From 6540becb0501e5baabaf60941b915b0fee822969 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 20 Jun 2025 12:35:08 +0000 Subject: [PATCH 001/881] zml/exe: don't wait for event Don't forget to delete them --- zml/exe.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/exe.zig b/zml/exe.zig index 0923f17c..d5c26795 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -251,7 +251,7 @@ pub const BaseExe = struct { for (events[0..sharding.num_partitions]) |e| { if (e) |ev| { - ev.await_(self.platform.pjrt_api) catch unreachable; + ev.deinit(self.platform.pjrt_api); } } } From 222e3d04c133ae81d38498386bd65bec5eace292 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 20 Jun 2025 12:35:31 +0000 Subject: [PATCH 002/881] zml/pjrtx: remove async blocking call to execute --- zml/pjrtx.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 1795efc3..9e3143d6 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -272,14 +272,14 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{ self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ .num_args = args.num_args, .arguments = @ptrCast(args.arguments), .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, .context = args.context, - } }); + }); } pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { From 2224423922552d7859900199efd0c46457de6a23 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 26 Jun 2025 12:37:28 +0000 Subject: [PATCH 003/881] zml/module.zig: disable gemm autotuning on ROCm This is taking way too much time for llmd --- zml/module.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/zml/module.zig b/zml/module.zig index f031e815..0c2f746c 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -899,6 +899,7 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m // Use lld from libllvm instead of invoking the ld.lld binary. // This saves us from having to sandbox it. setFlag(&options, "xla_gpu_use_inprocess_lld", true); + setFlag(&options, "xla_gpu_autotune_level", 0); }, else => {}, } From ace0bf4eaa0642c5af0f8cb3c8d7dc65ed6c5b54 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 26 Jun 2025 12:55:39 +0000 Subject: [PATCH 004/881] zml/tools/tracer.zig: make FakeTracer api compatible with CudaTracer --- zml/tools/tracer.zig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zml/tools/tracer.zig b/zml/tools/tracer.zig index 06bcd6fa..7efe748a 100644 --- a/zml/tools/tracer.zig +++ b/zml/tools/tracer.zig @@ -1,4 +1,5 @@ const builtin = @import("builtin"); + const c = @import("c"); pub const Tracer = switch (builtin.os.tag) { @@ -83,6 +84,10 @@ const FakeTracer = struct { return .{}; } + pub fn deinit(self: *const FakeTracer) void { + _ = self; + } + pub fn event(self: *const FakeTracer, message: [:0]const u8) void { _ = self; _ = message; From fea85cf7f06343fb9c1a1ceacb8c4121a3aae507 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 9 Jul 2025 11:34:29 +0000 Subject: [PATCH 005/881] stdx/math: added cdiv function --- stdx/math.zig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/stdx/math.zig b/stdx/math.zig index da0bec79..1e1e1318 100644 --- a/stdx/math.zig +++ b/stdx/math.zig @@ -21,3 +21,7 @@ pub inline fn roundeven(x: anytype) @TypeOf(x) { extern fn @"llvm.roundeven"(@TypeOf(x)) @TypeOf(x); }.@"llvm.roundeven"(x); } + +pub fn cdiv(numerator: anytype, denominator: anytype) @TypeOf(numerator) { + return (numerator + denominator - 1) / denominator; +} From 7c58136bbd846dc1bc5dc3edbc22a4f94c232267 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 11 Jul 2025 15:36:19 +0000 Subject: [PATCH 006/881] zml/nn: correctly parse RopeOpts.Scaling from JSON --- zml/nn.zig | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index ffdb7a9c..adc06237 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -175,17 +175,23 @@ pub const RopeOpts = struct { /// Read a Rope scaling config from HF config.json format. pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !Scaling { - const content = try std.json.Value.jsonParse(allocator, source, options); - if (content == .null) return .default; + return jsonParseFromValue( + allocator, + try std.json.innerParse(std.json.Value, allocator, source, options), + options, + ); + } - if (content != .object) return error.InvalidEnumTag; + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) std.json.ParseFromValueError!Scaling { + if (source == .null) return .default; - const obj = content.object; + if (source != .object) return error.InvalidEnumTag; + + const obj = source.object; const impl = obj.get("rope_type") orelse return error.MissingField; if (impl != .string) return error.InvalidEnumTag; if (std.mem.eql(u8, impl.string, "llama3")) { - // Note: leaky is fine here cause Llama3 struct don't need to allocate memory. - return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, undefined, content, .{ .ignore_unknown_fields = true }) }; + return .{ .llama3 = try std.json.innerParseFromValue(Llama3, allocator, source, options) }; } else { log.warn("Unsupported Rope implementation: {s}, will use the default one which will produce altered results", .{impl.string}); return .{ .default = {} }; From 328fd9b3cd046c1ab3359e7b9716d8f867985fb5 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 16 Jul 2025 13:51:24 +0000 Subject: [PATCH 007/881] zml/tools: temporarily disable CudaTracer for universal binaries --- zml/tools/tracer.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/tools/tracer.zig b/zml/tools/tracer.zig index 7efe748a..3d0c1b31 100644 --- a/zml/tools/tracer.zig +++ b/zml/tools/tracer.zig @@ -4,7 +4,7 @@ const c = @import("c"); pub const Tracer = switch (builtin.os.tag) { .macos => MacOsTracer, - .linux => if (@hasDecl(c, "ZML_RUNTIME_CUDA")) CudaTracer else FakeTracer, + .linux => FakeTracer, else => FakeTracer, }; From 52d36e1c7e0d17572039d9ffec88ad689b060d31 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Fri, 25 Jul 2025 11:35:04 +0200 Subject: [PATCH 008/881] runtimes/neuron: sandbox neuron dependencies (#290) The difference with other sandboxing PRs is that we also define a trampoline PJRT so file which will prove itself useful to circumvent PJRT version mismatch because of https://github.com/aws-neuron/aws-neuron-sdk/issues/1095 Additionally, we artificially create a empty repository so that we can sandbox the distroless deps inside the neuron module extension. --------- Co-authored-by: EC2 Default User --- MODULE.bazel | 2 +- MODULE.bazel.lock | 46 +++++++++-- bazel/simple_repository.bzl | 23 ++++++ examples/MODULE.bazel.lock | 46 +++++++++-- runtimes/common/packages.bzl | 11 +++ runtimes/neuron/BUILD.bazel | 79 ++++++++---------- runtimes/neuron/libpjrt_neuron.BUILD.bazel | 96 ++++++++++++++++++++++ runtimes/neuron/libpjrt_neuron.zig | 39 +++++++++ runtimes/neuron/neuron.bzl | 85 +++++++++++++------ runtimes/neuron/neuron.zig | 35 ++++++-- runtimes/neuron/packages.lock.json | 89 ++++---------------- runtimes/neuron/packages.yaml | 6 +- runtimes/neuron/requirements.lock.txt | 49 +++++------ runtimes/neuron/zmlxneuron.c | 5 +- 14 files changed, 412 insertions(+), 199 deletions(-) create mode 100644 bazel/simple_repository.bzl create mode 100644 runtimes/neuron/libpjrt_neuron.BUILD.bazel create mode 100644 runtimes/neuron/libpjrt_neuron.zig diff --git a/MODULE.bazel b/MODULE.bazel index 401868a1..ce4383ab 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -92,7 +92,7 @@ tpu = use_extension("//runtimes/tpu:tpu.bzl", "tpu_packages") use_repo(tpu, "libpjrt_tpu") neuron = use_extension("//runtimes/neuron:neuron.bzl", "neuron_packages") -use_repo(neuron, "aws-neuronx-collectives", "aws-neuronx-runtime-lib") +use_repo(neuron, "libpjrt_neuron") zls = use_extension("//third_party/zls:zls.bzl", "repo") use_repo(zls, "zls_aarch64-macos", "zls_x86_64-linux", "zls_x86_64-macos") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 1c74b30d..cca6007a 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1033,7 +1033,7 @@ "bzlTransitiveDigest": "fOwLlf5Rfxv4xkFCZwPlkSPUuZ9lAhMGf0SN43pc0xg=", "usagesDigest": "3+0h/pprjG6w7aHgvrT2H3LdTTVQQ79Mz0lATHIw+/I=", "recordedFileInputs": { - "@@//runtimes/neuron/requirements.lock.txt": "a5e5353b149b33dc067ce27ef67896d8ca3518e37a344e215e6604f98ea6d0fc", + "@@//runtimes/neuron/requirements.lock.txt": "fa0b5b967162e303415ee1dd7d2c1a448068935663ba5ac5b5f72a515fe3f6f6", "@@protobuf+//python/requirements.txt": "2bb341a0ba317fb8dc9ac79817b9b2d8f6a4d0c43e3fb652f0cab584eed420a4", "@@rules_fuzzing+//fuzzing/requirements.txt": "d4e91a1b3eca06d6e2877f67d3d8be498cb3ed6d28b9c14439b6f210ec621470", "@@rules_python+//tools/publish/requirements_darwin.txt": "6d10732737e5504291360b78fbf2b82abe39a5a021bd937f987f8433519b0f2c", @@ -1069,7 +1069,7 @@ ], "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "neuron_py_deps_311", - "requirement": "boto3==1.38.25 --hash=sha256:2f2cd517dd31d33ace0eefe567dc903fdf74221513e32f1e9445bdfac7554db7 --hash=sha256:85c1556a110896f68de8573a9b4757c81071448dbf6ffc1074941bfc8a43195e" + "requirement": "boto3==1.39.9 --hash=sha256:5bc85e9fdec4e21ef5ca2c22b4d51a3e32b53f3da36ce51f5a3ea4dbde07b132 --hash=sha256:e3d3a6b617e1575e7ec854c820a882ab2e189a0421e74dc0dca2c9e13d4370a5" } }, "neuron_py_deps_311_botocore": { @@ -1095,7 +1095,7 @@ ], "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "neuron_py_deps_311", - "requirement": "botocore==1.38.25 --hash=sha256:5a960bd990a11cdb78865e908a58ed712d87d9b419f55ad21c99d7d21f0d6726 --hash=sha256:8c73e97d9662a6c92be33dab66cd1e2b59797154c7ec379ce3bb5d6779d9d78c" + "requirement": "botocore==1.39.9 --hash=sha256:02f141c2849e4589a79feea245ce4ecc478d48b7865572445af8aae3b041772d --hash=sha256:a9691cbe03a3bc8b2720b3c36e5c5a2eecace6acd72bfb1107f00e75edaec4f3" } }, "neuron_py_deps_311_certifi": { @@ -1121,7 +1121,7 @@ ], "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "neuron_py_deps_311", - "requirement": "certifi==2025.4.26 --hash=sha256:0a816057ea3cdefcef70270d2c515e4506bbc954f417fa5ade2021213bb8f0c6 --hash=sha256:30350364dfe371162649852c63336a15c70c6510c2ad5015b21c2345311805f3" + "requirement": "certifi==2025.7.14 --hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 --hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995" } }, "neuron_py_deps_311_charset_normalizer": { @@ -1277,7 +1277,7 @@ ], "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "neuron_py_deps_311", - "requirement": "libneuronxla==2.2.3493.0+78c3e78c --hash=sha256:9ba54e5d481afc25ae83e964a7c076543991ad80b88839cd500abcac130595e8" + "requirement": "libneuronxla==2.2.4410.0+835a67fb --hash=sha256:37e5f08482ef1a9a844c2894de6d91c03400a092764e54ab67612456be439f16" } }, "neuron_py_deps_311_lockfile": { @@ -1381,7 +1381,7 @@ ], "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "neuron_py_deps_311", - "requirement": "neuronx-cc==2.18.121.0+9e31e41a --hash=sha256:b4c0959594af716aff94698e15cc3997cc39a66c853c25a1bbf5eb7bf71fd7ec" + "requirement": "neuronx-cc==2.19.8089.0+8ab9f450 --hash=sha256:fbb05d8a0da8f54954c03ff2bea659f627309110c00efd54f523d877ddd6b88d" } }, "neuron_py_deps_311_numpy": { @@ -1563,7 +1563,7 @@ ], "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "neuron_py_deps_311", - "requirement": "requests==2.31.0 --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1" + "requirement": "requests==2.32.4 --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422" } }, "neuron_py_deps_311_requests_unixsocket": { @@ -1615,7 +1615,7 @@ ], "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "neuron_py_deps_311", - "requirement": "s3transfer==0.13.0 --hash=sha256:0148ef34d6dd964d0d8cf4311b2b21c474693e57c2e069ec708ce043d2b527be --hash=sha256:f5e6db74eb7776a37208001113ea7aa97695368242b364d73e91c981ac522177" + "requirement": "s3transfer==0.13.1 --hash=sha256:a981aa7429be23fe6dfc13e80e4020057cbab622b08c0315288758d67cabc724 --hash=sha256:c3fdba22ba1bd367922f27ec8032d6a1cf5f10c934fb5d68cf60fd5a23d936cf" } }, "neuron_py_deps_311_scipy": { @@ -1696,6 +1696,32 @@ "requirement": "tqdm==4.67.1 --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2" } }, + "neuron_py_deps_311_typing_extensions": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@neuron_py_deps//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple", + "--extra-index-url", + "https://pip.repos.neuron.amazonaws.com", + "--find-links", + "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", + "--abi=cp311", + "--implementation=cp", + "--python-version=311", + "--platform=linux_x86_64", + "--platform=manylinux2014_x86_64" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "neuron_py_deps_311", + "requirement": "typing-extensions==4.14.1 --hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 --hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76" + } + }, "neuron_py_deps_311_urllib3": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { @@ -1719,7 +1745,7 @@ ], "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "neuron_py_deps_311", - "requirement": "urllib3==2.4.0 --hash=sha256:414bc6535b787febd7567804cc015fee39daab8ad86268f1310a9250697de466 --hash=sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813" + "requirement": "urllib3==2.5.0 --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc" } }, "pip_deps_310_numpy": { @@ -4148,6 +4174,7 @@ "scipy": "{\"neuron_py_deps_311_scipy\":[{\"version\":\"3.11\"}]}", "six": "{\"neuron_py_deps_311_six\":[{\"version\":\"3.11\"}]}", "tqdm": "{\"neuron_py_deps_311_tqdm\":[{\"version\":\"3.11\"}]}", + "typing_extensions": "{\"neuron_py_deps_311_typing_extensions\":[{\"version\":\"3.11\"}]}", "urllib3": "{\"neuron_py_deps_311_urllib3\":[{\"version\":\"3.11\"}]}" }, "packages": [ @@ -4176,6 +4203,7 @@ "scipy", "six", "tqdm", + "typing_extensions", "urllib3" ], "groups": {} diff --git a/bazel/simple_repository.bzl b/bazel/simple_repository.bzl new file mode 100644 index 00000000..9fd07136 --- /dev/null +++ b/bazel/simple_repository.bzl @@ -0,0 +1,23 @@ +_UNSET = "_UNSET" + +def _simple_repository_impl(rctx): + if (rctx.attr.build_file == None) == (rctx.attr.build_file_content == _UNSET): + fail("exactly one of `build_file` and `build_file_content` must be specified") + + if rctx.attr.build_file != None: + # Remove any existing BUILD.bazel in the repository to ensure + # the symlink to the defined build_file doesn't fail. + rctx.delete("BUILD.bazel") + rctx.symlink(rctx.attr.build_file, "BUILD.bazel") + else: + rctx.file("BUILD.bazel", rctx.attr.build_file_content) + +simple_repository = repository_rule( + implementation = _simple_repository_impl, + attrs = { + "build_file": attr.label(allow_single_file = True), + "build_file_content": attr.string(default = _UNSET), + }, + doc = "Makes an empty repository from just one BUILD.bazel file.", + local = True, +) diff --git a/examples/MODULE.bazel.lock b/examples/MODULE.bazel.lock index c2861d8d..aeb27f08 100644 --- a/examples/MODULE.bazel.lock +++ b/examples/MODULE.bazel.lock @@ -1265,7 +1265,7 @@ "@@rules_python+//tools/publish/requirements_darwin.txt": "6d10732737e5504291360b78fbf2b82abe39a5a021bd937f987f8433519b0f2c", "@@rules_python+//tools/publish/requirements_linux.txt": "cb63e4751377d1722a81fd5835b0526208620f1303feb473ed0dd45ae4d71ba9", "@@rules_python+//tools/publish/requirements_windows.txt": "f92da1a252df04e1931a0e16dfbc5a3c165a85b796b7fc5e56ee4e3846d82558", - "@@zml+//runtimes/neuron/requirements.lock.txt": "a5e5353b149b33dc067ce27ef67896d8ca3518e37a344e215e6604f98ea6d0fc" + "@@zml+//runtimes/neuron/requirements.lock.txt": "fa0b5b967162e303415ee1dd7d2c1a448068935663ba5ac5b5f72a515fe3f6f6" }, "recordedDirentsInputs": {}, "envVariables": { @@ -1296,7 +1296,7 @@ ], "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "neuron_py_deps_311", - "requirement": "boto3==1.38.25 --hash=sha256:2f2cd517dd31d33ace0eefe567dc903fdf74221513e32f1e9445bdfac7554db7 --hash=sha256:85c1556a110896f68de8573a9b4757c81071448dbf6ffc1074941bfc8a43195e" + "requirement": "boto3==1.39.9 --hash=sha256:5bc85e9fdec4e21ef5ca2c22b4d51a3e32b53f3da36ce51f5a3ea4dbde07b132 --hash=sha256:e3d3a6b617e1575e7ec854c820a882ab2e189a0421e74dc0dca2c9e13d4370a5" } }, "neuron_py_deps_311_botocore": { @@ -1322,7 +1322,7 @@ ], "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "neuron_py_deps_311", - "requirement": "botocore==1.38.25 --hash=sha256:5a960bd990a11cdb78865e908a58ed712d87d9b419f55ad21c99d7d21f0d6726 --hash=sha256:8c73e97d9662a6c92be33dab66cd1e2b59797154c7ec379ce3bb5d6779d9d78c" + "requirement": "botocore==1.39.9 --hash=sha256:02f141c2849e4589a79feea245ce4ecc478d48b7865572445af8aae3b041772d --hash=sha256:a9691cbe03a3bc8b2720b3c36e5c5a2eecace6acd72bfb1107f00e75edaec4f3" } }, "neuron_py_deps_311_certifi": { @@ -1348,7 +1348,7 @@ ], "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "neuron_py_deps_311", - "requirement": "certifi==2025.4.26 --hash=sha256:0a816057ea3cdefcef70270d2c515e4506bbc954f417fa5ade2021213bb8f0c6 --hash=sha256:30350364dfe371162649852c63336a15c70c6510c2ad5015b21c2345311805f3" + "requirement": "certifi==2025.7.14 --hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 --hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995" } }, "neuron_py_deps_311_charset_normalizer": { @@ -1504,7 +1504,7 @@ ], "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "neuron_py_deps_311", - "requirement": "libneuronxla==2.2.3493.0+78c3e78c --hash=sha256:9ba54e5d481afc25ae83e964a7c076543991ad80b88839cd500abcac130595e8" + "requirement": "libneuronxla==2.2.4410.0+835a67fb --hash=sha256:37e5f08482ef1a9a844c2894de6d91c03400a092764e54ab67612456be439f16" } }, "neuron_py_deps_311_lockfile": { @@ -1608,7 +1608,7 @@ ], "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "neuron_py_deps_311", - "requirement": "neuronx-cc==2.18.121.0+9e31e41a --hash=sha256:b4c0959594af716aff94698e15cc3997cc39a66c853c25a1bbf5eb7bf71fd7ec" + "requirement": "neuronx-cc==2.19.8089.0+8ab9f450 --hash=sha256:fbb05d8a0da8f54954c03ff2bea659f627309110c00efd54f523d877ddd6b88d" } }, "neuron_py_deps_311_numpy": { @@ -1790,7 +1790,7 @@ ], "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "neuron_py_deps_311", - "requirement": "requests==2.31.0 --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1" + "requirement": "requests==2.32.4 --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422" } }, "neuron_py_deps_311_requests_unixsocket": { @@ -1842,7 +1842,7 @@ ], "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "neuron_py_deps_311", - "requirement": "s3transfer==0.13.0 --hash=sha256:0148ef34d6dd964d0d8cf4311b2b21c474693e57c2e069ec708ce043d2b527be --hash=sha256:f5e6db74eb7776a37208001113ea7aa97695368242b364d73e91c981ac522177" + "requirement": "s3transfer==0.13.1 --hash=sha256:a981aa7429be23fe6dfc13e80e4020057cbab622b08c0315288758d67cabc724 --hash=sha256:c3fdba22ba1bd367922f27ec8032d6a1cf5f10c934fb5d68cf60fd5a23d936cf" } }, "neuron_py_deps_311_scipy": { @@ -1923,6 +1923,32 @@ "requirement": "tqdm==4.67.1 --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2" } }, + "neuron_py_deps_311_typing_extensions": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@neuron_py_deps//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple", + "--extra-index-url", + "https://pip.repos.neuron.amazonaws.com", + "--find-links", + "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", + "--abi=cp311", + "--implementation=cp", + "--python-version=311", + "--platform=linux_x86_64", + "--platform=manylinux2014_x86_64" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "neuron_py_deps_311", + "requirement": "typing-extensions==4.14.1 --hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 --hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76" + } + }, "neuron_py_deps_311_urllib3": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { @@ -1946,7 +1972,7 @@ ], "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "neuron_py_deps_311", - "requirement": "urllib3==2.4.0 --hash=sha256:414bc6535b787febd7567804cc015fee39daab8ad86268f1310a9250697de466 --hash=sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813" + "requirement": "urllib3==2.5.0 --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc" } }, "pip_deps_310_numpy": { @@ -4375,6 +4401,7 @@ "scipy": "{\"neuron_py_deps_311_scipy\":[{\"version\":\"3.11\"}]}", "six": "{\"neuron_py_deps_311_six\":[{\"version\":\"3.11\"}]}", "tqdm": "{\"neuron_py_deps_311_tqdm\":[{\"version\":\"3.11\"}]}", + "typing_extensions": "{\"neuron_py_deps_311_typing_extensions\":[{\"version\":\"3.11\"}]}", "urllib3": "{\"neuron_py_deps_311_urllib3\":[{\"version\":\"3.11\"}]}" }, "packages": [ @@ -4403,6 +4430,7 @@ "scipy", "six", "tqdm", + "typing_extensions", "urllib3" ], "groups": {} diff --git a/runtimes/common/packages.bzl b/runtimes/common/packages.bzl index 8e3c1c49..abb2ab1d 100644 --- a/runtimes/common/packages.bzl +++ b/runtimes/common/packages.bzl @@ -13,6 +13,16 @@ def _cc_import(**kwargs): def _cc_library(**kwargs): return """cc_library({})""".format(_kwargs(**kwargs)) +def _cc_library_hdrs_glob(name, hdrs_glob, deps = [], **kwargs): + return """\ +cc_library( + name = "{name}", + hdrs = glob({hdrs_glob}), + deps = {deps}, + {kwargs} +) +""".format(name = name, hdrs_glob = repr(hdrs_glob), deps = repr(deps), kwargs = _kwargs(**kwargs)) + def _cc_import_glob_hdrs(name, hdrs_glob, shared_library, deps = [], **kwargs): return """\ cc_import( @@ -48,6 +58,7 @@ packages = struct( cc_import = _cc_import, cc_import_glob_hdrs = _cc_import_glob_hdrs, cc_library = _cc_library, + cc_library_hdrs_glob = _cc_library_hdrs_glob, filegroup = _filegroup, load_ = _load, patchelf = _patchelf, diff --git a/runtimes/neuron/BUILD.bazel b/runtimes/neuron/BUILD.bazel index 01a3684b..b65d7a38 100644 --- a/runtimes/neuron/BUILD.bazel +++ b/runtimes/neuron/BUILD.bazel @@ -1,12 +1,8 @@ -load("@bazel_skylib//rules:select_file.bzl", "select_file") load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_cc//cc:cc_shared_library.bzl", "cc_shared_library") load("@rules_python//python:pip.bzl", "compile_pip_requirements") load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") load("@rules_uv//uv:pip.bzl", "pip_compile") -load("@rules_zig//zig:defs.bzl", "zig_library") -load("@zml//bazel:cc_import.bzl", "cc_import") -load("@zml//bazel:runfiles.bzl", "runfiles_to_default") +load("@rules_zig//zig:defs.bzl", "BINARY_KIND", "zig_binary", "zig_library") load(":neuron.bzl", "py_binary_with_script") load(":pyenv.bzl", "pyenv_zig") @@ -20,6 +16,25 @@ zig_library( main = ":libneuronxla_pyenv_zig", ) +# A proxy PJRT Plugin that loads the Neuron PJRT Plugin +# and returns the instance from nested GetPjrtApi. +# +# Additionally, it provides a way to load implicit transitive dependencies +# of neuronx-cc (see add_needed of the patchelf target below). +# +# TODO(cerisier): Use a zig_cc_shared_library instead. +zig_binary( + name = "libpjrt_neuron_proxy", + kind = BINARY_KIND.shared_lib, + main = "libpjrt_neuron.zig", + copts = ["-lc"], + deps = [ + "//stdx", + "@rules_zig//zig/runfiles", + ], + visibility = ["@libpjrt_neuron//:__subpackages__"], +) + pip_compile( name = "update_requirements", args = [ @@ -75,48 +90,11 @@ compile_pip_requirements( ], ) -runfiles_to_default( - name = "libneuronxla_files", - deps = ["@neuron_py_deps//libneuronxla:pkg"], -) - -select_file( - name = "libneuronpjrt_so", - srcs = ":libneuronxla_files", - subpath = "site-packages/libneuronxla/libneuronpjrt.so", -) - -cc_import( - name = "libpjrt_neuron_", - add_needed = [ - "libpython3.11.so.1.0", - "libzmlxneuron.so.0", - ], - data = [":libneuronxla"], - shared_library = ":libneuronpjrt_so", - soname = "libpjrt_neuron.so", - deps = [ - ":zmlxneuron", - "@aws-neuronx-runtime-lib", - "@rules_python//python/cc:current_py_cc_libs", - ], -) - cc_library( name = "zmlxneuron_lib", srcs = ["zmlxneuron.c"], linkopts = ["-ldl"], -) - -cc_shared_library( - name = "zmlxneuron_", - shared_lib_name = "libzmlxneuron.so.0", - deps = [":zmlxneuron_lib"], -) - -cc_import( - name = "zmlxneuron", - shared_library = ":zmlxneuron_", + visibility = ["@libpjrt_neuron//:__subpackages__"], ) alias( @@ -132,7 +110,9 @@ cc_library( name = "libpjrt_neuron", hdrs = ["libpjrt_neuron.h"], defines = ["ZML_RUNTIME_NEURON"], - deps = [":libpjrt_neuron_"], + deps = [ + "@libpjrt_neuron", + ], ) zig_library( @@ -140,14 +120,21 @@ zig_library( import_name = "runtimes/neuron", main = "neuron.zig", visibility = ["//visibility:public"], + data = select({ + "//runtimes:neuron.enabled": [ + ":libneuronxla", + ], + "//conditions:default": [], + }), deps = [ "//pjrt", ] + select({ "//runtimes:neuron.enabled": [ - ":libneuronxla_pyenv", ":libpjrt_neuron", + ":libneuronxla_pyenv", ":libpython", "//async", + "//stdx", "@rules_zig//zig/runfiles", ], "//conditions:default": [":empty"], @@ -158,8 +145,6 @@ filegroup( name = "layers", srcs = [ "@apt_neuron//bash/amd64", - "@apt_neuron//libxml2/amd64", - "@apt_neuron//zlib1g/amd64", ], visibility = ["//visibility:public"], ) diff --git a/runtimes/neuron/libpjrt_neuron.BUILD.bazel b/runtimes/neuron/libpjrt_neuron.BUILD.bazel new file mode 100644 index 00000000..5df16d6b --- /dev/null +++ b/runtimes/neuron/libpjrt_neuron.BUILD.bazel @@ -0,0 +1,96 @@ +load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@zml//bazel:patchelf.bzl", "patchelf") +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") +load("@bazel_skylib//rules:select_file.bzl", "select_file") +load("@rules_cc//cc:cc_shared_library.bzl", "cc_shared_library") +load("@zml//bazel:runfiles.bzl", "runfiles_to_default") + +cc_shared_library( + name = "zmlxneuron_so_", + shared_lib_name = "libzmlxneuron.so.0", + deps = ["@zml//runtimes/neuron:zmlxneuron_lib"], +) + +copy_file( + name = "zmlxneuron_so", + src = ":zmlxneuron_so_", + out = "lib/libzmlxneuron.so.0", +) + +runfiles_to_default( + name = "libneuronxla_files", + deps = ["@neuron_py_deps//libneuronxla:pkg"], +) + +select_file( + name = "libneuronpjrt_so", + srcs = ":libneuronxla_files", + subpath = "site-packages/libneuronxla/libneuronpjrt.so", +) + +patchelf( + name = "libneuronpjrt.patchelf", + add_needed = [ + "libpython3.11.so.1.0", + "libzmlxneuron.so.0", + "libnccom.so.2", + ], + set_rpath = '$ORIGIN', + shared_library = ":libneuronpjrt_so", + soname = "libneuronpjrt.so", +) + +patchelf( + name = "libpjrt_neuron_proxy.patchelf", + set_rpath = '$ORIGIN', + add_needed = [ + "libz.so.1", + "libgomp.so.1", + ], + shared_library = "@zml//runtimes/neuron:libpjrt_neuron_proxy", + soname = "libpjrt_neuron.so", +) + +copy_to_directory( + name = "sandbox", + srcs = [ + ":zmlxneuron_so", + ":libneuronpjrt.patchelf", + ":libpjrt_neuron_proxy.patchelf", + "@aws-neuronx-runtime-lib//:libnrt.patchelf", + "@aws-neuronx-runtime-lib//:libncfw.patchelf", + "@aws-neuronx-collectives//:libnccom", + "@zlib1g", + "@libgomp1", + ], + replace_prefixes = { + "libneuronpjrt.patchelf": "lib", + "libpjrt_neuron_proxy.patchelf": "lib", + "libnrt.patchelf": "lib", + "libncfw.patchelf": "lib", + "lib/x86_64-linux-gnu": "lib", + "usr/lib/x86_64-linux-gnu": "lib", + "opt/neuron": "lib", + }, + add_directory_to_runfiles = True, + include_external_repositories = ["**"], +) + +cc_library( + name = "libpjrt_neuron", + data = [":sandbox"], + deps = [ + "@aws-neuronx-runtime-lib//:libnrt_headers", + ], + linkopts = [ + # Defer function call resolution until the function is called + # (lazy loading) rather than at load time. + # + # This is required because we want to let downstream use weak NRT symbols. + # + # We force it here because -z,now (which resolve all symbols at load time), + # is the default in most bazel CC toolchains as well as in certain linkers. + "-Wl,-z,lazy", + ], + visibility = ["@zml//runtimes/neuron:__subpackages__"], +) diff --git a/runtimes/neuron/libpjrt_neuron.zig b/runtimes/neuron/libpjrt_neuron.zig new file mode 100644 index 00000000..6485ecd0 --- /dev/null +++ b/runtimes/neuron/libpjrt_neuron.zig @@ -0,0 +1,39 @@ +const std = @import("std"); +const runfiles = @import("runfiles"); +const bazel_builtin = @import("bazel_builtin"); +const stdx = @import("stdx"); + +const log = std.log.scoped(.@"zml/runtimes/neuron"); + +pub export fn GetPjrtApi() *anyopaque { + var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); + defer arena.deinit(); + + var r_ = runfiles.Runfiles.create(.{ .allocator = arena.allocator() }) catch |err| { + stdx.debug.panic("Unable to find runfiles: {}", .{err}); + } orelse stdx.debug.panic("Runfiles not availeabwewefle", .{}); + + const source_repo = bazel_builtin.current_repository; + const r = r_.withSourceRepo(source_repo); + + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const sandbox_path = r.rlocation("libpjrt_neuron/sandbox", &path_buf) catch |err| { + stdx.debug.panic("Failed to find sandbox path for NEURON runtime: {}", .{err}); + } orelse stdx.debug.panic("No NEURON sandbox path found", .{}); + + var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; + const library = stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libneuronpjrt.so" }) catch unreachable; + + var lib: std.DynLib = blk: { + const handle = std.c.dlopen(library, .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { + stdx.debug.panic("Unable to dlopen plugin: {s}", .{library}); + }; + break :blk .{ .inner = .{ .handle = handle } }; + }; + + const sym = lib.lookup(*const fn () callconv(.C) *anyopaque, "GetPjrtApi") orelse { + stdx.debug.panic("Unable to find symbol GetPjrtApi in plugin: {s}", .{library}); + }; + + return sym(); +} diff --git a/runtimes/neuron/neuron.bzl b/runtimes/neuron/neuron.bzl index 4466ca96..89996fab 100644 --- a/runtimes/neuron/neuron.bzl +++ b/runtimes/neuron/neuron.bzl @@ -2,39 +2,55 @@ load("@python_versions//3.11:defs.bzl", _py_binary = "py_binary") load("@rules_python//python:defs.bzl", "PyInfo") load("@with_cfg.bzl", "with_cfg") load("//bazel:http_deb_archive.bzl", "http_deb_archive") +load("//bazel:simple_repository.bzl", "simple_repository") load("//runtimes/common:packages.bzl", "packages") BASE_URL = "https://apt.repos.neuron.amazonaws.com" STRIP_PREFIX = "opt/aws/neuron" -BUILD_FILE_PRELUDE = """\ -load("@zml//bazel:cc_import.bzl", "cc_import") +_BUILD_FILE_PRELUDE = """\ +package(default_visibility = ["//visibility:public"]) """ -_PACKAGES = { - "aws-neuronx-runtime-lib": packages.cc_import_glob_hdrs( - name = "aws-neuronx-runtime-lib", - hdrs_glob = [ - "include/ndl/**/*.h", - "include/nrt/**/*.h", - ], - includes = ["include"], - shared_library = "lib/libnrt.so.1", - rename_dynamic_symbols = { - "dlopen": "zmlxneuron_dlopen", - }, - visibility = ["@zml//runtimes/neuron:__subpackages__"], - deps = ["@aws-neuronx-collectives//:libnccom"], - ), - "aws-neuronx-collectives": "\n".join([ - packages.cc_import( - name = "libnccom", - shared_library = "lib/libnccom.so.2", - visibility = ["@aws-neuronx-runtime-lib//:__subpackages__"], +_UBUNTU_PACKAGES = { + "zlib1g": packages.filegroup(name = "zlib1g", srcs = ["lib/x86_64-linux-gnu/libz.so.1"]), + "libgomp1": packages.filegroup(name = "libgomp1", srcs = ["usr/lib/x86_64-linux-gnu/libgomp.so.1"]), +} + +_NEURON_PACKAGES = { + "aws-neuronx-runtime-lib": "\n".join([ + packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), + packages.cc_library_hdrs_glob( + name = "libnrt_headers", + hdrs_glob = [ + "include/ndl/**/*.h", + "include/nrt/**/*.h", + ], + includes = ["include"], + visibility = ["//visibility:public"], ), - packages.cc_import( - name = "libnccom-net", - shared_library = "lib/libnccom-net.so.0", + packages.patchelf( + name = "libnrt.patchelf", + shared_library = "lib/libnrt.so.1", + set_rpath = '$ORIGIN', + add_needed = [ + # readelf -d ./opt/aws/neuron/libl/libncfw.so + "libncfw.so.2", + ], + rename_dynamic_symbols = { + "dlopen": "zmlxneuron_dlopen", + }, + ), + packages.patchelf( + name = "libncfw.patchelf", + shared_library = "lib/libncfw.so", + soname = "libncfw.so.2", + ), + ]), + "aws-neuronx-collectives": "\n".join([ + packages.filegroup( + name = "libnccom", + srcs = ["lib/libnccom.so.2"], ), ]), } @@ -43,14 +59,29 @@ def _neuron_impl(mctx): loaded_packages = packages.read(mctx, [ "@zml//runtimes/neuron:packages.lock.json", ]) - for pkg_name, build_file_content in _PACKAGES.items(): + + simple_repository( + name = "libpjrt_neuron", + build_file = ":libpjrt_neuron.BUILD.bazel", + ) + + for pkg_name, build_file_content in _UBUNTU_PACKAGES.items(): + pkg = loaded_packages[pkg_name] + http_deb_archive( + name = pkg_name, + urls = pkg["urls"], + sha256 = pkg["sha256"], + build_file_content = _BUILD_FILE_PRELUDE + build_file_content, + ) + + for pkg_name, build_file_content in _NEURON_PACKAGES.items(): pkg = loaded_packages[pkg_name] http_deb_archive( name = pkg_name, urls = pkg["urls"], sha256 = pkg["sha256"], strip_prefix = STRIP_PREFIX, - build_file_content = BUILD_FILE_PRELUDE + build_file_content, + build_file_content = _BUILD_FILE_PRELUDE + build_file_content, ) return mctx.extension_metadata( diff --git a/runtimes/neuron/neuron.zig b/runtimes/neuron/neuron.zig index 9fdbcc4c..fc973424 100644 --- a/runtimes/neuron/neuron.zig +++ b/runtimes/neuron/neuron.zig @@ -7,6 +7,9 @@ const c = @import("c"); const libneuronxla_pyenv = @import("libneuronxla_pyenv"); const pjrt = @import("pjrt"); const runfiles = @import("runfiles"); +const stdx = @import("stdx"); + +const log = std.log.scoped(.@"zml/runtime/neuron"); pub fn isEnabled() bool { return @hasDecl(c, "ZML_RUNTIME_NEURON"); @@ -44,11 +47,7 @@ fn pyErrorOrExit(status: c.PyStatus) void { } } -fn initialize() !void { - var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); - const allocator = arena.allocator(); - defer arena.deinit(); - +fn initialize(allocator: std.mem.Allocator, r_: *runfiles.Runfiles) !void { { var preconfig: c.PyPreConfig = undefined; c.PyPreConfig_InitIsolatedConfig(&preconfig); @@ -60,7 +59,6 @@ fn initialize() !void { c.PyConfig_InitIsolatedConfig(&config); defer c.PyConfig_Clear(&config); - var r_ = try runfiles.Runfiles.create(.{ .allocator = allocator }) orelse return error.Unavailable; const r = r_.withSourceRepo(bazel_builtin.current_repository); var buf: [std.fs.max_path_bytes]u8 = undefined; @@ -135,7 +133,28 @@ pub fn load() !*const pjrt.Api { return error.Unavailable; } + var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); + defer arena.deinit(); + + var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator() }) orelse { + stdx.debug.panic("Unable to find runfiles", .{}); + }; + + const source_repo = bazel_builtin.current_repository; + const r = r_.withSourceRepo(source_repo); + + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const sandbox_path = try r.rlocation("libpjrt_neuron/sandbox", &path_buf) orelse { + log.err("Failed to find sandbox path for NEURON runtime", .{}); + return error.FileNotFound; + }; + setNeuronCCFlags(); - try initialize(); - return try asynk.callBlocking(pjrt.Api.loadFrom, .{"libpjrt_neuron.so"}); + try initialize(arena.allocator(), &r_); + + return blk: { + var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; + const path = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_neuron.so" }); + break :blk asynk.callBlocking(pjrt.Api.loadFrom, .{path}); + }; } diff --git a/runtimes/neuron/packages.lock.json b/runtimes/neuron/packages.lock.json index 2ca0809e..ff484ae5 100755 --- a/runtimes/neuron/packages.lock.json +++ b/runtimes/neuron/packages.lock.json @@ -143,22 +143,17 @@ "version": "12.2.0-14+deb12u1" } ], - "key": "zlib1g_1-1.2.13.dfsg-1_amd64", - "name": "zlib1g", - "sha256": "d7dd1d1411fedf27f5e27650a6eff20ef294077b568f4c8c5e51466dc7c08ce4", + "key": "libgomp1_12.2.0-14-p-deb12u1_amd64", + "name": "libgomp1", + "sha256": "48fec46bda7f5b1638b9e959889bfbc20491247d402d120bb152687eb48143d7", "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_amd64.deb" + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/g/gcc-12/libgomp1_12.2.0-14+deb12u1_amd64.deb" ], - "version": "1:1.2.13.dfsg-1" + "version": "12.2.0-14+deb12u1" }, { "arch": "amd64", "dependencies": [ - { - "key": "zlib1g_1-1.2.13.dfsg-1_amd64", - "name": "zlib1g", - "version": "1:1.2.13.dfsg-1" - }, { "key": "libc6_2.36-9-p-deb12u10_amd64", "name": "libc6", @@ -173,74 +168,26 @@ "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", "name": "gcc-12-base", "version": "12.2.0-14+deb12u1" - }, - { - "key": "liblzma5_5.4.1-1_amd64", - "name": "liblzma5", - "version": "5.4.1-1" - }, - { - "key": "libicu72_72.1-3_amd64", - "name": "libicu72", - "version": "72.1-3" - }, - { - "key": "libstdc-p--p-6_12.2.0-14-p-deb12u1_amd64", - "name": "libstdc++6", - "version": "12.2.0-14+deb12u1" } ], - "key": "libxml2_2.9.14-p-dfsg-1.3_deb12u1_amd64", - "name": "libxml2", - "sha256": "35b76cb7038fc1c940204a4f05f33ffb79d027353ce469397d9adcf8f9b3e1a7", + "key": "zlib1g_1-1.2.13.dfsg-1_amd64", + "name": "zlib1g", + "sha256": "d7dd1d1411fedf27f5e27650a6eff20ef294077b568f4c8c5e51466dc7c08ce4", "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/libx/libxml2/libxml2_2.9.14+dfsg-1.3~deb12u1_amd64.deb" + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_amd64.deb" ], - "version": "2.9.14+dfsg-1.3~deb12u1" + "version": "1:1.2.13.dfsg-1" }, { "arch": "amd64", "dependencies": [], - "key": "liblzma5_5.4.1-1_amd64", - "name": "liblzma5", - "sha256": "d321b9502b16aac534e1c691afbe3dc5e125e5091aa35bea026c59b25ebe82e7", - "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/x/xz-utils/liblzma5_5.4.1-1_amd64.deb" - ], - "version": "5.4.1-1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libicu72_72.1-3_amd64", - "name": "libicu72", - "sha256": "e239c1c9f52bee0ff627f291552d63691b765ec7c5cdf6de7c7ae4dec0275857", - "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/i/icu/libicu72_72.1-3_amd64.deb" - ], - "version": "72.1-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libstdc-p--p-6_12.2.0-14-p-deb12u1_amd64", - "name": "libstdc++6", - "sha256": "5cd3171216d4ab0fc911cfe9c35509bf2dd8f47761c43b7f6a4296701551a24d", - "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/g/gcc-12/libstdc++6_12.2.0-14+deb12u1_amd64.deb" - ], - "version": "12.2.0-14+deb12u1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "aws-neuronx-runtime-lib_2.25.57.0-166c7a468_amd64", + "key": "aws-neuronx-runtime-lib_2.26.42.0-2ff3b5c7d_amd64", "name": "aws-neuronx-runtime-lib", - "sha256": "81613a0b86fc704acb5a4d1096a5bf2cce3aead04aeb5063d14a95195ca8ca39", + "sha256": "57f6fd8e850f4054c2d229cd41a83fe1489ec9d9ee55bedf33a14cd6e8beed16", "urls": [ - "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-runtime-lib/aws-neuronx-runtime-lib_2.25.57.0-166c7a468_amd64.deb" + "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-runtime-lib/aws-neuronx-runtime-lib_2.26.42.0-2ff3b5c7d_amd64.deb" ], - "version": "2.25.57.0-166c7a468" + "version": "2.26.42.0-2ff3b5c7d" }, { "arch": "amd64", @@ -386,13 +333,13 @@ "version": "252.36-1~deb12u1" } ], - "key": "aws-neuronx-collectives_2.25.65.0-9858ac9a1_amd64", + "key": "aws-neuronx-collectives_2.26.43.0-47cc904ea_amd64", "name": "aws-neuronx-collectives", - "sha256": "41e2eb08433556b243bae12ea61a7417068374b674b5a4e3e88455d28b6ba345", + "sha256": "4886f5aeeea21e4d502d445654a838190e2174a52ffb757da2cc3a8bf09be3f8", "urls": [ - "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-collectives/aws-neuronx-collectives_2.25.65.0-9858ac9a1_amd64.deb" + "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-collectives/aws-neuronx-collectives_2.26.43.0-47cc904ea_amd64.deb" ], - "version": "2.25.65.0-9858ac9a1" + "version": "2.26.43.0-47cc904ea" }, { "arch": "amd64", diff --git a/runtimes/neuron/packages.yaml b/runtimes/neuron/packages.yaml index c0d576a4..6c3de28a 100644 --- a/runtimes/neuron/packages.yaml +++ b/runtimes/neuron/packages.yaml @@ -18,8 +18,8 @@ archs: - "amd64" packages: - - "bash" - - "zlib1g" - - "libxml2" + - "bash" # needed to launch neuronx-cc + - "libgomp1" # implicit dep of neuronx-cc + - "zlib1g" # implicit dep of libpython - "aws-neuronx-runtime-lib" - "aws-neuronx-collectives" diff --git a/runtimes/neuron/requirements.lock.txt b/runtimes/neuron/requirements.lock.txt index ab1e2880..c5994c0b 100644 --- a/runtimes/neuron/requirements.lock.txt +++ b/runtimes/neuron/requirements.lock.txt @@ -4,20 +4,20 @@ --extra-index-url https://pip.repos.neuron.amazonaws.com --find-links https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html -boto3==1.38.25 \ - --hash=sha256:2f2cd517dd31d33ace0eefe567dc903fdf74221513e32f1e9445bdfac7554db7 \ - --hash=sha256:85c1556a110896f68de8573a9b4757c81071448dbf6ffc1074941bfc8a43195e +boto3==1.39.9 \ + --hash=sha256:5bc85e9fdec4e21ef5ca2c22b4d51a3e32b53f3da36ce51f5a3ea4dbde07b132 \ + --hash=sha256:e3d3a6b617e1575e7ec854c820a882ab2e189a0421e74dc0dca2c9e13d4370a5 # via libneuronxla -botocore==1.38.25 \ - --hash=sha256:5a960bd990a11cdb78865e908a58ed712d87d9b419f55ad21c99d7d21f0d6726 \ - --hash=sha256:8c73e97d9662a6c92be33dab66cd1e2b59797154c7ec379ce3bb5d6779d9d78c +botocore==1.39.9 \ + --hash=sha256:02f141c2849e4589a79feea245ce4ecc478d48b7865572445af8aae3b041772d \ + --hash=sha256:a9691cbe03a3bc8b2720b3c36e5c5a2eecace6acd72bfb1107f00e75edaec4f3 # via # boto3 # libneuronxla # s3transfer -certifi==2025.4.26 \ - --hash=sha256:0a816057ea3cdefcef70270d2c515e4506bbc954f417fa5ade2021213bb8f0c6 \ - --hash=sha256:30350364dfe371162649852c63336a15c70c6510c2ad5015b21c2345311805f3 +certifi==2025.7.14 \ + --hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 \ + --hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995 # via requests charset-normalizer==3.4.2 \ --hash=sha256:005fa3432484527f9732ebd315da8da8001593e2cf46a3d817669f062c3d9ed4 \ @@ -169,8 +169,8 @@ jmespath==1.0.1 \ # via # boto3 # botocore -libneuronxla==2.2.3493.0+78c3e78c \ - --hash=sha256:9ba54e5d481afc25ae83e964a7c076543991ad80b88839cd500abcac130595e8 +libneuronxla==2.2.4410.0+835a67fb \ + --hash=sha256:37e5f08482ef1a9a844c2894de6d91c03400a092764e54ab67612456be439f16 # via -r runtimes/neuron/requirements.in lockfile==0.12.2 \ --hash=sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799 \ @@ -206,8 +206,8 @@ networkx==2.8.8 \ --hash=sha256:230d388117af870fce5647a3c52401fcf753e94720e6ea6b4197a5355648885e \ --hash=sha256:e435dfa75b1d7195c7b8378c3859f0445cd88c6b0375c181ed66823a9ceb7524 # via neuronx-cc -neuronx-cc==2.18.121.0+9e31e41a \ - --hash=sha256:b4c0959594af716aff94698e15cc3997cc39a66c853c25a1bbf5eb7bf71fd7ec +neuronx-cc==2.19.8089.0+8ab9f450 \ + --hash=sha256:fbb05d8a0da8f54954c03ff2bea659f627309110c00efd54f523d877ddd6b88d # via # -r runtimes/neuron/requirements.in # libneuronxla @@ -287,20 +287,19 @@ python-dateutil==2.9.0.post0 \ --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 # via botocore -requests==2.31.0 \ - --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \ - --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1 +requests==2.32.4 \ + --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c \ + --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422 # via # ec2-metadata - # neuronx-cc # requests-unixsocket requests-unixsocket==0.4.1 \ --hash=sha256:60c4942e9dbecc2f64d611039fb1dfc25da382083c6434ac0316dca3ff908f4d \ --hash=sha256:b2596158c356ecee68d27ba469a52211230ac6fb0cde8b66afb19f0ed47a1995 # via neuronx-cc -s3transfer==0.13.0 \ - --hash=sha256:0148ef34d6dd964d0d8cf4311b2b21c474693e57c2e069ec708ce043d2b527be \ - --hash=sha256:f5e6db74eb7776a37208001113ea7aa97695368242b364d73e91c981ac522177 +s3transfer==0.13.1 \ + --hash=sha256:a981aa7429be23fe6dfc13e80e4020057cbab622b08c0315288758d67cabc724 \ + --hash=sha256:c3fdba22ba1bd367922f27ec8032d6a1cf5f10c934fb5d68cf60fd5a23d936cf # via boto3 scipy==1.12.0 \ --hash=sha256:196ebad3a4882081f62a5bf4aeb7326aa34b110e533aab23e4374fcccb0890dc \ @@ -337,9 +336,13 @@ tqdm==4.67.1 \ --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 \ --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2 # via neuronx-cc -urllib3==2.4.0 \ - --hash=sha256:414bc6535b787febd7567804cc015fee39daab8ad86268f1310a9250697de466 \ - --hash=sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813 +typing-extensions==4.14.1 \ + --hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 \ + --hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76 + # via neuronx-cc +urllib3==2.5.0 \ + --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 \ + --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc # via # botocore # requests diff --git a/runtimes/neuron/zmlxneuron.c b/runtimes/neuron/zmlxneuron.c index ea5e11bc..a5740bef 100644 --- a/runtimes/neuron/zmlxneuron.c +++ b/runtimes/neuron/zmlxneuron.c @@ -6,7 +6,10 @@ void *zmlxneuron_dlopen(const char *filename, int flags) if (filename != NULL) { char *replacements[] = { - "libnccom.so", "libnccom.so.2", "libnccom-net.so", "libnccom-net.so.0", NULL, NULL, + "libnccom.so", "libnccom.so.2", + "libnrt.so", "libnrt.so.1", + "libncfw.so", "libncfw.so.2", + NULL, NULL, }; for (int i = 0; replacements[i] != NULL; i += 2) { From 5263a170d6a0718413c8acb10d4367abaabd26a4 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 1 Aug 2025 09:46:07 +0200 Subject: [PATCH 009/881] examples/llama: switch to `--hf-model-path` flag (#270) Instead of giving config, model weights and tokenizer paths, rely on `huggingface-cli` download. --- README.md | 31 +++++---- examples/llama/BUILD.bazel | 136 ++----------------------------------- examples/llama/main.zig | 62 ++++++++++------- 3 files changed, 60 insertions(+), 169 deletions(-) diff --git a/README.md b/README.md index a93714df..309d26c3 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ you, too, to build cool and exciting AI projects. To give you a glimpse of what you can do with ZML, here is an early demo: -
+
It shows a prototype running a LLaMA2 model sharded on 1 NVIDIA RTX 4090, 1 AMD 6800XT, and 1 Google Cloud TPU v2. All accelerators were hosted in different @@ -118,18 +118,21 @@ This model has restrictions, see [here](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct). It **requires approval from Meta on Huggingface**, which can take a few hours to get granted. -While waiting, you can already generate an access token to log into HuggingFace -from `bazel`; see [here](./docs/huggingface-access-token.md). - Once you've been granted access, you're ready to download a gated model like -`Meta-Llama-3.1-8B-Instruct`! +`Llama-3.1-8B-Instruct`! + +First, you need to download the model using the [huggingface-cli](https://huggingface.co/docs/huggingface_hub/en/guides/cli). + +``` +huggingface-cli download meta-llama/Llama-3.1-8B-Instruct --local-dir $HOME/Llama-3.1-8B-Instruct +``` + +Then, you can run the model. ``` -# requires token in $HOME/.cache/huggingface/token, as created by the -# `huggingface-cli login` command, or the `HUGGINGFACE_TOKEN` environment variable. cd examples -bazel run --config=release //llama:Llama-3.1-8B-Instruct -bazel run --config=release //llama:Llama-3.1-8B-Instruct -- --prompt="What is the capital of France?" +bazel run --config=release //llama -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct +bazel run --config=release //llama -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct --prompt="What is the capital of France?" ``` You can also try `Llama-3.1-70B-Instruct` if you have enough memory. @@ -140,6 +143,7 @@ Like the 8B model above, this model also requires approval. See [here](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) for access requirements. ``` +huggingface-cli download meta-llama/Llama-3.2-1B-Instruct --local-dir $HOME/Llama-3.2-1B-Instruct cd examples bazel run --config=release //llama:Llama-3.2-1B-Instruct bazel run --config=release //llama:Llama-3.2-1B-Instruct -- --prompt="What is the capital of France?" @@ -160,14 +164,15 @@ following arguments to the command line when compiling / running a model: The latter, avoiding compilation for CPU, cuts down compilation time. -So, to run the OpenLLama model from above on your host sporting an NVIDIA GPU, +So, to run the Llama 3.1 8B model from above on your host sporting an NVIDIA GPU, run the following: ``` cd examples -bazel run --config=release //llama:Llama-3.2-1B-Instruct \ - --@zml//runtimes:cuda=true \ - -- --prompt="What is the capital of France?" +bazel run --config=release //llama \ + --@zml//runtimes:cuda=true \ + -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct \ + --prompt="What is the capital of France?" ``` diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel index d657f4a1..839791b2 100644 --- a/examples/llama/BUILD.bazel +++ b/examples/llama/BUILD.bazel @@ -21,106 +21,6 @@ zig_cc_binary( ], ) -cc_binary( - name = "Llama-3.1-8B-Instruct", - args = [ - "--config=$(location @Meta-Llama-3.1-8B-Instruct//:config.json)", - "--weights=$(location @Meta-Llama-3.1-8B-Instruct//:model.safetensors.index.json)", - "--tokenizer=$(location @Meta-Llama-3.1-8B-Instruct//:tokenizer.json)", - ], - data = [ - "@Meta-Llama-3.1-8B-Instruct", - "@Meta-Llama-3.1-8B-Instruct//:config.json", - "@Meta-Llama-3.1-8B-Instruct//:model.safetensors.index.json", - "@Meta-Llama-3.1-8B-Instruct//:tokenizer.json", - ], - tags = [ - "manual", - ], - deps = [":llama_lib"], -) - -cc_binary( - name = "Llama-3.1-70B-Instruct", - args = [ - "--config=$(location @Meta-Llama-3.1-70B-Instruct//:config.json)", - "--weights=$(location @Meta-Llama-3.1-70B-Instruct//:model.safetensors.index.json)", - "--tokenizer=$(location @Meta-Llama-3.1-70B-Instruct//:tokenizer.json)", - ], - data = [ - "@Meta-Llama-3.1-70B-Instruct", - "@Meta-Llama-3.1-70B-Instruct//:config.json", - "@Meta-Llama-3.1-70B-Instruct//:model.safetensors.index.json", - "@Meta-Llama-3.1-70B-Instruct//:tokenizer.json", - ], - tags = [ - "manual", - ], - deps = [":llama_lib"], -) - -cc_binary( - name = "Llama-3.2-1B-Instruct", - args = [ - "--config=$(location @Meta-Llama-3.2-1B-Instruct//:config.json)", - "--weights=$(location @Meta-Llama-3.2-1B-Instruct//:model.safetensors)", - "--tokenizer=$(location @Meta-Llama-3.2-1B-Instruct//:tokenizer.json)", - ], - data = [ - "@Meta-Llama-3.2-1B-Instruct", - "@Meta-Llama-3.2-1B-Instruct//:config.json", - "@Meta-Llama-3.2-1B-Instruct//:model.safetensors", - "@Meta-Llama-3.2-1B-Instruct//:tokenizer.json", - ], - tags = [ - "manual", - ], - deps = [":llama_lib"], -) - -cc_binary( - name = "Llama-3.2-3B-Instruct", - args = [ - "--config=$(location @Meta-Llama-3.2-3B-Instruct//:config.json)", - "--weights=$(location @Meta-Llama-3.2-3B-Instruct//:model.safetensors.index.json)", - "--tokenizer=$(location @Meta-Llama-3.2-3B-Instruct//:tokenizer.json)", - ], - data = [ - "@Meta-Llama-3.2-3B-Instruct", - "@Meta-Llama-3.2-3B-Instruct//:config.json", - "@Meta-Llama-3.2-3B-Instruct//:model.safetensors.index.json", - "@Meta-Llama-3.2-3B-Instruct//:tokenizer.json", - ], - tags = [ - "manual", - ], - deps = [":llama_lib"], -) - -cc_binary( - name = "TinyLlama-Stories-15M", - args = [ - "--config=$(location :tinyllama_stories15M_json)", - "--weights=$(location @Karpathy-TinyLlama-Stories15M//file)", - "--tokenizer=$(location @Karpathy-TinyLlama-Tokenizer//file)", - "--prompt='Once upon a time, there was a little girl named Lily.'", - "--no-llama3=1", # don't do template prompt encoding, I'm a simple model - "--sharding=false", # don't shard me, I'm so small - ], - data = [ - ":tinyllama_stories15M_json", - "@Karpathy-TinyLlama-Stories15M//file", - "@Karpathy-TinyLlama-Tokenizer//file", - ], - deps = [":llama_lib"], -) - -write_file( - name = "tinyllama_stories15M_json", - out = "config.json", - content = ['{"bos_token_id":1,"eos_token_id":2,"hidden_act":"silu","hidden_size":288,"intermediate_size":768,"max_position_embeddings":256,"model_type":"llama","num_attention_heads":6,"num_hidden_layers":6,"num_key_value_heads":6,"rms_norm_eps":1e-05,"hf_rope_impl":false,"rope_scaling":null,"rope_theta":10000.0}'], -) - zig_cc_binary( name = "test-implementation", srcs = ["llama.zig"], @@ -169,7 +69,7 @@ Artificial Intelligence in Healthcare mtree_spec( name = "mtree", - srcs = [":Llama-3.2-1B-Instruct"], + srcs = [":llama"], tags = [ "manual", ], @@ -177,7 +77,7 @@ mtree_spec( tar( name = "archive", - srcs = [":Llama-3.2-1B-Instruct"], + srcs = [":llama"], args = [ "--options", "zstd:compression-level=9", @@ -189,36 +89,10 @@ tar( ], ) -expand_template( - name = "entrypoint", - data = [ - ":Llama-3.2-1B-Instruct", - "@Meta-Llama-3.2-1B-Instruct", - "@Meta-Llama-3.2-1B-Instruct//:config.json", - "@Meta-Llama-3.2-1B-Instruct//:model.safetensors", - "@Meta-Llama-3.2-1B-Instruct//:tokenizer.json", - ], - substitutions = { - ":config": "$(rlocationpath @Meta-Llama-3.2-1B-Instruct//:config.json)", - ":weights": "$(rlocationpath @Meta-Llama-3.2-1B-Instruct//:model.safetensors)", - ":tokenizer": "$(rlocationpath @Meta-Llama-3.2-1B-Instruct//:tokenizer.json)", - }, - tags = [ - "manual", - ], - template = [ - "./{}/Llama-3.2-1B-Instruct".format(package_name()), - "--config=./{}/Llama-3.2-1B-Instruct.runfiles/:config".format(package_name()), - "--weights=./{}/Llama-3.2-1B-Instruct.runfiles/:weights".format(package_name()), - "--tokenizer=./{}/Llama-3.2-1B-Instruct.runfiles/:tokenizer".format(package_name()), - ], -) - oci_image( name = "image_", base = "@distroless_cc_debian12_debug", - # entrypoint = ["./{}/Llama-3.2-1B-Instruct".format(package_name())], - entrypoint = ":entrypoint", + entrypoint = ["./{}/llama".format(package_name())], tags = [ "manual", ], @@ -241,7 +115,7 @@ oci_load( name = "load", image = ":image", repo_tags = [ - "distroless/llama-3.2-1b-instruct:latest", + "distroless/llama:latest", ], tags = [ "manual", @@ -252,7 +126,7 @@ oci_push( name = "push", image = ":image", remote_tags = ["latest"], - repository = "index.docker.io/steeve/llama-3.2-1b-instruct", + repository = "index.docker.io/steeve/llama", tags = [ "manual", ], diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 0b121e65..0401444f 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -151,9 +151,7 @@ pub fn generateText( const params = clap.parseParamsComptime( \\--help print this help \\--prompt the prompt - \\--config config.json path - \\--weights model weights path - \\--tokenizer tokenizer path + \\--hf-model-path path to the directory containing model weights, config and tokenizer \\--seed random seed (optional) \\--seq-len sequence length \\--create-options platform creation options JSON, defaults to {} @@ -199,18 +197,37 @@ pub fn asyncMain() !void { return; } - const config = blk: { - if (res.args.config) |config_json_path| { - var config_json_file = try asynk.File.open(config_json_path, .{ .mode = .read_only }); - defer config_json_file.close() catch unreachable; - var reader = std.json.reader(allocator, config_json_file.reader()); - defer reader.deinit(); - const config_obj = try std.json.parseFromTokenSourceLeaky(llama.LlamaLM.Config, allocator, &reader, .{ .ignore_unknown_fields = true }); - break :blk config_obj; - } else { - log.err("Missing --config", .{}); - return; + const hf_model_path = res.args.@"hf-model-path" orelse { + log.err("Missing --hf-model-path", .{}); + return; + }; + + const model_config_path = try std.fs.path.join(allocator, &.{ hf_model_path, "config.json" }); + defer allocator.free(model_config_path); + + const model_weights_path = b: { + const simple_path = try std.fs.path.join(allocator, &.{ hf_model_path, "model.safetensors" }); + if (asynk.File.access(simple_path, .{})) { + break :b simple_path; + } else |_| { + allocator.free(simple_path); } + + const sharded_path = try std.fs.path.join(allocator, &.{ hf_model_path, "model.safetensors.index.json" }); + break :b sharded_path; + }; + defer allocator.free(model_weights_path); + + const model_tokenizer_path = try std.fs.path.join(allocator, &.{ hf_model_path, "tokenizer.json" }); + defer allocator.free(model_tokenizer_path); + + const config = blk: { + var config_json_file = try asynk.File.open(model_config_path, .{ .mode = .read_only }); + defer config_json_file.close() catch unreachable; + var reader = std.json.reader(allocator, config_json_file.reader()); + defer reader.deinit(); + const config_obj = try std.json.parseFromTokenSourceLeaky(llama.LlamaLM.Config, allocator, &reader, .{ .ignore_unknown_fields = true }); + break :blk config_obj; }; var context = try zml.Context.init(); @@ -229,7 +246,7 @@ pub fn asyncMain() !void { create_opts.deinit(); context.printAvailablePlatforms(platform); - var ts = try zml.aio.detectFormatAndOpen(allocator, res.args.weights.?); + var ts = try zml.aio.detectFormatAndOpen(allocator, model_weights_path); defer ts.deinit(); var model_arena = std.heap.ArenaAllocator.init(allocator); @@ -281,7 +298,7 @@ pub fn asyncMain() !void { platform, }); - log.info("\tLoading Llama weights from {?s}...", .{res.args.weights}); + log.info("\tLoading Llama weights from {?s}...", .{model_weights_path}); var llama_weights = try zml.aio.loadBuffers(llama.LlamaLM, .{ config, llama_options }, ts, model_arena.allocator(), platform); defer zml.aio.unloadBuffers(&llama_weights); log.info("✅\tLoaded weights in {}", .{std.fmt.fmtDuration(start.read())}); @@ -296,16 +313,11 @@ pub fn asyncMain() !void { const kv_cache = try llama.KvCache.initBuffer(kv_shape, platform); var tokenizer = blk: { - if (res.args.tokenizer) |tok| { - log.info("Loading tokenizer from {s}", .{tok}); - var timer = try stdx.time.Timer.start(); - defer log.info("Loaded tokenizer from {s} [{}]", .{ tok, timer.read() }); + log.info("Loading tokenizer from {s}", .{model_tokenizer_path}); + var timer = try stdx.time.Timer.start(); + defer log.info("Loaded tokenizer from {s} [{}]", .{ model_tokenizer_path, timer.read() }); - break :blk try zml.tokenizer.Tokenizer.fromFile(model_arena.allocator(), tok); - } else { - log.err("Missing --tokenizer", .{}); - return; - } + break :blk try zml.tokenizer.Tokenizer.fromFile(model_arena.allocator(), model_tokenizer_path); }; errdefer tokenizer.deinit(); From 9f2ccf8a54f2bd466484ae6a9d697ba3ccb66c14 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 1 Aug 2025 16:42:29 +0200 Subject: [PATCH 010/881] package HF cli so users don't have to create Python venv (#293) --- MODULE.bazel | 10 + MODULE.bazel.lock | 353 ++++++++++++++++++++++- README.md | 12 +- docs/howtos/deploy_on_server.md | 14 +- docs/howtos/huggingface_access_token.md | 30 +- docs/huggingface-access-token.md | 37 --- docs/tutorials/getting_started.md | 10 +- examples/MODULE.bazel.lock | 355 +++++++++++++++++++++++- tools/BUILD.bazel | 27 ++ tools/hf_requirements.in | 1 + tools/hf_requirements.lock.txt | 203 ++++++++++++++ 11 files changed, 979 insertions(+), 73 deletions(-) delete mode 100644 docs/huggingface-access-token.md create mode 100644 tools/hf_requirements.in create mode 100644 tools/hf_requirements.lock.txt diff --git a/MODULE.bazel b/MODULE.bazel index ce4383ab..9ff10eb5 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -79,6 +79,16 @@ pip.parse( ) use_repo(pip, "neuron_py_deps") +# For downloading models +pip.parse( + download_only = True, + hub_name = "huggingface_hub", + python_version = "3.11", + requirements_lock = "//tools:hf_requirements.lock.txt", +) +use_repo(pip, "huggingface_hub") + + cpu = use_extension("//runtimes/cpu:cpu.bzl", "cpu_pjrt_plugin") use_repo(cpu, "libpjrt_cpu_darwin_amd64", "libpjrt_cpu_darwin_arm64", "libpjrt_cpu_linux_amd64") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index cca6007a..248424bf 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1031,9 +1031,10 @@ "@@rules_python+//python/extensions:pip.bzl%pip": { "general": { "bzlTransitiveDigest": "fOwLlf5Rfxv4xkFCZwPlkSPUuZ9lAhMGf0SN43pc0xg=", - "usagesDigest": "3+0h/pprjG6w7aHgvrT2H3LdTTVQQ79Mz0lATHIw+/I=", + "usagesDigest": "Y6wFcKuqojwXVUMUxBiLATIjxW3VxUxL71LIt7iDHkw=", "recordedFileInputs": { "@@//runtimes/neuron/requirements.lock.txt": "fa0b5b967162e303415ee1dd7d2c1a448068935663ba5ac5b5f72a515fe3f6f6", + "@@//tools/hf_requirements.lock.txt": "ffbcfcfd01c9a31db823affa4ab6122c509b2f1eeb4b853325d1dbd5cc9568da", "@@protobuf+//python/requirements.txt": "2bb341a0ba317fb8dc9ac79817b9b2d8f6a4d0c43e3fb652f0cab584eed420a4", "@@rules_fuzzing+//fuzzing/requirements.txt": "d4e91a1b3eca06d6e2877f67d3d8be498cb3ed6d28b9c14439b6f210ec621470", "@@rules_python+//tools/publish/requirements_darwin.txt": "6d10732737e5504291360b78fbf2b82abe39a5a021bd937f987f8433519b0f2c", @@ -1046,6 +1047,318 @@ "RULES_PYTHON_REPO_DEBUG_VERBOSITY": null }, "generatedRepoSpecs": { + "huggingface_hub_311_certifi": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "certifi==2025.7.14 --hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 --hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995" + } + }, + "huggingface_hub_311_charset_normalizer": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "charset-normalizer==3.4.2 --hash=sha256:005fa3432484527f9732ebd315da8da8001593e2cf46a3d817669f062c3d9ed4 --hash=sha256:046595208aae0120559a67693ecc65dd75d46f7bf687f159127046628178dc45 --hash=sha256:0c29de6a1a95f24b9a1aa7aefd27d2487263f00dfd55a77719b530788f75cff7 --hash=sha256:0c8c57f84ccfc871a48a47321cfa49ae1df56cd1d965a09abe84066f6853b9c0 --hash=sha256:0f5d9ed7f254402c9e7d35d2f5972c9bbea9040e99cd2861bd77dc68263277c7 --hash=sha256:18dd2e350387c87dabe711b86f83c9c78af772c748904d372ade190b5c7c9d4d --hash=sha256:1b1bde144d98e446b056ef98e59c256e9294f6b74d7af6846bf5ffdafd687a7d --hash=sha256:1c95a1e2902a8b722868587c0e1184ad5c55631de5afc0eb96bc4b0d738092c0 --hash=sha256:1cad5f45b3146325bb38d6855642f6fd609c3f7cad4dbaf75549bf3b904d3184 --hash=sha256:21b2899062867b0e1fde9b724f8aecb1af14f2778d69aacd1a5a1853a597a5db --hash=sha256:24498ba8ed6c2e0b56d4acbf83f2d989720a93b41d712ebd4f4979660db4417b --hash=sha256:25a23ea5c7edc53e0f29bae2c44fcb5a1aa10591aae107f2a2b2583a9c5cbc64 --hash=sha256:289200a18fa698949d2b39c671c2cc7a24d44096784e76614899a7ccf2574b7b --hash=sha256:28a1005facc94196e1fb3e82a3d442a9d9110b8434fc1ded7a24a2983c9888d8 --hash=sha256:32fc0341d72e0f73f80acb0a2c94216bd704f4f0bce10aedea38f30502b271ff --hash=sha256:36b31da18b8890a76ec181c3cf44326bf2c48e36d393ca1b72b3f484113ea344 --hash=sha256:3c21d4fca343c805a52c0c78edc01e3477f6dd1ad7c47653241cf2a206d4fc58 --hash=sha256:3fddb7e2c84ac87ac3a947cb4e66d143ca5863ef48e4a5ecb83bd48619e4634e --hash=sha256:43e0933a0eff183ee85833f341ec567c0980dae57c464d8a508e1b2ceb336471 --hash=sha256:4a476b06fbcf359ad25d34a057b7219281286ae2477cc5ff5e3f70a246971148 --hash=sha256:4e594135de17ab3866138f496755f302b72157d115086d100c3f19370839dd3a --hash=sha256:50bf98d5e563b83cc29471fa114366e6806bc06bc7a25fd59641e41445327836 --hash=sha256:5a9979887252a82fefd3d3ed2a8e3b937a7a809f65dcb1e068b090e165bbe99e --hash=sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63 --hash=sha256:5bf4545e3b962767e5c06fe1738f951f77d27967cb2caa64c28be7c4563e162c --hash=sha256:6333b3aa5a12c26b2a4d4e7335a28f1475e0e5e17d69d55141ee3cab736f66d1 --hash=sha256:65c981bdbd3f57670af8b59777cbfae75364b483fa8a9f420f08094531d54a01 --hash=sha256:68a328e5f55ec37c57f19ebb1fdc56a248db2e3e9ad769919a58672958e8f366 --hash=sha256:6a0289e4589e8bdfef02a80478f1dfcb14f0ab696b5a00e1f4b8a14a307a3c58 --hash=sha256:6b66f92b17849b85cad91259efc341dce9c1af48e2173bf38a85c6329f1033e5 --hash=sha256:6c9379d65defcab82d07b2a9dfbfc2e95bc8fe0ebb1b176a3190230a3ef0e07c --hash=sha256:6fc1f5b51fa4cecaa18f2bd7a003f3dd039dd615cd69a2afd6d3b19aed6775f2 --hash=sha256:70f7172939fdf8790425ba31915bfbe8335030f05b9913d7ae00a87d4395620a --hash=sha256:721c76e84fe669be19c5791da68232ca2e05ba5185575086e384352e2c309597 --hash=sha256:7222ffd5e4de8e57e03ce2cef95a4c43c98fcb72ad86909abdfc2c17d227fc1b --hash=sha256:75d10d37a47afee94919c4fab4c22b9bc2a8bf7d4f46f87363bcf0573f3ff4f5 --hash=sha256:76af085e67e56c8816c3ccf256ebd136def2ed9654525348cfa744b6802b69eb --hash=sha256:770cab594ecf99ae64c236bc9ee3439c3f46be49796e265ce0cc8bc17b10294f --hash=sha256:7a6ab32f7210554a96cd9e33abe3ddd86732beeafc7a28e9955cdf22ffadbab0 --hash=sha256:7c48ed483eb946e6c04ccbe02c6b4d1d48e51944b6db70f697e089c193404941 --hash=sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0 --hash=sha256:8075c35cd58273fee266c58c0c9b670947c19df5fb98e7b66710e04ad4e9ff86 --hash=sha256:8272b73e1c5603666618805fe821edba66892e2870058c94c53147602eab29c7 --hash=sha256:82d8fd25b7f4675d0c47cf95b594d4e7b158aca33b76aa63d07186e13c0e0ab7 --hash=sha256:844da2b5728b5ce0e32d863af26f32b5ce61bc4273a9c720a9f3aa9df73b1455 --hash=sha256:8755483f3c00d6c9a77f490c17e6ab0c8729e39e6390328e42521ef175380ae6 --hash=sha256:915f3849a011c1f593ab99092f3cecfcb4d65d8feb4a64cf1bf2d22074dc0ec4 --hash=sha256:926ca93accd5d36ccdabd803392ddc3e03e6d4cd1cf17deff3b989ab8e9dbcf0 --hash=sha256:982bb1e8b4ffda883b3d0a521e23abcd6fd17418f6d2c4118d257a10199c0ce3 --hash=sha256:98f862da73774290f251b9df8d11161b6cf25b599a66baf087c1ffe340e9bfd1 --hash=sha256:9cbfacf36cb0ec2897ce0ebc5d08ca44213af24265bd56eca54bee7923c48fd6 --hash=sha256:a370b3e078e418187da8c3674eddb9d983ec09445c99a3a263c2011993522981 --hash=sha256:a955b438e62efdf7e0b7b52a64dc5c3396e2634baa62471768a64bc2adb73d5c --hash=sha256:aa6af9e7d59f9c12b33ae4e9450619cf2488e2bbe9b44030905877f0b2324980 --hash=sha256:aa88ca0b1932e93f2d961bf3addbb2db902198dca337d88c89e1559e066e7645 --hash=sha256:aaeeb6a479c7667fbe1099af9617c83aaca22182d6cf8c53966491a0f1b7ffb7 --hash=sha256:aaf27faa992bfee0264dc1f03f4c75e9fcdda66a519db6b957a3f826e285cf12 --hash=sha256:b2680962a4848b3c4f155dc2ee64505a9c57186d0d56b43123b17ca3de18f0fa --hash=sha256:b2d318c11350e10662026ad0eb71bb51c7812fc8590825304ae0bdd4ac283acd --hash=sha256:b33de11b92e9f75a2b545d6e9b6f37e398d86c3e9e9653c4864eb7e89c5773ef --hash=sha256:b3daeac64d5b371dea99714f08ffc2c208522ec6b06fbc7866a450dd446f5c0f --hash=sha256:be1e352acbe3c78727a16a455126d9ff83ea2dfdcbc83148d2982305a04714c2 --hash=sha256:bee093bf902e1d8fc0ac143c88902c3dfc8941f7ea1d6a8dd2bcb786d33db03d --hash=sha256:c72fbbe68c6f32f251bdc08b8611c7b3060612236e960ef848e0a517ddbe76c5 --hash=sha256:c9e36a97bee9b86ef9a1cf7bb96747eb7a15c2f22bdb5b516434b00f2a599f02 --hash=sha256:cddf7bd982eaa998934a91f69d182aec997c6c468898efe6679af88283b498d3 --hash=sha256:cf713fe9a71ef6fd5adf7a79670135081cd4431c2943864757f0fa3a65b1fafd --hash=sha256:d11b54acf878eef558599658b0ffca78138c8c3655cf4f3a4a673c437e67732e --hash=sha256:d41c4d287cfc69060fa91cae9683eacffad989f1a10811995fa309df656ec214 --hash=sha256:d524ba3f1581b35c03cb42beebab4a13e6cdad7b36246bd22541fa585a56cccd --hash=sha256:daac4765328a919a805fa5e2720f3e94767abd632ae410a9062dff5412bae65a --hash=sha256:db4c7bf0e07fc3b7d89ac2a5880a6a8062056801b83ff56d8464b70f65482b6c --hash=sha256:dc7039885fa1baf9be153a0626e337aa7ec8bf96b0128605fb0d77788ddc1681 --hash=sha256:dccab8d5fa1ef9bfba0590ecf4d46df048d18ffe3eec01eeb73a42e0d9e7a8ba --hash=sha256:dedb8adb91d11846ee08bec4c8236c8549ac721c245678282dcb06b221aab59f --hash=sha256:e45ba65510e2647721e35323d6ef54c7974959f6081b58d4ef5d87c60c84919a --hash=sha256:e53efc7c7cee4c1e70661e2e112ca46a575f90ed9ae3fef200f2a25e954f4b28 --hash=sha256:e635b87f01ebc977342e2697d05b56632f5f879a4f15955dfe8cef2448b51691 --hash=sha256:e70e990b2137b29dc5564715de1e12701815dacc1d056308e2b17e9095372a82 --hash=sha256:e8082b26888e2f8b36a042a58307d5b917ef2b1cacab921ad3323ef91901c71a --hash=sha256:e8323a9b031aa0393768b87f04b4164a40037fb2a3c11ac06a03ffecd3618027 --hash=sha256:e92fca20c46e9f5e1bb485887d074918b13543b1c2a1185e69bb8d17ab6236a7 --hash=sha256:eb30abc20df9ab0814b5a2524f23d75dcf83cde762c161917a2b4b7b55b1e518 --hash=sha256:eba9904b0f38a143592d9fc0e19e2df0fa2e41c3c3745554761c5f6447eedabf --hash=sha256:ef8de666d6179b009dce7bcb2ad4c4a779f113f12caf8dc77f0162c29d20490b --hash=sha256:efd387a49825780ff861998cd959767800d54f8308936b21025326de4b5a42b9 --hash=sha256:f0aa37f3c979cf2546b73e8222bbfa3dc07a641585340179d768068e3455e544 --hash=sha256:f4074c5a429281bf056ddd4c5d3b740ebca4d43ffffe2ef4bf4d2d05114299da --hash=sha256:f69a27e45c43520f5487f27627059b64aaf160415589230992cec34c5e18a509 --hash=sha256:fb707f3e15060adf5b7ada797624a6c6e0138e2a26baa089df64c68ee98e040f --hash=sha256:fcbe676a55d7445b22c10967bceaaf0ee69407fbe0ece4d032b6eb8d4565982a --hash=sha256:fdb20a30fe1175ecabed17cbf7812f7b804b8a315a25f24678bcdf120a90077f" + } + }, + "huggingface_hub_311_filelock": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "filelock==3.18.0 --hash=sha256:adbc88eabb99d2fec8c9c1b229b171f18afa655400173ddc653d5d01501fb9f2 --hash=sha256:c401f4f8377c4464e6db25fff06205fd89bdd83b65eb0488ed1b160f780e21de" + } + }, + "huggingface_hub_311_fsspec": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "fsspec==2025.7.0 --hash=sha256:786120687ffa54b8283d942929540d8bc5ccfa820deb555a2b5d0ed2b737bf58 --hash=sha256:8b012e39f63c7d5f10474de957f3ab793b47b45ae7d39f2fb735f8bbe25c0e21" + } + }, + "huggingface_hub_311_hf_xet": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "hf-xet==1.1.5 --hash=sha256:69ebbcfd9ec44fdc2af73441619eeb06b94ee34511bbcf57cd423820090f5694 --hash=sha256:73e167d9807d166596b4b2f0b585c6d5bd84a26dea32843665a8b58f6edba245 --hash=sha256:83088ecea236d5113de478acb2339f92c95b4fb0462acaa30621fac02f5a534a --hash=sha256:9fa6e3ee5d61912c4a113e0708eaaef987047616465ac7aa30f7121a48fc1af8 --hash=sha256:ab34c4c3104133c495785d5d8bba3b1efc99de52c02e759cf711a91fd39d3a14 --hash=sha256:dbba1660e5d810bd0ea77c511a99e9242d920790d0e63c0e4673ed36c4022d18 --hash=sha256:f52c2fa3635b8c37c7764d8796dfa72706cc4eded19d638331161e82b0792e23 --hash=sha256:fc874b5c843e642f45fd85cda1ce599e123308ad2901ead23d3510a47ff506d1" + } + }, + "huggingface_hub_311_huggingface_hub": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "huggingface-hub==0.34.3 --hash=sha256:5444550099e2d86e68b2898b09e85878fbd788fc2957b506c6a79ce060e39492 --hash=sha256:d58130fd5aa7408480681475491c0abd7e835442082fbc3ef4d45b6c39f83853" + } + }, + "huggingface_hub_311_idna": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "idna==3.10 --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3" + } + }, + "huggingface_hub_311_packaging": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "packaging==25.0 --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f" + } + }, + "huggingface_hub_311_pyyaml": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "pyyaml==6.0.2 --hash=sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff --hash=sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48 --hash=sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086 --hash=sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e --hash=sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133 --hash=sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5 --hash=sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484 --hash=sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee --hash=sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5 --hash=sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68 --hash=sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a --hash=sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf --hash=sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99 --hash=sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8 --hash=sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85 --hash=sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19 --hash=sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc --hash=sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a --hash=sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1 --hash=sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317 --hash=sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c --hash=sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631 --hash=sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d --hash=sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652 --hash=sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5 --hash=sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e --hash=sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b --hash=sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8 --hash=sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476 --hash=sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706 --hash=sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563 --hash=sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237 --hash=sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b --hash=sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083 --hash=sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180 --hash=sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425 --hash=sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e --hash=sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f --hash=sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725 --hash=sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183 --hash=sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab --hash=sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774 --hash=sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725 --hash=sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e --hash=sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5 --hash=sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d --hash=sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290 --hash=sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44 --hash=sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed --hash=sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4 --hash=sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba --hash=sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12 --hash=sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4" + } + }, + "huggingface_hub_311_requests": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "requests==2.32.4 --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422" + } + }, + "huggingface_hub_311_tqdm": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "tqdm==4.67.1 --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2" + } + }, + "huggingface_hub_311_typing_extensions": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "typing-extensions==4.14.1 --hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 --hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76" + } + }, + "huggingface_hub_311_urllib3": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "urllib3==2.5.0 --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc" + } + }, "neuron_py_deps_311_boto3": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { @@ -4143,6 +4456,44 @@ ] } }, + "huggingface_hub": { + "repoRuleId": "@@rules_python+//python/private/pypi:hub_repository.bzl%hub_repository", + "attributes": { + "repo_name": "huggingface_hub", + "extra_hub_aliases": {}, + "whl_map": { + "certifi": "{\"huggingface_hub_311_certifi\":[{\"version\":\"3.11\"}]}", + "charset_normalizer": "{\"huggingface_hub_311_charset_normalizer\":[{\"version\":\"3.11\"}]}", + "filelock": "{\"huggingface_hub_311_filelock\":[{\"version\":\"3.11\"}]}", + "fsspec": "{\"huggingface_hub_311_fsspec\":[{\"version\":\"3.11\"}]}", + "hf_xet": "{\"huggingface_hub_311_hf_xet\":[{\"version\":\"3.11\"}]}", + "huggingface_hub": "{\"huggingface_hub_311_huggingface_hub\":[{\"version\":\"3.11\"}]}", + "idna": "{\"huggingface_hub_311_idna\":[{\"version\":\"3.11\"}]}", + "packaging": "{\"huggingface_hub_311_packaging\":[{\"version\":\"3.11\"}]}", + "pyyaml": "{\"huggingface_hub_311_pyyaml\":[{\"version\":\"3.11\"}]}", + "requests": "{\"huggingface_hub_311_requests\":[{\"version\":\"3.11\"}]}", + "tqdm": "{\"huggingface_hub_311_tqdm\":[{\"version\":\"3.11\"}]}", + "typing_extensions": "{\"huggingface_hub_311_typing_extensions\":[{\"version\":\"3.11\"}]}", + "urllib3": "{\"huggingface_hub_311_urllib3\":[{\"version\":\"3.11\"}]}" + }, + "packages": [ + "certifi", + "charset_normalizer", + "filelock", + "fsspec", + "hf_xet", + "huggingface_hub", + "idna", + "packaging", + "pyyaml", + "requests", + "tqdm", + "typing_extensions", + "urllib3" + ], + "groups": {} + } + }, "neuron_py_deps": { "repoRuleId": "@@rules_python+//python/private/pypi:hub_repository.bzl%hub_repository", "attributes": { diff --git a/README.md b/README.md index 309d26c3..940d056c 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ To give you a glimpse of what you can do with ZML, here is an early demo:
-It shows a prototype running a LLaMA2 model sharded on 1 NVIDIA RTX 4090, 1 AMD +It shows a prototype running a LLama3 model sharded on 1 NVIDIA RTX 4090, 1 AMD 6800XT, and 1 Google Cloud TPU v2. All accelerators were hosted in different locations, with activations being passed over a VPN. @@ -122,9 +122,11 @@ Once you've been granted access, you're ready to download a gated model like `Llama-3.1-8B-Instruct`! First, you need to download the model using the [huggingface-cli](https://huggingface.co/docs/huggingface_hub/en/guides/cli). +Note you don't need to install it yourself, +you can just use the packaged version `bazel run @zml//tools:hf --`. ``` -huggingface-cli download meta-llama/Llama-3.1-8B-Instruct --local-dir $HOME/Llama-3.1-8B-Instruct +bazel run @zml//tools:hf -- download meta-llama/Llama-3.1-8B-Instruct --local-dir $HOME/Llama-3.1-8B-Instruct --exclude='*.pth' ``` Then, you can run the model. @@ -143,10 +145,10 @@ Like the 8B model above, this model also requires approval. See [here](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) for access requirements. ``` -huggingface-cli download meta-llama/Llama-3.2-1B-Instruct --local-dir $HOME/Llama-3.2-1B-Instruct +bazel run @zml//tools:hf -- download meta-llama/Llama-3.2-1B-Instruct --local-dir $HOME/Llama-3.2-1B-Instruct --exclude='*.pth' cd examples -bazel run --config=release //llama:Llama-3.2-1B-Instruct -bazel run --config=release //llama:Llama-3.2-1B-Instruct -- --prompt="What is the capital of France?" +bazel run --config=release //llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct +bazel run --config=release //llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct -- --prompt="What is the capital of France?" ``` For a larger 3.2 model, you can also try `Llama-3.2-3B-Instruct`. diff --git a/docs/howtos/deploy_on_server.md b/docs/howtos/deploy_on_server.md index 31bbd759..b2961051 100644 --- a/docs/howtos/deploy_on_server.md +++ b/docs/howtos/deploy_on_server.md @@ -20,12 +20,12 @@ following arguments to the command line when compiling / running a model: - AWS Trainium/Inferentia 2: `--@zml//runtimes:neuron=true` - **AVOID CPU:** `--@zml//runtimes:cpu=false` -So, to run the OpenLLama model from above **on your development machine** +So, to run the Llama model from above **on your development machine** housing an NVIDIA GPU, run the following: ``` cd examples -bazel run --config=release //llama:OpenLLaMA-3B --@zml//runtimes:cuda=true +bazel run --config=release //llama --@zml//runtimes:cuda=true -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct ``` @@ -38,14 +38,14 @@ architectures: - Linux ARM64: `--platforms=@zml//platforms:linux_arm64` - MacOS ARM64: `--platforms=@zml//platforms:macos_arm64` -As an example, here is how you build above OpenLLama for CUDA on Linux X86_64: +As an example, here is how you build above Llama for CUDA on Linux X86_64: ``` cd examples -bazel build --config=release //llama:OpenLLaMA-3B \ - --@zml//runtimes:cuda=true \ - --@zml//runtimes:cpu=false \ - --platforms=@zml//platforms:linux_amd64 +bazel build --config=release //llama \ + --@zml//runtimes:cuda=true \ + --@zml//runtimes:cpu=false \ + --platforms=@zml//platforms:linux_amd64 ``` ### Creating the TAR diff --git a/docs/howtos/huggingface_access_token.md b/docs/howtos/huggingface_access_token.md index 77cbaf19..c2415c3d 100644 --- a/docs/howtos/huggingface_access_token.md +++ b/docs/howtos/huggingface_access_token.md @@ -1,24 +1,21 @@ +# Running Gated Huggingface Models with Token Authentication -# Huggingface Token Authentication +Some models have restrictions and may require some sort of approval or agreement +process, which, by consequence, **requires token-authentication with Huggingface**. -Some models have restrictions and may require some sort of approval or -agreement process, which, by consequence, **requires token-authentication with -Huggingface**. +The easiest way might be to use the `huggingface-cli login` command. -Here is how you can generate a **"read-only public repositories"** access token -to log into your account on Huggingface, directly from `bazel`, in order to -download models. +Alternatively, here is how you can generate a **"read-only public repositories"** +access token to log into your account on Huggingface, directly from `bazel`, in order to download models. * log in at [https://huggingface.co/settings/tokens](https://huggingface.co/settings/tokens). * click on "Create new token" -* give the token a name, eg `zml_public_repos` -* under _Repositories_, grant the following permission: "Read access to - contents of all public gated repos you can access". -* at the bottom, click on "Create token". +* give the token a name, eg `zml_public_repos`, +* under _Repositories_, grant the following permission: "Read access to contents of all public gated repos you can access". +* at the bottom click on "Create token". * copy the token by clicking `Copy`. **You won't be able to see it again.** * the token looks something like `hf_abCdEfGhijKlM`. -* store the token on your machine (replace the placeholder with your actual - token): +* store the token on your machine (replace the placeholder with your actual token): You can use the `HUGGINGFACE_TOKEN` environment variable to store the token or use its standard location: @@ -26,7 +23,7 @@ its standard location: mkdir -p $HOME/.cache/huggingface/; echo > "$HOME/.cache/huggingface/token" ``` -Now you're ready to download a gated model like `Meta-Llama-3-8b`! +Now you're ready to download a gated model like `Meta-Llama-3.2-1b`! **Example:** @@ -34,8 +31,7 @@ Now you're ready to download a gated model like `Meta-Llama-3-8b`! # requires token in $HOME/.cache/huggingface/token, as created by the # `huggingface-cli login` command, or the `HUGGINGFACE_TOKEN` environment variable. cd examples -bazel run --config=release //llama:Meta-Llama-3-8b -bazel run --config=release //llama:Meta-Llama-3-8b -- --promt="Once upon a time," +bazel run @zml//tools:hf -- download meta-llama/Llama-3.2-1B-Instruct --local-dir $HOME/Llama-3.2-1B-Instruct --exclude='*.pth' +bazel run --config=release //llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" ``` - diff --git a/docs/huggingface-access-token.md b/docs/huggingface-access-token.md deleted file mode 100644 index 633a6078..00000000 --- a/docs/huggingface-access-token.md +++ /dev/null @@ -1,37 +0,0 @@ -# Running Gated Huggingface Models with Token Authentication - -Some models have restrictions and may require some sort of approval or agreement -process, which, by consequence, **requires token-authentication with Huggingface**. - -The easiest way might be to use the `huggingface-cli login` command. - -Alternatively, here is how you can generate a **"read-only public repositories"** -access token to log into your account on Huggingface, directly from `bazel`, in order to download models. - -* log in at [https://huggingface.co/settings/tokens](https://huggingface.co/settings/tokens). -* click on "Create new token" -* give the token a name, eg `zml_public_repos`, -* under _Repositories_, grant the following permission: "Read access to contents of all public gated repos you can access". -* at the bottom click on "Create token". -* copy the token by clicking `Copy`. **You won't be able to see it again.** -* the token looks something like `hf_abCdEfGhijKlM`. -* store the token on your machine (replace the placeholder with your actual token): - -You can use the `HUGGINGFACE_TOKEN` environment variable to store the token or use -its standard location: -``` -mkdir -p $HOME/.cache/huggingface/; echo > "$HOME/.cache/huggingface/token" -``` - -Now you're ready to download a gated model like `Meta-Llama-3-8b`! - -**Example:** - -``` -# requires token in $HOME/.cache/huggingface/token, as created by the -# `huggingface-cli login` command, or the `HUGGINGFACE_TOKEN` environment variable. -cd examples -bazel run --config=release //llama:Meta-Llama-3-8b -bazel run --config=release //llama:Meta-Llama-3-8b -- --promt="Once upon a time," -``` - diff --git a/docs/tutorials/getting_started.md b/docs/tutorials/getting_started.md index 0f98da8d..b5d2fa17 100644 --- a/docs/tutorials/getting_started.md +++ b/docs/tutorials/getting_started.md @@ -75,8 +75,9 @@ Once you've been granted access, you're ready to download a gated model like # requires token in $HOME/.cache/huggingface/token, as created by the # `huggingface-cli login` command, or the `HUGGINGFACE_TOKEN` environment variable. cd examples -bazel run --config=release //llama:Llama-3.1-8B-Instruct -bazel run --config=release //llama:Llama-3.1-8B-Instruct -- --prompt="What is the capital of France?" +bazel run @zml//tools:hf -- download meta-llama/Llama-3.1-8B-Instruct --local-dir $HOME/Llama-3.1-8B-Instruct --exclude='*.pth' +bazel run --config=release //llama -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct +bazel run --config=release //llama -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct --prompt="What is the capital of France?" ``` You can also try `Llama-3.1-70B-Instruct` if you have enough memory. @@ -88,8 +89,9 @@ Like the 8B model above, this model also requires approval. See ``` cd examples -bazel run --config=release //llama:Llama-3.2-1B-Instruct -bazel run --config=release //llama:Llama-3.2-1B-Instruct -- --prompt="What is the capital of France?" +bazel run @zml//tools:hf -- download meta-llama/Llama-3.2-1B-Instruct --local-dir $HOME/Llama-3.2-1B-Instruct --exclude='*.pth' +bazel run --config=release //llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct +bazel run --config=release //llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" ``` For a larger 3.2 model, you can also try `Llama-3.2-3B-Instruct`. diff --git a/examples/MODULE.bazel.lock b/examples/MODULE.bazel.lock index aeb27f08..0f803d6f 100644 --- a/examples/MODULE.bazel.lock +++ b/examples/MODULE.bazel.lock @@ -1258,14 +1258,15 @@ "@@rules_python+//python/extensions:pip.bzl%pip": { "general": { "bzlTransitiveDigest": "fOwLlf5Rfxv4xkFCZwPlkSPUuZ9lAhMGf0SN43pc0xg=", - "usagesDigest": "+GFOusHEj2Xf1povwuB2aIeeSJ6WpIrOaatCm65zTnc=", + "usagesDigest": "+J++nGlVyZlgzi9ejqCZML6OBtqR8IvEcE8T0yFIFKA=", "recordedFileInputs": { "@@protobuf+//python/requirements.txt": "2bb341a0ba317fb8dc9ac79817b9b2d8f6a4d0c43e3fb652f0cab584eed420a4", "@@rules_fuzzing+//fuzzing/requirements.txt": "d4e91a1b3eca06d6e2877f67d3d8be498cb3ed6d28b9c14439b6f210ec621470", "@@rules_python+//tools/publish/requirements_darwin.txt": "6d10732737e5504291360b78fbf2b82abe39a5a021bd937f987f8433519b0f2c", "@@rules_python+//tools/publish/requirements_linux.txt": "cb63e4751377d1722a81fd5835b0526208620f1303feb473ed0dd45ae4d71ba9", "@@rules_python+//tools/publish/requirements_windows.txt": "f92da1a252df04e1931a0e16dfbc5a3c165a85b796b7fc5e56ee4e3846d82558", - "@@zml+//runtimes/neuron/requirements.lock.txt": "fa0b5b967162e303415ee1dd7d2c1a448068935663ba5ac5b5f72a515fe3f6f6" + "@@zml+//runtimes/neuron/requirements.lock.txt": "fa0b5b967162e303415ee1dd7d2c1a448068935663ba5ac5b5f72a515fe3f6f6", + "@@zml+//tools/hf_requirements.lock.txt": "ffbcfcfd01c9a31db823affa4ab6122c509b2f1eeb4b853325d1dbd5cc9568da" }, "recordedDirentsInputs": {}, "envVariables": { @@ -1273,6 +1274,318 @@ "RULES_PYTHON_REPO_DEBUG_VERBOSITY": null }, "generatedRepoSpecs": { + "huggingface_hub_311_certifi": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "certifi==2025.7.14 --hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 --hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995" + } + }, + "huggingface_hub_311_charset_normalizer": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "charset-normalizer==3.4.2 --hash=sha256:005fa3432484527f9732ebd315da8da8001593e2cf46a3d817669f062c3d9ed4 --hash=sha256:046595208aae0120559a67693ecc65dd75d46f7bf687f159127046628178dc45 --hash=sha256:0c29de6a1a95f24b9a1aa7aefd27d2487263f00dfd55a77719b530788f75cff7 --hash=sha256:0c8c57f84ccfc871a48a47321cfa49ae1df56cd1d965a09abe84066f6853b9c0 --hash=sha256:0f5d9ed7f254402c9e7d35d2f5972c9bbea9040e99cd2861bd77dc68263277c7 --hash=sha256:18dd2e350387c87dabe711b86f83c9c78af772c748904d372ade190b5c7c9d4d --hash=sha256:1b1bde144d98e446b056ef98e59c256e9294f6b74d7af6846bf5ffdafd687a7d --hash=sha256:1c95a1e2902a8b722868587c0e1184ad5c55631de5afc0eb96bc4b0d738092c0 --hash=sha256:1cad5f45b3146325bb38d6855642f6fd609c3f7cad4dbaf75549bf3b904d3184 --hash=sha256:21b2899062867b0e1fde9b724f8aecb1af14f2778d69aacd1a5a1853a597a5db --hash=sha256:24498ba8ed6c2e0b56d4acbf83f2d989720a93b41d712ebd4f4979660db4417b --hash=sha256:25a23ea5c7edc53e0f29bae2c44fcb5a1aa10591aae107f2a2b2583a9c5cbc64 --hash=sha256:289200a18fa698949d2b39c671c2cc7a24d44096784e76614899a7ccf2574b7b --hash=sha256:28a1005facc94196e1fb3e82a3d442a9d9110b8434fc1ded7a24a2983c9888d8 --hash=sha256:32fc0341d72e0f73f80acb0a2c94216bd704f4f0bce10aedea38f30502b271ff --hash=sha256:36b31da18b8890a76ec181c3cf44326bf2c48e36d393ca1b72b3f484113ea344 --hash=sha256:3c21d4fca343c805a52c0c78edc01e3477f6dd1ad7c47653241cf2a206d4fc58 --hash=sha256:3fddb7e2c84ac87ac3a947cb4e66d143ca5863ef48e4a5ecb83bd48619e4634e --hash=sha256:43e0933a0eff183ee85833f341ec567c0980dae57c464d8a508e1b2ceb336471 --hash=sha256:4a476b06fbcf359ad25d34a057b7219281286ae2477cc5ff5e3f70a246971148 --hash=sha256:4e594135de17ab3866138f496755f302b72157d115086d100c3f19370839dd3a --hash=sha256:50bf98d5e563b83cc29471fa114366e6806bc06bc7a25fd59641e41445327836 --hash=sha256:5a9979887252a82fefd3d3ed2a8e3b937a7a809f65dcb1e068b090e165bbe99e --hash=sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63 --hash=sha256:5bf4545e3b962767e5c06fe1738f951f77d27967cb2caa64c28be7c4563e162c --hash=sha256:6333b3aa5a12c26b2a4d4e7335a28f1475e0e5e17d69d55141ee3cab736f66d1 --hash=sha256:65c981bdbd3f57670af8b59777cbfae75364b483fa8a9f420f08094531d54a01 --hash=sha256:68a328e5f55ec37c57f19ebb1fdc56a248db2e3e9ad769919a58672958e8f366 --hash=sha256:6a0289e4589e8bdfef02a80478f1dfcb14f0ab696b5a00e1f4b8a14a307a3c58 --hash=sha256:6b66f92b17849b85cad91259efc341dce9c1af48e2173bf38a85c6329f1033e5 --hash=sha256:6c9379d65defcab82d07b2a9dfbfc2e95bc8fe0ebb1b176a3190230a3ef0e07c --hash=sha256:6fc1f5b51fa4cecaa18f2bd7a003f3dd039dd615cd69a2afd6d3b19aed6775f2 --hash=sha256:70f7172939fdf8790425ba31915bfbe8335030f05b9913d7ae00a87d4395620a --hash=sha256:721c76e84fe669be19c5791da68232ca2e05ba5185575086e384352e2c309597 --hash=sha256:7222ffd5e4de8e57e03ce2cef95a4c43c98fcb72ad86909abdfc2c17d227fc1b --hash=sha256:75d10d37a47afee94919c4fab4c22b9bc2a8bf7d4f46f87363bcf0573f3ff4f5 --hash=sha256:76af085e67e56c8816c3ccf256ebd136def2ed9654525348cfa744b6802b69eb --hash=sha256:770cab594ecf99ae64c236bc9ee3439c3f46be49796e265ce0cc8bc17b10294f --hash=sha256:7a6ab32f7210554a96cd9e33abe3ddd86732beeafc7a28e9955cdf22ffadbab0 --hash=sha256:7c48ed483eb946e6c04ccbe02c6b4d1d48e51944b6db70f697e089c193404941 --hash=sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0 --hash=sha256:8075c35cd58273fee266c58c0c9b670947c19df5fb98e7b66710e04ad4e9ff86 --hash=sha256:8272b73e1c5603666618805fe821edba66892e2870058c94c53147602eab29c7 --hash=sha256:82d8fd25b7f4675d0c47cf95b594d4e7b158aca33b76aa63d07186e13c0e0ab7 --hash=sha256:844da2b5728b5ce0e32d863af26f32b5ce61bc4273a9c720a9f3aa9df73b1455 --hash=sha256:8755483f3c00d6c9a77f490c17e6ab0c8729e39e6390328e42521ef175380ae6 --hash=sha256:915f3849a011c1f593ab99092f3cecfcb4d65d8feb4a64cf1bf2d22074dc0ec4 --hash=sha256:926ca93accd5d36ccdabd803392ddc3e03e6d4cd1cf17deff3b989ab8e9dbcf0 --hash=sha256:982bb1e8b4ffda883b3d0a521e23abcd6fd17418f6d2c4118d257a10199c0ce3 --hash=sha256:98f862da73774290f251b9df8d11161b6cf25b599a66baf087c1ffe340e9bfd1 --hash=sha256:9cbfacf36cb0ec2897ce0ebc5d08ca44213af24265bd56eca54bee7923c48fd6 --hash=sha256:a370b3e078e418187da8c3674eddb9d983ec09445c99a3a263c2011993522981 --hash=sha256:a955b438e62efdf7e0b7b52a64dc5c3396e2634baa62471768a64bc2adb73d5c --hash=sha256:aa6af9e7d59f9c12b33ae4e9450619cf2488e2bbe9b44030905877f0b2324980 --hash=sha256:aa88ca0b1932e93f2d961bf3addbb2db902198dca337d88c89e1559e066e7645 --hash=sha256:aaeeb6a479c7667fbe1099af9617c83aaca22182d6cf8c53966491a0f1b7ffb7 --hash=sha256:aaf27faa992bfee0264dc1f03f4c75e9fcdda66a519db6b957a3f826e285cf12 --hash=sha256:b2680962a4848b3c4f155dc2ee64505a9c57186d0d56b43123b17ca3de18f0fa --hash=sha256:b2d318c11350e10662026ad0eb71bb51c7812fc8590825304ae0bdd4ac283acd --hash=sha256:b33de11b92e9f75a2b545d6e9b6f37e398d86c3e9e9653c4864eb7e89c5773ef --hash=sha256:b3daeac64d5b371dea99714f08ffc2c208522ec6b06fbc7866a450dd446f5c0f --hash=sha256:be1e352acbe3c78727a16a455126d9ff83ea2dfdcbc83148d2982305a04714c2 --hash=sha256:bee093bf902e1d8fc0ac143c88902c3dfc8941f7ea1d6a8dd2bcb786d33db03d --hash=sha256:c72fbbe68c6f32f251bdc08b8611c7b3060612236e960ef848e0a517ddbe76c5 --hash=sha256:c9e36a97bee9b86ef9a1cf7bb96747eb7a15c2f22bdb5b516434b00f2a599f02 --hash=sha256:cddf7bd982eaa998934a91f69d182aec997c6c468898efe6679af88283b498d3 --hash=sha256:cf713fe9a71ef6fd5adf7a79670135081cd4431c2943864757f0fa3a65b1fafd --hash=sha256:d11b54acf878eef558599658b0ffca78138c8c3655cf4f3a4a673c437e67732e --hash=sha256:d41c4d287cfc69060fa91cae9683eacffad989f1a10811995fa309df656ec214 --hash=sha256:d524ba3f1581b35c03cb42beebab4a13e6cdad7b36246bd22541fa585a56cccd --hash=sha256:daac4765328a919a805fa5e2720f3e94767abd632ae410a9062dff5412bae65a --hash=sha256:db4c7bf0e07fc3b7d89ac2a5880a6a8062056801b83ff56d8464b70f65482b6c --hash=sha256:dc7039885fa1baf9be153a0626e337aa7ec8bf96b0128605fb0d77788ddc1681 --hash=sha256:dccab8d5fa1ef9bfba0590ecf4d46df048d18ffe3eec01eeb73a42e0d9e7a8ba --hash=sha256:dedb8adb91d11846ee08bec4c8236c8549ac721c245678282dcb06b221aab59f --hash=sha256:e45ba65510e2647721e35323d6ef54c7974959f6081b58d4ef5d87c60c84919a --hash=sha256:e53efc7c7cee4c1e70661e2e112ca46a575f90ed9ae3fef200f2a25e954f4b28 --hash=sha256:e635b87f01ebc977342e2697d05b56632f5f879a4f15955dfe8cef2448b51691 --hash=sha256:e70e990b2137b29dc5564715de1e12701815dacc1d056308e2b17e9095372a82 --hash=sha256:e8082b26888e2f8b36a042a58307d5b917ef2b1cacab921ad3323ef91901c71a --hash=sha256:e8323a9b031aa0393768b87f04b4164a40037fb2a3c11ac06a03ffecd3618027 --hash=sha256:e92fca20c46e9f5e1bb485887d074918b13543b1c2a1185e69bb8d17ab6236a7 --hash=sha256:eb30abc20df9ab0814b5a2524f23d75dcf83cde762c161917a2b4b7b55b1e518 --hash=sha256:eba9904b0f38a143592d9fc0e19e2df0fa2e41c3c3745554761c5f6447eedabf --hash=sha256:ef8de666d6179b009dce7bcb2ad4c4a779f113f12caf8dc77f0162c29d20490b --hash=sha256:efd387a49825780ff861998cd959767800d54f8308936b21025326de4b5a42b9 --hash=sha256:f0aa37f3c979cf2546b73e8222bbfa3dc07a641585340179d768068e3455e544 --hash=sha256:f4074c5a429281bf056ddd4c5d3b740ebca4d43ffffe2ef4bf4d2d05114299da --hash=sha256:f69a27e45c43520f5487f27627059b64aaf160415589230992cec34c5e18a509 --hash=sha256:fb707f3e15060adf5b7ada797624a6c6e0138e2a26baa089df64c68ee98e040f --hash=sha256:fcbe676a55d7445b22c10967bceaaf0ee69407fbe0ece4d032b6eb8d4565982a --hash=sha256:fdb20a30fe1175ecabed17cbf7812f7b804b8a315a25f24678bcdf120a90077f" + } + }, + "huggingface_hub_311_filelock": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "filelock==3.18.0 --hash=sha256:adbc88eabb99d2fec8c9c1b229b171f18afa655400173ddc653d5d01501fb9f2 --hash=sha256:c401f4f8377c4464e6db25fff06205fd89bdd83b65eb0488ed1b160f780e21de" + } + }, + "huggingface_hub_311_fsspec": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "fsspec==2025.7.0 --hash=sha256:786120687ffa54b8283d942929540d8bc5ccfa820deb555a2b5d0ed2b737bf58 --hash=sha256:8b012e39f63c7d5f10474de957f3ab793b47b45ae7d39f2fb735f8bbe25c0e21" + } + }, + "huggingface_hub_311_hf_xet": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "hf-xet==1.1.5 --hash=sha256:69ebbcfd9ec44fdc2af73441619eeb06b94ee34511bbcf57cd423820090f5694 --hash=sha256:73e167d9807d166596b4b2f0b585c6d5bd84a26dea32843665a8b58f6edba245 --hash=sha256:83088ecea236d5113de478acb2339f92c95b4fb0462acaa30621fac02f5a534a --hash=sha256:9fa6e3ee5d61912c4a113e0708eaaef987047616465ac7aa30f7121a48fc1af8 --hash=sha256:ab34c4c3104133c495785d5d8bba3b1efc99de52c02e759cf711a91fd39d3a14 --hash=sha256:dbba1660e5d810bd0ea77c511a99e9242d920790d0e63c0e4673ed36c4022d18 --hash=sha256:f52c2fa3635b8c37c7764d8796dfa72706cc4eded19d638331161e82b0792e23 --hash=sha256:fc874b5c843e642f45fd85cda1ce599e123308ad2901ead23d3510a47ff506d1" + } + }, + "huggingface_hub_311_huggingface_hub": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "huggingface-hub==0.34.3 --hash=sha256:5444550099e2d86e68b2898b09e85878fbd788fc2957b506c6a79ce060e39492 --hash=sha256:d58130fd5aa7408480681475491c0abd7e835442082fbc3ef4d45b6c39f83853" + } + }, + "huggingface_hub_311_idna": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "idna==3.10 --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3" + } + }, + "huggingface_hub_311_packaging": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "packaging==25.0 --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f" + } + }, + "huggingface_hub_311_pyyaml": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "pyyaml==6.0.2 --hash=sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff --hash=sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48 --hash=sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086 --hash=sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e --hash=sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133 --hash=sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5 --hash=sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484 --hash=sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee --hash=sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5 --hash=sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68 --hash=sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a --hash=sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf --hash=sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99 --hash=sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8 --hash=sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85 --hash=sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19 --hash=sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc --hash=sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a --hash=sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1 --hash=sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317 --hash=sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c --hash=sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631 --hash=sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d --hash=sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652 --hash=sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5 --hash=sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e --hash=sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b --hash=sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8 --hash=sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476 --hash=sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706 --hash=sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563 --hash=sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237 --hash=sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b --hash=sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083 --hash=sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180 --hash=sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425 --hash=sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e --hash=sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f --hash=sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725 --hash=sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183 --hash=sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab --hash=sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774 --hash=sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725 --hash=sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e --hash=sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5 --hash=sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d --hash=sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290 --hash=sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44 --hash=sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed --hash=sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4 --hash=sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba --hash=sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12 --hash=sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4" + } + }, + "huggingface_hub_311_requests": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "requests==2.32.4 --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422" + } + }, + "huggingface_hub_311_tqdm": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "tqdm==4.67.1 --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2" + } + }, + "huggingface_hub_311_typing_extensions": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "typing-extensions==4.14.1 --hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 --hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76" + } + }, + "huggingface_hub_311_urllib3": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@huggingface_hub//{name}:{target}", + "download_only": true, + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple" + ], + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "huggingface_hub_311", + "requirement": "urllib3==2.5.0 --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc" + } + }, "neuron_py_deps_311_boto3": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { @@ -4370,6 +4683,44 @@ ] } }, + "huggingface_hub": { + "repoRuleId": "@@rules_python+//python/private/pypi:hub_repository.bzl%hub_repository", + "attributes": { + "repo_name": "huggingface_hub", + "extra_hub_aliases": {}, + "whl_map": { + "certifi": "{\"huggingface_hub_311_certifi\":[{\"version\":\"3.11\"}]}", + "charset_normalizer": "{\"huggingface_hub_311_charset_normalizer\":[{\"version\":\"3.11\"}]}", + "filelock": "{\"huggingface_hub_311_filelock\":[{\"version\":\"3.11\"}]}", + "fsspec": "{\"huggingface_hub_311_fsspec\":[{\"version\":\"3.11\"}]}", + "hf_xet": "{\"huggingface_hub_311_hf_xet\":[{\"version\":\"3.11\"}]}", + "huggingface_hub": "{\"huggingface_hub_311_huggingface_hub\":[{\"version\":\"3.11\"}]}", + "idna": "{\"huggingface_hub_311_idna\":[{\"version\":\"3.11\"}]}", + "packaging": "{\"huggingface_hub_311_packaging\":[{\"version\":\"3.11\"}]}", + "pyyaml": "{\"huggingface_hub_311_pyyaml\":[{\"version\":\"3.11\"}]}", + "requests": "{\"huggingface_hub_311_requests\":[{\"version\":\"3.11\"}]}", + "tqdm": "{\"huggingface_hub_311_tqdm\":[{\"version\":\"3.11\"}]}", + "typing_extensions": "{\"huggingface_hub_311_typing_extensions\":[{\"version\":\"3.11\"}]}", + "urllib3": "{\"huggingface_hub_311_urllib3\":[{\"version\":\"3.11\"}]}" + }, + "packages": [ + "certifi", + "charset_normalizer", + "filelock", + "fsspec", + "hf_xet", + "huggingface_hub", + "idna", + "packaging", + "pyyaml", + "requests", + "tqdm", + "typing_extensions", + "urllib3" + ], + "groups": {} + } + }, "neuron_py_deps": { "repoRuleId": "@@rules_python+//python/private/pypi:hub_repository.bzl%hub_repository", "attributes": { diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 5dc3d219..ef43fe7b 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -1,7 +1,34 @@ load("@rules_python//python:py_library.bzl", "py_library") +load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") + +load("@rules_uv//uv:pip.bzl", "pip_compile") py_library( name = "zml_utils", srcs = ["zml_utils.py"], visibility = ["//visibility:public"], ) + +py_console_script_binary( + name = "hf", + script = "hf", + pkg = "@huggingface_hub//huggingface_hub:pkg", + visibility = ["//visibility:public"], +) + +pip_compile( + name = "hf_update_requirements", + args = [ + "--generate-hashes", + "--emit-index-url", + "--emit-find-links", + "--no-strip-extras", + "--index-strategy=unsafe-best-match", + "--upgrade", + ], + requirements_in = "hf_requirements.in", + requirements_txt = "hf_requirements.lock.txt", + tags = [ + "manual", + ], +) diff --git a/tools/hf_requirements.in b/tools/hf_requirements.in new file mode 100644 index 00000000..6b964ccc --- /dev/null +++ b/tools/hf_requirements.in @@ -0,0 +1 @@ +huggingface_hub diff --git a/tools/hf_requirements.lock.txt b/tools/hf_requirements.lock.txt new file mode 100644 index 00000000..558b47be --- /dev/null +++ b/tools/hf_requirements.lock.txt @@ -0,0 +1,203 @@ +# This file was autogenerated by uv via the following command: +# bazel run @@//tools:hf_update_requirements +--index-url https://pypi.org/simple + +certifi==2025.7.14 \ + --hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 \ + --hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995 + # via requests +charset-normalizer==3.4.2 \ + --hash=sha256:005fa3432484527f9732ebd315da8da8001593e2cf46a3d817669f062c3d9ed4 \ + --hash=sha256:046595208aae0120559a67693ecc65dd75d46f7bf687f159127046628178dc45 \ + --hash=sha256:0c29de6a1a95f24b9a1aa7aefd27d2487263f00dfd55a77719b530788f75cff7 \ + --hash=sha256:0c8c57f84ccfc871a48a47321cfa49ae1df56cd1d965a09abe84066f6853b9c0 \ + --hash=sha256:0f5d9ed7f254402c9e7d35d2f5972c9bbea9040e99cd2861bd77dc68263277c7 \ + --hash=sha256:18dd2e350387c87dabe711b86f83c9c78af772c748904d372ade190b5c7c9d4d \ + --hash=sha256:1b1bde144d98e446b056ef98e59c256e9294f6b74d7af6846bf5ffdafd687a7d \ + --hash=sha256:1c95a1e2902a8b722868587c0e1184ad5c55631de5afc0eb96bc4b0d738092c0 \ + --hash=sha256:1cad5f45b3146325bb38d6855642f6fd609c3f7cad4dbaf75549bf3b904d3184 \ + --hash=sha256:21b2899062867b0e1fde9b724f8aecb1af14f2778d69aacd1a5a1853a597a5db \ + --hash=sha256:24498ba8ed6c2e0b56d4acbf83f2d989720a93b41d712ebd4f4979660db4417b \ + --hash=sha256:25a23ea5c7edc53e0f29bae2c44fcb5a1aa10591aae107f2a2b2583a9c5cbc64 \ + --hash=sha256:289200a18fa698949d2b39c671c2cc7a24d44096784e76614899a7ccf2574b7b \ + --hash=sha256:28a1005facc94196e1fb3e82a3d442a9d9110b8434fc1ded7a24a2983c9888d8 \ + --hash=sha256:32fc0341d72e0f73f80acb0a2c94216bd704f4f0bce10aedea38f30502b271ff \ + --hash=sha256:36b31da18b8890a76ec181c3cf44326bf2c48e36d393ca1b72b3f484113ea344 \ + --hash=sha256:3c21d4fca343c805a52c0c78edc01e3477f6dd1ad7c47653241cf2a206d4fc58 \ + --hash=sha256:3fddb7e2c84ac87ac3a947cb4e66d143ca5863ef48e4a5ecb83bd48619e4634e \ + --hash=sha256:43e0933a0eff183ee85833f341ec567c0980dae57c464d8a508e1b2ceb336471 \ + --hash=sha256:4a476b06fbcf359ad25d34a057b7219281286ae2477cc5ff5e3f70a246971148 \ + --hash=sha256:4e594135de17ab3866138f496755f302b72157d115086d100c3f19370839dd3a \ + --hash=sha256:50bf98d5e563b83cc29471fa114366e6806bc06bc7a25fd59641e41445327836 \ + --hash=sha256:5a9979887252a82fefd3d3ed2a8e3b937a7a809f65dcb1e068b090e165bbe99e \ + --hash=sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63 \ + --hash=sha256:5bf4545e3b962767e5c06fe1738f951f77d27967cb2caa64c28be7c4563e162c \ + --hash=sha256:6333b3aa5a12c26b2a4d4e7335a28f1475e0e5e17d69d55141ee3cab736f66d1 \ + --hash=sha256:65c981bdbd3f57670af8b59777cbfae75364b483fa8a9f420f08094531d54a01 \ + --hash=sha256:68a328e5f55ec37c57f19ebb1fdc56a248db2e3e9ad769919a58672958e8f366 \ + --hash=sha256:6a0289e4589e8bdfef02a80478f1dfcb14f0ab696b5a00e1f4b8a14a307a3c58 \ + --hash=sha256:6b66f92b17849b85cad91259efc341dce9c1af48e2173bf38a85c6329f1033e5 \ + --hash=sha256:6c9379d65defcab82d07b2a9dfbfc2e95bc8fe0ebb1b176a3190230a3ef0e07c \ + --hash=sha256:6fc1f5b51fa4cecaa18f2bd7a003f3dd039dd615cd69a2afd6d3b19aed6775f2 \ + --hash=sha256:70f7172939fdf8790425ba31915bfbe8335030f05b9913d7ae00a87d4395620a \ + --hash=sha256:721c76e84fe669be19c5791da68232ca2e05ba5185575086e384352e2c309597 \ + --hash=sha256:7222ffd5e4de8e57e03ce2cef95a4c43c98fcb72ad86909abdfc2c17d227fc1b \ + --hash=sha256:75d10d37a47afee94919c4fab4c22b9bc2a8bf7d4f46f87363bcf0573f3ff4f5 \ + --hash=sha256:76af085e67e56c8816c3ccf256ebd136def2ed9654525348cfa744b6802b69eb \ + --hash=sha256:770cab594ecf99ae64c236bc9ee3439c3f46be49796e265ce0cc8bc17b10294f \ + --hash=sha256:7a6ab32f7210554a96cd9e33abe3ddd86732beeafc7a28e9955cdf22ffadbab0 \ + --hash=sha256:7c48ed483eb946e6c04ccbe02c6b4d1d48e51944b6db70f697e089c193404941 \ + --hash=sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0 \ + --hash=sha256:8075c35cd58273fee266c58c0c9b670947c19df5fb98e7b66710e04ad4e9ff86 \ + --hash=sha256:8272b73e1c5603666618805fe821edba66892e2870058c94c53147602eab29c7 \ + --hash=sha256:82d8fd25b7f4675d0c47cf95b594d4e7b158aca33b76aa63d07186e13c0e0ab7 \ + --hash=sha256:844da2b5728b5ce0e32d863af26f32b5ce61bc4273a9c720a9f3aa9df73b1455 \ + --hash=sha256:8755483f3c00d6c9a77f490c17e6ab0c8729e39e6390328e42521ef175380ae6 \ + --hash=sha256:915f3849a011c1f593ab99092f3cecfcb4d65d8feb4a64cf1bf2d22074dc0ec4 \ + --hash=sha256:926ca93accd5d36ccdabd803392ddc3e03e6d4cd1cf17deff3b989ab8e9dbcf0 \ + --hash=sha256:982bb1e8b4ffda883b3d0a521e23abcd6fd17418f6d2c4118d257a10199c0ce3 \ + --hash=sha256:98f862da73774290f251b9df8d11161b6cf25b599a66baf087c1ffe340e9bfd1 \ + --hash=sha256:9cbfacf36cb0ec2897ce0ebc5d08ca44213af24265bd56eca54bee7923c48fd6 \ + --hash=sha256:a370b3e078e418187da8c3674eddb9d983ec09445c99a3a263c2011993522981 \ + --hash=sha256:a955b438e62efdf7e0b7b52a64dc5c3396e2634baa62471768a64bc2adb73d5c \ + --hash=sha256:aa6af9e7d59f9c12b33ae4e9450619cf2488e2bbe9b44030905877f0b2324980 \ + --hash=sha256:aa88ca0b1932e93f2d961bf3addbb2db902198dca337d88c89e1559e066e7645 \ + --hash=sha256:aaeeb6a479c7667fbe1099af9617c83aaca22182d6cf8c53966491a0f1b7ffb7 \ + --hash=sha256:aaf27faa992bfee0264dc1f03f4c75e9fcdda66a519db6b957a3f826e285cf12 \ + --hash=sha256:b2680962a4848b3c4f155dc2ee64505a9c57186d0d56b43123b17ca3de18f0fa \ + --hash=sha256:b2d318c11350e10662026ad0eb71bb51c7812fc8590825304ae0bdd4ac283acd \ + --hash=sha256:b33de11b92e9f75a2b545d6e9b6f37e398d86c3e9e9653c4864eb7e89c5773ef \ + --hash=sha256:b3daeac64d5b371dea99714f08ffc2c208522ec6b06fbc7866a450dd446f5c0f \ + --hash=sha256:be1e352acbe3c78727a16a455126d9ff83ea2dfdcbc83148d2982305a04714c2 \ + --hash=sha256:bee093bf902e1d8fc0ac143c88902c3dfc8941f7ea1d6a8dd2bcb786d33db03d \ + --hash=sha256:c72fbbe68c6f32f251bdc08b8611c7b3060612236e960ef848e0a517ddbe76c5 \ + --hash=sha256:c9e36a97bee9b86ef9a1cf7bb96747eb7a15c2f22bdb5b516434b00f2a599f02 \ + --hash=sha256:cddf7bd982eaa998934a91f69d182aec997c6c468898efe6679af88283b498d3 \ + --hash=sha256:cf713fe9a71ef6fd5adf7a79670135081cd4431c2943864757f0fa3a65b1fafd \ + --hash=sha256:d11b54acf878eef558599658b0ffca78138c8c3655cf4f3a4a673c437e67732e \ + --hash=sha256:d41c4d287cfc69060fa91cae9683eacffad989f1a10811995fa309df656ec214 \ + --hash=sha256:d524ba3f1581b35c03cb42beebab4a13e6cdad7b36246bd22541fa585a56cccd \ + --hash=sha256:daac4765328a919a805fa5e2720f3e94767abd632ae410a9062dff5412bae65a \ + --hash=sha256:db4c7bf0e07fc3b7d89ac2a5880a6a8062056801b83ff56d8464b70f65482b6c \ + --hash=sha256:dc7039885fa1baf9be153a0626e337aa7ec8bf96b0128605fb0d77788ddc1681 \ + --hash=sha256:dccab8d5fa1ef9bfba0590ecf4d46df048d18ffe3eec01eeb73a42e0d9e7a8ba \ + --hash=sha256:dedb8adb91d11846ee08bec4c8236c8549ac721c245678282dcb06b221aab59f \ + --hash=sha256:e45ba65510e2647721e35323d6ef54c7974959f6081b58d4ef5d87c60c84919a \ + --hash=sha256:e53efc7c7cee4c1e70661e2e112ca46a575f90ed9ae3fef200f2a25e954f4b28 \ + --hash=sha256:e635b87f01ebc977342e2697d05b56632f5f879a4f15955dfe8cef2448b51691 \ + --hash=sha256:e70e990b2137b29dc5564715de1e12701815dacc1d056308e2b17e9095372a82 \ + --hash=sha256:e8082b26888e2f8b36a042a58307d5b917ef2b1cacab921ad3323ef91901c71a \ + --hash=sha256:e8323a9b031aa0393768b87f04b4164a40037fb2a3c11ac06a03ffecd3618027 \ + --hash=sha256:e92fca20c46e9f5e1bb485887d074918b13543b1c2a1185e69bb8d17ab6236a7 \ + --hash=sha256:eb30abc20df9ab0814b5a2524f23d75dcf83cde762c161917a2b4b7b55b1e518 \ + --hash=sha256:eba9904b0f38a143592d9fc0e19e2df0fa2e41c3c3745554761c5f6447eedabf \ + --hash=sha256:ef8de666d6179b009dce7bcb2ad4c4a779f113f12caf8dc77f0162c29d20490b \ + --hash=sha256:efd387a49825780ff861998cd959767800d54f8308936b21025326de4b5a42b9 \ + --hash=sha256:f0aa37f3c979cf2546b73e8222bbfa3dc07a641585340179d768068e3455e544 \ + --hash=sha256:f4074c5a429281bf056ddd4c5d3b740ebca4d43ffffe2ef4bf4d2d05114299da \ + --hash=sha256:f69a27e45c43520f5487f27627059b64aaf160415589230992cec34c5e18a509 \ + --hash=sha256:fb707f3e15060adf5b7ada797624a6c6e0138e2a26baa089df64c68ee98e040f \ + --hash=sha256:fcbe676a55d7445b22c10967bceaaf0ee69407fbe0ece4d032b6eb8d4565982a \ + --hash=sha256:fdb20a30fe1175ecabed17cbf7812f7b804b8a315a25f24678bcdf120a90077f + # via requests +filelock==3.18.0 \ + --hash=sha256:adbc88eabb99d2fec8c9c1b229b171f18afa655400173ddc653d5d01501fb9f2 \ + --hash=sha256:c401f4f8377c4464e6db25fff06205fd89bdd83b65eb0488ed1b160f780e21de + # via huggingface-hub +fsspec==2025.7.0 \ + --hash=sha256:786120687ffa54b8283d942929540d8bc5ccfa820deb555a2b5d0ed2b737bf58 \ + --hash=sha256:8b012e39f63c7d5f10474de957f3ab793b47b45ae7d39f2fb735f8bbe25c0e21 + # via huggingface-hub +hf-xet==1.1.5 \ + --hash=sha256:69ebbcfd9ec44fdc2af73441619eeb06b94ee34511bbcf57cd423820090f5694 \ + --hash=sha256:73e167d9807d166596b4b2f0b585c6d5bd84a26dea32843665a8b58f6edba245 \ + --hash=sha256:83088ecea236d5113de478acb2339f92c95b4fb0462acaa30621fac02f5a534a \ + --hash=sha256:9fa6e3ee5d61912c4a113e0708eaaef987047616465ac7aa30f7121a48fc1af8 \ + --hash=sha256:ab34c4c3104133c495785d5d8bba3b1efc99de52c02e759cf711a91fd39d3a14 \ + --hash=sha256:dbba1660e5d810bd0ea77c511a99e9242d920790d0e63c0e4673ed36c4022d18 \ + --hash=sha256:f52c2fa3635b8c37c7764d8796dfa72706cc4eded19d638331161e82b0792e23 \ + --hash=sha256:fc874b5c843e642f45fd85cda1ce599e123308ad2901ead23d3510a47ff506d1 + # via huggingface-hub +huggingface-hub==0.34.3 \ + --hash=sha256:5444550099e2d86e68b2898b09e85878fbd788fc2957b506c6a79ce060e39492 \ + --hash=sha256:d58130fd5aa7408480681475491c0abd7e835442082fbc3ef4d45b6c39f83853 + # via -r tools/hf_requirements.in +idna==3.10 \ + --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \ + --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 + # via requests +packaging==25.0 \ + --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 \ + --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f + # via huggingface-hub +pyyaml==6.0.2 \ + --hash=sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff \ + --hash=sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48 \ + --hash=sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086 \ + --hash=sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e \ + --hash=sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133 \ + --hash=sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5 \ + --hash=sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484 \ + --hash=sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee \ + --hash=sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5 \ + --hash=sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68 \ + --hash=sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a \ + --hash=sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf \ + --hash=sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99 \ + --hash=sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8 \ + --hash=sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85 \ + --hash=sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19 \ + --hash=sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc \ + --hash=sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a \ + --hash=sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1 \ + --hash=sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317 \ + --hash=sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c \ + --hash=sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631 \ + --hash=sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d \ + --hash=sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652 \ + --hash=sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5 \ + --hash=sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e \ + --hash=sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b \ + --hash=sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8 \ + --hash=sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476 \ + --hash=sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706 \ + --hash=sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563 \ + --hash=sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237 \ + --hash=sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b \ + --hash=sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083 \ + --hash=sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180 \ + --hash=sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425 \ + --hash=sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e \ + --hash=sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f \ + --hash=sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725 \ + --hash=sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183 \ + --hash=sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab \ + --hash=sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774 \ + --hash=sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725 \ + --hash=sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e \ + --hash=sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5 \ + --hash=sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d \ + --hash=sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290 \ + --hash=sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44 \ + --hash=sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed \ + --hash=sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4 \ + --hash=sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba \ + --hash=sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12 \ + --hash=sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4 + # via huggingface-hub +requests==2.32.4 \ + --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c \ + --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422 + # via huggingface-hub +tqdm==4.67.1 \ + --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 \ + --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2 + # via huggingface-hub +typing-extensions==4.14.1 \ + --hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 \ + --hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76 + # via huggingface-hub +urllib3==2.5.0 \ + --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 \ + --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc + # via requests From 8279e4303802354f0aee811f7ab7da1e0415bcae Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 19 Aug 2025 16:04:43 +0200 Subject: [PATCH 011/881] pjrt,zml: remove profiler (#306) Closes #303 Progress towards #301 --- pjrt/BUILD.bazel | 46 +--- pjrt/convert/trace_container.zig | 366 ------------------------------- pjrt/convert/xplane_schema.zig | 297 ------------------------- pjrt/pjrt.zig | 14 -- pjrt/profiler.zig | 194 ---------------- pjrt/xspace_to_json.zig | 47 ---- zml/pjrtx.zig | 8 - zml/platform.zig | 7 - 8 files changed, 1 insertion(+), 978 deletions(-) delete mode 100644 pjrt/convert/trace_container.zig delete mode 100644 pjrt/convert/xplane_schema.zig delete mode 100644 pjrt/profiler.zig delete mode 100644 pjrt/xspace_to_json.zig diff --git a/pjrt/BUILD.bazel b/pjrt/BUILD.bazel index 682b569c..470a4157 100644 --- a/pjrt/BUILD.bazel +++ b/pjrt/BUILD.bazel @@ -1,28 +1,17 @@ load("@rules_zig//zig:defs.bzl", "zig_library") -load("@zml//bazel:zig.bzl", "zig_cc_binary") load("@zml//bazel:zig_srcs.bzl", "zig_srcs") -load("@zml//bazel:zig_proto_library.bzl", "zig_proto_library") zig_library( name = "pjrt", - srcs = [ - "convert/trace_container.zig", - "convert/xplane_schema.zig", - "ffi.zig", - "profiler.zig", - ], + srcs = ["ffi.zig"], main = "pjrt.zig", visibility = ["//visibility:public"], deps = [ - ":profiler_options_proto", - ":trace_events_proto", - ":xplane_proto", "//stdx", "@xla//xla/ffi/api:c_api", "@xla//xla/pjrt/c:pjrt_c_api_ffi_extension_hdrs", "@xla//xla/pjrt/c:pjrt_c_api_gpu_extension_hdrs", "@xla//xla/pjrt/c:pjrt_c_api_hdrs", - "@xla//xla/pjrt/c:pjrt_c_api_profiler_extension_hdrs", "@xla//xla/pjrt/c:pjrt_c_api_triton_extension_hdrs", ], ) @@ -31,36 +20,3 @@ zig_srcs( name = "sources", zig_lib = ":pjrt", ) - -zig_proto_library( - name = "profiler_options_proto", - import_name = "//tsl:profiler_options_proto", - deps = ["@xla//third_party/tsl/tsl/profiler/protobuf:profiler_options_proto"], -) - -zig_proto_library( - name = "xplane_proto", - import_name = "//tsl:xplane_proto", - deps = ["@xla//third_party/tsl/tsl/profiler/protobuf:xplane_proto"], -) - -zig_proto_library( - name = "trace_events_proto", - import_name = "//tsl:trace_events_proto", - deps = ["@xla//third_party/tsl/tsl/profiler/protobuf:trace_events_proto"], -) - -zig_cc_binary( - name = "xspace_to_json", - srcs = [ - "convert/trace_container.zig", - "convert/xplane_schema.zig", - ], - main = "xspace_to_json.zig", - visibility = ["//visibility:public"], - deps = [ - ":trace_events_proto", - ":xplane_proto", - "//stdx", - ], -) diff --git a/pjrt/convert/trace_container.zig b/pjrt/convert/trace_container.zig deleted file mode 100644 index 6346547b..00000000 --- a/pjrt/convert/trace_container.zig +++ /dev/null @@ -1,366 +0,0 @@ -const std = @import("std"); - -const trace_events_proto = @import("//tsl:trace_events_proto"); -const xplane_proto = @import("//tsl:xplane_proto"); - -const xplane_schema = @import("xplane_schema.zig"); - -// Constants used as trace_viewer PID (device_id in trace_events.proto). -// PID 0 is unused. -// Support up to 500 accelerator devices. -const first_device_id = 1; -const last_device_id = 500; -// Support Upto 200 custom planes as fake devices (i.e., planes with a -// "/custom:" prefix). See `::custom_plane_prefix` for more -// information -const first_custom_plane_device_id = last_device_id + 1; -const max_custom_plane_devices_per_host = 200; -const last_custom_plane_device_id = first_custom_plane_device_id + max_custom_plane_devices_per_host - 1; - -// Host threads are shown as a single fake device. -pub const host_threads_device_id = last_custom_plane_device_id + 1; - -pub const xla_async_op_line_name = "Async XLA Ops"; - -pub const host_threads_plane_name = "/host:CPU"; -pub const gpu_plane_prefix = "/device:GPU:"; -pub const tpu_plane_prefix = "/device:TPU:"; -pub const custom_plane_prefix = "/device:CUSTOM:"; - -pub const TraceContainer = struct { - arena: std.heap.ArenaAllocator, - events: std.ArrayListUnmanaged(TraceEvent) = .{}, - devices: std.AutoArrayHashMapUnmanaged(u32, Device) = .{}, - - pub const Device = struct { - name: []const u8, - device_id: u32, - resources: std.AutoArrayHashMapUnmanaged(i64, Resource) = .{}, - }; - - pub const Resource = struct { - name: []const u8, - sort_index: i64, - }; - - pub const TraceEvent = struct { - device_id: u32 = 0, - resource_id: i64 = 0, - name: []const u8 = &[_]u8{}, - timestamp_ps: u128 = 0, - duration_ps: u64 = 0, - args: std.StringArrayHashMapUnmanaged([]const u8) = .{}, - }; - - pub fn init(allocator: std.mem.Allocator) TraceContainer { - return .{ - .arena = std.heap.ArenaAllocator.init(allocator), - }; - } - - pub fn deinit(self: *TraceContainer) void { - self.arena.deinit(); - } - - pub fn parseXSpaceBytes(self: *TraceContainer, pb_buffer: []const u8, max_events: ?usize) !void { - const arena = self.arena.allocator(); - - const xspace = try xplane_proto.XSpace.decode(pb_buffer, arena); - try self.fromXSpace(arena, xspace, max_events); - } - - fn findPlaneWithName(space: xplane_proto.XSpace, name: []const u8) ?*xplane_proto.XPlane { - for (space.planes.items) |*v| { - if (std.mem.eql(u8, v.name.getSlice(), name)) return v; - } - return null; - } - - fn findPlanesWithPrefix( - out: *std.ArrayList(*const xplane_proto.XPlane), - space: xplane_proto.XSpace, - prefix: []const u8, - ) !void { - for (space.planes.items) |*p| { - if (std.mem.startsWith(u8, p.name.getSlice(), prefix)) { - try out.append(p); - } - } - } - - fn xplaneToTraceEvents(self: *TraceContainer, allocator: std.mem.Allocator, device_id: u32, xplane: *const XPlaneHashed) !void { - // Convert devices and resources. - const device_entry = try self.devices.getOrPutValue(allocator, device_id, .{ .name = xplane.name(), .device_id = device_id }); - var device = device_entry.value_ptr.*; - defer device_entry.value_ptr.* = device; - - try device.resources.ensureUnusedCapacity(allocator, xplane.plane.lines.items.len); - const sort_by_ordinal = (device_id == host_threads_device_id); - - // Convert events. - for (xplane.plane.lines.items, 0..) |*xline, ordinal| { - const resource_id = if (xline.display_id != 0) xline.display_id else xline.id; - const resource_name = if (xline.display_name.isEmpty()) xline.name.getSlice() else xline.display_name.getSlice(); - device.resources.putAssumeCapacity(resource_id, .{ - .name = resource_name, - .sort_index = if (sort_by_ordinal) @intCast(ordinal) else resource_id, - }); - - if (std.mem.eql(u8, resource_name, xla_async_op_line_name)) continue; - - for (xline.events.items) |*xevent| { - const event_type = xplane.getEventType(xevent.metadata_id); - if (event_type.isInternalEvent()) continue; - var event = try self.createEvent(allocator); - event.device_id = device_id; - event.resource_id = resource_id; - - if (xplane.event_metadata_by_id.get(xevent.metadata_id)) |metadata| { - try event.args.ensureUnusedCapacity(allocator, 1 + metadata.stats.items.len); - - if (metadata.display_name != .Empty) { - event.name = metadata.display_name.getSlice(); - event.args.putAssumeCapacity("long_name", metadata.name.getSlice()); - } else { - event.name = metadata.name.getSlice(); - } - - event.timestamp_ps = (@as(u128, @intCast(xline.timestamp_ns)) * 1000) + @as(u128, @intCast(xevent.data.?.offset_ps)); - event.duration_ps = @intCast(xevent.duration_ps); - - for (metadata.stats.items) |xstat| { - if (xstat.value == null) continue; - var stat_buffer = std.ArrayList(u8).init(allocator); - try xplane.xstatToString(xstat, stat_buffer.writer().any()); - const stat_str = try stat_buffer.toOwnedSlice(); - const stat_type = xplane.getStatType(xstat.metadata_id); - if (stat_type.isInternalStat()) continue; - if (stat_type == .step_name) event.name = stat_str; - event.args.putAssumeCapacity(xplane.getStatMetadataName(xstat.metadata_id), stat_str); - } - } - - try event.args.ensureUnusedCapacity(allocator, xevent.stats.items.len); - for (xevent.stats.items) |xstat| { - if (xstat.value == null) continue; - var stat_buffer = std.ArrayList(u8).init(allocator); - try xplane.xstatToString(xstat, stat_buffer.writer().any()); - const stat_str = try stat_buffer.toOwnedSlice(); - const stat_type = xplane.getStatType(xstat.metadata_id); - if (stat_type.isInternalStat()) continue; - if (stat_type == .step_name) event.name = stat_str; - event.args.putAssumeCapacity(xplane.getStatMetadataName(xstat.metadata_id), stat_str); - } - } - } - } - - fn fromXSpace(self: *TraceContainer, allocator: std.mem.Allocator, xspace: xplane_proto.XSpace, max_events: ?usize) !void { - if (findPlaneWithName(xspace, host_threads_plane_name)) |hp| { - const xplane = try XPlaneHashed.init(allocator, hp); - try self.xplaneToTraceEvents(allocator, host_threads_device_id, &xplane); - } - - var device_planes = std.ArrayList(*const xplane_proto.XPlane).init(allocator); - defer device_planes.deinit(); - - try findPlanesWithPrefix(&device_planes, xspace, gpu_plane_prefix); - // We don't expect GPU and TPU planes and custom devices to be present in the same XSpace. - if (device_planes.items.len == 0) { - try findPlanesWithPrefix(&device_planes, xspace, tpu_plane_prefix); - } - if (device_planes.items.len == 0) { - try findPlanesWithPrefix(&device_planes, xspace, custom_plane_prefix); - } - - for (device_planes.items) |dp| { - var xplane = try XPlaneHashed.init(allocator, dp); - defer xplane.deinit(allocator); - const device_id: u32 = first_device_id + @as(u32, @intCast(xplane.plane.id)); - try self.xplaneToTraceEvents(allocator, device_id, &xplane); - } - - // Trace viewer (non-streaming) has scalability issues, we need to drop - // events to avoid loading failure for trace viewer. - if (max_events) |limit| self.capEvents(limit); - } - - pub fn createEvent(self: *TraceContainer, allocator: std.mem.Allocator) !*TraceEvent { - try self.events.append(allocator, .{}); - return &self.events.items[self.events.items.len - 1]; - } - - pub fn capEvents(self: *TraceContainer, max_count: u64) void { - const total_count = self.events.items.len; - if (total_count <= max_count) { - // Nothing to do. Events are not known sorted after return. - return; - } - // sort the events according to start time. - // TODO: partial sort would improve performance. - std.mem.sort(TraceEvent, self.events.items, {}, struct { - pub fn call(_: void, lhs: TraceEvent, rhs: TraceEvent) bool { - return lhs.timestamp_ps < rhs.timestamp_ps; - } - }.call); - self.events.shrinkRetainingCapacity(max_count); - } - - pub fn toJson(self: *TraceContainer, writer: anytype) !void { - try writer.writeAll( - \\{"displayTimeUnit":"ns","metadata":{"highres-ticks":true},"traceEvents":[ - ); - - self.devices.sort(struct { - keys: []const u32, - pub fn lessThan(ctx: @This(), lhs: usize, rhs: usize) bool { - return ctx.keys[lhs] < ctx.keys[rhs]; - } - }{ .keys = self.devices.keys() }); - - for (self.devices.keys(), self.devices.values()) |device_id, *device| { - if (device.name.len != 0) { - try writer.print( - \\{{"ph":"M","pid":{d},"name":"process_name","args":{{"name":"{s}"}}}}, - , .{ device_id, device.name }); - } - try writer.print( - \\{{"ph":"M","pid":{d},"name":"process_sort_index","args":{{"sort_index":{d}}}}}, - , .{ - device_id, - device_id, - }); - - device.resources.sort(struct { - keys: []const i64, - pub fn lessThan(ctx: @This(), lhs: usize, rhs: usize) bool { - return ctx.keys[lhs] < ctx.keys[rhs]; - } - }{ .keys = device.resources.keys() }); - - for (device.resources.keys(), device.resources.values()) |resource_id, resource| { - if (resource.name.len != 0) { - try writer.print( - \\{{"ph":"M","pid":{d},"tid":{d},"name":"thread_name","args":{{"name":"{s}"}}}}, - , .{ - device_id, - resource_id, - resource.name, - }); - } - try writer.print( - \\{{"ph":"M","pid":{d},"tid":{d},"name":"thread_sort_index","args":{{"sort_index":{d}}}}}, - , .{ device_id, resource_id, resource.sort_index }); - } - } - - for (self.events.items) |*event| { - const duration_ps = @max(event.duration_ps, 1); - try writer.print( - \\{{"ph":"X","pid":{d},"tid":{d},"ts":{d:.17},"dur":{d:.17},"name":"{s}" - , .{ - event.device_id, - event.resource_id, - picoToMicro(event.timestamp_ps), - picoToMicro(duration_ps), - event.name, - }); - if (event.args.count() != 0) { - try writer.writeAll( - \\,"args":{ - ); - event.args.sort(struct { - keys: []const []const u8, - - pub fn lessThan(ctx: @This(), lhs: usize, rhs: usize) bool { - return std.mem.order(u8, ctx.keys[lhs], ctx.keys[rhs]).compare(std.math.CompareOperator.lt); - } - }{ .keys = event.args.keys() }); - - for (event.args.keys(), event.args.values(), 0..) |key, value, i| { - if (i < event.args.count() - 1) { - try writer.print( - \\"{s}":"{s}", - , .{ key, value }); - } else { - // Last item has closing bracket rather than trailing comma. - try writer.print( - \\"{s}":"{s}"}} - , .{ key, value }); - } - } - } - try writer.writeAll("},"); - } - try writer.writeAll("{}]}"); - } -}; - -fn picoToMicro(p: anytype) f64 { - return @as(f64, @floatFromInt(p)) / 1E6; -} - -pub const XPlaneHashed = struct { - plane: *const xplane_proto.XPlane, - event_metadata_by_id: std.AutoHashMapUnmanaged(i64, *const xplane_proto.XEventMetadata) = .{}, - stat_metadata_by_id: std.AutoHashMapUnmanaged(i64, *const xplane_proto.XStatMetadata) = .{}, - - pub fn init( - allocator: std.mem.Allocator, - plane: *const xplane_proto.XPlane, - ) !XPlaneHashed { - var res: XPlaneHashed = .{ .plane = plane }; - - try res.event_metadata_by_id.ensureUnusedCapacity(allocator, @intCast(plane.event_metadata.items.len)); - // build event metadata map - for (plane.event_metadata.items) |*event_metadata| { - res.event_metadata_by_id.putAssumeCapacity(event_metadata.key, &event_metadata.value.?); - } - - // build stat metadata map - try res.stat_metadata_by_id.ensureUnusedCapacity(allocator, @intCast(plane.stat_metadata.items.len)); - for (plane.stat_metadata.items) |*stat_metadata| { - res.stat_metadata_by_id.putAssumeCapacity(stat_metadata.key, &stat_metadata.value.?); - } - - return res; - } - - pub fn deinit(self: *XPlaneHashed, allocator: std.mem.Allocator) void { - self.stat_metadata_by_id.deinit(allocator); - self.event_metadata_by_id.deinit(allocator); - } - - pub fn name(self: XPlaneHashed) []const u8 { - return self.plane.name.getSlice(); - } - - pub fn getEventType(self: XPlaneHashed, event_metadata_id: i64) xplane_schema.HostEventType { - if (self.event_metadata_by_id.get(event_metadata_id)) |v| { - return xplane_schema.HostEventType.fromString(v.name.getSlice()); - } else return .unknown; - } - - pub fn getStatMetadataName(self: XPlaneHashed, stat_metadata_id: i64) []const u8 { - if (self.stat_metadata_by_id.get(stat_metadata_id)) |v| { - return v.name.getSlice(); - } else return &[_]u8{}; - } - - pub fn getStatType(self: XPlaneHashed, stat_metadata_id: i64) xplane_schema.StatType { - if (self.stat_metadata_by_id.get(stat_metadata_id)) |v| { - return xplane_schema.StatType.fromString(v.name.getSlice()); - } else return .unknown; - } - - pub fn xstatToString(self: XPlaneHashed, stat: xplane_proto.XStat, writer: anytype) !void { - if (stat.value == null) return; - - switch (stat.value.?) { - inline .int64_value, .uint64_value, .double_value => |v| try writer.print("{d}", .{v}), - .str_value => |*v| try writer.writeAll(v.getSlice()), - .bytes_value => try writer.writeAll(""), - .ref_value => |v| try writer.writeAll(self.getStatMetadataName(@intCast(v))), - } - } -}; diff --git a/pjrt/convert/xplane_schema.zig b/pjrt/convert/xplane_schema.zig deleted file mode 100644 index ef70462d..00000000 --- a/pjrt/convert/xplane_schema.zig +++ /dev/null @@ -1,297 +0,0 @@ -const std = @import("std"); - -// `HostEventType` uses the unconventional casing/formatting -// so that the string representation of the enum used in the -// protobuf encoding directly maps to the zig enum tag name. -pub const HostEventType = enum(u16) { - unknown = 0, - TraceContext, - SessionRun, - FunctionRun, - RunGraph, - RunGraphDone, - TfOpRun, - EagerExecute, - @"ExecutorState::Process", - ExecutorDoneCallback, - MemoryAllocation, - MemoryDeallocation, - // Performance counter related. - RemotePerfCounter, - // tf.data captured function events. - @"InstantiatedCapturedFunction::Run", - @"InstantiatedCapturedFunction::RunWithBorrowedArgs", - @"InstantiatedCapturedFunction::RunInstantiated", - @"InstantiatedCapturedFunction::RunAsync", - // Loop ops. - ParallelForOp, - ForeverOp, - @"WhileOp-EvalCond", - @"WhileOp-StartBody", - ForOp, - // tf.data related. - @"IteratorGetNextOp::DoCompute", - @"IteratorGetNextAsOptionalOp::DoCompute", - Iterator, - @"Iterator::Prefetch::Generator", - PrefetchProduce, - PrefetchConsume, - ParallelInterleaveProduce, - ParallelInterleaveConsume, - ParallelInterleaveInitializeInput, - ParallelMapProduce, - ParallelMapConsume, - MapAndBatchProduce, - MapAndBatchConsume, - ParseExampleProduce, - ParseExampleConsume, - ParallelBatchProduce, - ParallelBatchConsume, - // Batching related. - BatchingSessionRun, - ProcessBatch, - BrainSessionRun, - ConcatInputTensors, - MergeInputTensors, - ScheduleWithoutSplit, - ScheduleWithSplit, - ScheduleWithEagerSplit, - @"ASBSQueue::Schedule", - // TFRT related. - TfrtModelRun, - // Serving related. - ServingModelRun, - // GPU related. - KernelLaunch, - KernelExecute, - // TPU related - EnqueueRequestLocked, - RunProgramRequest, - HostCallbackRequest, - TransferH2DRequest, - TransferPreprocessedH2DRequest, - TransferD2HRequest, - OnDeviceSendRequest, - OnDeviceRecvRequest, - OnDeviceSendRecvLocalRequest, - CustomWait, - OnDeviceSendRequestMulti, - OnDeviceRecvRequestMulti, - PjrtAsyncWait, - DoEnqueueProgram, - DoEnqueueContinuationProgram, - WriteHbm, - ReadHbm, - TpuExecuteOp, - CompleteCallbacks, - @"tpu::System::TransferToDevice=>IssueEvent", - @"tpu::System::TransferToDevice=>IssueEvent=>Done", - @"tpu::System::TransferFromDevice=>IssueEvent", - @"tpu::System::TransferFromDevice=>IssueEvent=>Done", - @"tpu::System::Execute", - @"TPUPartitionedCallOp-InitializeVarOnTPU", - @"TPUPartitionedCallOp-ExecuteRemote", - @"TPUPartitionedCallOp-ExecuteLocal", - Linearize, - Delinearize, - @"TransferBufferFromDevice-FastPath", - - pub fn fromString(event_name: []const u8) HostEventType { - return std.meta.stringToEnum(HostEventType, event_name) orelse .unknown; - } - - pub fn isInternalEvent(event_type: HostEventType) bool { - // TODO(b/162102421): Introduce a prefix for internal event names. - return switch (event_type) { - .MemoryAllocation, - .MemoryDeallocation, - .PrefetchProduce, - .PrefetchConsume, - .ParallelInterleaveProduce, - .ParallelInterleaveConsume, - .ParallelInterleaveInitializeInput, - .ParallelMapProduce, - .ParallelMapConsume, - .MapAndBatchProduce, - .MapAndBatchConsume, - .ParseExampleProduce, - .ParseExampleConsume, - => true, - else => false, - }; - } -}; - -// `StatType` uses the unconventional casing/formatting -// so that the string representation of the enum used in the -// protobuf encoding directly maps to the zig enum tag name. -pub const StatType = enum(u16) { - unknown = 0, - // TraceMe arguments. - id, - device_ordinal, - chip_ordinal, - node_ordinal, - model_id, - queue_id, - queue_addr, - request_id, - run_id, - replica_id, - graph_type, - step_num, - iter_num, - index_on_host, - allocator_name, - bytes_reserved, - bytes_allocated, - bytes_available, - fragmentation, - peak_bytes_in_use, - requested_bytes, - allocation_bytes, - addr, - region_type, - data_type, - shape, - layout, - kpi_name, - kpi_value, - element_id, - parent_id, - core_type, - // XPlane semantics related. - _pt, - _ct, - _p, - _c, - _r, - _a, - // Device trace arguments. - device_id, - device_type_string, - context_id, - correlation_id, - // TODO(b/176137043): These "details" should differentiate between activity - // and API event sources. - memcpy_details, - memalloc_details, - MemFree_details, - Memset_details, - MemoryResidency_details, - nvtx_range, - kernel_details, - stream, - // Stats added when processing traces. - group_id, - flow, - step_name, - tf_op, - hlo_op, - deduplicated_name, - hlo_category, - hlo_module, - program_id, - equation, - is_eager, - is_func, - tf_function_call, - tracing_count, - flops, - model_flops, - bytes_accessed, - memory_access_breakdown, - source, - model_name, - model_version, - bytes_transferred, - queue, - dcn_collective_info, - // Performance counter related. - @"Raw Value", - @"Scaled Value", - @"Thread Id", - matrix_unit_utilization_percent, - // XLA metadata map related. - @"Hlo Proto", - // Device capability related. - clock_rate, - // For GPU, this is the number of SMs. - core_count, - memory_bandwidth, - memory_size, - compute_cap_major, - compute_cap_minor, - peak_teraflops_per_second, - peak_hbm_bw_gigabytes_per_second, - peak_sram_rd_bw_gigabytes_per_second, - peak_sram_wr_bw_gigabytes_per_second, - device_vendor, - // Batching related. - batch_size_after_padding, - padding_amount, - batching_input_task_size, - // GPU occupancy metrics - theoretical_occupancy_pct, - occupancy_min_grid_size, - occupancy_suggested_block_size, - // Aggregated Stats - self_duration_ps, - min_duration_ps, - total_profile_duration_ps, - max_iteration_num, - device_type, - uses_megacore, - symbol_id, - tf_op_name, - dma_stall_duration_ps, - key, - payload_size_bytes, - duration_us, - buffer_size, - transfers, - // Dcn message Stats - dcn_label, - dcn_source_slice_id, - dcn_source_per_slice_device_id, - dcn_destination_slice_id, - dcn_destination_per_slice_device_id, - dcn_chunk, - dcn_loop_index, - @"EdgeTPU Model information", - @"EdgeTPU Model Profile information", - @"EdgeTPU MLIR", - dropped_traces, - cuda_graph_id, - // Many events have `.cuda_graph_id`, such as graph sub events when tracing is in - // node level. Yet `.cuda_graph_exec_id` is used only for CudaGraphExecution events - // on the GPU device when tracing is in graph level. - cuda_graph_exec_id, - cuda_graph_orig_id, - step_idle_time_ps, - gpu_device_name, - source_stack, - device_offset_ps, - device_duration_ps, - - pub fn fromString(stat_name: []const u8) StatType { - return std.meta.stringToEnum(StatType, stat_name) orelse .unknown; - } - - pub fn isInternalStat(stat_type: StatType) bool { - return switch (stat_type) { - .kernel_details, - ._pt, - ._p, - ._ct, - ._c, - ._r, - .flops, - .bytes_accessed, - .program_id, - .symbol_id, - => true, - else => false, - }; - } -}; diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 58d9ac29..00749d5d 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -5,7 +5,6 @@ const c = @import("c"); const stdx = @import("stdx"); pub const ffi = @import("ffi.zig"); -pub const Profiler = @import("profiler.zig").Profiler; const log = std.log.scoped(.pjrt); @@ -389,19 +388,6 @@ pub const Client = opaque { }; } - /// Returns the Profiler for this API. - /// Not all platform have a profiling api, for those the profiler object will do nothing. - /// Platforms with known profiler extensions: cuda, xpu - pub fn getProfiler(self: *const Client, api: *const Api, options: Profiler.Options) Profiler { - if (api.version().minor >= 45) { - if (api.lookupExtension(c.PJRT_Profiler_Extension, c.PJRT_Extension_Type_Profiler)) |ext| { - return Profiler.init(ext.profiler_api.*, options); - } - } - log.warn("No profiler found for platform: {}", .{self}); - return Profiler.init(null, null); - } - pub fn deserializeAndLoad(self: *const Client, api: *const Api, bytes: []const u8) ApiError!*LoadedExecutable { const ret = try api.call(.PJRT_Executable_DeserializeAndLoad, .{ .client = self.inner(), diff --git a/pjrt/profiler.zig b/pjrt/profiler.zig deleted file mode 100644 index 77f15208..00000000 --- a/pjrt/profiler.zig +++ /dev/null @@ -1,194 +0,0 @@ -const std = @import("std"); -const c = @import("c"); -const tsl_proto = @import("//tsl:profiler_options_proto"); - -const log = std.log.scoped(.@"pjrt/profiler"); -const TraceContainer = @import("convert/trace_container.zig").TraceContainer; - -/// Pjrt Profiler extension -pub const Profiler = struct { - api: ?c.PLUGIN_Profiler_Api, - inner: *c.PLUGIN_Profiler, - last_error: ?*Error = null, - status: Status = .ready, - - pub const Status = enum { ready, started, stopped, done }; - pub const Error = c.PLUGIN_Profiler_Error; - pub const Options = tsl_proto.ProfileOptions; - - pub const default_options: Options = .{ - .version = 1, - .device_type = .UNSPECIFIED, // profile all devices - .include_dataset_ops = false, // tensorflow specific - .host_tracer_level = 2, - .device_tracer_level = 1, - .python_tracer_level = 0, - .enable_hlo_proto = true, - .start_timestamp_ns = 0, - .duration_ms = 0, - .repository_path = .Empty, - }; - - pub fn init(api: ?c.PLUGIN_Profiler_Api, options: ?Options) Profiler { - if (api == null) { - return .{ .api = null, .inner = undefined }; - } - var options_with_timestamp = options orelse default_options; - options_with_timestamp.start_timestamp_ns = @truncate(@max(0, std.time.nanoTimestamp())); - - var buffer: [std.fs.max_path_bytes + @sizeOf(Options) * 4]u8 = undefined; - var fba = std.heap.FixedBufferAllocator.init(&buffer); - const byte_options = options_with_timestamp.encode(fba.allocator()) catch unreachable; - var args: c.PLUGIN_Profiler_Create_Args = .{ - .options = byte_options.ptr, - .options_size = byte_options.len, - .profiler = undefined, // out - }; - var res: Profiler = .{ .api = api, .inner = undefined }; - res.check(api.?.create.?(&args)) catch unreachable; - - res.inner = args.profiler.?; - return res; - } - - fn transition(self: *Profiler, fn_name: []const u8, expected: Status, next: Status) void { - if (self.status == expected) { - self.status = next; - return; - } - std.debug.panic("Profiler can't `{s}()`. Current status: {}, expected: {}", .{ fn_name, self.status, expected }); - } - - pub fn start(self: *Profiler) void { - self.transition("start", .ready, .started); - if (self.api == null) return; - var args: c.PLUGIN_Profiler_Start_Args = .{ .profiler = self.inner }; - self.check(self.api.?.start.?(&args)) catch unreachable; - } - - pub fn stop(self: *Profiler) void { - self.transition("stop", .started, .stopped); - if (self.api == null) return; - - var args: c.PLUGIN_Profiler_Stop_Args = .{ .profiler = self.inner }; - self.check(self.api.?.stop.?(&args)) catch unreachable; - } - - pub fn collectData(self: *Profiler, allocator: std.mem.Allocator) !ProfilingData { - self.transition("collect_data", .stopped, .done); - if (self.api == null) return .{ .external = &.{} }; - - var args: c.PLUGIN_Profiler_CollectData_Args = .{ - .struct_size = c.PLUGIN_Profiler_CollectData_Args_STRUCT_SIZE, - .profiler = self.inner, - .buffer = null, - .buffer_size_in_bytes = 0, - }; - try self.check(self.api.?.collect_data.?(&args)); - std.debug.assert(args.buffer_size_in_bytes > 0); - return if (args.buffer == null) blk: { - log.debug("Plugin profiler wants us to allocate {d} bytes for profile data", .{args.buffer_size_in_bytes}); - // The plugin want us to allocate memory for it: - const buffer = try allocator.alloc(u8, args.buffer_size_in_bytes); - args.buffer = buffer.ptr; - try self.check(self.api.?.collect_data.?(&args)); - break :blk .{ .owned = buffer }; - } else blk: { - log.debug("Plugin profiler has {d} bytes of profile data", .{args.buffer_size_in_bytes}); - // Drop sentinel. The profiler plugin returns a null terminated string. - // But this is creating issues if we save the sentinel on disk, - // because it will trip up protobuf readers. - var data = args.buffer[0..args.buffer_size_in_bytes]; - data = if (data.len > 0 and data[data.len - 1] == 0) data[0 .. data.len - 1] else data; - break :blk .{ .external = data }; - }; - } - - pub fn dumpDataTo( - self: *Profiler, - allocator: std.mem.Allocator, - dir: std.fs.Dir, - file_name: []const u8, - ) !void { - const profile_data = try self.collectData(allocator); - defer profile_data.free(allocator); - - if (profile_data.items().len == 0) return; - - const file = try dir.createFile(file_name, .{ .truncate = true }); - defer file.close(); - log.info("Writing profiling data to {s} ({} bytes)", .{ file_name, profile_data.items().len }); - return try file.writeAll(profile_data.items()); - } - - pub fn dumpAsJsonTo( - self: *Profiler, - allocator: std.mem.Allocator, - dir: std.fs.Dir, - file_name: []const u8, - ) !void { - log.info("Writing profiling data to {s}", .{file_name}); - var output_file = try dir.createFile(file_name, .{}); - defer output_file.close(); - var buffered_writer = std.io.bufferedWriter(output_file.writer()); - try self.dumpAsJsonToWriter(allocator, buffered_writer.writer()); - try buffered_writer.flush(); - } - - pub fn dumpAsJsonToWriter( - self: *Profiler, - allocator: std.mem.Allocator, - writer: anytype, - ) !void { - const profile_data = try self.collectData(allocator); - defer profile_data.free(allocator); - - if (profile_data.items().len == 0) { - log.warn("No profile data was collected: {}", .{self}); - return; - } - - var converter = TraceContainer.init(allocator); - defer converter.deinit(); - try converter.parseXSpaceBytes(profile_data.items(), 1_000_000); - - try converter.toJson(writer); - } - - fn check(self: *Profiler, c_error: ?*Error) !void { - if (c_error) |err| { - self.last_error = err; - return error.PjrtProfilerError; - } - } - - pub fn deinit(self: Profiler) void { - switch (self.status) { - .started => log.warn("Profiler was never stopped", .{}), - .stopped => log.warn("Profiler data was never collected", .{}), - else => {}, - } - if (self.api == null) return; - - var args: c.PLUGIN_Profiler_Destroy_Args = .{ .profiler = self.inner }; - _ = self.api.?.destroy.?(&args); - } -}; - -const ProfilingData = union(enum) { - owned: []const u8, - external: []const u8, - - pub fn items(self: ProfilingData) []const u8 { - return switch (self) { - inline else => |x| x, - }; - } - - pub fn free(self: ProfilingData, allocator: std.mem.Allocator) void { - switch (self) { - .owned => |data| allocator.free(data), - .external => {}, - } - } -}; diff --git a/pjrt/xspace_to_json.zig b/pjrt/xspace_to_json.zig deleted file mode 100644 index e5c0698b..00000000 --- a/pjrt/xspace_to_json.zig +++ /dev/null @@ -1,47 +0,0 @@ -const std = @import("std"); -const stdx = @import("stdx"); -const flags = stdx.flags; - -const TraceContainer = @import("convert/trace_container.zig").TraceContainer; - -const CliArgs = struct { - pub const help = - \\ llama --path=path_to_profiling_data - ; - path: []const u8, - max_events: ?usize = null, -}; - -pub fn main() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{ .thread_safe = true }){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - var args = std.process.args(); - const cli_args = flags.parse(&args, CliArgs); - - var fd = try std.fs.openFileAbsolute(cli_args.path, .{}); - defer fd.close(); - - const pb_buffer = try fd.readToEndAlloc(allocator, (try fd.stat()).size); - defer allocator.free(pb_buffer); - if (pb_buffer.len == 0) return error.EmptyBuffer; - - var converter = TraceContainer.init(allocator); - defer converter.deinit(); - try converter.parseXSpaceBytes(pb_buffer, cli_args.max_events); - - var path_buffer: [1028]u8 = undefined; - - const output_path = try std.fmt.bufPrint(&path_buffer, "{s}/{s}.json", .{ - std.fs.path.dirname(cli_args.path) orelse "", - std.fs.path.stem(cli_args.path), - }); - - var output_file = try std.fs.createFileAbsolute(output_path, .{}); - defer output_file.close(); - - try converter.toJson(output_file.writer().any()); - - std.debug.print("Wrote JSON to {s}\n", .{output_path}); -} diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 1795efc3..b12bdb20 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -5,7 +5,6 @@ const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); pub const ffi = pjrt.ffi; -pub const Profiler = pjrt.Profiler; pub const ApiError = pjrt.ApiError; pub const ErrorCode = pjrt.ErrorCode; pub const ExecuteContext = pjrt.ExecuteContext; @@ -109,13 +108,6 @@ pub const Client = opaque { return try asynk.callBlocking(compileSync, .{ self, api, allocator, module, compile_options_pb }); } - /// Returns the Profiler for this API. - /// Not all platform have a profiling api, for those the profiler object will do nothing. - /// Platforms with known profiler extensions: cuda, xpu - pub fn getProfiler(self: *const Client, api: *const Api, options: pjrt.Profiler.Options) pjrt.Profiler { - return self.inner().getProfiler(api, options); - } - pub fn addressableMemories(self: *const Client, api: *const Api) []*const Memory { return self.inner().addressableMemories(api); } diff --git a/zml/platform.zig b/zml/platform.zig index a7bc87c9..6f48137d 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -85,13 +85,6 @@ pub const Platform = struct { pub fn deinit(self: *Platform) void { self.pjrt_client.deinit(self.pjrt_api); } - - /// Returns the Profiler for this API. - /// Not all platform have a profiling api, for those the profiler object will do nothing. - /// Platforms with known profiler extensions: cuda, xpu - pub fn getProfiler(self: Platform, options: ?pjrt.Profiler.Options) pjrt.Profiler { - return self.pjrt_client.getProfiler(self.pjrt_api, options orelse pjrt.Profiler.default_options); - } }; const _CreateOptions = struct { From 2044cd2de5c9507080f81336d1b1d048ef112208 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 19 Aug 2025 16:14:54 +0200 Subject: [PATCH 012/881] zml: remove zig-protobuf, switch to upb (#307) Closes #301 --- MODULE.bazel | 20 +- MODULE.bazel.lock | 4714 +---------------- bazel/zig_proto_library.bzl | 2 +- examples/MODULE.bazel.lock | 208 +- .../sentencepiece.bazel | 6 +- upb/BUILD.bazel | 12 + upb/upb.zig | 149 + zml/BUILD.bazel | 23 +- zml/module.zig | 166 +- 9 files changed, 597 insertions(+), 4703 deletions(-) create mode 100644 upb/BUILD.bazel create mode 100644 upb/upb.zig diff --git a/MODULE.bazel b/MODULE.bazel index 9ff10eb5..cffd6f0e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -10,7 +10,7 @@ bazel_dep(name = "libxev", version = "20250718.0-9f785d2") bazel_dep(name = "patchelf", version = "0.18.0") bazel_dep(name = "pcre2", version = "10.43") bazel_dep(name = "platforms", version = "0.0.11") -bazel_dep(name = "protobuf", version = "29.2") +bazel_dep(name = "protobuf", version = "32.0", repo_name = "com_google_protobuf") bazel_dep(name = "rules_cc", version = "0.1.1") bazel_dep(name = "rules_distroless", version = "0.5.1") bazel_dep(name = "rules_proto", version = "7.1.0") @@ -20,27 +20,12 @@ bazel_dep(name = "rules_uv", version = "0.65.0") bazel_dep(name = "rules_zig", version = "20250714.0-b14a4f1") bazel_dep(name = "sentencepiece", version = "20240618.0-d7ace0a") bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.2.4") -bazel_dep(name = "toolchains_protoc", version = "0.4.1") bazel_dep(name = "with_cfg.bzl", version = "0.9.1") bazel_dep(name = "xla", version = "20250718.0-6319f0d") -bazel_dep(name = "zig-protobuf", version = "20250716.0-97f1e31") bazel_dep(name = "zig-yaml", version = "20240903.0-83d5fdf") bazel_dep(name = "buildifier_prebuilt", version = "8.0.3", dev_dependency = True) -# Optional: choose a version of protoc rather than the latest. -protoc = use_extension("@toolchains_protoc//protoc:extensions.bzl", "protoc") -protoc.toolchain( - # Creates a repository to satisfy well-known-types dependencies such as - # deps=["@com_google_protobuf//:any_proto"] - google_protobuf = "com_google_protobuf", - # Pin to any version of protoc - version = "v29.2", -) -use_repo(protoc, "com_google_protobuf", "toolchains_protoc_hub") - -register_toolchains("@toolchains_protoc_hub//:all") - zig = use_extension("@rules_zig//zig:extensions.bzl", "zig") zig.index(file = "//bazel:zig_index.json") zig.toolchain(zig_version = "0.14.1") @@ -88,7 +73,6 @@ pip.parse( ) use_repo(pip, "huggingface_hub") - cpu = use_extension("//runtimes/cpu:cpu.bzl", "cpu_pjrt_plugin") use_repo(cpu, "libpjrt_cpu_darwin_amd64", "libpjrt_cpu_darwin_arm64", "libpjrt_cpu_linux_amd64") @@ -160,12 +144,14 @@ apt.install( manifest = "//runtimes/cuda:packages.yaml", ) use_repo(apt, "apt_cuda") + apt.install( name = "apt_rocm", lock = "//runtimes/rocm:packages.lock.json", manifest = "//runtimes/rocm:packages.yaml", ) use_repo(apt, "apt_rocm") + apt.install( name = "apt_neuron", lock = "//runtimes/neuron:packages.lock.json", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 248424bf..412d1fcd 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -11,6 +11,8 @@ "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20240722.0.bcr.2/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/abseil-cpp/20250127.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/abseil-cpp/20250512.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/apple_support/1.11.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/apple_support/1.15.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/apple_support/1.17.1/MODULE.bazel": "not found", @@ -29,6 +31,7 @@ "https://bazel-registry.zml.ai/modules/bazel_features/1.19.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.20.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.21.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/bazel_features/1.23.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.24.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.25.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.27.0/MODULE.bazel": "not found", @@ -56,7 +59,9 @@ "https://bazel-registry.zml.ai/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/googletest/1.14.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/googletest/1.15.2/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/googletest/1.17.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/jsoncpp/1.9.5/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/jsoncpp/1.9.6/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/libpfm/4.11.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/nlohmann_json/3.6.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/patchelf/0.18.0/MODULE.bazel": "not found", @@ -78,14 +83,15 @@ "https://bazel-registry.zml.ai/modules/protobuf/29.0-rc3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/protobuf/29.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/protobuf/29.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/29.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/protobuf/3.19.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/protobuf/32.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/pybind11_bazel/2.11.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/pybind11_bazel/2.12.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/re2/2023-09-01/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/re2/2024-07-02/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_android/0.1.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_apple/3.16.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_apple/3.22.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.10/MODULE.bazel": "not found", @@ -117,6 +123,7 @@ "https://bazel-registry.zml.ai/modules/rules_java/8.11.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_java/8.3.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_java/8.5.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_java/8.6.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_java/8.8.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_jvm_external/4.4.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_jvm_external/5.1/MODULE.bazel": "not found", @@ -124,6 +131,7 @@ "https://bazel-registry.zml.ai/modules/rules_jvm_external/5.3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_jvm_external/6.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_jvm_external/6.3/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_jvm_external/6.7/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_kotlin/1.9.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_kotlin/1.9.6/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_license/0.0.3/MODULE.bazel": "not found", @@ -153,11 +161,13 @@ "https://bazel-registry.zml.ai/modules/rules_python/0.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_python/0.40.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_python/1.3.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_python/1.4.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_rust/0.62.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.2.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.3.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.4.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/2.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_uv/0.65.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", @@ -177,6 +187,7 @@ "https://bazel-registry.zml.ai/modules/zlib/1.2.11/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/zlib/1.2.13/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/zlib/1.3.1/MODULE.bazel": "not found", "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2", @@ -189,7 +200,9 @@ "https://bcr.bazel.build/modules/abseil-cpp/20240116.1/MODULE.bazel": "37bcdb4440fbb61df6a1c296ae01b327f19e9bb521f9b8e26ec854b6f97309ed", "https://bcr.bazel.build/modules/abseil-cpp/20240116.2/MODULE.bazel": "73939767a4686cd9a520d16af5ab440071ed75cec1a876bf2fcfaf1f71987a16", "https://bcr.bazel.build/modules/abseil-cpp/20240722.0.bcr.2/MODULE.bazel": "c3661b44c9d3f17f0b65ffb544896aaeb89127398ea867537babac18133a002a", - "https://bcr.bazel.build/modules/abseil-cpp/20240722.0.bcr.2/source.json": "464f3012c8f40cd51facbfc6962563971c6bd55f45ba16bd98313f2de73340c5", + "https://bcr.bazel.build/modules/abseil-cpp/20250127.1/MODULE.bazel": "c4a89e7ceb9bf1e25cf84a9f830ff6b817b72874088bf5141b314726e46a57c1", + "https://bcr.bazel.build/modules/abseil-cpp/20250512.1/MODULE.bazel": "d209fdb6f36ffaf61c509fcc81b19e81b411a999a934a032e10cd009a0226215", + "https://bcr.bazel.build/modules/abseil-cpp/20250512.1/source.json": "d725d73707d01bb46ab3ca59ba408b8e9bd336642ca77a2269d4bfb8bbfd413d", "https://bcr.bazel.build/modules/apple_support/1.11.1/MODULE.bazel": "1843d7cd8a58369a444fc6000e7304425fba600ff641592161d9f15b179fb896", "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", "https://bcr.bazel.build/modules/apple_support/1.17.1/MODULE.bazel": "655c922ab1209978a94ef6ca7d9d43e940cd97d9c172fb55f94d91ac53f8610b", @@ -211,6 +224,7 @@ "https://bcr.bazel.build/modules/bazel_features/1.19.0/MODULE.bazel": "59adcdf28230d220f0067b1f435b8537dd033bfff8db21335ef9217919c7fb58", "https://bcr.bazel.build/modules/bazel_features/1.20.0/MODULE.bazel": "8b85300b9c8594752e0721a37210e34879d23adc219ed9dc8f4104a4a1750920", "https://bcr.bazel.build/modules/bazel_features/1.21.0/MODULE.bazel": "675642261665d8eea09989aa3b8afb5c37627f1be178382c320d1b46afba5e3b", + "https://bcr.bazel.build/modules/bazel_features/1.23.0/MODULE.bazel": "fd1ac84bc4e97a5a0816b7fd7d4d4f6d837b0047cf4cbd81652d616af3a6591a", "https://bcr.bazel.build/modules/bazel_features/1.24.0/MODULE.bazel": "4796b4c25b47053e9bbffa792b3792d07e228ff66cd0405faef56a978708acd4", "https://bcr.bazel.build/modules/bazel_features/1.25.0/MODULE.bazel": "e2e60a10a6da64bbf533f15ca652bf61a033e41c2ed734d79a9a08ba87f68c1a", "https://bcr.bazel.build/modules/bazel_features/1.27.0/MODULE.bazel": "621eeee06c4458a9121d1f104efb80f39d34deff4984e778359c60eaf1a8cb65", @@ -242,9 +256,11 @@ "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6", "https://bcr.bazel.build/modules/googletest/1.14.0/MODULE.bazel": "cfbcbf3e6eac06ef9d85900f64424708cc08687d1b527f0ef65aa7517af8118f", "https://bcr.bazel.build/modules/googletest/1.15.2/MODULE.bazel": "6de1edc1d26cafb0ea1a6ab3f4d4192d91a312fd2d360b63adaa213cd00b2108", - "https://bcr.bazel.build/modules/googletest/1.15.2/source.json": "dbdda654dcb3a0d7a8bc5d0ac5fc7e150b58c2a986025ae5bc634bb2cb61f470", + "https://bcr.bazel.build/modules/googletest/1.17.0/MODULE.bazel": "dbec758171594a705933a29fcf69293d2468c49ec1f2ebca65c36f504d72df46", + "https://bcr.bazel.build/modules/googletest/1.17.0/source.json": "38e4454b25fc30f15439c0378e57909ab1fd0a443158aa35aec685da727cd713", "https://bcr.bazel.build/modules/jsoncpp/1.9.5/MODULE.bazel": "31271aedc59e815656f5736f282bb7509a97c7ecb43e927ac1a37966e0578075", - "https://bcr.bazel.build/modules/jsoncpp/1.9.5/source.json": "4108ee5085dd2885a341c7fab149429db457b3169b86eb081fa245eadf69169d", + "https://bcr.bazel.build/modules/jsoncpp/1.9.6/MODULE.bazel": "2f8d20d3b7d54143213c4dfc3d98225c42de7d666011528dc8fe91591e2e17b0", + "https://bcr.bazel.build/modules/jsoncpp/1.9.6/source.json": "a04756d367a2126c3541682864ecec52f92cdee80a35735a3cb249ce015ca000", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/source.json": "f448c6e8963fdfa7eb831457df83ad63d3d6355018f6574fb017e8169deb43a9", @@ -270,9 +286,9 @@ "https://bcr.bazel.build/modules/protobuf/29.0-rc3/MODULE.bazel": "33c2dfa286578573afc55a7acaea3cada4122b9631007c594bf0729f41c8de92", "https://bcr.bazel.build/modules/protobuf/29.0/MODULE.bazel": "319dc8bf4c679ff87e71b1ccfb5a6e90a6dbc4693501d471f48662ac46d04e4e", "https://bcr.bazel.build/modules/protobuf/29.1/MODULE.bazel": "557c3457560ff49e122ed76c0bc3397a64af9574691cb8201b4e46d4ab2ecb95", - "https://bcr.bazel.build/modules/protobuf/29.2/MODULE.bazel": "5435497c190d86f79b0568698c45044df7c8d97692886cda9fe9cf9053aea712", - "https://bcr.bazel.build/modules/protobuf/29.2/source.json": "fe7090cc34072609b26d9beafb122916dabc1d47ba61b242c26c4b06c51384ab", "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0", + "https://bcr.bazel.build/modules/protobuf/32.0/MODULE.bazel": "0741cf24f8e1185286578069060e905ed67d68eef5990bfa3dea3fc1afba14c7", + "https://bcr.bazel.build/modules/protobuf/32.0/source.json": "1e278267d3642ab361dc460cc1f2d8e607e8292b9b3f799a47181a3acf4f3294", "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e", "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/MODULE.bazel": "e6f4c20442eaa7c90d7190d8dc539d0ab422f95c65a57cc59562170c58ae3d34", "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/source.json": "6900fdc8a9e95866b8c0d4ad4aba4d4236317b5c1cd04c502df3f0d33afed680", @@ -282,6 +298,7 @@ "https://bcr.bazel.build/modules/re2/2024-07-02/MODULE.bazel": "0eadc4395959969297cbcf31a249ff457f2f1d456228c67719480205aa306daa", "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", + "https://bcr.bazel.build/modules/rules_apple/3.16.0/MODULE.bazel": "0d1caf0b8375942ce98ea944be754a18874041e4e0459401d925577624d3a54a", "https://bcr.bazel.build/modules/rules_apple/3.22.0/MODULE.bazel": "0fd8c630d6cecc18abe84114ee21c1a0fb759b45063d3d2600ec22669e93ea43", "https://bcr.bazel.build/modules/rules_apple/3.22.0/source.json": "ee41f63c292bd876fd264a830cf45500619a063e2fa3a18cca6fa12b153d68e3", "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", @@ -301,7 +318,6 @@ "https://bcr.bazel.build/modules/rules_distroless/0.5.1/source.json": "c6b9ff7f325bfed89c3671757f14c1d1bc6077d0fcef809b8aa2d007cac7dd1d", "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", "https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/MODULE.bazel": "40c97d1144356f52905566c55811f13b299453a14ac7769dfba2ac38192337a8", - "https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/source.json": "c8b1e2c717646f1702290959a3302a178fb639d987ab61d548105019f11e527e", "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", "https://bcr.bazel.build/modules/rules_java/5.3.5/MODULE.bazel": "a4ec4f2db570171e3e5eb753276ee4b389bae16b96207e9d3230895c99644b86", "https://bcr.bazel.build/modules/rules_java/6.0.0/MODULE.bazel": "8a43b7df601a7ec1af61d79345c17b31ea1fedc6711fd4abfd013ea612978e39", @@ -318,6 +334,7 @@ "https://bcr.bazel.build/modules/rules_java/8.11.0/source.json": "302b52a39259a85aa06ca3addb9787864ca3e03b432a5f964ea68244397e7544", "https://bcr.bazel.build/modules/rules_java/8.3.2/MODULE.bazel": "7336d5511ad5af0b8615fdc7477535a2e4e723a357b6713af439fe8cf0195017", "https://bcr.bazel.build/modules/rules_java/8.5.1/MODULE.bazel": "d8a9e38cc5228881f7055a6079f6f7821a073df3744d441978e7a43e20226939", + "https://bcr.bazel.build/modules/rules_java/8.6.1/MODULE.bazel": "f4808e2ab5b0197f094cabce9f4b006a27766beb6a9975931da07099560ca9c2", "https://bcr.bazel.build/modules/rules_java/8.8.0/MODULE.bazel": "de589d0880911ac007abd521b9f0ddcd8b0dbd05c8553e6f8124a050b83acf7d", "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7", "https://bcr.bazel.build/modules/rules_jvm_external/5.1/MODULE.bazel": "33f6f999e03183f7d088c9be518a63467dfd0be94a11d0055fe2d210f89aa909", @@ -325,7 +342,8 @@ "https://bcr.bazel.build/modules/rules_jvm_external/5.3/MODULE.bazel": "bf93870767689637164657731849fb887ad086739bd5d360d90007a581d5527d", "https://bcr.bazel.build/modules/rules_jvm_external/6.1/MODULE.bazel": "75b5fec090dbd46cf9b7d8ea08cf84a0472d92ba3585b476f44c326eda8059c4", "https://bcr.bazel.build/modules/rules_jvm_external/6.3/MODULE.bazel": "c998e060b85f71e00de5ec552019347c8bca255062c990ac02d051bb80a38df0", - "https://bcr.bazel.build/modules/rules_jvm_external/6.3/source.json": "6f5f5a5a4419ae4e37c35a5bb0a6ae657ed40b7abc5a5189111b47fcebe43197", + "https://bcr.bazel.build/modules/rules_jvm_external/6.7/MODULE.bazel": "e717beabc4d091ecb2c803c2d341b88590e9116b8bf7947915eeb33aab4f96dd", + "https://bcr.bazel.build/modules/rules_jvm_external/6.7/source.json": "5426f412d0a7fc6b611643376c7e4a82dec991491b9ce5cb1cfdd25fe2e92be4", "https://bcr.bazel.build/modules/rules_kotlin/1.9.0/MODULE.bazel": "ef85697305025e5a61f395d4eaede272a5393cee479ace6686dba707de804d59", "https://bcr.bazel.build/modules/rules_kotlin/1.9.6/MODULE.bazel": "d269a01a18ee74d0335450b10f62c9ed81f2321d7958a2934e44272fe82dcef3", "https://bcr.bazel.build/modules/rules_kotlin/1.9.6/source.json": "2faa4794364282db7c06600b7e5e34867a564ae91bda7cae7c29c64e9466b7d5", @@ -360,7 +378,8 @@ "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", "https://bcr.bazel.build/modules/rules_python/0.40.0/MODULE.bazel": "9d1a3cd88ed7d8e39583d9ffe56ae8a244f67783ae89b60caafc9f5cf318ada7", "https://bcr.bazel.build/modules/rules_python/1.3.0/MODULE.bazel": "8361d57eafb67c09b75bf4bbe6be360e1b8f4f18118ab48037f2bd50aa2ccb13", - "https://bcr.bazel.build/modules/rules_python/1.3.0/source.json": "25932f917cd279c7baefa6cb1d3fa8750a7a29de522024449b19af6eab51f4a0", + "https://bcr.bazel.build/modules/rules_python/1.4.1/MODULE.bazel": "8991ad45bdc25018301d6b7e1d3626afc3c8af8aaf4bc04f23d0b99c938b73a6", + "https://bcr.bazel.build/modules/rules_python/1.4.1/source.json": "8ec8c90c70ccacc4de8ca1b97f599e756fb59173e898ee08b733006650057c07", "https://bcr.bazel.build/modules/rules_rust/0.62.0/MODULE.bazel": "6a15b57982e278793c684f426e19166e62e73f1bd45fe3b6bcedd0b901177b37", "https://bcr.bazel.build/modules/rules_rust/0.62.0/source.json": "1b3a6551a585ee47cafa21550c5eb87c6f3a56bb9761f9e3421ff1102f220437", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", @@ -368,6 +387,7 @@ "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", "https://bcr.bazel.build/modules/rules_shell/0.4.1/source.json": "4757bd277fe1567763991c4425b483477bb82e35e777a56fd846eb5cceda324a", "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", + "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", "https://bcr.bazel.build/modules/rules_swift/2.4.0/source.json": "a6577f57f9febbdc015a01f2a8f3487422032f134d6c61d18ed8e8ca3b9acc7c", "https://bcr.bazel.build/modules/rules_uv/0.65.0/MODULE.bazel": "a700505a80ad0ad65f062c9335458d137f504b98c5937b338807bb5cbe305e73", @@ -394,15 +414,16 @@ "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", "https://bcr.bazel.build/modules/zlib/1.2.13/MODULE.bazel": "aa6deb1b83c18ffecd940c4119aff9567cd0a671d7bba756741cb2ef043a29d5", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "af322bc08976524477c79d1e45e241b6efbeb918c497e8840b8ab116802dda79", - "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/source.json": "2be409ac3c7601245958cd4fcdff4288be79ed23bd690b4b951f500d54ee6e7d", + "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "eec517b5bbe5492629466e11dae908d043364302283de25581e3eb944326c4ca", + "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/source.json": "22bc55c47af97246cfc093d0acf683a7869377de362b5d1c552c2c2e16b7a806", "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198" }, "selectedYankedVersions": {}, "moduleExtensions": { "@@apple_support+//crosstool:setup.bzl%apple_cc_configure_extension": { "general": { - "bzlTransitiveDigest": "4vzWoFey2mnm603onNMfaDX4DRoTrnIgZawOBMnWpoI=", - "usagesDigest": "g16sWEKaACd08DhNP3oNi7nAS8e2q6cva2PQQJGC3sM=", + "bzlTransitiveDigest": "JJV/C5bhYKZip41t/GGZvqIMmmrcwr0Ydybkn97uXxQ=", + "usagesDigest": "39X2JjPCOAk6sThDALGv1L4q85GNjda2yfszm/phxxw=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -432,8 +453,8 @@ }, "@@aspect_rules_py+//py:extensions.bzl%py_tools": { "general": { - "bzlTransitiveDigest": "YIPHOwcYJl8X9QiZwJZ6IKTETxJTMYqApef63o81SeQ=", - "usagesDigest": "/X+U31L/0W+mPo15khS6kjLevIqdkqh+JgfyWHDdHH8=", + "bzlTransitiveDigest": "geB/ya+5nbONwFUgzta9iVBeMKtviiSBwlZnE7Avcgs=", + "usagesDigest": "tVlELaqQFX7HPl1uFzRY9m7nVV7tY9roq1qfK75ApwE=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -468,6 +489,12 @@ "platform": "windows_amd64" } }, + "bsd_tar_windows_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", + "attributes": { + "platform": "windows_arm64" + } + }, "bsd_tar_toolchains": { "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%tar_toolchains_repo", "attributes": { @@ -475,25 +502,25 @@ } }, "rules_py_tools.darwin_amd64": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:tools.bzl%prebuilt_tool_repo", + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", "attributes": { "platform": "darwin_amd64" } }, "rules_py_tools.darwin_arm64": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:tools.bzl%prebuilt_tool_repo", + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", "attributes": { "platform": "darwin_arm64" } }, "rules_py_tools.linux_amd64": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:tools.bzl%prebuilt_tool_repo", + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", "attributes": { "platform": "linux_amd64" } }, "rules_py_tools.linux_arm64": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:tools.bzl%prebuilt_tool_repo", + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", "attributes": { "platform": "linux_arm64" } @@ -528,145 +555,16 @@ ], [ "aspect_rules_py+", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@buildifier_prebuilt+//:defs.bzl%buildifier_prebuilt_deps_extension": { - "general": { - "bzlTransitiveDigest": "ezSrAolp2o8ezFDLQE9qsNwhqDOW0g2hQJG4NEzsLUo=", - "usagesDigest": "E92htUw9elHOT0ZHIEdwgPVnslFqL2iu3pcplYXXAxk=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "buildifier_darwin_amd64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildifier-darwin-amd64" - ], - "downloaded_file_path": "buildifier", - "executable": true, - "sha256": "b7a3152cde0b3971b1107f2274afe778c5c154dcdf6c9c669a231e3c004f047e" - } - }, - "buildifier_darwin_arm64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildifier-darwin-arm64" - ], - "downloaded_file_path": "buildifier", - "executable": true, - "sha256": "674c663f7b5cd03c002f8ca834a8c1c008ccb527a0a2a132d08a7a355883b22d" - } - }, - "buildifier_linux_amd64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildifier-linux-amd64" - ], - "downloaded_file_path": "buildifier", - "executable": true, - "sha256": "c969487c1af85e708576c8dfdd0bb4681eae58aad79e68ae48882c70871841b7" - } - }, - "buildifier_linux_arm64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildifier-linux-arm64" - ], - "downloaded_file_path": "buildifier", - "executable": true, - "sha256": "bdd9b92e2c65d46affeecaefb54e68d34c272d1f4a8c5b54929a3e92ab78820a" - } - }, - "buildifier_windows_amd64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildifier-windows-amd64.exe" - ], - "downloaded_file_path": "buildifier.exe", - "executable": true, - "sha256": "63a242f57e253efe7b9573d739c08a3d0e628efd84015c8dad17d87b6429e443" - } - }, - "buildozer_darwin_amd64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildozer-darwin-amd64" - ], - "downloaded_file_path": "buildozer", - "executable": true, - "sha256": "7aa7929faddcee5a879135356bb4d38af3e895893c78d361e7cdb8dfb3e7d5e4" - } - }, - "buildozer_darwin_arm64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildozer-darwin-arm64" - ], - "downloaded_file_path": "buildozer", - "executable": true, - "sha256": "33b776862c31bfb4e2e6e88186cae02746d9084da8b7341930e77d42718277cc" - } - }, - "buildozer_linux_amd64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildozer-linux-amd64" - ], - "downloaded_file_path": "buildozer", - "executable": true, - "sha256": "775d17118aecd023352e75611dbe7c12a340e23615ffb270a1dec3f339239949" - } - }, - "buildozer_linux_arm64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildozer-linux-arm64" - ], - "downloaded_file_path": "buildozer", - "executable": true, - "sha256": "058ad60f279a7b1db339bb287aed2056a992eddfe1b5d2e56d86256bdc9a93b3" - } - }, - "buildozer_windows_amd64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildozer-windows-amd64.exe" - ], - "downloaded_file_path": "buildozer.exe", - "executable": true, - "sha256": "5ba106b25230e0e321b7259bee325ae6fa5b5771e54163ad865b1d18856f59ca" - } - }, - "buildifier_prebuilt_toolchains": { - "repoRuleId": "@@buildifier_prebuilt+//:defs.bzl%_buildifier_toolchain_setup", - "attributes": { - "assets_json": "[{\"arch\":\"amd64\",\"name\":\"buildifier\",\"platform\":\"darwin\",\"sha256\":\"b7a3152cde0b3971b1107f2274afe778c5c154dcdf6c9c669a231e3c004f047e\",\"version\":\"v8.0.3\"},{\"arch\":\"arm64\",\"name\":\"buildifier\",\"platform\":\"darwin\",\"sha256\":\"674c663f7b5cd03c002f8ca834a8c1c008ccb527a0a2a132d08a7a355883b22d\",\"version\":\"v8.0.3\"},{\"arch\":\"amd64\",\"name\":\"buildifier\",\"platform\":\"linux\",\"sha256\":\"c969487c1af85e708576c8dfdd0bb4681eae58aad79e68ae48882c70871841b7\",\"version\":\"v8.0.3\"},{\"arch\":\"arm64\",\"name\":\"buildifier\",\"platform\":\"linux\",\"sha256\":\"bdd9b92e2c65d46affeecaefb54e68d34c272d1f4a8c5b54929a3e92ab78820a\",\"version\":\"v8.0.3\"},{\"arch\":\"amd64\",\"name\":\"buildifier\",\"platform\":\"windows\",\"sha256\":\"63a242f57e253efe7b9573d739c08a3d0e628efd84015c8dad17d87b6429e443\",\"version\":\"v8.0.3\"},{\"arch\":\"amd64\",\"name\":\"buildozer\",\"platform\":\"darwin\",\"sha256\":\"7aa7929faddcee5a879135356bb4d38af3e895893c78d361e7cdb8dfb3e7d5e4\",\"version\":\"v8.0.3\"},{\"arch\":\"arm64\",\"name\":\"buildozer\",\"platform\":\"darwin\",\"sha256\":\"33b776862c31bfb4e2e6e88186cae02746d9084da8b7341930e77d42718277cc\",\"version\":\"v8.0.3\"},{\"arch\":\"amd64\",\"name\":\"buildozer\",\"platform\":\"linux\",\"sha256\":\"775d17118aecd023352e75611dbe7c12a340e23615ffb270a1dec3f339239949\",\"version\":\"v8.0.3\"},{\"arch\":\"arm64\",\"name\":\"buildozer\",\"platform\":\"linux\",\"sha256\":\"058ad60f279a7b1db339bb287aed2056a992eddfe1b5d2e56d86256bdc9a93b3\",\"version\":\"v8.0.3\"},{\"arch\":\"amd64\",\"name\":\"buildozer\",\"platform\":\"windows\",\"sha256\":\"5ba106b25230e0e321b7259bee325ae6fa5b5771e54163ad865b1d18856f59ca\",\"version\":\"v8.0.3\"}]" - } - } - }, - "recordedRepoMappingEntries": [ + "aspect_tools_telemetry_report", + "aspect_tools_telemetry++telemetry+aspect_tools_telemetry_report" + ], [ - "buildifier_prebuilt+", + "aspect_rules_py+", "bazel_skylib", "bazel_skylib+" ], [ - "buildifier_prebuilt+", + "aspect_rules_py+", "bazel_tools", "bazel_tools" ] @@ -703,174 +601,9 @@ ] } }, - "@@rules_apple+//apple:apple.bzl%provisioning_profile_repository_extension": { - "general": { - "bzlTransitiveDigest": "mVdP3ANttcsLRv2k6Sbzs06sQDCgBTB8CPMicPSeCW4=", - "usagesDigest": "+Mri48un5kxPRrX8WFbwbgbUsG6JygfULZLaltX0weQ=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "local_provisioning_profiles": { - "repoRuleId": "@@rules_apple+//apple/internal:local_provisioning_profiles.bzl%provisioning_profile_repository", - "attributes": {} - } - }, - "recordedRepoMappingEntries": [ - [ - "apple_support+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "bazel_tools", - "rules_cc", - "rules_cc+" - ], - [ - "rules_apple+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "rules_apple+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_apple+", - "build_bazel_apple_support", - "apple_support+" - ], - [ - "rules_apple+", - "build_bazel_rules_swift", - "rules_swift+" - ], - [ - "rules_cc+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_cc+", - "rules_cc", - "rules_cc+" - ], - [ - "rules_swift+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "rules_swift+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_swift+", - "build_bazel_apple_support", - "apple_support+" - ], - [ - "rules_swift+", - "build_bazel_rules_swift", - "rules_swift+" - ], - [ - "rules_swift+", - "build_bazel_rules_swift_local_config", - "rules_swift++non_module_deps+build_bazel_rules_swift_local_config" - ] - ] - } - }, - "@@rules_fuzzing+//fuzzing/private:extensions.bzl%non_module_dependencies": { - "general": { - "bzlTransitiveDigest": "Wn0Wqc8tJukC1aqT+Fqun/4Xrnr+iTThN24Fn3rz3FU=", - "usagesDigest": "wy6ISK6UOcBEjj/mvJ/S3WeXoO67X+1llb9yPyFtPgc=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "platforms": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.8/platforms-0.0.8.tar.gz", - "https://github.com/bazelbuild/platforms/releases/download/0.0.8/platforms-0.0.8.tar.gz" - ], - "sha256": "8150406605389ececb6da07cbcb509d5637a3ab9a24bc69b1101531367d89d74" - } - }, - "rules_python": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "sha256": "d70cd72a7a4880f0000a6346253414825c19cdd40a28289bdf67b8e6480edff8", - "strip_prefix": "rules_python-0.28.0", - "url": "https://github.com/bazelbuild/rules_python/releases/download/0.28.0/rules_python-0.28.0.tar.gz" - } - }, - "bazel_skylib": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "sha256": "cd55a062e763b9349921f0f5db8c3933288dc8ba4f76dd9416aac68acee3cb94", - "urls": [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz", - "https://github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz" - ] - } - }, - "com_google_absl": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/abseil/abseil-cpp/archive/refs/tags/20240116.1.zip" - ], - "strip_prefix": "abseil-cpp-20240116.1", - "integrity": "sha256-7capMWOvWyoYbUaHF/b+I2U6XLMaHmky8KugWvfXYuk=" - } - }, - "rules_fuzzing_oss_fuzz": { - "repoRuleId": "@@rules_fuzzing+//fuzzing/private/oss_fuzz:repository.bzl%oss_fuzz_repository", - "attributes": {} - }, - "honggfuzz": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "build_file": "@@rules_fuzzing+//:honggfuzz.BUILD", - "sha256": "6b18ba13bc1f36b7b950c72d80f19ea67fbadc0ac0bb297ec89ad91f2eaa423e", - "url": "https://github.com/google/honggfuzz/archive/2.5.zip", - "strip_prefix": "honggfuzz-2.5" - } - }, - "rules_fuzzing_jazzer": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_jar", - "attributes": { - "sha256": "ee6feb569d88962d59cb59e8a31eb9d007c82683f3ebc64955fd5b96f277eec2", - "url": "https://repo1.maven.org/maven2/com/code-intelligence/jazzer/0.20.1/jazzer-0.20.1.jar" - } - }, - "rules_fuzzing_jazzer_api": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_jar", - "attributes": { - "sha256": "f5a60242bc408f7fa20fccf10d6c5c5ea1fcb3c6f44642fec5af88373ae7aa1b", - "url": "https://repo1.maven.org/maven2/com/code-intelligence/jazzer-api/0.20.1/jazzer-api-0.20.1.jar" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_fuzzing+", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { "general": { - "bzlTransitiveDigest": "fmfKdvTpZCJJntCdqlB6bYFsD3ax+7qZpeR0cGMDe8A=", + "bzlTransitiveDigest": "URQTcVYyatfFO8w2HD1Z1tb8lTBnhHn1wkiu5DYTSow=", "usagesDigest": "QI2z8ZUR+mqtbwsf2fLqYdJAkPOHdOV+tF2yVAUgRzw=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -934,8 +667,8 @@ }, "@@rules_multitool+//multitool:extension.bzl%multitool": { "general": { - "bzlTransitiveDigest": "r9TGCyer7o1g/TEqrdFBahcMZPkvD+oYDO4dUcGWYMs=", - "usagesDigest": "Bn6IoM2pzksz/oBIRwZyfLpBZBFrZnzat00Xcp/lWqw=", + "bzlTransitiveDigest": "CxG1tSKAppSqmYJtpLgGpxrOvrd/xewKD0M2kCNfvOo=", + "usagesDigest": "kKFKsjNo1vbiSGlyYYHdSAj1Z8xtmvq8JBs56ZEpNS4=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -1028,3796 +761,158 @@ ] } }, - "@@rules_python+//python/extensions:pip.bzl%pip": { - "general": { - "bzlTransitiveDigest": "fOwLlf5Rfxv4xkFCZwPlkSPUuZ9lAhMGf0SN43pc0xg=", - "usagesDigest": "Y6wFcKuqojwXVUMUxBiLATIjxW3VxUxL71LIt7iDHkw=", - "recordedFileInputs": { - "@@//runtimes/neuron/requirements.lock.txt": "fa0b5b967162e303415ee1dd7d2c1a448068935663ba5ac5b5f72a515fe3f6f6", - "@@//tools/hf_requirements.lock.txt": "ffbcfcfd01c9a31db823affa4ab6122c509b2f1eeb4b853325d1dbd5cc9568da", - "@@protobuf+//python/requirements.txt": "2bb341a0ba317fb8dc9ac79817b9b2d8f6a4d0c43e3fb652f0cab584eed420a4", - "@@rules_fuzzing+//fuzzing/requirements.txt": "d4e91a1b3eca06d6e2877f67d3d8be498cb3ed6d28b9c14439b6f210ec621470", - "@@rules_python+//tools/publish/requirements_darwin.txt": "6d10732737e5504291360b78fbf2b82abe39a5a021bd937f987f8433519b0f2c", - "@@rules_python+//tools/publish/requirements_linux.txt": "cb63e4751377d1722a81fd5835b0526208620f1303feb473ed0dd45ae4d71ba9", - "@@rules_python+//tools/publish/requirements_windows.txt": "f92da1a252df04e1931a0e16dfbc5a3c165a85b796b7fc5e56ee4e3846d82558" - }, - "recordedDirentsInputs": {}, - "envVariables": { - "RULES_PYTHON_REPO_DEBUG": null, - "RULES_PYTHON_REPO_DEBUG_VERBOSITY": null - }, - "generatedRepoSpecs": { - "huggingface_hub_311_certifi": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "certifi==2025.7.14 --hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 --hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995" - } - }, - "huggingface_hub_311_charset_normalizer": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "charset-normalizer==3.4.2 --hash=sha256:005fa3432484527f9732ebd315da8da8001593e2cf46a3d817669f062c3d9ed4 --hash=sha256:046595208aae0120559a67693ecc65dd75d46f7bf687f159127046628178dc45 --hash=sha256:0c29de6a1a95f24b9a1aa7aefd27d2487263f00dfd55a77719b530788f75cff7 --hash=sha256:0c8c57f84ccfc871a48a47321cfa49ae1df56cd1d965a09abe84066f6853b9c0 --hash=sha256:0f5d9ed7f254402c9e7d35d2f5972c9bbea9040e99cd2861bd77dc68263277c7 --hash=sha256:18dd2e350387c87dabe711b86f83c9c78af772c748904d372ade190b5c7c9d4d --hash=sha256:1b1bde144d98e446b056ef98e59c256e9294f6b74d7af6846bf5ffdafd687a7d --hash=sha256:1c95a1e2902a8b722868587c0e1184ad5c55631de5afc0eb96bc4b0d738092c0 --hash=sha256:1cad5f45b3146325bb38d6855642f6fd609c3f7cad4dbaf75549bf3b904d3184 --hash=sha256:21b2899062867b0e1fde9b724f8aecb1af14f2778d69aacd1a5a1853a597a5db --hash=sha256:24498ba8ed6c2e0b56d4acbf83f2d989720a93b41d712ebd4f4979660db4417b --hash=sha256:25a23ea5c7edc53e0f29bae2c44fcb5a1aa10591aae107f2a2b2583a9c5cbc64 --hash=sha256:289200a18fa698949d2b39c671c2cc7a24d44096784e76614899a7ccf2574b7b --hash=sha256:28a1005facc94196e1fb3e82a3d442a9d9110b8434fc1ded7a24a2983c9888d8 --hash=sha256:32fc0341d72e0f73f80acb0a2c94216bd704f4f0bce10aedea38f30502b271ff --hash=sha256:36b31da18b8890a76ec181c3cf44326bf2c48e36d393ca1b72b3f484113ea344 --hash=sha256:3c21d4fca343c805a52c0c78edc01e3477f6dd1ad7c47653241cf2a206d4fc58 --hash=sha256:3fddb7e2c84ac87ac3a947cb4e66d143ca5863ef48e4a5ecb83bd48619e4634e --hash=sha256:43e0933a0eff183ee85833f341ec567c0980dae57c464d8a508e1b2ceb336471 --hash=sha256:4a476b06fbcf359ad25d34a057b7219281286ae2477cc5ff5e3f70a246971148 --hash=sha256:4e594135de17ab3866138f496755f302b72157d115086d100c3f19370839dd3a --hash=sha256:50bf98d5e563b83cc29471fa114366e6806bc06bc7a25fd59641e41445327836 --hash=sha256:5a9979887252a82fefd3d3ed2a8e3b937a7a809f65dcb1e068b090e165bbe99e --hash=sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63 --hash=sha256:5bf4545e3b962767e5c06fe1738f951f77d27967cb2caa64c28be7c4563e162c --hash=sha256:6333b3aa5a12c26b2a4d4e7335a28f1475e0e5e17d69d55141ee3cab736f66d1 --hash=sha256:65c981bdbd3f57670af8b59777cbfae75364b483fa8a9f420f08094531d54a01 --hash=sha256:68a328e5f55ec37c57f19ebb1fdc56a248db2e3e9ad769919a58672958e8f366 --hash=sha256:6a0289e4589e8bdfef02a80478f1dfcb14f0ab696b5a00e1f4b8a14a307a3c58 --hash=sha256:6b66f92b17849b85cad91259efc341dce9c1af48e2173bf38a85c6329f1033e5 --hash=sha256:6c9379d65defcab82d07b2a9dfbfc2e95bc8fe0ebb1b176a3190230a3ef0e07c --hash=sha256:6fc1f5b51fa4cecaa18f2bd7a003f3dd039dd615cd69a2afd6d3b19aed6775f2 --hash=sha256:70f7172939fdf8790425ba31915bfbe8335030f05b9913d7ae00a87d4395620a --hash=sha256:721c76e84fe669be19c5791da68232ca2e05ba5185575086e384352e2c309597 --hash=sha256:7222ffd5e4de8e57e03ce2cef95a4c43c98fcb72ad86909abdfc2c17d227fc1b --hash=sha256:75d10d37a47afee94919c4fab4c22b9bc2a8bf7d4f46f87363bcf0573f3ff4f5 --hash=sha256:76af085e67e56c8816c3ccf256ebd136def2ed9654525348cfa744b6802b69eb --hash=sha256:770cab594ecf99ae64c236bc9ee3439c3f46be49796e265ce0cc8bc17b10294f --hash=sha256:7a6ab32f7210554a96cd9e33abe3ddd86732beeafc7a28e9955cdf22ffadbab0 --hash=sha256:7c48ed483eb946e6c04ccbe02c6b4d1d48e51944b6db70f697e089c193404941 --hash=sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0 --hash=sha256:8075c35cd58273fee266c58c0c9b670947c19df5fb98e7b66710e04ad4e9ff86 --hash=sha256:8272b73e1c5603666618805fe821edba66892e2870058c94c53147602eab29c7 --hash=sha256:82d8fd25b7f4675d0c47cf95b594d4e7b158aca33b76aa63d07186e13c0e0ab7 --hash=sha256:844da2b5728b5ce0e32d863af26f32b5ce61bc4273a9c720a9f3aa9df73b1455 --hash=sha256:8755483f3c00d6c9a77f490c17e6ab0c8729e39e6390328e42521ef175380ae6 --hash=sha256:915f3849a011c1f593ab99092f3cecfcb4d65d8feb4a64cf1bf2d22074dc0ec4 --hash=sha256:926ca93accd5d36ccdabd803392ddc3e03e6d4cd1cf17deff3b989ab8e9dbcf0 --hash=sha256:982bb1e8b4ffda883b3d0a521e23abcd6fd17418f6d2c4118d257a10199c0ce3 --hash=sha256:98f862da73774290f251b9df8d11161b6cf25b599a66baf087c1ffe340e9bfd1 --hash=sha256:9cbfacf36cb0ec2897ce0ebc5d08ca44213af24265bd56eca54bee7923c48fd6 --hash=sha256:a370b3e078e418187da8c3674eddb9d983ec09445c99a3a263c2011993522981 --hash=sha256:a955b438e62efdf7e0b7b52a64dc5c3396e2634baa62471768a64bc2adb73d5c --hash=sha256:aa6af9e7d59f9c12b33ae4e9450619cf2488e2bbe9b44030905877f0b2324980 --hash=sha256:aa88ca0b1932e93f2d961bf3addbb2db902198dca337d88c89e1559e066e7645 --hash=sha256:aaeeb6a479c7667fbe1099af9617c83aaca22182d6cf8c53966491a0f1b7ffb7 --hash=sha256:aaf27faa992bfee0264dc1f03f4c75e9fcdda66a519db6b957a3f826e285cf12 --hash=sha256:b2680962a4848b3c4f155dc2ee64505a9c57186d0d56b43123b17ca3de18f0fa --hash=sha256:b2d318c11350e10662026ad0eb71bb51c7812fc8590825304ae0bdd4ac283acd --hash=sha256:b33de11b92e9f75a2b545d6e9b6f37e398d86c3e9e9653c4864eb7e89c5773ef --hash=sha256:b3daeac64d5b371dea99714f08ffc2c208522ec6b06fbc7866a450dd446f5c0f --hash=sha256:be1e352acbe3c78727a16a455126d9ff83ea2dfdcbc83148d2982305a04714c2 --hash=sha256:bee093bf902e1d8fc0ac143c88902c3dfc8941f7ea1d6a8dd2bcb786d33db03d --hash=sha256:c72fbbe68c6f32f251bdc08b8611c7b3060612236e960ef848e0a517ddbe76c5 --hash=sha256:c9e36a97bee9b86ef9a1cf7bb96747eb7a15c2f22bdb5b516434b00f2a599f02 --hash=sha256:cddf7bd982eaa998934a91f69d182aec997c6c468898efe6679af88283b498d3 --hash=sha256:cf713fe9a71ef6fd5adf7a79670135081cd4431c2943864757f0fa3a65b1fafd --hash=sha256:d11b54acf878eef558599658b0ffca78138c8c3655cf4f3a4a673c437e67732e --hash=sha256:d41c4d287cfc69060fa91cae9683eacffad989f1a10811995fa309df656ec214 --hash=sha256:d524ba3f1581b35c03cb42beebab4a13e6cdad7b36246bd22541fa585a56cccd --hash=sha256:daac4765328a919a805fa5e2720f3e94767abd632ae410a9062dff5412bae65a --hash=sha256:db4c7bf0e07fc3b7d89ac2a5880a6a8062056801b83ff56d8464b70f65482b6c --hash=sha256:dc7039885fa1baf9be153a0626e337aa7ec8bf96b0128605fb0d77788ddc1681 --hash=sha256:dccab8d5fa1ef9bfba0590ecf4d46df048d18ffe3eec01eeb73a42e0d9e7a8ba --hash=sha256:dedb8adb91d11846ee08bec4c8236c8549ac721c245678282dcb06b221aab59f --hash=sha256:e45ba65510e2647721e35323d6ef54c7974959f6081b58d4ef5d87c60c84919a --hash=sha256:e53efc7c7cee4c1e70661e2e112ca46a575f90ed9ae3fef200f2a25e954f4b28 --hash=sha256:e635b87f01ebc977342e2697d05b56632f5f879a4f15955dfe8cef2448b51691 --hash=sha256:e70e990b2137b29dc5564715de1e12701815dacc1d056308e2b17e9095372a82 --hash=sha256:e8082b26888e2f8b36a042a58307d5b917ef2b1cacab921ad3323ef91901c71a --hash=sha256:e8323a9b031aa0393768b87f04b4164a40037fb2a3c11ac06a03ffecd3618027 --hash=sha256:e92fca20c46e9f5e1bb485887d074918b13543b1c2a1185e69bb8d17ab6236a7 --hash=sha256:eb30abc20df9ab0814b5a2524f23d75dcf83cde762c161917a2b4b7b55b1e518 --hash=sha256:eba9904b0f38a143592d9fc0e19e2df0fa2e41c3c3745554761c5f6447eedabf --hash=sha256:ef8de666d6179b009dce7bcb2ad4c4a779f113f12caf8dc77f0162c29d20490b --hash=sha256:efd387a49825780ff861998cd959767800d54f8308936b21025326de4b5a42b9 --hash=sha256:f0aa37f3c979cf2546b73e8222bbfa3dc07a641585340179d768068e3455e544 --hash=sha256:f4074c5a429281bf056ddd4c5d3b740ebca4d43ffffe2ef4bf4d2d05114299da --hash=sha256:f69a27e45c43520f5487f27627059b64aaf160415589230992cec34c5e18a509 --hash=sha256:fb707f3e15060adf5b7ada797624a6c6e0138e2a26baa089df64c68ee98e040f --hash=sha256:fcbe676a55d7445b22c10967bceaaf0ee69407fbe0ece4d032b6eb8d4565982a --hash=sha256:fdb20a30fe1175ecabed17cbf7812f7b804b8a315a25f24678bcdf120a90077f" - } - }, - "huggingface_hub_311_filelock": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "filelock==3.18.0 --hash=sha256:adbc88eabb99d2fec8c9c1b229b171f18afa655400173ddc653d5d01501fb9f2 --hash=sha256:c401f4f8377c4464e6db25fff06205fd89bdd83b65eb0488ed1b160f780e21de" - } - }, - "huggingface_hub_311_fsspec": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "fsspec==2025.7.0 --hash=sha256:786120687ffa54b8283d942929540d8bc5ccfa820deb555a2b5d0ed2b737bf58 --hash=sha256:8b012e39f63c7d5f10474de957f3ab793b47b45ae7d39f2fb735f8bbe25c0e21" - } - }, - "huggingface_hub_311_hf_xet": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "hf-xet==1.1.5 --hash=sha256:69ebbcfd9ec44fdc2af73441619eeb06b94ee34511bbcf57cd423820090f5694 --hash=sha256:73e167d9807d166596b4b2f0b585c6d5bd84a26dea32843665a8b58f6edba245 --hash=sha256:83088ecea236d5113de478acb2339f92c95b4fb0462acaa30621fac02f5a534a --hash=sha256:9fa6e3ee5d61912c4a113e0708eaaef987047616465ac7aa30f7121a48fc1af8 --hash=sha256:ab34c4c3104133c495785d5d8bba3b1efc99de52c02e759cf711a91fd39d3a14 --hash=sha256:dbba1660e5d810bd0ea77c511a99e9242d920790d0e63c0e4673ed36c4022d18 --hash=sha256:f52c2fa3635b8c37c7764d8796dfa72706cc4eded19d638331161e82b0792e23 --hash=sha256:fc874b5c843e642f45fd85cda1ce599e123308ad2901ead23d3510a47ff506d1" - } - }, - "huggingface_hub_311_huggingface_hub": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "huggingface-hub==0.34.3 --hash=sha256:5444550099e2d86e68b2898b09e85878fbd788fc2957b506c6a79ce060e39492 --hash=sha256:d58130fd5aa7408480681475491c0abd7e835442082fbc3ef4d45b6c39f83853" - } - }, - "huggingface_hub_311_idna": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "idna==3.10 --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3" - } - }, - "huggingface_hub_311_packaging": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "packaging==25.0 --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f" - } - }, - "huggingface_hub_311_pyyaml": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "pyyaml==6.0.2 --hash=sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff --hash=sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48 --hash=sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086 --hash=sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e --hash=sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133 --hash=sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5 --hash=sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484 --hash=sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee --hash=sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5 --hash=sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68 --hash=sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a --hash=sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf --hash=sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99 --hash=sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8 --hash=sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85 --hash=sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19 --hash=sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc --hash=sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a --hash=sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1 --hash=sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317 --hash=sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c --hash=sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631 --hash=sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d --hash=sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652 --hash=sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5 --hash=sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e --hash=sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b --hash=sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8 --hash=sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476 --hash=sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706 --hash=sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563 --hash=sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237 --hash=sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b --hash=sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083 --hash=sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180 --hash=sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425 --hash=sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e --hash=sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f --hash=sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725 --hash=sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183 --hash=sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab --hash=sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774 --hash=sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725 --hash=sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e --hash=sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5 --hash=sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d --hash=sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290 --hash=sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44 --hash=sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed --hash=sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4 --hash=sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba --hash=sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12 --hash=sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4" - } - }, - "huggingface_hub_311_requests": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "requests==2.32.4 --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422" - } - }, - "huggingface_hub_311_tqdm": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "tqdm==4.67.1 --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2" - } - }, - "huggingface_hub_311_typing_extensions": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "typing-extensions==4.14.1 --hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 --hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76" - } - }, - "huggingface_hub_311_urllib3": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "urllib3==2.5.0 --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc" - } - }, - "neuron_py_deps_311_boto3": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "boto3==1.39.9 --hash=sha256:5bc85e9fdec4e21ef5ca2c22b4d51a3e32b53f3da36ce51f5a3ea4dbde07b132 --hash=sha256:e3d3a6b617e1575e7ec854c820a882ab2e189a0421e74dc0dca2c9e13d4370a5" - } - }, - "neuron_py_deps_311_botocore": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "botocore==1.39.9 --hash=sha256:02f141c2849e4589a79feea245ce4ecc478d48b7865572445af8aae3b041772d --hash=sha256:a9691cbe03a3bc8b2720b3c36e5c5a2eecace6acd72bfb1107f00e75edaec4f3" - } - }, - "neuron_py_deps_311_certifi": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "certifi==2025.7.14 --hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 --hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995" - } - }, - "neuron_py_deps_311_charset_normalizer": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "charset-normalizer==3.4.2 --hash=sha256:005fa3432484527f9732ebd315da8da8001593e2cf46a3d817669f062c3d9ed4 --hash=sha256:046595208aae0120559a67693ecc65dd75d46f7bf687f159127046628178dc45 --hash=sha256:0c29de6a1a95f24b9a1aa7aefd27d2487263f00dfd55a77719b530788f75cff7 --hash=sha256:0c8c57f84ccfc871a48a47321cfa49ae1df56cd1d965a09abe84066f6853b9c0 --hash=sha256:0f5d9ed7f254402c9e7d35d2f5972c9bbea9040e99cd2861bd77dc68263277c7 --hash=sha256:18dd2e350387c87dabe711b86f83c9c78af772c748904d372ade190b5c7c9d4d --hash=sha256:1b1bde144d98e446b056ef98e59c256e9294f6b74d7af6846bf5ffdafd687a7d --hash=sha256:1c95a1e2902a8b722868587c0e1184ad5c55631de5afc0eb96bc4b0d738092c0 --hash=sha256:1cad5f45b3146325bb38d6855642f6fd609c3f7cad4dbaf75549bf3b904d3184 --hash=sha256:21b2899062867b0e1fde9b724f8aecb1af14f2778d69aacd1a5a1853a597a5db --hash=sha256:24498ba8ed6c2e0b56d4acbf83f2d989720a93b41d712ebd4f4979660db4417b --hash=sha256:25a23ea5c7edc53e0f29bae2c44fcb5a1aa10591aae107f2a2b2583a9c5cbc64 --hash=sha256:289200a18fa698949d2b39c671c2cc7a24d44096784e76614899a7ccf2574b7b --hash=sha256:28a1005facc94196e1fb3e82a3d442a9d9110b8434fc1ded7a24a2983c9888d8 --hash=sha256:32fc0341d72e0f73f80acb0a2c94216bd704f4f0bce10aedea38f30502b271ff --hash=sha256:36b31da18b8890a76ec181c3cf44326bf2c48e36d393ca1b72b3f484113ea344 --hash=sha256:3c21d4fca343c805a52c0c78edc01e3477f6dd1ad7c47653241cf2a206d4fc58 --hash=sha256:3fddb7e2c84ac87ac3a947cb4e66d143ca5863ef48e4a5ecb83bd48619e4634e --hash=sha256:43e0933a0eff183ee85833f341ec567c0980dae57c464d8a508e1b2ceb336471 --hash=sha256:4a476b06fbcf359ad25d34a057b7219281286ae2477cc5ff5e3f70a246971148 --hash=sha256:4e594135de17ab3866138f496755f302b72157d115086d100c3f19370839dd3a --hash=sha256:50bf98d5e563b83cc29471fa114366e6806bc06bc7a25fd59641e41445327836 --hash=sha256:5a9979887252a82fefd3d3ed2a8e3b937a7a809f65dcb1e068b090e165bbe99e --hash=sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63 --hash=sha256:5bf4545e3b962767e5c06fe1738f951f77d27967cb2caa64c28be7c4563e162c --hash=sha256:6333b3aa5a12c26b2a4d4e7335a28f1475e0e5e17d69d55141ee3cab736f66d1 --hash=sha256:65c981bdbd3f57670af8b59777cbfae75364b483fa8a9f420f08094531d54a01 --hash=sha256:68a328e5f55ec37c57f19ebb1fdc56a248db2e3e9ad769919a58672958e8f366 --hash=sha256:6a0289e4589e8bdfef02a80478f1dfcb14f0ab696b5a00e1f4b8a14a307a3c58 --hash=sha256:6b66f92b17849b85cad91259efc341dce9c1af48e2173bf38a85c6329f1033e5 --hash=sha256:6c9379d65defcab82d07b2a9dfbfc2e95bc8fe0ebb1b176a3190230a3ef0e07c --hash=sha256:6fc1f5b51fa4cecaa18f2bd7a003f3dd039dd615cd69a2afd6d3b19aed6775f2 --hash=sha256:70f7172939fdf8790425ba31915bfbe8335030f05b9913d7ae00a87d4395620a --hash=sha256:721c76e84fe669be19c5791da68232ca2e05ba5185575086e384352e2c309597 --hash=sha256:7222ffd5e4de8e57e03ce2cef95a4c43c98fcb72ad86909abdfc2c17d227fc1b --hash=sha256:75d10d37a47afee94919c4fab4c22b9bc2a8bf7d4f46f87363bcf0573f3ff4f5 --hash=sha256:76af085e67e56c8816c3ccf256ebd136def2ed9654525348cfa744b6802b69eb --hash=sha256:770cab594ecf99ae64c236bc9ee3439c3f46be49796e265ce0cc8bc17b10294f --hash=sha256:7a6ab32f7210554a96cd9e33abe3ddd86732beeafc7a28e9955cdf22ffadbab0 --hash=sha256:7c48ed483eb946e6c04ccbe02c6b4d1d48e51944b6db70f697e089c193404941 --hash=sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0 --hash=sha256:8075c35cd58273fee266c58c0c9b670947c19df5fb98e7b66710e04ad4e9ff86 --hash=sha256:8272b73e1c5603666618805fe821edba66892e2870058c94c53147602eab29c7 --hash=sha256:82d8fd25b7f4675d0c47cf95b594d4e7b158aca33b76aa63d07186e13c0e0ab7 --hash=sha256:844da2b5728b5ce0e32d863af26f32b5ce61bc4273a9c720a9f3aa9df73b1455 --hash=sha256:8755483f3c00d6c9a77f490c17e6ab0c8729e39e6390328e42521ef175380ae6 --hash=sha256:915f3849a011c1f593ab99092f3cecfcb4d65d8feb4a64cf1bf2d22074dc0ec4 --hash=sha256:926ca93accd5d36ccdabd803392ddc3e03e6d4cd1cf17deff3b989ab8e9dbcf0 --hash=sha256:982bb1e8b4ffda883b3d0a521e23abcd6fd17418f6d2c4118d257a10199c0ce3 --hash=sha256:98f862da73774290f251b9df8d11161b6cf25b599a66baf087c1ffe340e9bfd1 --hash=sha256:9cbfacf36cb0ec2897ce0ebc5d08ca44213af24265bd56eca54bee7923c48fd6 --hash=sha256:a370b3e078e418187da8c3674eddb9d983ec09445c99a3a263c2011993522981 --hash=sha256:a955b438e62efdf7e0b7b52a64dc5c3396e2634baa62471768a64bc2adb73d5c --hash=sha256:aa6af9e7d59f9c12b33ae4e9450619cf2488e2bbe9b44030905877f0b2324980 --hash=sha256:aa88ca0b1932e93f2d961bf3addbb2db902198dca337d88c89e1559e066e7645 --hash=sha256:aaeeb6a479c7667fbe1099af9617c83aaca22182d6cf8c53966491a0f1b7ffb7 --hash=sha256:aaf27faa992bfee0264dc1f03f4c75e9fcdda66a519db6b957a3f826e285cf12 --hash=sha256:b2680962a4848b3c4f155dc2ee64505a9c57186d0d56b43123b17ca3de18f0fa --hash=sha256:b2d318c11350e10662026ad0eb71bb51c7812fc8590825304ae0bdd4ac283acd --hash=sha256:b33de11b92e9f75a2b545d6e9b6f37e398d86c3e9e9653c4864eb7e89c5773ef --hash=sha256:b3daeac64d5b371dea99714f08ffc2c208522ec6b06fbc7866a450dd446f5c0f --hash=sha256:be1e352acbe3c78727a16a455126d9ff83ea2dfdcbc83148d2982305a04714c2 --hash=sha256:bee093bf902e1d8fc0ac143c88902c3dfc8941f7ea1d6a8dd2bcb786d33db03d --hash=sha256:c72fbbe68c6f32f251bdc08b8611c7b3060612236e960ef848e0a517ddbe76c5 --hash=sha256:c9e36a97bee9b86ef9a1cf7bb96747eb7a15c2f22bdb5b516434b00f2a599f02 --hash=sha256:cddf7bd982eaa998934a91f69d182aec997c6c468898efe6679af88283b498d3 --hash=sha256:cf713fe9a71ef6fd5adf7a79670135081cd4431c2943864757f0fa3a65b1fafd --hash=sha256:d11b54acf878eef558599658b0ffca78138c8c3655cf4f3a4a673c437e67732e --hash=sha256:d41c4d287cfc69060fa91cae9683eacffad989f1a10811995fa309df656ec214 --hash=sha256:d524ba3f1581b35c03cb42beebab4a13e6cdad7b36246bd22541fa585a56cccd --hash=sha256:daac4765328a919a805fa5e2720f3e94767abd632ae410a9062dff5412bae65a --hash=sha256:db4c7bf0e07fc3b7d89ac2a5880a6a8062056801b83ff56d8464b70f65482b6c --hash=sha256:dc7039885fa1baf9be153a0626e337aa7ec8bf96b0128605fb0d77788ddc1681 --hash=sha256:dccab8d5fa1ef9bfba0590ecf4d46df048d18ffe3eec01eeb73a42e0d9e7a8ba --hash=sha256:dedb8adb91d11846ee08bec4c8236c8549ac721c245678282dcb06b221aab59f --hash=sha256:e45ba65510e2647721e35323d6ef54c7974959f6081b58d4ef5d87c60c84919a --hash=sha256:e53efc7c7cee4c1e70661e2e112ca46a575f90ed9ae3fef200f2a25e954f4b28 --hash=sha256:e635b87f01ebc977342e2697d05b56632f5f879a4f15955dfe8cef2448b51691 --hash=sha256:e70e990b2137b29dc5564715de1e12701815dacc1d056308e2b17e9095372a82 --hash=sha256:e8082b26888e2f8b36a042a58307d5b917ef2b1cacab921ad3323ef91901c71a --hash=sha256:e8323a9b031aa0393768b87f04b4164a40037fb2a3c11ac06a03ffecd3618027 --hash=sha256:e92fca20c46e9f5e1bb485887d074918b13543b1c2a1185e69bb8d17ab6236a7 --hash=sha256:eb30abc20df9ab0814b5a2524f23d75dcf83cde762c161917a2b4b7b55b1e518 --hash=sha256:eba9904b0f38a143592d9fc0e19e2df0fa2e41c3c3745554761c5f6447eedabf --hash=sha256:ef8de666d6179b009dce7bcb2ad4c4a779f113f12caf8dc77f0162c29d20490b --hash=sha256:efd387a49825780ff861998cd959767800d54f8308936b21025326de4b5a42b9 --hash=sha256:f0aa37f3c979cf2546b73e8222bbfa3dc07a641585340179d768068e3455e544 --hash=sha256:f4074c5a429281bf056ddd4c5d3b740ebca4d43ffffe2ef4bf4d2d05114299da --hash=sha256:f69a27e45c43520f5487f27627059b64aaf160415589230992cec34c5e18a509 --hash=sha256:fb707f3e15060adf5b7ada797624a6c6e0138e2a26baa089df64c68ee98e040f --hash=sha256:fcbe676a55d7445b22c10967bceaaf0ee69407fbe0ece4d032b6eb8d4565982a --hash=sha256:fdb20a30fe1175ecabed17cbf7812f7b804b8a315a25f24678bcdf120a90077f" - } - }, - "neuron_py_deps_311_ec2_metadata": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "ec2-metadata==2.14.0 --hash=sha256:b2f83381722072efa28f65dc37e726c0abd3a0532f227b3fa6a2ad68460c7feb --hash=sha256:fc4f9e893da1c049761221ec3d2208cce4872953b8960e1ed1f7120ff4ca3512" - } - }, - "neuron_py_deps_311_idna": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "idna==3.10 --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3" - } - }, - "neuron_py_deps_311_islpy": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "islpy==2023.2.5 --hash=sha256:06446395028b88701fd84b685b1c82b9de493728906f96f576a4dca69d12dd1f --hash=sha256:0c1689d9ef2ccfb3fcb3108a37c8fa21561dac3615ec05508021628a3197d81c --hash=sha256:0f723e6d8fda7608e6329ee9a786b2812afefec01f6f45e4ac812594608d2990 --hash=sha256:20484f2ef8d8f5827e18eaac064bfbd4d0e9cb877030ee1297a544bc80e126db --hash=sha256:21571052bac1a319ef4937886559a41c85eddfc29389c7f9be6d8ddaa08dbd4a --hash=sha256:24b469e1e390860cb78a13e5a933727fed42fd991eb6dbb758754410224a28a2 --hash=sha256:2a64abecd9a966e6223633315e20b3abd9eb1faa8887122c675a35a7f4885212 --hash=sha256:31c48b641824bd6b50e97659a6842c445feeb359115148c3debcbb862cb85c1f --hash=sha256:3ee7815b1d49ac1efca0a42a5c038f88e87be01b9ddf598aadea0ef84b930a46 --hash=sha256:3f50ec187bbd61cef4400af6885e65afd4092e3c4dc84b5d5c98554de6d06988 --hash=sha256:4e97e23ba815afa37b8919ebb0d1c7ec7a9cb2c1491ea7ba89a886a8d213169c --hash=sha256:58b5b86a9f5f58d0e8165c7e2215e7441eeea3466277e44f24788aef8b2b8311 --hash=sha256:662a1c6f83feb77397eaef0027c5a57b98e249ecc454df6131c8cfe1541964dc --hash=sha256:679431e17ad40d3c496d5fd3e9cbb824047775c00ade143dd59f848110b287f7 --hash=sha256:69a13fa79fe9e901376e2c75bc49770e9027c55d9699077f8417e5bbe9e8b137 --hash=sha256:6a336af9106eb92c3b7babe919d9e1e0a629143941d32d474e47cc55cf64f647 --hash=sha256:6ed6c49803e53db5b0dbba9ec2fd3f896a8b9ea68f4a1cd203e5e7490f8e1c8e --hash=sha256:773e85ba8768e14bd2b813d4a7e8e8140fe628cddddbd4fcb3452b023ed57f84 --hash=sha256:7acbb16a6f55a6020cfcbd5823bbd3dd36ca03ef0713f9f7a47404b0aad7cba4 --hash=sha256:80674c7a9aef8d2657872264dc59aeb66ce92067603ead43be51a56a6efc058a --hash=sha256:819b16c4add84b42f227b451c2f773b8ae06489313d482771fc54c8806c1a2af --hash=sha256:831b7a8f290af99e24addcb5cc5f4e73f7d047d50202ad84c7d208cadbe24f15 --hash=sha256:a66c7baf86d6fcef8e2170483f45a241dca0811e0c78cc71efa30bc74bfc6bc8 --hash=sha256:b27f1aa0ec0ab8182e47aaab4e363a42221ae00f9cba1f7cdde4d687120965f3 --hash=sha256:b3582a7d19ee47f29c8c6b40643f69e5a52c0f0489ed179408cfd4d9616210c9 --hash=sha256:b45c69990d0f7a9fbbba4a28b552c3b93c80a75d17b52032851f2dde13ebd661 --hash=sha256:c0d67b1b8644ac8cf5dee3b2efbf8029891f5db1a625ee5259e2fcf6c33a2d55 --hash=sha256:c2fdf13ae6465c511d75b42ed18b2289b49ecc7f1764e7fa24340b05d0392e80 --hash=sha256:c6665d90beb3f37b2dcb95570bf8700e58884c5ad03dd131568287a639c03a5e --hash=sha256:cea357471c345e804fa31c9f1eacfb3f67eda26b3e8fd417c33556822076565b --hash=sha256:d35750ea2a3964325a2e67196806cf02ed771c2d2b2d47fb2bf60f58bf933cc4 --hash=sha256:d5bedd79939dac4403b573abe5e13f6fc6a4502ebe8ed7de79b2fbfdd7856f3f --hash=sha256:d75b4521f3291b92fc6796ebb7eabf5c6e318d44a578390f678d7bfaedde13b8 --hash=sha256:d77acfe6653d044d14b8e93482e0282f42b947d40f5a98a13ee6b705936aaa18 --hash=sha256:dd74398baf36938abb7c2a9d9828cc1a2e549c6c80485cec8b06b29abfaad18f --hash=sha256:e33812a9922d967da814b17bc778a2c0dd1dabcd7ff685d86677bd745e8a3d65 --hash=sha256:e372461fb5ba22f9955cdf180d6187ac691f0ffa972874af181bf60501c738cd --hash=sha256:ea705d8b5244409fdc32affc689b010e88efbc8d378072bcba8edcf51c797de7 --hash=sha256:ed0c62e64d160735ed1a7fc314d4c0f501711c282bb84a00ceaf590b4c4bd61f --hash=sha256:eeec97588bd5282c983bd73f9010156e5e251e9e5307af547e2cb53b02d4639a" - } - }, - "neuron_py_deps_311_jmespath": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "jmespath==1.0.1 --hash=sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 --hash=sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" - } - }, - "neuron_py_deps_311_libneuronxla": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "libneuronxla==2.2.4410.0+835a67fb --hash=sha256:37e5f08482ef1a9a844c2894de6d91c03400a092764e54ab67612456be439f16" - } - }, - "neuron_py_deps_311_lockfile": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "lockfile==0.12.2 --hash=sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799 --hash=sha256:6c3cb24f344923d30b2785d5ad75182c8ea7ac1b6171b08657258ec7429d50fa" - } - }, - "neuron_py_deps_311_ml_dtypes": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "ml-dtypes==0.5.1 --hash=sha256:023ce2f502efd4d6c1e0472cc58ce3640d051d40e71e27386bed33901e201327 --hash=sha256:05f23447a1c20ddf4dc7c2c661aa9ed93fcb2658f1017c204d1e758714dc28a8 --hash=sha256:12651420130ee7cc13059fc56dac6ad300c3af3848b802d475148c9defd27c23 --hash=sha256:141b2ea2f20bb10802ddca55d91fe21231ef49715cfc971998e8f2a9838f3dbe --hash=sha256:15ad0f3b0323ce96c24637a88a6f44f6713c64032f27277b069f285c3cf66478 --hash=sha256:1b7fbe5571fdf28fd3aaab3ef4aafc847de9ebf263be959958c1ca58ec8eadf5 --hash=sha256:26ebcc69d7b779c8f129393e99732961b5cc33fcff84090451f448c89b0e01b4 --hash=sha256:6f462f5eca22fb66d7ff9c4744a3db4463af06c49816c4b6ac89b16bfcdc592e --hash=sha256:6f76232163b5b9c34291b54621ee60417601e2e4802a188a0ea7157cd9b323f4 --hash=sha256:7000b6e4d8ef07542c05044ec5d8bbae1df083b3f56822c3da63993a113e716f --hash=sha256:810512e2eccdfc3b41eefa3a27402371a3411453a1efc7e9c000318196140fed --hash=sha256:8f2c028954f16ede77902b223a8da2d9cbb3892375b85809a5c3cfb1587960c4 --hash=sha256:9626d0bca1fb387d5791ca36bacbba298c5ef554747b7ebeafefb4564fc83566 --hash=sha256:ac5b58559bb84a95848ed6984eb8013249f90b6bab62aa5acbad876e256002c9 --hash=sha256:ad4953c5eb9c25a56d11a913c2011d7e580a435ef5145f804d98efa14477d390 --hash=sha256:aefedc579ece2f8fb38f876aa7698204ee4c372d0e54f1c1ffa8ca580b54cc60 --hash=sha256:afb2009ac98da274e893e03162f6269398b2b00d947e7057ee2469a921d58135 --hash=sha256:b8a9d46b4df5ae2135a8e8e72b465448ebbc1559997f4f9304a9ecc3413efb5b --hash=sha256:bd73f51957949069573ff783563486339a9285d72e2f36c18e0c1aa9ca7eb190 --hash=sha256:bf9975bda82a99dc935f2ae4c83846d86df8fd6ba179614acac8e686910851da --hash=sha256:c09526488c3a9e8b7a23a388d4974b670a9a3dd40c5c8a61db5593ce9b725bab --hash=sha256:c9945669d3dadf8acb40ec2e57d38c985d8c285ea73af57fc5b09872c516106d --hash=sha256:d13755f8e8445b3870114e5b6240facaa7cb0c3361e54beba3e07fa912a6e12b --hash=sha256:fd918d4e6a4e0c110e2e05be7a7814d10dc1b95872accbf6512b80a109b71ae1" - } - }, - "neuron_py_deps_311_networkx": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "networkx==2.8.8 --hash=sha256:230d388117af870fce5647a3c52401fcf753e94720e6ea6b4197a5355648885e --hash=sha256:e435dfa75b1d7195c7b8378c3859f0445cd88c6b0375c181ed66823a9ceb7524" - } - }, - "neuron_py_deps_311_neuronx_cc": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "neuronx-cc==2.19.8089.0+8ab9f450 --hash=sha256:fbb05d8a0da8f54954c03ff2bea659f627309110c00efd54f523d877ddd6b88d" - } - }, - "neuron_py_deps_311_numpy": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "numpy==1.26.4 --hash=sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b --hash=sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818 --hash=sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20 --hash=sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0 --hash=sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010 --hash=sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a --hash=sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea --hash=sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c --hash=sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71 --hash=sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110 --hash=sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be --hash=sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a --hash=sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a --hash=sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5 --hash=sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed --hash=sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd --hash=sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c --hash=sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e --hash=sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0 --hash=sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c --hash=sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a --hash=sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b --hash=sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0 --hash=sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6 --hash=sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2 --hash=sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a --hash=sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30 --hash=sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218 --hash=sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5 --hash=sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07 --hash=sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2 --hash=sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4 --hash=sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764 --hash=sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef --hash=sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3 --hash=sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f" - } - }, - "neuron_py_deps_311_pgzip": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "pgzip==0.3.5 --hash=sha256:4e13ab66ecface5c51c5af51d8cd676aa51675cf85df000f501a86cf38c208c1 --hash=sha256:dd35510f59f6bd6b64e31c4baf90c10cdbb2775235fcc079b14b404fbd7f46bf" - } - }, - "neuron_py_deps_311_protobuf": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "protobuf==6.31.1 --hash=sha256:0414e3aa5a5f3ff423828e1e6a6e907d6c65c1d5b7e6e975793d5590bdeecc16 --hash=sha256:426f59d2964864a1a366254fa703b8632dcec0790d8862d30034d8245e1cd447 --hash=sha256:4ee898bf66f7a8b0bd21bce523814e6fbd8c6add948045ce958b73af7e8878c6 --hash=sha256:6f1227473dc43d44ed644425268eb7c2e488ae245d51c6866d19fe158e207402 --hash=sha256:720a6c7e6b77288b85063569baae8536671b39f15cc22037ec7045658d80489e --hash=sha256:7fa17d5a29c2e04b7d90e5e32388b8bfd0e7107cd8e616feef7ed3fa6bdab5c9 --hash=sha256:8764cf4587791e7564051b35524b72844f845ad0bb011704c3736cce762d8fe9 --hash=sha256:a40fc12b84c154884d7d4c4ebd675d5b3b5283e155f324049ae396b95ddebc39 --hash=sha256:d8cac4c982f0b957a4dc73a80e2ea24fab08e679c0de9deb835f4a12d69aca9a" - } - }, - "neuron_py_deps_311_psutil": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "psutil==7.0.0 --hash=sha256:101d71dc322e3cffd7cea0650b09b3d08b8e7c4109dd6809fe452dfd00e58b25 --hash=sha256:1e744154a6580bc968a0195fd25e80432d3afec619daf145b9e5ba16cc1d688e --hash=sha256:1fcee592b4c6f146991ca55919ea3d1f8926497a713ed7faaf8225e174581e91 --hash=sha256:39db632f6bb862eeccf56660871433e111b6ea58f2caea825571951d4b6aa3da --hash=sha256:4b1388a4f6875d7e2aff5c4ca1cc16c545ed41dd8bb596cefea80111db353a34 --hash=sha256:4cf3d4eb1aa9b348dec30105c55cd9b7d4629285735a102beb4441e38db90553 --hash=sha256:7be9c3eba38beccb6495ea33afd982a44074b78f28c434a1f51cc07fd315c456 --hash=sha256:84df4eb63e16849689f76b1ffcb36db7b8de703d1bc1fe41773db487621b6c17 --hash=sha256:a5f098451abc2828f7dc6b58d44b532b22f2088f4999a937557b603ce72b1993 --hash=sha256:ba3fcef7523064a6c9da440fc4d6bd07da93ac726b5733c29027d7dc95b39d99" - } - }, - "neuron_py_deps_311_python_daemon": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "python-daemon==3.1.2 --hash=sha256:b906833cef63502994ad48e2eab213259ed9bb18d54fa8774dcba2ff7864cec6 --hash=sha256:f7b04335adc473de877f5117e26d5f1142f4c9f7cd765408f0877757be5afbf4" - } - }, - "neuron_py_deps_311_python_dateutil": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "python-dateutil==2.9.0.post0 --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" - } - }, - "neuron_py_deps_311_requests": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "requests==2.32.4 --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422" - } - }, - "neuron_py_deps_311_requests_unixsocket": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "requests-unixsocket==0.4.1 --hash=sha256:60c4942e9dbecc2f64d611039fb1dfc25da382083c6434ac0316dca3ff908f4d --hash=sha256:b2596158c356ecee68d27ba469a52211230ac6fb0cde8b66afb19f0ed47a1995" - } - }, - "neuron_py_deps_311_s3transfer": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "s3transfer==0.13.1 --hash=sha256:a981aa7429be23fe6dfc13e80e4020057cbab622b08c0315288758d67cabc724 --hash=sha256:c3fdba22ba1bd367922f27ec8032d6a1cf5f10c934fb5d68cf60fd5a23d936cf" - } - }, - "neuron_py_deps_311_scipy": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "scipy==1.12.0 --hash=sha256:196ebad3a4882081f62a5bf4aeb7326aa34b110e533aab23e4374fcccb0890dc --hash=sha256:408c68423f9de16cb9e602528be4ce0d6312b05001f3de61fe9ec8b1263cad08 --hash=sha256:4bf5abab8a36d20193c698b0f1fc282c1d083c94723902c447e5d2f1780936a3 --hash=sha256:4c1020cad92772bf44b8e4cdabc1df5d87376cb219742549ef69fc9fd86282dd --hash=sha256:5adfad5dbf0163397beb4aca679187d24aec085343755fcdbdeb32b3679f254c --hash=sha256:5e32847e08da8d895ce09d108a494d9eb78974cf6de23063f93306a3e419960c --hash=sha256:6546dc2c11a9df6926afcbdd8a3edec28566e4e785b915e849348c6dd9f3f490 --hash=sha256:730badef9b827b368f351eacae2e82da414e13cf8bd5051b4bdfd720271a5371 --hash=sha256:75ea2a144096b5e39402e2ff53a36fecfd3b960d786b7efd3c180e29c39e53f2 --hash=sha256:78e4402e140879387187f7f25d91cc592b3501a2e51dfb320f48dfb73565f10b --hash=sha256:8b8066bce124ee5531d12a74b617d9ac0ea59245246410e19bca549656d9a40a --hash=sha256:8bee4993817e204d761dba10dbab0774ba5a8612e57e81319ea04d84945375ba --hash=sha256:913d6e7956c3a671de3b05ccb66b11bc293f56bfdef040583a7221d9e22a2e35 --hash=sha256:95e5c750d55cf518c398a8240571b0e0782c2d5a703250872f36eaf737751338 --hash=sha256:9c39f92041f490422924dfdb782527a4abddf4707616e07b021de33467f917bc --hash=sha256:a24024d45ce9a675c1fb8494e8e5244efea1c7a09c60beb1eeb80373d0fecc70 --hash=sha256:a7ebda398f86e56178c2fa94cad15bf457a218a54a35c2a7b4490b9f9cb2676c --hash=sha256:b360f1b6b2f742781299514e99ff560d1fe9bd1bff2712894b52abe528d1fd1e --hash=sha256:bba1b0c7256ad75401c73e4b3cf09d1f176e9bd4248f0d3112170fb2ec4db067 --hash=sha256:c3003652496f6e7c387b1cf63f4bb720951cfa18907e998ea551e6de51a04467 --hash=sha256:e53958531a7c695ff66c2e7bb7b79560ffdc562e2051644c5576c39ff8efb563 --hash=sha256:e646d8571804a304e1da01040d21577685ce8e2db08ac58e543eaca063453e1c --hash=sha256:e7e76cc48638228212c747ada851ef355c2bb5e7f939e10952bc504c11f4e372 --hash=sha256:f5f00ebaf8de24d14b8449981a2842d404152774c1a1d880c901bf454cb8e2a1 --hash=sha256:f7ce148dffcd64ade37b2df9315541f9adad6efcaa86866ee7dd5db0c8f041c3" - } - }, - "neuron_py_deps_311_six": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "six==1.17.0 --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81" - } - }, - "neuron_py_deps_311_tqdm": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "tqdm==4.67.1 --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2" - } - }, - "neuron_py_deps_311_typing_extensions": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "typing-extensions==4.14.1 --hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 --hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76" - } - }, - "neuron_py_deps_311_urllib3": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "urllib3==2.5.0 --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc" - } - }, - "pip_deps_310_numpy": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@pip_deps//{name}:{target}", - "python_interpreter_target": "@@rules_python++python+python_3_10_host//:python", - "repo": "pip_deps_310", - "requirement": "numpy<=1.26.1" - } - }, - "pip_deps_310_setuptools": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@pip_deps//{name}:{target}", - "python_interpreter_target": "@@rules_python++python+python_3_10_host//:python", - "repo": "pip_deps_310", - "requirement": "setuptools<=70.3.0" - } - }, - "pip_deps_311_numpy": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@pip_deps//{name}:{target}", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "pip_deps_311", - "requirement": "numpy<=1.26.1" - } - }, - "pip_deps_311_setuptools": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@pip_deps//{name}:{target}", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "pip_deps_311", - "requirement": "setuptools<=70.3.0" - } - }, - "pip_deps_312_numpy": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@pip_deps//{name}:{target}", - "python_interpreter_target": "@@rules_python++python+python_3_12_host//:python", - "repo": "pip_deps_312", - "requirement": "numpy<=1.26.1" - } - }, - "pip_deps_312_setuptools": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@pip_deps//{name}:{target}", - "python_interpreter_target": "@@rules_python++python+python_3_12_host//:python", - "repo": "pip_deps_312", - "requirement": "setuptools<=70.3.0" - } - }, - "pip_deps_38_numpy": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@pip_deps//{name}:{target}", - "python_interpreter_target": "@@rules_python++python+python_3_8_host//:python", - "repo": "pip_deps_38", - "requirement": "numpy<=1.26.1" - } - }, - "pip_deps_38_setuptools": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@pip_deps//{name}:{target}", - "python_interpreter_target": "@@rules_python++python+python_3_8_host//:python", - "repo": "pip_deps_38", - "requirement": "setuptools<=70.3.0" - } - }, - "pip_deps_39_numpy": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@pip_deps//{name}:{target}", - "python_interpreter_target": "@@rules_python++python+python_3_9_host//:python", - "repo": "pip_deps_39", - "requirement": "numpy<=1.26.1" - } - }, - "pip_deps_39_setuptools": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@pip_deps//{name}:{target}", - "python_interpreter_target": "@@rules_python++python+python_3_9_host//:python", - "repo": "pip_deps_39", - "requirement": "setuptools<=70.3.0" - } - }, - "rules_fuzzing_py_deps_310_absl_py": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_fuzzing_py_deps//{name}:{target}", - "extra_pip_args": [ - "--require-hashes" - ], - "python_interpreter_target": "@@rules_python++python+python_3_10_host//:python", - "repo": "rules_fuzzing_py_deps_310", - "requirement": "absl-py==2.0.0 --hash=sha256:9a28abb62774ae4e8edbe2dd4c49ffcd45a6a848952a5eccc6a49f3f0fc1e2f3" - } - }, - "rules_fuzzing_py_deps_310_six": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_fuzzing_py_deps//{name}:{target}", - "extra_pip_args": [ - "--require-hashes" - ], - "python_interpreter_target": "@@rules_python++python+python_3_10_host//:python", - "repo": "rules_fuzzing_py_deps_310", - "requirement": "six==1.16.0 --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - } - }, - "rules_fuzzing_py_deps_311_absl_py": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_fuzzing_py_deps//{name}:{target}", - "extra_pip_args": [ - "--require-hashes" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_fuzzing_py_deps_311", - "requirement": "absl-py==2.0.0 --hash=sha256:9a28abb62774ae4e8edbe2dd4c49ffcd45a6a848952a5eccc6a49f3f0fc1e2f3" - } - }, - "rules_fuzzing_py_deps_311_six": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_fuzzing_py_deps//{name}:{target}", - "extra_pip_args": [ - "--require-hashes" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_fuzzing_py_deps_311", - "requirement": "six==1.16.0 --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - } - }, - "rules_fuzzing_py_deps_312_absl_py": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_fuzzing_py_deps//{name}:{target}", - "extra_pip_args": [ - "--require-hashes" - ], - "python_interpreter_target": "@@rules_python++python+python_3_12_host//:python", - "repo": "rules_fuzzing_py_deps_312", - "requirement": "absl-py==2.0.0 --hash=sha256:9a28abb62774ae4e8edbe2dd4c49ffcd45a6a848952a5eccc6a49f3f0fc1e2f3" - } - }, - "rules_fuzzing_py_deps_312_six": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_fuzzing_py_deps//{name}:{target}", - "extra_pip_args": [ - "--require-hashes" - ], - "python_interpreter_target": "@@rules_python++python+python_3_12_host//:python", - "repo": "rules_fuzzing_py_deps_312", - "requirement": "six==1.16.0 --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - } - }, - "rules_fuzzing_py_deps_38_absl_py": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_fuzzing_py_deps//{name}:{target}", - "extra_pip_args": [ - "--require-hashes" - ], - "python_interpreter_target": "@@rules_python++python+python_3_8_host//:python", - "repo": "rules_fuzzing_py_deps_38", - "requirement": "absl-py==2.0.0 --hash=sha256:9a28abb62774ae4e8edbe2dd4c49ffcd45a6a848952a5eccc6a49f3f0fc1e2f3" - } - }, - "rules_fuzzing_py_deps_38_six": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_fuzzing_py_deps//{name}:{target}", - "extra_pip_args": [ - "--require-hashes" - ], - "python_interpreter_target": "@@rules_python++python+python_3_8_host//:python", - "repo": "rules_fuzzing_py_deps_38", - "requirement": "six==1.16.0 --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - } - }, - "rules_fuzzing_py_deps_39_absl_py": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_fuzzing_py_deps//{name}:{target}", - "extra_pip_args": [ - "--require-hashes" - ], - "python_interpreter_target": "@@rules_python++python+python_3_9_host//:python", - "repo": "rules_fuzzing_py_deps_39", - "requirement": "absl-py==2.0.0 --hash=sha256:9a28abb62774ae4e8edbe2dd4c49ffcd45a6a848952a5eccc6a49f3f0fc1e2f3" - } - }, - "rules_fuzzing_py_deps_39_six": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_fuzzing_py_deps//{name}:{target}", - "extra_pip_args": [ - "--require-hashes" - ], - "python_interpreter_target": "@@rules_python++python+python_3_9_host//:python", - "repo": "rules_fuzzing_py_deps_39", - "requirement": "six==1.16.0 --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - } - }, - "rules_python_publish_deps_311_backports_tarfile_py3_none_any_77e284d7": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "backports.tarfile-1.2.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "backports-tarfile==1.2.0", - "sha256": "77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34", - "urls": [ - "https://files.pythonhosted.org/packages/b9/fa/123043af240e49752f1c4bd24da5053b6bd00cad78c2be53c0d1e8b975bc/backports.tarfile-1.2.0-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_backports_tarfile_sdist_d75e02c2": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "backports_tarfile-1.2.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "backports-tarfile==1.2.0", - "sha256": "d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991", - "urls": [ - "https://files.pythonhosted.org/packages/86/72/cd9b395f25e290e633655a100af28cb253e4393396264a98bd5f5951d50f/backports_tarfile-1.2.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_certifi_py3_none_any_922820b5": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "certifi-2024.8.30-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "certifi==2024.8.30", - "sha256": "922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8", - "urls": [ - "https://files.pythonhosted.org/packages/12/90/3c9ff0512038035f59d279fddeb79f5f1eccd8859f06d6163c58798b9487/certifi-2024.8.30-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_certifi_sdist_bec941d2": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "certifi-2024.8.30.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "certifi==2024.8.30", - "sha256": "bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9", - "urls": [ - "https://files.pythonhosted.org/packages/b0/ee/9b19140fe824b367c04c5e1b369942dd754c4c5462d5674002f75c4dedc1/certifi-2024.8.30.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_cffi_cp311_cp311_manylinux_2_17_aarch64_a1ed2dd2": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cffi==1.17.1", - "sha256": "a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41", - "urls": [ - "https://files.pythonhosted.org/packages/2e/ea/70ce63780f096e16ce8588efe039d3c4f91deb1dc01e9c73a287939c79a6/cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" - ] - } - }, - "rules_python_publish_deps_311_cffi_cp311_cp311_manylinux_2_17_s390x_a24ed04c": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cffi==1.17.1", - "sha256": "a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6", - "urls": [ - "https://files.pythonhosted.org/packages/62/12/ce8710b5b8affbcdd5c6e367217c242524ad17a02fe5beec3ee339f69f85/cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl" - ] - } - }, - "rules_python_publish_deps_311_cffi_cp311_cp311_manylinux_2_17_x86_64_610faea7": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cffi==1.17.1", - "sha256": "610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d", - "urls": [ - "https://files.pythonhosted.org/packages/ff/6b/d45873c5e0242196f042d555526f92aa9e0c32355a1be1ff8c27f077fd37/cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - ] - } - }, - "rules_python_publish_deps_311_cffi_cp311_cp311_musllinux_1_1_aarch64_a9b15d49": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cffi==1.17.1", - "sha256": "a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6", - "urls": [ - "https://files.pythonhosted.org/packages/1a/52/d9a0e523a572fbccf2955f5abe883cfa8bcc570d7faeee06336fbd50c9fc/cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl" - ] - } - }, - "rules_python_publish_deps_311_cffi_cp311_cp311_musllinux_1_1_x86_64_fc48c783": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cffi==1.17.1", - "sha256": "fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b", - "urls": [ - "https://files.pythonhosted.org/packages/f8/4a/34599cac7dfcd888ff54e801afe06a19c17787dfd94495ab0c8d35fe99fb/cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl" - ] - } - }, - "rules_python_publish_deps_311_cffi_sdist_1c39c601": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "cffi-1.17.1.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cffi==1.17.1", - "sha256": "1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", - "urls": [ - "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_charset_normalizer_cp311_cp311_macosx_10_9_universal2_0d99dd8f": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_universal2.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "charset-normalizer==3.4.0", - "sha256": "0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c", - "urls": [ - "https://files.pythonhosted.org/packages/9c/61/73589dcc7a719582bf56aae309b6103d2762b526bffe189d635a7fcfd998/charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_universal2.whl" - ] - } - }, - "rules_python_publish_deps_311_charset_normalizer_cp311_cp311_macosx_10_9_x86_64_c57516e5": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "charset-normalizer==3.4.0", - "sha256": "c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944", - "urls": [ - "https://files.pythonhosted.org/packages/77/d5/8c982d58144de49f59571f940e329ad6e8615e1e82ef84584c5eeb5e1d72/charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl" - ] - } - }, - "rules_python_publish_deps_311_charset_normalizer_cp311_cp311_macosx_11_0_arm64_6dba5d19": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "charset_normalizer-3.4.0-cp311-cp311-macosx_11_0_arm64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "charset-normalizer==3.4.0", - "sha256": "6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee", - "urls": [ - "https://files.pythonhosted.org/packages/bf/19/411a64f01ee971bed3231111b69eb56f9331a769072de479eae7de52296d/charset_normalizer-3.4.0-cp311-cp311-macosx_11_0_arm64.whl" - ] - } - }, - "rules_python_publish_deps_311_charset_normalizer_cp311_cp311_manylinux_2_17_aarch64_bf4475b8": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "charset-normalizer==3.4.0", - "sha256": "bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c", - "urls": [ - "https://files.pythonhosted.org/packages/4c/92/97509850f0d00e9f14a46bc751daabd0ad7765cff29cdfb66c68b6dad57f/charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" - ] - } - }, - "rules_python_publish_deps_311_charset_normalizer_cp311_cp311_manylinux_2_17_s390x_8ff4e7cd": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "charset-normalizer==3.4.0", - "sha256": "8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea", - "urls": [ - "https://files.pythonhosted.org/packages/13/bc/87c2c9f2c144bedfa62f894c3007cd4530ba4b5351acb10dc786428a50f0/charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl" - ] - } - }, - "rules_python_publish_deps_311_charset_normalizer_cp311_cp311_manylinux_2_17_x86_64_3710a975": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "charset-normalizer==3.4.0", - "sha256": "3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc", - "urls": [ - "https://files.pythonhosted.org/packages/eb/5b/6f10bad0f6461fa272bfbbdf5d0023b5fb9bc6217c92bf068fa5a99820f5/charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - ] - } - }, - "rules_python_publish_deps_311_charset_normalizer_cp311_cp311_musllinux_1_2_aarch64_47334db7": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "charset-normalizer==3.4.0", - "sha256": "47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594", - "urls": [ - "https://files.pythonhosted.org/packages/d7/a1/493919799446464ed0299c8eef3c3fad0daf1c3cd48bff9263c731b0d9e2/charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl" - ] - } - }, - "rules_python_publish_deps_311_charset_normalizer_cp311_cp311_musllinux_1_2_s390x_63bc5c4a": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "charset-normalizer==3.4.0", - "sha256": "63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129", - "urls": [ - "https://files.pythonhosted.org/packages/8d/c9/27e41d481557be53d51e60750b85aa40eaf52b841946b3cdeff363105737/charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl" - ] - } - }, - "rules_python_publish_deps_311_charset_normalizer_cp311_cp311_musllinux_1_2_x86_64_bcb4f8ea": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "charset-normalizer==3.4.0", - "sha256": "bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236", - "urls": [ - "https://files.pythonhosted.org/packages/ee/44/4f62042ca8cdc0cabf87c0fc00ae27cd8b53ab68be3605ba6d071f742ad3/charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl" - ] - } - }, - "rules_python_publish_deps_311_charset_normalizer_cp311_cp311_win_amd64_cee4373f": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "charset_normalizer-3.4.0-cp311-cp311-win_amd64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "charset-normalizer==3.4.0", - "sha256": "cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27", - "urls": [ - "https://files.pythonhosted.org/packages/0b/6e/b13bd47fa9023b3699e94abf565b5a2f0b0be6e9ddac9812182596ee62e4/charset_normalizer-3.4.0-cp311-cp311-win_amd64.whl" - ] - } - }, - "rules_python_publish_deps_311_charset_normalizer_py3_none_any_fe9f97fe": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "charset_normalizer-3.4.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "charset-normalizer==3.4.0", - "sha256": "fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079", - "urls": [ - "https://files.pythonhosted.org/packages/bf/9b/08c0432272d77b04803958a4598a51e2a4b51c06640af8b8f0f908c18bf2/charset_normalizer-3.4.0-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_charset_normalizer_sdist_223217c3": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "charset_normalizer-3.4.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "charset-normalizer==3.4.0", - "sha256": "223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e", - "urls": [ - "https://files.pythonhosted.org/packages/f2/4f/e1808dc01273379acc506d18f1504eb2d299bd4131743b9fc54d7be4df1e/charset_normalizer-3.4.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_cryptography_cp39_abi3_manylinux_2_17_aarch64_846da004": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "cryptography-43.0.3-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cryptography==43.0.3", - "sha256": "846da004a5804145a5f441b8530b4bf35afbf7da70f82409f151695b127213d5", - "urls": [ - "https://files.pythonhosted.org/packages/2f/78/55356eb9075d0be6e81b59f45c7b48df87f76a20e73893872170471f3ee8/cryptography-43.0.3-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" - ] - } - }, - "rules_python_publish_deps_311_cryptography_cp39_abi3_manylinux_2_17_x86_64_0f996e72": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "cryptography-43.0.3-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cryptography==43.0.3", - "sha256": "0f996e7268af62598f2fc1204afa98a3b5712313a55c4c9d434aef49cadc91d4", - "urls": [ - "https://files.pythonhosted.org/packages/2a/2c/488776a3dc843f95f86d2f957ca0fc3407d0242b50bede7fad1e339be03f/cryptography-43.0.3-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - ] - } - }, - "rules_python_publish_deps_311_cryptography_cp39_abi3_manylinux_2_28_aarch64_f7b178f1": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "cryptography-43.0.3-cp39-abi3-manylinux_2_28_aarch64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cryptography==43.0.3", - "sha256": "f7b178f11ed3664fd0e995a47ed2b5ff0a12d893e41dd0494f406d1cf555cab7", - "urls": [ - "https://files.pythonhosted.org/packages/7c/04/2345ca92f7a22f601a9c62961741ef7dd0127c39f7310dffa0041c80f16f/cryptography-43.0.3-cp39-abi3-manylinux_2_28_aarch64.whl" - ] - } - }, - "rules_python_publish_deps_311_cryptography_cp39_abi3_manylinux_2_28_x86_64_c2e6fc39": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "cryptography-43.0.3-cp39-abi3-manylinux_2_28_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cryptography==43.0.3", - "sha256": "c2e6fc39c4ab499049df3bdf567f768a723a5e8464816e8f009f121a5a9f4405", - "urls": [ - "https://files.pythonhosted.org/packages/ac/25/e715fa0bc24ac2114ed69da33adf451a38abb6f3f24ec207908112e9ba53/cryptography-43.0.3-cp39-abi3-manylinux_2_28_x86_64.whl" - ] - } - }, - "rules_python_publish_deps_311_cryptography_cp39_abi3_musllinux_1_2_aarch64_e1be4655": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "cryptography-43.0.3-cp39-abi3-musllinux_1_2_aarch64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cryptography==43.0.3", - "sha256": "e1be4655c7ef6e1bbe6b5d0403526601323420bcf414598955968c9ef3eb7d16", - "urls": [ - "https://files.pythonhosted.org/packages/21/ce/b9c9ff56c7164d8e2edfb6c9305045fbc0df4508ccfdb13ee66eb8c95b0e/cryptography-43.0.3-cp39-abi3-musllinux_1_2_aarch64.whl" - ] - } - }, - "rules_python_publish_deps_311_cryptography_cp39_abi3_musllinux_1_2_x86_64_df6b6c6d": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "cryptography-43.0.3-cp39-abi3-musllinux_1_2_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cryptography==43.0.3", - "sha256": "df6b6c6d742395dd77a23ea3728ab62f98379eff8fb61be2744d4679ab678f73", - "urls": [ - "https://files.pythonhosted.org/packages/2a/33/b3682992ab2e9476b9c81fff22f02c8b0a1e6e1d49ee1750a67d85fd7ed2/cryptography-43.0.3-cp39-abi3-musllinux_1_2_x86_64.whl" - ] - } - }, - "rules_python_publish_deps_311_cryptography_sdist_315b9001": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "cryptography-43.0.3.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cryptography==43.0.3", - "sha256": "315b9001266a492a6ff443b61238f956b214dbec9910a081ba5b6646a055a805", - "urls": [ - "https://files.pythonhosted.org/packages/0d/05/07b55d1fa21ac18c3a8c79f764e2514e6f6a9698f1be44994f5adf0d29db/cryptography-43.0.3.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_docutils_py3_none_any_dafca5b9": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "docutils-0.21.2-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "docutils==0.21.2", - "sha256": "dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2", - "urls": [ - "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_docutils_sdist_3a6b1873": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "docutils-0.21.2.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "docutils==0.21.2", - "sha256": "3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f", - "urls": [ - "https://files.pythonhosted.org/packages/ae/ed/aefcc8cd0ba62a0560c3c18c33925362d46c6075480bfa4df87b28e169a9/docutils-0.21.2.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_idna_py3_none_any_946d195a": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "idna-3.10-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "idna==3.10", - "sha256": "946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", - "urls": [ - "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_idna_sdist_12f65c9b": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "idna-3.10.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "idna==3.10", - "sha256": "12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", - "urls": [ - "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_importlib_metadata_py3_none_any_45e54197": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "importlib_metadata-8.5.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "importlib-metadata==8.5.0", - "sha256": "45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b", - "urls": [ - "https://files.pythonhosted.org/packages/a0/d9/a1e041c5e7caa9a05c925f4bdbdfb7f006d1f74996af53467bc394c97be7/importlib_metadata-8.5.0-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_importlib_metadata_sdist_71522656": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "importlib_metadata-8.5.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "importlib-metadata==8.5.0", - "sha256": "71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7", - "urls": [ - "https://files.pythonhosted.org/packages/cd/12/33e59336dca5be0c398a7482335911a33aa0e20776128f038019f1a95f1b/importlib_metadata-8.5.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_jaraco_classes_py3_none_any_f662826b": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "jaraco.classes-3.4.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "jaraco-classes==3.4.0", - "sha256": "f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790", - "urls": [ - "https://files.pythonhosted.org/packages/7f/66/b15ce62552d84bbfcec9a4873ab79d993a1dd4edb922cbfccae192bd5b5f/jaraco.classes-3.4.0-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_jaraco_classes_sdist_47a024b5": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "jaraco.classes-3.4.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "jaraco-classes==3.4.0", - "sha256": "47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd", - "urls": [ - "https://files.pythonhosted.org/packages/06/c0/ed4a27bc5571b99e3cff68f8a9fa5b56ff7df1c2251cc715a652ddd26402/jaraco.classes-3.4.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_jaraco_context_py3_none_any_f797fc48": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "jaraco.context-6.0.1-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "jaraco-context==6.0.1", - "sha256": "f797fc481b490edb305122c9181830a3a5b76d84ef6d1aef2fb9b47ab956f9e4", - "urls": [ - "https://files.pythonhosted.org/packages/ff/db/0c52c4cf5e4bd9f5d7135ec7669a3a767af21b3a308e1ed3674881e52b62/jaraco.context-6.0.1-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_jaraco_context_sdist_9bae4ea5": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "jaraco_context-6.0.1.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "jaraco-context==6.0.1", - "sha256": "9bae4ea555cf0b14938dc0aee7c9f32ed303aa20a3b73e7dc80111628792d1b3", - "urls": [ - "https://files.pythonhosted.org/packages/df/ad/f3777b81bf0b6e7bc7514a1656d3e637b2e8e15fab2ce3235730b3e7a4e6/jaraco_context-6.0.1.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_jaraco_functools_py3_none_any_ad159f13": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "jaraco.functools-4.1.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "jaraco-functools==4.1.0", - "sha256": "ad159f13428bc4acbf5541ad6dec511f91573b90fba04df61dafa2a1231cf649", - "urls": [ - "https://files.pythonhosted.org/packages/9f/4f/24b319316142c44283d7540e76c7b5a6dbd5db623abd86bb7b3491c21018/jaraco.functools-4.1.0-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_jaraco_functools_sdist_70f7e0e2": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "jaraco_functools-4.1.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "jaraco-functools==4.1.0", - "sha256": "70f7e0e2ae076498e212562325e805204fc092d7b4c17e0e86c959e249701a9d", - "urls": [ - "https://files.pythonhosted.org/packages/ab/23/9894b3df5d0a6eb44611c36aec777823fc2e07740dabbd0b810e19594013/jaraco_functools-4.1.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_jeepney_py3_none_any_c0a454ad": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "jeepney-0.8.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "jeepney==0.8.0", - "sha256": "c0a454ad016ca575060802ee4d590dd912e35c122fa04e70306de3d076cce755", - "urls": [ - "https://files.pythonhosted.org/packages/ae/72/2a1e2290f1ab1e06f71f3d0f1646c9e4634e70e1d37491535e19266e8dc9/jeepney-0.8.0-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_jeepney_sdist_5efe48d2": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "jeepney-0.8.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "jeepney==0.8.0", - "sha256": "5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806", - "urls": [ - "https://files.pythonhosted.org/packages/d6/f4/154cf374c2daf2020e05c3c6a03c91348d59b23c5366e968feb198306fdf/jeepney-0.8.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_keyring_py3_none_any_5426f817": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "keyring-25.4.1-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "keyring==25.4.1", - "sha256": "5426f817cf7f6f007ba5ec722b1bcad95a75b27d780343772ad76b17cb47b0bf", - "urls": [ - "https://files.pythonhosted.org/packages/83/25/e6d59e5f0a0508d0dca8bb98c7f7fd3772fc943ac3f53d5ab18a218d32c0/keyring-25.4.1-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_keyring_sdist_b07ebc55": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "keyring-25.4.1.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "keyring==25.4.1", - "sha256": "b07ebc55f3e8ed86ac81dd31ef14e81ace9dd9c3d4b5d77a6e9a2016d0d71a1b", - "urls": [ - "https://files.pythonhosted.org/packages/a5/1c/2bdbcfd5d59dc6274ffb175bc29aa07ecbfab196830e0cfbde7bd861a2ea/keyring-25.4.1.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_markdown_it_py_py3_none_any_35521684": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "markdown_it_py-3.0.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "markdown-it-py==3.0.0", - "sha256": "355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", - "urls": [ - "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_markdown_it_py_sdist_e3f60a94": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "markdown-it-py-3.0.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "markdown-it-py==3.0.0", - "sha256": "e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb", - "urls": [ - "https://files.pythonhosted.org/packages/38/71/3b932df36c1a044d397a1f92d1cf91ee0a503d91e470cbd670aa66b07ed0/markdown-it-py-3.0.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_mdurl_py3_none_any_84008a41": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "mdurl-0.1.2-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "mdurl==0.1.2", - "sha256": "84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", - "urls": [ - "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_mdurl_sdist_bb413d29": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "mdurl-0.1.2.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "mdurl==0.1.2", - "sha256": "bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", - "urls": [ - "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_more_itertools_py3_none_any_037b0d32": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "more_itertools-10.5.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "more-itertools==10.5.0", - "sha256": "037b0d3203ce90cca8ab1defbbdac29d5f993fc20131f3664dc8d6acfa872aef", - "urls": [ - "https://files.pythonhosted.org/packages/48/7e/3a64597054a70f7c86eb0a7d4fc315b8c1ab932f64883a297bdffeb5f967/more_itertools-10.5.0-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_more_itertools_sdist_5482bfef": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "more-itertools-10.5.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "more-itertools==10.5.0", - "sha256": "5482bfef7849c25dc3c6dd53a6173ae4795da2a41a80faea6700d9f5846c5da6", - "urls": [ - "https://files.pythonhosted.org/packages/51/78/65922308c4248e0eb08ebcbe67c95d48615cc6f27854b6f2e57143e9178f/more-itertools-10.5.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_nh3_cp37_abi3_macosx_10_12_x86_64_14c5a72e": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "nh3-0.2.18-cp37-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "nh3==0.2.18", - "sha256": "14c5a72e9fe82aea5fe3072116ad4661af5cf8e8ff8fc5ad3450f123e4925e86", - "urls": [ - "https://files.pythonhosted.org/packages/b3/89/1daff5d9ba5a95a157c092c7c5f39b8dd2b1ddb4559966f808d31cfb67e0/nh3-0.2.18-cp37-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl" - ] - } - }, - "rules_python_publish_deps_311_nh3_cp37_abi3_macosx_10_12_x86_64_7b7c2a3c": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "nh3-0.2.18-cp37-abi3-macosx_10_12_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "nh3==0.2.18", - "sha256": "7b7c2a3c9eb1a827d42539aa64091640bd275b81e097cd1d8d82ef91ffa2e811", - "urls": [ - "https://files.pythonhosted.org/packages/2c/b6/42fc3c69cabf86b6b81e4c051a9b6e249c5ba9f8155590222c2622961f58/nh3-0.2.18-cp37-abi3-macosx_10_12_x86_64.whl" - ] - } - }, - "rules_python_publish_deps_311_nh3_cp37_abi3_manylinux_2_17_aarch64_42c64511": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "nh3-0.2.18-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "nh3==0.2.18", - "sha256": "42c64511469005058cd17cc1537578eac40ae9f7200bedcfd1fc1a05f4f8c200", - "urls": [ - "https://files.pythonhosted.org/packages/45/b9/833f385403abaf0023c6547389ec7a7acf141ddd9d1f21573723a6eab39a/nh3-0.2.18-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" - ] - } - }, - "rules_python_publish_deps_311_nh3_cp37_abi3_manylinux_2_17_armv7l_0411beb0": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "nh3-0.2.18-cp37-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "nh3==0.2.18", - "sha256": "0411beb0589eacb6734f28d5497ca2ed379eafab8ad8c84b31bb5c34072b7164", - "urls": [ - "https://files.pythonhosted.org/packages/05/2b/85977d9e11713b5747595ee61f381bc820749daf83f07b90b6c9964cf932/nh3-0.2.18-cp37-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl" - ] - } - }, - "rules_python_publish_deps_311_nh3_cp37_abi3_manylinux_2_17_ppc64_5f36b271": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "nh3-0.2.18-cp37-abi3-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "nh3==0.2.18", - "sha256": "5f36b271dae35c465ef5e9090e1fdaba4a60a56f0bb0ba03e0932a66f28b9189", - "urls": [ - "https://files.pythonhosted.org/packages/72/f2/5c894d5265ab80a97c68ca36f25c8f6f0308abac649aaf152b74e7e854a8/nh3-0.2.18-cp37-abi3-manylinux_2_17_ppc64.manylinux2014_ppc64.whl" - ] - } - }, - "rules_python_publish_deps_311_nh3_cp37_abi3_manylinux_2_17_s390x_19aaba96": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "nh3-0.2.18-cp37-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "nh3==0.2.18", - "sha256": "19aaba96e0f795bd0a6c56291495ff59364f4300d4a39b29a0abc9cb3774a84b", - "urls": [ - "https://files.pythonhosted.org/packages/c2/a8/3bb02d0c60a03ad3a112b76c46971e9480efa98a8946677b5a59f60130ca/nh3-0.2.18-cp37-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl" - ] - } - }, - "rules_python_publish_deps_311_nh3_cp37_abi3_manylinux_2_17_x86_64_de3ceed6": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "nh3-0.2.18-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "nh3==0.2.18", - "sha256": "de3ceed6e661954871d6cd78b410213bdcb136f79aafe22aa7182e028b8c7307", - "urls": [ - "https://files.pythonhosted.org/packages/1b/63/6ab90d0e5225ab9780f6c9fb52254fa36b52bb7c188df9201d05b647e5e1/nh3-0.2.18-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - ] - } - }, - "rules_python_publish_deps_311_nh3_cp37_abi3_musllinux_1_2_aarch64_f0eca9ca": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "nh3-0.2.18-cp37-abi3-musllinux_1_2_aarch64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "nh3==0.2.18", - "sha256": "f0eca9ca8628dbb4e916ae2491d72957fdd35f7a5d326b7032a345f111ac07fe", - "urls": [ - "https://files.pythonhosted.org/packages/a3/da/0c4e282bc3cff4a0adf37005fa1fb42257673fbc1bbf7d1ff639ec3d255a/nh3-0.2.18-cp37-abi3-musllinux_1_2_aarch64.whl" - ] - } - }, - "rules_python_publish_deps_311_nh3_cp37_abi3_musllinux_1_2_armv7l_3a157ab1": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "nh3-0.2.18-cp37-abi3-musllinux_1_2_armv7l.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "nh3==0.2.18", - "sha256": "3a157ab149e591bb638a55c8c6bcb8cdb559c8b12c13a8affaba6cedfe51713a", - "urls": [ - "https://files.pythonhosted.org/packages/de/81/c291231463d21da5f8bba82c8167a6d6893cc5419b0639801ee5d3aeb8a9/nh3-0.2.18-cp37-abi3-musllinux_1_2_armv7l.whl" - ] - } - }, - "rules_python_publish_deps_311_nh3_cp37_abi3_musllinux_1_2_x86_64_36c95d4b": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "nh3-0.2.18-cp37-abi3-musllinux_1_2_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "nh3==0.2.18", - "sha256": "36c95d4b70530b320b365659bb5034341316e6a9b30f0b25fa9c9eff4c27a204", - "urls": [ - "https://files.pythonhosted.org/packages/eb/61/73a007c74c37895fdf66e0edcd881f5eaa17a348ff02f4bb4bc906d61085/nh3-0.2.18-cp37-abi3-musllinux_1_2_x86_64.whl" - ] - } - }, - "rules_python_publish_deps_311_nh3_cp37_abi3_win_amd64_8ce0f819": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "nh3-0.2.18-cp37-abi3-win_amd64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "nh3==0.2.18", - "sha256": "8ce0f819d2f1933953fca255db2471ad58184a60508f03e6285e5114b6254844", - "urls": [ - "https://files.pythonhosted.org/packages/26/8d/53c5b19c4999bdc6ba95f246f4ef35ca83d7d7423e5e38be43ad66544e5d/nh3-0.2.18-cp37-abi3-win_amd64.whl" - ] - } - }, - "rules_python_publish_deps_311_nh3_sdist_94a16692": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "nh3-0.2.18.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "nh3==0.2.18", - "sha256": "94a166927e53972a9698af9542ace4e38b9de50c34352b962f4d9a7d4c927af4", - "urls": [ - "https://files.pythonhosted.org/packages/62/73/10df50b42ddb547a907deeb2f3c9823022580a7a47281e8eae8e003a9639/nh3-0.2.18.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_pkginfo_py3_none_any_889a6da2": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "pkginfo-1.10.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "pkginfo==1.10.0", - "sha256": "889a6da2ed7ffc58ab5b900d888ddce90bce912f2d2de1dc1c26f4cb9fe65097", - "urls": [ - "https://files.pythonhosted.org/packages/56/09/054aea9b7534a15ad38a363a2bd974c20646ab1582a387a95b8df1bfea1c/pkginfo-1.10.0-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_pkginfo_sdist_5df73835": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "pkginfo-1.10.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "pkginfo==1.10.0", - "sha256": "5df73835398d10db79f8eecd5cd86b1f6d29317589ea70796994d49399af6297", - "urls": [ - "https://files.pythonhosted.org/packages/2f/72/347ec5be4adc85c182ed2823d8d1c7b51e13b9a6b0c1aae59582eca652df/pkginfo-1.10.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_pycparser_py3_none_any_c3702b6d": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "pycparser-2.22-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "pycparser==2.22", - "sha256": "c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", - "urls": [ - "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_pycparser_sdist_491c8be9": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "pycparser-2.22.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "pycparser==2.22", - "sha256": "491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6", - "urls": [ - "https://files.pythonhosted.org/packages/1d/b2/31537cf4b1ca988837256c910a668b553fceb8f069bedc4b1c826024b52c/pycparser-2.22.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_pygments_py3_none_any_b8e6aca0": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "pygments-2.18.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "pygments==2.18.0", - "sha256": "b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a", - "urls": [ - "https://files.pythonhosted.org/packages/f7/3f/01c8b82017c199075f8f788d0d906b9ffbbc5a47dc9918a945e13d5a2bda/pygments-2.18.0-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_pygments_sdist_786ff802": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "pygments-2.18.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "pygments==2.18.0", - "sha256": "786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199", - "urls": [ - "https://files.pythonhosted.org/packages/8e/62/8336eff65bcbc8e4cb5d05b55faf041285951b6e80f33e2bff2024788f31/pygments-2.18.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_pywin32_ctypes_py3_none_any_8a151337": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_windows_x86_64" - ], - "filename": "pywin32_ctypes-0.2.3-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "pywin32-ctypes==0.2.3", - "sha256": "8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8", - "urls": [ - "https://files.pythonhosted.org/packages/de/3d/8161f7711c017e01ac9f008dfddd9410dff3674334c233bde66e7ba65bbf/pywin32_ctypes-0.2.3-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_pywin32_ctypes_sdist_d162dc04": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "pywin32-ctypes-0.2.3.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "pywin32-ctypes==0.2.3", - "sha256": "d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755", - "urls": [ - "https://files.pythonhosted.org/packages/85/9f/01a1a99704853cb63f253eea009390c88e7131c67e66a0a02099a8c917cb/pywin32-ctypes-0.2.3.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_readme_renderer_py3_none_any_2fbca89b": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "readme_renderer-44.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "readme-renderer==44.0", - "sha256": "2fbca89b81a08526aadf1357a8c2ae889ec05fb03f5da67f9769c9a592166151", - "urls": [ - "https://files.pythonhosted.org/packages/e1/67/921ec3024056483db83953ae8e48079ad62b92db7880013ca77632921dd0/readme_renderer-44.0-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_readme_renderer_sdist_8712034e": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "readme_renderer-44.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "readme-renderer==44.0", - "sha256": "8712034eabbfa6805cacf1402b4eeb2a73028f72d1166d6f5cb7f9c047c5d1e1", - "urls": [ - "https://files.pythonhosted.org/packages/5a/a9/104ec9234c8448c4379768221ea6df01260cd6c2ce13182d4eac531c8342/readme_renderer-44.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_requests_py3_none_any_70761cfe": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "requests-2.32.3-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "requests==2.32.3", - "sha256": "70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6", - "urls": [ - "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_requests_sdist_55365417": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "requests-2.32.3.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "requests==2.32.3", - "sha256": "55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", - "urls": [ - "https://files.pythonhosted.org/packages/63/70/2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_requests_toolbelt_py2_none_any_cccfdd66": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "requests_toolbelt-1.0.0-py2.py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "requests-toolbelt==1.0.0", - "sha256": "cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", - "urls": [ - "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_requests_toolbelt_sdist_7681a0a3": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "requests-toolbelt-1.0.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "requests-toolbelt==1.0.0", - "sha256": "7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6", - "urls": [ - "https://files.pythonhosted.org/packages/f3/61/d7545dafb7ac2230c70d38d31cbfe4cc64f7144dc41f6e4e4b78ecd9f5bb/requests-toolbelt-1.0.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_rfc3986_py2_none_any_50b1502b": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "rfc3986-2.0.0-py2.py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "rfc3986==2.0.0", - "sha256": "50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd", - "urls": [ - "https://files.pythonhosted.org/packages/ff/9a/9afaade874b2fa6c752c36f1548f718b5b83af81ed9b76628329dab81c1b/rfc3986-2.0.0-py2.py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_rfc3986_sdist_97aacf9d": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "rfc3986-2.0.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "rfc3986==2.0.0", - "sha256": "97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c", - "urls": [ - "https://files.pythonhosted.org/packages/85/40/1520d68bfa07ab5a6f065a186815fb6610c86fe957bc065754e47f7b0840/rfc3986-2.0.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_rich_py3_none_any_6049d5e6": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "rich-13.9.4-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "rich==13.9.4", - "sha256": "6049d5e6ec054bf2779ab3358186963bac2ea89175919d699e378b99738c2a90", - "urls": [ - "https://files.pythonhosted.org/packages/19/71/39c7c0d87f8d4e6c020a393182060eaefeeae6c01dab6a84ec346f2567df/rich-13.9.4-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_rich_sdist_43959497": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "rich-13.9.4.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "rich==13.9.4", - "sha256": "439594978a49a09530cff7ebc4b5c7103ef57baf48d5ea3184f21d9a2befa098", - "urls": [ - "https://files.pythonhosted.org/packages/ab/3a/0316b28d0761c6734d6bc14e770d85506c986c85ffb239e688eeaab2c2bc/rich-13.9.4.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_secretstorage_py3_none_any_f356e662": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "SecretStorage-3.3.3-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "secretstorage==3.3.3", - "sha256": "f356e6628222568e3af06f2eba8df495efa13b3b63081dafd4f7d9a7b7bc9f99", - "urls": [ - "https://files.pythonhosted.org/packages/54/24/b4293291fa1dd830f353d2cb163295742fa87f179fcc8a20a306a81978b7/SecretStorage-3.3.3-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_secretstorage_sdist_2403533e": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "SecretStorage-3.3.3.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "secretstorage==3.3.3", - "sha256": "2403533ef369eca6d2ba81718576c5e0f564d5cca1b58f73a8b23e7d4eeebd77", - "urls": [ - "https://files.pythonhosted.org/packages/53/a4/f48c9d79cb507ed1373477dbceaba7401fd8a23af63b837fa61f1dcd3691/SecretStorage-3.3.3.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_twine_py3_none_any_215dbe7b": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "twine-5.1.1-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "twine==5.1.1", - "sha256": "215dbe7b4b94c2c50a7315c0275d2258399280fbb7d04182c7e55e24b5f93997", - "urls": [ - "https://files.pythonhosted.org/packages/5d/ec/00f9d5fd040ae29867355e559a94e9a8429225a0284a3f5f091a3878bfc0/twine-5.1.1-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_twine_sdist_9aa08251": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "twine-5.1.1.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "twine==5.1.1", - "sha256": "9aa0825139c02b3434d913545c7b847a21c835e11597f5255842d457da2322db", - "urls": [ - "https://files.pythonhosted.org/packages/77/68/bd982e5e949ef8334e6f7dcf76ae40922a8750aa2e347291ae1477a4782b/twine-5.1.1.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_urllib3_py3_none_any_ca899ca0": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "urllib3-2.2.3-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "urllib3==2.2.3", - "sha256": "ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac", - "urls": [ - "https://files.pythonhosted.org/packages/ce/d9/5f4c13cecde62396b0d3fe530a50ccea91e7dfc1ccf0e09c228841bb5ba8/urllib3-2.2.3-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_urllib3_sdist_e7d814a8": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "urllib3-2.2.3.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "urllib3==2.2.3", - "sha256": "e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9", - "urls": [ - "https://files.pythonhosted.org/packages/ed/63/22ba4ebfe7430b76388e7cd448d5478814d3032121827c12a2cc287e2260/urllib3-2.2.3.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_zipp_py3_none_any_a817ac80": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "zipp-3.20.2-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "zipp==3.20.2", - "sha256": "a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350", - "urls": [ - "https://files.pythonhosted.org/packages/62/8b/5ba542fa83c90e09eac972fc9baca7a88e7e7ca4b221a89251954019308b/zipp-3.20.2-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_zipp_sdist_bc9eb26f": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "zipp-3.20.2.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "zipp==3.20.2", - "sha256": "bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29", - "urls": [ - "https://files.pythonhosted.org/packages/54/bf/5c0000c44ebc80123ecbdddba1f5dcd94a5ada602a9c225d84b5aaa55e86/zipp-3.20.2.tar.gz" - ] - } - }, - "huggingface_hub": { - "repoRuleId": "@@rules_python+//python/private/pypi:hub_repository.bzl%hub_repository", - "attributes": { - "repo_name": "huggingface_hub", - "extra_hub_aliases": {}, - "whl_map": { - "certifi": "{\"huggingface_hub_311_certifi\":[{\"version\":\"3.11\"}]}", - "charset_normalizer": "{\"huggingface_hub_311_charset_normalizer\":[{\"version\":\"3.11\"}]}", - "filelock": "{\"huggingface_hub_311_filelock\":[{\"version\":\"3.11\"}]}", - "fsspec": "{\"huggingface_hub_311_fsspec\":[{\"version\":\"3.11\"}]}", - "hf_xet": "{\"huggingface_hub_311_hf_xet\":[{\"version\":\"3.11\"}]}", - "huggingface_hub": "{\"huggingface_hub_311_huggingface_hub\":[{\"version\":\"3.11\"}]}", - "idna": "{\"huggingface_hub_311_idna\":[{\"version\":\"3.11\"}]}", - "packaging": "{\"huggingface_hub_311_packaging\":[{\"version\":\"3.11\"}]}", - "pyyaml": "{\"huggingface_hub_311_pyyaml\":[{\"version\":\"3.11\"}]}", - "requests": "{\"huggingface_hub_311_requests\":[{\"version\":\"3.11\"}]}", - "tqdm": "{\"huggingface_hub_311_tqdm\":[{\"version\":\"3.11\"}]}", - "typing_extensions": "{\"huggingface_hub_311_typing_extensions\":[{\"version\":\"3.11\"}]}", - "urllib3": "{\"huggingface_hub_311_urllib3\":[{\"version\":\"3.11\"}]}" - }, - "packages": [ - "certifi", - "charset_normalizer", - "filelock", - "fsspec", - "hf_xet", - "huggingface_hub", - "idna", - "packaging", - "pyyaml", - "requests", - "tqdm", - "typing_extensions", - "urllib3" - ], - "groups": {} - } - }, - "neuron_py_deps": { - "repoRuleId": "@@rules_python+//python/private/pypi:hub_repository.bzl%hub_repository", - "attributes": { - "repo_name": "neuron_py_deps", - "extra_hub_aliases": {}, - "whl_map": { - "boto3": "{\"neuron_py_deps_311_boto3\":[{\"version\":\"3.11\"}]}", - "botocore": "{\"neuron_py_deps_311_botocore\":[{\"version\":\"3.11\"}]}", - "certifi": "{\"neuron_py_deps_311_certifi\":[{\"version\":\"3.11\"}]}", - "charset_normalizer": "{\"neuron_py_deps_311_charset_normalizer\":[{\"version\":\"3.11\"}]}", - "ec2_metadata": "{\"neuron_py_deps_311_ec2_metadata\":[{\"version\":\"3.11\"}]}", - "idna": "{\"neuron_py_deps_311_idna\":[{\"version\":\"3.11\"}]}", - "islpy": "{\"neuron_py_deps_311_islpy\":[{\"version\":\"3.11\"}]}", - "jmespath": "{\"neuron_py_deps_311_jmespath\":[{\"version\":\"3.11\"}]}", - "libneuronxla": "{\"neuron_py_deps_311_libneuronxla\":[{\"version\":\"3.11\"}]}", - "lockfile": "{\"neuron_py_deps_311_lockfile\":[{\"version\":\"3.11\"}]}", - "ml_dtypes": "{\"neuron_py_deps_311_ml_dtypes\":[{\"version\":\"3.11\"}]}", - "networkx": "{\"neuron_py_deps_311_networkx\":[{\"version\":\"3.11\"}]}", - "neuronx_cc": "{\"neuron_py_deps_311_neuronx_cc\":[{\"version\":\"3.11\"}]}", - "numpy": "{\"neuron_py_deps_311_numpy\":[{\"version\":\"3.11\"}]}", - "pgzip": "{\"neuron_py_deps_311_pgzip\":[{\"version\":\"3.11\"}]}", - "protobuf": "{\"neuron_py_deps_311_protobuf\":[{\"version\":\"3.11\"}]}", - "psutil": "{\"neuron_py_deps_311_psutil\":[{\"version\":\"3.11\"}]}", - "python_daemon": "{\"neuron_py_deps_311_python_daemon\":[{\"version\":\"3.11\"}]}", - "python_dateutil": "{\"neuron_py_deps_311_python_dateutil\":[{\"version\":\"3.11\"}]}", - "requests": "{\"neuron_py_deps_311_requests\":[{\"version\":\"3.11\"}]}", - "requests_unixsocket": "{\"neuron_py_deps_311_requests_unixsocket\":[{\"version\":\"3.11\"}]}", - "s3transfer": "{\"neuron_py_deps_311_s3transfer\":[{\"version\":\"3.11\"}]}", - "scipy": "{\"neuron_py_deps_311_scipy\":[{\"version\":\"3.11\"}]}", - "six": "{\"neuron_py_deps_311_six\":[{\"version\":\"3.11\"}]}", - "tqdm": "{\"neuron_py_deps_311_tqdm\":[{\"version\":\"3.11\"}]}", - "typing_extensions": "{\"neuron_py_deps_311_typing_extensions\":[{\"version\":\"3.11\"}]}", - "urllib3": "{\"neuron_py_deps_311_urllib3\":[{\"version\":\"3.11\"}]}" - }, - "packages": [ - "boto3", - "botocore", - "certifi", - "charset_normalizer", - "ec2_metadata", - "idna", - "islpy", - "jmespath", - "libneuronxla", - "lockfile", - "ml_dtypes", - "networkx", - "neuronx_cc", - "numpy", - "pgzip", - "protobuf", - "psutil", - "python_daemon", - "python_dateutil", - "requests", - "requests_unixsocket", - "s3transfer", - "scipy", - "six", - "tqdm", - "typing_extensions", - "urllib3" - ], - "groups": {} - } - }, - "pip_deps": { - "repoRuleId": "@@rules_python+//python/private/pypi:hub_repository.bzl%hub_repository", - "attributes": { - "repo_name": "pip_deps", - "extra_hub_aliases": {}, - "whl_map": { - "numpy": "{\"pip_deps_310_numpy\":[{\"version\":\"3.10\"}],\"pip_deps_311_numpy\":[{\"version\":\"3.11\"}],\"pip_deps_312_numpy\":[{\"version\":\"3.12\"}],\"pip_deps_38_numpy\":[{\"version\":\"3.8\"}],\"pip_deps_39_numpy\":[{\"version\":\"3.9\"}]}", - "setuptools": "{\"pip_deps_310_setuptools\":[{\"version\":\"3.10\"}],\"pip_deps_311_setuptools\":[{\"version\":\"3.11\"}],\"pip_deps_312_setuptools\":[{\"version\":\"3.12\"}],\"pip_deps_38_setuptools\":[{\"version\":\"3.8\"}],\"pip_deps_39_setuptools\":[{\"version\":\"3.9\"}]}" - }, - "packages": [ - "numpy", - "setuptools" - ], - "groups": {} - } - }, - "rules_fuzzing_py_deps": { - "repoRuleId": "@@rules_python+//python/private/pypi:hub_repository.bzl%hub_repository", - "attributes": { - "repo_name": "rules_fuzzing_py_deps", - "extra_hub_aliases": {}, - "whl_map": { - "absl_py": "{\"rules_fuzzing_py_deps_310_absl_py\":[{\"version\":\"3.10\"}],\"rules_fuzzing_py_deps_311_absl_py\":[{\"version\":\"3.11\"}],\"rules_fuzzing_py_deps_312_absl_py\":[{\"version\":\"3.12\"}],\"rules_fuzzing_py_deps_38_absl_py\":[{\"version\":\"3.8\"}],\"rules_fuzzing_py_deps_39_absl_py\":[{\"version\":\"3.9\"}]}", - "six": "{\"rules_fuzzing_py_deps_310_six\":[{\"version\":\"3.10\"}],\"rules_fuzzing_py_deps_311_six\":[{\"version\":\"3.11\"}],\"rules_fuzzing_py_deps_312_six\":[{\"version\":\"3.12\"}],\"rules_fuzzing_py_deps_38_six\":[{\"version\":\"3.8\"}],\"rules_fuzzing_py_deps_39_six\":[{\"version\":\"3.9\"}]}" - }, - "packages": [ - "absl_py", - "six" - ], - "groups": {} - } - }, - "rules_python_publish_deps": { - "repoRuleId": "@@rules_python+//python/private/pypi:hub_repository.bzl%hub_repository", - "attributes": { - "repo_name": "rules_python_publish_deps", - "extra_hub_aliases": {}, - "whl_map": { - "backports_tarfile": "{\"rules_python_publish_deps_311_backports_tarfile_py3_none_any_77e284d7\":[{\"filename\":\"backports.tarfile-1.2.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_backports_tarfile_sdist_d75e02c2\":[{\"filename\":\"backports_tarfile-1.2.0.tar.gz\",\"version\":\"3.11\"}]}", - "certifi": "{\"rules_python_publish_deps_311_certifi_py3_none_any_922820b5\":[{\"filename\":\"certifi-2024.8.30-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_certifi_sdist_bec941d2\":[{\"filename\":\"certifi-2024.8.30.tar.gz\",\"version\":\"3.11\"}]}", - "cffi": "{\"rules_python_publish_deps_311_cffi_cp311_cp311_manylinux_2_17_aarch64_a1ed2dd2\":[{\"filename\":\"cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_cffi_cp311_cp311_manylinux_2_17_s390x_a24ed04c\":[{\"filename\":\"cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_cffi_cp311_cp311_manylinux_2_17_x86_64_610faea7\":[{\"filename\":\"cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_cffi_cp311_cp311_musllinux_1_1_aarch64_a9b15d49\":[{\"filename\":\"cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_cffi_cp311_cp311_musllinux_1_1_x86_64_fc48c783\":[{\"filename\":\"cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_cffi_sdist_1c39c601\":[{\"filename\":\"cffi-1.17.1.tar.gz\",\"version\":\"3.11\"}]}", - "charset_normalizer": "{\"rules_python_publish_deps_311_charset_normalizer_cp311_cp311_macosx_10_9_universal2_0d99dd8f\":[{\"filename\":\"charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_universal2.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_charset_normalizer_cp311_cp311_macosx_10_9_x86_64_c57516e5\":[{\"filename\":\"charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_charset_normalizer_cp311_cp311_macosx_11_0_arm64_6dba5d19\":[{\"filename\":\"charset_normalizer-3.4.0-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_charset_normalizer_cp311_cp311_manylinux_2_17_aarch64_bf4475b8\":[{\"filename\":\"charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_charset_normalizer_cp311_cp311_manylinux_2_17_s390x_8ff4e7cd\":[{\"filename\":\"charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_charset_normalizer_cp311_cp311_manylinux_2_17_x86_64_3710a975\":[{\"filename\":\"charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_charset_normalizer_cp311_cp311_musllinux_1_2_aarch64_47334db7\":[{\"filename\":\"charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_charset_normalizer_cp311_cp311_musllinux_1_2_s390x_63bc5c4a\":[{\"filename\":\"charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_charset_normalizer_cp311_cp311_musllinux_1_2_x86_64_bcb4f8ea\":[{\"filename\":\"charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_charset_normalizer_cp311_cp311_win_amd64_cee4373f\":[{\"filename\":\"charset_normalizer-3.4.0-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_charset_normalizer_py3_none_any_fe9f97fe\":[{\"filename\":\"charset_normalizer-3.4.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_charset_normalizer_sdist_223217c3\":[{\"filename\":\"charset_normalizer-3.4.0.tar.gz\",\"version\":\"3.11\"}]}", - "cryptography": "{\"rules_python_publish_deps_311_cryptography_cp39_abi3_manylinux_2_17_aarch64_846da004\":[{\"filename\":\"cryptography-43.0.3-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_cryptography_cp39_abi3_manylinux_2_17_x86_64_0f996e72\":[{\"filename\":\"cryptography-43.0.3-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_cryptography_cp39_abi3_manylinux_2_28_aarch64_f7b178f1\":[{\"filename\":\"cryptography-43.0.3-cp39-abi3-manylinux_2_28_aarch64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_cryptography_cp39_abi3_manylinux_2_28_x86_64_c2e6fc39\":[{\"filename\":\"cryptography-43.0.3-cp39-abi3-manylinux_2_28_x86_64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_cryptography_cp39_abi3_musllinux_1_2_aarch64_e1be4655\":[{\"filename\":\"cryptography-43.0.3-cp39-abi3-musllinux_1_2_aarch64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_cryptography_cp39_abi3_musllinux_1_2_x86_64_df6b6c6d\":[{\"filename\":\"cryptography-43.0.3-cp39-abi3-musllinux_1_2_x86_64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_cryptography_sdist_315b9001\":[{\"filename\":\"cryptography-43.0.3.tar.gz\",\"version\":\"3.11\"}]}", - "docutils": "{\"rules_python_publish_deps_311_docutils_py3_none_any_dafca5b9\":[{\"filename\":\"docutils-0.21.2-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_docutils_sdist_3a6b1873\":[{\"filename\":\"docutils-0.21.2.tar.gz\",\"version\":\"3.11\"}]}", - "idna": "{\"rules_python_publish_deps_311_idna_py3_none_any_946d195a\":[{\"filename\":\"idna-3.10-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_idna_sdist_12f65c9b\":[{\"filename\":\"idna-3.10.tar.gz\",\"version\":\"3.11\"}]}", - "importlib_metadata": "{\"rules_python_publish_deps_311_importlib_metadata_py3_none_any_45e54197\":[{\"filename\":\"importlib_metadata-8.5.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_importlib_metadata_sdist_71522656\":[{\"filename\":\"importlib_metadata-8.5.0.tar.gz\",\"version\":\"3.11\"}]}", - "jaraco_classes": "{\"rules_python_publish_deps_311_jaraco_classes_py3_none_any_f662826b\":[{\"filename\":\"jaraco.classes-3.4.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_jaraco_classes_sdist_47a024b5\":[{\"filename\":\"jaraco.classes-3.4.0.tar.gz\",\"version\":\"3.11\"}]}", - "jaraco_context": "{\"rules_python_publish_deps_311_jaraco_context_py3_none_any_f797fc48\":[{\"filename\":\"jaraco.context-6.0.1-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_jaraco_context_sdist_9bae4ea5\":[{\"filename\":\"jaraco_context-6.0.1.tar.gz\",\"version\":\"3.11\"}]}", - "jaraco_functools": "{\"rules_python_publish_deps_311_jaraco_functools_py3_none_any_ad159f13\":[{\"filename\":\"jaraco.functools-4.1.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_jaraco_functools_sdist_70f7e0e2\":[{\"filename\":\"jaraco_functools-4.1.0.tar.gz\",\"version\":\"3.11\"}]}", - "jeepney": "{\"rules_python_publish_deps_311_jeepney_py3_none_any_c0a454ad\":[{\"filename\":\"jeepney-0.8.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_jeepney_sdist_5efe48d2\":[{\"filename\":\"jeepney-0.8.0.tar.gz\",\"version\":\"3.11\"}]}", - "keyring": "{\"rules_python_publish_deps_311_keyring_py3_none_any_5426f817\":[{\"filename\":\"keyring-25.4.1-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_keyring_sdist_b07ebc55\":[{\"filename\":\"keyring-25.4.1.tar.gz\",\"version\":\"3.11\"}]}", - "markdown_it_py": "{\"rules_python_publish_deps_311_markdown_it_py_py3_none_any_35521684\":[{\"filename\":\"markdown_it_py-3.0.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_markdown_it_py_sdist_e3f60a94\":[{\"filename\":\"markdown-it-py-3.0.0.tar.gz\",\"version\":\"3.11\"}]}", - "mdurl": "{\"rules_python_publish_deps_311_mdurl_py3_none_any_84008a41\":[{\"filename\":\"mdurl-0.1.2-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_mdurl_sdist_bb413d29\":[{\"filename\":\"mdurl-0.1.2.tar.gz\",\"version\":\"3.11\"}]}", - "more_itertools": "{\"rules_python_publish_deps_311_more_itertools_py3_none_any_037b0d32\":[{\"filename\":\"more_itertools-10.5.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_more_itertools_sdist_5482bfef\":[{\"filename\":\"more-itertools-10.5.0.tar.gz\",\"version\":\"3.11\"}]}", - "nh3": "{\"rules_python_publish_deps_311_nh3_cp37_abi3_macosx_10_12_x86_64_14c5a72e\":[{\"filename\":\"nh3-0.2.18-cp37-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_nh3_cp37_abi3_macosx_10_12_x86_64_7b7c2a3c\":[{\"filename\":\"nh3-0.2.18-cp37-abi3-macosx_10_12_x86_64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_nh3_cp37_abi3_manylinux_2_17_aarch64_42c64511\":[{\"filename\":\"nh3-0.2.18-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_nh3_cp37_abi3_manylinux_2_17_armv7l_0411beb0\":[{\"filename\":\"nh3-0.2.18-cp37-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_nh3_cp37_abi3_manylinux_2_17_ppc64_5f36b271\":[{\"filename\":\"nh3-0.2.18-cp37-abi3-manylinux_2_17_ppc64.manylinux2014_ppc64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_nh3_cp37_abi3_manylinux_2_17_s390x_19aaba96\":[{\"filename\":\"nh3-0.2.18-cp37-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_nh3_cp37_abi3_manylinux_2_17_x86_64_de3ceed6\":[{\"filename\":\"nh3-0.2.18-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_nh3_cp37_abi3_musllinux_1_2_aarch64_f0eca9ca\":[{\"filename\":\"nh3-0.2.18-cp37-abi3-musllinux_1_2_aarch64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_nh3_cp37_abi3_musllinux_1_2_armv7l_3a157ab1\":[{\"filename\":\"nh3-0.2.18-cp37-abi3-musllinux_1_2_armv7l.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_nh3_cp37_abi3_musllinux_1_2_x86_64_36c95d4b\":[{\"filename\":\"nh3-0.2.18-cp37-abi3-musllinux_1_2_x86_64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_nh3_cp37_abi3_win_amd64_8ce0f819\":[{\"filename\":\"nh3-0.2.18-cp37-abi3-win_amd64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_nh3_sdist_94a16692\":[{\"filename\":\"nh3-0.2.18.tar.gz\",\"version\":\"3.11\"}]}", - "pkginfo": "{\"rules_python_publish_deps_311_pkginfo_py3_none_any_889a6da2\":[{\"filename\":\"pkginfo-1.10.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_pkginfo_sdist_5df73835\":[{\"filename\":\"pkginfo-1.10.0.tar.gz\",\"version\":\"3.11\"}]}", - "pycparser": "{\"rules_python_publish_deps_311_pycparser_py3_none_any_c3702b6d\":[{\"filename\":\"pycparser-2.22-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_pycparser_sdist_491c8be9\":[{\"filename\":\"pycparser-2.22.tar.gz\",\"version\":\"3.11\"}]}", - "pygments": "{\"rules_python_publish_deps_311_pygments_py3_none_any_b8e6aca0\":[{\"filename\":\"pygments-2.18.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_pygments_sdist_786ff802\":[{\"filename\":\"pygments-2.18.0.tar.gz\",\"version\":\"3.11\"}]}", - "pywin32_ctypes": "{\"rules_python_publish_deps_311_pywin32_ctypes_py3_none_any_8a151337\":[{\"filename\":\"pywin32_ctypes-0.2.3-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_pywin32_ctypes_sdist_d162dc04\":[{\"filename\":\"pywin32-ctypes-0.2.3.tar.gz\",\"version\":\"3.11\"}]}", - "readme_renderer": "{\"rules_python_publish_deps_311_readme_renderer_py3_none_any_2fbca89b\":[{\"filename\":\"readme_renderer-44.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_readme_renderer_sdist_8712034e\":[{\"filename\":\"readme_renderer-44.0.tar.gz\",\"version\":\"3.11\"}]}", - "requests": "{\"rules_python_publish_deps_311_requests_py3_none_any_70761cfe\":[{\"filename\":\"requests-2.32.3-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_requests_sdist_55365417\":[{\"filename\":\"requests-2.32.3.tar.gz\",\"version\":\"3.11\"}]}", - "requests_toolbelt": "{\"rules_python_publish_deps_311_requests_toolbelt_py2_none_any_cccfdd66\":[{\"filename\":\"requests_toolbelt-1.0.0-py2.py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_requests_toolbelt_sdist_7681a0a3\":[{\"filename\":\"requests-toolbelt-1.0.0.tar.gz\",\"version\":\"3.11\"}]}", - "rfc3986": "{\"rules_python_publish_deps_311_rfc3986_py2_none_any_50b1502b\":[{\"filename\":\"rfc3986-2.0.0-py2.py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_rfc3986_sdist_97aacf9d\":[{\"filename\":\"rfc3986-2.0.0.tar.gz\",\"version\":\"3.11\"}]}", - "rich": "{\"rules_python_publish_deps_311_rich_py3_none_any_6049d5e6\":[{\"filename\":\"rich-13.9.4-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_rich_sdist_43959497\":[{\"filename\":\"rich-13.9.4.tar.gz\",\"version\":\"3.11\"}]}", - "secretstorage": "{\"rules_python_publish_deps_311_secretstorage_py3_none_any_f356e662\":[{\"filename\":\"SecretStorage-3.3.3-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_secretstorage_sdist_2403533e\":[{\"filename\":\"SecretStorage-3.3.3.tar.gz\",\"version\":\"3.11\"}]}", - "twine": "{\"rules_python_publish_deps_311_twine_py3_none_any_215dbe7b\":[{\"filename\":\"twine-5.1.1-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_twine_sdist_9aa08251\":[{\"filename\":\"twine-5.1.1.tar.gz\",\"version\":\"3.11\"}]}", - "urllib3": "{\"rules_python_publish_deps_311_urllib3_py3_none_any_ca899ca0\":[{\"filename\":\"urllib3-2.2.3-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_urllib3_sdist_e7d814a8\":[{\"filename\":\"urllib3-2.2.3.tar.gz\",\"version\":\"3.11\"}]}", - "zipp": "{\"rules_python_publish_deps_311_zipp_py3_none_any_a817ac80\":[{\"filename\":\"zipp-3.20.2-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_zipp_sdist_bc9eb26f\":[{\"filename\":\"zipp-3.20.2.tar.gz\",\"version\":\"3.11\"}]}" - }, - "packages": [ - "backports_tarfile", - "certifi", - "charset_normalizer", - "docutils", - "idna", - "importlib_metadata", - "jaraco_classes", - "jaraco_context", - "jaraco_functools", - "keyring", - "markdown_it_py", - "mdurl", - "more_itertools", - "nh3", - "pkginfo", - "pygments", - "readme_renderer", - "requests", - "requests_toolbelt", - "rfc3986", - "rich", - "twine", - "urllib3", - "zipp" - ], - "groups": {} - } - } - }, - "moduleExtensionMetadata": { - "useAllRepos": "NO", - "reproducible": false - }, - "recordedRepoMappingEntries": [ - [ - "bazel_features+", - "bazel_features_globals", - "bazel_features++version_extension+bazel_features_globals" - ], - [ - "bazel_features+", - "bazel_features_version", - "bazel_features++version_extension+bazel_features_version" - ], - [ - "rules_python+", - "bazel_features", - "bazel_features+" - ], - [ - "rules_python+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "rules_python+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_python+", - "pypi__build", - "rules_python++internal_deps+pypi__build" - ], - [ - "rules_python+", - "pypi__click", - "rules_python++internal_deps+pypi__click" - ], - [ - "rules_python+", - "pypi__colorama", - "rules_python++internal_deps+pypi__colorama" - ], - [ - "rules_python+", - "pypi__importlib_metadata", - "rules_python++internal_deps+pypi__importlib_metadata" - ], - [ - "rules_python+", - "pypi__installer", - "rules_python++internal_deps+pypi__installer" - ], - [ - "rules_python+", - "pypi__more_itertools", - "rules_python++internal_deps+pypi__more_itertools" - ], - [ - "rules_python+", - "pypi__packaging", - "rules_python++internal_deps+pypi__packaging" - ], - [ - "rules_python+", - "pypi__pep517", - "rules_python++internal_deps+pypi__pep517" - ], - [ - "rules_python+", - "pypi__pip", - "rules_python++internal_deps+pypi__pip" - ], - [ - "rules_python+", - "pypi__pip_tools", - "rules_python++internal_deps+pypi__pip_tools" - ], - [ - "rules_python+", - "pypi__pyproject_hooks", - "rules_python++internal_deps+pypi__pyproject_hooks" - ], - [ - "rules_python+", - "pypi__setuptools", - "rules_python++internal_deps+pypi__setuptools" - ], - [ - "rules_python+", - "pypi__tomli", - "rules_python++internal_deps+pypi__tomli" - ], - [ - "rules_python+", - "pypi__wheel", - "rules_python++internal_deps+pypi__wheel" - ], - [ - "rules_python+", - "pypi__zipp", - "rules_python++internal_deps+pypi__zipp" - ], - [ - "rules_python+", - "pythons_hub", - "rules_python++python+pythons_hub" - ], - [ - "rules_python++python+pythons_hub", - "python_3_10_host", - "rules_python++python+python_3_10_host" - ], - [ - "rules_python++python+pythons_hub", - "python_3_11_host", - "rules_python++python+python_3_11_host" - ], - [ - "rules_python++python+pythons_hub", - "python_3_12_host", - "rules_python++python+python_3_12_host" - ], - [ - "rules_python++python+pythons_hub", - "python_3_8_host", - "rules_python++python+python_3_8_host" - ], - [ - "rules_python++python+pythons_hub", - "python_3_9_host", - "rules_python++python+python_3_9_host" - ] - ] - } - }, "@@rules_python+//python/uv:uv.bzl%uv": { "general": { - "bzlTransitiveDigest": "UwuEebIvWhq1IDSj2VbBXlMc20ec8cBFyxShHe64sy8=", - "usagesDigest": "vJ5RHUxAnV24M5swNGiAnkdxMx3Hp/iOLmNANTC5Xc8=", + "bzlTransitiveDigest": "Vd/0cN91U7O//KVXEBJsnr64LUyHdC6DRiDTTnQTrUw=", + "usagesDigest": "t11Aj3mDwK2nwRAqSmqylVpvomzPYWiSVUL4xbuiMm4=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { + "uv_0_6_5_aarch64_apple_darwin": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.6.5", + "platform": "aarch64-apple-darwin", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-aarch64-apple-darwin.tar.gz" + ], + "sha256": "4952c6275778fbc7e5cf6b5739ffd944c47063134d6ea43a7037b0a028ef771e" + } + }, + "uv_0_6_5_aarch64_unknown_linux_gnu": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.6.5", + "platform": "aarch64-unknown-linux-gnu", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-aarch64-unknown-linux-gnu.tar.gz" + ], + "sha256": "ebf0707ab949b948ad14250ed93f968fec954e20d12ce6bc9fce74b0ce7e4900" + } + }, + "uv_0_6_5_powerpc64_unknown_linux_gnu": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.6.5", + "platform": "powerpc64-unknown-linux-gnu", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-powerpc64-unknown-linux-gnu.tar.gz" + ], + "sha256": "1f8fd8dde91258380dbcd4b9fc65f15a7dc3af6cfffefb9a4f99d32c4c67436d" + } + }, + "uv_0_6_5_powerpc64le_unknown_linux_gnu": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.6.5", + "platform": "powerpc64le-unknown-linux-gnu", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-powerpc64le-unknown-linux-gnu.tar.gz" + ], + "sha256": "d2b6e19ca7ecd8d7e0d1ddced965aa91a3c822d07a7f7657743945245c1f627a" + } + }, + "uv_0_6_5_s390x_unknown_linux_gnu": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.6.5", + "platform": "s390x-unknown-linux-gnu", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-s390x-unknown-linux-gnu.tar.gz" + ], + "sha256": "43a3e39f19fd5d098bb8c896b6c0f99a146253c271d78409cf31c2f55f2763d0" + } + }, + "uv_0_6_5_x86_64_apple_darwin": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.6.5", + "platform": "x86_64-apple-darwin", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-x86_64-apple-darwin.tar.gz" + ], + "sha256": "d336d862694e1c36afea55eb15f5df02aa77c04a168b781df515d641e23285f0" + } + }, + "uv_0_6_5_x86_64_pc_windows_msvc": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.6.5", + "platform": "x86_64-pc-windows-msvc", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-x86_64-pc-windows-msvc.zip" + ], + "sha256": "a3e9f0913189c38b804bf6bb3d348af305a8ceabc690be92b6e111245a821640" + } + }, + "uv_0_6_5_x86_64_unknown_linux_gnu": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.6.5", + "platform": "x86_64-unknown-linux-gnu", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-x86_64-unknown-linux-gnu.tar.gz" + ], + "sha256": "8fc9895719a1291ecd193cb86f9282ff3649cef797d29eacc74c4f573aab1e2f" + } + }, "uv": { "repoRuleId": "@@rules_python+//python/uv/private:uv_toolchains_repo.bzl%uv_toolchains_repo", "attributes": { "toolchain_type": "'@@rules_python+//python/uv:uv_toolchain_type'", "toolchain_names": [ - "none" + "0_6_5_aarch64_apple_darwin", + "0_6_5_aarch64_unknown_linux_gnu", + "0_6_5_powerpc64_unknown_linux_gnu", + "0_6_5_powerpc64le_unknown_linux_gnu", + "0_6_5_s390x_unknown_linux_gnu", + "0_6_5_x86_64_apple_darwin", + "0_6_5_x86_64_pc_windows_msvc", + "0_6_5_x86_64_unknown_linux_gnu" ], "toolchain_implementations": { - "none": "'@@rules_python+//python:none'" + "0_6_5_aarch64_apple_darwin": "@uv_0_6_5_aarch64_apple_darwin//:uv_toolchain", + "0_6_5_aarch64_unknown_linux_gnu": "@uv_0_6_5_aarch64_unknown_linux_gnu//:uv_toolchain", + "0_6_5_powerpc64_unknown_linux_gnu": "@uv_0_6_5_powerpc64_unknown_linux_gnu//:uv_toolchain", + "0_6_5_powerpc64le_unknown_linux_gnu": "@uv_0_6_5_powerpc64le_unknown_linux_gnu//:uv_toolchain", + "0_6_5_s390x_unknown_linux_gnu": "@uv_0_6_5_s390x_unknown_linux_gnu//:uv_toolchain", + "0_6_5_x86_64_apple_darwin": "@uv_0_6_5_x86_64_apple_darwin//:uv_toolchain", + "0_6_5_x86_64_pc_windows_msvc": "@uv_0_6_5_x86_64_pc_windows_msvc//:uv_toolchain", + "0_6_5_x86_64_unknown_linux_gnu": "@uv_0_6_5_x86_64_unknown_linux_gnu//:uv_toolchain" }, "toolchain_compatible_with": { - "none": [ - "@platforms//:incompatible" + "0_6_5_aarch64_apple_darwin": [ + "'@@platforms//os:macos'", + "'@@platforms//cpu:aarch64'" + ], + "0_6_5_aarch64_unknown_linux_gnu": [ + "'@@platforms//os:linux'", + "'@@platforms//cpu:aarch64'" + ], + "0_6_5_powerpc64_unknown_linux_gnu": [ + "'@@platforms//os:linux'", + "'@@platforms//cpu:ppc'" + ], + "0_6_5_powerpc64le_unknown_linux_gnu": [ + "'@@platforms//os:linux'", + "'@@platforms//cpu:ppc64le'" + ], + "0_6_5_s390x_unknown_linux_gnu": [ + "'@@platforms//os:linux'", + "'@@platforms//cpu:s390x'" + ], + "0_6_5_x86_64_apple_darwin": [ + "'@@platforms//os:macos'", + "'@@platforms//cpu:x86_64'" + ], + "0_6_5_x86_64_pc_windows_msvc": [ + "'@@platforms//os:windows'", + "'@@platforms//cpu:x86_64'" + ], + "0_6_5_x86_64_unknown_linux_gnu": [ + "'@@platforms//os:linux'", + "'@@platforms//cpu:x86_64'" ] }, "toolchain_target_settings": {} @@ -4825,6 +920,11 @@ } }, "recordedRepoMappingEntries": [ + [ + "rules_python+", + "bazel_tools", + "bazel_tools" + ], [ "rules_python+", "platforms", @@ -4835,11 +935,11 @@ }, "@@rules_rust+//crate_universe:extensions.bzl%crate": { "general": { - "bzlTransitiveDigest": "YWlcbEZ/1fdqy0HNBaFalAKz/RswQbhU7oDvWUZm7cQ=", + "bzlTransitiveDigest": "HHPRUh/mbCXdC11w2axMRLWeweoYK1bLuS1e2NpvcoM=", "usagesDigest": "mup6mgSFatk34Q0znMZPpIEOFZyk9v3xf6jRIXEFTxc=", "recordedFileInputs": { - "@@//zml/tokenizer/hftokenizers/Cargo.lock": "93198160b29c4607994b405ffa0ad1c29b08df1ad22ec9a711874914bbaadbcb", - "@@//zml/tokenizer/hftokenizers/Cargo.toml": "eaf4574383be4d13a653a9c91c052979b05630f1c251b87bc8386263165dc204" + "@@//zml/tokenizer/hftokenizers/Cargo.lock": "d78ffab37ef69bf8ff45a4d686b25175f30e8fed4d8375612e84dcf6db0cfc9c", + "@@//zml/tokenizer/hftokenizers/Cargo.toml": "eeccbf5ff71c5f26b94cb289b3fc64c5fc6eb8708b815f39ce5f16bf225b6774" }, "recordedDirentsInputs": {}, "envVariables": { @@ -4856,9 +956,9 @@ "repoRuleId": "@@rules_rust+//crate_universe:extensions.bzl%_generate_repo", "attributes": { "contents": { - "BUILD.bazel": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files(\n [\n \"cargo-bazel.json\",\n \"crates.bzl\",\n \"defs.bzl\",\n ] + glob(\n allow_empty = True,\n include = [\"*.bazel\"],\n ),\n)\n\nfilegroup(\n name = \"srcs\",\n srcs = glob(\n allow_empty = True,\n include = [\n \"*.bazel\",\n \"*.bzl\",\n ],\n ),\n)\n\n# Workspace Member Dependencies\nalias(\n name = \"tokenizers-0.21.2\",\n actual = \"@crates__tokenizers-0.21.2//:tokenizers\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"tokenizers\",\n actual = \"@crates__tokenizers-0.21.2//:tokenizers\",\n tags = [\"manual\"],\n)\n", + "BUILD.bazel": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files(\n [\n \"cargo-bazel.json\",\n \"crates.bzl\",\n \"defs.bzl\",\n ] + glob(\n allow_empty = True,\n include = [\"*.bazel\"],\n ),\n)\n\nfilegroup(\n name = \"srcs\",\n srcs = glob(\n allow_empty = True,\n include = [\n \"*.bazel\",\n \"*.bzl\",\n ],\n ),\n)\n\n# Workspace Member Dependencies\nalias(\n name = \"tokenizers-0.21.4\",\n actual = \"@crates__tokenizers-0.21.4//:tokenizers\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"tokenizers\",\n actual = \"@crates__tokenizers-0.21.4//:tokenizers\",\n tags = [\"manual\"],\n)\n", "alias_rules.bzl": "\"\"\"Alias that transitions its target to `compilation_mode=opt`. Use `transition_alias=\"opt\"` to enable.\"\"\"\n\nload(\"@rules_cc//cc:defs.bzl\", \"CcInfo\")\nload(\"@rules_rust//rust:rust_common.bzl\", \"COMMON_PROVIDERS\")\n\ndef _transition_alias_impl(ctx):\n # `ctx.attr.actual` is a list of 1 item due to the transition\n providers = [ctx.attr.actual[0][provider] for provider in COMMON_PROVIDERS]\n if CcInfo in ctx.attr.actual[0]:\n providers.append(ctx.attr.actual[0][CcInfo])\n return providers\n\ndef _change_compilation_mode(compilation_mode):\n def _change_compilation_mode_impl(_settings, _attr):\n return {\n \"//command_line_option:compilation_mode\": compilation_mode,\n }\n\n return transition(\n implementation = _change_compilation_mode_impl,\n inputs = [],\n outputs = [\n \"//command_line_option:compilation_mode\",\n ],\n )\n\ndef _transition_alias_rule(compilation_mode):\n return rule(\n implementation = _transition_alias_impl,\n provides = COMMON_PROVIDERS,\n attrs = {\n \"actual\": attr.label(\n mandatory = True,\n doc = \"`rust_library()` target to transition to `compilation_mode=opt`.\",\n providers = COMMON_PROVIDERS,\n cfg = _change_compilation_mode(compilation_mode),\n ),\n \"_allowlist_function_transition\": attr.label(\n default = \"@bazel_tools//tools/allowlists/function_transition_allowlist\",\n ),\n },\n doc = \"Transitions a Rust library crate to the `compilation_mode=opt`.\",\n )\n\ntransition_alias_dbg = _transition_alias_rule(\"dbg\")\ntransition_alias_fastbuild = _transition_alias_rule(\"fastbuild\")\ntransition_alias_opt = _transition_alias_rule(\"opt\")\n", - "defs.bzl": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\"\"\"\n# `crates_repository` API\n\n- [aliases](#aliases)\n- [crate_deps](#crate_deps)\n- [all_crate_deps](#all_crate_deps)\n- [crate_repositories](#crate_repositories)\n\n\"\"\"\n\nload(\"@bazel_tools//tools/build_defs/repo:git.bzl\", \"new_git_repository\")\nload(\"@bazel_tools//tools/build_defs/repo:http.bzl\", \"http_archive\")\nload(\"@bazel_tools//tools/build_defs/repo:utils.bzl\", \"maybe\")\nload(\"@bazel_skylib//lib:selects.bzl\", \"selects\")\nload(\"@rules_rust//crate_universe/private:local_crate_mirror.bzl\", \"local_crate_mirror\")\n\n###############################################################################\n# MACROS API\n###############################################################################\n\n# An identifier that represent common dependencies (unconditional).\n_COMMON_CONDITION = \"\"\n\ndef _flatten_dependency_maps(all_dependency_maps):\n \"\"\"Flatten a list of dependency maps into one dictionary.\n\n Dependency maps have the following structure:\n\n ```python\n DEPENDENCIES_MAP = {\n # The first key in the map is a Bazel package\n # name of the workspace this file is defined in.\n \"workspace_member_package\": {\n\n # Not all dependencies are supported for all platforms.\n # the condition key is the condition required to be true\n # on the host platform.\n \"condition\": {\n\n # An alias to a crate target. # The label of the crate target the\n # Aliases are only crate names. # package name refers to.\n \"package_name\": \"@full//:label\",\n }\n }\n }\n ```\n\n Args:\n all_dependency_maps (list): A list of dicts as described above\n\n Returns:\n dict: A dictionary as described above\n \"\"\"\n dependencies = {}\n\n for workspace_deps_map in all_dependency_maps:\n for pkg_name, conditional_deps_map in workspace_deps_map.items():\n if pkg_name not in dependencies:\n non_frozen_map = dict()\n for key, values in conditional_deps_map.items():\n non_frozen_map.update({key: dict(values.items())})\n dependencies.setdefault(pkg_name, non_frozen_map)\n continue\n\n for condition, deps_map in conditional_deps_map.items():\n # If the condition has not been recorded, do so and continue\n if condition not in dependencies[pkg_name]:\n dependencies[pkg_name].setdefault(condition, dict(deps_map.items()))\n continue\n\n # Alert on any miss-matched dependencies\n inconsistent_entries = []\n for crate_name, crate_label in deps_map.items():\n existing = dependencies[pkg_name][condition].get(crate_name)\n if existing and existing != crate_label:\n inconsistent_entries.append((crate_name, existing, crate_label))\n dependencies[pkg_name][condition].update({crate_name: crate_label})\n\n return dependencies\n\ndef crate_deps(deps, package_name = None):\n \"\"\"Finds the fully qualified label of the requested crates for the package where this macro is called.\n\n Args:\n deps (list): The desired list of crate targets.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()`.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if not deps:\n return []\n\n if package_name == None:\n package_name = native.package_name()\n\n # Join both sets of dependencies\n dependencies = _flatten_dependency_maps([\n _NORMAL_DEPENDENCIES,\n _NORMAL_DEV_DEPENDENCIES,\n _PROC_MACRO_DEPENDENCIES,\n _PROC_MACRO_DEV_DEPENDENCIES,\n _BUILD_DEPENDENCIES,\n _BUILD_PROC_MACRO_DEPENDENCIES,\n ]).pop(package_name, {})\n\n # Combine all conditional packages so we can easily index over a flat list\n # TODO: Perhaps this should actually return select statements and maintain\n # the conditionals of the dependencies\n flat_deps = {}\n for deps_set in dependencies.values():\n for crate_name, crate_label in deps_set.items():\n flat_deps.update({crate_name: crate_label})\n\n missing_crates = []\n crate_targets = []\n for crate_target in deps:\n if crate_target not in flat_deps:\n missing_crates.append(crate_target)\n else:\n crate_targets.append(flat_deps[crate_target])\n\n if missing_crates:\n fail(\"Could not find crates `{}` among dependencies of `{}`. Available dependencies were `{}`\".format(\n missing_crates,\n package_name,\n dependencies,\n ))\n\n return crate_targets\n\ndef all_crate_deps(\n normal = False, \n normal_dev = False, \n proc_macro = False, \n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Finds the fully qualified label of all requested direct crate dependencies \\\n for the package where this macro is called.\n\n If no parameters are set, all normal dependencies are returned. Setting any one flag will\n otherwise impact the contents of the returned list.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_dependency_maps = []\n if normal:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n if normal_dev:\n all_dependency_maps.append(_NORMAL_DEV_DEPENDENCIES)\n if proc_macro:\n all_dependency_maps.append(_PROC_MACRO_DEPENDENCIES)\n if proc_macro_dev:\n all_dependency_maps.append(_PROC_MACRO_DEV_DEPENDENCIES)\n if build:\n all_dependency_maps.append(_BUILD_DEPENDENCIES)\n if build_proc_macro:\n all_dependency_maps.append(_BUILD_PROC_MACRO_DEPENDENCIES)\n\n # Default to always using normal dependencies\n if not all_dependency_maps:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n\n dependencies = _flatten_dependency_maps(all_dependency_maps).pop(package_name, None)\n\n if not dependencies:\n if dependencies == None:\n fail(\"Tried to get all_crate_deps for package \" + package_name + \" but that package had no Cargo.toml file\")\n else:\n return []\n\n crate_deps = list(dependencies.pop(_COMMON_CONDITION, {}).values())\n for condition, deps in dependencies.items():\n crate_deps += selects.with_or({\n tuple(_CONDITIONS[condition]): deps.values(),\n \"//conditions:default\": [],\n })\n\n return crate_deps\n\ndef aliases(\n normal = False,\n normal_dev = False,\n proc_macro = False,\n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Produces a map of Crate alias names to their original label\n\n If no dependency kinds are specified, `normal` and `proc_macro` are used by default.\n Setting any one flag will otherwise determine the contents of the returned dict.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n dict: The aliases of all associated packages\n \"\"\"\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_aliases_maps = []\n if normal:\n all_aliases_maps.append(_NORMAL_ALIASES)\n if normal_dev:\n all_aliases_maps.append(_NORMAL_DEV_ALIASES)\n if proc_macro:\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n if proc_macro_dev:\n all_aliases_maps.append(_PROC_MACRO_DEV_ALIASES)\n if build:\n all_aliases_maps.append(_BUILD_ALIASES)\n if build_proc_macro:\n all_aliases_maps.append(_BUILD_PROC_MACRO_ALIASES)\n\n # Default to always using normal aliases\n if not all_aliases_maps:\n all_aliases_maps.append(_NORMAL_ALIASES)\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n\n aliases = _flatten_dependency_maps(all_aliases_maps).pop(package_name, None)\n\n if not aliases:\n return dict()\n\n common_items = aliases.pop(_COMMON_CONDITION, {}).items()\n\n # If there are only common items in the dictionary, immediately return them\n if not len(aliases.keys()) == 1:\n return dict(common_items)\n\n # Build a single select statement where each conditional has accounted for the\n # common set of aliases.\n crate_aliases = {\"//conditions:default\": dict(common_items)}\n for condition, deps in aliases.items():\n condition_triples = _CONDITIONS[condition]\n for triple in condition_triples:\n if triple in crate_aliases:\n crate_aliases[triple].update(deps)\n else:\n crate_aliases.update({triple: dict(deps.items() + common_items)})\n\n return select(crate_aliases)\n\n###############################################################################\n# WORKSPACE MEMBER DEPS AND ALIASES\n###############################################################################\n\n_NORMAL_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n _COMMON_CONDITION: {\n \"tokenizers\": Label(\"@crates//:tokenizers-0.21.2\"),\n },\n },\n}\n\n\n_NORMAL_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n _COMMON_CONDITION: {\n },\n },\n}\n\n\n_NORMAL_DEV_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_NORMAL_DEV_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEV_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEV_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_CONDITIONS = {\n \"aarch64-apple-darwin\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\"],\n \"aarch64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\"],\n \"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\",\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\": [],\n \"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\": [],\n \"cfg(any())\": [],\n \"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\": [],\n \"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\": [],\n \"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\": [],\n \"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\",\"@rules_rust//rust/platform:x86_64-apple-darwin\"],\n \"cfg(not(all(target_arch = \\\"arm\\\", target_os = \\\"none\\\")))\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\",\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\",\"@rules_rust//rust/platform:x86_64-apple-darwin\",\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"cfg(target_os = \\\"netbsd\\\")\": [],\n \"cfg(target_os = \\\"solaris\\\")\": [],\n \"cfg(target_os = \\\"vxworks\\\")\": [],\n \"x86_64-apple-darwin\": [\"@rules_rust//rust/platform:x86_64-apple-darwin\"],\n \"x86_64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n}\n\n###############################################################################\n\ndef crate_repositories():\n \"\"\"A macro for defining repositories for all generated crates.\n\n Returns:\n A list of repos visible to the module through the module extension.\n \"\"\"\n maybe(\n http_archive,\n name = \"crates__ahash-0.8.12\",\n sha256 = \"5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ahash/0.8.12/download\"],\n strip_prefix = \"ahash-0.8.12\",\n build_file = Label(\"@crates//crates:BUILD.ahash-0.8.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__aho-corasick-1.1.3\",\n sha256 = \"8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/aho-corasick/1.1.3/download\"],\n strip_prefix = \"aho-corasick-1.1.3\",\n build_file = Label(\"@crates//crates:BUILD.aho-corasick-1.1.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__base64-0.13.1\",\n sha256 = \"9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/base64/0.13.1/download\"],\n strip_prefix = \"base64-0.13.1\",\n build_file = Label(\"@crates//crates:BUILD.base64-0.13.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__bit-set-0.8.0\",\n sha256 = \"08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/bit-set/0.8.0/download\"],\n strip_prefix = \"bit-set-0.8.0\",\n build_file = Label(\"@crates//crates:BUILD.bit-set-0.8.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__bit-vec-0.8.0\",\n sha256 = \"5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/bit-vec/0.8.0/download\"],\n strip_prefix = \"bit-vec-0.8.0\",\n build_file = Label(\"@crates//crates:BUILD.bit-vec-0.8.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__bitflags-2.9.1\",\n sha256 = \"1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/bitflags/2.9.1/download\"],\n strip_prefix = \"bitflags-2.9.1\",\n build_file = Label(\"@crates//crates:BUILD.bitflags-2.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__bumpalo-3.16.0\",\n sha256 = \"79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/bumpalo/3.16.0/download\"],\n strip_prefix = \"bumpalo-3.16.0\",\n build_file = Label(\"@crates//crates:BUILD.bumpalo-3.16.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__byteorder-1.5.0\",\n sha256 = \"1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/byteorder/1.5.0/download\"],\n strip_prefix = \"byteorder-1.5.0\",\n build_file = Label(\"@crates//crates:BUILD.byteorder-1.5.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__castaway-0.2.4\",\n sha256 = \"dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/castaway/0.2.4/download\"],\n strip_prefix = \"castaway-0.2.4\",\n build_file = Label(\"@crates//crates:BUILD.castaway-0.2.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__cfg-if-1.0.0\",\n sha256 = \"baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/cfg-if/1.0.0/download\"],\n strip_prefix = \"cfg-if-1.0.0\",\n build_file = Label(\"@crates//crates:BUILD.cfg-if-1.0.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__compact_str-0.9.0\",\n sha256 = \"3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/compact_str/0.9.0/download\"],\n strip_prefix = \"compact_str-0.9.0\",\n build_file = Label(\"@crates//crates:BUILD.compact_str-0.9.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-deque-0.8.6\",\n sha256 = \"9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-deque/0.8.6/download\"],\n strip_prefix = \"crossbeam-deque-0.8.6\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-deque-0.8.6.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-epoch-0.9.18\",\n sha256 = \"5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-epoch/0.9.18/download\"],\n strip_prefix = \"crossbeam-epoch-0.9.18\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-epoch-0.9.18.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-utils-0.8.21\",\n sha256 = \"d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-utils/0.8.21/download\"],\n strip_prefix = \"crossbeam-utils-0.8.21\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-utils-0.8.21.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling-0.20.10\",\n sha256 = \"6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling/0.20.10/download\"],\n strip_prefix = \"darling-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling_core-0.20.10\",\n sha256 = \"95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling_core/0.20.10/download\"],\n strip_prefix = \"darling_core-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling_core-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling_macro-0.20.10\",\n sha256 = \"d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling_macro/0.20.10/download\"],\n strip_prefix = \"darling_macro-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling_macro-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__dary_heap-0.3.7\",\n sha256 = \"04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/dary_heap/0.3.7/download\"],\n strip_prefix = \"dary_heap-0.3.7\",\n build_file = Label(\"@crates//crates:BUILD.dary_heap-0.3.7.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder-0.20.2\",\n sha256 = \"507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder/0.20.2/download\"],\n strip_prefix = \"derive_builder-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder_core-0.20.2\",\n sha256 = \"2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder_core/0.20.2/download\"],\n strip_prefix = \"derive_builder_core-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder_core-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder_macro-0.20.2\",\n sha256 = \"ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder_macro/0.20.2/download\"],\n strip_prefix = \"derive_builder_macro-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder_macro-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__either-1.13.0\",\n sha256 = \"60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/either/1.13.0/download\"],\n strip_prefix = \"either-1.13.0\",\n build_file = Label(\"@crates//crates:BUILD.either-1.13.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__esaxx-rs-0.1.10\",\n sha256 = \"d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/esaxx-rs/0.1.10/download\"],\n strip_prefix = \"esaxx-rs-0.1.10\",\n build_file = Label(\"@crates//crates:BUILD.esaxx-rs-0.1.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__fancy-regex-0.14.0\",\n sha256 = \"6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/fancy-regex/0.14.0/download\"],\n strip_prefix = \"fancy-regex-0.14.0\",\n build_file = Label(\"@crates//crates:BUILD.fancy-regex-0.14.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__fnv-1.0.7\",\n sha256 = \"3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/fnv/1.0.7/download\"],\n strip_prefix = \"fnv-1.0.7\",\n build_file = Label(\"@crates//crates:BUILD.fnv-1.0.7.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__getrandom-0.3.3\",\n sha256 = \"26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/getrandom/0.3.3/download\"],\n strip_prefix = \"getrandom-0.3.3\",\n build_file = Label(\"@crates//crates:BUILD.getrandom-0.3.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ident_case-1.0.1\",\n sha256 = \"b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ident_case/1.0.1/download\"],\n strip_prefix = \"ident_case-1.0.1\",\n build_file = Label(\"@crates//crates:BUILD.ident_case-1.0.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__itertools-0.14.0\",\n sha256 = \"2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/itertools/0.14.0/download\"],\n strip_prefix = \"itertools-0.14.0\",\n build_file = Label(\"@crates//crates:BUILD.itertools-0.14.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__itoa-1.0.14\",\n sha256 = \"d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/itoa/1.0.14/download\"],\n strip_prefix = \"itoa-1.0.14\",\n build_file = Label(\"@crates//crates:BUILD.itoa-1.0.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__js-sys-0.3.77\",\n sha256 = \"1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/js-sys/0.3.77/download\"],\n strip_prefix = \"js-sys-0.3.77\",\n build_file = Label(\"@crates//crates:BUILD.js-sys-0.3.77.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__libc-0.2.169\",\n sha256 = \"b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/libc/0.2.169/download\"],\n strip_prefix = \"libc-0.2.169\",\n build_file = Label(\"@crates//crates:BUILD.libc-0.2.169.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__log-0.4.22\",\n sha256 = \"a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/log/0.4.22/download\"],\n strip_prefix = \"log-0.4.22\",\n build_file = Label(\"@crates//crates:BUILD.log-0.4.22.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__macro_rules_attribute-0.2.0\",\n sha256 = \"8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/macro_rules_attribute/0.2.0/download\"],\n strip_prefix = \"macro_rules_attribute-0.2.0\",\n build_file = Label(\"@crates//crates:BUILD.macro_rules_attribute-0.2.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__macro_rules_attribute-proc_macro-0.2.0\",\n sha256 = \"b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/macro_rules_attribute-proc_macro/0.2.0/download\"],\n strip_prefix = \"macro_rules_attribute-proc_macro-0.2.0\",\n build_file = Label(\"@crates//crates:BUILD.macro_rules_attribute-proc_macro-0.2.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__memchr-2.7.4\",\n sha256 = \"78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/memchr/2.7.4/download\"],\n strip_prefix = \"memchr-2.7.4\",\n build_file = Label(\"@crates//crates:BUILD.memchr-2.7.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__minimal-lexical-0.2.1\",\n sha256 = \"68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/minimal-lexical/0.2.1/download\"],\n strip_prefix = \"minimal-lexical-0.2.1\",\n build_file = Label(\"@crates//crates:BUILD.minimal-lexical-0.2.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__monostate-0.1.13\",\n sha256 = \"0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/monostate/0.1.13/download\"],\n strip_prefix = \"monostate-0.1.13\",\n build_file = Label(\"@crates//crates:BUILD.monostate-0.1.13.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__monostate-impl-0.1.13\",\n sha256 = \"a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/monostate-impl/0.1.13/download\"],\n strip_prefix = \"monostate-impl-0.1.13\",\n build_file = Label(\"@crates//crates:BUILD.monostate-impl-0.1.13.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__nom-7.1.3\",\n sha256 = \"d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/nom/7.1.3/download\"],\n strip_prefix = \"nom-7.1.3\",\n build_file = Label(\"@crates//crates:BUILD.nom-7.1.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__once_cell-1.20.2\",\n sha256 = \"1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/once_cell/1.20.2/download\"],\n strip_prefix = \"once_cell-1.20.2\",\n build_file = Label(\"@crates//crates:BUILD.once_cell-1.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__paste-1.0.15\",\n sha256 = \"57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/paste/1.0.15/download\"],\n strip_prefix = \"paste-1.0.15\",\n build_file = Label(\"@crates//crates:BUILD.paste-1.0.15.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ppv-lite86-0.2.20\",\n sha256 = \"77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ppv-lite86/0.2.20/download\"],\n strip_prefix = \"ppv-lite86-0.2.20\",\n build_file = Label(\"@crates//crates:BUILD.ppv-lite86-0.2.20.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__proc-macro2-1.0.92\",\n sha256 = \"37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/proc-macro2/1.0.92/download\"],\n strip_prefix = \"proc-macro2-1.0.92\",\n build_file = Label(\"@crates//crates:BUILD.proc-macro2-1.0.92.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__quote-1.0.37\",\n sha256 = \"b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/quote/1.0.37/download\"],\n strip_prefix = \"quote-1.0.37\",\n build_file = Label(\"@crates//crates:BUILD.quote-1.0.37.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__r-efi-5.3.0\",\n sha256 = \"69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/r-efi/5.3.0/download\"],\n strip_prefix = \"r-efi-5.3.0\",\n build_file = Label(\"@crates//crates:BUILD.r-efi-5.3.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand-0.9.1\",\n sha256 = \"9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand/0.9.1/download\"],\n strip_prefix = \"rand-0.9.1\",\n build_file = Label(\"@crates//crates:BUILD.rand-0.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand_chacha-0.9.0\",\n sha256 = \"d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand_chacha/0.9.0/download\"],\n strip_prefix = \"rand_chacha-0.9.0\",\n build_file = Label(\"@crates//crates:BUILD.rand_chacha-0.9.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand_core-0.9.3\",\n sha256 = \"99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand_core/0.9.3/download\"],\n strip_prefix = \"rand_core-0.9.3\",\n build_file = Label(\"@crates//crates:BUILD.rand_core-0.9.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-1.10.0\",\n sha256 = \"b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon/1.10.0/download\"],\n strip_prefix = \"rayon-1.10.0\",\n build_file = Label(\"@crates//crates:BUILD.rayon-1.10.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-cond-0.4.0\",\n sha256 = \"2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon-cond/0.4.0/download\"],\n strip_prefix = \"rayon-cond-0.4.0\",\n build_file = Label(\"@crates//crates:BUILD.rayon-cond-0.4.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-core-1.12.1\",\n sha256 = \"1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon-core/1.12.1/download\"],\n strip_prefix = \"rayon-core-1.12.1\",\n build_file = Label(\"@crates//crates:BUILD.rayon-core-1.12.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-1.11.1\",\n sha256 = \"b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex/1.11.1/download\"],\n strip_prefix = \"regex-1.11.1\",\n build_file = Label(\"@crates//crates:BUILD.regex-1.11.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-automata-0.4.9\",\n sha256 = \"809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-automata/0.4.9/download\"],\n strip_prefix = \"regex-automata-0.4.9\",\n build_file = Label(\"@crates//crates:BUILD.regex-automata-0.4.9.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-syntax-0.8.5\",\n sha256 = \"2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-syntax/0.8.5/download\"],\n strip_prefix = \"regex-syntax-0.8.5\",\n build_file = Label(\"@crates//crates:BUILD.regex-syntax-0.8.5.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rustversion-1.0.21\",\n sha256 = \"8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rustversion/1.0.21/download\"],\n strip_prefix = \"rustversion-1.0.21\",\n build_file = Label(\"@crates//crates:BUILD.rustversion-1.0.21.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ryu-1.0.18\",\n sha256 = \"f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ryu/1.0.18/download\"],\n strip_prefix = \"ryu-1.0.18\",\n build_file = Label(\"@crates//crates:BUILD.ryu-1.0.18.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde-1.0.216\",\n sha256 = \"0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde/1.0.216/download\"],\n strip_prefix = \"serde-1.0.216\",\n build_file = Label(\"@crates//crates:BUILD.serde-1.0.216.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde_derive-1.0.216\",\n sha256 = \"46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde_derive/1.0.216/download\"],\n strip_prefix = \"serde_derive-1.0.216\",\n build_file = Label(\"@crates//crates:BUILD.serde_derive-1.0.216.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde_json-1.0.134\",\n sha256 = \"d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde_json/1.0.134/download\"],\n strip_prefix = \"serde_json-1.0.134\",\n build_file = Label(\"@crates//crates:BUILD.serde_json-1.0.134.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__smallvec-1.13.2\",\n sha256 = \"3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/smallvec/1.13.2/download\"],\n strip_prefix = \"smallvec-1.13.2\",\n build_file = Label(\"@crates//crates:BUILD.smallvec-1.13.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__spm_precompiled-0.1.4\",\n sha256 = \"5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/spm_precompiled/0.1.4/download\"],\n strip_prefix = \"spm_precompiled-0.1.4\",\n build_file = Label(\"@crates//crates:BUILD.spm_precompiled-0.1.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__static_assertions-1.1.0\",\n sha256 = \"a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/static_assertions/1.1.0/download\"],\n strip_prefix = \"static_assertions-1.1.0\",\n build_file = Label(\"@crates//crates:BUILD.static_assertions-1.1.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__strsim-0.11.1\",\n sha256 = \"7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/strsim/0.11.1/download\"],\n strip_prefix = \"strsim-0.11.1\",\n build_file = Label(\"@crates//crates:BUILD.strsim-0.11.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__syn-2.0.91\",\n sha256 = \"d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/syn/2.0.91/download\"],\n strip_prefix = \"syn-2.0.91\",\n build_file = Label(\"@crates//crates:BUILD.syn-2.0.91.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__thiserror-2.0.12\",\n sha256 = \"567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/thiserror/2.0.12/download\"],\n strip_prefix = \"thiserror-2.0.12\",\n build_file = Label(\"@crates//crates:BUILD.thiserror-2.0.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__thiserror-impl-2.0.12\",\n sha256 = \"7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/thiserror-impl/2.0.12/download\"],\n strip_prefix = \"thiserror-impl-2.0.12\",\n build_file = Label(\"@crates//crates:BUILD.thiserror-impl-2.0.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__tokenizers-0.21.2\",\n sha256 = \"4c3846d8588abed0daba25a0e47edd58ea15e450a6088b2575f5116fdb0b27ca\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/tokenizers/0.21.2/download\"],\n strip_prefix = \"tokenizers-0.21.2\",\n build_file = Label(\"@crates//crates:BUILD.tokenizers-0.21.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-ident-1.0.14\",\n sha256 = \"adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-ident/1.0.14/download\"],\n strip_prefix = \"unicode-ident-1.0.14\",\n build_file = Label(\"@crates//crates:BUILD.unicode-ident-1.0.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-normalization-alignments-0.1.12\",\n sha256 = \"43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-normalization-alignments/0.1.12/download\"],\n strip_prefix = \"unicode-normalization-alignments-0.1.12\",\n build_file = Label(\"@crates//crates:BUILD.unicode-normalization-alignments-0.1.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-segmentation-1.12.0\",\n sha256 = \"f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-segmentation/1.12.0/download\"],\n strip_prefix = \"unicode-segmentation-1.12.0\",\n build_file = Label(\"@crates//crates:BUILD.unicode-segmentation-1.12.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode_categories-0.1.1\",\n sha256 = \"39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode_categories/0.1.1/download\"],\n strip_prefix = \"unicode_categories-0.1.1\",\n build_file = Label(\"@crates//crates:BUILD.unicode_categories-0.1.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__version_check-0.9.5\",\n sha256 = \"0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/version_check/0.9.5/download\"],\n strip_prefix = \"version_check-0.9.5\",\n build_file = Label(\"@crates//crates:BUILD.version_check-0.9.5.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasi-0.14.2-wasi-0.2.4\",\n sha256 = \"9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasi/0.14.2+wasi-0.2.4/download\"],\n strip_prefix = \"wasi-0.14.2+wasi-0.2.4\",\n build_file = Label(\"@crates//crates:BUILD.wasi-0.14.2+wasi-0.2.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasm-bindgen-0.2.100\",\n sha256 = \"1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasm-bindgen/0.2.100/download\"],\n strip_prefix = \"wasm-bindgen-0.2.100\",\n build_file = Label(\"@crates//crates:BUILD.wasm-bindgen-0.2.100.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasm-bindgen-backend-0.2.100\",\n sha256 = \"2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasm-bindgen-backend/0.2.100/download\"],\n strip_prefix = \"wasm-bindgen-backend-0.2.100\",\n build_file = Label(\"@crates//crates:BUILD.wasm-bindgen-backend-0.2.100.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasm-bindgen-macro-0.2.100\",\n sha256 = \"7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasm-bindgen-macro/0.2.100/download\"],\n strip_prefix = \"wasm-bindgen-macro-0.2.100\",\n build_file = Label(\"@crates//crates:BUILD.wasm-bindgen-macro-0.2.100.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasm-bindgen-macro-support-0.2.100\",\n sha256 = \"8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasm-bindgen-macro-support/0.2.100/download\"],\n strip_prefix = \"wasm-bindgen-macro-support-0.2.100\",\n build_file = Label(\"@crates//crates:BUILD.wasm-bindgen-macro-support-0.2.100.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasm-bindgen-shared-0.2.100\",\n sha256 = \"1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasm-bindgen-shared/0.2.100/download\"],\n strip_prefix = \"wasm-bindgen-shared-0.2.100\",\n build_file = Label(\"@crates//crates:BUILD.wasm-bindgen-shared-0.2.100.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wit-bindgen-rt-0.39.0\",\n sha256 = \"6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wit-bindgen-rt/0.39.0/download\"],\n strip_prefix = \"wit-bindgen-rt-0.39.0\",\n build_file = Label(\"@crates//crates:BUILD.wit-bindgen-rt-0.39.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-0.7.35\",\n sha256 = \"1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy/0.7.35/download\"],\n strip_prefix = \"zerocopy-0.7.35\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-0.7.35.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-0.8.26\",\n sha256 = \"1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy/0.8.26/download\"],\n strip_prefix = \"zerocopy-0.8.26\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-0.8.26.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-derive-0.7.35\",\n sha256 = \"fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy-derive/0.7.35/download\"],\n strip_prefix = \"zerocopy-derive-0.7.35\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-derive-0.7.35.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-derive-0.8.26\",\n sha256 = \"9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy-derive/0.8.26/download\"],\n strip_prefix = \"zerocopy-derive-0.8.26\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-derive-0.8.26.bazel\"),\n )\n\n return [\n struct(repo=\"crates__tokenizers-0.21.2\", is_dev_dep = False),\n ]\n" + "defs.bzl": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\"\"\"\n# `crates_repository` API\n\n- [aliases](#aliases)\n- [crate_deps](#crate_deps)\n- [all_crate_deps](#all_crate_deps)\n- [crate_repositories](#crate_repositories)\n\n\"\"\"\n\nload(\"@bazel_tools//tools/build_defs/repo:git.bzl\", \"new_git_repository\")\nload(\"@bazel_tools//tools/build_defs/repo:http.bzl\", \"http_archive\")\nload(\"@bazel_tools//tools/build_defs/repo:utils.bzl\", \"maybe\")\nload(\"@bazel_skylib//lib:selects.bzl\", \"selects\")\nload(\"@rules_rust//crate_universe/private:local_crate_mirror.bzl\", \"local_crate_mirror\")\n\n###############################################################################\n# MACROS API\n###############################################################################\n\n# An identifier that represent common dependencies (unconditional).\n_COMMON_CONDITION = \"\"\n\ndef _flatten_dependency_maps(all_dependency_maps):\n \"\"\"Flatten a list of dependency maps into one dictionary.\n\n Dependency maps have the following structure:\n\n ```python\n DEPENDENCIES_MAP = {\n # The first key in the map is a Bazel package\n # name of the workspace this file is defined in.\n \"workspace_member_package\": {\n\n # Not all dependencies are supported for all platforms.\n # the condition key is the condition required to be true\n # on the host platform.\n \"condition\": {\n\n # An alias to a crate target. # The label of the crate target the\n # Aliases are only crate names. # package name refers to.\n \"package_name\": \"@full//:label\",\n }\n }\n }\n ```\n\n Args:\n all_dependency_maps (list): A list of dicts as described above\n\n Returns:\n dict: A dictionary as described above\n \"\"\"\n dependencies = {}\n\n for workspace_deps_map in all_dependency_maps:\n for pkg_name, conditional_deps_map in workspace_deps_map.items():\n if pkg_name not in dependencies:\n non_frozen_map = dict()\n for key, values in conditional_deps_map.items():\n non_frozen_map.update({key: dict(values.items())})\n dependencies.setdefault(pkg_name, non_frozen_map)\n continue\n\n for condition, deps_map in conditional_deps_map.items():\n # If the condition has not been recorded, do so and continue\n if condition not in dependencies[pkg_name]:\n dependencies[pkg_name].setdefault(condition, dict(deps_map.items()))\n continue\n\n # Alert on any miss-matched dependencies\n inconsistent_entries = []\n for crate_name, crate_label in deps_map.items():\n existing = dependencies[pkg_name][condition].get(crate_name)\n if existing and existing != crate_label:\n inconsistent_entries.append((crate_name, existing, crate_label))\n dependencies[pkg_name][condition].update({crate_name: crate_label})\n\n return dependencies\n\ndef crate_deps(deps, package_name = None):\n \"\"\"Finds the fully qualified label of the requested crates for the package where this macro is called.\n\n Args:\n deps (list): The desired list of crate targets.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()`.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if not deps:\n return []\n\n if package_name == None:\n package_name = native.package_name()\n\n # Join both sets of dependencies\n dependencies = _flatten_dependency_maps([\n _NORMAL_DEPENDENCIES,\n _NORMAL_DEV_DEPENDENCIES,\n _PROC_MACRO_DEPENDENCIES,\n _PROC_MACRO_DEV_DEPENDENCIES,\n _BUILD_DEPENDENCIES,\n _BUILD_PROC_MACRO_DEPENDENCIES,\n ]).pop(package_name, {})\n\n # Combine all conditional packages so we can easily index over a flat list\n # TODO: Perhaps this should actually return select statements and maintain\n # the conditionals of the dependencies\n flat_deps = {}\n for deps_set in dependencies.values():\n for crate_name, crate_label in deps_set.items():\n flat_deps.update({crate_name: crate_label})\n\n missing_crates = []\n crate_targets = []\n for crate_target in deps:\n if crate_target not in flat_deps:\n missing_crates.append(crate_target)\n else:\n crate_targets.append(flat_deps[crate_target])\n\n if missing_crates:\n fail(\"Could not find crates `{}` among dependencies of `{}`. Available dependencies were `{}`\".format(\n missing_crates,\n package_name,\n dependencies,\n ))\n\n return crate_targets\n\ndef all_crate_deps(\n normal = False, \n normal_dev = False, \n proc_macro = False, \n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Finds the fully qualified label of all requested direct crate dependencies \\\n for the package where this macro is called.\n\n If no parameters are set, all normal dependencies are returned. Setting any one flag will\n otherwise impact the contents of the returned list.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_dependency_maps = []\n if normal:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n if normal_dev:\n all_dependency_maps.append(_NORMAL_DEV_DEPENDENCIES)\n if proc_macro:\n all_dependency_maps.append(_PROC_MACRO_DEPENDENCIES)\n if proc_macro_dev:\n all_dependency_maps.append(_PROC_MACRO_DEV_DEPENDENCIES)\n if build:\n all_dependency_maps.append(_BUILD_DEPENDENCIES)\n if build_proc_macro:\n all_dependency_maps.append(_BUILD_PROC_MACRO_DEPENDENCIES)\n\n # Default to always using normal dependencies\n if not all_dependency_maps:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n\n dependencies = _flatten_dependency_maps(all_dependency_maps).pop(package_name, None)\n\n if not dependencies:\n if dependencies == None:\n fail(\"Tried to get all_crate_deps for package \" + package_name + \" but that package had no Cargo.toml file\")\n else:\n return []\n\n crate_deps = list(dependencies.pop(_COMMON_CONDITION, {}).values())\n for condition, deps in dependencies.items():\n crate_deps += selects.with_or({\n tuple(_CONDITIONS[condition]): deps.values(),\n \"//conditions:default\": [],\n })\n\n return crate_deps\n\ndef aliases(\n normal = False,\n normal_dev = False,\n proc_macro = False,\n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Produces a map of Crate alias names to their original label\n\n If no dependency kinds are specified, `normal` and `proc_macro` are used by default.\n Setting any one flag will otherwise determine the contents of the returned dict.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n dict: The aliases of all associated packages\n \"\"\"\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_aliases_maps = []\n if normal:\n all_aliases_maps.append(_NORMAL_ALIASES)\n if normal_dev:\n all_aliases_maps.append(_NORMAL_DEV_ALIASES)\n if proc_macro:\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n if proc_macro_dev:\n all_aliases_maps.append(_PROC_MACRO_DEV_ALIASES)\n if build:\n all_aliases_maps.append(_BUILD_ALIASES)\n if build_proc_macro:\n all_aliases_maps.append(_BUILD_PROC_MACRO_ALIASES)\n\n # Default to always using normal aliases\n if not all_aliases_maps:\n all_aliases_maps.append(_NORMAL_ALIASES)\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n\n aliases = _flatten_dependency_maps(all_aliases_maps).pop(package_name, None)\n\n if not aliases:\n return dict()\n\n common_items = aliases.pop(_COMMON_CONDITION, {}).items()\n\n # If there are only common items in the dictionary, immediately return them\n if not len(aliases.keys()) == 1:\n return dict(common_items)\n\n # Build a single select statement where each conditional has accounted for the\n # common set of aliases.\n crate_aliases = {\"//conditions:default\": dict(common_items)}\n for condition, deps in aliases.items():\n condition_triples = _CONDITIONS[condition]\n for triple in condition_triples:\n if triple in crate_aliases:\n crate_aliases[triple].update(deps)\n else:\n crate_aliases.update({triple: dict(deps.items() + common_items)})\n\n return select(crate_aliases)\n\n###############################################################################\n# WORKSPACE MEMBER DEPS AND ALIASES\n###############################################################################\n\n_NORMAL_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n _COMMON_CONDITION: {\n \"tokenizers\": Label(\"@crates//:tokenizers-0.21.4\"),\n },\n },\n}\n\n\n_NORMAL_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n _COMMON_CONDITION: {\n },\n },\n}\n\n\n_NORMAL_DEV_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_NORMAL_DEV_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEV_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEV_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_CONDITIONS = {\n \"aarch64-apple-darwin\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\"],\n \"aarch64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\"],\n \"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\",\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\": [],\n \"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\": [],\n \"cfg(any())\": [],\n \"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\": [],\n \"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\": [],\n \"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\": [],\n \"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\",\"@rules_rust//rust/platform:x86_64-apple-darwin\"],\n \"cfg(not(all(target_arch = \\\"arm\\\", target_os = \\\"none\\\")))\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\",\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\",\"@rules_rust//rust/platform:x86_64-apple-darwin\",\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"cfg(target_os = \\\"netbsd\\\")\": [],\n \"cfg(target_os = \\\"solaris\\\")\": [],\n \"cfg(target_os = \\\"vxworks\\\")\": [],\n \"cfg(windows)\": [],\n \"x86_64-apple-darwin\": [\"@rules_rust//rust/platform:x86_64-apple-darwin\"],\n \"x86_64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n}\n\n###############################################################################\n\ndef crate_repositories():\n \"\"\"A macro for defining repositories for all generated crates.\n\n Returns:\n A list of repos visible to the module through the module extension.\n \"\"\"\n maybe(\n http_archive,\n name = \"crates__ahash-0.8.12\",\n sha256 = \"5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ahash/0.8.12/download\"],\n strip_prefix = \"ahash-0.8.12\",\n build_file = Label(\"@crates//crates:BUILD.ahash-0.8.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__aho-corasick-1.1.3\",\n sha256 = \"8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/aho-corasick/1.1.3/download\"],\n strip_prefix = \"aho-corasick-1.1.3\",\n build_file = Label(\"@crates//crates:BUILD.aho-corasick-1.1.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__base64-0.13.1\",\n sha256 = \"9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/base64/0.13.1/download\"],\n strip_prefix = \"base64-0.13.1\",\n build_file = Label(\"@crates//crates:BUILD.base64-0.13.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__bitflags-2.9.1\",\n sha256 = \"1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/bitflags/2.9.1/download\"],\n strip_prefix = \"bitflags-2.9.1\",\n build_file = Label(\"@crates//crates:BUILD.bitflags-2.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__byteorder-1.5.0\",\n sha256 = \"1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/byteorder/1.5.0/download\"],\n strip_prefix = \"byteorder-1.5.0\",\n build_file = Label(\"@crates//crates:BUILD.byteorder-1.5.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__castaway-0.2.4\",\n sha256 = \"dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/castaway/0.2.4/download\"],\n strip_prefix = \"castaway-0.2.4\",\n build_file = Label(\"@crates//crates:BUILD.castaway-0.2.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__cc-1.2.33\",\n sha256 = \"3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/cc/1.2.33/download\"],\n strip_prefix = \"cc-1.2.33\",\n build_file = Label(\"@crates//crates:BUILD.cc-1.2.33.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__cfg-if-1.0.0\",\n sha256 = \"baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/cfg-if/1.0.0/download\"],\n strip_prefix = \"cfg-if-1.0.0\",\n build_file = Label(\"@crates//crates:BUILD.cfg-if-1.0.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__compact_str-0.9.0\",\n sha256 = \"3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/compact_str/0.9.0/download\"],\n strip_prefix = \"compact_str-0.9.0\",\n build_file = Label(\"@crates//crates:BUILD.compact_str-0.9.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-deque-0.8.6\",\n sha256 = \"9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-deque/0.8.6/download\"],\n strip_prefix = \"crossbeam-deque-0.8.6\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-deque-0.8.6.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-epoch-0.9.18\",\n sha256 = \"5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-epoch/0.9.18/download\"],\n strip_prefix = \"crossbeam-epoch-0.9.18\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-epoch-0.9.18.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-utils-0.8.21\",\n sha256 = \"d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-utils/0.8.21/download\"],\n strip_prefix = \"crossbeam-utils-0.8.21\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-utils-0.8.21.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling-0.20.10\",\n sha256 = \"6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling/0.20.10/download\"],\n strip_prefix = \"darling-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling_core-0.20.10\",\n sha256 = \"95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling_core/0.20.10/download\"],\n strip_prefix = \"darling_core-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling_core-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling_macro-0.20.10\",\n sha256 = \"d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling_macro/0.20.10/download\"],\n strip_prefix = \"darling_macro-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling_macro-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__dary_heap-0.3.7\",\n sha256 = \"04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/dary_heap/0.3.7/download\"],\n strip_prefix = \"dary_heap-0.3.7\",\n build_file = Label(\"@crates//crates:BUILD.dary_heap-0.3.7.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder-0.20.2\",\n sha256 = \"507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder/0.20.2/download\"],\n strip_prefix = \"derive_builder-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder_core-0.20.2\",\n sha256 = \"2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder_core/0.20.2/download\"],\n strip_prefix = \"derive_builder_core-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder_core-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder_macro-0.20.2\",\n sha256 = \"ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder_macro/0.20.2/download\"],\n strip_prefix = \"derive_builder_macro-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder_macro-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__either-1.13.0\",\n sha256 = \"60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/either/1.13.0/download\"],\n strip_prefix = \"either-1.13.0\",\n build_file = Label(\"@crates//crates:BUILD.either-1.13.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__esaxx-rs-0.1.10\",\n sha256 = \"d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/esaxx-rs/0.1.10/download\"],\n strip_prefix = \"esaxx-rs-0.1.10\",\n build_file = Label(\"@crates//crates:BUILD.esaxx-rs-0.1.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__fnv-1.0.7\",\n sha256 = \"3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/fnv/1.0.7/download\"],\n strip_prefix = \"fnv-1.0.7\",\n build_file = Label(\"@crates//crates:BUILD.fnv-1.0.7.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__getrandom-0.3.3\",\n sha256 = \"26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/getrandom/0.3.3/download\"],\n strip_prefix = \"getrandom-0.3.3\",\n build_file = Label(\"@crates//crates:BUILD.getrandom-0.3.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ident_case-1.0.1\",\n sha256 = \"b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ident_case/1.0.1/download\"],\n strip_prefix = \"ident_case-1.0.1\",\n build_file = Label(\"@crates//crates:BUILD.ident_case-1.0.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__itertools-0.14.0\",\n sha256 = \"2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/itertools/0.14.0/download\"],\n strip_prefix = \"itertools-0.14.0\",\n build_file = Label(\"@crates//crates:BUILD.itertools-0.14.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__itoa-1.0.14\",\n sha256 = \"d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/itoa/1.0.14/download\"],\n strip_prefix = \"itoa-1.0.14\",\n build_file = Label(\"@crates//crates:BUILD.itoa-1.0.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__libc-0.2.169\",\n sha256 = \"b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/libc/0.2.169/download\"],\n strip_prefix = \"libc-0.2.169\",\n build_file = Label(\"@crates//crates:BUILD.libc-0.2.169.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__log-0.4.22\",\n sha256 = \"a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/log/0.4.22/download\"],\n strip_prefix = \"log-0.4.22\",\n build_file = Label(\"@crates//crates:BUILD.log-0.4.22.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__macro_rules_attribute-0.2.0\",\n sha256 = \"8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/macro_rules_attribute/0.2.0/download\"],\n strip_prefix = \"macro_rules_attribute-0.2.0\",\n build_file = Label(\"@crates//crates:BUILD.macro_rules_attribute-0.2.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__macro_rules_attribute-proc_macro-0.2.0\",\n sha256 = \"b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/macro_rules_attribute-proc_macro/0.2.0/download\"],\n strip_prefix = \"macro_rules_attribute-proc_macro-0.2.0\",\n build_file = Label(\"@crates//crates:BUILD.macro_rules_attribute-proc_macro-0.2.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__memchr-2.7.4\",\n sha256 = \"78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/memchr/2.7.4/download\"],\n strip_prefix = \"memchr-2.7.4\",\n build_file = Label(\"@crates//crates:BUILD.memchr-2.7.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__minimal-lexical-0.2.1\",\n sha256 = \"68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/minimal-lexical/0.2.1/download\"],\n strip_prefix = \"minimal-lexical-0.2.1\",\n build_file = Label(\"@crates//crates:BUILD.minimal-lexical-0.2.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__monostate-0.1.13\",\n sha256 = \"0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/monostate/0.1.13/download\"],\n strip_prefix = \"monostate-0.1.13\",\n build_file = Label(\"@crates//crates:BUILD.monostate-0.1.13.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__monostate-impl-0.1.13\",\n sha256 = \"a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/monostate-impl/0.1.13/download\"],\n strip_prefix = \"monostate-impl-0.1.13\",\n build_file = Label(\"@crates//crates:BUILD.monostate-impl-0.1.13.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__nom-7.1.3\",\n sha256 = \"d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/nom/7.1.3/download\"],\n strip_prefix = \"nom-7.1.3\",\n build_file = Label(\"@crates//crates:BUILD.nom-7.1.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__once_cell-1.20.2\",\n sha256 = \"1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/once_cell/1.20.2/download\"],\n strip_prefix = \"once_cell-1.20.2\",\n build_file = Label(\"@crates//crates:BUILD.once_cell-1.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__onig-6.5.1\",\n sha256 = \"336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/onig/6.5.1/download\"],\n strip_prefix = \"onig-6.5.1\",\n build_file = Label(\"@crates//crates:BUILD.onig-6.5.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__onig_sys-69.9.1\",\n sha256 = \"c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/onig_sys/69.9.1/download\"],\n strip_prefix = \"onig_sys-69.9.1\",\n build_file = Label(\"@crates//crates:BUILD.onig_sys-69.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__paste-1.0.15\",\n sha256 = \"57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/paste/1.0.15/download\"],\n strip_prefix = \"paste-1.0.15\",\n build_file = Label(\"@crates//crates:BUILD.paste-1.0.15.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__pkg-config-0.3.32\",\n sha256 = \"7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/pkg-config/0.3.32/download\"],\n strip_prefix = \"pkg-config-0.3.32\",\n build_file = Label(\"@crates//crates:BUILD.pkg-config-0.3.32.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ppv-lite86-0.2.20\",\n sha256 = \"77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ppv-lite86/0.2.20/download\"],\n strip_prefix = \"ppv-lite86-0.2.20\",\n build_file = Label(\"@crates//crates:BUILD.ppv-lite86-0.2.20.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__proc-macro2-1.0.92\",\n sha256 = \"37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/proc-macro2/1.0.92/download\"],\n strip_prefix = \"proc-macro2-1.0.92\",\n build_file = Label(\"@crates//crates:BUILD.proc-macro2-1.0.92.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__quote-1.0.37\",\n sha256 = \"b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/quote/1.0.37/download\"],\n strip_prefix = \"quote-1.0.37\",\n build_file = Label(\"@crates//crates:BUILD.quote-1.0.37.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__r-efi-5.3.0\",\n sha256 = \"69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/r-efi/5.3.0/download\"],\n strip_prefix = \"r-efi-5.3.0\",\n build_file = Label(\"@crates//crates:BUILD.r-efi-5.3.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand-0.9.1\",\n sha256 = \"9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand/0.9.1/download\"],\n strip_prefix = \"rand-0.9.1\",\n build_file = Label(\"@crates//crates:BUILD.rand-0.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand_chacha-0.9.0\",\n sha256 = \"d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand_chacha/0.9.0/download\"],\n strip_prefix = \"rand_chacha-0.9.0\",\n build_file = Label(\"@crates//crates:BUILD.rand_chacha-0.9.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand_core-0.9.3\",\n sha256 = \"99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand_core/0.9.3/download\"],\n strip_prefix = \"rand_core-0.9.3\",\n build_file = Label(\"@crates//crates:BUILD.rand_core-0.9.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-1.10.0\",\n sha256 = \"b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon/1.10.0/download\"],\n strip_prefix = \"rayon-1.10.0\",\n build_file = Label(\"@crates//crates:BUILD.rayon-1.10.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-cond-0.4.0\",\n sha256 = \"2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon-cond/0.4.0/download\"],\n strip_prefix = \"rayon-cond-0.4.0\",\n build_file = Label(\"@crates//crates:BUILD.rayon-cond-0.4.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-core-1.12.1\",\n sha256 = \"1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon-core/1.12.1/download\"],\n strip_prefix = \"rayon-core-1.12.1\",\n build_file = Label(\"@crates//crates:BUILD.rayon-core-1.12.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-1.11.1\",\n sha256 = \"b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex/1.11.1/download\"],\n strip_prefix = \"regex-1.11.1\",\n build_file = Label(\"@crates//crates:BUILD.regex-1.11.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-automata-0.4.9\",\n sha256 = \"809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-automata/0.4.9/download\"],\n strip_prefix = \"regex-automata-0.4.9\",\n build_file = Label(\"@crates//crates:BUILD.regex-automata-0.4.9.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-syntax-0.8.5\",\n sha256 = \"2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-syntax/0.8.5/download\"],\n strip_prefix = \"regex-syntax-0.8.5\",\n build_file = Label(\"@crates//crates:BUILD.regex-syntax-0.8.5.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rustversion-1.0.21\",\n sha256 = \"8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rustversion/1.0.21/download\"],\n strip_prefix = \"rustversion-1.0.21\",\n build_file = Label(\"@crates//crates:BUILD.rustversion-1.0.21.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ryu-1.0.18\",\n sha256 = \"f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ryu/1.0.18/download\"],\n strip_prefix = \"ryu-1.0.18\",\n build_file = Label(\"@crates//crates:BUILD.ryu-1.0.18.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde-1.0.216\",\n sha256 = \"0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde/1.0.216/download\"],\n strip_prefix = \"serde-1.0.216\",\n build_file = Label(\"@crates//crates:BUILD.serde-1.0.216.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde_derive-1.0.216\",\n sha256 = \"46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde_derive/1.0.216/download\"],\n strip_prefix = \"serde_derive-1.0.216\",\n build_file = Label(\"@crates//crates:BUILD.serde_derive-1.0.216.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde_json-1.0.134\",\n sha256 = \"d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde_json/1.0.134/download\"],\n strip_prefix = \"serde_json-1.0.134\",\n build_file = Label(\"@crates//crates:BUILD.serde_json-1.0.134.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__shlex-1.3.0\",\n sha256 = \"0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/shlex/1.3.0/download\"],\n strip_prefix = \"shlex-1.3.0\",\n build_file = Label(\"@crates//crates:BUILD.shlex-1.3.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__smallvec-1.13.2\",\n sha256 = \"3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/smallvec/1.13.2/download\"],\n strip_prefix = \"smallvec-1.13.2\",\n build_file = Label(\"@crates//crates:BUILD.smallvec-1.13.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__spm_precompiled-0.1.4\",\n sha256 = \"5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/spm_precompiled/0.1.4/download\"],\n strip_prefix = \"spm_precompiled-0.1.4\",\n build_file = Label(\"@crates//crates:BUILD.spm_precompiled-0.1.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__static_assertions-1.1.0\",\n sha256 = \"a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/static_assertions/1.1.0/download\"],\n strip_prefix = \"static_assertions-1.1.0\",\n build_file = Label(\"@crates//crates:BUILD.static_assertions-1.1.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__strsim-0.11.1\",\n sha256 = \"7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/strsim/0.11.1/download\"],\n strip_prefix = \"strsim-0.11.1\",\n build_file = Label(\"@crates//crates:BUILD.strsim-0.11.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__syn-2.0.91\",\n sha256 = \"d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/syn/2.0.91/download\"],\n strip_prefix = \"syn-2.0.91\",\n build_file = Label(\"@crates//crates:BUILD.syn-2.0.91.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__thiserror-2.0.12\",\n sha256 = \"567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/thiserror/2.0.12/download\"],\n strip_prefix = \"thiserror-2.0.12\",\n build_file = Label(\"@crates//crates:BUILD.thiserror-2.0.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__thiserror-impl-2.0.12\",\n sha256 = \"7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/thiserror-impl/2.0.12/download\"],\n strip_prefix = \"thiserror-impl-2.0.12\",\n build_file = Label(\"@crates//crates:BUILD.thiserror-impl-2.0.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__tokenizers-0.21.4\",\n sha256 = \"a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/tokenizers/0.21.4/download\"],\n strip_prefix = \"tokenizers-0.21.4\",\n build_file = Label(\"@crates//crates:BUILD.tokenizers-0.21.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-ident-1.0.14\",\n sha256 = \"adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-ident/1.0.14/download\"],\n strip_prefix = \"unicode-ident-1.0.14\",\n build_file = Label(\"@crates//crates:BUILD.unicode-ident-1.0.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-normalization-alignments-0.1.12\",\n sha256 = \"43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-normalization-alignments/0.1.12/download\"],\n strip_prefix = \"unicode-normalization-alignments-0.1.12\",\n build_file = Label(\"@crates//crates:BUILD.unicode-normalization-alignments-0.1.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-segmentation-1.12.0\",\n sha256 = \"f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-segmentation/1.12.0/download\"],\n strip_prefix = \"unicode-segmentation-1.12.0\",\n build_file = Label(\"@crates//crates:BUILD.unicode-segmentation-1.12.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode_categories-0.1.1\",\n sha256 = \"39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode_categories/0.1.1/download\"],\n strip_prefix = \"unicode_categories-0.1.1\",\n build_file = Label(\"@crates//crates:BUILD.unicode_categories-0.1.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__version_check-0.9.5\",\n sha256 = \"0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/version_check/0.9.5/download\"],\n strip_prefix = \"version_check-0.9.5\",\n build_file = Label(\"@crates//crates:BUILD.version_check-0.9.5.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasi-0.14.2-wasi-0.2.4\",\n sha256 = \"9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasi/0.14.2+wasi-0.2.4/download\"],\n strip_prefix = \"wasi-0.14.2+wasi-0.2.4\",\n build_file = Label(\"@crates//crates:BUILD.wasi-0.14.2+wasi-0.2.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wit-bindgen-rt-0.39.0\",\n sha256 = \"6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wit-bindgen-rt/0.39.0/download\"],\n strip_prefix = \"wit-bindgen-rt-0.39.0\",\n build_file = Label(\"@crates//crates:BUILD.wit-bindgen-rt-0.39.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-0.7.35\",\n sha256 = \"1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy/0.7.35/download\"],\n strip_prefix = \"zerocopy-0.7.35\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-0.7.35.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-0.8.26\",\n sha256 = \"1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy/0.8.26/download\"],\n strip_prefix = \"zerocopy-0.8.26\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-0.8.26.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-derive-0.7.35\",\n sha256 = \"fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy-derive/0.7.35/download\"],\n strip_prefix = \"zerocopy-derive-0.7.35\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-derive-0.7.35.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-derive-0.8.26\",\n sha256 = \"9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy-derive/0.8.26/download\"],\n strip_prefix = \"zerocopy-derive-0.8.26\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-derive-0.8.26.bazel\"),\n )\n\n return [\n struct(repo=\"crates__tokenizers-0.21.4\", is_dev_dep = False),\n ]\n" } } }, @@ -4910,38 +1010,6 @@ "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"base64\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=base64\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.13.1\",\n)\n" } }, - "crates__bit-set-0.8.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/bit-set/0.8.0/download" - ], - "strip_prefix": "bit-set-0.8.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"bit_set\",\n deps = [\n \"@crates__bit-vec-0.8.0//:bit_vec\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=bit-set\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.0\",\n)\n" - } - }, - "crates__bit-vec-0.8.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/bit-vec/0.8.0/download" - ], - "strip_prefix": "bit-vec-0.8.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"bit_vec\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=bit-vec\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.0\",\n)\n" - } - }, "crates__bitflags-2.9.1": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { @@ -4958,22 +1026,6 @@ "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"bitflags\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=bitflags\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.9.1\",\n)\n" } }, - "crates__bumpalo-3.16.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/bumpalo/3.16.0/download" - ], - "strip_prefix": "bumpalo-3.16.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"bumpalo\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=bumpalo\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"3.16.0\",\n)\n" - } - }, "crates__byteorder-1.5.0": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { @@ -5006,6 +1058,22 @@ "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"castaway\",\n proc_macro_deps = [\n \"@crates__rustversion-1.0.21//:rustversion\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=castaway\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.4\",\n)\n" } }, + "crates__cc-1.2.33": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/cc/1.2.33/download" + ], + "strip_prefix": "cc-1.2.33", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"cc\",\n deps = [\n \"@crates__shlex-1.3.0//:shlex\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=cc\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.2.33\",\n)\n" + } + }, "crates__cfg-if-1.0.0": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { @@ -5227,23 +1295,7 @@ "https://static.crates.io/crates/esaxx-rs/0.1.10/download" ], "strip_prefix": "esaxx-rs-0.1.10", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"esaxx_rs\",\n deps = [\n \"@crates__esaxx-rs-0.1.10//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=esaxx-rs\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.10\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2018\",\n pkg_name = \"esaxx-rs\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=esaxx-rs\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.1.10\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__fancy-regex-0.14.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/fancy-regex/0.14.0/download" - ], - "strip_prefix": "fancy-regex-0.14.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"fancy_regex\",\n deps = [\n \"@crates__bit-set-0.8.0//:bit_set\",\n \"@crates__regex-automata-0.4.9//:regex_automata\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"perf\",\n \"std\",\n \"unicode\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=fancy-regex\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.14.0\",\n)\n" + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"esaxx_rs\",\n deps = [\n \"@crates__esaxx-rs-0.1.10//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"cc\",\n \"cpp\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=esaxx-rs\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.10\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"cc\",\n \"cpp\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n deps = [\n \"@crates__cc-1.2.33//:cc\",\n ],\n edition = \"2018\",\n pkg_name = \"esaxx-rs\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=esaxx-rs\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.1.10\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" } }, "crates__fnv-1.0.7": { @@ -5275,7 +1327,7 @@ "https://static.crates.io/crates/getrandom/0.3.3/download" ], "strip_prefix": "getrandom-0.3.3", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"getrandom\",\n deps = [\n \"@crates__cfg-if-1.0.0//:cfg_if\",\n \"@crates__getrandom-0.3.3//:build_script_build\",\n ] + select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(any(target_os = \"macos\", target_os = \"openbsd\", target_os = \"vita\", target_os = \"emscripten\"))\n ],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(all(any(target_os = \"linux\", target_os = \"android\"), not(any(all(target_os = \"linux\", target_env = \"\"), getrandom_backend = \"custom\", getrandom_backend = \"linux_raw\", getrandom_backend = \"rdrand\", getrandom_backend = \"rndr\"))))\n ],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(any(target_os = \"macos\", target_os = \"openbsd\", target_os = \"vita\", target_os = \"emscripten\"))\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(all(any(target_os = \"linux\", target_os = \"android\"), not(any(all(target_os = \"linux\", target_env = \"\"), getrandom_backend = \"custom\", getrandom_backend = \"linux_raw\", getrandom_backend = \"rdrand\", getrandom_backend = \"rndr\"))))\n ],\n \"//conditions:default\": [],\n }),\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n \"wasm_js\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=getrandom\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.3\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n \"wasm_js\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"getrandom\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=getrandom\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.3.3\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"getrandom\",\n deps = [\n \"@crates__cfg-if-1.0.0//:cfg_if\",\n \"@crates__getrandom-0.3.3//:build_script_build\",\n ] + select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(any(target_os = \"macos\", target_os = \"openbsd\", target_os = \"vita\", target_os = \"emscripten\"))\n ],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(all(any(target_os = \"linux\", target_os = \"android\"), not(any(all(target_os = \"linux\", target_env = \"\"), getrandom_backend = \"custom\", getrandom_backend = \"linux_raw\", getrandom_backend = \"rdrand\", getrandom_backend = \"rndr\"))))\n ],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(any(target_os = \"macos\", target_os = \"openbsd\", target_os = \"vita\", target_os = \"emscripten\"))\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(all(any(target_os = \"linux\", target_os = \"android\"), not(any(all(target_os = \"linux\", target_env = \"\"), getrandom_backend = \"custom\", getrandom_backend = \"linux_raw\", getrandom_backend = \"rdrand\", getrandom_backend = \"rndr\"))))\n ],\n \"//conditions:default\": [],\n }),\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=getrandom\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.3\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"getrandom\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=getrandom\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.3.3\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" } }, "crates__ident_case-1.0.1": { @@ -5326,22 +1378,6 @@ "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"itoa\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=itoa\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.14\",\n)\n" } }, - "crates__js-sys-0.3.77": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/js-sys/0.3.77/download" - ], - "strip_prefix": "js-sys-0.3.77", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"js_sys\",\n deps = [\n \"@crates__once_cell-1.20.2//:once_cell\",\n \"@crates__wasm-bindgen-0.2.100//:wasm_bindgen\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=js-sys\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.77\",\n)\n" - } - }, "crates__libc-0.2.169": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { @@ -5499,7 +1535,39 @@ "https://static.crates.io/crates/once_cell/1.20.2/download" ], "strip_prefix": "once_cell-1.20.2", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"once_cell\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"race\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=once_cell\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.20.2\",\n)\n" + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"once_cell\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"default\",\n \"race\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=once_cell\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.20.2\",\n)\n" + } + }, + "crates__onig-6.5.1": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/onig/6.5.1/download" + ], + "strip_prefix": "onig-6.5.1", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"onig\",\n deps = [\n \"@crates__bitflags-2.9.1//:bitflags\",\n \"@crates__once_cell-1.20.2//:once_cell\",\n \"@crates__onig_sys-69.9.1//:onig_sys\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=onig\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"6.5.1\",\n)\n" + } + }, + "crates__onig_sys-69.9.1": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/onig_sys/69.9.1/download" + ], + "strip_prefix": "onig_sys-69.9.1", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"onig_sys\",\n deps = [\n \"@crates__onig_sys-69.9.1//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=onig_sys\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"69.9.1\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n deps = [\n \"@crates__cc-1.2.33//:cc\",\n \"@crates__pkg-config-0.3.32//:pkg_config\",\n ],\n edition = \"2021\",\n links = \"onig\",\n pkg_name = \"onig_sys\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=onig_sys\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"69.9.1\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" } }, "crates__paste-1.0.15": { @@ -5518,6 +1586,22 @@ "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"paste\",\n deps = [\n \"@crates__paste-1.0.15//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=paste\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.15\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2018\",\n pkg_name = \"paste\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=paste\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.15\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" } }, + "crates__pkg-config-0.3.32": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/pkg-config/0.3.32/download" + ], + "strip_prefix": "pkg-config-0.3.32", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"pkg_config\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=pkg-config\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.32\",\n)\n" + } + }, "crates__ppv-lite86-0.2.20": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { @@ -5707,7 +1791,7 @@ "https://static.crates.io/crates/regex-automata/0.4.9/download" ], "strip_prefix": "regex-automata-0.4.9", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"regex_automata\",\n deps = [\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__memchr-2.7.4//:memchr\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"dfa\",\n \"dfa-build\",\n \"dfa-onepass\",\n \"dfa-search\",\n \"hybrid\",\n \"meta\",\n \"nfa\",\n \"nfa-backtrack\",\n \"nfa-pikevm\",\n \"nfa-thompson\",\n \"perf\",\n \"perf-inline\",\n \"perf-literal\",\n \"perf-literal-multisubstring\",\n \"perf-literal-substring\",\n \"std\",\n \"syntax\",\n \"unicode\",\n \"unicode-age\",\n \"unicode-bool\",\n \"unicode-case\",\n \"unicode-gencat\",\n \"unicode-perl\",\n \"unicode-script\",\n \"unicode-segment\",\n \"unicode-word-boundary\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=regex-automata\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.4.9\",\n)\n" + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"regex_automata\",\n deps = [\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__memchr-2.7.4//:memchr\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"dfa-onepass\",\n \"hybrid\",\n \"meta\",\n \"nfa-backtrack\",\n \"nfa-pikevm\",\n \"nfa-thompson\",\n \"perf-inline\",\n \"perf-literal\",\n \"perf-literal-multisubstring\",\n \"perf-literal-substring\",\n \"std\",\n \"syntax\",\n \"unicode\",\n \"unicode-age\",\n \"unicode-bool\",\n \"unicode-case\",\n \"unicode-gencat\",\n \"unicode-perl\",\n \"unicode-script\",\n \"unicode-segment\",\n \"unicode-word-boundary\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=regex-automata\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.4.9\",\n)\n" } }, "crates__regex-syntax-0.8.5": { @@ -5806,6 +1890,22 @@ "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"serde_json\",\n deps = [\n \"@crates__itoa-1.0.14//:itoa\",\n \"@crates__memchr-2.7.4//:memchr\",\n \"@crates__ryu-1.0.18//:ryu\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__serde_json-1.0.134//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde_json\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.134\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"serde_json\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde_json\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.134\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" } }, + "crates__shlex-1.3.0": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/shlex/1.3.0/download" + ], + "strip_prefix": "shlex-1.3.0", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"shlex\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=shlex\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.3.0\",\n)\n" + } + }, "crates__smallvec-1.13.2": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { @@ -5883,7 +1983,7 @@ "https://static.crates.io/crates/syn/2.0.91/download" ], "strip_prefix": "syn-2.0.91", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"syn\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__unicode-ident-1.0.14//:unicode_ident\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"clone-impls\",\n \"default\",\n \"derive\",\n \"extra-traits\",\n \"full\",\n \"parsing\",\n \"printing\",\n \"proc-macro\",\n \"visit\",\n \"visit-mut\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=syn\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.0.91\",\n)\n" + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"syn\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__unicode-ident-1.0.14//:unicode_ident\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"clone-impls\",\n \"default\",\n \"derive\",\n \"extra-traits\",\n \"full\",\n \"parsing\",\n \"printing\",\n \"proc-macro\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=syn\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.0.91\",\n)\n" } }, "crates__thiserror-2.0.12": { @@ -5918,20 +2018,20 @@ "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"thiserror_impl\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=thiserror-impl\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.0.12\",\n)\n" } }, - "crates__tokenizers-0.21.2": { + "crates__tokenizers-0.21.4": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { "patch_args": [], "patch_tool": "", "patches": [], "remote_patch_strip": 1, - "sha256": "4c3846d8588abed0daba25a0e47edd58ea15e450a6088b2575f5116fdb0b27ca", + "sha256": "a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476", "type": "tar.gz", "urls": [ - "https://static.crates.io/crates/tokenizers/0.21.2/download" + "https://static.crates.io/crates/tokenizers/0.21.4/download" ], - "strip_prefix": "tokenizers-0.21.2", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"tokenizers\",\n deps = [\n \"@crates__ahash-0.8.12//:ahash\",\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__compact_str-0.9.0//:compact_str\",\n \"@crates__dary_heap-0.3.7//:dary_heap\",\n \"@crates__derive_builder-0.20.2//:derive_builder\",\n \"@crates__esaxx-rs-0.1.10//:esaxx_rs\",\n \"@crates__fancy-regex-0.14.0//:fancy_regex\",\n \"@crates__getrandom-0.3.3//:getrandom\",\n \"@crates__itertools-0.14.0//:itertools\",\n \"@crates__log-0.4.22//:log\",\n \"@crates__macro_rules_attribute-0.2.0//:macro_rules_attribute\",\n \"@crates__monostate-0.1.13//:monostate\",\n \"@crates__rand-0.9.1//:rand\",\n \"@crates__rayon-1.10.0//:rayon\",\n \"@crates__rayon-cond-0.4.0//:rayon_cond\",\n \"@crates__regex-1.11.1//:regex\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__serde_json-1.0.134//:serde_json\",\n \"@crates__spm_precompiled-0.1.4//:spm_precompiled\",\n \"@crates__thiserror-2.0.12//:thiserror\",\n \"@crates__unicode-normalization-alignments-0.1.12//:unicode_normalization_alignments\",\n \"@crates__unicode-segmentation-1.12.0//:unicode_segmentation\",\n \"@crates__unicode_categories-0.1.1//:unicode_categories\",\n ],\n proc_macro_deps = [\n \"@crates__paste-1.0.15//:paste\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"fancy-regex\",\n \"unstable_wasm\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=tokenizers\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.21.2\",\n)\n" + "strip_prefix": "tokenizers-0.21.4", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"tokenizers\",\n deps = [\n \"@crates__ahash-0.8.12//:ahash\",\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__compact_str-0.9.0//:compact_str\",\n \"@crates__dary_heap-0.3.7//:dary_heap\",\n \"@crates__derive_builder-0.20.2//:derive_builder\",\n \"@crates__esaxx-rs-0.1.10//:esaxx_rs\",\n \"@crates__getrandom-0.3.3//:getrandom\",\n \"@crates__itertools-0.14.0//:itertools\",\n \"@crates__log-0.4.22//:log\",\n \"@crates__macro_rules_attribute-0.2.0//:macro_rules_attribute\",\n \"@crates__monostate-0.1.13//:monostate\",\n \"@crates__onig-6.5.1//:onig\",\n \"@crates__rand-0.9.1//:rand\",\n \"@crates__rayon-1.10.0//:rayon\",\n \"@crates__rayon-cond-0.4.0//:rayon_cond\",\n \"@crates__regex-1.11.1//:regex\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__serde_json-1.0.134//:serde_json\",\n \"@crates__spm_precompiled-0.1.4//:spm_precompiled\",\n \"@crates__thiserror-2.0.12//:thiserror\",\n \"@crates__unicode-normalization-alignments-0.1.12//:unicode_normalization_alignments\",\n \"@crates__unicode-segmentation-1.12.0//:unicode_segmentation\",\n \"@crates__unicode_categories-0.1.1//:unicode_categories\",\n ],\n proc_macro_deps = [\n \"@crates__paste-1.0.15//:paste\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"esaxx_fast\",\n \"onig\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=tokenizers\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.21.4\",\n)\n" } }, "crates__unicode-ident-1.0.14": { @@ -6030,86 +2130,6 @@ "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wasi\",\n deps = [\n \"@crates__wit-bindgen-rt-0.39.0//:wit_bindgen_rt\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasi\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.14.2+wasi-0.2.4\",\n)\n" } }, - "crates__wasm-bindgen-0.2.100": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/wasm-bindgen/0.2.100/download" - ], - "strip_prefix": "wasm-bindgen-0.2.100", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wasm_bindgen\",\n deps = [\n \"@crates__cfg-if-1.0.0//:cfg_if\",\n \"@crates__once_cell-1.20.2//:once_cell\",\n \"@crates__wasm-bindgen-0.2.100//:build_script_build\",\n ],\n proc_macro_deps = [\n \"@crates__wasm-bindgen-macro-0.2.100//:wasm_bindgen_macro\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.100\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"wasm-bindgen\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.2.100\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__wasm-bindgen-backend-0.2.100": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/wasm-bindgen-backend/0.2.100/download" - ], - "strip_prefix": "wasm-bindgen-backend-0.2.100", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wasm_bindgen_backend\",\n deps = [\n \"@crates__bumpalo-3.16.0//:bumpalo\",\n \"@crates__log-0.4.22//:log\",\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n \"@crates__wasm-bindgen-shared-0.2.100//:wasm_bindgen_shared\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen-backend\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.100\",\n)\n" - } - }, - "crates__wasm-bindgen-macro-0.2.100": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/wasm-bindgen-macro/0.2.100/download" - ], - "strip_prefix": "wasm-bindgen-macro-0.2.100", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"wasm_bindgen_macro\",\n deps = [\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__wasm-bindgen-macro-support-0.2.100//:wasm_bindgen_macro_support\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen-macro\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.100\",\n)\n" - } - }, - "crates__wasm-bindgen-macro-support-0.2.100": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/wasm-bindgen-macro-support/0.2.100/download" - ], - "strip_prefix": "wasm-bindgen-macro-support-0.2.100", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wasm_bindgen_macro_support\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n \"@crates__wasm-bindgen-backend-0.2.100//:wasm_bindgen_backend\",\n \"@crates__wasm-bindgen-shared-0.2.100//:wasm_bindgen_shared\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen-macro-support\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.100\",\n)\n" - } - }, - "crates__wasm-bindgen-shared-0.2.100": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/wasm-bindgen-shared/0.2.100/download" - ], - "strip_prefix": "wasm-bindgen-shared-0.2.100", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wasm_bindgen_shared\",\n deps = [\n \"@crates__unicode-ident-1.0.14//:unicode_ident\",\n \"@crates__wasm-bindgen-shared-0.2.100//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen-shared\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.100\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n links = \"wasm_bindgen\",\n pkg_name = \"wasm-bindgen-shared\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen-shared\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.2.100\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, "crates__wit-bindgen-rt-0.39.0": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { @@ -6386,160 +2406,6 @@ ] ] } - }, - "@@rules_swift+//swift:extensions.bzl%non_module_deps": { - "general": { - "bzlTransitiveDigest": "3iOPDsadEKzPUpfznfK5UYw/yuGpZzq7zrjrB/oEo6g=", - "usagesDigest": "9w18ec4dj90mX/JqpR2tBU2YVc1GU9yNxi/d7q6lLVA=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "com_github_apple_swift_protobuf": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-protobuf/archive/1.20.2.tar.gz" - ], - "sha256": "3fb50bd4d293337f202d917b6ada22f9548a0a0aed9d9a4d791e6fbd8a246ebb", - "strip_prefix": "swift-protobuf-1.20.2/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_protobuf/BUILD.overlay" - } - }, - "com_github_grpc_grpc_swift": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/grpc/grpc-swift/archive/1.16.0.tar.gz" - ], - "sha256": "58b60431d0064969f9679411264b82e40a217ae6bd34e17096d92cc4e47556a5", - "strip_prefix": "grpc-swift-1.16.0/", - "build_file": "@@rules_swift+//third_party:com_github_grpc_grpc_swift/BUILD.overlay" - } - }, - "com_github_apple_swift_docc_symbolkit": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-docc-symbolkit/archive/refs/tags/swift-5.10-RELEASE.tar.gz" - ], - "sha256": "de1d4b6940468ddb53b89df7aa1a81323b9712775b0e33e8254fa0f6f7469a97", - "strip_prefix": "swift-docc-symbolkit-swift-5.10-RELEASE", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_docc_symbolkit/BUILD.overlay" - } - }, - "com_github_apple_swift_nio": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio/archive/2.42.0.tar.gz" - ], - "sha256": "e3304bc3fb53aea74a3e54bd005ede11f6dc357117d9b1db642d03aea87194a0", - "strip_prefix": "swift-nio-2.42.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_http2": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-http2/archive/1.26.0.tar.gz" - ], - "sha256": "f0edfc9d6a7be1d587e5b403f2d04264bdfae59aac1d74f7d974a9022c6d2b25", - "strip_prefix": "swift-nio-http2-1.26.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_http2/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_transport_services": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-transport-services/archive/1.15.0.tar.gz" - ], - "sha256": "f3498dafa633751a52b9b7f741f7ac30c42bcbeb3b9edca6d447e0da8e693262", - "strip_prefix": "swift-nio-transport-services-1.15.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_transport_services/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_extras": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-extras/archive/1.4.0.tar.gz" - ], - "sha256": "4684b52951d9d9937bb3e8ccd6b5daedd777021ef2519ea2f18c4c922843b52b", - "strip_prefix": "swift-nio-extras-1.4.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_extras/BUILD.overlay" - } - }, - "com_github_apple_swift_log": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-log/archive/1.4.4.tar.gz" - ], - "sha256": "48fe66426c784c0c20031f15dc17faf9f4c9037c192bfac2f643f65cb2321ba0", - "strip_prefix": "swift-log-1.4.4/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_log/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_ssl": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-ssl/archive/2.23.0.tar.gz" - ], - "sha256": "4787c63f61dd04d99e498adc3d1a628193387e41efddf8de19b8db04544d016d", - "strip_prefix": "swift-nio-ssl-2.23.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_ssl/BUILD.overlay" - } - }, - "com_github_apple_swift_collections": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-collections/archive/1.0.4.tar.gz" - ], - "sha256": "d9e4c8a91c60fb9c92a04caccbb10ded42f4cb47b26a212bc6b39cc390a4b096", - "strip_prefix": "swift-collections-1.0.4/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_collections/BUILD.overlay" - } - }, - "com_github_apple_swift_atomics": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-atomics/archive/1.1.0.tar.gz" - ], - "sha256": "1bee7f469f7e8dc49f11cfa4da07182fbc79eab000ec2c17bfdce468c5d276fb", - "strip_prefix": "swift-atomics-1.1.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_atomics/BUILD.overlay" - } - }, - "build_bazel_rules_swift_index_import": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "build_file": "@@rules_swift+//third_party:build_bazel_rules_swift_index_import/BUILD.overlay", - "canonical_id": "index-import-5.8", - "urls": [ - "https://github.com/MobileNativeFoundation/index-import/releases/download/5.8.0.1/index-import.tar.gz" - ], - "sha256": "28c1ffa39d99e74ed70623899b207b41f79214c498c603915aef55972a851a15" - } - }, - "build_bazel_rules_swift_local_config": { - "repoRuleId": "@@rules_swift+//swift/internal:swift_autoconfiguration.bzl%swift_autoconfiguration", - "attributes": {} - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_swift+", - "bazel_tools", - "bazel_tools" - ] - ] - } } } } diff --git a/bazel/zig_proto_library.bzl b/bazel/zig_proto_library.bzl index 02a903df..fa82d877 100644 --- a/bazel/zig_proto_library.bzl +++ b/bazel/zig_proto_library.bzl @@ -1,6 +1,6 @@ """Starlark implementation of zig_proto_library""" -load("@protobuf//bazel/common:proto_info.bzl", "ProtoInfo") +load("@com_google_protobuf//bazel/common:proto_info.bzl", "ProtoInfo") load("@rules_proto//proto:defs.bzl", "proto_common") load( "@rules_zig//zig/private/providers:zig_module_info.bzl", diff --git a/examples/MODULE.bazel.lock b/examples/MODULE.bazel.lock index 0f803d6f..ffa809e4 100644 --- a/examples/MODULE.bazel.lock +++ b/examples/MODULE.bazel.lock @@ -11,6 +11,8 @@ "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20240722.0.bcr.2/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/abseil-cpp/20250127.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/abseil-cpp/20250512.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/apple_support/1.11.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/apple_support/1.15.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/apple_support/1.17.1/MODULE.bazel": "not found", @@ -30,6 +32,7 @@ "https://bazel-registry.zml.ai/modules/bazel_features/1.19.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.20.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.21.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/bazel_features/1.23.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.24.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.25.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.27.0/MODULE.bazel": "not found", @@ -56,7 +59,9 @@ "https://bazel-registry.zml.ai/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/googletest/1.14.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/googletest/1.15.2/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/googletest/1.17.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/jsoncpp/1.9.5/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/jsoncpp/1.9.6/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/libpfm/4.11.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/nlohmann_json/3.6.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/patchelf/0.18.0/MODULE.bazel": "not found", @@ -78,14 +83,15 @@ "https://bazel-registry.zml.ai/modules/protobuf/29.0-rc3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/protobuf/29.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/protobuf/29.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/29.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/protobuf/3.19.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/protobuf/32.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/pybind11_bazel/2.11.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/pybind11_bazel/2.12.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/re2/2023-09-01/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/re2/2024-07-02/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_android/0.1.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_apple/3.16.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_apple/3.22.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.10/MODULE.bazel": "not found", @@ -117,6 +123,7 @@ "https://bazel-registry.zml.ai/modules/rules_java/8.11.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_java/8.3.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_java/8.5.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_java/8.6.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_java/8.8.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_jvm_external/4.4.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_jvm_external/5.1/MODULE.bazel": "not found", @@ -124,6 +131,7 @@ "https://bazel-registry.zml.ai/modules/rules_jvm_external/5.3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_jvm_external/6.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_jvm_external/6.3/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_jvm_external/6.7/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_kotlin/1.9.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_kotlin/1.9.6/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_license/0.0.3/MODULE.bazel": "not found", @@ -154,12 +162,14 @@ "https://bazel-registry.zml.ai/modules/rules_python/0.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_python/0.40.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_python/1.3.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_python/1.4.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_rust/0.60.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_rust/0.62.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.2.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.3.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.4.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/2.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_uv/0.65.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", @@ -180,6 +190,7 @@ "https://bazel-registry.zml.ai/modules/zlib/1.2.11/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/zlib/1.2.13/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/zlib/1.3.1/MODULE.bazel": "not found", "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2", @@ -192,7 +203,9 @@ "https://bcr.bazel.build/modules/abseil-cpp/20240116.1/MODULE.bazel": "37bcdb4440fbb61df6a1c296ae01b327f19e9bb521f9b8e26ec854b6f97309ed", "https://bcr.bazel.build/modules/abseil-cpp/20240116.2/MODULE.bazel": "73939767a4686cd9a520d16af5ab440071ed75cec1a876bf2fcfaf1f71987a16", "https://bcr.bazel.build/modules/abseil-cpp/20240722.0.bcr.2/MODULE.bazel": "c3661b44c9d3f17f0b65ffb544896aaeb89127398ea867537babac18133a002a", - "https://bcr.bazel.build/modules/abseil-cpp/20240722.0.bcr.2/source.json": "464f3012c8f40cd51facbfc6962563971c6bd55f45ba16bd98313f2de73340c5", + "https://bcr.bazel.build/modules/abseil-cpp/20250127.1/MODULE.bazel": "c4a89e7ceb9bf1e25cf84a9f830ff6b817b72874088bf5141b314726e46a57c1", + "https://bcr.bazel.build/modules/abseil-cpp/20250512.1/MODULE.bazel": "d209fdb6f36ffaf61c509fcc81b19e81b411a999a934a032e10cd009a0226215", + "https://bcr.bazel.build/modules/abseil-cpp/20250512.1/source.json": "d725d73707d01bb46ab3ca59ba408b8e9bd336642ca77a2269d4bfb8bbfd413d", "https://bcr.bazel.build/modules/apple_support/1.11.1/MODULE.bazel": "1843d7cd8a58369a444fc6000e7304425fba600ff641592161d9f15b179fb896", "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", "https://bcr.bazel.build/modules/apple_support/1.17.1/MODULE.bazel": "655c922ab1209978a94ef6ca7d9d43e940cd97d9c172fb55f94d91ac53f8610b", @@ -215,6 +228,7 @@ "https://bcr.bazel.build/modules/bazel_features/1.19.0/MODULE.bazel": "59adcdf28230d220f0067b1f435b8537dd033bfff8db21335ef9217919c7fb58", "https://bcr.bazel.build/modules/bazel_features/1.20.0/MODULE.bazel": "8b85300b9c8594752e0721a37210e34879d23adc219ed9dc8f4104a4a1750920", "https://bcr.bazel.build/modules/bazel_features/1.21.0/MODULE.bazel": "675642261665d8eea09989aa3b8afb5c37627f1be178382c320d1b46afba5e3b", + "https://bcr.bazel.build/modules/bazel_features/1.23.0/MODULE.bazel": "fd1ac84bc4e97a5a0816b7fd7d4d4f6d837b0047cf4cbd81652d616af3a6591a", "https://bcr.bazel.build/modules/bazel_features/1.24.0/MODULE.bazel": "4796b4c25b47053e9bbffa792b3792d07e228ff66cd0405faef56a978708acd4", "https://bcr.bazel.build/modules/bazel_features/1.25.0/MODULE.bazel": "e2e60a10a6da64bbf533f15ca652bf61a033e41c2ed734d79a9a08ba87f68c1a", "https://bcr.bazel.build/modules/bazel_features/1.27.0/MODULE.bazel": "621eeee06c4458a9121d1f104efb80f39d34deff4984e778359c60eaf1a8cb65", @@ -245,9 +259,11 @@ "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6", "https://bcr.bazel.build/modules/googletest/1.14.0/MODULE.bazel": "cfbcbf3e6eac06ef9d85900f64424708cc08687d1b527f0ef65aa7517af8118f", "https://bcr.bazel.build/modules/googletest/1.15.2/MODULE.bazel": "6de1edc1d26cafb0ea1a6ab3f4d4192d91a312fd2d360b63adaa213cd00b2108", - "https://bcr.bazel.build/modules/googletest/1.15.2/source.json": "dbdda654dcb3a0d7a8bc5d0ac5fc7e150b58c2a986025ae5bc634bb2cb61f470", + "https://bcr.bazel.build/modules/googletest/1.17.0/MODULE.bazel": "dbec758171594a705933a29fcf69293d2468c49ec1f2ebca65c36f504d72df46", + "https://bcr.bazel.build/modules/googletest/1.17.0/source.json": "38e4454b25fc30f15439c0378e57909ab1fd0a443158aa35aec685da727cd713", "https://bcr.bazel.build/modules/jsoncpp/1.9.5/MODULE.bazel": "31271aedc59e815656f5736f282bb7509a97c7ecb43e927ac1a37966e0578075", - "https://bcr.bazel.build/modules/jsoncpp/1.9.5/source.json": "4108ee5085dd2885a341c7fab149429db457b3169b86eb081fa245eadf69169d", + "https://bcr.bazel.build/modules/jsoncpp/1.9.6/MODULE.bazel": "2f8d20d3b7d54143213c4dfc3d98225c42de7d666011528dc8fe91591e2e17b0", + "https://bcr.bazel.build/modules/jsoncpp/1.9.6/source.json": "a04756d367a2126c3541682864ecec52f92cdee80a35735a3cb249ce015ca000", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/source.json": "f448c6e8963fdfa7eb831457df83ad63d3d6355018f6574fb017e8169deb43a9", @@ -273,9 +289,9 @@ "https://bcr.bazel.build/modules/protobuf/29.0-rc3/MODULE.bazel": "33c2dfa286578573afc55a7acaea3cada4122b9631007c594bf0729f41c8de92", "https://bcr.bazel.build/modules/protobuf/29.0/MODULE.bazel": "319dc8bf4c679ff87e71b1ccfb5a6e90a6dbc4693501d471f48662ac46d04e4e", "https://bcr.bazel.build/modules/protobuf/29.1/MODULE.bazel": "557c3457560ff49e122ed76c0bc3397a64af9574691cb8201b4e46d4ab2ecb95", - "https://bcr.bazel.build/modules/protobuf/29.2/MODULE.bazel": "5435497c190d86f79b0568698c45044df7c8d97692886cda9fe9cf9053aea712", - "https://bcr.bazel.build/modules/protobuf/29.2/source.json": "fe7090cc34072609b26d9beafb122916dabc1d47ba61b242c26c4b06c51384ab", "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0", + "https://bcr.bazel.build/modules/protobuf/32.0/MODULE.bazel": "0741cf24f8e1185286578069060e905ed67d68eef5990bfa3dea3fc1afba14c7", + "https://bcr.bazel.build/modules/protobuf/32.0/source.json": "1e278267d3642ab361dc460cc1f2d8e607e8292b9b3f799a47181a3acf4f3294", "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e", "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/MODULE.bazel": "e6f4c20442eaa7c90d7190d8dc539d0ab422f95c65a57cc59562170c58ae3d34", "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/source.json": "6900fdc8a9e95866b8c0d4ad4aba4d4236317b5c1cd04c502df3f0d33afed680", @@ -285,6 +301,7 @@ "https://bcr.bazel.build/modules/re2/2024-07-02/MODULE.bazel": "0eadc4395959969297cbcf31a249ff457f2f1d456228c67719480205aa306daa", "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", + "https://bcr.bazel.build/modules/rules_apple/3.16.0/MODULE.bazel": "0d1caf0b8375942ce98ea944be754a18874041e4e0459401d925577624d3a54a", "https://bcr.bazel.build/modules/rules_apple/3.22.0/MODULE.bazel": "0fd8c630d6cecc18abe84114ee21c1a0fb759b45063d3d2600ec22669e93ea43", "https://bcr.bazel.build/modules/rules_apple/3.22.0/source.json": "ee41f63c292bd876fd264a830cf45500619a063e2fa3a18cca6fa12b153d68e3", "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", @@ -304,7 +321,6 @@ "https://bcr.bazel.build/modules/rules_distroless/0.5.1/source.json": "c6b9ff7f325bfed89c3671757f14c1d1bc6077d0fcef809b8aa2d007cac7dd1d", "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", "https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/MODULE.bazel": "40c97d1144356f52905566c55811f13b299453a14ac7769dfba2ac38192337a8", - "https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/source.json": "c8b1e2c717646f1702290959a3302a178fb639d987ab61d548105019f11e527e", "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", "https://bcr.bazel.build/modules/rules_java/5.3.5/MODULE.bazel": "a4ec4f2db570171e3e5eb753276ee4b389bae16b96207e9d3230895c99644b86", "https://bcr.bazel.build/modules/rules_java/6.0.0/MODULE.bazel": "8a43b7df601a7ec1af61d79345c17b31ea1fedc6711fd4abfd013ea612978e39", @@ -321,6 +337,7 @@ "https://bcr.bazel.build/modules/rules_java/8.11.0/source.json": "302b52a39259a85aa06ca3addb9787864ca3e03b432a5f964ea68244397e7544", "https://bcr.bazel.build/modules/rules_java/8.3.2/MODULE.bazel": "7336d5511ad5af0b8615fdc7477535a2e4e723a357b6713af439fe8cf0195017", "https://bcr.bazel.build/modules/rules_java/8.5.1/MODULE.bazel": "d8a9e38cc5228881f7055a6079f6f7821a073df3744d441978e7a43e20226939", + "https://bcr.bazel.build/modules/rules_java/8.6.1/MODULE.bazel": "f4808e2ab5b0197f094cabce9f4b006a27766beb6a9975931da07099560ca9c2", "https://bcr.bazel.build/modules/rules_java/8.8.0/MODULE.bazel": "de589d0880911ac007abd521b9f0ddcd8b0dbd05c8553e6f8124a050b83acf7d", "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7", "https://bcr.bazel.build/modules/rules_jvm_external/5.1/MODULE.bazel": "33f6f999e03183f7d088c9be518a63467dfd0be94a11d0055fe2d210f89aa909", @@ -328,7 +345,8 @@ "https://bcr.bazel.build/modules/rules_jvm_external/5.3/MODULE.bazel": "bf93870767689637164657731849fb887ad086739bd5d360d90007a581d5527d", "https://bcr.bazel.build/modules/rules_jvm_external/6.1/MODULE.bazel": "75b5fec090dbd46cf9b7d8ea08cf84a0472d92ba3585b476f44c326eda8059c4", "https://bcr.bazel.build/modules/rules_jvm_external/6.3/MODULE.bazel": "c998e060b85f71e00de5ec552019347c8bca255062c990ac02d051bb80a38df0", - "https://bcr.bazel.build/modules/rules_jvm_external/6.3/source.json": "6f5f5a5a4419ae4e37c35a5bb0a6ae657ed40b7abc5a5189111b47fcebe43197", + "https://bcr.bazel.build/modules/rules_jvm_external/6.7/MODULE.bazel": "e717beabc4d091ecb2c803c2d341b88590e9116b8bf7947915eeb33aab4f96dd", + "https://bcr.bazel.build/modules/rules_jvm_external/6.7/source.json": "5426f412d0a7fc6b611643376c7e4a82dec991491b9ce5cb1cfdd25fe2e92be4", "https://bcr.bazel.build/modules/rules_kotlin/1.9.0/MODULE.bazel": "ef85697305025e5a61f395d4eaede272a5393cee479ace6686dba707de804d59", "https://bcr.bazel.build/modules/rules_kotlin/1.9.6/MODULE.bazel": "d269a01a18ee74d0335450b10f62c9ed81f2321d7958a2934e44272fe82dcef3", "https://bcr.bazel.build/modules/rules_kotlin/1.9.6/source.json": "2faa4794364282db7c06600b7e5e34867a564ae91bda7cae7c29c64e9466b7d5", @@ -365,7 +383,8 @@ "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", "https://bcr.bazel.build/modules/rules_python/0.40.0/MODULE.bazel": "9d1a3cd88ed7d8e39583d9ffe56ae8a244f67783ae89b60caafc9f5cf318ada7", "https://bcr.bazel.build/modules/rules_python/1.3.0/MODULE.bazel": "8361d57eafb67c09b75bf4bbe6be360e1b8f4f18118ab48037f2bd50aa2ccb13", - "https://bcr.bazel.build/modules/rules_python/1.3.0/source.json": "25932f917cd279c7baefa6cb1d3fa8750a7a29de522024449b19af6eab51f4a0", + "https://bcr.bazel.build/modules/rules_python/1.4.1/MODULE.bazel": "8991ad45bdc25018301d6b7e1d3626afc3c8af8aaf4bc04f23d0b99c938b73a6", + "https://bcr.bazel.build/modules/rules_python/1.4.1/source.json": "8ec8c90c70ccacc4de8ca1b97f599e756fb59173e898ee08b733006650057c07", "https://bcr.bazel.build/modules/rules_rust/0.60.0/MODULE.bazel": "911ff2a12d01ac574fd6dfec0b05fa976ff8693d8c2420db637a9f98f697b0ae", "https://bcr.bazel.build/modules/rules_rust/0.62.0/MODULE.bazel": "6a15b57982e278793c684f426e19166e62e73f1bd45fe3b6bcedd0b901177b37", "https://bcr.bazel.build/modules/rules_rust/0.62.0/source.json": "1b3a6551a585ee47cafa21550c5eb87c6f3a56bb9761f9e3421ff1102f220437", @@ -374,6 +393,7 @@ "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", "https://bcr.bazel.build/modules/rules_shell/0.4.1/source.json": "4757bd277fe1567763991c4425b483477bb82e35e777a56fd846eb5cceda324a", "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", + "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", "https://bcr.bazel.build/modules/rules_swift/2.4.0/source.json": "a6577f57f9febbdc015a01f2a8f3487422032f134d6c61d18ed8e8ca3b9acc7c", "https://bcr.bazel.build/modules/rules_uv/0.65.0/MODULE.bazel": "a700505a80ad0ad65f062c9335458d137f504b98c5937b338807bb5cbe305e73", @@ -401,7 +421,8 @@ "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", "https://bcr.bazel.build/modules/zlib/1.2.13/MODULE.bazel": "aa6deb1b83c18ffecd940c4119aff9567cd0a671d7bba756741cb2ef043a29d5", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "af322bc08976524477c79d1e45e241b6efbeb918c497e8840b8ab116802dda79", - "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/source.json": "2be409ac3c7601245958cd4fcdff4288be79ed23bd690b4b951f500d54ee6e7d", + "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "eec517b5bbe5492629466e11dae908d043364302283de25581e3eb944326c4ca", + "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/source.json": "22bc55c47af97246cfc093d0acf683a7869377de362b5d1c552c2c2e16b7a806", "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198" }, "selectedYankedVersions": {}, @@ -710,171 +731,6 @@ ] } }, - "@@rules_apple+//apple:apple.bzl%provisioning_profile_repository_extension": { - "general": { - "bzlTransitiveDigest": "mVdP3ANttcsLRv2k6Sbzs06sQDCgBTB8CPMicPSeCW4=", - "usagesDigest": "+Mri48un5kxPRrX8WFbwbgbUsG6JygfULZLaltX0weQ=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "local_provisioning_profiles": { - "repoRuleId": "@@rules_apple+//apple/internal:local_provisioning_profiles.bzl%provisioning_profile_repository", - "attributes": {} - } - }, - "recordedRepoMappingEntries": [ - [ - "apple_support+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "bazel_tools", - "rules_cc", - "rules_cc+" - ], - [ - "rules_apple+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "rules_apple+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_apple+", - "build_bazel_apple_support", - "apple_support+" - ], - [ - "rules_apple+", - "build_bazel_rules_swift", - "rules_swift+" - ], - [ - "rules_cc+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_cc+", - "rules_cc", - "rules_cc+" - ], - [ - "rules_swift+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "rules_swift+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_swift+", - "build_bazel_apple_support", - "apple_support+" - ], - [ - "rules_swift+", - "build_bazel_rules_swift", - "rules_swift+" - ], - [ - "rules_swift+", - "build_bazel_rules_swift_local_config", - "rules_swift++non_module_deps+build_bazel_rules_swift_local_config" - ] - ] - } - }, - "@@rules_fuzzing+//fuzzing/private:extensions.bzl%non_module_dependencies": { - "general": { - "bzlTransitiveDigest": "Wn0Wqc8tJukC1aqT+Fqun/4Xrnr+iTThN24Fn3rz3FU=", - "usagesDigest": "wy6ISK6UOcBEjj/mvJ/S3WeXoO67X+1llb9yPyFtPgc=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "platforms": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.8/platforms-0.0.8.tar.gz", - "https://github.com/bazelbuild/platforms/releases/download/0.0.8/platforms-0.0.8.tar.gz" - ], - "sha256": "8150406605389ececb6da07cbcb509d5637a3ab9a24bc69b1101531367d89d74" - } - }, - "rules_python": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "sha256": "d70cd72a7a4880f0000a6346253414825c19cdd40a28289bdf67b8e6480edff8", - "strip_prefix": "rules_python-0.28.0", - "url": "https://github.com/bazelbuild/rules_python/releases/download/0.28.0/rules_python-0.28.0.tar.gz" - } - }, - "bazel_skylib": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "sha256": "cd55a062e763b9349921f0f5db8c3933288dc8ba4f76dd9416aac68acee3cb94", - "urls": [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz", - "https://github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz" - ] - } - }, - "com_google_absl": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/abseil/abseil-cpp/archive/refs/tags/20240116.1.zip" - ], - "strip_prefix": "abseil-cpp-20240116.1", - "integrity": "sha256-7capMWOvWyoYbUaHF/b+I2U6XLMaHmky8KugWvfXYuk=" - } - }, - "rules_fuzzing_oss_fuzz": { - "repoRuleId": "@@rules_fuzzing+//fuzzing/private/oss_fuzz:repository.bzl%oss_fuzz_repository", - "attributes": {} - }, - "honggfuzz": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "build_file": "@@rules_fuzzing+//:honggfuzz.BUILD", - "sha256": "6b18ba13bc1f36b7b950c72d80f19ea67fbadc0ac0bb297ec89ad91f2eaa423e", - "url": "https://github.com/google/honggfuzz/archive/2.5.zip", - "strip_prefix": "honggfuzz-2.5" - } - }, - "rules_fuzzing_jazzer": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_jar", - "attributes": { - "sha256": "ee6feb569d88962d59cb59e8a31eb9d007c82683f3ebc64955fd5b96f277eec2", - "url": "https://repo1.maven.org/maven2/com/code-intelligence/jazzer/0.20.1/jazzer-0.20.1.jar" - } - }, - "rules_fuzzing_jazzer_api": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_jar", - "attributes": { - "sha256": "f5a60242bc408f7fa20fccf10d6c5c5ea1fcb3c6f44642fec5af88373ae7aa1b", - "url": "https://repo1.maven.org/maven2/com/code-intelligence/jazzer-api/0.20.1/jazzer-api-0.20.1.jar" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_fuzzing+", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { "general": { "bzlTransitiveDigest": "fmfKdvTpZCJJntCdqlB6bYFsD3ax+7qZpeR0cGMDe8A=", @@ -5027,7 +4883,7 @@ "@@rules_python+//python/uv:uv.bzl%uv": { "general": { "bzlTransitiveDigest": "UwuEebIvWhq1IDSj2VbBXlMc20ec8cBFyxShHe64sy8=", - "usagesDigest": "vJ5RHUxAnV24M5swNGiAnkdxMx3Hp/iOLmNANTC5Xc8=", + "usagesDigest": "4JapxcpS0mL3524k0TZJffAtVyuRjDHZvN9kBRxxF1U=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, diff --git a/third_party/com_google_sentencepiece/sentencepiece.bazel b/third_party/com_google_sentencepiece/sentencepiece.bazel index ecaa4361..f73793b3 100644 --- a/third_party/com_google_sentencepiece/sentencepiece.bazel +++ b/third_party/com_google_sentencepiece/sentencepiece.bazel @@ -1,6 +1,6 @@ load("@bazel_skylib//rules:copy_file.bzl", "copy_file") -load("@protobuf//bazel:cc_proto_library.bzl", "cc_proto_library") -load("@protobuf//bazel:proto_library.bzl", "proto_library") +load("@com_google_protobuf//bazel:cc_proto_library.bzl", "cc_proto_library") +load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") package( default_visibility = ["//visibility:public"], @@ -90,7 +90,7 @@ cc_library( ":darts_clone", ":sentencepiece_cc_proto", ":sentencepiece_model_cc_proto", - "@protobuf//:protobuf_lite", + "@com_google_protobuf//:protobuf_lite", "@zml//third_party/com_google_sentencepiece:absl", ], ) diff --git a/upb/BUILD.bazel b/upb/BUILD.bazel new file mode 100644 index 00000000..ca8ebc3b --- /dev/null +++ b/upb/BUILD.bazel @@ -0,0 +1,12 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +cc_library( + name = "empty", +) + +zig_library( + name = "upb", + main = "upb.zig", + deps = [":empty"], + visibility = ["//visibility:public"], +) diff --git a/upb/upb.zig b/upb/upb.zig new file mode 100644 index 00000000..3033554c --- /dev/null +++ b/upb/upb.zig @@ -0,0 +1,149 @@ +const std = @import("std"); + +const c = @import("c"); + +export fn _upb_MiniTable_StrongReference_dont_copy_me__upb_internal_use_only(mt: *c.upb_MiniTable) callconv(.c) void { + const unused: *volatile c.upb_MiniTable = mt; + _ = &unused; +} + +pub const SerializeOptions = packed struct(c_int) { + deterministic: bool = false, + skip_unknown: bool = false, + check_required: bool = false, + _ignored: u29 = 0, +}; + +pub const SerializeError = error{ + MaxDepthExceeded, + MissingRequired, + Unknown, +} || std.mem.Allocator.Error; + +pub const ParseOptions = packed struct { + alias_string: bool = false, + check_required: bool = false, + experimental_allow_unlinked: bool = false, + always_validate_utf8: bool = false, + disable_fast_table: bool = false, + _ignored: u27 = 0, +}; + +pub const ParseError = error{ + Malformed, + BadUtf8, + MaxDepthExceeded, + MissingRequired, + UnlinkedSubMessage, + Unknown, +} || std.mem.Allocator.Error; + +pub fn stringView(data: ?[]const u8) c.upb_StringView { + return if (data) |d| c.upb_StringView_FromDataAndSize(d.ptr, d.len) else .{}; +} + +pub fn slice(sv: c.upb_StringView) ?[]const u8 { + return if (sv.data) |d| d[0..sv.size] else null; +} + +fn ProtoName(comptime UpbType: type) []const u8 { + const needle = ".struct_"; + const type_name = @typeName(UpbType); + const idx = std.mem.indexOf(u8, type_name, needle) orelse @compileError("Type name is invalid"); + return type_name[idx + needle.len ..]; +} + +fn Minitable(comptime UpbType: type) *const c.upb_MiniTable { + const field_name = comptime blk: { + const name = ProtoName(UpbType); + var it = std.mem.tokenizeScalar(u8, name, '_'); + while (it.next()) |_| { + const new_name = name[0..it.index] ++ "_" ++ name[it.index..] ++ "_msg_init"; + if (@hasDecl(c, new_name)) { + break :blk new_name; + } + } else { + @compileError("Unable to find minitable for type:" ++ @typeName(UpbType)); + } + }; + return &@field(c, field_name); +} + +pub fn serializeEx(ptr: anytype, arena: *c.upb_Arena, opts: SerializeOptions) SerializeError![]const u8 { + var buf: [*c]u8 = undefined; + var size: usize = undefined; + return switch (c.upb_Encode(@ptrCast(ptr), Minitable(@TypeOf(ptr.*)), @bitCast(opts), arena, &buf, &size)) { + c.kUpb_EncodeStatus_Ok => buf[0..size], + c.kUpb_EncodeStatus_OutOfMemory => std.mem.Allocator.Error.OutOfMemory, + c.kUpb_EncodeStatus_MaxDepthExceeded => SerializeError.MaxDepthExceeded, + c.kUpb_EncodeStatus_MissingRequired => SerializeError.MissingRequired, + else => return SerializeError.Unknown, + }; +} + +pub fn serialize(ptr: anytype, arena: *c.upb_Arena) SerializeError![]const u8 { + return try serializeEx(ptr, arena, .{}); +} + +pub fn parseEx(comptime UpbType: type, arena: *c.upb_Arena, data: []const u8, opts: ParseOptions) ParseError!*UpbType { + const obj = try new(UpbType, arena); + return switch (c.upb_Decode(@ptrCast(@constCast(data)), data.len, @alignCast(@ptrCast(obj)), Minitable(UpbType), null, @bitCast(opts), arena)) { + c.kUpb_DecodeStatus_Ok => obj, + c.kUpb_DecodeStatus_Malformed => ParseError.Malformed, + c.kUpb_DecodeStatus_OutOfMemory => std.mem.Allocator.Error.OutOfMemory, + c.kUpb_DecodeStatus_BadUtf8 => ParseError.BadUtf8, + c.kUpb_DecodeStatus_MaxDepthExceeded => ParseError.MaxDepthExceeded, + c.kUpb_DecodeStatus_MissingRequired => ParseError.MissingRequired, + c.kUpb_DecodeStatus_UnlinkedSubMessage => ParseError.UnlinkedSubMessage, + else => ParseError.Unknown, + }; +} + +pub fn parse(comptime UpbType: type, arena: *c.upb_Arena, data: []const u8) ParseError!*UpbType { + return parseEx(arena, data, .{}); +} + +pub fn new(comptime UpbType: type, arena: *c.upb_Arena) std.mem.Allocator.Error!*UpbType { + const new_fn = @field(c, ProtoName(UpbType) ++ "_new"); + return @ptrCast(new_fn(arena) orelse return std.mem.Allocator.Error.OutOfMemory); +} + +pub const Allocator = struct { + upb_alloc: c.upb_alloc, + allocator: std.mem.Allocator, + + pub fn init(allocator: std.mem.Allocator) Allocator { + return .{ + .upb_alloc = .{ + .func = &alloc_impl, + }, + .allocator = allocator, + }; + } + + pub fn inner(self: *Allocator) *c.upb_alloc { + return &self.upb_alloc; + } + + fn alloc_impl(alloc: [*c]c.upb_alloc, ptr: ?*anyopaque, oldsize: usize, size: usize, actual_size: [*c]usize) callconv(.c) ?*anyopaque { + const upb_alloc: *c.upb_alloc = alloc orelse return null; + const self: *Allocator = @fieldParentPtr("upb_alloc", upb_alloc); + defer { + if (actual_size) |as| { + as.* = size; + } + } + if (ptr) |ptr_| { + const ptr_as_slice: []u8 = @as([*c]u8, @ptrCast(ptr_))[0..oldsize]; + if (size == 0) { + self.allocator.free(ptr_as_slice); + return null; + } else if (size != oldsize) { + return (self.allocator.realloc(ptr_as_slice, size) catch return null).ptr; + } + @panic("Unsupported case"); + } + + return (self.allocator.alloc(u8, size) catch return null).ptr; + } +}; diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 218c484f..556ae031 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -1,9 +1,19 @@ +load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_c_proto_library") load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library") load("//bazel:zig.bzl", "zig_cc_test") -load("//bazel:zig_proto_library.bzl", "zig_proto_library") load("//bazel:zig_srcs.bzl", "zig_srcs") +upb_c_proto_library( + name = "xla_data_upb", + deps = ["@xla//xla:xla_data_proto"], +) + +upb_c_proto_library( + name = "xla_compile_options_upb", + deps = ["@xla//xla/pjrt/proto:compile_options_proto"], +) + cc_library( name = "posix", hdrs = ["posix.h"], @@ -25,26 +35,21 @@ zig_library( visibility = ["//visibility:public"], deps = [ ":posix", - ":xla_proto", + ":xla_compile_options_upb", + ":xla_data_upb", "//async", "//mlir", "//mlir/dialects", "//pjrt", "//runtimes", "//stdx", + "//upb", "//zml/tokenizer", "//zml/tools", "@rules_zig//zig/runfiles", ], ) -zig_proto_library( - name = "xla_proto", - import_name = "//xla:xla_proto", - deps = ["@xla//xla/pjrt/proto:compile_options_proto"], -) - - # All ZML Tests zig_cc_test( diff --git a/zml/module.zig b/zml/module.zig index f031e815..1f9beaf4 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -1,10 +1,11 @@ const std = @import("std"); const asynk = @import("async"); +const c = @import("c"); const dialect = @import("mlir/dialects"); const mlir = @import("mlir"); const stdx = @import("stdx"); -const xla_pb = @import("//xla:xla_proto"); +const upb = @import("upb"); const BaseExe = @import("exe.zig").BaseExe; const Buffer = @import("buffer.zig").Buffer; @@ -834,6 +835,28 @@ fn storePjrtExecutable(platform: Platform, loaded_executable: *pjrt.LoadedExecut try loaded_executable_file.writeAll(serialize_result.bytes); } +fn setXlaOverrideFlag(map: *c.upb_Map, flag: []const u8, value: anytype, upb_arena: *c.upb_Arena) !void { + const result = c.upb_Map_Set( + map, + .{ .str_val = upb.stringView(flag) }, + .{ .msg_val = blk: { + const field = try upb.new(c.xla_OptionOverrideProto, upb_arena); + switch (@typeInfo(@TypeOf(value))) { + .bool => c.xla_OptionOverrideProto_set_bool_field(field, value), + .comptime_int, .int => c.xla_OptionOverrideProto_set_int_field(field, @intCast(value)), + .comptime_float, .float => c.xla_OptionOverrideProto_set_double_field(field, @floatCast(value)), + else => c.xla_OptionOverrideProto_set_string_field(field, upb.stringView(value)), + } + break :blk @ptrCast(field); + } }, + upb_arena, + ); + + if (result == false) { + return std.mem.Allocator.Error.OutOfMemory; + } +} + fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, module: mlir.Module, xla_dump_to_: ?[]const u8) !*pjrt.LoadedExecutable { const tracer = Tracer.init("ai.zml.compilation"); const compile_frame = tracer.frameStart("pjrt compilation"); @@ -841,86 +864,83 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m const sharding = platform.sharding(); - var options: xla_pb.CompileOptionsProto = .{ - .executable_build_options = .{ - .device_ordinal = -1, - .num_replicas = sharding.num_replicas, - .num_partitions = sharding.num_partitions, - .use_spmd_partitioning = sharding.num_partitions > 1 or sharding.num_replicas > 1, - .device_assignment = .{ - .replica_count = sharding.num_replicas, - .computation_count = sharding.num_partitions, - // Filled below. - .computation_devices = .{}, + var upb_alloc: upb.Allocator = .init(arena); + const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); + defer c.upb_Arena_Free(upb_arena); + + const options = blk: { + const options = try upb.new(c.xla_CompileOptionsProto, upb_arena); + c.xla_CompileOptionsProto_set_executable_build_options(options, executable_build_options_blk: { + const exec_build_options = try upb.new(c.xla_ExecutableBuildOptionsProto, upb_arena); + + c.xla_ExecutableBuildOptionsProto_set_device_ordinal(exec_build_options, -1); + c.xla_ExecutableBuildOptionsProto_set_num_replicas(exec_build_options, sharding.num_replicas); + c.xla_ExecutableBuildOptionsProto_set_num_partitions(exec_build_options, sharding.num_partitions); + c.xla_ExecutableBuildOptionsProto_set_use_spmd_partitioning(exec_build_options, sharding.num_partitions > 1 or sharding.num_replicas > 1); + + c.xla_ExecutableBuildOptionsProto_set_device_assignment(exec_build_options, device_assignment_blk: { + const device_assignment = try upb.new(c.xla_DeviceAssignmentProto, upb_arena); + + c.xla_DeviceAssignmentProto_set_replica_count(device_assignment, sharding.num_replicas); + c.xla_DeviceAssignmentProto_set_computation_count(device_assignment, sharding.num_partitions); + + const computation_devices = c.xla_DeviceAssignmentProto_resize_computation_devices(device_assignment, sharding.num_partitions, upb_arena); + for (computation_devices[0..sharding.num_partitions], 0..) |*computation_device, i| { + computation_device.* = try upb.new(c.xla_DeviceAssignmentProto_ComputationDevice, upb_arena); + _ = c.xla_DeviceAssignmentProto_ComputationDevice_add_replica_device_ids(computation_device.*, @intCast(i), upb_arena); + } + break :device_assignment_blk device_assignment; + }); + + break :executable_build_options_blk exec_build_options; + }); + + const overrides_map = c._xla_CompileOptionsProto_env_option_overrides_mutable_upb_map(options, upb_arena); + switch (platform.target) { + .cuda => { + // NVIDIA recommends these settings + // https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_triton_gemm", false, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_latency_hiding_scheduler", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_llvm_module_compilation_parallelism", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_libnvptxcompiler", true, upb_arena); }, - }, + .rocm => { + // Disable Triton GEMM on ROCM. For some reason it's much, much slower when + // enabled on CDNA and it's used on RDNA. Disable it altogether. + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_triton_gemm", false, upb_arena); + // Use lld from libllvm instead of invoking the ld.lld binary. + // This saves us from having to sandbox it. + try setXlaOverrideFlag(overrides_map, "xla_gpu_use_inprocess_lld", true, upb_arena); + }, + else => {}, + } + + if (xla_dump_to_ orelse platform.compilation_options.xla_dump_to) |xla_dump_to| { + try setXlaOverrideFlag(overrides_map, "xla_dump_to", xla_dump_to, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_dump_hlo_as_proto", true, upb_arena); + if (platform.compilation_options.xla_dump_fusion_visualization) { + try setXlaOverrideFlag(overrides_map, "xla_dump_fusion_visualization", true, upb_arena); + } + if (platform.compilation_options.xla_dump_hlo_pass_re) |re| { + try setXlaOverrideFlag(overrides_map, "xla_dump_hlo_pass_re", re, upb_arena); + } + } + + break :blk options; }; - const computation_devices = &options.executable_build_options.?.device_assignment.?.computation_devices; - try computation_devices.ensureTotalCapacity(arena, sharding.num_partitions); - const replica_device_ids = try arena.alloc(i64, sharding.num_partitions); - for (0..sharding.num_partitions) |i| { - replica_device_ids[i] = @intCast(i); - computation_devices.appendAssumeCapacity(.{ .replica_device_ids = .fromOwnedSlice(replica_device_ids[i .. i + 1]) }); - } - // Let the arena deinit, zig-protobuf deinit is very slow. - try options.env_option_overrides.ensureUnusedCapacity(arena, 16); - if (xla_dump_to_ orelse platform.compilation_options.xla_dump_to) |xla_dump_to| { - setFlag(&options, "xla_dump_to", xla_dump_to); - setFlag(&options, "xla_dump_hlo_as_proto", true); - if (platform.compilation_options.xla_dump_fusion_visualization) { - setFlag(&options, "xla_dump_fusion_visualization", true); - } - if (platform.compilation_options.xla_dump_hlo_pass_re) |re| { - setFlag(&options, "xla_dump_hlo_pass_re", re); - } - } - switch (platform.target) { - .cuda => { - // NVIDIA recommends these settings - // https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables - setFlag(&options, "xla_gpu_enable_triton_gemm", false); - setFlag(&options, "xla_gpu_enable_latency_hiding_scheduler", true); - setFlag(&options, "xla_gpu_enable_llvm_module_compilation_parallelism", true); - setFlag(&options, "xla_gpu_enable_libnvptxcompiler", true); - // setFlag(&options, "xla_gpu_enable_cudnn_fmha", true); - // setFlag(&options, "xla_gpu_fused_attention_use_cudnn_rng", true); - // setFlag(&options, "xla_gpu_enable_cudnn_layer_norm", true); - // setFlag(&options, "xla_gpu_enable_custom_fusions", true); - // setFlags(&options, "xla_gpu_enable_address_computation_fusion", true); - // setFlag(&options, "xla_gpu_enable_dynamic_slice_fusion", true); - // setFlag(&options, "xla_gpu_enable_while_loop_double_buffering", true); - // setFlag(&options, "xla_gpu_use_runtime_fusion", true); - }, - .rocm => { - // Disable Triton GEMM on ROCM. For some reason it's much, much slower when - // enabled on CDNA and it's used on RDNA. Disable it altogether. - setFlag(&options, "xla_gpu_enable_triton_gemm", false); - // Use lld from libllvm instead of invoking the ld.lld binary. - // This saves us from having to sandbox it. - setFlag(&options, "xla_gpu_use_inprocess_lld", true); - }, - else => {}, - } - - const options_bytes = try options.encode(arena); - - const loaded_executable = try platform.pjrt_client.compile(platform.pjrt_api, arena, module, options_bytes); + const loaded_executable = try platform.pjrt_client.compile( + platform.pjrt_api, + arena, + module, + try upb.serialize(options, upb_arena), + ); errdefer loaded_executable.deinit(); return loaded_executable; } -fn setFlag(options: *xla_pb.CompileOptionsProto, comptime flag: [:0]const u8, value: anytype) void { - const option: xla_pb.OptionOverrideProto = switch (@typeInfo(@TypeOf(value))) { - .bool => .{ .value = .{ .bool_field = value } }, - .comptime_int, .int => .{ .value = .{ .int_field = value } }, - .comptime_float, .float => .{ .value = .{ .double_field = value } }, - else => .{ .value = .{ .string_field = .{ .Const = value } } }, - }; - options.env_option_overrides.appendAssumeCapacity(.{ .key = .{ .Const = flag }, .value = option }); -} - /// Visit the given struct and recursively counts the number of tensors found. pub fn countTensors(v: anytype) usize { const LocalContext = struct { From 3b81c5630ed72e47386967f3e51f648b3a104958 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 19 Aug 2025 19:42:50 +0200 Subject: [PATCH 013/881] zml: remove examples workspace (#308) Closes #302 --- .bazelignore | 4 +- .bazelrc | 116 +- .bazelrc-examples | 2 - .bazelrc-zml | 2 - .bazelversion | 2 +- .github/workflows/commit-examples.yaml | 75 - .github/workflows/commit-framework.yaml | 69 - .github/workflows/commit.yaml | 70 + .nvim.lua | 37 +- MODULE.bazel | 55 +- MODULE.bazel.lock | 1015 ++- README.md | 18 +- examples/bazel.sh => bazel.sh | 0 build.zig | 1 - docs/howtos/add_weights.md | 30 +- docs/howtos/deploy_on_server.md | 8 +- docs/howtos/dockerize_models.md | 4 +- docs/howtos/huggingface_access_token.md | 4 +- docs/tutorials/getting_started.md | 13 +- docs/tutorials/write_first_model.md | 3 +- examples/.bazelignore | 3 - examples/.bazelrc | 119 - examples/.bazelversion | 1 - examples/.gitignore | 30 - examples/.nvim.lua | 36 - examples/.vscode/extensions.json | 8 - examples/.vscode/settings.json | 37 - examples/MODULE.bazel | 204 - examples/MODULE.bazel.lock | 6628 ----------------- examples/benchmark/BUILD.bazel | 14 +- examples/llama/BUILD.bazel | 146 +- examples/loader/BUILD.bazel | 11 - examples/loader/main.zig | 65 - examples/mnist/BUILD.bazel | 106 +- examples/modernbert/BUILD.bazel | 84 +- examples/platform_mappings | 29 - examples/simple_layer/BUILD.bazel | 35 +- examples/tools/buildifier.sh | 4 - examples/tools/zls.sh | 4 - examples/zls.build.json | 2 - mlir/BUILD.bazel | 8 +- mlir/dialects/BUILD.bazel | 8 - platform_mappings | 30 +- runtimes/neuron/BUILD.bazel | 17 +- .../com_github_hejsil_clap}/BUILD.bazel | 0 .../com_github_hejsil_clap/clap.bazel | 0 .../com_github_hejsil_clap/repo.bzl | 13 +- .../mnist/BUILD.bazel | 0 third_party/mnist/repo.bzl | 9 + third_party/non_module_deps.bzl | 6 +- third_party/org_swig_swig/BUILD.bazel | 0 upb/upb.zig | 5 +- zml/tokenizer/hftokenizers/BUILD.bazel | 9 +- 53 files changed, 1253 insertions(+), 7946 deletions(-) mode change 120000 => 100644 .bazelignore mode change 120000 => 100644 .bazelrc delete mode 100644 .bazelrc-examples delete mode 100644 .bazelrc-zml mode change 120000 => 100644 .bazelversion delete mode 100644 .github/workflows/commit-examples.yaml delete mode 100644 .github/workflows/commit-framework.yaml create mode 100644 .github/workflows/commit.yaml mode change 120000 => 100644 .nvim.lua rename examples/bazel.sh => bazel.sh (100%) mode change 120000 => 100644 build.zig delete mode 100644 examples/.bazelignore delete mode 100644 examples/.bazelrc delete mode 100644 examples/.bazelversion delete mode 100644 examples/.gitignore delete mode 100644 examples/.nvim.lua delete mode 100644 examples/.vscode/extensions.json delete mode 100644 examples/.vscode/settings.json delete mode 100644 examples/MODULE.bazel delete mode 100644 examples/MODULE.bazel.lock delete mode 100644 examples/loader/BUILD.bazel delete mode 100644 examples/loader/main.zig delete mode 100644 examples/platform_mappings delete mode 100755 examples/tools/buildifier.sh delete mode 100755 examples/tools/zls.sh delete mode 100644 examples/zls.build.json mode change 120000 => 100644 platform_mappings rename {examples => third_party/com_github_hejsil_clap}/BUILD.bazel (100%) rename {examples/third_party => third_party}/com_github_hejsil_clap/clap.bazel (100%) rename examples/third_party/non_module_deps.bzl => third_party/com_github_hejsil_clap/repo.bzl (55%) rename examples/build.zig => third_party/mnist/BUILD.bazel (100%) create mode 100644 third_party/mnist/repo.bzl create mode 100644 third_party/org_swig_swig/BUILD.bazel diff --git a/.bazelignore b/.bazelignore deleted file mode 120000 index 8fdc75e7..00000000 --- a/.bazelignore +++ /dev/null @@ -1 +0,0 @@ -examples/.bazelignore \ No newline at end of file diff --git a/.bazelignore b/.bazelignore new file mode 100644 index 00000000..1ef6fd4c --- /dev/null +++ b/.bazelignore @@ -0,0 +1,3 @@ +.* +third_party/modules +.zig-cache diff --git a/.bazelrc b/.bazelrc deleted file mode 120000 index b9f681ed..00000000 --- a/.bazelrc +++ /dev/null @@ -1 +0,0 @@ -examples/.bazelrc \ No newline at end of file diff --git a/.bazelrc b/.bazelrc new file mode 100644 index 00000000..928a29e1 --- /dev/null +++ b/.bazelrc @@ -0,0 +1,115 @@ +# Faster hash function +startup --digest_function=blake3 + +# Disable WORKSPACE +common --enable_workspace=false + +# Don't need for now +common --lockfile_mode=update + +# Needed by LLVM and/or XLA +common --experimental_repo_remote_exec +common --experimental_remote_cache_eviction_retries=5 + +# Self explanatory +common --enable_platform_specific_config + +# Use local and ZML registries +common --registry=file://%workspace%/third_party/ +common --registry=https://bazel-registry.zml.ai +common --registry=https://bcr.bazel.build + +# Ensure host tools are built optimized +common --host_compilation_mode=opt + +# Suppress all warnings, LLVM and friends are too noisy +common --copt='-w' +common --host_copt='-w' + +# Enable C++17 +common --cxxopt='-std=c++17' +common --host_cxxopt='-std=c++17' +common --enable_runfiles + +# Ensure the local env doesn't leak into the build +common --incompatible_strict_action_env + +# We don't to check the built files in the sandbox or repositories +common --experimental_check_output_files=false + +# Performance +common --legacy_external_runfiles=false + +common --experimental_output_directory_naming_scheme=diff_against_dynamic_baseline + +# Mount the zig cache into the sandboxes +build:macos --sandbox_add_mount_pair=/var/tmp +build:linux --sandbox_add_mount_pair=/tmp + +# Ensure workers are sandboxed +build --worker_sandboxing + +# Allows symlinks with spaces +build --experimental_inprocess_symlink_creation + +# Build in debug for now, by default +common --compilation_mode=dbg + +# Various cc options +common --experimental_cc_static_library +common --experimental_cc_shared_library +common --incompatible_use_cc_configure_from_rules_cc +common --experimental_starlark_cc_import + +common --experimental_platform_in_output_dir + +# Faster sandbox speeds +build --experimental_inmemory_sandbox_stashes + +# clang by default +common --compiler=clang + +# Enable resolving proto compiler from toolchains +common --incompatible_enable_proto_toolchain_resolution + +# Allow empty globs since some legacy rules +common --incompatible_disallow_empty_glob=false + +common:macos --action_env=MACOSX_DEPLOYMENT_TARGET=13.0 +common:macos --macos_sdk_version=13.0 + +# Used by the tsl overlay in python_init_repositories for the bazelified xla module 20241025.0-4663f04 and later +common --repo_env=HERMETIC_PYTHON_VERSION=3.11 + +# Don't detect the local C++ toolchain, be fully hermetic +# This is done only on Linux for now since we use the local toolchain on macOS +common:linux --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 + +# Print test errors in the console +test --test_output=errors + +common:silent --noshow_progress +common:silent --ui_event_filters=,+error,+fail +common:silent --show_result=0 +common:silent --logging=0 + +common --@rules_zig//zig/settings:copt=-fPIC +common --@rules_zig//zig/settings:copt=-fcompiler-rt + +common --@toolchains_llvm_bootstrapped//config:experimental_stub_libgcc_s=True + +# --config=debug will used optmized backend, but all the frontend zig code will be compiled in debug mode. +common:debug --compilation_mode=opt +common:debug --@rules_zig//zig/settings:mode=debug +common:debug --strategy=ZigBuildLib=local +common:debug --strategy=ZigBuildObj=local +common:debug --strategy=ZigBuildTestLib=local + +# --config=alldebug build everything in debug mode, including the backend. +common:alldebug --compilation_mode=dbg +common:alldebug --@rules_zig//zig/settings:mode=debug +common:alldebug --strategy=local + +common:release --compilation_mode=opt +common:release --copt=-Ofast +common:release --@rules_zig//zig/settings:mode=release_safe diff --git a/.bazelrc-examples b/.bazelrc-examples deleted file mode 100644 index 1f00f2a6..00000000 --- a/.bazelrc-examples +++ /dev/null @@ -1,2 +0,0 @@ -common --override_module=zml=%workspace%/../../ -common --registry=file://%workspace%/../third_party/ diff --git a/.bazelrc-zml b/.bazelrc-zml deleted file mode 100644 index 4ffe30c1..00000000 --- a/.bazelrc-zml +++ /dev/null @@ -1,2 +0,0 @@ -common --override_module=zml=./ -common --registry=file://%workspace%/third_party/ diff --git a/.bazelversion b/.bazelversion deleted file mode 120000 index 8ee3d828..00000000 --- a/.bazelversion +++ /dev/null @@ -1 +0,0 @@ -examples/.bazelversion \ No newline at end of file diff --git a/.bazelversion b/.bazelversion new file mode 100644 index 00000000..56b6be4e --- /dev/null +++ b/.bazelversion @@ -0,0 +1 @@ +8.3.1 diff --git a/.github/workflows/commit-examples.yaml b/.github/workflows/commit-examples.yaml deleted file mode 100644 index 6cc18da6..00000000 --- a/.github/workflows/commit-examples.yaml +++ /dev/null @@ -1,75 +0,0 @@ -name: examples-tests -on: - push: - branches: - - master - tags: - - '*' - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ !contains(github.ref, 'master')}} - -jobs: - examples-tests: - runs-on: ["runs-on","runner=4cpu-linux-x64", "image=ubuntu24-amd64", "run-id=${{ github.run_id }}", "extras=s3-cache"] - steps: - - uses: runs-on/action@v1 - - name: Checkout - uses: actions/checkout@v4 - - - uses: runs-on/snapshot@v1 - with: - path: /home/runner/.cache/bazel-disk - volume_size: 350 - - - name: TMP fix for the mount user - run: sudo chown -R runner:runner /home/runner/.cache/ - - - name: Setup Zig - uses: mlugg/setup-zig@v2 - with: - version: 0.14.1 - mirror: https://mirror.zml.ai/zig - - - name: Zig Format - run: zig fmt --check $(git ls-files --exclude-standard -- '*.zig' | xargs) - - - name: No autofix in Zig code - run: bash -c "! git grep -l '// autofix' -- '*.zig'" - - - uses: bazel-contrib/setup-bazel@0.15.0 - with: - bazelisk-version: 1.26.0 - bazelisk-cache: false - bazelrc: | - common --config=debug - common --verbose_failures - common --keep_going - common --test_output=streamed - common --color=yes - common --show_timestamps - common --disk_cache=/home/runner/.cache/bazel-disk - common --strip=always - - - name: Examples / bazel mod explain - working-directory: examples/ - run: bazel mod explain - - - name: Examples / bazel mod graph - working-directory: examples/ - run: bazel mod graph - - - name: Examples / bazel query //... - working-directory: examples/ - run: bazel query //... - - - name: Examples / bazel build //... - working-directory: examples/ - run: bazel build //... - - # Todo: uncomment when we have our first tests in examples, Bazel return code is not 0 if there are no tests - # - name: Examples / bazel test //... - # working-directory: examples/ - # run: bazel test //... diff --git a/.github/workflows/commit-framework.yaml b/.github/workflows/commit-framework.yaml deleted file mode 100644 index 1603e847..00000000 --- a/.github/workflows/commit-framework.yaml +++ /dev/null @@ -1,69 +0,0 @@ -name: framework-tests -on: - push: - branches: - - master - tags: - - '*' - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ !contains(github.ref, 'master')}} - -jobs: - framework-tests: - runs-on: ["runs-on","runner=4cpu-linux-x64", "image=ubuntu24-amd64", "run-id=${{ github.run_id }}", "extras=s3-cache"] - steps: - - uses: runs-on/action@v1 - - name: Checkout - uses: actions/checkout@v4 - - - uses: runs-on/snapshot@v1 - with: - path: /home/runner/.cache/bazel-disk - volume_size: 350 - - - name: TMP fix for the mount user - run: sudo chown -R runner:runner /home/runner/.cache/ - - - name: Setup Zig - uses: mlugg/setup-zig@v2 - with: - version: 0.14.1 - mirror: https://mirror.zml.ai/zig - - - name: Zig Format - run: zig fmt --check $(git ls-files --exclude-standard -- '*.zig' | xargs) - - - name: No autofix in Zig code - run: bash -c "! git grep -l '// autofix' -- '*.zig'" - - - uses: bazel-contrib/setup-bazel@0.15.0 - with: - bazelisk-version: 1.26.0 - bazelisk-cache: false - bazelrc: | - common --config=debug - common --verbose_failures - common --keep_going - common --test_output=streamed - common --color=yes - common --show_timestamps - common --disk_cache=/home/runner/.cache/bazel-disk - common --strip=always - - - name: ZML / bazel mod explain - run: bazel mod explain - - - name: ZML / bazel mod graph - run: bazel mod graph - - - name: ZML / bazel query //... - run: bazel query //... - - - name: ZML / bazel build //... - run: bazel build //... - - - name: ZML / bazel test //... - run: bazel test //... diff --git a/.github/workflows/commit.yaml b/.github/workflows/commit.yaml new file mode 100644 index 00000000..b6e7148d --- /dev/null +++ b/.github/workflows/commit.yaml @@ -0,0 +1,70 @@ +name: framework-tests +on: + push: + branches: + - master + tags: + - "*" + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ !contains(github.ref, 'master')}} + +jobs: + framework-tests: + runs-on: + [ + "runs-on", + "runner=4cpu-linux-x64", + "image=ubuntu24-amd64", + "run-id=${{ github.run_id }}", + "extras=s3-cache", + ] + steps: + - uses: runs-on/action@v1 + - name: Checkout + uses: actions/checkout@v4 + + - uses: runs-on/snapshot@v1 + with: + path: /home/runner/.cache/bazel-disk + volume_size: 350 + + - name: TMP fix for the mount user + run: sudo chown -R runner:runner /home/runner/.cache/ + + - name: Setup Zig + uses: mlugg/setup-zig@v2 + with: + version: 0.14.1 + mirror: https://mirror.zml.ai/zig + + - name: Zig Format + run: zig fmt --check $(git ls-files --exclude-standard -- '*.zig' | xargs) + + - name: No autofix in Zig code + run: bash -c "! git grep -l '// autofix' -- '*.zig'" + + - uses: bazel-contrib/setup-bazel@0.15.0 + with: + bazelisk-version: 1.26.0 + bazelisk-cache: false + bazelrc: | + common --config=debug + common --verbose_failures + common --keep_going + common --test_output=streamed + common --color=yes + common --show_timestamps + common --disk_cache=/home/runner/.cache/bazel-disk + common --strip=always + + - name: ZML / bazel mod explain + run: bazel mod explain + + - name: ZML / bazel mod graph + run: bazel mod graph + + - name: ZML / bazel test //... + run: bazel test //examples/... //mlir/... //zml/... diff --git a/.nvim.lua b/.nvim.lua deleted file mode 120000 index 2a8b1ccb..00000000 --- a/.nvim.lua +++ /dev/null @@ -1 +0,0 @@ -examples/.nvim.lua \ No newline at end of file diff --git a/.nvim.lua b/.nvim.lua new file mode 100644 index 00000000..a1f9ad95 --- /dev/null +++ b/.nvim.lua @@ -0,0 +1,36 @@ +-- Get the current working directory (CWD) +local cwd = vim.fn.getcwd() + +-- Load the local config file if it exists +local f, err = loadfile(".nvim.local.lua") +if not err then + f() +else + local capabilities = vim.lsp.protocol.make_client_capabilities() + + -- Prepend CWD to relative paths + local zls_cmd = cwd .. "/../zml/tools/zls.sh" + + vim.lsp.config["zls"] = { + capabilities = capabilities, + cmd = { zls_cmd }, + root_marker = { "build.zig" }, + filetypes = {"zig"}, + } + vim.lsp.enable('zls') + + vim.api.nvim_create_autocmd('BufWritePre',{ + pattern = {"*.zig", "*.zon"}, + callback = function(ev) + vim.lsp.buf.code_action({ + context = { only = { "source.organizeImports" } }, + apply = true, + }) + vim.lsp.buf.code_action({ + context = { only = { "source.fixAll" } }, + apply = true, + }) + vim.loop.sleep(100) + end + }) +end diff --git a/MODULE.bazel b/MODULE.bazel index cffd6f0e..d48734d2 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -2,25 +2,26 @@ module( name = "zml", ) -bazel_dep(name = "abseil-cpp", version = "20240722.0.bcr.2") +bazel_dep(name = "abseil-cpp", version = "20250814.0") bazel_dep(name = "aspect_bazel_lib", version = "2.14.0") bazel_dep(name = "aspect_rules_py", version = "1.3.2") -bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "bazel_skylib", version = "1.8.1") bazel_dep(name = "libxev", version = "20250718.0-9f785d2") bazel_dep(name = "patchelf", version = "0.18.0") bazel_dep(name = "pcre2", version = "10.43") -bazel_dep(name = "platforms", version = "0.0.11") +bazel_dep(name = "platforms", version = "1.0.0") bazel_dep(name = "protobuf", version = "32.0", repo_name = "com_google_protobuf") -bazel_dep(name = "rules_cc", version = "0.1.1") +bazel_dep(name = "rules_cc", version = "0.2.0") bazel_dep(name = "rules_distroless", version = "0.5.1") bazel_dep(name = "rules_proto", version = "7.1.0") -bazel_dep(name = "rules_python", version = "0.40.0") +bazel_dep(name = "rules_oci", version = "2.2.6") +bazel_dep(name = "rules_python", version = "1.5.3") bazel_dep(name = "rules_rust", version = "0.62.0") bazel_dep(name = "rules_uv", version = "0.65.0") bazel_dep(name = "rules_zig", version = "20250714.0-b14a4f1") bazel_dep(name = "sentencepiece", version = "20240618.0-d7ace0a") bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.2.4") -bazel_dep(name = "with_cfg.bzl", version = "0.9.1") +bazel_dep(name = "with_cfg.bzl", version = "0.11.0") bazel_dep(name = "xla", version = "20250718.0-6319f0d") bazel_dep(name = "zig-yaml", version = "20240903.0-83d5fdf") @@ -80,13 +81,13 @@ cuda = use_extension("//runtimes/cuda:cuda.bzl", "cuda_packages") use_repo(cuda, "libpjrt_cuda") rocm = use_extension("//runtimes/rocm:rocm.bzl", "rocm_packages") -use_repo(rocm, "libpjrt_rocm") +use_repo(rocm, "hipblaslt", "libpjrt_rocm", "rocblas") tpu = use_extension("//runtimes/tpu:tpu.bzl", "tpu_packages") use_repo(tpu, "libpjrt_tpu") neuron = use_extension("//runtimes/neuron:neuron.bzl", "neuron_packages") -use_repo(neuron, "libpjrt_neuron") +use_repo(neuron, "aws-neuronx-collectives", "aws-neuronx-runtime-lib", "libgomp1", "libpjrt_neuron", "zlib1g") zls = use_extension("//third_party/zls:zls.bzl", "repo") use_repo(zls, "zls_aarch64-macos", "zls_x86_64-linux", "zls_x86_64-macos") @@ -135,7 +136,7 @@ crate.from_cargo( use_repo(crate, "crates") non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") -use_repo(non_module_deps, "com_google_sentencepiece", "org_swig_swig") +use_repo(non_module_deps, "com_github_hejsil_clap", "com_google_sentencepiece", "mnist", "org_swig_swig") apt = use_extension("@rules_distroless//apt:extensions.bzl", "apt") apt.install( @@ -158,3 +159,39 @@ apt.install( manifest = "//runtimes/neuron:packages.yaml", ) use_repo(apt, "apt_neuron") + +oci = use_extension("@rules_oci//oci:extensions.bzl", "oci") +oci.pull( + name = "distroless_cc_debian12", + digest = "sha256:eccec5274132c1be0ce5d2c8e6fe41033e64af5e987ccee9007826e4c012069d", + image = "gcr.io/distroless/cc-debian12", + platforms = ["linux/amd64"], +) +use_repo(oci, "distroless_cc_debian12", "distroless_cc_debian12_linux_amd64") + +huggingface = use_extension("@zml//bazel:huggingface.bzl", "huggingface") + +################################################################################ +# ModernBERT +huggingface.model( + name = "ModernBERT-base", + build_file_content = """exports_files(glob(["**"]), visibility = ["//visibility:public"])""", + commit = "94032bb66234a691cf6248265170006a7ced4970", + includes = [ + "model.safetensors", + "tokenizer.json", + ], + model = "answerdotai/ModernBERT-base", +) +huggingface.model( + name = "ModernBERT-large", + build_file_content = """exports_files(glob(["**"]), visibility = ["//visibility:public"])""", + commit = "4bbcbf40bed02ce487125bcb3c897ea9bdc88340", + includes = [ + "model.safetensors", + "tokenizer.json", + ], + model = "answerdotai/ModernBERT-large", +) +use_repo(huggingface, "ModernBERT-base", "ModernBERT-large") +################################################################################ diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 412d1fcd..4ba6927d 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -10,15 +10,16 @@ "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20240722.0.bcr.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20250127.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20250512.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/abseil-cpp/20250814.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/apple_support/1.11.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/apple_support/1.15.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/apple_support/1.17.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/apple_support/1.21.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.10.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.9.4/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_rules_py/1.3.2/MODULE.bazel": "not found", @@ -35,7 +36,9 @@ "https://bazel-registry.zml.ai/modules/bazel_features/1.24.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.25.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.27.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/bazel_features/1.28.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.3.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/bazel_features/1.30.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.4.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.9.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.9.1/MODULE.bazel": "not found", @@ -50,6 +53,7 @@ "https://bazel-registry.zml.ai/modules/bazel_skylib/1.6.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_skylib/1.7.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_skylib/1.7.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/bazel_skylib/1.8.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/6.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/7.3.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/8.0.3/MODULE.bazel": "not found", @@ -74,6 +78,7 @@ "https://bazel-registry.zml.ai/modules/platforms/0.0.7/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/platforms/0.0.8/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/platforms/0.0.9/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/platforms/1.0.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/protobuf/21.7/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/protobuf/23.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/protobuf/24.4/MODULE.bazel": "not found", @@ -105,6 +110,7 @@ "https://bazel-registry.zml.ai/modules/rules_cc/0.0.8/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.9/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.1.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_cc/0.2.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_distroless/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_fuzzing/0.5.2/MODULE.bazel": "not found", @@ -120,9 +126,10 @@ "https://bazel-registry.zml.ai/modules/rules_java/7.2.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_java/7.3.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_java/7.6.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/8.11.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_java/8.12.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_java/8.3.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_java/8.5.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_java/8.6.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_java/8.6.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_java/8.8.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_jvm_external/4.4.2/MODULE.bazel": "not found", @@ -140,6 +147,7 @@ "https://bazel-registry.zml.ai/modules/rules_license/0.0.8/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_license/1.0.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_multitool/0.11.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_oci/2.2.6/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_pkg/0.7.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_pkg/0.9.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_pkg/1.0.1/MODULE.bazel": "not found", @@ -162,6 +170,7 @@ "https://bazel-registry.zml.ai/modules/rules_python/0.40.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_python/1.3.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_python/1.4.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_python/1.5.3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_rust/0.62.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.2.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.3.0/MODULE.bazel": "not found", @@ -172,6 +181,7 @@ "https://bazel-registry.zml.ai/modules/rules_uv/0.65.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.3/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/stardoc/0.5.4/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.6/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.6.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.7.0/MODULE.bazel": "not found", @@ -179,14 +189,12 @@ "https://bazel-registry.zml.ai/modules/stardoc/0.7.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/toolchains_llvm_bootstrapped/0.2.4/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/toolchains_protoc/0.4.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/upb/0.0.0-20230516-61a97ef/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/with_cfg.bzl/0.9.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/with_cfg.bzl/0.11.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/with_cfg.bzl/0.9.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/zlib/1.2.11/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/zlib/1.2.13/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/zlib/1.3.1/MODULE.bazel": "not found", "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", @@ -199,10 +207,10 @@ "https://bcr.bazel.build/modules/abseil-cpp/20240116.0/MODULE.bazel": "98dc378d64c12a4e4741ad3362f87fb737ee6a0886b2d90c3cdbb4d93ea3e0bf", "https://bcr.bazel.build/modules/abseil-cpp/20240116.1/MODULE.bazel": "37bcdb4440fbb61df6a1c296ae01b327f19e9bb521f9b8e26ec854b6f97309ed", "https://bcr.bazel.build/modules/abseil-cpp/20240116.2/MODULE.bazel": "73939767a4686cd9a520d16af5ab440071ed75cec1a876bf2fcfaf1f71987a16", - "https://bcr.bazel.build/modules/abseil-cpp/20240722.0.bcr.2/MODULE.bazel": "c3661b44c9d3f17f0b65ffb544896aaeb89127398ea867537babac18133a002a", "https://bcr.bazel.build/modules/abseil-cpp/20250127.1/MODULE.bazel": "c4a89e7ceb9bf1e25cf84a9f830ff6b817b72874088bf5141b314726e46a57c1", "https://bcr.bazel.build/modules/abseil-cpp/20250512.1/MODULE.bazel": "d209fdb6f36ffaf61c509fcc81b19e81b411a999a934a032e10cd009a0226215", - "https://bcr.bazel.build/modules/abseil-cpp/20250512.1/source.json": "d725d73707d01bb46ab3ca59ba408b8e9bd336642ca77a2269d4bfb8bbfd413d", + "https://bcr.bazel.build/modules/abseil-cpp/20250814.0/MODULE.bazel": "c43c16ca2c432566cdb78913964497259903ebe8fb7d9b57b38e9f1425b427b8", + "https://bcr.bazel.build/modules/abseil-cpp/20250814.0/source.json": "b88bff599ceaf0f56c264c749b1606f8485cec3b8c38ba30f88a4df9af142861", "https://bcr.bazel.build/modules/apple_support/1.11.1/MODULE.bazel": "1843d7cd8a58369a444fc6000e7304425fba600ff641592161d9f15b179fb896", "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", "https://bcr.bazel.build/modules/apple_support/1.17.1/MODULE.bazel": "655c922ab1209978a94ef6ca7d9d43e940cd97d9c172fb55f94d91ac53f8610b", @@ -211,6 +219,7 @@ "https://bcr.bazel.build/modules/aspect_bazel_lib/2.10.0/MODULE.bazel": "ae12288421b12faeb09bc4cfb3c05945ce66df5c55fca8a2f2f66ae9f21c7acb", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/source.json": "0cf1826853b0bef8b5cd19c0610d717500f5521aa2b38b72b2ec302ac5e7526c", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "780d1a6522b28f5edb7ea09630748720721dfe27690d65a2d33aa7509de77e07", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "812d2dd42f65dca362152101fbec418029cc8fd34cbad1a2fde905383d705838", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.9.4/MODULE.bazel": "ccc41028429f894b02fde7ef67d416cba3ba5084ed9ddb9bb6107aa82d118776", "https://bcr.bazel.build/modules/aspect_rules_py/1.3.2/MODULE.bazel": "0bd9f9f787a70a725b9829dd17c19e89ea86b1e06f25df505f780d77a96f55a5", @@ -228,8 +237,10 @@ "https://bcr.bazel.build/modules/bazel_features/1.24.0/MODULE.bazel": "4796b4c25b47053e9bbffa792b3792d07e228ff66cd0405faef56a978708acd4", "https://bcr.bazel.build/modules/bazel_features/1.25.0/MODULE.bazel": "e2e60a10a6da64bbf533f15ca652bf61a033e41c2ed734d79a9a08ba87f68c1a", "https://bcr.bazel.build/modules/bazel_features/1.27.0/MODULE.bazel": "621eeee06c4458a9121d1f104efb80f39d34deff4984e778359c60eaf1a8cb65", - "https://bcr.bazel.build/modules/bazel_features/1.27.0/source.json": "ed8cf0ef05c858dce3661689d0a2b110ff398e63994e178e4f1f7555a8067fed", + "https://bcr.bazel.build/modules/bazel_features/1.28.0/MODULE.bazel": "4b4200e6cbf8fa335b2c3f43e1d6ef3e240319c33d43d60cc0fbd4b87ece299d", "https://bcr.bazel.build/modules/bazel_features/1.3.0/MODULE.bazel": "cdcafe83ec318cda34e02948e81d790aab8df7a929cec6f6969f13a489ccecd9", + "https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87", + "https://bcr.bazel.build/modules/bazel_features/1.30.0/source.json": "b07e17f067fe4f69f90b03b36ef1e08fe0d1f3cac254c1241a1818773e3423bc", "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", "https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b", "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", @@ -244,7 +255,8 @@ "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/MODULE.bazel": "8fdee2dbaace6c252131c00e1de4b165dc65af02ea278476187765e1a617b917", "https://bcr.bazel.build/modules/bazel_skylib/1.7.0/MODULE.bazel": "0db596f4563de7938de764cc8deeabec291f55e8ec15299718b93c4423e9796d", "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/MODULE.bazel": "3120d80c5861aa616222ec015332e5f8d3171e062e3e804a2a0253e1be26e59b", - "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/source.json": "f121b43eeefc7c29efbd51b83d08631e2347297c95aac9764a701f2a6a2bb953", + "https://bcr.bazel.build/modules/bazel_skylib/1.8.1/MODULE.bazel": "88ade7293becda963e0e3ea33e7d54d3425127e0a326e0d17da085a5f1f03ff6", + "https://bcr.bazel.build/modules/bazel_skylib/1.8.1/source.json": "7ebaefba0b03efe59cac88ed5bbc67bcf59a3eff33af937345ede2a38b2d368a", "https://bcr.bazel.build/modules/buildifier_prebuilt/6.4.0/MODULE.bazel": "37389c6b5a40c59410b4226d3bb54b08637f393d66e2fa57925c6fcf68e64bf4", "https://bcr.bazel.build/modules/buildifier_prebuilt/7.3.1/MODULE.bazel": "537faf0ad9f5892910074b8e43b4c91c96f1d5d86b6ed04bdbe40cf68aa48b68", "https://bcr.bazel.build/modules/buildifier_prebuilt/8.0.3/MODULE.bazel": "4791b031727d1183c183a0b8fd5fc0fe8d8834c45f9a22efd26be6c962b9cfee", @@ -270,13 +282,14 @@ "https://bcr.bazel.build/modules/pcre2/10.43/source.json": "8b4149e707094f1d5b57df7216539c3415226e814085c4d960bd9f3d49581b88", "https://bcr.bazel.build/modules/platforms/0.0.10/MODULE.bazel": "8cb8efaf200bdeb2150d93e162c40f388529a25852b332cec879373771e48ed5", "https://bcr.bazel.build/modules/platforms/0.0.11/MODULE.bazel": "0daefc49732e227caa8bfa834d65dc52e8cc18a2faf80df25e8caea151a9413f", - "https://bcr.bazel.build/modules/platforms/0.0.11/source.json": "f7e188b79ebedebfe75e9e1d098b8845226c7992b307e28e1496f23112e8fc29", "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee", "https://bcr.bazel.build/modules/platforms/0.0.5/MODULE.bazel": "5733b54ea419d5eaf7997054bb55f6a1d0b5ff8aedf0176fef9eea44f3acda37", "https://bcr.bazel.build/modules/platforms/0.0.6/MODULE.bazel": "ad6eeef431dc52aefd2d77ed20a4b353f8ebf0f4ecdd26a807d2da5aa8cd0615", "https://bcr.bazel.build/modules/platforms/0.0.7/MODULE.bazel": "72fd4a0ede9ee5c021f6a8dd92b503e089f46c227ba2813ff183b71616034814", "https://bcr.bazel.build/modules/platforms/0.0.8/MODULE.bazel": "9f142c03e348f6d263719f5074b21ef3adf0b139ee4c5133e2aa35664da9eb2d", "https://bcr.bazel.build/modules/platforms/0.0.9/MODULE.bazel": "4a87a60c927b56ddd67db50c89acaa62f4ce2a1d2149ccb63ffd871d5ce29ebc", + "https://bcr.bazel.build/modules/platforms/1.0.0/MODULE.bazel": "f05feb42b48f1b3c225e4ccf351f367be0371411a803198ec34a389fb22aa580", + "https://bcr.bazel.build/modules/platforms/1.0.0/source.json": "f4ff1fd412e0246fd38c82328eb209130ead81d62dcd5a9e40910f867f733d96", "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7", "https://bcr.bazel.build/modules/protobuf/23.1/MODULE.bazel": "88b393b3eb4101d18129e5db51847cd40a5517a53e81216144a8c32dfeeca52a", "https://bcr.bazel.build/modules/protobuf/24.4/MODULE.bazel": "7bc7ce5f2abf36b3b7b7c8218d3acdebb9426aeb35c2257c96445756f970eb12", @@ -313,7 +326,8 @@ "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", "https://bcr.bazel.build/modules/rules_cc/0.1.1/MODULE.bazel": "2f0222a6f229f0bf44cd711dc13c858dad98c62d52bd51d8fc3a764a83125513", - "https://bcr.bazel.build/modules/rules_cc/0.1.1/source.json": "d61627377bd7dd1da4652063e368d9366fc9a73920bfa396798ad92172cf645c", + "https://bcr.bazel.build/modules/rules_cc/0.2.0/MODULE.bazel": "b5c17f90458caae90d2ccd114c81970062946f49f355610ed89bebf954f5783c", + "https://bcr.bazel.build/modules/rules_cc/0.2.0/source.json": "5f7f4e578e950adbf194217d4b607237a8197fc53ba46c367b3d61a86ecf35c2", "https://bcr.bazel.build/modules/rules_distroless/0.5.1/MODULE.bazel": "2a63f4744d30749128105da5f96adf7caf5628e37548293f89e7fa39c3b3f2c2", "https://bcr.bazel.build/modules/rules_distroless/0.5.1/source.json": "c6b9ff7f325bfed89c3671757f14c1d1bc6077d0fcef809b8aa2d007cac7dd1d", "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", @@ -330,10 +344,11 @@ "https://bcr.bazel.build/modules/rules_java/7.2.0/MODULE.bazel": "06c0334c9be61e6cef2c8c84a7800cef502063269a5af25ceb100b192453d4ab", "https://bcr.bazel.build/modules/rules_java/7.3.2/MODULE.bazel": "50dece891cfdf1741ea230d001aa9c14398062f2b7c066470accace78e412bc2", "https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe", - "https://bcr.bazel.build/modules/rules_java/8.11.0/MODULE.bazel": "c3d280bc5ff1038dcb3bacb95d3f6b83da8dd27bba57820ec89ea4085da767ad", - "https://bcr.bazel.build/modules/rules_java/8.11.0/source.json": "302b52a39259a85aa06ca3addb9787864ca3e03b432a5f964ea68244397e7544", + "https://bcr.bazel.build/modules/rules_java/8.12.0/MODULE.bazel": "8e6590b961f2defdfc2811c089c75716cb2f06c8a4edeb9a8d85eaa64ee2a761", + "https://bcr.bazel.build/modules/rules_java/8.12.0/source.json": "cbd5d55d9d38d4008a7d00bee5b5a5a4b6031fcd4a56515c9accbcd42c7be2ba", "https://bcr.bazel.build/modules/rules_java/8.3.2/MODULE.bazel": "7336d5511ad5af0b8615fdc7477535a2e4e723a357b6713af439fe8cf0195017", "https://bcr.bazel.build/modules/rules_java/8.5.1/MODULE.bazel": "d8a9e38cc5228881f7055a6079f6f7821a073df3744d441978e7a43e20226939", + "https://bcr.bazel.build/modules/rules_java/8.6.0/MODULE.bazel": "9c064c434606d75a086f15ade5edb514308cccd1544c2b2a89bbac4310e41c71", "https://bcr.bazel.build/modules/rules_java/8.6.1/MODULE.bazel": "f4808e2ab5b0197f094cabce9f4b006a27766beb6a9975931da07099560ca9c2", "https://bcr.bazel.build/modules/rules_java/8.8.0/MODULE.bazel": "de589d0880911ac007abd521b9f0ddcd8b0dbd05c8553e6f8124a050b83acf7d", "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7", @@ -355,6 +370,8 @@ "https://bcr.bazel.build/modules/rules_license/1.0.0/source.json": "a52c89e54cc311196e478f8382df91c15f7a2bfdf4c6cd0e2675cc2ff0b56efb", "https://bcr.bazel.build/modules/rules_multitool/0.11.0/MODULE.bazel": "8d9dda78d2398e136300d3ef4fbcc89ede7c32c158d8c016fa7d032df41c4aaf", "https://bcr.bazel.build/modules/rules_multitool/0.11.0/source.json": "0b86574a1eaff37c33aafaff095ea16d6ac846beb94ffc74c4fcf626f8f80681", + "https://bcr.bazel.build/modules/rules_oci/2.2.6/MODULE.bazel": "2ba6ddd679269e00aeffe9ca04faa2d0ca4129650982c9246d0d459fe2da47d9", + "https://bcr.bazel.build/modules/rules_oci/2.2.6/source.json": "94e7decb8f95d9465b0bbea71c65064cd16083be1350c7468f131818641dc4a5", "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc", "https://bcr.bazel.build/modules/rules_pkg/0.9.1/MODULE.bazel": "af00144208c4be503bc920d043ba3284fb37385b3f6160b4a4daf4df80b4b823", "https://bcr.bazel.build/modules/rules_pkg/1.0.1/MODULE.bazel": "5b1df97dbc29623bccdf2b0dcd0f5cb08e2f2c9050aab1092fd39a41e82686ff", @@ -379,7 +396,8 @@ "https://bcr.bazel.build/modules/rules_python/0.40.0/MODULE.bazel": "9d1a3cd88ed7d8e39583d9ffe56ae8a244f67783ae89b60caafc9f5cf318ada7", "https://bcr.bazel.build/modules/rules_python/1.3.0/MODULE.bazel": "8361d57eafb67c09b75bf4bbe6be360e1b8f4f18118ab48037f2bd50aa2ccb13", "https://bcr.bazel.build/modules/rules_python/1.4.1/MODULE.bazel": "8991ad45bdc25018301d6b7e1d3626afc3c8af8aaf4bc04f23d0b99c938b73a6", - "https://bcr.bazel.build/modules/rules_python/1.4.1/source.json": "8ec8c90c70ccacc4de8ca1b97f599e756fb59173e898ee08b733006650057c07", + "https://bcr.bazel.build/modules/rules_python/1.5.3/MODULE.bazel": "d0b7fb08458ca7fd80a26bc00c9e0f1d011609cc3da0381faa2eccd88c6ebd98", + "https://bcr.bazel.build/modules/rules_python/1.5.3/source.json": "06961e322e15331a2d88115a65af5d3f77cc46793f9d9aa0f928b95287337f12", "https://bcr.bazel.build/modules/rules_rust/0.62.0/MODULE.bazel": "6a15b57982e278793c684f426e19166e62e73f1bd45fe3b6bcedd0b901177b37", "https://bcr.bazel.build/modules/rules_rust/0.62.0/source.json": "1b3a6551a585ee47cafa21550c5eb87c6f3a56bb9761f9e3421ff1102f220437", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", @@ -394,6 +412,7 @@ "https://bcr.bazel.build/modules/rules_uv/0.65.0/source.json": "315237fb5f44ada2d537aabb2f89c6ba2296569ed63fc7340ca66d65f5730081", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", + "https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4", "https://bcr.bazel.build/modules/stardoc/0.5.6/MODULE.bazel": "c43dabc564990eeab55e25ed61c07a1aadafe9ece96a4efabb3f8bf9063b71ef", "https://bcr.bazel.build/modules/stardoc/0.6.2/MODULE.bazel": "7060193196395f5dd668eda046ccbeacebfd98efc77fed418dbe2b82ffaa39fd", "https://bcr.bazel.build/modules/stardoc/0.7.0/MODULE.bazel": "05e3d6d30c099b6770e97da986c53bd31844d7f13d41412480ea265ac9e8079c", @@ -404,16 +423,13 @@ "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/source.json": "32bd87e5f4d7acc57c5b2ff7c325ae3061d5e242c0c4c214ae87e0f1c13e54cb", "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.2.4/MODULE.bazel": "859f02226bc313a622f56b816582fb55d5aaced4a1ecde7397dd4a9ebd24321e", "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.2.4/source.json": "43c5fd6b585cbf0b642157b6843a25425d541fabe638acb7bc908846c4bda8ca", - "https://bcr.bazel.build/modules/toolchains_protoc/0.4.1/MODULE.bazel": "05d6c16474a7a96002dd5512c444aa6965ebcf95f3e0c47001aa18269fbecec9", - "https://bcr.bazel.build/modules/toolchains_protoc/0.4.1/source.json": "474d926c8e845762faaa42e792fc66cd36794daee4c8af900f673fad0b403a8d", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", "https://bcr.bazel.build/modules/upb/0.0.0-20230516-61a97ef/MODULE.bazel": "c0df5e35ad55e264160417fd0875932ee3c9dda63d9fccace35ac62f45e1b6f9", - "https://bcr.bazel.build/modules/with_cfg.bzl/0.9.1/MODULE.bazel": "5b880a01ab5b3bdb30931075d6b8e58075f999400f7a544632dc7b4a575b3c1e", + "https://bcr.bazel.build/modules/with_cfg.bzl/0.11.0/MODULE.bazel": "bcb1dba64b74542648a93d6313e160b2670f22395581edd8c4ffadb811618ef7", + "https://bcr.bazel.build/modules/with_cfg.bzl/0.11.0/source.json": "19102ff7f611a7a821e11d866868ad541abb4f5757e37e1c40adb6834ebf6205", "https://bcr.bazel.build/modules/with_cfg.bzl/0.9.2/MODULE.bazel": "8008021cb802e87e22b3e2b6f403c0529d4b865703fdcd40e2f36c23a651291c", - "https://bcr.bazel.build/modules/with_cfg.bzl/0.9.2/source.json": "06d276bd336d6c9f069d4a68729de4e5d322129665f871c2b328a14b26b7cd99", "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", "https://bcr.bazel.build/modules/zlib/1.2.13/MODULE.bazel": "aa6deb1b83c18ffecd940c4119aff9567cd0a671d7bba756741cb2ef043a29d5", - "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "af322bc08976524477c79d1e45e241b6efbeb918c497e8840b8ab116802dda79", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "eec517b5bbe5492629466e11dae908d043364302283de25581e3eb944326c4ca", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/source.json": "22bc55c47af97246cfc093d0acf683a7869377de362b5d1c552c2c2e16b7a806", "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198" @@ -422,8 +438,8 @@ "moduleExtensions": { "@@apple_support+//crosstool:setup.bzl%apple_cc_configure_extension": { "general": { - "bzlTransitiveDigest": "JJV/C5bhYKZip41t/GGZvqIMmmrcwr0Ydybkn97uXxQ=", - "usagesDigest": "39X2JjPCOAk6sThDALGv1L4q85GNjda2yfszm/phxxw=", + "bzlTransitiveDigest": "4vzWoFey2mnm603onNMfaDX4DRoTrnIgZawOBMnWpoI=", + "usagesDigest": "Er40Iux4rJmj16bfvk5XcY7UiJ/P97xDmaxJ0KpWmLs=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -453,8 +469,8 @@ }, "@@aspect_rules_py+//py:extensions.bzl%py_tools": { "general": { - "bzlTransitiveDigest": "geB/ya+5nbONwFUgzta9iVBeMKtviiSBwlZnE7Avcgs=", - "usagesDigest": "tVlELaqQFX7HPl1uFzRY9m7nVV7tY9roq1qfK75ApwE=", + "bzlTransitiveDigest": "YcVwG9jheqdm2/gbTPG6E6bzLLpuwTdLTs45NueVdyo=", + "usagesDigest": "/X+U31L/0W+mPo15khS6kjLevIqdkqh+JgfyWHDdHH8=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -489,12 +505,6 @@ "platform": "windows_amd64" } }, - "bsd_tar_windows_arm64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "windows_arm64" - } - }, "bsd_tar_toolchains": { "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%tar_toolchains_repo", "attributes": { @@ -502,25 +512,25 @@ } }, "rules_py_tools.darwin_amd64": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:tools.bzl%prebuilt_tool_repo", "attributes": { "platform": "darwin_amd64" } }, "rules_py_tools.darwin_arm64": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:tools.bzl%prebuilt_tool_repo", "attributes": { "platform": "darwin_arm64" } }, "rules_py_tools.linux_amd64": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:tools.bzl%prebuilt_tool_repo", "attributes": { "platform": "linux_amd64" } }, "rules_py_tools.linux_arm64": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:tools.bzl%prebuilt_tool_repo", "attributes": { "platform": "linux_arm64" } @@ -555,16 +565,145 @@ ], [ "aspect_rules_py+", - "aspect_tools_telemetry_report", - "aspect_tools_telemetry++telemetry+aspect_tools_telemetry_report" - ], + "bazel_tools", + "bazel_tools" + ] + ] + } + }, + "@@buildifier_prebuilt+//:defs.bzl%buildifier_prebuilt_deps_extension": { + "general": { + "bzlTransitiveDigest": "2b8+Z7ONtWAGRdJuUABpUTkgdRtGsS/rswXQSm8q/ac=", + "usagesDigest": "E92htUw9elHOT0ZHIEdwgPVnslFqL2iu3pcplYXXAxk=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "buildifier_darwin_amd64": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "urls": [ + "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildifier-darwin-amd64" + ], + "downloaded_file_path": "buildifier", + "executable": true, + "sha256": "b7a3152cde0b3971b1107f2274afe778c5c154dcdf6c9c669a231e3c004f047e" + } + }, + "buildifier_darwin_arm64": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "urls": [ + "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildifier-darwin-arm64" + ], + "downloaded_file_path": "buildifier", + "executable": true, + "sha256": "674c663f7b5cd03c002f8ca834a8c1c008ccb527a0a2a132d08a7a355883b22d" + } + }, + "buildifier_linux_amd64": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "urls": [ + "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildifier-linux-amd64" + ], + "downloaded_file_path": "buildifier", + "executable": true, + "sha256": "c969487c1af85e708576c8dfdd0bb4681eae58aad79e68ae48882c70871841b7" + } + }, + "buildifier_linux_arm64": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "urls": [ + "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildifier-linux-arm64" + ], + "downloaded_file_path": "buildifier", + "executable": true, + "sha256": "bdd9b92e2c65d46affeecaefb54e68d34c272d1f4a8c5b54929a3e92ab78820a" + } + }, + "buildifier_windows_amd64": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "urls": [ + "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildifier-windows-amd64.exe" + ], + "downloaded_file_path": "buildifier.exe", + "executable": true, + "sha256": "63a242f57e253efe7b9573d739c08a3d0e628efd84015c8dad17d87b6429e443" + } + }, + "buildozer_darwin_amd64": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "urls": [ + "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildozer-darwin-amd64" + ], + "downloaded_file_path": "buildozer", + "executable": true, + "sha256": "7aa7929faddcee5a879135356bb4d38af3e895893c78d361e7cdb8dfb3e7d5e4" + } + }, + "buildozer_darwin_arm64": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "urls": [ + "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildozer-darwin-arm64" + ], + "downloaded_file_path": "buildozer", + "executable": true, + "sha256": "33b776862c31bfb4e2e6e88186cae02746d9084da8b7341930e77d42718277cc" + } + }, + "buildozer_linux_amd64": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "urls": [ + "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildozer-linux-amd64" + ], + "downloaded_file_path": "buildozer", + "executable": true, + "sha256": "775d17118aecd023352e75611dbe7c12a340e23615ffb270a1dec3f339239949" + } + }, + "buildozer_linux_arm64": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "urls": [ + "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildozer-linux-arm64" + ], + "downloaded_file_path": "buildozer", + "executable": true, + "sha256": "058ad60f279a7b1db339bb287aed2056a992eddfe1b5d2e56d86256bdc9a93b3" + } + }, + "buildozer_windows_amd64": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "urls": [ + "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildozer-windows-amd64.exe" + ], + "downloaded_file_path": "buildozer.exe", + "executable": true, + "sha256": "5ba106b25230e0e321b7259bee325ae6fa5b5771e54163ad865b1d18856f59ca" + } + }, + "buildifier_prebuilt_toolchains": { + "repoRuleId": "@@buildifier_prebuilt+//:defs.bzl%_buildifier_toolchain_setup", + "attributes": { + "assets_json": "[{\"arch\":\"amd64\",\"name\":\"buildifier\",\"platform\":\"darwin\",\"sha256\":\"b7a3152cde0b3971b1107f2274afe778c5c154dcdf6c9c669a231e3c004f047e\",\"version\":\"v8.0.3\"},{\"arch\":\"arm64\",\"name\":\"buildifier\",\"platform\":\"darwin\",\"sha256\":\"674c663f7b5cd03c002f8ca834a8c1c008ccb527a0a2a132d08a7a355883b22d\",\"version\":\"v8.0.3\"},{\"arch\":\"amd64\",\"name\":\"buildifier\",\"platform\":\"linux\",\"sha256\":\"c969487c1af85e708576c8dfdd0bb4681eae58aad79e68ae48882c70871841b7\",\"version\":\"v8.0.3\"},{\"arch\":\"arm64\",\"name\":\"buildifier\",\"platform\":\"linux\",\"sha256\":\"bdd9b92e2c65d46affeecaefb54e68d34c272d1f4a8c5b54929a3e92ab78820a\",\"version\":\"v8.0.3\"},{\"arch\":\"amd64\",\"name\":\"buildifier\",\"platform\":\"windows\",\"sha256\":\"63a242f57e253efe7b9573d739c08a3d0e628efd84015c8dad17d87b6429e443\",\"version\":\"v8.0.3\"},{\"arch\":\"amd64\",\"name\":\"buildozer\",\"platform\":\"darwin\",\"sha256\":\"7aa7929faddcee5a879135356bb4d38af3e895893c78d361e7cdb8dfb3e7d5e4\",\"version\":\"v8.0.3\"},{\"arch\":\"arm64\",\"name\":\"buildozer\",\"platform\":\"darwin\",\"sha256\":\"33b776862c31bfb4e2e6e88186cae02746d9084da8b7341930e77d42718277cc\",\"version\":\"v8.0.3\"},{\"arch\":\"amd64\",\"name\":\"buildozer\",\"platform\":\"linux\",\"sha256\":\"775d17118aecd023352e75611dbe7c12a340e23615ffb270a1dec3f339239949\",\"version\":\"v8.0.3\"},{\"arch\":\"arm64\",\"name\":\"buildozer\",\"platform\":\"linux\",\"sha256\":\"058ad60f279a7b1db339bb287aed2056a992eddfe1b5d2e56d86256bdc9a93b3\",\"version\":\"v8.0.3\"},{\"arch\":\"amd64\",\"name\":\"buildozer\",\"platform\":\"windows\",\"sha256\":\"5ba106b25230e0e321b7259bee325ae6fa5b5771e54163ad865b1d18856f59ca\",\"version\":\"v8.0.3\"}]" + } + } + }, + "recordedRepoMappingEntries": [ [ - "aspect_rules_py+", + "buildifier_prebuilt+", "bazel_skylib", "bazel_skylib+" ], [ - "aspect_rules_py+", + "buildifier_prebuilt+", "bazel_tools", "bazel_tools" ] @@ -573,7 +712,7 @@ }, "@@pybind11_bazel+//:internal_configure.bzl%internal_configure_extension": { "general": { - "bzlTransitiveDigest": "BscKLx1uTqPejNWXrpBVNole5HrQ/iL/+SPa+29Shuk=", + "bzlTransitiveDigest": "YX3gHhSTjjghUwEDKUr1WO9E6h/uiehMZuE9vZup5+g=", "usagesDigest": "D1r3lfzMuUBFxgG8V6o0bQTLMk3GkaGOaPzw53wrwyw=", "recordedFileInputs": { "@@pybind11_bazel+//MODULE.bazel": "45d98c531db5f8430f847b195357dca36665d9c926cbbee77a85105fc6865d27" @@ -601,6 +740,88 @@ ] } }, + "@@rules_apple+//apple:apple.bzl%provisioning_profile_repository_extension": { + "general": { + "bzlTransitiveDigest": "mVdP3ANttcsLRv2k6Sbzs06sQDCgBTB8CPMicPSeCW4=", + "usagesDigest": "+Mri48un5kxPRrX8WFbwbgbUsG6JygfULZLaltX0weQ=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "local_provisioning_profiles": { + "repoRuleId": "@@rules_apple+//apple/internal:local_provisioning_profiles.bzl%provisioning_profile_repository", + "attributes": {} + } + }, + "recordedRepoMappingEntries": [ + [ + "apple_support+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "bazel_tools", + "rules_cc", + "rules_cc+" + ], + [ + "rules_apple+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "rules_apple+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_apple+", + "build_bazel_apple_support", + "apple_support+" + ], + [ + "rules_apple+", + "build_bazel_rules_swift", + "rules_swift+" + ], + [ + "rules_cc+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_cc+", + "rules_cc", + "rules_cc+" + ], + [ + "rules_swift+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "rules_swift+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_swift+", + "build_bazel_apple_support", + "apple_support+" + ], + [ + "rules_swift+", + "build_bazel_rules_swift", + "rules_swift+" + ], + [ + "rules_swift+", + "build_bazel_rules_swift_local_config", + "rules_swift++non_module_deps+build_bazel_rules_swift_local_config" + ] + ] + } + }, "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { "general": { "bzlTransitiveDigest": "URQTcVYyatfFO8w2HD1Z1tb8lTBnhHn1wkiu5DYTSow=", @@ -668,7 +889,7 @@ "@@rules_multitool+//multitool:extension.bzl%multitool": { "general": { "bzlTransitiveDigest": "CxG1tSKAppSqmYJtpLgGpxrOvrd/xewKD0M2kCNfvOo=", - "usagesDigest": "kKFKsjNo1vbiSGlyYYHdSAj1Z8xtmvq8JBs56ZEpNS4=", + "usagesDigest": "Bn6IoM2pzksz/oBIRwZyfLpBZBFrZnzat00Xcp/lWqw=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -761,158 +982,216 @@ ] } }, - "@@rules_python+//python/uv:uv.bzl%uv": { + "@@rules_oci+//oci:extensions.bzl%oci": { "general": { - "bzlTransitiveDigest": "Vd/0cN91U7O//KVXEBJsnr64LUyHdC6DRiDTTnQTrUw=", - "usagesDigest": "t11Aj3mDwK2nwRAqSmqylVpvomzPYWiSVUL4xbuiMm4=", + "bzlTransitiveDigest": "XTbZNic3alDB0IiYqOdBi1njq1/HjCKwaI1nW+qtDUw=", + "usagesDigest": "vK9qUa3pyGJik2hb1NI2fKVJrjIszor+wN/44XMkKi0=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { - "uv_0_6_5_aarch64_apple_darwin": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "distroless_cc_debian12_linux_amd64": { + "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_pull", "attributes": { - "version": "0.6.5", - "platform": "aarch64-apple-darwin", - "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-aarch64-apple-darwin.tar.gz" - ], - "sha256": "4952c6275778fbc7e5cf6b5739ffd944c47063134d6ea43a7037b0a028ef771e" + "www_authenticate_challenges": {}, + "scheme": "https", + "registry": "gcr.io", + "repository": "distroless/cc-debian12", + "identifier": "sha256:eccec5274132c1be0ce5d2c8e6fe41033e64af5e987ccee9007826e4c012069d", + "platform": "linux/amd64", + "target_name": "distroless_cc_debian12_linux_amd64", + "bazel_tags": [] } }, - "uv_0_6_5_aarch64_unknown_linux_gnu": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "distroless_cc_debian12": { + "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_alias", "attributes": { - "version": "0.6.5", - "platform": "aarch64-unknown-linux-gnu", - "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-aarch64-unknown-linux-gnu.tar.gz" - ], - "sha256": "ebf0707ab949b948ad14250ed93f968fec954e20d12ce6bc9fce74b0ce7e4900" + "target_name": "distroless_cc_debian12", + "www_authenticate_challenges": {}, + "scheme": "https", + "registry": "gcr.io", + "repository": "distroless/cc-debian12", + "identifier": "sha256:eccec5274132c1be0ce5d2c8e6fe41033e64af5e987ccee9007826e4c012069d", + "platforms": { + "@@platforms//cpu:x86_64": "@distroless_cc_debian12_linux_amd64" + }, + "bzlmod_repository": "distroless_cc_debian12", + "reproducible": true } }, - "uv_0_6_5_powerpc64_unknown_linux_gnu": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "oci_crane_darwin_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", "attributes": { - "version": "0.6.5", - "platform": "powerpc64-unknown-linux-gnu", - "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-powerpc64-unknown-linux-gnu.tar.gz" - ], - "sha256": "1f8fd8dde91258380dbcd4b9fc65f15a7dc3af6cfffefb9a4f99d32c4c67436d" + "platform": "darwin_amd64", + "crane_version": "v0.18.0" } }, - "uv_0_6_5_powerpc64le_unknown_linux_gnu": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "oci_crane_darwin_arm64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", "attributes": { - "version": "0.6.5", - "platform": "powerpc64le-unknown-linux-gnu", - "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-powerpc64le-unknown-linux-gnu.tar.gz" - ], - "sha256": "d2b6e19ca7ecd8d7e0d1ddced965aa91a3c822d07a7f7657743945245c1f627a" + "platform": "darwin_arm64", + "crane_version": "v0.18.0" } }, - "uv_0_6_5_s390x_unknown_linux_gnu": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "oci_crane_linux_arm64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", "attributes": { - "version": "0.6.5", - "platform": "s390x-unknown-linux-gnu", - "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-s390x-unknown-linux-gnu.tar.gz" - ], - "sha256": "43a3e39f19fd5d098bb8c896b6c0f99a146253c271d78409cf31c2f55f2763d0" + "platform": "linux_arm64", + "crane_version": "v0.18.0" } }, - "uv_0_6_5_x86_64_apple_darwin": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "oci_crane_linux_armv6": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", "attributes": { - "version": "0.6.5", - "platform": "x86_64-apple-darwin", - "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-x86_64-apple-darwin.tar.gz" - ], - "sha256": "d336d862694e1c36afea55eb15f5df02aa77c04a168b781df515d641e23285f0" + "platform": "linux_armv6", + "crane_version": "v0.18.0" } }, - "uv_0_6_5_x86_64_pc_windows_msvc": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "oci_crane_linux_i386": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", "attributes": { - "version": "0.6.5", - "platform": "x86_64-pc-windows-msvc", - "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-x86_64-pc-windows-msvc.zip" - ], - "sha256": "a3e9f0913189c38b804bf6bb3d348af305a8ceabc690be92b6e111245a821640" + "platform": "linux_i386", + "crane_version": "v0.18.0" } }, - "uv_0_6_5_x86_64_unknown_linux_gnu": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "oci_crane_linux_s390x": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", "attributes": { - "version": "0.6.5", - "platform": "x86_64-unknown-linux-gnu", - "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-x86_64-unknown-linux-gnu.tar.gz" - ], - "sha256": "8fc9895719a1291ecd193cb86f9282ff3649cef797d29eacc74c4f573aab1e2f" + "platform": "linux_s390x", + "crane_version": "v0.18.0" } }, + "oci_crane_linux_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "linux_amd64", + "crane_version": "v0.18.0" + } + }, + "oci_crane_windows_armv6": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "windows_armv6", + "crane_version": "v0.18.0" + } + }, + "oci_crane_windows_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "windows_amd64", + "crane_version": "v0.18.0" + } + }, + "oci_crane_toolchains": { + "repoRuleId": "@@rules_oci+//oci/private:toolchains_repo.bzl%toolchains_repo", + "attributes": { + "toolchain_type": "@rules_oci//oci:crane_toolchain_type", + "toolchain": "@oci_crane_{platform}//:crane_toolchain" + } + }, + "oci_regctl_darwin_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "darwin_amd64" + } + }, + "oci_regctl_darwin_arm64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "darwin_arm64" + } + }, + "oci_regctl_linux_arm64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "linux_arm64" + } + }, + "oci_regctl_linux_s390x": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "linux_s390x" + } + }, + "oci_regctl_linux_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "linux_amd64" + } + }, + "oci_regctl_windows_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "windows_amd64" + } + }, + "oci_regctl_toolchains": { + "repoRuleId": "@@rules_oci+//oci/private:toolchains_repo.bzl%toolchains_repo", + "attributes": { + "toolchain_type": "@rules_oci//oci:regctl_toolchain_type", + "toolchain": "@oci_regctl_{platform}//:regctl_toolchain" + } + } + }, + "moduleExtensionMetadata": { + "explicitRootModuleDirectDeps": [ + "distroless_cc_debian12", + "distroless_cc_debian12_linux_amd64" + ], + "explicitRootModuleDirectDevDeps": [], + "useAllRepos": "NO", + "reproducible": false + }, + "recordedRepoMappingEntries": [ + [ + "aspect_bazel_lib+", + "bazel_tools", + "bazel_tools" + ], + [ + "bazel_features+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_oci+", + "aspect_bazel_lib", + "aspect_bazel_lib+" + ], + [ + "rules_oci+", + "bazel_features", + "bazel_features+" + ], + [ + "rules_oci+", + "bazel_skylib", + "bazel_skylib+" + ] + ] + } + }, + "@@rules_python+//python/uv:uv.bzl%uv": { + "general": { + "bzlTransitiveDigest": "Vd/0cN91U7O//KVXEBJsnr64LUyHdC6DRiDTTnQTrUw=", + "usagesDigest": "4FQg4uUSAYXV3gvUvHtsV28NjaoGjNF9sJNQ66s1Fl8=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { "uv": { "repoRuleId": "@@rules_python+//python/uv/private:uv_toolchains_repo.bzl%uv_toolchains_repo", "attributes": { "toolchain_type": "'@@rules_python+//python/uv:uv_toolchain_type'", "toolchain_names": [ - "0_6_5_aarch64_apple_darwin", - "0_6_5_aarch64_unknown_linux_gnu", - "0_6_5_powerpc64_unknown_linux_gnu", - "0_6_5_powerpc64le_unknown_linux_gnu", - "0_6_5_s390x_unknown_linux_gnu", - "0_6_5_x86_64_apple_darwin", - "0_6_5_x86_64_pc_windows_msvc", - "0_6_5_x86_64_unknown_linux_gnu" + "none" ], "toolchain_implementations": { - "0_6_5_aarch64_apple_darwin": "@uv_0_6_5_aarch64_apple_darwin//:uv_toolchain", - "0_6_5_aarch64_unknown_linux_gnu": "@uv_0_6_5_aarch64_unknown_linux_gnu//:uv_toolchain", - "0_6_5_powerpc64_unknown_linux_gnu": "@uv_0_6_5_powerpc64_unknown_linux_gnu//:uv_toolchain", - "0_6_5_powerpc64le_unknown_linux_gnu": "@uv_0_6_5_powerpc64le_unknown_linux_gnu//:uv_toolchain", - "0_6_5_s390x_unknown_linux_gnu": "@uv_0_6_5_s390x_unknown_linux_gnu//:uv_toolchain", - "0_6_5_x86_64_apple_darwin": "@uv_0_6_5_x86_64_apple_darwin//:uv_toolchain", - "0_6_5_x86_64_pc_windows_msvc": "@uv_0_6_5_x86_64_pc_windows_msvc//:uv_toolchain", - "0_6_5_x86_64_unknown_linux_gnu": "@uv_0_6_5_x86_64_unknown_linux_gnu//:uv_toolchain" + "none": "'@@rules_python+//python:none'" }, "toolchain_compatible_with": { - "0_6_5_aarch64_apple_darwin": [ - "'@@platforms//os:macos'", - "'@@platforms//cpu:aarch64'" - ], - "0_6_5_aarch64_unknown_linux_gnu": [ - "'@@platforms//os:linux'", - "'@@platforms//cpu:aarch64'" - ], - "0_6_5_powerpc64_unknown_linux_gnu": [ - "'@@platforms//os:linux'", - "'@@platforms//cpu:ppc'" - ], - "0_6_5_powerpc64le_unknown_linux_gnu": [ - "'@@platforms//os:linux'", - "'@@platforms//cpu:ppc64le'" - ], - "0_6_5_s390x_unknown_linux_gnu": [ - "'@@platforms//os:linux'", - "'@@platforms//cpu:s390x'" - ], - "0_6_5_x86_64_apple_darwin": [ - "'@@platforms//os:macos'", - "'@@platforms//cpu:x86_64'" - ], - "0_6_5_x86_64_pc_windows_msvc": [ - "'@@platforms//os:windows'", - "'@@platforms//cpu:x86_64'" - ], - "0_6_5_x86_64_unknown_linux_gnu": [ - "'@@platforms//os:linux'", - "'@@platforms//cpu:x86_64'" + "none": [ + "@platforms//:incompatible" ] }, "toolchain_target_settings": {} @@ -935,11 +1214,11 @@ }, "@@rules_rust+//crate_universe:extensions.bzl%crate": { "general": { - "bzlTransitiveDigest": "HHPRUh/mbCXdC11w2axMRLWeweoYK1bLuS1e2NpvcoM=", + "bzlTransitiveDigest": "QaxJz0tRTK5Cb0Lu/nLbkmNxQTb6xlpvX7v9XZdv02k=", "usagesDigest": "mup6mgSFatk34Q0znMZPpIEOFZyk9v3xf6jRIXEFTxc=", "recordedFileInputs": { - "@@//zml/tokenizer/hftokenizers/Cargo.lock": "d78ffab37ef69bf8ff45a4d686b25175f30e8fed4d8375612e84dcf6db0cfc9c", - "@@//zml/tokenizer/hftokenizers/Cargo.toml": "eeccbf5ff71c5f26b94cb289b3fc64c5fc6eb8708b815f39ce5f16bf225b6774" + "@@//zml/tokenizer/hftokenizers/Cargo.lock": "93198160b29c4607994b405ffa0ad1c29b08df1ad22ec9a711874914bbaadbcb", + "@@//zml/tokenizer/hftokenizers/Cargo.toml": "eaf4574383be4d13a653a9c91c052979b05630f1c251b87bc8386263165dc204" }, "recordedDirentsInputs": {}, "envVariables": { @@ -956,9 +1235,9 @@ "repoRuleId": "@@rules_rust+//crate_universe:extensions.bzl%_generate_repo", "attributes": { "contents": { - "BUILD.bazel": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files(\n [\n \"cargo-bazel.json\",\n \"crates.bzl\",\n \"defs.bzl\",\n ] + glob(\n allow_empty = True,\n include = [\"*.bazel\"],\n ),\n)\n\nfilegroup(\n name = \"srcs\",\n srcs = glob(\n allow_empty = True,\n include = [\n \"*.bazel\",\n \"*.bzl\",\n ],\n ),\n)\n\n# Workspace Member Dependencies\nalias(\n name = \"tokenizers-0.21.4\",\n actual = \"@crates__tokenizers-0.21.4//:tokenizers\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"tokenizers\",\n actual = \"@crates__tokenizers-0.21.4//:tokenizers\",\n tags = [\"manual\"],\n)\n", + "BUILD.bazel": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files(\n [\n \"cargo-bazel.json\",\n \"crates.bzl\",\n \"defs.bzl\",\n ] + glob(\n allow_empty = True,\n include = [\"*.bazel\"],\n ),\n)\n\nfilegroup(\n name = \"srcs\",\n srcs = glob(\n allow_empty = True,\n include = [\n \"*.bazel\",\n \"*.bzl\",\n ],\n ),\n)\n\n# Workspace Member Dependencies\nalias(\n name = \"tokenizers-0.21.2\",\n actual = \"@crates__tokenizers-0.21.2//:tokenizers\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"tokenizers\",\n actual = \"@crates__tokenizers-0.21.2//:tokenizers\",\n tags = [\"manual\"],\n)\n", "alias_rules.bzl": "\"\"\"Alias that transitions its target to `compilation_mode=opt`. Use `transition_alias=\"opt\"` to enable.\"\"\"\n\nload(\"@rules_cc//cc:defs.bzl\", \"CcInfo\")\nload(\"@rules_rust//rust:rust_common.bzl\", \"COMMON_PROVIDERS\")\n\ndef _transition_alias_impl(ctx):\n # `ctx.attr.actual` is a list of 1 item due to the transition\n providers = [ctx.attr.actual[0][provider] for provider in COMMON_PROVIDERS]\n if CcInfo in ctx.attr.actual[0]:\n providers.append(ctx.attr.actual[0][CcInfo])\n return providers\n\ndef _change_compilation_mode(compilation_mode):\n def _change_compilation_mode_impl(_settings, _attr):\n return {\n \"//command_line_option:compilation_mode\": compilation_mode,\n }\n\n return transition(\n implementation = _change_compilation_mode_impl,\n inputs = [],\n outputs = [\n \"//command_line_option:compilation_mode\",\n ],\n )\n\ndef _transition_alias_rule(compilation_mode):\n return rule(\n implementation = _transition_alias_impl,\n provides = COMMON_PROVIDERS,\n attrs = {\n \"actual\": attr.label(\n mandatory = True,\n doc = \"`rust_library()` target to transition to `compilation_mode=opt`.\",\n providers = COMMON_PROVIDERS,\n cfg = _change_compilation_mode(compilation_mode),\n ),\n \"_allowlist_function_transition\": attr.label(\n default = \"@bazel_tools//tools/allowlists/function_transition_allowlist\",\n ),\n },\n doc = \"Transitions a Rust library crate to the `compilation_mode=opt`.\",\n )\n\ntransition_alias_dbg = _transition_alias_rule(\"dbg\")\ntransition_alias_fastbuild = _transition_alias_rule(\"fastbuild\")\ntransition_alias_opt = _transition_alias_rule(\"opt\")\n", - "defs.bzl": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\"\"\"\n# `crates_repository` API\n\n- [aliases](#aliases)\n- [crate_deps](#crate_deps)\n- [all_crate_deps](#all_crate_deps)\n- [crate_repositories](#crate_repositories)\n\n\"\"\"\n\nload(\"@bazel_tools//tools/build_defs/repo:git.bzl\", \"new_git_repository\")\nload(\"@bazel_tools//tools/build_defs/repo:http.bzl\", \"http_archive\")\nload(\"@bazel_tools//tools/build_defs/repo:utils.bzl\", \"maybe\")\nload(\"@bazel_skylib//lib:selects.bzl\", \"selects\")\nload(\"@rules_rust//crate_universe/private:local_crate_mirror.bzl\", \"local_crate_mirror\")\n\n###############################################################################\n# MACROS API\n###############################################################################\n\n# An identifier that represent common dependencies (unconditional).\n_COMMON_CONDITION = \"\"\n\ndef _flatten_dependency_maps(all_dependency_maps):\n \"\"\"Flatten a list of dependency maps into one dictionary.\n\n Dependency maps have the following structure:\n\n ```python\n DEPENDENCIES_MAP = {\n # The first key in the map is a Bazel package\n # name of the workspace this file is defined in.\n \"workspace_member_package\": {\n\n # Not all dependencies are supported for all platforms.\n # the condition key is the condition required to be true\n # on the host platform.\n \"condition\": {\n\n # An alias to a crate target. # The label of the crate target the\n # Aliases are only crate names. # package name refers to.\n \"package_name\": \"@full//:label\",\n }\n }\n }\n ```\n\n Args:\n all_dependency_maps (list): A list of dicts as described above\n\n Returns:\n dict: A dictionary as described above\n \"\"\"\n dependencies = {}\n\n for workspace_deps_map in all_dependency_maps:\n for pkg_name, conditional_deps_map in workspace_deps_map.items():\n if pkg_name not in dependencies:\n non_frozen_map = dict()\n for key, values in conditional_deps_map.items():\n non_frozen_map.update({key: dict(values.items())})\n dependencies.setdefault(pkg_name, non_frozen_map)\n continue\n\n for condition, deps_map in conditional_deps_map.items():\n # If the condition has not been recorded, do so and continue\n if condition not in dependencies[pkg_name]:\n dependencies[pkg_name].setdefault(condition, dict(deps_map.items()))\n continue\n\n # Alert on any miss-matched dependencies\n inconsistent_entries = []\n for crate_name, crate_label in deps_map.items():\n existing = dependencies[pkg_name][condition].get(crate_name)\n if existing and existing != crate_label:\n inconsistent_entries.append((crate_name, existing, crate_label))\n dependencies[pkg_name][condition].update({crate_name: crate_label})\n\n return dependencies\n\ndef crate_deps(deps, package_name = None):\n \"\"\"Finds the fully qualified label of the requested crates for the package where this macro is called.\n\n Args:\n deps (list): The desired list of crate targets.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()`.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if not deps:\n return []\n\n if package_name == None:\n package_name = native.package_name()\n\n # Join both sets of dependencies\n dependencies = _flatten_dependency_maps([\n _NORMAL_DEPENDENCIES,\n _NORMAL_DEV_DEPENDENCIES,\n _PROC_MACRO_DEPENDENCIES,\n _PROC_MACRO_DEV_DEPENDENCIES,\n _BUILD_DEPENDENCIES,\n _BUILD_PROC_MACRO_DEPENDENCIES,\n ]).pop(package_name, {})\n\n # Combine all conditional packages so we can easily index over a flat list\n # TODO: Perhaps this should actually return select statements and maintain\n # the conditionals of the dependencies\n flat_deps = {}\n for deps_set in dependencies.values():\n for crate_name, crate_label in deps_set.items():\n flat_deps.update({crate_name: crate_label})\n\n missing_crates = []\n crate_targets = []\n for crate_target in deps:\n if crate_target not in flat_deps:\n missing_crates.append(crate_target)\n else:\n crate_targets.append(flat_deps[crate_target])\n\n if missing_crates:\n fail(\"Could not find crates `{}` among dependencies of `{}`. Available dependencies were `{}`\".format(\n missing_crates,\n package_name,\n dependencies,\n ))\n\n return crate_targets\n\ndef all_crate_deps(\n normal = False, \n normal_dev = False, \n proc_macro = False, \n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Finds the fully qualified label of all requested direct crate dependencies \\\n for the package where this macro is called.\n\n If no parameters are set, all normal dependencies are returned. Setting any one flag will\n otherwise impact the contents of the returned list.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_dependency_maps = []\n if normal:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n if normal_dev:\n all_dependency_maps.append(_NORMAL_DEV_DEPENDENCIES)\n if proc_macro:\n all_dependency_maps.append(_PROC_MACRO_DEPENDENCIES)\n if proc_macro_dev:\n all_dependency_maps.append(_PROC_MACRO_DEV_DEPENDENCIES)\n if build:\n all_dependency_maps.append(_BUILD_DEPENDENCIES)\n if build_proc_macro:\n all_dependency_maps.append(_BUILD_PROC_MACRO_DEPENDENCIES)\n\n # Default to always using normal dependencies\n if not all_dependency_maps:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n\n dependencies = _flatten_dependency_maps(all_dependency_maps).pop(package_name, None)\n\n if not dependencies:\n if dependencies == None:\n fail(\"Tried to get all_crate_deps for package \" + package_name + \" but that package had no Cargo.toml file\")\n else:\n return []\n\n crate_deps = list(dependencies.pop(_COMMON_CONDITION, {}).values())\n for condition, deps in dependencies.items():\n crate_deps += selects.with_or({\n tuple(_CONDITIONS[condition]): deps.values(),\n \"//conditions:default\": [],\n })\n\n return crate_deps\n\ndef aliases(\n normal = False,\n normal_dev = False,\n proc_macro = False,\n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Produces a map of Crate alias names to their original label\n\n If no dependency kinds are specified, `normal` and `proc_macro` are used by default.\n Setting any one flag will otherwise determine the contents of the returned dict.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n dict: The aliases of all associated packages\n \"\"\"\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_aliases_maps = []\n if normal:\n all_aliases_maps.append(_NORMAL_ALIASES)\n if normal_dev:\n all_aliases_maps.append(_NORMAL_DEV_ALIASES)\n if proc_macro:\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n if proc_macro_dev:\n all_aliases_maps.append(_PROC_MACRO_DEV_ALIASES)\n if build:\n all_aliases_maps.append(_BUILD_ALIASES)\n if build_proc_macro:\n all_aliases_maps.append(_BUILD_PROC_MACRO_ALIASES)\n\n # Default to always using normal aliases\n if not all_aliases_maps:\n all_aliases_maps.append(_NORMAL_ALIASES)\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n\n aliases = _flatten_dependency_maps(all_aliases_maps).pop(package_name, None)\n\n if not aliases:\n return dict()\n\n common_items = aliases.pop(_COMMON_CONDITION, {}).items()\n\n # If there are only common items in the dictionary, immediately return them\n if not len(aliases.keys()) == 1:\n return dict(common_items)\n\n # Build a single select statement where each conditional has accounted for the\n # common set of aliases.\n crate_aliases = {\"//conditions:default\": dict(common_items)}\n for condition, deps in aliases.items():\n condition_triples = _CONDITIONS[condition]\n for triple in condition_triples:\n if triple in crate_aliases:\n crate_aliases[triple].update(deps)\n else:\n crate_aliases.update({triple: dict(deps.items() + common_items)})\n\n return select(crate_aliases)\n\n###############################################################################\n# WORKSPACE MEMBER DEPS AND ALIASES\n###############################################################################\n\n_NORMAL_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n _COMMON_CONDITION: {\n \"tokenizers\": Label(\"@crates//:tokenizers-0.21.4\"),\n },\n },\n}\n\n\n_NORMAL_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n _COMMON_CONDITION: {\n },\n },\n}\n\n\n_NORMAL_DEV_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_NORMAL_DEV_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEV_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEV_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_CONDITIONS = {\n \"aarch64-apple-darwin\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\"],\n \"aarch64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\"],\n \"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\",\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\": [],\n \"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\": [],\n \"cfg(any())\": [],\n \"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\": [],\n \"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\": [],\n \"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\": [],\n \"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\",\"@rules_rust//rust/platform:x86_64-apple-darwin\"],\n \"cfg(not(all(target_arch = \\\"arm\\\", target_os = \\\"none\\\")))\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\",\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\",\"@rules_rust//rust/platform:x86_64-apple-darwin\",\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"cfg(target_os = \\\"netbsd\\\")\": [],\n \"cfg(target_os = \\\"solaris\\\")\": [],\n \"cfg(target_os = \\\"vxworks\\\")\": [],\n \"cfg(windows)\": [],\n \"x86_64-apple-darwin\": [\"@rules_rust//rust/platform:x86_64-apple-darwin\"],\n \"x86_64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n}\n\n###############################################################################\n\ndef crate_repositories():\n \"\"\"A macro for defining repositories for all generated crates.\n\n Returns:\n A list of repos visible to the module through the module extension.\n \"\"\"\n maybe(\n http_archive,\n name = \"crates__ahash-0.8.12\",\n sha256 = \"5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ahash/0.8.12/download\"],\n strip_prefix = \"ahash-0.8.12\",\n build_file = Label(\"@crates//crates:BUILD.ahash-0.8.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__aho-corasick-1.1.3\",\n sha256 = \"8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/aho-corasick/1.1.3/download\"],\n strip_prefix = \"aho-corasick-1.1.3\",\n build_file = Label(\"@crates//crates:BUILD.aho-corasick-1.1.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__base64-0.13.1\",\n sha256 = \"9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/base64/0.13.1/download\"],\n strip_prefix = \"base64-0.13.1\",\n build_file = Label(\"@crates//crates:BUILD.base64-0.13.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__bitflags-2.9.1\",\n sha256 = \"1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/bitflags/2.9.1/download\"],\n strip_prefix = \"bitflags-2.9.1\",\n build_file = Label(\"@crates//crates:BUILD.bitflags-2.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__byteorder-1.5.0\",\n sha256 = \"1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/byteorder/1.5.0/download\"],\n strip_prefix = \"byteorder-1.5.0\",\n build_file = Label(\"@crates//crates:BUILD.byteorder-1.5.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__castaway-0.2.4\",\n sha256 = \"dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/castaway/0.2.4/download\"],\n strip_prefix = \"castaway-0.2.4\",\n build_file = Label(\"@crates//crates:BUILD.castaway-0.2.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__cc-1.2.33\",\n sha256 = \"3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/cc/1.2.33/download\"],\n strip_prefix = \"cc-1.2.33\",\n build_file = Label(\"@crates//crates:BUILD.cc-1.2.33.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__cfg-if-1.0.0\",\n sha256 = \"baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/cfg-if/1.0.0/download\"],\n strip_prefix = \"cfg-if-1.0.0\",\n build_file = Label(\"@crates//crates:BUILD.cfg-if-1.0.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__compact_str-0.9.0\",\n sha256 = \"3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/compact_str/0.9.0/download\"],\n strip_prefix = \"compact_str-0.9.0\",\n build_file = Label(\"@crates//crates:BUILD.compact_str-0.9.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-deque-0.8.6\",\n sha256 = \"9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-deque/0.8.6/download\"],\n strip_prefix = \"crossbeam-deque-0.8.6\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-deque-0.8.6.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-epoch-0.9.18\",\n sha256 = \"5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-epoch/0.9.18/download\"],\n strip_prefix = \"crossbeam-epoch-0.9.18\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-epoch-0.9.18.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-utils-0.8.21\",\n sha256 = \"d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-utils/0.8.21/download\"],\n strip_prefix = \"crossbeam-utils-0.8.21\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-utils-0.8.21.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling-0.20.10\",\n sha256 = \"6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling/0.20.10/download\"],\n strip_prefix = \"darling-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling_core-0.20.10\",\n sha256 = \"95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling_core/0.20.10/download\"],\n strip_prefix = \"darling_core-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling_core-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling_macro-0.20.10\",\n sha256 = \"d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling_macro/0.20.10/download\"],\n strip_prefix = \"darling_macro-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling_macro-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__dary_heap-0.3.7\",\n sha256 = \"04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/dary_heap/0.3.7/download\"],\n strip_prefix = \"dary_heap-0.3.7\",\n build_file = Label(\"@crates//crates:BUILD.dary_heap-0.3.7.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder-0.20.2\",\n sha256 = \"507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder/0.20.2/download\"],\n strip_prefix = \"derive_builder-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder_core-0.20.2\",\n sha256 = \"2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder_core/0.20.2/download\"],\n strip_prefix = \"derive_builder_core-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder_core-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder_macro-0.20.2\",\n sha256 = \"ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder_macro/0.20.2/download\"],\n strip_prefix = \"derive_builder_macro-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder_macro-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__either-1.13.0\",\n sha256 = \"60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/either/1.13.0/download\"],\n strip_prefix = \"either-1.13.0\",\n build_file = Label(\"@crates//crates:BUILD.either-1.13.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__esaxx-rs-0.1.10\",\n sha256 = \"d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/esaxx-rs/0.1.10/download\"],\n strip_prefix = \"esaxx-rs-0.1.10\",\n build_file = Label(\"@crates//crates:BUILD.esaxx-rs-0.1.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__fnv-1.0.7\",\n sha256 = \"3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/fnv/1.0.7/download\"],\n strip_prefix = \"fnv-1.0.7\",\n build_file = Label(\"@crates//crates:BUILD.fnv-1.0.7.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__getrandom-0.3.3\",\n sha256 = \"26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/getrandom/0.3.3/download\"],\n strip_prefix = \"getrandom-0.3.3\",\n build_file = Label(\"@crates//crates:BUILD.getrandom-0.3.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ident_case-1.0.1\",\n sha256 = \"b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ident_case/1.0.1/download\"],\n strip_prefix = \"ident_case-1.0.1\",\n build_file = Label(\"@crates//crates:BUILD.ident_case-1.0.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__itertools-0.14.0\",\n sha256 = \"2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/itertools/0.14.0/download\"],\n strip_prefix = \"itertools-0.14.0\",\n build_file = Label(\"@crates//crates:BUILD.itertools-0.14.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__itoa-1.0.14\",\n sha256 = \"d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/itoa/1.0.14/download\"],\n strip_prefix = \"itoa-1.0.14\",\n build_file = Label(\"@crates//crates:BUILD.itoa-1.0.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__libc-0.2.169\",\n sha256 = \"b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/libc/0.2.169/download\"],\n strip_prefix = \"libc-0.2.169\",\n build_file = Label(\"@crates//crates:BUILD.libc-0.2.169.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__log-0.4.22\",\n sha256 = \"a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/log/0.4.22/download\"],\n strip_prefix = \"log-0.4.22\",\n build_file = Label(\"@crates//crates:BUILD.log-0.4.22.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__macro_rules_attribute-0.2.0\",\n sha256 = \"8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/macro_rules_attribute/0.2.0/download\"],\n strip_prefix = \"macro_rules_attribute-0.2.0\",\n build_file = Label(\"@crates//crates:BUILD.macro_rules_attribute-0.2.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__macro_rules_attribute-proc_macro-0.2.0\",\n sha256 = \"b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/macro_rules_attribute-proc_macro/0.2.0/download\"],\n strip_prefix = \"macro_rules_attribute-proc_macro-0.2.0\",\n build_file = Label(\"@crates//crates:BUILD.macro_rules_attribute-proc_macro-0.2.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__memchr-2.7.4\",\n sha256 = \"78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/memchr/2.7.4/download\"],\n strip_prefix = \"memchr-2.7.4\",\n build_file = Label(\"@crates//crates:BUILD.memchr-2.7.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__minimal-lexical-0.2.1\",\n sha256 = \"68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/minimal-lexical/0.2.1/download\"],\n strip_prefix = \"minimal-lexical-0.2.1\",\n build_file = Label(\"@crates//crates:BUILD.minimal-lexical-0.2.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__monostate-0.1.13\",\n sha256 = \"0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/monostate/0.1.13/download\"],\n strip_prefix = \"monostate-0.1.13\",\n build_file = Label(\"@crates//crates:BUILD.monostate-0.1.13.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__monostate-impl-0.1.13\",\n sha256 = \"a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/monostate-impl/0.1.13/download\"],\n strip_prefix = \"monostate-impl-0.1.13\",\n build_file = Label(\"@crates//crates:BUILD.monostate-impl-0.1.13.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__nom-7.1.3\",\n sha256 = \"d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/nom/7.1.3/download\"],\n strip_prefix = \"nom-7.1.3\",\n build_file = Label(\"@crates//crates:BUILD.nom-7.1.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__once_cell-1.20.2\",\n sha256 = \"1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/once_cell/1.20.2/download\"],\n strip_prefix = \"once_cell-1.20.2\",\n build_file = Label(\"@crates//crates:BUILD.once_cell-1.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__onig-6.5.1\",\n sha256 = \"336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/onig/6.5.1/download\"],\n strip_prefix = \"onig-6.5.1\",\n build_file = Label(\"@crates//crates:BUILD.onig-6.5.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__onig_sys-69.9.1\",\n sha256 = \"c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/onig_sys/69.9.1/download\"],\n strip_prefix = \"onig_sys-69.9.1\",\n build_file = Label(\"@crates//crates:BUILD.onig_sys-69.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__paste-1.0.15\",\n sha256 = \"57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/paste/1.0.15/download\"],\n strip_prefix = \"paste-1.0.15\",\n build_file = Label(\"@crates//crates:BUILD.paste-1.0.15.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__pkg-config-0.3.32\",\n sha256 = \"7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/pkg-config/0.3.32/download\"],\n strip_prefix = \"pkg-config-0.3.32\",\n build_file = Label(\"@crates//crates:BUILD.pkg-config-0.3.32.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ppv-lite86-0.2.20\",\n sha256 = \"77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ppv-lite86/0.2.20/download\"],\n strip_prefix = \"ppv-lite86-0.2.20\",\n build_file = Label(\"@crates//crates:BUILD.ppv-lite86-0.2.20.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__proc-macro2-1.0.92\",\n sha256 = \"37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/proc-macro2/1.0.92/download\"],\n strip_prefix = \"proc-macro2-1.0.92\",\n build_file = Label(\"@crates//crates:BUILD.proc-macro2-1.0.92.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__quote-1.0.37\",\n sha256 = \"b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/quote/1.0.37/download\"],\n strip_prefix = \"quote-1.0.37\",\n build_file = Label(\"@crates//crates:BUILD.quote-1.0.37.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__r-efi-5.3.0\",\n sha256 = \"69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/r-efi/5.3.0/download\"],\n strip_prefix = \"r-efi-5.3.0\",\n build_file = Label(\"@crates//crates:BUILD.r-efi-5.3.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand-0.9.1\",\n sha256 = \"9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand/0.9.1/download\"],\n strip_prefix = \"rand-0.9.1\",\n build_file = Label(\"@crates//crates:BUILD.rand-0.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand_chacha-0.9.0\",\n sha256 = \"d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand_chacha/0.9.0/download\"],\n strip_prefix = \"rand_chacha-0.9.0\",\n build_file = Label(\"@crates//crates:BUILD.rand_chacha-0.9.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand_core-0.9.3\",\n sha256 = \"99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand_core/0.9.3/download\"],\n strip_prefix = \"rand_core-0.9.3\",\n build_file = Label(\"@crates//crates:BUILD.rand_core-0.9.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-1.10.0\",\n sha256 = \"b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon/1.10.0/download\"],\n strip_prefix = \"rayon-1.10.0\",\n build_file = Label(\"@crates//crates:BUILD.rayon-1.10.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-cond-0.4.0\",\n sha256 = \"2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon-cond/0.4.0/download\"],\n strip_prefix = \"rayon-cond-0.4.0\",\n build_file = Label(\"@crates//crates:BUILD.rayon-cond-0.4.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-core-1.12.1\",\n sha256 = \"1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon-core/1.12.1/download\"],\n strip_prefix = \"rayon-core-1.12.1\",\n build_file = Label(\"@crates//crates:BUILD.rayon-core-1.12.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-1.11.1\",\n sha256 = \"b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex/1.11.1/download\"],\n strip_prefix = \"regex-1.11.1\",\n build_file = Label(\"@crates//crates:BUILD.regex-1.11.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-automata-0.4.9\",\n sha256 = \"809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-automata/0.4.9/download\"],\n strip_prefix = \"regex-automata-0.4.9\",\n build_file = Label(\"@crates//crates:BUILD.regex-automata-0.4.9.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-syntax-0.8.5\",\n sha256 = \"2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-syntax/0.8.5/download\"],\n strip_prefix = \"regex-syntax-0.8.5\",\n build_file = Label(\"@crates//crates:BUILD.regex-syntax-0.8.5.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rustversion-1.0.21\",\n sha256 = \"8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rustversion/1.0.21/download\"],\n strip_prefix = \"rustversion-1.0.21\",\n build_file = Label(\"@crates//crates:BUILD.rustversion-1.0.21.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ryu-1.0.18\",\n sha256 = \"f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ryu/1.0.18/download\"],\n strip_prefix = \"ryu-1.0.18\",\n build_file = Label(\"@crates//crates:BUILD.ryu-1.0.18.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde-1.0.216\",\n sha256 = \"0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde/1.0.216/download\"],\n strip_prefix = \"serde-1.0.216\",\n build_file = Label(\"@crates//crates:BUILD.serde-1.0.216.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde_derive-1.0.216\",\n sha256 = \"46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde_derive/1.0.216/download\"],\n strip_prefix = \"serde_derive-1.0.216\",\n build_file = Label(\"@crates//crates:BUILD.serde_derive-1.0.216.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde_json-1.0.134\",\n sha256 = \"d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde_json/1.0.134/download\"],\n strip_prefix = \"serde_json-1.0.134\",\n build_file = Label(\"@crates//crates:BUILD.serde_json-1.0.134.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__shlex-1.3.0\",\n sha256 = \"0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/shlex/1.3.0/download\"],\n strip_prefix = \"shlex-1.3.0\",\n build_file = Label(\"@crates//crates:BUILD.shlex-1.3.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__smallvec-1.13.2\",\n sha256 = \"3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/smallvec/1.13.2/download\"],\n strip_prefix = \"smallvec-1.13.2\",\n build_file = Label(\"@crates//crates:BUILD.smallvec-1.13.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__spm_precompiled-0.1.4\",\n sha256 = \"5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/spm_precompiled/0.1.4/download\"],\n strip_prefix = \"spm_precompiled-0.1.4\",\n build_file = Label(\"@crates//crates:BUILD.spm_precompiled-0.1.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__static_assertions-1.1.0\",\n sha256 = \"a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/static_assertions/1.1.0/download\"],\n strip_prefix = \"static_assertions-1.1.0\",\n build_file = Label(\"@crates//crates:BUILD.static_assertions-1.1.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__strsim-0.11.1\",\n sha256 = \"7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/strsim/0.11.1/download\"],\n strip_prefix = \"strsim-0.11.1\",\n build_file = Label(\"@crates//crates:BUILD.strsim-0.11.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__syn-2.0.91\",\n sha256 = \"d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/syn/2.0.91/download\"],\n strip_prefix = \"syn-2.0.91\",\n build_file = Label(\"@crates//crates:BUILD.syn-2.0.91.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__thiserror-2.0.12\",\n sha256 = \"567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/thiserror/2.0.12/download\"],\n strip_prefix = \"thiserror-2.0.12\",\n build_file = Label(\"@crates//crates:BUILD.thiserror-2.0.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__thiserror-impl-2.0.12\",\n sha256 = \"7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/thiserror-impl/2.0.12/download\"],\n strip_prefix = \"thiserror-impl-2.0.12\",\n build_file = Label(\"@crates//crates:BUILD.thiserror-impl-2.0.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__tokenizers-0.21.4\",\n sha256 = \"a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/tokenizers/0.21.4/download\"],\n strip_prefix = \"tokenizers-0.21.4\",\n build_file = Label(\"@crates//crates:BUILD.tokenizers-0.21.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-ident-1.0.14\",\n sha256 = \"adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-ident/1.0.14/download\"],\n strip_prefix = \"unicode-ident-1.0.14\",\n build_file = Label(\"@crates//crates:BUILD.unicode-ident-1.0.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-normalization-alignments-0.1.12\",\n sha256 = \"43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-normalization-alignments/0.1.12/download\"],\n strip_prefix = \"unicode-normalization-alignments-0.1.12\",\n build_file = Label(\"@crates//crates:BUILD.unicode-normalization-alignments-0.1.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-segmentation-1.12.0\",\n sha256 = \"f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-segmentation/1.12.0/download\"],\n strip_prefix = \"unicode-segmentation-1.12.0\",\n build_file = Label(\"@crates//crates:BUILD.unicode-segmentation-1.12.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode_categories-0.1.1\",\n sha256 = \"39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode_categories/0.1.1/download\"],\n strip_prefix = \"unicode_categories-0.1.1\",\n build_file = Label(\"@crates//crates:BUILD.unicode_categories-0.1.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__version_check-0.9.5\",\n sha256 = \"0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/version_check/0.9.5/download\"],\n strip_prefix = \"version_check-0.9.5\",\n build_file = Label(\"@crates//crates:BUILD.version_check-0.9.5.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasi-0.14.2-wasi-0.2.4\",\n sha256 = \"9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasi/0.14.2+wasi-0.2.4/download\"],\n strip_prefix = \"wasi-0.14.2+wasi-0.2.4\",\n build_file = Label(\"@crates//crates:BUILD.wasi-0.14.2+wasi-0.2.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wit-bindgen-rt-0.39.0\",\n sha256 = \"6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wit-bindgen-rt/0.39.0/download\"],\n strip_prefix = \"wit-bindgen-rt-0.39.0\",\n build_file = Label(\"@crates//crates:BUILD.wit-bindgen-rt-0.39.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-0.7.35\",\n sha256 = \"1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy/0.7.35/download\"],\n strip_prefix = \"zerocopy-0.7.35\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-0.7.35.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-0.8.26\",\n sha256 = \"1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy/0.8.26/download\"],\n strip_prefix = \"zerocopy-0.8.26\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-0.8.26.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-derive-0.7.35\",\n sha256 = \"fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy-derive/0.7.35/download\"],\n strip_prefix = \"zerocopy-derive-0.7.35\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-derive-0.7.35.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-derive-0.8.26\",\n sha256 = \"9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy-derive/0.8.26/download\"],\n strip_prefix = \"zerocopy-derive-0.8.26\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-derive-0.8.26.bazel\"),\n )\n\n return [\n struct(repo=\"crates__tokenizers-0.21.4\", is_dev_dep = False),\n ]\n" + "defs.bzl": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\"\"\"\n# `crates_repository` API\n\n- [aliases](#aliases)\n- [crate_deps](#crate_deps)\n- [all_crate_deps](#all_crate_deps)\n- [crate_repositories](#crate_repositories)\n\n\"\"\"\n\nload(\"@bazel_tools//tools/build_defs/repo:git.bzl\", \"new_git_repository\")\nload(\"@bazel_tools//tools/build_defs/repo:http.bzl\", \"http_archive\")\nload(\"@bazel_tools//tools/build_defs/repo:utils.bzl\", \"maybe\")\nload(\"@bazel_skylib//lib:selects.bzl\", \"selects\")\nload(\"@rules_rust//crate_universe/private:local_crate_mirror.bzl\", \"local_crate_mirror\")\n\n###############################################################################\n# MACROS API\n###############################################################################\n\n# An identifier that represent common dependencies (unconditional).\n_COMMON_CONDITION = \"\"\n\ndef _flatten_dependency_maps(all_dependency_maps):\n \"\"\"Flatten a list of dependency maps into one dictionary.\n\n Dependency maps have the following structure:\n\n ```python\n DEPENDENCIES_MAP = {\n # The first key in the map is a Bazel package\n # name of the workspace this file is defined in.\n \"workspace_member_package\": {\n\n # Not all dependencies are supported for all platforms.\n # the condition key is the condition required to be true\n # on the host platform.\n \"condition\": {\n\n # An alias to a crate target. # The label of the crate target the\n # Aliases are only crate names. # package name refers to.\n \"package_name\": \"@full//:label\",\n }\n }\n }\n ```\n\n Args:\n all_dependency_maps (list): A list of dicts as described above\n\n Returns:\n dict: A dictionary as described above\n \"\"\"\n dependencies = {}\n\n for workspace_deps_map in all_dependency_maps:\n for pkg_name, conditional_deps_map in workspace_deps_map.items():\n if pkg_name not in dependencies:\n non_frozen_map = dict()\n for key, values in conditional_deps_map.items():\n non_frozen_map.update({key: dict(values.items())})\n dependencies.setdefault(pkg_name, non_frozen_map)\n continue\n\n for condition, deps_map in conditional_deps_map.items():\n # If the condition has not been recorded, do so and continue\n if condition not in dependencies[pkg_name]:\n dependencies[pkg_name].setdefault(condition, dict(deps_map.items()))\n continue\n\n # Alert on any miss-matched dependencies\n inconsistent_entries = []\n for crate_name, crate_label in deps_map.items():\n existing = dependencies[pkg_name][condition].get(crate_name)\n if existing and existing != crate_label:\n inconsistent_entries.append((crate_name, existing, crate_label))\n dependencies[pkg_name][condition].update({crate_name: crate_label})\n\n return dependencies\n\ndef crate_deps(deps, package_name = None):\n \"\"\"Finds the fully qualified label of the requested crates for the package where this macro is called.\n\n Args:\n deps (list): The desired list of crate targets.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()`.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if not deps:\n return []\n\n if package_name == None:\n package_name = native.package_name()\n\n # Join both sets of dependencies\n dependencies = _flatten_dependency_maps([\n _NORMAL_DEPENDENCIES,\n _NORMAL_DEV_DEPENDENCIES,\n _PROC_MACRO_DEPENDENCIES,\n _PROC_MACRO_DEV_DEPENDENCIES,\n _BUILD_DEPENDENCIES,\n _BUILD_PROC_MACRO_DEPENDENCIES,\n ]).pop(package_name, {})\n\n # Combine all conditional packages so we can easily index over a flat list\n # TODO: Perhaps this should actually return select statements and maintain\n # the conditionals of the dependencies\n flat_deps = {}\n for deps_set in dependencies.values():\n for crate_name, crate_label in deps_set.items():\n flat_deps.update({crate_name: crate_label})\n\n missing_crates = []\n crate_targets = []\n for crate_target in deps:\n if crate_target not in flat_deps:\n missing_crates.append(crate_target)\n else:\n crate_targets.append(flat_deps[crate_target])\n\n if missing_crates:\n fail(\"Could not find crates `{}` among dependencies of `{}`. Available dependencies were `{}`\".format(\n missing_crates,\n package_name,\n dependencies,\n ))\n\n return crate_targets\n\ndef all_crate_deps(\n normal = False, \n normal_dev = False, \n proc_macro = False, \n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Finds the fully qualified label of all requested direct crate dependencies \\\n for the package where this macro is called.\n\n If no parameters are set, all normal dependencies are returned. Setting any one flag will\n otherwise impact the contents of the returned list.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_dependency_maps = []\n if normal:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n if normal_dev:\n all_dependency_maps.append(_NORMAL_DEV_DEPENDENCIES)\n if proc_macro:\n all_dependency_maps.append(_PROC_MACRO_DEPENDENCIES)\n if proc_macro_dev:\n all_dependency_maps.append(_PROC_MACRO_DEV_DEPENDENCIES)\n if build:\n all_dependency_maps.append(_BUILD_DEPENDENCIES)\n if build_proc_macro:\n all_dependency_maps.append(_BUILD_PROC_MACRO_DEPENDENCIES)\n\n # Default to always using normal dependencies\n if not all_dependency_maps:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n\n dependencies = _flatten_dependency_maps(all_dependency_maps).pop(package_name, None)\n\n if not dependencies:\n if dependencies == None:\n fail(\"Tried to get all_crate_deps for package \" + package_name + \" but that package had no Cargo.toml file\")\n else:\n return []\n\n crate_deps = list(dependencies.pop(_COMMON_CONDITION, {}).values())\n for condition, deps in dependencies.items():\n crate_deps += selects.with_or({\n tuple(_CONDITIONS[condition]): deps.values(),\n \"//conditions:default\": [],\n })\n\n return crate_deps\n\ndef aliases(\n normal = False,\n normal_dev = False,\n proc_macro = False,\n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Produces a map of Crate alias names to their original label\n\n If no dependency kinds are specified, `normal` and `proc_macro` are used by default.\n Setting any one flag will otherwise determine the contents of the returned dict.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n dict: The aliases of all associated packages\n \"\"\"\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_aliases_maps = []\n if normal:\n all_aliases_maps.append(_NORMAL_ALIASES)\n if normal_dev:\n all_aliases_maps.append(_NORMAL_DEV_ALIASES)\n if proc_macro:\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n if proc_macro_dev:\n all_aliases_maps.append(_PROC_MACRO_DEV_ALIASES)\n if build:\n all_aliases_maps.append(_BUILD_ALIASES)\n if build_proc_macro:\n all_aliases_maps.append(_BUILD_PROC_MACRO_ALIASES)\n\n # Default to always using normal aliases\n if not all_aliases_maps:\n all_aliases_maps.append(_NORMAL_ALIASES)\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n\n aliases = _flatten_dependency_maps(all_aliases_maps).pop(package_name, None)\n\n if not aliases:\n return dict()\n\n common_items = aliases.pop(_COMMON_CONDITION, {}).items()\n\n # If there are only common items in the dictionary, immediately return them\n if not len(aliases.keys()) == 1:\n return dict(common_items)\n\n # Build a single select statement where each conditional has accounted for the\n # common set of aliases.\n crate_aliases = {\"//conditions:default\": dict(common_items)}\n for condition, deps in aliases.items():\n condition_triples = _CONDITIONS[condition]\n for triple in condition_triples:\n if triple in crate_aliases:\n crate_aliases[triple].update(deps)\n else:\n crate_aliases.update({triple: dict(deps.items() + common_items)})\n\n return select(crate_aliases)\n\n###############################################################################\n# WORKSPACE MEMBER DEPS AND ALIASES\n###############################################################################\n\n_NORMAL_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n _COMMON_CONDITION: {\n \"tokenizers\": Label(\"@crates//:tokenizers-0.21.2\"),\n },\n },\n}\n\n\n_NORMAL_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n _COMMON_CONDITION: {\n },\n },\n}\n\n\n_NORMAL_DEV_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_NORMAL_DEV_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEV_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEV_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_CONDITIONS = {\n \"aarch64-apple-darwin\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\"],\n \"aarch64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\"],\n \"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\",\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\": [],\n \"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\": [],\n \"cfg(any())\": [],\n \"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\": [],\n \"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\": [],\n \"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\": [],\n \"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\",\"@rules_rust//rust/platform:x86_64-apple-darwin\"],\n \"cfg(not(all(target_arch = \\\"arm\\\", target_os = \\\"none\\\")))\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\",\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\",\"@rules_rust//rust/platform:x86_64-apple-darwin\",\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"cfg(target_os = \\\"netbsd\\\")\": [],\n \"cfg(target_os = \\\"solaris\\\")\": [],\n \"cfg(target_os = \\\"vxworks\\\")\": [],\n \"x86_64-apple-darwin\": [\"@rules_rust//rust/platform:x86_64-apple-darwin\"],\n \"x86_64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n}\n\n###############################################################################\n\ndef crate_repositories():\n \"\"\"A macro for defining repositories for all generated crates.\n\n Returns:\n A list of repos visible to the module through the module extension.\n \"\"\"\n maybe(\n http_archive,\n name = \"crates__ahash-0.8.12\",\n sha256 = \"5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ahash/0.8.12/download\"],\n strip_prefix = \"ahash-0.8.12\",\n build_file = Label(\"@crates//crates:BUILD.ahash-0.8.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__aho-corasick-1.1.3\",\n sha256 = \"8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/aho-corasick/1.1.3/download\"],\n strip_prefix = \"aho-corasick-1.1.3\",\n build_file = Label(\"@crates//crates:BUILD.aho-corasick-1.1.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__base64-0.13.1\",\n sha256 = \"9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/base64/0.13.1/download\"],\n strip_prefix = \"base64-0.13.1\",\n build_file = Label(\"@crates//crates:BUILD.base64-0.13.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__bit-set-0.8.0\",\n sha256 = \"08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/bit-set/0.8.0/download\"],\n strip_prefix = \"bit-set-0.8.0\",\n build_file = Label(\"@crates//crates:BUILD.bit-set-0.8.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__bit-vec-0.8.0\",\n sha256 = \"5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/bit-vec/0.8.0/download\"],\n strip_prefix = \"bit-vec-0.8.0\",\n build_file = Label(\"@crates//crates:BUILD.bit-vec-0.8.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__bitflags-2.9.1\",\n sha256 = \"1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/bitflags/2.9.1/download\"],\n strip_prefix = \"bitflags-2.9.1\",\n build_file = Label(\"@crates//crates:BUILD.bitflags-2.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__bumpalo-3.16.0\",\n sha256 = \"79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/bumpalo/3.16.0/download\"],\n strip_prefix = \"bumpalo-3.16.0\",\n build_file = Label(\"@crates//crates:BUILD.bumpalo-3.16.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__byteorder-1.5.0\",\n sha256 = \"1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/byteorder/1.5.0/download\"],\n strip_prefix = \"byteorder-1.5.0\",\n build_file = Label(\"@crates//crates:BUILD.byteorder-1.5.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__castaway-0.2.4\",\n sha256 = \"dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/castaway/0.2.4/download\"],\n strip_prefix = \"castaway-0.2.4\",\n build_file = Label(\"@crates//crates:BUILD.castaway-0.2.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__cfg-if-1.0.0\",\n sha256 = \"baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/cfg-if/1.0.0/download\"],\n strip_prefix = \"cfg-if-1.0.0\",\n build_file = Label(\"@crates//crates:BUILD.cfg-if-1.0.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__compact_str-0.9.0\",\n sha256 = \"3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/compact_str/0.9.0/download\"],\n strip_prefix = \"compact_str-0.9.0\",\n build_file = Label(\"@crates//crates:BUILD.compact_str-0.9.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-deque-0.8.6\",\n sha256 = \"9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-deque/0.8.6/download\"],\n strip_prefix = \"crossbeam-deque-0.8.6\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-deque-0.8.6.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-epoch-0.9.18\",\n sha256 = \"5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-epoch/0.9.18/download\"],\n strip_prefix = \"crossbeam-epoch-0.9.18\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-epoch-0.9.18.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-utils-0.8.21\",\n sha256 = \"d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-utils/0.8.21/download\"],\n strip_prefix = \"crossbeam-utils-0.8.21\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-utils-0.8.21.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling-0.20.10\",\n sha256 = \"6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling/0.20.10/download\"],\n strip_prefix = \"darling-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling_core-0.20.10\",\n sha256 = \"95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling_core/0.20.10/download\"],\n strip_prefix = \"darling_core-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling_core-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling_macro-0.20.10\",\n sha256 = \"d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling_macro/0.20.10/download\"],\n strip_prefix = \"darling_macro-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling_macro-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__dary_heap-0.3.7\",\n sha256 = \"04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/dary_heap/0.3.7/download\"],\n strip_prefix = \"dary_heap-0.3.7\",\n build_file = Label(\"@crates//crates:BUILD.dary_heap-0.3.7.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder-0.20.2\",\n sha256 = \"507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder/0.20.2/download\"],\n strip_prefix = \"derive_builder-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder_core-0.20.2\",\n sha256 = \"2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder_core/0.20.2/download\"],\n strip_prefix = \"derive_builder_core-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder_core-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder_macro-0.20.2\",\n sha256 = \"ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder_macro/0.20.2/download\"],\n strip_prefix = \"derive_builder_macro-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder_macro-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__either-1.13.0\",\n sha256 = \"60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/either/1.13.0/download\"],\n strip_prefix = \"either-1.13.0\",\n build_file = Label(\"@crates//crates:BUILD.either-1.13.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__esaxx-rs-0.1.10\",\n sha256 = \"d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/esaxx-rs/0.1.10/download\"],\n strip_prefix = \"esaxx-rs-0.1.10\",\n build_file = Label(\"@crates//crates:BUILD.esaxx-rs-0.1.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__fancy-regex-0.14.0\",\n sha256 = \"6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/fancy-regex/0.14.0/download\"],\n strip_prefix = \"fancy-regex-0.14.0\",\n build_file = Label(\"@crates//crates:BUILD.fancy-regex-0.14.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__fnv-1.0.7\",\n sha256 = \"3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/fnv/1.0.7/download\"],\n strip_prefix = \"fnv-1.0.7\",\n build_file = Label(\"@crates//crates:BUILD.fnv-1.0.7.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__getrandom-0.3.3\",\n sha256 = \"26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/getrandom/0.3.3/download\"],\n strip_prefix = \"getrandom-0.3.3\",\n build_file = Label(\"@crates//crates:BUILD.getrandom-0.3.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ident_case-1.0.1\",\n sha256 = \"b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ident_case/1.0.1/download\"],\n strip_prefix = \"ident_case-1.0.1\",\n build_file = Label(\"@crates//crates:BUILD.ident_case-1.0.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__itertools-0.14.0\",\n sha256 = \"2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/itertools/0.14.0/download\"],\n strip_prefix = \"itertools-0.14.0\",\n build_file = Label(\"@crates//crates:BUILD.itertools-0.14.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__itoa-1.0.14\",\n sha256 = \"d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/itoa/1.0.14/download\"],\n strip_prefix = \"itoa-1.0.14\",\n build_file = Label(\"@crates//crates:BUILD.itoa-1.0.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__js-sys-0.3.77\",\n sha256 = \"1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/js-sys/0.3.77/download\"],\n strip_prefix = \"js-sys-0.3.77\",\n build_file = Label(\"@crates//crates:BUILD.js-sys-0.3.77.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__libc-0.2.169\",\n sha256 = \"b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/libc/0.2.169/download\"],\n strip_prefix = \"libc-0.2.169\",\n build_file = Label(\"@crates//crates:BUILD.libc-0.2.169.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__log-0.4.22\",\n sha256 = \"a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/log/0.4.22/download\"],\n strip_prefix = \"log-0.4.22\",\n build_file = Label(\"@crates//crates:BUILD.log-0.4.22.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__macro_rules_attribute-0.2.0\",\n sha256 = \"8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/macro_rules_attribute/0.2.0/download\"],\n strip_prefix = \"macro_rules_attribute-0.2.0\",\n build_file = Label(\"@crates//crates:BUILD.macro_rules_attribute-0.2.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__macro_rules_attribute-proc_macro-0.2.0\",\n sha256 = \"b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/macro_rules_attribute-proc_macro/0.2.0/download\"],\n strip_prefix = \"macro_rules_attribute-proc_macro-0.2.0\",\n build_file = Label(\"@crates//crates:BUILD.macro_rules_attribute-proc_macro-0.2.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__memchr-2.7.4\",\n sha256 = \"78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/memchr/2.7.4/download\"],\n strip_prefix = \"memchr-2.7.4\",\n build_file = Label(\"@crates//crates:BUILD.memchr-2.7.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__minimal-lexical-0.2.1\",\n sha256 = \"68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/minimal-lexical/0.2.1/download\"],\n strip_prefix = \"minimal-lexical-0.2.1\",\n build_file = Label(\"@crates//crates:BUILD.minimal-lexical-0.2.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__monostate-0.1.13\",\n sha256 = \"0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/monostate/0.1.13/download\"],\n strip_prefix = \"monostate-0.1.13\",\n build_file = Label(\"@crates//crates:BUILD.monostate-0.1.13.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__monostate-impl-0.1.13\",\n sha256 = \"a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/monostate-impl/0.1.13/download\"],\n strip_prefix = \"monostate-impl-0.1.13\",\n build_file = Label(\"@crates//crates:BUILD.monostate-impl-0.1.13.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__nom-7.1.3\",\n sha256 = \"d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/nom/7.1.3/download\"],\n strip_prefix = \"nom-7.1.3\",\n build_file = Label(\"@crates//crates:BUILD.nom-7.1.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__once_cell-1.20.2\",\n sha256 = \"1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/once_cell/1.20.2/download\"],\n strip_prefix = \"once_cell-1.20.2\",\n build_file = Label(\"@crates//crates:BUILD.once_cell-1.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__paste-1.0.15\",\n sha256 = \"57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/paste/1.0.15/download\"],\n strip_prefix = \"paste-1.0.15\",\n build_file = Label(\"@crates//crates:BUILD.paste-1.0.15.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ppv-lite86-0.2.20\",\n sha256 = \"77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ppv-lite86/0.2.20/download\"],\n strip_prefix = \"ppv-lite86-0.2.20\",\n build_file = Label(\"@crates//crates:BUILD.ppv-lite86-0.2.20.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__proc-macro2-1.0.92\",\n sha256 = \"37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/proc-macro2/1.0.92/download\"],\n strip_prefix = \"proc-macro2-1.0.92\",\n build_file = Label(\"@crates//crates:BUILD.proc-macro2-1.0.92.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__quote-1.0.37\",\n sha256 = \"b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/quote/1.0.37/download\"],\n strip_prefix = \"quote-1.0.37\",\n build_file = Label(\"@crates//crates:BUILD.quote-1.0.37.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__r-efi-5.3.0\",\n sha256 = \"69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/r-efi/5.3.0/download\"],\n strip_prefix = \"r-efi-5.3.0\",\n build_file = Label(\"@crates//crates:BUILD.r-efi-5.3.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand-0.9.1\",\n sha256 = \"9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand/0.9.1/download\"],\n strip_prefix = \"rand-0.9.1\",\n build_file = Label(\"@crates//crates:BUILD.rand-0.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand_chacha-0.9.0\",\n sha256 = \"d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand_chacha/0.9.0/download\"],\n strip_prefix = \"rand_chacha-0.9.0\",\n build_file = Label(\"@crates//crates:BUILD.rand_chacha-0.9.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand_core-0.9.3\",\n sha256 = \"99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand_core/0.9.3/download\"],\n strip_prefix = \"rand_core-0.9.3\",\n build_file = Label(\"@crates//crates:BUILD.rand_core-0.9.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-1.10.0\",\n sha256 = \"b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon/1.10.0/download\"],\n strip_prefix = \"rayon-1.10.0\",\n build_file = Label(\"@crates//crates:BUILD.rayon-1.10.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-cond-0.4.0\",\n sha256 = \"2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon-cond/0.4.0/download\"],\n strip_prefix = \"rayon-cond-0.4.0\",\n build_file = Label(\"@crates//crates:BUILD.rayon-cond-0.4.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-core-1.12.1\",\n sha256 = \"1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon-core/1.12.1/download\"],\n strip_prefix = \"rayon-core-1.12.1\",\n build_file = Label(\"@crates//crates:BUILD.rayon-core-1.12.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-1.11.1\",\n sha256 = \"b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex/1.11.1/download\"],\n strip_prefix = \"regex-1.11.1\",\n build_file = Label(\"@crates//crates:BUILD.regex-1.11.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-automata-0.4.9\",\n sha256 = \"809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-automata/0.4.9/download\"],\n strip_prefix = \"regex-automata-0.4.9\",\n build_file = Label(\"@crates//crates:BUILD.regex-automata-0.4.9.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-syntax-0.8.5\",\n sha256 = \"2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-syntax/0.8.5/download\"],\n strip_prefix = \"regex-syntax-0.8.5\",\n build_file = Label(\"@crates//crates:BUILD.regex-syntax-0.8.5.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rustversion-1.0.21\",\n sha256 = \"8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rustversion/1.0.21/download\"],\n strip_prefix = \"rustversion-1.0.21\",\n build_file = Label(\"@crates//crates:BUILD.rustversion-1.0.21.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ryu-1.0.18\",\n sha256 = \"f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ryu/1.0.18/download\"],\n strip_prefix = \"ryu-1.0.18\",\n build_file = Label(\"@crates//crates:BUILD.ryu-1.0.18.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde-1.0.216\",\n sha256 = \"0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde/1.0.216/download\"],\n strip_prefix = \"serde-1.0.216\",\n build_file = Label(\"@crates//crates:BUILD.serde-1.0.216.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde_derive-1.0.216\",\n sha256 = \"46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde_derive/1.0.216/download\"],\n strip_prefix = \"serde_derive-1.0.216\",\n build_file = Label(\"@crates//crates:BUILD.serde_derive-1.0.216.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde_json-1.0.134\",\n sha256 = \"d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde_json/1.0.134/download\"],\n strip_prefix = \"serde_json-1.0.134\",\n build_file = Label(\"@crates//crates:BUILD.serde_json-1.0.134.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__smallvec-1.13.2\",\n sha256 = \"3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/smallvec/1.13.2/download\"],\n strip_prefix = \"smallvec-1.13.2\",\n build_file = Label(\"@crates//crates:BUILD.smallvec-1.13.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__spm_precompiled-0.1.4\",\n sha256 = \"5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/spm_precompiled/0.1.4/download\"],\n strip_prefix = \"spm_precompiled-0.1.4\",\n build_file = Label(\"@crates//crates:BUILD.spm_precompiled-0.1.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__static_assertions-1.1.0\",\n sha256 = \"a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/static_assertions/1.1.0/download\"],\n strip_prefix = \"static_assertions-1.1.0\",\n build_file = Label(\"@crates//crates:BUILD.static_assertions-1.1.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__strsim-0.11.1\",\n sha256 = \"7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/strsim/0.11.1/download\"],\n strip_prefix = \"strsim-0.11.1\",\n build_file = Label(\"@crates//crates:BUILD.strsim-0.11.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__syn-2.0.91\",\n sha256 = \"d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/syn/2.0.91/download\"],\n strip_prefix = \"syn-2.0.91\",\n build_file = Label(\"@crates//crates:BUILD.syn-2.0.91.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__thiserror-2.0.12\",\n sha256 = \"567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/thiserror/2.0.12/download\"],\n strip_prefix = \"thiserror-2.0.12\",\n build_file = Label(\"@crates//crates:BUILD.thiserror-2.0.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__thiserror-impl-2.0.12\",\n sha256 = \"7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/thiserror-impl/2.0.12/download\"],\n strip_prefix = \"thiserror-impl-2.0.12\",\n build_file = Label(\"@crates//crates:BUILD.thiserror-impl-2.0.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__tokenizers-0.21.2\",\n sha256 = \"4c3846d8588abed0daba25a0e47edd58ea15e450a6088b2575f5116fdb0b27ca\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/tokenizers/0.21.2/download\"],\n strip_prefix = \"tokenizers-0.21.2\",\n build_file = Label(\"@crates//crates:BUILD.tokenizers-0.21.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-ident-1.0.14\",\n sha256 = \"adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-ident/1.0.14/download\"],\n strip_prefix = \"unicode-ident-1.0.14\",\n build_file = Label(\"@crates//crates:BUILD.unicode-ident-1.0.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-normalization-alignments-0.1.12\",\n sha256 = \"43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-normalization-alignments/0.1.12/download\"],\n strip_prefix = \"unicode-normalization-alignments-0.1.12\",\n build_file = Label(\"@crates//crates:BUILD.unicode-normalization-alignments-0.1.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-segmentation-1.12.0\",\n sha256 = \"f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-segmentation/1.12.0/download\"],\n strip_prefix = \"unicode-segmentation-1.12.0\",\n build_file = Label(\"@crates//crates:BUILD.unicode-segmentation-1.12.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode_categories-0.1.1\",\n sha256 = \"39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode_categories/0.1.1/download\"],\n strip_prefix = \"unicode_categories-0.1.1\",\n build_file = Label(\"@crates//crates:BUILD.unicode_categories-0.1.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__version_check-0.9.5\",\n sha256 = \"0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/version_check/0.9.5/download\"],\n strip_prefix = \"version_check-0.9.5\",\n build_file = Label(\"@crates//crates:BUILD.version_check-0.9.5.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasi-0.14.2-wasi-0.2.4\",\n sha256 = \"9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasi/0.14.2+wasi-0.2.4/download\"],\n strip_prefix = \"wasi-0.14.2+wasi-0.2.4\",\n build_file = Label(\"@crates//crates:BUILD.wasi-0.14.2+wasi-0.2.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasm-bindgen-0.2.100\",\n sha256 = \"1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasm-bindgen/0.2.100/download\"],\n strip_prefix = \"wasm-bindgen-0.2.100\",\n build_file = Label(\"@crates//crates:BUILD.wasm-bindgen-0.2.100.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasm-bindgen-backend-0.2.100\",\n sha256 = \"2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasm-bindgen-backend/0.2.100/download\"],\n strip_prefix = \"wasm-bindgen-backend-0.2.100\",\n build_file = Label(\"@crates//crates:BUILD.wasm-bindgen-backend-0.2.100.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasm-bindgen-macro-0.2.100\",\n sha256 = \"7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasm-bindgen-macro/0.2.100/download\"],\n strip_prefix = \"wasm-bindgen-macro-0.2.100\",\n build_file = Label(\"@crates//crates:BUILD.wasm-bindgen-macro-0.2.100.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasm-bindgen-macro-support-0.2.100\",\n sha256 = \"8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasm-bindgen-macro-support/0.2.100/download\"],\n strip_prefix = \"wasm-bindgen-macro-support-0.2.100\",\n build_file = Label(\"@crates//crates:BUILD.wasm-bindgen-macro-support-0.2.100.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasm-bindgen-shared-0.2.100\",\n sha256 = \"1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasm-bindgen-shared/0.2.100/download\"],\n strip_prefix = \"wasm-bindgen-shared-0.2.100\",\n build_file = Label(\"@crates//crates:BUILD.wasm-bindgen-shared-0.2.100.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wit-bindgen-rt-0.39.0\",\n sha256 = \"6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wit-bindgen-rt/0.39.0/download\"],\n strip_prefix = \"wit-bindgen-rt-0.39.0\",\n build_file = Label(\"@crates//crates:BUILD.wit-bindgen-rt-0.39.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-0.7.35\",\n sha256 = \"1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy/0.7.35/download\"],\n strip_prefix = \"zerocopy-0.7.35\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-0.7.35.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-0.8.26\",\n sha256 = \"1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy/0.8.26/download\"],\n strip_prefix = \"zerocopy-0.8.26\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-0.8.26.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-derive-0.7.35\",\n sha256 = \"fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy-derive/0.7.35/download\"],\n strip_prefix = \"zerocopy-derive-0.7.35\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-derive-0.7.35.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-derive-0.8.26\",\n sha256 = \"9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy-derive/0.8.26/download\"],\n strip_prefix = \"zerocopy-derive-0.8.26\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-derive-0.8.26.bazel\"),\n )\n\n return [\n struct(repo=\"crates__tokenizers-0.21.2\", is_dev_dep = False),\n ]\n" } } }, @@ -1010,6 +1289,38 @@ "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"base64\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=base64\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.13.1\",\n)\n" } }, + "crates__bit-set-0.8.0": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/bit-set/0.8.0/download" + ], + "strip_prefix": "bit-set-0.8.0", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"bit_set\",\n deps = [\n \"@crates__bit-vec-0.8.0//:bit_vec\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=bit-set\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.0\",\n)\n" + } + }, + "crates__bit-vec-0.8.0": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/bit-vec/0.8.0/download" + ], + "strip_prefix": "bit-vec-0.8.0", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"bit_vec\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=bit-vec\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.0\",\n)\n" + } + }, "crates__bitflags-2.9.1": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { @@ -1026,6 +1337,22 @@ "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"bitflags\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=bitflags\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.9.1\",\n)\n" } }, + "crates__bumpalo-3.16.0": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/bumpalo/3.16.0/download" + ], + "strip_prefix": "bumpalo-3.16.0", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"bumpalo\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=bumpalo\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"3.16.0\",\n)\n" + } + }, "crates__byteorder-1.5.0": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { @@ -1058,22 +1385,6 @@ "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"castaway\",\n proc_macro_deps = [\n \"@crates__rustversion-1.0.21//:rustversion\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=castaway\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.4\",\n)\n" } }, - "crates__cc-1.2.33": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/cc/1.2.33/download" - ], - "strip_prefix": "cc-1.2.33", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"cc\",\n deps = [\n \"@crates__shlex-1.3.0//:shlex\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=cc\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.2.33\",\n)\n" - } - }, "crates__cfg-if-1.0.0": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { @@ -1295,7 +1606,23 @@ "https://static.crates.io/crates/esaxx-rs/0.1.10/download" ], "strip_prefix": "esaxx-rs-0.1.10", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"esaxx_rs\",\n deps = [\n \"@crates__esaxx-rs-0.1.10//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"cc\",\n \"cpp\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=esaxx-rs\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.10\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"cc\",\n \"cpp\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n deps = [\n \"@crates__cc-1.2.33//:cc\",\n ],\n edition = \"2018\",\n pkg_name = \"esaxx-rs\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=esaxx-rs\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.1.10\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"esaxx_rs\",\n deps = [\n \"@crates__esaxx-rs-0.1.10//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=esaxx-rs\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.10\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2018\",\n pkg_name = \"esaxx-rs\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=esaxx-rs\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.1.10\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + } + }, + "crates__fancy-regex-0.14.0": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/fancy-regex/0.14.0/download" + ], + "strip_prefix": "fancy-regex-0.14.0", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"fancy_regex\",\n deps = [\n \"@crates__bit-set-0.8.0//:bit_set\",\n \"@crates__regex-automata-0.4.9//:regex_automata\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"perf\",\n \"std\",\n \"unicode\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=fancy-regex\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.14.0\",\n)\n" } }, "crates__fnv-1.0.7": { @@ -1327,7 +1654,7 @@ "https://static.crates.io/crates/getrandom/0.3.3/download" ], "strip_prefix": "getrandom-0.3.3", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"getrandom\",\n deps = [\n \"@crates__cfg-if-1.0.0//:cfg_if\",\n \"@crates__getrandom-0.3.3//:build_script_build\",\n ] + select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(any(target_os = \"macos\", target_os = \"openbsd\", target_os = \"vita\", target_os = \"emscripten\"))\n ],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(all(any(target_os = \"linux\", target_os = \"android\"), not(any(all(target_os = \"linux\", target_env = \"\"), getrandom_backend = \"custom\", getrandom_backend = \"linux_raw\", getrandom_backend = \"rdrand\", getrandom_backend = \"rndr\"))))\n ],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(any(target_os = \"macos\", target_os = \"openbsd\", target_os = \"vita\", target_os = \"emscripten\"))\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(all(any(target_os = \"linux\", target_os = \"android\"), not(any(all(target_os = \"linux\", target_env = \"\"), getrandom_backend = \"custom\", getrandom_backend = \"linux_raw\", getrandom_backend = \"rdrand\", getrandom_backend = \"rndr\"))))\n ],\n \"//conditions:default\": [],\n }),\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=getrandom\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.3\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"getrandom\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=getrandom\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.3.3\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"getrandom\",\n deps = [\n \"@crates__cfg-if-1.0.0//:cfg_if\",\n \"@crates__getrandom-0.3.3//:build_script_build\",\n ] + select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(any(target_os = \"macos\", target_os = \"openbsd\", target_os = \"vita\", target_os = \"emscripten\"))\n ],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(all(any(target_os = \"linux\", target_os = \"android\"), not(any(all(target_os = \"linux\", target_env = \"\"), getrandom_backend = \"custom\", getrandom_backend = \"linux_raw\", getrandom_backend = \"rdrand\", getrandom_backend = \"rndr\"))))\n ],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(any(target_os = \"macos\", target_os = \"openbsd\", target_os = \"vita\", target_os = \"emscripten\"))\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(all(any(target_os = \"linux\", target_os = \"android\"), not(any(all(target_os = \"linux\", target_env = \"\"), getrandom_backend = \"custom\", getrandom_backend = \"linux_raw\", getrandom_backend = \"rdrand\", getrandom_backend = \"rndr\"))))\n ],\n \"//conditions:default\": [],\n }),\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n \"wasm_js\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=getrandom\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.3\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n \"wasm_js\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"getrandom\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=getrandom\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.3.3\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" } }, "crates__ident_case-1.0.1": { @@ -1378,6 +1705,22 @@ "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"itoa\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=itoa\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.14\",\n)\n" } }, + "crates__js-sys-0.3.77": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/js-sys/0.3.77/download" + ], + "strip_prefix": "js-sys-0.3.77", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"js_sys\",\n deps = [\n \"@crates__once_cell-1.20.2//:once_cell\",\n \"@crates__wasm-bindgen-0.2.100//:wasm_bindgen\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=js-sys\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.77\",\n)\n" + } + }, "crates__libc-0.2.169": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { @@ -1535,39 +1878,7 @@ "https://static.crates.io/crates/once_cell/1.20.2/download" ], "strip_prefix": "once_cell-1.20.2", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"once_cell\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"default\",\n \"race\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=once_cell\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.20.2\",\n)\n" - } - }, - "crates__onig-6.5.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/onig/6.5.1/download" - ], - "strip_prefix": "onig-6.5.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"onig\",\n deps = [\n \"@crates__bitflags-2.9.1//:bitflags\",\n \"@crates__once_cell-1.20.2//:once_cell\",\n \"@crates__onig_sys-69.9.1//:onig_sys\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=onig\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"6.5.1\",\n)\n" - } - }, - "crates__onig_sys-69.9.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/onig_sys/69.9.1/download" - ], - "strip_prefix": "onig_sys-69.9.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"onig_sys\",\n deps = [\n \"@crates__onig_sys-69.9.1//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=onig_sys\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"69.9.1\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n deps = [\n \"@crates__cc-1.2.33//:cc\",\n \"@crates__pkg-config-0.3.32//:pkg_config\",\n ],\n edition = \"2021\",\n links = \"onig\",\n pkg_name = \"onig_sys\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=onig_sys\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"69.9.1\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"once_cell\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"race\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=once_cell\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.20.2\",\n)\n" } }, "crates__paste-1.0.15": { @@ -1586,22 +1897,6 @@ "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"paste\",\n deps = [\n \"@crates__paste-1.0.15//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=paste\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.15\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2018\",\n pkg_name = \"paste\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=paste\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.15\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" } }, - "crates__pkg-config-0.3.32": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/pkg-config/0.3.32/download" - ], - "strip_prefix": "pkg-config-0.3.32", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"pkg_config\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=pkg-config\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.32\",\n)\n" - } - }, "crates__ppv-lite86-0.2.20": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { @@ -1791,7 +2086,7 @@ "https://static.crates.io/crates/regex-automata/0.4.9/download" ], "strip_prefix": "regex-automata-0.4.9", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"regex_automata\",\n deps = [\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__memchr-2.7.4//:memchr\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"dfa-onepass\",\n \"hybrid\",\n \"meta\",\n \"nfa-backtrack\",\n \"nfa-pikevm\",\n \"nfa-thompson\",\n \"perf-inline\",\n \"perf-literal\",\n \"perf-literal-multisubstring\",\n \"perf-literal-substring\",\n \"std\",\n \"syntax\",\n \"unicode\",\n \"unicode-age\",\n \"unicode-bool\",\n \"unicode-case\",\n \"unicode-gencat\",\n \"unicode-perl\",\n \"unicode-script\",\n \"unicode-segment\",\n \"unicode-word-boundary\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=regex-automata\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.4.9\",\n)\n" + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"regex_automata\",\n deps = [\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__memchr-2.7.4//:memchr\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"dfa\",\n \"dfa-build\",\n \"dfa-onepass\",\n \"dfa-search\",\n \"hybrid\",\n \"meta\",\n \"nfa\",\n \"nfa-backtrack\",\n \"nfa-pikevm\",\n \"nfa-thompson\",\n \"perf\",\n \"perf-inline\",\n \"perf-literal\",\n \"perf-literal-multisubstring\",\n \"perf-literal-substring\",\n \"std\",\n \"syntax\",\n \"unicode\",\n \"unicode-age\",\n \"unicode-bool\",\n \"unicode-case\",\n \"unicode-gencat\",\n \"unicode-perl\",\n \"unicode-script\",\n \"unicode-segment\",\n \"unicode-word-boundary\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=regex-automata\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.4.9\",\n)\n" } }, "crates__regex-syntax-0.8.5": { @@ -1890,22 +2185,6 @@ "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"serde_json\",\n deps = [\n \"@crates__itoa-1.0.14//:itoa\",\n \"@crates__memchr-2.7.4//:memchr\",\n \"@crates__ryu-1.0.18//:ryu\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__serde_json-1.0.134//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde_json\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.134\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"serde_json\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde_json\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.134\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" } }, - "crates__shlex-1.3.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/shlex/1.3.0/download" - ], - "strip_prefix": "shlex-1.3.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"shlex\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=shlex\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.3.0\",\n)\n" - } - }, "crates__smallvec-1.13.2": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { @@ -1983,7 +2262,7 @@ "https://static.crates.io/crates/syn/2.0.91/download" ], "strip_prefix": "syn-2.0.91", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"syn\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__unicode-ident-1.0.14//:unicode_ident\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"clone-impls\",\n \"default\",\n \"derive\",\n \"extra-traits\",\n \"full\",\n \"parsing\",\n \"printing\",\n \"proc-macro\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=syn\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.0.91\",\n)\n" + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"syn\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__unicode-ident-1.0.14//:unicode_ident\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"clone-impls\",\n \"default\",\n \"derive\",\n \"extra-traits\",\n \"full\",\n \"parsing\",\n \"printing\",\n \"proc-macro\",\n \"visit\",\n \"visit-mut\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=syn\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.0.91\",\n)\n" } }, "crates__thiserror-2.0.12": { @@ -2018,20 +2297,20 @@ "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"thiserror_impl\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=thiserror-impl\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.0.12\",\n)\n" } }, - "crates__tokenizers-0.21.4": { + "crates__tokenizers-0.21.2": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { "patch_args": [], "patch_tool": "", "patches": [], "remote_patch_strip": 1, - "sha256": "a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476", + "sha256": "4c3846d8588abed0daba25a0e47edd58ea15e450a6088b2575f5116fdb0b27ca", "type": "tar.gz", "urls": [ - "https://static.crates.io/crates/tokenizers/0.21.4/download" + "https://static.crates.io/crates/tokenizers/0.21.2/download" ], - "strip_prefix": "tokenizers-0.21.4", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"tokenizers\",\n deps = [\n \"@crates__ahash-0.8.12//:ahash\",\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__compact_str-0.9.0//:compact_str\",\n \"@crates__dary_heap-0.3.7//:dary_heap\",\n \"@crates__derive_builder-0.20.2//:derive_builder\",\n \"@crates__esaxx-rs-0.1.10//:esaxx_rs\",\n \"@crates__getrandom-0.3.3//:getrandom\",\n \"@crates__itertools-0.14.0//:itertools\",\n \"@crates__log-0.4.22//:log\",\n \"@crates__macro_rules_attribute-0.2.0//:macro_rules_attribute\",\n \"@crates__monostate-0.1.13//:monostate\",\n \"@crates__onig-6.5.1//:onig\",\n \"@crates__rand-0.9.1//:rand\",\n \"@crates__rayon-1.10.0//:rayon\",\n \"@crates__rayon-cond-0.4.0//:rayon_cond\",\n \"@crates__regex-1.11.1//:regex\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__serde_json-1.0.134//:serde_json\",\n \"@crates__spm_precompiled-0.1.4//:spm_precompiled\",\n \"@crates__thiserror-2.0.12//:thiserror\",\n \"@crates__unicode-normalization-alignments-0.1.12//:unicode_normalization_alignments\",\n \"@crates__unicode-segmentation-1.12.0//:unicode_segmentation\",\n \"@crates__unicode_categories-0.1.1//:unicode_categories\",\n ],\n proc_macro_deps = [\n \"@crates__paste-1.0.15//:paste\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"esaxx_fast\",\n \"onig\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=tokenizers\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.21.4\",\n)\n" + "strip_prefix": "tokenizers-0.21.2", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"tokenizers\",\n deps = [\n \"@crates__ahash-0.8.12//:ahash\",\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__compact_str-0.9.0//:compact_str\",\n \"@crates__dary_heap-0.3.7//:dary_heap\",\n \"@crates__derive_builder-0.20.2//:derive_builder\",\n \"@crates__esaxx-rs-0.1.10//:esaxx_rs\",\n \"@crates__fancy-regex-0.14.0//:fancy_regex\",\n \"@crates__getrandom-0.3.3//:getrandom\",\n \"@crates__itertools-0.14.0//:itertools\",\n \"@crates__log-0.4.22//:log\",\n \"@crates__macro_rules_attribute-0.2.0//:macro_rules_attribute\",\n \"@crates__monostate-0.1.13//:monostate\",\n \"@crates__rand-0.9.1//:rand\",\n \"@crates__rayon-1.10.0//:rayon\",\n \"@crates__rayon-cond-0.4.0//:rayon_cond\",\n \"@crates__regex-1.11.1//:regex\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__serde_json-1.0.134//:serde_json\",\n \"@crates__spm_precompiled-0.1.4//:spm_precompiled\",\n \"@crates__thiserror-2.0.12//:thiserror\",\n \"@crates__unicode-normalization-alignments-0.1.12//:unicode_normalization_alignments\",\n \"@crates__unicode-segmentation-1.12.0//:unicode_segmentation\",\n \"@crates__unicode_categories-0.1.1//:unicode_categories\",\n ],\n proc_macro_deps = [\n \"@crates__paste-1.0.15//:paste\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"fancy-regex\",\n \"unstable_wasm\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=tokenizers\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.21.2\",\n)\n" } }, "crates__unicode-ident-1.0.14": { @@ -2130,6 +2409,86 @@ "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wasi\",\n deps = [\n \"@crates__wit-bindgen-rt-0.39.0//:wit_bindgen_rt\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasi\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.14.2+wasi-0.2.4\",\n)\n" } }, + "crates__wasm-bindgen-0.2.100": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/wasm-bindgen/0.2.100/download" + ], + "strip_prefix": "wasm-bindgen-0.2.100", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wasm_bindgen\",\n deps = [\n \"@crates__cfg-if-1.0.0//:cfg_if\",\n \"@crates__once_cell-1.20.2//:once_cell\",\n \"@crates__wasm-bindgen-0.2.100//:build_script_build\",\n ],\n proc_macro_deps = [\n \"@crates__wasm-bindgen-macro-0.2.100//:wasm_bindgen_macro\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.100\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"wasm-bindgen\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.2.100\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + } + }, + "crates__wasm-bindgen-backend-0.2.100": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/wasm-bindgen-backend/0.2.100/download" + ], + "strip_prefix": "wasm-bindgen-backend-0.2.100", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wasm_bindgen_backend\",\n deps = [\n \"@crates__bumpalo-3.16.0//:bumpalo\",\n \"@crates__log-0.4.22//:log\",\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n \"@crates__wasm-bindgen-shared-0.2.100//:wasm_bindgen_shared\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen-backend\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.100\",\n)\n" + } + }, + "crates__wasm-bindgen-macro-0.2.100": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/wasm-bindgen-macro/0.2.100/download" + ], + "strip_prefix": "wasm-bindgen-macro-0.2.100", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"wasm_bindgen_macro\",\n deps = [\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__wasm-bindgen-macro-support-0.2.100//:wasm_bindgen_macro_support\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen-macro\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.100\",\n)\n" + } + }, + "crates__wasm-bindgen-macro-support-0.2.100": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/wasm-bindgen-macro-support/0.2.100/download" + ], + "strip_prefix": "wasm-bindgen-macro-support-0.2.100", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wasm_bindgen_macro_support\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n \"@crates__wasm-bindgen-backend-0.2.100//:wasm_bindgen_backend\",\n \"@crates__wasm-bindgen-shared-0.2.100//:wasm_bindgen_shared\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen-macro-support\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.100\",\n)\n" + } + }, + "crates__wasm-bindgen-shared-0.2.100": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/wasm-bindgen-shared/0.2.100/download" + ], + "strip_prefix": "wasm-bindgen-shared-0.2.100", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wasm_bindgen_shared\",\n deps = [\n \"@crates__unicode-ident-1.0.14//:unicode_ident\",\n \"@crates__wasm-bindgen-shared-0.2.100//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen-shared\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.100\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n links = \"wasm_bindgen\",\n pkg_name = \"wasm-bindgen-shared\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen-shared\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.2.100\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + } + }, "crates__wit-bindgen-rt-0.39.0": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { @@ -2266,8 +2625,8 @@ }, "@@rules_rust+//crate_universe/private:internal_extensions.bzl%cu_nr": { "general": { - "bzlTransitiveDigest": "xy12v7lhrvLqCz2j/bAJgLlp6Q1jvGETwmcyBW8zeSQ=", - "usagesDigest": "WcrwUq7tMYKrrQoXBAJOrk0OAZY0JyWHpnidg71TX10=", + "bzlTransitiveDigest": "scFQEdObSJq+bJLzhX96j7iqyB2vDibsYijCpXjRAxo=", + "usagesDigest": "UWnpRlkSFx6zlujUh+aOuBGwtJYbR60HyNKAvijQ99g=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -2406,6 +2765,160 @@ ] ] } + }, + "@@rules_swift+//swift:extensions.bzl%non_module_deps": { + "general": { + "bzlTransitiveDigest": "kLWKyo+sC0n/GT9xPbQUxIl+Z0N0H8RcxfqCOyVLuOQ=", + "usagesDigest": "9w18ec4dj90mX/JqpR2tBU2YVc1GU9yNxi/d7q6lLVA=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "com_github_apple_swift_protobuf": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-protobuf/archive/1.20.2.tar.gz" + ], + "sha256": "3fb50bd4d293337f202d917b6ada22f9548a0a0aed9d9a4d791e6fbd8a246ebb", + "strip_prefix": "swift-protobuf-1.20.2/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_protobuf/BUILD.overlay" + } + }, + "com_github_grpc_grpc_swift": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/grpc/grpc-swift/archive/1.16.0.tar.gz" + ], + "sha256": "58b60431d0064969f9679411264b82e40a217ae6bd34e17096d92cc4e47556a5", + "strip_prefix": "grpc-swift-1.16.0/", + "build_file": "@@rules_swift+//third_party:com_github_grpc_grpc_swift/BUILD.overlay" + } + }, + "com_github_apple_swift_docc_symbolkit": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-docc-symbolkit/archive/refs/tags/swift-5.10-RELEASE.tar.gz" + ], + "sha256": "de1d4b6940468ddb53b89df7aa1a81323b9712775b0e33e8254fa0f6f7469a97", + "strip_prefix": "swift-docc-symbolkit-swift-5.10-RELEASE", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_docc_symbolkit/BUILD.overlay" + } + }, + "com_github_apple_swift_nio": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio/archive/2.42.0.tar.gz" + ], + "sha256": "e3304bc3fb53aea74a3e54bd005ede11f6dc357117d9b1db642d03aea87194a0", + "strip_prefix": "swift-nio-2.42.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_http2": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-http2/archive/1.26.0.tar.gz" + ], + "sha256": "f0edfc9d6a7be1d587e5b403f2d04264bdfae59aac1d74f7d974a9022c6d2b25", + "strip_prefix": "swift-nio-http2-1.26.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_http2/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_transport_services": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-transport-services/archive/1.15.0.tar.gz" + ], + "sha256": "f3498dafa633751a52b9b7f741f7ac30c42bcbeb3b9edca6d447e0da8e693262", + "strip_prefix": "swift-nio-transport-services-1.15.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_transport_services/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_extras": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-extras/archive/1.4.0.tar.gz" + ], + "sha256": "4684b52951d9d9937bb3e8ccd6b5daedd777021ef2519ea2f18c4c922843b52b", + "strip_prefix": "swift-nio-extras-1.4.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_extras/BUILD.overlay" + } + }, + "com_github_apple_swift_log": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-log/archive/1.4.4.tar.gz" + ], + "sha256": "48fe66426c784c0c20031f15dc17faf9f4c9037c192bfac2f643f65cb2321ba0", + "strip_prefix": "swift-log-1.4.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_log/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_ssl": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-ssl/archive/2.23.0.tar.gz" + ], + "sha256": "4787c63f61dd04d99e498adc3d1a628193387e41efddf8de19b8db04544d016d", + "strip_prefix": "swift-nio-ssl-2.23.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_ssl/BUILD.overlay" + } + }, + "com_github_apple_swift_collections": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-collections/archive/1.0.4.tar.gz" + ], + "sha256": "d9e4c8a91c60fb9c92a04caccbb10ded42f4cb47b26a212bc6b39cc390a4b096", + "strip_prefix": "swift-collections-1.0.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_collections/BUILD.overlay" + } + }, + "com_github_apple_swift_atomics": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-atomics/archive/1.1.0.tar.gz" + ], + "sha256": "1bee7f469f7e8dc49f11cfa4da07182fbc79eab000ec2c17bfdce468c5d276fb", + "strip_prefix": "swift-atomics-1.1.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_atomics/BUILD.overlay" + } + }, + "build_bazel_rules_swift_index_import": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file": "@@rules_swift+//third_party:build_bazel_rules_swift_index_import/BUILD.overlay", + "canonical_id": "index-import-5.8", + "urls": [ + "https://github.com/MobileNativeFoundation/index-import/releases/download/5.8.0.1/index-import.tar.gz" + ], + "sha256": "28c1ffa39d99e74ed70623899b207b41f79214c498c603915aef55972a851a15" + } + }, + "build_bazel_rules_swift_local_config": { + "repoRuleId": "@@rules_swift+//swift/internal:swift_autoconfiguration.bzl%swift_autoconfiguration", + "attributes": {} + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_swift+", + "bazel_tools", + "bazel_tools" + ] + ] + } } } } diff --git a/README.md b/README.md index 940d056c..c77dcf1d 100644 --- a/README.md +++ b/README.md @@ -103,11 +103,10 @@ compile it, and classify a randomly picked example from the test dataset. On the command line: ``` -cd examples -bazel run --config=release //mnist +bazel run --config=release //examples/mnist # or -./bazel.sh run --config=release //mnist +./bazel.sh run --config=release //examples/mnist ``` @@ -132,9 +131,8 @@ bazel run @zml//tools:hf -- download meta-llama/Llama-3.1-8B-Instruct --local-di Then, you can run the model. ``` -cd examples -bazel run --config=release //llama -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct -bazel run --config=release //llama -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct --prompt="What is the capital of France?" +bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct +bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct --prompt="What is the capital of France?" ``` You can also try `Llama-3.1-70B-Instruct` if you have enough memory. @@ -146,9 +144,8 @@ Like the 8B model above, this model also requires approval. See ``` bazel run @zml//tools:hf -- download meta-llama/Llama-3.2-1B-Instruct --local-dir $HOME/Llama-3.2-1B-Instruct --exclude='*.pth' -cd examples -bazel run --config=release //llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct -bazel run --config=release //llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct -- --prompt="What is the capital of France?" +bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct +bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct -- --prompt="What is the capital of France?" ``` For a larger 3.2 model, you can also try `Llama-3.2-3B-Instruct`. @@ -170,8 +167,7 @@ So, to run the Llama 3.1 8B model from above on your host sporting an NVIDIA GPU run the following: ``` -cd examples -bazel run --config=release //llama \ +bazel run --config=release //examples/llama \ --@zml//runtimes:cuda=true \ -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct \ --prompt="What is the capital of France?" diff --git a/examples/bazel.sh b/bazel.sh similarity index 100% rename from examples/bazel.sh rename to bazel.sh diff --git a/build.zig b/build.zig deleted file mode 120000 index 0468ff14..00000000 --- a/build.zig +++ /dev/null @@ -1 +0,0 @@ -examples/build.zig \ No newline at end of file diff --git a/build.zig b/build.zig new file mode 100644 index 00000000..e69de29b diff --git a/docs/howtos/add_weights.md b/docs/howtos/add_weights.md index 99b30b00..1c151c44 100644 --- a/docs/howtos/add_weights.md +++ b/docs/howtos/add_weights.md @@ -5,7 +5,7 @@ Our [first model](../tutorials/write_first_model.md) did not need any weights fi We just created weights and biases at runtime. But real-world models typically need weights files, and maybe some other -supporting files. +supporting files. We recommend, for easy deployments, you upload those files. In many instances, you will use a site like [🤗 Hugging Face](https://huggingface.co). @@ -14,21 +14,14 @@ We also recommend to add a `weights.bzl` file to your project root directory, so you don't "pollute" your build file with long URLs and SHAs: ```python -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") def _weights_impl(mctx): - http_file( - name = "com_github_zml_cdn_mnist", - downloaded_file_path = "mnist.pt", - sha256 = "d8a25252e28915e147720c19223721f0f53e3317493727ca754a2dd672450ba9", - url = "https://github.com/ggerganov/ggml/raw/18703ad600cc68dbdb04d57434c876989a841d12/examples/mnist/models/mnist/mnist_model.state_dict", - ) - - http_file( - name = "com_github_zml_cdn_mnist_data", - downloaded_file_path = "mnist.ylc", - sha256 = "0fa7898d509279e482958e8ce81c8e77db3f2f8254e26661ceb7762c4d494ce7", - url = "https://github.com/ggerganov/ggml/raw/18703ad600cc68dbdb04d57434c876989a841d12/examples/mnist/models/mnist/t10k-images.idx3-ubyte", + http_archive( + name = "mnist", + sha256 = "075905e433ea0cce13c3fc08832448ab86225d089b5d412be67f59c29388fb19", + url = "https://mirror.zml.ai/data/mnist.tar.zst", + build_file_content = """exports_files(glob(["**"]), visibility = ["//visibility:public"])""", ) return mctx.extension_metadata( @@ -54,12 +47,12 @@ the following way: zig_cc_binary( name = "mnist", args = [ - "$(location @com_github_zml_cdn_mnist//file)", - "$(location @com_github_zml_cdn_mnist_data//file)", + "$(location @mnist//:mnist.pt)", + "$(location @mnist//:t10k-images.idx3-ubyte)", ], data = [ - "@com_github_zml_cdn_mnist//file", - "@com_github_zml_cdn_mnist_data//file", + "@mnist//:mnist.pt", + "@mnist//:t10k-images.idx3-ubyte", ], main = "mnist.zig", deps = [ @@ -74,4 +67,3 @@ See how: - we use `data = [ ... ]` to reference the files in `weights.bzl` - we use `args = [ ... ]` to pass the files as command-line arguments to the MNIST executable at runtime, automatically. - diff --git a/docs/howtos/deploy_on_server.md b/docs/howtos/deploy_on_server.md index b2961051..4c5b8f25 100644 --- a/docs/howtos/deploy_on_server.md +++ b/docs/howtos/deploy_on_server.md @@ -24,8 +24,7 @@ So, to run the Llama model from above **on your development machine** housing an NVIDIA GPU, run the following: ``` -cd examples -bazel run --config=release //llama --@zml//runtimes:cuda=true -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct +bazel run --config=release //examples/llama --@zml//runtimes:cuda=true -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct ``` @@ -41,8 +40,7 @@ architectures: As an example, here is how you build above Llama for CUDA on Linux X86_64: ``` -cd examples -bazel build --config=release //llama \ +bazel build --config=release //examples/llama \ --@zml//runtimes:cuda=true \ --@zml//runtimes:cpu=false \ --platforms=@zml//platforms:linux_amd64 @@ -83,7 +81,6 @@ tar( ... and then build the TAR archive: ``` -# cd examples bazel build --config=release //mnist:archive \ --@zml//runtimes:cuda=true \ --@zml//runtimes:cpu=false \ @@ -134,4 +131,3 @@ INFO: Running command line: bazel-bin/mnist/mnist ../_main~_repo_rules~com_githu You see the command line right up there. On the server, you just need to replace `../` with the 'runfiles' directory of your TAR. - diff --git a/docs/howtos/dockerize_models.md b/docs/howtos/dockerize_models.md index 7bce7b08..af296f8a 100644 --- a/docs/howtos/dockerize_models.md +++ b/docs/howtos/dockerize_models.md @@ -133,8 +133,7 @@ platform_transition_filegroup( And that's almost it! You can already build the image: ``` -# cd examples -bazel build --config=release //simple_layer:image +bazel build --config=release //examples/simple_layer:image INFO: Analyzed target //simple_layer:image (1 packages loaded, 8 targets configured). INFO: Found 1 target... @@ -375,4 +374,3 @@ MNIST model, including weights and dataset, to the docker registry: ``` bazel run //mnist:push --@zml//runtimes:cuda=true -- --repository index.docker.io/my_org/zml_mnist ``` - diff --git a/docs/howtos/huggingface_access_token.md b/docs/howtos/huggingface_access_token.md index c2415c3d..797c6dfb 100644 --- a/docs/howtos/huggingface_access_token.md +++ b/docs/howtos/huggingface_access_token.md @@ -30,8 +30,6 @@ Now you're ready to download a gated model like `Meta-Llama-3.2-1b`! ``` # requires token in $HOME/.cache/huggingface/token, as created by the # `huggingface-cli login` command, or the `HUGGINGFACE_TOKEN` environment variable. -cd examples bazel run @zml//tools:hf -- download meta-llama/Llama-3.2-1B-Instruct --local-dir $HOME/Llama-3.2-1B-Instruct --exclude='*.pth' -bazel run --config=release //llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" +bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" ``` - diff --git a/docs/tutorials/getting_started.md b/docs/tutorials/getting_started.md index b5d2fa17..d5debcbc 100644 --- a/docs/tutorials/getting_started.md +++ b/docs/tutorials/getting_started.md @@ -49,7 +49,6 @@ compile it, and classify a randomly picked example from the test dataset. On the command line: ``` -cd examples bazel run --config=release //mnist ``` @@ -74,9 +73,8 @@ Once you've been granted access, you're ready to download a gated model like ``` # requires token in $HOME/.cache/huggingface/token, as created by the # `huggingface-cli login` command, or the `HUGGINGFACE_TOKEN` environment variable. -cd examples bazel run @zml//tools:hf -- download meta-llama/Llama-3.1-8B-Instruct --local-dir $HOME/Llama-3.1-8B-Instruct --exclude='*.pth' -bazel run --config=release //llama -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct +bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct bazel run --config=release //llama -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct --prompt="What is the capital of France?" ``` @@ -88,9 +86,8 @@ Like the 8B model above, this model also requires approval. See [here](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) for access requirements. ``` -cd examples bazel run @zml//tools:hf -- download meta-llama/Llama-3.2-1B-Instruct --local-dir $HOME/Llama-3.2-1B-Instruct --exclude='*.pth' -bazel run --config=release //llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct +bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct bazel run --config=release //llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" ``` @@ -121,9 +118,8 @@ So, to run the OpenLLama model from above on your host sporting an NVIDIA GPU, run the following: ``` -cd examples -bazel run --config=release //llama:Llama-3.2-1B-Instruct \ - --@zml//runtimes:cuda=true \ +bazel run --config=release //examples/llama:Llama-3.2-1B-Instruct \ + --@zml//runtimes:cuda=true \ -- --prompt="What is the capital of France?" ``` @@ -140,4 +136,3 @@ You might also want to check out the [documentation](../README.md), start [writing your first model](../tutorials/write_first_model.md), or read about more high-level [ZML concepts](../learn/concepts.md). - diff --git a/docs/tutorials/write_first_model.md b/docs/tutorials/write_first_model.md index 4d755d1d..e48cd7e8 100644 --- a/docs/tutorials/write_first_model.md +++ b/docs/tutorials/write_first_model.md @@ -381,8 +381,7 @@ With everything in place now, running the model is easy: ``` # run release (--config=release) -cd examples -bazel run --config=release //simple_layer +bazel run --config=release //examples/simple_layer # compile and run debug version bazel run //simple_layer diff --git a/examples/.bazelignore b/examples/.bazelignore deleted file mode 100644 index a8c9c763..00000000 --- a/examples/.bazelignore +++ /dev/null @@ -1,3 +0,0 @@ -examples -third_party/modules -.zig-cache diff --git a/examples/.bazelrc b/examples/.bazelrc deleted file mode 100644 index d41c85ec..00000000 --- a/examples/.bazelrc +++ /dev/null @@ -1,119 +0,0 @@ -# If we are running from the ZML repo, try to load the .bazelrc-zml above -try-import %workspace%/../.bazelrc-examples -try-import %workspace%/.bazelrc-zml - -# Faster hash function -startup --digest_function=blake3 - -# Disable WORKSPACE -common --enable_workspace=false - -# Don't need for now -common --lockfile_mode=update - -# Needed by LLVM and/or XLA -common --experimental_repo_remote_exec -common --experimental_remote_cache_eviction_retries=5 - -# Self explanatory -common --enable_platform_specific_config - -# Ensure host tools are built optimized -build --host_compilation_mode=opt - -# Suppress all warnings, LLVM and friends are too noisy -build --copt='-w' -build --host_copt='-w' - -# Enable C++17 -build --cxxopt='-std=c++17' -build --host_cxxopt='-std=c++17' -build --enable_runfiles - -# Ensure the local env doesn't leak into the build -build --incompatible_strict_action_env - -# We don't to check the built files in the sandbox or repositories -build --experimental_check_output_files=false -fetch --experimental_check_output_files=false -query --experimental_check_output_files=false - -# Performance -build --legacy_external_runfiles=false - -build --experimental_output_directory_naming_scheme=diff_against_dynamic_baseline -# Mount the zig cache into the sandboxes -build:macos --sandbox_add_mount_pair=/var/tmp -build:linux --sandbox_add_mount_pair=/tmp - -# Ensure workers are sandboxed -build --worker_sandboxing - -# Allows symlinks with spaces -build --experimental_inprocess_symlink_creation - -# Build in debug for now, by default -build --compilation_mode=dbg - -# Various cc options -common --experimental_cc_static_library -common --experimental_cc_shared_library -common --incompatible_use_cc_configure_from_rules_cc -common --experimental_starlark_cc_import - - -common --experimental_platform_in_output_dir - -# Faster sandbox speeds -build --experimental_inmemory_sandbox_stashes - -# clang by default -build --compiler=clang - -# Enable resolving proto compiler from toolchains -common --incompatible_enable_proto_toolchain_resolution - -# Allow empty globs since some legacy rules -common --incompatible_disallow_empty_glob=false - -build:macos --action_env=MACOSX_DEPLOYMENT_TARGET=13.0 -build:macos --macos_sdk_version=13.0 - -# Used by the tsl overlay in python_init_repositories for the bazelified xla module 20241025.0-4663f04 and later -build --repo_env=HERMETIC_PYTHON_VERSION=3.11 - -# Don't detect the local C++ toolchain, be fully hermetic -# This is done only on Linux for now since we use the local toolchain on macOS -build:linux --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 - -# Print test errors in the console -test --test_output=errors - -common:silent --noshow_progress -common:silent --ui_event_filters=,+error,+fail -common:silent --show_result=0 -common:silent --logging=0 - -common --registry=https://bazel-registry.zml.ai -common --registry=https://bcr.bazel.build - -common --@rules_zig//zig/settings:copt=-fPIC -common --@rules_zig//zig/settings:copt=-fcompiler-rt - -common --@toolchains_llvm_bootstrapped//config:experimental_stub_libgcc_s=True - -# --config=debug will used optmized backend, but all the frontend zig code will be compiled in debug mode. -common:debug --compilation_mode=opt -common:debug --@rules_zig//zig/settings:mode=debug -common:debug --strategy=ZigBuildLib=local -common:debug --strategy=ZigBuildObj=local -common:debug --strategy=ZigBuildTestLib=local - -# --config=alldebug build everything in debug mode, including the backend. -common:alldebug --compilation_mode=dbg -common:alldebug --@rules_zig//zig/settings:mode=debug -common:alldebug --strategy=local - -common:release --compilation_mode=opt -common:release --copt=-Ofast -common:release --@rules_zig//zig/settings:mode=release_safe diff --git a/examples/.bazelversion b/examples/.bazelversion deleted file mode 100644 index 2b0aa212..00000000 --- a/examples/.bazelversion +++ /dev/null @@ -1 +0,0 @@ -8.2.1 diff --git a/examples/.gitignore b/examples/.gitignore deleted file mode 100644 index 6add214f..00000000 --- a/examples/.gitignore +++ /dev/null @@ -1,30 +0,0 @@ -# Bazel symlinks -bazel-* - -# macOS -.DS_Store - -# Local Checkouts -/stablehlo/ -/llvm-project/ -/xla/ - -# Just in case, Zig caches -.zig-cache/ -zig-out/ - -# Models -*.safetensors -*.pb -*.trace.json -*.pt -*.bin - -# Data files -*.wav -*.ecdc -.venv - -# Editor specific -## Neovim -.nvim.local.lua diff --git a/examples/.nvim.lua b/examples/.nvim.lua deleted file mode 100644 index a1f9ad95..00000000 --- a/examples/.nvim.lua +++ /dev/null @@ -1,36 +0,0 @@ --- Get the current working directory (CWD) -local cwd = vim.fn.getcwd() - --- Load the local config file if it exists -local f, err = loadfile(".nvim.local.lua") -if not err then - f() -else - local capabilities = vim.lsp.protocol.make_client_capabilities() - - -- Prepend CWD to relative paths - local zls_cmd = cwd .. "/../zml/tools/zls.sh" - - vim.lsp.config["zls"] = { - capabilities = capabilities, - cmd = { zls_cmd }, - root_marker = { "build.zig" }, - filetypes = {"zig"}, - } - vim.lsp.enable('zls') - - vim.api.nvim_create_autocmd('BufWritePre',{ - pattern = {"*.zig", "*.zon"}, - callback = function(ev) - vim.lsp.buf.code_action({ - context = { only = { "source.organizeImports" } }, - apply = true, - }) - vim.lsp.buf.code_action({ - context = { only = { "source.fixAll" } }, - apply = true, - }) - vim.loop.sleep(100) - end - }) -end diff --git a/examples/.vscode/extensions.json b/examples/.vscode/extensions.json deleted file mode 100644 index 194cdc41..00000000 --- a/examples/.vscode/extensions.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "recommendations": [ - "BazelBuild.vscode-bazel", - "llvm-vs-code-extensions.vscode-mlir", - "ms-azuretools.vscode-docker", - "ziglang.vscode-zig", - ] -} diff --git a/examples/.vscode/settings.json b/examples/.vscode/settings.json deleted file mode 100644 index 82050af6..00000000 --- a/examples/.vscode/settings.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "editor.formatOnSave": true, - "files.exclude": { - "**/.git": true, - "**/.svn": true, - "**/.hg": true, - "**/CVS": true, - "**/.DS_Store": true, - "**/bazel-*": true - }, - "files.watcherExclude": { - "**/bazel-*": true, - }, - // "files.trimTrailingWhitespace": true, - "files.insertFinalNewline": true, - "bazel.buildifierFixOnFormat": true, - "bazel.queriesShareServer": true, - "bazel.pathsToIgnore": [ - "." - ], - "bazel.buildifierExecutable": "@buildifier_prebuilt//:buildifier", - "C_Cpp.formatting": "clangFormat", - "C_Cpp.clang_format_style": "Microsoft", - "zig.formattingProvider": "zls", - "zig.path": "x", - "zig.zls.path": "${workspaceFolder}/tools/zls.sh", - "[zig]": { - "editor.suggest.insertMode": "replace", - "editor.stickyScroll.defaultModel": "foldingProviderModel", - "editor.codeActionsOnSave": { - "source.fixAll": "explicit", - } - }, - "rust-analyzer.linkedProjects": [ - "${workspaceFolder}/zml/tokenizer/hftokenizers/Cargo.toml" - ] -} diff --git a/examples/MODULE.bazel b/examples/MODULE.bazel deleted file mode 100644 index 201a5af2..00000000 --- a/examples/MODULE.bazel +++ /dev/null @@ -1,204 +0,0 @@ -module(name = "examples") - -bazel_dep(name = "aspect_bazel_lib", version = "2.14.0") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.11") -bazel_dep(name = "rules_cc", version = "0.1.1") -bazel_dep(name = "rules_oci", version = "2.2.6") -bazel_dep(name = "rules_rust", version = "0.60.0") -bazel_dep(name = "rules_zig", version = "20250714.0-b14a4f1") -bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.2.4") -bazel_dep(name = "zml", version = "0.1.0") - -non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") -use_repo(non_module_deps, "com_github_hejsil_clap") - -oci = use_extension("@rules_oci//oci:extensions.bzl", "oci") -oci.pull( - name = "distroless_cc_debian12", - digest = "sha256:1850aee2ff72864350058d83d681c757d45c885986d15fcca7309b9e5c69f39a", - image = "gcr.io/distroless/cc-debian12", - platforms = [ - "linux/amd64", - ], -) -use_repo(oci, "distroless_cc_debian12", "distroless_cc_debian12_linux_amd64") -oci.pull( - name = "distroless_cc_debian12_debug", - digest = "sha256:ae6f470336acbf2aeffea3db70ec0e74d69bee7270cdb5fa2f28fe840fad57fe", - image = "gcr.io/distroless/cc-debian12", - platforms = [ - "linux/amd64", - ], -) -use_repo(oci, "distroless_cc_debian12_debug", "distroless_cc_debian12_debug_linux_amd64") - -# Mnist weights -http_file = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") - -http_file( - name = "com_github_ggerganov_ggml_mnist", - downloaded_file_path = "mnist.pt", - sha256 = "d8a25252e28915e147720c19223721f0f53e3317493727ca754a2dd672450ba9", - url = "https://github.com/ggerganov/ggml/raw/18703ad600cc68dbdb04d57434c876989a841d12/examples/mnist/models/mnist/mnist_model.state_dict", -) - -http_file( - name = "com_github_ggerganov_ggml_mnist_data", - downloaded_file_path = "mnist.ylc", - sha256 = "0fa7898d509279e482958e8ce81c8e77db3f2f8254e26661ceb7762c4d494ce7", - url = "https://github.com/ggerganov/ggml/raw/18703ad600cc68dbdb04d57434c876989a841d12/examples/mnist/models/mnist/t10k-images.idx3-ubyte", -) - -# Llama 3.2 - -huggingface = use_extension("@zml//bazel:huggingface.bzl", "huggingface") - -huggingface.model( - name = "Meta-Llama-3.2-1B-Instruct", - build_file_content = """\ -package(default_visibility = ["//visibility:public"]) -filegroup( - name = "Meta-Llama-3.2-1B-Instruct", - srcs = glob(["*.json", "*.safetensors"]), -) -""", - commit = "9213176726f574b556790deb65791e0c5aa438b6", - includes = [ - "*.safetensors", - "*.json", - ], - model = "meta-llama/Llama-3.2-1B-Instruct", -) -use_repo(huggingface, "Meta-Llama-3.2-1B-Instruct") - -huggingface.model( - name = "Meta-Llama-3.2-3B-Instruct", - build_file_content = """\ -package(default_visibility = ["//visibility:public"]) -filegroup( - name = "Meta-Llama-3.2-3B-Instruct", - srcs = glob(["*.json", "*.safetensors"]), -) -""", - commit = "0cb88a4f764b7a12671c53f0838cd831a0843b95", - includes = [ - "*.safetensors", - "*.json", - ], - model = "meta-llama/Llama-3.2-3B-Instruct", -) -use_repo(huggingface, "Meta-Llama-3.2-3B-Instruct") - -# Llama 3.1 - -huggingface.model( - name = "Meta-Llama-3.1-8B-Instruct", - build_file_content = """\ -package(default_visibility = ["//visibility:public"]) -filegroup( - name = "Meta-Llama-3.1-8B-Instruct", - srcs = glob(["*.json", "*.safetensors"]), -) -""", - commit = "5206a32e0bd3067aef1ce90f5528ade7d866253f", - includes = [ - "*.safetensors", - "*.json", - ], - model = "meta-llama/Meta-Llama-3.1-8B-Instruct", -) -use_repo(huggingface, "Meta-Llama-3.1-8B-Instruct") - -huggingface.model( - name = "Meta-Llama-3.1-70B-Instruct", - build_file_content = """\ -package(default_visibility = ["//visibility:public"]) -filegroup( - name = "Meta-Llama-3.1-70B-Instruct", - srcs = glob(["*.json", "*.safetensors"]), -) -""", - commit = "945c8663693130f8be2ee66210e062158b2a9693", - includes = [ - "*.safetensors", - "*.json", - ], - model = "meta-llama/Meta-Llama-3.1-70B-Instruct", -) -use_repo(huggingface, "Meta-Llama-3.1-70B-Instruct") - -http_file( - name = "Karpathy-TinyLlama-Stories15M", - downloaded_file_path = "stories15M.tinyllama", - sha256 = "cd590644d963867a2b6e5a1107f51fad663c41d79c149fbecbbb1f95fa81f49a", - url = "https://huggingface.co/karpathy/tinyllamas/resolve/0bd21da7698eaf29a0d7de3992de8a46ef624add/stories15M.bin?download=true", -) - -http_file( - name = "Karpathy-TinyLlama-Tokenizer", - downloaded_file_path = "stories260K.tinyllama", - sha256 = "50a52ef822ee9e83de5ce9d0be0a025a773d019437f58b5ff9dcafb063ece361", - url = "https://github.com/karpathy/llama2.c/raw/c02865df300f3bd9e567ce061000dc23bf785a17/tokenizer.bin", -) - -# ModernBERT -huggingface.model( - name = "ModernBERT-base", - build_file_content = """\ -package(default_visibility = ["//visibility:public"]) -filegroup( - name = "model", - srcs = ["model.safetensors"], -) - -filegroup( - name = "tokenizer", - srcs = ["tokenizer.json"], -) -""", - commit = "94032bb66234a691cf6248265170006a7ced4970", - includes = [ - "model.safetensors", - "tokenizer.json", - ], - model = "answerdotai/ModernBERT-base", -) -use_repo(huggingface, "ModernBERT-base") - -huggingface.model( - name = "ModernBERT-large", - build_file_content = """\ -package(default_visibility = ["//visibility:public"]) -filegroup( - name = "model", - srcs = ["model.safetensors"], -) - -filegroup( - name = "tokenizer", - srcs = ["tokenizer.json"], -) -""", - commit = "4bbcbf40bed02ce487125bcb3c897ea9bdc88340", - includes = [ - "model.safetensors", - "tokenizer.json", - ], - model = "answerdotai/ModernBERT-large", -) -use_repo(huggingface, "ModernBERT-large") - -rust = use_extension("@rules_rust//rust:extensions.bzl", "rust") -rust.toolchain( - edition = "2021", - versions = ["1.86.0"], - extra_target_triples = [ - "aarch64-apple-darwin", - "x86_64-apple-darwin", - "aarch64-unknown-linux-gnu", - "x86_64-unknown-linux-gnu", - ], -) -use_repo(rust, "rust_toolchains") -register_toolchains("@rust_toolchains//:all") diff --git a/examples/MODULE.bazel.lock b/examples/MODULE.bazel.lock deleted file mode 100644 index ffa809e4..00000000 --- a/examples/MODULE.bazel.lock +++ /dev/null @@ -1,6628 +0,0 @@ -{ - "lockFileVersion": 18, - "registryFileHashes": { - "https://bazel-registry.zml.ai/modules/abseil-cpp/20210324.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20211102.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20230125.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20230802.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20230802.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20240722.0.bcr.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20250127.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20250512.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/apple_support/1.11.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/apple_support/1.15.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/apple_support/1.17.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/apple_support/1.21.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.10.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.9.4/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/aspect_rules_py/1.3.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.10.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.11.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.15.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.17.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.18.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.19.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.20.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.21.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.23.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.24.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.25.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.27.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.3.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.4.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.9.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.9.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.0.3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.2.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.2.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.3.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.4.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.4.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.5.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.6.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.7.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.7.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/6.4.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/7.3.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/buildozer/7.1.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/google_benchmark/1.8.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/googletest/1.11.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/googletest/1.14.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/googletest/1.15.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/googletest/1.17.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/jsoncpp/1.9.5/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/jsoncpp/1.9.6/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/libpfm/4.11.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/nlohmann_json/3.6.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/patchelf/0.18.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/pcre2/10.43/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.10/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.11/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.4/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.5/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.6/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.7/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.8/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.9/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/21.7/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/23.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/24.4/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/27.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/27.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/29.0-rc2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/29.0-rc3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/29.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/29.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/3.19.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/32.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/pybind11_bazel/2.11.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/pybind11_bazel/2.12.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/re2/2023-09-01/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/re2/2024-07-02/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_android/0.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_apple/3.16.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_apple/3.22.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.10/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.13/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.14/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.15/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.16/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.17/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.6/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.8/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.9/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_distroless/0.5.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_fuzzing/0.5.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/4.0.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/5.3.5/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/6.0.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/6.3.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/6.4.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/6.5.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/7.1.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/7.10.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/7.12.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/7.2.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/7.3.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/7.6.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/8.11.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/8.3.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/8.5.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/8.6.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/8.8.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_jvm_external/4.4.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_jvm_external/5.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_jvm_external/5.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_jvm_external/5.3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_jvm_external/6.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_jvm_external/6.3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_jvm_external/6.7/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_kotlin/1.9.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_kotlin/1.9.6/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_license/0.0.3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_license/0.0.4/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_license/0.0.7/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_license/0.0.8/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_license/1.0.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_multitool/0.11.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_oci/2.2.6/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_pkg/0.7.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_pkg/0.9.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_pkg/1.0.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_proto/4.0.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_proto/6.0.0-rc1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_proto/6.0.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_proto/7.0.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_proto/7.1.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.10.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.23.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.25.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.28.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.29.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.31.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.33.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.34.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.39.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.4.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.40.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/1.3.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/1.4.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_rust/0.60.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_rust/0.62.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_shell/0.2.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_shell/0.3.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_shell/0.4.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_swift/2.4.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_uv/0.65.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/stardoc/0.5.3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/stardoc/0.5.4/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/stardoc/0.5.6/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/stardoc/0.6.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/stardoc/0.7.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/stardoc/0.7.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/stardoc/0.7.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/toolchains_llvm_bootstrapped/0.2.4/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/toolchains_protoc/0.4.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/upb/0.0.0-20230516-61a97ef/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/with_cfg.bzl/0.9.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/with_cfg.bzl/0.9.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/zlib/1.2.11/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/zlib/1.2.13/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/zlib/1.3.1/MODULE.bazel": "not found", - "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", - "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2", - "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/MODULE.bazel": "70390338f7a5106231d20620712f7cccb659cd0e9d073d1991c038eb9fc57589", - "https://bcr.bazel.build/modules/abseil-cpp/20230125.1/MODULE.bazel": "89047429cb0207707b2dface14ba7f8df85273d484c2572755be4bab7ce9c3a0", - "https://bcr.bazel.build/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "1c8cec495288dccd14fdae6e3f95f772c1c91857047a098fad772034264cc8cb", - "https://bcr.bazel.build/modules/abseil-cpp/20230802.0/MODULE.bazel": "d253ae36a8bd9ee3c5955384096ccb6baf16a1b1e93e858370da0a3b94f77c16", - "https://bcr.bazel.build/modules/abseil-cpp/20230802.1/MODULE.bazel": "fa92e2eb41a04df73cdabeec37107316f7e5272650f81d6cc096418fe647b915", - "https://bcr.bazel.build/modules/abseil-cpp/20240116.0/MODULE.bazel": "98dc378d64c12a4e4741ad3362f87fb737ee6a0886b2d90c3cdbb4d93ea3e0bf", - "https://bcr.bazel.build/modules/abseil-cpp/20240116.1/MODULE.bazel": "37bcdb4440fbb61df6a1c296ae01b327f19e9bb521f9b8e26ec854b6f97309ed", - "https://bcr.bazel.build/modules/abseil-cpp/20240116.2/MODULE.bazel": "73939767a4686cd9a520d16af5ab440071ed75cec1a876bf2fcfaf1f71987a16", - "https://bcr.bazel.build/modules/abseil-cpp/20240722.0.bcr.2/MODULE.bazel": "c3661b44c9d3f17f0b65ffb544896aaeb89127398ea867537babac18133a002a", - "https://bcr.bazel.build/modules/abseil-cpp/20250127.1/MODULE.bazel": "c4a89e7ceb9bf1e25cf84a9f830ff6b817b72874088bf5141b314726e46a57c1", - "https://bcr.bazel.build/modules/abseil-cpp/20250512.1/MODULE.bazel": "d209fdb6f36ffaf61c509fcc81b19e81b411a999a934a032e10cd009a0226215", - "https://bcr.bazel.build/modules/abseil-cpp/20250512.1/source.json": "d725d73707d01bb46ab3ca59ba408b8e9bd336642ca77a2269d4bfb8bbfd413d", - "https://bcr.bazel.build/modules/apple_support/1.11.1/MODULE.bazel": "1843d7cd8a58369a444fc6000e7304425fba600ff641592161d9f15b179fb896", - "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", - "https://bcr.bazel.build/modules/apple_support/1.17.1/MODULE.bazel": "655c922ab1209978a94ef6ca7d9d43e940cd97d9c172fb55f94d91ac53f8610b", - "https://bcr.bazel.build/modules/apple_support/1.21.1/MODULE.bazel": "5809fa3efab15d1f3c3c635af6974044bac8a4919c62238cce06acee8a8c11f1", - "https://bcr.bazel.build/modules/apple_support/1.21.1/source.json": "c5228fea88af2d0aa4ef17355c5d3884ce78717f475c3eba55b55bca23a4a560", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.10.0/MODULE.bazel": "ae12288421b12faeb09bc4cfb3c05945ce66df5c55fca8a2f2f66ae9f21c7acb", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/source.json": "0cf1826853b0bef8b5cd19c0610d717500f5521aa2b38b72b2ec302ac5e7526c", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "780d1a6522b28f5edb7ea09630748720721dfe27690d65a2d33aa7509de77e07", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "812d2dd42f65dca362152101fbec418029cc8fd34cbad1a2fde905383d705838", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.9.4/MODULE.bazel": "ccc41028429f894b02fde7ef67d416cba3ba5084ed9ddb9bb6107aa82d118776", - "https://bcr.bazel.build/modules/aspect_rules_py/1.3.2/MODULE.bazel": "0bd9f9f787a70a725b9829dd17c19e89ea86b1e06f25df505f780d77a96f55a5", - "https://bcr.bazel.build/modules/aspect_rules_py/1.3.2/source.json": "cafc6d1570e4503ff93cfd91309cdf1c96f468f3555993bd1654e75528a9bdd3", - "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", - "https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101", - "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", - "https://bcr.bazel.build/modules/bazel_features/1.15.0/MODULE.bazel": "d38ff6e517149dc509406aca0db3ad1efdd890a85e049585b7234d04238e2a4d", - "https://bcr.bazel.build/modules/bazel_features/1.17.0/MODULE.bazel": "039de32d21b816b47bd42c778e0454217e9c9caac4a3cf8e15c7231ee3ddee4d", - "https://bcr.bazel.build/modules/bazel_features/1.18.0/MODULE.bazel": "1be0ae2557ab3a72a57aeb31b29be347bcdc5d2b1eb1e70f39e3851a7e97041a", - "https://bcr.bazel.build/modules/bazel_features/1.19.0/MODULE.bazel": "59adcdf28230d220f0067b1f435b8537dd033bfff8db21335ef9217919c7fb58", - "https://bcr.bazel.build/modules/bazel_features/1.20.0/MODULE.bazel": "8b85300b9c8594752e0721a37210e34879d23adc219ed9dc8f4104a4a1750920", - "https://bcr.bazel.build/modules/bazel_features/1.21.0/MODULE.bazel": "675642261665d8eea09989aa3b8afb5c37627f1be178382c320d1b46afba5e3b", - "https://bcr.bazel.build/modules/bazel_features/1.23.0/MODULE.bazel": "fd1ac84bc4e97a5a0816b7fd7d4d4f6d837b0047cf4cbd81652d616af3a6591a", - "https://bcr.bazel.build/modules/bazel_features/1.24.0/MODULE.bazel": "4796b4c25b47053e9bbffa792b3792d07e228ff66cd0405faef56a978708acd4", - "https://bcr.bazel.build/modules/bazel_features/1.25.0/MODULE.bazel": "e2e60a10a6da64bbf533f15ca652bf61a033e41c2ed734d79a9a08ba87f68c1a", - "https://bcr.bazel.build/modules/bazel_features/1.27.0/MODULE.bazel": "621eeee06c4458a9121d1f104efb80f39d34deff4984e778359c60eaf1a8cb65", - "https://bcr.bazel.build/modules/bazel_features/1.27.0/source.json": "ed8cf0ef05c858dce3661689d0a2b110ff398e63994e178e4f1f7555a8067fed", - "https://bcr.bazel.build/modules/bazel_features/1.3.0/MODULE.bazel": "cdcafe83ec318cda34e02948e81d790aab8df7a929cec6f6969f13a489ccecd9", - "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", - "https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b", - "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", - "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", - "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", - "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", - "https://bcr.bazel.build/modules/bazel_skylib/1.2.1/MODULE.bazel": "f35baf9da0efe45fa3da1696ae906eea3d615ad41e2e3def4aeb4e8bc0ef9a7a", - "https://bcr.bazel.build/modules/bazel_skylib/1.3.0/MODULE.bazel": "20228b92868bf5cfc41bda7afc8a8ba2a543201851de39d990ec957b513579c5", - "https://bcr.bazel.build/modules/bazel_skylib/1.4.1/MODULE.bazel": "a0dcb779424be33100dcae821e9e27e4f2901d9dfd5333efe5ac6a8d7ab75e1d", - "https://bcr.bazel.build/modules/bazel_skylib/1.4.2/MODULE.bazel": "3bd40978e7a1fac911d5989e6b09d8f64921865a45822d8b09e815eaa726a651", - "https://bcr.bazel.build/modules/bazel_skylib/1.5.0/MODULE.bazel": "32880f5e2945ce6a03d1fbd588e9198c0a959bb42297b2cfaf1685b7bc32e138", - "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/MODULE.bazel": "8fdee2dbaace6c252131c00e1de4b165dc65af02ea278476187765e1a617b917", - "https://bcr.bazel.build/modules/bazel_skylib/1.7.0/MODULE.bazel": "0db596f4563de7938de764cc8deeabec291f55e8ec15299718b93c4423e9796d", - "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/MODULE.bazel": "3120d80c5861aa616222ec015332e5f8d3171e062e3e804a2a0253e1be26e59b", - "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/source.json": "f121b43eeefc7c29efbd51b83d08631e2347297c95aac9764a701f2a6a2bb953", - "https://bcr.bazel.build/modules/buildifier_prebuilt/6.4.0/MODULE.bazel": "37389c6b5a40c59410b4226d3bb54b08637f393d66e2fa57925c6fcf68e64bf4", - "https://bcr.bazel.build/modules/buildifier_prebuilt/7.3.1/MODULE.bazel": "537faf0ad9f5892910074b8e43b4c91c96f1d5d86b6ed04bdbe40cf68aa48b68", - "https://bcr.bazel.build/modules/buildifier_prebuilt/7.3.1/source.json": "55153a5e6ca9c8a7e266c4b46b951e8a010d25ec6062bc35d5d4f89925796bad", - "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", - "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", - "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb", - "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", - "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6", - "https://bcr.bazel.build/modules/googletest/1.14.0/MODULE.bazel": "cfbcbf3e6eac06ef9d85900f64424708cc08687d1b527f0ef65aa7517af8118f", - "https://bcr.bazel.build/modules/googletest/1.15.2/MODULE.bazel": "6de1edc1d26cafb0ea1a6ab3f4d4192d91a312fd2d360b63adaa213cd00b2108", - "https://bcr.bazel.build/modules/googletest/1.17.0/MODULE.bazel": "dbec758171594a705933a29fcf69293d2468c49ec1f2ebca65c36f504d72df46", - "https://bcr.bazel.build/modules/googletest/1.17.0/source.json": "38e4454b25fc30f15439c0378e57909ab1fd0a443158aa35aec685da727cd713", - "https://bcr.bazel.build/modules/jsoncpp/1.9.5/MODULE.bazel": "31271aedc59e815656f5736f282bb7509a97c7ecb43e927ac1a37966e0578075", - "https://bcr.bazel.build/modules/jsoncpp/1.9.6/MODULE.bazel": "2f8d20d3b7d54143213c4dfc3d98225c42de7d666011528dc8fe91591e2e17b0", - "https://bcr.bazel.build/modules/jsoncpp/1.9.6/source.json": "a04756d367a2126c3541682864ecec52f92cdee80a35735a3cb249ce015ca000", - "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", - "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", - "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/source.json": "f448c6e8963fdfa7eb831457df83ad63d3d6355018f6574fb017e8169deb43a9", - "https://bcr.bazel.build/modules/patchelf/0.18.0/MODULE.bazel": "15a6beff7e828d585c5bd0f9f93589df117b5594e9d19e43096c77de58b9ae5f", - "https://bcr.bazel.build/modules/patchelf/0.18.0/source.json": "57caf6bcaa5ba515c6fb1c2eacee00735afbeb1ffacb34a57553fb139c8e4333", - "https://bcr.bazel.build/modules/pcre2/10.43/MODULE.bazel": "08eaa025111bd0fedc14a8187c2905fa6ee4501fbe558193e9bf6cc3e2cdf23c", - "https://bcr.bazel.build/modules/pcre2/10.43/source.json": "8b4149e707094f1d5b57df7216539c3415226e814085c4d960bd9f3d49581b88", - "https://bcr.bazel.build/modules/platforms/0.0.10/MODULE.bazel": "8cb8efaf200bdeb2150d93e162c40f388529a25852b332cec879373771e48ed5", - "https://bcr.bazel.build/modules/platforms/0.0.11/MODULE.bazel": "0daefc49732e227caa8bfa834d65dc52e8cc18a2faf80df25e8caea151a9413f", - "https://bcr.bazel.build/modules/platforms/0.0.11/source.json": "f7e188b79ebedebfe75e9e1d098b8845226c7992b307e28e1496f23112e8fc29", - "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee", - "https://bcr.bazel.build/modules/platforms/0.0.5/MODULE.bazel": "5733b54ea419d5eaf7997054bb55f6a1d0b5ff8aedf0176fef9eea44f3acda37", - "https://bcr.bazel.build/modules/platforms/0.0.6/MODULE.bazel": "ad6eeef431dc52aefd2d77ed20a4b353f8ebf0f4ecdd26a807d2da5aa8cd0615", - "https://bcr.bazel.build/modules/platforms/0.0.7/MODULE.bazel": "72fd4a0ede9ee5c021f6a8dd92b503e089f46c227ba2813ff183b71616034814", - "https://bcr.bazel.build/modules/platforms/0.0.8/MODULE.bazel": "9f142c03e348f6d263719f5074b21ef3adf0b139ee4c5133e2aa35664da9eb2d", - "https://bcr.bazel.build/modules/platforms/0.0.9/MODULE.bazel": "4a87a60c927b56ddd67db50c89acaa62f4ce2a1d2149ccb63ffd871d5ce29ebc", - "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7", - "https://bcr.bazel.build/modules/protobuf/23.1/MODULE.bazel": "88b393b3eb4101d18129e5db51847cd40a5517a53e81216144a8c32dfeeca52a", - "https://bcr.bazel.build/modules/protobuf/24.4/MODULE.bazel": "7bc7ce5f2abf36b3b7b7c8218d3acdebb9426aeb35c2257c96445756f970eb12", - "https://bcr.bazel.build/modules/protobuf/27.0/MODULE.bazel": "7873b60be88844a0a1d8f80b9d5d20cfbd8495a689b8763e76c6372998d3f64c", - "https://bcr.bazel.build/modules/protobuf/27.1/MODULE.bazel": "703a7b614728bb06647f965264967a8ef1c39e09e8f167b3ca0bb1fd80449c0d", - "https://bcr.bazel.build/modules/protobuf/29.0-rc2/MODULE.bazel": "6241d35983510143049943fc0d57937937122baf1b287862f9dc8590fc4c37df", - "https://bcr.bazel.build/modules/protobuf/29.0-rc3/MODULE.bazel": "33c2dfa286578573afc55a7acaea3cada4122b9631007c594bf0729f41c8de92", - "https://bcr.bazel.build/modules/protobuf/29.0/MODULE.bazel": "319dc8bf4c679ff87e71b1ccfb5a6e90a6dbc4693501d471f48662ac46d04e4e", - "https://bcr.bazel.build/modules/protobuf/29.1/MODULE.bazel": "557c3457560ff49e122ed76c0bc3397a64af9574691cb8201b4e46d4ab2ecb95", - "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0", - "https://bcr.bazel.build/modules/protobuf/32.0/MODULE.bazel": "0741cf24f8e1185286578069060e905ed67d68eef5990bfa3dea3fc1afba14c7", - "https://bcr.bazel.build/modules/protobuf/32.0/source.json": "1e278267d3642ab361dc460cc1f2d8e607e8292b9b3f799a47181a3acf4f3294", - "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e", - "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/MODULE.bazel": "e6f4c20442eaa7c90d7190d8dc539d0ab422f95c65a57cc59562170c58ae3d34", - "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/source.json": "6900fdc8a9e95866b8c0d4ad4aba4d4236317b5c1cd04c502df3f0d33afed680", - "https://bcr.bazel.build/modules/re2/2023-09-01/MODULE.bazel": "cb3d511531b16cfc78a225a9e2136007a48cf8a677e4264baeab57fe78a80206", - "https://bcr.bazel.build/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "b4963dda9b31080be1905ef085ecd7dd6cd47c05c79b9cdf83ade83ab2ab271a", - "https://bcr.bazel.build/modules/re2/2024-07-02.bcr.1/source.json": "2ff292be6ef3340325ce8a045ecc326e92cbfab47c7cbab4bd85d28971b97ac4", - "https://bcr.bazel.build/modules/re2/2024-07-02/MODULE.bazel": "0eadc4395959969297cbcf31a249ff457f2f1d456228c67719480205aa306daa", - "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", - "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", - "https://bcr.bazel.build/modules/rules_apple/3.16.0/MODULE.bazel": "0d1caf0b8375942ce98ea944be754a18874041e4e0459401d925577624d3a54a", - "https://bcr.bazel.build/modules/rules_apple/3.22.0/MODULE.bazel": "0fd8c630d6cecc18abe84114ee21c1a0fb759b45063d3d2600ec22669e93ea43", - "https://bcr.bazel.build/modules/rules_apple/3.22.0/source.json": "ee41f63c292bd876fd264a830cf45500619a063e2fa3a18cca6fa12b153d68e3", - "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", - "https://bcr.bazel.build/modules/rules_cc/0.0.10/MODULE.bazel": "ec1705118f7eaedd6e118508d3d26deba2a4e76476ada7e0e3965211be012002", - "https://bcr.bazel.build/modules/rules_cc/0.0.13/MODULE.bazel": "0e8529ed7b323dad0775ff924d2ae5af7640b23553dfcd4d34344c7e7a867191", - "https://bcr.bazel.build/modules/rules_cc/0.0.14/MODULE.bazel": "5e343a3aac88b8d7af3b1b6d2093b55c347b8eefc2e7d1442f7a02dc8fea48ac", - "https://bcr.bazel.build/modules/rules_cc/0.0.15/MODULE.bazel": "6704c35f7b4a72502ee81f61bf88706b54f06b3cbe5558ac17e2e14666cd5dcc", - "https://bcr.bazel.build/modules/rules_cc/0.0.16/MODULE.bazel": "7661303b8fc1b4d7f532e54e9d6565771fea666fbdf839e0a86affcd02defe87", - "https://bcr.bazel.build/modules/rules_cc/0.0.17/MODULE.bazel": "2ae1d8f4238ec67d7185d8861cb0a2cdf4bc608697c331b95bf990e69b62e64a", - "https://bcr.bazel.build/modules/rules_cc/0.0.2/MODULE.bazel": "6915987c90970493ab97393024c156ea8fb9f3bea953b2f3ec05c34f19b5695c", - "https://bcr.bazel.build/modules/rules_cc/0.0.6/MODULE.bazel": "abf360251023dfe3efcef65ab9d56beefa8394d4176dd29529750e1c57eaa33f", - "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", - "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", - "https://bcr.bazel.build/modules/rules_cc/0.1.1/MODULE.bazel": "2f0222a6f229f0bf44cd711dc13c858dad98c62d52bd51d8fc3a764a83125513", - "https://bcr.bazel.build/modules/rules_cc/0.1.1/source.json": "d61627377bd7dd1da4652063e368d9366fc9a73920bfa396798ad92172cf645c", - "https://bcr.bazel.build/modules/rules_distroless/0.5.1/MODULE.bazel": "2a63f4744d30749128105da5f96adf7caf5628e37548293f89e7fa39c3b3f2c2", - "https://bcr.bazel.build/modules/rules_distroless/0.5.1/source.json": "c6b9ff7f325bfed89c3671757f14c1d1bc6077d0fcef809b8aa2d007cac7dd1d", - "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", - "https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/MODULE.bazel": "40c97d1144356f52905566c55811f13b299453a14ac7769dfba2ac38192337a8", - "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", - "https://bcr.bazel.build/modules/rules_java/5.3.5/MODULE.bazel": "a4ec4f2db570171e3e5eb753276ee4b389bae16b96207e9d3230895c99644b86", - "https://bcr.bazel.build/modules/rules_java/6.0.0/MODULE.bazel": "8a43b7df601a7ec1af61d79345c17b31ea1fedc6711fd4abfd013ea612978e39", - "https://bcr.bazel.build/modules/rules_java/6.3.0/MODULE.bazel": "a97c7678c19f236a956ad260d59c86e10a463badb7eb2eda787490f4c969b963", - "https://bcr.bazel.build/modules/rules_java/6.4.0/MODULE.bazel": "e986a9fe25aeaa84ac17ca093ef13a4637f6107375f64667a15999f77db6c8f6", - "https://bcr.bazel.build/modules/rules_java/6.5.2/MODULE.bazel": "1d440d262d0e08453fa0c4d8f699ba81609ed0e9a9a0f02cd10b3e7942e61e31", - "https://bcr.bazel.build/modules/rules_java/7.1.0/MODULE.bazel": "30d9135a2b6561c761bd67bd4990da591e6bdc128790ce3e7afd6a3558b2fb64", - "https://bcr.bazel.build/modules/rules_java/7.10.0/MODULE.bazel": "530c3beb3067e870561739f1144329a21c851ff771cd752a49e06e3dc9c2e71a", - "https://bcr.bazel.build/modules/rules_java/7.12.2/MODULE.bazel": "579c505165ee757a4280ef83cda0150eea193eed3bef50b1004ba88b99da6de6", - "https://bcr.bazel.build/modules/rules_java/7.2.0/MODULE.bazel": "06c0334c9be61e6cef2c8c84a7800cef502063269a5af25ceb100b192453d4ab", - "https://bcr.bazel.build/modules/rules_java/7.3.2/MODULE.bazel": "50dece891cfdf1741ea230d001aa9c14398062f2b7c066470accace78e412bc2", - "https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe", - "https://bcr.bazel.build/modules/rules_java/8.11.0/MODULE.bazel": "c3d280bc5ff1038dcb3bacb95d3f6b83da8dd27bba57820ec89ea4085da767ad", - "https://bcr.bazel.build/modules/rules_java/8.11.0/source.json": "302b52a39259a85aa06ca3addb9787864ca3e03b432a5f964ea68244397e7544", - "https://bcr.bazel.build/modules/rules_java/8.3.2/MODULE.bazel": "7336d5511ad5af0b8615fdc7477535a2e4e723a357b6713af439fe8cf0195017", - "https://bcr.bazel.build/modules/rules_java/8.5.1/MODULE.bazel": "d8a9e38cc5228881f7055a6079f6f7821a073df3744d441978e7a43e20226939", - "https://bcr.bazel.build/modules/rules_java/8.6.1/MODULE.bazel": "f4808e2ab5b0197f094cabce9f4b006a27766beb6a9975931da07099560ca9c2", - "https://bcr.bazel.build/modules/rules_java/8.8.0/MODULE.bazel": "de589d0880911ac007abd521b9f0ddcd8b0dbd05c8553e6f8124a050b83acf7d", - "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7", - "https://bcr.bazel.build/modules/rules_jvm_external/5.1/MODULE.bazel": "33f6f999e03183f7d088c9be518a63467dfd0be94a11d0055fe2d210f89aa909", - "https://bcr.bazel.build/modules/rules_jvm_external/5.2/MODULE.bazel": "d9351ba35217ad0de03816ef3ed63f89d411349353077348a45348b096615036", - "https://bcr.bazel.build/modules/rules_jvm_external/5.3/MODULE.bazel": "bf93870767689637164657731849fb887ad086739bd5d360d90007a581d5527d", - "https://bcr.bazel.build/modules/rules_jvm_external/6.1/MODULE.bazel": "75b5fec090dbd46cf9b7d8ea08cf84a0472d92ba3585b476f44c326eda8059c4", - "https://bcr.bazel.build/modules/rules_jvm_external/6.3/MODULE.bazel": "c998e060b85f71e00de5ec552019347c8bca255062c990ac02d051bb80a38df0", - "https://bcr.bazel.build/modules/rules_jvm_external/6.7/MODULE.bazel": "e717beabc4d091ecb2c803c2d341b88590e9116b8bf7947915eeb33aab4f96dd", - "https://bcr.bazel.build/modules/rules_jvm_external/6.7/source.json": "5426f412d0a7fc6b611643376c7e4a82dec991491b9ce5cb1cfdd25fe2e92be4", - "https://bcr.bazel.build/modules/rules_kotlin/1.9.0/MODULE.bazel": "ef85697305025e5a61f395d4eaede272a5393cee479ace6686dba707de804d59", - "https://bcr.bazel.build/modules/rules_kotlin/1.9.6/MODULE.bazel": "d269a01a18ee74d0335450b10f62c9ed81f2321d7958a2934e44272fe82dcef3", - "https://bcr.bazel.build/modules/rules_kotlin/1.9.6/source.json": "2faa4794364282db7c06600b7e5e34867a564ae91bda7cae7c29c64e9466b7d5", - "https://bcr.bazel.build/modules/rules_license/0.0.3/MODULE.bazel": "627e9ab0247f7d1e05736b59dbb1b6871373de5ad31c3011880b4133cafd4bd0", - "https://bcr.bazel.build/modules/rules_license/0.0.4/MODULE.bazel": "6a88dd22800cf1f9f79ba32cacad0d3a423ed28efa2c2ed5582eaa78dd3ac1e5", - "https://bcr.bazel.build/modules/rules_license/0.0.7/MODULE.bazel": "088fbeb0b6a419005b89cf93fe62d9517c0a2b8bb56af3244af65ecfe37e7d5d", - "https://bcr.bazel.build/modules/rules_license/0.0.8/MODULE.bazel": "5669c6fe49b5134dbf534db681ad3d67a2d49cfc197e4a95f1ca2fd7f3aebe96", - "https://bcr.bazel.build/modules/rules_license/1.0.0/MODULE.bazel": "a7fda60eefdf3d8c827262ba499957e4df06f659330bbe6cdbdb975b768bb65c", - "https://bcr.bazel.build/modules/rules_license/1.0.0/source.json": "a52c89e54cc311196e478f8382df91c15f7a2bfdf4c6cd0e2675cc2ff0b56efb", - "https://bcr.bazel.build/modules/rules_multitool/0.11.0/MODULE.bazel": "8d9dda78d2398e136300d3ef4fbcc89ede7c32c158d8c016fa7d032df41c4aaf", - "https://bcr.bazel.build/modules/rules_multitool/0.11.0/source.json": "0b86574a1eaff37c33aafaff095ea16d6ac846beb94ffc74c4fcf626f8f80681", - "https://bcr.bazel.build/modules/rules_oci/2.2.6/MODULE.bazel": "2ba6ddd679269e00aeffe9ca04faa2d0ca4129650982c9246d0d459fe2da47d9", - "https://bcr.bazel.build/modules/rules_oci/2.2.6/source.json": "94e7decb8f95d9465b0bbea71c65064cd16083be1350c7468f131818641dc4a5", - "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc", - "https://bcr.bazel.build/modules/rules_pkg/0.9.1/MODULE.bazel": "af00144208c4be503bc920d043ba3284fb37385b3f6160b4a4daf4df80b4b823", - "https://bcr.bazel.build/modules/rules_pkg/1.0.1/MODULE.bazel": "5b1df97dbc29623bccdf2b0dcd0f5cb08e2f2c9050aab1092fd39a41e82686ff", - "https://bcr.bazel.build/modules/rules_pkg/1.0.1/source.json": "bd82e5d7b9ce2d31e380dd9f50c111d678c3bdaca190cb76b0e1c71b05e1ba8a", - "https://bcr.bazel.build/modules/rules_proto/4.0.0/MODULE.bazel": "a7a7b6ce9bee418c1a760b3d84f83a299ad6952f9903c67f19e4edd964894e06", - "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "e8dff86b0971688790ae75528fe1813f71809b5afd57facb44dad9e8eca631b7", - "https://bcr.bazel.build/modules/rules_proto/6.0.0-rc1/MODULE.bazel": "1e5b502e2e1a9e825eef74476a5a1ee524a92297085015a052510b09a1a09483", - "https://bcr.bazel.build/modules/rules_proto/6.0.2/MODULE.bazel": "ce916b775a62b90b61888052a416ccdda405212b6aaeb39522f7dc53431a5e73", - "https://bcr.bazel.build/modules/rules_proto/7.0.2/MODULE.bazel": "bf81793bd6d2ad89a37a40693e56c61b0ee30f7a7fdbaf3eabbf5f39de47dea2", - "https://bcr.bazel.build/modules/rules_proto/7.1.0/MODULE.bazel": "002d62d9108f75bb807cd56245d45648f38275cb3a99dcd45dfb864c5d74cb96", - "https://bcr.bazel.build/modules/rules_proto/7.1.0/source.json": "39f89066c12c24097854e8f57ab8558929f9c8d474d34b2c00ac04630ad8940e", - "https://bcr.bazel.build/modules/rules_python/0.10.2/MODULE.bazel": "cc82bc96f2997baa545ab3ce73f196d040ffb8756fd2d66125a530031cd90e5f", - "https://bcr.bazel.build/modules/rules_python/0.23.1/MODULE.bazel": "49ffccf0511cb8414de28321f5fcf2a31312b47c40cc21577144b7447f2bf300", - "https://bcr.bazel.build/modules/rules_python/0.25.0/MODULE.bazel": "72f1506841c920a1afec76975b35312410eea3aa7b63267436bfb1dd91d2d382", - "https://bcr.bazel.build/modules/rules_python/0.28.0/MODULE.bazel": "cba2573d870babc976664a912539b320cbaa7114cd3e8f053c720171cde331ed", - "https://bcr.bazel.build/modules/rules_python/0.29.0/MODULE.bazel": "2ac8cd70524b4b9ec49a0b8284c79e4cd86199296f82f6e0d5da3f783d660c82", - "https://bcr.bazel.build/modules/rules_python/0.31.0/MODULE.bazel": "93a43dc47ee570e6ec9f5779b2e64c1476a6ce921c48cc9a1678a91dd5f8fd58", - "https://bcr.bazel.build/modules/rules_python/0.33.2/MODULE.bazel": "3e036c4ad8d804a4dad897d333d8dce200d943df4827cb849840055be8d2e937", - "https://bcr.bazel.build/modules/rules_python/0.34.0/MODULE.bazel": "1d623d026e075b78c9fde483a889cda7996f5da4f36dffb24c246ab30f06513a", - "https://bcr.bazel.build/modules/rules_python/0.39.0/MODULE.bazel": "8154faf3b171e09d3ecfde0704d2e6db50bc0aba9db374421a3d5dda2e28ab91", - "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", - "https://bcr.bazel.build/modules/rules_python/0.40.0/MODULE.bazel": "9d1a3cd88ed7d8e39583d9ffe56ae8a244f67783ae89b60caafc9f5cf318ada7", - "https://bcr.bazel.build/modules/rules_python/1.3.0/MODULE.bazel": "8361d57eafb67c09b75bf4bbe6be360e1b8f4f18118ab48037f2bd50aa2ccb13", - "https://bcr.bazel.build/modules/rules_python/1.4.1/MODULE.bazel": "8991ad45bdc25018301d6b7e1d3626afc3c8af8aaf4bc04f23d0b99c938b73a6", - "https://bcr.bazel.build/modules/rules_python/1.4.1/source.json": "8ec8c90c70ccacc4de8ca1b97f599e756fb59173e898ee08b733006650057c07", - "https://bcr.bazel.build/modules/rules_rust/0.60.0/MODULE.bazel": "911ff2a12d01ac574fd6dfec0b05fa976ff8693d8c2420db637a9f98f697b0ae", - "https://bcr.bazel.build/modules/rules_rust/0.62.0/MODULE.bazel": "6a15b57982e278793c684f426e19166e62e73f1bd45fe3b6bcedd0b901177b37", - "https://bcr.bazel.build/modules/rules_rust/0.62.0/source.json": "1b3a6551a585ee47cafa21550c5eb87c6f3a56bb9761f9e3421ff1102f220437", - "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", - "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", - "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", - "https://bcr.bazel.build/modules/rules_shell/0.4.1/source.json": "4757bd277fe1567763991c4425b483477bb82e35e777a56fd846eb5cceda324a", - "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", - "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", - "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", - "https://bcr.bazel.build/modules/rules_swift/2.4.0/source.json": "a6577f57f9febbdc015a01f2a8f3487422032f134d6c61d18ed8e8ca3b9acc7c", - "https://bcr.bazel.build/modules/rules_uv/0.65.0/MODULE.bazel": "a700505a80ad0ad65f062c9335458d137f504b98c5937b338807bb5cbe305e73", - "https://bcr.bazel.build/modules/rules_uv/0.65.0/source.json": "315237fb5f44ada2d537aabb2f89c6ba2296569ed63fc7340ca66d65f5730081", - "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", - "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", - "https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4", - "https://bcr.bazel.build/modules/stardoc/0.5.6/MODULE.bazel": "c43dabc564990eeab55e25ed61c07a1aadafe9ece96a4efabb3f8bf9063b71ef", - "https://bcr.bazel.build/modules/stardoc/0.6.2/MODULE.bazel": "7060193196395f5dd668eda046ccbeacebfd98efc77fed418dbe2b82ffaa39fd", - "https://bcr.bazel.build/modules/stardoc/0.7.0/MODULE.bazel": "05e3d6d30c099b6770e97da986c53bd31844d7f13d41412480ea265ac9e8079c", - "https://bcr.bazel.build/modules/stardoc/0.7.1/MODULE.bazel": "3548faea4ee5dda5580f9af150e79d0f6aea934fc60c1cc50f4efdd9420759e7", - "https://bcr.bazel.build/modules/stardoc/0.7.2/MODULE.bazel": "fc152419aa2ea0f51c29583fab1e8c99ddefd5b3778421845606ee628629e0e5", - "https://bcr.bazel.build/modules/stardoc/0.7.2/source.json": "58b029e5e901d6802967754adf0a9056747e8176f017cfe3607c0851f4d42216", - "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "5e463fbfba7b1701d957555ed45097d7f984211330106ccd1352c6e0af0dcf91", - "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/source.json": "32bd87e5f4d7acc57c5b2ff7c325ae3061d5e242c0c4c214ae87e0f1c13e54cb", - "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.2.4/MODULE.bazel": "859f02226bc313a622f56b816582fb55d5aaced4a1ecde7397dd4a9ebd24321e", - "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.2.4/source.json": "43c5fd6b585cbf0b642157b6843a25425d541fabe638acb7bc908846c4bda8ca", - "https://bcr.bazel.build/modules/toolchains_protoc/0.4.1/MODULE.bazel": "05d6c16474a7a96002dd5512c444aa6965ebcf95f3e0c47001aa18269fbecec9", - "https://bcr.bazel.build/modules/toolchains_protoc/0.4.1/source.json": "474d926c8e845762faaa42e792fc66cd36794daee4c8af900f673fad0b403a8d", - "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", - "https://bcr.bazel.build/modules/upb/0.0.0-20230516-61a97ef/MODULE.bazel": "c0df5e35ad55e264160417fd0875932ee3c9dda63d9fccace35ac62f45e1b6f9", - "https://bcr.bazel.build/modules/with_cfg.bzl/0.9.1/MODULE.bazel": "5b880a01ab5b3bdb30931075d6b8e58075f999400f7a544632dc7b4a575b3c1e", - "https://bcr.bazel.build/modules/with_cfg.bzl/0.9.2/MODULE.bazel": "8008021cb802e87e22b3e2b6f403c0529d4b865703fdcd40e2f36c23a651291c", - "https://bcr.bazel.build/modules/with_cfg.bzl/0.9.2/source.json": "06d276bd336d6c9f069d4a68729de4e5d322129665f871c2b328a14b26b7cd99", - "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", - "https://bcr.bazel.build/modules/zlib/1.2.13/MODULE.bazel": "aa6deb1b83c18ffecd940c4119aff9567cd0a671d7bba756741cb2ef043a29d5", - "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "af322bc08976524477c79d1e45e241b6efbeb918c497e8840b8ab116802dda79", - "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "eec517b5bbe5492629466e11dae908d043364302283de25581e3eb944326c4ca", - "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/source.json": "22bc55c47af97246cfc093d0acf683a7869377de362b5d1c552c2c2e16b7a806", - "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198" - }, - "selectedYankedVersions": {}, - "moduleExtensions": { - "@@apple_support+//crosstool:setup.bzl%apple_cc_configure_extension": { - "general": { - "bzlTransitiveDigest": "4vzWoFey2mnm603onNMfaDX4DRoTrnIgZawOBMnWpoI=", - "usagesDigest": "g16sWEKaACd08DhNP3oNi7nAS8e2q6cva2PQQJGC3sM=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "local_config_apple_cc_toolchains": { - "repoRuleId": "@@apple_support+//crosstool:setup.bzl%_apple_cc_autoconf_toolchains", - "attributes": {} - }, - "local_config_apple_cc": { - "repoRuleId": "@@apple_support+//crosstool:setup.bzl%_apple_cc_autoconf", - "attributes": {} - } - }, - "recordedRepoMappingEntries": [ - [ - "apple_support+", - "bazel_tools", - "bazel_tools" - ], - [ - "bazel_tools", - "rules_cc", - "rules_cc+" - ] - ] - } - }, - "@@aspect_rules_py+//py:extensions.bzl%py_tools": { - "general": { - "bzlTransitiveDigest": "YIPHOwcYJl8X9QiZwJZ6IKTETxJTMYqApef63o81SeQ=", - "usagesDigest": "/X+U31L/0W+mPo15khS6kjLevIqdkqh+JgfyWHDdHH8=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "bsd_tar_darwin_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "bsd_tar_darwin_arm64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "bsd_tar_linux_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "bsd_tar_linux_arm64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "bsd_tar_windows_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "windows_amd64" - } - }, - "bsd_tar_toolchains": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%tar_toolchains_repo", - "attributes": { - "user_repository_name": "bsd_tar" - } - }, - "rules_py_tools.darwin_amd64": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:tools.bzl%prebuilt_tool_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "rules_py_tools.darwin_arm64": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:tools.bzl%prebuilt_tool_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "rules_py_tools.linux_amd64": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:tools.bzl%prebuilt_tool_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "rules_py_tools.linux_arm64": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:tools.bzl%prebuilt_tool_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "rules_py_tools": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%toolchains_repo", - "attributes": { - "user_repository_name": "rules_py_tools" - } - }, - "rules_py_pex_2_3_1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://files.pythonhosted.org/packages/e7/d0/fbda2a4d41d62d86ce53f5ae4fbaaee8c34070f75bb7ca009090510ae874/pex-2.3.1-py2.py3-none-any.whl" - ], - "sha256": "64692a5bf6f298403aab930d22f0d836ae4736c5bc820e262e9092fe8c56f830", - "downloaded_file_path": "pex-2.3.1-py2.py3-none-any.whl" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "aspect_bazel_lib+", - "bazel_tools", - "bazel_tools" - ], - [ - "aspect_rules_py+", - "aspect_bazel_lib", - "aspect_bazel_lib+" - ], - [ - "aspect_rules_py+", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@buildifier_prebuilt+//:defs.bzl%buildifier_prebuilt_deps_extension": { - "general": { - "bzlTransitiveDigest": "KZE4I/C8xG6s/UMpbc11AQBqGwhM3g15djtTmNRKxes=", - "usagesDigest": "eWMDBEn8E8CrwAPXrlrjIap2pseSMhxDyDdrntHBOOE=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "buildifier_darwin_amd64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v7.3.1/buildifier-darwin-amd64" - ], - "downloaded_file_path": "buildifier", - "executable": true, - "sha256": "375f823103d01620aaec20a0c29c6cbca99f4fd0725ae30b93655c6704f44d71" - } - }, - "buildifier_darwin_arm64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v7.3.1/buildifier-darwin-arm64" - ], - "downloaded_file_path": "buildifier", - "executable": true, - "sha256": "5a6afc6ac7a09f5455ba0b89bd99d5ae23b4174dc5dc9d6c0ed5ce8caac3f813" - } - }, - "buildifier_linux_amd64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v7.3.1/buildifier-linux-amd64" - ], - "downloaded_file_path": "buildifier", - "executable": true, - "sha256": "5474cc5128a74e806783d54081f581662c4be8ae65022f557e9281ed5dc88009" - } - }, - "buildifier_linux_arm64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v7.3.1/buildifier-linux-arm64" - ], - "downloaded_file_path": "buildifier", - "executable": true, - "sha256": "0bf86c4bfffaf4f08eed77bde5b2082e4ae5039a11e2e8b03984c173c34a561c" - } - }, - "buildifier_windows_amd64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v7.3.1/buildifier-windows-amd64.exe" - ], - "downloaded_file_path": "buildifier.exe", - "executable": true, - "sha256": "370cd576075ad29930a82f5de132f1a1de4084c784a82514bd4da80c85acf4a8" - } - }, - "buildozer_darwin_amd64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v7.3.1/buildozer-darwin-amd64" - ], - "downloaded_file_path": "buildozer", - "executable": true, - "sha256": "854c9583efc166602276802658cef3f224d60898cfaa60630b33d328db3b0de2" - } - }, - "buildozer_darwin_arm64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v7.3.1/buildozer-darwin-arm64" - ], - "downloaded_file_path": "buildozer", - "executable": true, - "sha256": "31b1bfe20d7d5444be217af78f94c5c43799cdf847c6ce69794b7bf3319c5364" - } - }, - "buildozer_linux_amd64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v7.3.1/buildozer-linux-amd64" - ], - "downloaded_file_path": "buildozer", - "executable": true, - "sha256": "3305e287b3fcc68b9a35fd8515ee617452cd4e018f9e6886b6c7cdbcba8710d4" - } - }, - "buildozer_linux_arm64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v7.3.1/buildozer-linux-arm64" - ], - "downloaded_file_path": "buildozer", - "executable": true, - "sha256": "0b5a2a717ac4fc911e1fec8d92af71dbb4fe95b10e5213da0cc3d56cea64a328" - } - }, - "buildozer_windows_amd64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v7.3.1/buildozer-windows-amd64.exe" - ], - "downloaded_file_path": "buildozer.exe", - "executable": true, - "sha256": "58d41ce53257c5594c9bc86d769f580909269f68de114297f46284fbb9023dcf" - } - }, - "buildifier_prebuilt_toolchains": { - "repoRuleId": "@@buildifier_prebuilt+//:defs.bzl%_buildifier_toolchain_setup", - "attributes": { - "assets_json": "[{\"arch\":\"amd64\",\"name\":\"buildifier\",\"platform\":\"darwin\",\"sha256\":\"375f823103d01620aaec20a0c29c6cbca99f4fd0725ae30b93655c6704f44d71\",\"version\":\"v7.3.1\"},{\"arch\":\"arm64\",\"name\":\"buildifier\",\"platform\":\"darwin\",\"sha256\":\"5a6afc6ac7a09f5455ba0b89bd99d5ae23b4174dc5dc9d6c0ed5ce8caac3f813\",\"version\":\"v7.3.1\"},{\"arch\":\"amd64\",\"name\":\"buildifier\",\"platform\":\"linux\",\"sha256\":\"5474cc5128a74e806783d54081f581662c4be8ae65022f557e9281ed5dc88009\",\"version\":\"v7.3.1\"},{\"arch\":\"arm64\",\"name\":\"buildifier\",\"platform\":\"linux\",\"sha256\":\"0bf86c4bfffaf4f08eed77bde5b2082e4ae5039a11e2e8b03984c173c34a561c\",\"version\":\"v7.3.1\"},{\"arch\":\"amd64\",\"name\":\"buildifier\",\"platform\":\"windows\",\"sha256\":\"370cd576075ad29930a82f5de132f1a1de4084c784a82514bd4da80c85acf4a8\",\"version\":\"v7.3.1\"},{\"arch\":\"amd64\",\"name\":\"buildozer\",\"platform\":\"darwin\",\"sha256\":\"854c9583efc166602276802658cef3f224d60898cfaa60630b33d328db3b0de2\",\"version\":\"v7.3.1\"},{\"arch\":\"arm64\",\"name\":\"buildozer\",\"platform\":\"darwin\",\"sha256\":\"31b1bfe20d7d5444be217af78f94c5c43799cdf847c6ce69794b7bf3319c5364\",\"version\":\"v7.3.1\"},{\"arch\":\"amd64\",\"name\":\"buildozer\",\"platform\":\"linux\",\"sha256\":\"3305e287b3fcc68b9a35fd8515ee617452cd4e018f9e6886b6c7cdbcba8710d4\",\"version\":\"v7.3.1\"},{\"arch\":\"arm64\",\"name\":\"buildozer\",\"platform\":\"linux\",\"sha256\":\"0b5a2a717ac4fc911e1fec8d92af71dbb4fe95b10e5213da0cc3d56cea64a328\",\"version\":\"v7.3.1\"},{\"arch\":\"amd64\",\"name\":\"buildozer\",\"platform\":\"windows\",\"sha256\":\"58d41ce53257c5594c9bc86d769f580909269f68de114297f46284fbb9023dcf\",\"version\":\"v7.3.1\"}]" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "buildifier_prebuilt+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "buildifier_prebuilt+", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@pybind11_bazel+//:internal_configure.bzl%internal_configure_extension": { - "general": { - "bzlTransitiveDigest": "BscKLx1uTqPejNWXrpBVNole5HrQ/iL/+SPa+29Shuk=", - "usagesDigest": "D1r3lfzMuUBFxgG8V6o0bQTLMk3GkaGOaPzw53wrwyw=", - "recordedFileInputs": { - "@@pybind11_bazel+//MODULE.bazel": "45d98c531db5f8430f847b195357dca36665d9c926cbbee77a85105fc6865d27" - }, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "pybind11": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "build_file": "@@pybind11_bazel+//:pybind11-BUILD.bazel", - "strip_prefix": "pybind11-2.12.0", - "urls": [ - "https://github.com/pybind/pybind11/archive/v2.12.0.zip" - ] - } - } - }, - "recordedRepoMappingEntries": [ - [ - "pybind11_bazel+", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { - "general": { - "bzlTransitiveDigest": "fmfKdvTpZCJJntCdqlB6bYFsD3ax+7qZpeR0cGMDe8A=", - "usagesDigest": "QI2z8ZUR+mqtbwsf2fLqYdJAkPOHdOV+tF2yVAUgRzw=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "com_github_jetbrains_kotlin_git": { - "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:compiler.bzl%kotlin_compiler_git_repository", - "attributes": { - "urls": [ - "https://github.com/JetBrains/kotlin/releases/download/v1.9.23/kotlin-compiler-1.9.23.zip" - ], - "sha256": "93137d3aab9afa9b27cb06a824c2324195c6b6f6179d8a8653f440f5bd58be88" - } - }, - "com_github_jetbrains_kotlin": { - "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:compiler.bzl%kotlin_capabilities_repository", - "attributes": { - "git_repository_name": "com_github_jetbrains_kotlin_git", - "compiler_version": "1.9.23" - } - }, - "com_github_google_ksp": { - "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:ksp.bzl%ksp_compiler_plugin_repository", - "attributes": { - "urls": [ - "https://github.com/google/ksp/releases/download/1.9.23-1.0.20/artifacts.zip" - ], - "sha256": "ee0618755913ef7fd6511288a232e8fad24838b9af6ea73972a76e81053c8c2d", - "strip_version": "1.9.23-1.0.20" - } - }, - "com_github_pinterest_ktlint": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "sha256": "01b2e0ef893383a50dbeb13970fe7fa3be36ca3e83259e01649945b09d736985", - "urls": [ - "https://github.com/pinterest/ktlint/releases/download/1.3.0/ktlint" - ], - "executable": true - } - }, - "rules_android": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "sha256": "cd06d15dd8bb59926e4d65f9003bfc20f9da4b2519985c27e190cddc8b7a7806", - "strip_prefix": "rules_android-0.1.1", - "urls": [ - "https://github.com/bazelbuild/rules_android/archive/v0.1.1.zip" - ] - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_kotlin+", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@rules_multitool+//multitool:extension.bzl%multitool": { - "general": { - "bzlTransitiveDigest": "r9TGCyer7o1g/TEqrdFBahcMZPkvD+oYDO4dUcGWYMs=", - "usagesDigest": "Bn6IoM2pzksz/oBIRwZyfLpBZBFrZnzat00Xcp/lWqw=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "multitool.linux_arm64": { - "repoRuleId": "@@rules_multitool+//multitool/private:multitool.bzl%_env_specific_tools", - "attributes": { - "lockfiles": [ - "@@rules_uv+//uv/private:uv.lock.json" - ], - "os": "linux", - "cpu": "arm64" - } - }, - "multitool.linux_x86_64": { - "repoRuleId": "@@rules_multitool+//multitool/private:multitool.bzl%_env_specific_tools", - "attributes": { - "lockfiles": [ - "@@rules_uv+//uv/private:uv.lock.json" - ], - "os": "linux", - "cpu": "x86_64" - } - }, - "multitool.macos_arm64": { - "repoRuleId": "@@rules_multitool+//multitool/private:multitool.bzl%_env_specific_tools", - "attributes": { - "lockfiles": [ - "@@rules_uv+//uv/private:uv.lock.json" - ], - "os": "macos", - "cpu": "arm64" - } - }, - "multitool.macos_x86_64": { - "repoRuleId": "@@rules_multitool+//multitool/private:multitool.bzl%_env_specific_tools", - "attributes": { - "lockfiles": [ - "@@rules_uv+//uv/private:uv.lock.json" - ], - "os": "macos", - "cpu": "x86_64" - } - }, - "multitool.windows_arm64": { - "repoRuleId": "@@rules_multitool+//multitool/private:multitool.bzl%_env_specific_tools", - "attributes": { - "lockfiles": [ - "@@rules_uv+//uv/private:uv.lock.json" - ], - "os": "windows", - "cpu": "arm64" - } - }, - "multitool.windows_x86_64": { - "repoRuleId": "@@rules_multitool+//multitool/private:multitool.bzl%_env_specific_tools", - "attributes": { - "lockfiles": [ - "@@rules_uv+//uv/private:uv.lock.json" - ], - "os": "windows", - "cpu": "x86_64" - } - }, - "multitool": { - "repoRuleId": "@@rules_multitool+//multitool/private:multitool.bzl%_multitool_hub", - "attributes": { - "lockfiles": [ - "@@rules_uv+//uv/private:uv.lock.json" - ] - } - } - }, - "recordedRepoMappingEntries": [ - [ - "bazel_features+", - "bazel_features_globals", - "bazel_features++version_extension+bazel_features_globals" - ], - [ - "bazel_features+", - "bazel_features_version", - "bazel_features++version_extension+bazel_features_version" - ], - [ - "rules_multitool+", - "bazel_features", - "bazel_features+" - ] - ] - } - }, - "@@rules_oci+//oci:extensions.bzl%oci": { - "general": { - "bzlTransitiveDigest": "P7xQtBwlrvlBaS61l0lPAfbIuxhc7X1Sr0sGXefXRAs=", - "usagesDigest": "O2LO3OVHqt//Iqq92QHpuV4tJyqlNa2Z+84PJp7hlwg=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "distroless_cc_debian12_linux_amd64": { - "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_pull", - "attributes": { - "www_authenticate_challenges": {}, - "scheme": "https", - "registry": "gcr.io", - "repository": "distroless/cc-debian12", - "identifier": "sha256:1850aee2ff72864350058d83d681c757d45c885986d15fcca7309b9e5c69f39a", - "platform": "linux/amd64", - "target_name": "distroless_cc_debian12_linux_amd64", - "bazel_tags": [] - } - }, - "distroless_cc_debian12": { - "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_alias", - "attributes": { - "target_name": "distroless_cc_debian12", - "www_authenticate_challenges": {}, - "scheme": "https", - "registry": "gcr.io", - "repository": "distroless/cc-debian12", - "identifier": "sha256:1850aee2ff72864350058d83d681c757d45c885986d15fcca7309b9e5c69f39a", - "platforms": { - "@@platforms//cpu:x86_64": "@distroless_cc_debian12_linux_amd64" - }, - "bzlmod_repository": "distroless_cc_debian12", - "reproducible": true - } - }, - "distroless_cc_debian12_debug_linux_amd64": { - "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_pull", - "attributes": { - "www_authenticate_challenges": {}, - "scheme": "https", - "registry": "gcr.io", - "repository": "distroless/cc-debian12", - "identifier": "sha256:ae6f470336acbf2aeffea3db70ec0e74d69bee7270cdb5fa2f28fe840fad57fe", - "platform": "linux/amd64", - "target_name": "distroless_cc_debian12_debug_linux_amd64", - "bazel_tags": [] - } - }, - "distroless_cc_debian12_debug": { - "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_alias", - "attributes": { - "target_name": "distroless_cc_debian12_debug", - "www_authenticate_challenges": {}, - "scheme": "https", - "registry": "gcr.io", - "repository": "distroless/cc-debian12", - "identifier": "sha256:ae6f470336acbf2aeffea3db70ec0e74d69bee7270cdb5fa2f28fe840fad57fe", - "platforms": { - "@@platforms//cpu:x86_64": "@distroless_cc_debian12_debug_linux_amd64" - }, - "bzlmod_repository": "distroless_cc_debian12_debug", - "reproducible": true - } - }, - "oci_crane_darwin_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "darwin_amd64", - "crane_version": "v0.18.0" - } - }, - "oci_crane_darwin_arm64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "darwin_arm64", - "crane_version": "v0.18.0" - } - }, - "oci_crane_linux_arm64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "linux_arm64", - "crane_version": "v0.18.0" - } - }, - "oci_crane_linux_armv6": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "linux_armv6", - "crane_version": "v0.18.0" - } - }, - "oci_crane_linux_i386": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "linux_i386", - "crane_version": "v0.18.0" - } - }, - "oci_crane_linux_s390x": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "linux_s390x", - "crane_version": "v0.18.0" - } - }, - "oci_crane_linux_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "linux_amd64", - "crane_version": "v0.18.0" - } - }, - "oci_crane_windows_armv6": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "windows_armv6", - "crane_version": "v0.18.0" - } - }, - "oci_crane_windows_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "windows_amd64", - "crane_version": "v0.18.0" - } - }, - "oci_crane_toolchains": { - "repoRuleId": "@@rules_oci+//oci/private:toolchains_repo.bzl%toolchains_repo", - "attributes": { - "toolchain_type": "@rules_oci//oci:crane_toolchain_type", - "toolchain": "@oci_crane_{platform}//:crane_toolchain" - } - }, - "oci_regctl_darwin_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "darwin_amd64" - } - }, - "oci_regctl_darwin_arm64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "darwin_arm64" - } - }, - "oci_regctl_linux_arm64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "linux_arm64" - } - }, - "oci_regctl_linux_s390x": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "linux_s390x" - } - }, - "oci_regctl_linux_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "linux_amd64" - } - }, - "oci_regctl_windows_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "windows_amd64" - } - }, - "oci_regctl_toolchains": { - "repoRuleId": "@@rules_oci+//oci/private:toolchains_repo.bzl%toolchains_repo", - "attributes": { - "toolchain_type": "@rules_oci//oci:regctl_toolchain_type", - "toolchain": "@oci_regctl_{platform}//:regctl_toolchain" - } - } - }, - "moduleExtensionMetadata": { - "explicitRootModuleDirectDeps": [ - "distroless_cc_debian12", - "distroless_cc_debian12_linux_amd64", - "distroless_cc_debian12_debug", - "distroless_cc_debian12_debug_linux_amd64" - ], - "explicitRootModuleDirectDevDeps": [], - "useAllRepos": "NO", - "reproducible": false - }, - "recordedRepoMappingEntries": [ - [ - "aspect_bazel_lib+", - "bazel_tools", - "bazel_tools" - ], - [ - "bazel_features+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_oci+", - "aspect_bazel_lib", - "aspect_bazel_lib+" - ], - [ - "rules_oci+", - "bazel_features", - "bazel_features+" - ], - [ - "rules_oci+", - "bazel_skylib", - "bazel_skylib+" - ] - ] - } - }, - "@@rules_python+//python/extensions:pip.bzl%pip": { - "general": { - "bzlTransitiveDigest": "fOwLlf5Rfxv4xkFCZwPlkSPUuZ9lAhMGf0SN43pc0xg=", - "usagesDigest": "+J++nGlVyZlgzi9ejqCZML6OBtqR8IvEcE8T0yFIFKA=", - "recordedFileInputs": { - "@@protobuf+//python/requirements.txt": "2bb341a0ba317fb8dc9ac79817b9b2d8f6a4d0c43e3fb652f0cab584eed420a4", - "@@rules_fuzzing+//fuzzing/requirements.txt": "d4e91a1b3eca06d6e2877f67d3d8be498cb3ed6d28b9c14439b6f210ec621470", - "@@rules_python+//tools/publish/requirements_darwin.txt": "6d10732737e5504291360b78fbf2b82abe39a5a021bd937f987f8433519b0f2c", - "@@rules_python+//tools/publish/requirements_linux.txt": "cb63e4751377d1722a81fd5835b0526208620f1303feb473ed0dd45ae4d71ba9", - "@@rules_python+//tools/publish/requirements_windows.txt": "f92da1a252df04e1931a0e16dfbc5a3c165a85b796b7fc5e56ee4e3846d82558", - "@@zml+//runtimes/neuron/requirements.lock.txt": "fa0b5b967162e303415ee1dd7d2c1a448068935663ba5ac5b5f72a515fe3f6f6", - "@@zml+//tools/hf_requirements.lock.txt": "ffbcfcfd01c9a31db823affa4ab6122c509b2f1eeb4b853325d1dbd5cc9568da" - }, - "recordedDirentsInputs": {}, - "envVariables": { - "RULES_PYTHON_REPO_DEBUG": null, - "RULES_PYTHON_REPO_DEBUG_VERBOSITY": null - }, - "generatedRepoSpecs": { - "huggingface_hub_311_certifi": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "certifi==2025.7.14 --hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 --hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995" - } - }, - "huggingface_hub_311_charset_normalizer": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "charset-normalizer==3.4.2 --hash=sha256:005fa3432484527f9732ebd315da8da8001593e2cf46a3d817669f062c3d9ed4 --hash=sha256:046595208aae0120559a67693ecc65dd75d46f7bf687f159127046628178dc45 --hash=sha256:0c29de6a1a95f24b9a1aa7aefd27d2487263f00dfd55a77719b530788f75cff7 --hash=sha256:0c8c57f84ccfc871a48a47321cfa49ae1df56cd1d965a09abe84066f6853b9c0 --hash=sha256:0f5d9ed7f254402c9e7d35d2f5972c9bbea9040e99cd2861bd77dc68263277c7 --hash=sha256:18dd2e350387c87dabe711b86f83c9c78af772c748904d372ade190b5c7c9d4d --hash=sha256:1b1bde144d98e446b056ef98e59c256e9294f6b74d7af6846bf5ffdafd687a7d --hash=sha256:1c95a1e2902a8b722868587c0e1184ad5c55631de5afc0eb96bc4b0d738092c0 --hash=sha256:1cad5f45b3146325bb38d6855642f6fd609c3f7cad4dbaf75549bf3b904d3184 --hash=sha256:21b2899062867b0e1fde9b724f8aecb1af14f2778d69aacd1a5a1853a597a5db --hash=sha256:24498ba8ed6c2e0b56d4acbf83f2d989720a93b41d712ebd4f4979660db4417b --hash=sha256:25a23ea5c7edc53e0f29bae2c44fcb5a1aa10591aae107f2a2b2583a9c5cbc64 --hash=sha256:289200a18fa698949d2b39c671c2cc7a24d44096784e76614899a7ccf2574b7b --hash=sha256:28a1005facc94196e1fb3e82a3d442a9d9110b8434fc1ded7a24a2983c9888d8 --hash=sha256:32fc0341d72e0f73f80acb0a2c94216bd704f4f0bce10aedea38f30502b271ff --hash=sha256:36b31da18b8890a76ec181c3cf44326bf2c48e36d393ca1b72b3f484113ea344 --hash=sha256:3c21d4fca343c805a52c0c78edc01e3477f6dd1ad7c47653241cf2a206d4fc58 --hash=sha256:3fddb7e2c84ac87ac3a947cb4e66d143ca5863ef48e4a5ecb83bd48619e4634e --hash=sha256:43e0933a0eff183ee85833f341ec567c0980dae57c464d8a508e1b2ceb336471 --hash=sha256:4a476b06fbcf359ad25d34a057b7219281286ae2477cc5ff5e3f70a246971148 --hash=sha256:4e594135de17ab3866138f496755f302b72157d115086d100c3f19370839dd3a --hash=sha256:50bf98d5e563b83cc29471fa114366e6806bc06bc7a25fd59641e41445327836 --hash=sha256:5a9979887252a82fefd3d3ed2a8e3b937a7a809f65dcb1e068b090e165bbe99e --hash=sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63 --hash=sha256:5bf4545e3b962767e5c06fe1738f951f77d27967cb2caa64c28be7c4563e162c --hash=sha256:6333b3aa5a12c26b2a4d4e7335a28f1475e0e5e17d69d55141ee3cab736f66d1 --hash=sha256:65c981bdbd3f57670af8b59777cbfae75364b483fa8a9f420f08094531d54a01 --hash=sha256:68a328e5f55ec37c57f19ebb1fdc56a248db2e3e9ad769919a58672958e8f366 --hash=sha256:6a0289e4589e8bdfef02a80478f1dfcb14f0ab696b5a00e1f4b8a14a307a3c58 --hash=sha256:6b66f92b17849b85cad91259efc341dce9c1af48e2173bf38a85c6329f1033e5 --hash=sha256:6c9379d65defcab82d07b2a9dfbfc2e95bc8fe0ebb1b176a3190230a3ef0e07c --hash=sha256:6fc1f5b51fa4cecaa18f2bd7a003f3dd039dd615cd69a2afd6d3b19aed6775f2 --hash=sha256:70f7172939fdf8790425ba31915bfbe8335030f05b9913d7ae00a87d4395620a --hash=sha256:721c76e84fe669be19c5791da68232ca2e05ba5185575086e384352e2c309597 --hash=sha256:7222ffd5e4de8e57e03ce2cef95a4c43c98fcb72ad86909abdfc2c17d227fc1b --hash=sha256:75d10d37a47afee94919c4fab4c22b9bc2a8bf7d4f46f87363bcf0573f3ff4f5 --hash=sha256:76af085e67e56c8816c3ccf256ebd136def2ed9654525348cfa744b6802b69eb --hash=sha256:770cab594ecf99ae64c236bc9ee3439c3f46be49796e265ce0cc8bc17b10294f --hash=sha256:7a6ab32f7210554a96cd9e33abe3ddd86732beeafc7a28e9955cdf22ffadbab0 --hash=sha256:7c48ed483eb946e6c04ccbe02c6b4d1d48e51944b6db70f697e089c193404941 --hash=sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0 --hash=sha256:8075c35cd58273fee266c58c0c9b670947c19df5fb98e7b66710e04ad4e9ff86 --hash=sha256:8272b73e1c5603666618805fe821edba66892e2870058c94c53147602eab29c7 --hash=sha256:82d8fd25b7f4675d0c47cf95b594d4e7b158aca33b76aa63d07186e13c0e0ab7 --hash=sha256:844da2b5728b5ce0e32d863af26f32b5ce61bc4273a9c720a9f3aa9df73b1455 --hash=sha256:8755483f3c00d6c9a77f490c17e6ab0c8729e39e6390328e42521ef175380ae6 --hash=sha256:915f3849a011c1f593ab99092f3cecfcb4d65d8feb4a64cf1bf2d22074dc0ec4 --hash=sha256:926ca93accd5d36ccdabd803392ddc3e03e6d4cd1cf17deff3b989ab8e9dbcf0 --hash=sha256:982bb1e8b4ffda883b3d0a521e23abcd6fd17418f6d2c4118d257a10199c0ce3 --hash=sha256:98f862da73774290f251b9df8d11161b6cf25b599a66baf087c1ffe340e9bfd1 --hash=sha256:9cbfacf36cb0ec2897ce0ebc5d08ca44213af24265bd56eca54bee7923c48fd6 --hash=sha256:a370b3e078e418187da8c3674eddb9d983ec09445c99a3a263c2011993522981 --hash=sha256:a955b438e62efdf7e0b7b52a64dc5c3396e2634baa62471768a64bc2adb73d5c --hash=sha256:aa6af9e7d59f9c12b33ae4e9450619cf2488e2bbe9b44030905877f0b2324980 --hash=sha256:aa88ca0b1932e93f2d961bf3addbb2db902198dca337d88c89e1559e066e7645 --hash=sha256:aaeeb6a479c7667fbe1099af9617c83aaca22182d6cf8c53966491a0f1b7ffb7 --hash=sha256:aaf27faa992bfee0264dc1f03f4c75e9fcdda66a519db6b957a3f826e285cf12 --hash=sha256:b2680962a4848b3c4f155dc2ee64505a9c57186d0d56b43123b17ca3de18f0fa --hash=sha256:b2d318c11350e10662026ad0eb71bb51c7812fc8590825304ae0bdd4ac283acd --hash=sha256:b33de11b92e9f75a2b545d6e9b6f37e398d86c3e9e9653c4864eb7e89c5773ef --hash=sha256:b3daeac64d5b371dea99714f08ffc2c208522ec6b06fbc7866a450dd446f5c0f --hash=sha256:be1e352acbe3c78727a16a455126d9ff83ea2dfdcbc83148d2982305a04714c2 --hash=sha256:bee093bf902e1d8fc0ac143c88902c3dfc8941f7ea1d6a8dd2bcb786d33db03d --hash=sha256:c72fbbe68c6f32f251bdc08b8611c7b3060612236e960ef848e0a517ddbe76c5 --hash=sha256:c9e36a97bee9b86ef9a1cf7bb96747eb7a15c2f22bdb5b516434b00f2a599f02 --hash=sha256:cddf7bd982eaa998934a91f69d182aec997c6c468898efe6679af88283b498d3 --hash=sha256:cf713fe9a71ef6fd5adf7a79670135081cd4431c2943864757f0fa3a65b1fafd --hash=sha256:d11b54acf878eef558599658b0ffca78138c8c3655cf4f3a4a673c437e67732e --hash=sha256:d41c4d287cfc69060fa91cae9683eacffad989f1a10811995fa309df656ec214 --hash=sha256:d524ba3f1581b35c03cb42beebab4a13e6cdad7b36246bd22541fa585a56cccd --hash=sha256:daac4765328a919a805fa5e2720f3e94767abd632ae410a9062dff5412bae65a --hash=sha256:db4c7bf0e07fc3b7d89ac2a5880a6a8062056801b83ff56d8464b70f65482b6c --hash=sha256:dc7039885fa1baf9be153a0626e337aa7ec8bf96b0128605fb0d77788ddc1681 --hash=sha256:dccab8d5fa1ef9bfba0590ecf4d46df048d18ffe3eec01eeb73a42e0d9e7a8ba --hash=sha256:dedb8adb91d11846ee08bec4c8236c8549ac721c245678282dcb06b221aab59f --hash=sha256:e45ba65510e2647721e35323d6ef54c7974959f6081b58d4ef5d87c60c84919a --hash=sha256:e53efc7c7cee4c1e70661e2e112ca46a575f90ed9ae3fef200f2a25e954f4b28 --hash=sha256:e635b87f01ebc977342e2697d05b56632f5f879a4f15955dfe8cef2448b51691 --hash=sha256:e70e990b2137b29dc5564715de1e12701815dacc1d056308e2b17e9095372a82 --hash=sha256:e8082b26888e2f8b36a042a58307d5b917ef2b1cacab921ad3323ef91901c71a --hash=sha256:e8323a9b031aa0393768b87f04b4164a40037fb2a3c11ac06a03ffecd3618027 --hash=sha256:e92fca20c46e9f5e1bb485887d074918b13543b1c2a1185e69bb8d17ab6236a7 --hash=sha256:eb30abc20df9ab0814b5a2524f23d75dcf83cde762c161917a2b4b7b55b1e518 --hash=sha256:eba9904b0f38a143592d9fc0e19e2df0fa2e41c3c3745554761c5f6447eedabf --hash=sha256:ef8de666d6179b009dce7bcb2ad4c4a779f113f12caf8dc77f0162c29d20490b --hash=sha256:efd387a49825780ff861998cd959767800d54f8308936b21025326de4b5a42b9 --hash=sha256:f0aa37f3c979cf2546b73e8222bbfa3dc07a641585340179d768068e3455e544 --hash=sha256:f4074c5a429281bf056ddd4c5d3b740ebca4d43ffffe2ef4bf4d2d05114299da --hash=sha256:f69a27e45c43520f5487f27627059b64aaf160415589230992cec34c5e18a509 --hash=sha256:fb707f3e15060adf5b7ada797624a6c6e0138e2a26baa089df64c68ee98e040f --hash=sha256:fcbe676a55d7445b22c10967bceaaf0ee69407fbe0ece4d032b6eb8d4565982a --hash=sha256:fdb20a30fe1175ecabed17cbf7812f7b804b8a315a25f24678bcdf120a90077f" - } - }, - "huggingface_hub_311_filelock": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "filelock==3.18.0 --hash=sha256:adbc88eabb99d2fec8c9c1b229b171f18afa655400173ddc653d5d01501fb9f2 --hash=sha256:c401f4f8377c4464e6db25fff06205fd89bdd83b65eb0488ed1b160f780e21de" - } - }, - "huggingface_hub_311_fsspec": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "fsspec==2025.7.0 --hash=sha256:786120687ffa54b8283d942929540d8bc5ccfa820deb555a2b5d0ed2b737bf58 --hash=sha256:8b012e39f63c7d5f10474de957f3ab793b47b45ae7d39f2fb735f8bbe25c0e21" - } - }, - "huggingface_hub_311_hf_xet": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "hf-xet==1.1.5 --hash=sha256:69ebbcfd9ec44fdc2af73441619eeb06b94ee34511bbcf57cd423820090f5694 --hash=sha256:73e167d9807d166596b4b2f0b585c6d5bd84a26dea32843665a8b58f6edba245 --hash=sha256:83088ecea236d5113de478acb2339f92c95b4fb0462acaa30621fac02f5a534a --hash=sha256:9fa6e3ee5d61912c4a113e0708eaaef987047616465ac7aa30f7121a48fc1af8 --hash=sha256:ab34c4c3104133c495785d5d8bba3b1efc99de52c02e759cf711a91fd39d3a14 --hash=sha256:dbba1660e5d810bd0ea77c511a99e9242d920790d0e63c0e4673ed36c4022d18 --hash=sha256:f52c2fa3635b8c37c7764d8796dfa72706cc4eded19d638331161e82b0792e23 --hash=sha256:fc874b5c843e642f45fd85cda1ce599e123308ad2901ead23d3510a47ff506d1" - } - }, - "huggingface_hub_311_huggingface_hub": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "huggingface-hub==0.34.3 --hash=sha256:5444550099e2d86e68b2898b09e85878fbd788fc2957b506c6a79ce060e39492 --hash=sha256:d58130fd5aa7408480681475491c0abd7e835442082fbc3ef4d45b6c39f83853" - } - }, - "huggingface_hub_311_idna": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "idna==3.10 --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3" - } - }, - "huggingface_hub_311_packaging": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "packaging==25.0 --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f" - } - }, - "huggingface_hub_311_pyyaml": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "pyyaml==6.0.2 --hash=sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff --hash=sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48 --hash=sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086 --hash=sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e --hash=sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133 --hash=sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5 --hash=sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484 --hash=sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee --hash=sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5 --hash=sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68 --hash=sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a --hash=sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf --hash=sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99 --hash=sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8 --hash=sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85 --hash=sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19 --hash=sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc --hash=sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a --hash=sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1 --hash=sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317 --hash=sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c --hash=sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631 --hash=sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d --hash=sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652 --hash=sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5 --hash=sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e --hash=sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b --hash=sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8 --hash=sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476 --hash=sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706 --hash=sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563 --hash=sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237 --hash=sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b --hash=sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083 --hash=sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180 --hash=sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425 --hash=sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e --hash=sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f --hash=sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725 --hash=sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183 --hash=sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab --hash=sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774 --hash=sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725 --hash=sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e --hash=sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5 --hash=sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d --hash=sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290 --hash=sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44 --hash=sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed --hash=sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4 --hash=sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba --hash=sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12 --hash=sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4" - } - }, - "huggingface_hub_311_requests": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "requests==2.32.4 --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422" - } - }, - "huggingface_hub_311_tqdm": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "tqdm==4.67.1 --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2" - } - }, - "huggingface_hub_311_typing_extensions": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "typing-extensions==4.14.1 --hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 --hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76" - } - }, - "huggingface_hub_311_urllib3": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@huggingface_hub//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "huggingface_hub_311", - "requirement": "urllib3==2.5.0 --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc" - } - }, - "neuron_py_deps_311_boto3": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "boto3==1.39.9 --hash=sha256:5bc85e9fdec4e21ef5ca2c22b4d51a3e32b53f3da36ce51f5a3ea4dbde07b132 --hash=sha256:e3d3a6b617e1575e7ec854c820a882ab2e189a0421e74dc0dca2c9e13d4370a5" - } - }, - "neuron_py_deps_311_botocore": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "botocore==1.39.9 --hash=sha256:02f141c2849e4589a79feea245ce4ecc478d48b7865572445af8aae3b041772d --hash=sha256:a9691cbe03a3bc8b2720b3c36e5c5a2eecace6acd72bfb1107f00e75edaec4f3" - } - }, - "neuron_py_deps_311_certifi": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "certifi==2025.7.14 --hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 --hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995" - } - }, - "neuron_py_deps_311_charset_normalizer": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "charset-normalizer==3.4.2 --hash=sha256:005fa3432484527f9732ebd315da8da8001593e2cf46a3d817669f062c3d9ed4 --hash=sha256:046595208aae0120559a67693ecc65dd75d46f7bf687f159127046628178dc45 --hash=sha256:0c29de6a1a95f24b9a1aa7aefd27d2487263f00dfd55a77719b530788f75cff7 --hash=sha256:0c8c57f84ccfc871a48a47321cfa49ae1df56cd1d965a09abe84066f6853b9c0 --hash=sha256:0f5d9ed7f254402c9e7d35d2f5972c9bbea9040e99cd2861bd77dc68263277c7 --hash=sha256:18dd2e350387c87dabe711b86f83c9c78af772c748904d372ade190b5c7c9d4d --hash=sha256:1b1bde144d98e446b056ef98e59c256e9294f6b74d7af6846bf5ffdafd687a7d --hash=sha256:1c95a1e2902a8b722868587c0e1184ad5c55631de5afc0eb96bc4b0d738092c0 --hash=sha256:1cad5f45b3146325bb38d6855642f6fd609c3f7cad4dbaf75549bf3b904d3184 --hash=sha256:21b2899062867b0e1fde9b724f8aecb1af14f2778d69aacd1a5a1853a597a5db --hash=sha256:24498ba8ed6c2e0b56d4acbf83f2d989720a93b41d712ebd4f4979660db4417b --hash=sha256:25a23ea5c7edc53e0f29bae2c44fcb5a1aa10591aae107f2a2b2583a9c5cbc64 --hash=sha256:289200a18fa698949d2b39c671c2cc7a24d44096784e76614899a7ccf2574b7b --hash=sha256:28a1005facc94196e1fb3e82a3d442a9d9110b8434fc1ded7a24a2983c9888d8 --hash=sha256:32fc0341d72e0f73f80acb0a2c94216bd704f4f0bce10aedea38f30502b271ff --hash=sha256:36b31da18b8890a76ec181c3cf44326bf2c48e36d393ca1b72b3f484113ea344 --hash=sha256:3c21d4fca343c805a52c0c78edc01e3477f6dd1ad7c47653241cf2a206d4fc58 --hash=sha256:3fddb7e2c84ac87ac3a947cb4e66d143ca5863ef48e4a5ecb83bd48619e4634e --hash=sha256:43e0933a0eff183ee85833f341ec567c0980dae57c464d8a508e1b2ceb336471 --hash=sha256:4a476b06fbcf359ad25d34a057b7219281286ae2477cc5ff5e3f70a246971148 --hash=sha256:4e594135de17ab3866138f496755f302b72157d115086d100c3f19370839dd3a --hash=sha256:50bf98d5e563b83cc29471fa114366e6806bc06bc7a25fd59641e41445327836 --hash=sha256:5a9979887252a82fefd3d3ed2a8e3b937a7a809f65dcb1e068b090e165bbe99e --hash=sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63 --hash=sha256:5bf4545e3b962767e5c06fe1738f951f77d27967cb2caa64c28be7c4563e162c --hash=sha256:6333b3aa5a12c26b2a4d4e7335a28f1475e0e5e17d69d55141ee3cab736f66d1 --hash=sha256:65c981bdbd3f57670af8b59777cbfae75364b483fa8a9f420f08094531d54a01 --hash=sha256:68a328e5f55ec37c57f19ebb1fdc56a248db2e3e9ad769919a58672958e8f366 --hash=sha256:6a0289e4589e8bdfef02a80478f1dfcb14f0ab696b5a00e1f4b8a14a307a3c58 --hash=sha256:6b66f92b17849b85cad91259efc341dce9c1af48e2173bf38a85c6329f1033e5 --hash=sha256:6c9379d65defcab82d07b2a9dfbfc2e95bc8fe0ebb1b176a3190230a3ef0e07c --hash=sha256:6fc1f5b51fa4cecaa18f2bd7a003f3dd039dd615cd69a2afd6d3b19aed6775f2 --hash=sha256:70f7172939fdf8790425ba31915bfbe8335030f05b9913d7ae00a87d4395620a --hash=sha256:721c76e84fe669be19c5791da68232ca2e05ba5185575086e384352e2c309597 --hash=sha256:7222ffd5e4de8e57e03ce2cef95a4c43c98fcb72ad86909abdfc2c17d227fc1b --hash=sha256:75d10d37a47afee94919c4fab4c22b9bc2a8bf7d4f46f87363bcf0573f3ff4f5 --hash=sha256:76af085e67e56c8816c3ccf256ebd136def2ed9654525348cfa744b6802b69eb --hash=sha256:770cab594ecf99ae64c236bc9ee3439c3f46be49796e265ce0cc8bc17b10294f --hash=sha256:7a6ab32f7210554a96cd9e33abe3ddd86732beeafc7a28e9955cdf22ffadbab0 --hash=sha256:7c48ed483eb946e6c04ccbe02c6b4d1d48e51944b6db70f697e089c193404941 --hash=sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0 --hash=sha256:8075c35cd58273fee266c58c0c9b670947c19df5fb98e7b66710e04ad4e9ff86 --hash=sha256:8272b73e1c5603666618805fe821edba66892e2870058c94c53147602eab29c7 --hash=sha256:82d8fd25b7f4675d0c47cf95b594d4e7b158aca33b76aa63d07186e13c0e0ab7 --hash=sha256:844da2b5728b5ce0e32d863af26f32b5ce61bc4273a9c720a9f3aa9df73b1455 --hash=sha256:8755483f3c00d6c9a77f490c17e6ab0c8729e39e6390328e42521ef175380ae6 --hash=sha256:915f3849a011c1f593ab99092f3cecfcb4d65d8feb4a64cf1bf2d22074dc0ec4 --hash=sha256:926ca93accd5d36ccdabd803392ddc3e03e6d4cd1cf17deff3b989ab8e9dbcf0 --hash=sha256:982bb1e8b4ffda883b3d0a521e23abcd6fd17418f6d2c4118d257a10199c0ce3 --hash=sha256:98f862da73774290f251b9df8d11161b6cf25b599a66baf087c1ffe340e9bfd1 --hash=sha256:9cbfacf36cb0ec2897ce0ebc5d08ca44213af24265bd56eca54bee7923c48fd6 --hash=sha256:a370b3e078e418187da8c3674eddb9d983ec09445c99a3a263c2011993522981 --hash=sha256:a955b438e62efdf7e0b7b52a64dc5c3396e2634baa62471768a64bc2adb73d5c --hash=sha256:aa6af9e7d59f9c12b33ae4e9450619cf2488e2bbe9b44030905877f0b2324980 --hash=sha256:aa88ca0b1932e93f2d961bf3addbb2db902198dca337d88c89e1559e066e7645 --hash=sha256:aaeeb6a479c7667fbe1099af9617c83aaca22182d6cf8c53966491a0f1b7ffb7 --hash=sha256:aaf27faa992bfee0264dc1f03f4c75e9fcdda66a519db6b957a3f826e285cf12 --hash=sha256:b2680962a4848b3c4f155dc2ee64505a9c57186d0d56b43123b17ca3de18f0fa --hash=sha256:b2d318c11350e10662026ad0eb71bb51c7812fc8590825304ae0bdd4ac283acd --hash=sha256:b33de11b92e9f75a2b545d6e9b6f37e398d86c3e9e9653c4864eb7e89c5773ef --hash=sha256:b3daeac64d5b371dea99714f08ffc2c208522ec6b06fbc7866a450dd446f5c0f --hash=sha256:be1e352acbe3c78727a16a455126d9ff83ea2dfdcbc83148d2982305a04714c2 --hash=sha256:bee093bf902e1d8fc0ac143c88902c3dfc8941f7ea1d6a8dd2bcb786d33db03d --hash=sha256:c72fbbe68c6f32f251bdc08b8611c7b3060612236e960ef848e0a517ddbe76c5 --hash=sha256:c9e36a97bee9b86ef9a1cf7bb96747eb7a15c2f22bdb5b516434b00f2a599f02 --hash=sha256:cddf7bd982eaa998934a91f69d182aec997c6c468898efe6679af88283b498d3 --hash=sha256:cf713fe9a71ef6fd5adf7a79670135081cd4431c2943864757f0fa3a65b1fafd --hash=sha256:d11b54acf878eef558599658b0ffca78138c8c3655cf4f3a4a673c437e67732e --hash=sha256:d41c4d287cfc69060fa91cae9683eacffad989f1a10811995fa309df656ec214 --hash=sha256:d524ba3f1581b35c03cb42beebab4a13e6cdad7b36246bd22541fa585a56cccd --hash=sha256:daac4765328a919a805fa5e2720f3e94767abd632ae410a9062dff5412bae65a --hash=sha256:db4c7bf0e07fc3b7d89ac2a5880a6a8062056801b83ff56d8464b70f65482b6c --hash=sha256:dc7039885fa1baf9be153a0626e337aa7ec8bf96b0128605fb0d77788ddc1681 --hash=sha256:dccab8d5fa1ef9bfba0590ecf4d46df048d18ffe3eec01eeb73a42e0d9e7a8ba --hash=sha256:dedb8adb91d11846ee08bec4c8236c8549ac721c245678282dcb06b221aab59f --hash=sha256:e45ba65510e2647721e35323d6ef54c7974959f6081b58d4ef5d87c60c84919a --hash=sha256:e53efc7c7cee4c1e70661e2e112ca46a575f90ed9ae3fef200f2a25e954f4b28 --hash=sha256:e635b87f01ebc977342e2697d05b56632f5f879a4f15955dfe8cef2448b51691 --hash=sha256:e70e990b2137b29dc5564715de1e12701815dacc1d056308e2b17e9095372a82 --hash=sha256:e8082b26888e2f8b36a042a58307d5b917ef2b1cacab921ad3323ef91901c71a --hash=sha256:e8323a9b031aa0393768b87f04b4164a40037fb2a3c11ac06a03ffecd3618027 --hash=sha256:e92fca20c46e9f5e1bb485887d074918b13543b1c2a1185e69bb8d17ab6236a7 --hash=sha256:eb30abc20df9ab0814b5a2524f23d75dcf83cde762c161917a2b4b7b55b1e518 --hash=sha256:eba9904b0f38a143592d9fc0e19e2df0fa2e41c3c3745554761c5f6447eedabf --hash=sha256:ef8de666d6179b009dce7bcb2ad4c4a779f113f12caf8dc77f0162c29d20490b --hash=sha256:efd387a49825780ff861998cd959767800d54f8308936b21025326de4b5a42b9 --hash=sha256:f0aa37f3c979cf2546b73e8222bbfa3dc07a641585340179d768068e3455e544 --hash=sha256:f4074c5a429281bf056ddd4c5d3b740ebca4d43ffffe2ef4bf4d2d05114299da --hash=sha256:f69a27e45c43520f5487f27627059b64aaf160415589230992cec34c5e18a509 --hash=sha256:fb707f3e15060adf5b7ada797624a6c6e0138e2a26baa089df64c68ee98e040f --hash=sha256:fcbe676a55d7445b22c10967bceaaf0ee69407fbe0ece4d032b6eb8d4565982a --hash=sha256:fdb20a30fe1175ecabed17cbf7812f7b804b8a315a25f24678bcdf120a90077f" - } - }, - "neuron_py_deps_311_ec2_metadata": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "ec2-metadata==2.14.0 --hash=sha256:b2f83381722072efa28f65dc37e726c0abd3a0532f227b3fa6a2ad68460c7feb --hash=sha256:fc4f9e893da1c049761221ec3d2208cce4872953b8960e1ed1f7120ff4ca3512" - } - }, - "neuron_py_deps_311_idna": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "idna==3.10 --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3" - } - }, - "neuron_py_deps_311_islpy": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "islpy==2023.2.5 --hash=sha256:06446395028b88701fd84b685b1c82b9de493728906f96f576a4dca69d12dd1f --hash=sha256:0c1689d9ef2ccfb3fcb3108a37c8fa21561dac3615ec05508021628a3197d81c --hash=sha256:0f723e6d8fda7608e6329ee9a786b2812afefec01f6f45e4ac812594608d2990 --hash=sha256:20484f2ef8d8f5827e18eaac064bfbd4d0e9cb877030ee1297a544bc80e126db --hash=sha256:21571052bac1a319ef4937886559a41c85eddfc29389c7f9be6d8ddaa08dbd4a --hash=sha256:24b469e1e390860cb78a13e5a933727fed42fd991eb6dbb758754410224a28a2 --hash=sha256:2a64abecd9a966e6223633315e20b3abd9eb1faa8887122c675a35a7f4885212 --hash=sha256:31c48b641824bd6b50e97659a6842c445feeb359115148c3debcbb862cb85c1f --hash=sha256:3ee7815b1d49ac1efca0a42a5c038f88e87be01b9ddf598aadea0ef84b930a46 --hash=sha256:3f50ec187bbd61cef4400af6885e65afd4092e3c4dc84b5d5c98554de6d06988 --hash=sha256:4e97e23ba815afa37b8919ebb0d1c7ec7a9cb2c1491ea7ba89a886a8d213169c --hash=sha256:58b5b86a9f5f58d0e8165c7e2215e7441eeea3466277e44f24788aef8b2b8311 --hash=sha256:662a1c6f83feb77397eaef0027c5a57b98e249ecc454df6131c8cfe1541964dc --hash=sha256:679431e17ad40d3c496d5fd3e9cbb824047775c00ade143dd59f848110b287f7 --hash=sha256:69a13fa79fe9e901376e2c75bc49770e9027c55d9699077f8417e5bbe9e8b137 --hash=sha256:6a336af9106eb92c3b7babe919d9e1e0a629143941d32d474e47cc55cf64f647 --hash=sha256:6ed6c49803e53db5b0dbba9ec2fd3f896a8b9ea68f4a1cd203e5e7490f8e1c8e --hash=sha256:773e85ba8768e14bd2b813d4a7e8e8140fe628cddddbd4fcb3452b023ed57f84 --hash=sha256:7acbb16a6f55a6020cfcbd5823bbd3dd36ca03ef0713f9f7a47404b0aad7cba4 --hash=sha256:80674c7a9aef8d2657872264dc59aeb66ce92067603ead43be51a56a6efc058a --hash=sha256:819b16c4add84b42f227b451c2f773b8ae06489313d482771fc54c8806c1a2af --hash=sha256:831b7a8f290af99e24addcb5cc5f4e73f7d047d50202ad84c7d208cadbe24f15 --hash=sha256:a66c7baf86d6fcef8e2170483f45a241dca0811e0c78cc71efa30bc74bfc6bc8 --hash=sha256:b27f1aa0ec0ab8182e47aaab4e363a42221ae00f9cba1f7cdde4d687120965f3 --hash=sha256:b3582a7d19ee47f29c8c6b40643f69e5a52c0f0489ed179408cfd4d9616210c9 --hash=sha256:b45c69990d0f7a9fbbba4a28b552c3b93c80a75d17b52032851f2dde13ebd661 --hash=sha256:c0d67b1b8644ac8cf5dee3b2efbf8029891f5db1a625ee5259e2fcf6c33a2d55 --hash=sha256:c2fdf13ae6465c511d75b42ed18b2289b49ecc7f1764e7fa24340b05d0392e80 --hash=sha256:c6665d90beb3f37b2dcb95570bf8700e58884c5ad03dd131568287a639c03a5e --hash=sha256:cea357471c345e804fa31c9f1eacfb3f67eda26b3e8fd417c33556822076565b --hash=sha256:d35750ea2a3964325a2e67196806cf02ed771c2d2b2d47fb2bf60f58bf933cc4 --hash=sha256:d5bedd79939dac4403b573abe5e13f6fc6a4502ebe8ed7de79b2fbfdd7856f3f --hash=sha256:d75b4521f3291b92fc6796ebb7eabf5c6e318d44a578390f678d7bfaedde13b8 --hash=sha256:d77acfe6653d044d14b8e93482e0282f42b947d40f5a98a13ee6b705936aaa18 --hash=sha256:dd74398baf36938abb7c2a9d9828cc1a2e549c6c80485cec8b06b29abfaad18f --hash=sha256:e33812a9922d967da814b17bc778a2c0dd1dabcd7ff685d86677bd745e8a3d65 --hash=sha256:e372461fb5ba22f9955cdf180d6187ac691f0ffa972874af181bf60501c738cd --hash=sha256:ea705d8b5244409fdc32affc689b010e88efbc8d378072bcba8edcf51c797de7 --hash=sha256:ed0c62e64d160735ed1a7fc314d4c0f501711c282bb84a00ceaf590b4c4bd61f --hash=sha256:eeec97588bd5282c983bd73f9010156e5e251e9e5307af547e2cb53b02d4639a" - } - }, - "neuron_py_deps_311_jmespath": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "jmespath==1.0.1 --hash=sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 --hash=sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" - } - }, - "neuron_py_deps_311_libneuronxla": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "libneuronxla==2.2.4410.0+835a67fb --hash=sha256:37e5f08482ef1a9a844c2894de6d91c03400a092764e54ab67612456be439f16" - } - }, - "neuron_py_deps_311_lockfile": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "lockfile==0.12.2 --hash=sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799 --hash=sha256:6c3cb24f344923d30b2785d5ad75182c8ea7ac1b6171b08657258ec7429d50fa" - } - }, - "neuron_py_deps_311_ml_dtypes": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "ml-dtypes==0.5.1 --hash=sha256:023ce2f502efd4d6c1e0472cc58ce3640d051d40e71e27386bed33901e201327 --hash=sha256:05f23447a1c20ddf4dc7c2c661aa9ed93fcb2658f1017c204d1e758714dc28a8 --hash=sha256:12651420130ee7cc13059fc56dac6ad300c3af3848b802d475148c9defd27c23 --hash=sha256:141b2ea2f20bb10802ddca55d91fe21231ef49715cfc971998e8f2a9838f3dbe --hash=sha256:15ad0f3b0323ce96c24637a88a6f44f6713c64032f27277b069f285c3cf66478 --hash=sha256:1b7fbe5571fdf28fd3aaab3ef4aafc847de9ebf263be959958c1ca58ec8eadf5 --hash=sha256:26ebcc69d7b779c8f129393e99732961b5cc33fcff84090451f448c89b0e01b4 --hash=sha256:6f462f5eca22fb66d7ff9c4744a3db4463af06c49816c4b6ac89b16bfcdc592e --hash=sha256:6f76232163b5b9c34291b54621ee60417601e2e4802a188a0ea7157cd9b323f4 --hash=sha256:7000b6e4d8ef07542c05044ec5d8bbae1df083b3f56822c3da63993a113e716f --hash=sha256:810512e2eccdfc3b41eefa3a27402371a3411453a1efc7e9c000318196140fed --hash=sha256:8f2c028954f16ede77902b223a8da2d9cbb3892375b85809a5c3cfb1587960c4 --hash=sha256:9626d0bca1fb387d5791ca36bacbba298c5ef554747b7ebeafefb4564fc83566 --hash=sha256:ac5b58559bb84a95848ed6984eb8013249f90b6bab62aa5acbad876e256002c9 --hash=sha256:ad4953c5eb9c25a56d11a913c2011d7e580a435ef5145f804d98efa14477d390 --hash=sha256:aefedc579ece2f8fb38f876aa7698204ee4c372d0e54f1c1ffa8ca580b54cc60 --hash=sha256:afb2009ac98da274e893e03162f6269398b2b00d947e7057ee2469a921d58135 --hash=sha256:b8a9d46b4df5ae2135a8e8e72b465448ebbc1559997f4f9304a9ecc3413efb5b --hash=sha256:bd73f51957949069573ff783563486339a9285d72e2f36c18e0c1aa9ca7eb190 --hash=sha256:bf9975bda82a99dc935f2ae4c83846d86df8fd6ba179614acac8e686910851da --hash=sha256:c09526488c3a9e8b7a23a388d4974b670a9a3dd40c5c8a61db5593ce9b725bab --hash=sha256:c9945669d3dadf8acb40ec2e57d38c985d8c285ea73af57fc5b09872c516106d --hash=sha256:d13755f8e8445b3870114e5b6240facaa7cb0c3361e54beba3e07fa912a6e12b --hash=sha256:fd918d4e6a4e0c110e2e05be7a7814d10dc1b95872accbf6512b80a109b71ae1" - } - }, - "neuron_py_deps_311_networkx": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "networkx==2.8.8 --hash=sha256:230d388117af870fce5647a3c52401fcf753e94720e6ea6b4197a5355648885e --hash=sha256:e435dfa75b1d7195c7b8378c3859f0445cd88c6b0375c181ed66823a9ceb7524" - } - }, - "neuron_py_deps_311_neuronx_cc": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "neuronx-cc==2.19.8089.0+8ab9f450 --hash=sha256:fbb05d8a0da8f54954c03ff2bea659f627309110c00efd54f523d877ddd6b88d" - } - }, - "neuron_py_deps_311_numpy": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "numpy==1.26.4 --hash=sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b --hash=sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818 --hash=sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20 --hash=sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0 --hash=sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010 --hash=sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a --hash=sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea --hash=sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c --hash=sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71 --hash=sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110 --hash=sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be --hash=sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a --hash=sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a --hash=sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5 --hash=sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed --hash=sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd --hash=sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c --hash=sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e --hash=sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0 --hash=sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c --hash=sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a --hash=sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b --hash=sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0 --hash=sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6 --hash=sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2 --hash=sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a --hash=sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30 --hash=sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218 --hash=sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5 --hash=sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07 --hash=sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2 --hash=sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4 --hash=sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764 --hash=sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef --hash=sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3 --hash=sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f" - } - }, - "neuron_py_deps_311_pgzip": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "pgzip==0.3.5 --hash=sha256:4e13ab66ecface5c51c5af51d8cd676aa51675cf85df000f501a86cf38c208c1 --hash=sha256:dd35510f59f6bd6b64e31c4baf90c10cdbb2775235fcc079b14b404fbd7f46bf" - } - }, - "neuron_py_deps_311_protobuf": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "protobuf==6.31.1 --hash=sha256:0414e3aa5a5f3ff423828e1e6a6e907d6c65c1d5b7e6e975793d5590bdeecc16 --hash=sha256:426f59d2964864a1a366254fa703b8632dcec0790d8862d30034d8245e1cd447 --hash=sha256:4ee898bf66f7a8b0bd21bce523814e6fbd8c6add948045ce958b73af7e8878c6 --hash=sha256:6f1227473dc43d44ed644425268eb7c2e488ae245d51c6866d19fe158e207402 --hash=sha256:720a6c7e6b77288b85063569baae8536671b39f15cc22037ec7045658d80489e --hash=sha256:7fa17d5a29c2e04b7d90e5e32388b8bfd0e7107cd8e616feef7ed3fa6bdab5c9 --hash=sha256:8764cf4587791e7564051b35524b72844f845ad0bb011704c3736cce762d8fe9 --hash=sha256:a40fc12b84c154884d7d4c4ebd675d5b3b5283e155f324049ae396b95ddebc39 --hash=sha256:d8cac4c982f0b957a4dc73a80e2ea24fab08e679c0de9deb835f4a12d69aca9a" - } - }, - "neuron_py_deps_311_psutil": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "psutil==7.0.0 --hash=sha256:101d71dc322e3cffd7cea0650b09b3d08b8e7c4109dd6809fe452dfd00e58b25 --hash=sha256:1e744154a6580bc968a0195fd25e80432d3afec619daf145b9e5ba16cc1d688e --hash=sha256:1fcee592b4c6f146991ca55919ea3d1f8926497a713ed7faaf8225e174581e91 --hash=sha256:39db632f6bb862eeccf56660871433e111b6ea58f2caea825571951d4b6aa3da --hash=sha256:4b1388a4f6875d7e2aff5c4ca1cc16c545ed41dd8bb596cefea80111db353a34 --hash=sha256:4cf3d4eb1aa9b348dec30105c55cd9b7d4629285735a102beb4441e38db90553 --hash=sha256:7be9c3eba38beccb6495ea33afd982a44074b78f28c434a1f51cc07fd315c456 --hash=sha256:84df4eb63e16849689f76b1ffcb36db7b8de703d1bc1fe41773db487621b6c17 --hash=sha256:a5f098451abc2828f7dc6b58d44b532b22f2088f4999a937557b603ce72b1993 --hash=sha256:ba3fcef7523064a6c9da440fc4d6bd07da93ac726b5733c29027d7dc95b39d99" - } - }, - "neuron_py_deps_311_python_daemon": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "python-daemon==3.1.2 --hash=sha256:b906833cef63502994ad48e2eab213259ed9bb18d54fa8774dcba2ff7864cec6 --hash=sha256:f7b04335adc473de877f5117e26d5f1142f4c9f7cd765408f0877757be5afbf4" - } - }, - "neuron_py_deps_311_python_dateutil": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "python-dateutil==2.9.0.post0 --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" - } - }, - "neuron_py_deps_311_requests": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "requests==2.32.4 --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422" - } - }, - "neuron_py_deps_311_requests_unixsocket": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "requests-unixsocket==0.4.1 --hash=sha256:60c4942e9dbecc2f64d611039fb1dfc25da382083c6434ac0316dca3ff908f4d --hash=sha256:b2596158c356ecee68d27ba469a52211230ac6fb0cde8b66afb19f0ed47a1995" - } - }, - "neuron_py_deps_311_s3transfer": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "s3transfer==0.13.1 --hash=sha256:a981aa7429be23fe6dfc13e80e4020057cbab622b08c0315288758d67cabc724 --hash=sha256:c3fdba22ba1bd367922f27ec8032d6a1cf5f10c934fb5d68cf60fd5a23d936cf" - } - }, - "neuron_py_deps_311_scipy": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "scipy==1.12.0 --hash=sha256:196ebad3a4882081f62a5bf4aeb7326aa34b110e533aab23e4374fcccb0890dc --hash=sha256:408c68423f9de16cb9e602528be4ce0d6312b05001f3de61fe9ec8b1263cad08 --hash=sha256:4bf5abab8a36d20193c698b0f1fc282c1d083c94723902c447e5d2f1780936a3 --hash=sha256:4c1020cad92772bf44b8e4cdabc1df5d87376cb219742549ef69fc9fd86282dd --hash=sha256:5adfad5dbf0163397beb4aca679187d24aec085343755fcdbdeb32b3679f254c --hash=sha256:5e32847e08da8d895ce09d108a494d9eb78974cf6de23063f93306a3e419960c --hash=sha256:6546dc2c11a9df6926afcbdd8a3edec28566e4e785b915e849348c6dd9f3f490 --hash=sha256:730badef9b827b368f351eacae2e82da414e13cf8bd5051b4bdfd720271a5371 --hash=sha256:75ea2a144096b5e39402e2ff53a36fecfd3b960d786b7efd3c180e29c39e53f2 --hash=sha256:78e4402e140879387187f7f25d91cc592b3501a2e51dfb320f48dfb73565f10b --hash=sha256:8b8066bce124ee5531d12a74b617d9ac0ea59245246410e19bca549656d9a40a --hash=sha256:8bee4993817e204d761dba10dbab0774ba5a8612e57e81319ea04d84945375ba --hash=sha256:913d6e7956c3a671de3b05ccb66b11bc293f56bfdef040583a7221d9e22a2e35 --hash=sha256:95e5c750d55cf518c398a8240571b0e0782c2d5a703250872f36eaf737751338 --hash=sha256:9c39f92041f490422924dfdb782527a4abddf4707616e07b021de33467f917bc --hash=sha256:a24024d45ce9a675c1fb8494e8e5244efea1c7a09c60beb1eeb80373d0fecc70 --hash=sha256:a7ebda398f86e56178c2fa94cad15bf457a218a54a35c2a7b4490b9f9cb2676c --hash=sha256:b360f1b6b2f742781299514e99ff560d1fe9bd1bff2712894b52abe528d1fd1e --hash=sha256:bba1b0c7256ad75401c73e4b3cf09d1f176e9bd4248f0d3112170fb2ec4db067 --hash=sha256:c3003652496f6e7c387b1cf63f4bb720951cfa18907e998ea551e6de51a04467 --hash=sha256:e53958531a7c695ff66c2e7bb7b79560ffdc562e2051644c5576c39ff8efb563 --hash=sha256:e646d8571804a304e1da01040d21577685ce8e2db08ac58e543eaca063453e1c --hash=sha256:e7e76cc48638228212c747ada851ef355c2bb5e7f939e10952bc504c11f4e372 --hash=sha256:f5f00ebaf8de24d14b8449981a2842d404152774c1a1d880c901bf454cb8e2a1 --hash=sha256:f7ce148dffcd64ade37b2df9315541f9adad6efcaa86866ee7dd5db0c8f041c3" - } - }, - "neuron_py_deps_311_six": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "six==1.17.0 --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81" - } - }, - "neuron_py_deps_311_tqdm": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "tqdm==4.67.1 --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2" - } - }, - "neuron_py_deps_311_typing_extensions": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "typing-extensions==4.14.1 --hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 --hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76" - } - }, - "neuron_py_deps_311_urllib3": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@neuron_py_deps//{name}:{target}", - "download_only": true, - "experimental_target_platforms": [ - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--extra-index-url", - "https://pip.repos.neuron.amazonaws.com", - "--find-links", - "https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html", - "--abi=cp311", - "--implementation=cp", - "--python-version=311", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "neuron_py_deps_311", - "requirement": "urllib3==2.5.0 --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc" - } - }, - "pip_deps_310_numpy": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@pip_deps//{name}:{target}", - "python_interpreter_target": "@@rules_python++python+python_3_10_host//:python", - "repo": "pip_deps_310", - "requirement": "numpy<=1.26.1" - } - }, - "pip_deps_310_setuptools": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@pip_deps//{name}:{target}", - "python_interpreter_target": "@@rules_python++python+python_3_10_host//:python", - "repo": "pip_deps_310", - "requirement": "setuptools<=70.3.0" - } - }, - "pip_deps_311_numpy": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@pip_deps//{name}:{target}", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "pip_deps_311", - "requirement": "numpy<=1.26.1" - } - }, - "pip_deps_311_setuptools": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@pip_deps//{name}:{target}", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "pip_deps_311", - "requirement": "setuptools<=70.3.0" - } - }, - "pip_deps_312_numpy": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@pip_deps//{name}:{target}", - "python_interpreter_target": "@@rules_python++python+python_3_12_host//:python", - "repo": "pip_deps_312", - "requirement": "numpy<=1.26.1" - } - }, - "pip_deps_312_setuptools": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@pip_deps//{name}:{target}", - "python_interpreter_target": "@@rules_python++python+python_3_12_host//:python", - "repo": "pip_deps_312", - "requirement": "setuptools<=70.3.0" - } - }, - "pip_deps_38_numpy": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@pip_deps//{name}:{target}", - "python_interpreter_target": "@@rules_python++python+python_3_8_host//:python", - "repo": "pip_deps_38", - "requirement": "numpy<=1.26.1" - } - }, - "pip_deps_38_setuptools": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@pip_deps//{name}:{target}", - "python_interpreter_target": "@@rules_python++python+python_3_8_host//:python", - "repo": "pip_deps_38", - "requirement": "setuptools<=70.3.0" - } - }, - "pip_deps_39_numpy": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@pip_deps//{name}:{target}", - "python_interpreter_target": "@@rules_python++python+python_3_9_host//:python", - "repo": "pip_deps_39", - "requirement": "numpy<=1.26.1" - } - }, - "pip_deps_39_setuptools": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@pip_deps//{name}:{target}", - "python_interpreter_target": "@@rules_python++python+python_3_9_host//:python", - "repo": "pip_deps_39", - "requirement": "setuptools<=70.3.0" - } - }, - "rules_fuzzing_py_deps_310_absl_py": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_fuzzing_py_deps//{name}:{target}", - "extra_pip_args": [ - "--require-hashes" - ], - "python_interpreter_target": "@@rules_python++python+python_3_10_host//:python", - "repo": "rules_fuzzing_py_deps_310", - "requirement": "absl-py==2.0.0 --hash=sha256:9a28abb62774ae4e8edbe2dd4c49ffcd45a6a848952a5eccc6a49f3f0fc1e2f3" - } - }, - "rules_fuzzing_py_deps_310_six": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_fuzzing_py_deps//{name}:{target}", - "extra_pip_args": [ - "--require-hashes" - ], - "python_interpreter_target": "@@rules_python++python+python_3_10_host//:python", - "repo": "rules_fuzzing_py_deps_310", - "requirement": "six==1.16.0 --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - } - }, - "rules_fuzzing_py_deps_311_absl_py": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_fuzzing_py_deps//{name}:{target}", - "extra_pip_args": [ - "--require-hashes" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_fuzzing_py_deps_311", - "requirement": "absl-py==2.0.0 --hash=sha256:9a28abb62774ae4e8edbe2dd4c49ffcd45a6a848952a5eccc6a49f3f0fc1e2f3" - } - }, - "rules_fuzzing_py_deps_311_six": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_fuzzing_py_deps//{name}:{target}", - "extra_pip_args": [ - "--require-hashes" - ], - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_fuzzing_py_deps_311", - "requirement": "six==1.16.0 --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - } - }, - "rules_fuzzing_py_deps_312_absl_py": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_fuzzing_py_deps//{name}:{target}", - "extra_pip_args": [ - "--require-hashes" - ], - "python_interpreter_target": "@@rules_python++python+python_3_12_host//:python", - "repo": "rules_fuzzing_py_deps_312", - "requirement": "absl-py==2.0.0 --hash=sha256:9a28abb62774ae4e8edbe2dd4c49ffcd45a6a848952a5eccc6a49f3f0fc1e2f3" - } - }, - "rules_fuzzing_py_deps_312_six": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_fuzzing_py_deps//{name}:{target}", - "extra_pip_args": [ - "--require-hashes" - ], - "python_interpreter_target": "@@rules_python++python+python_3_12_host//:python", - "repo": "rules_fuzzing_py_deps_312", - "requirement": "six==1.16.0 --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - } - }, - "rules_fuzzing_py_deps_38_absl_py": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_fuzzing_py_deps//{name}:{target}", - "extra_pip_args": [ - "--require-hashes" - ], - "python_interpreter_target": "@@rules_python++python+python_3_8_host//:python", - "repo": "rules_fuzzing_py_deps_38", - "requirement": "absl-py==2.0.0 --hash=sha256:9a28abb62774ae4e8edbe2dd4c49ffcd45a6a848952a5eccc6a49f3f0fc1e2f3" - } - }, - "rules_fuzzing_py_deps_38_six": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_fuzzing_py_deps//{name}:{target}", - "extra_pip_args": [ - "--require-hashes" - ], - "python_interpreter_target": "@@rules_python++python+python_3_8_host//:python", - "repo": "rules_fuzzing_py_deps_38", - "requirement": "six==1.16.0 --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - } - }, - "rules_fuzzing_py_deps_39_absl_py": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_fuzzing_py_deps//{name}:{target}", - "extra_pip_args": [ - "--require-hashes" - ], - "python_interpreter_target": "@@rules_python++python+python_3_9_host//:python", - "repo": "rules_fuzzing_py_deps_39", - "requirement": "absl-py==2.0.0 --hash=sha256:9a28abb62774ae4e8edbe2dd4c49ffcd45a6a848952a5eccc6a49f3f0fc1e2f3" - } - }, - "rules_fuzzing_py_deps_39_six": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_fuzzing_py_deps//{name}:{target}", - "extra_pip_args": [ - "--require-hashes" - ], - "python_interpreter_target": "@@rules_python++python+python_3_9_host//:python", - "repo": "rules_fuzzing_py_deps_39", - "requirement": "six==1.16.0 --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - } - }, - "rules_python_publish_deps_311_backports_tarfile_py3_none_any_77e284d7": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "backports.tarfile-1.2.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "backports-tarfile==1.2.0", - "sha256": "77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34", - "urls": [ - "https://files.pythonhosted.org/packages/b9/fa/123043af240e49752f1c4bd24da5053b6bd00cad78c2be53c0d1e8b975bc/backports.tarfile-1.2.0-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_backports_tarfile_sdist_d75e02c2": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "backports_tarfile-1.2.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "backports-tarfile==1.2.0", - "sha256": "d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991", - "urls": [ - "https://files.pythonhosted.org/packages/86/72/cd9b395f25e290e633655a100af28cb253e4393396264a98bd5f5951d50f/backports_tarfile-1.2.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_certifi_py3_none_any_922820b5": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "certifi-2024.8.30-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "certifi==2024.8.30", - "sha256": "922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8", - "urls": [ - "https://files.pythonhosted.org/packages/12/90/3c9ff0512038035f59d279fddeb79f5f1eccd8859f06d6163c58798b9487/certifi-2024.8.30-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_certifi_sdist_bec941d2": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "certifi-2024.8.30.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "certifi==2024.8.30", - "sha256": "bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9", - "urls": [ - "https://files.pythonhosted.org/packages/b0/ee/9b19140fe824b367c04c5e1b369942dd754c4c5462d5674002f75c4dedc1/certifi-2024.8.30.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_cffi_cp311_cp311_manylinux_2_17_aarch64_a1ed2dd2": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cffi==1.17.1", - "sha256": "a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41", - "urls": [ - "https://files.pythonhosted.org/packages/2e/ea/70ce63780f096e16ce8588efe039d3c4f91deb1dc01e9c73a287939c79a6/cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" - ] - } - }, - "rules_python_publish_deps_311_cffi_cp311_cp311_manylinux_2_17_s390x_a24ed04c": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cffi==1.17.1", - "sha256": "a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6", - "urls": [ - "https://files.pythonhosted.org/packages/62/12/ce8710b5b8affbcdd5c6e367217c242524ad17a02fe5beec3ee339f69f85/cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl" - ] - } - }, - "rules_python_publish_deps_311_cffi_cp311_cp311_manylinux_2_17_x86_64_610faea7": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cffi==1.17.1", - "sha256": "610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d", - "urls": [ - "https://files.pythonhosted.org/packages/ff/6b/d45873c5e0242196f042d555526f92aa9e0c32355a1be1ff8c27f077fd37/cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - ] - } - }, - "rules_python_publish_deps_311_cffi_cp311_cp311_musllinux_1_1_aarch64_a9b15d49": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cffi==1.17.1", - "sha256": "a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6", - "urls": [ - "https://files.pythonhosted.org/packages/1a/52/d9a0e523a572fbccf2955f5abe883cfa8bcc570d7faeee06336fbd50c9fc/cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl" - ] - } - }, - "rules_python_publish_deps_311_cffi_cp311_cp311_musllinux_1_1_x86_64_fc48c783": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cffi==1.17.1", - "sha256": "fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b", - "urls": [ - "https://files.pythonhosted.org/packages/f8/4a/34599cac7dfcd888ff54e801afe06a19c17787dfd94495ab0c8d35fe99fb/cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl" - ] - } - }, - "rules_python_publish_deps_311_cffi_sdist_1c39c601": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "cffi-1.17.1.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cffi==1.17.1", - "sha256": "1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", - "urls": [ - "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_charset_normalizer_cp311_cp311_macosx_10_9_universal2_0d99dd8f": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_universal2.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "charset-normalizer==3.4.0", - "sha256": "0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c", - "urls": [ - "https://files.pythonhosted.org/packages/9c/61/73589dcc7a719582bf56aae309b6103d2762b526bffe189d635a7fcfd998/charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_universal2.whl" - ] - } - }, - "rules_python_publish_deps_311_charset_normalizer_cp311_cp311_macosx_10_9_x86_64_c57516e5": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "charset-normalizer==3.4.0", - "sha256": "c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944", - "urls": [ - "https://files.pythonhosted.org/packages/77/d5/8c982d58144de49f59571f940e329ad6e8615e1e82ef84584c5eeb5e1d72/charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl" - ] - } - }, - "rules_python_publish_deps_311_charset_normalizer_cp311_cp311_macosx_11_0_arm64_6dba5d19": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "charset_normalizer-3.4.0-cp311-cp311-macosx_11_0_arm64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "charset-normalizer==3.4.0", - "sha256": "6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee", - "urls": [ - "https://files.pythonhosted.org/packages/bf/19/411a64f01ee971bed3231111b69eb56f9331a769072de479eae7de52296d/charset_normalizer-3.4.0-cp311-cp311-macosx_11_0_arm64.whl" - ] - } - }, - "rules_python_publish_deps_311_charset_normalizer_cp311_cp311_manylinux_2_17_aarch64_bf4475b8": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "charset-normalizer==3.4.0", - "sha256": "bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c", - "urls": [ - "https://files.pythonhosted.org/packages/4c/92/97509850f0d00e9f14a46bc751daabd0ad7765cff29cdfb66c68b6dad57f/charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" - ] - } - }, - "rules_python_publish_deps_311_charset_normalizer_cp311_cp311_manylinux_2_17_s390x_8ff4e7cd": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "charset-normalizer==3.4.0", - "sha256": "8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea", - "urls": [ - "https://files.pythonhosted.org/packages/13/bc/87c2c9f2c144bedfa62f894c3007cd4530ba4b5351acb10dc786428a50f0/charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl" - ] - } - }, - "rules_python_publish_deps_311_charset_normalizer_cp311_cp311_manylinux_2_17_x86_64_3710a975": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "charset-normalizer==3.4.0", - "sha256": "3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc", - "urls": [ - "https://files.pythonhosted.org/packages/eb/5b/6f10bad0f6461fa272bfbbdf5d0023b5fb9bc6217c92bf068fa5a99820f5/charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - ] - } - }, - "rules_python_publish_deps_311_charset_normalizer_cp311_cp311_musllinux_1_2_aarch64_47334db7": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "charset-normalizer==3.4.0", - "sha256": "47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594", - "urls": [ - "https://files.pythonhosted.org/packages/d7/a1/493919799446464ed0299c8eef3c3fad0daf1c3cd48bff9263c731b0d9e2/charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl" - ] - } - }, - "rules_python_publish_deps_311_charset_normalizer_cp311_cp311_musllinux_1_2_s390x_63bc5c4a": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "charset-normalizer==3.4.0", - "sha256": "63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129", - "urls": [ - "https://files.pythonhosted.org/packages/8d/c9/27e41d481557be53d51e60750b85aa40eaf52b841946b3cdeff363105737/charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl" - ] - } - }, - "rules_python_publish_deps_311_charset_normalizer_cp311_cp311_musllinux_1_2_x86_64_bcb4f8ea": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "charset-normalizer==3.4.0", - "sha256": "bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236", - "urls": [ - "https://files.pythonhosted.org/packages/ee/44/4f62042ca8cdc0cabf87c0fc00ae27cd8b53ab68be3605ba6d071f742ad3/charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl" - ] - } - }, - "rules_python_publish_deps_311_charset_normalizer_cp311_cp311_win_amd64_cee4373f": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "charset_normalizer-3.4.0-cp311-cp311-win_amd64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "charset-normalizer==3.4.0", - "sha256": "cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27", - "urls": [ - "https://files.pythonhosted.org/packages/0b/6e/b13bd47fa9023b3699e94abf565b5a2f0b0be6e9ddac9812182596ee62e4/charset_normalizer-3.4.0-cp311-cp311-win_amd64.whl" - ] - } - }, - "rules_python_publish_deps_311_charset_normalizer_py3_none_any_fe9f97fe": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "charset_normalizer-3.4.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "charset-normalizer==3.4.0", - "sha256": "fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079", - "urls": [ - "https://files.pythonhosted.org/packages/bf/9b/08c0432272d77b04803958a4598a51e2a4b51c06640af8b8f0f908c18bf2/charset_normalizer-3.4.0-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_charset_normalizer_sdist_223217c3": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "charset_normalizer-3.4.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "charset-normalizer==3.4.0", - "sha256": "223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e", - "urls": [ - "https://files.pythonhosted.org/packages/f2/4f/e1808dc01273379acc506d18f1504eb2d299bd4131743b9fc54d7be4df1e/charset_normalizer-3.4.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_cryptography_cp39_abi3_manylinux_2_17_aarch64_846da004": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "cryptography-43.0.3-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cryptography==43.0.3", - "sha256": "846da004a5804145a5f441b8530b4bf35afbf7da70f82409f151695b127213d5", - "urls": [ - "https://files.pythonhosted.org/packages/2f/78/55356eb9075d0be6e81b59f45c7b48df87f76a20e73893872170471f3ee8/cryptography-43.0.3-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" - ] - } - }, - "rules_python_publish_deps_311_cryptography_cp39_abi3_manylinux_2_17_x86_64_0f996e72": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "cryptography-43.0.3-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cryptography==43.0.3", - "sha256": "0f996e7268af62598f2fc1204afa98a3b5712313a55c4c9d434aef49cadc91d4", - "urls": [ - "https://files.pythonhosted.org/packages/2a/2c/488776a3dc843f95f86d2f957ca0fc3407d0242b50bede7fad1e339be03f/cryptography-43.0.3-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - ] - } - }, - "rules_python_publish_deps_311_cryptography_cp39_abi3_manylinux_2_28_aarch64_f7b178f1": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "cryptography-43.0.3-cp39-abi3-manylinux_2_28_aarch64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cryptography==43.0.3", - "sha256": "f7b178f11ed3664fd0e995a47ed2b5ff0a12d893e41dd0494f406d1cf555cab7", - "urls": [ - "https://files.pythonhosted.org/packages/7c/04/2345ca92f7a22f601a9c62961741ef7dd0127c39f7310dffa0041c80f16f/cryptography-43.0.3-cp39-abi3-manylinux_2_28_aarch64.whl" - ] - } - }, - "rules_python_publish_deps_311_cryptography_cp39_abi3_manylinux_2_28_x86_64_c2e6fc39": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "cryptography-43.0.3-cp39-abi3-manylinux_2_28_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cryptography==43.0.3", - "sha256": "c2e6fc39c4ab499049df3bdf567f768a723a5e8464816e8f009f121a5a9f4405", - "urls": [ - "https://files.pythonhosted.org/packages/ac/25/e715fa0bc24ac2114ed69da33adf451a38abb6f3f24ec207908112e9ba53/cryptography-43.0.3-cp39-abi3-manylinux_2_28_x86_64.whl" - ] - } - }, - "rules_python_publish_deps_311_cryptography_cp39_abi3_musllinux_1_2_aarch64_e1be4655": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "cryptography-43.0.3-cp39-abi3-musllinux_1_2_aarch64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cryptography==43.0.3", - "sha256": "e1be4655c7ef6e1bbe6b5d0403526601323420bcf414598955968c9ef3eb7d16", - "urls": [ - "https://files.pythonhosted.org/packages/21/ce/b9c9ff56c7164d8e2edfb6c9305045fbc0df4508ccfdb13ee66eb8c95b0e/cryptography-43.0.3-cp39-abi3-musllinux_1_2_aarch64.whl" - ] - } - }, - "rules_python_publish_deps_311_cryptography_cp39_abi3_musllinux_1_2_x86_64_df6b6c6d": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "cryptography-43.0.3-cp39-abi3-musllinux_1_2_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cryptography==43.0.3", - "sha256": "df6b6c6d742395dd77a23ea3728ab62f98379eff8fb61be2744d4679ab678f73", - "urls": [ - "https://files.pythonhosted.org/packages/2a/33/b3682992ab2e9476b9c81fff22f02c8b0a1e6e1d49ee1750a67d85fd7ed2/cryptography-43.0.3-cp39-abi3-musllinux_1_2_x86_64.whl" - ] - } - }, - "rules_python_publish_deps_311_cryptography_sdist_315b9001": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "cryptography-43.0.3.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "cryptography==43.0.3", - "sha256": "315b9001266a492a6ff443b61238f956b214dbec9910a081ba5b6646a055a805", - "urls": [ - "https://files.pythonhosted.org/packages/0d/05/07b55d1fa21ac18c3a8c79f764e2514e6f6a9698f1be44994f5adf0d29db/cryptography-43.0.3.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_docutils_py3_none_any_dafca5b9": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "docutils-0.21.2-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "docutils==0.21.2", - "sha256": "dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2", - "urls": [ - "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_docutils_sdist_3a6b1873": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "docutils-0.21.2.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "docutils==0.21.2", - "sha256": "3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f", - "urls": [ - "https://files.pythonhosted.org/packages/ae/ed/aefcc8cd0ba62a0560c3c18c33925362d46c6075480bfa4df87b28e169a9/docutils-0.21.2.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_idna_py3_none_any_946d195a": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "idna-3.10-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "idna==3.10", - "sha256": "946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", - "urls": [ - "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_idna_sdist_12f65c9b": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "idna-3.10.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "idna==3.10", - "sha256": "12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", - "urls": [ - "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_importlib_metadata_py3_none_any_45e54197": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "importlib_metadata-8.5.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "importlib-metadata==8.5.0", - "sha256": "45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b", - "urls": [ - "https://files.pythonhosted.org/packages/a0/d9/a1e041c5e7caa9a05c925f4bdbdfb7f006d1f74996af53467bc394c97be7/importlib_metadata-8.5.0-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_importlib_metadata_sdist_71522656": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "importlib_metadata-8.5.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "importlib-metadata==8.5.0", - "sha256": "71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7", - "urls": [ - "https://files.pythonhosted.org/packages/cd/12/33e59336dca5be0c398a7482335911a33aa0e20776128f038019f1a95f1b/importlib_metadata-8.5.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_jaraco_classes_py3_none_any_f662826b": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "jaraco.classes-3.4.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "jaraco-classes==3.4.0", - "sha256": "f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790", - "urls": [ - "https://files.pythonhosted.org/packages/7f/66/b15ce62552d84bbfcec9a4873ab79d993a1dd4edb922cbfccae192bd5b5f/jaraco.classes-3.4.0-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_jaraco_classes_sdist_47a024b5": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "jaraco.classes-3.4.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "jaraco-classes==3.4.0", - "sha256": "47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd", - "urls": [ - "https://files.pythonhosted.org/packages/06/c0/ed4a27bc5571b99e3cff68f8a9fa5b56ff7df1c2251cc715a652ddd26402/jaraco.classes-3.4.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_jaraco_context_py3_none_any_f797fc48": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "jaraco.context-6.0.1-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "jaraco-context==6.0.1", - "sha256": "f797fc481b490edb305122c9181830a3a5b76d84ef6d1aef2fb9b47ab956f9e4", - "urls": [ - "https://files.pythonhosted.org/packages/ff/db/0c52c4cf5e4bd9f5d7135ec7669a3a767af21b3a308e1ed3674881e52b62/jaraco.context-6.0.1-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_jaraco_context_sdist_9bae4ea5": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "jaraco_context-6.0.1.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "jaraco-context==6.0.1", - "sha256": "9bae4ea555cf0b14938dc0aee7c9f32ed303aa20a3b73e7dc80111628792d1b3", - "urls": [ - "https://files.pythonhosted.org/packages/df/ad/f3777b81bf0b6e7bc7514a1656d3e637b2e8e15fab2ce3235730b3e7a4e6/jaraco_context-6.0.1.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_jaraco_functools_py3_none_any_ad159f13": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "jaraco.functools-4.1.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "jaraco-functools==4.1.0", - "sha256": "ad159f13428bc4acbf5541ad6dec511f91573b90fba04df61dafa2a1231cf649", - "urls": [ - "https://files.pythonhosted.org/packages/9f/4f/24b319316142c44283d7540e76c7b5a6dbd5db623abd86bb7b3491c21018/jaraco.functools-4.1.0-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_jaraco_functools_sdist_70f7e0e2": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "jaraco_functools-4.1.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "jaraco-functools==4.1.0", - "sha256": "70f7e0e2ae076498e212562325e805204fc092d7b4c17e0e86c959e249701a9d", - "urls": [ - "https://files.pythonhosted.org/packages/ab/23/9894b3df5d0a6eb44611c36aec777823fc2e07740dabbd0b810e19594013/jaraco_functools-4.1.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_jeepney_py3_none_any_c0a454ad": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "jeepney-0.8.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "jeepney==0.8.0", - "sha256": "c0a454ad016ca575060802ee4d590dd912e35c122fa04e70306de3d076cce755", - "urls": [ - "https://files.pythonhosted.org/packages/ae/72/2a1e2290f1ab1e06f71f3d0f1646c9e4634e70e1d37491535e19266e8dc9/jeepney-0.8.0-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_jeepney_sdist_5efe48d2": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "jeepney-0.8.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "jeepney==0.8.0", - "sha256": "5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806", - "urls": [ - "https://files.pythonhosted.org/packages/d6/f4/154cf374c2daf2020e05c3c6a03c91348d59b23c5366e968feb198306fdf/jeepney-0.8.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_keyring_py3_none_any_5426f817": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "keyring-25.4.1-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "keyring==25.4.1", - "sha256": "5426f817cf7f6f007ba5ec722b1bcad95a75b27d780343772ad76b17cb47b0bf", - "urls": [ - "https://files.pythonhosted.org/packages/83/25/e6d59e5f0a0508d0dca8bb98c7f7fd3772fc943ac3f53d5ab18a218d32c0/keyring-25.4.1-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_keyring_sdist_b07ebc55": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "keyring-25.4.1.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "keyring==25.4.1", - "sha256": "b07ebc55f3e8ed86ac81dd31ef14e81ace9dd9c3d4b5d77a6e9a2016d0d71a1b", - "urls": [ - "https://files.pythonhosted.org/packages/a5/1c/2bdbcfd5d59dc6274ffb175bc29aa07ecbfab196830e0cfbde7bd861a2ea/keyring-25.4.1.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_markdown_it_py_py3_none_any_35521684": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "markdown_it_py-3.0.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "markdown-it-py==3.0.0", - "sha256": "355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", - "urls": [ - "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_markdown_it_py_sdist_e3f60a94": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "markdown-it-py-3.0.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "markdown-it-py==3.0.0", - "sha256": "e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb", - "urls": [ - "https://files.pythonhosted.org/packages/38/71/3b932df36c1a044d397a1f92d1cf91ee0a503d91e470cbd670aa66b07ed0/markdown-it-py-3.0.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_mdurl_py3_none_any_84008a41": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "mdurl-0.1.2-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "mdurl==0.1.2", - "sha256": "84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", - "urls": [ - "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_mdurl_sdist_bb413d29": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "mdurl-0.1.2.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "mdurl==0.1.2", - "sha256": "bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", - "urls": [ - "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_more_itertools_py3_none_any_037b0d32": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "more_itertools-10.5.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "more-itertools==10.5.0", - "sha256": "037b0d3203ce90cca8ab1defbbdac29d5f993fc20131f3664dc8d6acfa872aef", - "urls": [ - "https://files.pythonhosted.org/packages/48/7e/3a64597054a70f7c86eb0a7d4fc315b8c1ab932f64883a297bdffeb5f967/more_itertools-10.5.0-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_more_itertools_sdist_5482bfef": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "more-itertools-10.5.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "more-itertools==10.5.0", - "sha256": "5482bfef7849c25dc3c6dd53a6173ae4795da2a41a80faea6700d9f5846c5da6", - "urls": [ - "https://files.pythonhosted.org/packages/51/78/65922308c4248e0eb08ebcbe67c95d48615cc6f27854b6f2e57143e9178f/more-itertools-10.5.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_nh3_cp37_abi3_macosx_10_12_x86_64_14c5a72e": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "nh3-0.2.18-cp37-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "nh3==0.2.18", - "sha256": "14c5a72e9fe82aea5fe3072116ad4661af5cf8e8ff8fc5ad3450f123e4925e86", - "urls": [ - "https://files.pythonhosted.org/packages/b3/89/1daff5d9ba5a95a157c092c7c5f39b8dd2b1ddb4559966f808d31cfb67e0/nh3-0.2.18-cp37-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl" - ] - } - }, - "rules_python_publish_deps_311_nh3_cp37_abi3_macosx_10_12_x86_64_7b7c2a3c": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "nh3-0.2.18-cp37-abi3-macosx_10_12_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "nh3==0.2.18", - "sha256": "7b7c2a3c9eb1a827d42539aa64091640bd275b81e097cd1d8d82ef91ffa2e811", - "urls": [ - "https://files.pythonhosted.org/packages/2c/b6/42fc3c69cabf86b6b81e4c051a9b6e249c5ba9f8155590222c2622961f58/nh3-0.2.18-cp37-abi3-macosx_10_12_x86_64.whl" - ] - } - }, - "rules_python_publish_deps_311_nh3_cp37_abi3_manylinux_2_17_aarch64_42c64511": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "nh3-0.2.18-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "nh3==0.2.18", - "sha256": "42c64511469005058cd17cc1537578eac40ae9f7200bedcfd1fc1a05f4f8c200", - "urls": [ - "https://files.pythonhosted.org/packages/45/b9/833f385403abaf0023c6547389ec7a7acf141ddd9d1f21573723a6eab39a/nh3-0.2.18-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" - ] - } - }, - "rules_python_publish_deps_311_nh3_cp37_abi3_manylinux_2_17_armv7l_0411beb0": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "nh3-0.2.18-cp37-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "nh3==0.2.18", - "sha256": "0411beb0589eacb6734f28d5497ca2ed379eafab8ad8c84b31bb5c34072b7164", - "urls": [ - "https://files.pythonhosted.org/packages/05/2b/85977d9e11713b5747595ee61f381bc820749daf83f07b90b6c9964cf932/nh3-0.2.18-cp37-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl" - ] - } - }, - "rules_python_publish_deps_311_nh3_cp37_abi3_manylinux_2_17_ppc64_5f36b271": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "nh3-0.2.18-cp37-abi3-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "nh3==0.2.18", - "sha256": "5f36b271dae35c465ef5e9090e1fdaba4a60a56f0bb0ba03e0932a66f28b9189", - "urls": [ - "https://files.pythonhosted.org/packages/72/f2/5c894d5265ab80a97c68ca36f25c8f6f0308abac649aaf152b74e7e854a8/nh3-0.2.18-cp37-abi3-manylinux_2_17_ppc64.manylinux2014_ppc64.whl" - ] - } - }, - "rules_python_publish_deps_311_nh3_cp37_abi3_manylinux_2_17_s390x_19aaba96": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "nh3-0.2.18-cp37-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "nh3==0.2.18", - "sha256": "19aaba96e0f795bd0a6c56291495ff59364f4300d4a39b29a0abc9cb3774a84b", - "urls": [ - "https://files.pythonhosted.org/packages/c2/a8/3bb02d0c60a03ad3a112b76c46971e9480efa98a8946677b5a59f60130ca/nh3-0.2.18-cp37-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl" - ] - } - }, - "rules_python_publish_deps_311_nh3_cp37_abi3_manylinux_2_17_x86_64_de3ceed6": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "nh3-0.2.18-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "nh3==0.2.18", - "sha256": "de3ceed6e661954871d6cd78b410213bdcb136f79aafe22aa7182e028b8c7307", - "urls": [ - "https://files.pythonhosted.org/packages/1b/63/6ab90d0e5225ab9780f6c9fb52254fa36b52bb7c188df9201d05b647e5e1/nh3-0.2.18-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - ] - } - }, - "rules_python_publish_deps_311_nh3_cp37_abi3_musllinux_1_2_aarch64_f0eca9ca": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "nh3-0.2.18-cp37-abi3-musllinux_1_2_aarch64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "nh3==0.2.18", - "sha256": "f0eca9ca8628dbb4e916ae2491d72957fdd35f7a5d326b7032a345f111ac07fe", - "urls": [ - "https://files.pythonhosted.org/packages/a3/da/0c4e282bc3cff4a0adf37005fa1fb42257673fbc1bbf7d1ff639ec3d255a/nh3-0.2.18-cp37-abi3-musllinux_1_2_aarch64.whl" - ] - } - }, - "rules_python_publish_deps_311_nh3_cp37_abi3_musllinux_1_2_armv7l_3a157ab1": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "nh3-0.2.18-cp37-abi3-musllinux_1_2_armv7l.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "nh3==0.2.18", - "sha256": "3a157ab149e591bb638a55c8c6bcb8cdb559c8b12c13a8affaba6cedfe51713a", - "urls": [ - "https://files.pythonhosted.org/packages/de/81/c291231463d21da5f8bba82c8167a6d6893cc5419b0639801ee5d3aeb8a9/nh3-0.2.18-cp37-abi3-musllinux_1_2_armv7l.whl" - ] - } - }, - "rules_python_publish_deps_311_nh3_cp37_abi3_musllinux_1_2_x86_64_36c95d4b": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "nh3-0.2.18-cp37-abi3-musllinux_1_2_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "nh3==0.2.18", - "sha256": "36c95d4b70530b320b365659bb5034341316e6a9b30f0b25fa9c9eff4c27a204", - "urls": [ - "https://files.pythonhosted.org/packages/eb/61/73a007c74c37895fdf66e0edcd881f5eaa17a348ff02f4bb4bc906d61085/nh3-0.2.18-cp37-abi3-musllinux_1_2_x86_64.whl" - ] - } - }, - "rules_python_publish_deps_311_nh3_cp37_abi3_win_amd64_8ce0f819": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "nh3-0.2.18-cp37-abi3-win_amd64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "nh3==0.2.18", - "sha256": "8ce0f819d2f1933953fca255db2471ad58184a60508f03e6285e5114b6254844", - "urls": [ - "https://files.pythonhosted.org/packages/26/8d/53c5b19c4999bdc6ba95f246f4ef35ca83d7d7423e5e38be43ad66544e5d/nh3-0.2.18-cp37-abi3-win_amd64.whl" - ] - } - }, - "rules_python_publish_deps_311_nh3_sdist_94a16692": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "nh3-0.2.18.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "nh3==0.2.18", - "sha256": "94a166927e53972a9698af9542ace4e38b9de50c34352b962f4d9a7d4c927af4", - "urls": [ - "https://files.pythonhosted.org/packages/62/73/10df50b42ddb547a907deeb2f3c9823022580a7a47281e8eae8e003a9639/nh3-0.2.18.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_pkginfo_py3_none_any_889a6da2": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "pkginfo-1.10.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "pkginfo==1.10.0", - "sha256": "889a6da2ed7ffc58ab5b900d888ddce90bce912f2d2de1dc1c26f4cb9fe65097", - "urls": [ - "https://files.pythonhosted.org/packages/56/09/054aea9b7534a15ad38a363a2bd974c20646ab1582a387a95b8df1bfea1c/pkginfo-1.10.0-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_pkginfo_sdist_5df73835": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "pkginfo-1.10.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "pkginfo==1.10.0", - "sha256": "5df73835398d10db79f8eecd5cd86b1f6d29317589ea70796994d49399af6297", - "urls": [ - "https://files.pythonhosted.org/packages/2f/72/347ec5be4adc85c182ed2823d8d1c7b51e13b9a6b0c1aae59582eca652df/pkginfo-1.10.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_pycparser_py3_none_any_c3702b6d": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "pycparser-2.22-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "pycparser==2.22", - "sha256": "c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", - "urls": [ - "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_pycparser_sdist_491c8be9": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "pycparser-2.22.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "pycparser==2.22", - "sha256": "491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6", - "urls": [ - "https://files.pythonhosted.org/packages/1d/b2/31537cf4b1ca988837256c910a668b553fceb8f069bedc4b1c826024b52c/pycparser-2.22.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_pygments_py3_none_any_b8e6aca0": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "pygments-2.18.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "pygments==2.18.0", - "sha256": "b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a", - "urls": [ - "https://files.pythonhosted.org/packages/f7/3f/01c8b82017c199075f8f788d0d906b9ffbbc5a47dc9918a945e13d5a2bda/pygments-2.18.0-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_pygments_sdist_786ff802": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "pygments-2.18.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "pygments==2.18.0", - "sha256": "786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199", - "urls": [ - "https://files.pythonhosted.org/packages/8e/62/8336eff65bcbc8e4cb5d05b55faf041285951b6e80f33e2bff2024788f31/pygments-2.18.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_pywin32_ctypes_py3_none_any_8a151337": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_windows_x86_64" - ], - "filename": "pywin32_ctypes-0.2.3-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "pywin32-ctypes==0.2.3", - "sha256": "8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8", - "urls": [ - "https://files.pythonhosted.org/packages/de/3d/8161f7711c017e01ac9f008dfddd9410dff3674334c233bde66e7ba65bbf/pywin32_ctypes-0.2.3-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_pywin32_ctypes_sdist_d162dc04": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "pywin32-ctypes-0.2.3.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "pywin32-ctypes==0.2.3", - "sha256": "d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755", - "urls": [ - "https://files.pythonhosted.org/packages/85/9f/01a1a99704853cb63f253eea009390c88e7131c67e66a0a02099a8c917cb/pywin32-ctypes-0.2.3.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_readme_renderer_py3_none_any_2fbca89b": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "readme_renderer-44.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "readme-renderer==44.0", - "sha256": "2fbca89b81a08526aadf1357a8c2ae889ec05fb03f5da67f9769c9a592166151", - "urls": [ - "https://files.pythonhosted.org/packages/e1/67/921ec3024056483db83953ae8e48079ad62b92db7880013ca77632921dd0/readme_renderer-44.0-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_readme_renderer_sdist_8712034e": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "readme_renderer-44.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "readme-renderer==44.0", - "sha256": "8712034eabbfa6805cacf1402b4eeb2a73028f72d1166d6f5cb7f9c047c5d1e1", - "urls": [ - "https://files.pythonhosted.org/packages/5a/a9/104ec9234c8448c4379768221ea6df01260cd6c2ce13182d4eac531c8342/readme_renderer-44.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_requests_py3_none_any_70761cfe": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "requests-2.32.3-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "requests==2.32.3", - "sha256": "70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6", - "urls": [ - "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_requests_sdist_55365417": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "requests-2.32.3.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "requests==2.32.3", - "sha256": "55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", - "urls": [ - "https://files.pythonhosted.org/packages/63/70/2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_requests_toolbelt_py2_none_any_cccfdd66": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "requests_toolbelt-1.0.0-py2.py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "requests-toolbelt==1.0.0", - "sha256": "cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", - "urls": [ - "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_requests_toolbelt_sdist_7681a0a3": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "requests-toolbelt-1.0.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "requests-toolbelt==1.0.0", - "sha256": "7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6", - "urls": [ - "https://files.pythonhosted.org/packages/f3/61/d7545dafb7ac2230c70d38d31cbfe4cc64f7144dc41f6e4e4b78ecd9f5bb/requests-toolbelt-1.0.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_rfc3986_py2_none_any_50b1502b": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "rfc3986-2.0.0-py2.py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "rfc3986==2.0.0", - "sha256": "50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd", - "urls": [ - "https://files.pythonhosted.org/packages/ff/9a/9afaade874b2fa6c752c36f1548f718b5b83af81ed9b76628329dab81c1b/rfc3986-2.0.0-py2.py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_rfc3986_sdist_97aacf9d": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "rfc3986-2.0.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "rfc3986==2.0.0", - "sha256": "97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c", - "urls": [ - "https://files.pythonhosted.org/packages/85/40/1520d68bfa07ab5a6f065a186815fb6610c86fe957bc065754e47f7b0840/rfc3986-2.0.0.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_rich_py3_none_any_6049d5e6": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "rich-13.9.4-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "rich==13.9.4", - "sha256": "6049d5e6ec054bf2779ab3358186963bac2ea89175919d699e378b99738c2a90", - "urls": [ - "https://files.pythonhosted.org/packages/19/71/39c7c0d87f8d4e6c020a393182060eaefeeae6c01dab6a84ec346f2567df/rich-13.9.4-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_rich_sdist_43959497": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "rich-13.9.4.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "rich==13.9.4", - "sha256": "439594978a49a09530cff7ebc4b5c7103ef57baf48d5ea3184f21d9a2befa098", - "urls": [ - "https://files.pythonhosted.org/packages/ab/3a/0316b28d0761c6734d6bc14e770d85506c986c85ffb239e688eeaab2c2bc/rich-13.9.4.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_secretstorage_py3_none_any_f356e662": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "filename": "SecretStorage-3.3.3-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "secretstorage==3.3.3", - "sha256": "f356e6628222568e3af06f2eba8df495efa13b3b63081dafd4f7d9a7b7bc9f99", - "urls": [ - "https://files.pythonhosted.org/packages/54/24/b4293291fa1dd830f353d2cb163295742fa87f179fcc8a20a306a81978b7/SecretStorage-3.3.3-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_secretstorage_sdist_2403533e": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "SecretStorage-3.3.3.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "secretstorage==3.3.3", - "sha256": "2403533ef369eca6d2ba81718576c5e0f564d5cca1b58f73a8b23e7d4eeebd77", - "urls": [ - "https://files.pythonhosted.org/packages/53/a4/f48c9d79cb507ed1373477dbceaba7401fd8a23af63b837fa61f1dcd3691/SecretStorage-3.3.3.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_twine_py3_none_any_215dbe7b": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "twine-5.1.1-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "twine==5.1.1", - "sha256": "215dbe7b4b94c2c50a7315c0275d2258399280fbb7d04182c7e55e24b5f93997", - "urls": [ - "https://files.pythonhosted.org/packages/5d/ec/00f9d5fd040ae29867355e559a94e9a8429225a0284a3f5f091a3878bfc0/twine-5.1.1-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_twine_sdist_9aa08251": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "twine-5.1.1.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "twine==5.1.1", - "sha256": "9aa0825139c02b3434d913545c7b847a21c835e11597f5255842d457da2322db", - "urls": [ - "https://files.pythonhosted.org/packages/77/68/bd982e5e949ef8334e6f7dcf76ae40922a8750aa2e347291ae1477a4782b/twine-5.1.1.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_urllib3_py3_none_any_ca899ca0": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "urllib3-2.2.3-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "urllib3==2.2.3", - "sha256": "ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac", - "urls": [ - "https://files.pythonhosted.org/packages/ce/d9/5f4c13cecde62396b0d3fe530a50ccea91e7dfc1ccf0e09c228841bb5ba8/urllib3-2.2.3-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_urllib3_sdist_e7d814a8": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "urllib3-2.2.3.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "urllib3==2.2.3", - "sha256": "e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9", - "urls": [ - "https://files.pythonhosted.org/packages/ed/63/22ba4ebfe7430b76388e7cd448d5478814d3032121827c12a2cc287e2260/urllib3-2.2.3.tar.gz" - ] - } - }, - "rules_python_publish_deps_311_zipp_py3_none_any_a817ac80": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "zipp-3.20.2-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "zipp==3.20.2", - "sha256": "a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350", - "urls": [ - "https://files.pythonhosted.org/packages/62/8b/5ba542fa83c90e09eac972fc9baca7a88e7e7ca4b221a89251954019308b/zipp-3.20.2-py3-none-any.whl" - ] - } - }, - "rules_python_publish_deps_311_zipp_sdist_bc9eb26f": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@rules_python_publish_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple" - ], - "filename": "zipp-3.20.2.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "rules_python_publish_deps_311", - "requirement": "zipp==3.20.2", - "sha256": "bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29", - "urls": [ - "https://files.pythonhosted.org/packages/54/bf/5c0000c44ebc80123ecbdddba1f5dcd94a5ada602a9c225d84b5aaa55e86/zipp-3.20.2.tar.gz" - ] - } - }, - "huggingface_hub": { - "repoRuleId": "@@rules_python+//python/private/pypi:hub_repository.bzl%hub_repository", - "attributes": { - "repo_name": "huggingface_hub", - "extra_hub_aliases": {}, - "whl_map": { - "certifi": "{\"huggingface_hub_311_certifi\":[{\"version\":\"3.11\"}]}", - "charset_normalizer": "{\"huggingface_hub_311_charset_normalizer\":[{\"version\":\"3.11\"}]}", - "filelock": "{\"huggingface_hub_311_filelock\":[{\"version\":\"3.11\"}]}", - "fsspec": "{\"huggingface_hub_311_fsspec\":[{\"version\":\"3.11\"}]}", - "hf_xet": "{\"huggingface_hub_311_hf_xet\":[{\"version\":\"3.11\"}]}", - "huggingface_hub": "{\"huggingface_hub_311_huggingface_hub\":[{\"version\":\"3.11\"}]}", - "idna": "{\"huggingface_hub_311_idna\":[{\"version\":\"3.11\"}]}", - "packaging": "{\"huggingface_hub_311_packaging\":[{\"version\":\"3.11\"}]}", - "pyyaml": "{\"huggingface_hub_311_pyyaml\":[{\"version\":\"3.11\"}]}", - "requests": "{\"huggingface_hub_311_requests\":[{\"version\":\"3.11\"}]}", - "tqdm": "{\"huggingface_hub_311_tqdm\":[{\"version\":\"3.11\"}]}", - "typing_extensions": "{\"huggingface_hub_311_typing_extensions\":[{\"version\":\"3.11\"}]}", - "urllib3": "{\"huggingface_hub_311_urllib3\":[{\"version\":\"3.11\"}]}" - }, - "packages": [ - "certifi", - "charset_normalizer", - "filelock", - "fsspec", - "hf_xet", - "huggingface_hub", - "idna", - "packaging", - "pyyaml", - "requests", - "tqdm", - "typing_extensions", - "urllib3" - ], - "groups": {} - } - }, - "neuron_py_deps": { - "repoRuleId": "@@rules_python+//python/private/pypi:hub_repository.bzl%hub_repository", - "attributes": { - "repo_name": "neuron_py_deps", - "extra_hub_aliases": {}, - "whl_map": { - "boto3": "{\"neuron_py_deps_311_boto3\":[{\"version\":\"3.11\"}]}", - "botocore": "{\"neuron_py_deps_311_botocore\":[{\"version\":\"3.11\"}]}", - "certifi": "{\"neuron_py_deps_311_certifi\":[{\"version\":\"3.11\"}]}", - "charset_normalizer": "{\"neuron_py_deps_311_charset_normalizer\":[{\"version\":\"3.11\"}]}", - "ec2_metadata": "{\"neuron_py_deps_311_ec2_metadata\":[{\"version\":\"3.11\"}]}", - "idna": "{\"neuron_py_deps_311_idna\":[{\"version\":\"3.11\"}]}", - "islpy": "{\"neuron_py_deps_311_islpy\":[{\"version\":\"3.11\"}]}", - "jmespath": "{\"neuron_py_deps_311_jmespath\":[{\"version\":\"3.11\"}]}", - "libneuronxla": "{\"neuron_py_deps_311_libneuronxla\":[{\"version\":\"3.11\"}]}", - "lockfile": "{\"neuron_py_deps_311_lockfile\":[{\"version\":\"3.11\"}]}", - "ml_dtypes": "{\"neuron_py_deps_311_ml_dtypes\":[{\"version\":\"3.11\"}]}", - "networkx": "{\"neuron_py_deps_311_networkx\":[{\"version\":\"3.11\"}]}", - "neuronx_cc": "{\"neuron_py_deps_311_neuronx_cc\":[{\"version\":\"3.11\"}]}", - "numpy": "{\"neuron_py_deps_311_numpy\":[{\"version\":\"3.11\"}]}", - "pgzip": "{\"neuron_py_deps_311_pgzip\":[{\"version\":\"3.11\"}]}", - "protobuf": "{\"neuron_py_deps_311_protobuf\":[{\"version\":\"3.11\"}]}", - "psutil": "{\"neuron_py_deps_311_psutil\":[{\"version\":\"3.11\"}]}", - "python_daemon": "{\"neuron_py_deps_311_python_daemon\":[{\"version\":\"3.11\"}]}", - "python_dateutil": "{\"neuron_py_deps_311_python_dateutil\":[{\"version\":\"3.11\"}]}", - "requests": "{\"neuron_py_deps_311_requests\":[{\"version\":\"3.11\"}]}", - "requests_unixsocket": "{\"neuron_py_deps_311_requests_unixsocket\":[{\"version\":\"3.11\"}]}", - "s3transfer": "{\"neuron_py_deps_311_s3transfer\":[{\"version\":\"3.11\"}]}", - "scipy": "{\"neuron_py_deps_311_scipy\":[{\"version\":\"3.11\"}]}", - "six": "{\"neuron_py_deps_311_six\":[{\"version\":\"3.11\"}]}", - "tqdm": "{\"neuron_py_deps_311_tqdm\":[{\"version\":\"3.11\"}]}", - "typing_extensions": "{\"neuron_py_deps_311_typing_extensions\":[{\"version\":\"3.11\"}]}", - "urllib3": "{\"neuron_py_deps_311_urllib3\":[{\"version\":\"3.11\"}]}" - }, - "packages": [ - "boto3", - "botocore", - "certifi", - "charset_normalizer", - "ec2_metadata", - "idna", - "islpy", - "jmespath", - "libneuronxla", - "lockfile", - "ml_dtypes", - "networkx", - "neuronx_cc", - "numpy", - "pgzip", - "protobuf", - "psutil", - "python_daemon", - "python_dateutil", - "requests", - "requests_unixsocket", - "s3transfer", - "scipy", - "six", - "tqdm", - "typing_extensions", - "urllib3" - ], - "groups": {} - } - }, - "pip_deps": { - "repoRuleId": "@@rules_python+//python/private/pypi:hub_repository.bzl%hub_repository", - "attributes": { - "repo_name": "pip_deps", - "extra_hub_aliases": {}, - "whl_map": { - "numpy": "{\"pip_deps_310_numpy\":[{\"version\":\"3.10\"}],\"pip_deps_311_numpy\":[{\"version\":\"3.11\"}],\"pip_deps_312_numpy\":[{\"version\":\"3.12\"}],\"pip_deps_38_numpy\":[{\"version\":\"3.8\"}],\"pip_deps_39_numpy\":[{\"version\":\"3.9\"}]}", - "setuptools": "{\"pip_deps_310_setuptools\":[{\"version\":\"3.10\"}],\"pip_deps_311_setuptools\":[{\"version\":\"3.11\"}],\"pip_deps_312_setuptools\":[{\"version\":\"3.12\"}],\"pip_deps_38_setuptools\":[{\"version\":\"3.8\"}],\"pip_deps_39_setuptools\":[{\"version\":\"3.9\"}]}" - }, - "packages": [ - "numpy", - "setuptools" - ], - "groups": {} - } - }, - "rules_fuzzing_py_deps": { - "repoRuleId": "@@rules_python+//python/private/pypi:hub_repository.bzl%hub_repository", - "attributes": { - "repo_name": "rules_fuzzing_py_deps", - "extra_hub_aliases": {}, - "whl_map": { - "absl_py": "{\"rules_fuzzing_py_deps_310_absl_py\":[{\"version\":\"3.10\"}],\"rules_fuzzing_py_deps_311_absl_py\":[{\"version\":\"3.11\"}],\"rules_fuzzing_py_deps_312_absl_py\":[{\"version\":\"3.12\"}],\"rules_fuzzing_py_deps_38_absl_py\":[{\"version\":\"3.8\"}],\"rules_fuzzing_py_deps_39_absl_py\":[{\"version\":\"3.9\"}]}", - "six": "{\"rules_fuzzing_py_deps_310_six\":[{\"version\":\"3.10\"}],\"rules_fuzzing_py_deps_311_six\":[{\"version\":\"3.11\"}],\"rules_fuzzing_py_deps_312_six\":[{\"version\":\"3.12\"}],\"rules_fuzzing_py_deps_38_six\":[{\"version\":\"3.8\"}],\"rules_fuzzing_py_deps_39_six\":[{\"version\":\"3.9\"}]}" - }, - "packages": [ - "absl_py", - "six" - ], - "groups": {} - } - }, - "rules_python_publish_deps": { - "repoRuleId": "@@rules_python+//python/private/pypi:hub_repository.bzl%hub_repository", - "attributes": { - "repo_name": "rules_python_publish_deps", - "extra_hub_aliases": {}, - "whl_map": { - "backports_tarfile": "{\"rules_python_publish_deps_311_backports_tarfile_py3_none_any_77e284d7\":[{\"filename\":\"backports.tarfile-1.2.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_backports_tarfile_sdist_d75e02c2\":[{\"filename\":\"backports_tarfile-1.2.0.tar.gz\",\"version\":\"3.11\"}]}", - "certifi": "{\"rules_python_publish_deps_311_certifi_py3_none_any_922820b5\":[{\"filename\":\"certifi-2024.8.30-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_certifi_sdist_bec941d2\":[{\"filename\":\"certifi-2024.8.30.tar.gz\",\"version\":\"3.11\"}]}", - "cffi": "{\"rules_python_publish_deps_311_cffi_cp311_cp311_manylinux_2_17_aarch64_a1ed2dd2\":[{\"filename\":\"cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_cffi_cp311_cp311_manylinux_2_17_s390x_a24ed04c\":[{\"filename\":\"cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_cffi_cp311_cp311_manylinux_2_17_x86_64_610faea7\":[{\"filename\":\"cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_cffi_cp311_cp311_musllinux_1_1_aarch64_a9b15d49\":[{\"filename\":\"cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_cffi_cp311_cp311_musllinux_1_1_x86_64_fc48c783\":[{\"filename\":\"cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_cffi_sdist_1c39c601\":[{\"filename\":\"cffi-1.17.1.tar.gz\",\"version\":\"3.11\"}]}", - "charset_normalizer": "{\"rules_python_publish_deps_311_charset_normalizer_cp311_cp311_macosx_10_9_universal2_0d99dd8f\":[{\"filename\":\"charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_universal2.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_charset_normalizer_cp311_cp311_macosx_10_9_x86_64_c57516e5\":[{\"filename\":\"charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_charset_normalizer_cp311_cp311_macosx_11_0_arm64_6dba5d19\":[{\"filename\":\"charset_normalizer-3.4.0-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_charset_normalizer_cp311_cp311_manylinux_2_17_aarch64_bf4475b8\":[{\"filename\":\"charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_charset_normalizer_cp311_cp311_manylinux_2_17_s390x_8ff4e7cd\":[{\"filename\":\"charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_charset_normalizer_cp311_cp311_manylinux_2_17_x86_64_3710a975\":[{\"filename\":\"charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_charset_normalizer_cp311_cp311_musllinux_1_2_aarch64_47334db7\":[{\"filename\":\"charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_charset_normalizer_cp311_cp311_musllinux_1_2_s390x_63bc5c4a\":[{\"filename\":\"charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_charset_normalizer_cp311_cp311_musllinux_1_2_x86_64_bcb4f8ea\":[{\"filename\":\"charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_charset_normalizer_cp311_cp311_win_amd64_cee4373f\":[{\"filename\":\"charset_normalizer-3.4.0-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_charset_normalizer_py3_none_any_fe9f97fe\":[{\"filename\":\"charset_normalizer-3.4.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_charset_normalizer_sdist_223217c3\":[{\"filename\":\"charset_normalizer-3.4.0.tar.gz\",\"version\":\"3.11\"}]}", - "cryptography": "{\"rules_python_publish_deps_311_cryptography_cp39_abi3_manylinux_2_17_aarch64_846da004\":[{\"filename\":\"cryptography-43.0.3-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_cryptography_cp39_abi3_manylinux_2_17_x86_64_0f996e72\":[{\"filename\":\"cryptography-43.0.3-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_cryptography_cp39_abi3_manylinux_2_28_aarch64_f7b178f1\":[{\"filename\":\"cryptography-43.0.3-cp39-abi3-manylinux_2_28_aarch64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_cryptography_cp39_abi3_manylinux_2_28_x86_64_c2e6fc39\":[{\"filename\":\"cryptography-43.0.3-cp39-abi3-manylinux_2_28_x86_64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_cryptography_cp39_abi3_musllinux_1_2_aarch64_e1be4655\":[{\"filename\":\"cryptography-43.0.3-cp39-abi3-musllinux_1_2_aarch64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_cryptography_cp39_abi3_musllinux_1_2_x86_64_df6b6c6d\":[{\"filename\":\"cryptography-43.0.3-cp39-abi3-musllinux_1_2_x86_64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_cryptography_sdist_315b9001\":[{\"filename\":\"cryptography-43.0.3.tar.gz\",\"version\":\"3.11\"}]}", - "docutils": "{\"rules_python_publish_deps_311_docutils_py3_none_any_dafca5b9\":[{\"filename\":\"docutils-0.21.2-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_docutils_sdist_3a6b1873\":[{\"filename\":\"docutils-0.21.2.tar.gz\",\"version\":\"3.11\"}]}", - "idna": "{\"rules_python_publish_deps_311_idna_py3_none_any_946d195a\":[{\"filename\":\"idna-3.10-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_idna_sdist_12f65c9b\":[{\"filename\":\"idna-3.10.tar.gz\",\"version\":\"3.11\"}]}", - "importlib_metadata": "{\"rules_python_publish_deps_311_importlib_metadata_py3_none_any_45e54197\":[{\"filename\":\"importlib_metadata-8.5.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_importlib_metadata_sdist_71522656\":[{\"filename\":\"importlib_metadata-8.5.0.tar.gz\",\"version\":\"3.11\"}]}", - "jaraco_classes": "{\"rules_python_publish_deps_311_jaraco_classes_py3_none_any_f662826b\":[{\"filename\":\"jaraco.classes-3.4.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_jaraco_classes_sdist_47a024b5\":[{\"filename\":\"jaraco.classes-3.4.0.tar.gz\",\"version\":\"3.11\"}]}", - "jaraco_context": "{\"rules_python_publish_deps_311_jaraco_context_py3_none_any_f797fc48\":[{\"filename\":\"jaraco.context-6.0.1-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_jaraco_context_sdist_9bae4ea5\":[{\"filename\":\"jaraco_context-6.0.1.tar.gz\",\"version\":\"3.11\"}]}", - "jaraco_functools": "{\"rules_python_publish_deps_311_jaraco_functools_py3_none_any_ad159f13\":[{\"filename\":\"jaraco.functools-4.1.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_jaraco_functools_sdist_70f7e0e2\":[{\"filename\":\"jaraco_functools-4.1.0.tar.gz\",\"version\":\"3.11\"}]}", - "jeepney": "{\"rules_python_publish_deps_311_jeepney_py3_none_any_c0a454ad\":[{\"filename\":\"jeepney-0.8.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_jeepney_sdist_5efe48d2\":[{\"filename\":\"jeepney-0.8.0.tar.gz\",\"version\":\"3.11\"}]}", - "keyring": "{\"rules_python_publish_deps_311_keyring_py3_none_any_5426f817\":[{\"filename\":\"keyring-25.4.1-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_keyring_sdist_b07ebc55\":[{\"filename\":\"keyring-25.4.1.tar.gz\",\"version\":\"3.11\"}]}", - "markdown_it_py": "{\"rules_python_publish_deps_311_markdown_it_py_py3_none_any_35521684\":[{\"filename\":\"markdown_it_py-3.0.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_markdown_it_py_sdist_e3f60a94\":[{\"filename\":\"markdown-it-py-3.0.0.tar.gz\",\"version\":\"3.11\"}]}", - "mdurl": "{\"rules_python_publish_deps_311_mdurl_py3_none_any_84008a41\":[{\"filename\":\"mdurl-0.1.2-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_mdurl_sdist_bb413d29\":[{\"filename\":\"mdurl-0.1.2.tar.gz\",\"version\":\"3.11\"}]}", - "more_itertools": "{\"rules_python_publish_deps_311_more_itertools_py3_none_any_037b0d32\":[{\"filename\":\"more_itertools-10.5.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_more_itertools_sdist_5482bfef\":[{\"filename\":\"more-itertools-10.5.0.tar.gz\",\"version\":\"3.11\"}]}", - "nh3": "{\"rules_python_publish_deps_311_nh3_cp37_abi3_macosx_10_12_x86_64_14c5a72e\":[{\"filename\":\"nh3-0.2.18-cp37-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_nh3_cp37_abi3_macosx_10_12_x86_64_7b7c2a3c\":[{\"filename\":\"nh3-0.2.18-cp37-abi3-macosx_10_12_x86_64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_nh3_cp37_abi3_manylinux_2_17_aarch64_42c64511\":[{\"filename\":\"nh3-0.2.18-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_nh3_cp37_abi3_manylinux_2_17_armv7l_0411beb0\":[{\"filename\":\"nh3-0.2.18-cp37-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_nh3_cp37_abi3_manylinux_2_17_ppc64_5f36b271\":[{\"filename\":\"nh3-0.2.18-cp37-abi3-manylinux_2_17_ppc64.manylinux2014_ppc64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_nh3_cp37_abi3_manylinux_2_17_s390x_19aaba96\":[{\"filename\":\"nh3-0.2.18-cp37-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_nh3_cp37_abi3_manylinux_2_17_x86_64_de3ceed6\":[{\"filename\":\"nh3-0.2.18-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_nh3_cp37_abi3_musllinux_1_2_aarch64_f0eca9ca\":[{\"filename\":\"nh3-0.2.18-cp37-abi3-musllinux_1_2_aarch64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_nh3_cp37_abi3_musllinux_1_2_armv7l_3a157ab1\":[{\"filename\":\"nh3-0.2.18-cp37-abi3-musllinux_1_2_armv7l.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_nh3_cp37_abi3_musllinux_1_2_x86_64_36c95d4b\":[{\"filename\":\"nh3-0.2.18-cp37-abi3-musllinux_1_2_x86_64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_nh3_cp37_abi3_win_amd64_8ce0f819\":[{\"filename\":\"nh3-0.2.18-cp37-abi3-win_amd64.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_nh3_sdist_94a16692\":[{\"filename\":\"nh3-0.2.18.tar.gz\",\"version\":\"3.11\"}]}", - "pkginfo": "{\"rules_python_publish_deps_311_pkginfo_py3_none_any_889a6da2\":[{\"filename\":\"pkginfo-1.10.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_pkginfo_sdist_5df73835\":[{\"filename\":\"pkginfo-1.10.0.tar.gz\",\"version\":\"3.11\"}]}", - "pycparser": "{\"rules_python_publish_deps_311_pycparser_py3_none_any_c3702b6d\":[{\"filename\":\"pycparser-2.22-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_pycparser_sdist_491c8be9\":[{\"filename\":\"pycparser-2.22.tar.gz\",\"version\":\"3.11\"}]}", - "pygments": "{\"rules_python_publish_deps_311_pygments_py3_none_any_b8e6aca0\":[{\"filename\":\"pygments-2.18.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_pygments_sdist_786ff802\":[{\"filename\":\"pygments-2.18.0.tar.gz\",\"version\":\"3.11\"}]}", - "pywin32_ctypes": "{\"rules_python_publish_deps_311_pywin32_ctypes_py3_none_any_8a151337\":[{\"filename\":\"pywin32_ctypes-0.2.3-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_pywin32_ctypes_sdist_d162dc04\":[{\"filename\":\"pywin32-ctypes-0.2.3.tar.gz\",\"version\":\"3.11\"}]}", - "readme_renderer": "{\"rules_python_publish_deps_311_readme_renderer_py3_none_any_2fbca89b\":[{\"filename\":\"readme_renderer-44.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_readme_renderer_sdist_8712034e\":[{\"filename\":\"readme_renderer-44.0.tar.gz\",\"version\":\"3.11\"}]}", - "requests": "{\"rules_python_publish_deps_311_requests_py3_none_any_70761cfe\":[{\"filename\":\"requests-2.32.3-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_requests_sdist_55365417\":[{\"filename\":\"requests-2.32.3.tar.gz\",\"version\":\"3.11\"}]}", - "requests_toolbelt": "{\"rules_python_publish_deps_311_requests_toolbelt_py2_none_any_cccfdd66\":[{\"filename\":\"requests_toolbelt-1.0.0-py2.py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_requests_toolbelt_sdist_7681a0a3\":[{\"filename\":\"requests-toolbelt-1.0.0.tar.gz\",\"version\":\"3.11\"}]}", - "rfc3986": "{\"rules_python_publish_deps_311_rfc3986_py2_none_any_50b1502b\":[{\"filename\":\"rfc3986-2.0.0-py2.py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_rfc3986_sdist_97aacf9d\":[{\"filename\":\"rfc3986-2.0.0.tar.gz\",\"version\":\"3.11\"}]}", - "rich": "{\"rules_python_publish_deps_311_rich_py3_none_any_6049d5e6\":[{\"filename\":\"rich-13.9.4-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_rich_sdist_43959497\":[{\"filename\":\"rich-13.9.4.tar.gz\",\"version\":\"3.11\"}]}", - "secretstorage": "{\"rules_python_publish_deps_311_secretstorage_py3_none_any_f356e662\":[{\"filename\":\"SecretStorage-3.3.3-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_secretstorage_sdist_2403533e\":[{\"filename\":\"SecretStorage-3.3.3.tar.gz\",\"version\":\"3.11\"}]}", - "twine": "{\"rules_python_publish_deps_311_twine_py3_none_any_215dbe7b\":[{\"filename\":\"twine-5.1.1-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_twine_sdist_9aa08251\":[{\"filename\":\"twine-5.1.1.tar.gz\",\"version\":\"3.11\"}]}", - "urllib3": "{\"rules_python_publish_deps_311_urllib3_py3_none_any_ca899ca0\":[{\"filename\":\"urllib3-2.2.3-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_urllib3_sdist_e7d814a8\":[{\"filename\":\"urllib3-2.2.3.tar.gz\",\"version\":\"3.11\"}]}", - "zipp": "{\"rules_python_publish_deps_311_zipp_py3_none_any_a817ac80\":[{\"filename\":\"zipp-3.20.2-py3-none-any.whl\",\"version\":\"3.11\"}],\"rules_python_publish_deps_311_zipp_sdist_bc9eb26f\":[{\"filename\":\"zipp-3.20.2.tar.gz\",\"version\":\"3.11\"}]}" - }, - "packages": [ - "backports_tarfile", - "certifi", - "charset_normalizer", - "docutils", - "idna", - "importlib_metadata", - "jaraco_classes", - "jaraco_context", - "jaraco_functools", - "keyring", - "markdown_it_py", - "mdurl", - "more_itertools", - "nh3", - "pkginfo", - "pygments", - "readme_renderer", - "requests", - "requests_toolbelt", - "rfc3986", - "rich", - "twine", - "urllib3", - "zipp" - ], - "groups": {} - } - } - }, - "moduleExtensionMetadata": { - "useAllRepos": "NO", - "reproducible": false - }, - "recordedRepoMappingEntries": [ - [ - "bazel_features+", - "bazel_features_globals", - "bazel_features++version_extension+bazel_features_globals" - ], - [ - "bazel_features+", - "bazel_features_version", - "bazel_features++version_extension+bazel_features_version" - ], - [ - "rules_python+", - "bazel_features", - "bazel_features+" - ], - [ - "rules_python+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "rules_python+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_python+", - "pypi__build", - "rules_python++internal_deps+pypi__build" - ], - [ - "rules_python+", - "pypi__click", - "rules_python++internal_deps+pypi__click" - ], - [ - "rules_python+", - "pypi__colorama", - "rules_python++internal_deps+pypi__colorama" - ], - [ - "rules_python+", - "pypi__importlib_metadata", - "rules_python++internal_deps+pypi__importlib_metadata" - ], - [ - "rules_python+", - "pypi__installer", - "rules_python++internal_deps+pypi__installer" - ], - [ - "rules_python+", - "pypi__more_itertools", - "rules_python++internal_deps+pypi__more_itertools" - ], - [ - "rules_python+", - "pypi__packaging", - "rules_python++internal_deps+pypi__packaging" - ], - [ - "rules_python+", - "pypi__pep517", - "rules_python++internal_deps+pypi__pep517" - ], - [ - "rules_python+", - "pypi__pip", - "rules_python++internal_deps+pypi__pip" - ], - [ - "rules_python+", - "pypi__pip_tools", - "rules_python++internal_deps+pypi__pip_tools" - ], - [ - "rules_python+", - "pypi__pyproject_hooks", - "rules_python++internal_deps+pypi__pyproject_hooks" - ], - [ - "rules_python+", - "pypi__setuptools", - "rules_python++internal_deps+pypi__setuptools" - ], - [ - "rules_python+", - "pypi__tomli", - "rules_python++internal_deps+pypi__tomli" - ], - [ - "rules_python+", - "pypi__wheel", - "rules_python++internal_deps+pypi__wheel" - ], - [ - "rules_python+", - "pypi__zipp", - "rules_python++internal_deps+pypi__zipp" - ], - [ - "rules_python+", - "pythons_hub", - "rules_python++python+pythons_hub" - ], - [ - "rules_python++python+pythons_hub", - "python_3_10_host", - "rules_python++python+python_3_10_host" - ], - [ - "rules_python++python+pythons_hub", - "python_3_11_host", - "rules_python++python+python_3_11_host" - ], - [ - "rules_python++python+pythons_hub", - "python_3_12_host", - "rules_python++python+python_3_12_host" - ], - [ - "rules_python++python+pythons_hub", - "python_3_8_host", - "rules_python++python+python_3_8_host" - ], - [ - "rules_python++python+pythons_hub", - "python_3_9_host", - "rules_python++python+python_3_9_host" - ] - ] - } - }, - "@@rules_python+//python/uv:uv.bzl%uv": { - "general": { - "bzlTransitiveDigest": "UwuEebIvWhq1IDSj2VbBXlMc20ec8cBFyxShHe64sy8=", - "usagesDigest": "4JapxcpS0mL3524k0TZJffAtVyuRjDHZvN9kBRxxF1U=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "uv": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_toolchains_repo.bzl%uv_toolchains_repo", - "attributes": { - "toolchain_type": "'@@rules_python+//python/uv:uv_toolchain_type'", - "toolchain_names": [ - "none" - ], - "toolchain_implementations": { - "none": "'@@rules_python+//python:none'" - }, - "toolchain_compatible_with": { - "none": [ - "@platforms//:incompatible" - ] - }, - "toolchain_target_settings": {} - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_python+", - "platforms", - "platforms" - ] - ] - } - }, - "@@rules_rust+//crate_universe:extensions.bzl%crate": { - "general": { - "bzlTransitiveDigest": "YWlcbEZ/1fdqy0HNBaFalAKz/RswQbhU7oDvWUZm7cQ=", - "usagesDigest": "641905SjTVLGtibJNQNQklx/63ZDVAy9joJU2P7P4ho=", - "recordedFileInputs": { - "@@zml+//zml/tokenizer/hftokenizers/Cargo.lock": "93198160b29c4607994b405ffa0ad1c29b08df1ad22ec9a711874914bbaadbcb", - "@@zml+//zml/tokenizer/hftokenizers/Cargo.toml": "eaf4574383be4d13a653a9c91c052979b05630f1c251b87bc8386263165dc204" - }, - "recordedDirentsInputs": {}, - "envVariables": { - "CARGO_BAZEL_DEBUG": null, - "CARGO_BAZEL_GENERATOR_SHA256": null, - "CARGO_BAZEL_GENERATOR_URL": null, - "CARGO_BAZEL_ISOLATED": null, - "CARGO_BAZEL_REPIN": null, - "CARGO_BAZEL_REPIN_ONLY": null, - "REPIN": null - }, - "generatedRepoSpecs": { - "crates": { - "repoRuleId": "@@rules_rust+//crate_universe:extensions.bzl%_generate_repo", - "attributes": { - "contents": { - "BUILD.bazel": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files(\n [\n \"cargo-bazel.json\",\n \"crates.bzl\",\n \"defs.bzl\",\n ] + glob(\n allow_empty = True,\n include = [\"*.bazel\"],\n ),\n)\n\nfilegroup(\n name = \"srcs\",\n srcs = glob(\n allow_empty = True,\n include = [\n \"*.bazel\",\n \"*.bzl\",\n ],\n ),\n)\n\n# Workspace Member Dependencies\nalias(\n name = \"tokenizers-0.21.2\",\n actual = \"@crates__tokenizers-0.21.2//:tokenizers\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"tokenizers\",\n actual = \"@crates__tokenizers-0.21.2//:tokenizers\",\n tags = [\"manual\"],\n)\n", - "alias_rules.bzl": "\"\"\"Alias that transitions its target to `compilation_mode=opt`. Use `transition_alias=\"opt\"` to enable.\"\"\"\n\nload(\"@rules_cc//cc:defs.bzl\", \"CcInfo\")\nload(\"@rules_rust//rust:rust_common.bzl\", \"COMMON_PROVIDERS\")\n\ndef _transition_alias_impl(ctx):\n # `ctx.attr.actual` is a list of 1 item due to the transition\n providers = [ctx.attr.actual[0][provider] for provider in COMMON_PROVIDERS]\n if CcInfo in ctx.attr.actual[0]:\n providers.append(ctx.attr.actual[0][CcInfo])\n return providers\n\ndef _change_compilation_mode(compilation_mode):\n def _change_compilation_mode_impl(_settings, _attr):\n return {\n \"//command_line_option:compilation_mode\": compilation_mode,\n }\n\n return transition(\n implementation = _change_compilation_mode_impl,\n inputs = [],\n outputs = [\n \"//command_line_option:compilation_mode\",\n ],\n )\n\ndef _transition_alias_rule(compilation_mode):\n return rule(\n implementation = _transition_alias_impl,\n provides = COMMON_PROVIDERS,\n attrs = {\n \"actual\": attr.label(\n mandatory = True,\n doc = \"`rust_library()` target to transition to `compilation_mode=opt`.\",\n providers = COMMON_PROVIDERS,\n cfg = _change_compilation_mode(compilation_mode),\n ),\n \"_allowlist_function_transition\": attr.label(\n default = \"@bazel_tools//tools/allowlists/function_transition_allowlist\",\n ),\n },\n doc = \"Transitions a Rust library crate to the `compilation_mode=opt`.\",\n )\n\ntransition_alias_dbg = _transition_alias_rule(\"dbg\")\ntransition_alias_fastbuild = _transition_alias_rule(\"fastbuild\")\ntransition_alias_opt = _transition_alias_rule(\"opt\")\n", - "defs.bzl": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\"\"\"\n# `crates_repository` API\n\n- [aliases](#aliases)\n- [crate_deps](#crate_deps)\n- [all_crate_deps](#all_crate_deps)\n- [crate_repositories](#crate_repositories)\n\n\"\"\"\n\nload(\"@bazel_tools//tools/build_defs/repo:git.bzl\", \"new_git_repository\")\nload(\"@bazel_tools//tools/build_defs/repo:http.bzl\", \"http_archive\")\nload(\"@bazel_tools//tools/build_defs/repo:utils.bzl\", \"maybe\")\nload(\"@bazel_skylib//lib:selects.bzl\", \"selects\")\nload(\"@rules_rust//crate_universe/private:local_crate_mirror.bzl\", \"local_crate_mirror\")\n\n###############################################################################\n# MACROS API\n###############################################################################\n\n# An identifier that represent common dependencies (unconditional).\n_COMMON_CONDITION = \"\"\n\ndef _flatten_dependency_maps(all_dependency_maps):\n \"\"\"Flatten a list of dependency maps into one dictionary.\n\n Dependency maps have the following structure:\n\n ```python\n DEPENDENCIES_MAP = {\n # The first key in the map is a Bazel package\n # name of the workspace this file is defined in.\n \"workspace_member_package\": {\n\n # Not all dependencies are supported for all platforms.\n # the condition key is the condition required to be true\n # on the host platform.\n \"condition\": {\n\n # An alias to a crate target. # The label of the crate target the\n # Aliases are only crate names. # package name refers to.\n \"package_name\": \"@full//:label\",\n }\n }\n }\n ```\n\n Args:\n all_dependency_maps (list): A list of dicts as described above\n\n Returns:\n dict: A dictionary as described above\n \"\"\"\n dependencies = {}\n\n for workspace_deps_map in all_dependency_maps:\n for pkg_name, conditional_deps_map in workspace_deps_map.items():\n if pkg_name not in dependencies:\n non_frozen_map = dict()\n for key, values in conditional_deps_map.items():\n non_frozen_map.update({key: dict(values.items())})\n dependencies.setdefault(pkg_name, non_frozen_map)\n continue\n\n for condition, deps_map in conditional_deps_map.items():\n # If the condition has not been recorded, do so and continue\n if condition not in dependencies[pkg_name]:\n dependencies[pkg_name].setdefault(condition, dict(deps_map.items()))\n continue\n\n # Alert on any miss-matched dependencies\n inconsistent_entries = []\n for crate_name, crate_label in deps_map.items():\n existing = dependencies[pkg_name][condition].get(crate_name)\n if existing and existing != crate_label:\n inconsistent_entries.append((crate_name, existing, crate_label))\n dependencies[pkg_name][condition].update({crate_name: crate_label})\n\n return dependencies\n\ndef crate_deps(deps, package_name = None):\n \"\"\"Finds the fully qualified label of the requested crates for the package where this macro is called.\n\n Args:\n deps (list): The desired list of crate targets.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()`.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if not deps:\n return []\n\n if package_name == None:\n package_name = native.package_name()\n\n # Join both sets of dependencies\n dependencies = _flatten_dependency_maps([\n _NORMAL_DEPENDENCIES,\n _NORMAL_DEV_DEPENDENCIES,\n _PROC_MACRO_DEPENDENCIES,\n _PROC_MACRO_DEV_DEPENDENCIES,\n _BUILD_DEPENDENCIES,\n _BUILD_PROC_MACRO_DEPENDENCIES,\n ]).pop(package_name, {})\n\n # Combine all conditional packages so we can easily index over a flat list\n # TODO: Perhaps this should actually return select statements and maintain\n # the conditionals of the dependencies\n flat_deps = {}\n for deps_set in dependencies.values():\n for crate_name, crate_label in deps_set.items():\n flat_deps.update({crate_name: crate_label})\n\n missing_crates = []\n crate_targets = []\n for crate_target in deps:\n if crate_target not in flat_deps:\n missing_crates.append(crate_target)\n else:\n crate_targets.append(flat_deps[crate_target])\n\n if missing_crates:\n fail(\"Could not find crates `{}` among dependencies of `{}`. Available dependencies were `{}`\".format(\n missing_crates,\n package_name,\n dependencies,\n ))\n\n return crate_targets\n\ndef all_crate_deps(\n normal = False, \n normal_dev = False, \n proc_macro = False, \n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Finds the fully qualified label of all requested direct crate dependencies \\\n for the package where this macro is called.\n\n If no parameters are set, all normal dependencies are returned. Setting any one flag will\n otherwise impact the contents of the returned list.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_dependency_maps = []\n if normal:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n if normal_dev:\n all_dependency_maps.append(_NORMAL_DEV_DEPENDENCIES)\n if proc_macro:\n all_dependency_maps.append(_PROC_MACRO_DEPENDENCIES)\n if proc_macro_dev:\n all_dependency_maps.append(_PROC_MACRO_DEV_DEPENDENCIES)\n if build:\n all_dependency_maps.append(_BUILD_DEPENDENCIES)\n if build_proc_macro:\n all_dependency_maps.append(_BUILD_PROC_MACRO_DEPENDENCIES)\n\n # Default to always using normal dependencies\n if not all_dependency_maps:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n\n dependencies = _flatten_dependency_maps(all_dependency_maps).pop(package_name, None)\n\n if not dependencies:\n if dependencies == None:\n fail(\"Tried to get all_crate_deps for package \" + package_name + \" but that package had no Cargo.toml file\")\n else:\n return []\n\n crate_deps = list(dependencies.pop(_COMMON_CONDITION, {}).values())\n for condition, deps in dependencies.items():\n crate_deps += selects.with_or({\n tuple(_CONDITIONS[condition]): deps.values(),\n \"//conditions:default\": [],\n })\n\n return crate_deps\n\ndef aliases(\n normal = False,\n normal_dev = False,\n proc_macro = False,\n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Produces a map of Crate alias names to their original label\n\n If no dependency kinds are specified, `normal` and `proc_macro` are used by default.\n Setting any one flag will otherwise determine the contents of the returned dict.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n dict: The aliases of all associated packages\n \"\"\"\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_aliases_maps = []\n if normal:\n all_aliases_maps.append(_NORMAL_ALIASES)\n if normal_dev:\n all_aliases_maps.append(_NORMAL_DEV_ALIASES)\n if proc_macro:\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n if proc_macro_dev:\n all_aliases_maps.append(_PROC_MACRO_DEV_ALIASES)\n if build:\n all_aliases_maps.append(_BUILD_ALIASES)\n if build_proc_macro:\n all_aliases_maps.append(_BUILD_PROC_MACRO_ALIASES)\n\n # Default to always using normal aliases\n if not all_aliases_maps:\n all_aliases_maps.append(_NORMAL_ALIASES)\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n\n aliases = _flatten_dependency_maps(all_aliases_maps).pop(package_name, None)\n\n if not aliases:\n return dict()\n\n common_items = aliases.pop(_COMMON_CONDITION, {}).items()\n\n # If there are only common items in the dictionary, immediately return them\n if not len(aliases.keys()) == 1:\n return dict(common_items)\n\n # Build a single select statement where each conditional has accounted for the\n # common set of aliases.\n crate_aliases = {\"//conditions:default\": dict(common_items)}\n for condition, deps in aliases.items():\n condition_triples = _CONDITIONS[condition]\n for triple in condition_triples:\n if triple in crate_aliases:\n crate_aliases[triple].update(deps)\n else:\n crate_aliases.update({triple: dict(deps.items() + common_items)})\n\n return select(crate_aliases)\n\n###############################################################################\n# WORKSPACE MEMBER DEPS AND ALIASES\n###############################################################################\n\n_NORMAL_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n _COMMON_CONDITION: {\n \"tokenizers\": Label(\"@crates//:tokenizers-0.21.2\"),\n },\n },\n}\n\n\n_NORMAL_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n _COMMON_CONDITION: {\n },\n },\n}\n\n\n_NORMAL_DEV_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_NORMAL_DEV_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEV_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEV_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_CONDITIONS = {\n \"aarch64-apple-darwin\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\"],\n \"aarch64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\"],\n \"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\",\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\": [],\n \"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\": [],\n \"cfg(any())\": [],\n \"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\": [],\n \"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\": [],\n \"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\": [],\n \"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\",\"@rules_rust//rust/platform:x86_64-apple-darwin\"],\n \"cfg(not(all(target_arch = \\\"arm\\\", target_os = \\\"none\\\")))\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\",\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\",\"@rules_rust//rust/platform:x86_64-apple-darwin\",\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"cfg(target_os = \\\"netbsd\\\")\": [],\n \"cfg(target_os = \\\"solaris\\\")\": [],\n \"cfg(target_os = \\\"vxworks\\\")\": [],\n \"x86_64-apple-darwin\": [\"@rules_rust//rust/platform:x86_64-apple-darwin\"],\n \"x86_64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n}\n\n###############################################################################\n\ndef crate_repositories():\n \"\"\"A macro for defining repositories for all generated crates.\n\n Returns:\n A list of repos visible to the module through the module extension.\n \"\"\"\n maybe(\n http_archive,\n name = \"crates__ahash-0.8.12\",\n sha256 = \"5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ahash/0.8.12/download\"],\n strip_prefix = \"ahash-0.8.12\",\n build_file = Label(\"@crates//crates:BUILD.ahash-0.8.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__aho-corasick-1.1.3\",\n sha256 = \"8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/aho-corasick/1.1.3/download\"],\n strip_prefix = \"aho-corasick-1.1.3\",\n build_file = Label(\"@crates//crates:BUILD.aho-corasick-1.1.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__base64-0.13.1\",\n sha256 = \"9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/base64/0.13.1/download\"],\n strip_prefix = \"base64-0.13.1\",\n build_file = Label(\"@crates//crates:BUILD.base64-0.13.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__bit-set-0.8.0\",\n sha256 = \"08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/bit-set/0.8.0/download\"],\n strip_prefix = \"bit-set-0.8.0\",\n build_file = Label(\"@crates//crates:BUILD.bit-set-0.8.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__bit-vec-0.8.0\",\n sha256 = \"5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/bit-vec/0.8.0/download\"],\n strip_prefix = \"bit-vec-0.8.0\",\n build_file = Label(\"@crates//crates:BUILD.bit-vec-0.8.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__bitflags-2.9.1\",\n sha256 = \"1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/bitflags/2.9.1/download\"],\n strip_prefix = \"bitflags-2.9.1\",\n build_file = Label(\"@crates//crates:BUILD.bitflags-2.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__bumpalo-3.16.0\",\n sha256 = \"79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/bumpalo/3.16.0/download\"],\n strip_prefix = \"bumpalo-3.16.0\",\n build_file = Label(\"@crates//crates:BUILD.bumpalo-3.16.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__byteorder-1.5.0\",\n sha256 = \"1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/byteorder/1.5.0/download\"],\n strip_prefix = \"byteorder-1.5.0\",\n build_file = Label(\"@crates//crates:BUILD.byteorder-1.5.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__castaway-0.2.4\",\n sha256 = \"dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/castaway/0.2.4/download\"],\n strip_prefix = \"castaway-0.2.4\",\n build_file = Label(\"@crates//crates:BUILD.castaway-0.2.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__cfg-if-1.0.0\",\n sha256 = \"baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/cfg-if/1.0.0/download\"],\n strip_prefix = \"cfg-if-1.0.0\",\n build_file = Label(\"@crates//crates:BUILD.cfg-if-1.0.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__compact_str-0.9.0\",\n sha256 = \"3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/compact_str/0.9.0/download\"],\n strip_prefix = \"compact_str-0.9.0\",\n build_file = Label(\"@crates//crates:BUILD.compact_str-0.9.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-deque-0.8.6\",\n sha256 = \"9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-deque/0.8.6/download\"],\n strip_prefix = \"crossbeam-deque-0.8.6\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-deque-0.8.6.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-epoch-0.9.18\",\n sha256 = \"5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-epoch/0.9.18/download\"],\n strip_prefix = \"crossbeam-epoch-0.9.18\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-epoch-0.9.18.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-utils-0.8.21\",\n sha256 = \"d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-utils/0.8.21/download\"],\n strip_prefix = \"crossbeam-utils-0.8.21\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-utils-0.8.21.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling-0.20.10\",\n sha256 = \"6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling/0.20.10/download\"],\n strip_prefix = \"darling-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling_core-0.20.10\",\n sha256 = \"95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling_core/0.20.10/download\"],\n strip_prefix = \"darling_core-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling_core-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling_macro-0.20.10\",\n sha256 = \"d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling_macro/0.20.10/download\"],\n strip_prefix = \"darling_macro-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling_macro-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__dary_heap-0.3.7\",\n sha256 = \"04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/dary_heap/0.3.7/download\"],\n strip_prefix = \"dary_heap-0.3.7\",\n build_file = Label(\"@crates//crates:BUILD.dary_heap-0.3.7.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder-0.20.2\",\n sha256 = \"507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder/0.20.2/download\"],\n strip_prefix = \"derive_builder-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder_core-0.20.2\",\n sha256 = \"2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder_core/0.20.2/download\"],\n strip_prefix = \"derive_builder_core-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder_core-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder_macro-0.20.2\",\n sha256 = \"ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder_macro/0.20.2/download\"],\n strip_prefix = \"derive_builder_macro-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder_macro-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__either-1.13.0\",\n sha256 = \"60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/either/1.13.0/download\"],\n strip_prefix = \"either-1.13.0\",\n build_file = Label(\"@crates//crates:BUILD.either-1.13.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__esaxx-rs-0.1.10\",\n sha256 = \"d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/esaxx-rs/0.1.10/download\"],\n strip_prefix = \"esaxx-rs-0.1.10\",\n build_file = Label(\"@crates//crates:BUILD.esaxx-rs-0.1.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__fancy-regex-0.14.0\",\n sha256 = \"6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/fancy-regex/0.14.0/download\"],\n strip_prefix = \"fancy-regex-0.14.0\",\n build_file = Label(\"@crates//crates:BUILD.fancy-regex-0.14.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__fnv-1.0.7\",\n sha256 = \"3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/fnv/1.0.7/download\"],\n strip_prefix = \"fnv-1.0.7\",\n build_file = Label(\"@crates//crates:BUILD.fnv-1.0.7.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__getrandom-0.3.3\",\n sha256 = \"26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/getrandom/0.3.3/download\"],\n strip_prefix = \"getrandom-0.3.3\",\n build_file = Label(\"@crates//crates:BUILD.getrandom-0.3.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ident_case-1.0.1\",\n sha256 = \"b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ident_case/1.0.1/download\"],\n strip_prefix = \"ident_case-1.0.1\",\n build_file = Label(\"@crates//crates:BUILD.ident_case-1.0.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__itertools-0.14.0\",\n sha256 = \"2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/itertools/0.14.0/download\"],\n strip_prefix = \"itertools-0.14.0\",\n build_file = Label(\"@crates//crates:BUILD.itertools-0.14.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__itoa-1.0.14\",\n sha256 = \"d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/itoa/1.0.14/download\"],\n strip_prefix = \"itoa-1.0.14\",\n build_file = Label(\"@crates//crates:BUILD.itoa-1.0.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__js-sys-0.3.77\",\n sha256 = \"1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/js-sys/0.3.77/download\"],\n strip_prefix = \"js-sys-0.3.77\",\n build_file = Label(\"@crates//crates:BUILD.js-sys-0.3.77.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__libc-0.2.169\",\n sha256 = \"b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/libc/0.2.169/download\"],\n strip_prefix = \"libc-0.2.169\",\n build_file = Label(\"@crates//crates:BUILD.libc-0.2.169.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__log-0.4.22\",\n sha256 = \"a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/log/0.4.22/download\"],\n strip_prefix = \"log-0.4.22\",\n build_file = Label(\"@crates//crates:BUILD.log-0.4.22.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__macro_rules_attribute-0.2.0\",\n sha256 = \"8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/macro_rules_attribute/0.2.0/download\"],\n strip_prefix = \"macro_rules_attribute-0.2.0\",\n build_file = Label(\"@crates//crates:BUILD.macro_rules_attribute-0.2.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__macro_rules_attribute-proc_macro-0.2.0\",\n sha256 = \"b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/macro_rules_attribute-proc_macro/0.2.0/download\"],\n strip_prefix = \"macro_rules_attribute-proc_macro-0.2.0\",\n build_file = Label(\"@crates//crates:BUILD.macro_rules_attribute-proc_macro-0.2.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__memchr-2.7.4\",\n sha256 = \"78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/memchr/2.7.4/download\"],\n strip_prefix = \"memchr-2.7.4\",\n build_file = Label(\"@crates//crates:BUILD.memchr-2.7.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__minimal-lexical-0.2.1\",\n sha256 = \"68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/minimal-lexical/0.2.1/download\"],\n strip_prefix = \"minimal-lexical-0.2.1\",\n build_file = Label(\"@crates//crates:BUILD.minimal-lexical-0.2.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__monostate-0.1.13\",\n sha256 = \"0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/monostate/0.1.13/download\"],\n strip_prefix = \"monostate-0.1.13\",\n build_file = Label(\"@crates//crates:BUILD.monostate-0.1.13.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__monostate-impl-0.1.13\",\n sha256 = \"a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/monostate-impl/0.1.13/download\"],\n strip_prefix = \"monostate-impl-0.1.13\",\n build_file = Label(\"@crates//crates:BUILD.monostate-impl-0.1.13.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__nom-7.1.3\",\n sha256 = \"d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/nom/7.1.3/download\"],\n strip_prefix = \"nom-7.1.3\",\n build_file = Label(\"@crates//crates:BUILD.nom-7.1.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__once_cell-1.20.2\",\n sha256 = \"1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/once_cell/1.20.2/download\"],\n strip_prefix = \"once_cell-1.20.2\",\n build_file = Label(\"@crates//crates:BUILD.once_cell-1.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__paste-1.0.15\",\n sha256 = \"57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/paste/1.0.15/download\"],\n strip_prefix = \"paste-1.0.15\",\n build_file = Label(\"@crates//crates:BUILD.paste-1.0.15.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ppv-lite86-0.2.20\",\n sha256 = \"77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ppv-lite86/0.2.20/download\"],\n strip_prefix = \"ppv-lite86-0.2.20\",\n build_file = Label(\"@crates//crates:BUILD.ppv-lite86-0.2.20.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__proc-macro2-1.0.92\",\n sha256 = \"37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/proc-macro2/1.0.92/download\"],\n strip_prefix = \"proc-macro2-1.0.92\",\n build_file = Label(\"@crates//crates:BUILD.proc-macro2-1.0.92.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__quote-1.0.37\",\n sha256 = \"b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/quote/1.0.37/download\"],\n strip_prefix = \"quote-1.0.37\",\n build_file = Label(\"@crates//crates:BUILD.quote-1.0.37.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__r-efi-5.3.0\",\n sha256 = \"69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/r-efi/5.3.0/download\"],\n strip_prefix = \"r-efi-5.3.0\",\n build_file = Label(\"@crates//crates:BUILD.r-efi-5.3.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand-0.9.1\",\n sha256 = \"9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand/0.9.1/download\"],\n strip_prefix = \"rand-0.9.1\",\n build_file = Label(\"@crates//crates:BUILD.rand-0.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand_chacha-0.9.0\",\n sha256 = \"d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand_chacha/0.9.0/download\"],\n strip_prefix = \"rand_chacha-0.9.0\",\n build_file = Label(\"@crates//crates:BUILD.rand_chacha-0.9.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand_core-0.9.3\",\n sha256 = \"99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand_core/0.9.3/download\"],\n strip_prefix = \"rand_core-0.9.3\",\n build_file = Label(\"@crates//crates:BUILD.rand_core-0.9.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-1.10.0\",\n sha256 = \"b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon/1.10.0/download\"],\n strip_prefix = \"rayon-1.10.0\",\n build_file = Label(\"@crates//crates:BUILD.rayon-1.10.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-cond-0.4.0\",\n sha256 = \"2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon-cond/0.4.0/download\"],\n strip_prefix = \"rayon-cond-0.4.0\",\n build_file = Label(\"@crates//crates:BUILD.rayon-cond-0.4.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-core-1.12.1\",\n sha256 = \"1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon-core/1.12.1/download\"],\n strip_prefix = \"rayon-core-1.12.1\",\n build_file = Label(\"@crates//crates:BUILD.rayon-core-1.12.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-1.11.1\",\n sha256 = \"b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex/1.11.1/download\"],\n strip_prefix = \"regex-1.11.1\",\n build_file = Label(\"@crates//crates:BUILD.regex-1.11.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-automata-0.4.9\",\n sha256 = \"809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-automata/0.4.9/download\"],\n strip_prefix = \"regex-automata-0.4.9\",\n build_file = Label(\"@crates//crates:BUILD.regex-automata-0.4.9.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-syntax-0.8.5\",\n sha256 = \"2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-syntax/0.8.5/download\"],\n strip_prefix = \"regex-syntax-0.8.5\",\n build_file = Label(\"@crates//crates:BUILD.regex-syntax-0.8.5.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rustversion-1.0.21\",\n sha256 = \"8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rustversion/1.0.21/download\"],\n strip_prefix = \"rustversion-1.0.21\",\n build_file = Label(\"@crates//crates:BUILD.rustversion-1.0.21.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ryu-1.0.18\",\n sha256 = \"f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ryu/1.0.18/download\"],\n strip_prefix = \"ryu-1.0.18\",\n build_file = Label(\"@crates//crates:BUILD.ryu-1.0.18.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde-1.0.216\",\n sha256 = \"0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde/1.0.216/download\"],\n strip_prefix = \"serde-1.0.216\",\n build_file = Label(\"@crates//crates:BUILD.serde-1.0.216.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde_derive-1.0.216\",\n sha256 = \"46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde_derive/1.0.216/download\"],\n strip_prefix = \"serde_derive-1.0.216\",\n build_file = Label(\"@crates//crates:BUILD.serde_derive-1.0.216.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde_json-1.0.134\",\n sha256 = \"d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde_json/1.0.134/download\"],\n strip_prefix = \"serde_json-1.0.134\",\n build_file = Label(\"@crates//crates:BUILD.serde_json-1.0.134.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__smallvec-1.13.2\",\n sha256 = \"3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/smallvec/1.13.2/download\"],\n strip_prefix = \"smallvec-1.13.2\",\n build_file = Label(\"@crates//crates:BUILD.smallvec-1.13.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__spm_precompiled-0.1.4\",\n sha256 = \"5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/spm_precompiled/0.1.4/download\"],\n strip_prefix = \"spm_precompiled-0.1.4\",\n build_file = Label(\"@crates//crates:BUILD.spm_precompiled-0.1.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__static_assertions-1.1.0\",\n sha256 = \"a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/static_assertions/1.1.0/download\"],\n strip_prefix = \"static_assertions-1.1.0\",\n build_file = Label(\"@crates//crates:BUILD.static_assertions-1.1.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__strsim-0.11.1\",\n sha256 = \"7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/strsim/0.11.1/download\"],\n strip_prefix = \"strsim-0.11.1\",\n build_file = Label(\"@crates//crates:BUILD.strsim-0.11.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__syn-2.0.91\",\n sha256 = \"d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/syn/2.0.91/download\"],\n strip_prefix = \"syn-2.0.91\",\n build_file = Label(\"@crates//crates:BUILD.syn-2.0.91.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__thiserror-2.0.12\",\n sha256 = \"567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/thiserror/2.0.12/download\"],\n strip_prefix = \"thiserror-2.0.12\",\n build_file = Label(\"@crates//crates:BUILD.thiserror-2.0.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__thiserror-impl-2.0.12\",\n sha256 = \"7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/thiserror-impl/2.0.12/download\"],\n strip_prefix = \"thiserror-impl-2.0.12\",\n build_file = Label(\"@crates//crates:BUILD.thiserror-impl-2.0.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__tokenizers-0.21.2\",\n sha256 = \"4c3846d8588abed0daba25a0e47edd58ea15e450a6088b2575f5116fdb0b27ca\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/tokenizers/0.21.2/download\"],\n strip_prefix = \"tokenizers-0.21.2\",\n build_file = Label(\"@crates//crates:BUILD.tokenizers-0.21.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-ident-1.0.14\",\n sha256 = \"adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-ident/1.0.14/download\"],\n strip_prefix = \"unicode-ident-1.0.14\",\n build_file = Label(\"@crates//crates:BUILD.unicode-ident-1.0.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-normalization-alignments-0.1.12\",\n sha256 = \"43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-normalization-alignments/0.1.12/download\"],\n strip_prefix = \"unicode-normalization-alignments-0.1.12\",\n build_file = Label(\"@crates//crates:BUILD.unicode-normalization-alignments-0.1.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-segmentation-1.12.0\",\n sha256 = \"f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-segmentation/1.12.0/download\"],\n strip_prefix = \"unicode-segmentation-1.12.0\",\n build_file = Label(\"@crates//crates:BUILD.unicode-segmentation-1.12.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode_categories-0.1.1\",\n sha256 = \"39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode_categories/0.1.1/download\"],\n strip_prefix = \"unicode_categories-0.1.1\",\n build_file = Label(\"@crates//crates:BUILD.unicode_categories-0.1.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__version_check-0.9.5\",\n sha256 = \"0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/version_check/0.9.5/download\"],\n strip_prefix = \"version_check-0.9.5\",\n build_file = Label(\"@crates//crates:BUILD.version_check-0.9.5.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasi-0.14.2-wasi-0.2.4\",\n sha256 = \"9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasi/0.14.2+wasi-0.2.4/download\"],\n strip_prefix = \"wasi-0.14.2+wasi-0.2.4\",\n build_file = Label(\"@crates//crates:BUILD.wasi-0.14.2+wasi-0.2.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasm-bindgen-0.2.100\",\n sha256 = \"1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasm-bindgen/0.2.100/download\"],\n strip_prefix = \"wasm-bindgen-0.2.100\",\n build_file = Label(\"@crates//crates:BUILD.wasm-bindgen-0.2.100.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasm-bindgen-backend-0.2.100\",\n sha256 = \"2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasm-bindgen-backend/0.2.100/download\"],\n strip_prefix = \"wasm-bindgen-backend-0.2.100\",\n build_file = Label(\"@crates//crates:BUILD.wasm-bindgen-backend-0.2.100.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasm-bindgen-macro-0.2.100\",\n sha256 = \"7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasm-bindgen-macro/0.2.100/download\"],\n strip_prefix = \"wasm-bindgen-macro-0.2.100\",\n build_file = Label(\"@crates//crates:BUILD.wasm-bindgen-macro-0.2.100.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasm-bindgen-macro-support-0.2.100\",\n sha256 = \"8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasm-bindgen-macro-support/0.2.100/download\"],\n strip_prefix = \"wasm-bindgen-macro-support-0.2.100\",\n build_file = Label(\"@crates//crates:BUILD.wasm-bindgen-macro-support-0.2.100.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasm-bindgen-shared-0.2.100\",\n sha256 = \"1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasm-bindgen-shared/0.2.100/download\"],\n strip_prefix = \"wasm-bindgen-shared-0.2.100\",\n build_file = Label(\"@crates//crates:BUILD.wasm-bindgen-shared-0.2.100.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wit-bindgen-rt-0.39.0\",\n sha256 = \"6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wit-bindgen-rt/0.39.0/download\"],\n strip_prefix = \"wit-bindgen-rt-0.39.0\",\n build_file = Label(\"@crates//crates:BUILD.wit-bindgen-rt-0.39.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-0.7.35\",\n sha256 = \"1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy/0.7.35/download\"],\n strip_prefix = \"zerocopy-0.7.35\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-0.7.35.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-0.8.26\",\n sha256 = \"1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy/0.8.26/download\"],\n strip_prefix = \"zerocopy-0.8.26\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-0.8.26.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-derive-0.7.35\",\n sha256 = \"fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy-derive/0.7.35/download\"],\n strip_prefix = \"zerocopy-derive-0.7.35\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-derive-0.7.35.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-derive-0.8.26\",\n sha256 = \"9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy-derive/0.8.26/download\"],\n strip_prefix = \"zerocopy-derive-0.8.26\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-derive-0.8.26.bazel\"),\n )\n\n return [\n struct(repo=\"crates__tokenizers-0.21.2\", is_dev_dep = False),\n ]\n" - } - } - }, - "crates__ahash-0.8.12": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/ahash/0.8.12/download" - ], - "strip_prefix": "ahash-0.8.12", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"ahash\",\n deps = [\n \"@crates__ahash-0.8.12//:build_script_build\",\n \"@crates__cfg-if-1.0.0//:cfg_if\",\n \"@crates__getrandom-0.3.3//:getrandom\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__zerocopy-0.8.26//:zerocopy\",\n ] + select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [\n \"@crates__once_cell-1.20.2//:once_cell\", # cfg(not(all(target_arch = \"arm\", target_os = \"none\")))\n ],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [\n \"@crates__once_cell-1.20.2//:once_cell\", # cfg(not(all(target_arch = \"arm\", target_os = \"none\")))\n ],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [\n \"@crates__once_cell-1.20.2//:once_cell\", # cfg(not(all(target_arch = \"arm\", target_os = \"none\")))\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [\n \"@crates__once_cell-1.20.2//:once_cell\", # cfg(not(all(target_arch = \"arm\", target_os = \"none\")))\n ],\n \"//conditions:default\": [],\n }),\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"getrandom\",\n \"runtime-rng\",\n \"serde\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=ahash\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.12\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"getrandom\",\n \"runtime-rng\",\n \"serde\",\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n deps = [\n \"@crates__version_check-0.9.5//:version_check\",\n ],\n edition = \"2018\",\n pkg_name = \"ahash\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=ahash\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.8.12\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__aho-corasick-1.1.3": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/aho-corasick/1.1.3/download" - ], - "strip_prefix": "aho-corasick-1.1.3", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"aho_corasick\",\n deps = [\n \"@crates__memchr-2.7.4//:memchr\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"perf-literal\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=aho-corasick\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.1.3\",\n)\n" - } - }, - "crates__base64-0.13.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/base64/0.13.1/download" - ], - "strip_prefix": "base64-0.13.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"base64\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=base64\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.13.1\",\n)\n" - } - }, - "crates__bit-set-0.8.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/bit-set/0.8.0/download" - ], - "strip_prefix": "bit-set-0.8.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"bit_set\",\n deps = [\n \"@crates__bit-vec-0.8.0//:bit_vec\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=bit-set\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.0\",\n)\n" - } - }, - "crates__bit-vec-0.8.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/bit-vec/0.8.0/download" - ], - "strip_prefix": "bit-vec-0.8.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"bit_vec\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=bit-vec\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.0\",\n)\n" - } - }, - "crates__bitflags-2.9.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/bitflags/2.9.1/download" - ], - "strip_prefix": "bitflags-2.9.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"bitflags\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=bitflags\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.9.1\",\n)\n" - } - }, - "crates__bumpalo-3.16.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/bumpalo/3.16.0/download" - ], - "strip_prefix": "bumpalo-3.16.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"bumpalo\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=bumpalo\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"3.16.0\",\n)\n" - } - }, - "crates__byteorder-1.5.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/byteorder/1.5.0/download" - ], - "strip_prefix": "byteorder-1.5.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"byteorder\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=byteorder\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.5.0\",\n)\n" - } - }, - "crates__castaway-0.2.4": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/castaway/0.2.4/download" - ], - "strip_prefix": "castaway-0.2.4", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"castaway\",\n proc_macro_deps = [\n \"@crates__rustversion-1.0.21//:rustversion\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=castaway\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.4\",\n)\n" - } - }, - "crates__cfg-if-1.0.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/cfg-if/1.0.0/download" - ], - "strip_prefix": "cfg-if-1.0.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"cfg_if\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=cfg-if\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.0\",\n)\n" - } - }, - "crates__compact_str-0.9.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/compact_str/0.9.0/download" - ], - "strip_prefix": "compact_str-0.9.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"compact_str\",\n deps = [\n \"@crates__castaway-0.2.4//:castaway\",\n \"@crates__cfg-if-1.0.0//:cfg_if\",\n \"@crates__itoa-1.0.14//:itoa\",\n \"@crates__ryu-1.0.18//:ryu\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__static_assertions-1.1.0//:static_assertions\",\n ],\n proc_macro_deps = [\n \"@crates__rustversion-1.0.21//:rustversion\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"serde\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=compact_str\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.9.0\",\n)\n" - } - }, - "crates__crossbeam-deque-0.8.6": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/crossbeam-deque/0.8.6/download" - ], - "strip_prefix": "crossbeam-deque-0.8.6", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"crossbeam_deque\",\n deps = [\n \"@crates__crossbeam-epoch-0.9.18//:crossbeam_epoch\",\n \"@crates__crossbeam-utils-0.8.21//:crossbeam_utils\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=crossbeam-deque\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.6\",\n)\n" - } - }, - "crates__crossbeam-epoch-0.9.18": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/crossbeam-epoch/0.9.18/download" - ], - "strip_prefix": "crossbeam-epoch-0.9.18", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"crossbeam_epoch\",\n deps = [\n \"@crates__crossbeam-utils-0.8.21//:crossbeam_utils\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=crossbeam-epoch\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.9.18\",\n)\n" - } - }, - "crates__crossbeam-utils-0.8.21": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/crossbeam-utils/0.8.21/download" - ], - "strip_prefix": "crossbeam-utils-0.8.21", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"crossbeam_utils\",\n deps = [\n \"@crates__crossbeam-utils-0.8.21//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=crossbeam-utils\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.21\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"crossbeam-utils\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=crossbeam-utils\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.8.21\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__darling-0.20.10": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/darling/0.20.10/download" - ], - "strip_prefix": "darling-0.20.10", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"darling\",\n deps = [\n \"@crates__darling_core-0.20.10//:darling_core\",\n ],\n proc_macro_deps = [\n \"@crates__darling_macro-0.20.10//:darling_macro\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"suggestions\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=darling\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.20.10\",\n)\n" - } - }, - "crates__darling_core-0.20.10": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/darling_core/0.20.10/download" - ], - "strip_prefix": "darling_core-0.20.10", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"darling_core\",\n deps = [\n \"@crates__fnv-1.0.7//:fnv\",\n \"@crates__ident_case-1.0.1//:ident_case\",\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__strsim-0.11.1//:strsim\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"strsim\",\n \"suggestions\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=darling_core\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.20.10\",\n)\n" - } - }, - "crates__darling_macro-0.20.10": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/darling_macro/0.20.10/download" - ], - "strip_prefix": "darling_macro-0.20.10", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"darling_macro\",\n deps = [\n \"@crates__darling_core-0.20.10//:darling_core\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=darling_macro\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.20.10\",\n)\n" - } - }, - "crates__dary_heap-0.3.7": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/dary_heap/0.3.7/download" - ], - "strip_prefix": "dary_heap-0.3.7", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"dary_heap\",\n deps = [\n \"@crates__serde-1.0.216//:serde\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"serde\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=dary_heap\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.7\",\n)\n" - } - }, - "crates__derive_builder-0.20.2": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/derive_builder/0.20.2/download" - ], - "strip_prefix": "derive_builder-0.20.2", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"derive_builder\",\n proc_macro_deps = [\n \"@crates__derive_builder_macro-0.20.2//:derive_builder_macro\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=derive_builder\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.20.2\",\n)\n" - } - }, - "crates__derive_builder_core-0.20.2": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/derive_builder_core/0.20.2/download" - ], - "strip_prefix": "derive_builder_core-0.20.2", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"derive_builder_core\",\n deps = [\n \"@crates__darling-0.20.10//:darling\",\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"lib_has_std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=derive_builder_core\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.20.2\",\n)\n" - } - }, - "crates__derive_builder_macro-0.20.2": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/derive_builder_macro/0.20.2/download" - ], - "strip_prefix": "derive_builder_macro-0.20.2", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"derive_builder_macro\",\n deps = [\n \"@crates__derive_builder_core-0.20.2//:derive_builder_core\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"lib_has_std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=derive_builder_macro\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.20.2\",\n)\n" - } - }, - "crates__either-1.13.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/either/1.13.0/download" - ], - "strip_prefix": "either-1.13.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"either\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"use_std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=either\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.13.0\",\n)\n" - } - }, - "crates__esaxx-rs-0.1.10": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/esaxx-rs/0.1.10/download" - ], - "strip_prefix": "esaxx-rs-0.1.10", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"esaxx_rs\",\n deps = [\n \"@crates__esaxx-rs-0.1.10//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=esaxx-rs\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.10\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2018\",\n pkg_name = \"esaxx-rs\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=esaxx-rs\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.1.10\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__fancy-regex-0.14.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/fancy-regex/0.14.0/download" - ], - "strip_prefix": "fancy-regex-0.14.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"fancy_regex\",\n deps = [\n \"@crates__bit-set-0.8.0//:bit_set\",\n \"@crates__regex-automata-0.4.9//:regex_automata\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"perf\",\n \"std\",\n \"unicode\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=fancy-regex\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.14.0\",\n)\n" - } - }, - "crates__fnv-1.0.7": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/fnv/1.0.7/download" - ], - "strip_prefix": "fnv-1.0.7", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"fnv\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=fnv\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.7\",\n)\n" - } - }, - "crates__getrandom-0.3.3": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/getrandom/0.3.3/download" - ], - "strip_prefix": "getrandom-0.3.3", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"getrandom\",\n deps = [\n \"@crates__cfg-if-1.0.0//:cfg_if\",\n \"@crates__getrandom-0.3.3//:build_script_build\",\n ] + select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(any(target_os = \"macos\", target_os = \"openbsd\", target_os = \"vita\", target_os = \"emscripten\"))\n ],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(all(any(target_os = \"linux\", target_os = \"android\"), not(any(all(target_os = \"linux\", target_env = \"\"), getrandom_backend = \"custom\", getrandom_backend = \"linux_raw\", getrandom_backend = \"rdrand\", getrandom_backend = \"rndr\"))))\n ],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(any(target_os = \"macos\", target_os = \"openbsd\", target_os = \"vita\", target_os = \"emscripten\"))\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(all(any(target_os = \"linux\", target_os = \"android\"), not(any(all(target_os = \"linux\", target_env = \"\"), getrandom_backend = \"custom\", getrandom_backend = \"linux_raw\", getrandom_backend = \"rdrand\", getrandom_backend = \"rndr\"))))\n ],\n \"//conditions:default\": [],\n }),\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n \"wasm_js\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=getrandom\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.3\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n \"wasm_js\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"getrandom\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=getrandom\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.3.3\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__ident_case-1.0.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/ident_case/1.0.1/download" - ], - "strip_prefix": "ident_case-1.0.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"ident_case\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=ident_case\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.1\",\n)\n" - } - }, - "crates__itertools-0.14.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/itertools/0.14.0/download" - ], - "strip_prefix": "itertools-0.14.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"itertools\",\n deps = [\n \"@crates__either-1.13.0//:either\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"use_alloc\",\n \"use_std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=itertools\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.14.0\",\n)\n" - } - }, - "crates__itoa-1.0.14": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/itoa/1.0.14/download" - ], - "strip_prefix": "itoa-1.0.14", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"itoa\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=itoa\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.14\",\n)\n" - } - }, - "crates__js-sys-0.3.77": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/js-sys/0.3.77/download" - ], - "strip_prefix": "js-sys-0.3.77", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"js_sys\",\n deps = [\n \"@crates__once_cell-1.20.2//:once_cell\",\n \"@crates__wasm-bindgen-0.2.100//:wasm_bindgen\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=js-sys\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.77\",\n)\n" - } - }, - "crates__libc-0.2.169": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/libc/0.2.169/download" - ], - "strip_prefix": "libc-0.2.169", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"libc\",\n deps = [\n \"@crates__libc-0.2.169//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=libc\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.169\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"libc\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=libc\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.2.169\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__log-0.4.22": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/log/0.4.22/download" - ], - "strip_prefix": "log-0.4.22", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"log\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=log\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.4.22\",\n)\n" - } - }, - "crates__macro_rules_attribute-0.2.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/macro_rules_attribute/0.2.0/download" - ], - "strip_prefix": "macro_rules_attribute-0.2.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"macro_rules_attribute\",\n proc_macro_deps = [\n \"@crates__macro_rules_attribute-proc_macro-0.2.0//:macro_rules_attribute_proc_macro\",\n \"@crates__paste-1.0.15//:paste\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=macro_rules_attribute\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.0\",\n)\n" - } - }, - "crates__macro_rules_attribute-proc_macro-0.2.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/macro_rules_attribute-proc_macro/0.2.0/download" - ], - "strip_prefix": "macro_rules_attribute-proc_macro-0.2.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"macro_rules_attribute_proc_macro\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"mod.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=macro_rules_attribute-proc_macro\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.0\",\n)\n" - } - }, - "crates__memchr-2.7.4": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/memchr/2.7.4/download" - ], - "strip_prefix": "memchr-2.7.4", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"memchr\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=memchr\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.7.4\",\n)\n" - } - }, - "crates__minimal-lexical-0.2.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/minimal-lexical/0.2.1/download" - ], - "strip_prefix": "minimal-lexical-0.2.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"minimal_lexical\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=minimal-lexical\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.1\",\n)\n" - } - }, - "crates__monostate-0.1.13": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/monostate/0.1.13/download" - ], - "strip_prefix": "monostate-0.1.13", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"monostate\",\n deps = [\n \"@crates__serde-1.0.216//:serde\",\n ],\n proc_macro_deps = [\n \"@crates__monostate-impl-0.1.13//:monostate_impl\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=monostate\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.13\",\n)\n" - } - }, - "crates__monostate-impl-0.1.13": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/monostate-impl/0.1.13/download" - ], - "strip_prefix": "monostate-impl-0.1.13", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"monostate_impl\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=monostate-impl\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.13\",\n)\n" - } - }, - "crates__nom-7.1.3": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/nom/7.1.3/download" - ], - "strip_prefix": "nom-7.1.3", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"nom\",\n deps = [\n \"@crates__memchr-2.7.4//:memchr\",\n \"@crates__minimal-lexical-0.2.1//:minimal_lexical\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=nom\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"7.1.3\",\n)\n" - } - }, - "crates__once_cell-1.20.2": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/once_cell/1.20.2/download" - ], - "strip_prefix": "once_cell-1.20.2", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"once_cell\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"race\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=once_cell\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.20.2\",\n)\n" - } - }, - "crates__paste-1.0.15": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/paste/1.0.15/download" - ], - "strip_prefix": "paste-1.0.15", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"paste\",\n deps = [\n \"@crates__paste-1.0.15//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=paste\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.15\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2018\",\n pkg_name = \"paste\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=paste\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.15\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__ppv-lite86-0.2.20": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/ppv-lite86/0.2.20/download" - ], - "strip_prefix": "ppv-lite86-0.2.20", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"ppv_lite86\",\n deps = [\n \"@crates__zerocopy-0.7.35//:zerocopy\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"simd\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=ppv-lite86\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.20\",\n)\n" - } - }, - "crates__proc-macro2-1.0.92": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/proc-macro2/1.0.92/download" - ], - "strip_prefix": "proc-macro2-1.0.92", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"proc_macro2\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:build_script_build\",\n \"@crates__unicode-ident-1.0.14//:unicode_ident\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"proc-macro\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=proc-macro2\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.92\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"proc-macro\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"proc-macro2\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=proc-macro2\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.92\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__quote-1.0.37": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/quote/1.0.37/download" - ], - "strip_prefix": "quote-1.0.37", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"quote\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"proc-macro\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=quote\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.37\",\n)\n" - } - }, - "crates__r-efi-5.3.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/r-efi/5.3.0/download" - ], - "strip_prefix": "r-efi-5.3.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"r_efi\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=r-efi\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"5.3.0\",\n)\n" - } - }, - "crates__rand-0.9.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/rand/0.9.1/download" - ], - "strip_prefix": "rand-0.9.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"rand\",\n deps = [\n \"@crates__rand_chacha-0.9.0//:rand_chacha\",\n \"@crates__rand_core-0.9.3//:rand_core\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"default\",\n \"os_rng\",\n \"small_rng\",\n \"std\",\n \"std_rng\",\n \"thread_rng\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rand\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.9.1\",\n)\n" - } - }, - "crates__rand_chacha-0.9.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/rand_chacha/0.9.0/download" - ], - "strip_prefix": "rand_chacha-0.9.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"rand_chacha\",\n deps = [\n \"@crates__ppv-lite86-0.2.20//:ppv_lite86\",\n \"@crates__rand_core-0.9.3//:rand_core\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rand_chacha\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.9.0\",\n)\n" - } - }, - "crates__rand_core-0.9.3": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/rand_core/0.9.3/download" - ], - "strip_prefix": "rand_core-0.9.3", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"rand_core\",\n deps = [\n \"@crates__getrandom-0.3.3//:getrandom\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"os_rng\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rand_core\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.9.3\",\n)\n" - } - }, - "crates__rayon-1.10.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/rayon/1.10.0/download" - ], - "strip_prefix": "rayon-1.10.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"rayon\",\n deps = [\n \"@crates__either-1.13.0//:either\",\n \"@crates__rayon-core-1.12.1//:rayon_core\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rayon\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.10.0\",\n)\n" - } - }, - "crates__rayon-cond-0.4.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/rayon-cond/0.4.0/download" - ], - "strip_prefix": "rayon-cond-0.4.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"rayon_cond\",\n deps = [\n \"@crates__either-1.13.0//:either\",\n \"@crates__itertools-0.14.0//:itertools\",\n \"@crates__rayon-1.10.0//:rayon\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rayon-cond\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.4.0\",\n)\n" - } - }, - "crates__rayon-core-1.12.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/rayon-core/1.12.1/download" - ], - "strip_prefix": "rayon-core-1.12.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"rayon_core\",\n deps = [\n \"@crates__crossbeam-deque-0.8.6//:crossbeam_deque\",\n \"@crates__crossbeam-utils-0.8.21//:crossbeam_utils\",\n \"@crates__rayon-core-1.12.1//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rayon-core\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.12.1\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n links = \"rayon-core\",\n pkg_name = \"rayon-core\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rayon-core\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.12.1\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__regex-1.11.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/regex/1.11.1/download" - ], - "strip_prefix": "regex-1.11.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"regex\",\n deps = [\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__memchr-2.7.4//:memchr\",\n \"@crates__regex-automata-0.4.9//:regex_automata\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"perf\",\n \"perf-backtrack\",\n \"perf-cache\",\n \"perf-dfa\",\n \"perf-inline\",\n \"perf-literal\",\n \"perf-onepass\",\n \"std\",\n \"unicode\",\n \"unicode-age\",\n \"unicode-bool\",\n \"unicode-case\",\n \"unicode-gencat\",\n \"unicode-perl\",\n \"unicode-script\",\n \"unicode-segment\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=regex\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.11.1\",\n)\n" - } - }, - "crates__regex-automata-0.4.9": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/regex-automata/0.4.9/download" - ], - "strip_prefix": "regex-automata-0.4.9", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"regex_automata\",\n deps = [\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__memchr-2.7.4//:memchr\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"dfa\",\n \"dfa-build\",\n \"dfa-onepass\",\n \"dfa-search\",\n \"hybrid\",\n \"meta\",\n \"nfa\",\n \"nfa-backtrack\",\n \"nfa-pikevm\",\n \"nfa-thompson\",\n \"perf\",\n \"perf-inline\",\n \"perf-literal\",\n \"perf-literal-multisubstring\",\n \"perf-literal-substring\",\n \"std\",\n \"syntax\",\n \"unicode\",\n \"unicode-age\",\n \"unicode-bool\",\n \"unicode-case\",\n \"unicode-gencat\",\n \"unicode-perl\",\n \"unicode-script\",\n \"unicode-segment\",\n \"unicode-word-boundary\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=regex-automata\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.4.9\",\n)\n" - } - }, - "crates__regex-syntax-0.8.5": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/regex-syntax/0.8.5/download" - ], - "strip_prefix": "regex-syntax-0.8.5", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"regex_syntax\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n \"unicode\",\n \"unicode-age\",\n \"unicode-bool\",\n \"unicode-case\",\n \"unicode-gencat\",\n \"unicode-perl\",\n \"unicode-script\",\n \"unicode-segment\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=regex-syntax\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.5\",\n)\n" - } - }, - "crates__rustversion-1.0.21": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/rustversion/1.0.21/download" - ], - "strip_prefix": "rustversion-1.0.21", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"rustversion\",\n deps = [\n \"@crates__rustversion-1.0.21//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rustversion\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.21\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build/build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2018\",\n pkg_name = \"rustversion\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rustversion\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.21\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__ryu-1.0.18": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/ryu/1.0.18/download" - ], - "strip_prefix": "ryu-1.0.18", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"ryu\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=ryu\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.18\",\n)\n" - } - }, - "crates__serde-1.0.216": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/serde/1.0.216/download" - ], - "strip_prefix": "serde-1.0.216", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"serde\",\n deps = [\n \"@crates__serde-1.0.216//:build_script_build\",\n ],\n proc_macro_deps = [\n \"@crates__serde_derive-1.0.216//:serde_derive\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"default\",\n \"derive\",\n \"serde_derive\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.216\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"default\",\n \"derive\",\n \"serde_derive\",\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2018\",\n pkg_name = \"serde\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.216\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__serde_derive-1.0.216": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/serde_derive/1.0.216/download" - ], - "strip_prefix": "serde_derive-1.0.216", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"serde_derive\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde_derive\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.216\",\n)\n" - } - }, - "crates__serde_json-1.0.134": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/serde_json/1.0.134/download" - ], - "strip_prefix": "serde_json-1.0.134", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"serde_json\",\n deps = [\n \"@crates__itoa-1.0.14//:itoa\",\n \"@crates__memchr-2.7.4//:memchr\",\n \"@crates__ryu-1.0.18//:ryu\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__serde_json-1.0.134//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde_json\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.134\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"serde_json\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde_json\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.134\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__smallvec-1.13.2": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/smallvec/1.13.2/download" - ], - "strip_prefix": "smallvec-1.13.2", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"smallvec\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=smallvec\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.13.2\",\n)\n" - } - }, - "crates__spm_precompiled-0.1.4": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/spm_precompiled/0.1.4/download" - ], - "strip_prefix": "spm_precompiled-0.1.4", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"spm_precompiled\",\n deps = [\n \"@crates__base64-0.13.1//:base64\",\n \"@crates__nom-7.1.3//:nom\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__unicode-segmentation-1.12.0//:unicode_segmentation\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=spm_precompiled\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.4\",\n)\n" - } - }, - "crates__static_assertions-1.1.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/static_assertions/1.1.0/download" - ], - "strip_prefix": "static_assertions-1.1.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"static_assertions\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=static_assertions\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.1.0\",\n)\n" - } - }, - "crates__strsim-0.11.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/strsim/0.11.1/download" - ], - "strip_prefix": "strsim-0.11.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"strsim\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=strsim\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.11.1\",\n)\n" - } - }, - "crates__syn-2.0.91": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/syn/2.0.91/download" - ], - "strip_prefix": "syn-2.0.91", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"syn\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__unicode-ident-1.0.14//:unicode_ident\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"clone-impls\",\n \"default\",\n \"derive\",\n \"extra-traits\",\n \"full\",\n \"parsing\",\n \"printing\",\n \"proc-macro\",\n \"visit\",\n \"visit-mut\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=syn\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.0.91\",\n)\n" - } - }, - "crates__thiserror-2.0.12": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/thiserror/2.0.12/download" - ], - "strip_prefix": "thiserror-2.0.12", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"thiserror\",\n deps = [\n \"@crates__thiserror-2.0.12//:build_script_build\",\n ],\n proc_macro_deps = [\n \"@crates__thiserror-impl-2.0.12//:thiserror_impl\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=thiserror\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.0.12\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"thiserror\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=thiserror\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"2.0.12\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__thiserror-impl-2.0.12": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/thiserror-impl/2.0.12/download" - ], - "strip_prefix": "thiserror-impl-2.0.12", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"thiserror_impl\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=thiserror-impl\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.0.12\",\n)\n" - } - }, - "crates__tokenizers-0.21.2": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "4c3846d8588abed0daba25a0e47edd58ea15e450a6088b2575f5116fdb0b27ca", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/tokenizers/0.21.2/download" - ], - "strip_prefix": "tokenizers-0.21.2", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"tokenizers\",\n deps = [\n \"@crates__ahash-0.8.12//:ahash\",\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__compact_str-0.9.0//:compact_str\",\n \"@crates__dary_heap-0.3.7//:dary_heap\",\n \"@crates__derive_builder-0.20.2//:derive_builder\",\n \"@crates__esaxx-rs-0.1.10//:esaxx_rs\",\n \"@crates__fancy-regex-0.14.0//:fancy_regex\",\n \"@crates__getrandom-0.3.3//:getrandom\",\n \"@crates__itertools-0.14.0//:itertools\",\n \"@crates__log-0.4.22//:log\",\n \"@crates__macro_rules_attribute-0.2.0//:macro_rules_attribute\",\n \"@crates__monostate-0.1.13//:monostate\",\n \"@crates__rand-0.9.1//:rand\",\n \"@crates__rayon-1.10.0//:rayon\",\n \"@crates__rayon-cond-0.4.0//:rayon_cond\",\n \"@crates__regex-1.11.1//:regex\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__serde_json-1.0.134//:serde_json\",\n \"@crates__spm_precompiled-0.1.4//:spm_precompiled\",\n \"@crates__thiserror-2.0.12//:thiserror\",\n \"@crates__unicode-normalization-alignments-0.1.12//:unicode_normalization_alignments\",\n \"@crates__unicode-segmentation-1.12.0//:unicode_segmentation\",\n \"@crates__unicode_categories-0.1.1//:unicode_categories\",\n ],\n proc_macro_deps = [\n \"@crates__paste-1.0.15//:paste\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"fancy-regex\",\n \"unstable_wasm\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=tokenizers\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.21.2\",\n)\n" - } - }, - "crates__unicode-ident-1.0.14": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/unicode-ident/1.0.14/download" - ], - "strip_prefix": "unicode-ident-1.0.14", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"unicode_ident\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=unicode-ident\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.14\",\n)\n" - } - }, - "crates__unicode-normalization-alignments-0.1.12": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/unicode-normalization-alignments/0.1.12/download" - ], - "strip_prefix": "unicode-normalization-alignments-0.1.12", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"unicode_normalization_alignments\",\n deps = [\n \"@crates__smallvec-1.13.2//:smallvec\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=unicode-normalization-alignments\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.12\",\n)\n" - } - }, - "crates__unicode-segmentation-1.12.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/unicode-segmentation/1.12.0/download" - ], - "strip_prefix": "unicode-segmentation-1.12.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"unicode_segmentation\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=unicode-segmentation\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.12.0\",\n)\n" - } - }, - "crates__unicode_categories-0.1.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/unicode_categories/0.1.1/download" - ], - "strip_prefix": "unicode_categories-0.1.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"unicode_categories\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=unicode_categories\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.1\",\n)\n" - } - }, - "crates__version_check-0.9.5": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/version_check/0.9.5/download" - ], - "strip_prefix": "version_check-0.9.5", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"version_check\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=version_check\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.9.5\",\n)\n" - } - }, - "crates__wasi-0.14.2-wasi-0.2.4": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/wasi/0.14.2+wasi-0.2.4/download" - ], - "strip_prefix": "wasi-0.14.2+wasi-0.2.4", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wasi\",\n deps = [\n \"@crates__wit-bindgen-rt-0.39.0//:wit_bindgen_rt\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasi\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.14.2+wasi-0.2.4\",\n)\n" - } - }, - "crates__wasm-bindgen-0.2.100": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/wasm-bindgen/0.2.100/download" - ], - "strip_prefix": "wasm-bindgen-0.2.100", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wasm_bindgen\",\n deps = [\n \"@crates__cfg-if-1.0.0//:cfg_if\",\n \"@crates__once_cell-1.20.2//:once_cell\",\n \"@crates__wasm-bindgen-0.2.100//:build_script_build\",\n ],\n proc_macro_deps = [\n \"@crates__wasm-bindgen-macro-0.2.100//:wasm_bindgen_macro\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.100\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"wasm-bindgen\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.2.100\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__wasm-bindgen-backend-0.2.100": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/wasm-bindgen-backend/0.2.100/download" - ], - "strip_prefix": "wasm-bindgen-backend-0.2.100", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wasm_bindgen_backend\",\n deps = [\n \"@crates__bumpalo-3.16.0//:bumpalo\",\n \"@crates__log-0.4.22//:log\",\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n \"@crates__wasm-bindgen-shared-0.2.100//:wasm_bindgen_shared\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen-backend\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.100\",\n)\n" - } - }, - "crates__wasm-bindgen-macro-0.2.100": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/wasm-bindgen-macro/0.2.100/download" - ], - "strip_prefix": "wasm-bindgen-macro-0.2.100", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"wasm_bindgen_macro\",\n deps = [\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__wasm-bindgen-macro-support-0.2.100//:wasm_bindgen_macro_support\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen-macro\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.100\",\n)\n" - } - }, - "crates__wasm-bindgen-macro-support-0.2.100": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/wasm-bindgen-macro-support/0.2.100/download" - ], - "strip_prefix": "wasm-bindgen-macro-support-0.2.100", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wasm_bindgen_macro_support\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n \"@crates__wasm-bindgen-backend-0.2.100//:wasm_bindgen_backend\",\n \"@crates__wasm-bindgen-shared-0.2.100//:wasm_bindgen_shared\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen-macro-support\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.100\",\n)\n" - } - }, - "crates__wasm-bindgen-shared-0.2.100": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/wasm-bindgen-shared/0.2.100/download" - ], - "strip_prefix": "wasm-bindgen-shared-0.2.100", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wasm_bindgen_shared\",\n deps = [\n \"@crates__unicode-ident-1.0.14//:unicode_ident\",\n \"@crates__wasm-bindgen-shared-0.2.100//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen-shared\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.100\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n links = \"wasm_bindgen\",\n pkg_name = \"wasm-bindgen-shared\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen-shared\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.2.100\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__wit-bindgen-rt-0.39.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/wit-bindgen-rt/0.39.0/download" - ], - "strip_prefix": "wit-bindgen-rt-0.39.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wit_bindgen_rt\",\n deps = [\n \"@crates__wit-bindgen-rt-0.39.0//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wit-bindgen-rt\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.39.0\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"wit-bindgen-rt\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wit-bindgen-rt\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.39.0\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__zerocopy-0.7.35": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/zerocopy/0.7.35/download" - ], - "strip_prefix": "zerocopy-0.7.35", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"zerocopy\",\n deps = [\n \"@crates__byteorder-1.5.0//:byteorder\",\n ],\n proc_macro_deps = [\n \"@crates__zerocopy-derive-0.7.35//:zerocopy_derive\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"byteorder\",\n \"default\",\n \"derive\",\n \"simd\",\n \"zerocopy-derive\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=zerocopy\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.7.35\",\n)\n" - } - }, - "crates__zerocopy-0.8.26": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/zerocopy/0.8.26/download" - ], - "strip_prefix": "zerocopy-0.8.26", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"zerocopy\",\n deps = [\n \"@crates__zerocopy-0.8.26//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"simd\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=zerocopy\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.26\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"simd\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"zerocopy\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=zerocopy\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.8.26\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__zerocopy-derive-0.7.35": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/zerocopy-derive/0.7.35/download" - ], - "strip_prefix": "zerocopy-derive-0.7.35", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"zerocopy_derive\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=zerocopy-derive\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.7.35\",\n)\n" - } - }, - "crates__zerocopy-derive-0.8.26": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/zerocopy-derive/0.8.26/download" - ], - "strip_prefix": "zerocopy-derive-0.8.26", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"zerocopy_derive\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=zerocopy-derive\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.26\",\n)\n" - } - } - }, - "moduleExtensionMetadata": { - "useAllRepos": "NO", - "reproducible": false - }, - "recordedRepoMappingEntries": [ - [ - "bazel_features+", - "bazel_features_globals", - "bazel_features++version_extension+bazel_features_globals" - ], - [ - "bazel_features+", - "bazel_features_version", - "bazel_features++version_extension+bazel_features_version" - ], - [ - "rules_cc+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_cc+", - "rules_cc", - "rules_cc+" - ], - [ - "rules_rust+", - "bazel_features", - "bazel_features+" - ], - [ - "rules_rust+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "rules_rust+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_rust+", - "rules_cc", - "rules_cc+" - ], - [ - "rules_rust+", - "rules_rust", - "rules_rust+" - ] - ] - } - }, - "@@rules_rust+//crate_universe/private:internal_extensions.bzl%cu_nr": { - "general": { - "bzlTransitiveDigest": "xJbZ33HMriukI2w0kbBh0FnM9oSo52MbobnlB5Ry4T0=", - "usagesDigest": "UWnpRlkSFx6zlujUh+aOuBGwtJYbR60HyNKAvijQ99g=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "cargo_bazel_bootstrap": { - "repoRuleId": "@@rules_rust+//cargo/private:cargo_bootstrap.bzl%cargo_bootstrap_repository", - "attributes": { - "srcs": [ - "@@rules_rust+//crate_universe:src/api.rs", - "@@rules_rust+//crate_universe:src/api/lockfile.rs", - "@@rules_rust+//crate_universe:src/cli.rs", - "@@rules_rust+//crate_universe:src/cli/generate.rs", - "@@rules_rust+//crate_universe:src/cli/query.rs", - "@@rules_rust+//crate_universe:src/cli/render.rs", - "@@rules_rust+//crate_universe:src/cli/splice.rs", - "@@rules_rust+//crate_universe:src/cli/vendor.rs", - "@@rules_rust+//crate_universe:src/config.rs", - "@@rules_rust+//crate_universe:src/context.rs", - "@@rules_rust+//crate_universe:src/context/crate_context.rs", - "@@rules_rust+//crate_universe:src/context/platforms.rs", - "@@rules_rust+//crate_universe:src/lib.rs", - "@@rules_rust+//crate_universe:src/lockfile.rs", - "@@rules_rust+//crate_universe:src/main.rs", - "@@rules_rust+//crate_universe:src/metadata.rs", - "@@rules_rust+//crate_universe:src/metadata/cargo_bin.rs", - "@@rules_rust+//crate_universe:src/metadata/cargo_tree_resolver.rs", - "@@rules_rust+//crate_universe:src/metadata/cargo_tree_rustc_wrapper.bat", - "@@rules_rust+//crate_universe:src/metadata/cargo_tree_rustc_wrapper.sh", - "@@rules_rust+//crate_universe:src/metadata/dependency.rs", - "@@rules_rust+//crate_universe:src/metadata/metadata_annotation.rs", - "@@rules_rust+//crate_universe:src/metadata/workspace_discoverer.rs", - "@@rules_rust+//crate_universe:src/rendering.rs", - "@@rules_rust+//crate_universe:src/rendering/template_engine.rs", - "@@rules_rust+//crate_universe:src/rendering/templates/module_bzl.j2", - "@@rules_rust+//crate_universe:src/rendering/templates/partials/header.j2", - "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/aliases_map.j2", - "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/deps_map.j2", - "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/repo_git.j2", - "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/repo_http.j2", - "@@rules_rust+//crate_universe:src/rendering/templates/vendor_module.j2", - "@@rules_rust+//crate_universe:src/rendering/verbatim/alias_rules.bzl", - "@@rules_rust+//crate_universe:src/select.rs", - "@@rules_rust+//crate_universe:src/splicing.rs", - "@@rules_rust+//crate_universe:src/splicing/cargo_config.rs", - "@@rules_rust+//crate_universe:src/splicing/crate_index_lookup.rs", - "@@rules_rust+//crate_universe:src/splicing/splicer.rs", - "@@rules_rust+//crate_universe:src/test.rs", - "@@rules_rust+//crate_universe:src/utils.rs", - "@@rules_rust+//crate_universe:src/utils/starlark.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/glob.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/label.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/select.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/select_dict.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/select_list.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/select_scalar.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/select_set.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/serialize.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/target_compatible_with.rs", - "@@rules_rust+//crate_universe:src/utils/symlink.rs", - "@@rules_rust+//crate_universe:src/utils/target_triple.rs" - ], - "binary": "cargo-bazel", - "cargo_lockfile": "@@rules_rust+//crate_universe:Cargo.lock", - "cargo_toml": "@@rules_rust+//crate_universe:Cargo.toml", - "version": "1.86.0", - "timeout": 900, - "rust_toolchain_cargo_template": "@rust_host_tools//:bin/{tool}", - "rust_toolchain_rustc_template": "@rust_host_tools//:bin/{tool}", - "compressed_windows_toolchain_names": false - } - } - }, - "moduleExtensionMetadata": { - "explicitRootModuleDirectDeps": [ - "cargo_bazel_bootstrap" - ], - "explicitRootModuleDirectDevDeps": [], - "useAllRepos": "NO", - "reproducible": false - }, - "recordedRepoMappingEntries": [ - [ - "bazel_features+", - "bazel_features_globals", - "bazel_features++version_extension+bazel_features_globals" - ], - [ - "bazel_features+", - "bazel_features_version", - "bazel_features++version_extension+bazel_features_version" - ], - [ - "rules_cc+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_cc+", - "rules_cc", - "rules_cc+" - ], - [ - "rules_rust+", - "bazel_features", - "bazel_features+" - ], - [ - "rules_rust+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "rules_rust+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_rust+", - "cui", - "rules_rust++cu+cui" - ], - [ - "rules_rust+", - "rules_cc", - "rules_cc+" - ], - [ - "rules_rust+", - "rules_rust", - "rules_rust+" - ], - [ - "rules_rust+", - "rules_rust_ctve", - "rules_rust++i2+rules_rust_ctve" - ] - ] - } - }, - "@@rules_swift+//swift:extensions.bzl%non_module_deps": { - "general": { - "bzlTransitiveDigest": "3iOPDsadEKzPUpfznfK5UYw/yuGpZzq7zrjrB/oEo6g=", - "usagesDigest": "9w18ec4dj90mX/JqpR2tBU2YVc1GU9yNxi/d7q6lLVA=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "com_github_apple_swift_protobuf": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-protobuf/archive/1.20.2.tar.gz" - ], - "sha256": "3fb50bd4d293337f202d917b6ada22f9548a0a0aed9d9a4d791e6fbd8a246ebb", - "strip_prefix": "swift-protobuf-1.20.2/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_protobuf/BUILD.overlay" - } - }, - "com_github_grpc_grpc_swift": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/grpc/grpc-swift/archive/1.16.0.tar.gz" - ], - "sha256": "58b60431d0064969f9679411264b82e40a217ae6bd34e17096d92cc4e47556a5", - "strip_prefix": "grpc-swift-1.16.0/", - "build_file": "@@rules_swift+//third_party:com_github_grpc_grpc_swift/BUILD.overlay" - } - }, - "com_github_apple_swift_docc_symbolkit": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-docc-symbolkit/archive/refs/tags/swift-5.10-RELEASE.tar.gz" - ], - "sha256": "de1d4b6940468ddb53b89df7aa1a81323b9712775b0e33e8254fa0f6f7469a97", - "strip_prefix": "swift-docc-symbolkit-swift-5.10-RELEASE", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_docc_symbolkit/BUILD.overlay" - } - }, - "com_github_apple_swift_nio": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio/archive/2.42.0.tar.gz" - ], - "sha256": "e3304bc3fb53aea74a3e54bd005ede11f6dc357117d9b1db642d03aea87194a0", - "strip_prefix": "swift-nio-2.42.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_http2": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-http2/archive/1.26.0.tar.gz" - ], - "sha256": "f0edfc9d6a7be1d587e5b403f2d04264bdfae59aac1d74f7d974a9022c6d2b25", - "strip_prefix": "swift-nio-http2-1.26.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_http2/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_transport_services": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-transport-services/archive/1.15.0.tar.gz" - ], - "sha256": "f3498dafa633751a52b9b7f741f7ac30c42bcbeb3b9edca6d447e0da8e693262", - "strip_prefix": "swift-nio-transport-services-1.15.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_transport_services/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_extras": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-extras/archive/1.4.0.tar.gz" - ], - "sha256": "4684b52951d9d9937bb3e8ccd6b5daedd777021ef2519ea2f18c4c922843b52b", - "strip_prefix": "swift-nio-extras-1.4.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_extras/BUILD.overlay" - } - }, - "com_github_apple_swift_log": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-log/archive/1.4.4.tar.gz" - ], - "sha256": "48fe66426c784c0c20031f15dc17faf9f4c9037c192bfac2f643f65cb2321ba0", - "strip_prefix": "swift-log-1.4.4/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_log/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_ssl": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-ssl/archive/2.23.0.tar.gz" - ], - "sha256": "4787c63f61dd04d99e498adc3d1a628193387e41efddf8de19b8db04544d016d", - "strip_prefix": "swift-nio-ssl-2.23.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_ssl/BUILD.overlay" - } - }, - "com_github_apple_swift_collections": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-collections/archive/1.0.4.tar.gz" - ], - "sha256": "d9e4c8a91c60fb9c92a04caccbb10ded42f4cb47b26a212bc6b39cc390a4b096", - "strip_prefix": "swift-collections-1.0.4/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_collections/BUILD.overlay" - } - }, - "com_github_apple_swift_atomics": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-atomics/archive/1.1.0.tar.gz" - ], - "sha256": "1bee7f469f7e8dc49f11cfa4da07182fbc79eab000ec2c17bfdce468c5d276fb", - "strip_prefix": "swift-atomics-1.1.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_atomics/BUILD.overlay" - } - }, - "build_bazel_rules_swift_index_import": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "build_file": "@@rules_swift+//third_party:build_bazel_rules_swift_index_import/BUILD.overlay", - "canonical_id": "index-import-5.8", - "urls": [ - "https://github.com/MobileNativeFoundation/index-import/releases/download/5.8.0.1/index-import.tar.gz" - ], - "sha256": "28c1ffa39d99e74ed70623899b207b41f79214c498c603915aef55972a851a15" - } - }, - "build_bazel_rules_swift_local_config": { - "repoRuleId": "@@rules_swift+//swift/internal:swift_autoconfiguration.bzl%swift_autoconfiguration", - "attributes": {} - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_swift+", - "bazel_tools", - "bazel_tools" - ] - ] - } - } - } -} diff --git a/examples/benchmark/BUILD.bazel b/examples/benchmark/BUILD.bazel index 7088dae8..c4a0d011 100644 --- a/examples/benchmark/BUILD.bazel +++ b/examples/benchmark/BUILD.bazel @@ -1,11 +1,17 @@ -load("@zml//bazel:zig.bzl", "zig_cc_binary") +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load("//bazel:zig.bzl", "zig_cc_binary") zig_cc_binary( name = "benchmark", main = "main.zig", deps = [ - "@zml//async", - "@zml//stdx", - "@zml//zml", + "//async", + "//stdx", + "//zml", ], ) + +build_test( + name = "test", + targets = [":benchmark"], +) diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel index 839791b2..f6c29c97 100644 --- a/examples/llama/BUILD.bazel +++ b/examples/llama/BUILD.bazel @@ -1,11 +1,8 @@ -load("@aspect_bazel_lib//lib:expand_template.bzl", "expand_template") load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") -load("@bazel_skylib//rules:native_binary.bzl", "native_test") -load("@bazel_skylib//rules:write_file.bzl", "write_file") -load("@rules_cc//cc:cc_binary.bzl", "cc_binary") +load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") -load("@zml//bazel:zig.bzl", "zig_cc_binary") +load("//bazel:zig.bzl", "zig_cc_binary") zig_cc_binary( name = "llama", @@ -14,65 +11,58 @@ zig_cc_binary( ], main = "main.zig", deps = [ + "//async", + "//stdx", + "//zml", "@com_github_hejsil_clap//:clap", - "@zml//async", - "@zml//stdx", - "@zml//zml", ], ) -zig_cc_binary( - name = "test-implementation", - srcs = ["llama.zig"], - args = [ - "--weights=$(location @Meta-Llama-3.1-8B-Instruct//:model.safetensors.index.json)", - "--config=$(location @Meta-Llama-3.1-8B-Instruct//:config.json)", - ], - data = [ - "@Meta-Llama-3.1-8B-Instruct//:config.json", - "@Meta-Llama-3.1-8B-Instruct//:model.safetensors.index.json", - ], - main = "test.zig", - tags = [ - "manual", - ], - deps = [ - "@zml//async", - "@zml//stdx", - "@zml//zml", - ], -) +# TODO(steeve): Re-enable when rules_zig linkmode is fixed. +# +# zig_cc_binary( +# name = "test-implementation", +# srcs = ["llama.zig"], +# args = [ +# "--weights=$(location @Meta-Llama-3.1-8B-Instruct//:model.safetensors.index.json)", +# "--config=$(location @Meta-Llama-3.1-8B-Instruct//:config.json)", +# ], +# data = [ +# "@Meta-Llama-3.1-8B-Instruct//:config.json", +# "@Meta-Llama-3.1-8B-Instruct//:model.safetensors.index.json", +# ], +# main = "test.zig", +# deps = [ +# "//async", +# "//stdx", +# "//zml", +# ], +# ) -native_test( - name = "test_tokenizer", - src = "@zml//zml/tokenizer:main", - # Note: all Llama-3.x tokenizers are the same, - # but using the 3.2-1B version because downloading the tokenizer triggers downloading the model. - args = [ - "--tokenizer=$(location @Meta-Llama-3.2-1B-Instruct//:tokenizer.json)", - """--prompt='Examples of titles: -📉 Stock Market Trends -🍪 Perfect Chocolate Chip Recipe -Evolution of Music Streaming -Remote Work Productivity Tips -Artificial Intelligence in Healthcare -🎮 Video Game Development Insights -'""", - # this correspond to encoding with HF tokenizers, with bos=False - "--expected=41481,315,15671,512,9468,241,231,12937,8152,50730,198,9468,235,103,24118,39520,32013,26371,198,35212,3294,315,10948,45910,198,25732,5664,5761,1968,26788,198,9470,16895,22107,304,39435,198,9468,236,106,8519,4140,11050,73137,198", - ], - data = ["@Meta-Llama-3.2-1B-Instruct//:tokenizer.json"], - tags = [ - "manual", - ], -) +# native_test( +# name = "test_tokenizer", +# src = "//zml/tokenizer:main", +# # Note: all Llama-3.x tokenizers are the same, +# # but using the 3.2-1B version because downloading the tokenizer triggers downloading the model. +# args = [ +# "--tokenizer=$(location @Meta-Llama-3.2-1B-Instruct//:tokenizer.json)", +# """--prompt='Examples of titles: +# 📉 Stock Market Trends +# 🍪 Perfect Chocolate Chip Recipe +# Evolution of Music Streaming +# Remote Work Productivity Tips +# Artificial Intelligence in Healthcare +# 🎮 Video Game Development Insights +# '""", +# # this correspond to encoding with HF tokenizers, with bos=False +# "--expected=41481,315,15671,512,9468,241,231,12937,8152,50730,198,9468,235,103,24118,39520,32013,26371,198,35212,3294,315,10948,45910,198,25732,5664,5761,1968,26788,198,9470,16895,22107,304,39435,198,9468,236,106,8519,4140,11050,73137,198", +# ], +# data = ["@Meta-Llama-3.2-1B-Instruct//:tokenizer.json"], +# ) mtree_spec( name = "mtree", srcs = [":llama"], - tags = [ - "manual", - ], ) tar( @@ -80,54 +70,48 @@ tar( srcs = [":llama"], args = [ "--options", - "zstd:compression-level=9", + ",".join([ + "zstd:compression-level=9", + "zstd:threads=16", + ]), ], compress = "zstd", mtree = ":mtree", - tags = [ - "manual", - ], ) oci_image( name = "image_", - base = "@distroless_cc_debian12_debug", - entrypoint = ["./{}/llama".format(package_name())], - tags = [ - "manual", - ], - tars = [ - "@zml//runtimes:layers", - ":archive", + base = "@distroless_cc_debian12", + entrypoint = ["/{}/llama".format(package_name())], + target_compatible_with = [ + "@platforms//os:linux", ], + tars = [":archive"], ) platform_transition_filegroup( name = "image", srcs = [":image_"], - tags = [ - "manual", - ], - target_platform = "@zml//platforms:linux_amd64", + tags = ["manual"], + target_platform = "//platforms:linux_amd64", ) oci_load( name = "load", image = ":image", - repo_tags = [ - "distroless/llama:latest", - ], - tags = [ - "manual", - ], + repo_tags = ["zmlai/llama:latest"], + tags = ["manual"], ) oci_push( name = "push", image = ":image", remote_tags = ["latest"], - repository = "index.docker.io/steeve/llama", - tags = [ - "manual", - ], + repository = "index.docker.io/zmlai/llama", + tags = ["manual"], +) + +build_test( + name = "test", + targets = [":llama"], ) diff --git a/examples/loader/BUILD.bazel b/examples/loader/BUILD.bazel deleted file mode 100644 index a80b40b4..00000000 --- a/examples/loader/BUILD.bazel +++ /dev/null @@ -1,11 +0,0 @@ -load("@zml//bazel:zig.bzl", "zig_cc_binary") - -zig_cc_binary( - name = "safetensors", - main = "main.zig", - deps = [ - "@zml//async", - "@zml//stdx", - "@zml//zml", - ], -) diff --git a/examples/loader/main.zig b/examples/loader/main.zig deleted file mode 100644 index d258b8a3..00000000 --- a/examples/loader/main.zig +++ /dev/null @@ -1,65 +0,0 @@ -const std = @import("std"); - -const asynk = @import("async"); -const stdx = @import("stdx"); -const zml = @import("zml"); - -pub fn main() !void { - try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); -} - -pub fn asyncMain() !void { - // Short lived allocations - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - var args = std.process.args(); - // Skip executable path - _ = args.next().?; - - const file = if (args.next()) |path| blk: { - std.debug.print("File path: {s}\n", .{path}); - break :blk path; - } else { - std.debug.print("Missing file path argument\n", .{}); - std.debug.print("Try: bazel run --config=release //loader:safetensors -- /path/to/mymodel.safetensors or /path/to/model.safetensors.index.json \n", .{}); - std.process.exit(0); - }; - - var buffer_store = try zml.aio.safetensors.open(allocator, file); - defer buffer_store.deinit(); - - var context = try zml.Context.init(); - defer context.deinit(); - - const platform = context.autoPlatform(.{}); - context.printAvailablePlatforms(platform); - - var buffers = try gpa.allocator().alloc(zml.Buffer, buffer_store.buffers.count()); - defer { - // Note we don't pass an allocator to buf.deinit() cause its allocated on the device. - for (buffers) |*buf| buf.deinit(); - gpa.allocator().free(buffers); - } - - var total_bytes: usize = 0; - var timer = try std.time.Timer.start(); - - var it = buffer_store.buffers.iterator(); - var i: usize = 0; - std.debug.print("\nStart to read {d} buffers from store..\n", .{buffer_store.buffers.count()}); - - while (it.next()) |entry| : (i += 1) { - const host_buffer = entry.value_ptr.*; - total_bytes += host_buffer.shape().byteSize(); - std.debug.print("Buffer: {s} ({any} / {any})\n", .{ entry.key_ptr.*, i + 1, buffer_store.buffers.count() }); - buffers[i] = try zml.Buffer.from(platform, host_buffer, .{}); - } - - const stop = timer.read(); - const time_in_s = stdx.math.divFloat(f64, stop, std.time.ns_per_s); - const mbs = stdx.math.divFloat(f64, total_bytes, 1024 * 1024); - - std.debug.print("\nLoading speed: {d:.2} MB/s\n\n", .{mbs / time_in_s}); -} diff --git a/examples/mnist/BUILD.bazel b/examples/mnist/BUILD.bazel index be8b8e95..f8eeae34 100644 --- a/examples/mnist/BUILD.bazel +++ b/examples/mnist/BUILD.bazel @@ -1,108 +1,24 @@ -load("@aspect_bazel_lib//lib:expand_template.bzl", "expand_template") -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") -load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") -load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") -load("@zml//bazel:zig.bzl", "zig_cc_binary") +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load("//bazel:zig.bzl", "zig_cc_binary") -# Executable zig_cc_binary( name = "mnist", args = [ - "$(location @com_github_ggerganov_ggml_mnist//file)", - "$(location @com_github_ggerganov_ggml_mnist_data//file)", + "$(location @mnist//:mnist.pt)", + "$(location @mnist//:t10k-images.idx3-ubyte)", ], data = [ - "@com_github_ggerganov_ggml_mnist//file", - "@com_github_ggerganov_ggml_mnist_data//file", + "@mnist//:mnist.pt", + "@mnist//:t10k-images.idx3-ubyte", ], main = "mnist.zig", deps = [ - "@zml//async", - "@zml//zml", + "//async", + "//zml", ], ) -mtree_spec( - name = "mtree", - srcs = [":mnist"], -) - -tar( - name = "archive", - srcs = [":mnist"], - args = [ - "--options", - "zstd:compression-level=9", - ], - compress = "zstd", - mtree = ":mtree", -) - -expand_template( - name = "entrypoint", - data = [ - ":mnist", - "@com_github_ggerganov_ggml_mnist//file", - "@com_github_ggerganov_ggml_mnist_data//file", - ], - substitutions = { - ":model": "$(rlocationpath @com_github_ggerganov_ggml_mnist//file)", - ":data": "$(rlocationpath @com_github_ggerganov_ggml_mnist_data//file)", - }, - template = [ - "./{}/mnist".format(package_name()), - "./{}/mnist.runfiles/:model".format(package_name()), - "./{}/mnist.runfiles/:data".format(package_name()), - ], -) - -oci_image( - name = "image_", - base = "@distroless_cc_debian12", - entrypoint = ":entrypoint", - target_compatible_with = [ - "@platforms//os:linux", - ], - tars = [":archive"], -) - -platform_transition_filegroup( - name = "image", - srcs = [":image_"], - target_compatible_with = [ - "@platforms//os:linux", - ], - target_platform = "@zml//platforms:linux_amd64", -) - -oci_load( - name = "load", - image = ":image", - repo_tags = [ - "distroless/mnist:latest", - ], - target_compatible_with = [ - "@platforms//os:linux", - ], -) - -oci_push( - name = "push", - image = ":image", - remote_tags = ["latest"], - repository = "index.docker.io/steeve/mnist", - target_compatible_with = [ - "@platforms//os:linux", - ], -) - -oci_load( - name = "debug_image", - image = ":image", - repo_tags = [ - "distroless/mnist:latest", - ], - target_compatible_with = [ - "@platforms//os:linux", - ], +build_test( + name = "test", + targets = [":mnist"], ) diff --git a/examples/modernbert/BUILD.bazel b/examples/modernbert/BUILD.bazel index f2b09ad6..5f8291c8 100644 --- a/examples/modernbert/BUILD.bazel +++ b/examples/modernbert/BUILD.bazel @@ -1,70 +1,46 @@ -load("@rules_cc//cc:cc_binary.bzl", "cc_binary") -load("@zml//bazel:zig.bzl", "zig_cc_binary") +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load("//bazel:zig.bzl", "zig_cc_binary") zig_cc_binary( name = "modernbert", srcs = ["modernbert.zig"], - main = "main.zig", - deps = [ - "@com_github_hejsil_clap//:clap", - "@zml//async", - "@zml//stdx", - "@zml//zml", - ], -) - -cc_binary( - name = "ModernBERT-base", args = [ "--model=$(location @ModernBERT-base//:model.safetensors)", - "--tokenizer=$(location @ModernBERT-base//:tokenizer)", + "--tokenizer=$(location @ModernBERT-base//:tokenizer.json)", "--num-attention-heads=12", "--tie-word-embeddings=true", ], data = [ "@ModernBERT-base//:model.safetensors", - "@ModernBERT-base//:tokenizer", - ], - tags = [ - "manual", - ], - deps = [":modernbert_lib"], -) - -cc_binary( - name = "ModernBERT-large", - args = [ - "--model=$(location @ModernBERT-large//:model.safetensors)", - "--tokenizer=$(location @ModernBERT-large//:tokenizer)", - "--num-attention-heads=16", - "--tie-word-embeddings=true", - ], - data = [ - "@ModernBERT-large//:model.safetensors", - "@ModernBERT-large//:tokenizer", - ], - tags = [ - "manual", - ], - deps = [":modernbert_lib"], -) - -zig_cc_binary( - name = "test-implementation", - srcs = ["modernbert.zig"], - args = [ - "--model=$(location @ModernBERT-base//:model.safetensors)", - ], - data = [ - "@ModernBERT-base//:model.safetensors", - ], - main = "test.zig", - tags = [ - "manual", + "@ModernBERT-base//:tokenizer.json", ], + main = "main.zig", deps = [ + "//async", + "//stdx", + "//zml", "@com_github_hejsil_clap//:clap", - "@zml//async", - "@zml//zml", ], ) + +# zig_cc_binary( +# name = "test-implementation", +# srcs = ["modernbert.zig"], +# args = [ +# "--model=$(location @ModernBERT-base//:model.safetensors)", +# ], +# data = [ +# "@ModernBERT-base//:model.safetensors", +# ], +# main = "test.zig", +# deps = [ +# "//async", +# "//zml", +# "@com_github_hejsil_clap//:clap", +# ], +# ) + +build_test( + name = "test", + targets = [":modernbert"], +) diff --git a/examples/platform_mappings b/examples/platform_mappings deleted file mode 100644 index ed02329d..00000000 --- a/examples/platform_mappings +++ /dev/null @@ -1,29 +0,0 @@ -platforms: - @zml//platforms:linux_amd64 - --cpu=k8 - - @zml//platforms:linux_arm64 - --cpu=aarch64 - - @zml//platforms:macos_arm64 - --cpu=darwin_arm64 - --apple_platform_type=macos - - @zml//platforms:macos_amd64 - --cpu=darwin_amd64 - --apple_platform_type=macos - -flags: - --cpu=darwin_arm64 - --apple_platform_type=macos - @zml//platforms:macos_arm64 - - --cpu=darwin_amd64 - --apple_platform_type=macos - @zml//platforms:macos_amd64 - - --cpu=k8 - @zml//platforms:linux_amd64 - - --cpu=aarch64 - @zml//platforms:linux_arm64 diff --git a/examples/simple_layer/BUILD.bazel b/examples/simple_layer/BUILD.bazel index 8d747677..98988398 100644 --- a/examples/simple_layer/BUILD.bazel +++ b/examples/simple_layer/BUILD.bazel @@ -1,14 +1,15 @@ load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") -load("@zml//bazel:zig.bzl", "zig_cc_binary") +load("//bazel:zig.bzl", "zig_cc_binary") zig_cc_binary( name = "simple_layer", main = "main.zig", deps = [ - "@zml//async", - "@zml//zml", + "//async", + "//zml", ], ) @@ -24,7 +25,10 @@ tar( srcs = [":simple_layer"], args = [ "--options", - "zstd:compression-level=9", + ",".join([ + "zstd:compression-level=9", + "zstd:threads=16", + ]), ], compress = "zstd", mtree = ":mtree", @@ -34,7 +38,7 @@ tar( oci_image( name = "image_", base = "@distroless_cc_debian12", - entrypoint = ["./{}/simple_layer".format(package_name())], + entrypoint = ["/{}/simple_layer".format(package_name())], target_compatible_with = [ "@platforms//os:linux", ], @@ -45,10 +49,8 @@ oci_image( platform_transition_filegroup( name = "image", srcs = [":image_"], - target_compatible_with = [ - "@platforms//os:linux", - ], - target_platform = "@zml//platforms:linux_amd64", + tags = ["manual"], + target_platform = "//platforms:linux_amd64", ) # Load will immediatly load the image (eg: docker load) @@ -58,9 +60,7 @@ oci_load( repo_tags = [ "distroless/simple_layer:latest", ], - target_compatible_with = [ - "@platforms//os:linux", - ], + tags = ["manual"], ) # Bazel target for pushing the Linux image to the docker registry @@ -69,8 +69,11 @@ oci_push( image = ":image", remote_tags = ["latest"], # override with -- --repository foo.bar/org/image - repository = "index.docker.io/renerocksai/simple_layer", - target_compatible_with = [ - "@platforms//os:linux", - ], + repository = "index.docker.io/zmlai/simple_layer", + tags = ["manual"], +) + +build_test( + name = "test", + targets = [":simple_layer"], ) diff --git a/examples/tools/buildifier.sh b/examples/tools/buildifier.sh deleted file mode 100755 index 0e2b7ad7..00000000 --- a/examples/tools/buildifier.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -cd "$(dirname "${BASH_SOURCE[0]}")" -cd "$(bazel info workspace)" -exec bazel run -- @buildifier_prebuilt//:buildifier "$@" diff --git a/examples/tools/zls.sh b/examples/tools/zls.sh deleted file mode 100755 index 86827c87..00000000 --- a/examples/tools/zls.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -cd "$(dirname "${BASH_SOURCE[0]}")" -cd "$(bazel info workspace)" -exec bazel run -- @zml//:zls "${@}" diff --git a/examples/zls.build.json b/examples/zls.build.json deleted file mode 100644 index 2c63c085..00000000 --- a/examples/zls.build.json +++ /dev/null @@ -1,2 +0,0 @@ -{ -} diff --git a/mlir/BUILD.bazel b/mlir/BUILD.bazel index 63b9ede6..ae329430 100644 --- a/mlir/BUILD.bazel +++ b/mlir/BUILD.bazel @@ -1,8 +1,7 @@ load("@rules_cc//cc:defs.bzl", "cc_library") - load("@rules_zig//zig:defs.bzl", "zig_library") -load("//bazel:zig_srcs.bzl", "zig_srcs") load("//bazel:zig.bzl", "zig_cc_test") +load("//bazel:zig_srcs.bzl", "zig_srcs") cc_library( name = "c", @@ -33,11 +32,6 @@ zig_cc_test( deps = [":mlir"], ) -cc_static_library( - name="mlir_static", - deps = ["c"] -) - zig_srcs( name = "sources", zig_bin = ":test_test_lib", diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/BUILD.bazel index bcb3a1bb..c18f12d8 100644 --- a/mlir/dialects/BUILD.bazel +++ b/mlir/dialects/BUILD.bazel @@ -30,14 +30,6 @@ zig_srcs( zig_bin = ":test_test_lib", ) -cc_static_library( - name="mlir_static", - deps = [ - "//mlir:c", - "@stablehlo//:stablehlo_dialect_capi", - ] -) - zig_library( name = "stablehlo", import_name = "mlir/dialects/stablehlo", diff --git a/platform_mappings b/platform_mappings deleted file mode 120000 index 4a67a02f..00000000 --- a/platform_mappings +++ /dev/null @@ -1 +0,0 @@ -examples/platform_mappings \ No newline at end of file diff --git a/platform_mappings b/platform_mappings new file mode 100644 index 00000000..ed02329d --- /dev/null +++ b/platform_mappings @@ -0,0 +1,29 @@ +platforms: + @zml//platforms:linux_amd64 + --cpu=k8 + + @zml//platforms:linux_arm64 + --cpu=aarch64 + + @zml//platforms:macos_arm64 + --cpu=darwin_arm64 + --apple_platform_type=macos + + @zml//platforms:macos_amd64 + --cpu=darwin_amd64 + --apple_platform_type=macos + +flags: + --cpu=darwin_arm64 + --apple_platform_type=macos + @zml//platforms:macos_arm64 + + --cpu=darwin_amd64 + --apple_platform_type=macos + @zml//platforms:macos_amd64 + + --cpu=k8 + @zml//platforms:linux_amd64 + + --cpu=aarch64 + @zml//platforms:linux_arm64 diff --git a/runtimes/neuron/BUILD.bazel b/runtimes/neuron/BUILD.bazel index b65d7a38..deab87aa 100644 --- a/runtimes/neuron/BUILD.bazel +++ b/runtimes/neuron/BUILD.bazel @@ -18,21 +18,21 @@ zig_library( # A proxy PJRT Plugin that loads the Neuron PJRT Plugin # and returns the instance from nested GetPjrtApi. -# +# # Additionally, it provides a way to load implicit transitive dependencies # of neuronx-cc (see add_needed of the patchelf target below). # # TODO(cerisier): Use a zig_cc_shared_library instead. zig_binary( name = "libpjrt_neuron_proxy", + copts = ["-lc"], kind = BINARY_KIND.shared_lib, main = "libpjrt_neuron.zig", - copts = ["-lc"], + visibility = ["@libpjrt_neuron//:__subpackages__"], deps = [ "//stdx", "@rules_zig//zig/runfiles", ], - visibility = ["@libpjrt_neuron//:__subpackages__"], ) pip_compile( @@ -48,9 +48,6 @@ pip_compile( python_platform = "x86_64-unknown-linux-gnu", requirements_in = "requirements.in", requirements_txt = "requirements.lock.txt", - tags = [ - "manual", - ], ) py_console_script_binary( @@ -117,21 +114,21 @@ cc_library( zig_library( name = "neuron", - import_name = "runtimes/neuron", - main = "neuron.zig", - visibility = ["//visibility:public"], data = select({ "//runtimes:neuron.enabled": [ ":libneuronxla", ], "//conditions:default": [], }), + import_name = "runtimes/neuron", + main = "neuron.zig", + visibility = ["//visibility:public"], deps = [ "//pjrt", ] + select({ "//runtimes:neuron.enabled": [ - ":libpjrt_neuron", ":libneuronxla_pyenv", + ":libpjrt_neuron", ":libpython", "//async", "//stdx", diff --git a/examples/BUILD.bazel b/third_party/com_github_hejsil_clap/BUILD.bazel similarity index 100% rename from examples/BUILD.bazel rename to third_party/com_github_hejsil_clap/BUILD.bazel diff --git a/examples/third_party/com_github_hejsil_clap/clap.bazel b/third_party/com_github_hejsil_clap/clap.bazel similarity index 100% rename from examples/third_party/com_github_hejsil_clap/clap.bazel rename to third_party/com_github_hejsil_clap/clap.bazel diff --git a/examples/third_party/non_module_deps.bzl b/third_party/com_github_hejsil_clap/repo.bzl similarity index 55% rename from examples/third_party/non_module_deps.bzl rename to third_party/com_github_hejsil_clap/repo.bzl index d003b744..f665c74a 100644 --- a/examples/third_party/non_module_deps.bzl +++ b/third_party/com_github_hejsil_clap/repo.bzl @@ -1,20 +1,9 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") -def _non_module_deps_impl(mctx): - +def repo(): new_git_repository( name = "com_github_hejsil_clap", remote = "https://github.com/Hejsil/zig-clap.git", commit = "068c38f89814079635692c7d0be9f58508c86173", build_file = "//:third_party/com_github_hejsil_clap/clap.bazel", ) - - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -non_module_deps = module_extension( - implementation = _non_module_deps_impl, -) diff --git a/examples/build.zig b/third_party/mnist/BUILD.bazel similarity index 100% rename from examples/build.zig rename to third_party/mnist/BUILD.bazel diff --git a/third_party/mnist/repo.bzl b/third_party/mnist/repo.bzl new file mode 100644 index 00000000..025845eb --- /dev/null +++ b/third_party/mnist/repo.bzl @@ -0,0 +1,9 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +def repo(): + http_archive( + name = "mnist", + sha256 = "075905e433ea0cce13c3fc08832448ab86225d089b5d412be67f59c29388fb19", + url = "https://mirror.zml.ai/data/mnist.tar.zst", + build_file_content = """exports_files(glob(["**"]), visibility = ["//visibility:public"])""", + ) diff --git a/third_party/non_module_deps.bzl b/third_party/non_module_deps.bzl index 004a45e9..af92c417 100644 --- a/third_party/non_module_deps.bzl +++ b/third_party/non_module_deps.bzl @@ -1,9 +1,13 @@ -load("//:third_party/org_swig_swig/repo.bzl", org_swig_swig = "repo") +load("//third_party/com_github_hejsil_clap:repo.bzl", com_github_hejsil_clap = "repo") load("//third_party/com_google_sentencepiece:repo.bzl", com_google_sentencepiece = "repo") +load("//third_party/mnist:repo.bzl", mnist = "repo") +load("//third_party/org_swig_swig:repo.bzl", org_swig_swig = "repo") def _non_module_deps_impl(mctx): com_google_sentencepiece() org_swig_swig() + com_github_hejsil_clap() + mnist() return mctx.extension_metadata( reproducible = True, diff --git a/third_party/org_swig_swig/BUILD.bazel b/third_party/org_swig_swig/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/upb/upb.zig b/upb/upb.zig index 3033554c..84702859 100644 --- a/upb/upb.zig +++ b/upb/upb.zig @@ -126,6 +126,7 @@ pub const Allocator = struct { } fn alloc_impl(alloc: [*c]c.upb_alloc, ptr: ?*anyopaque, oldsize: usize, size: usize, actual_size: [*c]usize) callconv(.c) ?*anyopaque { + const PointerAlignedSlice = [*c]align(@alignOf(*anyopaque)) u8; const upb_alloc: *c.upb_alloc = alloc orelse return null; const self: *Allocator = @fieldParentPtr("upb_alloc", upb_alloc); defer { @@ -134,7 +135,7 @@ pub const Allocator = struct { } } if (ptr) |ptr_| { - const ptr_as_slice: []u8 = @as([*c]u8, @ptrCast(ptr_))[0..oldsize]; + const ptr_as_slice: []u8 = @as(PointerAlignedSlice, @ptrCast(@alignCast(ptr_)))[0..oldsize]; if (size == 0) { self.allocator.free(ptr_as_slice); return null; @@ -144,6 +145,6 @@ pub const Allocator = struct { @panic("Unsupported case"); } - return (self.allocator.alloc(u8, size) catch return null).ptr; + return @ptrCast(self.allocator.alignedAlloc(u8, @alignOf(*anyopaque), size) catch return null); } }; diff --git a/zml/tokenizer/hftokenizers/BUILD.bazel b/zml/tokenizer/hftokenizers/BUILD.bazel index 68188e38..54a01c8b 100644 --- a/zml/tokenizer/hftokenizers/BUILD.bazel +++ b/zml/tokenizer/hftokenizers/BUILD.bazel @@ -1,3 +1,4 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_rust//rust:defs.bzl", "rust_static_library") load("@rules_zig//zig:defs.bzl", "zig_library") @@ -28,11 +29,3 @@ zig_library( "//ffi:zig", ], ) - -cc_static_library( - name="hftokenizer_static", - deps = [ - ":hftokenizers_rs", - "//ffi:cc", - ] -) From aa87c024c152f6832048e0bf7da9318673d4a645 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 19 Aug 2025 19:44:20 +0200 Subject: [PATCH 014/881] workspace: re-add .vscode folder removed by mistake in #308 --- .vscode | 1 - .vscode/extensions.json | 8 +++++++ .vscode/settings.json | 37 ++++++++++++++++++++++++++++++++ examples/.vscode/extensions.json | 8 +++++++ examples/.vscode/settings.json | 37 ++++++++++++++++++++++++++++++++ 5 files changed, 90 insertions(+), 1 deletion(-) delete mode 120000 .vscode create mode 100644 .vscode/extensions.json create mode 100644 .vscode/settings.json create mode 100644 examples/.vscode/extensions.json create mode 100644 examples/.vscode/settings.json diff --git a/.vscode b/.vscode deleted file mode 120000 index fb3442e0..00000000 --- a/.vscode +++ /dev/null @@ -1 +0,0 @@ -examples/.vscode \ No newline at end of file diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 00000000..194cdc41 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,8 @@ +{ + "recommendations": [ + "BazelBuild.vscode-bazel", + "llvm-vs-code-extensions.vscode-mlir", + "ms-azuretools.vscode-docker", + "ziglang.vscode-zig", + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..82050af6 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,37 @@ +{ + "editor.formatOnSave": true, + "files.exclude": { + "**/.git": true, + "**/.svn": true, + "**/.hg": true, + "**/CVS": true, + "**/.DS_Store": true, + "**/bazel-*": true + }, + "files.watcherExclude": { + "**/bazel-*": true, + }, + // "files.trimTrailingWhitespace": true, + "files.insertFinalNewline": true, + "bazel.buildifierFixOnFormat": true, + "bazel.queriesShareServer": true, + "bazel.pathsToIgnore": [ + "." + ], + "bazel.buildifierExecutable": "@buildifier_prebuilt//:buildifier", + "C_Cpp.formatting": "clangFormat", + "C_Cpp.clang_format_style": "Microsoft", + "zig.formattingProvider": "zls", + "zig.path": "x", + "zig.zls.path": "${workspaceFolder}/tools/zls.sh", + "[zig]": { + "editor.suggest.insertMode": "replace", + "editor.stickyScroll.defaultModel": "foldingProviderModel", + "editor.codeActionsOnSave": { + "source.fixAll": "explicit", + } + }, + "rust-analyzer.linkedProjects": [ + "${workspaceFolder}/zml/tokenizer/hftokenizers/Cargo.toml" + ] +} diff --git a/examples/.vscode/extensions.json b/examples/.vscode/extensions.json new file mode 100644 index 00000000..194cdc41 --- /dev/null +++ b/examples/.vscode/extensions.json @@ -0,0 +1,8 @@ +{ + "recommendations": [ + "BazelBuild.vscode-bazel", + "llvm-vs-code-extensions.vscode-mlir", + "ms-azuretools.vscode-docker", + "ziglang.vscode-zig", + ] +} diff --git a/examples/.vscode/settings.json b/examples/.vscode/settings.json new file mode 100644 index 00000000..82050af6 --- /dev/null +++ b/examples/.vscode/settings.json @@ -0,0 +1,37 @@ +{ + "editor.formatOnSave": true, + "files.exclude": { + "**/.git": true, + "**/.svn": true, + "**/.hg": true, + "**/CVS": true, + "**/.DS_Store": true, + "**/bazel-*": true + }, + "files.watcherExclude": { + "**/bazel-*": true, + }, + // "files.trimTrailingWhitespace": true, + "files.insertFinalNewline": true, + "bazel.buildifierFixOnFormat": true, + "bazel.queriesShareServer": true, + "bazel.pathsToIgnore": [ + "." + ], + "bazel.buildifierExecutable": "@buildifier_prebuilt//:buildifier", + "C_Cpp.formatting": "clangFormat", + "C_Cpp.clang_format_style": "Microsoft", + "zig.formattingProvider": "zls", + "zig.path": "x", + "zig.zls.path": "${workspaceFolder}/tools/zls.sh", + "[zig]": { + "editor.suggest.insertMode": "replace", + "editor.stickyScroll.defaultModel": "foldingProviderModel", + "editor.codeActionsOnSave": { + "source.fixAll": "explicit", + } + }, + "rust-analyzer.linkedProjects": [ + "${workspaceFolder}/zml/tokenizer/hftokenizers/Cargo.toml" + ] +} From de4a0e0beb0111b6385551075ee63e1dba30a320 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 19 Aug 2025 23:35:01 +0200 Subject: [PATCH 015/881] third_party/xla: de-modularize xla (#309) This PR intends to remove XLA as a bzlmod and transfer it as a non bzlmod dep. This is because this module will never be upstreamed as is, so keep it private. Also, we fetch llvm-raw and stablehlo from it, which is fine. While there, dummify the various local_config XLA symbols to please the imports, as we don't use those parts in ZML itself. Closes #305 --- .bazelrc | 3 - MODULE.bazel | 44 +- MODULE.bazel.lock | 716 +++++++++--------- README.md | 6 +- third_party/com_google_sentencepiece/repo.bzl | 9 +- third_party/non_module_deps.bzl | 2 + third_party/xla/BUILD.bazel | 0 third_party/xla/llvm.bzl | 30 + .../0001-bazel-migration-to-bazel-8.1.1.patch | 41 + ...ler-registration-API-to-the-FFI-PjRt.patch | 135 ++++ ...ove-unconventional-C-code-in-headers.patch | 124 +++ third_party/xla/repo.bzl | 14 + third_party/xla/xla.bzl | 87 +++ tools/BUILD.bazel | 2 +- tools/hf/BUILD.bazel | 16 + tools/hf/pyproject.toml | 4 + tools/hf/requirements.lock.txt | 188 +++++ tools/hf_requirements.in | 1 - tools/hf_requirements.lock.txt | 203 ----- zml/tokenizer/hftokenizers/Cargo.lock | 149 ++-- zml/tokenizer/hftokenizers/Cargo.toml | 8 +- 21 files changed, 1079 insertions(+), 703 deletions(-) create mode 100644 third_party/xla/BUILD.bazel create mode 100644 third_party/xla/llvm.bzl create mode 100644 third_party/xla/patches/0001-bazel-migration-to-bazel-8.1.1.patch create mode 100644 third_party/xla/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch create mode 100644 third_party/xla/patches/0003-Remove-unconventional-C-code-in-headers.patch create mode 100644 third_party/xla/repo.bzl create mode 100644 third_party/xla/xla.bzl create mode 100644 tools/hf/BUILD.bazel create mode 100644 tools/hf/pyproject.toml create mode 100644 tools/hf/requirements.lock.txt delete mode 100644 tools/hf_requirements.in delete mode 100644 tools/hf_requirements.lock.txt diff --git a/.bazelrc b/.bazelrc index 928a29e1..ca73486e 100644 --- a/.bazelrc +++ b/.bazelrc @@ -78,9 +78,6 @@ common --incompatible_disallow_empty_glob=false common:macos --action_env=MACOSX_DEPLOYMENT_TARGET=13.0 common:macos --macos_sdk_version=13.0 -# Used by the tsl overlay in python_init_repositories for the bazelified xla module 20241025.0-4663f04 and later -common --repo_env=HERMETIC_PYTHON_VERSION=3.11 - # Don't detect the local C++ toolchain, be fully hermetic # This is done only on Linux for now since we use the local toolchain on macOS common:linux --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 diff --git a/MODULE.bazel b/MODULE.bazel index d48734d2..263ede18 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -3,29 +3,28 @@ module( ) bazel_dep(name = "abseil-cpp", version = "20250814.0") -bazel_dep(name = "aspect_bazel_lib", version = "2.14.0") -bazel_dep(name = "aspect_rules_py", version = "1.3.2") +bazel_dep(name = "aspect_bazel_lib", version = "2.20.0") +bazel_dep(name = "aspect_rules_py", version = "1.6.3") bazel_dep(name = "bazel_skylib", version = "1.8.1") bazel_dep(name = "libxev", version = "20250718.0-9f785d2") bazel_dep(name = "patchelf", version = "0.18.0") -bazel_dep(name = "pcre2", version = "10.43") +bazel_dep(name = "pcre2", version = "10.45") bazel_dep(name = "platforms", version = "1.0.0") bazel_dep(name = "protobuf", version = "32.0", repo_name = "com_google_protobuf") bazel_dep(name = "rules_cc", version = "0.2.0") bazel_dep(name = "rules_distroless", version = "0.5.1") -bazel_dep(name = "rules_proto", version = "7.1.0") +bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep(name = "rules_oci", version = "2.2.6") +bazel_dep(name = "rules_proto", version = "7.1.0") bazel_dep(name = "rules_python", version = "1.5.3") -bazel_dep(name = "rules_rust", version = "0.62.0") -bazel_dep(name = "rules_uv", version = "0.65.0") +bazel_dep(name = "rules_rust", version = "0.63.0") +bazel_dep(name = "rules_uv", version = "0.87.0") bazel_dep(name = "rules_zig", version = "20250714.0-b14a4f1") -bazel_dep(name = "sentencepiece", version = "20240618.0-d7ace0a") bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.2.4") bazel_dep(name = "with_cfg.bzl", version = "0.11.0") -bazel_dep(name = "xla", version = "20250718.0-6319f0d") bazel_dep(name = "zig-yaml", version = "20240903.0-83d5fdf") -bazel_dep(name = "buildifier_prebuilt", version = "8.0.3", dev_dependency = True) +bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2", dev_dependency = True) zig = use_extension("@rules_zig//zig:extensions.bzl", "zig") zig.index(file = "//bazel:zig_index.json") @@ -45,6 +44,9 @@ register_toolchains( "@toolchains_llvm_bootstrapped//toolchain:linux_aarch64", ) +uv = use_extension("@rules_python//python/uv:uv.bzl", "uv", dev_dependency = True) +uv.configure(version = "0.6.5") + python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain(python_version = "3.11") use_repo(python, "python_versions") @@ -70,7 +72,7 @@ pip.parse( download_only = True, hub_name = "huggingface_hub", python_version = "3.11", - requirements_lock = "//tools:hf_requirements.lock.txt", + requirements_lock = "//tools/hf:requirements.lock.txt", ) use_repo(pip, "huggingface_hub") @@ -94,15 +96,26 @@ use_repo(zls, "zls_aarch64-macos", "zls_x86_64-linux", "zls_x86_64-macos") register_toolchains("//third_party/zls:all") -xla = use_extension("@xla//:xla.bzl", "xla") +non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") +use_repo(non_module_deps, "com_github_hejsil_clap", "com_google_sentencepiece", "mnist", "org_swig_swig", "xla") + +xla = use_extension("//third_party/xla:xla.bzl", "xla") use_repo( xla, + "com_github_grpc_grpc", "llvm-raw", + "local_config_cuda", + "local_config_remote_execution", + "local_config_rocm", + "local_config_tensorrt", + "python_version_repo", + "rules_ml_toolchain", "stablehlo", "triton", + "tsl", ) -llvm = use_extension("@xla//:llvm.bzl", "llvm") +llvm = use_extension("//third_party/xla:llvm.bzl", "llvm") llvm.configure() use_repo(llvm, "llvm-project") @@ -115,7 +128,7 @@ rust.toolchain( "aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu", ], - versions = ["1.86.0"], + versions = ["1.89.0"], ) use_repo(rust, "rust_toolchains") @@ -135,9 +148,6 @@ crate.from_cargo( ) use_repo(crate, "crates") -non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") -use_repo(non_module_deps, "com_github_hejsil_clap", "com_google_sentencepiece", "mnist", "org_swig_swig") - apt = use_extension("@rules_distroless//apt:extensions.bzl", "apt") apt.install( name = "apt_cuda", @@ -145,14 +155,12 @@ apt.install( manifest = "//runtimes/cuda:packages.yaml", ) use_repo(apt, "apt_cuda") - apt.install( name = "apt_rocm", lock = "//runtimes/rocm:packages.lock.json", manifest = "//runtimes/rocm:packages.yaml", ) use_repo(apt, "apt_rocm") - apt.install( name = "apt_neuron", lock = "//runtimes/neuron:packages.lock.json", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 4ba6927d..4f754c09 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -7,7 +7,6 @@ "https://bazel-registry.zml.ai/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20230802.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20230802.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20250127.1/MODULE.bazel": "not found", @@ -16,13 +15,15 @@ "https://bazel-registry.zml.ai/modules/apple_support/1.11.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/apple_support/1.15.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/apple_support/1.17.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/apple_support/1.21.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.10.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.0.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.16.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.20.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.9.4/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/aspect_rules_py/1.3.2/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/aspect_rules_py/1.6.3/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/aspect_tools_telemetry/0.2.6/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.1.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.10.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.11.0/MODULE.bazel": "not found", @@ -34,8 +35,6 @@ "https://bazel-registry.zml.ai/modules/bazel_features/1.21.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.23.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.24.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.25.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.27.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.28.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.3.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.30.0/MODULE.bazel": "not found", @@ -55,8 +54,7 @@ "https://bazel-registry.zml.ai/modules/bazel_skylib/1.7.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_skylib/1.8.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/6.4.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/7.3.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/8.0.3/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/8.2.0.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/buildozer/7.1.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/google_benchmark/1.8.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/googletest/1.11.0/MODULE.bazel": "not found", @@ -64,12 +62,14 @@ "https://bazel-registry.zml.ai/modules/googletest/1.14.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/googletest/1.15.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/googletest/1.17.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/jq.bzl/0.1.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/jsoncpp/1.9.5/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/jsoncpp/1.9.6/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/libpfm/4.11.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/nlohmann_json/3.6.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/package_metadata/0.0.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/patchelf/0.18.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/pcre2/10.43/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/pcre2/10.45/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/platforms/0.0.10/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/platforms/0.0.11/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/platforms/0.0.4/MODULE.bazel": "not found", @@ -97,7 +97,6 @@ "https://bazel-registry.zml.ai/modules/re2/2024-07-02/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_android/0.1.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_apple/3.16.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_apple/3.22.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.10/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.13/MODULE.bazel": "not found", @@ -142,14 +141,11 @@ "https://bazel-registry.zml.ai/modules/rules_kotlin/1.9.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_kotlin/1.9.6/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_license/0.0.3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_license/0.0.4/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_license/0.0.7/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_license/0.0.8/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_license/1.0.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_multitool/0.11.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_oci/2.2.6/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_pkg/0.7.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_pkg/0.9.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_pkg/1.0.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_proto/4.0.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "not found", @@ -165,20 +161,17 @@ "https://bazel-registry.zml.ai/modules/rules_python/0.31.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_python/0.33.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_python/0.34.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.39.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_python/0.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_python/0.40.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/1.3.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_python/1.4.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_python/1.5.3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_rust/0.62.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_rust/0.63.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.2.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.3.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.4.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_swift/2.4.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_uv/0.65.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_uv/0.87.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.4/MODULE.bazel": "not found", @@ -188,13 +181,14 @@ "https://bazel-registry.zml.ai/modules/stardoc/0.7.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.7.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/tar.bzl/0.2.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/toolchains_llvm_bootstrapped/0.2.4/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/upb/0.0.0-20230516-61a97ef/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/with_cfg.bzl/0.11.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/with_cfg.bzl/0.9.2/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/yq.bzl/0.1.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/zlib/1.2.11/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/zlib/1.2.13/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/zlib/1.3.1/MODULE.bazel": "not found", "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", @@ -204,7 +198,6 @@ "https://bcr.bazel.build/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "1c8cec495288dccd14fdae6e3f95f772c1c91857047a098fad772034264cc8cb", "https://bcr.bazel.build/modules/abseil-cpp/20230802.0/MODULE.bazel": "d253ae36a8bd9ee3c5955384096ccb6baf16a1b1e93e858370da0a3b94f77c16", "https://bcr.bazel.build/modules/abseil-cpp/20230802.1/MODULE.bazel": "fa92e2eb41a04df73cdabeec37107316f7e5272650f81d6cc096418fe647b915", - "https://bcr.bazel.build/modules/abseil-cpp/20240116.0/MODULE.bazel": "98dc378d64c12a4e4741ad3362f87fb737ee6a0886b2d90c3cdbb4d93ea3e0bf", "https://bcr.bazel.build/modules/abseil-cpp/20240116.1/MODULE.bazel": "37bcdb4440fbb61df6a1c296ae01b327f19e9bb521f9b8e26ec854b6f97309ed", "https://bcr.bazel.build/modules/abseil-cpp/20240116.2/MODULE.bazel": "73939767a4686cd9a520d16af5ab440071ed75cec1a876bf2fcfaf1f71987a16", "https://bcr.bazel.build/modules/abseil-cpp/20250127.1/MODULE.bazel": "c4a89e7ceb9bf1e25cf84a9f830ff6b817b72874088bf5141b314726e46a57c1", @@ -214,16 +207,19 @@ "https://bcr.bazel.build/modules/apple_support/1.11.1/MODULE.bazel": "1843d7cd8a58369a444fc6000e7304425fba600ff641592161d9f15b179fb896", "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", "https://bcr.bazel.build/modules/apple_support/1.17.1/MODULE.bazel": "655c922ab1209978a94ef6ca7d9d43e940cd97d9c172fb55f94d91ac53f8610b", - "https://bcr.bazel.build/modules/apple_support/1.21.1/MODULE.bazel": "5809fa3efab15d1f3c3c635af6974044bac8a4919c62238cce06acee8a8c11f1", - "https://bcr.bazel.build/modules/apple_support/1.21.1/source.json": "c5228fea88af2d0aa4ef17355c5d3884ce78717f475c3eba55b55bca23a4a560", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.10.0/MODULE.bazel": "ae12288421b12faeb09bc4cfb3c05945ce66df5c55fca8a2f2f66ae9f21c7acb", + "https://bcr.bazel.build/modules/apple_support/1.17.1/source.json": "6b2b8c74d14e8d485528a938e44bdb72a5ba17632b9e14ef6e68a5ee96c8347f", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.0.0/MODULE.bazel": "e118477db5c49419a88d78ebc7a2c2cea9d49600fe0f490c1903324a2c16ecd9", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/source.json": "0cf1826853b0bef8b5cd19c0610d717500f5521aa2b38b72b2ec302ac5e7526c", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.16.0/MODULE.bazel": "852f9ebbda017572a7c113a2434592dd3b2f55cd9a0faea3d4be5a09a59e4900", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.20.0/MODULE.bazel": "c5565bac49e1973227225b441fad1c938d498d83df62dc5da95b2fab0f0626a2", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.20.0/source.json": "3eaada79dd3c65b6c57d5fc33c57ffd2896c4ebd78c4c9001a790a70f7f50e61", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "780d1a6522b28f5edb7ea09630748720721dfe27690d65a2d33aa7509de77e07", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "812d2dd42f65dca362152101fbec418029cc8fd34cbad1a2fde905383d705838", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.9.4/MODULE.bazel": "ccc41028429f894b02fde7ef67d416cba3ba5084ed9ddb9bb6107aa82d118776", - "https://bcr.bazel.build/modules/aspect_rules_py/1.3.2/MODULE.bazel": "0bd9f9f787a70a725b9829dd17c19e89ea86b1e06f25df505f780d77a96f55a5", - "https://bcr.bazel.build/modules/aspect_rules_py/1.3.2/source.json": "cafc6d1570e4503ff93cfd91309cdf1c96f468f3555993bd1654e75528a9bdd3", + "https://bcr.bazel.build/modules/aspect_rules_py/1.6.3/MODULE.bazel": "a34d989f9a258f478b40f69a0c1dd0a810b65173681b459c296ffeebabd997b0", + "https://bcr.bazel.build/modules/aspect_rules_py/1.6.3/source.json": "aac06d362804a5a4c1424efc37fc1ded6c6ac7d8be84ef28f735944f8a1f884d", + "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.2.6/MODULE.bazel": "cafb8781ad591bc57cc765dca5fefab08cf9f65af363d162b79d49205c7f8af7", + "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.2.6/source.json": "4d98137d5f74f01e00c6efa8bf591c02718e6c5f31f0bcc73983ea514dd02a12", "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", "https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101", "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", @@ -235,8 +231,6 @@ "https://bcr.bazel.build/modules/bazel_features/1.21.0/MODULE.bazel": "675642261665d8eea09989aa3b8afb5c37627f1be178382c320d1b46afba5e3b", "https://bcr.bazel.build/modules/bazel_features/1.23.0/MODULE.bazel": "fd1ac84bc4e97a5a0816b7fd7d4d4f6d837b0047cf4cbd81652d616af3a6591a", "https://bcr.bazel.build/modules/bazel_features/1.24.0/MODULE.bazel": "4796b4c25b47053e9bbffa792b3792d07e228ff66cd0405faef56a978708acd4", - "https://bcr.bazel.build/modules/bazel_features/1.25.0/MODULE.bazel": "e2e60a10a6da64bbf533f15ca652bf61a033e41c2ed734d79a9a08ba87f68c1a", - "https://bcr.bazel.build/modules/bazel_features/1.27.0/MODULE.bazel": "621eeee06c4458a9121d1f104efb80f39d34deff4984e778359c60eaf1a8cb65", "https://bcr.bazel.build/modules/bazel_features/1.28.0/MODULE.bazel": "4b4200e6cbf8fa335b2c3f43e1d6ef3e240319c33d43d60cc0fbd4b87ece299d", "https://bcr.bazel.build/modules/bazel_features/1.3.0/MODULE.bazel": "cdcafe83ec318cda34e02948e81d790aab8df7a929cec6f6969f13a489ccecd9", "https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87", @@ -258,9 +252,8 @@ "https://bcr.bazel.build/modules/bazel_skylib/1.8.1/MODULE.bazel": "88ade7293becda963e0e3ea33e7d54d3425127e0a326e0d17da085a5f1f03ff6", "https://bcr.bazel.build/modules/bazel_skylib/1.8.1/source.json": "7ebaefba0b03efe59cac88ed5bbc67bcf59a3eff33af937345ede2a38b2d368a", "https://bcr.bazel.build/modules/buildifier_prebuilt/6.4.0/MODULE.bazel": "37389c6b5a40c59410b4226d3bb54b08637f393d66e2fa57925c6fcf68e64bf4", - "https://bcr.bazel.build/modules/buildifier_prebuilt/7.3.1/MODULE.bazel": "537faf0ad9f5892910074b8e43b4c91c96f1d5d86b6ed04bdbe40cf68aa48b68", - "https://bcr.bazel.build/modules/buildifier_prebuilt/8.0.3/MODULE.bazel": "4791b031727d1183c183a0b8fd5fc0fe8d8834c45f9a22efd26be6c962b9cfee", - "https://bcr.bazel.build/modules/buildifier_prebuilt/8.0.3/source.json": "68ee16210562f325ff961aabeb4d46ed10e6638b5e842cd48d8d83dea220dfbd", + "https://bcr.bazel.build/modules/buildifier_prebuilt/8.2.0.2/MODULE.bazel": "a9b689711d5b69f9db741649b218c119b9fdf82924ba390415037e09798edd03", + "https://bcr.bazel.build/modules/buildifier_prebuilt/8.2.0.2/source.json": "51eb0a4b38aaaeab7fa64361576d616c4d8bfd0f17a0a10184aeab7084d79f8e", "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb", @@ -270,16 +263,20 @@ "https://bcr.bazel.build/modules/googletest/1.15.2/MODULE.bazel": "6de1edc1d26cafb0ea1a6ab3f4d4192d91a312fd2d360b63adaa213cd00b2108", "https://bcr.bazel.build/modules/googletest/1.17.0/MODULE.bazel": "dbec758171594a705933a29fcf69293d2468c49ec1f2ebca65c36f504d72df46", "https://bcr.bazel.build/modules/googletest/1.17.0/source.json": "38e4454b25fc30f15439c0378e57909ab1fd0a443158aa35aec685da727cd713", + "https://bcr.bazel.build/modules/jq.bzl/0.1.0/MODULE.bazel": "2ce69b1af49952cd4121a9c3055faa679e748ce774c7f1fda9657f936cae902f", + "https://bcr.bazel.build/modules/jq.bzl/0.1.0/source.json": "746bf13cac0860f091df5e4911d0c593971cd8796b5ad4e809b2f8e133eee3d5", "https://bcr.bazel.build/modules/jsoncpp/1.9.5/MODULE.bazel": "31271aedc59e815656f5736f282bb7509a97c7ecb43e927ac1a37966e0578075", "https://bcr.bazel.build/modules/jsoncpp/1.9.6/MODULE.bazel": "2f8d20d3b7d54143213c4dfc3d98225c42de7d666011528dc8fe91591e2e17b0", "https://bcr.bazel.build/modules/jsoncpp/1.9.6/source.json": "a04756d367a2126c3541682864ecec52f92cdee80a35735a3cb249ce015ca000", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/source.json": "f448c6e8963fdfa7eb831457df83ad63d3d6355018f6574fb017e8169deb43a9", + "https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92", + "https://bcr.bazel.build/modules/package_metadata/0.0.2/source.json": "e53a759a72488d2c0576f57491ef2da0cf4aab05ac0997314012495935531b73", "https://bcr.bazel.build/modules/patchelf/0.18.0/MODULE.bazel": "15a6beff7e828d585c5bd0f9f93589df117b5594e9d19e43096c77de58b9ae5f", "https://bcr.bazel.build/modules/patchelf/0.18.0/source.json": "57caf6bcaa5ba515c6fb1c2eacee00735afbeb1ffacb34a57553fb139c8e4333", - "https://bcr.bazel.build/modules/pcre2/10.43/MODULE.bazel": "08eaa025111bd0fedc14a8187c2905fa6ee4501fbe558193e9bf6cc3e2cdf23c", - "https://bcr.bazel.build/modules/pcre2/10.43/source.json": "8b4149e707094f1d5b57df7216539c3415226e814085c4d960bd9f3d49581b88", + "https://bcr.bazel.build/modules/pcre2/10.45/MODULE.bazel": "9bdc838aa2e21be030d7181385e45a5c87b14575bb7ff35d10b5be3d56913bed", + "https://bcr.bazel.build/modules/pcre2/10.45/source.json": "c0b10fd1afbfc978aad9a83e84533880b6c533026fa930d73bb95df34291bc4b", "https://bcr.bazel.build/modules/platforms/0.0.10/MODULE.bazel": "8cb8efaf200bdeb2150d93e162c40f388529a25852b332cec879373771e48ed5", "https://bcr.bazel.build/modules/platforms/0.0.11/MODULE.bazel": "0daefc49732e227caa8bfa834d65dc52e8cc18a2faf80df25e8caea151a9413f", "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee", @@ -312,8 +309,7 @@ "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", "https://bcr.bazel.build/modules/rules_apple/3.16.0/MODULE.bazel": "0d1caf0b8375942ce98ea944be754a18874041e4e0459401d925577624d3a54a", - "https://bcr.bazel.build/modules/rules_apple/3.22.0/MODULE.bazel": "0fd8c630d6cecc18abe84114ee21c1a0fb759b45063d3d2600ec22669e93ea43", - "https://bcr.bazel.build/modules/rules_apple/3.22.0/source.json": "ee41f63c292bd876fd264a830cf45500619a063e2fa3a18cca6fa12b153d68e3", + "https://bcr.bazel.build/modules/rules_apple/3.16.0/source.json": "d8b5fe461272018cc07cfafce11fe369c7525330804c37eec5a82f84cd475366", "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", "https://bcr.bazel.build/modules/rules_cc/0.0.10/MODULE.bazel": "ec1705118f7eaedd6e118508d3d26deba2a4e76476ada7e0e3965211be012002", "https://bcr.bazel.build/modules/rules_cc/0.0.13/MODULE.bazel": "0e8529ed7b323dad0775ff924d2ae5af7640b23553dfcd4d34344c7e7a867191", @@ -363,9 +359,7 @@ "https://bcr.bazel.build/modules/rules_kotlin/1.9.6/MODULE.bazel": "d269a01a18ee74d0335450b10f62c9ed81f2321d7958a2934e44272fe82dcef3", "https://bcr.bazel.build/modules/rules_kotlin/1.9.6/source.json": "2faa4794364282db7c06600b7e5e34867a564ae91bda7cae7c29c64e9466b7d5", "https://bcr.bazel.build/modules/rules_license/0.0.3/MODULE.bazel": "627e9ab0247f7d1e05736b59dbb1b6871373de5ad31c3011880b4133cafd4bd0", - "https://bcr.bazel.build/modules/rules_license/0.0.4/MODULE.bazel": "6a88dd22800cf1f9f79ba32cacad0d3a423ed28efa2c2ed5582eaa78dd3ac1e5", "https://bcr.bazel.build/modules/rules_license/0.0.7/MODULE.bazel": "088fbeb0b6a419005b89cf93fe62d9517c0a2b8bb56af3244af65ecfe37e7d5d", - "https://bcr.bazel.build/modules/rules_license/0.0.8/MODULE.bazel": "5669c6fe49b5134dbf534db681ad3d67a2d49cfc197e4a95f1ca2fd7f3aebe96", "https://bcr.bazel.build/modules/rules_license/1.0.0/MODULE.bazel": "a7fda60eefdf3d8c827262ba499957e4df06f659330bbe6cdbdb975b768bb65c", "https://bcr.bazel.build/modules/rules_license/1.0.0/source.json": "a52c89e54cc311196e478f8382df91c15f7a2bfdf4c6cd0e2675cc2ff0b56efb", "https://bcr.bazel.build/modules/rules_multitool/0.11.0/MODULE.bazel": "8d9dda78d2398e136300d3ef4fbcc89ede7c32c158d8c016fa7d032df41c4aaf", @@ -373,7 +367,6 @@ "https://bcr.bazel.build/modules/rules_oci/2.2.6/MODULE.bazel": "2ba6ddd679269e00aeffe9ca04faa2d0ca4129650982c9246d0d459fe2da47d9", "https://bcr.bazel.build/modules/rules_oci/2.2.6/source.json": "94e7decb8f95d9465b0bbea71c65064cd16083be1350c7468f131818641dc4a5", "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc", - "https://bcr.bazel.build/modules/rules_pkg/0.9.1/MODULE.bazel": "af00144208c4be503bc920d043ba3284fb37385b3f6160b4a4daf4df80b4b823", "https://bcr.bazel.build/modules/rules_pkg/1.0.1/MODULE.bazel": "5b1df97dbc29623bccdf2b0dcd0f5cb08e2f2c9050aab1092fd39a41e82686ff", "https://bcr.bazel.build/modules/rules_pkg/1.0.1/source.json": "bd82e5d7b9ce2d31e380dd9f50c111d678c3bdaca190cb76b0e1c71b05e1ba8a", "https://bcr.bazel.build/modules/rules_proto/4.0.0/MODULE.bazel": "a7a7b6ce9bee418c1a760b3d84f83a299ad6952f9903c67f19e4edd964894e06", @@ -391,25 +384,22 @@ "https://bcr.bazel.build/modules/rules_python/0.31.0/MODULE.bazel": "93a43dc47ee570e6ec9f5779b2e64c1476a6ce921c48cc9a1678a91dd5f8fd58", "https://bcr.bazel.build/modules/rules_python/0.33.2/MODULE.bazel": "3e036c4ad8d804a4dad897d333d8dce200d943df4827cb849840055be8d2e937", "https://bcr.bazel.build/modules/rules_python/0.34.0/MODULE.bazel": "1d623d026e075b78c9fde483a889cda7996f5da4f36dffb24c246ab30f06513a", - "https://bcr.bazel.build/modules/rules_python/0.39.0/MODULE.bazel": "8154faf3b171e09d3ecfde0704d2e6db50bc0aba9db374421a3d5dda2e28ab91", "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", "https://bcr.bazel.build/modules/rules_python/0.40.0/MODULE.bazel": "9d1a3cd88ed7d8e39583d9ffe56ae8a244f67783ae89b60caafc9f5cf318ada7", - "https://bcr.bazel.build/modules/rules_python/1.3.0/MODULE.bazel": "8361d57eafb67c09b75bf4bbe6be360e1b8f4f18118ab48037f2bd50aa2ccb13", "https://bcr.bazel.build/modules/rules_python/1.4.1/MODULE.bazel": "8991ad45bdc25018301d6b7e1d3626afc3c8af8aaf4bc04f23d0b99c938b73a6", "https://bcr.bazel.build/modules/rules_python/1.5.3/MODULE.bazel": "d0b7fb08458ca7fd80a26bc00c9e0f1d011609cc3da0381faa2eccd88c6ebd98", "https://bcr.bazel.build/modules/rules_python/1.5.3/source.json": "06961e322e15331a2d88115a65af5d3f77cc46793f9d9aa0f928b95287337f12", - "https://bcr.bazel.build/modules/rules_rust/0.62.0/MODULE.bazel": "6a15b57982e278793c684f426e19166e62e73f1bd45fe3b6bcedd0b901177b37", - "https://bcr.bazel.build/modules/rules_rust/0.62.0/source.json": "1b3a6551a585ee47cafa21550c5eb87c6f3a56bb9761f9e3421ff1102f220437", + "https://bcr.bazel.build/modules/rules_rust/0.63.0/MODULE.bazel": "4144e1606661c7168d23e8b4e7c5f6fb28ef519d9d5d63e0bd789d1b2a4611f8", + "https://bcr.bazel.build/modules/rules_rust/0.63.0/source.json": "638d4731ad05d31835ba45cffc06e8dc1cca01692a681daf831378cf952ee7e6", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", "https://bcr.bazel.build/modules/rules_shell/0.4.1/source.json": "4757bd277fe1567763991c4425b483477bb82e35e777a56fd846eb5cceda324a", "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", - "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", - "https://bcr.bazel.build/modules/rules_swift/2.4.0/source.json": "a6577f57f9febbdc015a01f2a8f3487422032f134d6c61d18ed8e8ca3b9acc7c", - "https://bcr.bazel.build/modules/rules_uv/0.65.0/MODULE.bazel": "a700505a80ad0ad65f062c9335458d137f504b98c5937b338807bb5cbe305e73", - "https://bcr.bazel.build/modules/rules_uv/0.65.0/source.json": "315237fb5f44ada2d537aabb2f89c6ba2296569ed63fc7340ca66d65f5730081", + "https://bcr.bazel.build/modules/rules_swift/2.1.1/source.json": "40fc69dfaac64deddbb75bd99cdac55f4427d9ca0afbe408576a65428427a186", + "https://bcr.bazel.build/modules/rules_uv/0.87.0/MODULE.bazel": "1f50b350458384f5c327ce04d943361435c772a80834536fc0eeebe92f3a7b95", + "https://bcr.bazel.build/modules/rules_uv/0.87.0/source.json": "2d75593d5c1126f6b7a014d14fb0771a24b7bd2e1de700f3e5e522ff0554b95c", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", "https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4", @@ -421,6 +411,8 @@ "https://bcr.bazel.build/modules/stardoc/0.7.2/source.json": "58b029e5e901d6802967754adf0a9056747e8176f017cfe3607c0851f4d42216", "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "5e463fbfba7b1701d957555ed45097d7f984211330106ccd1352c6e0af0dcf91", "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/source.json": "32bd87e5f4d7acc57c5b2ff7c325ae3061d5e242c0c4c214ae87e0f1c13e54cb", + "https://bcr.bazel.build/modules/tar.bzl/0.2.1/MODULE.bazel": "52d1c00a80a8cc67acbd01649e83d8dd6a9dc426a6c0b754a04fe8c219c76468", + "https://bcr.bazel.build/modules/tar.bzl/0.2.1/source.json": "600ac6ff61744667a439e7b814ae59c1f29632c3984fccf8000c64c9db8d7bb6", "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.2.4/MODULE.bazel": "859f02226bc313a622f56b816582fb55d5aaced4a1ecde7397dd4a9ebd24321e", "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.2.4/source.json": "43c5fd6b585cbf0b642157b6843a25425d541fabe638acb7bc908846c4bda8ca", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", @@ -428,8 +420,9 @@ "https://bcr.bazel.build/modules/with_cfg.bzl/0.11.0/MODULE.bazel": "bcb1dba64b74542648a93d6313e160b2670f22395581edd8c4ffadb811618ef7", "https://bcr.bazel.build/modules/with_cfg.bzl/0.11.0/source.json": "19102ff7f611a7a821e11d866868ad541abb4f5757e37e1c40adb6834ebf6205", "https://bcr.bazel.build/modules/with_cfg.bzl/0.9.2/MODULE.bazel": "8008021cb802e87e22b3e2b6f403c0529d4b865703fdcd40e2f36c23a651291c", + "https://bcr.bazel.build/modules/yq.bzl/0.1.1/MODULE.bazel": "9039681f9bcb8958ee2c87ffc74bdafba9f4369096a2b5634b88abc0eaefa072", + "https://bcr.bazel.build/modules/yq.bzl/0.1.1/source.json": "2d2bad780a9f2b9195a4a370314d2c17ae95eaa745cefc2e12fbc49759b15aa3", "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", - "https://bcr.bazel.build/modules/zlib/1.2.13/MODULE.bazel": "aa6deb1b83c18ffecd940c4119aff9567cd0a671d7bba756741cb2ef043a29d5", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "eec517b5bbe5492629466e11dae908d043364302283de25581e3eb944326c4ca", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/source.json": "22bc55c47af97246cfc093d0acf683a7869377de362b5d1c552c2c2e16b7a806", "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198" @@ -438,8 +431,8 @@ "moduleExtensions": { "@@apple_support+//crosstool:setup.bzl%apple_cc_configure_extension": { "general": { - "bzlTransitiveDigest": "4vzWoFey2mnm603onNMfaDX4DRoTrnIgZawOBMnWpoI=", - "usagesDigest": "Er40Iux4rJmj16bfvk5XcY7UiJ/P97xDmaxJ0KpWmLs=", + "bzlTransitiveDigest": "JJV/C5bhYKZip41t/GGZvqIMmmrcwr0Ydybkn97uXxQ=", + "usagesDigest": "39X2JjPCOAk6sThDALGv1L4q85GNjda2yfszm/phxxw=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -469,8 +462,8 @@ }, "@@aspect_rules_py+//py:extensions.bzl%py_tools": { "general": { - "bzlTransitiveDigest": "YcVwG9jheqdm2/gbTPG6E6bzLLpuwTdLTs45NueVdyo=", - "usagesDigest": "/X+U31L/0W+mPo15khS6kjLevIqdkqh+JgfyWHDdHH8=", + "bzlTransitiveDigest": "geB/ya+5nbONwFUgzta9iVBeMKtviiSBwlZnE7Avcgs=", + "usagesDigest": "tVlELaqQFX7HPl1uFzRY9m7nVV7tY9roq1qfK75ApwE=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -505,6 +498,12 @@ "platform": "windows_amd64" } }, + "bsd_tar_windows_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", + "attributes": { + "platform": "windows_arm64" + } + }, "bsd_tar_toolchains": { "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%tar_toolchains_repo", "attributes": { @@ -512,25 +511,25 @@ } }, "rules_py_tools.darwin_amd64": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:tools.bzl%prebuilt_tool_repo", + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", "attributes": { "platform": "darwin_amd64" } }, "rules_py_tools.darwin_arm64": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:tools.bzl%prebuilt_tool_repo", + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", "attributes": { "platform": "darwin_arm64" } }, "rules_py_tools.linux_amd64": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:tools.bzl%prebuilt_tool_repo", + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", "attributes": { "platform": "linux_amd64" } }, "rules_py_tools.linux_arm64": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:tools.bzl%prebuilt_tool_repo", + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", "attributes": { "platform": "linux_arm64" } @@ -563,6 +562,16 @@ "aspect_bazel_lib", "aspect_bazel_lib+" ], + [ + "aspect_rules_py+", + "aspect_tools_telemetry_report", + "aspect_tools_telemetry++telemetry+aspect_tools_telemetry_report" + ], + [ + "aspect_rules_py+", + "bazel_skylib", + "bazel_skylib+" + ], [ "aspect_rules_py+", "bazel_tools", @@ -571,141 +580,34 @@ ] } }, - "@@buildifier_prebuilt+//:defs.bzl%buildifier_prebuilt_deps_extension": { + "@@aspect_tools_telemetry+//:extension.bzl%telemetry": { "general": { - "bzlTransitiveDigest": "2b8+Z7ONtWAGRdJuUABpUTkgdRtGsS/rswXQSm8q/ac=", - "usagesDigest": "E92htUw9elHOT0ZHIEdwgPVnslFqL2iu3pcplYXXAxk=", + "bzlTransitiveDigest": "aVNc9R1NtH1OtynCXvZ9j7WtILv+X93Qxzmkyn4x++8=", + "usagesDigest": "bLd5DJLF7d7awlSuBaWvY0qEqBLja5XVKriiQmSNEi8=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { - "buildifier_darwin_amd64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "aspect_tools_telemetry_report": { + "repoRuleId": "@@aspect_tools_telemetry+//:extension.bzl%tel_repository", "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildifier-darwin-amd64" - ], - "downloaded_file_path": "buildifier", - "executable": true, - "sha256": "b7a3152cde0b3971b1107f2274afe778c5c154dcdf6c9c669a231e3c004f047e" - } - }, - "buildifier_darwin_arm64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildifier-darwin-arm64" - ], - "downloaded_file_path": "buildifier", - "executable": true, - "sha256": "674c663f7b5cd03c002f8ca834a8c1c008ccb527a0a2a132d08a7a355883b22d" - } - }, - "buildifier_linux_amd64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildifier-linux-amd64" - ], - "downloaded_file_path": "buildifier", - "executable": true, - "sha256": "c969487c1af85e708576c8dfdd0bb4681eae58aad79e68ae48882c70871841b7" - } - }, - "buildifier_linux_arm64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildifier-linux-arm64" - ], - "downloaded_file_path": "buildifier", - "executable": true, - "sha256": "bdd9b92e2c65d46affeecaefb54e68d34c272d1f4a8c5b54929a3e92ab78820a" - } - }, - "buildifier_windows_amd64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildifier-windows-amd64.exe" - ], - "downloaded_file_path": "buildifier.exe", - "executable": true, - "sha256": "63a242f57e253efe7b9573d739c08a3d0e628efd84015c8dad17d87b6429e443" - } - }, - "buildozer_darwin_amd64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildozer-darwin-amd64" - ], - "downloaded_file_path": "buildozer", - "executable": true, - "sha256": "7aa7929faddcee5a879135356bb4d38af3e895893c78d361e7cdb8dfb3e7d5e4" - } - }, - "buildozer_darwin_arm64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildozer-darwin-arm64" - ], - "downloaded_file_path": "buildozer", - "executable": true, - "sha256": "33b776862c31bfb4e2e6e88186cae02746d9084da8b7341930e77d42718277cc" - } - }, - "buildozer_linux_amd64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildozer-linux-amd64" - ], - "downloaded_file_path": "buildozer", - "executable": true, - "sha256": "775d17118aecd023352e75611dbe7c12a340e23615ffb270a1dec3f339239949" - } - }, - "buildozer_linux_arm64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildozer-linux-arm64" - ], - "downloaded_file_path": "buildozer", - "executable": true, - "sha256": "058ad60f279a7b1db339bb287aed2056a992eddfe1b5d2e56d86256bdc9a93b3" - } - }, - "buildozer_windows_amd64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v8.0.3/buildozer-windows-amd64.exe" - ], - "downloaded_file_path": "buildozer.exe", - "executable": true, - "sha256": "5ba106b25230e0e321b7259bee325ae6fa5b5771e54163ad865b1d18856f59ca" - } - }, - "buildifier_prebuilt_toolchains": { - "repoRuleId": "@@buildifier_prebuilt+//:defs.bzl%_buildifier_toolchain_setup", - "attributes": { - "assets_json": "[{\"arch\":\"amd64\",\"name\":\"buildifier\",\"platform\":\"darwin\",\"sha256\":\"b7a3152cde0b3971b1107f2274afe778c5c154dcdf6c9c669a231e3c004f047e\",\"version\":\"v8.0.3\"},{\"arch\":\"arm64\",\"name\":\"buildifier\",\"platform\":\"darwin\",\"sha256\":\"674c663f7b5cd03c002f8ca834a8c1c008ccb527a0a2a132d08a7a355883b22d\",\"version\":\"v8.0.3\"},{\"arch\":\"amd64\",\"name\":\"buildifier\",\"platform\":\"linux\",\"sha256\":\"c969487c1af85e708576c8dfdd0bb4681eae58aad79e68ae48882c70871841b7\",\"version\":\"v8.0.3\"},{\"arch\":\"arm64\",\"name\":\"buildifier\",\"platform\":\"linux\",\"sha256\":\"bdd9b92e2c65d46affeecaefb54e68d34c272d1f4a8c5b54929a3e92ab78820a\",\"version\":\"v8.0.3\"},{\"arch\":\"amd64\",\"name\":\"buildifier\",\"platform\":\"windows\",\"sha256\":\"63a242f57e253efe7b9573d739c08a3d0e628efd84015c8dad17d87b6429e443\",\"version\":\"v8.0.3\"},{\"arch\":\"amd64\",\"name\":\"buildozer\",\"platform\":\"darwin\",\"sha256\":\"7aa7929faddcee5a879135356bb4d38af3e895893c78d361e7cdb8dfb3e7d5e4\",\"version\":\"v8.0.3\"},{\"arch\":\"arm64\",\"name\":\"buildozer\",\"platform\":\"darwin\",\"sha256\":\"33b776862c31bfb4e2e6e88186cae02746d9084da8b7341930e77d42718277cc\",\"version\":\"v8.0.3\"},{\"arch\":\"amd64\",\"name\":\"buildozer\",\"platform\":\"linux\",\"sha256\":\"775d17118aecd023352e75611dbe7c12a340e23615ffb270a1dec3f339239949\",\"version\":\"v8.0.3\"},{\"arch\":\"arm64\",\"name\":\"buildozer\",\"platform\":\"linux\",\"sha256\":\"058ad60f279a7b1db339bb287aed2056a992eddfe1b5d2e56d86256bdc9a93b3\",\"version\":\"v8.0.3\"},{\"arch\":\"amd64\",\"name\":\"buildozer\",\"platform\":\"windows\",\"sha256\":\"5ba106b25230e0e321b7259bee325ae6fa5b5771e54163ad865b1d18856f59ca\",\"version\":\"v8.0.3\"}]" + "deps": { + "aspect_rules_py": "1.6.3", + "aspect_tools_telemetry": "0.2.6" + } } } }, "recordedRepoMappingEntries": [ [ - "buildifier_prebuilt+", - "bazel_skylib", - "bazel_skylib+" + "aspect_tools_telemetry+", + "aspect_bazel_lib", + "aspect_bazel_lib+" ], [ - "buildifier_prebuilt+", - "bazel_tools", - "bazel_tools" + "aspect_tools_telemetry+", + "bazel_skylib", + "bazel_skylib+" ] ] } @@ -742,8 +644,8 @@ }, "@@rules_apple+//apple:apple.bzl%provisioning_profile_repository_extension": { "general": { - "bzlTransitiveDigest": "mVdP3ANttcsLRv2k6Sbzs06sQDCgBTB8CPMicPSeCW4=", - "usagesDigest": "+Mri48un5kxPRrX8WFbwbgbUsG6JygfULZLaltX0weQ=", + "bzlTransitiveDigest": "BRNIrN/yKvsUnyouziBgSAp7x+TIcFZhks4bYDTh9iA=", + "usagesDigest": "vsJl8Rw5NL+5Ag2wdUDoTeRF/5klkXO8545Iy7U1Q08=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -822,6 +724,34 @@ ] } }, + "@@rules_apple+//apple:extensions.bzl%non_module_deps": { + "general": { + "bzlTransitiveDigest": "MABz1Esd3zxY4bS7aJoMhMKwBUdwC27e+3XkbPb3QcQ=", + "usagesDigest": "M3VqFpeTCo4qmrNKGZw0dxBHvTYDrfV3cscGzlSAhQ4=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "xctestrunner": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/google/xctestrunner/archive/b7698df3d435b6491b4b4c0f9fc7a63fbed5e3a6.tar.gz" + ], + "strip_prefix": "xctestrunner-b7698df3d435b6491b4b4c0f9fc7a63fbed5e3a6", + "sha256": "ae3a063c985a8633cb7eb566db21656f8db8eb9a0edb8c182312c7f0db53730d" + } + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_apple+", + "bazel_tools", + "bazel_tools" + ] + ] + } + }, "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { "general": { "bzlTransitiveDigest": "URQTcVYyatfFO8w2HD1Z1tb8lTBnhHn1wkiu5DYTSow=", @@ -889,7 +819,7 @@ "@@rules_multitool+//multitool:extension.bzl%multitool": { "general": { "bzlTransitiveDigest": "CxG1tSKAppSqmYJtpLgGpxrOvrd/xewKD0M2kCNfvOo=", - "usagesDigest": "Bn6IoM2pzksz/oBIRwZyfLpBZBFrZnzat00Xcp/lWqw=", + "usagesDigest": "kKFKsjNo1vbiSGlyYYHdSAj1Z8xtmvq8JBs56ZEpNS4=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -984,7 +914,7 @@ }, "@@rules_oci+//oci:extensions.bzl%oci": { "general": { - "bzlTransitiveDigest": "XTbZNic3alDB0IiYqOdBi1njq1/HjCKwaI1nW+qtDUw=", + "bzlTransitiveDigest": "4LJBCkExDO/zET3DzuMS4pDRRr4V0gc6YgFMhM6wIjI=", "usagesDigest": "vK9qUa3pyGJik2hb1NI2fKVJrjIszor+wN/44XMkKi0=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -1174,24 +1104,155 @@ "@@rules_python+//python/uv:uv.bzl%uv": { "general": { "bzlTransitiveDigest": "Vd/0cN91U7O//KVXEBJsnr64LUyHdC6DRiDTTnQTrUw=", - "usagesDigest": "4FQg4uUSAYXV3gvUvHtsV28NjaoGjNF9sJNQ66s1Fl8=", + "usagesDigest": "t11Aj3mDwK2nwRAqSmqylVpvomzPYWiSVUL4xbuiMm4=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { + "uv_0_6_5_aarch64_apple_darwin": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.6.5", + "platform": "aarch64-apple-darwin", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-aarch64-apple-darwin.tar.gz" + ], + "sha256": "4952c6275778fbc7e5cf6b5739ffd944c47063134d6ea43a7037b0a028ef771e" + } + }, + "uv_0_6_5_aarch64_unknown_linux_gnu": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.6.5", + "platform": "aarch64-unknown-linux-gnu", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-aarch64-unknown-linux-gnu.tar.gz" + ], + "sha256": "ebf0707ab949b948ad14250ed93f968fec954e20d12ce6bc9fce74b0ce7e4900" + } + }, + "uv_0_6_5_powerpc64_unknown_linux_gnu": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.6.5", + "platform": "powerpc64-unknown-linux-gnu", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-powerpc64-unknown-linux-gnu.tar.gz" + ], + "sha256": "1f8fd8dde91258380dbcd4b9fc65f15a7dc3af6cfffefb9a4f99d32c4c67436d" + } + }, + "uv_0_6_5_powerpc64le_unknown_linux_gnu": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.6.5", + "platform": "powerpc64le-unknown-linux-gnu", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-powerpc64le-unknown-linux-gnu.tar.gz" + ], + "sha256": "d2b6e19ca7ecd8d7e0d1ddced965aa91a3c822d07a7f7657743945245c1f627a" + } + }, + "uv_0_6_5_s390x_unknown_linux_gnu": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.6.5", + "platform": "s390x-unknown-linux-gnu", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-s390x-unknown-linux-gnu.tar.gz" + ], + "sha256": "43a3e39f19fd5d098bb8c896b6c0f99a146253c271d78409cf31c2f55f2763d0" + } + }, + "uv_0_6_5_x86_64_apple_darwin": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.6.5", + "platform": "x86_64-apple-darwin", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-x86_64-apple-darwin.tar.gz" + ], + "sha256": "d336d862694e1c36afea55eb15f5df02aa77c04a168b781df515d641e23285f0" + } + }, + "uv_0_6_5_x86_64_pc_windows_msvc": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.6.5", + "platform": "x86_64-pc-windows-msvc", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-x86_64-pc-windows-msvc.zip" + ], + "sha256": "a3e9f0913189c38b804bf6bb3d348af305a8ceabc690be92b6e111245a821640" + } + }, + "uv_0_6_5_x86_64_unknown_linux_gnu": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.6.5", + "platform": "x86_64-unknown-linux-gnu", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-x86_64-unknown-linux-gnu.tar.gz" + ], + "sha256": "8fc9895719a1291ecd193cb86f9282ff3649cef797d29eacc74c4f573aab1e2f" + } + }, "uv": { "repoRuleId": "@@rules_python+//python/uv/private:uv_toolchains_repo.bzl%uv_toolchains_repo", "attributes": { "toolchain_type": "'@@rules_python+//python/uv:uv_toolchain_type'", "toolchain_names": [ - "none" + "0_6_5_aarch64_apple_darwin", + "0_6_5_aarch64_unknown_linux_gnu", + "0_6_5_powerpc64_unknown_linux_gnu", + "0_6_5_powerpc64le_unknown_linux_gnu", + "0_6_5_s390x_unknown_linux_gnu", + "0_6_5_x86_64_apple_darwin", + "0_6_5_x86_64_pc_windows_msvc", + "0_6_5_x86_64_unknown_linux_gnu" ], "toolchain_implementations": { - "none": "'@@rules_python+//python:none'" + "0_6_5_aarch64_apple_darwin": "@uv_0_6_5_aarch64_apple_darwin//:uv_toolchain", + "0_6_5_aarch64_unknown_linux_gnu": "@uv_0_6_5_aarch64_unknown_linux_gnu//:uv_toolchain", + "0_6_5_powerpc64_unknown_linux_gnu": "@uv_0_6_5_powerpc64_unknown_linux_gnu//:uv_toolchain", + "0_6_5_powerpc64le_unknown_linux_gnu": "@uv_0_6_5_powerpc64le_unknown_linux_gnu//:uv_toolchain", + "0_6_5_s390x_unknown_linux_gnu": "@uv_0_6_5_s390x_unknown_linux_gnu//:uv_toolchain", + "0_6_5_x86_64_apple_darwin": "@uv_0_6_5_x86_64_apple_darwin//:uv_toolchain", + "0_6_5_x86_64_pc_windows_msvc": "@uv_0_6_5_x86_64_pc_windows_msvc//:uv_toolchain", + "0_6_5_x86_64_unknown_linux_gnu": "@uv_0_6_5_x86_64_unknown_linux_gnu//:uv_toolchain" }, "toolchain_compatible_with": { - "none": [ - "@platforms//:incompatible" + "0_6_5_aarch64_apple_darwin": [ + "'@@platforms//os:macos'", + "'@@platforms//cpu:aarch64'" + ], + "0_6_5_aarch64_unknown_linux_gnu": [ + "'@@platforms//os:linux'", + "'@@platforms//cpu:aarch64'" + ], + "0_6_5_powerpc64_unknown_linux_gnu": [ + "'@@platforms//os:linux'", + "'@@platforms//cpu:ppc'" + ], + "0_6_5_powerpc64le_unknown_linux_gnu": [ + "'@@platforms//os:linux'", + "'@@platforms//cpu:ppc64le'" + ], + "0_6_5_s390x_unknown_linux_gnu": [ + "'@@platforms//os:linux'", + "'@@platforms//cpu:s390x'" + ], + "0_6_5_x86_64_apple_darwin": [ + "'@@platforms//os:macos'", + "'@@platforms//cpu:x86_64'" + ], + "0_6_5_x86_64_pc_windows_msvc": [ + "'@@platforms//os:windows'", + "'@@platforms//cpu:x86_64'" + ], + "0_6_5_x86_64_unknown_linux_gnu": [ + "'@@platforms//os:linux'", + "'@@platforms//cpu:x86_64'" ] }, "toolchain_target_settings": {} @@ -1214,11 +1275,11 @@ }, "@@rules_rust+//crate_universe:extensions.bzl%crate": { "general": { - "bzlTransitiveDigest": "QaxJz0tRTK5Cb0Lu/nLbkmNxQTb6xlpvX7v9XZdv02k=", + "bzlTransitiveDigest": "HHPRUh/mbCXdC11w2axMRLWeweoYK1bLuS1e2NpvcoM=", "usagesDigest": "mup6mgSFatk34Q0znMZPpIEOFZyk9v3xf6jRIXEFTxc=", "recordedFileInputs": { - "@@//zml/tokenizer/hftokenizers/Cargo.lock": "93198160b29c4607994b405ffa0ad1c29b08df1ad22ec9a711874914bbaadbcb", - "@@//zml/tokenizer/hftokenizers/Cargo.toml": "eaf4574383be4d13a653a9c91c052979b05630f1c251b87bc8386263165dc204" + "@@//zml/tokenizer/hftokenizers/Cargo.lock": "d78ffab37ef69bf8ff45a4d686b25175f30e8fed4d8375612e84dcf6db0cfc9c", + "@@//zml/tokenizer/hftokenizers/Cargo.toml": "eeccbf5ff71c5f26b94cb289b3fc64c5fc6eb8708b815f39ce5f16bf225b6774" }, "recordedDirentsInputs": {}, "envVariables": { @@ -1235,9 +1296,9 @@ "repoRuleId": "@@rules_rust+//crate_universe:extensions.bzl%_generate_repo", "attributes": { "contents": { - "BUILD.bazel": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files(\n [\n \"cargo-bazel.json\",\n \"crates.bzl\",\n \"defs.bzl\",\n ] + glob(\n allow_empty = True,\n include = [\"*.bazel\"],\n ),\n)\n\nfilegroup(\n name = \"srcs\",\n srcs = glob(\n allow_empty = True,\n include = [\n \"*.bazel\",\n \"*.bzl\",\n ],\n ),\n)\n\n# Workspace Member Dependencies\nalias(\n name = \"tokenizers-0.21.2\",\n actual = \"@crates__tokenizers-0.21.2//:tokenizers\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"tokenizers\",\n actual = \"@crates__tokenizers-0.21.2//:tokenizers\",\n tags = [\"manual\"],\n)\n", + "BUILD.bazel": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files(\n [\n \"cargo-bazel.json\",\n \"crates.bzl\",\n \"defs.bzl\",\n ] + glob(\n allow_empty = True,\n include = [\"*.bazel\"],\n ),\n)\n\nfilegroup(\n name = \"srcs\",\n srcs = glob(\n allow_empty = True,\n include = [\n \"*.bazel\",\n \"*.bzl\",\n ],\n ),\n)\n\n# Workspace Member Dependencies\nalias(\n name = \"tokenizers-0.21.4\",\n actual = \"@crates__tokenizers-0.21.4//:tokenizers\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"tokenizers\",\n actual = \"@crates__tokenizers-0.21.4//:tokenizers\",\n tags = [\"manual\"],\n)\n", "alias_rules.bzl": "\"\"\"Alias that transitions its target to `compilation_mode=opt`. Use `transition_alias=\"opt\"` to enable.\"\"\"\n\nload(\"@rules_cc//cc:defs.bzl\", \"CcInfo\")\nload(\"@rules_rust//rust:rust_common.bzl\", \"COMMON_PROVIDERS\")\n\ndef _transition_alias_impl(ctx):\n # `ctx.attr.actual` is a list of 1 item due to the transition\n providers = [ctx.attr.actual[0][provider] for provider in COMMON_PROVIDERS]\n if CcInfo in ctx.attr.actual[0]:\n providers.append(ctx.attr.actual[0][CcInfo])\n return providers\n\ndef _change_compilation_mode(compilation_mode):\n def _change_compilation_mode_impl(_settings, _attr):\n return {\n \"//command_line_option:compilation_mode\": compilation_mode,\n }\n\n return transition(\n implementation = _change_compilation_mode_impl,\n inputs = [],\n outputs = [\n \"//command_line_option:compilation_mode\",\n ],\n )\n\ndef _transition_alias_rule(compilation_mode):\n return rule(\n implementation = _transition_alias_impl,\n provides = COMMON_PROVIDERS,\n attrs = {\n \"actual\": attr.label(\n mandatory = True,\n doc = \"`rust_library()` target to transition to `compilation_mode=opt`.\",\n providers = COMMON_PROVIDERS,\n cfg = _change_compilation_mode(compilation_mode),\n ),\n \"_allowlist_function_transition\": attr.label(\n default = \"@bazel_tools//tools/allowlists/function_transition_allowlist\",\n ),\n },\n doc = \"Transitions a Rust library crate to the `compilation_mode=opt`.\",\n )\n\ntransition_alias_dbg = _transition_alias_rule(\"dbg\")\ntransition_alias_fastbuild = _transition_alias_rule(\"fastbuild\")\ntransition_alias_opt = _transition_alias_rule(\"opt\")\n", - "defs.bzl": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\"\"\"\n# `crates_repository` API\n\n- [aliases](#aliases)\n- [crate_deps](#crate_deps)\n- [all_crate_deps](#all_crate_deps)\n- [crate_repositories](#crate_repositories)\n\n\"\"\"\n\nload(\"@bazel_tools//tools/build_defs/repo:git.bzl\", \"new_git_repository\")\nload(\"@bazel_tools//tools/build_defs/repo:http.bzl\", \"http_archive\")\nload(\"@bazel_tools//tools/build_defs/repo:utils.bzl\", \"maybe\")\nload(\"@bazel_skylib//lib:selects.bzl\", \"selects\")\nload(\"@rules_rust//crate_universe/private:local_crate_mirror.bzl\", \"local_crate_mirror\")\n\n###############################################################################\n# MACROS API\n###############################################################################\n\n# An identifier that represent common dependencies (unconditional).\n_COMMON_CONDITION = \"\"\n\ndef _flatten_dependency_maps(all_dependency_maps):\n \"\"\"Flatten a list of dependency maps into one dictionary.\n\n Dependency maps have the following structure:\n\n ```python\n DEPENDENCIES_MAP = {\n # The first key in the map is a Bazel package\n # name of the workspace this file is defined in.\n \"workspace_member_package\": {\n\n # Not all dependencies are supported for all platforms.\n # the condition key is the condition required to be true\n # on the host platform.\n \"condition\": {\n\n # An alias to a crate target. # The label of the crate target the\n # Aliases are only crate names. # package name refers to.\n \"package_name\": \"@full//:label\",\n }\n }\n }\n ```\n\n Args:\n all_dependency_maps (list): A list of dicts as described above\n\n Returns:\n dict: A dictionary as described above\n \"\"\"\n dependencies = {}\n\n for workspace_deps_map in all_dependency_maps:\n for pkg_name, conditional_deps_map in workspace_deps_map.items():\n if pkg_name not in dependencies:\n non_frozen_map = dict()\n for key, values in conditional_deps_map.items():\n non_frozen_map.update({key: dict(values.items())})\n dependencies.setdefault(pkg_name, non_frozen_map)\n continue\n\n for condition, deps_map in conditional_deps_map.items():\n # If the condition has not been recorded, do so and continue\n if condition not in dependencies[pkg_name]:\n dependencies[pkg_name].setdefault(condition, dict(deps_map.items()))\n continue\n\n # Alert on any miss-matched dependencies\n inconsistent_entries = []\n for crate_name, crate_label in deps_map.items():\n existing = dependencies[pkg_name][condition].get(crate_name)\n if existing and existing != crate_label:\n inconsistent_entries.append((crate_name, existing, crate_label))\n dependencies[pkg_name][condition].update({crate_name: crate_label})\n\n return dependencies\n\ndef crate_deps(deps, package_name = None):\n \"\"\"Finds the fully qualified label of the requested crates for the package where this macro is called.\n\n Args:\n deps (list): The desired list of crate targets.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()`.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if not deps:\n return []\n\n if package_name == None:\n package_name = native.package_name()\n\n # Join both sets of dependencies\n dependencies = _flatten_dependency_maps([\n _NORMAL_DEPENDENCIES,\n _NORMAL_DEV_DEPENDENCIES,\n _PROC_MACRO_DEPENDENCIES,\n _PROC_MACRO_DEV_DEPENDENCIES,\n _BUILD_DEPENDENCIES,\n _BUILD_PROC_MACRO_DEPENDENCIES,\n ]).pop(package_name, {})\n\n # Combine all conditional packages so we can easily index over a flat list\n # TODO: Perhaps this should actually return select statements and maintain\n # the conditionals of the dependencies\n flat_deps = {}\n for deps_set in dependencies.values():\n for crate_name, crate_label in deps_set.items():\n flat_deps.update({crate_name: crate_label})\n\n missing_crates = []\n crate_targets = []\n for crate_target in deps:\n if crate_target not in flat_deps:\n missing_crates.append(crate_target)\n else:\n crate_targets.append(flat_deps[crate_target])\n\n if missing_crates:\n fail(\"Could not find crates `{}` among dependencies of `{}`. Available dependencies were `{}`\".format(\n missing_crates,\n package_name,\n dependencies,\n ))\n\n return crate_targets\n\ndef all_crate_deps(\n normal = False, \n normal_dev = False, \n proc_macro = False, \n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Finds the fully qualified label of all requested direct crate dependencies \\\n for the package where this macro is called.\n\n If no parameters are set, all normal dependencies are returned. Setting any one flag will\n otherwise impact the contents of the returned list.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_dependency_maps = []\n if normal:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n if normal_dev:\n all_dependency_maps.append(_NORMAL_DEV_DEPENDENCIES)\n if proc_macro:\n all_dependency_maps.append(_PROC_MACRO_DEPENDENCIES)\n if proc_macro_dev:\n all_dependency_maps.append(_PROC_MACRO_DEV_DEPENDENCIES)\n if build:\n all_dependency_maps.append(_BUILD_DEPENDENCIES)\n if build_proc_macro:\n all_dependency_maps.append(_BUILD_PROC_MACRO_DEPENDENCIES)\n\n # Default to always using normal dependencies\n if not all_dependency_maps:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n\n dependencies = _flatten_dependency_maps(all_dependency_maps).pop(package_name, None)\n\n if not dependencies:\n if dependencies == None:\n fail(\"Tried to get all_crate_deps for package \" + package_name + \" but that package had no Cargo.toml file\")\n else:\n return []\n\n crate_deps = list(dependencies.pop(_COMMON_CONDITION, {}).values())\n for condition, deps in dependencies.items():\n crate_deps += selects.with_or({\n tuple(_CONDITIONS[condition]): deps.values(),\n \"//conditions:default\": [],\n })\n\n return crate_deps\n\ndef aliases(\n normal = False,\n normal_dev = False,\n proc_macro = False,\n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Produces a map of Crate alias names to their original label\n\n If no dependency kinds are specified, `normal` and `proc_macro` are used by default.\n Setting any one flag will otherwise determine the contents of the returned dict.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n dict: The aliases of all associated packages\n \"\"\"\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_aliases_maps = []\n if normal:\n all_aliases_maps.append(_NORMAL_ALIASES)\n if normal_dev:\n all_aliases_maps.append(_NORMAL_DEV_ALIASES)\n if proc_macro:\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n if proc_macro_dev:\n all_aliases_maps.append(_PROC_MACRO_DEV_ALIASES)\n if build:\n all_aliases_maps.append(_BUILD_ALIASES)\n if build_proc_macro:\n all_aliases_maps.append(_BUILD_PROC_MACRO_ALIASES)\n\n # Default to always using normal aliases\n if not all_aliases_maps:\n all_aliases_maps.append(_NORMAL_ALIASES)\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n\n aliases = _flatten_dependency_maps(all_aliases_maps).pop(package_name, None)\n\n if not aliases:\n return dict()\n\n common_items = aliases.pop(_COMMON_CONDITION, {}).items()\n\n # If there are only common items in the dictionary, immediately return them\n if not len(aliases.keys()) == 1:\n return dict(common_items)\n\n # Build a single select statement where each conditional has accounted for the\n # common set of aliases.\n crate_aliases = {\"//conditions:default\": dict(common_items)}\n for condition, deps in aliases.items():\n condition_triples = _CONDITIONS[condition]\n for triple in condition_triples:\n if triple in crate_aliases:\n crate_aliases[triple].update(deps)\n else:\n crate_aliases.update({triple: dict(deps.items() + common_items)})\n\n return select(crate_aliases)\n\n###############################################################################\n# WORKSPACE MEMBER DEPS AND ALIASES\n###############################################################################\n\n_NORMAL_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n _COMMON_CONDITION: {\n \"tokenizers\": Label(\"@crates//:tokenizers-0.21.2\"),\n },\n },\n}\n\n\n_NORMAL_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n _COMMON_CONDITION: {\n },\n },\n}\n\n\n_NORMAL_DEV_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_NORMAL_DEV_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEV_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEV_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_CONDITIONS = {\n \"aarch64-apple-darwin\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\"],\n \"aarch64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\"],\n \"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\",\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\": [],\n \"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\": [],\n \"cfg(any())\": [],\n \"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\": [],\n \"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\": [],\n \"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\": [],\n \"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\",\"@rules_rust//rust/platform:x86_64-apple-darwin\"],\n \"cfg(not(all(target_arch = \\\"arm\\\", target_os = \\\"none\\\")))\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\",\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\",\"@rules_rust//rust/platform:x86_64-apple-darwin\",\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"cfg(target_os = \\\"netbsd\\\")\": [],\n \"cfg(target_os = \\\"solaris\\\")\": [],\n \"cfg(target_os = \\\"vxworks\\\")\": [],\n \"x86_64-apple-darwin\": [\"@rules_rust//rust/platform:x86_64-apple-darwin\"],\n \"x86_64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n}\n\n###############################################################################\n\ndef crate_repositories():\n \"\"\"A macro for defining repositories for all generated crates.\n\n Returns:\n A list of repos visible to the module through the module extension.\n \"\"\"\n maybe(\n http_archive,\n name = \"crates__ahash-0.8.12\",\n sha256 = \"5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ahash/0.8.12/download\"],\n strip_prefix = \"ahash-0.8.12\",\n build_file = Label(\"@crates//crates:BUILD.ahash-0.8.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__aho-corasick-1.1.3\",\n sha256 = \"8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/aho-corasick/1.1.3/download\"],\n strip_prefix = \"aho-corasick-1.1.3\",\n build_file = Label(\"@crates//crates:BUILD.aho-corasick-1.1.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__base64-0.13.1\",\n sha256 = \"9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/base64/0.13.1/download\"],\n strip_prefix = \"base64-0.13.1\",\n build_file = Label(\"@crates//crates:BUILD.base64-0.13.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__bit-set-0.8.0\",\n sha256 = \"08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/bit-set/0.8.0/download\"],\n strip_prefix = \"bit-set-0.8.0\",\n build_file = Label(\"@crates//crates:BUILD.bit-set-0.8.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__bit-vec-0.8.0\",\n sha256 = \"5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/bit-vec/0.8.0/download\"],\n strip_prefix = \"bit-vec-0.8.0\",\n build_file = Label(\"@crates//crates:BUILD.bit-vec-0.8.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__bitflags-2.9.1\",\n sha256 = \"1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/bitflags/2.9.1/download\"],\n strip_prefix = \"bitflags-2.9.1\",\n build_file = Label(\"@crates//crates:BUILD.bitflags-2.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__bumpalo-3.16.0\",\n sha256 = \"79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/bumpalo/3.16.0/download\"],\n strip_prefix = \"bumpalo-3.16.0\",\n build_file = Label(\"@crates//crates:BUILD.bumpalo-3.16.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__byteorder-1.5.0\",\n sha256 = \"1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/byteorder/1.5.0/download\"],\n strip_prefix = \"byteorder-1.5.0\",\n build_file = Label(\"@crates//crates:BUILD.byteorder-1.5.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__castaway-0.2.4\",\n sha256 = \"dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/castaway/0.2.4/download\"],\n strip_prefix = \"castaway-0.2.4\",\n build_file = Label(\"@crates//crates:BUILD.castaway-0.2.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__cfg-if-1.0.0\",\n sha256 = \"baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/cfg-if/1.0.0/download\"],\n strip_prefix = \"cfg-if-1.0.0\",\n build_file = Label(\"@crates//crates:BUILD.cfg-if-1.0.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__compact_str-0.9.0\",\n sha256 = \"3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/compact_str/0.9.0/download\"],\n strip_prefix = \"compact_str-0.9.0\",\n build_file = Label(\"@crates//crates:BUILD.compact_str-0.9.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-deque-0.8.6\",\n sha256 = \"9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-deque/0.8.6/download\"],\n strip_prefix = \"crossbeam-deque-0.8.6\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-deque-0.8.6.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-epoch-0.9.18\",\n sha256 = \"5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-epoch/0.9.18/download\"],\n strip_prefix = \"crossbeam-epoch-0.9.18\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-epoch-0.9.18.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-utils-0.8.21\",\n sha256 = \"d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-utils/0.8.21/download\"],\n strip_prefix = \"crossbeam-utils-0.8.21\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-utils-0.8.21.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling-0.20.10\",\n sha256 = \"6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling/0.20.10/download\"],\n strip_prefix = \"darling-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling_core-0.20.10\",\n sha256 = \"95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling_core/0.20.10/download\"],\n strip_prefix = \"darling_core-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling_core-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling_macro-0.20.10\",\n sha256 = \"d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling_macro/0.20.10/download\"],\n strip_prefix = \"darling_macro-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling_macro-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__dary_heap-0.3.7\",\n sha256 = \"04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/dary_heap/0.3.7/download\"],\n strip_prefix = \"dary_heap-0.3.7\",\n build_file = Label(\"@crates//crates:BUILD.dary_heap-0.3.7.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder-0.20.2\",\n sha256 = \"507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder/0.20.2/download\"],\n strip_prefix = \"derive_builder-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder_core-0.20.2\",\n sha256 = \"2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder_core/0.20.2/download\"],\n strip_prefix = \"derive_builder_core-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder_core-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder_macro-0.20.2\",\n sha256 = \"ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder_macro/0.20.2/download\"],\n strip_prefix = \"derive_builder_macro-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder_macro-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__either-1.13.0\",\n sha256 = \"60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/either/1.13.0/download\"],\n strip_prefix = \"either-1.13.0\",\n build_file = Label(\"@crates//crates:BUILD.either-1.13.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__esaxx-rs-0.1.10\",\n sha256 = \"d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/esaxx-rs/0.1.10/download\"],\n strip_prefix = \"esaxx-rs-0.1.10\",\n build_file = Label(\"@crates//crates:BUILD.esaxx-rs-0.1.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__fancy-regex-0.14.0\",\n sha256 = \"6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/fancy-regex/0.14.0/download\"],\n strip_prefix = \"fancy-regex-0.14.0\",\n build_file = Label(\"@crates//crates:BUILD.fancy-regex-0.14.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__fnv-1.0.7\",\n sha256 = \"3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/fnv/1.0.7/download\"],\n strip_prefix = \"fnv-1.0.7\",\n build_file = Label(\"@crates//crates:BUILD.fnv-1.0.7.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__getrandom-0.3.3\",\n sha256 = \"26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/getrandom/0.3.3/download\"],\n strip_prefix = \"getrandom-0.3.3\",\n build_file = Label(\"@crates//crates:BUILD.getrandom-0.3.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ident_case-1.0.1\",\n sha256 = \"b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ident_case/1.0.1/download\"],\n strip_prefix = \"ident_case-1.0.1\",\n build_file = Label(\"@crates//crates:BUILD.ident_case-1.0.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__itertools-0.14.0\",\n sha256 = \"2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/itertools/0.14.0/download\"],\n strip_prefix = \"itertools-0.14.0\",\n build_file = Label(\"@crates//crates:BUILD.itertools-0.14.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__itoa-1.0.14\",\n sha256 = \"d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/itoa/1.0.14/download\"],\n strip_prefix = \"itoa-1.0.14\",\n build_file = Label(\"@crates//crates:BUILD.itoa-1.0.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__js-sys-0.3.77\",\n sha256 = \"1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/js-sys/0.3.77/download\"],\n strip_prefix = \"js-sys-0.3.77\",\n build_file = Label(\"@crates//crates:BUILD.js-sys-0.3.77.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__libc-0.2.169\",\n sha256 = \"b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/libc/0.2.169/download\"],\n strip_prefix = \"libc-0.2.169\",\n build_file = Label(\"@crates//crates:BUILD.libc-0.2.169.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__log-0.4.22\",\n sha256 = \"a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/log/0.4.22/download\"],\n strip_prefix = \"log-0.4.22\",\n build_file = Label(\"@crates//crates:BUILD.log-0.4.22.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__macro_rules_attribute-0.2.0\",\n sha256 = \"8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/macro_rules_attribute/0.2.0/download\"],\n strip_prefix = \"macro_rules_attribute-0.2.0\",\n build_file = Label(\"@crates//crates:BUILD.macro_rules_attribute-0.2.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__macro_rules_attribute-proc_macro-0.2.0\",\n sha256 = \"b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/macro_rules_attribute-proc_macro/0.2.0/download\"],\n strip_prefix = \"macro_rules_attribute-proc_macro-0.2.0\",\n build_file = Label(\"@crates//crates:BUILD.macro_rules_attribute-proc_macro-0.2.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__memchr-2.7.4\",\n sha256 = \"78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/memchr/2.7.4/download\"],\n strip_prefix = \"memchr-2.7.4\",\n build_file = Label(\"@crates//crates:BUILD.memchr-2.7.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__minimal-lexical-0.2.1\",\n sha256 = \"68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/minimal-lexical/0.2.1/download\"],\n strip_prefix = \"minimal-lexical-0.2.1\",\n build_file = Label(\"@crates//crates:BUILD.minimal-lexical-0.2.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__monostate-0.1.13\",\n sha256 = \"0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/monostate/0.1.13/download\"],\n strip_prefix = \"monostate-0.1.13\",\n build_file = Label(\"@crates//crates:BUILD.monostate-0.1.13.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__monostate-impl-0.1.13\",\n sha256 = \"a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/monostate-impl/0.1.13/download\"],\n strip_prefix = \"monostate-impl-0.1.13\",\n build_file = Label(\"@crates//crates:BUILD.monostate-impl-0.1.13.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__nom-7.1.3\",\n sha256 = \"d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/nom/7.1.3/download\"],\n strip_prefix = \"nom-7.1.3\",\n build_file = Label(\"@crates//crates:BUILD.nom-7.1.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__once_cell-1.20.2\",\n sha256 = \"1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/once_cell/1.20.2/download\"],\n strip_prefix = \"once_cell-1.20.2\",\n build_file = Label(\"@crates//crates:BUILD.once_cell-1.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__paste-1.0.15\",\n sha256 = \"57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/paste/1.0.15/download\"],\n strip_prefix = \"paste-1.0.15\",\n build_file = Label(\"@crates//crates:BUILD.paste-1.0.15.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ppv-lite86-0.2.20\",\n sha256 = \"77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ppv-lite86/0.2.20/download\"],\n strip_prefix = \"ppv-lite86-0.2.20\",\n build_file = Label(\"@crates//crates:BUILD.ppv-lite86-0.2.20.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__proc-macro2-1.0.92\",\n sha256 = \"37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/proc-macro2/1.0.92/download\"],\n strip_prefix = \"proc-macro2-1.0.92\",\n build_file = Label(\"@crates//crates:BUILD.proc-macro2-1.0.92.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__quote-1.0.37\",\n sha256 = \"b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/quote/1.0.37/download\"],\n strip_prefix = \"quote-1.0.37\",\n build_file = Label(\"@crates//crates:BUILD.quote-1.0.37.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__r-efi-5.3.0\",\n sha256 = \"69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/r-efi/5.3.0/download\"],\n strip_prefix = \"r-efi-5.3.0\",\n build_file = Label(\"@crates//crates:BUILD.r-efi-5.3.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand-0.9.1\",\n sha256 = \"9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand/0.9.1/download\"],\n strip_prefix = \"rand-0.9.1\",\n build_file = Label(\"@crates//crates:BUILD.rand-0.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand_chacha-0.9.0\",\n sha256 = \"d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand_chacha/0.9.0/download\"],\n strip_prefix = \"rand_chacha-0.9.0\",\n build_file = Label(\"@crates//crates:BUILD.rand_chacha-0.9.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand_core-0.9.3\",\n sha256 = \"99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand_core/0.9.3/download\"],\n strip_prefix = \"rand_core-0.9.3\",\n build_file = Label(\"@crates//crates:BUILD.rand_core-0.9.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-1.10.0\",\n sha256 = \"b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon/1.10.0/download\"],\n strip_prefix = \"rayon-1.10.0\",\n build_file = Label(\"@crates//crates:BUILD.rayon-1.10.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-cond-0.4.0\",\n sha256 = \"2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon-cond/0.4.0/download\"],\n strip_prefix = \"rayon-cond-0.4.0\",\n build_file = Label(\"@crates//crates:BUILD.rayon-cond-0.4.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-core-1.12.1\",\n sha256 = \"1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon-core/1.12.1/download\"],\n strip_prefix = \"rayon-core-1.12.1\",\n build_file = Label(\"@crates//crates:BUILD.rayon-core-1.12.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-1.11.1\",\n sha256 = \"b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex/1.11.1/download\"],\n strip_prefix = \"regex-1.11.1\",\n build_file = Label(\"@crates//crates:BUILD.regex-1.11.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-automata-0.4.9\",\n sha256 = \"809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-automata/0.4.9/download\"],\n strip_prefix = \"regex-automata-0.4.9\",\n build_file = Label(\"@crates//crates:BUILD.regex-automata-0.4.9.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-syntax-0.8.5\",\n sha256 = \"2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-syntax/0.8.5/download\"],\n strip_prefix = \"regex-syntax-0.8.5\",\n build_file = Label(\"@crates//crates:BUILD.regex-syntax-0.8.5.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rustversion-1.0.21\",\n sha256 = \"8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rustversion/1.0.21/download\"],\n strip_prefix = \"rustversion-1.0.21\",\n build_file = Label(\"@crates//crates:BUILD.rustversion-1.0.21.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ryu-1.0.18\",\n sha256 = \"f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ryu/1.0.18/download\"],\n strip_prefix = \"ryu-1.0.18\",\n build_file = Label(\"@crates//crates:BUILD.ryu-1.0.18.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde-1.0.216\",\n sha256 = \"0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde/1.0.216/download\"],\n strip_prefix = \"serde-1.0.216\",\n build_file = Label(\"@crates//crates:BUILD.serde-1.0.216.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde_derive-1.0.216\",\n sha256 = \"46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde_derive/1.0.216/download\"],\n strip_prefix = \"serde_derive-1.0.216\",\n build_file = Label(\"@crates//crates:BUILD.serde_derive-1.0.216.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde_json-1.0.134\",\n sha256 = \"d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde_json/1.0.134/download\"],\n strip_prefix = \"serde_json-1.0.134\",\n build_file = Label(\"@crates//crates:BUILD.serde_json-1.0.134.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__smallvec-1.13.2\",\n sha256 = \"3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/smallvec/1.13.2/download\"],\n strip_prefix = \"smallvec-1.13.2\",\n build_file = Label(\"@crates//crates:BUILD.smallvec-1.13.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__spm_precompiled-0.1.4\",\n sha256 = \"5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/spm_precompiled/0.1.4/download\"],\n strip_prefix = \"spm_precompiled-0.1.4\",\n build_file = Label(\"@crates//crates:BUILD.spm_precompiled-0.1.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__static_assertions-1.1.0\",\n sha256 = \"a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/static_assertions/1.1.0/download\"],\n strip_prefix = \"static_assertions-1.1.0\",\n build_file = Label(\"@crates//crates:BUILD.static_assertions-1.1.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__strsim-0.11.1\",\n sha256 = \"7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/strsim/0.11.1/download\"],\n strip_prefix = \"strsim-0.11.1\",\n build_file = Label(\"@crates//crates:BUILD.strsim-0.11.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__syn-2.0.91\",\n sha256 = \"d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/syn/2.0.91/download\"],\n strip_prefix = \"syn-2.0.91\",\n build_file = Label(\"@crates//crates:BUILD.syn-2.0.91.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__thiserror-2.0.12\",\n sha256 = \"567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/thiserror/2.0.12/download\"],\n strip_prefix = \"thiserror-2.0.12\",\n build_file = Label(\"@crates//crates:BUILD.thiserror-2.0.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__thiserror-impl-2.0.12\",\n sha256 = \"7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/thiserror-impl/2.0.12/download\"],\n strip_prefix = \"thiserror-impl-2.0.12\",\n build_file = Label(\"@crates//crates:BUILD.thiserror-impl-2.0.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__tokenizers-0.21.2\",\n sha256 = \"4c3846d8588abed0daba25a0e47edd58ea15e450a6088b2575f5116fdb0b27ca\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/tokenizers/0.21.2/download\"],\n strip_prefix = \"tokenizers-0.21.2\",\n build_file = Label(\"@crates//crates:BUILD.tokenizers-0.21.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-ident-1.0.14\",\n sha256 = \"adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-ident/1.0.14/download\"],\n strip_prefix = \"unicode-ident-1.0.14\",\n build_file = Label(\"@crates//crates:BUILD.unicode-ident-1.0.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-normalization-alignments-0.1.12\",\n sha256 = \"43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-normalization-alignments/0.1.12/download\"],\n strip_prefix = \"unicode-normalization-alignments-0.1.12\",\n build_file = Label(\"@crates//crates:BUILD.unicode-normalization-alignments-0.1.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-segmentation-1.12.0\",\n sha256 = \"f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-segmentation/1.12.0/download\"],\n strip_prefix = \"unicode-segmentation-1.12.0\",\n build_file = Label(\"@crates//crates:BUILD.unicode-segmentation-1.12.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode_categories-0.1.1\",\n sha256 = \"39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode_categories/0.1.1/download\"],\n strip_prefix = \"unicode_categories-0.1.1\",\n build_file = Label(\"@crates//crates:BUILD.unicode_categories-0.1.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__version_check-0.9.5\",\n sha256 = \"0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/version_check/0.9.5/download\"],\n strip_prefix = \"version_check-0.9.5\",\n build_file = Label(\"@crates//crates:BUILD.version_check-0.9.5.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasi-0.14.2-wasi-0.2.4\",\n sha256 = \"9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasi/0.14.2+wasi-0.2.4/download\"],\n strip_prefix = \"wasi-0.14.2+wasi-0.2.4\",\n build_file = Label(\"@crates//crates:BUILD.wasi-0.14.2+wasi-0.2.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasm-bindgen-0.2.100\",\n sha256 = \"1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasm-bindgen/0.2.100/download\"],\n strip_prefix = \"wasm-bindgen-0.2.100\",\n build_file = Label(\"@crates//crates:BUILD.wasm-bindgen-0.2.100.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasm-bindgen-backend-0.2.100\",\n sha256 = \"2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasm-bindgen-backend/0.2.100/download\"],\n strip_prefix = \"wasm-bindgen-backend-0.2.100\",\n build_file = Label(\"@crates//crates:BUILD.wasm-bindgen-backend-0.2.100.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasm-bindgen-macro-0.2.100\",\n sha256 = \"7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasm-bindgen-macro/0.2.100/download\"],\n strip_prefix = \"wasm-bindgen-macro-0.2.100\",\n build_file = Label(\"@crates//crates:BUILD.wasm-bindgen-macro-0.2.100.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasm-bindgen-macro-support-0.2.100\",\n sha256 = \"8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasm-bindgen-macro-support/0.2.100/download\"],\n strip_prefix = \"wasm-bindgen-macro-support-0.2.100\",\n build_file = Label(\"@crates//crates:BUILD.wasm-bindgen-macro-support-0.2.100.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasm-bindgen-shared-0.2.100\",\n sha256 = \"1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasm-bindgen-shared/0.2.100/download\"],\n strip_prefix = \"wasm-bindgen-shared-0.2.100\",\n build_file = Label(\"@crates//crates:BUILD.wasm-bindgen-shared-0.2.100.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wit-bindgen-rt-0.39.0\",\n sha256 = \"6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wit-bindgen-rt/0.39.0/download\"],\n strip_prefix = \"wit-bindgen-rt-0.39.0\",\n build_file = Label(\"@crates//crates:BUILD.wit-bindgen-rt-0.39.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-0.7.35\",\n sha256 = \"1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy/0.7.35/download\"],\n strip_prefix = \"zerocopy-0.7.35\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-0.7.35.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-0.8.26\",\n sha256 = \"1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy/0.8.26/download\"],\n strip_prefix = \"zerocopy-0.8.26\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-0.8.26.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-derive-0.7.35\",\n sha256 = \"fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy-derive/0.7.35/download\"],\n strip_prefix = \"zerocopy-derive-0.7.35\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-derive-0.7.35.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-derive-0.8.26\",\n sha256 = \"9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy-derive/0.8.26/download\"],\n strip_prefix = \"zerocopy-derive-0.8.26\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-derive-0.8.26.bazel\"),\n )\n\n return [\n struct(repo=\"crates__tokenizers-0.21.2\", is_dev_dep = False),\n ]\n" + "defs.bzl": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\"\"\"\n# `crates_repository` API\n\n- [aliases](#aliases)\n- [crate_deps](#crate_deps)\n- [all_crate_deps](#all_crate_deps)\n- [crate_repositories](#crate_repositories)\n\n\"\"\"\n\nload(\"@bazel_tools//tools/build_defs/repo:git.bzl\", \"new_git_repository\")\nload(\"@bazel_tools//tools/build_defs/repo:http.bzl\", \"http_archive\")\nload(\"@bazel_tools//tools/build_defs/repo:utils.bzl\", \"maybe\")\nload(\"@bazel_skylib//lib:selects.bzl\", \"selects\")\nload(\"@rules_rust//crate_universe/private:local_crate_mirror.bzl\", \"local_crate_mirror\")\n\n###############################################################################\n# MACROS API\n###############################################################################\n\n# An identifier that represent common dependencies (unconditional).\n_COMMON_CONDITION = \"\"\n\ndef _flatten_dependency_maps(all_dependency_maps):\n \"\"\"Flatten a list of dependency maps into one dictionary.\n\n Dependency maps have the following structure:\n\n ```python\n DEPENDENCIES_MAP = {\n # The first key in the map is a Bazel package\n # name of the workspace this file is defined in.\n \"workspace_member_package\": {\n\n # Not all dependencies are supported for all platforms.\n # the condition key is the condition required to be true\n # on the host platform.\n \"condition\": {\n\n # An alias to a crate target. # The label of the crate target the\n # Aliases are only crate names. # package name refers to.\n \"package_name\": \"@full//:label\",\n }\n }\n }\n ```\n\n Args:\n all_dependency_maps (list): A list of dicts as described above\n\n Returns:\n dict: A dictionary as described above\n \"\"\"\n dependencies = {}\n\n for workspace_deps_map in all_dependency_maps:\n for pkg_name, conditional_deps_map in workspace_deps_map.items():\n if pkg_name not in dependencies:\n non_frozen_map = dict()\n for key, values in conditional_deps_map.items():\n non_frozen_map.update({key: dict(values.items())})\n dependencies.setdefault(pkg_name, non_frozen_map)\n continue\n\n for condition, deps_map in conditional_deps_map.items():\n # If the condition has not been recorded, do so and continue\n if condition not in dependencies[pkg_name]:\n dependencies[pkg_name].setdefault(condition, dict(deps_map.items()))\n continue\n\n # Alert on any miss-matched dependencies\n inconsistent_entries = []\n for crate_name, crate_label in deps_map.items():\n existing = dependencies[pkg_name][condition].get(crate_name)\n if existing and existing != crate_label:\n inconsistent_entries.append((crate_name, existing, crate_label))\n dependencies[pkg_name][condition].update({crate_name: crate_label})\n\n return dependencies\n\ndef crate_deps(deps, package_name = None):\n \"\"\"Finds the fully qualified label of the requested crates for the package where this macro is called.\n\n Args:\n deps (list): The desired list of crate targets.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()`.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if not deps:\n return []\n\n if package_name == None:\n package_name = native.package_name()\n\n # Join both sets of dependencies\n dependencies = _flatten_dependency_maps([\n _NORMAL_DEPENDENCIES,\n _NORMAL_DEV_DEPENDENCIES,\n _PROC_MACRO_DEPENDENCIES,\n _PROC_MACRO_DEV_DEPENDENCIES,\n _BUILD_DEPENDENCIES,\n _BUILD_PROC_MACRO_DEPENDENCIES,\n ]).pop(package_name, {})\n\n # Combine all conditional packages so we can easily index over a flat list\n # TODO: Perhaps this should actually return select statements and maintain\n # the conditionals of the dependencies\n flat_deps = {}\n for deps_set in dependencies.values():\n for crate_name, crate_label in deps_set.items():\n flat_deps.update({crate_name: crate_label})\n\n missing_crates = []\n crate_targets = []\n for crate_target in deps:\n if crate_target not in flat_deps:\n missing_crates.append(crate_target)\n else:\n crate_targets.append(flat_deps[crate_target])\n\n if missing_crates:\n fail(\"Could not find crates `{}` among dependencies of `{}`. Available dependencies were `{}`\".format(\n missing_crates,\n package_name,\n dependencies,\n ))\n\n return crate_targets\n\ndef all_crate_deps(\n normal = False, \n normal_dev = False, \n proc_macro = False, \n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Finds the fully qualified label of all requested direct crate dependencies \\\n for the package where this macro is called.\n\n If no parameters are set, all normal dependencies are returned. Setting any one flag will\n otherwise impact the contents of the returned list.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_dependency_maps = []\n if normal:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n if normal_dev:\n all_dependency_maps.append(_NORMAL_DEV_DEPENDENCIES)\n if proc_macro:\n all_dependency_maps.append(_PROC_MACRO_DEPENDENCIES)\n if proc_macro_dev:\n all_dependency_maps.append(_PROC_MACRO_DEV_DEPENDENCIES)\n if build:\n all_dependency_maps.append(_BUILD_DEPENDENCIES)\n if build_proc_macro:\n all_dependency_maps.append(_BUILD_PROC_MACRO_DEPENDENCIES)\n\n # Default to always using normal dependencies\n if not all_dependency_maps:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n\n dependencies = _flatten_dependency_maps(all_dependency_maps).pop(package_name, None)\n\n if not dependencies:\n if dependencies == None:\n fail(\"Tried to get all_crate_deps for package \" + package_name + \" but that package had no Cargo.toml file\")\n else:\n return []\n\n crate_deps = list(dependencies.pop(_COMMON_CONDITION, {}).values())\n for condition, deps in dependencies.items():\n crate_deps += selects.with_or({\n tuple(_CONDITIONS[condition]): deps.values(),\n \"//conditions:default\": [],\n })\n\n return crate_deps\n\ndef aliases(\n normal = False,\n normal_dev = False,\n proc_macro = False,\n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Produces a map of Crate alias names to their original label\n\n If no dependency kinds are specified, `normal` and `proc_macro` are used by default.\n Setting any one flag will otherwise determine the contents of the returned dict.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n dict: The aliases of all associated packages\n \"\"\"\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_aliases_maps = []\n if normal:\n all_aliases_maps.append(_NORMAL_ALIASES)\n if normal_dev:\n all_aliases_maps.append(_NORMAL_DEV_ALIASES)\n if proc_macro:\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n if proc_macro_dev:\n all_aliases_maps.append(_PROC_MACRO_DEV_ALIASES)\n if build:\n all_aliases_maps.append(_BUILD_ALIASES)\n if build_proc_macro:\n all_aliases_maps.append(_BUILD_PROC_MACRO_ALIASES)\n\n # Default to always using normal aliases\n if not all_aliases_maps:\n all_aliases_maps.append(_NORMAL_ALIASES)\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n\n aliases = _flatten_dependency_maps(all_aliases_maps).pop(package_name, None)\n\n if not aliases:\n return dict()\n\n common_items = aliases.pop(_COMMON_CONDITION, {}).items()\n\n # If there are only common items in the dictionary, immediately return them\n if not len(aliases.keys()) == 1:\n return dict(common_items)\n\n # Build a single select statement where each conditional has accounted for the\n # common set of aliases.\n crate_aliases = {\"//conditions:default\": dict(common_items)}\n for condition, deps in aliases.items():\n condition_triples = _CONDITIONS[condition]\n for triple in condition_triples:\n if triple in crate_aliases:\n crate_aliases[triple].update(deps)\n else:\n crate_aliases.update({triple: dict(deps.items() + common_items)})\n\n return select(crate_aliases)\n\n###############################################################################\n# WORKSPACE MEMBER DEPS AND ALIASES\n###############################################################################\n\n_NORMAL_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n _COMMON_CONDITION: {\n \"tokenizers\": Label(\"@crates//:tokenizers-0.21.4\"),\n },\n },\n}\n\n\n_NORMAL_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n _COMMON_CONDITION: {\n },\n },\n}\n\n\n_NORMAL_DEV_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_NORMAL_DEV_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEV_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEV_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_CONDITIONS = {\n \"aarch64-apple-darwin\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\"],\n \"aarch64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\"],\n \"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\",\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\": [],\n \"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\": [],\n \"cfg(any())\": [],\n \"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\": [],\n \"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\": [],\n \"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\": [],\n \"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\",\"@rules_rust//rust/platform:x86_64-apple-darwin\"],\n \"cfg(not(all(target_arch = \\\"arm\\\", target_os = \\\"none\\\")))\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\",\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\",\"@rules_rust//rust/platform:x86_64-apple-darwin\",\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"cfg(target_os = \\\"netbsd\\\")\": [],\n \"cfg(target_os = \\\"solaris\\\")\": [],\n \"cfg(target_os = \\\"vxworks\\\")\": [],\n \"cfg(windows)\": [],\n \"x86_64-apple-darwin\": [\"@rules_rust//rust/platform:x86_64-apple-darwin\"],\n \"x86_64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n}\n\n###############################################################################\n\ndef crate_repositories():\n \"\"\"A macro for defining repositories for all generated crates.\n\n Returns:\n A list of repos visible to the module through the module extension.\n \"\"\"\n maybe(\n http_archive,\n name = \"crates__ahash-0.8.12\",\n sha256 = \"5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ahash/0.8.12/download\"],\n strip_prefix = \"ahash-0.8.12\",\n build_file = Label(\"@crates//crates:BUILD.ahash-0.8.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__aho-corasick-1.1.3\",\n sha256 = \"8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/aho-corasick/1.1.3/download\"],\n strip_prefix = \"aho-corasick-1.1.3\",\n build_file = Label(\"@crates//crates:BUILD.aho-corasick-1.1.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__base64-0.13.1\",\n sha256 = \"9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/base64/0.13.1/download\"],\n strip_prefix = \"base64-0.13.1\",\n build_file = Label(\"@crates//crates:BUILD.base64-0.13.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__bitflags-2.9.1\",\n sha256 = \"1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/bitflags/2.9.1/download\"],\n strip_prefix = \"bitflags-2.9.1\",\n build_file = Label(\"@crates//crates:BUILD.bitflags-2.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__byteorder-1.5.0\",\n sha256 = \"1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/byteorder/1.5.0/download\"],\n strip_prefix = \"byteorder-1.5.0\",\n build_file = Label(\"@crates//crates:BUILD.byteorder-1.5.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__castaway-0.2.4\",\n sha256 = \"dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/castaway/0.2.4/download\"],\n strip_prefix = \"castaway-0.2.4\",\n build_file = Label(\"@crates//crates:BUILD.castaway-0.2.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__cc-1.2.33\",\n sha256 = \"3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/cc/1.2.33/download\"],\n strip_prefix = \"cc-1.2.33\",\n build_file = Label(\"@crates//crates:BUILD.cc-1.2.33.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__cfg-if-1.0.0\",\n sha256 = \"baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/cfg-if/1.0.0/download\"],\n strip_prefix = \"cfg-if-1.0.0\",\n build_file = Label(\"@crates//crates:BUILD.cfg-if-1.0.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__compact_str-0.9.0\",\n sha256 = \"3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/compact_str/0.9.0/download\"],\n strip_prefix = \"compact_str-0.9.0\",\n build_file = Label(\"@crates//crates:BUILD.compact_str-0.9.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-deque-0.8.6\",\n sha256 = \"9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-deque/0.8.6/download\"],\n strip_prefix = \"crossbeam-deque-0.8.6\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-deque-0.8.6.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-epoch-0.9.18\",\n sha256 = \"5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-epoch/0.9.18/download\"],\n strip_prefix = \"crossbeam-epoch-0.9.18\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-epoch-0.9.18.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-utils-0.8.21\",\n sha256 = \"d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-utils/0.8.21/download\"],\n strip_prefix = \"crossbeam-utils-0.8.21\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-utils-0.8.21.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling-0.20.10\",\n sha256 = \"6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling/0.20.10/download\"],\n strip_prefix = \"darling-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling_core-0.20.10\",\n sha256 = \"95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling_core/0.20.10/download\"],\n strip_prefix = \"darling_core-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling_core-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling_macro-0.20.10\",\n sha256 = \"d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling_macro/0.20.10/download\"],\n strip_prefix = \"darling_macro-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling_macro-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__dary_heap-0.3.7\",\n sha256 = \"04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/dary_heap/0.3.7/download\"],\n strip_prefix = \"dary_heap-0.3.7\",\n build_file = Label(\"@crates//crates:BUILD.dary_heap-0.3.7.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder-0.20.2\",\n sha256 = \"507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder/0.20.2/download\"],\n strip_prefix = \"derive_builder-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder_core-0.20.2\",\n sha256 = \"2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder_core/0.20.2/download\"],\n strip_prefix = \"derive_builder_core-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder_core-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder_macro-0.20.2\",\n sha256 = \"ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder_macro/0.20.2/download\"],\n strip_prefix = \"derive_builder_macro-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder_macro-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__either-1.13.0\",\n sha256 = \"60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/either/1.13.0/download\"],\n strip_prefix = \"either-1.13.0\",\n build_file = Label(\"@crates//crates:BUILD.either-1.13.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__esaxx-rs-0.1.10\",\n sha256 = \"d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/esaxx-rs/0.1.10/download\"],\n strip_prefix = \"esaxx-rs-0.1.10\",\n build_file = Label(\"@crates//crates:BUILD.esaxx-rs-0.1.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__fnv-1.0.7\",\n sha256 = \"3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/fnv/1.0.7/download\"],\n strip_prefix = \"fnv-1.0.7\",\n build_file = Label(\"@crates//crates:BUILD.fnv-1.0.7.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__getrandom-0.3.3\",\n sha256 = \"26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/getrandom/0.3.3/download\"],\n strip_prefix = \"getrandom-0.3.3\",\n build_file = Label(\"@crates//crates:BUILD.getrandom-0.3.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ident_case-1.0.1\",\n sha256 = \"b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ident_case/1.0.1/download\"],\n strip_prefix = \"ident_case-1.0.1\",\n build_file = Label(\"@crates//crates:BUILD.ident_case-1.0.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__itertools-0.14.0\",\n sha256 = \"2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/itertools/0.14.0/download\"],\n strip_prefix = \"itertools-0.14.0\",\n build_file = Label(\"@crates//crates:BUILD.itertools-0.14.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__itoa-1.0.14\",\n sha256 = \"d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/itoa/1.0.14/download\"],\n strip_prefix = \"itoa-1.0.14\",\n build_file = Label(\"@crates//crates:BUILD.itoa-1.0.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__libc-0.2.169\",\n sha256 = \"b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/libc/0.2.169/download\"],\n strip_prefix = \"libc-0.2.169\",\n build_file = Label(\"@crates//crates:BUILD.libc-0.2.169.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__log-0.4.22\",\n sha256 = \"a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/log/0.4.22/download\"],\n strip_prefix = \"log-0.4.22\",\n build_file = Label(\"@crates//crates:BUILD.log-0.4.22.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__macro_rules_attribute-0.2.0\",\n sha256 = \"8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/macro_rules_attribute/0.2.0/download\"],\n strip_prefix = \"macro_rules_attribute-0.2.0\",\n build_file = Label(\"@crates//crates:BUILD.macro_rules_attribute-0.2.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__macro_rules_attribute-proc_macro-0.2.0\",\n sha256 = \"b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/macro_rules_attribute-proc_macro/0.2.0/download\"],\n strip_prefix = \"macro_rules_attribute-proc_macro-0.2.0\",\n build_file = Label(\"@crates//crates:BUILD.macro_rules_attribute-proc_macro-0.2.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__memchr-2.7.4\",\n sha256 = \"78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/memchr/2.7.4/download\"],\n strip_prefix = \"memchr-2.7.4\",\n build_file = Label(\"@crates//crates:BUILD.memchr-2.7.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__minimal-lexical-0.2.1\",\n sha256 = \"68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/minimal-lexical/0.2.1/download\"],\n strip_prefix = \"minimal-lexical-0.2.1\",\n build_file = Label(\"@crates//crates:BUILD.minimal-lexical-0.2.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__monostate-0.1.13\",\n sha256 = \"0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/monostate/0.1.13/download\"],\n strip_prefix = \"monostate-0.1.13\",\n build_file = Label(\"@crates//crates:BUILD.monostate-0.1.13.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__monostate-impl-0.1.13\",\n sha256 = \"a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/monostate-impl/0.1.13/download\"],\n strip_prefix = \"monostate-impl-0.1.13\",\n build_file = Label(\"@crates//crates:BUILD.monostate-impl-0.1.13.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__nom-7.1.3\",\n sha256 = \"d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/nom/7.1.3/download\"],\n strip_prefix = \"nom-7.1.3\",\n build_file = Label(\"@crates//crates:BUILD.nom-7.1.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__once_cell-1.20.2\",\n sha256 = \"1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/once_cell/1.20.2/download\"],\n strip_prefix = \"once_cell-1.20.2\",\n build_file = Label(\"@crates//crates:BUILD.once_cell-1.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__onig-6.5.1\",\n sha256 = \"336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/onig/6.5.1/download\"],\n strip_prefix = \"onig-6.5.1\",\n build_file = Label(\"@crates//crates:BUILD.onig-6.5.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__onig_sys-69.9.1\",\n sha256 = \"c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/onig_sys/69.9.1/download\"],\n strip_prefix = \"onig_sys-69.9.1\",\n build_file = Label(\"@crates//crates:BUILD.onig_sys-69.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__paste-1.0.15\",\n sha256 = \"57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/paste/1.0.15/download\"],\n strip_prefix = \"paste-1.0.15\",\n build_file = Label(\"@crates//crates:BUILD.paste-1.0.15.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__pkg-config-0.3.32\",\n sha256 = \"7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/pkg-config/0.3.32/download\"],\n strip_prefix = \"pkg-config-0.3.32\",\n build_file = Label(\"@crates//crates:BUILD.pkg-config-0.3.32.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ppv-lite86-0.2.20\",\n sha256 = \"77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ppv-lite86/0.2.20/download\"],\n strip_prefix = \"ppv-lite86-0.2.20\",\n build_file = Label(\"@crates//crates:BUILD.ppv-lite86-0.2.20.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__proc-macro2-1.0.92\",\n sha256 = \"37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/proc-macro2/1.0.92/download\"],\n strip_prefix = \"proc-macro2-1.0.92\",\n build_file = Label(\"@crates//crates:BUILD.proc-macro2-1.0.92.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__quote-1.0.37\",\n sha256 = \"b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/quote/1.0.37/download\"],\n strip_prefix = \"quote-1.0.37\",\n build_file = Label(\"@crates//crates:BUILD.quote-1.0.37.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__r-efi-5.3.0\",\n sha256 = \"69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/r-efi/5.3.0/download\"],\n strip_prefix = \"r-efi-5.3.0\",\n build_file = Label(\"@crates//crates:BUILD.r-efi-5.3.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand-0.9.1\",\n sha256 = \"9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand/0.9.1/download\"],\n strip_prefix = \"rand-0.9.1\",\n build_file = Label(\"@crates//crates:BUILD.rand-0.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand_chacha-0.9.0\",\n sha256 = \"d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand_chacha/0.9.0/download\"],\n strip_prefix = \"rand_chacha-0.9.0\",\n build_file = Label(\"@crates//crates:BUILD.rand_chacha-0.9.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand_core-0.9.3\",\n sha256 = \"99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand_core/0.9.3/download\"],\n strip_prefix = \"rand_core-0.9.3\",\n build_file = Label(\"@crates//crates:BUILD.rand_core-0.9.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-1.10.0\",\n sha256 = \"b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon/1.10.0/download\"],\n strip_prefix = \"rayon-1.10.0\",\n build_file = Label(\"@crates//crates:BUILD.rayon-1.10.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-cond-0.4.0\",\n sha256 = \"2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon-cond/0.4.0/download\"],\n strip_prefix = \"rayon-cond-0.4.0\",\n build_file = Label(\"@crates//crates:BUILD.rayon-cond-0.4.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-core-1.12.1\",\n sha256 = \"1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon-core/1.12.1/download\"],\n strip_prefix = \"rayon-core-1.12.1\",\n build_file = Label(\"@crates//crates:BUILD.rayon-core-1.12.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-1.11.1\",\n sha256 = \"b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex/1.11.1/download\"],\n strip_prefix = \"regex-1.11.1\",\n build_file = Label(\"@crates//crates:BUILD.regex-1.11.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-automata-0.4.9\",\n sha256 = \"809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-automata/0.4.9/download\"],\n strip_prefix = \"regex-automata-0.4.9\",\n build_file = Label(\"@crates//crates:BUILD.regex-automata-0.4.9.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-syntax-0.8.5\",\n sha256 = \"2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-syntax/0.8.5/download\"],\n strip_prefix = \"regex-syntax-0.8.5\",\n build_file = Label(\"@crates//crates:BUILD.regex-syntax-0.8.5.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rustversion-1.0.21\",\n sha256 = \"8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rustversion/1.0.21/download\"],\n strip_prefix = \"rustversion-1.0.21\",\n build_file = Label(\"@crates//crates:BUILD.rustversion-1.0.21.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ryu-1.0.18\",\n sha256 = \"f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ryu/1.0.18/download\"],\n strip_prefix = \"ryu-1.0.18\",\n build_file = Label(\"@crates//crates:BUILD.ryu-1.0.18.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde-1.0.216\",\n sha256 = \"0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde/1.0.216/download\"],\n strip_prefix = \"serde-1.0.216\",\n build_file = Label(\"@crates//crates:BUILD.serde-1.0.216.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde_derive-1.0.216\",\n sha256 = \"46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde_derive/1.0.216/download\"],\n strip_prefix = \"serde_derive-1.0.216\",\n build_file = Label(\"@crates//crates:BUILD.serde_derive-1.0.216.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde_json-1.0.134\",\n sha256 = \"d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde_json/1.0.134/download\"],\n strip_prefix = \"serde_json-1.0.134\",\n build_file = Label(\"@crates//crates:BUILD.serde_json-1.0.134.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__shlex-1.3.0\",\n sha256 = \"0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/shlex/1.3.0/download\"],\n strip_prefix = \"shlex-1.3.0\",\n build_file = Label(\"@crates//crates:BUILD.shlex-1.3.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__smallvec-1.13.2\",\n sha256 = \"3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/smallvec/1.13.2/download\"],\n strip_prefix = \"smallvec-1.13.2\",\n build_file = Label(\"@crates//crates:BUILD.smallvec-1.13.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__spm_precompiled-0.1.4\",\n sha256 = \"5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/spm_precompiled/0.1.4/download\"],\n strip_prefix = \"spm_precompiled-0.1.4\",\n build_file = Label(\"@crates//crates:BUILD.spm_precompiled-0.1.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__static_assertions-1.1.0\",\n sha256 = \"a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/static_assertions/1.1.0/download\"],\n strip_prefix = \"static_assertions-1.1.0\",\n build_file = Label(\"@crates//crates:BUILD.static_assertions-1.1.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__strsim-0.11.1\",\n sha256 = \"7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/strsim/0.11.1/download\"],\n strip_prefix = \"strsim-0.11.1\",\n build_file = Label(\"@crates//crates:BUILD.strsim-0.11.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__syn-2.0.91\",\n sha256 = \"d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/syn/2.0.91/download\"],\n strip_prefix = \"syn-2.0.91\",\n build_file = Label(\"@crates//crates:BUILD.syn-2.0.91.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__thiserror-2.0.12\",\n sha256 = \"567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/thiserror/2.0.12/download\"],\n strip_prefix = \"thiserror-2.0.12\",\n build_file = Label(\"@crates//crates:BUILD.thiserror-2.0.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__thiserror-impl-2.0.12\",\n sha256 = \"7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/thiserror-impl/2.0.12/download\"],\n strip_prefix = \"thiserror-impl-2.0.12\",\n build_file = Label(\"@crates//crates:BUILD.thiserror-impl-2.0.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__tokenizers-0.21.4\",\n sha256 = \"a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/tokenizers/0.21.4/download\"],\n strip_prefix = \"tokenizers-0.21.4\",\n build_file = Label(\"@crates//crates:BUILD.tokenizers-0.21.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-ident-1.0.14\",\n sha256 = \"adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-ident/1.0.14/download\"],\n strip_prefix = \"unicode-ident-1.0.14\",\n build_file = Label(\"@crates//crates:BUILD.unicode-ident-1.0.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-normalization-alignments-0.1.12\",\n sha256 = \"43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-normalization-alignments/0.1.12/download\"],\n strip_prefix = \"unicode-normalization-alignments-0.1.12\",\n build_file = Label(\"@crates//crates:BUILD.unicode-normalization-alignments-0.1.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-segmentation-1.12.0\",\n sha256 = \"f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-segmentation/1.12.0/download\"],\n strip_prefix = \"unicode-segmentation-1.12.0\",\n build_file = Label(\"@crates//crates:BUILD.unicode-segmentation-1.12.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode_categories-0.1.1\",\n sha256 = \"39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode_categories/0.1.1/download\"],\n strip_prefix = \"unicode_categories-0.1.1\",\n build_file = Label(\"@crates//crates:BUILD.unicode_categories-0.1.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__version_check-0.9.5\",\n sha256 = \"0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/version_check/0.9.5/download\"],\n strip_prefix = \"version_check-0.9.5\",\n build_file = Label(\"@crates//crates:BUILD.version_check-0.9.5.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasi-0.14.2-wasi-0.2.4\",\n sha256 = \"9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasi/0.14.2+wasi-0.2.4/download\"],\n strip_prefix = \"wasi-0.14.2+wasi-0.2.4\",\n build_file = Label(\"@crates//crates:BUILD.wasi-0.14.2+wasi-0.2.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wit-bindgen-rt-0.39.0\",\n sha256 = \"6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wit-bindgen-rt/0.39.0/download\"],\n strip_prefix = \"wit-bindgen-rt-0.39.0\",\n build_file = Label(\"@crates//crates:BUILD.wit-bindgen-rt-0.39.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-0.7.35\",\n sha256 = \"1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy/0.7.35/download\"],\n strip_prefix = \"zerocopy-0.7.35\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-0.7.35.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-0.8.26\",\n sha256 = \"1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy/0.8.26/download\"],\n strip_prefix = \"zerocopy-0.8.26\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-0.8.26.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-derive-0.7.35\",\n sha256 = \"fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy-derive/0.7.35/download\"],\n strip_prefix = \"zerocopy-derive-0.7.35\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-derive-0.7.35.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-derive-0.8.26\",\n sha256 = \"9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy-derive/0.8.26/download\"],\n strip_prefix = \"zerocopy-derive-0.8.26\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-derive-0.8.26.bazel\"),\n )\n\n return [\n struct(repo=\"crates__tokenizers-0.21.4\", is_dev_dep = False),\n ]\n" } } }, @@ -1289,38 +1350,6 @@ "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"base64\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=base64\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.13.1\",\n)\n" } }, - "crates__bit-set-0.8.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/bit-set/0.8.0/download" - ], - "strip_prefix": "bit-set-0.8.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"bit_set\",\n deps = [\n \"@crates__bit-vec-0.8.0//:bit_vec\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=bit-set\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.0\",\n)\n" - } - }, - "crates__bit-vec-0.8.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/bit-vec/0.8.0/download" - ], - "strip_prefix": "bit-vec-0.8.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"bit_vec\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=bit-vec\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.0\",\n)\n" - } - }, "crates__bitflags-2.9.1": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { @@ -1337,22 +1366,6 @@ "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"bitflags\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=bitflags\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.9.1\",\n)\n" } }, - "crates__bumpalo-3.16.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/bumpalo/3.16.0/download" - ], - "strip_prefix": "bumpalo-3.16.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"bumpalo\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=bumpalo\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"3.16.0\",\n)\n" - } - }, "crates__byteorder-1.5.0": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { @@ -1385,6 +1398,22 @@ "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"castaway\",\n proc_macro_deps = [\n \"@crates__rustversion-1.0.21//:rustversion\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=castaway\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.4\",\n)\n" } }, + "crates__cc-1.2.33": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/cc/1.2.33/download" + ], + "strip_prefix": "cc-1.2.33", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"cc\",\n deps = [\n \"@crates__shlex-1.3.0//:shlex\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=cc\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.2.33\",\n)\n" + } + }, "crates__cfg-if-1.0.0": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { @@ -1606,23 +1635,7 @@ "https://static.crates.io/crates/esaxx-rs/0.1.10/download" ], "strip_prefix": "esaxx-rs-0.1.10", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"esaxx_rs\",\n deps = [\n \"@crates__esaxx-rs-0.1.10//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=esaxx-rs\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.10\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2018\",\n pkg_name = \"esaxx-rs\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=esaxx-rs\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.1.10\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__fancy-regex-0.14.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/fancy-regex/0.14.0/download" - ], - "strip_prefix": "fancy-regex-0.14.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"fancy_regex\",\n deps = [\n \"@crates__bit-set-0.8.0//:bit_set\",\n \"@crates__regex-automata-0.4.9//:regex_automata\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"perf\",\n \"std\",\n \"unicode\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=fancy-regex\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.14.0\",\n)\n" + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"esaxx_rs\",\n deps = [\n \"@crates__esaxx-rs-0.1.10//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"cc\",\n \"cpp\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=esaxx-rs\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.10\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"cc\",\n \"cpp\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n deps = [\n \"@crates__cc-1.2.33//:cc\",\n ],\n edition = \"2018\",\n pkg_name = \"esaxx-rs\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=esaxx-rs\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.1.10\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" } }, "crates__fnv-1.0.7": { @@ -1654,7 +1667,7 @@ "https://static.crates.io/crates/getrandom/0.3.3/download" ], "strip_prefix": "getrandom-0.3.3", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"getrandom\",\n deps = [\n \"@crates__cfg-if-1.0.0//:cfg_if\",\n \"@crates__getrandom-0.3.3//:build_script_build\",\n ] + select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(any(target_os = \"macos\", target_os = \"openbsd\", target_os = \"vita\", target_os = \"emscripten\"))\n ],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(all(any(target_os = \"linux\", target_os = \"android\"), not(any(all(target_os = \"linux\", target_env = \"\"), getrandom_backend = \"custom\", getrandom_backend = \"linux_raw\", getrandom_backend = \"rdrand\", getrandom_backend = \"rndr\"))))\n ],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(any(target_os = \"macos\", target_os = \"openbsd\", target_os = \"vita\", target_os = \"emscripten\"))\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(all(any(target_os = \"linux\", target_os = \"android\"), not(any(all(target_os = \"linux\", target_env = \"\"), getrandom_backend = \"custom\", getrandom_backend = \"linux_raw\", getrandom_backend = \"rdrand\", getrandom_backend = \"rndr\"))))\n ],\n \"//conditions:default\": [],\n }),\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n \"wasm_js\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=getrandom\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.3\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n \"wasm_js\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"getrandom\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=getrandom\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.3.3\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"getrandom\",\n deps = [\n \"@crates__cfg-if-1.0.0//:cfg_if\",\n \"@crates__getrandom-0.3.3//:build_script_build\",\n ] + select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(any(target_os = \"macos\", target_os = \"openbsd\", target_os = \"vita\", target_os = \"emscripten\"))\n ],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(all(any(target_os = \"linux\", target_os = \"android\"), not(any(all(target_os = \"linux\", target_env = \"\"), getrandom_backend = \"custom\", getrandom_backend = \"linux_raw\", getrandom_backend = \"rdrand\", getrandom_backend = \"rndr\"))))\n ],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(any(target_os = \"macos\", target_os = \"openbsd\", target_os = \"vita\", target_os = \"emscripten\"))\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(all(any(target_os = \"linux\", target_os = \"android\"), not(any(all(target_os = \"linux\", target_env = \"\"), getrandom_backend = \"custom\", getrandom_backend = \"linux_raw\", getrandom_backend = \"rdrand\", getrandom_backend = \"rndr\"))))\n ],\n \"//conditions:default\": [],\n }),\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=getrandom\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.3\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"getrandom\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=getrandom\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.3.3\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" } }, "crates__ident_case-1.0.1": { @@ -1705,22 +1718,6 @@ "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"itoa\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=itoa\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.14\",\n)\n" } }, - "crates__js-sys-0.3.77": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/js-sys/0.3.77/download" - ], - "strip_prefix": "js-sys-0.3.77", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"js_sys\",\n deps = [\n \"@crates__once_cell-1.20.2//:once_cell\",\n \"@crates__wasm-bindgen-0.2.100//:wasm_bindgen\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=js-sys\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.77\",\n)\n" - } - }, "crates__libc-0.2.169": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { @@ -1878,7 +1875,39 @@ "https://static.crates.io/crates/once_cell/1.20.2/download" ], "strip_prefix": "once_cell-1.20.2", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"once_cell\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"race\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=once_cell\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.20.2\",\n)\n" + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"once_cell\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"default\",\n \"race\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=once_cell\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.20.2\",\n)\n" + } + }, + "crates__onig-6.5.1": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/onig/6.5.1/download" + ], + "strip_prefix": "onig-6.5.1", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"onig\",\n deps = [\n \"@crates__bitflags-2.9.1//:bitflags\",\n \"@crates__once_cell-1.20.2//:once_cell\",\n \"@crates__onig_sys-69.9.1//:onig_sys\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=onig\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"6.5.1\",\n)\n" + } + }, + "crates__onig_sys-69.9.1": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/onig_sys/69.9.1/download" + ], + "strip_prefix": "onig_sys-69.9.1", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"onig_sys\",\n deps = [\n \"@crates__onig_sys-69.9.1//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=onig_sys\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"69.9.1\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n deps = [\n \"@crates__cc-1.2.33//:cc\",\n \"@crates__pkg-config-0.3.32//:pkg_config\",\n ],\n edition = \"2021\",\n links = \"onig\",\n pkg_name = \"onig_sys\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=onig_sys\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"69.9.1\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" } }, "crates__paste-1.0.15": { @@ -1897,6 +1926,22 @@ "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"paste\",\n deps = [\n \"@crates__paste-1.0.15//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=paste\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.15\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2018\",\n pkg_name = \"paste\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=paste\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.15\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" } }, + "crates__pkg-config-0.3.32": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/pkg-config/0.3.32/download" + ], + "strip_prefix": "pkg-config-0.3.32", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"pkg_config\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=pkg-config\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.32\",\n)\n" + } + }, "crates__ppv-lite86-0.2.20": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { @@ -2086,7 +2131,7 @@ "https://static.crates.io/crates/regex-automata/0.4.9/download" ], "strip_prefix": "regex-automata-0.4.9", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"regex_automata\",\n deps = [\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__memchr-2.7.4//:memchr\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"dfa\",\n \"dfa-build\",\n \"dfa-onepass\",\n \"dfa-search\",\n \"hybrid\",\n \"meta\",\n \"nfa\",\n \"nfa-backtrack\",\n \"nfa-pikevm\",\n \"nfa-thompson\",\n \"perf\",\n \"perf-inline\",\n \"perf-literal\",\n \"perf-literal-multisubstring\",\n \"perf-literal-substring\",\n \"std\",\n \"syntax\",\n \"unicode\",\n \"unicode-age\",\n \"unicode-bool\",\n \"unicode-case\",\n \"unicode-gencat\",\n \"unicode-perl\",\n \"unicode-script\",\n \"unicode-segment\",\n \"unicode-word-boundary\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=regex-automata\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.4.9\",\n)\n" + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"regex_automata\",\n deps = [\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__memchr-2.7.4//:memchr\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"dfa-onepass\",\n \"hybrid\",\n \"meta\",\n \"nfa-backtrack\",\n \"nfa-pikevm\",\n \"nfa-thompson\",\n \"perf-inline\",\n \"perf-literal\",\n \"perf-literal-multisubstring\",\n \"perf-literal-substring\",\n \"std\",\n \"syntax\",\n \"unicode\",\n \"unicode-age\",\n \"unicode-bool\",\n \"unicode-case\",\n \"unicode-gencat\",\n \"unicode-perl\",\n \"unicode-script\",\n \"unicode-segment\",\n \"unicode-word-boundary\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=regex-automata\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.4.9\",\n)\n" } }, "crates__regex-syntax-0.8.5": { @@ -2185,6 +2230,22 @@ "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"serde_json\",\n deps = [\n \"@crates__itoa-1.0.14//:itoa\",\n \"@crates__memchr-2.7.4//:memchr\",\n \"@crates__ryu-1.0.18//:ryu\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__serde_json-1.0.134//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde_json\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.134\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"serde_json\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde_json\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.134\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" } }, + "crates__shlex-1.3.0": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/shlex/1.3.0/download" + ], + "strip_prefix": "shlex-1.3.0", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"shlex\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=shlex\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.3.0\",\n)\n" + } + }, "crates__smallvec-1.13.2": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { @@ -2262,7 +2323,7 @@ "https://static.crates.io/crates/syn/2.0.91/download" ], "strip_prefix": "syn-2.0.91", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"syn\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__unicode-ident-1.0.14//:unicode_ident\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"clone-impls\",\n \"default\",\n \"derive\",\n \"extra-traits\",\n \"full\",\n \"parsing\",\n \"printing\",\n \"proc-macro\",\n \"visit\",\n \"visit-mut\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=syn\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.0.91\",\n)\n" + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"syn\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__unicode-ident-1.0.14//:unicode_ident\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"clone-impls\",\n \"default\",\n \"derive\",\n \"extra-traits\",\n \"full\",\n \"parsing\",\n \"printing\",\n \"proc-macro\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=syn\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.0.91\",\n)\n" } }, "crates__thiserror-2.0.12": { @@ -2297,20 +2358,20 @@ "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"thiserror_impl\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=thiserror-impl\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.0.12\",\n)\n" } }, - "crates__tokenizers-0.21.2": { + "crates__tokenizers-0.21.4": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { "patch_args": [], "patch_tool": "", "patches": [], "remote_patch_strip": 1, - "sha256": "4c3846d8588abed0daba25a0e47edd58ea15e450a6088b2575f5116fdb0b27ca", + "sha256": "a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476", "type": "tar.gz", "urls": [ - "https://static.crates.io/crates/tokenizers/0.21.2/download" + "https://static.crates.io/crates/tokenizers/0.21.4/download" ], - "strip_prefix": "tokenizers-0.21.2", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"tokenizers\",\n deps = [\n \"@crates__ahash-0.8.12//:ahash\",\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__compact_str-0.9.0//:compact_str\",\n \"@crates__dary_heap-0.3.7//:dary_heap\",\n \"@crates__derive_builder-0.20.2//:derive_builder\",\n \"@crates__esaxx-rs-0.1.10//:esaxx_rs\",\n \"@crates__fancy-regex-0.14.0//:fancy_regex\",\n \"@crates__getrandom-0.3.3//:getrandom\",\n \"@crates__itertools-0.14.0//:itertools\",\n \"@crates__log-0.4.22//:log\",\n \"@crates__macro_rules_attribute-0.2.0//:macro_rules_attribute\",\n \"@crates__monostate-0.1.13//:monostate\",\n \"@crates__rand-0.9.1//:rand\",\n \"@crates__rayon-1.10.0//:rayon\",\n \"@crates__rayon-cond-0.4.0//:rayon_cond\",\n \"@crates__regex-1.11.1//:regex\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__serde_json-1.0.134//:serde_json\",\n \"@crates__spm_precompiled-0.1.4//:spm_precompiled\",\n \"@crates__thiserror-2.0.12//:thiserror\",\n \"@crates__unicode-normalization-alignments-0.1.12//:unicode_normalization_alignments\",\n \"@crates__unicode-segmentation-1.12.0//:unicode_segmentation\",\n \"@crates__unicode_categories-0.1.1//:unicode_categories\",\n ],\n proc_macro_deps = [\n \"@crates__paste-1.0.15//:paste\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"fancy-regex\",\n \"unstable_wasm\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=tokenizers\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.21.2\",\n)\n" + "strip_prefix": "tokenizers-0.21.4", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"tokenizers\",\n deps = [\n \"@crates__ahash-0.8.12//:ahash\",\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__compact_str-0.9.0//:compact_str\",\n \"@crates__dary_heap-0.3.7//:dary_heap\",\n \"@crates__derive_builder-0.20.2//:derive_builder\",\n \"@crates__esaxx-rs-0.1.10//:esaxx_rs\",\n \"@crates__getrandom-0.3.3//:getrandom\",\n \"@crates__itertools-0.14.0//:itertools\",\n \"@crates__log-0.4.22//:log\",\n \"@crates__macro_rules_attribute-0.2.0//:macro_rules_attribute\",\n \"@crates__monostate-0.1.13//:monostate\",\n \"@crates__onig-6.5.1//:onig\",\n \"@crates__rand-0.9.1//:rand\",\n \"@crates__rayon-1.10.0//:rayon\",\n \"@crates__rayon-cond-0.4.0//:rayon_cond\",\n \"@crates__regex-1.11.1//:regex\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__serde_json-1.0.134//:serde_json\",\n \"@crates__spm_precompiled-0.1.4//:spm_precompiled\",\n \"@crates__thiserror-2.0.12//:thiserror\",\n \"@crates__unicode-normalization-alignments-0.1.12//:unicode_normalization_alignments\",\n \"@crates__unicode-segmentation-1.12.0//:unicode_segmentation\",\n \"@crates__unicode_categories-0.1.1//:unicode_categories\",\n ],\n proc_macro_deps = [\n \"@crates__paste-1.0.15//:paste\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"esaxx_fast\",\n \"onig\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=tokenizers\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.21.4\",\n)\n" } }, "crates__unicode-ident-1.0.14": { @@ -2409,86 +2470,6 @@ "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wasi\",\n deps = [\n \"@crates__wit-bindgen-rt-0.39.0//:wit_bindgen_rt\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasi\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.14.2+wasi-0.2.4\",\n)\n" } }, - "crates__wasm-bindgen-0.2.100": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/wasm-bindgen/0.2.100/download" - ], - "strip_prefix": "wasm-bindgen-0.2.100", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wasm_bindgen\",\n deps = [\n \"@crates__cfg-if-1.0.0//:cfg_if\",\n \"@crates__once_cell-1.20.2//:once_cell\",\n \"@crates__wasm-bindgen-0.2.100//:build_script_build\",\n ],\n proc_macro_deps = [\n \"@crates__wasm-bindgen-macro-0.2.100//:wasm_bindgen_macro\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.100\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"wasm-bindgen\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.2.100\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__wasm-bindgen-backend-0.2.100": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/wasm-bindgen-backend/0.2.100/download" - ], - "strip_prefix": "wasm-bindgen-backend-0.2.100", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wasm_bindgen_backend\",\n deps = [\n \"@crates__bumpalo-3.16.0//:bumpalo\",\n \"@crates__log-0.4.22//:log\",\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n \"@crates__wasm-bindgen-shared-0.2.100//:wasm_bindgen_shared\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen-backend\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.100\",\n)\n" - } - }, - "crates__wasm-bindgen-macro-0.2.100": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/wasm-bindgen-macro/0.2.100/download" - ], - "strip_prefix": "wasm-bindgen-macro-0.2.100", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"wasm_bindgen_macro\",\n deps = [\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__wasm-bindgen-macro-support-0.2.100//:wasm_bindgen_macro_support\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen-macro\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.100\",\n)\n" - } - }, - "crates__wasm-bindgen-macro-support-0.2.100": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/wasm-bindgen-macro-support/0.2.100/download" - ], - "strip_prefix": "wasm-bindgen-macro-support-0.2.100", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wasm_bindgen_macro_support\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n \"@crates__wasm-bindgen-backend-0.2.100//:wasm_bindgen_backend\",\n \"@crates__wasm-bindgen-shared-0.2.100//:wasm_bindgen_shared\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen-macro-support\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.100\",\n)\n" - } - }, - "crates__wasm-bindgen-shared-0.2.100": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/wasm-bindgen-shared/0.2.100/download" - ], - "strip_prefix": "wasm-bindgen-shared-0.2.100", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wasm_bindgen_shared\",\n deps = [\n \"@crates__unicode-ident-1.0.14//:unicode_ident\",\n \"@crates__wasm-bindgen-shared-0.2.100//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen-shared\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.100\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n links = \"wasm_bindgen\",\n pkg_name = \"wasm-bindgen-shared\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasm-bindgen-shared\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.2.100\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, "crates__wit-bindgen-rt-0.39.0": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { @@ -2625,8 +2606,8 @@ }, "@@rules_rust+//crate_universe/private:internal_extensions.bzl%cu_nr": { "general": { - "bzlTransitiveDigest": "scFQEdObSJq+bJLzhX96j7iqyB2vDibsYijCpXjRAxo=", - "usagesDigest": "UWnpRlkSFx6zlujUh+aOuBGwtJYbR60HyNKAvijQ99g=", + "bzlTransitiveDigest": "A72J7FdqiIlmYkdw1rlcP/8uguFzwKWCs3Q3Mn4fVpQ=", + "usagesDigest": "3vKI8uvqTpJCf+t8aU6UD5d5cUWinWhtMjKkRpCLR+A=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -2768,8 +2749,8 @@ }, "@@rules_swift+//swift:extensions.bzl%non_module_deps": { "general": { - "bzlTransitiveDigest": "kLWKyo+sC0n/GT9xPbQUxIl+Z0N0H8RcxfqCOyVLuOQ=", - "usagesDigest": "9w18ec4dj90mX/JqpR2tBU2YVc1GU9yNxi/d7q6lLVA=", + "bzlTransitiveDigest": "zLT+pQh+AVbyxn970hzqSTfnjUqwnk0PUnP/REyVSMY=", + "usagesDigest": "mhACFnrdMv9Wi0Mt67bxocJqviRkDSV+Ee5Mqdj5akA=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -2916,6 +2897,11 @@ "rules_swift+", "bazel_tools", "bazel_tools" + ], + [ + "rules_swift+", + "build_bazel_rules_swift", + "rules_swift+" ] ] } diff --git a/README.md b/README.md index c77dcf1d..bdfbf7f5 100644 --- a/README.md +++ b/README.md @@ -122,10 +122,10 @@ Once you've been granted access, you're ready to download a gated model like First, you need to download the model using the [huggingface-cli](https://huggingface.co/docs/huggingface_hub/en/guides/cli). Note you don't need to install it yourself, -you can just use the packaged version `bazel run @zml//tools:hf --`. +you can just use the packaged version `bazel run //tools/hf --`. ``` -bazel run @zml//tools:hf -- download meta-llama/Llama-3.1-8B-Instruct --local-dir $HOME/Llama-3.1-8B-Instruct --exclude='*.pth' +bazel run //tools/hf -- download meta-llama/Llama-3.1-8B-Instruct --local-dir $HOME/Llama-3.1-8B-Instruct --exclude='*.pth' ``` Then, you can run the model. @@ -143,7 +143,7 @@ Like the 8B model above, this model also requires approval. See [here](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) for access requirements. ``` -bazel run @zml//tools:hf -- download meta-llama/Llama-3.2-1B-Instruct --local-dir $HOME/Llama-3.2-1B-Instruct --exclude='*.pth' +bazel run //tools/hf -- download meta-llama/Llama-3.2-1B-Instruct --local-dir $HOME/Llama-3.2-1B-Instruct --exclude='*.pth' bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct -- --prompt="What is the capital of France?" ``` diff --git a/third_party/com_google_sentencepiece/repo.bzl b/third_party/com_google_sentencepiece/repo.bzl index 89b97510..8b2a11aa 100644 --- a/third_party/com_google_sentencepiece/repo.bzl +++ b/third_party/com_google_sentencepiece/repo.bzl @@ -1,9 +1,10 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") def repo(): - git_repository( + http_archive( name = "com_google_sentencepiece", - remote = "https://github.com/google/sentencepiece.git", - commit = "273449044caa593c2fd7eb7550cb3ab2cff93f1a", + url = "https://github.com/google/sentencepiece/releases/download/v0.2.1/sentencepiece-0.2.1.tar.gz", + strip_prefix = "sentencepiece-0.2.1/sentencepiece", + sha256 = "8138cec27c2f2282f4a34d9a016e3374cd40e5c6e9cb335063db66a0a3b71fad", build_file = "//third_party/com_google_sentencepiece:sentencepiece.bazel", ) diff --git a/third_party/non_module_deps.bzl b/third_party/non_module_deps.bzl index af92c417..5c769468 100644 --- a/third_party/non_module_deps.bzl +++ b/third_party/non_module_deps.bzl @@ -2,12 +2,14 @@ load("//third_party/com_github_hejsil_clap:repo.bzl", com_github_hejsil_clap = " load("//third_party/com_google_sentencepiece:repo.bzl", com_google_sentencepiece = "repo") load("//third_party/mnist:repo.bzl", mnist = "repo") load("//third_party/org_swig_swig:repo.bzl", org_swig_swig = "repo") +load("//third_party/xla:repo.bzl", xla = "repo") def _non_module_deps_impl(mctx): com_google_sentencepiece() org_swig_swig() com_github_hejsil_clap() mnist() + xla() return mctx.extension_metadata( reproducible = True, diff --git a/third_party/xla/BUILD.bazel b/third_party/xla/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/third_party/xla/llvm.bzl b/third_party/xla/llvm.bzl new file mode 100644 index 00000000..b4a2fe4f --- /dev/null +++ b/third_party/xla/llvm.bzl @@ -0,0 +1,30 @@ +load("@llvm-raw//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") + +def _llvm_impl(mctx): + _targets = {} + for mod in mctx.modules: + for conf in mod.tags.configure: + for target in conf.targets: + _targets[target] = True + _llvm_configure( + name = "llvm-project", + targets = _targets.keys(), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +llvm = module_extension( + implementation = _llvm_impl, + tag_classes = { + "configure": tag_class( + attrs = { + "targets": attr.string_list( + default = [], + ), + }, + ), + }, +) diff --git a/third_party/xla/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/xla/patches/0001-bazel-migration-to-bazel-8.1.1.patch new file mode 100644 index 00000000..8924cf4b --- /dev/null +++ b/third_party/xla/patches/0001-bazel-migration-to-bazel-8.1.1.patch @@ -0,0 +1,41 @@ +From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 +From: Jean-Baptiste Dalido +Date: Mon, 6 Jan 2025 13:33:13 +0100 +Subject: [PATCH] bazel: migration to bazel 8.0.1 + +--- + .bazelversion | 2 +- + third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/.bazelversion b/.bazelversion +index f22d756da3..fa5fce04b3 100644 +--- a/.bazelversion ++++ b/.bazelversion +@@ -1 +1 @@ +-7.4.1 ++8.1.1 +\ No newline at end of file +diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl +index d62531152d..71d80a5a99 100644 +--- a/third_party/gpus/cuda_configure.bzl ++++ b/third_party/gpus/cuda_configure.bzl +@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. + load( + "@bazel_tools//tools/cpp:lib_cc_configure.bzl", + "escape_string", +- "get_env_var", + ) + load( + "@bazel_tools//tools/cpp:windows_cc_configure.bzl", +- "find_msvc_tool", + "find_vc_path", + "setup_vc_env_vars", + ) ++load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") ++load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") + load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") + load( + "//third_party/remote_config:common.bzl", +-- +2.39.3 (Apple Git-146) diff --git a/third_party/xla/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/xla/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch new file mode 100644 index 00000000..938ef401 --- /dev/null +++ b/third_party/xla/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch @@ -0,0 +1,135 @@ +From 2ae9bb9d24b569c2c6bfab3c54b428103614944d Mon Sep 17 00:00:00 2001 +From: Hugo Mano +Date: Tue, 27 May 2025 11:48:17 +0200 +Subject: [PATCH 1/8] Added FFI handler registration API to the FFI PjRt + +PR: https://github.com/openxla/xla/pull/13420 +--- + xla/pjrt/c/BUILD | 5 +++++ + xla/pjrt/c/pjrt_c_api_ffi_extension.h | 21 ++++++++++++++++++ + xla/pjrt/c/pjrt_c_api_ffi_internal.cc | 32 ++++++++++++++++++++++++++- + 3 files changed, 57 insertions(+), 1 deletion(-) + +diff --git a/xla/pjrt/c/BUILD b/xla/pjrt/c/BUILD +index 79f18fa0bc..0f33dd8a6e 100644 +--- a/xla/pjrt/c/BUILD ++++ b/xla/pjrt/c/BUILD +@@ -69,8 +69,13 @@ cc_library( + ":pjrt_c_api_wrapper_impl", + "//xla/ffi:execution_context", + "//xla/ffi:type_id_registry", ++ "//xla/ffi:ffi_api", ++ "//xla/ffi/api:c_api", ++ "//xla/ffi/api:ffi", ++ "//xla/service:custom_call_target_registry", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings:string_view", ++ "@com_google_absl//absl/strings:str_format", + ], + ) + +diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h +index 995a2c7e50..b8f10bc2f7 100644 +--- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h ++++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h +@@ -69,10 +69,31 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); + // Adds a user data to the execute context. + typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); + ++typedef enum PJRT_FFI_Handler_TraitsBits { ++ PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE = 1u << 0, ++} PJRT_FFI_Handler_TraitsBits; ++ ++struct PJRT_FFI_Register_Handler_Args { ++ size_t struct_size; ++ const char* target_name; ++ size_t target_name_size; ++ int api_version; // 0 for an untyped call, 1 -- for typed ++ void* handler; ++ const char* platform_name; ++ size_t platform_name_size; ++ PJRT_FFI_Handler_TraitsBits traits; ++}; ++PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, traits); ++ ++// Registers an FFI call handler for a specific platform. ++typedef PJRT_Error* PJRT_FFI_Register_Handler( ++ PJRT_FFI_Register_Handler_Args* args); ++ + typedef struct PJRT_FFI_Extension { + PJRT_Extension_Base base; + PJRT_FFI_TypeID_Register* type_id_register; + PJRT_FFI_UserData_Add* user_data_add; ++ PJRT_FFI_Register_Handler* register_handler; + } PJRT_FFI; + PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Extension, user_data_add); + +diff --git a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc +index 5fa88eab33..763270331b 100644 +--- a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc ++++ b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc +@@ -13,16 +13,20 @@ See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ + +-#include "xla/pjrt/c/pjrt_c_api_ffi_internal.h" ++#include + + #include "absl/status/status.h" ++#include "absl/strings/str_format.h" + #include "absl/strings/string_view.h" ++#include "xla/ffi/api/c_api.h" + #include "xla/ffi/execution_context.h" + #include "xla/ffi/type_id_registry.h" ++#include "xla/ffi/ffi_api.h" + #include "xla/pjrt/c/pjrt_c_api.h" + #include "xla/pjrt/c/pjrt_c_api_ffi_extension.h" + #include "xla/pjrt/c/pjrt_c_api_helpers.h" + #include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" ++#include "xla/service/custom_call_target_registry.h" + + namespace pjrt { + +@@ -68,6 +72,31 @@ static PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args) { + return nullptr; + } + ++static PJRT_Error* PJRT_FFI_Register_Handler( ++ PJRT_FFI_Register_Handler_Args* args) { ++ PJRT_RETURN_IF_ERROR(ActualStructSizeIsGreaterOrEqual( ++ "PJRT_FFI_Register_Handler_Args", ++ PJRT_FFI_Register_Handler_Args_STRUCT_SIZE, args->struct_size)); ++ std::string target_name(args->target_name, args->target_name_size); ++ std::string platform_name(args->platform_name, args->platform_name_size); ++ switch (args->api_version) { ++ case 0: ++ xla::CustomCallTargetRegistry::Global()->Register( ++ target_name, args->handler, platform_name); ++ return nullptr; ++ case 1: ++ xla::ffi::Ffi::RegisterStaticHandler( ++ xla::ffi::GetXlaFfiApi(), target_name, platform_name, ++ reinterpret_cast(args->handler)); ++ return nullptr; ++ default: ++ return new PJRT_Error{absl::UnimplementedError( ++ absl::StrFormat("API version %d not supported for PJRT GPU plugin. " ++ "Supported versions are 0 and 1.", ++ args->api_version))}; ++ } ++} ++ + PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { + return { + PJRT_Extension_Base{ +@@ -77,6 +106,7 @@ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { + }, + /*type_id_register=*/PJRT_FFI_TypeID_Register, + /*user_data_add=*/PJRT_FFI_UserData_Add, ++ /*register_handler=*/PJRT_FFI_Register_Handler, + }; + } + +-- +2.39.5 (Apple Git-154) + diff --git a/third_party/xla/patches/0003-Remove-unconventional-C-code-in-headers.patch b/third_party/xla/patches/0003-Remove-unconventional-C-code-in-headers.patch new file mode 100644 index 00000000..6df5b1b3 --- /dev/null +++ b/third_party/xla/patches/0003-Remove-unconventional-C-code-in-headers.patch @@ -0,0 +1,124 @@ +From 6078da86a46b6f0d983dccb9ae4f36fc90640247 Mon Sep 17 00:00:00 2001 +From: Hugo Mano +Date: Fri, 11 Jul 2025 14:05:16 +0200 +Subject: [PATCH] zml patch + +--- + third_party/stablehlo/workspace.bzl | 1 + + third_party/stablehlo/zml.patch | 93 +++++++++++++++++++++++++++++ + 2 files changed, 94 insertions(+) + create mode 100644 third_party/stablehlo/zml.patch + +diff --git a/third_party/stablehlo/workspace.bzl b/third_party/stablehlo/workspace.bzl +index d9d5063744..44980948d0 100644 +--- a/third_party/stablehlo/workspace.bzl ++++ b/third_party/stablehlo/workspace.bzl +@@ -15,5 +15,6 @@ def repo(): + urls = tf_mirror_urls("https://github.com/openxla/stablehlo/archive/{commit}.zip".format(commit = STABLEHLO_COMMIT)), + patch_file = [ + "//third_party/stablehlo:temporary.patch", # Autogenerated, don't remove. ++ "//third_party/stablehlo:zml.patch", # Autogenerated, don't remove. + ], + ) +diff --git a/third_party/stablehlo/zml.patch b/third_party/stablehlo/zml.patch +new file mode 100644 +index 0000000000..2a09384582 +--- /dev/null ++++ b/third_party/stablehlo/zml.patch +@@ -0,0 +1,93 @@ ++From e38ab68376dd8a17ebf4469d2c8350f521310182 Mon Sep 17 00:00:00 2001 ++From: Hugo Mano ++Date: Fri, 11 Jul 2025 12:08:35 +0200 ++Subject: [PATCH] zml patch ++ ++--- ++ stablehlo/dialect/Serialization.cpp | 5 ++--- ++ stablehlo/dialect/Serialization.h | 3 +-- ++ stablehlo/integrations/c/StablehloDialectApi.cpp | 3 +-- ++ stablehlo/integrations/c/StablehloDialectApi.h | 2 +- ++ stablehlo/tools/StablehloTranslateMain.cpp | 2 +- ++ 5 files changed, 6 insertions(+), 9 deletions(-) ++ ++diff --git a/stablehlo/dialect/Serialization.cpp b/stablehlo/dialect/Serialization.cpp ++index cb89d673..4370d588 100644 ++--- a/stablehlo/dialect/Serialization.cpp +++++ b/stablehlo/dialect/Serialization.cpp ++@@ -39,8 +39,7 @@ namespace stablehlo { ++ ++ LogicalResult serializePortableArtifact(ModuleOp module, ++ StringRef targetVersion, ++- raw_ostream& os, ++- bool allowOtherDialects) { +++ raw_ostream& os) { ++ MLIRContext* context = module.getContext(); ++ ++ // Convert StableHLO --> VHLO. ++@@ -49,7 +48,7 @@ LogicalResult serializePortableArtifact(ModuleOp module, ++ { ++ PassManager pm(context); ++ StablehloLegalizeToVhloPassOptions options; ++- options.allowOtherDialects = allowOtherDialects; +++ options.allowOtherDialects = false; ++ pm.addPass(stablehlo::createStablehloLegalizeToVhloPass(options)); ++ if (!succeeded(pm.run(module))) { ++ return failure(); ++diff --git a/stablehlo/dialect/Serialization.h b/stablehlo/dialect/Serialization.h ++index 811ca97b..abe95e63 100644 ++--- a/stablehlo/dialect/Serialization.h +++++ b/stablehlo/dialect/Serialization.h ++@@ -34,8 +34,7 @@ namespace stablehlo { ++ // unsupported dialects. ++ LogicalResult serializePortableArtifact(ModuleOp module, ++ StringRef targetVersion, ++- raw_ostream& os, ++- bool allowOtherDialects = false); +++ raw_ostream& os); ++ ++ // Read StableHLO portable artifact ++ // ++diff --git a/stablehlo/integrations/c/StablehloDialectApi.cpp b/stablehlo/integrations/c/StablehloDialectApi.cpp ++index 343f8d0b..8f52e4d5 100644 ++--- a/stablehlo/integrations/c/StablehloDialectApi.cpp +++++ b/stablehlo/integrations/c/StablehloDialectApi.cpp ++@@ -81,8 +81,7 @@ MlirLogicalResult stablehloSerializePortableArtifactFromModule( ++ MlirStringCallback callback, void *userData, bool allowOtherDialects) { ++ mlir::detail::CallbackOstream stream(callback, userData); ++ if (failed(mlir::stablehlo::serializePortableArtifact( ++- unwrap(moduleStr), unwrap(targetVersion), stream, ++- allowOtherDialects))) +++ unwrap(moduleStr), unwrap(targetVersion), stream))) ++ return mlirLogicalResultFailure(); ++ return mlirLogicalResultSuccess(); ++ } ++diff --git a/stablehlo/integrations/c/StablehloDialectApi.h b/stablehlo/integrations/c/StablehloDialectApi.h ++index 385156bf..24d11c1d 100644 ++--- a/stablehlo/integrations/c/StablehloDialectApi.h +++++ b/stablehlo/integrations/c/StablehloDialectApi.h ++@@ -93,7 +93,7 @@ stablehloSerializePortableArtifactFromModule(MlirModule moduleStr, ++ MlirStringRef targetVersion, ++ MlirStringCallback callback, ++ void* userData, ++- bool allowOtherDialects = false); +++ bool allowOtherDialects); ++ ++ // Read a StableHLO program from a portable artifact, returning the module as ++ // MLIR bytecode. Note, this bytecode returned is not a portable artifact, ++diff --git a/stablehlo/tools/StablehloTranslateMain.cpp b/stablehlo/tools/StablehloTranslateMain.cpp ++index fdf0d6a9..8d5c8752 100644 ++--- a/stablehlo/tools/StablehloTranslateMain.cpp +++++ b/stablehlo/tools/StablehloTranslateMain.cpp ++@@ -323,7 +323,7 @@ TranslateFromMLIRRegistration serializeRegistration( ++ } ++ ++ return stablehlo::serializePortableArtifact( ++- module, targetVersion, os, allowOtherDialectsOption.getValue()); +++ module, targetVersion, os); ++ }, ++ [](DialectRegistry ®istry) { ++ mlir::registerAllDialects(registry); ++-- ++2.39.5 (Apple Git-154) ++ +-- +2.39.5 (Apple Git-154) + diff --git a/third_party/xla/repo.bzl b/third_party/xla/repo.bzl new file mode 100644 index 00000000..8c005dc8 --- /dev/null +++ b/third_party/xla/repo.bzl @@ -0,0 +1,14 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + git_repository( + name = "xla", + remote = "https://github.com/openxla/xla.git", + commit = "ef07e787ea1303fa2f8d8a175d24d434bfb84107", + patch_args = ["-p1"], + patches = [ + "//third_party/xla:patches/0001-bazel-migration-to-bazel-8.1.1.patch", + "//third_party/xla:patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch", + "//third_party/xla:patches/0003-Remove-unconventional-C-code-in-headers.patch", + ], + ) diff --git a/third_party/xla/xla.bzl b/third_party/xla/xla.bzl new file mode 100644 index 00000000..b0aff388 --- /dev/null +++ b/third_party/xla/xla.bzl @@ -0,0 +1,87 @@ +load("@xla//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls", "tf_vendored") +load("@xla//third_party/llvm:workspace.bzl", llvm = "repo") +load("@xla//third_party/stablehlo:workspace.bzl", stablehlo = "repo") +load("@xla//third_party/triton:workspace.bzl", triton = "repo") + +_BZL_HELPERS = """\ +always_newer_than = lambda wanted_ver, if_true, if_false = []: if_true +always_if_true = lambda if_true, if_false = []: if_true +always_if_false = lambda if_true, if_false = []: if_false +always_false = lambda *args, **kwargs: False +""" + +def _simple_files_impl(rctx): + rctx.file("BUILD.bazel", "") + for f, content in rctx.attr.files.items(): + rctx.file(f, content) + +simple_files = repository_rule( + implementation = _simple_files_impl, + attrs = { + "files": attr.string_dict(), + }, +) + +def _dummy_repos(mctx): + simple_files(name = "local_config_cuda", files = { + "cuda/BUILD.bazel": "", + "cuda/build_defs.bzl": _BZL_HELPERS + """\ +cuda_library = always_false +if_cuda = always_if_false +if_cuda_is_configured = always_if_false +if_cuda_newer_than = always_newer_than +is_cuda_configured = always_false +""", + }) + simple_files(name = "local_config_rocm", files = { + "rocm/BUILD.bazel": "", + "rocm/build_defs.bzl": _BZL_HELPERS + """\ +if_rocm = always_if_false +if_rocm_is_configured = always_if_false +if_rocm_newer_than = always_newer_than +is_rocm_configured = always_false +if_gpu_is_configured = always_if_false +if_cuda_or_rocm = always_if_false +""", + }) + simple_files(name = "local_config_remote_execution", files = { + "remote_execution.bzl": """gpu_test_tags = lambda: []""", + }) + simple_files(name = "local_config_tensorrt", files = { + "build_defs.bzl": _BZL_HELPERS + """if_tensorrt = always_if_false""", + }) + simple_files(name = "python_version_repo", files = { + "py_version.bzl": """USE_PYWRAP_RULES = False""", + }) + simple_files(name = "rules_ml_toolchain", files = { + "third_party/gpus/BUILD.bazel": "", + "third_party/gpus/nvidia_common_rules.bzl": """cuda_rpath_flags = lambda *args, **kwargs: []""", + }) + +def _xla_impl(mctx): + llvm("llvm-raw") + stablehlo() + triton() + + tf_http_archive( + name = "com_github_grpc_grpc", + sha256 = "afbc5d78d6ba6d509cc6e264de0d49dcd7304db435cbf2d630385bacf49e066c", + strip_prefix = "grpc-1.68.2", + patch_file = [ + "//third_party/grpc:grpc.patch", + ], + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/refs/tags/v1.68.2.tar.gz"), + ) + tf_vendored(name = "tsl", relpath = "third_party/tsl") + + _dummy_repos(mctx) + + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +xla = module_extension( + implementation = _xla_impl, +) diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index ef43fe7b..b27a9664 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -1,6 +1,6 @@ load("@rules_python//python:py_library.bzl", "py_library") load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") - +load("@rules_python//python/uv:lock.bzl", uv_lock = "lock") load("@rules_uv//uv:pip.bzl", "pip_compile") py_library( diff --git a/tools/hf/BUILD.bazel b/tools/hf/BUILD.bazel new file mode 100644 index 00000000..154f165a --- /dev/null +++ b/tools/hf/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") +load("@rules_python//python/uv:lock.bzl", uv_lock = "lock") + +py_console_script_binary( + name = "hf", + pkg = "@huggingface_hub//huggingface_hub", + tags = ["manual"], +) + +uv_lock( + name = "requirements", + srcs = ["pyproject.toml"], + out = "requirements.lock.txt", + args = ["--upgrade"], + tags = ["manual"], +) diff --git a/tools/hf/pyproject.toml b/tools/hf/pyproject.toml new file mode 100644 index 00000000..d65bcf2c --- /dev/null +++ b/tools/hf/pyproject.toml @@ -0,0 +1,4 @@ +[project] +name = "hf" +version = "0.1.0" +dependencies = ["huggingface_hub"] diff --git a/tools/hf/requirements.lock.txt b/tools/hf/requirements.lock.txt new file mode 100644 index 00000000..f4288672 --- /dev/null +++ b/tools/hf/requirements.lock.txt @@ -0,0 +1,188 @@ +# This file was autogenerated by uv via the following command: +# bazel run //tools/hf:requirements.update +certifi==2025.8.3 \ + --hash=sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407 \ + --hash=sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5 + # via requests +charset-normalizer==3.4.3 \ + --hash=sha256:00237675befef519d9af72169d8604a067d92755e84fe76492fef5441db05b91 \ + --hash=sha256:02425242e96bcf29a49711b0ca9f37e451da7c70562bc10e8ed992a5a7a25cc0 \ + --hash=sha256:027b776c26d38b7f15b26a5da1044f376455fb3766df8fc38563b4efbc515154 \ + --hash=sha256:07a0eae9e2787b586e129fdcbe1af6997f8d0e5abaa0bc98c0e20e124d67e601 \ + --hash=sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884 \ + --hash=sha256:0e78314bdc32fa80696f72fa16dc61168fda4d6a0c014e0380f9d02f0e5d8a07 \ + --hash=sha256:0f2be7e0cf7754b9a30eb01f4295cc3d4358a479843b31f328afd210e2c7598c \ + --hash=sha256:13faeacfe61784e2559e690fc53fa4c5ae97c6fcedb8eb6fb8d0a15b475d2c64 \ + --hash=sha256:14c2a87c65b351109f6abfc424cab3927b3bdece6f706e4d12faaf3d52ee5efe \ + --hash=sha256:1606f4a55c0fd363d754049cdf400175ee96c992b1f8018b993941f221221c5f \ + --hash=sha256:16a8770207946ac75703458e2c743631c79c59c5890c80011d536248f8eaa432 \ + --hash=sha256:18343b2d246dc6761a249ba1fb13f9ee9a2bcd95decc767319506056ea4ad4dc \ + --hash=sha256:18b97b8404387b96cdbd30ad660f6407799126d26a39ca65729162fd810a99aa \ + --hash=sha256:1bb60174149316da1c35fa5233681f7c0f9f514509b8e399ab70fea5f17e45c9 \ + --hash=sha256:1e8ac75d72fa3775e0b7cb7e4629cec13b7514d928d15ef8ea06bca03ef01cae \ + --hash=sha256:1ef99f0456d3d46a50945c98de1774da86f8e992ab5c77865ea8b8195341fc19 \ + --hash=sha256:2001a39612b241dae17b4687898843f254f8748b796a2e16f1051a17078d991d \ + --hash=sha256:23b6b24d74478dc833444cbd927c338349d6ae852ba53a0d02a2de1fce45b96e \ + --hash=sha256:252098c8c7a873e17dd696ed98bbe91dbacd571da4b87df3736768efa7a792e4 \ + --hash=sha256:257f26fed7d7ff59921b78244f3cd93ed2af1800ff048c33f624c87475819dd7 \ + --hash=sha256:2c322db9c8c89009a990ef07c3bcc9f011a3269bc06782f916cd3d9eed7c9312 \ + --hash=sha256:30a96e1e1f865f78b030d65241c1ee850cdf422d869e9028e2fc1d5e4db73b92 \ + --hash=sha256:30d006f98569de3459c2fc1f2acde170b7b2bd265dc1943e87e1a4efe1b67c31 \ + --hash=sha256:31a9a6f775f9bcd865d88ee350f0ffb0e25936a7f930ca98995c05abf1faf21c \ + --hash=sha256:320e8e66157cc4e247d9ddca8e21f427efc7a04bbd0ac8a9faf56583fa543f9f \ + --hash=sha256:34a7f768e3f985abdb42841e20e17b330ad3aaf4bb7e7aeeb73db2e70f077b99 \ + --hash=sha256:3653fad4fe3ed447a596ae8638b437f827234f01a8cd801842e43f3d0a6b281b \ + --hash=sha256:3cd35b7e8aedeb9e34c41385fda4f73ba609e561faedfae0a9e75e44ac558a15 \ + --hash=sha256:3cfb2aad70f2c6debfbcb717f23b7eb55febc0bb23dcffc0f076009da10c6392 \ + --hash=sha256:416175faf02e4b0810f1f38bcb54682878a4af94059a1cd63b8747244420801f \ + --hash=sha256:41d1fc408ff5fdfb910200ec0e74abc40387bccb3252f3f27c0676731df2b2c8 \ + --hash=sha256:42e5088973e56e31e4fa58eb6bd709e42fc03799c11c42929592889a2e54c491 \ + --hash=sha256:4ca4c094de7771a98d7fbd67d9e5dbf1eb73efa4f744a730437d8a3a5cf994f0 \ + --hash=sha256:511729f456829ef86ac41ca78c63a5cb55240ed23b4b737faca0eb1abb1c41bc \ + --hash=sha256:53cd68b185d98dde4ad8990e56a58dea83a4162161b1ea9272e5c9182ce415e0 \ + --hash=sha256:585f3b2a80fbd26b048a0be90c5aae8f06605d3c92615911c3a2b03a8a3b796f \ + --hash=sha256:5b413b0b1bfd94dbf4023ad6945889f374cd24e3f62de58d6bb102c4d9ae534a \ + --hash=sha256:5d8d01eac18c423815ed4f4a2ec3b439d654e55ee4ad610e153cf02faf67ea40 \ + --hash=sha256:6aab0f181c486f973bc7262a97f5aca3ee7e1437011ef0c2ec04b5a11d16c927 \ + --hash=sha256:6cf8fd4c04756b6b60146d98cd8a77d0cdae0e1ca20329da2ac85eed779b6849 \ + --hash=sha256:6fb70de56f1859a3f71261cbe41005f56a7842cc348d3aeb26237560bfa5e0ce \ + --hash=sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14 \ + --hash=sha256:70bfc5f2c318afece2f5838ea5e4c3febada0be750fcf4775641052bbba14d05 \ + --hash=sha256:73dc19b562516fc9bcf6e5d6e596df0b4eb98d87e4f79f3ae71840e6ed21361c \ + --hash=sha256:74d77e25adda8581ffc1c720f1c81ca082921329452eba58b16233ab1842141c \ + --hash=sha256:78deba4d8f9590fe4dae384aeff04082510a709957e968753ff3c48399f6f92a \ + --hash=sha256:86df271bf921c2ee3818f0522e9a5b8092ca2ad8b065ece5d7d9d0e9f4849bcc \ + --hash=sha256:88ab34806dea0671532d3f82d82b85e8fc23d7b2dd12fa837978dad9bb392a34 \ + --hash=sha256:8999f965f922ae054125286faf9f11bc6932184b93011d138925a1773830bbe9 \ + --hash=sha256:8dcfc373f888e4fb39a7bc57e93e3b845e7f462dacc008d9749568b1c4ece096 \ + --hash=sha256:939578d9d8fd4299220161fdd76e86c6a251987476f5243e8864a7844476ba14 \ + --hash=sha256:96b2b3d1a83ad55310de8c7b4a2d04d9277d5591f40761274856635acc5fcb30 \ + --hash=sha256:a2d08ac246bb48479170408d6c19f6385fa743e7157d716e144cad849b2dd94b \ + --hash=sha256:b256ee2e749283ef3ddcff51a675ff43798d92d746d1a6e4631bf8c707d22d0b \ + --hash=sha256:b5e3b2d152e74e100a9e9573837aba24aab611d39428ded46f4e4022ea7d1942 \ + --hash=sha256:b89bc04de1d83006373429975f8ef9e7932534b8cc9ca582e4db7d20d91816db \ + --hash=sha256:bd28b817ea8c70215401f657edef3a8aa83c29d447fb0b622c35403780ba11d5 \ + --hash=sha256:c60e092517a73c632ec38e290eba714e9627abe9d301c8c8a12ec32c314a2a4b \ + --hash=sha256:c6dbd0ccdda3a2ba7c2ecd9d77b37f3b5831687d8dc1b6ca5f56a4880cc7b7ce \ + --hash=sha256:c6e490913a46fa054e03699c70019ab869e990270597018cef1d8562132c2669 \ + --hash=sha256:c6f162aabe9a91a309510d74eeb6507fab5fff92337a15acbe77753d88d9dcf0 \ + --hash=sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018 \ + --hash=sha256:cc34f233c9e71701040d772aa7490318673aa7164a0efe3172b2981218c26d93 \ + --hash=sha256:cc9370a2da1ac13f0153780040f465839e6cccb4a1e44810124b4e22483c93fe \ + --hash=sha256:ccf600859c183d70eb47e05a44cd80a4ce77394d1ac0f79dbd2dd90a69a3a049 \ + --hash=sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a \ + --hash=sha256:cf1ebb7d78e1ad8ec2a8c4732c7be2e736f6e5123a4146c5b89c9d1f585f8cef \ + --hash=sha256:d0e909868420b7049dafd3a31d45125b31143eec59235311fc4c57ea26a4acd2 \ + --hash=sha256:d22dbedd33326a4a5190dd4fe9e9e693ef12160c77382d9e87919bce54f3d4ca \ + --hash=sha256:d716a916938e03231e86e43782ca7878fb602a125a91e7acb8b5112e2e96ac16 \ + --hash=sha256:d79c198e27580c8e958906f803e63cddb77653731be08851c7df0b1a14a8fc0f \ + --hash=sha256:d95bfb53c211b57198bb91c46dd5a2d8018b3af446583aab40074bf7988401cb \ + --hash=sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1 \ + --hash=sha256:ec557499516fc90fd374bf2e32349a2887a876fbf162c160e3c01b6849eaf557 \ + --hash=sha256:fb6fecfd65564f208cbf0fba07f107fb661bcd1a7c389edbced3f7a493f70e37 \ + --hash=sha256:fb731e5deb0c7ef82d698b0f4c5bb724633ee2a489401594c5c88b02e6cb15f7 \ + --hash=sha256:fb7f67a1bfa6e40b438170ebdc8158b78dc465a5a67b6dde178a46987b244a72 \ + --hash=sha256:fd10de089bcdcd1be95a2f73dbe6254798ec1bda9f450d5828c96f93e2536b9c \ + --hash=sha256:fdabf8315679312cfa71302f9bd509ded4f2f263fb5b765cf1433b39106c3cc9 + # via requests +filelock==3.19.1 \ + --hash=sha256:66eda1888b0171c998b35be2bcc0f6d75c388a7ce20c3f3f37aa8e96c2dddf58 \ + --hash=sha256:d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d + # via huggingface-hub +fsspec==2025.7.0 \ + --hash=sha256:786120687ffa54b8283d942929540d8bc5ccfa820deb555a2b5d0ed2b737bf58 \ + --hash=sha256:8b012e39f63c7d5f10474de957f3ab793b47b45ae7d39f2fb735f8bbe25c0e21 + # via huggingface-hub +hf-xet==1.1.7 \ + --hash=sha256:18b61bbae92d56ae731b92087c44efcac216071182c603fc535f8e29ec4b09b8 \ + --hash=sha256:20cec8db4561338824a3b5f8c19774055b04a8df7fff0cb1ff2cb1a0c1607b80 \ + --hash=sha256:2e356da7d284479ae0f1dea3cf5a2f74fdf925d6dca84ac4341930d892c7cb34 \ + --hash=sha256:60dae4b44d520819e54e216a2505685248ec0adbdb2dd4848b17aa85a0375cde \ + --hash=sha256:6efaaf1a5a9fc3a501d3e71e88a6bfebc69ee3a716d0e713a931c8b8d920038f \ + --hash=sha256:713f2bff61b252f8523739969f247aa354ad8e6d869b8281e174e2ea1bb8d604 \ + --hash=sha256:751571540f9c1fbad9afcf222a5fb96daf2384bf821317b8bfb0c59d86078513 \ + --hash=sha256:b109f4c11e01c057fc82004c9e51e6cdfe2cb230637644ade40c599739067b2e + # via huggingface-hub +huggingface-hub==0.34.4 \ + --hash=sha256:9b365d781739c93ff90c359844221beef048403f1bc1f1c123c191257c3c890a \ + --hash=sha256:a4228daa6fb001be3f4f4bdaf9a0db00e1739235702848df00885c9b5742c85c + # via hf (tools/hf/pyproject.toml) +idna==3.10 \ + --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \ + --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 + # via requests +packaging==25.0 \ + --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 \ + --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f + # via huggingface-hub +pyyaml==6.0.2 \ + --hash=sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff \ + --hash=sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48 \ + --hash=sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086 \ + --hash=sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e \ + --hash=sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133 \ + --hash=sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5 \ + --hash=sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484 \ + --hash=sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee \ + --hash=sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5 \ + --hash=sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68 \ + --hash=sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a \ + --hash=sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf \ + --hash=sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99 \ + --hash=sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8 \ + --hash=sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85 \ + --hash=sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19 \ + --hash=sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc \ + --hash=sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a \ + --hash=sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1 \ + --hash=sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317 \ + --hash=sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c \ + --hash=sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631 \ + --hash=sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d \ + --hash=sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652 \ + --hash=sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5 \ + --hash=sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e \ + --hash=sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b \ + --hash=sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8 \ + --hash=sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476 \ + --hash=sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706 \ + --hash=sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563 \ + --hash=sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237 \ + --hash=sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b \ + --hash=sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083 \ + --hash=sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180 \ + --hash=sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425 \ + --hash=sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e \ + --hash=sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f \ + --hash=sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725 \ + --hash=sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183 \ + --hash=sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab \ + --hash=sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774 \ + --hash=sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725 \ + --hash=sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e \ + --hash=sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5 \ + --hash=sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d \ + --hash=sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290 \ + --hash=sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44 \ + --hash=sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed \ + --hash=sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4 \ + --hash=sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba \ + --hash=sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12 \ + --hash=sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4 + # via huggingface-hub +requests==2.32.4 \ + --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c \ + --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422 + # via huggingface-hub +tqdm==4.67.1 \ + --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 \ + --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2 + # via huggingface-hub +typing-extensions==4.14.1 \ + --hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 \ + --hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76 + # via huggingface-hub +urllib3==2.5.0 \ + --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 \ + --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc + # via requests diff --git a/tools/hf_requirements.in b/tools/hf_requirements.in deleted file mode 100644 index 6b964ccc..00000000 --- a/tools/hf_requirements.in +++ /dev/null @@ -1 +0,0 @@ -huggingface_hub diff --git a/tools/hf_requirements.lock.txt b/tools/hf_requirements.lock.txt deleted file mode 100644 index 558b47be..00000000 --- a/tools/hf_requirements.lock.txt +++ /dev/null @@ -1,203 +0,0 @@ -# This file was autogenerated by uv via the following command: -# bazel run @@//tools:hf_update_requirements ---index-url https://pypi.org/simple - -certifi==2025.7.14 \ - --hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 \ - --hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995 - # via requests -charset-normalizer==3.4.2 \ - --hash=sha256:005fa3432484527f9732ebd315da8da8001593e2cf46a3d817669f062c3d9ed4 \ - --hash=sha256:046595208aae0120559a67693ecc65dd75d46f7bf687f159127046628178dc45 \ - --hash=sha256:0c29de6a1a95f24b9a1aa7aefd27d2487263f00dfd55a77719b530788f75cff7 \ - --hash=sha256:0c8c57f84ccfc871a48a47321cfa49ae1df56cd1d965a09abe84066f6853b9c0 \ - --hash=sha256:0f5d9ed7f254402c9e7d35d2f5972c9bbea9040e99cd2861bd77dc68263277c7 \ - --hash=sha256:18dd2e350387c87dabe711b86f83c9c78af772c748904d372ade190b5c7c9d4d \ - --hash=sha256:1b1bde144d98e446b056ef98e59c256e9294f6b74d7af6846bf5ffdafd687a7d \ - --hash=sha256:1c95a1e2902a8b722868587c0e1184ad5c55631de5afc0eb96bc4b0d738092c0 \ - --hash=sha256:1cad5f45b3146325bb38d6855642f6fd609c3f7cad4dbaf75549bf3b904d3184 \ - --hash=sha256:21b2899062867b0e1fde9b724f8aecb1af14f2778d69aacd1a5a1853a597a5db \ - --hash=sha256:24498ba8ed6c2e0b56d4acbf83f2d989720a93b41d712ebd4f4979660db4417b \ - --hash=sha256:25a23ea5c7edc53e0f29bae2c44fcb5a1aa10591aae107f2a2b2583a9c5cbc64 \ - --hash=sha256:289200a18fa698949d2b39c671c2cc7a24d44096784e76614899a7ccf2574b7b \ - --hash=sha256:28a1005facc94196e1fb3e82a3d442a9d9110b8434fc1ded7a24a2983c9888d8 \ - --hash=sha256:32fc0341d72e0f73f80acb0a2c94216bd704f4f0bce10aedea38f30502b271ff \ - --hash=sha256:36b31da18b8890a76ec181c3cf44326bf2c48e36d393ca1b72b3f484113ea344 \ - --hash=sha256:3c21d4fca343c805a52c0c78edc01e3477f6dd1ad7c47653241cf2a206d4fc58 \ - --hash=sha256:3fddb7e2c84ac87ac3a947cb4e66d143ca5863ef48e4a5ecb83bd48619e4634e \ - --hash=sha256:43e0933a0eff183ee85833f341ec567c0980dae57c464d8a508e1b2ceb336471 \ - --hash=sha256:4a476b06fbcf359ad25d34a057b7219281286ae2477cc5ff5e3f70a246971148 \ - --hash=sha256:4e594135de17ab3866138f496755f302b72157d115086d100c3f19370839dd3a \ - --hash=sha256:50bf98d5e563b83cc29471fa114366e6806bc06bc7a25fd59641e41445327836 \ - --hash=sha256:5a9979887252a82fefd3d3ed2a8e3b937a7a809f65dcb1e068b090e165bbe99e \ - --hash=sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63 \ - --hash=sha256:5bf4545e3b962767e5c06fe1738f951f77d27967cb2caa64c28be7c4563e162c \ - --hash=sha256:6333b3aa5a12c26b2a4d4e7335a28f1475e0e5e17d69d55141ee3cab736f66d1 \ - --hash=sha256:65c981bdbd3f57670af8b59777cbfae75364b483fa8a9f420f08094531d54a01 \ - --hash=sha256:68a328e5f55ec37c57f19ebb1fdc56a248db2e3e9ad769919a58672958e8f366 \ - --hash=sha256:6a0289e4589e8bdfef02a80478f1dfcb14f0ab696b5a00e1f4b8a14a307a3c58 \ - --hash=sha256:6b66f92b17849b85cad91259efc341dce9c1af48e2173bf38a85c6329f1033e5 \ - --hash=sha256:6c9379d65defcab82d07b2a9dfbfc2e95bc8fe0ebb1b176a3190230a3ef0e07c \ - --hash=sha256:6fc1f5b51fa4cecaa18f2bd7a003f3dd039dd615cd69a2afd6d3b19aed6775f2 \ - --hash=sha256:70f7172939fdf8790425ba31915bfbe8335030f05b9913d7ae00a87d4395620a \ - --hash=sha256:721c76e84fe669be19c5791da68232ca2e05ba5185575086e384352e2c309597 \ - --hash=sha256:7222ffd5e4de8e57e03ce2cef95a4c43c98fcb72ad86909abdfc2c17d227fc1b \ - --hash=sha256:75d10d37a47afee94919c4fab4c22b9bc2a8bf7d4f46f87363bcf0573f3ff4f5 \ - --hash=sha256:76af085e67e56c8816c3ccf256ebd136def2ed9654525348cfa744b6802b69eb \ - --hash=sha256:770cab594ecf99ae64c236bc9ee3439c3f46be49796e265ce0cc8bc17b10294f \ - --hash=sha256:7a6ab32f7210554a96cd9e33abe3ddd86732beeafc7a28e9955cdf22ffadbab0 \ - --hash=sha256:7c48ed483eb946e6c04ccbe02c6b4d1d48e51944b6db70f697e089c193404941 \ - --hash=sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0 \ - --hash=sha256:8075c35cd58273fee266c58c0c9b670947c19df5fb98e7b66710e04ad4e9ff86 \ - --hash=sha256:8272b73e1c5603666618805fe821edba66892e2870058c94c53147602eab29c7 \ - --hash=sha256:82d8fd25b7f4675d0c47cf95b594d4e7b158aca33b76aa63d07186e13c0e0ab7 \ - --hash=sha256:844da2b5728b5ce0e32d863af26f32b5ce61bc4273a9c720a9f3aa9df73b1455 \ - --hash=sha256:8755483f3c00d6c9a77f490c17e6ab0c8729e39e6390328e42521ef175380ae6 \ - --hash=sha256:915f3849a011c1f593ab99092f3cecfcb4d65d8feb4a64cf1bf2d22074dc0ec4 \ - --hash=sha256:926ca93accd5d36ccdabd803392ddc3e03e6d4cd1cf17deff3b989ab8e9dbcf0 \ - --hash=sha256:982bb1e8b4ffda883b3d0a521e23abcd6fd17418f6d2c4118d257a10199c0ce3 \ - --hash=sha256:98f862da73774290f251b9df8d11161b6cf25b599a66baf087c1ffe340e9bfd1 \ - --hash=sha256:9cbfacf36cb0ec2897ce0ebc5d08ca44213af24265bd56eca54bee7923c48fd6 \ - --hash=sha256:a370b3e078e418187da8c3674eddb9d983ec09445c99a3a263c2011993522981 \ - --hash=sha256:a955b438e62efdf7e0b7b52a64dc5c3396e2634baa62471768a64bc2adb73d5c \ - --hash=sha256:aa6af9e7d59f9c12b33ae4e9450619cf2488e2bbe9b44030905877f0b2324980 \ - --hash=sha256:aa88ca0b1932e93f2d961bf3addbb2db902198dca337d88c89e1559e066e7645 \ - --hash=sha256:aaeeb6a479c7667fbe1099af9617c83aaca22182d6cf8c53966491a0f1b7ffb7 \ - --hash=sha256:aaf27faa992bfee0264dc1f03f4c75e9fcdda66a519db6b957a3f826e285cf12 \ - --hash=sha256:b2680962a4848b3c4f155dc2ee64505a9c57186d0d56b43123b17ca3de18f0fa \ - --hash=sha256:b2d318c11350e10662026ad0eb71bb51c7812fc8590825304ae0bdd4ac283acd \ - --hash=sha256:b33de11b92e9f75a2b545d6e9b6f37e398d86c3e9e9653c4864eb7e89c5773ef \ - --hash=sha256:b3daeac64d5b371dea99714f08ffc2c208522ec6b06fbc7866a450dd446f5c0f \ - --hash=sha256:be1e352acbe3c78727a16a455126d9ff83ea2dfdcbc83148d2982305a04714c2 \ - --hash=sha256:bee093bf902e1d8fc0ac143c88902c3dfc8941f7ea1d6a8dd2bcb786d33db03d \ - --hash=sha256:c72fbbe68c6f32f251bdc08b8611c7b3060612236e960ef848e0a517ddbe76c5 \ - --hash=sha256:c9e36a97bee9b86ef9a1cf7bb96747eb7a15c2f22bdb5b516434b00f2a599f02 \ - --hash=sha256:cddf7bd982eaa998934a91f69d182aec997c6c468898efe6679af88283b498d3 \ - --hash=sha256:cf713fe9a71ef6fd5adf7a79670135081cd4431c2943864757f0fa3a65b1fafd \ - --hash=sha256:d11b54acf878eef558599658b0ffca78138c8c3655cf4f3a4a673c437e67732e \ - --hash=sha256:d41c4d287cfc69060fa91cae9683eacffad989f1a10811995fa309df656ec214 \ - --hash=sha256:d524ba3f1581b35c03cb42beebab4a13e6cdad7b36246bd22541fa585a56cccd \ - --hash=sha256:daac4765328a919a805fa5e2720f3e94767abd632ae410a9062dff5412bae65a \ - --hash=sha256:db4c7bf0e07fc3b7d89ac2a5880a6a8062056801b83ff56d8464b70f65482b6c \ - --hash=sha256:dc7039885fa1baf9be153a0626e337aa7ec8bf96b0128605fb0d77788ddc1681 \ - --hash=sha256:dccab8d5fa1ef9bfba0590ecf4d46df048d18ffe3eec01eeb73a42e0d9e7a8ba \ - --hash=sha256:dedb8adb91d11846ee08bec4c8236c8549ac721c245678282dcb06b221aab59f \ - --hash=sha256:e45ba65510e2647721e35323d6ef54c7974959f6081b58d4ef5d87c60c84919a \ - --hash=sha256:e53efc7c7cee4c1e70661e2e112ca46a575f90ed9ae3fef200f2a25e954f4b28 \ - --hash=sha256:e635b87f01ebc977342e2697d05b56632f5f879a4f15955dfe8cef2448b51691 \ - --hash=sha256:e70e990b2137b29dc5564715de1e12701815dacc1d056308e2b17e9095372a82 \ - --hash=sha256:e8082b26888e2f8b36a042a58307d5b917ef2b1cacab921ad3323ef91901c71a \ - --hash=sha256:e8323a9b031aa0393768b87f04b4164a40037fb2a3c11ac06a03ffecd3618027 \ - --hash=sha256:e92fca20c46e9f5e1bb485887d074918b13543b1c2a1185e69bb8d17ab6236a7 \ - --hash=sha256:eb30abc20df9ab0814b5a2524f23d75dcf83cde762c161917a2b4b7b55b1e518 \ - --hash=sha256:eba9904b0f38a143592d9fc0e19e2df0fa2e41c3c3745554761c5f6447eedabf \ - --hash=sha256:ef8de666d6179b009dce7bcb2ad4c4a779f113f12caf8dc77f0162c29d20490b \ - --hash=sha256:efd387a49825780ff861998cd959767800d54f8308936b21025326de4b5a42b9 \ - --hash=sha256:f0aa37f3c979cf2546b73e8222bbfa3dc07a641585340179d768068e3455e544 \ - --hash=sha256:f4074c5a429281bf056ddd4c5d3b740ebca4d43ffffe2ef4bf4d2d05114299da \ - --hash=sha256:f69a27e45c43520f5487f27627059b64aaf160415589230992cec34c5e18a509 \ - --hash=sha256:fb707f3e15060adf5b7ada797624a6c6e0138e2a26baa089df64c68ee98e040f \ - --hash=sha256:fcbe676a55d7445b22c10967bceaaf0ee69407fbe0ece4d032b6eb8d4565982a \ - --hash=sha256:fdb20a30fe1175ecabed17cbf7812f7b804b8a315a25f24678bcdf120a90077f - # via requests -filelock==3.18.0 \ - --hash=sha256:adbc88eabb99d2fec8c9c1b229b171f18afa655400173ddc653d5d01501fb9f2 \ - --hash=sha256:c401f4f8377c4464e6db25fff06205fd89bdd83b65eb0488ed1b160f780e21de - # via huggingface-hub -fsspec==2025.7.0 \ - --hash=sha256:786120687ffa54b8283d942929540d8bc5ccfa820deb555a2b5d0ed2b737bf58 \ - --hash=sha256:8b012e39f63c7d5f10474de957f3ab793b47b45ae7d39f2fb735f8bbe25c0e21 - # via huggingface-hub -hf-xet==1.1.5 \ - --hash=sha256:69ebbcfd9ec44fdc2af73441619eeb06b94ee34511bbcf57cd423820090f5694 \ - --hash=sha256:73e167d9807d166596b4b2f0b585c6d5bd84a26dea32843665a8b58f6edba245 \ - --hash=sha256:83088ecea236d5113de478acb2339f92c95b4fb0462acaa30621fac02f5a534a \ - --hash=sha256:9fa6e3ee5d61912c4a113e0708eaaef987047616465ac7aa30f7121a48fc1af8 \ - --hash=sha256:ab34c4c3104133c495785d5d8bba3b1efc99de52c02e759cf711a91fd39d3a14 \ - --hash=sha256:dbba1660e5d810bd0ea77c511a99e9242d920790d0e63c0e4673ed36c4022d18 \ - --hash=sha256:f52c2fa3635b8c37c7764d8796dfa72706cc4eded19d638331161e82b0792e23 \ - --hash=sha256:fc874b5c843e642f45fd85cda1ce599e123308ad2901ead23d3510a47ff506d1 - # via huggingface-hub -huggingface-hub==0.34.3 \ - --hash=sha256:5444550099e2d86e68b2898b09e85878fbd788fc2957b506c6a79ce060e39492 \ - --hash=sha256:d58130fd5aa7408480681475491c0abd7e835442082fbc3ef4d45b6c39f83853 - # via -r tools/hf_requirements.in -idna==3.10 \ - --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \ - --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 - # via requests -packaging==25.0 \ - --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 \ - --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f - # via huggingface-hub -pyyaml==6.0.2 \ - --hash=sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff \ - --hash=sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48 \ - --hash=sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086 \ - --hash=sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e \ - --hash=sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133 \ - --hash=sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5 \ - --hash=sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484 \ - --hash=sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee \ - --hash=sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5 \ - --hash=sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68 \ - --hash=sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a \ - --hash=sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf \ - --hash=sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99 \ - --hash=sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8 \ - --hash=sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85 \ - --hash=sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19 \ - --hash=sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc \ - --hash=sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a \ - --hash=sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1 \ - --hash=sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317 \ - --hash=sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c \ - --hash=sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631 \ - --hash=sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d \ - --hash=sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652 \ - --hash=sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5 \ - --hash=sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e \ - --hash=sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b \ - --hash=sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8 \ - --hash=sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476 \ - --hash=sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706 \ - --hash=sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563 \ - --hash=sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237 \ - --hash=sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b \ - --hash=sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083 \ - --hash=sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180 \ - --hash=sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425 \ - --hash=sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e \ - --hash=sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f \ - --hash=sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725 \ - --hash=sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183 \ - --hash=sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab \ - --hash=sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774 \ - --hash=sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725 \ - --hash=sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e \ - --hash=sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5 \ - --hash=sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d \ - --hash=sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290 \ - --hash=sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44 \ - --hash=sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed \ - --hash=sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4 \ - --hash=sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba \ - --hash=sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12 \ - --hash=sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4 - # via huggingface-hub -requests==2.32.4 \ - --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c \ - --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422 - # via huggingface-hub -tqdm==4.67.1 \ - --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 \ - --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2 - # via huggingface-hub -typing-extensions==4.14.1 \ - --hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 \ - --hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76 - # via huggingface-hub -urllib3==2.5.0 \ - --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 \ - --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc - # via requests diff --git a/zml/tokenizer/hftokenizers/Cargo.lock b/zml/tokenizer/hftokenizers/Cargo.lock index 1a3714bf..68d1a36c 100644 --- a/zml/tokenizer/hftokenizers/Cargo.lock +++ b/zml/tokenizer/hftokenizers/Cargo.lock @@ -31,33 +31,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" -[[package]] -name = "bit-set" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" -dependencies = [ - "bit-vec", -] - -[[package]] -name = "bit-vec" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" - [[package]] name = "bitflags" version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" -[[package]] -name = "bumpalo" -version = "3.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" - [[package]] name = "byteorder" version = "1.5.0" @@ -73,6 +52,15 @@ dependencies = [ "rustversion", ] +[[package]] +name = "cc" +version = "1.2.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f" +dependencies = [ + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -205,16 +193,8 @@ name = "esaxx-rs" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" - -[[package]] -name = "fancy-regex" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298" dependencies = [ - "bit-set", - "regex-automata", - "regex-syntax", + "cc", ] [[package]] @@ -230,11 +210,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi", "wasi", - "wasm-bindgen", ] [[package]] @@ -258,16 +236,6 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" -[[package]] -name = "js-sys" -version = "0.3.77" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" -dependencies = [ - "once_cell", - "wasm-bindgen", -] - [[package]] name = "libc" version = "0.2.169" @@ -345,12 +313,40 @@ version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +[[package]] +name = "onig" +version = "6.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0" +dependencies = [ + "bitflags", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "paste" version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + [[package]] name = "ppv-lite86" version = "0.2.20" @@ -517,6 +513,12 @@ dependencies = [ "serde", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "smallvec" version = "1.13.2" @@ -580,9 +582,9 @@ dependencies = [ [[package]] name = "tokenizers" -version = "0.21.2" +version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c3846d8588abed0daba25a0e47edd58ea15e450a6088b2575f5116fdb0b27ca" +checksum = "a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476" dependencies = [ "ahash", "aho-corasick", @@ -590,12 +592,12 @@ dependencies = [ "dary_heap", "derive_builder", "esaxx-rs", - "fancy-regex", "getrandom", "itertools", "log", "macro_rules_attribute", "monostate", + "onig", "paste", "rand", "rayon", @@ -653,63 +655,6 @@ dependencies = [ "wit-bindgen-rt", ] -[[package]] -name = "wasm-bindgen" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" -dependencies = [ - "cfg-if", - "once_cell", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" -dependencies = [ - "unicode-ident", -] - [[package]] name = "wit-bindgen-rt" version = "0.39.0" diff --git a/zml/tokenizer/hftokenizers/Cargo.toml b/zml/tokenizer/hftokenizers/Cargo.toml index 924187b1..8054cd49 100644 --- a/zml/tokenizer/hftokenizers/Cargo.toml +++ b/zml/tokenizer/hftokenizers/Cargo.toml @@ -4,9 +4,11 @@ version = "0.1.0" edition = "2021" [dependencies] -# onig-sys fails to build with zig cc, disable it via the unstable_wasm feature, which switches -# the regex library to using fancy. -tokenizers = { version = "0.21.2", default-features = false, features = ["unstable_wasm"] } +tokenizers = { version = "0.21.4", default-features = false, features = [ + # We don't need/want the progressbar feature + "onig", + "esaxx_fast", +] } [lib] name = "zml_tokenizer_hftokenizers" From cd8a65334f4eb817540a3565acaa266f9074d42e Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Wed, 20 Aug 2025 00:15:55 +0200 Subject: [PATCH 016/881] zml: remove nemo, sentencepiece loaders, remove zig-yaml (#310) Progress towards #298 --- MODULE.bazel | 3 +- zml/aio/json.zig | 107 -------------------------------------- zml/aio/nemo.zig | 58 --------------------- zml/aio/safetensors.zig | 21 +++----- zml/aio/sentencepiece.zig | 38 -------------- zml/aio/yaml.zig | 93 --------------------------------- zml/zml.zig | 56 +++++++++----------- 7 files changed, 34 insertions(+), 342 deletions(-) delete mode 100644 zml/aio/json.zig delete mode 100644 zml/aio/nemo.zig delete mode 100644 zml/aio/sentencepiece.zig delete mode 100644 zml/aio/yaml.zig diff --git a/MODULE.bazel b/MODULE.bazel index 263ede18..e433c542 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -22,7 +22,6 @@ bazel_dep(name = "rules_uv", version = "0.87.0") bazel_dep(name = "rules_zig", version = "20250714.0-b14a4f1") bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.2.4") bazel_dep(name = "with_cfg.bzl", version = "0.11.0") -bazel_dep(name = "zig-yaml", version = "20240903.0-83d5fdf") bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2", dev_dependency = True) @@ -155,12 +154,14 @@ apt.install( manifest = "//runtimes/cuda:packages.yaml", ) use_repo(apt, "apt_cuda") + apt.install( name = "apt_rocm", lock = "//runtimes/rocm:packages.lock.json", manifest = "//runtimes/rocm:packages.yaml", ) use_repo(apt, "apt_rocm") + apt.install( name = "apt_neuron", lock = "//runtimes/neuron:packages.lock.json", diff --git a/zml/aio/json.zig b/zml/aio/json.zig deleted file mode 100644 index 9c02a5db..00000000 --- a/zml/aio/json.zig +++ /dev/null @@ -1,107 +0,0 @@ -const asynk = @import("async"); -const std = @import("std"); -const zml = @import("../zml.zig"); - -const StringBuilder = std.ArrayListUnmanaged(u8); -const Allocator = std.mem.Allocator; - -pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore { - const file = try std.fs.cwd().openFile(path, .{}); - defer file.close(); - var res: zml.aio.BufferStore = .{ - .arena = std.heap.ArenaAllocator.init(allocator), - }; - errdefer res.arena.deinit(); - const arena = res.arena.allocator(); - - const json_data = try file.reader().readAllAlloc(arena, (try file.metadata()).size()); - const metadata = try std.json.parseFromSliceLeaky(std.json.Value, allocator, json_data, .{ .allocate = .alloc_if_needed }); - - var it = metadata.object.iterator(); - while (it.next()) |entry| { - var prefix_buf: [1024]u8 = undefined; - try parseMetadata(allocator, &res, StringBuilder.initBuffer(&prefix_buf), entry.value_ptr.*); - } - - return res; -} - -pub fn parseMetadata(allocator: Allocator, store: *zml.aio.BufferStore, prefix: StringBuilder, val: std.json.Value) !void { - const metadata = &store._metadata; - const key = prefix.items; - return switch (val) { - .null => try metadata.put(allocator, try allocator.dupe(u8, key), .null), - .bool => |v| try metadata.put(allocator, try allocator.dupe(u8, key), .{ .bool = v }), - .integer => |v| try metadata.put(allocator, try allocator.dupe(u8, key), .{ .int = v }), - .float => |v| try metadata.put(allocator, try allocator.dupe(u8, key), .{ .float = v }), - .number_string, .string => |v| try metadata.put(allocator, try allocator.dupe(u8, key), .{ .string = try allocator.dupe(u8, v) }), - .array => |v| { - if (v.items.len == 0) return; - return if (validSlice(v)) |item_type| { - const data: zml.aio.Metadata = switch (item_type) { - .bool => blk: { - const values = try allocator.alloc(bool, v.items.len); - for (v.items, 0..) |item, i| values[i] = item.bool; - break :blk .{ .array_bool = values }; - }, - .integer => blk: { - const values = try allocator.alloc(i64, v.items.len); - for (v.items, 0..) |item, i| values[i] = item.integer; - break :blk .{ .array_int = values }; - }, - .float => blk: { - const values = try allocator.alloc(f64, v.items.len); - for (v.items, 0..) |item, i| values[i] = item.float; - break :blk .{ .array_float = values }; - }, - inline .string, .number_string => |tag| blk: { - const values = try allocator.alloc([]const u8, v.items.len); - for (v.items, 0..) |item, i| { - values[i] = @field(item, @tagName(tag)); - } - break :blk .{ .array_string = values }; - }, - .null, .array, .object => unreachable, - }; - try metadata.put(allocator, try allocator.dupe(u8, key), data); - } else { - for (v.items, 0..) |item, i| { - var new_prefix = prefix; - if (prefix.items.len > 0) - new_prefix.appendAssumeCapacity('.'); - new_prefix.items.len += std.fmt.formatIntBuf(new_prefix.unusedCapacitySlice(), i, 10, .lower, .{}); - try parseMetadata(allocator, store, new_prefix, item); - } - }; - }, - .object => |v| { - var obj_iter = v.iterator(); - while (obj_iter.next()) |entry| { - var new_prefix = prefix; - if (prefix.items.len > 0) - new_prefix.appendAssumeCapacity('.'); - new_prefix.appendSliceAssumeCapacity(entry.key_ptr.*); - try parseMetadata(allocator, store, new_prefix, entry.value_ptr.*); - } - }, - }; -} - -/// We can only create a Zig slice out of json array, if all values -/// in the array have the same type. -fn validSlice(v: std.json.Array) ?std.meta.Tag(std.json.Value) { - if (v.items.len == 0) return null; - - const item_type: std.meta.Tag(std.json.Value) = v.items[0]; - switch (item_type) { - .null, .array, .object => return null, - else => {}, - } - - for (v.items[1..]) |item| { - if (item != item_type) - return null; - } - - return item_type; -} diff --git a/zml/aio/nemo.zig b/zml/aio/nemo.zig deleted file mode 100644 index c48e5116..00000000 --- a/zml/aio/nemo.zig +++ /dev/null @@ -1,58 +0,0 @@ -const std = @import("std"); -const log = std.log.scoped(.@"zml/aio"); - -const asynk = @import("async"); -const yaml = @import("zig-yaml"); - -const eval = @import("torch/eval.zig"); -const zml = @import("../zml.zig"); -const File = @import("torch/file.zig").File; -const StringBuilder = std.ArrayListUnmanaged(u8); - -pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore { - var res: zml.aio.BufferStore = .{ - .arena = std.heap.ArenaAllocator.init(allocator), - }; - errdefer res.arena.deinit(); - - // TODO(cryptodeal): this is incorrect, you should use a temporary arena for all intermediary allocations. - const arena = res.arena.allocator(); - - // TODO(cryptodeal): mapped_file will never be close in case of success. - // You need to store it inside the result. - var mapped_file = try zml.aio.MemoryMappedFile.init(try asynk.File.open(path, .{})); - errdefer mapped_file.deinit(); - - var file_name_buffer: [std.fs.max_path_bytes]u8 = undefined; - var link_name_buffer: [std.fs.max_path_bytes]u8 = undefined; - var tar_iter = std.tar.iterator( - mapped_file.file.reader(), - .{ - .file_name_buffer = &file_name_buffer, - .link_name_buffer = &link_name_buffer, - }, - ); - while (try tar_iter.next()) |file| { - if (std.mem.endsWith(u8, file.name, ".yaml")) { - const yaml_data = try file.reader().readAllAlloc(arena, file.size); - const parsed = try yaml.Yaml.load(arena, yaml_data); - - var prefix_buf: [1024]u8 = undefined; - try zml.aio.yaml.parseMetadata(arena, &res, StringBuilder.initBuffer(&prefix_buf), parsed.docs.items[0]); - } else if (std.mem.endsWith(u8, file.name, ".ckpt") or std.mem.endsWith(u8, file.name, ".pt")) { - const start = try mapped_file.file.getPos(); - var torch_file = try File.fromTarFile(arena, mapped_file, file); - const ops = try torch_file.parsePickle(arena); - const values = try eval.evaluate(arena, ops, true); - - try torch_file.parseModel(values, &res); - // Since we directly manipulate the file handle pointer, - // reset to the end of file so iterator does not error - // and avoid `skipBytes`. - try mapped_file.file.seekTo(start + file.size); - file.unread_bytes.* = 0; - } else if (std.mem.eql(u8, file.name, "./model_weights/")) @panic(".NeMo sharded weights are not yet supported") else continue; - } - - return res; -} diff --git a/zml/aio/safetensors.zig b/zml/aio/safetensors.zig index 581ab77f..8d674807 100644 --- a/zml/aio/safetensors.zig +++ b/zml/aio/safetensors.zig @@ -1,12 +1,13 @@ -const asynk = @import("async"); const std = @import("std"); -const zml = @import("../zml.zig"); -const json = @import("json.zig"); -const HostBuffer = zml.HostBuffer; +const Allocator = std.mem.Allocator; + +const asynk = @import("async"); + const MemoryMappedFile = @import("../aio.zig").MemoryMappedFile; +const zml = @import("../zml.zig"); +const HostBuffer = zml.HostBuffer; const StringBuilder = std.ArrayListUnmanaged(u8); -const Allocator = std.mem.Allocator; const log = std.log.scoped(.@"zml/io"); pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore { @@ -55,11 +56,6 @@ fn loadFromIndex(allocator: Allocator, store: *zml.aio.BufferStore, files: *std. const full_filename = try std.fs.path.join(allocator, &.{ std.fs.path.dirname(path).?, filename }); try loadFile(allocator, store, files, full_filename); } - - if (index.object.get("__metadata__")) |metadata| { - var prefix_buf: [1024]u8 = undefined; - try json.parseMetadata(allocator, store, StringBuilder.initBuffer(&prefix_buf), metadata); - } } fn loadFile(allocator: Allocator, store: *zml.aio.BufferStore, files: *std.ArrayList(MemoryMappedFile), path: []const u8) !void { @@ -89,11 +85,6 @@ fn loadFile(allocator: Allocator, store: *zml.aio.BufferStore, files: *std.Array var it = metadata.object.iterator(); while (it.next()) |entry| { const key = entry.key_ptr.*; - if (std.mem.eql(u8, key, "__metadata__")) { - var prefix_buf: [1024]u8 = undefined; - try json.parseMetadata(allocator, store, StringBuilder.initBuffer(&prefix_buf), entry.value_ptr.*); - continue; - } const val = entry.value_ptr.*; const shape_field = val.object.get("shape").?.array; if (shape_field.items.len > zml.Shape.MAX_RANK) { diff --git a/zml/aio/sentencepiece.zig b/zml/aio/sentencepiece.zig deleted file mode 100644 index 298d8fd8..00000000 --- a/zml/aio/sentencepiece.zig +++ /dev/null @@ -1,38 +0,0 @@ -const std = @import("std"); -const asynk = @import("async"); -const zml = @import("../zml.zig"); - -const sentencepiece_proto = @import("//sentencepiece:model_proto"); -const Normalizer = zml.tokenizer.Normalizer; - -fn parseTokenId(id: ?i32) u32 { - if (id) |idx| { - if (idx > 0) return @intCast(idx); - } - - return std.math.maxInt(u32); -} - -pub fn normalizerFromSpec(spec: sentencepiece_proto.NormalizerSpec) Normalizer { - std.log.info("NormalizerSpec: {}", .{spec}); - if (spec.normalization_rule_tsv) |rule_tsv| { - if (!rule_tsv.isEmpty()) { - std.debug.panic("SentencePiece model with normalization rules not supported: model.normalizer_spec.normalization_rule_tsv: {s}", .{spec.normalization_rule_tsv.?.getSlice()}); - } - } - if (!std.mem.eql(u8, spec.name.?.getSlice(), "identity")) std.debug.panic("Normalizer only supports NormalizerSpec with name \"identity\", got \"{s}\"", .{spec.name.?.getSlice()}); - if (!spec.escape_whitespaces.?) std.debug.panic("Normalizer only supports NormalizerSpec with \"escape_whitespaces\" flag set", .{}); - if (spec.remove_extra_whitespaces) |_| {} else std.debug.panic("Normalizer only supports NormalizerSpec with \"remove_extra_whitespaces\" flag set", .{}); - - return Normalizer.init( - .{ - .remove_extra_whitespaces = spec.remove_extra_whitespaces orelse false, - .add_dummy_prefix = spec.add_dummy_prefix orelse false, - .add_dummy_suffix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - .use_nfc = false, - }, - if (spec.escape_whitespaces orelse false) Normalizer.sentencepiece_space else null, - ); -} diff --git a/zml/aio/yaml.zig b/zml/aio/yaml.zig deleted file mode 100644 index 2fc46902..00000000 --- a/zml/aio/yaml.zig +++ /dev/null @@ -1,93 +0,0 @@ -const std = @import("std"); -const yaml = @import("zig-yaml"); -const zml = @import("../zml.zig"); - -const Allocator = std.mem.Allocator; - -const StringBuilder = std.ArrayListUnmanaged(u8); - -pub fn open(allocator: Allocator, path: []const u8) !zml.aio.BufferStore { - const file = try std.fs.cwd().openFile(path, .{}); - defer file.close(); - var res: zml.aio.BufferStore = .{ - .arena = std.heap.ArenaAllocator.init(allocator), - }; - errdefer res.arena.deinit(); - const arena = res.arena.allocator(); - - const yaml_data = try file.reader().readAllAlloc(arena, (try file.metadata()).size()); - const parsed = try yaml.Yaml.load(arena, yaml_data); - - var prefix_buf: [1024]u8 = undefined; - try parseMetadata(arena, &res, StringBuilder.initBuffer(&prefix_buf), parsed.docs.items[0]); - return res; -} - -pub fn parseMetadata(allocator: Allocator, store: *zml.aio.BufferStore, key: StringBuilder, val: yaml.Value) !void { - switch (val) { - .int => |v| try store._metadata.put(allocator, try allocator.dupe(u8, key.items), .{ .int = v }), - .float => |v| try store._metadata.put(allocator, try allocator.dupe(u8, key.items), .{ .float = v }), - .string => |v| try store._metadata.put(allocator, try allocator.dupe(u8, key.items), .{ .string = v }), - .list => |v| switch (validSlice(v)) { - true => { - if (v.len == 0) return; - switch (v[0]) { - .int => { - const values = try allocator.alloc(i64, v.len); - errdefer allocator.free(values); - for (v, 0..) |item, i| values[i] = item.int; - try store._metadata.put(allocator, try allocator.dupe(u8, key.items), .{ .array_int = values }); - }, - .float => { - const values = try allocator.alloc(f64, v.len); - errdefer allocator.free(values); - for (v, 0..) |item, i| values[i] = item.float; - try store._metadata.put(allocator, try allocator.dupe(u8, key.items), .{ .array_float = values }); - }, - .string => { - const values = try allocator.alloc([]const u8, v.len); - errdefer allocator.free(values); - for (v, 0..) |item, i| { - values[i] = try allocator.dupe(u8, item.string); - } - try store._metadata.put(allocator, try allocator.dupe(u8, key.items), .{ .array_string = values }); - }, - .list => unreachable, - else => {}, - } - }, - false => for (v, 0..) |item, i| { - var new_key = key; - if (key.items.len > 0) - new_key.appendAssumeCapacity('.'); - new_key.items.len += std.fmt.formatIntBuf(new_key.unusedCapacitySlice(), i, 10, .lower, .{}); - try parseMetadata(allocator, store, new_key, item); - }, - }, - .map => { - var map_iter = val.map.iterator(); - while (map_iter.next()) |entry| { - var new_prefix = key; - if (key.items.len > 0) - new_prefix.appendAssumeCapacity('.'); - new_prefix.appendSliceAssumeCapacity(entry.key_ptr.*); - try parseMetadata(allocator, store, new_prefix, entry.value_ptr.*); - } - }, - else => {}, - } -} - -fn validSlice(v: []yaml.Value) bool { - if (v.len == 0) return false; - const item_type = std.meta.activeTag(v[0]); - switch (item_type) { - .empty, .list, .map => return false, - else => {}, - } - - for (v[1..]) |item| - if (item_type != std.meta.activeTag(item)) return false; - - return true; -} diff --git a/zml/zml.zig b/zml/zml.zig index 45abfccd..b5cd8039 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -3,37 +3,20 @@ //! compiling it for various accelerators and targets, and executing it. //! +// Namespaces +const std = @import("std"); + +pub const tokenizer = @import("zml/tokenizer"); + +pub const aio = @import("aio.zig"); pub const Buffer = @import("buffer.zig").Buffer; pub const Bufferized = @import("tensor.zig").Bufferized; pub const CompilationOptions = @import("platform.zig").CompilationOptions; +pub const context = @import("context.zig"); pub const Context = @import("context.zig").Context; pub const Data = @import("dtype.zig").Data; pub const DataType = @import("dtype.zig").DataType; -pub const HostBuffer = @import("hostbuffer.zig").HostBuffer; -pub const Platform = @import("platform.zig").Platform; -pub const Shape = @import("shape.zig").Shape; -pub const ShapeOf = @import("tensor.zig").ShapeOf; -pub const Target = @import("platform.zig").Target; -pub const Tensor = @import("tensor.zig").Tensor; - -// Namespaces -pub const context = @import("context.zig"); pub const exe = @import("exe.zig"); -pub const floats = @import("floats.zig"); -pub const helpers = @import("helpers.zig"); -pub const nn = @import("nn.zig"); -pub const module = @import("module.zig"); -pub const meta = @import("meta.zig"); -pub const platform = @import("platform.zig"); -pub const mlir = @import("mlirx.zig"); -pub const pjrt = @import("pjrtx.zig"); -pub const testing = @import("testing.zig"); -pub const torch = @import("torch.zig"); - -// pub const tokenizer = @import("tokenizer.zig"); -pub const tokenizer = @import("zml/tokenizer"); - -pub const call = ops.call; pub const compile = exe.compile; pub const compileWithPrefix = exe.compileWithPrefix; pub const compileFn = exe.compileFn; @@ -41,19 +24,32 @@ pub const compileModel = exe.compileModel; pub const FnExe = exe.FnExe; pub const ModuleExe = exe.ModuleExe; pub const ModuleSignature = exe.ModuleSignature; - +pub const floats = @import("floats.zig"); +pub const helpers = @import("helpers.zig"); +pub const HostBuffer = @import("hostbuffer.zig").HostBuffer; +pub const meta = @import("meta.zig"); +pub const mlir = @import("mlirx.zig"); +pub const module = @import("module.zig"); +pub const nn = @import("nn.zig"); pub const ops = @import("ops.zig"); +pub const call = ops.call; +pub const pjrt = @import("pjrtx.zig"); +pub const platform = @import("platform.zig"); +pub const Platform = @import("platform.zig").Platform; +pub const Shape = @import("shape.zig").Shape; +pub const ShapeOf = @import("tensor.zig").ShapeOf; +pub const Target = @import("platform.zig").Target; +pub const Tensor = @import("tensor.zig").Tensor; +pub const testing = @import("testing.zig"); +pub const torch = @import("torch.zig"); + +// pub const tokenizer = @import("tokenizer.zig"); pub const tools = struct { pub const Tracer = @import("tools/tracer.zig").Tracer; }; -pub const aio = @import("aio.zig"); -pub const sentencepiece = @import("aio/sentencepiece.zig"); - pub const log = std.log.scoped(.zml); -const std = @import("std"); - test { // NOTE : testing entrypoint. // Don't forget to import your module if you want to declare tests declarations that will be run by //zml:test From 24575c22323f03f59ee9f996521dfaec5c3c922f Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 21 Aug 2025 13:12:15 +0200 Subject: [PATCH 017/881] workspace: switch from zig_cc_binary to zig_binary (#311) Following the rules_zig bump that adds the `linkmode` setting, we can use the C linker directly. We can now remove the `zig_cc_binary` hack. --- .bazelrc | 10 +-- MODULE.bazel | 2 +- async/BUILD.bazel | 9 +-- bazel/zig.bzl | 63 ---------------- bazel/zig_srcs.bzl | 11 +-- docs/howtos/dockerize_models.md | 8 +- docs/howtos/howto_torch2zml.md | 8 +- docs/tutorials/write_first_model.md | 4 +- examples/benchmark/BUILD.bazel | 4 +- examples/llama/BUILD.bazel | 8 +- examples/mnist/BUILD.bazel | 4 +- examples/modernbert/BUILD.bazel | 4 +- examples/simple_layer/BUILD.bazel | 4 +- mlir/BUILD.bazel | 14 +--- mlir/dialects/BUILD.bazel | 31 +------- mlir/dialects/stablehlo/BUILD.bazel | 19 +++++ mlir/dialects/{ => stablehlo}/stablehlo.zig | 0 mlir/{c.h => mlir.h} | 0 pjrt/BUILD.bazel | 6 -- runtimes/neuron/BUILD.bazel | 9 +-- runtimes/rocm/BUILD.bazel | 2 +- stdx/BUILD.bazel | 8 +- stdx/fs.zig | 23 ++++++ .../rules_zig/20250821.0-be53625/MODULE.bazel | 75 +++++++++++++++++++ .../rules_zig/20250821.0-be53625/source.json | 5 ++ third_party/modules/rules_zig/metadata.json | 21 +++--- zml/BUILD.bazel | 11 +-- zml/tokenizer/BUILD.bazel | 5 +- zml/tokenizer/sentencepiece/BUILD.bazel | 6 -- 29 files changed, 183 insertions(+), 191 deletions(-) delete mode 100644 bazel/zig.bzl create mode 100644 mlir/dialects/stablehlo/BUILD.bazel rename mlir/dialects/{ => stablehlo}/stablehlo.zig (100%) rename mlir/{c.h => mlir.h} (100%) create mode 100644 third_party/modules/rules_zig/20250821.0-be53625/MODULE.bazel create mode 100644 third_party/modules/rules_zig/20250821.0-be53625/source.json diff --git a/.bazelrc b/.bazelrc index ca73486e..003a9e82 100644 --- a/.bazelrc +++ b/.bazelrc @@ -75,6 +75,11 @@ common --incompatible_enable_proto_toolchain_resolution # Allow empty globs since some legacy rules common --incompatible_disallow_empty_glob=false +# Use the C linker to link Zig code +common --@rules_zig//zig/settings:linkmode=cc + +common --@toolchains_llvm_bootstrapped//config:experimental_stub_libgcc_s=True + common:macos --action_env=MACOSX_DEPLOYMENT_TARGET=13.0 common:macos --macos_sdk_version=13.0 @@ -90,11 +95,6 @@ common:silent --ui_event_filters=,+error,+fail common:silent --show_result=0 common:silent --logging=0 -common --@rules_zig//zig/settings:copt=-fPIC -common --@rules_zig//zig/settings:copt=-fcompiler-rt - -common --@toolchains_llvm_bootstrapped//config:experimental_stub_libgcc_s=True - # --config=debug will used optmized backend, but all the frontend zig code will be compiled in debug mode. common:debug --compilation_mode=opt common:debug --@rules_zig//zig/settings:mode=debug diff --git a/MODULE.bazel b/MODULE.bazel index e433c542..105eeca0 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -19,7 +19,7 @@ bazel_dep(name = "rules_proto", version = "7.1.0") bazel_dep(name = "rules_python", version = "1.5.3") bazel_dep(name = "rules_rust", version = "0.63.0") bazel_dep(name = "rules_uv", version = "0.87.0") -bazel_dep(name = "rules_zig", version = "20250714.0-b14a4f1") +bazel_dep(name = "rules_zig", version = "20250821.0-be53625") bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.2.4") bazel_dep(name = "with_cfg.bzl", version = "0.11.0") diff --git a/async/BUILD.bazel b/async/BUILD.bazel index 65e44d0e..3f3c7dca 100644 --- a/async/BUILD.bazel +++ b/async/BUILD.bazel @@ -1,6 +1,4 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") -load("@zml//bazel:zig_srcs.bzl", "zig_srcs") - zig_library( name = "async", @@ -23,11 +21,6 @@ zig_library( zig_test( name = "test", - deps = [":async"], testonly = False, -) - -zig_srcs( - name = "sources", - zig_bin = ":test", + deps = [":async"], ) diff --git a/bazel/zig.bzl b/bazel/zig.bzl deleted file mode 100644 index 26773a5e..00000000 --- a/bazel/zig.bzl +++ /dev/null @@ -1,63 +0,0 @@ -load("@rules_cc//cc:cc_binary.bzl", "cc_binary") -load("@rules_cc//cc:cc_test.bzl", "cc_test") -load("@rules_zig//zig:defs.bzl", "BINARY_KIND", "zig_binary") - -def zig_cc_binary( - name, - copts = [], - args = None, - env = None, - data = [], - deps = [], - tags = [], - visibility = None, - **kwargs): - zig_binary( - name = "{}_lib".format(name), - kind = BINARY_KIND.static_lib, - copts = copts + ["-lc"], - deps = deps, - visibility = visibility, - **kwargs - ) - cc_binary( - name = name, - args = args, - env = env, - data = data, - deps = [":{}_lib".format(name)], - tags = tags, - visibility = visibility, - ) - -def zig_cc_test( - name, - copts = [], - env = None, - data = [], - deps = [], - test_runner = None, - tags = [], - visibility = None, - **kwargs): - zig_binary( - name = "{}_test_lib".format(name), - kind = BINARY_KIND.test_lib, - test_runner = test_runner, - tags = tags, - copts = copts + ["-lc"], - deps = deps + [ - "@rules_zig//zig/lib:libc", - ], - visibility = visibility, - **kwargs - ) - cc_test( - name = name, - env = env, - data = data, - deps = [":{}_test_lib".format(name)], - tags = tags, - visibility = visibility, - linkstatic = True, - ) diff --git a/bazel/zig_srcs.bzl b/bazel/zig_srcs.bzl index 2aa79568..ee4adb36 100644 --- a/bazel/zig_srcs.bzl +++ b/bazel/zig_srcs.bzl @@ -1,7 +1,7 @@ load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") -load("@rules_zig//zig:defs.bzl", "zig_binary", "BINARY_KIND") +load("@rules_zig//zig:defs.bzl", "zig_static_library") -def zig_srcs(name, zig_bin="", zig_lib=""): +def zig_srcs(name, zig_bin = "", zig_lib = ""): """For a given zig_library, recursively extract all zig sources into a tarball. This also includes the files translated from C headers. @@ -10,21 +10,22 @@ def zig_srcs(name, zig_bin="", zig_lib=""): """ if zig_bin == "": zig_bin = "{}_bin".format(name) - zig_binary( + zig_static_library( name = zig_bin, - kind = BINARY_KIND.bc, - tags = ["manual", "@rules_zig//zig/lib:libc"], + tags = ["manual"], deps = [zig_lib], ) native.filegroup( name = "{}_files".format(name), srcs = [zig_bin], + tags = ["manual"], output_group = "srcs", ) mtree_spec( name = "{}_mtree".format(name), srcs = [":{}_files".format(name)], + tags = ["manual"], ) tar( name = name, diff --git a/docs/howtos/dockerize_models.md b/docs/howtos/dockerize_models.md index af296f8a..0d2d8451 100644 --- a/docs/howtos/dockerize_models.md +++ b/docs/howtos/dockerize_models.md @@ -41,9 +41,9 @@ use their rules to define our 5 additional targets: load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") -load("@zml//bazel:zig.bzl", "zig_cc_binary") +load("@rules_zig//zig:defs.bzl", "zig_binary") -zig_cc_binary( +zig_binary( name = "simple_layer", main = "main.zig", deps = [ @@ -248,10 +248,10 @@ load("@aspect_bazel_lib//lib:expand_template.bzl", "expand_template") load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") -load("@zml//bazel:zig.bzl", "zig_cc_binary") +load("@rules_zig//zig:defs.bzl", "zig_binary") # The executable -zig_cc_binary( +zig_binary( name = "mnist", args = [ "$(location @com_github_ggerganov_ggml_mnist//file)", diff --git a/docs/howtos/howto_torch2zml.md b/docs/howtos/howto_torch2zml.md index a5785f44..026e2013 100644 --- a/docs/howtos/howto_torch2zml.md +++ b/docs/howtos/howto_torch2zml.md @@ -150,9 +150,9 @@ pub fn asyncMain() !void { And add a `zig_cc_binary` target in `my_project/BUILD.bazel`: ```python -load("@zml//bazel:zig.bzl", "zig_cc_binary") +load("@rules_zig//zig:defs.bzl", "zig_binary") -zig_cc_binary( +zig_binary( name = "torch2zml", main = "torch2zml.zig", deps = [ @@ -246,7 +246,7 @@ We can test the MLP layer with the `zml.testing.testLayer` utility: ```zig pub fn asyncMain() !void { ... - + var ctx = try zml.Context.init(); defer ctx.deinit(); const platform = ctx.autoPlatform(.{}); @@ -293,5 +293,5 @@ During this phase, you have three kinds of errors that can appear: "C,W,H" of tensors, you can port this to actual tensor attributes using `x.withTags(.{.c, .w, .h})`, and use those tags (eg `.c`) to refer to axes instead of offsets. E.g. in Pytorch: `x.sum(0) # reduce over channel axis` - becomes `x.sum(.c)`. More on this topic in + becomes `x.sum(.c)`. More on this topic in ["Working with tensors"](../tutorials/working_with_tensors.md). diff --git a/docs/tutorials/write_first_model.md b/docs/tutorials/write_first_model.md index e48cd7e8..7cea735c 100644 --- a/docs/tutorials/write_first_model.md +++ b/docs/tutorials/write_first_model.md @@ -352,9 +352,9 @@ As mentioned already, ZML uses Bazel; so to build our model, we just need to create a simple `BUILD.bazel` file, next to the `main.zig` file, like this: ```python -load("@zml//bazel:zig.bzl", "zig_cc_binary") +load("@rules_zig//zig:defs.bzl", "zig_binary") -zig_cc_binary( +zig_binary( name = "simple_layer", main = "main.zig", deps = [ diff --git a/examples/benchmark/BUILD.bazel b/examples/benchmark/BUILD.bazel index c4a0d011..ff55ed6b 100644 --- a/examples/benchmark/BUILD.bazel +++ b/examples/benchmark/BUILD.bazel @@ -1,7 +1,7 @@ load("@bazel_skylib//rules:build_test.bzl", "build_test") -load("//bazel:zig.bzl", "zig_cc_binary") +load("@rules_zig//zig:defs.bzl", "zig_binary") -zig_cc_binary( +zig_binary( name = "benchmark", main = "main.zig", deps = [ diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel index f6c29c97..6f4cd70c 100644 --- a/examples/llama/BUILD.bazel +++ b/examples/llama/BUILD.bazel @@ -2,9 +2,9 @@ load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") -load("//bazel:zig.bzl", "zig_cc_binary") +load("@rules_zig//zig:defs.bzl", "zig_binary") -zig_cc_binary( +zig_binary( name = "llama", srcs = [ "llama.zig", @@ -18,9 +18,9 @@ zig_cc_binary( ], ) -# TODO(steeve): Re-enable when rules_zig linkmode is fixed. +# TODO(steeve): Re-enable with a small, tokenless huggingface model. # -# zig_cc_binary( +# zig_test( # name = "test-implementation", # srcs = ["llama.zig"], # args = [ diff --git a/examples/mnist/BUILD.bazel b/examples/mnist/BUILD.bazel index f8eeae34..dfc635ba 100644 --- a/examples/mnist/BUILD.bazel +++ b/examples/mnist/BUILD.bazel @@ -1,7 +1,7 @@ load("@bazel_skylib//rules:build_test.bzl", "build_test") -load("//bazel:zig.bzl", "zig_cc_binary") +load("@rules_zig//zig:defs.bzl", "zig_binary") -zig_cc_binary( +zig_binary( name = "mnist", args = [ "$(location @mnist//:mnist.pt)", diff --git a/examples/modernbert/BUILD.bazel b/examples/modernbert/BUILD.bazel index 5f8291c8..081127ef 100644 --- a/examples/modernbert/BUILD.bazel +++ b/examples/modernbert/BUILD.bazel @@ -1,7 +1,7 @@ load("@bazel_skylib//rules:build_test.bzl", "build_test") -load("//bazel:zig.bzl", "zig_cc_binary") +load("@rules_zig//zig:defs.bzl", "zig_binary") -zig_cc_binary( +zig_binary( name = "modernbert", srcs = ["modernbert.zig"], args = [ diff --git a/examples/simple_layer/BUILD.bazel b/examples/simple_layer/BUILD.bazel index 98988398..b10b6dad 100644 --- a/examples/simple_layer/BUILD.bazel +++ b/examples/simple_layer/BUILD.bazel @@ -2,9 +2,9 @@ load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") -load("//bazel:zig.bzl", "zig_cc_binary") +load("@rules_zig//zig:defs.bzl", "zig_binary") -zig_cc_binary( +zig_binary( name = "simple_layer", main = "main.zig", deps = [ diff --git a/mlir/BUILD.bazel b/mlir/BUILD.bazel index ae329430..87982a7e 100644 --- a/mlir/BUILD.bazel +++ b/mlir/BUILD.bazel @@ -1,11 +1,9 @@ load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library") -load("//bazel:zig.bzl", "zig_cc_test") -load("//bazel:zig_srcs.bzl", "zig_srcs") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") cc_library( name = "c", - hdrs = ["c.h"], + hdrs = ["mlir.h"], visibility = ["//mlir:__subpackages__"], deps = [ "@llvm-project//mlir:CAPIArith", @@ -18,7 +16,6 @@ cc_library( zig_library( name = "mlir", - copts = ["-lc"], main = "mlir.zig", visibility = ["//visibility:public"], deps = [ @@ -27,12 +24,7 @@ zig_library( ], ) -zig_cc_test( +zig_test( name = "test", deps = [":mlir"], ) - -zig_srcs( - name = "sources", - zig_bin = ":test_test_lib", -) diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/BUILD.bazel index c18f12d8..845775e3 100644 --- a/mlir/dialects/BUILD.bazel +++ b/mlir/dialects/BUILD.bazel @@ -1,6 +1,4 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") -load("//bazel:zig.bzl", "zig_cc_test") -load("//bazel:zig_srcs.bzl", "zig_srcs") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "dialects", @@ -15,35 +13,14 @@ zig_library( main = "dialects.zig", visibility = ["//visibility:public"], deps = [ - ":stablehlo", "//mlir", + "//mlir/dialects/stablehlo", ], ) -zig_cc_test( +zig_test( name = "test", - deps = [":dialects"], -) - -zig_srcs( - name = "sources", - zig_bin = ":test_test_lib", -) - -zig_library( - name = "stablehlo", - import_name = "mlir/dialects/stablehlo", - main = "stablehlo.zig", - visibility = ["//mlir/dialects:__subpackages__"], deps = [ - "//mlir", - "//mlir:c", - "//stdx", - "@stablehlo//:stablehlo_dialect_capi", + ":dialects", ], ) - -zig_cc_test( - name = "stablehlo_test", - deps = [":stablehlo"], -) diff --git a/mlir/dialects/stablehlo/BUILD.bazel b/mlir/dialects/stablehlo/BUILD.bazel new file mode 100644 index 00000000..45e69239 --- /dev/null +++ b/mlir/dialects/stablehlo/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "stablehlo", + import_name = "mlir/dialects/stablehlo", + main = "stablehlo.zig", + visibility = ["//mlir/dialects:__subpackages__"], + deps = [ + "//mlir", + "//mlir:c", + "//stdx", + "@stablehlo//:stablehlo_dialect_capi", + ], +) + +zig_test( + name = "test", + deps = [":stablehlo"], +) diff --git a/mlir/dialects/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig similarity index 100% rename from mlir/dialects/stablehlo.zig rename to mlir/dialects/stablehlo/stablehlo.zig diff --git a/mlir/c.h b/mlir/mlir.h similarity index 100% rename from mlir/c.h rename to mlir/mlir.h diff --git a/pjrt/BUILD.bazel b/pjrt/BUILD.bazel index 470a4157..173bea7e 100644 --- a/pjrt/BUILD.bazel +++ b/pjrt/BUILD.bazel @@ -1,5 +1,4 @@ load("@rules_zig//zig:defs.bzl", "zig_library") -load("@zml//bazel:zig_srcs.bzl", "zig_srcs") zig_library( name = "pjrt", @@ -15,8 +14,3 @@ zig_library( "@xla//xla/pjrt/c:pjrt_c_api_triton_extension_hdrs", ], ) - -zig_srcs( - name = "sources", - zig_lib = ":pjrt", -) diff --git a/runtimes/neuron/BUILD.bazel b/runtimes/neuron/BUILD.bazel index deab87aa..0fbe667f 100644 --- a/runtimes/neuron/BUILD.bazel +++ b/runtimes/neuron/BUILD.bazel @@ -2,7 +2,7 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_python//python:pip.bzl", "compile_pip_requirements") load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") load("@rules_uv//uv:pip.bzl", "pip_compile") -load("@rules_zig//zig:defs.bzl", "BINARY_KIND", "zig_binary", "zig_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_shared_library") load(":neuron.bzl", "py_binary_with_script") load(":pyenv.bzl", "pyenv_zig") @@ -21,12 +21,9 @@ zig_library( # # Additionally, it provides a way to load implicit transitive dependencies # of neuronx-cc (see add_needed of the patchelf target below). -# -# TODO(cerisier): Use a zig_cc_shared_library instead. -zig_binary( +zig_shared_library( name = "libpjrt_neuron_proxy", - copts = ["-lc"], - kind = BINARY_KIND.shared_lib, + copts = ["-fno-stack-check"], main = "libpjrt_neuron.zig", visibility = ["@libpjrt_neuron//:__subpackages__"], deps = [ diff --git a/runtimes/rocm/BUILD.bazel b/runtimes/rocm/BUILD.bazel index bab048b1..536ccb97 100644 --- a/runtimes/rocm/BUILD.bazel +++ b/runtimes/rocm/BUILD.bazel @@ -1,5 +1,5 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library") -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") cc_library( name = "zmlxrocm_lib", diff --git a/stdx/BUILD.bazel b/stdx/BUILD.bazel index 171d136e..76389c50 100644 --- a/stdx/BUILD.bazel +++ b/stdx/BUILD.bazel @@ -1,5 +1,4 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") -load("@zml//bazel:zig_srcs.bzl", "zig_srcs") zig_library( name = "stdx", @@ -22,11 +21,6 @@ zig_library( zig_test( name = "test", - deps = [":stdx"], testonly = False, -) - -zig_srcs( - name = "sources", - zig_bin = ":test", + deps = [":stdx"], ) diff --git a/stdx/fs.zig b/stdx/fs.zig index 3f278e32..ea8a5fce 100644 --- a/stdx/fs.zig +++ b/stdx/fs.zig @@ -1,5 +1,28 @@ const std = @import("std"); +extern fn dladdr(addr: *anyopaque, info: *Dl_info) c_int; + +const Dl_info = extern struct { + dli_fname: [*c]const u8, + dli_fbase: *anyopaque, + dli_sname: [*c]const u8, + dli_saddr: *anyopaque, +}; + +fn selfSharedObjectPathImpl(addr: usize) []const u8 { + var info: Dl_info = undefined; + _ = dladdr(@ptrFromInt(addr), &info); + return std.mem.span(info.dli_fname); +} + +pub fn selfSharedObjectPath() []const u8 { + return selfSharedObjectPathImpl(@returnAddress()); +} + +pub fn selfSharedObjectDirPath() []const u8 { + return std.fs.path.dirname(selfSharedObjectPathImpl(@returnAddress())).?; +} + pub const path = struct { pub fn bufJoin(buf: []u8, paths: []const []const u8) ![]u8 { var fa: std.heap.FixedBufferAllocator = .init(buf); diff --git a/third_party/modules/rules_zig/20250821.0-be53625/MODULE.bazel b/third_party/modules/rules_zig/20250821.0-be53625/MODULE.bazel new file mode 100644 index 00000000..3c64eb32 --- /dev/null +++ b/third_party/modules/rules_zig/20250821.0-be53625/MODULE.bazel @@ -0,0 +1,75 @@ +module( + name = "rules_zig", + version = "20250821.0-be53625", + compatibility_level = 1, +) + +bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "platforms", version = "0.0.10") + +zig = use_extension("//zig:extensions.bzl", "zig") +zig.index(file = "//zig/private:versions.json") +use_repo(zig, "zig_toolchains") + +register_toolchains("@rules_zig//zig/target:all") + +register_toolchains("@zig_toolchains//:all") + +zig_dev = use_extension( + "//zig:extensions.bzl", + "zig", + dev_dependency = True, +) +zig_dev.toolchain(zig_version = "0.13.0") +zig_dev.toolchain(zig_version = "0.12.1") +zig_dev.toolchain(zig_version = "0.12.0") +zig_dev.toolchain(zig_version = "0.11.0") + +bazel_dep(name = "rules_cc", version = "0.0.9") + +bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") +bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") +bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) +bazel_dep( + name = "buildifier_prebuilt", + version = "7.3.1", + dev_dependency = True, +) +bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) +bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) +bazel_dep( + name = "rules_bazel_integration_test", + version = "0.25.0", + dev_dependency = True, +) + +bazel_binaries = use_extension( + "@rules_bazel_integration_test//:extensions.bzl", + "bazel_binaries", + dev_dependency = True, +) + +# NOTE: Keep in sync with WORKSPACE. +bazel_binaries.download(version_file = "//:.bazelversion") +bazel_binaries.download(version = "7.0.0") +use_repo( + bazel_binaries, + "bazel_binaries", + "bazel_binaries_bazelisk", + "build_bazel_bazel_.bazelversion", + "build_bazel_bazel_7_0_0", +) + +# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. +# However, if we do not include it explicitly, then the runfiles resolution for +# cgrindel_bazel_starlib/shlib/lib/message.sh fails in +# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked +# through the rules_multirun target //util:update. +bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) + +# Hack to get around a cc_common.link limitation. +# See https://github.com/bazelbuild/bazel/pull/23838 +cc_common_link = use_repo_rule("//zig:extensions.bzl", "cc_common_link") + +cc_common_link(name = "build_bazel_rules_android") diff --git a/third_party/modules/rules_zig/20250821.0-be53625/source.json b/third_party/modules/rules_zig/20250821.0-be53625/source.json new file mode 100644 index 00000000..959b2dbc --- /dev/null +++ b/third_party/modules/rules_zig/20250821.0-be53625/source.json @@ -0,0 +1,5 @@ +{ + "strip_prefix": "rules_zig-be53625afb13e73856ee4cab38d5aad9f86f63ef", + "url": "https://github.com/zml/rules_zig/archive/be53625afb13e73856ee4cab38d5aad9f86f63ef.tar.gz", + "integrity": "sha256-IsqEl3BHW/vttyHpVBx5WzO3an/K3eZZx8RJk7nVx8s=" +} diff --git a/third_party/modules/rules_zig/metadata.json b/third_party/modules/rules_zig/metadata.json index 00e9f2f8..00757b88 100644 --- a/third_party/modules/rules_zig/metadata.json +++ b/third_party/modules/rules_zig/metadata.json @@ -7,18 +7,17 @@ "name": "ZML Engineering Team" } ], - "repository": [ - "github:zml/rules_zig" - ], + "repository": ["github:zml/rules_zig"], "versions": [ - "20240904.0-010da15", - "20240909.0-37f17ff", - "20240912.0-41bfe84", - "20240913.0-1957d05", - "20250314.0-b9739c6", - "20250519.0-233b207", - "20250613.0-567662a", - "20250714.0-b14a4f1" + "20240904.0-010da15", + "20240909.0-37f17ff", + "20240912.0-41bfe84", + "20240913.0-1957d05", + "20250314.0-b9739c6", + "20250519.0-233b207", + "20250613.0-567662a", + "20250714.0-b14a4f1", + "20250821.0-be53625" ], "yanked_versions": {} } diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 556ae031..7901965f 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -1,8 +1,6 @@ load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_c_proto_library") load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library") -load("//bazel:zig.bzl", "zig_cc_test") -load("//bazel:zig_srcs.bzl", "zig_srcs") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") upb_c_proto_library( name = "xla_data_upb", @@ -52,7 +50,7 @@ zig_library( # All ZML Tests -zig_cc_test( +zig_test( name = "test", data = [ "aio/torch/simple.pt", @@ -67,8 +65,3 @@ filegroup( srcs = ["test_runner.zig"], visibility = ["//visibility:public"], ) - -zig_srcs( - name = "sources", - zig_bin = ":test_test_lib", -) diff --git a/zml/tokenizer/BUILD.bazel b/zml/tokenizer/BUILD.bazel index d1bb0def..4489d774 100644 --- a/zml/tokenizer/BUILD.bazel +++ b/zml/tokenizer/BUILD.bazel @@ -1,5 +1,4 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") -load("@zml//bazel:zig.bzl", "zig_cc_binary") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") zig_library( name = "tokenizer", @@ -15,7 +14,7 @@ zig_library( ], ) -zig_cc_binary( +zig_binary( name = "main", main = "main.zig", visibility = ["//visibility:public"], diff --git a/zml/tokenizer/sentencepiece/BUILD.bazel b/zml/tokenizer/sentencepiece/BUILD.bazel index 457d811b..354b289a 100644 --- a/zml/tokenizer/sentencepiece/BUILD.bazel +++ b/zml/tokenizer/sentencepiece/BUILD.bazel @@ -1,6 +1,5 @@ load("@rules_zig//zig:defs.bzl", "zig_library") load("//bazel:swig.bzl", "swig_cc_library") -load("//bazel:zig_srcs.bzl", "zig_srcs") swig_cc_library( name = "sentencepiece_swig", @@ -22,8 +21,3 @@ zig_library( "//ffi:zig", ], ) - -zig_srcs( - name = "sources", - zig_lib = ":sentencepiece", -) From b0978b4b2dfe2134f24bc73dacb4f0b25a7edfe8 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 21 Aug 2025 18:15:30 +0200 Subject: [PATCH 018/881] runtimes/rocm: implement zmlxrocm in Zig (#312) Also, sandbox `amdgpu.ids` and restore safetensors json parsing. --- .editorconfig | 0 bazel/cc_import.bzl | 2 +- bazel/patchelf.bzl | 8 +- examples/.vscode/extensions.json | 8 -- examples/.vscode/settings.json | 37 --------- runtimes/cuda/libpjrt_cuda.BUILD.bazel | 2 +- runtimes/neuron/neuron.bzl | 4 +- runtimes/rocm/BUILD.bazel | 19 ++--- runtimes/rocm/libpjrt_rocm.BUILD.bazel | 57 ++++++------- runtimes/rocm/packages.lock.json | 51 +++++++++++- runtimes/rocm/packages.yaml | 4 +- runtimes/rocm/rocm.bzl | 26 +++--- runtimes/rocm/rocm.zig | 38 ++------- runtimes/rocm/zmlxrocm.c | 52 ------------ runtimes/rocm/zmlxrocm.zig | 50 ++++++++++++ zml/aio/json.zig | 107 +++++++++++++++++++++++++ zml/aio/safetensors.zig | 19 +++-- 17 files changed, 280 insertions(+), 204 deletions(-) create mode 100644 .editorconfig delete mode 100644 examples/.vscode/extensions.json delete mode 100644 examples/.vscode/settings.json delete mode 100644 runtimes/rocm/zmlxrocm.c create mode 100644 runtimes/rocm/zmlxrocm.zig create mode 100644 zml/aio/json.zig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..e69de29b diff --git a/bazel/cc_import.bzl b/bazel/cc_import.bzl index d65d90da..7c0e8de5 100644 --- a/bazel/cc_import.bzl +++ b/bazel/cc_import.bzl @@ -54,7 +54,7 @@ def cc_import( patched_name = "{}.patchelf".format(name) patchelf( name = patched_name, - shared_library = shared_library, + src = shared_library, soname = soname, add_needed = add_needed, remove_needed = remove_needed, diff --git a/bazel/patchelf.bzl b/bazel/patchelf.bzl index 265c3979..ac79033c 100644 --- a/bazel/patchelf.bzl +++ b/bazel/patchelf.bzl @@ -1,5 +1,5 @@ def _patchelf_impl(ctx): - output_name = ctx.file.shared_library.basename + output_name = ctx.file.src.basename if ctx.attr.soname: output_name = ctx.attr.soname output = ctx.actions.declare_file("{}/{}".format(ctx.attr.name, output_name)) @@ -43,9 +43,9 @@ def _patchelf_impl(ctx): ctx.actions.write(renamed_syms, "") ctx.actions.run_shell( - inputs = [ctx.file.shared_library, renamed_syms], + inputs = [ctx.file.src, renamed_syms], outputs = [output], - arguments = [ctx.executable._patchelf.path, ctx.file.shared_library.path, output.path], + arguments = [ctx.executable._patchelf.path, ctx.file.src.path, output.path], command = "\n".join(commands), tools = [ctx.executable._patchelf], ) @@ -59,7 +59,7 @@ def _patchelf_impl(ctx): patchelf = rule( implementation = _patchelf_impl, attrs = { - "shared_library": attr.label(allow_single_file = True, mandatory = True), + "src": attr.label(allow_single_file = True, mandatory = True), "soname": attr.string(), "add_needed": attr.string_list(), "remove_needed": attr.string_list(), diff --git a/examples/.vscode/extensions.json b/examples/.vscode/extensions.json deleted file mode 100644 index 194cdc41..00000000 --- a/examples/.vscode/extensions.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "recommendations": [ - "BazelBuild.vscode-bazel", - "llvm-vs-code-extensions.vscode-mlir", - "ms-azuretools.vscode-docker", - "ziglang.vscode-zig", - ] -} diff --git a/examples/.vscode/settings.json b/examples/.vscode/settings.json deleted file mode 100644 index 82050af6..00000000 --- a/examples/.vscode/settings.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "editor.formatOnSave": true, - "files.exclude": { - "**/.git": true, - "**/.svn": true, - "**/.hg": true, - "**/CVS": true, - "**/.DS_Store": true, - "**/bazel-*": true - }, - "files.watcherExclude": { - "**/bazel-*": true, - }, - // "files.trimTrailingWhitespace": true, - "files.insertFinalNewline": true, - "bazel.buildifierFixOnFormat": true, - "bazel.queriesShareServer": true, - "bazel.pathsToIgnore": [ - "." - ], - "bazel.buildifierExecutable": "@buildifier_prebuilt//:buildifier", - "C_Cpp.formatting": "clangFormat", - "C_Cpp.clang_format_style": "Microsoft", - "zig.formattingProvider": "zls", - "zig.path": "x", - "zig.zls.path": "${workspaceFolder}/tools/zls.sh", - "[zig]": { - "editor.suggest.insertMode": "replace", - "editor.stickyScroll.defaultModel": "foldingProviderModel", - "editor.codeActionsOnSave": { - "source.fixAll": "explicit", - } - }, - "rust-analyzer.linkedProjects": [ - "${workspaceFolder}/zml/tokenizer/hftokenizers/Cargo.toml" - ] -} diff --git a/runtimes/cuda/libpjrt_cuda.BUILD.bazel b/runtimes/cuda/libpjrt_cuda.BUILD.bazel index d49d1357..6cb5d74b 100644 --- a/runtimes/cuda/libpjrt_cuda.BUILD.bazel +++ b/runtimes/cuda/libpjrt_cuda.BUILD.bazel @@ -10,7 +10,7 @@ cc_shared_library( patchelf( name = "libpjrt_cuda.patchelf", - shared_library = "libpjrt_cuda.so", + src = "libpjrt_cuda.so", add_needed = [ "libzmlxcuda.so.0", ], diff --git a/runtimes/neuron/neuron.bzl b/runtimes/neuron/neuron.bzl index 89996fab..a277b357 100644 --- a/runtimes/neuron/neuron.bzl +++ b/runtimes/neuron/neuron.bzl @@ -31,7 +31,7 @@ _NEURON_PACKAGES = { ), packages.patchelf( name = "libnrt.patchelf", - shared_library = "lib/libnrt.so.1", + src = "lib/libnrt.so.1", set_rpath = '$ORIGIN', add_needed = [ # readelf -d ./opt/aws/neuron/libl/libncfw.so @@ -43,7 +43,7 @@ _NEURON_PACKAGES = { ), packages.patchelf( name = "libncfw.patchelf", - shared_library = "lib/libncfw.so", + src = "lib/libncfw.so", soname = "libncfw.so.2", ), ]), diff --git a/runtimes/rocm/BUILD.bazel b/runtimes/rocm/BUILD.bazel index 536ccb97..f94e8069 100644 --- a/runtimes/rocm/BUILD.bazel +++ b/runtimes/rocm/BUILD.bazel @@ -1,13 +1,14 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_shared_library") -cc_library( - name = "zmlxrocm_lib", - srcs = ["zmlxrocm.c"], - linkopts = [ - "-lc", - "-ldl", - ], +zig_shared_library( + name = "zmlxrocm", + main = "zmlxrocm.zig", + # Use Clang's compiler-rt, but disable stack checking + # to avoid requiring on the _zig_probe_stack symbol. + copts = ["-fno-stack-check"], + shared_lib_name = "libzmlxrocm.so.0", + deps = ["//stdx"], visibility = ["@libpjrt_rocm//:__subpackages__"], ) @@ -51,6 +52,6 @@ zig_library( filegroup( name = "layers", - srcs = ["@libpjrt_rocm//:amdgpu_ids_layer"], + srcs = [], visibility = ["//visibility:public"], ) diff --git a/runtimes/rocm/libpjrt_rocm.BUILD.bazel b/runtimes/rocm/libpjrt_rocm.BUILD.bazel index 9b1a143e..7a59252b 100644 --- a/runtimes/rocm/libpjrt_rocm.BUILD.bazel +++ b/runtimes/rocm/libpjrt_rocm.BUILD.bazel @@ -20,26 +20,12 @@ config_setting( flag_values = {":hipblaslt": "True"}, ) -cc_shared_library( - name = "zmlxrocm_so", - shared_lib_name = "lib/libzmlxrocm.so.0", - deps = ["@zml//runtimes/rocm:zmlxrocm_lib"], -) - patchelf( - name = "libpjrt_rocm.patchelf", - shared_library = "libpjrt_rocm.so", + name = "libpjrt_rocm_so", + src = "libpjrt_rocm.so", add_needed = [ "libzmlxrocm.so.0", - # So that RPATH is taken into account. - "librocblas.so.4", - "libMIOpen.so.1", - ] + select({ - "_hipblaslt": [ - "libhipblaslt.so.0", - ], - "//conditions:default": [], - }), + ], rename_dynamic_symbols = { "dlopen": "zmlxrocm_dlopen", }, @@ -49,49 +35,52 @@ patchelf( copy_to_directory( name = "sandbox", srcs = [ - ":zmlxrocm_so", - ":libpjrt_rocm.patchelf", + ":libpjrt_rocm_so", "@comgr//:amd_comgr", - "@hip-runtime-amd//:amdhip_patched", + "@hip-runtime-amd//:amdhip", "@hip-runtime-amd//:hiprtc", "@hipblaslt//:hipblaslt", "@hipfft", "@hipsolver", "@hsa-amd-aqlprofile//:hsa-amd-aqlprofile", "@hsa-rocr//:hsa-runtime", + "@libdrm-amdgpu-amdgpu1", + "@libdrm-amdgpu-common//:amdgpu_ids", + "@libdrm2-amdgpu", + "@libelf1", + "@libnuma1", + "@libtinfo6", + "@libzstd1", "@miopen-hip//:MIOpen", "@rccl", "@rocblas//:rocblas", "@rocblas//:runfiles", + "@rocfft", "@rocm-core", "@rocm-device-libs//:runfiles", "@rocm-smi-lib//:rocm_smi", "@rocprofiler-register", - "@rocfft", "@rocsolver", "@roctracer", "@roctracer//:roctx", - "@libelf1", - "@libdrm2-amdgpu", - "@libnuma1", - "@libzstd1", - "@libdrm-amdgpu-amdgpu1", - "@libtinfo6", "@zlib1g", + "@zml//runtimes/rocm:zmlxrocm", ] + select({ ":_hipblaslt": ["@hipblaslt//:runfiles"], "//conditions:default": [], }), replace_prefixes = { - "libpjrt_rocm.patchelf": "lib", - "lib/x86_64-linux-gnu": "lib", - "usr/lib/x86_64-linux-gnu": "lib", - "libelf1": "lib", + "amdhip": "lib", "hipblaslt": "lib", - "rocblas": "lib", - "opt/amdgpu/lib/x86_64-linux-gnu": "lib", + "lib/x86_64-linux-gnu": "lib", "libdrm-amdgpu-amdgpu1": "lib", - "amdhip_patched": "lib", + "libelf1": "lib", + "libpjrt_rocm_so": "lib", + "opt/amdgpu/lib/x86_64-linux-gnu": "lib", + "opt/amdgpu/share": "share", + "rocblas": "lib", + "runtimes/rocm": "lib", + "usr/lib/x86_64-linux-gnu": "lib", }, add_directory_to_runfiles = True, include_external_repositories = ["**"], diff --git a/runtimes/rocm/packages.lock.json b/runtimes/rocm/packages.lock.json index 42a3aafd..213b57c9 100755 --- a/runtimes/rocm/packages.lock.json +++ b/runtimes/rocm/packages.lock.json @@ -1277,7 +1277,13 @@ }, { "arch": "amd64", - "dependencies": [], + "dependencies": [ + { + "key": "rocm-core_6.4.1.60401-83_22.04_amd64", + "name": "rocm-core", + "version": "6.4.1.60401-83~22.04" + } + ], "key": "roctracer_4.1.60401.60401-83_22.04_amd64", "name": "roctracer", "sha256": "58cead537cf07c8a8770bfe28346c3b3c92cc4297b51e307c9032b04434b187c", @@ -4153,6 +4159,49 @@ "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/r/rocfft/rocfft_1.0.32.60401-83~22.04_amd64.deb" ], "version": "1.0.32.60401-83~22.04" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "hipblaslt_0.12.1.60401-83_22.04_amd64", + "name": "hipblaslt", + "version": "0.12.1.60401-83~22.04" + }, + { + "key": "rocm-core_6.4.1.60401-83_22.04_amd64", + "name": "rocm-core", + "version": "6.4.1.60401-83~22.04" + }, + { + "key": "roctracer_4.1.60401.60401-83_22.04_amd64", + "name": "roctracer", + "version": "4.1.60401.60401-83~22.04" + }, + { + "key": "hipblas-common-dev_1.0.0.60401-83_22.04_amd64", + "name": "hipblas-common-dev", + "version": "1.0.0.60401-83~22.04" + } + ], + "key": "hipblaslt-dev_0.12.1.60401-83_22.04_amd64", + "name": "hipblaslt-dev", + "sha256": "46eb2285c76d246b162eb54cc7f9e5cb7bcdd0aa83d57ecaea440e57260f2f4a", + "urls": [ + "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/h/hipblaslt-dev/hipblaslt-dev_0.12.1.60401-83~22.04_amd64.deb" + ], + "version": "0.12.1.60401-83~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "hipblas-common-dev_1.0.0.60401-83_22.04_amd64", + "name": "hipblas-common-dev", + "sha256": "5df3e4a8a1959cbf94106f7bf87d7fb71bf06e726cde00c6092ef29bbd8156f0", + "urls": [ + "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/h/hipblas-common-dev/hipblas-common-dev_1.0.0.60401-83~22.04_amd64.deb" + ], + "version": "1.0.0.60401-83~22.04" } ], "version": 1 diff --git a/runtimes/rocm/packages.yaml b/runtimes/rocm/packages.yaml index fd425f4f..fcb6583d 100644 --- a/runtimes/rocm/packages.yaml +++ b/runtimes/rocm/packages.yaml @@ -35,7 +35,7 @@ packages: - "rocsolver" - "hipsolver" - "hipfft" - # - "roctracer" + - "roctracer" - "hipblaslt" - # - "hipblaslt-dev" + - "hipblaslt-dev" - "hip-runtime-amd" diff --git a/runtimes/rocm/rocm.bzl b/runtimes/rocm/rocm.bzl index b38f69be..1d11da3e 100644 --- a/runtimes/rocm/rocm.bzl +++ b/runtimes/rocm/rocm.bzl @@ -14,7 +14,7 @@ _UBUNTU_PACKAGES = { packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), packages.patchelf( name = "libelf1", - shared_library = "usr/lib/x86_64-linux-gnu/libelf.so.1", + src = "usr/lib/x86_64-linux-gnu/libelf.so.1", set_rpath = "$ORIGIN", ), ]), @@ -25,8 +25,12 @@ _UBUNTU_PACKAGES = { packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), packages.patchelf( name = "libdrm-amdgpu-amdgpu1", - shared_library = "opt/amdgpu/lib/x86_64-linux-gnu/libdrm_amdgpu.so.1", + src = "opt/amdgpu/lib/x86_64-linux-gnu/libdrm_amdgpu.so.1", + add_needed = ["libzmlxrocm.so.0"], set_rpath = "$ORIGIN", + rename_dynamic_symbols = { + "fopen64": "zmlxrocm_fopen64", + }, ), ]), "libtinfo6": packages.filegroup(name = "libtinfo6", srcs = ["lib/x86_64-linux-gnu/libtinfo.so.6"]), @@ -38,14 +42,7 @@ _ROCM_PACKAGES = { "rocm-smi-lib": packages.filegroup(name = "rocm_smi", srcs = ["lib/librocm_smi64.so.7"]), "hsa-rocr": packages.filegroup(name = "hsa-runtime", srcs = ["lib/libhsa-runtime64.so.1"]), "hsa-amd-aqlprofile": packages.filegroup(name = "hsa-amd-aqlprofile", srcs = ["lib/libhsa-amd-aqlprofile64.so.1"]), - "comgr": "\n".join([ - packages.filegroup( - name = "amd_comgr", - srcs = [ - "lib/libamd_comgr.so.3", - ], - ), - ]), + "comgr": packages.filegroup(name = "amd_comgr", srcs = ["lib/libamd_comgr.so.3"]), "rocprofiler-register": packages.filegroup(name = "rocprofiler-register", srcs = ["lib/librocprofiler-register.so.0"]), "miopen-hip": "\n".join([ packages.filegroup(name = "MIOpen", srcs = ["lib/libMIOpen.so.1"]), @@ -59,7 +56,7 @@ _ROCM_PACKAGES = { packages.load_("@zml//runtimes/rocm:gfx.bzl", "bytecode_select"), packages.patchelf( name = "rocblas", - shared_library = "lib/librocblas.so.4", + src = "lib/librocblas.so.4", add_needed = ["libzmlxrocm.so.0"], rename_dynamic_symbols = { "dlopen": "zmlxrocm_dlopen", @@ -90,7 +87,7 @@ _ROCM_PACKAGES = { packages.load_("@zml//runtimes/rocm:gfx.bzl", "bytecode_select"), packages.patchelf( name = "hipblaslt", - shared_library = "lib/libhipblaslt.so.0", + src = "lib/libhipblaslt.so.0", add_needed = ["libzmlxrocm.so.0"], rename_dynamic_symbols = { "dlopen": "zmlxrocm_dlopen", @@ -116,10 +113,9 @@ _ROCM_PACKAGES = { "hipfft": packages.filegroup(name = "hipfft", srcs = ["lib/libhipfft.so.0"]), "hip-runtime-amd": "\n".join([ packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), - packages.filegroup(name = "amdhip", srcs = ["lib/libamdhip64.so.6"]), packages.patchelf( - name = "amdhip_patched", - shared_library = ":amdhip", + name = "amdhip", + src = "lib/libamdhip64.so.6", add_needed = ["libzmlxrocm.so.0"], rename_dynamic_symbols = { "dlopen": "zmlxrocm_dlopen", diff --git a/runtimes/rocm/rocm.zig b/runtimes/rocm/rocm.zig index 98be6031..0861f92c 100644 --- a/runtimes/rocm/rocm.zig +++ b/runtimes/rocm/rocm.zig @@ -1,5 +1,5 @@ -const builtin = @import("builtin"); const std = @import("std"); +const builtin = @import("builtin"); const asynk = @import("async"); const bazel_builtin = @import("bazel_builtin"); @@ -10,20 +10,6 @@ const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/runtime/rocm"); -const ROCmEnvEntry = struct { - name: [:0]const u8, - rpath: []const u8, - dirname: bool, - mandatory: bool, -}; - -const rocm_env_entries: []const ROCmEnvEntry = &.{ - .{ .name = "HIPBLASLT_EXT_OP_LIBRARY_PATH", .rpath = "/lib/hipblaslt/library/hipblasltExtOpLibrary.dat", .dirname = false, .mandatory = false }, - .{ .name = "HIPBLASLT_TENSILE_LIBPATH", .rpath = "/lib/hipblaslt/library/TensileManifest.txt", .dirname = true, .mandatory = false }, - .{ .name = "ROCBLAS_TENSILE_LIBPATH", .rpath = "/lib/rocblas/library/TensileManifest.txt", .dirname = true, .mandatory = true }, - .{ .name = "ROCM_PATH", .rpath = "/", .dirname = false, .mandatory = true }, -}; - pub fn isEnabled() bool { return @hasDecl(c, "ZML_RUNTIME_ROCM"); } @@ -35,23 +21,9 @@ fn hasRocmDevices() bool { return true; } -fn setupRocmEnv(allocator: std.mem.Allocator, rocm_data_dir: []const u8) !void { - for (rocm_env_entries) |entry| { - var real_path: []const u8 = std.fmt.allocPrintZ(allocator, "{s}/{s}", .{ rocm_data_dir, entry.rpath }) catch null orelse { - if (entry.mandatory) { - stdx.debug.panic("Unable to find {s} in {s}\n", .{ entry.name, bazel_builtin.current_repository }); - } - continue; - }; - - if (entry.dirname) { - real_path = std.fs.path.dirname(real_path) orelse { - stdx.debug.panic("Unable to dirname on {s}", .{real_path}); - }; - } - - _ = c.setenv(entry.name, try allocator.dupeZ(u8, real_path), 1); - } +fn setupRocmEnv(rocm_data_dir: []const u8) !void { + var buf: [std.fs.max_path_bytes]u8 = undefined; + _ = c.setenv("ROCM_PATH", try stdx.fs.path.bufJoinZ(&buf, &.{rocm_data_dir}), 1); // must be zero terminated } pub fn load() !*const pjrt.Api { @@ -81,7 +53,7 @@ pub fn load() !*const pjrt.Api { return error.FileNotFound; }; - try setupRocmEnv(arena.allocator(), sandbox_path); + try setupRocmEnv(sandbox_path); var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; const lib_path = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_rocm.so" }); diff --git a/runtimes/rocm/zmlxrocm.c b/runtimes/rocm/zmlxrocm.c deleted file mode 100644 index 4c81c309..00000000 --- a/runtimes/rocm/zmlxrocm.c +++ /dev/null @@ -1,52 +0,0 @@ -#include -#include -#include -#include - -void *zmlxrocm_dlopen(const char *filename, int flags) __attribute__((visibility("default"))) -{ - if (filename != NULL) - { - char *replacements[] = { - "librocm-core.so", - "librocm-core.so.1", - "librocm_smi64.so", - "librocm_smi64.so.7", - "libhsa-runtime64.so", - "libhsa-runtime64.so.1", - "libhsa-amd-aqlprofile64.so", - "libhsa-amd-aqlprofile64.so.1", - "libamd_comgr.so", - "libamd_comgr.so.3", - "librocprofiler-register.so", - "librocprofiler-register.so.0", - "libMIOpen.so", - "libMIOpen.so.1", - "librccl.so", - "librccl.so.1", - "librocblas.so", - "librocblas.so.4", - "libroctracer64.so", - "libroctracer64.so.4", - "libroctx64.so", - "libroctx64.so.4", - "libhipblaslt.so", - "libhipblaslt.so.0", - "libamdhip64.so", - "libamdhip64.so.6", - "libhiprtc.so", - "libhiprtc.so.6", - NULL, - NULL, - }; - for (int i = 0; replacements[i] != NULL; i += 2) - { - if (strcmp(filename, replacements[i]) == 0) - { - filename = replacements[i + 1]; - break; - } - } - } - return dlopen(filename, flags); -} diff --git a/runtimes/rocm/zmlxrocm.zig b/runtimes/rocm/zmlxrocm.zig new file mode 100644 index 00000000..19f840c3 --- /dev/null +++ b/runtimes/rocm/zmlxrocm.zig @@ -0,0 +1,50 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +pub export fn zmlxrocm_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque { + const replacements: std.StaticStringMap([:0]const u8) = .initComptime(.{ + .{ "librocm-core.so", "librocm-core.so.1" }, + .{ "librocm_smi64.so", "librocm_smi64.so.7" }, + .{ "libhsa-runtime64.so", "libhsa-runtime64.so.1" }, + .{ "libhsa-amd-aqlprofile64.so", "libhsa-amd-aqlprofile64.so.1" }, + .{ "libamd_comgr.so", "libamd_comgr.so.3" }, + .{ "librocprofiler-register.so", "librocprofiler-register.so.0" }, + .{ "libMIOpen.so", "libMIOpen.so.1" }, + .{ "librccl.so", "librccl.so.1" }, + .{ "librocblas.so", "librocblas.so.4" }, + .{ "libroctracer64.so", "libroctracer64.so.4" }, + .{ "libroctx64.so", "libroctx64.so.4" }, + .{ "libhipblaslt.so", "libhipblaslt.so.0" }, + .{ "libamdhip64.so", "libamdhip64.so.6" }, + .{ "libhiprtc.so", "libhiprtc.so.6" }, + }); + + var buf: [std.fs.max_path_bytes]u8 = undefined; + const new_filename: [*c]const u8 = if (filename) |f| blk: { + const replacement = replacements.get(std.fs.path.basename(std.mem.span(f))) orelse break :blk f; + break :blk stdx.fs.path.bufJoinZ(&buf, &.{ + stdx.fs.selfSharedObjectDirPath(), + replacement, + }) catch unreachable; + } else null; + + return std.c.dlopen(new_filename, @bitCast(flags)); +} + +pub export fn zmlxrocm_fopen64(pathname: [*c]const u8, mode: [*c]const u8) ?*std.c.FILE { + const replacements: std.StaticStringMap([]const u8) = .initComptime(.{ + .{ "/opt/amdgpu/share/libdrm/amdgpu.ids", "../share/libdrm/amdgpu.ids" }, + }); + + var buf: [std.fs.max_path_bytes]u8 = undefined; + const new_pathname: [*c]const u8 = blk: { + const replacement = replacements.get(std.mem.span(pathname)) orelse break :blk pathname; + break :blk stdx.fs.path.bufJoinZ(&buf, &.{ + stdx.fs.selfSharedObjectDirPath(), + replacement, + }) catch unreachable; + }; + + return std.c.fopen64(new_pathname, mode); +} diff --git a/zml/aio/json.zig b/zml/aio/json.zig new file mode 100644 index 00000000..9c02a5db --- /dev/null +++ b/zml/aio/json.zig @@ -0,0 +1,107 @@ +const asynk = @import("async"); +const std = @import("std"); +const zml = @import("../zml.zig"); + +const StringBuilder = std.ArrayListUnmanaged(u8); +const Allocator = std.mem.Allocator; + +pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore { + const file = try std.fs.cwd().openFile(path, .{}); + defer file.close(); + var res: zml.aio.BufferStore = .{ + .arena = std.heap.ArenaAllocator.init(allocator), + }; + errdefer res.arena.deinit(); + const arena = res.arena.allocator(); + + const json_data = try file.reader().readAllAlloc(arena, (try file.metadata()).size()); + const metadata = try std.json.parseFromSliceLeaky(std.json.Value, allocator, json_data, .{ .allocate = .alloc_if_needed }); + + var it = metadata.object.iterator(); + while (it.next()) |entry| { + var prefix_buf: [1024]u8 = undefined; + try parseMetadata(allocator, &res, StringBuilder.initBuffer(&prefix_buf), entry.value_ptr.*); + } + + return res; +} + +pub fn parseMetadata(allocator: Allocator, store: *zml.aio.BufferStore, prefix: StringBuilder, val: std.json.Value) !void { + const metadata = &store._metadata; + const key = prefix.items; + return switch (val) { + .null => try metadata.put(allocator, try allocator.dupe(u8, key), .null), + .bool => |v| try metadata.put(allocator, try allocator.dupe(u8, key), .{ .bool = v }), + .integer => |v| try metadata.put(allocator, try allocator.dupe(u8, key), .{ .int = v }), + .float => |v| try metadata.put(allocator, try allocator.dupe(u8, key), .{ .float = v }), + .number_string, .string => |v| try metadata.put(allocator, try allocator.dupe(u8, key), .{ .string = try allocator.dupe(u8, v) }), + .array => |v| { + if (v.items.len == 0) return; + return if (validSlice(v)) |item_type| { + const data: zml.aio.Metadata = switch (item_type) { + .bool => blk: { + const values = try allocator.alloc(bool, v.items.len); + for (v.items, 0..) |item, i| values[i] = item.bool; + break :blk .{ .array_bool = values }; + }, + .integer => blk: { + const values = try allocator.alloc(i64, v.items.len); + for (v.items, 0..) |item, i| values[i] = item.integer; + break :blk .{ .array_int = values }; + }, + .float => blk: { + const values = try allocator.alloc(f64, v.items.len); + for (v.items, 0..) |item, i| values[i] = item.float; + break :blk .{ .array_float = values }; + }, + inline .string, .number_string => |tag| blk: { + const values = try allocator.alloc([]const u8, v.items.len); + for (v.items, 0..) |item, i| { + values[i] = @field(item, @tagName(tag)); + } + break :blk .{ .array_string = values }; + }, + .null, .array, .object => unreachable, + }; + try metadata.put(allocator, try allocator.dupe(u8, key), data); + } else { + for (v.items, 0..) |item, i| { + var new_prefix = prefix; + if (prefix.items.len > 0) + new_prefix.appendAssumeCapacity('.'); + new_prefix.items.len += std.fmt.formatIntBuf(new_prefix.unusedCapacitySlice(), i, 10, .lower, .{}); + try parseMetadata(allocator, store, new_prefix, item); + } + }; + }, + .object => |v| { + var obj_iter = v.iterator(); + while (obj_iter.next()) |entry| { + var new_prefix = prefix; + if (prefix.items.len > 0) + new_prefix.appendAssumeCapacity('.'); + new_prefix.appendSliceAssumeCapacity(entry.key_ptr.*); + try parseMetadata(allocator, store, new_prefix, entry.value_ptr.*); + } + }, + }; +} + +/// We can only create a Zig slice out of json array, if all values +/// in the array have the same type. +fn validSlice(v: std.json.Array) ?std.meta.Tag(std.json.Value) { + if (v.items.len == 0) return null; + + const item_type: std.meta.Tag(std.json.Value) = v.items[0]; + switch (item_type) { + .null, .array, .object => return null, + else => {}, + } + + for (v.items[1..]) |item| { + if (item != item_type) + return null; + } + + return item_type; +} diff --git a/zml/aio/safetensors.zig b/zml/aio/safetensors.zig index 8d674807..581ab77f 100644 --- a/zml/aio/safetensors.zig +++ b/zml/aio/safetensors.zig @@ -1,13 +1,12 @@ -const std = @import("std"); -const Allocator = std.mem.Allocator; - const asynk = @import("async"); - -const MemoryMappedFile = @import("../aio.zig").MemoryMappedFile; +const std = @import("std"); const zml = @import("../zml.zig"); +const json = @import("json.zig"); const HostBuffer = zml.HostBuffer; +const MemoryMappedFile = @import("../aio.zig").MemoryMappedFile; const StringBuilder = std.ArrayListUnmanaged(u8); +const Allocator = std.mem.Allocator; const log = std.log.scoped(.@"zml/io"); pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore { @@ -56,6 +55,11 @@ fn loadFromIndex(allocator: Allocator, store: *zml.aio.BufferStore, files: *std. const full_filename = try std.fs.path.join(allocator, &.{ std.fs.path.dirname(path).?, filename }); try loadFile(allocator, store, files, full_filename); } + + if (index.object.get("__metadata__")) |metadata| { + var prefix_buf: [1024]u8 = undefined; + try json.parseMetadata(allocator, store, StringBuilder.initBuffer(&prefix_buf), metadata); + } } fn loadFile(allocator: Allocator, store: *zml.aio.BufferStore, files: *std.ArrayList(MemoryMappedFile), path: []const u8) !void { @@ -85,6 +89,11 @@ fn loadFile(allocator: Allocator, store: *zml.aio.BufferStore, files: *std.Array var it = metadata.object.iterator(); while (it.next()) |entry| { const key = entry.key_ptr.*; + if (std.mem.eql(u8, key, "__metadata__")) { + var prefix_buf: [1024]u8 = undefined; + try json.parseMetadata(allocator, store, StringBuilder.initBuffer(&prefix_buf), entry.value_ptr.*); + continue; + } const val = entry.value_ptr.*; const shape_field = val.object.get("shape").?.array; if (shape_field.items.len > zml.Shape.MAX_RANK) { From 0c796302e2cd5b945179fb624eff9980465e8b39 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 21 Aug 2025 18:34:09 +0200 Subject: [PATCH 019/881] runtimes/cuda: implement zmlxcuda in Zig (#313) --- runtimes/cuda/BUILD.bazel | 13 +++++--- runtimes/cuda/cuda.bzl | 43 +++++++++++--------------- runtimes/cuda/libpjrt_cuda.BUILD.bazel | 36 ++++++++++----------- runtimes/cuda/zmlxcuda.c | 40 ------------------------ runtimes/cuda/zmlxcuda.zig | 28 +++++++++++++++++ 5 files changed, 72 insertions(+), 88 deletions(-) delete mode 100644 runtimes/cuda/zmlxcuda.c create mode 100644 runtimes/cuda/zmlxcuda.zig diff --git a/runtimes/cuda/BUILD.bazel b/runtimes/cuda/BUILD.bazel index 0913c058..38904d47 100644 --- a/runtimes/cuda/BUILD.bazel +++ b/runtimes/cuda/BUILD.bazel @@ -1,9 +1,14 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_shared_library") -cc_library( - name = "zmlxcuda_lib", - srcs = ["zmlxcuda.c"], +zig_shared_library( + name = "zmlxcuda", + main = "zmlxcuda.zig", + # Use Clang's compiler-rt, but disable stack checking + # to avoid requiring on the _zig_probe_stack symbol. + copts = ["-fno-stack-check"], + shared_lib_name = "libzmlxcuda.so.0", + deps = ["//stdx"], visibility = ["@libpjrt_cuda//:__subpackages__"], ) diff --git a/runtimes/cuda/cuda.bzl b/runtimes/cuda/cuda.bzl index 2d27705f..33806948 100644 --- a/runtimes/cuda/cuda.bzl +++ b/runtimes/cuda/cuda.bzl @@ -31,12 +31,12 @@ CUDA_PACKAGES = { ), #TODO: Remove me as soon we use the Driver API in tracer.zig packages.filegroup( - name = "so_files", + name = "cuda_cudart", srcs = ["lib/libcudart.so.12"], ), ]), "cuda_cupti": packages.filegroup( - name = "so_files", + name = "cuda_cupti", srcs = ["lib/libcupti.so.12"], ), "cuda_nvtx": "\n".join([ @@ -46,42 +46,35 @@ CUDA_PACKAGES = { # visibility = ["//visibility:public"], # ), packages.filegroup( - name = "so_files", + name = "cuda_nvtx", srcs = ["lib/libnvToolsExt.so.1"], ), ]), "libcufft": packages.filegroup( - name = "so_files", + name = "libcufft", srcs = ["lib/libcufft.so.11"], ), "libcusolver": packages.filegroup( - name = "so_files", + name = "libcusolver", srcs = ["lib/libcusolver.so.11"], ), "libcusparse": packages.filegroup( - name = "so_files", + name = "libcusparse", srcs = ["lib/libcusparse.so.12"], ), "libnvjitlink": packages.filegroup( - name = "so_files", + name = "libnvjitlink", srcs = ["lib/libnvJitLink.so.12"], ), "cuda_nvcc": "\n".join([ packages.filegroup( - name = "ptxas", - srcs = ["bin/ptxas"], - ), - packages.filegroup( - name = "nvlink", - srcs = ["bin/nvlink"], - ), - packages.filegroup( - name = "libdevice", - srcs = ["nvvm/libdevice/libdevice.10.bc"], - ), - packages.filegroup( - name = "so_files", - srcs = ["nvvm/lib64/libnvvm.so.4"], + name = "cuda_nvcc", + srcs = [ + "bin/ptxas", + "bin/nvlink", + "nvvm/libdevice/libdevice.10.bc", + "nvvm/lib64/libnvvm.so.4", + ], ), packages.cc_import( name = "nvptxcompiler", @@ -90,7 +83,7 @@ CUDA_PACKAGES = { ]), "cuda_nvrtc": "\n".join([ packages.filegroup( - name = "so_files", + name = "cuda_nvrtc", srcs = [ "lib/libnvrtc.so.12", "lib/libnvrtc-builtins.so.12.8", @@ -99,7 +92,7 @@ CUDA_PACKAGES = { ]), "libcublas": "\n".join([ packages.filegroup( - name = "so_files", + name = "libcublas", srcs = [ "lib/libcublasLt.so.12", "lib/libcublas.so.12", @@ -111,7 +104,7 @@ CUDA_PACKAGES = { CUDNN_PACKAGES = { "cudnn": "\n".join([ packages.filegroup( - name = "so_files", + name = "cudnn", srcs = [ "lib/libcudnn.so.9", "lib/libcudnn_adv.so.9", @@ -193,7 +186,7 @@ def _cuda_impl(mctx): type = "zip", sha256 = "362aed5963fb9ea2ed2f264409baae30143498fd0e5c503aeaa1badd88cdc54a", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + packages.filegroup( - name = "so_files", + name = "nccl", srcs = ["nvidia/nccl/lib/libnccl.so.2"], ), ) diff --git a/runtimes/cuda/libpjrt_cuda.BUILD.bazel b/runtimes/cuda/libpjrt_cuda.BUILD.bazel index 6cb5d74b..031ec507 100644 --- a/runtimes/cuda/libpjrt_cuda.BUILD.bazel +++ b/runtimes/cuda/libpjrt_cuda.BUILD.bazel @@ -9,7 +9,7 @@ cc_shared_library( ) patchelf( - name = "libpjrt_cuda.patchelf", + name = "libpjrt_cuda_so", src = "libpjrt_cuda.so", add_needed = [ "libzmlxcuda.so.0", @@ -23,30 +23,28 @@ patchelf( copy_to_directory( name = "sandbox", srcs = [ - ":zmlxcuda_so", - ":libpjrt_cuda.patchelf", - "@cuda_nvcc//:libdevice", - "@cuda_nvcc//:ptxas", - "@cuda_nvcc//:nvlink", - "@cuda_cupti//:so_files", - "@cuda_nvtx//:so_files", - "@cuda_nvcc//:so_files", - "@cuda_nvrtc//:so_files", - "@cuda_cudart//:so_files", - "@cudnn//:so_files", - "@libcublas//:so_files", - "@libcufft//:so_files", - "@libcusolver//:so_files", - "@libcusparse//:so_files", - "@libnvjitlink//:so_files", - "@nccl//:so_files", + ":libpjrt_cuda_so", + "@cuda_cudart", + "@cuda_cupti", + "@cuda_nvcc", + "@cuda_nvrtc", + "@cuda_nvtx", + "@cudnn", + "@libcublas", + "@libcufft", + "@libcusolver", + "@libcusparse", + "@libnvjitlink", + "@nccl", "@zlib1g", + "@zml//runtimes/cuda:zmlxcuda", ], replace_prefixes = { "nvidia/nccl/lib": "lib", "nvvm/lib64": "lib", - "libpjrt_cuda.patchelf": "lib", + "libpjrt_cuda_so": "lib", "lib/x86_64-linux-gnu": "lib", + "runtimes/cuda": "lib", }, add_directory_to_runfiles = False, include_external_repositories = ["**"], diff --git a/runtimes/cuda/zmlxcuda.c b/runtimes/cuda/zmlxcuda.c deleted file mode 100644 index 2bd1d32a..00000000 --- a/runtimes/cuda/zmlxcuda.c +++ /dev/null @@ -1,40 +0,0 @@ -#include -#include - -void *zmlxcuda_dlopen(const char *filename, int flags) -{ - if (filename != NULL) - { - char *replacements[] = { - "libcublas.so", - "libcublas.so.12", - "libcublasLt.so", - "libcublasLt.so.12", - "libcudart.so", - "libcudart.so.12", - "libcudnn.so", - "libcudnn.so.9", - "libcufft.so", - "libcufft.so.11", - "libcupti.so", - "libcupti.so.12", - "libcusolver.so", - "libcusolver.so.11", - "libcusparse.so", - "libcusparse.so.12", - "libnccl.so", - "libnccl.so.2", - NULL, - NULL, - }; - for (int i = 0; replacements[i] != NULL; i += 2) - { - if (strcmp(filename, replacements[i]) == 0) - { - filename = replacements[i + 1]; - break; - } - } - } - return dlopen(filename, flags); -} diff --git a/runtimes/cuda/zmlxcuda.zig b/runtimes/cuda/zmlxcuda.zig new file mode 100644 index 00000000..1ccd4baa --- /dev/null +++ b/runtimes/cuda/zmlxcuda.zig @@ -0,0 +1,28 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +pub export fn zmlxcuda_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque { + const replacements: std.StaticStringMap([:0]const u8) = .initComptime(.{ + .{ "libcublas.so", "libcublas.so.12" }, + .{ "libcublasLt.so", "libcublasLt.so.12" }, + .{ "libcudart.so", "libcudart.so.12" }, + .{ "libcudnn.so", "libcudnn.so.9" }, + .{ "libcufft.so", "libcufft.so.11" }, + .{ "libcupti.so", "libcupti.so.12" }, + .{ "libcusolver.so", "libcusolver.so.11" }, + .{ "libcusparse.so", "libcusparse.so.12" }, + .{ "libnccl.so", "libnccl.so.2" }, + }); + + var buf: [std.fs.max_path_bytes]u8 = undefined; + const new_filename: [*c]const u8 = if (filename) |f| blk: { + const replacement = replacements.get(std.fs.path.basename(std.mem.span(f))) orelse break :blk f; + break :blk stdx.fs.path.bufJoinZ(&buf, &.{ + stdx.fs.selfSharedObjectDirPath(), + replacement, + }) catch unreachable; + } else null; + + return std.c.dlopen(new_filename, @bitCast(flags)); +} From fc384eb43ec4b2da1705692a4cd442267254e8c2 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 22 Aug 2025 16:28:23 +0200 Subject: [PATCH 020/881] runtimes/neuron: fix neuron runtime (#314) This PR fixes the neuron runtime with the following: Proxy the PJRT Api method to enforce the client struct sizes since the neuron PJRT plugin doesn't use `>=` but `==` to assert them, breaking PJRT compatibility guarantees. Fixes https://github.com/aws-neuron/aws-neuron-sdk/issues/1095 Reimplement `libneuronxla` in Zig to control neuronx-cc sandboxing and invocation. Implement a python bootstrapper in Zig to create a full blown `neuronx-cc` executable, avoiding the infamous chicken and egg problem of python executables boostrapping when sandboxed (due to fixed path shebangs). --------- Co-authored-by: Corentin Kerisit --- MODULE.bazel | 3 + runtimes/neuron/BUILD.bazel | 161 +++++------ runtimes/neuron/empty.py | 0 runtimes/neuron/libneuronxla.zig | 249 +++++++++++++++++ runtimes/neuron/libpjrt_neuron.BUILD.bazel | 70 ++--- runtimes/neuron/libpjrt_neuron.zig | 209 ++++++++++++-- runtimes/neuron/neuron.bzl | 2 +- runtimes/neuron/neuron.zig | 99 +------ runtimes/neuron/neuronx-cc.zig | 89 ++++++ runtimes/neuron/{libpjrt_neuron.h => nrt.h} | 0 runtimes/neuron/pyenv.bzl | 41 --- runtimes/neuron/requirements.lock.txt | 294 ++++++++++---------- runtimes/neuron/zmlxneuron.c | 24 -- upb/BUILD.bazel | 8 +- upb/upb.h | 1 + upb/upb.zig | 10 +- 16 files changed, 804 insertions(+), 456 deletions(-) delete mode 100644 runtimes/neuron/empty.py create mode 100644 runtimes/neuron/libneuronxla.zig create mode 100644 runtimes/neuron/neuronx-cc.zig rename runtimes/neuron/{libpjrt_neuron.h => nrt.h} (100%) delete mode 100644 runtimes/neuron/pyenv.bzl delete mode 100644 runtimes/neuron/zmlxneuron.c create mode 100644 upb/upb.h diff --git a/MODULE.bazel b/MODULE.bazel index 105eeca0..09230384 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -59,6 +59,9 @@ pip.parse( "--python-version=311", "--platform=linux_x86_64", "--platform=manylinux2014_x86_64", + "--platform=manylinux_2_27_x86_64", + "--platform=manylinux_2_28_x86_64", + "--platform=manylinux_2_29_x86_64", ], hub_name = "neuron_py_deps", python_version = "3.11", diff --git a/runtimes/neuron/BUILD.bazel b/runtimes/neuron/BUILD.bazel index 0fbe667f..184925a5 100644 --- a/runtimes/neuron/BUILD.bazel +++ b/runtimes/neuron/BUILD.bazel @@ -1,20 +1,8 @@ +load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_c_proto_library") load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_python//python:pip.bzl", "compile_pip_requirements") -load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") -load("@rules_uv//uv:pip.bzl", "pip_compile") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_shared_library") -load(":neuron.bzl", "py_binary_with_script") -load(":pyenv.bzl", "pyenv_zig") - -pyenv_zig( - name = "libneuronxla_pyenv_zig", - deps = [":libneuronxla"], -) - -zig_library( - name = "libneuronxla_pyenv", - main = ":libneuronxla_pyenv_zig", -) +load("@rules_python//python/uv:lock.bzl", uv_lock = "lock") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_shared_library") +load("@zml//bazel:runfiles.bzl", "runfiles_to_default") # A proxy PJRT Plugin that loads the Neuron PJRT Plugin # and returns the instance from nested GetPjrtApi. @@ -22,101 +10,107 @@ zig_library( # Additionally, it provides a way to load implicit transitive dependencies # of neuronx-cc (see add_needed of the patchelf target below). zig_shared_library( - name = "libpjrt_neuron_proxy", + name = "libpjrt_neuron", copts = ["-fno-stack-check"], main = "libpjrt_neuron.zig", visibility = ["@libpjrt_neuron//:__subpackages__"], deps = [ + ":libpython", "//stdx", "@rules_zig//zig/runfiles", + "@xla//xla/pjrt/c:pjrt_c_api_hdrs", ], ) -pip_compile( - name = "update_requirements", - args = [ - "--generate-hashes", - "--emit-index-url", - "--emit-find-links", - "--no-strip-extras", - "--index-strategy=unsafe-best-match", - "--upgrade", - ], - python_platform = "x86_64-unknown-linux-gnu", - requirements_in = "requirements.in", - requirements_txt = "requirements.lock.txt", -) - -py_console_script_binary( +zig_binary( name = "neuronx-cc", - binary_rule = py_binary_with_script, - pkg = "@neuron_py_deps//neuronx_cc", - visibility = ["//visibility:public"], + data = ["@neuron_py_deps//neuronx_cc"], + linkopts = ["-Wl,-rpath,$ORIGIN/../lib"], + main = "neuronx-cc.zig", + tags = ["manual"], + deps = [":libpython"], ) -py_binary_with_script( - name = "libneuronxla", - srcs = ["empty.py"], - main = "empty.py", - deps = [ - ":neuronx-cc", - "@neuron_py_deps//libneuronxla", - ], +runfiles_to_default( + name = "neuronx-cc_files", + visibility = ["@libpjrt_neuron//:__subpackages__"], + deps = [":neuronx-cc"], ) cc_library( name = "libpython", hdrs = ["libpython.h"], - visibility = ["//visibility:public"], deps = [ "@rules_python//python/cc:current_py_cc_headers", "@rules_python//python/cc:current_py_cc_libs", ], ) -compile_pip_requirements( - name = "requirements", - src = "requirements.in", - py_binary = py_binary_with_script, - requirements_txt = "requirements.lock.txt", - tags = [ - "manual", - ], -) - -cc_library( - name = "zmlxneuron_lib", - srcs = ["zmlxneuron.c"], - linkopts = ["-ldl"], - visibility = ["@libpjrt_neuron//:__subpackages__"], -) - -alias( - name = "bootstrap", - actual = "@rules_python//python/config_settings:bootstrap_impl", -) - cc_library( name = "empty", ) cc_library( - name = "libpjrt_neuron", - hdrs = ["libpjrt_neuron.h"], + name = "zmlxneuron", defines = ["ZML_RUNTIME_NEURON"], - deps = [ - "@libpjrt_neuron", +) + +cc_library( + name = "libnrt_headers", + hdrs = ["nrt.h"], + deps = ["@libpjrt_neuron//:libnrt_headers"], +) + +filegroup( + name = "layers", + srcs = [], + visibility = ["//visibility:public"], +) + +upb_c_proto_library( + name = "xla_data_upb", + deps = ["@xla//xla:xla_data_proto"], +) + +upb_c_proto_library( + name = "hlo_proto_upb", + deps = ["@xla//xla/service:hlo_proto"], +) + +zig_shared_library( + name = "libneuronxla", + copts = [ + "-fno-stack-check", + "-fPIC", ], + main = "libneuronxla.zig", + shared_lib_name = "libneuronxla.so", + visibility = ["@libpjrt_neuron//:__subpackages__"], + deps = [ + ":hlo_proto_upb", + ":libpython", + ":xla_data_upb", + "//stdx", + "//upb", + ], +) + +uv_lock( + name = "requirements", + srcs = ["requirements.in"], + out = "requirements.lock.txt", + args = [ + "--emit-index-url", + "--emit-find-links", + "--index-strategy=unsafe-best-match", + "--upgrade", + ], + python_version = "3.11", + tags = ["manual"], ) zig_library( name = "neuron", - data = select({ - "//runtimes:neuron.enabled": [ - ":libneuronxla", - ], - "//conditions:default": [], - }), import_name = "runtimes/neuron", main = "neuron.zig", visibility = ["//visibility:public"], @@ -124,21 +118,14 @@ zig_library( "//pjrt", ] + select({ "//runtimes:neuron.enabled": [ - ":libneuronxla_pyenv", - ":libpjrt_neuron", + ":libnrt_headers", ":libpython", + ":zmlxneuron", "//async", "//stdx", + "@libpjrt_neuron", "@rules_zig//zig/runfiles", ], "//conditions:default": [":empty"], }), ) - -filegroup( - name = "layers", - srcs = [ - "@apt_neuron//bash/amd64", - ], - visibility = ["//visibility:public"], -) diff --git a/runtimes/neuron/empty.py b/runtimes/neuron/empty.py deleted file mode 100644 index e69de29b..00000000 diff --git a/runtimes/neuron/libneuronxla.zig b/runtimes/neuron/libneuronxla.zig new file mode 100644 index 00000000..ae5710ee --- /dev/null +++ b/runtimes/neuron/libneuronxla.zig @@ -0,0 +1,249 @@ +const std = @import("std"); + +const c = @import("c"); +const stdx = @import("stdx"); +const upb = @import("upb"); + +const log = std.log.scoped(.@"zml/runtimes/neuron/libneuronxla"); + +pub fn makeTempDir(buf: []u8, prefix: []const u8) ![]const u8 { + const tmp_dir = std.posix.getenv("TMPDIR") orelse "/tmp"; + const ret = try std.fmt.bufPrint(buf, "{s}{s}{s}{d}", .{ + tmp_dir, + std.fs.path.sep_str_posix, + prefix, + std.time.microTimestamp(), + }); + try std.fs.makeDirAbsolute(ret); + return ret; +} + +var module_def: c.PyModuleDef = .{ + .m_base = .{}, + .m_name = "libneuronxla", + .m_size = 0, + .m_methods = @constCast(&[_]c.PyMethodDef{ + .{ + .ml_name = "hook", + .ml_meth = @ptrCast(&hook), + .ml_flags = c.METH_NOARGS, + .ml_doc = "Return a greeting from Zig.", + }, + .{ + .ml_name = "neuronx_cc", + .ml_meth = @ptrCast(&neuronx_cc), + .ml_flags = c.METH_FASTCALL, + .ml_doc = "Return a greeting from Zig.", + }, + .{}, + }), + .m_slots = @constCast(&[_]c.PyModuleDef_Slot{ + .{ .slot = c.Py_mod_exec, .value = @constCast(@ptrCast(&module_exec)) }, + .{}, + }), + .m_traverse = null, + .m_clear = null, + .m_free = null, +}; + +fn module_exec(module: ?*c.PyObject) callconv(.c) c_int { + _ = module; + return 0; +} + +fn hook(self: ?*c.PyObject, args: ?*c.PyObject) callconv(.c) ?*c.PyObject { + _ = self; + _ = args; + const none = c.Py_None(); + defer c.Py_IncRef(none); + return none; +} + +pub fn PyBytes_AsStringAndSize(object: *c.PyObject) []u8 { + var buf: [*c]u8 = undefined; + var len: c.Py_ssize_t = undefined; + _ = c.PyBytes_AsStringAndSize(object, &buf, &len); + return buf[0..@intCast(len)]; +} + +fn wrapNeffAsCustomCall(allocator: std.mem.Allocator, hlo_code: []const u8, neff_file_path: []const u8) ![]const u8 { + var upb_alloc: upb.Allocator = .init(allocator); + const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); + + const hlo_module = try upb.parse(c.xla_HloModuleProto, upb_arena, hlo_code); + + const entry = blk: { + var size: usize = undefined; + const computations = c.xla_HloModuleProto_mutable_computations(hlo_module, &size)[0..size]; + for (computations) |comp| { + if (c.xla_HloComputationProto_id(comp) == c.xla_HloModuleProto_entry_computation_id(hlo_module)) { + break :blk comp; + } + } else return error.ComputationNotFound; + }; + + const entry_instructions = blk: { + var size: usize = undefined; + break :blk c.xla_HloComputationProto_instructions(entry, &size)[0..size]; + }; + c.xla_HloComputationProto_clear_instructions(entry); + + const fused_root = blk: { + for (entry_instructions) |instruction| { + if (c.xla_HloInstructionProto_id(instruction) == c.xla_HloComputationProto_root_id(entry)) { + break :blk try upb.shallowClone(c.xla_HloInstructionProto, upb_arena, instruction); + } + } else return error.ComputationNotFound; + }; + + c.xla_HloInstructionProto_set_opcode(fused_root, upb.stringView("custom-call")); + c.xla_HloInstructionProto_set_custom_call_target(fused_root, upb.stringView("AwsNeuronNeff")); + c.xla_HloInstructionProto_set_backend_config(fused_root, blk: { + const neff_file = try std.fs.openFileAbsolute(neff_file_path, .{}); + defer neff_file.close(); + const stat = try neff_file.stat(); + const neff_buf = try allocator.alloc(u8, stat.size); + const size = try neff_file.readAll(neff_buf); + break :blk upb.stringView(neff_buf[0..size]); + }); + + const parameters_len = blk: { + var size: usize = undefined; + _ = c.xla_ProgramShapeProto_parameters( + c.xla_HloComputationProto_program_shape(entry), + &size, + ); + break :blk size; + }; + + { + var operand_ids: std.ArrayListUnmanaged(i64) = .initBuffer(c.xla_HloInstructionProto_resize_operand_ids(fused_root, parameters_len + 1, upb_arena)[0 .. parameters_len + 1]); + var new_instructions: std.ArrayListUnmanaged(*const c.xla_HloInstructionProto) = .initBuffer(@ptrCast(c.xla_HloComputationProto_resize_instructions(entry, parameters_len + 1, upb_arena)[0 .. parameters_len + 1])); + for (entry_instructions) |instruction| { + if (std.mem.eql(u8, upb.slice(c.xla_HloInstructionProto_opcode(instruction)) orelse continue, "parameter")) { + const id = c.xla_HloInstructionProto_id(instruction); + operand_ids.appendAssumeCapacity(id); + new_instructions.appendAssumeCapacity(instruction); + } + } + new_instructions.appendAssumeCapacity(fused_root); + } + + { + const fa = c.xla_HloInstructionProto_mutable_frontend_attributes(fused_root, upb_arena); + const map = c._xla_FrontendAttributes_map_mutable_upb_map(fa, upb_arena); + _ = c.upb_Map_Set( + map, + .{ .str_val = upb.stringView("valid_inputs") }, + .{ .str_val = blk: { + const valid_inputs_value = try allocator.alloc(u8, parameters_len * 2 - 1); + for (valid_inputs_value, 0..) |*char, i| { + char.* = if (i % 2 == 0) '1' else ','; + } + break :blk upb.stringView(valid_inputs_value); + } }, + upb_arena, + ); + } + + return try upb.serialize(hlo_module, upb_arena); +} + +fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) !?*c.PyObject { + _ = self; + + var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); + defer arena.deinit(); + + const args = args_[0..@intCast(nargs_)]; + + const code = PyBytes_AsStringAndSize(args[0]); + const platform_version = PyBytes_AsStringAndSize(args[2]); + + const target = std.StaticStringMap([]const u8).initComptime(.{ + .{ "1.0", "inf1" }, + .{ "2.0", "trn1" }, + .{ "3.0", "trn2" }, + }).get(platform_version) orelse { + log.err("Unknown platform version: {s}\n", .{platform_version}); + return error.UnknownPlatformVersion; + }; + + var tmp_dir_buf: [std.fs.max_path_bytes]u8 = undefined; + const tmp_dir = try makeTempDir(&tmp_dir_buf, "zml-neuronxcc-"); + defer std.fs.deleteTreeAbsolute(tmp_dir) catch |err| { + log.err("Error deleting temporary directory {s}: {}\n", .{ tmp_dir, err }); + }; + + const code_file = try std.fs.path.join(arena.allocator(), &.{ tmp_dir, "file.code" }); + { + const file = try std.fs.cwd().createFile(code_file, .{ .truncate = true }); + defer file.close(); + try file.writeAll(code); + } + + const neff_file = try std.fs.path.join(arena.allocator(), &.{ tmp_dir, "file.neff" }); + + var neuronx_cc_buf: [std.fs.max_path_bytes]u8 = undefined; + var child = std.process.Child.init(&.{ + try stdx.fs.path.bufJoin(&neuronx_cc_buf, &.{ + stdx.fs.selfSharedObjectDirPath(), + "..", + "bin", + "neuronx-cc", + }), + "compile", + "--framework=XLA", + "--target", + target, + "--verbose=info", + "--enable-internal-neff-wrapper", + "--output", + neff_file, + "--optlevel=1", + // generic is the default, but it fails on transformers, force it + "--model-type=transformer", + // disable it, we do our own + "--auto-cast=none", + "--enable-fast-loading-neuron-binaries", + code_file, + }, arena.allocator()); + child.stdin_behavior = .Ignore; + child.stdout_behavior = .Inherit; + child.stderr_behavior = .Inherit; + child.cwd = tmp_dir; + _ = try child.spawnAndWait(); + + std.debug.print(">>>> {s}\n", .{tmp_dir}); + + const neff_hlo_bytes = wrapNeffAsCustomCall(arena.allocator(), code, neff_file) catch |err| { + log.err("Error wrapping NEFF as custom call: {}\n", .{err}); + return err; + }; + + return c.PyTuple_Pack( + 2, + c.PyLong_FromLongLong(0), + c.PyBytes_FromStringAndSize(@ptrCast(neff_hlo_bytes), @intCast(neff_hlo_bytes.len)), + ); +} + +fn neuronx_cc(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) callconv(.c) ?*c.PyObject { + return neuronx_cc_(self, args_, nargs_) catch |err| { + log.err("Error in neuronx_cc: {}\n", .{err}); + + const none = c.Py_None(); + c.Py_IncRef(none); + const tuple = c.PyTuple_New(2) orelse { + c.Py_DecRef(none); + return null; + }; + _ = c.PyTuple_SetItem(tuple, 0, c.PyLong_FromLongLong(400).?); + _ = c.PyTuple_SetItem(tuple, 1, none); + return tuple; + }; +} + +pub export fn PyInit_libneuronxla() callconv(.c) ?*c.PyObject { + return c.PyModuleDef_Init(&module_def); +} diff --git a/runtimes/neuron/libpjrt_neuron.BUILD.bazel b/runtimes/neuron/libpjrt_neuron.BUILD.bazel index 5df16d6b..01c0bede 100644 --- a/runtimes/neuron/libpjrt_neuron.BUILD.bazel +++ b/runtimes/neuron/libpjrt_neuron.BUILD.bazel @@ -4,18 +4,7 @@ load("@bazel_skylib//rules:copy_file.bzl", "copy_file") load("@bazel_skylib//rules:select_file.bzl", "select_file") load("@rules_cc//cc:cc_shared_library.bzl", "cc_shared_library") load("@zml//bazel:runfiles.bzl", "runfiles_to_default") - -cc_shared_library( - name = "zmlxneuron_so_", - shared_lib_name = "libzmlxneuron.so.0", - deps = ["@zml//runtimes/neuron:zmlxneuron_lib"], -) - -copy_file( - name = "zmlxneuron_so", - src = ":zmlxneuron_so_", - out = "lib/libzmlxneuron.so.0", -) +load("@rules_python//python:py_binary.bzl", "py_binary") runfiles_to_default( name = "libneuronxla_files", @@ -23,40 +12,33 @@ runfiles_to_default( ) select_file( - name = "libneuronpjrt_so", + name = "libneuronpjrt_so_orig", srcs = ":libneuronxla_files", subpath = "site-packages/libneuronxla/libneuronpjrt.so", ) patchelf( - name = "libneuronpjrt.patchelf", - add_needed = [ - "libpython3.11.so.1.0", - "libzmlxneuron.so.0", - "libnccom.so.2", - ], + name = "libneuronpjrt_so", set_rpath = '$ORIGIN', - shared_library = ":libneuronpjrt_so", + src = ":libneuronpjrt_so_orig", soname = "libneuronpjrt.so", ) patchelf( - name = "libpjrt_neuron_proxy.patchelf", + name = "libpjrt_neuron_so", set_rpath = '$ORIGIN', - add_needed = [ - "libz.so.1", - "libgomp.so.1", - ], - shared_library = "@zml//runtimes/neuron:libpjrt_neuron_proxy", + src = "@zml//runtimes/neuron:libpjrt_neuron", soname = "libpjrt_neuron.so", ) copy_to_directory( name = "sandbox", srcs = [ - ":zmlxneuron_so", - ":libneuronpjrt.patchelf", - ":libpjrt_neuron_proxy.patchelf", + ":libneuronpjrt_so", + ":libpjrt_neuron_so", + "@zml//runtimes/neuron:neuronx-cc_files", + "@zml//runtimes/neuron:libneuronxla", + "@rules_python//python:current_py_toolchain", "@aws-neuronx-runtime-lib//:libnrt.patchelf", "@aws-neuronx-runtime-lib//:libncfw.patchelf", "@aws-neuronx-collectives//:libnccom", @@ -64,21 +46,35 @@ copy_to_directory( "@libgomp1", ], replace_prefixes = { - "libneuronpjrt.patchelf": "lib", - "libpjrt_neuron_proxy.patchelf": "lib", + "runtimes/neuron/neuronx-cc": "bin/neuronx-cc", + "runtimes/neuron/libneuronxla.so": "site-packages/libneuronxla.so", + "libneuronpjrt_so": "lib", + "libpjrt_neuron_so": "lib", "libnrt.patchelf": "lib", "libncfw.patchelf": "lib", "lib/x86_64-linux-gnu": "lib", "usr/lib/x86_64-linux-gnu": "lib", "opt/neuron": "lib", }, - add_directory_to_runfiles = True, + exclude_srcs_patterns = [ + "**/tests/**", + "**/include/**", + "**/_solib_*/**", + ], + include_srcs_patterns = [ + "runtimes/neuron/neuronx-cc", + "**/lib**", + "lib/python*/**", + "site-packages/**", + ], + allow_overwrites = True, + add_directory_to_runfiles = False, include_external_repositories = ["**"], + hardlink = "on", ) cc_library( - name = "libpjrt_neuron", - data = [":sandbox"], + name = "libnrt_headers", deps = [ "@aws-neuronx-runtime-lib//:libnrt_headers", ], @@ -94,3 +90,9 @@ cc_library( ], visibility = ["@zml//runtimes/neuron:__subpackages__"], ) + +cc_library( + name = "libpjrt_neuron", + data = [":sandbox"], + visibility = ["@zml//runtimes/neuron:__subpackages__"], +) diff --git a/runtimes/neuron/libpjrt_neuron.zig b/runtimes/neuron/libpjrt_neuron.zig index 6485ecd0..9a13d5ed 100644 --- a/runtimes/neuron/libpjrt_neuron.zig +++ b/runtimes/neuron/libpjrt_neuron.zig @@ -1,39 +1,202 @@ const std = @import("std"); -const runfiles = @import("runfiles"); + const bazel_builtin = @import("bazel_builtin"); +const c = @import("c"); +const runfiles = @import("runfiles"); const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/runtimes/neuron"); -pub export fn GetPjrtApi() *anyopaque { - var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); - defer arena.deinit(); +fn findFreeTcpPort() !u16 { + var address = std.net.Address.initIp4(.{ 127, 0, 0, 1 }, 0); + const sockfd = try std.posix.socket( + std.posix.AF.INET, + std.posix.SOCK.STREAM, + std.posix.IPPROTO.TCP, + ); + defer std.posix.close(sockfd); + var socklen = address.getOsSockLen(); + try std.posix.bind(sockfd, &address.any, socklen); + try std.posix.getsockname(sockfd, &address.any, &socklen); + return address.getPort(); +} - var r_ = runfiles.Runfiles.create(.{ .allocator = arena.allocator() }) catch |err| { - stdx.debug.panic("Unable to find runfiles: {}", .{err}); - } orelse stdx.debug.panic("Runfiles not availeabwewefle", .{}); +pub export fn zmlxneuron_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque { + const replacements: std.StaticStringMap([:0]const u8) = .initComptime(.{ + .{ "libnccom.so", "libnccom.so.2" }, + .{ "libnrt.so", "libnrt.so.1" }, + .{ "libncfw.so", "libncfw.so.2" }, + }); - const source_repo = bazel_builtin.current_repository; - const r = r_.withSourceRepo(source_repo); + var buf: [std.fs.max_path_bytes]u8 = undefined; + const new_filename: [*c]const u8 = if (filename) |f| blk: { + const replacement = replacements.get(std.fs.path.basename(std.mem.span(f))) orelse break :blk f; + break :blk stdx.fs.path.bufJoinZ(&buf, &.{ + stdx.fs.selfSharedObjectDirPath(), + replacement, + }) catch unreachable; + } else null; - var path_buf: [std.fs.max_path_bytes]u8 = undefined; - const sandbox_path = r.rlocation("libpjrt_neuron/sandbox", &path_buf) catch |err| { - stdx.debug.panic("Failed to find sandbox path for NEURON runtime: {}", .{err}); - } orelse stdx.debug.panic("No NEURON sandbox path found", .{}); + return std.c.dlopen(new_filename, @bitCast(flags)); +} - var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; - const library = stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libneuronpjrt.so" }) catch unreachable; +extern fn setenv(name: [*:0]const u8, value: [*:0]const u8, overwrite: c_int) c_int; +fn setupNeuronEnv() !void { + var buf: [256]u8 = undefined; + _ = setenv( + "NEURON_RT_ROOT_COMM_ID", + try std.fmt.bufPrintZ(&buf, "127.0.0.1:{d}", .{try findFreeTcpPort()}), + 1, + ); + _ = setenv( + "NEURON_INTERNAL_PJRT_C_API_VERSION", + std.fmt.comptimePrint("{d}.{d}", .{ + c.PJRT_API_MAJOR, + c.PJRT_API_MINOR, + }), + 1, + ); + _ = setenv( + "NEURON_RT_STOCHASTIC_ROUNDING_EN", + "1", + 1, + ); +} - var lib: std.DynLib = blk: { - const handle = std.c.dlopen(library, .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { - stdx.debug.panic("Unable to dlopen plugin: {s}", .{library}); +fn pyStatusCheck(status: c.PyStatus) void { + if (c.PyStatus_Exception(status) != 0) { + if (c.PyStatus_IsExit(status) != 0) { + std.process.exit(@intCast(status.exitcode)); + } + c.Py_ExitStatusException(status); + } +} + +fn toPosixPathW(file_path: []const u8) error{NameTooLong}![std.posix.PATH_MAX - 1:0]c.wchar_t { + if (file_path.len >= std.posix.PATH_MAX) return error.NameTooLong; + + var path_with_null: [std.posix.PATH_MAX - 1:0]c.wchar_t = undefined; + const len = c.mbstowcs(&path_with_null, file_path.ptr, file_path.len); + path_with_null[len] = 0; + return path_with_null; +} + +fn setupPythonEnv(sandbox_path: []const u8) !void { + const Static = struct { + var py_config: c.PyConfig = undefined; + }; + + { + var preconfig: c.PyPreConfig = undefined; + c.PyPreConfig_InitIsolatedConfig(&preconfig); + preconfig.utf8_mode = 1; + pyStatusCheck(c.Py_PreInitialize(&preconfig)); + } + + c.PyConfig_InitIsolatedConfig(&Static.py_config); + + Static.py_config.module_search_paths_set = 1; + Static.py_config.optimization_level = 2; + Static.py_config.write_bytecode = 0; + + { + var buf: [std.fs.max_path_bytes]u8 = undefined; + const home = try std.fmt.bufPrintZ(&buf, "{f}{d}.{d}", .{ + std.fs.path.fmtJoin(&.{ + sandbox_path, + "lib", + "python", + }), + c.PY_MAJOR_VERSION, + c.PY_MINOR_VERSION, + }); + pyStatusCheck(c.PyConfig_SetBytesString(&Static.py_config, &Static.py_config.home, home)); + pyStatusCheck(c.PyWideStringList_Append(&Static.py_config.module_search_paths, &try toPosixPathW(home))); + } + + { + var buf: [std.fs.max_path_bytes]u8 = undefined; + const site_packages = try stdx.fs.path.bufJoin(&buf, &.{ + sandbox_path, + "site-packages", + }); + pyStatusCheck(c.PyWideStringList_Append(&Static.py_config.module_search_paths, &try toPosixPathW(site_packages))); + } + + pyStatusCheck(c.Py_InitializeFromConfig(&Static.py_config)); + + // release the GIL + _ = c.PyEval_SaveThread(); +} + +// Duplicates a PJRT Api object while being careful about struct size differences +fn dupePjrtApi(api: *c.PJRT_Api) c.PJRT_Api { + var ret: c.PJRT_Api = undefined; + const struct_size = @min(@sizeOf(c.PJRT_Api), api.struct_size); + @memcpy( + std.mem.asBytes(&ret)[0..struct_size], + std.mem.asBytes(api)[0..struct_size], + ); + return ret; +} + +fn getPjrtApi() !*c.PJRT_Api { + const Static = struct { + var inner: *c.PJRT_Api = undefined; + var proxy: c.PJRT_Api = undefined; + }; + + var sandbox_path_buf: [std.fs.max_path_bytes]u8 = undefined; + const sandbox_path = try stdx.fs.path.bufJoin(&sandbox_path_buf, &.{ + stdx.fs.selfSharedObjectDirPath(), + "..", + }); + + try setupNeuronEnv(); + try setupPythonEnv(sandbox_path); + + Static.inner = blk: { + const GetPjrtApi_inner = GetPjrtApi_blk: { + var lib: std.DynLib = .{ + .inner = .{ + .handle = handle_blk: { + var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; + const library = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libneuronpjrt.so" }); + break :handle_blk std.c.dlopen(library, .{ .LAZY = true, .NODELETE = true }) orelse { + log.err("Unable to dlopen plugin: {?s}", .{std.mem.span(std.c.dlerror())}); + return error.DlOpenFailed; + }; + }, + }, + }; + + break :GetPjrtApi_blk lib.lookup(*const fn () callconv(.c) *c.PJRT_Api, "GetPjrtApi") orelse { + log.err("Unable to find symbol GetPjrtApi in plugin: {?s}", .{std.mem.span(std.c.dlerror())}); + return error.SymbolNotFound; + }; }; - break :blk .{ .inner = .{ .handle = handle } }; + + break :blk GetPjrtApi_inner(); }; + + Static.proxy = dupePjrtApi(Static.inner); + // Setup the API proxy functions + Static.proxy.PJRT_Plugin_Attributes = &struct { + const STRUCT_SIZE = 24; // according to the failing assertion + + fn call(args: [*c]c.PJRT_Plugin_Attributes_Args) callconv(.c) ?*c.PJRT_Error { + var new_args = args.*; + new_args.struct_size = @min(new_args.struct_size, STRUCT_SIZE); + return Static.inner.PJRT_Plugin_Attributes.?(&new_args); + } + }.call; - const sym = lib.lookup(*const fn () callconv(.C) *anyopaque, "GetPjrtApi") orelse { - stdx.debug.panic("Unable to find symbol GetPjrtApi in plugin: {s}", .{library}); + return &Static.proxy; +} + +pub export fn GetPjrtApi() ?*c.PJRT_Api { + return getPjrtApi() catch |err| { + log.err("Failed to get PJRT API: {}", .{err}); + return null; }; - - return sym(); } diff --git a/runtimes/neuron/neuron.bzl b/runtimes/neuron/neuron.bzl index a277b357..e03e8ca6 100644 --- a/runtimes/neuron/neuron.bzl +++ b/runtimes/neuron/neuron.bzl @@ -32,7 +32,7 @@ _NEURON_PACKAGES = { packages.patchelf( name = "libnrt.patchelf", src = "lib/libnrt.so.1", - set_rpath = '$ORIGIN', + set_rpath = "$ORIGIN", add_needed = [ # readelf -d ./opt/aws/neuron/libl/libncfw.so "libncfw.so.2", diff --git a/runtimes/neuron/neuron.zig b/runtimes/neuron/neuron.zig index fc973424..09d6a218 100644 --- a/runtimes/neuron/neuron.zig +++ b/runtimes/neuron/neuron.zig @@ -1,10 +1,9 @@ -const builtin = @import("builtin"); const std = @import("std"); +const builtin = @import("builtin"); const asynk = @import("async"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); -const libneuronxla_pyenv = @import("libneuronxla_pyenv"); const pjrt = @import("pjrt"); const runfiles = @import("runfiles"); const stdx = @import("stdx"); @@ -26,99 +25,12 @@ fn isRunningOnEC2() !bool { var f = try asynk.File.open("/sys/devices/virtual/dmi/id/sys_vendor", .{ .mode = .read_only }); defer f.close() catch {}; - var buf = [_]u8{0} ** AmazonEC2.len; + var buf: [AmazonEC2.len]u8 = undefined; _ = try f.reader().readAll(&buf); return std.mem.eql(u8, &buf, AmazonEC2); } -fn toWchar(str: []const u8, out: [:0]c.wchar_t) [:0]c.wchar_t { - const len = c.mbstowcs(out.ptr, str.ptr, str.len); - out[len] = 0; - return out[0..len :0]; -} - -fn pyErrorOrExit(status: c.PyStatus) void { - if (c.PyStatus_Exception(status) != 0) { - if (c.PyStatus_IsExit(status) != 0) { - std.process.exit(@intCast(status.exitcode)); - } - c.Py_ExitStatusException(status); - } -} - -fn initialize(allocator: std.mem.Allocator, r_: *runfiles.Runfiles) !void { - { - var preconfig: c.PyPreConfig = undefined; - c.PyPreConfig_InitIsolatedConfig(&preconfig); - preconfig.utf8_mode = 1; - pyErrorOrExit(c.Py_PreInitialize(&preconfig)); - } - - var config: c.PyConfig = undefined; - c.PyConfig_InitIsolatedConfig(&config); - defer c.PyConfig_Clear(&config); - - const r = r_.withSourceRepo(bazel_builtin.current_repository); - - var buf: [std.fs.max_path_bytes]u8 = undefined; - var wbuf: [std.fs.max_path_bytes:0]c.wchar_t = undefined; - - { - const path = (try r.rlocation(libneuronxla_pyenv.home, &buf)).?; - const wpath = toWchar(std.fs.path.dirname(path).?, &wbuf); - pyErrorOrExit(c.PyConfig_SetString(&config, &config.home, wpath.ptr)); - } - - { - config.module_search_paths_set = 1; - for (libneuronxla_pyenv.modules) |module| { - const path = (try r.rlocation(module, &buf)).?; - const wline = toWchar(std.fs.path.dirname(path).?, &wbuf); - pyErrorOrExit(c.PyWideStringList_Append(&config.module_search_paths, wline.ptr)); - } - } - - { - const neuronx_cc = (try r.rlocation("zml/runtimes/neuron/neuronx-cc/neuronx-cc", &buf)).?; - const neuronx_cc_path = std.fs.path.dirname(neuronx_cc).?; - const path = std.posix.getenv("PATH") orelse ""; - const new_path = try std.fmt.allocPrintZ(allocator, "{s}:{s}", .{ neuronx_cc_path, path }); - _ = c.setenv("PATH", new_path.ptr, 1); - } - - pyErrorOrExit(c.Py_InitializeFromConfig(&config)); - - // release the GIL - _ = c.PyEval_SaveThread(); -} - -fn comptimeStrJoin(comptime separator: [:0]const u8, comptime slices: []const [:0]const u8) [:0]const u8 { - comptime var ret = slices[0]; - inline for (slices[1..]) |slice| { - ret = ret ++ separator ++ slice; - } - return ret; -} - -pub fn setNeuronCCFlags() void { - // See neuronxcc reference: - // https://awsdocs-neuron.readthedocs-hosted.com/en/latest/compiler/neuronx-cc/api-reference-guide/neuron-compiler-cli-reference-guide.html#neuron-compiler-cli-reference-guide - _ = c.setenv("NEURON_CC_FLAGS", comptimeStrJoin(" ", &.{ - // 30% faster, no visible speed difference on llama - "--optlevel=1", - // generic is the default, but it fails on transformers, force it - "--model-type=transformer", - // disable it, we do our own - "--auto-cast=none", - "--enable-fast-loading-neuron-binaries", - }), 1); - - // Enable stochastic rounding - // https://awsdocs-neuron.readthedocs-hosted.com/en/latest/general/arch/neuron-features/rounding-modes.html - _ = c.setenv("NEURON_RT_STOCHASTIC_ROUNDING_EN", "1", 1); -} - pub fn load() !*const pjrt.Api { if (comptime !isEnabled()) { return error.Unavailable; @@ -143,15 +55,12 @@ pub fn load() !*const pjrt.Api { const source_repo = bazel_builtin.current_repository; const r = r_.withSourceRepo(source_repo); - var path_buf: [std.fs.max_path_bytes]u8 = undefined; - const sandbox_path = try r.rlocation("libpjrt_neuron/sandbox", &path_buf) orelse { + var sandbox_path_buf: [std.fs.max_path_bytes]u8 = undefined; + const sandbox_path = try r.rlocation("libpjrt_neuron/sandbox", &sandbox_path_buf) orelse { log.err("Failed to find sandbox path for NEURON runtime", .{}); return error.FileNotFound; }; - setNeuronCCFlags(); - try initialize(arena.allocator(), &r_); - return blk: { var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; const path = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_neuron.so" }); diff --git a/runtimes/neuron/neuronx-cc.zig b/runtimes/neuron/neuronx-cc.zig new file mode 100644 index 00000000..828a1d42 --- /dev/null +++ b/runtimes/neuron/neuronx-cc.zig @@ -0,0 +1,89 @@ +const std = @import("std"); + +const c = @import("c"); +const runfiles = @import("runfiles"); + +fn pyStatusCheck(status: c.PyStatus) void { + if (c.PyStatus_Exception(status) != 0) { + if (c.PyStatus_IsExit(status) != 0) { + std.process.exit(@intCast(status.exitcode)); + } + c.Py_ExitStatusException(status); + } +} + +pub fn toPosixPathW(file_path: []const u8) error{NameTooLong}![std.posix.PATH_MAX - 1:0]c.wchar_t { + if (file_path.len >= std.posix.PATH_MAX) return error.NameTooLong; + + var path_with_null: [std.posix.PATH_MAX - 1:0]c.wchar_t = undefined; + const len = c.mbstowcs(&path_with_null, file_path.ptr, file_path.len); + path_with_null[len] = 0; + return path_with_null; +} + +pub fn main() !void { + var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); + defer arena.deinit(); + + { + var preconfig: c.PyPreConfig = undefined; + c.PyPreConfig_InitIsolatedConfig(&preconfig); + preconfig.utf8_mode = 1; + pyStatusCheck(c.Py_PreInitialize(&preconfig)); + } + + var config: c.PyConfig = undefined; + c.PyConfig_InitIsolatedConfig(&config); + defer c.PyConfig_Clear(&config); + + config.module_search_paths_set = 1; + config.optimization_level = 2; + config.write_bytecode = 0; + + _ = c.PyConfig_SetBytesArgv(&config, @intCast(std.os.argv.len), @ptrCast(std.os.argv)); + + var self_exe_dir_buf: [std.fs.max_path_bytes]u8 = undefined; + const self_exe_dir = try std.fs.selfExeDirPath(&self_exe_dir_buf); + + { + var buf: [std.fs.max_path_bytes]u8 = undefined; + const home = try std.fmt.bufPrintZ(&buf, "{f}{d}.{d}", .{ + std.fs.path.fmtJoin(&.{ + self_exe_dir, + "..", + "lib", + "python", + }), + c.PY_MAJOR_VERSION, + c.PY_MINOR_VERSION, + }); + pyStatusCheck(c.PyConfig_SetBytesString(&config, &config.home, home)); + pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(home))); + } + + { + var buf: [std.fs.max_path_bytes]u8 = undefined; + const site_packages = try std.fmt.bufPrint(&buf, "{f}", .{ + std.fs.path.fmtJoin(&.{ + self_exe_dir, + "..", + "site-packages", + }), + }); + pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(site_packages))); + } + + pyStatusCheck(c.Py_InitializeFromConfig(&config)); + defer c.Py_Finalize(); + + const neuronxcc_main = blk: { + const module = c.PyImport_ImportModule("neuronxcc.driver.CommandDriver"); + std.debug.print(">>> MODULE: {any}\n", .{module}); + defer c.Py_DecRef(module); + break :blk c.PyObject_GetAttrString(module, "main"); + }; + defer c.Py_DecRef(neuronxcc_main); + + const result = c.PyObject_CallNoArgs(neuronxcc_main); + defer c.Py_DecRef(result); +} diff --git a/runtimes/neuron/libpjrt_neuron.h b/runtimes/neuron/nrt.h similarity index 100% rename from runtimes/neuron/libpjrt_neuron.h rename to runtimes/neuron/nrt.h diff --git a/runtimes/neuron/pyenv.bzl b/runtimes/neuron/pyenv.bzl deleted file mode 100644 index 11c27dbd..00000000 --- a/runtimes/neuron/pyenv.bzl +++ /dev/null @@ -1,41 +0,0 @@ -load("@bazel_skylib//lib:paths.bzl", "paths") -load("@rules_python//python:defs.bzl", "PyInfo") - -def runfile_path(ctx, runfile): - return paths.normalize(ctx.workspace_name + "/" + runfile.short_path) - -def _pyenv_zig_impl(ctx): - py_toolchain = ctx.toolchains["@rules_python//python:toolchain_type"].py3_runtime - content = "pub const home: [:0]const u8 = {};\n".format(repr(runfile_path(ctx, py_toolchain.interpreter))) - - modules = [] - for file in py_toolchain.files.to_list(): - if file.basename == "__hello__.py": - modules.append(repr(runfile_path(ctx, file))) - - imports = depset([], transitive = [ - dep[PyInfo].imports - for dep in ctx.attr.deps - ]) - modules.extend([ - repr("{}/__init__.py".format(imp)) - for imp in imports.to_list() - ]) - content += "pub const modules: []const [:0]const u8 = &.{{ {} }};\n".format(", ".join(modules)) - - f = ctx.actions.declare_file("{}.zig".format(ctx.label.name)) - ctx.actions.write(f, content) - - return [ - DefaultInfo( - files = depset([f]), - ), - ] - -pyenv_zig = rule( - implementation = _pyenv_zig_impl, - attrs = { - "deps": attr.label_list(providers = [PyInfo]), - }, - toolchains = ["@rules_python//python:toolchain_type"], -) diff --git a/runtimes/neuron/requirements.lock.txt b/runtimes/neuron/requirements.lock.txt index c5994c0b..9332b2f4 100644 --- a/runtimes/neuron/requirements.lock.txt +++ b/runtimes/neuron/requirements.lock.txt @@ -1,121 +1,108 @@ # This file was autogenerated by uv via the following command: -# bazel run @@//runtimes/neuron:update_requirements +# bazel run //runtimes/neuron:requirements.update --index-url https://pypi.org/simple --extra-index-url https://pip.repos.neuron.amazonaws.com --find-links https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html -boto3==1.39.9 \ - --hash=sha256:5bc85e9fdec4e21ef5ca2c22b4d51a3e32b53f3da36ce51f5a3ea4dbde07b132 \ - --hash=sha256:e3d3a6b617e1575e7ec854c820a882ab2e189a0421e74dc0dca2c9e13d4370a5 +boto3==1.40.14 \ + --hash=sha256:ab5798a03582d09c0de132d080c9aee53d5647b6461261a5b7621170ec80d92b \ + --hash=sha256:d1d9998fc2b9619fc796c859d263ac81793d783e79331be62931b353dd1b68b9 # via libneuronxla -botocore==1.39.9 \ - --hash=sha256:02f141c2849e4589a79feea245ce4ecc478d48b7865572445af8aae3b041772d \ - --hash=sha256:a9691cbe03a3bc8b2720b3c36e5c5a2eecace6acd72bfb1107f00e75edaec4f3 +botocore==1.40.14 \ + --hash=sha256:06c852be83543c8d45e18a530abcad31659db8b16bcb66fd371c930d38017e7c \ + --hash=sha256:1810494c8c4190f20f9e17f2da4f7ee91eda863f429c6d690ea17ec41a8f83c4 # via # boto3 # libneuronxla # s3transfer -certifi==2025.7.14 \ - --hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 \ - --hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995 +certifi==2025.8.3 \ + --hash=sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407 \ + --hash=sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5 # via requests -charset-normalizer==3.4.2 \ - --hash=sha256:005fa3432484527f9732ebd315da8da8001593e2cf46a3d817669f062c3d9ed4 \ - --hash=sha256:046595208aae0120559a67693ecc65dd75d46f7bf687f159127046628178dc45 \ - --hash=sha256:0c29de6a1a95f24b9a1aa7aefd27d2487263f00dfd55a77719b530788f75cff7 \ - --hash=sha256:0c8c57f84ccfc871a48a47321cfa49ae1df56cd1d965a09abe84066f6853b9c0 \ - --hash=sha256:0f5d9ed7f254402c9e7d35d2f5972c9bbea9040e99cd2861bd77dc68263277c7 \ - --hash=sha256:18dd2e350387c87dabe711b86f83c9c78af772c748904d372ade190b5c7c9d4d \ - --hash=sha256:1b1bde144d98e446b056ef98e59c256e9294f6b74d7af6846bf5ffdafd687a7d \ - --hash=sha256:1c95a1e2902a8b722868587c0e1184ad5c55631de5afc0eb96bc4b0d738092c0 \ - --hash=sha256:1cad5f45b3146325bb38d6855642f6fd609c3f7cad4dbaf75549bf3b904d3184 \ - --hash=sha256:21b2899062867b0e1fde9b724f8aecb1af14f2778d69aacd1a5a1853a597a5db \ - --hash=sha256:24498ba8ed6c2e0b56d4acbf83f2d989720a93b41d712ebd4f4979660db4417b \ - --hash=sha256:25a23ea5c7edc53e0f29bae2c44fcb5a1aa10591aae107f2a2b2583a9c5cbc64 \ - --hash=sha256:289200a18fa698949d2b39c671c2cc7a24d44096784e76614899a7ccf2574b7b \ - --hash=sha256:28a1005facc94196e1fb3e82a3d442a9d9110b8434fc1ded7a24a2983c9888d8 \ - --hash=sha256:32fc0341d72e0f73f80acb0a2c94216bd704f4f0bce10aedea38f30502b271ff \ - --hash=sha256:36b31da18b8890a76ec181c3cf44326bf2c48e36d393ca1b72b3f484113ea344 \ - --hash=sha256:3c21d4fca343c805a52c0c78edc01e3477f6dd1ad7c47653241cf2a206d4fc58 \ - --hash=sha256:3fddb7e2c84ac87ac3a947cb4e66d143ca5863ef48e4a5ecb83bd48619e4634e \ - --hash=sha256:43e0933a0eff183ee85833f341ec567c0980dae57c464d8a508e1b2ceb336471 \ - --hash=sha256:4a476b06fbcf359ad25d34a057b7219281286ae2477cc5ff5e3f70a246971148 \ - --hash=sha256:4e594135de17ab3866138f496755f302b72157d115086d100c3f19370839dd3a \ - --hash=sha256:50bf98d5e563b83cc29471fa114366e6806bc06bc7a25fd59641e41445327836 \ - --hash=sha256:5a9979887252a82fefd3d3ed2a8e3b937a7a809f65dcb1e068b090e165bbe99e \ - --hash=sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63 \ - --hash=sha256:5bf4545e3b962767e5c06fe1738f951f77d27967cb2caa64c28be7c4563e162c \ - --hash=sha256:6333b3aa5a12c26b2a4d4e7335a28f1475e0e5e17d69d55141ee3cab736f66d1 \ - --hash=sha256:65c981bdbd3f57670af8b59777cbfae75364b483fa8a9f420f08094531d54a01 \ - --hash=sha256:68a328e5f55ec37c57f19ebb1fdc56a248db2e3e9ad769919a58672958e8f366 \ - --hash=sha256:6a0289e4589e8bdfef02a80478f1dfcb14f0ab696b5a00e1f4b8a14a307a3c58 \ - --hash=sha256:6b66f92b17849b85cad91259efc341dce9c1af48e2173bf38a85c6329f1033e5 \ - --hash=sha256:6c9379d65defcab82d07b2a9dfbfc2e95bc8fe0ebb1b176a3190230a3ef0e07c \ - --hash=sha256:6fc1f5b51fa4cecaa18f2bd7a003f3dd039dd615cd69a2afd6d3b19aed6775f2 \ - --hash=sha256:70f7172939fdf8790425ba31915bfbe8335030f05b9913d7ae00a87d4395620a \ - --hash=sha256:721c76e84fe669be19c5791da68232ca2e05ba5185575086e384352e2c309597 \ - --hash=sha256:7222ffd5e4de8e57e03ce2cef95a4c43c98fcb72ad86909abdfc2c17d227fc1b \ - --hash=sha256:75d10d37a47afee94919c4fab4c22b9bc2a8bf7d4f46f87363bcf0573f3ff4f5 \ - --hash=sha256:76af085e67e56c8816c3ccf256ebd136def2ed9654525348cfa744b6802b69eb \ - --hash=sha256:770cab594ecf99ae64c236bc9ee3439c3f46be49796e265ce0cc8bc17b10294f \ - --hash=sha256:7a6ab32f7210554a96cd9e33abe3ddd86732beeafc7a28e9955cdf22ffadbab0 \ - --hash=sha256:7c48ed483eb946e6c04ccbe02c6b4d1d48e51944b6db70f697e089c193404941 \ - --hash=sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0 \ - --hash=sha256:8075c35cd58273fee266c58c0c9b670947c19df5fb98e7b66710e04ad4e9ff86 \ - --hash=sha256:8272b73e1c5603666618805fe821edba66892e2870058c94c53147602eab29c7 \ - --hash=sha256:82d8fd25b7f4675d0c47cf95b594d4e7b158aca33b76aa63d07186e13c0e0ab7 \ - --hash=sha256:844da2b5728b5ce0e32d863af26f32b5ce61bc4273a9c720a9f3aa9df73b1455 \ - --hash=sha256:8755483f3c00d6c9a77f490c17e6ab0c8729e39e6390328e42521ef175380ae6 \ - --hash=sha256:915f3849a011c1f593ab99092f3cecfcb4d65d8feb4a64cf1bf2d22074dc0ec4 \ - --hash=sha256:926ca93accd5d36ccdabd803392ddc3e03e6d4cd1cf17deff3b989ab8e9dbcf0 \ - --hash=sha256:982bb1e8b4ffda883b3d0a521e23abcd6fd17418f6d2c4118d257a10199c0ce3 \ - --hash=sha256:98f862da73774290f251b9df8d11161b6cf25b599a66baf087c1ffe340e9bfd1 \ - --hash=sha256:9cbfacf36cb0ec2897ce0ebc5d08ca44213af24265bd56eca54bee7923c48fd6 \ - --hash=sha256:a370b3e078e418187da8c3674eddb9d983ec09445c99a3a263c2011993522981 \ - --hash=sha256:a955b438e62efdf7e0b7b52a64dc5c3396e2634baa62471768a64bc2adb73d5c \ - --hash=sha256:aa6af9e7d59f9c12b33ae4e9450619cf2488e2bbe9b44030905877f0b2324980 \ - --hash=sha256:aa88ca0b1932e93f2d961bf3addbb2db902198dca337d88c89e1559e066e7645 \ - --hash=sha256:aaeeb6a479c7667fbe1099af9617c83aaca22182d6cf8c53966491a0f1b7ffb7 \ - --hash=sha256:aaf27faa992bfee0264dc1f03f4c75e9fcdda66a519db6b957a3f826e285cf12 \ - --hash=sha256:b2680962a4848b3c4f155dc2ee64505a9c57186d0d56b43123b17ca3de18f0fa \ - --hash=sha256:b2d318c11350e10662026ad0eb71bb51c7812fc8590825304ae0bdd4ac283acd \ - --hash=sha256:b33de11b92e9f75a2b545d6e9b6f37e398d86c3e9e9653c4864eb7e89c5773ef \ - --hash=sha256:b3daeac64d5b371dea99714f08ffc2c208522ec6b06fbc7866a450dd446f5c0f \ - --hash=sha256:be1e352acbe3c78727a16a455126d9ff83ea2dfdcbc83148d2982305a04714c2 \ - --hash=sha256:bee093bf902e1d8fc0ac143c88902c3dfc8941f7ea1d6a8dd2bcb786d33db03d \ - --hash=sha256:c72fbbe68c6f32f251bdc08b8611c7b3060612236e960ef848e0a517ddbe76c5 \ - --hash=sha256:c9e36a97bee9b86ef9a1cf7bb96747eb7a15c2f22bdb5b516434b00f2a599f02 \ - --hash=sha256:cddf7bd982eaa998934a91f69d182aec997c6c468898efe6679af88283b498d3 \ - --hash=sha256:cf713fe9a71ef6fd5adf7a79670135081cd4431c2943864757f0fa3a65b1fafd \ - --hash=sha256:d11b54acf878eef558599658b0ffca78138c8c3655cf4f3a4a673c437e67732e \ - --hash=sha256:d41c4d287cfc69060fa91cae9683eacffad989f1a10811995fa309df656ec214 \ - --hash=sha256:d524ba3f1581b35c03cb42beebab4a13e6cdad7b36246bd22541fa585a56cccd \ - --hash=sha256:daac4765328a919a805fa5e2720f3e94767abd632ae410a9062dff5412bae65a \ - --hash=sha256:db4c7bf0e07fc3b7d89ac2a5880a6a8062056801b83ff56d8464b70f65482b6c \ - --hash=sha256:dc7039885fa1baf9be153a0626e337aa7ec8bf96b0128605fb0d77788ddc1681 \ - --hash=sha256:dccab8d5fa1ef9bfba0590ecf4d46df048d18ffe3eec01eeb73a42e0d9e7a8ba \ - --hash=sha256:dedb8adb91d11846ee08bec4c8236c8549ac721c245678282dcb06b221aab59f \ - --hash=sha256:e45ba65510e2647721e35323d6ef54c7974959f6081b58d4ef5d87c60c84919a \ - --hash=sha256:e53efc7c7cee4c1e70661e2e112ca46a575f90ed9ae3fef200f2a25e954f4b28 \ - --hash=sha256:e635b87f01ebc977342e2697d05b56632f5f879a4f15955dfe8cef2448b51691 \ - --hash=sha256:e70e990b2137b29dc5564715de1e12701815dacc1d056308e2b17e9095372a82 \ - --hash=sha256:e8082b26888e2f8b36a042a58307d5b917ef2b1cacab921ad3323ef91901c71a \ - --hash=sha256:e8323a9b031aa0393768b87f04b4164a40037fb2a3c11ac06a03ffecd3618027 \ - --hash=sha256:e92fca20c46e9f5e1bb485887d074918b13543b1c2a1185e69bb8d17ab6236a7 \ - --hash=sha256:eb30abc20df9ab0814b5a2524f23d75dcf83cde762c161917a2b4b7b55b1e518 \ - --hash=sha256:eba9904b0f38a143592d9fc0e19e2df0fa2e41c3c3745554761c5f6447eedabf \ - --hash=sha256:ef8de666d6179b009dce7bcb2ad4c4a779f113f12caf8dc77f0162c29d20490b \ - --hash=sha256:efd387a49825780ff861998cd959767800d54f8308936b21025326de4b5a42b9 \ - --hash=sha256:f0aa37f3c979cf2546b73e8222bbfa3dc07a641585340179d768068e3455e544 \ - --hash=sha256:f4074c5a429281bf056ddd4c5d3b740ebca4d43ffffe2ef4bf4d2d05114299da \ - --hash=sha256:f69a27e45c43520f5487f27627059b64aaf160415589230992cec34c5e18a509 \ - --hash=sha256:fb707f3e15060adf5b7ada797624a6c6e0138e2a26baa089df64c68ee98e040f \ - --hash=sha256:fcbe676a55d7445b22c10967bceaaf0ee69407fbe0ece4d032b6eb8d4565982a \ - --hash=sha256:fdb20a30fe1175ecabed17cbf7812f7b804b8a315a25f24678bcdf120a90077f +charset-normalizer==3.4.3 \ + --hash=sha256:00237675befef519d9af72169d8604a067d92755e84fe76492fef5441db05b91 \ + --hash=sha256:02425242e96bcf29a49711b0ca9f37e451da7c70562bc10e8ed992a5a7a25cc0 \ + --hash=sha256:027b776c26d38b7f15b26a5da1044f376455fb3766df8fc38563b4efbc515154 \ + --hash=sha256:07a0eae9e2787b586e129fdcbe1af6997f8d0e5abaa0bc98c0e20e124d67e601 \ + --hash=sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884 \ + --hash=sha256:0e78314bdc32fa80696f72fa16dc61168fda4d6a0c014e0380f9d02f0e5d8a07 \ + --hash=sha256:0f2be7e0cf7754b9a30eb01f4295cc3d4358a479843b31f328afd210e2c7598c \ + --hash=sha256:13faeacfe61784e2559e690fc53fa4c5ae97c6fcedb8eb6fb8d0a15b475d2c64 \ + --hash=sha256:14c2a87c65b351109f6abfc424cab3927b3bdece6f706e4d12faaf3d52ee5efe \ + --hash=sha256:1606f4a55c0fd363d754049cdf400175ee96c992b1f8018b993941f221221c5f \ + --hash=sha256:16a8770207946ac75703458e2c743631c79c59c5890c80011d536248f8eaa432 \ + --hash=sha256:18343b2d246dc6761a249ba1fb13f9ee9a2bcd95decc767319506056ea4ad4dc \ + --hash=sha256:18b97b8404387b96cdbd30ad660f6407799126d26a39ca65729162fd810a99aa \ + --hash=sha256:1bb60174149316da1c35fa5233681f7c0f9f514509b8e399ab70fea5f17e45c9 \ + --hash=sha256:1e8ac75d72fa3775e0b7cb7e4629cec13b7514d928d15ef8ea06bca03ef01cae \ + --hash=sha256:1ef99f0456d3d46a50945c98de1774da86f8e992ab5c77865ea8b8195341fc19 \ + --hash=sha256:2001a39612b241dae17b4687898843f254f8748b796a2e16f1051a17078d991d \ + --hash=sha256:23b6b24d74478dc833444cbd927c338349d6ae852ba53a0d02a2de1fce45b96e \ + --hash=sha256:252098c8c7a873e17dd696ed98bbe91dbacd571da4b87df3736768efa7a792e4 \ + --hash=sha256:257f26fed7d7ff59921b78244f3cd93ed2af1800ff048c33f624c87475819dd7 \ + --hash=sha256:2c322db9c8c89009a990ef07c3bcc9f011a3269bc06782f916cd3d9eed7c9312 \ + --hash=sha256:30a96e1e1f865f78b030d65241c1ee850cdf422d869e9028e2fc1d5e4db73b92 \ + --hash=sha256:30d006f98569de3459c2fc1f2acde170b7b2bd265dc1943e87e1a4efe1b67c31 \ + --hash=sha256:31a9a6f775f9bcd865d88ee350f0ffb0e25936a7f930ca98995c05abf1faf21c \ + --hash=sha256:320e8e66157cc4e247d9ddca8e21f427efc7a04bbd0ac8a9faf56583fa543f9f \ + --hash=sha256:34a7f768e3f985abdb42841e20e17b330ad3aaf4bb7e7aeeb73db2e70f077b99 \ + --hash=sha256:3653fad4fe3ed447a596ae8638b437f827234f01a8cd801842e43f3d0a6b281b \ + --hash=sha256:3cd35b7e8aedeb9e34c41385fda4f73ba609e561faedfae0a9e75e44ac558a15 \ + --hash=sha256:3cfb2aad70f2c6debfbcb717f23b7eb55febc0bb23dcffc0f076009da10c6392 \ + --hash=sha256:416175faf02e4b0810f1f38bcb54682878a4af94059a1cd63b8747244420801f \ + --hash=sha256:41d1fc408ff5fdfb910200ec0e74abc40387bccb3252f3f27c0676731df2b2c8 \ + --hash=sha256:42e5088973e56e31e4fa58eb6bd709e42fc03799c11c42929592889a2e54c491 \ + --hash=sha256:4ca4c094de7771a98d7fbd67d9e5dbf1eb73efa4f744a730437d8a3a5cf994f0 \ + --hash=sha256:511729f456829ef86ac41ca78c63a5cb55240ed23b4b737faca0eb1abb1c41bc \ + --hash=sha256:53cd68b185d98dde4ad8990e56a58dea83a4162161b1ea9272e5c9182ce415e0 \ + --hash=sha256:585f3b2a80fbd26b048a0be90c5aae8f06605d3c92615911c3a2b03a8a3b796f \ + --hash=sha256:5b413b0b1bfd94dbf4023ad6945889f374cd24e3f62de58d6bb102c4d9ae534a \ + --hash=sha256:5d8d01eac18c423815ed4f4a2ec3b439d654e55ee4ad610e153cf02faf67ea40 \ + --hash=sha256:6aab0f181c486f973bc7262a97f5aca3ee7e1437011ef0c2ec04b5a11d16c927 \ + --hash=sha256:6cf8fd4c04756b6b60146d98cd8a77d0cdae0e1ca20329da2ac85eed779b6849 \ + --hash=sha256:6fb70de56f1859a3f71261cbe41005f56a7842cc348d3aeb26237560bfa5e0ce \ + --hash=sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14 \ + --hash=sha256:70bfc5f2c318afece2f5838ea5e4c3febada0be750fcf4775641052bbba14d05 \ + --hash=sha256:73dc19b562516fc9bcf6e5d6e596df0b4eb98d87e4f79f3ae71840e6ed21361c \ + --hash=sha256:74d77e25adda8581ffc1c720f1c81ca082921329452eba58b16233ab1842141c \ + --hash=sha256:78deba4d8f9590fe4dae384aeff04082510a709957e968753ff3c48399f6f92a \ + --hash=sha256:86df271bf921c2ee3818f0522e9a5b8092ca2ad8b065ece5d7d9d0e9f4849bcc \ + --hash=sha256:88ab34806dea0671532d3f82d82b85e8fc23d7b2dd12fa837978dad9bb392a34 \ + --hash=sha256:8999f965f922ae054125286faf9f11bc6932184b93011d138925a1773830bbe9 \ + --hash=sha256:8dcfc373f888e4fb39a7bc57e93e3b845e7f462dacc008d9749568b1c4ece096 \ + --hash=sha256:939578d9d8fd4299220161fdd76e86c6a251987476f5243e8864a7844476ba14 \ + --hash=sha256:96b2b3d1a83ad55310de8c7b4a2d04d9277d5591f40761274856635acc5fcb30 \ + --hash=sha256:a2d08ac246bb48479170408d6c19f6385fa743e7157d716e144cad849b2dd94b \ + --hash=sha256:b256ee2e749283ef3ddcff51a675ff43798d92d746d1a6e4631bf8c707d22d0b \ + --hash=sha256:b5e3b2d152e74e100a9e9573837aba24aab611d39428ded46f4e4022ea7d1942 \ + --hash=sha256:b89bc04de1d83006373429975f8ef9e7932534b8cc9ca582e4db7d20d91816db \ + --hash=sha256:bd28b817ea8c70215401f657edef3a8aa83c29d447fb0b622c35403780ba11d5 \ + --hash=sha256:c60e092517a73c632ec38e290eba714e9627abe9d301c8c8a12ec32c314a2a4b \ + --hash=sha256:c6dbd0ccdda3a2ba7c2ecd9d77b37f3b5831687d8dc1b6ca5f56a4880cc7b7ce \ + --hash=sha256:c6e490913a46fa054e03699c70019ab869e990270597018cef1d8562132c2669 \ + --hash=sha256:c6f162aabe9a91a309510d74eeb6507fab5fff92337a15acbe77753d88d9dcf0 \ + --hash=sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018 \ + --hash=sha256:cc34f233c9e71701040d772aa7490318673aa7164a0efe3172b2981218c26d93 \ + --hash=sha256:cc9370a2da1ac13f0153780040f465839e6cccb4a1e44810124b4e22483c93fe \ + --hash=sha256:ccf600859c183d70eb47e05a44cd80a4ce77394d1ac0f79dbd2dd90a69a3a049 \ + --hash=sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a \ + --hash=sha256:cf1ebb7d78e1ad8ec2a8c4732c7be2e736f6e5123a4146c5b89c9d1f585f8cef \ + --hash=sha256:d0e909868420b7049dafd3a31d45125b31143eec59235311fc4c57ea26a4acd2 \ + --hash=sha256:d22dbedd33326a4a5190dd4fe9e9e693ef12160c77382d9e87919bce54f3d4ca \ + --hash=sha256:d716a916938e03231e86e43782ca7878fb602a125a91e7acb8b5112e2e96ac16 \ + --hash=sha256:d79c198e27580c8e958906f803e63cddb77653731be08851c7df0b1a14a8fc0f \ + --hash=sha256:d95bfb53c211b57198bb91c46dd5a2d8018b3af446583aab40074bf7988401cb \ + --hash=sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1 \ + --hash=sha256:ec557499516fc90fd374bf2e32349a2887a876fbf162c160e3c01b6849eaf557 \ + --hash=sha256:fb6fecfd65564f208cbf0fba07f107fb661bcd1a7c389edbced3f7a493f70e37 \ + --hash=sha256:fb731e5deb0c7ef82d698b0f4c5bb724633ee2a489401594c5c88b02e6cb15f7 \ + --hash=sha256:fb7f67a1bfa6e40b438170ebdc8158b78dc465a5a67b6dde178a46987b244a72 \ + --hash=sha256:fd10de089bcdcd1be95a2f73dbe6254798ec1bda9f450d5828c96f93e2536b9c \ + --hash=sha256:fdabf8315679312cfa71302f9bd509ded4f2f263fb5b765cf1433b39106c3cc9 # via requests -ec2-metadata==2.14.0 \ - --hash=sha256:b2f83381722072efa28f65dc37e726c0abd3a0532f227b3fa6a2ad68460c7feb \ - --hash=sha256:fc4f9e893da1c049761221ec3d2208cce4872953b8960e1ed1f7120ff4ca3512 +ec2-metadata==2.15.0 \ + --hash=sha256:b066b268091ca40369fdd644b90a34ecbfd78225bce6f0e269f0bf9ce3ab1b75 \ + --hash=sha256:daeec6b93229623942ac61f5ccc3026d0079ea1e7c84a5b18a3fb631db85600f # via neuronx-cc idna==3.10 \ --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \ @@ -169,45 +156,56 @@ jmespath==1.0.1 \ # via # boto3 # botocore -libneuronxla==2.2.4410.0+835a67fb \ - --hash=sha256:37e5f08482ef1a9a844c2894de6d91c03400a092764e54ab67612456be439f16 +libneuronxla==2.2.8201.0+f46ac1ef \ + --hash=sha256:c5bb9d2c4ab45a786561e09919ef6a3e3b7839bb7b18f539130e8c8659a29b1e # via -r runtimes/neuron/requirements.in lockfile==0.12.2 \ --hash=sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799 \ --hash=sha256:6c3cb24f344923d30b2785d5ad75182c8ea7ac1b6171b08657258ec7429d50fa # via python-daemon -ml-dtypes==0.5.1 \ - --hash=sha256:023ce2f502efd4d6c1e0472cc58ce3640d051d40e71e27386bed33901e201327 \ - --hash=sha256:05f23447a1c20ddf4dc7c2c661aa9ed93fcb2658f1017c204d1e758714dc28a8 \ - --hash=sha256:12651420130ee7cc13059fc56dac6ad300c3af3848b802d475148c9defd27c23 \ - --hash=sha256:141b2ea2f20bb10802ddca55d91fe21231ef49715cfc971998e8f2a9838f3dbe \ - --hash=sha256:15ad0f3b0323ce96c24637a88a6f44f6713c64032f27277b069f285c3cf66478 \ - --hash=sha256:1b7fbe5571fdf28fd3aaab3ef4aafc847de9ebf263be959958c1ca58ec8eadf5 \ - --hash=sha256:26ebcc69d7b779c8f129393e99732961b5cc33fcff84090451f448c89b0e01b4 \ - --hash=sha256:6f462f5eca22fb66d7ff9c4744a3db4463af06c49816c4b6ac89b16bfcdc592e \ - --hash=sha256:6f76232163b5b9c34291b54621ee60417601e2e4802a188a0ea7157cd9b323f4 \ - --hash=sha256:7000b6e4d8ef07542c05044ec5d8bbae1df083b3f56822c3da63993a113e716f \ - --hash=sha256:810512e2eccdfc3b41eefa3a27402371a3411453a1efc7e9c000318196140fed \ - --hash=sha256:8f2c028954f16ede77902b223a8da2d9cbb3892375b85809a5c3cfb1587960c4 \ - --hash=sha256:9626d0bca1fb387d5791ca36bacbba298c5ef554747b7ebeafefb4564fc83566 \ - --hash=sha256:ac5b58559bb84a95848ed6984eb8013249f90b6bab62aa5acbad876e256002c9 \ - --hash=sha256:ad4953c5eb9c25a56d11a913c2011d7e580a435ef5145f804d98efa14477d390 \ - --hash=sha256:aefedc579ece2f8fb38f876aa7698204ee4c372d0e54f1c1ffa8ca580b54cc60 \ - --hash=sha256:afb2009ac98da274e893e03162f6269398b2b00d947e7057ee2469a921d58135 \ - --hash=sha256:b8a9d46b4df5ae2135a8e8e72b465448ebbc1559997f4f9304a9ecc3413efb5b \ - --hash=sha256:bd73f51957949069573ff783563486339a9285d72e2f36c18e0c1aa9ca7eb190 \ - --hash=sha256:bf9975bda82a99dc935f2ae4c83846d86df8fd6ba179614acac8e686910851da \ - --hash=sha256:c09526488c3a9e8b7a23a388d4974b670a9a3dd40c5c8a61db5593ce9b725bab \ - --hash=sha256:c9945669d3dadf8acb40ec2e57d38c985d8c285ea73af57fc5b09872c516106d \ - --hash=sha256:d13755f8e8445b3870114e5b6240facaa7cb0c3361e54beba3e07fa912a6e12b \ - --hash=sha256:fd918d4e6a4e0c110e2e05be7a7814d10dc1b95872accbf6512b80a109b71ae1 +ml-dtypes==0.5.3 \ + --hash=sha256:01de48de4537dc3c46e684b969a40ec36594e7eeb7c69e9a093e7239f030a28a \ + --hash=sha256:0a1d68a7cb53e3f640b2b6a34d12c0542da3dd935e560fdf463c0c77f339fc20 \ + --hash=sha256:0cd5a6c711b5350f3cbc2ac28def81cd1c580075ccb7955e61e9d8f4bfd40d24 \ + --hash=sha256:0e44a3761f64bc009d71ddb6d6c71008ba21b53ab6ee588dadab65e2fa79eafc \ + --hash=sha256:156418abeeda48ea4797db6776db3c5bdab9ac7be197c1233771e0880c304057 \ + --hash=sha256:19f6c3a4f635c2fc9e2aa7d91416bd7a3d649b48350c51f7f715a09370a90d93 \ + --hash=sha256:1b255acada256d1fa8c35ed07b5f6d18bc21d1556f842fbc2d5718aea2cd9e55 \ + --hash=sha256:1db60c154989af253f6c4a34e8a540c2c9dce4d770784d426945e09908fbb177 \ + --hash=sha256:2db74788fc01914a3c7f7da0763427280adfc9cd377e9604b6b64eb8097284bd \ + --hash=sha256:4a177b882667c69422402df6ed5c3428ce07ac2c1f844d8a1314944651439458 \ + --hash=sha256:4cae435a68861660af81fa3c5af16b70ca11a17275c5b662d9c6f58294e0f113 \ + --hash=sha256:5103856a225465371fe119f2fef737402b705b810bd95ad5f348e6e1a6ae21af \ + --hash=sha256:58e39349d820b5702bb6f94ea0cb2dc8ec62ee81c0267d9622067d8333596a46 \ + --hash=sha256:5ab039ffb40f3dc0aeeeba84fd6c3452781b5e15bef72e2d10bcb33e4bbffc39 \ + --hash=sha256:5ee72568d46b9533ad54f78b1e1f3067c0534c5065120ea8ecc6f210d22748b3 \ + --hash=sha256:66c2756ae6cfd7f5224e355c893cfd617fa2f747b8bbd8996152cbdebad9a184 \ + --hash=sha256:6936283b56d74fbec431ca57ce58a90a908fdbd14d4e2d22eea6d72bb208a7b7 \ + --hash=sha256:8b1a6e231b0770f2894910f1dce6d2f31d65884dbf7668f9b08d73623cdca909 \ + --hash=sha256:8bb9cd1ce63096567f5f42851f5843b5a0ea11511e50039a7649619abfb4ba6d \ + --hash=sha256:93c36a08a6d158db44f2eb9ce3258e53f24a9a4a695325a689494f0fdbc71770 \ + --hash=sha256:95ce33057ba4d05df50b1f3cfefab22e351868a843b3b15a46c65836283670c9 \ + --hash=sha256:9849ce7267444c0a717c80c6900997de4f36e2815ce34ac560a3edb2d9a64cd2 \ + --hash=sha256:9d55ea7f7baf2aed61bf1872116cefc9d0c3693b45cae3916897ee27ef4b835e \ + --hash=sha256:a4f39b9bf6555fab9bfb536cf5fdd1c1c727e8d22312078702e9ff005354b37f \ + --hash=sha256:aec640bd94c4c85c0d11e2733bd13cbb10438fb004852996ec0efbc6cacdaf70 \ + --hash=sha256:aecbd7c5272c82e54d5b99d8435fd10915d1bc704b7df15e4d9ca8dc3902be61 \ + --hash=sha256:bda32ce212baa724e03c68771e5c69f39e584ea426bfe1a701cb01508ffc7035 \ + --hash=sha256:bdcf26c2dbc926b8a35ec8cbfad7eff1a8bd8239e12478caca83a1fc2c400dc2 \ + --hash=sha256:bdf40d2aaabd3913dec11840f0d0ebb1b93134f99af6a0a4fd88ffe924928ab4 \ + --hash=sha256:c205cac07d24a29840c163d6469f61069ce4b065518519216297fc2f261f8db9 \ + --hash=sha256:c3f5ae0309d9f888fd825c2e9d0241102fadaca81d888f26f845bc8c13c1e4ee \ + --hash=sha256:cd7c0bb22d4ff86d65ad61b5dd246812e8993fbc95b558553624c33e8b6903ea \ + --hash=sha256:d0f730a17cf4f343b2c7ad50cee3bd19e969e793d2be6ed911f43086460096e4 \ + --hash=sha256:da65e5fd3eea434ccb8984c3624bc234ddcc0d9f4c81864af611aaebcc08a50e \ + --hash=sha256:e12e29764a0e66a7a31e9b8bf1de5cc0423ea72979f45909acd4292de834ccd3 # via neuronx-cc networkx==2.8.8 \ --hash=sha256:230d388117af870fce5647a3c52401fcf753e94720e6ea6b4197a5355648885e \ --hash=sha256:e435dfa75b1d7195c7b8378c3859f0445cd88c6b0375c181ed66823a9ceb7524 # via neuronx-cc -neuronx-cc==2.19.8089.0+8ab9f450 \ - --hash=sha256:fbb05d8a0da8f54954c03ff2bea659f627309110c00efd54f523d877ddd6b88d +neuronx-cc==2.20.9961.0+0acef03a \ + --hash=sha256:22db1e09975f400b74edc2fab24a1b86d967386f060f5a29a3ec21756ecf5b0c # via # -r runtimes/neuron/requirements.in # libneuronxla @@ -256,16 +254,16 @@ pgzip==0.3.5 \ --hash=sha256:4e13ab66ecface5c51c5af51d8cd676aa51675cf85df000f501a86cf38c208c1 \ --hash=sha256:dd35510f59f6bd6b64e31c4baf90c10cdbb2775235fcc079b14b404fbd7f46bf # via neuronx-cc -protobuf==6.31.1 \ - --hash=sha256:0414e3aa5a5f3ff423828e1e6a6e907d6c65c1d5b7e6e975793d5590bdeecc16 \ - --hash=sha256:426f59d2964864a1a366254fa703b8632dcec0790d8862d30034d8245e1cd447 \ - --hash=sha256:4ee898bf66f7a8b0bd21bce523814e6fbd8c6add948045ce958b73af7e8878c6 \ - --hash=sha256:6f1227473dc43d44ed644425268eb7c2e488ae245d51c6866d19fe158e207402 \ - --hash=sha256:720a6c7e6b77288b85063569baae8536671b39f15cc22037ec7045658d80489e \ - --hash=sha256:7fa17d5a29c2e04b7d90e5e32388b8bfd0e7107cd8e616feef7ed3fa6bdab5c9 \ - --hash=sha256:8764cf4587791e7564051b35524b72844f845ad0bb011704c3736cce762d8fe9 \ - --hash=sha256:a40fc12b84c154884d7d4c4ebd675d5b3b5283e155f324049ae396b95ddebc39 \ - --hash=sha256:d8cac4c982f0b957a4dc73a80e2ea24fab08e679c0de9deb835f4a12d69aca9a +protobuf==6.32.0 \ + --hash=sha256:15eba1b86f193a407607112ceb9ea0ba9569aed24f93333fe9a497cf2fda37d3 \ + --hash=sha256:501fe6372fd1c8ea2a30b4d9be8f87955a64d6be9c88a973996cef5ef6f0abf1 \ + --hash=sha256:75a2aab2bd1aeb1f5dc7c5f33bcb11d82ea8c055c9becbb41c26a8c43fd7092c \ + --hash=sha256:7db8ed09024f115ac877a1427557b838705359f047b2ff2f2b2364892d19dacb \ + --hash=sha256:84f9e3c1ff6fb0308dbacb0950d8aa90694b0d0ee68e75719cb044b7078fe741 \ + --hash=sha256:a81439049127067fc49ec1d36e25c6ee1d1a2b7be930675f919258d03c04e7d2 \ + --hash=sha256:a8bdbb2f009cfc22a36d031f22a625a38b615b5e19e558a7b756b3279723e68e \ + --hash=sha256:ba377e5b67b908c8f3072a57b63e2c6a4cbd18aea4ed98d2584350dbf46f2783 \ + --hash=sha256:d52691e5bee6c860fff9a1c86ad26a13afbeb4b168cd4445c922b7e2cf85aaf0 # via neuronx-cc psutil==7.0.0 \ --hash=sha256:101d71dc322e3cffd7cea0650b09b3d08b8e7c4109dd6809fe452dfd00e58b25 \ @@ -287,9 +285,9 @@ python-dateutil==2.9.0.post0 \ --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 # via botocore -requests==2.32.4 \ - --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c \ - --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422 +requests==2.32.5 \ + --hash=sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6 \ + --hash=sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf # via # ec2-metadata # requests-unixsocket diff --git a/runtimes/neuron/zmlxneuron.c b/runtimes/neuron/zmlxneuron.c deleted file mode 100644 index a5740bef..00000000 --- a/runtimes/neuron/zmlxneuron.c +++ /dev/null @@ -1,24 +0,0 @@ -#include -#include - -void *zmlxneuron_dlopen(const char *filename, int flags) -{ - if (filename != NULL) - { - char *replacements[] = { - "libnccom.so", "libnccom.so.2", - "libnrt.so", "libnrt.so.1", - "libncfw.so", "libncfw.so.2", - NULL, NULL, - }; - for (int i = 0; replacements[i] != NULL; i += 2) - { - if (strcmp(filename, replacements[i]) == 0) - { - filename = replacements[i + 1]; - break; - } - } - } - return dlopen(filename, flags); -} diff --git a/upb/BUILD.bazel b/upb/BUILD.bazel index ca8ebc3b..316cad3e 100644 --- a/upb/BUILD.bazel +++ b/upb/BUILD.bazel @@ -1,12 +1,16 @@ load("@rules_zig//zig:defs.bzl", "zig_library") cc_library( - name = "empty", + name = "upb_c", + hdrs = ["upb.h"], + deps = [ + "@com_google_protobuf//upb/message:copy", + ], ) zig_library( name = "upb", main = "upb.zig", - deps = [":empty"], + deps = [":upb_c"], visibility = ["//visibility:public"], ) diff --git a/upb/upb.h b/upb/upb.h new file mode 100644 index 00000000..dbec3577 --- /dev/null +++ b/upb/upb.h @@ -0,0 +1 @@ +#include "upb/message/copy.h" diff --git a/upb/upb.zig b/upb/upb.zig index 84702859..27d1502b 100644 --- a/upb/upb.zig +++ b/upb/upb.zig @@ -38,6 +38,14 @@ pub const ParseError = error{ Unknown, } || std.mem.Allocator.Error; +pub fn deepClone(comptime UpbType: type, arena: *c.upb_Arena, msg: *const UpbType) std.mem.Allocator.Error!*UpbType { + return @ptrCast(c.upb_Message_DeepClone(@ptrCast(msg), Minitable(UpbType), arena) orelse return std.mem.Allocator.Error.OutOfMemory); +} + +pub fn shallowClone(comptime UpbType: type, arena: *c.upb_Arena, msg: *const UpbType) std.mem.Allocator.Error!*UpbType { + return @ptrCast(c.upb_Message_ShallowClone(@ptrCast(msg), Minitable(UpbType), arena) orelse return std.mem.Allocator.Error.OutOfMemory); +} + pub fn stringView(data: ?[]const u8) c.upb_StringView { return if (data) |d| c.upb_StringView_FromDataAndSize(d.ptr, d.len) else .{}; } @@ -100,7 +108,7 @@ pub fn parseEx(comptime UpbType: type, arena: *c.upb_Arena, data: []const u8, op } pub fn parse(comptime UpbType: type, arena: *c.upb_Arena, data: []const u8) ParseError!*UpbType { - return parseEx(arena, data, .{}); + return parseEx(UpbType, arena, data, .{}); } pub fn new(comptime UpbType: type, arena: *c.upb_Arena) std.mem.Allocator.Error!*UpbType { From 5f3a2e74c38734612f975878c0e3f2c179df6981 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 22 Aug 2025 16:37:36 +0200 Subject: [PATCH 021/881] workspace: buildifier pass, remove rules_uv, rework tools/hf deps (#315) Title says it all --- MODULE.bazel | 1 - MODULE.bazel.lock | 114 --------------------------------- ffi/BUILD.bazel | 1 + runtimes/common/packages.bzl | 2 - runtimes/cpu/BUILD.bazel | 5 +- runtimes/cuda/BUILD.bazel | 4 +- runtimes/cuda/cuda.bzl | 2 +- runtimes/cuda/cuda.zig | 9 +-- runtimes/rocm/BUILD.bazel | 4 +- runtimes/tpu/BUILD.bazel | 1 + tools/BUILD.bazel | 21 +----- tools/hf/BUILD.bazel | 2 +- tools/hf/pyproject.toml | 4 -- tools/hf/requirements.in | 1 + tools/hf/requirements.lock.txt | 26 ++++---- upb/BUILD.bazel | 3 +- zml/tools/BUILD.bazel | 3 +- 17 files changed, 32 insertions(+), 171 deletions(-) delete mode 100644 tools/hf/pyproject.toml create mode 100644 tools/hf/requirements.in diff --git a/MODULE.bazel b/MODULE.bazel index 09230384..7f34bc92 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -18,7 +18,6 @@ bazel_dep(name = "rules_oci", version = "2.2.6") bazel_dep(name = "rules_proto", version = "7.1.0") bazel_dep(name = "rules_python", version = "1.5.3") bazel_dep(name = "rules_rust", version = "0.63.0") -bazel_dep(name = "rules_uv", version = "0.87.0") bazel_dep(name = "rules_zig", version = "20250821.0-be53625") bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.2.4") bazel_dep(name = "with_cfg.bzl", version = "0.11.0") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 4f754c09..5fa256de 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -53,7 +53,6 @@ "https://bazel-registry.zml.ai/modules/bazel_skylib/1.7.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_skylib/1.7.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_skylib/1.8.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/6.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/8.2.0.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/buildozer/7.1.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/google_benchmark/1.8.2/MODULE.bazel": "not found", @@ -80,8 +79,6 @@ "https://bazel-registry.zml.ai/modules/platforms/0.0.9/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/platforms/1.0.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/protobuf/21.7/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/23.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/24.4/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/protobuf/27.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/protobuf/27.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/protobuf/29.0-rc2/MODULE.bazel": "not found", @@ -119,7 +116,6 @@ "https://bazel-registry.zml.ai/modules/rules_java/6.3.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_java/6.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_java/6.5.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/7.1.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_java/7.10.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_java/7.12.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_java/7.2.0/MODULE.bazel": "not found", @@ -143,7 +139,6 @@ "https://bazel-registry.zml.ai/modules/rules_license/0.0.3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_license/0.0.7/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_license/1.0.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_multitool/0.11.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_oci/2.2.6/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_pkg/0.7.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_pkg/1.0.1/MODULE.bazel": "not found", @@ -160,7 +155,6 @@ "https://bazel-registry.zml.ai/modules/rules_python/0.29.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_python/0.31.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_python/0.33.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.34.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_python/0.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_python/0.40.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_python/1.4.1/MODULE.bazel": "not found", @@ -171,7 +165,6 @@ "https://bazel-registry.zml.ai/modules/rules_shell/0.4.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_uv/0.87.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.4/MODULE.bazel": "not found", @@ -184,7 +177,6 @@ "https://bazel-registry.zml.ai/modules/tar.bzl/0.2.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/toolchains_llvm_bootstrapped/0.2.4/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/upb/0.0.0-20230516-61a97ef/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/with_cfg.bzl/0.11.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/with_cfg.bzl/0.9.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/yq.bzl/0.1.1/MODULE.bazel": "not found", @@ -251,7 +243,6 @@ "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/MODULE.bazel": "3120d80c5861aa616222ec015332e5f8d3171e062e3e804a2a0253e1be26e59b", "https://bcr.bazel.build/modules/bazel_skylib/1.8.1/MODULE.bazel": "88ade7293becda963e0e3ea33e7d54d3425127e0a326e0d17da085a5f1f03ff6", "https://bcr.bazel.build/modules/bazel_skylib/1.8.1/source.json": "7ebaefba0b03efe59cac88ed5bbc67bcf59a3eff33af937345ede2a38b2d368a", - "https://bcr.bazel.build/modules/buildifier_prebuilt/6.4.0/MODULE.bazel": "37389c6b5a40c59410b4226d3bb54b08637f393d66e2fa57925c6fcf68e64bf4", "https://bcr.bazel.build/modules/buildifier_prebuilt/8.2.0.2/MODULE.bazel": "a9b689711d5b69f9db741649b218c119b9fdf82924ba390415037e09798edd03", "https://bcr.bazel.build/modules/buildifier_prebuilt/8.2.0.2/source.json": "51eb0a4b38aaaeab7fa64361576d616c4d8bfd0f17a0a10184aeab7084d79f8e", "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", @@ -288,8 +279,6 @@ "https://bcr.bazel.build/modules/platforms/1.0.0/MODULE.bazel": "f05feb42b48f1b3c225e4ccf351f367be0371411a803198ec34a389fb22aa580", "https://bcr.bazel.build/modules/platforms/1.0.0/source.json": "f4ff1fd412e0246fd38c82328eb209130ead81d62dcd5a9e40910f867f733d96", "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7", - "https://bcr.bazel.build/modules/protobuf/23.1/MODULE.bazel": "88b393b3eb4101d18129e5db51847cd40a5517a53e81216144a8c32dfeeca52a", - "https://bcr.bazel.build/modules/protobuf/24.4/MODULE.bazel": "7bc7ce5f2abf36b3b7b7c8218d3acdebb9426aeb35c2257c96445756f970eb12", "https://bcr.bazel.build/modules/protobuf/27.0/MODULE.bazel": "7873b60be88844a0a1d8f80b9d5d20cfbd8495a689b8763e76c6372998d3f64c", "https://bcr.bazel.build/modules/protobuf/27.1/MODULE.bazel": "703a7b614728bb06647f965264967a8ef1c39e09e8f167b3ca0bb1fd80449c0d", "https://bcr.bazel.build/modules/protobuf/29.0-rc2/MODULE.bazel": "6241d35983510143049943fc0d57937937122baf1b287862f9dc8590fc4c37df", @@ -334,7 +323,6 @@ "https://bcr.bazel.build/modules/rules_java/6.3.0/MODULE.bazel": "a97c7678c19f236a956ad260d59c86e10a463badb7eb2eda787490f4c969b963", "https://bcr.bazel.build/modules/rules_java/6.4.0/MODULE.bazel": "e986a9fe25aeaa84ac17ca093ef13a4637f6107375f64667a15999f77db6c8f6", "https://bcr.bazel.build/modules/rules_java/6.5.2/MODULE.bazel": "1d440d262d0e08453fa0c4d8f699ba81609ed0e9a9a0f02cd10b3e7942e61e31", - "https://bcr.bazel.build/modules/rules_java/7.1.0/MODULE.bazel": "30d9135a2b6561c761bd67bd4990da591e6bdc128790ce3e7afd6a3558b2fb64", "https://bcr.bazel.build/modules/rules_java/7.10.0/MODULE.bazel": "530c3beb3067e870561739f1144329a21c851ff771cd752a49e06e3dc9c2e71a", "https://bcr.bazel.build/modules/rules_java/7.12.2/MODULE.bazel": "579c505165ee757a4280ef83cda0150eea193eed3bef50b1004ba88b99da6de6", "https://bcr.bazel.build/modules/rules_java/7.2.0/MODULE.bazel": "06c0334c9be61e6cef2c8c84a7800cef502063269a5af25ceb100b192453d4ab", @@ -362,8 +350,6 @@ "https://bcr.bazel.build/modules/rules_license/0.0.7/MODULE.bazel": "088fbeb0b6a419005b89cf93fe62d9517c0a2b8bb56af3244af65ecfe37e7d5d", "https://bcr.bazel.build/modules/rules_license/1.0.0/MODULE.bazel": "a7fda60eefdf3d8c827262ba499957e4df06f659330bbe6cdbdb975b768bb65c", "https://bcr.bazel.build/modules/rules_license/1.0.0/source.json": "a52c89e54cc311196e478f8382df91c15f7a2bfdf4c6cd0e2675cc2ff0b56efb", - "https://bcr.bazel.build/modules/rules_multitool/0.11.0/MODULE.bazel": "8d9dda78d2398e136300d3ef4fbcc89ede7c32c158d8c016fa7d032df41c4aaf", - "https://bcr.bazel.build/modules/rules_multitool/0.11.0/source.json": "0b86574a1eaff37c33aafaff095ea16d6ac846beb94ffc74c4fcf626f8f80681", "https://bcr.bazel.build/modules/rules_oci/2.2.6/MODULE.bazel": "2ba6ddd679269e00aeffe9ca04faa2d0ca4129650982c9246d0d459fe2da47d9", "https://bcr.bazel.build/modules/rules_oci/2.2.6/source.json": "94e7decb8f95d9465b0bbea71c65064cd16083be1350c7468f131818641dc4a5", "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc", @@ -383,7 +369,6 @@ "https://bcr.bazel.build/modules/rules_python/0.29.0/MODULE.bazel": "2ac8cd70524b4b9ec49a0b8284c79e4cd86199296f82f6e0d5da3f783d660c82", "https://bcr.bazel.build/modules/rules_python/0.31.0/MODULE.bazel": "93a43dc47ee570e6ec9f5779b2e64c1476a6ce921c48cc9a1678a91dd5f8fd58", "https://bcr.bazel.build/modules/rules_python/0.33.2/MODULE.bazel": "3e036c4ad8d804a4dad897d333d8dce200d943df4827cb849840055be8d2e937", - "https://bcr.bazel.build/modules/rules_python/0.34.0/MODULE.bazel": "1d623d026e075b78c9fde483a889cda7996f5da4f36dffb24c246ab30f06513a", "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", "https://bcr.bazel.build/modules/rules_python/0.40.0/MODULE.bazel": "9d1a3cd88ed7d8e39583d9ffe56ae8a244f67783ae89b60caafc9f5cf318ada7", "https://bcr.bazel.build/modules/rules_python/1.4.1/MODULE.bazel": "8991ad45bdc25018301d6b7e1d3626afc3c8af8aaf4bc04f23d0b99c938b73a6", @@ -398,8 +383,6 @@ "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", "https://bcr.bazel.build/modules/rules_swift/2.1.1/source.json": "40fc69dfaac64deddbb75bd99cdac55f4427d9ca0afbe408576a65428427a186", - "https://bcr.bazel.build/modules/rules_uv/0.87.0/MODULE.bazel": "1f50b350458384f5c327ce04d943361435c772a80834536fc0eeebe92f3a7b95", - "https://bcr.bazel.build/modules/rules_uv/0.87.0/source.json": "2d75593d5c1126f6b7a014d14fb0771a24b7bd2e1de700f3e5e522ff0554b95c", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", "https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4", @@ -416,7 +399,6 @@ "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.2.4/MODULE.bazel": "859f02226bc313a622f56b816582fb55d5aaced4a1ecde7397dd4a9ebd24321e", "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.2.4/source.json": "43c5fd6b585cbf0b642157b6843a25425d541fabe638acb7bc908846c4bda8ca", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", - "https://bcr.bazel.build/modules/upb/0.0.0-20230516-61a97ef/MODULE.bazel": "c0df5e35ad55e264160417fd0875932ee3c9dda63d9fccace35ac62f45e1b6f9", "https://bcr.bazel.build/modules/with_cfg.bzl/0.11.0/MODULE.bazel": "bcb1dba64b74542648a93d6313e160b2670f22395581edd8c4ffadb811618ef7", "https://bcr.bazel.build/modules/with_cfg.bzl/0.11.0/source.json": "19102ff7f611a7a821e11d866868ad541abb4f5757e37e1c40adb6834ebf6205", "https://bcr.bazel.build/modules/with_cfg.bzl/0.9.2/MODULE.bazel": "8008021cb802e87e22b3e2b6f403c0529d4b865703fdcd40e2f36c23a651291c", @@ -816,102 +798,6 @@ ] } }, - "@@rules_multitool+//multitool:extension.bzl%multitool": { - "general": { - "bzlTransitiveDigest": "CxG1tSKAppSqmYJtpLgGpxrOvrd/xewKD0M2kCNfvOo=", - "usagesDigest": "kKFKsjNo1vbiSGlyYYHdSAj1Z8xtmvq8JBs56ZEpNS4=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "multitool.linux_arm64": { - "repoRuleId": "@@rules_multitool+//multitool/private:multitool.bzl%_env_specific_tools", - "attributes": { - "lockfiles": [ - "@@rules_uv+//uv/private:uv.lock.json" - ], - "os": "linux", - "cpu": "arm64" - } - }, - "multitool.linux_x86_64": { - "repoRuleId": "@@rules_multitool+//multitool/private:multitool.bzl%_env_specific_tools", - "attributes": { - "lockfiles": [ - "@@rules_uv+//uv/private:uv.lock.json" - ], - "os": "linux", - "cpu": "x86_64" - } - }, - "multitool.macos_arm64": { - "repoRuleId": "@@rules_multitool+//multitool/private:multitool.bzl%_env_specific_tools", - "attributes": { - "lockfiles": [ - "@@rules_uv+//uv/private:uv.lock.json" - ], - "os": "macos", - "cpu": "arm64" - } - }, - "multitool.macos_x86_64": { - "repoRuleId": "@@rules_multitool+//multitool/private:multitool.bzl%_env_specific_tools", - "attributes": { - "lockfiles": [ - "@@rules_uv+//uv/private:uv.lock.json" - ], - "os": "macos", - "cpu": "x86_64" - } - }, - "multitool.windows_arm64": { - "repoRuleId": "@@rules_multitool+//multitool/private:multitool.bzl%_env_specific_tools", - "attributes": { - "lockfiles": [ - "@@rules_uv+//uv/private:uv.lock.json" - ], - "os": "windows", - "cpu": "arm64" - } - }, - "multitool.windows_x86_64": { - "repoRuleId": "@@rules_multitool+//multitool/private:multitool.bzl%_env_specific_tools", - "attributes": { - "lockfiles": [ - "@@rules_uv+//uv/private:uv.lock.json" - ], - "os": "windows", - "cpu": "x86_64" - } - }, - "multitool": { - "repoRuleId": "@@rules_multitool+//multitool/private:multitool.bzl%_multitool_hub", - "attributes": { - "lockfiles": [ - "@@rules_uv+//uv/private:uv.lock.json" - ] - } - } - }, - "recordedRepoMappingEntries": [ - [ - "bazel_features+", - "bazel_features_globals", - "bazel_features++version_extension+bazel_features_globals" - ], - [ - "bazel_features+", - "bazel_features_version", - "bazel_features++version_extension+bazel_features_version" - ], - [ - "rules_multitool+", - "bazel_features", - "bazel_features+" - ] - ] - } - }, "@@rules_oci+//oci:extensions.bzl%oci": { "general": { "bzlTransitiveDigest": "4LJBCkExDO/zET3DzuMS4pDRRr4V0gc6YgFMhM6wIjI=", diff --git a/ffi/BUILD.bazel b/ffi/BUILD.bazel index ed5f95bf..1bf8ca7f 100644 --- a/ffi/BUILD.bazel +++ b/ffi/BUILD.bazel @@ -1,3 +1,4 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library") cc_library( diff --git a/runtimes/common/packages.bzl b/runtimes/common/packages.bzl index abb2ab1d..633f98b1 100644 --- a/runtimes/common/packages.bzl +++ b/runtimes/common/packages.bzl @@ -1,5 +1,3 @@ -load("//bazel:http_deb_archive.bzl", "http_deb_archive") - _BUILD_FILE_DEFAULT_VISIBILITY = """\ package(default_visibility = ["//visibility:public"]) """ diff --git a/runtimes/cpu/BUILD.bazel b/runtimes/cpu/BUILD.bazel index fe803c24..40760fd2 100644 --- a/runtimes/cpu/BUILD.bazel +++ b/runtimes/cpu/BUILD.bazel @@ -1,4 +1,5 @@ load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library") config_setting( @@ -23,19 +24,19 @@ cc_library( copy_to_directory( name = "sandbox", - out = "sandbox/lib", srcs = select({ ":darwin_arm64": ["@libpjrt_cpu_darwin_arm64//:libpjrt_cpu"], ":darwin_amd64": ["@libpjrt_cpu_darwin_amd64//:libpjrt_cpu"], "@platforms//os:linux": ["@libpjrt_cpu_linux_amd64//:libpjrt_cpu"], }), + out = "sandbox/lib", include_external_repositories = ["**"], ) cc_library( name = "libpjrt_cpu", - defines = ["ZML_RUNTIME_CPU"], data = [":sandbox"], + defines = ["ZML_RUNTIME_CPU"], ) zig_library( diff --git a/runtimes/cuda/BUILD.bazel b/runtimes/cuda/BUILD.bazel index 38904d47..96cc5879 100644 --- a/runtimes/cuda/BUILD.bazel +++ b/runtimes/cuda/BUILD.bazel @@ -3,13 +3,13 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_shared_library") zig_shared_library( name = "zmlxcuda", - main = "zmlxcuda.zig", # Use Clang's compiler-rt, but disable stack checking # to avoid requiring on the _zig_probe_stack symbol. copts = ["-fno-stack-check"], + main = "zmlxcuda.zig", shared_lib_name = "libzmlxcuda.so.0", - deps = ["//stdx"], visibility = ["@libpjrt_cuda//:__subpackages__"], + deps = ["//stdx"], ) cc_library( diff --git a/runtimes/cuda/cuda.bzl b/runtimes/cuda/cuda.bzl index 33806948..9920103b 100644 --- a/runtimes/cuda/cuda.bzl +++ b/runtimes/cuda/cuda.bzl @@ -130,7 +130,7 @@ def _read_redist_json(mctx, url, sha256): def _cuda_impl(mctx): loaded_packages = packages.read(mctx, [ - "@zml//runtimes/cuda:packages.lock.json", + "@zml//runtimes/cuda:packages.lock.json", ]) CUDA_REDIST = _read_redist_json( mctx, diff --git a/runtimes/cuda/cuda.zig b/runtimes/cuda/cuda.zig index a7fc166e..c07c0e91 100644 --- a/runtimes/cuda/cuda.zig +++ b/runtimes/cuda/cuda.zig @@ -1,5 +1,5 @@ -const builtin = @import("builtin"); const std = @import("std"); +const builtin = @import("builtin"); const asynk = @import("async"); const bazel_builtin = @import("bazel_builtin"); @@ -22,12 +22,7 @@ fn hasNvidiaDevice() bool { } fn hasCudaPathInLDPath() bool { - const ldLibraryPath = c.getenv("LD_LIBRARY_PATH"); - - if (ldLibraryPath == null) { - return false; - } - + const ldLibraryPath = std.c.getenv("LD_LIBRARY_PATH") orelse return false; return std.ascii.indexOfIgnoreCase(std.mem.span(ldLibraryPath), nvidiaLibsPath) != null; } diff --git a/runtimes/rocm/BUILD.bazel b/runtimes/rocm/BUILD.bazel index f94e8069..7355dc99 100644 --- a/runtimes/rocm/BUILD.bazel +++ b/runtimes/rocm/BUILD.bazel @@ -3,13 +3,13 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_shared_library") zig_shared_library( name = "zmlxrocm", - main = "zmlxrocm.zig", # Use Clang's compiler-rt, but disable stack checking # to avoid requiring on the _zig_probe_stack symbol. copts = ["-fno-stack-check"], + main = "zmlxrocm.zig", shared_lib_name = "libzmlxrocm.so.0", - deps = ["//stdx"], visibility = ["@libpjrt_rocm//:__subpackages__"], + deps = ["//stdx"], ) alias( diff --git a/runtimes/tpu/BUILD.bazel b/runtimes/tpu/BUILD.bazel index 94ae63c1..8939fef1 100644 --- a/runtimes/tpu/BUILD.bazel +++ b/runtimes/tpu/BUILD.bazel @@ -1,3 +1,4 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library") cc_library( diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index b27a9664..af65de3c 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -1,7 +1,5 @@ load("@rules_python//python:py_library.bzl", "py_library") load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") -load("@rules_python//python/uv:lock.bzl", uv_lock = "lock") -load("@rules_uv//uv:pip.bzl", "pip_compile") py_library( name = "zml_utils", @@ -11,24 +9,7 @@ py_library( py_console_script_binary( name = "hf", - script = "hf", pkg = "@huggingface_hub//huggingface_hub:pkg", + script = "hf", visibility = ["//visibility:public"], ) - -pip_compile( - name = "hf_update_requirements", - args = [ - "--generate-hashes", - "--emit-index-url", - "--emit-find-links", - "--no-strip-extras", - "--index-strategy=unsafe-best-match", - "--upgrade", - ], - requirements_in = "hf_requirements.in", - requirements_txt = "hf_requirements.lock.txt", - tags = [ - "manual", - ], -) diff --git a/tools/hf/BUILD.bazel b/tools/hf/BUILD.bazel index 154f165a..21a5315f 100644 --- a/tools/hf/BUILD.bazel +++ b/tools/hf/BUILD.bazel @@ -9,7 +9,7 @@ py_console_script_binary( uv_lock( name = "requirements", - srcs = ["pyproject.toml"], + srcs = ["requirements.in"], out = "requirements.lock.txt", args = ["--upgrade"], tags = ["manual"], diff --git a/tools/hf/pyproject.toml b/tools/hf/pyproject.toml deleted file mode 100644 index d65bcf2c..00000000 --- a/tools/hf/pyproject.toml +++ /dev/null @@ -1,4 +0,0 @@ -[project] -name = "hf" -version = "0.1.0" -dependencies = ["huggingface_hub"] diff --git a/tools/hf/requirements.in b/tools/hf/requirements.in new file mode 100644 index 00000000..6b964ccc --- /dev/null +++ b/tools/hf/requirements.in @@ -0,0 +1 @@ +huggingface_hub diff --git a/tools/hf/requirements.lock.txt b/tools/hf/requirements.lock.txt index f4288672..3657ce18 100644 --- a/tools/hf/requirements.lock.txt +++ b/tools/hf/requirements.lock.txt @@ -93,20 +93,20 @@ fsspec==2025.7.0 \ --hash=sha256:786120687ffa54b8283d942929540d8bc5ccfa820deb555a2b5d0ed2b737bf58 \ --hash=sha256:8b012e39f63c7d5f10474de957f3ab793b47b45ae7d39f2fb735f8bbe25c0e21 # via huggingface-hub -hf-xet==1.1.7 \ - --hash=sha256:18b61bbae92d56ae731b92087c44efcac216071182c603fc535f8e29ec4b09b8 \ - --hash=sha256:20cec8db4561338824a3b5f8c19774055b04a8df7fff0cb1ff2cb1a0c1607b80 \ - --hash=sha256:2e356da7d284479ae0f1dea3cf5a2f74fdf925d6dca84ac4341930d892c7cb34 \ - --hash=sha256:60dae4b44d520819e54e216a2505685248ec0adbdb2dd4848b17aa85a0375cde \ - --hash=sha256:6efaaf1a5a9fc3a501d3e71e88a6bfebc69ee3a716d0e713a931c8b8d920038f \ - --hash=sha256:713f2bff61b252f8523739969f247aa354ad8e6d869b8281e174e2ea1bb8d604 \ - --hash=sha256:751571540f9c1fbad9afcf222a5fb96daf2384bf821317b8bfb0c59d86078513 \ - --hash=sha256:b109f4c11e01c057fc82004c9e51e6cdfe2cb230637644ade40c599739067b2e +hf-xet==1.1.8 \ + --hash=sha256:09e86514c3c4284ed8a57d6b0f3d089f9836a0af0a1ceb3c9dd664f1f3eaefef \ + --hash=sha256:25b9d43333bbef39aeae1616789ec329c21401a7fe30969d538791076227b591 \ + --hash=sha256:3d5f82e533fc51c7daad0f9b655d9c7811b5308e5890236828bd1dd3ed8fea74 \ + --hash=sha256:4171f31d87b13da4af1ed86c98cf763292e4720c088b4957cf9d564f92904ca9 \ + --hash=sha256:4a9b99ab721d385b83f4fc8ee4e0366b0b59dce03b5888a86029cc0ca634efbf \ + --hash=sha256:62a0043e441753bbc446dcb5a3fe40a4d03f5fb9f13589ef1df9ab19252beb53 \ + --hash=sha256:8e2dba5896bca3ab61d0bef4f01a1647004de59640701b37e37eaa57087bbd9d \ + --hash=sha256:bfe5700bc729be3d33d4e9a9b5cc17a951bf8c7ada7ba0c9198a6ab2053b7453 # via huggingface-hub huggingface-hub==0.34.4 \ --hash=sha256:9b365d781739c93ff90c359844221beef048403f1bc1f1c123c191257c3c890a \ --hash=sha256:a4228daa6fb001be3f4f4bdaf9a0db00e1739235702848df00885c9b5742c85c - # via hf (tools/hf/pyproject.toml) + # via -r tools/hf/requirements.in idna==3.10 \ --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \ --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 @@ -170,9 +170,9 @@ pyyaml==6.0.2 \ --hash=sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12 \ --hash=sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4 # via huggingface-hub -requests==2.32.4 \ - --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c \ - --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422 +requests==2.32.5 \ + --hash=sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6 \ + --hash=sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf # via huggingface-hub tqdm==4.67.1 \ --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 \ diff --git a/upb/BUILD.bazel b/upb/BUILD.bazel index 316cad3e..ebf973fb 100644 --- a/upb/BUILD.bazel +++ b/upb/BUILD.bazel @@ -1,3 +1,4 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library") cc_library( @@ -11,6 +12,6 @@ cc_library( zig_library( name = "upb", main = "upb.zig", - deps = [":upb_c"], visibility = ["//visibility:public"], + deps = [":upb_c"], ) diff --git a/zml/tools/BUILD.bazel b/zml/tools/BUILD.bazel index 7579fb9d..ff8c1b04 100644 --- a/zml/tools/BUILD.bazel +++ b/zml/tools/BUILD.bazel @@ -1,3 +1,4 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library") cc_library( @@ -25,5 +26,5 @@ zig_library( cc_static_library( name = "macos_static_tools", - deps = ["macos_c"] + deps = ["macos_c"], ) From 56ac3e27dd014dbc896d9f03fd16d8a59b05ed09 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Wed, 27 Aug 2025 09:43:39 +0200 Subject: [PATCH 022/881] zml: remove gguf and torch loaders (#317) Torch loader might be reintroduced later, if the need arises. --- examples/mnist/BUILD.bazel | 4 +- examples/mnist/mnist.zig | 4 +- third_party/mnist/repo.bzl | 4 +- zml/BUILD.bazel | 40 +- zml/aio.zig | 15 +- zml/aio/gguf.zig | 83 --- zml/aio/gguf/core.zig | 505 ------------- zml/aio/torch.zig | 42 -- zml/aio/torch/eval.zig | 435 ------------ zml/aio/torch/file.zig | 681 ------------------ zml/aio/torch/pickle.zig | 1065 ---------------------------- zml/aio/torch/py.zig | 430 ----------- zml/aio/torch/simple.pt | Bin 2708 -> 0 bytes zml/aio/torch/simple_test_4.pickle | Bin 130 -> 0 bytes 14 files changed, 35 insertions(+), 3273 deletions(-) delete mode 100644 zml/aio/gguf.zig delete mode 100644 zml/aio/gguf/core.zig delete mode 100644 zml/aio/torch.zig delete mode 100644 zml/aio/torch/eval.zig delete mode 100644 zml/aio/torch/file.zig delete mode 100644 zml/aio/torch/pickle.zig delete mode 100644 zml/aio/torch/py.zig delete mode 100644 zml/aio/torch/simple.pt delete mode 100644 zml/aio/torch/simple_test_4.pickle diff --git a/examples/mnist/BUILD.bazel b/examples/mnist/BUILD.bazel index dfc635ba..11ac0c66 100644 --- a/examples/mnist/BUILD.bazel +++ b/examples/mnist/BUILD.bazel @@ -4,11 +4,11 @@ load("@rules_zig//zig:defs.bzl", "zig_binary") zig_binary( name = "mnist", args = [ - "$(location @mnist//:mnist.pt)", + "$(location @mnist//:mnist.safetensors)", "$(location @mnist//:t10k-images.idx3-ubyte)", ], data = [ - "@mnist//:mnist.pt", + "@mnist//:mnist.safetensors", "@mnist//:t10k-images.idx3-ubyte", ], main = "mnist.zig", diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index d4cfa328..5c85f5b7 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -30,7 +30,7 @@ const Mnist = struct { var x = input.flattenAll().convert(.f32); const layers: []const Layer = &.{ self.fc1, self.fc2 }; for (layers) |layer| { - x = zml.call(layer, .forward, .{x}); + x = layer.forward(x); } return x.argMax(0).indices.convert(.u8); } @@ -66,7 +66,7 @@ pub fn asyncMain() !void { // Read model shapes. // Note this works because Mnist struct uses the same layer names as the pytorch model - var buffer_store = try zml.aio.torch.open(allocator, pt_model); + var buffer_store = try zml.aio.detectFormatAndOpen(allocator, pt_model); defer buffer_store.deinit(); const mnist_model = try zml.aio.populateModel(Mnist, allocator, buffer_store); diff --git a/third_party/mnist/repo.bzl b/third_party/mnist/repo.bzl index 025845eb..fdabe71a 100644 --- a/third_party/mnist/repo.bzl +++ b/third_party/mnist/repo.bzl @@ -3,7 +3,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") def repo(): http_archive( name = "mnist", - sha256 = "075905e433ea0cce13c3fc08832448ab86225d089b5d412be67f59c29388fb19", - url = "https://mirror.zml.ai/data/mnist.tar.zst", + sha256 = "9c7d9a5ef9c245084996f6d2ec66ef176e51186e6a5b22efdcc3828d644941ca", + url = "https://mirror.zml.ai/data/mnist_safetensors.tar.zst", build_file_content = """exports_files(glob(["**"]), visibility = ["//visibility:public"])""", ) diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 7901965f..e0722f70 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -19,15 +19,35 @@ cc_library( zig_library( name = "zml", - srcs = glob([ - "*.zig", - "aio/**/*.zig", - "nn/**/*.zig", - # TODO: test_runner.zig should not be here. - # It's here for now because it seems that test_runner property in zig_test is misbehaving. - # See https://github.com/zml/rules_zig/issues/2 + srcs = [ + "aio.zig", + "aio/json.zig", + "aio/safetensors.zig", + "aio/tinyllama.zig", + "buffer.zig", + "context.zig", + "dtype.zig", + "exe.zig", + "floats.zig", + "helpers.zig", + "hostbuffer.zig", + "meta.zig", + "mlirx.zig", + "module.zig", + "nn.zig", + "nn/cuda.zig", + "ops.zig", + "pjrtx.zig", + "platform.zig", + "posix.zig", + "quantization.zig", + "shape.zig", + "tensor.zig", "test_runner.zig", - ]), + "testing.zig", + "torch.zig", + "zml.zig", + ], copts = ["-lc"], main = "zml.zig", visibility = ["//visibility:public"], @@ -52,10 +72,6 @@ zig_library( zig_test( name = "test", - data = [ - "aio/torch/simple.pt", - "aio/torch/simple_test_4.pickle", - ], test_runner = ":test_runner", deps = [":zml"], ) diff --git a/zml/aio.zig b/zml/aio.zig index 467def71..08f352eb 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -4,24 +4,17 @@ const asynk = @import("async"); const c = @import("c"); const stdx = @import("stdx"); -pub const gguf = @import("aio/gguf.zig"); -// pub const nemo = @import("aio/nemo.zig"); pub const safetensors = @import("aio/safetensors.zig"); pub const tinyllama = @import("aio/tinyllama.zig"); -pub const torch = @import("aio/torch.zig"); -// pub const yaml = @import("aio/yaml.zig"); const HostBuffer = @import("hostbuffer.zig").HostBuffer; const posix = @import("posix.zig"); const zml = @import("zml.zig"); pub const log = std.log.scoped(.@"zml/aio"); + test { std.testing.refAllDecls(@This()); - std.testing.refAllDecls(gguf); - // std.testing.refAllDecls(nemo); std.testing.refAllDecls(safetensors); - std.testing.refAllDecls(torch); - // std.testing.refAllDecls(yaml); } // TODO error set for weight loading @@ -32,12 +25,6 @@ pub fn detectFormatAndOpen(allocator: std.mem.Allocator, model_path: []const u8) try safetensors.open(allocator, model_path) else if (std.mem.endsWith(u8, model_path, ".safetensors.index.json")) try safetensors.open(allocator, model_path) - else if (std.mem.endsWith(u8, model_path, ".gguf")) - try gguf.open(allocator, model_path) - else if (std.mem.endsWith(u8, model_path, ".pt")) - try torch.open(allocator, model_path) - else if (std.mem.endsWith(u8, model_path, ".tinyllama")) - try tinyllama.open(allocator, model_path) else { std.debug.panic("File extension not recognized: {s}", .{model_path}); }; diff --git a/zml/aio/gguf.zig b/zml/aio/gguf.zig deleted file mode 100644 index 3134f969..00000000 --- a/zml/aio/gguf.zig +++ /dev/null @@ -1,83 +0,0 @@ -const asynk = @import("async"); -const core = @import("gguf/core.zig"); -const std = @import("std"); -const zml = @import("../zml.zig"); - -const HostBuffer = @import("../hostbuffer.zig").HostBuffer; - -const Allocator = std.mem.Allocator; -const assert = std.debug.assert; - -const log = std.log.scoped(.@"zml/io"); - -pub fn open(allocator: Allocator, path: []const u8) !zml.aio.BufferStore { - var file = try core.GgufFile.open(path); - errdefer file.close(); - - var res: zml.aio.BufferStore = .{ - .arena = std.heap.ArenaAllocator.init(allocator), - }; - errdefer res.arena.deinit(); - const arena = res.arena.allocator(); - - res.files = try arena.dupe(zml.aio.MemoryMappedFile, &.{file.file}); - - // metadata must be read in order to read tensors - try loadMetadata(arena, &res, &file); - try loadBuffers(arena, &res, &file); - if (res.buffers.count() != file.header.tensor_count) { - log.warn("Expected to find {d} tensors in {s}, only found {d}", .{ file.header.tensor_count, path, res.buffers.count() }); - } - return res; -} - -fn loadMetadata(allocator: Allocator, store: *zml.aio.BufferStore, file: *core.GgufFile) !void { - try store._metadata.ensureTotalCapacity(allocator, @intCast(file.header.metadata_kv_count)); - - while (file.readMetadata(allocator)) |entry| { - log.info("Loading MetaData: {s}", .{entry.name}); - const res = store._metadata.getOrPutAssumeCapacity(entry.name); - if (res.found_existing) { - // This file seems invalid. Since most metadatas aren't required, continue ahead. - log.warn("Found duplicated metadata key: {s}", .{entry.name}); - continue; - } - res.value_ptr.* = switch (entry.val) { - .array => |arr| switch (arr.child) { - inline .uint8, .int8, .uint16, .int16, .uint32, .int32, .float32, .bool, .string, .uint64, .int64, .float64 => |tag| blk: { - const T = @FieldType(core.GgufValue, @tagName(tag)); - break :blk try zml.aio.Metadata.copySlice(allocator, std.mem.bytesAsSlice(T, arr.data)); - }, - else => blk: { - log.warn("ignoring array metadata", .{}); - break :blk .null; - }, - }, - inline else => |v| zml.aio.Metadata.wrap(v), - }; - } else |err| switch (err) { - error.EndOfMetadata => {}, - else => return err, - } -} - -fn loadBuffers(allocator: Allocator, store: *zml.aio.BufferStore, file: *core.GgufFile) !void { - try store.buffers.ensureTotalCapacity(allocator, @intCast(file.header.tensor_count)); - while (file.readTensorInfo(allocator)) |info| { - const res = store.buffers.getOrPutAssumeCapacity(info.name); - if (res.found_existing) { - // This file seems invalid. Try to continue anyway. - log.warn("Found duplicated tensor: {s}", .{info.name}); - continue; - } - - // TODO: handle quantized types - const dtype: zml.DataType = info.t.toDtype() orelse return error.UnsupportedGgufType; - const buffer = HostBuffer.fromBytes(zml.Shape.init(info.shape(), dtype), file.file.mappedSlice(info.start, info.byte_len)); - res.value_ptr.* = buffer; - // store the info index. - } else |err| switch (err) { - error.EndOfMetadata => {}, - else => return err, - } -} diff --git a/zml/aio/gguf/core.zig b/zml/aio/gguf/core.zig deleted file mode 100644 index 27102a28..00000000 --- a/zml/aio/gguf/core.zig +++ /dev/null @@ -1,505 +0,0 @@ -const asynk = @import("async"); -const std = @import("std"); -const zml = @import("../../zml.zig"); - -const assert = std.debug.assert; -const log = std.log.scoped(.@"zml/io"); - -pub const GgufErrors = error{ - ValueTypeMismatch, - InvalidGguf, - UnsupportedGgufType, - EndOfMetadata, - OutOfMemory, -}; - -// Enums and structures -pub const TensorType = enum(u32) { - f32 = 0, - f16 = 1, - q4_0 = 2, - q4_1 = 3, - deprecated_q4_2 = 4, - deprecated_q4_3 = 5, - q5_0 = 6, - q5_1 = 7, - q8_0 = 8, - q8_1 = 9, - // k-quantizations - q2_k = 10, - q3_k = 11, - q4_k = 12, - q5_k = 13, - q6_k = 14, - q8_k = 15, - i8 = 16, - i16 = 17, - i32 = 18, - - const MAX_KNOWN_ENUM = 18; - - pub fn canConvertQuant(self: TensorType) bool { - return switch (self) { - .q8_0, .q4_k, .q6_k, .q2_k, .q4_0, .q4_1 => true, - else => false, - }; - } - - pub fn toDtype(self: TensorType) ?zml.DataType { - return switch (self) { - .f32 => .f32, - .f16 => .f16, - .i8 => .i8, - .i16 => .i16, - .i32 => .i32, - else => null, - }; - } - - pub fn sizeOf(self: TensorType) usize { - return self.toDtype().?.sizeOf(); - } - - /// Return the tensor type features - pub fn getFeatures(t: TensorType) TensorTypeFeatures { - return switch (t) { - inline else => |val| @field(TENSOR_TYPE_FEATURES, @tagName(val)), - }; - } -}; - -/// GGUF tensor type to features lookup table. -pub const TensorTypeFeatures = struct { - items_per_block: u29, - bytes_per_block: u29, - - pub fn alignment(features: TensorTypeFeatures) u8 { - return std.math.log2_int(u29, features.bytes_per_block); - } -}; - -pub const TENSOR_TYPE_FEATURES: std.enums.EnumFieldStruct(TensorType, TensorTypeFeatures, null) = .{ - .f32 = .{ .items_per_block = 1, .bytes_per_block = @sizeOf(f32) }, - .f16 = .{ .items_per_block = 1, .bytes_per_block = @sizeOf(f16) }, - .q4_0 = .{ .items_per_block = 32, .bytes_per_block = 18 }, - .q4_1 = .{ .items_per_block = 32, .bytes_per_block = 20 }, - .deprecated_q4_2 = .{ .items_per_block = 0, .bytes_per_block = 0 }, - .deprecated_q4_3 = .{ .items_per_block = 0, .bytes_per_block = 0 }, - .q5_0 = .{ .items_per_block = 32, .bytes_per_block = 22 }, - .q5_1 = .{ .items_per_block = 32, .bytes_per_block = 24 }, - .q8_0 = .{ .items_per_block = 32, .bytes_per_block = 34 }, - .q8_1 = .{ .items_per_block = 32, .bytes_per_block = 40 }, - .q2_k = .{ .items_per_block = 256, .bytes_per_block = 82 }, - .q3_k = .{ .items_per_block = 256, .bytes_per_block = 110 }, - .q4_k = .{ .items_per_block = 256, .bytes_per_block = 144 }, - .q5_k = .{ .items_per_block = 256, .bytes_per_block = 176 }, - .q6_k = .{ .items_per_block = 256, .bytes_per_block = 210 }, - .q8_k = .{ .items_per_block = 256, .bytes_per_block = 292 }, - .i8 = .{ .items_per_block = 1, .bytes_per_block = @sizeOf(i8) }, - .i16 = .{ .items_per_block = 1, .bytes_per_block = @sizeOf(i16) }, - .i32 = .{ .items_per_block = 1, .bytes_per_block = @sizeOf(i32) }, -}; - -pub const GgufValueType = enum(u32) { - // The value is a 8-bit unsigned integer. - uint8 = 0, - // The value is a 8-bit signed integer. - int8 = 1, - // The value is a 16-bit unsigned little-endian integer. - uint16 = 2, - // The value is a 16-bit signed little-endian integer. - int16 = 3, - // The value is a 32-bit unsigned little-endian integer. - uint32 = 4, - // The value is a 32-bit signed little-endian integer. - int32 = 5, - // The value is a 32-bit IEEE754 floating point number. - float32 = 6, - // The value is a boolean. - // 1-byte value where 0 is false and 1 is true. - // Anything else is invalid, and should be treated as either the model - // being invalid or the reader being buggy. - bool = 7, - // The value is a UTF-8 non-null-terminated string, with length prepended. - string = 8, - // The value is an array of other values, with the length and type - // prepended. Arrays can be nested, and the length of the array is the - // number of elements in the array, not the number of bytes. - array = 9, - // The value is a 64-bit unsigned little-endian integer. - uint64 = 10, - // The value is a 64-bit signed little-endian integer. - int64 = 11, - // The value is a 64-bit IEEE754 floating point number. - float64 = 12, - // Special values used by the callbacks of gguf_do_with_value(). - array_start = 100, - array_end = 101, - - // Allow other values in case GGUF add more types without us noticing - _, - - pub fn sizeOf(self: GgufValueType) usize { - return switch (self) { - .uint8 => @sizeOf(u8), - .int8 => @sizeOf(i8), - .uint16 => @sizeOf(u16), - .int16 => @sizeOf(i16), - .uint32 => @sizeOf(u32), - .int32 => @sizeOf(i32), - .float32 => @sizeOf(f32), - .bool => @sizeOf(bool), - .uint64 => @sizeOf(u64), - .int64 => @sizeOf(i64), - .float64 => @sizeOf(f64), - .string => @sizeOf([]u8), - else => unreachable, - }; - } - - pub fn arrayTypeCheck(self: GgufValueType, comptime T: type) !void { - switch (self) { - .string => if (T != []u8 and T != []const u8) return error.ValueTypeMismatch, - .uint8 => if (T != u8) return error.ValueTypeMismatch, - .int8 => if (T != i8) return error.ValueTypeMismatch, - .uint16 => if (T != u16) return error.ValueTypeMismatch, - .int16 => if (T != i16) return error.ValueTypeMismatch, - .uint32 => if (T != u32) return error.ValueTypeMismatch, - .int32 => if (T != i32) return error.ValueTypeMismatch, - .float32 => if (T != f32) return error.ValueTypeMismatch, - .bool => if (T != bool) return error.ValueTypeMismatch, - .uint64 => if (T != u64) return error.ValueTypeMismatch, - .int64 => if (T != i64) return error.ValueTypeMismatch, - .float64 => if (T != f64) return error.ValueTypeMismatch, - else => {}, - } - } -}; - -pub const ValueType = enum(u8) { - uint8 = 0, - int8 = 1, - uint16 = 2, - int16 = 3, - uint32 = 4, - int32 = 5, - float32 = 6, - bool = 7, - string = 8, - array = 9, - uint64 = 10, - int64 = 11, - float64 = 12, -}; - -// Union of possible values. -pub const GgufValue = union(ValueType) { - uint8: u8, - int8: i8, - uint16: u16, - int16: i16, - uint32: u32, - int32: i32, - float32: f32, - bool: bool, - string: []const u8, - array: Array, - uint64: u64, - int64: i64, - float64: f64, - - pub const Array = struct { - // Any value type is valid, including arrays. - child: ValueType, - // Number of elements, not bytes - len: usize, - data: []u8, - }; -}; - -// Header -const GgufHeader = extern struct { - // Magic number to announce that this is a GGUF file. Must be `GUFF`. - magic: [4]u8, - // The version of the format implemented. - // Must be `3` for version described in this spec. - version: u32, - // The number of tensors in the file. - // This is explicit, instead of being included in the metadata, to ensure - // it is always present for loading the tensors. - tensor_count: usize, - // The number of metadata key-value pairs. - metadata_kv_count: usize, - - pub fn validate(self: GgufHeader) !void { - if (!std.mem.eql(u8, &self.magic, "GGUF")) { - log.err("Invalid GGUF file: wrong header {s}", .{self.magic}); - return error.InvalidGguf; - } - } -}; - -// Key representation in this library API. -pub const GgufMetadataKv = struct { - name: []const u8, - type_: GgufValueType, - val: GgufValue, -}; - -// Tensor representation in this library API. -const GGUF_TENSOR_MAX_DIM: usize = 8; // Future-proof: actual limit is 4. -pub const GgufTensorInfo = struct { - name: []const u8, - t: TensorType, // Tensor type (enum TensorType). - rank: usize, // Number of dimensions of the tensor. - dims: [GGUF_TENSOR_MAX_DIM]i64, // Dimensions (Eg. [512, 1024, 1, 1]). - start: usize, // Offset from start of data section. - byte_len: usize, // Total size in bytes. - num_weights: usize, // Total number of parameters. - - pub inline fn shape(info: GgufTensorInfo) []const i64 { - return info.dims[0..info.rank]; - } -}; - -// Return the value type name given the type ID. -fn getValueTypeName(t: u32) []const u8 { - if (@as(usize, @intCast(t)) >= GGUF_VALUE_NAME.len) return "unknown"; - return GGUF_VALUE_NAME[@intCast(t)]; -} - -const GGUF_VALUE_NAME = [_][]const u8{ - "uint8", "int8", "uint16", "int16", "uint32", "int32", - "float32", "bool", "string", "array", "uint64", "int64", - "float64", -}; - -/// GGUF file API -/// A memory-mapped view of a .gguf file. -/// Format used by GGML models: https://github.com/ggerganov/ggml/ -pub const GgufFile = struct { - header: GgufHeader, // GUFF file header info. - size: usize, // Total file size. - file: zml.aio.MemoryMappedFile, - left_kv: usize, // Number of key-value pairs yet to read. - left_tensors: usize, // Number of tensors yet to read. - off: usize, // Offset of the next item to parse. - alignment: usize = 32, // File data alignment. Default: 32 bytes. - - /// Open and memmap the given file. - pub fn open(path: []const u8) !GgufFile { - const file = try asynk.File.open(path, .{}); - const header = try file.reader().readStruct(GgufHeader); - try header.validate(); - return .{ - .header = header, - .size = (try file.stat()).size, - .file = try zml.aio.MemoryMappedFile.init(file), - .off = @sizeOf(GgufHeader), - .left_kv = header.metadata_kv_count, - .left_tensors = header.tensor_count, - }; - } - - /// Unmap the file memory and close the file handle. - pub fn close(self: *GgufFile) void { - self.file.deinit(); - } - - /// Set the context to read the first key-value entry in the GGUF - /// file and then all the rest. Is used when creating a new context - /// and also when you want to restart scanning the key-value - /// items in the file. - fn rewind(ctx: *GgufFile) void { - ctx.off = @sizeOf(GgufHeader); - ctx.left_kv = ctx.header.metadata_kv_count; - ctx.left_tensors = ctx.header.tensor_count; - } - - pub fn seek(self: *GgufFile, pos: usize) void { - assert(pos < self.size); - self.off = pos; - } - - fn readInt(self: *GgufFile, comptime T: type) !T { - if (self.off + @sizeOf(T) >= self.size) return error.InvalidGguf; - const res = self.file.file.reader().readInt(T, .little); - self.off += @sizeOf(T); - return res; - } - - fn readTensorType(self: *GgufFile) !TensorType { - const raw = try self.readInt(u32); - if (raw > TensorType.MAX_KNOWN_ENUM) { - log.err("Unsupported GGUF tensor type: {d}", .{raw}); - return error.UnsupportedGgufType; - } - return @enumFromInt(raw); - } - - fn readValueType(self: *GgufFile) !GgufValueType { - const raw = try self.readInt(u32); - const t: GgufValueType = @enumFromInt(raw); - switch (t) { - .uint8, .int8, .uint16, .int16, .uint32, .int32, .float32, .bool, .string, .array, .uint64, .int64, .float64, .array_start, .array_end => {}, - else => { - log.err("Unsupported GGUF value type: {s}", .{@tagName(t)}); - return error.UnsupportedGgufType; - }, - } - return t; - } - - pub fn readAlloc(self: *GgufFile, allocator: std.mem.Allocator, len: usize) ![]u8 { - const data = try allocator.alloc(u8, len); - const read = try self.file.file.reader().readAll(data); - if (read != data.len) return error.InvalidGguf; - self.off += len; - return data; - } - - pub fn skipBytes(self: *GgufFile, len: usize) !void { - try self.file.file.seekBy(@intCast(len)); - self.off += len; - } - - /// Read the len then the actual bytes. - pub fn readString(self: *GgufFile, allocator: std.mem.Allocator) ![]u8 { - const len: usize = try self.readInt(u64); - return self.readAlloc(allocator, len); - } - - pub fn skipString(self: *GgufFile) !void { - const len: usize = try self.readInt(u64); - return self.skipBytes(len); - } - - fn readArrayHeader(self: *GgufFile, allocator: std.mem.Allocator) !GgufValue.Array { - const child = try self.readValueType(); - if (@intFromEnum(child) > @intFromEnum(ValueType.float64)) { - return error.UnsupportedGgufType; - } - const len: usize = try self.readInt(u64); - const data = switch (child) { - // Since strings have variable lenghts, we need to read them one by one - .string => str: { - var data = try allocator.alloc([]u8, len); - for (0..len) |i| data[i] = try self.readString(allocator); - break :str std.mem.sliceAsBytes(data); - }, - else => try self.readAlloc(allocator, len * child.sizeOf()), - }; - return .{ - .child = @enumFromInt(@intFromEnum(child)), - .len = len, - .data = data, - }; - } - - fn readTypedValue(self: *GgufFile, allocator: std.mem.Allocator, t: GgufValueType) !GgufValue { - return switch (t) { - .uint8 => .{ .uint8 = try self.readInt(u8) }, - .int8 => .{ .int8 = try self.readInt(i8) }, - .uint16 => .{ .uint16 = try self.readInt(u16) }, - .int16 => .{ .int16 = try self.readInt(i16) }, - .uint32 => .{ .uint32 = try self.readInt(u32) }, - .int32 => .{ .int32 = try self.readInt(i32) }, - .float32 => .{ .float32 = @bitCast(try self.readInt(u32)) }, - .bool => .{ .bool = try self.readInt(u8) != 0 }, - .string => .{ .string = try self.readString(allocator) }, - .array => .{ .array = try self.readArrayHeader(allocator) }, - .uint64 => .{ .uint64 = try self.readInt(u64) }, - .int64 => .{ .int64 = try self.readInt(i64) }, - .float64 => .{ .float64 = @bitCast(try self.readInt(u64)) }, - else => error.UnsupportedGgufType, - }; - } - - /// Parses the next metadata entry. - /// Returns error.EndOfMetadata if there are no longer metadata to process in this GGUF file. - pub fn readMetadata(self: *GgufFile, allocator: std.mem.Allocator) !GgufMetadataKv { - if (self.left_kv == 0) return error.EndOfMetadata; - self.left_kv -= 1; - const name = try self.readString(allocator); - const type_ = try self.readValueType(); - const val: GgufValue = try self.readTypedValue(allocator, type_); - return .{ .name = name, .type_ = type_, .val = val }; - } - - // Set the data section offset. This function must be called exactly when - // all the key-values are consumed, in the context of the first call of - // ctx.getTensor(): this way we will be able to return tensor offsets - // as absolute positions and pointers to the mmapped file. - fn setDataOffset(self: *GgufFile) !void { - const base_off = self.off; - - assert(self.left_kv == 0 and self.left_tensors == self.header.tensor_count); - - for (0..self.left_tensors) |_| try self.skipTensor(); - const padding: usize = getAlignmentPadding(self.alignment, self.off); - self.file.data_offset = self.off + padding; - - try self.file.file.seekTo(base_off); - self.off = base_off; - } - - pub fn skipTensor(self: *GgufFile) !void { - try self.skipString(); // Skip name - const num_dim: u32 = try self.readInt(u32); - // dimensions, type, and offset. - try self.skipBytes(8 * num_dim + 4 + 8); - } - - /// Parses the next tensor entry. - /// Returns error.EndOfMetadata if there are no longer tensor metadata to process in this GGUF file. - pub fn readTensorInfo(self: *GgufFile, allocator: std.mem.Allocator) !GgufTensorInfo { - if (self.left_tensors == 0 or self.left_kv != 0) { - return error.EndOfMetadata; - } - - // We want to return tensor data with offsets relative to the start - // of the file, so that the user of the API is able to access tensors - // as it iterates over them. To do so, we need to perform a full - // scan if this is the first tensor info we are reading. - // TODO: explicitly set the data offset in - if (self.file.data_offset == 0) try self.setDataOffset(); - self.left_tensors -= 1; - const name = try self.readString(allocator); - const num_dim = try self.readInt(u32); - assert(@as(usize, @intCast(num_dim)) <= GGUF_TENSOR_MAX_DIM); - // Read the dimentions; unused dimensions are left `undefined`. - // Note: we reverse the order of the dimensions to match zml convention. - var dims: [GGUF_TENSOR_MAX_DIM]i64 = undefined; - var num_weights: usize = 1; - for (0..num_dim) |j| { - const d = try self.readInt(u64); - dims[num_dim - 1 - j] = @intCast(d); - num_weights *= d; - } - const t: TensorType = try self.readTensorType(); - const start = try self.readInt(u64); - // To accurately calculate the bytes used by this tensor on the GGUF - // file, we need to take into account that quantization methods store - // tensors as block of N weights. So first of all we need to understand - // the number of padding weights (since the last block may have just - // fewer weights stored inside, but still requires to be stored to its full - // length). Then we can do the math to see how many blocks we need, and - // multiply by the block size to obtain the final total size. - const tf = t.getFeatures(); - const byte_len: usize = (std.math.divCeil(usize, num_weights, tf.items_per_block) catch unreachable) * tf.bytes_per_block; - return .{ - .name = name, - .t = t, - .rank = num_dim, - .dims = dims, - .start = start, - .byte_len = byte_len, - .num_weights = num_weights, - }; - } -}; - -/// Given an offset or a length, returns the padding needed to align it to alignment. -fn getAlignmentPadding(alignment: usize, offset: usize) usize { - return @rem((alignment - @rem(offset, alignment)), alignment); -} diff --git a/zml/aio/torch.zig b/zml/aio/torch.zig deleted file mode 100644 index ff0d1163..00000000 --- a/zml/aio/torch.zig +++ /dev/null @@ -1,42 +0,0 @@ -const asynk = @import("async"); -const std = @import("std"); -const zml = @import("../zml.zig"); - -const eval = @import("torch/eval.zig"); -const py = @import("torch/py.zig"); -const File = @import("torch/file.zig").File; - -const StringBuilder = std.ArrayListUnmanaged(u8); -const log = std.log.scoped(.@"zml/aio"); - -test { - std.testing.refAllDecls(@This()); - std.testing.refAllDecls(eval); - std.testing.refAllDecls(py); - std.testing.refAllDecls(File); -} - -/// Opens and loads a BufferStore from the torch file at the given path. -pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore { - const file = asynk.File.open(path, .{}) catch |err| { - log.err("Failed to open {s}: {}", .{ path, err }); - return err; - }; - errdefer file.close() catch unreachable; - - // Temporary memory needed to parse the pytorch file. - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - const tmp_alloc = arena.allocator(); - - const mmap_file = try zml.aio.MemoryMappedFile.init(file); - var torch_file = try File.init(tmp_alloc, mmap_file); - - const ops = try torch_file.parsePickle(tmp_alloc); - const py_values = try eval.evaluate(tmp_alloc, ops, true); - - // file ownership is transferred to the BufferStore - var res = try zml.aio.BufferStore.init(allocator, &.{torch_file.buffer_file}); - try torch_file.parseModel(py_values, &res); - return res; -} diff --git a/zml/aio/torch/eval.zig b/zml/aio/torch/eval.zig deleted file mode 100644 index c7085a16..00000000 --- a/zml/aio/torch/eval.zig +++ /dev/null @@ -1,435 +0,0 @@ -const std = @import("std"); -const stdx = @import("stdx"); - -const py = @import("py.zig"); -const pickle = @import("pickle.zig"); - -const MAX_DEPTH: usize = 250; -const MAX_PROTOCOL: u8 = 5; - -pub const PickleMemo = struct { - map: std.AutoHashMap(u32, py.Any), - - pub fn init(allocator: std.mem.Allocator) PickleMemo { - return .{ - .map = std.AutoHashMap(u32, py.Any).init(allocator), - }; - } - - pub fn deinit(self: *PickleMemo) void { - const allocator = self.map.allocator; - var iterator = self.map.iterator(); - defer iterator.deinit(); - while (iterator.next()) |entry| { - entry.value_ptr.deinit(allocator); - } - self.map.deinit() catch unreachable; - self.* = undefined; - } - - pub fn resolve(self: *PickleMemo, allocator: std.mem.Allocator, op: py.Any, recursive: bool) !py.Any { - var used_op = op; - while (used_op == .ref) { - var count: usize = 0; - const val = self.map.get(op.ref) orelse { - return error.BadMemoRef; - }; - if (!recursive) { - return val.clone(allocator); - } - - count += 1; - if (count >= MAX_DEPTH or val != .ref) { - used_op = try val.clone(allocator); - break; - } - used_op = val; - } - if (used_op.containsRef()) { - switch (used_op) { - .app, .object, .global => |v| { - if (v.member.containsRef()) { - v.member = try self.resolve(allocator, v.member, recursive); - } - for (v.args) |*item| { - if (item.containsRef()) { - item.* = try self.resolve(allocator, item.*, recursive); - } - } - }, - .set_state => |v| { - if (v.obj.containsRef()) { - v.obj = try self.resolve(allocator, v.obj, recursive); - } - if (v.state.containsRef()) { - v.state = try self.resolve(allocator, v.state, recursive); - } - }, - .pers_id => |v| { - if (v.ref.containsRef()) { - v.ref = try self.resolve(allocator, v.ref, recursive); - } - }, - .seq => |*v| { - for (v.values) |*item| { - if (item.containsRef()) { - item.* = try self.resolve(allocator, item.*, recursive); - } - } - }, - else => {}, - } - } - return used_op; - } - - pub fn insert(self: *PickleMemo, mid: u32, val: py.Any) !void { - _ = try self.map.fetchPut(mid, val); - } - - pub fn resolveMut(self: *PickleMemo, op: *py.Any, recursive: bool) !*py.Any { - if (op.* != .ref) return op; - var lastmid = op.ref; - var count: usize = 0; - var val = self.map.get(lastmid) orelse { - return error.BadMemoRef; - }; - while (val == .ref) { - lastmid = val.ref; - if (!recursive) { - break; - } - count += 1; - if (count >= MAX_DEPTH) { - break; - } - val = self.map.get(lastmid) orelse { - return error.BadMemoRef; - }; - } - return (self.map.getPtr(lastmid) orelse { - return error.BadMemoRef; - }); - } - - const MemoError = py.Any.UnpickleError || error{BadMemoRef}; - - pub fn resolveAllRefsIter(self: *PickleMemo, allocator: std.mem.Allocator, depth: usize, vals: []py.Any, fix_values: bool) MemoError![]py.Any { - if (depth >= MAX_DEPTH) { - return vals; - } - const res = try allocator.alloc(py.Any, vals.len); - for (vals, 0..) |v, i| { - res[i] = try self.resolveAllRefs(allocator, depth + 1, v, fix_values); - } - return res; - } - - pub fn resolveAllRefs(self: *PickleMemo, allocator: std.mem.Allocator, depth: usize, val: py.Any, fix_values: bool) !py.Any { - var output: py.Any = switch (val) { - .ref => try self.resolve(allocator, val, true), - inline .app, .object, .global => |v, tag| @unionInit(py.Any, @tagName(tag), try py.Object.init( - allocator, - try self.resolveAllRefs(allocator, depth + 1, v.member, fix_values), - try self.resolveAllRefsIter(allocator, depth + 1, v.args, fix_values), - try self.resolveAllRefsIter(allocator, depth + 1, v.kwargs, fix_values), - )), - .set_state => |v| .{ .set_state = try py.SetState.init( - allocator, - try self.resolveAllRefs(allocator, depth + 1, v.obj, fix_values), - try self.resolveAllRefs(allocator, depth + 1, v.state, fix_values), - ) }, - .seq => |v| .{ .seq = .{ .type = v.type, .values = try self.resolveAllRefsIter(allocator, depth + 1, v.values, fix_values) } }, - .pers_id => |v| .{ .pers_id = try py.PersId.init(allocator, try self.resolveAllRefs(allocator, depth + 1, v.ref, fix_values)) }, - else => try val.clone(allocator), - }; - if (fix_values) { - output = try output.coerceFromRaw(allocator); - } - return output; - } -}; - -pub fn evaluate(arena: std.mem.Allocator, x: []const pickle.Op, resolve_refs: bool) ![]const py.Any { - var stack = std.ArrayList(py.Any).init(arena); - var memo = PickleMemo.init(arena); - - for (x) |op| { - switch (op) { - .mark => try stack.append(.{ .raw = op }), - .frame => {}, - .stop => break, - .pop => _ = try pop(&stack), - .pop_mark => _ = try popMark(&stack), - .dup => if (stack.getLastOrNull()) |item| - try stack.append(try item.clone(arena)) - else - return error.CannotDupEmptyStack, - .persid => |v| try stack.append(.{ .pers_id = try py.PersId.init(arena, .{ .string = try arena.dupe(u8, v) }) }), - .binpersid => try stack.append(.{ .pers_id = try py.PersId.init(arena, try pop(&stack)) }), - .reduce => try stack.append(.{ .global = blk: { - var args = try pop(&stack); - args = try memo.resolve(arena, args, true); - if (args != .seq) return error.InvalidInput; - var func = try pop(&stack); - func = try memo.resolve(arena, func, true); - break :blk try py.Object.init(arena, func, args.seq.values, &.{}); - } }), - .build => try stack.append(blk: { - const args = try memo.resolve(arena, try pop(&stack), true); - const member = try memo.resolve(arena, try pop(&stack), true); - break :blk .{ .set_state = try py.SetState.init(arena, member, args) }; - }), - .empty_dict => try stack.append(.{ .seq = .{ .type = .dict, .values = &[_]py.Any{} } }), - .get => |v| try stack.append(.{ .ref = v }), - .empty_list => try stack.append(.{ .seq = .{ .type = .list, .values = &[_]py.Any{} } }), - .put => |v| { - try memo.insert(v, try pop(&stack)); - try stack.append(.{ .ref = v }); - }, - .tuple => try stack.append(blk: { - const popped = try popMark(&stack); - break :blk .{ .seq = .{ .type = .tuple, .values = try arena.dupe(py.Any, popped) } }; - }), - .empty_tuple => try stack.append(.{ .seq = .{ .type = .tuple, .values = &[_]py.Any{} } }), - .setitem => { - const v = try memo.resolve(arena, try pop(&stack), true); - const k = try memo.resolve(arena, try pop(&stack), true); - const top = try lastMut(&stack); - const rtop = try memo.resolveMut(top, true); - switch (rtop.*) { - .global => |obj| { - try append(arena, &obj.kwargs, &.{ k, v }); - }, - .seq => |*dict| { - if (dict.type != .dict) return error.BadStackTopForSetItem; - try append(arena, &dict.values, &.{ k, v }); - }, - else => { - return error.BadStackTopForSetItem; - }, - } - }, - .setitems => { - const popped = try memo.resolveAllRefsIter(arena, 0, try popMark(&stack), true); - const top = try lastMut(&stack); - const rtop = try memo.resolveMut(top, true); - switch (rtop.*) { - .global => |obj| { - try append(arena, &obj.kwargs, popped); - }, - .seq => |*dict| { - if (dict.type != .dict) return error.BadStackTopForSetItems; - try append(arena, &dict.values, popped); - }, - else => { - return error.BadStackTopForSetItems; - }, - } - }, - .proto => |proto| stdx.debug.assert(proto <= MAX_PROTOCOL, "Unsupported protocol {d}", .{proto}), - .tuple1 => try stack.append(blk: { - const tup_values = try arena.alloc(py.Any, 1); - tup_values[0] = try pop(&stack); - break :blk .{ .seq = .{ .type = .tuple, .values = tup_values } }; - }), - .tuple2 => try stack.append(blk: { - const tup_values = try arena.alloc(py.Any, 2); - inline for (0..2) |i| tup_values[(tup_values.len - 1) - i] = try pop(&stack); - break :blk .{ .seq = .{ .type = .tuple, .values = tup_values } }; - }), - .tuple3 => try stack.append(blk: { - const tup_values = try arena.alloc(py.Any, 3); - inline for (0..3) |i| tup_values[(tup_values.len - 1) - i] = try pop(&stack); - break :blk .{ .seq = .{ .type = .tuple, .values = tup_values } }; - }), - .append => { - const v = try pop(&stack); - const top = try lastMut(&stack); - const rtop = try memo.resolveMut(top, true); - switch (rtop.*) { - .global => |obj| { - // can this happen ? - try append(arena, &obj.args, &.{v}); - }, - .seq => |*seq| { - if (seq.type != .list) return error.BadStackTopForAppend; - try append(arena, &seq.values, &.{v}); - }, - else => { - return error.BadStackTopForAppend; - }, - } - }, - .appends => { - const postmark = try popMark(&stack); - const top = try lastMut(&stack); - const rtop = try memo.resolveMut(top, true); - switch (rtop.*) { - .global => try append(arena, &rtop.global.args, postmark), - .seq => |*seq| { - if (seq.type != .list) return error.BadStackTopForAppend; - try append(arena, &seq.values, postmark); - }, - else => { - return error.BadStackTopForAppends; - }, - } - }, - .dict => try stack.append(.{ .seq = .{ - .type = .dict, - .values = try arena.dupe(py.Any, try popMark(&stack)), - } }), - .list => try stack.append(.{ .seq = .{ - .type = .list, - .values = try arena.dupe(py.Any, try popMark(&stack)), - } }), - .inst => |v| try stack.append(.{ .object = try py.Object.init( - arena, - try py.tuple(&.{ .{ .string = v.module }, .{ .string = v.class } }).clone(arena), - try arena.dupe(py.Any, try popMark(&stack)), - &.{}, - ) }), - .obj => try stack.append(blk: { - const mark = try findMark(&stack); - const args = try arena.dupe(py.Any, stack.items[mark + 2 ..]); - const member = stack.items[mark + 1]; - break :blk .{ .object = try py.Object.init(arena, member, args, &.{}) }; - }), - .newobj => try stack.append(blk: { - const args = try arena.alloc(py.Any, 1); - args[0] = try pop(&stack); - break :blk .{ .object = try py.Object.init(arena, try pop(&stack), args, &.{}) }; - }), - .empty_set => try stack.append(.{ .seq = .{ .type = .set, .values = &[_]py.Any{} } }), - .additems => { - const postmark = try popMark(&stack); - const top = try lastMut(&stack); - const rtop = try memo.resolveMut(top, true); - switch (rtop.*) { - .seq => |*seq| { - if (seq.type != .set) return error.BadStackTopForAppend; - try append(arena, &seq.values, postmark); - }, - else => { - return error.BadStackTopForAppends; - }, - } - }, - .frozenset => try stack.append(.{ .seq = .{ - .type = .frozen_set, - .values = try arena.dupe(py.Any, try popMark(&stack)), - } }), - .newobj_ex => try stack.append(blk: { - const kwargs, const args, const cls = .{ try pop(&stack), try pop(&stack), try pop(&stack) }; - break :blk .{ .object = try py.Object.init(arena, cls, args.seq.values, kwargs.seq.values) }; - }), - .stack_global => try stack.append(blk: { - const gn, const mn = .{ - try memo.resolve(arena, try pop(&stack), true), - try memo.resolve(arena, try pop(&stack), true), - }; - const new_seq: py.Sequence = .{ .type = .tuple, .values = try arena.dupe(py.Any, &.{ gn, mn }) }; - break :blk .{ .object = try py.Object.init(arena, .{ .seq = new_seq }, &.{}, &.{}) }; - }), - .memoize => { - const item = stack.getLastOrNull() orelse { - return error.StackUnderrun; - }; - try memo.insert(@intCast(memo.map.count()), try item.clone(arena)); - }, - else => try stack.append(.{ .raw = try op.clone(arena) }), - } - } - if (resolve_refs) { - return try memo.resolveAllRefsIter(arena, 0, stack.items, true); - } - return stack.toOwnedSlice(); -} - -fn append(allocator: std.mem.Allocator, current: *[]py.Any, values: []const py.Any) !void { - var array_list = std.ArrayListUnmanaged(py.Any).fromOwnedSlice(current.*); - try array_list.appendSlice(allocator, values); - current.* = array_list.items; -} - -test evaluate { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const allocator = arena.allocator(); - const file = try std.fs.cwd().openFile("zml/aio/torch/simple_test_4.pickle", .{ .mode = .read_only }); - var buffered_reader = std.io.bufferedReader(file.reader()); - const ops = try pickle.parse(allocator, buffered_reader.reader(), 4096); - - const vals = try evaluate(allocator, ops, true); - defer allocator.free(vals); - - try std.testing.expect(vals.len == 1); - try std.testing.expect(vals[0] == .seq); - try std.testing.expect(vals[0].seq.type == .dict); - const entries = vals[0].seq.values; - const expected: []const py.Any = &.{ - // Key, followed by its value - .{ .string = "hello" }, .{ .string = "world" }, - .{ .string = "int" }, .{ .int64 = 1 }, - .{ .string = "float" }, .{ .float64 = 3.141592 }, - .{ .string = "list" }, - .{ - .seq = .{ - .type = .list, - .values = @constCast(&[_]py.Any{ - .{ .int64 = 255 }, - .{ .int64 = 1234 }, - .{ .int64 = -123 }, - .{ .int64 = 1_000_000_000 }, - .{ .int64 = 999_000_000_000 }, - .{ .bigint = (try std.math.big.int.Managed.initSet(allocator, 999_000_000_000_000_000_000_000_000_000)).toConst() }, - }), - }, - }, - .{ .string = "bool" }, .{ .boolval = false }, - .{ .string = "tuple" }, - .{ .seq = .{ - .type = .tuple, - .values = @constCast(&[_]py.Any{ - .{ .string = "a" }, - .{ .int64 = 10 }, - }), - } }, - }; - - try std.testing.expectEqualDeep(expected, entries); -} - -pub fn pop(values: *std.ArrayList(py.Any)) error{StackUnderrun}!py.Any { - if (values.items.len == 0) { - return error.StackUnderrun; - } - return values.pop().?; -} - -fn popMark(values: *std.ArrayList(py.Any)) ![]py.Any { - const mark = try findMark(values); - const popping = values.items[mark + 1 ..]; - values.shrinkRetainingCapacity(mark); - return popping; -} - -fn lastMut(values: *std.ArrayList(py.Any)) !*py.Any { - if (values.items.len == 0) { - return error.UnexpectedEmptyStack; - } - return &values.items[values.items.len - 1]; -} - -fn findMark(values: *std.ArrayList(py.Any)) !usize { - const len = values.items.len; - for (0..len) |i| { - const idx = (len - 1) - i; - const val = values.items[idx]; - if (val == .raw and val.raw == .mark) { - return idx; - } - } - return error.MarkNotFound; -} diff --git a/zml/aio/torch/file.zig b/zml/aio/torch/file.zig deleted file mode 100644 index b3794bbb..00000000 --- a/zml/aio/torch/file.zig +++ /dev/null @@ -1,681 +0,0 @@ -const asynk = @import("async"); -const std = @import("std"); -const stdx = @import("stdx"); - -const zml = @import("../../zml.zig"); -const pickle = @import("pickle.zig"); -const py = @import("py.zig"); -const eval = @import("eval.zig"); -const HostBuffer = zml.HostBuffer; - -const testing = std.testing; -const log = std.log.scoped(.@"zml/aio"); - -// TODO(cryptodeal): use zml.aio.PrefixBuilder instead -const StringBuilder = std.ArrayListUnmanaged(u8); - -test { - std.testing.refAllDecls(@This()); - std.testing.refAllDecls(File); -} - -pub const File = struct { - buffer_file: zml.aio.MemoryMappedFile, - /// Map names to sub file - file_map: std.StringArrayHashMapUnmanaged(FileEntry) = .{}, - tar_file: ?TarStream = null, - is_zip_file: bool, - zip_prefix: []const u8 = &.{}, - pickle_subfile: struct { start: u64 = 0, len: usize }, - - pub const FileEntry = struct { - version_needed_to_extract: u16, - flags: u16, - compression_method: std.zip.CompressionMethod, - last_modification_time: u16, - last_modification_date: u16, - header_zip_offset: u64, - crc32: u32, - filename_len: u32, - compressed_size: u64, - uncompressed_size: u64, - file_offset: u64, - - pub fn init(entry: anytype) FileEntry { - return .{ - .version_needed_to_extract = entry.version_needed_to_extract, - .flags = @as(u16, @bitCast(entry.flags)), - .compression_method = entry.compression_method, - .last_modification_time = entry.last_modification_time, - .last_modification_date = entry.last_modification_date, - .header_zip_offset = entry.header_zip_offset, - .crc32 = entry.crc32, - .filename_len = entry.filename_len, - .compressed_size = entry.compressed_size, - .uncompressed_size = entry.uncompressed_size, - .file_offset = entry.file_offset, - }; - } - }; - - const magic = "PK\x03\x04"; - - pub fn fromTarFile(allocator: std.mem.Allocator, mapped: zml.aio.MemoryMappedFile, file: std.tar.Iterator(asynk.File.Reader).File) !File { - const tar_file = try TarStream.init(file); - const file_magic = try tar_file.reader().readBytesNoEof(magic.len); - try tar_file.seekTo(0); - var res: File = .{ - .buffer_file = mapped, - .tar_file = tar_file, - .is_zip_file = std.mem.eql(u8, &file_magic, magic), - .pickle_subfile = .{ .len = try tar_file.getEndPos() }, - }; - if (res.is_zip_file) { - try res.parseZipHeaders(allocator, tar_file.seekableStream()); - } - return res; - } - - pub fn init(allocator: std.mem.Allocator, mmap_file: zml.aio.MemoryMappedFile) !File { - const file_magic = try mmap_file.file.reader().readBytesNoEof(magic.len); - try mmap_file.file.seekTo(0); - var res: File = .{ - .buffer_file = mmap_file, - .is_zip_file = std.mem.eql(u8, &file_magic, magic), - .pickle_subfile = .{ .len = mmap_file.data.len }, - }; - - if (res.is_zip_file) { - try res.parseZipHeaders(allocator, mmap_file.file.seekableStream()); - } - return res; - } - - pub fn close(self: *File) void { - self.buffer_file.deinit(); - } - - pub fn parsePickle(self: *File, allocator: std.mem.Allocator) ![]const pickle.Op { - return if (self.tar_file) |tar_file| { - try tar_file.seekTo(self.pickle_subfile.start); - var buffered = std.io.bufferedReader(tar_file.reader()); - return try pickle.parse(allocator, buffered.reader(), self.pickle_subfile.len); - } else { - const file = self.buffer_file.file; - try file.seekTo(self.pickle_subfile.start); - var buffered = std.io.bufferedReader(file.reader()); - return try pickle.parse(allocator, buffered.reader(), self.pickle_subfile.len); - }; - } - - fn parseZipHeaders(self: *File, allocator: std.mem.Allocator, seekable_stream: anytype) !void { - var file_map: std.StringArrayHashMapUnmanaged(FileEntry) = .{}; - - var iter = try std.zip.Iterator(@TypeOf(seekable_stream)).init(seekable_stream); - var filename_buf: [std.fs.max_path_bytes]u8 = undefined; - while (try iter.next()) |entry| { - const filename = filename_buf[0..entry.filename_len]; - try seekable_stream.seekTo(entry.header_zip_offset + @sizeOf(std.zip.CentralDirectoryFileHeader)); - const len = try seekable_stream.context.reader().readAll(filename); - if (len != filename.len) return error.ZipBadFileOffset; - if (isBadFilename(filename)) return error.ZipBadFilename; - std.mem.replaceScalar(u8, filename, '\\', '/'); // normalize path separators - try file_map.put(allocator, try allocator.dupe(u8, filename), FileEntry.init(entry)); - } - - self.file_map = file_map; - var file_iter = file_map.iterator(); - while (file_iter.next()) |e| { - const entry = e.value_ptr.*; - const filename = e.key_ptr.*; - if (!std.mem.endsWith(u8, filename, "data.pkl")) continue; - - self.zip_prefix = filename[0 .. filename.len - "data.pkl".len]; - - const local_data_header_offset: u64 = local_data_header_offset: { - switch (entry.compression_method) { - .store => {}, - .deflate => { - // TODO(cryptodeal): handle decompress - @panic("TODO support use of `deflate`"); - }, - else => @panic("TODO support other modes of compression"), - } - const local_header = blk: { - try seekable_stream.seekTo(entry.file_offset); - break :blk try seekable_stream.context.reader().readStructEndian(std.zip.LocalFileHeader, .little); - }; - if (!std.mem.eql(u8, &local_header.signature, &std.zip.local_file_header_sig)) - return error.ZipBadFileOffset; - if (local_header.version_needed_to_extract != entry.version_needed_to_extract) - return error.ZipMismatchVersionNeeded; - if (local_header.last_modification_time != entry.last_modification_time) - return error.ZipMismatchModTime; - if (local_header.last_modification_date != entry.last_modification_date) - return error.ZipMismatchModDate; - - if (@as(u16, @bitCast(local_header.flags)) != entry.flags) - return error.ZipMismatchFlags; - if (local_header.crc32 != 0 and local_header.crc32 != entry.crc32) - return error.ZipMismatchCrc32; - if (local_header.compressed_size != 0 and - local_header.compressed_size != entry.compressed_size) - return error.ZipMismatchCompLen; - if (local_header.uncompressed_size != 0 and - local_header.uncompressed_size != entry.uncompressed_size) - return error.ZipMismatchUncompLen; - if (local_header.filename_len != entry.filename_len) - return error.ZipMismatchFilenameLen; - - break :local_data_header_offset @as(u64, local_header.filename_len) + - @as(u64, local_header.extra_len); - }; - - const local_data_file_offset: u64 = - @as(u64, entry.file_offset) + - @as(u64, @sizeOf(std.zip.LocalFileHeader)) + - local_data_header_offset; - self.pickle_subfile = .{ .start = local_data_file_offset, .len = entry.uncompressed_size }; - return; - } - - log.err("Could not find file ending in `data.pkl` in archive", .{}); - return error.PickleNotFound; - } - - fn basicTypeCheck(object: *const py.Object, module: []const u8, class: []const u8) bool { - return switch (object.member) { - .raw => |raw| return (std.mem.eql(u8, module, raw.global.module) and - std.mem.eql(u8, class, raw.global.class)), - else => false, - }; - } - - pub fn parseModel(self: File, values: []const py.Any, store: *zml.aio.BufferStore) !void { - var prefix_buf: [1024]u8 = undefined; - const allocator = store.arena.allocator(); - for (values) |item| { - try self.parseValue(allocator, store, StringBuilder.initBuffer(&prefix_buf), item); - } - } - - pub fn parseValue(self: File, allocator: std.mem.Allocator, store: *zml.aio.BufferStore, prefix: StringBuilder, v: py.Any) !void { - // log.warn("Parsing {}", .{v}); - switch (v) { - .app, .object, .global => |object| { - if (!(try self.parseTorchGlobal(allocator, store, prefix, v))) { - try self.parseValue(allocator, store, prefix, object.member); - for (object.args) |item| { - try self.parseValue(allocator, store, prefix, item); - } - if (object.kwargs.len % 2 != 0) return error.InvalidInput; - const n_kwargs = @divExact(object.kwargs.len, 2); - - for (0..n_kwargs) |i| { - const key, const val = object.kwargs[2 * i ..][0..2].*; - // kwargs can only be keyed by string. - if (key != .string) return error.InvalidInput; - // Handle Pytorch specific fields - const s = key.string; - if (std.mem.eql(u8, s, "_modules") or std.mem.eql(u8, s, "_parameters") or std.mem.eql(u8, s, "_buffers")) { - try self.parseValue(allocator, store, prefix, val); - } else { - var new_prefix = prefix; - if (prefix.items.len > 0) { - new_prefix.appendAssumeCapacity('.'); - } - new_prefix.appendSliceAssumeCapacity(s); - try self.parseValue(allocator, store, new_prefix, val); - } - } - } - }, - .set_state => |set_state| { - // `set_state` contains info about python struct being constructed - switch (set_state.obj) { - .object => |obj| switch (obj.member) { - .raw => |raw| switch (raw) { - .global => |global| { - // in this case, we can capture the name of the python type - // which can be used for codegen (e.g. `torch.nn.modules.conv.Conv2d`) - var new_prefix = prefix; - if (prefix.items.len > 0) { - new_prefix.appendAssumeCapacity('.'); - } - new_prefix.appendSliceAssumeCapacity("_gen_type_helper"); - const key = try allocator.dupe(u8, new_prefix.items); - const d = try store._metadata.getOrPut(allocator, key); - if (d.found_existing) { - log.err("Duplicate key: {s}", .{new_prefix.items}); - allocator.free(key); - } else { - const val = try std.mem.join(allocator, ".", &.{ global.module, global.class }); - d.value_ptr.* = .{ .string = val }; - } - }, - else => try self.parseValue(allocator, store, prefix, set_state.obj), // parse normally - }, - else => try self.parseValue(allocator, store, prefix, set_state.obj), // parse normally - }, - else => try self.parseValue(allocator, store, prefix, set_state.obj), // parse normally - } - try self.parseValue(allocator, store, prefix, set_state.state); - }, - .pers_id => |pers_id| try self.parseValue(allocator, store, prefix, pers_id.ref), - .seq => |seq| { - switch (seq.type) { - .list, .tuple, .set, .frozen_set => { - if (seq.values.len == 0) return; - var valid_slice = true; - switch (seq.values[0]) { - inline .int64, .float64, .boolval => |val0, tag| { - const ItemType = switch (tag) { - .int64 => i64, - .float64 => f64, - .boolval => bool, - else => unreachable, - }; - var values: std.ArrayListUnmanaged(ItemType) = .{}; - try values.append(allocator, val0); - for (seq.values[1..], 1..) |val, i| { - if (std.meta.activeTag(val) != tag) valid_slice = false; - if (valid_slice) { - try values.append(allocator, @field(val, @tagName(tag))); - } else { - var new_prefix = prefix; - if (prefix.items.len > 0) { - new_prefix.appendAssumeCapacity('.'); - } - new_prefix.items.len += std.fmt.formatIntBuf(new_prefix.unusedCapacitySlice(), i, 10, .lower, .{}); - try self.parseValue(allocator, store, new_prefix, val); - } - } - - if (valid_slice) { - try store._metadata.put( - allocator, - try allocator.dupe(u8, prefix.items), - try zml.aio.Metadata.copySlice(allocator, values.items), - ); - } else { - for (values.items, 0..) |val, i| { - var new_prefix = prefix; - if (prefix.items.len > 0) { - new_prefix.appendAssumeCapacity('.'); - } - new_prefix.items.len += std.fmt.formatIntBuf(new_prefix.unusedCapacitySlice(), i, 10, .lower, .{}); - const new_tag = switch (tag) { - .int64 => "int", - .float64 => "float", - .boolval => "bool", - else => unreachable, // we are already inside a switch - }; - try store._metadata.put(allocator, try allocator.dupe(u8, new_prefix.items), @unionInit(zml.aio.Metadata, new_tag, val)); - } - } - }, - else => { - for (seq.values, 0..) |item, i| { - var new_prefix = prefix; - if (v.isPrimitive()) { - if (prefix.items.len > 0) { - new_prefix.appendAssumeCapacity('.'); - } - new_prefix.items.len += std.fmt.formatIntBuf(new_prefix.unusedCapacitySlice(), i, 10, .lower, .{}); - } - try self.parseValue(allocator, store, new_prefix, item); - } - }, - } - }, - .dict => { - const n = @divExact(seq.values.len, 2); - log.debug("found dict with {} entries", .{n}); - for (0..n) |i| { - const key, const val = seq.values[2 * i ..][0..2].*; - switch (key) { - .string => |s| { - // Handle Pytorch specific fields. - if (std.mem.eql(u8, s, "_modules") or std.mem.eql(u8, s, "_parameters") or std.mem.eql(u8, s, "_buffers")) { - try self.parseValue(allocator, store, prefix, val); - } else { - var new_prefix = prefix; - if (prefix.items.len > 0) { - new_prefix.appendAssumeCapacity('.'); - } - new_prefix.appendSliceAssumeCapacity(s); - - try self.parseValue(allocator, store, new_prefix, val); - } - }, - .int64 => |int| { - var new_prefix = prefix; - if (prefix.items.len > 0) { - new_prefix.appendAssumeCapacity('.'); - } - new_prefix.items.len += std.fmt.formatIntBuf(new_prefix.unusedCapacitySlice(), int, 10, .lower, .{}); - try self.parseValue(allocator, store, new_prefix, val); - }, - inline else => |_, tag| { - log.debug("Ignoring unsupported key type found in torch file: {s}", .{@tagName(tag)}); - continue; - }, - } - } - }, - } - }, - .bytes => |val| { - const key = try allocator.dupe(u8, prefix.items); - const d = try store._metadata.getOrPut(allocator, key); - if (d.found_existing) { - log.warn("Duplicate key: {s}", .{prefix.items}); - allocator.free(key); - } else d.value_ptr.* = .{ .string = val }; - }, - inline .float64, .int64, .boolval, .bigint, .string => |val| { - const key = try allocator.dupe(u8, prefix.items); - const d = try store._metadata.getOrPut(allocator, key); - if (d.found_existing) { - log.warn("Duplicate key: {s}", .{prefix.items}); - allocator.free(key); - } else { - d.value_ptr.* = zml.aio.Metadata.wrap(val); - } - }, - else => {}, - } - } - - fn parseTorchGlobal(self: File, allocator: std.mem.Allocator, store: *zml.aio.BufferStore, prefix: StringBuilder, v: py.Any) !bool { - return switch (v) { - .global => |object| { - if (try self.parseTensor(allocator, object)) |host_buffer| { - const key = try allocator.dupe(u8, prefix.items); - const entry = try store.buffers.getOrPut(allocator, key); - if (entry.found_existing) { - log.warn("Duplicate key: {s}", .{prefix.items}); - allocator.free(key); - } - entry.value_ptr.* = host_buffer; - return true; - } else if (basicTypeCheck(object, "torch", "Size")) { - const size = object.args; - const key = try allocator.dupe(u8, prefix.items); - const entry = try store._metadata.getOrPut(allocator, key); - if (entry.found_existing) { - log.warn("Duplicate key: {s}", .{prefix.items}); - allocator.free(key); - } - const d = try allocator.alloc(i64, size.len); - for (d, 0..) |*di, i| di.* = size[i].int64; - entry.value_ptr.* = .{ .array_int = d }; - return true; - } else if (basicTypeCheck(object, "fractions", "Fraction")) { - const fraction_str = object.args[0].string; - if (std.mem.indexOfScalar(u8, fraction_str, '/')) |split_idx| { - { - var new_prefix = prefix; - new_prefix.appendSliceAssumeCapacity(".numerator"); - try store._metadata.put(allocator, try allocator.dupe(u8, new_prefix.items), .{ .int = try std.fmt.parseInt(i64, fraction_str[0..split_idx], 10) }); - } - { - var new_prefix = prefix; - new_prefix.appendSliceAssumeCapacity(".denominator"); - try store._metadata.put(allocator, try allocator.dupe(u8, new_prefix.items), .{ .int = try std.fmt.parseInt(i64, fraction_str[split_idx + 1 ..], 10) }); - } - return true; - } - } - return false; - }, - else => false, - }; - } - - fn parseTensor(self: File, tmp_allocator: std.mem.Allocator, object: *py.Object) !?zml.HostBuffer { - if (!basicTypeCheck(object, "torch._utils", "_rebuild_tensor_v2")) { - return null; - } - - const args = object.args; - if (args.len < 4 or - args[0] != .pers_id or - args[1] != .int64 or - args[2] != .seq or args[2].seq.type != .tuple or - args[3] != .seq or args[3].seq.type != .tuple) - { - log.err("Unexpected py.Any in call to torch._utils._rebuild_tensor_v2: {}", .{object.*}); - return error.InvalidInput; - } - - const pid: *py.PersId = args[0].pers_id; - var offset: u64 = @intCast(args[1].int64); - const raw_dims: py.Sequence = args[2].seq; - const raw_strides: py.Sequence = args[3].seq; - const dims = try parseDims(raw_dims.values); - var strides = try parseDims(raw_strides.values); - - const dtype, const storage_file = try parseStorage(pid.ref); - // Pytorch store "item" strides, while ZML uses byte strides. - for (strides.slice()) |*s| s.* *= dtype.sizeOf(); - // Same thing for the offset. - offset = offset * dtype.sizeOf(); - - const filename = try std.mem.join(tmp_allocator, "", &.{ self.zip_prefix, "data/", storage_file }); - defer tmp_allocator.free(filename); - - // The offset in the pickle is the offset inside the storage_file. - // But .pt are made of several files, so we need to append the file offset. - const storage = try self.getStorage(filename); - return HostBuffer.fromStridedSlice( - zml.Shape.init(dims.constSlice(), dtype), - storage[offset..], - strides.constSlice(), - ); - } - - fn parseStorage(val: py.Any) !struct { zml.DataType, []const u8 } { - if (val != .seq) return error.InvalidInput; - const sargs = val.seq.values; - if (val.seq.type == .tuple and - sargs.len >= 5 and - sargs[0] == .string and std.mem.eql(u8, sargs[0].string, "storage") and - sargs[1] == .raw and sargs[1].raw == .global and - sargs[2] == .string and - sargs[3] == .string) - { - const op = sargs[1].raw.global; - const storage_file = sargs[2].string; - // const sdev = sargs[3].string; - if (!std.mem.eql(u8, "torch", op.module) or - !std.mem.endsWith(u8, op.class, "Storage")) - return error.InvalidInput; - - return .{ - try storageToDtype(op.class), - storage_file, - }; - } else { - return error.InvalidInput; - } - } - - /// Given the name of one of the files in the .pt tarball, - /// return the slice of the memory-mapped .pt corresponding to it. - fn getStorage(self: File, filename: []const u8) ![]const u8 { - const maybe_entry = self.file_map.get(filename); - if (maybe_entry == null) { - std.log.err("Could not find file ending in `{s}` in archive", .{filename}); - return error.TensorNotFound; - } - const entry = maybe_entry.?; - const base_offset: u64 = if (self.tar_file) |t| t.start else 0; - const file_offset: u64 = base_offset + entry.file_offset; - const file = self.buffer_file.file; - try file.seekTo(entry.file_offset); - const local_header = try file.reader().readStructEndian(std.zip.LocalFileHeader, .little); - - if (!std.mem.eql(u8, &local_header.signature, &std.zip.local_file_header_sig)) - return error.ZipBadFileOffset; - if (local_header.compressed_size != 0 and - local_header.compressed_size != entry.compressed_size) - return error.ZipMismatchCompLen; - if (local_header.uncompressed_size != 0 and - local_header.uncompressed_size != entry.uncompressed_size) - return error.ZipMismatchUncompLen; - if (local_header.filename_len != entry.filename_len) - return error.ZipMismatchFilenameLen; - - const start = file_offset + - @sizeOf(std.zip.LocalFileHeader) + - @as(u64, local_header.filename_len) + - @as(u64, local_header.extra_len); - return self.buffer_file.mappedSlice(start, entry.uncompressed_size); - } - - fn parseDims(values: []py.Any) error{InvalidInput}!zml.Shape.DimsArray { - stdx.debug.assert(values.len <= zml.Tensor.MAX_RANK, "Found Pytorch tensor with unsupported rank {}", .{values.len}); - var result: zml.Shape.DimsArray = .{}; - for (values) |val| { - switch (val) { - .int64 => |d| result.appendAssumeCapacity(d), - else => return error.InvalidInput, - } - } - return result; - } -}; - -/// Convert from a torch.Storage to a `zml.DataType`. -/// TODO: make this future proof, storage type are going to get replaced with torch.UntypedStorage -/// See https://pytorch.org/docs/stable/storage.html -fn storageToDtype(storage_type: []const u8) !zml.DataType { - const torch_type = storage_type[0 .. storage_type.len - "Storage".len]; - const map = std.StaticStringMap(zml.DataType).initComptime(.{ - .{ "Double", .f64 }, - .{ "Float", .f32 }, - .{ "Half", .f16 }, - .{ "Long", .i64 }, - .{ "Int", .i32 }, - .{ "Short", .i16 }, - .{ "Char", .i8 }, - .{ "Byte", .u8 }, - .{ "Bool", .bool }, - .{ "BFloat16", .bf16 }, - .{ "ComplexDouble", .c128 }, - .{ "ComplexFloat", .c64 }, - // QUInt8Storage - // QInt8Storage - // QInt32Storage - // QUInt4x2Storage - // QUInt2x4Storage - }); - - return map.get(torch_type) orelse { - log.err("Unsupported torch storage type: {s}", .{storage_type}); - return error.UnsupportedDataType; - }; -} - -const TarStream = struct { - pub const SeekableStream = std.io.SeekableStream( - TarStream, - asynk.File.SeekError, - asynk.File.GetSeekPosError, - TarStream.seekTo, - TarStream.seekBy, - TarStream.getPos, - TarStream.getEndPos, - ); - - file: std.tar.Iterator(asynk.File.Reader).File, - start: usize, - - pub fn init(file: std.tar.Iterator(asynk.File.Reader).File) !TarStream { - return .{ - .file = file, - .start = try file.parent_reader.context.getPos(), - }; - } - - pub fn reader(file: TarStream) std.tar.Iterator(asynk.File.Reader).File.Reader { - return file.file.reader(); - } - - pub fn seekTo(self: TarStream, offset: u64) !void { - return self.file.parent_reader.context.seekTo(self.start + offset); - } - - pub fn seekBy(self: TarStream, offset: i64) !void { - return self.file.parent_reader.context.seekBy(offset); - } - - pub fn getPos(self: TarStream) !u64 { - return try self.file.parent_reader.context.getPos() - self.start; - } - - pub fn getEndPos(self: TarStream) !u64 { - return self.file.size; - } - - pub fn seekableStream(self: TarStream) TarStream.SeekableStream { - return .{ .context = self }; - } -}; - -test "Read pickle (zipped)" { - // test file created with following python snippet: - // - // import torch - // torch.manual_seed(0) - // model = torch.nn.Conv2d(2, 2, 3, stride=2, padding=[2, 4], dtype=torch.float16) - // tensor = torch.tensor([[2, 4, 3, 2]], dtype=torch.uint8) - // torch.save({ "model": model, "tensor": tensor}, "simple.pt") - const file = try asynk.File.open("zml/aio/torch/simple.pt", .{ .mode = .read_only }); - const mmap_file = try zml.aio.MemoryMappedFile.init(file); - var store = try zml.aio.BufferStore.init(testing.allocator, &.{mmap_file}); - defer store.deinit(); - - { - var tmp_arena = std.heap.ArenaAllocator.init(testing.allocator); - defer tmp_arena.deinit(); - const tmp_alloc = tmp_arena.allocator(); - var torch_file = try File.init(tmp_alloc, mmap_file); - // We don't close the file directly, it will be closed by the store. - - const ops = try torch_file.parsePickle(tmp_alloc); - try std.testing.expectEqual(302, ops.len); - - const py_values = try eval.evaluate(tmp_alloc, ops, true); - try torch_file.parseModel(py_values, &store); - } - - // now we have freed the tmp_arena. - // all data needed should have been copied into the store arena. - try zml.testing.expectEqualShapes( - zml.Shape.init(.{ 1, 4 }, .u8), - store.get("tensor").?.shape(), - ); - try zml.testing.expectEqualShapes( - zml.Shape.init(.{ 2, 2, 3, 3 }, .f16), - store.get("model.weight").?.shape(), - ); - try zml.testing.expectEqualShapes( - zml.Shape.init(.{2}, .f16), - store.get("model.bias").?.shape(), - ); -} - -fn isBadFilename(filename: []const u8) bool { - if (filename.len == 0 or filename[0] == '/') - return true; - - var it = std.mem.splitScalar(u8, filename, '/'); - while (it.next()) |part| { - if (std.mem.eql(u8, part, "..")) - return true; - } - - return false; -} diff --git a/zml/aio/torch/pickle.zig b/zml/aio/torch/pickle.zig deleted file mode 100644 index d9d2bcd5..00000000 --- a/zml/aio/torch/pickle.zig +++ /dev/null @@ -1,1065 +0,0 @@ -const std = @import("std"); - -const log = std.log.scoped(.@"zml/aio"); - -/// All possible pickle operators. -/// Reference: https://github.com/python/cpython/blob/3.13/Lib/pickletools.py -pub const OpCode = enum(u8) { - /// Push an integer or bool. - /// - /// The argument is a newline-terminated decimal literal string. - /// - /// The intent may have been that this always fit in a short Python int, - /// but INT can be generated in pickles written on a 64-bit box that - /// require a Python long on a 32-bit box. The difference between this - /// and LONG then is that INT skips a trailing 'L', and produces a short - /// int whenever possible. - /// - /// Another difference is due to that, when bool was introduced as a - /// distinct type in 2.3, builtin names True and False were also added to - /// 2.2.2, mapping to ints 1 and 0. For compatibility in both directions, - /// True gets pickled as INT + "I01\n", and False as INT + "I00\n". - /// Leading zeroes are never produced for a genuine integer. The 2.3 - /// (and later) unpicklers special-case these and return bool instead; - /// earlier unpicklers ignore the leading "0" and return the int. - int = 'I', - - /// Push a four-byte signed integer. - /// Introduced in protocol 1. - /// - /// This handles the full range of Python (short) integers on a 32-bit - /// box, directly as binary bytes (1 for the opcode and 4 for the integer). - /// If the integer is non-negative and fits in 1 or 2 bytes, pickling via - /// BININT1 or BININT2 saves space. - binint = 'J', - - /// Push a one-byte unsigned integer. - /// Introduced in protocol 1. - /// - /// This is a space optimization for pickling very small non-negative ints, - /// in range(256). - binint1 = 'K', - - /// Push a two-byte unsigned integer. - /// Introduced in protocol 1. - /// - /// This is a space optimization for pickling small positive ints, in - /// range(256, 2**16). Integers in range(256) can also be pickled via - /// BININT2, but BININT1 instead saves a byte. - binint2 = 'M', - - /// Push a long integer. - /// - /// The same as INT, except that the literal ends with 'L', and always - /// unpickles to a Python long. There doesn't seem a real purpose to the - /// trailing 'L'. - /// - /// Note that LONG takes time quadratic in the number of digits when - /// unpickling (this is simply due to the nature of decimal->binary - /// conversion). Proto 2 added linear-time (in C; still quadratic-time - /// in Python) LONG1 and LONG4 opcodes. - long = 'L', - - /// Long integer using one-byte length. - /// Introduced in protocol 2. - /// - /// A more efficient encoding of a Python long; the long1 encoding - long1 = 0x8a, - - /// Long integer using four-byte length. - /// Introduced in protocol 2. - /// - /// A more efficient encoding of a Python long; the long4 encoding - long4 = 0x8b, - - /// Push a Python string object. - /// - /// The argument is a repr-style string, with bracketing quote characters, - /// and perhaps embedded escapes. The argument extends until the next - /// newline character. These are usually decoded into a str instance - /// using the encoding given to the Unpickler constructor. or the default, - /// 'ASCII'. If the encoding given was 'bytes' however, they will be - /// decoded as bytes object instead. - string = 'S', - - /// Push a Python string object. - /// Introduced in protocol 1. - /// - /// There are two arguments: the first is a 4-byte little-endian - /// signed int giving the number of bytes in the string, and the - /// second is that many bytes, which are taken literally as the string - /// content. These are usually decoded into a str instance using the - /// encoding given to the Unpickler constructor. or the default, - /// 'ASCII'. If the encoding given was 'bytes' however, they will be - /// decoded as bytes object instead. - binstring = 'T', - - /// Push a Python string object. - /// Introduced in protocol 1. - /// - /// There are two arguments: the first is a 1-byte unsigned int giving - /// the number of bytes in the string, and the second is that many - /// bytes, which are taken literally as the string content. These are - /// usually decoded into a str instance using the encoding given to - /// the Unpickler constructor. or the default, 'ASCII'. If the - /// encoding given was 'bytes' however, they will be decoded as bytes - /// object instead. - short_binstring = 'U', - - /// Push a Python bytes object. - /// Introduced in protocol 3. - /// - /// There are two arguments: the first is a 4-byte little-endian unsigned int - /// giving the number of bytes, and the second is that many bytes, which are - /// taken literally as the bytes content. - binbytes = 'B', - - /// Push a Python bytes object. - /// Introduced in protocol 3. - /// - /// There are two arguments: the first is a 1-byte unsigned int giving - /// the number of bytes, and the second is that many bytes, which are taken - /// literally as the string content. - short_binbytes = 'C', - - /// Push a Python bytes object. - /// Introduced in protocol 4. - /// - /// There are two arguments: the first is an 8-byte unsigned int giving - /// the number of bytes in the string, and the second is that many bytes, - /// which are taken literally as the string content. - binbytes8 = 0x8e, - - /// Push a Python bytearray object. - /// Introduced in protocol 5. - /// - /// There are two arguments: the first is an 8-byte unsigned int giving - /// the number of bytes in the bytearray, and the second is that many bytes, - /// which are taken literally as the bytearray content. - bytearray8 = 0x96, - - /// Introduced in protocol 5. - next_buffer = 0x97, - - /// Introduced in protocol 5. - readonly_buffer = 0x98, - - none = 'N', - - /// Introduced in protocol 2. - newtrue = 0x88, - - /// Introduced in protocol 2. - newfalse = 0x89, - - /// Push a Python Unicode string object. - /// - /// The argument is a raw-unicode-escape encoding of a Unicode string, - /// and so may contain embedded escape sequences. The argument extends - /// until the next newline character. - unicode = 'V', - - /// Push a Python Unicode string object. - /// Introduced in protocol 4. - /// - /// There are two arguments: the first is a 1-byte little-endian signed int - /// giving the number of bytes in the string. The second is that many - /// bytes, and is the UTF-8 encoding of the Unicode string. - short_binunicode = 0x8c, - - /// Push a Python Unicode string object. - /// Introduced in protocol 1. - /// - /// There are two arguments: the first is a 4-byte little-endian unsigned int - /// giving the number of bytes in the string. The second is that many - /// bytes, and is the UTF-8 encoding of the Unicode string. - binunicode = 'X', - - /// Push a Python Unicode string object. - /// Introduced in protocol 4. - /// - /// There are two arguments: the first is an 8-byte little-endian signed int - /// giving the number of bytes in the string. The second is that many - /// bytes, and is the UTF-8 encoding of the Unicode string. - binunicode8 = 0x8d, - - /// Newline-terminated decimal float literal. - /// - /// The argument is repr(a_float), and in general requires 17 significant - /// digits for roundtrip conversion to be an identity (this is so for - /// IEEE-754 double precision values, which is what Python float maps to - /// on most boxes). - /// - /// In general, FLOAT cannot be used to transport infinities, NaNs, or - /// minus zero across boxes (or even on a single box, if the platform C - /// library can't read the strings it produces for such things -- Windows - /// is like that), but may do less damage than BINFLOAT on boxes with - /// greater precision or dynamic range than IEEE-754 double. - float = 'F', - - /// Float stored in binary form, with 8 bytes of data. - /// Introduced in protocol 1. - /// - /// This generally requires less than half the space of FLOAT encoding. - /// In general, BINFLOAT cannot be used to transport infinities, NaNs, or - /// minus zero, raises an exception if the exponent exceeds the range of - /// an IEEE-754 double, and retains no more than 53 bits of precision (if - /// there are more than that, "add a half and chop" rounding is used to - /// cut it back to 53 significant bits). - binfloat = 'G', - - /// Introduced in protocol 1. - empty_list = ']', - - /// Append an object to a list. - /// - /// Stack before: ... pylist anyobject - /// Stack after: ... pylist+[anyobject] - /// - /// although pylist is really extended in-place. - append = 'a', - - /// Extend a list by a slice of stack objects. - /// Introduced in protocol 1. - /// - /// Stack before: ... pylist markobject stackslice - /// Stack after: ... pylist+stackslice - /// - /// although pylist is really extended in-place. - appends = 'e', - - /// Build a list out of the topmost stack slice, after markobject. - /// - /// All the stack entries following the topmost markobject are placed into - /// a single Python list, which single list object replaces all of the - /// stack from the topmost markobject onward. For example, - /// - /// Stack before: ... markobject 1 2 3 'abc' - /// Stack after: ... [1, 2, 3, 'abc'] - list = 'l', - - /// Introduced in protocol 1. - empty_tuple = ')', - - /// Build a tuple out of the topmost stack slice, after markobject. - /// - /// All the stack entries following the topmost markobject are placed into - /// a single Python tuple, which single tuple object replaces all of the - /// stack from the topmost markobject onward. For example, - /// - /// Stack before: ... markobject 1 2 3 'abc' - /// Stack after: ... (1, 2, 3, 'abc') - tuple = 't', - - /// Build a one-tuple out of the topmost item on the stack. - /// Introduced in protocol 2. - /// - /// This code pops one value off the stack and pushes a tuple of - /// length 1 whose one item is that value back onto it. In other - /// words: - /// - /// stack[-1] = tuple(stack[-1:]) - tuple1 = 0x85, - - /// Build a two-tuple out of the top two items on the stack. - /// Introduced in protocol 2. - /// - /// This code pops two values off the stack and pushes a tuple of - /// length 2 whose items are those values back onto it. In other - /// words: - /// - /// stack[-2:] = [tuple(stack[-2:])] - tuple2 = 0x86, - - /// Build a three-tuple out of the top three items on the stack. - /// Introduced in protocol 2. - /// - /// This code pops three values off the stack and pushes a tuple of - /// length 3 whose items are those values back onto it. In other - /// words: - /// - /// stack[-3:] = [tuple(stack[-3:])] - tuple3 = 0x87, - - /// Introduced in protocol 1. - empty_dict = '}', - - /// Build a dict out of the topmost stack slice, after markobject. - /// - /// All the stack entries following the topmost markobject are placed into - /// a single Python dict, which single dict object replaces all of the - /// stack from the topmost markobject onward. The stack slice alternates - /// key, value, key, value, .... For example, - /// - /// Stack before: ... markobject 1 2 3 'abc' - /// Stack after: ... {1: 2, 3: 'abc'} - dict = 'd', - - /// Add a key+value pair to an existing dict. - /// - /// Stack before: ... pydict key value - /// Stack after: ... pydict - /// - /// where pydict has been modified via pydict[key] = value. - setitem = 's', - - /// Add an arbitrary number of key+value pairs to an existing dict. - /// Introduced in protocol 1. - /// - /// The slice of the stack following the topmost markobject is taken as - /// an alternating sequence of keys and values, added to the dict - /// immediately under the topmost markobject. Everything at and after the - /// topmost markobject is popped, leaving the mutated dict at the top - /// of the stack. - /// - /// Stack before: ... pydict markobject key_1 value_1 ... key_n value_n - /// Stack after: ... pydict - /// - /// where pydict has been modified via pydict[key_i] = value_i for i in - /// 1, 2, ..., n, and in that order. - setitems = 'u', - - /// Introduced in protocol 4. - empty_set = 0x8f, - - /// Add an arbitrary number of items to an existing set. - /// Introduced in protocol 4. - /// - /// The slice of the stack following the topmost markobject is taken as - /// a sequence of items, added to the set immediately under the topmost - /// markobject. Everything at and after the topmost markobject is popped, - /// leaving the mutated set at the top of the stack. - /// - /// Stack before: ... pyset markobject item_1 ... item_n - /// Stack after: ... pyset - /// - /// where pyset has been modified via pyset.add(item_i) = item_i for i in - /// 1, 2, ..., n, and in that order. - additems = 0x90, - - /// Build a frozenset out of the topmost slice, after markobject. - /// Introduced in protocol 4. - /// - /// All the stack entries following the topmost markobject are placed into - /// a single Python frozenset, which single frozenset object replaces all - /// of the stack from the topmost markobject onward. For example, - /// - /// Stack before: ... markobject 1 2 3 - /// Stack after: ... frozenset({1, 2, 3}) - frozenset = 0x91, - - pop = '0', - - dup = '2', - - /// Push markobject onto the stack. - /// - /// markobject is a unique object, used by other opcodes to identify a - /// region of the stack containing a variable number of objects for them - /// to work on. See markobject.doc for more detail. - mark = '(', - - /// Pop all the stack objects at and above the topmost markobject. - /// Introduced in protocol 1. - /// - /// When an opcode using a variable number of stack objects is done, - /// POP_MARK is used to remove those objects, and to remove the markobject - /// that delimited their starting position on the stack. - pop_mark = '1', - - /// Read an object from the memo and push it on the stack. - /// - /// The index of the memo object to push is given by the newline-terminated - /// decimal string following. BINGET and LONG_BINGET are space-optimized - /// versions. - get = 'g', - - /// Read an object from the memo and push it on the stack. - /// Introduced in protocol 1. - /// - /// The index of the memo object to push is given by the 1-byte unsigned - /// integer following. - binget = 'h', - - /// Read an object from the memo and push it on the stack. - /// Introduced in protocol 1. - /// - /// The index of the memo object to push is given by the 4-byte unsigned - /// little-endian integer following. - long_binget = 'j', - - /// Store the stack top into the memo. The stack is not popped. - /// - /// The index of the memo location to write into is given by the newline- - /// terminated decimal string following. BINPUT and LONG_BINPUT are - /// space-optimized versions. - put = 'p', - - /// Store the stack top into the memo. The stack is not popped. - /// Introduced in protocol 1. - /// - /// The index of the memo location to write into is given by the 1-byte - /// unsigned integer following. - binput = 'q', - - /// Store the stack top into the memo. The stack is not popped. - /// Introduced in protocol 1. - /// - /// The index of the memo location to write into is given by the 4-byte - /// unsigned little-endian integer following. - long_binput = 'r', - - /// Store the stack top into the memo. The stack is not popped. - /// Introduced in protocol 4. - /// - /// The index of the memo location to write is the number of - /// elements currently present in the memo. - memoize = 0x94, - - /// Extension code. - /// Introduced in protocol 2. - /// - /// This code and the similar EXT2 and EXT4 allow using a registry - /// of popular objects that are pickled by name, typically classes. - /// It is envisioned that through a global negotiation and - /// registration process, third parties can set up a mapping between - /// ints and object names. - /// - /// In order to guarantee pickle interchangeability, the extension - /// code registry ought to be global, although a range of codes may - /// be reserved for private use. - /// - /// EXT1 has a 1-byte integer argument. This is used to index into the - /// extension registry, and the object at that index is pushed on the stack. - ext1 = 0x82, - - /// Extension code. - /// Introduced in protocol 2. - /// - /// See EXT1. EXT2 has a two-byte integer argument. - ext2 = 0x83, - - /// Extension code. - /// Introduced in protocol 2. - /// - /// See EXT1. EXT4 has a four-byte integer argument. - ext4 = 0x84, - - /// Push a global object (module.attr) on the stack. - /// - /// Two newline-terminated strings follow the GLOBAL opcode. The first is - /// taken as a module name, and the second as a class name. The class - /// object module.class is pushed on the stack. More accurately, the - /// object returned by self.find_class(module, class) is pushed on the - /// stack, so unpickling subclasses can override this form of lookup. - global = 'c', - - /// Push a global object (module.attr) on the stack. - /// Introduced in protocol 4. - stack_global = 0x93, - - /// Push an object built from a callable and an argument tuple. - /// - /// The opcode is named to remind of the __reduce__() method. - /// - /// Stack before: ... callable pytuple - /// Stack after: ... callable(*pytuple) - /// - /// The callable and the argument tuple are the first two items returned - /// by a __reduce__ method. Applying the callable to the argtuple is - /// supposed to reproduce the original object, or at least get it started. - /// If the __reduce__ method returns a 3-tuple, the last component is an - /// argument to be passed to the object's __setstate__, and then the REDUCE - /// opcode is followed by code to create setstate's argument, and then a - /// BUILD opcode to apply __setstate__ to that argument. - /// - /// If not isinstance(callable, type), REDUCE complains unless the - /// callable has been registered with the copyreg module's - /// safe_constructors dict, or the callable has a magic - /// '__safe_for_unpickling__' attribute with a true value. I'm not sure - /// why it does this, but I've sure seen this complaint often enough when - /// I didn't want to . - reduce = 'R', - - /// Finish building an object, via __setstate__ or dict update. - /// - /// Stack before: ... anyobject argument - /// Stack after: ... anyobject - /// - /// where anyobject may have been mutated, as follows: - /// - /// If the object has a __setstate__ method, - /// - /// anyobject.__setstate__(argument) - /// - /// is called. - /// - /// Else the argument must be a dict, the object must have a __dict__, and - /// the object is updated via - /// - /// anyobject.__dict__.update(argument) - build = 'b', - - /// Build a class instance. - /// - /// This is the protocol 0 version of protocol 1's OBJ opcode. - /// INST is followed by two newline-terminated strings, giving a - /// module and class name, just as for the GLOBAL opcode (and see - /// GLOBAL for more details about that). self.find_class(module, name) - /// is used to get a class object. - /// - /// In addition, all the objects on the stack following the topmost - /// markobject are gathered into a tuple and popped (along with the - /// topmost markobject), just as for the TUPLE opcode. - /// - /// Now it gets complicated. If all of these are true: - /// - /// + The argtuple is empty (markobject was at the top of the stack - /// at the start). - /// - /// + The class object does not have a __getinitargs__ attribute. - /// - /// then we want to create an old-style class instance without invoking - /// its __init__() method (pickle has waffled on this over the years; not - /// calling __init__() is current wisdom). In this case, an instance of - /// an old-style dummy class is created, and then we try to rebind its - /// __class__ attribute to the desired class object. If this succeeds, - /// the new instance object is pushed on the stack, and we're done. - /// - /// Else (the argtuple is not empty, it's not an old-style class object, - /// or the class object does have a __getinitargs__ attribute), the code - /// first insists that the class object have a __safe_for_unpickling__ - /// attribute. Unlike as for the __safe_for_unpickling__ check in REDUCE, - /// it doesn't matter whether this attribute has a true or false value, it - /// only matters whether it exists (XXX this is a bug). If - /// __safe_for_unpickling__ doesn't exist, UnpicklingError is raised. - /// - /// Else (the class object does have a __safe_for_unpickling__ attr), - /// the class object obtained from INST's arguments is applied to the - /// argtuple obtained from the stack, and the resulting instance object - /// is pushed on the stack. - /// - /// NOTE: checks for __safe_for_unpickling__ went away in Python 2.3. - /// NOTE: the distinction between old-style and new-style classes does - /// not make sense in Python 3. - inst = 'i', - - /// Build a class instance. - /// Introduced in protocol 1. - /// - /// This is the protocol 1 version of protocol 0's INST opcode, and is - /// very much like it. The major difference is that the class object - /// is taken off the stack, allowing it to be retrieved from the memo - /// repeatedly if several instances of the same class are created. This - /// can be much more efficient (in both time and space) than repeatedly - /// embedding the module and class names in INST opcodes. - /// - /// Unlike INST, OBJ takes no arguments from the opcode stream. Instead - /// the class object is taken off the stack, immediately above the - /// topmost markobject: - /// - /// Stack before: ... markobject classobject stackslice - /// Stack after: ... new_instance_object - /// - /// As for INST, the remainder of the stack above the markobject is - /// gathered into an argument tuple, and then the logic seems identical, - /// except that no __safe_for_unpickling__ check is done (XXX this is - /// a bug). See INST for the gory details. - /// - /// NOTE: In Python 2.3, INST and OBJ are identical except for how they - /// get the class object. That was always the intent; the implementations - /// had diverged for accidental reasons. - obj = 'o', - - /// Build an object instance. - /// Introduced in protocol 2. - /// - /// The stack before should be thought of as containing a class - /// object followed by an argument tuple (the tuple being the stack - /// top). Call these cls and args. They are popped off the stack, - /// and the value returned by cls.__new__(cls, *args) is pushed back - /// onto the stack. - newobj = 0x81, - - /// Build an object instance. - /// Introduced in protocol 4. - /// - /// The stack before should be thought of as containing a class - /// object followed by an argument tuple and by a keyword argument dict - /// (the dict being the stack top). Call these cls and args. They are - /// popped off the stack, and the value returned by - /// cls.__new__(cls, *args, *kwargs) is pushed back onto the stack. - newobj_ex = 0x92, - - /// Protocol version indicator. - /// Introduced in protocol 2. - /// - /// For protocol 2 and above, a pickle must start with this opcode. - /// The argument is the protocol version, an int in range(2, 256). - proto = 0x80, - - /// Stop the unpickling machine. - /// - /// Every pickle ends with this opcode. The object at the top of the stack - /// is popped, and that's the result of unpickling. The stack should be - /// empty then. - stop = '.', - - /// Indicate the beginning of a new frame. - /// Introduced in protocol 4. - /// - /// The unpickler may use this opcode to safely prefetch data from its - /// underlying stream. - frame = 0x95, - - /// Push an object identified by a persistent ID. - /// - /// The pickle module doesn't define what a persistent ID means. PERSID's - /// argument is a newline-terminated str-style (no embedded escapes, no - /// bracketing quote characters) string, which *is* "the persistent ID". - /// The unpickler passes this string to self.persistent_load(). Whatever - /// object that returns is pushed on the stack. There is no implementation - /// of persistent_load() in Python's unpickler: it must be supplied by an - /// unpickler subclass. - persid = 'P', - - /// Push an object identified by a persistent ID. - /// Introduced in protocol 1. - /// - /// Like PERSID, except the persistent ID is popped off the stack (instead - /// of being a string embedded in the opcode bytestream). The persistent - /// ID is passed to self.persistent_load(), and whatever object that - /// returns is pushed on the stack. See PERSID for more detail. - binpersid = 'Q', - - _, -}; - -// The above enum was generated with the following Python code, -// run inside pickletools.py -// -// def generate_zig(): -// print("""/// All possible pickle operators. -// /// Reference: https://github.com/python/cpython/blob/3.13/Lib/pickletools.py -// pub const OpCode = enum(u8) { -// """) -// for op in opcodes: -// lines = [_cleanup(l) for l in op.doc.split("\n")[:-1]] -// if op.proto > 0: -// lines.insert(1, _cleanup(f"Introduced in protocol {op.proto}.")) -// doc = "\n".join(lines) -// op_code = op.code.__repr__() -// if op_code.startswith("'\\x"): -// op_code = "0x" + op_code[3:-1] -// print(f"""{doc} -// {op.name.lower()} = {op_code}, -// """) -// -// print(" _,") -// print("};") -// -// def _cleanup(line: str) -> str: -// indent = " " -// if (line.startswith(indent)): -// line = line[len(indent):] -// line = line.replace(". ", ". ") -// if line: -// line = " " + line -// line = " ///" + line -// return line - -/// A decoded Pickle operation in its natural state. -/// This is a bit different from Op enum, -/// because operators having same semantics, but different encoding have been merged. -/// ex: string, binstring, short_binstring -> string. -pub const Op = union(enum) { - int: i32, - // Python can represent arbitrary long integers - long: []const u8, - binlong: []const u8, - string: []const u8, - bytes: []const u8, - bytearray: []u8, - next_buffer, - readonly_buffer, - none, - bool: bool, - unicode: []const u8, - float: []const u8, - binfloat: f64, - empty_list, - append, - appends, - list, - empty_tuple, - tuple, - tuple1, - tuple2, - tuple3, - empty_dict, - dict, - setitem, - setitems, - empty_set, - additems, - frozenset, - pop, - dup, - mark, - pop_mark, - get: u32, - put: u32, - memoize, - ext1: u8, - ext2: i16, - ext4: i32, - global: PyType, - stack_global, - reduce, - build, - inst: PyType, - obj, - newobj, - newobj_ex, - proto: u8, - stop, - frame: u64, // new frame and its size - persid: []const u8, - binpersid, - - pub const PyType = struct { module: []const u8, class: []const u8 }; - - pub fn deinit(self: Op, allocator: std.mem.Allocator) void { - switch (self) { - // Use a switch on the type of the stored data, - // this is easier than listing every opcode. - inline else => |v| switch (@TypeOf(v)) { - void, bool, u8, u16, u32, u64, i16, i32, f64 => {}, - []const u8, []u8 => allocator.free(v), - PyType => { - allocator.free(v.module); - allocator.free(v.class); - }, - else => @compileError("please explicit how to free this new opcode: " ++ @typeName(@TypeOf(v))), - }, - } - } - - pub fn clone(self: Op, allocator: std.mem.Allocator) !Op { - return switch (self) { - // Use a switch on the type of the stored data, - // this is easier than listing every opcode. - inline else => |v, tag| switch (@TypeOf(v)) { - void, bool, u8, u16, u32, u64, i16, i32, f64 => self, - []const u8, []u8 => @unionInit(Op, @tagName(tag), try allocator.dupe(u8, v)), - PyType => @unionInit(Op, @tagName(tag), .{ - .module = try allocator.dupe(u8, v.module), - .class = try allocator.dupe(u8, v.class), - }), - else => @compileError("please explicit how to close this new opcode: " ++ @typeName(@TypeOf(v))), - }, - }; - } -}; - -/// Read a stream of bytes, and interpret it as a stream of Pickle operators. -pub fn parse(allocator: std.mem.Allocator, reader: anytype, max_line_len: usize) ![]const Op { - var results = std.ArrayList(Op).init(allocator); - errdefer results.deinit(); - const len = max_line_len; - var _buf: std.BoundedArray(u8, 12) = .{}; - - while (true) { - const b = try reader.readByte(); - const code: OpCode = @enumFromInt(b); - const op: Op = switch (code) { - .int => blk: { - _buf.len = 0; - try reader.streamUntilDelimiter(_buf.writer(), '\n', _buf.capacity() + 1); - const buf = _buf.constSlice(); - // Legacy hack, see OpCode.int documentation - // We do this parsing right away to simplify downstream code. - break :blk if (std.mem.eql(u8, "00", buf)) - .{ .bool = false } - else if (std.mem.eql(u8, "01", buf)) - .{ .bool = true } - else - .{ .int = try std.fmt.parseInt(i32, buf, 10) }; - }, - .binint => .{ .int = try reader.readInt(i32, .little) }, - .binint1 => .{ .int = try reader.readByte() }, - .binint2 => .{ .int = try reader.readInt(u16, .little) }, - // TODO: long should handle the trailing 'L' -> add a test. - .long => .{ .long = try reader.readUntilDelimiterAlloc(allocator, '\n', len) }, - .long1 => .{ .binlong = try _readSlice(reader, allocator, 1) }, - .long4 => .{ .binlong = try _readSlice(reader, allocator, 4) }, - .string => .{ .string = try reader.readUntilDelimiterAlloc(allocator, '\n', len) }, - .binstring => .{ .string = try _readSlice(reader, allocator, 4) }, - .short_binstring => .{ .string = try _readSlice(reader, allocator, 1) }, - .binbytes => .{ .bytes = try _readSlice(reader, allocator, 4) }, - .binbytes8 => .{ .bytes = try _readSlice(reader, allocator, 8) }, - .short_binbytes => .{ .bytes = try _readSlice(reader, allocator, 1) }, - .bytearray8 => .{ .bytearray = try _readSlice(reader, allocator, 8) }, - .next_buffer => .next_buffer, - .readonly_buffer => .readonly_buffer, - .none => .none, - .newtrue => .{ .bool = true }, - .newfalse => .{ .bool = false }, - .unicode => .{ .unicode = try reader.readUntilDelimiterAlloc(allocator, '\n', len) }, - .short_binunicode => .{ .unicode = try _readSlice(reader, allocator, 1) }, - .binunicode => .{ .unicode = try _readSlice(reader, allocator, 4) }, - .binunicode8 => .{ .unicode = try _readSlice(reader, allocator, 8) }, - .float => .{ .float = try reader.readUntilDelimiterAlloc(allocator, '\n', len) }, - .binfloat => .{ .binfloat = @bitCast(try reader.readInt(u64, .big)) }, - .empty_list => .empty_list, - .append => .append, - .appends => .appends, - .list => .list, - .empty_tuple => .empty_tuple, - .tuple => .tuple, - .tuple1 => .tuple1, - .tuple2 => .tuple2, - .tuple3 => .tuple3, - .empty_dict => .empty_dict, - .dict => .dict, - .setitem => .setitem, - .setitems => .setitems, - .empty_set => .empty_set, - .additems => .additems, - .frozenset => .frozenset, - .pop => .pop, - .dup => .dup, - .mark => .mark, - .pop_mark => .pop_mark, - // If we fail to parse delay the error to the evaluation. - .get => .{ - .get = _readDigits(u32, reader, &_buf) catch std.math.maxInt(u32), - }, - .binget => .{ .get = try reader.readByte() }, - .long_binget => .{ .get = try reader.readInt(u32, .little) }, - .put => blk: { - const buf = try reader.readUntilDelimiterAlloc(allocator, '\n', len); - defer allocator.free(buf); - const n = std.fmt.parseInt(u32, buf, 10) catch std.math.maxInt(u32); - break :blk .{ .put = n }; - }, - .binput => .{ .put = try reader.readByte() }, - .long_binput => .{ .put = try reader.readInt(u32, .little) }, - .memoize => .memoize, - .ext1 => .{ .ext1 = try reader.readByte() }, - .ext2 => .{ .ext2 = try reader.readInt(i16, .little) }, - .ext4 => .{ .ext4 = try reader.readInt(i32, .little) }, - .global => .{ .global = .{ - .module = try reader.readUntilDelimiterAlloc(allocator, '\n', len), - .class = try reader.readUntilDelimiterAlloc(allocator, '\n', len), - } }, - .stack_global => .stack_global, - .reduce => .reduce, - .build => .build, - .inst => .{ .inst = .{ - .module = try reader.readUntilDelimiterAlloc(allocator, '\n', len), - .class = try reader.readUntilDelimiterAlloc(allocator, '\n', len), - } }, - .obj => .obj, - .newobj => .newobj, - .newobj_ex => .newobj_ex, - .proto => blk: { - const version = try reader.readByte(); - if (version > 5) log.warn("zml.aio.torch.pickle.parse expects a Python pickle object of version <=5, got version {}. Will try to interpret anyway, but this may lead to more errors.", .{version}); - break :blk .{ .proto = version }; - }, - .stop => .stop, - // This is not documented in pickletools but in https://peps.python.org/pep-3154/ - // The frame size is stored right after the frame header. - // The loader is allowed to prefetch framesize from the underlying reader, - // and ops are not allowed to cross a frame boundary. - // We don't prefetch because we assume the reader is going to use some kind of buffered reader. - // We could try to enforce frame boundaries, but we would need to track - // how many bytes we are reading from the stream. - .frame => .{ .frame = try reader.readInt(u64, .little) }, - .persid => .{ .persid = try reader.readUntilDelimiterAlloc(allocator, '\n', len) }, - .binpersid => .binpersid, - _ => |unk_tag| { - log.err("Unknow pickle operator {}, note we are only supporting pickle protocol up to version 5.", .{unk_tag}); - return error.NotSupported; - }, - }; - try results.append(op); - if (op == .stop) break; - } - return results.toOwnedSlice(); -} - -test "parse protocol 4" { - const allocator = std.testing.allocator; - const file = try std.fs.cwd().openFile("zml/aio/torch/simple_test_4.pickle", .{ .mode = .read_only }); - var buffered_reader = std.io.bufferedReader(file.reader()); - const ops = try parse(allocator, buffered_reader.reader(), 4096); - defer { - // Test we are correctly freeing every allocation. - for (ops) |op| op.deinit(allocator); - allocator.free(ops); - } - - // this can be obtained by running: `python -m pickletools simple_test_4.pickle` - var expected = [_]Op{ - .{ .proto = 4 }, - .{ .frame = 119 }, - .empty_dict, - .memoize, - .mark, - .{ .unicode = "hello" }, - .memoize, - .{ .unicode = "world" }, - .memoize, - .{ .unicode = "int" }, - .memoize, - .{ .int = 1 }, - .{ .unicode = "float" }, - .memoize, - .{ .binfloat = 3.141592 }, - .{ .unicode = "list" }, - .memoize, - .empty_list, - .memoize, - .mark, - .{ .int = 255 }, - .{ .int = 1234 }, - .{ .int = -123 }, - .{ .int = 1_000_000_000 }, - .{ .binlong = &writeIntBuff(u48, 999_000_000_000) }, - .{ .binlong = &writeIntBuff(u104, 999_000_000_000_000_000_000_000_000_000) }, - .appends, - .{ .unicode = "bool" }, - .memoize, - .{ .bool = false }, - .{ .unicode = "tuple" }, - .memoize, - .{ .unicode = "a" }, - .memoize, - .{ .int = 10 }, - .tuple2, - .memoize, - .setitems, - .stop, - }; - try std.testing.expectEqualDeep(&expected, ops); -} - -test "parse protocol 0" { - // We also test protocol 0, cause it's more text oriented. - const allocator = std.testing.allocator; - const pickle_0 = - \\(dp0 - \\Vhello - \\p1 - \\Vworld - \\p2 - \\sVint - \\p3 - \\I1 - \\sVfloat - \\p4 - \\F3.141592 - \\sVlist - \\p5 - \\(lp6 - \\I255 - \\aI1234 - \\aI-123 - \\aI1000000000 - \\aL999000000000L - \\aL999000000000000000000000000000L - \\asVbool - \\p7 - \\I00 - \\sVtuple - \\p8 - \\(Va - \\p9 - \\I10 - \\tp10 - \\s. - ; - - var stream = std.io.fixedBufferStream(pickle_0); - const ops = try parse(allocator, stream.reader(), 4096); - defer { - // Test we are correctly freeing every allocation. - for (ops) |op| op.deinit(allocator); - allocator.free(ops); - } - - var expected = [_]Op{ - .mark, - .dict, - .{ .put = 0 }, - .{ .unicode = "hello" }, - .{ .put = 1 }, - .{ .unicode = "world" }, - .{ .put = 2 }, - .setitem, - .{ .unicode = "int" }, - .{ .put = 3 }, - .{ .int = 1 }, - .setitem, - .{ .unicode = "float" }, - .{ .put = 4 }, - .{ .float = "3.141592" }, - .setitem, - .{ .unicode = "list" }, - .{ .put = 5 }, - .mark, - .list, - .{ .put = 6 }, - .{ .int = 255 }, - .append, - .{ .int = 1234 }, - .append, - .{ .int = -123 }, - .append, - .{ .int = 1_000_000_000 }, - .append, - .{ .long = "999000000000L" }, - .append, - .{ .long = "999000000000000000000000000000L" }, - .append, - .setitem, - .{ .unicode = "bool" }, - .{ .put = 7 }, - .{ .bool = false }, - .setitem, - .{ .unicode = "tuple" }, - .{ .put = 8 }, - .mark, - .{ .unicode = "a" }, - .{ .put = 9 }, - .{ .int = 10 }, - .tuple, - .{ .put = 10 }, - .setitem, - .stop, - }; - try std.testing.expectEqualDeep(&expected, ops); -} - -fn _readDigits(comptime T: type, reader: anytype, buffer: *std.BoundedArray(u8, 12)) !T { - buffer.len = 0; - try reader.streamUntilDelimiter(buffer.writer(), '\n', 13); - return std.fmt.parseInt(T, buffer.constSlice(), 10); -} - -fn _readSlice(reader: anytype, allocator: std.mem.Allocator, comptime len_bytes: u8) ![]u8 { - const T = std.meta.Int(.unsigned, 8 * len_bytes); - const str_len: u64 = try reader.readInt(T, .little); - const buf = try allocator.alloc(u8, str_len); - errdefer allocator.free(buf); - _ = try reader.read(buf); - return buf; -} - -fn writeIntBuff(comptime T: type, value: T) [@divExact(@typeInfo(T).int.bits, 8)]u8 { - var res: [@divExact(@typeInfo(T).int.bits, 8)]u8 = undefined; - std.mem.writeInt(T, &res, value, .little); - return res; -} diff --git a/zml/aio/torch/py.zig b/zml/aio/torch/py.zig deleted file mode 100644 index c523f280..00000000 --- a/zml/aio/torch/py.zig +++ /dev/null @@ -1,430 +0,0 @@ -const std = @import("std"); -const math = std.math; -const log = std.log.scoped(.@"zml/aio"); - -const pickle = @import("pickle.zig"); - -/// Correspond to a function/constructor call -pub const Object = struct { - member: Any, - args: []Any, - kwargs: []Any, - - pub fn init(allocator: std.mem.Allocator, member: Any, args: []Any, kwargs: []Any) !*Object { - const self = try allocator.create(Object); - self.* = .{ .member = member, .args = args, .kwargs = kwargs }; - return self; - } - - pub fn clone(self: *Object, allocator: std.mem.Allocator) std.mem.Allocator.Error!*Object { - const res = try allocator.create(Object); - res.* = .{ - .member = try self.member.clone(allocator), - .args = try allocator.alloc(Any, self.args.len), - .kwargs = try allocator.alloc(Any, self.kwargs.len), - }; - for (self.args, 0..) |v, i| res.args[i] = try v.clone(allocator); - for (self.kwargs, 0..) |v, i| res.kwargs[i] = try v.clone(allocator); - return res; - } - - pub fn deinit(self: *Object, allocator: std.mem.Allocator) void { - self.member.deinit(allocator); - for (self.args) |*v| v.deinit(allocator); - allocator.free(self.args); - allocator.destroy(self); - } -}; - -/// Correspond to the __set_state__ call when pickle finishes building an object. -pub const SetState = struct { - obj: Any, - state: Any, - - pub fn init(allocator: std.mem.Allocator, obj: Any, state: Any) !*SetState { - const res = try allocator.create(SetState); - res.* = .{ .obj = obj, .state = state }; - return res; - } - - pub fn clone(self: *SetState, allocator: std.mem.Allocator) std.mem.Allocator.Error!*SetState { - const res = try allocator.create(SetState); - res.* = .{ .obj = try self.obj.clone(allocator), .state = try self.state.clone(allocator) }; - return res; - } - - pub fn deinit(self: *SetState, allocator: std.mem.Allocator) void { - self.obj.deinit(allocator); - self.state.deinit(allocator); - self.allocator.destroy(self); - } -}; - -/// The types of sequences that exist. -pub const SequenceType = enum { - list, - dict, - tuple, - set, - frozen_set, -}; - -pub const Sequence = struct { - type: SequenceType, - values: []Any, -}; - -pub fn tuple(values: []const Any) Any { - // tuple are readonly, but sequence in general aren't - return .{ .seq = .{ .type = .tuple, .values = @constCast(values) } }; -} - -pub const PersId = struct { - ref: Any, - - pub fn init(allocator: std.mem.Allocator, ref: Any) !*PersId { - const self = try allocator.create(PersId); - self.* = .{ .ref = ref }; - return self; - } - - pub fn clone(self: *PersId, allocator: std.mem.Allocator) std.mem.Allocator.Error!*PersId { - const res = try allocator.create(PersId); - res.* = .{ .ref = try self.ref.clone(allocator) }; - return res; - } - - pub fn deinit(self: *PersId, allocator: std.mem.Allocator) void { - self.ref.deinit(allocator); - allocator.destroy(self); - } -}; - -pub const Kind = enum { - raw, - ref, - app, - object, - set_state, - pers_id, - global, - seq, - string, - bytes, - int64, - bigint, - float64, - raw_num, - boolval, - none, -}; - -/// A pickle operator that has been interpreted. -pub const Any = union(Kind) { - /// Types that we can't handle or just had to give up on processing. - raw: pickle.Op, - - /// A reference. You might be able to look it up in the memo map - /// unless there's something weird going on like recursive references. - /// You generally shouldn't see this in the result unless bad things - /// are going on... - ref: u32, - - /// The result of applying a thing to another thing. We're not - /// Python so we don't really know what a "thing" is. - app: *Object, - - /// An object or something. The first tuple member is the - /// thing, the second one is the arguments it got applied to. - object: *Object, - - /// Correspond to the __set_state__ call when pickle finishes building an object. - /// The first tuple member is the target object, - /// the second one is the "state" argument - set_state: *SetState, - - /// References to persistant storage. They basically could be anything. - /// You kind of have to know what the thing you're trying to - /// interface wants to use as keys for persistant storage. - /// Good luck. - pers_id: *PersId, - - /// A global value of some kind. The first tuple member is - /// the thing, the second one is the arguments it got applied to. - global: *Object, - - /// A sequence. We don't really distinguish between them - /// much. The one exception is when the SequenceType is - /// Dict we try to split the flat list of `[k, v, k, v, k, v]` - /// into a list of tuples with the key and value. - seq: Sequence, - - /// A string, but not the crazy strings that have to be - /// unescaped as if they were Python strings. If you - /// need one of those, look for it inside a `Value.raw`. - string: []const u8, - - /// Some bytes. It might be a byte array or a binary - /// string that couldn't get UTF8 decoded. We do the best - /// we can. - bytes: []const u8, - - /// An integer, but not the crazy kind that comes as a string - /// that has to be parsed. You can look in `Value.raw_num` for - /// those. - int64: i64, - - /// An integer that can't fit in i64. - bigint: math.big.int.Const, - - /// An float, but not the crazy kind that comes as a string - /// that has to be parsed. You can look in `Value.raw_num` for - /// those. - float64: f64, - - /// Some kind of weird number we can't handle. - raw_num: pickle.Op, - - /// A boolean value. - boolval: bool, - - /// Python `None`. - none: void, - - pub fn deinit(self: *Any, allocator: std.mem.Allocator) void { - switch (self.*) { - .raw, .raw_num => |v| v.deinit(allocator), - inline .app, .object, .global, .build, .pers_id => |v| v.deinit(), - .seq => |v| { - for (v.values) |*val| val.deinit(allocator); - allocator.free(v.values); - }, - .string, .bytes => |v| allocator.free(v), - .bigint => |big| allocator.free(big.limbs), - else => {}, - } - self.* = undefined; - } - - inline fn writeIndents(indents: usize, writer: anytype) !void { - try writer.writeBytesNTimes(" ", indents); // resolve tab = 2 spaces - // try writer.writeByteNTimes('\t'); - } - - fn internalFormat(value: Any, indents: usize, writer: anytype) !void { - try writeIndents(indents, writer); - try writer.writeAll(".{\n"); - try writeIndents(indents + 1, writer); - try writer.print(".{s} = ", .{@tagName(std.meta.activeTag(value))}); - switch (value) { - inline .ref, .int64, .float64 => |v| try writer.print("{d} ", .{v}), - .app, .object, .global => |v| { - try writer.writeAll(".{\n"); - try writeIndents(indents + 2, writer); - try writer.writeAll(".fn ="); - try internalFormat(v.member, indents + 2, writer); - try writer.writeAll(",\n"); - try writeIndents(indents + 2, writer); - try writer.writeAll(".args = "); - if (v.args.len > 0) { - try writer.writeAll(".{\n"); - for (v.args, 0..) |arg, i| { - try internalFormat(arg, indents + 3, writer); - if (i < v.args.len - 1) try writer.writeAll(","); - try writer.writeByte('\n'); - } - try writeIndents(indents + 2, writer); - try writer.writeAll("},\n"); - } else { - try writer.writeAll(".{},\n"); - } - try writeIndents(indents + 2, writer); - try writer.writeAll(".kwargs ="); - if (v.kwargs.len > 0) { - try writer.writeAll(".{\n"); - for (v.kwargs, 0..) |arg, i| { - try internalFormat(arg, indents + 3, writer); - if (i < v.kwargs.len - 1) try writer.writeAll(","); - try writer.writeByte('\n'); - } - try writeIndents(indents + 2, writer); - try writer.writeAll("}\n"); - } else { - try writer.writeAll(".{}\n"); - } - try writeIndents(indents + 1, writer); - try writer.writeAll("}"); - }, - .set_state => |v| { - try writer.writeAll(".{\n"); - try internalFormat(v.obj, indents + 2, writer); - try writer.writeAll(",\n"); - try internalFormat(v.state, indents + 2, writer); - try writer.writeAll(",\n"); - try writeIndents(indents + 1, writer); - try writer.writeAll("}"); - }, - .pers_id => |v| { - try writer.writeByte('\n'); - try internalFormat(v.ref, indents + 2, writer); - }, - .seq => |v| { - try writer.writeAll(".{\n"); - try writeIndents(indents + 2, writer); - try writer.print(".{s},\n", .{@tagName(v.type)}); - try writeIndents(indents + 2, writer); - if (v.values.len > 0) { - try writer.writeAll(".{\n"); - for (v.values, 0..) |arg, i| { - try internalFormat(arg, indents + 3, writer); - if (i < v.values.len - 1) try writer.writeAll(","); - try writer.writeByte('\n'); - } - try writeIndents(indents + 2, writer); - try writer.writeAll("}\n"); - } else { - try writer.writeAll(".{}\n"); - } - - try writeIndents(indents + 1, writer); - try writer.writeAll("}"); - }, - .string => |v| try writer.print("\"{s}\"", .{v}), - .raw => |v| switch (v) { - .global => |py_type| try writer.print("\"{s}\", \"{s}\"", .{ py_type.module, py_type.class }), - else => try writer.print("{any}", .{v}), - }, - inline else => |v| { - try writer.print("{any}", .{v}); - }, - } - try writer.writeByte('\n'); - try writeIndents(indents, writer); - try writer.writeByte('}'); - } - - pub fn format(self: Any, comptime _: []const u8, _: std.fmt.FormatOptions, writer: anytype) !void { - return internalFormat(self, 0, writer); - } - - pub fn clone(self: Any, allocator: std.mem.Allocator) !Any { - return switch (self) { - inline .raw, .raw_num => |v, tag| @unionInit(Any, @tagName(tag), try v.clone(allocator)), - inline .app, .object, .global, .set_state, .pers_id => |v, tag| @unionInit(Any, @tagName(tag), try v.clone(allocator)), - .seq => |seq| { - const values = try allocator.alloc(Any, seq.values.len); - for (seq.values, 0..) |v, i| values[i] = try v.clone(allocator); - return .{ .seq = .{ .type = seq.type, .values = values } }; - }, - inline .string, .bytes => |v, tag| @unionInit(Any, @tagName(tag), try allocator.dupe(u8, v)), - .bigint => |v| .{ .bigint = (try v.toManaged(allocator)).toConst() }, - else => self, - }; - } - - pub fn isPrimitive(self: Any) bool { - return switch (self) { - .int64, .bigint, .float64, .string, .bytes, .boolval, .none => true, - .seq => |seq| { - for (seq.values) |v| { - if (!v.isPrimitive()) return false; - } - return true; - }, - else => false, - }; - } - - pub fn containsRef(self: Any) bool { - switch (self) { - .ref => return true, - .app, .object, .global => |v| { - if (v.member.containsRef()) return true; - for (v.args) |arg| if (arg.containsRef()) return true; - return false; - }, - .set_state => |v| { - if (v.obj.containsRef()) return true; - if (v.state.containsRef()) return true; - return false; - }, - .pers_id => |v| return v.ref.containsRef(), - .seq => |v| { - for (v.values) |val| if (val.containsRef()) return true; - return false; - }, - else => return false, - } - } - - pub const UnpickleError = error{ InvalidCharacter, OutOfMemory }; - - pub fn coerceFromRaw(self: Any, allocator: std.mem.Allocator) UnpickleError!Any { - return switch (self) { - .raw => |raw_val| switch (raw_val) { - .none => .none, - .bool => |b| .{ .boolval = b }, - .float => |b| .{ .float64 = std.fmt.parseFloat(f64, b) catch std.math.nan(f64) }, - .int => |val| .{ .int64 = val }, - .long => |digits| { - const n = std.fmt.parseInt(i64, digits[0 .. digits.len - 1], 10) catch |err| { - switch (err) { - error.Overflow => { - log.warn("Not parsing long integer: {s}", .{digits}); - return self; - }, - error.InvalidCharacter => return error.InvalidCharacter, - } - }; - return .{ .int64 = n }; - }, - .binlong => |bytes| if (bytes.len <= 8) - .{ .int64 = std.mem.readVarInt(i64, bytes, .little) } - else { - // Note: we need to copy here, because Zig big int limbs are usize aligned, - // whereas pickle big int are byte aligned. - const n_limbs = std.math.divCeil(usize, bytes.len, @sizeOf(math.big.Limb)) catch unreachable; - var big = (try math.big.int.Managed.initCapacity(allocator, n_limbs)).toMutable(); - big.readTwosComplement(bytes, bytes.len * 8, .little, .signed); - - return .{ .bigint = big.toConst() }; - }, - .binfloat => |val| .{ .float64 = val }, - .unicode => |s| .{ .string = s }, - inline .bytes, .bytearray => |b| .{ .bytes = b }, - // This isn't how Pickle actually works but we just try to UTF8 decode the - // string and if it fails, we make it a bytes value instead. If anyone - // actually cares they can just fix values themselves or recover the raw bytes - // from the UTF8 string (it's guaranteed to be reversible, as far as I know). - .string => |b| if (std.unicode.utf8ValidateSlice(b)) - .{ .string = b } - else - .{ .bytes = b }, - else => self, - }, - .app, .object, .global => |v| blk: { - v.member = try v.member.coerceFromRaw(allocator); - for (v.args) |*arg| { - arg.* = try arg.coerceFromRaw(allocator); - } - break :blk self; - }, - .set_state => |v| blk: { - v.obj = try v.obj.coerceFromRaw(allocator); - v.state = try v.state.coerceFromRaw(allocator); - break :blk self; - }, - .pers_id => |v| blk: { - v.ref = try v.ref.coerceFromRaw(allocator); - break :blk self; - }, - .seq => |v| blk: { - for (v.values) |*val| { - val.* = try val.coerceFromRaw(allocator); - } - break :blk self; - }, - else => self, - }; - } -}; diff --git a/zml/aio/torch/simple.pt b/zml/aio/torch/simple.pt deleted file mode 100644 index f6978e65eaa6a8429f4ced64c8370b7294eefa6f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2708 zcmbtWOKjX!6!pxc`M4!%6I%Ks6bLYZW-`BIlK#?8%JgGsr_&Zpp;(UnGGoQH=h|-) zTB#CUByAV4V8Idz#G+LpsObWU4Qtqe0HJJIuz>{<3nYZZz0dPc(~9y&FOJ{+oO|E9 z=lbQ9NQWeKcS~!=W~pD&J)>mJvr$v$daOKW=d!)hg_iMd_$o+Ibpx7Ta!qDK$lxwo z#T+Mw3l*FBF~fD{$vxe2ET;$?Zm1jaT`TLJeu;7B`Os+? zu5B}eTdw2Fr#+K-%sgrtT!yaU0(3{!PPABHR25T}rriM@@4!vpwpGLU?eaaAr&1V!T zs)X?uQ6e0+M3sJ}2mcRo=)YN_hzB@4UVyDP#LKqp@I(Q&tGN5pjHUaqqa>FQzqF|o z03={%sZR-W*oBUj@FMJ1LW}TZ;A)V=Qw7))xO)0JJX3&YD`H2rnab=eZQqc9c~0Gh z0Gi`ES{YX@AG75(9nEJPo;Ngnw`^`XnkM6l3@==S=ptamFRBA%J);|Q3%ZB9=`pS3 zx^q6fB)soc`IY(&Un>Nb*DzBXI2?S>E1fzeZHFG*)zPdE+?iZ%7vhXMcwcb~jG*S@G#0hA z1bG2g^xR6>hcik@L|WC+sG~qZG?y#PbA33g?m^#3?s=?bM(h>=;X;|=nV1@1unY$0 zqDlw;Fmev(D>DKjU^%=NaIDJf>}AgCynbEqIteP$u7krHM7t2>c>#T4Vs8f2dyB)x z0=!*`<(01P!>j(6e)0uQSGm)4~p)4cL=$oEzk3=y-nt3He!Pajc z%|zNvf|k*K4{5h<<_@*!#!&EPQnRj8uid=4aOtzu7#}v72Xt$GObt#Cc@{wL7%wl1M+${TH=J!j}L5 diff --git a/zml/aio/torch/simple_test_4.pickle b/zml/aio/torch/simple_test_4.pickle deleted file mode 100644 index 4a5176590cd0d4a7aa79b39b80fb411b6513808e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 130 zcmZo*nOe>O0ku;!dRQ}3b8_;h^stua7v-c(>0!>yE1BZW*u$EZlb={J#od8Z@zQxoXp~qDX~*Dy#M=NV)1JI|Np-i!>L);U2F_)Tr*!VbnyZWjaZwzWZC4rkF$AF fdsve4^K+(j_OO0wNq;?31IrBn|9AKNkd From 57fd9cda27a51fddaf3bae92180fea2d37e3f209 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Wed, 27 Aug 2025 14:11:22 +0200 Subject: [PATCH 023/881] workspace: zig 0.15 (#316) Much of the work was done by @gwenzek Sadly we drop the torch loader in the process, to be added back later ! --------- Co-authored-by: Guillaume Wenzek --- .github/workflows/commit-framework.yaml | 69 +++ BUILD.bazel | 1 + MODULE.bazel | 4 +- async/async.zig | 208 +++++++-- async/coro.zig | 2 +- async/coro_base.zig | 2 +- bazel/zig_index.json | 277 ++++++++++-- examples/llama/BUILD.bazel | 1 + examples/llama/llama.zig | 8 +- examples/llama/main.zig | 39 +- examples/modernbert/main.zig | 37 +- examples/simple_layer/main.zig | 17 +- ffi/zig_allocator.zig | 4 +- mlir/dialects/BUILD.bazel | 1 + mlir/dialects/func.zig | 3 +- mlir/dialects/stablehlo/stablehlo.zig | 22 +- mlir/mlir.zig | 120 +++-- pjrt/ffi.zig | 2 +- pjrt/pjrt.zig | 12 +- stdx/BUILD.bazel | 1 + stdx/bounded_array.zig | 412 ++++++++++++++++++ stdx/flags.zig | 3 +- stdx/fmt.zig | 32 +- stdx/signature.zig | 2 +- stdx/stdx.zig | 2 + stdx/time.zig | 11 +- third_party/com_github_hejsil_clap/repo.bzl | 2 +- .../rules_zig/20250827.0-35b6d57/MODULE.bazel | 75 ++++ .../rules_zig/20250827.0-35b6d57/source.json | 5 + third_party/modules/rules_zig/metadata.json | 3 +- tools/BUILD.bazel | 8 - upb/upb.zig | 2 +- zml/aio.zig | 8 +- zml/aio/json.zig | 2 +- zml/aio/safetensors.zig | 48 +- zml/buffer.zig | 14 +- zml/context.zig | 68 ++- zml/exe.zig | 4 +- zml/floats.zig | 31 +- zml/hostbuffer.zig | 53 ++- zml/meta.zig | 15 +- zml/module.zig | 28 +- zml/nn.zig | 26 +- zml/nn/cuda.zig | 3 +- zml/ops.zig | 50 +-- zml/pjrtx.zig | 6 +- zml/platform.zig | 2 +- zml/shape.zig | 53 ++- zml/tensor.zig | 151 ++++--- zml/test_runner.zig | 2 +- zml/testing.zig | 8 +- zml/tokenizer/BUILD.bazel | 1 + zml/tokenizer/hftokenizers/BUILD.bazel | 1 + zml/tokenizer/hftokenizers/hftokenizers.zig | 6 +- zml/tokenizer/homemade.zig | 76 ++-- zml/tokenizer/main.zig | 9 +- zml/tokenizer/sentencepiece/BUILD.bazel | 1 + zml/tokenizer/sentencepiece/sentencepiece.zig | 4 +- zml/torch.zig | 10 +- 59 files changed, 1499 insertions(+), 568 deletions(-) create mode 100644 .github/workflows/commit-framework.yaml create mode 100644 stdx/bounded_array.zig create mode 100644 third_party/modules/rules_zig/20250827.0-35b6d57/MODULE.bazel create mode 100644 third_party/modules/rules_zig/20250827.0-35b6d57/source.json diff --git a/.github/workflows/commit-framework.yaml b/.github/workflows/commit-framework.yaml new file mode 100644 index 00000000..1603e847 --- /dev/null +++ b/.github/workflows/commit-framework.yaml @@ -0,0 +1,69 @@ +name: framework-tests +on: + push: + branches: + - master + tags: + - '*' + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ !contains(github.ref, 'master')}} + +jobs: + framework-tests: + runs-on: ["runs-on","runner=4cpu-linux-x64", "image=ubuntu24-amd64", "run-id=${{ github.run_id }}", "extras=s3-cache"] + steps: + - uses: runs-on/action@v1 + - name: Checkout + uses: actions/checkout@v4 + + - uses: runs-on/snapshot@v1 + with: + path: /home/runner/.cache/bazel-disk + volume_size: 350 + + - name: TMP fix for the mount user + run: sudo chown -R runner:runner /home/runner/.cache/ + + - name: Setup Zig + uses: mlugg/setup-zig@v2 + with: + version: 0.14.1 + mirror: https://mirror.zml.ai/zig + + - name: Zig Format + run: zig fmt --check $(git ls-files --exclude-standard -- '*.zig' | xargs) + + - name: No autofix in Zig code + run: bash -c "! git grep -l '// autofix' -- '*.zig'" + + - uses: bazel-contrib/setup-bazel@0.15.0 + with: + bazelisk-version: 1.26.0 + bazelisk-cache: false + bazelrc: | + common --config=debug + common --verbose_failures + common --keep_going + common --test_output=streamed + common --color=yes + common --show_timestamps + common --disk_cache=/home/runner/.cache/bazel-disk + common --strip=always + + - name: ZML / bazel mod explain + run: bazel mod explain + + - name: ZML / bazel mod graph + run: bazel mod graph + + - name: ZML / bazel query //... + run: bazel query //... + + - name: ZML / bazel build //... + run: bazel build //... + + - name: ZML / bazel test //... + run: bazel test //... diff --git a/BUILD.bazel b/BUILD.bazel index 63540f5e..aa1656d4 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -11,6 +11,7 @@ zls_completion( visibility = ["//visibility:public"], deps = [ "//async", + "//examples/llama", "//stdx", "//zml", ], diff --git a/MODULE.bazel b/MODULE.bazel index 7f34bc92..72ad53a5 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -18,7 +18,7 @@ bazel_dep(name = "rules_oci", version = "2.2.6") bazel_dep(name = "rules_proto", version = "7.1.0") bazel_dep(name = "rules_python", version = "1.5.3") bazel_dep(name = "rules_rust", version = "0.63.0") -bazel_dep(name = "rules_zig", version = "20250821.0-be53625") +bazel_dep(name = "rules_zig", version = "20250827.0-35b6d57") bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.2.4") bazel_dep(name = "with_cfg.bzl", version = "0.11.0") @@ -26,7 +26,7 @@ bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2", dev_dependency = Tr zig = use_extension("@rules_zig//zig:extensions.bzl", "zig") zig.index(file = "//bazel:zig_index.json") -zig.toolchain(zig_version = "0.14.1") +zig.toolchain(zig_version = "0.15.1") zig.mirrors(urls = [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/", diff --git a/async/async.zig b/async/async.zig index f14608b8..e1052525 100644 --- a/async/async.zig +++ b/async/async.zig @@ -228,36 +228,88 @@ pub const AsyncThread = struct { }; pub fn getStdIn() !File { - return File.init(std.io.getStdIn()) catch @panic("Unable to open stdin"); + return File.initStreaming(std.fs.File.stdin()) catch @panic("Unable to open stdin"); } pub fn getStdOut() File { - return File.init(std.io.getStdOut()) catch @panic("Unable to open stdout"); + return File.initStreaming(std.fs.File.stdout()) catch @panic("Unable to open stdout"); } pub fn getStdErr() File { - return File.init(std.io.getStdErr()) catch @panic("Unable to open stderr"); + return File.initStreaming(std.fs.File.stderr()) catch @panic("Unable to open stderr"); } pub const File = struct { + pub const Reader = struct { + interface: std.Io.Reader, + file: File, + mode: std.fs.File.Reader.Mode = .positional, + pos: u64 = 0, + + fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { + const self: *Reader = @alignCast(@fieldParentPtr("interface", r)); + const dest = limit.slice(try w.writableSliceGreedy(1)); + const n = switch (self.mode) { + .streaming => self.file.read(dest), + .positional => self.file.pread(dest, self.pos), + else => @panic("UNSUPPORTED"), + } catch { + return std.Io.Reader.StreamError.ReadFailed; + }; + if (n == 0) { + return std.Io.Reader.StreamError.EndOfStream; + } + self.pos += n; + w.advance(n); + return n; + } + }; + + pub const Writer = struct { + interface: std.Io.Writer, + file: File, + mode: std.fs.File.Writer.Mode = .positional, + pos: u64 = 0, + + fn write(self: *Writer, buf: []const u8) !usize { + const n = switch (self.mode) { + .streaming => self.file.write(buf), + .positional => self.file.pwrite(buf, self.pos), + else => unreachable, + } catch { + return std.Io.Writer.Error.WriteFailed; + }; + self.pos += n; + return n; + } + + fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + // TODO: implement splat + _ = splat; + const self: *Writer = @alignCast(@fieldParentPtr("interface", w)); + var total: usize = 0; + if (w.buffered().len > 0) { + total += w.consume(try self.write(w.buffered())); + } + for (data) |d| { + const n = try self.write(d); + total += n; + if (n < d.len) { + return total; + } + } + return total; + } + }; + pub const SeekError = stdx.meta.FnSignature(File.seekTo, null).ReturnErrorSet.? || stdx.meta.FnSignature(File.seekBy, null).ReturnErrorSet.?; pub const GetSeekPosError = SeekError || stdx.meta.FnSignature(File.stat, null).ReturnErrorSet.?; - pub const Reader = std.io.GenericReader(File, stdx.meta.FnSignature(File.read, null).ReturnErrorSet.?, File.read); - pub const Writer = std.io.GenericWriter(File, stdx.meta.FnSignature(File.write, null).ReturnErrorSet.?, File.write); - pub const SeekableStream = std.io.SeekableStream( - File, - SeekError, - GetSeekPosError, - seekTo, - seekBy, - getPos, - getEndPos, - ); _handle: std.fs.File.Handle, inner: aio.File, + is_streaming: bool = false, - fn asFile(self: File) std.fs.File { + pub fn asFile(self: File) std.fs.File { return .{ .handle = self._handle }; } @@ -272,6 +324,14 @@ pub const File = struct { }; } + pub fn initStreaming(file_: std.fs.File) !File { + return .{ + ._handle = file_.handle, + .inner = aio.File.init(AsyncThread.current.executor, try xev.File.init(file_)), + .is_streaming = true, + }; + } + pub fn open(path: []const u8, flags: std.fs.File.OpenFlags) !File { return init(try callBlocking(std.fs.Dir.openFile, .{ std.fs.cwd(), path, flags })); } @@ -280,6 +340,21 @@ pub const File = struct { return try callBlocking(std.fs.Dir.access, .{ std.fs.cwd(), path, flags }); } + pub fn reader(self: File, buffer: []u8) Reader { + return .{ + .file = self, + .interface = .{ + .vtable = &.{ + .stream = Reader.stream, + }, + .buffer = buffer, + .seek = 0, + .end = 0, + }, + .mode = if (self.is_streaming) .streaming else .positional, + }; + } + pub fn read(self: File, buf: []u8) !usize { // NOTE(Corentin): Early return is required to avoid error with xev on Linux with io_uring backend. if (buf.len == 0) { @@ -310,6 +385,19 @@ pub const File = struct { return self.inner.write(.{ .slice = buf }); } + pub fn writer(self: File, buffer: []u8) Writer { + return .{ + .file = self, + .interface = .{ + .vtable = &.{ + .drain = Writer.drain, + }, + .buffer = buffer, + }, + .mode = if (self.is_streaming) .streaming else .positional, + }; + } + pub fn pwrite(self: File, buf: []const u8, offset: u64) !usize { return self.inner.pwrite(.{ .slice = buf }, offset); } @@ -318,18 +406,6 @@ pub const File = struct { return self.inner.close(); } - pub fn reader(self: File) Reader { - return .{ .context = self }; - } - - pub fn seekableStream(file: File) SeekableStream { - return .{ .context = file }; - } - - pub fn writer(self: File) Writer { - return .{ .context = self }; - } - pub fn stat(self: File) !std.fs.File.Stat { return try callBlocking(std.fs.File.stat, .{self.asFile()}); } @@ -375,8 +451,47 @@ pub const Socket = struct { pub const TCP = struct { const Inner = aio.TCP; - pub const Reader = std.io.GenericReader(TCP, stdx.meta.FnSignature(TCP.read, null).ReturnErrorSet.?, TCP.read); - pub const Writer = std.io.GenericWriter(TCP, stdx.meta.FnSignature(TCP.write, null).ReturnErrorSet.?, TCP.write); + pub const Reader = struct { + interface: std.Io.Reader, + socket: TCP, + + fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { + const self: *Reader = @alignCast(@fieldParentPtr("interface", r)); + const dest = limit.slice(try w.writableSliceGreedy(1)); + const n = self.socket.read(dest) catch { + return std.Io.Reader.StreamError.ReadFailed; + }; + w.advance(n); + return n; + } + }; + + pub const Writer = struct { + interface: std.Io.Writer, + socket: TCP, + + fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + // TODO: implement splat + _ = splat; + const self: *Writer = @alignCast(@fieldParentPtr("interface", w)); + var total: usize = 0; + if (w.buffered().len >= 0) { + total += w.consume(self.socket.write(w.buffered()) catch { + return std.Io.Writer.Error.WriteFailed; + }); + } + for (data) |d| { + const n = self.socket.write(d) catch { + return std.Io.Writer.Error.WriteFailed; + }; + total += n; + if (n < d.len) { + return total; + } + } + return total; + } + }; inner: aio.TCP, @@ -418,12 +533,30 @@ pub const Socket = struct { return self.inner.close(); } - pub fn reader(self: TCP) Reader { - return .{ .context = self }; + pub fn reader(self: TCP, buffer: []u8) Reader { + return .{ + .socket = self, + .interface = .{ + .vtable = &.{ + .stream = Reader.stream, + }, + .buffer = buffer, + .seek = 0, + .end = 0, + }, + }; } - pub fn writer(self: TCP) Writer { - return .{ .context = self }; + pub fn writer(self: TCP, buffer: []u8) Writer { + return .{ + .socket = self, + .interface = .{ + .vtable = &.{ + .drain = Writer.drain, + }, + .buffer = buffer, + }, + }; } }; @@ -564,9 +697,8 @@ pub fn logFn(comptime fallbackLogFn: LogFn) LogFn { const level_txt = comptime message_level.asText(); const prefix2 = if (scope == .default) ": " else "(" ++ @tagName(scope) ++ "): "; - const stderr = getStdErr().writer(); - var bw = std.io.bufferedWriter(stderr); - const writer = bw.writer(); + var buffer: [1024]u8 = undefined; + var stderr = getStdErr().writer(&buffer); var mutex = Self.mu orelse blk: { Self.mu = Mutex.init(); @@ -575,8 +707,8 @@ pub fn logFn(comptime fallbackLogFn: LogFn) LogFn { mutex.lock(); defer mutex.unlock(); nosuspend { - writer.print(level_txt ++ prefix2 ++ format ++ "\n", args) catch return; - bw.flush() catch return; + stderr.interface.print(level_txt ++ prefix2 ++ format ++ "\n", args) catch unreachable; + stderr.interface.flush() catch unreachable; } } }.call; diff --git a/async/coro.zig b/async/coro.zig index be26b95b..a4d0345e 100644 --- a/async/coro.zig +++ b/async/coro.zig @@ -156,7 +156,7 @@ const Coro = struct { return self; } - fn runcoro(from: *base.Coro, this: *base.Coro) callconv(.C) noreturn { + fn runcoro(from: *base.Coro, this: *base.Coro) callconv(.c) noreturn { const from_coro: *Coro = @fieldParentPtr("impl", from); const this_coro: *Coro = @fieldParentPtr("impl", this); log(.debug, "coro start {any}", .{this_coro.id}); diff --git a/async/coro_base.zig b/async/coro_base.zig index 9cd6f1c0..e1a4d6c1 100644 --- a/async/coro_base.zig +++ b/async/coro_base.zig @@ -49,7 +49,7 @@ pub const Coro = packed struct { const Func = *const fn ( from: *Coro, self: *Coro, - ) callconv(.C) noreturn; + ) callconv(.c) noreturn; pub fn init(func: Func, stack: []align(stack_alignment) u8) !Self { stdx.debug.assertComptime(@sizeOf(usize) == 8, "usize expected to take 8 bytes", .{}); diff --git a/bazel/zig_index.json b/bazel/zig_index.json index afe965ec..2dec4081 100644 --- a/bazel/zig_index.json +++ b/bazel/zig_index.json @@ -1,83 +1,264 @@ { "master": { - "version": "0.15.0-dev.650+4f3b59f70", - "date": "2025-05-29", + "version": "0.16.0-dev.27+83f773fc6", + "date": "2025-08-24", "docs": "https://ziglang.org/documentation/master/", "stdDocs": "https://ziglang.org/documentation/master/std/", "src": { - "tarball": "https://ziglang.org/builds/zig-0.15.0-dev.650+4f3b59f70.tar.xz", - "shasum": "c14764ee9fd16f4437f2e2e7092cc7ec7ff76469f719be04155c29fa3bcc52cd", - "size": "21279148" + "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.27+83f773fc6.tar.xz", + "shasum": "afddafbede9becaa6d94a544d3115893cc6aa6492a58545b4f638c58990586dc", + "size": "21370308" }, "bootstrap": { - "tarball": "https://ziglang.org/builds/zig-bootstrap-0.15.0-dev.650+4f3b59f70.tar.xz", - "shasum": "142343992733282138b88b2205f29173ce3febcf5d73de4a19111fb36630f5a6", - "size": "52649088" + "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.27+83f773fc6.tar.xz", + "shasum": "109cdda833baf951ce8116d3fdfd50b3770b02cff5c057f777601ca694f44b7c", + "size": "52732704" }, "x86_64-macos": { - "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.15.0-dev.650+4f3b59f70.tar.xz", - "shasum": "70117a96313ddfe57bd0b3fbbbdb2beffc27fb81f5993ba9b2825628bb9f5aaa", - "size": "55795264" + "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.27+83f773fc6.tar.xz", + "shasum": "1ac88b947a6001eb30d41c15c404847209dfe945a27dbce6d1b7d401e4aef325", + "size": "55817716" }, "aarch64-macos": { - "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.15.0-dev.650+4f3b59f70.tar.xz", - "shasum": "991ef1b1871852f87e1d53f65c3dfb3b3f7187a0dc1f8fe6e7be279227b868d3", - "size": "50625312" + "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.27+83f773fc6.tar.xz", + "shasum": "75243ad6d2e9fcd634862dda9003883484024fd2e067fc7e4a0f6c524cb6c86e", + "size": "50659200" }, "x86_64-linux": { - "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.15.0-dev.650+4f3b59f70.tar.xz", - "shasum": "2c2f65db1ad72d415b5a5bfb0ccd437bfc16e91b18a58035d28ea3177d07b1e2", - "size": "53712376" + "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.27+83f773fc6.tar.xz", + "shasum": "c260aefaf5bf10bbd48101217874ccc9c0a37513729dabbbfe09bcf18fff1b5b", + "size": "53759360" }, "aarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.15.0-dev.650+4f3b59f70.tar.xz", - "shasum": "98bfd9c33b737aa66d5cc19886b772cdc99801e66d8c891d400b0370fc626455", - "size": "49500380" + "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.27+83f773fc6.tar.xz", + "shasum": "658241c15ad827a89878b2e35e13261cd1c2aaefe9a8bb94ccb907725cf80564", + "size": "49485912" }, - "armv7a-linux": { - "tarball": "https://ziglang.org/builds/zig-armv7a-linux-0.15.0-dev.650+4f3b59f70.tar.xz", - "shasum": "c204b60e27930702a64d55768fbbf2c819109dd0400168e0daf2994f4e57ed8b", - "size": "50413108" + "arm-linux": { + "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.27+83f773fc6.tar.xz", + "shasum": "6039c11f41bd032ec3cbda2d92e38b409aa9d9e2935b21266033b820ecb1a668", + "size": "50475500" }, "riscv64-linux": { - "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.15.0-dev.650+4f3b59f70.tar.xz", - "shasum": "2212613f09d114296f542dc3ca4583d3e1cfa72642e0d192a450ef1704722295", - "size": "53645244" + "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.27+83f773fc6.tar.xz", + "shasum": "f56712849af75bb1bfda502c8cdfc52dcc41c18d3d7eddd329d94be85538d258", + "size": "53610616" }, "powerpc64le-linux": { - "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.15.0-dev.650+4f3b59f70.tar.xz", - "shasum": "6a68c45d8bb5a3c1c3a259ad5ef965011c02feb0d17e8c96abd76aa48af03fe9", - "size": "53559816" + "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.27+83f773fc6.tar.xz", + "shasum": "0562fdd578b5ae65a17e614454e67466314790a1d373279a4b054398f6a7b364", + "size": "53585484" }, "x86-linux": { - "tarball": "https://ziglang.org/builds/zig-x86-linux-0.15.0-dev.650+4f3b59f70.tar.xz", - "shasum": "802e860891aa12979c3279e6d0571d08c0ed0398f362ac3f2a2fd945ef59bd1c", - "size": "56328204" + "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.27+83f773fc6.tar.xz", + "shasum": "6eafb1b1d81118066dc03b17bed6612d7a9da057794f1d0f2451c249cf8a231d", + "size": "56334072" }, "loongarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.15.0-dev.650+4f3b59f70.tar.xz", - "shasum": "7fb8e715cdf44a2158afced5eecb3d2ae0f44d0a161f43ea8dfb04e95b22c89f", - "size": "50835068" + "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.27+83f773fc6.tar.xz", + "shasum": "4fddf62a6698dbece51f504b4de8360a55d24efb0c7a56117eef64c8b717d13f", + "size": "50823012" }, "s390x-linux": { - "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.15.0-dev.650+4f3b59f70.tar.xz", - "shasum": "cbbcbf324db5158232d1fd5a1efcc2d694e058ad66772495e291828fa5227450", - "size": "53386612" + "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.27+83f773fc6.tar.xz", + "shasum": "12ed086066c373e4098dc9f132e42bbc478a3be6bd5fc6967f39396c43543f89", + "size": "53525496" }, "x86_64-windows": { - "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.15.0-dev.650+4f3b59f70.zip", - "shasum": "0d4e148a60e859f0fa5ce6d293dcec7ba79a7e1da25df151b34e3ce71d0fa84f", - "size": "94297867" + "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.27+83f773fc6.zip", + "shasum": "ae987b8d93eec8a923bba26a74d20b5d15eac7ee9694c44db82c232247810a51", + "size": "93312425" }, "aarch64-windows": { - "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.15.0-dev.650+4f3b59f70.zip", - "shasum": "6b1a094cb4c666d0078531ce3d4f970dcf01f63e5b885d9af0f498726bdf5e43", - "size": "90200802" + "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.27+83f773fc6.zip", + "shasum": "2e3c95d044dd36d302a301fb029ea080b2bda4ef8e42bbb1b2a0246973952c53", + "size": "89157759" }, "x86-windows": { - "tarball": "https://ziglang.org/builds/zig-x86-windows-0.15.0-dev.650+4f3b59f70.zip", - "shasum": "f18b6f9472a27abea8edbfb43687a75b61298e6d4da42247218dc3fd23560aa7", - "size": "96232989" + "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.27+83f773fc6.zip", + "shasum": "b696f07f7104da430c7b37750bc6e36571c3ffcd5bc49240d8a19aa4b431049f", + "size": "95215201" + }, + "aarch64-freebsd": { + "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.27+83f773fc6.tar.xz", + "shasum": "0295a3f6bebb6d25fbc916cd4e3cff1285acab75b9591695069ad9a878301203", + "size": "49384624" + }, + "arm-freebsd": { + "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.27+83f773fc6.tar.xz", + "shasum": "a293670c7ee89ce6cf11f03dee297de2a31ffdc9d26f2c0e6e78fd66b11b2327", + "size": "50925296" + }, + "powerpc64-freebsd": { + "tarball": "https://ziglang.org/builds/zig-powerpc64-freebsd-0.16.0-dev.27+83f773fc6.tar.xz", + "shasum": "503922dd11374ddc44013ae3d7b07a9a17a1404da296bc30bf8fdc6d7273fd82", + "size": "52148468" + }, + "powerpc64le-freebsd": { + "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.27+83f773fc6.tar.xz", + "shasum": "1cf44603dec137a35551916ab5e542c36e6c1ec54df4efd7783f8ae66c80acc4", + "size": "53516496" + }, + "riscv64-freebsd": { + "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.27+83f773fc6.tar.xz", + "shasum": "c4b1c691328abacf7f7b07f1900dd44c2f2852a0b2611378a2531975abb6f260", + "size": "53716116" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.27+83f773fc6.tar.xz", + "shasum": "989caf4343e1dfb4c9a68f3d3146cac96729e6f6e58276001654307f56c43981", + "size": "53808720" + }, + "aarch64-netbsd": { + "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.27+83f773fc6.tar.xz", + "shasum": "bbbe2aa2163d3c25d7414795537088245f9ec3cb53f10518b5824e1a31c35ff9", + "size": "49387264" + }, + "arm-netbsd": { + "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.27+83f773fc6.tar.xz", + "shasum": "264346f44664055d61c91087d0c55fdd04e3bdf1f2f0f969cd5efcf47d54b247", + "size": "52034816" + }, + "x86-netbsd": { + "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.27+83f773fc6.tar.xz", + "shasum": "f5e3cdab5d7e24552eee2200b1e4ac45cd84f13d5d64ae0f558ad0210d6883d5", + "size": "56915916" + }, + "x86_64-netbsd": { + "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.27+83f773fc6.tar.xz", + "shasum": "b6418c9d5036d2328271078687dbf2d772279e65787546974943855423fc6545", + "size": "53809524" + } + }, + "0.15.1": { + "date": "2025-08-19", + "docs": "https://ziglang.org/documentation/master/", + "stdDocs": "https://ziglang.org/documentation/master/std/", + "notes": "https://ziglang.org/download/0.15.1/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.15.1/zig-0.15.1.tar.xz", + "shasum": "816c0303ab313f59766ce2097658c9fff7fafd1504f61f80f9507cd11652865f", + "size": "21359884" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.15.1/zig-bootstrap-0.15.1.tar.xz", + "shasum": "4c0cfbcf12da144955761ca43f89e3c74956bce978694fc1d0a63555f5c0a199", + "size": "52711548" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-macos-0.15.1.tar.xz", + "shasum": "9919392e0287cccc106dfbcbb46c7c1c3fa05d919567bb58d7eb16bca4116184", + "size": "55791880" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-macos-0.15.1.tar.xz", + "shasum": "c4bd624d901c1268f2deb9d8eb2d86a2f8b97bafa3f118025344242da2c54d7b", + "size": "50644996" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-linux-0.15.1.tar.xz", + "shasum": "c61c5da6edeea14ca51ecd5e4520c6f4189ef5250383db33d01848293bfafe05", + "size": "53734456" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-linux-0.15.1.tar.xz", + "shasum": "bb4a8d2ad735e7fba764c497ddf4243cb129fece4148da3222a7046d3f1f19fe", + "size": "49493872" + }, + "arm-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-arm-linux-0.15.1.tar.xz", + "shasum": "3f4bf3b06b67d14e3f38be30798488c1abe3cf5b33de570cd0e87bbf09b978ad", + "size": "50477464" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-riscv64-linux-0.15.1.tar.xz", + "shasum": "7ca7a3e621436fb31d66a253132fc39574a13d2a1b4d8458af4f2e7c6e4374fe", + "size": "53597792" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-powerpc64le-linux-0.15.1.tar.xz", + "shasum": "339e2106496be70b614e32d444298216e676c36d08f5cd7bee3dd1dbd4567fd7", + "size": "53566944" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86-linux-0.15.1.tar.xz", + "shasum": "dff166f25fdd06e8341d831a71211b5ba7411463a6b264bdefa8868438690b6a", + "size": "56311228" + }, + "loongarch64-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-loongarch64-linux-0.15.1.tar.xz", + "shasum": "0af18a012f3c4cffbef29ab5f42021484d92517f921a1380faacc89c50c1f89d", + "size": "50794376" + }, + "s390x-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-s390x-linux-0.15.1.tar.xz", + "shasum": "bcd13e5c88cf2d0da7100a48572195560069a94402c9bec3b316398204aa27e2", + "size": "53508068" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-windows-0.15.1.zip", + "shasum": "91e69e887ca8c943ce9a515df3af013d95a66a190a3df3f89221277ebad29e34", + "size": "92612958" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-windows-0.15.1.zip", + "shasum": "1f1bf16228b0ffcc882b713dc5e11a6db4219cb30997e13c72e8e723c2104ec6", + "size": "88458549" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86-windows-0.15.1.zip", + "shasum": "fb1c07cffbb43615d3158ab8b8f5db5da1d48875eca99e1d7a8a0064ff63fc5b", + "size": "94516052" + }, + "aarch64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-freebsd-0.15.1.tar.xz", + "shasum": "4d9d25c775828d49ea037b2284310c295d951793da8ebe94827a54fed4cca3ce", + "size": "49358464" + }, + "arm-freebsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-arm-freebsd-0.15.1.tar.xz", + "shasum": "9707f3a5f7e1a3d99c40db9a74de1acc61016a197ad289c2ad964f93cb213a18", + "size": "50904124" + }, + "powerpc64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-powerpc64-freebsd-0.15.1.tar.xz", + "shasum": "79448884372db04e62f77a46b92245fce805063e69534729537f75cb4681e7e3", + "size": "52099020" + }, + "powerpc64le-freebsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-powerpc64le-freebsd-0.15.1.tar.xz", + "shasum": "f18ee12ba9c98a20b8d2ad0410c679e7aa5591bc7917f169fd6b377833d2c7ad", + "size": "53480976" + }, + "riscv64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-riscv64-freebsd-0.15.1.tar.xz", + "shasum": "ee9f864a6fd8b57c1f4fdbb11daa06578746a6f8253afe3f5ddb5a76f2eddd2d", + "size": "53677800" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-freebsd-0.15.1.tar.xz", + "shasum": "9714f8ac3d3dc908b1599837c6167f857c1efaa930f0cfa840699458de7c3cd0", + "size": "53782112" + }, + "aarch64-netbsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-netbsd-0.15.1.tar.xz", + "shasum": "b2a528399777583b85b89c54ccd45488af7709d6dd29a27323ec2a229db40910", + "size": "49368000" + }, + "arm-netbsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-arm-netbsd-0.15.1.tar.xz", + "shasum": "93dc70109cbf5d2e022d20dfb56211978c4ea3c0b1e67aaabff947d8d1583aab", + "size": "52028184" + }, + "x86-netbsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86-netbsd-0.15.1.tar.xz", + "shasum": "a91b26051822ff17f3143f859b87dce5b4a13e90928bd6daa6f07a895d3410f0", + "size": "56881864" + }, + "x86_64-netbsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-netbsd-0.15.1.tar.xz", + "shasum": "6d7ba6eca5b4434351ebdb971b7303c9934514f9bb8481852251dbd5b52b03d6", + "size": "53794836" } }, "0.14.1": { diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel index 6f4cd70c..635eb087 100644 --- a/examples/llama/BUILD.bazel +++ b/examples/llama/BUILD.bazel @@ -10,6 +10,7 @@ zig_binary( "llama.zig", ], main = "main.zig", + visibility = ["//visibility:public"], deps = [ "//async", "//stdx", diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index ee9c852f..da487bb3 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -87,7 +87,7 @@ pub const LlamaLM = struct { kv_cache: KvCache, rng: Tensor.Rng, ) struct { Tensor, KvCache, Tensor.Rng } { - stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Llama ! Expected >=1d tokens and 0d token_index, got: {} and {}", .{ tokens_, token_index }); + stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Llama ! Expected >=1d tokens and 0d token_index, got: {f} and {f}", .{ tokens_, token_index }); const tokens = tokens_.withPartialTags(.{.s}); const out, const updated_kv_cache = zml.call(self.model, .forward, .{ tokens, token_index, kv_cache }); const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); @@ -200,8 +200,8 @@ pub const TransformerLayer = struct { kv_cache: KvCache, ) struct { Tensor, KvCache } { // Self Attention - //log.debug("TransformerLayer({}) -> {}", .{ x0, self.input_layernorm.forward(x0) }); - stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {}", .{x0}); + //log.debug("TransformerLayer({f}) -> {f}", .{ x0, self.input_layernorm.forward(x0) }); + stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {f}", .{x0}); const x0_normalized = zml.call(self.input_layernorm, .forward, .{x0}); const delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); @@ -336,7 +336,7 @@ pub const KvCache = struct { return .{ .k = try zml.Buffer.constant(platform, kv_shape, 1), .v = try zml.Buffer.constant(platform, kv_shape, 1), - .layer_index = try zml.Buffer.constant(platform, zml.Shape.init(.{}, .u32), 0), + .layer_index = try zml.Buffer.scalar(platform, 0, .u32), }; } diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 0401444f..af94fd21 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -1,19 +1,19 @@ +const std = @import("std"); + const asynk = @import("async"); const clap = @import("clap"); -const std = @import("std"); const stdx = @import("stdx"); const zml = @import("zml"); +const Buffer = zml.Buffer; +const Tensor = zml.Tensor; +const ShapeOf = zml.ShapeOf; const llama = @import("llama.zig"); - const LlamaLM = llama.LlamaLM; const Llama = llama.Llama; const KvCache = llama.KvCache; const TransformerLayer = llama.TransformerLayer; const SelfAttn = llama.SelfAttn; -const Buffer = zml.Buffer; -const Tensor = zml.Tensor; -const ShapeOf = zml.ShapeOf; const log = std.log.scoped(.llama); @@ -23,7 +23,7 @@ pub const std_options: std.Options = .{ }; pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: LlamaLM.Config, prompt: []const u8, skip_llama3_encoding: bool) ![]u32 { - var tokens = std.ArrayList(u32).init(allocator); + var tokens = std.array_list.Managed(u32).init(allocator); var encoder = try tokenizer.encoder(); defer encoder.deinit(); @@ -101,7 +101,7 @@ pub fn generateText( defer current_token.deinit(); // Here we collect the generated text - var output = std.ArrayList(u8).init(allocator); + var output = std.array_list.Managed(u8).init(allocator); defer output.deinit(); const output_tokens_len = max_seq_len - prompt_tok.len - 1; @@ -179,21 +179,22 @@ pub fn asyncMain() !void { .PATH = clap.parsers.string, }; var diag: clap.Diagnostic = .{}; - const stderr = std.io.getStdErr().writer(); + var stderr_buffer: [1024]u8 = undefined; + var stderr = std.fs.File.stderr().writer(&stderr_buffer); var res = clap.parse(clap.Help, ¶ms, parsers, .{ .diagnostic = &diag, .allocator = allocator, }) catch |err| { - diag.report(stderr, err) catch {}; - stderr.print("usage: ", .{}) catch {}; - clap.usage(stderr, clap.Help, ¶ms) catch {}; - stderr.print("\n", .{}) catch {}; + diag.report(&stderr.interface, err) catch {}; + stderr.interface.print("usage: ", .{}) catch {}; + clap.usage(&stderr.interface, clap.Help, ¶ms) catch {}; + stderr.interface.print("\n", .{}) catch {}; return; }; defer res.deinit(); if (res.args.help != 0) { - clap.help(std.io.getStdErr().writer(), clap.Help, ¶ms, .{}) catch {}; + clap.help(&stderr.interface, clap.Help, ¶ms, .{}) catch {}; return; } @@ -224,7 +225,9 @@ pub fn asyncMain() !void { const config = blk: { var config_json_file = try asynk.File.open(model_config_path, .{ .mode = .read_only }); defer config_json_file.close() catch unreachable; - var reader = std.json.reader(allocator, config_json_file.reader()); + var config_json_buffer: [256]u8 = undefined; + var config_reader = config_json_file.reader(&config_json_buffer); + var reader = std.json.Reader.init(allocator, &config_reader.interface); defer reader.deinit(); const config_obj = try std.json.parseFromTokenSourceLeaky(llama.LlamaLM.Config, allocator, &reader, .{ .ignore_unknown_fields = true }); break :blk config_obj; @@ -298,16 +301,16 @@ pub fn asyncMain() !void { platform, }); - log.info("\tLoading Llama weights from {?s}...", .{model_weights_path}); + log.info("\tLoading Llama weights from {s}...", .{model_weights_path}); var llama_weights = try zml.aio.loadBuffers(llama.LlamaLM, .{ config, llama_options }, ts, model_arena.allocator(), platform); defer zml.aio.unloadBuffers(&llama_weights); - log.info("✅\tLoaded weights in {}", .{std.fmt.fmtDuration(start.read())}); + log.info("✅\tLoaded weights in {D}", .{start.read()}); var llama_module_prefill = (try fut_mod_prefill.awaitt()).prepare(llama_weights); defer llama_module_prefill.deinit(); var llama_module = (try fut_mod.awaitt()).prepare(llama_weights); defer llama_module.deinit(); - log.info("✅\tCompiled model in {}", .{std.fmt.fmtDuration(start.read())}); + log.info("✅\tCompiled model in {D}", .{start.read()}); log.info("Creating KvCache", .{}); const kv_cache = try llama.KvCache.initBuffer(kv_shape, platform); @@ -315,7 +318,7 @@ pub fn asyncMain() !void { var tokenizer = blk: { log.info("Loading tokenizer from {s}", .{model_tokenizer_path}); var timer = try stdx.time.Timer.start(); - defer log.info("Loaded tokenizer from {s} [{}]", .{ model_tokenizer_path, timer.read() }); + defer log.info("Loaded tokenizer from {s} [{D}]", .{ model_tokenizer_path, timer.read() }); break :blk try zml.tokenizer.Tokenizer.fromFile(model_arena.allocator(), model_tokenizer_path); }; diff --git a/examples/modernbert/main.zig b/examples/modernbert/main.zig index 8e11e79f..95d5aff1 100644 --- a/examples/modernbert/main.zig +++ b/examples/modernbert/main.zig @@ -1,7 +1,4 @@ const std = @import("std"); -const log = std.log.scoped(.modernbert); - -const modernbert = @import("modernbert.zig"); const asynk = @import("async"); const clap = @import("clap"); @@ -9,6 +6,10 @@ const stdx = @import("stdx"); const zml = @import("zml"); const Tensor = zml.Tensor; +const modernbert = @import("modernbert.zig"); + +const log = std.log.scoped(.modernbert); + pub const std_options: std.Options = .{ .log_level = .info, .log_scope_levels = &[_]std.log.ScopeLevel{ @@ -42,20 +43,20 @@ pub fn main() !void { pub fn asyncMain() !void { const allocator = std.heap.c_allocator; - const stderr = std.io.getStdErr().writer(); + const stderr = std.fs.File.stderr(); var diag: clap.Diagnostic = .{}; var cli = clap.parse(clap.Help, ¶ms, clap_parsers, .{ .diagnostic = &diag, .allocator = allocator, }) catch |err| { - try diag.report(stderr, err); + try diag.reportToFile(stderr, err); try printUsageAndExit(stderr); }; defer cli.deinit(); if (cli.args.help != 0) { - try clap.help(stderr, clap.Help, ¶ms, .{}); + try clap.helpToFile(stderr, clap.Help, ¶ms, .{}); return; } @@ -80,7 +81,9 @@ pub fn asyncMain() !void { // Detects the format of the model file (base on filename) and open it. const model_file = cli.args.model orelse { - stderr.print("Error: missing --model=...\n\n", .{}) catch {}; + var buf: [256]u8 = undefined; + var writer = stderr.writer(&buf); + writer.interface.print("Error: missing --model=...\n\n", .{}) catch {}; printUsageAndExit(stderr); unreachable; }; @@ -96,7 +99,7 @@ pub fn asyncMain() !void { if (cli.args.tokenizer) |tok| { log.info("\tLoading tokenizer from {s}", .{tok}); var timer = try stdx.time.Timer.start(); - defer log.info("✅\tLoaded tokenizer from {s} [{}]", .{ tok, timer.read() }); + defer log.info("✅\tLoaded tokenizer from {s} [{D}]", .{ tok, timer.read() }); break :blk try zml.tokenizer.Tokenizer.fromFile(model_arena, tok); } else { @@ -123,13 +126,13 @@ pub fn asyncMain() !void { const seq_len = @as(i64, @intCast(cli.args.@"seq-len" orelse 256)); const input_shape = zml.Shape.init(.{ .b = 1, .s = seq_len }, .u32); - var start = try std.time.Timer.start(); + var start = try stdx.time.Timer.start(); // Load weights - log.info("\tLoading ModernBERT weights from {?s}...", .{model_file}); + log.info("\tLoading ModernBERT weights from {s}...", .{model_file}); var bert_weights = try zml.aio.loadBuffers(modernbert.ModernBertForMaskedLM, .{modernbert_options}, tensor_store, model_arena, platform); defer zml.aio.unloadBuffers(&bert_weights); - log.info("✅\tLoaded weights in {d}ms", .{start.read() / std.time.ns_per_ms}); + log.info("✅\tLoaded weights in {D}", .{start.read()}); // Compile the model log.info("\tCompiling ModernBERT model...", .{}); @@ -143,7 +146,7 @@ pub fn asyncMain() !void { }); var bert_module = (try fut_mod.awaitt()).prepare(bert_weights); defer bert_module.deinit(); - log.info("✅\tLoaded weights and compiled model in {d}ms", .{start.read() / std.time.ns_per_ms}); + log.info("✅\tLoaded weights and compiled model in {D}", .{start.read()}); const text = cli.args.text orelse "Paris is the [MASK] of France."; log.info("\tInput text: {s}", .{text}); @@ -213,7 +216,7 @@ pub fn unmask( } pub fn tokenize(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]const u32 { - var tokens = std.ArrayList(u32).init(allocator); + var tokens = std.array_list.Managed(u32).init(allocator); var encoder = try tokenizer.encoder(); defer encoder.deinit(); @@ -228,7 +231,7 @@ pub fn tokenize(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer } fn findMaskPositions(allocator: std.mem.Allocator, tokens: []const u32, mask_token: u32) ![]usize { - var mask_positions = std.ArrayList(usize).init(allocator); + var mask_positions = std.array_list.Managed(usize).init(allocator); defer mask_positions.deinit(); for (tokens, 0..) |token, i| { @@ -267,9 +270,7 @@ fn bool_parser(in: []const u8) error{}!bool { return std.mem.indexOfScalar(u8, "tTyY1", in[0]) != null; } -fn printUsageAndExit(stderr: anytype) noreturn { - stderr.print("usage: ", .{}) catch {}; - clap.usage(stderr, clap.Help, ¶ms) catch {}; - stderr.print("\n", .{}) catch {}; +fn printUsageAndExit(stderr: std.fs.File) noreturn { + clap.usageToFile(stderr, clap.Help, ¶ms) catch {}; std.process.exit(0); } diff --git a/examples/simple_layer/main.zig b/examples/simple_layer/main.zig index 3ffc1471..a84ab056 100644 --- a/examples/simple_layer/main.zig +++ b/examples/simple_layer/main.zig @@ -1,6 +1,7 @@ const std = @import("std"); -const zml = @import("zml"); + const asynk = @import("async"); +const zml = @import("zml"); /// Model definition const Layer = struct { @@ -38,9 +39,9 @@ pub fn asyncMain() !void { context.printAvailablePlatforms(platform); // Our weights and bias to use - var weights = [4]f16{ 2.0, 2.0, 2.0, 2.0 }; - var bias = [4]f16{ 1.0, 2.0, 3.0, 4.0 }; - const input_shape = zml.Shape.init(.{4}, .f16); + var weights = [4]f32{ 2.0, 2.0, 2.0, 2.0 }; + var bias = [4]f32{ 1.0, 2.0, 3.0, 4.0 }; + const input_shape = zml.Shape.init(.{4}, .f32); // We manually produce a BufferStore. You would not normally do that. // A BufferStore is usually created by loading model data from a file. @@ -80,8 +81,8 @@ pub fn asyncMain() !void { // prepare an input buffer // Here, we use zml.HostBuffer.fromSlice to show how you would create a HostBuffer // with a specific shape from an array. - // For situations where e.g. you have an [4]f16 array but need a .{2, 2} input shape. - var input = [4]f16{ 5.0, 5.0, 5.0, 5.0 }; + // For situations where e.g. you have an [4]f32 array but need a .{2, 2} input shape. + var input = [4]f32{ 5.0, 5.0, 5.0, 5.0 }; var input_buffer = try zml.Buffer.from(platform, zml.HostBuffer.fromSlice(input_shape, &input), .{}); defer input_buffer.deinit(); @@ -92,7 +93,7 @@ pub fn asyncMain() !void { // fetch the result to CPU memory const cpu_result = try result.toHostAlloc(arena); std.debug.print( - "\nThe result of {d} * {d} + {d} = {d}\n", - .{ &weights, &input, &bias, cpu_result.items(f16) }, + "\nThe result of {any} * {any} + {any} = {any}\n", + .{ &weights, &input, &bias, cpu_result.items(f32) }, ); } diff --git a/ffi/zig_allocator.zig b/ffi/zig_allocator.zig index 412dcd1e..35f5fc33 100644 --- a/ffi/zig_allocator.zig +++ b/ffi/zig_allocator.zig @@ -10,13 +10,13 @@ pub const ZigAllocator = struct { }; } - pub fn alloc(ctx: ?*const anyopaque, elem: usize, nelems: usize, alignment: usize) callconv(.C) ?*anyopaque { + pub fn alloc(ctx: ?*const anyopaque, elem: usize, nelems: usize, alignment: usize) callconv(.c) ?*anyopaque { const self: *const std.mem.Allocator = @ptrCast(@alignCast(ctx)); const ret = self.rawAlloc(elem * nelems, std.math.log2_int(usize, alignment), @returnAddress()) orelse return null; return @ptrCast(ret); } - pub fn free(ctx: ?*const anyopaque, ptr: ?*anyopaque, elem: usize, nelems: usize, alignment: usize) callconv(.C) void { + pub fn free(ctx: ?*const anyopaque, ptr: ?*anyopaque, elem: usize, nelems: usize, alignment: usize) callconv(.c) void { const self: *const std.mem.Allocator = @ptrCast(@alignCast(ctx)); const memory: [*c]u8 = @ptrCast(ptr); const size = elem * nelems; diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/BUILD.bazel index 845775e3..101f67ff 100644 --- a/mlir/dialects/BUILD.bazel +++ b/mlir/dialects/BUILD.bazel @@ -15,6 +15,7 @@ zig_library( deps = [ "//mlir", "//mlir/dialects/stablehlo", + "//stdx", ], ) diff --git a/mlir/dialects/func.zig b/mlir/dialects/func.zig index 9db36121..868ff3bc 100644 --- a/mlir/dialects/func.zig +++ b/mlir/dialects/func.zig @@ -1,6 +1,7 @@ const std = @import("std"); const mlir = @import("mlir"); +const stdx = @import("stdx"); pub fn func( ctx: mlir.Context, @@ -14,7 +15,7 @@ pub fn func( location: mlir.Location, }, ) mlir.Operation { - var attrs_tuple_buffer = std.BoundedArray(mlir.AttrTuple, 4){}; + var attrs_tuple_buffer = stdx.BoundedArray(mlir.AttrTuple, 4){}; attrs_tuple_buffer.appendAssumeCapacity(.{ "sym_name", .string(ctx, args.sym_name) }); attrs_tuple_buffer.appendAssumeCapacity(.{ "function_type", .type_(.function(ctx, args.args, args.results)) }); if (args.arg_attrs.len > 0) { diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 8010985d..0c60fd01 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -761,7 +761,7 @@ pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCa ); } - var attrs: std.BoundedArray(mlir.AttrTuple, 32) = .{}; + var attrs: stdx.BoundedArray(mlir.AttrTuple, 32) = .{}; attrs.appendSliceAssumeCapacity(&[_]mlir.AttrTuple{ .{ "api_version", .int(ctx, .i32, @intFromEnum(opts.api_version)) }, .{ "call_target_name", .string(ctx, opts.call_target_name) }, @@ -770,7 +770,7 @@ pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCa }); { - var output_operand_aliases: std.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; + var output_operand_aliases: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; for (opts.output_operand_aliases) |alias| { output_operand_aliases.appendAssumeCapacity( OutputOperandAliasAttribute.init(ctx, &.{}, alias, &.{}).asAttr(), @@ -789,14 +789,14 @@ pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCa }; if (opts.operand_layouts) |layouts| { - var operand_layouts: std.BoundedArray(mlir.Attribute, MAX_OPERANDS) = .{}; + var operand_layouts: stdx.BoundedArray(mlir.Attribute, MAX_OPERANDS) = .{}; for (layouts) |ol| { operand_layouts.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(ol.len)}, .index, ol)); } attrs.appendAssumeCapacity(.{ "operand_layouts", .array(ctx, operand_layouts.constSlice()) }); } else { const operand_layouts = blk: { - var ret: std.BoundedArray(mlir.Attribute, MAX_OPERANDS) = .{}; + var ret: stdx.BoundedArray(mlir.Attribute, MAX_OPERANDS) = .{}; for (inputs) |input| { const ranked_type = input.getType().as(mlir.RankedTensorType).?; const ol = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - ranked_type.getRank() ..]; @@ -808,14 +808,14 @@ pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCa } if (opts.result_layouts) |layouts| { - var result_layouts: std.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; + var result_layouts: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; for (layouts) |rl| { result_layouts.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(rl.len)}, .index, rl)); } attrs.appendAssumeCapacity(.{ "result_layouts", .array(ctx, result_layouts.constSlice()) }); } else { const result_layouts = blk: { - var ret: std.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; + var ret: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; for (res_types) |t| { const ranked_t = t.as(mlir.RankedTensorType).?; const rl = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - ranked_t.getRank() ..]; @@ -1271,7 +1271,7 @@ pub fn stablehloVersionFromCompatibilityRequirement(requirement: c.MlirStablehlo const WriterContext = @TypeOf(context); c.stablehloVersionFromCompatibilityRequirement(req, (struct { - pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.C) void { + pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.c) void { const inner_ctx: *WriterContext = @ptrCast(@alignCast(userdata)); _ = inner_ctx.writer.write(mlir.fromStringRef(mlir_str)) catch unreachable; } @@ -1292,7 +1292,7 @@ pub fn stablehloGetSmallerVersion(version1: []const u8, version2: []const u8) [] const WriterContext = @TypeOf(context); _ = c.stablehloGetSmallerVersion(mlir.stringRef(version1), mlir.stringRef(version2), (struct { - pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.C) void { + pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.c) void { const inner_ctx: *WriterContext = @ptrCast(@alignCast(userdata)); _ = inner_ctx.writer.write(mlir.fromStringRef(mlir_str)) catch unreachable; } @@ -1313,7 +1313,7 @@ pub fn getCurrentVersion() []const u8 { const ContextWriter = @TypeOf(writer_); c.stablehloGetCurrentVersion((struct { - pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.C) void { + pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.c) void { const writer: *ContextWriter = @ptrCast(@alignCast(userdata)); _ = writer.write(mlir.fromStringRef(mlir_str)) catch unreachable; } @@ -1339,7 +1339,7 @@ pub fn getMinimumVersion() []const u8 { const WriterContext = @TypeOf(context); c.stablehloGetMinimumVersion((struct { - pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.C) void { + pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.c) void { const inner_ctx: *WriterContext = @ptrCast(@alignCast(userdata)); _ = inner_ctx.writer.write(mlir.fromStringRef(mlir_str)) catch unreachable; } @@ -1358,7 +1358,7 @@ pub fn serializePortableArtifact(bytecode: []const u8, target_version: []const u const WriterContext = @TypeOf(context); try mlir.successOr(c.stablehloSerializePortableArtifactFromStringRef(mlir.stringRef(bytecode), mlir.stringRef(target_version), (struct { - pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.C) void { + pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.c) void { const inner_ctx: *WriterContext = @ptrCast(@alignCast(userdata)); _ = inner_ctx.writer.write(mlir.fromStringRef(mlir_str)) catch unreachable; } diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 10d26c93..a222c40a 100755 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -40,7 +40,7 @@ pub fn successOr(res: c.MlirLogicalResult, err: anytype) !void { } /// Alternative to MlirWrapperType -pub const MlirStrCallback = fn (c.MlirStringRef, ?*anyopaque) callconv(.C) void; +pub const MlirStrCallback = fn (c.MlirStringRef, ?*anyopaque) callconv(.c) void; pub const Registry = struct { _inner: c.MlirDialectRegistry, @@ -171,7 +171,7 @@ pub const PassManager = struct { } }; -fn _mlir_passpipeline_error(err: c.MlirStringRef, ctx: ?*anyopaque) callconv(.C) void { +fn _mlir_passpipeline_error(err: c.MlirStringRef, ctx: ?*anyopaque) callconv(.c) void { _ = ctx; std.debug.print(">>ERROR: {s}\n", .{err.data}); } @@ -754,7 +754,7 @@ pub const Operation = struct { state.addOperands(operands); } else if (args.variadic_operands) |operands_segments| { const MAX_SEGMENTS = 32; - var segments: std.BoundedArray(i32, MAX_SEGMENTS) = .{}; + var segments: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; for (operands_segments) |operands| { state.addOperands(operands); @@ -764,7 +764,7 @@ pub const Operation = struct { } else if (args.tt_variadic_operands) |operands_segments| { // stablehlo and triton seems to disagree on the expected type of operandSegmentSizes, let's fix that. const MAX_SEGMENTS = 32; - var segments: std.BoundedArray(i32, MAX_SEGMENTS) = .{}; + var segments: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; for (operands_segments) |operands| { state.addOperands(operands); @@ -811,7 +811,7 @@ pub const Operation = struct { @panic("Failed to create MLIR operation"); }; if (args.verify and new_op.verify() == false) { - log.err("Failed to verify MLIR operation:\n{}", .{new_op.mlirFormatter(.{ .debug_info = true })}); + log.err("Failed to verify MLIR operation:\n{f}", .{new_op.mlirFormatter(.{ .debug_info = true })}); @panic("Failed to verify MLIR operation"); } return new_op; @@ -888,7 +888,7 @@ pub const Operation = struct { c.mlirOperationWriteBytecode( self._inner, (struct { - pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.C) void { + pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { const inner_writer_context: *WriterContext = @ptrCast(@alignCast(ctx_)); _ = inner_writer_context.writer.write(str.data[0..str.length]) catch unreachable; } @@ -916,7 +916,7 @@ pub const Operation = struct { self._inner, cfg, (struct { - pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.C) void { + pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { const inner_writer_context: *WriterContext = @ptrCast(@alignCast(ctx_)); _ = inner_writer_context.writer.write(str.data[0..str.length]) catch |err| { inner_writer_context.write_error = err; @@ -939,29 +939,25 @@ pub const Operation = struct { op: Operation, flags: OpPrintingFlags, - pub fn format(self: @This(), comptime fmt: []const u8, options: std.fmt.FormatOptions, writer: anytype) !void { - _ = fmt; - _ = options; + pub fn format(self: @This(), writer: anytype) !void { self.op.print(writer, self.flags); } }; - pub fn print(self: Self, writer: anytype, flags: OpPrintingFlags) void { + pub fn print(self: Self, writer: *std.Io.Writer, flags: OpPrintingFlags) void { const pflags = flags.create(); defer c.mlirOpPrintingFlagsDestroy(pflags); - var writer_context = .{ .writer = writer }; - const WriterContext = @TypeOf(writer_context); c.mlirOperationPrintWithFlags( self._inner, pflags, (struct { - pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.C) void { - const inner_writer_context: *WriterContext = @ptrCast(@alignCast(ctx_)); - _ = inner_writer_context.writer.write(str.data[0..str.length]) catch unreachable; + pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { + const _writer: *std.Io.Writer = @ptrCast(@alignCast(ctx_)); + _writer.writeAll(str.data[0..str.length]) catch @panic("Mlir print failed"); } }).callback, - &writer_context, + writer, ); } @@ -991,7 +987,7 @@ pub const Operation = struct { c.mlirOperationWalk( self._inner, (struct { - pub fn callback(op: c.MlirOperation, ctx_: ?*anyopaque) callconv(.C) c.MlirWalkResult { + pub fn callback(op: c.MlirOperation, ctx_: ?*anyopaque) callconv(.c) c.MlirWalkResult { const inner_ctx_: *ContextType = @ptrCast(@alignCast(ctx_)); return @intFromEnum(walkfn(inner_ctx_.ctx, .{ ._inner = op })); } @@ -1017,24 +1013,26 @@ pub const Operation = struct { return c.mlirOperationRemoveAttributeByName(self._inner, stringRef(name_)); } + /// Hash the canonicalized IR, without debug information that can change across builds. pub fn hash(op: Operation, hasher: *std.hash.XxHash64) void { - const NoError = error{}; - const write = struct { - fn write(hasher_: *std.hash.XxHash64, bytes: []const u8) NoError!usize { - hasher_.update(bytes); - return bytes.len; - } - }.write; - const HashWriter = std.io.Writer(*std.hash.XxHash64, NoError, write); - const writer: HashWriter = .{ .context = hasher }; - - // Hash the canonicalized IR, without debug information that can change across builds. // Note: before we where using op.writeBytecode(writer), // but it crashes on some inputs, notably for unused variables. // So we use the text representation of the mlir. // See https://github.com/zml/zml/issues/97. - // Writes can't fail because we are writing to a hasher. - op.print(writer, .{ .debug_info = false }); + const flags = OpPrintingFlags.create(.{ .debug_info = false }); + defer c.mlirOpPrintingFlagsDestroy(flags); + + c.mlirOperationPrintWithFlags( + op._inner, + flags, + (struct { + pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { + const _hasher: *std.hash.XxHash64 = @ptrCast(@alignCast(ctx_)); + _hasher.update(str.data[0..str.length]); + } + }).callback, + hasher, + ); } }; @@ -1185,9 +1183,9 @@ pub const BlockArgument = struct { return @bitCast(c.mlirBlockArgumentGetArgNumber(arg._inner)); } - pub fn format(self: BlockArgument, comptime fmt: []const u8, options: std.fmt.FormatOptions, writer: anytype) !void { + pub fn format(self: BlockArgument, writer: anytype) !void { const value = Value{ ._inner = self._inner }; - return value.format(fmt, options, writer); + return value.format(writer); } }; @@ -1234,8 +1232,8 @@ pub const Type = struct { pub fn formatAny(SpecificType: type) fn (SpecificType, SpecificType) type { return struct { - pub fn format(self: SpecificType, comptime fmt: []const u8, options: std.fmt.FormatOptions, writer: anytype) !void { - return try Type.format(self.asType(), fmt, options, writer); + pub fn format(self: SpecificType, writer: anytype) !void { + return try Type.format(self.asType(), writer); } }; } @@ -1344,7 +1342,7 @@ pub fn IntegerType(comptime it: IntegerTypes) type { pub const eql = Type.eqlAny(Int); pub const format = helpers.format(Int, c.mlirTypePrint); - fn typeIsAIntegerExact(typ: c.MlirType) callconv(.C) bool { + fn typeIsAIntegerExact(typ: c.MlirType) callconv(.c) bool { const bit_width = Config[0]; const is_sign = Config[2]; return c.mlirTypeIsAInteger(typ) and (c.mlirIntegerTypeGetWidth(typ) == bit_width) and is_sign(typ); @@ -1422,20 +1420,20 @@ pub fn ComplexType(comptime ct: ComplexTypes) type { _inner: c.MlirType, const Complex = @This(); - fn mlirC64TypeGet(ctx: c.MlirContext) callconv(.C) c.MlirType { + fn mlirC64TypeGet(ctx: c.MlirContext) callconv(.c) c.MlirType { return c.mlirComplexTypeGet(c.mlirF32TypeGet(ctx)); } - fn mlirC128TypeGet(ctx: c.MlirContext) callconv(.C) c.MlirType { + fn mlirC128TypeGet(ctx: c.MlirContext) callconv(.c) c.MlirType { return c.mlirComplexTypeGet(c.mlirF64TypeGet(ctx)); } - fn mlirTypeIsAC64(typ: c.MlirType) callconv(.C) bool { + fn mlirTypeIsAC64(typ: c.MlirType) callconv(.c) bool { const element_type: c.MlirType = c.mlirComplexTypeGetElementType(typ); return c.mlirTypeIsAF32(element_type); } - fn mlirTypeIsAC128(typ: c.MlirType) callconv(.C) bool { + fn mlirTypeIsAC128(typ: c.MlirType) callconv(.c) bool { const element_type: c.MlirType = c.mlirComplexTypeGetElementType(typ); return c.mlirTypeIsAF64(element_type); } @@ -1446,7 +1444,7 @@ pub fn ComplexType(comptime ct: ComplexTypes) type { .unknown => .{ c.mlirTypeIsAComplex, null }, }; - fn typeIsAUnknownComplex(typ: c.MlirType) callconv(.C) bool { + fn typeIsAUnknownComplex(typ: c.MlirType) callconv(.c) bool { return c.mlirTypeIsAComplex(typ); } @@ -1685,7 +1683,7 @@ pub const Block = struct { .op_result => |parent_op| self.appendOperationRecursive(parent_op, opt), .block_argument => |arg| { // Hermetic blocks are not allowed to use arguments from other blocks. - stdx.debug.assert(opt == .open or self.eql(arg.block()), "Can't add {} from {?x} block to {?x} block", .{ arg, arg.block()._inner.ptr, self._inner.ptr }); + stdx.debug.assert(opt == .open or self.eql(arg.block()), "Can't add {f} from {*} block to {*} block", .{ arg, arg.block()._inner.ptr, self._inner.ptr }); }, .null => @panic("InvalidMlir"), } @@ -1694,7 +1692,7 @@ pub const Block = struct { pub fn appendOperationRecursive(self: Block, op: Operation, opt: RecursiveOpts) void { if (op.block()) |prev_block| { // Hermetic blocks are not allowed to reference values from other blocks. - stdx.debug.assert(opt == .open or self.equals(prev_block), "Can't add {} from {?x} block to {?x} block", .{ op, prev_block._inner.ptr, self._inner.ptr }); + stdx.debug.assert(opt == .open or self.equals(prev_block), "Can't add {} from {*} block to {*} block", .{ op, prev_block._inner.ptr, self._inner.ptr }); return; } for (0..op.numOperands()) |i| { @@ -1705,7 +1703,7 @@ pub const Block = struct { }; pub const helpers = struct { - pub fn eql(T: type, equal_fn: fn (@FieldType(T, "_inner"), @FieldType(T, "_inner")) callconv(.C) bool) fn (T, T) bool { + pub fn eql(T: type, equal_fn: fn (@FieldType(T, "_inner"), @FieldType(T, "_inner")) callconv(.c) bool) fn (T, T) bool { return struct { fn eql(a: T, b: T) bool { return equal_fn(a._inner, b._inner); @@ -1713,7 +1711,7 @@ pub const helpers = struct { }.eql; } - pub fn deinit(T: type, deinit_fn: fn (@FieldType(T, "_inner")) callconv(.C) void) fn (*T) void { + pub fn deinit(T: type, deinit_fn: fn (@FieldType(T, "_inner")) callconv(.c) void) fn (*T) void { return struct { fn deinit(a: *T) void { deinit_fn(a._inner); @@ -1722,7 +1720,7 @@ pub const helpers = struct { }.deinit; } - pub fn dump(T: type, dump_fn: fn (@FieldType(T, "_inner")) callconv(.C) void) fn (T) void { + pub fn dump(T: type, dump_fn: fn (@FieldType(T, "_inner")) callconv(.c) void) fn (T) void { return struct { fn dump(a: T) void { return dump_fn(a._inner); @@ -1730,7 +1728,7 @@ pub const helpers = struct { }.dump; } - pub fn isNull(T: type, is_null_fn: fn (@FieldType(T, "_inner")) callconv(.C) bool) fn (T) bool { + pub fn isNull(T: type, is_null_fn: fn (@FieldType(T, "_inner")) callconv(.c) bool) fn (T) bool { return struct { fn isNull(a: T) bool { return is_null_fn(a._inner); @@ -1738,21 +1736,13 @@ pub const helpers = struct { }.isNull; } - pub fn format(Any: type, print_fn: fn (@FieldType(Any, "_inner"), ?*const MlirStrCallback, ?*anyopaque) callconv(.C) void) type { + pub fn format(Any: type, print_fn: fn (@FieldType(Any, "_inner"), ?*const MlirStrCallback, ?*anyopaque) callconv(.c) void) type { return struct { - pub fn format( - self: Any, - comptime fmt: []const u8, - options: std.fmt.FormatOptions, - writer: anytype, - ) !void { - _ = fmt; - _ = options; - - const Writer = struct { - writer: @TypeOf(writer), - err: ?@TypeOf(writer).Error = null, - fn printCallback(mlir_str: c.MlirStringRef, opaque_ctx: ?*anyopaque) callconv(.C) void { + pub fn format(self: Any, writer: *std.Io.Writer) !void { + const WriterWithErr = struct { + writer: *std.Io.Writer, + err: ?std.Io.Writer.Error = null, + fn printCallback(mlir_str: c.MlirStringRef, opaque_ctx: ?*anyopaque) callconv(.c) void { var ctx: *@This() = @alignCast(@ptrCast(opaque_ctx)); if (ctx.err) |_| return; _ = ctx.writer.write(mlir_str.data[0..mlir_str.length]) catch |err| { @@ -1762,14 +1752,14 @@ pub const helpers = struct { } }; - var context: Writer = .{ .writer = writer }; - print_fn(self._inner, &Writer.printCallback, &context); + var context: WriterWithErr = .{ .writer = writer }; + print_fn(self._inner, &WriterWithErr.printCallback, &context); if (context.err) |err| return err; } }; } - pub fn wrapOr(T: type, is_null_fn: fn (@FieldType(T, "_inner")) callconv(.C) bool) fn (@FieldType(T, "_inner")) ?T { + pub fn wrapOr(T: type, is_null_fn: fn (@FieldType(T, "_inner")) callconv(.c) bool) fn (@FieldType(T, "_inner")) ?T { return struct { fn wrapOr(inner: @FieldType(T, "_inner")) ?T { if (is_null_fn(inner)) return null; @@ -1778,7 +1768,7 @@ pub const helpers = struct { }.wrapOr; } - pub fn init(T: type, inner: @FieldType(T, "_inner"), is_null_fn: fn (@FieldType(T, "_inner")) callconv(.C) bool) ?T { + pub fn init(T: type, inner: @FieldType(T, "_inner"), is_null_fn: fn (@FieldType(T, "_inner")) callconv(.c) bool) ?T { if (is_null_fn(inner)) return null; return .{ ._inner = inner }; } diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index d4fb6ee4..b79a789f 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -424,7 +424,7 @@ pub const CallFrame = extern struct { } }; -pub const Handler = fn (*CallFrame) callconv(.C) ?*Error; +pub const Handler = fn (*CallFrame) callconv(.c) ?*Error; pub const ErrorCode = enum(c.XLA_FFI_Error_Code) { cancelled = c.XLA_FFI_Error_Code_CANCELLED, diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 00749d5d..87f3d174 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -88,7 +88,7 @@ pub const Api = struct { return err; }, }; - const DynGetPjrtApi = lib.lookup(*const fn () callconv(.C) *const Api, "GetPjrtApi") orelse { + const DynGetPjrtApi = lib.lookup(*const fn () callconv(.c) *const Api, "GetPjrtApi") orelse { std.debug.panic("Unable to find GetPjrtApi symbol in library: {s}", .{library}); }; @@ -100,7 +100,7 @@ pub const Api = struct { } fn CallFnArgType(comptime func: Funcs) type { - const fti = @typeInfo(std.meta.FieldType(c.PJRT_Api, func)); + const fti = @typeInfo(@FieldType(c.PJRT_Api, @tagName(func))); const fn_ptr = @typeInfo(fti.optional.child); const fn_type_info = @typeInfo(fn_ptr.pointer.child); const arg_array_type_info = @typeInfo(fn_type_info.@"fn".params[0].type.?); @@ -403,8 +403,8 @@ pub const Client = opaque { element_type: BufferType, layout: MemoryLayout, device: *const Device, - on_delete_callback: *const fn (device_buffer_ptr: ?*anyopaque, ctx: ?*anyopaque) callconv(.C) void = &struct { - fn call(_: ?*anyopaque, _: ?*anyopaque) callconv(.C) void {} + on_delete_callback: *const fn (device_buffer_ptr: ?*anyopaque, ctx: ?*anyopaque) callconv(.c) void = &struct { + fn call(_: ?*anyopaque, _: ?*anyopaque) callconv(.c) void {} }.call, on_delete_callback_arg: ?*anyopaque = null, stream: ?*const Stream = null, @@ -637,7 +637,7 @@ pub const GetCostAnalysisError = std.mem.Allocator.Error || ApiError; pub const SerializeResult = struct { bytes: []const u8, handle: *anyopaque, - deleter: *const fn (?*anyopaque) callconv(.C) void, + deleter: *const fn (?*anyopaque) callconv(.c) void, pub fn deinit(self: *SerializeResult) void { self.deleter(self.handle); @@ -1036,7 +1036,7 @@ pub const Event = opaque { }); } - pub fn onReady(self: *Event, api: *const Api, func: *const fn (err: ?*Error, user_arg: ?*anyopaque) callconv(.C) void, user_arg: ?*anyopaque) ApiError!void { + pub fn onReady(self: *Event, api: *const Api, func: *const fn (err: ?*Error, user_arg: ?*anyopaque) callconv(.c) void, user_arg: ?*anyopaque) ApiError!void { _ = try api.call(.PJRT_Event_OnReady, .{ .event = self.inner(), .callback = @ptrCast(func), diff --git a/stdx/BUILD.bazel b/stdx/BUILD.bazel index 76389c50..90456d16 100644 --- a/stdx/BUILD.bazel +++ b/stdx/BUILD.bazel @@ -3,6 +3,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "stdx", srcs = [ + "bounded_array.zig", "debug.zig", "flags.zig", "fmt.zig", diff --git a/stdx/bounded_array.zig b/stdx/bounded_array.zig new file mode 100644 index 00000000..670201a4 --- /dev/null +++ b/stdx/bounded_array.zig @@ -0,0 +1,412 @@ +const std = @import("std"); +const assert = std.debug.assert; +const mem = std.mem; +const testing = std.testing; +const Alignment = std.mem.Alignment; + +/// A structure with an array and a length, that can be used as a slice. +/// +/// Useful to pass around small arrays whose exact size is only known at +/// runtime, but whose maximum size is known at comptime, without requiring +/// an `Allocator`. +/// +/// ```zig +/// var actual_size = 32; +/// var a = try BoundedArray(u8, 64).init(actual_size); +/// var slice = a.slice(); // a slice of the 64-byte array +/// var a_clone = a; // creates a copy - the structure doesn't use any internal pointers +/// ``` +pub fn BoundedArray(comptime T: type, comptime buffer_capacity: usize) type { + return BoundedArrayAligned(T, .of(T), buffer_capacity); +} + +/// A structure with an array, length and alignment, that can be used as a +/// slice. +/// +/// Useful to pass around small explicitly-aligned arrays whose exact size is +/// only known at runtime, but whose maximum size is known at comptime, without +/// requiring an `Allocator`. +/// ```zig +// var a = try BoundedArrayAligned(u8, 16, 2).init(0); +// try a.append(255); +// try a.append(255); +// const b = @ptrCast(*const [1]u16, a.constSlice().ptr); +// try testing.expectEqual(@as(u16, 65535), b[0]); +/// ``` +pub fn BoundedArrayAligned( + comptime T: type, + comptime alignment: Alignment, + comptime buffer_capacity: usize, +) type { + return struct { + const Self = @This(); + buffer: [buffer_capacity]T align(alignment.toByteUnits()) = undefined, + len: usize = 0, + + /// Set the actual length of the slice. + /// Returns error.Overflow if it exceeds the length of the backing array. + pub fn init(len: usize) error{Overflow}!Self { + if (len > buffer_capacity) return error.Overflow; + return Self{ .len = len }; + } + + /// View the internal array as a slice whose size was previously set. + pub fn slice(self: anytype) switch (@TypeOf(&self.buffer)) { + *align(alignment.toByteUnits()) [buffer_capacity]T => []align(alignment.toByteUnits()) T, + *align(alignment.toByteUnits()) const [buffer_capacity]T => []align(alignment.toByteUnits()) const T, + else => unreachable, + } { + return self.buffer[0..self.len]; + } + + /// View the internal array as a constant slice whose size was previously set. + pub fn constSlice(self: *const Self) []align(alignment.toByteUnits()) const T { + return self.slice(); + } + + /// Adjust the slice's length to `len`. + /// Does not initialize added items if any. + pub fn resize(self: *Self, len: usize) error{Overflow}!void { + if (len > buffer_capacity) return error.Overflow; + self.len = len; + } + + /// Remove all elements from the slice. + pub fn clear(self: *Self) void { + self.len = 0; + } + + /// Copy the content of an existing slice. + pub fn fromSlice(m: []const T) error{Overflow}!Self { + var list = try init(m.len); + @memcpy(list.slice(), m); + return list; + } + + /// Return the element at index `i` of the slice. + pub fn get(self: Self, i: usize) T { + return self.constSlice()[i]; + } + + /// Set the value of the element at index `i` of the slice. + pub fn set(self: *Self, i: usize, item: T) void { + self.slice()[i] = item; + } + + /// Return the maximum length of a slice. + pub fn capacity(self: Self) usize { + return self.buffer.len; + } + + /// Check that the slice can hold at least `additional_count` items. + pub fn ensureUnusedCapacity(self: Self, additional_count: usize) error{Overflow}!void { + if (self.len + additional_count > buffer_capacity) { + return error.Overflow; + } + } + + /// Increase length by 1, returning a pointer to the new item. + pub fn addOne(self: *Self) error{Overflow}!*T { + try self.ensureUnusedCapacity(1); + return self.addOneAssumeCapacity(); + } + + /// Increase length by 1, returning pointer to the new item. + /// Asserts that there is space for the new item. + pub fn addOneAssumeCapacity(self: *Self) *T { + assert(self.len < buffer_capacity); + self.len += 1; + return &self.slice()[self.len - 1]; + } + + /// Resize the slice, adding `n` new elements, which have `undefined` values. + /// The return value is a pointer to the array of uninitialized elements. + pub fn addManyAsArray(self: *Self, comptime n: usize) error{Overflow}!*align(alignment.toByteUnits()) [n]T { + const prev_len = self.len; + try self.resize(self.len + n); + return self.slice()[prev_len..][0..n]; + } + + /// Resize the slice, adding `n` new elements, which have `undefined` values. + /// The return value is a slice pointing to the uninitialized elements. + pub fn addManyAsSlice(self: *Self, n: usize) error{Overflow}![]align(alignment.toByteUnits()) T { + const prev_len = self.len; + try self.resize(self.len + n); + return self.slice()[prev_len..][0..n]; + } + + /// Remove and return the last element from the slice, or return `null` if the slice is empty. + pub fn pop(self: *Self) ?T { + if (self.len == 0) return null; + const item = self.get(self.len - 1); + self.len -= 1; + return item; + } + + /// Return a slice of only the extra capacity after items. + /// This can be useful for writing directly into it. + /// Note that such an operation must be followed up with a + /// call to `resize()` + pub fn unusedCapacitySlice(self: *Self) []align(alignment.toByteUnits()) T { + return self.buffer[self.len..]; + } + + /// Insert `item` at index `i` by moving `slice[n .. slice.len]` to make room. + /// This operation is O(N). + pub fn insert( + self: *Self, + i: usize, + item: T, + ) error{Overflow}!void { + if (i > self.len) { + return error.Overflow; + } + _ = try self.addOne(); + var s = self.slice(); + mem.copyBackwards(T, s[i + 1 .. s.len], s[i .. s.len - 1]); + self.buffer[i] = item; + } + + /// Insert slice `items` at index `i` by moving `slice[i .. slice.len]` to make room. + /// This operation is O(N). + pub fn insertSlice(self: *Self, i: usize, items: []const T) error{Overflow}!void { + try self.ensureUnusedCapacity(items.len); + self.len += items.len; + mem.copyBackwards(T, self.slice()[i + items.len .. self.len], self.constSlice()[i .. self.len - items.len]); + @memcpy(self.slice()[i..][0..items.len], items); + } + + /// Replace range of elements `slice[start..][0..len]` with `new_items`. + /// Grows slice if `len < new_items.len`. + /// Shrinks slice if `len > new_items.len`. + pub fn replaceRange( + self: *Self, + start: usize, + len: usize, + new_items: []const T, + ) error{Overflow}!void { + const after_range = start + len; + var range = self.slice()[start..after_range]; + + if (range.len == new_items.len) { + @memcpy(range[0..new_items.len], new_items); + } else if (range.len < new_items.len) { + const first = new_items[0..range.len]; + const rest = new_items[range.len..]; + @memcpy(range[0..first.len], first); + try self.insertSlice(after_range, rest); + } else { + @memcpy(range[0..new_items.len], new_items); + const after_subrange = start + new_items.len; + for (self.constSlice()[after_range..], 0..) |item, i| { + self.slice()[after_subrange..][i] = item; + } + self.len -= len - new_items.len; + } + } + + /// Extend the slice by 1 element. + pub fn append(self: *Self, item: T) error{Overflow}!void { + const new_item_ptr = try self.addOne(); + new_item_ptr.* = item; + } + + /// Extend the slice by 1 element, asserting the capacity is already + /// enough to store the new item. + pub fn appendAssumeCapacity(self: *Self, item: T) void { + const new_item_ptr = self.addOneAssumeCapacity(); + new_item_ptr.* = item; + } + + /// Remove the element at index `i`, shift elements after index + /// `i` forward, and return the removed element. + /// Asserts the slice has at least one item. + /// This operation is O(N). + pub fn orderedRemove(self: *Self, i: usize) T { + const newlen = self.len - 1; + if (newlen == i) return self.pop().?; + const old_item = self.get(i); + for (self.slice()[i..newlen], 0..) |*b, j| b.* = self.get(i + 1 + j); + self.set(newlen, undefined); + self.len = newlen; + return old_item; + } + + /// Remove the element at the specified index and return it. + /// The empty slot is filled from the end of the slice. + /// This operation is O(1). + pub fn swapRemove(self: *Self, i: usize) T { + if (self.len - 1 == i) return self.pop().?; + const old_item = self.get(i); + self.set(i, self.pop().?); + return old_item; + } + + /// Append the slice of items to the slice. + pub fn appendSlice(self: *Self, items: []const T) error{Overflow}!void { + try self.ensureUnusedCapacity(items.len); + self.appendSliceAssumeCapacity(items); + } + + /// Append the slice of items to the slice, asserting the capacity is already + /// enough to store the new items. + pub fn appendSliceAssumeCapacity(self: *Self, items: []const T) void { + const old_len = self.len; + self.len += items.len; + @memcpy(self.slice()[old_len..][0..items.len], items); + } + + /// Append a value to the slice `n` times. + /// Allocates more memory as necessary. + pub fn appendNTimes(self: *Self, value: T, n: usize) error{Overflow}!void { + const old_len = self.len; + try self.resize(old_len + n); + @memset(self.slice()[old_len..self.len], value); + } + + /// Append a value to the slice `n` times. + /// Asserts the capacity is enough. + pub fn appendNTimesAssumeCapacity(self: *Self, value: T, n: usize) void { + const old_len = self.len; + self.len += n; + assert(self.len <= buffer_capacity); + @memset(self.slice()[old_len..self.len], value); + } + + pub const Writer = if (T != u8) + @compileError("The Writer interface is only defined for BoundedArray(u8, ...) " ++ + "but the given type is BoundedArray(" ++ @typeName(T) ++ ", ...)") + else + std.io.GenericWriter(*Self, error{Overflow}, appendWrite); + + /// Initializes a writer which will write into the array. + pub fn writer(self: *Self) Writer { + return .{ .context = self }; + } + + /// Same as `appendSlice` except it returns the number of bytes written, which is always the same + /// as `m.len`. The purpose of this function existing is to match `std.io.GenericWriter` API. + fn appendWrite(self: *Self, m: []const u8) error{Overflow}!usize { + try self.appendSlice(m); + return m.len; + } + }; +} + +test BoundedArray { + var a = try BoundedArray(u8, 64).init(32); + + try testing.expectEqual(a.capacity(), 64); + try testing.expectEqual(a.slice().len, 32); + try testing.expectEqual(a.constSlice().len, 32); + + try a.resize(48); + try testing.expectEqual(a.len, 48); + + const x = [_]u8{1} ** 10; + a = try BoundedArray(u8, 64).fromSlice(&x); + try testing.expectEqualSlices(u8, &x, a.constSlice()); + + var a2 = a; + try testing.expectEqualSlices(u8, a.constSlice(), a2.constSlice()); + a2.set(0, 0); + try testing.expect(a.get(0) != a2.get(0)); + + try testing.expectError(error.Overflow, a.resize(100)); + try testing.expectError(error.Overflow, BoundedArray(u8, x.len - 1).fromSlice(&x)); + + try a.resize(0); + try a.ensureUnusedCapacity(a.capacity()); + (try a.addOne()).* = 0; + try a.ensureUnusedCapacity(a.capacity() - 1); + try testing.expectEqual(a.len, 1); + + const uninitialized = try a.addManyAsArray(4); + try testing.expectEqual(uninitialized.len, 4); + try testing.expectEqual(a.len, 5); + + try a.append(0xff); + try testing.expectEqual(a.len, 6); + try testing.expectEqual(a.pop(), 0xff); + + a.appendAssumeCapacity(0xff); + try testing.expectEqual(a.len, 6); + try testing.expectEqual(a.pop(), 0xff); + + try a.resize(1); + try testing.expectEqual(a.pop(), 0); + try testing.expectEqual(a.pop(), null); + var unused = a.unusedCapacitySlice(); + @memset(unused[0..8], 2); + unused[8] = 3; + unused[9] = 4; + try testing.expectEqual(unused.len, a.capacity()); + try a.resize(10); + + try a.insert(5, 0xaa); + try testing.expectEqual(a.len, 11); + try testing.expectEqual(a.get(5), 0xaa); + try testing.expectEqual(a.get(9), 3); + try testing.expectEqual(a.get(10), 4); + + try a.insert(11, 0xbb); + try testing.expectEqual(a.len, 12); + try testing.expectEqual(a.pop(), 0xbb); + + try a.appendSlice(&x); + try testing.expectEqual(a.len, 11 + x.len); + + try a.appendNTimes(0xbb, 5); + try testing.expectEqual(a.len, 11 + x.len + 5); + try testing.expectEqual(a.pop(), 0xbb); + + a.appendNTimesAssumeCapacity(0xcc, 5); + try testing.expectEqual(a.len, 11 + x.len + 5 - 1 + 5); + try testing.expectEqual(a.pop(), 0xcc); + + try testing.expectEqual(a.len, 29); + try a.replaceRange(1, 20, &x); + try testing.expectEqual(a.len, 29 + x.len - 20); + + try a.insertSlice(0, &x); + try testing.expectEqual(a.len, 29 + x.len - 20 + x.len); + + try a.replaceRange(1, 5, &x); + try testing.expectEqual(a.len, 29 + x.len - 20 + x.len + x.len - 5); + + try a.append(10); + try testing.expectEqual(a.pop(), 10); + + try a.append(20); + const removed = a.orderedRemove(5); + try testing.expectEqual(removed, 1); + try testing.expectEqual(a.len, 34); + + a.set(0, 0xdd); + a.set(a.len - 1, 0xee); + const swapped = a.swapRemove(0); + try testing.expectEqual(swapped, 0xdd); + try testing.expectEqual(a.get(0), 0xee); + + const added_slice = try a.addManyAsSlice(3); + try testing.expectEqual(added_slice.len, 3); + try testing.expectEqual(a.len, 36); + + while (a.pop()) |_| {} + const w = a.writer(); + const s = "hello, this is a test string"; + try w.writeAll(s); + try testing.expectEqualStrings(s, a.constSlice()); +} + +test "BoundedArrayAligned" { + var a = try BoundedArrayAligned(u8, .@"16", 4).init(0); + try a.append(0); + try a.append(0); + try a.append(255); + try a.append(255); + + const b = @as(*const [2]u16, @ptrCast(a.constSlice().ptr)); + try testing.expectEqual(@as(u16, 0), b[0]); + try testing.expectEqual(@as(u16, 65535), b[1]); +} diff --git a/stdx/flags.zig b/stdx/flags.zig index 7aab69ea..467cea69 100644 --- a/stdx/flags.zig +++ b/stdx/flags.zig @@ -47,8 +47,7 @@ const debug = @import("debug.zig"); /// Format and print an error message to stderr, then exit with an exit code of 1. pub fn fatal(comptime fmt_string: []const u8, args: anytype) noreturn { - const stderr = std.io.getStdErr().writer(); - stderr.print("error: " ++ fmt_string ++ "\n", args) catch {}; + std.debug.print("error: " ++ fmt_string ++ "\n", args); std.posix.exit(1); } diff --git a/stdx/fmt.zig b/stdx/fmt.zig index 3934de1d..259b3cd8 100644 --- a/stdx/fmt.zig +++ b/stdx/fmt.zig @@ -43,8 +43,8 @@ pub const IntFmt = struct { }; pub const FloatFmt = enum(u8) { - scientific = @intFromEnum(std.fmt.format_float.Format.scientific), - decimal = @intFromEnum(std.fmt.format_float.Format.decimal), + scientific = @intFromEnum(std.fmt.Number.Mode.scientific), + decimal = @intFromEnum(std.fmt.Number.Mode.decimal), hex, pub fn parseComptime(comptime fmt_: []const u8) FloatFmt { @@ -71,44 +71,34 @@ pub fn formatValue(value: anytype, full: FullFormatOptions, writer: anytype) !vo }; } -pub fn formatFloatValue(value: anytype, full: FullFormatOptions, writer: anytype) !void { - const formatFloat = std.fmt.format_float.formatFloat; - var buf: [std.fmt.format_float.bufferSize(.decimal, f64)]u8 = undefined; - +pub fn formatFloatValue(value: anytype, full: FullFormatOptions, writer: *std.Io.Writer) !void { const x = switch (@typeInfo(@TypeOf(value))) { .@"struct" => value.toF32(), .float => value, else => @compileError("formatFloatValue expects a float, got: " ++ @typeName(@TypeOf(value))), }; - const s_or_err = switch (full.fmt.float) { - .scientific => formatFloat(&buf, x, .{ .mode = .scientific, .precision = full.options.precision }), - .decimal => formatFloat(&buf, x, .{ .mode = .decimal, .precision = full.options.precision }), - .hex => hex: { - var buf_stream = std.io.fixedBufferStream(&buf); - std.fmt.formatFloatHexadecimal(x, full.options, buf_stream.writer()) catch unreachable; - break :hex buf_stream.getWritten(); - }, + try switch (full.fmt.float) { + .scientific => writer.printFloat(x, .{ .mode = .scientific, .precision = full.options.precision }), + .decimal => writer.printFloat(x, .{ .mode = .decimal, .precision = full.options.precision }), + .hex => writer.printFloatHexOptions(x, .{ .mode = .hex }), }; - - const s = s_or_err catch "(float)"; - return std.fmt.formatBuf(s, full.options, writer); } -pub fn formatIntValue(value: anytype, full: FullFormatOptions, writer: anytype) !void { +pub fn formatIntValue(value: anytype, full: FullFormatOptions, writer: *std.Io.Writer) !void { switch (@typeInfo(@TypeOf(value))) { .int => {}, else => @compileError("formatIntValue expects an int, got: " ++ @typeName(@TypeOf(value))), } - return std.fmt.formatInt(value, full.fmt.int.base, full.fmt.int.case, full.options, writer); + return writer.printInt(value, full.fmt.int.base, full.fmt.int.case, full.options); } -pub fn formatAnyValue(value: anytype, full: FullFormatOptions, writer: anytype) !void { +pub fn formatAnyValue(value: anytype, full: FullFormatOptions, writer: *std.Io.Writer) !void { var buf: [48]u8 = undefined; const s = std.fmt.bufPrint(&buf, "{any}", .{value}) catch blk: { buf[45..].* = "...".*; break :blk buf[0..]; }; - return std.fmt.formatBuf(s, full.options, writer); + return try writer.alignBufferOptions(s, full.options); } pub fn formatSliceCustom(fmt_func: anytype, values: anytype, full: FullFormatOptions, writer: anytype) !void { diff --git a/stdx/signature.zig b/stdx/signature.zig index 9938fb28..2f042ce8 100644 --- a/stdx/signature.zig +++ b/stdx/signature.zig @@ -26,7 +26,7 @@ pub fn ArgsTuple(comptime funcT: anytype, comptime ArgsT: ?type) type { var num_buf: [8]u8 = undefined; tuple_fields[i] = .{ .name = blk: { - const s = std.fmt.formatIntBuf(&num_buf, i, 10, .lower, .{}); + const s = std.fmt.printInt(&num_buf, i, 10, .lower, .{}); num_buf[s] = 0; break :blk num_buf[0..s :0]; }, diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 87cc2312..661dea61 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -1,3 +1,5 @@ +pub const BoundedArray = @import("bounded_array.zig").BoundedArray; +pub const BoundedArrayAligned = @import("bounded_array.zig").BoundedArrayAligned; pub const debug = @import("debug.zig"); pub const flags = @import("flags.zig"); pub const fmt = @import("fmt.zig"); diff --git a/stdx/time.zig b/stdx/time.zig index b77a489e..48d38fdf 100644 --- a/stdx/time.zig +++ b/stdx/time.zig @@ -11,14 +11,11 @@ pub const Duration = struct { return (1 * std.time.ns_per_s) / self.ns; } - pub fn format( - self: Duration, - comptime fmt: []const u8, - options: std.fmt.FormatOptions, - writer: anytype, - ) @TypeOf(writer).Error!void { - return try std.fmt.fmtDuration(self.ns).format(fmt, options, writer); + pub fn formatDuration(duration: Duration, writer: *std.io.Writer) std.io.Writer.Error!void { + try writer.printDuration(duration.ns, .{}); } + + pub const format = formatDuration; }; pub const Timer = struct { diff --git a/third_party/com_github_hejsil_clap/repo.bzl b/third_party/com_github_hejsil_clap/repo.bzl index f665c74a..4359f511 100644 --- a/third_party/com_github_hejsil_clap/repo.bzl +++ b/third_party/com_github_hejsil_clap/repo.bzl @@ -4,6 +4,6 @@ def repo(): new_git_repository( name = "com_github_hejsil_clap", remote = "https://github.com/Hejsil/zig-clap.git", - commit = "068c38f89814079635692c7d0be9f58508c86173", + commit = "5289e0753cd274d65344bef1c114284c633536ea", build_file = "//:third_party/com_github_hejsil_clap/clap.bazel", ) diff --git a/third_party/modules/rules_zig/20250827.0-35b6d57/MODULE.bazel b/third_party/modules/rules_zig/20250827.0-35b6d57/MODULE.bazel new file mode 100644 index 00000000..7620c53c --- /dev/null +++ b/third_party/modules/rules_zig/20250827.0-35b6d57/MODULE.bazel @@ -0,0 +1,75 @@ +module( + name = "rules_zig", + version = "20250827.0-35b6d57", + compatibility_level = 1, +) + +bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "platforms", version = "0.0.10") + +zig = use_extension("//zig:extensions.bzl", "zig") +zig.index(file = "//zig/private:versions.json") +use_repo(zig, "zig_toolchains") + +register_toolchains("@rules_zig//zig/target:all") + +register_toolchains("@zig_toolchains//:all") + +zig_dev = use_extension( + "//zig:extensions.bzl", + "zig", + dev_dependency = True, +) +zig_dev.toolchain(zig_version = "0.13.0") +zig_dev.toolchain(zig_version = "0.12.1") +zig_dev.toolchain(zig_version = "0.12.0") +zig_dev.toolchain(zig_version = "0.11.0") + +bazel_dep(name = "rules_cc", version = "0.0.9") + +bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") +bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") +bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) +bazel_dep( + name = "buildifier_prebuilt", + version = "7.3.1", + dev_dependency = True, +) +bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) +bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) +bazel_dep( + name = "rules_bazel_integration_test", + version = "0.25.0", + dev_dependency = True, +) + +bazel_binaries = use_extension( + "@rules_bazel_integration_test//:extensions.bzl", + "bazel_binaries", + dev_dependency = True, +) + +# NOTE: Keep in sync with WORKSPACE. +bazel_binaries.download(version_file = "//:.bazelversion") +bazel_binaries.download(version = "7.0.0") +use_repo( + bazel_binaries, + "bazel_binaries", + "bazel_binaries_bazelisk", + "build_bazel_bazel_.bazelversion", + "build_bazel_bazel_7_0_0", +) + +# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. +# However, if we do not include it explicitly, then the runfiles resolution for +# cgrindel_bazel_starlib/shlib/lib/message.sh fails in +# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked +# through the rules_multirun target //util:update. +bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) + +# Hack to get around a cc_common.link limitation. +# See https://github.com/bazelbuild/bazel/pull/23838 +cc_common_link = use_repo_rule("//zig:extensions.bzl", "cc_common_link") + +cc_common_link(name = "build_bazel_rules_android") diff --git a/third_party/modules/rules_zig/20250827.0-35b6d57/source.json b/third_party/modules/rules_zig/20250827.0-35b6d57/source.json new file mode 100644 index 00000000..e42d62ba --- /dev/null +++ b/third_party/modules/rules_zig/20250827.0-35b6d57/source.json @@ -0,0 +1,5 @@ +{ + "strip_prefix": "rules_zig-35b6d57e94f3eb08e978c1133314606f4f38e216", + "url": "https://github.com/zml/rules_zig/archive/35b6d57e94f3eb08e978c1133314606f4f38e216.tar.gz", + "integrity": "sha256-FDnAqynTD2LB3W/IaBgocmnLz8CA9nyHZYYntC4plUU=" +} diff --git a/third_party/modules/rules_zig/metadata.json b/third_party/modules/rules_zig/metadata.json index 00757b88..e7937f1e 100644 --- a/third_party/modules/rules_zig/metadata.json +++ b/third_party/modules/rules_zig/metadata.json @@ -17,7 +17,8 @@ "20250519.0-233b207", "20250613.0-567662a", "20250714.0-b14a4f1", - "20250821.0-be53625" + "20250821.0-be53625", + "20250827.0-35b6d57" ], "yanked_versions": {} } diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index af65de3c..5dc3d219 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -1,15 +1,7 @@ load("@rules_python//python:py_library.bzl", "py_library") -load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") py_library( name = "zml_utils", srcs = ["zml_utils.py"], visibility = ["//visibility:public"], ) - -py_console_script_binary( - name = "hf", - pkg = "@huggingface_hub//huggingface_hub:pkg", - script = "hf", - visibility = ["//visibility:public"], -) diff --git a/upb/upb.zig b/upb/upb.zig index 27d1502b..1a233ece 100644 --- a/upb/upb.zig +++ b/upb/upb.zig @@ -153,6 +153,6 @@ pub const Allocator = struct { @panic("Unsupported case"); } - return @ptrCast(self.allocator.alignedAlloc(u8, @alignOf(*anyopaque), size) catch return null); + return @ptrCast(self.allocator.alignedAlloc(u8, std.mem.Alignment.of(*anyopaque), size) catch return null); } }; diff --git a/zml/aio.zig b/zml/aio.zig index 08f352eb..67e1876c 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -608,7 +608,7 @@ fn findSimilarBufferKeys(original_key: []const u8, store: BufferStore, temp_allo if (std.mem.startsWith(u8, key, base_key)) { if (matches == 0) log.warn("Similar buffers found:", .{}); if (!shown_keys.contains(key)) { - log.warn(" - {s}: {}", .{ key, entry.value_ptr.*.shape() }); + log.warn(" - {s}: {f}", .{ key, entry.value_ptr.*.shape() }); shown_keys.put(key, {}) catch continue; matches += 1; } @@ -625,7 +625,7 @@ fn findSimilarBufferKeys(original_key: []const u8, store: BufferStore, temp_allo const key = entry.key_ptr.*; if (std.mem.indexOf(u8, key, component) != null and !shown_keys.contains(key)) { if (matches == 0) log.warn("Partial matches for '{s}':", .{component}); - log.warn(" - {s}: {}", .{ key, entry.value_ptr.*.shape() }); + log.warn(" - {s}: {f}", .{ key, entry.value_ptr.*.shape() }); shown_keys.put(key, {}) catch continue; matches += 1; if (matches >= 5) break; @@ -660,8 +660,8 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi return if (buffer_store.get(prefix)) |host_buffer| { // obj._shape has been set inside `loadModelBuffersWithPrefix`, before calling us. var buf_with_metadata = host_buffer; - log.debug("Loading buffer {s} ({})", .{ prefix, obj._shape }); - stdx.debug.assert(host_buffer.shape().eql(obj._shape), "loadModelBuffers expects to find the same shapes in the model and in the buffer store, got {} and {} for tensor {s}", .{ obj._shape, host_buffer, prefix }); + log.debug("Loading buffer {s} ({f})", .{ prefix, obj._shape }); + stdx.debug.assert(host_buffer.shape().eql(obj._shape), "loadModelBuffers expects to find the same shapes in the model and in the buffer store, got {f} and {f} for tensor {s}", .{ obj._shape, host_buffer, prefix }); buf_with_metadata._shape = obj._shape; obj.* = try zml.Buffer.from(platform, buf_with_metadata, .{}); } else { diff --git a/zml/aio/json.zig b/zml/aio/json.zig index 9c02a5db..a42e5619 100644 --- a/zml/aio/json.zig +++ b/zml/aio/json.zig @@ -69,7 +69,7 @@ pub fn parseMetadata(allocator: Allocator, store: *zml.aio.BufferStore, prefix: var new_prefix = prefix; if (prefix.items.len > 0) new_prefix.appendAssumeCapacity('.'); - new_prefix.items.len += std.fmt.formatIntBuf(new_prefix.unusedCapacitySlice(), i, 10, .lower, .{}); + new_prefix.items.len += std.fmt.printInt(new_prefix.unusedCapacitySlice(), i, 10, .lower, .{}); try parseMetadata(allocator, store, new_prefix, item); } }; diff --git a/zml/aio/safetensors.zig b/zml/aio/safetensors.zig index 581ab77f..08952b8f 100644 --- a/zml/aio/safetensors.zig +++ b/zml/aio/safetensors.zig @@ -1,12 +1,15 @@ -const asynk = @import("async"); const std = @import("std"); -const zml = @import("../zml.zig"); -const json = @import("json.zig"); -const HostBuffer = zml.HostBuffer; +const Allocator = std.mem.Allocator; + +const asynk = @import("async"); +const stdx = @import("stdx"); + const MemoryMappedFile = @import("../aio.zig").MemoryMappedFile; +const zml = @import("../zml.zig"); +const HostBuffer = zml.HostBuffer; +const json = @import("json.zig"); const StringBuilder = std.ArrayListUnmanaged(u8); -const Allocator = std.mem.Allocator; const log = std.log.scoped(.@"zml/io"); pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore { @@ -16,7 +19,7 @@ pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore errdefer res.arena.deinit(); const arena = res.arena.allocator(); - var files = std.ArrayList(MemoryMappedFile).init(arena); + var files = std.array_list.Managed(MemoryMappedFile).init(arena); errdefer files.deinit(); if (std.mem.endsWith(u8, path, ".safetensors.index.json")) { @@ -28,17 +31,19 @@ pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore return res; } -fn loadFromIndex(allocator: Allocator, store: *zml.aio.BufferStore, files: *std.ArrayList(MemoryMappedFile), path: []const u8) !void { +fn loadFromIndex(allocator: Allocator, store: *zml.aio.BufferStore, files: *std.array_list.Managed(MemoryMappedFile), path: []const u8) !void { const file = asynk.File.open(path, .{}) catch |err| { log.err("Failed to open {s}: {}", .{ path, err }); return err; }; errdefer file.close() catch unreachable; - var r = file.reader(); + var buffer: [4096]u8 = undefined; + var r = file.reader(&buffer); - const json_data = try allocator.alloc(u8, (try file.stat()).size); - _ = try r.readAtLeast(json_data, json_data.len); - const index = try std.json.parseFromSliceLeaky(std.json.Value, allocator, json_data, .{ .allocate = .alloc_if_needed }); + // const json_data = try allocator.alloc(u8, (try file.stat()).size); + var json_reader = std.json.Reader.init(allocator, &r.interface); + // _ = try r.readAtLeast(json_data, json_data.len); + const index = try std.json.parseFromTokenSourceLeaky(std.json.Value, allocator, &json_reader, .{ .allocate = .alloc_if_needed }); var loaded_files = std.StringHashMap(void).init(allocator); const weight_map = index.object.get("weight_map").?.object; @@ -62,23 +67,24 @@ fn loadFromIndex(allocator: Allocator, store: *zml.aio.BufferStore, files: *std. } } -fn loadFile(allocator: Allocator, store: *zml.aio.BufferStore, files: *std.ArrayList(MemoryMappedFile), path: []const u8) !void { +fn loadFile(allocator: Allocator, store: *zml.aio.BufferStore, files: *std.array_list.Managed(MemoryMappedFile), path: []const u8) !void { const file = asynk.File.open(path, .{}) catch |err| { log.err("Failed to open {s}: {}", .{ path, err }); return err; }; errdefer file.close() catch unreachable; - var r = file.reader(); + var buffer: [16 * 1024]u8 = undefined; + var r = file.reader(&buffer); - const json_header_length: usize = @intCast(try r.readInt(u64, std.builtin.Endian.little)); + const json_header_length: usize = @intCast(try r.interface.takeInt(u64, .little)); const json_data = try allocator.alloc(u8, json_header_length); - const n = try r.readAll(json_data); - if (n != json_header_length) { - log.err("Failed to read the full {} bytes of json header from file {s}", .{ n, path }); - return error.CorruptedFile; - } + try r.interface.readSliceAll(json_data); + // if (n != json_header_length) { + // log.err("Failed to read the full {} bytes of json header from file {s}", .{ n, path }); + // return error.CorruptedFile; + // } - const metadata = try std.json.parseFromSliceLeaky(std.json.Value, allocator, json_data[0..n], .{}); + const metadata = try std.json.parseFromSliceLeaky(std.json.Value, allocator, json_data, .{}); var buffer_file = try MemoryMappedFile.init(file); errdefer buffer_file.deinit(); buffer_file.data_offset = 8 + json_header_length; @@ -105,7 +111,7 @@ fn loadFile(allocator: Allocator, store: *zml.aio.BufferStore, files: *std.Array const start: usize = @intCast(offset_field.array.items[0].integer); const end: usize = @intCast(offset_field.array.items[1].integer); const dtype = try stringToDtype(val.object.get("dtype").?.string); - var dims: std.BoundedArray(i64, zml.Shape.MAX_RANK) = .{}; + var dims: stdx.BoundedArray(i64, zml.Shape.MAX_RANK) = .{}; for (shape_field.items) |d| { dims.appendAssumeCapacity(d.integer); } diff --git a/zml/buffer.zig b/zml/buffer.zig index 36f61255..3553bb94 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -45,7 +45,7 @@ pub const Buffer = struct { _shards: Shards, pub const MAX_NUM_SHARDS: u8 = Platform.MAX_NUM_DEVICES; - pub const Shards = std.BoundedArray(*pjrt.Buffer, MAX_NUM_SHARDS); + pub const Shards = stdx.BoundedArray(*pjrt.Buffer, MAX_NUM_SHARDS); pub const FromOptions = struct { wait: bool = true, @@ -67,7 +67,7 @@ pub const Buffer = struct { const n_partitions = platform.sharding().num_partitions; const chunk_size = if (sharding_ax) |ax| cs: { // This kind of sharding error should be detected earlier on. - stdx.debug.assert(@rem(host_buffer.dim(ax), n_partitions) == 0, "Buffer.from({}) expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ host_buffer, ax, n_partitions }); + stdx.debug.assert(@rem(host_buffer.dim(ax), n_partitions) == 0, "Buffer.from({f}) expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ host_buffer, ax, n_partitions }); break :cs @divExact(host_buffer.dim(ax), n_partitions); } else 0; @@ -201,7 +201,7 @@ pub const Buffer = struct { const duration_ms = stdx.math.divFloat(f32, start.read(), std.time.ns_per_ms); if (duration_ms > 100) { const size_gb = stdx.math.divFloat(f32, shape_.byteSize(), 1024 * 1024 * 1024); - log.info("Wrote constant({_}) to device ({d:.2}Gb) in {d:.0}ms: {d:.2}Gb/s", .{ shape_, size_gb, duration_ms, size_gb / duration_ms * 1000 }); + log.debug("Wrote constant({f}) to device ({d:.2}Gb) in {d:.0}ms: {d:.2}Gb/s", .{ shape_, size_gb, duration_ms, size_gb / duration_ms * 1000 }); } } @@ -301,7 +301,7 @@ pub const Buffer = struct { /// Fetches the content of the given buffer into a stack variable of the given type. pub fn getValue(self: Buffer, T: type) !T { - stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); + stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {f} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); var res: T = undefined; stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); const maybe_event = try self._shards.get(0).toHostBuffer(self._api, std.mem.asBytes(&res)); @@ -375,13 +375,9 @@ pub const Buffer = struct { pub fn format( self: Buffer, - comptime fmt: []const u8, - options: std.fmt.FormatOptions, writer: anytype, ) !void { - _ = fmt; - _ = options; - try writer.print("Buffer({_})", .{self._shape}); + try writer.print("Buffer({f})", .{self._shape}); } pub fn getMemory(self: Buffer) *const pjrt.Memory { diff --git a/zml/context.zig b/zml/context.zig index bde867f9..15acb93b 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -224,7 +224,7 @@ const CustomCall = struct { try ffi.register(platform.pjrt_api, "zmlHostBufferCallback", @tagName(platform.target), &hostBufferCallback, .{}); } - fn hostBufferCallback(call_frame: *pjrt.ffi.CallFrame) callconv(.C) ?*pjrt.ffi.Error { + fn hostBufferCallback(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { if (call_frame.registeringHook()) return null; const callback_attr = call_frame.attrs.getByName(.scalar, "callback") orelse unreachable; @@ -275,3 +275,69 @@ fn hostBufferFromPinnedBuffer(buffer_desc: *const pjrt.ffi.Buffer) HostBuffer { buffer_desc.data[0..buffer_shape.byteSize()], ); } + +pub const cuda = struct { + pub var streamSynchronize: StreamSynchronize = @ptrFromInt(0xdeadc00da00); + pub var cuLaunchHostFunc: CuLaunchHostFunc = @ptrFromInt(0xdeadc00da00); + var _memcpyAsync: MemcpyAsync = @ptrFromInt(0xdeadc00da00); + var _memcpyBlocking: MemcpyBlocking = @ptrFromInt(0xdeadc00da00); + + pub const MemcpyKind = enum(c_int) { + host_to_host = 0, + host_to_device = 1, + device_to_host = 2, + device_to_device = 3, + inferred = 4, + }; + + const MemcpyAsync = *const fn (dst: *anyopaque, src: *const anyopaque, count: usize, kind: MemcpyKind, stream: ?*anyopaque) callconv(.c) c_int; + const MemcpyBlocking = *const fn (dst: *anyopaque, src: *const anyopaque, count: usize, kind: MemcpyKind) callconv(.c) c_int; + const StreamSynchronize = *const fn (stream: *anyopaque) callconv(.c) c_int; + const CuLaunchHostFunc = *const fn (stream: *anyopaque, host_func: *const fn (user_data: *const anyopaque) callconv(.c) void, user_data: *const anyopaque) callconv(.c) c_int; + + pub fn init() void { + var cudart = std.DynLib.open("libcudart.so.12") catch { + log.err("cudart not found, callback will segfault", .{}); + return; + }; + defer cudart.close(); + + _memcpyAsync = cudart.lookup(MemcpyAsync, "cudaMemcpyAsync") orelse { + @panic("cudaMemcpyAsync not found"); + }; + _memcpyBlocking = cudart.lookup(MemcpyBlocking, "cudaMemcpy") orelse { + @panic("cudaMemcpy not found"); + }; + streamSynchronize = cudart.lookup(StreamSynchronize, "cudaStreamSynchronize") orelse { + @panic("cudaStreamSynchronize not found"); + }; + cuLaunchHostFunc = cudart.lookup(CuLaunchHostFunc, "cudaLaunchHostFunc") orelse { + @panic("cudaLaunchHostFunc not found"); + }; + } + + pub fn memcpyToHostBlocking(dst: []u8, src: *const anyopaque) void { + const err = _memcpyBlocking(dst.ptr, src, dst.len, .device_to_host); + check(err); + } + + pub fn memcpyToDeviceBlocking(dst: *anyopaque, src: []const u8) void { + const err = _memcpyBlocking(dst, src.ptr, src.len, .host_to_device); + check(err); + } + + pub fn memcpyToDeviceAsync(dst: *anyopaque, src: []const u8, stream: ?*anyopaque) void { + const err = _memcpyAsync(dst, src.ptr, src.len, .host_to_device, stream); + check(err); + } + + pub fn memcpyToHostAsync(dst: []u8, src: *const anyopaque, stream: ?*anyopaque) void { + const err = _memcpyAsync(dst.ptr, src, dst.len, .device_to_host, stream); + check(err); + } + + pub fn check(err: c_int) void { + if (err == 0) return; + stdx.debug.panic("CUDA error: {d}", .{err}); + } +}; diff --git a/zml/exe.zig b/zml/exe.zig index 0923f17c..625fecf9 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -394,8 +394,8 @@ fn fillBuffers(v: anytype, shapes: []const Shape, buffers: []const [*]*pjrt.Buff fn cb(ctx: *LocalContext, buffer: *const Buffer) void { // stdx.debug.assert(!buffer._data.isDeleted(), "Can't use {} (argument buffer {}) because its pjrt buffer has been donated", .{ buffer, ctx.index }); const model_sharding = ctx.buffers.len; - stdx.debug.assert(buffer._shards.len == model_sharding, "Can't feed a {}-sharded tensor into a {}-sharded model", .{ buffer._shards.len, ctx.buffers.len }); - stdx.debug.assert(ctx.shapes[ctx.index].eql(buffer.shape()), "Executable expected argument {} to have shape {}, got {}", .{ ctx.index, ctx.shapes[ctx.index], buffer.shape() }); + stdx.debug.assert(buffer._shards.len == model_sharding, "Can't feed a {d}-sharded tensor into a {d}-sharded model", .{ buffer._shards.len, ctx.buffers.len }); + stdx.debug.assert(ctx.shapes[ctx.index].eql(buffer.shape()), "Executable expected argument {} to have shape {f}, got {f}", .{ ctx.index, ctx.shapes[ctx.index], buffer.shape() }); for (buffer._shards.constSlice(), 0..) |shard, d| { ctx.buffers[d][ctx.index] = shard; } diff --git a/zml/floats.zig b/zml/floats.zig index 74a8b993..57bf5150 100644 --- a/zml/floats.zig +++ b/zml/floats.zig @@ -87,17 +87,10 @@ fn FloatHelpers(Float: type) type { return std.math.maxInt(std.meta.Int(.unsigned, exponent_bits - 1)); } - pub fn format( - float: Float, - comptime fmt: []const u8, - options: std.fmt.FormatOptions, - writer: anytype, - ) !void { - _ = options; - if (fmt.len == 1 and fmt[0] == '_') { - try writer.print("{{ .sign={}, .exp={}, .mantissa={} }}", .{ float.sign, float.exponent, float.mantissa }); - } else { - try writer.print("{" ++ fmt ++ "}", .{float.toF32()}); + pub fn formatNumber(x: Float, writer: *std.io.Writer, n: std.fmt.Number) std.io.Writer.Error!void { + switch (n.mode) { + .binary, .octal, .hex => try writer.print("{{ .sign={}, .exp={}, .mantissa={} }}", .{ x.sign, x.exponent, x.mantissa }), + else => try writer.printFloat(x.toF32(), n), } } }; @@ -113,7 +106,7 @@ pub const Float32 = packed struct(u32) { pub const neg = Helpers.neg; pub const fromF32 = Helpers.fromF32; pub const toF32 = Helpers.toF32; - pub const format = Helpers.format; + pub const formatNumber = Helpers.formatNumber; }; const f32_exp_bias = FloatHelpers(Float32).expBias(); @@ -128,7 +121,7 @@ pub const Float64 = packed struct(u64) { pub const neg = Helpers.neg; pub const fromF32 = Helpers.fromF32; pub const toF32 = Helpers.toF32; - pub const format = Helpers.format; + pub const formatNumber = Helpers.formatNumber; }; pub const Float8E4M3B11FNUZ = packed struct(u8) { @@ -151,7 +144,7 @@ pub const Float8E4M3B11FNUZ = packed struct(u8) { pub const neg = Helpers.neg; pub const fromF32 = Helpers.fromF32; pub const toF32 = Helpers.toF32; - pub const format = Helpers.format; + pub const formatNumber = Helpers.formatNumber; }; pub const Float8E4M3FN = packed struct(u8) { @@ -169,7 +162,7 @@ pub const Float8E4M3FN = packed struct(u8) { pub const neg = Helpers.neg; pub const fromF32 = Helpers.fromF32; pub const toF32 = Helpers.toF32; - pub const format = Helpers.format; + pub const formatNumber = Helpers.formatNumber; }; pub const Float8E4M3FNUZ = packed struct(u8) { @@ -192,7 +185,7 @@ pub const Float8E4M3FNUZ = packed struct(u8) { pub const neg = Helpers.neg; pub const fromF32 = Helpers.fromF32; pub const toF32 = Helpers.toF32; - pub const format = Helpers.format; + pub const formatNumber = Helpers.formatNumber; }; test "Float8E4" { @@ -247,7 +240,7 @@ pub const Float8E5M2 = packed struct(u8) { pub const neg = Helpers.neg; pub const fromF32 = Helpers.fromF32; pub const toF32 = Helpers.toF32; - pub const format = Helpers.format; + pub const formatNumber = Helpers.formatNumber; }; pub const Float8E5M2FNUZ = packed struct(u8) { @@ -266,7 +259,7 @@ pub const Float8E5M2FNUZ = packed struct(u8) { pub const neg = Helpers.neg; pub const fromF32 = Helpers.fromF32; pub const toF32 = Helpers.toF32; - pub const format = Helpers.format; + pub const formatNumber = Helpers.formatNumber; }; test "Float8E5" { @@ -322,7 +315,7 @@ pub const BFloat16 = packed struct(u16) { const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; pub const neg = Helpers.neg; - pub const format = Helpers.format; + pub const formatNumber = Helpers.formatNumber; }; test BFloat16 { diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 8f6fcb1c..117cb1e4 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -31,7 +31,7 @@ pub const HostBuffer = struct { return .{ ._shape = sh, ._strides = sh.computeStrides().buffer, - ._data = (try allocator.alignedAlloc(u8, 64, sh.byteSize())).ptr, + ._data = (try allocator.alignedAlloc(u8, .@"64", sh.byteSize())).ptr, ._memory = .{ .managed = .@"64" }, }; } @@ -170,8 +170,8 @@ pub const HostBuffer = struct { /// Strided buffers can't use this method. pub fn items(self: HostBuffer, comptime T: type) []const T { // TODO we should allow interpreting the output as @Vector(8, f32) when the tensor is f32. - stdx.debug.assert(DataType.fromZigType(T) == self.dtype(), "Can't reinterpret {} as {s}", .{ self, @typeName(T) }); - stdx.debug.assert(self.isContiguous(), "{} isn't contiguous, can't interpret as []const u8", .{self}); + stdx.debug.assert(DataType.fromZigType(T) == self.dtype(), "Can't reinterpret {f} as {s}", .{ self, @typeName(T) }); + stdx.debug.assert(self.isContiguous(), "{f} isn't contiguous, can't interpret as []const u8", .{self}); const ptr: [*]const T = @alignCast(@ptrCast(self._data)); return ptr[0..self._shape.count()]; } @@ -181,7 +181,7 @@ pub const HostBuffer = struct { } pub fn bytes(self: HostBuffer) []const u8 { - stdx.debug.assert(self.isContiguous(), "{} isn't contiguous, can't interpret as []const u8", .{self}); + stdx.debug.assert(self.isContiguous(), "{f} isn't contiguous, can't interpret as []const u8", .{self}); return self._data[0..self._shape.byteSize()]; } @@ -233,7 +233,7 @@ pub const HostBuffer = struct { } pub fn reshape(self: HostBuffer, shape_: anytype) HostBuffer { - stdx.debug.assert(self.isContiguous(), "reshape expects a contiguous tensor, got: {}", .{self}); + stdx.debug.assert(self.isContiguous(), "reshape expects a contiguous tensor, got: {f}", .{self}); var res = self; res._shape = self._shape.reshape(shape_); res._strides = res._shape.computeStrides().buffer; @@ -252,9 +252,9 @@ pub const HostBuffer = struct { const start: i64 = if (s.start < 0) s.start + d else s.start; var end = s.end orelse d; if (end < 0) end += d; - stdx.debug.assert(start >= 0 and start < d, "slice1d({}, {}) expects the slice start to be between 0 and {} got: {}", .{ self, ax, d, s }); - stdx.debug.assert(end >= 1 and end <= d, "slice1d({}, {}) expects the slice end to be between 1 and {} got: {}", .{ self, ax, d, s }); - stdx.debug.assert(start < end, "slice1d({}, {}) expects the slice start ({}) to be smaller than the end ({}), got: {}", .{ self, ax, start, end, s }); + stdx.debug.assert(start >= 0 and start < d, "slice1d({f}, {}) expects the slice start to be between 0 and {} got: {}", .{ self, ax, d, s }); + stdx.debug.assert(end >= 1 and end <= d, "slice1d({f}, {}) expects the slice end to be between 1 and {} got: {}", .{ self, ax, d, s }); + stdx.debug.assert(start < end, "slice1d({f}, {}) expects the slice start ({}) to be smaller than the end ({}), got: {}", .{ self, ax, start, end, s }); const offset: usize = @intCast(start * self._strides[ax]); const new_shape = self.shape().set(ax, end - start); @@ -308,9 +308,9 @@ pub const HostBuffer = struct { pub fn squeeze(self: HostBuffer, axis_: anytype) HostBuffer { const ax = self._shape.axis(axis_); - stdx.debug.assert(self.dim(ax) == 1, "squeeze expects a 1-d axis got {} in {}", .{ ax, self }); + stdx.debug.assert(self.dim(ax) == 1, "squeeze expects a 1-d axis got {} in {f}", .{ ax, self }); - var strd: std.BoundedArray(i64, Shape.MAX_RANK) = .{ .buffer = self._strides, .len = self.rank() }; + var strd: stdx.BoundedArray(i64, Shape.MAX_RANK) = .{ .buffer = self._strides, .len = self.rank() }; _ = strd.orderedRemove(ax); return .{ @@ -323,16 +323,11 @@ pub const HostBuffer = struct { pub fn format( self: HostBuffer, - comptime fmt: []const u8, - options: std.fmt.FormatOptions, writer: anytype, ) !void { - _ = options; - if (std.mem.eql(u8, fmt, "v")) { - try writer.print("HostBuffer(.{_})@0x{x}", .{ self._shape, @intFromPtr(self._data) }); - } else { - try writer.print("HostBuffer(.{_})", .{self._shape}); - } + // TODO debug option + // try writer.print("HostBuffer(.{f})@0x{x}", .{ self._shape, @intFromPtr(self._data) }); + try writer.print("HostBuffer(.{f})", .{self._shape}); } /// Formatter for a HostBuffer that also print the values not just the shape. @@ -344,21 +339,23 @@ pub const HostBuffer = struct { pub const PrettyPrinter = struct { x: HostBuffer, - pub fn format(self: PrettyPrinter, comptime fmt: []const u8, options: std.fmt.FormatOptions, writer: anytype) !void { + // TODO(0.15.0) revisit pretty printer + pub fn format(self: PrettyPrinter, writer: anytype) !void { const fmt_: stdx.fmt.Fmt = switch (self.x.dtype().class()) { - .integer => .parse(i32, fmt), - .float => .parse(f32, fmt), - else => .parse(void, fmt), + .integer => .parse(i32, "d"), + .float => .parse(f32, "d"), + else => .parse(void, ""), }; + const options: std.fmt.FormatOptions = .{}; try prettyPrint(self.x, writer, .{ .fmt = fmt_, .options = options }); } }; - pub fn prettyPrint(self: HostBuffer, writer: anytype, options: stdx.fmt.FullFormatOptions) !void { + pub fn prettyPrint(self: HostBuffer, writer: *std.Io.Writer, options: stdx.fmt.FullFormatOptions) !void { return self.prettyPrintIndented(writer, 4, 0, options); } - fn prettyPrintIndented(self: HostBuffer, writer: anytype, num_rows: u8, indent_level: u8, options: stdx.fmt.FullFormatOptions) !void { + fn prettyPrintIndented(self: HostBuffer, writer: *std.Io.Writer, num_rows: u8, indent_level: u8, options: stdx.fmt.FullFormatOptions) !void { if (self.rank() == 0) { // Special case input tensor is a scalar return switch (self.dtype()) { @@ -376,7 +373,7 @@ pub const HostBuffer = struct { if (self.rank() == 1) { // Print a contiguous slice of items from the buffer in one line. // The number of items printed is controlled by the user through format syntax. - try writer.writeByteNTimes(' ', indent_level); + try writer.splatByteAll(' ', indent_level); switch (self.dtype()) { inline else => |dt| { const values = self.items(dt.toZigType()); @@ -391,10 +388,10 @@ pub const HostBuffer = struct { return; } // TODO: consider removing the \n if dim is 1 for this axis. - try writer.writeByteNTimes(' ', indent_level); + try writer.splatByteAll(' ', indent_level); _ = try writer.write("{\n"); defer { - writer.writeByteNTimes(' ', indent_level) catch {}; + writer.splatByteAll(' ', indent_level) catch {}; _ = writer.write("},\n") catch {}; } @@ -409,7 +406,7 @@ pub const HostBuffer = struct { if (n < num_rows) return; // Skip middle rows if (n > 2 * num_rows) { - try writer.writeByteNTimes(' ', indent_level + 2); + try writer.splatByteAll(' ', indent_level + 2); _ = try writer.write("...\n"); } // Write last rows diff --git a/zml/meta.zig b/zml/meta.zig index ad50fe05..ee2fbc68 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -358,10 +358,11 @@ pub fn MapRestrict(From: type, To: type) type { const fields = union_info.fields; var union_fields: [fields.len]std.builtin.Type.UnionField = undefined; for (0.., fields) |i, field| { + const FT = map(field.type); union_fields[i] = .{ .name = field.name, - .type = map(field.type), - .alignment = 0, + .type = FT, + .alignment = @alignOf(FT), }; } return @Type(.{ .@"union" = .{ @@ -453,7 +454,7 @@ pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) void { else => {}, } - // Handle std.BoundedArray that contains uninitalized data. + // Handle stdx.BoundedArray that contains uninitalized data. if (@typeInfo(Child) == .@"struct" and @hasDecl(Child, "constSlice") and @hasDecl(Child, "slice")) { return visit(cb, ctx, if (mutating_cb) v.slice() else v.constSlice()); } @@ -511,7 +512,7 @@ test visit { const NestedAttrOptional = struct { nested: ?Attr }; const SimpleStruct = struct { prop: Attr }; const MultipleTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: ?Attr }; - const NestedTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: NestedAttr, prop4: NestedAttrOptional, prop5: std.BoundedArray(Attr, 8) }; + const NestedTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: NestedAttr, prop4: NestedAttrOptional, prop5: stdx.BoundedArray(Attr, 8) }; const LocalContext = struct { result: usize }; @@ -565,7 +566,7 @@ test visit { } { var context: LocalContext = .{ .result = 0 }; - const prop5: std.BoundedArray(Attr, 8) = .{ + const prop5: stdx.BoundedArray(Attr, 8) = .{ .buffer = @splat(.{ .data = 4 }), .len = 2, }; @@ -677,11 +678,11 @@ test zip { /// Given a func(X) -> Y or a func(Ctx, X) -> Y, /// finds all X in the given object, and write the result of func(X) into an arraylist. -pub fn collect(func: anytype, func_ctx: _CollectCtx(func), out: *std.ArrayList(stdx.meta.FnSignature(func, null).ReturnT), obj: anytype) error{OutOfMemory}!void { +pub fn collect(func: anytype, func_ctx: _CollectCtx(func), out: *std.array_list.Managed(stdx.meta.FnSignature(func, null).ReturnT), obj: anytype) error{OutOfMemory}!void { stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".params.len <= 2, "zml.meta.collect expects a func with two arguments, got: {}", .{@TypeOf(func)}); const LocalContext = struct { func_ctx: _CollectCtx(func), - out: *std.ArrayList(stdx.meta.FnSignature(func, null).ReturnT), + out: *std.array_list.Managed(stdx.meta.FnSignature(func, null).ReturnT), oom: bool = false, }; var context = LocalContext{ .func_ctx = func_ctx, .out = out }; diff --git a/zml/module.zig b/zml/module.zig index 1f9beaf4..4e40882e 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -51,7 +51,7 @@ pub const CompilationContext = struct { _module: mlir.Module, - _blocks: std.BoundedArray(TaggedBlock, 64) = .{}, + _blocks: stdx.BoundedArray(TaggedBlock, 64) = .{}, _fn_cache: FnCache = .{}, _block_args: TensorToBlockArg = .{}, @@ -63,7 +63,7 @@ pub const CompilationContext = struct { const TaggedBlock = struct { mlir.Block, mlir.Block.RecursiveOpts }; const TensorToBlockArg = std.AutoHashMapUnmanaged(Tensor._Id, struct { mlir.Value, Tensor._Donation }); - const AttributeList = std.BoundedArray(mlir.NamedAttribute, 3); + const AttributeList = stdx.BoundedArray(mlir.NamedAttribute, 3); pub fn init(allocator_: std.mem.Allocator, full_name: []const u8, platform: Platform) !CompilationContext { const mlir_registry = mlir.Registry.init() catch unreachable; @@ -185,7 +185,9 @@ pub const CompilationContext = struct { // Write the mlir to a file. All errors are discarded, since this is for debugging only. const mlir_name = "module.mlir"; if (cache_dir.createFile(mlir_name, .{ .truncate = true })) |file| { - module.op().print(file.writer(), .{ .debug_info = true, .debug_info_pretty_form = false }); + var write_buf: [4096]u8 = undefined; + var writer = file.writer(&write_buf); + module.op().print(&writer.interface, .{ .debug_info = true, .debug_info_pretty_form = false }); log.info("Wrote MLIR to {s}/{s}", .{ module_dir.?, mlir_name }); } else |_| { log.warn("Failed to open {s}", .{mlir_name}); @@ -219,7 +221,7 @@ pub const CompilationContext = struct { }; log.debug("******** ZML generated MLIR ********", .{}); - log.debug("{}", .{module.op().mlirFormatter(.{})}); + log.debug("{f}", .{module.op().mlirFormatter(.{})}); if (timer) |*t| { const time_ms = @divFloor(t.lap(), std.time.ns_per_ms); @@ -339,7 +341,7 @@ pub const CompilationContext = struct { const locations = try arena.alloc(mlir.Location, tensor_count); @memset(locations, mlir.Location.unknown(mlir_ctx)); - var input_shapes = try std.ArrayList(Shape).initCapacity(res_allocator, tensor_count); + var input_shapes: std.array_list.Managed(Shape) = try .initCapacity(res_allocator, tensor_count); meta.collect(Tensor.shape, {}, &input_shapes, args) catch unreachable; stdx.debug.internalAssert(input_shapes.items.len == tensor_count, "args have changed ?", .{}); @@ -416,7 +418,7 @@ pub const CompilationContext = struct { defer self._tracer.frameEnd(canonicalize_frame, "emitMlir.canonicalize"); self._mlir_canonicalizer.runOnOp(mlir_fn) catch |err| switch (err) { error.InvalidMlir => { - log.err("Failed to canonicalize invalid mlir: {}", .{mlir_fn.mlirFormatter(.{})}); + log.err("Failed to canonicalize invalid mlir: {f}", .{mlir_fn.mlirFormatter(.{})}); // user errors should have triggered a panic before we reach this. @panic("ZML generated invalid mlir. Please open a bug report"); }, @@ -464,7 +466,7 @@ pub const CompilationContext = struct { // This will break the day we writer another attribute before donation. // When the time come, do a more fancy lookup here to check if an argument // is donated twice. - stdx.debug.assert(attributes[a].len == 0, "Donation error ! Argument {} has been donated twice ! To {} and to {}", .{ a, index, attributes[a].buffer[0] }); + stdx.debug.assert(attributes[a].len == 0, "Donation error ! Argument {d} has been donated twice ! To {d} and to {any}", .{ a, index, attributes[a].buffer[0] }); attributes[a].appendAssumeCapacity(.named(ctx, "tf.aliasing_output", .int(ctx, .i32, @intCast(index)))); // log.debug("attribute: {}", .{attributes[a].constSlice()}); }, @@ -504,9 +506,9 @@ pub const CompilationContext = struct { var tensor_args = .{ model, Tensor{ ._shape = s, ._id = .{ .buffer_id = 1234 } }, Tensor{ ._shape = s, ._id = .{ .buffer_id = 1235 } } }; const f = try comp.emitMlir(Local._fwd, &tensor_args, .{ .name = "test.emitMlir.Local.forward", .kind = .main }); - var mlir_bytecode = std.ArrayList(u8).init(std.testing.allocator); + var mlir_bytecode = std.array_list.Managed(u8).init(std.testing.allocator); defer mlir_bytecode.deinit(); - try mlir_bytecode.writer().print("{}", .{f.mlir_fn.mlirFormatter(.{})}); + try mlir_bytecode.writer().print("{f}", .{f.mlir_fn.mlirFormatter(.{})}); // Check that the `x` input argument gives its buffer to the result tensor. // `%arg0` is the bias of the model, `%arg1` is `x`, `%arg2` is `y`. @@ -545,7 +547,7 @@ pub const CompilationContext = struct { pub fn getShardingAttr(self: CompilationContext, shape: Shape) mlir.Attribute { const ctx = self.mlirCtx(); const num_partitions = self.numPartitions(); - var sharding_str: std.BoundedArray(u8, 128) = .{}; + var sharding_str: stdx.BoundedArray(u8, 128) = .{}; writeShardingRepresentation(shape, num_partitions, sharding_str.writer()) catch unreachable; return mlir.Attribute.string(ctx, sharding_str.constSlice()); } @@ -622,7 +624,7 @@ pub const CompilationContext = struct { const full_name: [:0]const u8 = if (std.mem.eql(u8, "main", func_name)) try self.allocator().dupeZ(u8, func_name) else - try std.fmt.allocPrintZ(self.allocator(), "{s}_{x}", .{ func_name, key.input_hash }); + try std.fmt.allocPrintSentinel(self.allocator(), "{s}_{x}", .{ func_name, key.input_hash }, 0); var arg_id: u16 = 0; var tensor_args: @TypeOf(args) = args; @@ -702,7 +704,7 @@ pub const CompilationContext = struct { const res = ctx.self._block_args.getOrPutAssumeCapacity(tensor._id); if (res.found_existing) { - stdx.debug.panic("Failed compilation because received two tensors arguments with the same ID: {} and {} at index {} ({}).", .{ res.value_ptr.*[0], tensor, ctx.index, tensor._id }); + stdx.debug.panic("Failed compilation because received two tensors arguments with the same ID: {f} and {f} at index {} ({}).", .{ res.value_ptr.*[0], tensor, ctx.index, tensor._id }); } else { res.value_ptr.* = .{ arg_value, .{ .arg = @intCast(ctx.index) } }; } @@ -777,7 +779,7 @@ pub const CompilationContext = struct { .buffer_id, .arg_id => if (self._block_args.get(tensor._id)) |res| .{ res[0], res[1] } else { - log.err("Found unknown tensor id {}({})", .{ tensor, tensor._id }); + log.err("Found unknown tensor id {f}({})", .{ tensor, tensor._id }); @panic("Found unknown tensor id"); }, .mlir => |v| .{ v, tensor._donation }, diff --git a/zml/nn.zig b/zml/nn.zig index ffdb7a9c..631bce16 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -40,8 +40,8 @@ pub const TokenEmbedding = struct { weight: Tensor, pub fn forward(self: TokenEmbedding, idx: Tensor) Tensor { - stdx.debug.assert(idx.dtype().isInteger(), "TokenEmbedding expects an integer input, received: {}", .{idx}); - stdx.debug.assert(self.weight.rank() == 2, "TokenEmbedding expects it's weight Tensor to be a 2D matrix, got {}", .{self.weight}); + stdx.debug.assert(idx.dtype().isInteger(), "TokenEmbedding expects an integer input, received: {f}", .{idx}); + stdx.debug.assert(self.weight.rank() == 2, "TokenEmbedding expects it's weight Tensor to be a 2D matrix, got {f}", .{self.weight}); return self.weight.gatherValues(0, idx, .{}); } }; @@ -204,13 +204,13 @@ pub const RopeOpts = struct { /// - pos_idx: optional tensor which indicates which positions are needed. /// When not set `rope` return all positions from 0 to x.dim(.s) which is the max seq len. pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { - stdx.debug.assert(@mod(x.dim(.hd), 2) == 0, "rope expects a even head dim (.hd), got {}", .{x}); + stdx.debug.assert(@mod(x.dim(.hd), 2) == 0, "rope expects a even head dim (.hd), got {f}", .{x}); const idx = if (pos_idx) |idx| blk: { - stdx.debug.assert(x.shape().hasTags(.{.hd}), "rope expects x argument to have .hd axes got: rope(x={}, idx={})", .{ x, idx }); + stdx.debug.assert(x.shape().hasTags(.{.hd}), "rope expects x argument to have .hd axes got: rope(x={f}, idx={f})", .{ x, idx }); break :blk idx; } else blk: { - stdx.debug.assert(x.shape().hasTags(.{ .s, .hd }), "rope expects x argument to have both .s and .hd axes got: rope(x={})", .{x}); + stdx.debug.assert(x.shape().hasTags(.{ .s, .hd }), "rope expects x argument to have both .s and .hd axes got: rope(x={f})", .{x}); break :blk Tensor.arange(.{ .end = x.dim(.s) }, .f32).withTags(.{.s}); }; const x_real, const x_imag = splitRealImg(x, opts.layout); @@ -273,7 +273,7 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { switch (opts.scaling) { .default => {}, .custom => { - stdx.debug.assert(opts.scaling.custom.len == N, "rope expected custom inv_freq to match half head dimension {}, got {}", .{ N, opts.scaling.custom.len }); + stdx.debug.assert(opts.scaling.custom.len == N, "rope expected custom inv_freq to match half head dimension {d}, got {d}", .{ N, opts.scaling.custom.len }); @memcpy(inv_freq, opts.scaling.custom); }, .llama3 => |s| { @@ -318,7 +318,7 @@ test invFreq { var inv_freq: @TypeOf(llama_freq) = undefined; _invFreq(llama_conf, &inv_freq); for (llama_freq, inv_freq, 0..) |expected, actual, i| { - errdefer log.err("Mismatch at position {d}.\nExpected: {d}\nActual: {d}", .{ i, llama_freq, inv_freq }); + errdefer log.err("Mismatch at position {d}.\nExpected: {any}\nActual: {any}", .{ i, llama_freq, inv_freq }); try std.testing.expectApproxEqRel(expected, actual, 1e-5); } } @@ -462,7 +462,7 @@ pub fn upsample( ) Tensor { // TODO(james): make `nearest` compatible with resizeBilinear and resizeBicubic, and wrap them here. // resize* have API which are more explicit, this assume you want to scale the N-2 last axes. - stdx.debug.assert(3 <= input.rank() and input.rank() <= 5, "upsample is only implemented for (3,4,5)-D tensors, received {}", .{input}); + stdx.debug.assert(3 <= input.rank() and input.rank() <= 5, "upsample is only implemented for (3,4,5)-D tensors, received {f}", .{input}); stdx.debug.assert(opts.scale_factor.len == 1 or opts.scale_factor.len == input.rank() - 2, "scale factors", .{}); return switch (opts.mode) { .nearest => { @@ -791,7 +791,7 @@ pub fn causalAttnMask( attn_window_len: ?u32, ) Tensor { const attn_shape = Shape.init(attn_shape_, dtype); - stdx.debug.assert(attn_shape.rank() == 2, "causalAttnMask({}) shape need to be exactly 2 axes", .{attn_shape}); + stdx.debug.assert(attn_shape.rank() == 2, "causalAttnMask({f}) shape need to be exactly 2 axes", .{attn_shape}); const qlen = attn_shape.dim(-2); const q_idx = Tensor.iota(attn_shape, -2); const klen = attn_shape.dim(-1); @@ -843,7 +843,7 @@ pub const SdpaOpts = struct { pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { var q, var k, var v = .{ q_, k_, v_ }; - const err_template = "sdpa(q: {}, k: {}, v: {}, attn: {?}) is invalid ! "; + const err_template = "sdpa(q: {f}, k: {f}, v: {f}, attn: {?f}) is invalid ! "; const err_args = .{ q, k, v, opts.attn_mask }; stdx.debug.assert(q.shape().hasTags(.{ .h, .q, .hd }), err_template ++ "q is missing tags {{.h, .q, .hd}}", err_args); stdx.debug.assert(k.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "k is missing tags {{.h, .k, .hd}}", err_args); @@ -909,8 +909,8 @@ const SdpaMemEfficient = struct { chunking: SdpaChunks, fn forward(self: SdpaMemEfficient) Tensor { - stdx.debug.assert(@mod(self.q.dim(.q), self.chunking.q_chunk_size) == 0, "sdpaMemEfficient expects the chunk_size to exactly divise the seq_len, got: sdpaMemEfficient({}, {})", .{ self.q, self.chunking }); - stdx.debug.assert(@mod(self.k.dim(.k), self.chunking.k_chunk_size) == 0, "sdpaMemEfficient expects the chunk_size to exactly divise the seq_len, got: sdpaMemEfficient({}, {})", .{ self.k, self.chunking }); + stdx.debug.assert(@mod(self.q.dim(.q), self.chunking.q_chunk_size) == 0, "sdpaMemEfficient expects the chunk_size to exactly divise the seq_len, got: sdpaMemEfficient({f}, {})", .{ self.q, self.chunking }); + stdx.debug.assert(@mod(self.k.dim(.k), self.chunking.k_chunk_size) == 0, "sdpaMemEfficient expects the chunk_size to exactly divise the seq_len, got: sdpaMemEfficient({f}, {})", .{ self.k, self.chunking }); const n_q_chunks: u32 = @intCast(@divExact(self.q.dim(.q), self.chunking.q_chunk_size)); const ctx = zml.module.CompilationContext.current(); @@ -1054,7 +1054,7 @@ pub fn sdpaChunk(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) PartialSoft // Consider implementing sdpa from sdpaChunk. var q, var k, var v = .{ q_, k_, v_ }; - const err_template = "sdpa(q: {}, k: {}, v: {}, attn: {?}) is invalid ! "; + const err_template = "sdpa(q: {f}, k: {f}, v: {f}, attn: {?f}) is invalid ! "; const err_args = .{ q, k, v, opts.attn_mask }; stdx.debug.assert(q.shape().hasTags(.{ .h, .q, .hd }), err_template ++ "q is missing tags {{.h, .q, .hd}}", err_args); stdx.debug.assert(k.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "k is missing tags {{.h, .k, .hd}}", err_args); diff --git a/zml/nn/cuda.zig b/zml/nn/cuda.zig index d0de6543..4a682f31 100644 --- a/zml/nn/cuda.zig +++ b/zml/nn/cuda.zig @@ -51,7 +51,7 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { var fba = std.heap.FixedBufferAllocator.init(&buffer); const allocator = fba.allocator(); - const backend_config = std.fmt.allocPrintZ( + const backend_config = std.fmt.allocPrintSentinel( allocator, \\{{ \\ "operation_queue_id":"0", @@ -110,6 +110,7 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { q.dim(.q), k.dim(.k), }, + 0, ) catch unreachable; var bias = Tensor.constant(Shape.init(.{ .b = q.dim(.b), .h = q.dim(.h), .q = q.dim(.q), .k = k.dim(.k) }, q.dtype()), Data.init(q.dtype(), 0)); diff --git a/zml/ops.zig b/zml/ops.zig index e0af64e8..c26eac3c 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -155,7 +155,7 @@ pub fn reduce( // To that order, we initialize `result` to `inputs`, then we use stdx.meta.visit, // to find the correct mlir.Value, but we first broadcast before creating the final // Tensor struct. - var broadcasting_axes: std.BoundedArray(i64, Tensor.MAX_RANK) = .{}; + var broadcasting_axes: stdx.BoundedArray(i64, Tensor.MAX_RANK) = .{}; for (0..Tensor.MAX_RANK) |i| { if (std.mem.indexOfScalar(i64, axes, @intCast(i)) == null) { broadcasting_axes.append(@intCast(i)) catch unreachable; @@ -437,15 +437,15 @@ pub fn if_( @compileError("true_branch_fn and false_branch_fn return types don't match ! " ++ @typeName(TrueBlockSignature.Return) ++ " and " ++ @typeName(FalseBlockSignature.Return)); } - stdx.debug.assert(pred.dtype() == .bool and pred.count() == 1, "zml.ops.if_ expects the condition to have exactly one element of dtype .bool, got {}", .{pred}); + stdx.debug.assert(pred.dtype() == .bool and pred.count() == 1, "zml.ops.if_ expects the condition to have exactly one element of dtype .bool, got {f}", .{pred}); const ctx = CompilationContext.current(); const true_branch_block, const true_branch_res = ctx.makeBlock(.open, TrueBlockSignature, &true_branch_fn, blkctx, {}); const false_branch_block, const false_branch_res = ctx.makeBlock(.open, TrueBlockSignature, &false_branch_fn, blkctx, {}); - var true_shapes = std.ArrayList(Shape).init(ctx.allocator()); + var true_shapes = std.array_list.Managed(Shape).init(ctx.allocator()); defer true_shapes.deinit(); - var false_shapes = std.ArrayList(Shape).init(ctx.allocator()); + var false_shapes = std.array_list.Managed(Shape).init(ctx.allocator()); defer false_shapes.deinit(); var failed_to_collect = false; @@ -456,9 +456,9 @@ pub fn if_( failed_to_collect = true; }; if (!failed_to_collect) { - stdx.debug.assert(true_shapes.items.len == false_shapes.items.len, "zml.ops.if_ expects the true and false branch to produce the same number of tensors. Got: \n - true branch: {_}\n -false branch: {_}", .{ true_shapes.items, false_shapes.items }); + stdx.debug.assert(true_shapes.items.len == false_shapes.items.len, "zml.ops.if_ expects the true and false branch to produce the same number of tensors. Got: \n - true branch: {any}\n -false branch: {any}", .{ true_shapes.items, false_shapes.items }); for (true_shapes.items, false_shapes.items) |true_shape, false_shape| { - stdx.debug.assert(true_shape.eqlWithTags(false_shape), "zml.ops.if_ expects the true and false branch to produce tensors of the same shape. Got: \n - true branch: {_}\n -false branch: {_}", .{ true_shapes.items, false_shapes.items }); + stdx.debug.assert(true_shape.eqlWithTags(false_shape), "zml.ops.if_ expects the true and false branch to produce tensors of the same shape. Got: \n - true branch: {any}\n -false branch: {any}", .{ true_shapes.items, false_shapes.items }); } } @@ -751,7 +751,7 @@ pub fn fromMlirOperationWithTags(op: mlir.Operation, base: anytype) @TypeOf(base meta.visit((struct { fn cb(inner_ctx: *LocalContext, tensor: *Tensor) void { var new = Tensor.fromMlirValue(inner_ctx.op.result(inner_ctx.index)); - stdx.debug.internalAssert(new.rank() == tensor.rank(), "expected operand result to have rank {} but got {}", .{ tensor.rank(), new }); + stdx.debug.internalAssert(new.rank() == tensor.rank(), "expected operand result to have rank {} but got {f}", .{ tensor.rank(), new }); // copy tags and sharding info over // some ops can change dims eg reduceWindow, so we trust mlir here. new._shape._tags = tensor._shape._tags; @@ -932,7 +932,7 @@ pub fn scatter( const n_inputs = meta.count(Tensor, &inputs); const n_updates = meta.count(Tensor, &updates); - stdx.debug.assert(n_inputs == n_updates, "zml.ops.scatter expects the same number of tensors in inputs and updates, got {} and {}", .{ n_inputs, n_updates }); + stdx.debug.assert(n_inputs == n_updates, "zml.ops.scatter expects the same number of tensors in inputs and updates, got {d} and {d}", .{ n_inputs, n_updates }); // Note: I was a bit lazy here, and I only look at tags on the first tensor. // we probably should check all of them. @@ -944,7 +944,7 @@ pub fn scatter( // validate coord axes: all coord_axes should exist inside self for (indices_axes.constSlice()) |t| { - stdx.debug.assert(self._shape.hasTag(t) != null, "zml.ops.scatter expects axes of indices to be axes of inputs, got input={_} and indices={s}", .{ self, indices_axes.constSlice() }); + stdx.debug.assert(self._shape.hasTag(t) != null, "zml.ops.scatter expects axes of indices to be axes of inputs, got input={f} and indices={any}", .{ self, indices_axes.constSlice() }); } // Handle scalar indices by broadcasting them to the indices with the highest rank. @@ -958,8 +958,8 @@ pub fn scatter( break :blk higher_rank; }; for (indices_per_axis.slice()) |*idx| { - stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "zml.ops.scatter expects all indices tensor to have the same shape, got {_}", .{indices_per_axis.slice()}); - stdx.debug.assert(idx.dtype() == indices_shape.dtype(), "zml.ops.scatter expects all indices tensor to have the same dtype, got {_}", .{indices_per_axis.slice()}); + stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "zml.ops.scatter expects all indices tensor to have the same shape, got {any}", .{indices_per_axis.slice()}); + stdx.debug.assert(idx.dtype() == indices_shape.dtype(), "zml.ops.scatter expects all indices tensor to have the same dtype, got {any}", .{indices_per_axis.slice()}); idx.* = idx.broad(indices_shape); } @@ -972,7 +972,7 @@ pub fn scatter( var config = scatterConfig(self.shape(), update.shape(), indices_per_axis, indices_axes); const indices = scatterPrepareIndices(&config, self.shape(), update.shape(), &indices_per_axis, &indices_axes); // const n_indices_axes = update.rank() - _collectAxes(AxisKind, up_kind, .update_window).len; - // stdx.debug.assert(n_indices_axe == indices_axes.len, "scatter({_}, {any}) expects 'updates' to contain all axes from 'indices', got indices={s}, updates={_}", .{ self, index_tensors, indices_axes.constSlice(), update }); + // stdx.debug.assert(n_indices_axe == indices_axes.len, "scatter({f}, {any}) expects 'updates' to contain all axes from 'indices', got indices={s}, updates={f}", .{ self, index_tensors, indices_axes.constSlice(), update }); const mlir_ctx = ctx.mlirCtx(); var _scalar: T = inputs; @@ -985,10 +985,10 @@ pub fn scatter( const UpdateS = BlockSign(update_fn); const update_block, _ = ctx.makeBlock(.hermetic, UpdateS, update_fn, blkctx, .{ _scalar, _scalar }); - var input_values = std.ArrayList(mlir.Value).initCapacity(ctx.allocator(), n_inputs) catch @panic("OOM"); + var input_values = std.array_list.Managed(mlir.Value).initCapacity(ctx.allocator(), n_inputs) catch @panic("OOM"); defer input_values.deinit(); meta.collect(CompilationContext.getValue, ctx, &input_values, &inputs) catch unreachable; - var updates_values = std.ArrayList(mlir.Value).initCapacity(ctx.allocator(), n_updates) catch @panic("OOM"); + var updates_values = std.array_list.Managed(mlir.Value).initCapacity(ctx.allocator(), n_updates) catch @panic("OOM"); defer updates_values.deinit(); meta.collect(CompilationContext.getValue, ctx, &updates_values, &updates) catch unreachable; @@ -1029,8 +1029,8 @@ pub fn scatter( } const ScatterConfig = struct { - op_kind: std.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}, - up_kind: std.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}, + op_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}, + up_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}, indices_batch_axes: Shape.DimsArray = .{}, scatter_to_operand_axes: Shape.DimsArray = .{}, updates_transpose: Shape.AxesArray = .{}, @@ -1041,11 +1041,11 @@ const AxisKind = enum { batching, update_window, inserted_window, window_id }; fn scatterConfig( op: Shape, update: Shape, - indices_per_axis: std.BoundedArray(Tensor, Tensor.MAX_RANK), + indices_per_axis: stdx.BoundedArray(Tensor, Tensor.MAX_RANK), indices_axes: Shape.TagsArray, ) ScatterConfig { - var op_kind: std.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}; - var up_kind: std.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}; + var op_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}; + var up_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}; var indices_batch_axes: Shape.DimsArray = .{}; var scatter_to_operand_axes: Shape.DimsArray = .{}; var updates_transpose: Shape.AxesArray = .{}; @@ -1058,7 +1058,7 @@ fn scatterConfig( scatter_to_operand_axes.appendAssumeCapacity(op.axis(t)); } for (indices.tags()) |t| { - stdx.debug.assert(update.hasTag(t) != null, "scatter expects 'updates' to have all axes of 'indices', got self={_}, updates={_} and indices={_}", .{ op, update, indices }); + stdx.debug.assert(update.hasTag(t) != null, "scatter expects 'updates' to have all axes of 'indices', got self={f}, updates={f} and indices={f}", .{ op, update, indices }); updates_transpose.appendAssumeCapacity(update.axis(t)); } @@ -1094,11 +1094,11 @@ fn scatterConfig( if (indices.hasTag(t) != null) { up_kind.appendAssumeCapacity(.window_id); } else if (op.hasTag(t)) |self_ax| { - stdx.debug.assert(update.dim(up_ax) <= op.dim(self_ax), "scatter expects the slices described in 'updates' to fit inside 'op', but along axis .{s} it doesn't. Got op={_}, updates={_}.", .{ t, op, update }); + stdx.debug.assert(update.dim(up_ax) <= op.dim(self_ax), "scatter expects the slices described in 'updates' to fit inside 'op', but along axis .{s} it doesn't. Got op={f}, updates={f}.", .{ t, op, update }); up_kind.appendAssumeCapacity(.update_window); } else { // TODO: consider accepting untagged update here. - std.debug.panic("scatter expects 'updates' to be made of axes from op={_} and from indices={s}, got unknown tag {s} in {_}", .{ op, indices_axes.constSlice(), t, update }); + std.debug.panic("scatter expects 'updates' to be made of axes from op={f} and from indices={any}, got unknown tag {s} in {f}", .{ op, indices_axes.constSlice(), std.mem.sliceTo(t, 0), update }); } } } else { @@ -1174,7 +1174,7 @@ fn scatterPrepareIndices( cfg: *ScatterConfig, op: Shape, update: Shape, - indices_per_axis: *std.BoundedArray(Tensor, Tensor.MAX_RANK), + indices_per_axis: *stdx.BoundedArray(Tensor, Tensor.MAX_RANK), indices_axes: *Shape.TagsArray, ) Tensor { var old_scatter_to_op_axes = cfg.scatter_to_operand_axes; @@ -1194,7 +1194,7 @@ fn scatterPrepareIndices( // Reorder the axes so that in indices_per_axis is ordered like in op if possible. // TODO: transpose updates if needed - var indices: std.BoundedArray(Tensor, Tensor.MAX_RANK) = .{}; + var indices: stdx.BoundedArray(Tensor, Tensor.MAX_RANK) = .{}; var scatter_to_op_axes: Shape.DimsArray = .{}; while (old_scatter_to_op_axes.len > 0) { @@ -1209,7 +1209,7 @@ fn scatterPrepareIndices( for (scatter_to_op_axes.constSlice(), 0..) |sc_ax, i| { if (i != sc_ax) { - log.warn("Found a slow scatter pattern, which is going to generate a while loop: scatter({_}, {any}, {_}). Because the index axes aren't the major ones in the input tensor.", .{ op, scatter_to_op_axes.constSlice(), update }); + log.warn("Found a slow scatter pattern, which is going to generate a while loop: scatter({f}, {any}, {f}). Because the index axes aren't the major ones in the input tensor.", .{ op, scatter_to_op_axes.constSlice(), update }); break; } } diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index b12bdb20..84343a63 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -75,14 +75,14 @@ pub const Client = opaque { } fn compileSync(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { - var bytecode = std.ArrayList(u8).init(allocator); + var bytecode: std.array_list.Managed(u8) = .init(allocator); defer bytecode.deinit(); module.op().writeBytecodeWithConfig(bytecode.writer(), .{ .desiredEmitedVersion = 1 }) catch |err| { log.err("failed to write module bytecode: {}", .{err}); return err; }; - var serialized_buffer = std.ArrayList(u8).init(allocator); + var serialized_buffer: std.array_list.Managed(u8) = .init(allocator); defer serialized_buffer.deinit(); const stablehlo_version = blk: { @@ -220,7 +220,7 @@ pub const Event = opaque { }{}; try self.inner().onReady(api, &(struct { - fn call(err: ?*pjrt.Error, user_arg: ?*anyopaque) callconv(.C) void { + fn call(err: ?*pjrt.Error, user_arg: ?*anyopaque) callconv(.c) void { const ctx_: *@TypeOf(ctx) = @ptrCast(@alignCast(user_arg.?)); ctx_.err = err; ctx_.event.set(); diff --git a/zml/platform.zig b/zml/platform.zig index 6f48137d..0bd8a2a5 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -15,7 +15,7 @@ pub const CompilationOptions = struct { xla_dump_fusion_visualization: bool = false, xla_dump_hlo_pass_re: ?[]const u8 = null, sharding_enabled: bool = false, - sharding_axes: std.BoundedArray([*:0]const u8, 8) = .{}, + sharding_axes: stdx.BoundedArray([*:0]const u8, 8) = .{}, }; pub const Platform = struct { diff --git a/zml/shape.zig b/zml/shape.zig index ce16ec93..bd5f2a22 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -22,9 +22,9 @@ pub const Shape = struct { pub const TagUnknown = "_".ptr; const TagLast = "last".ptr; - pub const DimsArray = std.BoundedArray(i64, MAX_RANK); - pub const TagsArray = std.BoundedArray(Tag, MAX_RANK); - pub const AxesArray = std.BoundedArray(u3, MAX_RANK); + pub const DimsArray = stdx.BoundedArray(i64, MAX_RANK); + pub const TagsArray = stdx.BoundedArray(Tag, MAX_RANK); + pub const AxesArray = stdx.BoundedArray(u3, MAX_RANK); pub const ShardingInfo = @Vector(MAX_RANK, bool); const UnknownTags: TagsArray = .{ .len = 0, .buffer = [_]Tag{TagUnknown} ** MAX_RANK }; @@ -300,7 +300,7 @@ pub const Shape = struct { fn axisFromInt(self: Shape, a: isize) u3 { const rk: i8 = self.rank(); if (a < -rk or a > rk) { - stdx.debug.panic("Tensor {} doesn't have dimension: {d}", .{ self, a }); + stdx.debug.panic("Tensor {f} doesn't have dimension: {d}", .{ self, a }); } return if (a < 0) @intCast(a + rk) @@ -341,9 +341,9 @@ pub const Shape = struct { } fn axisFromTag(self: Shape, d: Tag) u3 { - stdx.debug.assert(d != TagUnknown, "The unknown tag .{s} can't be used to fetch axis in {}", .{ d, self }); + stdx.debug.assert(d != TagUnknown, "The unknown tag .{s} can't be used to fetch axis in {f}", .{ d, self }); return self.axisFromTagMaybe(d) orelse { - stdx.debug.panic("Tensor {} doesn't have dimension with tag: {s}", .{ self, d }); + stdx.debug.panic("Tensor {f} doesn't have dimension with tag: {s}", .{ self, d }); }; } @@ -357,7 +357,7 @@ pub const Shape = struct { pub fn count(self: Shape) usize { var res: i64 = 1; for (self.dims()) |d| { - stdx.debug.assert(d >= 0, "Can't count elements in shape with negative dimension: {}", .{self}); + stdx.debug.assert(d >= 0, "Can't count elements in shape with negative dimension: {f}", .{self}); res *= d; } return @intCast(res); @@ -388,12 +388,11 @@ pub const Shape = struct { /// Bare format {_}: "{.a=10, .b=20}, dtype=.f32" pub fn format( self: Shape, - comptime fmt: []const u8, - options: std.fmt.FormatOptions, writer: anytype, ) !void { - _ = options; - const bare_fmt = fmt.len == 1 and fmt[0] == '_'; + // TODO: impl alternative format + // const bare_fmt = fmt.len == 1 and fmt[0] == '_'; + const bare_fmt = true; _ = try writer.write(if (bare_fmt) "{" else "Shape({"); var need_comma = false; @@ -441,12 +440,12 @@ pub const Shape = struct { var new_shape: Shape = .{ ._dtype = self.dtype() }; new_shape._dims, new_shape._tags = parseDimensions(new_shape_); new_shape.inferMissingAxis(self.count()); - stdx.debug.assert(self.count() == new_shape.count(), "Can't reshape {d} to {d}", .{ self.dims(), new_shape.dims() }); + stdx.debug.assert(self.count() == new_shape.count(), "Can't reshape {any} to {any}", .{ self.dims(), new_shape.dims() }); return new_shape; } fn inferMissingAxis(self: *Shape, n_: usize) void { - stdx.debug.assert(std.mem.count(i64, self.dims(), &.{-1}) < 2, "Cannot infer multiple dimensions when reshaping to: {}", .{self.*}); + stdx.debug.assert(std.mem.count(i64, self.dims(), &.{-1}) < 2, "Cannot infer multiple dimensions when reshaping to: {f}", .{self.*}); const inferred_ax = std.mem.indexOfScalar(i64, self.dims(), -1) orelse return; // We can't use `self.count()` yet cause we have negative dims. @@ -524,7 +523,7 @@ pub const Shape = struct { } pub fn insertTag(self: Shape, axis_: anytype, d: i64, tag_: anytype) Shape { - stdx.debug.assert(self.rank() < MAX_RANK - 1, "Can't insert new axis in {}, it's already at max rank.", .{self}); + stdx.debug.assert(self.rank() < MAX_RANK - 1, "Can't insert new axis in {f}, it's already at max rank.", .{self}); const ax = if (@TypeOf(axis_) == EnumLiteral and axis_ == .last) self.rank() @@ -652,7 +651,7 @@ pub const Shape = struct { var res = self; if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, EnumLiteral)) { - stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {}, got {any}", .{ self, tagz }); + stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {any}", .{ self, tagz }); for (tagz, 0..) |tag_, i| { res._tags.set(i, toTag(tag_)); } @@ -660,7 +659,7 @@ pub const Shape = struct { } if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, EnumLiteral)) { - stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {}, got {}", .{ self, tagz }); + stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {}", .{ self, tagz }); inline for (tagz, 0..) |tag_, i| { res._tags.set(i, toTag(tag_)); } @@ -699,7 +698,7 @@ pub const Shape = struct { var res = self; if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, EnumLiteral)) { - stdx.debug.assert(tagz.len <= self.rank(), "Too many tags for shape {}, got {any}", .{ self, tagz }); + stdx.debug.assert(tagz.len <= self.rank(), "Too many tags for shape {f}, got {any}", .{ self, tagz }); for (tagz, self.rank() - tagz.len..) |tag_, i| { res._tags.set(i, toTag(tag_)); } @@ -707,7 +706,7 @@ pub const Shape = struct { } if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, EnumLiteral)) { - stdx.debug.assert(tagz.len <= self.rank(), "Too many tags for shape {}, got {}", .{ self, tagz }); + stdx.debug.assert(tagz.len <= self.rank(), "Too many tags for shape {f}, got {}", .{ self, tagz }); inline for (tagz, self.rank() - tagz.len..) |tag_, i| { res._tags.set(i, toTag(tag_)); } @@ -765,7 +764,7 @@ pub const Shape = struct { var res = self; inline for (std.meta.fields(T)) |field| { const new_field = @field(renames, field.name); - stdx.debug.assert(self.hasTag(new_field) == null, "{}.rename({any}) failed because of duplicated axis {}", .{ self, renames, new_field }); + stdx.debug.assert(self.hasTag(new_field) == null, "{f}.rename({any}) failed because of duplicated axis {}", .{ self, renames, new_field }); res._tags.set(self.axis(field), toTag(new_field)); } return res; @@ -785,9 +784,9 @@ pub const Shape = struct { } } - pub fn computeStrides(self: Shape) std.BoundedArray(i64, MAX_RANK) { + pub fn computeStrides(self: Shape) stdx.BoundedArray(i64, MAX_RANK) { const rk = self.rank(); - var strides: std.BoundedArray(i64, MAX_RANK) = .{ .len = rk }; + var strides: stdx.BoundedArray(i64, MAX_RANK) = .{ .len = rk }; if (rk == 0) return strides; const V = @Vector(MAX_RANK, i64); @@ -907,7 +906,7 @@ pub const Shape = struct { var new_dim: i64 = 1; for (axes__.constSlice(), first_axis..) |ax, counter| { new_dim *= self.dim(ax); - stdx.debug.assert(ax == counter, "Can't merge shape {} along non-contiguous axes {any}", .{ self, axes_ }); + stdx.debug.assert(ax == counter, "Can't merge shape {f} along non-contiguous axes {any}", .{ self, axes_ }); } var new_shape = self; @@ -991,10 +990,10 @@ pub const Shape = struct { return res; } - pub fn parseStruct(T: type, v: anytype) struct { std.BoundedArray(T, MAX_RANK), TagsArray } { + pub fn parseStruct(T: type, v: anytype) struct { stdx.BoundedArray(T, MAX_RANK), TagsArray } { const V = @TypeOf(v); - var vals_: std.BoundedArray(T, MAX_RANK) = .{}; + var vals_: stdx.BoundedArray(T, MAX_RANK) = .{}; var tags_: TagsArray = .{}; if (comptime stdx.meta.isSliceOf(V, T)) { @@ -1029,10 +1028,10 @@ pub const Shape = struct { } /// Parses a struct literal into a list of options for each axes. - pub fn parseAxesOptions(self: Shape, T: type, options: anytype, default: T) std.BoundedArray(T, MAX_RANK) { + pub fn parseAxesOptions(self: Shape, T: type, options: anytype, default: T) stdx.BoundedArray(T, MAX_RANK) { const V = @TypeOf(options); - var res: std.BoundedArray(T, MAX_RANK) = .{}; + var res: stdx.BoundedArray(T, MAX_RANK) = .{}; if (comptime stdx.meta.isSliceOf(V, T)) { stdx.debug.assert(options.len == self.rank(), "expects exactly {} options in slice, for {} got {}", .{ self.rank(), self, options.len }); for (options) |d| { @@ -1084,7 +1083,7 @@ pub const Shape = struct { for (0..other.rank()) |ax| { if (other.tag(ax) != Shape.TagUnknown) { if (self.hasTag(other.tag(ax))) |batching_ax| { - stdx.debug.assert(batching_ax == batching_axes and batching_ax == ax, "outer expects batching dims to be the first dims in both tensors, got outer({}, {})", .{ self, other }); + stdx.debug.assert(batching_ax == batching_axes and batching_ax == ax, "outer expects batching dims to be the first dims in both tensors, got outer({f}, {f})", .{ self, other }); batching_axes += 1; } } diff --git a/zml/tensor.zig b/zml/tensor.zig index 74bf1779..26f670d7 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -53,13 +53,12 @@ pub const Tensor = struct { pub fn format( self: Tensor, - comptime fmt: []const u8, - options: std.fmt.FormatOptions, writer: anytype, ) !void { - _ = options; - const bare_fmt = fmt.len == 1 and fmt[0] == '_'; - try writer.print(if (bare_fmt) "{_}" else "Tensor({_})", .{self._shape}); + // TODO(0.15.0) handle format + // const bare_fmt = fmt.len == 1 and fmt[0] == '_'; + const bare_fmt = false; + try writer.print(if (bare_fmt) "{f}" else "Tensor({f})", .{self._shape}); } /// Returns the shape of a Tensor. @@ -99,7 +98,7 @@ pub const Tensor = struct { if (builtin.mode == .Debug) { // Check that the MLIR value actually have the same shape. const other = fromMlirValue(val); - stdx.debug.internalAssert(sh.eql(other._shape), "Created a {} from Mlir value but expected {}", .{ other._shape, res._shape }); + stdx.debug.internalAssert(sh.eql(other._shape), "Created a {f} from Mlir value but expected {f}", .{ other._shape, res._shape }); } return res; @@ -145,7 +144,7 @@ pub const Tensor = struct { /// Returns the indices of each of the given axes. /// /// 'axis_' can be an integer or a tag. - pub fn axes(self: Tensor, axes_: anytype) std.BoundedArray(u3, Tensor.MAX_RANK) { + pub fn axes(self: Tensor, axes_: anytype) stdx.BoundedArray(u3, Tensor.MAX_RANK) { return self._shape.axes(axes_); } @@ -260,7 +259,7 @@ pub const Tensor = struct { /// For `reuseBuffer` to be effective, it needs to propagate all the way through the output. pub fn reuseBuffer(self: Tensor, origin: Tensor) Tensor { // Note: check donation docs, this may be too permissive. - stdx.debug.assert(self.byteSize() == origin.byteSize(), "Can't reuse buffers between tensors of different size: {} and {}", .{ self, origin }); + stdx.debug.assert(self.byteSize() == origin.byteSize(), "Can't reuse buffers between tensors of different size: {f} and {f}", .{ self, origin }); // TODO: should we store all donations inside the context ? var res = self; @@ -410,7 +409,7 @@ pub const Tensor = struct { stdx.debug.assert(self.dtype() == other.dtype(), "triangularSolve expects tensors to be of the same type, got {} and {}", .{ self.dtype(), other.dtype() }); stdx.debug.assert(self.rank() <= 2 and self.rank() == other.rank(), "triangularSolve expects tensors to have the same rank and be <= 2, got {} and {}", .{ self.rank(), other.rank() }); - const loc = self.getContext().location(@src(), "triangularSolve({_}, {})", .{ self, opts }); + const loc = self.getContext().location(@src(), "triangularSolve({f}, {})", .{ self, opts }); const op = dialect.stablehlo.triangular_solve(self.getContext().mlirCtx(), self.value(), other.value(), loc, opts); return _result(self._shape, op.result(0)); } @@ -435,7 +434,7 @@ pub const Tensor = struct { /// Returns a Tensor of complex number converted from a pair of real and imaginary Tensors. pub fn complex(re: Tensor, im: Tensor) Tensor { - stdx.debug.assert(re._shape.eql(im._shape), "complex expects tensor shapes to match, got {} and {}", .{ re._shape, im._shape }); + stdx.debug.assert(re._shape.eql(im._shape), "complex expects tensor shapes to match, got {f} and {f}", .{ re._shape, im._shape }); stdx.debug.assert(re.dtype() == .f32 or re.dtype() == .f64, "complex expects tensors type to be f32 or f64, got {}", .{re.dtype()}); const loc = re.getContext().mlirCtx().location(@src()); @@ -523,7 +522,7 @@ pub const Tensor = struct { }, }; - const loc = self.getContext().location(@src(), "fft({_},{})", .{ self, opts }); + const loc = self.getContext().location(@src(), "fft({f},{})", .{ self, opts }); const op = dialect.stablehlo.fft(self.getContext().mlirCtx(), self.value(), loc, opts); return _result(sh, op.result(0)); } @@ -551,7 +550,7 @@ pub const Tensor = struct { /// but it is not guaranteed to be deterministic between implementations. pub fn bitGenerator(self: Rng, sh: Shape) struct { Rng, Tensor } { const ctx = CompilationContext.current(); - const loc = ctx.location(@src(), "rand.bitGen({_})", .{sh}); + const loc = ctx.location(@src(), "rand.bitGen({f})", .{sh}); const op = dialect.stablehlo.rng_bit_generator( ctx.mlirCtx(), self.algorithm, @@ -589,7 +588,7 @@ pub const Tensor = struct { 16 => .u16, 32 => .u32, 64 => .u64, - else => stdx.debug.panic("uniform don't support non-byte aligned dtype. Got: {}", .{shape_}), + else => stdx.debug.panic("uniform don't support non-byte aligned dtype. Got: {f}", .{shape_}), }; const rng, const bits = self.bitGenerator(shape_.withDtype(uint_dtype)); @@ -674,7 +673,7 @@ pub const Tensor = struct { stdx.debug.assert(sh.dtype().isFloat(), "normal expects tensor type to be a float, got {}", .{sh.dtype()}); const ctx = CompilationContext.current(); - const loc = ctx.location(@src(), "rand.normal({_}, mean={},stddev={})", .{ sh, opts.mean, opts.stddev }); + const loc = ctx.location(@src(), "rand.normal({f}, mean={},stddev={})", .{ sh, opts.mean, opts.stddev }); const a = Tensor.constant(.{}, Data.init(sh.dtype(), opts.mean)); const b = Tensor.constant(.{}, Data.init(sh.dtype(), opts.stddev)); const res_shape = Tensor.constantTensor(HostBuffer.fromSlice(.{sh.rank()}, sh.dims())); @@ -1046,7 +1045,7 @@ pub const Tensor = struct { if (to == self.dtype()) { return self; } - const loc = self.getContext().location(@src(), "convert({_},to={s})", .{ self, @tagName(to) }); + const loc = self.getContext().location(@src(), "convert({f},to={s})", .{ self, @tagName(to) }); const mlir_ctx = self.getContext().mlirCtx(); const res_type = mlirx.tensorType(mlir_ctx, self.shape().withDtype(to)); @@ -1160,7 +1159,7 @@ pub const Tensor = struct { ) Tensor { stdx.debug.assert(lhs.dtype() == rhs.dtype(), "dotGeneral expects tensors to be of the same type, got {} and {}", .{ lhs.dtype(), rhs.dtype() }); - const Axes = std.BoundedArray(i64, MAX_RANK); + const Axes = stdx.BoundedArray(i64, MAX_RANK); var res_shape: Shape = .{ ._dtype = lhs.dtype() }; // Validate batching axes @@ -1168,7 +1167,7 @@ pub const Tensor = struct { var rhs_batching_axes: Axes = .{}; for (batching_axes) |b_axes| { const l, const r = b_axes; - stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects batching dimensions to be equal, got {} and {} in {} and {}", .{ l, r, lhs, rhs }); + stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects batching dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); var t = lhs._shape.tag(l); if (t == Shape.TagUnknown) t = rhs._shape.tag(r); res_shape = res_shape.appendDim(lhs._shape.dim(l), t); @@ -1181,7 +1180,7 @@ pub const Tensor = struct { var rhs_contracting_axes: Axes = .{}; for (contracting_axes) |c_axes| { const l, const r = c_axes; - stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects contracting dimensions to be equal, got {} and {} in {} and {}", .{ l, r, lhs, rhs }); + stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects contracting dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); lhs_contracting_axes.appendAssumeCapacity(lhs._shape.axis(l)); rhs_contracting_axes.appendAssumeCapacity(rhs._shape.axis(r)); } @@ -1209,7 +1208,7 @@ pub const Tensor = struct { } const mlir_ctx = lhs.getContext().mlirCtx(); - const loc = lhs.getContext().location(@src(), "dot({_},{_},contracting={any},batching={any}", .{ lhs, rhs, contracting_axes, batching_axes }); + const loc = lhs.getContext().location(@src(), "dot({f},{f},contracting={any},batching={any}", .{ lhs, rhs, contracting_axes, batching_axes }); const op = dialect.stablehlo.dot_general( mlir_ctx, lhs.value(), @@ -1406,7 +1405,7 @@ pub const Tensor = struct { else toI64(axes__); - stdx.debug.assert(permutation.len == self.rank(), "transpose expects input tensor rank and 'axes_' length to be equal, got {_} and {d}", .{ self, permutation[0..@min(permutation.len, MAX_RANK + 2)] }); + stdx.debug.assert(permutation.len == self.rank(), "transpose expects input tensor rank and 'axes_' length to be equal, got {f} and {any}", .{ self, permutation[0..@min(permutation.len, MAX_RANK + 2)] }); if (std.mem.eql(i64, permutation, no_op[0..self.rank()])) { return self; @@ -1417,7 +1416,7 @@ pub const Tensor = struct { return self.reshape(res_shape); } - const loc = self.getContext().location(@src(), "transpose({_}, {d})", .{ self, permutation }); + const loc = self.getContext().location(@src(), "transpose({f}, {any})", .{ self, permutation }); const op = dialect.stablehlo.transpose( self.getContext().mlirCtx(), self.value(), @@ -1505,7 +1504,7 @@ pub const Tensor = struct { // stdx.debug.assert(a + 1 < self.rank(), "Can't flatten {} on the last axis {}.", .{ self, axis }); const new_shape = old_shape.remove(a + 1).set(a, old_shape.dim(a) * old_shape.dim(a + 1)); - const loc = self.getContext().location(@src(), "flatten({_},{})", .{ self, axis_ }); + const loc = self.getContext().location(@src(), "flatten({f},{})", .{ self, axis_ }); const reshaped_val = dialect.stablehlo.reshape( self.getContext().mlirCtx(), self.value(), @@ -1684,7 +1683,7 @@ pub const Tensor = struct { const res_shape = shape0.insertTag(axis_, 1, tag); for (tensors[1..]) |tensor| { - stdx.debug.assert(shape0.eqlWithTags(tensor._shape), "stack expects tensor shapes to match, got {} and {}", .{ shape0, tensor._shape }); + stdx.debug.assert(shape0.eqlWithTags(tensor._shape), "stack expects tensor shapes to match, got {f} and {f}", .{ shape0, tensor._shape }); } var reshaped: [32]Tensor = undefined; @@ -1859,7 +1858,7 @@ pub const Tensor = struct { const dt: DataType = if (sh.dim(a) <= std.math.maxInt(i32)) .i32 else .i64; const res_shape = sh.withDtype(dt); const ctx = CompilationContext.current(); - const loc = ctx.location(@src(), "iota({_}, {})", .{ res_shape, a }); + const loc = ctx.location(@src(), "iota({f}, {})", .{ res_shape, a }); const mlir_ctx = ctx.mlirCtx(); var op = dialect.stablehlo.iota( @@ -1931,7 +1930,7 @@ pub const Tensor = struct { pub fn constant(dimz: anytype, val: Data) Tensor { const sh = Shape.init(dimz, val.dtype()); const ctx = CompilationContext.current().mlirCtx(); - const loc = CompilationContext.current().location(@src(), "dims={d}, value={}", .{ sh, val }); + const loc = CompilationContext.current().location(@src(), "dims={f}, value={}", .{ sh, val }); var constant_op = if (mlirx.denseElementAttrType(val.dtype())) |elem_type| dialect.stablehlo.constant(ctx, &.{}, elem_type, val.constSlice(), loc) @@ -1951,7 +1950,7 @@ pub const Tensor = struct { pub fn constantTensor(val: HostBuffer) Tensor { const ctx = CompilationContext.current().mlirCtx(); const loc = ctx.location(@src()); - const elem_type = mlirx.denseElementAttrType(val.dtype()) orelse std.debug.panic("constantTensor expects a dtype that can be serialized to MLIR, like f32 or i32, got {}", .{val.shape()}); + const elem_type = mlirx.denseElementAttrType(val.dtype()) orelse std.debug.panic("constantTensor expects a dtype that can be serialized to MLIR, like f32 or i32, got {f}", .{val.shape()}); const constant_op = dialect.stablehlo.constant(ctx, val.shape().dims(), elem_type, val.bytes(), loc); return _result(val.shape(), constant_op.result(0)); } @@ -1975,10 +1974,10 @@ pub const Tensor = struct { /// you will lose the tags. /// To avoid use favorise `.broad(shape)` when working with tagged tensors. pub fn broadcast(self: Tensor, output_shape: Shape, axes_: []const i64) Tensor { - stdx.debug.assert(axes_.len == self.rank(), "broadcast expects axes_ to map all axes from self to axes of the output shape, got broadcast({}, {}, {d})", .{ self, output_shape, axes_ }); + stdx.debug.assert(axes_.len == self.rank(), "broadcast expects axes_ to map all axes from self to axes of the output shape, got broadcast({f}, {f}, {any})", .{ self, output_shape, axes_ }); for (0.., axes_) |self_ax, other_ax| { const d = self.dim(self_ax); - stdx.debug.assert(d == 1 or d == output_shape.dim(other_ax), "broadcast expects shape axes to either be 1-sized or to match the target size. got broadcast({}, {}, {d}), error on self axis {} mapping to other axis {}", .{ self, output_shape, axes_, self_ax, other_ax }); + stdx.debug.assert(d == 1 or d == output_shape.dim(other_ax), "broadcast expects shape axes to either be 1-sized or to match the target size. got broadcast({f}, {f}, {any}), error on self axis {d} mapping to other axis {d}", .{ self, output_shape, axes_, self_ax, other_ax }); } const res_shape = output_shape.withDtype(self.dtype()); @@ -1989,7 +1988,7 @@ pub const Tensor = struct { } const ctx = self.getContext(); const result_type = mlirx.tensorType(ctx.mlirCtx(), res_shape); - const loc = ctx.location(@src(), "broadcast({_}, {_}, axes={d})", .{ self, res_shape, axes_ }); + const loc = ctx.location(@src(), "broadcast({f}, {f}, axes={any})", .{ self, res_shape, axes_ }); const broadcast_op = dialect.stablehlo.broadcast_in_dim(ctx.mlirCtx(), self.value(), axes_, result_type, loc); return _result(res_shape, broadcast_op.result(0)); @@ -1997,7 +1996,7 @@ pub const Tensor = struct { /// Broadcasts a Tensor to the given shape, adding axes at the beginning. pub fn broadcastLeft(self: Tensor, output_shape: Shape) Tensor { - stdx.debug.assert(self.rank() <= output_shape.rank(), "broadcastLeft expects tensor rank to be less than output tensor rank, got {} and {}", .{ self.rank(), output_shape.rank() }); + stdx.debug.assert(self.rank() <= output_shape.rank(), "broadcastLeft expects tensor rank to be less than output tensor rank, got {d} and {d}", .{ self.rank(), output_shape.rank() }); const a = output_shape.rank() - self.rank(); if (self.rank() == output_shape.rank() and std.mem.eql(i64, self.dims(), output_shape.dims())) { @@ -2009,7 +2008,7 @@ pub const Tensor = struct { /// Broadcasts a Tensor to the given shape, adding axes at the end. pub fn broadcastRight(self: Tensor, output_shape: Shape) Tensor { - stdx.debug.assert(self.rank() <= output_shape.rank(), "broadcastRight expects tensor rank to be less than output tensor rank, got {} and {}", .{ self.rank(), output_shape.rank() }); + stdx.debug.assert(self.rank() <= output_shape.rank(), "broadcastRight expects tensor rank to be less than output tensor rank, got {d} and {d}", .{ self.rank(), output_shape.rank() }); if (self.rank() == output_shape.rank() and self._shape.eql(output_shape)) { return self; @@ -2022,7 +2021,7 @@ pub const Tensor = struct { pub fn broad(self: Tensor, other: Shape) Tensor { // TODO: broad is too restrictive because sometime you only want to specify one specific axis // Note: if you code below, make sure to update Shape.canBroadcastTo. - stdx.debug.assert(self._shape.canBroadcastTo(other), "Can't broadcast {} to {}", .{ self, other }); + stdx.debug.assert(self._shape.canBroadcastTo(other), "Can't broadcast {f} to {f}", .{ self, other }); // Already the right shape if (std.mem.eql(i64, self.dims(), other.dims())) return self; @@ -2036,7 +2035,7 @@ pub const Tensor = struct { } // check that each axis of self maps to an axis of other - var axes_: std.BoundedArray(i64, MAX_RANK) = .{}; + var axes_: stdx.BoundedArray(i64, MAX_RANK) = .{}; for (self._shape.tags()) |t| { axes_.appendAssumeCapacity(@intCast(other.axis(t))); } @@ -2047,14 +2046,14 @@ pub const Tensor = struct { pub fn reshape(self: Tensor, output_shape_: anytype) Tensor { const output_shape = self._shape.reshape(output_shape_); const tensor_type = mlirx.tensorType(self.getContext().mlirCtx(), output_shape); - const loc = self.getContext().location(@src(), "reshape({any})", .{output_shape}); + const loc = self.getContext().location(@src(), "reshape({f})", .{output_shape}); const reshape_value = dialect.stablehlo.reshape(self.getContext().mlirCtx(), self.value(), tensor_type, loc); return _result(output_shape, reshape_value.result(0)); } /// Converts the given 1 element Tensor into a 0-rank Tensor. pub fn asScalar(self: Tensor) Tensor { - stdx.debug.assert(self.count() == 1, "Tensor.asScalar expects an input with exactly 1-element got {}", .{self}); + stdx.debug.assert(self.count() == 1, "Tensor.asScalar expects an input with exactly 1-element got {f}", .{self}); return self.reshape(.{}); } @@ -2177,12 +2176,12 @@ pub const Tensor = struct { stdx.debug.assert(coord_axes_.len > 0, "gatherValues expects 1 or more axes to operate one, received none. Example: `x.gatherValues(.a, indices, .{{}})`", .{}); for (coord_axes_.constSlice(), 0..) |a, i| { if (i > 0) { - stdx.debug.assert(a == coord_axes_.get(i - 1) + 1, "gatherValues expects 'coord_axes' to be sequential. But {any} aren't sequential in {}", .{ coord_axes, self }); + stdx.debug.assert(a == coord_axes_.get(i - 1) + 1, "gatherValues expects 'coord_axes' to be sequential. But {any} aren't sequential in {f}", .{ coord_axes, self }); } } const AxisKind = enum { batching, offset, collapsed, indices }; - var self_kind: std.BoundedArray(AxisKind, MAX_RANK) = .{}; + var self_kind: stdx.BoundedArray(AxisKind, MAX_RANK) = .{}; var indices_batch_axes: Shape.DimsArray = .{}; for (self._shape.tags(), 0..self.rank()) |t, self_ax| { const maybe_coord_ax = std.mem.indexOfScalar(u3, coord_axes_.constSlice(), @intCast(self_ax)); @@ -2191,7 +2190,7 @@ pub const Tensor = struct { // Note: tags are required for batching. self_kind.appendAssumeCapacity(.batching); indices_batch_axes.appendAssumeCapacity(id_ax); - stdx.debug.assert(maybe_coord_ax == null, "gatherValues expects axes to appear at most twice. Axis {s} has been found both in 'self={any}', in 'coord_axes_={any}' and in 'indices={}'", .{ self._shape._tags.get(self_ax), self, coord_axes, indices }); + stdx.debug.assert(maybe_coord_ax == null, "gatherValues expects axes to appear at most twice. Axis {s} has been found both in 'self={f}', in 'coord_axes_={any}' and in 'indices={f}'", .{ self._shape._tags.get(self_ax), self, coord_axes, indices }); } else if (maybe_coord_ax) |_| { // for gatherValues we collapsed all gathered axes // (contrary to gatherSlices where we collapse none) @@ -2208,13 +2207,13 @@ pub const Tensor = struct { indices.rank() else blk: { const ax = indices._shape.hasTag(.coord) orelse indices._shape.axis(-1); - stdx.debug.assert(indices.dim(ax) == coord_axes_.len, "gatherValues with axes={any}, expects indices to be of shape [..., {}], got: {}", .{ coord_axes, coord_axes_.len, indices }); + stdx.debug.assert(indices.dim(ax) == coord_axes_.len, "gatherValues with axes={any}, expects indices to be of shape [..., {}], got: {f}", .{ coord_axes, coord_axes_.len, indices }); break :blk ax; }; // compute res shape var res_shape = Shape.init(.{}, self.dtype()); - var res_kind: std.BoundedArray(AxisKind, MAX_RANK) = .{}; + var res_kind: stdx.BoundedArray(AxisKind, MAX_RANK) = .{}; for (self_kind.constSlice(), 0..) |kind, ax_usize| { const ax: u3 = @intCast(ax_usize); if (ax == coord_axes_.get(0)) { @@ -2275,7 +2274,7 @@ pub const Tensor = struct { ); const mlir_shape = fromMlirValue(gather_op.result(0)).shape(); - stdx.debug.assert(mlir_shape.eql(res_shape), "gatherValues expects that batching indices appear in the same order in 'self' and 'indices', got: self={}, indices={}. You should transpose one or the other.", .{ self, indices }); + stdx.debug.assert(mlir_shape.eql(res_shape), "gatherValues expects that batching indices appear in the same order in 'self' and 'indices', got: self={f}, indices={f}. You should transpose one or the other.", .{ self, indices }); return _result(res_shape, gather_op.result(0)); } @@ -2347,29 +2346,29 @@ pub const Tensor = struct { /// and gatherSlices can copy data by group of C'*D elements. pub fn gatherSlices(self: Tensor, slice_shape_: anytype, indices: Tensor, opts: GatherOpts) Tensor { const slice_shape = if (@TypeOf(slice_shape_) == Shape) slice_shape_ else Shape.init(slice_shape_, .i32); - // scoped_log.debug("gatherSlice({}, {_}, {})", .{ self, slice_shape, indices }); + // scoped_log.debug("gatherSlice({}, {f}, {})", .{ self, slice_shape, indices }); const tagged_api = slice_shape.isFullyTagged(); if (tagged_api) { for (slice_shape.tags()) |t| { - stdx.debug.assert(self._shape.hasTag(t) != null, "gatherSlices expects `slices_shape` to only use tags from `self`. But {s} wasn't found in {}", .{ t, self }); + stdx.debug.assert(self._shape.hasTag(t) != null, "gatherSlices expects `slices_shape` to only use tags from `self`. But {s} wasn't found in {f}", .{ t, self }); } } else { // For untagged api, we require all slices to be specified. // Note: we could relax this and right align the slice. - stdx.debug.assert(slice_shape.rank() == self.rank(), "gatherSlices expects `slice_shape.rank()` to match `self.rank()`. Got: gatherSlices({}, slice={_}). To avoid specifying all axes in `slice_shape`, you can use tags.", .{ self, slice_shape }); + stdx.debug.assert(slice_shape.rank() == self.rank(), "gatherSlices expects `slice_shape.rank()` to match `self.rank()`. Got: gatherSlices({f}, slice={f}). To avoid specifying all axes in `slice_shape`, you can use tags.", .{ self, slice_shape }); } const index_coord_axis = indices._shape.hasTag(.coord) orelse indices._shape.axis(-1); - stdx.debug.assert(indices.dim(index_coord_axis) == slice_shape.rank(), "gatherSlices({}, slice={_}, indices) expects 'indices' to be a tensor [..., {}], got {}", .{ self, slice_shape, slice_shape.rank(), indices }); + stdx.debug.assert(indices.dim(index_coord_axis) == slice_shape.rank(), "gatherSlices({f}, slice={f}, indices) expects 'indices' to be a tensor [..., {}], got {f}", .{ self, slice_shape, slice_shape.rank(), indices }); // Compute result shape var res_shape = indices._shape.remove(index_coord_axis).withDtype(self.dtype()); var slice_dims = self._shape._dims; - var self_batch_axes: std.BoundedArray(i64, MAX_RANK) = .{}; - var indices_batch_axes: std.BoundedArray(i64, MAX_RANK) = .{}; - var start_index_map: std.BoundedArray(i64, MAX_RANK) = .{}; - var self_offset_axes: std.BoundedArray(i64, MAX_RANK) = .{}; + var self_batch_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; + var indices_batch_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; + var start_index_map: stdx.BoundedArray(i64, MAX_RANK) = .{}; + var self_offset_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; for (self._shape.tags(), 0..self.rank()) |t, self_ax| { const maybe_slice_ax: ?u3 = if (tagged_api) slice_shape.hasTag(t) else @intCast(self_ax); @@ -2379,12 +2378,12 @@ pub const Tensor = struct { self_batch_axes.appendAssumeCapacity(@intCast(self_ax)); indices_batch_axes.appendAssumeCapacity(indices._shape.axis(t)); slice_dims.set(self_ax, 1); - stdx.debug.assert(slice_shape.hasTag(t) == null, "gatherSlices expect axes to be either batches or slices axes. Axis {s} has been found both in `slices={_}` and `indices={}`", .{ t, slice_shape, indices }); + stdx.debug.assert(slice_shape.hasTag(t) == null, "gatherSlices expect axes to be either batches or slices axes. Axis {s} has been found both in `slices={f}` and `indices={f}`", .{ t, slice_shape, indices }); } else if (maybe_slice_ax) |slice_ax| { // Specified axes contains the start offset of the slices, // and are collected in `start_index_map`. const slice_dim = slice_shape.dim(slice_ax); - stdx.debug.assert(slice_dim <= self._shape.dim(self_ax), "gatherSlices expects `slice_shape` to be smaller than `self.shape()`. On axis {s}, got {} > {}.", .{ t, slice_shape, self._shape }); + stdx.debug.assert(slice_dim <= self._shape.dim(self_ax), "gatherSlices expects `slice_shape` to be smaller than `self.shape()`. On axis {s}, got {f} > {f}.", .{ t, slice_shape, self._shape }); slice_dims.set(self_ax, slice_dim); res_shape = res_shape.appendDim(slice_dim, t); start_index_map.appendAssumeCapacity(@intCast(self_ax)); @@ -2396,7 +2395,7 @@ pub const Tensor = struct { } } - const loc = self.getContext().location(@src(), "gatherSlices({_}, slice_shape={_}, idx={_})", .{ self, slice_shape, indices }); + const loc = self.getContext().location(@src(), "gatherSlices({f}, slice_shape={f}, idx={f})", .{ self, slice_shape, indices }); const gather_op = dialect.stablehlo.gather( self.getContext().mlirCtx(), self.value(), @@ -3172,7 +3171,7 @@ pub const Tensor = struct { /// Note: this doesn't support tagging, if you have tags, /// you should use `dynamicSlice` directly. pub fn dynamicSlice1d(self: Tensor, axis_: i8, slice_: DynSlice) Tensor { - stdx.debug.assert(slice_.start.rank() == 0, "dynamicSlice1d expects 'slice_.start' tensor rank to be a scalar, got {}", .{slice_.start}); + stdx.debug.assert(slice_.start.rank() == 0, "dynamicSlice1d expects 'slice_.start' tensor rank to be a scalar, got {f}", .{slice_.start}); const a = self.axis(axis_); const new_shape = self._shape.set(a, slice_.len); @@ -3226,17 +3225,17 @@ pub const Tensor = struct { const offset = slice_.start; const len = slice_.len; if (slices_tags.len == 0) { - stdx.debug.assert(self.rank() == slices.len, "dynamicSlice expects tensor rank and 'slices_' length to be equal, got {} and {}", .{ self.rank(), slices.len }); + stdx.debug.assert(self.rank() == slices.len, "dynamicSlice expects tensor rank and 'slices_' length to be equal, got {d} and {d}", .{ self.rank(), slices.len }); offset_values[i] = offset.value(); res_shape._dims.set(i, len); - stdx.debug.assert(len <= self.dim(i), "dynamicSlice expects slices 'len' to be less than or equal to their corresponding dimension in input tensor, got {} and {} for index {}", .{ len, self.dim(i), i }); + stdx.debug.assert(len <= self.dim(i), "dynamicSlice expects slices 'len' to be less than or equal to their corresponding dimension in input tensor, got {d} and {d} for index {d}", .{ len, self.dim(i), i }); } else { const t = slices_tags.get(i); - const a = res_shape.hasTag(t) orelse stdx.debug.panic("dynamicSlice expects input tensor to have tags used in 'slices_' but {s} is missing (input shape is {})", .{ t, self._shape }); + const a = res_shape.hasTag(t) orelse stdx.debug.panic("dynamicSlice expects input tensor to have tags used in 'slices_' but {s} is missing (input shape is {f})", .{ t, self._shape }); - stdx.debug.assert(len <= self.dim(a), "dynamicSlice expects slices 'len' to be less than their corresponding dimension in input tensor, got {} and {} for axis {s}", .{ len, self.dim(a), t }); + stdx.debug.assert(len <= self.dim(a), "dynamicSlice expects slices 'len' to be less than their corresponding dimension in input tensor, got {d} and {d} for axis {s}", .{ len, self.dim(a), t }); offset_values[a] = offset.value(); res_shape._dims.set(a, len); @@ -3304,14 +3303,14 @@ pub const Tensor = struct { if (tagged_api) { // Check that all update tags are known. for (update._shape._tags.constSlice()) |t| { - stdx.debug.assert(self._shape.hasTag(t) != null, "dynamicUpdateSlice expects 'update_' tensor tags to be a subset of input tensor tags but {s} is missing (input shape is {})", .{ t, self._shape }); + stdx.debug.assert(self._shape.hasTag(t) != null, "dynamicUpdateSlice expects 'update_' tensor tags to be a subset of input tensor tags but {s} is missing (input shape is {f})", .{ t, self._shape }); } var update_shape = self._shape; var prev_ax: i8 = -1; for (self._shape.tags(), 0..) |t, self_ax| { if (update._shape.hasTag(t)) |up_ax| { - stdx.debug.assert(up_ax == prev_ax + 1, "dynamicUpdateSlice expects 'update_' and input tensor axis to have the same order, got {} and {}. (hint: you need to explicitly transpose 'update_')", .{ update_, self }); + stdx.debug.assert(up_ax == prev_ax + 1, "dynamicUpdateSlice expects 'update_' and input tensor axis to have the same order, got {f} and {f}. (hint: you need to explicitly transpose 'update_')", .{ update_, self }); update_shape._dims.set(self_ax, update.dim(up_ax)); prev_ax = up_ax; @@ -3322,7 +3321,7 @@ pub const Tensor = struct { update = update.reshape(update_shape); } - stdx.debug.assert(self.rank() == update.rank(), "dynamicUpdateSlice expects input and computed update tensors to have the same rank, got {} and {}", .{ self, update }); + stdx.debug.assert(self.rank() == update.rank(), "dynamicUpdateSlice expects input and computed update tensors to have the same rank, got {f} and {f}", .{ self, update }); for (self.dims(), update.dims(), 0..) |self_d, up_d, ax| { const t = self._shape.debugTag(ax); @@ -3350,7 +3349,7 @@ pub const Tensor = struct { // This is only allowed when using tagged sliced. offset_values = .{zero} ** MAX_RANK; for (offset.constSlice(), offset_tags.constSlice()) |start, t| { - const a = self._shape.hasTag(t) orelse stdx.debug.panic("dynamicUpdateSlice expects input tensor to have tags used in 'offset_' but {s} is missing (input shape is {})", .{ t, self._shape }); + const a = self._shape.hasTag(t) orelse stdx.debug.panic("dynamicUpdateSlice expects input tensor to have tags used in 'offset_' but {s} is missing (input shape is {f})", .{ t, self._shape }); offset_values[a] = start.value(); } } @@ -3469,12 +3468,12 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise result of the given 'cmp' comparison between the two input Tensors. pub fn cmp(self: Tensor, direction: dialect.stablehlo.ComparisonDirection.Direction, other: Tensor) Tensor { - stdx.debug.assert(self.dtype() == other.dtype(), "cmp expects input tensors to be of the same type, got {} and {}", .{ self.dtype(), other.dtype() }); + stdx.debug.assert(self.dtype() == other.dtype(), "cmp expects input tensors to be of the same type, got {t} and {t}", .{ self.dtype(), other.dtype() }); if (self.rank() == 0 and other.rank() != 0) return self.broadcast(other._shape, &.{}).cmp(direction, other); if (self.rank() != 0 and other.rank() == 0) return self.cmp(direction, other.broadcast(self._shape, &.{})); - stdx.debug.assert(self._shape.eql(other._shape), "cmp expects input tensor shapes to match, got {} and {}", .{ self._shape, other._shape }); + stdx.debug.assert(self._shape.eql(other._shape), "cmp expects input tensor shapes to match, got {f} and {f}", .{ self._shape, other._shape }); const loc = self.getContext().location(@src(), "cmp(.{s})", .{@tagName(direction)}); const op = dialect.stablehlo.compare( @@ -3492,7 +3491,7 @@ pub const Tensor = struct { /// For each vector in the input tensor, /// creates a diagonal-matrix where diagonal values are set to the vector values. pub fn toDiagonal(self: Tensor, axis_: anytype, new_tags: [2]EnumLiteral) Tensor { - stdx.debug.assert(self.rank() < MAX_RANK - 1, "toDiagonal expects input up to {} rank, got {}", .{ MAX_RANK - 1, self }); + stdx.debug.assert(self.rank() < MAX_RANK - 1, "toDiagonal expects input up to {d} rank, got {f}", .{ MAX_RANK - 1, self }); const a = self.axis(axis_); const d = self.dim(a); var res_shape = self._shape; @@ -3622,8 +3621,8 @@ pub const Tensor = struct { return bool_tensor.select(on_true, on_false.broad(bool_tensor.shape())); } - stdx.debug.assert(bool_tensor._shape.eqlDims(on_true._shape), "select expects input tensor and 'on_true' tensor dimensions to match, got {} and {}", .{ bool_tensor._shape, on_true._shape }); - stdx.debug.assert(bool_tensor._shape.eqlDims(on_false._shape), "select expects input tensor and 'on_false' tensor dimensions to match, got {} and {}", .{ bool_tensor._shape, on_false._shape }); + stdx.debug.assert(bool_tensor._shape.eqlDims(on_true._shape), "select expects input tensor and 'on_true' tensor dimensions to match, got {f} and {f}", .{ bool_tensor._shape, on_true._shape }); + stdx.debug.assert(bool_tensor._shape.eqlDims(on_false._shape), "select expects input tensor and 'on_false' tensor dimensions to match, got {f} and {f}", .{ bool_tensor._shape, on_false._shape }); const loc = bool_tensor.getContext().mlirCtx().location(@src()); const op = dialect.stablehlo.select( @@ -3762,7 +3761,7 @@ pub const Tensor = struct { /// /// - res[a, b, c, d] == (A[a], B[b], C[c], D[d]) pub fn cartesianProductStacked(vectors: []const Tensor) Tensor { - var out = std.BoundedArray(Tensor, Tensor.MAX_RANK).init(vectors.len) catch unreachable; + var out = stdx.BoundedArray(Tensor, Tensor.MAX_RANK).init(vectors.len) catch unreachable; _cartesianProduct(vectors, out.slice()); return Tensor.stack(out.constSlice(), .last, .coord); @@ -3801,7 +3800,7 @@ pub const Tensor = struct { ) fn (Tensor, Tensor) Tensor { return struct { pub fn binaryOpHelper(self: Tensor, other: Tensor) Tensor { - stdx.debug.assert(self.dtype() == other.dtype(), "{s} expects tensor to be of same type, got {} and {}", .{ op_name, self, other }); + stdx.debug.assert(self.dtype() == other.dtype(), "{s} expects tensor to be of same type, got {f} and {f}", .{ op_name, self, other }); if (self.rank() == 0 and other.rank() != 0) { return binaryOpHelper(self.broad(other._shape), other); @@ -3811,10 +3810,10 @@ pub const Tensor = struct { return binaryOpHelper(self, other.broad(self._shape)); } - stdx.debug.assert(self._shape.eql(other._shape), "{s} expects tensor shapes to match, got {} and {}", .{ op_name, self._shape, other._shape }); + stdx.debug.assert(self._shape.eql(other._shape), "{s} expects tensor shapes to match, got {f} and {f}", .{ op_name, self._shape, other._shape }); const ctx = self.getContext(); - const location = ctx.location(src, "{s}({_}, {_})", .{ op_name, self, other }); + const location = ctx.location(src, "{s}({f}, {f})", .{ op_name, self, other }); const ret = @call(.auto, op_fn, .{ ctx.mlirCtx(), self.value(), other.value(), location }); return _result(self._shape, ret.result(0)); } @@ -3837,7 +3836,7 @@ pub const Tensor = struct { fn printCallback(_: ?*anyopaque, inputs: []const HostBuffer, outputs: []const HostBuffer) void { const host_buffer = inputs[0]; - std.log.defaultLog(.info, .zml, "Device buffer: {}: {}", .{ host_buffer.shape(), host_buffer.pretty() }); + std.log.defaultLog(.info, .zml, "Device buffer: {f}: {f}", .{ host_buffer.shape(), host_buffer.pretty() }); // This is true because of the operand aliases. // Since the result is already pointing to the input we don't need to modify the buffer. std.debug.assert(host_buffer._data == outputs[0]._data); @@ -4060,8 +4059,8 @@ test shapesOf { } } -pub fn _collectAxes(T: type, bounded_array: std.BoundedArray(T, Tensor.MAX_RANK), value: T) std.BoundedArray(i64, Tensor.MAX_RANK) { - var res: std.BoundedArray(i64, Tensor.MAX_RANK) = .{}; +pub fn _collectAxes(T: type, bounded_array: stdx.BoundedArray(T, Tensor.MAX_RANK), value: T) stdx.BoundedArray(i64, Tensor.MAX_RANK) { + var res: stdx.BoundedArray(i64, Tensor.MAX_RANK) = .{}; for (bounded_array.constSlice(), 0..) |v, ax| { if (v == value) { res.appendAssumeCapacity(@intCast(ax)); @@ -4070,12 +4069,12 @@ pub fn _collectAxes(T: type, bounded_array: std.BoundedArray(T, Tensor.MAX_RANK) return res; } -fn _parseGatherCoord(self: Tensor, axes_: anytype) struct { bool, std.BoundedArray(u3, Tensor.MAX_RANK) } { +fn _parseGatherCoord(self: Tensor, axes_: anytype) struct { bool, stdx.BoundedArray(u3, Tensor.MAX_RANK) } { const AxesT = @TypeOf(axes_); const axes_is_scalar = AxesT == EnumLiteral or AxesT == comptime_int or @typeInfo(AxesT) == .int; const coord_axes = if (axes_is_scalar) - std.BoundedArray(u3, Tensor.MAX_RANK).fromSlice(&.{self.axis(axes_)}) catch unreachable + stdx.BoundedArray(u3, Tensor.MAX_RANK).fromSlice(&.{self.axis(axes_)}) catch unreachable else self.axes(axes_); @@ -4099,7 +4098,7 @@ inline fn toI64(values: anytype) []i64 { } fn transposeIsJustAReshape(x: Shape, permutation: []const i64) bool { - var perm: std.BoundedArray(struct { u8, bool }, Tensor.MAX_RANK) = .{}; + var perm: stdx.BoundedArray(struct { u8, bool }, Tensor.MAX_RANK) = .{}; // Don't rewrite on invalid inputs. if (permutation.len > x.rank()) return false; for (permutation) |ax| { diff --git a/zml/test_runner.zig b/zml/test_runner.zig index ed8ee113..8b8d6555 100644 --- a/zml/test_runner.zig +++ b/zml/test_runner.zig @@ -31,7 +31,7 @@ pub fn asyncMain() !void { .root_name = "Test", .estimated_total_items = test_fn_list.len, }); - const have_tty = std.io.getStdErr().isTty(); + const have_tty = std.fs.File.stderr().isTty(); var args = std.process.args(); // Skip executable path diff --git a/zml/testing.zig b/zml/testing.zig index 0dcf3941..b236400b 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -51,10 +51,10 @@ pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { if (should_free_left) left.deinit(allocator); if (should_free_right) right.deinit(allocator); } - errdefer log.err("\n--> Left: {}\n--> Right: {}", .{ left.pretty(), right.pretty() }); + errdefer log.err("\n--> Left: {f}\n--> Right: {f}", .{ left.pretty(), right.pretty() }); if (!std.mem.eql(i64, left.shape().dims(), right.shape().dims())) { - log.err("left.shape() {} != right.shape() {}", .{ left.shape(), right.shape() }); + log.err("left.shape() {f} != right.shape() {f}", .{ left.shape(), right.shape() }); return error.TestUnexpectedResult; } if (left.dtype() != right.dtype() and !(left.dtype() == .f16 and right.dtype() == .bf16)) { @@ -89,7 +89,7 @@ pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { const right_data = right.items(R); for (left_data, right_data, 0..) |l, r, i| { if (!approxEq(f32, zml.floats.floatCast(f32, l), zml.floats.floatCast(f32, r), tolerance)) { - log.err("left.data != right_data.\n < {d:.3} \n > {d:.3}\n error at idx {d}: {d:.3} != {d:.3}", .{ center(left_data, i), center(right_data, i), i, left_data[i], right_data[i] }); + log.err("left.data != right_data.\n < {any:.3} \n > {any:.3}\n error at idx {any}: {any:.3} != {any:.3}", .{ center(left_data, i), center(right_data, i), i, left_data[i], right_data[i] }); return error.TestUnexpectedResult; } } @@ -108,7 +108,7 @@ pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { pub fn expectEqualShapes(expected: zml.Shape, actual: zml.Shape) error{TestExpectedEqual}!void { if (expected.eqlWithTags(actual)) return; - std.debug.print("Expected {}, got {}", .{ expected, actual }); + std.debug.print("Expected {f}, got {f}", .{ expected, actual }); return error.TestExpectedEqual; } diff --git a/zml/tokenizer/BUILD.bazel b/zml/tokenizer/BUILD.bazel index 4489d774..9b2f9c71 100644 --- a/zml/tokenizer/BUILD.bazel +++ b/zml/tokenizer/BUILD.bazel @@ -9,6 +9,7 @@ zig_library( deps = [ "//async", "//ffi:zig", + "//stdx", "//zml/tokenizer/hftokenizers", "//zml/tokenizer/sentencepiece", ], diff --git a/zml/tokenizer/hftokenizers/BUILD.bazel b/zml/tokenizer/hftokenizers/BUILD.bazel index 54a01c8b..a07d855b 100644 --- a/zml/tokenizer/hftokenizers/BUILD.bazel +++ b/zml/tokenizer/hftokenizers/BUILD.bazel @@ -27,5 +27,6 @@ zig_library( deps = [ ":hftokenizers_cc", "//ffi:zig", + "//stdx", ], ) diff --git a/zml/tokenizer/hftokenizers/hftokenizers.zig b/zml/tokenizer/hftokenizers/hftokenizers.zig index 538a25a1..7b36dbe8 100644 --- a/zml/tokenizer/hftokenizers/hftokenizers.zig +++ b/zml/tokenizer/hftokenizers/hftokenizers.zig @@ -1,6 +1,8 @@ const std = @import("std"); + const c = @import("c"); const ffi = @import("ffi"); +const stdx = @import("stdx"); pub const Encoder = struct { inner: *HFTokenizer, @@ -33,8 +35,8 @@ pub const Encoder = struct { }; pub const Decoder = struct { - const StringBuffer = std.BoundedArray(u8, 128); - const TokensIdsBuffer = std.BoundedArray(u32, 4); + const StringBuffer = stdx.BoundedArray(u8, 128); + const TokensIdsBuffer = stdx.BoundedArray(u32, 4); inner: *HFTokenizer, current_string: ?[]const u8 = null, diff --git a/zml/tokenizer/homemade.zig b/zml/tokenizer/homemade.zig index 98f574c0..9b5898d8 100644 --- a/zml/tokenizer/homemade.zig +++ b/zml/tokenizer/homemade.zig @@ -2,10 +2,11 @@ //! Disclaimer this is not a very robust implementation: //! In particular the normalization is pretty minimalist, only works with ascii, and don't do unicode normalization. //! Mostly used for testing models that don't have an official HF/sentencepiece tokenizer. -const builtin = @import("builtin"); const std = @import("std"); - const testing = std.testing; +const builtin = @import("builtin"); + +const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/tokenizer"); @@ -87,12 +88,11 @@ pub const Tokenizer = struct { } /// Reads a new word directly into the tokenizer arena. - pub fn readTokenInto(self: *Tokenizer, score: f32, len: usize, tok_reader: anytype) !void { + pub fn readTokenInto(self: *Tokenizer, score: f32, len: usize, tok_reader: *std.Io.Reader) !void { const arena = self.arena_state.allocator(); const token = try arena.alloc(u8, len); - const n = try tok_reader.readAll(token); - std.debug.assert(n == len); + try tok_reader.readSliceAll(token); return self.addOwnedToken(score, token); } @@ -190,9 +190,9 @@ pub const Tokenizer = struct { if (options.debug) { var _debug_buf: [256]u8 = undefined; var _debug_alloc = std.heap.FixedBufferAllocator.init(&_debug_buf); - var debug_progress = std.ArrayList(u8).init(_debug_alloc.allocator()); + var debug_progress = std.array_list.Managed(u8).init(_debug_alloc.allocator()); self.decodeWithOpts(&debug_progress, tok_buff[0..num_tokens], .{ .sep = "|" }) catch {}; - log.debug("tokens: {d} -> {s}", .{ tok_buff[0..num_tokens], debug_progress.items }); + log.debug("tokens: {any} -> {s}", .{ tok_buff[0..num_tokens], debug_progress.items }); } var best_score: f32 = -1e10; var best_token: u32 = 0; @@ -312,7 +312,7 @@ pub const Tokenizer = struct { /// Note that if the tokenizer allows sub-unicode bytes, it's possible /// the output is not valid utf8. pub fn decode(self: *const Tokenizer, allocator: std.mem.Allocator, input: []const u32) error{OutOfMemory}![]u8 { - var output = std.ArrayList(u8).init(allocator); + var output = std.array_list.Managed(u8).init(allocator); errdefer output.deinit(); try self.decodeWithOpts(&output, input, .{}); @@ -321,7 +321,7 @@ pub const Tokenizer = struct { pub fn decodeWithOpts( self: *const Tokenizer, - output: *std.ArrayList(u8), + output: *std.array_list.Managed(u8), input: []const u32, opts: struct { sep: []const u8 = "" }, ) error{OutOfMemory}!void { @@ -363,7 +363,8 @@ pub const Tokenizer = struct { // First lookup the byte fallback entry. // Note: we assume upper case, but we could try both upper and lower case if needed. - _ = std.fmt.bufPrintIntToSlice(byte_fallback_buf[3..5], c, 16, .upper, .{ .fill = '0', .width = 2 }); + var writer: std.Io.Writer = .fixed(byte_fallback_buf[3..5]); + try writer.printInt(c, 16, .upper, .{ .fill = '0', .width = 2 }); const entry = tokenizer.token_lookup.getEntry(&byte_fallback_buf) orelse { log.err("Tokenizer has \"byte_fallback\" = true, but doesn't contains the byte fallback token {s}", .{byte_fallback_buf}); return error.InvalidInput; @@ -443,8 +444,8 @@ pub const Encoder = struct { }; pub const Decoder = struct { - const StringBuffer = std.BoundedArray(u8, 128); - const TokensIdsBuffer = std.BoundedArray(u32, 4); + const StringBuffer = stdx.BoundedArray(u8, 128); + const TokensIdsBuffer = stdx.BoundedArray(u32, 4); inner: *Tokenizer, arena: std.heap.ArenaAllocator, @@ -571,7 +572,7 @@ test CharTokenIterator { { tokenizer.byte_fallback = false; var it: CharTokenIterator = .{ .input = "ζℳL" }; - var res: std.BoundedArray(u32, 8) = .{}; + var res: stdx.BoundedArray(u32, 8) = .{}; while (try it.nextCodepointToken(&tokenizer)) |token| { res.appendAssumeCapacity(token); } @@ -582,7 +583,7 @@ test CharTokenIterator { { tokenizer.byte_fallback = true; var it: CharTokenIterator = .{ .input = "ζℳL" }; - var res: std.BoundedArray(u32, 8) = .{}; + var res: stdx.BoundedArray(u32, 8) = .{}; while (try it.nextCodepointToken(&tokenizer)) |token| { res.appendAssumeCapacity(token); } @@ -596,7 +597,7 @@ pub const Normalizer = struct { /// Space token used by sentencepiece derived tokenizer. pub const sentencepiece_space = "▁"; // \xe2\x96\x81 - _whitespace: std.BoundedArray(u8, 8) = .{}, + _whitespace: stdx.BoundedArray(u8, 8) = .{}, flags: packed struct { remove_extra_whitespaces: bool, @@ -610,7 +611,7 @@ pub const Normalizer = struct { split_on_punct_ascii: bool, }, - pub fn init(flags: std.meta.FieldType(Normalizer, .flags), escaped_whitespace: ?[]const u8) Normalizer { + pub fn init(flags: @FieldType(Normalizer, "flags"), escaped_whitespace: ?[]const u8) Normalizer { var res: Normalizer = .{ .flags = flags }; if (escaped_whitespace) |escaped| { res._whitespace.appendSliceAssumeCapacity(escaped); @@ -622,7 +623,7 @@ pub const Normalizer = struct { return if (self._whitespace.len > 1) self._whitespace.constSlice() else null; } - fn addSlice(data: []const u8, consumed: usize, normalized: *std.ArrayList(u8), normalized_to_origin: *std.ArrayList(usize)) !void { + fn addSlice(data: []const u8, consumed: usize, normalized: *std.array_list.Managed(u8), normalized_to_origin: *std.array_list.Managed(usize)) !void { try normalized.appendSlice(data); for (data) |_| try normalized_to_origin.append(consumed); } @@ -672,9 +673,9 @@ pub const Normalizer = struct { // Pre-allocate outputs const space = self.escapedSpace() orelse " "; const overhead = if (self.flags.split_on_punct_ascii) space.len + 1 else space.len; - var normalized = try std.ArrayList(u8).initCapacity(allocator, trimmed_input.len * overhead + 2 * space.len); + var normalized = try std.array_list.Managed(u8).initCapacity(allocator, trimmed_input.len * overhead + 2 * space.len); errdefer normalized.deinit(); - var normalized_to_origin = try std.ArrayList(usize).initCapacity(allocator, normalized.capacity); + var normalized_to_origin = try std.array_list.Managed(usize).initCapacity(allocator, normalized.capacity); errdefer normalized_to_origin.deinit(); // If the spec asks for it, add a whitespace at the beginning. @@ -965,7 +966,7 @@ test Normalizer { /// This implementation precompupte a mapping between bytes encoded with GPT2 algorithm, /// into utf8 bytes, and do lookups at runtime. pub const Gpt2TextDecoder = struct { - const Code = std.BoundedArray(u8, 2); + const Code = stdx.BoundedArray(u8, 2); // TODO: benchmark this is more efficient than doing the conversion at runtime. code_to_byte: std.AutoArrayHashMap(Code, u8), @@ -982,7 +983,7 @@ pub const Gpt2TextDecoder = struct { var code: Code = .{ .buffer = .{ 0, 0 }, .len = 0 }; // 0-init const i: u8 = @intCast(index); if (isPrintableByte(i)) { - if (std.ascii.isASCII(i)) { + if (std.ascii.isAscii(i)) { code.appendAssumeCapacity(i); } else { const codepoint: u21 = @as(u21, @intCast(i)); @@ -1005,7 +1006,7 @@ pub const Gpt2TextDecoder = struct { /// Transform bytes representing text under the gpt2 encoding, /// and write to the `unicode` buffer utf-8 bytes. - pub fn decode(self: Gpt2TextDecoder, unicode: *std.ArrayList(u8), bytes: []const u8) ![]const u8 { + pub fn decode(self: Gpt2TextDecoder, unicode: *std.array_list.Managed(u8), bytes: []const u8) ![]const u8 { const start = unicode.items.len; var it = std.unicode.Utf8Iterator{ .i = 0, .bytes = bytes }; while (it.nextCodepointSlice()) |codepoint| { @@ -1029,7 +1030,7 @@ test Gpt2TextDecoder { var decoder = try Gpt2TextDecoder.init(testing.allocator); defer decoder.deinit(); - var out = std.ArrayList(u8).init(testing.allocator); + var out = std.array_list.Managed(u8).init(testing.allocator); defer out.deinit(); // Ascii is not changed. @@ -1076,7 +1077,7 @@ pub fn fromHfJson(allocator: std.mem.Allocator, tokenizer_path: []const u8) !Tok // Buffer containing all concatenated tokens. // Reserve a big chunk, to avoid grow event, but release over-allocated memory. - var all_tokens = try std.ArrayList(u8).initCapacity(tokenizer.arena_state.allocator(), file_content.len); + var all_tokens = try std.array_list.Managed(u8).initCapacity(tokenizer.arena_state.allocator(), file_content.len); const original_alloc = all_tokens.items.ptr; // A re-alloc event here means we have invalidated all slices inside the tokenizer. // If this is too annoying we could switch to a custom type instead of slices. @@ -1164,7 +1165,7 @@ pub fn fromHfJson(allocator: std.mem.Allocator, tokenizer_path: []const u8) !Tok } /// Returns a copy of the given string, stored inside the given ArrayList. -fn dup(buffer: *std.ArrayList(u8), str: []const u8) ![]const u8 { +fn dup(buffer: *std.array_list.Managed(u8), str: []const u8) ![]const u8 { const n = buffer.items.len; try buffer.appendSlice(str); return buffer.items[n..]; @@ -1175,7 +1176,7 @@ fn objectGet( object: std.json.ObjectMap, comptime kind: std.meta.FieldEnum(std.json.Value), key: []const u8, -) ?std.meta.FieldType(std.json.Value, kind) { +) ?@FieldType(std.json.Value, @tagName(kind)) { const val = object.get(key) orelse return null; if (val != kind) return null; return @field(val, @tagName(kind)); @@ -1184,10 +1185,11 @@ fn objectGet( pub fn fromTinyLlamaFile(allocator: std.mem.Allocator, tokenizer_path: []const u8, vocab_size: u32) !Tokenizer { const tokenizer_file = try std.fs.cwd().openFile(tokenizer_path, .{}); defer tokenizer_file.close(); - var tok_reader = std.io.bufferedReader(tokenizer_file.reader()); - const r = tok_reader.reader(); + var read_buff: [4096]u8 = undefined; + var tok_reader = tokenizer_file.reader(&read_buff); + const r: *std.Io.Reader = &tok_reader.interface; - const max_token_len = try r.readInt(u32, .little); + const max_token_len = try readValueLE(u32, r); const special_tokens: Tokenizer.SpecialTokens = .{ .unk = 0, .bos = 1, @@ -1195,11 +1197,11 @@ pub fn fromTinyLlamaFile(allocator: std.mem.Allocator, tokenizer_path: []const u }; var tokenizer = try Tokenizer.init(allocator, vocab_size, max_token_len, null, special_tokens, true); var i: u32 = 0; - while (readToken(&tokenizer, &r)) : (i += 1) { + while (readToken(&tokenizer, r)) : (i += 1) { // Pass } else |_| { if (i < vocab_size) { - log.info("Read {d} words out of {?d}", .{ i, vocab_size }); + log.info("Read {d} words out of {d}", .{ i, vocab_size }); } tokenizer.vocab_size = i; } @@ -1207,8 +1209,14 @@ pub fn fromTinyLlamaFile(allocator: std.mem.Allocator, tokenizer_path: []const u return tokenizer; } -fn readToken(tokenizer: *Tokenizer, tok_reader: anytype) !void { - const score: f32 = @bitCast(try tok_reader.readInt(u32, .little)); - const len: usize = @intCast(try tok_reader.readInt(u32, .little)); +fn readToken(tokenizer: *Tokenizer, tok_reader: *std.Io.Reader) !void { + const score: f32 = try readValueLE(f32, tok_reader); + const len: usize = try readValueLE(u32, tok_reader); try tokenizer.readTokenInto(score, len, tok_reader); } + +fn readValueLE(T: type, reader: *std.Io.Reader) !T { + var res: [1]T = undefined; + try reader.readSliceEndian(T, &res, .little); + return res[0]; +} diff --git a/zml/tokenizer/main.zig b/zml/tokenizer/main.zig index d3554f1d..b6f91dc2 100644 --- a/zml/tokenizer/main.zig +++ b/zml/tokenizer/main.zig @@ -1,10 +1,11 @@ const std = @import("std"); -const log = std.log.scoped(.@"//zml/tokenizer"); const asynk = @import("async"); const stdx = @import("stdx"); const zml_tokenizer = @import("zml/tokenizer"); +const log = std.log.scoped(.@"//zml/tokenizer"); + const Flags = struct { tokenizer: []const u8, prompt: []const u8, @@ -35,7 +36,7 @@ pub fn asyncMain() !void { const prompt_tok = try encoder.encode(args.prompt); - log.info("Input: {s}\nOutput: {d}", .{ args.prompt, prompt_tok }); + log.info("Input: {s}\nOutput: {any}", .{ args.prompt, prompt_tok }); var errors: u8 = 0; { @@ -47,14 +48,14 @@ pub fn asyncMain() !void { } if (args.expected.len > 0) { - var expected = try std.ArrayList(u32).initCapacity(allocator, args.prompt.len); + var expected = try std.array_list.Managed(u32).initCapacity(allocator, args.prompt.len); var it = std.mem.splitSequence(u8, args.expected, ","); while (it.next()) |int_token| { const tok = try std.fmt.parseInt(u32, int_token, 10); try expected.append(tok); } if (!std.mem.eql(u32, expected.items, prompt_tok)) { - log.err("Doesn't match expected: {d}", .{expected.items}); + log.err("Doesn't match expected: {any}", .{expected.items}); errors += 1; } } diff --git a/zml/tokenizer/sentencepiece/BUILD.bazel b/zml/tokenizer/sentencepiece/BUILD.bazel index 354b289a..dba86ddb 100644 --- a/zml/tokenizer/sentencepiece/BUILD.bazel +++ b/zml/tokenizer/sentencepiece/BUILD.bazel @@ -19,5 +19,6 @@ zig_library( deps = [ ":sentencepiece_swig", "//ffi:zig", + "//stdx", ], ) diff --git a/zml/tokenizer/sentencepiece/sentencepiece.zig b/zml/tokenizer/sentencepiece/sentencepiece.zig index 7f4edc69..c3623cb5 100644 --- a/zml/tokenizer/sentencepiece/sentencepiece.zig +++ b/zml/tokenizer/sentencepiece/sentencepiece.zig @@ -1,6 +1,8 @@ const std = @import("std"); + const c = @import("c"); const ffi = @import("ffi"); +const stdx = @import("stdx"); const StringToTokenRatio = 3; @@ -81,7 +83,7 @@ pub const Encoder = struct { pub const Decoder = struct { const StringBufferSize = 64; - const StringBuffer = std.BoundedArray(u8, StringBufferSize); + const StringBuffer = stdx.BoundedArray(u8, StringBufferSize); const TokenIdsBufferSize = 4; inner: *SentencePieceProcessor, diff --git a/zml/torch.zig b/zml/torch.zig index 5889353d..e982aaec 100644 --- a/zml/torch.zig +++ b/zml/torch.zig @@ -14,7 +14,7 @@ const Tensor = zml.Tensor; /// * `matmul(.{10}, .{10}) -> .{}` /// * `matmul(.{10}, .{10}) -> .{}` pub fn matmul(lhs: Tensor, rhs: Tensor) Tensor { - stdx.debug.assert(lhs.rank() >= 1 and rhs.rank() >= 1, "Can't matmul({}, {}) ! The two tensors need to have at least rank 1.", .{ lhs, rhs }); + stdx.debug.assert(lhs.rank() >= 1 and rhs.rank() >= 1, "Can't matmul({f}, {f}) ! The two tensors need to have at least rank 1.", .{ lhs, rhs }); const contracting = [_][2]i8{.{ -1, if (rhs.rank() >= 2) rhs.rank() - 2 else 0 }}; if (lhs.rank() == 1 or rhs.rank() <= 2) { @@ -22,7 +22,7 @@ pub fn matmul(lhs: Tensor, rhs: Tensor) Tensor { return lhs.dotGeneral(rhs, &contracting, &.{}); } - stdx.debug.assert(lhs.rank() == 2, "Can't matmul({}, {}) ! One of the two tensors need to have a rank less than 2.", .{ lhs, rhs }); + stdx.debug.assert(lhs.rank() == 2, "Can't matmul({f}, {f}) ! One of the two tensors need to have a rank less than 2.", .{ lhs, rhs }); // Pytorch treats the extra dimensions of rhs has batching dimensions, // and implicitly broadcast lhs along those. @@ -91,7 +91,7 @@ pub fn unsqueeze( self: Tensor, axis_: anytype, ) Tensor { - stdx.debug.assert(self.rank() < Tensor.MAX_RANK - 1, "Can't unsqueeze {}, it's already at max rank.", .{self}); + stdx.debug.assert(self.rank() < Tensor.MAX_RANK - 1, "Can't unsqueeze {f}, it's already at max rank.", .{self}); const a = switch (@typeInfo(@TypeOf(axis_))) { .int, .comptime_int => if (axis_ < 0) @as(i8, self.rank()) + 1 + axis_ @@ -125,9 +125,9 @@ test unsqueeze { /// ref: https://pytorch.org/docs/stable/generated/torch.nn.PixelShuffle.html#pixelshuffle pub fn pixelShuffle(tensor: Tensor, upscale_factor: u32) Tensor { const shape = tensor.shape(); - stdx.debug.assert(shape.hasTags(.{ .c, .w, .h }), "pixelShuffle({}) is invalide. Missing tags {{.c, .w, .h}}", .{tensor}); + stdx.debug.assert(shape.hasTags(.{ .c, .w, .h }), "pixelShuffle({f}) is invalide. Missing tags {{.c, .w, .h}}", .{tensor}); - stdx.debug.assert(@mod(shape.dim(.c), upscale_factor * upscale_factor) == 0, "pixelShuffle({}) is invalide. Number of channels {}, isn't divisible by upscale factor {}**2", .{ tensor, shape.dim(.c), upscale_factor }); + stdx.debug.assert(@mod(shape.dim(.c), upscale_factor * upscale_factor) == 0, "pixelShuffle({f}) is invalide. Number of channels {}, isn't divisible by upscale factor {}**2", .{ tensor, shape.dim(.c), upscale_factor }); const s = tensor.splitAxis(.c, .{ .c = -1, .upscale_h = upscale_factor, .upscale_w = upscale_factor }); const perm = s.shape().contiguousPerm(.{ .h, .upscale_h, .w, .upscale_w }); From 8994a37792c8f03ab00fc000349336885fc0f89a Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 2 Sep 2025 15:16:09 +0200 Subject: [PATCH 024/881] bump runtimes/* code for 0.15 (#318) Fix #316, close #304 - Bump platform specific Zig code that doesn't get built in CI. - Restore pytorch loader, using `std.fs.File` instead of `async.File`, allowing to reuse `std.zip` - Update `zig fmt` in CI to `0.15.1` and ran it on the codebase (third_party excluded) - Remove `stdx.io` which was pointing to remove `std.io.BufferededReader`. The Neuron target is still broken though: `bazel build //runtimes/neuron:libneuronxla` As well as the Cuda plugin in `--config=debug` mode: `bazel build --config=debug //runtimes/cuda:zmlxcuda` --- .github/workflows/commit-framework.yaml | 4 +- .github/workflows/commit.yaml | 4 +- mlir/mlir.zig | 4 +- pjrt/ffi.zig | 6 +- pjrt/pjrt.zig | 8 +- runtimes/cuda/cuda.zig | 2 +- runtimes/neuron/libneuronxla.zig | 2 +- runtimes/neuron/neuron.zig | 8 +- runtimes/rocm/rocm.zig | 2 +- runtimes/tpu/tpu.zig | 16 +- stdx/BUILD.bazel | 1 - stdx/io.zig | 4 - stdx/stdx.zig | 1 - upb/upb.zig | 2 +- zml/BUILD.bazel | 9 + zml/aio.zig | 12 +- zml/aio/torch.zig | 42 + zml/aio/torch/eval.zig | 436 +++++++++ zml/aio/torch/file.zig | 526 +++++++++++ zml/aio/torch/pickle.zig | 1066 +++++++++++++++++++++++ zml/aio/torch/py.zig | 430 +++++++++ zml/aio/torch/simple.pt | Bin 0 -> 2708 bytes zml/aio/torch/simple_test_4.pickle | Bin 0 -> 130 bytes zml/hostbuffer.zig | 2 +- zml/module.zig | 2 +- zml/platform.zig | 4 +- zml/tools/tracer.zig | 11 +- 27 files changed, 2557 insertions(+), 47 deletions(-) delete mode 100644 stdx/io.zig create mode 100644 zml/aio/torch.zig create mode 100644 zml/aio/torch/eval.zig create mode 100644 zml/aio/torch/file.zig create mode 100644 zml/aio/torch/pickle.zig create mode 100644 zml/aio/torch/py.zig create mode 100644 zml/aio/torch/simple.pt create mode 100644 zml/aio/torch/simple_test_4.pickle diff --git a/.github/workflows/commit-framework.yaml b/.github/workflows/commit-framework.yaml index 1603e847..141630fe 100644 --- a/.github/workflows/commit-framework.yaml +++ b/.github/workflows/commit-framework.yaml @@ -30,11 +30,11 @@ jobs: - name: Setup Zig uses: mlugg/setup-zig@v2 with: - version: 0.14.1 + version: 0.15.1 mirror: https://mirror.zml.ai/zig - name: Zig Format - run: zig fmt --check $(git ls-files --exclude-standard -- '*.zig' | xargs) + run: zig fmt --check $(git ls-files --exclude-standard -- ':!third_party/*' '*.zig' | xargs) - name: No autofix in Zig code run: bash -c "! git grep -l '// autofix' -- '*.zig'" diff --git a/.github/workflows/commit.yaml b/.github/workflows/commit.yaml index b6e7148d..1ce4b5c9 100644 --- a/.github/workflows/commit.yaml +++ b/.github/workflows/commit.yaml @@ -37,11 +37,11 @@ jobs: - name: Setup Zig uses: mlugg/setup-zig@v2 with: - version: 0.14.1 + version: 0.15.1 mirror: https://mirror.zml.ai/zig - name: Zig Format - run: zig fmt --check $(git ls-files --exclude-standard -- '*.zig' | xargs) + run: zig fmt --check $(git ls-files --exclude-standard -- ':!third_party/*' '*.zig' | xargs) - name: No autofix in Zig code run: bash -c "! git grep -l '// autofix' -- '*.zig'" diff --git a/mlir/mlir.zig b/mlir/mlir.zig index a222c40a..a6721914 100755 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -605,7 +605,7 @@ pub fn DenseElementsAttribute(comptime dt: DenseElementsAttributeTypes) type { pub fn items(self: Attr) []const dt.ZigType() { const raw_bytes: [*]const u8 = c.mlirDenseElementsAttrGetRawData(self._inner) orelse unreachable; - const ptr: [*]const dt.ZigType() = @alignCast(@ptrCast(raw_bytes)); + const ptr: [*]const dt.ZigType() = @ptrCast(@alignCast(raw_bytes)); // Note the mlir API returns us the number of elements, not the number of bytes, // that's why we track the element type at comptime to allow items to work. return ptr[0..self.len()]; @@ -1743,7 +1743,7 @@ pub const helpers = struct { writer: *std.Io.Writer, err: ?std.Io.Writer.Error = null, fn printCallback(mlir_str: c.MlirStringRef, opaque_ctx: ?*anyopaque) callconv(.c) void { - var ctx: *@This() = @alignCast(@ptrCast(opaque_ctx)); + var ctx: *@This() = @ptrCast(@alignCast(opaque_ctx)); if (ctx.err) |_| return; _ = ctx.writer.write(mlir_str.data[0..mlir_str.length]) catch |err| { ctx.err = err; diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index b79a789f..d2517a04 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -359,7 +359,7 @@ pub const Attrs = extern struct { value: *const anyopaque, pub fn get(self: Scalar, T: type) T { - const ptr: *const T = @alignCast(@ptrCast(self.value)); + const ptr: *const T = @ptrCast(@alignCast(self.value)); return ptr.*; } }; @@ -370,13 +370,13 @@ pub const Attrs = extern struct { data: [*]const u8, pub fn slice(self: Array, T: type) []const T { - const ptr: [*]const T = @alignCast(@ptrCast(self.data)); + const ptr: [*]const T = @ptrCast(@alignCast(self.data)); return ptr[0..self.len]; } }; pub fn slice(self: Array, T: type) []const T { - const ptr: [*]const T = @alignCast(@ptrCast(self.data)); + const ptr: [*]const T = @ptrCast(@alignCast(self.data)); return ptr[0..self.len]; } diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 87f3d174..24581496 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -58,7 +58,7 @@ pub const ApiError = error{ fn InnerMixin(comptime innerT: type) type { return struct { fn inner(self: anytype) *innerT { - return @ptrCast(@constCast(@alignCast(self))); + return @ptrCast(@alignCast(@constCast(self))); } }; } @@ -125,10 +125,10 @@ pub const Api = struct { } pub fn lookupExtension(self: *const Api, comptime ExtensionT: type, ext_id: c_int) ?*const ExtensionT { - var cur: [*c]const c.PJRT_Extension_Base = @alignCast(@ptrCast(self.inner.extension_start)); + var cur: [*c]const c.PJRT_Extension_Base = @ptrCast(@alignCast(self.inner.extension_start)); while (cur != null) : (cur = cur.*.next) { if (cur.*.type == ext_id) { - return @alignCast(@ptrCast(cur)); + return @ptrCast(@alignCast(cur)); } } @@ -432,7 +432,7 @@ pub const Client = opaque { .client = self.inner(), }) catch unreachable; if (ret.addressable_memories) |memories| { - return @constCast(@ptrCast(memories[0..ret.num_addressable_memories])); + return @ptrCast(@constCast(memories[0..ret.num_addressable_memories])); } return &.{}; } diff --git a/runtimes/cuda/cuda.zig b/runtimes/cuda/cuda.zig index c07c0e91..42b3b5f4 100644 --- a/runtimes/cuda/cuda.zig +++ b/runtimes/cuda/cuda.zig @@ -28,7 +28,7 @@ fn hasCudaPathInLDPath() bool { fn setupXlaGpuCudaDirFlag(allocator: std.mem.Allocator, sandbox: []const u8) !void { const xla_flags = std.process.getEnvVarOwned(allocator, "XLA_FLAGS") catch ""; - const new_xla_flagsZ = try std.fmt.allocPrintZ(allocator, "{s} --xla_gpu_cuda_data_dir={s}", .{ xla_flags, sandbox }); + const new_xla_flagsZ = try std.fmt.allocPrintSentinel(allocator, "{s} --xla_gpu_cuda_data_dir={s}", .{ xla_flags, sandbox }, 0); _ = c.setenv("XLA_FLAGS", new_xla_flagsZ, 1); } diff --git a/runtimes/neuron/libneuronxla.zig b/runtimes/neuron/libneuronxla.zig index ae5710ee..0010d28e 100644 --- a/runtimes/neuron/libneuronxla.zig +++ b/runtimes/neuron/libneuronxla.zig @@ -38,7 +38,7 @@ var module_def: c.PyModuleDef = .{ .{}, }), .m_slots = @constCast(&[_]c.PyModuleDef_Slot{ - .{ .slot = c.Py_mod_exec, .value = @constCast(@ptrCast(&module_exec)) }, + .{ .slot = c.Py_mod_exec, .value = @ptrCast(@constCast(&module_exec)) }, .{}, }), .m_traverse = null, diff --git a/runtimes/neuron/neuron.zig b/runtimes/neuron/neuron.zig index 09d6a218..4d2efa1e 100644 --- a/runtimes/neuron/neuron.zig +++ b/runtimes/neuron/neuron.zig @@ -25,10 +25,10 @@ fn isRunningOnEC2() !bool { var f = try asynk.File.open("/sys/devices/virtual/dmi/id/sys_vendor", .{ .mode = .read_only }); defer f.close() catch {}; - var buf: [AmazonEC2.len]u8 = undefined; - _ = try f.reader().readAll(&buf); + var content: [AmazonEC2.len]u8 = undefined; + const n_read = try f.pread(&content, 0); - return std.mem.eql(u8, &buf, AmazonEC2); + return std.mem.eql(u8, content[0..n_read], AmazonEC2); } pub fn load() !*const pjrt.Api { @@ -45,7 +45,7 @@ pub fn load() !*const pjrt.Api { return error.Unavailable; } - var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); + var arena = std.heap.ArenaAllocator.init(std.heap.smp_allocator); defer arena.deinit(); var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator() }) orelse { diff --git a/runtimes/rocm/rocm.zig b/runtimes/rocm/rocm.zig index 0861f92c..9be8b07a 100644 --- a/runtimes/rocm/rocm.zig +++ b/runtimes/rocm/rocm.zig @@ -37,7 +37,7 @@ pub fn load() !*const pjrt.Api { return error.Unavailable; } - var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); + var arena = std.heap.ArenaAllocator.init(std.heap.smp_allocator); defer arena.deinit(); var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator() }) orelse { diff --git a/runtimes/tpu/tpu.zig b/runtimes/tpu/tpu.zig index dc12b437..96af0be3 100644 --- a/runtimes/tpu/tpu.zig +++ b/runtimes/tpu/tpu.zig @@ -1,12 +1,12 @@ -const builtin = @import("builtin"); const std = @import("std"); +const builtin = @import("builtin"); const asynk = @import("async"); -const pjrt = @import("pjrt"); -const c = @import("c"); -const stdx = @import("stdx"); const bazel_builtin = @import("bazel_builtin"); +const c = @import("c"); +const pjrt = @import("pjrt"); const runfiles = @import("runfiles"); +const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/runtime/tpu"); @@ -25,10 +25,10 @@ fn isOnGCP() !bool { var f = try asynk.File.open("/sys/devices/virtual/dmi/id/product_name", .{ .mode = .read_only }); defer f.close() catch {}; - var buf = [_]u8{0} ** GoogleComputeEngine.len; - _ = try f.reader().readAll(&buf); + var content: [GoogleComputeEngine.len]u8 = undefined; + const n_read = try f.pread(&content, 0); - return std.mem.eql(u8, &buf, GoogleComputeEngine); + return std.mem.eql(u8, content[0..n_read], GoogleComputeEngine); } pub fn load() !*const pjrt.Api { @@ -42,7 +42,7 @@ pub fn load() !*const pjrt.Api { return error.Unavailable; } - var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); + var arena = std.heap.ArenaAllocator.init(std.heap.smp_allocator); defer arena.deinit(); var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator() }) orelse { diff --git a/stdx/BUILD.bazel b/stdx/BUILD.bazel index 90456d16..e44bda25 100644 --- a/stdx/BUILD.bazel +++ b/stdx/BUILD.bazel @@ -8,7 +8,6 @@ zig_library( "flags.zig", "fmt.zig", "fs.zig", - "io.zig", "json.zig", "math.zig", "meta.zig", diff --git a/stdx/io.zig b/stdx/io.zig deleted file mode 100644 index 390647b8..00000000 --- a/stdx/io.zig +++ /dev/null @@ -1,4 +0,0 @@ -const std = @import("std"); - -pub const BufferedAnyWriter = std.io.BufferedWriter(4096, std.io.AnyWriter); -pub const BufferedAnyReader = std.io.BufferedReader(4096, std.io.AnyReader); diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 661dea61..9bfa532a 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -4,7 +4,6 @@ pub const debug = @import("debug.zig"); pub const flags = @import("flags.zig"); pub const fmt = @import("fmt.zig"); pub const fs = @import("fs.zig"); -pub const io = @import("io.zig"); pub const json = @import("json.zig"); pub const math = @import("math.zig"); pub const meta = @import("meta.zig"); diff --git a/upb/upb.zig b/upb/upb.zig index 1a233ece..2a9b1d52 100644 --- a/upb/upb.zig +++ b/upb/upb.zig @@ -95,7 +95,7 @@ pub fn serialize(ptr: anytype, arena: *c.upb_Arena) SerializeError![]const u8 { pub fn parseEx(comptime UpbType: type, arena: *c.upb_Arena, data: []const u8, opts: ParseOptions) ParseError!*UpbType { const obj = try new(UpbType, arena); - return switch (c.upb_Decode(@ptrCast(@constCast(data)), data.len, @alignCast(@ptrCast(obj)), Minitable(UpbType), null, @bitCast(opts), arena)) { + return switch (c.upb_Decode(@ptrCast(@constCast(data)), data.len, @ptrCast(@alignCast(obj)), Minitable(UpbType), null, @bitCast(opts), arena)) { c.kUpb_DecodeStatus_Ok => obj, c.kUpb_DecodeStatus_Malformed => ParseError.Malformed, c.kUpb_DecodeStatus_OutOfMemory => std.mem.Allocator.Error.OutOfMemory, diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index e0722f70..aaa44f04 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -24,6 +24,11 @@ zig_library( "aio/json.zig", "aio/safetensors.zig", "aio/tinyllama.zig", + "aio/torch.zig", + "aio/torch/eval.zig", + "aio/torch/file.zig", + "aio/torch/pickle.zig", + "aio/torch/py.zig", "buffer.zig", "context.zig", "dtype.zig", @@ -72,6 +77,10 @@ zig_library( zig_test( name = "test", + data = [ + "aio/torch/simple.pt", + "aio/torch/simple_test_4.pickle", + ], test_runner = ":test_runner", deps = [":zml"], ) diff --git a/zml/aio.zig b/zml/aio.zig index 67e1876c..1b926a91 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -5,16 +5,16 @@ const c = @import("c"); const stdx = @import("stdx"); pub const safetensors = @import("aio/safetensors.zig"); -pub const tinyllama = @import("aio/tinyllama.zig"); +pub const torch = @import("aio/torch.zig"); const HostBuffer = @import("hostbuffer.zig").HostBuffer; const posix = @import("posix.zig"); const zml = @import("zml.zig"); pub const log = std.log.scoped(.@"zml/aio"); - test { std.testing.refAllDecls(@This()); std.testing.refAllDecls(safetensors); + std.testing.refAllDecls(torch); } // TODO error set for weight loading @@ -25,6 +25,12 @@ pub fn detectFormatAndOpen(allocator: std.mem.Allocator, model_path: []const u8) try safetensors.open(allocator, model_path) else if (std.mem.endsWith(u8, model_path, ".safetensors.index.json")) try safetensors.open(allocator, model_path) + else if (std.mem.endsWith(u8, model_path, ".pt")) + try torch.open(allocator, model_path) + // else if (std.mem.endsWith(u8, model_path, ".gguf")) + // try gguf.open(allocator, model_path) + // else if (std.mem.endsWith(u8, model_path, ".tinyllama")) + // try tinyllama.open(allocator, model_path) else { std.debug.panic("File extension not recognized: {s}", .{model_path}); }; @@ -384,7 +390,7 @@ fn _populateStruct( partial_struct = partial_struct or field_found; if (!field_found) { if (field.default_value_ptr) |v| { - @field(obj, field.name) = @as(*const field.type, @alignCast(@ptrCast(v))).*; + @field(obj, field.name) = @as(*const field.type, @ptrCast(@alignCast(v))).*; } else { if (partial_struct) { log.warn("Incomplete metadata '{0s}': {1s}. Missing field: '{2s}'. '{0s}' will be ignored.", .{ prefix, @typeName(T), field.name }); diff --git a/zml/aio/torch.zig b/zml/aio/torch.zig new file mode 100644 index 00000000..79487f64 --- /dev/null +++ b/zml/aio/torch.zig @@ -0,0 +1,42 @@ +const std = @import("std"); + +const asynk = @import("async"); + +const zml = @import("../zml.zig"); +const eval = @import("torch/eval.zig"); +const File = @import("torch/file.zig").File; + +const StringBuilder = std.ArrayListUnmanaged(u8); +const log = std.log.scoped(.@"zml/aio"); + +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(eval); + std.testing.refAllDecls(@import("torch/py.zig")); + std.testing.refAllDecls(File); +} + +/// Opens and loads a BufferStore from the torch file at the given path. +pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore { + const file = asynk.File.open(path, .{}) catch |err| { + log.err("Failed to open {s}: {}", .{ path, err }); + return err; + }; + errdefer file.close() catch unreachable; + + // Temporary memory needed to parse the pytorch file. + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + const tmp_alloc = arena.allocator(); + + const mmap_file = try zml.aio.MemoryMappedFile.init(file); + var torch_file = try asynk.callBlocking(File.init, .{ tmp_alloc, mmap_file }); + + const ops = try torch_file.parsePickle(tmp_alloc); + const py_values = try eval.evaluate(tmp_alloc, ops, true); + + // file ownership is transferred to the BufferStore + var res = try zml.aio.BufferStore.init(allocator, &.{torch_file.mmap_file}); + try torch_file.parseModel(py_values, &res); + return res; +} diff --git a/zml/aio/torch/eval.zig b/zml/aio/torch/eval.zig new file mode 100644 index 00000000..42565ae5 --- /dev/null +++ b/zml/aio/torch/eval.zig @@ -0,0 +1,436 @@ +const std = @import("std"); +const stdx = @import("stdx"); + +const py = @import("py.zig"); +const pickle = @import("pickle.zig"); + +const MAX_DEPTH: usize = 250; +const MAX_PROTOCOL: u8 = 5; + +pub const PickleMemo = struct { + map: std.AutoHashMap(u32, py.Any), + + pub fn init(allocator: std.mem.Allocator) PickleMemo { + return .{ + .map = std.AutoHashMap(u32, py.Any).init(allocator), + }; + } + + pub fn deinit(self: *PickleMemo) void { + const allocator = self.map.allocator; + var iterator = self.map.iterator(); + defer iterator.deinit(); + while (iterator.next()) |entry| { + entry.value_ptr.deinit(allocator); + } + self.map.deinit() catch unreachable; + self.* = undefined; + } + + pub fn resolve(self: *PickleMemo, allocator: std.mem.Allocator, op: py.Any, recursive: bool) !py.Any { + var used_op = op; + while (used_op == .ref) { + var count: usize = 0; + const val = self.map.get(op.ref) orelse { + return error.BadMemoRef; + }; + if (!recursive) { + return val.clone(allocator); + } + + count += 1; + if (count >= MAX_DEPTH or val != .ref) { + used_op = try val.clone(allocator); + break; + } + used_op = val; + } + if (used_op.containsRef()) { + switch (used_op) { + .app, .object, .global => |v| { + if (v.member.containsRef()) { + v.member = try self.resolve(allocator, v.member, recursive); + } + for (v.args) |*item| { + if (item.containsRef()) { + item.* = try self.resolve(allocator, item.*, recursive); + } + } + }, + .set_state => |v| { + if (v.obj.containsRef()) { + v.obj = try self.resolve(allocator, v.obj, recursive); + } + if (v.state.containsRef()) { + v.state = try self.resolve(allocator, v.state, recursive); + } + }, + .pers_id => |v| { + if (v.ref.containsRef()) { + v.ref = try self.resolve(allocator, v.ref, recursive); + } + }, + .seq => |*v| { + for (v.values) |*item| { + if (item.containsRef()) { + item.* = try self.resolve(allocator, item.*, recursive); + } + } + }, + else => {}, + } + } + return used_op; + } + + pub fn insert(self: *PickleMemo, mid: u32, val: py.Any) !void { + _ = try self.map.fetchPut(mid, val); + } + + pub fn resolveMut(self: *PickleMemo, op: *py.Any, recursive: bool) !*py.Any { + if (op.* != .ref) return op; + var lastmid = op.ref; + var count: usize = 0; + var val = self.map.get(lastmid) orelse { + return error.BadMemoRef; + }; + while (val == .ref) { + lastmid = val.ref; + if (!recursive) { + break; + } + count += 1; + if (count >= MAX_DEPTH) { + break; + } + val = self.map.get(lastmid) orelse { + return error.BadMemoRef; + }; + } + return (self.map.getPtr(lastmid) orelse { + return error.BadMemoRef; + }); + } + + const MemoError = py.Any.UnpickleError || error{BadMemoRef}; + + pub fn resolveAllRefsIter(self: *PickleMemo, allocator: std.mem.Allocator, depth: usize, vals: []py.Any, fix_values: bool) MemoError![]py.Any { + if (depth >= MAX_DEPTH) { + return vals; + } + const res = try allocator.alloc(py.Any, vals.len); + for (vals, 0..) |v, i| { + res[i] = try self.resolveAllRefs(allocator, depth + 1, v, fix_values); + } + return res; + } + + pub fn resolveAllRefs(self: *PickleMemo, allocator: std.mem.Allocator, depth: usize, val: py.Any, fix_values: bool) !py.Any { + var output: py.Any = switch (val) { + .ref => try self.resolve(allocator, val, true), + inline .app, .object, .global => |v, tag| @unionInit(py.Any, @tagName(tag), try py.Object.init( + allocator, + try self.resolveAllRefs(allocator, depth + 1, v.member, fix_values), + try self.resolveAllRefsIter(allocator, depth + 1, v.args, fix_values), + try self.resolveAllRefsIter(allocator, depth + 1, v.kwargs, fix_values), + )), + .set_state => |v| .{ .set_state = try py.SetState.init( + allocator, + try self.resolveAllRefs(allocator, depth + 1, v.obj, fix_values), + try self.resolveAllRefs(allocator, depth + 1, v.state, fix_values), + ) }, + .seq => |v| .{ .seq = .{ .type = v.type, .values = try self.resolveAllRefsIter(allocator, depth + 1, v.values, fix_values) } }, + .pers_id => |v| .{ .pers_id = try py.PersId.init(allocator, try self.resolveAllRefs(allocator, depth + 1, v.ref, fix_values)) }, + else => try val.clone(allocator), + }; + if (fix_values) { + output = try output.coerceFromRaw(allocator); + } + return output; + } +}; + +pub fn evaluate(arena: std.mem.Allocator, x: []const pickle.Op, resolve_refs: bool) ![]const py.Any { + var stack: std.ArrayList(py.Any) = .{}; + var memo = PickleMemo.init(arena); + + for (x) |op| { + switch (op) { + .mark => try stack.append(arena, .{ .raw = op }), + .frame => {}, + .stop => break, + .pop => _ = try pop(&stack), + .pop_mark => _ = try popMark(&stack), + .dup => if (stack.getLastOrNull()) |item| + try stack.append(arena, try item.clone(arena)) + else + return error.CannotDupEmptyStack, + .persid => |v| try stack.append(arena, .{ .pers_id = try py.PersId.init(arena, .{ .string = try arena.dupe(u8, v) }) }), + .binpersid => try stack.append(arena, .{ .pers_id = try py.PersId.init(arena, try pop(&stack)) }), + .reduce => try stack.append(arena, .{ .global = blk: { + var args = try pop(&stack); + args = try memo.resolve(arena, args, true); + if (args != .seq) return error.InvalidInput; + var func = try pop(&stack); + func = try memo.resolve(arena, func, true); + break :blk try py.Object.init(arena, func, args.seq.values, &.{}); + } }), + .build => try stack.append(arena, blk: { + const args = try memo.resolve(arena, try pop(&stack), true); + const member = try memo.resolve(arena, try pop(&stack), true); + break :blk .{ .set_state = try py.SetState.init(arena, member, args) }; + }), + .empty_dict => try stack.append(arena, .{ .seq = .{ .type = .dict, .values = &[_]py.Any{} } }), + .get => |v| try stack.append(arena, .{ .ref = v }), + .empty_list => try stack.append(arena, .{ .seq = .{ .type = .list, .values = &[_]py.Any{} } }), + .put => |v| { + try memo.insert(v, try pop(&stack)); + try stack.append(arena, .{ .ref = v }); + }, + .tuple => try stack.append(arena, blk: { + const popped = try popMark(&stack); + break :blk .{ .seq = .{ .type = .tuple, .values = try arena.dupe(py.Any, popped) } }; + }), + .empty_tuple => try stack.append(arena, .{ .seq = .{ .type = .tuple, .values = &[_]py.Any{} } }), + .setitem => { + const v = try memo.resolve(arena, try pop(&stack), true); + const k = try memo.resolve(arena, try pop(&stack), true); + const top = try lastMut(&stack); + const rtop = try memo.resolveMut(top, true); + switch (rtop.*) { + .global => |obj| { + try append(arena, &obj.kwargs, &.{ k, v }); + }, + .seq => |*dict| { + if (dict.type != .dict) return error.BadStackTopForSetItem; + try append(arena, &dict.values, &.{ k, v }); + }, + else => { + return error.BadStackTopForSetItem; + }, + } + }, + .setitems => { + const popped = try memo.resolveAllRefsIter(arena, 0, try popMark(&stack), true); + const top = try lastMut(&stack); + const rtop = try memo.resolveMut(top, true); + switch (rtop.*) { + .global => |obj| { + try append(arena, &obj.kwargs, popped); + }, + .seq => |*dict| { + if (dict.type != .dict) return error.BadStackTopForSetItems; + try append(arena, &dict.values, popped); + }, + else => { + return error.BadStackTopForSetItems; + }, + } + }, + .proto => |proto| stdx.debug.assert(proto <= MAX_PROTOCOL, "Unsupported protocol {d}", .{proto}), + .tuple1 => try stack.append(arena, blk: { + const tup_values = try arena.alloc(py.Any, 1); + tup_values[0] = try pop(&stack); + break :blk .{ .seq = .{ .type = .tuple, .values = tup_values } }; + }), + .tuple2 => try stack.append(arena, blk: { + const tup_values = try arena.alloc(py.Any, 2); + inline for (0..2) |i| tup_values[(tup_values.len - 1) - i] = try pop(&stack); + break :blk .{ .seq = .{ .type = .tuple, .values = tup_values } }; + }), + .tuple3 => try stack.append(arena, blk: { + const tup_values = try arena.alloc(py.Any, 3); + inline for (0..3) |i| tup_values[(tup_values.len - 1) - i] = try pop(&stack); + break :blk .{ .seq = .{ .type = .tuple, .values = tup_values } }; + }), + .append => { + const v = try pop(&stack); + const top = try lastMut(&stack); + const rtop = try memo.resolveMut(top, true); + switch (rtop.*) { + .global => |obj| { + // can this happen ? + try append(arena, &obj.args, &.{v}); + }, + .seq => |*seq| { + if (seq.type != .list) return error.BadStackTopForAppend; + try append(arena, &seq.values, &.{v}); + }, + else => { + return error.BadStackTopForAppend; + }, + } + }, + .appends => { + const postmark = try popMark(&stack); + const top = try lastMut(&stack); + const rtop = try memo.resolveMut(top, true); + switch (rtop.*) { + .global => try append(arena, &rtop.global.args, postmark), + .seq => |*seq| { + if (seq.type != .list) return error.BadStackTopForAppend; + try append(arena, &seq.values, postmark); + }, + else => { + return error.BadStackTopForAppends; + }, + } + }, + .dict => try stack.append(arena, .{ .seq = .{ + .type = .dict, + .values = try arena.dupe(py.Any, try popMark(&stack)), + } }), + .list => try stack.append(arena, .{ .seq = .{ + .type = .list, + .values = try arena.dupe(py.Any, try popMark(&stack)), + } }), + .inst => |v| try stack.append(arena, .{ .object = try py.Object.init( + arena, + try py.tuple(&.{ .{ .string = v.module }, .{ .string = v.class } }).clone(arena), + try arena.dupe(py.Any, try popMark(&stack)), + &.{}, + ) }), + .obj => try stack.append(arena, blk: { + const mark = try findMark(&stack); + const args = try arena.dupe(py.Any, stack.items[mark + 2 ..]); + const member = stack.items[mark + 1]; + break :blk .{ .object = try py.Object.init(arena, member, args, &.{}) }; + }), + .newobj => try stack.append(arena, blk: { + const args = try arena.alloc(py.Any, 1); + args[0] = try pop(&stack); + break :blk .{ .object = try py.Object.init(arena, try pop(&stack), args, &.{}) }; + }), + .empty_set => try stack.append(arena, .{ .seq = .{ .type = .set, .values = &[_]py.Any{} } }), + .additems => { + const postmark = try popMark(&stack); + const top = try lastMut(&stack); + const rtop = try memo.resolveMut(top, true); + switch (rtop.*) { + .seq => |*seq| { + if (seq.type != .set) return error.BadStackTopForAppend; + try append(arena, &seq.values, postmark); + }, + else => { + return error.BadStackTopForAppends; + }, + } + }, + .frozenset => try stack.append(arena, .{ .seq = .{ + .type = .frozen_set, + .values = try arena.dupe(py.Any, try popMark(&stack)), + } }), + .newobj_ex => try stack.append(arena, blk: { + const kwargs, const args, const cls = .{ try pop(&stack), try pop(&stack), try pop(&stack) }; + break :blk .{ .object = try py.Object.init(arena, cls, args.seq.values, kwargs.seq.values) }; + }), + .stack_global => try stack.append(arena, blk: { + const gn, const mn = .{ + try memo.resolve(arena, try pop(&stack), true), + try memo.resolve(arena, try pop(&stack), true), + }; + const new_seq: py.Sequence = .{ .type = .tuple, .values = try arena.dupe(py.Any, &.{ gn, mn }) }; + break :blk .{ .object = try py.Object.init(arena, .{ .seq = new_seq }, &.{}, &.{}) }; + }), + .memoize => { + const item = stack.getLastOrNull() orelse { + return error.StackUnderrun; + }; + try memo.insert(@intCast(memo.map.count()), try item.clone(arena)); + }, + else => try stack.append(arena, .{ .raw = try op.clone(arena) }), + } + } + if (resolve_refs) { + return try memo.resolveAllRefsIter(arena, 0, stack.items, true); + } + return stack.toOwnedSlice(arena); +} + +fn append(allocator: std.mem.Allocator, current: *[]py.Any, values: []const py.Any) !void { + var array_list = std.ArrayListUnmanaged(py.Any).fromOwnedSlice(current.*); + try array_list.appendSlice(allocator, values); + current.* = array_list.items; +} + +test evaluate { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + const file = try std.fs.cwd().openFile("zml/aio/torch/simple_test_4.pickle", .{ .mode = .read_only }); + var reader_buffer: [1024]u8 = undefined; + var reader = file.reader(&reader_buffer); + const ops = try pickle.parse(allocator, &reader.interface); + + const vals = try evaluate(allocator, ops, true); + defer allocator.free(vals); + + try std.testing.expect(vals.len == 1); + try std.testing.expect(vals[0] == .seq); + try std.testing.expect(vals[0].seq.type == .dict); + const entries = vals[0].seq.values; + const expected: []const py.Any = &.{ + // Key, followed by its value + .{ .string = "hello" }, .{ .string = "world" }, + .{ .string = "int" }, .{ .int64 = 1 }, + .{ .string = "float" }, .{ .float64 = 3.141592 }, + .{ .string = "list" }, + .{ + .seq = .{ + .type = .list, + .values = @constCast(&[_]py.Any{ + .{ .int64 = 255 }, + .{ .int64 = 1234 }, + .{ .int64 = -123 }, + .{ .int64 = 1_000_000_000 }, + .{ .int64 = 999_000_000_000 }, + .{ .bigint = (try std.math.big.int.Managed.initSet(allocator, 999_000_000_000_000_000_000_000_000_000)).toConst() }, + }), + }, + }, + .{ .string = "bool" }, .{ .boolval = false }, + .{ .string = "tuple" }, + .{ .seq = .{ + .type = .tuple, + .values = @constCast(&[_]py.Any{ + .{ .string = "a" }, + .{ .int64 = 10 }, + }), + } }, + }; + + try std.testing.expectEqualDeep(expected, entries); +} + +pub fn pop(values: *std.ArrayList(py.Any)) error{StackUnderrun}!py.Any { + if (values.items.len == 0) { + return error.StackUnderrun; + } + return values.pop().?; +} + +fn popMark(values: *std.ArrayList(py.Any)) ![]py.Any { + const mark = try findMark(values); + const popping = values.items[mark + 1 ..]; + values.shrinkRetainingCapacity(mark); + return popping; +} + +fn lastMut(values: *std.ArrayList(py.Any)) !*py.Any { + if (values.items.len == 0) { + return error.UnexpectedEmptyStack; + } + return &values.items[values.items.len - 1]; +} + +fn findMark(values: *std.ArrayList(py.Any)) !usize { + const len = values.items.len; + for (0..len) |i| { + const idx = (len - 1) - i; + const val = values.items[idx]; + if (val == .raw and val.raw == .mark) { + return idx; + } + } + return error.MarkNotFound; +} diff --git a/zml/aio/torch/file.zig b/zml/aio/torch/file.zig new file mode 100644 index 00000000..0307d51c --- /dev/null +++ b/zml/aio/torch/file.zig @@ -0,0 +1,526 @@ +const std = @import("std"); +const testing = std.testing; + +const asynk = @import("async"); +const stdx = @import("stdx"); + +const zml = @import("../../zml.zig"); +const HostBuffer = zml.HostBuffer; +const eval = @import("eval.zig"); +const pickle = @import("pickle.zig"); +const py = @import("py.zig"); + +const log = std.log.scoped(.@"zml/aio"); + +// TODO(cryptodeal): use zml.aio.PrefixBuilder instead +const StringBuilder = std.ArrayListUnmanaged(u8); + +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(File); +} + +pub const File = struct { + mmap_file: zml.aio.MemoryMappedFile, + /// Map names to sub file + file_map: std.StringArrayHashMapUnmanaged([]const u8) = .{}, + zip_prefix: []const u8, + pickle_subfile: []const u8, + + const magic = "PK\x03\x04"; + + pub fn init(allocator: std.mem.Allocator, mmap_file: zml.aio.MemoryMappedFile) !File { + var pkl: []const u8 = mmap_file.data; + var zip_prefix: []const u8 = &.{}; + var file_map: std.StringArrayHashMapUnmanaged([]const u8) = .{}; + if (std.mem.eql(u8, mmap_file.data[0..magic.len], magic)) { + // We are dealing with a zip file. + // Let's look for the `data.pkl` file and keep a map of all other files. + // The other files will be the tensor storage and will be reference from `data.pkl`. + var header_parsing_buffer: [4096]u8 = undefined; + + // std.zip requires on a std.fs.File and don't leverage std.Io.Reader directly. + // So we use the synchronous API to parse the headers, + // then we rely only on the memory map data to parse the pickle and load the buffers. + // To mitigate this we use `async.launchBlocking` in `torch.open`. + const raw_file: std.fs.File = .{ .handle = mmap_file.file._handle }; + var reader = raw_file.reader(&header_parsing_buffer); + var it: std.zip.Iterator = try .init(&reader); + + while (try it.next()) |header| { + if (header.filename_len == 0) { + continue; + } + if (header.compression_method != .store) { + return error.Unsupported; + } + + const filename = mmap_file.data[header.header_zip_offset + @sizeOf(std.zip.CentralDirectoryFileHeader) ..][0..header.filename_len]; + + var local_reader: std.Io.Reader = .fixed(mmap_file.data); + local_reader.discardAll(header.file_offset) catch return error.InvalidZipFile; + const local_header = local_reader.takeStruct(std.zip.LocalFileHeader, .little) catch return error.InvalidZipFile; + local_reader.discardAll(local_header.filename_len) catch return error.InvalidZipFile; + local_reader.discardAll(local_header.extra_len) catch return error.InvalidZipFile; + + // normalize path separators + const file_content = mmap_file.data[local_reader.seek..][0..header.compressed_size]; + const my_filename: []u8 = try allocator.dupe(u8, filename); + std.mem.replaceScalar(u8, my_filename, '\\', '/'); + + try file_map.put(allocator, my_filename, file_content); + + if (std.mem.endsWith(u8, filename, "data.pkl")) { + pkl = file_content; + zip_prefix = filename[0 .. filename.len - "data.pkl".len]; + } + } + + if (pkl.len == 0) { + log.err("Could not find file ending in `data.pkl` in archive", .{}); + return error.PickleNotFound; + } + } + + return .{ + .mmap_file = mmap_file, + .file_map = file_map, + .pickle_subfile = pkl, + .zip_prefix = zip_prefix, + }; + } + + pub fn close(self: *File) void { + self.mmap_file.deinit(); + } + + pub fn parsePickle(self: *File, allocator: std.mem.Allocator) ![]const pickle.Op { + var reader: std.Io.Reader = .fixed(self.pickle_subfile); + return try pickle.parse(allocator, &reader); + } + + fn basicTypeCheck(object: *const py.Object, module: []const u8, class: []const u8) bool { + return switch (object.member) { + .raw => |raw| return (std.mem.eql(u8, module, raw.global.module) and + std.mem.eql(u8, class, raw.global.class)), + else => false, + }; + } + + pub fn parseModel(self: File, values: []const py.Any, store: *zml.aio.BufferStore) !void { + var prefix_buf: [1024]u8 = undefined; + const allocator = store.arena.allocator(); + for (values) |item| { + try self.parseValue(allocator, store, StringBuilder.initBuffer(&prefix_buf), item); + } + } + + pub fn parseValue(self: File, allocator: std.mem.Allocator, store: *zml.aio.BufferStore, prefix: StringBuilder, v: py.Any) !void { + // log.warn("Parsing {}", .{v}); + switch (v) { + .app, .object, .global => |object| { + if (!(try self.parseTorchGlobal(allocator, store, prefix, v))) { + try self.parseValue(allocator, store, prefix, object.member); + for (object.args) |item| { + try self.parseValue(allocator, store, prefix, item); + } + if (object.kwargs.len % 2 != 0) return error.InvalidInput; + const n_kwargs = @divExact(object.kwargs.len, 2); + + for (0..n_kwargs) |i| { + const key, const val = object.kwargs[2 * i ..][0..2].*; + // kwargs can only be keyed by string. + if (key != .string) return error.InvalidInput; + // Handle Pytorch specific fields + const s = key.string; + if (std.mem.eql(u8, s, "_modules") or std.mem.eql(u8, s, "_parameters") or std.mem.eql(u8, s, "_buffers")) { + try self.parseValue(allocator, store, prefix, val); + } else { + var new_prefix = prefix; + if (prefix.items.len > 0) { + new_prefix.appendAssumeCapacity('.'); + } + new_prefix.appendSliceAssumeCapacity(s); + try self.parseValue(allocator, store, new_prefix, val); + } + } + } + }, + .set_state => |set_state| { + // `set_state` contains info about python struct being constructed + switch (set_state.obj) { + .object => |obj| switch (obj.member) { + .raw => |raw| switch (raw) { + .global => |global| { + // in this case, we can capture the name of the python type + // which can be used for codegen (e.g. `torch.nn.modules.conv.Conv2d`) + var new_prefix = prefix; + if (prefix.items.len > 0) { + new_prefix.appendAssumeCapacity('.'); + } + new_prefix.appendSliceAssumeCapacity("_gen_type_helper"); + const key = try allocator.dupe(u8, new_prefix.items); + const d = try store._metadata.getOrPut(allocator, key); + if (d.found_existing) { + log.err("Duplicate key: {s}", .{new_prefix.items}); + allocator.free(key); + } else { + const val = try std.mem.join(allocator, ".", &.{ global.module, global.class }); + d.value_ptr.* = .{ .string = val }; + } + }, + else => try self.parseValue(allocator, store, prefix, set_state.obj), // parse normally + }, + else => try self.parseValue(allocator, store, prefix, set_state.obj), // parse normally + }, + else => try self.parseValue(allocator, store, prefix, set_state.obj), // parse normally + } + try self.parseValue(allocator, store, prefix, set_state.state); + }, + .pers_id => |pers_id| try self.parseValue(allocator, store, prefix, pers_id.ref), + .seq => |seq| { + switch (seq.type) { + .list, .tuple, .set, .frozen_set => { + if (seq.values.len == 0) return; + var valid_slice = true; + switch (seq.values[0]) { + inline .int64, .float64, .boolval => |val0, tag| { + const ItemType = switch (tag) { + .int64 => i64, + .float64 => f64, + .boolval => bool, + else => unreachable, + }; + var values: std.ArrayListUnmanaged(ItemType) = .{}; + try values.append(allocator, val0); + for (seq.values[1..], 1..) |val, i| { + if (std.meta.activeTag(val) != tag) valid_slice = false; + if (valid_slice) { + try values.append(allocator, @field(val, @tagName(tag))); + } else { + var new_prefix = prefix; + if (prefix.items.len > 0) { + new_prefix.appendAssumeCapacity('.'); + } + new_prefix.items.len += std.fmt.printInt(new_prefix.unusedCapacitySlice(), i, 10, .lower, .{}); + try self.parseValue(allocator, store, new_prefix, val); + } + } + + if (valid_slice) { + try store._metadata.put( + allocator, + try allocator.dupe(u8, prefix.items), + try zml.aio.Metadata.copySlice(allocator, values.items), + ); + } else { + for (values.items, 0..) |val, i| { + var new_prefix = prefix; + if (prefix.items.len > 0) { + new_prefix.appendAssumeCapacity('.'); + } + new_prefix.items.len += std.fmt.printInt(new_prefix.unusedCapacitySlice(), i, 10, .lower, .{}); + const new_tag = switch (tag) { + .int64 => "int", + .float64 => "float", + .boolval => "bool", + else => unreachable, // we are already inside a switch + }; + try store._metadata.put(allocator, try allocator.dupe(u8, new_prefix.items), @unionInit(zml.aio.Metadata, new_tag, val)); + } + } + }, + else => { + for (seq.values, 0..) |item, i| { + var new_prefix = prefix; + if (v.isPrimitive()) { + if (prefix.items.len > 0) { + new_prefix.appendAssumeCapacity('.'); + } + new_prefix.items.len += std.fmt.printInt(new_prefix.unusedCapacitySlice(), i, 10, .lower, .{}); + } + try self.parseValue(allocator, store, new_prefix, item); + } + }, + } + }, + .dict => { + const n = @divExact(seq.values.len, 2); + log.debug("found dict with {} entries", .{n}); + for (0..n) |i| { + const key, const val = seq.values[2 * i ..][0..2].*; + switch (key) { + .string => |s| { + // Handle Pytorch specific fields. + if (std.mem.eql(u8, s, "_modules") or std.mem.eql(u8, s, "_parameters") or std.mem.eql(u8, s, "_buffers")) { + try self.parseValue(allocator, store, prefix, val); + } else { + var new_prefix = prefix; + if (prefix.items.len > 0) { + new_prefix.appendAssumeCapacity('.'); + } + new_prefix.appendSliceAssumeCapacity(s); + + try self.parseValue(allocator, store, new_prefix, val); + } + }, + .int64 => |int| { + var new_prefix = prefix; + if (prefix.items.len > 0) { + new_prefix.appendAssumeCapacity('.'); + } + new_prefix.items.len += std.fmt.printInt(new_prefix.unusedCapacitySlice(), int, 10, .lower, .{}); + try self.parseValue(allocator, store, new_prefix, val); + }, + inline else => |_, tag| { + log.debug("Ignoring unsupported key type found in torch file: {s}", .{@tagName(tag)}); + continue; + }, + } + } + }, + } + }, + .bytes => |val| { + const key = try allocator.dupe(u8, prefix.items); + const d = try store._metadata.getOrPut(allocator, key); + if (d.found_existing) { + log.warn("Duplicate key: {s}", .{prefix.items}); + allocator.free(key); + } else d.value_ptr.* = .{ .string = val }; + }, + inline .float64, .int64, .boolval, .bigint, .string => |val| { + const key = try allocator.dupe(u8, prefix.items); + const d = try store._metadata.getOrPut(allocator, key); + if (d.found_existing) { + log.warn("Duplicate key: {s}", .{prefix.items}); + allocator.free(key); + } else { + d.value_ptr.* = zml.aio.Metadata.wrap(val); + } + }, + else => {}, + } + } + + fn parseTorchGlobal(self: File, allocator: std.mem.Allocator, store: *zml.aio.BufferStore, prefix: StringBuilder, v: py.Any) !bool { + return switch (v) { + .global => |object| { + if (try self.parseTensor(allocator, object)) |host_buffer| { + const key = try allocator.dupe(u8, prefix.items); + const entry = try store.buffers.getOrPut(allocator, key); + if (entry.found_existing) { + log.warn("Duplicate key: {s}", .{prefix.items}); + allocator.free(key); + } + entry.value_ptr.* = host_buffer; + return true; + } else if (basicTypeCheck(object, "torch", "Size")) { + const size = object.args; + const key = try allocator.dupe(u8, prefix.items); + const entry = try store._metadata.getOrPut(allocator, key); + if (entry.found_existing) { + log.warn("Duplicate key: {s}", .{prefix.items}); + allocator.free(key); + } + const d = try allocator.alloc(i64, size.len); + for (d, 0..) |*di, i| di.* = size[i].int64; + entry.value_ptr.* = .{ .array_int = d }; + return true; + } else if (basicTypeCheck(object, "fractions", "Fraction")) { + const fraction_str = object.args[0].string; + if (std.mem.indexOfScalar(u8, fraction_str, '/')) |split_idx| { + { + var new_prefix = prefix; + new_prefix.appendSliceAssumeCapacity(".numerator"); + try store._metadata.put(allocator, try allocator.dupe(u8, new_prefix.items), .{ .int = try std.fmt.parseInt(i64, fraction_str[0..split_idx], 10) }); + } + { + var new_prefix = prefix; + new_prefix.appendSliceAssumeCapacity(".denominator"); + try store._metadata.put(allocator, try allocator.dupe(u8, new_prefix.items), .{ .int = try std.fmt.parseInt(i64, fraction_str[split_idx + 1 ..], 10) }); + } + return true; + } + } + return false; + }, + else => false, + }; + } + + fn parseTensor(self: File, tmp_allocator: std.mem.Allocator, object: *py.Object) !?zml.HostBuffer { + if (!basicTypeCheck(object, "torch._utils", "_rebuild_tensor_v2")) { + return null; + } + + const args = object.args; + if (args.len < 4 or + args[0] != .pers_id or + args[1] != .int64 or + args[2] != .seq or args[2].seq.type != .tuple or + args[3] != .seq or args[3].seq.type != .tuple) + { + log.err("Unexpected py.Any in call to torch._utils._rebuild_tensor_v2: {}", .{object.*}); + return error.InvalidInput; + } + + const pid: *py.PersId = args[0].pers_id; + var offset: u64 = @intCast(args[1].int64); + const raw_dims: py.Sequence = args[2].seq; + const raw_strides: py.Sequence = args[3].seq; + const dims = try parseDims(raw_dims.values); + var strides = try parseDims(raw_strides.values); + + const dtype, const storage_file = try parseStorage(pid.ref); + // Pytorch store "item" strides, while ZML uses byte strides. + for (strides.slice()) |*s| s.* *= dtype.sizeOf(); + // Same thing for the offset. + offset = offset * dtype.sizeOf(); + + const filename = try std.mem.join(tmp_allocator, "", &.{ self.zip_prefix, "data/", storage_file }); + defer tmp_allocator.free(filename); + + // The offset in the pickle is the offset inside the storage_file. + // But .pt are made of several files, so we need to append the file offset. + const storage = try self.getStorage(filename); + return HostBuffer.fromStridedSlice( + zml.Shape.init(dims.constSlice(), dtype), + storage[offset..], + strides.constSlice(), + ); + } + + fn parseStorage(val: py.Any) !struct { zml.DataType, []const u8 } { + if (val != .seq) return error.InvalidInput; + const sargs = val.seq.values; + if (val.seq.type == .tuple and + sargs.len >= 5 and + sargs[0] == .string and std.mem.eql(u8, sargs[0].string, "storage") and + sargs[1] == .raw and sargs[1].raw == .global and + sargs[2] == .string and + sargs[3] == .string) + { + const op = sargs[1].raw.global; + const storage_file = sargs[2].string; + // const sdev = sargs[3].string; + if (!std.mem.eql(u8, "torch", op.module) or + !std.mem.endsWith(u8, op.class, "Storage")) + return error.InvalidInput; + + return .{ + try storageToDtype(op.class), + storage_file, + }; + } else { + return error.InvalidInput; + } + } + + /// Given the name of one of the files in the .pt tarball, + /// return the slice of the memory-mapped .pt corresponding to it. + fn getStorage(self: File, filename: []const u8) ![]const u8 { + return self.file_map.get(filename) orelse { + std.log.err("Could not find file ending in `{s}` in archive", .{filename}); + return error.TensorNotFound; + }; + } + + fn parseDims(values: []py.Any) error{InvalidInput}!zml.Shape.DimsArray { + stdx.debug.assert(values.len <= zml.Tensor.MAX_RANK, "Found Pytorch tensor with unsupported rank {}", .{values.len}); + var result: zml.Shape.DimsArray = .{}; + for (values) |val| { + switch (val) { + .int64 => |d| result.appendAssumeCapacity(d), + else => return error.InvalidInput, + } + } + return result; + } +}; + +/// Convert from a torch.Storage to a `zml.DataType`. +/// TODO: make this future proof, storage type are going to get replaced with torch.UntypedStorage +/// See https://pytorch.org/docs/stable/storage.html +fn storageToDtype(storage_type: []const u8) !zml.DataType { + const torch_type = storage_type[0 .. storage_type.len - "Storage".len]; + const map = std.StaticStringMap(zml.DataType).initComptime(.{ + .{ "Double", .f64 }, + .{ "Float", .f32 }, + .{ "Half", .f16 }, + .{ "Long", .i64 }, + .{ "Int", .i32 }, + .{ "Short", .i16 }, + .{ "Char", .i8 }, + .{ "Byte", .u8 }, + .{ "Bool", .bool }, + .{ "BFloat16", .bf16 }, + .{ "ComplexDouble", .c128 }, + .{ "ComplexFloat", .c64 }, + // QUInt8Storage + // QInt8Storage + // QInt32Storage + // QUInt4x2Storage + // QUInt2x4Storage + }); + + return map.get(torch_type) orelse { + log.err("Unsupported torch storage type: {s}", .{storage_type}); + return error.UnsupportedDataType; + }; +} + +test "Read pickle (zipped)" { + // test file created with following python snippet: + // + // import torch + // torch.manual_seed(0) + // model = torch.nn.Conv2d(2, 2, 3, stride=2, padding=[2, 4], dtype=torch.float16) + // tensor = torch.tensor([[2, 4, 3, 2]], dtype=torch.uint8) + // torch.save({ "model": model, "tensor": tensor}, "simple.pt") + const file = try asynk.File.open("zml/aio/torch/simple.pt", .{ .mode = .read_only }); + const mmap_file = try zml.aio.MemoryMappedFile.init(file); + var store = try zml.aio.BufferStore.init(testing.allocator, &.{mmap_file}); + defer store.deinit(); + + { + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + var torch_file = try File.init(arena.allocator(), mmap_file); + // We don't close the file directly, it will be closed by the store. + + const ops = try torch_file.parsePickle(arena.allocator()); + try std.testing.expectEqual(302, ops.len); + + const py_values = try eval.evaluate(arena.allocator(), ops, true); + try torch_file.parseModel(py_values, &store); + } + + // now we have freed the arena. + // all data needed should have been copied into the store arena. + try zml.testing.expectEqualShapes( + zml.Shape.init(.{ 1, 4 }, .u8), + store.get("tensor").?.shape(), + ); + try zml.testing.expectEqualShapes( + zml.Shape.init(.{ 2, 2, 3, 3 }, .f16), + store.get("model.weight").?.shape(), + ); + try zml.testing.expectEqualShapes( + zml.Shape.init(.{2}, .f16), + store.get("model.bias").?.shape(), + ); +} + +fn isBadFilename(filename: []const u8) bool { + if (filename.len == 0 or filename[0] == '/') + return true; + + var it = std.mem.splitScalar(u8, filename, '/'); + while (it.next()) |part| { + if (std.mem.eql(u8, part, "..")) + return true; + } + + return false; +} diff --git a/zml/aio/torch/pickle.zig b/zml/aio/torch/pickle.zig new file mode 100644 index 00000000..9174eb10 --- /dev/null +++ b/zml/aio/torch/pickle.zig @@ -0,0 +1,1066 @@ +const std = @import("std"); + +const log = std.log.scoped(.@"zml/aio"); + +/// All possible pickle operators. +/// Reference: https://github.com/python/cpython/blob/3.13/Lib/pickletools.py +pub const OpCode = enum(u8) { + /// Push an integer or bool. + /// + /// The argument is a newline-terminated decimal literal string. + /// + /// The intent may have been that this always fit in a short Python int, + /// but INT can be generated in pickles written on a 64-bit box that + /// require a Python long on a 32-bit box. The difference between this + /// and LONG then is that INT skips a trailing 'L', and produces a short + /// int whenever possible. + /// + /// Another difference is due to that, when bool was introduced as a + /// distinct type in 2.3, builtin names True and False were also added to + /// 2.2.2, mapping to ints 1 and 0. For compatibility in both directions, + /// True gets pickled as INT + "I01\n", and False as INT + "I00\n". + /// Leading zeroes are never produced for a genuine integer. The 2.3 + /// (and later) unpicklers special-case these and return bool instead; + /// earlier unpicklers ignore the leading "0" and return the int. + int = 'I', + + /// Push a four-byte signed integer. + /// Introduced in protocol 1. + /// + /// This handles the full range of Python (short) integers on a 32-bit + /// box, directly as binary bytes (1 for the opcode and 4 for the integer). + /// If the integer is non-negative and fits in 1 or 2 bytes, pickling via + /// BININT1 or BININT2 saves space. + binint = 'J', + + /// Push a one-byte unsigned integer. + /// Introduced in protocol 1. + /// + /// This is a space optimization for pickling very small non-negative ints, + /// in range(256). + binint1 = 'K', + + /// Push a two-byte unsigned integer. + /// Introduced in protocol 1. + /// + /// This is a space optimization for pickling small positive ints, in + /// range(256, 2**16). Integers in range(256) can also be pickled via + /// BININT2, but BININT1 instead saves a byte. + binint2 = 'M', + + /// Push a long integer. + /// + /// The same as INT, except that the literal ends with 'L', and always + /// unpickles to a Python long. There doesn't seem a real purpose to the + /// trailing 'L'. + /// + /// Note that LONG takes time quadratic in the number of digits when + /// unpickling (this is simply due to the nature of decimal->binary + /// conversion). Proto 2 added linear-time (in C; still quadratic-time + /// in Python) LONG1 and LONG4 opcodes. + long = 'L', + + /// Long integer using one-byte length. + /// Introduced in protocol 2. + /// + /// A more efficient encoding of a Python long; the long1 encoding + long1 = 0x8a, + + /// Long integer using four-byte length. + /// Introduced in protocol 2. + /// + /// A more efficient encoding of a Python long; the long4 encoding + long4 = 0x8b, + + /// Push a Python string object. + /// + /// The argument is a repr-style string, with bracketing quote characters, + /// and perhaps embedded escapes. The argument extends until the next + /// newline character. These are usually decoded into a str instance + /// using the encoding given to the Unpickler constructor. or the default, + /// 'ASCII'. If the encoding given was 'bytes' however, they will be + /// decoded as bytes object instead. + string = 'S', + + /// Push a Python string object. + /// Introduced in protocol 1. + /// + /// There are two arguments: the first is a 4-byte little-endian + /// signed int giving the number of bytes in the string, and the + /// second is that many bytes, which are taken literally as the string + /// content. These are usually decoded into a str instance using the + /// encoding given to the Unpickler constructor. or the default, + /// 'ASCII'. If the encoding given was 'bytes' however, they will be + /// decoded as bytes object instead. + binstring = 'T', + + /// Push a Python string object. + /// Introduced in protocol 1. + /// + /// There are two arguments: the first is a 1-byte unsigned int giving + /// the number of bytes in the string, and the second is that many + /// bytes, which are taken literally as the string content. These are + /// usually decoded into a str instance using the encoding given to + /// the Unpickler constructor. or the default, 'ASCII'. If the + /// encoding given was 'bytes' however, they will be decoded as bytes + /// object instead. + short_binstring = 'U', + + /// Push a Python bytes object. + /// Introduced in protocol 3. + /// + /// There are two arguments: the first is a 4-byte little-endian unsigned int + /// giving the number of bytes, and the second is that many bytes, which are + /// taken literally as the bytes content. + binbytes = 'B', + + /// Push a Python bytes object. + /// Introduced in protocol 3. + /// + /// There are two arguments: the first is a 1-byte unsigned int giving + /// the number of bytes, and the second is that many bytes, which are taken + /// literally as the string content. + short_binbytes = 'C', + + /// Push a Python bytes object. + /// Introduced in protocol 4. + /// + /// There are two arguments: the first is an 8-byte unsigned int giving + /// the number of bytes in the string, and the second is that many bytes, + /// which are taken literally as the string content. + binbytes8 = 0x8e, + + /// Push a Python bytearray object. + /// Introduced in protocol 5. + /// + /// There are two arguments: the first is an 8-byte unsigned int giving + /// the number of bytes in the bytearray, and the second is that many bytes, + /// which are taken literally as the bytearray content. + bytearray8 = 0x96, + + /// Introduced in protocol 5. + next_buffer = 0x97, + + /// Introduced in protocol 5. + readonly_buffer = 0x98, + + none = 'N', + + /// Introduced in protocol 2. + newtrue = 0x88, + + /// Introduced in protocol 2. + newfalse = 0x89, + + /// Push a Python Unicode string object. + /// + /// The argument is a raw-unicode-escape encoding of a Unicode string, + /// and so may contain embedded escape sequences. The argument extends + /// until the next newline character. + unicode = 'V', + + /// Push a Python Unicode string object. + /// Introduced in protocol 4. + /// + /// There are two arguments: the first is a 1-byte little-endian signed int + /// giving the number of bytes in the string. The second is that many + /// bytes, and is the UTF-8 encoding of the Unicode string. + short_binunicode = 0x8c, + + /// Push a Python Unicode string object. + /// Introduced in protocol 1. + /// + /// There are two arguments: the first is a 4-byte little-endian unsigned int + /// giving the number of bytes in the string. The second is that many + /// bytes, and is the UTF-8 encoding of the Unicode string. + binunicode = 'X', + + /// Push a Python Unicode string object. + /// Introduced in protocol 4. + /// + /// There are two arguments: the first is an 8-byte little-endian signed int + /// giving the number of bytes in the string. The second is that many + /// bytes, and is the UTF-8 encoding of the Unicode string. + binunicode8 = 0x8d, + + /// Newline-terminated decimal float literal. + /// + /// The argument is repr(a_float), and in general requires 17 significant + /// digits for roundtrip conversion to be an identity (this is so for + /// IEEE-754 double precision values, which is what Python float maps to + /// on most boxes). + /// + /// In general, FLOAT cannot be used to transport infinities, NaNs, or + /// minus zero across boxes (or even on a single box, if the platform C + /// library can't read the strings it produces for such things -- Windows + /// is like that), but may do less damage than BINFLOAT on boxes with + /// greater precision or dynamic range than IEEE-754 double. + float = 'F', + + /// Float stored in binary form, with 8 bytes of data. + /// Introduced in protocol 1. + /// + /// This generally requires less than half the space of FLOAT encoding. + /// In general, BINFLOAT cannot be used to transport infinities, NaNs, or + /// minus zero, raises an exception if the exponent exceeds the range of + /// an IEEE-754 double, and retains no more than 53 bits of precision (if + /// there are more than that, "add a half and chop" rounding is used to + /// cut it back to 53 significant bits). + binfloat = 'G', + + /// Introduced in protocol 1. + empty_list = ']', + + /// Append an object to a list. + /// + /// Stack before: ... pylist anyobject + /// Stack after: ... pylist+[anyobject] + /// + /// although pylist is really extended in-place. + append = 'a', + + /// Extend a list by a slice of stack objects. + /// Introduced in protocol 1. + /// + /// Stack before: ... pylist markobject stackslice + /// Stack after: ... pylist+stackslice + /// + /// although pylist is really extended in-place. + appends = 'e', + + /// Build a list out of the topmost stack slice, after markobject. + /// + /// All the stack entries following the topmost markobject are placed into + /// a single Python list, which single list object replaces all of the + /// stack from the topmost markobject onward. For example, + /// + /// Stack before: ... markobject 1 2 3 'abc' + /// Stack after: ... [1, 2, 3, 'abc'] + list = 'l', + + /// Introduced in protocol 1. + empty_tuple = ')', + + /// Build a tuple out of the topmost stack slice, after markobject. + /// + /// All the stack entries following the topmost markobject are placed into + /// a single Python tuple, which single tuple object replaces all of the + /// stack from the topmost markobject onward. For example, + /// + /// Stack before: ... markobject 1 2 3 'abc' + /// Stack after: ... (1, 2, 3, 'abc') + tuple = 't', + + /// Build a one-tuple out of the topmost item on the stack. + /// Introduced in protocol 2. + /// + /// This code pops one value off the stack and pushes a tuple of + /// length 1 whose one item is that value back onto it. In other + /// words: + /// + /// stack[-1] = tuple(stack[-1:]) + tuple1 = 0x85, + + /// Build a two-tuple out of the top two items on the stack. + /// Introduced in protocol 2. + /// + /// This code pops two values off the stack and pushes a tuple of + /// length 2 whose items are those values back onto it. In other + /// words: + /// + /// stack[-2:] = [tuple(stack[-2:])] + tuple2 = 0x86, + + /// Build a three-tuple out of the top three items on the stack. + /// Introduced in protocol 2. + /// + /// This code pops three values off the stack and pushes a tuple of + /// length 3 whose items are those values back onto it. In other + /// words: + /// + /// stack[-3:] = [tuple(stack[-3:])] + tuple3 = 0x87, + + /// Introduced in protocol 1. + empty_dict = '}', + + /// Build a dict out of the topmost stack slice, after markobject. + /// + /// All the stack entries following the topmost markobject are placed into + /// a single Python dict, which single dict object replaces all of the + /// stack from the topmost markobject onward. The stack slice alternates + /// key, value, key, value, .... For example, + /// + /// Stack before: ... markobject 1 2 3 'abc' + /// Stack after: ... {1: 2, 3: 'abc'} + dict = 'd', + + /// Add a key+value pair to an existing dict. + /// + /// Stack before: ... pydict key value + /// Stack after: ... pydict + /// + /// where pydict has been modified via pydict[key] = value. + setitem = 's', + + /// Add an arbitrary number of key+value pairs to an existing dict. + /// Introduced in protocol 1. + /// + /// The slice of the stack following the topmost markobject is taken as + /// an alternating sequence of keys and values, added to the dict + /// immediately under the topmost markobject. Everything at and after the + /// topmost markobject is popped, leaving the mutated dict at the top + /// of the stack. + /// + /// Stack before: ... pydict markobject key_1 value_1 ... key_n value_n + /// Stack after: ... pydict + /// + /// where pydict has been modified via pydict[key_i] = value_i for i in + /// 1, 2, ..., n, and in that order. + setitems = 'u', + + /// Introduced in protocol 4. + empty_set = 0x8f, + + /// Add an arbitrary number of items to an existing set. + /// Introduced in protocol 4. + /// + /// The slice of the stack following the topmost markobject is taken as + /// a sequence of items, added to the set immediately under the topmost + /// markobject. Everything at and after the topmost markobject is popped, + /// leaving the mutated set at the top of the stack. + /// + /// Stack before: ... pyset markobject item_1 ... item_n + /// Stack after: ... pyset + /// + /// where pyset has been modified via pyset.add(item_i) = item_i for i in + /// 1, 2, ..., n, and in that order. + additems = 0x90, + + /// Build a frozenset out of the topmost slice, after markobject. + /// Introduced in protocol 4. + /// + /// All the stack entries following the topmost markobject are placed into + /// a single Python frozenset, which single frozenset object replaces all + /// of the stack from the topmost markobject onward. For example, + /// + /// Stack before: ... markobject 1 2 3 + /// Stack after: ... frozenset({1, 2, 3}) + frozenset = 0x91, + + pop = '0', + + dup = '2', + + /// Push markobject onto the stack. + /// + /// markobject is a unique object, used by other opcodes to identify a + /// region of the stack containing a variable number of objects for them + /// to work on. See markobject.doc for more detail. + mark = '(', + + /// Pop all the stack objects at and above the topmost markobject. + /// Introduced in protocol 1. + /// + /// When an opcode using a variable number of stack objects is done, + /// POP_MARK is used to remove those objects, and to remove the markobject + /// that delimited their starting position on the stack. + pop_mark = '1', + + /// Read an object from the memo and push it on the stack. + /// + /// The index of the memo object to push is given by the newline-terminated + /// decimal string following. BINGET and LONG_BINGET are space-optimized + /// versions. + get = 'g', + + /// Read an object from the memo and push it on the stack. + /// Introduced in protocol 1. + /// + /// The index of the memo object to push is given by the 1-byte unsigned + /// integer following. + binget = 'h', + + /// Read an object from the memo and push it on the stack. + /// Introduced in protocol 1. + /// + /// The index of the memo object to push is given by the 4-byte unsigned + /// little-endian integer following. + long_binget = 'j', + + /// Store the stack top into the memo. The stack is not popped. + /// + /// The index of the memo location to write into is given by the newline- + /// terminated decimal string following. BINPUT and LONG_BINPUT are + /// space-optimized versions. + put = 'p', + + /// Store the stack top into the memo. The stack is not popped. + /// Introduced in protocol 1. + /// + /// The index of the memo location to write into is given by the 1-byte + /// unsigned integer following. + binput = 'q', + + /// Store the stack top into the memo. The stack is not popped. + /// Introduced in protocol 1. + /// + /// The index of the memo location to write into is given by the 4-byte + /// unsigned little-endian integer following. + long_binput = 'r', + + /// Store the stack top into the memo. The stack is not popped. + /// Introduced in protocol 4. + /// + /// The index of the memo location to write is the number of + /// elements currently present in the memo. + memoize = 0x94, + + /// Extension code. + /// Introduced in protocol 2. + /// + /// This code and the similar EXT2 and EXT4 allow using a registry + /// of popular objects that are pickled by name, typically classes. + /// It is envisioned that through a global negotiation and + /// registration process, third parties can set up a mapping between + /// ints and object names. + /// + /// In order to guarantee pickle interchangeability, the extension + /// code registry ought to be global, although a range of codes may + /// be reserved for private use. + /// + /// EXT1 has a 1-byte integer argument. This is used to index into the + /// extension registry, and the object at that index is pushed on the stack. + ext1 = 0x82, + + /// Extension code. + /// Introduced in protocol 2. + /// + /// See EXT1. EXT2 has a two-byte integer argument. + ext2 = 0x83, + + /// Extension code. + /// Introduced in protocol 2. + /// + /// See EXT1. EXT4 has a four-byte integer argument. + ext4 = 0x84, + + /// Push a global object (module.attr) on the stack. + /// + /// Two newline-terminated strings follow the GLOBAL opcode. The first is + /// taken as a module name, and the second as a class name. The class + /// object module.class is pushed on the stack. More accurately, the + /// object returned by self.find_class(module, class) is pushed on the + /// stack, so unpickling subclasses can override this form of lookup. + global = 'c', + + /// Push a global object (module.attr) on the stack. + /// Introduced in protocol 4. + stack_global = 0x93, + + /// Push an object built from a callable and an argument tuple. + /// + /// The opcode is named to remind of the __reduce__() method. + /// + /// Stack before: ... callable pytuple + /// Stack after: ... callable(*pytuple) + /// + /// The callable and the argument tuple are the first two items returned + /// by a __reduce__ method. Applying the callable to the argtuple is + /// supposed to reproduce the original object, or at least get it started. + /// If the __reduce__ method returns a 3-tuple, the last component is an + /// argument to be passed to the object's __setstate__, and then the REDUCE + /// opcode is followed by code to create setstate's argument, and then a + /// BUILD opcode to apply __setstate__ to that argument. + /// + /// If not isinstance(callable, type), REDUCE complains unless the + /// callable has been registered with the copyreg module's + /// safe_constructors dict, or the callable has a magic + /// '__safe_for_unpickling__' attribute with a true value. I'm not sure + /// why it does this, but I've sure seen this complaint often enough when + /// I didn't want to . + reduce = 'R', + + /// Finish building an object, via __setstate__ or dict update. + /// + /// Stack before: ... anyobject argument + /// Stack after: ... anyobject + /// + /// where anyobject may have been mutated, as follows: + /// + /// If the object has a __setstate__ method, + /// + /// anyobject.__setstate__(argument) + /// + /// is called. + /// + /// Else the argument must be a dict, the object must have a __dict__, and + /// the object is updated via + /// + /// anyobject.__dict__.update(argument) + build = 'b', + + /// Build a class instance. + /// + /// This is the protocol 0 version of protocol 1's OBJ opcode. + /// INST is followed by two newline-terminated strings, giving a + /// module and class name, just as for the GLOBAL opcode (and see + /// GLOBAL for more details about that). self.find_class(module, name) + /// is used to get a class object. + /// + /// In addition, all the objects on the stack following the topmost + /// markobject are gathered into a tuple and popped (along with the + /// topmost markobject), just as for the TUPLE opcode. + /// + /// Now it gets complicated. If all of these are true: + /// + /// + The argtuple is empty (markobject was at the top of the stack + /// at the start). + /// + /// + The class object does not have a __getinitargs__ attribute. + /// + /// then we want to create an old-style class instance without invoking + /// its __init__() method (pickle has waffled on this over the years; not + /// calling __init__() is current wisdom). In this case, an instance of + /// an old-style dummy class is created, and then we try to rebind its + /// __class__ attribute to the desired class object. If this succeeds, + /// the new instance object is pushed on the stack, and we're done. + /// + /// Else (the argtuple is not empty, it's not an old-style class object, + /// or the class object does have a __getinitargs__ attribute), the code + /// first insists that the class object have a __safe_for_unpickling__ + /// attribute. Unlike as for the __safe_for_unpickling__ check in REDUCE, + /// it doesn't matter whether this attribute has a true or false value, it + /// only matters whether it exists (XXX this is a bug). If + /// __safe_for_unpickling__ doesn't exist, UnpicklingError is raised. + /// + /// Else (the class object does have a __safe_for_unpickling__ attr), + /// the class object obtained from INST's arguments is applied to the + /// argtuple obtained from the stack, and the resulting instance object + /// is pushed on the stack. + /// + /// NOTE: checks for __safe_for_unpickling__ went away in Python 2.3. + /// NOTE: the distinction between old-style and new-style classes does + /// not make sense in Python 3. + inst = 'i', + + /// Build a class instance. + /// Introduced in protocol 1. + /// + /// This is the protocol 1 version of protocol 0's INST opcode, and is + /// very much like it. The major difference is that the class object + /// is taken off the stack, allowing it to be retrieved from the memo + /// repeatedly if several instances of the same class are created. This + /// can be much more efficient (in both time and space) than repeatedly + /// embedding the module and class names in INST opcodes. + /// + /// Unlike INST, OBJ takes no arguments from the opcode stream. Instead + /// the class object is taken off the stack, immediately above the + /// topmost markobject: + /// + /// Stack before: ... markobject classobject stackslice + /// Stack after: ... new_instance_object + /// + /// As for INST, the remainder of the stack above the markobject is + /// gathered into an argument tuple, and then the logic seems identical, + /// except that no __safe_for_unpickling__ check is done (XXX this is + /// a bug). See INST for the gory details. + /// + /// NOTE: In Python 2.3, INST and OBJ are identical except for how they + /// get the class object. That was always the intent; the implementations + /// had diverged for accidental reasons. + obj = 'o', + + /// Build an object instance. + /// Introduced in protocol 2. + /// + /// The stack before should be thought of as containing a class + /// object followed by an argument tuple (the tuple being the stack + /// top). Call these cls and args. They are popped off the stack, + /// and the value returned by cls.__new__(cls, *args) is pushed back + /// onto the stack. + newobj = 0x81, + + /// Build an object instance. + /// Introduced in protocol 4. + /// + /// The stack before should be thought of as containing a class + /// object followed by an argument tuple and by a keyword argument dict + /// (the dict being the stack top). Call these cls and args. They are + /// popped off the stack, and the value returned by + /// cls.__new__(cls, *args, *kwargs) is pushed back onto the stack. + newobj_ex = 0x92, + + /// Protocol version indicator. + /// Introduced in protocol 2. + /// + /// For protocol 2 and above, a pickle must start with this opcode. + /// The argument is the protocol version, an int in range(2, 256). + proto = 0x80, + + /// Stop the unpickling machine. + /// + /// Every pickle ends with this opcode. The object at the top of the stack + /// is popped, and that's the result of unpickling. The stack should be + /// empty then. + stop = '.', + + /// Indicate the beginning of a new frame. + /// Introduced in protocol 4. + /// + /// The unpickler may use this opcode to safely prefetch data from its + /// underlying stream. + frame = 0x95, + + /// Push an object identified by a persistent ID. + /// + /// The pickle module doesn't define what a persistent ID means. PERSID's + /// argument is a newline-terminated str-style (no embedded escapes, no + /// bracketing quote characters) string, which *is* "the persistent ID". + /// The unpickler passes this string to self.persistent_load(). Whatever + /// object that returns is pushed on the stack. There is no implementation + /// of persistent_load() in Python's unpickler: it must be supplied by an + /// unpickler subclass. + persid = 'P', + + /// Push an object identified by a persistent ID. + /// Introduced in protocol 1. + /// + /// Like PERSID, except the persistent ID is popped off the stack (instead + /// of being a string embedded in the opcode bytestream). The persistent + /// ID is passed to self.persistent_load(), and whatever object that + /// returns is pushed on the stack. See PERSID for more detail. + binpersid = 'Q', + + _, +}; + +// The above enum was generated with the following Python code, +// run inside pickletools.py +// +// def generate_zig(): +// print("""/// All possible pickle operators. +// /// Reference: https://github.com/python/cpython/blob/3.13/Lib/pickletools.py +// pub const OpCode = enum(u8) { +// """) +// for op in opcodes: +// lines = [_cleanup(l) for l in op.doc.split("\n")[:-1]] +// if op.proto > 0: +// lines.insert(1, _cleanup(f"Introduced in protocol {op.proto}.")) +// doc = "\n".join(lines) +// op_code = op.code.__repr__() +// if op_code.startswith("'\\x"): +// op_code = "0x" + op_code[3:-1] +// print(f"""{doc} +// {op.name.lower()} = {op_code}, +// """) +// +// print(" _,") +// print("};") +// +// def _cleanup(line: str) -> str: +// indent = " " +// if (line.startswith(indent)): +// line = line[len(indent):] +// line = line.replace(". ", ". ") +// if line: +// line = " " + line +// line = " ///" + line +// return line + +/// A decoded Pickle operation in its natural state. +/// This is a bit different from Op enum, +/// because operators having same semantics, but different encoding have been merged. +/// ex: string, binstring, short_binstring -> string. +pub const Op = union(enum) { + int: i32, + // Python can represent arbitrary long integers + long: []const u8, + binlong: []const u8, + string: []const u8, + bytes: []const u8, + bytearray: []u8, + next_buffer, + readonly_buffer, + none, + bool: bool, + unicode: []const u8, + float: []const u8, + binfloat: f64, + empty_list, + append, + appends, + list, + empty_tuple, + tuple, + tuple1, + tuple2, + tuple3, + empty_dict, + dict, + setitem, + setitems, + empty_set, + additems, + frozenset, + pop, + dup, + mark, + pop_mark, + get: u32, + put: u32, + memoize, + ext1: u8, + ext2: i16, + ext4: i32, + global: PyType, + stack_global, + reduce, + build, + inst: PyType, + obj, + newobj, + newobj_ex, + proto: u8, + stop, + frame: u64, // new frame and its size + persid: []const u8, + binpersid, + + pub const PyType = struct { module: []const u8, class: []const u8 }; + + pub fn deinit(self: Op, allocator: std.mem.Allocator) void { + switch (self) { + // Use a switch on the type of the stored data, + // this is easier than listing every opcode. + inline else => |v| switch (@TypeOf(v)) { + void, bool, u8, u16, u32, u64, i16, i32, f64 => {}, + []const u8, []u8 => allocator.free(v), + PyType => { + allocator.free(v.module); + allocator.free(v.class); + }, + else => @compileError("please explicit how to free this new opcode: " ++ @typeName(@TypeOf(v))), + }, + } + } + + pub fn clone(self: Op, allocator: std.mem.Allocator) !Op { + return switch (self) { + // Use a switch on the type of the stored data, + // this is easier than listing every opcode. + inline else => |v, tag| switch (@TypeOf(v)) { + void, bool, u8, u16, u32, u64, i16, i32, f64 => self, + []const u8, []u8 => @unionInit(Op, @tagName(tag), try allocator.dupe(u8, v)), + PyType => @unionInit(Op, @tagName(tag), .{ + .module = try allocator.dupe(u8, v.module), + .class = try allocator.dupe(u8, v.class), + }), + else => @compileError("please explicit how to close this new opcode: " ++ @typeName(@TypeOf(v))), + }, + }; + } +}; + +/// Read a stream of bytes, and interpret it as a stream of Pickle operators. +/// The given allocator needs to be an arena cause we are not aligning allocations to avoid copies. +pub fn parse(arena: std.mem.Allocator, reader: *std.Io.Reader) ![]const Op { + // It's not very efficient to interleave the results with the data copied from the stream, + // because growth event in the results ArrayList will lead to fragmentation. + // Trying to mitigate that by using a generous default size. + var results: std.ArrayListUnmanaged(Op) = try .initCapacity(arena, 512); + errdefer results.deinit(arena); + var alloc_writer = try std.Io.Writer.Allocating.initCapacity(arena, 512); + + while (true) { + const code: OpCode = @enumFromInt(try reader.takeByte()); + const op: Op = switch (code) { + .int => int: { + const bytes = try reader.takeDelimiterExclusive('\n'); + // Legacy hack, see OpCode.int documentation + // We do this parsing right away to simplify downstream code. + break :int if (bytes.len == 2 and bytes[0] == '0' and bytes[1] == '0') + .{ .bool = false } + else if (bytes.len == 2 and bytes[0] == '0' and bytes[1] == '1') + .{ .bool = true } + else + .{ .int = try std.fmt.parseInt(i32, bytes, 10) }; + }, + .binint => .{ .int = try reader.takeInt(i32, .little) }, + .binint1 => .{ .int = try reader.takeByte() }, + .binint2 => .{ .int = try reader.takeInt(u16, .little) }, + // TODO: long should handle the trailing 'L' -> add a test. + .long => .{ .long = try readLine(reader, &alloc_writer) }, + .long1 => .{ .binlong = try _readSlice(reader, arena, 1) }, + .long4 => .{ .binlong = try _readSlice(reader, arena, 4) }, + .string => .{ .string = try readLine(reader, &alloc_writer) }, + .binstring => .{ .string = try _readSlice(reader, arena, 4) }, + .short_binstring => .{ .string = try _readSlice(reader, arena, 1) }, + .binbytes => .{ .bytes = try _readSlice(reader, arena, 4) }, + .binbytes8 => .{ .bytes = try _readSlice(reader, arena, 8) }, + .short_binbytes => .{ .bytes = try _readSlice(reader, arena, 1) }, + .bytearray8 => .{ .bytearray = try _readSlice(reader, arena, 8) }, + .next_buffer => .next_buffer, + .readonly_buffer => .readonly_buffer, + .none => .none, + .newtrue => .{ .bool = true }, + .newfalse => .{ .bool = false }, + .unicode => .{ .unicode = try readLine(reader, &alloc_writer) }, + .short_binunicode => .{ .unicode = try _readSlice(reader, arena, 1) }, + .binunicode => .{ .unicode = try _readSlice(reader, arena, 4) }, + .binunicode8 => .{ .unicode = try _readSlice(reader, arena, 8) }, + .float => .{ .float = try readLine(reader, &alloc_writer) }, + .binfloat => .{ .binfloat = @bitCast(try reader.takeInt(u64, .big)) }, + .empty_list => .empty_list, + .append => .append, + .appends => .appends, + .list => .list, + .empty_tuple => .empty_tuple, + .tuple => .tuple, + .tuple1 => .tuple1, + .tuple2 => .tuple2, + .tuple3 => .tuple3, + .empty_dict => .empty_dict, + .dict => .dict, + .setitem => .setitem, + .setitems => .setitems, + .empty_set => .empty_set, + .additems => .additems, + .frozenset => .frozenset, + .pop => .pop, + .dup => .dup, + .mark => .mark, + .pop_mark => .pop_mark, + // If we fail to parse delay the error to the evaluation. + .get => get: { + const digits = try reader.takeDelimiterExclusive('\n'); + break :get .{ .get = std.fmt.parseInt(u32, digits, 10) catch std.math.maxInt(u32) }; + }, + .binget => .{ .get = try reader.takeByte() }, + .long_binget => .{ .get = try reader.takeInt(u32, .little) }, + .put => put: { + const digits = try reader.takeDelimiterExclusive('\n'); + break :put .{ .put = std.fmt.parseInt(u32, digits, 10) catch std.math.maxInt(u32) }; + }, + .binput => .{ .put = try reader.takeByte() }, + .long_binput => .{ .put = try reader.takeInt(u32, .little) }, + .memoize => .memoize, + .ext1 => .{ .ext1 = try reader.takeByte() }, + .ext2 => .{ .ext2 = try reader.takeInt(i16, .little) }, + .ext4 => .{ .ext4 = try reader.takeInt(i32, .little) }, + .global => .{ .global = .{ + .module = try readLine(reader, &alloc_writer), + .class = try readLine(reader, &alloc_writer), + } }, + .stack_global => .stack_global, + .reduce => .reduce, + .build => .build, + .inst => .{ .inst = .{ + .module = try readLine(reader, &alloc_writer), + .class = try readLine(reader, &alloc_writer), + } }, + .obj => .obj, + .newobj => .newobj, + .newobj_ex => .newobj_ex, + .proto => blk: { + const version = try reader.takeByte(); + if (version > 5) log.warn("zml.aio.torch.pickle.parse expects a Python pickle object of version <=5, got version {}. Will try to interpret anyway, but this may lead to more errors.", .{version}); + break :blk .{ .proto = version }; + }, + .stop => .stop, + .frame => frame: { + // This is not documented in pickletools but in https://peps.python.org/pep-3154/ + // The loader is allowed to prefetch framesize from the underlying reader, + // and ops are not allowed to cross a frame boundary. + const frame_size = try reader.takeInt(u64, .little); + reader.fill(@min(frame_size, reader.buffer.len)) catch |err| switch (err) { + error.EndOfStream => {}, + else => return err, + }; + break :frame .{ .frame = frame_size }; + }, + .persid => .{ .persid = try readLine(reader, &alloc_writer) }, + .binpersid => .binpersid, + _ => |unk_tag| { + log.err("Unknow pickle operator {}, note we are only supporting pickle protocol up to version 5.", .{unk_tag}); + return error.NotSupported; + }, + }; + try results.append(arena, op); + if (op == .stop) break; + } + return results.items; +} + +test "parse protocol 4" { + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + + const file = try std.fs.cwd().openFile("zml/aio/torch/simple_test_4.pickle", .{ .mode = .read_only }); + var read_buffer: [1024]u8 = undefined; + var reader = file.reader(&read_buffer); + const ops = try parse(arena.allocator(), &reader.interface); + + // this can be obtained by running: `python -m pickletools simple_test_4.pickle` + var expected = [_]Op{ + .{ .proto = 4 }, + .{ .frame = 119 }, + .empty_dict, + .memoize, + .mark, + .{ .unicode = "hello" }, + .memoize, + .{ .unicode = "world" }, + .memoize, + .{ .unicode = "int" }, + .memoize, + .{ .int = 1 }, + .{ .unicode = "float" }, + .memoize, + .{ .binfloat = 3.141592 }, + .{ .unicode = "list" }, + .memoize, + .empty_list, + .memoize, + .mark, + .{ .int = 255 }, + .{ .int = 1234 }, + .{ .int = -123 }, + .{ .int = 1_000_000_000 }, + .{ .binlong = &writeIntBuff(u48, 999_000_000_000) }, + .{ .binlong = &writeIntBuff(u104, 999_000_000_000_000_000_000_000_000_000) }, + .appends, + .{ .unicode = "bool" }, + .memoize, + .{ .bool = false }, + .{ .unicode = "tuple" }, + .memoize, + .{ .unicode = "a" }, + .memoize, + .{ .int = 10 }, + .tuple2, + .memoize, + .setitems, + .stop, + }; + try std.testing.expectEqualDeep(&expected, ops); +} + +test "parse protocol 0" { + // We also test protocol 0, cause it's more text oriented. + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + + const pickle_0 = + \\(dp0 + \\Vhello + \\p1 + \\Vworld + \\p2 + \\sVint + \\p3 + \\I1 + \\sVfloat + \\p4 + \\F3.141592 + \\sVlist + \\p5 + \\(lp6 + \\I255 + \\aI1234 + \\aI-123 + \\aI1000000000 + \\aL999000000000L + \\aL999000000000000000000000000000L + \\asVbool + \\p7 + \\I00 + \\sVtuple + \\p8 + \\(Va + \\p9 + \\I10 + \\tp10 + \\s. + ; + + var reader: std.Io.Reader = .fixed(pickle_0); + const ops = try parse(arena.allocator(), &reader); + + var expected = [_]Op{ + .mark, + .dict, + .{ .put = 0 }, + .{ .unicode = "hello" }, + .{ .put = 1 }, + .{ .unicode = "world" }, + .{ .put = 2 }, + .setitem, + .{ .unicode = "int" }, + .{ .put = 3 }, + .{ .int = 1 }, + .setitem, + .{ .unicode = "float" }, + .{ .put = 4 }, + .{ .float = "3.141592" }, + .setitem, + .{ .unicode = "list" }, + .{ .put = 5 }, + .mark, + .list, + .{ .put = 6 }, + .{ .int = 255 }, + .append, + .{ .int = 1234 }, + .append, + .{ .int = -123 }, + .append, + .{ .int = 1_000_000_000 }, + .append, + .{ .long = "999000000000L" }, + .append, + .{ .long = "999000000000000000000000000000L" }, + .append, + .setitem, + .{ .unicode = "bool" }, + .{ .put = 7 }, + .{ .bool = false }, + .setitem, + .{ .unicode = "tuple" }, + .{ .put = 8 }, + .mark, + .{ .unicode = "a" }, + .{ .put = 9 }, + .{ .int = 10 }, + .tuple, + .{ .put = 10 }, + .setitem, + .stop, + }; + try std.testing.expectEqualDeep(&expected, ops); +} + +fn _readSlice(reader: anytype, allocator: std.mem.Allocator, comptime len_bytes: u8) ![]u8 { + const T = std.meta.Int(.unsigned, 8 * len_bytes); + const str_len: u64 = try reader.takeInt(T, .little); + const buf = try allocator.alloc(u8, str_len); + _ = try reader.readSliceAll(buf); + return buf; +} + +fn writeIntBuff(comptime T: type, value: T) [@divExact(@typeInfo(T).int.bits, 8)]u8 { + var res: [@divExact(@typeInfo(T).int.bits, 8)]u8 = undefined; + std.mem.writeInt(T, &res, value, .little); + return res; +} + +fn readLine(reader: *std.Io.Reader, alloc_writer: *std.Io.Writer.Allocating) ![]const u8 { + const n = try reader.streamDelimiter(&alloc_writer.writer, '\n'); + std.debug.assert(try reader.takeByte() == '\n'); + const w = &alloc_writer.writer; + std.debug.assert(w.end == n); + const items = w.buffer[0..n]; + w.buffer = w.buffer[n + 1 ..]; + w.end = 0; + return items; +} diff --git a/zml/aio/torch/py.zig b/zml/aio/torch/py.zig new file mode 100644 index 00000000..c523f280 --- /dev/null +++ b/zml/aio/torch/py.zig @@ -0,0 +1,430 @@ +const std = @import("std"); +const math = std.math; +const log = std.log.scoped(.@"zml/aio"); + +const pickle = @import("pickle.zig"); + +/// Correspond to a function/constructor call +pub const Object = struct { + member: Any, + args: []Any, + kwargs: []Any, + + pub fn init(allocator: std.mem.Allocator, member: Any, args: []Any, kwargs: []Any) !*Object { + const self = try allocator.create(Object); + self.* = .{ .member = member, .args = args, .kwargs = kwargs }; + return self; + } + + pub fn clone(self: *Object, allocator: std.mem.Allocator) std.mem.Allocator.Error!*Object { + const res = try allocator.create(Object); + res.* = .{ + .member = try self.member.clone(allocator), + .args = try allocator.alloc(Any, self.args.len), + .kwargs = try allocator.alloc(Any, self.kwargs.len), + }; + for (self.args, 0..) |v, i| res.args[i] = try v.clone(allocator); + for (self.kwargs, 0..) |v, i| res.kwargs[i] = try v.clone(allocator); + return res; + } + + pub fn deinit(self: *Object, allocator: std.mem.Allocator) void { + self.member.deinit(allocator); + for (self.args) |*v| v.deinit(allocator); + allocator.free(self.args); + allocator.destroy(self); + } +}; + +/// Correspond to the __set_state__ call when pickle finishes building an object. +pub const SetState = struct { + obj: Any, + state: Any, + + pub fn init(allocator: std.mem.Allocator, obj: Any, state: Any) !*SetState { + const res = try allocator.create(SetState); + res.* = .{ .obj = obj, .state = state }; + return res; + } + + pub fn clone(self: *SetState, allocator: std.mem.Allocator) std.mem.Allocator.Error!*SetState { + const res = try allocator.create(SetState); + res.* = .{ .obj = try self.obj.clone(allocator), .state = try self.state.clone(allocator) }; + return res; + } + + pub fn deinit(self: *SetState, allocator: std.mem.Allocator) void { + self.obj.deinit(allocator); + self.state.deinit(allocator); + self.allocator.destroy(self); + } +}; + +/// The types of sequences that exist. +pub const SequenceType = enum { + list, + dict, + tuple, + set, + frozen_set, +}; + +pub const Sequence = struct { + type: SequenceType, + values: []Any, +}; + +pub fn tuple(values: []const Any) Any { + // tuple are readonly, but sequence in general aren't + return .{ .seq = .{ .type = .tuple, .values = @constCast(values) } }; +} + +pub const PersId = struct { + ref: Any, + + pub fn init(allocator: std.mem.Allocator, ref: Any) !*PersId { + const self = try allocator.create(PersId); + self.* = .{ .ref = ref }; + return self; + } + + pub fn clone(self: *PersId, allocator: std.mem.Allocator) std.mem.Allocator.Error!*PersId { + const res = try allocator.create(PersId); + res.* = .{ .ref = try self.ref.clone(allocator) }; + return res; + } + + pub fn deinit(self: *PersId, allocator: std.mem.Allocator) void { + self.ref.deinit(allocator); + allocator.destroy(self); + } +}; + +pub const Kind = enum { + raw, + ref, + app, + object, + set_state, + pers_id, + global, + seq, + string, + bytes, + int64, + bigint, + float64, + raw_num, + boolval, + none, +}; + +/// A pickle operator that has been interpreted. +pub const Any = union(Kind) { + /// Types that we can't handle or just had to give up on processing. + raw: pickle.Op, + + /// A reference. You might be able to look it up in the memo map + /// unless there's something weird going on like recursive references. + /// You generally shouldn't see this in the result unless bad things + /// are going on... + ref: u32, + + /// The result of applying a thing to another thing. We're not + /// Python so we don't really know what a "thing" is. + app: *Object, + + /// An object or something. The first tuple member is the + /// thing, the second one is the arguments it got applied to. + object: *Object, + + /// Correspond to the __set_state__ call when pickle finishes building an object. + /// The first tuple member is the target object, + /// the second one is the "state" argument + set_state: *SetState, + + /// References to persistant storage. They basically could be anything. + /// You kind of have to know what the thing you're trying to + /// interface wants to use as keys for persistant storage. + /// Good luck. + pers_id: *PersId, + + /// A global value of some kind. The first tuple member is + /// the thing, the second one is the arguments it got applied to. + global: *Object, + + /// A sequence. We don't really distinguish between them + /// much. The one exception is when the SequenceType is + /// Dict we try to split the flat list of `[k, v, k, v, k, v]` + /// into a list of tuples with the key and value. + seq: Sequence, + + /// A string, but not the crazy strings that have to be + /// unescaped as if they were Python strings. If you + /// need one of those, look for it inside a `Value.raw`. + string: []const u8, + + /// Some bytes. It might be a byte array or a binary + /// string that couldn't get UTF8 decoded. We do the best + /// we can. + bytes: []const u8, + + /// An integer, but not the crazy kind that comes as a string + /// that has to be parsed. You can look in `Value.raw_num` for + /// those. + int64: i64, + + /// An integer that can't fit in i64. + bigint: math.big.int.Const, + + /// An float, but not the crazy kind that comes as a string + /// that has to be parsed. You can look in `Value.raw_num` for + /// those. + float64: f64, + + /// Some kind of weird number we can't handle. + raw_num: pickle.Op, + + /// A boolean value. + boolval: bool, + + /// Python `None`. + none: void, + + pub fn deinit(self: *Any, allocator: std.mem.Allocator) void { + switch (self.*) { + .raw, .raw_num => |v| v.deinit(allocator), + inline .app, .object, .global, .build, .pers_id => |v| v.deinit(), + .seq => |v| { + for (v.values) |*val| val.deinit(allocator); + allocator.free(v.values); + }, + .string, .bytes => |v| allocator.free(v), + .bigint => |big| allocator.free(big.limbs), + else => {}, + } + self.* = undefined; + } + + inline fn writeIndents(indents: usize, writer: anytype) !void { + try writer.writeBytesNTimes(" ", indents); // resolve tab = 2 spaces + // try writer.writeByteNTimes('\t'); + } + + fn internalFormat(value: Any, indents: usize, writer: anytype) !void { + try writeIndents(indents, writer); + try writer.writeAll(".{\n"); + try writeIndents(indents + 1, writer); + try writer.print(".{s} = ", .{@tagName(std.meta.activeTag(value))}); + switch (value) { + inline .ref, .int64, .float64 => |v| try writer.print("{d} ", .{v}), + .app, .object, .global => |v| { + try writer.writeAll(".{\n"); + try writeIndents(indents + 2, writer); + try writer.writeAll(".fn ="); + try internalFormat(v.member, indents + 2, writer); + try writer.writeAll(",\n"); + try writeIndents(indents + 2, writer); + try writer.writeAll(".args = "); + if (v.args.len > 0) { + try writer.writeAll(".{\n"); + for (v.args, 0..) |arg, i| { + try internalFormat(arg, indents + 3, writer); + if (i < v.args.len - 1) try writer.writeAll(","); + try writer.writeByte('\n'); + } + try writeIndents(indents + 2, writer); + try writer.writeAll("},\n"); + } else { + try writer.writeAll(".{},\n"); + } + try writeIndents(indents + 2, writer); + try writer.writeAll(".kwargs ="); + if (v.kwargs.len > 0) { + try writer.writeAll(".{\n"); + for (v.kwargs, 0..) |arg, i| { + try internalFormat(arg, indents + 3, writer); + if (i < v.kwargs.len - 1) try writer.writeAll(","); + try writer.writeByte('\n'); + } + try writeIndents(indents + 2, writer); + try writer.writeAll("}\n"); + } else { + try writer.writeAll(".{}\n"); + } + try writeIndents(indents + 1, writer); + try writer.writeAll("}"); + }, + .set_state => |v| { + try writer.writeAll(".{\n"); + try internalFormat(v.obj, indents + 2, writer); + try writer.writeAll(",\n"); + try internalFormat(v.state, indents + 2, writer); + try writer.writeAll(",\n"); + try writeIndents(indents + 1, writer); + try writer.writeAll("}"); + }, + .pers_id => |v| { + try writer.writeByte('\n'); + try internalFormat(v.ref, indents + 2, writer); + }, + .seq => |v| { + try writer.writeAll(".{\n"); + try writeIndents(indents + 2, writer); + try writer.print(".{s},\n", .{@tagName(v.type)}); + try writeIndents(indents + 2, writer); + if (v.values.len > 0) { + try writer.writeAll(".{\n"); + for (v.values, 0..) |arg, i| { + try internalFormat(arg, indents + 3, writer); + if (i < v.values.len - 1) try writer.writeAll(","); + try writer.writeByte('\n'); + } + try writeIndents(indents + 2, writer); + try writer.writeAll("}\n"); + } else { + try writer.writeAll(".{}\n"); + } + + try writeIndents(indents + 1, writer); + try writer.writeAll("}"); + }, + .string => |v| try writer.print("\"{s}\"", .{v}), + .raw => |v| switch (v) { + .global => |py_type| try writer.print("\"{s}\", \"{s}\"", .{ py_type.module, py_type.class }), + else => try writer.print("{any}", .{v}), + }, + inline else => |v| { + try writer.print("{any}", .{v}); + }, + } + try writer.writeByte('\n'); + try writeIndents(indents, writer); + try writer.writeByte('}'); + } + + pub fn format(self: Any, comptime _: []const u8, _: std.fmt.FormatOptions, writer: anytype) !void { + return internalFormat(self, 0, writer); + } + + pub fn clone(self: Any, allocator: std.mem.Allocator) !Any { + return switch (self) { + inline .raw, .raw_num => |v, tag| @unionInit(Any, @tagName(tag), try v.clone(allocator)), + inline .app, .object, .global, .set_state, .pers_id => |v, tag| @unionInit(Any, @tagName(tag), try v.clone(allocator)), + .seq => |seq| { + const values = try allocator.alloc(Any, seq.values.len); + for (seq.values, 0..) |v, i| values[i] = try v.clone(allocator); + return .{ .seq = .{ .type = seq.type, .values = values } }; + }, + inline .string, .bytes => |v, tag| @unionInit(Any, @tagName(tag), try allocator.dupe(u8, v)), + .bigint => |v| .{ .bigint = (try v.toManaged(allocator)).toConst() }, + else => self, + }; + } + + pub fn isPrimitive(self: Any) bool { + return switch (self) { + .int64, .bigint, .float64, .string, .bytes, .boolval, .none => true, + .seq => |seq| { + for (seq.values) |v| { + if (!v.isPrimitive()) return false; + } + return true; + }, + else => false, + }; + } + + pub fn containsRef(self: Any) bool { + switch (self) { + .ref => return true, + .app, .object, .global => |v| { + if (v.member.containsRef()) return true; + for (v.args) |arg| if (arg.containsRef()) return true; + return false; + }, + .set_state => |v| { + if (v.obj.containsRef()) return true; + if (v.state.containsRef()) return true; + return false; + }, + .pers_id => |v| return v.ref.containsRef(), + .seq => |v| { + for (v.values) |val| if (val.containsRef()) return true; + return false; + }, + else => return false, + } + } + + pub const UnpickleError = error{ InvalidCharacter, OutOfMemory }; + + pub fn coerceFromRaw(self: Any, allocator: std.mem.Allocator) UnpickleError!Any { + return switch (self) { + .raw => |raw_val| switch (raw_val) { + .none => .none, + .bool => |b| .{ .boolval = b }, + .float => |b| .{ .float64 = std.fmt.parseFloat(f64, b) catch std.math.nan(f64) }, + .int => |val| .{ .int64 = val }, + .long => |digits| { + const n = std.fmt.parseInt(i64, digits[0 .. digits.len - 1], 10) catch |err| { + switch (err) { + error.Overflow => { + log.warn("Not parsing long integer: {s}", .{digits}); + return self; + }, + error.InvalidCharacter => return error.InvalidCharacter, + } + }; + return .{ .int64 = n }; + }, + .binlong => |bytes| if (bytes.len <= 8) + .{ .int64 = std.mem.readVarInt(i64, bytes, .little) } + else { + // Note: we need to copy here, because Zig big int limbs are usize aligned, + // whereas pickle big int are byte aligned. + const n_limbs = std.math.divCeil(usize, bytes.len, @sizeOf(math.big.Limb)) catch unreachable; + var big = (try math.big.int.Managed.initCapacity(allocator, n_limbs)).toMutable(); + big.readTwosComplement(bytes, bytes.len * 8, .little, .signed); + + return .{ .bigint = big.toConst() }; + }, + .binfloat => |val| .{ .float64 = val }, + .unicode => |s| .{ .string = s }, + inline .bytes, .bytearray => |b| .{ .bytes = b }, + // This isn't how Pickle actually works but we just try to UTF8 decode the + // string and if it fails, we make it a bytes value instead. If anyone + // actually cares they can just fix values themselves or recover the raw bytes + // from the UTF8 string (it's guaranteed to be reversible, as far as I know). + .string => |b| if (std.unicode.utf8ValidateSlice(b)) + .{ .string = b } + else + .{ .bytes = b }, + else => self, + }, + .app, .object, .global => |v| blk: { + v.member = try v.member.coerceFromRaw(allocator); + for (v.args) |*arg| { + arg.* = try arg.coerceFromRaw(allocator); + } + break :blk self; + }, + .set_state => |v| blk: { + v.obj = try v.obj.coerceFromRaw(allocator); + v.state = try v.state.coerceFromRaw(allocator); + break :blk self; + }, + .pers_id => |v| blk: { + v.ref = try v.ref.coerceFromRaw(allocator); + break :blk self; + }, + .seq => |v| blk: { + for (v.values) |*val| { + val.* = try val.coerceFromRaw(allocator); + } + break :blk self; + }, + else => self, + }; + } +}; diff --git a/zml/aio/torch/simple.pt b/zml/aio/torch/simple.pt new file mode 100644 index 0000000000000000000000000000000000000000..f6978e65eaa6a8429f4ced64c8370b7294eefa6f GIT binary patch literal 2708 zcmbtWOKjX!6!pxc`M4!%6I%Ks6bLYZW-`BIlK#?8%JgGsr_&Zpp;(UnGGoQH=h|-) zTB#CUByAV4V8Idz#G+LpsObWU4Qtqe0HJJIuz>{<3nYZZz0dPc(~9y&FOJ{+oO|E9 z=lbQ9NQWeKcS~!=W~pD&J)>mJvr$v$daOKW=d!)hg_iMd_$o+Ibpx7Ta!qDK$lxwo z#T+Mw3l*FBF~fD{$vxe2ET;$?Zm1jaT`TLJeu;7B`Os+? zu5B}eTdw2Fr#+K-%sgrtT!yaU0(3{!PPABHR25T}rriM@@4!vpwpGLU?eaaAr&1V!T zs)X?uQ6e0+M3sJ}2mcRo=)YN_hzB@4UVyDP#LKqp@I(Q&tGN5pjHUaqqa>FQzqF|o z03={%sZR-W*oBUj@FMJ1LW}TZ;A)V=Qw7))xO)0JJX3&YD`H2rnab=eZQqc9c~0Gh z0Gi`ES{YX@AG75(9nEJPo;Ngnw`^`XnkM6l3@==S=ptamFRBA%J);|Q3%ZB9=`pS3 zx^q6fB)soc`IY(&Un>Nb*DzBXI2?S>E1fzeZHFG*)zPdE+?iZ%7vhXMcwcb~jG*S@G#0hA z1bG2g^xR6>hcik@L|WC+sG~qZG?y#PbA33g?m^#3?s=?bM(h>=;X;|=nV1@1unY$0 zqDlw;Fmev(D>DKjU^%=NaIDJf>}AgCynbEqIteP$u7krHM7t2>c>#T4Vs8f2dyB)x z0=!*`<(01P!>j(6e)0uQSGm)4~p)4cL=$oEzk3=y-nt3He!Pajc z%|zNvf|k*K4{5h<<_@*!#!&EPQnRj8uid=4aOtzu7#}v72Xt$GObt#Cc@{wL7%wl1M+${TH=J!j}L5 literal 0 HcmV?d00001 diff --git a/zml/aio/torch/simple_test_4.pickle b/zml/aio/torch/simple_test_4.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4a5176590cd0d4a7aa79b39b80fb411b6513808e GIT binary patch literal 130 zcmZo*nOe>O0ku;!dRQ}3b8_;h^stua7v-c(>0!>yE1BZW*u$EZlb={J#od8Z@zQxoXp~qDX~*Dy#M=NV)1JI|Np-i!>L);U2F_)Tr*!VbnyZWjaZwzWZC4rkF$AF fdsve4^K+(j_OO0wNq;?31IrBn|9AKNkd literal 0 HcmV?d00001 diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 117cb1e4..c2099b0d 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -172,7 +172,7 @@ pub const HostBuffer = struct { // TODO we should allow interpreting the output as @Vector(8, f32) when the tensor is f32. stdx.debug.assert(DataType.fromZigType(T) == self.dtype(), "Can't reinterpret {f} as {s}", .{ self, @typeName(T) }); stdx.debug.assert(self.isContiguous(), "{f} isn't contiguous, can't interpret as []const u8", .{self}); - const ptr: [*]const T = @alignCast(@ptrCast(self._data)); + const ptr: [*]const T = @ptrCast(@alignCast(self._data)); return ptr[0..self._shape.count()]; } diff --git a/zml/module.zig b/zml/module.zig index 4e40882e..e57f942c 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -664,7 +664,7 @@ pub const CompilationContext = struct { // Create the result tensor object by combining the operand results, // as well as the registered shapes and donations. // Note: this assume res can be stack-allocated. - var res = @as(*const stdx.meta.FnResult(func), @alignCast(@ptrCast(function.res_tensors))).*; + var res = @as(*const stdx.meta.FnResult(func), @ptrCast(@alignCast(function.res_tensors))).*; const LocalContext = struct { index: usize = 0, op: mlir.Operation, function: MlirFn, donations: []Tensor._Donation }; var context: LocalContext = .{ .op = op, .function = function, .donations = donations }; meta.visit((struct { diff --git a/zml/platform.zig b/zml/platform.zig index 0bd8a2a5..838cac48 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -113,7 +113,7 @@ const _CreateOptions = struct { /// "Best-Fit with Coalescing" algorithm bfc: Options, /// use cudaMallocAsync - @"async": Options, + async: Options, /// use raw cuMalloc platform, @@ -129,7 +129,7 @@ const _CreateOptions = struct { .platform => { values.appendAssumeCapacity(pjrt.NamedValue.fromString("allocator", "platform")); }, - .bfc, .@"async" => |opt| { + .bfc, .async => |opt| { values.appendAssumeCapacity(pjrt.NamedValue.from("allocator", self.allocator)); values.appendAssumeCapacity(pjrt.NamedValue.from("preallocate", opt.preallocate)); if (opt.memory_fraction > 0) { diff --git a/zml/tools/tracer.zig b/zml/tools/tracer.zig index 06bcd6fa..d369b1e2 100644 --- a/zml/tools/tracer.zig +++ b/zml/tools/tracer.zig @@ -1,4 +1,5 @@ const builtin = @import("builtin"); + const c = @import("c"); pub const Tracer = switch (builtin.os.tag) { @@ -11,15 +12,15 @@ const CudaTracer = struct { // Those symbols are defined in cudaProfiler.h but their implementation is in libcuda.so // They will be bound at call time after libcuda.so is loaded (as a needed dependency of libpjrt_cuda.so). - const cuProfilerStart = @extern(*const fn () callconv(.C) c_int, .{ .name = "cuProfilerStart", .linkage = .weak }) orelse unreachable; - const cuProfilerStop = @extern(*const fn () callconv(.C) c_int, .{ .name = "cuProfilerStop", .linkage = .weak }) orelse unreachable; + const cuProfilerStart = @extern(*const fn () callconv(.c) c_int, .{ .name = "cuProfilerStart", .linkage = .weak }) orelse unreachable; + const cuProfilerStop = @extern(*const fn () callconv(.c) c_int, .{ .name = "cuProfilerStop", .linkage = .weak }) orelse unreachable; // Those symbols are defined in nvToolsExt.h which we don't want to provide. // However, we link with libnvToolsExt.so which provides them. // They will be bound at call time after libnvToolsExt.so is loaded (manually dlopen'ed by us). - const nvtxMarkA = @extern(*const fn ([*:0]const u8) callconv(.C) void, .{ .name = "nvtxMarkA", .linkage = .weak }) orelse unreachable; - const nvtxRangeStartA = @extern(*const fn ([*:0]const u8) callconv(.C) c_int, .{ .name = "nvtxRangeStartA", .linkage = .weak }) orelse unreachable; - const nvtxRangeEnd = @extern(*const fn (c_int) callconv(.C) void, .{ .name = "nvtxRangeEnd", .linkage = .weak }) orelse unreachable; + const nvtxMarkA = @extern(*const fn ([*:0]const u8) callconv(.c) void, .{ .name = "nvtxMarkA", .linkage = .weak }) orelse unreachable; + const nvtxRangeStartA = @extern(*const fn ([*:0]const u8) callconv(.c) c_int, .{ .name = "nvtxRangeStartA", .linkage = .weak }) orelse unreachable; + const nvtxRangeEnd = @extern(*const fn (c_int) callconv(.c) void, .{ .name = "nvtxRangeEnd", .linkage = .weak }) orelse unreachable; pub fn init(name: [:0]const u8) CudaTracer { _ = name; From 5d6c04ec6939738b9bfb59fc2f693e1fce5556d5 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 3 Sep 2025 14:23:22 +0200 Subject: [PATCH 025/881] ci: revamp commit workflow (#320) - Deduplicate workflows (introduced by https://github.com/zml/zml/commit/57fd9cda27a51fddaf3bae92180fea2d37e3f20) - Fix linking issue on Linux since upb was introduced - Fix duplicated `_start` symbol for some tests targets - Temporarily use `-c opt` instead of `--config=debug` to bypass static archive bug --- .bazelrc | 3 + .../{commit-framework.yaml => ci.yaml} | 7 +- .github/workflows/commit.yaml | 70 ------------------- async/BUILD.bazel | 1 + platforms/BUILD.bazel | 6 ++ stdx/BUILD.bazel | 1 + 6 files changed, 15 insertions(+), 73 deletions(-) rename .github/workflows/{commit-framework.yaml => ci.yaml} (93%) delete mode 100644 .github/workflows/commit.yaml diff --git a/.bazelrc b/.bazelrc index 003a9e82..cbbb4386 100644 --- a/.bazelrc +++ b/.bazelrc @@ -87,6 +87,9 @@ common:macos --macos_sdk_version=13.0 # This is done only on Linux for now since we use the local toolchain on macOS common:linux --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 +# Required for protobuf upb on Linux +common:linux --copt=-fPIC + # Print test errors in the console test --test_output=errors diff --git a/.github/workflows/commit-framework.yaml b/.github/workflows/ci.yaml similarity index 93% rename from .github/workflows/commit-framework.yaml rename to .github/workflows/ci.yaml index 141630fe..7af596eb 100644 --- a/.github/workflows/commit-framework.yaml +++ b/.github/workflows/ci.yaml @@ -1,4 +1,4 @@ -name: framework-tests +name: ci on: push: branches: @@ -12,7 +12,7 @@ concurrency: cancel-in-progress: ${{ !contains(github.ref, 'master')}} jobs: - framework-tests: + ci: runs-on: ["runs-on","runner=4cpu-linux-x64", "image=ubuntu24-amd64", "run-id=${{ github.run_id }}", "extras=s3-cache"] steps: - uses: runs-on/action@v1 @@ -43,8 +43,9 @@ jobs: with: bazelisk-version: 1.26.0 bazelisk-cache: false + # todo: use `--config=debug` instead of `-c opt` when static archive bug is fixed bazelrc: | - common --config=debug + common -c opt common --verbose_failures common --keep_going common --test_output=streamed diff --git a/.github/workflows/commit.yaml b/.github/workflows/commit.yaml deleted file mode 100644 index 1ce4b5c9..00000000 --- a/.github/workflows/commit.yaml +++ /dev/null @@ -1,70 +0,0 @@ -name: framework-tests -on: - push: - branches: - - master - tags: - - "*" - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ !contains(github.ref, 'master')}} - -jobs: - framework-tests: - runs-on: - [ - "runs-on", - "runner=4cpu-linux-x64", - "image=ubuntu24-amd64", - "run-id=${{ github.run_id }}", - "extras=s3-cache", - ] - steps: - - uses: runs-on/action@v1 - - name: Checkout - uses: actions/checkout@v4 - - - uses: runs-on/snapshot@v1 - with: - path: /home/runner/.cache/bazel-disk - volume_size: 350 - - - name: TMP fix for the mount user - run: sudo chown -R runner:runner /home/runner/.cache/ - - - name: Setup Zig - uses: mlugg/setup-zig@v2 - with: - version: 0.15.1 - mirror: https://mirror.zml.ai/zig - - - name: Zig Format - run: zig fmt --check $(git ls-files --exclude-standard -- ':!third_party/*' '*.zig' | xargs) - - - name: No autofix in Zig code - run: bash -c "! git grep -l '// autofix' -- '*.zig'" - - - uses: bazel-contrib/setup-bazel@0.15.0 - with: - bazelisk-version: 1.26.0 - bazelisk-cache: false - bazelrc: | - common --config=debug - common --verbose_failures - common --keep_going - common --test_output=streamed - common --color=yes - common --show_timestamps - common --disk_cache=/home/runner/.cache/bazel-disk - common --strip=always - - - name: ZML / bazel mod explain - run: bazel mod explain - - - name: ZML / bazel mod graph - run: bazel mod graph - - - name: ZML / bazel test //... - run: bazel test //examples/... //mlir/... //zml/... diff --git a/async/BUILD.bazel b/async/BUILD.bazel index 3f3c7dca..17cdcf92 100644 --- a/async/BUILD.bazel +++ b/async/BUILD.bazel @@ -10,6 +10,7 @@ zig_library( "executor.zig", "stack.zig", ], + copts = ["-lc"], extra_srcs = glob(["asm/*.s"]), main = "async.zig", visibility = ["//visibility:public"], diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index 45fddc0b..62a36b81 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -4,6 +4,9 @@ platform( "@platforms//cpu:x86_64", "@platforms//os:linux", ], + flags = [ + "--force_pic=true", # required for protobuf upb on Linux + ], visibility = ["//visibility:public"], ) @@ -13,6 +16,9 @@ platform( "@platforms//cpu:aarch64", "@platforms//os:linux", ], + flags = [ + "--force_pic=true", # required for protobuf upb on Linux + ], visibility = ["//visibility:public"], ) diff --git a/stdx/BUILD.bazel b/stdx/BUILD.bazel index e44bda25..2594731f 100644 --- a/stdx/BUILD.bazel +++ b/stdx/BUILD.bazel @@ -15,6 +15,7 @@ zig_library( "signature.zig", "time.zig", ], + copts = ["-lc"], main = "stdx.zig", visibility = ["//visibility:public"], ) From 10d3da2449f035ef112371434d6a9372f746a077 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 3 Sep 2025 14:24:04 +0200 Subject: [PATCH 026/881] xla: bump to b3fbfee (#321) - Temporarily disable libnvptxcompiler usage since it's not available in the PjRT CUDA plugin v13.0 - Add nvshmem in the sandbox, required by the PjRT CUDA plugin --- pjrt/pjrt.zig | 1 - runtimes/cpu/cpu.bzl | 12 +- runtimes/cuda/cuda.bzl | 39 ++++- runtimes/cuda/libpjrt_cuda.BUILD.bazel | 2 +- runtimes/rocm/rocm.bzl | 4 +- runtimes/tpu/tpu.bzl | 4 +- .../0001-bazel-migration-to-bazel-8.1.1.patch | 41 ------ ...ler-registration-API-to-the-FFI-PjRt.patch | 135 ------------------ ...ove-unconventional-C-code-in-headers.patch | 124 ---------------- third_party/xla/repo.bzl | 6 +- third_party/xla/xla.bzl | 10 +- zml/module.zig | 1 - 12 files changed, 54 insertions(+), 325 deletions(-) delete mode 100644 third_party/xla/patches/0001-bazel-migration-to-bazel-8.1.1.patch delete mode 100644 third_party/xla/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch delete mode 100644 third_party/xla/patches/0003-Remove-unconventional-C-code-in-headers.patch diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 24581496..6e358562 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -1308,7 +1308,6 @@ pub const FFI = extern struct { options: RegisterFfiOptions, ) ApiError!void { var ret = pjrtStruct(c.PJRT_FFI_Register_Handler_Args{ - .api_version = 1, .target_name = target_name.ptr, .target_name_size = target_name.len, .handler = @ptrCast(@constCast(func)), diff --git a/runtimes/cpu/cpu.bzl b/runtimes/cpu/cpu.bzl index e4d5536d..dbc69d7b 100644 --- a/runtimes/cpu/cpu.bzl +++ b/runtimes/cpu/cpu.bzl @@ -23,22 +23,22 @@ def _cpu_pjrt_plugin_impl(mctx): http_archive( name = "libpjrt_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "cf5ea44b14a6ddc320c5b1d7bb88328dda099571e106b17a214b6ec586e321b8", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v12.0.0/pjrt-cpu_linux-amd64.tar.gz", + sha256 = "124dc500291a5930f910ca23533520e22c90797110b29fd2c0d8274475f4a220", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v13.0.0/pjrt-cpu_linux-amd64.tar.gz", ) http_archive( name = "libpjrt_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "1b48c26b0d2709730df6921a86d9ae97ab71d3f2cfa17fd1a370e85235370914", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v12.0.0/pjrt-cpu_darwin-amd64.tar.gz", + sha256 = "6e5b59874880f4db37c53fb1d52520d410b0078f9d2606a90762c6c622693c26", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v13.0.0/pjrt-cpu_darwin-amd64.tar.gz", ) http_archive( name = "libpjrt_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "66dc6c65933a6d7985b1c69837d8abe13750cd1f06704427f6989c3f952d3511", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v12.0.0/pjrt-cpu_darwin-arm64.tar.gz", + sha256 = "e54bc3c3b71313c49e38fc342851bffe2cd3a55e3990839947d49496a1a71270", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v13.0.0/pjrt-cpu_darwin-arm64.tar.gz", ) return mctx.extension_metadata( diff --git a/runtimes/cuda/cuda.bzl b/runtimes/cuda/cuda.bzl index 9920103b..a99d4607 100644 --- a/runtimes/cuda/cuda.bzl +++ b/runtimes/cuda/cuda.bzl @@ -17,6 +17,10 @@ CUDNN_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cudnn/redis CUDNN_VERSION = "9.8.0" CUDNN_REDIST_JSON_SHA256 = "a1599fa1f8dcb81235157be5de5ab7d3936e75dfc4e1e442d07970afad3c4843" +NVSHMEM_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/nvshmem/redist/" +NVSHMEM_VERSION = "3.2.5" +NVSHMEM_REDIST_JSON_SHA256 = "6945425d3bfd24de23c045996f93ec720c010379bfd6f0860ac5f2716659442d" + _UBUNTU_PACKAGES = { "zlib1g": packages.filegroup(name = "zlib1g", srcs = ["lib/x86_64-linux-gnu/libz.so.1"]), } @@ -119,6 +123,17 @@ CUDNN_PACKAGES = { ]), } +NVSHMEM_PACKAGES = { + "libnvshmem": packages.filegroup( + name = "libnvshmem", + srcs = [ + "lib/libnvshmem_host.so.3", + "lib/nvshmem_bootstrap_uid.so.3", + "lib/nvshmem_transport_ibrc.so.3", + ], + ), +} + def _read_redist_json(mctx, url, sha256): fname = ".{}.json".format(sha256) mctx.download( @@ -138,6 +153,12 @@ def _cuda_impl(mctx): sha256 = CUDA_REDIST_JSON_SHA256, ) + NVSHMEM_REDIST = _read_redist_json( + mctx, + url = NVSHMEM_REDIST_PREFIX + "redistrib_{}.json".format(NVSHMEM_VERSION), + sha256 = NVSHMEM_REDIST_JSON_SHA256, + ) + CUDNN_REDIST = _read_redist_json( mctx, url = CUDNN_REDIST_PREFIX + "redistrib_{}.json".format(CUDNN_VERSION), @@ -180,6 +201,20 @@ def _cuda_impl(mctx): strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), ) + for pkg, build_file_content in NVSHMEM_PACKAGES.items(): + pkg_data = NVSHMEM_REDIST[pkg] + arch_data = pkg_data.get(ARCH) + if not arch_data: + continue + arch_data = arch_data.get("cuda12", arch_data) + http_archive( + name = pkg, + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, + url = NVSHMEM_REDIST_PREFIX + arch_data["relative_path"], + sha256 = arch_data["sha256"], + strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), + ) + http_archive( name = "nccl", urls = ["https://files.pythonhosted.org/packages/11/0c/8c78b7603f4e685624a3ea944940f1e75f36d71bd6504330511f4a0e1557/nvidia_nccl_cu12-2.25.1-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl"], @@ -194,8 +229,8 @@ def _cuda_impl(mctx): http_archive( name = "libpjrt_cuda", build_file = "libpjrt_cuda.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v11.0.0/pjrt-cuda_linux-amd64.tar.gz", - sha256 = "08fa022a6067ddfb5c951bdf11ddc398e63de21fdcacc9ffd07f70b1463482c2", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v13.0.0/pjrt-cuda_linux-amd64.tar.gz", + sha256 = "6cdac9bac6db904e4423c9745c61000cf3acaf3c7da8016ab0016f076869048a", ) return mctx.extension_metadata( diff --git a/runtimes/cuda/libpjrt_cuda.BUILD.bazel b/runtimes/cuda/libpjrt_cuda.BUILD.bazel index 031ec507..f63f0ff9 100644 --- a/runtimes/cuda/libpjrt_cuda.BUILD.bazel +++ b/runtimes/cuda/libpjrt_cuda.BUILD.bazel @@ -1,5 +1,4 @@ load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") -load("@zml//bazel:cc_import.bzl", "cc_import") load("@zml//bazel:patchelf.bzl", "patchelf") cc_shared_library( @@ -30,6 +29,7 @@ copy_to_directory( "@cuda_nvrtc", "@cuda_nvtx", "@cudnn", + "@libnvshmem", "@libcublas", "@libcufft", "@libcusolver", diff --git a/runtimes/rocm/rocm.bzl b/runtimes/rocm/rocm.bzl index 1d11da3e..e71f1a92 100644 --- a/runtimes/rocm/rocm.bzl +++ b/runtimes/rocm/rocm.bzl @@ -153,8 +153,8 @@ def _rocm_impl(mctx): http_archive( name = "libpjrt_rocm", build_file = "libpjrt_rocm.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v12.0.0/pjrt-rocm_linux-amd64.tar.gz", - sha256 = "709982b959595750545a01d125adf4893c42f05c60ec290425276bba8aa49f64", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v13.0.0/pjrt-rocm_linux-amd64.tar.gz", + sha256 = "945c43c68325c0e91cd41eaa594a9f9f6e78da7cc06892d83bf345b69f7bd714", ) return mctx.extension_metadata( diff --git a/runtimes/tpu/tpu.bzl b/runtimes/tpu/tpu.bzl index d8384152..e637d8dd 100644 --- a/runtimes/tpu/tpu.bzl +++ b/runtimes/tpu/tpu.bzl @@ -4,9 +4,9 @@ def _tpu_impl(mctx): # https://storage.googleapis.com/jax-releases/libtpu_releases.html http_archive( name = "libpjrt_tpu", - url = "https://storage.googleapis.com/libtpu-nightly-releases/wheels/libtpu-nightly/libtpu_nightly-0.1.dev20250102+nightly-py3-none-linux_x86_64.whl", + url = "https://storage.googleapis.com/libtpu-nightly-releases/wheels/libtpu-nightly/libtpu_nightly-0.1.dev20250807+nightly-py3-none-manylinux_2_31_x86_64.whl", type = "zip", - sha256 = "df8339b4f852bd19ad4ed380facc08f28c04e214e9dabb88863e70907b08817e", + sha256 = "41c19fa5ae4a32fbd05f0260527ba2d93afb6cf128e6c4de7773e9011c7b3df5", build_file = "libpjrt_tpu.BUILD.bazel", ) return mctx.extension_metadata( diff --git a/third_party/xla/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/xla/patches/0001-bazel-migration-to-bazel-8.1.1.patch deleted file mode 100644 index 8924cf4b..00000000 --- a/third_party/xla/patches/0001-bazel-migration-to-bazel-8.1.1.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 -From: Jean-Baptiste Dalido -Date: Mon, 6 Jan 2025 13:33:13 +0100 -Subject: [PATCH] bazel: migration to bazel 8.0.1 - ---- - .bazelversion | 2 +- - third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/.bazelversion b/.bazelversion -index f22d756da3..fa5fce04b3 100644 ---- a/.bazelversion -+++ b/.bazelversion -@@ -1 +1 @@ --7.4.1 -+8.1.1 -\ No newline at end of file -diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl -index d62531152d..71d80a5a99 100644 ---- a/third_party/gpus/cuda_configure.bzl -+++ b/third_party/gpus/cuda_configure.bzl -@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. - load( - "@bazel_tools//tools/cpp:lib_cc_configure.bzl", - "escape_string", -- "get_env_var", - ) - load( - "@bazel_tools//tools/cpp:windows_cc_configure.bzl", -- "find_msvc_tool", - "find_vc_path", - "setup_vc_env_vars", - ) -+load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") -+load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") - load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") - load( - "//third_party/remote_config:common.bzl", --- -2.39.3 (Apple Git-146) diff --git a/third_party/xla/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/xla/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch deleted file mode 100644 index 938ef401..00000000 --- a/third_party/xla/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch +++ /dev/null @@ -1,135 +0,0 @@ -From 2ae9bb9d24b569c2c6bfab3c54b428103614944d Mon Sep 17 00:00:00 2001 -From: Hugo Mano -Date: Tue, 27 May 2025 11:48:17 +0200 -Subject: [PATCH 1/8] Added FFI handler registration API to the FFI PjRt - -PR: https://github.com/openxla/xla/pull/13420 ---- - xla/pjrt/c/BUILD | 5 +++++ - xla/pjrt/c/pjrt_c_api_ffi_extension.h | 21 ++++++++++++++++++ - xla/pjrt/c/pjrt_c_api_ffi_internal.cc | 32 ++++++++++++++++++++++++++- - 3 files changed, 57 insertions(+), 1 deletion(-) - -diff --git a/xla/pjrt/c/BUILD b/xla/pjrt/c/BUILD -index 79f18fa0bc..0f33dd8a6e 100644 ---- a/xla/pjrt/c/BUILD -+++ b/xla/pjrt/c/BUILD -@@ -69,8 +69,13 @@ cc_library( - ":pjrt_c_api_wrapper_impl", - "//xla/ffi:execution_context", - "//xla/ffi:type_id_registry", -+ "//xla/ffi:ffi_api", -+ "//xla/ffi/api:c_api", -+ "//xla/ffi/api:ffi", -+ "//xla/service:custom_call_target_registry", - "@com_google_absl//absl/status", - "@com_google_absl//absl/strings:string_view", -+ "@com_google_absl//absl/strings:str_format", - ], - ) - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -index 995a2c7e50..b8f10bc2f7 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -@@ -69,10 +69,31 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); - // Adds a user data to the execute context. - typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); - -+typedef enum PJRT_FFI_Handler_TraitsBits { -+ PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE = 1u << 0, -+} PJRT_FFI_Handler_TraitsBits; -+ -+struct PJRT_FFI_Register_Handler_Args { -+ size_t struct_size; -+ const char* target_name; -+ size_t target_name_size; -+ int api_version; // 0 for an untyped call, 1 -- for typed -+ void* handler; -+ const char* platform_name; -+ size_t platform_name_size; -+ PJRT_FFI_Handler_TraitsBits traits; -+}; -+PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, traits); -+ -+// Registers an FFI call handler for a specific platform. -+typedef PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args); -+ - typedef struct PJRT_FFI_Extension { - PJRT_Extension_Base base; - PJRT_FFI_TypeID_Register* type_id_register; - PJRT_FFI_UserData_Add* user_data_add; -+ PJRT_FFI_Register_Handler* register_handler; - } PJRT_FFI; - PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Extension, user_data_add); - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -index 5fa88eab33..763270331b 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -+++ b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -@@ -13,16 +13,20 @@ See the License for the specific language governing permissions and - limitations under the License. - ==============================================================================*/ - --#include "xla/pjrt/c/pjrt_c_api_ffi_internal.h" -+#include - - #include "absl/status/status.h" -+#include "absl/strings/str_format.h" - #include "absl/strings/string_view.h" -+#include "xla/ffi/api/c_api.h" - #include "xla/ffi/execution_context.h" - #include "xla/ffi/type_id_registry.h" -+#include "xla/ffi/ffi_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" - #include "xla/pjrt/c/pjrt_c_api_ffi_extension.h" - #include "xla/pjrt/c/pjrt_c_api_helpers.h" - #include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" -+#include "xla/service/custom_call_target_registry.h" - - namespace pjrt { - -@@ -68,6 +72,31 @@ static PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args) { - return nullptr; - } - -+static PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args) { -+ PJRT_RETURN_IF_ERROR(ActualStructSizeIsGreaterOrEqual( -+ "PJRT_FFI_Register_Handler_Args", -+ PJRT_FFI_Register_Handler_Args_STRUCT_SIZE, args->struct_size)); -+ std::string target_name(args->target_name, args->target_name_size); -+ std::string platform_name(args->platform_name, args->platform_name_size); -+ switch (args->api_version) { -+ case 0: -+ xla::CustomCallTargetRegistry::Global()->Register( -+ target_name, args->handler, platform_name); -+ return nullptr; -+ case 1: -+ xla::ffi::Ffi::RegisterStaticHandler( -+ xla::ffi::GetXlaFfiApi(), target_name, platform_name, -+ reinterpret_cast(args->handler)); -+ return nullptr; -+ default: -+ return new PJRT_Error{absl::UnimplementedError( -+ absl::StrFormat("API version %d not supported for PJRT GPU plugin. " -+ "Supported versions are 0 and 1.", -+ args->api_version))}; -+ } -+} -+ - PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - return { - PJRT_Extension_Base{ -@@ -77,6 +106,7 @@ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - }, - /*type_id_register=*/PJRT_FFI_TypeID_Register, - /*user_data_add=*/PJRT_FFI_UserData_Add, -+ /*register_handler=*/PJRT_FFI_Register_Handler, - }; - } - --- -2.39.5 (Apple Git-154) - diff --git a/third_party/xla/patches/0003-Remove-unconventional-C-code-in-headers.patch b/third_party/xla/patches/0003-Remove-unconventional-C-code-in-headers.patch deleted file mode 100644 index 6df5b1b3..00000000 --- a/third_party/xla/patches/0003-Remove-unconventional-C-code-in-headers.patch +++ /dev/null @@ -1,124 +0,0 @@ -From 6078da86a46b6f0d983dccb9ae4f36fc90640247 Mon Sep 17 00:00:00 2001 -From: Hugo Mano -Date: Fri, 11 Jul 2025 14:05:16 +0200 -Subject: [PATCH] zml patch - ---- - third_party/stablehlo/workspace.bzl | 1 + - third_party/stablehlo/zml.patch | 93 +++++++++++++++++++++++++++++ - 2 files changed, 94 insertions(+) - create mode 100644 third_party/stablehlo/zml.patch - -diff --git a/third_party/stablehlo/workspace.bzl b/third_party/stablehlo/workspace.bzl -index d9d5063744..44980948d0 100644 ---- a/third_party/stablehlo/workspace.bzl -+++ b/third_party/stablehlo/workspace.bzl -@@ -15,5 +15,6 @@ def repo(): - urls = tf_mirror_urls("https://github.com/openxla/stablehlo/archive/{commit}.zip".format(commit = STABLEHLO_COMMIT)), - patch_file = [ - "//third_party/stablehlo:temporary.patch", # Autogenerated, don't remove. -+ "//third_party/stablehlo:zml.patch", # Autogenerated, don't remove. - ], - ) -diff --git a/third_party/stablehlo/zml.patch b/third_party/stablehlo/zml.patch -new file mode 100644 -index 0000000000..2a09384582 ---- /dev/null -+++ b/third_party/stablehlo/zml.patch -@@ -0,0 +1,93 @@ -+From e38ab68376dd8a17ebf4469d2c8350f521310182 Mon Sep 17 00:00:00 2001 -+From: Hugo Mano -+Date: Fri, 11 Jul 2025 12:08:35 +0200 -+Subject: [PATCH] zml patch -+ -+--- -+ stablehlo/dialect/Serialization.cpp | 5 ++--- -+ stablehlo/dialect/Serialization.h | 3 +-- -+ stablehlo/integrations/c/StablehloDialectApi.cpp | 3 +-- -+ stablehlo/integrations/c/StablehloDialectApi.h | 2 +- -+ stablehlo/tools/StablehloTranslateMain.cpp | 2 +- -+ 5 files changed, 6 insertions(+), 9 deletions(-) -+ -+diff --git a/stablehlo/dialect/Serialization.cpp b/stablehlo/dialect/Serialization.cpp -+index cb89d673..4370d588 100644 -+--- a/stablehlo/dialect/Serialization.cpp -++++ b/stablehlo/dialect/Serialization.cpp -+@@ -39,8 +39,7 @@ namespace stablehlo { -+ -+ LogicalResult serializePortableArtifact(ModuleOp module, -+ StringRef targetVersion, -+- raw_ostream& os, -+- bool allowOtherDialects) { -++ raw_ostream& os) { -+ MLIRContext* context = module.getContext(); -+ -+ // Convert StableHLO --> VHLO. -+@@ -49,7 +48,7 @@ LogicalResult serializePortableArtifact(ModuleOp module, -+ { -+ PassManager pm(context); -+ StablehloLegalizeToVhloPassOptions options; -+- options.allowOtherDialects = allowOtherDialects; -++ options.allowOtherDialects = false; -+ pm.addPass(stablehlo::createStablehloLegalizeToVhloPass(options)); -+ if (!succeeded(pm.run(module))) { -+ return failure(); -+diff --git a/stablehlo/dialect/Serialization.h b/stablehlo/dialect/Serialization.h -+index 811ca97b..abe95e63 100644 -+--- a/stablehlo/dialect/Serialization.h -++++ b/stablehlo/dialect/Serialization.h -+@@ -34,8 +34,7 @@ namespace stablehlo { -+ // unsupported dialects. -+ LogicalResult serializePortableArtifact(ModuleOp module, -+ StringRef targetVersion, -+- raw_ostream& os, -+- bool allowOtherDialects = false); -++ raw_ostream& os); -+ -+ // Read StableHLO portable artifact -+ // -+diff --git a/stablehlo/integrations/c/StablehloDialectApi.cpp b/stablehlo/integrations/c/StablehloDialectApi.cpp -+index 343f8d0b..8f52e4d5 100644 -+--- a/stablehlo/integrations/c/StablehloDialectApi.cpp -++++ b/stablehlo/integrations/c/StablehloDialectApi.cpp -+@@ -81,8 +81,7 @@ MlirLogicalResult stablehloSerializePortableArtifactFromModule( -+ MlirStringCallback callback, void *userData, bool allowOtherDialects) { -+ mlir::detail::CallbackOstream stream(callback, userData); -+ if (failed(mlir::stablehlo::serializePortableArtifact( -+- unwrap(moduleStr), unwrap(targetVersion), stream, -+- allowOtherDialects))) -++ unwrap(moduleStr), unwrap(targetVersion), stream))) -+ return mlirLogicalResultFailure(); -+ return mlirLogicalResultSuccess(); -+ } -+diff --git a/stablehlo/integrations/c/StablehloDialectApi.h b/stablehlo/integrations/c/StablehloDialectApi.h -+index 385156bf..24d11c1d 100644 -+--- a/stablehlo/integrations/c/StablehloDialectApi.h -++++ b/stablehlo/integrations/c/StablehloDialectApi.h -+@@ -93,7 +93,7 @@ stablehloSerializePortableArtifactFromModule(MlirModule moduleStr, -+ MlirStringRef targetVersion, -+ MlirStringCallback callback, -+ void* userData, -+- bool allowOtherDialects = false); -++ bool allowOtherDialects); -+ -+ // Read a StableHLO program from a portable artifact, returning the module as -+ // MLIR bytecode. Note, this bytecode returned is not a portable artifact, -+diff --git a/stablehlo/tools/StablehloTranslateMain.cpp b/stablehlo/tools/StablehloTranslateMain.cpp -+index fdf0d6a9..8d5c8752 100644 -+--- a/stablehlo/tools/StablehloTranslateMain.cpp -++++ b/stablehlo/tools/StablehloTranslateMain.cpp -+@@ -323,7 +323,7 @@ TranslateFromMLIRRegistration serializeRegistration( -+ } -+ -+ return stablehlo::serializePortableArtifact( -+- module, targetVersion, os, allowOtherDialectsOption.getValue()); -++ module, targetVersion, os); -+ }, -+ [](DialectRegistry ®istry) { -+ mlir::registerAllDialects(registry); -+-- -+2.39.5 (Apple Git-154) -+ --- -2.39.5 (Apple Git-154) - diff --git a/third_party/xla/repo.bzl b/third_party/xla/repo.bzl index 8c005dc8..e9a8b406 100644 --- a/third_party/xla/repo.bzl +++ b/third_party/xla/repo.bzl @@ -4,11 +4,9 @@ def repo(): git_repository( name = "xla", remote = "https://github.com/openxla/xla.git", - commit = "ef07e787ea1303fa2f8d8a175d24d434bfb84107", + commit = "b3fbfeeb076f2b536897180f4a274680ed9d52eb", patch_args = ["-p1"], patches = [ - "//third_party/xla:patches/0001-bazel-migration-to-bazel-8.1.1.patch", - "//third_party/xla:patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch", - "//third_party/xla:patches/0003-Remove-unconventional-C-code-in-headers.patch", + # patches live in the patches directory ], ) diff --git a/third_party/xla/xla.bzl b/third_party/xla/xla.bzl index b0aff388..3780bc51 100644 --- a/third_party/xla/xla.bzl +++ b/third_party/xla/xla.bzl @@ -65,12 +65,10 @@ def _xla_impl(mctx): tf_http_archive( name = "com_github_grpc_grpc", - sha256 = "afbc5d78d6ba6d509cc6e264de0d49dcd7304db435cbf2d630385bacf49e066c", - strip_prefix = "grpc-1.68.2", - patch_file = [ - "//third_party/grpc:grpc.patch", - ], - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/refs/tags/v1.68.2.tar.gz"), + sha256 = "dd6a2fa311ba8441bbefd2764c55b99136ff10f7ea42954be96006a2723d33fc", + strip_prefix = "grpc-1.74.0", + patch_file = ["//third_party/grpc:grpc.patch"], + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/refs/tags/v1.74.0.tar.gz"), ) tf_vendored(name = "tsl", relpath = "third_party/tsl") diff --git a/zml/module.zig b/zml/module.zig index e57f942c..33d22f78 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -905,7 +905,6 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_triton_gemm", false, upb_arena); try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_latency_hiding_scheduler", true, upb_arena); try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_llvm_module_compilation_parallelism", true, upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_libnvptxcompiler", true, upb_arena); }, .rocm => { // Disable Triton GEMM on ROCM. For some reason it's much, much slower when From 3f95bdd9001c865e0dd2d7682a86b0de4396be89 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 4 Sep 2025 16:57:34 +0200 Subject: [PATCH 027/881] zml: callback api (#319) Resuming work from @hugomano #244 This is still a bit experimental. The core API is here, but there are some caveats. ZML should help with copying buffers to host memory (which is often needed) but I'm not sure the current implementation using PJRT is good: - does it allow graph capture - what's the overhead ? We should probably use directly cuMemcpy here instead of going through PJRT. Current implem should work with all plugins though. This fix `tensor.print()` I've also made a slightly more interesting examples/callback that manually calls a cuda kernel. (there are other way to achieve this but this shows you can do whatever you want with `zml.callback` Note: the feature was originally named `custom_call` but it was conflicting with existing bindings for xla builtin custom calls, that are fully handled by the backend. So I renamed it to `callback` --------- Co-authored-by: Hugo Mano --- examples/callback/BUILD.bazel | 11 ++ examples/callback/main.zig | 218 +++++++++++++++++++++++ pjrt/ffi.zig | 148 +++++++--------- pjrt/pjrt.zig | 32 +--- runtimes/cuda/BUILD.bazel | 2 +- stdx/fmt.zig | 160 +++++++---------- zml/BUILD.bazel | 3 +- zml/buffer.zig | 57 +++++-- zml/callback.zig | 313 ++++++++++++++++++++++++++++++++++ zml/context.zig | 172 +++---------------- zml/exe.zig | 49 ++++-- zml/hostbuffer.zig | 37 ++-- zml/module.zig | 8 +- zml/ops.zig | 27 --- zml/pjrtx.zig | 11 +- zml/platform.zig | 16 +- zml/tensor.zig | 18 +- zml/testing.zig | 7 +- zml/zml.zig | 3 +- 19 files changed, 825 insertions(+), 467 deletions(-) create mode 100644 examples/callback/BUILD.bazel create mode 100644 examples/callback/main.zig create mode 100644 zml/callback.zig diff --git a/examples/callback/BUILD.bazel b/examples/callback/BUILD.bazel new file mode 100644 index 00000000..3f55040f --- /dev/null +++ b/examples/callback/BUILD.bazel @@ -0,0 +1,11 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary") + +zig_binary( + name = "callback", + main = "main.zig", + deps = [ + "@zml//async", + "@zml//zml", + "@zml//runtimes", + ], +) diff --git a/examples/callback/main.zig b/examples/callback/main.zig new file mode 100644 index 00000000..be3b3095 --- /dev/null +++ b/examples/callback/main.zig @@ -0,0 +1,218 @@ +const std = @import("std"); + +const asynk = @import("async"); +const runtimes = @import("runtimes"); +const zml = @import("zml"); +const cu = zml.platform_specific; + +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +const log = std.log.scoped(.@"examples/custom_call"); + +/// Demonstration of the custom_call mechanism. +/// +/// * defines a function to compute the grayscale version of an image. +/// * using simple for loop on CPU +/// * using a custom PTX kernel on Cuda, with manual cuLaunchKernel +/// * Use `zml.customCall` to create an executable calling our callback and our kernel. +pub const GrayScale = struct { + // Mandatory fields to work with ZML custom call api + pub var type_id: zml.pjrt.ffi.TypeId = undefined; + pub const callback_config: zml.callback.Config = .{}; + platform: zml.Platform, + + // Custom field to store our cuda module and function. + cu_data: [2]*anyopaque, + + // Set by ZML before `call` is entered + results: [1]zml.Buffer = undefined, + stream: *zml.pjrt.Stream = undefined, + + pub fn init(platform: zml.Platform) !GrayScale { + var cu_data: [2]*anyopaque = undefined; + if (comptime runtimes.isEnabled(.cuda)) { + var module: cu.CUmodule = undefined; + try cuda.check(cuda.moduleLoadData.?(&module, grayscale_ptx)); + log.info("Loaded Grayscale cuda module", .{}); + var function: cu.CUfunction = undefined; + try cuda.check(cuda.moduleGetFunction.?(&function, module, "rgba_to_grayscale")); + log.info("Found Grayscale cuda function", .{}); + cu_data = .{ module.?, function.? }; + } + return .{ + .platform = platform, + .cu_data = cu_data, + }; + } + + pub fn deinit(self: *GrayScale) void { + if (comptime runtimes.isEnabled(.cuda)) { + const module: cu.CUmodule = @ptrCast(self.cu_data[0]); + _ = cuda.moduleUnload.?(module); + } + } + + pub fn call(self: *GrayScale, rgb_d: zml.Buffer) !void { + switch (self.platform.target) { + .cpu => grayScaleCpu(rgb_d, self.results[0]), + // Only try to compile `grayScaleCuda` if we have cuda symbols. + .cuda => if (comptime runtimes.isEnabled(.cuda)) + try self.grayScaleCuda(rgb_d, self.results[0]) + else + unreachable, + else => @panic("Platform not supported"), + } + } + + pub fn grayScaleCpu(rgb_d: zml.Buffer, gray_d: zml.Buffer) void { + const rgb_h = rgb_d.asHostBuffer().items(u8); + const gray_h = gray_d.asHostBuffer().mutItems(u8); + + for (gray_h, 0..) |*gray, i| { + const px = rgb_h[i * 3 .. i * 3 + 3]; + const R: u32 = @intCast(px[0]); + const G: u32 = @intCast(px[1]); + const B: u32 = @intCast(px[2]); + const gray_u32: u32 = @divFloor(299 * R + 587 * G + 114 * B, 1000); + gray.* = @intCast(gray_u32); + } + } + + pub fn grayScaleCuda(self: GrayScale, rgb_d: zml.Buffer, gray_d: zml.Buffer) !void { + var args: [2][]u8 = .{ + rgb_d.opaqueDeviceMemoryDataPointer()[0..rgb_d.shape().byteSize()], + gray_d.opaqueDeviceMemoryDataPointer()[0..gray_d.shape().byteSize()], + }; + var args_ptr: [2:null]?*anyopaque = .{ @ptrCast(&args[0]), @ptrCast(&args[1]) }; + // This is a naive kernel with one block per pixel. + try cuda.check(cuda.launchKernel.?( + @ptrCast(self.cu_data[1]), // function + @intCast(rgb_d.shape().count() / 3), // num blocks x + 1, // num blocks y + 1, // num blocks z + 1, // num grids x + 1, // num grids y + 1, // num grids z + 0, // shared mem + @ptrCast(self.stream), + &args_ptr, + null, + )); + // Note: no explicit synchronization, we just enqueue work in the stream. + } + + const cuda = struct { + // Here we leverage ZML sandboxing to access cuda symbols and their definitions. + const moduleLoadData = @extern(*const @TypeOf(cu.cuModuleLoadData), .{ .name = "cuModuleLoadData", .linkage = .weak }); + const moduleUnload = @extern(*const @TypeOf(cu.cuModuleUnload), .{ .name = "cuModuleUnload", .linkage = .weak }); + const moduleGetFunction = @extern(*const @TypeOf(cu.cuModuleGetFunction), .{ .name = "cuModuleGetFunction", .linkage = .weak }); + const launchKernel = @extern(*const @TypeOf(cu.cuLaunchKernel), .{ .name = "cuLaunchKernel", .linkage = .weak }); + + pub fn check(result: cu.CUresult) error{CudaError}!void { + if (result == cu.CUDA_SUCCESS) return; + std.log.err("cuda error: {}", .{result}); + return error.CudaError; + } + }; +}; + +pub fn grayscale(rgb: zml.Tensor) zml.Tensor { + const gray_shape = rgb.shape().setDim(0, @divExact(rgb.dim(0), 3)); + const result = zml.callback.call(GrayScale, .{rgb.print()}, &.{gray_shape}); + return result[0]; +} + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.smp_allocator, asyncMain); +} + +pub fn asyncMain() !void { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform(.{}); + context.printAvailablePlatforms(platform); + + // Register our custom call + try zml.callback.register(GrayScale, platform); + + // Compile MLIR code containing our custom call. + const rgb_shape = zml.Shape.init(.{12 * 3}, .u8); + const exe = try zml.compileFn(allocator, grayscale, .{rgb_shape}, platform); + defer exe.deinit(); + + // Provide runtime information needed by our custom call. + var gray_op: GrayScale = try .init(platform); + defer gray_op.deinit(); + try exe.bind(GrayScale, &gray_op); + + // Load data and run the executable. + const rgb_h: [12][3]u8 = @splat(.{ 0xFF, 0xAA, 0x00 }); + const rgb_d = try zml.Buffer.fromBytes(platform, rgb_shape, @ptrCast(&rgb_h)); + defer rgb_d.deinit(); + + var gray_d: zml.Buffer = exe.call(.{rgb_d}); + defer gray_d.deinit(); + + // Inspect results + const gray_h = try gray_d.toHostAlloc(allocator); + defer gray_h.deinit(allocator); + + std.debug.print("Grayscale conversion of {any} -> {d}\n", .{ rgb_h, gray_h }); + try std.testing.expectEqualSlices(u8, &@as([12]u8, @splat(0xB0)), gray_h.items(u8)); +} + +// Compiled with Zig and a little help from `https://github.com/gwenzek/cudaz` +const grayscale_ptx = + \\.version 4.0 + \\.target sm_32 + \\.address_size 64 + \\ + \\.visible .entry rgba_to_grayscale( + \\ .param .align 8 .b8 rgba_to_grayscale_param_0[16], + \\ .param .align 8 .b8 rgba_to_grayscale_param_1[16] + \\) + \\{ + \\ .reg .pred %p<2>; + \\ .reg .b16 %rs<4>; + \\ .reg .b32 %r<13>; + \\ .reg .b64 %rd<9>; + \\ + \\ ld.param.u64 %rd5, [rgba_to_grayscale_param_1+8]; + \\ mov.u32 %r1, %tid.x; + \\ mov.u32 %r3, %ntid.x; + \\ mov.u32 %r2, %ctaid.x; + \\ mad.lo.s32 %r4, %r2, %r3, %r1; + \\ cvt.u64.u32 %rd1, %r4; + \\ setp.gt.u64 %p1, %rd5, %rd1; + \\ @%p1 bra $L__BB0_2; + \\ bra.uni $L__BB0_1; + \\$L__BB0_2: + \\ ld.param.u64 %rd4, [rgba_to_grayscale_param_1]; + \\ ld.param.u64 %rd2, [rgba_to_grayscale_param_0]; + \\ cvt.u32.u64 %r5, %rd1; + \\ mul.lo.s32 %r6, %r5, 3; + \\ cvt.u64.u32 %rd6, %r6; + \\ add.s64 %rd7, %rd2, %rd6; + \\ ld.u8 %rs1, [%rd7]; + \\ ld.u8 %rs2, [%rd7+1]; + \\ ld.u8 %rs3, [%rd7+2]; + \\ mul.wide.u16 %r7, %rs1, 299; + \\ mul.wide.u16 %r8, %rs2, 587; + \\ add.s32 %r9, %r8, %r7; + \\ mul.wide.u16 %r10, %rs3, 114; + \\ add.s32 %r11, %r9, %r10; + \\ mul.hi.u32 %r12, %r11, 4294968; + \\ add.s64 %rd8, %rd4, %rd1; + \\ st.u8 [%rd8], %r12; + \\$L__BB0_1: + \\ ret; + \\} +; diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index d2517a04..2ca639a7 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -2,12 +2,21 @@ const std = @import("std"); const c = @import("c"); +pub const TypeId = c.XLA_FFI_TypeId; const stdx = @import("stdx"); -const pjrtStruct = @import("pjrt.zig").pjrtStruct; +const pjrt = @import("pjrt.zig"); +const Stream = @import("pjrt.zig").Stream; const log = std.log.scoped(.pjrt); +comptime { + if (@typeInfo(TypeId).@"struct".fields.len != 1) @compileError("TypeId has changed"); +} + +/// The signature of a generic custom call. +pub const Handler = fn (*CallFrame) callconv(.c) ?*Error; + pub const ApiVersion = extern struct { pub const major = c.XLA_FFI_API_MAJOR; pub const minor = c.XLA_FFI_API_MINOR; @@ -29,13 +38,13 @@ pub const ExtensionBase = extern struct { }; // Based of https://github.com/openxla/xla/blob/145f836bd5175dc5dd262f716a0c59af2b0297a0/xla/ffi/api/c_api.h#L449 -pub const HandlerTraits = packed struct(u32) { +pub const HandlerTraits = packed struct(c_uint) { /// Calls to FFI handler are safe to trace into the command buffer. /// It means that calls to FFI handler always launch exactly the same device operations (can depend on attribute values) /// that can be captured and then replayed. - command_buffer_compatible: u1, + command_buffer_compatible: bool, - __unassigned__: u31, + __unassigned__: u31 = 0, }; pub const Metadata = extern struct { @@ -49,25 +58,6 @@ pub const MetadataExtension = extern struct { metadata: ?*Metadata, }; -pub const ApiError = error{ - Cancelled, - Unknown, - InvalidArgument, - DeadlineExceeded, - NotFound, - AlreadyExists, - PermissionDenied, - ResourceExhausted, - FailedPrecondition, - Aborted, - OutOfRange, - Unimplemented, - Internal, - Unavailable, - DataLoss, - Unauthenticated, -}; - fn TransmuteMixin(comptime T: type, comptime InnerT: type) type { return struct { pub fn to(self: anytype) switch (@TypeOf(self)) { @@ -91,8 +81,8 @@ fn TransmuteMixin(comptime T: type, comptime InnerT: type) type { pub const Api = opaque { pub const inner = TransmuteMixin(Api, c.XLA_FFI_Api).to; - pub fn stream(self: *const Api, context: *const ExecutionContext) *Stream { - var ret = pjrtStruct(c.XLA_FFI_Stream_Get_Args{ + pub fn stream(self: *const Api, context: *const ExecutionContext) *pjrt.Stream { + var ret = pjrt.pjrtStruct(c.XLA_FFI_Stream_Get_Args{ .ctx = @constCast(context.inner()), }); const result = self.inner().XLA_FFI_Stream_Get.?(&ret); @@ -107,8 +97,8 @@ pub const Api = opaque { return @ptrCast(ret.stream.?); } - pub fn allocateDeviceMemory(self: *const Api, context: *const ExecutionContext, size: usize, alignment: usize) ApiError!*anyopaque { - var ret = pjrtStruct(c.XLA_FFI_DeviceMemory_Allocate_Args{ + pub fn allocateDeviceMemory(self: *const Api, context: *const ExecutionContext, size: usize, alignment: usize) pjrt.ApiError!*anyopaque { + var ret = pjrt.pjrtStruct(c.XLA_FFI_DeviceMemory_Allocate_Args{ .ctx = @constCast(context.inner()), .size = size, .alignment = alignment, @@ -127,8 +117,8 @@ pub const Api = opaque { return ret.data.?; } - pub fn freeDeviceMemory(self: *const Api, context: *const ExecutionContext, data: *anyopaque, size: usize) ApiError!void { - var ret = pjrtStruct(c.XLA_FFI_DeviceMemory_Free_Args{ + pub fn freeDeviceMemory(self: *const Api, context: *const ExecutionContext, data: *anyopaque, size: usize) pjrt.ApiError!void { + var ret = pjrt.pjrtStruct(c.XLA_FFI_DeviceMemory_Free_Args{ .ctx = @constCast(context.inner()), .size = size, .data = data, @@ -163,33 +153,31 @@ pub const ExecutionStage = enum(c.XLA_FFI_ExecutionStage) { pub const ExecutionContext = opaque { pub const inner = TransmuteMixin(ExecutionContext, c.XLA_FFI_ExecutionContext).to; - pub fn Context(comptime T: type) type { - return struct { - pub fn get(self: *const ExecutionContext, api: *const Api) ApiError!*T { - const type_id: TypeId = .{ .type_id = T.type_id }; - var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Get_Args{ - .ctx = @constCast(self.inner()), - .type_id = @constCast(&type_id.toCStruct()), - }); - const result = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); - - if (result) |ffi_error| { - const err = Error.fromInner(ffi_error); - defer err.destroy(api); - log.err("[ExecutionContext.get] {s}", .{err.getMessage(api)}); - - // TODO(Corentin): Retrieve error code from Error when implemented in XLA. - return error.Unknown; - } - - if (ret.data == null) return error.NotFound; - return @ptrCast(@alignCast(ret.data.?)); - } + pub fn getContext(self: *const ExecutionContext, type_id: TypeId, api: *const Api) pjrt.ApiError!*anyopaque { + var ret: c.XLA_FFI_ExecutionContext_Get_Args = .{ + .struct_size = pjrt.pjrtStructSize(c.XLA_FFI_ExecutionContext_Get_Args), + .extension_start = api.inner().extension_start, + .ctx = @ptrCast(@constCast(self)), + .type_id = @constCast(&type_id), + .data = undefined, // set by XLA_FFI_ExecutionContext_Get. }; + const maybe_err = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); + + if (maybe_err) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(api); + log.err("[ExecutionContext.get] {s}", .{err.getMessage(api)}); + + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } + + if (ret.data == null) return error.NotFound; + return ret.data.?; } - pub fn getDeviceOrdinal(self: *const ExecutionContext, api: *const Api) ApiError!i32 { - var ret = pjrtStruct(c.XLA_FFI_DeviceOrdinal_Get_Args{ + pub fn getDeviceOrdinal(self: *const ExecutionContext, api: *const Api) pjrt.ApiError!i32 { + var ret = pjrt.pjrtStruct(c.XLA_FFI_DeviceOrdinal_Get_Args{ .ctx = @constCast(self.inner()), }); const result = api.inner().XLA_FFI_DeviceOrdinal_Get.?(&ret); @@ -206,8 +194,10 @@ pub const ExecutionContext = opaque { return ret.device_ordinal; } - pub fn scheduleTask(self: *const ExecutionContext, api: *const Api, task: *const Task, data: *anyopaque) ApiError!void { - var ret = pjrtStruct(c.XLA_FFI_ThreadPool_Schedule_Args{ + const Task = fn (*anyopaque) void; + + pub fn scheduleTask(self: *const ExecutionContext, api: *const Api, task: *const Task, data: *anyopaque) pjrt.ApiError!void { + var ret = pjrt.pjrtStruct(c.XLA_FFI_ThreadPool_Schedule_Args{ .ctx = @constCast(self.inner()), .task = @ptrCast(@alignCast(task)), .data = @ptrCast(@alignCast(data)), @@ -225,23 +215,9 @@ pub const ExecutionContext = opaque { return error.Unknown; } } - - fn getTypeId(type_name: []const u8) TypeId { - const id: i64 = @bitCast(std.hash.Fnv1a_64.hash(type_name)); - - return .{ - .type_id = id, - }; - } }; -const TypeId = c.XLA_FFI_TypeId; - -const Task = fn (*anyopaque) void; - -const Stream = @import("pjrt.zig").Stream; - -const ByteSpan = extern struct { +pub const ByteSpan = extern struct { ptr: [*]const u8, len: usize, @@ -252,7 +228,7 @@ const ByteSpan = extern struct { pub const DataType = enum(c.XLA_FFI_DataType) { invalid = c.XLA_FFI_DataType_INVALID, - pred = c.XLA_FFI_DataType_PRED, + bool = c.XLA_FFI_DataType_PRED, i8 = c.XLA_FFI_DataType_S8, i16 = c.XLA_FFI_DataType_S16, i32 = c.XLA_FFI_DataType_S32, @@ -399,6 +375,8 @@ pub const Attrs = extern struct { } }; +/// All informations needed by the user callback, +/// including the list of input/ouput buffers to work on. pub const CallFrame = extern struct { struct_size: usize, extension_start: ?*ExtensionBase, @@ -422,9 +400,11 @@ pub const CallFrame = extern struct { } return false; } -}; -pub const Handler = fn (*CallFrame) callconv(.c) ?*Error; + pub fn stream(call_frame: CallFrame) ?*const pjrt.Stream { + return call_frame.api.stream(call_frame.ctx); + } +}; pub const ErrorCode = enum(c.XLA_FFI_Error_Code) { cancelled = c.XLA_FFI_Error_Code_CANCELLED, @@ -444,7 +424,7 @@ pub const ErrorCode = enum(c.XLA_FFI_Error_Code) { data_loss = c.XLA_FFI_Error_Code_DATA_LOSS, unauthenticated = c.XLA_FFI_Error_Code_UNAUTHENTICATED, - pub fn toApiError(code: ErrorCode) ApiError { + pub fn toApiError(code: ErrorCode) pjrt.ApiError { return switch (code) { .cancelled => error.Cancelled, .unknown => error.Unknown, @@ -470,8 +450,10 @@ pub const Error = opaque { pub const inner = TransmuteMixin(Error, c.XLA_FFI_Error).to; pub const fromInner = TransmuteMixin(Error, c.XLA_FFI_Error).from; + pub const ok: ?*Error = null; + pub fn create(api: *const Api, error_code: ErrorCode, message: []const u8) *Error { - var ret = pjrtStruct(c.XLA_FFI_Error_Create_Args{ + var ret = pjrt.pjrtStruct(c.XLA_FFI_Error_Create_Args{ .message = message.ptr, .errc = @intFromEnum(error_code), }); @@ -479,12 +461,12 @@ pub const Error = opaque { } pub fn destroy(err: *Error, api: *const Api) void { - var ret = pjrtStruct(c.XLA_FFI_Error_Destroy_Args{ .@"error" = err.inner() }); + var ret = pjrt.pjrtStruct(c.XLA_FFI_Error_Destroy_Args{ .@"error" = err.inner() }); api.inner().XLA_FFI_Error_Destroy.?(&ret); } pub fn getMessage(err: *Error, api: *const Api) [:0]const u8 { - var ret = pjrtStruct(c.XLA_FFI_Error_GetMessage_Args{ + var ret = pjrt.pjrtStruct(c.XLA_FFI_Error_GetMessage_Args{ .@"error" = err.inner(), }); api.inner().XLA_FFI_Error_GetMessage.?(&ret); @@ -496,8 +478,8 @@ pub const Future = opaque { pub const inner = TransmuteMixin(Future, c.XLA_FFI_Future).to; pub const fromInner = TransmuteMixin(Future, c.XLA_FFI_Future).from; - pub fn create(api: *const Api) ApiError!*Future { - var ret = pjrtStruct(c.XLA_FFI_Future_Create_Args{}); + pub fn create(api: *const Api) pjrt.ApiError!*Future { + var ret = pjrt.pjrtStruct(c.XLA_FFI_Future_Create_Args{}); const result = api.inner().XLA_FFI_Future_Create.?(&ret); if (result) |ffi_error| { @@ -512,8 +494,8 @@ pub const Future = opaque { return fromInner(ret.future.?); } - pub fn setAvailable(self: *Future, api: *const Api) ApiError!void { - var ret = pjrtStruct(c.XLA_FFI_Future_SetAvailable_Args{ + pub fn setAvailable(self: *Future, api: *const Api) pjrt.ApiError!void { + var ret = pjrt.pjrtStruct(c.XLA_FFI_Future_SetAvailable_Args{ .future = self.inner(), }); @@ -529,8 +511,8 @@ pub const Future = opaque { } } - pub fn setError(self: *Future, api: *const Api, err: *Error) ApiError!void { - var ret = pjrtStruct(c.XLA_FFI_Future_SetError_Args{ + pub fn setError(self: *Future, api: *const Api, err: *Error) pjrt.ApiError!void { + var ret = pjrt.pjrtStruct(c.XLA_FFI_Future_SetError_Args{ .future = self.inner(), .@"error" = err.inner(), }); diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 6e358562..08ccf217 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -20,7 +20,7 @@ test { // as the way PJRT does it is not very robust. // // 1. https://github.com/openxla/xla/issues/10032 -fn pjrtStructSize(comptime T: type) usize { +pub fn pjrtStructSize(comptime T: type) usize { // unsafe on purpose, we want this to fail if that ever changes const typedef_name = comptime blk: { const needle = ".struct_"; @@ -164,7 +164,7 @@ pub const Api = struct { return @ptrCast(ret.context.?); } - pub fn ffi(api: *const Api) ?FFI { + pub fn ffi(api: *const Api) ?Ffi { if (api.lookupExtension(c.PJRT_FFI_Extension, c.PJRT_Extension_Type_FFI)) |ext| { return .{ .inner = ext }; } @@ -1278,7 +1278,7 @@ pub const NamedValue = extern struct { } }; -pub const FFI = extern struct { +pub const Ffi = extern struct { inner: *const c.PJRT_FFI, pub const UserData = extern struct { @@ -1293,19 +1293,15 @@ pub const FFI = extern struct { } }; - pub const RegisterFfiOptions = struct { - traits: RegisterHandlerTraits = @enumFromInt(0), - }; - // todo : support all missing handlers available in GPU plugin extension: handler_instantiate, handler_prepare, handler_initialize // introduced by https://github.com/openxla/xla/commit/ef85a7bcc308313492ebc50295a8a08b4e51b8f5 pub fn register( - self: *const FFI, + self: *const Ffi, api: *const Api, target_name: []const u8, platform_name: []const u8, func: *const ffi.Handler, - options: RegisterFfiOptions, + traits: ffi.HandlerTraits, ) ApiError!void { var ret = pjrtStruct(c.PJRT_FFI_Register_Handler_Args{ .target_name = target_name.ptr, @@ -1313,7 +1309,7 @@ pub const FFI = extern struct { .handler = @ptrCast(@constCast(func)), .platform_name = platform_name.ptr, .platform_name_size = platform_name.len, - .traits = @intFromEnum(options.traits), + .traits = @bitCast(traits), }); const result = self.inner.register_handler.?(&ret); if (result) |pjrt_c_error| { @@ -1323,8 +1319,7 @@ pub const FFI = extern struct { } } - pub fn registerTypeId(self: *const FFI, api: *const Api, T: type) ApiError!void { - const type_name = @typeName(T); + pub fn registerTypeId(self: *const Ffi, api: *const Api, type_name: []const u8) ApiError!ffi.TypeId { var ret = pjrtStruct(c.PJRT_FFI_TypeID_Register_Args{ .type_name = type_name.ptr, .type_name_size = type_name.len, @@ -1336,10 +1331,10 @@ pub const FFI = extern struct { return pjrt_error.getCode(api).toApiError(); } - T.type_id = ret.type_id; + return .{ .type_id = ret.type_id }; } - pub fn addUserData(self: *const FFI, api: *const Api, context: *ExecuteContext, user_data: UserData) ApiError!void { + pub fn addUserData(self: *const Ffi, api: *const Api, context: *ExecuteContext, user_data: UserData) ApiError!void { var ret = pjrtStruct(c.PJRT_FFI_UserData_Add_Args{ .context = @ptrCast(context), .user_data = user_data.toCStruct(), @@ -1352,12 +1347,3 @@ pub const FFI = extern struct { } } }; - -pub const RegisterHandlerTraits = enum(c.PJRT_FFI_Handler_TraitsBits) { - command_buffer_compatible = c.PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE, - _, -}; - -pub const CustomCallRegistry = extern struct { - inner: *const c.PJRT_FFI_Register_Handler, -}; diff --git a/runtimes/cuda/BUILD.bazel b/runtimes/cuda/BUILD.bazel index 96cc5879..83109cf0 100644 --- a/runtimes/cuda/BUILD.bazel +++ b/runtimes/cuda/BUILD.bazel @@ -5,7 +5,7 @@ zig_shared_library( name = "zmlxcuda", # Use Clang's compiler-rt, but disable stack checking # to avoid requiring on the _zig_probe_stack symbol. - copts = ["-fno-stack-check"], + copts = ["-fno-stack-check", "-fllvm"], main = "zmlxcuda.zig", shared_lib_name = "libzmlxcuda.so.0", visibility = ["@libpjrt_cuda//:__subpackages__"], diff --git a/stdx/fmt.zig b/stdx/fmt.zig index 259b3cd8..145db768 100644 --- a/stdx/fmt.zig +++ b/stdx/fmt.zig @@ -1,145 +1,117 @@ const std = @import("std"); -pub const Fmt = union(enum) { - int: IntFmt, - float: FloatFmt, - generic: void, +pub fn slice(any_slice: anytype) FmtSlice(std.meta.Elem(@TypeOf(any_slice))) { + return .{ .slice = any_slice }; +} - pub fn parse(T: type, comptime fmt_: []const u8) Fmt { - return switch (@typeInfo(T)) { - .float, .comptime_float => .{ .float = FloatFmt.parseComptime(fmt_) }, - .int, .comptime_int => .{ .int = IntFmt.parseComptime(fmt_) }, - else => .{ .generic = {} }, - }; - } -}; +fn FmtSlice(T: type) type { + return struct { + slice: []const T, -pub const FullFormatOptions = struct { - fmt: Fmt, - options: std.fmt.FormatOptions, -}; - -pub const IntFmt = struct { - base: u8, - case: std.fmt.Case = .lower, - - pub fn parseComptime(comptime fmt_: []const u8) IntFmt { - return parse(fmt_) catch @panic("invalid fmt for int: " ++ fmt_); - } - - pub fn parse(fmt_: []const u8) error{InvalidArgument}!IntFmt { - return if (fmt_.len == 0 or std.mem.eql(u8, fmt_, "d")) - .{ .base = 10, .case = .lower } - else if (std.mem.eql(u8, fmt_, "x")) - .{ .base = 16, .case = .lower } - else if (std.mem.eql(u8, fmt_, "X")) - .{ .base = 16, .case = .upper } - else if (std.mem.eql(u8, fmt_, "o")) - .{ .base = 8, .case = .upper } - else - // TODO: unicode/ascii - error.InvalidArgument; - } -}; - -pub const FloatFmt = enum(u8) { - scientific = @intFromEnum(std.fmt.Number.Mode.scientific), - decimal = @intFromEnum(std.fmt.Number.Mode.decimal), - hex, - - pub fn parseComptime(comptime fmt_: []const u8) FloatFmt { - return parse(fmt_) catch @panic("invalid fmt for float: " ++ fmt_); - } - - pub fn parse(fmt_: []const u8) error{InvalidArgument}!FloatFmt { - return if (fmt_.len == 0 or std.mem.eql(u8, fmt_, "e")) - .scientific - else if (std.mem.eql(u8, fmt_, "d")) - .decimal - else if (std.mem.eql(u8, fmt_, "x")) - .hex - else - error.InvalidArgument; - } -}; - -pub fn formatValue(value: anytype, full: FullFormatOptions, writer: anytype) !void { - return switch (@typeInfo(@TypeOf(value))) { - .comptime_float, .float => try formatFloatValue(value, full, writer), - .comptime_int, .int => try formatIntValue(value, full, writer), - else => try formatAnyValue(value, full, writer), + pub fn formatNumber(f: @This(), writer: *std.io.Writer, n: std.fmt.Number) std.io.Writer.Error!void { + return switch (@typeInfo(T)) { + .comptime_float, .float => try formatFloatSlice(f.slice, n, writer), + .comptime_int, .int => try formatIntSlice(f.slice, n, writer), + .bool => try formatBoolSlice(f.slice, n, writer), + .@"struct" => if (@hasField(T, "re") and @hasField(T, "im")) { + try formatComplexSlice(f.slice, n, writer); + } else if (@hasDecl(T, "toF32")) { + try formatFloatSlice(f.slice, n, writer); + } else { + try formatSliceAny(f.slice, n, writer); + }, + else => @compileError("FmtSlice doesn't support type: " ++ @typeName(T)), + }; + } }; } -pub fn formatFloatValue(value: anytype, full: FullFormatOptions, writer: *std.Io.Writer) !void { +pub fn formatFloat(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { const x = switch (@typeInfo(@TypeOf(value))) { .@"struct" => value.toF32(), .float => value, - else => @compileError("formatFloatValue expects a float, got: " ++ @typeName(@TypeOf(value))), - }; - try switch (full.fmt.float) { - .scientific => writer.printFloat(x, .{ .mode = .scientific, .precision = full.options.precision }), - .decimal => writer.printFloat(x, .{ .mode = .decimal, .precision = full.options.precision }), - .hex => writer.printFloatHexOptions(x, .{ .mode = .hex }), + else => @compileError("formatFloat expects a float, got: " ++ @typeName(@TypeOf(value))), }; + return writer.printFloat(x, spec); } -pub fn formatIntValue(value: anytype, full: FullFormatOptions, writer: *std.Io.Writer) !void { +pub fn formatInt(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { switch (@typeInfo(@TypeOf(value))) { .int => {}, - else => @compileError("formatIntValue expects an int, got: " ++ @typeName(@TypeOf(value))), + else => @compileError("formatInt expects an int, got: " ++ @typeName(@TypeOf(value))), } - return writer.printInt(value, full.fmt.int.base, full.fmt.int.case, full.options); + return writer.printInt(value, spec.mode.base().?, spec.case, .{ .alignment = spec.alignment, .fill = spec.fill }); } -pub fn formatAnyValue(value: anytype, full: FullFormatOptions, writer: *std.Io.Writer) !void { +pub fn formatComplex(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { + try writer.writeAll(".{.re="); + try writer.printFloat(value.re, spec); + try writer.writeAll(", .im="); + try writer.printFloat(value.im, spec); + try writer.writeAll("}"); +} + +pub fn formatBool(value: bool, spec: std.fmt.Number, writer: *std.Io.Writer) !void { + try writer.alignBufferOptions(if (value) "1" else "0", .{ .alignment = spec.alignment, .fill = spec.fill }); +} + +pub fn formatAny(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { var buf: [48]u8 = undefined; - const s = std.fmt.bufPrint(&buf, "{any}", .{value}) catch blk: { + const T = @TypeOf(value); + const fmt = if (@hasDecl(T, "formatNumber")) "{d}" else "{f}"; + + const s = std.fmt.bufPrint(&buf, fmt, .{value}) catch blk: { buf[45..].* = "...".*; break :blk buf[0..]; }; - return try writer.alignBufferOptions(s, full.options); + return try writer.alignBufferOptions(s, .{ .alignment = spec.alignment, .fill = spec.fill }); } -pub fn formatSliceCustom(fmt_func: anytype, values: anytype, full: FullFormatOptions, writer: anytype) !void { - - // Write first rows - const num_cols: usize = full.options.width orelse 12; +pub fn formatSliceCustom(fmt_func: anytype, values: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { + // use the format "width" for the number of columns instead of individual width. + const num_cols: usize = spec.width orelse 12; + var my_options = spec; + my_options.width = null; const n: usize = values.len; + _ = try writer.write("{"); if (n <= num_cols) { for (values, 0..) |v, i| { // Force inlining so that the switch and the buffer can be done once. - try @call(.always_inline, fmt_func, .{ v, full, writer }); + try @call(.always_inline, fmt_func, .{ v, my_options, writer }); if (i < n - 1) _ = try writer.write(","); } } else { const half = @divFloor(num_cols, 2); for (values[0..half]) |v| { - try @call(.always_inline, fmt_func, .{ v, full, writer }); + try @call(.always_inline, fmt_func, .{ v, my_options, writer }); _ = try writer.write(","); } _ = try writer.write(" ..., "); for (values[n - half ..], 0..) |v, i| { - try @call(.always_inline, fmt_func, .{ v, full, writer }); + try @call(.always_inline, fmt_func, .{ v, my_options, writer }); if (i < half - 1) _ = try writer.write(","); } } _ = try writer.write("}"); } -pub fn formatAny(values: anytype, full: FullFormatOptions, writer: anytype) !void { - return try formatSliceCustom(formatAnyValue, values, full, writer); +pub fn formatSliceAny(values: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatAny, values, spec, writer); } -pub fn formatFloatSlice(values: anytype, full: FullFormatOptions, writer: anytype) !void { - return try formatSliceCustom(formatFloatValue, values, full, writer); +pub fn formatFloatSlice(values: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatFloat, values, spec, writer); } -pub fn formatIntSlice(values: anytype, full: FullFormatOptions, writer: anytype) !void { - return try formatSliceCustom(formatIntValue, values, full, writer); +pub fn formatIntSlice(values: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatInt, values, spec, writer); } -pub fn formatAnySlice(values: anytype, full: FullFormatOptions, writer: anytype) !void { - return try formatSliceCustom(formatAnyValue, values, full, writer); +pub fn formatComplexSlice(values: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatComplex, values, spec, writer); +} + +pub fn formatBoolSlice(values: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatBool, values, spec, writer); } diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index aaa44f04..3d2ae081 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -31,6 +31,7 @@ zig_library( "aio/torch/py.zig", "buffer.zig", "context.zig", + "callback.zig", "dtype.zig", "exe.zig", "floats.zig", @@ -53,7 +54,7 @@ zig_library( "torch.zig", "zml.zig", ], - copts = ["-lc"], + copts = ["-lc", "-freference-trace=20"], main = "zml.zig", visibility = ["//visibility:public"], deps = [ diff --git a/zml/buffer.zig b/zml/buffer.zig index 3553bb94..f502d703 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -49,7 +49,7 @@ pub const Buffer = struct { pub const FromOptions = struct { wait: bool = true, - memory: ?pjrt.Memory.Kind = null, + memory: ?Memory = null, }; /// Copies the content of the given buffer from host memory to the accelerator memory. @@ -89,15 +89,20 @@ pub const Buffer = struct { .byte_strides = byte_strides, .host_buffer_semantics = .ImmutableUntilTransferCompletes, }; - if (opts.memory) |memory_kind| { - const memories = try devices[i].addressableMemories(platform.pjrt_api); - const memory = for (memories) |m| { - const kind = m.kind(platform.pjrt_api); - if (kind == memory_kind) break m; - } else return error.NotFound; - args.memory = memory; - } else { + if (platform.target == .cpu or opts.memory == null) { args.device = devices[i]; + } else { + const memory = opts.memory.?; + const device_memories = try devices[i].addressableMemories(platform.pjrt_api); + // TODO measure the cost of this and consider caching on Zig side inside the platform. + const selected_memory = for (device_memories) |m| { + const kind = m.kind(platform.pjrt_api); + if (kind == memory.toPjrtMemory()) break m; + } else { + log.warn("Platform {s} doesn't have memory {s}", .{ @tagName(platform.target), @tagName(memory) }); + return error.NotFound; + }; + args.memory = selected_memory; } const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); @@ -179,10 +184,10 @@ pub const Buffer = struct { return try from(platform, host_buffer, opts); } - pub fn asPinnedHostBuffer(self: Buffer) HostBuffer { - // TODO restore assert + pub fn asHostBuffer(self: Buffer) HostBuffer { + // TODO: skip this check on cpu // const memory = self.getMemory().kind(self._api); - // stdx.debug.assert(memory == .pinned_host, "asPinnedHostBuffer({}) expects a buffer allocated on host memory, got {}. see `toMemory`", .{ self, memory }); + // stdx.debug.assert((memory == .pinned_host) or (memory == .unpinned_host), "asHostBuffer({f}) expects a buffer allocated on host memory, got {t}. see `copyToMemory`", .{ self, memory }); const ptr: [*]u8 = @ptrCast(self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable); return HostBuffer.fromBytes(self._shape, ptr[0..self._shape.byteSize()]); } @@ -299,6 +304,12 @@ pub const Buffer = struct { }; } + pub fn opaqueDeviceMemoryDataPointer(self: Buffer) [*]u8 { + stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer", .{}); + const opaque_ptr: *anyopaque = self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable; + return @ptrCast(opaque_ptr); + } + /// Fetches the content of the given buffer into a stack variable of the given type. pub fn getValue(self: Buffer, T: type) !T { stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {f} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); @@ -390,13 +401,31 @@ pub const Buffer = struct { return @reduce(.Or, self._shape._sharding_info); } - pub fn copyToMemory(self: Buffer, memory: *const pjrt.Memory) !Buffer { + pub const CopyToMemoryOpts = struct { + wait: bool = true, + }; + + pub fn copyToMemory(self: Buffer, platform: Platform, memory: Memory, opts: CopyToMemoryOpts) !Buffer { + const pjrt_memory = platform.pjrt_client.memoryByKind(self._api, memory.toPjrtMemory()); + if (pjrt_memory == null) { + stdx.debug.panic("Memory destination `{s}` for {f}", .{ memory.pjrtName(), self }); + } + var new_shards: Buffer.Shards = .{}; for (self._shards.slice()) |shard| { - const new_shard = try shard.copyToMemory(self._api, memory); + const new_shard = try shard.copyToMemory(self._api, pjrt_memory.?); new_shards.appendAssumeCapacity(new_shard); } + if (opts.wait) { + for (new_shards.constSlice()) |shard| { + const event = shard.getReadyEvent(self._api); + if (event) |e| { + try e.awaitBlocking(self._api); + } + } + } + return Buffer{ ._shape = self._shape, ._shards = new_shards, ._api = self._api }; } diff --git a/zml/callback.zig b/zml/callback.zig new file mode 100644 index 00000000..4d6ab90e --- /dev/null +++ b/zml/callback.zig @@ -0,0 +1,313 @@ +const std = @import("std"); + +const asynk = @import("async"); +const mlir = @import("mlir"); +const pjrt = @import("pjrt"); +const stablehlo = @import("mlir/dialects").stablehlo; +const stdx = @import("stdx"); + +const Buffer = @import("buffer.zig").Buffer; +const CompilationContext = @import("module.zig").CompilationContext; +const DataType = @import("dtype.zig").DataType; +const HostBuffer = @import("hostbuffer.zig").HostBuffer; +const mlirx = @import("mlirx.zig"); +const pjrtx = @import("pjrtx.zig"); +const Platform = @import("platform.zig").Platform; +const Shape = @import("shape.zig").Shape; +const Tensor = @import("tensor.zig").Tensor; + +const log = std.log.scoped(.@"zml/callback"); + +/// Inserts a user-defined callback into the computation graph. +/// The callback is defined with a struct, that store runtime information needed by the callback. +/// +/// ❗Experimental API❗ +/// +/// ```zig +/// pub const MyCallback = struct { +/// // a unique type_id will be set by the PJRT plugin during registration. +/// pub var type_id: pjrt.ffi.TypeId = undefined; +/// +/// pub const callback_config: zml.callback.Config = .{ +/// // assumption this custom call makes about the input / output buffers +/// }; +/// +/// // Required, this will tell the callback in which env it runs. +/// platform: zml.Platform, +/// // data needed by the callback +/// my_data: []const u8, +/// +/// // storage modified by the runtime to tell the callback where it should write its results. +/// // Normally the callback doesn't need to allocate as the input and output buffers are given. +/// results: [1]Buffer = undefined, +/// +/// pub fn init(my_data: []const u8) !MyCallback { +/// return .{ .my_data = my_data }; +/// } +/// +/// pub fn call(callback: *MyCallback, input: Buffer) !void { +/// // Do something with `input` and `callback.my_data`, write the results inside `callback.results[0]` +/// } +/// }; +/// ``` +/// +/// See eg the implementation of the `zml.callback.Print` callback, for a practical example. +/// +/// Note calling this during the compilation of a module, isn't enough: +/// +/// * backend need to be made aware of the callback, see `zml.Platform.registerCallback` +/// * executable need to know the specific data needed by `MyCallback`, see `zml.Exe.bind` +pub fn call( + comptime Callback: type, + inputs: TensorArgs(Callback), + output_shapes: []const Shape, +) []Tensor { + checkIsValidCallback(Callback); + + const ctx = CompilationContext.current(); + const allocator = ctx.allocator(); + const mlir_ctx = ctx.mlirCtx(); + const platform = ctx._platform; + const pjrt_api = platform.pjrt_api; + + if (pjrt_api.ffi() == null) { + stdx.debug.panic("Custom calls are not supported for target {s}", .{@tagName(platform.target)}); + } + + const output_tensors = allocator.alloc(Tensor, output_shapes.len) catch @panic("OOM"); + // Note: we don't always free output_tensor, because it's returned to the caller. + // It's also why we allocate it first so that it doesn't fragment the arena. + errdefer allocator.free(output_tensors); + + const output_types = allocator.alloc(mlir.Type, output_shapes.len) catch @panic("OOM"); + defer allocator.free(output_types); + for (output_types, output_shapes) |*output_type, output_shape| { + output_type.* = mlirx.tensorType(mlir_ctx, output_shape); + } + const input_values = allocator.alloc(mlir.Value, inputs.len) catch @panic("OOM"); + defer allocator.free(input_values); + for (input_values, inputs) |*input_value, input_tensor| { + input_value.* = input_tensor.value(); + } + + const target_name = "zml$" ++ @typeName(Callback); + const op = stablehlo.custom_call( + mlir_ctx, + input_values, + .{ + .call_target_name = target_name, + .api_version = .typed_ffi, + .backend_config = .dict(mlir_ctx, &.{}), + .additional_attributes = &.{.{ "mhlo.frontend_attributes", .dict(mlir_ctx, &.{}) }}, + .has_side_effect = true, + .output_operand_aliases = Callback.callback_config.output_operand_aliases, + }, + output_types, + mlir_ctx.location(@src()), + ); + + for (output_tensors, output_shapes, 0..) |*output_tensor, output_shape, i| { + output_tensor.* = Tensor._result(output_shape, op.result(i)); + } + return output_tensors; +} + +/// Describe properties of a callback +/// +/// * output_operand_aliases: the callback reuse input buffer to write the output +/// * copy_inputs_to_host_pinned: the callback need to work on host visible buffers +/// * traits: PJRT specified properties of the callback +pub const Config = struct { + output_operand_aliases: []const i64 = &.{}, + copy_inputs_to_host_pinned: bool = false, + // TODO: document precisely what `command_buffer_compatible` is doing and its limitations. + traits: pjrt.ffi.HandlerTraits = .{ .command_buffer_compatible = false }, + // TODO: handle sharded inputs +}; + +/// Compile-time check that a callback has all informations we require. +pub fn checkIsValidCallback(Callback: type) void { + stdx.debug.assertComptime(@hasDecl(Callback, "call"), "Expected callback {} to have a call method", .{Callback}); + const ArgsT = stdx.meta.FnArgs(Callback.call); + inline for (@typeInfo(ArgsT).@"struct".fields[1..]) |field| { + stdx.debug.assertComptime(field.type == Buffer, "Expected callback {}.call arguments to be of type zml.Buffer, got {}", .{ Callback, field.type }); + } + + stdx.debug.assertComptime(@hasDecl(Callback, "type_id") and @TypeOf(Callback.type_id) == pjrt.ffi.TypeId, "Expected callback {} to have a field `pub var type_id: pjrt.ffi.TypeId`", .{Callback}); + stdx.debug.assertComptime(@hasDecl(Callback, "callback_config") and @TypeOf(Callback.callback_config) == Config, "Expected callback {} to have a field `pub const callback_config: zml.CustomCallOptions`", .{Callback}); +} + +pub fn register(Callback: type, platform: Platform) pjrt.ApiError!void { + checkIsValidCallback(Callback); + + const ffi = platform.pjrt_api.ffi() orelse return error.Unavailable; + const target_name = "zml$" ++ @typeName(Callback); + + const proxy_cb = proxy(Callback); + Callback.type_id = try ffi.registerTypeId(platform.pjrt_api, @typeName(Callback)); + try ffi.register(platform.pjrt_api, target_name, @tagName(platform.target), &proxy_cb, Callback.callback_config.traits); + log.debug("Registered custom call {} with target name \"{s}\"", .{ Callback, target_name }); +} + +fn proxy(Callback: type) pjrt.ffi.Handler { + return struct { + pub fn cb(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { + return CallbackImpl(Callback, call_frame); + } + }.cb; +} + +fn CallbackImpl(comptime Callback: type, call_frame: *pjrt.ffi.CallFrame) ?*pjrt.ffi.Error { + if (call_frame.registeringHook()) return null; + + const opts = Callback.callback_config; + + const execution_context = call_frame.ctx; + log.debug("Custom call {s} called !", .{@typeName(Callback)}); + const user_ctx_opaque = execution_context.getContext(Callback.type_id, call_frame.api) catch { + log.err("{} user data was never given for current executable", .{Callback}); + return .create(call_frame.api, .failed_precondition, "failed to fetch user context" ++ @typeName(Callback)); + }; + const user_ctx: *Callback = @ptrCast(@alignCast(user_ctx_opaque)); + // We actually have one more constraint here, we force the Callback to have a platform field, + // and to correctly set it. + // Is this good ? We could also simplify this by registering ourselves the `Platform` type id. + const platform: Platform = user_ctx.platform; + + // Hook to get a cuda stream in the callback. + if (@hasField(Callback, "stream") and platform.target != .cpu) { + const stream = call_frame.api.stream(execution_context); + user_ctx.stream = stream; + } + + var callback_args: std.meta.ArgsTuple(@TypeOf(Callback.call)) = undefined; + callback_args[0] = user_ctx; + + inline for (1..callback_args.len, call_frame.args.buffers()) |i, ffi_buffer| { + const shape = shapeFromFfi(ffi_buffer); + var zml_buffer: Buffer = if (platform.target == .cpu) + .asViewOfHostBuffer(platform, .fromBytes(shape, ffi_buffer.data[0..shape.byteSize()])) + else + .asViewOfDeviceBuffer(platform, shape, null, ffi_buffer.data); + if (opts.copy_inputs_to_host_pinned and platform.target != .cpu) { + log.debug("Copying argument {d} {f} {*} to host_pinned memory !", .{ i, zml_buffer, zml_buffer.opaqueDeviceMemoryDataPointer() }); + zml_buffer = zml_buffer.copyToMemory(platform, .host_pinned, .{ .wait = true }) catch |err| { + log.err("Failed to copy input buffer {d} {f} {*} to host_pinned: {}", .{ i, zml_buffer, zml_buffer.opaqueDeviceMemoryDataPointer(), err }); + return .create(call_frame.api, .resource_exhausted, "host pinned OOM"); + }; + log.debug("--> {f} {*} ({})", .{ zml_buffer, zml_buffer.opaqueDeviceMemoryDataPointer(), @as(*const f32, @ptrCast(@alignCast(zml_buffer.opaqueDeviceMemoryDataPointer()))).* }); + } + callback_args[i] = zml_buffer; + } + + defer { + if (opts.copy_inputs_to_host_pinned and platform.target != .cpu) { + inline for (1..callback_args.len) |i| callback_args[i].deinit(); + } + } + + for (0..call_frame.results.len) |i| { + const ffi_buffer = call_frame.results.buffers()[i]; + const ffi_buffer_shape = shapeFromFfi(ffi_buffer); + + if (platform.target == .cpu) { + user_ctx.results[i] = Buffer.asViewOfHostBuffer(platform, HostBuffer.fromBytes(ffi_buffer_shape, ffi_buffer.data[0..ffi_buffer_shape.byteSize()])); + } else { + user_ctx.results[i] = Buffer.asViewOfDeviceBuffer(platform, shapeFromFfi(ffi_buffer), null, ffi_buffer.data); + } + } + + @call(.auto, Callback.call, callback_args) catch |err| { + log.err("Callback {} failed with {}", .{ Callback, err }); + return .create(call_frame.api, .internal, "internal callback error"); + }; + + return .ok; +} + +/// Internal custom calls. +/// These are not meant to be used by users, but rather by the library itself. +pub const internal_callbacks = [_]type{ + Print, +}; + +pub fn registerInternalCallbacks(platform: Platform) !void { + inline for (internal_callbacks) |Callback| { + try register(Callback, platform); + // log.debug("Registered internal custom call {s} with type_id {d}", .{ @typeName(Callback), Callback.type_id.type_id }); + } +} + +/// Allocate user data data needed by the ZML provided custom calls. +pub fn bindInternalCallbacks( + arena: std.mem.Allocator, + platform: Platform, + ffi: pjrt.Ffi, + execute_context: *pjrt.ExecuteContext, +) (std.mem.Allocator.Error || pjrt.ApiError)!void { + // Atm we don't have a mechanism to detect which ZML callbacks the executable needs, + // so we always allocate. + { + // Print + const print_ptr = try arena.create(Print); + print_ptr.* = try .init(platform); + try addUserData(Print, platform.pjrt_api, ffi, execute_context, print_ptr); + } +} + +pub fn addUserData( + Callback: type, + api: *const pjrt.Api, + ffi: pjrt.Ffi, + execute_context: *pjrt.ExecuteContext, + user_data: *Callback, +) pjrt.ApiError!void { + try ffi.addUserData( + api, + execute_context, + .{ .type_id = Callback.type_id.type_id, .user_data = @ptrCast(user_data) }, + ); + log.debug("Bound {s}@{x} with type id {d} on {any}", .{ @typeName(Callback), @intFromPtr(user_data), Callback.type_id.type_id, execute_context }); +} + +/// The print callback +pub const Print = struct { + // a unique type_id will be set by the PJRT plugin during registration. + pub var type_id: pjrt.ffi.TypeId = undefined; + + pub const callback_config: Config = .{ + // Print callback pretends to modify the given input buffer, but just returns it unmodified. + .output_operand_aliases = &.{0}, + // It also needs PJRT to copy the data on the host first so it can print it. + .copy_inputs_to_host_pinned = true, + // Print is fairly predictable and can be captured in an execution graph. + .traits = .{ .command_buffer_compatible = false }, + }; + + platform: Platform, + results: [1]Buffer = undefined, + + pub fn init(platform: Platform) !Print { + return .{ .platform = platform }; + } + + pub fn call(_: *Print, input: Buffer) !void { + std.log.defaultLog(.info, .zml, "Device buffer: {f}: {d:20.3}", .{ input, input.asHostBuffer() }); + } +}; + +fn shapeFromFfi(ffi_buffer: *const pjrt.ffi.Buffer) Shape { + const dt: DataType = switch (ffi_buffer.dtype) { + .invalid => stdx.debug.panic("Invalid FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), + .token, .f8e4m3, .f8e3m4 => stdx.debug.panic("Unsupported FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), + inline else => |t| @field(DataType, @tagName(t)), + }; + return Shape.init(ffi_buffer.dims(), dt); +} + +fn TensorArgs(Callback: type) type { + const ArgsT = stdx.meta.FnArgs(Callback.call); + + const args = @typeInfo(ArgsT).@"struct".fields; + return [args.len - 1]Tensor; +} diff --git a/zml/context.zig b/zml/context.zig index 15acb93b..27d635a0 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -7,16 +7,19 @@ const runfiles = @import("runfiles"); const runtimes = @import("runtimes"); const stdx = @import("stdx"); -const DataType = @import("dtype.zig").DataType; -const HostBuffer = @import("hostbuffer.zig").HostBuffer; const pjrt = @import("pjrtx.zig"); -const Platform = @import("platform.zig").Platform; -const Shape = @import("shape.zig").Shape; -const Target = @import("platform.zig").Target; -const zml_platform = @import("platform.zig"); -const PjrtApiMap = std.EnumArray(Target, ?*const pjrt.Api); -const PlatformsMap = std.EnumArray(Target, ?Platform); +const zml = struct { + pub const callback = @import("callback.zig"); + pub const HostBuffer = @import("hostbuffer.zig").HostBuffer; + pub const Platform = @import("platform.zig").Platform; + pub const platform = @import("platform.zig"); + pub const Shape = @import("shape.zig").Shape; + pub const Target = @import("platform.zig").Target; +}; + +const PjrtApiMap = std.EnumArray(zml.Target, ?*const pjrt.Api); +const PlatformsMap = std.EnumArray(zml.Target, ?zml.Platform); const log = std.log.scoped(.@"zml/context"); test { @@ -94,7 +97,7 @@ pub const Context = struct { return .{ .platforms = PlatformsMap.initFill(null) }; } - fn platformToLibrary(comptime target: Target) []const u8 { + fn platformToLibrary(comptime target: zml.Target) []const u8 { const ext = switch (builtin.os.tag) { .windows => ".dll", .macos, .ios, .watchos => ".dylib", @@ -105,7 +108,7 @@ pub const Context = struct { }; } - pub fn pjrtApi(target: Target) *const pjrt.Api { + pub fn pjrtApi(target: zml.Target) *const pjrt.Api { return Context.apis.get(target).?; } @@ -119,12 +122,12 @@ pub const Context = struct { self.* = undefined; } - const prefered_targets = [_]Target{ .tpu, .neuron, .cuda, .rocm, .cpu }; + const prefered_targets = [_]zml.Target{ .tpu, .neuron, .cuda, .rocm, .cpu }; /// Automatically selects the best Platform loaded in the current Context. /// /// For example, if supported, this will select a platform corresponding to an accelerator (GPU, TPU, ...). - pub fn autoPlatform(self: *Context, opts: Platform.CreateOptions) Platform { + pub fn autoPlatform(self: *Context, opts: zml.Platform.CreateOptions) zml.Platform { stdx.debug.assert(prefered_targets.len == apis.values.len, "New target need to be inserted inside `zml.Context.preferred_targets`", .{}); return self.platformByPreferences(opts, &prefered_targets); @@ -133,7 +136,7 @@ pub const Context = struct { /// Given a list of preferred targets to select the best Platform /// /// For example, if supported, this will select a platform corresponding to an accelerator (GPU, TPU, ...). - pub fn platformByPreferences(self: *Context, opts: Platform.CreateOptions, prefered: []const Target) Platform { + pub fn platformByPreferences(self: *Context, opts: zml.Platform.CreateOptions, prefered: []const zml.Target) zml.Platform { // Try prefered targets. for (prefered) |target| { if (apis.get(target) == null) continue; @@ -150,7 +153,7 @@ pub const Context = struct { // CPU should only be use as fallback. if (target == .cpu) continue; if (entry.value.* == null) continue; - if (std.mem.indexOfScalar(Target, prefered, target) != null) continue; + if (std.mem.indexOfScalar(zml.Target, prefered, target) != null) continue; return self.platform(target, opts) catch |err| { log.err("Failed to load platform .{s}: {}", .{ @tagName(target), err }); continue; @@ -164,25 +167,25 @@ pub const Context = struct { }; } - pub fn platform(self: *Context, target: Target, opts: Platform.CreateOptions) !Platform { + pub fn platform(self: *Context, target: zml.Target, opts: zml.Platform.CreateOptions) !zml.Platform { if (self.platforms.get(target)) |p| { return p; } const api = Context.apis.get(target); if (api == null) return error.PlatformNotCompiled; - const p = try Platform.init(target, api.?, opts); + const p = try zml.Platform.init(target, api.?, opts); if (p.getDevices().len == 0) { log.err("No device found for platform {} !", .{target}); return error.NoDevicesFound; } - try CustomCall.registerZmlCustomCalls(p); - self.platforms.set(target, p); + try zml.callback.registerInternalCallbacks(p); + return p; } - pub fn printAvailablePlatforms(self: Context, selected: Platform) void { + pub fn printAvailablePlatforms(self: Context, selected: zml.Platform) void { // List available targets log.info("Available Platforms:", .{}); const selected_prefix = "✅"; @@ -190,7 +193,7 @@ pub const Context = struct { const selected_postfix = "(AUTO-SELECTED)"; const not_selected_postfix = ""; - for (zml_platform.available_targets) |target| { + for (zml.platform.available_targets) |target| { log.info(" {s} {s} {s}", .{ if (target == selected.target) selected_prefix else not_selected_prefix, @tagName(target), @@ -211,133 +214,4 @@ pub const Context = struct { } } } - - pub const HostCallback = fn (?*anyopaque, []const HostBuffer, []const HostBuffer) void; -}; - -const CustomCall = struct { - pub fn registerZmlCustomCalls(platform: Platform) !void { - const ffi = platform.pjrt_api.ffi() orelse { - log.warn("Registering custom calls failed: No FFI Extension found in {s} PJRT Plugin.", .{@tagName(platform.target)}); - return; - }; - try ffi.register(platform.pjrt_api, "zmlHostBufferCallback", @tagName(platform.target), &hostBufferCallback, .{}); - } - - fn hostBufferCallback(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { - if (call_frame.registeringHook()) return null; - - const callback_attr = call_frame.attrs.getByName(.scalar, "callback") orelse unreachable; - std.debug.assert(callback_attr.dtype == .u64); - const callback: *const Context.HostCallback = @ptrFromInt(callback_attr.get(usize)); - - const user_ctx_ptr = call_frame.attrs.getByName(.scalar, "user_context") orelse unreachable; - std.debug.assert(user_ctx_ptr.dtype == .u64); - const user_ctx: ?*anyopaque = @ptrFromInt(user_ctx_ptr.get(usize)); - - const input_buffers = stdx.stackSlice(8, HostBuffer, call_frame.args.len); - for (input_buffers, 0..) |*b, i| { - b.* = hostBufferFromPinnedBuffer(call_frame.args.buffers()[i]); - } - - const output_buffers = stdx.stackSlice(8, HostBuffer, call_frame.results.len); - for (output_buffers, 0..) |*b, i| { - b.* = hostBufferFromPinnedBuffer(call_frame.results.buffers()[i]); - } - - callback(user_ctx, input_buffers, output_buffers); - return null; - } -}; - -fn getShape(buffer_desc: *const pjrt.ffi.Buffer) Shape { - // log.warn("received buffer {}", .{buffer_desc}); - const dt: DataType = switch (buffer_desc.dtype) { - .invalid => @panic("invalid ffi"), - .pred => .bool, - .i8 => .i8, - .i16 => .i16, - .i32 => .i32, - .i64 => .i64, - .token, .f8e4m3, .f8e3m4 => @panic("Unsupported ffi type"), - inline else => |t| @field(DataType, @tagName(t)), - }; - return Shape.init(buffer_desc.dims(), dt); -} - -/// Create a HostBuffer from a ffi description of a buffer. -/// Normally the ffi describe device buffer but we assume they are located in pinned memory, -/// and therefore the data pointer is readable both from host and from device. -fn hostBufferFromPinnedBuffer(buffer_desc: *const pjrt.ffi.Buffer) HostBuffer { - const buffer_shape = getShape(buffer_desc); - return HostBuffer.fromBytes( - buffer_shape, - buffer_desc.data[0..buffer_shape.byteSize()], - ); -} - -pub const cuda = struct { - pub var streamSynchronize: StreamSynchronize = @ptrFromInt(0xdeadc00da00); - pub var cuLaunchHostFunc: CuLaunchHostFunc = @ptrFromInt(0xdeadc00da00); - var _memcpyAsync: MemcpyAsync = @ptrFromInt(0xdeadc00da00); - var _memcpyBlocking: MemcpyBlocking = @ptrFromInt(0xdeadc00da00); - - pub const MemcpyKind = enum(c_int) { - host_to_host = 0, - host_to_device = 1, - device_to_host = 2, - device_to_device = 3, - inferred = 4, - }; - - const MemcpyAsync = *const fn (dst: *anyopaque, src: *const anyopaque, count: usize, kind: MemcpyKind, stream: ?*anyopaque) callconv(.c) c_int; - const MemcpyBlocking = *const fn (dst: *anyopaque, src: *const anyopaque, count: usize, kind: MemcpyKind) callconv(.c) c_int; - const StreamSynchronize = *const fn (stream: *anyopaque) callconv(.c) c_int; - const CuLaunchHostFunc = *const fn (stream: *anyopaque, host_func: *const fn (user_data: *const anyopaque) callconv(.c) void, user_data: *const anyopaque) callconv(.c) c_int; - - pub fn init() void { - var cudart = std.DynLib.open("libcudart.so.12") catch { - log.err("cudart not found, callback will segfault", .{}); - return; - }; - defer cudart.close(); - - _memcpyAsync = cudart.lookup(MemcpyAsync, "cudaMemcpyAsync") orelse { - @panic("cudaMemcpyAsync not found"); - }; - _memcpyBlocking = cudart.lookup(MemcpyBlocking, "cudaMemcpy") orelse { - @panic("cudaMemcpy not found"); - }; - streamSynchronize = cudart.lookup(StreamSynchronize, "cudaStreamSynchronize") orelse { - @panic("cudaStreamSynchronize not found"); - }; - cuLaunchHostFunc = cudart.lookup(CuLaunchHostFunc, "cudaLaunchHostFunc") orelse { - @panic("cudaLaunchHostFunc not found"); - }; - } - - pub fn memcpyToHostBlocking(dst: []u8, src: *const anyopaque) void { - const err = _memcpyBlocking(dst.ptr, src, dst.len, .device_to_host); - check(err); - } - - pub fn memcpyToDeviceBlocking(dst: *anyopaque, src: []const u8) void { - const err = _memcpyBlocking(dst, src.ptr, src.len, .host_to_device); - check(err); - } - - pub fn memcpyToDeviceAsync(dst: *anyopaque, src: []const u8, stream: ?*anyopaque) void { - const err = _memcpyAsync(dst, src.ptr, src.len, .host_to_device, stream); - check(err); - } - - pub fn memcpyToHostAsync(dst: []u8, src: *const anyopaque, stream: ?*anyopaque) void { - const err = _memcpyAsync(dst.ptr, src, dst.len, .device_to_host, stream); - check(err); - } - - pub fn check(err: c_int) void { - if (err == 0) return; - stdx.debug.panic("CUDA error: {d}", .{err}); - } }; diff --git a/zml/exe.zig b/zml/exe.zig index 625fecf9..d4c426b7 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -5,6 +5,7 @@ const stdx = @import("stdx"); const aio = @import("aio.zig"); const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("tensor.zig").Bufferized; +const callback = @import("callback.zig"); const CompilationContext = @import("module.zig").CompilationContext; const meta = @import("meta.zig"); const pjrt = @import("pjrtx.zig"); @@ -154,7 +155,7 @@ pub const BaseExe = struct { exe: *pjrt.LoadedExecutable, /// The execution context for this executable. - context: ?*pjrt.ExecuteContext = null, + execute_context: ?*pjrt.ExecuteContext, /// Pre-allocated slice of buffers to use as inputs when the module is called. input_per_device: []const [*]*pjrt.Buffer, @@ -205,9 +206,18 @@ pub const BaseExe = struct { const all_shapes = try allocator.alloc(Shape, n_in + n_out); @memcpy(all_shapes[0..n_in], args.input_shapes); @memcpy(all_shapes[n_in..], args.result_shapes); + + var execute_context: ?*pjrt.ExecuteContext = null; + if (platform.pjrt_api.ffi()) |ffi| { + log.info("Created context execution {*} for {*}", .{ execute_context, exe }); + execute_context = try platform.pjrt_api.createExecuteContext(); + try callback.bindInternalCallbacks(allocator, platform, ffi, execute_context.?); + } + return .{ .platform = platform, .exe = exe, + .execute_context = execute_context, .ready_buffer_count = 0, .input_buffer_count = @intCast(n_in), .num_devices = args.n_devices, @@ -220,7 +230,7 @@ pub const BaseExe = struct { } pub fn deinit(self: BaseExe) void { - if (self.context) |ctx| { + if (self.execute_context) |ctx| { ctx.deinit(self.platform.pjrt_api); } self._arena.deinit(); @@ -244,16 +254,16 @@ pub const BaseExe = struct { // even if it has been marked as "can be donated" during compilation. // TODO: expose it ? .non_donatable_input_indices = &.{}, - .context = self.context, + .context = self.execute_context, }) catch |err| { std.debug.panic("PJRT_LoadedExecutable_Execute failed with: {}", .{err}); }; - for (events[0..sharding.num_partitions]) |e| { - if (e) |ev| { - ev.await_(self.platform.pjrt_api) catch unreachable; - } - } + // for (events[0..sharding.num_partitions]) |e| { + // if (e) |ev| { + // ev.await_(self.platform.pjrt_api) catch unreachable; + // } + // } } pub fn _unsafeAssignResults(self: BaseExe, T: type, result: *T) void { @@ -285,6 +295,17 @@ pub const BaseExe = struct { stdx.debug.internalAssert(local_ctx.index == self.result_shapes.len, "Pjrt call returned {} tensors, but the return type {s}, contains {} Buffers. Note that modules need to have a comptime know number of returned tensors.", .{ self.output_per_device.len, @typeName(T), local_ctx.index }); } + pub fn bind(exe: BaseExe, Callback: type, op: *Callback) !void { + stdx.debug.assert(exe.execute_context != null, "Exe doesn't have an execution context", .{}); + const pjrt_api = exe.platform.pjrt_api; + + if (pjrt_api.ffi()) |ffi| { + try callback.addUserData(Callback, pjrt_api, ffi, exe.execute_context.?, op); + } else { + stdx.debug.panic("Callbacks are not supported for target {s}", .{@tagName(exe.platform.target)}); + } + } + pub fn serialize(self: BaseExe, writer: anytype) !void { var executable = try self.exe.getExecutable(self.platform.pjrt_api); var serialize_result = try executable.serialize(self.platform.pjrt_api); @@ -314,11 +335,11 @@ pub const BaseExe = struct { pub fn clone(self: BaseExe, parent_allocator: std.mem.Allocator) !BaseExe { var exe: BaseExe = try .init(parent_allocator, self.platform, self.exe, .{ - .n_in = self.input_buffer_count, + .input_shapes = self.input_shapes, .result_shapes = self.result_shapes, .n_devices = self.num_devices, }); - exe.context = self.context; + exe.execute_context = self.execute_context; return exe; } }; @@ -348,6 +369,14 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { return new; } + /// For a given customCall inside this executable, + /// provide a pointer to runtime data. + /// The caller keeps memory ownership and need to ensure that the value + /// stays alive as long as the executable. + pub fn bind(self: Self, comptime T: type, value: *T) !void { + try self.inner.bind(T, value); + } + pub fn serialize(self: Self, writer: anytype) !void { return try self.inner.serialize(writer); } diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index c2099b0d..848fee86 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -325,37 +325,18 @@ pub const HostBuffer = struct { self: HostBuffer, writer: anytype, ) !void { - // TODO debug option - // try writer.print("HostBuffer(.{f})@0x{x}", .{ self._shape, @intFromPtr(self._data) }); try writer.print("HostBuffer(.{f})", .{self._shape}); } - /// Formatter for a HostBuffer that also print the values not just the shape. - /// Usage: `std.log.info("my buffer: {}", .{buffer.pretty()});` - pub fn pretty(self: HostBuffer) PrettyPrinter { - return .{ .x = self }; + pub fn formatNumber(self: HostBuffer, writer: *std.io.Writer, n: std.fmt.Number) std.io.Writer.Error!void { + return self.prettyPrintIndented(writer, 4, 0, n); } - pub const PrettyPrinter = struct { - x: HostBuffer, - - // TODO(0.15.0) revisit pretty printer - pub fn format(self: PrettyPrinter, writer: anytype) !void { - const fmt_: stdx.fmt.Fmt = switch (self.x.dtype().class()) { - .integer => .parse(i32, "d"), - .float => .parse(f32, "d"), - else => .parse(void, ""), - }; - const options: std.fmt.FormatOptions = .{}; - try prettyPrint(self.x, writer, .{ .fmt = fmt_, .options = options }); - } - }; - - pub fn prettyPrint(self: HostBuffer, writer: *std.Io.Writer, options: stdx.fmt.FullFormatOptions) !void { + pub fn prettyPrint(self: HostBuffer, writer: *std.Io.Writer, options: std.fmt.Number) !void { return self.prettyPrintIndented(writer, 4, 0, options); } - fn prettyPrintIndented(self: HostBuffer, writer: *std.Io.Writer, num_rows: u8, indent_level: u8, options: stdx.fmt.FullFormatOptions) !void { + fn prettyPrintIndented(self: HostBuffer, writer: *std.Io.Writer, num_rows: u8, indent_level: u8, options: std.fmt.Number) !void { if (self.rank() == 0) { // Special case input tensor is a scalar return switch (self.dtype()) { @@ -363,9 +344,10 @@ pub const HostBuffer = struct { const val: dt.toZigType() = self.items(dt.toZigType())[0]; return switch (comptime dt.class()) { // Since we have custom floats, we need to explicitly convert to float32 ourselves. - .float => stdx.fmt.formatFloatValue(floats.floatCast(f32, val), options, writer), - .integer => stdx.fmt.formatIntValue(val, options, writer), - .bool, .complex => stdx.fmt.formatAnyValue(val, options, writer), + .float => stdx.fmt.formatFloat(floats.floatCast(f32, val), options, writer), + .integer => stdx.fmt.formatInt(val, options, writer), + .bool => stdx.fmt.formatBool(val, options, writer), + .complex => stdx.fmt.formatComplex(val, options, writer), }; }, }; @@ -380,7 +362,8 @@ pub const HostBuffer = struct { switch (comptime dt.class()) { .float => try stdx.fmt.formatFloatSlice(values, options, writer), .integer => try stdx.fmt.formatIntSlice(values, options, writer), - .bool, .complex => try stdx.fmt.formatAnySlice(values, options, writer), + .complex => try stdx.fmt.formatComplexSlice(values, options, writer), + .bool => try stdx.fmt.formatBoolSlice(values, options, writer), } }, } diff --git a/zml/module.zig b/zml/module.zig index 33d22f78..6263d3fd 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -1178,9 +1178,9 @@ pub fn hash(hasher: *std.hash.Wyhash, key: anytype, comptime strat: std.hash.Str .@"anyframe", .@"fn" => hash(hasher, @intFromPtr(key), strat), .pointer => |info| switch (info.size) { .one => switch (strat) { - .shallow => hash(hasher, @intFromPtr(key), .Shallow), - .deep => hash(hasher, key.*, .Shallow), - .deeprecursive => switch (@typeInfo(info.child)) { + .Shallow => hash(hasher, @intFromPtr(key), .Shallow), + .Deep => hash(hasher, key.*, .Shallow), + .DeepRecursive => switch (@typeInfo(info.child)) { .@"opaque", .@"fn" => hash(hasher, @intFromPtr(key), .Shallow), else => hash(hasher, key.*, .DeepRecursive), }, @@ -1196,7 +1196,7 @@ pub fn hash(hasher: *std.hash.Wyhash, key: anytype, comptime strat: std.hash.Str .many, .c, => switch (strat) { - .shallow => hash(hasher, @intFromPtr(key), .Shallow), + .Shallow => hash(hasher, @intFromPtr(key), .Shallow), else => @compileError( \\ unknown-length pointers and C pointers cannot be hashed deeply. \\ Consider providing your own hash function. diff --git a/zml/ops.zig b/zml/ops.zig index c26eac3c..ee125086 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -764,33 +764,6 @@ pub fn fromMlirOperationWithTags(op: mlir.Operation, base: anytype) @TypeOf(base return res; } -pub const HostCallbackOpt = struct { - has_side_effect: bool = false, - output_operand_aliases: []const i64 = &.{}, -}; - -pub fn addHostCallback( - callback: *const Context.HostCallback, - blkctx: ?*anyopaque, - inputs: []const Tensor, - output_shapes: []const Shape, - opts: HostCallbackOpt, -) []Tensor { - return customCall( - "zmlHostBufferCallback", - inputs, - output_shapes, - .{ - .callback = @intFromPtr(callback), - .user_context = @intFromPtr(blkctx), - }, - .{ - .has_side_effect = opts.has_side_effect, - .output_operand_aliases = opts.output_operand_aliases, - }, - ); -} - pub const TritonOps = struct { debug: bool = false, name: [:0]const u8, diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 84343a63..8658aebe 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -207,6 +207,13 @@ pub const Event = opaque { return self.inner().getEventError(api); } + pub fn awaitBlocking(self: *Event, api: *const Api) ApiError!void { + if (self.isReady(api)) { + return; + } + try self.inner().await_(api); + } + pub fn await_(self: *Event, api: *const Api) ApiError!void { defer self.deinit(api); @@ -264,14 +271,14 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{ self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ .num_args = args.num_args, .arguments = @ptrCast(args.arguments), .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, .context = args.context, - } }); + }); } pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { diff --git a/zml/platform.zig b/zml/platform.zig index 838cac48..0bc792e9 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -22,6 +22,11 @@ pub const Platform = struct { target: Target, pjrt_api: *const pjrt.Api, pjrt_client: *pjrt.Client, + + // This make the pjrt struct quite fat, but is only used during compilation. + // TODO: Reconsider having it here, and maybe pass explicitly to compile, + // or create an intermediary struct: + // `const comp = platform.compiler(compile_opts); const exe = comp.compile(...);` compilation_options: CompilationOptions = .{}, pub const MAX_NUM_DEVICES: u8 = 32; @@ -71,17 +76,6 @@ pub const Platform = struct { return res; } - pub fn registerFFIType(self: Platform, comptime T: type) !void { - if (self.pjrt_api.ffi()) |ffi| { - if (!@hasDecl(T, "type_id")) { - stdx.debug.panic("registerFFIType requires type {s} to have a `type_id` i64 field ", .{@typeName(T)}); - } - try ffi.registerTypeId(self.pjrt_api, T); - } else { - stdx.debug.panic("registerFFIType is not available for target {s}", .{@tagName(self.target)}); - } - } - pub fn deinit(self: *Platform) void { self.pjrt_client.deinit(self.pjrt_api); } diff --git a/zml/tensor.zig b/zml/tensor.zig index 26f670d7..63ab44d4 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -5,6 +5,7 @@ const mlir = @import("mlir"); const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; +const callback = @import("callback.zig"); const CompilationContext = @import("module.zig").CompilationContext; const Data = @import("dtype.zig").Data; const DataType = @import("dtype.zig").DataType; @@ -3824,22 +3825,7 @@ pub const Tensor = struct { /// Only for debug purpose, it inserts device to host synchronization /// so it will slow down the program execution. pub fn print(input: Tensor) Tensor { - // TODO: find a way of doing print that doesn't involve a H2D copy. - return ops.addHostCallback( - &printCallback, - null, - &.{input}, - &.{input.shape()}, - .{ .output_operand_aliases = &.{0} }, - )[0]; - } - - fn printCallback(_: ?*anyopaque, inputs: []const HostBuffer, outputs: []const HostBuffer) void { - const host_buffer = inputs[0]; - std.log.defaultLog(.info, .zml, "Device buffer: {f}: {f}", .{ host_buffer.shape(), host_buffer.pretty() }); - // This is true because of the operand aliases. - // Since the result is already pointing to the input we don't need to modify the buffer. - std.debug.assert(host_buffer._data == outputs[0]._data); + return callback.call(callback.Print, .{input}, &.{input.shape()})[0]; } }; diff --git a/zml/testing.zig b/zml/testing.zig index b236400b..5fec596d 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -51,14 +51,13 @@ pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { if (should_free_left) left.deinit(allocator); if (should_free_right) right.deinit(allocator); } - errdefer log.err("\n--> Left: {f}\n--> Right: {f}", .{ left.pretty(), right.pretty() }); - + errdefer log.err("\n--> Left: {0f}{0d:24.3}\n--> Right: {1f}{1d:24.3}", .{ left, right }); if (!std.mem.eql(i64, left.shape().dims(), right.shape().dims())) { log.err("left.shape() {f} != right.shape() {f}", .{ left.shape(), right.shape() }); return error.TestUnexpectedResult; } if (left.dtype() != right.dtype() and !(left.dtype() == .f16 and right.dtype() == .bf16)) { - log.err("left.dtype ({}) != right.dtype ({})", .{ left.dtype(), right.dtype() }); + log.err("left.dtype ({f}) != right.dtype ({f})", .{ left.shape(), right.shape() }); return error.TestUnexpectedResult; } switch (left.dtype()) { @@ -89,7 +88,7 @@ pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { const right_data = right.items(R); for (left_data, right_data, 0..) |l, r, i| { if (!approxEq(f32, zml.floats.floatCast(f32, l), zml.floats.floatCast(f32, r), tolerance)) { - log.err("left.data != right_data.\n < {any:.3} \n > {any:.3}\n error at idx {any}: {any:.3} != {any:.3}", .{ center(left_data, i), center(right_data, i), i, left_data[i], right_data[i] }); + log.err("left.data != right_data.\n < {d:40.3} \n > {d:40.3}\n error at idx {d}: {d:.3} != {d:.3}", .{ stdx.fmt.slice(center(left_data, i)), stdx.fmt.slice(center(right_data, i)), i, left_data[i], right_data[i] }); return error.TestUnexpectedResult; } } diff --git a/zml/zml.zig b/zml/zml.zig index b5cd8039..55559f5c 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -6,11 +6,13 @@ // Namespaces const std = @import("std"); +pub const platform_specific = @import("c"); pub const tokenizer = @import("zml/tokenizer"); pub const aio = @import("aio.zig"); pub const Buffer = @import("buffer.zig").Buffer; pub const Bufferized = @import("tensor.zig").Bufferized; +pub const callback = @import("callback.zig"); pub const CompilationOptions = @import("platform.zig").CompilationOptions; pub const context = @import("context.zig"); pub const Context = @import("context.zig").Context; @@ -43,7 +45,6 @@ pub const Tensor = @import("tensor.zig").Tensor; pub const testing = @import("testing.zig"); pub const torch = @import("torch.zig"); -// pub const tokenizer = @import("tokenizer.zig"); pub const tools = struct { pub const Tracer = @import("tools/tracer.zig").Tracer; }; From 988f2711c55f5a9b726d129ed560ebd58a0cb75f Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 11 Sep 2025 10:28:16 +0200 Subject: [PATCH 028/881] zml: loadById (#322) We already assign a unique tensor id to each tensor object created by a BufferStore. But we weren't using the id to load find the corresponding buffer in the store, instead relying on the name. This is allow to decouple the shape of the model in the struct vs the shape of the model inside the safetensor file. I also moved the id logic out of Tensor and into BufferStore. I also updated llama/main.zig to use the new logic. Bonus: * fix to `zml.call` tag hashing leading to different MLIR for every recompile. * expose Cpu device count to control the number of CPU "devices" --- examples/llama/llama.zig | 157 +++++++++--------- examples/llama/main.zig | 185 +++++++++++---------- examples/simple_layer/main.zig | 17 +- stdx/fmt.zig | 55 ++++++ stdx/stdx.zig | 6 +- zml/aio.zig | 294 +++++++++++++++++++++------------ zml/aio/safetensors.zig | 4 +- zml/aio/tinyllama.zig | 4 +- zml/aio/torch.zig | 2 +- zml/aio/torch/file.zig | 2 +- zml/exe.zig | 4 +- zml/meta.zig | 1 + zml/module.zig | 2 +- zml/platform.zig | 15 +- zml/tensor.zig | 12 -- zml/testing.zig | 2 +- 16 files changed, 460 insertions(+), 302 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index da487bb3..5d0e87d5 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -18,19 +18,22 @@ pub const LlamaLM = struct { int: u32, ints: []u32, }), - num_hidden_layers: usize, - num_attention_heads: usize, - num_key_value_heads: usize, + head_dim: ?u32, + hidden_size: u32, + num_hidden_layers: u32, + num_attention_heads: u32, + num_key_value_heads: u32, rope_theta: f32, - max_position_embeddings: usize, + max_position_embeddings: u32, rms_norm_eps: f32, hf_rope_impl: bool = true, + tie_word_embeddings: bool = false, rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = {} }, }; pub const Options = struct { sampling_strategy: ?zml.nn.SamplingStrategy, - max_seq_len: usize, + max_seq_len: u32, }; lm_head: ?zml.nn.Linear, @@ -40,39 +43,79 @@ pub const LlamaLM = struct { gen_opts: zml.nn.SamplingStrategy = .{}, config: Config, - pub fn init(self: *LlamaLM, config: Config, options: Options) void { - self.config = config; - self.gen_opts = options.sampling_strategy orelse .{}; - self.model.max_seq_len = @intCast(options.max_seq_len); - self.model.num_heads = @intCast(config.num_attention_heads); - self.model.num_kv_heads = @intCast(config.num_key_value_heads); - self.model.rope_opts = .{ + pub fn init(allocator: std.mem.Allocator, config: Config, options: Options, store: zml.aio.BufferStore) !LlamaLM { + const rope_opts: zml.nn.RopeOpts = .{ .layout = if (config.hf_rope_impl) .sequential else .interleaved, .freq_base = config.rope_theta, .scaling = config.rope_scaling, }; - self.model.norm.eps = config.rms_norm_eps; - for (self.model.layers) |*layer| { - layer.self_attn.num_heads = self.model.num_heads; - layer.self_attn.num_kv_heads = self.model.num_kv_heads; - layer.self_attn.rope_opts = self.model.rope_opts; - layer.input_layernorm.eps = config.rms_norm_eps; - layer.post_attention_layernorm.eps = config.rms_norm_eps; - layer.mlp.up_proj.weight = layer.mlp.up_proj.weight.withSharding(.{0}); - layer.mlp.gate_proj.weight = layer.mlp.gate_proj.weight.withSharding(.{0}); - layer.mlp.down_proj.weight = layer.mlp.down_proj.weight.withSharding(.{1}); - layer.self_attn.q_proj.weight = layer.self_attn.q_proj.weight.withSharding(.{0}); - layer.self_attn.k_proj.weight = layer.self_attn.k_proj.weight.withSharding(.{0}); - layer.self_attn.v_proj.weight = layer.self_attn.v_proj.weight.withSharding(.{0}); - layer.self_attn.o_proj.weight = layer.self_attn.o_proj.weight.withSharding(.{1}); + const layers = try allocator.alloc(TransformerLayer, config.num_hidden_layers); + var prefix = try zml.aio.PrefixBuilder.initCapacity(allocator, 1024); + try prefix.push(stdx.noalloc, "model.layers"); + for (0.., layers) |i, *layer| { + try prefix.pushDigit(stdx.noalloc, i); + defer prefix.pop(); + var self_attn = try zml.aio.populateModelWithPrefix(SelfAttn, allocator, store, prefix.concat("self_attn")); + self_attn.num_heads = config.num_attention_heads; + self_attn.num_kv_heads = config.num_key_value_heads; + self_attn.rope_opts = rope_opts; + self_attn.q_proj.weight = self_attn.q_proj.weight.withSharding(.{0}); + self_attn.k_proj.weight = self_attn.k_proj.weight.withSharding(.{0}); + self_attn.v_proj.weight = self_attn.v_proj.weight.withSharding(.{0}); + self_attn.o_proj.weight = self_attn.o_proj.weight.withSharding(.{1}); + + var input_layernorm = try zml.aio.populateModelWithPrefix(RmsNorm, allocator, store, prefix.concat("input_layernorm")); + input_layernorm.eps = config.rms_norm_eps; + + var post_attention_layernorm = try zml.aio.populateModelWithPrefix(RmsNorm, allocator, store, prefix.concat("post_attention_layernorm")); + post_attention_layernorm.eps = config.rms_norm_eps; + + var mlp = try zml.aio.populateModelWithPrefix(Mlp, allocator, store, prefix.concat("mlp")); + mlp.up_proj.weight = mlp.up_proj.weight.withSharding(.{0}); + mlp.gate_proj.weight = mlp.gate_proj.weight.withSharding(.{0}); + mlp.down_proj.weight = mlp.down_proj.weight.withSharding(.{1}); + + layer.* = .{ + .self_attn = self_attn, + .input_layernorm = input_layernorm, + .post_attention_layernorm = post_attention_layernorm, + .mlp = mlp, + }; } - // TODO(Corentin): Fix lm_head sharding when top-k sampling is enabled. - // It currently crashes/compilation fails - if (self.gen_opts.topk == 1 and self.lm_head != null) { - self.lm_head.?.weight = self.lm_head.?.weight.withSharding(.{0}); + var lm_head: ?zml.nn.Linear = null; + if (!config.tie_word_embeddings) { + lm_head = .{ .weight = store.getTensor("lm_head.weight") }; + if (options.sampling_strategy) |gen_opts| { + if (gen_opts.topk == 1) + lm_head.?.weight = lm_head.?.weight.withSharding(.{0}); + } } + + return .{ + .config = config, + .gen_opts = options.sampling_strategy orelse .{}, + .model = .{ + // Weights + .layers = layers, + .embed_tokens = .{ .weight = store.getTensor("model.embed_tokens.weight") }, + .norm = .{ + .weight = store.getTensor("model.norm.weight"), + .eps = config.rms_norm_eps, + }, + // Push down some configs + .max_seq_len = options.max_seq_len, + .num_heads = config.num_attention_heads, + .num_kv_heads = config.num_key_value_heads, + .rope_opts = .{ + .layout = if (config.hf_rope_impl) .sequential else .interleaved, + .freq_base = config.rope_theta, + .scaling = config.rope_scaling, + }, + }, + .lm_head = lm_head, + }; } /// Predicts the token at `token_index` position. @@ -129,36 +172,13 @@ pub const Llama = struct { layers: []TransformerLayer, max_seq_len: u32 = 0, - num_heads: i64 = 32, - num_kv_heads: i64 = 32, + num_heads: u32 = 32, + num_kv_heads: u32 = 32, rope_opts: zml.nn.RopeOpts = .{ .layout = .interleaved, .freq_base = 10_000, }, - const Shape = struct { - s: u32, - layer: u16, - hd: u16, - nh: u16, - nkvh: u16, - dtype: zml.DataType, - }; - - pub fn shape(self: Llama) Shape { - const key_dim = self.layers[0].self_attn.k_proj.weight.dim(0); - const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; - - return .{ - .s = self.max_seq_len, - .layer = @intCast(self.layers.len), - .hd = @intCast(@divExact(key_dim, num_kv_heads)), - .nh = @intCast(self.num_heads), - .nkvh = @intCast(num_kv_heads), - .dtype = self.embed_tokens.weight.dtype(), - }; - } - /// Forward one token, using KV cache for previous tokens. /// Returns result and updated KV cache. pub fn forward(self: Llama, tokens: Tensor, token_index: Tensor, kv_cache: KvCache) struct { Tensor, KvCache } { @@ -177,14 +197,6 @@ pub const Llama = struct { pub fn embed(embed_tokens_: zml.nn.TokenEmbedding, tokens_: Tensor) Tensor { return zml.call(embed_tokens_, .forward, .{tokens_}).withPartialTags(.{.d}); } - - fn initKvCache(self: Llama, embed_shape: zml.Shape) KvCache { - const dims = self.shape(); - var kv_shape = embed_shape.insert(0, .{ .layer = dims.layer }).rename(.{ .s = .k }).splitAxes(.{ .d = .{ .h = dims.nkvh, .hd = dims.hd } }); - const perm = kv_shape.contiguousPerm(.{ .k, .h, .hd }); - kv_shape = kv_shape.transpose(perm.constSlice()); - return KvCache.init(kv_shape); - } }; pub const TransformerLayer = struct { @@ -245,6 +257,9 @@ pub const SelfAttn = struct { k_proj: zml.nn.Linear, v_proj: zml.nn.Linear, + q_norm: ?RmsNorm, + k_norm: ?RmsNorm, + o_proj: zml.nn.Linear, num_heads: i64 = undefined, num_kv_heads: i64 = 0, @@ -282,6 +297,8 @@ pub const SelfAttn = struct { break :b temp.add(token_index.broad(temp.shape())); }; + if (self.q_norm) |norm| q = norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); + if (self.k_norm) |norm| k = norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); q = zml.nn.rope(q, pos_index, self.rope_opts); k = zml.nn.rope(k, pos_index, self.rope_opts); q = q.rename(.{ .s = .q }); @@ -298,16 +315,6 @@ pub const SelfAttn = struct { const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); return .{ zml.call(self.o_proj, .forward, .{attn}), new_kv_cache }; } - - fn initKvCache(key_shape: zml.Shape) KvCache { - // When we call initKvCache, we haven't renamed .s to .k yet. - var kv_shape = key_shape.insert(0, .{ .layer = 1 }).rename(.{ .s = .k }); - const perm = kv_shape.contiguousPerm(.{ .h, .k, .hd }); - kv_shape = kv_shape.transpose(perm.constSlice()); - var res = KvCache.init(kv_shape); - res.layer_index = Tensor.scalar(0, .u32); - return res; - } }; pub const KvCache = struct { @@ -334,8 +341,8 @@ pub const KvCache = struct { pub fn initBuffer(kv_shape: zml.Shape, platform: zml.Platform) !zml.Bufferized(KvCache) { return .{ - .k = try zml.Buffer.constant(platform, kv_shape, 1), - .v = try zml.Buffer.constant(platform, kv_shape, 1), + .k = try zml.Buffer.uninitialized(platform, kv_shape, .{}), + .v = try zml.Buffer.uninitialized(platform, kv_shape, .{}), .layer_index = try zml.Buffer.scalar(platform, 0, .u32), }; } diff --git a/examples/llama/main.zig b/examples/llama/main.zig index af94fd21..c1c8d805 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -22,35 +22,42 @@ pub const std_options: std.Options = .{ .logFn = asynk.logFn(std.log.defaultLog), }; +const params = clap.parseParamsComptime( + \\--help print this help + \\--prompt the prompt + \\--hf-model-path path to the directory containing model weights, config and tokenizer + \\--seed random seed (optional) + \\--seq-len sequence length + \\--create-options platform creation options JSON, defaults to {} + \\--no-llama3 skip prompt template + \\--sharding default: true: sharding on or off +); + pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: LlamaLM.Config, prompt: []const u8, skip_llama3_encoding: bool) ![]u32 { - var tokens = std.array_list.Managed(u32).init(allocator); var encoder = try tokenizer.encoder(); defer encoder.deinit(); if (skip_llama3_encoding) { - // Copy to the arraylist so the ownership is the same in both branches. - try tokens.appendSlice(try encoder.encode(prompt)); - return tokens.toOwnedSlice(); + // Copy so the ownership is the same in both branches. + return try allocator.dupe(u32, try encoder.encode(prompt)); } - const start_header_id = tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; - const end_header_id = tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; - const eot_id = tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; - const newline_id = (try encoder.encode("\n"))[0]; + const start_header = tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; + const end_header = tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; + const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; + const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; + const eot = tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; + const newline = (try encoder.encode("\n"))[0]; - try tokens.append(config.bos_token_id); + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); + try tokens.appendSlice(allocator, &.{ config.bos_token_id, start_header, user, end_header, newline }); - try tokens.append(start_header_id); - try tokens.appendSlice(try encoder.encode("user")); - try tokens.appendSlice(&.{ end_header_id, newline_id }); + try tokens.appendSlice(allocator, try encoder.encode(prompt)); + try tokens.appendSlice(allocator, &.{ eot, newline }); - try tokens.appendSlice(try encoder.encode(prompt)); - try tokens.appendSlice(&.{ eot_id, newline_id }); - try tokens.append(start_header_id); - try tokens.appendSlice(try encoder.encode("assistant")); - try tokens.appendSlice(&.{ end_header_id, newline_id }); + try tokens.appendSlice(allocator, &.{ start_header, assistant, end_header, newline }); - return tokens.toOwnedSlice(); + return tokens.toOwnedSlice(allocator); } pub fn generateText( @@ -64,7 +71,8 @@ pub fn generateText( seed: u128, prompt: []const u8, skip_llama3_encoding: bool, -) ![]const u8 { + writer: *std.Io.Writer, +) !void { const prompt_tok: []const u32 = try tokenizePrompt(allocator, tokenizer, config, prompt, skip_llama3_encoding); defer allocator.free(prompt_tok); @@ -72,7 +80,7 @@ pub fn generateText( defer tokenizer_decoder.deinit(); const platform = mod_generate.platform(); - const max_seq_len = llama_.model.shape().s; + const max_seq_len = llama_.model.max_seq_len; // init RNG and buffers var rng = try zml.Tensor.Rng.init(platform, seed); @@ -100,10 +108,6 @@ pub fn generateText( var current_token = try zml.Buffer.fromSlice(platform, .{1}, &generated_token_buffer); defer current_token.deinit(); - // Here we collect the generated text - var output = std.array_list.Managed(u8).init(allocator); - defer output.deinit(); - const output_tokens_len = max_seq_len - prompt_tok.len - 1; const start = std.time.microTimestamp(); @@ -114,9 +118,9 @@ pub fn generateText( // collect and print generated sequence num_tokens_generated += 1; const generated_token = generated_token_buffer[0]; - const chunk = try tokenizer_decoder.next(generated_token) orelse unreachable; - try output.appendSlice(chunk); - std.debug.print("{s}", .{chunk}); + if (try tokenizer_decoder.next(generated_token)) |chunk| { + try writer.writeAll(chunk); + } // check for eos if (i == output_tokens_len) break :generation; @@ -145,22 +149,6 @@ pub fn generateText( const speed = @as(f64, @floatFromInt(num_tokens_generated)) / duration; std.debug.print("\n", .{}); log.info("✅ Generated {d} tokens in {:.3}s: {d:.3}tok/s", .{ num_tokens_generated, duration, speed }); - return output.toOwnedSlice(); -} - -const params = clap.parseParamsComptime( - \\--help print this help - \\--prompt the prompt - \\--hf-model-path path to the directory containing model weights, config and tokenizer - \\--seed random seed (optional) - \\--seq-len sequence length - \\--create-options platform creation options JSON, defaults to {} - \\--no-llama3 skip prompt template - \\--sharding default: true: sharding on or off -); - -pub fn bool_parser(in: []const u8) error{}!bool { - return std.mem.indexOfScalar(u8, "tTyY1", in[0]) != null; } pub fn main() !void { @@ -174,31 +162,33 @@ pub fn asyncMain() !void { const parsers = comptime .{ .BOOL = bool_parser, - .UINT = clap.parsers.int(usize, 0), + .UINT = clap.parsers.int(u32, 0), .STRING = clap.parsers.string, .PATH = clap.parsers.string, }; var diag: clap.Diagnostic = .{}; var stderr_buffer: [1024]u8 = undefined; var stderr = std.fs.File.stderr().writer(&stderr_buffer); - var res = clap.parse(clap.Help, ¶ms, parsers, .{ + defer stderr.interface.flush() catch {}; + + var cli = clap.parse(clap.Help, ¶ms, parsers, .{ .diagnostic = &diag, .allocator = allocator, }) catch |err| { diag.report(&stderr.interface, err) catch {}; - stderr.interface.print("usage: ", .{}) catch {}; + stderr.interface.writeAll("usage: ") catch {}; clap.usage(&stderr.interface, clap.Help, ¶ms) catch {}; - stderr.interface.print("\n", .{}) catch {}; + stderr.interface.writeAll("\n") catch {}; return; }; - defer res.deinit(); + defer cli.deinit(); - if (res.args.help != 0) { + if (cli.args.help != 0) { clap.help(&stderr.interface, clap.Help, ¶ms, .{}) catch {}; return; } - const hf_model_path = res.args.@"hf-model-path" orelse { + const hf_model_path = cli.args.@"hf-model-path" orelse { log.err("Missing --hf-model-path", .{}); return; }; @@ -238,80 +228,87 @@ pub fn asyncMain() !void { const compilation_options = zml.CompilationOptions{ .xla_dump_to = "/tmp/zml/llama", - .sharding_enabled = res.args.sharding orelse true, + .sharding_enabled = cli.args.sharding orelse true, }; // initialize ZML platform with optional create options // eg: --create-options='{"cuda":{"allocator":{"bfc":{"memory_fraction": 0.99}}}}' - const create_opts_json = res.args.@"create-options" orelse "{}"; + const create_opts_json = cli.args.@"create-options" orelse "{}"; const create_opts = try std.json.parseFromSlice(zml.Platform.CreateOptions, allocator, create_opts_json, .{}); const platform = context.autoPlatform(create_opts.value).withCompilationOptions(compilation_options); create_opts.deinit(); context.printAvailablePlatforms(platform); - var ts = try zml.aio.detectFormatAndOpen(allocator, model_weights_path); - defer ts.deinit(); - - var model_arena = std.heap.ArenaAllocator.init(allocator); - var model_instance = try zml.aio.populateModel(llama.LlamaLM, model_arena.allocator(), ts); + var store = try zml.aio.detectFormatAndOpen(allocator, model_weights_path); + defer store.deinit(); + // Write metadata from the config file into the LlamaLm struct. + const seq_len: u32 = cli.args.@"seq-len" orelse 256; const llama_options: llama.LlamaLM.Options = .{ - .max_seq_len = @intCast(res.args.@"seq-len" orelse 256), + .max_seq_len = seq_len, .sampling_strategy = .{ .topk = 1, .temperature = 1.0, }, }; - model_instance.init(config, llama_options); - const dims = model_instance.model.shape(); - const dtype = model_instance.model.embed_tokens.weight.dtype(); + // Contains memory for llama_tensors and llama_buffers. + var compiler_arena = std.heap.ArenaAllocator.init(allocator); + defer compiler_arena.deinit(); - const tokens_shape_prefill = zml.Shape.init(.{ .s = llama_options.max_seq_len }, .u32); - const tokens_shape = zml.Shape.init(.{ .s = 1 }, .u32); + // Initialize the Llama struct and map the content of the .safetensors to the model tensors. + const llama_tensors: llama.LlamaLM = try .init(compiler_arena.allocator(), config, llama_options, store); + + // Specify shapes of input arguments + const prefill_tokens_shape = zml.Shape.init(.{ .s = llama_options.max_seq_len }, .u32); + const gen_tokens_shape = zml.Shape.init(.{ .s = 1 }, .u32); const token_idx_shape = zml.Shape.init(.{}, .u32); - const kv_shape = zml.Shape.init(.{ .layer = model_instance.model.layers.len, .k = dims.s, .h = dims.nkvh, .hd = dims.hd }, dtype).withSharding(.{.h}); - + const dtype = llama_tensors.model.embed_tokens.weight.dtype(); + const kv_shape = zml.Shape.init(.{ + .layer = llama_tensors.model.layers.len, + .k = seq_len, + .h = config.num_key_value_heads, + .hd = config.head_dim orelse @divExact(config.hidden_size, config.num_attention_heads), + }, dtype).withSharding(.{.h}); const kv_cache_shape: zml.ShapeOf(llama.KvCache) = llama.KvCache.initShape(kv_shape); const rng_shape = zml.Tensor.Rng.shape(); + // Compile the model twice, one for prefill, one for generation. var start = try std.time.Timer.start(); - var fut_mod_prefill = try asynk.asyncc(zml.compile, .{ - allocator, llama.LlamaLM.forward, .{ config, llama_options }, + var fut_mod_prefill = try asynk.asyncc(zml.compileModel, .{ + allocator, llama.LlamaLM.forward, llama_tensors, .{ - tokens_shape_prefill, + prefill_tokens_shape, token_idx_shape, kv_cache_shape, rng_shape, }, - ts, platform, }); - var fut_mod = try asynk.asyncc(zml.compile, .{ - allocator, llama.LlamaLM.forward, .{ config, llama_options }, + var fut_mod = try asynk.asyncc(zml.compileModel, .{ + allocator, llama.LlamaLM.forward, llama_tensors, .{ - tokens_shape, + gen_tokens_shape, token_idx_shape, kv_cache_shape, rng_shape, }, - ts, platform, }); + // While we are still compiling load the weights to the device. log.info("\tLoading Llama weights from {s}...", .{model_weights_path}); - var llama_weights = try zml.aio.loadBuffers(llama.LlamaLM, .{ config, llama_options }, ts, model_arena.allocator(), platform); - defer zml.aio.unloadBuffers(&llama_weights); + var llama_buffers = try store.loadModelById(llama.LlamaLM, compiler_arena.allocator(), llama_tensors, platform); + defer zml.aio.unloadBuffers(&llama_buffers); log.info("✅\tLoaded weights in {D}", .{start.read()}); - var llama_module_prefill = (try fut_mod_prefill.awaitt()).prepare(llama_weights); + var llama_module_prefill = (try fut_mod_prefill.awaitt()).prepare(llama_buffers); defer llama_module_prefill.deinit(); - var llama_module = (try fut_mod.awaitt()).prepare(llama_weights); + var llama_module = (try fut_mod.awaitt()).prepare(llama_buffers); defer llama_module.deinit(); log.info("✅\tCompiled model in {D}", .{start.read()}); - log.info("Creating KvCache", .{}); const kv_cache = try llama.KvCache.initBuffer(kv_shape, platform); @@ -320,16 +317,34 @@ pub fn asyncMain() !void { var timer = try stdx.time.Timer.start(); defer log.info("Loaded tokenizer from {s} [{D}]", .{ model_tokenizer_path, timer.read() }); - break :blk try zml.tokenizer.Tokenizer.fromFile(model_arena.allocator(), model_tokenizer_path); + break :blk try zml.tokenizer.Tokenizer.fromFile(allocator, model_tokenizer_path); }; errdefer tokenizer.deinit(); - const prompt = res.args.prompt orelse "What is the capital of France?"; + const prompt = cli.args.prompt orelse "What is the capital of France?"; log.info("✅\tPrompt: {s}", .{prompt}); - const seed = res.args.seed orelse @as(u128, @bitCast(std.time.nanoTimestamp())); - const skip_llama3_encoding = res.args.@"no-llama3" orelse false; - const generated_text = try generateText(config, model_instance, llama_module_prefill, llama_module, kv_cache, tokenizer, allocator, seed, prompt[0..], skip_llama3_encoding); - // generated text will be printed token by token. - defer allocator.free(generated_text); + // Unbuffered writing of the tokens to stdout. + var stdout = std.fs.File.stdout().writer(&.{}); + + const seed: u128 = cli.args.seed orelse @bitCast(std.time.nanoTimestamp()); + const skip_llama3_encoding = cli.args.@"no-llama3" orelse false; + + try generateText( + config, + llama_tensors, + llama_module_prefill, + llama_module, + kv_cache, + tokenizer, + allocator, + seed, + prompt[0..], + skip_llama3_encoding, + &stdout.interface, + ); +} + +fn bool_parser(in: []const u8) error{}!bool { + return std.mem.indexOfScalar(u8, "tTyY1", in[0]) != null; } diff --git a/examples/simple_layer/main.zig b/examples/simple_layer/main.zig index a84ab056..e0b89a94 100644 --- a/examples/simple_layer/main.zig +++ b/examples/simple_layer/main.zig @@ -45,19 +45,14 @@ pub fn asyncMain() !void { // We manually produce a BufferStore. You would not normally do that. // A BufferStore is usually created by loading model data from a file. - var buffers: zml.aio.BufferStore.Buffers = .{}; - try buffers.put(arena, "weight", zml.HostBuffer.fromArray(&weights)); - try buffers.put(arena, "bias", zml.HostBuffer.fromArray(&bias)); - - // the actual BufferStore - const buffer_store: zml.aio.BufferStore = .{ - .arena = arena_state, - .buffers = buffers, - }; + var store: zml.aio.BufferStore = .init(allocator); + defer store.deinit(); + try store.buffers.put(store.arena.allocator(), "weight", zml.HostBuffer.fromArray(&weights)); + try store.buffers.put(store.arena.allocator(), "bias", zml.HostBuffer.fromArray(&bias)); // A clone of our model, consisting of shapes. We only need shapes for compiling. // We use the BufferStore to infer the shapes. - var model_shapes = try zml.aio.populateModel(Layer, allocator, buffer_store); + var model_shapes = try zml.aio.populateModel(Layer, allocator, store); model_shapes.weight = model_shapes.weight.withSharding(.{-1}); model_shapes.bias = model_shapes.bias.?.withSharding(.{-1}); @@ -68,7 +63,7 @@ pub fn asyncMain() !void { // Produce a bufferized weights struct from the fake BufferStore. // This is like the inferred shapes, but with actual values. // We will need to send those to the computation device later. - var model_weights = try zml.aio.loadModelBuffers(Layer, model_shapes, buffer_store, arena, platform); + var model_weights = try zml.aio.loadModelBuffers(Layer, model_shapes, store, arena, platform); defer zml.aio.unloadBuffers(&model_weights); // for good practice // Wait for compilation to finish diff --git a/stdx/fmt.zig b/stdx/fmt.zig index 145db768..03bf7a35 100644 --- a/stdx/fmt.zig +++ b/stdx/fmt.zig @@ -1,5 +1,6 @@ const std = @import("std"); +/// Properly format a slice of numbers. pub fn slice(any_slice: anytype) FmtSlice(std.meta.Elem(@TypeOf(any_slice))) { return .{ .slice = any_slice }; } @@ -115,3 +116,57 @@ pub fn formatComplexSlice(values: anytype, spec: std.fmt.Number, writer: *std.Io pub fn formatBoolSlice(values: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { return try formatSliceCustom(formatBool, values, spec, writer); } + +/// Format a struct using `format` method of subfields when possible. +pub fn any(any_val: anytype) FmtAny(@TypeOf(any_val)) { + return .{ .data = any_val }; +} + +fn FmtAny(Data: type) type { + return struct { + data: Data, + + pub inline fn format(self: @This(), writer: *std.io.Writer) std.io.Writer.Error!void { + try printValue(writer, .{}, self.data, std.options.fmt_max_depth); + } + }; +} + +/// Fix up of std.io.Writer.printValue that uses `format` method of subfields when possible +fn printValue( + w: *std.io.Writer, + options: std.fmt.Options, + value: anytype, + max_depth: usize, +) std.io.Writer.Error!void { + const T = @TypeOf(value); + if (std.meta.hasMethod(T, "format")) { + return try value.format(w); + } + if (std.meta.hasMethod(T, "formatNumber")) { + return try value.formatNumber(w, options.toNumber(.decimal, .lower)); + } + + if (max_depth == 0) { + try w.writeAll(".{ ... }"); + return; + } + + switch (@typeInfo(T)) { + .@"struct" => |info| { + try w.writeAll(".{ "); + inline for (info.fields, 0..) |f, i| { + if (i > 0) try w.writeAll(", "); + + if (!info.is_tuple) { + try w.writeByte('.'); + try w.writeAll(f.name); + try w.writeAll(" = "); + } + try printValue(w, options, @field(value, f.name), max_depth - 1); + } + try w.writeAll(" }"); + }, + inline else => try w.printValue("any", options, value, max_depth - 1), + } +} diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 9bfa532a..46983493 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -1,3 +1,6 @@ +const std = @import("std"); +const builtin = @import("builtin"); + pub const BoundedArray = @import("bounded_array.zig").BoundedArray; pub const BoundedArrayAligned = @import("bounded_array.zig").BoundedArrayAligned; pub const debug = @import("debug.zig"); @@ -11,7 +14,6 @@ pub const queue = @import("queue.zig"); pub const time = @import("time.zig"); test { - const std = @import("std"); std.testing.refAllDecls(@This()); } @@ -20,3 +22,5 @@ pub inline fn stackSlice(comptime max_len: usize, T: type, len: usize) []T { var storage: [max_len]T = undefined; return storage[0..len]; } + +pub const noalloc: std.mem.Allocator = if (builtin.mode == .ReleaseFast) undefined else std.testing.failing_allocator; diff --git a/zml/aio.zig b/zml/aio.zig index 1b926a91..a778a627 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -43,8 +43,8 @@ pub fn detectFormatAndOpen(allocator: std.mem.Allocator, model_path: []const u8) /// whose shape is read from the "a.b" tensor. /// * If `Model` contains a list of layers, then the field: /// `Model.layers[2].a.b` will be populated from the "layers.2.a.b" tensor. -pub fn populateModel(comptime Model: type, allocator: std.mem.Allocator, buffer_store: BufferStore) !Model { - return populateModelWithPrefix(Model, allocator, buffer_store, ""); +pub fn populateModel(comptime Model: type, allocator: std.mem.Allocator, store: BufferStore) !Model { + return populateModelWithPrefix(Model, allocator, store, ""); } /// Creates a Model struct with tensor shapes read from the given TensorStore, @@ -62,8 +62,7 @@ pub fn populateModelWithPrefix(comptime Model: type, allocator: std.mem.Allocato try prefix_builder.push(allocator, prefix); defer prefix_builder.deinit(allocator); - const unique_id = zml.Tensor._reserveIdRange(@intCast(store.buffers.count())); - const ok = _populateStruct(allocator, &prefix_builder, unique_id, store, &model, true) catch |err| { + const ok = _populateStruct(allocator, &prefix_builder, store, &model, true) catch |err| { std.debug.panic("Can't populate model of type {s}: {s}", .{ @typeName(type), @errorName(err) }); }; if (!ok) return error.TensorNotFound; @@ -74,17 +73,28 @@ pub fn populateModelWithPrefix(comptime Model: type, allocator: std.mem.Allocato pub const BufferStore = struct { pub const Buffers = std.StringArrayHashMapUnmanaged(HostBuffer); pub const Metadatas = std.StringArrayHashMapUnmanaged(Metadata); + var _unique_store_id: std.atomic.Value(u64) = .init(0); + const _store_id_range: u64 = 1024 * 1024 * 1024; arena: std.heap.ArenaAllocator, files: []MemoryMappedFile = &.{}, buffers: Buffers = .{}, _metadata: Metadatas = .{}, + _unique_id: u64, - /// Create an empty BufferStore. Takes owneship of the given files. - pub fn init(allocator: std.mem.Allocator, files: []const MemoryMappedFile) error{OutOfMemory}!BufferStore { - var self: zml.aio.BufferStore = .{ + /// Create an empty BufferStore. + /// Takes owneship of the given files. + pub fn init(allocator: std.mem.Allocator) BufferStore { + return .{ .arena = std.heap.ArenaAllocator.init(allocator), + ._unique_id = _unique_store_id.fetchAdd(_store_id_range, .monotonic), }; + } + + /// Create an empty BufferStore. + /// Takes owneship of the given files. + pub fn initWithFiles(allocator: std.mem.Allocator, files: []const MemoryMappedFile) error{OutOfMemory}!BufferStore { + var self: BufferStore = .init(allocator); self.files = try self.arena.allocator().dupe(MemoryMappedFile, files); return self; } @@ -100,6 +110,61 @@ pub const BufferStore = struct { return self.buffers.get(key); } + pub fn loadBufferById(self: BufferStore, x: zml.Tensor, platform: zml.Platform) !zml.Buffer { + var host_buffer: zml.HostBuffer = switch (x._id) { + .buffer_id => |id| hb: { + if (id < self._unique_id or self._unique_id + _store_id_range <= id) { + @panic("`store.loadBufferById()` only works on Tensor created by `store.getTensor()`, using the same store object."); + } + break :hb self.buffers.values()[id - self._unique_id]; + }, + else => @panic("`store.loadBufferById()` only works on Tensor created by `store.getTensor()`"), + }; + + // Use the sharding information stored in the tensor. + host_buffer._shape = x.shape(); + return try host_buffer.toDevice(platform); + } + + /// Creates a bufferized version of a model from the given BufferStore. + /// + /// This will represent the weights of the model, loaded on a specific platform. + /// It can be used with a `module.Exe` (a compiled version of the same Model), to make a + /// `module.ExeWithWeights` ready to be called. + pub fn loadModelById(self: BufferStore, Model: type, allocator: std.mem.Allocator, model: Model, platform: zml.Platform) !zml.Bufferized(Model) { + const Ctx = struct { + platform: *const zml.Platform, + store: *const BufferStore, + + pub fn cb(ctx: @This(), x: zml.Tensor) zml.Buffer { + return ctx.store.loadBufferById(x, ctx.platform.*) catch @panic("Failed to load buffer to device"); + } + }; + + var res: zml.Bufferized(Model) = undefined; + try zml.meta.mapAlloc(Ctx.cb, allocator, .{ .platform = &platform, .store = &self }, model, &res); + return res; + } + + pub fn getTensor(self: BufferStore, key: []const u8) zml.Tensor { + return self.getTensorOrNull(key) orelse { + log.err("Tensor not found: {s}", .{key}); + self.findSimilarBufferKeys(std.heap.smp_allocator, key); + @panic("Tensor not found"); + }; + } + + pub fn getTensorOrNull(self: BufferStore, key: []const u8) ?zml.Tensor { + return if (self.buffers.getIndex(key)) |entry_idx| + .{ + ._shape = self.buffers.values()[entry_idx].shape(), + ._id = .{ .buffer_id = self._unique_id + entry_idx }, + ._donation = .input_buffer, + } + else + return null; + } + /// Count layers starting with the given prefix. pub fn countLayers(self: BufferStore, prefix: []const u8) usize { // Note: This is kinda inefficient @@ -140,6 +205,58 @@ pub const BufferStore = struct { return null; } + + /// Assists in debuggigng `BufferNotFound` error + /// This is useful when a buffer key is not found and you want to identify possible alternatives (or typos) + pub fn findSimilarBufferKeys(store: BufferStore, tmp_alloc: std.mem.Allocator, original_key: []const u8) void { + const suffixes = [_][]const u8{ "", ".weight", ".bias" }; + var shown_keys = std.StringHashMap(void).init(tmp_alloc); + defer shown_keys.deinit(); + + // remove suffix .weight and .bias + var base_key = original_key; + for (suffixes) |suffix| { + if (std.mem.endsWith(u8, original_key, suffix)) { + base_key = original_key[0 .. original_key.len - suffix.len]; + break; + } + } + + // first test: look for exact matches + var matches: usize = 0; + var it = store.buffers.iterator(); + while (it.next()) |entry| { + const key = entry.key_ptr.*; + if (std.mem.startsWith(u8, key, base_key)) { + if (matches == 0) log.warn("Similar buffers found:", .{}); + if (!shown_keys.contains(key)) { + log.warn(" - {s}: {f}", .{ key, entry.value_ptr.*.shape() }); + shown_keys.put(key, {}) catch continue; + matches += 1; + } + } + } + + // second test: progressive partial matches + if (matches == 0) { + var components = std.mem.splitScalar(u8, base_key, '.'); + while (components.next()) |component| { + matches = 0; + it = store.buffers.iterator(); + while (it.next()) |entry| { + const key = entry.key_ptr.*; + if (std.mem.indexOf(u8, key, component) != null and !shown_keys.contains(key)) { + if (matches == 0) log.warn("Partial matches for '{s}':", .{component}); + log.warn(" - {s}: {f}", .{ key, entry.value_ptr.*.shape() }); + shown_keys.put(key, {}) catch continue; + matches += 1; + if (matches >= 5) break; + } + } + if (matches > 0) break; + } + } + } }; pub const Metadata = union(enum) { @@ -260,17 +377,35 @@ pub const MemoryMappedFile = struct { /// Helper handling prefix building. /// /// This allows to easily push/pop prefixes and handles the generation of the string with the correct format. -const PrefixBuilder = struct { +pub const PrefixBuilder = struct { /// Stores the computed prefix. - data: std.ArrayListUnmanaged(u8) = .{}, + data: std.ArrayList(u8) = .{}, /// Stack storing the size of the intermediary prefix. - subprefixes: std.ArrayListUnmanaged(u32) = .{}, + subprefixes: std.ArrayList(u32) = .{}, + + pub fn initCapacity(allocator: std.mem.Allocator, capacity: usize) !PrefixBuilder { + return .{ + .data = try .initCapacity(allocator, capacity), + .subprefixes = try .initCapacity(allocator, @divFloor(capacity, 4)), + }; + } pub fn deinit(self: *PrefixBuilder, allocator: std.mem.Allocator) void { self.data.deinit(allocator); self.subprefixes.deinit(allocator); } + pub fn items(self: PrefixBuilder) []const u8 { + return self.data.items; + } + + pub fn concat(self: *PrefixBuilder, prefix: []const u8) []const u8 { + self.push(stdx.noalloc, prefix) catch unreachable; + const res = self.items(); + self.pop(); + return res; + } + pub fn push(self: *PrefixBuilder, allocator: std.mem.Allocator, prefix: []const u8) !void { const old_len: u32 = @intCast(self.data.items.len); try self.subprefixes.append(allocator, old_len); @@ -301,13 +436,20 @@ const PrefixBuilder = struct { const last_prefix_len = self.subprefixes.pop() orelse unreachable; self.data.shrinkRetainingCapacity(last_prefix_len); } + + pub fn checkpoint(self: PrefixBuilder) [2]usize { + return .{ self.data.items.len, self.subprefixes.items.len }; + } + + pub fn restore(self: *PrefixBuilder, ckpt: [2]usize) void { + self.data.items.len, self.subprefixes.items.len = ckpt; + } }; fn _populateStruct( allocator: std.mem.Allocator, prefix_builder: *PrefixBuilder, - unique_id: u64, - buffer_store: BufferStore, + store: BufferStore, obj: anytype, required: bool, ) !bool { @@ -322,17 +464,12 @@ fn _populateStruct( const prefix = prefix_builder.data.items; if (T == zml.Tensor) { - return if (buffer_store.buffers.getIndex(prefix)) |entry_idx| { - const buffer = buffer_store.get(prefix).?; - obj.* = zml.Tensor{ - ._shape = buffer.shape(), - ._id = .{ .buffer_id = unique_id + entry_idx }, - ._donation = .input_buffer, - }; + return if (store.getTensorOrNull(prefix)) |tensor| { + obj.* = tensor; return true; } else { if (required) { - log.err("Tensor not found: {s} ({d})", .{ prefix, buffer_store.buffers.count() }); + log.err("Tensor not found: {s} ({d})", .{ prefix, store.buffers.count() }); } return false; }; @@ -343,14 +480,14 @@ fn _populateStruct( if (ptr_info.size == .slice) { obj.* = &.{}; - const len = buffer_store.countLayers(prefix); + const len = store.countLayers(prefix); if (len > 0) { obj.* = try allocator.alloc(ptr_info.child, len); for (obj.*, 0..) |*value, i| { try prefix_builder.pushDigit(allocator, i); defer prefix_builder.pop(); - const found = try _populateStruct(allocator, prefix_builder, unique_id, buffer_store, value, required); + const found = try _populateStruct(allocator, prefix_builder, store, value, required); if (!found) { log.err("Not able to load {s} as {s}", .{ prefix_builder.data.items, @typeName(ptr_info.child) }); return false; @@ -369,7 +506,7 @@ fn _populateStruct( for (obj, 0..) |*value, i| { try prefix_builder.pushDigit(allocator, i); defer prefix_builder.pop(); - const found = try _populateStruct(allocator, prefix_builder, unique_id, buffer_store, value, required); + const found = try _populateStruct(allocator, prefix_builder, store, value, required); if (!found) { log.err("Not able to load {s} as {s}", .{ prefix_builder.data.items, @typeName(arr_info.child) }); return false; @@ -386,15 +523,20 @@ fn _populateStruct( var has_default = false; if (field.default_value_ptr) |_| has_default = true; - const field_found = try _populateStruct(allocator, prefix_builder, unique_id, buffer_store, &@field(obj, field.name), required and !has_default); - partial_struct = partial_struct or field_found; - if (!field_found) { - if (field.default_value_ptr) |v| { - @field(obj, field.name) = @as(*const field.type, @ptrCast(@alignCast(v))).*; - } else { - if (partial_struct) { - log.warn("Incomplete metadata '{0s}': {1s}. Missing field: '{2s}'. '{0s}' will be ignored.", .{ prefix, @typeName(T), field.name }); - obj.* = undefined; + + if (zml.meta.Contains(field.type, zml.Tensor)) { + const field_found = try _populateStruct(allocator, prefix_builder, store, &@field(obj, field.name), required and !has_default); + partial_struct = partial_struct or field_found; + if (!field_found) { + if (field.default_value_ptr) |v| { + @field(obj, field.name) = @as(*const field.type, @ptrCast(@alignCast(v))).*; + } else { + if (partial_struct) { + log.warn("Incomplete struct '{0s}': {1s}. Missing field: '{2s}'. '{0s}' will be ignored.", .{ prefix, @typeName(T), field.name }); + obj.* = undefined; + return false; + } + return false; } @@ -406,7 +548,7 @@ fn _populateStruct( }, .optional => |opt_info| { obj.* = @as(opt_info.child, undefined); - const found = try _populateStruct(allocator, prefix_builder, unique_id, buffer_store, &(obj.*.?), false); + const found = try _populateStruct(allocator, prefix_builder, store, &(obj.*.?), false); if (!found) obj.* = null; return true; }, @@ -449,7 +591,7 @@ test populateModel { var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_state.deinit(); - var store: BufferStore = .{ .arena = arena_state }; + var store: BufferStore = .init(arena_state.allocator()); try store.buffers.ensureUnusedCapacity(arena_state.allocator(), 16); store.buffers.putAssumeCapacity("a", Model._newHostBuffer(10)); store.buffers.putAssumeCapacity("b.a", Model._newHostBuffer(20)); @@ -494,11 +636,11 @@ test populateModel { pub fn loadBuffers( comptime Model: type, init_args: if (@hasDecl(Model, "init")) stdx.meta.Tail(stdx.meta.FnArgs(Model.init)) else void, - buffer_store: BufferStore, + store: BufferStore, allocator: std.mem.Allocator, platform: zml.Platform, ) !zml.Bufferized(Model) { - return loadBuffersWithPrefix(Model, init_args, buffer_store, allocator, platform, ""); + return loadBuffersWithPrefix(Model, init_args, store, allocator, platform, ""); } /// Creates a bufferized version of a Model from the given BufferStore with a specified prefix. @@ -512,7 +654,7 @@ pub fn loadBuffers( pub fn loadBuffersWithPrefix( comptime Model: type, init_args: if (@hasDecl(Model, "init")) stdx.meta.Tail(stdx.meta.FnArgs(Model.init)) else void, - buffer_store: BufferStore, + store: BufferStore, allocator: std.mem.Allocator, platform: zml.Platform, prefix: []const u8, @@ -522,14 +664,14 @@ pub fn loadBuffersWithPrefix( const arena = arena_state.allocator(); // Get model structure with tensor shapes from the buffer store with prefix - var model: Model = try zml.aio.populateModelWithPrefix(Model, arena, buffer_store, prefix); + var model: Model = try zml.aio.populateModelWithPrefix(Model, arena, store, prefix); // If the Model has a "init" function, call it with the given parameters. if (@hasDecl(Model, "init")) { @call(.auto, Model.init, .{&model} ++ init_args); } - return loadModelBuffersWithPrefix(Model, model, buffer_store, allocator, platform, prefix); + return loadModelBuffersWithPrefix(Model, model, store, allocator, platform, prefix); } /// Creates a bufferized version of a Model from the given BufferStore. For details about @@ -541,11 +683,11 @@ pub fn loadBuffersWithPrefix( pub fn loadModelBuffers( comptime Model: type, model: Model, - buffer_store: BufferStore, + store: BufferStore, allocator: std.mem.Allocator, platform: zml.Platform, ) !zml.Bufferized(Model) { - return try loadModelBuffersWithPrefix(Model, model, buffer_store, allocator, platform, ""); + return try loadModelBuffersWithPrefix(Model, model, store, allocator, platform, ""); } /// Creates a bufferized version of a Model from the given BufferStore and the given prefix. @@ -557,7 +699,7 @@ pub fn loadModelBuffers( pub fn loadModelBuffersWithPrefix( comptime Model: type, model: Model, - buffer_store: BufferStore, + store: BufferStore, allocator: std.mem.Allocator, platform: zml.Platform, prefix: []const u8, @@ -576,7 +718,7 @@ pub fn loadModelBuffersWithPrefix( try prefix_builder.push(allocator, prefix); defer prefix_builder.deinit(allocator); - try visitStructAndLoadBuffer(allocator, &prefix_builder, buffer_store, &res, platform); + try visitStructAndLoadBuffer(allocator, &prefix_builder, store, &res, platform); return res; } @@ -590,58 +732,6 @@ pub fn unloadBuffers(model: anytype) void { }).cb, {}, model); } -/// Assists in debuggigng `BufferNotFound` error -/// This is useful when a buffer key is not found and you want to identify possible alternatives (or typos) -fn findSimilarBufferKeys(original_key: []const u8, store: BufferStore, temp_allocator: std.mem.Allocator) void { - const suffixes = [_][]const u8{ "", ".weight", ".bias" }; - var shown_keys = std.StringHashMap(void).init(temp_allocator); - defer shown_keys.deinit(); - - // remove suffix .weight and .bias - var base_key = original_key; - for (suffixes) |suffix| { - if (std.mem.endsWith(u8, original_key, suffix)) { - base_key = original_key[0 .. original_key.len - suffix.len]; - break; - } - } - - // first test: look for exact matches - var matches: usize = 0; - var it = store.buffers.iterator(); - while (it.next()) |entry| { - const key = entry.key_ptr.*; - if (std.mem.startsWith(u8, key, base_key)) { - if (matches == 0) log.warn("Similar buffers found:", .{}); - if (!shown_keys.contains(key)) { - log.warn(" - {s}: {f}", .{ key, entry.value_ptr.*.shape() }); - shown_keys.put(key, {}) catch continue; - matches += 1; - } - } - } - - // second test: progressive partial matches - if (matches == 0) { - var components = std.mem.splitScalar(u8, base_key, '.'); - while (components.next()) |component| { - matches = 0; - it = store.buffers.iterator(); - while (it.next()) |entry| { - const key = entry.key_ptr.*; - if (std.mem.indexOf(u8, key, component) != null and !shown_keys.contains(key)) { - if (matches == 0) log.warn("Partial matches for '{s}':", .{component}); - log.warn(" - {s}: {f}", .{ key, entry.value_ptr.*.shape() }); - shown_keys.put(key, {}) catch continue; - matches += 1; - if (matches >= 5) break; - } - } - if (matches > 0) break; - } - } -} - /// deinit all buffers in the given struct pub fn awaitAll(buffers: anytype) !void { zml.meta.visit((struct { @@ -651,7 +741,7 @@ pub fn awaitAll(buffers: anytype) !void { }).cb, {}, buffers); } -fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *PrefixBuilder, buffer_store: BufferStore, obj: anytype, platform: zml.Platform) !void { +fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *PrefixBuilder, store: BufferStore, obj: anytype, platform: zml.Platform) !void { const err_msg = "visitStructAndLoadBuffer must be called with a pointer to type. Received "; const type_info, const T = switch (@typeInfo(@TypeOf(obj))) { .pointer => |ptr_info| switch (ptr_info.size) { @@ -663,7 +753,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi const prefix = prefix_builder.data.items; if (T == zml.Buffer) { - return if (buffer_store.get(prefix)) |host_buffer| { + return if (store.get(prefix)) |host_buffer| { // obj._shape has been set inside `loadModelBuffersWithPrefix`, before calling us. var buf_with_metadata = host_buffer; log.debug("Loading buffer {s} ({f})", .{ prefix, obj._shape }); @@ -673,7 +763,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi } else { log.err("Buffer not found: {s}", .{prefix}); - findSimilarBufferKeys(prefix, buffer_store, allocator); + store.findSimilarBufferKeys(allocator, prefix); return error.BufferNotFound; }; @@ -686,7 +776,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi try prefix_builder.pushDigit(allocator, i); defer prefix_builder.pop(); - try visitStructAndLoadBuffer(allocator, prefix_builder, buffer_store, value, platform); + try visitStructAndLoadBuffer(allocator, prefix_builder, store, value, platform); } } else stdx.debug.compileError("type not supported by visitStructAndLoadBuffer: {}", .{T}); }, @@ -694,7 +784,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi for (obj, 0..) |*value, i| { try prefix_builder.pushDigit(allocator, i); defer prefix_builder.pop(); - try visitStructAndLoadBuffer(allocator, prefix_builder, buffer_store, value, platform); + try visitStructAndLoadBuffer(allocator, prefix_builder, store, value, platform); } }, @@ -704,12 +794,12 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi try prefix_builder.push(allocator, field.name); defer prefix_builder.pop(); - try visitStructAndLoadBuffer(allocator, prefix_builder, buffer_store, &@field(obj, field.name), platform); + try visitStructAndLoadBuffer(allocator, prefix_builder, store, &@field(obj, field.name), platform); } }, .optional => { if (obj.*) |*obj_val| { - try visitStructAndLoadBuffer(allocator, prefix_builder, buffer_store, obj_val, platform); + try visitStructAndLoadBuffer(allocator, prefix_builder, store, obj_val, platform); } }, else => {}, diff --git a/zml/aio/safetensors.zig b/zml/aio/safetensors.zig index 08952b8f..49ba5fca 100644 --- a/zml/aio/safetensors.zig +++ b/zml/aio/safetensors.zig @@ -13,9 +13,7 @@ const StringBuilder = std.ArrayListUnmanaged(u8); const log = std.log.scoped(.@"zml/io"); pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore { - var res: zml.aio.BufferStore = .{ - .arena = std.heap.ArenaAllocator.init(allocator), - }; + var res: zml.aio.BufferStore = .init(allocator); errdefer res.arena.deinit(); const arena = res.arena.allocator(); diff --git a/zml/aio/tinyllama.zig b/zml/aio/tinyllama.zig index 8114c90f..c51319d6 100644 --- a/zml/aio/tinyllama.zig +++ b/zml/aio/tinyllama.zig @@ -25,9 +25,7 @@ const TinyLlamaConfig = extern struct { /// For convenience we use the same layer names /// than the one used by the Llama-3.1 models. pub fn open(allocator: std.mem.Allocator, model_path: []const u8) !zml.aio.BufferStore { - var res: zml.aio.BufferStore = .{ - .arena = std.heap.ArenaAllocator.init(allocator), - }; + var res: zml.aio.BufferStore = .init(allocator); errdefer res.arena.deinit(); const arena = res.arena.allocator(); diff --git a/zml/aio/torch.zig b/zml/aio/torch.zig index 79487f64..3f1511d7 100644 --- a/zml/aio/torch.zig +++ b/zml/aio/torch.zig @@ -36,7 +36,7 @@ pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore const py_values = try eval.evaluate(tmp_alloc, ops, true); // file ownership is transferred to the BufferStore - var res = try zml.aio.BufferStore.init(allocator, &.{torch_file.mmap_file}); + var res = try zml.aio.BufferStore.initWithFiles(allocator, &.{mmap_file}); try torch_file.parseModel(py_values, &res); return res; } diff --git a/zml/aio/torch/file.zig b/zml/aio/torch/file.zig index 0307d51c..fe8811fd 100644 --- a/zml/aio/torch/file.zig +++ b/zml/aio/torch/file.zig @@ -480,7 +480,7 @@ test "Read pickle (zipped)" { // torch.save({ "model": model, "tensor": tensor}, "simple.pt") const file = try asynk.File.open("zml/aio/torch/simple.pt", .{ .mode = .read_only }); const mmap_file = try zml.aio.MemoryMappedFile.init(file); - var store = try zml.aio.BufferStore.init(testing.allocator, &.{mmap_file}); + var store = try zml.aio.BufferStore.initWithFiles(testing.allocator, &.{mmap_file}); defer store.deinit(); { diff --git a/zml/exe.zig b/zml/exe.zig index d4c426b7..dffc16c2 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -77,7 +77,7 @@ pub fn compileModel( ) !FnExe(func) { const ModelT = ModuleSignature(func).ModelT; const name = @typeName(ModelT) ++ ".forward"; - log.info("Compiling {s} with {}", .{ name, args_shapes }); + log.info("Compiling {s} with {f}", .{ name, stdx.fmt.any(args_shapes) }); var context = try CompilationContext.init(allocator, name, platform); defer context.deinit(); @@ -209,9 +209,9 @@ pub const BaseExe = struct { var execute_context: ?*pjrt.ExecuteContext = null; if (platform.pjrt_api.ffi()) |ffi| { - log.info("Created context execution {*} for {*}", .{ execute_context, exe }); execute_context = try platform.pjrt_api.createExecuteContext(); try callback.bindInternalCallbacks(allocator, platform, ffi, execute_context.?); + // log.info("Created context execution {*} for {*}", .{ execute_context, exe }); } return .{ diff --git a/zml/meta.zig b/zml/meta.zig index ee2fbc68..855318a5 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -141,6 +141,7 @@ test MapType { /// Any `To` struct inside `from` will be copied over to the target. pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam(cb, 0), from: anytype, to: anytype) !void { // TODO: handle tuple to slice conversion + // TODO: handle error bubbling up const From = FnParam(cb, 1); const To = stdx.meta.FnResult(cb); const FromStruct = @TypeOf(from); diff --git a/zml/module.zig b/zml/module.zig index 6263d3fd..1b63ac7f 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -1129,7 +1129,7 @@ pub fn hashShape(hasher: *std.hash.Wyhash, shape: Shape) void { hash(hasher, shape._dtype, .Shallow); hash(hasher, shape._sharding_info, .Shallow); for (shape.tags()) |tag| { - hash(hasher, @intFromPtr(tag), .Shallow); + hashArray(hasher, std.mem.span(tag), .Shallow); } } diff --git a/zml/platform.zig b/zml/platform.zig index 0bc792e9..6e13d7f0 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -82,9 +82,7 @@ pub const Platform = struct { }; const _CreateOptions = struct { - // XLA CPU client doesn't read options - // https://github.com/openxla/xla/blob/42496a28c374bd35f493cc5dbde74805407245dc/xla/pjrt/c/pjrt_c_api_cpu_internal.cc#L33-L46 - cpu: struct {} = .{}, + cpu: Cpu = .{ .device_count = 4 }, // bump memory fraction from XLA defaults of 75% to 90%. // Even on a 8GB GPU it should leave enough space for the Cuda driver @@ -94,6 +92,14 @@ const _CreateOptions = struct { tpu: struct {} = .{}, neuron: struct {} = .{}, + pub const Cpu = struct { + device_count: u32, + + fn writeNamedValues(self: Cpu, values: *std.ArrayListUnmanaged(pjrt.NamedValue)) void { + values.appendAssumeCapacity(pjrt.NamedValue.from("cpu_device_count", @as(i64, self.device_count))); + } + }; + pub const Cuda = struct { allocator: Allocator = .{ .bfc = .{} }, // TODO support all of https://github.com/openxla/xla/blob/3d31c48c719d331d432132b3e0c2c5ce52650675/xla/pjrt/c/pjrt_c_api_gpu_internal.cc#L76-L86 @@ -118,7 +124,7 @@ const _CreateOptions = struct { }; }; - pub fn writeNamedValues(self: Cuda, values: *std.ArrayListUnmanaged(pjrt.NamedValue)) void { + fn writeNamedValues(self: Cuda, values: *std.ArrayListUnmanaged(pjrt.NamedValue)) void { switch (self.allocator) { .platform => { values.appendAssumeCapacity(pjrt.NamedValue.fromString("allocator", "platform")); @@ -142,6 +148,7 @@ const _CreateOptions = struct { var values = std.ArrayListUnmanaged(pjrt.NamedValue).fromOwnedSlice(out); values.shrinkRetainingCapacity(0); switch (target) { + .cpu => self.cpu.writeNamedValues(&values), .cuda => self.cuda.writeNamedValues(&values), inline else => |t| { stdx.debug.assertComptime(@hasField(_CreateOptions, @tagName(t)), "zml.platform.CreateOptions doesn't list target {s}", .{@tagName(t)}); diff --git a/zml/tensor.zig b/zml/tensor.zig index 63ab44d4..9d55ebf3 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -269,18 +269,6 @@ pub const Tensor = struct { return res; } - var _global_tensor_counter: u64 = 0; - - /// Internal use - pub fn _reserveIdRange(len: u32) u64 { - return @atomicRmw(u64, &_global_tensor_counter, .Add, len, .seq_cst); - } - - /// Internal use - pub fn setUniqueId(self: *Tensor) void { - self._id = .{ .buffer_id = _reserveIdRange(1) }; - } - /// Returns a Tensor containing the absolute value of each element of the input Tensor. pub fn abs(self: Tensor) Tensor { const loc = self.getContext().mlirCtx().location(@src()); diff --git a/zml/testing.zig b/zml/testing.zig index 5fec596d..59778eaf 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -298,7 +298,7 @@ test testLayer { }; // create a buffer store containing the activations: - var activations = try zml.aio.BufferStore.init(std.testing.allocator, &.{}); + var activations = zml.aio.BufferStore.init(std.testing.allocator); defer activations.deinit(); { const input = zml.HostBuffer.fromArray(&[2]f32{ 1, -1 }); From 03865b2f3db304d99740694f94f40852c081ad9b Mon Sep 17 00:00:00 2001 From: Emmanuel Ferdman Date: Tue, 16 Sep 2025 12:39:51 +0300 Subject: [PATCH 029/881] Update `bazel.sh` source (#323) ## PR Summary This small PR fixes the `bazel.sh` source. Signed-off-by: Emmanuel Ferdman --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bdfbf7f5..dbfc2ae8 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ We use `bazel` to build ZML and its dependencies. The only prerequisite is for `bazel`. **Please note: If you do not wish to install `bazel`** system-wide, we provide -[examples/bazel.sh](examples/bazel.sh) which downloads it to your home folder +[bazel.sh](bazel.sh) which downloads it to your home folder and runs it. **Install Bazel** (recommended): From df77b33d8ee6a0e3ce2800ab71a8f2d67ad9cb99 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 18 Sep 2025 10:05:54 +0200 Subject: [PATCH 030/881] async: remove asynk, asyncc, awaitt, await_ (#325) --- async/async.zig | 6 ++---- docs/howtos/howto_torch2zml.md | 4 ++-- docs/tutorials/write_first_model.md | 27 +++++++++++---------------- examples/benchmark/main.zig | 10 +++++----- examples/callback/main.zig | 6 +++--- examples/llama/main.zig | 18 +++++++++--------- examples/llama/test.zig | 6 +++--- examples/mnist/mnist.zig | 12 ++++++------ examples/modernbert/main.zig | 10 +++++----- examples/modernbert/modernbert.zig | 2 +- examples/modernbert/test.zig | 4 ++-- examples/simple_layer/main.zig | 8 ++++---- pjrt/pjrt.zig | 2 +- runtimes/cpu/cpu.zig | 4 ++-- runtimes/cuda/cuda.zig | 6 +++--- runtimes/neuron/neuron.zig | 8 ++++---- runtimes/rocm/rocm.zig | 4 ++-- runtimes/tpu/tpu.zig | 6 +++--- third_party/zls/zls.bzl | 8 ++++---- zml/aio.zig | 12 ++++++------ zml/aio/json.zig | 2 +- zml/aio/safetensors.zig | 6 +++--- zml/aio/tinyllama.zig | 4 ++-- zml/aio/torch.zig | 6 +++--- zml/aio/torch/file.zig | 4 ++-- zml/buffer.zig | 14 +++++++------- zml/callback.zig | 2 +- zml/exe.zig | 2 +- zml/module.zig | 4 ++-- zml/pjrtx.zig | 12 ++++++------ zml/test_runner.zig | 4 ++-- zml/tokenizer/main.zig | 4 ++-- zml/tokenizer/tokenizer.zig | 8 ++++---- 33 files changed, 114 insertions(+), 121 deletions(-) diff --git a/async/async.zig b/async/async.zig index e1052525..08e56516 100644 --- a/async/async.zig +++ b/async/async.zig @@ -52,9 +52,7 @@ fn FrameExx(comptime func: anytype, comptime argsT: type, comptime returnT: type inner: FrameT, - pub const wait = await_; - pub const await_ = awaitt; - pub fn awaitt(self: *Self) returnT { + pub fn await(self: *Self) returnT { defer { self.inner.deinit(); AsyncThread.current.stack_allocator.destroy(&self.inner._frame.stack); @@ -65,7 +63,7 @@ fn FrameExx(comptime func: anytype, comptime argsT: type, comptime returnT: type }; } -pub fn asyncc(comptime func: anytype, args: anytype) !FrameEx(func, @TypeOf(args)) { +pub fn async(comptime func: anytype, args: anytype) !FrameEx(func, @TypeOf(args)) { const Signature = stdx.meta.FnSignature(func, @TypeOf(args)); const new_stack = try AsyncThread.current.stack_allocator.create(); return .{ diff --git a/docs/howtos/howto_torch2zml.md b/docs/howtos/howto_torch2zml.md index 026e2013..5c751c53 100644 --- a/docs/howtos/howto_torch2zml.md +++ b/docs/howtos/howto_torch2zml.md @@ -118,13 +118,13 @@ model. Put the following in `my_project/torch2zml.zig`. const std = @import("std"); const log = std.log; -const asynk = @import("async"); +const async = @import("async"); const zml = @import("zml"); pub fn main() !void { var gpa = std.heap.GeneralPurposeAllocator(.{}){}; defer _ = gpa.deinit(); - try asynk.AsyncThread.main(gpa.allocator(), asyncMain, .{}); + try async.AsyncThread.main(gpa.allocator(), asyncMain, .{}); } pub fn asyncMain() !void { diff --git a/docs/tutorials/write_first_model.md b/docs/tutorials/write_first_model.md index 7cea735c..ebaf6c2f 100644 --- a/docs/tutorials/write_first_model.md +++ b/docs/tutorials/write_first_model.md @@ -99,10 +99,7 @@ Let's start by writing some Zig code, importing ZML and often-used modules: ```zig const std = @import("std"); const zml = @import("zml"); -const asynk = @import("async"); - -// shortcut to the asyncc function in the asynk module -const asyncc = asynk.asyncc; +const async = @import("async"); ``` You will use above lines probably in all ZML projects. Also, note that **ZML is @@ -154,7 +151,7 @@ like this: pub fn main() !void { var gpa = std.heap.GeneralPurposeAllocator(.{}){}; defer _ = gpa.deinit(); - try asynk.AsyncThread.main(gpa.allocator(), asyncMain); + try async.AsyncThread.main(gpa.allocator(), asyncMain); } @@ -251,7 +248,7 @@ const model_shapes = try zml.aio.populateModel(Layer, allocator, bs); // Start compiling. This uses the inferred shapes from the BufferStore. // The shape of the input tensor, we have to pass in manually. -var compilation = try asyncc( +var compilation = try async.async( zml.compileModel, .{ allocator, Layer.forward, model_shapes, .{input_shape}, platform }, ); @@ -263,11 +260,11 @@ var model_weights = try zml.aio.loadBuffers(Layer, .{}, bs, arena, platform); defer zml.aio.unloadBuffers(&model_weights); // for good practice // Wait for compilation to finish -const compiled = try compilation.awaitt(); +const compiled = try compilation.await(); ``` -Compiling is happening in the background via the `asyncc` function. We call -`asyncc` with the `zml.compileModel` function and its arguments +Compiling is happening in the background via the `async` function. We call +`async` with the `zml.compileModel` function and its arguments separately. The arguments themselves are basically the shapes of the weights in the BufferStore, the `.forward` function name in order to compile `Layer.forward`, the shape of the input tensor(s), and the platform for which to @@ -371,7 +368,7 @@ top of the Zig file: ```zig const zml = @import("zml"); -const asynk = @import("async"); +const async = @import("async"); ``` @@ -418,9 +415,7 @@ You can access the complete source code of this walkthrough here: ```zig const std = @import("std"); const zml = @import("zml"); -const asynk = @import("async"); - -const asyncc = asynk.asyncc; +const async = @import("async"); /// Model definition const Layer = struct { @@ -439,7 +434,7 @@ const Layer = struct { pub fn main() !void { var gpa = std.heap.GeneralPurposeAllocator(.{}){}; defer _ = gpa.deinit(); - try asynk.AsyncThread.main(gpa.allocator(), asyncMain); + try async.AsyncThread.main(gpa.allocator(), asyncMain); } pub fn asyncMain() !void { @@ -482,7 +477,7 @@ pub fn asyncMain() !void { // Start compiling. This uses the inferred shapes from the BufferStore. // The shape of the input tensor, we have to pass in manually. - var compilation = try asyncc(zml.compileModel, .{ allocator, Layer.forward, model_shapes, .{input_shape}, platform }); + var compilation = try async.async(zml.compileModel, .{ allocator, Layer.forward, model_shapes, .{input_shape}, platform }); // Produce a bufferized weights struct from the fake BufferStore. // This is like the inferred shapes, but with actual values. @@ -491,7 +486,7 @@ pub fn asyncMain() !void { defer zml.aio.unloadBuffers(&model_weights); // for good practice // Wait for compilation to finish - const compiled = try compilation.awaitt(); + const compiled = try compilation.await(); // pass the model weights to the compiled module to create an executable // module diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index 6428eda9..fb6bccb8 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -1,13 +1,13 @@ const std = @import("std"); const zml = @import("zml"); const stdx = @import("stdx"); -const asynk = @import("async"); +const async = @import("async"); const flags = stdx.flags; // set log level to debug to print the generated IR pub const std_options: std.Options = .{ .log_level = .warn, - .logFn = asynk.logFn(std.log.defaultLog), + .logFn = async.logFn(std.log.defaultLog), }; pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { @@ -15,7 +15,7 @@ pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { } pub fn main() !void { - try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); + try async.AsyncThread.main(std.heap.c_allocator, asyncMain); } pub fn asyncMain() !void { @@ -53,10 +53,10 @@ pub fn asyncMain() !void { // Start compiling. // The shape of the input tensor, we have to pass in manually. timer.reset(); - var compilation = try asynk.asyncc(zml.compileFn, .{ allocator, benchmark, .{ a_shape, b_shape }, platform }); + var compilation = try async.async(zml.compileFn, .{ allocator, benchmark, .{ a_shape, b_shape }, platform }); // Wait for compilation to finish - const executable = try compilation.awaitt(); + const executable = try compilation.await(); defer executable.deinit(); const compilation_elapsed = timer.lap() / std.time.ns_per_ms; std.debug.print("-" ** 160 ++ "\n\n", .{}); diff --git a/examples/callback/main.zig b/examples/callback/main.zig index be3b3095..107c5f99 100644 --- a/examples/callback/main.zig +++ b/examples/callback/main.zig @@ -1,13 +1,13 @@ const std = @import("std"); -const asynk = @import("async"); +const async = @import("async"); const runtimes = @import("runtimes"); const zml = @import("zml"); const cu = zml.platform_specific; pub const std_options: std.Options = .{ .log_level = .info, - .logFn = asynk.logFn(std.log.defaultLog), + .logFn = async.logFn(std.log.defaultLog), }; const log = std.log.scoped(.@"examples/custom_call"); @@ -126,7 +126,7 @@ pub fn grayscale(rgb: zml.Tensor) zml.Tensor { } pub fn main() !void { - try asynk.AsyncThread.main(std.heap.smp_allocator, asyncMain); + try async.AsyncThread.main(std.heap.smp_allocator, asyncMain); } pub fn asyncMain() !void { diff --git a/examples/llama/main.zig b/examples/llama/main.zig index c1c8d805..dc31af5a 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -1,6 +1,6 @@ const std = @import("std"); -const asynk = @import("async"); +const async = @import("async"); const clap = @import("clap"); const stdx = @import("stdx"); const zml = @import("zml"); @@ -19,7 +19,7 @@ const log = std.log.scoped(.llama); pub const std_options: std.Options = .{ .log_level = .info, - .logFn = asynk.logFn(std.log.defaultLog), + .logFn = async.logFn(std.log.defaultLog), }; const params = clap.parseParamsComptime( @@ -152,7 +152,7 @@ pub fn generateText( } pub fn main() !void { - try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); + try async.AsyncThread.main(std.heap.c_allocator, asyncMain); } pub fn asyncMain() !void { @@ -198,7 +198,7 @@ pub fn asyncMain() !void { const model_weights_path = b: { const simple_path = try std.fs.path.join(allocator, &.{ hf_model_path, "model.safetensors" }); - if (asynk.File.access(simple_path, .{})) { + if (async.File.access(simple_path, .{})) { break :b simple_path; } else |_| { allocator.free(simple_path); @@ -213,7 +213,7 @@ pub fn asyncMain() !void { defer allocator.free(model_tokenizer_path); const config = blk: { - var config_json_file = try asynk.File.open(model_config_path, .{ .mode = .read_only }); + var config_json_file = try async.File.open(model_config_path, .{ .mode = .read_only }); defer config_json_file.close() catch unreachable; var config_json_buffer: [256]u8 = undefined; var config_reader = config_json_file.reader(&config_json_buffer); @@ -276,7 +276,7 @@ pub fn asyncMain() !void { // Compile the model twice, one for prefill, one for generation. var start = try std.time.Timer.start(); - var fut_mod_prefill = try asynk.asyncc(zml.compileModel, .{ + var fut_mod_prefill = try async.async(zml.compileModel, .{ allocator, llama.LlamaLM.forward, llama_tensors, .{ prefill_tokens_shape, @@ -287,7 +287,7 @@ pub fn asyncMain() !void { platform, }); - var fut_mod = try asynk.asyncc(zml.compileModel, .{ + var fut_mod = try async.async(zml.compileModel, .{ allocator, llama.LlamaLM.forward, llama_tensors, .{ gen_tokens_shape, @@ -304,9 +304,9 @@ pub fn asyncMain() !void { defer zml.aio.unloadBuffers(&llama_buffers); log.info("✅\tLoaded weights in {D}", .{start.read()}); - var llama_module_prefill = (try fut_mod_prefill.awaitt()).prepare(llama_buffers); + var llama_module_prefill = (try fut_mod_prefill.await()).prepare(llama_buffers); defer llama_module_prefill.deinit(); - var llama_module = (try fut_mod.awaitt()).prepare(llama_buffers); + var llama_module = (try fut_mod.await()).prepare(llama_buffers); defer llama_module.deinit(); log.info("✅\tCompiled model in {D}", .{start.read()}); log.info("Creating KvCache", .{}); diff --git a/examples/llama/test.zig b/examples/llama/test.zig index 42c0b493..0e1edba0 100644 --- a/examples/llama/test.zig +++ b/examples/llama/test.zig @@ -1,4 +1,4 @@ -const asynk = @import("async"); +const async = @import("async"); const std = @import("std"); const stdx = @import("stdx"); const zml = @import("zml"); @@ -10,7 +10,7 @@ const LlamaLM = llama_mod.LlamaLM; const Tensor = zml.Tensor; pub fn main() !void { - try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); + try async.AsyncThread.main(std.heap.c_allocator, asyncMain); } pub fn asyncMain() !void { @@ -58,7 +58,7 @@ pub fn asyncMain() !void { const num_kv_heads: i64 = cli_args.num_kv_heads orelse buffer_store.metadata("num_kv_heads", .int) orelse num_heads; const config = blk: { - var config_json_file = try asynk.File.open(cli_args.config, .{ .mode = .read_only }); + var config_json_file = try async.File.open(cli_args.config, .{ .mode = .read_only }); defer config_json_file.close() catch unreachable; var reader = std.json.reader(allocator, config_json_file.reader()); defer reader.deinit(); diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 5c85f5b7..632cafda 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -1,13 +1,13 @@ const std = @import("std"); -const asynk = @import("async"); +const async = @import("async"); const zml = @import("zml"); const log = std.log.scoped(.mnist); pub const std_options: std.Options = .{ .log_level = .info, - .logFn = asynk.logFn(std.log.defaultLog), + .logFn = async.logFn(std.log.defaultLog), }; /// Model definition @@ -37,7 +37,7 @@ const Mnist = struct { }; pub fn main() !void { - try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); + try async.AsyncThread.main(std.heap.c_allocator, asyncMain); } pub fn asyncMain() !void { @@ -75,14 +75,14 @@ pub fn asyncMain() !void { // Start compiling log.info("Compiling model to MLIR....", .{}); var start_time = try std.time.Timer.start(); - var compilation = try asynk.asyncc(zml.compile, .{ allocator, Mnist.forward, .{}, .{zml.Shape.init(.{ 28, 28 }, .u8)}, buffer_store, platform }); + var compilation = try async.async(zml.compile, .{ allocator, Mnist.forward, .{}, .{zml.Shape.init(.{ 28, 28 }, .u8)}, buffer_store, platform }); // While compiling, start loading weights on the platform var model_weights = try zml.aio.loadModelBuffers(Mnist, mnist_model, buffer_store, arena, platform); defer zml.aio.unloadBuffers(&model_weights); // Wait for end of compilation and end of weights loading. - const compiled_mnist = try compilation.awaitt(); + const compiled_mnist = try compilation.await(); log.info("✅ Compiled model in {d}ms", .{start_time.read() / std.time.ns_per_ms}); const mnist = compiled_mnist.prepare(model_weights); @@ -92,7 +92,7 @@ pub fn asyncMain() !void { log.info("Starting inference...", .{}); // Load a random digit image from the dataset. - const dataset = try asynk.File.open(t10kfilename, .{ .mode = .read_only }); + const dataset = try async.File.open(t10kfilename, .{ .mode = .read_only }); defer dataset.close() catch unreachable; var rng = std.Random.Xoshiro256.init(@intCast(std.time.timestamp())); diff --git a/examples/modernbert/main.zig b/examples/modernbert/main.zig index 95d5aff1..1206eb94 100644 --- a/examples/modernbert/main.zig +++ b/examples/modernbert/main.zig @@ -1,6 +1,6 @@ const std = @import("std"); -const asynk = @import("async"); +const async = @import("async"); const clap = @import("clap"); const stdx = @import("stdx"); const zml = @import("zml"); @@ -15,7 +15,7 @@ pub const std_options: std.Options = .{ .log_scope_levels = &[_]std.log.ScopeLevel{ .{ .scope = .modernbert, .level = .info }, }, - .logFn = asynk.logFn(std.log.defaultLog), + .logFn = async.logFn(std.log.defaultLog), }; const params = clap.parseParamsComptime( @@ -38,7 +38,7 @@ const clap_parsers = .{ }; pub fn main() !void { - try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); + try async.AsyncThread.main(std.heap.c_allocator, asyncMain); } pub fn asyncMain() !void { @@ -136,7 +136,7 @@ pub fn asyncMain() !void { // Compile the model log.info("\tCompiling ModernBERT model...", .{}); - var fut_mod = try asynk.asyncc(zml.compile, .{ + var fut_mod = try async.async(zml.compile, .{ allocator, modernbert.ModernBertForMaskedLM.forward, .{modernbert_options}, @@ -144,7 +144,7 @@ pub fn asyncMain() !void { tensor_store, platform, }); - var bert_module = (try fut_mod.awaitt()).prepare(bert_weights); + var bert_module = (try fut_mod.await()).prepare(bert_weights); defer bert_module.deinit(); log.info("✅\tLoaded weights and compiled model in {D}", .{start.read()}); diff --git a/examples/modernbert/modernbert.zig b/examples/modernbert/modernbert.zig index 79585f14..2e6eb4ab 100644 --- a/examples/modernbert/modernbert.zig +++ b/examples/modernbert/modernbert.zig @@ -1,6 +1,6 @@ const std = @import("std"); -const asynk = @import("async"); +const async = @import("async"); const stdx = @import("stdx"); const zml = @import("zml"); const Tensor = zml.Tensor; diff --git a/examples/modernbert/test.zig b/examples/modernbert/test.zig index 8dd8bbba..5a6d4496 100644 --- a/examples/modernbert/test.zig +++ b/examples/modernbert/test.zig @@ -1,7 +1,7 @@ const clap = @import("clap"); const std = @import("std"); const zml = @import("zml"); -const asynk = @import("async"); +const async = @import("async"); const log = std.log; const Tensor = zml.Tensor; const modernbert_module = @import("modernbert.zig"); @@ -20,7 +20,7 @@ fn printUsageAndExit(stderr: anytype) noreturn { std.process.exit(0); } pub fn main() !void { - try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); + try async.AsyncThread.main(std.heap.c_allocator, asyncMain); } pub fn asyncMain() !void { diff --git a/examples/simple_layer/main.zig b/examples/simple_layer/main.zig index e0b89a94..57d5d16b 100644 --- a/examples/simple_layer/main.zig +++ b/examples/simple_layer/main.zig @@ -1,6 +1,6 @@ const std = @import("std"); -const asynk = @import("async"); +const async = @import("async"); const zml = @import("zml"); /// Model definition @@ -18,7 +18,7 @@ const Layer = struct { }; pub fn main() !void { - try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); + try async.AsyncThread.main(std.heap.c_allocator, asyncMain); } pub fn asyncMain() !void { @@ -58,7 +58,7 @@ pub fn asyncMain() !void { // Start compiling. This uses the inferred shapes from the BufferStore. // The shape of the input tensor, we have to pass in manually. - var compilation = try asynk.asyncc(zml.compileModel, .{ allocator, Layer.forward, model_shapes, .{input_shape}, platform }); + var compilation = try async.async(zml.compileModel, .{ allocator, Layer.forward, model_shapes, .{input_shape}, platform }); // Produce a bufferized weights struct from the fake BufferStore. // This is like the inferred shapes, but with actual values. @@ -67,7 +67,7 @@ pub fn asyncMain() !void { defer zml.aio.unloadBuffers(&model_weights); // for good practice // Wait for compilation to finish - const compiled = try compilation.awaitt(); + const compiled = try compilation.await(); // pass the model weights to the compiled module to create an executable module var executable = compiled.prepare(model_weights); diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 08ccf217..d4d49dcd 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -1030,7 +1030,7 @@ pub const Event = opaque { return @ptrCast(result); } - pub fn await_(self: *const Event, api: *const Api) ApiError!void { + pub fn await(self: *const Event, api: *const Api) ApiError!void { _ = try api.call(.PJRT_Event_Await, .{ .event = self.inner(), }); diff --git a/runtimes/cpu/cpu.zig b/runtimes/cpu/cpu.zig index ff22cf56..a3b8cde0 100644 --- a/runtimes/cpu/cpu.zig +++ b/runtimes/cpu/cpu.zig @@ -1,6 +1,6 @@ const builtin = @import("builtin"); -const asynk = @import("async"); +const async = @import("async"); const c = @import("c"); const pjrt = @import("pjrt"); const bazel_builtin = @import("bazel_builtin"); @@ -44,6 +44,6 @@ pub fn load() !*const pjrt.Api { var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; const path = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "libpjrt_cpu" ++ ext }); - break :blk asynk.callBlocking(pjrt.Api.loadFrom, .{path}); + break :blk async.callBlocking(pjrt.Api.loadFrom, .{path}); }; } diff --git a/runtimes/cuda/cuda.zig b/runtimes/cuda/cuda.zig index 42b3b5f4..bf52dc07 100644 --- a/runtimes/cuda/cuda.zig +++ b/runtimes/cuda/cuda.zig @@ -1,7 +1,7 @@ const std = @import("std"); const builtin = @import("builtin"); -const asynk = @import("async"); +const async = @import("async"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); @@ -17,7 +17,7 @@ pub fn isEnabled() bool { } fn hasNvidiaDevice() bool { - asynk.File.access("/dev/nvidiactl", .{ .mode = .read_only }) catch return false; + async.File.access("/dev/nvidiactl", .{ .mode = .read_only }) catch return false; return true; } @@ -79,6 +79,6 @@ pub fn load() !*const pjrt.Api { return blk: { var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; const path = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_cuda.so" }); - break :blk asynk.callBlocking(pjrt.Api.loadFrom, .{path}); + break :blk async.callBlocking(pjrt.Api.loadFrom, .{path}); }; } diff --git a/runtimes/neuron/neuron.zig b/runtimes/neuron/neuron.zig index 4d2efa1e..d40106f0 100644 --- a/runtimes/neuron/neuron.zig +++ b/runtimes/neuron/neuron.zig @@ -1,7 +1,7 @@ const std = @import("std"); const builtin = @import("builtin"); -const asynk = @import("async"); +const async = @import("async"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); @@ -15,14 +15,14 @@ pub fn isEnabled() bool { } fn hasNeuronDevice() bool { - asynk.File.access("/dev/neuron0", .{ .mode = .read_only }) catch return false; + async.File.access("/dev/neuron0", .{ .mode = .read_only }) catch return false; return true; } fn isRunningOnEC2() !bool { const AmazonEC2 = "Amazon EC2"; - var f = try asynk.File.open("/sys/devices/virtual/dmi/id/sys_vendor", .{ .mode = .read_only }); + var f = try async.File.open("/sys/devices/virtual/dmi/id/sys_vendor", .{ .mode = .read_only }); defer f.close() catch {}; var content: [AmazonEC2.len]u8 = undefined; @@ -64,6 +64,6 @@ pub fn load() !*const pjrt.Api { return blk: { var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; const path = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_neuron.so" }); - break :blk asynk.callBlocking(pjrt.Api.loadFrom, .{path}); + break :blk async.callBlocking(pjrt.Api.loadFrom, .{path}); }; } diff --git a/runtimes/rocm/rocm.zig b/runtimes/rocm/rocm.zig index 9be8b07a..c5c12285 100644 --- a/runtimes/rocm/rocm.zig +++ b/runtimes/rocm/rocm.zig @@ -1,7 +1,7 @@ const std = @import("std"); const builtin = @import("builtin"); -const asynk = @import("async"); +const async = @import("async"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); @@ -16,7 +16,7 @@ pub fn isEnabled() bool { fn hasRocmDevices() bool { inline for (&.{ "/dev/kfd", "/dev/dri" }) |path| { - asynk.File.access(path, .{ .mode = .read_only }) catch return false; + async.File.access(path, .{ .mode = .read_only }) catch return false; } return true; } diff --git a/runtimes/tpu/tpu.zig b/runtimes/tpu/tpu.zig index 96af0be3..fd972c8a 100644 --- a/runtimes/tpu/tpu.zig +++ b/runtimes/tpu/tpu.zig @@ -1,7 +1,7 @@ const std = @import("std"); const builtin = @import("builtin"); -const asynk = @import("async"); +const async = @import("async"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); @@ -22,7 +22,7 @@ fn isOnGCP() !bool { // TODO: abstract that in the client and fail init const GoogleComputeEngine = "Google Compute Engine"; - var f = try asynk.File.open("/sys/devices/virtual/dmi/id/product_name", .{ .mode = .read_only }); + var f = try async.File.open("/sys/devices/virtual/dmi/id/product_name", .{ .mode = .read_only }); defer f.close() catch {}; var content: [GoogleComputeEngine.len]u8 = undefined; @@ -61,6 +61,6 @@ pub fn load() !*const pjrt.Api { return blk: { var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; const path = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_tpu.so" }); - break :blk asynk.callBlocking(pjrt.Api.loadFrom, .{path}); + break :blk async.callBlocking(pjrt.Api.loadFrom, .{path}); }; } diff --git a/third_party/zls/zls.bzl b/third_party/zls/zls.bzl index 4cc5caa1..9e28da65 100644 --- a/third_party/zls/zls.bzl +++ b/third_party/zls/zls.bzl @@ -1,24 +1,24 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -_VERSION = "0.14.0" +_VERSION = "0.15.0" _ARCH = { "x86_64-linux": struct( - sha256 = "661f8d402ba3dc9b04b6e9bc3026495be7b838d2f18d148db2bd98bd699c1360", + sha256 = "508bfe3fd637d2a02f07f3fc7da8900351f407116b03685c5dae26b4f01a30de", exec_compatible_with = [ "@platforms//os:linux", "@platforms//cpu:x86_64", ], ), "x86_64-macos": struct( - sha256 = "baee69e4645deeccb42970b4a01f573592209dc1cf72e32893c59ca06af511dc", + sha256 = "46c31838bfef5adcc7fee82428c3ec2b9abbfae38242639afea5f242ee133d93", exec_compatible_with = [ "@platforms//os:macos", "@platforms//cpu:x86_64", ], ), "aarch64-macos": struct( - sha256 = "dfb627e1f9603583678f552d8035a12dce878215c0a507b32d6f1b9d074d6c4d", + sha256 = "76c7a23190f67e67970024065f689c2c49b3c7b0fc16876fb24ef199fb05fc2a", exec_compatible_with = [ "@platforms//os:macos", "@platforms//cpu:aarch64", diff --git a/zml/aio.zig b/zml/aio.zig index a778a627..38a9914f 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -1,6 +1,6 @@ const std = @import("std"); -const asynk = @import("async"); +const async = @import("async"); const c = @import("c"); const stdx = @import("stdx"); @@ -336,13 +336,13 @@ pub const Metadata = union(enum) { /// This struct is meant to be wrapped into a format specific struct, like io.gguf.File. pub const MemoryMappedFile = struct { /// underlying file handle - file: asynk.File, + file: async.File, data: []align(std.heap.page_size_min) const u8, data_offset: u64 = 0, - pub fn init(file: asynk.File) !MemoryMappedFile { + pub fn init(file: async.File) !MemoryMappedFile { const data_len: usize = (try file.stat()).size; - const data_ = try asynk.callBlocking(std.posix.mmap, .{ + const data_ = try async.callBlocking(std.posix.mmap, .{ null, data_len, std.posix.PROT.READ, @@ -351,7 +351,7 @@ pub const MemoryMappedFile = struct { 0, }); - try asynk.callBlocking(posix.madvise, .{ + try async.callBlocking(posix.madvise, .{ data_.ptr, @as(usize, @intCast(data_.len)), @as(u32, @intCast(c.MADV_SEQUENTIAL)), @@ -736,7 +736,7 @@ pub fn unloadBuffers(model: anytype) void { pub fn awaitAll(buffers: anytype) !void { zml.meta.visit((struct { fn cb(_: void, buffer: *zml.Buffer) void { - buffer.* = buffer.awaitt() catch unreachable; + buffer.* = buffer.await() catch unreachable; } }).cb, {}, buffers); } diff --git a/zml/aio/json.zig b/zml/aio/json.zig index a42e5619..2c504bb3 100644 --- a/zml/aio/json.zig +++ b/zml/aio/json.zig @@ -1,4 +1,4 @@ -const asynk = @import("async"); +const async = @import("async"); const std = @import("std"); const zml = @import("../zml.zig"); diff --git a/zml/aio/safetensors.zig b/zml/aio/safetensors.zig index 49ba5fca..f1cdf293 100644 --- a/zml/aio/safetensors.zig +++ b/zml/aio/safetensors.zig @@ -1,7 +1,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; -const asynk = @import("async"); +const async = @import("async"); const stdx = @import("stdx"); const MemoryMappedFile = @import("../aio.zig").MemoryMappedFile; @@ -30,7 +30,7 @@ pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore } fn loadFromIndex(allocator: Allocator, store: *zml.aio.BufferStore, files: *std.array_list.Managed(MemoryMappedFile), path: []const u8) !void { - const file = asynk.File.open(path, .{}) catch |err| { + const file = async.File.open(path, .{}) catch |err| { log.err("Failed to open {s}: {}", .{ path, err }); return err; }; @@ -66,7 +66,7 @@ fn loadFromIndex(allocator: Allocator, store: *zml.aio.BufferStore, files: *std. } fn loadFile(allocator: Allocator, store: *zml.aio.BufferStore, files: *std.array_list.Managed(MemoryMappedFile), path: []const u8) !void { - const file = asynk.File.open(path, .{}) catch |err| { + const file = async.File.open(path, .{}) catch |err| { log.err("Failed to open {s}: {}", .{ path, err }); return err; }; diff --git a/zml/aio/tinyllama.zig b/zml/aio/tinyllama.zig index c51319d6..5adca927 100644 --- a/zml/aio/tinyllama.zig +++ b/zml/aio/tinyllama.zig @@ -1,6 +1,6 @@ /// Tools to load models from https://huggingface.co/karpathy/tinyllamas/ /// Originally made to be run with https://github.com/karpathy/llama2.c -const asynk = @import("async"); +const async = @import("async"); const std = @import("std"); const stdx = @import("stdx"); @@ -29,7 +29,7 @@ pub fn open(allocator: std.mem.Allocator, model_path: []const u8) !zml.aio.Buffe errdefer res.arena.deinit(); const arena = res.arena.allocator(); - const file = try asynk.File.open(model_path, .{}); + const file = try async.File.open(model_path, .{}); res.files = try arena.alloc(zml.aio.MemoryMappedFile, 1); res.files[0] = try zml.aio.MemoryMappedFile.init(file); diff --git a/zml/aio/torch.zig b/zml/aio/torch.zig index 3f1511d7..fe487218 100644 --- a/zml/aio/torch.zig +++ b/zml/aio/torch.zig @@ -1,6 +1,6 @@ const std = @import("std"); -const asynk = @import("async"); +const async = @import("async"); const zml = @import("../zml.zig"); const eval = @import("torch/eval.zig"); @@ -18,7 +18,7 @@ test { /// Opens and loads a BufferStore from the torch file at the given path. pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore { - const file = asynk.File.open(path, .{}) catch |err| { + const file = async.File.open(path, .{}) catch |err| { log.err("Failed to open {s}: {}", .{ path, err }); return err; }; @@ -30,7 +30,7 @@ pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore const tmp_alloc = arena.allocator(); const mmap_file = try zml.aio.MemoryMappedFile.init(file); - var torch_file = try asynk.callBlocking(File.init, .{ tmp_alloc, mmap_file }); + var torch_file = try async.callBlocking(File.init, .{ tmp_alloc, mmap_file }); const ops = try torch_file.parsePickle(tmp_alloc); const py_values = try eval.evaluate(tmp_alloc, ops, true); diff --git a/zml/aio/torch/file.zig b/zml/aio/torch/file.zig index fe8811fd..1afafdd5 100644 --- a/zml/aio/torch/file.zig +++ b/zml/aio/torch/file.zig @@ -1,7 +1,7 @@ const std = @import("std"); const testing = std.testing; -const asynk = @import("async"); +const async = @import("async"); const stdx = @import("stdx"); const zml = @import("../../zml.zig"); @@ -478,7 +478,7 @@ test "Read pickle (zipped)" { // model = torch.nn.Conv2d(2, 2, 3, stride=2, padding=[2, 4], dtype=torch.float16) // tensor = torch.tensor([[2, 4, 3, 2]], dtype=torch.uint8) // torch.save({ "model": model, "tensor": tensor}, "simple.pt") - const file = try asynk.File.open("zml/aio/torch/simple.pt", .{ .mode = .read_only }); + const file = try async.File.open("zml/aio/torch/simple.pt", .{ .mode = .read_only }); const mmap_file = try zml.aio.MemoryMappedFile.init(file); var store = try zml.aio.BufferStore.initWithFiles(testing.allocator, &.{mmap_file}); defer store.deinit(); diff --git a/zml/buffer.zig b/zml/buffer.zig index f502d703..6dcc42f2 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -1,6 +1,6 @@ const std = @import("std"); -const asynk = @import("async"); +const async = @import("async"); const stdx = @import("stdx"); const DataType = @import("dtype.zig").DataType; @@ -115,16 +115,16 @@ pub const Buffer = struct { } if (opts.wait) { - res = try res.awaitt(); + res = try res.await(); } return res; } - pub fn awaitt(self: Buffer) !Buffer { + pub fn await(self: Buffer) !Buffer { for (self._shards.constSlice()) |buffer| { if (buffer.getReadyEvent(self._api)) |ev| { - try ev.await_(self._api); + try ev.await(self._api); } } @@ -317,7 +317,7 @@ pub const Buffer = struct { stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); const maybe_event = try self._shards.get(0).toHostBuffer(self._api, std.mem.asBytes(&res)); if (maybe_event) |event| { - try event.await_(self._api); + try event.await(self._api); } return res; } @@ -329,7 +329,7 @@ pub const Buffer = struct { stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); const maybe_event = try self._shards.get(0).toHostBuffer(self._api, output); if (maybe_event) |event| { - try event.await_(self._api); + try event.await(self._api); } return HostBuffer.fromBytes(self.shape(), output); } @@ -341,7 +341,7 @@ pub const Buffer = struct { stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); const maybe_event = try self._shards.get(0).toHostBuffer(self._api, @constCast(output.bytes())); if (maybe_event) |event| { - try event.await_(self._api); + try event.await(self._api); } return output; } diff --git a/zml/callback.zig b/zml/callback.zig index 4d6ab90e..4d063533 100644 --- a/zml/callback.zig +++ b/zml/callback.zig @@ -1,6 +1,6 @@ const std = @import("std"); -const asynk = @import("async"); +const async = @import("async"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); const stablehlo = @import("mlir/dialects").stablehlo; diff --git a/zml/exe.zig b/zml/exe.zig index dffc16c2..9e3d35e4 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -261,7 +261,7 @@ pub const BaseExe = struct { // for (events[0..sharding.num_partitions]) |e| { // if (e) |ev| { - // ev.await_(self.platform.pjrt_api) catch unreachable; + // ev.await(self.platform.pjrt_api) catch unreachable; // } // } } diff --git a/zml/module.zig b/zml/module.zig index 1b63ac7f..294d76b4 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -1,6 +1,6 @@ const std = @import("std"); -const asynk = @import("async"); +const async = @import("async"); const c = @import("c"); const dialect = @import("mlir/dialects"); const mlir = @import("mlir"); @@ -162,7 +162,7 @@ pub const CompilationContext = struct { var timer = std.time.Timer.start() catch null; const tensor_args = try self.tensorFromShapes(stdx.meta.FnArgs(func), arena, args); // Run in a dedicated thread because compilation relies on `threadlocal`. - const f = try asynk.callBlocking(CompilationContext.emitMlir, .{ self, func, &tensor_args, CompilationContext.EmitMlirOpts{ .name = "main", .kind = .main } }); + const f = try async.callBlocking(CompilationContext.emitMlir, .{ self, func, &tensor_args, CompilationContext.EmitMlirOpts{ .name = "main", .kind = .main } }); const module = self._module; module.getBody().appendOperation(f.mlir_fn); diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 8658aebe..f3c8c349 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -1,6 +1,6 @@ const std = @import("std"); -const asynk = @import("async"); +const async = @import("async"); const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); @@ -66,7 +66,7 @@ pub const Client = opaque { } pub fn deserializeAndLoad(self: *const Client, api: *const Api, bytes: []const u8) ApiError!*LoadedExecutable { - return @ptrCast(try asynk.callBlocking(pjrt.Client.deserializeAndLoad, .{ self.inner(), api, bytes })); + return @ptrCast(try async.callBlocking(pjrt.Client.deserializeAndLoad, .{ self.inner(), api, bytes })); } pub const CreateViewOfDeviceBufferArgs = pjrt.Client.CreateViewOfDeviceBufferArgs; @@ -105,7 +105,7 @@ pub const Client = opaque { } pub fn compile(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { - return try asynk.callBlocking(compileSync, .{ self, api, allocator, module, compile_options_pb }); + return try async.callBlocking(compileSync, .{ self, api, allocator, module, compile_options_pb }); } pub fn addressableMemories(self: *const Client, api: *const Api) []*const Memory { @@ -211,10 +211,10 @@ pub const Event = opaque { if (self.isReady(api)) { return; } - try self.inner().await_(api); + try self.inner().await(api); } - pub fn await_(self: *Event, api: *const Api) ApiError!void { + pub fn await(self: *Event, api: *const Api) ApiError!void { defer self.deinit(api); if (self.isReady(api)) { @@ -223,7 +223,7 @@ pub const Event = opaque { var ctx = struct { err: ?*pjrt.Error = null, - event: asynk.threading.ResetEventSingle = .{}, + event: async.threading.ResetEventSingle = .{}, }{}; try self.inner().onReady(api, &(struct { diff --git a/zml/test_runner.zig b/zml/test_runner.zig index 8b8d6555..85acadef 100644 --- a/zml/test_runner.zig +++ b/zml/test_runner.zig @@ -3,7 +3,7 @@ const std = @import("std"); const testing = std.testing; const builtin = @import("builtin"); -const asynk = @import("async"); +const async = @import("async"); // note: std_options.log_level does not respect testing.log_level // ref: https://github.com/ziglang/zig/issues/5738 @@ -19,7 +19,7 @@ var fba = std.heap.FixedBufferAllocator.init(&fba_buffer); pub fn main() anyerror!void { testing.log_level = log_level; - try asynk.AsyncThread.main(testing.allocator, asyncMain); + try async.AsyncThread.main(testing.allocator, asyncMain); } pub fn asyncMain() !void { diff --git a/zml/tokenizer/main.zig b/zml/tokenizer/main.zig index b6f91dc2..fc568388 100644 --- a/zml/tokenizer/main.zig +++ b/zml/tokenizer/main.zig @@ -1,6 +1,6 @@ const std = @import("std"); -const asynk = @import("async"); +const async = @import("async"); const stdx = @import("stdx"); const zml_tokenizer = @import("zml/tokenizer"); @@ -14,7 +14,7 @@ const Flags = struct { }; pub fn main() !void { - try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); + try async.AsyncThread.main(std.heap.c_allocator, asyncMain); } pub fn asyncMain() !void { diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig index 443e2fb8..c8dfb120 100644 --- a/zml/tokenizer/tokenizer.zig +++ b/zml/tokenizer/tokenizer.zig @@ -1,6 +1,6 @@ const std = @import("std"); -const asynk = @import("async"); +const async = @import("async"); const hftokenizers = @import("hftokenizers"); const sentencepiece = @import("sentencepiece"); @@ -98,15 +98,15 @@ pub const Tokenizer = union(Tokenizers) { pub fn fromFile(allocator: std.mem.Allocator, model: []const u8) !Tokenizer { if (std.mem.endsWith(u8, model, ".pb")) { - return .{ .sentencepiece = try asynk.callBlocking(sentencepiece.SentencePieceProcessor.fromFile, .{model}) }; + return .{ .sentencepiece = try async.callBlocking(sentencepiece.SentencePieceProcessor.fromFile, .{model}) }; } if (std.mem.endsWith(u8, model, ".json")) { - return .{ .hftokenizers = try asynk.callBlocking(hftokenizers.HFTokenizer.fromFile, .{model}) }; + return .{ .hftokenizers = try async.callBlocking(hftokenizers.HFTokenizer.fromFile, .{model}) }; } if (std.mem.endsWith(u8, model, ".tinyllama")) { const tokenizer = try allocator.create(homemade.Tokenizer); - tokenizer.* = try asynk.callBlocking(homemade.fromTinyLlamaFile, .{ allocator, model, 32000 }); + tokenizer.* = try async.callBlocking(homemade.fromTinyLlamaFile, .{ allocator, model, 32000 }); return .{ .homemade = tokenizer }; } From dda7f6c3f0afb6de039b6a409b014856d8a894a7 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 18 Sep 2025 11:36:23 +0200 Subject: [PATCH 031/881] zml: remove old writer interface APIs (#324) We still had a few old reader/writer left over. I think we are good now --- async/coro.zig | 26 +- examples/modernbert/main.zig | 17 +- mlir/dialects/stablehlo/stablehlo.zig | 86 +++--- mlir/mlir.zig | 320 +++++++++----------- pjrt/ffi.zig | 12 +- pjrt/pjrt.zig | 9 +- runtimes/neuron/libneuronxla.zig | 4 +- stdx/fmt.zig | 4 + zml/aio.zig | 9 +- zml/aio/json.zig | 10 +- zml/aio/safetensors.zig | 16 +- zml/aio/torch.zig | 1 - zml/aio/torch/eval.zig | 2 +- zml/aio/torch/file.zig | 4 +- zml/aio/torch/pickle.zig | 2 +- zml/aio/torch/py.zig | 15 +- zml/buffer.zig | 5 +- zml/exe.zig | 8 +- zml/hostbuffer.zig | 5 +- zml/meta.zig | 30 +- zml/mlirx.zig | 2 +- zml/module.zig | 54 ++-- zml/ops.zig | 46 ++- zml/pjrtx.zig | 20 +- zml/platform.zig | 6 +- zml/shape.zig | 12 +- zml/tensor.zig | 24 +- zml/testing.zig | 2 +- zml/tokenizer/BUILD.bazel | 13 +- zml/tokenizer/hftokenizers/hftokenizers.zig | 3 +- zml/tokenizer/homemade.zig | 139 +++++---- zml/tokenizer/main.zig | 4 +- 32 files changed, 433 insertions(+), 477 deletions(-) diff --git a/async/coro.zig b/async/coro.zig index a4d0345e..67042967 100644 --- a/async/coro.zig +++ b/async/coro.zig @@ -159,26 +159,21 @@ const Coro = struct { fn runcoro(from: *base.Coro, this: *base.Coro) callconv(.c) noreturn { const from_coro: *Coro = @fieldParentPtr("impl", from); const this_coro: *Coro = @fieldParentPtr("impl", this); - log(.debug, "coro start {any}", .{this_coro.id}); + log(.debug, "coro start {f}", .{this_coro.*}); @call(.auto, this_coro.func, .{}); this_coro.status = .Done; thread_state.switchOut(from_coro); // Never returns - stdx.debug.panic("Cannot resume an already completed coroutine {any}", .{this_coro.id}); + stdx.debug.panic("Cannot resume an already completed coroutine {f}", .{this_coro.*}); } pub fn getStorage(self: Coro, comptime T: type) *T { return @ptrCast(@alignCast(self.storage)); } - pub fn format(self: Coro, comptime fmt: []const u8, options: std.fmt.FormatOptions, writer: anytype) !void { - _ = fmt; - _ = options; - try writer.print("Coro{{.id = {any}, .status = {s}}}", .{ - self.id, - @tagName(self.status), - }); + pub fn format(self: Coro, writer: *std.Io.Writer) !void { + try writer.print("Coro{{.id = {any}, .status = {t}}}", .{ self.id, self.status }); } }; @@ -292,7 +287,7 @@ const ThreadState = struct { /// Called from resume fn switchIn(self: *ThreadState, target: Frame) void { - log(.debug, "coro resume {any} from {any}", .{ target.id, self.current().id }); + log(.debug, "coro resume {f} from {f}", .{ target.id, self.current().id }); // Switch to target, setting this coro as the resumer. self.switchTo(target, true); @@ -307,7 +302,7 @@ const ThreadState = struct { /// Called from suspend fn switchOut(self: *ThreadState, target: Frame) void { - log(.debug, "coro suspend {any} to {any}", .{ self.current().id, target.id }); + log(.debug, "coro suspend {f} to {f}", .{ self.current().id, target.id }); self.switchTo(target, false); } @@ -384,13 +379,8 @@ const CoroId = struct { self.invocation += 1; } - pub fn format(self: @This(), comptime fmt: []const u8, options: std.fmt.FormatOptions, writer: anytype) !void { - _ = fmt; - _ = options; - try writer.print("CoroId{{.cid={d}, .i={d}}}", .{ - self.id.coro, - self.invocation, - }); + pub fn format(self: @This(), writer: *std.Io.Writer) !void { + try writer.print("CoroId{{.cid={d}, .i={d}}}", .{ self.id.coro, self.invocation }); } }; }; diff --git a/examples/modernbert/main.zig b/examples/modernbert/main.zig index 1206eb94..0df16b39 100644 --- a/examples/modernbert/main.zig +++ b/examples/modernbert/main.zig @@ -216,27 +216,26 @@ pub fn unmask( } pub fn tokenize(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]const u32 { - var tokens = std.array_list.Managed(u32).init(allocator); var encoder = try tokenizer.encoder(); defer encoder.deinit(); const bos = tokenizer.tokenToId("[CLS]") orelse return error.NoSuchToken; const eos = tokenizer.tokenToId("[SEP]") orelse return error.NoSuchToken; - try tokens.append(bos); - try tokens.appendSlice(try encoder.encode(prompt)); - try tokens.append(eos); + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); + try tokens.append(allocator, bos); + try tokens.appendSlice(allocator, try encoder.encode(prompt)); + try tokens.append(allocator, eos); - return tokens.toOwnedSlice(); + return tokens.toOwnedSlice(allocator); } fn findMaskPositions(allocator: std.mem.Allocator, tokens: []const u32, mask_token: u32) ![]usize { - var mask_positions = std.array_list.Managed(usize).init(allocator); - defer mask_positions.deinit(); + var mask_positions: std.ArrayList(usize) = .empty; for (tokens, 0..) |token, i| { if (token == mask_token) { - try mask_positions.append(i); + try mask_positions.append(allocator, i); } } @@ -247,7 +246,7 @@ fn findMaskPositions(allocator: std.mem.Allocator, tokens: []const u32, mask_tok if (mask_positions.items.len > 1) log.warn("Currently only supporting one [MASK] per input", .{}); - return mask_positions.toOwnedSlice(); + return mask_positions.toOwnedSlice(allocator); } fn prepareTensorInputs( diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 0c60fd01..7c7c1394 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -750,13 +750,13 @@ pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCa if (@intFromEnum(opts.api_version) < @intFromEnum(CustomCallOpts.ApiVersion.typed_ffi)) { stdx.debug.assert( backend_config.isA(mlir.StringAttribute), - "API version < 4 requires a string as backend_config, got {}", + "API version < 4 requires a string as backend_config, got {f}", .{backend_config}, ); } else { stdx.debug.assert( backend_config.isA(mlir.DictionaryAttribute), - "API version >= 4 requires a dictionary as backend_config, got {}", + "API version >= 4 requires a dictionary as backend_config, got {f}", .{backend_config}, ); } @@ -1285,20 +1285,21 @@ pub fn stablehloVersionFromCompatibilityRequirement(requirement: c.MlirStablehlo } pub fn stablehloGetSmallerVersion(version1: []const u8, version2: []const u8) []const u8 { - var buf: [32]u8 = undefined; + const Cmp = struct { + v1: []const u8, + v1_is_smaller: bool = undefined, - var stream = std.io.fixedBufferStream(&buf); - var context = .{ .writer = stream.writer() }; - const WriterContext = @TypeOf(context); - - _ = c.stablehloGetSmallerVersion(mlir.stringRef(version1), mlir.stringRef(version2), (struct { - pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.c) void { - const inner_ctx: *WriterContext = @ptrCast(@alignCast(userdata)); - _ = inner_ctx.writer.write(mlir.fromStringRef(mlir_str)) catch unreachable; + pub fn smallerCb(smaller_version: c.MlirStringRef, opaque_cmp: ?*anyopaque) callconv(.c) void { + var cmp: *@This() = @ptrCast(@alignCast(opaque_cmp)); + cmp.v1_is_smaller = std.mem.eql(u8, cmp.v1, smaller_version.data[0..smaller_version.length]); } - }).callback, &context); + }; - return if (std.mem.eql(u8, buf[0..stream.pos], version1)) version1 else version2; + var cmp_ctx: Cmp = .{ .v1 = version1 }; + const cmp_res = c.stablehloGetSmallerVersion(mlir.stringRef(version1), mlir.stringRef(version2), Cmp.smallerCb, &cmp_ctx); + + std.debug.assert(cmp_res.value != 0); + return if (cmp_ctx.v1_is_smaller) version1 else version2; } pub fn getCurrentVersion() []const u8 { @@ -1308,18 +1309,9 @@ pub fn getCurrentVersion() []const u8 { var once = std.once(call); fn call() void { - var stream = std.io.fixedBufferStream(&buf); - var writer_ = stream.writer(); - const ContextWriter = @TypeOf(writer_); - - c.stablehloGetCurrentVersion((struct { - pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.c) void { - const writer: *ContextWriter = @ptrCast(@alignCast(userdata)); - _ = writer.write(mlir.fromStringRef(mlir_str)) catch unreachable; - } - }).callback, &writer_); - - str = buf[0..stream.pos]; + var writer: std.Io.Writer = .fixed(&buf); + c.stablehloGetCurrentVersion(printCallbackNoFail, &writer); + str = writer.buffered(); } }; @@ -1334,18 +1326,9 @@ pub fn getMinimumVersion() []const u8 { var once = std.once(call); fn call() void { - var stream = std.io.fixedBufferStream(&buf); - var context = .{ .writer = stream.writer() }; - const WriterContext = @TypeOf(context); - - c.stablehloGetMinimumVersion((struct { - pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.c) void { - const inner_ctx: *WriterContext = @ptrCast(@alignCast(userdata)); - _ = inner_ctx.writer.write(mlir.fromStringRef(mlir_str)) catch unreachable; - } - }).callback, &context); - - str = buf[0..stream.pos]; + var writer: std.Io.Writer = .fixed(&buf); + c.stablehloGetMinimumVersion(printCallbackNoFail, &writer); + str = writer.buffered(); } }; @@ -1353,14 +1336,25 @@ pub fn getMinimumVersion() []const u8 { return state.str; } -pub fn serializePortableArtifact(bytecode: []const u8, target_version: []const u8, writer: anytype) !void { - var context = .{ .writer = writer }; - const WriterContext = @TypeOf(context); +pub fn serializePortableArtifact( + bytecode: []const u8, + target_version: []const u8, + writer: *std.Io.Writer, +) error{ InvalidMlirBytecodeVersion, WriteFailed }!void { + var writer_err: mlir.WriterWithErr = .{ .writer = writer }; + try mlir.successOr( + c.stablehloSerializePortableArtifactFromStringRef( + mlir.stringRef(bytecode), + mlir.stringRef(target_version), + mlir.WriterWithErr.printCallback, + &writer_err, + ), + error.InvalidMlirBytecodeVersion, + ); + return try writer_err.check(); +} - try mlir.successOr(c.stablehloSerializePortableArtifactFromStringRef(mlir.stringRef(bytecode), mlir.stringRef(target_version), (struct { - pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.c) void { - const inner_ctx: *WriterContext = @ptrCast(@alignCast(userdata)); - _ = inner_ctx.writer.write(mlir.fromStringRef(mlir_str)) catch unreachable; - } - }).callback, &context), error.InvalidMlirBytecodeVersion); +fn printCallbackNoFail(mlir_str: c.MlirStringRef, opaque_writer: ?*anyopaque) callconv(.c) void { + const writer: *std.Io.Writer = @ptrCast(@alignCast(opaque_writer)); + writer.writeAll(mlir.fromStringRef(mlir_str)) catch @panic("Failed to write MLIR"); } diff --git a/mlir/mlir.zig b/mlir/mlir.zig index a6721914..15dbdab8 100755 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -7,18 +7,20 @@ const stdx = @import("stdx"); const log = std.log.scoped(.mlir); test { - std.testing.refAllDecls(@This()); + std.testing.refAllDeclsRecursive(@This()); _ = try Context.init(); } -const Error = error{ +pub const Error = error{ /// Invalid Mlir was created. InvalidMlir, /// Another Mlir error. Check the log for more context. MlirUnexpected, /// A resource/executor was not found. NotFound, + /// Bytecode version incompatibility. + InvalidMlirBytecodeVersion, OutOfMemory, }; @@ -35,64 +37,61 @@ pub fn registerPasses(comptime passes: []const u8) void { @field(c, "mlirRegister" ++ passes ++ "Passes")(); } -pub fn successOr(res: c.MlirLogicalResult, err: anytype) !void { - return if (res.value == 0) err; +pub fn successOr(res: c.MlirLogicalResult, err: anytype) @TypeOf(err)!void { + return if (res.value == 0) err else {}; } -/// Alternative to MlirWrapperType -pub const MlirStrCallback = fn (c.MlirStringRef, ?*anyopaque) callconv(.c) void; - pub const Registry = struct { _inner: c.MlirDialectRegistry, pub const deinit = helpers.deinit(Registry, c.mlirDialectRegistryDestroy); pub fn init() !Registry { - return helpers.init(Registry, c.mlirDialectRegistryCreate(), c.mlirDialectRegistryIsNull) orelse Error.MlirUnexpected; + const registry = c.mlirDialectRegistryCreate(); + return .{ ._inner = .{ .ptr = registry.ptr orelse return Error.MlirUnexpected } }; } }; pub const Context = struct { _inner: c.MlirContext, - const Self = Context; pub const deinit = helpers.deinit(Context, c.mlirContextDestroy); pub const wrapOr = helpers.wrapOr(Context, c.mlirContextIsNull); - pub fn init() !Self { - return Self.wrapOr(c.mlirContextCreate()) orelse Error.MlirUnexpected; + pub fn init() !Context { + return Context.wrapOr(c.mlirContextCreate()) orelse Error.MlirUnexpected; } - pub fn initWithRegistry(registry: Registry, threadingEnabled: bool) !Self { - return Self.wrapOr( + pub fn initWithRegistry(registry: Registry, threadingEnabled: bool) !Context { + return Context.wrapOr( c.mlirContextCreateWithRegistry(registry._inner, threadingEnabled), ) orelse Error.InvalidMlir; } - pub fn setMultiThreading(self: *Self, enabled: bool) void { + pub fn setMultiThreading(self: *Context, enabled: bool) void { c.mlirContextEnableMultithreading(self._inner, enabled); } - pub fn appendDialectRegistry(self: *Self, registry: Registry) void { + pub fn appendDialectRegistry(self: *Context, registry: Registry) void { c.mlirContextAppendDialectRegistry(self._inner, registry._inner); } - pub fn loadAllAvailableDialects(self: *Self) void { + pub fn loadAllAvailableDialects(self: *Context) void { c.mlirContextLoadAllAvailableDialects(self._inner); } - pub fn numRegisteredDialects(self: Self) usize { + pub fn numRegisteredDialects(self: Context) usize { return @intCast(c.mlirContextGetNumRegisteredDialects(self._inner)); } - pub fn numLoadedDialects(self: Self) usize { + pub fn numLoadedDialects(self: Context) usize { return @intCast(c.mlirContextGetNumLoadedDialects(self._inner)); } - pub fn isRegisteredOperation(self: Self, op: [:0]const u8) bool { + pub fn isRegisteredOperation(self: Context, op: [:0]const u8) bool { return c.mlirContextIsRegisteredOperation(self._inner, stringRef(op)); } - pub fn location(self: Self, src: std.builtin.SourceLocation) Location { + pub fn location(self: Context, src: std.builtin.SourceLocation) Location { return Location.fromSrc(self, src); } }; @@ -103,9 +102,7 @@ pub const Module = struct { pub const deinit = helpers.deinit(Module, c.mlirModuleDestroy); pub const wrapOr = helpers.wrapOr(Module, c.mlirModuleIsNull); - const Self = Module; - - pub fn init(loc: Location) Self { + pub fn init(loc: Location) Module { return .{ ._inner = c.mlirModuleCreateEmpty(loc._inner) }; } @@ -142,29 +139,26 @@ pub const PassManager = struct { pub const deinit = helpers.deinit(PassManager, c.mlirPassManagerDestroy); pub const wrapOr = helpers.wrapOr(PassManager, c.mlirPassManagerIsNull); - const Self = PassManager; - - pub fn init(ctx: Context) !Self { - return Self.wrapOr( + pub fn init(ctx: Context) !PassManager { + return PassManager.wrapOr( c.mlirPassManagerCreate(ctx._inner), ) orelse Error.MlirUnexpected; } - pub fn initOnOperation(ctx: Context, op: [:0]const u8) !Self { - return Self.wrapOr( + pub fn initOnOperation(ctx: Context, op: [:0]const u8) !PassManager { + return PassManager.wrapOr( c.mlirPassManagerCreateOnOperation(ctx._inner, stringRef(op)), ) orelse Error.MlirUnexpected; } - pub fn asOpPassManager(self: Self) OpPassManager { + pub fn asOpPassManager(self: PassManager) OpPassManager { return .{ ._inner = c.mlirPassManagerGetAsOpPassManager(self._inner) }; } - pub fn enableIRPrinting(self: *Self) void { - c.mlirPassManagerEnableIRPrinting(self._inner); - } + // TODO mlirPassManagerEnableIRPrinting + // pub fn enableIRPrinting(self: *PassManager) void {} - pub fn runOnOp(self: *Self, op: Operation) error{InvalidMlir}!void { + pub fn runOnOp(self: *PassManager, op: Operation) error{InvalidMlir}!void { if (c.mlirPassManagerRunOnOp(self._inner, op._inner).value == 0) { return Error.InvalidMlir; } @@ -193,21 +187,20 @@ pub const OpPassManager = struct { pub const Identifier = struct { _inner: c.MlirIdentifier, - const Self = Identifier; - pub fn get(ctx: Context, str_: [:0]const u8) Self { + pub fn get(ctx: Context, str_: [:0]const u8) Identifier { return .{ ._inner = c.mlirIdentifierGet(ctx._inner, stringRef(str_)) }; } - pub fn context(self: Self) Context { + pub fn context(self: Identifier) Context { return .{ ._inner = c.mlirIdentifierGetContext(self._inner) }; } - pub fn str(self: Self) []const u8 { + pub fn str(self: Identifier) []const u8 { return fromStringRef(c.mlirIdentifierStr(self._inner)); } - pub fn equals(self: Self, other: Self) bool { + pub fn equals(self: Identifier, other: Identifier) bool { return c.mlirIdentifierEqual(self._inner, other._inner); } }; @@ -322,6 +315,14 @@ pub const Attribute = struct { return DictionaryAttribute.init(ctx, attrs).asAttr(); } + + pub fn eqlAny(Attr: type) fn (Attr, Attr) bool { + return struct { + fn eql(a: Attr, b: Attr) bool { + return a.asAttr().eql(b.asAttr()); + } + }.eql; + } }; pub const NamedAttribute = extern struct { @@ -349,15 +350,14 @@ pub const NamedAttribute = extern struct { pub const StringAttribute = struct { _inner: c.MlirAttribute, pub const is_a_fn = c.mlirAttributeIsAString; - const Self = StringAttribute; - pub const asAttr = Attribute.fromAny(Self); - pub const eql = Attribute.eqlAny(Self); + pub const asAttr = Attribute.fromAny(StringAttribute); + pub const eql = Attribute.eqlAny(StringAttribute); - pub fn init(ctx: Context, str: []const u8) Self { + pub fn init(ctx: Context, str: []const u8) StringAttribute { return .{ ._inner = c.mlirStringAttrGet(ctx._inner, stringRef(str)) }; } - pub fn value(self: Self) []const u8 { + pub fn value(self: StringAttribute) []const u8 { return fromStringRef(c.mlirStringAttrGetValue(self._inner)); } }; @@ -365,15 +365,14 @@ pub const StringAttribute = struct { pub const BoolAttribute = struct { _inner: c.MlirAttribute, pub const is_a_fn = c.mlirAttributeIsABool; - const Self = BoolAttribute; - pub const asAttr = Attribute.fromAny(Self); - pub const eql = Attribute.eqlAny(Self); + pub const asAttr = Attribute.fromAny(BoolAttribute); + pub const eql = Attribute.eqlAny(BoolAttribute); - pub fn init(ctx: Context, value_: bool) Self { + pub fn init(ctx: Context, value_: bool) BoolAttribute { return .{ ._inner = c.mlirBoolAttrGet(ctx._inner, if (value_) 1 else 0) }; } - pub fn value(self: Self) bool { + pub fn value(self: BoolAttribute) bool { return c.mlirBoolAttrGetValue(self._inner); } }; @@ -397,19 +396,18 @@ pub const TypeAttribute = struct { pub const ArrayAttribute = struct { _inner: c.MlirAttribute, pub const is_a_fn = c.mlirAttributeIsAArray; - const Self = ArrayAttribute; - pub const asAttr = Attribute.fromAny(Self); - pub const eql = Attribute.eqlAny(Self); + pub const asAttr = Attribute.fromAny(ArrayAttribute); + pub const eql = Attribute.eqlAny(ArrayAttribute); - pub fn init(ctx: Context, attrs: []const Attribute) Self { + pub fn init(ctx: Context, attrs: []const Attribute) ArrayAttribute { return .{ ._inner = c.mlirArrayAttrGet(ctx._inner, @intCast(attrs.len), @ptrCast(attrs.ptr)) }; } - pub fn size(self: Self) usize { + pub fn size(self: ArrayAttribute) usize { return @intCast(c.mlirArrayAttrGetNumElements(self._inner)); } - pub fn get(self: Self, index: usize) Attribute { + pub fn get(self: ArrayAttribute, index: usize) Attribute { return .{ ._inner = c.mlirArrayAttrGetElement(self._inner, @intCast(index)) }; } }; @@ -590,13 +588,13 @@ pub fn DenseElementsAttribute(comptime dt: DenseElementsAttributeTypes) type { pub fn init(shaped_type: Type, slice: []const dt.ZigType()) Attr { const raw_bytes = std.mem.sliceAsBytes(slice); - const res: Attr = .{ ._inner = c.mlirDenseElementsAttrRawBufferGet( + const attr: Attr = .{ ._inner = c.mlirDenseElementsAttrRawBufferGet( shaped_type._inner, @intCast(raw_bytes.len), @ptrCast(raw_bytes.ptr), ) }; - std.debug.assert(Attribute.wrapOr(res._inner) != null); - return res; + std.debug.assert(attr._inner.ptr != null); + return attr; } pub fn len(self: Attr) usize { @@ -717,7 +715,7 @@ pub const DictionaryAttribute = struct { } pub fn getByName(self: DictionaryAttribute, name: [:0]const u8) ?Attribute { - return Attribute.wrapOr(c.mlirDictionaryAttrGetElementByName(self._inner, name)); + return Attribute.wrapOr(c.mlirDictionaryAttrGetElementByName(self._inner, stringRef(name))); } }; @@ -728,8 +726,7 @@ pub const Operation = struct { pub const dump = helpers.dump(Operation, c.mlirOperationDestroy); pub const deinit = helpers.deinit(Operation, c.mlirOperationDestroy); pub const wrapOr = helpers.wrapOr(Operation, c.mlirOperationIsNull); - - pub const eql = Attribute.eqlAny(Self); + pub const eql = helpers.eql(Operation, c.mlirOperationEqual); pub fn init(state: *OperationState) !Self { return Self.wrapOr(c.mlirOperationCreate(&state._inner)) orelse Error.InvalidMlir; @@ -881,52 +878,33 @@ pub const Operation = struct { return .{ ._inner = c.mlirOperationGetContext(self._inner) }; } - pub fn writeBytecode(self: Self, writer: anytype) void { - var writer_context = .{ .writer = writer }; - const WriterContext = @TypeOf(writer_context); - + pub fn writeBytecode(self: Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + var writer_with_err: WriterWithErr = .{ .writer = writer }; c.mlirOperationWriteBytecode( self._inner, - (struct { - pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { - const inner_writer_context: *WriterContext = @ptrCast(@alignCast(ctx_)); - _ = inner_writer_context.writer.write(str.data[0..str.length]) catch unreachable; - } - }).callback, - &writer_context, + WriterWithErr.printCallback, + &writer_with_err, ); + return writer_with_err.check(); } - pub fn writeBytecodeWithConfig(self: Self, writer: anytype, config: struct { + pub fn writeBytecodeWithConfig(self: Self, writer: *std.Io.Writer, config: struct { desiredEmitedVersion: ?i64 = null, - }) !void { + }) error{ InvalidMlirBytecodeVersion, WriteFailed }!void { const cfg = c.mlirBytecodeWriterConfigCreate(); defer c.mlirBytecodeWriterConfigDestroy(cfg); if (config.desiredEmitedVersion) |v| { c.mlirBytecodeWriterConfigDesiredEmitVersion(cfg, v); } - const WriterContext = struct { - writer: @TypeOf(writer), - write_error: ?@TypeOf(writer).Error = null, - }; - var writer_context: WriterContext = .{ .writer = writer }; - + var writer_with_err: WriterWithErr = .{ .writer = writer }; try successOr(c.mlirOperationWriteBytecodeWithConfig( self._inner, cfg, - (struct { - pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { - const inner_writer_context: *WriterContext = @ptrCast(@alignCast(ctx_)); - _ = inner_writer_context.writer.write(str.data[0..str.length]) catch |err| { - inner_writer_context.write_error = err; - }; - } - }).callback, - &writer_context, + &WriterWithErr.printCallback, + &writer_with_err, ), error.InvalidMlirBytecodeVersion); - - if (writer_context.write_error) |err| return err; + return writer_with_err.check(); } /// Enable a full dump of the IR. @@ -939,26 +917,18 @@ pub const Operation = struct { op: Operation, flags: OpPrintingFlags, - pub fn format(self: @This(), writer: anytype) !void { - self.op.print(writer, self.flags); + pub fn format(self: MlirFormatter, writer: *std.Io.Writer) !void { + try self.op.print(writer, self.flags); } }; - pub fn print(self: Self, writer: *std.Io.Writer, flags: OpPrintingFlags) void { + pub fn print(self: Self, writer: *std.Io.Writer, flags: OpPrintingFlags) std.Io.Writer.Error!void { const pflags = flags.create(); defer c.mlirOpPrintingFlagsDestroy(pflags); - c.mlirOperationPrintWithFlags( - self._inner, - pflags, - (struct { - pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { - const _writer: *std.Io.Writer = @ptrCast(@alignCast(ctx_)); - _writer.writeAll(str.data[0..str.length]) catch @panic("Mlir print failed"); - } - }).callback, - writer, - ); + var writer_err: WriterWithErr = .{ .writer = writer }; + c.mlirOperationPrintWithFlags(self._inner, pflags, WriterWithErr.printCallback, &writer_err); + return writer_err.check(); } pub fn verify(self: Self) bool { @@ -1065,27 +1035,25 @@ pub const OpPrintingFlags = struct { pub const OpOperand = struct { _inner: c.MlirOpOperand, - const Self = OpOperand; + pub const wrapOr = helpers.wrapOr(OpOperand, c.mlirOpOperandIsNull); - pub fn owner(self: Self) Operation { + pub fn owner(self: OpOperand) Operation { return .{ ._inner = c.mlirOpOperandGetOwner(self._inner) }; } - pub fn number(self: Self) usize { + pub fn number(self: OpOperand) usize { return @intCast(c.mlirOpOperandGetOperandNumber(self._inner)); } - pub fn nextUse(self: Self) ?Self { - return Self.wrapOr( - c.mlirOpOperandGetNextUse(self._inner), - ); + pub fn nextUse(self: OpOperand) ?OpOperand { + return wrapOr(c.mlirOpOperandGetNextUse(self._inner)); } }; pub const Region = struct { _inner: c.MlirRegion, - pub const eql = helpers.eql(Region, c.mlirBlockEqual); + pub const eql = helpers.eql(Region, c.mlirRegionEqual); pub const deinit = helpers.deinit(Region, c.mlirRegionDestroy); pub const wrapOr = helpers.wrapOr(Region, c.mlirRegionIsNull); @@ -1121,7 +1089,7 @@ pub const Value = struct { pub const dump = helpers.dump(Value, c.mlirValueDump); pub const eql = helpers.eql(Value, c.mlirValueEqual); - pub const format = helpers.format(Value, c.mlirValuePrint).format; + pub const format = helpers.format(Value, c.mlirValuePrint); pub const wrapOr = helpers.wrapOr(Value, c.mlirValueIsNull); pub fn getType(val: Value) Type { @@ -1183,7 +1151,7 @@ pub const BlockArgument = struct { return @bitCast(c.mlirBlockArgumentGetArgNumber(arg._inner)); } - pub fn format(self: BlockArgument, writer: anytype) !void { + pub fn format(self: BlockArgument, writer: *std.Io.Writer) !void { const value = Value{ ._inner = self._inner }; return value.format(writer); } @@ -1192,7 +1160,7 @@ pub const BlockArgument = struct { pub const Type = struct { _inner: c.MlirType, - pub const dump = helpers.eql(Type, c.mlirTypeDump); + pub const dump = helpers.dump(Type, c.mlirTypeDump); pub const eql = helpers.eql(Type, c.mlirTypeEqual); pub const format = helpers.format(Type, c.mlirTypePrint); pub const wrapOr = helpers.wrapOr(Type, c.mlirTypeIsNull); @@ -1230,14 +1198,6 @@ pub const Type = struct { }.eql; } - pub fn formatAny(SpecificType: type) fn (SpecificType, SpecificType) type { - return struct { - pub fn format(self: SpecificType, writer: anytype) !void { - return try Type.format(self.asType(), writer); - } - }; - } - pub fn index(ctx: Context) Type { return IndexType.init(ctx).asType(); } @@ -1280,7 +1240,7 @@ pub const IndexType = struct { pub const asType = Type.fromAny(IndexType); pub const eql = Type.eqlAny(IndexType); - pub const format = Type.formatAny(IndexType).format; + pub const format = helpers.format(IndexType, c.mlirTypePrint); pub fn init(ctx: Context) IndexType { return .{ ._inner = c.mlirIndexTypeGet(ctx._inner) }; @@ -1452,7 +1412,7 @@ pub fn ComplexType(comptime ct: ComplexTypes) type { pub const asType = Type.fromAny(Complex); pub const eql = Type.eqlAny(Complex); - pub const format = Type.formatAny(Complex).format; + pub const format = helpers.format(Complex, c.mlirTypePrint); pub const ComplexTypeType: ComplexTypes = ct; pub const init = if (ct != .unknown) struct { @@ -1468,51 +1428,50 @@ pub const TupleType = struct { _inner: c.MlirType, pub const is_a_fn = c.mlirTypeIsATuple; - const Self = TupleType; - - pub fn init(ctx: Context, elements: []const Type) !Self { - return Self.wrapOr(c.mlirTupleTypeGet( + pub fn init(ctx: Context, elements: []const Type) !TupleType { + const tuple_type = c.mlirTupleTypeGet( ctx._inner, @intCast(elements.len), @ptrCast(elements.ptr), - )) orelse Error.InvalidMlir; + ); + return .{ ._inner = .{ .ptr = tuple_type.ptr orelse return error.InvalidMlir } }; } - pub fn getNumTypes(self: Self) usize { + pub fn getNumTypes(self: TupleType) usize { return @intCast(c.mlirTupleTypeGetNumTypes(self._inner)); } - pub fn getElementType(self: Self, index: usize) Type { + pub fn getElementType(self: TupleType, index: usize) Type { return .{ ._inner = c.mlirTupleTypeGetType(self._inner, @intCast(index)) }; } - pub const asType = Type.fromAny(Self); + pub const asType = Type.fromAny(TupleType); }; pub const FunctionType = struct { _inner: c.MlirType, pub const is_a_fn = c.mlirTypeIsAFunction; - const Self = FunctionType; - pub const asType = Type.fromAny(Self); - pub const eql = Type.eqlAny(IndexType); + pub const asType = Type.fromAny(FunctionType); + pub const eql = Type.eqlAny(FunctionType); - pub fn init(ctx: Context, args: []const Type, results: []const Type) !Self { - const func = Type.wrapOr(c.mlirFunctionTypeGet( + pub fn init(ctx: Context, args: []const Type, results: []const Type) !FunctionType { + const func_type = c.mlirFunctionTypeGet( ctx._inner, @intCast(args.len), @ptrCast(args.ptr), @intCast(results.len), @ptrCast(results.ptr), - )) orelse return Error.InvalidMlir; - return func.as(Self).?; + ); + return .{ ._inner = .{ .ptr = func_type.ptr orelse return error.InvalidMlir } }; } }; pub const RankedTensorType = struct { _inner: c.MlirType, pub const is_a_fn = c.mlirTypeIsARankedTensor; + pub const asType = Type.fromAny(RankedTensorType); pub const eql = Type.eqlAny(RankedTensorType); - pub const format = helpers.format(Type, c.mlirTypePrint); + pub const format = helpers.format(RankedTensorType, c.mlirTypePrint); pub fn init(dimensions: []const i64, elemType: Type) RankedTensorType { return .{ ._inner = c.mlirRankedTensorTypeGet( @@ -1534,20 +1493,16 @@ pub const RankedTensorType = struct { pub fn getDimension(self: RankedTensorType, dim: usize) i64 { return c.mlirShapedTypeGetDimSize(self._inner, @intCast(dim)); } - - pub const asType = Type.fromAny(RankedTensorType); }; pub const Dialect = struct { _inner: c.MlirDialect, - const Self = Dialect; - - pub fn getContext(self: Self) Context { + pub fn getContext(self: Dialect) Context { return .{ ._inner = c.mlirDialectGetContext(self._inner) }; } - pub fn getNamespace(self: Self) []const u8 { + pub fn getNamespace(self: Dialect) []const u8 { return fromStringRef(c.mlirDialectGetNamespace(self._inner)); } }; @@ -1579,7 +1534,7 @@ pub const DialectHandle = struct { pub const Location = struct { _inner: c.MlirLocation, - pub const eql = helpers.eql(Type, c.mlirLocationEqual); + pub const eql = helpers.eql(Location, c.mlirLocationEqual); pub const format = helpers.format(Location, c.mlirLocationPrint); pub fn fromSrc(ctx: Context, src: std.builtin.SourceLocation) Location { @@ -1613,17 +1568,17 @@ pub const Location = struct { ) }; } - pub fn named(loc: Location, ctx: Context, loc_name: [:0]const u8) Location { + pub fn named(loc: Location, ctx: Context, loc_name: []const u8) Location { return .{ ._inner = c.mlirLocationNameGet(ctx._inner, stringRef(loc_name), loc._inner) }; } pub fn namedFmt(loc: Location, ctx: Context, comptime fmt: [:0]const u8, args: anytype) Location { var buf: [256]u8 = undefined; - var stream = std.io.fixedBufferStream(&buf); - std.fmt.format(stream.writer(), fmt, args) catch { + var writer: std.Io.Writer = .fixed(&buf); + writer.print(fmt, args) catch { buf[256 - 3 ..].* = "...".*; }; - return loc.named(ctx, @ptrCast(stream.getWritten())); + return loc.named(ctx, writer.buffered()); } pub fn unknown(ctx: Context) Location { @@ -1636,7 +1591,6 @@ pub const Block = struct { pub const wrapOr = helpers.wrapOr(Block, c.mlirBlockIsNull); pub const deinit = helpers.deinit(Block, c.mlirBlockDestroy); - pub const eql = helpers.eql(Block, c.mlirBlockEqual); pub fn init(args: []const Type, locs: []const Location) !Block { @@ -1736,27 +1690,15 @@ pub const helpers = struct { }.isNull; } - pub fn format(Any: type, print_fn: fn (@FieldType(Any, "_inner"), ?*const MlirStrCallback, ?*anyopaque) callconv(.c) void) type { + pub fn format( + Any: type, + print_fn: fn (@FieldType(Any, "_inner"), ?*const MlirStrCallback, ?*anyopaque) callconv(.c) void, + ) fn (Any, *std.Io.Writer) std.Io.Writer.Error!void { return struct { - pub fn format(self: Any, writer: *std.Io.Writer) !void { - const WriterWithErr = struct { - writer: *std.Io.Writer, - err: ?std.Io.Writer.Error = null, - fn printCallback(mlir_str: c.MlirStringRef, opaque_ctx: ?*anyopaque) callconv(.c) void { - var ctx: *@This() = @ptrCast(@alignCast(opaque_ctx)); - if (ctx.err) |_| return; - _ = ctx.writer.write(mlir_str.data[0..mlir_str.length]) catch |err| { - ctx.err = err; - return; - }; - } - }; - - var context: WriterWithErr = .{ .writer = writer }; - print_fn(self._inner, &WriterWithErr.printCallback, &context); - if (context.err) |err| return err; + pub fn format(self: Any, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try callPrintFn(Any, self, print_fn, writer); } - }; + }.format; } pub fn wrapOr(T: type, is_null_fn: fn (@FieldType(T, "_inner")) callconv(.c) bool) fn (@FieldType(T, "_inner")) ?T { @@ -1767,9 +1709,35 @@ pub const helpers = struct { } }.wrapOr; } +}; - pub fn init(T: type, inner: @FieldType(T, "_inner"), is_null_fn: fn (@FieldType(T, "_inner")) callconv(.c) bool) ?T { - if (is_null_fn(inner)) return null; - return .{ ._inner = inner }; +pub const MlirStrCallback = fn (c.MlirStringRef, ?*anyopaque) callconv(.c) void; + +pub fn callPrintFn( + T: type, + value: T, + print_fn: fn (@FieldType(T, "_inner"), ?*const MlirStrCallback, ?*anyopaque) callconv(.c) void, + writer: *std.Io.Writer, +) std.Io.Writer.Error!void { + var writer_with_err: WriterWithErr = .{ .writer = writer }; + print_fn(value._inner, &WriterWithErr.printCallback, &writer_with_err); + return writer_with_err.check(); +} + +pub const WriterWithErr = struct { + writer: *std.Io.Writer, + err: ?std.Io.Writer.Error = null, + + pub fn printCallback(mlir_str: c.MlirStringRef, opaque_ctx: ?*anyopaque) callconv(.c) void { + var ctx: *WriterWithErr = @ptrCast(@alignCast(opaque_ctx)); + if (ctx.err) |_| return; + ctx.writer.writeAll(fromStringRef(mlir_str)) catch |err| { + ctx.err = err; + return; + }; + } + + pub fn check(self: WriterWithErr) std.Io.Writer.Error!void { + if (self.err) |err| return err; } }; diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index 2ca639a7..6a49294e 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -265,16 +265,8 @@ pub const Buffer = extern struct { return self._dims[0..self.rank]; } - pub fn format( - buffer: Buffer, - comptime fmt: []const u8, - options: std.fmt.FormatOptions, - writer: anytype, - ) !void { - _ = fmt; - _ = options; - - try writer.print("FfiBuffer({d}, .{s})@0x{x}", .{ buffer.dims(), @tagName(buffer.dtype), @intFromPtr(buffer.data) }); + pub fn format(buffer: Buffer, writer: *std.Io.Writer) !void { + try writer.print("FfiBuffer({any}, .{t})@0x{x}", .{ buffer.dims(), buffer.dtype, @intFromPtr(buffer.data) }); } }; diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index d4d49dcd..25e45516 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -1257,14 +1257,7 @@ pub const NamedValue = extern struct { }) }; } - pub fn format( - self: NamedValue, - comptime fmt: []const u8, - options: std.fmt.FormatOptions, - writer: anytype, - ) !void { - _ = fmt; - _ = options; + pub fn format(self: NamedValue, writer: *std.Io.Writer) !void { try writer.print("{s}{{ .name = {s},", .{ @typeName(NamedValue), self.inner.name[0..self.inner.name_size] }); const u = self.inner.unnamed_0; switch (self.kind()) { diff --git a/runtimes/neuron/libneuronxla.zig b/runtimes/neuron/libneuronxla.zig index 0010d28e..59d8ae6b 100644 --- a/runtimes/neuron/libneuronxla.zig +++ b/runtimes/neuron/libneuronxla.zig @@ -117,8 +117,8 @@ fn wrapNeffAsCustomCall(allocator: std.mem.Allocator, hlo_code: []const u8, neff }; { - var operand_ids: std.ArrayListUnmanaged(i64) = .initBuffer(c.xla_HloInstructionProto_resize_operand_ids(fused_root, parameters_len + 1, upb_arena)[0 .. parameters_len + 1]); - var new_instructions: std.ArrayListUnmanaged(*const c.xla_HloInstructionProto) = .initBuffer(@ptrCast(c.xla_HloComputationProto_resize_instructions(entry, parameters_len + 1, upb_arena)[0 .. parameters_len + 1])); + var operand_ids: std.ArrayList(i64) = .initBuffer(c.xla_HloInstructionProto_resize_operand_ids(fused_root, parameters_len + 1, upb_arena)[0 .. parameters_len + 1]); + var new_instructions: std.ArrayList(*const c.xla_HloInstructionProto) = .initBuffer(@ptrCast(c.xla_HloComputationProto_resize_instructions(entry, parameters_len + 1, upb_arena)[0 .. parameters_len + 1])); for (entry_instructions) |instruction| { if (std.mem.eql(u8, upb.slice(c.xla_HloInstructionProto_opcode(instruction)) orelse continue, "parameter")) { const id = c.xla_HloInstructionProto_id(instruction); diff --git a/stdx/fmt.zig b/stdx/fmt.zig index 03bf7a35..4e38488a 100644 --- a/stdx/fmt.zig +++ b/stdx/fmt.zig @@ -9,6 +9,10 @@ fn FmtSlice(T: type) type { return struct { slice: []const T, + pub fn format(f: @This(), writer: *std.io.Writer) std.io.Writer.Error!void { + return try formatSliceAny(f.slice, .{}, writer); + } + pub fn formatNumber(f: @This(), writer: *std.io.Writer, n: std.fmt.Number) std.io.Writer.Error!void { return switch (@typeInfo(T)) { .comptime_float, .float => try formatFloatSlice(f.slice, n, writer), diff --git a/zml/aio.zig b/zml/aio.zig index 38a9914f..47cc75f3 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -315,14 +315,7 @@ pub const Metadata = union(enum) { }; } - pub fn format( - self: Metadata, - comptime fmt: []const u8, - options: std.fmt.FormatOptions, - writer: anytype, - ) !void { - _ = fmt; - _ = options; + pub fn format(self: Metadata, writer: *std.Io.Writer) !void { switch (self) { .null => _ = try writer.write("null"), inline .bool, .array_bool => |b| try writer.print("{any}", .{b}), diff --git a/zml/aio/json.zig b/zml/aio/json.zig index 2c504bb3..6f465134 100644 --- a/zml/aio/json.zig +++ b/zml/aio/json.zig @@ -1,10 +1,10 @@ -const async = @import("async"); const std = @import("std"); + +const async = @import("async"); + const zml = @import("../zml.zig"); -const StringBuilder = std.ArrayListUnmanaged(u8); -const Allocator = std.mem.Allocator; - +const StringBuilder = std.ArrayList(u8); pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore { const file = try std.fs.cwd().openFile(path, .{}); defer file.close(); @@ -26,7 +26,7 @@ pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore return res; } -pub fn parseMetadata(allocator: Allocator, store: *zml.aio.BufferStore, prefix: StringBuilder, val: std.json.Value) !void { +pub fn parseMetadata(allocator: std.mem.Allocator, store: *zml.aio.BufferStore, prefix: StringBuilder, val: std.json.Value) !void { const metadata = &store._metadata; const key = prefix.items; return switch (val) { diff --git a/zml/aio/safetensors.zig b/zml/aio/safetensors.zig index f1cdf293..b49928ac 100644 --- a/zml/aio/safetensors.zig +++ b/zml/aio/safetensors.zig @@ -9,7 +9,6 @@ const zml = @import("../zml.zig"); const HostBuffer = zml.HostBuffer; const json = @import("json.zig"); -const StringBuilder = std.ArrayListUnmanaged(u8); const log = std.log.scoped(.@"zml/io"); pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore { @@ -17,19 +16,18 @@ pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore errdefer res.arena.deinit(); const arena = res.arena.allocator(); - var files = std.array_list.Managed(MemoryMappedFile).init(arena); - errdefer files.deinit(); + var files: std.ArrayList(MemoryMappedFile) = .empty; if (std.mem.endsWith(u8, path, ".safetensors.index.json")) { try loadFromIndex(arena, &res, &files, path); } else { try loadFile(arena, &res, &files, path); } - res.files = try files.toOwnedSlice(); + res.files = try files.toOwnedSlice(allocator); return res; } -fn loadFromIndex(allocator: Allocator, store: *zml.aio.BufferStore, files: *std.array_list.Managed(MemoryMappedFile), path: []const u8) !void { +fn loadFromIndex(allocator: Allocator, store: *zml.aio.BufferStore, files: *std.ArrayList(MemoryMappedFile), path: []const u8) !void { const file = async.File.open(path, .{}) catch |err| { log.err("Failed to open {s}: {}", .{ path, err }); return err; @@ -61,11 +59,11 @@ fn loadFromIndex(allocator: Allocator, store: *zml.aio.BufferStore, files: *std. if (index.object.get("__metadata__")) |metadata| { var prefix_buf: [1024]u8 = undefined; - try json.parseMetadata(allocator, store, StringBuilder.initBuffer(&prefix_buf), metadata); + try json.parseMetadata(allocator, store, .initBuffer(&prefix_buf), metadata); } } -fn loadFile(allocator: Allocator, store: *zml.aio.BufferStore, files: *std.array_list.Managed(MemoryMappedFile), path: []const u8) !void { +fn loadFile(allocator: Allocator, store: *zml.aio.BufferStore, files: *std.ArrayList(MemoryMappedFile), path: []const u8) !void { const file = async.File.open(path, .{}) catch |err| { log.err("Failed to open {s}: {}", .{ path, err }); return err; @@ -87,7 +85,7 @@ fn loadFile(allocator: Allocator, store: *zml.aio.BufferStore, files: *std.array errdefer buffer_file.deinit(); buffer_file.data_offset = 8 + json_header_length; - try files.append(buffer_file); + try files.append(allocator, buffer_file); errdefer _ = files.pop(); var it = metadata.object.iterator(); @@ -95,7 +93,7 @@ fn loadFile(allocator: Allocator, store: *zml.aio.BufferStore, files: *std.array const key = entry.key_ptr.*; if (std.mem.eql(u8, key, "__metadata__")) { var prefix_buf: [1024]u8 = undefined; - try json.parseMetadata(allocator, store, StringBuilder.initBuffer(&prefix_buf), entry.value_ptr.*); + try json.parseMetadata(allocator, store, .initBuffer(&prefix_buf), entry.value_ptr.*); continue; } const val = entry.value_ptr.*; diff --git a/zml/aio/torch.zig b/zml/aio/torch.zig index fe487218..7c244f61 100644 --- a/zml/aio/torch.zig +++ b/zml/aio/torch.zig @@ -6,7 +6,6 @@ const zml = @import("../zml.zig"); const eval = @import("torch/eval.zig"); const File = @import("torch/file.zig").File; -const StringBuilder = std.ArrayListUnmanaged(u8); const log = std.log.scoped(.@"zml/aio"); test { diff --git a/zml/aio/torch/eval.zig b/zml/aio/torch/eval.zig index 42565ae5..01604e73 100644 --- a/zml/aio/torch/eval.zig +++ b/zml/aio/torch/eval.zig @@ -348,7 +348,7 @@ pub fn evaluate(arena: std.mem.Allocator, x: []const pickle.Op, resolve_refs: bo } fn append(allocator: std.mem.Allocator, current: *[]py.Any, values: []const py.Any) !void { - var array_list = std.ArrayListUnmanaged(py.Any).fromOwnedSlice(current.*); + var array_list = std.ArrayList(py.Any).fromOwnedSlice(current.*); try array_list.appendSlice(allocator, values); current.* = array_list.items; } diff --git a/zml/aio/torch/file.zig b/zml/aio/torch/file.zig index 1afafdd5..c1588b4e 100644 --- a/zml/aio/torch/file.zig +++ b/zml/aio/torch/file.zig @@ -13,7 +13,7 @@ const py = @import("py.zig"); const log = std.log.scoped(.@"zml/aio"); // TODO(cryptodeal): use zml.aio.PrefixBuilder instead -const StringBuilder = std.ArrayListUnmanaged(u8); +const StringBuilder = std.ArrayList(u8); test { std.testing.refAllDecls(@This()); @@ -191,7 +191,7 @@ pub const File = struct { .boolval => bool, else => unreachable, }; - var values: std.ArrayListUnmanaged(ItemType) = .{}; + var values: std.ArrayList(ItemType) = .{}; try values.append(allocator, val0); for (seq.values[1..], 1..) |val, i| { if (std.meta.activeTag(val) != tag) valid_slice = false; diff --git a/zml/aio/torch/pickle.zig b/zml/aio/torch/pickle.zig index 9174eb10..5724ab0a 100644 --- a/zml/aio/torch/pickle.zig +++ b/zml/aio/torch/pickle.zig @@ -768,7 +768,7 @@ pub fn parse(arena: std.mem.Allocator, reader: *std.Io.Reader) ![]const Op { // It's not very efficient to interleave the results with the data copied from the stream, // because growth event in the results ArrayList will lead to fragmentation. // Trying to mitigate that by using a generous default size. - var results: std.ArrayListUnmanaged(Op) = try .initCapacity(arena, 512); + var results: std.ArrayList(Op) = try .initCapacity(arena, 512); errdefer results.deinit(arena); var alloc_writer = try std.Io.Writer.Allocating.initCapacity(arena, 512); diff --git a/zml/aio/torch/py.zig b/zml/aio/torch/py.zig index c523f280..539c28a0 100644 --- a/zml/aio/torch/py.zig +++ b/zml/aio/torch/py.zig @@ -1,9 +1,10 @@ const std = @import("std"); const math = std.math; -const log = std.log.scoped(.@"zml/aio"); const pickle = @import("pickle.zig"); +const log = std.log.scoped(.@"zml/aio"); + /// Correspond to a function/constructor call pub const Object = struct { member: Any, @@ -206,12 +207,16 @@ pub const Any = union(Kind) { self.* = undefined; } - inline fn writeIndents(indents: usize, writer: anytype) !void { + inline fn writeIndents(indents: usize, writer: *std.Io.Writer) !void { try writer.writeBytesNTimes(" ", indents); // resolve tab = 2 spaces // try writer.writeByteNTimes('\t'); } - fn internalFormat(value: Any, indents: usize, writer: anytype) !void { + pub fn format(self: Any, writer: *std.Io.Writer) !void { + return internalFormat(self, 0, writer); + } + + fn internalFormat(value: Any, indents: usize, writer: *std.Io.Writer) !void { try writeIndents(indents, writer); try writer.writeAll(".{\n"); try writeIndents(indents + 1, writer); @@ -303,10 +308,6 @@ pub const Any = union(Kind) { try writer.writeByte('}'); } - pub fn format(self: Any, comptime _: []const u8, _: std.fmt.FormatOptions, writer: anytype) !void { - return internalFormat(self, 0, writer); - } - pub fn clone(self: Any, allocator: std.mem.Allocator) !Any { return switch (self) { inline .raw, .raw_num => |v, tag| @unionInit(Any, @tagName(tag), try v.clone(allocator)), diff --git a/zml/buffer.zig b/zml/buffer.zig index 6dcc42f2..36a55d88 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -384,10 +384,7 @@ pub const Buffer = struct { return res; } - pub fn format( - self: Buffer, - writer: anytype, - ) !void { + pub fn format(self: Buffer, writer: *std.Io.Writer) !void { try writer.print("Buffer({f})", .{self._shape}); } diff --git a/zml/exe.zig b/zml/exe.zig index 9e3d35e4..37c98269 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -95,6 +95,8 @@ pub fn compileFn( ) !FnExe(func) { var pretty_name = try prettyFnName(func, allocator); defer pretty_name.deinit(allocator); + log.info("Compiling {s} with {f}", .{ pretty_name.items, stdx.fmt.any(args) }); + var context = try CompilationContext.init(allocator, pretty_name.items, platform); defer context.deinit(); @@ -306,7 +308,7 @@ pub const BaseExe = struct { } } - pub fn serialize(self: BaseExe, writer: anytype) !void { + pub fn serialize(self: BaseExe, writer: *std.Io.Writer) !void { var executable = try self.exe.getExecutable(self.platform.pjrt_api); var serialize_result = try executable.serialize(self.platform.pjrt_api); defer serialize_result.deinit(); @@ -377,7 +379,7 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { try self.inner.bind(T, value); } - pub fn serialize(self: Self, writer: anytype) !void { + pub fn serialize(self: Self, writer: *std.Io.Writer) !void { return try self.inner.serialize(writer); } @@ -437,7 +439,7 @@ fn fillBuffers(v: anytype, shapes: []const Shape, buffers: []const [*]*pjrt.Buff fn prettyFnName( comptime func: anytype, allocator: std.mem.Allocator, -) !std.ArrayListUnmanaged(u8) { +) !std.ArrayList(u8) { const full_noisy_name = @typeName(@TypeOf(func)); const og_len = full_noisy_name.len; const buffer = try allocator.alloc(u8, og_len); diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 848fee86..88419b7d 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -321,10 +321,7 @@ pub const HostBuffer = struct { }; } - pub fn format( - self: HostBuffer, - writer: anytype, - ) !void { + pub fn format(self: HostBuffer, writer: *std.Io.Writer) !void { try writer.print("HostBuffer(.{f})", .{self._shape}); } diff --git a/zml/meta.zig b/zml/meta.zig index 855318a5..891a27ec 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -679,28 +679,38 @@ test zip { /// Given a func(X) -> Y or a func(Ctx, X) -> Y, /// finds all X in the given object, and write the result of func(X) into an arraylist. -pub fn collect(func: anytype, func_ctx: _CollectCtx(func), out: *std.array_list.Managed(stdx.meta.FnSignature(func, null).ReturnT), obj: anytype) error{OutOfMemory}!void { +pub fn collectAlloc( + func: anytype, + func_ctx: _CollectCtx(func), + allocator: std.mem.Allocator, + obj: anytype, +) std.mem.Allocator.Error![]stdx.meta.FnSignature(func, null).ReturnT { stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".params.len <= 2, "zml.meta.collect expects a func with two arguments, got: {}", .{@TypeOf(func)}); - const LocalContext = struct { + + const CollectAllocCtx = struct { func_ctx: _CollectCtx(func), - out: *std.array_list.Managed(stdx.meta.FnSignature(func, null).ReturnT), + allocator: std.mem.Allocator, + out: std.ArrayList(stdx.meta.FnSignature(func, null).ReturnT) = .empty, oom: bool = false, - }; - var context = LocalContext{ .func_ctx = func_ctx, .out = out }; - visit((struct { - fn cb(ctx: *LocalContext, val: *const _CollectArg(func)) void { + + fn cb(ctx: *@This(), val: *const _CollectArg(func)) void { if (ctx.oom) return; const res = if (_CollectCtx(func) == void) func(val.*) else func(ctx.func_ctx, val.*); - ctx.out.append(res) catch { + ctx.out.append(ctx.allocator, res) catch { ctx.oom = true; }; } - }).cb, &context, obj); + }; + var context = CollectAllocCtx{ .func_ctx = func_ctx, .allocator = allocator }; + visit(CollectAllocCtx.cb, &context, obj); if (context.oom) return error.OutOfMemory; + + return context.out.toOwnedSlice(allocator); } /// Given a func(X) -> Y or a func(Ctx, X) -> Y, -/// finds all X in the given object, and write the result of func(X) into an arraylist. +/// finds all X in the given object, and write the result of func(X) into a given slice. +/// Asserts that the number of X found is equal to the slice len. pub fn collectBuf(func: anytype, func_ctx: _CollectCtx(func), obj: anytype, out: []stdx.meta.FnResult(func)) void { stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".params.len <= 2, "zml.meta.collectBuf expects a func with one or two arguments, got: {}", .{@TypeOf(func)}); const LocalContext = struct { diff --git a/zml/mlirx.zig b/zml/mlirx.zig index 28f40383..f50a2efa 100644 --- a/zml/mlirx.zig +++ b/zml/mlirx.zig @@ -96,6 +96,6 @@ pub const Type = struct { } } - std.debug.panic("Could not convert mlir.Type to DataType: {}", .{mlir_type}); + std.debug.panic("Could not convert mlir.Type to DataType: {f}", .{mlir_type}); } }; diff --git a/zml/module.zig b/zml/module.zig index 294d76b4..5c27607e 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -187,7 +187,8 @@ pub const CompilationContext = struct { if (cache_dir.createFile(mlir_name, .{ .truncate = true })) |file| { var write_buf: [4096]u8 = undefined; var writer = file.writer(&write_buf); - module.op().print(&writer.interface, .{ .debug_info = true, .debug_info_pretty_form = false }); + try module.op().print(&writer.interface, .{ .debug_info = true, .debug_info_pretty_form = false }); + try writer.interface.flush(); log.info("Wrote MLIR to {s}/{s}", .{ module_dir.?, mlir_name }); } else |_| { log.warn("Failed to open {s}", .{mlir_name}); @@ -341,12 +342,11 @@ pub const CompilationContext = struct { const locations = try arena.alloc(mlir.Location, tensor_count); @memset(locations, mlir.Location.unknown(mlir_ctx)); - var input_shapes: std.array_list.Managed(Shape) = try .initCapacity(res_allocator, tensor_count); - meta.collect(Tensor.shape, {}, &input_shapes, args) catch unreachable; - stdx.debug.internalAssert(input_shapes.items.len == tensor_count, "args have changed ?", .{}); + const input_shapes = try res_allocator.alloc(Shape, tensor_count); + meta.collectBuf(Tensor.shape, {}, args, input_shapes); const input_types = try arena.alloc(mlir.Type, tensor_count); - for (input_types, input_shapes.items) |*t, sh| t.* = mlirx.tensorType(mlir_ctx, sh); + for (input_types, input_shapes) |*t, sh| t.* = mlirx.tensorType(mlir_ctx, sh); const og_block_args = self._block_args; defer { @@ -399,7 +399,7 @@ pub const CompilationContext = struct { self.addDonationsAttributes(arg_attrs, fn_res_donations); self.addOutputMemoryKindAttributes(res_attrs, fn_res_output_memory_kind); if (self._platform.sharding().num_partitions > 1) { - self.addShardingAttributes(arg_attrs, res_attrs, input_shapes.items, fn_res_shapes); + self.addShardingAttributes(arg_attrs, res_attrs, input_shapes, fn_res_shapes); } } @@ -427,7 +427,7 @@ pub const CompilationContext = struct { return .{ .mlir_fn = mlir_fn, .name = opts.name, - .args_shapes = input_shapes.items, + .args_shapes = input_shapes, .res_tensors = fn_res, .res_types = fn_res_types, .res_shapes = fn_res_shapes, @@ -506,9 +506,9 @@ pub const CompilationContext = struct { var tensor_args = .{ model, Tensor{ ._shape = s, ._id = .{ .buffer_id = 1234 } }, Tensor{ ._shape = s, ._id = .{ .buffer_id = 1235 } } }; const f = try comp.emitMlir(Local._fwd, &tensor_args, .{ .name = "test.emitMlir.Local.forward", .kind = .main }); - var mlir_bytecode = std.array_list.Managed(u8).init(std.testing.allocator); - defer mlir_bytecode.deinit(); - try mlir_bytecode.writer().print("{f}", .{f.mlir_fn.mlirFormatter(.{})}); + var mlir_code: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer mlir_code.deinit(); + try f.mlir_fn.print(&mlir_code.writer, .{}); // Check that the `x` input argument gives its buffer to the result tensor. // `%arg0` is the bias of the model, `%arg1` is `x`, `%arg2` is `y`. @@ -518,8 +518,8 @@ pub const CompilationContext = struct { var buf = template.*; for (0..2) |i| { const alias_attr = std.fmt.bufPrint(&buf, template, .{i}) catch unreachable; - std.testing.expect(std.mem.indexOf(u8, mlir_bytecode.items, alias_attr) != null) catch |err| { - log.warn("Didn't produced the expected IR:\n{s}", .{mlir_bytecode.items}); + std.testing.expect(std.mem.indexOf(u8, mlir_code.written(), alias_attr) != null) catch |err| { + log.warn("Didn't produced the expected IR:\n{s}", .{mlir_code.written()}); return err; }; } @@ -547,12 +547,14 @@ pub const CompilationContext = struct { pub fn getShardingAttr(self: CompilationContext, shape: Shape) mlir.Attribute { const ctx = self.mlirCtx(); const num_partitions = self.numPartitions(); - var sharding_str: stdx.BoundedArray(u8, 128) = .{}; - writeShardingRepresentation(shape, num_partitions, sharding_str.writer()) catch unreachable; - return mlir.Attribute.string(ctx, sharding_str.constSlice()); + // This is big enough, see test below for examples values + var sharding_str_buf: [64]u8 = undefined; + var writer: std.Io.Writer = .fixed(&sharding_str_buf); + writeShardingRepresentation(shape, num_partitions, &writer) catch unreachable; + return mlir.Attribute.string(ctx, writer.buffered()); } - fn writeShardingRepresentation(shape: Shape, num_partitions: u8, writer: anytype) @TypeOf(writer).Error!void { + fn writeShardingRepresentation(shape: Shape, num_partitions: u8, writer: *std.Io.Writer) std.Io.Writer.Error!void { const n_sharded: u8 = @popCount(@as(u8, @bitCast(shape._sharding_info))); if (n_sharded == 0 or num_partitions == 1) { try writer.writeAll("{replicated}"); @@ -567,26 +569,26 @@ pub const CompilationContext = struct { } test writeShardingRepresentation { - var rule: [64]u8 = undefined; + var attr_buf: [64]u8 = undefined; const x = Shape.init(.{ 16, 8 }, .f32); // By default tensors are replicated. { - var fbs = std.io.fixedBufferStream(&rule); - try writeShardingRepresentation(x, 4, fbs.writer()); - try std.testing.expectEqualStrings("{replicated}", fbs.getWritten()); + var writer: std.Io.Writer = .fixed(&attr_buf); + try writeShardingRepresentation(x, 4, &writer); + try std.testing.expectEqualStrings("{replicated}", writer.buffered()); } // Shard along first axis. { - var fbs = std.io.fixedBufferStream(&rule); - try writeShardingRepresentation(x.withSharding(.{0}), 4, fbs.writer()); - try std.testing.expectEqualStrings("{devices=[4,1]<=[4]}", fbs.getWritten()); + var writer: std.Io.Writer = .fixed(&attr_buf); + try writeShardingRepresentation(x.withSharding(.{0}), 4, &writer); + try std.testing.expectEqualStrings("{devices=[4,1]<=[4]}", writer.buffered()); } // Also shard along second axis. { - var fbs = std.io.fixedBufferStream(&rule); - try writeShardingRepresentation(x.withSharding(.{ 0, 1 }), 2, fbs.writer()); - try std.testing.expectEqualStrings("{devices=[2,2]<=[2]}", fbs.getWritten()); + var writer: std.Io.Writer = .fixed(&attr_buf); + try writeShardingRepresentation(x.withSharding(.{ 0, 1 }), 2, &writer); + try std.testing.expectEqualStrings("{devices=[2,2]<=[2]}", writer.buffered()); } } diff --git a/zml/ops.zig b/zml/ops.zig index ee125086..edbcd4a2 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -443,29 +443,23 @@ pub fn if_( const true_branch_block, const true_branch_res = ctx.makeBlock(.open, TrueBlockSignature, &true_branch_fn, blkctx, {}); const false_branch_block, const false_branch_res = ctx.makeBlock(.open, TrueBlockSignature, &false_branch_fn, blkctx, {}); - var true_shapes = std.array_list.Managed(Shape).init(ctx.allocator()); - defer true_shapes.deinit(); - var false_shapes = std.array_list.Managed(Shape).init(ctx.allocator()); - defer false_shapes.deinit(); + check: { + const arena = ctx.allocator(); + const true_shapes = meta.collectAlloc(Tensor.shape, {}, arena, &true_branch_res) catch break :check; + defer arena.free(true_shapes); - var failed_to_collect = false; - meta.collect(Tensor.shape, {}, &true_shapes, &true_branch_res) catch { - failed_to_collect = true; - }; - meta.collect(Tensor.shape, {}, &false_shapes, &false_branch_res) catch { - failed_to_collect = true; - }; - if (!failed_to_collect) { - stdx.debug.assert(true_shapes.items.len == false_shapes.items.len, "zml.ops.if_ expects the true and false branch to produce the same number of tensors. Got: \n - true branch: {any}\n -false branch: {any}", .{ true_shapes.items, false_shapes.items }); - for (true_shapes.items, false_shapes.items) |true_shape, false_shape| { - stdx.debug.assert(true_shape.eqlWithTags(false_shape), "zml.ops.if_ expects the true and false branch to produce tensors of the same shape. Got: \n - true branch: {any}\n -false branch: {any}", .{ true_shapes.items, false_shapes.items }); + const false_shapes = meta.collectAlloc(Tensor.shape, {}, arena, &false_branch_res) catch break :check; + defer arena.free(false_shapes); + + stdx.debug.assert(true_shapes.len == false_shapes.len, "zml.ops.if_ expects the true and false branch to produce the same number of tensors. Got: \n - true branch: {f}\n -false branch: {f}", .{ stdx.fmt.slice(true_shapes), stdx.fmt.slice(false_shapes) }); + for (true_shapes, false_shapes) |true_shape, false_shape| { + stdx.debug.assert(true_shape.eqlWithTags(false_shape), "zml.ops.if_ expects the true and false branch to produce tensors of the same shape. Got: \n - true branch: {f}\n -false branch: {f}", .{ stdx.fmt.slice(true_shapes), stdx.fmt.slice(false_shapes) }); } } - const scalar_pred = if (pred.rank() == 0) pred else pred.flattenAll().squeeze(0); const loc = ctx.mlirCtx().location(@src()); const op = mlir.Operation.make(ctx.mlirCtx(), "stablehlo.if", .{ - .operands = &.{scalar_pred.value()}, + .operands = &.{pred.asScalar().value()}, .result_type_inference = true, .blocks = &.{ true_branch_block, false_branch_block }, // We can't verify right away, cause the weights captured by the if haven't been added yet. @@ -958,18 +952,20 @@ pub fn scatter( const UpdateS = BlockSign(update_fn); const update_block, _ = ctx.makeBlock(.hermetic, UpdateS, update_fn, blkctx, .{ _scalar, _scalar }); - var input_values = std.array_list.Managed(mlir.Value).initCapacity(ctx.allocator(), n_inputs) catch @panic("OOM"); - defer input_values.deinit(); - meta.collect(CompilationContext.getValue, ctx, &input_values, &inputs) catch unreachable; - var updates_values = std.array_list.Managed(mlir.Value).initCapacity(ctx.allocator(), n_updates) catch @panic("OOM"); - defer updates_values.deinit(); - meta.collect(CompilationContext.getValue, ctx, &updates_values, &updates) catch unreachable; + const arena = ctx.allocator(); + const input_values = arena.alloc(mlir.Value, n_inputs) catch @panic("OOM"); + defer arena.free(input_values); + meta.collectBuf(CompilationContext.getValue, ctx, &inputs, input_values); + + const updates_values = arena.alloc(mlir.Value, n_updates) catch @panic("OOM"); + defer arena.free(updates_values); + meta.collectBuf(CompilationContext.getValue, ctx, &updates, updates_values); const op = dialect.stablehlo.scatter( mlir_ctx, - input_values.items, + input_values, &.{indices.value()}, - updates_values.items, + updates_values, update_block, .{ .update_window_dims = _collectAxes(AxisKind, config.up_kind, .update_window).constSlice(), diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index f3c8c349..5932619a 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -75,14 +75,17 @@ pub const Client = opaque { } fn compileSync(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { - var bytecode: std.array_list.Managed(u8) = .init(allocator); + var bytecode: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); defer bytecode.deinit(); - module.op().writeBytecodeWithConfig(bytecode.writer(), .{ .desiredEmitedVersion = 1 }) catch |err| { + module.op().writeBytecodeWithConfig(&bytecode.writer, .{ .desiredEmitedVersion = 1 }) catch |err| { log.err("failed to write module bytecode: {}", .{err}); - return err; + return switch (err) { + std.Io.Writer.Error.WriteFailed => error.OutOfMemory, + else => |e| e, + }; }; - var serialized_buffer: std.array_list.Managed(u8) = .init(allocator); + var serialized_buffer: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); defer serialized_buffer.deinit(); const stablehlo_version = blk: { @@ -92,13 +95,16 @@ pub const Client = opaque { break :blk dialects.stablehlo.getMinimumVersion(); }; - dialects.stablehlo.serializePortableArtifact(bytecode.items, stablehlo_version, serialized_buffer.writer()) catch |err| { + dialects.stablehlo.serializePortableArtifact(bytecode.written(), stablehlo_version, &serialized_buffer.writer) catch |err| { log.err("failed to serialize to portable artifact: {}", .{err}); - return err; + return switch (err) { + std.Io.Writer.Error.WriteFailed => error.OutOfMemory, + else => |e| e, + }; }; return @ptrCast(try self.inner().compile(api, .{ - .bytecode = serialized_buffer.items, + .bytecode = serialized_buffer.written(), .bytecode_format = .mlir, .compile_options_pb = compile_options_pb, })); diff --git a/zml/platform.zig b/zml/platform.zig index 6e13d7f0..a6f4e145 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -95,7 +95,7 @@ const _CreateOptions = struct { pub const Cpu = struct { device_count: u32, - fn writeNamedValues(self: Cpu, values: *std.ArrayListUnmanaged(pjrt.NamedValue)) void { + fn writeNamedValues(self: Cpu, values: *std.ArrayList(pjrt.NamedValue)) void { values.appendAssumeCapacity(pjrt.NamedValue.from("cpu_device_count", @as(i64, self.device_count))); } }; @@ -124,7 +124,7 @@ const _CreateOptions = struct { }; }; - fn writeNamedValues(self: Cuda, values: *std.ArrayListUnmanaged(pjrt.NamedValue)) void { + fn writeNamedValues(self: Cuda, values: *std.ArrayList(pjrt.NamedValue)) void { switch (self.allocator) { .platform => { values.appendAssumeCapacity(pjrt.NamedValue.fromString("allocator", "platform")); @@ -145,7 +145,7 @@ const _CreateOptions = struct { }; pub fn toNamedValues(self: _CreateOptions, target: Target, out: []pjrt.NamedValue) []pjrt.NamedValue { - var values = std.ArrayListUnmanaged(pjrt.NamedValue).fromOwnedSlice(out); + var values = std.ArrayList(pjrt.NamedValue).fromOwnedSlice(out); values.shrinkRetainingCapacity(0); switch (target) { .cpu => self.cpu.writeNamedValues(&values), diff --git a/zml/shape.zig b/zml/shape.zig index bd5f2a22..a9daffa7 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -386,14 +386,8 @@ pub const Shape = struct { /// Format the shape. /// Default format: "Shape({.a=10, .b=20}, dtype=.f32)" /// Bare format {_}: "{.a=10, .b=20}, dtype=.f32" - pub fn format( - self: Shape, - writer: anytype, - ) !void { - // TODO: impl alternative format - // const bare_fmt = fmt.len == 1 and fmt[0] == '_'; - const bare_fmt = true; - _ = try writer.write(if (bare_fmt) "{" else "Shape({"); + pub fn format(self: Shape, writer: *std.Io.Writer) !void { + _ = try writer.writeByte('{'); var need_comma = false; for (self.dims(), 0..) |d, i| { @@ -411,7 +405,7 @@ pub const Shape = struct { } if (need_comma) try writer.writeByte(','); _ = try writer.write(@tagName(self.dtype())); - _ = try writer.write(if (bare_fmt) "}" else "})"); + _ = try writer.writeByte('}'); } /// Broadcasts a Tensor to the given shape, extending dimensions if needed. diff --git a/zml/tensor.zig b/zml/tensor.zig index 9d55ebf3..50b8c561 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -52,10 +52,7 @@ pub const Tensor = struct { return CompilationContext.current(); } - pub fn format( - self: Tensor, - writer: anytype, - ) !void { + pub fn format(self: Tensor, writer: *std.Io.Writer) !void { // TODO(0.15.0) handle format // const bare_fmt = fmt.len == 1 and fmt[0] == '_'; const bare_fmt = false; @@ -1146,7 +1143,7 @@ pub const Tensor = struct { contracting_axes: []const [2]i8, batching_axes: []const [2]i8, ) Tensor { - stdx.debug.assert(lhs.dtype() == rhs.dtype(), "dotGeneral expects tensors to be of the same type, got {} and {}", .{ lhs.dtype(), rhs.dtype() }); + stdx.debug.assert(lhs.dtype() == rhs.dtype(), "dotGeneral expects tensors to be of the same type, got {f} and {f}", .{ lhs, rhs }); const Axes = stdx.BoundedArray(i64, MAX_RANK); @@ -1156,7 +1153,7 @@ pub const Tensor = struct { var rhs_batching_axes: Axes = .{}; for (batching_axes) |b_axes| { const l, const r = b_axes; - stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects batching dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); + stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects batching dimensions to be equal, got {d} and {d} in {f} and {f}", .{ l, r, lhs, rhs }); var t = lhs._shape.tag(l); if (t == Shape.TagUnknown) t = rhs._shape.tag(r); res_shape = res_shape.appendDim(lhs._shape.dim(l), t); @@ -1521,14 +1518,7 @@ pub const Tensor = struct { const to_the_end = std.math.maxInt(i64); - pub fn format( - self: Slice, - comptime fmt: []const u8, - options: std.fmt.FormatOptions, - writer: anytype, - ) !void { - _ = fmt; - _ = options; + pub fn format(self: Slice, writer: *std.Io.Writer) !void { if (self.singleton) { try writer.print("[{}]", .{self.start}); } else if (self.end == to_the_end and self.step == 1) { @@ -2043,7 +2033,7 @@ pub const Tensor = struct { /// Converts the given 1 element Tensor into a 0-rank Tensor. pub fn asScalar(self: Tensor) Tensor { stdx.debug.assert(self.count() == 1, "Tensor.asScalar expects an input with exactly 1-element got {f}", .{self}); - return self.reshape(.{}); + return if (self.rank() == 0) self else self.reshape(.{}); } pub const Pad = struct { @@ -2582,7 +2572,7 @@ pub const Tensor = struct { /// that requires host<->device synchronization. /// ZML tries to generate the easiest to optimize IR, and will warn you if it generates known problematic IR. pub fn scatterSlices(self: Tensor, indices: anytype, updates: Tensor, opts: ScatterOpts) Tensor { - scoped_log.debug("scatterSlices({}, {any}, {})", .{ self, indices, updates }); + scoped_log.debug("scatterSlices({f}, {any}, {f})", .{ self, indices, updates }); const UpdateType = @TypeOf(ScatterOpts.increment); @@ -3087,7 +3077,7 @@ pub const Tensor = struct { const tail_chunk_size: i64 = @rem(d, chunk_size); const allocator = self.getContext().allocator(); - var chunks = std.ArrayListUnmanaged(Tensor).initCapacity(allocator, n_chunks + 1) catch @panic("OOM"); + var chunks = std.ArrayList(Tensor).initCapacity(allocator, n_chunks + 1) catch @panic("OOM"); for (0..n_chunks) |i| { const start: i64 = @as(i64, @intCast(i)) * chunk_size; chunks.appendAssumeCapacity( diff --git a/zml/testing.zig b/zml/testing.zig index 59778eaf..c9ad9efa 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -229,7 +229,7 @@ pub fn testLayerOut( const FetchCtx = struct { store: zml.aio.BufferStore, index: u32, - prefix: std.ArrayListUnmanaged(u8), + prefix: std.ArrayList(u8), platform: zml.Platform, fn fetch(ctx: *@This(), x: zml.Tensor) zml.Buffer { diff --git a/zml/tokenizer/BUILD.bazel b/zml/tokenizer/BUILD.bazel index 9b2f9c71..b7af9e29 100644 --- a/zml/tokenizer/BUILD.bazel +++ b/zml/tokenizer/BUILD.bazel @@ -1,4 +1,4 @@ -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_test") zig_library( name = "tokenizer", @@ -25,3 +25,14 @@ zig_binary( "//stdx", ], ) + +zig_test( + name = "test", + main = "homemade.zig", + visibility = ["//visibility:public"], + deps = [ + "//async", + "//ffi:zig", + "//stdx", + ], +) diff --git a/zml/tokenizer/hftokenizers/hftokenizers.zig b/zml/tokenizer/hftokenizers/hftokenizers.zig index 7b36dbe8..01b9c997 100644 --- a/zml/tokenizer/hftokenizers/hftokenizers.zig +++ b/zml/tokenizer/hftokenizers/hftokenizers.zig @@ -110,6 +110,7 @@ pub const HFTokenizer = opaque { } pub fn tokenToId(self: *HFTokenizer, token: []const u8) ?u32 { - return c.hftokenizers_token_to_id(@ptrCast(self), ffi.ZigSlice.from(token)); + const id = c.hftokenizers_token_to_id(@ptrCast(self), ffi.ZigSlice.from(token)); + return if (id == std.math.maxInt(u32)) null else id; } }; diff --git a/zml/tokenizer/homemade.zig b/zml/tokenizer/homemade.zig index 9b5898d8..93383b0f 100644 --- a/zml/tokenizer/homemade.zig +++ b/zml/tokenizer/homemade.zig @@ -189,10 +189,9 @@ pub const Tokenizer = struct { // Step by step visualization of the progress. if (options.debug) { var _debug_buf: [256]u8 = undefined; - var _debug_alloc = std.heap.FixedBufferAllocator.init(&_debug_buf); - var debug_progress = std.array_list.Managed(u8).init(_debug_alloc.allocator()); - self.decodeWithOpts(&debug_progress, tok_buff[0..num_tokens], .{ .sep = "|" }) catch {}; - log.debug("tokens: {any} -> {s}", .{ tok_buff[0..num_tokens], debug_progress.items }); + var debug_progress: std.Io.Writer = .fixed(&_debug_buf); + self.decodeWithOpts(tok_buff[0..num_tokens], &debug_progress, .{ .sep = "|" }) catch {}; + log.debug("tokens: {any} -> {s}", .{ tok_buff[0..num_tokens], debug_progress.buffered() }); } var best_score: f32 = -1e10; var best_token: u32 = 0; @@ -311,22 +310,19 @@ pub const Tokenizer = struct { /// Converts the given slice of tokens back into bytes. /// Note that if the tokenizer allows sub-unicode bytes, it's possible /// the output is not valid utf8. - pub fn decode(self: *const Tokenizer, allocator: std.mem.Allocator, input: []const u32) error{OutOfMemory}![]u8 { - var output = std.array_list.Managed(u8).init(allocator); - errdefer output.deinit(); - - try self.decodeWithOpts(&output, input, .{}); - return output.toOwnedSlice(); + pub fn decode(self: *const Tokenizer, input: []const u32, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try self.decodeWithOpts(input, writer, .{}); } pub fn decodeWithOpts( self: *const Tokenizer, - output: *std.array_list.Managed(u8), input: []const u32, + output: *std.Io.Writer, opts: struct { sep: []const u8 = "" }, - ) error{OutOfMemory}!void { + ) std.Io.Writer.Error!void { const escaped = if (self.normalizer) |n| n.escapedSpace() else null; // Flag used to indicate if the first dummy whitespace has been consumed. + var first_token: bool = true; for (input) |id| { // Retrieve the slice corresponding to the id. var piece = self.lookupPiece(id); @@ -337,12 +333,13 @@ pub const Tokenizer = struct { while (std.mem.startsWith(u8, piece, escspc)) { piece = piece[escspc.len..]; // don't output a space at beginning of text. - if (output.items.len > 0) try output.append(' '); + if (!first_token) try output.writeByte(' '); } } - try output.appendSlice(piece); - if (opts.sep.len > 0) try output.appendSlice(opts.sep); + try output.writeAll(piece); + first_token = false; + try output.writeAll(opts.sep); } } @@ -472,10 +469,13 @@ pub const Decoder = struct { } pub fn decode(self: *Decoder, ids: []const u32) ![]const u8 { + // TODO: revisite tokenizer api to use std.Io.Writer. self.reset(); - const res = try self.inner.decode(self.arena.allocator(), ids); - self.current_string = res; - return res; + // Reuse the same warmup than in init, to maximize contiguous writes. + var arena_writer = std.Io.Writer.Allocating.initCapacity(self.arena.allocator(), 4096) catch unreachable; + try self.inner.decode(ids, &arena_writer.writer); + self.current_string = arena_writer.written(); + return self.current_string.?; } pub fn string(self: *const Decoder) []const u8 { @@ -623,9 +623,9 @@ pub const Normalizer = struct { return if (self._whitespace.len > 1) self._whitespace.constSlice() else null; } - fn addSlice(data: []const u8, consumed: usize, normalized: *std.array_list.Managed(u8), normalized_to_origin: *std.array_list.Managed(usize)) !void { - try normalized.appendSlice(data); - for (data) |_| try normalized_to_origin.append(consumed); + fn addSlice(allocator: std.mem.Allocator, data: []const u8, consumed: usize, normalized: *std.ArrayList(u8), normalized_to_origin: *std.ArrayList(usize)) !void { + try normalized.appendSlice(allocator, data); + for (data) |_| try normalized_to_origin.append(allocator, consumed); } pub const Result = struct { @@ -673,13 +673,13 @@ pub const Normalizer = struct { // Pre-allocate outputs const space = self.escapedSpace() orelse " "; const overhead = if (self.flags.split_on_punct_ascii) space.len + 1 else space.len; - var normalized = try std.array_list.Managed(u8).initCapacity(allocator, trimmed_input.len * overhead + 2 * space.len); - errdefer normalized.deinit(); - var normalized_to_origin = try std.array_list.Managed(usize).initCapacity(allocator, normalized.capacity); - errdefer normalized_to_origin.deinit(); + var normalized: std.ArrayList(u8) = try .initCapacity(allocator, trimmed_input.len * overhead + 2 * space.len); + errdefer normalized.deinit(allocator); + var normalized_to_origin: std.ArrayList(usize) = try .initCapacity(allocator, normalized.capacity); + errdefer normalized_to_origin.deinit(allocator); // If the spec asks for it, add a whitespace at the beginning. - if (self.flags.add_dummy_prefix) try addSlice(space, consumed, &normalized, &normalized_to_origin); + if (self.flags.add_dummy_prefix) try addSlice(allocator, space, consumed, &normalized, &normalized_to_origin); var is_prev_space: bool = true; var is_prev_word: bool = false; @@ -706,23 +706,23 @@ pub const Normalizer = struct { var byte = slice[0]; if (self.escapedSpace() != null and byte == ' ') { // replace the space token by the special token - try addSlice(space, origin, &normalized, &normalized_to_origin); + try addSlice(allocator, space, origin, &normalized, &normalized_to_origin); is_prev_word = false; break :ascii; } else if (self.flags.split_on_punct_ascii) { if (is_prev_word and isPunct(slice)) { // Insert a space, but continue handling the rest - try addSlice(space, origin, &normalized, &normalized_to_origin); + try addSlice(allocator, space, origin, &normalized, &normalized_to_origin); } } if (self.flags.lower_case_ascii) { byte = std.ascii.toLower(byte); } - try normalized.append(byte); - try normalized_to_origin.append(origin); + try normalized.append(allocator, byte); + try normalized_to_origin.append(allocator, origin); } else { // we can safely copy to the output. - try addSlice(slice, origin, &normalized, &normalized_to_origin); + try addSlice(allocator, slice, origin, &normalized, &normalized_to_origin); } is_prev_word = !is_prev_space and !isPunct(slice); } @@ -732,20 +732,20 @@ pub const Normalizer = struct { while (std.mem.endsWith(u8, normalized.items, space)) { const length = normalized.items.len - space.len; consumed = normalized_to_origin.items[length]; - try normalized.resize(length); - try normalized_to_origin.resize(length); + try normalized.resize(allocator, length); + try normalized_to_origin.resize(allocator, length); } } - try normalized_to_origin.append(consumed); + try normalized_to_origin.append(allocator, consumed); std.debug.assert(normalized_to_origin.items.len == normalized.items.len + 1); - if (self.flags.add_dummy_suffix) try addSlice(space, consumed, &normalized, &normalized_to_origin); + if (self.flags.add_dummy_suffix) try addSlice(allocator, space, consumed, &normalized, &normalized_to_origin); return .{ - .normalized = try normalized.toOwnedSlice(), - .normalized_to_origin = try normalized_to_origin.toOwnedSlice(), + .normalized_to_origin = try normalized_to_origin.toOwnedSlice(allocator), + .normalized = try normalized.toOwnedSlice(allocator), }; } @@ -1006,8 +1006,7 @@ pub const Gpt2TextDecoder = struct { /// Transform bytes representing text under the gpt2 encoding, /// and write to the `unicode` buffer utf-8 bytes. - pub fn decode(self: Gpt2TextDecoder, unicode: *std.array_list.Managed(u8), bytes: []const u8) ![]const u8 { - const start = unicode.items.len; + pub fn decode(self: Gpt2TextDecoder, bytes: []const u8, writer: *std.Io.Writer) (error{InvalidInput} || std.Io.Writer.Error)!void { var it = std.unicode.Utf8Iterator{ .i = 0, .bytes = bytes }; while (it.nextCodepointSlice()) |codepoint| { const code: Code = switch (codepoint.len) { @@ -1016,9 +1015,8 @@ pub const Gpt2TextDecoder = struct { else => return error.InvalidInput, }; const byte = self.code_to_byte.get(code) orelse return error.InvalidInput; - try unicode.append(byte); + try writer.writeByte(byte); } - return unicode.items[start..]; } inline fn isPrintableByte(c: u8) bool { @@ -1030,15 +1028,33 @@ test Gpt2TextDecoder { var decoder = try Gpt2TextDecoder.init(testing.allocator); defer decoder.deinit(); - var out = std.array_list.Managed(u8).init(testing.allocator); - defer out.deinit(); + var buf: [128]u8 = undefined; // Ascii is not changed. - try testing.expectEqualStrings("getTitle", try decoder.decode(&out, "getTitle")); + { + var out: std.Io.Writer = .fixed(&buf); + try decoder.decode("getTitle", &out); + try testing.expectEqualStrings("getTitle", out.buffered()); + } + + { + var out: std.Io.Writer = .fixed(&buf); + try decoder.decode("Ċ", &out); + try testing.expectEqualStrings("\n", out.buffered()); + } + // Leading space are represented with 'Ġ' - try testing.expectEqualStrings(" UINavigationController", try decoder.decode(&out, "ĠUINavigationController")); + { + var out: std.Io.Writer = .fixed(&buf); + try decoder.decode("ĠUINavigationController", &out); + try testing.expectEqualStrings(" UINavigationController", out.buffered()); + } // Russian is wild - try testing.expectEqualStrings(" работ", try decoder.decode(&out, "ĠÑĢабоÑĤ")); + { + var out: std.Io.Writer = .fixed(&buf); + try decoder.decode("ĠÑĢабоÑĤ", &out); + try testing.expectEqualStrings(" работ", out.buffered()); + } } /// Open a json file in HF format and load the vocab from it. @@ -1077,12 +1093,12 @@ pub fn fromHfJson(allocator: std.mem.Allocator, tokenizer_path: []const u8) !Tok // Buffer containing all concatenated tokens. // Reserve a big chunk, to avoid grow event, but release over-allocated memory. - var all_tokens = try std.array_list.Managed(u8).initCapacity(tokenizer.arena_state.allocator(), file_content.len); - const original_alloc = all_tokens.items.ptr; + var all_tokens: std.Io.Writer.Allocating = try .initCapacity(tokenizer.arena_state.allocator(), file_content.len); + const original_alloc = all_tokens.writer.buffer.ptr; // A re-alloc event here means we have invalidated all slices inside the tokenizer. // If this is too annoying we could switch to a custom type instead of slices. defer { - std.debug.assert(all_tokens.items.ptr == original_alloc); + std.debug.assert(all_tokens.writer.buffer.ptr == original_alloc); } // gpt2 based tokenizer got a special way of encoding unicode. @@ -1094,16 +1110,18 @@ pub fn fromHfJson(allocator: std.mem.Allocator, tokenizer_path: []const u8) !Tok defer gpt2_decoder.deinit(); var it = vocab.iterator(); while (it.next()) |kv| { - const token = gpt2_decoder.decode(&all_tokens, kv.key_ptr.*) catch |err| { + const n = all_tokens.writer.buffered().len; + gpt2_decoder.decode(kv.key_ptr.*, &all_tokens.writer) catch |err| { switch (err) { error.InvalidInput => { is_gpt2_vocab = false; break; }, - else => return err, + error.WriteFailed => return error.OutOfMemory, } }; const idx: u32 = @intCast(kv.value_ptr.*.integer); + const token = all_tokens.written()[n..]; tokenizer.addOwnedTokenByIndex(idx, @floatFromInt(vocab_size - idx), token); } @@ -1126,15 +1144,16 @@ pub fn fromHfJson(allocator: std.mem.Allocator, tokenizer_path: []const u8) !Tok if (token_obj != .object) return error.InvalidFormat; const v = objectGet(token_obj.object, .string, "content") orelse return error.InvalidFormat; const id: u32 = @intCast(objectGet(token_obj.object, .integer, "id") orelse return error.InvalidFormat); - const token = try if (is_gpt2_vocab) - gpt2_decoder.decode(&all_tokens, v) + const n = all_tokens.written().len; + try if (is_gpt2_vocab) + gpt2_decoder.decode(v, &all_tokens.writer) else - dup(&all_tokens, v); - + all_tokens.writer.writeAll(v); + const token = all_tokens.written()[n..]; tokenizer.addOwnedTokenByIndex(id, 0, token); } // We won't add more tokens here, let release. - all_tokens.shrinkAndFree(all_tokens.items.len); + _ = try all_tokens.toOwnedSlice(); var unk = tokenizer.lookup(""); if (objectGet(model, .integer, "unk_token")) |unk_tok| { @@ -1165,10 +1184,10 @@ pub fn fromHfJson(allocator: std.mem.Allocator, tokenizer_path: []const u8) !Tok } /// Returns a copy of the given string, stored inside the given ArrayList. -fn dup(buffer: *std.array_list.Managed(u8), str: []const u8) ![]const u8 { - const n = buffer.items.len; - try buffer.appendSlice(str); - return buffer.items[n..]; +fn dup(allocating: *std.Io.Writer.Allocating, str: []const u8) std.Io.Writer.Error![]const u8 { + const n = allocating.written().len; + try allocating.writer.writeAll(str); + return allocating.written()[n..]; } /// Returns the given entry in a json object only if it has the right type. diff --git a/zml/tokenizer/main.zig b/zml/tokenizer/main.zig index fc568388..5d78cb80 100644 --- a/zml/tokenizer/main.zig +++ b/zml/tokenizer/main.zig @@ -48,11 +48,11 @@ pub fn asyncMain() !void { } if (args.expected.len > 0) { - var expected = try std.array_list.Managed(u32).initCapacity(allocator, args.prompt.len); + var expected: std.ArrayList(u32) = try .initCapacity(allocator, args.prompt.len); var it = std.mem.splitSequence(u8, args.expected, ","); while (it.next()) |int_token| { const tok = try std.fmt.parseInt(u32, int_token, 10); - try expected.append(tok); + try expected.append(allocator, tok); } if (!std.mem.eql(u32, expected.items, prompt_tok)) { log.err("Doesn't match expected: {any}", .{expected.items}); From 1480f77a55eee7af170dd3db285c10c02ed917c7 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 18 Sep 2025 15:35:25 +0200 Subject: [PATCH 032/881] zml: add the platform tag inside the Buffer (#326) CPU buffers behave differently from Cuda/Rocm/TPU buffers, and in particular they are addressable from main memory. This was annoying and prevented us to add some safety checks that weren't valid for CPU. I'm adding a tag to buffers so that we know which target they refer to. Also there is a bug in PJRT unitialized which doesn't respected the memory. I've opened https://github.com/openxla/xla/pull/31292 but in the meantime I've a workaround in ZML that uses the regular .fromHostBuffer. (I've also a patched version of PJRT plugins in https://github.com/zml/pjrt-artifacts/pull/41 but not merged yet) --- docs/howtos/add_weights.md | 2 +- examples/llama/main.zig | 2 +- pjrt/pjrt.zig | 59 +++++++---- stdx/flags.zig | 20 +++- zml/aio.zig | 7 +- zml/buffer.zig | 209 +++++++++++++++++++------------------ zml/callback.zig | 10 +- zml/exe.zig | 5 + zml/hostbuffer.zig | 2 +- zml/module.zig | 10 +- zml/pjrtx.zig | 4 +- zml/platform.zig | 59 +++++++++-- zml/tensor.zig | 12 ++- 13 files changed, 258 insertions(+), 143 deletions(-) diff --git a/docs/howtos/add_weights.md b/docs/howtos/add_weights.md index 1c151c44..2caf2fc9 100644 --- a/docs/howtos/add_weights.md +++ b/docs/howtos/add_weights.md @@ -44,7 +44,7 @@ Then, in your `BUILD.bazel`, you can refer to the files you defined above, in the following way: ```python -zig_cc_binary( +zig_binary( name = "mnist", args = [ "$(location @mnist//:mnist.pt)", diff --git a/examples/llama/main.zig b/examples/llama/main.zig index dc31af5a..6d577695 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -93,7 +93,7 @@ pub fn generateText( var prefill_tokens = try zml.Buffer.fromSlice(platform, .{max_seq_len}, prefill_buffer); defer prefill_tokens.deinit(); - var prefill_token_pos = try zml.Buffer.constant(platform, zml.Shape.init(.{}, .u32), 0); + var prefill_token_pos = try zml.Buffer.scalar(platform, 0, .u32); defer prefill_token_pos.deinit(); const prefilled_tokens, const kv_cache, rng = mod_prefill.call(.{ prefill_tokens, prefill_token_pos, kv_cache_, rng }); diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 25e45516..a3ef83da 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -360,9 +360,11 @@ pub const Client = opaque { buffer_type: BufferType, dims: []const i64, byte_strides: ?[]const i64, - device: ?*const Device = null, host_buffer_semantics: HostBufferSemantics, - memory: ?*const Memory = null, + dst: union(enum) { + device: *const Device, + memory: *const Memory, + }, }; pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) ApiError!struct { *Buffer, ?*Event } { @@ -375,11 +377,11 @@ pub const Client = opaque { .byte_strides = if (args.byte_strides) |bs| @ptrCast(@constCast(bs.ptr)) else null, .num_byte_strides = if (args.byte_strides) |bs| bs.len else 0, .host_buffer_semantics = @intFromEnum(args.host_buffer_semantics), - .device = @ptrCast(@constCast(args.device)), - .memory = @ptrCast(@constCast(args.memory)), + .device = if (args.dst == .device) @ptrCast(@constCast(args.dst.device)) else null, + .memory = if (args.dst == .memory) @ptrCast(@constCast(args.dst.memory)) else null, .device_layout = null, // TODO - .done_with_host_buffer = null, - .buffer = null, + .done_with_host_buffer = null, // out + .buffer = null, // out }); return .{ @@ -430,7 +432,7 @@ pub const Client = opaque { pub fn addressableMemories(self: *const Client, api: *const Api) []*const Memory { const ret = api.call(.PJRT_Client_AddressableMemories, .{ .client = self.inner(), - }) catch unreachable; + }) catch return &.{}; if (ret.addressable_memories) |memories| { return @ptrCast(@constCast(memories[0..ret.num_addressable_memories])); } @@ -474,8 +476,10 @@ pub const Client = opaque { dims: []const i64, element_type: BufferType, layout: MemoryLayout, - device: ?*const Device = null, - memory: ?*const Memory = null, + dst: union(enum) { + device: *const Device, + memory: *const Memory, + }, }; pub fn createUninitializedBuffer(self: *const Client, api: *const Api, args: CreateUninitializedBufferArgs) ApiError!*Buffer { @@ -486,8 +490,8 @@ pub const Client = opaque { .shape_num_dims = @intCast(args.dims.len), .shape_element_type = @intFromEnum(args.element_type), .shape_layout = @ptrCast(&layout), - .device = @ptrCast(@constCast(args.device)), - .memory = @ptrCast(@constCast(args.memory)), + .device = if (args.dst == .device) @ptrCast(@constCast(args.dst.device)) else null, + .memory = if (args.dst == .memory) @ptrCast(@constCast(args.dst.memory)) else null, }); return @ptrCast(ret.buffer.?); } @@ -530,6 +534,8 @@ pub const MemoryStats = struct { pool_bytes_is_set: bool, // out peak_pool_bytes: u64, // out peak_pool_bytes_is_set: bool, // out + + pub const zeroes = std.mem.zeroes(MemoryStats); }; pub const Device = opaque { @@ -556,10 +562,11 @@ pub const Device = opaque { return @intCast(ret.local_hardware_id); } - pub fn addressableMemories(self: *const Device, api: *const Api) ApiError![]const *Memory { - const ret = try api.call(.PJRT_Device_AddressableMemories, .{ - .device = self.inner(), - }); + pub fn addressableMemories(self: *const Device, api: *const Api) []const *Memory { + const ret = api.call( + .PJRT_Device_AddressableMemories, + .{ .device = self.inner() }, + ) catch return &.{}; return @ptrCast(ret.memories[0..ret.num_memories]); } @@ -728,7 +735,6 @@ pub const LoadedExecutable = opaque { _ = api.call(.PJRT_LoadedExecutable_Destroy, .{ .executable = self.inner(), }) catch {}; - self.* = undefined; } pub fn delete(self: *LoadedExecutable, api: *const Api) void { @@ -759,6 +765,7 @@ pub const LoadedExecutable = opaque { non_donatable_input_indices: []const i64 = &.{}, context: ?*ExecuteContext, }; + pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ApiError!void { var options = pjrtStruct(c.PJRT_ExecuteOptions{ .send_callbacks = null, @@ -1048,8 +1055,16 @@ pub const Event = opaque { pub const Memory = opaque { pub const Kind = enum { device, - pinned_host, - unpinned_host, + host_pinned, + host_unpinned, + + pub fn pjrtName(k: Kind) []const u8 { + return switch (k) { + .device => "device", + .host_pinned => "pinned_host", + .host_unpinned => "unpinned_host", + }; + } }; const inner = InnerMixin(c.PJRT_Memory).inner; @@ -1061,8 +1076,12 @@ pub const Memory = opaque { pub fn kind(self: *const Memory, api: *const Api) Kind { const ret = api.call(.PJRT_Memory_Kind, .{ .memory = self.inner() }) catch unreachable; - const kind_ = ret.kind orelse unreachable; - return std.meta.stringToEnum(Kind, kind_[0..ret.kind_size]) orelse unreachable; + return switch (ret.kind_size) { + "device".len => .device, + "pinned_host".len => .host_pinned, + "unpinned_host".len => .host_unpinned, + else => @panic("Memory kind not supported"), + }; } pub fn kindId(self: *const Memory, api: *const Api) u32 { diff --git a/stdx/flags.zig b/stdx/flags.zig index 467cea69..667814f2 100644 --- a/stdx/flags.zig +++ b/stdx/flags.zig @@ -41,8 +41,9 @@ //! caller to manage the lifetime. The caller should be skipping program name. const std = @import("std"); -const builtin = @import("builtin"); const assert = std.debug.assert; +const builtin = @import("builtin"); + const debug = @import("debug.zig"); /// Format and print an error message to stderr, then exit with an exit code of 1. @@ -285,7 +286,7 @@ fn parse_flags(args: *std.process.ArgIterator, comptime Flags: type) Flags { fn assert_valid_value_type(comptime T: type) void { comptime { - if (T == []const u8 or T == [:0]const u8 or T == ByteSize or @typeInfo(T) == .int) return; + if (T == []const u8 or T == [:0]const u8 or T == ByteSize or @typeInfo(T) == .int or @typeInfo(T) == .float) return; if (@typeInfo(T) == .@"enum") { const info = @typeInfo(T).@"enum"; @@ -347,6 +348,7 @@ fn parse_value(comptime T: type, flag: []const u8, value: [:0]const u8) T { if (V == []const u8 or V == [:0]const u8) return value; if (V == ByteSize) return parse_value_size(flag, value); if (@typeInfo(V) == .int) return parse_value_int(V, flag, value); + if (@typeInfo(V) == .float) return parse_value_float(V, flag, value); if (@typeInfo(V) == .@"enum") return parse_value_enum(V, flag, value); comptime unreachable; } @@ -515,6 +517,20 @@ fn parse_value_int(comptime T: type, flag: []const u8, value: [:0]const u8) T { }; } +/// Parse string value into a float, providing a nice error message for the user. +fn parse_value_float(comptime T: type, flag: []const u8, value: [:0]const u8) T { + assert((flag[0] == '-' and flag[1] == '-') or flag[0] == '<'); + + return std.fmt.parseFloat(T, value) catch |err| { + switch (err) { + error.InvalidCharacter => fatal( + "{s}: expected a decimal value, but found '{s}' (invalid character)", + .{ flag, value }, + ), + } + }; +} + fn parse_value_enum(comptime E: type, flag: []const u8, value: [:0]const u8) E { assert((flag[0] == '-' and flag[1] == '-') or flag[0] == '<'); comptime assert(@typeInfo(E).@"enum".is_exhaustive); diff --git a/zml/aio.zig b/zml/aio.zig index 47cc75f3..8114e0a1 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -116,12 +116,17 @@ pub const BufferStore = struct { if (id < self._unique_id or self._unique_id + _store_id_range <= id) { @panic("`store.loadBufferById()` only works on Tensor created by `store.getTensor()`, using the same store object."); } + if (platform.target != .cpu) mem_debug: { + const stats = platform.getDevices()[0].memoryStats(platform.pjrt_api) catch break :mem_debug; + log.debug("Loading {s} -> {f} {d:>10} bytes ({d:>10} allocated / {d:>10} reserved)", .{ self.buffers.keys()[id - self._unique_id], x._shape, x.shape().byteSize(), stats.bytes_in_use, stats.bytes_reserved }); + } break :hb self.buffers.values()[id - self._unique_id]; }, else => @panic("`store.loadBufferById()` only works on Tensor created by `store.getTensor()`"), }; // Use the sharding information stored in the tensor. + std.debug.assert(host_buffer.shape().eql(x.shape())); host_buffer._shape = x.shape(); return try host_buffer.toDevice(platform); } @@ -703,7 +708,7 @@ pub fn loadModelBuffersWithPrefix( var res: zml.Bufferized(Model) = undefined; try zml.meta.mapAlloc(struct { pub fn initBuffer(_: void, tensor: zml.Tensor) zml.Buffer { - return .{ ._shape = tensor.shape(), ._api = undefined, ._shards = undefined }; + return .{ ._shape = tensor.shape(), ._api = undefined, ._shards = undefined, ._target = undefined }; } }.initBuffer, allocator, {}, model, &res); diff --git a/zml/buffer.zig b/zml/buffer.zig index 36a55d88..c9373e5f 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -8,6 +8,7 @@ const HostBuffer = @import("hostbuffer.zig").HostBuffer; const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; +const Target = @import("platform.zig").Target; test { std.testing.refAllDecls(@This()); @@ -22,40 +23,22 @@ const log = std.log.scoped(.zml); /// * loading weights from disk directly to the `device zml.aio.loadBuffers` /// * can be created by calling `HostBuffer.toDevice(platform)`. pub const Buffer = struct { - pub const Memory = enum { - host, - host_pinned, - device, - - pub fn toPjrtMemory(self: Memory) pjrt.Memory.Kind { - return switch (self) { - .host => .unpinned_host, - .host_pinned => .pinned_host, - .device => .device, - }; - } - - pub fn pjrtName(self: Memory) []const u8 { - return @tagName(self.toPjrtMemory()); - } - }; - _shape: Shape, _api: *const pjrt.Api, + _target: Target, _shards: Shards, pub const MAX_NUM_SHARDS: u8 = Platform.MAX_NUM_DEVICES; pub const Shards = stdx.BoundedArray(*pjrt.Buffer, MAX_NUM_SHARDS); - pub const FromOptions = struct { - wait: bool = true, - memory: ?Memory = null, - }; + pub const Memory = pjrt.Memory.Kind; + pub const FromOptions = struct { wait: bool = true, memory: Memory = .device }; /// Copies the content of the given buffer from host memory to the accelerator memory. pub fn from(platform: Platform, host_buffer: HostBuffer, opts: FromOptions) !Buffer { var res: Buffer = .{ ._api = platform.pjrt_api, + ._target = platform.target, ._shape = host_buffer.shape(), ._shards = .{}, }; @@ -82,35 +65,22 @@ pub const Buffer = struct { break :buf host_buffer.slice1d(ax, .{ .start = start, .end = start + chunk_size }); } else host_buffer; - var args = pjrt.Client.BufferFromHostBufferArgs{ + const args = pjrt.Client.BufferFromHostBufferArgs{ .data = buf._data, .buffer_type = buffer_type, .dims = buf.shape().dims(), .byte_strides = byte_strides, .host_buffer_semantics = .ImmutableUntilTransferCompletes, + // CPU has no distinctions between memories. + .dst = if (platform.target == .cpu or opts.memory == .device) + .{ .device = devices[i] } + else + .{ .memory = platform.memoryForDevice(opts.memory, devices[i]) }, }; - if (platform.target == .cpu or opts.memory == null) { - args.device = devices[i]; - } else { - const memory = opts.memory.?; - const device_memories = try devices[i].addressableMemories(platform.pjrt_api); - // TODO measure the cost of this and consider caching on Zig side inside the platform. - const selected_memory = for (device_memories) |m| { - const kind = m.kind(platform.pjrt_api); - if (kind == memory.toPjrtMemory()) break m; - } else { - log.warn("Platform {s} doesn't have memory {s}", .{ @tagName(platform.target), @tagName(memory) }); - return error.NotFound; - }; - args.memory = selected_memory; - } const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); - if (event) |ev| { - ev.deinit(platform.pjrt_api); - } - + if (event) |ev| ev.deinit(platform.pjrt_api); res._shards.appendAssumeCapacity(pjrt_buffer); } @@ -131,6 +101,15 @@ pub const Buffer = struct { return self; } + pub fn awaitBlocking(self: Buffer) !Buffer { + for (self._shards.constSlice()) |buffer| { + if (buffer.getReadyEvent(self._api)) |ev| { + try ev.awaitBlocking(self._api); + } + } + return self; + } + /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. pub fn fromPjrtBuffers(platform: Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); @@ -139,6 +118,7 @@ pub const Buffer = struct { shards.appendSliceAssumeCapacity(pjrt_buffers); return .{ ._api = platform.pjrt_api, + ._target = platform.target, ._shape = shape_, ._shards = shards, }; @@ -185,9 +165,10 @@ pub const Buffer = struct { } pub fn asHostBuffer(self: Buffer) HostBuffer { - // TODO: skip this check on cpu - // const memory = self.getMemory().kind(self._api); - // stdx.debug.assert((memory == .pinned_host) or (memory == .unpinned_host), "asHostBuffer({f}) expects a buffer allocated on host memory, got {t}. see `copyToMemory`", .{ self, memory }); + if (self._target != .cpu) { + const memory = self.getMemory().kind(self._api); + stdx.debug.assert((memory == .host_pinned) or (memory == .host_unpinned), "asHostBuffer({f}) expects a buffer allocated on host memory, got {t}. see `copyToMemory`", .{ self, memory }); + } const ptr: [*]u8 = @ptrCast(self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable); return HostBuffer.fromBytes(self._shape, ptr[0..self._shape.byteSize()]); } @@ -200,7 +181,7 @@ pub const Buffer = struct { } /// Creates a Buffer with a single element repeated manytime. - pub fn constant(platform: Platform, shape_: Shape, val: anytype) !Buffer { + pub fn constant(platform: Platform, shape_: Shape, val: anytype, opts: FromOptions) !Buffer { var start = try std.time.Timer.start(); defer { const duration_ms = stdx.math.divFloat(f32, start.read(), std.time.ns_per_ms); @@ -210,6 +191,8 @@ pub const Buffer = struct { } } + // Constant is always blocking because it uses pointer to stack memory. + const cst_opts: FromOptions = .{ .memory = opts.memory, .wait = true }; // Convert val to the requested dtype. const x = shape_.dtype().constant(val); const byte_size = shape_.dtype().sizeOf(); @@ -222,7 +205,7 @@ pub const Buffer = struct { ._strides = @splat(0), ._data = x.constSlice().ptr, }; - return try from(platform, host_buffer, .{ .wait = true }); + return try from(platform, host_buffer, cst_opts); } // To speed up copies, duplicate the scalar value into a vector, @@ -245,14 +228,14 @@ pub const Buffer = struct { else => unreachable, } const host_buffer: HostBuffer = .{ ._shape = shape_, ._strides = strides, ._data = &bytes }; - return try from(platform, host_buffer, .{ .wait = true }); + return try from(platform, host_buffer, cst_opts); } test constant { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const x = try constant(platform, Shape.init(.{ 4, 3, 2 }, .u16), 42); + const x = try constant(platform, Shape.init(.{ 4, 3, 2 }, .u16), 42, .{ .wait = true }); const y = try x.getValue([4 * 3 * 2]u16); try std.testing.expectEqual([_]u16{42} ** (4 * 3 * 2), y); } @@ -271,14 +254,6 @@ pub const Buffer = struct { /// Creates a Buffer from a pointer into device memory. /// This allows to interface with other libraries producing buffers. pub fn asViewOfDeviceBuffer(platform: Platform, shape_: Shape, stream: ?*const pjrt.Stream, device_data: *anyopaque) Buffer { - const minor_to_major: [Shape.MAX_RANK]i64 = comptime blk: { - var res: [Shape.MAX_RANK]i64 = undefined; - for (0..Shape.MAX_RANK) |i| { - res[i] = @intCast(Shape.MAX_RANK - i - 1); - } - break :blk res; - }; - const pjrt_buffer = platform.pjrt_client.createViewOfDeviceBuffer(platform.pjrt_api, .{ .data = device_data, .element_type = bufferTypeFromDtype(shape_.dtype()), @@ -287,7 +262,7 @@ pub const Buffer = struct { .device = platform.getDevices()[0], .layout = .{ .tiled = .{ - .minor_to_major = minor_to_major[Shape.MAX_RANK - shape_.rank() ..], + .minor_to_major = minorToMajor(shape_.rank()), .tile_dims = &.{}, .tile_dims_sizes = &.{}, }, @@ -299,15 +274,16 @@ pub const Buffer = struct { shards.appendAssumeCapacity(pjrt_buffer); return .{ ._api = platform.pjrt_api, + ._target = platform.target, ._shape = shape_, ._shards = shards, }; } - pub fn opaqueDeviceMemoryDataPointer(self: Buffer) [*]u8 { + pub fn devicePtr(self: Buffer) u64 { stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer", .{}); const opaque_ptr: *anyopaque = self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable; - return @ptrCast(opaque_ptr); + return @intFromPtr(opaque_ptr); } /// Fetches the content of the given buffer into a stack variable of the given type. @@ -350,6 +326,7 @@ pub const Buffer = struct { /// Depending on the platform, the memory is typically not released to the OS /// but just marked as available in the memory pool. pub fn deinit(self: *const Buffer) void { + // log.warn("Unloading {f} {d} bytes", .{ self._shape, self._shape.byteSize() }); for (self._shards.constSlice()) |buffer| { buffer.deinit(self._api); } @@ -385,7 +362,7 @@ pub const Buffer = struct { } pub fn format(self: Buffer, writer: *std.Io.Writer) !void { - try writer.print("Buffer({f})", .{self._shape}); + try writer.print("Buffer({f})@{x}", .{ self._shape, self.devicePtr() }); } pub fn getMemory(self: Buffer) *const pjrt.Memory { @@ -402,10 +379,10 @@ pub const Buffer = struct { wait: bool = true, }; - pub fn copyToMemory(self: Buffer, platform: Platform, memory: Memory, opts: CopyToMemoryOpts) !Buffer { - const pjrt_memory = platform.pjrt_client.memoryByKind(self._api, memory.toPjrtMemory()); + pub fn copyToMemory(self: Buffer, platform: Platform, memory: pjrt.Memory.Kind, opts: CopyToMemoryOpts) !Buffer { + const pjrt_memory = platform.pjrt_client.memoryByKind(self._api, memory); if (pjrt_memory == null) { - stdx.debug.panic("Memory destination `{s}` for {f}", .{ memory.pjrtName(), self }); + stdx.debug.panic("Memory destination `{t}` for {f}", .{ memory, self }); } var new_shards: Buffer.Shards = .{}; @@ -423,35 +400,34 @@ pub const Buffer = struct { } } - return Buffer{ ._shape = self._shape, ._shards = new_shards, ._api = self._api }; + return Buffer{ ._api = self._api, ._target = platform.target, ._shape = self._shape, ._shards = new_shards }; } - pub const UnitializedOptions = struct { - memory: ?pjrt.Memory.Kind = null, - }; + pub const UnitializedOptions = struct { memory: Memory = .device }; pub fn uninitialized(platform: Platform, shape_: Shape, opts: UnitializedOptions) !Buffer { + if (opts.memory != .device) { + // XLA uninitialized doesn't respect memory see https://github.com/openxla/xla/pull/31292 + // TODO: use uninitialized when it works again. + const host_buffer: HostBuffer = try .empty(std.heap.smp_allocator, shape_); + defer host_buffer.deinit(std.heap.smp_allocator); + return try .from(platform, host_buffer, .{ .wait = true, .memory = opts.memory }); + } + var res: Buffer = .{ ._api = platform.pjrt_api, ._shape = shape_, ._shards = .{}, + ._target = platform.target, }; errdefer for (res._shards.slice()) |shard| { shard.deinit(platform.pjrt_api); }; - const minor_to_major: [Shape.MAX_RANK]i64 = comptime blk: { - var minor_to_major: [Shape.MAX_RANK]i64 = undefined; - for (0..Shape.MAX_RANK) |i| { - minor_to_major[i] = @intCast(Shape.MAX_RANK - i - 1); - } - break :blk minor_to_major; - }; - - // TODO: support more advanced sharding specs stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); const sharding_ax: ?u3 = std.simd.firstTrue(shape_._sharding_info); const n_partitions = platform.sharding().num_partitions; + const shard_shape = if (sharding_ax) |ax| s: { // This kind of sharding error should be detected earlier on. stdx.debug.assert(@rem(shape_.dim(ax), n_partitions) == 0, "Buffer.uninitialized() expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ ax, n_partitions }); @@ -459,37 +435,56 @@ pub const Buffer = struct { break :s shard_shape; } else shape_; - const buffer_type = bufferTypeFromDtype(shape_.dtype()); + var args = pjrt.Client.CreateUninitializedBufferArgs{ + .dims = shard_shape.dims(), + .element_type = bufferTypeFromDtype(shape_.dtype()), + .layout = .{ + .tiled = .{ + .minor_to_major = minorToMajor(shape_.rank()), + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }, + // set per device, see below. + .dst = undefined, + }; + const devices = platform.getDevices(); for (0..n_partitions) |i| { - var args = pjrt.Client.CreateUninitializedBufferArgs{ - .dims = shard_shape.dims(), - .element_type = buffer_type, - .layout = .{ - .tiled = .{ - .minor_to_major = minor_to_major[Shape.MAX_RANK - shape_.rank() ..], - .tile_dims = &.{}, - .tile_dims_sizes = &.{}, - }, - }, - }; - if (opts.memory) |memory_kind| { - const memories = try devices[i].addressableMemories(platform.pjrt_api); - const memory = for (memories) |m| { - const kind = m.kind(platform.pjrt_api); - if (kind == memory_kind) break m; - } else return error.NotFound; - args.memory = memory; - } else { - args.device = devices[i]; - } - const pjrt_buffer = try platform.pjrt_client.createUnitializedBuffer(platform.pjrt_api, args); + args.dst = if (platform.target == .cpu or opts.memory == .device) + .{ .device = devices[i] } + else + .{ .memory = platform.memoryForDevice(opts.memory, devices[i]) }; - res._shards.appendAssumeCapacity(pjrt_buffer); + const shard = try platform.pjrt_client.createUnitializedBuffer(platform.pjrt_api, args); + res._shards.appendAssumeCapacity(shard); } return res; } + + test uninitialized { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const host_visible_memories: []const Memory = &.{ .host_pinned, .host_unpinned }; + for (host_visible_memories) |memory| { + const x = try uninitialized(platform, .init(.{6}, .u8), .{ .memory = memory }); + const x_ptr: [*]u8 = @ptrFromInt(x.devicePtr()); + @memcpy(x_ptr, &[_]u8{ 104, 101, 108, 108, 111, 33 }); + + const y = try x.getValue([6]u8); + try std.testing.expectEqualStrings("hello!", &y); + } + } + + pub fn isDeleted(self: Buffer) bool { + const deleted: bool = self._shards.get(0).isDeleted(self._api); + for (self._shards.slice()[1..]) |shard| { + stdx.debug.assert(shard.isDeleted(self._api) == deleted, "Buffer has some shards deleted but not others.", .{}); + } + return deleted; + } }; pub fn bufferTypeFromDtype(dt: DataType) pjrt.BufferType { @@ -517,3 +512,15 @@ test bufferTypeFromDtype { try std.testing.expectEqual(dt, bufferTypeFromDtype(dtypeFromBufferType(dt))); } } + +const _MINOR_TO_MAJOR = blk: { + var ret: [Shape.MAX_RANK]i64 = undefined; + for (0..Shape.MAX_RANK) |i| { + ret[i] = @intCast(Shape.MAX_RANK - i - 1); + } + break :blk ret; +}; + +fn minorToMajor(rank: u8) []const i64 { + return _MINOR_TO_MAJOR[_MINOR_TO_MAJOR.len - rank ..]; +} diff --git a/zml/callback.zig b/zml/callback.zig index 4d063533..5af96d15 100644 --- a/zml/callback.zig +++ b/zml/callback.zig @@ -99,7 +99,7 @@ pub fn call( .api_version = .typed_ffi, .backend_config = .dict(mlir_ctx, &.{}), .additional_attributes = &.{.{ "mhlo.frontend_attributes", .dict(mlir_ctx, &.{}) }}, - .has_side_effect = true, + .has_side_effect = Callback.callback_config.has_side_effect, .output_operand_aliases = Callback.callback_config.output_operand_aliases, }, output_types, @@ -123,6 +123,7 @@ pub const Config = struct { // TODO: document precisely what `command_buffer_compatible` is doing and its limitations. traits: pjrt.ffi.HandlerTraits = .{ .command_buffer_compatible = false }, // TODO: handle sharded inputs + has_side_effect: bool = true, }; /// Compile-time check that a callback has all informations we require. @@ -190,12 +191,12 @@ fn CallbackImpl(comptime Callback: type, call_frame: *pjrt.ffi.CallFrame) ?*pjrt else .asViewOfDeviceBuffer(platform, shape, null, ffi_buffer.data); if (opts.copy_inputs_to_host_pinned and platform.target != .cpu) { - log.debug("Copying argument {d} {f} {*} to host_pinned memory !", .{ i, zml_buffer, zml_buffer.opaqueDeviceMemoryDataPointer() }); + log.debug("Copying argument {d} {f} {x} to host_pinned memory !", .{ i, zml_buffer, zml_buffer.devicePtr() }); zml_buffer = zml_buffer.copyToMemory(platform, .host_pinned, .{ .wait = true }) catch |err| { - log.err("Failed to copy input buffer {d} {f} {*} to host_pinned: {}", .{ i, zml_buffer, zml_buffer.opaqueDeviceMemoryDataPointer(), err }); + log.err("Failed to copy input buffer {d} {f} {x} to host_pinned: {}", .{ i, zml_buffer, zml_buffer.devicePtr(), err }); return .create(call_frame.api, .resource_exhausted, "host pinned OOM"); }; - log.debug("--> {f} {*} ({})", .{ zml_buffer, zml_buffer.opaqueDeviceMemoryDataPointer(), @as(*const f32, @ptrCast(@alignCast(zml_buffer.opaqueDeviceMemoryDataPointer()))).* }); + log.debug("--> {f} {x}", .{ zml_buffer, zml_buffer.devicePtr() }); } callback_args[i] = zml_buffer; } @@ -282,6 +283,7 @@ pub const Print = struct { .copy_inputs_to_host_pinned = true, // Print is fairly predictable and can be captured in an execution graph. .traits = .{ .command_buffer_compatible = false }, + .has_side_effect = false, }; platform: Platform, diff --git a/zml/exe.zig b/zml/exe.zig index 37c98269..15d33b0c 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -235,6 +235,7 @@ pub const BaseExe = struct { if (self.execute_context) |ctx| { ctx.deinit(self.platform.pjrt_api); } + self.exe.deinit(self.platform.pjrt_api); self._arena.deinit(); } @@ -395,6 +396,10 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { self.inner._unsafeAssignResults(Bufferized(ReturnT), &result); return result; } + + pub fn clone(self: Self, allocator: std.mem.Allocator) !Self { + return .{ .inner = try self.inner.clone(allocator) }; + } }; } diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 88419b7d..3185348c 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -322,7 +322,7 @@ pub const HostBuffer = struct { } pub fn format(self: HostBuffer, writer: *std.Io.Writer) !void { - try writer.print("HostBuffer(.{f})", .{self._shape}); + try writer.print("HostBuffer(.{f})@{x}", .{ self._shape, @intFromPtr(self._data) }); } pub fn formatNumber(self: HostBuffer, writer: *std.io.Writer, n: std.fmt.Number) std.io.Writer.Error!void { diff --git a/zml/module.zig b/zml/module.zig index 5c27607e..7ba63b23 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -221,6 +221,12 @@ pub const CompilationContext = struct { break :blk loaded_executable; }; + { + const exe = try loaded_executable.getExecutable(self._platform.pjrt_api); + const stats = try exe.getCompiledMemoryStats(self._platform.pjrt_api); + log.debug("Compiled {s}: {any}", .{ self._name, stats }); + } + log.debug("******** ZML generated MLIR ********", .{}); log.debug("{f}", .{module.op().mlirFormatter(.{})}); @@ -881,6 +887,9 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m c.xla_ExecutableBuildOptionsProto_set_num_replicas(exec_build_options, sharding.num_replicas); c.xla_ExecutableBuildOptionsProto_set_num_partitions(exec_build_options, sharding.num_partitions); c.xla_ExecutableBuildOptionsProto_set_use_spmd_partitioning(exec_build_options, sharding.num_partitions > 1 or sharding.num_replicas > 1); + if (platform.compilation_options.device_memory_size > 0) { + c.xla_ExecutableBuildOptionsProto_set_device_memory_size(exec_build_options, @intCast(platform.compilation_options.device_memory_size)); + } c.xla_ExecutableBuildOptionsProto_set_device_assignment(exec_build_options, device_assignment_blk: { const device_assignment = try upb.new(c.xla_DeviceAssignmentProto, upb_arena); @@ -895,7 +904,6 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m } break :device_assignment_blk device_assignment; }); - break :executable_build_options_blk exec_build_options; }); diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 5932619a..46c40523 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -243,7 +243,9 @@ pub const Event = opaque { if (ctx.err) |e| { defer e.deinit(api); - return e.getCode(api).toApiError(); + const err_code = e.getCode(api).toApiError(); + log.err("{t} {s}", .{ err_code, e.getMessage(api) }); + return err_code; } } }; diff --git a/zml/platform.zig b/zml/platform.zig index a6f4e145..410d60bc 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -10,12 +10,17 @@ const log = std.log.scoped(.zml); pub const available_targets = std.enums.values(Target); +test { + std.testing.refAllDecls(@This()); +} + pub const CompilationOptions = struct { xla_dump_to: ?[]const u8 = null, xla_dump_fusion_visualization: bool = false, xla_dump_hlo_pass_re: ?[]const u8 = null, sharding_enabled: bool = false, sharding_axes: stdx.BoundedArray([*:0]const u8, 8) = .{}, + device_memory_size: u64 = 0, }; pub const Platform = struct { @@ -29,7 +34,7 @@ pub const Platform = struct { // `const comp = platform.compiler(compile_opts); const exe = comp.compile(...);` compilation_options: CompilationOptions = .{}, - pub const MAX_NUM_DEVICES: u8 = 32; + pub const MAX_NUM_DEVICES: u8 = if (runtimes.isEnabled(.tpu)) 32 else 8; pub const CreateOptions = _CreateOptions; pub fn init(target: Target, api: *const pjrt.Api, options: CreateOptions) !Platform { @@ -79,6 +84,44 @@ pub const Platform = struct { pub fn deinit(self: *Platform) void { self.pjrt_client.deinit(self.pjrt_api); } + + pub fn memoryForDevice(platform: Platform, memory: pjrt.Memory.Kind, device: *const pjrt.Device) *const pjrt.Memory { + const memory_target: pjrt.Memory.Kind = switch (memory) { + .host_unpinned => switch (platform.target) { + // Cuda doesn't have host_unpinned. + .cuda => .host_pinned, + else => .host_unpinned, + }, + inline else => |t| t, + }; + // TODO measure the cost of this and consider caching. + const device_memories = device.addressableMemories(platform.pjrt_api); + for (device_memories) |m| { + if (memory_target == m.kind(platform.pjrt_api)) { + return m; + } + } + log.err("Platform {t} doesn't have memory {t}", .{ platform.target, memory }); + @panic("Memory kind not found"); + } + + test memoryForDevice { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const memory_fields = @typeInfo(pjrt.Memory.Kind).@"enum".fields; + inline for (memory_fields) |field| { + for (platform.getDevices()) |dev| { + _ = platform.memoryForDevice(@field(pjrt.Memory.Kind, field.name), dev); + } + } + } + + pub fn memoryStats(platform: Platform, device_id: usize) pjrt.MemoryStats { + if (platform.target == .cpu) return .zeroes; + + const device = platform.getDevices()[device_id]; + return device.memoryStats(platform.pjrt_api) catch .zeroes; + } }; const _CreateOptions = struct { @@ -127,17 +170,21 @@ const _CreateOptions = struct { fn writeNamedValues(self: Cuda, values: *std.ArrayList(pjrt.NamedValue)) void { switch (self.allocator) { .platform => { - values.appendAssumeCapacity(pjrt.NamedValue.fromString("allocator", "platform")); + values.appendAssumeCapacity(.fromString("allocator", "platform")); }, .bfc, .async => |opt| { - values.appendAssumeCapacity(pjrt.NamedValue.from("allocator", self.allocator)); - values.appendAssumeCapacity(pjrt.NamedValue.from("preallocate", opt.preallocate)); + values.appendAssumeCapacity(.fromString("allocator", switch (self.allocator) { + .bfc => "bfc", + .async => "cuda_async", + .platform => unreachable, + })); + values.appendAssumeCapacity(.from("preallocate", opt.preallocate)); if (opt.memory_fraction > 0) { - values.appendAssumeCapacity(pjrt.NamedValue.from("memory_fraction", opt.memory_fraction)); + values.appendAssumeCapacity(.from("memory_fraction", opt.memory_fraction)); } if (opt.collective_memory_size_mb > 0) { const collective = @as(i64, opt.collective_memory_size_mb) * 1024 * 1024; - values.appendAssumeCapacity(pjrt.NamedValue.from("collective_memory_size", collective)); + values.appendAssumeCapacity(.from("collective_memory_size", collective)); } }, } diff --git a/zml/tensor.zig b/zml/tensor.zig index 50b8c561..ff12d91d 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -202,10 +202,8 @@ pub const Tensor = struct { const mlir_ctx = ctx.mlirCtx(); if (ctx.target() == .cpu) return self; - const memory_kind = @tagName(kind.toPjrtMemory()); - const frontend_attributes = mlir.Attribute.dict(mlir_ctx, &.{ - .{ "_xla_buffer_placement", .string(mlir_ctx, memory_kind) }, + .{ "_xla_buffer_placement", .string(mlir_ctx, kind.pjrtName()) }, }); const op = dialect.stablehlo.custom_call(mlir_ctx, &.{self.value()}, .{ @@ -311,6 +309,11 @@ pub const Tensor = struct { return _result(res_shape, op.result(0)); } + /// Returns the given tensor as one contiguous buffer of bytes. + pub fn bytes(self: Tensor) Tensor { + return self.bitCast(.u8).flattenAll().withTags(.{.bytes}); + } + /// Returns a Tensor containing the element-wise number of leading 0 bits in the input Tensor. pub fn countLeadingZeros(self: Tensor) Tensor { const loc = self.getContext().mlirCtx().location(@src()); @@ -2683,7 +2686,7 @@ pub const Tensor = struct { } { // Test with actual values and batching along axis .a - const operand = try zml.Buffer.constant(platform, Shape.init(.{ .a = 2, .b = 3, .c = 4, .d = 2 }, .u16), 0); + const operand = try zml.Buffer.constant(platform, Shape.init(.{ .a = 2, .b = 3, .c = 4, .d = 2 }, .u16), 0, .{}); defer operand.deinit(); const start_indices = (try zml.Buffer.fromArray( platform, @@ -2704,6 +2707,7 @@ pub const Tensor = struct { platform, Shape.init(.{ .n = 2, .a = 2, .m = 3, .c = 2, .d = 2 }, .u16), 1, + .{}, ); defer values.deinit(); From 21cd3ed532e02f4ea3c857df824e9d23b6636b77 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 22 Sep 2025 09:11:05 +0200 Subject: [PATCH 033/881] mlir,pjrt,zml: expose all missing dtype (#328) Adding the following missing data types: - u2 - i1 - i2 - f4e2m1fn - f8e3m4 - f8e4m3 - f8e8m0fnu (ground truth is PJRT) I've also realized the code in from/to Float32 wasn't correct and was truncating to zero to often. Hence the rewrite and bug fix. --- mlir/mlir.zig | 21 +++- pjrt/pjrt.zig | 6 + zml/dtype.zig | 45 +++++-- zml/floats.zig | 312 ++++++++++++++++++++++++++++++++++++------------ zml/mlirx.zig | 15 ++- zml/tensor.zig | 44 ++++++- zml/testing.zig | 6 +- 7 files changed, 355 insertions(+), 94 deletions(-) diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 15dbdab8..a3b9114c 100755 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -414,9 +414,9 @@ pub const ArrayAttribute = struct { pub fn IntegerAttribute(comptime it: IntegerTypes) type { const ZigType, const getter = comptime switch (it) { - .i1, .i4, .i8, .i16, .i32, .i64 => .{ i64, c.mlirIntegerAttrGetValueInt }, + .i1, .i2, .i4, .i8, .i16, .i32, .i64 => .{ i64, c.mlirIntegerAttrGetValueInt }, .si4, .si8, .si16, .si32, .si64 => .{ i64, c.mlirIntegerAttrGetValueSInt }, - .u4, .u8, .u16, .u32, .u64 => .{ u64, c.mlirIntegerAttrGetValueUInt }, + .u2, .u4, .u8, .u16, .u32, .u64 => .{ u64, c.mlirIntegerAttrGetValueUInt }, .unknown => @compileError("IntegerAttribute(unknown)"), }; @@ -1249,6 +1249,7 @@ pub const IndexType = struct { pub const IntegerTypes = enum { i1, + i2, i4, i8, i16, @@ -1259,6 +1260,7 @@ pub const IntegerTypes = enum { si16, si32, si64, + u2, u4, u8, u16, @@ -1271,6 +1273,7 @@ pub const IntegerTypes = enum { pub fn IntegerType(comptime it: IntegerTypes) type { const Config = switch (it) { .i1 => .{ 1, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, + .i2 => .{ 2, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, .i4 => .{ 4, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, .i8 => .{ 8, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, .i16 => .{ 16, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, @@ -1281,6 +1284,7 @@ pub fn IntegerType(comptime it: IntegerTypes) type { .si16 => .{ 16, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, .si32 => .{ 32, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, .si64 => .{ 64, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, + .u2 => .{ 2, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, .u4 => .{ 4, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, .u8 => .{ 8, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, .u16 => .{ 16, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, @@ -1320,11 +1324,15 @@ pub fn IntegerType(comptime it: IntegerTypes) type { } pub const FloatTypes = enum { + f4e2m1fn, + f8e3m4, + f8e4m3, f8e4m3b11fnuz, f8e4m3fn, f8e4m3fnuz, f8e5m2, f8e5m2fnuz, + f8e8m0fnu, bf16, f16, f32, @@ -1338,12 +1346,19 @@ pub const FloatTypes = enum { }; pub fn FloatType(comptime ft: FloatTypes) type { - const Config = switch (ft) { + const Config: struct { + *const fn (c.MlirType) callconv(.c) bool, + *const fn (c.MlirContext) callconv(.c) c.MlirType, + } = switch (ft) { + .f4e2m1fn => .{ c.mlirTypeIsAFloat4E2M1FN, c.mlirFloat4E2M1FNTypeGet }, + .f8e3m4 => .{ c.mlirTypeIsAFloat8E3M4, c.mlirFloat8E3M4TypeGet }, + .f8e4m3 => .{ c.mlirTypeIsAFloat8E4M3, c.mlirFloat8E4M3TypeGet }, .f8e4m3b11fnuz => .{ c.mlirTypeIsAFloat8E4M3B11FNUZ, c.mlirFloat8E4M3B11FNUZTypeGet }, .f8e4m3fn => .{ c.mlirTypeIsAFloat8E4M3FN, c.mlirFloat8E4M3FNTypeGet }, .f8e4m3fnuz => .{ c.mlirTypeIsAFloat8E4M3FNUZ, c.mlirFloat8E4M3FNUZTypeGet }, .f8e5m2 => .{ c.mlirTypeIsAFloat8E5M2, c.mlirFloat8E5M2TypeGet }, .f8e5m2fnuz => .{ c.mlirTypeIsAFloat8E5M2FNUZ, c.mlirFloat8E5M2FNUZTypeGet }, + .f8e8m0fnu => .{ c.mlirTypeIsAFloat8E8M0FNU, c.mlirFloat8E8M0FNUTypeGet }, .bf16 => .{ c.mlirTypeIsABF16, c.mlirBF16TypeGet }, .f16 => .{ c.mlirTypeIsAF16, c.mlirF16TypeGet }, .f32 => .{ c.mlirTypeIsAF32, c.mlirF32TypeGet }, diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index a3ef83da..cad1ce69 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -800,11 +800,13 @@ pub const LoadedExecutable = opaque { pub const BufferType = enum(c.PJRT_Buffer_Type) { invalid = c.PJRT_Buffer_Type_INVALID, bool = c.PJRT_Buffer_Type_PRED, + i2 = c.PJRT_Buffer_Type_S2, i4 = c.PJRT_Buffer_Type_S4, i8 = c.PJRT_Buffer_Type_S8, i16 = c.PJRT_Buffer_Type_S16, i32 = c.PJRT_Buffer_Type_S32, i64 = c.PJRT_Buffer_Type_S64, + u2 = c.PJRT_Buffer_Type_U2, u4 = c.PJRT_Buffer_Type_U4, u8 = c.PJRT_Buffer_Type_U8, u16 = c.PJRT_Buffer_Type_U16, @@ -821,6 +823,10 @@ pub const BufferType = enum(c.PJRT_Buffer_Type) { f8e4m3b11fnuz = c.PJRT_Buffer_Type_F8E4M3B11FNUZ, f8e5m2fnuz = c.PJRT_Buffer_Type_F8E5M2FNUZ, f8e4m3fnuz = c.PJRT_Buffer_Type_F8E4M3FNUZ, + f8e4m3 = c.PJRT_Buffer_Type_F8E4M3, + f8e3m4 = c.PJRT_Buffer_Type_F8E3M4, + f8e8m0 = c.PJRT_Buffer_Type_F8E8M0FNU, + f4e2m1 = c.PJRT_Buffer_Type_F4E2M1FN, }; pub const MemoryLayoutType = enum(c.PJRT_Buffer_MemoryLayout_Type) { diff --git a/zml/dtype.zig b/zml/dtype.zig index b0deb8f4..b14f483e 100644 --- a/zml/dtype.zig +++ b/zml/dtype.zig @@ -12,20 +12,26 @@ test { pub const DataType = enum(u8) { bool, // Note: the support of the float8 is a bit spotty, f8e4m3b11fnuz seems to be the most supported one on Cuda. + f4e2m1, + f8e3m4, + f8e4m3, f8e4m3b11fnuz, f8e4m3fn, f8e4m3fnuz, f8e5m2, f8e5m2fnuz, + f8e8m0, bf16, f16, f32, f64, + i2, i4, i8, i16, i32, i64, + u2, u4, u8, u16, @@ -50,8 +56,21 @@ pub const DataType = enum(u8) { pub fn class(self: DataType) Class { return switch (self) { .bool => .bool, - .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16, .f16, .f32, .f64 => .float, - .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => .integer, + .f4e2m1, + .f8e3m4, + .f8e4m3, + .f8e4m3b11fnuz, + .f8e4m3fn, + .f8e4m3fnuz, + .f8e5m2, + .f8e5m2fnuz, + .f8e8m0, + .bf16, + .f16, + .f32, + .f64, + => .float, + .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => .integer, .c64, .c128 => .complex, }; } @@ -70,21 +89,27 @@ pub const DataType = enum(u8) { pub fn fromZigType(comptime T: type) DataType { return switch (T) { + floats.Float4E2M1 => .f4e2m1, + floats.Float8E3M4 => .f8e3m4, + floats.Float8E4M3 => .f8e4m3, floats.Float8E4M3B11FNUZ => .f8e4m3b11fnuz, floats.Float8E4M3FN => .f8e4m3fn, floats.Float8E4M3FNUZ => .f8e4m3fnuz, floats.Float8E5M2 => .f8e5m2, floats.Float8E5M2FNUZ => .f8e5m2fnuz, + floats.Float8E8M0 => .f8e8m0, floats.BFloat16 => .bf16, f16 => .f16, f32 => .f32, f64 => .f64, bool => .bool, + i2 => .i2, i4 => .i4, i8 => .i8, i16 => .i16, i32 => .i32, i64 => .i64, + u2 => .u2, u4 => .u4, u8 => .u8, u16 => .u16, @@ -192,10 +217,10 @@ pub const DataType = enum(u8) { pub fn maxValue(dtype: DataType) Data { return switch (dtype) { .bool => .{ .bool = true }, - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), - inline .f8e5m2, .bf16 => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).inf), + inline .f4e2m1, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz, .f8e8m0 => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), + inline .f8e3m4, .f8e4m3, .f8e5m2, .bf16 => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).inf), inline .f16, .f32, .f64 => |tag| @unionInit(Data, @tagName(tag), std.math.inf(@FieldType(Data, @tagName(tag)))), - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Data, @tagName(tag), std.math.maxInt(@FieldType(Data, @tagName(tag)))), + inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Data, @tagName(tag), std.math.maxInt(@FieldType(Data, @tagName(tag)))), inline .c64, .c128 => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), }; } @@ -207,20 +232,26 @@ pub const DataType = enum(u8) { pub const Data = union(DataType) { bool: bool, + f4e2m1: floats.Float4E2M1, + f8e3m4: floats.Float8E3M4, + f8e4m3: floats.Float8E4M3, f8e4m3b11fnuz: floats.Float8E4M3B11FNUZ, f8e4m3fn: floats.Float8E4M3FN, f8e4m3fnuz: floats.Float8E4M3FNUZ, f8e5m2: floats.Float8E5M2, f8e5m2fnuz: floats.Float8E5M2FNUZ, + f8e8m0: floats.Float8E8M0, bf16: floats.BFloat16, f16: f16, f32: f32, f64: f64, + i2: i2, i4: i4, i8: i8, i16: i16, i32: i32, i64: i64, + u2: u2, u4: u4, u8: u8, u16: u16, @@ -244,7 +275,7 @@ pub const Data = union(DataType) { .comptime_int, .int, .comptime_float, .float => .{ .bool = value != 0 }, else => @panic("Could not create Data of type bool from value of type " ++ @typeName(T)), }, - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16 => |tag| switch (Ti) { + inline .f4e2m1, .f8e3m4, .f8e4m3, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .f8e8m0, .bf16 => |tag| switch (Ti) { .comptime_int, .int => @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).fromF32(@floatFromInt(value))), .comptime_float, .float => @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).fromF32(@floatCast(value))), else => @panic("Could not create Data of type bf16 from value of type " ++ @typeName(T)), @@ -254,7 +285,7 @@ pub const Data = union(DataType) { .comptime_float, .float => @unionInit(Data, @tagName(tag), @floatCast(value)), else => @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), }, - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| switch (Ti) { + inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| switch (Ti) { .comptime_int => blk: { const OutT = @FieldType(Data, @tagName(tag)); if (value >= std.math.minInt(OutT) and value <= std.math.maxInt(OutT)) { diff --git a/zml/floats.zig b/zml/floats.zig index 57bf5150..7d995c44 100644 --- a/zml/floats.zig +++ b/zml/floats.zig @@ -5,10 +5,6 @@ test { std.testing.refAllDecls(@This()); } -fn allBitsOne(v: anytype) bool { - return v == std.math.maxInt(@TypeOf(v)); -} - fn FloatHelpers(Float: type) type { const info = @typeInfo(Float); const err_msg = "FloatHelpers expect a packed struct { mantissa: uXX, exponent: uXX, sign: u1}"; @@ -23,9 +19,11 @@ fn FloatHelpers(Float: type) type { } return struct { - const sign_bits: u8 = @typeInfo(@FieldType(Float, "sign")).int.bits; const mantissa_bits: u8 = @typeInfo(@FieldType(Float, "mantissa")).int.bits; const exponent_bits: u8 = @typeInfo(@FieldType(Float, "exponent")).int.bits; + const f32_mantissa_bits: u8 = @typeInfo(@FieldType(Float32, "mantissa")).int.bits; + const exp_bias: i16 = std.math.maxInt(std.meta.Int(.unsigned, exponent_bits - 1)); + const exp_off: u8 = FloatHelpers(Float32).exp_bias - exp_bias; pub const zero: Float = .{ .sign = 0, .exponent = 0, .mantissa = 0 }; @@ -39,52 +37,74 @@ fn FloatHelpers(Float: type) type { /// Lossy conversion from f32, similar to @floatCast pub fn fromF32(f: f32) Float { + @setRuntimeSafety(false); + const vf32: Float32 = @bitCast(f); - const exp_bias = comptime expBias(); - const exponent = @as(u16, vf32.exponent) + exp_bias -| FloatHelpers(Float32).expBias(); + const exponent: i16 = @as(i16, vf32.exponent) - exp_off; const overflow = exponent > std.math.maxInt(@FieldType(Float, "exponent")); if (overflow) { - return if (@hasDecl(Float, "inf")) { - return if (vf32.sign == 0) Float.inf else Float.minus_inf; - } else Float.nan; + @branchHint(.unlikely); + return if (@hasDecl(Float, "inf")) + if (vf32.sign == 0) Float.inf else Float.minus_inf + else + Float.nan; } - return .{ - .sign = vf32.sign, - .exponent = @intCast(exponent), - .mantissa = truncMantissa(vf32.mantissa), - }; + + return if (exponent <= 0) + .{ + .sign = vf32.sign, + .exponent = 0, + .mantissa = shiftMantissa(vf32.mantissa, @intCast(-exponent)), + } + else + .{ + .sign = vf32.sign, + .exponent = @intCast(exponent), + .mantissa = truncMantissa(vf32.mantissa), + }; } /// Lossless conversion to f32. pub fn toF32(x: Float) f32 { - var vf32: Float32 = undefined; - if (@hasDecl(Float, "isInf") and x.isInf()) { + @setRuntimeSafety(false); + + if (x == zero) return 0.0; + if (isInf(x)) { + @branchHint(.unlikely); return if (x.sign == 0) std.math.inf(f32) else -std.math.inf(f32); } - vf32 = .{ - .sign = x.sign, - .exponent = if (x.exponent == 0) 0 else @intCast(@as(i16, x.exponent) + f32_exp_bias - expBias()), - .mantissa = f32Mantissa(x), - }; + + const vf32: Float32 = if (x.exponent > 0) + .{ + .sign = x.sign, + .exponent = @as(u8, x.exponent) + exp_off, + .mantissa = f32Mantissa(x), + } + else + .{ + .sign = x.sign, + .exponent = exp_off - @clz(x.mantissa), + .mantissa = @as(u23, x.mantissa) << @clz(x.mantissa), + }; return @bitCast(vf32); } - fn truncMantissa(x: anytype) @FieldType(Float, "mantissa") { - @setRuntimeSafety(false); - const off = @bitSizeOf(@TypeOf(x)) - mantissa_bits; - return @intCast(x >> off); + fn truncMantissa(f32_mantissa: u32) @FieldType(Float, "mantissa") { + const rounding_val: u32 = @as(u32, 1) << (f32_mantissa_bits - mantissa_bits - 1); + return @truncate((f32_mantissa + rounding_val) >> (f32_mantissa_bits - mantissa_bits)); + } + + fn shiftMantissa(f32_mantissa: u32, underflow: u8) @FieldType(Float, "mantissa") { + const upper_bit: u32 = @as(u32, 1) << f32_mantissa_bits; + const full_mant32: u32 = f32_mantissa | upper_bit; + // divide the mantissa proportionally to the exponent underflow + const shifted_mant: u32 = full_mant32 >> @truncate(underflow + 1); + return truncMantissa(shifted_mant); } fn f32Mantissa(x: Float) @FieldType(Float32, "mantissa") { - @setRuntimeSafety(false); - const Res = @FieldType(Float32, "mantissa"); - const f32_mantissa_bits = @bitSizeOf(Res); - - return @shlExact(@as(Res, x.mantissa), f32_mantissa_bits - mantissa_bits); - } - - fn expBias() u8 { - return std.math.maxInt(std.meta.Int(.unsigned, exponent_bits - 1)); + const T = @FieldType(Float32, "mantissa"); + return @as(T, x.mantissa) << f32_mantissa_bits - mantissa_bits; } pub fn formatNumber(x: Float, writer: *std.io.Writer, n: std.fmt.Number) std.io.Writer.Error!void { @@ -101,6 +121,9 @@ pub const Float32 = packed struct(u32) { exponent: u8, sign: u1, + pub const inf: Float32 = .{ .sign = 0, .exponent = std.math.maxInt(u8), .mantissa = 0 }; + pub const minus_inf = neg(inf); + const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; pub const neg = Helpers.neg; @@ -129,11 +152,7 @@ pub const Float8E4M3B11FNUZ = packed struct(u8) { exponent: u4, sign: u1, - pub const nan: Float8E4M3B11FNUZ = .{ - .sign = 1, - .exponent = 0, - .mantissa = 0, - }; + pub const nan: Float8E4M3B11FNUZ = .{ .sign = 1, .exponent = 0, .mantissa = 0 }; pub fn isNan(self: Float8E4M3B11FNUZ) bool { return self.sign == 1 and self.exponent == 0 and self.mantissa == 0; @@ -155,7 +174,7 @@ pub const Float8E4M3FN = packed struct(u8) { pub const nan: Float8E4M3FN = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = std.math.maxInt(u3) }; pub fn isNan(self: Float8E4M3FN) bool { - return allBitsOne(self.exponent) and allBitsOne(self.mantissa); + return self.exponent == nan.exponent and self.mantissa == nan.mantissa; } const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; @@ -170,11 +189,7 @@ pub const Float8E4M3FNUZ = packed struct(u8) { exponent: u4, sign: u1, - pub const nan: Float8E4M3FNUZ = .{ - .sign = 1, - .exponent = 0, - .mantissa = 0, - }; + pub const nan: Float8E4M3FNUZ = .{ .sign = 1, .exponent = 0, .mantissa = 0 }; pub fn isNan(self: Float8E4M3FNUZ) bool { return self.sign == 1 and self.exponent == 0 and self.mantissa == 0; @@ -189,9 +204,10 @@ pub const Float8E4M3FNUZ = packed struct(u8) { }; test "Float8E4" { + // With 4 bits of exponents power of two can be represented exactly up to 64. const test_case_e4: TestCase = .{ - .lossless = &[_]f32{ 0, 1.0, -2, 1.0 / 64.0, -128 }, - .lossy = &[_]f32{3.02344107628}, + .lossless = &[_]f32{ 0, 1.0, -2, 1.0 / 64.0, -128, -1.125 / 64.0 }, + .lossy = &[_]f32{ 3.02344107628, 1.0 / 128.0, 1.0 / 512.0 }, }; inline for (.{ @@ -200,7 +216,11 @@ test "Float8E4" { Float8E4M3FNUZ, }) |Float8T| { try testCustomFloat(Float8T, test_case_e4); - try std.testing.expectEqual(0.0, Float8T.fromF32(1.0 / 128.0).toF32()); + try std.testing.expectEqual(0.0, Float8T.fromF32(1.0 / 2048.0).toF32()); + if (@hasDecl(Float8T, "inf")) { + try std.testing.expectEqual(Float8T.inf, Float8T.fromF32(128.0)); + try std.testing.expectEqual(Float8T.inf.neg(), Float8T.fromF32(-128.0)); + } } } @@ -209,31 +229,19 @@ pub const Float8E5M2 = packed struct(u8) { exponent: u5, sign: u1, - pub const nan: Float8E5M2 = .{ - .sign = 0, - .exponent = std.math.maxInt(u5), - .mantissa = 1, - }; + pub const nan: Float8E5M2 = .{ .sign = 0, .exponent = std.math.maxInt(u5), .mantissa = 1 }; pub fn isNan(self: Float8E5M2) bool { - return allBitsOne(self.exponent) and self.mantissa != 0; + return self.exponent == nan.exponent and self.mantissa != 0; } - pub const minus_inf: Float8E5M2 = .{ - .sign = 1, - .exponent = std.math.maxInt(u5), - .mantissa = 0, - }; - pub const inf: Float8E5M2 = .{ .sign = 0, .exponent = std.math.maxInt(u5), .mantissa = 0, }; - pub fn isInf(self: Float8E5M2) bool { - return allBitsOne(self.exponent) and self.mantissa == 0; - } + pub const minus_inf: Float8E5M2 = .neg(inf); const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; @@ -283,22 +291,16 @@ pub const BFloat16 = packed struct(u16) { return allBitsOne(self.exponent) and self.mantissa != 0; } - pub const minus_inf: BFloat16 = .{ - .sign = 1, - .exponent = std.math.maxInt(u8), - .mantissa = 0, - }; - pub const inf: BFloat16 = .{ .sign = 0, .exponent = std.math.maxInt(u8), .mantissa = 0, }; - pub fn isInf(self: BFloat16) bool { - return allBitsOne(self.exponent) and self.mantissa == 0; - } + pub const minus_inf: BFloat16 = .neg(inf); + // Specialized versions of to/from F32. Since BFloat16 has the same exponent range than F32, + // no overflow/underflow can happen, simplifiying conversion logic. pub fn toF32(self: BFloat16) f32 { // Pad the BF16 with zeros 0 return @bitCast([2]u16{ 0, @bitCast(self) }); @@ -333,11 +335,167 @@ test BFloat16 { }); } -pub fn floatCast(T: type, x: anytype) T { - return switch (@TypeOf(x)) { - f64, f32, f16 => @floatCast(x), - else => @floatCast(x.toF32()), +pub const Float8E4M3 = packed struct(u8) { + mantissa: u3, + exponent: u4, + sign: u1, + + pub const nan: Float8E4M3 = @bitCast(0xFF); + + pub fn isNan(self: Float8E4M3) bool { + return self == nan or self == comptime nan.neg(); + } + + pub const inf: Float8E4M3 = .{ + .sign = 0, + .exponent = std.math.maxInt(u4), + .mantissa = 0, }; + + pub const minus_inf = neg(inf); + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const toF32 = Helpers.toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +pub const Float8E3M4 = packed struct(u8) { + mantissa: u4, + exponent: u3, + sign: u1, + + pub const nan: Float8E3M4 = @bitCast(0xFF); + + pub fn isNan(self: Float8E3M4) bool { + return self == nan or self == comptime nan.neg(); + } + + pub const inf: Float8E3M4 = .{ + .sign = 0, + .exponent = std.math.maxInt(u3), + .mantissa = 0, + }; + + pub const minus_inf = neg(inf); + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const toF32 = Helpers.toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +pub const Float8E8M0 = packed struct(u8) { + mantissa: u0 = 0, + exponent: u8, + sign: u0 = 0, + + pub const min_scale: f32 = @bitCast(Float32{ .sign = 0, .exponent = 0, .mantissa = 0b1 << 22 }); + + /// Lossy conversion from f32, similar to @floatCast + pub fn fromF32(f: f32) Float8E8M0 { + const vf32: Float32 = @bitCast(f); + return .{ .exponent = @intCast(vf32.exponent) }; + } + + /// Lossless conversion to f32. + pub fn toF32(x: Float8E8M0) f32 { + if (x.exponent == 0) return min_scale; + const vf32: Float32 = .{ + .sign = 0, + .exponent = x.exponent, + .mantissa = 0, + }; + return @bitCast(vf32); + } + + const Helpers = FloatHelpers(@This()); + pub const formatNumber = Helpers.formatNumber; +}; + +test Float8E8M0 { + try std.testing.expectEqual(Float8E8M0{ .exponent = 127 }, Float8E8M0.fromF32(1.0)); + // try std.testing.expectEqual(5.877472e-39, Float8E8M0.toF32(.{ .exponent = 0})); + + try testCustomFloat(Float8E8M0, .{ + .lossless = &[_]f32{ Float8E8M0.min_scale, 1.0, 64.0, 1.0 / 128.0, std.math.pow(f32, 2.0, 127) }, + .lossy = &[_]f32{1.00001}, + }); +} + +pub const Float4E2M1 = packed struct(u4) { + mantissa: u1, + exponent: u2, + sign: u1, + + pub const nan: Float4E2M1 = @bitCast(@as(u4, 0xF)); + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const formatNumber = Helpers.formatNumber; + + pub const values = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; + + pub fn toF32(x: Float4E2M1) f32 { + // the baseline toF32 doesn't work correctly: + // 0b0001 and 0b1001 shoud map to ±0.5, but are mapped to ±epsilon + return values[@as(u4, @bitCast(x))]; + } + + test toF32 { + var to_f32_res: [16]f32 = undefined; + for (&to_f32_res, 0..) |*r, i| { + const x_f4: Float4E2M1 = @bitCast(@as(u4, @intCast(i))); + r.* = x_f4.toF32(); + } + try std.testing.expectEqualSlices(f32, &Float4E2M1.values, &to_f32_res); + } + + test fromF32 { + // the baseline fromF32 doesn't work correctly: + // ±0.5 should map to 0b0001/0b1001 but are map to ±0.0 instead. + // TODO: it probably affects other types. + var from_f32_res: [16]Float4E2M1 = undefined; + for (&from_f32_res, 0..) |*r, i| { + r.* = .fromF32(Float4E2M1.values[i]); + } + try std.testing.expectEqualSlices(u4, &.{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, @ptrCast(&from_f32_res)); + } +}; + +pub fn floatCast(T: type, x: anytype) T { + return switch (T) { + f64, f32, f16 => switch (@TypeOf(x)) { + f64, f32, f16 => @floatCast(x), + else => @floatCast(x.toF32()), + }, + else => switch (@TypeOf(x)) { + f64, f32, f16 => .fromF32(x), + else => .fromF32(x.toF32()), + }, + }; +} + +pub fn isInf(x: anytype) bool { + const Float = @TypeOf(x); + switch (Float) { + f64, f32, f16 => return std.math.isInf(x), + else => {}, + } + if (!@hasDecl(Float, "inf")) return false; + + const FBits = std.meta.Int(.unsigned, @bitSizeOf(Float)); + const remove_sign = ~@as(FBits, 0) >> 1; + return @as(FBits, @bitCast(x)) & remove_sign == @as(FBits, @bitCast(Float.inf)); +} + +fn allBitsOne(v: anytype) bool { + return v == std.math.maxInt(@TypeOf(v)); } const TestCase = struct { diff --git a/zml/mlirx.zig b/zml/mlirx.zig index f50a2efa..286166aa 100644 --- a/zml/mlirx.zig +++ b/zml/mlirx.zig @@ -35,20 +35,26 @@ pub const Type = struct { pub fn fromDType(ctx: mlir.Context, dt: dtype.DataType) mlir.Type { return switch (dt) { .bool => .int(ctx, .i1), + .f4e2m1 => .float(ctx, .f4e2m1fn), + .f8e3m4 => .float(ctx, .f8e3m4), + .f8e4m3 => .float(ctx, .f8e4m3), .f8e4m3b11fnuz => .float(ctx, .f8e4m3b11fnuz), .f8e4m3fn => .float(ctx, .f8e4m3fn), .f8e4m3fnuz => .float(ctx, .f8e4m3fnuz), .f8e5m2 => .float(ctx, .f8e5m2), .f8e5m2fnuz => .float(ctx, .f8e5m2fnuz), + .f8e8m0 => .float(ctx, .f8e8m0fnu), .bf16 => .float(ctx, .bf16), .f16 => .float(ctx, .f16), .f32 => .float(ctx, .f32), .f64 => .float(ctx, .f64), + .i2 => .int(ctx, .i2), .i4 => .int(ctx, .i4), .i8 => .int(ctx, .i8), .i16 => .int(ctx, .i16), .i32 => .int(ctx, .i32), .i64 => .int(ctx, .i64), + .u2 => .int(ctx, .u2), .u4 => .int(ctx, .u4), .u8 => .int(ctx, .u8), .u16 => .int(ctx, .u16), @@ -62,23 +68,28 @@ pub const Type = struct { pub fn toDType(mlir_type: mlir.Type) dtype.DataType { const mapping = .{ .{ .bool, mlir.IntegerType(.i1) }, - + .{ .f4e2m1, mlir.FloatType(.f4e2m1fn) }, + .{ .f8e3m4, mlir.FloatType(.f8e3m4) }, + .{ .f8e4m3, mlir.FloatType(.f8e4m3) }, .{ .f8e4m3b11fnuz, mlir.FloatType(.f8e4m3b11fnuz) }, .{ .f8e4m3fn, mlir.FloatType(.f8e4m3fn) }, .{ .f8e4m3fnuz, mlir.FloatType(.f8e4m3fnuz) }, .{ .f8e5m2, mlir.FloatType(.f8e5m2) }, .{ .f8e5m2fnuz, mlir.FloatType(.f8e5m2fnuz) }, + .{ .f8e8m0, mlir.FloatType(.f8e8m0fnu) }, .{ .bf16, mlir.FloatType(.bf16) }, .{ .f16, mlir.FloatType(.f16) }, .{ .f32, mlir.FloatType(.f32) }, .{ .f64, mlir.FloatType(.f64) }, + .{ .i2, mlir.IntegerType(.i2) }, .{ .i4, mlir.IntegerType(.i4) }, .{ .i8, mlir.IntegerType(.i8) }, .{ .i16, mlir.IntegerType(.i16) }, .{ .i32, mlir.IntegerType(.i32) }, .{ .i64, mlir.IntegerType(.i64) }, + .{ .u2, mlir.IntegerType(.u2) }, .{ .u4, mlir.IntegerType(.u4) }, .{ .u8, mlir.IntegerType(.u8) }, .{ .u16, mlir.IntegerType(.u16) }, @@ -89,6 +100,8 @@ pub const Type = struct { .{ .c128, mlir.ComplexType(.c128) }, }; + // TODO: this seems quite slow to have all of those functions calls. + // Maybe we should memoize the ptr of a set of mlir types when creating the context. inline for (mapping) |entry| { const dt, const mlirT = entry; if (mlirT.is_a_fn(mlir_type._inner)) { diff --git a/zml/tensor.zig b/zml/tensor.zig index ff12d91d..373d5efa 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1042,6 +1042,40 @@ pub const Tensor = struct { return _result(self._shape.withDtype(to), op.result(0)); } + test convert { + const floats = @import("floats.zig"); + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + // f4e2m1 + { + const x = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; + var x_f4: [x.len]floats.Float4E2M1 = undefined; + for (&x_f4, &x) |*xi_f4, xi| xi_f4.* = .fromF32(xi); + + const x_d = try zml.Buffer.fromArray(platform, x); + const x_f4_xla_d = try zml.testing.compileAndCall(platform, Tensor.convert, .{ x_d, .f4e2m1 }); + + const x_f4_xla = x_f4_xla_d.getValue(@TypeOf(x_f4)); + errdefer std.log.warn("convert(.f4e2m1) failed !\ninput f32:\n{e}\nzml.floats computed:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x), x_f4, x_f4_xla }); + try std.testing.expectEqualDeep(x_f4, x_f4_xla); + } + + // f8e3m4 + { + const x = [_]f32{ 1.1 / 4.0, 1.1 / 8.0, 1.1 / 16.0, 1.1 / 32.0, 1.1 / 64.0, 1.1 / 128.0 }; + var x_f8e3: [x.len]floats.Float8E3M4 = undefined; + for (&x_f8e3, &x) |*xi_f8e3, xi| xi_f8e3.* = .fromF32(xi); + + const x_d = try zml.Buffer.fromArray(platform, x); + const x_f8e3_xla_d = try zml.testing.compileAndCall(platform, Tensor.convert, .{ x_d, .f8e3m4 }); + + const x_f8e3_xla = x_f8e3_xla_d.getValue(@TypeOf(x_f8e3)); + errdefer std.log.warn("convert(.f8e3m4) failed !\ninput f32:\n{e}\nzml.floats computed:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x), x_f8e3, x_f8e3_xla }); + try std.testing.expectEqualDeep(x_f8e3, x_f8e3_xla); + } + } + /// Returns a Tensor containing the element-wise rounding operation of the input Tensor. pub fn round(self: Tensor) Tensor { const loc = self.getContext().mlirCtx().location(@src()); @@ -3844,11 +3878,11 @@ fn getPoolResDims(dt: DataType, in_dims: []const i64, base_dilations: @Vector(Te } fn getComparisonType(ctx: mlir.Context, dtype: DataType) dialect.stablehlo.CompareType { - return dialect.stablehlo.CompareType.init(ctx, switch (dtype) { - .i4, .i8, .i16, .i32, .i64 => .SIGNED, - .bool, .u4, .u8, .u16, .u32, .u64 => .UNSIGNED, - .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16, .f16, .f32, .f64 => .FLOAT, - .c64, .c128 => @panic("Can't compare complex numbers"), + return dialect.stablehlo.CompareType.init(ctx, switch (dtype.class()) { + .bool => .UNSIGNED, + .integer => if (dtype.isSignedInt()) .SIGNED else .UNSIGNED, + .float => .FLOAT, + .complex => @panic("Can't compare complex numbers"), }); } diff --git a/zml/testing.zig b/zml/testing.zig index c9ad9efa..851bd8c1 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -65,11 +65,15 @@ pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { .f16, .f32, .f64, + .f4e2m1, + .f8e3m4, + .f8e4m3, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, + .f8e8m0, => |t| { const L = t.toZigType(); const left_data = left.items(L); @@ -96,7 +100,7 @@ pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { else => unreachable, } }, - inline .bool, .u4, .u8, .u16, .u32, .u64, .i4, .i8, .i16, .i32, .i64 => |t| { + inline .bool, .u2, .u4, .u8, .u16, .u32, .u64, .i2, .i4, .i8, .i16, .i32, .i64 => |t| { const T = t.toZigType(); return std.testing.expectEqualSlices(T, left.items(T), right.items(T)); }, From 0312d6f6789c10ad0a4202e3bbaeaaab8ceed7be Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 25 Sep 2025 12:29:46 +0200 Subject: [PATCH 034/881] zml: gather (#330) Revamp the `gather` api to match `scatter` (prerequesite of #329 ) ## Gather new api Now the syntax is to give a named list of indices. Each name must correspond to one axis in the gathered tensor. This match `scatter`, and avoid the user to have to concatenate indices in the same order than the input axes, which is error prone cause any screw up will compile but index at the wrong offsets. For 1 axis it doesn't change much: ``` // before self.weight.withTags(.{ .voc, .d }).gather(.{.voc}, .{ idx }, .{}); // after self.weight.withTags(.{ .voc, .d }).gather(.{ .voc = idx }, .{}); ``` But for two it helps a lot: ``` // before self.weight.withTags(.{ .voc, .d }).gather(.{.voc, .d}, .{ .concatenate(&.{idx_voc, idx_d}, .last)}, .{}); // after self.weight.withTags(.{ .voc, .d }).gather(.{ .voc = idx_voc, .d = idx_d }, .{}); ``` Also adds a `gather_` that offers the same apis but with axis position instead of names. This one is more suitable inside library functions when dealing with unknown names, making it complicated to create the "indices" struct. I think we could generalize this pattern of offering "non-tagged" api with an underscore suffix, simplifying the job of LSP. ## topk new API I realized that most of the code calling topk was renaming the output axis. The idea is that logprob.topk(.voc) doesn't return you a tensor with voc entry but only top_voc entries. For the values it doesn't make much a difference but for the indices it's embarrassing and it break usages of gather/scatter. Now the user of `topK` can either use the unnamed api and pass a number, of if using the name api must give a new name for the indexed axis. ## Yarn Support Yarn rope embeddings. It's equivalent to the "llama3" embedding except they do the interpolation in time space instead of freq space. --- examples/mnist/mnist.zig | 2 +- examples/modernbert/modernbert.zig | 2 +- zml/buffer.zig | 32 +- zml/nn.zig | 496 ++++++------------------ zml/pjrtx.zig | 59 ++- zml/quantization.zig | 38 +- zml/shape.zig | 8 + zml/tensor.zig | 367 +++++++++++------- zml/testing.zig | 2 +- zml/tokenizer/BUILD.bazel | 7 +- zml/tokenizer/sentencepiece/BUILD.bazel | 11 + zml/tokenizer/tokenizer.zig | 7 +- zml/torch.zig | 17 +- 13 files changed, 446 insertions(+), 602 deletions(-) diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 632cafda..4130bf7a 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -27,7 +27,7 @@ const Mnist = struct { /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { // std.log.info("Compiling for target: {s}", .{@tagName(input.getContext().target())}); - var x = input.flattenAll().convert(.f32); + var x = input.flatten().convert(.f32); const layers: []const Layer = &.{ self.fc1, self.fc2 }; for (layers) |layer| { x = layer.forward(x); diff --git a/examples/modernbert/modernbert.zig b/examples/modernbert/modernbert.zig index 2e6eb4ab..1ba35578 100644 --- a/examples/modernbert/modernbert.zig +++ b/examples/modernbert/modernbert.zig @@ -43,7 +43,7 @@ pub const ModernBertForMaskedLM = struct { const biased_logits = logits.add(self.decoder.bias.withTags(.{.voc}).broad(logits.shape())); const probabilities = biased_logits.softmax(.voc); - return probabilities.topK(5, .voc, .{ .descending = true }); + return probabilities.topK(.{ .best_words = .voc }, 5, .{ .descending = true }); } }; diff --git a/zml/buffer.zig b/zml/buffer.zig index c9373e5f..a4a9b194 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -54,7 +54,7 @@ pub const Buffer = struct { break :cs @divExact(host_buffer.dim(ax), n_partitions); } else 0; - const buffer_type = bufferTypeFromDtype(host_buffer.shape().dtype()); + const buffer_type = pjrt.bufferTypeFromDtype(host_buffer.shape().dtype()); const byte_strides = host_buffer.strides(); const devices = platform.getDevices(); @@ -256,7 +256,7 @@ pub const Buffer = struct { pub fn asViewOfDeviceBuffer(platform: Platform, shape_: Shape, stream: ?*const pjrt.Stream, device_data: *anyopaque) Buffer { const pjrt_buffer = platform.pjrt_client.createViewOfDeviceBuffer(platform.pjrt_api, .{ .data = device_data, - .element_type = bufferTypeFromDtype(shape_.dtype()), + .element_type = pjrt.bufferTypeFromDtype(shape_.dtype()), .dims = shape_.dims(), // TODO: exposes sharding in the API. .device = platform.getDevices()[0], @@ -437,7 +437,7 @@ pub const Buffer = struct { var args = pjrt.Client.CreateUninitializedBufferArgs{ .dims = shard_shape.dims(), - .element_type = bufferTypeFromDtype(shape_.dtype()), + .element_type = pjrt.bufferTypeFromDtype(shape_.dtype()), .layout = .{ .tiled = .{ .minor_to_major = minorToMajor(shape_.rank()), @@ -487,32 +487,6 @@ pub const Buffer = struct { } }; -pub fn bufferTypeFromDtype(dt: DataType) pjrt.BufferType { - return switch (dt) { - inline else => |tag| @field(pjrt.BufferType, @tagName(tag)), - }; -} - -pub fn dtypeFromBufferType(pjrt_type: pjrt.BufferType) DataType { - return switch (pjrt_type) { - .invalid => @panic("Found an invalid pjrt buffer"), - inline else => |tag| @field(DataType, @tagName(tag)), - }; -} - -test bufferTypeFromDtype { - inline for (@typeInfo(DataType).@"enum".fields) |field| { - const dt: DataType = @enumFromInt(field.value); - try std.testing.expectEqual(dt, dtypeFromBufferType(bufferTypeFromDtype(dt))); - } - - inline for (@typeInfo(pjrt.BufferType).@"enum".fields) |field| { - const dt: pjrt.BufferType = @enumFromInt(field.value); - if (dt == .invalid) continue; - try std.testing.expectEqual(dt, bufferTypeFromDtype(dtypeFromBufferType(dt))); - } -} - const _MINOR_TO_MAJOR = blk: { var ret: [Shape.MAX_RANK]i64 = undefined; for (0..Shape.MAX_RANK) |i| { diff --git a/zml/nn.zig b/zml/nn.zig index 631bce16..4fbb17cd 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -32,7 +32,7 @@ pub const Linear = struct { } // log.debug("Linear({*}): {d} -> {d} -> {d}", .{ self, x.dims(), y.dims(), if (self.bias) |bias| y.add(bias).dims() else y.dims() }); - return if (self.bias) |bias| y.add(bias.broadcastLeft(y.shape())) else y; + return if (self.bias) |bias| y.add(bias.broadcast(y.shape(), &.{y.axis(-1)})) else y; } }; @@ -42,7 +42,7 @@ pub const TokenEmbedding = struct { pub fn forward(self: TokenEmbedding, idx: Tensor) Tensor { stdx.debug.assert(idx.dtype().isInteger(), "TokenEmbedding expects an integer input, received: {f}", .{idx}); stdx.debug.assert(self.weight.rank() == 2, "TokenEmbedding expects it's weight Tensor to be a 2D matrix, got {f}", .{self.weight}); - return self.weight.gatherValues(0, idx, .{}); + return self.weight.withTags(.{ .voc, .d }).gather(.{ .voc = idx }, .{}); } }; @@ -125,18 +125,18 @@ pub fn normalizeVariance(x: Tensor, eps: f32) Tensor { // Upcast to improve precision const xf32 = x.convert(.f32); const mean = xf32.sum(-1).scale(1.0 / N); - const mean_dev = xf32.sub(mean.broadcastRight(xf32.shape())); - const variance = mean_dev.mul(mean_dev).sum(-1).scale(1.0 / N); + const mean_dev = xf32.sub(mean); + const variance = mean_dev.mul(mean_dev).sum(-1).divByConst(N); const rsqrt = Tensor.rsqrt(variance.addConstant(eps)); - return mean_dev.mul(rsqrt.broadcastRight(mean_dev.shape())).convert(x.dtype()); + return mean_dev.mul(rsqrt).convert(x.dtype()); } // ref: https://pytorch.org/docs/stable/generated/torch.nn.functional.normalize.html // Implementation equivalent to `nn.functional.normalize(tensor, dim=-1)` call pub fn normalizeL2(input: Tensor, eps: f32) Tensor { - const inv_norm = input.pow(Tensor.scalar(2, input.dtype())).sum(-1).addConstant(eps).rsqrt(); - return input.mul(inv_norm.broad(input.shape())); + const inv_norm = input.powByConst(2).sum(-1).addConstant(eps).rsqrt(); + return input.mul(inv_norm); } test normalizeL2 { @@ -165,12 +165,22 @@ pub const RopeOpts = struct { default: void, custom: []const f32, llama3: Llama3, + yarn: Yarn, pub const Llama3 = struct { factor: f32, high_freq_factor: f32, low_freq_factor: f32, original_max_position_embeddings: u32, + truncate: bool = true, + }; + + pub const Yarn = struct { + beta_fast: f32 = 32.0, + beta_slow: f32 = 1.0, + factor: f32, + truncate: bool = true, + original_max_position_embeddings: u32, }; /// Read a Rope scaling config from HF config.json format. @@ -185,12 +195,21 @@ pub const RopeOpts = struct { if (impl != .string) return error.InvalidEnumTag; if (std.mem.eql(u8, impl.string, "llama3")) { // Note: leaky is fine here cause Llama3 struct don't need to allocate memory. - return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, undefined, content, .{ .ignore_unknown_fields = true }) }; + return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, stdx.noalloc, content, .{ .ignore_unknown_fields = true }) }; + } else if (std.mem.eql(u8, impl.string, "yarn")) { + return .{ .yarn = try std.json.parseFromValueLeaky(Yarn, stdx.noalloc, content, .{ .ignore_unknown_fields = true }) }; } else { log.warn("Unsupported Rope implementation: {s}, will use the default one which will produce altered results", .{impl.string}); return .{ .default = {} }; } } + + pub fn attentionScaling(scaling: Scaling) f32 { + return switch (scaling) { + .yarn => |yarn| 0.1 * @log(yarn.factor) + 1.0, + else => 1.0, + }; + } }; }; @@ -218,8 +237,9 @@ pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { // compute sin and cos in f32 before downcasting to x type. const inv_freq = invFreq(x.dim(.hd), opts).withTags(.{.hd}); const inv_freq_pos = Tensor.outer(idx.convert(.f32), inv_freq); - const cos = inv_freq_pos.cos().convert(x.dtype()).broad(x_real.shape()); - const sin = inv_freq_pos.sin().convert(x.dtype()).broad(x_real.shape()); + const scaling = opts.scaling.attentionScaling(); + const cos = inv_freq_pos.cos().scale(scaling).convert(x.dtype()).broad(x_real.shape()); + const sin = inv_freq_pos.sin().scale(scaling).convert(x.dtype()).broad(x_real.shape()); // apply rotation const y_real = x_real.mul(cos).sub(x_imag.mul(sin)); @@ -250,7 +270,7 @@ pub fn mergeRealImg(x_real: Tensor, x_imag: Tensor, layout: RopeOpts.Layout) Ten .interleaved => Tensor.concatenate(&.{ x_real.appendAxes(.{.interleaved_real_img}), x_imag.appendAxes(.{.interleaved_real_img}), - }, -1).flatten(-2), + }, -1).reshape(x_imag.shape().setDim(-1, -1)), }; } @@ -259,6 +279,8 @@ pub fn invFreq(N: i64, opts: RopeOpts) Tensor { const allocator = zml.module.CompilationContext.current().allocator(); const N_half: usize = @intCast(@divExact(N, 2)); const inv_freq = allocator.alloc(f32, N_half) catch @panic("OOM"); + defer allocator.free(inv_freq); + _invFreq(opts, inv_freq); return zml.Tensor.constantTensor(.fromSlice(.{@divExact(N, 2)}, inv_freq)); } @@ -299,10 +321,38 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { } } }, + .yarn => |s| { + const N_f: f64 = @floatFromInt(inv_freq.len); + const M: f64 = @floatFromInt(s.original_max_position_embeddings); + const f_high = s.beta_fast * (2 * std.math.pi) / M; + const f_low = s.beta_slow * (2 * std.math.pi) / M; + const downscaling = 1.0 / s.factor; + + // This isn't a typo: low n have a high frequency, high n have a low frequency. + var n_low: f64 = -@log(f_high) / @log(opts.freq_base) * N_f; + var n_high: f64 = -@log(f_low) / @log(opts.freq_base) * N_f; + if (s.truncate) { + n_high = std.math.ceil(n_high); + n_low = std.math.floor(n_low); + } + std.debug.assert(n_high > n_low); + for (0..N, inv_freq) |n, f| { + if (f > f_high) { + // High freq match default implem + } else if (f < f_low) { + // Downscaling for low freq + inv_freq[n] *= downscaling; + } else { + // Yarn use lerp too but not in the frequency space, in the time space. + const lerp: f64 = (n_high - @as(f64, @floatFromInt(n))) / (n_high - n_low); + inv_freq[n] *= @floatCast(lerp + (1 - lerp) * downscaling); + } + } + }, } } -test invFreq { +test "invFreq Llama3" { // Llama 3.2-1B config const llama_conf: RopeOpts = .{ .freq_base = 500_000, @@ -323,6 +373,28 @@ test invFreq { } } +test "invFreq Yarn" { + const yarn_conf: RopeOpts = .{ + .freq_base = 150_000, + .scaling = .{ .yarn = .{ + .factor = 32.0, + .beta_fast = 32.0, + .beta_slow = 1.0, + .original_max_position_embeddings = 4096, + .truncate = true, + } }, + }; + const yarn_freq = [_]f32{ 1.000000000000e+00, 6.890442967415e-01, 4.747820496559e-01, 3.271458745003e-01, 2.254180014133e-01, 1.553229838610e-01, 1.070244237781e-01, 7.374456524849e-02, 5.081327259541e-02, 3.162075206637e-02, 1.945096626878e-02, 1.179219130427e-02, 7.015713956207e-03, 4.069554619491e-03, 2.277272054926e-03, 1.206130953506e-03, 5.809474969283e-04, 2.279478358105e-04, 3.830881178146e-05, 2.639646845637e-05, 1.818833698053e-05, 1.253256959899e-05, 8.635495760245e-06, 5.950239483354e-06, 4.099978468730e-06, 2.825066758305e-06, 1.946596285052e-06, 1.341290953860e-06, 9.242089618056e-07, 6.368209142238e-07, 4.387978549403e-07, 3.023511396805e-07 }; + + var inv_freq: @TypeOf(yarn_freq) = undefined; + _invFreq(yarn_conf, &inv_freq); + for (yarn_freq, inv_freq, 0..) |expected, actual, i| { + errdefer log.err("Mismatch at position {d}.\nExpected: {d}\nActual: {d}", .{ i, stdx.fmt.slice(&yarn_freq), stdx.fmt.slice(&inv_freq) }); + try std.testing.expectApproxEqRel(expected, actual, 1e-5); + } + try std.testing.expectApproxEqRel(1.3465735902799727, yarn_conf.scaling.attentionScaling(), 1e-5); +} + test "real/img" { const platform = zml.testing.env(); @@ -332,8 +404,8 @@ test "real/img" { const real, const imag = splitRealImg(x, layout); const y = mergeRealImg(real, imag, layout); const real2, const imag2 = splitRealImg(y, layout); - return real.cmp(.EQ, real2).flatten(0).convert(.i32).sum(-1).add( - imag.cmp(.EQ, imag2).flatten(0).convert(.i32).sum(-1), + return real.cmp(.EQ, real2).flatten().convert(.i32).sum(-1).add( + imag.cmp(.EQ, imag2).flatten().convert(.i32).sum(-1), ); } @@ -349,8 +421,8 @@ test "real/img" { Tensor.arange(.{ .start = 3, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), }, 1); - return real.cmp(.EQ, x_real).flatten(0).convert(.i32).sum(-1).add( - imag.cmp(.EQ, x_imag).flatten(0).convert(.i32).sum(-1), + return real.cmp(.EQ, x_real).flatten().convert(.i32).sum(-1).add( + imag.cmp(.EQ, x_imag).flatten().convert(.i32).sum(-1), ); } @@ -366,8 +438,8 @@ test "real/img" { Tensor.arange(.{ .start = 3, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), }, 1); - return real.cmp(.EQ, x_real).flatten(0).convert(.i32).sum(-1).add( - imag.cmp(.EQ, x_imag).flatten(0).convert(.i32).sum(-1), + return real.cmp(.EQ, x_real).flatten().convert(.i32).sum(-1).add( + imag.cmp(.EQ, x_imag).flatten().convert(.i32).sum(-1), ); } @@ -377,8 +449,8 @@ test "real/img" { const x_real = Tensor.arange(.{ .start = 0, .end = 20, .step = 2 }, .f32).reshape(.{ 5, 2 }); const x_imag = Tensor.arange(.{ .start = 1, .end = 20, .step = 2 }, .f32).reshape(.{ 5, 2 }); - return real.cmp(.EQ, x_real).flatten(0).convert(.i32).sum(-1).add( - imag.cmp(.EQ, x_imag).flatten(0).convert(.i32).sum(-1), + return real.cmp(.EQ, x_real).flatten().convert(.i32).sum(-1).add( + imag.cmp(.EQ, x_imag).flatten().convert(.i32).sum(-1), ); } }; @@ -484,21 +556,21 @@ pub fn nearest(input: Tensor, scale_factor: []const f64) Tensor { out_shape._dims.set(i + 2, @intFromFloat(@floor(@as(f64, @floatFromInt(out_shape.dim(i + 2))) * sf))); } // TODO(james): remove this implicit two batching dims - var sd: [3]usize = undefined; + var sd: [3]u3 = undefined; var len_sd: usize = 0; for (2..input.rank()) |i| { if (input.dim(i) != out_shape.dim(i)) { - sd[len_sd] = i; + sd[len_sd] = @intCast(i); len_sd += 1; } } - const spatial_dims = sd[0..len_sd]; + const spatial_axes = sd[0..len_sd]; var res = input; - for (spatial_dims) |d| { - const n = out_shape.dim(d); - const ratio = stdx.math.divFloat(f32, input.dim(d), n); + for (spatial_axes) |ax| { + const n = out_shape.dim(ax); + const ratio = stdx.math.divFloat(f32, input.dim(ax), n); const offsets = Tensor.arange(.{ .end = n }, .f32).addConstant(0.5).scale(ratio).floor().convert(.i32); - res = res.gatherValues(d, offsets, .{ .indices_are_sorted = true }); + res = res.gather_(&.{ax}, &.{offsets}, .{ .indices_are_sorted = true }); } return res; } @@ -670,10 +742,11 @@ test resizeBilinear { } pub fn resizeLinear1d(image: Tensor, axis: i8, new_len: u63, opt: ResizeOpts) Tensor { - const res_shape = image.shape().set(axis, new_len); + const ax = image.axis(axis); + const res_shape = image.shape().set(ax, new_len); const dtype = opt.precision orelse if (image.dtype().class() == .integer) .f32 else image.dtype(); - const og_len = opt.original_len orelse Tensor.scalar(image.dim(axis), dtype); + const og_len = opt.original_len orelse Tensor.scalar(image.dim(ax), dtype); const ratio = og_len.convert(dtype).scale(stdx.math.divFloat(f32, 1, new_len)); const scaled = Tensor.arange(.{ .end = new_len }, dtype).mul(ratio); const left = scaled.floor(); @@ -682,11 +755,11 @@ pub fn resizeLinear1d(image: Tensor, axis: i8, new_len: u63, opt: ResizeOpts) Te // TODO: check that two gather isn't too bad perf wise. // Normally we should use gatherSlices to collect the values 2 by 2, // but gatherSlices messes up with the order of axes. - const left_val = image.gatherValues(axis, left.convert(.i32), .{ .indices_are_sorted = true }).convert(dtype); - const right_val = image.gatherValues(axis, right.convert(.i32), .{ .indices_are_sorted = true }).convert(dtype); + const left_val = image.gather_(&.{ax}, &.{left.convert(.i32)}, .{ .indices_are_sorted = true }).convert(dtype); + const right_val = image.gather_(&.{ax}, &.{right.convert(.i32)}, .{ .indices_are_sorted = true }).convert(dtype); - const left_weight = right.sub(scaled).broadcast(res_shape, &.{axis}); - const right_weight = scaled.sub(left).broadcast(res_shape, &.{axis}); + const left_weight = right.sub(scaled).broadcast(res_shape, &.{ax}); + const right_weight = scaled.sub(left).broadcast(res_shape, &.{ax}); const res = left_val.mul(left_weight).add(right_val.mul(right_weight)); return res.convert(image.dtype()).withTags(image.shape().tags()); @@ -853,13 +926,10 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { return cuda.sdpa(q, k, v, opts); } - if (q.dim(.h) != k.dim(.h)) { - stdx.debug.assert(@mod(q.dim(.h), k.dim(.h)) == 0, err_template ++ "Different number of heads for keys and queries, but can't repeat keys.", err_args); - // Note: we don't try to repeat queries. - // Repeating keys is the interesting optimisation cause it reduces KV cache memory usage. - const num_rep: u63 = @intCast(@divExact(q.dim(.h), k.dim(.h))); - k, v = .{ k.repeat1d(.h, num_rep), v.repeat1d(.h, num_rep) }; - } + // Handle different numbers of head by splitting q heads. + // This is a bit error prone in the sense that it depends of the layout of q heads. + // This is the Llama convention though. + q = q.splitAxis(.h, .{ .h = k.dim(.h), .hq = .auto }); const attn_mask = if (opts.attn_mask) |m| m else null; const dims = helpers.collectDims(.{ .h, .q, .k, .hd }, &.{ q, k, v, attn_mask }, .strict) catch { @@ -875,339 +945,7 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { attn_weights = attn_weights.convert(.f32).softmax(.k).convert(q.dtype()); var attn = attn_weights.dot(v, .{.k}); - return attn.transpose(q.shape()); -} - -pub const SdpaChunks = struct { q_chunk_size: u32, k_chunk_size: u32 }; - -pub fn sdpaMemEfficient( - q: Tensor, - k: Tensor, - v: Tensor, - sdpa_opts: SdpaOpts, - chunking: SdpaChunks, -) Tensor { - const sdpa_mem_efficient: SdpaMemEfficient = .{ - .q = q, - .k = k, - .v = v, - .sdpa_opts = sdpa_opts, - .chunking = .{ - .q_chunk_size = @intCast(@min(q.dim(.q), chunking.q_chunk_size)), - .k_chunk_size = @intCast(@min(k.dim(.k), chunking.k_chunk_size)), - }, - }; - - return sdpa_mem_efficient.forward(); -} - -const SdpaMemEfficient = struct { - q: Tensor, - k: Tensor, - v: Tensor, - sdpa_opts: SdpaOpts, - chunking: SdpaChunks, - - fn forward(self: SdpaMemEfficient) Tensor { - stdx.debug.assert(@mod(self.q.dim(.q), self.chunking.q_chunk_size) == 0, "sdpaMemEfficient expects the chunk_size to exactly divise the seq_len, got: sdpaMemEfficient({f}, {})", .{ self.q, self.chunking }); - stdx.debug.assert(@mod(self.k.dim(.k), self.chunking.k_chunk_size) == 0, "sdpaMemEfficient expects the chunk_size to exactly divise the seq_len, got: sdpaMemEfficient({f}, {})", .{ self.k, self.chunking }); - const n_q_chunks: u32 = @intCast(@divExact(self.q.dim(.q), self.chunking.q_chunk_size)); - - const ctx = zml.module.CompilationContext.current(); - const q_chunks = ctx.allocator().alloc(zml.Tensor, n_q_chunks) catch unreachable; - defer ctx.allocator().free(q_chunks); - for (0..n_q_chunks) |i| { - const idx: u32 = @intCast(i); - const q_slice: zml.Tensor.DynSlice = .{ - .start = Tensor.scalar(idx * self.chunking.q_chunk_size, .i32), - .len = self.chunking.q_chunk_size, - }; - const q_chunk = self.q.dynamicSlice(.{ .q = q_slice }); - const attn_chunk = if (self.sdpa_opts.attn_mask) |attn_mask| attn_mask.dynamicSlice(.{ .q = q_slice }) else null; - - var chunk: SdpaMemEfficient = self; - chunk.q = q_chunk; - chunk.sdpa_opts.attn_mask = attn_chunk; - q_chunks[i] = chunk.scanKeyVal(); - } - - const res = zml.Tensor.concatenate(q_chunks, .q); - return res.transpose(self.q.shape()); - } - - fn nextQueriesChunk(self: SdpaMemEfficient, idx: Tensor) Tensor { - const q_slice: zml.Tensor.DynSlice = .{ - .start = idx.scale(self.chunking.q_chunk_size), - .len = self.chunking.q_chunk_size, - }; - const q_chunk = self.q.dynamicSlice(.{ .q = q_slice }); - const attn_chunk = if (self.sdpa_opts.attn_mask) |attn_mask| attn_mask.dynamicSlice(.{ .q = q_slice }) else null; - - var chunk: SdpaMemEfficient = self; - chunk.q = q_chunk; - chunk.sdpa_opts.attn_mask = attn_chunk; - return chunk.scanKeyVal(); - } - - fn scanKeyVal(self: SdpaMemEfficient) Tensor { - const n_chunks = @divExact(self.k.dim(.k), self.chunking.k_chunk_size); - return if (n_chunks <= 4) { - // Unrolled version - var partial_softmax: ?PartialSoftmax = null; - for (0..@intCast(n_chunks)) |idx| { - const next = self.nextKeyValChunk(Tensor.scalar(idx, .i32)); - partial_softmax = if (partial_softmax) |prev| prev.merge(next) else next; - } - return partial_softmax.?.finalize(); - } else { - // stablehlo.while version - const partial_softmax, _ = zml.ops.while_(hasNextKeyValChunk, nextKeyValChunkMerge, self, .{ PartialSoftmax.zeros(self.q.shape(), .f32), Tensor.scalar(0, .i32) }); - return partial_softmax.finalize(); - }; - } - - fn nextKeyValChunkMerge(self: SdpaMemEfficient, prev: PartialSoftmax, idx: Tensor) struct { PartialSoftmax, Tensor } { - const next = self.nextKeyValChunk(idx); - return .{ prev.merge(next), idx.addConstant(1) }; - } - - fn nextKeyValChunk(self: SdpaMemEfficient, idx: Tensor) PartialSoftmax { - const k_slice: zml.Tensor.DynSlice = .{ - .start = idx.scale(self.chunking.k_chunk_size), - .len = self.chunking.k_chunk_size, - }; - - const k_chunk = self.k.dynamicSlice(.{ .k = k_slice }); - const v_chunk = self.v.dynamicSlice(.{ .k = k_slice }); - const attn_chunk = if (self.sdpa_opts.attn_mask) |mask| mask.dynamicSlice(.{ .k = k_slice }) else null; - - return sdpaChunk(self.q, k_chunk, v_chunk, .{ .attn_mask = attn_chunk }); - } - - pub fn hasNextKeyValChunk(self: SdpaMemEfficient, _: PartialSoftmax, idx: Tensor) zml.Tensor { - const n_chunks = @divExact(self.k.dim(.k), self.chunking.k_chunk_size); - return idx.cmp(.LT, Tensor.scalar(n_chunks, idx.dtype())); - } -}; - -pub const PartialSoftmax = struct { - values: Tensor, - exp_sum: Tensor, - max_value: Tensor, - - pub fn zeros(q_shape: Shape, exp_sum_precision: DataType) PartialSoftmax { - return .{ - .values = Tensor.constant(q_shape, q_shape.dtype().zero()), - .exp_sum = Tensor.constant(q_shape.setDim(.hd, 1), exp_sum_precision.zero()), - .max_value = Tensor.constant(q_shape.setDim(.hd, 1), q_shape.dtype().minValue()), - }; - } - - pub fn merge(self: PartialSoftmax, other: PartialSoftmax) PartialSoftmax { - // Rescale self and other using the new global_max. - const global_max = self.max_value.maximum(other.max_value); - const new_self = self.rescale(global_max); - const new_other = other.rescale(global_max); - - // Now that self and other are using the same scale, we can just add them: - return .{ - .max_value = global_max, - .values = new_self.values.add(new_other.values), - .exp_sum = new_self.exp_sum.add(new_other.exp_sum), - }; - } - - /// Update max_value and rescale attn and exp_sum accordingly. - pub fn rescale(self: PartialSoftmax, max_value: Tensor) PartialSoftmax { - const max_diff_exp = self.max_value.sub(max_value).exp(); - const sum_dtype = self.exp_sum.dtype(); - return .{ - .max_value = max_value, - .values = self.values.mul(max_diff_exp.broad(self.values.shape())), - .exp_sum = self.exp_sum.mul(max_diff_exp.convert(sum_dtype)), - }; - } - - /// Divides the intermediary results by the exp_sum to get the proper attention values. - pub fn finalize(self: PartialSoftmax) Tensor { - return self.values.div(self.exp_sum.broad(self.values.shape()).convert(self.values.dtype())); - } -}; - -/// Compute softmax over a chunk. -/// Returns intermediary results to allow aggregating later. -pub fn partialSoftmax(self: Tensor, axis: anytype) PartialSoftmax { - const a = self.axis(axis); - const max_val = self.max(a).maximum(Tensor.scalar(-1e16, self.dtype())); - const out = self.sub(max_val.broad(self.shape())).exp(); - return .{ - .values = out, - .exp_sum = out.convert(.f32).sum(a), - .max_value = max_val, - }; -} - -/// Compute sdpa on a chunk, and computes a partial softmax. -/// q: (B, H, Sq, H_dim) ⊙ k: (B, H, Sk, H_dim) -> qk: (B, H, Sq, Sk) -pub fn sdpaChunk(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) PartialSoftmax { - // this is a dupe of sdpa, but return the PartialSoftmax instead of true Attn. - // Consider implementing sdpa from sdpaChunk. - var q, var k, var v = .{ q_, k_, v_ }; - - const err_template = "sdpa(q: {f}, k: {f}, v: {f}, attn: {?f}) is invalid ! "; - const err_args = .{ q, k, v, opts.attn_mask }; - stdx.debug.assert(q.shape().hasTags(.{ .h, .q, .hd }), err_template ++ "q is missing tags {{.h, .q, .hd}}", err_args); - stdx.debug.assert(k.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "k is missing tags {{.h, .k, .hd}}", err_args); - stdx.debug.assert(v.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "v is missing tags {{.h, .k, .hd}}", err_args); - - if (q.dim(.h) != k.dim(.h)) { - stdx.debug.assert(@mod(q.dim(.h), k.dim(.h)) == 0, err_template ++ "Different number of heads for keys and queries, but can't repeat keys.", err_args); - // Note: we don't try to repeat queries. - // Repeating keys is the interesting optimisation cause it reduces KV cache memory usage. - const num_rep: u63 = @intCast(@divExact(q.dim(.h), k.dim(.h))); - k, v = .{ k.repeat1d(.h, num_rep), v.repeat1d(.h, num_rep) }; - } - const attn_mask = if (opts.attn_mask) |m| m else null; - - const dims = helpers.collectDims(.{ .h, .q, .k, .hd }, &.{ q, k, v, attn_mask }, .strict) catch { - stdx.debug.panic(err_template ++ "Inputs have incompatible shapes.", err_args); - }; - const sqrtHeadDim: f32 = 1.0 / std.math.sqrt(@as(f32, @floatFromInt(dims.hd))); - const head_scaling = if (opts.scale) |s| s else Tensor.scalar(sqrtHeadDim, k.dtype()); - k = k.mul(head_scaling.convert(k.dtype())); - - var attn_weights = q.dot(k, .{.hd}); - // log.debug("attn_weights : {}", .{attn_weights}); - // log.debug("attn_mask : {?}", .{attn_mask}); - if (attn_mask) |mask| attn_weights = attn_weights.add(mask.broad(attn_weights.shape())); - - const partial = partialSoftmax(attn_weights, .k); - const attn = partial.values.dot(v, .{.k}).transpose(q.shape()); - - return .{ - .values = attn, - // The renaming is because the above dot projected values.k into .hd, - // do the same thing on the other tensors. - // This work because dot is a linear operation, and commutes with `PartialSoftmax.finalize` - .exp_sum = partial.exp_sum.rename(.{ .k = .hd }).transpose(attn.shape()), - .max_value = partial.max_value.rename(.{ .k = .hd }).transpose(attn.shape()), - }; -} - -test sdpaMemEfficient { - const platform = zml.testing.env(); - const allocator = std.testing.allocator; - - // Note we use small input vectors to have the tests run reasonably fast, - // but don't expect speed ups with this small sizes. - const rng = try zml.compileFn(allocator, Tensor.Rng.normal, .{ Shape.init(.{ 1, 10, 512, 64 }, .f32), .{ .mean = 0, .stddev = 1 } }, platform); - defer rng.deinit(); - - const rng_mask = try zml.compileFn(allocator, Tensor.Rng.normal, .{ Shape.init(.{ 512, 512 }, .f32), .{ .mean = 0, .stddev = 1 } }, platform); - defer rng_mask.deinit(); - - // Note: we pass void here, cause Rng.normal doesn't take any runtime inputs. - const q = rng.call({}).withTags(.{ .b, .h, .q, .hd }); - const k = rng.call({}).withTags(.{ .b, .h, .k, .hd }); - const v = rng.call({}).withTags(.{ .b, .h, .k, .hd }); - const mask = rng_mask.call({}).withTags(.{ .q, .k }); - - const ref_res = try zml.testing.compileAndCall( - platform, - sdpa, - .{ q, k, v, .{ .attn_mask = mask, .scale = null } }, - ); - try std.testing.expectEqualSlices(i64, q.shape().dims(), ref_res.shape().dims()); - { - // 4 k_chunks - const res = try zml.testing.compileAndCall( - platform, - sdpaMemEfficient, - .{ - q, - k, - v, - .{ .attn_mask = mask, .scale = null }, - .{ .q_chunk_size = 256, .k_chunk_size = @divExact(512, 4) }, - }, - ); - - try zml.testing.expectClose(ref_res, res, 2e-3); - } - { - // 16 k_chunks - const res = try zml.testing.compileAndCall( - platform, - sdpaMemEfficient, - .{ - q, - k, - v, - .{ .attn_mask = mask, .scale = null }, - .{ .q_chunk_size = 256, .k_chunk_size = @divExact(512, 16) }, - }, - ); - - try zml.testing.expectClose(ref_res, res, 2e-3); - } -} - -test "sdpaMemEfficient transposed" { - const platform = zml.testing.env(); - const allocator = std.testing.allocator; - - // Note we use small input vectors to have the tests run reasonably fast, - // but don't expect speed ups with this small sizes. - const rng = try zml.compileFn(allocator, Tensor.Rng.normal, .{ Shape.init(.{ 1, 512, 10, 64 }, .f32), .{ .mean = 0, .stddev = 1 } }, platform); - defer rng.deinit(); - - const rng_mask = try zml.compileFn(allocator, Tensor.Rng.normal, .{ Shape.init(.{ 512, 512 }, .f32), .{ .mean = 0, .stddev = 1 } }, platform); - defer rng_mask.deinit(); - - // Note: we pass void here, cause Rng.normal doesn't take any runtime inputs. - const q = rng.call({}).withTags(.{ .b, .q, .h, .hd }); - const k = rng.call({}).withTags(.{ .b, .k, .h, .hd }); - const v = rng.call({}).withTags(.{ .b, .k, .h, .hd }); - const mask = rng_mask.call({}).withTags(.{ .q, .k }); - - const ref_res = try zml.testing.compileAndCall( - platform, - sdpa, - .{ q, k, v, .{ .attn_mask = mask, .scale = null } }, - ); - try std.testing.expectEqualSlices(i64, q.shape().dims(), ref_res.shape().dims()); - - { - const res = try zml.testing.compileAndCall( - platform, - sdpaMemEfficient, - .{ - q, - k, - v, - .{ .attn_mask = mask, .scale = null }, - .{ .q_chunk_size = @divExact(512, 2), .k_chunk_size = @divExact(512, 4) }, - }, - ); - - try zml.testing.expectClose(ref_res, res, 1e-3); - } - - { - const res = try zml.testing.compileAndCall( - platform, - sdpaMemEfficient, - .{ - q, - k, - v, - .{ .attn_mask = mask, .scale = null }, - .{ .q_chunk_size = 512, .k_chunk_size = @divExact(512, 4) }, - }, - ); - - try zml.testing.expectClose(ref_res, res, 1e-3); - } + return attn.transpose(q.shape()).merge(.{ .h = .{ .h, .hq } }); } /// Options controlling generation. The default values correspond to greedy decoding. @@ -1225,9 +963,9 @@ pub fn sampleTokens(activations: Tensor, opts: SamplingStrategy, rng: Tensor.Rng return .{ next_tokens, rng }; } - const topk = activations.topK(opts.topk, .voc, .{}); - // After the topk, we don't have .voc values, anymore, only topk. - var x = topk.values.rename(.{ .voc = .topk }); + const topk = activations.topK(.{ .topk = .voc }, opts.topk, .{}); + // After the topk, we don't have .voc values, anymore, only .topk. + var x = topk.values; if (opts.temperature != 1.0) { x = x.scale(1 / opts.temperature); } @@ -1242,7 +980,7 @@ pub fn sampleTokens(activations: Tensor, opts: SamplingStrategy, rng: Tensor.Rng // topk_idx is indices into topk.values ! so in the range [0, topk] // Convert for the original indices from the full [0, voc] range. - const next_tokens = topk.indices.gatherValues(.voc, topk_idx.squeeze(.topk), .{}); + const next_tokens = topk.indices.gather(.{ .topk = topk_idx.squeeze(.topk) }, .{}); // log.debug("sampleTokens({}) -> {} -> {} -> {}", .{ activations, topk.indices, topk_idx, next_tokens }); return .{ next_tokens, next_rng }; } @@ -1330,7 +1068,7 @@ pub fn sampleTokensDynamic(logits: Tensor, opts: DynamicSamplingStrategy, rng: T x = x.add(gumbel_noise); const topk_idx = x.argMax(.topk).indices; - const next_tokens = topk_indices.gatherValues(.voc, topk_idx.squeeze(.topk), .{}); + const next_tokens = topk_indices.gather(.{ .voc = topk_idx.squeeze(.topk) }, .{}); return .{ next_tokens, next_rng }; } @@ -1339,7 +1077,7 @@ fn fixupLogits(logits: Tensor, opts: DynamicSamplingStrategy) [2]Tensor { // First reduce the vocab size to a reasonable sub set of candidate. const full_topk = if (opts.max_top_k > 0) - logits.topK(opts.max_top_k, .voc, .{ .descending = true }) + logits.topK(.{ .voc = .voc }, opts.max_top_k, .{ .descending = true }) else logits.sort(.voc, .{ .descending = true }); diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 46c40523..f09ffa1e 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -5,25 +5,30 @@ const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); pub const ffi = pjrt.ffi; -pub const ApiError = pjrt.ApiError; -pub const ErrorCode = pjrt.ErrorCode; -pub const ExecuteContext = pjrt.ExecuteContext; -pub const BufferType = pjrt.BufferType; -pub const Device = pjrt.Device; -pub const MemoryStats = pjrt.MemoryStats; -pub const DeviceDescription = pjrt.DeviceDescription; pub const Api = pjrt.Api; -pub const NamedValue = pjrt.NamedValue; +pub const ApiError = pjrt.ApiError; +pub const BufferType = pjrt.BufferType; pub const ClientInitError = pjrt.ClientInitError; -pub const Error = pjrt.Error; -pub const GetCostAnalysisError = pjrt.GetCostAnalysisError; -pub const SerializeResult = pjrt.SerializeResult; -pub const Executable = pjrt.Executable; pub const CompiledMemoryStats = pjrt.CompiledMemoryStats; +pub const Device = pjrt.Device; +pub const DeviceDescription = pjrt.DeviceDescription; +pub const Error = pjrt.Error; +pub const ErrorCode = pjrt.ErrorCode; +pub const Executable = pjrt.Executable; +pub const ExecuteContext = pjrt.ExecuteContext; pub const ExecuteError = ApiError; +pub const GetCostAnalysisError = pjrt.GetCostAnalysisError; pub const Memory = pjrt.Memory; +pub const MemoryStats = pjrt.MemoryStats; +pub const NamedValue = pjrt.NamedValue; +pub const Profiler = pjrt.Profiler; +pub const SerializeResult = pjrt.SerializeResult; pub const Stream = pjrt.Stream; +const zml = struct { + pub const DataType = @import("dtype.zig").DataType; +}; + const log = std.log.scoped(.zml); pub const CompileError = std.mem.Allocator.Error || error{InvalidMlirBytecodeVersion} || ApiError; @@ -165,8 +170,8 @@ pub const Buffer = opaque { return @ptrCast(try self.inner().toHostBuffer(api, dst)); } - pub fn getElementType(self: *const Buffer, api: *const Api) BufferType { - return self.inner().getElementType(api); + pub fn getElementType(self: *const Buffer, api: *const Api) zml.DataType { + return dtypeFromBufferType(self.inner().getElementType(api)); } pub fn getDimensions(self: *const Buffer, api: *const Api) []const i64 { @@ -329,3 +334,29 @@ pub const AsyncHostToDeviceTransferManager = opaque { return self.inner().addMetadata(api, transfer_metadata); } }; + +pub fn bufferTypeFromDtype(dt: zml.DataType) pjrt.BufferType { + return switch (dt) { + inline else => |tag| @field(pjrt.BufferType, @tagName(tag)), + }; +} + +pub fn dtypeFromBufferType(pjrt_type: pjrt.BufferType) zml.DataType { + return switch (pjrt_type) { + .invalid => @panic("Found an invalid pjrt buffer"), + inline else => |tag| @field(zml.DataType, @tagName(tag)), + }; +} + +test bufferTypeFromDtype { + inline for (@typeInfo(zml.DataType).@"enum".fields) |field| { + const dt: zml.DataType = @enumFromInt(field.value); + try std.testing.expectEqual(dt, dtypeFromBufferType(bufferTypeFromDtype(dt))); + } + + inline for (@typeInfo(pjrt.BufferType).@"enum".fields) |field| { + const dt: pjrt.BufferType = @enumFromInt(field.value); + if (dt == .invalid) continue; + try std.testing.expectEqual(dt, bufferTypeFromDtype(dtypeFromBufferType(dt))); + } +} diff --git a/zml/quantization.zig b/zml/quantization.zig index 2a850a4d..90a35bda 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -1,9 +1,11 @@ const std = @import("std"); -const zml = @import("zml.zig"); - const Allocator = std.mem.Allocator; + +const zml = @import("zml.zig"); const module = zml.module; +// TODO add tests, use modern zml + pub fn Q4_0(comptime dtype: zml.DataType) type { return struct { const Self = @This(); @@ -38,38 +40,12 @@ pub fn Q4_0(comptime dtype: zml.DataType) type { const scales = extractScales(block_count, input); const weights = extractWeights(block_count, input); - return scales.reshape(.{ block_count, 1 }) - .broadcastLeft(zml.Shape.init(.{ block_count, 32 }, .f32)) + return scales.broadcast(weights.shape(), &.{0}) .mul(weights) .convert(dtype) - .reshape(.{block_count * 32}) .reshape(shape); } - pub fn scaleIndices(block_count: u63) zml.Tensor { - // indices1 is the offsets of the scale bytes, repeated block_count times. - const indices1 = zml.Tensor.arange(.{ .start = 0, .end = 2 }, .i32).repeat1d(0, block_count); - - // indices2 is the offsets of the blocks, repeated for each scale byte, repeated block_count times. - const indices2 = zml.Tensor.arange(.{ .start = 0, .end = block_stride * block_count, .step = block_stride }, .i32) - .reshape(.{ block_count, 1 }).broadcastLeft(zml.Shape.init(.{ block_count, 2 }, .i32)).reshape(.{2 * block_count}); - - // indices is the sum of the two, which is the offsets to all the bytes we are interested in. - return indices1.add(indices2); - } - - pub fn weightIndices(block_count: u63) zml.Tensor { - // indices1 is the offsets of the data bytes, repeated block_count times. - const indices1 = zml.Tensor.arange(.{ .start = 2, .end = 18 }, .i32).repeat1d(0, block_count); - - // indices2 is the offsets of the blocks, repeated for each data byte, repeated block_count times. - const indices2 = zml.Tensor.arange(.{ .start = 0, .end = block_stride * block_count, .step = block_stride }, .i32) - .reshape(.{ block_count, 1 }).broadcastLeft(zml.Shape.init(.{ block_count, 16 }, .i32)).reshape(.{16 * block_count}); - - // indices is the sum of the two, which is the offsets to all the bytes we are interested in. - return indices1.add(indices2); - } - pub fn extractScales(block_count: u63, input: zml.Tensor) zml.Tensor { // The goal here is to get the first two bytes of every 18-bytes block in the input. For that, // we generate a list of indices that we will use to gather from the input. @@ -85,7 +61,7 @@ pub fn Q4_0(comptime dtype: zml.DataType) type { const indices = indices1.add(indices2); // We select the values we are interested in with the indices, group them by pair and bitcast them to f16, then convert them to f32. - const scales = input.gatherValues(0, indices, .{ .indices_are_sorted = true }).reshape(.{ block_count, 2 }).bitCast(.f16).convert(.f32); + const scales = input.gather_(&.{0}, &.{indices}, .{ .indices_are_sorted = true }).reshape(.{ block_count, 2 }).bitCast(.f16).convert(.f32); return scales; } @@ -107,7 +83,7 @@ pub fn Q4_0(comptime dtype: zml.DataType) type { // NOTE(Corendos): i4 is not supported by bitcast convert, so we need the following workaround. // We select the values we are interested in with the indices, these are our quantized_weights. - const quantized_weights = input.gatherValues(0, indices, .{ .indices_are_sorted = true }); + const quantized_weights = input.gather_(&.{0}, &.{indices}, .{ .indices_are_sorted = true }); const lb_weights = quantized_weights .logical(.And, zml.Tensor.constant(.{16 * block_count}, zml.Data.init(.u8, 0xf))) .bitCast(.i8); diff --git a/zml/shape.zig b/zml/shape.zig index a9daffa7..564f33b5 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -413,6 +413,13 @@ pub const Shape = struct { // Already the right shape if (std.mem.eql(i64, self.dims(), other.dims())) return true; + if (std.mem.eql(Tag, self.tags(), other.tags())) { + for (0..self.rank()) |i| { + if (self.dim(i) != 1 and self.dim(i) != other.dim(i)) return false; + } + return true; + } + // Non ambiguous broadcasting // TODO: broad is error prone because of this: // it will happily broadcast .{ .a = 10, .b = 1 } to .{ .b = 10, .a = 5 } @@ -424,6 +431,7 @@ pub const Shape = struct { } for (self.dims(), self.tags()) |d, t| { + // TODO this is also wrong when the axes are in different order in the two shapes. const other_ax = other.hasTag(t) orelse return false; if (d != 1 and d != other.dim(other_ax)) return false; } diff --git a/zml/tensor.zig b/zml/tensor.zig index 373d5efa..78219cdc 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -311,7 +311,7 @@ pub const Tensor = struct { /// Returns the given tensor as one contiguous buffer of bytes. pub fn bytes(self: Tensor) Tensor { - return self.bitCast(.u8).flattenAll().withTags(.{.bytes}); + return self.bitCast(.u8).flatten().withTags(.{.bytes}); } /// Returns a Tensor containing the element-wise number of leading 0 bits in the input Tensor. @@ -700,7 +700,7 @@ pub const Tensor = struct { pub fn gumbelStats(rand: Rng, target_dist: Tensor) struct { Rng, Stats } { const s = Shape.init(.{ .n = 1024, .d = 4 }, .f32); const rng, const data = rand.gumbel(s); - const flat = data.flattenAll(); + const flat = data.flatten(); const mean_ = flat.mean(0); const variance = flat.sub(mean_.broad(flat.shape())).pow(Tensor.scalar(2, .f32)).mean(0); @@ -719,7 +719,7 @@ pub const Tensor = struct { break :blk powers; }; const values = Tensor.constantTensor(HostBuffer.fromArray(&powers)).withTags(.{.d}); - const counts = values.gatherValues(.d, samples, .{}).sum(.n).bitCast(.u16); + const counts = values.gather(.{ .d = samples }, .{}).sum(.n).bitCast(.u16); const actual_dist = counts.reshape(target_dist.shape()).convert(target_dist.dtype()).divByConst(s.dim(.n)); return .{ rng, .{ .mean = mean_, .variance = variance, .actual_dist = actual_dist } }; } @@ -993,17 +993,22 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise addition of the input Tensor with a constant. pub fn addConstant(self: Tensor, b: anytype) Tensor { - return self.add(Tensor.scalar(b, self.dtype())); + return self.add(.scalar(b, self.dtype())); } /// Returns a Tensor containing the element-wise division of the input Tensor by a constant. pub fn divByConst(self: Tensor, b: anytype) Tensor { - return self.div(Tensor.scalar(b, self.dtype())); + return self.div(.scalar(b, self.dtype())); + } + + /// Returns a Tensor containing the element-wise power of the input Tensor by a constant. + pub fn powByConst(self: Tensor, b: anytype) Tensor { + return self.pow(.scalar(b, self.dtype())); } /// Returns a Tensor containing the element-wise multiplication of the input Tensor by a constant. pub inline fn scale(self: Tensor, val: anytype) Tensor { - return self.mul(Tensor.scalar(val, self.dtype())); + return self.mul(.scalar(val, self.dtype())); } pub const LogicalOp = enum { OR, XOR, AND }; @@ -1325,7 +1330,7 @@ pub const Tensor = struct { const row_mask = max_val.cmp(.GT, Tensor.scalar(-std.math.inf(f64), self.dtype())); const exp_diff_max = self.sub(self.max(a).broad(self._shape)).exp(); - const res = exp_diff_max.div(exp_diff_max.sum(a).broad(self._shape)); + const res = exp_diff_max.div(exp_diff_max.sum(a)); // If a row is full -inf return full 0 instead of full nan, // this fix attention when mask hides a full row. @@ -1519,27 +1524,7 @@ pub const Tensor = struct { return self.transpose(perm.constSlice()); } - /// Flattens the given axis and the next one, into one new axis. - pub fn flatten(self: Tensor, axis_: anytype) Tensor { - // TODO: move to torch.zig, this is equivalent to merge - const old_shape = self._shape; - const a = self.axis(axis_); - // stdx.debug.assert(a + 1 < self.rank(), "Can't flatten {} on the last axis {}.", .{ self, axis }); - const new_shape = old_shape.remove(a + 1).set(a, old_shape.dim(a) * old_shape.dim(a + 1)); - - const loc = self.getContext().location(@src(), "flatten({f},{})", .{ self, axis_ }); - const reshaped_val = dialect.stablehlo.reshape( - self.getContext().mlirCtx(), - self.value(), - mlirx.tensorType(self.getContext().mlirCtx(), new_shape), - loc, - ); - // log.debug("flatten({d}, {d}) -> {d}", .{ self.dims(), axis_, new_shape[0 .. self.rank() - 1] }); - return _result(new_shape, reshaped_val.result(0)); - } - - pub inline fn flattenAll(self: Tensor) Tensor { - // TODO: rename to just flatten, once flatten is moved to torch + pub inline fn flatten(self: Tensor) Tensor { return self.reshape(.{self.count()}); } @@ -1727,14 +1712,12 @@ pub const Tensor = struct { } const a = self.axis(axis_); - const broadshape = self._shape.insert(a + 1, .{n_rep}); - const repeat_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a + 1); + const res_shape = self._shape.setDim(a, self.dim(a) * n_rep); - var res = self.broadcast(broadshape, repeat_dims.dims()).flatten(a); - // Restor the tag that has been lost by flatten. - res._shape._tags.set(a, self._shape.tag(a)); + const broadshape = self._shape.insert(a, .{n_rep}); + const repeat_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a); - return res; + return self.broadcast(broadshape, repeat_dims.dims()).reshape(res_shape); } /// Repeats a Tensor several times along the given axes. @@ -1751,15 +1734,47 @@ pub const Tensor = struct { return res; } + test repeat1d { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const Local = struct { + fn repeat1d(x: Tensor, axis_: u3, n_reps: u32) Tensor { + return x.repeat1d(axis_, n_reps); + } + }; + + { + const inputs: [3]u8 = .{ 1, 2, 3 }; + const expectations: [6]u8 = .{ 1, 2, 3, 1, 2, 3 }; + + const input = try zml.Buffer.fromArray(platform, inputs); + const output = try zml.testing.compileAndCall(platform, Local.repeat1d, .{ input, 0, 2 }); + + try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + } + { + const inputs: [2][3]u8 = .{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }; + const expectations: [2][6]u8 = .{ .{ 1, 2, 3, 1, 2, 3 }, .{ 4, 5, 6, 4, 5, 6 } }; + + const input = try zml.Buffer.fromArray(platform, inputs); + const output = try zml.testing.compileAndCall(platform, Local.repeat1d, .{ input, 1, 2 }); + + try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + } + } + /// Repeats in line each value along the given axis. /// - /// * stutter1d([0, 1, 2, 3], 0, 2) = [0, 0, 1, 1, 2, 2, 3, 3] - pub fn stutter1d(self: Tensor, axis_: i64, n_rep: u63) Tensor { + /// * stutter1d([0, 1, 2, 3], -1, 2) = [0, 0, 1, 1, 2, 2, 3, 3] + /// This is equivalent to repeat(ax+1) unless ax is the last axis. + pub fn stutter1d(self: Tensor, axis_: i8, n_rep: u63) Tensor { const a = self.axis(axis_); const broadshape = self._shape.insert(a + 1, .{n_rep}); - const stutter_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a + 1); + const res_shape = self._shape.setDim(a, self.dim(a) * n_rep); - return self.broadcast(broadshape, stutter_dims.dims()).flatten(a); + const stutter_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a + 1); + return self.broadcast(broadshape, stutter_dims.dims()).reshape(res_shape); } /// Repeats in line each value along the given axes. @@ -1775,6 +1790,45 @@ pub const Tensor = struct { return res; } + test stutter1d { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const Local = struct { + fn stutter1d(x: Tensor, axis_: u3, n_reps: u32) Tensor { + return x.stutter1d(axis_, n_reps); + } + }; + + { + const inputs: [3]u8 = .{ 1, 2, 3 }; + const expectations: [6]u8 = .{ 1, 1, 2, 2, 3, 3 }; + + const input = try zml.Buffer.fromArray(platform, inputs); + const output = try zml.testing.compileAndCall(platform, Local.stutter1d, .{ input, 0, 2 }); + + try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + } + { + const inputs: [2][3]u8 = .{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }; + const expectations: [2][6]u8 = .{ .{ 1, 1, 2, 2, 3, 3 }, .{ 4, 4, 5, 5, 6, 6 } }; + + const input = try zml.Buffer.fromArray(platform, inputs); + const output = try zml.testing.compileAndCall(platform, Local.stutter1d, .{ input, 1, 2 }); + + try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + } + { + const inputs: [2][3]u8 = .{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }; + const expectations: [2][6]u8 = .{ .{ 1, 2, 3, 1, 2, 3 }, .{ 4, 5, 6, 4, 5, 6 } }; + + const input = try zml.Buffer.fromArray(platform, inputs); + const output = try zml.testing.compileAndCall(platform, Local.stutter1d, .{ input, 0, 2 }); + + try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + } + } + /// Returns a Tensor containing the element-wise negation of the input Tensor. pub fn negate(self: Tensor) Tensor { const loc = self.getContext().mlirCtx().location(@src()); @@ -2160,88 +2214,98 @@ pub const Tensor = struct { pub const GatherOpts = struct { indices_are_sorted: bool = false }; - /// For each coordinate in `indices`, - /// `gatherValues` extracts a single value of the given tensor. + /// `gather` extracts slices from the given tensor at the specified offsets. + /// example: `values.gather(.{ .a = idx }, .{})` + /// + /// * indices is a named list of integer tensors: eg `.{ .a = idx }`. + /// Each names specify a gathering axis, it must refer to an axis of self, + /// and the corresponding idx Tensor must contains valid indices into axis .a. + /// All indices must have the same shape or broadcast to the same shape. /// - /// * axes_ is a single axis, or a tuple of axis: .b, or .{ .b, .c } - /// * indices is an integer tensor /// * result is a tensor whose shape is similar to the input shape /// where the gathered axes have been replaced by axes from 'indices'. /// /// Some example input for the base case where we work on one axis: - /// - gatherValues(f:[a]->float, .a, ind:[n]->int)[n] == f[ind[n]] - /// - gatherValues(f:[a, b], .a, ind:[n])[n, b] == f[ind[n], b] - /// - gatherValues(f: [a,b,c], .{.b}, ind: [n,m])[a, n, m, c] == f[a, ind[n, m], c] + /// - gather(f:[a], .{ .a = idx:[n]})[n] == f[idx[n]] + /// - gather(f:[a, b], .a, idx:[n])[n, b] == f[idx[n], b] + /// - gather(f:[a,b,c], .{.b = idx:[n,m]})[a, n, m, c] == f[a, idx[n, m], c] /// /// If an axis in common between `self` and `indices`, /// it is treated as a "batching" axis, meaning that semantically - /// the operator is doing a gatherValues one time per dimension of this axis: - /// - gatherValues(f: [a,b,c], .{.b}, ind: [a,n])[a, n] == f[a, ind[a, n]] + /// the operator is doing a gather one time per dimension of this axis: + /// - gather(f: [a,b,c], .{.b=idx: [a,n]})[a, n] == f[a, idx[a, n]] /// - /// It is an error to have an axis present in `self`, `axes_` and `indices`. + /// It's possible to pass several indices: + /// - gather(f: [a,b,c], .{.b=idx_b[n], .c=idx_c[n]})[a, n] == f[a, idx_b[n], idx_c[n]] + /// - gather(f: [a,b,c,d], .{.b=idx_b[a,n], .c=idx_c[a, n]})[a, n, d] == f[a, idx_b[a, n], idx_c[a, n], d] /// - /// If several axes are passed, then the last axis of indices is treated as coordinates: - /// - gatherValues(f: [a,b,c], .{.b, .c}, ind: [n,2])[a, n] == f[a, ind[n][0], ind[n][1]] - /// - gatherValues(f: [a,b,c,d], .{.b, .c}, ind: [a, n,2])[a, n, d] == f[a, ind[a, n][0], ind[a, n][1], d] + /// If `self` isn't tagged, you can use `gather_` to specify gathered axis by their position but batching won't be available. /// - /// It is possible to use gatherValues without tags, but batching won't be available. - pub fn gatherValues(self: Tensor, coord_axes: anytype, indices: Tensor, opts: GatherOpts) Tensor { - // scoped_log.debug("gatherValues({}, {any}, {})", .{ self, coord_axes, indices }); - const single_coord, const coord_axes_ = _parseGatherCoord(self, coord_axes); + /// For performance it's better to have batching and gathering axes of `self` be the first one, + /// so that gather can + pub fn gather(self: Tensor, _indices: anytype, opts: GatherOpts) Tensor { + const idx_per_axis, const idx_tags = Shape.parseStruct(Tensor, _indices); + var idx_axes: Shape.AxesArray = .{}; + for (idx_tags.slice()) |t| { + idx_axes.appendAssumeCapacity(self.axis(t)); + } - stdx.debug.assert(coord_axes_.len > 0, "gatherValues expects 1 or more axes to operate one, received none. Example: `x.gatherValues(.a, indices, .{{}})`", .{}); - for (coord_axes_.constSlice(), 0..) |a, i| { + // TODO: sort indices following self.shape instead of asking the user to do it. + return self.gather_(idx_axes.slice(), idx_per_axis.slice(), opts); + } + + pub fn gather_(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, opts: GatherOpts) Tensor { + stdx.debug.assert(idx_axes.len > 0, "gather expects 1 or more axes to operate one, received none. Example: `x.gather(.a, indices, .{{}})`", .{}); + for (idx_axes, 0..) |a, i| { if (i > 0) { - stdx.debug.assert(a == coord_axes_.get(i - 1) + 1, "gatherValues expects 'coord_axes' to be sequential. But {any} aren't sequential in {f}", .{ coord_axes, self }); + stdx.debug.assert(a == idx_axes[i - 1] + 1, "gather expects 'idx_axes' to be sequential. But {any} aren't sequential in {f}", .{ idx_axes, self }); } } + var indices_shape = idx_per_axis[0].shape(); + for (idx_per_axis[1..]) |idx| { + if (idx.rank() > indices_shape.rank()) { + indices_shape = idx.shape(); + } + } + for (idx_per_axis) |idx| { + stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "gather indices can't be broadcasted together {any}", .{idx_per_axis}); + } + + var idx_batch_axes: Shape.DimsArray = .{}; const AxisKind = enum { batching, offset, collapsed, indices }; - var self_kind: stdx.BoundedArray(AxisKind, MAX_RANK) = .{}; - var indices_batch_axes: Shape.DimsArray = .{}; + var self_kind: stdx.BoundedArray(AxisKind, MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; + for (self._shape.tags(), 0..self.rank()) |t, self_ax| { - const maybe_coord_ax = std.mem.indexOfScalar(u3, coord_axes_.constSlice(), @intCast(self_ax)); - if (indices._shape.hasTag(t)) |id_ax| { + const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, 1, @intCast(self_ax)); + if (indices_shape.hasTag(t)) |id_ax| { // tag is both in self and indices -> it's a batching dim // Note: tags are required for batching. - self_kind.appendAssumeCapacity(.batching); - indices_batch_axes.appendAssumeCapacity(id_ax); - stdx.debug.assert(maybe_coord_ax == null, "gatherValues expects axes to appear at most twice. Axis {s} has been found both in 'self={f}', in 'coord_axes_={any}' and in 'indices={f}'", .{ self._shape._tags.get(self_ax), self, coord_axes, indices }); - } else if (maybe_coord_ax) |_| { - // for gatherValues we collapsed all gathered axes - // (contrary to gatherSlices where we collapse none) - self_kind.appendAssumeCapacity(.collapsed); + self_kind.buffer[self_ax] = .batching; + idx_batch_axes.appendAssumeCapacity(id_ax); + stdx.debug.assert(!is_gather_axis, "gather expects axes to appear at most twice. Axis {s} has been found both in 'self={f}', in 'idx_axes={any}' and in 'indices={f}'", .{ t, self, idx_axes, indices_shape }); + } else if (is_gather_axis) { + // we collapsed all gathered axes + self_kind.buffer[self_ax] = .collapsed; + // idx_kind.buffer[id_ax] = .indices; } else { - self_kind.appendAssumeCapacity(.offset); + self_kind.buffer[self_ax] = .offset; } } - // When we receive several coord_axes we need an extra dimension to store - // one index per axis, which makes the coordinates of one value. - // Otherwi se stablehlo uses the "indices.rank()" default value. - const index_coord_axis = if (single_coord) - indices.rank() - else blk: { - const ax = indices._shape.hasTag(.coord) orelse indices._shape.axis(-1); - stdx.debug.assert(indices.dim(ax) == coord_axes_.len, "gatherValues with axes={any}, expects indices to be of shape [..., {}], got: {f}", .{ coord_axes, coord_axes_.len, indices }); - break :blk ax; - }; - // compute res shape var res_shape = Shape.init(.{}, self.dtype()); var res_kind: stdx.BoundedArray(AxisKind, MAX_RANK) = .{}; - for (self_kind.constSlice(), 0..) |kind, ax_usize| { + for (self_kind.slice(), 0..) |kind, ax_usize| { const ax: u3 = @intCast(ax_usize); - if (ax == coord_axes_.get(0)) { + if (ax == idx_axes[0]) { // The first val_ax is special cause this is the place where we insert indices axes. - for (indices._shape.tags(), 0..indices.rank()) |t, id_ax| { - if (id_ax == index_coord_axis) continue; - if (std.mem.indexOfScalar(i64, indices_batch_axes.constSlice(), @intCast(id_ax))) |_| { - // batching dim are already in res - continue; - } + for (0.., indices_shape.tags(), indices_shape.dims()) |id_axis_order, id_axis, id_inserted_dim| { + const is_batching_axis = std.mem.containsAtLeastScalar(i64, idx_batch_axes.constSlice(), 1, @intCast(id_axis_order)); + // Batching axis is already in self. + if (is_batching_axis) continue; - res_shape = res_shape.appendDim(indices.dim(id_ax), t); + res_shape = res_shape.appendDim(id_inserted_dim, id_axis); res_kind.appendAssumeCapacity(.indices); } } @@ -2257,12 +2321,12 @@ pub const Tensor = struct { // This is not a gather, but a dynamicSlice. // Sometimes the backend recognize this pattern, but not always. // So let us handle that. - if (indices.count() == 1) { - return self.dynamicSlice1d(coord_axes_.get(0), .{ .start = indices.flattenAll().squeeze(0), .len = 1 }).reshape(res_shape); + if (indices_shape.count() == 1 and idx_axes.len == 1) { + return self.dynamicSlice1d(idx_axes[0], .{ .start = idx_per_axis[0].asScalar(), .len = 1 }).reshape(res_shape); } var slice_dims: Shape.DimsArray = .{}; - for (self_kind.constSlice(), self.dims()) |k, d| { + for (self_kind.slice(), self.dims()) |k, d| { slice_dims.appendAssumeCapacity(switch (k) { .batching, .collapsed => 1, .offset => d, @@ -2270,7 +2334,9 @@ pub const Tensor = struct { }); } - // scoped_log.debug("gatherValues --> {} {any}", .{ res_shape, res_kind.constSlice() }); + // TODO: try changing .last by other axis and see the perf impact. + const indices = Tensor.stack(idx_per_axis, .last, .coord); + // scoped_log.debug("gather --> {} {any}", .{ res_shape, res_kind.constSlice() }); const loc = self.getContext().mlirCtx().location(@src()); const gather_op = dialect.stablehlo.gather( self.getContext().mlirCtx(), @@ -2282,22 +2348,30 @@ pub const Tensor = struct { .offset_dims = _collectAxes(AxisKind, res_kind, .offset).constSlice(), .collapsed_slice_dims = _collectAxes(AxisKind, self_kind, .collapsed).constSlice(), .operand_batching_dims = _collectAxes(AxisKind, self_kind, .batching).constSlice(), - .start_indices_batching_dims = indices_batch_axes.constSlice(), + .start_indices_batching_dims = idx_batch_axes.constSlice(), .start_index_map = _collectAxes(AxisKind, self_kind, .collapsed).constSlice(), - .index_vector_dim = index_coord_axis, + .index_vector_dim = indices.axis(.coord), .indices_are_sorted = opts.indices_are_sorted, }, ); const mlir_shape = fromMlirValue(gather_op.result(0)).shape(); - stdx.debug.assert(mlir_shape.eql(res_shape), "gatherValues expects that batching indices appear in the same order in 'self' and 'indices', got: self={f}, indices={f}. You should transpose one or the other.", .{ self, indices }); + stdx.debug.assert(mlir_shape.eql(res_shape), "gather expects that batching indices appear in the same order in 'self' and 'indices', got: self={f}, indices={f}. You should transpose one or the other.", .{ self, indices }); return _result(res_shape, gather_op.result(0)); } - test gatherValues { + test gather { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const Local = struct { + pub fn _idx(idx_shape: anytype) Tensor { + return Tensor.constant(idx_shape, .{ .i32 = 0 }); + } + }; + + const idx = Local._idx; + { // Only test shapes var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); @@ -2306,32 +2380,38 @@ pub const Tensor = struct { defer comp.deactivate(); inline for (.{ - .{ .{ .a = 10 }, .a, .{}, .{} }, - .{ .{ .a = 10 }, .a, .{ .n = 8 }, .{ .n = 8 } }, - .{ .{ .a = 10, .b = 20 }, .a, .{}, .{ .b = 20 } }, - .{ .{ .a = 10, .b = 20 }, .a, .{ .n = 8 }, .{ .n = 8, .b = 20 } }, - .{ .{ .a = 10, .b = 20 }, 0, .{ .n = 8 }, .{ .n = 8, .b = 20 } }, + .{ .{ .a = 10 }, .{ .a = idx(.{}) }, .{} }, + .{ .{ .a = 10 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{}) }, .{ .b = 20 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8, .b = 20 } }, + // .{ .{ .a = 10, .b = 20 }, 0, idx(.{ .n = 8 }), .{ .n = 8, .b = 20 } }, // Favor val shape, instead of indices shape. - .{ .{ .a = 10, .b = 20 }, .b, .{ .n = 8 }, .{ .a = 10, .n = 8 } }, - .{ .{ .a = 10, .b = 20, .c = 30 }, .b, .{ .n = 8 }, .{ .a = 10, .n = 8, .c = 30 } }, + .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .n = 8 }) }, .{ .a = 10, .n = 8 } }, + .{ .{ .a = 10, .b = 20, .c = 30 }, .{ .b = idx(.{ .n = 8 }) }, .{ .a = 10, .n = 8, .c = 30 } }, // batching axes are implicits. - .{ .{ .a = 10, .b = 20 }, .b, .{ .a = 10 }, .{ .a = 10 } }, - .{ .{ .a = 10, .b = 20 }, .a, .{ .b = 20 }, .{ .b = 20 } }, - .{ .{ .a = 10, .b = 20 }, .b, .{ .a = 10, .n = 8 }, .{ .a = 10, .n = 8 } }, + .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10 }) }, .{ .a = 10 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .b = 20 }) }, .{ .b = 20 } }, + .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .n = 8 } }, // stablehlo.gather is biased toward indices shape (like gatherSlice). // This make it awkward to use when you have both batching dimension and new indices dimensions. // For now we reject those, and let user explicitly transpose self or indices if needed. - // .{ .{ .a = 10, .b = 20 }, .b, .{ .n = 8, .a = 10 }, .{ .a = 10, .n = 8 } }, + // .{ .{ .a = 10, .b = 20 }, .{.b = idx(.{ .n = 8, .a = 10 })}, .{ .a = 10, .n = 8 } }, // Also handle tuples - .{ .{ .a = 10, .b = 20 }, .{ .a, .b }, .{ .n = 8, ._ = 2 }, .{ .n = 8 } }, - .{ .{ 10, 20 }, .{ -2, -1 }, .{ 8, 2 }, .{8} }, - // and 1-tuple - .{ .{ .a = 10, .b = 20 }, .{.b}, .{ .n = 8, ._ = 1 }, .{ .a = 10, .n = 8 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }), .b = idx(.{ .n = 8 }) }, .{ .n = 8 } }, }) |testcase| { - const x_shape, const tag, const idx_shape, const res_shape = testcase; + const x_shape, const indices, const res_shape = testcase; const x = Tensor.constant(x_shape, .{ .f16 = 0 }); - const idx = Tensor.constant(idx_shape, .{ .i32 = 0 }); - const y = gatherValues(x, tag, idx, .{}); + const y = gather(x, indices, .{}); + try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); + try std.testing.expect(y.value().owner().verify()); + } + + inline for (.{ + .{ .{ 10, 20 }, &[_]u3{ 0, 1 }, &[_]Tensor{ idx(.{8}), idx(.{8}) }, .{8} }, + }) |testcase| { + const x_shape, const idx_axes, const idx_per_axis, const res_shape = testcase; + const x = Tensor.constant(x_shape, .{ .f16 = 0 }); + const y = gather_(x, idx_axes, idx_per_axis, .{}); try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); try std.testing.expect(y.value().owner().verify()); } @@ -2966,13 +3046,24 @@ pub const Tensor = struct { } /// Returns a Tensor representing the result of Top-K over the given axis. - pub fn topK(self: Tensor, k: u32, axis_: anytype, opts: struct { descending: bool = true }) SortRes { - const a = self.axis(axis_); - const result = self.sort(a, .{ .descending = opts.descending }); - return .{ - .values = result.values.slice1d(a, .{ .end = k }), - .indices = result.indices.slice1d(a, .{ .end = k }), + pub fn topK(self: Tensor, named_axis_: anytype, k: u32, opts: struct { descending: bool = true }) SortRes { + const err_msg = "topK named axis should be an integer or a named axis, eg `x.topK(.{{ .best_token = .token }}, 16)` or `x.topK(-1, 16)`"; + const has_name: ?[:0]const u8, const a = switch (@typeInfo(@TypeOf(named_axis_))) { + .int, .comptime_int => .{ null, self.axis(@as(i64, @intCast(named_axis_))) }, + .@"struct" => |info| blk: { + stdx.debug.assertComptime(info.fields.len == 1, err_msg, .{}); + break :blk .{ info.fields[0].name, self.axis(@field(named_axis_, info.fields[0].name)) }; + }, + else => stdx.debug.compileError(err_msg, .{}), }; + var result = self.sort(a, .{ .descending = opts.descending }); + result.values = result.values.slice1d(a, .{ .end = k }); + result.indices = result.indices.slice1d(a, .{ .end = k }); + if (has_name) |new_name| { + result.values._shape._tags.set(a, new_name.ptr); + result.indices._shape._tags.set(a, new_name.ptr); + } + return result; } pub const MaxPoolRes = ArgMaxRes; @@ -3827,11 +3918,20 @@ pub const Tensor = struct { return binaryOpHelper(self, other.broad(self._shape)); } - stdx.debug.assert(self._shape.eql(other._shape), "{s} expects tensor shapes to match, got {f} and {f}", .{ op_name, self._shape, other._shape }); + var other_ = other; + var same_shape = self._shape.eql(other._shape); + if (!same_shape and std.mem.eql(Shape.Tag, self._shape.tags(), other._shape.tags()) and other._shape.canBroadcastTo(self._shape)) { + // Only a restrictive version of broadcasting is allowed here, where all the tags matches already. + // Typical use case: `x.div(x.sum(.a))` + same_shape = true; + other_ = other.broad(self._shape); + } + + stdx.debug.assert(same_shape, "{s} expects tensor shapes to match, got {f} and {f}", .{ op_name, self._shape, other._shape }); const ctx = self.getContext(); - const location = ctx.location(src, "{s}({f}, {f})", .{ op_name, self, other }); - const ret = @call(.auto, op_fn, .{ ctx.mlirCtx(), self.value(), other.value(), location }); + const location = ctx.location(src, "{s}({f}, {f})", .{ op_name, self, other_ }); + const ret = @call(.auto, op_fn, .{ ctx.mlirCtx(), self.value(), other_.value(), location }); return _result(self._shape, ret.result(0)); } }.binaryOpHelper; @@ -3904,21 +4004,14 @@ test "Tensor.maxPool1d" { const x = try zml.Buffer.fromSlice(platform, .{ 2, 2, 5 }, &data); const result = try zml.testing.compileAndCall(platform, MaxPool._fwd, .{x}); - try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2 }, .f32), result.values.shape()); - try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2 }, .i32), result.indices.shape()); - const buffer = result.values.getValue([2][2][2]f32); + try zml.testing.expectEqualShapes(.init(.{ 2, 2, 2 }, .f32), result.values.shape()); + try zml.testing.expectEqualShapes(.init(.{ 2, 2, 2 }, .i32), result.indices.shape()); try std.testing.expectEqualDeep( [2][2][2]f32{ - [2][2]f32{ - [2]f32{ 2, 4 }, - [2]f32{ 7, 9 }, - }, - [2][2]f32{ - [2]f32{ 12, 14 }, - [2]f32{ 17, 19 }, - }, + .{ .{ 2, 4 }, .{ 7, 9 } }, + .{ .{ 12, 14 }, .{ 17, 19 } }, }, - buffer, + result.values.getValue([2][2][2]f32), ); } diff --git a/zml/testing.zig b/zml/testing.zig index 851bd8c1..38952dd6 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -33,7 +33,7 @@ pub fn approxEq(comptime Float: type, l: Float, r: Float, tolerance: Float) bool return closeRel or closeAbs; } -/// Testing utility. Accepts both Tensor and HostBuffer but Tensor will be copied to the +/// Testing utility. Accepts both zml.Buffer and zml.HostBuffer but zml.Buffer will be copied to the /// host for comparison ! pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { const allocator = if (builtin.is_test) std.testing.allocator else std.heap.smp_allocator; diff --git a/zml/tokenizer/BUILD.bazel b/zml/tokenizer/BUILD.bazel index b7af9e29..012e218c 100644 --- a/zml/tokenizer/BUILD.bazel +++ b/zml/tokenizer/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_test") +load("@zml//bazel:zig_srcs.bzl", "zig_srcs") zig_library( name = "tokenizer", @@ -7,7 +8,6 @@ zig_library( main = "tokenizer.zig", visibility = ["//visibility:public"], deps = [ - "//async", "//ffi:zig", "//stdx", "//zml/tokenizer/hftokenizers", @@ -36,3 +36,8 @@ zig_test( "//stdx", ], ) + +zig_srcs( + name = "sources", + zig_lib = ":tokenizer", +) diff --git a/zml/tokenizer/sentencepiece/BUILD.bazel b/zml/tokenizer/sentencepiece/BUILD.bazel index dba86ddb..10ac080b 100644 --- a/zml/tokenizer/sentencepiece/BUILD.bazel +++ b/zml/tokenizer/sentencepiece/BUILD.bazel @@ -1,5 +1,6 @@ load("@rules_zig//zig:defs.bzl", "zig_library") load("//bazel:swig.bzl", "swig_cc_library") +load("//bazel:zig_srcs.bzl", "zig_srcs") swig_cc_library( name = "sentencepiece_swig", @@ -22,3 +23,13 @@ zig_library( "//stdx", ], ) + +zig_srcs( + name = "sources", + zig_lib = ":sentencepiece", +) + +cc_static_library( + name="sentencepiece_static", + deps = [":sentencepiece_swig"] +) diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig index c8dfb120..5109ce31 100644 --- a/zml/tokenizer/tokenizer.zig +++ b/zml/tokenizer/tokenizer.zig @@ -1,6 +1,5 @@ const std = @import("std"); -const async = @import("async"); const hftokenizers = @import("hftokenizers"); const sentencepiece = @import("sentencepiece"); @@ -98,15 +97,15 @@ pub const Tokenizer = union(Tokenizers) { pub fn fromFile(allocator: std.mem.Allocator, model: []const u8) !Tokenizer { if (std.mem.endsWith(u8, model, ".pb")) { - return .{ .sentencepiece = try async.callBlocking(sentencepiece.SentencePieceProcessor.fromFile, .{model}) }; + return .{ .sentencepiece = try sentencepiece.SentencePieceProcessor.fromFile(model) }; } if (std.mem.endsWith(u8, model, ".json")) { - return .{ .hftokenizers = try async.callBlocking(hftokenizers.HFTokenizer.fromFile, .{model}) }; + return .{ .hftokenizers = try hftokenizers.HFTokenizer.fromFile(model) }; } if (std.mem.endsWith(u8, model, ".tinyllama")) { const tokenizer = try allocator.create(homemade.Tokenizer); - tokenizer.* = try async.callBlocking(homemade.fromTinyLlamaFile, .{ allocator, model, 32000 }); + tokenizer.* = try homemade.fromTinyLlamaFile(allocator, model, 32000); return .{ .homemade = tokenizer }; } diff --git a/zml/torch.zig b/zml/torch.zig index e982aaec..09888aee 100644 --- a/zml/torch.zig +++ b/zml/torch.zig @@ -169,7 +169,7 @@ test pixelShuffle { /// /// Note: at the difference of Pytorch, shifts need to be explicitly repeated, even if they are the same for all axes. /// ref: https://pytorch.org/docs/stable/generated/torch.roll.html -pub fn roll(self: Tensor, shifts: []const i64, axes_: []const u8) Tensor { +pub fn roll(self: Tensor, shifts: []const i64, axes_: []const i8) Tensor { // TODO(hugo) accept following syntax: x.roll(.{ .a = 5, .b = 8 }) stdx.debug.assert(self.rank() > 0 and shifts.len == axes_.len, "Shifts length ({d}) and dims length ({d}) are not equal, we expect the same length.", .{ shifts.len, axes_.len }); @@ -180,11 +180,11 @@ pub fn roll(self: Tensor, shifts: []const i64, axes_: []const u8) Tensor { return roll(first_dim_rolled, tail_shifts, tail_dims); } - const a = axes_[0]; + const a = self.axis(axes_[0]); const start = @mod(self.dim(a) - shifts[0], self.dim(a)); const idx = Tensor.arange(.{ .start = start, .end = start + self.dim(a) }, .f32); const divisor: f32 = @floatFromInt(self.dim(a)); - return self.gatherValues(a, idx.fmod(divisor).convert(.i32), .{}); + return self.gather_(&.{a}, &.{idx.fmod(divisor).convert(.i32)}, .{}); } test roll { @@ -194,7 +194,7 @@ test roll { const res = try zml.testing.compileAndCall( platform, roll, - .{ input, &[_]i64{ 2, 1 }, &[_]u8{ 0, 1 } }, + .{ input, &[_]i64{ 2, 1 }, &[_]i8{ 0, 1 } }, ); const expectation = zml.HostBuffer.fromSlice(.{ 4, 2 }, &[_]f32{ 6, 5, 8, 7, 2, 1, 4, 3 }); @@ -274,3 +274,12 @@ test meshgrid { ); } } + +/// Flattens the given axis and the next one, into one new axis. +pub fn flatten(self: Tensor, axis_: anytype) Tensor { + const old_shape = self._shape; + const a = self.axis(axis_); + stdx.debug.assert(a + 1 < self.rank(), "Can't flatten {f} on the last axis {}.", .{ self, axis_ }); + const new_shape = old_shape.mergeAxis(a, .{ a, a + 1 }); + return self.reshape(new_shape); +} From e9b42f0ca86f91a7c9c9557846dca7faf2945919 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 25 Sep 2025 14:03:37 +0200 Subject: [PATCH 035/881] examples: gpt oss (#329) This implement support for OpenAI "gpt-oss" models. It's mostly a take over of #292 branch, but MoE code is kept inside examples folder cause it's still a bit warm. In particular the routing logic can break apart when there are undefined memory in the inputs, where it wasn't an issue for normal MLP. I only tested with the 20B version. --- examples/gpt_oss/BUILD.bazel | 16 + examples/gpt_oss/GptOss.zig | 741 +++++++++++++++++++++++++++++++++++ examples/gpt_oss/README.md | 19 + examples/gpt_oss/main.zig | 376 ++++++++++++++++++ stdx/fmt.zig | 54 +-- tools/zml_utils.py | 10 - zml/aio/safetensors.zig | 2 +- zml/hostbuffer.zig | 15 +- zml/nn.zig | 20 +- zml/tensor.zig | 79 ++-- 10 files changed, 1258 insertions(+), 74 deletions(-) create mode 100644 examples/gpt_oss/BUILD.bazel create mode 100644 examples/gpt_oss/GptOss.zig create mode 100644 examples/gpt_oss/README.md create mode 100644 examples/gpt_oss/main.zig diff --git a/examples/gpt_oss/BUILD.bazel b/examples/gpt_oss/BUILD.bazel new file mode 100644 index 00000000..7efba06c --- /dev/null +++ b/examples/gpt_oss/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_test") + +zig_binary( + name = "gpt_oss", + srcs = [ + "GptOss.zig", + ], + main = "main.zig", + deps = [ + "@com_github_hejsil_clap//:clap", + "@zml//async", + "@zml//stdx", + "@zml//zml", + ], + visibility = ["//visibility:public"], +) diff --git a/examples/gpt_oss/GptOss.zig b/examples/gpt_oss/GptOss.zig new file mode 100644 index 00000000..bce16daf --- /dev/null +++ b/examples/gpt_oss/GptOss.zig @@ -0,0 +1,741 @@ +///! GptOss architecture, using huggingface transformers naming. +///! Dimensions of activations: {.b, .s, .d} +const std = @import("std"); + +const stdx = @import("stdx"); +const zml = @import("zml"); + +const GptOss = @This(); +const log = std.log.scoped(.GptOss); + +pub const Config = struct { + bos_token_id: u32 = 199998, + eos_token_id: stdx.json.Union(union(enum) { + int: u32, + ints: []const u32, + }), + head_dim: u32, + num_hidden_layers: u32, + num_attention_heads: u32, + num_key_value_heads: u32, + experts_per_token: u32, + rope_theta: f32, + max_position_embeddings: u32, + rms_norm_eps: f32, + sliding_window: u32, + hf_rope_impl: bool = true, + rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = {} }, +}; + +pub const Options = struct { + sampling_strategy: zml.nn.SamplingStrategy, + max_seq_len: u32, + max_prompt_len: u32, + tokens_per_expert_ratio: f32, +}; + +pub const Mode = union(enum) { + /// In prefill mode we pass the actual len of the prompt + prefill: zml.Tensor, + /// In gen mode we pass the position of the next token + gen: zml.Tensor, +}; + +lm_head: ?zml.nn.Linear, +model: Model, + +config: Config, +options: Options, + +pub fn init(allocator: std.mem.Allocator, store: zml.aio.BufferStore, config: Config, options: Options) !GptOss { + var self: GptOss = .{ + .config = config, + .options = options, + .model = .{ + .max_seq_len = @intCast(options.max_seq_len), + .num_heads = @intCast(config.num_attention_heads), + .num_kv_heads = @intCast(config.num_key_value_heads), + .rope_opts = .{ + .layout = if (config.hf_rope_impl) .sequential else .interleaved, + .freq_base = config.rope_theta, + .scaling = config.rope_scaling, + }, + + .embed_tokens = .{ + .weight = store.getTensor("model.embed_tokens.weight").withSharding(.{1}), + }, + .layers = try allocator.alloc(TransformerLayer, config.num_hidden_layers), + .norm = .{ + .weight = store.getTensor("model.norm.weight"), + .eps = config.rms_norm_eps, + }, + }, + .lm_head = .{ .weight = store.getTensor("lm_head.weight").withSharding(.{0}) }, + }; + + var prefix: zml.aio.PrefixBuilder = try .initCapacity(allocator, 1024); + try prefix.push(stdx.noalloc, "model.layers"); + for (self.model.layers, 0..) |*layer, i| { + try prefix.pushDigit(stdx.noalloc, i); + defer prefix.pop(); + + var self_attn: SelfAttn = .{ + .sinks = store.getTensor(prefix.concat("self_attn.sinks")), + .q_proj = try zml.aio.populateModelWithPrefix(zml.nn.Linear, allocator, store, prefix.concat("self_attn.q_proj")), + .k_proj = try zml.aio.populateModelWithPrefix(zml.nn.Linear, allocator, store, prefix.concat("self_attn.k_proj")), + .v_proj = try zml.aio.populateModelWithPrefix(zml.nn.Linear, allocator, store, prefix.concat("self_attn.v_proj")), + .o_proj = try zml.aio.populateModelWithPrefix(zml.nn.Linear, allocator, store, prefix.concat("self_attn.o_proj")), + + .sliding_window = if (i % 2 == 0) config.sliding_window else null, + .num_heads = self.model.num_heads, + .num_kv_heads = self.model.num_kv_heads, + .rope_opts = self.model.rope_opts, + }; + + self_attn.q_proj.weight = self_attn.q_proj.weight.withSharding(.{0}); + self_attn.k_proj.weight = self_attn.k_proj.weight.withSharding(.{0}); + self_attn.v_proj.weight = self_attn.v_proj.weight.withSharding(.{0}); + self_attn.o_proj.weight = self_attn.o_proj.weight.withSharding(.{1}); + + const on_disk_moe = try zml.aio.populateModelWithPrefix(MoE.OnDisk, allocator, store, prefix.concat("mlp")); + var moe = on_disk_moe.rewrite(config.experts_per_token, options); + { + moe.experts.gate_up_proj.blocks = moe.experts.gate_up_proj.blocks.withSharding(.{.expert}); + moe.experts.down_proj.blocks = moe.experts.down_proj.blocks.withSharding(.{.expert}); + } + + layer.* = .{ + .input_layernorm = .{ + .weight = store.getTensor(prefix.concat("input_layernorm.weight")), + .eps = config.rms_norm_eps, + }, + .post_attention_layernorm = .{ + .weight = store.getTensor(prefix.concat("post_attention_layernorm.weight")), + .eps = config.rms_norm_eps, + }, + .self_attn = self_attn, + .mlp = moe, + }; + } + + // TODO(Corentin): Fix lm_head sharding when top-k sampling is enabled. + // It currently crashes/compilation fails + if (self.options.sampling_strategy.topk == 1 and self.lm_head != null) { + self.lm_head.?.weight = self.lm_head.?.weight.withSharding(.{0}); + } + + return self; +} + +/// Predicts the token at `token_index` position. +/// Returns: +/// - updated `tokens`, +/// - updated KV cache +/// - a Rng state to allow for probabilistic generation +pub fn forward( + self: GptOss, + tokens_: zml.Tensor, + mode: Mode, + kv_cache: KvCache, + rng: zml.Tensor.Rng, +) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { + const tokens = tokens_.withPartialTags(.{.s}); + + // token index is the position in the kv cache where to write results. + const token_index: zml.Tensor = switch (mode) { + .gen => |token_index| token_index, + .prefill => .scalar(0, .u32), + }; + var out, const updated_kv_cache = zml.call(self.model, .forward, .{ tokens, token_index, kv_cache }); + + switch (mode) { + // In prefill we only pass the last token to the lm head. + .prefill => |prompt_len| out = out.gather(.{ .s = prompt_len.convert(.i32).addConstant(-1) }, .{ .indices_are_sorted = true }), + .gen => {}, + } + + var new_token, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.options.sampling_strategy); + new_token = new_token.convert(.u32); + new_token = switch (mode) { + .gen => new_token.reuseBuffer(tokens), + .prefill => new_token.appendAxes(.{.s}), + }; + return .{ new_token, updated_kv_cache, new_rng }; +} + +fn sampleTokens( + self: GptOss, + lm_head_: ?zml.nn.Linear, + out_: zml.Tensor, + rng: zml.Tensor.Rng, + opts: zml.nn.SamplingStrategy, +) struct { zml.Tensor, zml.Tensor.Rng } { + const out = out_.withPartialTags(.{.d}); + + var logits = blk: { + if (lm_head_) |lm_head| { + break :blk zml.call(lm_head, .forward, .{out}); + } else { + break :blk self.model.embed_tokens.weight.withTags(.{ .voc, .d }).dot(out, .{.d}); + } + }; + + if (logits.shape().hasTag(.voc) == null) + logits = logits.rename(.{ .d = .voc }); + + const next_tokens, const new_rng = zml.nn.sampleTokens(logits, opts, rng); + return .{ next_tokens, new_rng }; +} + +pub fn loadBuffers(self: GptOss, allocator: std.mem.Allocator, store: zml.aio.BufferStore, platform: zml.Platform) !zml.Bufferized(GptOss) { + var prefix: zml.aio.PrefixBuilder = try .initCapacity(allocator, 256); + defer prefix.deinit(allocator); + + const noalloc = stdx.noalloc; + const loaded: zml.Bufferized(GptOss) = .{ + .model = .{ + .embed_tokens = try store.loadModelById(zml.nn.TokenEmbedding, noalloc, self.model.embed_tokens, platform), + .layers = try allocator.alloc(zml.Bufferized(TransformerLayer), self.model.layers.len), + .norm = try store.loadModelById(RmsNorm, noalloc, self.model.norm, platform), + }, + .lm_head = try store.loadModelById(?zml.nn.Linear, noalloc, self.lm_head, platform), + }; + + prefix.push(noalloc, "model.layers") catch unreachable; + for (loaded.model.layers, self.model.layers, 0..) |*d_layer, layer, layer_id| { + const ckpt = prefix.checkpoint(); + defer prefix.restore(ckpt); + + prefix.pushDigit(noalloc, layer_id) catch unreachable; + d_layer.* = .{ + .input_layernorm = try store.loadModelById(RmsNorm, noalloc, layer.input_layernorm, platform), + .self_attn = try store.loadModelById(SelfAttn, noalloc, layer.self_attn, platform), + .post_attention_layernorm = try store.loadModelById(RmsNorm, noalloc, layer.post_attention_layernorm, platform), + .mlp = try store.loadModelById(MoE, noalloc, layer.mlp, platform), + }; + } + + return loaded; +} + +pub const Model = struct { + embed_tokens: zml.nn.TokenEmbedding, + norm: RmsNorm, + layers: []TransformerLayer, + + max_seq_len: u32 = 0, + num_heads: i64 = 32, + num_kv_heads: i64 = 32, + rope_opts: zml.nn.RopeOpts = .{ + .layout = .interleaved, + .freq_base = 10_000, + }, + + /// Forward one token, using KV cache for previous tokens. + /// Returns result and updated KV cache. + pub fn forward(self: Model, tokens: zml.Tensor, token_index: zml.Tensor, kv_cache: KvCache) struct { zml.Tensor, KvCache } { + const embeds = embed(self.embed_tokens, tokens); + var hidden = embeds; + + var updated_kv_cache = kv_cache; + for (self.layers, 0..) |layer, i| { + hidden, updated_kv_cache = zml.call(layer, .forward, .{ hidden, token_index, updated_kv_cache.atLayer(i) }); + } + const output = zml.call(self.norm, .forward, .{hidden}); + + return .{ output, updated_kv_cache.reuseBuffer(kv_cache) }; + } + + pub fn embed(embed_tokens_: zml.nn.TokenEmbedding, tokens_: zml.Tensor) zml.Tensor { + return zml.call(embed_tokens_, .forward, .{tokens_}).withPartialTags(.{.d}); + } +}; + +pub const TransformerLayer = struct { + input_layernorm: RmsNorm, + self_attn: SelfAttn, + post_attention_layernorm: RmsNorm, + mlp: MoE, + + pub fn forward( + self: TransformerLayer, + x0: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache, + ) struct { zml.Tensor, KvCache } { + // Self Attention + //log.debug("TransformerLayer({}) -> {}", .{ x0, self.input_layernorm.forward(x0) }); + stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {f}", .{x0}); + + const x0_normalized = zml.call(self.input_layernorm, .forward, .{x0}); + const delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); + const x1 = x0.add(delta0); + + // Fully Connected + const x1_normalized = zml.call(self.post_attention_layernorm, .forward, .{x1}); + const x2 = zml.call(self.mlp, .forward, .{x1_normalized}).add(x1); + + return .{ x2.reuseBuffer(x0), updated_kv_cache }; + } +}; + +const RmsNorm = struct { + weight: zml.Tensor, + eps: f32 = 1e-6, + + /// L2 normalization of input tensor along `.d` axis. + pub fn forward(self: RmsNorm, input: zml.Tensor) zml.Tensor { + const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); + // Note: contrary to Llama here the full layer is done in .f32, not just the variance computation. + const normalized = zml.nn.rmsNorm(x.convert(.f32), .d, self.eps); + return normalized.mul(self.weight.convert(.f32).withTags(.{.d}).broad(x.shape())).convert(input.dtype()); + } +}; + +const MoE = struct { + experts: Mlp, + router: zml.nn.Linear, + moe_opts: MoeOpts, + + pub fn forward(self: MoE, input: zml.Tensor) zml.Tensor { + log.warn("compiling moe with {f}", .{input}); + // Note: GptOss applies softmax on the routing score. + // We delay the softmax to mixtureOfExperts where the actual routing is done. + // This allow to do re-routing without introducing nans. + const gating = self.router.forward(input); + return mixtureOfExperts(Mlp, self.experts, input, gating, self.moe_opts); + } + + pub const OnDisk = struct { + router: zml.nn.Linear, + experts: struct { + down_proj_bias: zml.Tensor, + down_proj_blocks: zml.Tensor, + down_proj_scales: zml.Tensor, + gate_up_proj_bias: zml.Tensor, + gate_up_proj_blocks: zml.Tensor, + gate_up_proj_scales: zml.Tensor, + }, + + pub fn rewrite(on_disk: OnDisk, experts_per_token: u32, options: Options) MoE { + const e = on_disk.experts; + return .{ + .experts = .{ + .gate_up_proj = .{ + // We need to bitcast the scale cause safetensors doesn't encode f8 types correctly + .scale = e.gate_up_proj_scales.withTags(.{ .expert, .out, .d }), + // We don't bitcast here because PJRT doesn't handle packed host buffers + .blocks = e.gate_up_proj_blocks.withTags(.{ .expert, .out, .d, .d_block }), + .blocks_dtype = .f4e2m1, + .bias = e.gate_up_proj_bias.withTags(.{ .expert, .d }), + }, + .down_proj = .{ + .blocks = e.down_proj_blocks.withTags(.{ .expert, .out, .d, .d_block }), + .blocks_dtype = .f4e2m1, + .scale = e.down_proj_scales.withTags(.{ .expert, .out, .d }), + .bias = e.down_proj_bias.withTags(.{ .expert, .d }), + }, + }, + + .router = .{ + .weight = on_disk.router.weight.withTags(.{ .expert, .d }), + .bias = on_disk.router.bias.?.withTags(.{.expert}), + }, + + .moe_opts = .{ + .experts_per_token = experts_per_token, + .tokens_per_expert_ratio = options.tokens_per_expert_ratio, + .normalization = .softmax, + }, + }; + } + }; +}; + +pub const Mlp = struct { + gate_up_proj: BlockScaledLinear, // {.out = intermediate_size * 2, .d = hidden_size / block_size, .d_block = block_size } + down_proj: BlockScaledLinear, // {.out = hidden_size * 2, .d = intermediate_size / block_size, .d_block = block_size } + + pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { + const dt = x.dtype(); + var gate, var up = zml.nn.splitRealImg(self.gate_up_proj.forward(x), .interleaved); + gate = .minimum(gate, .scalar(7, dt)); + up = .clamp(up, .scalar(-7, dt), .scalar(7, dt)); + + const out = gate.quickGelu().mul(up.addConstant(1)); + return zml.call(self.down_proj, .forward, .{out}); + } + + pub fn format(self: Mlp, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("Mlp(gate_up_proj=.{f}, down_proj=.{f})", .{ self.gate_up_proj, self.down_proj }); + } +}; + +pub const SelfAttn = struct { + q_proj: zml.nn.Linear, + k_proj: zml.nn.Linear, + v_proj: zml.nn.Linear, + sinks: zml.Tensor, + + o_proj: zml.nn.Linear, + + sliding_window: ?u32, + num_heads: i64, + num_kv_heads: i64, + rope_opts: zml.nn.RopeOpts, + + /// Self Attention. + /// - If token_index is set, x is assumed to be the representation of one new token, + /// and kv_cache will be read for the previous tokens. + /// - If token_index is not set, x is assumed to be the representation of all tokens + /// since the beginning of the sequence, and kv_cache won't be read. + /// In both case, kv_cache will be updated with the computed key and value. + /// x: {.b, .s, .d } -> .{.b, .s, .d} + pub fn forward( + self: SelfAttn, + x: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache, + ) struct { zml.Tensor, KvCache } { + const num_kv_heads = self.num_kv_heads; + var q = zml.call(self.q_proj, .forward, .{x}).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }).withSharding(.{.h}); + var k = zml.call(self.k_proj, .forward, .{x}).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{.h}); + var v = zml.call(self.v_proj, .forward, .{x}).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{.h}); + + // Generate the attention mask. + const seq_len = kv_cache.k.dim(.k); + var attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, x.dtype(), self.sliding_window); + + // Note: in Pytorch it would be very inefficient to generate the full attn_mask, + // then slice into it, but XLA is able to optimize this correctly. + attn_mask = attn_mask.gatherSlices(zml.Shape.init(.{ .q = x.dim(.s) }, attn_mask.dtype()), token_index.reshape(.{ .coord = 1 }), .{}); + + // In self-attention, .s axis is used both for keys and queries. + const pos_index = b: { + const temp = zml.Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()).withTags(.{.s}).broad(zml.Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); + break :b temp.add(token_index.broad(temp.shape())); + }; + + q = zml.nn.rope(q, pos_index, self.rope_opts); + k = zml.nn.rope(k, pos_index, self.rope_opts); + q = q.rename(.{ .s = .q }); + k = k.rename(.{ .s = .k }); + v = v.rename(.{ .s = .k }); + + const dtype = q.dtype(); + const new_kv_cache = kv_cache.update(k, v, token_index); + k = new_kv_cache.keys().convert(dtype); + v = new_kv_cache.values().convert(dtype); + + // TODO ringbuffer kv cache. + + const softmax_bias = self.sinks.withTags(.{.h}); + const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask, .softmax_bias = softmax_bias }); + const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); + return .{ zml.call(self.o_proj, .forward, .{attn}), new_kv_cache }; + } +}; + +pub const KvCache = struct { + k: zml.Tensor, + v: zml.Tensor, + layer_index: zml.Tensor, + + pub fn init(kv_shape: zml.Shape) KvCache { + // The KV-cache is initialized with ones to detect reads of uninitialized memory. + return .{ + .k = .constant(kv_shape, kv_shape.dtype().one()).withSharding(.{.h}), + .v = .constant(kv_shape, kv_shape.dtype().one()).withSharding(.{.h}), + .layer_index = .scalar(-1, .u32), + }; + } + + pub fn initShape(kv_shape: zml.Shape) zml.ShapeOf(KvCache) { + return .{ + .k = kv_shape, + .v = kv_shape, + .layer_index = zml.Shape.init(.{}, .u32), + }; + } + + pub fn initBuffer(kv_shape: zml.Shape, platform: zml.Platform) !zml.Bufferized(KvCache) { + return .{ + .k = try zml.Buffer.uninitialized(platform, kv_shape, .{}), + .v = try zml.Buffer.uninitialized(platform, kv_shape, .{}), + .layer_index = try zml.Buffer.uninitialized(platform, .scalar(.u32), .{}), + }; + } + + pub fn keys(self: KvCache) zml.Tensor { + return self.k.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn values(self: KvCache) zml.Tensor { + return self.v.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn update(self: KvCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: ?zml.Tensor) KvCache { + const idx = if (token_index) |idx| idx else zml.Tensor.arange(.{ .end = new_k.dim(.k) }, .u32); + + return .{ + .k = self.k.scatterSlices( + .{ .k = idx, .layer = self.layer_index }, + new_k.convert(self.k.dtype()), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.k), + .v = self.v.scatterSlices( + .{ .k = idx, .layer = self.layer_index }, + new_v.convert(self.v.dtype()), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.v), + .layer_index = self.layer_index, + }; + } + + pub fn atLayer(self: KvCache, layer_index: usize) KvCache { + return .{ + .k = self.k, + .v = self.v, + .layer_index = .scalar(layer_index, .u32), + }; + } + + pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { + return .{ + .k = self.k.reuseBuffer(other.k), + .v = self.v.reuseBuffer(other.v), + .layer_index = self.layer_index.reuseBuffer(other.layer_index), + }; + } +}; + +pub const BlockScaledLinear = struct { + blocks: zml.Tensor, + scale: zml.Tensor, + bias: ?zml.Tensor = null, + blocks_dtype: zml.DataType, + + pub fn forward(self: BlockScaledLinear, x: zml.Tensor) zml.Tensor { + const ctx = x.getContext(); + const res_shape = x.shape().setDim(-1, self.blocks.dim(-3)); + + // Bitcast to our actual type. This allows to load weights in a packed layout. + const blocks_0 = self.blocks.bitCast(self.blocks_dtype); + const blocks = blocks_0.merge(.{ .d_block = .{ .d_block, .bitcast } }); + + const scale = self.scale.bitCast(.f8e8m0); + + // log.warn("BlockScaledLinear({}): {f} -> {f}", .{ self, x, res_shape }); + const y = switch (ctx._platform.target) { + else => y: { + var dequantized_weight: zml.Tensor = .mul( + blocks.convert(x.dtype()), + scale.convert(x.dtype()).appendAxes(.{.d_block}), + ); + var y = x.dot(dequantized_weight.merge(.{ .d = .{ .d, .d_block } }), .{.d}); + // std.log.warn("output shape: {f}", .{y}); + std.debug.assert(y.shape().eql(res_shape)); + y._shape = res_shape; + break :y y; + }, + }; + return if (self.bias) |bias| y.add(bias.broad(y.shape())) else y; + } + + pub fn format(self: BlockScaledLinear, writer: *std.Io.Writer) !void { + try writer.print("BlockScaledLinear(blocks={f}, scale={f}, bias={?f}, dt={t})", .{ self.blocks, self.scale, self.bias, self.blocks_dtype }); + } +}; + +const MoeOpts = struct { + experts_per_token: u32, + tokens_per_expert_ratio: ?f32 = 0.0, + normalization: Normalization, + + pub const Normalization = enum { linear, softmax }; +}; + +/// We have three algorithms, +/// * one for single-stream inference (naive), +/// * one for small batch sized with exact precision that sends all tokens to all experts. +/// this isn't too costly as long as the batch size is small and the experts are IO bound. +/// * one for big batch size that assign a fixed compute budget per expert and +/// experts chose the tokens they want to handle. This introduces noise since it's possible +/// a token doesn't get their requested expert. +/// The parameter `tokens_per_expert_ratio` control how much compute budget is granted: +/// expert_budget = ratio * (num_tokens * experts_per_token / num_experts). +/// Bigger values of ratio will ensure it's rare a token doesn't get it's top 2 tokens. +/// +/// The preferred algorithm is the batched one, +/// it is selected as soon there is enough tokens to guarantee that experts will be active most of the time. +/// +/// - input: .{ .s, .d } per-entry vector +/// - gating: .{ .s, .expert } per-entry expert-affinity +/// - experts: .{ .expert, .d_out, .d } expert layer (need to have a .forward method). +/// -> output: .{ .s, .d_out } +pub fn mixtureOfExperts(Expert: type, experts: Expert, input: zml.Tensor, gating: zml.Tensor, opts: MoeOpts) zml.Tensor { + log.warn("mixtureOfExperts({s}, {f}, {f}, {})", .{ @typeName(Expert), input, gating, opts }); + const num_tokens: u32 = @intCast(input.dim(.s)); + const num_experts = gating.dim(.expert); + stdx.debug.assert(opts.experts_per_token > 0, "mixtureOfExperts expects opts.experts_per_token > 0, got {}", .{opts}); + + if (num_tokens == 1) { + return moePerTokenRouting(Expert, experts, input, gating, opts); + } + + const tokens_per_expert: u32 = if (opts.tokens_per_expert_ratio) |ratio| tpe: { + const compute_budget = ratio * @as(f32, @floatFromInt(num_tokens * opts.experts_per_token)); + var tpe: u32 = @intFromFloat(stdx.math.divFloat(f32, compute_budget, num_experts)); + // Round to next multiple of 8 to avoid weird shapes. + if (tpe % 8 != 0) tpe += 8 - (tpe % 8); + break :tpe tpe; + } else num_tokens; + + if (3 * tokens_per_expert <= 2 * num_tokens) { + const routing, const tokens_ids_per_expert = dispatchTokens(gating, .{ + .tokens_per_expert = tokens_per_expert, + .experts_per_token = opts.experts_per_token, + .normalization = opts.normalization, + }); + const scores_per_expert = routing.transpose(.{ .expert, .s }).gather(.{ .s = tokens_ids_per_expert }, .{}); + const input_per_expert = input.gather(.{ .s = tokens_ids_per_expert }, .{}); + var output_per_expert = experts.forward(input_per_expert); + output_per_expert = output_per_expert.mul(scores_per_expert.convert(output_per_expert.dtype()).broad(output_per_expert.shape())); + + // Reverse engineer the normal output shape that one expert would have produced for all tokens. + // If this fall short, we could use the "sliced_expert" strategy and call forward ourselves. + const output_shape = output_per_expert.shape().drop(.expert).rename(.{ .top_token = .s }).setDim(.s, num_tokens); + const output = zml.Tensor.scatterSlices( + .constant(output_shape, output_shape.dtype().zero()), + .{ .s = tokens_ids_per_expert }, + output_per_expert, + .{ .update_fn = zml.Tensor.ScatterOpts.increment }, + ); + + log.warn("mixtureOfExperts({s}, {f}, {f}) -> fixed budget impl tpe: {d}, tokens: {d}", .{ @typeName(Expert), input, gating, tokens_per_expert, num_tokens }); + return output; + } else { + return mixtureOfExpertsAllToAll(Expert, experts, input, gating, opts); + } +} + +/// Few tokens: most experts are unused, experts have at most one token. +/// Select active experts and compute with that. +pub fn moePerTokenRouting(Expert: type, experts: Expert, input: zml.Tensor, gating: zml.Tensor, opts: MoeOpts) zml.Tensor { + const num_tokens: u32 = @intCast(input.dim(.s)); + stdx.debug.assert(num_tokens < 32, "Trying to unroll a lot of tokens !", .{}); + const per_token_outputs = input.getContext().allocator().alloc(zml.Tensor, num_tokens) catch @panic("OOM"); + + const routing = gating.topK(.{ .top_expert = .expert }, opts.experts_per_token, .{}); + const per_token_score = switch (opts.normalization) { + .linear => routing.values.div(routing.values.sum(.top_expert)), + .softmax => routing.values.softmax(.top_expert), + }; + + for (per_token_outputs, 0..num_tokens) |*output, tok_id| { + for (0..opts.experts_per_token) |expert_rank| { + const expert_id = routing.indices.choose(.{ .s = tok_id, .top_expert = expert_rank }).asScalar(); + const expert_score = per_token_score.choose(.{ .s = tok_id, .top_expert = expert_rank }).asScalar(); + + var sliced_expert: Expert = undefined; + zml.meta.mapAlloc(struct { + pub fn cb(expert_id_: zml.Tensor, expert_weight: zml.Tensor) zml.Tensor { + return expert_weight.gather(.{ .expert = expert_id_ }, .{}); + } + }.cb, stdx.noalloc, expert_id, experts, &sliced_expert) catch unreachable; + + // TODO how does this work when the two experts are on different gpus? + // does the compute overlap ? + var expert_output = sliced_expert.forward(input.choose(.{ .s = tok_id })); + expert_output = .mul( + expert_output, + expert_score.convert(input.dtype()).broad(expert_output.shape()), + ); + output.* = if (expert_rank > 0) output.add(expert_output) else expert_output; + } + } + + log.warn("mixtureOfExperts({s}, {f}, {f}) -> single-stream impl", .{ @typeName(Expert), input, gating }); + return .stack(per_token_outputs, 0, .s); +} + +/// Send all tokens to all experts, and apply gating. +pub fn mixtureOfExpertsAllToAll(Expert: type, experts: Expert, input: zml.Tensor, gating: zml.Tensor, opts: MoeOpts) zml.Tensor { + log.warn("mixtureOfExperts({s}, {f}, {f}) -> all to all impl", .{ @typeName(Expert), input, gating }); + const num_experts = gating.dim(.expert); + const hard_gating = hardGating(gating, opts).print(); + // TODO: `input.insertAxes(0, .{.expert}).repeat1d(.expert, num_experts)` is too verbose for just broadcasting along a new axis` + const output_per_expert = experts.forward(input.insertAxes(0, .{.expert}).repeat1d(.expert, @intCast(num_experts))); + return output_per_expert.dot(hard_gating.convert(input.dtype()), .expert); +} + +/// Given `(token, expert) -> scores`, +/// keeps only the top-k expert per token, and normalize the scores accordingly. +/// Non selected experts will have a 0 score. +pub fn hardGating(gating: zml.Tensor, opts: MoeOpts) zml.Tensor { + const routing = gating.topK(.{ .top_expert = .expert }, opts.experts_per_token, .{}); + + const per_token_score = switch (opts.normalization) { + .linear => routing.values.div(routing.values.sum(.top_expert)), + .softmax => routing.values.softmax(.top_expert), + }; + + return zml.Tensor.scatterSlices( + .zeroes(gating.shape()), + .{ .expert = routing.indices }, + per_token_score, + .{ .indices_are_unique = true }, + ); +} + +/// Lot of tokens, each experts chose their tokens. +/// It means that some tokens may have only one expert assigned. +/// Each token will get assigned to at least one expert IIF the input gating is sums up to 1 (typically softmax output). +/// Returns the actual `(token, expert) -> scores` used. +pub fn dispatchTokens( + gating: zml.Tensor, + opts: struct { + tokens_per_expert: u32, + experts_per_token: u32, + normalization: MoeOpts.Normalization, + }, +) [2]zml.Tensor { + const num_experts = gating.dim(.expert); + + const token_pref = gating.argsort(.expert, .{ .descending = true }); + var expert_rank: zml.Tensor = .scatterSlices( + .zeroes(gating.shape().withDtype(.i32)), + .{ .expert = token_pref }, + .addConstant(.iota(gating.shape(), .expert), 1), + .{ .indices_are_unique = true }, + ); + // The pow(expert_rank) here means that we strongly favor top 1 over top 2 and top 2 over top 3. + // expert_routing: (expert, top_token) -> token + const expert_routing = gating.pow(expert_rank.convert(gating.dtype())).topK(.{ .top_token = .s }, opts.tokens_per_expert, .{}); + const scores_per_expert = gating.gather(.{ .s = expert_routing.indices }, .{}); + + // Update the gating coefficient to account for the expert routing. + // Each (token, expert) which can't be computed within the given budget is left to 0. + const gating_v2: zml.Tensor = .scatterSlices( + .zeroes(gating.shape()), + .{ .s = expert_routing.indices }, + scores_per_expert, + .{ .indices_are_unique = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ); + // Now set to zero the scores (token, expert) for tokens that have been assigned more than experts_per_token. + const lowest_experts = gating_v2.topK(.{ .top_expert = .expert }, @intCast(num_experts - opts.experts_per_token), .{ .descending = false }); + var gating_v3: zml.Tensor = .scatterSlices( + gating_v2, + .{ .expert = lowest_experts.indices }, + .zeroes(lowest_experts.values.shape()), + .{ .indices_are_unique = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ); + // Then normalize so the sum of experts scores for one token sums up to 1. + gating_v3 = switch (opts.normalization) { + .linear => gating_v3.div(gating_v3.sum(.expert)), + .softmax => gating_v3.softmax(.expert), + }; + const tokens_ids_per_expert = expert_routing.indices.transpose(.{ .expert, .top_token }); + + return .{ gating_v3, tokens_ids_per_expert }; +} diff --git a/examples/gpt_oss/README.md b/examples/gpt_oss/README.md new file mode 100644 index 00000000..d569d1c9 --- /dev/null +++ b/examples/gpt_oss/README.md @@ -0,0 +1,19 @@ +# GptOss + +Download the model (you can change 20b to 120b for the bigger version if you have the required hardware). + +``` +bazel run //tools/hf -- download openai/gpt-oss-20b --local-dir /tmp/models/openai/gpt-oss-20b --exclude metal/ original/ +``` + +Run the model, use `--@zml//runtimes:xxx` flags to target the relevant platform for your hardware. + +``` +bazel run --config=release //examples/gpt_oss \ + --@zml//runtimes:cuda=true \ + --@zml//runtimes:cpu=false \ + --@zml//runtimes:tpu=false \ + --@zml//runtimes:rocm=false \ + -- --hf-model-path=/tmp/models/openai/gpt-oss-20b/ \ + --prompt='What are some spooky funfacts about animal for a Halloween party ?' +``` diff --git a/examples/gpt_oss/main.zig b/examples/gpt_oss/main.zig new file mode 100644 index 00000000..72aead20 --- /dev/null +++ b/examples/gpt_oss/main.zig @@ -0,0 +1,376 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const async = @import("async"); +const clap = @import("clap"); +const stdx = @import("stdx"); +const zml = @import("zml"); +const Buffer = zml.Buffer; +const Tensor = zml.Tensor; +const ShapeOf = zml.ShapeOf; + +const GptOss = @import("GptOss.zig"); + +const log = std.log.scoped(.GptOss); + +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = async.logFn(std.log.defaultLog), +}; + +const cli_params = clap.parseParamsComptime( + \\--help print this help + \\--prompt the prompt + \\--hf-model-path path to the directory containing model weights, config and tokenizer + \\--seed random seed (optional) + \\--seq-len max sequence length + \\--prompt-len max prompt length + \\--temperature temperature (default 1.0) + \\--topk topk (default 10) + \\--expert-budget token budget per expert + \\--platform-options platform options, using Zon syntax, eg '.{.cuda=.{.allocator=.{.async=.{.memory_fraction=0.95}}}}' + \\--nochat skip prompt template + \\--sharding default: true: sharding on or off +); + +pub fn tokenizePrompt(tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8, no_chat: bool, out: []u32) ![]u32 { + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + if (no_chat) { + const tokens = try encoder.encode(prompt); + if (tokens.len > out.len) return error.PromptTooLong; + @memcpy(out[0..tokens.len], tokens); + return out[0..tokens.len]; + } + + const start_header = tokenizer.tokenToId("<|start|>") orelse return error.NoSuchToken; + const end_header_start_message = tokenizer.tokenToId("<|message|>") orelse return error.NoSuchToken; + const end_message = tokenizer.tokenToId("<|end|>") orelse return error.NoSuchToken; + + var tokens: std.ArrayList(u32) = .initBuffer(out); + + const system_prompt = try encoder.encode("You are ChatGPT, a large language model trained by OpenAI.\n"); + if (system_prompt.len + 4 > tokens.unusedCapacitySlice().len) return error.PromptTooLong; + tokens.appendSliceAssumeCapacity(&.{ start_header, tokenizer.tokenToId("system").?, end_header_start_message }); + tokens.appendSliceAssumeCapacity(system_prompt); + tokens.appendAssumeCapacity(end_message); + + const user_prompt = try encoder.encode(prompt); + if (user_prompt.len + 9 > tokens.unusedCapacitySlice().len) return error.PromptTooLong; + tokens.appendSliceAssumeCapacity(&.{ start_header, tokenizer.tokenToId("user").?, end_header_start_message }); + tokens.appendSliceAssumeCapacity(user_prompt); + tokens.appendSliceAssumeCapacity(&.{ + end_message, + start_header, + tokenizer.tokenToId("assistant").?, + tokenizer.tokenToId("<|channel|>") orelse return error.NoSuchToken, + tokenizer.tokenToId("analysis") orelse return error.NoSuchToken, + end_header_start_message, + }); + + return tokens.items; +} + +pub fn generateText( + config: GptOss.Config, + options: GptOss.Options, + mod_prefill: zml.ModuleExe(GptOss.forward), + mod_generate: zml.ModuleExe(GptOss.forward), + kv_cache_: zml.Bufferized(GptOss.KvCache), + tokenizer: zml.tokenizer.Tokenizer, + allocator: std.mem.Allocator, + seed: u128, + prompt_tok: []const u32, + output: *std.Io.Writer, +) !void { + var tokenizer_decoder = try tokenizer.decoder(); + defer tokenizer_decoder.deinit(); + + const platform = mod_generate.platform(); + + // init RNG and buffers + var rng = try zml.Tensor.Rng.init(platform, seed); + var generated_token_buffer = [_]u32{undefined}; + + var current_token, var kv_cache = prefill: { + // prepare device buffers for the prefill tokens and their positions + const prefill_buffer = try allocator.alloc(u32, options.max_prompt_len); + @memcpy(prefill_buffer[0..prompt_tok.len], prompt_tok); + + var prefill_tokens = try zml.Buffer.fromSlice(platform, .{options.max_prompt_len}, prefill_buffer); + defer prefill_tokens.deinit(); + var prefill_token_pos = try zml.Buffer.scalar(platform, prompt_tok.len, .u32); + defer prefill_token_pos.deinit(); + + const first_token, const kv_cache, rng = mod_prefill.call(.{ prefill_tokens, .{ .prefill = prefill_token_pos }, kv_cache_, rng }); + + // extract the first generated token + _ = try first_token.toHost(std.mem.sliceAsBytes(&generated_token_buffer)); + log.warn("first_token: {d}", .{generated_token_buffer[0]}); + break :prefill .{ first_token, kv_cache }; + }; + defer zml.aio.unloadBuffers(&kv_cache); + defer current_token.deinit(); + + const output_tokens_len = options.max_seq_len - prompt_tok.len - 1; + const start = std.time.microTimestamp(); + + // One token has already been generated by the prefill. + var num_tokens_generated: usize = 1; + + generation: for (0..output_tokens_len + 1) |i| { + // collect and print generated sequence + num_tokens_generated += 1; + const generated_token = generated_token_buffer[0]; + if (try tokenizer_decoder.next(generated_token)) |chunk| { + try output.writeAll(chunk); + } + + // check for eos + if (i == output_tokens_len) break :generation; + switch (config.eos_token_id.value) { + .int => |eos| if (generated_token == @as(u32, @intCast(eos))) break :generation, + .ints => |eos_list| { + for (eos_list) |eos| { + if (generated_token == @as(u32, @intCast(eos))) break :generation; + } + }, + } + + // current token pos needs to go into a zml.Buffer + const token_pos_buffer = &[_]u32{@intCast(prompt_tok.len + i)}; + const token_pos = try zml.Buffer.fromSlice(platform, .{}, token_pos_buffer); + defer token_pos.deinit(); + + // call to generate the next token + current_token, kv_cache, rng = mod_generate.call(.{ current_token, .{ .gen = token_pos }, kv_cache, rng }); + + // extract the generated token from the buffer + _ = try current_token.toHost(std.mem.sliceAsBytes(&generated_token_buffer)); + } + const end = std.time.microTimestamp(); + const duration = stdx.math.divFloat(f64, end - start, std.time.us_per_s); + const speed = @as(f64, @floatFromInt(num_tokens_generated)) / duration; + + log.info("✅ Generated {d} tokens in {:.3}s: {d:.3}tok/s", .{ num_tokens_generated, duration, speed }); +} + +pub fn main() !void { + try async.AsyncThread.main(std.heap.smp_allocator, asyncMain); +} + +pub fn asyncMain() !void { + log.info(" GptOss was compiled with {}", .{@import("builtin").mode}); + + var allocator: std.mem.Allocator = alloc: { + if (builtin.mode == .Debug) { + var dbg_alloc: std.heap.DebugAllocator(.{ + .never_unmap = true, + .retain_metadata = true, + }) = .init; + break :alloc dbg_alloc.allocator(); + } + break :alloc std.heap.smp_allocator; + }; + + const cli = ClapBoilerplate.parseCli(allocator); + defer cli.deinit(); + + const hf_model_path = cli.args.@"hf-model-path" orelse { + log.err("Missing --hf-model-path", .{}); + return; + }; + + const config = config: { + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + + const model_config_path = try std.fs.path.join(arena.allocator(), &.{ hf_model_path, "config.json" }); + + var config_json_file = try async.File.open(model_config_path, .{ .mode = .read_only }); + defer config_json_file.close() catch unreachable; + + var config_reader = config_json_file.reader(try arena.allocator().alloc(u8, 256)); + var reader = std.json.Reader.init(allocator, &config_reader.interface); + defer reader.deinit(); + var config = try std.json.parseFromTokenSourceLeaky(GptOss.Config, arena.allocator(), &reader, .{ .ignore_unknown_fields = true }); + + // From generation_config.json + config.eos_token_id = .{ .value = .{ .ints = &.{ 200002, 199999, 200012 } } }; + break :config config; + }; + + var context = try zml.Context.init(); + defer context.deinit(); + + // initialize ZML platform + const platform: zml.Platform = platform: { + const arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + + // eg: --platform-options='.{.cuda=.{.allocator=.{.bfc=.{.memory_fraction=0.99}}}}' + // eg: --platform-options='.{.cpu=.{.device_count=8}}' + const platform_opts = std.zon.parse.fromSlice(zml.Platform.CreateOptions, allocator, @ptrCast(cli.args.@"platform-options" orelse ".{}"), null, .{ .free_on_error = false }) catch |err| { + log.err("Failed to parse --platform-options as json ({}): {s}", .{ err, cli.args.@"platform-options".? }); + return err; + }; + + const compilation_options = zml.CompilationOptions{ + .xla_dump_to = "/tmp/zml/gpt_oss", + .sharding_enabled = cli.args.sharding orelse true, + }; + + const platform = context + .autoPlatform(platform_opts) + .withCompilationOptions(compilation_options); + context.printAvailablePlatforms(platform); + + break :platform platform; + }; + + const options: GptOss.Options = .{ + .max_seq_len = cli.args.@"seq-len" orelse 8192, + .max_prompt_len = cli.args.@"prompt-len" orelse 256, + .tokens_per_expert_ratio = cli.args.@"expert-budget" orelse 4.0, + .sampling_strategy = .{ + .topk = cli.args.topk orelse 10, + .temperature = 1.0, + }, + }; + + var compiler_arena = std.heap.ArenaAllocator.init(allocator); + defer compiler_arena.deinit(); + + const model_weights_path = try std.fs.path.join(allocator, &.{ hf_model_path, "model.safetensors.index.json" }); + defer allocator.free(model_weights_path); + + var store = try zml.aio.detectFormatAndOpen(allocator, model_weights_path); + defer store.deinit(); + + const model: GptOss = try GptOss.init(compiler_arena.allocator(), store, config, options); + + const tokens_shape_prefill = zml.Shape.init(.{ .s = options.max_prompt_len }, .u32); + const tokens_shape = zml.Shape.init(.{ .s = 1 }, .u32); + + const dtype = model.model.embed_tokens.weight.dtype(); + + const kv_shape = zml.Shape.init(.{ + .layer = model.model.layers.len, + .k = options.max_seq_len, + .h = config.num_key_value_heads, + .hd = config.head_dim, + }, dtype).withSharding(.{.h}); + + const kv_cache_shape: zml.ShapeOf(GptOss.KvCache) = GptOss.KvCache.initShape(kv_shape); + const rng_shape = zml.Tensor.Rng.shape(); + + var start = try std.time.Timer.start(); + var fut_mod_prefill = try async.async(zml.compileModel, .{ + allocator, GptOss.forward, model, + .{ + tokens_shape_prefill, + zml.ShapeOf(GptOss.Mode){ .prefill = .scalar(.u32) }, + kv_cache_shape, + rng_shape, + }, + platform, + }); + + var fut_mod = try async.async(zml.compileModel, .{ + allocator, GptOss.forward, model, + .{ + tokens_shape, + zml.ShapeOf(GptOss.Mode){ .gen = .scalar(.u32) }, + kv_cache_shape, + rng_shape, + }, + platform, + }); + + log.info("\tLoading GptOss weights from {s}...", .{model_weights_path}); + var gpt_oss_weights = try model.loadBuffers(compiler_arena.allocator(), store, platform); + defer zml.aio.unloadBuffers(&gpt_oss_weights); + log.info("✅\tLoaded weights in {D}", .{start.read()}); + + var module_prefill = (try fut_mod_prefill.await()).prepare(gpt_oss_weights); + defer module_prefill.deinit(); + var module_gen = (try fut_mod.await()).prepare(gpt_oss_weights); + defer module_gen.deinit(); + log.info("✅\tCompiled model in {D}", .{start.read()}); + + log.info("Creating KvCache", .{}); + const kv_cache = try GptOss.KvCache.initBuffer(kv_shape, platform); + + var tokenizer = blk: { + const model_tokenizer_path = try std.fs.path.join(allocator, &.{ hf_model_path, "tokenizer.json" }); + defer allocator.free(model_tokenizer_path); + + log.info("Loading tokenizer from {s}", .{model_tokenizer_path}); + var timer = try stdx.time.Timer.start(); + defer log.info("Loaded tokenizer from {s} [{f}]", .{ model_tokenizer_path, timer.read() }); + + break :blk try zml.tokenizer.Tokenizer.fromFile(allocator, model_tokenizer_path); + }; + errdefer tokenizer.deinit(); + + const prompt = cli.args.prompt orelse "What are some fun facts about animals?"; + log.info("✅\tPrompt: {s}", .{prompt}); + + const no_chat = cli.args.nochat orelse false; + const prompt_tok_buf = try allocator.alloc(u32, options.max_prompt_len); + defer allocator.free(prompt_tok_buf); + + const prompt_tok = tokenizePrompt(tokenizer, prompt, no_chat, prompt_tok_buf) catch |err| switch (err) { + error.PromptTooLong => std.debug.panic("Prompt too long, expected at most {d} tokens. Consider increasing --max-prompt-len", .{prompt_tok_buf.len}), + else => |e| return e, + }; + log.info("\t Tokenized prompt: {any} ({d} tokens)", .{ prompt_tok, prompt_tok.len }); + + const seed = cli.args.seed orelse @as(u128, @bitCast(std.time.nanoTimestamp())); + + // Unbuffered writing of the tokens to stdout. + // generated text will be printed token by token. + var output = std.fs.File.stdout().writer(&.{}); + + try generateText(config, options, module_prefill, module_gen, kv_cache, tokenizer, allocator, seed, prompt_tok, &output.interface); +} + +const ClapBoilerplate = struct { + pub const Cli = clap.Result(clap.Help, &cli_params, parsers); + + fn bool_parser(in: []const u8) error{}!bool { + return std.mem.indexOfScalar(u8, "tTyY1", in[0]) != null; + } + + const parsers = .{ + .BOOL = bool_parser, + .UINT = clap.parsers.int(u32, 0), + .FLOAT = clap.parsers.float(f32), + .STRING = clap.parsers.string, + .PATH = clap.parsers.string, + }; + + pub fn parseCli(allocator: std.mem.Allocator) Cli { + var diag: clap.Diagnostic = .{}; + var stderr_buffer: [1024]u8 = undefined; + var stderr = std.fs.File.stderr().writer(&stderr_buffer); + const cli = clap.parse(clap.Help, &cli_params, parsers, .{ + .diagnostic = &diag, + .allocator = allocator, + }) catch |err| { + diag.report(&stderr.interface, err) catch {}; + stderr.interface.print("usage: ", .{}) catch {}; + clap.usage(&stderr.interface, clap.Help, &cli_params) catch {}; + stderr.interface.print("\n", .{}) catch {}; + stderr.interface.flush() catch {}; + std.process.exit(1); + }; + if (cli.args.help != 0) { + clap.help(&stderr.interface, clap.Help, &cli_params, .{}) catch {}; + stderr.interface.flush() catch {}; + std.process.exit(0); + } + return cli; + } +}; diff --git a/stdx/fmt.zig b/stdx/fmt.zig index 4e38488a..e9df9131 100644 --- a/stdx/fmt.zig +++ b/stdx/fmt.zig @@ -10,20 +10,20 @@ fn FmtSlice(T: type) type { slice: []const T, pub fn format(f: @This(), writer: *std.io.Writer) std.io.Writer.Error!void { - return try formatSliceAny(f.slice, .{}, writer); + return try formatSliceAny(f.slice, .{}, 1, writer); } pub fn formatNumber(f: @This(), writer: *std.io.Writer, n: std.fmt.Number) std.io.Writer.Error!void { return switch (@typeInfo(T)) { - .comptime_float, .float => try formatFloatSlice(f.slice, n, writer), - .comptime_int, .int => try formatIntSlice(f.slice, n, writer), - .bool => try formatBoolSlice(f.slice, n, writer), + .comptime_float, .float => try formatFloatSlice(f.slice, n, 1, writer), + .comptime_int, .int => try formatIntSlice(f.slice, n, 1, writer), + .bool => try formatBoolSlice(f.slice, n, 1, writer), .@"struct" => if (@hasField(T, "re") and @hasField(T, "im")) { - try formatComplexSlice(f.slice, n, writer); + try formatComplexSlice(f.slice, n, 1, writer); } else if (@hasDecl(T, "toF32")) { - try formatFloatSlice(f.slice, n, writer); + try formatFloatSlice(f.slice, n, 1, writer); } else { - try formatSliceAny(f.slice, n, writer); + try formatSliceAny(f.slice, n, 1, writer); }, else => @compileError("FmtSlice doesn't support type: " ++ @typeName(T)), }; @@ -72,53 +72,55 @@ pub fn formatAny(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) ! return try writer.alignBufferOptions(s, .{ .alignment = spec.alignment, .fill = spec.fill }); } -pub fn formatSliceCustom(fmt_func: anytype, values: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { +pub fn formatSliceCustom(fmt_func: anytype, values: anytype, spec: std.fmt.Number, stride: i64, writer: *std.Io.Writer) !void { // use the format "width" for the number of columns instead of individual width. const num_cols: usize = spec.width orelse 12; var my_options = spec; my_options.width = null; - const n: usize = values.len; + // TODO: handle negative strides + const strd: usize = @intCast(stride); + const n: usize = @divTrunc(values.len, strd); _ = try writer.write("{"); if (n <= num_cols) { - for (values, 0..) |v, i| { + for (0..n) |i| { // Force inlining so that the switch and the buffer can be done once. - try @call(.always_inline, fmt_func, .{ v, my_options, writer }); + try @call(.always_inline, fmt_func, .{ values[i * strd], my_options, writer }); if (i < n - 1) _ = try writer.write(","); } } else { const half = @divFloor(num_cols, 2); - for (values[0..half]) |v| { - try @call(.always_inline, fmt_func, .{ v, my_options, writer }); + for (0..half) |i| { + try @call(.always_inline, fmt_func, .{ values[i * strd], my_options, writer }); _ = try writer.write(","); } _ = try writer.write(" ..., "); - for (values[n - half ..], 0..) |v, i| { - try @call(.always_inline, fmt_func, .{ v, my_options, writer }); - if (i < half - 1) _ = try writer.write(","); + for (n - half..n) |i| { + try @call(.always_inline, fmt_func, .{ values[i * strd], my_options, writer }); + if (i < n - 1) _ = try writer.write(","); } } _ = try writer.write("}"); } -pub fn formatSliceAny(values: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { - return try formatSliceCustom(formatAny, values, spec, writer); +pub fn formatSliceAny(values: anytype, spec: std.fmt.Number, stride: i64, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatAny, values, spec, stride, writer); } -pub fn formatFloatSlice(values: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { - return try formatSliceCustom(formatFloat, values, spec, writer); +pub fn formatFloatSlice(values: anytype, spec: std.fmt.Number, stride: i64, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatFloat, values, spec, stride, writer); } -pub fn formatIntSlice(values: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { - return try formatSliceCustom(formatInt, values, spec, writer); +pub fn formatIntSlice(values: anytype, spec: std.fmt.Number, stride: i64, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatInt, values, spec, stride, writer); } -pub fn formatComplexSlice(values: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { - return try formatSliceCustom(formatComplex, values, spec, writer); +pub fn formatComplexSlice(values: anytype, spec: std.fmt.Number, stride: i64, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatComplex, values, spec, stride, writer); } -pub fn formatBoolSlice(values: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { - return try formatSliceCustom(formatBool, values, spec, writer); +pub fn formatBoolSlice(values: anytype, spec: std.fmt.Number, stride: i64, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatBool, values, spec, stride, writer); } /// Format a struct using `format` method of subfields when possible. diff --git a/tools/zml_utils.py b/tools/zml_utils.py index a0053e77..9bddc023 100644 --- a/tools/zml_utils.py +++ b/tools/zml_utils.py @@ -21,16 +21,6 @@ import torch log = logging.getLogger(__name__) -builtin_open = builtins.open - - -def log_and_open(file, *args, **kwargs): - print("open:", file, args, kwargs) - return builtin_open(file, *args, **kwargs) - - -builtins.open = log_and_open - class ActivationCollector: """Wrap a given torch.nn.Module and collect all its intermediary activations. diff --git a/zml/aio/safetensors.zig b/zml/aio/safetensors.zig index b49928ac..de18f446 100644 --- a/zml/aio/safetensors.zig +++ b/zml/aio/safetensors.zig @@ -23,7 +23,7 @@ pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore } else { try loadFile(arena, &res, &files, path); } - res.files = try files.toOwnedSlice(allocator); + res.files = try files.toOwnedSlice(arena); return res; } diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 3185348c..2052970a 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -355,12 +355,17 @@ pub const HostBuffer = struct { try writer.splatByteAll(' ', indent_level); switch (self.dtype()) { inline else => |dt| { - const values = self.items(dt.toZigType()); + const T = dt.toZigType(); + const n: i64 = self._shape.dim(0); + // TODO: handle negative strides + const byte_stride: i64 = self._strides[0]; + const elem_strides: i64 = @divExact(byte_stride, @sizeOf(T)); + const values: []const T = @ptrCast(@alignCast(self._data[0..@intCast(n * byte_stride)])); switch (comptime dt.class()) { - .float => try stdx.fmt.formatFloatSlice(values, options, writer), - .integer => try stdx.fmt.formatIntSlice(values, options, writer), - .complex => try stdx.fmt.formatComplexSlice(values, options, writer), - .bool => try stdx.fmt.formatBoolSlice(values, options, writer), + .float => try stdx.fmt.formatFloatSlice(values, options, elem_strides, writer), + .integer => try stdx.fmt.formatIntSlice(values, options, elem_strides, writer), + .complex => try stdx.fmt.formatComplexSlice(values, options, elem_strides, writer), + .bool => try stdx.fmt.formatBoolSlice(values, options, elem_strides, writer), } }, } diff --git a/zml/nn.zig b/zml/nn.zig index 4fbb17cd..8bfd2f2d 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -110,9 +110,8 @@ pub const LayerNorm = struct { pub fn rmsNorm(x: Tensor, axis: anytype, eps: f32) Tensor { const ax = x.axis(axis); // upcast to improve precision - const xf32 = x.convert(.f32); - const mean = xf32.mul(xf32).mean(ax); - const rsqrt = Tensor.rsqrt(mean.addConstant(eps)).convert(x.dtype()); + const variance = x.convert(.f32).powByConst(2).mean(ax); + const rsqrt = Tensor.rsqrt(variance.addConstant(eps)).convert(x.dtype()); return x.mul(rsqrt.broad(x.shape())); } @@ -816,7 +815,7 @@ pub fn resizeCubic1d(image: Tensor, axis: i8, new_len: u63, opt: ResizeOpts) Ten Tensor.constant(t.shape(), dtype.one()), t, t.mul(t), - t.pow(Tensor.scalar(3, dtype)), + t.powByConst(3), }, .last, ._interpolated); std.debug.assert(pos.dim(0) == new_len); @@ -894,6 +893,7 @@ pub fn causalAttnMask( pub const SdpaOpts = struct { attn_mask: ?Tensor = null, scale: ?Tensor = null, + softmax_bias: ?Tensor = null, allow_cudnn: bool = true, // TODO: put a callback instead of all this field, // so that @@ -922,7 +922,7 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { stdx.debug.assert(k.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "k is missing tags {{.h, .k, .hd}}", err_args); stdx.debug.assert(v.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "v is missing tags {{.h, .k, .hd}}", err_args); - if (opts.allow_cudnn and cuda.canUseCudnnSdpa(q.shape())) { + if (opts.allow_cudnn and cuda.canUseCudnnSdpa(q.shape()) and opts.softmax_bias == null) { return cuda.sdpa(q, k, v, opts); } @@ -940,9 +940,15 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { k = k.mul(head_scaling.convert(k.dtype())); var attn_weights = q.dot(k, .{.hd}); - // log.debug("attn_weights : {}, attn_mask : {?}", .{ attn_weights, attn_mask }); + // log.debug("attn_weights : {f}, attn_mask : {?f}", .{ attn_weights, attn_mask }); if (attn_mask) |mask| attn_weights = attn_weights.add(mask.broad(attn_weights.shape())); - attn_weights = attn_weights.convert(.f32).softmax(.k).convert(q.dtype()); + attn_weights = attn_weights.convert(.f32); + attn_weights = if (opts.softmax_bias) |softmax_bias| attn: { + // The split is needed because we also split q ourselves. + // TODO: consider letting the user do that. + const bias = softmax_bias.splitAxis(.h, .{ .h = k.dim(.h), .hq = .auto }); + break :attn attn_weights.convert(.f32).softmaxBiased(.k, bias).convert(q.dtype()); + } else attn_weights.convert(.f32).softmax(.k).convert(q.dtype()); var attn = attn_weights.dot(v, .{.k}); return attn.transpose(q.shape()).merge(.{ .h = .{ .h, .hq } }); diff --git a/zml/tensor.zig b/zml/tensor.zig index 78219cdc..58798a02 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1105,13 +1105,16 @@ pub const Tensor = struct { /// Axes with the same tag on both sides, and which aren't contracting, /// are considered "batching axes". pub fn dot(lhs: Tensor, rhs: Tensor, comptime contracting: anytype) Tensor { - var contracting_axes: [contracting.len][2]i8 = undefined; - inline for (contracting, 0..) |c, i| { - contracting_axes[i] = .{ lhs.axis(c), rhs.axis(c) }; + var contracting_axes: stdx.BoundedArray([2]i8, MAX_RANK) = .{}; + if (@TypeOf(contracting) == EnumLiteral) { + contracting_axes.appendAssumeCapacity(.{ lhs.axis(contracting), rhs.axis(contracting) }); + } else { + inline for (contracting) |c| { + contracting_axes.appendAssumeCapacity(.{ lhs.axis(c), rhs.axis(c) }); + } } - var batching_axes: [MAX_RANK][2]i8 = undefined; - var n_batching: u8 = 0; + var batching_axes: stdx.BoundedArray([2]i8, MAX_RANK) = .{}; for (lhs._shape.tags(), 0..) |l, li| { stdx.debug.assert(l != Shape.TagUnknown, "Can't use `dot(..., {any})` on {any}, it need to be explictily tagged.", .{ contracting, lhs }); @@ -1119,20 +1122,19 @@ pub const Tensor = struct { stdx.debug.assert(r != Shape.TagUnknown, "Can't use `dot(..., {any})` on {any}, it need to be explictily tagged.", .{ contracting, rhs }); if (l == r) { - for (contracting_axes) |ct| { + for (contracting_axes.slice()) |ct| { if (l == lhs._shape.tag(ct[0])) { break; } } else { // tag is both in lhs and rhs but not in contracting -> it's a batching dim. - batching_axes[n_batching] = .{ @intCast(li), @intCast(ri) }; - n_batching += 1; + batching_axes.appendAssumeCapacity(.{ @intCast(li), @intCast(ri) }); } } } } - return dotGeneral(lhs, rhs, contracting_axes[0..], batching_axes[0..n_batching]); + return dotGeneral(lhs, rhs, contracting_axes.slice(), batching_axes.slice()); } test dot { @@ -1318,23 +1320,39 @@ pub const Tensor = struct { /// Returns a Tensor containing the Sigmoid Linear Unit (SiLU) activation function applied to each element of the input Tensor. /// /// silu(x) = x σ(x) - /// https://paperswithcode.com/method/silu pub fn silu(x: Tensor) Tensor { - return x.mul(x.sigmoid()); + return x.mul(.sigmoid(x)); } - /// Returns a Tensor containing the softmax function applied to each element of the input Tensor. + /// Computes softmax along the given axis. + /// y[i] = exp(x[i]) / ( Σ_k exp(x[k]) + bias ) pub fn softmax(self: Tensor, axis_: anytype) Tensor { const a = self.axis(axis_); const max_val = self.max(a); - const row_mask = max_val.cmp(.GT, Tensor.scalar(-std.math.inf(f64), self.dtype())); + const row_mask = max_val.cmp(.GT, .scalar(-std.math.inf(f64), self.dtype())); - const exp_diff_max = self.sub(self.max(a).broad(self._shape)).exp(); - const res = exp_diff_max.div(exp_diff_max.sum(a)); + const exp_diff_max = self.sub(max_val).convert(.f32).exp(); + const res = exp_diff_max.div(exp_diff_max.sum(a)).convert(self.dtype()); // If a row is full -inf return full 0 instead of full nan, // this fix attention when mask hides a full row. - return row_mask.broad(self.shape()).select(res, Tensor.scalar(0, self.dtype())); + return row_mask.broad(self.shape()).select(res, .scalar(0, self.dtype())); + } + + /// Computes softmax, but adds a bias to the sum of exponentiel. + /// y[i] = exp(x[i]) / ( Σ_k exp(x[k]) + bias ) + pub fn softmaxBiased(self: Tensor, axis_: anytype, bias: ?Tensor) Tensor { + const a = self.axis(axis_); + + if (bias == null) return self.softmax(axis_); + const b = bias.?.convert(self.dtype()).broad(self.shape().setDim(a, 1)); + const max_val: Tensor = maximum(self.max(a), b); + const exp_diff_max = self.sub(max_val).exp(); + const bias_diff_max = b.sub(max_val).exp(); + const res = exp_diff_max.div(exp_diff_max.sum(a).add(bias_diff_max)); + + // The bias means that denominator won't be 0, we don't need to handle that case. + return res; } /// Returns a Tensor containing the log of the sum of exponential over the given axis. @@ -1534,21 +1552,31 @@ pub const Tensor = struct { step: i32 = 1, singleton: bool = false, + const full = .{ .start = 0, .end = to_the_end, .step = 1 }; + pub fn single(offset: i64) Slice { return .{ .start = offset, .end = offset + 1, .singleton = true }; } + pub fn absolute(s: Slice, d: i64) Slice { + const start = if (s.start < 0) d + s.start else s.start; + const end = if (s.end == to_the_end) d else if (s.end < 0) d + s.end else s.end; + const res: Slice = .{ .start = start, .end = end, .step = s.step, .singleton = s.singleton }; + stdx.debug.assert(start < end, "Slice {f} is invalid for axis of dimension {d} (resolved to {f}", .{ s, d, res }); + return res; + } + const to_the_end = std.math.maxInt(i64); pub fn format(self: Slice, writer: *std.Io.Writer) !void { if (self.singleton) { - try writer.print("[{}]", .{self.start}); + try writer.print("[{d}]", .{self.start}); } else if (self.end == to_the_end and self.step == 1) { - try writer.print("[{}..]", .{self.start}); + try writer.print("[{d}..]", .{self.start}); } else if (self.step == 1) { - try writer.print("[{}..{}]", .{ self.start, self.end }); + try writer.print("[{d}..{d}]", .{ self.start, self.end }); } else { - try writer.print("[{}..{}:{}]", .{ self.start, self.end, self.step }); + try writer.print("[{d}..{d}:{d}]", .{ self.start, self.end, self.step }); } } }; @@ -1570,11 +1598,7 @@ pub const Tensor = struct { for (slices, 0..) |s, a| { stdx.debug.assert(s.step > 0, "slice expects 'step' to be positive, got {} at index {}", .{ s.step, a }); - const args: Slice = .{ - .start = self.wrapIndex(a, s.start), - .end = if (s.end == Slice.to_the_end) self.dim(a) else self.wrapIndex(a, s.end), - .step = s.step, - }; + const args: Slice = s.absolute(self.dim(a)); start_indices[a] = args.start; limit_indices[a] = args.end; strides[a] = args.step; @@ -2016,6 +2040,10 @@ pub const Tensor = struct { return _result(sh, constant_op.result(0)).convert(val.dtype()); } + pub fn zeroes(sh: Shape) Tensor { + return .constant(sh, sh.dtype().zero()); + } + /// Embeds a buffer with concrete values into an Mlir program. pub fn constantTensor(val: HostBuffer) Tensor { const ctx = CompilationContext.current().mlirCtx(); @@ -3047,6 +3075,7 @@ pub const Tensor = struct { /// Returns a Tensor representing the result of Top-K over the given axis. pub fn topK(self: Tensor, named_axis_: anytype, k: u32, opts: struct { descending: bool = true }) SortRes { + stdx.debug.assert(k > 0, "topK expects a k > 0, got 0", .{}); const err_msg = "topK named axis should be an integer or a named axis, eg `x.topK(.{{ .best_token = .token }}, 16)` or `x.topK(-1, 16)`"; const has_name: ?[:0]const u8, const a = switch (@typeInfo(@TypeOf(named_axis_))) { .int, .comptime_int => .{ null, self.axis(@as(i64, @intCast(named_axis_))) }, From 25bd4d08ee0e991860355138397da72a8efdde61 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 25 Sep 2025 14:05:26 +0200 Subject: [PATCH 036/881] zml: cleanup some todos (#331) --- zml/aio/torch/file.zig | 9 +++------ zml/aio/torch/pickle.zig | 5 ++--- zml/floats.zig | 6 +----- zml/hostbuffer.zig | 5 ----- zml/tensor.zig | 12 ++++-------- 5 files changed, 10 insertions(+), 27 deletions(-) diff --git a/zml/aio/torch/file.zig b/zml/aio/torch/file.zig index c1588b4e..42da1192 100644 --- a/zml/aio/torch/file.zig +++ b/zml/aio/torch/file.zig @@ -12,9 +12,6 @@ const py = @import("py.zig"); const log = std.log.scoped(.@"zml/aio"); -// TODO(cryptodeal): use zml.aio.PrefixBuilder instead -const StringBuilder = std.ArrayList(u8); - test { std.testing.refAllDecls(@This()); std.testing.refAllDecls(File); @@ -111,11 +108,11 @@ pub const File = struct { var prefix_buf: [1024]u8 = undefined; const allocator = store.arena.allocator(); for (values) |item| { - try self.parseValue(allocator, store, StringBuilder.initBuffer(&prefix_buf), item); + try self.parseValue(allocator, store, .initBuffer(&prefix_buf), item); } } - pub fn parseValue(self: File, allocator: std.mem.Allocator, store: *zml.aio.BufferStore, prefix: StringBuilder, v: py.Any) !void { + pub fn parseValue(self: File, allocator: std.mem.Allocator, store: *zml.aio.BufferStore, prefix: std.ArrayList(u8), v: py.Any) !void { // log.warn("Parsing {}", .{v}); switch (v) { .app, .object, .global => |object| { @@ -303,7 +300,7 @@ pub const File = struct { } } - fn parseTorchGlobal(self: File, allocator: std.mem.Allocator, store: *zml.aio.BufferStore, prefix: StringBuilder, v: py.Any) !bool { + fn parseTorchGlobal(self: File, allocator: std.mem.Allocator, store: *zml.aio.BufferStore, prefix: std.ArrayList(u8), v: py.Any) !bool { return switch (v) { .global => |object| { if (try self.parseTensor(allocator, object)) |host_buffer| { diff --git a/zml/aio/torch/pickle.zig b/zml/aio/torch/pickle.zig index 5724ab0a..6ed3a1b2 100644 --- a/zml/aio/torch/pickle.zig +++ b/zml/aio/torch/pickle.zig @@ -789,7 +789,6 @@ pub fn parse(arena: std.mem.Allocator, reader: *std.Io.Reader) ![]const Op { .binint => .{ .int = try reader.takeInt(i32, .little) }, .binint1 => .{ .int = try reader.takeByte() }, .binint2 => .{ .int = try reader.takeInt(u16, .little) }, - // TODO: long should handle the trailing 'L' -> add a test. .long => .{ .long = try readLine(reader, &alloc_writer) }, .long1 => .{ .binlong = try _readSlice(reader, arena, 1) }, .long4 => .{ .binlong = try _readSlice(reader, arena, 4) }, @@ -902,7 +901,7 @@ test "parse protocol 4" { const ops = try parse(arena.allocator(), &reader.interface); // this can be obtained by running: `python -m pickletools simple_test_4.pickle` - var expected = [_]Op{ + const expected: []const Op = &.{ .{ .proto = 4 }, .{ .frame = 119 }, .empty_dict, @@ -943,7 +942,7 @@ test "parse protocol 4" { .setitems, .stop, }; - try std.testing.expectEqualDeep(&expected, ops); + try std.testing.expectEqualDeep(expected, ops); } test "parse protocol 0" { diff --git a/zml/floats.zig b/zml/floats.zig index 7d995c44..54240485 100644 --- a/zml/floats.zig +++ b/zml/floats.zig @@ -442,8 +442,7 @@ pub const Float4E2M1 = packed struct(u4) { pub const values = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; pub fn toF32(x: Float4E2M1) f32 { - // the baseline toF32 doesn't work correctly: - // 0b0001 and 0b1001 shoud map to ±0.5, but are mapped to ±epsilon + // faster implementation return values[@as(u4, @bitCast(x))]; } @@ -457,9 +456,6 @@ pub const Float4E2M1 = packed struct(u4) { } test fromF32 { - // the baseline fromF32 doesn't work correctly: - // ±0.5 should map to 0b0001/0b1001 but are map to ±0.0 instead. - // TODO: it probably affects other types. var from_f32_res: [16]Float4E2M1 = undefined; for (&from_f32_res, 0..) |*r, i| { r.* = .fromF32(Float4E2M1.values[i]); diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 2052970a..0469f1ec 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -202,11 +202,6 @@ pub const HostBuffer = struct { return self._strides[0..self._shape.rank()]; } - // TODO: rename .data into ._data and make it a [*]u8 - // pub fn data(self: HostBuffer) []const u8 { - // return self.data; - // } - pub inline fn rank(self: HostBuffer) u4 { return self._shape.rank(); } diff --git a/zml/tensor.zig b/zml/tensor.zig index 58798a02..d5acc541 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -53,10 +53,7 @@ pub const Tensor = struct { } pub fn format(self: Tensor, writer: *std.Io.Writer) !void { - // TODO(0.15.0) handle format - // const bare_fmt = fmt.len == 1 and fmt[0] == '_'; - const bare_fmt = false; - try writer.print(if (bare_fmt) "{f}" else "Tensor({f})", .{self._shape}); + try writer.print("Tensor({f})", .{self._shape}); } /// Returns the shape of a Tensor. @@ -4081,11 +4078,10 @@ test "Tensor.maxPool2d" { ); } +/// Return a clone of a type with Tensors replaced by Buffer. +/// Non-Tensor metadata is stripped out of the resulting struct. +/// Recursively descends into the type. pub fn Bufferized(comptime T: type) type { - // TODO: we should strip out the non-buffer fields. - // Currently it's confusing cause the Bufferized struct contains field that are never read. - // Also it will simplify the layout of the Bufferized struct. - // accelerating the calls to execute. return meta.MapRestrict(Tensor, Buffer).map(T); } From b04d483ecd7b4f54bfe84ba05824bb21e3508a87 Mon Sep 17 00:00:00 2001 From: 0xC0 Date: Fri, 26 Sep 2025 05:25:00 -0400 Subject: [PATCH 037/881] runtimes/cpu: fix darwin arm64 PJRT tarball checksum (#335) Update sha256 to match v13.0.0 pjrt-cpu_darwin-arm64.tar.gz; unblocks Bazel fetch on macOS arm64. --- runtimes/cpu/cpu.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtimes/cpu/cpu.bzl b/runtimes/cpu/cpu.bzl index dbc69d7b..2a095f49 100644 --- a/runtimes/cpu/cpu.bzl +++ b/runtimes/cpu/cpu.bzl @@ -37,7 +37,7 @@ def _cpu_pjrt_plugin_impl(mctx): http_archive( name = "libpjrt_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "e54bc3c3b71313c49e38fc342851bffe2cd3a55e3990839947d49496a1a71270", + sha256 = "a6354bfed828a011e6d809eda2230e10c40c80044c67fe618b2a9615c047f092", url = "https://github.com/zml/pjrt-artifacts/releases/download/v13.0.0/pjrt-cpu_darwin-arm64.tar.gz", ) From 53ad5830893b6bf86995d7293e701fea83dc4fdc Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Dalido Date: Fri, 26 Sep 2025 17:47:15 +0200 Subject: [PATCH 038/881] workspace: add CI for cuda and rocm platforms, split jobs (#327) --- .github/workflows/ci.yaml | 85 ++++++++++++++++++++++++++++++++++----- .zed/settings.json | 3 ++ 2 files changed, 79 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7af596eb..b033adec 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2,9 +2,9 @@ name: ci on: push: branches: - - master + - master tags: - - '*' + - "*" pull_request: concurrency: @@ -12,8 +12,81 @@ concurrency: cancel-in-progress: ${{ !contains(github.ref, 'master')}} jobs: + platforms: + strategy: + matrix: + runtimes: + - target: cuda + bazel_opts: "--@zml//runtimes:cuda=true --@zml//runtimes:cpu=false" + enable_cache: false + user: actions + runs-on: ["self-hosted", "platform=cuda"] + - target: rocm + bazel_opts: "--@zml//runtimes:rocm=true --@zml//runtimes:cpu=false" + enable_cache: false + user: actions + runs-on: ["self-hosted", "platform=rocm"] + - target: cpu + bazel_opts: "--@zml//runtimes:cpu=true" + enable_cache: true + user: runner + runs-on: + [ + "runs-on", + "runner=4cpu-linux-x64", + "image=ubuntu24-amd64", + "run-id=${{ github.run_id }}", + "extras=s3-cache", + ] + runs-on: ${{ matrix.runtimes.runs-on }} + steps: + - uses: runs-on/action@v1 + - name: Checkout + uses: actions/checkout@v4 + + - uses: runs-on/snapshot@v1 + if: ${{ matrix.runtimes.enable_cache }} + with: + path: /home/${{ matrix.runtimes.user }}/.cache/bazel-disk + volume_size: 350 + + - name: TMP fix for the mount user + if: ${{ matrix.runtimes.enable_cache }} + run: sudo chown -R ${{ matrix.runtimes.user }}:${{ matrix.runtimes.user }} /home/${{ matrix.runtimes.user }}/.cache/ + + - name: Ensure self hosted cache permissions + if: ${{ !matrix.runtimes.enable_cache }} + run: sudo chmod -R 777 /tmp/zig-cache && sudo chmod -R 777 /tmp/zml + + - uses: bazel-contrib/setup-bazel@0.15.0 + with: + bazelisk-version: 1.26.0 + bazelisk-cache: false + # todo: use `--config=debug` instead of `-c opt` when static archive bug is fixed + bazelrc: | + common -c opt + common --verbose_failures + common --keep_going + common --test_output=streamed + common --color=yes + common --show_timestamps + common --disk_cache=/home/${{ matrix.runtimes.user }}/.cache/bazel-disk + common --strip=always + + - name: ZML / bazel build //... + run: bazel build ${{ matrix.runtimes.bazel_opts }} //... + + - name: ZML / bazel test //... + run: bazel test ${{ matrix.runtimes.bazel_opts }} //... ci: - runs-on: ["runs-on","runner=4cpu-linux-x64", "image=ubuntu24-amd64", "run-id=${{ github.run_id }}", "extras=s3-cache"] + runs-on: + [ + "runs-on", + "runner=4cpu-linux-x64", + "image=ubuntu24-amd64", + "run-id=${{ github.run_id }}", + "extras=s3-cache", + ] steps: - uses: runs-on/action@v1 - name: Checkout @@ -62,9 +135,3 @@ jobs: - name: ZML / bazel query //... run: bazel query //... - - - name: ZML / bazel build //... - run: bazel build //... - - - name: ZML / bazel test //... - run: bazel test //... diff --git a/.zed/settings.json b/.zed/settings.json index 955aea11..26af5ce3 100644 --- a/.zed/settings.json +++ b/.zed/settings.json @@ -10,6 +10,9 @@ "source.fixAll": true, "source.organizeImports": true } + }, + "YAML": { + "tab_size": 2 } }, "file_scan_exclusions": [ From 36d583dd586dbaab243e624bb78b6b8b8422271e Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Mon, 13 Oct 2025 14:34:33 +0200 Subject: [PATCH 039/881] workspace: use upstreamed rules_zig instead of our fork (#340) This PR uses the new release of rules_zig where we upstreamed all the work from the zml rules_zig fork. Not much changes from the POV of the public API except for `copts` becoming `zigopts`. Also, the global `c` module is only available to Zig modules that depend on a non empty `cc_library`. This is behavior that can probably be modified in `rules_zig` later on. Finally, ZLS support had to be completely rethought to comply with the change in shape of ZigModuleInfo providers and subsequent rules. This PR is blocked until `rules_zig` exposes `ZigModuleInfo` for `zig_binaries`. --- .bazelrc | 2 +- BUILD.bazel | 8 +- MODULE.bazel | 10 +- MODULE.bazel.lock | 245 +++++++++++++++++- async/BUILD.bazel | 2 +- examples/mnist/BUILD.bazel | 1 + runtimes/cuda/BUILD.bazel | 3 +- runtimes/neuron/BUILD.bazel | 5 +- runtimes/rocm/BUILD.bazel | 3 +- runtimes/tpu/BUILD.bazel | 1 + stdx/BUILD.bazel | 2 +- .../libxev/20251010.0-9f785d2/MODULE.bazel | 1 + .../20251010.0-9f785d2/overlay/BUILD.bazel | 13 + .../20251010.0-9f785d2/overlay/MODULE.bazel | 7 + .../libxev/20251010.0-9f785d2/source.json | 11 + third_party/modules/libxev/metadata.json | 1 + third_party/zls/BUILD.bazel | 75 +++++- third_party/zls/cat.zig | 31 +++ third_party/zls/resolved_toolchain.bzl | 24 ++ third_party/zls/zls.bzl | 187 ------------- third_party/zls/zls.completion.tpl | 10 + third_party/zls/zls_build_runner.zig | 50 ++++ third_party/zls/zls_completion.bzl | 69 +++++ third_party/zls/zls_runner.zig | 123 +++++++++ third_party/zls/zls_toolchain.bzl | 123 +++++++++ third_party/zls/zls_write_build_config.bzl | 138 ++++++++++ third_party/zls/zls_write_runner_zig_src.bzl | 45 ++++ tools/zls.sh | 2 +- zml/BUILD.bazel | 2 +- 29 files changed, 973 insertions(+), 221 deletions(-) create mode 120000 third_party/modules/libxev/20251010.0-9f785d2/MODULE.bazel create mode 100644 third_party/modules/libxev/20251010.0-9f785d2/overlay/BUILD.bazel create mode 100644 third_party/modules/libxev/20251010.0-9f785d2/overlay/MODULE.bazel create mode 100644 third_party/modules/libxev/20251010.0-9f785d2/source.json create mode 100644 third_party/zls/cat.zig create mode 100644 third_party/zls/resolved_toolchain.bzl delete mode 100644 third_party/zls/zls.bzl create mode 100644 third_party/zls/zls.completion.tpl create mode 100644 third_party/zls/zls_build_runner.zig create mode 100644 third_party/zls/zls_completion.bzl create mode 100644 third_party/zls/zls_runner.zig create mode 100644 third_party/zls/zls_toolchain.bzl create mode 100644 third_party/zls/zls_write_build_config.bzl create mode 100644 third_party/zls/zls_write_runner_zig_src.bzl diff --git a/.bazelrc b/.bazelrc index cbbb4386..5e0695b6 100644 --- a/.bazelrc +++ b/.bazelrc @@ -76,7 +76,7 @@ common --incompatible_enable_proto_toolchain_resolution common --incompatible_disallow_empty_glob=false # Use the C linker to link Zig code -common --@rules_zig//zig/settings:linkmode=cc +common --@rules_zig//zig/settings:use_cc_common_link=True common --@toolchains_llvm_bootstrapped//config:experimental_stub_libgcc_s=True diff --git a/BUILD.bazel b/BUILD.bazel index aa1656d4..97b2cf13 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,10 +1,4 @@ -load("@rules_zig//zig:defs.bzl", "zls_completion") -load("@zml//third_party/zls:zls.bzl", "zls_runner") - -zls_runner( - name = "zls", - target = ":completion", -) +load("//third_party/zls:zls_completion.bzl", "zls_completion") zls_completion( name = "completion", diff --git a/MODULE.bazel b/MODULE.bazel index 72ad53a5..1b638ce0 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -6,7 +6,7 @@ bazel_dep(name = "abseil-cpp", version = "20250814.0") bazel_dep(name = "aspect_bazel_lib", version = "2.20.0") bazel_dep(name = "aspect_rules_py", version = "1.6.3") bazel_dep(name = "bazel_skylib", version = "1.8.1") -bazel_dep(name = "libxev", version = "20250718.0-9f785d2") +bazel_dep(name = "libxev", version = "20251010.0-9f785d2") bazel_dep(name = "patchelf", version = "0.18.0") bazel_dep(name = "pcre2", version = "10.45") bazel_dep(name = "platforms", version = "1.0.0") @@ -18,7 +18,7 @@ bazel_dep(name = "rules_oci", version = "2.2.6") bazel_dep(name = "rules_proto", version = "7.1.0") bazel_dep(name = "rules_python", version = "1.5.3") bazel_dep(name = "rules_rust", version = "0.63.0") -bazel_dep(name = "rules_zig", version = "20250827.0-35b6d57") +bazel_dep(name = "rules_zig", version = "0.12.1") bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.2.4") bazel_dep(name = "with_cfg.bzl", version = "0.11.0") @@ -92,10 +92,10 @@ use_repo(tpu, "libpjrt_tpu") neuron = use_extension("//runtimes/neuron:neuron.bzl", "neuron_packages") use_repo(neuron, "aws-neuronx-collectives", "aws-neuronx-runtime-lib", "libgomp1", "libpjrt_neuron", "zlib1g") -zls = use_extension("//third_party/zls:zls.bzl", "repo") -use_repo(zls, "zls_aarch64-macos", "zls_x86_64-linux", "zls_x86_64-macos") +zls = use_extension("//third_party/zls:zls_toolchain.bzl", "zls_toolchains") +use_repo(zls, "zls_toolchains") -register_toolchains("//third_party/zls:all") +register_toolchains("@zls_toolchains//:all") non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") use_repo(non_module_deps, "com_github_hejsil_clap", "com_google_sentencepiece", "mnist", "org_swig_swig", "xla") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 5fa256de..e5f0dd8e 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -18,7 +18,9 @@ "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.0.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.16.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.19.3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.20.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.21.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.9.4/MODULE.bazel": "not found", @@ -53,8 +55,10 @@ "https://bazel-registry.zml.ai/modules/bazel_skylib/1.7.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_skylib/1.7.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_skylib/1.8.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/bazel_skylib/1.8.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/8.2.0.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/buildozer/7.1.2/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/gawk/5.3.2.bcr.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/google_benchmark/1.8.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/googletest/1.11.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "not found", @@ -107,6 +111,7 @@ "https://bazel-registry.zml.ai/modules/rules_cc/0.0.9/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.1.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_cc/0.2.9/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_distroless/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_fuzzing/0.5.2/MODULE.bazel": "not found", @@ -165,6 +170,7 @@ "https://bazel-registry.zml.ai/modules/rules_shell/0.4.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_zig/0.12.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.4/MODULE.bazel": "not found", @@ -175,6 +181,7 @@ "https://bazel-registry.zml.ai/modules/stardoc/0.7.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/tar.bzl/0.2.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/tar.bzl/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/toolchains_llvm_bootstrapped/0.2.4/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/with_cfg.bzl/0.11.0/MODULE.bazel": "not found", @@ -203,8 +210,10 @@ "https://bcr.bazel.build/modules/aspect_bazel_lib/2.0.0/MODULE.bazel": "e118477db5c49419a88d78ebc7a2c2cea9d49600fe0f490c1903324a2c16ecd9", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.16.0/MODULE.bazel": "852f9ebbda017572a7c113a2434592dd3b2f55cd9a0faea3d4be5a09a59e4900", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.3/MODULE.bazel": "253d739ba126f62a5767d832765b12b59e9f8d2bc88cc1572f4a73e46eb298ca", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.20.0/MODULE.bazel": "c5565bac49e1973227225b441fad1c938d498d83df62dc5da95b2fab0f0626a2", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.20.0/source.json": "3eaada79dd3c65b6c57d5fc33c57ffd2896c4ebd78c4c9001a790a70f7f50e61", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.21.2/MODULE.bazel": "276347663a25b0d5bd6cad869252bea3e160c4d980e764b15f3bae7f80b30624", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.21.2/source.json": "f42051fa42629f0e59b7ac2adf0a55749144b11f1efcd8c697f0ee247181e526", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "780d1a6522b28f5edb7ea09630748720721dfe27690d65a2d33aa7509de77e07", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "812d2dd42f65dca362152101fbec418029cc8fd34cbad1a2fde905383d705838", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.9.4/MODULE.bazel": "ccc41028429f894b02fde7ef67d416cba3ba5084ed9ddb9bb6107aa82d118776", @@ -242,11 +251,14 @@ "https://bcr.bazel.build/modules/bazel_skylib/1.7.0/MODULE.bazel": "0db596f4563de7938de764cc8deeabec291f55e8ec15299718b93c4423e9796d", "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/MODULE.bazel": "3120d80c5861aa616222ec015332e5f8d3171e062e3e804a2a0253e1be26e59b", "https://bcr.bazel.build/modules/bazel_skylib/1.8.1/MODULE.bazel": "88ade7293becda963e0e3ea33e7d54d3425127e0a326e0d17da085a5f1f03ff6", - "https://bcr.bazel.build/modules/bazel_skylib/1.8.1/source.json": "7ebaefba0b03efe59cac88ed5bbc67bcf59a3eff33af937345ede2a38b2d368a", + "https://bcr.bazel.build/modules/bazel_skylib/1.8.2/MODULE.bazel": "69ad6927098316848b34a9142bcc975e018ba27f08c4ff403f50c1b6e646ca67", + "https://bcr.bazel.build/modules/bazel_skylib/1.8.2/source.json": "34a3c8bcf233b835eb74be9d628899bb32999d3e0eadef1947a0a562a2b16ffb", "https://bcr.bazel.build/modules/buildifier_prebuilt/8.2.0.2/MODULE.bazel": "a9b689711d5b69f9db741649b218c119b9fdf82924ba390415037e09798edd03", "https://bcr.bazel.build/modules/buildifier_prebuilt/8.2.0.2/source.json": "51eb0a4b38aaaeab7fa64361576d616c4d8bfd0f17a0a10184aeab7084d79f8e", "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", + "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.1/MODULE.bazel": "cdf8cbe5ee750db04b78878c9633cc76e80dcf4416cbe982ac3a9222f80713c8", + "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.1/source.json": "fa7b512dfcb5eafd90ce3959cf42a2a6fe96144ebbb4b3b3928054895f2afac2", "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb", "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6", @@ -312,7 +324,8 @@ "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", "https://bcr.bazel.build/modules/rules_cc/0.1.1/MODULE.bazel": "2f0222a6f229f0bf44cd711dc13c858dad98c62d52bd51d8fc3a764a83125513", "https://bcr.bazel.build/modules/rules_cc/0.2.0/MODULE.bazel": "b5c17f90458caae90d2ccd114c81970062946f49f355610ed89bebf954f5783c", - "https://bcr.bazel.build/modules/rules_cc/0.2.0/source.json": "5f7f4e578e950adbf194217d4b607237a8197fc53ba46c367b3d61a86ecf35c2", + "https://bcr.bazel.build/modules/rules_cc/0.2.9/MODULE.bazel": "34263f1dca62ea664265438cef714d7db124c03e1ed55ebb4f1dc860164308d1", + "https://bcr.bazel.build/modules/rules_cc/0.2.9/source.json": "4e49b40effcbd14fbfb233eb929de42dfff7b66538b4ffda310ad501638e7986", "https://bcr.bazel.build/modules/rules_distroless/0.5.1/MODULE.bazel": "2a63f4744d30749128105da5f96adf7caf5628e37548293f89e7fa39c3b3f2c2", "https://bcr.bazel.build/modules/rules_distroless/0.5.1/source.json": "c6b9ff7f325bfed89c3671757f14c1d1bc6077d0fcef809b8aa2d007cac7dd1d", "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", @@ -383,6 +396,8 @@ "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", "https://bcr.bazel.build/modules/rules_swift/2.1.1/source.json": "40fc69dfaac64deddbb75bd99cdac55f4427d9ca0afbe408576a65428427a186", + "https://bcr.bazel.build/modules/rules_zig/0.12.1/MODULE.bazel": "5953094b681c212eebe3dd4275809c52b9177aeaa058d8f1bff433822c0dcd01", + "https://bcr.bazel.build/modules/rules_zig/0.12.1/source.json": "d880cdc3473bfcf172c9aad9292e856430496e198d805539bb8be98e79f58a7c", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", "https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4", @@ -395,7 +410,8 @@ "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "5e463fbfba7b1701d957555ed45097d7f984211330106ccd1352c6e0af0dcf91", "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/source.json": "32bd87e5f4d7acc57c5b2ff7c325ae3061d5e242c0c4c214ae87e0f1c13e54cb", "https://bcr.bazel.build/modules/tar.bzl/0.2.1/MODULE.bazel": "52d1c00a80a8cc67acbd01649e83d8dd6a9dc426a6c0b754a04fe8c219c76468", - "https://bcr.bazel.build/modules/tar.bzl/0.2.1/source.json": "600ac6ff61744667a439e7b814ae59c1f29632c3984fccf8000c64c9db8d7bb6", + "https://bcr.bazel.build/modules/tar.bzl/0.5.1/MODULE.bazel": "7c2eb3dcfc53b0f3d6f9acdfd911ca803eaf92aadf54f8ca6e4c1f3aee288351", + "https://bcr.bazel.build/modules/tar.bzl/0.5.1/source.json": "deed3094f7cc779ed1d37a68403847b0e38d9dd9d931e03cb90825f3368b515f", "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.2.4/MODULE.bazel": "859f02226bc313a622f56b816582fb55d5aaced4a1ecde7397dd4a9ebd24321e", "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.2.4/source.json": "43c5fd6b585cbf0b642157b6843a25425d541fabe638acb7bc908846c4bda8ca", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", @@ -413,7 +429,7 @@ "moduleExtensions": { "@@apple_support+//crosstool:setup.bzl%apple_cc_configure_extension": { "general": { - "bzlTransitiveDigest": "JJV/C5bhYKZip41t/GGZvqIMmmrcwr0Ydybkn97uXxQ=", + "bzlTransitiveDigest": "N6yHPMaF17im7GO6Gu+cdCtGXs+lAlBIhCeB1/ZZoPk=", "usagesDigest": "39X2JjPCOAk6sThDALGv1L4q85GNjda2yfszm/phxxw=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -444,7 +460,7 @@ }, "@@aspect_rules_py+//py:extensions.bzl%py_tools": { "general": { - "bzlTransitiveDigest": "geB/ya+5nbONwFUgzta9iVBeMKtviiSBwlZnE7Avcgs=", + "bzlTransitiveDigest": "Udhh0icjIe0dHLjbgfmj2tzBz0geB4SXJl0FoPcpF84=", "usagesDigest": "tVlELaqQFX7HPl1uFzRY9m7nVV7tY9roq1qfK75ApwE=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -626,7 +642,7 @@ }, "@@rules_apple+//apple:apple.bzl%provisioning_profile_repository_extension": { "general": { - "bzlTransitiveDigest": "BRNIrN/yKvsUnyouziBgSAp7x+TIcFZhks4bYDTh9iA=", + "bzlTransitiveDigest": "7XgPyOlTI0qJefux0gxRds30ZZCLPjr/7R9gFtU/jgE=", "usagesDigest": "vsJl8Rw5NL+5Ag2wdUDoTeRF/5klkXO8545Iy7U1Q08=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -800,7 +816,7 @@ }, "@@rules_oci+//oci:extensions.bzl%oci": { "general": { - "bzlTransitiveDigest": "4LJBCkExDO/zET3DzuMS4pDRRr4V0gc6YgFMhM6wIjI=", + "bzlTransitiveDigest": "PTQgYMhZAtC3X0ttcTBbDaaOBjIGyz0G4vX/ThEUHDY=", "usagesDigest": "vK9qUa3pyGJik2hb1NI2fKVJrjIszor+wN/44XMkKi0=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -1161,7 +1177,7 @@ }, "@@rules_rust+//crate_universe:extensions.bzl%crate": { "general": { - "bzlTransitiveDigest": "HHPRUh/mbCXdC11w2axMRLWeweoYK1bLuS1e2NpvcoM=", + "bzlTransitiveDigest": "uv5JLD8zajzRT1Hh+FYGrK5o1QjgTE4Ohde5NQMsUlk=", "usagesDigest": "mup6mgSFatk34Q0znMZPpIEOFZyk9v3xf6jRIXEFTxc=", "recordedFileInputs": { "@@//zml/tokenizer/hftokenizers/Cargo.lock": "d78ffab37ef69bf8ff45a4d686b25175f30e8fed4d8375612e84dcf6db0cfc9c", @@ -2492,7 +2508,7 @@ }, "@@rules_rust+//crate_universe/private:internal_extensions.bzl%cu_nr": { "general": { - "bzlTransitiveDigest": "A72J7FdqiIlmYkdw1rlcP/8uguFzwKWCs3Q3Mn4fVpQ=", + "bzlTransitiveDigest": "BEf6td+6l3OplJk+TxURVgZT9CK8AKbw6H/uSseljfQ=", "usagesDigest": "3vKI8uvqTpJCf+t8aU6UD5d5cUWinWhtMjKkRpCLR+A=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -2791,6 +2807,215 @@ ] ] } + }, + "@@rules_zig+//zig:extensions.bzl%zig": { + "general": { + "bzlTransitiveDigest": "wV3euhBZSI0D2ZlqbY1YW592mghFY88d9H6zBMQRaJY=", + "usagesDigest": "T0xLzuDAWCJjFcphcIcT31t1HM/Yg7RxtfOUb+XmxTw=", + "recordedFileInputs": { + "@@//bazel/zig_index.json": "42ea73420b15741770d5d8d0ded1311a2dfda1f963f0802dea3418f6e060ffef", + "@@rules_zig+//zig/private/versions.json": "6eb85ebaee72c4e6fbc82d8ad73f69e0899d839b70a7e19caa792ab6a5fa0c36" + }, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "zig_0.15.1_aarch64-linux": { + "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", + "attributes": { + "url": "https://ziglang.org/download/0.15.1/zig-aarch64-linux-0.15.1.tar.xz", + "mirrors": [ + "https://mirror.zml.ai/zig", + "https://ziglang.org/builds/" + ], + "sha256": "bb4a8d2ad735e7fba764c497ddf4243cb129fece4148da3222a7046d3f1f19fe", + "zig_version": "0.15.1", + "platform": "aarch64-linux" + } + }, + "zig_0.15.1_aarch64-macos": { + "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", + "attributes": { + "url": "https://ziglang.org/download/0.15.1/zig-aarch64-macos-0.15.1.tar.xz", + "mirrors": [ + "https://mirror.zml.ai/zig", + "https://ziglang.org/builds/" + ], + "sha256": "c4bd624d901c1268f2deb9d8eb2d86a2f8b97bafa3f118025344242da2c54d7b", + "zig_version": "0.15.1", + "platform": "aarch64-macos" + } + }, + "zig_0.15.1_aarch64-windows": { + "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", + "attributes": { + "url": "https://ziglang.org/download/0.15.1/zig-aarch64-windows-0.15.1.zip", + "mirrors": [ + "https://mirror.zml.ai/zig", + "https://ziglang.org/builds/" + ], + "sha256": "1f1bf16228b0ffcc882b713dc5e11a6db4219cb30997e13c72e8e723c2104ec6", + "zig_version": "0.15.1", + "platform": "aarch64-windows" + } + }, + "zig_0.15.1_x86_64-linux": { + "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", + "attributes": { + "url": "https://ziglang.org/download/0.15.1/zig-x86_64-linux-0.15.1.tar.xz", + "mirrors": [ + "https://mirror.zml.ai/zig", + "https://ziglang.org/builds/" + ], + "sha256": "c61c5da6edeea14ca51ecd5e4520c6f4189ef5250383db33d01848293bfafe05", + "zig_version": "0.15.1", + "platform": "x86_64-linux" + } + }, + "zig_0.15.1_x86_64-macos": { + "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", + "attributes": { + "url": "https://ziglang.org/download/0.15.1/zig-x86_64-macos-0.15.1.tar.xz", + "mirrors": [ + "https://mirror.zml.ai/zig", + "https://ziglang.org/builds/" + ], + "sha256": "9919392e0287cccc106dfbcbb46c7c1c3fa05d919567bb58d7eb16bca4116184", + "zig_version": "0.15.1", + "platform": "x86_64-macos" + } + }, + "zig_0.15.1_x86_64-windows": { + "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", + "attributes": { + "url": "https://ziglang.org/download/0.15.1/zig-x86_64-windows-0.15.1.zip", + "mirrors": [ + "https://mirror.zml.ai/zig", + "https://ziglang.org/builds/" + ], + "sha256": "91e69e887ca8c943ce9a515df3af013d95a66a190a3df3f89221277ebad29e34", + "zig_version": "0.15.1", + "platform": "x86_64-windows" + } + }, + "zig_toolchains": { + "repoRuleId": "@@rules_zig+//zig/private/repo:toolchains_repo.bzl%toolchains_repo", + "attributes": { + "names": [ + "zig_0.15.1_aarch64-linux", + "zig_0.15.1_aarch64-macos", + "zig_0.15.1_aarch64-windows", + "zig_0.15.1_x86_64-linux", + "zig_0.15.1_x86_64-macos", + "zig_0.15.1_x86_64-windows" + ], + "labels": [ + "@zig_0.15.1_aarch64-linux//:zig_toolchain", + "@zig_0.15.1_aarch64-macos//:zig_toolchain", + "@zig_0.15.1_aarch64-windows//:zig_toolchain", + "@zig_0.15.1_x86_64-linux//:zig_toolchain", + "@zig_0.15.1_x86_64-macos//:zig_toolchain", + "@zig_0.15.1_x86_64-windows//:zig_toolchain" + ], + "zig_versions": [ + "0.15.1", + "0.15.1", + "0.15.1", + "0.15.1", + "0.15.1", + "0.15.1" + ], + "exec_lengths": [ + 2, + 2, + 2, + 2, + 2, + 2 + ], + "exec_constraints": [ + "@platforms//os:linux", + "@platforms//cpu:aarch64", + "@platforms//os:macos", + "@platforms//cpu:aarch64", + "@platforms//os:windows", + "@platforms//cpu:aarch64", + "@platforms//os:linux", + "@platforms//cpu:x86_64", + "@platforms//os:macos", + "@platforms//cpu:x86_64", + "@platforms//os:windows", + "@platforms//cpu:x86_64" + ] + } + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_zig+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "rules_zig+", + "bazel_tools", + "bazel_tools" + ] + ] + } + }, + "@@tar.bzl+//tar:extensions.bzl%toolchains": { + "general": { + "bzlTransitiveDigest": "ARFnga1qqjy0bT/hVunmIZnM4oZCkD8F253Z8POuLXs=", + "usagesDigest": "maF8qsAIqeH1ey8pxP0gNZbvJt34kLZvTFeQ0ntrJVA=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "bsd_tar_toolchains": { + "repoRuleId": "@@tar.bzl+//tar/toolchain:toolchain.bzl%tar_toolchains_repo", + "attributes": { + "user_repository_name": "bsd_tar_toolchains" + } + }, + "bsd_tar_toolchains_darwin_amd64": { + "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", + "attributes": { + "platform": "darwin_amd64" + } + }, + "bsd_tar_toolchains_darwin_arm64": { + "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", + "attributes": { + "platform": "darwin_arm64" + } + }, + "bsd_tar_toolchains_linux_amd64": { + "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", + "attributes": { + "platform": "linux_amd64" + } + }, + "bsd_tar_toolchains_linux_arm64": { + "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", + "attributes": { + "platform": "linux_arm64" + } + }, + "bsd_tar_toolchains_windows_amd64": { + "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", + "attributes": { + "platform": "windows_amd64" + } + }, + "bsd_tar_toolchains_windows_arm64": { + "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", + "attributes": { + "platform": "windows_arm64" + } + } + }, + "recordedRepoMappingEntries": [] + } } } } diff --git a/async/BUILD.bazel b/async/BUILD.bazel index 17cdcf92..82fc0966 100644 --- a/async/BUILD.bazel +++ b/async/BUILD.bazel @@ -10,7 +10,7 @@ zig_library( "executor.zig", "stack.zig", ], - copts = ["-lc"], + zigopts = ["-lc"], extra_srcs = glob(["asm/*.s"]), main = "async.zig", visibility = ["//visibility:public"], diff --git a/examples/mnist/BUILD.bazel b/examples/mnist/BUILD.bazel index 11ac0c66..22e9d675 100644 --- a/examples/mnist/BUILD.bazel +++ b/examples/mnist/BUILD.bazel @@ -11,6 +11,7 @@ zig_binary( "@mnist//:mnist.safetensors", "@mnist//:t10k-images.idx3-ubyte", ], + zigopts = ["-fllvm"], main = "mnist.zig", deps = [ "//async", diff --git a/runtimes/cuda/BUILD.bazel b/runtimes/cuda/BUILD.bazel index 83109cf0..19eecbc1 100644 --- a/runtimes/cuda/BUILD.bazel +++ b/runtimes/cuda/BUILD.bazel @@ -5,7 +5,7 @@ zig_shared_library( name = "zmlxcuda", # Use Clang's compiler-rt, but disable stack checking # to avoid requiring on the _zig_probe_stack symbol. - copts = ["-fno-stack-check", "-fllvm"], + zigopts = ["-fno-stack-check", "-fllvm"], main = "zmlxcuda.zig", shared_lib_name = "libzmlxcuda.so.0", visibility = ["@libpjrt_cuda//:__subpackages__"], @@ -14,6 +14,7 @@ zig_shared_library( cc_library( name = "empty", + defines = ["ZML_RUNTIME_CUDA_DISABLED"], ) cc_library( diff --git a/runtimes/neuron/BUILD.bazel b/runtimes/neuron/BUILD.bazel index 184925a5..f1553fff 100644 --- a/runtimes/neuron/BUILD.bazel +++ b/runtimes/neuron/BUILD.bazel @@ -11,7 +11,7 @@ load("@zml//bazel:runfiles.bzl", "runfiles_to_default") # of neuronx-cc (see add_needed of the patchelf target below). zig_shared_library( name = "libpjrt_neuron", - copts = ["-fno-stack-check"], + zigopts = ["-fno-stack-check"], main = "libpjrt_neuron.zig", visibility = ["@libpjrt_neuron//:__subpackages__"], deps = [ @@ -48,6 +48,7 @@ cc_library( cc_library( name = "empty", + defines = ["ZML_RUNTIME_NEURON_DISABLED"], ) cc_library( @@ -79,7 +80,7 @@ upb_c_proto_library( zig_shared_library( name = "libneuronxla", - copts = [ + zigopts = [ "-fno-stack-check", "-fPIC", ], diff --git a/runtimes/rocm/BUILD.bazel b/runtimes/rocm/BUILD.bazel index 7355dc99..7f9be788 100644 --- a/runtimes/rocm/BUILD.bazel +++ b/runtimes/rocm/BUILD.bazel @@ -5,7 +5,7 @@ zig_shared_library( name = "zmlxrocm", # Use Clang's compiler-rt, but disable stack checking # to avoid requiring on the _zig_probe_stack symbol. - copts = ["-fno-stack-check"], + zigopts = ["-fno-stack-check"], main = "zmlxrocm.zig", shared_lib_name = "libzmlxrocm.so.0", visibility = ["@libpjrt_rocm//:__subpackages__"], @@ -24,6 +24,7 @@ alias( cc_library( name = "empty", + defines = ["ZML_RUNTIME_ROCM_DISABLED"], ) cc_library( diff --git a/runtimes/tpu/BUILD.bazel b/runtimes/tpu/BUILD.bazel index 8939fef1..fe513d3c 100644 --- a/runtimes/tpu/BUILD.bazel +++ b/runtimes/tpu/BUILD.bazel @@ -3,6 +3,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library") cc_library( name = "empty", + defines = ["ZML_RUNTIME_TPU_DISABLED"], ) cc_library( diff --git a/stdx/BUILD.bazel b/stdx/BUILD.bazel index 2594731f..23c270e0 100644 --- a/stdx/BUILD.bazel +++ b/stdx/BUILD.bazel @@ -15,7 +15,7 @@ zig_library( "signature.zig", "time.zig", ], - copts = ["-lc"], + zigopts = ["-lc"], main = "stdx.zig", visibility = ["//visibility:public"], ) diff --git a/third_party/modules/libxev/20251010.0-9f785d2/MODULE.bazel b/third_party/modules/libxev/20251010.0-9f785d2/MODULE.bazel new file mode 120000 index 00000000..c8a52909 --- /dev/null +++ b/third_party/modules/libxev/20251010.0-9f785d2/MODULE.bazel @@ -0,0 +1 @@ +./overlay/MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20251010.0-9f785d2/overlay/BUILD.bazel b/third_party/modules/libxev/20251010.0-9f785d2/overlay/BUILD.bazel new file mode 100644 index 00000000..872ef7fb --- /dev/null +++ b/third_party/modules/libxev/20251010.0-9f785d2/overlay/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "xev", + srcs = glob([ + "src/*.zig", + "src/backend/*.zig", + "src/linux/*.zig", + "src/watcher/*.zig", + ]), + main = "src/main.zig", + visibility = ["//visibility:public"], +) diff --git a/third_party/modules/libxev/20251010.0-9f785d2/overlay/MODULE.bazel b/third_party/modules/libxev/20251010.0-9f785d2/overlay/MODULE.bazel new file mode 100644 index 00000000..66f5a554 --- /dev/null +++ b/third_party/modules/libxev/20251010.0-9f785d2/overlay/MODULE.bazel @@ -0,0 +1,7 @@ +module( + name = "libxev", + version = "20251010.0-9f785d2", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "0.12.1") diff --git a/third_party/modules/libxev/20251010.0-9f785d2/source.json b/third_party/modules/libxev/20251010.0-9f785d2/source.json new file mode 100644 index 00000000..daf74448 --- /dev/null +++ b/third_party/modules/libxev/20251010.0-9f785d2/source.json @@ -0,0 +1,11 @@ +{ + "strip_prefix": "libxev-9f785d202ddccf7a625e799250579253977978b6", + "url": "https://github.com/mitchellh/libxev/archive/9f785d202ddccf7a625e799250579253977978b6.tar.gz", + "integrity": "sha256-HaExxsAsATjH/pCP9NMo/EjVrG2KVX3SiTtZzmS8R90=", + "overlay": { + "MODULE.bazel": "", + "BUILD.bazel": "" + }, + "patch_strip": 1 +} + diff --git a/third_party/modules/libxev/metadata.json b/third_party/modules/libxev/metadata.json index 6d9687b0..1fd34d18 100644 --- a/third_party/modules/libxev/metadata.json +++ b/third_party/modules/libxev/metadata.json @@ -21,6 +21,7 @@ "20250222.0-07bcffa", "20250313.0-5773f46", "20250716.0-9f785d2", + "20251010.0-9f785d2" ], "yanked_versions": {} } diff --git a/third_party/zls/BUILD.bazel b/third_party/zls/BUILD.bazel index c00511f4..97064a40 100644 --- a/third_party/zls/BUILD.bazel +++ b/third_party/zls/BUILD.bazel @@ -1,5 +1,74 @@ -load(":zls.bzl", "targets") +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("@rules_zig//zig:defs.bzl", "zig_binary") +load(":resolved_toolchain.bzl", "resolved_toolchain") -toolchain_type(name = "toolchain_type") +# This is the ZLS binary toolchain type. Zig rule authors should use this +# target in the `toolchains` parameter in order to access the appropriate ZLS +# binary for the current execution platform. +toolchain_type( + name = "toolchain_type", + visibility = ["//visibility:public"], +) -targets() +resolved_toolchain( + name = "resolved_toolchain", + # Marked manual so that `bazel test //...` passes + # even if no toolchain is registered. + tags = ["manual"], + visibility = ["//visibility:public"], +) + +exports_files( + [ + "cat.zig", + "zls_build_runner.zig", + "zls_runner.zig", + "zls.completion.tpl", + ], +) + +zig_binary( + name = "runner", + main = "runner.zig", + data = [ + "//zig:resolved_toolchain", + ], + deps = ["@rules_zig//zig/runfiles"], +) + +bzl_library( + name = "zls_completion", + srcs = ["zls_completion.bzl"], + visibility = ["//visibility:public"], + deps = [ + "//zig/private/common:zig_build", + "//zig/private/providers:zig_module_info", + ], +) + +bzl_library( + name = "zls_toolchain", + srcs = ["zls_toolchain.bzl"], + visibility = ["//visibility:public"], + deps = ["@bazel_tools//tools/build_defs/repo:http.bzl"], +) + +bzl_library( + name = "zls_runner", + srcs = ["zls_runner.bzl"], + visibility = ["//visibility:public"], +) + +# Execute `bazel run //util:update_filegroups` to update this target. +filegroup( + name = "all_files", + srcs = [ + ":BUILD.bazel", + ":zls_build_runner.zig", + ":zls.completion.tpl", + ":zls_completion.bzl", + ":zls_runner.zig", + ":zls_toolchain.bzl", + ], + visibility = ["//zig:__pkg__"], +) diff --git a/third_party/zls/cat.zig b/third_party/zls/cat.zig new file mode 100644 index 00000000..ebca6f5a --- /dev/null +++ b/third_party/zls/cat.zig @@ -0,0 +1,31 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +pub fn main() !void { + const gpa = std.heap.page_allocator; + + const args = try std.process.argsAlloc(gpa); + defer std.process.argsFree(gpa, args); + + const file_path = args[1]; + var file = try std.fs.cwd().openFile(file_path, .{ .mode = .read_only}); + defer file.close(); + + if (builtin.zig_version.major == 0 and builtin.zig_version.minor >= 15) { + var write_buffer: [512]u8 = undefined; + var writer = std.fs.File.stdout().writer(&write_buffer); + var stdout = &writer.interface; + + var reader = file.reader(&.{}); + _ = try reader.interface.streamRemaining(stdout); + try stdout.flush(); + } else { + var stdout = std.io.getStdOut().writer(); + var buffer: [4096]u8 = undefined; + while (true) { + const n = try file.read(&buffer); + if (n == 0) break; // EOF + try stdout.writeAll(buffer[0..n]); + } + } +} diff --git a/third_party/zls/resolved_toolchain.bzl b/third_party/zls/resolved_toolchain.bzl new file mode 100644 index 00000000..c5e38e63 --- /dev/null +++ b/third_party/zls/resolved_toolchain.bzl @@ -0,0 +1,24 @@ +"""This module implements an alias rule to the resolved toolchain.""" + +DOC = """\ +Exposes a concrete toolchain which is the result of Bazel resolving the +toolchain for the execution or target platform. +Workaround for https://github.com/bazelbuild/bazel/issues/14009 +""" + +# Forward all the providers +def _resolved_toolchain_impl(ctx): + toolchain_info = ctx.toolchains["//third_party/zls:toolchain_type"] + return [ + toolchain_info, + toolchain_info.default, + toolchain_info.zlstoolchaininfo, + ] + +# Copied from java_toolchain_alias +# https://cs.opensource.google/bazel/bazel/+/master:tools/jdk/java_toolchain_alias.bzl +resolved_toolchain = rule( + implementation = _resolved_toolchain_impl, + toolchains = ["//third_party/zls:toolchain_type"], + doc = DOC, +) diff --git a/third_party/zls/zls.bzl b/third_party/zls/zls.bzl deleted file mode 100644 index 9e28da65..00000000 --- a/third_party/zls/zls.bzl +++ /dev/null @@ -1,187 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -_VERSION = "0.15.0" - -_ARCH = { - "x86_64-linux": struct( - sha256 = "508bfe3fd637d2a02f07f3fc7da8900351f407116b03685c5dae26b4f01a30de", - exec_compatible_with = [ - "@platforms//os:linux", - "@platforms//cpu:x86_64", - ], - ), - "x86_64-macos": struct( - sha256 = "46c31838bfef5adcc7fee82428c3ec2b9abbfae38242639afea5f242ee133d93", - exec_compatible_with = [ - "@platforms//os:macos", - "@platforms//cpu:x86_64", - ], - ), - "aarch64-macos": struct( - sha256 = "76c7a23190f67e67970024065f689c2c49b3c7b0fc16876fb24ef199fb05fc2a", - exec_compatible_with = [ - "@platforms//os:macos", - "@platforms//cpu:aarch64", - ], - ), -} - -ZlsInfo = provider( - fields = { - "bin": "ZLS binary", - }, -) - -def _zls_toolchain_impl(ctx): - default = DefaultInfo( - files = depset(direct = [ctx.file.zls]), - ) - zlsinfo = ZlsInfo( - bin = ctx.file.zls, - ) - toolchain_info = platform_common.ToolchainInfo( - default = default, - zlsinfo = zlsinfo, - ) - - return [ - default, - zlsinfo, - toolchain_info, - ] - -zls_toolchain = rule( - implementation = _zls_toolchain_impl, - attrs = { - "zls": attr.label( - executable = True, - allow_single_file = True, - cfg = "exec", - ), - }, -) - -def _repo_impl(mctx): - for arch, config in _ARCH.items(): - http_archive( - name = "zls_{}".format(arch), - url = "https://github.com/zigtools/zls/releases/download/{version}/zls-{arch}.tar.xz".format( - version = _VERSION, - arch = arch, - ), - sha256 = config.sha256, - build_file_content = """\ -load("@zml//third_party/zls:zls.bzl", "zls_toolchain") -zls_toolchain(name = "toolchain", zls = "zls") -""", - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -repo = module_extension( - implementation = _repo_impl, -) - -def targets(): - for arch, config in _ARCH.items(): - native.toolchain( - name = "toolchain_{}".format(arch), - exec_compatible_with = config.exec_compatible_with, - target_compatible_with = config.exec_compatible_with, - toolchain = "@zls_{}//:toolchain".format(arch), - toolchain_type = "@zml//third_party/zls:toolchain_type", - ) - -def build_runner_tpl(target): - return """\ -const std = @import("std"); - -pub fn main() !void {{ - var gpa: std.heap.GeneralPurposeAllocator(.{{}}) = .{{}}; - defer _ = gpa.deinit(); - var arena_ = std.heap.ArenaAllocator.init(gpa.allocator()); - defer arena_.deinit(); - const arena = arena_.allocator(); - - const build_workspace_directory = try std.process.getEnvVarOwned(arena, "BUILD_WORKSPACE_DIRECTORY"); - var child = std.process.Child.init(&.{{ - "bazel", - "run", - {target}, - }}, arena); - child.stdin_behavior = .Ignore; - child.stdout_behavior = .Inherit; - child.stderr_behavior = .Inherit; - child.cwd = build_workspace_directory; - _ = try child.spawnAndWait(); -}} -""".format(target = repr(target)) - -def runner_tpl(zls, zig_exe_path, zig_lib_path, zig_cache, build_runner): - return """\ -#!/bin/bash -set -eo pipefail - -json_config="$(mktemp)" -cat < ${{json_config}} -{{ - "build_runner_path": "$(realpath {build_runner})", - "global_cache_path": "$(realpath {zig_cache})", - "zig_exe_path": "$(realpath {zig_exe_path})", - "zig_lib_path": "$(realpath {zig_lib_path})" -}} -EOF - -exec {zls} "${{@}}" --config-path "${{json_config}}" -""".format( - zig_lib_path = zig_lib_path, - zig_exe_path = zig_exe_path, - zig_cache = zig_cache, - zls = zls, - build_runner = build_runner, - ) - -def _zls_runner_impl(ctx): - zigtoolchaininfo = ctx.toolchains["@rules_zig//zig:toolchain_type"].zigtoolchaininfo - zlsinfo = ctx.toolchains["@zml//third_party/zls:toolchain_type"].zlsinfo - - build_runner = ctx.actions.declare_file(ctx.label.name + ".build_runner.zig") - ctx.actions.write(build_runner, build_runner_tpl(str(ctx.attr.target.label))) - - zls_runner = ctx.actions.declare_file(ctx.label.name + ".zls_runner.sh") - ctx.actions.write(zls_runner, runner_tpl( - zig_cache = zigtoolchaininfo.zig_cache, - zig_exe_path = zigtoolchaininfo.zig_exe.short_path, - zig_lib_path = zigtoolchaininfo.zig_lib.short_path, - zls = zlsinfo.bin.short_path, - build_runner = build_runner.short_path, - )) - - return [ - DefaultInfo( - files = depset([zls_runner]), - executable = zls_runner, - runfiles = ctx.runfiles( - files = [ - build_runner, - zlsinfo.bin, - ], - transitive_files = zigtoolchaininfo.zig_files, - ), - ), - ] - -zls_runner = rule( - implementation = _zls_runner_impl, - attrs = { - "target": attr.label(mandatory = True, executable = True, cfg = "exec"), - }, - executable = True, - toolchains = [ - "@rules_zig//zig:toolchain_type", - "@zml//third_party/zls:toolchain_type", - ], -) diff --git a/third_party/zls/zls.completion.tpl b/third_party/zls/zls.completion.tpl new file mode 100644 index 00000000..154b4c17 --- /dev/null +++ b/third_party/zls/zls.completion.tpl @@ -0,0 +1,10 @@ +{ + "deps_build_roots": [], + "packages": [ + @@PACKAGES@@ + ], + "include_dirs": [], + "top_level_steps": [], + "available_options": {}, + "c_macros": [] +} diff --git a/third_party/zls/zls_build_runner.zig b/third_party/zls/zls_build_runner.zig new file mode 100644 index 00000000..9ec12c9b --- /dev/null +++ b/third_party/zls/zls_build_runner.zig @@ -0,0 +1,50 @@ +const builtin = @import("builtin"); +const std = @import("std"); + +pub fn main() !void { + var gpa: std.heap.GeneralPurposeAllocator(.{}) = .{}; + defer _ = gpa.deinit(); + var arena_ = std.heap.ArenaAllocator.init(gpa.allocator()); + defer arena_.deinit(); + const arena = arena_.allocator(); + + const build_workspace_directory = try std.process.getEnvVarOwned(arena, "BUILD_WORKSPACE_DIRECTORY"); + var child = std.process.Child.init(&.{ + "bazel", + "run", + "@@__TARGET__@@", + }, arena); + child.stdin_behavior = .Ignore; + child.stdout_behavior = .Pipe; + child.stderr_behavior = .Inherit; + child.cwd = build_workspace_directory; + + try child.spawn(); + + var out = child.stdout.?; + + const file = try out.readToEndAlloc(arena, 16 * 1024 * 1024); + + const need = std.mem.replacementSize( + u8, + file, + "@@__BUILD_WORKSPACE_DIRECTORY__@@", + build_workspace_directory, + ); + + const replaced = try arena.alloc(u8, need); + + _ = std.mem.replace( + u8, + file, + "@@__BUILD_WORKSPACE_DIRECTORY__@@", + build_workspace_directory, + replaced, + ); + + if (comptime builtin.zig_version.order(.{ .major = 0, .minor = 15, .patch = 0 }).compare(.gte)) { + _ = try std.fs.File.stdout().writeAll(replaced); + } else { + _ = try std.io.getStdOut().writeAll(replaced); + } +} diff --git a/third_party/zls/zls_completion.bzl b/third_party/zls/zls_completion.bzl new file mode 100644 index 00000000..02f11f31 --- /dev/null +++ b/third_party/zls/zls_completion.bzl @@ -0,0 +1,69 @@ +"""Implementation of the zls_completion macro.""" +load("@aspect_bazel_lib//lib:utils.bzl", "utils") +load("@bazel_skylib//rules:expand_template.bzl", "expand_template") +load("@rules_zig//zig:defs.bzl", "zig_binary") +load(":zls_write_build_config.bzl", "zls_write_build_config") +load(":zls_write_runner_zig_src.bzl", "zls_write_runner_zig_src") + +def zls_completion(name, deps, **kwargs): + """Entry point for ZLS completion. + + Args: + name: The name of the completion target. + deps: The List of Zig modules to include for completion. + **kwargs: Additional keyword arguments passed to the `zig_binary` + """ + # Generate the ZLS BuildConfig file. + # It contains the list of Zig packages alongside their main Zig file paths. + build_config = name + ".build_config" + zls_write_build_config( + name = build_config, + out = name + ".build_config.json", + deps = deps, + ) + + # Create a target that will be invoked by ZLS using our customer build_runner. + build_config_printer = "print_{}_build_config".format(name) + zig_binary( + name = build_config_printer, + main = Label("//third_party/zls:cat.zig"), + data = [ + ":{}".format(build_config), + ], + args = [ + "$(location :{})".format(build_config), + ], + visibility = ["//visibility:private"], + ) + + # Generate the Zig build runner that will be used by ZLS to query the build config. + expand_template( + name = name + ".build_runner", + out = name + ".build_runner.zig", + substitutions = { + "@@__TARGET__@@": str(utils.to_label(build_config_printer)), + }, + template = Label(":zls_build_runner.zig"), + ) + + # Generate the Zig source file for the ZLS runner binary which embeds the + # rlocationpath of all runtime dependencies of the ZLS runner binary. + zls_write_runner_zig_src( + name = name + ".runner", + out = name + ".runner.zig", + build_runner = ":" + name + ".build_runner.zig", + ) + + zig_binary( + name = name, + main = name + ".runner", + data = [ + "@rules_zig//zig:resolved_toolchain", + "@zml//third_party/zls:resolved_toolchain", + name + ".build_runner", + ], + deps = [ + "@rules_zig//zig/runfiles", + ], + **kwargs + ) diff --git a/third_party/zls/zls_runner.zig b/third_party/zls/zls_runner.zig new file mode 100644 index 00000000..81555e05 --- /dev/null +++ b/third_party/zls/zls_runner.zig @@ -0,0 +1,123 @@ +/// Custom ZLS launcher. +/// +/// Sets up paths to ZLS dependencies with Bazel runfiles. +/// +/// This file is used as a template by `zls_write_runner_zig_src.bzl`. +const std = @import("std"); +const runfiles = @import("runfiles"); +const bazel_builtin = @import("bazel_builtin"); + +fn getRandomFilename(buf: *[std.fs.max_name_bytes]u8, extension: []const u8) ![]const u8 { + const random_bytes_count = 12; + const sub_path_len = comptime std.fs.base64_encoder.calcSize(random_bytes_count); + + var random_bytes: [random_bytes_count]u8 = undefined; + std.crypto.random.bytes(&random_bytes); + var random_name: [sub_path_len]u8 = undefined; + _ = std.fs.base64_encoder.encode(&random_name, &random_bytes); + + const fmt_template = "/tmp/{s}{s}"; + const fmt_args = .{ + @as([]const u8, &random_name), + extension, + }; + return std.fmt.bufPrint(buf, fmt_template, fmt_args) catch @panic("OOM"); +} + +const Config = struct { + /// Override the Zig library path. Will be automatically resolved using the 'zig_exe_path'. + zig_lib_path: ?[]const u8 = null, + + /// Specify the path to the Zig executable (not the directory). If unset, zig is looked up in `PATH`. e.g. `/path/to/zig-templeos-armless-1.0.0/zig`. + zig_exe_path: ?[]const u8 = null, + + /// Specify a custom build runner to resolve build system information. + build_runner_path: ?[]const u8 = null, + + /// Path to a directory that will be used as zig's cache. Will default to `${KnownFolders.Cache}/zls`. + global_cache_path: ?[]const u8 = null, +}; + +pub fn main() !void { + const gpa = std.heap.page_allocator; + var arena = std.heap.ArenaAllocator.init(gpa); + const allocator = arena.allocator(); + + var r_ = try runfiles.Runfiles.create(.{ .allocator = allocator }) orelse + return error.RunfilesNotFound; + defer r_.deinit(allocator); + + const r = r_.withSourceRepo(bazel_builtin.current_repository); + + const zls_bin_rpath = "@@__ZLS_BIN_RPATH__@@"; + var zls_bin_path_buf: [std.fs.max_path_bytes]u8 = undefined; + const zls_bin_path = try r.rlocation(zls_bin_rpath, &zls_bin_path_buf) orelse + return error.RLocationNotFound; + + const zig_exe_rpath = "@@__ZIG_EXE_RPATH__@@"; + var zig_exe_path_buf: [std.fs.max_path_bytes]u8 = undefined; + const zig_exe_path = blk2: { + if (zig_exe_rpath[0] == '/') { + break :blk2 zig_exe_rpath; + } else { + break :blk2 try r.rlocation(zig_exe_rpath, &zig_exe_path_buf) orelse + return error.RLocationNotFound; + } + }; + + const zig_lib_path = "@@__ZIG_LIB_PATH__@@"; + const zig_lib_computed_path = blk: { + if (zig_lib_path[0] == '/') { + break :blk zig_lib_path; + } else { + break :blk null; + } + }; + + const zls_build_runner_rpath = "@@__ZLS_BUILD_RUNNER_RPATH__@@"; + var zls_build_runner_path_buf: [std.fs.max_path_bytes]u8 = undefined; + const zls_build_runner_path = try r.rlocation(zls_build_runner_rpath, &zls_build_runner_path_buf) orelse + return error.RLocationNotFound; + + const global_cache_path = "@@__GLOBAL_CACHE_PATH__@@"; + + const config: Config = .{ + .zig_exe_path = zig_exe_path, + .zig_lib_path = zig_lib_computed_path, + .build_runner_path = zls_build_runner_path, + .global_cache_path = global_cache_path, + }; + + var buf: [std.fs.max_name_bytes]u8 = undefined; + const tmp_file_path = try getRandomFilename(&buf, ".json"); + + { + var tmp_file = try std.fs.createFileAbsolute(tmp_file_path, .{}); + defer tmp_file.close(); + + var out_buf: [4096]u8 = undefined; + var tmp_file_writer = tmp_file.writer(&out_buf); + + const formatter = std.json.fmt(config, .{ .whitespace = .indent_2 }); + + try tmp_file_writer.interface.print("{f}", .{formatter}); + try tmp_file_writer.interface.flush(); + } + + const args = std.process.argsAlloc(allocator) catch return error.ArgsAllocFailed; + defer std.process.argsFree(allocator, args); + + const exec_args_len = args.len - 1 + 3; // Skip args[0] and add "zls" + --config-path + tmp_file_path + var exec_args = try allocator.alloc([]const u8, exec_args_len); + + exec_args[0] = zls_bin_path; // "zls" + + // ${@} + for (args[1..], 0..) |arg, i| { + exec_args[1 + i] = arg; + } + exec_args[exec_args.len - 2] = "--config-path"; + exec_args[exec_args.len - 1] = tmp_file_path; + + return std.process.execv(arena.allocator(), exec_args); +} diff --git a/third_party/zls/zls_toolchain.bzl b/third_party/zls/zls_toolchain.bzl new file mode 100644 index 00000000..b5f7e6c1 --- /dev/null +++ b/third_party/zls/zls_toolchain.bzl @@ -0,0 +1,123 @@ +"""Implementation of the zls_toolchain rule.""" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +_VERSION = "0.15.0" + +_ZLS_PLATFORMS = { + "x86_64-linux": struct( + compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + ), + "x86_64-macos": struct( + compatible_with = [ + "@platforms//os:macos", + "@platforms//cpu:x86_64", + ], + ), + "aarch64-macos": struct( + compatible_with = [ + "@platforms//os:macos", + "@platforms//cpu:aarch64", + ], + ), +} + +_ZLS_VERSIONS = { + "x86_64-linux": "508bfe3fd637d2a02f07f3fc7da8900351f407116b03685c5dae26b4f01a30de", + "x86_64-macos": "46c31838bfef5adcc7fee82428c3ec2b9abbfae38242639afea5f242ee133d93", + "aarch64-macos": "76c7a23190f67e67970024065f689c2c49b3c7b0fc16876fb24ef199fb05fc2a", +} + +ZlsToolchainInfo = provider( + doc = "Provide info for executing zls", + fields = { + "bin": "Executable ZLS binary", + }, +) + +def _zls_toolchain_impl(ctx): + binary = ctx.file.bin + + default_info = DefaultInfo( + files = depset([binary]), + runfiles = ctx.runfiles(files = [binary]), + ) + zlstoolchaininfo = ZlsToolchainInfo( + bin = binary, + ) + toolchain_info = platform_common.ToolchainInfo( + default = default_info, + zlstoolchaininfo = zlstoolchaininfo, + ) + + return [ + default_info, + zlstoolchaininfo, + toolchain_info, + ] + +zls_toolchain = rule( + implementation = _zls_toolchain_impl, + attrs = { + "bin": attr.label( + executable = True, + allow_single_file = True, + cfg = "exec", + ), + }, +) + +def _zls_toolchains_repos_impl(mctx): + for platform, _ in _ZLS_PLATFORMS.items(): + http_archive( + name = "zls_{}".format(platform), + url = "https://github.com/zigtools/zls/releases/download/{version}/zls-{platform}.tar.xz".format( + version = _VERSION, + platform = platform, + ), + sha256 = _ZLS_VERSIONS[platform], + build_file_content = """\ +load("@zml//third_party/zls:zls_toolchain.bzl", "zls_toolchain") +zls_toolchain(name = "toolchain", bin = "zls") +""", + ) + + zls_toolchains_repo(name = "zls_toolchains") + + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = ["zls_toolchains"], + root_module_direct_dev_deps = [], + ) + +zls_toolchains = module_extension( + implementation = _zls_toolchains_repos_impl, +) + +def _zls_toolchains_repo_impl(rctx): + build_content = """# @generated by @zml//third_party/zls:zls_toolchain.bzl""" + + for platform, config in _ZLS_PLATFORMS.items(): + build_content += """ +toolchain( + name = "{platform}_toolchain", + exec_compatible_with = {compatible_with}, + target_compatible_with = {compatible_with}, + toolchain = "@zls_{platform}//:toolchain", + toolchain_type = "@zml//third_party/zls:toolchain_type", +) +""".format( + platform = platform, + compatible_with = config.compatible_with, + ) + + rctx.file("BUILD.bazel", build_content) + +zls_toolchains_repo = repository_rule( + _zls_toolchains_repo_impl, + doc = """Creates a repository with toolchain definitions for all known platforms + which can be registered or selected.""", +) diff --git a/third_party/zls/zls_write_build_config.bzl b/third_party/zls/zls_write_build_config.bzl new file mode 100644 index 00000000..27f9be2e --- /dev/null +++ b/third_party/zls/zls_write_build_config.bzl @@ -0,0 +1,138 @@ +"""Generates the ZLS build config file.""" + +load("@rules_cc//cc/common:cc_common.bzl", "cc_common") +load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") +load("@rules_zig//zig/private:cc_helper.bzl", "need_translate_c") +load("@rules_zig//zig/private/common:translate_c.bzl", "zig_translate_c") +load("@rules_zig//zig/private/common:zig_cache.bzl", "zig_cache_output") +load("@rules_zig//zig/private/common:zig_lib_dir.bzl", "zig_lib_dir") +load("@rules_zig//zig/private/providers:zig_module_info.bzl", "ZigModuleInfo", "zig_module_info") +load( + "@rules_zig//zig/private/providers:zig_target_info.bzl", + "zig_target_platform", +) + +def _zls_construct_zig_module_info_impl(target, ctx): + """Aspect that constructs ZigModuleInfo for zig_binary and zig_library rules.""" + if ZigModuleInfo in target: + return [] + + if ctx.rule.kind not in ("zig_binary", "zig_static_library", "zig_shared_library"): + return [] + + cdeps = [] + zdeps = [] + for dep in ctx.rule.attr.deps: + if ZigModuleInfo in dep: + zdeps.append(dep[ZigModuleInfo]) + elif CcInfo in dep: + cdeps.append(dep[CcInfo]) + + root_module_is_only_dep = len(ctx.rule.attr.deps) == 1 and ZigModuleInfo in ctx.rule.attr.deps[0] + if root_module_is_only_dep: + root_module = ctx.rule.attr.deps[0][ZigModuleInfo] + else: + root_module = zig_module_info( + name = ctx.rule.attr.name, + canonical_name = target.label.name, + main = ctx.rule.file.main, + srcs = ctx.rule.files.srcs, + extra_srcs = ctx.rule.files.extra_srcs, + deps = zdeps, # [bazel_builtin_module(ctx)], + cdeps = cdeps, + zigopts = [], + ) + + return [ + root_module, + ] + +zls_construct_zig_module_info = aspect( + implementation = _zls_construct_zig_module_info_impl, +) + +def _add_context(context): + return json.encode(struct(name = context.name, path = "@@__BUILD_WORKSPACE_DIRECTORY__@@/" + context.main)) + +def _zls_write_build_config_impl(ctx): + zigtoolchaininfo = ctx.toolchains["@rules_zig//zig:toolchain_type"].zigtoolchaininfo + zigtargetinfo = ctx.toolchains["@rules_zig//zig/target:toolchain_type"].zigtargetinfo + + c_module_contexts = [] + c_module_inputs = [] + cc_info = cc_common.merge_cc_infos(cc_infos = [dep[ZigModuleInfo].cc_info for dep in ctx.attr.deps if dep[ZigModuleInfo].cc_info]) + if need_translate_c(cc_info): + global_args = ctx.actions.args() + global_args.use_param_file("@%s") + + zig_target_platform( + target = zigtargetinfo, + args = global_args, + ) + + zig_lib_dir( + zigtoolchaininfo = zigtoolchaininfo, + args = global_args, + ) + + zig_cache_output( + zigtoolchaininfo = zigtoolchaininfo, + args = global_args, + ) + + c_module = zig_translate_c( + ctx = ctx, + name = "c", + zigtoolchaininfo = zigtoolchaininfo, + global_args = global_args, + cc_infos = [cc_info], + ) + + c_module_contexts = [depset(direct = [c_module.module_context])] + c_module_inputs = [c_module.transitive_inputs] + + contexts = depset( + direct = [dep[ZigModuleInfo].module_context for dep in ctx.attr.deps], + transitive = [dep[ZigModuleInfo].transitive_module_contexts for dep in ctx.attr.deps] + c_module_contexts, + ) + + template_dict = ctx.actions.template_dict() + template_dict.add_joined("@@PACKAGES@@", contexts, map_each = _add_context, join_with = ",\n ") + + config = ctx.outputs.out + ctx.actions.expand_template( + output = config, + template = ctx.file._zls_completion_tpl, + computed_substitutions = template_dict, + ) + + return [ + DefaultInfo( + files = depset(direct = [config]), + # Not really runfiles but we need to pull the output of translate_c... + runfiles = ctx.runfiles(transitive_files = depset(transitive = c_module_inputs)), + ), + ] + +zls_write_build_config = rule( + implementation = _zls_write_build_config_impl, + attrs = { + "deps": attr.label_list( + providers = [ZigModuleInfo], + mandatory = True, + aspects = [zls_construct_zig_module_info], + ), + "out": attr.output( + mandatory = True, + ), + "_zls_completion_tpl": attr.label( + default = ":zls.completion.tpl", + allow_single_file = True, + ), + }, + toolchains = [ + "@rules_zig//zig:toolchain_type", + "@rules_zig//zig/target:toolchain_type", + ], + fragments = ["cpp"], +) diff --git a/third_party/zls/zls_write_runner_zig_src.bzl b/third_party/zls/zls_write_runner_zig_src.bzl new file mode 100644 index 00000000..ac89e08e --- /dev/null +++ b/third_party/zls/zls_write_runner_zig_src.bzl @@ -0,0 +1,45 @@ +"""Generates the ZLS runner source file.""" + +load("@aspect_bazel_lib//lib:paths.bzl", "to_rlocation_path") + +def _zls_write_runner_zig_src_impl(ctx): + zigtoolchaininfo = ctx.toolchains["@rules_zig//zig:toolchain_type"].zigtoolchaininfo + zlstoolchaininfo = ctx.toolchains["//third_party/zls:toolchain_type"].zlstoolchaininfo + + zls_runner = ctx.outputs.out + ctx.actions.expand_template( + output = zls_runner, + template = ctx.file._runner_tpl, + substitutions = { + "@@__ZIG_EXE_RPATH__@@": zigtoolchaininfo.zig_exe_rpath, + "@@__ZIG_LIB_PATH__@@": zigtoolchaininfo.zig_lib_rpath, + "@@__ZLS_BIN_RPATH__@@": to_rlocation_path(ctx, zlstoolchaininfo.bin), + "@@__ZLS_BUILD_RUNNER_RPATH__@@": to_rlocation_path(ctx, ctx.file.build_runner), + "@@__GLOBAL_CACHE_PATH__@@": zigtoolchaininfo.zig_cache, + }, + ) + + return [ + DefaultInfo(files = depset([zls_runner])), + ] + +zls_write_runner_zig_src = rule( + implementation = _zls_write_runner_zig_src_impl, + attrs = { + "out": attr.output( + mandatory = True, + ), + "build_runner": attr.label( + allow_single_file = True, + mandatory = True, + ), + "_runner_tpl": attr.label( + default = "zls_runner.zig", + allow_single_file = True, + ), + }, + toolchains = [ + "@rules_zig//zig:toolchain_type", + "//third_party/zls:toolchain_type", + ], +) diff --git a/tools/zls.sh b/tools/zls.sh index 86827c87..8428ee01 100755 --- a/tools/zls.sh +++ b/tools/zls.sh @@ -1,4 +1,4 @@ #!/bin/bash cd "$(dirname "${BASH_SOURCE[0]}")" cd "$(bazel info workspace)" -exec bazel run -- @zml//:zls "${@}" +exec bazel run -- @zml//:completion "${@}" diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 3d2ae081..47a3b5cf 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -54,7 +54,7 @@ zig_library( "torch.zig", "zml.zig", ], - copts = ["-lc", "-freference-trace=20"], + zigopts = ["-lc", "-freference-trace=20"], main = "zml.zig", visibility = ["//visibility:public"], deps = [ From 876ca22ff2bd855435827c6f485b8c3154a2d9d0 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 15 Oct 2025 15:06:58 +0200 Subject: [PATCH 040/881] add patches to http_deb_archive (#338) add patches to http_deb_archive add documentation for command_buffer_compatible pinToCore Optional head dim for Llama 3.1 --- bazel/http_deb_archive.bzl | 1 + examples/llama/llama.zig | 2 +- stdx/stdx.zig | 10 ++++++++++ zml/callback.zig | 8 +++++++- 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/bazel/http_deb_archive.bzl b/bazel/http_deb_archive.bzl index f847e6f3..91b52f5c 100644 --- a/bazel/http_deb_archive.bzl +++ b/bazel/http_deb_archive.bzl @@ -39,6 +39,7 @@ http_deb_archive = repository_rule( "strip_prefix": attr.string(), "build_file": attr.label(allow_single_file = True), "build_file_content": attr.string(), + "patches": attr.label_list(allow_files = True), "workspace_file": attr.label(allow_single_file = True), "workspace_file_content": attr.string(), }, diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 5d0e87d5..606598fb 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -18,7 +18,7 @@ pub const LlamaLM = struct { int: u32, ints: []u32, }), - head_dim: ?u32, + head_dim: ?u32 = null, hidden_size: u32, num_hidden_layers: u32, num_attention_heads: u32, diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 46983493..3ec8b8bb 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -24,3 +24,13 @@ pub inline fn stackSlice(comptime max_len: usize, T: type, len: usize) []T { } pub const noalloc: std.mem.Allocator = if (builtin.mode == .ReleaseFast) undefined else std.testing.failing_allocator; + +pub fn pinToCore(core_id: usize) void { + if (builtin.os.tag == .linux) { + const CPUSet = std.bit_set.ArrayBitSet(usize, std.os.linux.CPU_SETSIZE * @sizeOf(usize)); + + var set: CPUSet = .initEmpty(); + set.set(core_id); + std.os.linux.sched_setaffinity(0, @ptrCast(&set.masks)) catch {}; + } +} diff --git a/zml/callback.zig b/zml/callback.zig index 5af96d15..ea7a4b23 100644 --- a/zml/callback.zig +++ b/zml/callback.zig @@ -120,7 +120,13 @@ pub fn call( pub const Config = struct { output_operand_aliases: []const i64 = &.{}, copy_inputs_to_host_pinned: bool = false, - // TODO: document precisely what `command_buffer_compatible` is doing and its limitations. + + /// Indicates that the handler is compatible with command buffers (ie Cuda graphs). + /// The Cuda backend traces the execution of the callback and records it as a Cuda graph. + /// To be compatible with CUDA graphs, the callback need: + /// 1. to not use any synchronous operations + /// 2. to not allocate any new host/device memory + /// See: https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#prohibited-and-unhandled-operations. traits: pjrt.ffi.HandlerTraits = .{ .command_buffer_compatible = false }, // TODO: handle sharded inputs has_side_effect: bool = true, From bc05231f30742c8b3f1c4073b62e7f96891e7b58 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Thu, 16 Oct 2025 11:48:09 +0200 Subject: [PATCH 041/881] workspace: use libxev from the BCR (#341) This marks the last PR for getting out of our custom registry. Module is now at https://registry.bazel.build/modules/libxev and @steeve is a maintainer as well. --- MODULE.bazel | 2 +- MODULE.bazel.lock | 8 ++++---- async/BUILD.bazel | 2 +- .../zigcoro/20240829.0-fc1db29/overlay/BUILD.bazel | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 1b638ce0..49f7507c 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -6,7 +6,7 @@ bazel_dep(name = "abseil-cpp", version = "20250814.0") bazel_dep(name = "aspect_bazel_lib", version = "2.20.0") bazel_dep(name = "aspect_rules_py", version = "1.6.3") bazel_dep(name = "bazel_skylib", version = "1.8.1") -bazel_dep(name = "libxev", version = "20251010.0-9f785d2") +bazel_dep(name = "libxev", version = "0.0.0-20251014.0-9f785d2") bazel_dep(name = "patchelf", version = "0.18.0") bazel_dep(name = "pcre2", version = "10.45") bazel_dep(name = "platforms", version = "1.0.0") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index e5f0dd8e..3bec3fef 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -69,6 +69,7 @@ "https://bazel-registry.zml.ai/modules/jsoncpp/1.9.5/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/jsoncpp/1.9.6/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/libpfm/4.11.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/libxev/0.0.0-20251014.0-9f785d2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/nlohmann_json/3.6.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/package_metadata/0.0.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/patchelf/0.18.0/MODULE.bazel": "not found", @@ -170,7 +171,6 @@ "https://bazel-registry.zml.ai/modules/rules_shell/0.4.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_zig/0.12.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.4/MODULE.bazel": "not found", @@ -272,6 +272,8 @@ "https://bcr.bazel.build/modules/jsoncpp/1.9.6/MODULE.bazel": "2f8d20d3b7d54143213c4dfc3d98225c42de7d666011528dc8fe91591e2e17b0", "https://bcr.bazel.build/modules/jsoncpp/1.9.6/source.json": "a04756d367a2126c3541682864ecec52f92cdee80a35735a3cb249ce015ca000", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", + "https://bcr.bazel.build/modules/libxev/0.0.0-20251014.0-9f785d2/MODULE.bazel": "d8bda6be48f8c0613d3393cfe15143467a7ee17d7e702fa15719ddc3e9fb646a", + "https://bcr.bazel.build/modules/libxev/0.0.0-20251014.0-9f785d2/source.json": "52ac2605138fd00de1413a13b2b8c5ceb4adc0d8e052133db888400f6768746f", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/source.json": "f448c6e8963fdfa7eb831457df83ad63d3d6355018f6574fb017e8169deb43a9", "https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92", @@ -396,8 +398,6 @@ "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", "https://bcr.bazel.build/modules/rules_swift/2.1.1/source.json": "40fc69dfaac64deddbb75bd99cdac55f4427d9ca0afbe408576a65428427a186", - "https://bcr.bazel.build/modules/rules_zig/0.12.1/MODULE.bazel": "5953094b681c212eebe3dd4275809c52b9177aeaa058d8f1bff433822c0dcd01", - "https://bcr.bazel.build/modules/rules_zig/0.12.1/source.json": "d880cdc3473bfcf172c9aad9292e856430496e198d805539bb8be98e79f58a7c", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", "https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4", @@ -2811,7 +2811,7 @@ "@@rules_zig+//zig:extensions.bzl%zig": { "general": { "bzlTransitiveDigest": "wV3euhBZSI0D2ZlqbY1YW592mghFY88d9H6zBMQRaJY=", - "usagesDigest": "T0xLzuDAWCJjFcphcIcT31t1HM/Yg7RxtfOUb+XmxTw=", + "usagesDigest": "FR3I3Uy7g7MHBta/AuDtKPMdikDVo0hgQZNdLunZDX4=", "recordedFileInputs": { "@@//bazel/zig_index.json": "42ea73420b15741770d5d8d0ded1311a2dfda1f963f0802dea3418f6e060ffef", "@@rules_zig+//zig/private/versions.json": "6eb85ebaee72c4e6fbc82d8ad73f69e0899d839b70a7e19caa792ab6a5fa0c36" diff --git a/async/BUILD.bazel b/async/BUILD.bazel index 82fc0966..647ac4d4 100644 --- a/async/BUILD.bazel +++ b/async/BUILD.bazel @@ -16,7 +16,7 @@ zig_library( visibility = ["//visibility:public"], deps = [ "//stdx", - "@libxev//:xev", + "@libxev", ], ) diff --git a/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/BUILD.bazel b/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/BUILD.bazel index 29227e97..2aa934ca 100644 --- a/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/BUILD.bazel +++ b/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/BUILD.bazel @@ -24,6 +24,6 @@ zig_library( visibility = ["//visibility:public"], deps = [ ":libcoro_options", - "@libxev//:xev", + "@libxev", ], ) From 319cf48b2ca5b6117d23794fd456a1b54225e2c4 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Fri, 17 Oct 2025 14:51:29 +0200 Subject: [PATCH 042/881] pjrt/cpu: fix compilation without cpu (#343) Zig modules that want to access the "c" module needs to have a non empty CcCompilationContext for now. So basically depending on an empty cc_library is not enough, it needs to contain something (defines or headers or includes or wathever) --- runtimes/cpu/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/runtimes/cpu/BUILD.bazel b/runtimes/cpu/BUILD.bazel index 40760fd2..04156daf 100644 --- a/runtimes/cpu/BUILD.bazel +++ b/runtimes/cpu/BUILD.bazel @@ -20,6 +20,7 @@ config_setting( cc_library( name = "empty", + defines = ["ZML_RUNTIME_CPU_DISABLED"], ) copy_to_directory( From 1a263cf4e25e6e7c8998229bff92db7ff0a01fa7 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Fri, 17 Oct 2025 14:51:46 +0200 Subject: [PATCH 043/881] chore: remove unused target (#346) --- runtimes/cuda/libpjrt_cuda.BUILD.bazel | 6 ------ 1 file changed, 6 deletions(-) diff --git a/runtimes/cuda/libpjrt_cuda.BUILD.bazel b/runtimes/cuda/libpjrt_cuda.BUILD.bazel index f63f0ff9..b63a30fb 100644 --- a/runtimes/cuda/libpjrt_cuda.BUILD.bazel +++ b/runtimes/cuda/libpjrt_cuda.BUILD.bazel @@ -1,12 +1,6 @@ load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@zml//bazel:patchelf.bzl", "patchelf") -cc_shared_library( - name = "zmlxcuda_so", - shared_lib_name = "lib/libzmlxcuda.so.0", - deps = ["@zml//runtimes/cuda:zmlxcuda_lib"], -) - patchelf( name = "libpjrt_cuda_so", src = "libpjrt_cuda.so", From 2ff320253f6e15a5e19f946de6a9030e12a9e834 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 17 Oct 2025 14:55:49 +0200 Subject: [PATCH 044/881] ci: handle no /tmp/zig-cache (#342) --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b033adec..e27a8763 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -56,7 +56,7 @@ jobs: - name: Ensure self hosted cache permissions if: ${{ !matrix.runtimes.enable_cache }} - run: sudo chmod -R 777 /tmp/zig-cache && sudo chmod -R 777 /tmp/zml + run: mkdir -p /tmp/zig-cache && mkdir -p /tmp/zml && sudo chmod -R 777 /tmp/zig-cache && sudo chmod -R 777 /tmp/zml - uses: bazel-contrib/setup-bazel@0.15.0 with: From 2613c1137c68b9e0d3786c040149f2b34613d60e Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Fri, 17 Oct 2025 19:04:17 +0200 Subject: [PATCH 045/881] workspace: use rules_zig archive_override pre 0.12.2 (#345) Fixes shared_library and PJRT loading. Fixes os/log.h non inclusion. --- MODULE.bazel | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/MODULE.bazel b/MODULE.bazel index 49f7507c..6b13e17e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -24,6 +24,14 @@ bazel_dep(name = "with_cfg.bzl", version = "0.11.0") bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2", dev_dependency = True) +# Remove this when rules_zig@0.12.2 is released. +archive_override( + module_name = "rules_zig", + urls = ["https://github.com/aherrmann/rules_zig/archive/cbf03e45c9ffa7d23a50790929649805a08afbf9.tar.gz"], + strip_prefix = "rules_zig-cbf03e45c9ffa7d23a50790929649805a08afbf9", + integrity = "sha256-lu6gNjulEIqG68I7DRyJzWInDVk3lL+mCj2J14vYKwc=", +) + zig = use_extension("@rules_zig//zig:extensions.bzl", "zig") zig.index(file = "//bazel:zig_index.json") zig.toolchain(zig_version = "0.15.1") From 892766c65277f38b5cd3f10e7222fd03988cc2b0 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Fri, 17 Oct 2025 19:04:39 +0200 Subject: [PATCH 046/881] zls: add missing cc toolchain (#347) --- third_party/zls/BUILD.bazel | 34 +--------------------- third_party/zls/zls_write_build_config.bzl | 3 +- 2 files changed, 3 insertions(+), 34 deletions(-) diff --git a/third_party/zls/BUILD.bazel b/third_party/zls/BUILD.bazel index 97064a40..6c1560ea 100644 --- a/third_party/zls/BUILD.bazel +++ b/third_party/zls/BUILD.bazel @@ -27,38 +27,6 @@ exports_files( ], ) -zig_binary( - name = "runner", - main = "runner.zig", - data = [ - "//zig:resolved_toolchain", - ], - deps = ["@rules_zig//zig/runfiles"], -) - -bzl_library( - name = "zls_completion", - srcs = ["zls_completion.bzl"], - visibility = ["//visibility:public"], - deps = [ - "//zig/private/common:zig_build", - "//zig/private/providers:zig_module_info", - ], -) - -bzl_library( - name = "zls_toolchain", - srcs = ["zls_toolchain.bzl"], - visibility = ["//visibility:public"], - deps = ["@bazel_tools//tools/build_defs/repo:http.bzl"], -) - -bzl_library( - name = "zls_runner", - srcs = ["zls_runner.bzl"], - visibility = ["//visibility:public"], -) - # Execute `bazel run //util:update_filegroups` to update this target. filegroup( name = "all_files", @@ -70,5 +38,5 @@ filegroup( ":zls_runner.zig", ":zls_toolchain.bzl", ], - visibility = ["//zig:__pkg__"], + visibility = ["@rules_zig//zig:__pkg__"], ) diff --git a/third_party/zls/zls_write_build_config.bzl b/third_party/zls/zls_write_build_config.bzl index 27f9be2e..22c1f83c 100644 --- a/third_party/zls/zls_write_build_config.bzl +++ b/third_party/zls/zls_write_build_config.bzl @@ -2,6 +2,7 @@ load("@rules_cc//cc/common:cc_common.bzl", "cc_common") load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") +load("@rules_cc//cc:find_cc_toolchain.bzl", "use_cc_toolchain") load("@rules_zig//zig/private:cc_helper.bzl", "need_translate_c") load("@rules_zig//zig/private/common:translate_c.bzl", "zig_translate_c") load("@rules_zig//zig/private/common:zig_cache.bzl", "zig_cache_output") @@ -133,6 +134,6 @@ zls_write_build_config = rule( toolchains = [ "@rules_zig//zig:toolchain_type", "@rules_zig//zig/target:toolchain_type", - ], + ] + use_cc_toolchain(mandatory = False), fragments = ["cpp"], ) From f1dce257fee7dad6c0d7f91aa782a1195e9646db Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Fri, 17 Oct 2025 19:05:29 +0200 Subject: [PATCH 047/881] all: provide -fllvm to all targets that requires it (#348) Until zig x86 backend is fixed. --- MODULE.bazel.lock | 91 ++++++++++++++++++++------------------ examples/mnist/BUILD.bazel | 1 - runtimes/cuda/BUILD.bazel | 2 +- upb/BUILD.bazel | 4 ++ zml/BUILD.bazel | 7 ++- zml/tokenizer/BUILD.bazel | 4 ++ 6 files changed, 62 insertions(+), 47 deletions(-) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 3bec3fef..fe78fb19 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -171,6 +171,7 @@ "https://bazel-registry.zml.ai/modules/rules_shell/0.4.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_zig/0.12.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.4/MODULE.bazel": "not found", @@ -398,6 +399,8 @@ "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", "https://bcr.bazel.build/modules/rules_swift/2.1.1/source.json": "40fc69dfaac64deddbb75bd99cdac55f4427d9ca0afbe408576a65428427a186", + "https://bcr.bazel.build/modules/rules_zig/0.12.1/MODULE.bazel": "5953094b681c212eebe3dd4275809c52b9177aeaa058d8f1bff433822c0dcd01", + "https://bcr.bazel.build/modules/rules_zig/0.12.1/source.json": "d880cdc3473bfcf172c9aad9292e856430496e198d805539bb8be98e79f58a7c", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", "https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4", @@ -2811,89 +2814,89 @@ "@@rules_zig+//zig:extensions.bzl%zig": { "general": { "bzlTransitiveDigest": "wV3euhBZSI0D2ZlqbY1YW592mghFY88d9H6zBMQRaJY=", - "usagesDigest": "FR3I3Uy7g7MHBta/AuDtKPMdikDVo0hgQZNdLunZDX4=", + "usagesDigest": "PFS+UEa30JWWU52AUTNHD1nDzqpwjSts6jIKzD7gVJU=", "recordedFileInputs": { - "@@//bazel/zig_index.json": "42ea73420b15741770d5d8d0ded1311a2dfda1f963f0802dea3418f6e060ffef", + "@@//bazel/zig_index.json": "e6c6b29fa227fa9b1806dc03af2b6dab00f52f34d39878557074e1626027259f", "@@rules_zig+//zig/private/versions.json": "6eb85ebaee72c4e6fbc82d8ad73f69e0899d839b70a7e19caa792ab6a5fa0c36" }, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { - "zig_0.15.1_aarch64-linux": { + "zig_0.15.2_aarch64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.15.1/zig-aarch64-linux-0.15.1.tar.xz", + "url": "https://ziglang.org/download/0.15.2/zig-aarch64-linux-0.15.2.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "bb4a8d2ad735e7fba764c497ddf4243cb129fece4148da3222a7046d3f1f19fe", - "zig_version": "0.15.1", + "sha256": "958ed7d1e00d0ea76590d27666efbf7a932281b3d7ba0c6b01b0ff26498f667f", + "zig_version": "0.15.2", "platform": "aarch64-linux" } }, - "zig_0.15.1_aarch64-macos": { + "zig_0.15.2_aarch64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.15.1/zig-aarch64-macos-0.15.1.tar.xz", + "url": "https://ziglang.org/download/0.15.2/zig-aarch64-macos-0.15.2.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "c4bd624d901c1268f2deb9d8eb2d86a2f8b97bafa3f118025344242da2c54d7b", - "zig_version": "0.15.1", + "sha256": "3cc2bab367e185cdfb27501c4b30b1b0653c28d9f73df8dc91488e66ece5fa6b", + "zig_version": "0.15.2", "platform": "aarch64-macos" } }, - "zig_0.15.1_aarch64-windows": { + "zig_0.15.2_aarch64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.15.1/zig-aarch64-windows-0.15.1.zip", + "url": "https://ziglang.org/download/0.15.2/zig-aarch64-windows-0.15.2.zip", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "1f1bf16228b0ffcc882b713dc5e11a6db4219cb30997e13c72e8e723c2104ec6", - "zig_version": "0.15.1", + "sha256": "b926465f8872bf983422257cd9ec248bb2b270996fbe8d57872cca13b56fc370", + "zig_version": "0.15.2", "platform": "aarch64-windows" } }, - "zig_0.15.1_x86_64-linux": { + "zig_0.15.2_x86_64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.15.1/zig-x86_64-linux-0.15.1.tar.xz", + "url": "https://ziglang.org/download/0.15.2/zig-x86_64-linux-0.15.2.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "c61c5da6edeea14ca51ecd5e4520c6f4189ef5250383db33d01848293bfafe05", - "zig_version": "0.15.1", + "sha256": "02aa270f183da276e5b5920b1dac44a63f1a49e55050ebde3aecc9eb82f93239", + "zig_version": "0.15.2", "platform": "x86_64-linux" } }, - "zig_0.15.1_x86_64-macos": { + "zig_0.15.2_x86_64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.15.1/zig-x86_64-macos-0.15.1.tar.xz", + "url": "https://ziglang.org/download/0.15.2/zig-x86_64-macos-0.15.2.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "9919392e0287cccc106dfbcbb46c7c1c3fa05d919567bb58d7eb16bca4116184", - "zig_version": "0.15.1", + "sha256": "375b6909fc1495d16fc2c7db9538f707456bfc3373b14ee83fdd3e22b3d43f7f", + "zig_version": "0.15.2", "platform": "x86_64-macos" } }, - "zig_0.15.1_x86_64-windows": { + "zig_0.15.2_x86_64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.15.1/zig-x86_64-windows-0.15.1.zip", + "url": "https://ziglang.org/download/0.15.2/zig-x86_64-windows-0.15.2.zip", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "91e69e887ca8c943ce9a515df3af013d95a66a190a3df3f89221277ebad29e34", - "zig_version": "0.15.1", + "sha256": "3a0ed1e8799a2f8ce2a6e6290a9ff22e6906f8227865911fb7ddedc3cc14cb0c", + "zig_version": "0.15.2", "platform": "x86_64-windows" } }, @@ -2901,28 +2904,28 @@ "repoRuleId": "@@rules_zig+//zig/private/repo:toolchains_repo.bzl%toolchains_repo", "attributes": { "names": [ - "zig_0.15.1_aarch64-linux", - "zig_0.15.1_aarch64-macos", - "zig_0.15.1_aarch64-windows", - "zig_0.15.1_x86_64-linux", - "zig_0.15.1_x86_64-macos", - "zig_0.15.1_x86_64-windows" + "zig_0.15.2_aarch64-linux", + "zig_0.15.2_aarch64-macos", + "zig_0.15.2_aarch64-windows", + "zig_0.15.2_x86_64-linux", + "zig_0.15.2_x86_64-macos", + "zig_0.15.2_x86_64-windows" ], "labels": [ - "@zig_0.15.1_aarch64-linux//:zig_toolchain", - "@zig_0.15.1_aarch64-macos//:zig_toolchain", - "@zig_0.15.1_aarch64-windows//:zig_toolchain", - "@zig_0.15.1_x86_64-linux//:zig_toolchain", - "@zig_0.15.1_x86_64-macos//:zig_toolchain", - "@zig_0.15.1_x86_64-windows//:zig_toolchain" + "@zig_0.15.2_aarch64-linux//:zig_toolchain", + "@zig_0.15.2_aarch64-macos//:zig_toolchain", + "@zig_0.15.2_aarch64-windows//:zig_toolchain", + "@zig_0.15.2_x86_64-linux//:zig_toolchain", + "@zig_0.15.2_x86_64-macos//:zig_toolchain", + "@zig_0.15.2_x86_64-windows//:zig_toolchain" ], "zig_versions": [ - "0.15.1", - "0.15.1", - "0.15.1", - "0.15.1", - "0.15.1", - "0.15.1" + "0.15.2", + "0.15.2", + "0.15.2", + "0.15.2", + "0.15.2", + "0.15.2" ], "exec_lengths": [ 2, diff --git a/examples/mnist/BUILD.bazel b/examples/mnist/BUILD.bazel index 22e9d675..11ac0c66 100644 --- a/examples/mnist/BUILD.bazel +++ b/examples/mnist/BUILD.bazel @@ -11,7 +11,6 @@ zig_binary( "@mnist//:mnist.safetensors", "@mnist//:t10k-images.idx3-ubyte", ], - zigopts = ["-fllvm"], main = "mnist.zig", deps = [ "//async", diff --git a/runtimes/cuda/BUILD.bazel b/runtimes/cuda/BUILD.bazel index 19eecbc1..cf266d7b 100644 --- a/runtimes/cuda/BUILD.bazel +++ b/runtimes/cuda/BUILD.bazel @@ -5,7 +5,7 @@ zig_shared_library( name = "zmlxcuda", # Use Clang's compiler-rt, but disable stack checking # to avoid requiring on the _zig_probe_stack symbol. - zigopts = ["-fno-stack-check", "-fllvm"], + zigopts = ["-fno-stack-check"], main = "zmlxcuda.zig", shared_lib_name = "libzmlxcuda.so.0", visibility = ["@libpjrt_cuda//:__subpackages__"], diff --git a/upb/BUILD.bazel b/upb/BUILD.bazel index ebf973fb..3273aebb 100644 --- a/upb/BUILD.bazel +++ b/upb/BUILD.bazel @@ -14,4 +14,8 @@ zig_library( main = "upb.zig", visibility = ["//visibility:public"], deps = [":upb_c"], + zigopts = [ + #TODO(cerisier): Remove me when this is done inside rules_zig. + "-fllvm", + ], ) diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 47a3b5cf..be9d5558 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -54,7 +54,12 @@ zig_library( "torch.zig", "zml.zig", ], - zigopts = ["-lc", "-freference-trace=20"], + zigopts = [ + "-lc", + "-freference-trace=20", + #TODO(cerisier): Remove me when this is done inside rules_zig. + "-fllvm", + ], main = "zml.zig", visibility = ["//visibility:public"], deps = [ diff --git a/zml/tokenizer/BUILD.bazel b/zml/tokenizer/BUILD.bazel index 012e218c..d7c8f6de 100644 --- a/zml/tokenizer/BUILD.bazel +++ b/zml/tokenizer/BUILD.bazel @@ -19,6 +19,10 @@ zig_binary( name = "main", main = "main.zig", visibility = ["//visibility:public"], + zigopts = [ + #TODO(cerisier): Remove me when this is done inside rules_zig. + "-fllvm" + ], deps = [ ":tokenizer", "//async", From 711ed46f5b389557313ebfeb836a68cbb69c2dbc Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Fri, 17 Oct 2025 19:05:47 +0200 Subject: [PATCH 048/881] workspace: update zig version to 0.15.2 (#349) --- .github/workflows/ci.yaml | 2 +- MODULE.bazel | 2 +- bazel/zig_index.json | 289 +++++++++++++++++++++++++++----------- 3 files changed, 212 insertions(+), 81 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e27a8763..67e7ba75 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -103,7 +103,7 @@ jobs: - name: Setup Zig uses: mlugg/setup-zig@v2 with: - version: 0.15.1 + version: 0.15.2 mirror: https://mirror.zml.ai/zig - name: Zig Format diff --git a/MODULE.bazel b/MODULE.bazel index 6b13e17e..a8a83539 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -34,7 +34,7 @@ archive_override( zig = use_extension("@rules_zig//zig:extensions.bzl", "zig") zig.index(file = "//bazel:zig_index.json") -zig.toolchain(zig_version = "0.15.1") +zig.toolchain(zig_version = "0.15.2") zig.mirrors(urls = [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/", diff --git a/bazel/zig_index.json b/bazel/zig_index.json index 2dec4081..d708f008 100644 --- a/bazel/zig_index.json +++ b/bazel/zig_index.json @@ -1,139 +1,270 @@ { "master": { - "version": "0.16.0-dev.27+83f773fc6", - "date": "2025-08-24", + "version": "0.16.0-dev.732+2f3234c76", + "date": "2025-10-14", "docs": "https://ziglang.org/documentation/master/", "stdDocs": "https://ziglang.org/documentation/master/std/", "src": { - "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.27+83f773fc6.tar.xz", - "shasum": "afddafbede9becaa6d94a544d3115893cc6aa6492a58545b4f638c58990586dc", - "size": "21370308" + "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.732+2f3234c76.tar.xz", + "shasum": "57cd3a69985ad3ccf3330c801959d193ee9a45a83e3c3ffcdd5dac4d15fe6886", + "size": "21457132" }, "bootstrap": { - "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.27+83f773fc6.tar.xz", - "shasum": "109cdda833baf951ce8116d3fdfd50b3770b02cff5c057f777601ca694f44b7c", - "size": "52732704" + "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.732+2f3234c76.tar.xz", + "shasum": "54c02c70217657bdf88309d62a90df1bcd17662125790a28f19dddde3b90f7aa", + "size": "54219768" }, "x86_64-macos": { - "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.27+83f773fc6.tar.xz", - "shasum": "1ac88b947a6001eb30d41c15c404847209dfe945a27dbce6d1b7d401e4aef325", - "size": "55817716" + "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.732+2f3234c76.tar.xz", + "shasum": "3067dde8a2e11627c25b890df6f3fb4a88fb96e11cc1ef343c27c8a6d803402f", + "size": "56569768" }, "aarch64-macos": { - "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.27+83f773fc6.tar.xz", - "shasum": "75243ad6d2e9fcd634862dda9003883484024fd2e067fc7e4a0f6c524cb6c86e", - "size": "50659200" + "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.732+2f3234c76.tar.xz", + "shasum": "ff75284a7a084f700594b094a22911add95fe0cd7a484dcb999f51946bfca3e9", + "size": "51435252" }, "x86_64-linux": { - "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.27+83f773fc6.tar.xz", - "shasum": "c260aefaf5bf10bbd48101217874ccc9c0a37513729dabbbfe09bcf18fff1b5b", - "size": "53759360" + "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.732+2f3234c76.tar.xz", + "shasum": "f4d7bd32dbf88a2b4a020d8f6af7d4af0dca7db5a7a556ed832a1f1f684729dd", + "size": "54481728" }, "aarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.27+83f773fc6.tar.xz", - "shasum": "658241c15ad827a89878b2e35e13261cd1c2aaefe9a8bb94ccb907725cf80564", - "size": "49485912" + "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.732+2f3234c76.tar.xz", + "shasum": "7eb157b88e5c04b74653f30af2f8a078448504c80332dc3efa9b03a9b20a9552", + "size": "50239776" }, "arm-linux": { - "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.27+83f773fc6.tar.xz", - "shasum": "6039c11f41bd032ec3cbda2d92e38b409aa9d9e2935b21266033b820ecb1a668", - "size": "50475500" + "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.732+2f3234c76.tar.xz", + "shasum": "4efa2363c4d5736eed5de54f42a051f70201e2eb49514dc8691e8469a3b5ec08", + "size": "51089884" }, "riscv64-linux": { - "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.27+83f773fc6.tar.xz", - "shasum": "f56712849af75bb1bfda502c8cdfc52dcc41c18d3d7eddd329d94be85538d258", - "size": "53610616" + "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.732+2f3234c76.tar.xz", + "shasum": "05ef8e27a3c74c877f73ee844dfb8c15162f37811a7924000e46d55f1bac18fd", + "size": "54282992" }, "powerpc64le-linux": { - "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.27+83f773fc6.tar.xz", - "shasum": "0562fdd578b5ae65a17e614454e67466314790a1d373279a4b054398f6a7b364", - "size": "53585484" + "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.732+2f3234c76.tar.xz", + "shasum": "105a5f5ad1f040d4119377036944dff092f970d4c6e9d5585573fe81a489516b", + "size": "54288284" }, "x86-linux": { - "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.27+83f773fc6.tar.xz", - "shasum": "6eafb1b1d81118066dc03b17bed6612d7a9da057794f1d0f2451c249cf8a231d", - "size": "56334072" + "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.732+2f3234c76.tar.xz", + "shasum": "1d05a15af12aaf97d9dee58f4707831c146f7cedec21a11180bd471b97221af4", + "size": "57162212" }, "loongarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.27+83f773fc6.tar.xz", - "shasum": "4fddf62a6698dbece51f504b4de8360a55d24efb0c7a56117eef64c8b717d13f", - "size": "50823012" + "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.732+2f3234c76.tar.xz", + "shasum": "856e9155ccf551dfcd9d83a9b135f7a532dcc3f8ef29a1a7ca513e5e7cf1d8be", + "size": "51527580" }, "s390x-linux": { - "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.27+83f773fc6.tar.xz", - "shasum": "12ed086066c373e4098dc9f132e42bbc478a3be6bd5fc6967f39396c43543f89", - "size": "53525496" + "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.732+2f3234c76.tar.xz", + "shasum": "35ded5aab84f397eeeba384d6dfb9869bfd6c8962125421df82363f009d37302", + "size": "53991088" }, "x86_64-windows": { - "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.27+83f773fc6.zip", - "shasum": "ae987b8d93eec8a923bba26a74d20b5d15eac7ee9694c44db82c232247810a51", - "size": "93312425" + "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.732+2f3234c76.zip", + "shasum": "14594707e551dabe476e350f08439d506231725ff1c270180cf5eb5d943e33ad", + "size": "94274022" }, "aarch64-windows": { - "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.27+83f773fc6.zip", - "shasum": "2e3c95d044dd36d302a301fb029ea080b2bda4ef8e42bbb1b2a0246973952c53", - "size": "89157759" + "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.732+2f3234c76.zip", + "shasum": "fd07bca688fa7f4a89d9595f6475b61839ed34dd4e98322b7ee594fde12aafbe", + "size": "90225621" }, "x86-windows": { - "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.27+83f773fc6.zip", - "shasum": "b696f07f7104da430c7b37750bc6e36571c3ffcd5bc49240d8a19aa4b431049f", - "size": "95215201" + "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.732+2f3234c76.zip", + "shasum": "c65ddde1112852498be49ae136608e2bb9f7a1f78347447f39589288fba281fc", + "size": "96083498" }, "aarch64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.27+83f773fc6.tar.xz", - "shasum": "0295a3f6bebb6d25fbc916cd4e3cff1285acab75b9591695069ad9a878301203", - "size": "49384624" + "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.732+2f3234c76.tar.xz", + "shasum": "3e63836c990e835e8257223664ace605c9aff32c38b546213e12e8e0004236a0", + "size": "50184132" }, "arm-freebsd": { - "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.27+83f773fc6.tar.xz", - "shasum": "a293670c7ee89ce6cf11f03dee297de2a31ffdc9d26f2c0e6e78fd66b11b2327", - "size": "50925296" + "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.732+2f3234c76.tar.xz", + "shasum": "29580f93f4e2e1c947f09f2ac27cc9b035744e27093d3838d7c01315b70dbf80", + "size": "51776420" }, "powerpc64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-powerpc64-freebsd-0.16.0-dev.27+83f773fc6.tar.xz", - "shasum": "503922dd11374ddc44013ae3d7b07a9a17a1404da296bc30bf8fdc6d7273fd82", - "size": "52148468" + "tarball": "https://ziglang.org/builds/zig-powerpc64-freebsd-0.16.0-dev.732+2f3234c76.tar.xz", + "shasum": "6a8ac2bafea198104c09be1d9d314bfb8255281efc6a1550d92642d5965d25ab", + "size": "52866216" }, "powerpc64le-freebsd": { - "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.27+83f773fc6.tar.xz", - "shasum": "1cf44603dec137a35551916ab5e542c36e6c1ec54df4efd7783f8ae66c80acc4", - "size": "53516496" + "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.732+2f3234c76.tar.xz", + "shasum": "5acbe627380efa3d746dfc01d0fb49744610caf2acd935364ce2f82ae754bb60", + "size": "54293208" }, "riscv64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.27+83f773fc6.tar.xz", - "shasum": "c4b1c691328abacf7f7b07f1900dd44c2f2852a0b2611378a2531975abb6f260", - "size": "53716116" + "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.732+2f3234c76.tar.xz", + "shasum": "52e11b1de3b1ec121418228a8d627027e087fe0d2178a9952901acdec75c513d", + "size": "54400788" }, "x86_64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.27+83f773fc6.tar.xz", - "shasum": "989caf4343e1dfb4c9a68f3d3146cac96729e6f6e58276001654307f56c43981", - "size": "53808720" + "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.732+2f3234c76.tar.xz", + "shasum": "a3b42e2719c9b20b0b53a028d4324a96ff6ab3da47c8fa7e3c46d9809397ef7b", + "size": "54620776" }, "aarch64-netbsd": { - "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.27+83f773fc6.tar.xz", - "shasum": "bbbe2aa2163d3c25d7414795537088245f9ec3cb53f10518b5824e1a31c35ff9", - "size": "49387264" + "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.732+2f3234c76.tar.xz", + "shasum": "99e9b13e4b1c456dafcd7a72cde8c94b165cc162a1e6cf83f2330f4117527c56", + "size": "50150340" }, "arm-netbsd": { - "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.27+83f773fc6.tar.xz", - "shasum": "264346f44664055d61c91087d0c55fdd04e3bdf1f2f0f969cd5efcf47d54b247", - "size": "52034816" + "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.732+2f3234c76.tar.xz", + "shasum": "33734775600ac79a75382140963063bcf012d8ffe04f426ca17001405b857d10", + "size": "52805340" }, "x86-netbsd": { - "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.27+83f773fc6.tar.xz", - "shasum": "f5e3cdab5d7e24552eee2200b1e4ac45cd84f13d5d64ae0f558ad0210d6883d5", - "size": "56915916" + "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.732+2f3234c76.tar.xz", + "shasum": "e5e8894b82ac787229cea9354bc2eb549292702e011ab9370d9c8825ad94f119", + "size": "57766016" }, "x86_64-netbsd": { - "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.27+83f773fc6.tar.xz", - "shasum": "b6418c9d5036d2328271078687dbf2d772279e65787546974943855423fc6545", - "size": "53809524" + "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.732+2f3234c76.tar.xz", + "shasum": "48a9f23fe3b19fc0794ec18767a3500bd544c397913efd58fc8cd624601fccd5", + "size": "54555756" + } + }, + "0.15.2": { + "version": "0.15.2", + "date": "2025-10-11", + "docs": "https://ziglang.org/documentation/0.15.2/", + "stdDocs": "https://ziglang.org/documentation/0.15.2/std/", + "src": { + "tarball": "https://ziglang.org/download/0.15.2/zig-0.15.2.tar.xz", + "shasum": "d9b30c7aa983fcff5eed2084d54ae83eaafe7ff3a84d8fb754d854165a6e521c", + "size": "21366268" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.15.2/zig-bootstrap-0.15.2.tar.xz", + "shasum": "a6845459501df3c3264ebc587b02a7094ad14f4f3f7287c48f04457e784d0d85", + "size": "52718988" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-macos-0.15.2.tar.xz", + "shasum": "375b6909fc1495d16fc2c7db9538f707456bfc3373b14ee83fdd3e22b3d43f7f", + "size": "55800460" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-macos-0.15.2.tar.xz", + "shasum": "3cc2bab367e185cdfb27501c4b30b1b0653c28d9f73df8dc91488e66ece5fa6b", + "size": "50635984" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-linux-0.15.2.tar.xz", + "shasum": "02aa270f183da276e5b5920b1dac44a63f1a49e55050ebde3aecc9eb82f93239", + "size": "53733924" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-linux-0.15.2.tar.xz", + "shasum": "958ed7d1e00d0ea76590d27666efbf7a932281b3d7ba0c6b01b0ff26498f667f", + "size": "49471996" + }, + "arm-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-arm-linux-0.15.2.tar.xz", + "shasum": "7d8401495065dae45d6249c68d5faf10508f8203c86362ccb698aeaafc66b7cd", + "size": "50454216" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-riscv64-linux-0.15.2.tar.xz", + "shasum": "493512bdca485be3c6a9b0f69dcb4cbe4587f3af8e1be282fdd827108ba39930", + "size": "53613656" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-powerpc64le-linux-0.15.2.tar.xz", + "shasum": "e182c5f8d30fc7f97d17d2ffef1488826aa3afaa51e5f0dbe14c597a98b45778", + "size": "53570632" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86-linux-0.15.2.tar.xz", + "shasum": "4c6e23f39daa305e274197bfdff0d56ffd1750fc1de226ae10505c0eff52d7a5", + "size": "56334032" + }, + "loongarch64-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-loongarch64-linux-0.15.2.tar.xz", + "shasum": "95d6a03e0415dbe30ae9c00d58b5446fd681ebf3d4044a8b707e1784c8074a61", + "size": "50800332" + }, + "s390x-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-s390x-linux-0.15.2.tar.xz", + "shasum": "cd4b295f43b1b7ca3d603368caea82dbd00fce6691affb7527e607c0b6eafe8e", + "size": "53486436" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-windows-0.15.2.zip", + "shasum": "3a0ed1e8799a2f8ce2a6e6290a9ff22e6906f8227865911fb7ddedc3cc14cb0c", + "size": "92614574" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-windows-0.15.2.zip", + "shasum": "b926465f8872bf983422257cd9ec248bb2b270996fbe8d57872cca13b56fc370", + "size": "88464585" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86-windows-0.15.2.zip", + "shasum": "7a6dfc00f4cc09ec46d3e10eb06f42538e92b6285e34debea7462edaf371da98", + "size": "94516930" + }, + "aarch64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-freebsd-0.15.2.tar.xz", + "shasum": "c62efd319f86663eb7747709dfca259205edba8eaee98efc96a51ce40a9437de", + "size": "49363332" + }, + "arm-freebsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-arm-freebsd-0.15.2.tar.xz", + "shasum": "1d5d06428b2d4d31cf4e99008602f2096a9c652dd1025820a4aa66326fbf3dc4", + "size": "50900824" + }, + "powerpc64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-powerpc64-freebsd-0.15.2.tar.xz", + "shasum": "2bdb5a38069e667936d3378dfa332d67c21bb454cb17bea3c3b03faaa02a74af", + "size": "52099240" + }, + "powerpc64le-freebsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-powerpc64le-freebsd-0.15.2.tar.xz", + "shasum": "940894eb954c2077d0643f4deba8113239436fd0dcbc88160163b1b23fe3f035", + "size": "53459116" + }, + "riscv64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-riscv64-freebsd-0.15.2.tar.xz", + "shasum": "9aecdcf32fb63944c11389c10993aecd5e443a82d456693cc5ad230b5fe584ad", + "size": "53658752" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-freebsd-0.15.2.tar.xz", + "shasum": "5509ff57cd3f219165caed0da10221739af82742b9edfcda3f7bfaf4da7212dd", + "size": "53776208" + }, + "aarch64-netbsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-netbsd-0.15.2.tar.xz", + "shasum": "357c1e62df6581ec1726546f9a827973233cb31ac371d2b0e92456b306a0c6c0", + "size": "49365256" + }, + "arm-netbsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-arm-netbsd-0.15.2.tar.xz", + "shasum": "1c43871791d49dbabd224da307b995076e493e34c36734cb21c4f6cbc208f8c3", + "size": "52021696" + }, + "x86-netbsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86-netbsd-0.15.2.tar.xz", + "shasum": "7f18fc4720426a6ca77bbfb7b970c31a61c37ce4a867b53830762958e921cdde", + "size": "56893368" + }, + "x86_64-netbsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-netbsd-0.15.2.tar.xz", + "shasum": "11a3b9f4b66bece34a79c0e98e91136baf0ff9795ddf60e442ca6c8ad73b9204", + "size": "53778132" } }, "0.15.1": { "date": "2025-08-19", - "docs": "https://ziglang.org/documentation/master/", - "stdDocs": "https://ziglang.org/documentation/master/std/", + "docs": "https://ziglang.org/documentation/0.15.1/", + "stdDocs": "https://ziglang.org/documentation/0.15.1/std/", "notes": "https://ziglang.org/download/0.15.1/release-notes.html", "src": { "tarball": "https://ziglang.org/download/0.15.1/zig-0.15.1.tar.xz", From 8ceacce06779c31e993f804b10a422127577a485 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Fri, 17 Oct 2025 19:06:05 +0200 Subject: [PATCH 049/881] runtimes/neuron: fix $ORIGIN now that zigopts can use Make Variables (#350) zigopts is not subject to Make Variables expansion and so $ should be escaped. --- runtimes/neuron/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtimes/neuron/BUILD.bazel b/runtimes/neuron/BUILD.bazel index f1553fff..1d6c0280 100644 --- a/runtimes/neuron/BUILD.bazel +++ b/runtimes/neuron/BUILD.bazel @@ -25,7 +25,7 @@ zig_shared_library( zig_binary( name = "neuronx-cc", data = ["@neuron_py_deps//neuronx_cc"], - linkopts = ["-Wl,-rpath,$ORIGIN/../lib"], + linkopts = ["-Wl,-rpath,$$ORIGIN/../lib"], main = "neuronx-cc.zig", tags = ["manual"], deps = [":libpython"], From a5b925f46b7363dbdf37dc67a51000f225371aa4 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Fri, 17 Oct 2025 19:06:17 +0200 Subject: [PATCH 050/881] examples/callback: fix compilation (#351) Left over from when CI was not working most probably. --- examples/callback/main.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/callback/main.zig b/examples/callback/main.zig index 107c5f99..955ce9ac 100644 --- a/examples/callback/main.zig +++ b/examples/callback/main.zig @@ -83,8 +83,8 @@ pub const GrayScale = struct { pub fn grayScaleCuda(self: GrayScale, rgb_d: zml.Buffer, gray_d: zml.Buffer) !void { var args: [2][]u8 = .{ - rgb_d.opaqueDeviceMemoryDataPointer()[0..rgb_d.shape().byteSize()], - gray_d.opaqueDeviceMemoryDataPointer()[0..gray_d.shape().byteSize()], + @as([*]u8, @ptrFromInt(rgb_d.devicePtr()))[0..rgb_d.shape().byteSize()], + @as([*]u8, @ptrFromInt(gray_d.devicePtr()))[0..gray_d.shape().byteSize()], }; var args_ptr: [2:null]?*anyopaque = .{ @ptrCast(&args[0]), @ptrCast(&args[1]) }; // This is a naive kernel with one block per pixel. From 773253dfaf625176f1543a9a28b6caf87f6afb3e Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Fri, 17 Oct 2025 21:18:11 +0200 Subject: [PATCH 051/881] zml: use .host_pinned memory for ROCm (#353) It was failing with: ``` error(zml): Platform rocm doesn't have memory host_unpinned ``` --- zml/platform.zig | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zml/platform.zig b/zml/platform.zig index 410d60bc..c5872a57 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -89,7 +89,9 @@ pub const Platform = struct { const memory_target: pjrt.Memory.Kind = switch (memory) { .host_unpinned => switch (platform.target) { // Cuda doesn't have host_unpinned. - .cuda => .host_pinned, + // ROCm doesn't seem to have it either. + // TODO(gwenzek): investigate why it was not forced before. + .cuda, .rocm => .host_pinned, else => .host_unpinned, }, inline else => |t| t, From 7d825caf2d53ecd074ccf7036e395f6e128b7b8a Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Fri, 17 Oct 2025 21:19:08 +0200 Subject: [PATCH 052/881] zml/aio/torch/pickle: disable test until implement is fixed (#352) 0.15.2 made this test fail with: 112/117 aio.torch.pickle.test.parse protocol 0...error(zml/aio): Unknow pickle operator @enumFromInt(10), note we are only supporting pickle protocol up to version 5. cc @gwenzek --- zml/aio/torch/pickle.zig | 283 ++++++++++++++++++++------------------- 1 file changed, 142 insertions(+), 141 deletions(-) diff --git a/zml/aio/torch/pickle.zig b/zml/aio/torch/pickle.zig index 6ed3a1b2..1a04541a 100644 --- a/zml/aio/torch/pickle.zig +++ b/zml/aio/torch/pickle.zig @@ -891,153 +891,154 @@ pub fn parse(arena: std.mem.Allocator, reader: *std.Io.Reader) ![]const Op { return results.items; } -test "parse protocol 4" { - var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); - defer arena.deinit(); +//TODO(gwenzek): re-enable these tests when the bug has been fixed. +// test "parse protocol 4" { +// var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); +// defer arena.deinit(); - const file = try std.fs.cwd().openFile("zml/aio/torch/simple_test_4.pickle", .{ .mode = .read_only }); - var read_buffer: [1024]u8 = undefined; - var reader = file.reader(&read_buffer); - const ops = try parse(arena.allocator(), &reader.interface); +// const file = try std.fs.cwd().openFile("zml/aio/torch/simple_test_4.pickle", .{ .mode = .read_only }); +// var read_buffer: [1024]u8 = undefined; +// var reader = file.reader(&read_buffer); +// const ops = try parse(arena.allocator(), &reader.interface); - // this can be obtained by running: `python -m pickletools simple_test_4.pickle` - const expected: []const Op = &.{ - .{ .proto = 4 }, - .{ .frame = 119 }, - .empty_dict, - .memoize, - .mark, - .{ .unicode = "hello" }, - .memoize, - .{ .unicode = "world" }, - .memoize, - .{ .unicode = "int" }, - .memoize, - .{ .int = 1 }, - .{ .unicode = "float" }, - .memoize, - .{ .binfloat = 3.141592 }, - .{ .unicode = "list" }, - .memoize, - .empty_list, - .memoize, - .mark, - .{ .int = 255 }, - .{ .int = 1234 }, - .{ .int = -123 }, - .{ .int = 1_000_000_000 }, - .{ .binlong = &writeIntBuff(u48, 999_000_000_000) }, - .{ .binlong = &writeIntBuff(u104, 999_000_000_000_000_000_000_000_000_000) }, - .appends, - .{ .unicode = "bool" }, - .memoize, - .{ .bool = false }, - .{ .unicode = "tuple" }, - .memoize, - .{ .unicode = "a" }, - .memoize, - .{ .int = 10 }, - .tuple2, - .memoize, - .setitems, - .stop, - }; - try std.testing.expectEqualDeep(expected, ops); -} +// // this can be obtained by running: `python -m pickletools simple_test_4.pickle` +// const expected: []const Op = &.{ +// .{ .proto = 4 }, +// .{ .frame = 119 }, +// .empty_dict, +// .memoize, +// .mark, +// .{ .unicode = "hello" }, +// .memoize, +// .{ .unicode = "world" }, +// .memoize, +// .{ .unicode = "int" }, +// .memoize, +// .{ .int = 1 }, +// .{ .unicode = "float" }, +// .memoize, +// .{ .binfloat = 3.141592 }, +// .{ .unicode = "list" }, +// .memoize, +// .empty_list, +// .memoize, +// .mark, +// .{ .int = 255 }, +// .{ .int = 1234 }, +// .{ .int = -123 }, +// .{ .int = 1_000_000_000 }, +// .{ .binlong = &writeIntBuff(u48, 999_000_000_000) }, +// .{ .binlong = &writeIntBuff(u104, 999_000_000_000_000_000_000_000_000_000) }, +// .appends, +// .{ .unicode = "bool" }, +// .memoize, +// .{ .bool = false }, +// .{ .unicode = "tuple" }, +// .memoize, +// .{ .unicode = "a" }, +// .memoize, +// .{ .int = 10 }, +// .tuple2, +// .memoize, +// .setitems, +// .stop, +// }; +// try std.testing.expectEqualDeep(expected, ops); +// } -test "parse protocol 0" { - // We also test protocol 0, cause it's more text oriented. - var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); - defer arena.deinit(); +// test "parse protocol 0" { +// // We also test protocol 0, cause it's more text oriented. +// var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); +// defer arena.deinit(); - const pickle_0 = - \\(dp0 - \\Vhello - \\p1 - \\Vworld - \\p2 - \\sVint - \\p3 - \\I1 - \\sVfloat - \\p4 - \\F3.141592 - \\sVlist - \\p5 - \\(lp6 - \\I255 - \\aI1234 - \\aI-123 - \\aI1000000000 - \\aL999000000000L - \\aL999000000000000000000000000000L - \\asVbool - \\p7 - \\I00 - \\sVtuple - \\p8 - \\(Va - \\p9 - \\I10 - \\tp10 - \\s. - ; +// const pickle_0 = +// \\(dp0 +// \\Vhello +// \\p1 +// \\Vworld +// \\p2 +// \\sVint +// \\p3 +// \\I1 +// \\sVfloat +// \\p4 +// \\F3.141592 +// \\sVlist +// \\p5 +// \\(lp6 +// \\I255 +// \\aI1234 +// \\aI-123 +// \\aI1000000000 +// \\aL999000000000L +// \\aL999000000000000000000000000000L +// \\asVbool +// \\p7 +// \\I00 +// \\sVtuple +// \\p8 +// \\(Va +// \\p9 +// \\I10 +// \\tp10 +// \\s. +// ; - var reader: std.Io.Reader = .fixed(pickle_0); - const ops = try parse(arena.allocator(), &reader); +// var reader: std.Io.Reader = .fixed(pickle_0); +// const ops = try parse(arena.allocator(), &reader); - var expected = [_]Op{ - .mark, - .dict, - .{ .put = 0 }, - .{ .unicode = "hello" }, - .{ .put = 1 }, - .{ .unicode = "world" }, - .{ .put = 2 }, - .setitem, - .{ .unicode = "int" }, - .{ .put = 3 }, - .{ .int = 1 }, - .setitem, - .{ .unicode = "float" }, - .{ .put = 4 }, - .{ .float = "3.141592" }, - .setitem, - .{ .unicode = "list" }, - .{ .put = 5 }, - .mark, - .list, - .{ .put = 6 }, - .{ .int = 255 }, - .append, - .{ .int = 1234 }, - .append, - .{ .int = -123 }, - .append, - .{ .int = 1_000_000_000 }, - .append, - .{ .long = "999000000000L" }, - .append, - .{ .long = "999000000000000000000000000000L" }, - .append, - .setitem, - .{ .unicode = "bool" }, - .{ .put = 7 }, - .{ .bool = false }, - .setitem, - .{ .unicode = "tuple" }, - .{ .put = 8 }, - .mark, - .{ .unicode = "a" }, - .{ .put = 9 }, - .{ .int = 10 }, - .tuple, - .{ .put = 10 }, - .setitem, - .stop, - }; - try std.testing.expectEqualDeep(&expected, ops); -} +// var expected = [_]Op{ +// .mark, +// .dict, +// .{ .put = 0 }, +// .{ .unicode = "hello" }, +// .{ .put = 1 }, +// .{ .unicode = "world" }, +// .{ .put = 2 }, +// .setitem, +// .{ .unicode = "int" }, +// .{ .put = 3 }, +// .{ .int = 1 }, +// .setitem, +// .{ .unicode = "float" }, +// .{ .put = 4 }, +// .{ .float = "3.141592" }, +// .setitem, +// .{ .unicode = "list" }, +// .{ .put = 5 }, +// .mark, +// .list, +// .{ .put = 6 }, +// .{ .int = 255 }, +// .append, +// .{ .int = 1234 }, +// .append, +// .{ .int = -123 }, +// .append, +// .{ .int = 1_000_000_000 }, +// .append, +// .{ .long = "999000000000L" }, +// .append, +// .{ .long = "999000000000000000000000000000L" }, +// .append, +// .setitem, +// .{ .unicode = "bool" }, +// .{ .put = 7 }, +// .{ .bool = false }, +// .setitem, +// .{ .unicode = "tuple" }, +// .{ .put = 8 }, +// .mark, +// .{ .unicode = "a" }, +// .{ .put = 9 }, +// .{ .int = 10 }, +// .tuple, +// .{ .put = 10 }, +// .setitem, +// .stop, +// }; +// try std.testing.expectEqualDeep(&expected, ops); +// } fn _readSlice(reader: anytype, allocator: std.mem.Allocator, comptime len_bytes: u8) ![]u8 { const T = std.meta.Int(.unsigned, 8 * len_bytes); From 13aa52b1d7e5c78b82593779c0ab5ca4fc0db50b Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 20 Oct 2025 12:28:45 +0200 Subject: [PATCH 053/881] tools/hf: add hf_transfer implicit dependency (#344) Fix #339 --- MODULE.bazel | 6 +- MODULE.bazel.lock | 7 +- tools/hf/requirements.in | 1 + tools/hf/requirements.lock.txt | 405 ++++++++++++++++++++------------- 4 files changed, 248 insertions(+), 171 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index a8a83539..d8ad79f0 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -3,15 +3,15 @@ module( ) bazel_dep(name = "abseil-cpp", version = "20250814.0") -bazel_dep(name = "aspect_bazel_lib", version = "2.20.0") +bazel_dep(name = "aspect_bazel_lib", version = "2.21.2") bazel_dep(name = "aspect_rules_py", version = "1.6.3") -bazel_dep(name = "bazel_skylib", version = "1.8.1") +bazel_dep(name = "bazel_skylib", version = "1.8.2") bazel_dep(name = "libxev", version = "0.0.0-20251014.0-9f785d2") bazel_dep(name = "patchelf", version = "0.18.0") bazel_dep(name = "pcre2", version = "10.45") bazel_dep(name = "platforms", version = "1.0.0") bazel_dep(name = "protobuf", version = "32.0", repo_name = "com_google_protobuf") -bazel_dep(name = "rules_cc", version = "0.2.0") +bazel_dep(name = "rules_cc", version = "0.2.9") bazel_dep(name = "rules_distroless", version = "0.5.1") bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep(name = "rules_oci", version = "2.2.6") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index fe78fb19..0ecf36d6 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -19,7 +19,6 @@ "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.16.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.19.3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.20.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.21.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "not found", @@ -171,7 +170,6 @@ "https://bazel-registry.zml.ai/modules/rules_shell/0.4.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_zig/0.12.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.4/MODULE.bazel": "not found", @@ -212,7 +210,6 @@ "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.16.0/MODULE.bazel": "852f9ebbda017572a7c113a2434592dd3b2f55cd9a0faea3d4be5a09a59e4900", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.3/MODULE.bazel": "253d739ba126f62a5767d832765b12b59e9f8d2bc88cc1572f4a73e46eb298ca", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.20.0/MODULE.bazel": "c5565bac49e1973227225b441fad1c938d498d83df62dc5da95b2fab0f0626a2", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.21.2/MODULE.bazel": "276347663a25b0d5bd6cad869252bea3e160c4d980e764b15f3bae7f80b30624", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.21.2/source.json": "f42051fa42629f0e59b7ac2adf0a55749144b11f1efcd8c697f0ee247181e526", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "780d1a6522b28f5edb7ea09630748720721dfe27690d65a2d33aa7509de77e07", @@ -399,8 +396,6 @@ "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", "https://bcr.bazel.build/modules/rules_swift/2.1.1/source.json": "40fc69dfaac64deddbb75bd99cdac55f4427d9ca0afbe408576a65428427a186", - "https://bcr.bazel.build/modules/rules_zig/0.12.1/MODULE.bazel": "5953094b681c212eebe3dd4275809c52b9177aeaa058d8f1bff433822c0dcd01", - "https://bcr.bazel.build/modules/rules_zig/0.12.1/source.json": "d880cdc3473bfcf172c9aad9292e856430496e198d805539bb8be98e79f58a7c", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", "https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4", @@ -2814,7 +2809,7 @@ "@@rules_zig+//zig:extensions.bzl%zig": { "general": { "bzlTransitiveDigest": "wV3euhBZSI0D2ZlqbY1YW592mghFY88d9H6zBMQRaJY=", - "usagesDigest": "PFS+UEa30JWWU52AUTNHD1nDzqpwjSts6jIKzD7gVJU=", + "usagesDigest": "yjAIKp9S0o0qAI7IuTPORFscLuBNsX9/I86qX5/4j1Y=", "recordedFileInputs": { "@@//bazel/zig_index.json": "e6c6b29fa227fa9b1806dc03af2b6dab00f52f34d39878557074e1626027259f", "@@rules_zig+//zig/private/versions.json": "6eb85ebaee72c4e6fbc82d8ad73f69e0899d839b70a7e19caa792ab6a5fa0c36" diff --git a/tools/hf/requirements.in b/tools/hf/requirements.in index 6b964ccc..96187ec3 100644 --- a/tools/hf/requirements.in +++ b/tools/hf/requirements.in @@ -1 +1,2 @@ huggingface_hub +hf_transfer diff --git a/tools/hf/requirements.lock.txt b/tools/hf/requirements.lock.txt index 3657ce18..9d6c0dea 100644 --- a/tools/hf/requirements.lock.txt +++ b/tools/hf/requirements.lock.txt @@ -1,174 +1,255 @@ # This file was autogenerated by uv via the following command: # bazel run //tools/hf:requirements.update -certifi==2025.8.3 \ - --hash=sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407 \ - --hash=sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5 +certifi==2025.10.5 \ + --hash=sha256:0f212c2744a9bb6de0c56639a6f68afe01ecd92d91f14ae897c4fe7bbeeef0de \ + --hash=sha256:47c09d31ccf2acf0be3f701ea53595ee7e0b8fa08801c6624be771df09ae7b43 # via requests -charset-normalizer==3.4.3 \ - --hash=sha256:00237675befef519d9af72169d8604a067d92755e84fe76492fef5441db05b91 \ - --hash=sha256:02425242e96bcf29a49711b0ca9f37e451da7c70562bc10e8ed992a5a7a25cc0 \ - --hash=sha256:027b776c26d38b7f15b26a5da1044f376455fb3766df8fc38563b4efbc515154 \ - --hash=sha256:07a0eae9e2787b586e129fdcbe1af6997f8d0e5abaa0bc98c0e20e124d67e601 \ - --hash=sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884 \ - --hash=sha256:0e78314bdc32fa80696f72fa16dc61168fda4d6a0c014e0380f9d02f0e5d8a07 \ - --hash=sha256:0f2be7e0cf7754b9a30eb01f4295cc3d4358a479843b31f328afd210e2c7598c \ - --hash=sha256:13faeacfe61784e2559e690fc53fa4c5ae97c6fcedb8eb6fb8d0a15b475d2c64 \ - --hash=sha256:14c2a87c65b351109f6abfc424cab3927b3bdece6f706e4d12faaf3d52ee5efe \ - --hash=sha256:1606f4a55c0fd363d754049cdf400175ee96c992b1f8018b993941f221221c5f \ - --hash=sha256:16a8770207946ac75703458e2c743631c79c59c5890c80011d536248f8eaa432 \ - --hash=sha256:18343b2d246dc6761a249ba1fb13f9ee9a2bcd95decc767319506056ea4ad4dc \ - --hash=sha256:18b97b8404387b96cdbd30ad660f6407799126d26a39ca65729162fd810a99aa \ - --hash=sha256:1bb60174149316da1c35fa5233681f7c0f9f514509b8e399ab70fea5f17e45c9 \ - --hash=sha256:1e8ac75d72fa3775e0b7cb7e4629cec13b7514d928d15ef8ea06bca03ef01cae \ - --hash=sha256:1ef99f0456d3d46a50945c98de1774da86f8e992ab5c77865ea8b8195341fc19 \ - --hash=sha256:2001a39612b241dae17b4687898843f254f8748b796a2e16f1051a17078d991d \ - --hash=sha256:23b6b24d74478dc833444cbd927c338349d6ae852ba53a0d02a2de1fce45b96e \ - --hash=sha256:252098c8c7a873e17dd696ed98bbe91dbacd571da4b87df3736768efa7a792e4 \ - --hash=sha256:257f26fed7d7ff59921b78244f3cd93ed2af1800ff048c33f624c87475819dd7 \ - --hash=sha256:2c322db9c8c89009a990ef07c3bcc9f011a3269bc06782f916cd3d9eed7c9312 \ - --hash=sha256:30a96e1e1f865f78b030d65241c1ee850cdf422d869e9028e2fc1d5e4db73b92 \ - --hash=sha256:30d006f98569de3459c2fc1f2acde170b7b2bd265dc1943e87e1a4efe1b67c31 \ - --hash=sha256:31a9a6f775f9bcd865d88ee350f0ffb0e25936a7f930ca98995c05abf1faf21c \ - --hash=sha256:320e8e66157cc4e247d9ddca8e21f427efc7a04bbd0ac8a9faf56583fa543f9f \ - --hash=sha256:34a7f768e3f985abdb42841e20e17b330ad3aaf4bb7e7aeeb73db2e70f077b99 \ - --hash=sha256:3653fad4fe3ed447a596ae8638b437f827234f01a8cd801842e43f3d0a6b281b \ - --hash=sha256:3cd35b7e8aedeb9e34c41385fda4f73ba609e561faedfae0a9e75e44ac558a15 \ - --hash=sha256:3cfb2aad70f2c6debfbcb717f23b7eb55febc0bb23dcffc0f076009da10c6392 \ - --hash=sha256:416175faf02e4b0810f1f38bcb54682878a4af94059a1cd63b8747244420801f \ - --hash=sha256:41d1fc408ff5fdfb910200ec0e74abc40387bccb3252f3f27c0676731df2b2c8 \ - --hash=sha256:42e5088973e56e31e4fa58eb6bd709e42fc03799c11c42929592889a2e54c491 \ - --hash=sha256:4ca4c094de7771a98d7fbd67d9e5dbf1eb73efa4f744a730437d8a3a5cf994f0 \ - --hash=sha256:511729f456829ef86ac41ca78c63a5cb55240ed23b4b737faca0eb1abb1c41bc \ - --hash=sha256:53cd68b185d98dde4ad8990e56a58dea83a4162161b1ea9272e5c9182ce415e0 \ - --hash=sha256:585f3b2a80fbd26b048a0be90c5aae8f06605d3c92615911c3a2b03a8a3b796f \ - --hash=sha256:5b413b0b1bfd94dbf4023ad6945889f374cd24e3f62de58d6bb102c4d9ae534a \ - --hash=sha256:5d8d01eac18c423815ed4f4a2ec3b439d654e55ee4ad610e153cf02faf67ea40 \ - --hash=sha256:6aab0f181c486f973bc7262a97f5aca3ee7e1437011ef0c2ec04b5a11d16c927 \ - --hash=sha256:6cf8fd4c04756b6b60146d98cd8a77d0cdae0e1ca20329da2ac85eed779b6849 \ - --hash=sha256:6fb70de56f1859a3f71261cbe41005f56a7842cc348d3aeb26237560bfa5e0ce \ - --hash=sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14 \ - --hash=sha256:70bfc5f2c318afece2f5838ea5e4c3febada0be750fcf4775641052bbba14d05 \ - --hash=sha256:73dc19b562516fc9bcf6e5d6e596df0b4eb98d87e4f79f3ae71840e6ed21361c \ - --hash=sha256:74d77e25adda8581ffc1c720f1c81ca082921329452eba58b16233ab1842141c \ - --hash=sha256:78deba4d8f9590fe4dae384aeff04082510a709957e968753ff3c48399f6f92a \ - --hash=sha256:86df271bf921c2ee3818f0522e9a5b8092ca2ad8b065ece5d7d9d0e9f4849bcc \ - --hash=sha256:88ab34806dea0671532d3f82d82b85e8fc23d7b2dd12fa837978dad9bb392a34 \ - --hash=sha256:8999f965f922ae054125286faf9f11bc6932184b93011d138925a1773830bbe9 \ - --hash=sha256:8dcfc373f888e4fb39a7bc57e93e3b845e7f462dacc008d9749568b1c4ece096 \ - --hash=sha256:939578d9d8fd4299220161fdd76e86c6a251987476f5243e8864a7844476ba14 \ - --hash=sha256:96b2b3d1a83ad55310de8c7b4a2d04d9277d5591f40761274856635acc5fcb30 \ - --hash=sha256:a2d08ac246bb48479170408d6c19f6385fa743e7157d716e144cad849b2dd94b \ - --hash=sha256:b256ee2e749283ef3ddcff51a675ff43798d92d746d1a6e4631bf8c707d22d0b \ - --hash=sha256:b5e3b2d152e74e100a9e9573837aba24aab611d39428ded46f4e4022ea7d1942 \ - --hash=sha256:b89bc04de1d83006373429975f8ef9e7932534b8cc9ca582e4db7d20d91816db \ - --hash=sha256:bd28b817ea8c70215401f657edef3a8aa83c29d447fb0b622c35403780ba11d5 \ - --hash=sha256:c60e092517a73c632ec38e290eba714e9627abe9d301c8c8a12ec32c314a2a4b \ - --hash=sha256:c6dbd0ccdda3a2ba7c2ecd9d77b37f3b5831687d8dc1b6ca5f56a4880cc7b7ce \ - --hash=sha256:c6e490913a46fa054e03699c70019ab869e990270597018cef1d8562132c2669 \ - --hash=sha256:c6f162aabe9a91a309510d74eeb6507fab5fff92337a15acbe77753d88d9dcf0 \ - --hash=sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018 \ - --hash=sha256:cc34f233c9e71701040d772aa7490318673aa7164a0efe3172b2981218c26d93 \ - --hash=sha256:cc9370a2da1ac13f0153780040f465839e6cccb4a1e44810124b4e22483c93fe \ - --hash=sha256:ccf600859c183d70eb47e05a44cd80a4ce77394d1ac0f79dbd2dd90a69a3a049 \ - --hash=sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a \ - --hash=sha256:cf1ebb7d78e1ad8ec2a8c4732c7be2e736f6e5123a4146c5b89c9d1f585f8cef \ - --hash=sha256:d0e909868420b7049dafd3a31d45125b31143eec59235311fc4c57ea26a4acd2 \ - --hash=sha256:d22dbedd33326a4a5190dd4fe9e9e693ef12160c77382d9e87919bce54f3d4ca \ - --hash=sha256:d716a916938e03231e86e43782ca7878fb602a125a91e7acb8b5112e2e96ac16 \ - --hash=sha256:d79c198e27580c8e958906f803e63cddb77653731be08851c7df0b1a14a8fc0f \ - --hash=sha256:d95bfb53c211b57198bb91c46dd5a2d8018b3af446583aab40074bf7988401cb \ - --hash=sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1 \ - --hash=sha256:ec557499516fc90fd374bf2e32349a2887a876fbf162c160e3c01b6849eaf557 \ - --hash=sha256:fb6fecfd65564f208cbf0fba07f107fb661bcd1a7c389edbced3f7a493f70e37 \ - --hash=sha256:fb731e5deb0c7ef82d698b0f4c5bb724633ee2a489401594c5c88b02e6cb15f7 \ - --hash=sha256:fb7f67a1bfa6e40b438170ebdc8158b78dc465a5a67b6dde178a46987b244a72 \ - --hash=sha256:fd10de089bcdcd1be95a2f73dbe6254798ec1bda9f450d5828c96f93e2536b9c \ - --hash=sha256:fdabf8315679312cfa71302f9bd509ded4f2f263fb5b765cf1433b39106c3cc9 +charset-normalizer==3.4.4 \ + --hash=sha256:027f6de494925c0ab2a55eab46ae5129951638a49a34d87f4c3eda90f696b4ad \ + --hash=sha256:077fbb858e903c73f6c9db43374fd213b0b6a778106bc7032446a8e8b5b38b93 \ + --hash=sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394 \ + --hash=sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89 \ + --hash=sha256:0f04b14ffe5fdc8c4933862d8306109a2c51e0704acfa35d51598eb45a1e89fc \ + --hash=sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86 \ + --hash=sha256:194f08cbb32dc406d6e1aea671a68be0823673db2832b38405deba2fb0d88f63 \ + --hash=sha256:1bee1e43c28aa63cb16e5c14e582580546b08e535299b8b6158a7c9c768a1f3d \ + --hash=sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f \ + --hash=sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8 \ + --hash=sha256:244bfb999c71b35de57821b8ea746b24e863398194a4014e4c76adc2bbdfeff0 \ + --hash=sha256:2677acec1a2f8ef614c6888b5b4ae4060cc184174a938ed4e8ef690e15d3e505 \ + --hash=sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161 \ + --hash=sha256:2aaba3b0819274cc41757a1da876f810a3e4d7b6eb25699253a4effef9e8e4af \ + --hash=sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152 \ + --hash=sha256:2c9d3c380143a1fedbff95a312aa798578371eb29da42106a29019368a475318 \ + --hash=sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72 \ + --hash=sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4 \ + --hash=sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e \ + --hash=sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3 \ + --hash=sha256:44c2a8734b333e0578090c4cd6b16f275e07aa6614ca8715e6c038e865e70576 \ + --hash=sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c \ + --hash=sha256:4902828217069c3c5c71094537a8e623f5d097858ac6ca8252f7b4d10b7560f1 \ + --hash=sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8 \ + --hash=sha256:4fe7859a4e3e8457458e2ff592f15ccb02f3da787fcd31e0183879c3ad4692a1 \ + --hash=sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2 \ + --hash=sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44 \ + --hash=sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26 \ + --hash=sha256:5947809c8a2417be3267efc979c47d76a079758166f7d43ef5ae8e9f92751f88 \ + --hash=sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016 \ + --hash=sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede \ + --hash=sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf \ + --hash=sha256:5cb4d72eea50c8868f5288b7f7f33ed276118325c1dfd3957089f6b519e1382a \ + --hash=sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc \ + --hash=sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0 \ + --hash=sha256:64b55f9dce520635f018f907ff1b0df1fdc31f2795a922fb49dd14fbcdf48c84 \ + --hash=sha256:6515f3182dbe4ea06ced2d9e8666d97b46ef4c75e326b79bb624110f122551db \ + --hash=sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1 \ + --hash=sha256:6aee717dcfead04c6eb1ce3bd29ac1e22663cdea57f943c87d1eab9a025438d7 \ + --hash=sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed \ + --hash=sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8 \ + --hash=sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133 \ + --hash=sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e \ + --hash=sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef \ + --hash=sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14 \ + --hash=sha256:778d2e08eda00f4256d7f672ca9fef386071c9202f5e4607920b86d7803387f2 \ + --hash=sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0 \ + --hash=sha256:798d75d81754988d2565bff1b97ba5a44411867c0cf32b77a7e8f8d84796b10d \ + --hash=sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828 \ + --hash=sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f \ + --hash=sha256:7c308f7e26e4363d79df40ca5b2be1c6ba9f02bdbccfed5abddb7859a6ce72cf \ + --hash=sha256:7fa17817dc5625de8a027cb8b26d9fefa3ea28c8253929b8d6649e705d2835b6 \ + --hash=sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328 \ + --hash=sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090 \ + --hash=sha256:837c2ce8c5a65a2035be9b3569c684358dfbf109fd3b6969630a87535495ceaa \ + --hash=sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381 \ + --hash=sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c \ + --hash=sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb \ + --hash=sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc \ + --hash=sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a \ + --hash=sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec \ + --hash=sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc \ + --hash=sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac \ + --hash=sha256:9cd98cdc06614a2f768d2b7286d66805f94c48cde050acdbbb7db2600ab3197e \ + --hash=sha256:9d1bb833febdff5c8927f922386db610b49db6e0d4f4ee29601d71e7c2694313 \ + --hash=sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569 \ + --hash=sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3 \ + --hash=sha256:a61900df84c667873b292c3de315a786dd8dac506704dea57bc957bd31e22c7d \ + --hash=sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525 \ + --hash=sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894 \ + --hash=sha256:a8bf8d0f749c5757af2142fe7903a9df1d2e8aa3841559b2bad34b08d0e2bcf3 \ + --hash=sha256:a9768c477b9d7bd54bc0c86dbaebdec6f03306675526c9927c0e8a04e8f94af9 \ + --hash=sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a \ + --hash=sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9 \ + --hash=sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14 \ + --hash=sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25 \ + --hash=sha256:b5d84d37db046c5ca74ee7bb47dd6cbc13f80665fdde3e8040bdd3fb015ecb50 \ + --hash=sha256:b7cf1017d601aa35e6bb650b6ad28652c9cd78ee6caff19f3c28d03e1c80acbf \ + --hash=sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1 \ + --hash=sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3 \ + --hash=sha256:c4ef880e27901b6cc782f1b95f82da9313c0eb95c3af699103088fa0ac3ce9ac \ + --hash=sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e \ + --hash=sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815 \ + --hash=sha256:cb01158d8b88ee68f15949894ccc6712278243d95f344770fa7593fa2d94410c \ + --hash=sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6 \ + --hash=sha256:cc00f04ed596e9dc0da42ed17ac5e596c6ccba999ba6bd92b0e0aef2f170f2d6 \ + --hash=sha256:cd09d08005f958f370f539f186d10aec3377d55b9eeb0d796025d4886119d76e \ + --hash=sha256:cd4b7ca9984e5e7985c12bc60a6f173f3c958eae74f3ef6624bb6b26e2abbae4 \ + --hash=sha256:ce8a0633f41a967713a59c4139d29110c07e826d131a316b50ce11b1d79b4f84 \ + --hash=sha256:cead0978fc57397645f12578bfd2d5ea9138ea0fac82b2f63f7f7c6877986a69 \ + --hash=sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15 \ + --hash=sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191 \ + --hash=sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0 \ + --hash=sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897 \ + --hash=sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd \ + --hash=sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2 \ + --hash=sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794 \ + --hash=sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d \ + --hash=sha256:e912091979546adf63357d7e2ccff9b44f026c075aeaf25a52d0e95ad2281074 \ + --hash=sha256:eaabd426fe94daf8fd157c32e571c85cb12e66692f15516a83a03264b08d06c3 \ + --hash=sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224 \ + --hash=sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838 \ + --hash=sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a \ + --hash=sha256:f155a433c2ec037d4e8df17d18922c3a0d9b3232a396690f17175d2946f0218d \ + --hash=sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d \ + --hash=sha256:f34be2938726fc13801220747472850852fe6b1ea75869a048d6f896838c896f \ + --hash=sha256:f820802628d2694cb7e56db99213f930856014862f3fd943d290ea8438d07ca8 \ + --hash=sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490 \ + --hash=sha256:f8e160feb2aed042cd657a72acc0b481212ed28b1b9a95c0cee1621b524e1966 \ + --hash=sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9 \ + --hash=sha256:fa09f53c465e532f4d3db095e0c55b615f010ad81803d383195b6b5ca6cbf5f3 \ + --hash=sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e \ + --hash=sha256:fd44c878ea55ba351104cb93cc85e74916eb8fa440ca7903e57575e97394f608 # via requests -filelock==3.19.1 \ - --hash=sha256:66eda1888b0171c998b35be2bcc0f6d75c388a7ce20c3f3f37aa8e96c2dddf58 \ - --hash=sha256:d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d +filelock==3.20.0 \ + --hash=sha256:339b4732ffda5cd79b13f4e2711a31b0365ce445d95d243bb996273d072546a2 \ + --hash=sha256:711e943b4ec6be42e1d4e6690b48dc175c822967466bb31c0c293f34334c13f4 # via huggingface-hub -fsspec==2025.7.0 \ - --hash=sha256:786120687ffa54b8283d942929540d8bc5ccfa820deb555a2b5d0ed2b737bf58 \ - --hash=sha256:8b012e39f63c7d5f10474de957f3ab793b47b45ae7d39f2fb735f8bbe25c0e21 +fsspec==2025.9.0 \ + --hash=sha256:19fd429483d25d28b65ec68f9f4adc16c17ea2c7c7bf54ec61360d478fb19c19 \ + --hash=sha256:530dc2a2af60a414a832059574df4a6e10cce927f6f4a78209390fe38955cfb7 # via huggingface-hub -hf-xet==1.1.8 \ - --hash=sha256:09e86514c3c4284ed8a57d6b0f3d089f9836a0af0a1ceb3c9dd664f1f3eaefef \ - --hash=sha256:25b9d43333bbef39aeae1616789ec329c21401a7fe30969d538791076227b591 \ - --hash=sha256:3d5f82e533fc51c7daad0f9b655d9c7811b5308e5890236828bd1dd3ed8fea74 \ - --hash=sha256:4171f31d87b13da4af1ed86c98cf763292e4720c088b4957cf9d564f92904ca9 \ - --hash=sha256:4a9b99ab721d385b83f4fc8ee4e0366b0b59dce03b5888a86029cc0ca634efbf \ - --hash=sha256:62a0043e441753bbc446dcb5a3fe40a4d03f5fb9f13589ef1df9ab19252beb53 \ - --hash=sha256:8e2dba5896bca3ab61d0bef4f01a1647004de59640701b37e37eaa57087bbd9d \ - --hash=sha256:bfe5700bc729be3d33d4e9a9b5cc17a951bf8c7ada7ba0c9198a6ab2053b7453 - # via huggingface-hub -huggingface-hub==0.34.4 \ - --hash=sha256:9b365d781739c93ff90c359844221beef048403f1bc1f1c123c191257c3c890a \ - --hash=sha256:a4228daa6fb001be3f4f4bdaf9a0db00e1739235702848df00885c9b5742c85c +hf-transfer==0.1.9 \ + --hash=sha256:035572865dab29d17e783fbf1e84cf1cb24f3fcf8f1b17db1cfc7fdf139f02bf \ + --hash=sha256:0d991376f0eac70a60f0cbc95602aa708a6f7c8617f28b4945c1431d67b8e3c8 \ + --hash=sha256:16f208fc678911c37e11aa7b586bc66a37d02e636208f18b6bc53d29b5df40ad \ + --hash=sha256:1a6bd16c667ebe89a069ca163060127a794fa3a3525292c900b8c8cc47985b0d \ + --hash=sha256:2c7fc1b85f4d0f76e452765d7648c9f4bfd0aedb9ced2ae1ebfece2d8cfaf8e2 \ + --hash=sha256:3a736dfbb2c84f5a2c975478ad200c0c8bfcb58a25a35db402678fb87ce17fa4 \ + --hash=sha256:3ebc4ab9023414880c8b1d3c38174d1c9989eb5022d37e814fa91a3060123eb0 \ + --hash=sha256:435cc3cdc8524ce57b074032b8fd76eed70a4224d2091232fa6a8cef8fd6803e \ + --hash=sha256:504b8427fd785dd8546d53b9fafe6e436bd7a3adf76b9dce556507650a7b4567 \ + --hash=sha256:57fd9880da1ee0f47250f735f791fab788f0aa1ee36afc49f761349869c8b4d9 \ + --hash=sha256:5828057e313de59300dd1abb489444bc452efe3f479d3c55b31a8f680936ba42 \ + --hash=sha256:5d561f0520f493c66b016d99ceabe69c23289aa90be38dd802d2aef279f15751 \ + --hash=sha256:6e94e8822da79573c9b6ae4d6b2f847c59a7a06c5327d7db20751b68538dc4f6 \ + --hash=sha256:8669dbcc7a3e2e8d61d42cd24da9c50d57770bd74b445c65123291ca842a7e7a \ + --hash=sha256:8674026f21ed369aa2a0a4b46000aca850fc44cd2b54af33a172ce5325b4fc82 \ + --hash=sha256:89a23f58b7b7effbc047b8ca286f131b17728c99a9f972723323003ffd1bb916 \ + --hash=sha256:8fd0167c4407a3bc4cdd0307e65ada2294ec04f1813d8a69a5243e379b22e9d8 \ + --hash=sha256:a5b366d34cd449fe9b20ef25941e6eef0460a2f74e7389f02e673e1f88ebd538 \ + --hash=sha256:cdca9bfb89e6f8f281890cc61a8aff2d3cecaff7e1a4d275574d96ca70098557 \ + --hash=sha256:d2fde99d502093ade3ab1b53f80da18480e9902aa960dab7f74fb1b9e5bc5746 \ + --hash=sha256:dc7fff1345980d6c0ebb92c811d24afa4b98b3e07ed070c8e38cc91fd80478c5 \ + --hash=sha256:e66acf91df4a8b72f60223059df3003062a5ae111757187ed1a06750a30e911b \ + --hash=sha256:e6ac4eddcd99575ed3735ed911ddf9d1697e2bd13aa3f0ad7e3904dd4863842e \ + --hash=sha256:ee8b10afedcb75f71091bcc197c526a6ebf5c58bbbadb34fdeee6160f55f619f \ + --hash=sha256:fc6bd19e1cc177c66bdef15ef8636ad3bde79d5a4f608c158021153b4573509d # via -r tools/hf/requirements.in -idna==3.10 \ - --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \ - --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 +hf-xet==1.1.10 \ + --hash=sha256:0a0005fd08f002180f7a12d4e13b22be277725bc23ed0529f8add5c7a6309c06 \ + --hash=sha256:408aef343800a2102374a883f283ff29068055c111f003ff840733d3b715bb97 \ + --hash=sha256:5f54b19cc347c13235ae7ee98b330c26dd65ef1df47e5316ffb1e87713ca7045 \ + --hash=sha256:686083aca1a6669bc85c21c0563551cbcdaa5cf7876a91f3d074a030b577231d \ + --hash=sha256:6b6bceb6361c80c1cc42b5a7b4e3efd90e64630bcf11224dcac50ef30a47e435 \ + --hash=sha256:71081925383b66b24eedff3013f8e6bbd41215c3338be4b94ba75fd75b21513b \ + --hash=sha256:eae7c1fc8a664e54753ffc235e11427ca61f4b0477d757cc4eb9ae374b69f09c \ + --hash=sha256:f900481cf6e362a6c549c61ff77468bd59d6dd082f3170a36acfef2eb6a6793f + # via huggingface-hub +huggingface-hub==0.35.3 \ + --hash=sha256:0e3a01829c19d86d03793e4577816fe3bdfc1602ac62c7fb220d593d351224ba \ + --hash=sha256:350932eaa5cc6a4747efae85126ee220e4ef1b54e29d31c3b45c5612ddf0b32a + # via -r tools/hf/requirements.in +idna==3.11 \ + --hash=sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea \ + --hash=sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902 # via requests packaging==25.0 \ --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 \ --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f # via huggingface-hub -pyyaml==6.0.2 \ - --hash=sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff \ - --hash=sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48 \ - --hash=sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086 \ - --hash=sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e \ - --hash=sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133 \ - --hash=sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5 \ - --hash=sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484 \ - --hash=sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee \ - --hash=sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5 \ - --hash=sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68 \ - --hash=sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a \ - --hash=sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf \ - --hash=sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99 \ - --hash=sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8 \ - --hash=sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85 \ - --hash=sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19 \ - --hash=sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc \ - --hash=sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a \ - --hash=sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1 \ - --hash=sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317 \ - --hash=sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c \ - --hash=sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631 \ - --hash=sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d \ - --hash=sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652 \ - --hash=sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5 \ - --hash=sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e \ - --hash=sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b \ - --hash=sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8 \ - --hash=sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476 \ - --hash=sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706 \ - --hash=sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563 \ - --hash=sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237 \ - --hash=sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b \ - --hash=sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083 \ - --hash=sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180 \ - --hash=sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425 \ - --hash=sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e \ - --hash=sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f \ - --hash=sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725 \ - --hash=sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183 \ - --hash=sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab \ - --hash=sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774 \ - --hash=sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725 \ - --hash=sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e \ - --hash=sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5 \ - --hash=sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d \ - --hash=sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290 \ - --hash=sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44 \ - --hash=sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed \ - --hash=sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4 \ - --hash=sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba \ - --hash=sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12 \ - --hash=sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4 +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 # via huggingface-hub requests==2.32.5 \ --hash=sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6 \ @@ -178,9 +259,9 @@ tqdm==4.67.1 \ --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 \ --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2 # via huggingface-hub -typing-extensions==4.14.1 \ - --hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 \ - --hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76 +typing-extensions==4.15.0 \ + --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ + --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 # via huggingface-hub urllib3==2.5.0 \ --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 \ From e58a5359f9c0fdcbfe7b4f8e64c9a23bf309d73d Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 22 Oct 2025 17:43:44 +0200 Subject: [PATCH 054/881] zml/aio: fix pickle loader for Zig 0.15.2 (#354) zig 0.15.2 changed the meaning of `reader.takeDelimiterExclusive` to not consume the separator. I rewrote the code to use `takeDelimiterInclusive` since we want to consume the delimiter. --- examples/callback/main.zig | 8 +- third_party/zls/cat.zig | 2 +- third_party/zls/zls_runner.zig | 2 +- zml/aio/torch/pickle.zig | 318 +++++++++++++++++---------------- zml/buffer.zig | 9 +- zml/callback.zig | 6 +- zml/tools/BUILD.bazel | 5 +- zml/tools/tracer.zig | 3 +- 8 files changed, 177 insertions(+), 176 deletions(-) diff --git a/examples/callback/main.zig b/examples/callback/main.zig index 955ce9ac..5f207001 100644 --- a/examples/callback/main.zig +++ b/examples/callback/main.zig @@ -82,11 +82,7 @@ pub const GrayScale = struct { } pub fn grayScaleCuda(self: GrayScale, rgb_d: zml.Buffer, gray_d: zml.Buffer) !void { - var args: [2][]u8 = .{ - @as([*]u8, @ptrFromInt(rgb_d.devicePtr()))[0..rgb_d.shape().byteSize()], - @as([*]u8, @ptrFromInt(gray_d.devicePtr()))[0..gray_d.shape().byteSize()], - }; - var args_ptr: [2:null]?*anyopaque = .{ @ptrCast(&args[0]), @ptrCast(&args[1]) }; + var args: [2:null]?*anyopaque = .{ rgb_d.devicePtr(), gray_d.devicePtr() }; // This is a naive kernel with one block per pixel. try cuda.check(cuda.launchKernel.?( @ptrCast(self.cu_data[1]), // function @@ -98,7 +94,7 @@ pub const GrayScale = struct { 1, // num grids z 0, // shared mem @ptrCast(self.stream), - &args_ptr, + &args, null, )); // Note: no explicit synchronization, we just enqueue work in the stream. diff --git a/third_party/zls/cat.zig b/third_party/zls/cat.zig index ebca6f5a..acfa8ea0 100644 --- a/third_party/zls/cat.zig +++ b/third_party/zls/cat.zig @@ -8,7 +8,7 @@ pub fn main() !void { defer std.process.argsFree(gpa, args); const file_path = args[1]; - var file = try std.fs.cwd().openFile(file_path, .{ .mode = .read_only}); + var file = try std.fs.cwd().openFile(file_path, .{ .mode = .read_only }); defer file.close(); if (builtin.zig_version.major == 0 and builtin.zig_version.minor >= 15) { diff --git a/third_party/zls/zls_runner.zig b/third_party/zls/zls_runner.zig index 81555e05..0f011ec4 100644 --- a/third_party/zls/zls_runner.zig +++ b/third_party/zls/zls_runner.zig @@ -1,5 +1,5 @@ /// Custom ZLS launcher. -/// +/// /// Sets up paths to ZLS dependencies with Bazel runfiles. /// /// This file is used as a template by `zls_write_runner_zig_src.bzl`. diff --git a/zml/aio/torch/pickle.zig b/zml/aio/torch/pickle.zig index 1a04541a..8172ddc7 100644 --- a/zml/aio/torch/pickle.zig +++ b/zml/aio/torch/pickle.zig @@ -762,9 +762,13 @@ pub const Op = union(enum) { } }; +pub const ParseError = error{ + UnknownPickleOp, +} || std.fmt.ParseIntError || std.Io.Reader.DelimiterError || std.mem.Allocator.Error; + /// Read a stream of bytes, and interpret it as a stream of Pickle operators. /// The given allocator needs to be an arena cause we are not aligning allocations to avoid copies. -pub fn parse(arena: std.mem.Allocator, reader: *std.Io.Reader) ![]const Op { +pub fn parse(arena: std.mem.Allocator, reader: *std.Io.Reader) ParseError![]const Op { // It's not very efficient to interleave the results with the data copied from the stream, // because growth event in the results ArrayList will lead to fragmentation. // Trying to mitigate that by using a generous default size. @@ -776,7 +780,8 @@ pub fn parse(arena: std.mem.Allocator, reader: *std.Io.Reader) ![]const Op { const code: OpCode = @enumFromInt(try reader.takeByte()); const op: Op = switch (code) { .int => int: { - const bytes = try reader.takeDelimiterExclusive('\n'); + const bytes_with_ln = try reader.takeDelimiterInclusive('\n'); + const bytes = bytes_with_ln[0 .. bytes_with_ln.len - 1]; // Legacy hack, see OpCode.int documentation // We do this parsing right away to simplify downstream code. break :int if (bytes.len == 2 and bytes[0] == '0' and bytes[1] == '0') @@ -830,17 +835,10 @@ pub fn parse(arena: std.mem.Allocator, reader: *std.Io.Reader) ![]const Op { .dup => .dup, .mark => .mark, .pop_mark => .pop_mark, - // If we fail to parse delay the error to the evaluation. - .get => get: { - const digits = try reader.takeDelimiterExclusive('\n'); - break :get .{ .get = std.fmt.parseInt(u32, digits, 10) catch std.math.maxInt(u32) }; - }, + .get => .{ .get = try readTextU32(reader) }, .binget => .{ .get = try reader.takeByte() }, .long_binget => .{ .get = try reader.takeInt(u32, .little) }, - .put => put: { - const digits = try reader.takeDelimiterExclusive('\n'); - break :put .{ .put = std.fmt.parseInt(u32, digits, 10) catch std.math.maxInt(u32) }; - }, + .put => .{ .put = try readTextU32(reader) }, .binput => .{ .put = try reader.takeByte() }, .long_binput => .{ .put = try reader.takeInt(u32, .little) }, .memoize => .memoize, @@ -882,7 +880,7 @@ pub fn parse(arena: std.mem.Allocator, reader: *std.Io.Reader) ![]const Op { .binpersid => .binpersid, _ => |unk_tag| { log.err("Unknow pickle operator {}, note we are only supporting pickle protocol up to version 5.", .{unk_tag}); - return error.NotSupported; + return error.UnknownPickleOp; }, }; try results.append(arena, op); @@ -891,154 +889,153 @@ pub fn parse(arena: std.mem.Allocator, reader: *std.Io.Reader) ![]const Op { return results.items; } -//TODO(gwenzek): re-enable these tests when the bug has been fixed. -// test "parse protocol 4" { -// var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); -// defer arena.deinit(); +test "parse protocol 4" { + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); -// const file = try std.fs.cwd().openFile("zml/aio/torch/simple_test_4.pickle", .{ .mode = .read_only }); -// var read_buffer: [1024]u8 = undefined; -// var reader = file.reader(&read_buffer); -// const ops = try parse(arena.allocator(), &reader.interface); + const file = try std.fs.cwd().openFile("zml/aio/torch/simple_test_4.pickle", .{ .mode = .read_only }); + var read_buffer: [1024]u8 = undefined; + var reader = file.reader(&read_buffer); + const ops = try parse(arena.allocator(), &reader.interface); -// // this can be obtained by running: `python -m pickletools simple_test_4.pickle` -// const expected: []const Op = &.{ -// .{ .proto = 4 }, -// .{ .frame = 119 }, -// .empty_dict, -// .memoize, -// .mark, -// .{ .unicode = "hello" }, -// .memoize, -// .{ .unicode = "world" }, -// .memoize, -// .{ .unicode = "int" }, -// .memoize, -// .{ .int = 1 }, -// .{ .unicode = "float" }, -// .memoize, -// .{ .binfloat = 3.141592 }, -// .{ .unicode = "list" }, -// .memoize, -// .empty_list, -// .memoize, -// .mark, -// .{ .int = 255 }, -// .{ .int = 1234 }, -// .{ .int = -123 }, -// .{ .int = 1_000_000_000 }, -// .{ .binlong = &writeIntBuff(u48, 999_000_000_000) }, -// .{ .binlong = &writeIntBuff(u104, 999_000_000_000_000_000_000_000_000_000) }, -// .appends, -// .{ .unicode = "bool" }, -// .memoize, -// .{ .bool = false }, -// .{ .unicode = "tuple" }, -// .memoize, -// .{ .unicode = "a" }, -// .memoize, -// .{ .int = 10 }, -// .tuple2, -// .memoize, -// .setitems, -// .stop, -// }; -// try std.testing.expectEqualDeep(expected, ops); -// } + // this can be obtained by running: `python -m pickletools simple_test_4.pickle` + const expected: []const Op = &.{ + .{ .proto = 4 }, + .{ .frame = 119 }, + .empty_dict, + .memoize, + .mark, + .{ .unicode = "hello" }, + .memoize, + .{ .unicode = "world" }, + .memoize, + .{ .unicode = "int" }, + .memoize, + .{ .int = 1 }, + .{ .unicode = "float" }, + .memoize, + .{ .binfloat = 3.141592 }, + .{ .unicode = "list" }, + .memoize, + .empty_list, + .memoize, + .mark, + .{ .int = 255 }, + .{ .int = 1234 }, + .{ .int = -123 }, + .{ .int = 1_000_000_000 }, + .{ .binlong = &writeIntBuff(u48, 999_000_000_000) }, + .{ .binlong = &writeIntBuff(u104, 999_000_000_000_000_000_000_000_000_000) }, + .appends, + .{ .unicode = "bool" }, + .memoize, + .{ .bool = false }, + .{ .unicode = "tuple" }, + .memoize, + .{ .unicode = "a" }, + .memoize, + .{ .int = 10 }, + .tuple2, + .memoize, + .setitems, + .stop, + }; + try std.testing.expectEqualDeep(expected, ops); +} -// test "parse protocol 0" { -// // We also test protocol 0, cause it's more text oriented. -// var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); -// defer arena.deinit(); +test "parse protocol 0" { + // We also test protocol 0, cause it's more text oriented. + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); -// const pickle_0 = -// \\(dp0 -// \\Vhello -// \\p1 -// \\Vworld -// \\p2 -// \\sVint -// \\p3 -// \\I1 -// \\sVfloat -// \\p4 -// \\F3.141592 -// \\sVlist -// \\p5 -// \\(lp6 -// \\I255 -// \\aI1234 -// \\aI-123 -// \\aI1000000000 -// \\aL999000000000L -// \\aL999000000000000000000000000000L -// \\asVbool -// \\p7 -// \\I00 -// \\sVtuple -// \\p8 -// \\(Va -// \\p9 -// \\I10 -// \\tp10 -// \\s. -// ; + const pickle_0 = + \\(dp0 + \\Vhello + \\p1 + \\Vworld + \\p2 + \\sVint + \\p3 + \\I1 + \\sVfloat + \\p4 + \\F3.141592 + \\sVlist + \\p5 + \\(lp6 + \\I255 + \\aI1234 + \\aI-123 + \\aI1000000000 + \\aL999000000000L + \\aL999000000000000000000000000000L + \\asVbool + \\p7 + \\I00 + \\sVtuple + \\p8 + \\(Va + \\p9 + \\I10 + \\tp10 + \\s. + ; -// var reader: std.Io.Reader = .fixed(pickle_0); -// const ops = try parse(arena.allocator(), &reader); + var reader: std.Io.Reader = .fixed(pickle_0); + const ops = try parse(arena.allocator(), &reader); -// var expected = [_]Op{ -// .mark, -// .dict, -// .{ .put = 0 }, -// .{ .unicode = "hello" }, -// .{ .put = 1 }, -// .{ .unicode = "world" }, -// .{ .put = 2 }, -// .setitem, -// .{ .unicode = "int" }, -// .{ .put = 3 }, -// .{ .int = 1 }, -// .setitem, -// .{ .unicode = "float" }, -// .{ .put = 4 }, -// .{ .float = "3.141592" }, -// .setitem, -// .{ .unicode = "list" }, -// .{ .put = 5 }, -// .mark, -// .list, -// .{ .put = 6 }, -// .{ .int = 255 }, -// .append, -// .{ .int = 1234 }, -// .append, -// .{ .int = -123 }, -// .append, -// .{ .int = 1_000_000_000 }, -// .append, -// .{ .long = "999000000000L" }, -// .append, -// .{ .long = "999000000000000000000000000000L" }, -// .append, -// .setitem, -// .{ .unicode = "bool" }, -// .{ .put = 7 }, -// .{ .bool = false }, -// .setitem, -// .{ .unicode = "tuple" }, -// .{ .put = 8 }, -// .mark, -// .{ .unicode = "a" }, -// .{ .put = 9 }, -// .{ .int = 10 }, -// .tuple, -// .{ .put = 10 }, -// .setitem, -// .stop, -// }; -// try std.testing.expectEqualDeep(&expected, ops); -// } + var expected = [_]Op{ + .mark, + .dict, + .{ .put = 0 }, + .{ .unicode = "hello" }, + .{ .put = 1 }, + .{ .unicode = "world" }, + .{ .put = 2 }, + .setitem, + .{ .unicode = "int" }, + .{ .put = 3 }, + .{ .int = 1 }, + .setitem, + .{ .unicode = "float" }, + .{ .put = 4 }, + .{ .float = "3.141592" }, + .setitem, + .{ .unicode = "list" }, + .{ .put = 5 }, + .mark, + .list, + .{ .put = 6 }, + .{ .int = 255 }, + .append, + .{ .int = 1234 }, + .append, + .{ .int = -123 }, + .append, + .{ .int = 1_000_000_000 }, + .append, + .{ .long = "999000000000L" }, + .append, + .{ .long = "999000000000000000000000000000L" }, + .append, + .setitem, + .{ .unicode = "bool" }, + .{ .put = 7 }, + .{ .bool = false }, + .setitem, + .{ .unicode = "tuple" }, + .{ .put = 8 }, + .mark, + .{ .unicode = "a" }, + .{ .put = 9 }, + .{ .int = 10 }, + .tuple, + .{ .put = 10 }, + .setitem, + .stop, + }; + try std.testing.expectEqualDeep(&expected, ops); +} fn _readSlice(reader: anytype, allocator: std.mem.Allocator, comptime len_bytes: u8) ![]u8 { const T = std.meta.Int(.unsigned, 8 * len_bytes); @@ -1054,8 +1051,11 @@ fn writeIntBuff(comptime T: type, value: T) [@divExact(@typeInfo(T).int.bits, 8) return res; } -fn readLine(reader: *std.Io.Reader, alloc_writer: *std.Io.Writer.Allocating) ![]const u8 { - const n = try reader.streamDelimiter(&alloc_writer.writer, '\n'); +fn readLine(reader: *std.Io.Reader, alloc_writer: *std.Io.Writer.Allocating) ParseError![]const u8 { + const n = reader.streamDelimiter(&alloc_writer.writer, '\n') catch |err| switch (err) { + error.WriteFailed => return error.OutOfMemory, + else => |e| return e, + }; std.debug.assert(try reader.takeByte() == '\n'); const w = &alloc_writer.writer; std.debug.assert(w.end == n); @@ -1064,3 +1064,9 @@ fn readLine(reader: *std.Io.Reader, alloc_writer: *std.Io.Writer.Allocating) ![] w.end = 0; return items; } + +fn readTextU32(reader: *std.Io.Reader) ParseError!u32 { + // Note we use takeDelimiterInclusive because the newline must always be there. + const digits = try reader.takeDelimiterInclusive('\n'); + return try std.fmt.parseInt(u32, digits[0 .. digits.len - 1], 10); +} diff --git a/zml/buffer.zig b/zml/buffer.zig index a4a9b194..e76039d2 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -280,10 +280,9 @@ pub const Buffer = struct { }; } - pub fn devicePtr(self: Buffer) u64 { + pub fn devicePtr(self: Buffer) *anyopaque { stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer", .{}); - const opaque_ptr: *anyopaque = self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable; - return @intFromPtr(opaque_ptr); + return self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable; } /// Fetches the content of the given buffer into a stack variable of the given type. @@ -362,7 +361,7 @@ pub const Buffer = struct { } pub fn format(self: Buffer, writer: *std.Io.Writer) !void { - try writer.print("Buffer({f})@{x}", .{ self._shape, self.devicePtr() }); + try writer.print("Buffer({f})@{x}", .{ self._shape, @intFromPtr(self.devicePtr()) }); } pub fn getMemory(self: Buffer) *const pjrt.Memory { @@ -470,7 +469,7 @@ pub const Buffer = struct { const host_visible_memories: []const Memory = &.{ .host_pinned, .host_unpinned }; for (host_visible_memories) |memory| { const x = try uninitialized(platform, .init(.{6}, .u8), .{ .memory = memory }); - const x_ptr: [*]u8 = @ptrFromInt(x.devicePtr()); + const x_ptr: [*]u8 = @ptrCast(x.devicePtr()); @memcpy(x_ptr, &[_]u8{ 104, 101, 108, 108, 111, 33 }); const y = try x.getValue([6]u8); diff --git a/zml/callback.zig b/zml/callback.zig index ea7a4b23..709e6c87 100644 --- a/zml/callback.zig +++ b/zml/callback.zig @@ -197,12 +197,12 @@ fn CallbackImpl(comptime Callback: type, call_frame: *pjrt.ffi.CallFrame) ?*pjrt else .asViewOfDeviceBuffer(platform, shape, null, ffi_buffer.data); if (opts.copy_inputs_to_host_pinned and platform.target != .cpu) { - log.debug("Copying argument {d} {f} {x} to host_pinned memory !", .{ i, zml_buffer, zml_buffer.devicePtr() }); + // log.debug("Copying argument {d} {f} {x} to host_pinned memory !", .{ i, zml_buffer, @intFromPtr(zml_buffer.devicePtr()) }); zml_buffer = zml_buffer.copyToMemory(platform, .host_pinned, .{ .wait = true }) catch |err| { - log.err("Failed to copy input buffer {d} {f} {x} to host_pinned: {}", .{ i, zml_buffer, zml_buffer.devicePtr(), err }); + log.err("Failed to copy input buffer {d} {f} {x} to host_pinned: {}", .{ i, zml_buffer, @intFromPtr(zml_buffer.devicePtr()), err }); return .create(call_frame.api, .resource_exhausted, "host pinned OOM"); }; - log.debug("--> {f} {x}", .{ zml_buffer, zml_buffer.devicePtr() }); + // log.debug("--> {f} {x}", .{ zml_buffer, @intFromPtr(zml_buffer.devicePtr()) }); } callback_args[i] = zml_buffer; } diff --git a/zml/tools/BUILD.bazel b/zml/tools/BUILD.bazel index ff8c1b04..21ad5654 100644 --- a/zml/tools/BUILD.bazel +++ b/zml/tools/BUILD.bazel @@ -17,9 +17,8 @@ zig_library( main = "tools.zig", visibility = ["//visibility:public"], deps = select({ - "@platforms//os:macos": [ - ":macos_c", - ], + # TODO(cerisier): fix MacOsTracer + # "@platforms//os:macos": [ ":macos_c" ], "//conditions:default": [], }), ) diff --git a/zml/tools/tracer.zig b/zml/tools/tracer.zig index d369b1e2..3946cb6c 100644 --- a/zml/tools/tracer.zig +++ b/zml/tools/tracer.zig @@ -3,7 +3,8 @@ const builtin = @import("builtin"); const c = @import("c"); pub const Tracer = switch (builtin.os.tag) { - .macos => MacOsTracer, + // TODO(cerisier): fix MacOsTracer + // .macos => MacOsTracer, .linux => if (@hasDecl(c, "ZML_RUNTIME_CUDA")) CudaTracer else FakeTracer, else => FakeTracer, }; From e2c27b790b0ddfd3da37030f3f8f3e3d0df476fb Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Tue, 28 Oct 2025 17:06:48 +0100 Subject: [PATCH 055/881] workspace: default to LLVM backend for all zig compile actions (#356) Use the new zigopt global setting from rules_zig to set -fllvm to all zig copmile actions. The Zig self-hosted backend is more fragile that we thought before. It fails in numerous cases, in debug, but also in release sometimes. USe LLVM backend until it is more stable. --- .bazelrc | 2 ++ MODULE.bazel | 10 +--------- MODULE.bazel.lock | 11 +++++++++-- upb/BUILD.bazel | 4 ---- zml/BUILD.bazel | 2 -- zml/tokenizer/BUILD.bazel | 4 ---- 6 files changed, 12 insertions(+), 21 deletions(-) diff --git a/.bazelrc b/.bazelrc index 5e0695b6..2b7fbd0e 100644 --- a/.bazelrc +++ b/.bazelrc @@ -77,6 +77,8 @@ common --incompatible_disallow_empty_glob=false # Use the C linker to link Zig code common --@rules_zig//zig/settings:use_cc_common_link=True +# Zig self hosted backend is too fragile for now +common --@rules_zig//zig/settings:zigopt=-fllvm common --@toolchains_llvm_bootstrapped//config:experimental_stub_libgcc_s=True diff --git a/MODULE.bazel b/MODULE.bazel index d8ad79f0..588646c5 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -18,20 +18,12 @@ bazel_dep(name = "rules_oci", version = "2.2.6") bazel_dep(name = "rules_proto", version = "7.1.0") bazel_dep(name = "rules_python", version = "1.5.3") bazel_dep(name = "rules_rust", version = "0.63.0") -bazel_dep(name = "rules_zig", version = "0.12.1") +bazel_dep(name = "rules_zig", version = "0.12.2") bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.2.4") bazel_dep(name = "with_cfg.bzl", version = "0.11.0") bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2", dev_dependency = True) -# Remove this when rules_zig@0.12.2 is released. -archive_override( - module_name = "rules_zig", - urls = ["https://github.com/aherrmann/rules_zig/archive/cbf03e45c9ffa7d23a50790929649805a08afbf9.tar.gz"], - strip_prefix = "rules_zig-cbf03e45c9ffa7d23a50790929649805a08afbf9", - integrity = "sha256-lu6gNjulEIqG68I7DRyJzWInDVk3lL+mCj2J14vYKwc=", -) - zig = use_extension("@rules_zig//zig:extensions.bzl", "zig") zig.index(file = "//bazel:zig_index.json") zig.toolchain(zig_version = "0.15.2") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 0ecf36d6..cc876eae 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -111,6 +111,7 @@ "https://bazel-registry.zml.ai/modules/rules_cc/0.0.9/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.1.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_cc/0.2.11/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.9/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_distroless/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "not found", @@ -170,6 +171,8 @@ "https://bazel-registry.zml.ai/modules/rules_shell/0.4.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_zig/0.12.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_zig/0.12.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.4/MODULE.bazel": "not found", @@ -324,8 +327,9 @@ "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", "https://bcr.bazel.build/modules/rules_cc/0.1.1/MODULE.bazel": "2f0222a6f229f0bf44cd711dc13c858dad98c62d52bd51d8fc3a764a83125513", "https://bcr.bazel.build/modules/rules_cc/0.2.0/MODULE.bazel": "b5c17f90458caae90d2ccd114c81970062946f49f355610ed89bebf954f5783c", + "https://bcr.bazel.build/modules/rules_cc/0.2.11/MODULE.bazel": "e94f24f065bf2191dba2dace951814378b66a94bb3bcc48077492fe0508059b5", + "https://bcr.bazel.build/modules/rules_cc/0.2.11/source.json": "4d555dc20c9c135b21b2e403cf0ce8393fb65711b2305979ce053df4ee3e78de", "https://bcr.bazel.build/modules/rules_cc/0.2.9/MODULE.bazel": "34263f1dca62ea664265438cef714d7db124c03e1ed55ebb4f1dc860164308d1", - "https://bcr.bazel.build/modules/rules_cc/0.2.9/source.json": "4e49b40effcbd14fbfb233eb929de42dfff7b66538b4ffda310ad501638e7986", "https://bcr.bazel.build/modules/rules_distroless/0.5.1/MODULE.bazel": "2a63f4744d30749128105da5f96adf7caf5628e37548293f89e7fa39c3b3f2c2", "https://bcr.bazel.build/modules/rules_distroless/0.5.1/source.json": "c6b9ff7f325bfed89c3671757f14c1d1bc6077d0fcef809b8aa2d007cac7dd1d", "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", @@ -396,6 +400,9 @@ "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", "https://bcr.bazel.build/modules/rules_swift/2.1.1/source.json": "40fc69dfaac64deddbb75bd99cdac55f4427d9ca0afbe408576a65428427a186", + "https://bcr.bazel.build/modules/rules_zig/0.12.1/MODULE.bazel": "5953094b681c212eebe3dd4275809c52b9177aeaa058d8f1bff433822c0dcd01", + "https://bcr.bazel.build/modules/rules_zig/0.12.2/MODULE.bazel": "09d2ca5486c5c3bbcaa61057958d35f9b4e1e981e08a5a28ad516bab87e8c4ef", + "https://bcr.bazel.build/modules/rules_zig/0.12.2/source.json": "4cef85ce1b625967ce9c063b8716920e8e7067c38321981a7bfd581ad588f4ba", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", "https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4", @@ -2809,7 +2816,7 @@ "@@rules_zig+//zig:extensions.bzl%zig": { "general": { "bzlTransitiveDigest": "wV3euhBZSI0D2ZlqbY1YW592mghFY88d9H6zBMQRaJY=", - "usagesDigest": "yjAIKp9S0o0qAI7IuTPORFscLuBNsX9/I86qX5/4j1Y=", + "usagesDigest": "87SI2jgNGLvFKH9GAVUH7PVF1DJqfUvlXiSe9uiKbnk=", "recordedFileInputs": { "@@//bazel/zig_index.json": "e6c6b29fa227fa9b1806dc03af2b6dab00f52f34d39878557074e1626027259f", "@@rules_zig+//zig/private/versions.json": "6eb85ebaee72c4e6fbc82d8ad73f69e0899d839b70a7e19caa792ab6a5fa0c36" diff --git a/upb/BUILD.bazel b/upb/BUILD.bazel index 3273aebb..ebf973fb 100644 --- a/upb/BUILD.bazel +++ b/upb/BUILD.bazel @@ -14,8 +14,4 @@ zig_library( main = "upb.zig", visibility = ["//visibility:public"], deps = [":upb_c"], - zigopts = [ - #TODO(cerisier): Remove me when this is done inside rules_zig. - "-fllvm", - ], ) diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index be9d5558..efe4abb0 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -57,8 +57,6 @@ zig_library( zigopts = [ "-lc", "-freference-trace=20", - #TODO(cerisier): Remove me when this is done inside rules_zig. - "-fllvm", ], main = "zml.zig", visibility = ["//visibility:public"], diff --git a/zml/tokenizer/BUILD.bazel b/zml/tokenizer/BUILD.bazel index d7c8f6de..012e218c 100644 --- a/zml/tokenizer/BUILD.bazel +++ b/zml/tokenizer/BUILD.bazel @@ -19,10 +19,6 @@ zig_binary( name = "main", main = "main.zig", visibility = ["//visibility:public"], - zigopts = [ - #TODO(cerisier): Remove me when this is done inside rules_zig. - "-fllvm" - ], deps = [ ":tokenizer", "//async", From 06842784d3d43e51c7dfa323b146e7fb59d107b2 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Tue, 28 Oct 2025 17:08:23 +0100 Subject: [PATCH 056/881] workspace: use apple_support cc toolchains vs rules_cc (#355) This is the recommended toolchain and is where all further improvements is done. --- MODULE.bazel | 5 ++- MODULE.bazel.lock | 79 ++++++++++++++++++++++++++--------------------- 2 files changed, 47 insertions(+), 37 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 588646c5..73468e4f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -11,7 +11,10 @@ bazel_dep(name = "patchelf", version = "0.18.0") bazel_dep(name = "pcre2", version = "10.45") bazel_dep(name = "platforms", version = "1.0.0") bazel_dep(name = "protobuf", version = "32.0", repo_name = "com_google_protobuf") -bazel_dep(name = "rules_cc", version = "0.2.9") +# Needs to be added before rules_cc so that the cc toolchain declared by +# apple_support wins over the one in rules_cc. +bazel_dep(name = "apple_support", version = "1.24.2") +bazel_dep(name = "rules_cc", version = "0.2.12") bazel_dep(name = "rules_distroless", version = "0.5.1") bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep(name = "rules_oci", version = "2.2.6") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index cc876eae..7a3758f1 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -15,6 +15,7 @@ "https://bazel-registry.zml.ai/modules/apple_support/1.11.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/apple_support/1.15.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/apple_support/1.17.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/apple_support/1.24.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.0.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.16.0/MODULE.bazel": "not found", @@ -36,6 +37,7 @@ "https://bazel-registry.zml.ai/modules/bazel_features/1.21.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.23.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.24.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/bazel_features/1.27.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.28.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.3.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.30.0/MODULE.bazel": "not found", @@ -112,6 +114,8 @@ "https://bazel-registry.zml.ai/modules/rules_cc/0.1.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.11/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_cc/0.2.12/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_cc/0.2.8/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.9/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_distroless/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "not found", @@ -208,7 +212,8 @@ "https://bcr.bazel.build/modules/apple_support/1.11.1/MODULE.bazel": "1843d7cd8a58369a444fc6000e7304425fba600ff641592161d9f15b179fb896", "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", "https://bcr.bazel.build/modules/apple_support/1.17.1/MODULE.bazel": "655c922ab1209978a94ef6ca7d9d43e940cd97d9c172fb55f94d91ac53f8610b", - "https://bcr.bazel.build/modules/apple_support/1.17.1/source.json": "6b2b8c74d14e8d485528a938e44bdb72a5ba17632b9e14ef6e68a5ee96c8347f", + "https://bcr.bazel.build/modules/apple_support/1.24.2/MODULE.bazel": "0e62471818affb9f0b26f128831d5c40b074d32e6dda5a0d3852847215a41ca4", + "https://bcr.bazel.build/modules/apple_support/1.24.2/source.json": "2c22c9827093250406c5568da6c54e6fdf0ef06238def3d99c71b12feb057a8d", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.0.0/MODULE.bazel": "e118477db5c49419a88d78ebc7a2c2cea9d49600fe0f490c1903324a2c16ecd9", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.16.0/MODULE.bazel": "852f9ebbda017572a7c113a2434592dd3b2f55cd9a0faea3d4be5a09a59e4900", @@ -233,6 +238,7 @@ "https://bcr.bazel.build/modules/bazel_features/1.21.0/MODULE.bazel": "675642261665d8eea09989aa3b8afb5c37627f1be178382c320d1b46afba5e3b", "https://bcr.bazel.build/modules/bazel_features/1.23.0/MODULE.bazel": "fd1ac84bc4e97a5a0816b7fd7d4d4f6d837b0047cf4cbd81652d616af3a6591a", "https://bcr.bazel.build/modules/bazel_features/1.24.0/MODULE.bazel": "4796b4c25b47053e9bbffa792b3792d07e228ff66cd0405faef56a978708acd4", + "https://bcr.bazel.build/modules/bazel_features/1.27.0/MODULE.bazel": "621eeee06c4458a9121d1f104efb80f39d34deff4984e778359c60eaf1a8cb65", "https://bcr.bazel.build/modules/bazel_features/1.28.0/MODULE.bazel": "4b4200e6cbf8fa335b2c3f43e1d6ef3e240319c33d43d60cc0fbd4b87ece299d", "https://bcr.bazel.build/modules/bazel_features/1.3.0/MODULE.bazel": "cdcafe83ec318cda34e02948e81d790aab8df7a929cec6f6969f13a489ccecd9", "https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87", @@ -328,7 +334,9 @@ "https://bcr.bazel.build/modules/rules_cc/0.1.1/MODULE.bazel": "2f0222a6f229f0bf44cd711dc13c858dad98c62d52bd51d8fc3a764a83125513", "https://bcr.bazel.build/modules/rules_cc/0.2.0/MODULE.bazel": "b5c17f90458caae90d2ccd114c81970062946f49f355610ed89bebf954f5783c", "https://bcr.bazel.build/modules/rules_cc/0.2.11/MODULE.bazel": "e94f24f065bf2191dba2dace951814378b66a94bb3bcc48077492fe0508059b5", - "https://bcr.bazel.build/modules/rules_cc/0.2.11/source.json": "4d555dc20c9c135b21b2e403cf0ce8393fb65711b2305979ce053df4ee3e78de", + "https://bcr.bazel.build/modules/rules_cc/0.2.12/MODULE.bazel": "4216c383ce3223c7dfbd3afcb09c7056265156150b2a3bd3fd9d02949a7ee1bc", + "https://bcr.bazel.build/modules/rules_cc/0.2.12/source.json": "593d822a2ce7b70d5fad3195f7eb64d519dde94d7c20b0b79e6c23891a79b2bd", + "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", "https://bcr.bazel.build/modules/rules_cc/0.2.9/MODULE.bazel": "34263f1dca62ea664265438cef714d7db124c03e1ed55ebb4f1dc860164308d1", "https://bcr.bazel.build/modules/rules_distroless/0.5.1/MODULE.bazel": "2a63f4744d30749128105da5f96adf7caf5628e37548293f89e7fa39c3b3f2c2", "https://bcr.bazel.build/modules/rules_distroless/0.5.1/source.json": "c6b9ff7f325bfed89c3671757f14c1d1bc6077d0fcef809b8aa2d007cac7dd1d", @@ -432,37 +440,6 @@ }, "selectedYankedVersions": {}, "moduleExtensions": { - "@@apple_support+//crosstool:setup.bzl%apple_cc_configure_extension": { - "general": { - "bzlTransitiveDigest": "N6yHPMaF17im7GO6Gu+cdCtGXs+lAlBIhCeB1/ZZoPk=", - "usagesDigest": "39X2JjPCOAk6sThDALGv1L4q85GNjda2yfszm/phxxw=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "local_config_apple_cc_toolchains": { - "repoRuleId": "@@apple_support+//crosstool:setup.bzl%_apple_cc_autoconf_toolchains", - "attributes": {} - }, - "local_config_apple_cc": { - "repoRuleId": "@@apple_support+//crosstool:setup.bzl%_apple_cc_autoconf", - "attributes": {} - } - }, - "recordedRepoMappingEntries": [ - [ - "apple_support+", - "bazel_tools", - "bazel_tools" - ], - [ - "bazel_tools", - "rules_cc", - "rules_cc+" - ] - ] - } - }, "@@aspect_rules_py+//py:extensions.bzl%py_tools": { "general": { "bzlTransitiveDigest": "Udhh0icjIe0dHLjbgfmj2tzBz0geB4SXJl0FoPcpF84=", @@ -647,7 +624,7 @@ }, "@@rules_apple+//apple:apple.bzl%provisioning_profile_repository_extension": { "general": { - "bzlTransitiveDigest": "7XgPyOlTI0qJefux0gxRds30ZZCLPjr/7R9gFtU/jgE=", + "bzlTransitiveDigest": "Oa+WNwUQ6db3WoQOwZ8qU+aRE5kJufCF45z4ZCs6298=", "usagesDigest": "vsJl8Rw5NL+5Ag2wdUDoTeRF/5klkXO8545Iy7U1Q08=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -696,6 +673,16 @@ ], [ "rules_cc+", + "cc_compatibility_proxy", + "rules_cc++compatibility_proxy+cc_compatibility_proxy" + ], + [ + "rules_cc+", + "rules_cc", + "rules_cc+" + ], + [ + "rules_cc++compatibility_proxy+cc_compatibility_proxy", "rules_cc", "rules_cc+" ], @@ -1182,7 +1169,7 @@ }, "@@rules_rust+//crate_universe:extensions.bzl%crate": { "general": { - "bzlTransitiveDigest": "uv5JLD8zajzRT1Hh+FYGrK5o1QjgTE4Ohde5NQMsUlk=", + "bzlTransitiveDigest": "ZaON2eA6LKrwcPU8xDGi0W3+6wD1r6kjb0pSYFC4Se8=", "usagesDigest": "mup6mgSFatk34Q0znMZPpIEOFZyk9v3xf6jRIXEFTxc=", "recordedFileInputs": { "@@//zml/tokenizer/hftokenizers/Cargo.lock": "d78ffab37ef69bf8ff45a4d686b25175f30e8fed4d8375612e84dcf6db0cfc9c", @@ -2480,6 +2467,16 @@ ], [ "rules_cc+", + "cc_compatibility_proxy", + "rules_cc++compatibility_proxy+cc_compatibility_proxy" + ], + [ + "rules_cc+", + "rules_cc", + "rules_cc+" + ], + [ + "rules_cc++compatibility_proxy+cc_compatibility_proxy", "rules_cc", "rules_cc+" ], @@ -2513,7 +2510,7 @@ }, "@@rules_rust+//crate_universe/private:internal_extensions.bzl%cu_nr": { "general": { - "bzlTransitiveDigest": "BEf6td+6l3OplJk+TxURVgZT9CK8AKbw6H/uSseljfQ=", + "bzlTransitiveDigest": "XkpmtSAoMsVR5D0IWKWoGVKSPjnIiH3NEyMQqyFvqm8=", "usagesDigest": "3vKI8uvqTpJCf+t8aU6UD5d5cUWinWhtMjKkRpCLR+A=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -2613,6 +2610,16 @@ ], [ "rules_cc+", + "cc_compatibility_proxy", + "rules_cc++compatibility_proxy+cc_compatibility_proxy" + ], + [ + "rules_cc+", + "rules_cc", + "rules_cc+" + ], + [ + "rules_cc++compatibility_proxy+cc_compatibility_proxy", "rules_cc", "rules_cc+" ], From e5bc971c7514699147ebfb9f146784b2f003387e Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 12 Nov 2025 11:55:48 +0100 Subject: [PATCH 057/881] xla: bump to 9a77a88 (#357) --- MODULE.bazel | 5 +++ pjrt/pjrt.zig | 21 +++++++-- runtimes/cpu/cpu.bzl | 12 ++--- runtimes/cuda/cuda.bzl | 4 +- runtimes/rocm/rocm.bzl | 6 +-- runtimes/rocm/zmlxrocm.zig | 2 + ...header-C-compliant-for-PJRT-FFI-exte.patch | 45 +++++++++++++++++++ third_party/xla/repo.bzl | 4 +- third_party/xla/xla.bzl | 22 ++++++++- zml/callback.zig | 3 +- 10 files changed, 106 insertions(+), 18 deletions(-) create mode 100644 third_party/xla/patches/0001-PjRT-C-API-male-header-C-compliant-for-PJRT-FFI-exte.patch diff --git a/MODULE.bazel b/MODULE.bazel index 73468e4f..22e95e70 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -11,6 +11,7 @@ bazel_dep(name = "patchelf", version = "0.18.0") bazel_dep(name = "pcre2", version = "10.45") bazel_dep(name = "platforms", version = "1.0.0") bazel_dep(name = "protobuf", version = "32.0", repo_name = "com_google_protobuf") + # Needs to be added before rules_cc so that the cc toolchain declared by # apple_support wins over the one in rules_cc. bazel_dep(name = "apple_support", version = "1.24.2") @@ -111,10 +112,14 @@ use_repo( "local_config_cuda", "local_config_remote_execution", "local_config_rocm", + "local_config_sycl", "local_config_tensorrt", "python_version_repo", "rules_ml_toolchain", + "rules_shell", "stablehlo", + "sycl_configure", + "sycl_configure_ext", "triton", "tsl", ) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index cad1ce69..f9f30bda 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -1311,6 +1311,20 @@ pub const Ffi = extern struct { } }; + pub const TypeInfo = struct { + deleter: ?*const fn (*anyopaque) callconv(.c) void = null, + serialize: ?*const fn () callconv(.c) void = null, + deserialize: ?*const fn () callconv(.c) void = null, + + pub fn toCStruct(self: TypeInfo) c.PJRT_FFI_Type_Info { + return .{ + .deleter = @ptrCast(self.deleter), + .serialize = @ptrCast(self.serialize), + .deserialize = @ptrCast(self.deserialize), + }; + } + }; + // todo : support all missing handlers available in GPU plugin extension: handler_instantiate, handler_prepare, handler_initialize // introduced by https://github.com/openxla/xla/commit/ef85a7bcc308313492ebc50295a8a08b4e51b8f5 pub fn register( @@ -1337,13 +1351,14 @@ pub const Ffi = extern struct { } } - pub fn registerTypeId(self: *const Ffi, api: *const Api, type_name: []const u8) ApiError!ffi.TypeId { - var ret = pjrtStruct(c.PJRT_FFI_TypeID_Register_Args{ + pub fn registerTypeId(self: *const Ffi, api: *const Api, type_name: []const u8, type_info: ?*const c.PJRT_FFI_Type_Info) ApiError!ffi.TypeId { + var ret = pjrtStruct(c.PJRT_FFI_Type_Register_Args{ .type_name = type_name.ptr, .type_name_size = type_name.len, .type_id = 0, // let the plugin assign a unique type ID + .type_info = @ptrCast(@constCast(type_info)), }); - const result = self.inner.type_id_register.?(&ret); + const result = self.inner.type_register.?(&ret); if (result) |pjrt_c_error| { const pjrt_error: *Error = @ptrCast(pjrt_c_error); return pjrt_error.getCode(api).toApiError(); diff --git a/runtimes/cpu/cpu.bzl b/runtimes/cpu/cpu.bzl index 2a095f49..675ff3b4 100644 --- a/runtimes/cpu/cpu.bzl +++ b/runtimes/cpu/cpu.bzl @@ -23,22 +23,22 @@ def _cpu_pjrt_plugin_impl(mctx): http_archive( name = "libpjrt_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "124dc500291a5930f910ca23533520e22c90797110b29fd2c0d8274475f4a220", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v13.0.0/pjrt-cpu_linux-amd64.tar.gz", + sha256 = "ecc26dc792d2577474348eb48f3989aba8c3bb8d3cbd6df77ccf43357092a700", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v14.0.1/pjrt-cpu_linux-amd64.tar.gz", ) http_archive( name = "libpjrt_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "6e5b59874880f4db37c53fb1d52520d410b0078f9d2606a90762c6c622693c26", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v13.0.0/pjrt-cpu_darwin-amd64.tar.gz", + sha256 = "4a21db4ecd015fb772614ce4b491551d483ce11321c8784e3d0e07a9a425d5eb", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v14.0.1/pjrt-cpu_darwin-amd64.tar.gz", ) http_archive( name = "libpjrt_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "a6354bfed828a011e6d809eda2230e10c40c80044c67fe618b2a9615c047f092", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v13.0.0/pjrt-cpu_darwin-arm64.tar.gz", + sha256 = "e0ab4492468999ae7861a27837427846a708f4346fdae9ad1e84b80e1313566a", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v14.0.1/pjrt-cpu_darwin-arm64.tar.gz", ) return mctx.extension_metadata( diff --git a/runtimes/cuda/cuda.bzl b/runtimes/cuda/cuda.bzl index a99d4607..0b1a35bc 100644 --- a/runtimes/cuda/cuda.bzl +++ b/runtimes/cuda/cuda.bzl @@ -229,8 +229,8 @@ def _cuda_impl(mctx): http_archive( name = "libpjrt_cuda", build_file = "libpjrt_cuda.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v13.0.0/pjrt-cuda_linux-amd64.tar.gz", - sha256 = "6cdac9bac6db904e4423c9745c61000cf3acaf3c7da8016ab0016f076869048a", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v14.0.1/pjrt-cuda_linux-amd64.tar.gz", + sha256 = "4b618f05f9cd4cd14966717f7a521b1aa80b425999755870ce2d1caf45685578", ) return mctx.extension_metadata( diff --git a/runtimes/rocm/rocm.bzl b/runtimes/rocm/rocm.bzl index e71f1a92..f06daacb 100644 --- a/runtimes/rocm/rocm.bzl +++ b/runtimes/rocm/rocm.bzl @@ -121,7 +121,7 @@ _ROCM_PACKAGES = { "dlopen": "zmlxrocm_dlopen", }, ), - packages.filegroup(name = "hiprtc", srcs = ["lib/libhiprtc.so.6"]), + packages.filegroup(name = "hiprtc", srcs = ["lib/libhiprtc.so.6", "lib/libhiprtc-builtins.so.6"]), ]), "hipsolver": packages.filegroup(name = "hipsolver", srcs = ["lib/libhipsolver.so.0"]), } @@ -153,8 +153,8 @@ def _rocm_impl(mctx): http_archive( name = "libpjrt_rocm", build_file = "libpjrt_rocm.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v13.0.0/pjrt-rocm_linux-amd64.tar.gz", - sha256 = "945c43c68325c0e91cd41eaa594a9f9f6e78da7cc06892d83bf345b69f7bd714", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v14.0.1/pjrt-rocm_linux-amd64.tar.gz", + sha256 = "087858044f17bc06b70d7cbffc33e7f2bf590d732f3ce2c24425e41453ea1cf4", ) return mctx.extension_metadata( diff --git a/runtimes/rocm/zmlxrocm.zig b/runtimes/rocm/zmlxrocm.zig index 19f840c3..fccfbe9c 100644 --- a/runtimes/rocm/zmlxrocm.zig +++ b/runtimes/rocm/zmlxrocm.zig @@ -11,7 +11,9 @@ pub export fn zmlxrocm_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque .{ "libamd_comgr.so", "libamd_comgr.so.3" }, .{ "librocprofiler-register.so", "librocprofiler-register.so.0" }, .{ "libMIOpen.so", "libMIOpen.so.1" }, + .{ "libMIOpen.so.1", "libMIOpen.so.1" }, .{ "librccl.so", "librccl.so.1" }, + .{ "librocblas.so.4", "librocblas.so.4" }, .{ "librocblas.so", "librocblas.so.4" }, .{ "libroctracer64.so", "libroctracer64.so.4" }, .{ "libroctx64.so", "libroctx64.so.4" }, diff --git a/third_party/xla/patches/0001-PjRT-C-API-male-header-C-compliant-for-PJRT-FFI-exte.patch b/third_party/xla/patches/0001-PjRT-C-API-male-header-C-compliant-for-PJRT-FFI-exte.patch new file mode 100644 index 00000000..bfb65808 --- /dev/null +++ b/third_party/xla/patches/0001-PjRT-C-API-male-header-C-compliant-for-PJRT-FFI-exte.patch @@ -0,0 +1,45 @@ +From 0d88ac9b06c8bc78db817d85e90cd60d38e6561a Mon Sep 17 00:00:00 2001 +From: Hugo Mano +Date: Mon, 3 Nov 2025 16:54:54 +0100 +Subject: [PATCH] PjRT C API: make PJRT FFI C extension header compliant + + +XLA PR: https://github.com/openxla/xla/pull/33470 + +--- + xla/pjrt/c/pjrt_c_api_ffi_extension.h | 8 ++++---- + 1 file changed, 4 insertions(+), 4 deletions(-) + +diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h +index e756650911..33b78238b9 100644 +--- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h ++++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h +@@ -32,13 +32,13 @@ extern "C" { + // See: https://en.wikipedia.org/wiki/Foreign_function_interface + #define PJRT_API_FFI_EXTENSION_VERSION 3 + +-struct PJRT_FFI_Type_Info { ++typedef struct PJRT_FFI_Type_Info { + void (*deleter)(void* object); + void (*serialize)(); // placeholder for future use + void (*deserialize)(); // placeholder for future use +-}; ++} PJRT_FFI_Type_Info; + +-struct PJRT_FFI_Type_Register_Args { ++typedef struct PJRT_FFI_Type_Register_Args { + size_t struct_size; + PJRT_Extension_Base* extension_start; + +@@ -46,7 +46,7 @@ struct PJRT_FFI_Type_Register_Args { + size_t type_name_size; + int64_t type_id; // in-out + PJRT_FFI_Type_Info* type_info; +-}; ++} PJRT_FFI_Type_Register_Args; + PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Type_Register_Args, type_info); + + // Registers external type in a static type registry. If `type_id` is set to `0` +-- +2.50.1 (Apple Git-155) + diff --git a/third_party/xla/repo.bzl b/third_party/xla/repo.bzl index e9a8b406..4e45d2d3 100644 --- a/third_party/xla/repo.bzl +++ b/third_party/xla/repo.bzl @@ -4,9 +4,9 @@ def repo(): git_repository( name = "xla", remote = "https://github.com/openxla/xla.git", - commit = "b3fbfeeb076f2b536897180f4a274680ed9d52eb", + commit = "9a77a882bb2bc75cb8c29620ff8cd0fd089bdc86", patch_args = ["-p1"], patches = [ - # patches live in the patches directory + "third_party/xla/patches/0001-PjRT-C-API-male-header-C-compliant-for-PJRT-FFI-exte.patch", ], ) diff --git a/third_party/xla/xla.bzl b/third_party/xla/xla.bzl index 3780bc51..8444ffdb 100644 --- a/third_party/xla/xla.bzl +++ b/third_party/xla/xla.bzl @@ -42,6 +42,15 @@ if_rocm_newer_than = always_newer_than is_rocm_configured = always_false if_gpu_is_configured = always_if_false if_cuda_or_rocm = always_if_false +""", + }) + simple_files(name = "local_config_sycl", files = { + "BUILD.bazel": "", + "sycl/BUILD.bazel": "", + "crosstool/BUILD.bazel": "", + "sycl/build_defs.bzl": _BZL_HELPERS + """\ +if_sycl = always_if_false +if_sycl_is_configured = always_if_false """, }) simple_files(name = "local_config_remote_execution", files = { @@ -56,6 +65,17 @@ if_cuda_or_rocm = always_if_false simple_files(name = "rules_ml_toolchain", files = { "third_party/gpus/BUILD.bazel": "", "third_party/gpus/nvidia_common_rules.bzl": """cuda_rpath_flags = lambda *args, **kwargs: []""", + "third_party/extensions/sycl_configure.bzl": "", + }) + simple_files(name = "sycl_configure_ext", files = {}) + simple_files(name = "sycl_configure", files = {}) + simple_files(name = "rules_shell", files = { + "BUILD.bazel": "", + "shell/BUILD.bazel": "", + "shell/sh_binary.bzl": """ +def sh_binary(**kwargs): + native.sh_binary(**kwargs) +""", }) def _xla_impl(mctx): @@ -70,7 +90,7 @@ def _xla_impl(mctx): patch_file = ["//third_party/grpc:grpc.patch"], urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/refs/tags/v1.74.0.tar.gz"), ) - tf_vendored(name = "tsl", relpath = "third_party/tsl") + tf_vendored(name = "tsl", path = "third_party/tsl") _dummy_repos(mctx) diff --git a/zml/callback.zig b/zml/callback.zig index 709e6c87..04d09de7 100644 --- a/zml/callback.zig +++ b/zml/callback.zig @@ -151,7 +151,8 @@ pub fn register(Callback: type, platform: Platform) pjrt.ApiError!void { const target_name = "zml$" ++ @typeName(Callback); const proxy_cb = proxy(Callback); - Callback.type_id = try ffi.registerTypeId(platform.pjrt_api, @typeName(Callback)); + const type_info: pjrt.Ffi.TypeInfo = .{}; + Callback.type_id = try ffi.registerTypeId(platform.pjrt_api, @typeName(Callback), &type_info.toCStruct()); try ffi.register(platform.pjrt_api, target_name, @tagName(platform.target), &proxy_cb, Callback.callback_config.traits); log.debug("Registered custom call {} with target name \"{s}\"", .{ Callback, target_name }); } From 42e9f801e6e0a97eeeaecd712162936103216c41 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 17 Nov 2025 13:51:35 -0800 Subject: [PATCH 058/881] WIP --- BUILD.bazel | 22 + MODULE.bazel | 2 +- MODULE.bazel.lock | 88 +- bazel/zig_index.json | 3086 +++++++++++++++++++++--------------------- 4 files changed, 1610 insertions(+), 1588 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 97b2cf13..72cbb96b 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,3 +1,5 @@ +load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_c_proto_library") +load("@rules_zig//zig:defs.bzl", "zig_binary") load("//third_party/zls:zls_completion.bzl", "zls_completion") zls_completion( @@ -10,3 +12,23 @@ zls_completion( "//zml", ], ) + +zig_binary( + name = "main", + main = "main.zig", + deps = [ + ":xla_compile_options_upb", + ":xla_data_upb", + "//upb", + ], +) + +upb_c_proto_library( + name = "xla_data_upb", + deps = ["@xla//xla:xla_data_proto"], +) + +upb_c_proto_library( + name = "xla_compile_options_upb", + deps = ["@xla//xla/pjrt/proto:compile_options_proto"], +) diff --git a/MODULE.bazel b/MODULE.bazel index 22e95e70..1d0bce76 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -30,7 +30,7 @@ bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2", dev_dependency = Tr zig = use_extension("@rules_zig//zig:extensions.bzl", "zig") zig.index(file = "//bazel:zig_index.json") -zig.toolchain(zig_version = "0.15.2") +zig.toolchain(zig_version = "0.16.0-dev.1316+181b25ce4") zig.mirrors(urls = [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 7a3758f1..c2af45d8 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -2823,89 +2823,89 @@ "@@rules_zig+//zig:extensions.bzl%zig": { "general": { "bzlTransitiveDigest": "wV3euhBZSI0D2ZlqbY1YW592mghFY88d9H6zBMQRaJY=", - "usagesDigest": "87SI2jgNGLvFKH9GAVUH7PVF1DJqfUvlXiSe9uiKbnk=", + "usagesDigest": "Hdjgt35tQvLgxXPxkTE4XGRdMneQAuafpvd5+TlL/7M=", "recordedFileInputs": { - "@@//bazel/zig_index.json": "e6c6b29fa227fa9b1806dc03af2b6dab00f52f34d39878557074e1626027259f", + "@@//bazel/zig_index.json": "af50f44e3ea95eee3ab0cc894be8fe981d54238d6ddd3dafb3e5fc5a7ae3b8d5", "@@rules_zig+//zig/private/versions.json": "6eb85ebaee72c4e6fbc82d8ad73f69e0899d839b70a7e19caa792ab6a5fa0c36" }, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { - "zig_0.15.2_aarch64-linux": { + "zig_0.16.0-dev.1316_P181b25ce4_aarch64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.15.2/zig-aarch64-linux-0.15.2.tar.xz", + "url": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1316+181b25ce4.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "958ed7d1e00d0ea76590d27666efbf7a932281b3d7ba0c6b01b0ff26498f667f", - "zig_version": "0.15.2", + "sha256": "0f4cbfdab91b67d7cc0ec5d822a4cf0cf39da658d35e7eaa4cff3876d87444b3", + "zig_version": "0.16.0-dev.1316+181b25ce4", "platform": "aarch64-linux" } }, - "zig_0.15.2_aarch64-macos": { + "zig_0.16.0-dev.1316_P181b25ce4_aarch64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.15.2/zig-aarch64-macos-0.15.2.tar.xz", + "url": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1316+181b25ce4.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "3cc2bab367e185cdfb27501c4b30b1b0653c28d9f73df8dc91488e66ece5fa6b", - "zig_version": "0.15.2", + "sha256": "c8121f65e34bb8f14b721d61e3e5ea234271ca9c1c35594092de6c12ff2cd19f", + "zig_version": "0.16.0-dev.1316+181b25ce4", "platform": "aarch64-macos" } }, - "zig_0.15.2_aarch64-windows": { + "zig_0.16.0-dev.1316_P181b25ce4_aarch64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.15.2/zig-aarch64-windows-0.15.2.zip", + "url": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1316+181b25ce4.zip", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "b926465f8872bf983422257cd9ec248bb2b270996fbe8d57872cca13b56fc370", - "zig_version": "0.15.2", + "sha256": "19ab6ebda0a8dc185174a8e19c279f6a8cd6c0a7cbfeaf4c34bdc5afbeeb53d0", + "zig_version": "0.16.0-dev.1316+181b25ce4", "platform": "aarch64-windows" } }, - "zig_0.15.2_x86_64-linux": { + "zig_0.16.0-dev.1316_P181b25ce4_x86_64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.15.2/zig-x86_64-linux-0.15.2.tar.xz", + "url": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1316+181b25ce4.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "02aa270f183da276e5b5920b1dac44a63f1a49e55050ebde3aecc9eb82f93239", - "zig_version": "0.15.2", + "sha256": "3c5af46217c87fc3b5e94e25ea68882690c82495aeac7028230f69025f92cd47", + "zig_version": "0.16.0-dev.1316+181b25ce4", "platform": "x86_64-linux" } }, - "zig_0.15.2_x86_64-macos": { + "zig_0.16.0-dev.1316_P181b25ce4_x86_64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.15.2/zig-x86_64-macos-0.15.2.tar.xz", + "url": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1316+181b25ce4.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "375b6909fc1495d16fc2c7db9538f707456bfc3373b14ee83fdd3e22b3d43f7f", - "zig_version": "0.15.2", + "sha256": "7ac36a40a2ed06d4030fe55737e162e507f9be1e654c34a3c7b22d7f1fd213a3", + "zig_version": "0.16.0-dev.1316+181b25ce4", "platform": "x86_64-macos" } }, - "zig_0.15.2_x86_64-windows": { + "zig_0.16.0-dev.1316_P181b25ce4_x86_64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.15.2/zig-x86_64-windows-0.15.2.zip", + "url": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1316+181b25ce4.zip", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "3a0ed1e8799a2f8ce2a6e6290a9ff22e6906f8227865911fb7ddedc3cc14cb0c", - "zig_version": "0.15.2", + "sha256": "bf021b998f93c71ad5970cd58a3f2083401191137e432112e5a60c75c25b35b2", + "zig_version": "0.16.0-dev.1316+181b25ce4", "platform": "x86_64-windows" } }, @@ -2913,28 +2913,28 @@ "repoRuleId": "@@rules_zig+//zig/private/repo:toolchains_repo.bzl%toolchains_repo", "attributes": { "names": [ - "zig_0.15.2_aarch64-linux", - "zig_0.15.2_aarch64-macos", - "zig_0.15.2_aarch64-windows", - "zig_0.15.2_x86_64-linux", - "zig_0.15.2_x86_64-macos", - "zig_0.15.2_x86_64-windows" + "zig_0.16.0-dev.1316_P181b25ce4_aarch64-linux", + "zig_0.16.0-dev.1316_P181b25ce4_aarch64-macos", + "zig_0.16.0-dev.1316_P181b25ce4_aarch64-windows", + "zig_0.16.0-dev.1316_P181b25ce4_x86_64-linux", + "zig_0.16.0-dev.1316_P181b25ce4_x86_64-macos", + "zig_0.16.0-dev.1316_P181b25ce4_x86_64-windows" ], "labels": [ - "@zig_0.15.2_aarch64-linux//:zig_toolchain", - "@zig_0.15.2_aarch64-macos//:zig_toolchain", - "@zig_0.15.2_aarch64-windows//:zig_toolchain", - "@zig_0.15.2_x86_64-linux//:zig_toolchain", - "@zig_0.15.2_x86_64-macos//:zig_toolchain", - "@zig_0.15.2_x86_64-windows//:zig_toolchain" + "@zig_0.16.0-dev.1316_P181b25ce4_aarch64-linux//:zig_toolchain", + "@zig_0.16.0-dev.1316_P181b25ce4_aarch64-macos//:zig_toolchain", + "@zig_0.16.0-dev.1316_P181b25ce4_aarch64-windows//:zig_toolchain", + "@zig_0.16.0-dev.1316_P181b25ce4_x86_64-linux//:zig_toolchain", + "@zig_0.16.0-dev.1316_P181b25ce4_x86_64-macos//:zig_toolchain", + "@zig_0.16.0-dev.1316_P181b25ce4_x86_64-windows//:zig_toolchain" ], "zig_versions": [ - "0.15.2", - "0.15.2", - "0.15.2", - "0.15.2", - "0.15.2", - "0.15.2" + "0.16.0-dev.1316+181b25ce4", + "0.16.0-dev.1316+181b25ce4", + "0.16.0-dev.1316+181b25ce4", + "0.16.0-dev.1316+181b25ce4", + "0.16.0-dev.1316+181b25ce4", + "0.16.0-dev.1316+181b25ce4" ], "exec_lengths": [ 2, diff --git a/bazel/zig_index.json b/bazel/zig_index.json index d708f008..16212acb 100644 --- a/bazel/zig_index.json +++ b/bazel/zig_index.json @@ -1,1568 +1,1568 @@ { - "master": { - "version": "0.16.0-dev.732+2f3234c76", - "date": "2025-10-14", - "docs": "https://ziglang.org/documentation/master/", - "stdDocs": "https://ziglang.org/documentation/master/std/", - "src": { - "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.732+2f3234c76.tar.xz", - "shasum": "57cd3a69985ad3ccf3330c801959d193ee9a45a83e3c3ffcdd5dac4d15fe6886", - "size": "21457132" + "master": { + "version": "0.16.0-dev.1316+181b25ce4", + "date": "2025-11-13", + "docs": "https://ziglang.org/documentation/master/", + "stdDocs": "https://ziglang.org/documentation/master/std/", + "src": { + "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.1316+181b25ce4.tar.xz", + "shasum": "1f100b72af97c49f6c220dd1a118b2e3d24376db7500e2e0c2ca1c45e4ddfa6e", + "size": "21532448" + }, + "bootstrap": { + "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.1316+181b25ce4.tar.xz", + "shasum": "705a565be067f3eea6692094fc50f86c274127f65b6700fbd1407c39ccbb1035", + "size": "54297044" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1316+181b25ce4.tar.xz", + "shasum": "7ac36a40a2ed06d4030fe55737e162e507f9be1e654c34a3c7b22d7f1fd213a3", + "size": "56658972" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1316+181b25ce4.tar.xz", + "shasum": "c8121f65e34bb8f14b721d61e3e5ea234271ca9c1c35594092de6c12ff2cd19f", + "size": "51541680" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1316+181b25ce4.tar.xz", + "shasum": "3c5af46217c87fc3b5e94e25ea68882690c82495aeac7028230f69025f92cd47", + "size": "54601016" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1316+181b25ce4.tar.xz", + "shasum": "0f4cbfdab91b67d7cc0ec5d822a4cf0cf39da658d35e7eaa4cff3876d87444b3", + "size": "50329600" + }, + "arm-linux": { + "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.1316+181b25ce4.tar.xz", + "shasum": "41245740e7212dcf1416cba42c4af280a5b9a2842f75d4e1475aa06cddac2601", + "size": "51223720" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.1316+181b25ce4.tar.xz", + "shasum": "87c2327c8ea455a899cfa53cbb6d083f6c5f8b3ed318787aaaf343b31dc0b81a", + "size": "54400728" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.1316+181b25ce4.tar.xz", + "shasum": "51bd89e809757f8affdd9f2206b200a0bc3bf0c22ac46f6968706ef056646ce0", + "size": "54401180" + }, + "x86-linux": { + "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.1316+181b25ce4.tar.xz", + "shasum": "292dad4e3adf9a311d232d81f93653cb862b88c46bff787a23fdb1708b8b9086", + "size": "57269752" + }, + "loongarch64-linux": { + "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.1316+181b25ce4.tar.xz", + "shasum": "b073137e46fb307d6224b5bf662116631738723fb020da5224c9fffdaf7d88c5", + "size": "51645988" + }, + "s390x-linux": { + "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.1316+181b25ce4.tar.xz", + "shasum": "20ec44ae170b4163e0a3ec683754e9a17c007f6a703718e8d679b97877297e7b", + "size": "54094164" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1316+181b25ce4.zip", + "shasum": "bf021b998f93c71ad5970cd58a3f2083401191137e432112e5a60c75c25b35b2", + "size": "94482164" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1316+181b25ce4.zip", + "shasum": "19ab6ebda0a8dc185174a8e19c279f6a8cd6c0a7cbfeaf4c34bdc5afbeeb53d0", + "size": "90458422" + }, + "x86-windows": { + "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.1316+181b25ce4.zip", + "shasum": "7a4f6463c92ddf81a9dceaaa953257688cc93f0366013681b1c91d1e57f08572", + "size": "96320381" + }, + "aarch64-freebsd": { + "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.1316+181b25ce4.tar.xz", + "shasum": "09cea8e2ef61b91e2138e16d4cb32823edec41a2676fcfc7ff9e9dfd3175e3a6", + "size": "50296404" + }, + "arm-freebsd": { + "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.1316+181b25ce4.tar.xz", + "shasum": "213a3394ac0aefffab7997367c6f6510565b7e8e8c7829c469a13db191b8ad56", + "size": "51889452" + }, + "powerpc64-freebsd": { + "tarball": "https://ziglang.org/builds/zig-powerpc64-freebsd-0.16.0-dev.1316+181b25ce4.tar.xz", + "shasum": "79e9e0c49f30aab6c973f79840ef8107924af5c7dc2cd19692cf564ae77531b3", + "size": "52979080" + }, + "powerpc64le-freebsd": { + "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.1316+181b25ce4.tar.xz", + "shasum": "630b4b54a43f18d92134901396aafeb49996a9b6beb4d873bab70136819737a9", + "size": "54416056" + }, + "riscv64-freebsd": { + "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.1316+181b25ce4.tar.xz", + "shasum": "e422a73e4c4616ca07a4e7d70db5dcf3a1924ebbc17727b5f9f392b0c7e0ce20", + "size": "54545596" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.1316+181b25ce4.tar.xz", + "shasum": "e8ccc582b936ae144465d9ff390cc7a1910d322b5006d856d715d9b507f7911a", + "size": "54689648" + }, + "aarch64-netbsd": { + "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.1316+181b25ce4.tar.xz", + "shasum": "6565b48ea59fd3842650f3f930389b465f2a7ee81182596e771a701420e5f74a", + "size": "50258944" + }, + "arm-netbsd": { + "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.1316+181b25ce4.tar.xz", + "shasum": "328912165bf0523fd6122645fb0566e996ad399c819d9435bc7f6be8467b5495", + "size": "52915556" + }, + "x86-netbsd": { + "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.1316+181b25ce4.tar.xz", + "shasum": "2a3c24cdf88088285a9b8b3ceab705b09edc98574351e513e201cac3f86fd14b", + "size": "57879744" + }, + "x86_64-netbsd": { + "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.1316+181b25ce4.tar.xz", + "shasum": "c3bb94dae6f5e6bd78c45c4c271c8c939c08179ea66b2676377f21dada933f15", + "size": "54641936" + } }, - "bootstrap": { - "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.732+2f3234c76.tar.xz", - "shasum": "54c02c70217657bdf88309d62a90df1bcd17662125790a28f19dddde3b90f7aa", - "size": "54219768" + "0.15.2": { + "version": "0.15.2", + "date": "2025-10-11", + "docs": "https://ziglang.org/documentation/0.15.2/", + "stdDocs": "https://ziglang.org/documentation/0.15.2/std/", + "src": { + "tarball": "https://ziglang.org/download/0.15.2/zig-0.15.2.tar.xz", + "shasum": "d9b30c7aa983fcff5eed2084d54ae83eaafe7ff3a84d8fb754d854165a6e521c", + "size": "21366268" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.15.2/zig-bootstrap-0.15.2.tar.xz", + "shasum": "a6845459501df3c3264ebc587b02a7094ad14f4f3f7287c48f04457e784d0d85", + "size": "52718988" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-macos-0.15.2.tar.xz", + "shasum": "375b6909fc1495d16fc2c7db9538f707456bfc3373b14ee83fdd3e22b3d43f7f", + "size": "55800460" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-macos-0.15.2.tar.xz", + "shasum": "3cc2bab367e185cdfb27501c4b30b1b0653c28d9f73df8dc91488e66ece5fa6b", + "size": "50635984" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-linux-0.15.2.tar.xz", + "shasum": "02aa270f183da276e5b5920b1dac44a63f1a49e55050ebde3aecc9eb82f93239", + "size": "53733924" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-linux-0.15.2.tar.xz", + "shasum": "958ed7d1e00d0ea76590d27666efbf7a932281b3d7ba0c6b01b0ff26498f667f", + "size": "49471996" + }, + "arm-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-arm-linux-0.15.2.tar.xz", + "shasum": "7d8401495065dae45d6249c68d5faf10508f8203c86362ccb698aeaafc66b7cd", + "size": "50454216" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-riscv64-linux-0.15.2.tar.xz", + "shasum": "493512bdca485be3c6a9b0f69dcb4cbe4587f3af8e1be282fdd827108ba39930", + "size": "53613656" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-powerpc64le-linux-0.15.2.tar.xz", + "shasum": "e182c5f8d30fc7f97d17d2ffef1488826aa3afaa51e5f0dbe14c597a98b45778", + "size": "53570632" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86-linux-0.15.2.tar.xz", + "shasum": "4c6e23f39daa305e274197bfdff0d56ffd1750fc1de226ae10505c0eff52d7a5", + "size": "56334032" + }, + "loongarch64-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-loongarch64-linux-0.15.2.tar.xz", + "shasum": "95d6a03e0415dbe30ae9c00d58b5446fd681ebf3d4044a8b707e1784c8074a61", + "size": "50800332" + }, + "s390x-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-s390x-linux-0.15.2.tar.xz", + "shasum": "cd4b295f43b1b7ca3d603368caea82dbd00fce6691affb7527e607c0b6eafe8e", + "size": "53486436" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-windows-0.15.2.zip", + "shasum": "3a0ed1e8799a2f8ce2a6e6290a9ff22e6906f8227865911fb7ddedc3cc14cb0c", + "size": "92614574" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-windows-0.15.2.zip", + "shasum": "b926465f8872bf983422257cd9ec248bb2b270996fbe8d57872cca13b56fc370", + "size": "88464585" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86-windows-0.15.2.zip", + "shasum": "7a6dfc00f4cc09ec46d3e10eb06f42538e92b6285e34debea7462edaf371da98", + "size": "94516930" + }, + "aarch64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-freebsd-0.15.2.tar.xz", + "shasum": "c62efd319f86663eb7747709dfca259205edba8eaee98efc96a51ce40a9437de", + "size": "49363332" + }, + "arm-freebsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-arm-freebsd-0.15.2.tar.xz", + "shasum": "1d5d06428b2d4d31cf4e99008602f2096a9c652dd1025820a4aa66326fbf3dc4", + "size": "50900824" + }, + "powerpc64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-powerpc64-freebsd-0.15.2.tar.xz", + "shasum": "2bdb5a38069e667936d3378dfa332d67c21bb454cb17bea3c3b03faaa02a74af", + "size": "52099240" + }, + "powerpc64le-freebsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-powerpc64le-freebsd-0.15.2.tar.xz", + "shasum": "940894eb954c2077d0643f4deba8113239436fd0dcbc88160163b1b23fe3f035", + "size": "53459116" + }, + "riscv64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-riscv64-freebsd-0.15.2.tar.xz", + "shasum": "9aecdcf32fb63944c11389c10993aecd5e443a82d456693cc5ad230b5fe584ad", + "size": "53658752" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-freebsd-0.15.2.tar.xz", + "shasum": "5509ff57cd3f219165caed0da10221739af82742b9edfcda3f7bfaf4da7212dd", + "size": "53776208" + }, + "aarch64-netbsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-netbsd-0.15.2.tar.xz", + "shasum": "357c1e62df6581ec1726546f9a827973233cb31ac371d2b0e92456b306a0c6c0", + "size": "49365256" + }, + "arm-netbsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-arm-netbsd-0.15.2.tar.xz", + "shasum": "1c43871791d49dbabd224da307b995076e493e34c36734cb21c4f6cbc208f8c3", + "size": "52021696" + }, + "x86-netbsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86-netbsd-0.15.2.tar.xz", + "shasum": "7f18fc4720426a6ca77bbfb7b970c31a61c37ce4a867b53830762958e921cdde", + "size": "56893368" + }, + "x86_64-netbsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-netbsd-0.15.2.tar.xz", + "shasum": "11a3b9f4b66bece34a79c0e98e91136baf0ff9795ddf60e442ca6c8ad73b9204", + "size": "53778132" + } }, - "x86_64-macos": { - "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.732+2f3234c76.tar.xz", - "shasum": "3067dde8a2e11627c25b890df6f3fb4a88fb96e11cc1ef343c27c8a6d803402f", - "size": "56569768" + "0.15.1": { + "date": "2025-08-19", + "docs": "https://ziglang.org/documentation/master/", + "stdDocs": "https://ziglang.org/documentation/master/std/", + "notes": "https://ziglang.org/download/0.15.1/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.15.1/zig-0.15.1.tar.xz", + "shasum": "816c0303ab313f59766ce2097658c9fff7fafd1504f61f80f9507cd11652865f", + "size": "21359884" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.15.1/zig-bootstrap-0.15.1.tar.xz", + "shasum": "4c0cfbcf12da144955761ca43f89e3c74956bce978694fc1d0a63555f5c0a199", + "size": "52711548" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-macos-0.15.1.tar.xz", + "shasum": "9919392e0287cccc106dfbcbb46c7c1c3fa05d919567bb58d7eb16bca4116184", + "size": "55791880" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-macos-0.15.1.tar.xz", + "shasum": "c4bd624d901c1268f2deb9d8eb2d86a2f8b97bafa3f118025344242da2c54d7b", + "size": "50644996" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-linux-0.15.1.tar.xz", + "shasum": "c61c5da6edeea14ca51ecd5e4520c6f4189ef5250383db33d01848293bfafe05", + "size": "53734456" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-linux-0.15.1.tar.xz", + "shasum": "bb4a8d2ad735e7fba764c497ddf4243cb129fece4148da3222a7046d3f1f19fe", + "size": "49493872" + }, + "arm-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-arm-linux-0.15.1.tar.xz", + "shasum": "3f4bf3b06b67d14e3f38be30798488c1abe3cf5b33de570cd0e87bbf09b978ad", + "size": "50477464" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-riscv64-linux-0.15.1.tar.xz", + "shasum": "7ca7a3e621436fb31d66a253132fc39574a13d2a1b4d8458af4f2e7c6e4374fe", + "size": "53597792" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-powerpc64le-linux-0.15.1.tar.xz", + "shasum": "339e2106496be70b614e32d444298216e676c36d08f5cd7bee3dd1dbd4567fd7", + "size": "53566944" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86-linux-0.15.1.tar.xz", + "shasum": "dff166f25fdd06e8341d831a71211b5ba7411463a6b264bdefa8868438690b6a", + "size": "56311228" + }, + "loongarch64-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-loongarch64-linux-0.15.1.tar.xz", + "shasum": "0af18a012f3c4cffbef29ab5f42021484d92517f921a1380faacc89c50c1f89d", + "size": "50794376" + }, + "s390x-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-s390x-linux-0.15.1.tar.xz", + "shasum": "bcd13e5c88cf2d0da7100a48572195560069a94402c9bec3b316398204aa27e2", + "size": "53508068" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-windows-0.15.1.zip", + "shasum": "91e69e887ca8c943ce9a515df3af013d95a66a190a3df3f89221277ebad29e34", + "size": "92612958" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-windows-0.15.1.zip", + "shasum": "1f1bf16228b0ffcc882b713dc5e11a6db4219cb30997e13c72e8e723c2104ec6", + "size": "88458549" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86-windows-0.15.1.zip", + "shasum": "fb1c07cffbb43615d3158ab8b8f5db5da1d48875eca99e1d7a8a0064ff63fc5b", + "size": "94516052" + }, + "aarch64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-freebsd-0.15.1.tar.xz", + "shasum": "4d9d25c775828d49ea037b2284310c295d951793da8ebe94827a54fed4cca3ce", + "size": "49358464" + }, + "arm-freebsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-arm-freebsd-0.15.1.tar.xz", + "shasum": "9707f3a5f7e1a3d99c40db9a74de1acc61016a197ad289c2ad964f93cb213a18", + "size": "50904124" + }, + "powerpc64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-powerpc64-freebsd-0.15.1.tar.xz", + "shasum": "79448884372db04e62f77a46b92245fce805063e69534729537f75cb4681e7e3", + "size": "52099020" + }, + "powerpc64le-freebsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-powerpc64le-freebsd-0.15.1.tar.xz", + "shasum": "f18ee12ba9c98a20b8d2ad0410c679e7aa5591bc7917f169fd6b377833d2c7ad", + "size": "53480976" + }, + "riscv64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-riscv64-freebsd-0.15.1.tar.xz", + "shasum": "ee9f864a6fd8b57c1f4fdbb11daa06578746a6f8253afe3f5ddb5a76f2eddd2d", + "size": "53677800" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-freebsd-0.15.1.tar.xz", + "shasum": "9714f8ac3d3dc908b1599837c6167f857c1efaa930f0cfa840699458de7c3cd0", + "size": "53782112" + }, + "aarch64-netbsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-netbsd-0.15.1.tar.xz", + "shasum": "b2a528399777583b85b89c54ccd45488af7709d6dd29a27323ec2a229db40910", + "size": "49368000" + }, + "arm-netbsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-arm-netbsd-0.15.1.tar.xz", + "shasum": "93dc70109cbf5d2e022d20dfb56211978c4ea3c0b1e67aaabff947d8d1583aab", + "size": "52028184" + }, + "x86-netbsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86-netbsd-0.15.1.tar.xz", + "shasum": "a91b26051822ff17f3143f859b87dce5b4a13e90928bd6daa6f07a895d3410f0", + "size": "56881864" + }, + "x86_64-netbsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-netbsd-0.15.1.tar.xz", + "shasum": "6d7ba6eca5b4434351ebdb971b7303c9934514f9bb8481852251dbd5b52b03d6", + "size": "53794836" + } }, - "aarch64-macos": { - "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.732+2f3234c76.tar.xz", - "shasum": "ff75284a7a084f700594b094a22911add95fe0cd7a484dcb999f51946bfca3e9", - "size": "51435252" + "0.14.1": { + "date": "2025-05-21", + "docs": "https://ziglang.org/documentation/0.14.1/", + "stdDocs": "https://ziglang.org/documentation/0.14.1/std/", + "src": { + "tarball": "https://ziglang.org/download/0.14.1/zig-0.14.1.tar.xz", + "shasum": "237f8abcc8c3fd68c70c66cdbf63dce4fb5ad4a2e6225ac925e3d5b4c388f203", + "size": "17787696" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.14.1/zig-bootstrap-0.14.1.tar.xz", + "shasum": "89b2fce50bfbb1eee29c382193d22c6eb0c7da3a96b5ba6d05e0af2945b3ca3d", + "size": "48041028" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.14.1/zig-x86_64-macos-0.14.1.tar.xz", + "shasum": "b0f8bdfb9035783db58dd6c19d7dea89892acc3814421853e5752fe4573e5f43", + "size": "51044512" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.14.1/zig-aarch64-macos-0.14.1.tar.xz", + "shasum": "39f3dc5e79c22088ce878edc821dedb4ca5a1cd9f5ef915e9b3cc3053e8faefa", + "size": "45903552" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.14.1/zig-x86_64-linux-0.14.1.tar.xz", + "shasum": "24aeeec8af16c381934a6cd7d95c807a8cb2cf7df9fa40d359aa884195c4716c", + "size": "49086504" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.14.1/zig-aarch64-linux-0.14.1.tar.xz", + "shasum": "f7a654acc967864f7a050ddacfaa778c7504a0eca8d2b678839c21eea47c992b", + "size": "44954692" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.14.1/zig-armv7a-linux-0.14.1.tar.xz", + "shasum": "1b34d9ecfaeb3b360e86c0bc233e1a8a2bbed2d40f2d4f20c12bde2128714324", + "size": "46137456" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.14.1/zig-riscv64-linux-0.14.1.tar.xz", + "shasum": "005f214f74dbafb7b4d8bd305f4e9d25048f711d9ec6fa7b3d4fca177e11b882", + "size": "48094380" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.14.1/zig-powerpc64le-linux-0.14.1.tar.xz", + "shasum": "15523e748efd9224ae164482e5cc1f8c11b035246b9749fb5b00758484f384a1", + "size": "48720956" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.14.1/zig-x86-linux-0.14.1.tar.xz", + "shasum": "4bce6347fa112247443cb0952c19e560d1f90b910506cf895fd07a7b8d1c4a76", + "size": "51643520" + }, + "loongarch64-linux": { + "tarball": "https://ziglang.org/download/0.14.1/zig-loongarch64-linux-0.14.1.tar.xz", + "shasum": "fdc344789be6becbe220bf5ad035156e653cc148f4da270188cbac87729f17e0", + "size": "45825404" + }, + "s390x-linux": { + "tarball": "https://ziglang.org/download/0.14.1/zig-s390x-linux-0.14.1.tar.xz", + "shasum": "8e1bb63532ba22725f0a4a45e5c2920a8af508d8ca81e6007c330eec2b7033a6", + "size": "55602032" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.14.1/zig-x86_64-windows-0.14.1.zip", + "shasum": "554f5378228923ffd558eac35e21af020c73789d87afeabf4bfd16f2e6feed2c", + "size": "82229343" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.14.1/zig-aarch64-windows-0.14.1.zip", + "shasum": "b5aac0ccc40dd91e8311b1f257717d8e3903b5fefb8f659de6d65a840ad1d0e7", + "size": "78125379" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.14.1/zig-x86-windows-0.14.1.zip", + "shasum": "3ee730c2a5523570dc4dc1b724f3e4f30174ebc1fa109ca472a719586a473b18", + "size": "83983932" + } }, - "x86_64-linux": { - "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.732+2f3234c76.tar.xz", - "shasum": "f4d7bd32dbf88a2b4a020d8f6af7d4af0dca7db5a7a556ed832a1f1f684729dd", - "size": "54481728" + "0.14.0": { + "date": "2025-03-05", + "docs": "https://ziglang.org/documentation/0.14.0/", + "stdDocs": "https://ziglang.org/documentation/0.14.0/std/", + "notes": "https://ziglang.org/download/0.14.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.14.0/zig-0.14.0.tar.xz", + "shasum": "c76638c03eb204c4432ae092f6fa07c208567e110fbd4d862d131a7332584046", + "size": "17772188" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.14.0/zig-bootstrap-0.14.0.tar.xz", + "shasum": "bf3fcb22be0b83f4791748adb567d3304779d66d7bf9b1bd557ef6c2e0232807", + "size": "48029040" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.14.0/zig-macos-x86_64-0.14.0.tar.xz", + "shasum": "685816166f21f0b8d6fc7aa6a36e91396dcd82ca6556dfbe3e329deffc01fec3", + "size": "51039964" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.14.0/zig-macos-aarch64-0.14.0.tar.xz", + "shasum": "b71e4b7c4b4be9953657877f7f9e6f7ee89114c716da7c070f4a238220e95d7e", + "size": "45902412" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.14.0/zig-linux-x86_64-0.14.0.tar.xz", + "shasum": "473ec26806133cf4d1918caf1a410f8403a13d979726a9045b421b685031a982", + "size": "49091960" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.14.0/zig-linux-aarch64-0.14.0.tar.xz", + "shasum": "ab64e3ea277f6fc5f3d723dcd95d9ce1ab282c8ed0f431b4de880d30df891e4f", + "size": "44922728" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.14.0/zig-linux-armv7a-0.14.0.tar.xz", + "shasum": "a67dbfa9bdf769228ec994f2098698c619f930883ca5ef638f50eee2d7788d10", + "size": "46112980" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.14.0/zig-linux-riscv64-0.14.0.tar.xz", + "shasum": "a2b14d3de326d3fd095548ef38bf5a67b15dadd62fbcc90836d63cc4355f8ef7", + "size": "48069188" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.14.0/zig-linux-powerpc64le-0.14.0.tar.xz", + "shasum": "3eabd60876ebc2748de8eb57b4b8cfa78861ba9bf7c6dd83f4e3e1d271d7c45e", + "size": "48707620" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.14.0/zig-linux-x86-0.14.0.tar.xz", + "shasum": "55d1ba21de5109686ffa675b9cc1dd66930093c202995a637ce3e397816e4c08", + "size": "51621460" + }, + "loongarch64-linux": { + "tarball": "https://ziglang.org/download/0.14.0/zig-linux-loongarch64-0.14.0.tar.xz", + "shasum": "31a2f07df55f8f528b92d540db9aae6c0b38643c34dc1ac33a0111d855e996ae", + "size": "45821860" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.14.0/zig-windows-x86_64-0.14.0.zip", + "shasum": "f53e5f9011ba20bbc3e0e6d0a9441b31eb227a97bac0e7d24172f1b8b27b4371", + "size": "82219809" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.14.0/zig-windows-aarch64-0.14.0.zip", + "shasum": "03e984383ebb8f85293557cfa9f48ee8698e7c400239570c9ff1aef3bffaf046", + "size": "78113283" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.14.0/zig-windows-x86-0.14.0.zip", + "shasum": "1a867d808cf4fa9184358395d94441390b6b24ee8d00d356ca11ea7cbfd3a4ec", + "size": "83970029" + } }, - "aarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.732+2f3234c76.tar.xz", - "shasum": "7eb157b88e5c04b74653f30af2f8a078448504c80332dc3efa9b03a9b20a9552", - "size": "50239776" + "0.13.0": { + "date": "2024-06-07", + "docs": "https://ziglang.org/documentation/0.13.0/", + "stdDocs": "https://ziglang.org/documentation/0.13.0/std/", + "notes": "https://ziglang.org/download/0.13.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.13.0/zig-0.13.0.tar.xz", + "shasum": "06c73596beeccb71cc073805bdb9c0e05764128f16478fa53bf17dfabc1d4318", + "size": "17220728" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.13.0/zig-bootstrap-0.13.0.tar.xz", + "shasum": "cd446c084b5da7bc42e8ad9b4e1c910a957f2bf3f82bcc02888102cd0827c139", + "size": "46440356" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.13.0/zig-freebsd-x86_64-0.13.0.tar.xz", + "shasum": "adc1ffc9be56533b2f1c7191f9e435ad55db00414ff2829d951ef63d95aaad8c", + "size": "47177744" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.13.0/zig-macos-x86_64-0.13.0.tar.xz", + "shasum": "8b06ed1091b2269b700b3b07f8e3be3b833000841bae5aa6a09b1a8b4773effd", + "size": "48857012" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.13.0/zig-macos-aarch64-0.13.0.tar.xz", + "shasum": "46fae219656545dfaf4dce12fb4e8685cec5b51d721beee9389ab4194d43394c", + "size": "44892040" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.13.0/zig-linux-x86_64-0.13.0.tar.xz", + "shasum": "d45312e61ebcc48032b77bc4cf7fd6915c11fa16e4aad116b66c9468211230ea", + "size": "47082308" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.13.0/zig-linux-aarch64-0.13.0.tar.xz", + "shasum": "041ac42323837eb5624068acd8b00cd5777dac4cf91179e8dad7a7e90dd0c556", + "size": "43090688" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.13.0/zig-linux-armv7a-0.13.0.tar.xz", + "shasum": "4b0550239c2cd884cc03ddeb2b9934708f4b073ad59a96fccbfe09f7e4f54233", + "size": "43998916" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.13.0/zig-linux-riscv64-0.13.0.tar.xz", + "shasum": "9f7f3c685894ff80f43eaf3cad1598f4844ac46f4308374237c7f912f7907bb3", + "size": "45540956" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.13.0/zig-linux-powerpc64le-0.13.0.tar.xz", + "shasum": "6a467622448e830e8f85d20cabed151498af2b0a62f87b8c083b2fe127e60417", + "size": "46574596" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.13.0/zig-linux-x86-0.13.0.tar.xz", + "shasum": "876159cc1e15efb571e61843b39a2327f8925951d48b9a7a03048c36f72180f7", + "size": "52062336" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.13.0/zig-windows-x86_64-0.13.0.zip", + "shasum": "d859994725ef9402381e557c60bb57497215682e355204d754ee3df75ee3c158", + "size": "79163968" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.13.0/zig-windows-aarch64-0.13.0.zip", + "shasum": "95ff88427af7ba2b4f312f45d2377ce7a033e5e3c620c8caaa396a9aba20efda", + "size": "75119033" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.13.0/zig-windows-x86-0.13.0.zip", + "shasum": "eb3d533c3cf868bff7e74455dc005d18fd836c42e50b27106b31e9fec6dffc4a", + "size": "83274739" + } }, - "arm-linux": { - "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.732+2f3234c76.tar.xz", - "shasum": "4efa2363c4d5736eed5de54f42a051f70201e2eb49514dc8691e8469a3b5ec08", - "size": "51089884" + "0.12.1": { + "date": "2024-06-08", + "docs": "https://ziglang.org/documentation/0.12.1/", + "stdDocs": "https://ziglang.org/documentation/0.12.1/std/", + "src": { + "tarball": "https://ziglang.org/download/0.12.1/zig-0.12.1.tar.xz", + "shasum": "cca0bf5686fe1a15405bd535661811fac7663f81664d2204ea4590ce49a6e9ba", + "size": "17110932" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.12.1/zig-bootstrap-0.12.1.tar.xz", + "shasum": "e533e2cb6ef60edda0ae3f2ca5c6504557db6e985e3c3a80159beb32279ed341", + "size": "45542004" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.12.1/zig-freebsd-x86_64-0.12.1.tar.xz", + "shasum": "30eaa28fa7bc21d01f88528d75ae4b392ae4970406675d5ac712a4937a605123", + "size": "45590080" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.12.1/zig-macos-x86_64-0.12.1.tar.xz", + "shasum": "68f309c6e431d56eb42648d7fe86e8028a23464d401a467831e27c26f1a8d9c9", + "size": "47202232" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.12.1/zig-macos-aarch64-0.12.1.tar.xz", + "shasum": "6587860dbbc070e1ee069e1a3d18ced83b7ba7a80bf67b2c57caf7c9ce5208b1", + "size": "43451512" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.12.1/zig-linux-x86_64-0.12.1.tar.xz", + "shasum": "8860fc9725c2d9297a63008f853e9b11e3c5a2441217f99c1e3104cc6fa4a443", + "size": "45512024" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.12.1/zig-linux-aarch64-0.12.1.tar.xz", + "shasum": "27d4fef393e8d8b5f3b1d19f4dd43bfdb469b4ed17bbc4c2283c1b1fe650ef7f", + "size": "41867324" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.12.1/zig-linux-armv7a-0.12.1.tar.xz", + "shasum": "27493c922fd1454137ed6cbe6b6bec00352838fcd32e6e74f4f9187011816157", + "size": "42665508" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.12.1/zig-linux-riscv64-0.12.1.tar.xz", + "shasum": "463511a863acc16911cff6801de97623e6de296aab7b15dcda5f6fd078b400b5", + "size": "43932908" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.12.1/zig-linux-powerpc64le-0.12.1.tar.xz", + "shasum": "462d6f10350f3c5d6fc4c9d6cfdede93d69d0103af026889a15f65c5de791d39", + "size": "45227716" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.12.1/zig-linux-x86-0.12.1.tar.xz", + "shasum": "c36ac019ca0fc3167e50d17e2affd3d072a06c519761737d0639adfdf2dcfddd", + "size": "50555428" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.12.1/zig-windows-x86_64-0.12.1.zip", + "shasum": "52459b147c2de4d7c28f6b1a4b3d571c114e96836bf8e31c953a7d2f5e94251c", + "size": "76470574" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.12.1/zig-windows-aarch64-0.12.1.zip", + "shasum": "e1286114a11be4695a6ad5cf0ba6a0e5f489bb3b029a5237de93598133f0c13a", + "size": "72998386" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.12.1/zig-windows-x86-0.12.1.zip", + "shasum": "4f0cc9258527e7b8bcf742772b3069122086a5cd857b38a1c08002462ac81f80", + "size": "80979711" + } }, - "riscv64-linux": { - "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.732+2f3234c76.tar.xz", - "shasum": "05ef8e27a3c74c877f73ee844dfb8c15162f37811a7924000e46d55f1bac18fd", - "size": "54282992" + "0.12.0": { + "date": "2024-04-20", + "docs": "https://ziglang.org/documentation/0.12.0/", + "stdDocs": "https://ziglang.org/documentation/0.12.0/std/", + "notes": "https://ziglang.org/download/0.12.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.12.0/zig-0.12.0.tar.xz", + "shasum": "a6744ef84b6716f976dad923075b2f54dc4f785f200ae6c8ea07997bd9d9bd9a", + "size": "17099152" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.12.0/zig-bootstrap-0.12.0.tar.xz", + "shasum": "3efc643d56421fa68072af94d5512cb71c61acf1c32512f77c0b4590bff63187", + "size": "45527312" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.12.0/zig-freebsd-x86_64-0.12.0.tar.xz", + "shasum": "bd49957d1157850b337ee1cf3c00af83585cff98e1ebc3c524a267e7422a2d7b", + "size": "45578364" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.12.0/zig-macos-x86_64-0.12.0.tar.xz", + "shasum": "4d411bf413e7667821324da248e8589278180dbc197f4f282b7dbb599a689311", + "size": "47185720" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.12.0/zig-macos-aarch64-0.12.0.tar.xz", + "shasum": "294e224c14fd0822cfb15a35cf39aa14bd9967867999bf8bdfe3db7ddec2a27f", + "size": "43447724" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.12.0/zig-linux-x86_64-0.12.0.tar.xz", + "shasum": "c7ae866b8a76a568e2d5cfd31fe89cdb629bdd161fdd5018b29a4a0a17045cad", + "size": "45480516" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.12.0/zig-linux-aarch64-0.12.0.tar.xz", + "shasum": "754f1029484079b7e0ca3b913a0a2f2a6afd5a28990cb224fe8845e72f09de63", + "size": "41849060" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.12.0/zig-linux-armv7a-0.12.0.tar.xz", + "shasum": "b48221f4c64416d257f0f9f77d8727dccf7de92aeabe59744ee6e70d650a97bc", + "size": "42638808" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.12.0/zig-linux-riscv64-0.12.0.tar.xz", + "shasum": "bb2d1a78b01595a9c00ffd2e12ab46e32f8b6798f76aec643ff78e5b4f5c5afd", + "size": "43917444" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.12.0/zig-linux-powerpc64le-0.12.0.tar.xz", + "shasum": "9218beecfb9250e9eff863f58f987dca7077e3258dd263c40269086127f9679b", + "size": "45216736" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.12.0/zig-linux-x86-0.12.0.tar.xz", + "shasum": "fb752fceb88749a80d625a6efdb23bea8208962b5150d6d14c92d20efda629a5", + "size": "50498940" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.12.0/zig-windows-x86_64-0.12.0.zip", + "shasum": "2199eb4c2000ddb1fba85ba78f1fcf9c1fb8b3e57658f6a627a8e513131893f5", + "size": "76442958" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.12.0/zig-windows-aarch64-0.12.0.zip", + "shasum": "04c6b92689241ca7a8a59b5f12d2ca2820c09d5043c3c4808b7e93e41c7bf97b", + "size": "72976876" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.12.0/zig-windows-x86-0.12.0.zip", + "shasum": "497dc9fd415cadf948872f137d6cc0870507488f79db9547b8f2adb73cda9981", + "size": "80950440" + } }, - "powerpc64le-linux": { - "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.732+2f3234c76.tar.xz", - "shasum": "105a5f5ad1f040d4119377036944dff092f970d4c6e9d5585573fe81a489516b", - "size": "54288284" + "0.11.0": { + "date": "2023-08-04", + "docs": "https://ziglang.org/documentation/0.11.0/", + "stdDocs": "https://ziglang.org/documentation/0.11.0/std/", + "notes": "https://ziglang.org/download/0.11.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.11.0/zig-0.11.0.tar.xz", + "shasum": "72014e700e50c0d3528cef3adf80b76b26ab27730133e8202716a187a799e951", + "size": "15275316" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.11.0/zig-bootstrap-0.11.0.tar.xz", + "shasum": "38dd9e17433c7ce5687c48fa0a757462cbfcbe75d9d5087d14ebbe00efd21fdc", + "size": "43227592" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.11.0/zig-freebsd-x86_64-0.11.0.tar.xz", + "shasum": "ea430327f9178377b79264a1d492868dcff056cd76d43a6fb00719203749e958", + "size": "46432140" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.11.0/zig-macos-x86_64-0.11.0.tar.xz", + "shasum": "1c1c6b9a906b42baae73656e24e108fd8444bb50b6e8fd03e9e7a3f8b5f05686", + "size": "47189164" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.11.0/zig-macos-aarch64-0.11.0.tar.xz", + "shasum": "c6ebf927bb13a707d74267474a9f553274e64906fd21bf1c75a20bde8cadf7b2", + "size": "43855096" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.11.0/zig-linux-x86_64-0.11.0.tar.xz", + "shasum": "2d00e789fec4f71790a6e7bf83ff91d564943c5ee843c5fd966efc474b423047", + "size": "44961892" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.11.0/zig-linux-aarch64-0.11.0.tar.xz", + "shasum": "956eb095d8ba44ac6ebd27f7c9956e47d92937c103bf754745d0a39cdaa5d4c6", + "size": "41492432" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.11.0/zig-linux-armv7a-0.11.0.tar.xz", + "shasum": "aebe8bbeca39f13f9b7304465f9aee01ab005d243836bd40f4ec808093dccc9b", + "size": "42240664" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.11.0/zig-linux-riscv64-0.11.0.tar.xz", + "shasum": "24a478937eddb507e96d60bd4da00de9092b3f0920190eb45c4c99c946b00ed5", + "size": "43532324" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.11.0/zig-linux-powerpc64le-0.11.0.tar.xz", + "shasum": "75260e87325e820a278cf9e74f130c7b3d84c0b5197afb2e3c85eff3fcedd48d", + "size": "44656184" + }, + "powerpc-linux": { + "tarball": "https://ziglang.org/download/0.11.0/zig-linux-powerpc-0.11.0.tar.xz", + "shasum": "70a5f9668a66fb2a91a7c3488b15bcb568e1f9f44b95cd10075c138ad8c42864", + "size": "44539972" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.11.0/zig-linux-x86-0.11.0.tar.xz", + "shasum": "7b0dc3e0e070ae0e0d2240b1892af6a1f9faac3516cae24e57f7a0e7b04662a8", + "size": "49824456" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.11.0/zig-windows-x86_64-0.11.0.zip", + "shasum": "142caa3b804d86b4752556c9b6b039b7517a08afa3af842645c7e2dcd125f652", + "size": "77216743" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.11.0/zig-windows-aarch64-0.11.0.zip", + "shasum": "5d4bd13db5ecb0ddc749231e00f125c1d31087d708e9ff9b45c4f4e13e48c661", + "size": "73883137" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.11.0/zig-windows-x86-0.11.0.zip", + "shasum": "e72b362897f28c671633e650aa05289f2e62b154efcca977094456c8dac3aefa", + "size": "81576961" + } }, - "x86-linux": { - "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.732+2f3234c76.tar.xz", - "shasum": "1d05a15af12aaf97d9dee58f4707831c146f7cedec21a11180bd471b97221af4", - "size": "57162212" + "0.10.1": { + "date": "2023-01-19", + "docs": "https://ziglang.org/documentation/0.10.1/", + "stdDocs": "https://ziglang.org/documentation/0.10.1/std/", + "notes": "https://ziglang.org/download/0.10.1/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.10.1/zig-0.10.1.tar.xz", + "shasum": "69459bc804333df077d441ef052ffa143d53012b655a51f04cfef1414c04168c", + "size": "15143112" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.10.1/zig-bootstrap-0.10.1.tar.xz", + "shasum": "9f5781210b9be8f832553d160851635780f9bd71816065351ab29cfd8968f5e9", + "size": "43971816" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.10.1/zig-macos-x86_64-0.10.1.tar.xz", + "shasum": "02483550b89d2a3070c2ed003357fd6e6a3059707b8ee3fbc0c67f83ca898437", + "size": "45119596" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.10.1/zig-macos-aarch64-0.10.1.tar.xz", + "shasum": "b9b00477ec5fa1f1b89f35a7d2a58688e019910ab80a65eac2a7417162737656", + "size": "40517896" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.10.1/zig-linux-x86_64-0.10.1.tar.xz", + "shasum": "6699f0e7293081b42428f32c9d9c983854094bd15fee5489f12c4cf4518cc380", + "size": "44085596" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.10.1/zig-linux-aarch64-0.10.1.tar.xz", + "shasum": "db0761664f5f22aa5bbd7442a1617dd696c076d5717ddefcc9d8b95278f71f5d", + "size": "40321280" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.10.1/zig-linux-riscv64-0.10.1.tar.xz", + "shasum": "9db5b59a5112b8beb995094ba800e88b0060e9cf7cfadf4dc3e666c9010dc77b", + "size": "42196008" + }, + "i386-linux": { + "tarball": "https://ziglang.org/download/0.10.1/zig-linux-i386-0.10.1.tar.xz", + "shasum": "8c710ca5966b127b0ee3efba7310601ee57aab3dd6052a082ebc446c5efb2316", + "size": "48367388" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.10.1/zig-windows-x86_64-0.10.1.zip", + "shasum": "5768004e5e274c7969c3892e891596e51c5df2b422d798865471e05049988125", + "size": "73259729" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.10.1/zig-windows-aarch64-0.10.1.zip", + "shasum": "ece93b0d77b2ab03c40db99ef7ccbc63e0b6bd658af12b97898960f621305428", + "size": "69417459" + } }, - "loongarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.732+2f3234c76.tar.xz", - "shasum": "856e9155ccf551dfcd9d83a9b135f7a532dcc3f8ef29a1a7ca513e5e7cf1d8be", - "size": "51527580" + "0.10.0": { + "date": "2022-10-31", + "docs": "https://ziglang.org/documentation/0.10.0/", + "stdDocs": "https://ziglang.org/documentation/0.10.0/std/", + "notes": "https://ziglang.org/download/0.10.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.10.0/zig-0.10.0.tar.xz", + "shasum": "d8409f7aafc624770dcd050c8fa7e62578be8e6a10956bca3c86e8531c64c136", + "size": "14530912" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.10.0/zig-bootstrap-0.10.0.tar.xz", + "shasum": "c13dc70c4ff4c09f749adc0d473cbd3942991dd4d1bd2d860fbf257d8c1bbabf", + "size": "45625516" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.10.0/zig-freebsd-x86_64-0.10.0.tar.xz", + "shasum": "dd77afa2a8676afbf39f7d6068eda81b0723afd728642adaac43cb2106253d65", + "size": "44056504" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.10.0/zig-linux-aarch64-0.10.0.tar.xz", + "shasum": "09ef50c8be73380799804169197820ee78760723b0430fa823f56ed42b06ea0f", + "size": "40387688" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.10.0/zig-linux-armv7a-0.10.0.tar.xz", + "shasum": "7201b2e89cd7cc2dde95d39485fd7d5641ba67dc6a9a58c036cb4c308d2e82de", + "size": "50805936" + }, + "i386-linux": { + "tarball": "https://ziglang.org/download/0.10.0/zig-linux-i386-0.10.0.tar.xz", + "shasum": "dac8134f1328c50269f3e50b334298ec7916cb3b0ef76927703ddd1c96fd0115", + "size": "48451732" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.10.0/zig-linux-riscv64-0.10.0.tar.xz", + "shasum": "2a126f3401a7a7efc4b454f0a85c133db1af5a9dfee117f172213b7cbd47bfba", + "size": "42272968" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.10.0/zig-linux-x86_64-0.10.0.tar.xz", + "shasum": "631ec7bcb649cd6795abe40df044d2473b59b44e10be689c15632a0458ddea55", + "size": "44142400" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.10.0/zig-macos-aarch64-0.10.0.tar.xz", + "shasum": "02f7a7839b6a1e127eeae22ea72c87603fb7298c58bc35822a951479d53c7557", + "size": "40602664" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.10.0/zig-macos-x86_64-0.10.0.tar.xz", + "shasum": "3a22cb6c4749884156a94ea9b60f3a28cf4e098a69f08c18fbca81c733ebfeda", + "size": "45175104" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.10.0/zig-windows-x86_64-0.10.0.zip", + "shasum": "a66e2ff555c6e48781de1bcb0662ef28ee4b88af3af2a577f7b1950e430897ee", + "size": "73181558" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.10.0/zig-windows-aarch64-0.10.0.zip", + "shasum": "1bbda8d123d44f3ae4fa90d0da04b1e9093c3f9ddae3429a4abece1e1c0bf19a", + "size": "69332389" + } }, - "s390x-linux": { - "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.732+2f3234c76.tar.xz", - "shasum": "35ded5aab84f397eeeba384d6dfb9869bfd6c8962125421df82363f009d37302", - "size": "53991088" + "0.9.1": { + "date": "2022-02-14", + "docs": "https://ziglang.org/documentation/0.9.1/", + "stdDocs": "https://ziglang.org/documentation/0.9.1/std/", + "notes": "https://ziglang.org/download/0.9.1/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.9.1/zig-0.9.1.tar.xz", + "shasum": "38cf4e84481f5facc766ba72783e7462e08d6d29a5d47e3b75c8ee3142485210", + "size": "13940828" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.9.1/zig-bootstrap-0.9.1.tar.xz", + "shasum": "0a8e221c71860d8975c15662b3ed3bd863e81c4fe383455a596e5e0e490d6109", + "size": "42488812" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.9.1/zig-freebsd-x86_64-0.9.1.tar.xz", + "shasum": "4e06009bd3ede34b72757eec1b5b291b30aa0d5046dadd16ecb6b34a02411254", + "size": "39028848" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.9.1/zig-linux-aarch64-0.9.1.tar.xz", + "shasum": "5d99a39cded1870a3fa95d4de4ce68ac2610cca440336cfd252ffdddc2b90e66", + "size": "37034860" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.9.1/zig-linux-armv7a-0.9.1.tar.xz", + "shasum": "6de64456cb4757a555816611ea697f86fba7681d8da3e1863fa726a417de49be", + "size": "37974652" + }, + "i386-linux": { + "tarball": "https://ziglang.org/download/0.9.1/zig-linux-i386-0.9.1.tar.xz", + "shasum": "e776844fecd2e62fc40d94718891057a1dbca1816ff6013369e9a38c874374ca", + "size": "44969172" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.9.1/zig-linux-riscv64-0.9.1.tar.xz", + "shasum": "208dea53662c2c52777bd9e3076115d2126a4f71aed7f2ff3b8fe224dc3881aa", + "size": "39390868" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.9.1/zig-linux-x86_64-0.9.1.tar.xz", + "shasum": "be8da632c1d3273f766b69244d80669fe4f5e27798654681d77c992f17c237d7", + "size": "41011464" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.9.1/zig-macos-aarch64-0.9.1.tar.xz", + "shasum": "8c473082b4f0f819f1da05de2dbd0c1e891dff7d85d2c12b6ee876887d438287", + "size": "38995640" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.9.1/zig-macos-x86_64-0.9.1.tar.xz", + "shasum": "2d94984972d67292b55c1eb1c00de46580e9916575d083003546e9a01166754c", + "size": "43713044" + }, + "i386-windows": { + "tarball": "https://ziglang.org/download/0.9.1/zig-windows-i386-0.9.1.zip", + "shasum": "74a640ed459914b96bcc572183a8db687bed0af08c30d2ea2f8eba03ae930f69", + "size": "67929868" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.9.1/zig-windows-x86_64-0.9.1.zip", + "shasum": "443da53387d6ae8ba6bac4b3b90e9fef4ecbe545e1c5fa3a89485c36f5c0e3a2", + "size": "65047697" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.9.1/zig-windows-aarch64-0.9.1.zip", + "shasum": "621bf95f54dc3ff71466c5faae67479419951d7489e40e87fd26d195825fb842", + "size": "61478151" + } }, - "x86_64-windows": { - "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.732+2f3234c76.zip", - "shasum": "14594707e551dabe476e350f08439d506231725ff1c270180cf5eb5d943e33ad", - "size": "94274022" + "0.9.0": { + "date": "2021-12-20", + "docs": "https://ziglang.org/documentation/0.9.0/", + "stdDocs": "https://ziglang.org/documentation/0.9.0/std/", + "notes": "https://ziglang.org/download/0.9.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.9.0/zig-0.9.0.tar.xz", + "shasum": "cd1be83b12f8269cc5965e59877b49fdd8fa638efb6995ac61eb4cea36a2e381", + "size": "13928772" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.9.0/zig-bootstrap-0.9.0.tar.xz", + "shasum": "16b0bdf0bc0a5ed1e0950e08481413d806192e06443a512347526647b2baeabc", + "size": "42557736" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.9.0/zig-freebsd-x86_64-0.9.0.tar.xz", + "shasum": "c95afe679b7cc4110dc2ecd3606c83a699718b7a958d6627f74c20886333e194", + "size": "41293236" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.9.0/zig-linux-aarch64-0.9.0.tar.xz", + "shasum": "1524fedfdbade2dbc9bae1ed98ad38fa7f2114c9a3e94da0d652573c75efbc5a", + "size": "40008396" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.9.0/zig-linux-armv7a-0.9.0.tar.xz", + "shasum": "50225dee6e6448a63ee96383a34d9fe3bba34ae8da1a0c8619bde2cdfc1df87d", + "size": "41196876" + }, + "i386-linux": { + "tarball": "https://ziglang.org/download/0.9.0/zig-linux-i386-0.9.0.tar.xz", + "shasum": "b0dcf688349268c883292acdd55eaa3c13d73b9146e4b990fad95b84a2ac528b", + "size": "47408656" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.9.0/zig-linux-riscv64-0.9.0.tar.xz", + "shasum": "85466de07504767ed37f59782672ad41bbdf43d6480fafd07f45543278b07620", + "size": "44171420" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.9.0/zig-linux-x86_64-0.9.0.tar.xz", + "shasum": "5c55344a877d557fb1b28939785474eb7f4f2f327aab55293998f501f7869fa6", + "size": "43420796" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.9.0/zig-macos-aarch64-0.9.0.tar.xz", + "shasum": "3991c70594d61d09fb4b316157a7c1d87b1d4ec159e7a5ecd11169ff74cad832", + "size": "39013392" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.9.0/zig-macos-x86_64-0.9.0.tar.xz", + "shasum": "c5280eeec4d6e5ea5ce5b448dc9a7c4bdd85ecfed4c1b96aa0835e48b36eccf0", + "size": "43764596" + }, + "i386-windows": { + "tarball": "https://ziglang.org/download/0.9.0/zig-windows-i386-0.9.0.zip", + "shasum": "bb839434afc75092015cf4c33319d31463c18512bc01dd719aedf5dcbc368466", + "size": "67946715" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.9.0/zig-windows-x86_64-0.9.0.zip", + "shasum": "084ea2646850aaf068234b0f1a92b914ed629be47075e835f8a67d55c21d880e", + "size": "65045849" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.9.0/zig-windows-aarch64-0.9.0.zip", + "shasum": "f9018725e3fb2e8992b17c67034726971156eb190685018a9ac8c3a9f7a22340", + "size": "61461921" + } }, - "aarch64-windows": { - "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.732+2f3234c76.zip", - "shasum": "fd07bca688fa7f4a89d9595f6475b61839ed34dd4e98322b7ee594fde12aafbe", - "size": "90225621" + "0.8.1": { + "date": "2021-09-06", + "docs": "https://ziglang.org/documentation/0.8.1/", + "stdDocs": "https://ziglang.org/documentation/0.8.1/std/", + "notes": "https://ziglang.org/download/0.8.1/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.8.1/zig-0.8.1.tar.xz", + "shasum": "8c428e14a0a89cb7a15a6768424a37442292858cdb695e2eb503fa3c7bf47f1a", + "size": "12650228" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.8.1/zig-bootstrap-0.8.1.tar.xz", + "shasum": "fa1239247f830ecd51c42537043f5220e4d1dfefdc54356fa419616a0efb3902", + "size": "43613464" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.8.1/zig-freebsd-x86_64-0.8.1.tar.xz", + "shasum": "fc4f6478bcf3a9fce1b8ef677a91694f476dd35be6d6c9c4f44a8b76eedbe176", + "size": "39150924" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.8.1/zig-linux-aarch64-0.8.1.tar.xz", + "shasum": "2166dc9f2d8df387e8b4122883bb979d739281e1ff3f3d5483fec3a23b957510", + "size": "37605932" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.8.1/zig-linux-armv7a-0.8.1.tar.xz", + "shasum": "5ba58141805e2519f38cf8e715933cbf059f4f3dade92c71838cce341045de05", + "size": "39185876" + }, + "i386-linux": { + "tarball": "https://ziglang.org/download/0.8.1/zig-linux-i386-0.8.1.tar.xz", + "shasum": "2f3e84f30492b5f1c5f97cecc0166f07a8a8d50c5f85dbb3a6ef2a4ee6f915e6", + "size": "44782932" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.8.1/zig-linux-riscv64-0.8.1.tar.xz", + "shasum": "4adfaf147b025917c03367462fe5018aaa9edbc6439ef9cd0da2b074ae960554", + "size": "41234480" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.8.1/zig-linux-x86_64-0.8.1.tar.xz", + "shasum": "6c032fc61b5d77a3f3cf781730fa549f8f059ffdb3b3f6ad1c2994d2b2d87983", + "size": "41250060" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.8.1/zig-macos-aarch64-0.8.1.tar.xz", + "shasum": "5351297e3b8408213514b29c0a938002c5cf9f97eee28c2f32920e1227fd8423", + "size": "35340712" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.8.1/zig-macos-x86_64-0.8.1.tar.xz", + "shasum": "16b0e1defe4c1807f2e128f72863124bffdd906cefb21043c34b673bf85cd57f", + "size": "39946200" + }, + "i386-windows": { + "tarball": "https://ziglang.org/download/0.8.1/zig-windows-i386-0.8.1.zip", + "shasum": "099605051eb0452a947c8eab8fbbc7e43833c8376d267e94e41131c289a1c535", + "size": "64152358" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.8.1/zig-windows-x86_64-0.8.1.zip", + "shasum": "43573db14cd238f7111d6bdf37492d363f11ecd1eba802567a172f277d003926", + "size": "61897838" + } }, - "x86-windows": { - "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.732+2f3234c76.zip", - "shasum": "c65ddde1112852498be49ae136608e2bb9f7a1f78347447f39589288fba281fc", - "size": "96083498" + "0.8.0": { + "date": "2021-06-04", + "docs": "https://ziglang.org/documentation/0.8.0/", + "stdDocs": "https://ziglang.org/documentation/0.8.0/std/", + "notes": "https://ziglang.org/download/0.8.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.8.0/zig-0.8.0.tar.xz", + "shasum": "03a828d00c06b2e3bb8b7ff706997fd76bf32503b08d759756155b6e8c981e77", + "size": "12614896" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.8.0/zig-bootstrap-0.8.0.tar.xz", + "shasum": "10600bc9c01f92e343f40d6ecc0ad05d67d27c3e382bce75524c0639cd8ca178", + "size": "43574248" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.8.0/zig-freebsd-x86_64-0.8.0.tar.xz", + "shasum": "0d3ccc436c8c0f50fd55462f72f8492d98723c7218ffc2a8a1831967d81b4bdc", + "size": "39125332" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.8.0/zig-linux-aarch64-0.8.0.tar.xz", + "shasum": "ee204ca2c2037952cf3f8b10c609373a08a291efa4af7b3c73be0f2b27720470", + "size": "37575428" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.8.0/zig-linux-armv7a-0.8.0.tar.xz", + "shasum": "d00b8bd97b79f45d6f5da956983bafeaa082e6c2ae8c6e1c6d4faa22fa29b320", + "size": "38884212" + }, + "i386-linux": { + "tarball": "https://ziglang.org/download/0.8.0/zig-linux-i386-0.8.0.tar.xz", + "shasum": "96e43ee6ed81c3c63401f456bd1c58ee6d42373a43cb324f5cf4974ca0998865", + "size": "42136032" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.8.0/zig-linux-riscv64-0.8.0.tar.xz", + "shasum": "75997527a78cdab64c40c43d9df39c01c4cdb557bb3992a869838371a204cfea", + "size": "40016268" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.8.0/zig-linux-x86_64-0.8.0.tar.xz", + "shasum": "502625d3da3ae595c5f44a809a87714320b7a40e6dff4a895b5fa7df3391d01e", + "size": "41211184" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.8.0/zig-macos-aarch64-0.8.0.tar.xz", + "shasum": "b32d13f66d0e1ff740b3326d66a469ee6baddbd7211fa111c066d3bd57683111", + "size": "35292180" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.8.0/zig-macos-x86_64-0.8.0.tar.xz", + "shasum": "279f9360b5cb23103f0395dc4d3d0d30626e699b1b4be55e98fd985b62bc6fbe", + "size": "39969312" + }, + "i386-windows": { + "tarball": "https://ziglang.org/download/0.8.0/zig-windows-i386-0.8.0.zip", + "shasum": "b6ec9aa6cd6f3872fcb30d43ff411802d82008a0c4142ee49e208a09b2c1c5fe", + "size": "61507213" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.8.0/zig-windows-x86_64-0.8.0.zip", + "shasum": "8580fbbf3afb72e9b495c7f8aeac752a03475ae0bbcf5d787f3775c7e1f4f807", + "size": "61766193" + } }, - "aarch64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.732+2f3234c76.tar.xz", - "shasum": "3e63836c990e835e8257223664ace605c9aff32c38b546213e12e8e0004236a0", - "size": "50184132" + "0.7.1": { + "date": "2020-12-13", + "docs": "https://ziglang.org/documentation/0.7.1/", + "stdDocs": "https://ziglang.org/documentation/0.7.1/std/", + "notes": "https://ziglang.org/download/0.7.1/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.7.1/zig-0.7.1.tar.xz", + "shasum": "2db3b944ab368d955b48743d9f7c963b8f96de1a441ba5a35e197237cc6dae44", + "size": "10711824" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.7.1/zig-bootstrap-0.7.1.tar.xz", + "shasum": "040f27c1fae4b0cac0a2782aecdb691f6a2f8e89db6a6ed35024c31c304fd9b2", + "size": "40232612" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.7.1/zig-freebsd-x86_64-0.7.1.tar.xz", + "shasum": "e73c1dca35791a3183fdd5ecde0443ebbe180942efceafe651886034fb8def09", + "size": "39066808" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.7.1/zig-linux-aarch64-0.7.1.tar.xz", + "shasum": "48ec90eba407e4587ddef7eecef25fec7e13587eb98e3b83c5f2f5fff2a5cbe7", + "size": "33780552" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.7.1/zig-linux-armv7a-0.7.1.tar.xz", + "shasum": "5a0662e07b4c4968665e1f97558f8591f6facec45d2e0ff5715e661743107ceb", + "size": "35813504" + }, + "i386-linux": { + "tarball": "https://ziglang.org/download/0.7.1/zig-linux-i386-0.7.1.tar.xz", + "shasum": "4882e052e5f83690bd0334bb4fc1702b5403cb3a3d2aa63fd7d6043d8afecba3", + "size": "39230912" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.7.1/zig-linux-riscv64-0.7.1.tar.xz", + "shasum": "187294bfd35983348c3fe042901b42e67e7e36ab7f77a5f969d21c0051f4d21f", + "size": "37454812" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.7.1/zig-linux-x86_64-0.7.1.tar.xz", + "shasum": "18c7b9b200600f8bcde1cd8d7f1f578cbc3676241ce36d771937ce19a8159b8d", + "size": "37848176" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.7.1/zig-macos-x86_64-0.7.1.tar.xz", + "shasum": "845cb17562978af0cf67e3993f4e33330525eaf01ead9386df9105111e3bc519", + "size": "36211076" + }, + "i386-windows": { + "tarball": "https://ziglang.org/download/0.7.1/zig-windows-i386-0.7.1.zip", + "shasum": "a1b9a7421e13153e07fd2e2c93ff29aad64d83105b8fcdafa633dbe689caf1c0", + "size": "54374983" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.7.1/zig-windows-x86_64-0.7.1.zip", + "shasum": "4818a8a65b4672bc52c0ae7f14d014e0eb8caf10f12c0745176820384cea296a", + "size": "54909997" + } }, - "arm-freebsd": { - "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.732+2f3234c76.tar.xz", - "shasum": "29580f93f4e2e1c947f09f2ac27cc9b035744e27093d3838d7c01315b70dbf80", - "size": "51776420" + "0.7.0": { + "date": "2020-11-08", + "docs": "https://ziglang.org/documentation/0.7.0/", + "stdDocs": "https://ziglang.org/documentation/0.7.0/std/", + "notes": "https://ziglang.org/download/0.7.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.7.0/zig-0.7.0.tar.xz", + "shasum": "0efd2cf6c3b05723db80e9cf193bc55150bba84ca41f855a90f53fc756445f83", + "size": "10683920" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.7.0/zig-bootstrap-0.7.0.tar.xz", + "shasum": "f073beaf5c53c8c57c0d374cbfcb332ef92ad703173edba0d9e0f2ed28401b72", + "size": "40200436" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.7.0/zig-freebsd-x86_64-0.7.0.tar.xz", + "shasum": "a0c926272ee4ae720034b4a6a1dc98399d76156dd84182554740f0ca8a41fc99", + "size": "34798992" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.7.0/zig-linux-aarch64-0.7.0.tar.xz", + "shasum": "f89933bac87d44be82325754ff88423020c81c7032a6fc41cfeb81e982eeab9b", + "size": "33096140" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.7.0/zig-linux-armv7a-0.7.0.tar.xz", + "shasum": "011c267e25a96ee160505a560c441daa045359a9d50e13ab1bada9d75c95db2d", + "size": "35157584" + }, + "i386-linux": { + "tarball": "https://ziglang.org/download/0.7.0/zig-linux-i386-0.7.0.tar.xz", + "shasum": "4bb2072cd363bcb1cbeb4872ff5cbc1f683b02d0cc1f90c46e3ea7422ce53222", + "size": "38530596" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.7.0/zig-linux-riscv64-0.7.0.tar.xz", + "shasum": "40dff81faa6f232ac40abbf88b9371f3cc932b6e09c423b94387c9ea580cb7be", + "size": "36759992" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.7.0/zig-linux-x86_64-0.7.0.tar.xz", + "shasum": "e619b1c6094c095b932767f527aee2507f847ea981513ff8a08aab0fd730e0ac", + "size": "37154432" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.7.0/zig-macos-aarch64-0.7.0.tar.xz", + "shasum": "338238035734db74ea4f30e500a4893bf741d38305c10952d5e39fa05bdb057d", + "size": "33739424" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.7.0/zig-macos-x86_64-0.7.0.tar.xz", + "shasum": "94063f9a311cbbf7a2e0a12295e09437182cf950f18cb0eb30ea9893f3677f24", + "size": "35258328" + }, + "i386-windows": { + "tarball": "https://ziglang.org/download/0.7.0/zig-windows-i386-0.7.0.zip", + "shasum": "b1e520aacbfbd645ff3521b3eb4d44166d9a0288b8725e4b001f8b50a425eb2e", + "size": "53390517" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.7.0/zig-windows-x86_64-0.7.0.zip", + "shasum": "965f56c0a36f9cda2125e3a348bc654f7f155e2804c3667d231775ec228f8553", + "size": "53943784" + } }, - "powerpc64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-powerpc64-freebsd-0.16.0-dev.732+2f3234c76.tar.xz", - "shasum": "6a8ac2bafea198104c09be1d9d314bfb8255281efc6a1550d92642d5965d25ab", - "size": "52866216" + "0.6.0": { + "date": "2020-04-13", + "docs": "https://ziglang.org/documentation/0.6.0/", + "stdDocs": "https://ziglang.org/documentation/0.6.0/std/", + "notes": "https://ziglang.org/download/0.6.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.6.0/zig-0.6.0.tar.xz", + "shasum": "5d167dc19354282dd35dd17b38e99e1763713b9be8a4ba9e9e69284e059e7204", + "size": "10349552" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.6.0/zig-bootstrap-0.6.0.tar.xz", + "shasum": "5e0e4dc878b3dd0c1852a442b174f0732e8c07869a8fcd226b71a93b89b381ab", + "size": "38469948" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.6.0/zig-freebsd-x86_64-0.6.0.tar.xz", + "shasum": "190ff79c1eb56805a315d7c7a51082e32f62926250c0702b36760c225e1634a3", + "size": "36974604" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.6.0/zig-linux-aarch64-0.6.0.tar.xz", + "shasum": "e7520efd42cfa02be48c2e430d08fe1f3cbb999d21d9f0d3ffd0febb976b2f41", + "size": "37090044" + }, + "armv6kz-linux": { + "tarball": "https://ziglang.org/download/0.6.0/zig-linux-armv6kz-0.6.0.tar.xz", + "shasum": "36b6493b3fed43eb1f0000e765798ad31a6bb7d7fd3f553ac1c3761dbc919b82", + "size": "39133452" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.6.0/zig-linux-armv7a-0.6.0.tar.xz", + "shasum": "946969abe357def95ca9cbbfcebfcf2d90cf967bcd3f48ee87662e32d91d8f35", + "size": "39143748" + }, + "i386-linux": { + "tarball": "https://ziglang.org/download/0.6.0/zig-linux-i386-0.6.0.tar.xz", + "shasum": "a97a2f9ae21575743cdd763c1917d49400d83fc562ef64582b18bade43eb24ce", + "size": "44877640" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.6.0/zig-linux-riscv64-0.6.0.tar.xz", + "shasum": "68ddee43f7503c8ae5f26a921f3602c34719a02ed2241f528c0b8b888cc14b38", + "size": "41993144" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.6.0/zig-linux-x86_64-0.6.0.tar.xz", + "shasum": "08fd3c757963630645441c2772362e9c2294020c44f14fce1b89f45de0dc1253", + "size": "44766320" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.6.0/zig-macos-x86_64-0.6.0.tar.xz", + "shasum": "17270360e87ddc49f737e760047b2fac49f1570a824a306119b1194ac4093895", + "size": "42573184" + }, + "i386-windows": { + "tarball": "https://ziglang.org/download/0.6.0/zig-windows-i386-0.6.0.zip", + "shasum": "3b0a02618743e92175990dc6d1a787bb95ff62c4cda016f1c14c7786f575f8ca", + "size": "60446431" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.6.0/zig-windows-x86_64-0.6.0.zip", + "shasum": "c3b897832523e1026e10b2d8d55d7f895185c0a27a63681f3a23219c3f1c38f4", + "size": "49065511" + } }, - "powerpc64le-freebsd": { - "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.732+2f3234c76.tar.xz", - "shasum": "5acbe627380efa3d746dfc01d0fb49744610caf2acd935364ce2f82ae754bb60", - "size": "54293208" + "0.5.0": { + "date": "2019-09-30", + "docs": "https://ziglang.org/documentation/0.5.0/", + "notes": "https://ziglang.org/download/0.5.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.5.0/zig-0.5.0.tar.xz", + "shasum": "55ae16960f152bcb9cf98b4f8570902d0e559a141abf927f0d3555b7cc838a31", + "size": "10956132" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.5.0/zig-freebsd-x86_64-0.5.0.tar.xz", + "shasum": "9e1f4d36c3d584c0aa01f20eb4cd0a0eef3eee5af23e483b8414de55feab6ab6", + "size": "33650744" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.5.0/zig-macos-x86_64-0.5.0.tar.xz", + "shasum": "28702cc05745c7c0bd450487d5f4091bf0a1ad279b35eb9a640ce3e3a15b300d", + "size": "37898664" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.5.0/zig-windows-x86_64-0.5.0.zip", + "shasum": "58141323db8d84a5af62746be5f9140bc161ee760ef33dc91a887bf9ac021976", + "size": "44871804" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.5.0/zig-linux-x86_64-0.5.0.tar.xz", + "shasum": "43e8f8a8b8556edd373ddf9c1ef3ca6cf852d4d09fe07d5736d12fefedd2b4f7", + "size": "40895068" + } }, - "riscv64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.732+2f3234c76.tar.xz", - "shasum": "52e11b1de3b1ec121418228a8d627027e087fe0d2178a9952901acdec75c513d", - "size": "54400788" + "0.4.0": { + "date": "2019-04-08", + "docs": "https://ziglang.org/documentation/0.4.0/", + "notes": "https://ziglang.org/download/0.4.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.4.0/zig-0.4.0.tar.xz", + "shasum": "fec1f3f6b359a3d942e0a7f9157b3b30cde83927627a0e1ea95c54de3c526cfc", + "size": "5348776" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.4.0/zig-freebsd-x86_64-0.4.0.tar.xz", + "shasum": "3d557c91ac36d8262eb1733bb5f261c95944f9b635e43386e3d00a3272818c30", + "size": "27269672" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.4.0/zig-macos-x86_64-0.4.0.tar.xz", + "shasum": "67c932982484d017c5111e54af9f33f15e8e05c6bc5346a55e04052159c964a8", + "size": "30841504" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.4.0/zig-windows-x86_64-0.4.0.zip", + "shasum": "fbc3dd205e064c263063f69f600bedb18e3d0aa2efa747a63ef6cafb6d73f127", + "size": "35800101" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.4.0/zig-linux-x86_64-0.4.0.tar.xz", + "shasum": "fb1954e2fb556a01f8079a08130e88f70084e08978ff853bb2b1986d8c39d84e", + "size": "32876100" + } }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.732+2f3234c76.tar.xz", - "shasum": "a3b42e2719c9b20b0b53a028d4324a96ff6ab3da47c8fa7e3c46d9809397ef7b", - "size": "54620776" + "0.3.0": { + "date": "2018-09-28", + "docs": "https://ziglang.org/documentation/0.3.0/", + "notes": "https://ziglang.org/download/0.3.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.3.0/zig-0.3.0.tar.xz", + "shasum": "d70af604f3a8622f3393d93abb3e056bf60351e32d121e6fa4fe03d8d41e1f5a", + "size": "2335592" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.3.0/zig-macos-x86_64-0.3.0.tar.xz", + "shasum": "19dec1f1943ab7be26823376d466f7e456143deb34e17502778a949034dc2e7e", + "size": "23712696" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.3.0/zig-windows-x86_64-0.3.0.zip", + "shasum": "bb568c03950958f8bb3472139c3ab5ed74547c8c694ab50f404c202faf51baf4", + "size": "22524425" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.3.0/zig-linux-x86_64-0.3.0.tar.xz", + "shasum": "b378d0aae30cb54f28494e7bc4efbc9bfb6326f47bfb302e8b5287af777b2f3c", + "size": "25209304" + } }, - "aarch64-netbsd": { - "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.732+2f3234c76.tar.xz", - "shasum": "99e9b13e4b1c456dafcd7a72cde8c94b165cc162a1e6cf83f2330f4117527c56", - "size": "50150340" + "0.2.0": { + "date": "2018-03-15", + "docs": "https://ziglang.org/documentation/0.2.0/", + "notes": "https://ziglang.org/download/0.2.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.2.0/zig-0.2.0.tar.xz", + "shasum": "29c9beb172737f4d5019b88ceae829ae8bc6512fb4386cfbf895ae2b42aa6965", + "size": "1940832" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.2.0/zig-win64-0.2.0.zip", + "shasum": "4f8a2979941a1f081ec8e545cca0b72608c0db1c5a3fd377a94db40649dcd3d4", + "size": "21076274" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.2.0/zig-linux-x86_64-0.2.0.tar.xz", + "shasum": "209c6fb745d42474c0a73d6f291c7ae3a38b6a1b6b641eea285a7f840cc1a890", + "size": "22551928" + } }, - "arm-netbsd": { - "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.732+2f3234c76.tar.xz", - "shasum": "33734775600ac79a75382140963063bcf012d8ffe04f426ca17001405b857d10", - "size": "52805340" - }, - "x86-netbsd": { - "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.732+2f3234c76.tar.xz", - "shasum": "e5e8894b82ac787229cea9354bc2eb549292702e011ab9370d9c8825ad94f119", - "size": "57766016" - }, - "x86_64-netbsd": { - "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.732+2f3234c76.tar.xz", - "shasum": "48a9f23fe3b19fc0794ec18767a3500bd544c397913efd58fc8cd624601fccd5", - "size": "54555756" + "0.1.1": { + "date": "2017-10-17", + "docs": "https://ziglang.org/documentation/0.1.1/", + "notes": "https://ziglang.org/download/0.1.1/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.1.1/zig-0.1.1.tar.xz", + "shasum": "ffca0cfb263485287e19cc997b08701fcd5f24b700345bcdc3dd8074f5a104e0", + "size": "1659716" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.1.1/zig-win64-0.1.1.zip", + "shasum": "6fc88bef531af7e567fe30bf60da1487b86833cbee84c7a2f3e317030aa5b660", + "size": "19757776" + } } - }, - "0.15.2": { - "version": "0.15.2", - "date": "2025-10-11", - "docs": "https://ziglang.org/documentation/0.15.2/", - "stdDocs": "https://ziglang.org/documentation/0.15.2/std/", - "src": { - "tarball": "https://ziglang.org/download/0.15.2/zig-0.15.2.tar.xz", - "shasum": "d9b30c7aa983fcff5eed2084d54ae83eaafe7ff3a84d8fb754d854165a6e521c", - "size": "21366268" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.15.2/zig-bootstrap-0.15.2.tar.xz", - "shasum": "a6845459501df3c3264ebc587b02a7094ad14f4f3f7287c48f04457e784d0d85", - "size": "52718988" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-macos-0.15.2.tar.xz", - "shasum": "375b6909fc1495d16fc2c7db9538f707456bfc3373b14ee83fdd3e22b3d43f7f", - "size": "55800460" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-macos-0.15.2.tar.xz", - "shasum": "3cc2bab367e185cdfb27501c4b30b1b0653c28d9f73df8dc91488e66ece5fa6b", - "size": "50635984" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-linux-0.15.2.tar.xz", - "shasum": "02aa270f183da276e5b5920b1dac44a63f1a49e55050ebde3aecc9eb82f93239", - "size": "53733924" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-linux-0.15.2.tar.xz", - "shasum": "958ed7d1e00d0ea76590d27666efbf7a932281b3d7ba0c6b01b0ff26498f667f", - "size": "49471996" - }, - "arm-linux": { - "tarball": "https://ziglang.org/download/0.15.2/zig-arm-linux-0.15.2.tar.xz", - "shasum": "7d8401495065dae45d6249c68d5faf10508f8203c86362ccb698aeaafc66b7cd", - "size": "50454216" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.15.2/zig-riscv64-linux-0.15.2.tar.xz", - "shasum": "493512bdca485be3c6a9b0f69dcb4cbe4587f3af8e1be282fdd827108ba39930", - "size": "53613656" - }, - "powerpc64le-linux": { - "tarball": "https://ziglang.org/download/0.15.2/zig-powerpc64le-linux-0.15.2.tar.xz", - "shasum": "e182c5f8d30fc7f97d17d2ffef1488826aa3afaa51e5f0dbe14c597a98b45778", - "size": "53570632" - }, - "x86-linux": { - "tarball": "https://ziglang.org/download/0.15.2/zig-x86-linux-0.15.2.tar.xz", - "shasum": "4c6e23f39daa305e274197bfdff0d56ffd1750fc1de226ae10505c0eff52d7a5", - "size": "56334032" - }, - "loongarch64-linux": { - "tarball": "https://ziglang.org/download/0.15.2/zig-loongarch64-linux-0.15.2.tar.xz", - "shasum": "95d6a03e0415dbe30ae9c00d58b5446fd681ebf3d4044a8b707e1784c8074a61", - "size": "50800332" - }, - "s390x-linux": { - "tarball": "https://ziglang.org/download/0.15.2/zig-s390x-linux-0.15.2.tar.xz", - "shasum": "cd4b295f43b1b7ca3d603368caea82dbd00fce6691affb7527e607c0b6eafe8e", - "size": "53486436" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-windows-0.15.2.zip", - "shasum": "3a0ed1e8799a2f8ce2a6e6290a9ff22e6906f8227865911fb7ddedc3cc14cb0c", - "size": "92614574" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-windows-0.15.2.zip", - "shasum": "b926465f8872bf983422257cd9ec248bb2b270996fbe8d57872cca13b56fc370", - "size": "88464585" - }, - "x86-windows": { - "tarball": "https://ziglang.org/download/0.15.2/zig-x86-windows-0.15.2.zip", - "shasum": "7a6dfc00f4cc09ec46d3e10eb06f42538e92b6285e34debea7462edaf371da98", - "size": "94516930" - }, - "aarch64-freebsd": { - "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-freebsd-0.15.2.tar.xz", - "shasum": "c62efd319f86663eb7747709dfca259205edba8eaee98efc96a51ce40a9437de", - "size": "49363332" - }, - "arm-freebsd": { - "tarball": "https://ziglang.org/download/0.15.2/zig-arm-freebsd-0.15.2.tar.xz", - "shasum": "1d5d06428b2d4d31cf4e99008602f2096a9c652dd1025820a4aa66326fbf3dc4", - "size": "50900824" - }, - "powerpc64-freebsd": { - "tarball": "https://ziglang.org/download/0.15.2/zig-powerpc64-freebsd-0.15.2.tar.xz", - "shasum": "2bdb5a38069e667936d3378dfa332d67c21bb454cb17bea3c3b03faaa02a74af", - "size": "52099240" - }, - "powerpc64le-freebsd": { - "tarball": "https://ziglang.org/download/0.15.2/zig-powerpc64le-freebsd-0.15.2.tar.xz", - "shasum": "940894eb954c2077d0643f4deba8113239436fd0dcbc88160163b1b23fe3f035", - "size": "53459116" - }, - "riscv64-freebsd": { - "tarball": "https://ziglang.org/download/0.15.2/zig-riscv64-freebsd-0.15.2.tar.xz", - "shasum": "9aecdcf32fb63944c11389c10993aecd5e443a82d456693cc5ad230b5fe584ad", - "size": "53658752" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-freebsd-0.15.2.tar.xz", - "shasum": "5509ff57cd3f219165caed0da10221739af82742b9edfcda3f7bfaf4da7212dd", - "size": "53776208" - }, - "aarch64-netbsd": { - "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-netbsd-0.15.2.tar.xz", - "shasum": "357c1e62df6581ec1726546f9a827973233cb31ac371d2b0e92456b306a0c6c0", - "size": "49365256" - }, - "arm-netbsd": { - "tarball": "https://ziglang.org/download/0.15.2/zig-arm-netbsd-0.15.2.tar.xz", - "shasum": "1c43871791d49dbabd224da307b995076e493e34c36734cb21c4f6cbc208f8c3", - "size": "52021696" - }, - "x86-netbsd": { - "tarball": "https://ziglang.org/download/0.15.2/zig-x86-netbsd-0.15.2.tar.xz", - "shasum": "7f18fc4720426a6ca77bbfb7b970c31a61c37ce4a867b53830762958e921cdde", - "size": "56893368" - }, - "x86_64-netbsd": { - "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-netbsd-0.15.2.tar.xz", - "shasum": "11a3b9f4b66bece34a79c0e98e91136baf0ff9795ddf60e442ca6c8ad73b9204", - "size": "53778132" - } - }, - "0.15.1": { - "date": "2025-08-19", - "docs": "https://ziglang.org/documentation/0.15.1/", - "stdDocs": "https://ziglang.org/documentation/0.15.1/std/", - "notes": "https://ziglang.org/download/0.15.1/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.15.1/zig-0.15.1.tar.xz", - "shasum": "816c0303ab313f59766ce2097658c9fff7fafd1504f61f80f9507cd11652865f", - "size": "21359884" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.15.1/zig-bootstrap-0.15.1.tar.xz", - "shasum": "4c0cfbcf12da144955761ca43f89e3c74956bce978694fc1d0a63555f5c0a199", - "size": "52711548" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-macos-0.15.1.tar.xz", - "shasum": "9919392e0287cccc106dfbcbb46c7c1c3fa05d919567bb58d7eb16bca4116184", - "size": "55791880" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-macos-0.15.1.tar.xz", - "shasum": "c4bd624d901c1268f2deb9d8eb2d86a2f8b97bafa3f118025344242da2c54d7b", - "size": "50644996" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-linux-0.15.1.tar.xz", - "shasum": "c61c5da6edeea14ca51ecd5e4520c6f4189ef5250383db33d01848293bfafe05", - "size": "53734456" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-linux-0.15.1.tar.xz", - "shasum": "bb4a8d2ad735e7fba764c497ddf4243cb129fece4148da3222a7046d3f1f19fe", - "size": "49493872" - }, - "arm-linux": { - "tarball": "https://ziglang.org/download/0.15.1/zig-arm-linux-0.15.1.tar.xz", - "shasum": "3f4bf3b06b67d14e3f38be30798488c1abe3cf5b33de570cd0e87bbf09b978ad", - "size": "50477464" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.15.1/zig-riscv64-linux-0.15.1.tar.xz", - "shasum": "7ca7a3e621436fb31d66a253132fc39574a13d2a1b4d8458af4f2e7c6e4374fe", - "size": "53597792" - }, - "powerpc64le-linux": { - "tarball": "https://ziglang.org/download/0.15.1/zig-powerpc64le-linux-0.15.1.tar.xz", - "shasum": "339e2106496be70b614e32d444298216e676c36d08f5cd7bee3dd1dbd4567fd7", - "size": "53566944" - }, - "x86-linux": { - "tarball": "https://ziglang.org/download/0.15.1/zig-x86-linux-0.15.1.tar.xz", - "shasum": "dff166f25fdd06e8341d831a71211b5ba7411463a6b264bdefa8868438690b6a", - "size": "56311228" - }, - "loongarch64-linux": { - "tarball": "https://ziglang.org/download/0.15.1/zig-loongarch64-linux-0.15.1.tar.xz", - "shasum": "0af18a012f3c4cffbef29ab5f42021484d92517f921a1380faacc89c50c1f89d", - "size": "50794376" - }, - "s390x-linux": { - "tarball": "https://ziglang.org/download/0.15.1/zig-s390x-linux-0.15.1.tar.xz", - "shasum": "bcd13e5c88cf2d0da7100a48572195560069a94402c9bec3b316398204aa27e2", - "size": "53508068" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-windows-0.15.1.zip", - "shasum": "91e69e887ca8c943ce9a515df3af013d95a66a190a3df3f89221277ebad29e34", - "size": "92612958" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-windows-0.15.1.zip", - "shasum": "1f1bf16228b0ffcc882b713dc5e11a6db4219cb30997e13c72e8e723c2104ec6", - "size": "88458549" - }, - "x86-windows": { - "tarball": "https://ziglang.org/download/0.15.1/zig-x86-windows-0.15.1.zip", - "shasum": "fb1c07cffbb43615d3158ab8b8f5db5da1d48875eca99e1d7a8a0064ff63fc5b", - "size": "94516052" - }, - "aarch64-freebsd": { - "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-freebsd-0.15.1.tar.xz", - "shasum": "4d9d25c775828d49ea037b2284310c295d951793da8ebe94827a54fed4cca3ce", - "size": "49358464" - }, - "arm-freebsd": { - "tarball": "https://ziglang.org/download/0.15.1/zig-arm-freebsd-0.15.1.tar.xz", - "shasum": "9707f3a5f7e1a3d99c40db9a74de1acc61016a197ad289c2ad964f93cb213a18", - "size": "50904124" - }, - "powerpc64-freebsd": { - "tarball": "https://ziglang.org/download/0.15.1/zig-powerpc64-freebsd-0.15.1.tar.xz", - "shasum": "79448884372db04e62f77a46b92245fce805063e69534729537f75cb4681e7e3", - "size": "52099020" - }, - "powerpc64le-freebsd": { - "tarball": "https://ziglang.org/download/0.15.1/zig-powerpc64le-freebsd-0.15.1.tar.xz", - "shasum": "f18ee12ba9c98a20b8d2ad0410c679e7aa5591bc7917f169fd6b377833d2c7ad", - "size": "53480976" - }, - "riscv64-freebsd": { - "tarball": "https://ziglang.org/download/0.15.1/zig-riscv64-freebsd-0.15.1.tar.xz", - "shasum": "ee9f864a6fd8b57c1f4fdbb11daa06578746a6f8253afe3f5ddb5a76f2eddd2d", - "size": "53677800" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-freebsd-0.15.1.tar.xz", - "shasum": "9714f8ac3d3dc908b1599837c6167f857c1efaa930f0cfa840699458de7c3cd0", - "size": "53782112" - }, - "aarch64-netbsd": { - "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-netbsd-0.15.1.tar.xz", - "shasum": "b2a528399777583b85b89c54ccd45488af7709d6dd29a27323ec2a229db40910", - "size": "49368000" - }, - "arm-netbsd": { - "tarball": "https://ziglang.org/download/0.15.1/zig-arm-netbsd-0.15.1.tar.xz", - "shasum": "93dc70109cbf5d2e022d20dfb56211978c4ea3c0b1e67aaabff947d8d1583aab", - "size": "52028184" - }, - "x86-netbsd": { - "tarball": "https://ziglang.org/download/0.15.1/zig-x86-netbsd-0.15.1.tar.xz", - "shasum": "a91b26051822ff17f3143f859b87dce5b4a13e90928bd6daa6f07a895d3410f0", - "size": "56881864" - }, - "x86_64-netbsd": { - "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-netbsd-0.15.1.tar.xz", - "shasum": "6d7ba6eca5b4434351ebdb971b7303c9934514f9bb8481852251dbd5b52b03d6", - "size": "53794836" - } - }, - "0.14.1": { - "date": "2025-05-21", - "docs": "https://ziglang.org/documentation/0.14.1/", - "stdDocs": "https://ziglang.org/documentation/0.14.1/std/", - "src": { - "tarball": "https://ziglang.org/download/0.14.1/zig-0.14.1.tar.xz", - "shasum": "237f8abcc8c3fd68c70c66cdbf63dce4fb5ad4a2e6225ac925e3d5b4c388f203", - "size": "17787696" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.14.1/zig-bootstrap-0.14.1.tar.xz", - "shasum": "89b2fce50bfbb1eee29c382193d22c6eb0c7da3a96b5ba6d05e0af2945b3ca3d", - "size": "48041028" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.14.1/zig-x86_64-macos-0.14.1.tar.xz", - "shasum": "b0f8bdfb9035783db58dd6c19d7dea89892acc3814421853e5752fe4573e5f43", - "size": "51044512" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.14.1/zig-aarch64-macos-0.14.1.tar.xz", - "shasum": "39f3dc5e79c22088ce878edc821dedb4ca5a1cd9f5ef915e9b3cc3053e8faefa", - "size": "45903552" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.14.1/zig-x86_64-linux-0.14.1.tar.xz", - "shasum": "24aeeec8af16c381934a6cd7d95c807a8cb2cf7df9fa40d359aa884195c4716c", - "size": "49086504" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.14.1/zig-aarch64-linux-0.14.1.tar.xz", - "shasum": "f7a654acc967864f7a050ddacfaa778c7504a0eca8d2b678839c21eea47c992b", - "size": "44954692" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.14.1/zig-armv7a-linux-0.14.1.tar.xz", - "shasum": "1b34d9ecfaeb3b360e86c0bc233e1a8a2bbed2d40f2d4f20c12bde2128714324", - "size": "46137456" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.14.1/zig-riscv64-linux-0.14.1.tar.xz", - "shasum": "005f214f74dbafb7b4d8bd305f4e9d25048f711d9ec6fa7b3d4fca177e11b882", - "size": "48094380" - }, - "powerpc64le-linux": { - "tarball": "https://ziglang.org/download/0.14.1/zig-powerpc64le-linux-0.14.1.tar.xz", - "shasum": "15523e748efd9224ae164482e5cc1f8c11b035246b9749fb5b00758484f384a1", - "size": "48720956" - }, - "x86-linux": { - "tarball": "https://ziglang.org/download/0.14.1/zig-x86-linux-0.14.1.tar.xz", - "shasum": "4bce6347fa112247443cb0952c19e560d1f90b910506cf895fd07a7b8d1c4a76", - "size": "51643520" - }, - "loongarch64-linux": { - "tarball": "https://ziglang.org/download/0.14.1/zig-loongarch64-linux-0.14.1.tar.xz", - "shasum": "fdc344789be6becbe220bf5ad035156e653cc148f4da270188cbac87729f17e0", - "size": "45825404" - }, - "s390x-linux": { - "tarball": "https://ziglang.org/download/0.14.1/zig-s390x-linux-0.14.1.tar.xz", - "shasum": "8e1bb63532ba22725f0a4a45e5c2920a8af508d8ca81e6007c330eec2b7033a6", - "size": "55602032" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.14.1/zig-x86_64-windows-0.14.1.zip", - "shasum": "554f5378228923ffd558eac35e21af020c73789d87afeabf4bfd16f2e6feed2c", - "size": "82229343" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.14.1/zig-aarch64-windows-0.14.1.zip", - "shasum": "b5aac0ccc40dd91e8311b1f257717d8e3903b5fefb8f659de6d65a840ad1d0e7", - "size": "78125379" - }, - "x86-windows": { - "tarball": "https://ziglang.org/download/0.14.1/zig-x86-windows-0.14.1.zip", - "shasum": "3ee730c2a5523570dc4dc1b724f3e4f30174ebc1fa109ca472a719586a473b18", - "size": "83983932" - } - }, - "0.14.0": { - "date": "2025-03-05", - "docs": "https://ziglang.org/documentation/0.14.0/", - "stdDocs": "https://ziglang.org/documentation/0.14.0/std/", - "notes": "https://ziglang.org/download/0.14.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.14.0/zig-0.14.0.tar.xz", - "shasum": "c76638c03eb204c4432ae092f6fa07c208567e110fbd4d862d131a7332584046", - "size": "17772188" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.14.0/zig-bootstrap-0.14.0.tar.xz", - "shasum": "bf3fcb22be0b83f4791748adb567d3304779d66d7bf9b1bd557ef6c2e0232807", - "size": "48029040" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.14.0/zig-macos-x86_64-0.14.0.tar.xz", - "shasum": "685816166f21f0b8d6fc7aa6a36e91396dcd82ca6556dfbe3e329deffc01fec3", - "size": "51039964" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.14.0/zig-macos-aarch64-0.14.0.tar.xz", - "shasum": "b71e4b7c4b4be9953657877f7f9e6f7ee89114c716da7c070f4a238220e95d7e", - "size": "45902412" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.14.0/zig-linux-x86_64-0.14.0.tar.xz", - "shasum": "473ec26806133cf4d1918caf1a410f8403a13d979726a9045b421b685031a982", - "size": "49091960" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.14.0/zig-linux-aarch64-0.14.0.tar.xz", - "shasum": "ab64e3ea277f6fc5f3d723dcd95d9ce1ab282c8ed0f431b4de880d30df891e4f", - "size": "44922728" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.14.0/zig-linux-armv7a-0.14.0.tar.xz", - "shasum": "a67dbfa9bdf769228ec994f2098698c619f930883ca5ef638f50eee2d7788d10", - "size": "46112980" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.14.0/zig-linux-riscv64-0.14.0.tar.xz", - "shasum": "a2b14d3de326d3fd095548ef38bf5a67b15dadd62fbcc90836d63cc4355f8ef7", - "size": "48069188" - }, - "powerpc64le-linux": { - "tarball": "https://ziglang.org/download/0.14.0/zig-linux-powerpc64le-0.14.0.tar.xz", - "shasum": "3eabd60876ebc2748de8eb57b4b8cfa78861ba9bf7c6dd83f4e3e1d271d7c45e", - "size": "48707620" - }, - "x86-linux": { - "tarball": "https://ziglang.org/download/0.14.0/zig-linux-x86-0.14.0.tar.xz", - "shasum": "55d1ba21de5109686ffa675b9cc1dd66930093c202995a637ce3e397816e4c08", - "size": "51621460" - }, - "loongarch64-linux": { - "tarball": "https://ziglang.org/download/0.14.0/zig-linux-loongarch64-0.14.0.tar.xz", - "shasum": "31a2f07df55f8f528b92d540db9aae6c0b38643c34dc1ac33a0111d855e996ae", - "size": "45821860" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.14.0/zig-windows-x86_64-0.14.0.zip", - "shasum": "f53e5f9011ba20bbc3e0e6d0a9441b31eb227a97bac0e7d24172f1b8b27b4371", - "size": "82219809" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.14.0/zig-windows-aarch64-0.14.0.zip", - "shasum": "03e984383ebb8f85293557cfa9f48ee8698e7c400239570c9ff1aef3bffaf046", - "size": "78113283" - }, - "x86-windows": { - "tarball": "https://ziglang.org/download/0.14.0/zig-windows-x86-0.14.0.zip", - "shasum": "1a867d808cf4fa9184358395d94441390b6b24ee8d00d356ca11ea7cbfd3a4ec", - "size": "83970029" - } - }, - "0.13.0": { - "date": "2024-06-07", - "docs": "https://ziglang.org/documentation/0.13.0/", - "stdDocs": "https://ziglang.org/documentation/0.13.0/std/", - "notes": "https://ziglang.org/download/0.13.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.13.0/zig-0.13.0.tar.xz", - "shasum": "06c73596beeccb71cc073805bdb9c0e05764128f16478fa53bf17dfabc1d4318", - "size": "17220728" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.13.0/zig-bootstrap-0.13.0.tar.xz", - "shasum": "cd446c084b5da7bc42e8ad9b4e1c910a957f2bf3f82bcc02888102cd0827c139", - "size": "46440356" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.13.0/zig-freebsd-x86_64-0.13.0.tar.xz", - "shasum": "adc1ffc9be56533b2f1c7191f9e435ad55db00414ff2829d951ef63d95aaad8c", - "size": "47177744" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.13.0/zig-macos-x86_64-0.13.0.tar.xz", - "shasum": "8b06ed1091b2269b700b3b07f8e3be3b833000841bae5aa6a09b1a8b4773effd", - "size": "48857012" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.13.0/zig-macos-aarch64-0.13.0.tar.xz", - "shasum": "46fae219656545dfaf4dce12fb4e8685cec5b51d721beee9389ab4194d43394c", - "size": "44892040" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.13.0/zig-linux-x86_64-0.13.0.tar.xz", - "shasum": "d45312e61ebcc48032b77bc4cf7fd6915c11fa16e4aad116b66c9468211230ea", - "size": "47082308" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.13.0/zig-linux-aarch64-0.13.0.tar.xz", - "shasum": "041ac42323837eb5624068acd8b00cd5777dac4cf91179e8dad7a7e90dd0c556", - "size": "43090688" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.13.0/zig-linux-armv7a-0.13.0.tar.xz", - "shasum": "4b0550239c2cd884cc03ddeb2b9934708f4b073ad59a96fccbfe09f7e4f54233", - "size": "43998916" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.13.0/zig-linux-riscv64-0.13.0.tar.xz", - "shasum": "9f7f3c685894ff80f43eaf3cad1598f4844ac46f4308374237c7f912f7907bb3", - "size": "45540956" - }, - "powerpc64le-linux": { - "tarball": "https://ziglang.org/download/0.13.0/zig-linux-powerpc64le-0.13.0.tar.xz", - "shasum": "6a467622448e830e8f85d20cabed151498af2b0a62f87b8c083b2fe127e60417", - "size": "46574596" - }, - "x86-linux": { - "tarball": "https://ziglang.org/download/0.13.0/zig-linux-x86-0.13.0.tar.xz", - "shasum": "876159cc1e15efb571e61843b39a2327f8925951d48b9a7a03048c36f72180f7", - "size": "52062336" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.13.0/zig-windows-x86_64-0.13.0.zip", - "shasum": "d859994725ef9402381e557c60bb57497215682e355204d754ee3df75ee3c158", - "size": "79163968" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.13.0/zig-windows-aarch64-0.13.0.zip", - "shasum": "95ff88427af7ba2b4f312f45d2377ce7a033e5e3c620c8caaa396a9aba20efda", - "size": "75119033" - }, - "x86-windows": { - "tarball": "https://ziglang.org/download/0.13.0/zig-windows-x86-0.13.0.zip", - "shasum": "eb3d533c3cf868bff7e74455dc005d18fd836c42e50b27106b31e9fec6dffc4a", - "size": "83274739" - } - }, - "0.12.1": { - "date": "2024-06-08", - "docs": "https://ziglang.org/documentation/0.12.1/", - "stdDocs": "https://ziglang.org/documentation/0.12.1/std/", - "src": { - "tarball": "https://ziglang.org/download/0.12.1/zig-0.12.1.tar.xz", - "shasum": "cca0bf5686fe1a15405bd535661811fac7663f81664d2204ea4590ce49a6e9ba", - "size": "17110932" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.12.1/zig-bootstrap-0.12.1.tar.xz", - "shasum": "e533e2cb6ef60edda0ae3f2ca5c6504557db6e985e3c3a80159beb32279ed341", - "size": "45542004" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.12.1/zig-freebsd-x86_64-0.12.1.tar.xz", - "shasum": "30eaa28fa7bc21d01f88528d75ae4b392ae4970406675d5ac712a4937a605123", - "size": "45590080" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.12.1/zig-macos-x86_64-0.12.1.tar.xz", - "shasum": "68f309c6e431d56eb42648d7fe86e8028a23464d401a467831e27c26f1a8d9c9", - "size": "47202232" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.12.1/zig-macos-aarch64-0.12.1.tar.xz", - "shasum": "6587860dbbc070e1ee069e1a3d18ced83b7ba7a80bf67b2c57caf7c9ce5208b1", - "size": "43451512" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.12.1/zig-linux-x86_64-0.12.1.tar.xz", - "shasum": "8860fc9725c2d9297a63008f853e9b11e3c5a2441217f99c1e3104cc6fa4a443", - "size": "45512024" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.12.1/zig-linux-aarch64-0.12.1.tar.xz", - "shasum": "27d4fef393e8d8b5f3b1d19f4dd43bfdb469b4ed17bbc4c2283c1b1fe650ef7f", - "size": "41867324" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.12.1/zig-linux-armv7a-0.12.1.tar.xz", - "shasum": "27493c922fd1454137ed6cbe6b6bec00352838fcd32e6e74f4f9187011816157", - "size": "42665508" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.12.1/zig-linux-riscv64-0.12.1.tar.xz", - "shasum": "463511a863acc16911cff6801de97623e6de296aab7b15dcda5f6fd078b400b5", - "size": "43932908" - }, - "powerpc64le-linux": { - "tarball": "https://ziglang.org/download/0.12.1/zig-linux-powerpc64le-0.12.1.tar.xz", - "shasum": "462d6f10350f3c5d6fc4c9d6cfdede93d69d0103af026889a15f65c5de791d39", - "size": "45227716" - }, - "x86-linux": { - "tarball": "https://ziglang.org/download/0.12.1/zig-linux-x86-0.12.1.tar.xz", - "shasum": "c36ac019ca0fc3167e50d17e2affd3d072a06c519761737d0639adfdf2dcfddd", - "size": "50555428" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.12.1/zig-windows-x86_64-0.12.1.zip", - "shasum": "52459b147c2de4d7c28f6b1a4b3d571c114e96836bf8e31c953a7d2f5e94251c", - "size": "76470574" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.12.1/zig-windows-aarch64-0.12.1.zip", - "shasum": "e1286114a11be4695a6ad5cf0ba6a0e5f489bb3b029a5237de93598133f0c13a", - "size": "72998386" - }, - "x86-windows": { - "tarball": "https://ziglang.org/download/0.12.1/zig-windows-x86-0.12.1.zip", - "shasum": "4f0cc9258527e7b8bcf742772b3069122086a5cd857b38a1c08002462ac81f80", - "size": "80979711" - } - }, - "0.12.0": { - "date": "2024-04-20", - "docs": "https://ziglang.org/documentation/0.12.0/", - "stdDocs": "https://ziglang.org/documentation/0.12.0/std/", - "notes": "https://ziglang.org/download/0.12.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.12.0/zig-0.12.0.tar.xz", - "shasum": "a6744ef84b6716f976dad923075b2f54dc4f785f200ae6c8ea07997bd9d9bd9a", - "size": "17099152" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.12.0/zig-bootstrap-0.12.0.tar.xz", - "shasum": "3efc643d56421fa68072af94d5512cb71c61acf1c32512f77c0b4590bff63187", - "size": "45527312" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.12.0/zig-freebsd-x86_64-0.12.0.tar.xz", - "shasum": "bd49957d1157850b337ee1cf3c00af83585cff98e1ebc3c524a267e7422a2d7b", - "size": "45578364" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.12.0/zig-macos-x86_64-0.12.0.tar.xz", - "shasum": "4d411bf413e7667821324da248e8589278180dbc197f4f282b7dbb599a689311", - "size": "47185720" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.12.0/zig-macos-aarch64-0.12.0.tar.xz", - "shasum": "294e224c14fd0822cfb15a35cf39aa14bd9967867999bf8bdfe3db7ddec2a27f", - "size": "43447724" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.12.0/zig-linux-x86_64-0.12.0.tar.xz", - "shasum": "c7ae866b8a76a568e2d5cfd31fe89cdb629bdd161fdd5018b29a4a0a17045cad", - "size": "45480516" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.12.0/zig-linux-aarch64-0.12.0.tar.xz", - "shasum": "754f1029484079b7e0ca3b913a0a2f2a6afd5a28990cb224fe8845e72f09de63", - "size": "41849060" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.12.0/zig-linux-armv7a-0.12.0.tar.xz", - "shasum": "b48221f4c64416d257f0f9f77d8727dccf7de92aeabe59744ee6e70d650a97bc", - "size": "42638808" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.12.0/zig-linux-riscv64-0.12.0.tar.xz", - "shasum": "bb2d1a78b01595a9c00ffd2e12ab46e32f8b6798f76aec643ff78e5b4f5c5afd", - "size": "43917444" - }, - "powerpc64le-linux": { - "tarball": "https://ziglang.org/download/0.12.0/zig-linux-powerpc64le-0.12.0.tar.xz", - "shasum": "9218beecfb9250e9eff863f58f987dca7077e3258dd263c40269086127f9679b", - "size": "45216736" - }, - "x86-linux": { - "tarball": "https://ziglang.org/download/0.12.0/zig-linux-x86-0.12.0.tar.xz", - "shasum": "fb752fceb88749a80d625a6efdb23bea8208962b5150d6d14c92d20efda629a5", - "size": "50498940" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.12.0/zig-windows-x86_64-0.12.0.zip", - "shasum": "2199eb4c2000ddb1fba85ba78f1fcf9c1fb8b3e57658f6a627a8e513131893f5", - "size": "76442958" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.12.0/zig-windows-aarch64-0.12.0.zip", - "shasum": "04c6b92689241ca7a8a59b5f12d2ca2820c09d5043c3c4808b7e93e41c7bf97b", - "size": "72976876" - }, - "x86-windows": { - "tarball": "https://ziglang.org/download/0.12.0/zig-windows-x86-0.12.0.zip", - "shasum": "497dc9fd415cadf948872f137d6cc0870507488f79db9547b8f2adb73cda9981", - "size": "80950440" - } - }, - "0.11.0": { - "date": "2023-08-04", - "docs": "https://ziglang.org/documentation/0.11.0/", - "stdDocs": "https://ziglang.org/documentation/0.11.0/std/", - "notes": "https://ziglang.org/download/0.11.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.11.0/zig-0.11.0.tar.xz", - "shasum": "72014e700e50c0d3528cef3adf80b76b26ab27730133e8202716a187a799e951", - "size": "15275316" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.11.0/zig-bootstrap-0.11.0.tar.xz", - "shasum": "38dd9e17433c7ce5687c48fa0a757462cbfcbe75d9d5087d14ebbe00efd21fdc", - "size": "43227592" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.11.0/zig-freebsd-x86_64-0.11.0.tar.xz", - "shasum": "ea430327f9178377b79264a1d492868dcff056cd76d43a6fb00719203749e958", - "size": "46432140" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.11.0/zig-macos-x86_64-0.11.0.tar.xz", - "shasum": "1c1c6b9a906b42baae73656e24e108fd8444bb50b6e8fd03e9e7a3f8b5f05686", - "size": "47189164" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.11.0/zig-macos-aarch64-0.11.0.tar.xz", - "shasum": "c6ebf927bb13a707d74267474a9f553274e64906fd21bf1c75a20bde8cadf7b2", - "size": "43855096" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.11.0/zig-linux-x86_64-0.11.0.tar.xz", - "shasum": "2d00e789fec4f71790a6e7bf83ff91d564943c5ee843c5fd966efc474b423047", - "size": "44961892" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.11.0/zig-linux-aarch64-0.11.0.tar.xz", - "shasum": "956eb095d8ba44ac6ebd27f7c9956e47d92937c103bf754745d0a39cdaa5d4c6", - "size": "41492432" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.11.0/zig-linux-armv7a-0.11.0.tar.xz", - "shasum": "aebe8bbeca39f13f9b7304465f9aee01ab005d243836bd40f4ec808093dccc9b", - "size": "42240664" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.11.0/zig-linux-riscv64-0.11.0.tar.xz", - "shasum": "24a478937eddb507e96d60bd4da00de9092b3f0920190eb45c4c99c946b00ed5", - "size": "43532324" - }, - "powerpc64le-linux": { - "tarball": "https://ziglang.org/download/0.11.0/zig-linux-powerpc64le-0.11.0.tar.xz", - "shasum": "75260e87325e820a278cf9e74f130c7b3d84c0b5197afb2e3c85eff3fcedd48d", - "size": "44656184" - }, - "powerpc-linux": { - "tarball": "https://ziglang.org/download/0.11.0/zig-linux-powerpc-0.11.0.tar.xz", - "shasum": "70a5f9668a66fb2a91a7c3488b15bcb568e1f9f44b95cd10075c138ad8c42864", - "size": "44539972" - }, - "x86-linux": { - "tarball": "https://ziglang.org/download/0.11.0/zig-linux-x86-0.11.0.tar.xz", - "shasum": "7b0dc3e0e070ae0e0d2240b1892af6a1f9faac3516cae24e57f7a0e7b04662a8", - "size": "49824456" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.11.0/zig-windows-x86_64-0.11.0.zip", - "shasum": "142caa3b804d86b4752556c9b6b039b7517a08afa3af842645c7e2dcd125f652", - "size": "77216743" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.11.0/zig-windows-aarch64-0.11.0.zip", - "shasum": "5d4bd13db5ecb0ddc749231e00f125c1d31087d708e9ff9b45c4f4e13e48c661", - "size": "73883137" - }, - "x86-windows": { - "tarball": "https://ziglang.org/download/0.11.0/zig-windows-x86-0.11.0.zip", - "shasum": "e72b362897f28c671633e650aa05289f2e62b154efcca977094456c8dac3aefa", - "size": "81576961" - } - }, - "0.10.1": { - "date": "2023-01-19", - "docs": "https://ziglang.org/documentation/0.10.1/", - "stdDocs": "https://ziglang.org/documentation/0.10.1/std/", - "notes": "https://ziglang.org/download/0.10.1/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.10.1/zig-0.10.1.tar.xz", - "shasum": "69459bc804333df077d441ef052ffa143d53012b655a51f04cfef1414c04168c", - "size": "15143112" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.10.1/zig-bootstrap-0.10.1.tar.xz", - "shasum": "9f5781210b9be8f832553d160851635780f9bd71816065351ab29cfd8968f5e9", - "size": "43971816" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.10.1/zig-macos-x86_64-0.10.1.tar.xz", - "shasum": "02483550b89d2a3070c2ed003357fd6e6a3059707b8ee3fbc0c67f83ca898437", - "size": "45119596" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.10.1/zig-macos-aarch64-0.10.1.tar.xz", - "shasum": "b9b00477ec5fa1f1b89f35a7d2a58688e019910ab80a65eac2a7417162737656", - "size": "40517896" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.10.1/zig-linux-x86_64-0.10.1.tar.xz", - "shasum": "6699f0e7293081b42428f32c9d9c983854094bd15fee5489f12c4cf4518cc380", - "size": "44085596" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.10.1/zig-linux-aarch64-0.10.1.tar.xz", - "shasum": "db0761664f5f22aa5bbd7442a1617dd696c076d5717ddefcc9d8b95278f71f5d", - "size": "40321280" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.10.1/zig-linux-riscv64-0.10.1.tar.xz", - "shasum": "9db5b59a5112b8beb995094ba800e88b0060e9cf7cfadf4dc3e666c9010dc77b", - "size": "42196008" - }, - "i386-linux": { - "tarball": "https://ziglang.org/download/0.10.1/zig-linux-i386-0.10.1.tar.xz", - "shasum": "8c710ca5966b127b0ee3efba7310601ee57aab3dd6052a082ebc446c5efb2316", - "size": "48367388" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.10.1/zig-windows-x86_64-0.10.1.zip", - "shasum": "5768004e5e274c7969c3892e891596e51c5df2b422d798865471e05049988125", - "size": "73259729" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.10.1/zig-windows-aarch64-0.10.1.zip", - "shasum": "ece93b0d77b2ab03c40db99ef7ccbc63e0b6bd658af12b97898960f621305428", - "size": "69417459" - } - }, - "0.10.0": { - "date": "2022-10-31", - "docs": "https://ziglang.org/documentation/0.10.0/", - "stdDocs": "https://ziglang.org/documentation/0.10.0/std/", - "notes": "https://ziglang.org/download/0.10.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.10.0/zig-0.10.0.tar.xz", - "shasum": "d8409f7aafc624770dcd050c8fa7e62578be8e6a10956bca3c86e8531c64c136", - "size": "14530912" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.10.0/zig-bootstrap-0.10.0.tar.xz", - "shasum": "c13dc70c4ff4c09f749adc0d473cbd3942991dd4d1bd2d860fbf257d8c1bbabf", - "size": "45625516" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.10.0/zig-freebsd-x86_64-0.10.0.tar.xz", - "shasum": "dd77afa2a8676afbf39f7d6068eda81b0723afd728642adaac43cb2106253d65", - "size": "44056504" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.10.0/zig-linux-aarch64-0.10.0.tar.xz", - "shasum": "09ef50c8be73380799804169197820ee78760723b0430fa823f56ed42b06ea0f", - "size": "40387688" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.10.0/zig-linux-armv7a-0.10.0.tar.xz", - "shasum": "7201b2e89cd7cc2dde95d39485fd7d5641ba67dc6a9a58c036cb4c308d2e82de", - "size": "50805936" - }, - "i386-linux": { - "tarball": "https://ziglang.org/download/0.10.0/zig-linux-i386-0.10.0.tar.xz", - "shasum": "dac8134f1328c50269f3e50b334298ec7916cb3b0ef76927703ddd1c96fd0115", - "size": "48451732" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.10.0/zig-linux-riscv64-0.10.0.tar.xz", - "shasum": "2a126f3401a7a7efc4b454f0a85c133db1af5a9dfee117f172213b7cbd47bfba", - "size": "42272968" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.10.0/zig-linux-x86_64-0.10.0.tar.xz", - "shasum": "631ec7bcb649cd6795abe40df044d2473b59b44e10be689c15632a0458ddea55", - "size": "44142400" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.10.0/zig-macos-aarch64-0.10.0.tar.xz", - "shasum": "02f7a7839b6a1e127eeae22ea72c87603fb7298c58bc35822a951479d53c7557", - "size": "40602664" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.10.0/zig-macos-x86_64-0.10.0.tar.xz", - "shasum": "3a22cb6c4749884156a94ea9b60f3a28cf4e098a69f08c18fbca81c733ebfeda", - "size": "45175104" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.10.0/zig-windows-x86_64-0.10.0.zip", - "shasum": "a66e2ff555c6e48781de1bcb0662ef28ee4b88af3af2a577f7b1950e430897ee", - "size": "73181558" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.10.0/zig-windows-aarch64-0.10.0.zip", - "shasum": "1bbda8d123d44f3ae4fa90d0da04b1e9093c3f9ddae3429a4abece1e1c0bf19a", - "size": "69332389" - } - }, - "0.9.1": { - "date": "2022-02-14", - "docs": "https://ziglang.org/documentation/0.9.1/", - "stdDocs": "https://ziglang.org/documentation/0.9.1/std/", - "notes": "https://ziglang.org/download/0.9.1/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.9.1/zig-0.9.1.tar.xz", - "shasum": "38cf4e84481f5facc766ba72783e7462e08d6d29a5d47e3b75c8ee3142485210", - "size": "13940828" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.9.1/zig-bootstrap-0.9.1.tar.xz", - "shasum": "0a8e221c71860d8975c15662b3ed3bd863e81c4fe383455a596e5e0e490d6109", - "size": "42488812" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.9.1/zig-freebsd-x86_64-0.9.1.tar.xz", - "shasum": "4e06009bd3ede34b72757eec1b5b291b30aa0d5046dadd16ecb6b34a02411254", - "size": "39028848" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.9.1/zig-linux-aarch64-0.9.1.tar.xz", - "shasum": "5d99a39cded1870a3fa95d4de4ce68ac2610cca440336cfd252ffdddc2b90e66", - "size": "37034860" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.9.1/zig-linux-armv7a-0.9.1.tar.xz", - "shasum": "6de64456cb4757a555816611ea697f86fba7681d8da3e1863fa726a417de49be", - "size": "37974652" - }, - "i386-linux": { - "tarball": "https://ziglang.org/download/0.9.1/zig-linux-i386-0.9.1.tar.xz", - "shasum": "e776844fecd2e62fc40d94718891057a1dbca1816ff6013369e9a38c874374ca", - "size": "44969172" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.9.1/zig-linux-riscv64-0.9.1.tar.xz", - "shasum": "208dea53662c2c52777bd9e3076115d2126a4f71aed7f2ff3b8fe224dc3881aa", - "size": "39390868" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.9.1/zig-linux-x86_64-0.9.1.tar.xz", - "shasum": "be8da632c1d3273f766b69244d80669fe4f5e27798654681d77c992f17c237d7", - "size": "41011464" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.9.1/zig-macos-aarch64-0.9.1.tar.xz", - "shasum": "8c473082b4f0f819f1da05de2dbd0c1e891dff7d85d2c12b6ee876887d438287", - "size": "38995640" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.9.1/zig-macos-x86_64-0.9.1.tar.xz", - "shasum": "2d94984972d67292b55c1eb1c00de46580e9916575d083003546e9a01166754c", - "size": "43713044" - }, - "i386-windows": { - "tarball": "https://ziglang.org/download/0.9.1/zig-windows-i386-0.9.1.zip", - "shasum": "74a640ed459914b96bcc572183a8db687bed0af08c30d2ea2f8eba03ae930f69", - "size": "67929868" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.9.1/zig-windows-x86_64-0.9.1.zip", - "shasum": "443da53387d6ae8ba6bac4b3b90e9fef4ecbe545e1c5fa3a89485c36f5c0e3a2", - "size": "65047697" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.9.1/zig-windows-aarch64-0.9.1.zip", - "shasum": "621bf95f54dc3ff71466c5faae67479419951d7489e40e87fd26d195825fb842", - "size": "61478151" - } - }, - "0.9.0": { - "date": "2021-12-20", - "docs": "https://ziglang.org/documentation/0.9.0/", - "stdDocs": "https://ziglang.org/documentation/0.9.0/std/", - "notes": "https://ziglang.org/download/0.9.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.9.0/zig-0.9.0.tar.xz", - "shasum": "cd1be83b12f8269cc5965e59877b49fdd8fa638efb6995ac61eb4cea36a2e381", - "size": "13928772" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.9.0/zig-bootstrap-0.9.0.tar.xz", - "shasum": "16b0bdf0bc0a5ed1e0950e08481413d806192e06443a512347526647b2baeabc", - "size": "42557736" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.9.0/zig-freebsd-x86_64-0.9.0.tar.xz", - "shasum": "c95afe679b7cc4110dc2ecd3606c83a699718b7a958d6627f74c20886333e194", - "size": "41293236" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.9.0/zig-linux-aarch64-0.9.0.tar.xz", - "shasum": "1524fedfdbade2dbc9bae1ed98ad38fa7f2114c9a3e94da0d652573c75efbc5a", - "size": "40008396" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.9.0/zig-linux-armv7a-0.9.0.tar.xz", - "shasum": "50225dee6e6448a63ee96383a34d9fe3bba34ae8da1a0c8619bde2cdfc1df87d", - "size": "41196876" - }, - "i386-linux": { - "tarball": "https://ziglang.org/download/0.9.0/zig-linux-i386-0.9.0.tar.xz", - "shasum": "b0dcf688349268c883292acdd55eaa3c13d73b9146e4b990fad95b84a2ac528b", - "size": "47408656" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.9.0/zig-linux-riscv64-0.9.0.tar.xz", - "shasum": "85466de07504767ed37f59782672ad41bbdf43d6480fafd07f45543278b07620", - "size": "44171420" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.9.0/zig-linux-x86_64-0.9.0.tar.xz", - "shasum": "5c55344a877d557fb1b28939785474eb7f4f2f327aab55293998f501f7869fa6", - "size": "43420796" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.9.0/zig-macos-aarch64-0.9.0.tar.xz", - "shasum": "3991c70594d61d09fb4b316157a7c1d87b1d4ec159e7a5ecd11169ff74cad832", - "size": "39013392" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.9.0/zig-macos-x86_64-0.9.0.tar.xz", - "shasum": "c5280eeec4d6e5ea5ce5b448dc9a7c4bdd85ecfed4c1b96aa0835e48b36eccf0", - "size": "43764596" - }, - "i386-windows": { - "tarball": "https://ziglang.org/download/0.9.0/zig-windows-i386-0.9.0.zip", - "shasum": "bb839434afc75092015cf4c33319d31463c18512bc01dd719aedf5dcbc368466", - "size": "67946715" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.9.0/zig-windows-x86_64-0.9.0.zip", - "shasum": "084ea2646850aaf068234b0f1a92b914ed629be47075e835f8a67d55c21d880e", - "size": "65045849" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.9.0/zig-windows-aarch64-0.9.0.zip", - "shasum": "f9018725e3fb2e8992b17c67034726971156eb190685018a9ac8c3a9f7a22340", - "size": "61461921" - } - }, - "0.8.1": { - "date": "2021-09-06", - "docs": "https://ziglang.org/documentation/0.8.1/", - "stdDocs": "https://ziglang.org/documentation/0.8.1/std/", - "notes": "https://ziglang.org/download/0.8.1/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.8.1/zig-0.8.1.tar.xz", - "shasum": "8c428e14a0a89cb7a15a6768424a37442292858cdb695e2eb503fa3c7bf47f1a", - "size": "12650228" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.8.1/zig-bootstrap-0.8.1.tar.xz", - "shasum": "fa1239247f830ecd51c42537043f5220e4d1dfefdc54356fa419616a0efb3902", - "size": "43613464" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.8.1/zig-freebsd-x86_64-0.8.1.tar.xz", - "shasum": "fc4f6478bcf3a9fce1b8ef677a91694f476dd35be6d6c9c4f44a8b76eedbe176", - "size": "39150924" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.8.1/zig-linux-aarch64-0.8.1.tar.xz", - "shasum": "2166dc9f2d8df387e8b4122883bb979d739281e1ff3f3d5483fec3a23b957510", - "size": "37605932" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.8.1/zig-linux-armv7a-0.8.1.tar.xz", - "shasum": "5ba58141805e2519f38cf8e715933cbf059f4f3dade92c71838cce341045de05", - "size": "39185876" - }, - "i386-linux": { - "tarball": "https://ziglang.org/download/0.8.1/zig-linux-i386-0.8.1.tar.xz", - "shasum": "2f3e84f30492b5f1c5f97cecc0166f07a8a8d50c5f85dbb3a6ef2a4ee6f915e6", - "size": "44782932" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.8.1/zig-linux-riscv64-0.8.1.tar.xz", - "shasum": "4adfaf147b025917c03367462fe5018aaa9edbc6439ef9cd0da2b074ae960554", - "size": "41234480" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.8.1/zig-linux-x86_64-0.8.1.tar.xz", - "shasum": "6c032fc61b5d77a3f3cf781730fa549f8f059ffdb3b3f6ad1c2994d2b2d87983", - "size": "41250060" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.8.1/zig-macos-aarch64-0.8.1.tar.xz", - "shasum": "5351297e3b8408213514b29c0a938002c5cf9f97eee28c2f32920e1227fd8423", - "size": "35340712" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.8.1/zig-macos-x86_64-0.8.1.tar.xz", - "shasum": "16b0e1defe4c1807f2e128f72863124bffdd906cefb21043c34b673bf85cd57f", - "size": "39946200" - }, - "i386-windows": { - "tarball": "https://ziglang.org/download/0.8.1/zig-windows-i386-0.8.1.zip", - "shasum": "099605051eb0452a947c8eab8fbbc7e43833c8376d267e94e41131c289a1c535", - "size": "64152358" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.8.1/zig-windows-x86_64-0.8.1.zip", - "shasum": "43573db14cd238f7111d6bdf37492d363f11ecd1eba802567a172f277d003926", - "size": "61897838" - } - }, - "0.8.0": { - "date": "2021-06-04", - "docs": "https://ziglang.org/documentation/0.8.0/", - "stdDocs": "https://ziglang.org/documentation/0.8.0/std/", - "notes": "https://ziglang.org/download/0.8.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.8.0/zig-0.8.0.tar.xz", - "shasum": "03a828d00c06b2e3bb8b7ff706997fd76bf32503b08d759756155b6e8c981e77", - "size": "12614896" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.8.0/zig-bootstrap-0.8.0.tar.xz", - "shasum": "10600bc9c01f92e343f40d6ecc0ad05d67d27c3e382bce75524c0639cd8ca178", - "size": "43574248" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.8.0/zig-freebsd-x86_64-0.8.0.tar.xz", - "shasum": "0d3ccc436c8c0f50fd55462f72f8492d98723c7218ffc2a8a1831967d81b4bdc", - "size": "39125332" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.8.0/zig-linux-aarch64-0.8.0.tar.xz", - "shasum": "ee204ca2c2037952cf3f8b10c609373a08a291efa4af7b3c73be0f2b27720470", - "size": "37575428" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.8.0/zig-linux-armv7a-0.8.0.tar.xz", - "shasum": "d00b8bd97b79f45d6f5da956983bafeaa082e6c2ae8c6e1c6d4faa22fa29b320", - "size": "38884212" - }, - "i386-linux": { - "tarball": "https://ziglang.org/download/0.8.0/zig-linux-i386-0.8.0.tar.xz", - "shasum": "96e43ee6ed81c3c63401f456bd1c58ee6d42373a43cb324f5cf4974ca0998865", - "size": "42136032" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.8.0/zig-linux-riscv64-0.8.0.tar.xz", - "shasum": "75997527a78cdab64c40c43d9df39c01c4cdb557bb3992a869838371a204cfea", - "size": "40016268" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.8.0/zig-linux-x86_64-0.8.0.tar.xz", - "shasum": "502625d3da3ae595c5f44a809a87714320b7a40e6dff4a895b5fa7df3391d01e", - "size": "41211184" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.8.0/zig-macos-aarch64-0.8.0.tar.xz", - "shasum": "b32d13f66d0e1ff740b3326d66a469ee6baddbd7211fa111c066d3bd57683111", - "size": "35292180" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.8.0/zig-macos-x86_64-0.8.0.tar.xz", - "shasum": "279f9360b5cb23103f0395dc4d3d0d30626e699b1b4be55e98fd985b62bc6fbe", - "size": "39969312" - }, - "i386-windows": { - "tarball": "https://ziglang.org/download/0.8.0/zig-windows-i386-0.8.0.zip", - "shasum": "b6ec9aa6cd6f3872fcb30d43ff411802d82008a0c4142ee49e208a09b2c1c5fe", - "size": "61507213" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.8.0/zig-windows-x86_64-0.8.0.zip", - "shasum": "8580fbbf3afb72e9b495c7f8aeac752a03475ae0bbcf5d787f3775c7e1f4f807", - "size": "61766193" - } - }, - "0.7.1": { - "date": "2020-12-13", - "docs": "https://ziglang.org/documentation/0.7.1/", - "stdDocs": "https://ziglang.org/documentation/0.7.1/std/", - "notes": "https://ziglang.org/download/0.7.1/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.7.1/zig-0.7.1.tar.xz", - "shasum": "2db3b944ab368d955b48743d9f7c963b8f96de1a441ba5a35e197237cc6dae44", - "size": "10711824" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.7.1/zig-bootstrap-0.7.1.tar.xz", - "shasum": "040f27c1fae4b0cac0a2782aecdb691f6a2f8e89db6a6ed35024c31c304fd9b2", - "size": "40232612" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.7.1/zig-freebsd-x86_64-0.7.1.tar.xz", - "shasum": "e73c1dca35791a3183fdd5ecde0443ebbe180942efceafe651886034fb8def09", - "size": "39066808" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.7.1/zig-linux-aarch64-0.7.1.tar.xz", - "shasum": "48ec90eba407e4587ddef7eecef25fec7e13587eb98e3b83c5f2f5fff2a5cbe7", - "size": "33780552" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.7.1/zig-linux-armv7a-0.7.1.tar.xz", - "shasum": "5a0662e07b4c4968665e1f97558f8591f6facec45d2e0ff5715e661743107ceb", - "size": "35813504" - }, - "i386-linux": { - "tarball": "https://ziglang.org/download/0.7.1/zig-linux-i386-0.7.1.tar.xz", - "shasum": "4882e052e5f83690bd0334bb4fc1702b5403cb3a3d2aa63fd7d6043d8afecba3", - "size": "39230912" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.7.1/zig-linux-riscv64-0.7.1.tar.xz", - "shasum": "187294bfd35983348c3fe042901b42e67e7e36ab7f77a5f969d21c0051f4d21f", - "size": "37454812" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.7.1/zig-linux-x86_64-0.7.1.tar.xz", - "shasum": "18c7b9b200600f8bcde1cd8d7f1f578cbc3676241ce36d771937ce19a8159b8d", - "size": "37848176" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.7.1/zig-macos-x86_64-0.7.1.tar.xz", - "shasum": "845cb17562978af0cf67e3993f4e33330525eaf01ead9386df9105111e3bc519", - "size": "36211076" - }, - "i386-windows": { - "tarball": "https://ziglang.org/download/0.7.1/zig-windows-i386-0.7.1.zip", - "shasum": "a1b9a7421e13153e07fd2e2c93ff29aad64d83105b8fcdafa633dbe689caf1c0", - "size": "54374983" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.7.1/zig-windows-x86_64-0.7.1.zip", - "shasum": "4818a8a65b4672bc52c0ae7f14d014e0eb8caf10f12c0745176820384cea296a", - "size": "54909997" - } - }, - "0.7.0": { - "date": "2020-11-08", - "docs": "https://ziglang.org/documentation/0.7.0/", - "stdDocs": "https://ziglang.org/documentation/0.7.0/std/", - "notes": "https://ziglang.org/download/0.7.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.7.0/zig-0.7.0.tar.xz", - "shasum": "0efd2cf6c3b05723db80e9cf193bc55150bba84ca41f855a90f53fc756445f83", - "size": "10683920" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.7.0/zig-bootstrap-0.7.0.tar.xz", - "shasum": "f073beaf5c53c8c57c0d374cbfcb332ef92ad703173edba0d9e0f2ed28401b72", - "size": "40200436" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.7.0/zig-freebsd-x86_64-0.7.0.tar.xz", - "shasum": "a0c926272ee4ae720034b4a6a1dc98399d76156dd84182554740f0ca8a41fc99", - "size": "34798992" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.7.0/zig-linux-aarch64-0.7.0.tar.xz", - "shasum": "f89933bac87d44be82325754ff88423020c81c7032a6fc41cfeb81e982eeab9b", - "size": "33096140" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.7.0/zig-linux-armv7a-0.7.0.tar.xz", - "shasum": "011c267e25a96ee160505a560c441daa045359a9d50e13ab1bada9d75c95db2d", - "size": "35157584" - }, - "i386-linux": { - "tarball": "https://ziglang.org/download/0.7.0/zig-linux-i386-0.7.0.tar.xz", - "shasum": "4bb2072cd363bcb1cbeb4872ff5cbc1f683b02d0cc1f90c46e3ea7422ce53222", - "size": "38530596" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.7.0/zig-linux-riscv64-0.7.0.tar.xz", - "shasum": "40dff81faa6f232ac40abbf88b9371f3cc932b6e09c423b94387c9ea580cb7be", - "size": "36759992" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.7.0/zig-linux-x86_64-0.7.0.tar.xz", - "shasum": "e619b1c6094c095b932767f527aee2507f847ea981513ff8a08aab0fd730e0ac", - "size": "37154432" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.7.0/zig-macos-aarch64-0.7.0.tar.xz", - "shasum": "338238035734db74ea4f30e500a4893bf741d38305c10952d5e39fa05bdb057d", - "size": "33739424" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.7.0/zig-macos-x86_64-0.7.0.tar.xz", - "shasum": "94063f9a311cbbf7a2e0a12295e09437182cf950f18cb0eb30ea9893f3677f24", - "size": "35258328" - }, - "i386-windows": { - "tarball": "https://ziglang.org/download/0.7.0/zig-windows-i386-0.7.0.zip", - "shasum": "b1e520aacbfbd645ff3521b3eb4d44166d9a0288b8725e4b001f8b50a425eb2e", - "size": "53390517" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.7.0/zig-windows-x86_64-0.7.0.zip", - "shasum": "965f56c0a36f9cda2125e3a348bc654f7f155e2804c3667d231775ec228f8553", - "size": "53943784" - } - }, - "0.6.0": { - "date": "2020-04-13", - "docs": "https://ziglang.org/documentation/0.6.0/", - "stdDocs": "https://ziglang.org/documentation/0.6.0/std/", - "notes": "https://ziglang.org/download/0.6.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.6.0/zig-0.6.0.tar.xz", - "shasum": "5d167dc19354282dd35dd17b38e99e1763713b9be8a4ba9e9e69284e059e7204", - "size": "10349552" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.6.0/zig-bootstrap-0.6.0.tar.xz", - "shasum": "5e0e4dc878b3dd0c1852a442b174f0732e8c07869a8fcd226b71a93b89b381ab", - "size": "38469948" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.6.0/zig-freebsd-x86_64-0.6.0.tar.xz", - "shasum": "190ff79c1eb56805a315d7c7a51082e32f62926250c0702b36760c225e1634a3", - "size": "36974604" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.6.0/zig-linux-aarch64-0.6.0.tar.xz", - "shasum": "e7520efd42cfa02be48c2e430d08fe1f3cbb999d21d9f0d3ffd0febb976b2f41", - "size": "37090044" - }, - "armv6kz-linux": { - "tarball": "https://ziglang.org/download/0.6.0/zig-linux-armv6kz-0.6.0.tar.xz", - "shasum": "36b6493b3fed43eb1f0000e765798ad31a6bb7d7fd3f553ac1c3761dbc919b82", - "size": "39133452" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.6.0/zig-linux-armv7a-0.6.0.tar.xz", - "shasum": "946969abe357def95ca9cbbfcebfcf2d90cf967bcd3f48ee87662e32d91d8f35", - "size": "39143748" - }, - "i386-linux": { - "tarball": "https://ziglang.org/download/0.6.0/zig-linux-i386-0.6.0.tar.xz", - "shasum": "a97a2f9ae21575743cdd763c1917d49400d83fc562ef64582b18bade43eb24ce", - "size": "44877640" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.6.0/zig-linux-riscv64-0.6.0.tar.xz", - "shasum": "68ddee43f7503c8ae5f26a921f3602c34719a02ed2241f528c0b8b888cc14b38", - "size": "41993144" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.6.0/zig-linux-x86_64-0.6.0.tar.xz", - "shasum": "08fd3c757963630645441c2772362e9c2294020c44f14fce1b89f45de0dc1253", - "size": "44766320" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.6.0/zig-macos-x86_64-0.6.0.tar.xz", - "shasum": "17270360e87ddc49f737e760047b2fac49f1570a824a306119b1194ac4093895", - "size": "42573184" - }, - "i386-windows": { - "tarball": "https://ziglang.org/download/0.6.0/zig-windows-i386-0.6.0.zip", - "shasum": "3b0a02618743e92175990dc6d1a787bb95ff62c4cda016f1c14c7786f575f8ca", - "size": "60446431" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.6.0/zig-windows-x86_64-0.6.0.zip", - "shasum": "c3b897832523e1026e10b2d8d55d7f895185c0a27a63681f3a23219c3f1c38f4", - "size": "49065511" - } - }, - "0.5.0": { - "date": "2019-09-30", - "docs": "https://ziglang.org/documentation/0.5.0/", - "notes": "https://ziglang.org/download/0.5.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.5.0/zig-0.5.0.tar.xz", - "shasum": "55ae16960f152bcb9cf98b4f8570902d0e559a141abf927f0d3555b7cc838a31", - "size": "10956132" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.5.0/zig-freebsd-x86_64-0.5.0.tar.xz", - "shasum": "9e1f4d36c3d584c0aa01f20eb4cd0a0eef3eee5af23e483b8414de55feab6ab6", - "size": "33650744" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.5.0/zig-macos-x86_64-0.5.0.tar.xz", - "shasum": "28702cc05745c7c0bd450487d5f4091bf0a1ad279b35eb9a640ce3e3a15b300d", - "size": "37898664" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.5.0/zig-windows-x86_64-0.5.0.zip", - "shasum": "58141323db8d84a5af62746be5f9140bc161ee760ef33dc91a887bf9ac021976", - "size": "44871804" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.5.0/zig-linux-x86_64-0.5.0.tar.xz", - "shasum": "43e8f8a8b8556edd373ddf9c1ef3ca6cf852d4d09fe07d5736d12fefedd2b4f7", - "size": "40895068" - } - }, - "0.4.0": { - "date": "2019-04-08", - "docs": "https://ziglang.org/documentation/0.4.0/", - "notes": "https://ziglang.org/download/0.4.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.4.0/zig-0.4.0.tar.xz", - "shasum": "fec1f3f6b359a3d942e0a7f9157b3b30cde83927627a0e1ea95c54de3c526cfc", - "size": "5348776" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.4.0/zig-freebsd-x86_64-0.4.0.tar.xz", - "shasum": "3d557c91ac36d8262eb1733bb5f261c95944f9b635e43386e3d00a3272818c30", - "size": "27269672" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.4.0/zig-macos-x86_64-0.4.0.tar.xz", - "shasum": "67c932982484d017c5111e54af9f33f15e8e05c6bc5346a55e04052159c964a8", - "size": "30841504" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.4.0/zig-windows-x86_64-0.4.0.zip", - "shasum": "fbc3dd205e064c263063f69f600bedb18e3d0aa2efa747a63ef6cafb6d73f127", - "size": "35800101" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.4.0/zig-linux-x86_64-0.4.0.tar.xz", - "shasum": "fb1954e2fb556a01f8079a08130e88f70084e08978ff853bb2b1986d8c39d84e", - "size": "32876100" - } - }, - "0.3.0": { - "date": "2018-09-28", - "docs": "https://ziglang.org/documentation/0.3.0/", - "notes": "https://ziglang.org/download/0.3.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.3.0/zig-0.3.0.tar.xz", - "shasum": "d70af604f3a8622f3393d93abb3e056bf60351e32d121e6fa4fe03d8d41e1f5a", - "size": "2335592" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.3.0/zig-macos-x86_64-0.3.0.tar.xz", - "shasum": "19dec1f1943ab7be26823376d466f7e456143deb34e17502778a949034dc2e7e", - "size": "23712696" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.3.0/zig-windows-x86_64-0.3.0.zip", - "shasum": "bb568c03950958f8bb3472139c3ab5ed74547c8c694ab50f404c202faf51baf4", - "size": "22524425" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.3.0/zig-linux-x86_64-0.3.0.tar.xz", - "shasum": "b378d0aae30cb54f28494e7bc4efbc9bfb6326f47bfb302e8b5287af777b2f3c", - "size": "25209304" - } - }, - "0.2.0": { - "date": "2018-03-15", - "docs": "https://ziglang.org/documentation/0.2.0/", - "notes": "https://ziglang.org/download/0.2.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.2.0/zig-0.2.0.tar.xz", - "shasum": "29c9beb172737f4d5019b88ceae829ae8bc6512fb4386cfbf895ae2b42aa6965", - "size": "1940832" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.2.0/zig-win64-0.2.0.zip", - "shasum": "4f8a2979941a1f081ec8e545cca0b72608c0db1c5a3fd377a94db40649dcd3d4", - "size": "21076274" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.2.0/zig-linux-x86_64-0.2.0.tar.xz", - "shasum": "209c6fb745d42474c0a73d6f291c7ae3a38b6a1b6b641eea285a7f840cc1a890", - "size": "22551928" - } - }, - "0.1.1": { - "date": "2017-10-17", - "docs": "https://ziglang.org/documentation/0.1.1/", - "notes": "https://ziglang.org/download/0.1.1/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.1.1/zig-0.1.1.tar.xz", - "shasum": "ffca0cfb263485287e19cc997b08701fcd5f24b700345bcdc3dd8074f5a104e0", - "size": "1659716" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.1.1/zig-win64-0.1.1.zip", - "shasum": "6fc88bef531af7e567fe30bf60da1487b86833cbee84c7a2f3e317030aa5b660", - "size": "19757776" - } - } } From 20747c4dcbc718897bd2f757e1d38f45e7d9eb46 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Wed, 26 Nov 2025 17:47:50 +0100 Subject: [PATCH 059/881] WIP --- MODULE.bazel | 19 +- MODULE.bazel.lock | 203 ++- bazel/zig_index.json | 3086 +++++++++++++++++++++--------------------- 3 files changed, 1635 insertions(+), 1673 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 1d0bce76..30f3bcd4 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -10,7 +10,7 @@ bazel_dep(name = "libxev", version = "0.0.0-20251014.0-9f785d2") bazel_dep(name = "patchelf", version = "0.18.0") bazel_dep(name = "pcre2", version = "10.45") bazel_dep(name = "platforms", version = "1.0.0") -bazel_dep(name = "protobuf", version = "32.0", repo_name = "com_google_protobuf") +bazel_dep(name = "protobuf", version = "33.1", repo_name = "com_google_protobuf") # Needs to be added before rules_cc so that the cc toolchain declared by # apple_support wins over the one in rules_cc. @@ -30,7 +30,7 @@ bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2", dev_dependency = Tr zig = use_extension("@rules_zig//zig:extensions.bzl", "zig") zig.index(file = "//bazel:zig_index.json") -zig.toolchain(zig_version = "0.16.0-dev.1316+181b25ce4") +zig.toolchain(zig_version = "0.16.0-dev.1458+755a3d957") zig.mirrors(urls = [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/", @@ -42,8 +42,9 @@ register_toolchains("@rules_zig//zig/target:all") register_toolchains("@zig_toolchains//:all") register_toolchains( - "@toolchains_llvm_bootstrapped//toolchain:linux_amd64", - "@toolchains_llvm_bootstrapped//toolchain:linux_aarch64", + "@toolchains_llvm_bootstrapped//toolchain:macos_aarch64_to_linux_x86_64", + # "@toolchains_llvm_bootstrapped//toolchain:linux_amd64", + # "@toolchains_llvm_bootstrapped//toolchain:linux_aarch64", ) uv = use_extension("@rules_python//python/uv:uv.bzl", "uv", dev_dependency = True) @@ -214,3 +215,13 @@ huggingface.model( ) use_repo(huggingface, "ModernBERT-base", "ModernBERT-large") ################################################################################ + +local_path_override( + module_name = "rules_zig", + path = "../rules_zig", +) + +local_path_override( + module_name = "toolchains_llvm_bootstrapped", + path = "../toolchains_llvm_bootstrapped", +) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index c2af45d8..dfec8011 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -16,6 +16,7 @@ "https://bazel-registry.zml.ai/modules/apple_support/1.15.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/apple_support/1.17.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/apple_support/1.24.2/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/apple_support/1.24.5/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.0.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.16.0/MODULE.bazel": "not found", @@ -35,15 +36,17 @@ "https://bazel-registry.zml.ai/modules/bazel_features/1.19.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.20.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.21.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.23.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.24.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.27.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.28.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.3.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.30.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/bazel_features/1.33.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/bazel_features/1.34.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.4.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.9.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.9.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/bazel_lib/3.0.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_skylib/1.0.3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_skylib/1.1.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_skylib/1.2.0/MODULE.bazel": "not found", @@ -92,7 +95,7 @@ "https://bazel-registry.zml.ai/modules/protobuf/29.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/protobuf/29.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/protobuf/3.19.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/32.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/protobuf/33.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/pybind11_bazel/2.11.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/pybind11_bazel/2.12.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/re2/2023-09-01/MODULE.bazel": "not found", @@ -115,8 +118,9 @@ "https://bazel-registry.zml.ai/modules/rules_cc/0.2.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.11/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.12/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_cc/0.2.13/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_cc/0.2.14/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.8/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.2.9/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_distroless/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_fuzzing/0.5.2/MODULE.bazel": "not found", @@ -167,16 +171,15 @@ "https://bazel-registry.zml.ai/modules/rules_python/0.33.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_python/0.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_python/0.40.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/1.4.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_python/1.5.3/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_python/1.6.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_rust/0.63.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.2.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.3.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.4.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_shell/0.6.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_zig/0.12.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_zig/0.12.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.4/MODULE.bazel": "not found", @@ -188,10 +191,10 @@ "https://bazel-registry.zml.ai/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/tar.bzl/0.2.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/tar.bzl/0.5.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/toolchains_llvm_bootstrapped/0.2.4/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/tar.bzl/0.6.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/with_cfg.bzl/0.11.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/with_cfg.bzl/0.9.2/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/with_cfg.bzl/0.12.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/yq.bzl/0.1.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/zlib/1.2.11/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "not found", @@ -213,7 +216,8 @@ "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", "https://bcr.bazel.build/modules/apple_support/1.17.1/MODULE.bazel": "655c922ab1209978a94ef6ca7d9d43e940cd97d9c172fb55f94d91ac53f8610b", "https://bcr.bazel.build/modules/apple_support/1.24.2/MODULE.bazel": "0e62471818affb9f0b26f128831d5c40b074d32e6dda5a0d3852847215a41ca4", - "https://bcr.bazel.build/modules/apple_support/1.24.2/source.json": "2c22c9827093250406c5568da6c54e6fdf0ef06238def3d99c71b12feb057a8d", + "https://bcr.bazel.build/modules/apple_support/1.24.5/MODULE.bazel": "692784f32ee3175cc1ced78f4262a0d4b7110c89793cd0b63f950af2d9cc6bdc", + "https://bcr.bazel.build/modules/apple_support/1.24.5/source.json": "e81e4f6557a879aaaa104e84255710a02862b9380b12bd72e4a58bff32c70116", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.0.0/MODULE.bazel": "e118477db5c49419a88d78ebc7a2c2cea9d49600fe0f490c1903324a2c16ecd9", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.16.0/MODULE.bazel": "852f9ebbda017572a7c113a2434592dd3b2f55cd9a0faea3d4be5a09a59e4900", @@ -236,16 +240,19 @@ "https://bcr.bazel.build/modules/bazel_features/1.19.0/MODULE.bazel": "59adcdf28230d220f0067b1f435b8537dd033bfff8db21335ef9217919c7fb58", "https://bcr.bazel.build/modules/bazel_features/1.20.0/MODULE.bazel": "8b85300b9c8594752e0721a37210e34879d23adc219ed9dc8f4104a4a1750920", "https://bcr.bazel.build/modules/bazel_features/1.21.0/MODULE.bazel": "675642261665d8eea09989aa3b8afb5c37627f1be178382c320d1b46afba5e3b", - "https://bcr.bazel.build/modules/bazel_features/1.23.0/MODULE.bazel": "fd1ac84bc4e97a5a0816b7fd7d4d4f6d837b0047cf4cbd81652d616af3a6591a", "https://bcr.bazel.build/modules/bazel_features/1.24.0/MODULE.bazel": "4796b4c25b47053e9bbffa792b3792d07e228ff66cd0405faef56a978708acd4", "https://bcr.bazel.build/modules/bazel_features/1.27.0/MODULE.bazel": "621eeee06c4458a9121d1f104efb80f39d34deff4984e778359c60eaf1a8cb65", "https://bcr.bazel.build/modules/bazel_features/1.28.0/MODULE.bazel": "4b4200e6cbf8fa335b2c3f43e1d6ef3e240319c33d43d60cc0fbd4b87ece299d", "https://bcr.bazel.build/modules/bazel_features/1.3.0/MODULE.bazel": "cdcafe83ec318cda34e02948e81d790aab8df7a929cec6f6969f13a489ccecd9", "https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87", - "https://bcr.bazel.build/modules/bazel_features/1.30.0/source.json": "b07e17f067fe4f69f90b03b36ef1e08fe0d1f3cac254c1241a1818773e3423bc", + "https://bcr.bazel.build/modules/bazel_features/1.33.0/MODULE.bazel": "8b8dc9d2a4c88609409c3191165bccec0e4cb044cd7a72ccbe826583303459f6", + "https://bcr.bazel.build/modules/bazel_features/1.34.0/MODULE.bazel": "e8475ad7c8965542e0c7aac8af68eb48c4af904be3d614b6aa6274c092c2ea1e", + "https://bcr.bazel.build/modules/bazel_features/1.34.0/source.json": "dfa5c4b01110313153b484a735764d247fee5624bbab63d25289e43b151a657a", "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", "https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b", "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", + "https://bcr.bazel.build/modules/bazel_lib/3.0.0/MODULE.bazel": "22b70b80ac89ad3f3772526cd9feee2fa412c2b01933fea7ed13238a448d370d", + "https://bcr.bazel.build/modules/bazel_lib/3.0.0/source.json": "895f21909c6fba01d7c17914bb6c8e135982275a1b18cdaa4e62272217ef1751", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", @@ -307,8 +314,8 @@ "https://bcr.bazel.build/modules/protobuf/29.0/MODULE.bazel": "319dc8bf4c679ff87e71b1ccfb5a6e90a6dbc4693501d471f48662ac46d04e4e", "https://bcr.bazel.build/modules/protobuf/29.1/MODULE.bazel": "557c3457560ff49e122ed76c0bc3397a64af9574691cb8201b4e46d4ab2ecb95", "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0", - "https://bcr.bazel.build/modules/protobuf/32.0/MODULE.bazel": "0741cf24f8e1185286578069060e905ed67d68eef5990bfa3dea3fc1afba14c7", - "https://bcr.bazel.build/modules/protobuf/32.0/source.json": "1e278267d3642ab361dc460cc1f2d8e607e8292b9b3f799a47181a3acf4f3294", + "https://bcr.bazel.build/modules/protobuf/33.1/MODULE.bazel": "982c8a0cceab4d790076f72b7677faf836b0dfadc2b66a34aab7232116c4ae39", + "https://bcr.bazel.build/modules/protobuf/33.1/source.json": "992c237a40899425648213bf79b05f08c6e8dcd619f96cd944b4511b0276fbd8", "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e", "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/MODULE.bazel": "e6f4c20442eaa7c90d7190d8dc539d0ab422f95c65a57cc59562170c58ae3d34", "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/source.json": "6900fdc8a9e95866b8c0d4ad4aba4d4236317b5c1cd04c502df3f0d33afed680", @@ -335,9 +342,10 @@ "https://bcr.bazel.build/modules/rules_cc/0.2.0/MODULE.bazel": "b5c17f90458caae90d2ccd114c81970062946f49f355610ed89bebf954f5783c", "https://bcr.bazel.build/modules/rules_cc/0.2.11/MODULE.bazel": "e94f24f065bf2191dba2dace951814378b66a94bb3bcc48077492fe0508059b5", "https://bcr.bazel.build/modules/rules_cc/0.2.12/MODULE.bazel": "4216c383ce3223c7dfbd3afcb09c7056265156150b2a3bd3fd9d02949a7ee1bc", - "https://bcr.bazel.build/modules/rules_cc/0.2.12/source.json": "593d822a2ce7b70d5fad3195f7eb64d519dde94d7c20b0b79e6c23891a79b2bd", + "https://bcr.bazel.build/modules/rules_cc/0.2.13/MODULE.bazel": "eecdd666eda6be16a8d9dc15e44b5c75133405e820f620a234acc4b1fdc5aa37", + "https://bcr.bazel.build/modules/rules_cc/0.2.14/MODULE.bazel": "353c99ed148887ee89c54a17d4100ae7e7e436593d104b668476019023b58df8", + "https://bcr.bazel.build/modules/rules_cc/0.2.14/source.json": "55d0a4587c5592fad350f6e698530f4faf0e7dd15e69d43f8d87e220c78bea54", "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", - "https://bcr.bazel.build/modules/rules_cc/0.2.9/MODULE.bazel": "34263f1dca62ea664265438cef714d7db124c03e1ed55ebb4f1dc860164308d1", "https://bcr.bazel.build/modules/rules_distroless/0.5.1/MODULE.bazel": "2a63f4744d30749128105da5f96adf7caf5628e37548293f89e7fa39c3b3f2c2", "https://bcr.bazel.build/modules/rules_distroless/0.5.1/source.json": "c6b9ff7f325bfed89c3671757f14c1d1bc6077d0fcef809b8aa2d007cac7dd1d", "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", @@ -396,21 +404,19 @@ "https://bcr.bazel.build/modules/rules_python/0.33.2/MODULE.bazel": "3e036c4ad8d804a4dad897d333d8dce200d943df4827cb849840055be8d2e937", "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", "https://bcr.bazel.build/modules/rules_python/0.40.0/MODULE.bazel": "9d1a3cd88ed7d8e39583d9ffe56ae8a244f67783ae89b60caafc9f5cf318ada7", - "https://bcr.bazel.build/modules/rules_python/1.4.1/MODULE.bazel": "8991ad45bdc25018301d6b7e1d3626afc3c8af8aaf4bc04f23d0b99c938b73a6", "https://bcr.bazel.build/modules/rules_python/1.5.3/MODULE.bazel": "d0b7fb08458ca7fd80a26bc00c9e0f1d011609cc3da0381faa2eccd88c6ebd98", - "https://bcr.bazel.build/modules/rules_python/1.5.3/source.json": "06961e322e15331a2d88115a65af5d3f77cc46793f9d9aa0f928b95287337f12", + "https://bcr.bazel.build/modules/rules_python/1.6.0/MODULE.bazel": "7e04ad8f8d5bea40451cf80b1bd8262552aa73f841415d20db96b7241bd027d8", + "https://bcr.bazel.build/modules/rules_python/1.6.0/source.json": "e980f654cf66ec4928672f41fc66c4102b5ea54286acf4aecd23256c84211be6", "https://bcr.bazel.build/modules/rules_rust/0.63.0/MODULE.bazel": "4144e1606661c7168d23e8b4e7c5f6fb28ef519d9d5d63e0bd789d1b2a4611f8", "https://bcr.bazel.build/modules/rules_rust/0.63.0/source.json": "638d4731ad05d31835ba45cffc06e8dc1cca01692a681daf831378cf952ee7e6", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", - "https://bcr.bazel.build/modules/rules_shell/0.4.1/source.json": "4757bd277fe1567763991c4425b483477bb82e35e777a56fd846eb5cceda324a", + "https://bcr.bazel.build/modules/rules_shell/0.6.1/MODULE.bazel": "72e76b0eea4e81611ef5452aa82b3da34caca0c8b7b5c0c9584338aa93bae26b", + "https://bcr.bazel.build/modules/rules_shell/0.6.1/source.json": "20ec05cd5e592055e214b2da8ccb283c7f2a421ea0dc2acbf1aa792e11c03d0c", "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", "https://bcr.bazel.build/modules/rules_swift/2.1.1/source.json": "40fc69dfaac64deddbb75bd99cdac55f4427d9ca0afbe408576a65428427a186", - "https://bcr.bazel.build/modules/rules_zig/0.12.1/MODULE.bazel": "5953094b681c212eebe3dd4275809c52b9177aeaa058d8f1bff433822c0dcd01", - "https://bcr.bazel.build/modules/rules_zig/0.12.2/MODULE.bazel": "09d2ca5486c5c3bbcaa61057958d35f9b4e1e981e08a5a28ad516bab87e8c4ef", - "https://bcr.bazel.build/modules/rules_zig/0.12.2/source.json": "4cef85ce1b625967ce9c063b8716920e8e7067c38321981a7bfd581ad588f4ba", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", "https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4", @@ -424,13 +430,12 @@ "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/source.json": "32bd87e5f4d7acc57c5b2ff7c325ae3061d5e242c0c4c214ae87e0f1c13e54cb", "https://bcr.bazel.build/modules/tar.bzl/0.2.1/MODULE.bazel": "52d1c00a80a8cc67acbd01649e83d8dd6a9dc426a6c0b754a04fe8c219c76468", "https://bcr.bazel.build/modules/tar.bzl/0.5.1/MODULE.bazel": "7c2eb3dcfc53b0f3d6f9acdfd911ca803eaf92aadf54f8ca6e4c1f3aee288351", - "https://bcr.bazel.build/modules/tar.bzl/0.5.1/source.json": "deed3094f7cc779ed1d37a68403847b0e38d9dd9d931e03cb90825f3368b515f", - "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.2.4/MODULE.bazel": "859f02226bc313a622f56b816582fb55d5aaced4a1ecde7397dd4a9ebd24321e", - "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.2.4/source.json": "43c5fd6b585cbf0b642157b6843a25425d541fabe638acb7bc908846c4bda8ca", + "https://bcr.bazel.build/modules/tar.bzl/0.6.0/MODULE.bazel": "a3584b4edcfafcabd9b0ef9819808f05b372957bbdff41601429d5fd0aac2e7c", + "https://bcr.bazel.build/modules/tar.bzl/0.6.0/source.json": "4a620381df075a16cb3a7ed57bd1d05f7480222394c64a20fa51bdb636fda658", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", "https://bcr.bazel.build/modules/with_cfg.bzl/0.11.0/MODULE.bazel": "bcb1dba64b74542648a93d6313e160b2670f22395581edd8c4ffadb811618ef7", - "https://bcr.bazel.build/modules/with_cfg.bzl/0.11.0/source.json": "19102ff7f611a7a821e11d866868ad541abb4f5757e37e1c40adb6834ebf6205", - "https://bcr.bazel.build/modules/with_cfg.bzl/0.9.2/MODULE.bazel": "8008021cb802e87e22b3e2b6f403c0529d4b865703fdcd40e2f36c23a651291c", + "https://bcr.bazel.build/modules/with_cfg.bzl/0.12.0/MODULE.bazel": "b573395fe63aef4299ba095173e2f62ccfee5ad9bbf7acaa95dba73af9fc2b38", + "https://bcr.bazel.build/modules/with_cfg.bzl/0.12.0/source.json": "3f3fbaeafecaf629877ad152a2c9def21f8d330d91aa94c5dc75bbb98c10b8b8", "https://bcr.bazel.build/modules/yq.bzl/0.1.1/MODULE.bazel": "9039681f9bcb8958ee2c87ffc74bdafba9f4369096a2b5634b88abc0eaefa072", "https://bcr.bazel.build/modules/yq.bzl/0.1.1/source.json": "2d2bad780a9f2b9195a4a370314d2c17ae95eaa745cefc2e12fbc49759b15aa3", "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", @@ -998,7 +1003,7 @@ "@@rules_python+//python/uv:uv.bzl%uv": { "general": { "bzlTransitiveDigest": "Vd/0cN91U7O//KVXEBJsnr64LUyHdC6DRiDTTnQTrUw=", - "usagesDigest": "t11Aj3mDwK2nwRAqSmqylVpvomzPYWiSVUL4xbuiMm4=", + "usagesDigest": "IlyRxs7yhBn46gx9AgZj1f1kRAknvGnAoqPCzcMEPDA=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -1169,7 +1174,7 @@ }, "@@rules_rust+//crate_universe:extensions.bzl%crate": { "general": { - "bzlTransitiveDigest": "ZaON2eA6LKrwcPU8xDGi0W3+6wD1r6kjb0pSYFC4Se8=", + "bzlTransitiveDigest": "dgvl+ECngVEy/RaoBW5zRnnG7OibigZhl1yu8+X1gzg=", "usagesDigest": "mup6mgSFatk34Q0znMZPpIEOFZyk9v3xf6jRIXEFTxc=", "recordedFileInputs": { "@@//zml/tokenizer/hftokenizers/Cargo.lock": "d78ffab37ef69bf8ff45a4d686b25175f30e8fed4d8375612e84dcf6db0cfc9c", @@ -2823,89 +2828,89 @@ "@@rules_zig+//zig:extensions.bzl%zig": { "general": { "bzlTransitiveDigest": "wV3euhBZSI0D2ZlqbY1YW592mghFY88d9H6zBMQRaJY=", - "usagesDigest": "Hdjgt35tQvLgxXPxkTE4XGRdMneQAuafpvd5+TlL/7M=", + "usagesDigest": "kssHId7CLEGeadFinWbOWrcYqSqEA5uUw/GxX5yTP70=", "recordedFileInputs": { - "@@//bazel/zig_index.json": "af50f44e3ea95eee3ab0cc894be8fe981d54238d6ddd3dafb3e5fc5a7ae3b8d5", + "@@//bazel/zig_index.json": "4c9ee005881097e41652cd6e37a9ca921abf2e126188c67b74d7a153c7052536", "@@rules_zig+//zig/private/versions.json": "6eb85ebaee72c4e6fbc82d8ad73f69e0899d839b70a7e19caa792ab6a5fa0c36" }, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { - "zig_0.16.0-dev.1316_P181b25ce4_aarch64-linux": { + "zig_0.16.0-dev.1458_P755a3d957_aarch64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1316+181b25ce4.tar.xz", + "url": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1458+755a3d957.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "0f4cbfdab91b67d7cc0ec5d822a4cf0cf39da658d35e7eaa4cff3876d87444b3", - "zig_version": "0.16.0-dev.1316+181b25ce4", + "sha256": "63f22a3ee1fc7a8214ae21264b9773f4553ea11b94104233d4c4946984d6c3be", + "zig_version": "0.16.0-dev.1458+755a3d957", "platform": "aarch64-linux" } }, - "zig_0.16.0-dev.1316_P181b25ce4_aarch64-macos": { + "zig_0.16.0-dev.1458_P755a3d957_aarch64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1316+181b25ce4.tar.xz", + "url": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1458+755a3d957.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "c8121f65e34bb8f14b721d61e3e5ea234271ca9c1c35594092de6c12ff2cd19f", - "zig_version": "0.16.0-dev.1316+181b25ce4", + "sha256": "3dc8d405cea1ff394cd6720b8445ffea20c07bc1d9e4f7c68add9b4fd7ece554", + "zig_version": "0.16.0-dev.1458+755a3d957", "platform": "aarch64-macos" } }, - "zig_0.16.0-dev.1316_P181b25ce4_aarch64-windows": { + "zig_0.16.0-dev.1458_P755a3d957_aarch64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1316+181b25ce4.zip", + "url": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1458+755a3d957.zip", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "19ab6ebda0a8dc185174a8e19c279f6a8cd6c0a7cbfeaf4c34bdc5afbeeb53d0", - "zig_version": "0.16.0-dev.1316+181b25ce4", + "sha256": "e6a55c7ef706ecc8fe757a55d333620267b09339a1c7f3e08eb8e7e75bbb9ad9", + "zig_version": "0.16.0-dev.1458+755a3d957", "platform": "aarch64-windows" } }, - "zig_0.16.0-dev.1316_P181b25ce4_x86_64-linux": { + "zig_0.16.0-dev.1458_P755a3d957_x86_64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1316+181b25ce4.tar.xz", + "url": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1458+755a3d957.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "3c5af46217c87fc3b5e94e25ea68882690c82495aeac7028230f69025f92cd47", - "zig_version": "0.16.0-dev.1316+181b25ce4", + "sha256": "d1f819092c784fd821428501dac06387966179623ceea498be55a40e2fa8c8ef", + "zig_version": "0.16.0-dev.1458+755a3d957", "platform": "x86_64-linux" } }, - "zig_0.16.0-dev.1316_P181b25ce4_x86_64-macos": { + "zig_0.16.0-dev.1458_P755a3d957_x86_64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1316+181b25ce4.tar.xz", + "url": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1458+755a3d957.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "7ac36a40a2ed06d4030fe55737e162e507f9be1e654c34a3c7b22d7f1fd213a3", - "zig_version": "0.16.0-dev.1316+181b25ce4", + "sha256": "bebb2f5361cd65171ea78fc0fa7c180d5eaeec89da8a5d1985f3d7b97634d389", + "zig_version": "0.16.0-dev.1458+755a3d957", "platform": "x86_64-macos" } }, - "zig_0.16.0-dev.1316_P181b25ce4_x86_64-windows": { + "zig_0.16.0-dev.1458_P755a3d957_x86_64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1316+181b25ce4.zip", + "url": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1458+755a3d957.zip", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "bf021b998f93c71ad5970cd58a3f2083401191137e432112e5a60c75c25b35b2", - "zig_version": "0.16.0-dev.1316+181b25ce4", + "sha256": "bd1b710cf4c928407453f4da0bbc253c4db147223cc5df101b1f2fb5de01f624", + "zig_version": "0.16.0-dev.1458+755a3d957", "platform": "x86_64-windows" } }, @@ -2913,28 +2918,28 @@ "repoRuleId": "@@rules_zig+//zig/private/repo:toolchains_repo.bzl%toolchains_repo", "attributes": { "names": [ - "zig_0.16.0-dev.1316_P181b25ce4_aarch64-linux", - "zig_0.16.0-dev.1316_P181b25ce4_aarch64-macos", - "zig_0.16.0-dev.1316_P181b25ce4_aarch64-windows", - "zig_0.16.0-dev.1316_P181b25ce4_x86_64-linux", - "zig_0.16.0-dev.1316_P181b25ce4_x86_64-macos", - "zig_0.16.0-dev.1316_P181b25ce4_x86_64-windows" + "zig_0.16.0-dev.1458_P755a3d957_aarch64-linux", + "zig_0.16.0-dev.1458_P755a3d957_aarch64-macos", + "zig_0.16.0-dev.1458_P755a3d957_aarch64-windows", + "zig_0.16.0-dev.1458_P755a3d957_x86_64-linux", + "zig_0.16.0-dev.1458_P755a3d957_x86_64-macos", + "zig_0.16.0-dev.1458_P755a3d957_x86_64-windows" ], "labels": [ - "@zig_0.16.0-dev.1316_P181b25ce4_aarch64-linux//:zig_toolchain", - "@zig_0.16.0-dev.1316_P181b25ce4_aarch64-macos//:zig_toolchain", - "@zig_0.16.0-dev.1316_P181b25ce4_aarch64-windows//:zig_toolchain", - "@zig_0.16.0-dev.1316_P181b25ce4_x86_64-linux//:zig_toolchain", - "@zig_0.16.0-dev.1316_P181b25ce4_x86_64-macos//:zig_toolchain", - "@zig_0.16.0-dev.1316_P181b25ce4_x86_64-windows//:zig_toolchain" + "@zig_0.16.0-dev.1458_P755a3d957_aarch64-linux//:zig_toolchain", + "@zig_0.16.0-dev.1458_P755a3d957_aarch64-macos//:zig_toolchain", + "@zig_0.16.0-dev.1458_P755a3d957_aarch64-windows//:zig_toolchain", + "@zig_0.16.0-dev.1458_P755a3d957_x86_64-linux//:zig_toolchain", + "@zig_0.16.0-dev.1458_P755a3d957_x86_64-macos//:zig_toolchain", + "@zig_0.16.0-dev.1458_P755a3d957_x86_64-windows//:zig_toolchain" ], "zig_versions": [ - "0.16.0-dev.1316+181b25ce4", - "0.16.0-dev.1316+181b25ce4", - "0.16.0-dev.1316+181b25ce4", - "0.16.0-dev.1316+181b25ce4", - "0.16.0-dev.1316+181b25ce4", - "0.16.0-dev.1316+181b25ce4" + "0.16.0-dev.1458+755a3d957", + "0.16.0-dev.1458+755a3d957", + "0.16.0-dev.1458+755a3d957", + "0.16.0-dev.1458+755a3d957", + "0.16.0-dev.1458+755a3d957", + "0.16.0-dev.1458+755a3d957" ], "exec_lengths": [ 2, @@ -2974,60 +2979,6 @@ ] ] } - }, - "@@tar.bzl+//tar:extensions.bzl%toolchains": { - "general": { - "bzlTransitiveDigest": "ARFnga1qqjy0bT/hVunmIZnM4oZCkD8F253Z8POuLXs=", - "usagesDigest": "maF8qsAIqeH1ey8pxP0gNZbvJt34kLZvTFeQ0ntrJVA=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "bsd_tar_toolchains": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:toolchain.bzl%tar_toolchains_repo", - "attributes": { - "user_repository_name": "bsd_tar_toolchains" - } - }, - "bsd_tar_toolchains_darwin_amd64": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "bsd_tar_toolchains_darwin_arm64": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "bsd_tar_toolchains_linux_amd64": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "bsd_tar_toolchains_linux_arm64": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "bsd_tar_toolchains_windows_amd64": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "windows_amd64" - } - }, - "bsd_tar_toolchains_windows_arm64": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "windows_arm64" - } - } - }, - "recordedRepoMappingEntries": [] - } } } } diff --git a/bazel/zig_index.json b/bazel/zig_index.json index 16212acb..ec051715 100644 --- a/bazel/zig_index.json +++ b/bazel/zig_index.json @@ -1,1568 +1,1568 @@ { - "master": { - "version": "0.16.0-dev.1316+181b25ce4", - "date": "2025-11-13", - "docs": "https://ziglang.org/documentation/master/", - "stdDocs": "https://ziglang.org/documentation/master/std/", - "src": { - "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.1316+181b25ce4.tar.xz", - "shasum": "1f100b72af97c49f6c220dd1a118b2e3d24376db7500e2e0c2ca1c45e4ddfa6e", - "size": "21532448" - }, - "bootstrap": { - "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.1316+181b25ce4.tar.xz", - "shasum": "705a565be067f3eea6692094fc50f86c274127f65b6700fbd1407c39ccbb1035", - "size": "54297044" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1316+181b25ce4.tar.xz", - "shasum": "7ac36a40a2ed06d4030fe55737e162e507f9be1e654c34a3c7b22d7f1fd213a3", - "size": "56658972" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1316+181b25ce4.tar.xz", - "shasum": "c8121f65e34bb8f14b721d61e3e5ea234271ca9c1c35594092de6c12ff2cd19f", - "size": "51541680" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1316+181b25ce4.tar.xz", - "shasum": "3c5af46217c87fc3b5e94e25ea68882690c82495aeac7028230f69025f92cd47", - "size": "54601016" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1316+181b25ce4.tar.xz", - "shasum": "0f4cbfdab91b67d7cc0ec5d822a4cf0cf39da658d35e7eaa4cff3876d87444b3", - "size": "50329600" - }, - "arm-linux": { - "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.1316+181b25ce4.tar.xz", - "shasum": "41245740e7212dcf1416cba42c4af280a5b9a2842f75d4e1475aa06cddac2601", - "size": "51223720" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.1316+181b25ce4.tar.xz", - "shasum": "87c2327c8ea455a899cfa53cbb6d083f6c5f8b3ed318787aaaf343b31dc0b81a", - "size": "54400728" - }, - "powerpc64le-linux": { - "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.1316+181b25ce4.tar.xz", - "shasum": "51bd89e809757f8affdd9f2206b200a0bc3bf0c22ac46f6968706ef056646ce0", - "size": "54401180" - }, - "x86-linux": { - "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.1316+181b25ce4.tar.xz", - "shasum": "292dad4e3adf9a311d232d81f93653cb862b88c46bff787a23fdb1708b8b9086", - "size": "57269752" - }, - "loongarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.1316+181b25ce4.tar.xz", - "shasum": "b073137e46fb307d6224b5bf662116631738723fb020da5224c9fffdaf7d88c5", - "size": "51645988" - }, - "s390x-linux": { - "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.1316+181b25ce4.tar.xz", - "shasum": "20ec44ae170b4163e0a3ec683754e9a17c007f6a703718e8d679b97877297e7b", - "size": "54094164" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1316+181b25ce4.zip", - "shasum": "bf021b998f93c71ad5970cd58a3f2083401191137e432112e5a60c75c25b35b2", - "size": "94482164" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1316+181b25ce4.zip", - "shasum": "19ab6ebda0a8dc185174a8e19c279f6a8cd6c0a7cbfeaf4c34bdc5afbeeb53d0", - "size": "90458422" - }, - "x86-windows": { - "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.1316+181b25ce4.zip", - "shasum": "7a4f6463c92ddf81a9dceaaa953257688cc93f0366013681b1c91d1e57f08572", - "size": "96320381" - }, - "aarch64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.1316+181b25ce4.tar.xz", - "shasum": "09cea8e2ef61b91e2138e16d4cb32823edec41a2676fcfc7ff9e9dfd3175e3a6", - "size": "50296404" - }, - "arm-freebsd": { - "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.1316+181b25ce4.tar.xz", - "shasum": "213a3394ac0aefffab7997367c6f6510565b7e8e8c7829c469a13db191b8ad56", - "size": "51889452" - }, - "powerpc64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-powerpc64-freebsd-0.16.0-dev.1316+181b25ce4.tar.xz", - "shasum": "79e9e0c49f30aab6c973f79840ef8107924af5c7dc2cd19692cf564ae77531b3", - "size": "52979080" - }, - "powerpc64le-freebsd": { - "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.1316+181b25ce4.tar.xz", - "shasum": "630b4b54a43f18d92134901396aafeb49996a9b6beb4d873bab70136819737a9", - "size": "54416056" - }, - "riscv64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.1316+181b25ce4.tar.xz", - "shasum": "e422a73e4c4616ca07a4e7d70db5dcf3a1924ebbc17727b5f9f392b0c7e0ce20", - "size": "54545596" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.1316+181b25ce4.tar.xz", - "shasum": "e8ccc582b936ae144465d9ff390cc7a1910d322b5006d856d715d9b507f7911a", - "size": "54689648" - }, - "aarch64-netbsd": { - "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.1316+181b25ce4.tar.xz", - "shasum": "6565b48ea59fd3842650f3f930389b465f2a7ee81182596e771a701420e5f74a", - "size": "50258944" - }, - "arm-netbsd": { - "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.1316+181b25ce4.tar.xz", - "shasum": "328912165bf0523fd6122645fb0566e996ad399c819d9435bc7f6be8467b5495", - "size": "52915556" - }, - "x86-netbsd": { - "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.1316+181b25ce4.tar.xz", - "shasum": "2a3c24cdf88088285a9b8b3ceab705b09edc98574351e513e201cac3f86fd14b", - "size": "57879744" - }, - "x86_64-netbsd": { - "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.1316+181b25ce4.tar.xz", - "shasum": "c3bb94dae6f5e6bd78c45c4c271c8c939c08179ea66b2676377f21dada933f15", - "size": "54641936" - } + "master": { + "version": "0.16.0-dev.1458+755a3d957", + "date": "2025-11-24", + "docs": "https://ziglang.org/documentation/master/", + "stdDocs": "https://ziglang.org/documentation/master/std/", + "src": { + "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.1458+755a3d957.tar.xz", + "shasum": "5cd89a3f8dfdf34d009d5cf9547ec319ac1898779b42fa4437c7ab4dd36c13c9", + "size": "21698384" }, - "0.15.2": { - "version": "0.15.2", - "date": "2025-10-11", - "docs": "https://ziglang.org/documentation/0.15.2/", - "stdDocs": "https://ziglang.org/documentation/0.15.2/std/", - "src": { - "tarball": "https://ziglang.org/download/0.15.2/zig-0.15.2.tar.xz", - "shasum": "d9b30c7aa983fcff5eed2084d54ae83eaafe7ff3a84d8fb754d854165a6e521c", - "size": "21366268" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.15.2/zig-bootstrap-0.15.2.tar.xz", - "shasum": "a6845459501df3c3264ebc587b02a7094ad14f4f3f7287c48f04457e784d0d85", - "size": "52718988" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-macos-0.15.2.tar.xz", - "shasum": "375b6909fc1495d16fc2c7db9538f707456bfc3373b14ee83fdd3e22b3d43f7f", - "size": "55800460" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-macos-0.15.2.tar.xz", - "shasum": "3cc2bab367e185cdfb27501c4b30b1b0653c28d9f73df8dc91488e66ece5fa6b", - "size": "50635984" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-linux-0.15.2.tar.xz", - "shasum": "02aa270f183da276e5b5920b1dac44a63f1a49e55050ebde3aecc9eb82f93239", - "size": "53733924" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-linux-0.15.2.tar.xz", - "shasum": "958ed7d1e00d0ea76590d27666efbf7a932281b3d7ba0c6b01b0ff26498f667f", - "size": "49471996" - }, - "arm-linux": { - "tarball": "https://ziglang.org/download/0.15.2/zig-arm-linux-0.15.2.tar.xz", - "shasum": "7d8401495065dae45d6249c68d5faf10508f8203c86362ccb698aeaafc66b7cd", - "size": "50454216" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.15.2/zig-riscv64-linux-0.15.2.tar.xz", - "shasum": "493512bdca485be3c6a9b0f69dcb4cbe4587f3af8e1be282fdd827108ba39930", - "size": "53613656" - }, - "powerpc64le-linux": { - "tarball": "https://ziglang.org/download/0.15.2/zig-powerpc64le-linux-0.15.2.tar.xz", - "shasum": "e182c5f8d30fc7f97d17d2ffef1488826aa3afaa51e5f0dbe14c597a98b45778", - "size": "53570632" - }, - "x86-linux": { - "tarball": "https://ziglang.org/download/0.15.2/zig-x86-linux-0.15.2.tar.xz", - "shasum": "4c6e23f39daa305e274197bfdff0d56ffd1750fc1de226ae10505c0eff52d7a5", - "size": "56334032" - }, - "loongarch64-linux": { - "tarball": "https://ziglang.org/download/0.15.2/zig-loongarch64-linux-0.15.2.tar.xz", - "shasum": "95d6a03e0415dbe30ae9c00d58b5446fd681ebf3d4044a8b707e1784c8074a61", - "size": "50800332" - }, - "s390x-linux": { - "tarball": "https://ziglang.org/download/0.15.2/zig-s390x-linux-0.15.2.tar.xz", - "shasum": "cd4b295f43b1b7ca3d603368caea82dbd00fce6691affb7527e607c0b6eafe8e", - "size": "53486436" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-windows-0.15.2.zip", - "shasum": "3a0ed1e8799a2f8ce2a6e6290a9ff22e6906f8227865911fb7ddedc3cc14cb0c", - "size": "92614574" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-windows-0.15.2.zip", - "shasum": "b926465f8872bf983422257cd9ec248bb2b270996fbe8d57872cca13b56fc370", - "size": "88464585" - }, - "x86-windows": { - "tarball": "https://ziglang.org/download/0.15.2/zig-x86-windows-0.15.2.zip", - "shasum": "7a6dfc00f4cc09ec46d3e10eb06f42538e92b6285e34debea7462edaf371da98", - "size": "94516930" - }, - "aarch64-freebsd": { - "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-freebsd-0.15.2.tar.xz", - "shasum": "c62efd319f86663eb7747709dfca259205edba8eaee98efc96a51ce40a9437de", - "size": "49363332" - }, - "arm-freebsd": { - "tarball": "https://ziglang.org/download/0.15.2/zig-arm-freebsd-0.15.2.tar.xz", - "shasum": "1d5d06428b2d4d31cf4e99008602f2096a9c652dd1025820a4aa66326fbf3dc4", - "size": "50900824" - }, - "powerpc64-freebsd": { - "tarball": "https://ziglang.org/download/0.15.2/zig-powerpc64-freebsd-0.15.2.tar.xz", - "shasum": "2bdb5a38069e667936d3378dfa332d67c21bb454cb17bea3c3b03faaa02a74af", - "size": "52099240" - }, - "powerpc64le-freebsd": { - "tarball": "https://ziglang.org/download/0.15.2/zig-powerpc64le-freebsd-0.15.2.tar.xz", - "shasum": "940894eb954c2077d0643f4deba8113239436fd0dcbc88160163b1b23fe3f035", - "size": "53459116" - }, - "riscv64-freebsd": { - "tarball": "https://ziglang.org/download/0.15.2/zig-riscv64-freebsd-0.15.2.tar.xz", - "shasum": "9aecdcf32fb63944c11389c10993aecd5e443a82d456693cc5ad230b5fe584ad", - "size": "53658752" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-freebsd-0.15.2.tar.xz", - "shasum": "5509ff57cd3f219165caed0da10221739af82742b9edfcda3f7bfaf4da7212dd", - "size": "53776208" - }, - "aarch64-netbsd": { - "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-netbsd-0.15.2.tar.xz", - "shasum": "357c1e62df6581ec1726546f9a827973233cb31ac371d2b0e92456b306a0c6c0", - "size": "49365256" - }, - "arm-netbsd": { - "tarball": "https://ziglang.org/download/0.15.2/zig-arm-netbsd-0.15.2.tar.xz", - "shasum": "1c43871791d49dbabd224da307b995076e493e34c36734cb21c4f6cbc208f8c3", - "size": "52021696" - }, - "x86-netbsd": { - "tarball": "https://ziglang.org/download/0.15.2/zig-x86-netbsd-0.15.2.tar.xz", - "shasum": "7f18fc4720426a6ca77bbfb7b970c31a61c37ce4a867b53830762958e921cdde", - "size": "56893368" - }, - "x86_64-netbsd": { - "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-netbsd-0.15.2.tar.xz", - "shasum": "11a3b9f4b66bece34a79c0e98e91136baf0ff9795ddf60e442ca6c8ad73b9204", - "size": "53778132" - } + "bootstrap": { + "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.1458+755a3d957.tar.xz", + "shasum": "0a2f9586d101b904d12777abf765337d466a973a6ac98983c53f0a4fbf4920ff", + "size": "54461840" }, - "0.15.1": { - "date": "2025-08-19", - "docs": "https://ziglang.org/documentation/master/", - "stdDocs": "https://ziglang.org/documentation/master/std/", - "notes": "https://ziglang.org/download/0.15.1/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.15.1/zig-0.15.1.tar.xz", - "shasum": "816c0303ab313f59766ce2097658c9fff7fafd1504f61f80f9507cd11652865f", - "size": "21359884" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.15.1/zig-bootstrap-0.15.1.tar.xz", - "shasum": "4c0cfbcf12da144955761ca43f89e3c74956bce978694fc1d0a63555f5c0a199", - "size": "52711548" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-macos-0.15.1.tar.xz", - "shasum": "9919392e0287cccc106dfbcbb46c7c1c3fa05d919567bb58d7eb16bca4116184", - "size": "55791880" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-macos-0.15.1.tar.xz", - "shasum": "c4bd624d901c1268f2deb9d8eb2d86a2f8b97bafa3f118025344242da2c54d7b", - "size": "50644996" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-linux-0.15.1.tar.xz", - "shasum": "c61c5da6edeea14ca51ecd5e4520c6f4189ef5250383db33d01848293bfafe05", - "size": "53734456" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-linux-0.15.1.tar.xz", - "shasum": "bb4a8d2ad735e7fba764c497ddf4243cb129fece4148da3222a7046d3f1f19fe", - "size": "49493872" - }, - "arm-linux": { - "tarball": "https://ziglang.org/download/0.15.1/zig-arm-linux-0.15.1.tar.xz", - "shasum": "3f4bf3b06b67d14e3f38be30798488c1abe3cf5b33de570cd0e87bbf09b978ad", - "size": "50477464" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.15.1/zig-riscv64-linux-0.15.1.tar.xz", - "shasum": "7ca7a3e621436fb31d66a253132fc39574a13d2a1b4d8458af4f2e7c6e4374fe", - "size": "53597792" - }, - "powerpc64le-linux": { - "tarball": "https://ziglang.org/download/0.15.1/zig-powerpc64le-linux-0.15.1.tar.xz", - "shasum": "339e2106496be70b614e32d444298216e676c36d08f5cd7bee3dd1dbd4567fd7", - "size": "53566944" - }, - "x86-linux": { - "tarball": "https://ziglang.org/download/0.15.1/zig-x86-linux-0.15.1.tar.xz", - "shasum": "dff166f25fdd06e8341d831a71211b5ba7411463a6b264bdefa8868438690b6a", - "size": "56311228" - }, - "loongarch64-linux": { - "tarball": "https://ziglang.org/download/0.15.1/zig-loongarch64-linux-0.15.1.tar.xz", - "shasum": "0af18a012f3c4cffbef29ab5f42021484d92517f921a1380faacc89c50c1f89d", - "size": "50794376" - }, - "s390x-linux": { - "tarball": "https://ziglang.org/download/0.15.1/zig-s390x-linux-0.15.1.tar.xz", - "shasum": "bcd13e5c88cf2d0da7100a48572195560069a94402c9bec3b316398204aa27e2", - "size": "53508068" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-windows-0.15.1.zip", - "shasum": "91e69e887ca8c943ce9a515df3af013d95a66a190a3df3f89221277ebad29e34", - "size": "92612958" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-windows-0.15.1.zip", - "shasum": "1f1bf16228b0ffcc882b713dc5e11a6db4219cb30997e13c72e8e723c2104ec6", - "size": "88458549" - }, - "x86-windows": { - "tarball": "https://ziglang.org/download/0.15.1/zig-x86-windows-0.15.1.zip", - "shasum": "fb1c07cffbb43615d3158ab8b8f5db5da1d48875eca99e1d7a8a0064ff63fc5b", - "size": "94516052" - }, - "aarch64-freebsd": { - "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-freebsd-0.15.1.tar.xz", - "shasum": "4d9d25c775828d49ea037b2284310c295d951793da8ebe94827a54fed4cca3ce", - "size": "49358464" - }, - "arm-freebsd": { - "tarball": "https://ziglang.org/download/0.15.1/zig-arm-freebsd-0.15.1.tar.xz", - "shasum": "9707f3a5f7e1a3d99c40db9a74de1acc61016a197ad289c2ad964f93cb213a18", - "size": "50904124" - }, - "powerpc64-freebsd": { - "tarball": "https://ziglang.org/download/0.15.1/zig-powerpc64-freebsd-0.15.1.tar.xz", - "shasum": "79448884372db04e62f77a46b92245fce805063e69534729537f75cb4681e7e3", - "size": "52099020" - }, - "powerpc64le-freebsd": { - "tarball": "https://ziglang.org/download/0.15.1/zig-powerpc64le-freebsd-0.15.1.tar.xz", - "shasum": "f18ee12ba9c98a20b8d2ad0410c679e7aa5591bc7917f169fd6b377833d2c7ad", - "size": "53480976" - }, - "riscv64-freebsd": { - "tarball": "https://ziglang.org/download/0.15.1/zig-riscv64-freebsd-0.15.1.tar.xz", - "shasum": "ee9f864a6fd8b57c1f4fdbb11daa06578746a6f8253afe3f5ddb5a76f2eddd2d", - "size": "53677800" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-freebsd-0.15.1.tar.xz", - "shasum": "9714f8ac3d3dc908b1599837c6167f857c1efaa930f0cfa840699458de7c3cd0", - "size": "53782112" - }, - "aarch64-netbsd": { - "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-netbsd-0.15.1.tar.xz", - "shasum": "b2a528399777583b85b89c54ccd45488af7709d6dd29a27323ec2a229db40910", - "size": "49368000" - }, - "arm-netbsd": { - "tarball": "https://ziglang.org/download/0.15.1/zig-arm-netbsd-0.15.1.tar.xz", - "shasum": "93dc70109cbf5d2e022d20dfb56211978c4ea3c0b1e67aaabff947d8d1583aab", - "size": "52028184" - }, - "x86-netbsd": { - "tarball": "https://ziglang.org/download/0.15.1/zig-x86-netbsd-0.15.1.tar.xz", - "shasum": "a91b26051822ff17f3143f859b87dce5b4a13e90928bd6daa6f07a895d3410f0", - "size": "56881864" - }, - "x86_64-netbsd": { - "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-netbsd-0.15.1.tar.xz", - "shasum": "6d7ba6eca5b4434351ebdb971b7303c9934514f9bb8481852251dbd5b52b03d6", - "size": "53794836" - } + "x86_64-macos": { + "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1458+755a3d957.tar.xz", + "shasum": "bebb2f5361cd65171ea78fc0fa7c180d5eaeec89da8a5d1985f3d7b97634d389", + "size": "56838428" }, - "0.14.1": { - "date": "2025-05-21", - "docs": "https://ziglang.org/documentation/0.14.1/", - "stdDocs": "https://ziglang.org/documentation/0.14.1/std/", - "src": { - "tarball": "https://ziglang.org/download/0.14.1/zig-0.14.1.tar.xz", - "shasum": "237f8abcc8c3fd68c70c66cdbf63dce4fb5ad4a2e6225ac925e3d5b4c388f203", - "size": "17787696" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.14.1/zig-bootstrap-0.14.1.tar.xz", - "shasum": "89b2fce50bfbb1eee29c382193d22c6eb0c7da3a96b5ba6d05e0af2945b3ca3d", - "size": "48041028" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.14.1/zig-x86_64-macos-0.14.1.tar.xz", - "shasum": "b0f8bdfb9035783db58dd6c19d7dea89892acc3814421853e5752fe4573e5f43", - "size": "51044512" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.14.1/zig-aarch64-macos-0.14.1.tar.xz", - "shasum": "39f3dc5e79c22088ce878edc821dedb4ca5a1cd9f5ef915e9b3cc3053e8faefa", - "size": "45903552" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.14.1/zig-x86_64-linux-0.14.1.tar.xz", - "shasum": "24aeeec8af16c381934a6cd7d95c807a8cb2cf7df9fa40d359aa884195c4716c", - "size": "49086504" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.14.1/zig-aarch64-linux-0.14.1.tar.xz", - "shasum": "f7a654acc967864f7a050ddacfaa778c7504a0eca8d2b678839c21eea47c992b", - "size": "44954692" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.14.1/zig-armv7a-linux-0.14.1.tar.xz", - "shasum": "1b34d9ecfaeb3b360e86c0bc233e1a8a2bbed2d40f2d4f20c12bde2128714324", - "size": "46137456" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.14.1/zig-riscv64-linux-0.14.1.tar.xz", - "shasum": "005f214f74dbafb7b4d8bd305f4e9d25048f711d9ec6fa7b3d4fca177e11b882", - "size": "48094380" - }, - "powerpc64le-linux": { - "tarball": "https://ziglang.org/download/0.14.1/zig-powerpc64le-linux-0.14.1.tar.xz", - "shasum": "15523e748efd9224ae164482e5cc1f8c11b035246b9749fb5b00758484f384a1", - "size": "48720956" - }, - "x86-linux": { - "tarball": "https://ziglang.org/download/0.14.1/zig-x86-linux-0.14.1.tar.xz", - "shasum": "4bce6347fa112247443cb0952c19e560d1f90b910506cf895fd07a7b8d1c4a76", - "size": "51643520" - }, - "loongarch64-linux": { - "tarball": "https://ziglang.org/download/0.14.1/zig-loongarch64-linux-0.14.1.tar.xz", - "shasum": "fdc344789be6becbe220bf5ad035156e653cc148f4da270188cbac87729f17e0", - "size": "45825404" - }, - "s390x-linux": { - "tarball": "https://ziglang.org/download/0.14.1/zig-s390x-linux-0.14.1.tar.xz", - "shasum": "8e1bb63532ba22725f0a4a45e5c2920a8af508d8ca81e6007c330eec2b7033a6", - "size": "55602032" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.14.1/zig-x86_64-windows-0.14.1.zip", - "shasum": "554f5378228923ffd558eac35e21af020c73789d87afeabf4bfd16f2e6feed2c", - "size": "82229343" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.14.1/zig-aarch64-windows-0.14.1.zip", - "shasum": "b5aac0ccc40dd91e8311b1f257717d8e3903b5fefb8f659de6d65a840ad1d0e7", - "size": "78125379" - }, - "x86-windows": { - "tarball": "https://ziglang.org/download/0.14.1/zig-x86-windows-0.14.1.zip", - "shasum": "3ee730c2a5523570dc4dc1b724f3e4f30174ebc1fa109ca472a719586a473b18", - "size": "83983932" - } + "aarch64-macos": { + "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1458+755a3d957.tar.xz", + "shasum": "3dc8d405cea1ff394cd6720b8445ffea20c07bc1d9e4f7c68add9b4fd7ece554", + "size": "51687600" }, - "0.14.0": { - "date": "2025-03-05", - "docs": "https://ziglang.org/documentation/0.14.0/", - "stdDocs": "https://ziglang.org/documentation/0.14.0/std/", - "notes": "https://ziglang.org/download/0.14.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.14.0/zig-0.14.0.tar.xz", - "shasum": "c76638c03eb204c4432ae092f6fa07c208567e110fbd4d862d131a7332584046", - "size": "17772188" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.14.0/zig-bootstrap-0.14.0.tar.xz", - "shasum": "bf3fcb22be0b83f4791748adb567d3304779d66d7bf9b1bd557ef6c2e0232807", - "size": "48029040" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.14.0/zig-macos-x86_64-0.14.0.tar.xz", - "shasum": "685816166f21f0b8d6fc7aa6a36e91396dcd82ca6556dfbe3e329deffc01fec3", - "size": "51039964" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.14.0/zig-macos-aarch64-0.14.0.tar.xz", - "shasum": "b71e4b7c4b4be9953657877f7f9e6f7ee89114c716da7c070f4a238220e95d7e", - "size": "45902412" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.14.0/zig-linux-x86_64-0.14.0.tar.xz", - "shasum": "473ec26806133cf4d1918caf1a410f8403a13d979726a9045b421b685031a982", - "size": "49091960" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.14.0/zig-linux-aarch64-0.14.0.tar.xz", - "shasum": "ab64e3ea277f6fc5f3d723dcd95d9ce1ab282c8ed0f431b4de880d30df891e4f", - "size": "44922728" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.14.0/zig-linux-armv7a-0.14.0.tar.xz", - "shasum": "a67dbfa9bdf769228ec994f2098698c619f930883ca5ef638f50eee2d7788d10", - "size": "46112980" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.14.0/zig-linux-riscv64-0.14.0.tar.xz", - "shasum": "a2b14d3de326d3fd095548ef38bf5a67b15dadd62fbcc90836d63cc4355f8ef7", - "size": "48069188" - }, - "powerpc64le-linux": { - "tarball": "https://ziglang.org/download/0.14.0/zig-linux-powerpc64le-0.14.0.tar.xz", - "shasum": "3eabd60876ebc2748de8eb57b4b8cfa78861ba9bf7c6dd83f4e3e1d271d7c45e", - "size": "48707620" - }, - "x86-linux": { - "tarball": "https://ziglang.org/download/0.14.0/zig-linux-x86-0.14.0.tar.xz", - "shasum": "55d1ba21de5109686ffa675b9cc1dd66930093c202995a637ce3e397816e4c08", - "size": "51621460" - }, - "loongarch64-linux": { - "tarball": "https://ziglang.org/download/0.14.0/zig-linux-loongarch64-0.14.0.tar.xz", - "shasum": "31a2f07df55f8f528b92d540db9aae6c0b38643c34dc1ac33a0111d855e996ae", - "size": "45821860" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.14.0/zig-windows-x86_64-0.14.0.zip", - "shasum": "f53e5f9011ba20bbc3e0e6d0a9441b31eb227a97bac0e7d24172f1b8b27b4371", - "size": "82219809" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.14.0/zig-windows-aarch64-0.14.0.zip", - "shasum": "03e984383ebb8f85293557cfa9f48ee8698e7c400239570c9ff1aef3bffaf046", - "size": "78113283" - }, - "x86-windows": { - "tarball": "https://ziglang.org/download/0.14.0/zig-windows-x86-0.14.0.zip", - "shasum": "1a867d808cf4fa9184358395d94441390b6b24ee8d00d356ca11ea7cbfd3a4ec", - "size": "83970029" - } + "x86_64-linux": { + "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1458+755a3d957.tar.xz", + "shasum": "d1f819092c784fd821428501dac06387966179623ceea498be55a40e2fa8c8ef", + "size": "54753544" }, - "0.13.0": { - "date": "2024-06-07", - "docs": "https://ziglang.org/documentation/0.13.0/", - "stdDocs": "https://ziglang.org/documentation/0.13.0/std/", - "notes": "https://ziglang.org/download/0.13.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.13.0/zig-0.13.0.tar.xz", - "shasum": "06c73596beeccb71cc073805bdb9c0e05764128f16478fa53bf17dfabc1d4318", - "size": "17220728" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.13.0/zig-bootstrap-0.13.0.tar.xz", - "shasum": "cd446c084b5da7bc42e8ad9b4e1c910a957f2bf3f82bcc02888102cd0827c139", - "size": "46440356" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.13.0/zig-freebsd-x86_64-0.13.0.tar.xz", - "shasum": "adc1ffc9be56533b2f1c7191f9e435ad55db00414ff2829d951ef63d95aaad8c", - "size": "47177744" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.13.0/zig-macos-x86_64-0.13.0.tar.xz", - "shasum": "8b06ed1091b2269b700b3b07f8e3be3b833000841bae5aa6a09b1a8b4773effd", - "size": "48857012" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.13.0/zig-macos-aarch64-0.13.0.tar.xz", - "shasum": "46fae219656545dfaf4dce12fb4e8685cec5b51d721beee9389ab4194d43394c", - "size": "44892040" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.13.0/zig-linux-x86_64-0.13.0.tar.xz", - "shasum": "d45312e61ebcc48032b77bc4cf7fd6915c11fa16e4aad116b66c9468211230ea", - "size": "47082308" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.13.0/zig-linux-aarch64-0.13.0.tar.xz", - "shasum": "041ac42323837eb5624068acd8b00cd5777dac4cf91179e8dad7a7e90dd0c556", - "size": "43090688" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.13.0/zig-linux-armv7a-0.13.0.tar.xz", - "shasum": "4b0550239c2cd884cc03ddeb2b9934708f4b073ad59a96fccbfe09f7e4f54233", - "size": "43998916" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.13.0/zig-linux-riscv64-0.13.0.tar.xz", - "shasum": "9f7f3c685894ff80f43eaf3cad1598f4844ac46f4308374237c7f912f7907bb3", - "size": "45540956" - }, - "powerpc64le-linux": { - "tarball": "https://ziglang.org/download/0.13.0/zig-linux-powerpc64le-0.13.0.tar.xz", - "shasum": "6a467622448e830e8f85d20cabed151498af2b0a62f87b8c083b2fe127e60417", - "size": "46574596" - }, - "x86-linux": { - "tarball": "https://ziglang.org/download/0.13.0/zig-linux-x86-0.13.0.tar.xz", - "shasum": "876159cc1e15efb571e61843b39a2327f8925951d48b9a7a03048c36f72180f7", - "size": "52062336" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.13.0/zig-windows-x86_64-0.13.0.zip", - "shasum": "d859994725ef9402381e557c60bb57497215682e355204d754ee3df75ee3c158", - "size": "79163968" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.13.0/zig-windows-aarch64-0.13.0.zip", - "shasum": "95ff88427af7ba2b4f312f45d2377ce7a033e5e3c620c8caaa396a9aba20efda", - "size": "75119033" - }, - "x86-windows": { - "tarball": "https://ziglang.org/download/0.13.0/zig-windows-x86-0.13.0.zip", - "shasum": "eb3d533c3cf868bff7e74455dc005d18fd836c42e50b27106b31e9fec6dffc4a", - "size": "83274739" - } + "aarch64-linux": { + "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1458+755a3d957.tar.xz", + "shasum": "63f22a3ee1fc7a8214ae21264b9773f4553ea11b94104233d4c4946984d6c3be", + "size": "50495484" }, - "0.12.1": { - "date": "2024-06-08", - "docs": "https://ziglang.org/documentation/0.12.1/", - "stdDocs": "https://ziglang.org/documentation/0.12.1/std/", - "src": { - "tarball": "https://ziglang.org/download/0.12.1/zig-0.12.1.tar.xz", - "shasum": "cca0bf5686fe1a15405bd535661811fac7663f81664d2204ea4590ce49a6e9ba", - "size": "17110932" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.12.1/zig-bootstrap-0.12.1.tar.xz", - "shasum": "e533e2cb6ef60edda0ae3f2ca5c6504557db6e985e3c3a80159beb32279ed341", - "size": "45542004" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.12.1/zig-freebsd-x86_64-0.12.1.tar.xz", - "shasum": "30eaa28fa7bc21d01f88528d75ae4b392ae4970406675d5ac712a4937a605123", - "size": "45590080" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.12.1/zig-macos-x86_64-0.12.1.tar.xz", - "shasum": "68f309c6e431d56eb42648d7fe86e8028a23464d401a467831e27c26f1a8d9c9", - "size": "47202232" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.12.1/zig-macos-aarch64-0.12.1.tar.xz", - "shasum": "6587860dbbc070e1ee069e1a3d18ced83b7ba7a80bf67b2c57caf7c9ce5208b1", - "size": "43451512" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.12.1/zig-linux-x86_64-0.12.1.tar.xz", - "shasum": "8860fc9725c2d9297a63008f853e9b11e3c5a2441217f99c1e3104cc6fa4a443", - "size": "45512024" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.12.1/zig-linux-aarch64-0.12.1.tar.xz", - "shasum": "27d4fef393e8d8b5f3b1d19f4dd43bfdb469b4ed17bbc4c2283c1b1fe650ef7f", - "size": "41867324" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.12.1/zig-linux-armv7a-0.12.1.tar.xz", - "shasum": "27493c922fd1454137ed6cbe6b6bec00352838fcd32e6e74f4f9187011816157", - "size": "42665508" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.12.1/zig-linux-riscv64-0.12.1.tar.xz", - "shasum": "463511a863acc16911cff6801de97623e6de296aab7b15dcda5f6fd078b400b5", - "size": "43932908" - }, - "powerpc64le-linux": { - "tarball": "https://ziglang.org/download/0.12.1/zig-linux-powerpc64le-0.12.1.tar.xz", - "shasum": "462d6f10350f3c5d6fc4c9d6cfdede93d69d0103af026889a15f65c5de791d39", - "size": "45227716" - }, - "x86-linux": { - "tarball": "https://ziglang.org/download/0.12.1/zig-linux-x86-0.12.1.tar.xz", - "shasum": "c36ac019ca0fc3167e50d17e2affd3d072a06c519761737d0639adfdf2dcfddd", - "size": "50555428" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.12.1/zig-windows-x86_64-0.12.1.zip", - "shasum": "52459b147c2de4d7c28f6b1a4b3d571c114e96836bf8e31c953a7d2f5e94251c", - "size": "76470574" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.12.1/zig-windows-aarch64-0.12.1.zip", - "shasum": "e1286114a11be4695a6ad5cf0ba6a0e5f489bb3b029a5237de93598133f0c13a", - "size": "72998386" - }, - "x86-windows": { - "tarball": "https://ziglang.org/download/0.12.1/zig-windows-x86-0.12.1.zip", - "shasum": "4f0cc9258527e7b8bcf742772b3069122086a5cd857b38a1c08002462ac81f80", - "size": "80979711" - } + "arm-linux": { + "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.1458+755a3d957.tar.xz", + "shasum": "f3865ba1c3a19127f9a25962c919414c923c2c0bcddf060dd464f7fb5aefb044", + "size": "51360952" }, - "0.12.0": { - "date": "2024-04-20", - "docs": "https://ziglang.org/documentation/0.12.0/", - "stdDocs": "https://ziglang.org/documentation/0.12.0/std/", - "notes": "https://ziglang.org/download/0.12.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.12.0/zig-0.12.0.tar.xz", - "shasum": "a6744ef84b6716f976dad923075b2f54dc4f785f200ae6c8ea07997bd9d9bd9a", - "size": "17099152" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.12.0/zig-bootstrap-0.12.0.tar.xz", - "shasum": "3efc643d56421fa68072af94d5512cb71c61acf1c32512f77c0b4590bff63187", - "size": "45527312" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.12.0/zig-freebsd-x86_64-0.12.0.tar.xz", - "shasum": "bd49957d1157850b337ee1cf3c00af83585cff98e1ebc3c524a267e7422a2d7b", - "size": "45578364" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.12.0/zig-macos-x86_64-0.12.0.tar.xz", - "shasum": "4d411bf413e7667821324da248e8589278180dbc197f4f282b7dbb599a689311", - "size": "47185720" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.12.0/zig-macos-aarch64-0.12.0.tar.xz", - "shasum": "294e224c14fd0822cfb15a35cf39aa14bd9967867999bf8bdfe3db7ddec2a27f", - "size": "43447724" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.12.0/zig-linux-x86_64-0.12.0.tar.xz", - "shasum": "c7ae866b8a76a568e2d5cfd31fe89cdb629bdd161fdd5018b29a4a0a17045cad", - "size": "45480516" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.12.0/zig-linux-aarch64-0.12.0.tar.xz", - "shasum": "754f1029484079b7e0ca3b913a0a2f2a6afd5a28990cb224fe8845e72f09de63", - "size": "41849060" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.12.0/zig-linux-armv7a-0.12.0.tar.xz", - "shasum": "b48221f4c64416d257f0f9f77d8727dccf7de92aeabe59744ee6e70d650a97bc", - "size": "42638808" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.12.0/zig-linux-riscv64-0.12.0.tar.xz", - "shasum": "bb2d1a78b01595a9c00ffd2e12ab46e32f8b6798f76aec643ff78e5b4f5c5afd", - "size": "43917444" - }, - "powerpc64le-linux": { - "tarball": "https://ziglang.org/download/0.12.0/zig-linux-powerpc64le-0.12.0.tar.xz", - "shasum": "9218beecfb9250e9eff863f58f987dca7077e3258dd263c40269086127f9679b", - "size": "45216736" - }, - "x86-linux": { - "tarball": "https://ziglang.org/download/0.12.0/zig-linux-x86-0.12.0.tar.xz", - "shasum": "fb752fceb88749a80d625a6efdb23bea8208962b5150d6d14c92d20efda629a5", - "size": "50498940" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.12.0/zig-windows-x86_64-0.12.0.zip", - "shasum": "2199eb4c2000ddb1fba85ba78f1fcf9c1fb8b3e57658f6a627a8e513131893f5", - "size": "76442958" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.12.0/zig-windows-aarch64-0.12.0.zip", - "shasum": "04c6b92689241ca7a8a59b5f12d2ca2820c09d5043c3c4808b7e93e41c7bf97b", - "size": "72976876" - }, - "x86-windows": { - "tarball": "https://ziglang.org/download/0.12.0/zig-windows-x86-0.12.0.zip", - "shasum": "497dc9fd415cadf948872f137d6cc0870507488f79db9547b8f2adb73cda9981", - "size": "80950440" - } + "riscv64-linux": { + "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.1458+755a3d957.tar.xz", + "shasum": "8d83cf1bb3f81510c79a8e7859c79b113ae9d63cac610057c9483f1b89d716f5", + "size": "54551500" }, - "0.11.0": { - "date": "2023-08-04", - "docs": "https://ziglang.org/documentation/0.11.0/", - "stdDocs": "https://ziglang.org/documentation/0.11.0/std/", - "notes": "https://ziglang.org/download/0.11.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.11.0/zig-0.11.0.tar.xz", - "shasum": "72014e700e50c0d3528cef3adf80b76b26ab27730133e8202716a187a799e951", - "size": "15275316" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.11.0/zig-bootstrap-0.11.0.tar.xz", - "shasum": "38dd9e17433c7ce5687c48fa0a757462cbfcbe75d9d5087d14ebbe00efd21fdc", - "size": "43227592" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.11.0/zig-freebsd-x86_64-0.11.0.tar.xz", - "shasum": "ea430327f9178377b79264a1d492868dcff056cd76d43a6fb00719203749e958", - "size": "46432140" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.11.0/zig-macos-x86_64-0.11.0.tar.xz", - "shasum": "1c1c6b9a906b42baae73656e24e108fd8444bb50b6e8fd03e9e7a3f8b5f05686", - "size": "47189164" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.11.0/zig-macos-aarch64-0.11.0.tar.xz", - "shasum": "c6ebf927bb13a707d74267474a9f553274e64906fd21bf1c75a20bde8cadf7b2", - "size": "43855096" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.11.0/zig-linux-x86_64-0.11.0.tar.xz", - "shasum": "2d00e789fec4f71790a6e7bf83ff91d564943c5ee843c5fd966efc474b423047", - "size": "44961892" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.11.0/zig-linux-aarch64-0.11.0.tar.xz", - "shasum": "956eb095d8ba44ac6ebd27f7c9956e47d92937c103bf754745d0a39cdaa5d4c6", - "size": "41492432" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.11.0/zig-linux-armv7a-0.11.0.tar.xz", - "shasum": "aebe8bbeca39f13f9b7304465f9aee01ab005d243836bd40f4ec808093dccc9b", - "size": "42240664" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.11.0/zig-linux-riscv64-0.11.0.tar.xz", - "shasum": "24a478937eddb507e96d60bd4da00de9092b3f0920190eb45c4c99c946b00ed5", - "size": "43532324" - }, - "powerpc64le-linux": { - "tarball": "https://ziglang.org/download/0.11.0/zig-linux-powerpc64le-0.11.0.tar.xz", - "shasum": "75260e87325e820a278cf9e74f130c7b3d84c0b5197afb2e3c85eff3fcedd48d", - "size": "44656184" - }, - "powerpc-linux": { - "tarball": "https://ziglang.org/download/0.11.0/zig-linux-powerpc-0.11.0.tar.xz", - "shasum": "70a5f9668a66fb2a91a7c3488b15bcb568e1f9f44b95cd10075c138ad8c42864", - "size": "44539972" - }, - "x86-linux": { - "tarball": "https://ziglang.org/download/0.11.0/zig-linux-x86-0.11.0.tar.xz", - "shasum": "7b0dc3e0e070ae0e0d2240b1892af6a1f9faac3516cae24e57f7a0e7b04662a8", - "size": "49824456" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.11.0/zig-windows-x86_64-0.11.0.zip", - "shasum": "142caa3b804d86b4752556c9b6b039b7517a08afa3af842645c7e2dcd125f652", - "size": "77216743" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.11.0/zig-windows-aarch64-0.11.0.zip", - "shasum": "5d4bd13db5ecb0ddc749231e00f125c1d31087d708e9ff9b45c4f4e13e48c661", - "size": "73883137" - }, - "x86-windows": { - "tarball": "https://ziglang.org/download/0.11.0/zig-windows-x86-0.11.0.zip", - "shasum": "e72b362897f28c671633e650aa05289f2e62b154efcca977094456c8dac3aefa", - "size": "81576961" - } + "powerpc64le-linux": { + "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.1458+755a3d957.tar.xz", + "shasum": "92d3dba05203249fb9199abd0d937f16053253a8aa2eab5d76c3edeb84b46e02", + "size": "54579384" }, - "0.10.1": { - "date": "2023-01-19", - "docs": "https://ziglang.org/documentation/0.10.1/", - "stdDocs": "https://ziglang.org/documentation/0.10.1/std/", - "notes": "https://ziglang.org/download/0.10.1/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.10.1/zig-0.10.1.tar.xz", - "shasum": "69459bc804333df077d441ef052ffa143d53012b655a51f04cfef1414c04168c", - "size": "15143112" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.10.1/zig-bootstrap-0.10.1.tar.xz", - "shasum": "9f5781210b9be8f832553d160851635780f9bd71816065351ab29cfd8968f5e9", - "size": "43971816" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.10.1/zig-macos-x86_64-0.10.1.tar.xz", - "shasum": "02483550b89d2a3070c2ed003357fd6e6a3059707b8ee3fbc0c67f83ca898437", - "size": "45119596" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.10.1/zig-macos-aarch64-0.10.1.tar.xz", - "shasum": "b9b00477ec5fa1f1b89f35a7d2a58688e019910ab80a65eac2a7417162737656", - "size": "40517896" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.10.1/zig-linux-x86_64-0.10.1.tar.xz", - "shasum": "6699f0e7293081b42428f32c9d9c983854094bd15fee5489f12c4cf4518cc380", - "size": "44085596" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.10.1/zig-linux-aarch64-0.10.1.tar.xz", - "shasum": "db0761664f5f22aa5bbd7442a1617dd696c076d5717ddefcc9d8b95278f71f5d", - "size": "40321280" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.10.1/zig-linux-riscv64-0.10.1.tar.xz", - "shasum": "9db5b59a5112b8beb995094ba800e88b0060e9cf7cfadf4dc3e666c9010dc77b", - "size": "42196008" - }, - "i386-linux": { - "tarball": "https://ziglang.org/download/0.10.1/zig-linux-i386-0.10.1.tar.xz", - "shasum": "8c710ca5966b127b0ee3efba7310601ee57aab3dd6052a082ebc446c5efb2316", - "size": "48367388" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.10.1/zig-windows-x86_64-0.10.1.zip", - "shasum": "5768004e5e274c7969c3892e891596e51c5df2b422d798865471e05049988125", - "size": "73259729" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.10.1/zig-windows-aarch64-0.10.1.zip", - "shasum": "ece93b0d77b2ab03c40db99ef7ccbc63e0b6bd658af12b97898960f621305428", - "size": "69417459" - } + "x86-linux": { + "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.1458+755a3d957.tar.xz", + "shasum": "976b94b28079f9c083436fe2728d92f0fbf9353392a2b80945cf10c6b6faf42d", + "size": "57416272" }, - "0.10.0": { - "date": "2022-10-31", - "docs": "https://ziglang.org/documentation/0.10.0/", - "stdDocs": "https://ziglang.org/documentation/0.10.0/std/", - "notes": "https://ziglang.org/download/0.10.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.10.0/zig-0.10.0.tar.xz", - "shasum": "d8409f7aafc624770dcd050c8fa7e62578be8e6a10956bca3c86e8531c64c136", - "size": "14530912" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.10.0/zig-bootstrap-0.10.0.tar.xz", - "shasum": "c13dc70c4ff4c09f749adc0d473cbd3942991dd4d1bd2d860fbf257d8c1bbabf", - "size": "45625516" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.10.0/zig-freebsd-x86_64-0.10.0.tar.xz", - "shasum": "dd77afa2a8676afbf39f7d6068eda81b0723afd728642adaac43cb2106253d65", - "size": "44056504" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.10.0/zig-linux-aarch64-0.10.0.tar.xz", - "shasum": "09ef50c8be73380799804169197820ee78760723b0430fa823f56ed42b06ea0f", - "size": "40387688" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.10.0/zig-linux-armv7a-0.10.0.tar.xz", - "shasum": "7201b2e89cd7cc2dde95d39485fd7d5641ba67dc6a9a58c036cb4c308d2e82de", - "size": "50805936" - }, - "i386-linux": { - "tarball": "https://ziglang.org/download/0.10.0/zig-linux-i386-0.10.0.tar.xz", - "shasum": "dac8134f1328c50269f3e50b334298ec7916cb3b0ef76927703ddd1c96fd0115", - "size": "48451732" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.10.0/zig-linux-riscv64-0.10.0.tar.xz", - "shasum": "2a126f3401a7a7efc4b454f0a85c133db1af5a9dfee117f172213b7cbd47bfba", - "size": "42272968" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.10.0/zig-linux-x86_64-0.10.0.tar.xz", - "shasum": "631ec7bcb649cd6795abe40df044d2473b59b44e10be689c15632a0458ddea55", - "size": "44142400" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.10.0/zig-macos-aarch64-0.10.0.tar.xz", - "shasum": "02f7a7839b6a1e127eeae22ea72c87603fb7298c58bc35822a951479d53c7557", - "size": "40602664" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.10.0/zig-macos-x86_64-0.10.0.tar.xz", - "shasum": "3a22cb6c4749884156a94ea9b60f3a28cf4e098a69f08c18fbca81c733ebfeda", - "size": "45175104" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.10.0/zig-windows-x86_64-0.10.0.zip", - "shasum": "a66e2ff555c6e48781de1bcb0662ef28ee4b88af3af2a577f7b1950e430897ee", - "size": "73181558" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.10.0/zig-windows-aarch64-0.10.0.zip", - "shasum": "1bbda8d123d44f3ae4fa90d0da04b1e9093c3f9ddae3429a4abece1e1c0bf19a", - "size": "69332389" - } + "loongarch64-linux": { + "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.1458+755a3d957.tar.xz", + "shasum": "fd67f4c7930322862103fc930d328e0dd0626389611bd715377374c38845e5c0", + "size": "51808832" }, - "0.9.1": { - "date": "2022-02-14", - "docs": "https://ziglang.org/documentation/0.9.1/", - "stdDocs": "https://ziglang.org/documentation/0.9.1/std/", - "notes": "https://ziglang.org/download/0.9.1/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.9.1/zig-0.9.1.tar.xz", - "shasum": "38cf4e84481f5facc766ba72783e7462e08d6d29a5d47e3b75c8ee3142485210", - "size": "13940828" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.9.1/zig-bootstrap-0.9.1.tar.xz", - "shasum": "0a8e221c71860d8975c15662b3ed3bd863e81c4fe383455a596e5e0e490d6109", - "size": "42488812" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.9.1/zig-freebsd-x86_64-0.9.1.tar.xz", - "shasum": "4e06009bd3ede34b72757eec1b5b291b30aa0d5046dadd16ecb6b34a02411254", - "size": "39028848" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.9.1/zig-linux-aarch64-0.9.1.tar.xz", - "shasum": "5d99a39cded1870a3fa95d4de4ce68ac2610cca440336cfd252ffdddc2b90e66", - "size": "37034860" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.9.1/zig-linux-armv7a-0.9.1.tar.xz", - "shasum": "6de64456cb4757a555816611ea697f86fba7681d8da3e1863fa726a417de49be", - "size": "37974652" - }, - "i386-linux": { - "tarball": "https://ziglang.org/download/0.9.1/zig-linux-i386-0.9.1.tar.xz", - "shasum": "e776844fecd2e62fc40d94718891057a1dbca1816ff6013369e9a38c874374ca", - "size": "44969172" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.9.1/zig-linux-riscv64-0.9.1.tar.xz", - "shasum": "208dea53662c2c52777bd9e3076115d2126a4f71aed7f2ff3b8fe224dc3881aa", - "size": "39390868" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.9.1/zig-linux-x86_64-0.9.1.tar.xz", - "shasum": "be8da632c1d3273f766b69244d80669fe4f5e27798654681d77c992f17c237d7", - "size": "41011464" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.9.1/zig-macos-aarch64-0.9.1.tar.xz", - "shasum": "8c473082b4f0f819f1da05de2dbd0c1e891dff7d85d2c12b6ee876887d438287", - "size": "38995640" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.9.1/zig-macos-x86_64-0.9.1.tar.xz", - "shasum": "2d94984972d67292b55c1eb1c00de46580e9916575d083003546e9a01166754c", - "size": "43713044" - }, - "i386-windows": { - "tarball": "https://ziglang.org/download/0.9.1/zig-windows-i386-0.9.1.zip", - "shasum": "74a640ed459914b96bcc572183a8db687bed0af08c30d2ea2f8eba03ae930f69", - "size": "67929868" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.9.1/zig-windows-x86_64-0.9.1.zip", - "shasum": "443da53387d6ae8ba6bac4b3b90e9fef4ecbe545e1c5fa3a89485c36f5c0e3a2", - "size": "65047697" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.9.1/zig-windows-aarch64-0.9.1.zip", - "shasum": "621bf95f54dc3ff71466c5faae67479419951d7489e40e87fd26d195825fb842", - "size": "61478151" - } + "s390x-linux": { + "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.1458+755a3d957.tar.xz", + "shasum": "6e79f44ac4469163b572d1c26671aa8cb5c93cc544f5fc01cbdfde97cb320421", + "size": "54284392" }, - "0.9.0": { - "date": "2021-12-20", - "docs": "https://ziglang.org/documentation/0.9.0/", - "stdDocs": "https://ziglang.org/documentation/0.9.0/std/", - "notes": "https://ziglang.org/download/0.9.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.9.0/zig-0.9.0.tar.xz", - "shasum": "cd1be83b12f8269cc5965e59877b49fdd8fa638efb6995ac61eb4cea36a2e381", - "size": "13928772" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.9.0/zig-bootstrap-0.9.0.tar.xz", - "shasum": "16b0bdf0bc0a5ed1e0950e08481413d806192e06443a512347526647b2baeabc", - "size": "42557736" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.9.0/zig-freebsd-x86_64-0.9.0.tar.xz", - "shasum": "c95afe679b7cc4110dc2ecd3606c83a699718b7a958d6627f74c20886333e194", - "size": "41293236" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.9.0/zig-linux-aarch64-0.9.0.tar.xz", - "shasum": "1524fedfdbade2dbc9bae1ed98ad38fa7f2114c9a3e94da0d652573c75efbc5a", - "size": "40008396" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.9.0/zig-linux-armv7a-0.9.0.tar.xz", - "shasum": "50225dee6e6448a63ee96383a34d9fe3bba34ae8da1a0c8619bde2cdfc1df87d", - "size": "41196876" - }, - "i386-linux": { - "tarball": "https://ziglang.org/download/0.9.0/zig-linux-i386-0.9.0.tar.xz", - "shasum": "b0dcf688349268c883292acdd55eaa3c13d73b9146e4b990fad95b84a2ac528b", - "size": "47408656" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.9.0/zig-linux-riscv64-0.9.0.tar.xz", - "shasum": "85466de07504767ed37f59782672ad41bbdf43d6480fafd07f45543278b07620", - "size": "44171420" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.9.0/zig-linux-x86_64-0.9.0.tar.xz", - "shasum": "5c55344a877d557fb1b28939785474eb7f4f2f327aab55293998f501f7869fa6", - "size": "43420796" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.9.0/zig-macos-aarch64-0.9.0.tar.xz", - "shasum": "3991c70594d61d09fb4b316157a7c1d87b1d4ec159e7a5ecd11169ff74cad832", - "size": "39013392" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.9.0/zig-macos-x86_64-0.9.0.tar.xz", - "shasum": "c5280eeec4d6e5ea5ce5b448dc9a7c4bdd85ecfed4c1b96aa0835e48b36eccf0", - "size": "43764596" - }, - "i386-windows": { - "tarball": "https://ziglang.org/download/0.9.0/zig-windows-i386-0.9.0.zip", - "shasum": "bb839434afc75092015cf4c33319d31463c18512bc01dd719aedf5dcbc368466", - "size": "67946715" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.9.0/zig-windows-x86_64-0.9.0.zip", - "shasum": "084ea2646850aaf068234b0f1a92b914ed629be47075e835f8a67d55c21d880e", - "size": "65045849" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.9.0/zig-windows-aarch64-0.9.0.zip", - "shasum": "f9018725e3fb2e8992b17c67034726971156eb190685018a9ac8c3a9f7a22340", - "size": "61461921" - } + "x86_64-windows": { + "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1458+755a3d957.zip", + "shasum": "bd1b710cf4c928407453f4da0bbc253c4db147223cc5df101b1f2fb5de01f624", + "size": "94708257" }, - "0.8.1": { - "date": "2021-09-06", - "docs": "https://ziglang.org/documentation/0.8.1/", - "stdDocs": "https://ziglang.org/documentation/0.8.1/std/", - "notes": "https://ziglang.org/download/0.8.1/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.8.1/zig-0.8.1.tar.xz", - "shasum": "8c428e14a0a89cb7a15a6768424a37442292858cdb695e2eb503fa3c7bf47f1a", - "size": "12650228" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.8.1/zig-bootstrap-0.8.1.tar.xz", - "shasum": "fa1239247f830ecd51c42537043f5220e4d1dfefdc54356fa419616a0efb3902", - "size": "43613464" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.8.1/zig-freebsd-x86_64-0.8.1.tar.xz", - "shasum": "fc4f6478bcf3a9fce1b8ef677a91694f476dd35be6d6c9c4f44a8b76eedbe176", - "size": "39150924" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.8.1/zig-linux-aarch64-0.8.1.tar.xz", - "shasum": "2166dc9f2d8df387e8b4122883bb979d739281e1ff3f3d5483fec3a23b957510", - "size": "37605932" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.8.1/zig-linux-armv7a-0.8.1.tar.xz", - "shasum": "5ba58141805e2519f38cf8e715933cbf059f4f3dade92c71838cce341045de05", - "size": "39185876" - }, - "i386-linux": { - "tarball": "https://ziglang.org/download/0.8.1/zig-linux-i386-0.8.1.tar.xz", - "shasum": "2f3e84f30492b5f1c5f97cecc0166f07a8a8d50c5f85dbb3a6ef2a4ee6f915e6", - "size": "44782932" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.8.1/zig-linux-riscv64-0.8.1.tar.xz", - "shasum": "4adfaf147b025917c03367462fe5018aaa9edbc6439ef9cd0da2b074ae960554", - "size": "41234480" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.8.1/zig-linux-x86_64-0.8.1.tar.xz", - "shasum": "6c032fc61b5d77a3f3cf781730fa549f8f059ffdb3b3f6ad1c2994d2b2d87983", - "size": "41250060" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.8.1/zig-macos-aarch64-0.8.1.tar.xz", - "shasum": "5351297e3b8408213514b29c0a938002c5cf9f97eee28c2f32920e1227fd8423", - "size": "35340712" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.8.1/zig-macos-x86_64-0.8.1.tar.xz", - "shasum": "16b0e1defe4c1807f2e128f72863124bffdd906cefb21043c34b673bf85cd57f", - "size": "39946200" - }, - "i386-windows": { - "tarball": "https://ziglang.org/download/0.8.1/zig-windows-i386-0.8.1.zip", - "shasum": "099605051eb0452a947c8eab8fbbc7e43833c8376d267e94e41131c289a1c535", - "size": "64152358" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.8.1/zig-windows-x86_64-0.8.1.zip", - "shasum": "43573db14cd238f7111d6bdf37492d363f11ecd1eba802567a172f277d003926", - "size": "61897838" - } + "aarch64-windows": { + "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1458+755a3d957.zip", + "shasum": "e6a55c7ef706ecc8fe757a55d333620267b09339a1c7f3e08eb8e7e75bbb9ad9", + "size": "90682974" }, - "0.8.0": { - "date": "2021-06-04", - "docs": "https://ziglang.org/documentation/0.8.0/", - "stdDocs": "https://ziglang.org/documentation/0.8.0/std/", - "notes": "https://ziglang.org/download/0.8.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.8.0/zig-0.8.0.tar.xz", - "shasum": "03a828d00c06b2e3bb8b7ff706997fd76bf32503b08d759756155b6e8c981e77", - "size": "12614896" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.8.0/zig-bootstrap-0.8.0.tar.xz", - "shasum": "10600bc9c01f92e343f40d6ecc0ad05d67d27c3e382bce75524c0639cd8ca178", - "size": "43574248" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.8.0/zig-freebsd-x86_64-0.8.0.tar.xz", - "shasum": "0d3ccc436c8c0f50fd55462f72f8492d98723c7218ffc2a8a1831967d81b4bdc", - "size": "39125332" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.8.0/zig-linux-aarch64-0.8.0.tar.xz", - "shasum": "ee204ca2c2037952cf3f8b10c609373a08a291efa4af7b3c73be0f2b27720470", - "size": "37575428" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.8.0/zig-linux-armv7a-0.8.0.tar.xz", - "shasum": "d00b8bd97b79f45d6f5da956983bafeaa082e6c2ae8c6e1c6d4faa22fa29b320", - "size": "38884212" - }, - "i386-linux": { - "tarball": "https://ziglang.org/download/0.8.0/zig-linux-i386-0.8.0.tar.xz", - "shasum": "96e43ee6ed81c3c63401f456bd1c58ee6d42373a43cb324f5cf4974ca0998865", - "size": "42136032" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.8.0/zig-linux-riscv64-0.8.0.tar.xz", - "shasum": "75997527a78cdab64c40c43d9df39c01c4cdb557bb3992a869838371a204cfea", - "size": "40016268" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.8.0/zig-linux-x86_64-0.8.0.tar.xz", - "shasum": "502625d3da3ae595c5f44a809a87714320b7a40e6dff4a895b5fa7df3391d01e", - "size": "41211184" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.8.0/zig-macos-aarch64-0.8.0.tar.xz", - "shasum": "b32d13f66d0e1ff740b3326d66a469ee6baddbd7211fa111c066d3bd57683111", - "size": "35292180" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.8.0/zig-macos-x86_64-0.8.0.tar.xz", - "shasum": "279f9360b5cb23103f0395dc4d3d0d30626e699b1b4be55e98fd985b62bc6fbe", - "size": "39969312" - }, - "i386-windows": { - "tarball": "https://ziglang.org/download/0.8.0/zig-windows-i386-0.8.0.zip", - "shasum": "b6ec9aa6cd6f3872fcb30d43ff411802d82008a0c4142ee49e208a09b2c1c5fe", - "size": "61507213" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.8.0/zig-windows-x86_64-0.8.0.zip", - "shasum": "8580fbbf3afb72e9b495c7f8aeac752a03475ae0bbcf5d787f3775c7e1f4f807", - "size": "61766193" - } + "x86-windows": { + "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.1458+755a3d957.zip", + "shasum": "b53ab697230168494d626049d04858fb6834cc23737f879d2026cbf39cc214b2", + "size": "96547358" }, - "0.7.1": { - "date": "2020-12-13", - "docs": "https://ziglang.org/documentation/0.7.1/", - "stdDocs": "https://ziglang.org/documentation/0.7.1/std/", - "notes": "https://ziglang.org/download/0.7.1/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.7.1/zig-0.7.1.tar.xz", - "shasum": "2db3b944ab368d955b48743d9f7c963b8f96de1a441ba5a35e197237cc6dae44", - "size": "10711824" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.7.1/zig-bootstrap-0.7.1.tar.xz", - "shasum": "040f27c1fae4b0cac0a2782aecdb691f6a2f8e89db6a6ed35024c31c304fd9b2", - "size": "40232612" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.7.1/zig-freebsd-x86_64-0.7.1.tar.xz", - "shasum": "e73c1dca35791a3183fdd5ecde0443ebbe180942efceafe651886034fb8def09", - "size": "39066808" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.7.1/zig-linux-aarch64-0.7.1.tar.xz", - "shasum": "48ec90eba407e4587ddef7eecef25fec7e13587eb98e3b83c5f2f5fff2a5cbe7", - "size": "33780552" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.7.1/zig-linux-armv7a-0.7.1.tar.xz", - "shasum": "5a0662e07b4c4968665e1f97558f8591f6facec45d2e0ff5715e661743107ceb", - "size": "35813504" - }, - "i386-linux": { - "tarball": "https://ziglang.org/download/0.7.1/zig-linux-i386-0.7.1.tar.xz", - "shasum": "4882e052e5f83690bd0334bb4fc1702b5403cb3a3d2aa63fd7d6043d8afecba3", - "size": "39230912" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.7.1/zig-linux-riscv64-0.7.1.tar.xz", - "shasum": "187294bfd35983348c3fe042901b42e67e7e36ab7f77a5f969d21c0051f4d21f", - "size": "37454812" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.7.1/zig-linux-x86_64-0.7.1.tar.xz", - "shasum": "18c7b9b200600f8bcde1cd8d7f1f578cbc3676241ce36d771937ce19a8159b8d", - "size": "37848176" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.7.1/zig-macos-x86_64-0.7.1.tar.xz", - "shasum": "845cb17562978af0cf67e3993f4e33330525eaf01ead9386df9105111e3bc519", - "size": "36211076" - }, - "i386-windows": { - "tarball": "https://ziglang.org/download/0.7.1/zig-windows-i386-0.7.1.zip", - "shasum": "a1b9a7421e13153e07fd2e2c93ff29aad64d83105b8fcdafa633dbe689caf1c0", - "size": "54374983" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.7.1/zig-windows-x86_64-0.7.1.zip", - "shasum": "4818a8a65b4672bc52c0ae7f14d014e0eb8caf10f12c0745176820384cea296a", - "size": "54909997" - } + "aarch64-freebsd": { + "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.1458+755a3d957.tar.xz", + "shasum": "ce2a0974e6ffabff59592067f4d12dcc0c54a2074c06f7b6478e0f13f2042e00", + "size": "50450564" }, - "0.7.0": { - "date": "2020-11-08", - "docs": "https://ziglang.org/documentation/0.7.0/", - "stdDocs": "https://ziglang.org/documentation/0.7.0/std/", - "notes": "https://ziglang.org/download/0.7.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.7.0/zig-0.7.0.tar.xz", - "shasum": "0efd2cf6c3b05723db80e9cf193bc55150bba84ca41f855a90f53fc756445f83", - "size": "10683920" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.7.0/zig-bootstrap-0.7.0.tar.xz", - "shasum": "f073beaf5c53c8c57c0d374cbfcb332ef92ad703173edba0d9e0f2ed28401b72", - "size": "40200436" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.7.0/zig-freebsd-x86_64-0.7.0.tar.xz", - "shasum": "a0c926272ee4ae720034b4a6a1dc98399d76156dd84182554740f0ca8a41fc99", - "size": "34798992" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.7.0/zig-linux-aarch64-0.7.0.tar.xz", - "shasum": "f89933bac87d44be82325754ff88423020c81c7032a6fc41cfeb81e982eeab9b", - "size": "33096140" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.7.0/zig-linux-armv7a-0.7.0.tar.xz", - "shasum": "011c267e25a96ee160505a560c441daa045359a9d50e13ab1bada9d75c95db2d", - "size": "35157584" - }, - "i386-linux": { - "tarball": "https://ziglang.org/download/0.7.0/zig-linux-i386-0.7.0.tar.xz", - "shasum": "4bb2072cd363bcb1cbeb4872ff5cbc1f683b02d0cc1f90c46e3ea7422ce53222", - "size": "38530596" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.7.0/zig-linux-riscv64-0.7.0.tar.xz", - "shasum": "40dff81faa6f232ac40abbf88b9371f3cc932b6e09c423b94387c9ea580cb7be", - "size": "36759992" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.7.0/zig-linux-x86_64-0.7.0.tar.xz", - "shasum": "e619b1c6094c095b932767f527aee2507f847ea981513ff8a08aab0fd730e0ac", - "size": "37154432" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.7.0/zig-macos-aarch64-0.7.0.tar.xz", - "shasum": "338238035734db74ea4f30e500a4893bf741d38305c10952d5e39fa05bdb057d", - "size": "33739424" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.7.0/zig-macos-x86_64-0.7.0.tar.xz", - "shasum": "94063f9a311cbbf7a2e0a12295e09437182cf950f18cb0eb30ea9893f3677f24", - "size": "35258328" - }, - "i386-windows": { - "tarball": "https://ziglang.org/download/0.7.0/zig-windows-i386-0.7.0.zip", - "shasum": "b1e520aacbfbd645ff3521b3eb4d44166d9a0288b8725e4b001f8b50a425eb2e", - "size": "53390517" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.7.0/zig-windows-x86_64-0.7.0.zip", - "shasum": "965f56c0a36f9cda2125e3a348bc654f7f155e2804c3667d231775ec228f8553", - "size": "53943784" - } + "arm-freebsd": { + "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.1458+755a3d957.tar.xz", + "shasum": "ee9c87d8dcb336c807833488267c79ea27689ca86684a2cc1ce9fce2dcc3d373", + "size": "52049996" }, - "0.6.0": { - "date": "2020-04-13", - "docs": "https://ziglang.org/documentation/0.6.0/", - "stdDocs": "https://ziglang.org/documentation/0.6.0/std/", - "notes": "https://ziglang.org/download/0.6.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.6.0/zig-0.6.0.tar.xz", - "shasum": "5d167dc19354282dd35dd17b38e99e1763713b9be8a4ba9e9e69284e059e7204", - "size": "10349552" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.6.0/zig-bootstrap-0.6.0.tar.xz", - "shasum": "5e0e4dc878b3dd0c1852a442b174f0732e8c07869a8fcd226b71a93b89b381ab", - "size": "38469948" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.6.0/zig-freebsd-x86_64-0.6.0.tar.xz", - "shasum": "190ff79c1eb56805a315d7c7a51082e32f62926250c0702b36760c225e1634a3", - "size": "36974604" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.6.0/zig-linux-aarch64-0.6.0.tar.xz", - "shasum": "e7520efd42cfa02be48c2e430d08fe1f3cbb999d21d9f0d3ffd0febb976b2f41", - "size": "37090044" - }, - "armv6kz-linux": { - "tarball": "https://ziglang.org/download/0.6.0/zig-linux-armv6kz-0.6.0.tar.xz", - "shasum": "36b6493b3fed43eb1f0000e765798ad31a6bb7d7fd3f553ac1c3761dbc919b82", - "size": "39133452" - }, - "armv7a-linux": { - "tarball": "https://ziglang.org/download/0.6.0/zig-linux-armv7a-0.6.0.tar.xz", - "shasum": "946969abe357def95ca9cbbfcebfcf2d90cf967bcd3f48ee87662e32d91d8f35", - "size": "39143748" - }, - "i386-linux": { - "tarball": "https://ziglang.org/download/0.6.0/zig-linux-i386-0.6.0.tar.xz", - "shasum": "a97a2f9ae21575743cdd763c1917d49400d83fc562ef64582b18bade43eb24ce", - "size": "44877640" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.6.0/zig-linux-riscv64-0.6.0.tar.xz", - "shasum": "68ddee43f7503c8ae5f26a921f3602c34719a02ed2241f528c0b8b888cc14b38", - "size": "41993144" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.6.0/zig-linux-x86_64-0.6.0.tar.xz", - "shasum": "08fd3c757963630645441c2772362e9c2294020c44f14fce1b89f45de0dc1253", - "size": "44766320" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.6.0/zig-macos-x86_64-0.6.0.tar.xz", - "shasum": "17270360e87ddc49f737e760047b2fac49f1570a824a306119b1194ac4093895", - "size": "42573184" - }, - "i386-windows": { - "tarball": "https://ziglang.org/download/0.6.0/zig-windows-i386-0.6.0.zip", - "shasum": "3b0a02618743e92175990dc6d1a787bb95ff62c4cda016f1c14c7786f575f8ca", - "size": "60446431" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.6.0/zig-windows-x86_64-0.6.0.zip", - "shasum": "c3b897832523e1026e10b2d8d55d7f895185c0a27a63681f3a23219c3f1c38f4", - "size": "49065511" - } + "powerpc64-freebsd": { + "tarball": "https://ziglang.org/builds/zig-powerpc64-freebsd-0.16.0-dev.1458+755a3d957.tar.xz", + "shasum": "93956cdcd452994be83a1418ce9901f7a2b5f92768103663b2c2d2e39250260b", + "size": "53097748" }, - "0.5.0": { - "date": "2019-09-30", - "docs": "https://ziglang.org/documentation/0.5.0/", - "notes": "https://ziglang.org/download/0.5.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.5.0/zig-0.5.0.tar.xz", - "shasum": "55ae16960f152bcb9cf98b4f8570902d0e559a141abf927f0d3555b7cc838a31", - "size": "10956132" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.5.0/zig-freebsd-x86_64-0.5.0.tar.xz", - "shasum": "9e1f4d36c3d584c0aa01f20eb4cd0a0eef3eee5af23e483b8414de55feab6ab6", - "size": "33650744" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.5.0/zig-macos-x86_64-0.5.0.tar.xz", - "shasum": "28702cc05745c7c0bd450487d5f4091bf0a1ad279b35eb9a640ce3e3a15b300d", - "size": "37898664" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.5.0/zig-windows-x86_64-0.5.0.zip", - "shasum": "58141323db8d84a5af62746be5f9140bc161ee760ef33dc91a887bf9ac021976", - "size": "44871804" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.5.0/zig-linux-x86_64-0.5.0.tar.xz", - "shasum": "43e8f8a8b8556edd373ddf9c1ef3ca6cf852d4d09fe07d5736d12fefedd2b4f7", - "size": "40895068" - } + "powerpc64le-freebsd": { + "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.1458+755a3d957.tar.xz", + "shasum": "95f3b2b50a87dcf62c83c5312a5296b223fffff614f63b463f9a8b9e8d04d117", + "size": "54573692" }, - "0.4.0": { - "date": "2019-04-08", - "docs": "https://ziglang.org/documentation/0.4.0/", - "notes": "https://ziglang.org/download/0.4.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.4.0/zig-0.4.0.tar.xz", - "shasum": "fec1f3f6b359a3d942e0a7f9157b3b30cde83927627a0e1ea95c54de3c526cfc", - "size": "5348776" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.4.0/zig-freebsd-x86_64-0.4.0.tar.xz", - "shasum": "3d557c91ac36d8262eb1733bb5f261c95944f9b635e43386e3d00a3272818c30", - "size": "27269672" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.4.0/zig-macos-x86_64-0.4.0.tar.xz", - "shasum": "67c932982484d017c5111e54af9f33f15e8e05c6bc5346a55e04052159c964a8", - "size": "30841504" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.4.0/zig-windows-x86_64-0.4.0.zip", - "shasum": "fbc3dd205e064c263063f69f600bedb18e3d0aa2efa747a63ef6cafb6d73f127", - "size": "35800101" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.4.0/zig-linux-x86_64-0.4.0.tar.xz", - "shasum": "fb1954e2fb556a01f8079a08130e88f70084e08978ff853bb2b1986d8c39d84e", - "size": "32876100" - } + "riscv64-freebsd": { + "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.1458+755a3d957.tar.xz", + "shasum": "6eae893b95b2ebe6696a1d87dafd3737a30692bde1c2878dbf3180401e15c195", + "size": "54698508" }, - "0.3.0": { - "date": "2018-09-28", - "docs": "https://ziglang.org/documentation/0.3.0/", - "notes": "https://ziglang.org/download/0.3.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.3.0/zig-0.3.0.tar.xz", - "shasum": "d70af604f3a8622f3393d93abb3e056bf60351e32d121e6fa4fe03d8d41e1f5a", - "size": "2335592" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.3.0/zig-macos-x86_64-0.3.0.tar.xz", - "shasum": "19dec1f1943ab7be26823376d466f7e456143deb34e17502778a949034dc2e7e", - "size": "23712696" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.3.0/zig-windows-x86_64-0.3.0.zip", - "shasum": "bb568c03950958f8bb3472139c3ab5ed74547c8c694ab50f404c202faf51baf4", - "size": "22524425" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.3.0/zig-linux-x86_64-0.3.0.tar.xz", - "shasum": "b378d0aae30cb54f28494e7bc4efbc9bfb6326f47bfb302e8b5287af777b2f3c", - "size": "25209304" - } + "x86_64-freebsd": { + "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.1458+755a3d957.tar.xz", + "shasum": "3637c94591bcabfd403650a75e77bf799fd8a3adbaa00e1e678aa2cb1eff2d81", + "size": "54845396" }, - "0.2.0": { - "date": "2018-03-15", - "docs": "https://ziglang.org/documentation/0.2.0/", - "notes": "https://ziglang.org/download/0.2.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.2.0/zig-0.2.0.tar.xz", - "shasum": "29c9beb172737f4d5019b88ceae829ae8bc6512fb4386cfbf895ae2b42aa6965", - "size": "1940832" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.2.0/zig-win64-0.2.0.zip", - "shasum": "4f8a2979941a1f081ec8e545cca0b72608c0db1c5a3fd377a94db40649dcd3d4", - "size": "21076274" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.2.0/zig-linux-x86_64-0.2.0.tar.xz", - "shasum": "209c6fb745d42474c0a73d6f291c7ae3a38b6a1b6b641eea285a7f840cc1a890", - "size": "22551928" - } + "aarch64-netbsd": { + "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.1458+755a3d957.tar.xz", + "shasum": "9d69e952a4a8de168b5b8055979cfd2e54efbf71427c8f6f70deacf82dfdc4d1", + "size": "50389780" }, - "0.1.1": { - "date": "2017-10-17", - "docs": "https://ziglang.org/documentation/0.1.1/", - "notes": "https://ziglang.org/download/0.1.1/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.1.1/zig-0.1.1.tar.xz", - "shasum": "ffca0cfb263485287e19cc997b08701fcd5f24b700345bcdc3dd8074f5a104e0", - "size": "1659716" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.1.1/zig-win64-0.1.1.zip", - "shasum": "6fc88bef531af7e567fe30bf60da1487b86833cbee84c7a2f3e317030aa5b660", - "size": "19757776" - } + "arm-netbsd": { + "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.1458+755a3d957.tar.xz", + "shasum": "0f5b6bbc1749aa0491cc865c49bd6c714db2d7749f8732c8d2107409cbbb83c7", + "size": "53090212" + }, + "x86-netbsd": { + "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.1458+755a3d957.tar.xz", + "shasum": "b51b0d9594743dcf24b15009d815f87ceda27833adcac02dd96c679daedfba52", + "size": "58031544" + }, + "x86_64-netbsd": { + "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.1458+755a3d957.tar.xz", + "shasum": "f27f5b71ed7e35a4c22f279d1a2823db3d72b9d821d510465be1f0666427de64", + "size": "54795056" } + }, + "0.15.2": { + "version": "0.15.2", + "date": "2025-10-11", + "docs": "https://ziglang.org/documentation/0.15.2/", + "stdDocs": "https://ziglang.org/documentation/0.15.2/std/", + "src": { + "tarball": "https://ziglang.org/download/0.15.2/zig-0.15.2.tar.xz", + "shasum": "d9b30c7aa983fcff5eed2084d54ae83eaafe7ff3a84d8fb754d854165a6e521c", + "size": "21366268" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.15.2/zig-bootstrap-0.15.2.tar.xz", + "shasum": "a6845459501df3c3264ebc587b02a7094ad14f4f3f7287c48f04457e784d0d85", + "size": "52718988" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-macos-0.15.2.tar.xz", + "shasum": "375b6909fc1495d16fc2c7db9538f707456bfc3373b14ee83fdd3e22b3d43f7f", + "size": "55800460" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-macos-0.15.2.tar.xz", + "shasum": "3cc2bab367e185cdfb27501c4b30b1b0653c28d9f73df8dc91488e66ece5fa6b", + "size": "50635984" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-linux-0.15.2.tar.xz", + "shasum": "02aa270f183da276e5b5920b1dac44a63f1a49e55050ebde3aecc9eb82f93239", + "size": "53733924" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-linux-0.15.2.tar.xz", + "shasum": "958ed7d1e00d0ea76590d27666efbf7a932281b3d7ba0c6b01b0ff26498f667f", + "size": "49471996" + }, + "arm-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-arm-linux-0.15.2.tar.xz", + "shasum": "7d8401495065dae45d6249c68d5faf10508f8203c86362ccb698aeaafc66b7cd", + "size": "50454216" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-riscv64-linux-0.15.2.tar.xz", + "shasum": "493512bdca485be3c6a9b0f69dcb4cbe4587f3af8e1be282fdd827108ba39930", + "size": "53613656" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-powerpc64le-linux-0.15.2.tar.xz", + "shasum": "e182c5f8d30fc7f97d17d2ffef1488826aa3afaa51e5f0dbe14c597a98b45778", + "size": "53570632" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86-linux-0.15.2.tar.xz", + "shasum": "4c6e23f39daa305e274197bfdff0d56ffd1750fc1de226ae10505c0eff52d7a5", + "size": "56334032" + }, + "loongarch64-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-loongarch64-linux-0.15.2.tar.xz", + "shasum": "95d6a03e0415dbe30ae9c00d58b5446fd681ebf3d4044a8b707e1784c8074a61", + "size": "50800332" + }, + "s390x-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-s390x-linux-0.15.2.tar.xz", + "shasum": "cd4b295f43b1b7ca3d603368caea82dbd00fce6691affb7527e607c0b6eafe8e", + "size": "53486436" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-windows-0.15.2.zip", + "shasum": "3a0ed1e8799a2f8ce2a6e6290a9ff22e6906f8227865911fb7ddedc3cc14cb0c", + "size": "92614574" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-windows-0.15.2.zip", + "shasum": "b926465f8872bf983422257cd9ec248bb2b270996fbe8d57872cca13b56fc370", + "size": "88464585" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86-windows-0.15.2.zip", + "shasum": "7a6dfc00f4cc09ec46d3e10eb06f42538e92b6285e34debea7462edaf371da98", + "size": "94516930" + }, + "aarch64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-freebsd-0.15.2.tar.xz", + "shasum": "c62efd319f86663eb7747709dfca259205edba8eaee98efc96a51ce40a9437de", + "size": "49363332" + }, + "arm-freebsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-arm-freebsd-0.15.2.tar.xz", + "shasum": "1d5d06428b2d4d31cf4e99008602f2096a9c652dd1025820a4aa66326fbf3dc4", + "size": "50900824" + }, + "powerpc64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-powerpc64-freebsd-0.15.2.tar.xz", + "shasum": "2bdb5a38069e667936d3378dfa332d67c21bb454cb17bea3c3b03faaa02a74af", + "size": "52099240" + }, + "powerpc64le-freebsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-powerpc64le-freebsd-0.15.2.tar.xz", + "shasum": "940894eb954c2077d0643f4deba8113239436fd0dcbc88160163b1b23fe3f035", + "size": "53459116" + }, + "riscv64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-riscv64-freebsd-0.15.2.tar.xz", + "shasum": "9aecdcf32fb63944c11389c10993aecd5e443a82d456693cc5ad230b5fe584ad", + "size": "53658752" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-freebsd-0.15.2.tar.xz", + "shasum": "5509ff57cd3f219165caed0da10221739af82742b9edfcda3f7bfaf4da7212dd", + "size": "53776208" + }, + "aarch64-netbsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-netbsd-0.15.2.tar.xz", + "shasum": "357c1e62df6581ec1726546f9a827973233cb31ac371d2b0e92456b306a0c6c0", + "size": "49365256" + }, + "arm-netbsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-arm-netbsd-0.15.2.tar.xz", + "shasum": "1c43871791d49dbabd224da307b995076e493e34c36734cb21c4f6cbc208f8c3", + "size": "52021696" + }, + "x86-netbsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86-netbsd-0.15.2.tar.xz", + "shasum": "7f18fc4720426a6ca77bbfb7b970c31a61c37ce4a867b53830762958e921cdde", + "size": "56893368" + }, + "x86_64-netbsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-netbsd-0.15.2.tar.xz", + "shasum": "11a3b9f4b66bece34a79c0e98e91136baf0ff9795ddf60e442ca6c8ad73b9204", + "size": "53778132" + } + }, + "0.15.1": { + "date": "2025-08-19", + "docs": "https://ziglang.org/documentation/0.15.1/", + "stdDocs": "https://ziglang.org/documentation/0.15.1/std/", + "notes": "https://ziglang.org/download/0.15.1/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.15.1/zig-0.15.1.tar.xz", + "shasum": "816c0303ab313f59766ce2097658c9fff7fafd1504f61f80f9507cd11652865f", + "size": "21359884" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.15.1/zig-bootstrap-0.15.1.tar.xz", + "shasum": "4c0cfbcf12da144955761ca43f89e3c74956bce978694fc1d0a63555f5c0a199", + "size": "52711548" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-macos-0.15.1.tar.xz", + "shasum": "9919392e0287cccc106dfbcbb46c7c1c3fa05d919567bb58d7eb16bca4116184", + "size": "55791880" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-macos-0.15.1.tar.xz", + "shasum": "c4bd624d901c1268f2deb9d8eb2d86a2f8b97bafa3f118025344242da2c54d7b", + "size": "50644996" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-linux-0.15.1.tar.xz", + "shasum": "c61c5da6edeea14ca51ecd5e4520c6f4189ef5250383db33d01848293bfafe05", + "size": "53734456" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-linux-0.15.1.tar.xz", + "shasum": "bb4a8d2ad735e7fba764c497ddf4243cb129fece4148da3222a7046d3f1f19fe", + "size": "49493872" + }, + "arm-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-arm-linux-0.15.1.tar.xz", + "shasum": "3f4bf3b06b67d14e3f38be30798488c1abe3cf5b33de570cd0e87bbf09b978ad", + "size": "50477464" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-riscv64-linux-0.15.1.tar.xz", + "shasum": "7ca7a3e621436fb31d66a253132fc39574a13d2a1b4d8458af4f2e7c6e4374fe", + "size": "53597792" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-powerpc64le-linux-0.15.1.tar.xz", + "shasum": "339e2106496be70b614e32d444298216e676c36d08f5cd7bee3dd1dbd4567fd7", + "size": "53566944" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86-linux-0.15.1.tar.xz", + "shasum": "dff166f25fdd06e8341d831a71211b5ba7411463a6b264bdefa8868438690b6a", + "size": "56311228" + }, + "loongarch64-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-loongarch64-linux-0.15.1.tar.xz", + "shasum": "0af18a012f3c4cffbef29ab5f42021484d92517f921a1380faacc89c50c1f89d", + "size": "50794376" + }, + "s390x-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-s390x-linux-0.15.1.tar.xz", + "shasum": "bcd13e5c88cf2d0da7100a48572195560069a94402c9bec3b316398204aa27e2", + "size": "53508068" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-windows-0.15.1.zip", + "shasum": "91e69e887ca8c943ce9a515df3af013d95a66a190a3df3f89221277ebad29e34", + "size": "92612958" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-windows-0.15.1.zip", + "shasum": "1f1bf16228b0ffcc882b713dc5e11a6db4219cb30997e13c72e8e723c2104ec6", + "size": "88458549" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86-windows-0.15.1.zip", + "shasum": "fb1c07cffbb43615d3158ab8b8f5db5da1d48875eca99e1d7a8a0064ff63fc5b", + "size": "94516052" + }, + "aarch64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-freebsd-0.15.1.tar.xz", + "shasum": "4d9d25c775828d49ea037b2284310c295d951793da8ebe94827a54fed4cca3ce", + "size": "49358464" + }, + "arm-freebsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-arm-freebsd-0.15.1.tar.xz", + "shasum": "9707f3a5f7e1a3d99c40db9a74de1acc61016a197ad289c2ad964f93cb213a18", + "size": "50904124" + }, + "powerpc64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-powerpc64-freebsd-0.15.1.tar.xz", + "shasum": "79448884372db04e62f77a46b92245fce805063e69534729537f75cb4681e7e3", + "size": "52099020" + }, + "powerpc64le-freebsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-powerpc64le-freebsd-0.15.1.tar.xz", + "shasum": "f18ee12ba9c98a20b8d2ad0410c679e7aa5591bc7917f169fd6b377833d2c7ad", + "size": "53480976" + }, + "riscv64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-riscv64-freebsd-0.15.1.tar.xz", + "shasum": "ee9f864a6fd8b57c1f4fdbb11daa06578746a6f8253afe3f5ddb5a76f2eddd2d", + "size": "53677800" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-freebsd-0.15.1.tar.xz", + "shasum": "9714f8ac3d3dc908b1599837c6167f857c1efaa930f0cfa840699458de7c3cd0", + "size": "53782112" + }, + "aarch64-netbsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-netbsd-0.15.1.tar.xz", + "shasum": "b2a528399777583b85b89c54ccd45488af7709d6dd29a27323ec2a229db40910", + "size": "49368000" + }, + "arm-netbsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-arm-netbsd-0.15.1.tar.xz", + "shasum": "93dc70109cbf5d2e022d20dfb56211978c4ea3c0b1e67aaabff947d8d1583aab", + "size": "52028184" + }, + "x86-netbsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86-netbsd-0.15.1.tar.xz", + "shasum": "a91b26051822ff17f3143f859b87dce5b4a13e90928bd6daa6f07a895d3410f0", + "size": "56881864" + }, + "x86_64-netbsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-netbsd-0.15.1.tar.xz", + "shasum": "6d7ba6eca5b4434351ebdb971b7303c9934514f9bb8481852251dbd5b52b03d6", + "size": "53794836" + } + }, + "0.14.1": { + "date": "2025-05-21", + "docs": "https://ziglang.org/documentation/0.14.1/", + "stdDocs": "https://ziglang.org/documentation/0.14.1/std/", + "src": { + "tarball": "https://ziglang.org/download/0.14.1/zig-0.14.1.tar.xz", + "shasum": "237f8abcc8c3fd68c70c66cdbf63dce4fb5ad4a2e6225ac925e3d5b4c388f203", + "size": "17787696" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.14.1/zig-bootstrap-0.14.1.tar.xz", + "shasum": "89b2fce50bfbb1eee29c382193d22c6eb0c7da3a96b5ba6d05e0af2945b3ca3d", + "size": "48041028" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.14.1/zig-x86_64-macos-0.14.1.tar.xz", + "shasum": "b0f8bdfb9035783db58dd6c19d7dea89892acc3814421853e5752fe4573e5f43", + "size": "51044512" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.14.1/zig-aarch64-macos-0.14.1.tar.xz", + "shasum": "39f3dc5e79c22088ce878edc821dedb4ca5a1cd9f5ef915e9b3cc3053e8faefa", + "size": "45903552" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.14.1/zig-x86_64-linux-0.14.1.tar.xz", + "shasum": "24aeeec8af16c381934a6cd7d95c807a8cb2cf7df9fa40d359aa884195c4716c", + "size": "49086504" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.14.1/zig-aarch64-linux-0.14.1.tar.xz", + "shasum": "f7a654acc967864f7a050ddacfaa778c7504a0eca8d2b678839c21eea47c992b", + "size": "44954692" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.14.1/zig-armv7a-linux-0.14.1.tar.xz", + "shasum": "1b34d9ecfaeb3b360e86c0bc233e1a8a2bbed2d40f2d4f20c12bde2128714324", + "size": "46137456" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.14.1/zig-riscv64-linux-0.14.1.tar.xz", + "shasum": "005f214f74dbafb7b4d8bd305f4e9d25048f711d9ec6fa7b3d4fca177e11b882", + "size": "48094380" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.14.1/zig-powerpc64le-linux-0.14.1.tar.xz", + "shasum": "15523e748efd9224ae164482e5cc1f8c11b035246b9749fb5b00758484f384a1", + "size": "48720956" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.14.1/zig-x86-linux-0.14.1.tar.xz", + "shasum": "4bce6347fa112247443cb0952c19e560d1f90b910506cf895fd07a7b8d1c4a76", + "size": "51643520" + }, + "loongarch64-linux": { + "tarball": "https://ziglang.org/download/0.14.1/zig-loongarch64-linux-0.14.1.tar.xz", + "shasum": "fdc344789be6becbe220bf5ad035156e653cc148f4da270188cbac87729f17e0", + "size": "45825404" + }, + "s390x-linux": { + "tarball": "https://ziglang.org/download/0.14.1/zig-s390x-linux-0.14.1.tar.xz", + "shasum": "8e1bb63532ba22725f0a4a45e5c2920a8af508d8ca81e6007c330eec2b7033a6", + "size": "55602032" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.14.1/zig-x86_64-windows-0.14.1.zip", + "shasum": "554f5378228923ffd558eac35e21af020c73789d87afeabf4bfd16f2e6feed2c", + "size": "82229343" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.14.1/zig-aarch64-windows-0.14.1.zip", + "shasum": "b5aac0ccc40dd91e8311b1f257717d8e3903b5fefb8f659de6d65a840ad1d0e7", + "size": "78125379" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.14.1/zig-x86-windows-0.14.1.zip", + "shasum": "3ee730c2a5523570dc4dc1b724f3e4f30174ebc1fa109ca472a719586a473b18", + "size": "83983932" + } + }, + "0.14.0": { + "date": "2025-03-05", + "docs": "https://ziglang.org/documentation/0.14.0/", + "stdDocs": "https://ziglang.org/documentation/0.14.0/std/", + "notes": "https://ziglang.org/download/0.14.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.14.0/zig-0.14.0.tar.xz", + "shasum": "c76638c03eb204c4432ae092f6fa07c208567e110fbd4d862d131a7332584046", + "size": "17772188" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.14.0/zig-bootstrap-0.14.0.tar.xz", + "shasum": "bf3fcb22be0b83f4791748adb567d3304779d66d7bf9b1bd557ef6c2e0232807", + "size": "48029040" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.14.0/zig-macos-x86_64-0.14.0.tar.xz", + "shasum": "685816166f21f0b8d6fc7aa6a36e91396dcd82ca6556dfbe3e329deffc01fec3", + "size": "51039964" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.14.0/zig-macos-aarch64-0.14.0.tar.xz", + "shasum": "b71e4b7c4b4be9953657877f7f9e6f7ee89114c716da7c070f4a238220e95d7e", + "size": "45902412" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.14.0/zig-linux-x86_64-0.14.0.tar.xz", + "shasum": "473ec26806133cf4d1918caf1a410f8403a13d979726a9045b421b685031a982", + "size": "49091960" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.14.0/zig-linux-aarch64-0.14.0.tar.xz", + "shasum": "ab64e3ea277f6fc5f3d723dcd95d9ce1ab282c8ed0f431b4de880d30df891e4f", + "size": "44922728" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.14.0/zig-linux-armv7a-0.14.0.tar.xz", + "shasum": "a67dbfa9bdf769228ec994f2098698c619f930883ca5ef638f50eee2d7788d10", + "size": "46112980" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.14.0/zig-linux-riscv64-0.14.0.tar.xz", + "shasum": "a2b14d3de326d3fd095548ef38bf5a67b15dadd62fbcc90836d63cc4355f8ef7", + "size": "48069188" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.14.0/zig-linux-powerpc64le-0.14.0.tar.xz", + "shasum": "3eabd60876ebc2748de8eb57b4b8cfa78861ba9bf7c6dd83f4e3e1d271d7c45e", + "size": "48707620" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.14.0/zig-linux-x86-0.14.0.tar.xz", + "shasum": "55d1ba21de5109686ffa675b9cc1dd66930093c202995a637ce3e397816e4c08", + "size": "51621460" + }, + "loongarch64-linux": { + "tarball": "https://ziglang.org/download/0.14.0/zig-linux-loongarch64-0.14.0.tar.xz", + "shasum": "31a2f07df55f8f528b92d540db9aae6c0b38643c34dc1ac33a0111d855e996ae", + "size": "45821860" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.14.0/zig-windows-x86_64-0.14.0.zip", + "shasum": "f53e5f9011ba20bbc3e0e6d0a9441b31eb227a97bac0e7d24172f1b8b27b4371", + "size": "82219809" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.14.0/zig-windows-aarch64-0.14.0.zip", + "shasum": "03e984383ebb8f85293557cfa9f48ee8698e7c400239570c9ff1aef3bffaf046", + "size": "78113283" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.14.0/zig-windows-x86-0.14.0.zip", + "shasum": "1a867d808cf4fa9184358395d94441390b6b24ee8d00d356ca11ea7cbfd3a4ec", + "size": "83970029" + } + }, + "0.13.0": { + "date": "2024-06-07", + "docs": "https://ziglang.org/documentation/0.13.0/", + "stdDocs": "https://ziglang.org/documentation/0.13.0/std/", + "notes": "https://ziglang.org/download/0.13.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.13.0/zig-0.13.0.tar.xz", + "shasum": "06c73596beeccb71cc073805bdb9c0e05764128f16478fa53bf17dfabc1d4318", + "size": "17220728" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.13.0/zig-bootstrap-0.13.0.tar.xz", + "shasum": "cd446c084b5da7bc42e8ad9b4e1c910a957f2bf3f82bcc02888102cd0827c139", + "size": "46440356" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.13.0/zig-freebsd-x86_64-0.13.0.tar.xz", + "shasum": "adc1ffc9be56533b2f1c7191f9e435ad55db00414ff2829d951ef63d95aaad8c", + "size": "47177744" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.13.0/zig-macos-x86_64-0.13.0.tar.xz", + "shasum": "8b06ed1091b2269b700b3b07f8e3be3b833000841bae5aa6a09b1a8b4773effd", + "size": "48857012" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.13.0/zig-macos-aarch64-0.13.0.tar.xz", + "shasum": "46fae219656545dfaf4dce12fb4e8685cec5b51d721beee9389ab4194d43394c", + "size": "44892040" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.13.0/zig-linux-x86_64-0.13.0.tar.xz", + "shasum": "d45312e61ebcc48032b77bc4cf7fd6915c11fa16e4aad116b66c9468211230ea", + "size": "47082308" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.13.0/zig-linux-aarch64-0.13.0.tar.xz", + "shasum": "041ac42323837eb5624068acd8b00cd5777dac4cf91179e8dad7a7e90dd0c556", + "size": "43090688" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.13.0/zig-linux-armv7a-0.13.0.tar.xz", + "shasum": "4b0550239c2cd884cc03ddeb2b9934708f4b073ad59a96fccbfe09f7e4f54233", + "size": "43998916" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.13.0/zig-linux-riscv64-0.13.0.tar.xz", + "shasum": "9f7f3c685894ff80f43eaf3cad1598f4844ac46f4308374237c7f912f7907bb3", + "size": "45540956" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.13.0/zig-linux-powerpc64le-0.13.0.tar.xz", + "shasum": "6a467622448e830e8f85d20cabed151498af2b0a62f87b8c083b2fe127e60417", + "size": "46574596" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.13.0/zig-linux-x86-0.13.0.tar.xz", + "shasum": "876159cc1e15efb571e61843b39a2327f8925951d48b9a7a03048c36f72180f7", + "size": "52062336" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.13.0/zig-windows-x86_64-0.13.0.zip", + "shasum": "d859994725ef9402381e557c60bb57497215682e355204d754ee3df75ee3c158", + "size": "79163968" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.13.0/zig-windows-aarch64-0.13.0.zip", + "shasum": "95ff88427af7ba2b4f312f45d2377ce7a033e5e3c620c8caaa396a9aba20efda", + "size": "75119033" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.13.0/zig-windows-x86-0.13.0.zip", + "shasum": "eb3d533c3cf868bff7e74455dc005d18fd836c42e50b27106b31e9fec6dffc4a", + "size": "83274739" + } + }, + "0.12.1": { + "date": "2024-06-08", + "docs": "https://ziglang.org/documentation/0.12.1/", + "stdDocs": "https://ziglang.org/documentation/0.12.1/std/", + "src": { + "tarball": "https://ziglang.org/download/0.12.1/zig-0.12.1.tar.xz", + "shasum": "cca0bf5686fe1a15405bd535661811fac7663f81664d2204ea4590ce49a6e9ba", + "size": "17110932" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.12.1/zig-bootstrap-0.12.1.tar.xz", + "shasum": "e533e2cb6ef60edda0ae3f2ca5c6504557db6e985e3c3a80159beb32279ed341", + "size": "45542004" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.12.1/zig-freebsd-x86_64-0.12.1.tar.xz", + "shasum": "30eaa28fa7bc21d01f88528d75ae4b392ae4970406675d5ac712a4937a605123", + "size": "45590080" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.12.1/zig-macos-x86_64-0.12.1.tar.xz", + "shasum": "68f309c6e431d56eb42648d7fe86e8028a23464d401a467831e27c26f1a8d9c9", + "size": "47202232" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.12.1/zig-macos-aarch64-0.12.1.tar.xz", + "shasum": "6587860dbbc070e1ee069e1a3d18ced83b7ba7a80bf67b2c57caf7c9ce5208b1", + "size": "43451512" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.12.1/zig-linux-x86_64-0.12.1.tar.xz", + "shasum": "8860fc9725c2d9297a63008f853e9b11e3c5a2441217f99c1e3104cc6fa4a443", + "size": "45512024" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.12.1/zig-linux-aarch64-0.12.1.tar.xz", + "shasum": "27d4fef393e8d8b5f3b1d19f4dd43bfdb469b4ed17bbc4c2283c1b1fe650ef7f", + "size": "41867324" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.12.1/zig-linux-armv7a-0.12.1.tar.xz", + "shasum": "27493c922fd1454137ed6cbe6b6bec00352838fcd32e6e74f4f9187011816157", + "size": "42665508" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.12.1/zig-linux-riscv64-0.12.1.tar.xz", + "shasum": "463511a863acc16911cff6801de97623e6de296aab7b15dcda5f6fd078b400b5", + "size": "43932908" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.12.1/zig-linux-powerpc64le-0.12.1.tar.xz", + "shasum": "462d6f10350f3c5d6fc4c9d6cfdede93d69d0103af026889a15f65c5de791d39", + "size": "45227716" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.12.1/zig-linux-x86-0.12.1.tar.xz", + "shasum": "c36ac019ca0fc3167e50d17e2affd3d072a06c519761737d0639adfdf2dcfddd", + "size": "50555428" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.12.1/zig-windows-x86_64-0.12.1.zip", + "shasum": "52459b147c2de4d7c28f6b1a4b3d571c114e96836bf8e31c953a7d2f5e94251c", + "size": "76470574" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.12.1/zig-windows-aarch64-0.12.1.zip", + "shasum": "e1286114a11be4695a6ad5cf0ba6a0e5f489bb3b029a5237de93598133f0c13a", + "size": "72998386" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.12.1/zig-windows-x86-0.12.1.zip", + "shasum": "4f0cc9258527e7b8bcf742772b3069122086a5cd857b38a1c08002462ac81f80", + "size": "80979711" + } + }, + "0.12.0": { + "date": "2024-04-20", + "docs": "https://ziglang.org/documentation/0.12.0/", + "stdDocs": "https://ziglang.org/documentation/0.12.0/std/", + "notes": "https://ziglang.org/download/0.12.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.12.0/zig-0.12.0.tar.xz", + "shasum": "a6744ef84b6716f976dad923075b2f54dc4f785f200ae6c8ea07997bd9d9bd9a", + "size": "17099152" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.12.0/zig-bootstrap-0.12.0.tar.xz", + "shasum": "3efc643d56421fa68072af94d5512cb71c61acf1c32512f77c0b4590bff63187", + "size": "45527312" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.12.0/zig-freebsd-x86_64-0.12.0.tar.xz", + "shasum": "bd49957d1157850b337ee1cf3c00af83585cff98e1ebc3c524a267e7422a2d7b", + "size": "45578364" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.12.0/zig-macos-x86_64-0.12.0.tar.xz", + "shasum": "4d411bf413e7667821324da248e8589278180dbc197f4f282b7dbb599a689311", + "size": "47185720" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.12.0/zig-macos-aarch64-0.12.0.tar.xz", + "shasum": "294e224c14fd0822cfb15a35cf39aa14bd9967867999bf8bdfe3db7ddec2a27f", + "size": "43447724" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.12.0/zig-linux-x86_64-0.12.0.tar.xz", + "shasum": "c7ae866b8a76a568e2d5cfd31fe89cdb629bdd161fdd5018b29a4a0a17045cad", + "size": "45480516" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.12.0/zig-linux-aarch64-0.12.0.tar.xz", + "shasum": "754f1029484079b7e0ca3b913a0a2f2a6afd5a28990cb224fe8845e72f09de63", + "size": "41849060" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.12.0/zig-linux-armv7a-0.12.0.tar.xz", + "shasum": "b48221f4c64416d257f0f9f77d8727dccf7de92aeabe59744ee6e70d650a97bc", + "size": "42638808" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.12.0/zig-linux-riscv64-0.12.0.tar.xz", + "shasum": "bb2d1a78b01595a9c00ffd2e12ab46e32f8b6798f76aec643ff78e5b4f5c5afd", + "size": "43917444" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.12.0/zig-linux-powerpc64le-0.12.0.tar.xz", + "shasum": "9218beecfb9250e9eff863f58f987dca7077e3258dd263c40269086127f9679b", + "size": "45216736" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.12.0/zig-linux-x86-0.12.0.tar.xz", + "shasum": "fb752fceb88749a80d625a6efdb23bea8208962b5150d6d14c92d20efda629a5", + "size": "50498940" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.12.0/zig-windows-x86_64-0.12.0.zip", + "shasum": "2199eb4c2000ddb1fba85ba78f1fcf9c1fb8b3e57658f6a627a8e513131893f5", + "size": "76442958" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.12.0/zig-windows-aarch64-0.12.0.zip", + "shasum": "04c6b92689241ca7a8a59b5f12d2ca2820c09d5043c3c4808b7e93e41c7bf97b", + "size": "72976876" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.12.0/zig-windows-x86-0.12.0.zip", + "shasum": "497dc9fd415cadf948872f137d6cc0870507488f79db9547b8f2adb73cda9981", + "size": "80950440" + } + }, + "0.11.0": { + "date": "2023-08-04", + "docs": "https://ziglang.org/documentation/0.11.0/", + "stdDocs": "https://ziglang.org/documentation/0.11.0/std/", + "notes": "https://ziglang.org/download/0.11.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.11.0/zig-0.11.0.tar.xz", + "shasum": "72014e700e50c0d3528cef3adf80b76b26ab27730133e8202716a187a799e951", + "size": "15275316" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.11.0/zig-bootstrap-0.11.0.tar.xz", + "shasum": "38dd9e17433c7ce5687c48fa0a757462cbfcbe75d9d5087d14ebbe00efd21fdc", + "size": "43227592" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.11.0/zig-freebsd-x86_64-0.11.0.tar.xz", + "shasum": "ea430327f9178377b79264a1d492868dcff056cd76d43a6fb00719203749e958", + "size": "46432140" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.11.0/zig-macos-x86_64-0.11.0.tar.xz", + "shasum": "1c1c6b9a906b42baae73656e24e108fd8444bb50b6e8fd03e9e7a3f8b5f05686", + "size": "47189164" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.11.0/zig-macos-aarch64-0.11.0.tar.xz", + "shasum": "c6ebf927bb13a707d74267474a9f553274e64906fd21bf1c75a20bde8cadf7b2", + "size": "43855096" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.11.0/zig-linux-x86_64-0.11.0.tar.xz", + "shasum": "2d00e789fec4f71790a6e7bf83ff91d564943c5ee843c5fd966efc474b423047", + "size": "44961892" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.11.0/zig-linux-aarch64-0.11.0.tar.xz", + "shasum": "956eb095d8ba44ac6ebd27f7c9956e47d92937c103bf754745d0a39cdaa5d4c6", + "size": "41492432" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.11.0/zig-linux-armv7a-0.11.0.tar.xz", + "shasum": "aebe8bbeca39f13f9b7304465f9aee01ab005d243836bd40f4ec808093dccc9b", + "size": "42240664" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.11.0/zig-linux-riscv64-0.11.0.tar.xz", + "shasum": "24a478937eddb507e96d60bd4da00de9092b3f0920190eb45c4c99c946b00ed5", + "size": "43532324" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.11.0/zig-linux-powerpc64le-0.11.0.tar.xz", + "shasum": "75260e87325e820a278cf9e74f130c7b3d84c0b5197afb2e3c85eff3fcedd48d", + "size": "44656184" + }, + "powerpc-linux": { + "tarball": "https://ziglang.org/download/0.11.0/zig-linux-powerpc-0.11.0.tar.xz", + "shasum": "70a5f9668a66fb2a91a7c3488b15bcb568e1f9f44b95cd10075c138ad8c42864", + "size": "44539972" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.11.0/zig-linux-x86-0.11.0.tar.xz", + "shasum": "7b0dc3e0e070ae0e0d2240b1892af6a1f9faac3516cae24e57f7a0e7b04662a8", + "size": "49824456" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.11.0/zig-windows-x86_64-0.11.0.zip", + "shasum": "142caa3b804d86b4752556c9b6b039b7517a08afa3af842645c7e2dcd125f652", + "size": "77216743" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.11.0/zig-windows-aarch64-0.11.0.zip", + "shasum": "5d4bd13db5ecb0ddc749231e00f125c1d31087d708e9ff9b45c4f4e13e48c661", + "size": "73883137" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.11.0/zig-windows-x86-0.11.0.zip", + "shasum": "e72b362897f28c671633e650aa05289f2e62b154efcca977094456c8dac3aefa", + "size": "81576961" + } + }, + "0.10.1": { + "date": "2023-01-19", + "docs": "https://ziglang.org/documentation/0.10.1/", + "stdDocs": "https://ziglang.org/documentation/0.10.1/std/", + "notes": "https://ziglang.org/download/0.10.1/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.10.1/zig-0.10.1.tar.xz", + "shasum": "69459bc804333df077d441ef052ffa143d53012b655a51f04cfef1414c04168c", + "size": "15143112" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.10.1/zig-bootstrap-0.10.1.tar.xz", + "shasum": "9f5781210b9be8f832553d160851635780f9bd71816065351ab29cfd8968f5e9", + "size": "43971816" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.10.1/zig-macos-x86_64-0.10.1.tar.xz", + "shasum": "02483550b89d2a3070c2ed003357fd6e6a3059707b8ee3fbc0c67f83ca898437", + "size": "45119596" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.10.1/zig-macos-aarch64-0.10.1.tar.xz", + "shasum": "b9b00477ec5fa1f1b89f35a7d2a58688e019910ab80a65eac2a7417162737656", + "size": "40517896" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.10.1/zig-linux-x86_64-0.10.1.tar.xz", + "shasum": "6699f0e7293081b42428f32c9d9c983854094bd15fee5489f12c4cf4518cc380", + "size": "44085596" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.10.1/zig-linux-aarch64-0.10.1.tar.xz", + "shasum": "db0761664f5f22aa5bbd7442a1617dd696c076d5717ddefcc9d8b95278f71f5d", + "size": "40321280" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.10.1/zig-linux-riscv64-0.10.1.tar.xz", + "shasum": "9db5b59a5112b8beb995094ba800e88b0060e9cf7cfadf4dc3e666c9010dc77b", + "size": "42196008" + }, + "i386-linux": { + "tarball": "https://ziglang.org/download/0.10.1/zig-linux-i386-0.10.1.tar.xz", + "shasum": "8c710ca5966b127b0ee3efba7310601ee57aab3dd6052a082ebc446c5efb2316", + "size": "48367388" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.10.1/zig-windows-x86_64-0.10.1.zip", + "shasum": "5768004e5e274c7969c3892e891596e51c5df2b422d798865471e05049988125", + "size": "73259729" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.10.1/zig-windows-aarch64-0.10.1.zip", + "shasum": "ece93b0d77b2ab03c40db99ef7ccbc63e0b6bd658af12b97898960f621305428", + "size": "69417459" + } + }, + "0.10.0": { + "date": "2022-10-31", + "docs": "https://ziglang.org/documentation/0.10.0/", + "stdDocs": "https://ziglang.org/documentation/0.10.0/std/", + "notes": "https://ziglang.org/download/0.10.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.10.0/zig-0.10.0.tar.xz", + "shasum": "d8409f7aafc624770dcd050c8fa7e62578be8e6a10956bca3c86e8531c64c136", + "size": "14530912" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.10.0/zig-bootstrap-0.10.0.tar.xz", + "shasum": "c13dc70c4ff4c09f749adc0d473cbd3942991dd4d1bd2d860fbf257d8c1bbabf", + "size": "45625516" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.10.0/zig-freebsd-x86_64-0.10.0.tar.xz", + "shasum": "dd77afa2a8676afbf39f7d6068eda81b0723afd728642adaac43cb2106253d65", + "size": "44056504" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.10.0/zig-linux-aarch64-0.10.0.tar.xz", + "shasum": "09ef50c8be73380799804169197820ee78760723b0430fa823f56ed42b06ea0f", + "size": "40387688" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.10.0/zig-linux-armv7a-0.10.0.tar.xz", + "shasum": "7201b2e89cd7cc2dde95d39485fd7d5641ba67dc6a9a58c036cb4c308d2e82de", + "size": "50805936" + }, + "i386-linux": { + "tarball": "https://ziglang.org/download/0.10.0/zig-linux-i386-0.10.0.tar.xz", + "shasum": "dac8134f1328c50269f3e50b334298ec7916cb3b0ef76927703ddd1c96fd0115", + "size": "48451732" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.10.0/zig-linux-riscv64-0.10.0.tar.xz", + "shasum": "2a126f3401a7a7efc4b454f0a85c133db1af5a9dfee117f172213b7cbd47bfba", + "size": "42272968" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.10.0/zig-linux-x86_64-0.10.0.tar.xz", + "shasum": "631ec7bcb649cd6795abe40df044d2473b59b44e10be689c15632a0458ddea55", + "size": "44142400" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.10.0/zig-macos-aarch64-0.10.0.tar.xz", + "shasum": "02f7a7839b6a1e127eeae22ea72c87603fb7298c58bc35822a951479d53c7557", + "size": "40602664" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.10.0/zig-macos-x86_64-0.10.0.tar.xz", + "shasum": "3a22cb6c4749884156a94ea9b60f3a28cf4e098a69f08c18fbca81c733ebfeda", + "size": "45175104" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.10.0/zig-windows-x86_64-0.10.0.zip", + "shasum": "a66e2ff555c6e48781de1bcb0662ef28ee4b88af3af2a577f7b1950e430897ee", + "size": "73181558" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.10.0/zig-windows-aarch64-0.10.0.zip", + "shasum": "1bbda8d123d44f3ae4fa90d0da04b1e9093c3f9ddae3429a4abece1e1c0bf19a", + "size": "69332389" + } + }, + "0.9.1": { + "date": "2022-02-14", + "docs": "https://ziglang.org/documentation/0.9.1/", + "stdDocs": "https://ziglang.org/documentation/0.9.1/std/", + "notes": "https://ziglang.org/download/0.9.1/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.9.1/zig-0.9.1.tar.xz", + "shasum": "38cf4e84481f5facc766ba72783e7462e08d6d29a5d47e3b75c8ee3142485210", + "size": "13940828" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.9.1/zig-bootstrap-0.9.1.tar.xz", + "shasum": "0a8e221c71860d8975c15662b3ed3bd863e81c4fe383455a596e5e0e490d6109", + "size": "42488812" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.9.1/zig-freebsd-x86_64-0.9.1.tar.xz", + "shasum": "4e06009bd3ede34b72757eec1b5b291b30aa0d5046dadd16ecb6b34a02411254", + "size": "39028848" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.9.1/zig-linux-aarch64-0.9.1.tar.xz", + "shasum": "5d99a39cded1870a3fa95d4de4ce68ac2610cca440336cfd252ffdddc2b90e66", + "size": "37034860" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.9.1/zig-linux-armv7a-0.9.1.tar.xz", + "shasum": "6de64456cb4757a555816611ea697f86fba7681d8da3e1863fa726a417de49be", + "size": "37974652" + }, + "i386-linux": { + "tarball": "https://ziglang.org/download/0.9.1/zig-linux-i386-0.9.1.tar.xz", + "shasum": "e776844fecd2e62fc40d94718891057a1dbca1816ff6013369e9a38c874374ca", + "size": "44969172" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.9.1/zig-linux-riscv64-0.9.1.tar.xz", + "shasum": "208dea53662c2c52777bd9e3076115d2126a4f71aed7f2ff3b8fe224dc3881aa", + "size": "39390868" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.9.1/zig-linux-x86_64-0.9.1.tar.xz", + "shasum": "be8da632c1d3273f766b69244d80669fe4f5e27798654681d77c992f17c237d7", + "size": "41011464" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.9.1/zig-macos-aarch64-0.9.1.tar.xz", + "shasum": "8c473082b4f0f819f1da05de2dbd0c1e891dff7d85d2c12b6ee876887d438287", + "size": "38995640" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.9.1/zig-macos-x86_64-0.9.1.tar.xz", + "shasum": "2d94984972d67292b55c1eb1c00de46580e9916575d083003546e9a01166754c", + "size": "43713044" + }, + "i386-windows": { + "tarball": "https://ziglang.org/download/0.9.1/zig-windows-i386-0.9.1.zip", + "shasum": "74a640ed459914b96bcc572183a8db687bed0af08c30d2ea2f8eba03ae930f69", + "size": "67929868" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.9.1/zig-windows-x86_64-0.9.1.zip", + "shasum": "443da53387d6ae8ba6bac4b3b90e9fef4ecbe545e1c5fa3a89485c36f5c0e3a2", + "size": "65047697" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.9.1/zig-windows-aarch64-0.9.1.zip", + "shasum": "621bf95f54dc3ff71466c5faae67479419951d7489e40e87fd26d195825fb842", + "size": "61478151" + } + }, + "0.9.0": { + "date": "2021-12-20", + "docs": "https://ziglang.org/documentation/0.9.0/", + "stdDocs": "https://ziglang.org/documentation/0.9.0/std/", + "notes": "https://ziglang.org/download/0.9.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.9.0/zig-0.9.0.tar.xz", + "shasum": "cd1be83b12f8269cc5965e59877b49fdd8fa638efb6995ac61eb4cea36a2e381", + "size": "13928772" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.9.0/zig-bootstrap-0.9.0.tar.xz", + "shasum": "16b0bdf0bc0a5ed1e0950e08481413d806192e06443a512347526647b2baeabc", + "size": "42557736" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.9.0/zig-freebsd-x86_64-0.9.0.tar.xz", + "shasum": "c95afe679b7cc4110dc2ecd3606c83a699718b7a958d6627f74c20886333e194", + "size": "41293236" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.9.0/zig-linux-aarch64-0.9.0.tar.xz", + "shasum": "1524fedfdbade2dbc9bae1ed98ad38fa7f2114c9a3e94da0d652573c75efbc5a", + "size": "40008396" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.9.0/zig-linux-armv7a-0.9.0.tar.xz", + "shasum": "50225dee6e6448a63ee96383a34d9fe3bba34ae8da1a0c8619bde2cdfc1df87d", + "size": "41196876" + }, + "i386-linux": { + "tarball": "https://ziglang.org/download/0.9.0/zig-linux-i386-0.9.0.tar.xz", + "shasum": "b0dcf688349268c883292acdd55eaa3c13d73b9146e4b990fad95b84a2ac528b", + "size": "47408656" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.9.0/zig-linux-riscv64-0.9.0.tar.xz", + "shasum": "85466de07504767ed37f59782672ad41bbdf43d6480fafd07f45543278b07620", + "size": "44171420" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.9.0/zig-linux-x86_64-0.9.0.tar.xz", + "shasum": "5c55344a877d557fb1b28939785474eb7f4f2f327aab55293998f501f7869fa6", + "size": "43420796" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.9.0/zig-macos-aarch64-0.9.0.tar.xz", + "shasum": "3991c70594d61d09fb4b316157a7c1d87b1d4ec159e7a5ecd11169ff74cad832", + "size": "39013392" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.9.0/zig-macos-x86_64-0.9.0.tar.xz", + "shasum": "c5280eeec4d6e5ea5ce5b448dc9a7c4bdd85ecfed4c1b96aa0835e48b36eccf0", + "size": "43764596" + }, + "i386-windows": { + "tarball": "https://ziglang.org/download/0.9.0/zig-windows-i386-0.9.0.zip", + "shasum": "bb839434afc75092015cf4c33319d31463c18512bc01dd719aedf5dcbc368466", + "size": "67946715" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.9.0/zig-windows-x86_64-0.9.0.zip", + "shasum": "084ea2646850aaf068234b0f1a92b914ed629be47075e835f8a67d55c21d880e", + "size": "65045849" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.9.0/zig-windows-aarch64-0.9.0.zip", + "shasum": "f9018725e3fb2e8992b17c67034726971156eb190685018a9ac8c3a9f7a22340", + "size": "61461921" + } + }, + "0.8.1": { + "date": "2021-09-06", + "docs": "https://ziglang.org/documentation/0.8.1/", + "stdDocs": "https://ziglang.org/documentation/0.8.1/std/", + "notes": "https://ziglang.org/download/0.8.1/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.8.1/zig-0.8.1.tar.xz", + "shasum": "8c428e14a0a89cb7a15a6768424a37442292858cdb695e2eb503fa3c7bf47f1a", + "size": "12650228" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.8.1/zig-bootstrap-0.8.1.tar.xz", + "shasum": "fa1239247f830ecd51c42537043f5220e4d1dfefdc54356fa419616a0efb3902", + "size": "43613464" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.8.1/zig-freebsd-x86_64-0.8.1.tar.xz", + "shasum": "fc4f6478bcf3a9fce1b8ef677a91694f476dd35be6d6c9c4f44a8b76eedbe176", + "size": "39150924" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.8.1/zig-linux-aarch64-0.8.1.tar.xz", + "shasum": "2166dc9f2d8df387e8b4122883bb979d739281e1ff3f3d5483fec3a23b957510", + "size": "37605932" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.8.1/zig-linux-armv7a-0.8.1.tar.xz", + "shasum": "5ba58141805e2519f38cf8e715933cbf059f4f3dade92c71838cce341045de05", + "size": "39185876" + }, + "i386-linux": { + "tarball": "https://ziglang.org/download/0.8.1/zig-linux-i386-0.8.1.tar.xz", + "shasum": "2f3e84f30492b5f1c5f97cecc0166f07a8a8d50c5f85dbb3a6ef2a4ee6f915e6", + "size": "44782932" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.8.1/zig-linux-riscv64-0.8.1.tar.xz", + "shasum": "4adfaf147b025917c03367462fe5018aaa9edbc6439ef9cd0da2b074ae960554", + "size": "41234480" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.8.1/zig-linux-x86_64-0.8.1.tar.xz", + "shasum": "6c032fc61b5d77a3f3cf781730fa549f8f059ffdb3b3f6ad1c2994d2b2d87983", + "size": "41250060" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.8.1/zig-macos-aarch64-0.8.1.tar.xz", + "shasum": "5351297e3b8408213514b29c0a938002c5cf9f97eee28c2f32920e1227fd8423", + "size": "35340712" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.8.1/zig-macos-x86_64-0.8.1.tar.xz", + "shasum": "16b0e1defe4c1807f2e128f72863124bffdd906cefb21043c34b673bf85cd57f", + "size": "39946200" + }, + "i386-windows": { + "tarball": "https://ziglang.org/download/0.8.1/zig-windows-i386-0.8.1.zip", + "shasum": "099605051eb0452a947c8eab8fbbc7e43833c8376d267e94e41131c289a1c535", + "size": "64152358" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.8.1/zig-windows-x86_64-0.8.1.zip", + "shasum": "43573db14cd238f7111d6bdf37492d363f11ecd1eba802567a172f277d003926", + "size": "61897838" + } + }, + "0.8.0": { + "date": "2021-06-04", + "docs": "https://ziglang.org/documentation/0.8.0/", + "stdDocs": "https://ziglang.org/documentation/0.8.0/std/", + "notes": "https://ziglang.org/download/0.8.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.8.0/zig-0.8.0.tar.xz", + "shasum": "03a828d00c06b2e3bb8b7ff706997fd76bf32503b08d759756155b6e8c981e77", + "size": "12614896" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.8.0/zig-bootstrap-0.8.0.tar.xz", + "shasum": "10600bc9c01f92e343f40d6ecc0ad05d67d27c3e382bce75524c0639cd8ca178", + "size": "43574248" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.8.0/zig-freebsd-x86_64-0.8.0.tar.xz", + "shasum": "0d3ccc436c8c0f50fd55462f72f8492d98723c7218ffc2a8a1831967d81b4bdc", + "size": "39125332" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.8.0/zig-linux-aarch64-0.8.0.tar.xz", + "shasum": "ee204ca2c2037952cf3f8b10c609373a08a291efa4af7b3c73be0f2b27720470", + "size": "37575428" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.8.0/zig-linux-armv7a-0.8.0.tar.xz", + "shasum": "d00b8bd97b79f45d6f5da956983bafeaa082e6c2ae8c6e1c6d4faa22fa29b320", + "size": "38884212" + }, + "i386-linux": { + "tarball": "https://ziglang.org/download/0.8.0/zig-linux-i386-0.8.0.tar.xz", + "shasum": "96e43ee6ed81c3c63401f456bd1c58ee6d42373a43cb324f5cf4974ca0998865", + "size": "42136032" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.8.0/zig-linux-riscv64-0.8.0.tar.xz", + "shasum": "75997527a78cdab64c40c43d9df39c01c4cdb557bb3992a869838371a204cfea", + "size": "40016268" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.8.0/zig-linux-x86_64-0.8.0.tar.xz", + "shasum": "502625d3da3ae595c5f44a809a87714320b7a40e6dff4a895b5fa7df3391d01e", + "size": "41211184" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.8.0/zig-macos-aarch64-0.8.0.tar.xz", + "shasum": "b32d13f66d0e1ff740b3326d66a469ee6baddbd7211fa111c066d3bd57683111", + "size": "35292180" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.8.0/zig-macos-x86_64-0.8.0.tar.xz", + "shasum": "279f9360b5cb23103f0395dc4d3d0d30626e699b1b4be55e98fd985b62bc6fbe", + "size": "39969312" + }, + "i386-windows": { + "tarball": "https://ziglang.org/download/0.8.0/zig-windows-i386-0.8.0.zip", + "shasum": "b6ec9aa6cd6f3872fcb30d43ff411802d82008a0c4142ee49e208a09b2c1c5fe", + "size": "61507213" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.8.0/zig-windows-x86_64-0.8.0.zip", + "shasum": "8580fbbf3afb72e9b495c7f8aeac752a03475ae0bbcf5d787f3775c7e1f4f807", + "size": "61766193" + } + }, + "0.7.1": { + "date": "2020-12-13", + "docs": "https://ziglang.org/documentation/0.7.1/", + "stdDocs": "https://ziglang.org/documentation/0.7.1/std/", + "notes": "https://ziglang.org/download/0.7.1/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.7.1/zig-0.7.1.tar.xz", + "shasum": "2db3b944ab368d955b48743d9f7c963b8f96de1a441ba5a35e197237cc6dae44", + "size": "10711824" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.7.1/zig-bootstrap-0.7.1.tar.xz", + "shasum": "040f27c1fae4b0cac0a2782aecdb691f6a2f8e89db6a6ed35024c31c304fd9b2", + "size": "40232612" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.7.1/zig-freebsd-x86_64-0.7.1.tar.xz", + "shasum": "e73c1dca35791a3183fdd5ecde0443ebbe180942efceafe651886034fb8def09", + "size": "39066808" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.7.1/zig-linux-aarch64-0.7.1.tar.xz", + "shasum": "48ec90eba407e4587ddef7eecef25fec7e13587eb98e3b83c5f2f5fff2a5cbe7", + "size": "33780552" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.7.1/zig-linux-armv7a-0.7.1.tar.xz", + "shasum": "5a0662e07b4c4968665e1f97558f8591f6facec45d2e0ff5715e661743107ceb", + "size": "35813504" + }, + "i386-linux": { + "tarball": "https://ziglang.org/download/0.7.1/zig-linux-i386-0.7.1.tar.xz", + "shasum": "4882e052e5f83690bd0334bb4fc1702b5403cb3a3d2aa63fd7d6043d8afecba3", + "size": "39230912" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.7.1/zig-linux-riscv64-0.7.1.tar.xz", + "shasum": "187294bfd35983348c3fe042901b42e67e7e36ab7f77a5f969d21c0051f4d21f", + "size": "37454812" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.7.1/zig-linux-x86_64-0.7.1.tar.xz", + "shasum": "18c7b9b200600f8bcde1cd8d7f1f578cbc3676241ce36d771937ce19a8159b8d", + "size": "37848176" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.7.1/zig-macos-x86_64-0.7.1.tar.xz", + "shasum": "845cb17562978af0cf67e3993f4e33330525eaf01ead9386df9105111e3bc519", + "size": "36211076" + }, + "i386-windows": { + "tarball": "https://ziglang.org/download/0.7.1/zig-windows-i386-0.7.1.zip", + "shasum": "a1b9a7421e13153e07fd2e2c93ff29aad64d83105b8fcdafa633dbe689caf1c0", + "size": "54374983" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.7.1/zig-windows-x86_64-0.7.1.zip", + "shasum": "4818a8a65b4672bc52c0ae7f14d014e0eb8caf10f12c0745176820384cea296a", + "size": "54909997" + } + }, + "0.7.0": { + "date": "2020-11-08", + "docs": "https://ziglang.org/documentation/0.7.0/", + "stdDocs": "https://ziglang.org/documentation/0.7.0/std/", + "notes": "https://ziglang.org/download/0.7.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.7.0/zig-0.7.0.tar.xz", + "shasum": "0efd2cf6c3b05723db80e9cf193bc55150bba84ca41f855a90f53fc756445f83", + "size": "10683920" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.7.0/zig-bootstrap-0.7.0.tar.xz", + "shasum": "f073beaf5c53c8c57c0d374cbfcb332ef92ad703173edba0d9e0f2ed28401b72", + "size": "40200436" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.7.0/zig-freebsd-x86_64-0.7.0.tar.xz", + "shasum": "a0c926272ee4ae720034b4a6a1dc98399d76156dd84182554740f0ca8a41fc99", + "size": "34798992" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.7.0/zig-linux-aarch64-0.7.0.tar.xz", + "shasum": "f89933bac87d44be82325754ff88423020c81c7032a6fc41cfeb81e982eeab9b", + "size": "33096140" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.7.0/zig-linux-armv7a-0.7.0.tar.xz", + "shasum": "011c267e25a96ee160505a560c441daa045359a9d50e13ab1bada9d75c95db2d", + "size": "35157584" + }, + "i386-linux": { + "tarball": "https://ziglang.org/download/0.7.0/zig-linux-i386-0.7.0.tar.xz", + "shasum": "4bb2072cd363bcb1cbeb4872ff5cbc1f683b02d0cc1f90c46e3ea7422ce53222", + "size": "38530596" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.7.0/zig-linux-riscv64-0.7.0.tar.xz", + "shasum": "40dff81faa6f232ac40abbf88b9371f3cc932b6e09c423b94387c9ea580cb7be", + "size": "36759992" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.7.0/zig-linux-x86_64-0.7.0.tar.xz", + "shasum": "e619b1c6094c095b932767f527aee2507f847ea981513ff8a08aab0fd730e0ac", + "size": "37154432" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.7.0/zig-macos-aarch64-0.7.0.tar.xz", + "shasum": "338238035734db74ea4f30e500a4893bf741d38305c10952d5e39fa05bdb057d", + "size": "33739424" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.7.0/zig-macos-x86_64-0.7.0.tar.xz", + "shasum": "94063f9a311cbbf7a2e0a12295e09437182cf950f18cb0eb30ea9893f3677f24", + "size": "35258328" + }, + "i386-windows": { + "tarball": "https://ziglang.org/download/0.7.0/zig-windows-i386-0.7.0.zip", + "shasum": "b1e520aacbfbd645ff3521b3eb4d44166d9a0288b8725e4b001f8b50a425eb2e", + "size": "53390517" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.7.0/zig-windows-x86_64-0.7.0.zip", + "shasum": "965f56c0a36f9cda2125e3a348bc654f7f155e2804c3667d231775ec228f8553", + "size": "53943784" + } + }, + "0.6.0": { + "date": "2020-04-13", + "docs": "https://ziglang.org/documentation/0.6.0/", + "stdDocs": "https://ziglang.org/documentation/0.6.0/std/", + "notes": "https://ziglang.org/download/0.6.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.6.0/zig-0.6.0.tar.xz", + "shasum": "5d167dc19354282dd35dd17b38e99e1763713b9be8a4ba9e9e69284e059e7204", + "size": "10349552" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.6.0/zig-bootstrap-0.6.0.tar.xz", + "shasum": "5e0e4dc878b3dd0c1852a442b174f0732e8c07869a8fcd226b71a93b89b381ab", + "size": "38469948" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.6.0/zig-freebsd-x86_64-0.6.0.tar.xz", + "shasum": "190ff79c1eb56805a315d7c7a51082e32f62926250c0702b36760c225e1634a3", + "size": "36974604" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.6.0/zig-linux-aarch64-0.6.0.tar.xz", + "shasum": "e7520efd42cfa02be48c2e430d08fe1f3cbb999d21d9f0d3ffd0febb976b2f41", + "size": "37090044" + }, + "armv6kz-linux": { + "tarball": "https://ziglang.org/download/0.6.0/zig-linux-armv6kz-0.6.0.tar.xz", + "shasum": "36b6493b3fed43eb1f0000e765798ad31a6bb7d7fd3f553ac1c3761dbc919b82", + "size": "39133452" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.6.0/zig-linux-armv7a-0.6.0.tar.xz", + "shasum": "946969abe357def95ca9cbbfcebfcf2d90cf967bcd3f48ee87662e32d91d8f35", + "size": "39143748" + }, + "i386-linux": { + "tarball": "https://ziglang.org/download/0.6.0/zig-linux-i386-0.6.0.tar.xz", + "shasum": "a97a2f9ae21575743cdd763c1917d49400d83fc562ef64582b18bade43eb24ce", + "size": "44877640" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.6.0/zig-linux-riscv64-0.6.0.tar.xz", + "shasum": "68ddee43f7503c8ae5f26a921f3602c34719a02ed2241f528c0b8b888cc14b38", + "size": "41993144" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.6.0/zig-linux-x86_64-0.6.0.tar.xz", + "shasum": "08fd3c757963630645441c2772362e9c2294020c44f14fce1b89f45de0dc1253", + "size": "44766320" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.6.0/zig-macos-x86_64-0.6.0.tar.xz", + "shasum": "17270360e87ddc49f737e760047b2fac49f1570a824a306119b1194ac4093895", + "size": "42573184" + }, + "i386-windows": { + "tarball": "https://ziglang.org/download/0.6.0/zig-windows-i386-0.6.0.zip", + "shasum": "3b0a02618743e92175990dc6d1a787bb95ff62c4cda016f1c14c7786f575f8ca", + "size": "60446431" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.6.0/zig-windows-x86_64-0.6.0.zip", + "shasum": "c3b897832523e1026e10b2d8d55d7f895185c0a27a63681f3a23219c3f1c38f4", + "size": "49065511" + } + }, + "0.5.0": { + "date": "2019-09-30", + "docs": "https://ziglang.org/documentation/0.5.0/", + "notes": "https://ziglang.org/download/0.5.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.5.0/zig-0.5.0.tar.xz", + "shasum": "55ae16960f152bcb9cf98b4f8570902d0e559a141abf927f0d3555b7cc838a31", + "size": "10956132" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.5.0/zig-freebsd-x86_64-0.5.0.tar.xz", + "shasum": "9e1f4d36c3d584c0aa01f20eb4cd0a0eef3eee5af23e483b8414de55feab6ab6", + "size": "33650744" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.5.0/zig-macos-x86_64-0.5.0.tar.xz", + "shasum": "28702cc05745c7c0bd450487d5f4091bf0a1ad279b35eb9a640ce3e3a15b300d", + "size": "37898664" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.5.0/zig-windows-x86_64-0.5.0.zip", + "shasum": "58141323db8d84a5af62746be5f9140bc161ee760ef33dc91a887bf9ac021976", + "size": "44871804" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.5.0/zig-linux-x86_64-0.5.0.tar.xz", + "shasum": "43e8f8a8b8556edd373ddf9c1ef3ca6cf852d4d09fe07d5736d12fefedd2b4f7", + "size": "40895068" + } + }, + "0.4.0": { + "date": "2019-04-08", + "docs": "https://ziglang.org/documentation/0.4.0/", + "notes": "https://ziglang.org/download/0.4.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.4.0/zig-0.4.0.tar.xz", + "shasum": "fec1f3f6b359a3d942e0a7f9157b3b30cde83927627a0e1ea95c54de3c526cfc", + "size": "5348776" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.4.0/zig-freebsd-x86_64-0.4.0.tar.xz", + "shasum": "3d557c91ac36d8262eb1733bb5f261c95944f9b635e43386e3d00a3272818c30", + "size": "27269672" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.4.0/zig-macos-x86_64-0.4.0.tar.xz", + "shasum": "67c932982484d017c5111e54af9f33f15e8e05c6bc5346a55e04052159c964a8", + "size": "30841504" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.4.0/zig-windows-x86_64-0.4.0.zip", + "shasum": "fbc3dd205e064c263063f69f600bedb18e3d0aa2efa747a63ef6cafb6d73f127", + "size": "35800101" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.4.0/zig-linux-x86_64-0.4.0.tar.xz", + "shasum": "fb1954e2fb556a01f8079a08130e88f70084e08978ff853bb2b1986d8c39d84e", + "size": "32876100" + } + }, + "0.3.0": { + "date": "2018-09-28", + "docs": "https://ziglang.org/documentation/0.3.0/", + "notes": "https://ziglang.org/download/0.3.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.3.0/zig-0.3.0.tar.xz", + "shasum": "d70af604f3a8622f3393d93abb3e056bf60351e32d121e6fa4fe03d8d41e1f5a", + "size": "2335592" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.3.0/zig-macos-x86_64-0.3.0.tar.xz", + "shasum": "19dec1f1943ab7be26823376d466f7e456143deb34e17502778a949034dc2e7e", + "size": "23712696" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.3.0/zig-windows-x86_64-0.3.0.zip", + "shasum": "bb568c03950958f8bb3472139c3ab5ed74547c8c694ab50f404c202faf51baf4", + "size": "22524425" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.3.0/zig-linux-x86_64-0.3.0.tar.xz", + "shasum": "b378d0aae30cb54f28494e7bc4efbc9bfb6326f47bfb302e8b5287af777b2f3c", + "size": "25209304" + } + }, + "0.2.0": { + "date": "2018-03-15", + "docs": "https://ziglang.org/documentation/0.2.0/", + "notes": "https://ziglang.org/download/0.2.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.2.0/zig-0.2.0.tar.xz", + "shasum": "29c9beb172737f4d5019b88ceae829ae8bc6512fb4386cfbf895ae2b42aa6965", + "size": "1940832" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.2.0/zig-win64-0.2.0.zip", + "shasum": "4f8a2979941a1f081ec8e545cca0b72608c0db1c5a3fd377a94db40649dcd3d4", + "size": "21076274" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.2.0/zig-linux-x86_64-0.2.0.tar.xz", + "shasum": "209c6fb745d42474c0a73d6f291c7ae3a38b6a1b6b641eea285a7f840cc1a890", + "size": "22551928" + } + }, + "0.1.1": { + "date": "2017-10-17", + "docs": "https://ziglang.org/documentation/0.1.1/", + "notes": "https://ziglang.org/download/0.1.1/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.1.1/zig-0.1.1.tar.xz", + "shasum": "ffca0cfb263485287e19cc997b08701fcd5f24b700345bcdc3dd8074f5a104e0", + "size": "1659716" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.1.1/zig-win64-0.1.1.zip", + "shasum": "6fc88bef531af7e567fe30bf60da1487b86833cbee84c7a2f3e317030aa5b660", + "size": "19757776" + } + } } From 255a2b6518b17f05754fe1788c2760463f718d7e Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 27 Nov 2025 15:48:19 +0100 Subject: [PATCH 060/881] WIP --- main.zig | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 main.zig diff --git a/main.zig b/main.zig new file mode 100644 index 00000000..a2a52c95 --- /dev/null +++ b/main.zig @@ -0,0 +1,51 @@ +const std = @import("std"); +const upb = @import("upb"); +const c = @import("c"); + +pub fn main() !void { + var arena = std.heap.ArenaAllocator.init(std.heap.smp_allocator); + defer arena.deinit(); + + var upb_alloc: upb.Allocator = .init(arena.allocator()); + const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); + defer c.upb_Arena_Free(upb_arena); + + const sharding = struct { + num_replicas: i32 = 2, + num_partitions: i32 = 4, + }{}; + + const options = blk: { + const options = try upb.new(c.xla_CompileOptionsProto, upb_arena); + c.xla_CompileOptionsProto_set_executable_build_options(options, executable_build_options_blk: { + const exec_build_options = try upb.new(c.xla_ExecutableBuildOptionsProto, upb_arena); + + c.xla_ExecutableBuildOptionsProto_set_device_ordinal(exec_build_options, -1); + c.xla_ExecutableBuildOptionsProto_set_num_replicas(exec_build_options, sharding.num_replicas); + c.xla_ExecutableBuildOptionsProto_set_num_partitions(exec_build_options, sharding.num_partitions); + c.xla_ExecutableBuildOptionsProto_set_use_spmd_partitioning(exec_build_options, sharding.num_partitions > 1 or sharding.num_replicas > 1); + + c.xla_ExecutableBuildOptionsProto_set_device_assignment(exec_build_options, device_assignment_blk: { + const device_assignment = try upb.new(c.xla_DeviceAssignmentProto, upb_arena); + + c.xla_DeviceAssignmentProto_set_replica_count(device_assignment, sharding.num_replicas); + c.xla_DeviceAssignmentProto_set_computation_count(device_assignment, sharding.num_partitions); + + const computation_devices = c.xla_DeviceAssignmentProto_resize_computation_devices(device_assignment, sharding.num_partitions, upb_arena); + for (computation_devices[0..sharding.num_partitions], 0..) |*computation_device, i| { + computation_device.* = try upb.new(c.xla_DeviceAssignmentProto_ComputationDevice, upb_arena); + _ = c.xla_DeviceAssignmentProto_ComputationDevice_add_replica_device_ids(computation_device.*, @intCast(i), upb_arena); + } + break :device_assignment_blk device_assignment; + }); + break :executable_build_options_blk exec_build_options; + }); + + // const overrides_map = c._xla_CompileOptionsProto_env_option_overrides_mutable_upb_map(options, upb_arena); + + break :blk options; + }; + + _ = options; + +} From ebd95aafe004a304805e675eeb6fe8fab2d50878 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 28 Nov 2025 10:08:51 +0000 Subject: [PATCH 061/881] wip: xla plugin update + rocm 7 --- runtimes/rocm/BUILD.bazel | 5 + runtimes/rocm/libpjrt_rocm.BUILD.bazel | 86 +- runtimes/rocm/packages.lock.json | 6464 +++++++++++++++--------- runtimes/rocm/packages.yaml | 7 +- runtimes/rocm/rocm.bzl | 35 +- 5 files changed, 4150 insertions(+), 2447 deletions(-) diff --git a/runtimes/rocm/BUILD.bazel b/runtimes/rocm/BUILD.bazel index bab048b1..b9c132bb 100644 --- a/runtimes/rocm/BUILD.bazel +++ b/runtimes/rocm/BUILD.bazel @@ -54,3 +54,8 @@ filegroup( srcs = ["@libpjrt_rocm//:amdgpu_ids_layer"], visibility = ["//visibility:public"], ) + +exports_files( + ["libpjrt_rocm.0.7.1.so", "libpjrt_rocm.so"], + visibility = ["//visibility:public"], +) diff --git a/runtimes/rocm/libpjrt_rocm.BUILD.bazel b/runtimes/rocm/libpjrt_rocm.BUILD.bazel index 9b1a143e..0ea8e236 100644 --- a/runtimes/rocm/libpjrt_rocm.BUILD.bazel +++ b/runtimes/rocm/libpjrt_rocm.BUILD.bazel @@ -28,15 +28,16 @@ cc_shared_library( patchelf( name = "libpjrt_rocm.patchelf", - shared_library = "libpjrt_rocm.so", + #shared_library = "libpjrt_rocm.so", + shared_library = "@zml//runtimes/rocm:libpjrt_rocm.so", add_needed = [ "libzmlxrocm.so.0", # So that RPATH is taken into account. - "librocblas.so.4", - "libMIOpen.so.1", + #"librocblas.so.4", + #"libMIOpen.so.1", ] + select({ "_hipblaslt": [ - "libhipblaslt.so.0", + #"libhipblaslt.so.0", ], "//conditions:default": [], }), @@ -51,47 +52,50 @@ copy_to_directory( srcs = [ ":zmlxrocm_so", ":libpjrt_rocm.patchelf", - "@comgr//:amd_comgr", - "@hip-runtime-amd//:amdhip_patched", - "@hip-runtime-amd//:hiprtc", - "@hipblaslt//:hipblaslt", - "@hipfft", - "@hipsolver", - "@hsa-amd-aqlprofile//:hsa-amd-aqlprofile", - "@hsa-rocr//:hsa-runtime", - "@miopen-hip//:MIOpen", - "@rccl", - "@rocblas//:rocblas", - "@rocblas//:runfiles", - "@rocm-core", - "@rocm-device-libs//:runfiles", - "@rocm-smi-lib//:rocm_smi", - "@rocprofiler-register", - "@rocfft", - "@rocsolver", - "@roctracer", - "@roctracer//:roctx", - "@libelf1", - "@libdrm2-amdgpu", - "@libnuma1", - "@libzstd1", - "@libdrm-amdgpu-amdgpu1", - "@libtinfo6", - "@zlib1g", + #"@comgr//:amd_comgr", + #"@hip-runtime-amd//:amdhip_patched", + #"@hip-runtime-amd//:hiprtc", + #"@hipblaslt//:hipblaslt", + #"@hipfft", + #"@hipsolver", + #"@hsa-amd-aqlprofile//:hsa-amd-aqlprofile", + #"@hsa-rocr//:hsa-runtime", + #"@miopen-hip//:MIOpen", + #"@rccl", + #"@rocblas//:rocblas", + #"@rocblas//:runfiles", + #"@rocm-core", + #"@rocm-device-libs//:runfiles", + #"@rocm-smi-lib//:rocm_smi", + #"@rocprofiler-register", + #"@rocprofiler-sdk", + #"@rocfft", + #"@rocsolver", + #"@roctracer", + #"@roctracer//:roctx", + #"@libelf1", + #"@libdrm2-amdgpu", + #"@libnuma1", + #"@libzstd1", + #"@libdrm-amdgpu-amdgpu1", + #"@libtinfo6", + #"@zlib1g", ] + select({ - ":_hipblaslt": ["@hipblaslt//:runfiles"], + ":_hipblaslt": [ + #"@hipblaslt//:runfiles" + ], "//conditions:default": [], }), replace_prefixes = { - "libpjrt_rocm.patchelf": "lib", - "lib/x86_64-linux-gnu": "lib", - "usr/lib/x86_64-linux-gnu": "lib", - "libelf1": "lib", - "hipblaslt": "lib", - "rocblas": "lib", - "opt/amdgpu/lib/x86_64-linux-gnu": "lib", - "libdrm-amdgpu-amdgpu1": "lib", - "amdhip_patched": "lib", + #"libpjrt_rocm.patchelf": "lib", + #"lib/x86_64-linux-gnu": "lib", + #"usr/lib/x86_64-linux-gnu": "lib", + #"libelf1": "lib", + #"hipblaslt": "lib", + #"rocblas": "lib", + #"opt/amdgpu/lib/x86_64-linux-gnu": "lib", + #"libdrm-amdgpu-amdgpu1": "lib", + #"amdhip_patched": "lib", }, add_directory_to_runfiles = True, include_external_repositories = ["**"], diff --git a/runtimes/rocm/packages.lock.json b/runtimes/rocm/packages.lock.json index 42a3aafd..8d1d9ed2 100755 --- a/runtimes/rocm/packages.lock.json +++ b/runtimes/rocm/packages.lock.json @@ -4,37 +4,9 @@ "arch": "amd64", "dependencies": [ { - "key": "amdgpu-core_1-6.4.60401-2164967.22.04_amd64", - "name": "amdgpu-core", - "version": "1:6.4.60401-2164967.22.04" - } - ], - "key": "libdrm-amdgpu-common_1.0.0.60401-2164967.22.04_amd64", - "name": "libdrm-amdgpu-common", - "sha256": "e37673ce63a3f01d33d40b38f3aa8b5d08f5fc10c985909df5cd7acf5bb44267", - "urls": [ - "https://repo.radeon.com/amdgpu/6.4.1/ubuntu/pool/main/libd/libdrm-amdgpu-common/libdrm-amdgpu-common_1.0.0.60401-2164967.22.04_all.deb" - ], - "version": "1.0.0.60401-2164967.22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdgpu-core_1-6.4.60401-2164967.22.04_amd64", - "name": "amdgpu-core", - "sha256": "9d4244a588009ae3951fa4e1c9d20c243a663bda14043849ecff40c493a0e73f", - "urls": [ - "https://repo.radeon.com/amdgpu/6.4.1/ubuntu/pool/main/a/amdgpu-core/amdgpu-core_6.4.60401-2164967.22.04_all.deb" - ], - "version": "1:6.4.60401-2164967.22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "rocprofiler-register_0.4.0.60401-83_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70100-20_22.04_amd64", "name": "rocprofiler-register", - "version": "0.4.0.60401-83~22.04" + "version": "0.6.0.70100-20~22.04" }, { "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", @@ -62,311 +34,29 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", + "key": "rocm-core_7.1.0.70100-20_22.04_amd64", "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" + "version": "7.1.0.70100-20~22.04" }, { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" }, { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" }, { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" }, { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.124.60401-2164967.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.124.60401-2164967.22.04" - }, - { - "key": "libdrm-amdgpu-common_1.0.0.60401-2164967.22.04_amd64", - "name": "libdrm-amdgpu-common", - "version": "1.0.0.60401-2164967.22.04" - }, - { - "key": "amdgpu-core_1-6.4.60401-2164967.22.04_amd64", - "name": "amdgpu-core", - "version": "1:6.4.60401-2164967.22.04" - }, - { - "key": "libdrm2-amdgpu_1-2.4.124.60401-2164967.22.04_amd64", - "name": "libdrm2-amdgpu", - "version": "1:2.4.124.60401-2164967.22.04" - } - ], - "key": "hsa-rocr_1.15.0.60401-83_22.04_amd64", - "name": "hsa-rocr", - "sha256": "07a37ac162acbc5f054b0ea9cfd4e37e1640adee367d31aad15d504c49999372", - "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/h/hsa-rocr/hsa-rocr_1.15.0.60401-83~22.04_amd64.deb" - ], - "version": "1.15.0.60401-83~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", - "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" - } - ], - "key": "rocprofiler-register_0.4.0.60401-83_22.04_amd64", - "name": "rocprofiler-register", - "sha256": "da49a66ca3e6ee8b9491777c2b5170b6020e8308371e26b869d7af81bc50f571", - "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/r/rocprofiler-register/rocprofiler-register_0.4.0.60401-83~22.04_amd64.deb" - ], - "version": "0.4.0.60401-83~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "sha256": "83873058e692936a09649ede2f8e70b87dde1f3f5488db53da8081b81c79a5d9", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libstdc++6_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "sha256": "8ca7b573446d22ef46a97b4ec537c38acc8653f76391127253040acc7b74af5f", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libgcc-s1_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "sha256": "d8e36ba6cc55719d728c38166bf753650710878e02e30af2aa79aefa244eb275", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc6_2.35-0ubuntu3.10_amd64.deb" - ], - "version": "2.35-0ubuntu3.10" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "sha256": "3fa566e9f861a08736cbc5a97562d9d6e4f0c00450fbeafcb6d7583423b04a98", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libx/libxcrypt/libcrypt1_4.4.27-1_amd64.deb" - ], - "version": "1:4.4.27-1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "sha256": "80a1f5df097af3978f7df2808006ca77197865e5eb7a923712aee6876e7a14cb", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/gcc-12-base_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", - "name": "rocm-core", - "sha256": "221236cf665597beea8c6faeffa1569fc35a6ad4cab89d5c8931974120b42621", - "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/r/rocm-core/rocm-core_6.4.1.60401-83~22.04_amd64.deb" - ], - "version": "6.4.1.60401-83~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "sha256": "2cc781614b045b450a1e073ee3b0902f393ee58ede1d2ea4526ad805fcccf538", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/e/elfutils/libelf1_0.186-1ubuntu0.1_amd64.deb" - ], - "version": "0.186-1ubuntu0.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "sha256": "9dc17e51a1be2d9ed63b7b84ef0e4e29c5abe6f1bc62cb03e7181483cce8a2f2", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/z/zlib/zlib1g_1.2.11.dfsg-2ubuntu9.2_amd64.deb" - ], - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "sha256": "0721c89001fbbd1ada23e89da5d60e762763c1a7b3dc814a2e9a518480a8043d", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/numactl/libnuma1_2.0.14-3ubuntu2_amd64.deb" - ], - "version": "2.0.14-3ubuntu2" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libdrm-amdgpu-amdgpu1_1-2.4.124.60401-2164967.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "sha256": "5c01c079bf95f08e920ad41ab8133faef5ce6e5e6908a5b5fa55d2365557a497", - "urls": [ - "https://repo.radeon.com/amdgpu/6.4.1/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-amdgpu1_2.4.124.60401-2164967.22.04_amd64.deb" - ], - "version": "1:2.4.124.60401-2164967.22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libdrm2-amdgpu_1-2.4.124.60401-2164967.22.04_amd64", - "name": "libdrm2-amdgpu", - "sha256": "a0eb10e4765d721d83d93312e47247a618a491dcc73c39e5e4414adc87e6ea93", - "urls": [ - "https://repo.radeon.com/amdgpu/6.4.1/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm2-amdgpu_2.4.124.60401-2164967.22.04_amd64.deb" - ], - "version": "1:2.4.124.60401-2164967.22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", - "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" - } - ], - "key": "hsa-amd-aqlprofile_1.0.0.60401-83_22.04_amd64", - "name": "hsa-amd-aqlprofile", - "sha256": "9e0917b47d40318f73d4323bdc0fdaa27202931544bc4e89f706c4ddd9bd9428", - "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/h/hsa-amd-aqlprofile/hsa-amd-aqlprofile_1.0.0.60401-83~22.04_amd64.deb" - ], - "version": "1.0.0.60401-83~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", - "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" }, { "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", @@ -433,6 +123,16 @@ "name": "libacl1", "version": "2.3.1-1" }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, { "key": "liblzma5_5.2.5-2ubuntu1_amd64", "name": "liblzma5", @@ -458,137 +158,461 @@ "name": "libcom-err2", "version": "1.46.5-2ubuntu1.2" }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, { "key": "libnsl2_1.3.0-2build2_amd64", "name": "libnsl2", "version": "1.3.0-2build2" }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, { "key": "libncursesw6_6.3-2ubuntu0.1_amd64", "name": "libncursesw6", "version": "6.3-2ubuntu0.1" }, { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "version": "6.3-2ubuntu0.1" + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libelf1_0.186-1ubuntu0.1_amd64", + "name": "libelf1", + "version": "0.186-1ubuntu0.1" + }, + { + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" + }, + { + "key": "libdrm-amdgpu1_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-amdgpu1", + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "key": "libdrm2_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm2", + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "key": "libdrm-common_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-common", + "version": "2.4.113-2~ubuntu0.22.04.1" } ], - "key": "comgr_3.0.0.60401-83_22.04_amd64", - "name": "comgr", - "sha256": "ba02bdf830458668a39300ad082419f4bac3644bf5727c496846697e9fddf429", + "key": "hsa-rocr_1.18.0.70100-20_22.04_amd64", + "name": "hsa-rocr", + "sha256": "189a9bb319fcbcda6178af7064a860fcf8e0ea029b94d31d290eaf6c36ea8f97", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/c/comgr/comgr_3.0.0.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1/pool/main/h/hsa-rocr/hsa-rocr_1.18.0.70100-20~22.04_amd64.deb" ], - "version": "3.0.0.60401-83~22.04" + "version": "1.18.0.70100-20~22.04" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "rocm-core_7.1.0.70100-20_22.04_amd64", + "name": "rocm-core", + "version": "7.1.0.70100-20~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + } + ], + "key": "rocprofiler-register_0.6.0.70100-20_22.04_amd64", + "name": "rocprofiler-register", + "sha256": "522b2231c43a3b6533d62243d68b345738078beedff7906f50a0b15d0120560b", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1/pool/main/r/rocprofiler-register/rocprofiler-register_0.6.0.70100-20~22.04_amd64.deb" + ], + "version": "0.6.0.70100-20~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "sha256": "ae7db00ce8b093e50c994518b90203544e063b4bc574836a048bb142b950b2c9", + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "sha256": "83873058e692936a09649ede2f8e70b87dde1f3f5488db53da8081b81c79a5d9", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libz/libzstd/libzstd1_1.4.8+dfsg-3build1_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libstdc++6_12.3.0-1ubuntu1~22.04_amd64.deb" ], - "version": "1.4.8+dfsg-3build1" + "version": "12.3.0-1ubuntu1~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "sha256": "145736b5bec52cd4fc1b2c09517759383b95255f9fd304ace1de7d830295bb33", + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "sha256": "8ca7b573446d22ef46a97b4ec537c38acc8653f76391127253040acc7b74af5f", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libtinfo-dev_6.3-2ubuntu0.1_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libgcc-s1_12.3.0-1ubuntu1~22.04_amd64.deb" ], - "version": "6.3-2ubuntu0.1" + "version": "12.3.0-1ubuntu1~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "sha256": "1de196ffdd885b5953a44cde13e68b77dc7ceb9c9921482dc84e358bd1ad8b02", + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "sha256": "d8e36ba6cc55719d728c38166bf753650710878e02e30af2aa79aefa244eb275", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libncurses-dev_6.3-2ubuntu0.1_amd64.deb" - ], - "version": "6.3-2ubuntu0.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "sha256": "6382f5c1d7d0bf39cbb94e208ece2bdf556cb3a8dfe278748542c7b8b57a2e6d", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/ncurses-bin_6.3-2ubuntu0.1_amd64.deb" - ], - "version": "6.3-2ubuntu0.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "sha256": "1ec03fe34a7dfd8640a0564a3cad385a10b4f68ebf2190a4c08ccadb395133d3", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libtinfo6_6.3-2ubuntu0.1_amd64.deb" - ], - "version": "6.3-2ubuntu0.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "sha256": "c2e3b50af761886617462ca47ebc9303a83968129acca2add42b8eba0a7abca3", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc6-dev_2.35-0ubuntu3.10_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc6_2.35-0ubuntu3.10_amd64.deb" ], "version": "2.35-0ubuntu3.10" }, { "arch": "amd64", "dependencies": [], - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "sha256": "e93aa448fe708a4bbfcc40b2a8cab6f5f88896174a301caab2719f734e08a621", + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "sha256": "3fa566e9f861a08736cbc5a97562d9d6e4f0c00450fbeafcb6d7583423b04a98", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libn/libnsl/libnsl-dev_1.3.0-2build2_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libx/libxcrypt/libcrypt1_4.4.27-1_amd64.deb" ], - "version": "1.3.0-2build2" + "version": "1:4.4.27-1" }, { "arch": "amd64", "dependencies": [], - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "sha256": "f4254aff5a32ef3c9fca06d3f4f2eff71c93b9393cc1ecc718f7478a5d6a83da", + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "sha256": "80a1f5df097af3978f7df2808006ca77197865e5eb7a923712aee6876e7a14cb", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libt/libtirpc/libtirpc-dev_1.3.2-2ubuntu0.1_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/gcc-12-base_12.3.0-1ubuntu1~22.04_amd64.deb" ], - "version": "1.3.2-2ubuntu0.1" + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "rocm-core_7.1.0.70100-20_22.04_amd64", + "name": "rocm-core", + "sha256": "07e55d5192f7ea915dbefe24c5b8aa5c51b2ea65bade100a4e8b0f5df1d09a07", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1/pool/main/r/rocm-core/rocm-core_7.1.0.70100-20~22.04_amd64.deb" + ], + "version": "7.1.0.70100-20~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "sha256": "43dbcc12d790ed9360be4a1dc690ac0a7464a5e7c3170785886df84811a3f5a5", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3-defaults/python3_3.10.6-1~22.04.1_amd64.deb" + ], + "version": "3.10.6-1~22.04.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "sha256": "41b8f285f0a1661ece778a63592dcb556710cce3c559d83114fad245bfdf56ea", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3-defaults/libpython3-stdlib_3.10.6-1~22.04.1_amd64.deb" + ], + "version": "3.10.6-1~22.04.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "sha256": "123b338b5857a35e145c2bd7ce890434dbed2e7f6cd03505e86377f9e0aac40b", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3.10/libpython3.10-stdlib_3.10.12-1~22.04.10_amd64.deb" + ], + "version": "3.10.12-1~22.04.10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "sha256": "3a88f3c8c58a32c623b79f1caace557f63cb997c03fa18eaa31ca0285f635919", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/u/util-linux/libuuid1_2.37.2-4ubuntu3.4_amd64.deb" + ], + "version": "2.37.2-4ubuntu3.4" }, { "arch": "amd64", @@ -733,6 +757,28 @@ ], "version": "2.3.1-1" }, + { + "arch": "amd64", + "dependencies": [], + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "sha256": "9dc17e51a1be2d9ed63b7b84ef0e4e29c5abe6f1bc62cb03e7181483cce8a2f2", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/z/zlib/zlib1g_1.2.11.dfsg-2ubuntu9.2_amd64.deb" + ], + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "sha256": "ae7db00ce8b093e50c994518b90203544e063b4bc574836a048bb142b950b2c9", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libz/libzstd/libzstd1_1.4.8+dfsg-3build1_amd64.deb" + ], + "version": "1.4.8+dfsg-3build1" + }, { "arch": "amd64", "dependencies": [], @@ -791,1275 +837,14 @@ { "arch": "amd64", "dependencies": [], - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "sha256": "cfeef478f96ace59617f4f93c2497776b98a33c99bf3602af46844ccf9cba9d3", + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "sha256": "1ec03fe34a7dfd8640a0564a3cad385a10b4f68ebf2190a4c08ccadb395133d3", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libn/libnsl/libnsl2_1.3.0-2build2_amd64.deb" - ], - "version": "1.3.0-2build2" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "sha256": "447c6d8f7624a3d728a12232440d3f10e6e53bbf60cc202b7beb8a3bc9f7acc9", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/r/rpcsvc-proto/rpcsvc-proto_1.4.2-0ubuntu6_amd64.deb" - ], - "version": "1.4.2-0ubuntu6" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "sha256": "a79be2f6e45823dcc09e04d5e98c88ec88d07d5b8895d05b875a8ade8b345efa", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libx/libxcrypt/libcrypt-dev_4.4.27-1_amd64.deb" - ], - "version": "1:4.4.27-1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "sha256": "f534bb052e96efb9f9e08279b18950d1b1f2a7958f3a06c426fed7d2c1500840", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/l/linux/linux-libc-dev_5.15.0-143.153_amd64.deb" - ], - "version": "5.15.0-143.153" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "sha256": "d93f29bc5ad18f69a4fe292fd392dc17e0a9d9bf0a22ee11fcf43e6395b6ecf8", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc-dev-bin_2.35-0ubuntu3.10_amd64.deb" - ], - "version": "2.35-0ubuntu3.10" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "sha256": "88e5178ccdd4317e263dbe03db1a54a6e35ae1f1fd057585655c2b06ab487f6e", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libncursesw6_6.3-2ubuntu0.1_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libtinfo6_6.3-2ubuntu0.1_amd64.deb" ], "version": "6.3-2ubuntu0.1" }, - { - "arch": "amd64", - "dependencies": [], - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "sha256": "4a828e6cb2a721be6f4ed7b34256d3fc9aab1495b2b4923fd579d4dc589496fd", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libncurses6_6.3-2ubuntu0.1_amd64.deb" - ], - "version": "6.3-2ubuntu0.1" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", - "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" - }, - { - "key": "rocrand_3.3.0.60401-83_22.04_amd64", - "name": "rocrand", - "version": "3.3.0.60401-83~22.04" - }, - { - "key": "hipblaslt_0.12.1.60401-83_22.04_amd64", - "name": "hipblaslt", - "version": "0.12.1.60401-83~22.04" - }, - { - "key": "roctracer_4.1.60401.60401-83_22.04_amd64", - "name": "roctracer", - "version": "4.1.60401.60401-83~22.04" - }, - { - "key": "rocblas_4.4.0.60401-83_22.04_amd64", - "name": "rocblas", - "version": "4.4.0.60401-83~22.04" - }, - { - "key": "hip-runtime-amd_6.4.43483.60401-83_22.04_amd64", - "name": "hip-runtime-amd", - "version": "6.4.43483.60401-83~22.04" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "rocprofiler-register_0.4.0.60401-83_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.4.0.60401-83~22.04" - }, - { - "key": "comgr_3.0.0.60401-83_22.04_amd64", - "name": "comgr", - "version": "3.0.0.60401-83~22.04" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "rocminfo_1.0.0.60401-83_22.04_amd64", - "name": "rocminfo", - "version": "1.0.0.60401-83~22.04" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "pciutils_1-3.7.0-6_amd64", - "name": "pciutils", - "version": "1:3.7.0-6" - }, - { - "key": "libpci3_1-3.7.0-6_amd64", - "name": "libpci3", - "version": "1:3.7.0-6" - }, - { - "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", - "name": "pci.ids", - "version": "0.0~2022.01.22-1ubuntu0.1" - }, - { - "key": "libudev1_249.11-0ubuntu3.16_amd64", - "name": "libudev1", - "version": "249.11-0ubuntu3.16" - }, - { - "key": "libkmod2_29-1ubuntu1_amd64", - "name": "libkmod2", - "version": "29-1ubuntu1" - }, - { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.10-2166611.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.10-2166611.22.04" - }, - { - "key": "amdgpu-core_1-6.4.60401-2164967.22.04_amd64", - "name": "amdgpu-core", - "version": "1:6.4.60401-2164967.22.04" - }, - { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" - } - ], - "key": "miopen-hip_3.4.0.60401-83_22.04_amd64", - "name": "miopen-hip", - "sha256": "584431b94f2809f553633a887a7f88d308391114cc9547da305d5ddcd84bd3e5", - "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/m/miopen-hip/miopen-hip_3.4.0.60401-83~22.04_amd64.deb" - ], - "version": "3.4.0.60401-83~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "rocrand_3.3.0.60401-83_22.04_amd64", - "name": "rocrand", - "sha256": "8d77ce48dc4256bcdba122c9243febd083ffc9147c340ef793f362d8611e35d3", - "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/r/rocrand/rocrand_3.3.0.60401-83~22.04_amd64.deb" - ], - "version": "3.3.0.60401-83~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", - "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" - }, - { - "key": "roctracer_4.1.60401.60401-83_22.04_amd64", - "name": "roctracer", - "version": "4.1.60401.60401-83~22.04" - } - ], - "key": "hipblaslt_0.12.1.60401-83_22.04_amd64", - "name": "hipblaslt", - "sha256": "23e6facad7bd1a17223761e06e9c6d1561c5a9e80f052e5f1589d21ab7167e9e", - "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/h/hipblaslt/hipblaslt_0.12.1.60401-83~22.04_amd64.deb" - ], - "version": "0.12.1.60401-83~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "roctracer_4.1.60401.60401-83_22.04_amd64", - "name": "roctracer", - "sha256": "58cead537cf07c8a8770bfe28346c3b3c92cc4297b51e307c9032b04434b187c", - "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/r/roctracer/roctracer_4.1.60401.60401-83~22.04_amd64.deb" - ], - "version": "4.1.60401.60401-83~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", - "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" - }, - { - "key": "hip-runtime-amd_6.4.43483.60401-83_22.04_amd64", - "name": "hip-runtime-amd", - "version": "6.4.43483.60401-83~22.04" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "rocprofiler-register_0.4.0.60401-83_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.4.0.60401-83~22.04" - }, - { - "key": "comgr_3.0.0.60401-83_22.04_amd64", - "name": "comgr", - "version": "3.0.0.60401-83~22.04" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "rocminfo_1.0.0.60401-83_22.04_amd64", - "name": "rocminfo", - "version": "1.0.0.60401-83~22.04" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "pciutils_1-3.7.0-6_amd64", - "name": "pciutils", - "version": "1:3.7.0-6" - }, - { - "key": "libpci3_1-3.7.0-6_amd64", - "name": "libpci3", - "version": "1:3.7.0-6" - }, - { - "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", - "name": "pci.ids", - "version": "0.0~2022.01.22-1ubuntu0.1" - }, - { - "key": "libudev1_249.11-0ubuntu3.16_amd64", - "name": "libudev1", - "version": "249.11-0ubuntu3.16" - }, - { - "key": "libkmod2_29-1ubuntu1_amd64", - "name": "libkmod2", - "version": "29-1ubuntu1" - }, - { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.10-2166611.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.10-2166611.22.04" - }, - { - "key": "amdgpu-core_1-6.4.60401-2164967.22.04_amd64", - "name": "amdgpu-core", - "version": "1:6.4.60401-2164967.22.04" - }, - { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" - }, - { - "key": "hipblaslt_0.12.1.60401-83_22.04_amd64", - "name": "hipblaslt", - "version": "0.12.1.60401-83~22.04" - }, - { - "key": "roctracer_4.1.60401.60401-83_22.04_amd64", - "name": "roctracer", - "version": "4.1.60401.60401-83~22.04" - } - ], - "key": "rocblas_4.4.0.60401-83_22.04_amd64", - "name": "rocblas", - "sha256": "0f850a080c312d80c0b928b48efca4fb507c64e95536aaefe7f23c6fc0c1fe09", - "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/r/rocblas/rocblas_4.4.0.60401-83~22.04_amd64.deb" - ], - "version": "4.4.0.60401-83~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "rocprofiler-register_0.4.0.60401-83_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.4.0.60401-83~22.04" - }, - { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", - "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" - }, - { - "key": "comgr_3.0.0.60401-83_22.04_amd64", - "name": "comgr", - "version": "3.0.0.60401-83~22.04" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "rocminfo_1.0.0.60401-83_22.04_amd64", - "name": "rocminfo", - "version": "1.0.0.60401-83~22.04" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "pciutils_1-3.7.0-6_amd64", - "name": "pciutils", - "version": "1:3.7.0-6" - }, - { - "key": "libpci3_1-3.7.0-6_amd64", - "name": "libpci3", - "version": "1:3.7.0-6" - }, - { - "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", - "name": "pci.ids", - "version": "0.0~2022.01.22-1ubuntu0.1" - }, - { - "key": "libudev1_249.11-0ubuntu3.16_amd64", - "name": "libudev1", - "version": "249.11-0ubuntu3.16" - }, - { - "key": "libkmod2_29-1ubuntu1_amd64", - "name": "libkmod2", - "version": "29-1ubuntu1" - }, - { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.10-2166611.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.10-2166611.22.04" - }, - { - "key": "amdgpu-core_1-6.4.60401-2164967.22.04_amd64", - "name": "amdgpu-core", - "version": "1:6.4.60401-2164967.22.04" - }, - { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" - } - ], - "key": "hip-runtime-amd_6.4.43483.60401-83_22.04_amd64", - "name": "hip-runtime-amd", - "sha256": "dee21498aa4fd4dd267efad563b4cc7fc3363d60f46b626c23681631f5f0cf79", - "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/h/hip-runtime-amd/hip-runtime-amd_6.4.43483.60401-83~22.04_amd64.deb" - ], - "version": "6.4.43483.60401-83~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "rocminfo_1.0.0.60401-83_22.04_amd64", - "name": "rocminfo", - "sha256": "55d19b59464dd444002d12bf1fe8c762c15f2ae79f9dccbbf38360b42e3156b6", - "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/r/rocminfo/rocminfo_1.0.0.60401-83~22.04_amd64.deb" - ], - "version": "1.0.0.60401-83~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "sha256": "43dbcc12d790ed9360be4a1dc690ac0a7464a5e7c3170785886df84811a3f5a5", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3-defaults/python3_3.10.6-1~22.04.1_amd64.deb" - ], - "version": "3.10.6-1~22.04.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "sha256": "41b8f285f0a1661ece778a63592dcb556710cce3c559d83114fad245bfdf56ea", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3-defaults/libpython3-stdlib_3.10.6-1~22.04.1_amd64.deb" - ], - "version": "3.10.6-1~22.04.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "sha256": "123b338b5857a35e145c2bd7ce890434dbed2e7f6cd03505e86377f9e0aac40b", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3.10/libpython3.10-stdlib_3.10.12-1~22.04.10_amd64.deb" - ], - "version": "3.10.12-1~22.04.10" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "sha256": "3a88f3c8c58a32c623b79f1caace557f63cb997c03fa18eaa31ca0285f635919", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/u/util-linux/libuuid1_2.37.2-4ubuntu3.4_amd64.deb" - ], - "version": "2.37.2-4ubuntu3.4" - }, { "arch": "amd64", "dependencies": [], @@ -2093,6 +878,28 @@ ], "version": "8.1.2-1" }, + { + "arch": "amd64", + "dependencies": [], + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "sha256": "cfeef478f96ace59617f4f93c2497776b98a33c99bf3602af46844ccf9cba9d3", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libn/libnsl/libnsl2_1.3.0-2build2_amd64.deb" + ], + "version": "1.3.0-2build2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "sha256": "88e5178ccdd4317e263dbe03db1a54a6e35ae1f1fd057585655c2b06ab487f6e", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libncursesw6_6.3-2ubuntu0.1_amd64.deb" + ], + "version": "6.3-2ubuntu0.1" + }, { "arch": "amd64", "dependencies": [], @@ -2195,77 +1002,333 @@ { "arch": "amd64", "dependencies": [], - "key": "pciutils_1-3.7.0-6_amd64", - "name": "pciutils", - "sha256": "830f2957662723860d3ebac949b22867df00d9a4c5ab1c2bf98e10674dc1c756", + "key": "libelf1_0.186-1ubuntu0.1_amd64", + "name": "libelf1", + "sha256": "2cc781614b045b450a1e073ee3b0902f393ee58ede1d2ea4526ad805fcccf538", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pciutils/pciutils_3.7.0-6_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/e/elfutils/libelf1_0.186-1ubuntu0.1_amd64.deb" ], - "version": "1:3.7.0-6" + "version": "0.186-1ubuntu0.1" }, { "arch": "amd64", "dependencies": [], - "key": "libpci3_1-3.7.0-6_amd64", - "name": "libpci3", - "sha256": "624bea7b5a9073a5bc2dfa5d956571d6127c20400fcfcfca1e47b7448ba5d351", + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "sha256": "0721c89001fbbd1ada23e89da5d60e762763c1a7b3dc814a2e9a518480a8043d", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pciutils/libpci3_3.7.0-6_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/numactl/libnuma1_2.0.14-3ubuntu2_amd64.deb" ], - "version": "1:3.7.0-6" + "version": "2.0.14-3ubuntu2" }, { "arch": "amd64", "dependencies": [], - "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", - "name": "pci.ids", - "sha256": "026609ed71363cd6ed13ace2dac0c9813d94be3874c778539a8a64d6d2cc5566", + "key": "libdrm-amdgpu1_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-amdgpu1", + "sha256": "a284d1d55956ed2f5f1ece4e2df9104ff87052a6c27a8150574c30c757ecd4b8", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pci.ids/pci.ids_0.0~2022.01.22-1ubuntu0.1_all.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libd/libdrm/libdrm-amdgpu1_2.4.113-2~ubuntu0.22.04.1_amd64.deb" ], - "version": "0.0~2022.01.22-1ubuntu0.1" + "version": "2.4.113-2~ubuntu0.22.04.1" }, { "arch": "amd64", "dependencies": [], - "key": "libudev1_249.11-0ubuntu3.16_amd64", - "name": "libudev1", - "sha256": "451d4eaa84dbeabc045611937a78489710ad2c2baf277a63aacda7ee5226e3ea", + "key": "libdrm2_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm2", + "sha256": "555925f1caedfd98d89bae3bc11b4fe3401c2c4817dbbafcab25e1144582e946", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/s/systemd/libudev1_249.11-0ubuntu3.16_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libd/libdrm/libdrm2_2.4.113-2~ubuntu0.22.04.1_amd64.deb" ], - "version": "249.11-0ubuntu3.16" + "version": "2.4.113-2~ubuntu0.22.04.1" }, { "arch": "amd64", "dependencies": [], - "key": "libkmod2_29-1ubuntu1_amd64", - "name": "libkmod2", - "sha256": "f355cebb66405a84cc88bb313440abd39f97cf45d3f468ea9ec7ba34e11b6c8a", + "key": "libdrm-common_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-common", + "sha256": "35a306712d8b15b30c42ecd73ec087813eb01c0b3125dc8f7ca2b5134e133522", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/k/kmod/libkmod2_29-1ubuntu1_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libd/libdrm/libdrm-common_2.4.113-2~ubuntu0.22.04.1_all.deb" ], - "version": "29-1ubuntu1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "hsa-runtime-rocr4wsl-amdgpu_25.10-2166611.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "sha256": "5d726440fdedc0a14a818747adc1ce26e5447e88a4260a580afacfafbf68cfba", - "urls": [ - "https://repo.radeon.com/amdgpu/6.4.1/ubuntu/pool/main/h/hsa-runtime-rocr4wsl-amdgpu/hsa-runtime-rocr4wsl-amdgpu_25.10-2166611.22.04_amd64.deb" - ], - "version": "25.10-2166611.22.04" + "version": "2.4.113-2~ubuntu0.22.04.1" }, { "arch": "amd64", "dependencies": [ + { + "key": "rocm-core_7.1.0.70100-20_22.04_amd64", + "name": "rocm-core", + "version": "7.1.0.70100-20~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, { "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", "name": "libstdc++6", "version": "12.3.0-1ubuntu1~22.04" }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "hsa-rocr_1.18.0.70100-20_22.04_amd64", + "name": "hsa-rocr", + "version": "1.18.0.70100-20~22.04" + }, + { + "key": "rocprofiler-register_0.6.0.70100-20_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70100-20~22.04" + }, + { + "key": "libelf1_0.186-1ubuntu0.1_amd64", + "name": "libelf1", + "version": "0.186-1ubuntu0.1" + }, + { + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" + }, + { + "key": "libdrm-amdgpu1_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-amdgpu1", + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "key": "libdrm2_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm2", + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "key": "libdrm-common_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-common", + "version": "2.4.113-2~ubuntu0.22.04.1" + } + ], + "key": "hsa-amd-aqlprofile_1.0.0.70100-20_22.04_amd64", + "name": "hsa-amd-aqlprofile", + "sha256": "391d37c8c3289777c32c0eafef8e673dc8207a6635da9ca91093a251fa063fce", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1/pool/main/h/hsa-amd-aqlprofile/hsa-amd-aqlprofile_1.0.0.70100-20~22.04_amd64.deb" + ], + "version": "1.0.0.70100-20~22.04" + }, + { + "arch": "amd64", + "dependencies": [ { "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", "name": "libgcc-s1", @@ -2287,14 +1350,1739 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", - "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "rocm-smi-lib_7.5.0.60401-83_22.04_amd64", - "name": "rocm-smi-lib", - "version": "7.5.0.60401-83~22.04" + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "rocm-core_7.1.0.70100-20_22.04_amd64", + "name": "rocm-core", + "version": "7.1.0.70100-20~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", + "name": "libtinfo-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", + "name": "libncurses-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", + "name": "ncurses-bin", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libncurses6_6.3-2ubuntu0.1_amd64", + "name": "libncurses6", + "version": "6.3-2ubuntu0.1" + } + ], + "key": "comgr_3.0.0.70100-20_22.04_amd64", + "name": "comgr", + "sha256": "65e7c64e61585838ca55a33a8b7c8b6fa36f0a7b57bac8a4c2b9a3ab484d2b48", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1/pool/main/c/comgr/comgr_3.0.0.70100-20~22.04_amd64.deb" + ], + "version": "3.0.0.70100-20~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", + "name": "libtinfo-dev", + "sha256": "145736b5bec52cd4fc1b2c09517759383b95255f9fd304ace1de7d830295bb33", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libtinfo-dev_6.3-2ubuntu0.1_amd64.deb" + ], + "version": "6.3-2ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", + "name": "libncurses-dev", + "sha256": "1de196ffdd885b5953a44cde13e68b77dc7ceb9c9921482dc84e358bd1ad8b02", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libncurses-dev_6.3-2ubuntu0.1_amd64.deb" + ], + "version": "6.3-2ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", + "name": "ncurses-bin", + "sha256": "6382f5c1d7d0bf39cbb94e208ece2bdf556cb3a8dfe278748542c7b8b57a2e6d", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/ncurses-bin_6.3-2ubuntu0.1_amd64.deb" + ], + "version": "6.3-2ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "sha256": "c2e3b50af761886617462ca47ebc9303a83968129acca2add42b8eba0a7abca3", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc6-dev_2.35-0ubuntu3.10_amd64.deb" + ], + "version": "2.35-0ubuntu3.10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "sha256": "e93aa448fe708a4bbfcc40b2a8cab6f5f88896174a301caab2719f734e08a621", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libn/libnsl/libnsl-dev_1.3.0-2build2_amd64.deb" + ], + "version": "1.3.0-2build2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "sha256": "f4254aff5a32ef3c9fca06d3f4f2eff71c93b9393cc1ecc718f7478a5d6a83da", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libt/libtirpc/libtirpc-dev_1.3.2-2ubuntu0.1_amd64.deb" + ], + "version": "1.3.2-2ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "sha256": "447c6d8f7624a3d728a12232440d3f10e6e53bbf60cc202b7beb8a3bc9f7acc9", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/r/rpcsvc-proto/rpcsvc-proto_1.4.2-0ubuntu6_amd64.deb" + ], + "version": "1.4.2-0ubuntu6" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "sha256": "a79be2f6e45823dcc09e04d5e98c88ec88d07d5b8895d05b875a8ade8b345efa", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libx/libxcrypt/libcrypt-dev_4.4.27-1_amd64.deb" + ], + "version": "1:4.4.27-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "sha256": "f534bb052e96efb9f9e08279b18950d1b1f2a7958f3a06c426fed7d2c1500840", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/l/linux/linux-libc-dev_5.15.0-143.153_amd64.deb" + ], + "version": "5.15.0-143.153" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "sha256": "d93f29bc5ad18f69a4fe292fd392dc17e0a9d9bf0a22ee11fcf43e6395b6ecf8", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc-dev-bin_2.35-0ubuntu3.10_amd64.deb" + ], + "version": "2.35-0ubuntu3.10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libncurses6_6.3-2ubuntu0.1_amd64", + "name": "libncurses6", + "sha256": "4a828e6cb2a721be6f4ed7b34256d3fc9aab1495b2b4923fd579d4dc589496fd", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libncurses6_6.3-2ubuntu0.1_amd64.deb" + ], + "version": "6.3-2ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "rocprofiler-sdk-rocpd_1.0.0-20_22.04_amd64", + "name": "rocprofiler-sdk-rocpd", + "version": "1.0.0-20~22.04" + }, + { + "key": "rocm-core_7.1.0.70100-20_22.04_amd64", + "name": "rocm-core", + "version": "7.1.0.70100-20~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "rocprofiler-sdk-roctx_1.0.0-20_22.04_amd64", + "name": "rocprofiler-sdk-roctx", + "version": "1.0.0-20~22.04" + }, + { + "key": "rocprofiler-register_0.6.0.70100-20_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70100-20~22.04" + } + ], + "key": "rocprofiler-sdk_1.0.0-20_22.04_amd64", + "name": "rocprofiler-sdk", + "sha256": "8f030bbd32c4143e66af52de677d5bbfc2b2d3d943533478b06426ea164d5bd5", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1/pool/main/r/rocprofiler-sdk/rocprofiler-sdk_1.0.0-20~22.04_amd64.deb" + ], + "version": "1.0.0-20~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "rocprofiler-sdk-rocpd_1.0.0-20_22.04_amd64", + "name": "rocprofiler-sdk-rocpd", + "sha256": "0703408ab221d81c4c1441caf076497e9d3f587b1a735d9554a7f11b21d2ca95", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1/pool/main/r/rocprofiler-sdk-rocpd/rocprofiler-sdk-rocpd_1.0.0-20~22.04_amd64.deb" + ], + "version": "1.0.0-20~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "rocprofiler-sdk-roctx_1.0.0-20_22.04_amd64", + "name": "rocprofiler-sdk-roctx", + "sha256": "363d004404bc8d7fabd11fe064b74c3de4b7fb63224f3dd0d599f55b4d6d73d3", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1/pool/main/r/rocprofiler-sdk-roctx/rocprofiler-sdk-roctx_1.0.0-20~22.04_amd64.deb" + ], + "version": "1.0.0-20~22.04" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "rocm-core_7.1.0.70100-20_22.04_amd64", + "name": "rocm-core", + "version": "7.1.0.70100-20~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "rocrand_4.1.0.70100-20_22.04_amd64", + "name": "rocrand", + "version": "4.1.0.70100-20~22.04" + }, + { + "key": "hipblaslt_1.1.0.70100-20_22.04_amd64", + "name": "hipblaslt", + "version": "1.1.0.70100-20~22.04" + }, + { + "key": "roctracer_4.1.70100.70100-20_22.04_amd64", + "name": "roctracer", + "version": "4.1.70100.70100-20~22.04" + }, + { + "key": "rocblas_5.1.0.70100-20_22.04_amd64", + "name": "rocblas", + "version": "5.1.0.70100-20~22.04" + }, + { + "key": "hip-runtime-amd_7.1.25424.70100-20_22.04_amd64", + "name": "hip-runtime-amd", + "version": "7.1.25424.70100-20~22.04" + }, + { + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" + }, + { + "key": "rocprofiler-register_0.6.0.70100-20_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70100-20~22.04" + }, + { + "key": "comgr_3.0.0.70100-20_22.04_amd64", + "name": "comgr", + "version": "3.0.0.70100-20~22.04" + }, + { + "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", + "name": "libtinfo-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", + "name": "libncurses-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", + "name": "ncurses-bin", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libncurses6_6.3-2ubuntu0.1_amd64", + "name": "libncurses6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "rocminfo_1.0.0.70100-20_22.04_amd64", + "name": "rocminfo", + "version": "1.0.0.70100-20~22.04" + }, + { + "key": "pciutils_1-3.7.0-6_amd64", + "name": "pciutils", + "version": "1:3.7.0-6" + }, + { + "key": "libpci3_1-3.7.0-6_amd64", + "name": "libpci3", + "version": "1:3.7.0-6" + }, + { + "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", + "name": "pci.ids", + "version": "0.0~2022.01.22-1ubuntu0.1" + }, + { + "key": "libudev1_249.11-0ubuntu3.16_amd64", + "name": "libudev1", + "version": "249.11-0ubuntu3.16" + }, + { + "key": "libkmod2_29-1ubuntu1_amd64", + "name": "libkmod2", + "version": "29-1ubuntu1" + }, + { + "key": "hsa-rocr_1.18.0.70100-20_22.04_amd64", + "name": "hsa-rocr", + "version": "1.18.0.70100-20~22.04" + }, + { + "key": "libelf1_0.186-1ubuntu0.1_amd64", + "name": "libelf1", + "version": "0.186-1ubuntu0.1" + }, + { + "key": "libdrm-amdgpu1_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-amdgpu1", + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "key": "libdrm2_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm2", + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "key": "libdrm-common_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-common", + "version": "2.4.113-2~ubuntu0.22.04.1" + } + ], + "key": "miopen-hip_3.5.1.70100-20_22.04_amd64", + "name": "miopen-hip", + "sha256": "704d3d6408689caa3fb8dea476893eac9da552b5a60d47776bb9d9cb88c30e0e", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1/pool/main/m/miopen-hip/miopen-hip_3.5.1.70100-20~22.04_amd64.deb" + ], + "version": "3.5.1.70100-20~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "rocrand_4.1.0.70100-20_22.04_amd64", + "name": "rocrand", + "sha256": "ce51d36a6de1738bbd2a195f91d4f70e49e6062e5da225b88cb2be694629fbde", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1/pool/main/r/rocrand/rocrand_4.1.0.70100-20~22.04_amd64.deb" + ], + "version": "4.1.0.70100-20~22.04" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "rocm-core_7.1.0.70100-20_22.04_amd64", + "name": "rocm-core", + "version": "7.1.0.70100-20~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "roctracer_4.1.70100.70100-20_22.04_amd64", + "name": "roctracer", + "version": "4.1.70100.70100-20~22.04" + } + ], + "key": "hipblaslt_1.1.0.70100-20_22.04_amd64", + "name": "hipblaslt", + "sha256": "0f5e753be845eb09eea483d0004a9fb4e7d916fbca35b2999625e26253e72147", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1/pool/main/h/hipblaslt/hipblaslt_1.1.0.70100-20~22.04_amd64.deb" + ], + "version": "1.1.0.70100-20~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "roctracer_4.1.70100.70100-20_22.04_amd64", + "name": "roctracer", + "sha256": "90b9a31fed0d2803a8ba6afd1409ae059e6fbe56d028ebcd72f2d5902f1f770c", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1/pool/main/r/roctracer/roctracer_4.1.70100.70100-20~22.04_amd64.deb" + ], + "version": "4.1.70100.70100-20~22.04" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "rocm-core_7.1.0.70100-20_22.04_amd64", + "name": "rocm-core", + "version": "7.1.0.70100-20~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "hip-runtime-amd_7.1.25424.70100-20_22.04_amd64", + "name": "hip-runtime-amd", + "version": "7.1.25424.70100-20~22.04" + }, + { + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" + }, + { + "key": "rocprofiler-register_0.6.0.70100-20_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70100-20~22.04" + }, + { + "key": "comgr_3.0.0.70100-20_22.04_amd64", + "name": "comgr", + "version": "3.0.0.70100-20~22.04" + }, + { + "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", + "name": "libtinfo-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", + "name": "libncurses-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", + "name": "ncurses-bin", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libncurses6_6.3-2ubuntu0.1_amd64", + "name": "libncurses6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "rocminfo_1.0.0.70100-20_22.04_amd64", + "name": "rocminfo", + "version": "1.0.0.70100-20~22.04" + }, + { + "key": "pciutils_1-3.7.0-6_amd64", + "name": "pciutils", + "version": "1:3.7.0-6" + }, + { + "key": "libpci3_1-3.7.0-6_amd64", + "name": "libpci3", + "version": "1:3.7.0-6" + }, + { + "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", + "name": "pci.ids", + "version": "0.0~2022.01.22-1ubuntu0.1" + }, + { + "key": "libudev1_249.11-0ubuntu3.16_amd64", + "name": "libudev1", + "version": "249.11-0ubuntu3.16" + }, + { + "key": "libkmod2_29-1ubuntu1_amd64", + "name": "libkmod2", + "version": "29-1ubuntu1" + }, + { + "key": "hsa-rocr_1.18.0.70100-20_22.04_amd64", + "name": "hsa-rocr", + "version": "1.18.0.70100-20~22.04" + }, + { + "key": "libelf1_0.186-1ubuntu0.1_amd64", + "name": "libelf1", + "version": "0.186-1ubuntu0.1" + }, + { + "key": "libdrm-amdgpu1_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-amdgpu1", + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "key": "libdrm2_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm2", + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "key": "libdrm-common_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-common", + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "key": "hipblaslt_1.1.0.70100-20_22.04_amd64", + "name": "hipblaslt", + "version": "1.1.0.70100-20~22.04" + }, + { + "key": "roctracer_4.1.70100.70100-20_22.04_amd64", + "name": "roctracer", + "version": "4.1.70100.70100-20~22.04" + } + ], + "key": "rocblas_5.1.0.70100-20_22.04_amd64", + "name": "rocblas", + "sha256": "367ee19978ff7469c318528180e759f78cfa114aaf4cfe8bcc417481fab9b749", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1/pool/main/r/rocblas/rocblas_5.1.0.70100-20~22.04_amd64.deb" + ], + "version": "5.1.0.70100-20~22.04" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" + }, + { + "key": "rocprofiler-register_0.6.0.70100-20_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70100-20~22.04" + }, + { + "key": "rocm-core_7.1.0.70100-20_22.04_amd64", + "name": "rocm-core", + "version": "7.1.0.70100-20~22.04" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -2492,24 +3280,9 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "hip-runtime-amd_6.4.43483.60401-83_22.04_amd64", - "name": "hip-runtime-amd", - "version": "6.4.43483.60401-83~22.04" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "rocprofiler-register_0.4.0.60401-83_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.4.0.60401-83~22.04" - }, - { - "key": "comgr_3.0.0.60401-83_22.04_amd64", + "key": "comgr_3.0.0.70100-20_22.04_amd64", "name": "comgr", - "version": "3.0.0.60401-83~22.04" + "version": "3.0.0.70100-20~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -2567,9 +3340,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.60401-83_22.04_amd64", + "key": "rocminfo_1.0.0.70100-20_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.60401-83~22.04" + "version": "1.0.0.70100-20~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -2597,84 +3370,495 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.10-2166611.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.10-2166611.22.04" - }, - { - "key": "amdgpu-core_1-6.4.60401-2164967.22.04_amd64", - "name": "amdgpu-core", - "version": "1:6.4.60401-2164967.22.04" + "key": "hsa-rocr_1.18.0.70100-20_22.04_amd64", + "name": "hsa-rocr", + "version": "1.18.0.70100-20~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", "name": "libelf1", "version": "0.186-1ubuntu0.1" + }, + { + "key": "libdrm-amdgpu1_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-amdgpu1", + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "key": "libdrm2_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm2", + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "key": "libdrm-common_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-common", + "version": "2.4.113-2~ubuntu0.22.04.1" } ], - "key": "rccl_2.22.3.60401-83_22.04_amd64", - "name": "rccl", - "sha256": "6b5675f7bb81b03387a64478a896a671d10e83a5629e54a0c3a8bf503a7ac106", + "key": "hip-runtime-amd_7.1.25424.70100-20_22.04_amd64", + "name": "hip-runtime-amd", + "sha256": "e21a53a2537413baf03b2936a016c5b0701e12f8bc42872346b52252cb8bbd30", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/r/rccl/rccl_2.22.3.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1/pool/main/h/hip-runtime-amd/hip-runtime-amd_7.1.25424.70100-20~22.04_amd64.deb" ], - "version": "2.22.3.60401-83~22.04" + "version": "7.1.25424.70100-20~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocm-smi-lib_7.5.0.60401-83_22.04_amd64", - "name": "rocm-smi-lib", - "sha256": "3db3358e0ae302b4589e586d6b9dae2ae183d521e61979723d9eb918c7bfbde1", + "key": "rocminfo_1.0.0.70100-20_22.04_amd64", + "name": "rocminfo", + "sha256": "59ef33531a4e093780218a9f7ca929dcb51aa9d6f55596243acae64786c7e36c", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/r/rocm-smi-lib/rocm-smi-lib_7.5.0.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1/pool/main/r/rocminfo/rocminfo_1.0.0.70100-20~22.04_amd64.deb" ], - "version": "7.5.0.60401-83~22.04" + "version": "1.0.0.70100-20~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "pciutils_1-3.7.0-6_amd64", + "name": "pciutils", + "sha256": "830f2957662723860d3ebac949b22867df00d9a4c5ab1c2bf98e10674dc1c756", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pciutils/pciutils_3.7.0-6_amd64.deb" + ], + "version": "1:3.7.0-6" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libpci3_1-3.7.0-6_amd64", + "name": "libpci3", + "sha256": "624bea7b5a9073a5bc2dfa5d956571d6127c20400fcfcfca1e47b7448ba5d351", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pciutils/libpci3_3.7.0-6_amd64.deb" + ], + "version": "1:3.7.0-6" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", + "name": "pci.ids", + "sha256": "026609ed71363cd6ed13ace2dac0c9813d94be3874c778539a8a64d6d2cc5566", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pci.ids/pci.ids_0.0~2022.01.22-1ubuntu0.1_all.deb" + ], + "version": "0.0~2022.01.22-1ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libudev1_249.11-0ubuntu3.16_amd64", + "name": "libudev1", + "sha256": "451d4eaa84dbeabc045611937a78489710ad2c2baf277a63aacda7ee5226e3ea", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/s/systemd/libudev1_249.11-0ubuntu3.16_amd64.deb" + ], + "version": "249.11-0ubuntu3.16" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libkmod2_29-1ubuntu1_amd64", + "name": "libkmod2", + "sha256": "f355cebb66405a84cc88bb313440abd39f97cf45d3f468ea9ec7ba34e11b6c8a", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/k/kmod/libkmod2_29-1ubuntu1_amd64.deb" + ], + "version": "29-1ubuntu1" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "rocm-core_7.1.0.70100-20_22.04_amd64", "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" + "version": "7.1.0.70100-20~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "rocm-smi-lib_7.8.0.70100-20_22.04_amd64", + "name": "rocm-smi-lib", + "version": "7.8.0.70100-20~22.04" + }, + { + "key": "hip-runtime-amd_7.1.25424.70100-20_22.04_amd64", + "name": "hip-runtime-amd", + "version": "7.1.25424.70100-20~22.04" + }, + { + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" + }, + { + "key": "rocprofiler-register_0.6.0.70100-20_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70100-20~22.04" + }, + { + "key": "comgr_3.0.0.70100-20_22.04_amd64", + "name": "comgr", + "version": "3.0.0.70100-20~22.04" + }, + { + "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", + "name": "libtinfo-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", + "name": "libncurses-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", + "name": "ncurses-bin", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libncurses6_6.3-2ubuntu0.1_amd64", + "name": "libncurses6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "rocminfo_1.0.0.70100-20_22.04_amd64", + "name": "rocminfo", + "version": "1.0.0.70100-20~22.04" + }, + { + "key": "pciutils_1-3.7.0-6_amd64", + "name": "pciutils", + "version": "1:3.7.0-6" + }, + { + "key": "libpci3_1-3.7.0-6_amd64", + "name": "libpci3", + "version": "1:3.7.0-6" + }, + { + "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", + "name": "pci.ids", + "version": "0.0~2022.01.22-1ubuntu0.1" + }, + { + "key": "libudev1_249.11-0ubuntu3.16_amd64", + "name": "libudev1", + "version": "249.11-0ubuntu3.16" + }, + { + "key": "libkmod2_29-1ubuntu1_amd64", + "name": "libkmod2", + "version": "29-1ubuntu1" + }, + { + "key": "hsa-rocr_1.18.0.70100-20_22.04_amd64", + "name": "hsa-rocr", + "version": "1.18.0.70100-20~22.04" + }, + { + "key": "libelf1_0.186-1ubuntu0.1_amd64", + "name": "libelf1", + "version": "0.186-1ubuntu0.1" + }, + { + "key": "libdrm-amdgpu1_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-amdgpu1", + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "key": "libdrm2_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm2", + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "key": "libdrm-common_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-common", + "version": "2.4.113-2~ubuntu0.22.04.1" } ], - "key": "rocm-device-libs_1.0.0.60401-83_22.04_amd64", - "name": "rocm-device-libs", - "sha256": "080c1cbcade5a22cea800256eccc993116ae05b60f11785fda76beffe68c38bf", + "key": "rccl_2.27.7.70100-20_22.04_amd64", + "name": "rccl", + "sha256": "9e976a182ca64f3df924e4221c7374d63a6cbd9ebcb584bdd1fe64ed9dfb6a71", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/r/rocm-device-libs/rocm-device-libs_1.0.0.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1/pool/main/r/rccl/rccl_2.27.7.70100-20~22.04_amd64.deb" ], - "version": "1.0.0.60401-83~22.04" + "version": "2.27.7.70100-20~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "rocm-smi-lib_7.8.0.70100-20_22.04_amd64", + "name": "rocm-smi-lib", + "sha256": "d9c655dad3f075c2b94abbd339468e17381fe5ff3c1772429259b61012ea627f", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1/pool/main/r/rocm-smi-lib/rocm-smi-lib_7.8.0.70100-20~22.04_amd64.deb" + ], + "version": "7.8.0.70100-20~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "hipcc_1.1.1.60401-83_22.04_amd64", - "name": "hipcc", - "version": "1.1.1.60401-83~22.04" - }, - { - "key": "rocm-llvm_19.0.0.25184.60401-83_22.04_amd64", - "name": "rocm-llvm", - "version": "19.0.0.25184.60401-83~22.04" - }, - { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", + "key": "rocm-core_7.1.0.70100-20_22.04_amd64", "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" - }, - { - "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libgcc-11-dev", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", - "name": "libquadmath0", - "version": "12.3.0-1ubuntu1~22.04" + "version": "7.1.0.70100-20~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -2697,15 +3881,467 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libubsan1", - "version": "12.3.0-1ubuntu1~22.04" + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" }, { "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", "name": "libstdc++6", "version": "12.3.0-1ubuntu1~22.04" }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + } + ], + "key": "rocm-device-libs_1.0.0.70100-20_22.04_amd64", + "name": "rocm-device-libs", + "sha256": "8b18942b931b44ac3471d3b1ed08d271e0ed6b25f7771e745c3c7992cc147307", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1/pool/main/r/rocm-device-libs/rocm-device-libs_1.0.0.70100-20~22.04_amd64.deb" + ], + "version": "1.0.0.70100-20~22.04" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "hipcc_1.1.1.70100-20_22.04_amd64", + "name": "hipcc", + "version": "1.1.1.70100-20~22.04" + }, + { + "key": "rocm-llvm_20.0.0.25425.70100-20_22.04_amd64", + "name": "rocm-llvm", + "version": "20.0.0.25425.70100-20~22.04" + }, + { + "key": "rocm-core_7.1.0.70100-20_22.04_amd64", + "name": "rocm-core", + "version": "7.1.0.70100-20~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, { "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", "name": "libtsan0", @@ -2761,111 +4397,6 @@ "name": "libtirpc-dev", "version": "1.3.2-2ubuntu0.1" }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, { "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", "name": "rpcsvc-proto", @@ -2887,174 +4418,84 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "hsa-rocr-dev_1.15.0.60401-83_22.04_amd64", + "key": "hsa-rocr-dev_1.18.0.70100-20_22.04_amd64", "name": "hsa-rocr-dev", - "version": "1.15.0.60401-83~22.04" + "version": "1.18.0.70100-20~22.04" }, { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.10-2166611.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.10-2166611.22.04" + "key": "hsa-rocr_1.18.0.70100-20_22.04_amd64", + "name": "hsa-rocr", + "version": "1.18.0.70100-20~22.04" }, { - "key": "amdgpu-core_1-6.4.60401-2164967.22.04_amd64", - "name": "amdgpu-core", - "version": "1:6.4.60401-2164967.22.04" + "key": "rocprofiler-register_0.6.0.70100-20_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70100-20~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", "name": "libelf1", "version": "0.186-1ubuntu0.1" }, - { - "key": "libdrm-amdgpu-dev_1-2.4.124.60401-2164967.22.04_amd64", - "name": "libdrm-amdgpu-dev", - "version": "1:2.4.124.60401-2164967.22.04" - }, - { - "key": "valgrind_1-3.18.1-1ubuntu2_amd64", - "name": "valgrind", - "version": "1:3.18.1-1ubuntu2" - }, - { - "key": "libc6-dbg_2.35-0ubuntu3.10_amd64", - "name": "libc6-dbg", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libc6-i386_2.35-0ubuntu3.10_amd64", - "name": "libc6-i386", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libdrm-amdgpu-radeon1_1-2.4.124.60401-2164967.22.04_amd64", - "name": "libdrm-amdgpu-radeon1", - "version": "1:2.4.124.60401-2164967.22.04" - }, - { - "key": "libdrm2-amdgpu_1-2.4.124.60401-2164967.22.04_amd64", - "name": "libdrm2-amdgpu", - "version": "1:2.4.124.60401-2164967.22.04" - }, - { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.124.60401-2164967.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.124.60401-2164967.22.04" - }, - { - "key": "libdrm-amdgpu-common_1.0.0.60401-2164967.22.04_amd64", - "name": "libdrm-amdgpu-common", - "version": "1.0.0.60401-2164967.22.04" - }, - { - "key": "hip-runtime-amd_6.4.43483.60401-83_22.04_amd64", - "name": "hip-runtime-amd", - "version": "6.4.43483.60401-83~22.04" - }, { "key": "libnuma1_2.0.14-3ubuntu2_amd64", "name": "libnuma1", "version": "2.0.14-3ubuntu2" }, { - "key": "rocprofiler-register_0.4.0.60401-83_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.4.0.60401-83~22.04" + "key": "libdrm-amdgpu1_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-amdgpu1", + "version": "2.4.113-2~ubuntu0.22.04.1" }, { - "key": "comgr_3.0.0.60401-83_22.04_amd64", + "key": "libdrm2_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm2", + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "key": "libdrm-common_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-common", + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "key": "libdrm-dev_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-dev", + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "key": "libpciaccess-dev_0.16-3_amd64", + "name": "libpciaccess-dev", + "version": "0.16-3" + }, + { + "key": "libpciaccess0_0.16-3_amd64", + "name": "libpciaccess0", + "version": "0.16-3" + }, + { + "key": "libdrm-nouveau2_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-nouveau2", + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "key": "libdrm-radeon1_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-radeon1", + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "key": "libdrm-intel1_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-intel1", + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "key": "hip-runtime-amd_7.1.25424.70100-20_22.04_amd64", + "name": "hip-runtime-amd", + "version": "7.1.25424.70100-20~22.04" + }, + { + "key": "comgr_3.0.0.70100-20_22.04_amd64", "name": "comgr", - "version": "3.0.0.60401-83~22.04" + "version": "3.0.0.70100-20~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -3077,9 +4518,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.60401-83_22.04_amd64", + "key": "rocminfo_1.0.0.70100-20_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.60401-83~22.04" + "version": "1.0.0.70100-20~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -3122,35 +4563,35 @@ "version": "1:5.41-3ubuntu0.1" } ], - "key": "hip-dev_6.4.43483.60401-83_22.04_amd64", + "key": "hip-dev_7.1.25424.70100-20_22.04_amd64", "name": "hip-dev", - "sha256": "e10323a872b833c7846f7a6d204efa8126b2b6b1727f6da317b687ea4da8fb0e", + "sha256": "1fe1615a52bcd01e0417e3bc78d2e5e6f49a937a5217f72a95eb8e1ee58c0617", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/h/hip-dev/hip-dev_6.4.43483.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1/pool/main/h/hip-dev/hip-dev_7.1.25424.70100-20~22.04_amd64.deb" ], - "version": "6.4.43483.60401-83~22.04" + "version": "7.1.25424.70100-20~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "hipcc_1.1.1.60401-83_22.04_amd64", + "key": "hipcc_1.1.1.70100-20_22.04_amd64", "name": "hipcc", - "sha256": "71aabfdefdd126fe6145b0e21be6964371d550b4be1b955095db6cbb53f9e146", + "sha256": "4eb419ad347dfa97d3f4c0841e06c4c3610f25b43788763980c73e193c07d949", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/h/hipcc/hipcc_1.1.1.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1/pool/main/h/hipcc/hipcc_1.1.1.70100-20~22.04_amd64.deb" ], - "version": "1.1.1.60401-83~22.04" + "version": "1.1.1.70100-20~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocm-llvm_19.0.0.25184.60401-83_22.04_amd64", + "key": "rocm-llvm_20.0.0.25425.70100-20_22.04_amd64", "name": "rocm-llvm", - "sha256": "1dec6406f10c8f2b8fa2baefd1b7a8af844908d7da1493fca407ea696f5b0877", + "sha256": "a32b59322d4a3840987e142a908aa3eeed0104cb12db4db8609dd68a4a92194f", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/r/rocm-llvm/rocm-llvm_19.0.0.25184.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1/pool/main/r/rocm-llvm/rocm-llvm_20.0.0.25425.70100-20~22.04_amd64.deb" ], - "version": "19.0.0.25184.60401-83~22.04" + "version": "20.0.0.25425.70100-20~22.04" }, { "arch": "amd64", @@ -3276,68 +4717,79 @@ { "arch": "amd64", "dependencies": [], - "key": "hsa-rocr-dev_1.15.0.60401-83_22.04_amd64", + "key": "hsa-rocr-dev_1.18.0.70100-20_22.04_amd64", "name": "hsa-rocr-dev", - "sha256": "e1221293f8a5501ef01324c381e61d76a5d2e0f11032e4d33766dc53efc75813", + "sha256": "1c78d88dab92ce162e2dd4e80dbce42da8c8ccb5aa89001875fd58705fd3005b", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/h/hsa-rocr-dev/hsa-rocr-dev_1.15.0.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1/pool/main/h/hsa-rocr-dev/hsa-rocr-dev_1.18.0.70100-20~22.04_amd64.deb" ], - "version": "1.15.0.60401-83~22.04" + "version": "1.18.0.70100-20~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "libdrm-amdgpu-dev_1-2.4.124.60401-2164967.22.04_amd64", - "name": "libdrm-amdgpu-dev", - "sha256": "71c05b464d721e24c8dc2c9b6ca43f8cc9f92714f98c7c8d32022aef6ffb9df7", + "key": "libdrm-dev_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-dev", + "sha256": "9e6d24b88a938a673e79426b47e0c3abbea8a00434926a5a84caa3bc70efa293", "urls": [ - "https://repo.radeon.com/amdgpu/6.4.1/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-dev_2.4.124.60401-2164967.22.04_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libd/libdrm/libdrm-dev_2.4.113-2~ubuntu0.22.04.1_amd64.deb" ], - "version": "1:2.4.124.60401-2164967.22.04" + "version": "2.4.113-2~ubuntu0.22.04.1" }, { "arch": "amd64", "dependencies": [], - "key": "valgrind_1-3.18.1-1ubuntu2_amd64", - "name": "valgrind", - "sha256": "8376c1524d187246eb09b15531c83d83567bdbc609880622a05bc8126558a221", + "key": "libpciaccess-dev_0.16-3_amd64", + "name": "libpciaccess-dev", + "sha256": "44cd5bb81b2536582ca1ff6b5b09d582d0645c73d21f31fc7f879ad38dc843a2", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/v/valgrind/valgrind_3.18.1-1ubuntu2_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libp/libpciaccess/libpciaccess-dev_0.16-3_amd64.deb" ], - "version": "1:3.18.1-1ubuntu2" + "version": "0.16-3" }, { "arch": "amd64", "dependencies": [], - "key": "libc6-dbg_2.35-0ubuntu3.10_amd64", - "name": "libc6-dbg", - "sha256": "a69e6e8bf94a714b3a4686bb6161d0ce6fb1fc863e97c82b40dae83a650f0ade", + "key": "libpciaccess0_0.16-3_amd64", + "name": "libpciaccess0", + "sha256": "0f3c826d6cd56b7b46550d952709b8ede9f2636b98698da40a2054768ebdcffc", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc6-dbg_2.35-0ubuntu3.10_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libp/libpciaccess/libpciaccess0_0.16-3_amd64.deb" ], - "version": "2.35-0ubuntu3.10" + "version": "0.16-3" }, { "arch": "amd64", "dependencies": [], - "key": "libc6-i386_2.35-0ubuntu3.10_amd64", - "name": "libc6-i386", - "sha256": "1a4822354c437e51d47cde7b3db268b3d68ccdc6c2dd340926cdc31c5d8f5e73", + "key": "libdrm-nouveau2_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-nouveau2", + "sha256": "75ce233c7120443a15ab3e71829f7bf63258e6d2a025df366f54027d1d3e8d0c", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc6-i386_2.35-0ubuntu3.10_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libd/libdrm/libdrm-nouveau2_2.4.113-2~ubuntu0.22.04.1_amd64.deb" ], - "version": "2.35-0ubuntu3.10" + "version": "2.4.113-2~ubuntu0.22.04.1" }, { "arch": "amd64", "dependencies": [], - "key": "libdrm-amdgpu-radeon1_1-2.4.124.60401-2164967.22.04_amd64", - "name": "libdrm-amdgpu-radeon1", - "sha256": "1b5a5d927f604020efc5d3ccd857036cd43857cab05e8d357e63d2fd3fc59644", + "key": "libdrm-radeon1_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-radeon1", + "sha256": "579b0d2ccc7800309277fabe517b71a4160b7f26c4745db8e0df25f80576aef7", "urls": [ - "https://repo.radeon.com/amdgpu/6.4.1/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-radeon1_2.4.124.60401-2164967.22.04_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libd/libdrm/libdrm-radeon1_2.4.113-2~ubuntu0.22.04.1_amd64.deb" ], - "version": "1:2.4.124.60401-2164967.22.04" + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libdrm-intel1_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-intel1", + "sha256": "465de8142e166a59d0c0d6857cd23b76d04ab2680b37fcd16cf4d2ce631f8f92", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libd/libdrm/libdrm-intel1_2.4.113-2~ubuntu0.22.04.1_amd64.deb" + ], + "version": "2.4.113-2~ubuntu0.22.04.1" }, { "arch": "amd64", @@ -3376,19 +4828,9 @@ "arch": "amd64", "dependencies": [ { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", + "key": "rocm-core_7.1.0.70100-20_22.04_amd64", "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" - }, - { - "key": "rocblas_4.4.0.60401-83_22.04_amd64", - "name": "rocblas", - "version": "4.4.0.60401-83~22.04" - }, - { - "key": "hip-runtime-amd_6.4.43483.60401-83_22.04_amd64", - "name": "hip-runtime-amd", - "version": "6.4.43483.60401-83~22.04" + "version": "7.1.0.70100-20~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -3411,69 +4853,24 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" }, { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" }, { - "key": "rocprofiler-register_0.4.0.60401-83_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.4.0.60401-83~22.04" + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" }, { - "key": "comgr_3.0.0.60401-83_22.04_amd64", - "name": "comgr", - "version": "3.0.0.60401-83~22.04" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" }, { "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", @@ -3540,6 +4937,16 @@ "name": "libacl1", "version": "2.3.1-1" }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, { "key": "liblzma5_5.2.5-2ubuntu1_amd64", "name": "liblzma5", @@ -3566,65 +4973,10 @@ "version": "1.46.5-2ubuntu1.2" }, { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", "version": "6.3-2ubuntu0.1" }, - { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "rocminfo_1.0.0.60401-83_22.04_amd64", - "name": "rocminfo", - "version": "1.0.0.60401-83~22.04" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, { "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", "name": "libsqlite3-0", @@ -3640,11 +4992,26 @@ "name": "readline-common", "version": "8.1.2-1" }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, { "key": "libmpdec3_2.5.1-2build2_amd64", "name": "libmpdec3", "version": "2.5.1-2build2" }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, { "key": "libffi8_3.4.2-4_amd64", "name": "libffi8", @@ -3685,6 +5052,91 @@ "name": "python3-minimal", "version": "3.10.6-1~22.04.1" }, + { + "key": "rocblas_5.1.0.70100-20_22.04_amd64", + "name": "rocblas", + "version": "5.1.0.70100-20~22.04" + }, + { + "key": "hip-runtime-amd_7.1.25424.70100-20_22.04_amd64", + "name": "hip-runtime-amd", + "version": "7.1.25424.70100-20~22.04" + }, + { + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" + }, + { + "key": "rocprofiler-register_0.6.0.70100-20_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70100-20~22.04" + }, + { + "key": "comgr_3.0.0.70100-20_22.04_amd64", + "name": "comgr", + "version": "3.0.0.70100-20~22.04" + }, + { + "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", + "name": "libtinfo-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", + "name": "libncurses-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", + "name": "ncurses-bin", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libncurses6_6.3-2ubuntu0.1_amd64", + "name": "libncurses6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "rocminfo_1.0.0.70100-20_22.04_amd64", + "name": "rocminfo", + "version": "1.0.0.70100-20~22.04" + }, { "key": "pciutils_1-3.7.0-6_amd64", "name": "pciutils", @@ -3711,14 +5163,9 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.10-2166611.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.10-2166611.22.04" - }, - { - "key": "amdgpu-core_1-6.4.60401-2164967.22.04_amd64", - "name": "amdgpu-core", - "version": "1:6.4.60401-2164967.22.04" + "key": "hsa-rocr_1.18.0.70100-20_22.04_amd64", + "name": "hsa-rocr", + "version": "1.18.0.70100-20~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", @@ -3726,46 +5173,46 @@ "version": "0.186-1ubuntu0.1" }, { - "key": "hipblaslt_0.12.1.60401-83_22.04_amd64", - "name": "hipblaslt", - "version": "0.12.1.60401-83~22.04" + "key": "libdrm-amdgpu1_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-amdgpu1", + "version": "2.4.113-2~ubuntu0.22.04.1" }, { - "key": "roctracer_4.1.60401.60401-83_22.04_amd64", + "key": "libdrm2_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm2", + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "key": "libdrm-common_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-common", + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "key": "hipblaslt_1.1.0.70100-20_22.04_amd64", + "name": "hipblaslt", + "version": "1.1.0.70100-20~22.04" + }, + { + "key": "roctracer_4.1.70100.70100-20_22.04_amd64", "name": "roctracer", - "version": "4.1.60401.60401-83~22.04" + "version": "4.1.70100.70100-20~22.04" } ], - "key": "rocsolver_3.28.0.60401-83_22.04_amd64", + "key": "rocsolver_3.31.0.70100-20_22.04_amd64", "name": "rocsolver", - "sha256": "82451d985c3f9ea9472df45e044651a16cc7a8d510a338e57e0d7c19b04a7243", + "sha256": "d9455cffcc6900633ca3d6986c7444c43bb01c33014a2caeeb409ea1adb90fd4", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/r/rocsolver/rocsolver_3.28.0.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1/pool/main/r/rocsolver/rocsolver_3.31.0.70100-20~22.04_amd64.deb" ], - "version": "3.28.0.60401-83~22.04" + "version": "3.31.0.70100-20~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", + "key": "rocm-core_7.1.0.70100-20_22.04_amd64", "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" - }, - { - "key": "rocsolver_3.28.0.60401-83_22.04_amd64", - "name": "rocsolver", - "version": "3.28.0.60401-83~22.04" - }, - { - "key": "rocblas_4.4.0.60401-83_22.04_amd64", - "name": "rocblas", - "version": "4.4.0.60401-83~22.04" - }, - { - "key": "hip-runtime-amd_6.4.43483.60401-83_22.04_amd64", - "name": "hip-runtime-amd", - "version": "6.4.43483.60401-83~22.04" + "version": "7.1.0.70100-20~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -3788,69 +5235,24 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" }, { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" }, { - "key": "rocprofiler-register_0.4.0.60401-83_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.4.0.60401-83~22.04" + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" }, { - "key": "comgr_3.0.0.60401-83_22.04_amd64", - "name": "comgr", - "version": "3.0.0.60401-83~22.04" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" }, { "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", @@ -3917,6 +5319,16 @@ "name": "libacl1", "version": "2.3.1-1" }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, { "key": "liblzma5_5.2.5-2ubuntu1_amd64", "name": "liblzma5", @@ -3943,65 +5355,10 @@ "version": "1.46.5-2ubuntu1.2" }, { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", "version": "6.3-2ubuntu0.1" }, - { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "rocminfo_1.0.0.60401-83_22.04_amd64", - "name": "rocminfo", - "version": "1.0.0.60401-83~22.04" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, { "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", "name": "libsqlite3-0", @@ -4017,11 +5374,26 @@ "name": "readline-common", "version": "8.1.2-1" }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, { "key": "libmpdec3_2.5.1-2build2_amd64", "name": "libmpdec3", "version": "2.5.1-2build2" }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, { "key": "libffi8_3.4.2-4_amd64", "name": "libffi8", @@ -4062,6 +5434,96 @@ "name": "python3-minimal", "version": "3.10.6-1~22.04.1" }, + { + "key": "rocsolver_3.31.0.70100-20_22.04_amd64", + "name": "rocsolver", + "version": "3.31.0.70100-20~22.04" + }, + { + "key": "rocblas_5.1.0.70100-20_22.04_amd64", + "name": "rocblas", + "version": "5.1.0.70100-20~22.04" + }, + { + "key": "hip-runtime-amd_7.1.25424.70100-20_22.04_amd64", + "name": "hip-runtime-amd", + "version": "7.1.25424.70100-20~22.04" + }, + { + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" + }, + { + "key": "rocprofiler-register_0.6.0.70100-20_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70100-20~22.04" + }, + { + "key": "comgr_3.0.0.70100-20_22.04_amd64", + "name": "comgr", + "version": "3.0.0.70100-20~22.04" + }, + { + "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", + "name": "libtinfo-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", + "name": "libncurses-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", + "name": "ncurses-bin", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libncurses6_6.3-2ubuntu0.1_amd64", + "name": "libncurses6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "rocminfo_1.0.0.70100-20_22.04_amd64", + "name": "rocminfo", + "version": "1.0.0.70100-20~22.04" + }, { "key": "pciutils_1-3.7.0-6_amd64", "name": "pciutils", @@ -4088,14 +5550,9 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.10-2166611.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.10-2166611.22.04" - }, - { - "key": "amdgpu-core_1-6.4.60401-2164967.22.04_amd64", - "name": "amdgpu-core", - "version": "1:6.4.60401-2164967.22.04" + "key": "hsa-rocr_1.18.0.70100-20_22.04_amd64", + "name": "hsa-rocr", + "version": "1.18.0.70100-20~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", @@ -4103,56 +5560,291 @@ "version": "0.186-1ubuntu0.1" }, { - "key": "hipblaslt_0.12.1.60401-83_22.04_amd64", - "name": "hipblaslt", - "version": "0.12.1.60401-83~22.04" + "key": "libdrm-amdgpu1_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-amdgpu1", + "version": "2.4.113-2~ubuntu0.22.04.1" }, { - "key": "roctracer_4.1.60401.60401-83_22.04_amd64", + "key": "libdrm2_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm2", + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "key": "libdrm-common_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-common", + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "key": "hipblaslt_1.1.0.70100-20_22.04_amd64", + "name": "hipblaslt", + "version": "1.1.0.70100-20~22.04" + }, + { + "key": "roctracer_4.1.70100.70100-20_22.04_amd64", "name": "roctracer", - "version": "4.1.60401.60401-83~22.04" + "version": "4.1.70100.70100-20~22.04" } ], - "key": "hipsolver_2.4.0.60401-83_22.04_amd64", + "key": "hipsolver_3.1.0.70100-20_22.04_amd64", "name": "hipsolver", - "sha256": "d461a5a321aad9743dd91906b393615cc2066e7ca792092d4d7619aa14903612", + "sha256": "534728b44d4b3788feb27d175a92643d7c91e219a74309d6f876a920bbf8085c", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/h/hipsolver/hipsolver_2.4.0.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1/pool/main/h/hipsolver/hipsolver_3.1.0.70100-20~22.04_amd64.deb" ], - "version": "2.4.0.60401-83~22.04" + "version": "3.1.0.70100-20~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", + "key": "rocm-core_7.1.0.70100-20_22.04_amd64", "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" + "version": "7.1.0.70100-20~22.04" }, { - "key": "rocfft_1.0.32.60401-83_22.04_amd64", + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "rocfft_1.0.35.70100-20_22.04_amd64", "name": "rocfft", - "version": "1.0.32.60401-83~22.04" + "version": "1.0.35.70100-20~22.04" } ], - "key": "hipfft_1.0.18.60401-83_22.04_amd64", + "key": "hipfft_1.0.21.70100-20_22.04_amd64", "name": "hipfft", - "sha256": "64f2bc139adf53f00affdda5b189d60d491b93a7260c4cbcffc7b26af6569348", + "sha256": "08cc1d13df68bce6c109a2a4cf125926c4a74e35d15845f544515fc6d472fed2", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/h/hipfft/hipfft_1.0.18.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1/pool/main/h/hipfft/hipfft_1.0.21.70100-20~22.04_amd64.deb" ], - "version": "1.0.18.60401-83~22.04" + "version": "1.0.21.70100-20~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocfft_1.0.32.60401-83_22.04_amd64", + "key": "rocfft_1.0.35.70100-20_22.04_amd64", "name": "rocfft", - "sha256": "0c2ec9e4b492adb6d782db21d6438751d93be4c0ac64a9b8d0dd294a32ca7586", + "sha256": "dcf2c2ef0e4cc82302ef96c8fb78e27eeb9a681d6c48978986608f75000ac392", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/r/rocfft/rocfft_1.0.32.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1/pool/main/r/rocfft/rocfft_1.0.35.70100-20~22.04_amd64.deb" ], - "version": "1.0.32.60401-83~22.04" + "version": "1.0.35.70100-20~22.04" } ], "version": 1 diff --git a/runtimes/rocm/packages.yaml b/runtimes/rocm/packages.yaml index fd425f4f..c5038c53 100644 --- a/runtimes/rocm/packages.yaml +++ b/runtimes/rocm/packages.yaml @@ -5,9 +5,9 @@ version: 1 sources: - channel: jammy main - url: https://repo.radeon.com/amdgpu/6.4.1/ubuntu + url: https://repo.radeon.com/amdgpu/7.0.3/ubuntu - channel: jammy main - url: https://repo.radeon.com/rocm/apt/6.4.1 + url: https://repo.radeon.com/rocm/apt/7.1 - channel: jammy main url: https://snapshot.ubuntu.com/ubuntu/20250711T030400Z - channel: jammy-security main @@ -21,12 +21,13 @@ archs: # readelf -d libpjrt_rosm.so | grep NEEDED packages: # - "rocm-smi-lib" - - "libdrm-amdgpu-common" + #- "libdrm-amdgpu-common" # Do not use hsa-rocr from amdgpu repo because it is overriden by hsa-runtime-rocr4wsl-amdgpu - "hsa-rocr (<< 20)" - "hsa-amd-aqlprofile" - "comgr" - "rocprofiler-register" + - "rocprofiler-sdk" - "miopen-hip" - "rccl" - "rocm-device-libs" diff --git a/runtimes/rocm/rocm.bzl b/runtimes/rocm/rocm.bzl index b38f69be..dc724b41 100644 --- a/runtimes/rocm/rocm.bzl +++ b/runtimes/rocm/rocm.bzl @@ -6,10 +6,10 @@ _BUILD_FILE_DEFAULT_VISIBILITY = """\ package(default_visibility = ["//visibility:public"]) """ -_ROCM_STRIP_PREFIX = "opt/rocm-6.4.1" +_ROCM_STRIP_PREFIX = "opt/rocm-7.1.0" _UBUNTU_PACKAGES = { - "libdrm2-amdgpu": packages.filegroup(name = "libdrm2-amdgpu", srcs = ["opt/amdgpu/lib/x86_64-linux-gnu/libdrm.so.2"]), + #"libdrm2-amdgpu": packages.filegroup(name = "libdrm2-amdgpu", srcs = ["opt/amdgpu/lib/x86_64-linux-gnu/libdrm.so.2"]), "libelf1": "\n".join([ packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), packages.patchelf( @@ -18,24 +18,24 @@ _UBUNTU_PACKAGES = { set_rpath = "$ORIGIN", ), ]), - "libdrm-amdgpu-common": packages.filegroup(name = "amdgpu_ids", srcs = ["opt/amdgpu/share/libdrm/amdgpu.ids"]), + #"libdrm-amdgpu-common": packages.filegroup(name = "amdgpu_ids", srcs = ["opt/amdgpu/share/libdrm/amdgpu.ids"]), "libnuma1": packages.filegroup(name = "libnuma1", srcs = ["usr/lib/x86_64-linux-gnu/libnuma.so.1"]), "libzstd1": packages.filegroup(name = "libzstd1", srcs = ["usr/lib/x86_64-linux-gnu/libzstd.so.1"]), - "libdrm-amdgpu-amdgpu1": "\n".join([ - packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), - packages.patchelf( - name = "libdrm-amdgpu-amdgpu1", - shared_library = "opt/amdgpu/lib/x86_64-linux-gnu/libdrm_amdgpu.so.1", - set_rpath = "$ORIGIN", - ), - ]), + #"libdrm-amdgpu-amdgpu1": "\n".join([ + # packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), + # packages.patchelf( + # name = "libdrm-amdgpu-amdgpu1", + # shared_library = "opt/amdgpu/lib/x86_64-linux-gnu/libdrm_amdgpu.so.1", + # set_rpath = "$ORIGIN", + # ), + #]), "libtinfo6": packages.filegroup(name = "libtinfo6", srcs = ["lib/x86_64-linux-gnu/libtinfo.so.6"]), "zlib1g": packages.filegroup(name = "zlib1g", srcs = ["lib/x86_64-linux-gnu/libz.so.1"]), } _ROCM_PACKAGES = { "rocm-core": packages.filegroup(name = "rocm-core", srcs = ["lib/librocm-core.so.1"]), - "rocm-smi-lib": packages.filegroup(name = "rocm_smi", srcs = ["lib/librocm_smi64.so.7"]), + "rocm-smi-lib": packages.filegroup(name = "rocm_smi", srcs = ["lib/librocm_smi64.so.1"]), "hsa-rocr": packages.filegroup(name = "hsa-runtime", srcs = ["lib/libhsa-runtime64.so.1"]), "hsa-amd-aqlprofile": packages.filegroup(name = "hsa-amd-aqlprofile", srcs = ["lib/libhsa-amd-aqlprofile64.so.1"]), "comgr": "\n".join([ @@ -59,7 +59,7 @@ _ROCM_PACKAGES = { packages.load_("@zml//runtimes/rocm:gfx.bzl", "bytecode_select"), packages.patchelf( name = "rocblas", - shared_library = "lib/librocblas.so.4", + shared_library = "lib/librocblas.so.5", add_needed = ["libzmlxrocm.so.0"], rename_dynamic_symbols = { "dlopen": "zmlxrocm_dlopen", @@ -90,7 +90,7 @@ _ROCM_PACKAGES = { packages.load_("@zml//runtimes/rocm:gfx.bzl", "bytecode_select"), packages.patchelf( name = "hipblaslt", - shared_library = "lib/libhipblaslt.so.0", + shared_library = "lib/libhipblaslt.so.1", add_needed = ["libzmlxrocm.so.0"], rename_dynamic_symbols = { "dlopen": "zmlxrocm_dlopen", @@ -116,7 +116,7 @@ _ROCM_PACKAGES = { "hipfft": packages.filegroup(name = "hipfft", srcs = ["lib/libhipfft.so.0"]), "hip-runtime-amd": "\n".join([ packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), - packages.filegroup(name = "amdhip", srcs = ["lib/libamdhip64.so.6"]), + packages.filegroup(name = "amdhip", srcs = ["lib/libamdhip64.so.7"]), packages.patchelf( name = "amdhip_patched", shared_library = ":amdhip", @@ -125,9 +125,10 @@ _ROCM_PACKAGES = { "dlopen": "zmlxrocm_dlopen", }, ), - packages.filegroup(name = "hiprtc", srcs = ["lib/libhiprtc.so.6"]), + packages.filegroup(name = "hiprtc", srcs = ["lib/libhiprtc.so.7"]), ]), - "hipsolver": packages.filegroup(name = "hipsolver", srcs = ["lib/libhipsolver.so.0"]), + "hipsolver": packages.filegroup(name = "hipsolver", srcs = ["lib/libhipsolver.so.1"]), + "rocprofiler-sdk": packages.filegroup(name = "rocprofiler-sdk", srcs = ["lib/librocprofiler-sdk-attach.so.1"]), } def _rocm_impl(mctx): From 3fe06706e92998dc16f55321292e40f70def0c1f Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 1 Dec 2025 19:16:46 +0100 Subject: [PATCH 062/881] WORKING !!! --- MODULE.bazel | 22 +++- MODULE.bazel.lock | 134 ++++++++++--------- bazel/zig_index.json | 154 +++++++++++----------- third_party/arocc/BUILD.bazel | 0 third_party/arocc/arocc.bazel | 111 ++++++++++++++++ third_party/arocc/repo.bzl | 9 ++ third_party/non_module_deps.bzl | 4 + third_party/translate-c/BUILD.bazel | 0 third_party/translate-c/repo.bzl | 9 ++ third_party/translate-c/translate-c.bazel | 34 +++++ translate-c/BUILD.bazel | 17 +++ 11 files changed, 351 insertions(+), 143 deletions(-) create mode 100644 third_party/arocc/BUILD.bazel create mode 100644 third_party/arocc/arocc.bazel create mode 100644 third_party/arocc/repo.bzl create mode 100644 third_party/translate-c/BUILD.bazel create mode 100644 third_party/translate-c/repo.bzl create mode 100644 third_party/translate-c/translate-c.bazel create mode 100644 translate-c/BUILD.bazel diff --git a/MODULE.bazel b/MODULE.bazel index 30f3bcd4..c694df5e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -23,14 +23,14 @@ bazel_dep(name = "rules_proto", version = "7.1.0") bazel_dep(name = "rules_python", version = "1.5.3") bazel_dep(name = "rules_rust", version = "0.63.0") bazel_dep(name = "rules_zig", version = "0.12.2") -bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.2.4") +bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.3.1") bazel_dep(name = "with_cfg.bzl", version = "0.11.0") bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2", dev_dependency = True) zig = use_extension("@rules_zig//zig:extensions.bzl", "zig") zig.index(file = "//bazel:zig_index.json") -zig.toolchain(zig_version = "0.16.0-dev.1458+755a3d957") +zig.toolchain(zig_version = "0.16.0-dev.1484+d0ba6642b") zig.mirrors(urls = [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/", @@ -103,7 +103,7 @@ use_repo(zls, "zls_toolchains") register_toolchains("@zls_toolchains//:all") non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") -use_repo(non_module_deps, "com_github_hejsil_clap", "com_google_sentencepiece", "mnist", "org_swig_swig", "xla") +use_repo(non_module_deps, "arocc", "com_github_hejsil_clap", "com_google_sentencepiece", "mnist", "org_swig_swig", "translate-c", "xla") xla = use_extension("//third_party/xla:xla.bzl", "xla") use_repo( @@ -216,12 +216,20 @@ huggingface.model( use_repo(huggingface, "ModernBERT-base", "ModernBERT-large") ################################################################################ -local_path_override( +git_override( module_name = "rules_zig", - path = "../rules_zig", + commit = "2c91b54305772d7b43445168e7cf0037b5cbc9e2", + remote = "https://github.com/zml/rules_zig.git", ) -local_path_override( +git_override( module_name = "toolchains_llvm_bootstrapped", - path = "../toolchains_llvm_bootstrapped", + commit = "4902b6cadc4ca840645c97d52f985c6f67967b6f", + remote = "https://github.com/cerisier/toolchains_llvm_bootstrapped.git", +) + +git_override( + module_name = "apple_support", + commit = "6ddcf9c1c46be1a7eb8767c7dbec7714dd47efb8", + remote = "https://github.com/cerisier/apple_support.git", ) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index dfec8011..dc44f6dc 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -12,11 +12,6 @@ "https://bazel-registry.zml.ai/modules/abseil-cpp/20250127.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20250512.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20250814.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/apple_support/1.11.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/apple_support/1.15.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/apple_support/1.17.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/apple_support/1.24.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/apple_support/1.24.5/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.0.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.16.0/MODULE.bazel": "not found", @@ -118,9 +113,7 @@ "https://bazel-registry.zml.ai/modules/rules_cc/0.2.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.11/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.12/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.2.13/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.14/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.2.8/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_distroless/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_fuzzing/0.5.2/MODULE.bazel": "not found", @@ -212,12 +205,6 @@ "https://bcr.bazel.build/modules/abseil-cpp/20250512.1/MODULE.bazel": "d209fdb6f36ffaf61c509fcc81b19e81b411a999a934a032e10cd009a0226215", "https://bcr.bazel.build/modules/abseil-cpp/20250814.0/MODULE.bazel": "c43c16ca2c432566cdb78913964497259903ebe8fb7d9b57b38e9f1425b427b8", "https://bcr.bazel.build/modules/abseil-cpp/20250814.0/source.json": "b88bff599ceaf0f56c264c749b1606f8485cec3b8c38ba30f88a4df9af142861", - "https://bcr.bazel.build/modules/apple_support/1.11.1/MODULE.bazel": "1843d7cd8a58369a444fc6000e7304425fba600ff641592161d9f15b179fb896", - "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", - "https://bcr.bazel.build/modules/apple_support/1.17.1/MODULE.bazel": "655c922ab1209978a94ef6ca7d9d43e940cd97d9c172fb55f94d91ac53f8610b", - "https://bcr.bazel.build/modules/apple_support/1.24.2/MODULE.bazel": "0e62471818affb9f0b26f128831d5c40b074d32e6dda5a0d3852847215a41ca4", - "https://bcr.bazel.build/modules/apple_support/1.24.5/MODULE.bazel": "692784f32ee3175cc1ced78f4262a0d4b7110c89793cd0b63f950af2d9cc6bdc", - "https://bcr.bazel.build/modules/apple_support/1.24.5/source.json": "e81e4f6557a879aaaa104e84255710a02862b9380b12bd72e4a58bff32c70116", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.0.0/MODULE.bazel": "e118477db5c49419a88d78ebc7a2c2cea9d49600fe0f490c1903324a2c16ecd9", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.16.0/MODULE.bazel": "852f9ebbda017572a7c113a2434592dd3b2f55cd9a0faea3d4be5a09a59e4900", @@ -342,10 +329,8 @@ "https://bcr.bazel.build/modules/rules_cc/0.2.0/MODULE.bazel": "b5c17f90458caae90d2ccd114c81970062946f49f355610ed89bebf954f5783c", "https://bcr.bazel.build/modules/rules_cc/0.2.11/MODULE.bazel": "e94f24f065bf2191dba2dace951814378b66a94bb3bcc48077492fe0508059b5", "https://bcr.bazel.build/modules/rules_cc/0.2.12/MODULE.bazel": "4216c383ce3223c7dfbd3afcb09c7056265156150b2a3bd3fd9d02949a7ee1bc", - "https://bcr.bazel.build/modules/rules_cc/0.2.13/MODULE.bazel": "eecdd666eda6be16a8d9dc15e44b5c75133405e820f620a234acc4b1fdc5aa37", "https://bcr.bazel.build/modules/rules_cc/0.2.14/MODULE.bazel": "353c99ed148887ee89c54a17d4100ae7e7e436593d104b668476019023b58df8", "https://bcr.bazel.build/modules/rules_cc/0.2.14/source.json": "55d0a4587c5592fad350f6e698530f4faf0e7dd15e69d43f8d87e220c78bea54", - "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", "https://bcr.bazel.build/modules/rules_distroless/0.5.1/MODULE.bazel": "2a63f4744d30749128105da5f96adf7caf5628e37548293f89e7fa39c3b3f2c2", "https://bcr.bazel.build/modules/rules_distroless/0.5.1/source.json": "c6b9ff7f325bfed89c3671757f14c1d1bc6077d0fcef809b8aa2d007cac7dd1d", "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", @@ -445,6 +430,37 @@ }, "selectedYankedVersions": {}, "moduleExtensions": { + "@@apple_support+//crosstool:setup.bzl%apple_cc_configure_extension": { + "general": { + "bzlTransitiveDigest": "6HFMfNkxiYUs5AKI+9d2/9J5tWKUXMh2sUT0MsHY6jI=", + "usagesDigest": "si6rju4O264yIf33EKkDlbbpsFHy/ADOlY9r6T0faq8=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "local_config_apple_cc_toolchains": { + "repoRuleId": "@@apple_support+//crosstool:setup.bzl%_apple_cc_autoconf_toolchains", + "attributes": {} + }, + "local_config_apple_cc": { + "repoRuleId": "@@apple_support+//crosstool:setup.bzl%_apple_cc_autoconf", + "attributes": {} + } + }, + "recordedRepoMappingEntries": [ + [ + "apple_support+", + "bazel_tools", + "bazel_tools" + ], + [ + "bazel_tools", + "rules_cc", + "rules_cc+" + ] + ] + } + }, "@@aspect_rules_py+//py:extensions.bzl%py_tools": { "general": { "bzlTransitiveDigest": "Udhh0icjIe0dHLjbgfmj2tzBz0geB4SXJl0FoPcpF84=", @@ -2828,89 +2844,89 @@ "@@rules_zig+//zig:extensions.bzl%zig": { "general": { "bzlTransitiveDigest": "wV3euhBZSI0D2ZlqbY1YW592mghFY88d9H6zBMQRaJY=", - "usagesDigest": "kssHId7CLEGeadFinWbOWrcYqSqEA5uUw/GxX5yTP70=", + "usagesDigest": "eS/vW6tmN9XXr9OKW9PDWeVMh8wVIBSfvoOHCD420XU=", "recordedFileInputs": { - "@@//bazel/zig_index.json": "4c9ee005881097e41652cd6e37a9ca921abf2e126188c67b74d7a153c7052536", + "@@//bazel/zig_index.json": "83fdd8898d3a527681082cd98c02b91570aa439ae2209807169ad4e41326d75e", "@@rules_zig+//zig/private/versions.json": "6eb85ebaee72c4e6fbc82d8ad73f69e0899d839b70a7e19caa792ab6a5fa0c36" }, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { - "zig_0.16.0-dev.1458_P755a3d957_aarch64-linux": { + "zig_0.16.0-dev.1484_Pd0ba6642b_aarch64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1458+755a3d957.tar.xz", + "url": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1484+d0ba6642b.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "63f22a3ee1fc7a8214ae21264b9773f4553ea11b94104233d4c4946984d6c3be", - "zig_version": "0.16.0-dev.1458+755a3d957", + "sha256": "f8240e9708376f3e808c59dbaf69a2839eb1a14507432ee9f7d6817482346bf9", + "zig_version": "0.16.0-dev.1484+d0ba6642b", "platform": "aarch64-linux" } }, - "zig_0.16.0-dev.1458_P755a3d957_aarch64-macos": { + "zig_0.16.0-dev.1484_Pd0ba6642b_aarch64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1458+755a3d957.tar.xz", + "url": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1484+d0ba6642b.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "3dc8d405cea1ff394cd6720b8445ffea20c07bc1d9e4f7c68add9b4fd7ece554", - "zig_version": "0.16.0-dev.1458+755a3d957", + "sha256": "b521ed99eecad23575de1c138d9efc2fa4e3c3b9afd454af66c91707e3c89d32", + "zig_version": "0.16.0-dev.1484+d0ba6642b", "platform": "aarch64-macos" } }, - "zig_0.16.0-dev.1458_P755a3d957_aarch64-windows": { + "zig_0.16.0-dev.1484_Pd0ba6642b_aarch64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1458+755a3d957.zip", + "url": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1484+d0ba6642b.zip", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "e6a55c7ef706ecc8fe757a55d333620267b09339a1c7f3e08eb8e7e75bbb9ad9", - "zig_version": "0.16.0-dev.1458+755a3d957", + "sha256": "62beade8036e74f10009c133ae81454df690506c447c215baf891d828f38761e", + "zig_version": "0.16.0-dev.1484+d0ba6642b", "platform": "aarch64-windows" } }, - "zig_0.16.0-dev.1458_P755a3d957_x86_64-linux": { + "zig_0.16.0-dev.1484_Pd0ba6642b_x86_64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1458+755a3d957.tar.xz", + "url": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1484+d0ba6642b.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "d1f819092c784fd821428501dac06387966179623ceea498be55a40e2fa8c8ef", - "zig_version": "0.16.0-dev.1458+755a3d957", + "sha256": "3f90582cde6e4ea4a9b9b1dd283a6720131dfefbece1c85be2f4c82ab74a2e7b", + "zig_version": "0.16.0-dev.1484+d0ba6642b", "platform": "x86_64-linux" } }, - "zig_0.16.0-dev.1458_P755a3d957_x86_64-macos": { + "zig_0.16.0-dev.1484_Pd0ba6642b_x86_64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1458+755a3d957.tar.xz", + "url": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1484+d0ba6642b.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "bebb2f5361cd65171ea78fc0fa7c180d5eaeec89da8a5d1985f3d7b97634d389", - "zig_version": "0.16.0-dev.1458+755a3d957", + "sha256": "7055d0af0dd99af4e845a016e25fdd3be26c0e2c331b98902d2510c737e8b50e", + "zig_version": "0.16.0-dev.1484+d0ba6642b", "platform": "x86_64-macos" } }, - "zig_0.16.0-dev.1458_P755a3d957_x86_64-windows": { + "zig_0.16.0-dev.1484_Pd0ba6642b_x86_64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1458+755a3d957.zip", + "url": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1484+d0ba6642b.zip", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "bd1b710cf4c928407453f4da0bbc253c4db147223cc5df101b1f2fb5de01f624", - "zig_version": "0.16.0-dev.1458+755a3d957", + "sha256": "14cd84fa0977c6893257c10a89a87b730815f3a71426e9494feea2accb768144", + "zig_version": "0.16.0-dev.1484+d0ba6642b", "platform": "x86_64-windows" } }, @@ -2918,28 +2934,28 @@ "repoRuleId": "@@rules_zig+//zig/private/repo:toolchains_repo.bzl%toolchains_repo", "attributes": { "names": [ - "zig_0.16.0-dev.1458_P755a3d957_aarch64-linux", - "zig_0.16.0-dev.1458_P755a3d957_aarch64-macos", - "zig_0.16.0-dev.1458_P755a3d957_aarch64-windows", - "zig_0.16.0-dev.1458_P755a3d957_x86_64-linux", - "zig_0.16.0-dev.1458_P755a3d957_x86_64-macos", - "zig_0.16.0-dev.1458_P755a3d957_x86_64-windows" + "zig_0.16.0-dev.1484_Pd0ba6642b_aarch64-linux", + "zig_0.16.0-dev.1484_Pd0ba6642b_aarch64-macos", + "zig_0.16.0-dev.1484_Pd0ba6642b_aarch64-windows", + "zig_0.16.0-dev.1484_Pd0ba6642b_x86_64-linux", + "zig_0.16.0-dev.1484_Pd0ba6642b_x86_64-macos", + "zig_0.16.0-dev.1484_Pd0ba6642b_x86_64-windows" ], "labels": [ - "@zig_0.16.0-dev.1458_P755a3d957_aarch64-linux//:zig_toolchain", - "@zig_0.16.0-dev.1458_P755a3d957_aarch64-macos//:zig_toolchain", - "@zig_0.16.0-dev.1458_P755a3d957_aarch64-windows//:zig_toolchain", - "@zig_0.16.0-dev.1458_P755a3d957_x86_64-linux//:zig_toolchain", - "@zig_0.16.0-dev.1458_P755a3d957_x86_64-macos//:zig_toolchain", - "@zig_0.16.0-dev.1458_P755a3d957_x86_64-windows//:zig_toolchain" + "@zig_0.16.0-dev.1484_Pd0ba6642b_aarch64-linux//:zig_toolchain", + "@zig_0.16.0-dev.1484_Pd0ba6642b_aarch64-macos//:zig_toolchain", + "@zig_0.16.0-dev.1484_Pd0ba6642b_aarch64-windows//:zig_toolchain", + "@zig_0.16.0-dev.1484_Pd0ba6642b_x86_64-linux//:zig_toolchain", + "@zig_0.16.0-dev.1484_Pd0ba6642b_x86_64-macos//:zig_toolchain", + "@zig_0.16.0-dev.1484_Pd0ba6642b_x86_64-windows//:zig_toolchain" ], "zig_versions": [ - "0.16.0-dev.1458+755a3d957", - "0.16.0-dev.1458+755a3d957", - "0.16.0-dev.1458+755a3d957", - "0.16.0-dev.1458+755a3d957", - "0.16.0-dev.1458+755a3d957", - "0.16.0-dev.1458+755a3d957" + "0.16.0-dev.1484+d0ba6642b", + "0.16.0-dev.1484+d0ba6642b", + "0.16.0-dev.1484+d0ba6642b", + "0.16.0-dev.1484+d0ba6642b", + "0.16.0-dev.1484+d0ba6642b", + "0.16.0-dev.1484+d0ba6642b" ], "exec_lengths": [ 2, diff --git a/bazel/zig_index.json b/bazel/zig_index.json index ec051715..43dbe8a6 100644 --- a/bazel/zig_index.json +++ b/bazel/zig_index.json @@ -1,133 +1,133 @@ { "master": { - "version": "0.16.0-dev.1458+755a3d957", - "date": "2025-11-24", + "version": "0.16.0-dev.1484+d0ba6642b", + "date": "2025-11-26", "docs": "https://ziglang.org/documentation/master/", "stdDocs": "https://ziglang.org/documentation/master/std/", "src": { - "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.1458+755a3d957.tar.xz", - "shasum": "5cd89a3f8dfdf34d009d5cf9547ec319ac1898779b42fa4437c7ab4dd36c13c9", - "size": "21698384" + "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.1484+d0ba6642b.tar.xz", + "shasum": "a7ee7eae112fda7d3ddaab0fb110eee34692e0a1e12fd016e121a1f954c0a50c", + "size": "21703792" }, "bootstrap": { - "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.1458+755a3d957.tar.xz", - "shasum": "0a2f9586d101b904d12777abf765337d466a973a6ac98983c53f0a4fbf4920ff", - "size": "54461840" + "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.1484+d0ba6642b.tar.xz", + "shasum": "db7d3943bfb389fca4e77f8f55b1e17710b3453fe05fd3bfdc97fa5cf955b7d5", + "size": "54466044" }, "x86_64-macos": { - "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1458+755a3d957.tar.xz", - "shasum": "bebb2f5361cd65171ea78fc0fa7c180d5eaeec89da8a5d1985f3d7b97634d389", - "size": "56838428" + "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1484+d0ba6642b.tar.xz", + "shasum": "7055d0af0dd99af4e845a016e25fdd3be26c0e2c331b98902d2510c737e8b50e", + "size": "56848400" }, "aarch64-macos": { - "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1458+755a3d957.tar.xz", - "shasum": "3dc8d405cea1ff394cd6720b8445ffea20c07bc1d9e4f7c68add9b4fd7ece554", - "size": "51687600" + "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1484+d0ba6642b.tar.xz", + "shasum": "b521ed99eecad23575de1c138d9efc2fa4e3c3b9afd454af66c91707e3c89d32", + "size": "51712376" }, "x86_64-linux": { - "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1458+755a3d957.tar.xz", - "shasum": "d1f819092c784fd821428501dac06387966179623ceea498be55a40e2fa8c8ef", - "size": "54753544" + "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1484+d0ba6642b.tar.xz", + "shasum": "3f90582cde6e4ea4a9b9b1dd283a6720131dfefbece1c85be2f4c82ab74a2e7b", + "size": "54737932" }, "aarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1458+755a3d957.tar.xz", - "shasum": "63f22a3ee1fc7a8214ae21264b9773f4553ea11b94104233d4c4946984d6c3be", - "size": "50495484" + "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1484+d0ba6642b.tar.xz", + "shasum": "f8240e9708376f3e808c59dbaf69a2839eb1a14507432ee9f7d6817482346bf9", + "size": "50502404" }, "arm-linux": { - "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.1458+755a3d957.tar.xz", - "shasum": "f3865ba1c3a19127f9a25962c919414c923c2c0bcddf060dd464f7fb5aefb044", - "size": "51360952" + "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.1484+d0ba6642b.tar.xz", + "shasum": "5bf3b779b33ad4479e4380700393551df22804c6ca44e9126cbde0719b84a788", + "size": "51392924" }, "riscv64-linux": { - "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.1458+755a3d957.tar.xz", - "shasum": "8d83cf1bb3f81510c79a8e7859c79b113ae9d63cac610057c9483f1b89d716f5", - "size": "54551500" + "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.1484+d0ba6642b.tar.xz", + "shasum": "3dfc535a064d60443a61c0fc672860287e006db5667da3b345e614a4ca75a577", + "size": "54552624" }, "powerpc64le-linux": { - "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.1458+755a3d957.tar.xz", - "shasum": "92d3dba05203249fb9199abd0d937f16053253a8aa2eab5d76c3edeb84b46e02", - "size": "54579384" + "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.1484+d0ba6642b.tar.xz", + "shasum": "e8fa27e9bfaffe74ec85dc07133658587d3ab6f84947640acb92b4b24c06b348", + "size": "54592728" }, "x86-linux": { - "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.1458+755a3d957.tar.xz", - "shasum": "976b94b28079f9c083436fe2728d92f0fbf9353392a2b80945cf10c6b6faf42d", - "size": "57416272" + "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.1484+d0ba6642b.tar.xz", + "shasum": "ca98b5a992fbf4e4f66b8ce6ad37e1280e4cdc0e4d5aa4556b4dd84c2b9a4161", + "size": "57448984" }, "loongarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.1458+755a3d957.tar.xz", - "shasum": "fd67f4c7930322862103fc930d328e0dd0626389611bd715377374c38845e5c0", - "size": "51808832" + "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.1484+d0ba6642b.tar.xz", + "shasum": "d6a3829bd4c18603261e9b4a9e7ece7eb44ac3c9e784e0edacd8527a70e2f245", + "size": "51807684" }, "s390x-linux": { - "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.1458+755a3d957.tar.xz", - "shasum": "6e79f44ac4469163b572d1c26671aa8cb5c93cc544f5fc01cbdfde97cb320421", - "size": "54284392" + "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.1484+d0ba6642b.tar.xz", + "shasum": "bb1e5bba6d3b1e4d95bb9bf51042e3887b0aa1e55f39800eafbc49d1eece67bf", + "size": "54272976" }, "x86_64-windows": { - "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1458+755a3d957.zip", - "shasum": "bd1b710cf4c928407453f4da0bbc253c4db147223cc5df101b1f2fb5de01f624", - "size": "94708257" + "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1484+d0ba6642b.zip", + "shasum": "14cd84fa0977c6893257c10a89a87b730815f3a71426e9494feea2accb768144", + "size": "94726172" }, "aarch64-windows": { - "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1458+755a3d957.zip", - "shasum": "e6a55c7ef706ecc8fe757a55d333620267b09339a1c7f3e08eb8e7e75bbb9ad9", - "size": "90682974" + "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1484+d0ba6642b.zip", + "shasum": "62beade8036e74f10009c133ae81454df690506c447c215baf891d828f38761e", + "size": "90710289" }, "x86-windows": { - "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.1458+755a3d957.zip", - "shasum": "b53ab697230168494d626049d04858fb6834cc23737f879d2026cbf39cc214b2", - "size": "96547358" + "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.1484+d0ba6642b.zip", + "shasum": "f5331ec9511d5b85f51635aa9487b9ebb93acdde8b973c92019b1226eda9b853", + "size": "96574941" }, "aarch64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.1458+755a3d957.tar.xz", - "shasum": "ce2a0974e6ffabff59592067f4d12dcc0c54a2074c06f7b6478e0f13f2042e00", - "size": "50450564" + "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.1484+d0ba6642b.tar.xz", + "shasum": "5291d755fe5b5a43449c3d763298e33b8c370eb99572f298a1fc1296e72b3ddb", + "size": "50454224" }, "arm-freebsd": { - "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.1458+755a3d957.tar.xz", - "shasum": "ee9c87d8dcb336c807833488267c79ea27689ca86684a2cc1ce9fce2dcc3d373", - "size": "52049996" + "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.1484+d0ba6642b.tar.xz", + "shasum": "3949292db30948364bf7e5c2a012b24ae8449096218c8f3ed558b61391211fdc", + "size": "52058888" }, "powerpc64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-powerpc64-freebsd-0.16.0-dev.1458+755a3d957.tar.xz", - "shasum": "93956cdcd452994be83a1418ce9901f7a2b5f92768103663b2c2d2e39250260b", - "size": "53097748" + "tarball": "https://ziglang.org/builds/zig-powerpc64-freebsd-0.16.0-dev.1484+d0ba6642b.tar.xz", + "shasum": "1916ec21059513d70764017753f22df316b1375c6e1258aa3464aad443e9cda1", + "size": "53141628" }, "powerpc64le-freebsd": { - "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.1458+755a3d957.tar.xz", - "shasum": "95f3b2b50a87dcf62c83c5312a5296b223fffff614f63b463f9a8b9e8d04d117", - "size": "54573692" + "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.1484+d0ba6642b.tar.xz", + "shasum": "d3381907956625bc0585f1fdc66a14b9b418cca6fd7b79ff01e8d5af11f25694", + "size": "54590608" }, "riscv64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.1458+755a3d957.tar.xz", - "shasum": "6eae893b95b2ebe6696a1d87dafd3737a30692bde1c2878dbf3180401e15c195", - "size": "54698508" + "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.1484+d0ba6642b.tar.xz", + "shasum": "96d66952a130af6a958c25aa8f131333c2e8fa0fed6582004ce4b45d9d97708b", + "size": "54719008" }, "x86_64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.1458+755a3d957.tar.xz", - "shasum": "3637c94591bcabfd403650a75e77bf799fd8a3adbaa00e1e678aa2cb1eff2d81", - "size": "54845396" + "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.1484+d0ba6642b.tar.xz", + "shasum": "fa5bddd439bacb8401553299d8e66aa3008804312aed0702f5a317e96cc713c0", + "size": "54855936" }, "aarch64-netbsd": { - "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.1458+755a3d957.tar.xz", - "shasum": "9d69e952a4a8de168b5b8055979cfd2e54efbf71427c8f6f70deacf82dfdc4d1", - "size": "50389780" + "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.1484+d0ba6642b.tar.xz", + "shasum": "64fec627697895f4ae979b9f05586d27b8114f18ec6c067139798f9928077e1f", + "size": "50418132" }, "arm-netbsd": { - "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.1458+755a3d957.tar.xz", - "shasum": "0f5b6bbc1749aa0491cc865c49bd6c714db2d7749f8732c8d2107409cbbb83c7", - "size": "53090212" + "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.1484+d0ba6642b.tar.xz", + "shasum": "7e9d09a4971fb0a4b257527f0414b0adb04f83e0ffd6ebb067b3555095ab193b", + "size": "53085432" }, "x86-netbsd": { - "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.1458+755a3d957.tar.xz", - "shasum": "b51b0d9594743dcf24b15009d815f87ceda27833adcac02dd96c679daedfba52", - "size": "58031544" + "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.1484+d0ba6642b.tar.xz", + "shasum": "6fed94706492135b5a9db2421df435bdd3ca502d2a2eb1a4892d443340a05cac", + "size": "58054536" }, "x86_64-netbsd": { - "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.1458+755a3d957.tar.xz", - "shasum": "f27f5b71ed7e35a4c22f279d1a2823db3d72b9d821d510465be1f0666427de64", - "size": "54795056" + "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.1484+d0ba6642b.tar.xz", + "shasum": "3e375f4aa4c7e31391ec5bbef5e8a5eaf14750faf5510c09074b76d61525ef4a", + "size": "54805628" } }, "0.15.2": { diff --git a/third_party/arocc/BUILD.bazel b/third_party/arocc/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/third_party/arocc/arocc.bazel b/third_party/arocc/arocc.bazel new file mode 100644 index 00000000..3e3c62bd --- /dev/null +++ b/third_party/arocc/arocc.bazel @@ -0,0 +1,111 @@ +load("@bazel_skylib//rules:write_file.bzl", "write_file") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") + +write_file( + name = "system_defaults_zig", + out = "system_defaults.zig", + content = ["""\ +pub const enable_linker_build_id: bool = false; +pub const linker: []const u8 = ""; +pub const sysroot: []const u8 = ""; +pub const gcc_install_prefix: []const u8 = ""; +pub const rtlib: []const u8 = ""; +pub const unwindlib: []const u8 = ""; +"""], +) + +zig_library( + name = "system_defaults", + main = ":system_defaults_zig", +) + +write_file( + name = "aro_options_zig", + out = "aro_options.zig", + content = ["""\ +pub const enable_tracy: bool = false; +pub const enable_tracy_callstack: bool = false; +pub const enable_tracy_allocation: bool = false; +pub const debug_allocations: bool = false; +pub const version_str: []const u8 = ""; +"""], +) + +zig_library( + name = "build_options", + main = ":aro_options_zig", +) + +zig_library( + name = "zig", + srcs = glob(["deps/zig/**/*.zig"]), + main = "deps/zig/lib.zig", +) + +zig_library( + name = "backend", + main = "src/backend.zig", + deps = [ + ":build_options", + ":zig", + ], +) + +zig_library( + name = "aro", + srcs = glob(["src/**/*.zig"]), + main = "src/aro.zig", + visibility = ["//visibility:public"], + deps = [ + ":backend", + ":build_options", + ":system_defaults", + ] + [ + ":{}".format(name.replace("src/aro/", "")) + for name in glob(["src/aro/**/*.def"]) + ], +) + +zig_library( + name = "assembly_backend", + main = "src/assembly_backend.zig", + deps = [ + ":aro", + ], +) + +zig_binary( + name = "arocc", + srcs = glob(["src/**/*.zig"]), + main = "src/main.zig", + deps = [ + ":aro", + ":assembly_backend", + ":build_options", + ], + zigopts = ["-fm"], +) + +zig_binary( + name = "generate-def", + main = "build/generate_def.zig", +) + +[ + genrule( + name = "def_{}".format(name), + srcs = [name], + outs = ["{}.zig".format(name)], + cmd = """$(location :generate-def) $< $@""", + tools = [":generate-def"], + ) + for name in glob(["src/aro/**/*.def"]) +] + +[ + zig_library( + name = "{}".format(name.replace("src/aro/", "")), + main = ":def_{}".format(name), + ) + for name in glob(["src/aro/**/*.def"]) +] diff --git a/third_party/arocc/repo.bzl b/third_party/arocc/repo.bzl new file mode 100644 index 00000000..818ad68e --- /dev/null +++ b/third_party/arocc/repo.bzl @@ -0,0 +1,9 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") + +def repo(): + new_git_repository( + name = "arocc", + remote = "https://github.com/Vexu/arocc.git", + commit = "3c7b4545b2ac30b9da41dbc8eacd2f14af44bc6c", + build_file = "//third_party/arocc:arocc.bazel", + ) diff --git a/third_party/non_module_deps.bzl b/third_party/non_module_deps.bzl index 5c769468..02eee8fc 100644 --- a/third_party/non_module_deps.bzl +++ b/third_party/non_module_deps.bzl @@ -1,7 +1,9 @@ +load("//third_party/arocc:repo.bzl", arocc = "repo") load("//third_party/com_github_hejsil_clap:repo.bzl", com_github_hejsil_clap = "repo") load("//third_party/com_google_sentencepiece:repo.bzl", com_google_sentencepiece = "repo") load("//third_party/mnist:repo.bzl", mnist = "repo") load("//third_party/org_swig_swig:repo.bzl", org_swig_swig = "repo") +load("//third_party/translate-c:repo.bzl", translate_c = "repo") load("//third_party/xla:repo.bzl", xla = "repo") def _non_module_deps_impl(mctx): @@ -10,6 +12,8 @@ def _non_module_deps_impl(mctx): com_github_hejsil_clap() mnist() xla() + arocc() + translate_c() return mctx.extension_metadata( reproducible = True, diff --git a/third_party/translate-c/BUILD.bazel b/third_party/translate-c/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/third_party/translate-c/repo.bzl b/third_party/translate-c/repo.bzl new file mode 100644 index 00000000..0390cb87 --- /dev/null +++ b/third_party/translate-c/repo.bzl @@ -0,0 +1,9 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") + +def repo(): + new_git_repository( + name = "translate-c", + remote = "https://codeberg.org/ziglang/translate-c.git", + commit = "55f225f0a37b9627d459ed23b7df21f458f492d9", + build_file = "//third_party/translate-c:translate-c.bazel", + ) diff --git a/third_party/translate-c/translate-c.bazel b/third_party/translate-c/translate-c.bazel new file mode 100644 index 00000000..f71603e3 --- /dev/null +++ b/third_party/translate-c/translate-c.bazel @@ -0,0 +1,34 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_configure_binary", "zig_library") + +zig_library( + name = "c_builtins", + main = "lib/c_builtins.zig", + visibility = ["//visibility:public"], +) + +zig_library( + name = "helpers", + main = "lib/helpers.zig", + visibility = ["//visibility:public"], +) + +zig_binary( + name = "translate-c_raw", + srcs = glob(["src/**/*.zig"]), + main = "src/main.zig", + linkopts = ["-lc"], + visibility = ["//visibility:public"], + deps = [ + ":c_builtins", + ":helpers", + "@arocc//:aro", + ], +) + +zig_configure_binary( + name = "translate-c", + actual = ":translate-c_raw", + no_translate_c = 1, + use_cc_common_link = 0, + visibility = ["//visibility:public"], +) diff --git a/translate-c/BUILD.bazel b/translate-c/BUILD.bazel new file mode 100644 index 00000000..91f3fc51 --- /dev/null +++ b/translate-c/BUILD.bazel @@ -0,0 +1,17 @@ +alias( + name = "translate-c", + actual = "@translate-c", + visibility = ["//visibility:public"], +) + +alias( + name = "helpers", + actual = "@translate-c//:helpers", + visibility = ["//visibility:public"], +) + +alias( + name = "c_builtins", + actual = "@translate-c//:c_builtins", + visibility = ["//visibility:public"], +) From bff3193026e2f421bb2d867638daa02c9da3bf5f Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 1 Dec 2025 19:30:16 +0100 Subject: [PATCH 063/881] WIP --- MODULE.bazel | 15 ++++++++++++--- MODULE.bazel.lock | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index c694df5e..a8f47598 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -28,6 +28,7 @@ bazel_dep(name = "with_cfg.bzl", version = "0.11.0") bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2", dev_dependency = True) +### Zig toolchain zig = use_extension("@rules_zig//zig:extensions.bzl", "zig") zig.index(file = "//bazel:zig_index.json") zig.toolchain(zig_version = "0.16.0-dev.1484+d0ba6642b") @@ -40,12 +41,20 @@ use_repo(zig, "zig_toolchains") register_toolchains("@rules_zig//zig/target:all") register_toolchains("@zig_toolchains//:all") +### +### Apple C/C++ toolchain (needs to be first) +apple_cc_configure = use_extension("@apple_support//crosstool:setup.bzl", "apple_cc_configure_extension") +use_repo(apple_cc_configure, "local_config_apple_cc", "local_config_apple_cc_toolchains") + +register_toolchains("@local_config_apple_cc_toolchains//:all") +### + +### LLVM Bootstrapped toolchains register_toolchains( - "@toolchains_llvm_bootstrapped//toolchain:macos_aarch64_to_linux_x86_64", - # "@toolchains_llvm_bootstrapped//toolchain:linux_amd64", - # "@toolchains_llvm_bootstrapped//toolchain:linux_aarch64", + "@toolchains_llvm_bootstrapped//toolchain:all", ) +### uv = use_extension("@rules_python//python/uv:uv.bzl", "uv", dev_dependency = True) uv.configure(version = "0.6.5") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index dc44f6dc..aae72b5a 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -433,7 +433,7 @@ "@@apple_support+//crosstool:setup.bzl%apple_cc_configure_extension": { "general": { "bzlTransitiveDigest": "6HFMfNkxiYUs5AKI+9d2/9J5tWKUXMh2sUT0MsHY6jI=", - "usagesDigest": "si6rju4O264yIf33EKkDlbbpsFHy/ADOlY9r6T0faq8=", + "usagesDigest": "GXVjAqzvGUnh20kBPs+f0GSdAtaqLPet4sogeCfejZY=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, From 36760aec62c52b72a1c2765ff43bc3cda9b518cd Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 2 Dec 2025 14:31:52 +0100 Subject: [PATCH 064/881] WIPREADER --- MODULE.bazel | 12 +++-- main.zig | 4 +- third_party/zls/BUILD.bazel | 8 ++- third_party/zls/cat.zig | 33 +++++++++++- third_party/zls/zls_build_runner.zig | 59 ++++++++++++---------- third_party/zls/zls_runner.zig | 5 +- third_party/zls/zls_write_build_config.bzl | 19 +++++-- 7 files changed, 101 insertions(+), 39 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index a8f47598..19a3f7ff 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -2,6 +2,7 @@ module( name = "zml", ) +bazel_dep(name = "apple_support", version = "1.24.5") bazel_dep(name = "abseil-cpp", version = "20250814.0") bazel_dep(name = "aspect_bazel_lib", version = "2.21.2") bazel_dep(name = "aspect_rules_py", version = "1.6.3") @@ -14,7 +15,6 @@ bazel_dep(name = "protobuf", version = "33.1", repo_name = "com_google_protobuf" # Needs to be added before rules_cc so that the cc toolchain declared by # apple_support wins over the one in rules_cc. -bazel_dep(name = "apple_support", version = "1.24.2") bazel_dep(name = "rules_cc", version = "0.2.12") bazel_dep(name = "rules_distroless", version = "0.5.1") bazel_dep(name = "rules_license", version = "1.0.0") @@ -225,10 +225,14 @@ huggingface.model( use_repo(huggingface, "ModernBERT-base", "ModernBERT-large") ################################################################################ -git_override( +# git_override( +# module_name = "rules_zig", +# commit = "2c91b54305772d7b43445168e7cf0037b5cbc9e2", +# remote = "https://github.com/zml/rules_zig.git", +# ) +local_path_override( module_name = "rules_zig", - commit = "2c91b54305772d7b43445168e7cf0037b5cbc9e2", - remote = "https://github.com/zml/rules_zig.git", + path = "../rules_zig", ) git_override( diff --git a/main.zig b/main.zig index a2a52c95..83d4cd52 100644 --- a/main.zig +++ b/main.zig @@ -1,6 +1,7 @@ const std = @import("std"); -const upb = @import("upb"); + const c = @import("c"); +const upb = @import("upb"); pub fn main() !void { var arena = std.heap.ArenaAllocator.init(std.heap.smp_allocator); @@ -47,5 +48,4 @@ pub fn main() !void { }; _ = options; - } diff --git a/third_party/zls/BUILD.bazel b/third_party/zls/BUILD.bazel index 6c1560ea..dd8f026c 100644 --- a/third_party/zls/BUILD.bazel +++ b/third_party/zls/BUILD.bazel @@ -1,4 +1,3 @@ -load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("@rules_zig//zig:defs.bzl", "zig_binary") load(":resolved_toolchain.bzl", "resolved_toolchain") @@ -32,11 +31,16 @@ filegroup( name = "all_files", srcs = [ ":BUILD.bazel", - ":zls_build_runner.zig", ":zls.completion.tpl", + ":zls_build_runner.zig", ":zls_completion.bzl", ":zls_runner.zig", ":zls_toolchain.bzl", ], visibility = ["@rules_zig//zig:__pkg__"], ) + +zig_binary( + name = "workspace_printer", + main = "workspace_printer.zig", +) diff --git a/third_party/zls/cat.zig b/third_party/zls/cat.zig index acfa8ea0..25040ccf 100644 --- a/third_party/zls/cat.zig +++ b/third_party/zls/cat.zig @@ -1,7 +1,12 @@ const std = @import("std"); const builtin = @import("builtin"); -pub fn main() !void { +pub const main = if (builtin.zig_version.order(.{ .major = 0, .minor = 16, .patch = 0 }).compare(.gte)) + main_io +else + main_non_io; + +pub fn main_non_io() !void { const gpa = std.heap.page_allocator; const args = try std.process.argsAlloc(gpa); @@ -29,3 +34,29 @@ pub fn main() !void { } } } + +pub fn main_io() !void { + var arena_ = std.heap.ArenaAllocator.init(std.heap.smp_allocator); + defer arena_.deinit(); + const arena = arena_.allocator(); + + const args = try std.process.argsAlloc(arena); + defer std.process.argsFree(arena, args); + + const file_path = args[1]; + var io = std.Io.Threaded.init(arena); + defer io.deinit(); + + var file = try std.Io.Dir.cwd().openFile(io.io(), file_path, .{ .mode = .read_only }); + defer file.close(io.io()); + + var read_buffer: [8192]u8 = undefined; + var reader = file.reader(io.io(), &read_buffer); + + var write_buffer: [8192]u8 = undefined; + var writer = std.fs.File.stdout().writer(&write_buffer); + var stdout = &writer.interface; + + _ = try reader.interface.streamRemaining(stdout); + try stdout.flush(); +} diff --git a/third_party/zls/zls_build_runner.zig b/third_party/zls/zls_build_runner.zig index 9ec12c9b..675f61be 100644 --- a/third_party/zls/zls_build_runner.zig +++ b/third_party/zls/zls_build_runner.zig @@ -1,10 +1,8 @@ -const builtin = @import("builtin"); const std = @import("std"); +const builtin = @import("builtin"); pub fn main() !void { - var gpa: std.heap.GeneralPurposeAllocator(.{}) = .{}; - defer _ = gpa.deinit(); - var arena_ = std.heap.ArenaAllocator.init(gpa.allocator()); + var arena_ = std.heap.ArenaAllocator.init(std.heap.smp_allocator); defer arena_.deinit(); const arena = arena_.allocator(); @@ -15,36 +13,45 @@ pub fn main() !void { "@@__TARGET__@@", }, arena); child.stdin_behavior = .Ignore; - child.stdout_behavior = .Pipe; + child.stdout_behavior = .Inherit; child.stderr_behavior = .Inherit; child.cwd = build_workspace_directory; try child.spawn(); - var out = child.stdout.?; + // var out = child.stdout.?; + // _ = out; // autofix - const file = try out.readToEndAlloc(arena, 16 * 1024 * 1024); + // const file = if (builtin.zig_version.major == 0 and builtin.zig_version.minor >= 16) blk: { + // var threaded = std.Io.Threaded.init(arena); + // defer threaded.deinit(); + // const io = threaded.io(); - const need = std.mem.replacementSize( - u8, - file, - "@@__BUILD_WORKSPACE_DIRECTORY__@@", - build_workspace_directory, - ); + // var read_buffer: [8192]u8 = undefined; + // var reader = out.reader(io, &read_buffer); + // break :blk try reader.interface.allocRemaining(arena, .unlimited); + // } else try out.readToEndAlloc(arena, 16 * 1024 * 1024); - const replaced = try arena.alloc(u8, need); + // const need = std.mem.replacementSize( + // u8, + // file, + // "@@__BUILD_WORKSPACE_DIRECTORY__@@", + // build_workspace_directory, + // ); - _ = std.mem.replace( - u8, - file, - "@@__BUILD_WORKSPACE_DIRECTORY__@@", - build_workspace_directory, - replaced, - ); + // const replaced = try arena.alloc(u8, need); - if (comptime builtin.zig_version.order(.{ .major = 0, .minor = 15, .patch = 0 }).compare(.gte)) { - _ = try std.fs.File.stdout().writeAll(replaced); - } else { - _ = try std.io.getStdOut().writeAll(replaced); - } + // _ = std.mem.replace( + // u8, + // file, + // "@@__BUILD_WORKSPACE_DIRECTORY__@@", + // build_workspace_directory, + // replaced, + // ); + + // if (comptime builtin.zig_version.order(.{ .major = 0, .minor = 15, .patch = 0 }).compare(.gte)) { + // _ = try std.fs.File.stdout().writeAll(replaced); + // } else { + // _ = try std.io.getStdOut().writeAll(replaced); + // } } diff --git a/third_party/zls/zls_runner.zig b/third_party/zls/zls_runner.zig index 0f011ec4..0bfd0aca 100644 --- a/third_party/zls/zls_runner.zig +++ b/third_party/zls/zls_runner.zig @@ -43,7 +43,10 @@ pub fn main() !void { var arena = std.heap.ArenaAllocator.init(gpa); const allocator = arena.allocator(); - var r_ = try runfiles.Runfiles.create(.{ .allocator = allocator }) orelse + var threaded: std.Io.Threaded = .init(gpa); + defer threaded.deinit(); + + var r_ = try runfiles.Runfiles.create(.{ .allocator = allocator, .io = threaded.io() }) orelse return error.RunfilesNotFound; defer r_.deinit(allocator); diff --git a/third_party/zls/zls_write_build_config.bzl b/third_party/zls/zls_write_build_config.bzl index 22c1f83c..dfcafde9 100644 --- a/third_party/zls/zls_write_build_config.bzl +++ b/third_party/zls/zls_write_build_config.bzl @@ -1,8 +1,9 @@ """Generates the ZLS build config file.""" +load("@apple_support//lib:apple_support.bzl", "apple_support") +load("@rules_cc//cc:find_cc_toolchain.bzl", "use_cc_toolchain") load("@rules_cc//cc/common:cc_common.bzl", "cc_common") load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") -load("@rules_cc//cc:find_cc_toolchain.bzl", "use_cc_toolchain") load("@rules_zig//zig/private:cc_helper.bzl", "need_translate_c") load("@rules_zig//zig/private/common:translate_c.bzl", "zig_translate_c") load("@rules_zig//zig/private/common:zig_cache.bzl", "zig_cache_output") @@ -84,6 +85,7 @@ def _zls_write_build_config_impl(ctx): c_module = zig_translate_c( ctx = ctx, name = "c", + canonical_name = "c", zigtoolchaininfo = zigtoolchaininfo, global_args = global_args, cc_infos = [cc_info], @@ -130,10 +132,21 @@ zls_write_build_config = rule( default = ":zls.completion.tpl", allow_single_file = True, ), - }, + "_translate_c": attr.label( + default = Label("@rules_zig//zig/private/common:translate-c"), + cfg = "exec", + executable = True, + ), + "_c_helpers": attr.label( + default = Label("@rules_zig//zig/private/common:helpers"), + ), + "_c_builtins": attr.label( + default = Label("@rules_zig//zig/private/common:c_builtins"), + ), + } | apple_support.action_required_attrs() | apple_support.platform_constraint_attrs(), toolchains = [ "@rules_zig//zig:toolchain_type", "@rules_zig//zig/target:toolchain_type", ] + use_cc_toolchain(mandatory = False), - fragments = ["cpp"], + fragments = ["cpp", "apple"], ) From de6b1a570b1793fb4ebafb0ca28f5ccaeec1aa39 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 2 Dec 2025 15:21:25 +0100 Subject: [PATCH 065/881] WIP --- third_party/zls/string_replace_reader.zig | 28 +++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 third_party/zls/string_replace_reader.zig diff --git a/third_party/zls/string_replace_reader.zig b/third_party/zls/string_replace_reader.zig new file mode 100644 index 00000000..3b68bf67 --- /dev/null +++ b/third_party/zls/string_replace_reader.zig @@ -0,0 +1,28 @@ +const std = @import("std"); + +pub const StringReplaceReader = struct { + interface: std.Io.Reader = .{ + .buffer = &.{}, + .vtable = &.{ + .stream = stream, + }, + .seek = 0, + .end = 0, + }, + in: *std.Io.Reader, + pattern: []const u8, + replace_by: []const u8, + + pub fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { + const self: *StringReplaceReader = @fieldParentPtr("interface", r); + if (std.mem.eql(u8, try self.in.peek(self.pattern.len), self.pattern)) { + self.in.toss(self.pattern.len); + try w.writeAll(self.replace_by); + return self.replace_by.len; + } + return self.in.streamDelimiterLimit(w, self.pattern[0], limit) catch |err| switch (err) { + std.Io.Reader.StreamDelimiterLimitError.StreamTooLong => return 0, + else => |e| return e, + }; + } +}; From 133abd1c7f0de4f16dbbcc43349042e32726d3fc Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 2 Dec 2025 15:21:57 +0100 Subject: [PATCH 066/881] WIP --- third_party/zls/workspace_printer.zig | 119 ++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 third_party/zls/workspace_printer.zig diff --git a/third_party/zls/workspace_printer.zig b/third_party/zls/workspace_printer.zig new file mode 100644 index 00000000..0d51d671 --- /dev/null +++ b/third_party/zls/workspace_printer.zig @@ -0,0 +1,119 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const PATTERN = "@@__BUILD_WORKSPACE_DIRECTORY__@@"; + +pub const StringReplaceReader = struct { + interface: std.Io.Reader = .{ + .buffer = &.{}, + .vtable = &.{ + .stream = stream, + }, + .seek = 0, + .end = 0, + }, + in: *std.Io.Reader, + pattern: []const u8, + replace_by: []const u8, + + pub fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { + const self: *StringReplaceReader = @fieldParentPtr("interface", r); + if (std.mem.eql(u8, try self.in.peek(self.pattern.len), self.pattern)) { + self.in.toss(self.pattern.len); + try w.writeAll(self.replace_by); + return self.replace_by.len; + } + return self.in.streamDelimiterLimit(w, self.pattern[0], limit) catch |err| switch (err) { + std.Io.Reader.StreamDelimiterLimitError.StreamTooLong => return 0, + else => |e| return e, + }; + } +}; + +pub fn main_pre_015() !void { + const arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); + defer arena.deinit(); + + const args = try std.process.argsAlloc(arena.allocator()); + defer std.process.argsFree(arena.allocator(), args); + + const file_path = args[1]; + const content = try std.fs.cwd().readFileAlloc(file_path, arena.allocator(), std.math.maxInt(usize)); + const build_workspace_directory = try std.process.getEnvVarOwned(arena.allocator(), "BUILD_WORKSPACE_DIRECTORY"); + + const need = std.mem.replacementSize(u8, content, PATTERN, build_workspace_directory); + const replaced = try arena.allocator().alloc(u8, need); + _ = std.mem.replace(u8, content, PATTERN, build_workspace_directory, replaced); + try std.io.getStdOut().writeAll(replaced); +} + +pub fn main_015() !void { + const gpa = std.heap.page_allocator; + + const args = try std.process.argsAlloc(gpa); + defer std.process.argsFree(gpa, args); + + const file_path = args[1]; + var file = try std.fs.cwd().openFile(file_path, .{}); + defer file.close(); + + var read_buffer: [8192]u8 = undefined; + var reader = file.reader(io, &read_buffer); + + const build_workspace_directory = try std.process.getEnvVarOwned(arena.allocator(), "BUILD_WORKSPACE_DIRECTORY"); + var replacer: StringReplaceReader = .{ + .in = &reader.interface, + .pattern = "@@__BUILD_WORKSPACE_DIRECTORY__@@", + .replace_by = build_workspace_directory, + }; + + var write_buffer: [8192]u8 = undefined; + var stdout_writer = std.fs.File.stdout().writer(&write_buffer); + + var write_buffer: [8192]u8 = undefined; + var writer = std.fs.File.stdout().writer(&write_buffer); + _ = try replacer.interface.streamRemaining(&writer.interface); + try writer.interface.flush(); + + var reader = file.reader(&.{}); + _ = try reader.interface.streamRemaining(&stdout_writer.interface); + try stdout.flush(); +} + +pub fn main_016() !void { + var arena = std.heap.ArenaAllocator.init(std.heap.smp_allocator); + defer arena.deinit(); + + var threaded = std.Io.Threaded.init(arena.allocator()); + defer threaded.deinit(); + const io = threaded.io(); + + const args = try std.process.argsAlloc(arena.allocator()); + defer std.process.argsFree(arena.allocator(), args); + + const file_path = args[1]; + var file = try std.Io.Dir.cwd().openFile(io, file_path, .{ .mode = .read_only }); + defer file.close(io); + + var read_buffer: [8192]u8 = undefined; + var reader = file.reader(io, &read_buffer); + + const build_workspace_directory = try std.process.getEnvVarOwned(arena.allocator(), "BUILD_WORKSPACE_DIRECTORY"); + var replacer: StringReplaceReader = .{ + .in = &reader.interface, + .pattern = "@@__BUILD_WORKSPACE_DIRECTORY__@@", + .replace_by = build_workspace_directory, + }; + + var write_buffer: [8192]u8 = undefined; + var writer = std.fs.File.stdout().writer(&write_buffer); + _ = try replacer.interface.streamRemaining(&writer.interface); + try writer.interface.flush(); +} + +pub const main = if (builtin.zig_version.major == 0 and builtin.zig_version.minor >= 16) + main_016 +else if (builtin.zig_version.major == 0 and builtin.zig_version.minor == 15) + main_015 +else + main_pre_015; From 55b207354223b422b1651512a1dac9186dd40866 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Tue, 2 Dec 2025 16:23:25 +0100 Subject: [PATCH 067/881] fix workspace_printer for all versions of zig --- third_party/zls/BUILD.bazel | 2 +- third_party/zls/workspace_printer.zig | 23 +++++++++-------------- third_party/zls/zls_completion.bzl | 2 +- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/third_party/zls/BUILD.bazel b/third_party/zls/BUILD.bazel index dd8f026c..00b78c07 100644 --- a/third_party/zls/BUILD.bazel +++ b/third_party/zls/BUILD.bazel @@ -19,7 +19,7 @@ resolved_toolchain( exports_files( [ - "cat.zig", + "workspace_printer.zig", "zls_build_runner.zig", "zls_runner.zig", "zls.completion.tpl", diff --git a/third_party/zls/workspace_printer.zig b/third_party/zls/workspace_printer.zig index 0d51d671..52a3a652 100644 --- a/third_party/zls/workspace_printer.zig +++ b/third_party/zls/workspace_printer.zig @@ -31,14 +31,14 @@ pub const StringReplaceReader = struct { }; pub fn main_pre_015() !void { - const arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); + var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); const args = try std.process.argsAlloc(arena.allocator()); defer std.process.argsFree(arena.allocator(), args); const file_path = args[1]; - const content = try std.fs.cwd().readFileAlloc(file_path, arena.allocator(), std.math.maxInt(usize)); + const content = try std.fs.cwd().readFileAlloc(arena.allocator(), file_path, std.math.maxInt(usize)); const build_workspace_directory = try std.process.getEnvVarOwned(arena.allocator(), "BUILD_WORKSPACE_DIRECTORY"); const need = std.mem.replacementSize(u8, content, PATTERN, build_workspace_directory); @@ -48,17 +48,18 @@ pub fn main_pre_015() !void { } pub fn main_015() !void { - const gpa = std.heap.page_allocator; + var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); + defer arena.deinit(); - const args = try std.process.argsAlloc(gpa); - defer std.process.argsFree(gpa, args); + const args = try std.process.argsAlloc(arena.allocator()); + defer std.process.argsFree(arena.allocator(), args); const file_path = args[1]; var file = try std.fs.cwd().openFile(file_path, .{}); defer file.close(); var read_buffer: [8192]u8 = undefined; - var reader = file.reader(io, &read_buffer); + var reader = file.reader(&read_buffer); const build_workspace_directory = try std.process.getEnvVarOwned(arena.allocator(), "BUILD_WORKSPACE_DIRECTORY"); var replacer: StringReplaceReader = .{ @@ -70,14 +71,8 @@ pub fn main_015() !void { var write_buffer: [8192]u8 = undefined; var stdout_writer = std.fs.File.stdout().writer(&write_buffer); - var write_buffer: [8192]u8 = undefined; - var writer = std.fs.File.stdout().writer(&write_buffer); - _ = try replacer.interface.streamRemaining(&writer.interface); - try writer.interface.flush(); - - var reader = file.reader(&.{}); - _ = try reader.interface.streamRemaining(&stdout_writer.interface); - try stdout.flush(); + _ = try replacer.interface.streamRemaining(&stdout_writer.interface); + try stdout_writer.interface.flush(); } pub fn main_016() !void { diff --git a/third_party/zls/zls_completion.bzl b/third_party/zls/zls_completion.bzl index 02f11f31..f2b59387 100644 --- a/third_party/zls/zls_completion.bzl +++ b/third_party/zls/zls_completion.bzl @@ -26,7 +26,7 @@ def zls_completion(name, deps, **kwargs): build_config_printer = "print_{}_build_config".format(name) zig_binary( name = build_config_printer, - main = Label("//third_party/zls:cat.zig"), + main = Label("//third_party/zls:workspace_printer.zig"), data = [ ":{}".format(build_config), ], From ddb05f5cdef11b03f720038d1c3c861b28ef5337 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Tue, 2 Dec 2025 16:26:23 +0100 Subject: [PATCH 068/881] Remove cat --- third_party/zls/cat.zig | 62 ----------------------------------------- 1 file changed, 62 deletions(-) delete mode 100644 third_party/zls/cat.zig diff --git a/third_party/zls/cat.zig b/third_party/zls/cat.zig deleted file mode 100644 index 25040ccf..00000000 --- a/third_party/zls/cat.zig +++ /dev/null @@ -1,62 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); - -pub const main = if (builtin.zig_version.order(.{ .major = 0, .minor = 16, .patch = 0 }).compare(.gte)) - main_io -else - main_non_io; - -pub fn main_non_io() !void { - const gpa = std.heap.page_allocator; - - const args = try std.process.argsAlloc(gpa); - defer std.process.argsFree(gpa, args); - - const file_path = args[1]; - var file = try std.fs.cwd().openFile(file_path, .{ .mode = .read_only }); - defer file.close(); - - if (builtin.zig_version.major == 0 and builtin.zig_version.minor >= 15) { - var write_buffer: [512]u8 = undefined; - var writer = std.fs.File.stdout().writer(&write_buffer); - var stdout = &writer.interface; - - var reader = file.reader(&.{}); - _ = try reader.interface.streamRemaining(stdout); - try stdout.flush(); - } else { - var stdout = std.io.getStdOut().writer(); - var buffer: [4096]u8 = undefined; - while (true) { - const n = try file.read(&buffer); - if (n == 0) break; // EOF - try stdout.writeAll(buffer[0..n]); - } - } -} - -pub fn main_io() !void { - var arena_ = std.heap.ArenaAllocator.init(std.heap.smp_allocator); - defer arena_.deinit(); - const arena = arena_.allocator(); - - const args = try std.process.argsAlloc(arena); - defer std.process.argsFree(arena, args); - - const file_path = args[1]; - var io = std.Io.Threaded.init(arena); - defer io.deinit(); - - var file = try std.Io.Dir.cwd().openFile(io.io(), file_path, .{ .mode = .read_only }); - defer file.close(io.io()); - - var read_buffer: [8192]u8 = undefined; - var reader = file.reader(io.io(), &read_buffer); - - var write_buffer: [8192]u8 = undefined; - var writer = std.fs.File.stdout().writer(&write_buffer); - var stdout = &writer.interface; - - _ = try reader.interface.streamRemaining(stdout); - try stdout.flush(); -} From 299aaac7a7bdb452dbcb148a3b1605ddea5a371c Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Fri, 5 Dec 2025 16:08:15 +0100 Subject: [PATCH 069/881] cc_infos should be exposed as direct since translate_c use direct_public_headers --- third_party/zls/zls_write_build_config.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/zls/zls_write_build_config.bzl b/third_party/zls/zls_write_build_config.bzl index dfcafde9..1a758b67 100644 --- a/third_party/zls/zls_write_build_config.bzl +++ b/third_party/zls/zls_write_build_config.bzl @@ -62,7 +62,7 @@ def _zls_write_build_config_impl(ctx): c_module_contexts = [] c_module_inputs = [] - cc_info = cc_common.merge_cc_infos(cc_infos = [dep[ZigModuleInfo].cc_info for dep in ctx.attr.deps if dep[ZigModuleInfo].cc_info]) + cc_info = cc_common.merge_cc_infos(direct_cc_infos = [dep[ZigModuleInfo].cc_info for dep in ctx.attr.deps if dep[ZigModuleInfo].cc_info]) if need_translate_c(cc_info): global_args = ctx.actions.args() global_args.use_param_file("@%s") From eba929b8cc861f8951e561bf9b0ecfde7c33fdb6 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 1 Dec 2025 11:25:03 +0100 Subject: [PATCH 070/881] Initial rework of Context and Platform * Removed `zml.Context` for `zml.init()` and `zml.deinit()` * `zml.Platform` is responsible for loading the PJRT api when initialized. This replace the previous behavior of trying to load all enabled runtimes. --- zml2/BUILD.bazel | 44 ++++ zml2/dtype.zig | 375 +++++++++++++++++++++++++++++++++ zml2/floats.zig | 517 ++++++++++++++++++++++++++++++++++++++++++++++ zml2/main.zig | 15 ++ zml2/pjrtx.zig | 362 ++++++++++++++++++++++++++++++++ zml2/platform.zig | 190 +++++++++++++++++ zml2/posix.h | 2 + zml2/zml.zig | 61 ++++++ 8 files changed, 1566 insertions(+) create mode 100644 zml2/BUILD.bazel create mode 100644 zml2/dtype.zig create mode 100644 zml2/floats.zig create mode 100644 zml2/main.zig create mode 100644 zml2/pjrtx.zig create mode 100644 zml2/platform.zig create mode 100644 zml2/posix.h create mode 100644 zml2/zml.zig diff --git a/zml2/BUILD.bazel b/zml2/BUILD.bazel new file mode 100644 index 00000000..84eb6dda --- /dev/null +++ b/zml2/BUILD.bazel @@ -0,0 +1,44 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_binary") +load("@rules_cc//cc:defs.bzl", "cc_library") + +cc_library( + name = "posix", + hdrs = ["posix.h"], +) + +zig_library( + name = "zml", + srcs = [ + "zml.zig", + "platform.zig", + "pjrtx.zig", + "dtype.zig", + "floats.zig", + ], + zigopts = [ + "-lc", + "-freference-trace=20", + ], + main = "zml.zig", + visibility = ["//visibility:public"], + deps = [ + "//runtimes", + "//mlir", + "//mlir/dialects", + "//pjrt", + "//stdx", + "@rules_zig//zig/runfiles", + ":posix", + ], +) + +zig_binary( + name = "main", + args = [], + data = [], + main = "main.zig", + deps = [ + "//async", + "//zml2:zml", + ], +) diff --git a/zml2/dtype.zig b/zml2/dtype.zig new file mode 100644 index 00000000..b14f483e --- /dev/null +++ b/zml2/dtype.zig @@ -0,0 +1,375 @@ +const std = @import("std"); + +const floats = @import("floats.zig"); + +const C64 = std.math.Complex(f32); +const C128 = std.math.Complex(f64); + +test { + std.testing.refAllDecls(@This()); +} + +pub const DataType = enum(u8) { + bool, + // Note: the support of the float8 is a bit spotty, f8e4m3b11fnuz seems to be the most supported one on Cuda. + f4e2m1, + f8e3m4, + f8e4m3, + f8e4m3b11fnuz, + f8e4m3fn, + f8e4m3fnuz, + f8e5m2, + f8e5m2fnuz, + f8e8m0, + bf16, + f16, + f32, + f64, + i2, + i4, + i8, + i16, + i32, + i64, + u2, + u4, + u8, + u16, + u32, + u64, + c64, + c128, + + pub fn str(self: DataType) [:0]const u8 { + return switch (self) { + inline else => |tag| @tagName(tag), + }; + } + + pub const Class = enum(u8) { + bool, + float, + integer, + complex, + }; + + pub fn class(self: DataType) Class { + return switch (self) { + .bool => .bool, + .f4e2m1, + .f8e3m4, + .f8e4m3, + .f8e4m3b11fnuz, + .f8e4m3fn, + .f8e4m3fnuz, + .f8e5m2, + .f8e5m2fnuz, + .f8e8m0, + .bf16, + .f16, + .f32, + .f64, + => .float, + .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => .integer, + .c64, .c128 => .complex, + }; + } + + pub fn isInteger(self: DataType) bool { + return self.class() == .integer; + } + + pub fn isFloat(self: DataType) bool { + return self.class() == .float; + } + + pub fn isComplex(self: DataType) bool { + return self.class() == .complex; + } + + pub fn fromZigType(comptime T: type) DataType { + return switch (T) { + floats.Float4E2M1 => .f4e2m1, + floats.Float8E3M4 => .f8e3m4, + floats.Float8E4M3 => .f8e4m3, + floats.Float8E4M3B11FNUZ => .f8e4m3b11fnuz, + floats.Float8E4M3FN => .f8e4m3fn, + floats.Float8E4M3FNUZ => .f8e4m3fnuz, + floats.Float8E5M2 => .f8e5m2, + floats.Float8E5M2FNUZ => .f8e5m2fnuz, + floats.Float8E8M0 => .f8e8m0, + floats.BFloat16 => .bf16, + f16 => .f16, + f32 => .f32, + f64 => .f64, + bool => .bool, + i2 => .i2, + i4 => .i4, + i8 => .i8, + i16 => .i16, + i32 => .i32, + i64 => .i64, + u2 => .u2, + u4 => .u4, + u8 => .u8, + u16 => .u16, + u32 => .u32, + u64 => .u64, + C64 => .c64, + C128 => .c128, + else => @compileError("Unsupported Zig type: " ++ @typeName(T)), + }; + } + + pub fn fromSliceElementType(slice: anytype) DataType { + const type_info = @typeInfo(@TypeOf(slice)); + if (type_info != .pointer) { + @compileError("`initFromSlice` expects a slice, got " ++ @tagName(type_info)); + } + + return switch (type_info.pointer.size) { + .slice, .c, .many => DataType.fromZigType(type_info.pointer.child), + .one => b: { + const child_type_info = @typeInfo(type_info.pointer.child); + break :b DataType.fromZigType(child_type_info.array.child); + }, + }; + } + + pub fn toZigType(comptime dtype: DataType) type { + return @FieldType(Data, @tagName(dtype)); + } + + pub fn isSignedInt(dtype: DataType) bool { + return switch (dtype) { + .i4, .i8, .i16, .i32, .i64 => true, + else => false, + }; + } + + pub fn sizeOf(self: DataType) u16 { + return switch (self) { + inline else => |tag| @sizeOf(tag.toZigType()), + }; + } + + pub fn bitSizeOf(self: DataType) u16 { + return switch (self) { + inline else => |tag| @bitSizeOf(tag.toZigType()), + }; + } + + pub fn alignOf(self: DataType) u29 { + return switch (self) { + inline else => |tag| @alignOf(tag.toZigType()), + }; + } + + /// Try to find a type compatible with both dtype. + pub fn resolvePeerType(a: DataType, b: DataType) ?DataType { + if (a == b) { + return a; + } + + // only resolve types in the same class + if (a.class() != b.class()) { + return null; + } + + return if (a.sizeOf() >= b.sizeOf()) a else b; + } + + test resolvePeerType { + try std.testing.expectEqual(DataType.f16.resolvePeerType(.f16), .f16); + try std.testing.expectEqual(DataType.f32.resolvePeerType(.f32), .f32); + + try std.testing.expectEqual(DataType.f16.resolvePeerType(.f32), .f32); + try std.testing.expectEqual(DataType.f32.resolvePeerType(.f16), .f32); + + try std.testing.expectEqual(DataType.f32.resolvePeerType(.f64), .f64); + try std.testing.expectEqual(DataType.f64.resolvePeerType(.f32), .f64); + try std.testing.expectEqual(DataType.f32.resolvePeerType(.i32), null); + + try std.testing.expectEqual(DataType.c64.resolvePeerType(.c128), .c128); + try std.testing.expectEqual(DataType.c128.resolvePeerType(.i32), null); + try std.testing.expectEqual(DataType.c64.resolvePeerType(.f32), null); + } + + pub fn zero(dtype: DataType) Data { + return Data.init(dtype, 0); + } + + pub fn one(dtype: DataType) Data { + return Data.init(dtype, 1); + } + + pub fn minValue(dtype: DataType) Data { + return switch (dtype) { + .bool => .{ .bool = false }, + inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).zero), + inline .f8e5m2, .bf16 => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).minus_inf), + inline .f16, .f32, .f64 => |tag| @unionInit(Data, @tagName(tag), -std.math.inf(@FieldType(Data, @tagName(tag)))), + inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Data, @tagName(tag), std.math.minInt(@FieldType(Data, @tagName(tag)))), + inline else => |tag| @panic("Unsupported type: " ++ @tagName(tag)), + }; + } + + pub fn maxValue(dtype: DataType) Data { + return switch (dtype) { + .bool => .{ .bool = true }, + inline .f4e2m1, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz, .f8e8m0 => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), + inline .f8e3m4, .f8e4m3, .f8e5m2, .bf16 => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).inf), + inline .f16, .f32, .f64 => |tag| @unionInit(Data, @tagName(tag), std.math.inf(@FieldType(Data, @tagName(tag)))), + inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Data, @tagName(tag), std.math.maxInt(@FieldType(Data, @tagName(tag)))), + inline .c64, .c128 => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), + }; + } + + pub fn constant(dtype: DataType, value: anytype) Data { + return Data.init(dtype, value); + } +}; + +pub const Data = union(DataType) { + bool: bool, + f4e2m1: floats.Float4E2M1, + f8e3m4: floats.Float8E3M4, + f8e4m3: floats.Float8E4M3, + f8e4m3b11fnuz: floats.Float8E4M3B11FNUZ, + f8e4m3fn: floats.Float8E4M3FN, + f8e4m3fnuz: floats.Float8E4M3FNUZ, + f8e5m2: floats.Float8E5M2, + f8e5m2fnuz: floats.Float8E5M2FNUZ, + f8e8m0: floats.Float8E8M0, + bf16: floats.BFloat16, + f16: f16, + f32: f32, + f64: f64, + i2: i2, + i4: i4, + i8: i8, + i16: i16, + i32: i32, + i64: i64, + u2: u2, + u4: u4, + u8: u8, + u16: u16, + u32: u32, + u64: u64, + c64: C64, + c128: C128, + + /// Creates `Data` from a `value`. + /// + /// If the `dtype` and `@TypeOf(value)` are incompatible + /// or a cast from `value` to `FieldType(dtype)` would + /// be lossy, a panic occurs. + pub fn init(dtype_: DataType, value: anytype) Data { + const T = @TypeOf(value); + const Ti = @typeInfo(T); + + return switch (dtype_) { + .bool => switch (Ti) { + .bool => .{ .bool = value }, + .comptime_int, .int, .comptime_float, .float => .{ .bool = value != 0 }, + else => @panic("Could not create Data of type bool from value of type " ++ @typeName(T)), + }, + inline .f4e2m1, .f8e3m4, .f8e4m3, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .f8e8m0, .bf16 => |tag| switch (Ti) { + .comptime_int, .int => @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).fromF32(@floatFromInt(value))), + .comptime_float, .float => @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).fromF32(@floatCast(value))), + else => @panic("Could not create Data of type bf16 from value of type " ++ @typeName(T)), + }, + inline .f16, .f32, .f64 => |tag| switch (Ti) { + .comptime_int, .int => @unionInit(Data, @tagName(tag), @floatFromInt(value)), + .comptime_float, .float => @unionInit(Data, @tagName(tag), @floatCast(value)), + else => @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), + }, + inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| switch (Ti) { + .comptime_int => blk: { + const OutT = @FieldType(Data, @tagName(tag)); + if (value >= std.math.minInt(OutT) and value <= std.math.maxInt(OutT)) { + break :blk @unionInit(Data, @tagName(tag), @intCast(value)); + } else { + @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)); + } + }, + .int => @unionInit(Data, @tagName(tag), @intCast(value)), + else => @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), + }, + .c64 => switch (T) { + C64 => .{ .c64 = value }, + C128 => .{ .c64 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, + else => @panic("Could not create Data of type c64 from value of type " ++ @typeName(T)), + }, + .c128 => switch (T) { + C64 => .{ .c128 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, + C128 => .{ .c128 = value }, + else => @panic("Could not create Data of type c128 from value of type " ++ @typeName(T)), + }, + }; + } + + test init { + try std.testing.expectEqual(20.0, Data.init(.f16, 20).f16); + try std.testing.expectEqual(20.5, Data.init(.f16, 20.5).f16); + try std.testing.expectEqual(20, Data.init(.f16, @as(u8, 20)).f16); + try std.testing.expectEqual(-20, Data.init(.f16, @as(i8, -20)).f16); + try std.testing.expectEqual(2000.5, Data.init(.f16, @as(f32, 2000.5)).f16); + + try std.testing.expectEqual(true, Data.init(.bool, true).bool); + + try std.testing.expectEqual(10, Data.init(.u8, 10).u8); + try std.testing.expectEqual(10, Data.init(.u8, @as(u16, 10)).u8); + + try std.testing.expectEqual(10, Data.init(.i8, 10).i8); + try std.testing.expectEqual(10, Data.init(.i8, @as(u16, 10)).i8); + try std.testing.expectEqual(-10, Data.init(.i8, -10).i8); + try std.testing.expectEqual(-10, Data.init(.i8, @as(i16, -10)).i8); + + try std.testing.expectEqual(C64.init(1, 2), Data.init(.c64, C64.init(1, 2)).c64); + try std.testing.expectEqual(C64.init(1, 2), Data.init(.c64, C128.init(1, 2)).c64); + try std.testing.expectEqual(C128.init(1, 2), Data.init(.c128, C128.init(1, 2)).c128); + try std.testing.expectEqual(C128.init(1, 2), Data.init(.c128, C64.init(1, 2)).c128); + } + + pub fn dtype(self: Data) DataType { + return std.meta.activeTag(self); + } + + pub fn constSlice(data: *const Data) []const u8 { + return switch (data.*) { + inline else => |*value| std.mem.asBytes(value), + }; + } + + pub fn as(self: Data, comptime T: type) T { + // TODO allow more lossless conversions + switch (@typeInfo(T)) { + .bool => return self.bool, + .float => switch (self) { + inline .f16, .f32, .f64 => |v| return @floatCast(v), + inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16 => |v| return @floatCast(v.toF32()), + else => {}, + }, + .int => switch (self) { + inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |v| return @intCast(v), + else => {}, + }, + else => {}, + } + std.debug.panic("Unsupported conversion {} -> {s}", .{ self.dtype(), @typeName(T) }); + } +}; + +pub fn mantissaSize(dtype: DataType) usize { + return switch (dtype) { + .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz => 3, + .f8e5m2, .f8e5m2fnuz => 2, + .f16 => 10, + .bf16 => 7, + .f32 => 23, + .f64 => 52, + else => @panic("Can't get mantissa size for a non-float dtype"), + }; +} diff --git a/zml2/floats.zig b/zml2/floats.zig new file mode 100644 index 00000000..54240485 --- /dev/null +++ b/zml2/floats.zig @@ -0,0 +1,517 @@ +///! Conversion utilities between different Floating point formats. +const std = @import("std"); + +test { + std.testing.refAllDecls(@This()); +} + +fn FloatHelpers(Float: type) type { + const info = @typeInfo(Float); + const err_msg = "FloatHelpers expect a packed struct { mantissa: uXX, exponent: uXX, sign: u1}"; + if (info != .@"struct" or info.@"struct".backing_integer == null) { + @compileError(err_msg); + } + comptime { + for (info.@"struct".fields, &.{ "mantissa", "exponent", "sign" }) |field, expected_name| { + if (!std.mem.eql(u8, field.name, expected_name)) + @compileError(err_msg); + } + } + + return struct { + const mantissa_bits: u8 = @typeInfo(@FieldType(Float, "mantissa")).int.bits; + const exponent_bits: u8 = @typeInfo(@FieldType(Float, "exponent")).int.bits; + const f32_mantissa_bits: u8 = @typeInfo(@FieldType(Float32, "mantissa")).int.bits; + const exp_bias: i16 = std.math.maxInt(std.meta.Int(.unsigned, exponent_bits - 1)); + const exp_off: u8 = FloatHelpers(Float32).exp_bias - exp_bias; + + pub const zero: Float = .{ .sign = 0, .exponent = 0, .mantissa = 0 }; + + pub fn neg(x: Float) Float { + return .{ + .sign = x.sign ^ 1, + .exponent = x.exponent, + .mantissa = x.mantissa, + }; + } + + /// Lossy conversion from f32, similar to @floatCast + pub fn fromF32(f: f32) Float { + @setRuntimeSafety(false); + + const vf32: Float32 = @bitCast(f); + const exponent: i16 = @as(i16, vf32.exponent) - exp_off; + const overflow = exponent > std.math.maxInt(@FieldType(Float, "exponent")); + if (overflow) { + @branchHint(.unlikely); + return if (@hasDecl(Float, "inf")) + if (vf32.sign == 0) Float.inf else Float.minus_inf + else + Float.nan; + } + + return if (exponent <= 0) + .{ + .sign = vf32.sign, + .exponent = 0, + .mantissa = shiftMantissa(vf32.mantissa, @intCast(-exponent)), + } + else + .{ + .sign = vf32.sign, + .exponent = @intCast(exponent), + .mantissa = truncMantissa(vf32.mantissa), + }; + } + + /// Lossless conversion to f32. + pub fn toF32(x: Float) f32 { + @setRuntimeSafety(false); + + if (x == zero) return 0.0; + if (isInf(x)) { + @branchHint(.unlikely); + return if (x.sign == 0) std.math.inf(f32) else -std.math.inf(f32); + } + + const vf32: Float32 = if (x.exponent > 0) + .{ + .sign = x.sign, + .exponent = @as(u8, x.exponent) + exp_off, + .mantissa = f32Mantissa(x), + } + else + .{ + .sign = x.sign, + .exponent = exp_off - @clz(x.mantissa), + .mantissa = @as(u23, x.mantissa) << @clz(x.mantissa), + }; + return @bitCast(vf32); + } + + fn truncMantissa(f32_mantissa: u32) @FieldType(Float, "mantissa") { + const rounding_val: u32 = @as(u32, 1) << (f32_mantissa_bits - mantissa_bits - 1); + return @truncate((f32_mantissa + rounding_val) >> (f32_mantissa_bits - mantissa_bits)); + } + + fn shiftMantissa(f32_mantissa: u32, underflow: u8) @FieldType(Float, "mantissa") { + const upper_bit: u32 = @as(u32, 1) << f32_mantissa_bits; + const full_mant32: u32 = f32_mantissa | upper_bit; + // divide the mantissa proportionally to the exponent underflow + const shifted_mant: u32 = full_mant32 >> @truncate(underflow + 1); + return truncMantissa(shifted_mant); + } + + fn f32Mantissa(x: Float) @FieldType(Float32, "mantissa") { + const T = @FieldType(Float32, "mantissa"); + return @as(T, x.mantissa) << f32_mantissa_bits - mantissa_bits; + } + + pub fn formatNumber(x: Float, writer: *std.io.Writer, n: std.fmt.Number) std.io.Writer.Error!void { + switch (n.mode) { + .binary, .octal, .hex => try writer.print("{{ .sign={}, .exp={}, .mantissa={} }}", .{ x.sign, x.exponent, x.mantissa }), + else => try writer.printFloat(x.toF32(), n), + } + } + }; +} + +pub const Float32 = packed struct(u32) { + mantissa: u23, + exponent: u8, + sign: u1, + + pub const inf: Float32 = .{ .sign = 0, .exponent = std.math.maxInt(u8), .mantissa = 0 }; + pub const minus_inf = neg(inf); + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const toF32 = Helpers.toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +const f32_exp_bias = FloatHelpers(Float32).expBias(); + +pub const Float64 = packed struct(u64) { + mantissa: u52, + exponent: u11, + sign: u1, + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const toF32 = Helpers.toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +pub const Float8E4M3B11FNUZ = packed struct(u8) { + mantissa: u3, + exponent: u4, + sign: u1, + + pub const nan: Float8E4M3B11FNUZ = .{ .sign = 1, .exponent = 0, .mantissa = 0 }; + + pub fn isNan(self: Float8E4M3B11FNUZ) bool { + return self.sign == 1 and self.exponent == 0 and self.mantissa == 0; + } + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const toF32 = Helpers.toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +pub const Float8E4M3FN = packed struct(u8) { + mantissa: u3, + exponent: u4, + sign: u1, + + pub const nan: Float8E4M3FN = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = std.math.maxInt(u3) }; + + pub fn isNan(self: Float8E4M3FN) bool { + return self.exponent == nan.exponent and self.mantissa == nan.mantissa; + } + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const toF32 = Helpers.toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +pub const Float8E4M3FNUZ = packed struct(u8) { + mantissa: u3, + exponent: u4, + sign: u1, + + pub const nan: Float8E4M3FNUZ = .{ .sign = 1, .exponent = 0, .mantissa = 0 }; + + pub fn isNan(self: Float8E4M3FNUZ) bool { + return self.sign == 1 and self.exponent == 0 and self.mantissa == 0; + } + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const toF32 = Helpers.toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +test "Float8E4" { + // With 4 bits of exponents power of two can be represented exactly up to 64. + const test_case_e4: TestCase = .{ + .lossless = &[_]f32{ 0, 1.0, -2, 1.0 / 64.0, -128, -1.125 / 64.0 }, + .lossy = &[_]f32{ 3.02344107628, 1.0 / 128.0, 1.0 / 512.0 }, + }; + + inline for (.{ + Float8E4M3B11FNUZ, + Float8E4M3FN, + Float8E4M3FNUZ, + }) |Float8T| { + try testCustomFloat(Float8T, test_case_e4); + try std.testing.expectEqual(0.0, Float8T.fromF32(1.0 / 2048.0).toF32()); + if (@hasDecl(Float8T, "inf")) { + try std.testing.expectEqual(Float8T.inf, Float8T.fromF32(128.0)); + try std.testing.expectEqual(Float8T.inf.neg(), Float8T.fromF32(-128.0)); + } + } +} + +pub const Float8E5M2 = packed struct(u8) { + mantissa: u2, + exponent: u5, + sign: u1, + + pub const nan: Float8E5M2 = .{ .sign = 0, .exponent = std.math.maxInt(u5), .mantissa = 1 }; + + pub fn isNan(self: Float8E5M2) bool { + return self.exponent == nan.exponent and self.mantissa != 0; + } + + pub const inf: Float8E5M2 = .{ + .sign = 0, + .exponent = std.math.maxInt(u5), + .mantissa = 0, + }; + + pub const minus_inf: Float8E5M2 = .neg(inf); + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const toF32 = Helpers.toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +pub const Float8E5M2FNUZ = packed struct(u8) { + mantissa: u2, + exponent: u5, + sign: u1, + + pub const nan: Float8E5M2FNUZ = .{ .sign = 1, .exponent = 0, .mantissa = 0 }; + + pub fn isNan(self: Float8E5M2FNUZ) bool { + return self.sign == 1 and self.exponent == 0 and self.mantissa == 0; + } + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const toF32 = Helpers.toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +test "Float8E5" { + const test_case_e5: TestCase = .{ + .lossless = &[_]f32{ 0, 1.0, -2, 1.0 / 128.0, -128 }, + .lossy = &[_]f32{3.02344107628}, + }; + inline for (.{ Float8E5M2, Float8E5M2FNUZ }) |Float8T| { + try testCustomFloat(Float8T, test_case_e5); + } +} + +pub const BFloat16 = packed struct(u16) { + mantissa: u7, + exponent: u8, + sign: u1, + + pub const nan: BFloat16 = .{ .sign = 0, .exponent = std.math.maxInt(u8), .mantissa = 1 }; + + pub fn isNan(self: BFloat16) bool { + return allBitsOne(self.exponent) and self.mantissa != 0; + } + + pub const inf: BFloat16 = .{ + .sign = 0, + .exponent = std.math.maxInt(u8), + .mantissa = 0, + }; + + pub const minus_inf: BFloat16 = .neg(inf); + + // Specialized versions of to/from F32. Since BFloat16 has the same exponent range than F32, + // no overflow/underflow can happen, simplifiying conversion logic. + pub fn toF32(self: BFloat16) f32 { + // Pad the BF16 with zeros 0 + return @bitCast([2]u16{ 0, @bitCast(self) }); + } + + pub fn fromF32(float32: f32) BFloat16 { + var int: u32 = @bitCast(float32); + // Round up if needed. + int += 0x8000; + const parts: [2]u16 = @bitCast(int); + return @bitCast(parts[1]); + } + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const formatNumber = Helpers.formatNumber; +}; + +test BFloat16 { + // From https://en.wikipedia.org/wiki/Bfloat16_floating-point_format#Examples + try std.testing.expectEqual(BFloat16.fromF32(0), BFloat16{ .sign = 0, .exponent = 0, .mantissa = 0 }); + try std.testing.expectEqual(BFloat16.fromF32(-2), BFloat16{ .sign = 1, .exponent = 127 + 1, .mantissa = 0 }); + try std.testing.expectEqual(BFloat16.fromF32(3.02344107628), BFloat16{ .sign = 0, .exponent = 127 + 1, .mantissa = 66 }); + try std.testing.expectEqual(BFloat16.fromF32(1.0 / 128.0), BFloat16{ .sign = 0, .exponent = 127 - 7, .mantissa = 0 }); + try std.testing.expectEqual(std.mem.toBytes(BFloat16.inf.neg()), [_]u8{ 0x80, 0xff }); + try std.testing.expectEqual(BFloat16.inf, BFloat16.fromF32(std.math.inf(f32))); + + try testCustomFloat(BFloat16, .{ + .lossless = &[_]f32{ 0, -2, 1.0 / 128.0, -1e64, std.math.inf(f32) }, + .lossy = &[_]f32{3.02344107628}, + }); +} + +pub const Float8E4M3 = packed struct(u8) { + mantissa: u3, + exponent: u4, + sign: u1, + + pub const nan: Float8E4M3 = @bitCast(0xFF); + + pub fn isNan(self: Float8E4M3) bool { + return self == nan or self == comptime nan.neg(); + } + + pub const inf: Float8E4M3 = .{ + .sign = 0, + .exponent = std.math.maxInt(u4), + .mantissa = 0, + }; + + pub const minus_inf = neg(inf); + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const toF32 = Helpers.toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +pub const Float8E3M4 = packed struct(u8) { + mantissa: u4, + exponent: u3, + sign: u1, + + pub const nan: Float8E3M4 = @bitCast(0xFF); + + pub fn isNan(self: Float8E3M4) bool { + return self == nan or self == comptime nan.neg(); + } + + pub const inf: Float8E3M4 = .{ + .sign = 0, + .exponent = std.math.maxInt(u3), + .mantissa = 0, + }; + + pub const minus_inf = neg(inf); + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const toF32 = Helpers.toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +pub const Float8E8M0 = packed struct(u8) { + mantissa: u0 = 0, + exponent: u8, + sign: u0 = 0, + + pub const min_scale: f32 = @bitCast(Float32{ .sign = 0, .exponent = 0, .mantissa = 0b1 << 22 }); + + /// Lossy conversion from f32, similar to @floatCast + pub fn fromF32(f: f32) Float8E8M0 { + const vf32: Float32 = @bitCast(f); + return .{ .exponent = @intCast(vf32.exponent) }; + } + + /// Lossless conversion to f32. + pub fn toF32(x: Float8E8M0) f32 { + if (x.exponent == 0) return min_scale; + const vf32: Float32 = .{ + .sign = 0, + .exponent = x.exponent, + .mantissa = 0, + }; + return @bitCast(vf32); + } + + const Helpers = FloatHelpers(@This()); + pub const formatNumber = Helpers.formatNumber; +}; + +test Float8E8M0 { + try std.testing.expectEqual(Float8E8M0{ .exponent = 127 }, Float8E8M0.fromF32(1.0)); + // try std.testing.expectEqual(5.877472e-39, Float8E8M0.toF32(.{ .exponent = 0})); + + try testCustomFloat(Float8E8M0, .{ + .lossless = &[_]f32{ Float8E8M0.min_scale, 1.0, 64.0, 1.0 / 128.0, std.math.pow(f32, 2.0, 127) }, + .lossy = &[_]f32{1.00001}, + }); +} + +pub const Float4E2M1 = packed struct(u4) { + mantissa: u1, + exponent: u2, + sign: u1, + + pub const nan: Float4E2M1 = @bitCast(@as(u4, 0xF)); + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const formatNumber = Helpers.formatNumber; + + pub const values = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; + + pub fn toF32(x: Float4E2M1) f32 { + // faster implementation + return values[@as(u4, @bitCast(x))]; + } + + test toF32 { + var to_f32_res: [16]f32 = undefined; + for (&to_f32_res, 0..) |*r, i| { + const x_f4: Float4E2M1 = @bitCast(@as(u4, @intCast(i))); + r.* = x_f4.toF32(); + } + try std.testing.expectEqualSlices(f32, &Float4E2M1.values, &to_f32_res); + } + + test fromF32 { + var from_f32_res: [16]Float4E2M1 = undefined; + for (&from_f32_res, 0..) |*r, i| { + r.* = .fromF32(Float4E2M1.values[i]); + } + try std.testing.expectEqualSlices(u4, &.{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, @ptrCast(&from_f32_res)); + } +}; + +pub fn floatCast(T: type, x: anytype) T { + return switch (T) { + f64, f32, f16 => switch (@TypeOf(x)) { + f64, f32, f16 => @floatCast(x), + else => @floatCast(x.toF32()), + }, + else => switch (@TypeOf(x)) { + f64, f32, f16 => .fromF32(x), + else => .fromF32(x.toF32()), + }, + }; +} + +pub fn isInf(x: anytype) bool { + const Float = @TypeOf(x); + switch (Float) { + f64, f32, f16 => return std.math.isInf(x), + else => {}, + } + if (!@hasDecl(Float, "inf")) return false; + + const FBits = std.meta.Int(.unsigned, @bitSizeOf(Float)); + const remove_sign = ~@as(FBits, 0) >> 1; + return @as(FBits, @bitCast(x)) & remove_sign == @as(FBits, @bitCast(Float.inf)); +} + +fn allBitsOne(v: anytype) bool { + return v == std.math.maxInt(@TypeOf(v)); +} + +const TestCase = struct { + lossless: []const f32, + lossy: []const f32, + tolerance: f32 = 1e-2, +}; + +fn testCustomFloat(FloatT: type, test_case: TestCase) !void { + for (test_case.lossless) |x| { + try std.testing.expectEqual(x, FloatT.fromF32(x).toF32()); + } + for (test_case.lossy) |x| { + try expectApproxEqRel(f32, x, FloatT.fromF32(x).toF32(), test_case.tolerance); + } +} + +fn expectApproxEqRel(FloatT: type, x: FloatT, y: FloatT, tolerance: FloatT) !void { + if (!std.math.approxEqRel(f32, x, y, tolerance)) { + std.log.err("expected ~{d}, got {d}", .{ x, y }); + return error.TestUnexpectedResult; + } +} diff --git a/zml2/main.zig b/zml2/main.zig new file mode 100644 index 00000000..66d7a6cd --- /dev/null +++ b/zml2/main.zig @@ -0,0 +1,15 @@ +const std = @import("std"); +const async = @import("async"); +const zml = @import("zml"); + +pub fn main() !void { + try async.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +pub fn asyncMain() !void { + zml.init(); + defer zml.deinit(); + + var platform = try zml.Platform.init(.cpu, .{}); + defer platform.deinit(); +} diff --git a/zml2/pjrtx.zig b/zml2/pjrtx.zig new file mode 100644 index 00000000..f09ffa1e --- /dev/null +++ b/zml2/pjrtx.zig @@ -0,0 +1,362 @@ +const std = @import("std"); + +const async = @import("async"); +const dialects = @import("mlir/dialects"); +const mlir = @import("mlir"); +const pjrt = @import("pjrt"); +pub const ffi = pjrt.ffi; +pub const Api = pjrt.Api; +pub const ApiError = pjrt.ApiError; +pub const BufferType = pjrt.BufferType; +pub const ClientInitError = pjrt.ClientInitError; +pub const CompiledMemoryStats = pjrt.CompiledMemoryStats; +pub const Device = pjrt.Device; +pub const DeviceDescription = pjrt.DeviceDescription; +pub const Error = pjrt.Error; +pub const ErrorCode = pjrt.ErrorCode; +pub const Executable = pjrt.Executable; +pub const ExecuteContext = pjrt.ExecuteContext; +pub const ExecuteError = ApiError; +pub const GetCostAnalysisError = pjrt.GetCostAnalysisError; +pub const Memory = pjrt.Memory; +pub const MemoryStats = pjrt.MemoryStats; +pub const NamedValue = pjrt.NamedValue; +pub const Profiler = pjrt.Profiler; +pub const SerializeResult = pjrt.SerializeResult; +pub const Stream = pjrt.Stream; + +const zml = struct { + pub const DataType = @import("dtype.zig").DataType; +}; + +const log = std.log.scoped(.zml); + +pub const CompileError = std.mem.Allocator.Error || error{InvalidMlirBytecodeVersion} || ApiError; + +fn InnerMixin(comptime innerT: type) type { + return struct { + inline fn inner(self: anytype) if (@typeInfo(@TypeOf(self)).pointer.is_const) *const innerT else *innerT { + return @ptrCast(self); + } + }; +} + +pub const Client = opaque { + const inner = InnerMixin(pjrt.Client).inner; + + pub fn init(api: *const Api, options: []const NamedValue) ClientInitError!*Client { + return @ptrCast(try pjrt.Client.init(api, options)); + } + + pub fn deinit(self: *Client, api: *const Api) void { + self.inner().deinit(api); + } + + pub fn getPlatformName(self: *const Client, api: *const Api) []const u8 { + return self.inner().getPlatformName(api); + } + + pub fn getDevices(self: *const Client, api: *const Api) []const *const Device { + return self.inner().getDevices(api); + } + + pub fn getAddressableDevices(self: *const Client, api: *const Api) []const *const Device { + return self.inner().getAddressableDevices(api); + } + + pub const BufferFromHostBufferArgs = pjrt.Client.BufferFromHostBufferArgs; + pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) ApiError!struct { *Buffer, ?*Event } { + const buffer, const event_ = try self.inner().bufferFromHostBuffer(api, args); + return .{ @ptrCast(buffer), @ptrCast(event_) }; + } + + pub fn deserializeAndLoad(self: *const Client, api: *const Api, bytes: []const u8) ApiError!*LoadedExecutable { + return @ptrCast(try async.callBlocking(pjrt.Client.deserializeAndLoad, .{ self.inner(), api, bytes })); + } + + pub const CreateViewOfDeviceBufferArgs = pjrt.Client.CreateViewOfDeviceBufferArgs; + pub fn createViewOfDeviceBuffer(self: *const Client, api: *const Api, args: CreateViewOfDeviceBufferArgs) ApiError!*Buffer { + return @ptrCast(try self.inner().createViewOfDeviceBuffer(api, args)); + } + + fn compileSync(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { + var bytecode: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); + defer bytecode.deinit(); + module.op().writeBytecodeWithConfig(&bytecode.writer, .{ .desiredEmitedVersion = 1 }) catch |err| { + log.err("failed to write module bytecode: {}", .{err}); + return switch (err) { + std.Io.Writer.Error.WriteFailed => error.OutOfMemory, + else => |e| e, + }; + }; + + var serialized_buffer: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); + defer serialized_buffer.deinit(); + + const stablehlo_version = blk: { + if (api.stablehloCurrentVersion()) |requested_version| { + break :blk dialects.stablehlo.stablehloGetSmallerVersion(requested_version, dialects.stablehlo.getCurrentVersion()); + } + break :blk dialects.stablehlo.getMinimumVersion(); + }; + + dialects.stablehlo.serializePortableArtifact(bytecode.written(), stablehlo_version, &serialized_buffer.writer) catch |err| { + log.err("failed to serialize to portable artifact: {}", .{err}); + return switch (err) { + std.Io.Writer.Error.WriteFailed => error.OutOfMemory, + else => |e| e, + }; + }; + + return @ptrCast(try self.inner().compile(api, .{ + .bytecode = serialized_buffer.written(), + .bytecode_format = .mlir, + .compile_options_pb = compile_options_pb, + })); + } + + pub fn compile(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { + return try async.callBlocking(compileSync, .{ self, api, allocator, module, compile_options_pb }); + } + + pub fn addressableMemories(self: *const Client, api: *const Api) []*const Memory { + return self.inner().addressableMemories(api); + } + + pub fn memoryByKind(self: *const Client, api: *const Api, kind: Memory.Kind) ?*const Memory { + for (self.addressableMemories(api)) |mem| { + if (mem.kind(api) == kind) { + return mem; + } + } + return null; + } + + pub const CreateUninitializedBufferArgs = pjrt.Client.CreateUninitializedBufferArgs; + + pub fn createUnitializedBuffer(self: *const Client, api: *const Api, args: CreateUninitializedBufferArgs) ApiError!*Buffer { + return @ptrCast(try self.inner().createUninitializedBuffer(api, args)); + } +}; + +pub const Buffer = opaque { + pub const inner = InnerMixin(pjrt.Buffer).inner; + + pub fn deinit(self: *Buffer, api: *const Api) void { + self.inner().deinit(api); + } + + pub fn getDevice(self: *const Buffer, api: *const Api) ApiError!*Device { + return try self.inner().getDevice(api); + } + + pub fn delete(self: *Buffer, api: *const Api) void { + self.inner().delete(api); + } + + pub fn isDeleted(self: *const Buffer, api: *const Api) bool { + return self.inner().isDeleted(api); + } + + pub fn isOnCpu(self: *const Buffer, api: *const Api) bool { + return self.inner().isOnCpu(api); + } + + pub fn memory(self: *const Buffer, api: *const Api) *const Memory { + return self.inner().memory(api); + } + + pub fn toHostBuffer(self: *const Buffer, api: *const Api, dst: []u8) ApiError!?*Event { + return @ptrCast(try self.inner().toHostBuffer(api, dst)); + } + + pub fn getElementType(self: *const Buffer, api: *const Api) zml.DataType { + return dtypeFromBufferType(self.inner().getElementType(api)); + } + + pub fn getDimensions(self: *const Buffer, api: *const Api) []const i64 { + return self.inner().getDimensions(api); + } + + pub fn getUnpaddedDimensions(self: *const Buffer, api: *const Api) ApiError![]const i64 { + return try self.inner().getUnpaddedDimensions(api); + } + + pub fn getOnDeviceSizeInBytes(self: *const Buffer, api: *const Api) ApiError!usize { + return try self.inner().getOnDeviceSizeInBytes(api); + } + + pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!*Buffer { + return @ptrCast(self.inner().copyToDevice(api, device)); + } + + pub fn copyToMemory(self: *const Buffer, api: *const Api, memory_: *const Memory) ApiError!*Buffer { + return @ptrCast(try self.inner().copyToMemory(api, memory_)); + } + + pub fn getReadyEvent(self: *const Buffer, api: *const Api) ?*Event { + return @ptrCast(self.inner().getReadyEvent(api)); + } + + pub fn getOpaqueDeviceMemoryDataPointer(self: *const Buffer, api: *const Api) ApiError!*anyopaque { + return try self.inner().getOpaqueDeviceMemoryDataPointer(api); + } +}; + +pub const Event = opaque { + pub const inner = InnerMixin(pjrt.Event).inner; + + pub fn deinit(self: *Event, api: *const Api) void { + self.inner().deinit(api); + } + + pub fn isReady(self: *const Event, api: *const Api) bool { + return self.inner().isReady(api); + } + + pub fn getEventError(self: *const Event, api: *const Api) ?*Error { + return self.inner().getEventError(api); + } + + pub fn awaitBlocking(self: *Event, api: *const Api) ApiError!void { + if (self.isReady(api)) { + return; + } + try self.inner().await(api); + } + + pub fn await(self: *Event, api: *const Api) ApiError!void { + defer self.deinit(api); + + if (self.isReady(api)) { + return; + } + + var ctx = struct { + err: ?*pjrt.Error = null, + event: async.threading.ResetEventSingle = .{}, + }{}; + + try self.inner().onReady(api, &(struct { + fn call(err: ?*pjrt.Error, user_arg: ?*anyopaque) callconv(.c) void { + const ctx_: *@TypeOf(ctx) = @ptrCast(@alignCast(user_arg.?)); + ctx_.err = err; + ctx_.event.set(); + } + }.call), &ctx); + ctx.event.wait(); + + if (ctx.err) |e| { + defer e.deinit(api); + const err_code = e.getCode(api).toApiError(); + log.err("{t} {s}", .{ err_code, e.getMessage(api) }); + return err_code; + } + } +}; + +pub const LoadedExecutable = opaque { + const inner = InnerMixin(pjrt.LoadedExecutable).inner; + + pub fn deinit(self: *LoadedExecutable, api: *const Api) void { + self.inner().deinit(api); + } + + pub fn delete(self: *LoadedExecutable, api: *const Api) void { + self.inner().delete(api); + } + + pub fn isDeleted(self: *const LoadedExecutable, api: *const Api) bool { + return self.inner().isDeleted(api); + } + + pub fn getAddressableDevices(self: *const LoadedExecutable, api: *const Api) []*const Device { + return self.inner().getAddressableDevices(api); + } + + pub const ExecuteArgs = struct { + arguments: []const [*]const *const Buffer, + num_args: usize, + results: []const [*]*Buffer, + events: []?*Event, + non_donatable_input_indices: []const i64 = &.{}, + context: ?*ExecuteContext, + }; + + pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { + try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ + .num_args = args.num_args, + .arguments = @ptrCast(args.arguments), + .results = @ptrCast(args.results), + .events = @ptrCast(args.events), + .non_donatable_input_indices = args.non_donatable_input_indices, + .context = args.context, + }); + } + + pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { + return try self.inner().getExecutable(api); + } +}; + +pub const AsyncHostToDeviceTransferManager = opaque { + const inner = InnerMixin(pjrt.AsyncHostToDeviceTransferManager).inner; + + pub fn deinit(self: *AsyncHostToDeviceTransferManager, api: *const Api) void { + self.inner().deinit(api); + } + + pub fn transferData(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize, data: []const u8, offset: i64, is_last_transfer: bool) ApiError!*Event { + return @ptrCast(try self.inner().transferData(api, buffer_index, data, offset, is_last_transfer)); + } + + pub fn retrieveBuffer(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize) ApiError!*Buffer { + return @ptrCast(try self.inner().retrieveBuffer(api, buffer_index)); + } + + pub fn device(self: *AsyncHostToDeviceTransferManager, api: *const Api) *Device { + return @ptrCast(self.inner().device(api)); + } + + pub fn bufferCount(self: *AsyncHostToDeviceTransferManager, api: *const Api) usize { + return self.inner().bufferCount(api); + } + + pub fn bufferSize(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize) usize { + return self.inner().bufferSize(api, buffer_index); + } + + pub fn setBufferError(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize, error_code: ErrorCode, error_message: []const u8) void { + self.inner().setBufferError(api, buffer_index, error_code, error_message); + } + + pub fn addMetadata(self: *AsyncHostToDeviceTransferManager, api: *const Api, transfer_metadata: []const NamedValue) void { + return self.inner().addMetadata(api, transfer_metadata); + } +}; + +pub fn bufferTypeFromDtype(dt: zml.DataType) pjrt.BufferType { + return switch (dt) { + inline else => |tag| @field(pjrt.BufferType, @tagName(tag)), + }; +} + +pub fn dtypeFromBufferType(pjrt_type: pjrt.BufferType) zml.DataType { + return switch (pjrt_type) { + .invalid => @panic("Found an invalid pjrt buffer"), + inline else => |tag| @field(zml.DataType, @tagName(tag)), + }; +} + +test bufferTypeFromDtype { + inline for (@typeInfo(zml.DataType).@"enum".fields) |field| { + const dt: zml.DataType = @enumFromInt(field.value); + try std.testing.expectEqual(dt, dtypeFromBufferType(bufferTypeFromDtype(dt))); + } + + inline for (@typeInfo(pjrt.BufferType).@"enum".fields) |field| { + const dt: pjrt.BufferType = @enumFromInt(field.value); + if (dt == .invalid) continue; + try std.testing.expectEqual(dt, bufferTypeFromDtype(dtypeFromBufferType(dt))); + } +} diff --git a/zml2/platform.zig b/zml2/platform.zig new file mode 100644 index 00000000..d942290b --- /dev/null +++ b/zml2/platform.zig @@ -0,0 +1,190 @@ +const std = @import("std"); + +const runtimes = @import("runtimes"); +pub const Target = runtimes.Platform; +const stdx = @import("stdx"); + +const pjrt = @import("pjrtx.zig"); + +const log = std.log.scoped(.zml); + +pub const CompilationOptions = struct { + xla_dump_to: ?[]const u8 = null, + xla_dump_fusion_visualization: bool = false, + xla_dump_hlo_pass_re: ?[]const u8 = null, + sharding_enabled: bool = false, + sharding_axes: stdx.BoundedArray([*:0]const u8, 8) = .{}, + device_memory_size: u64 = 0, +}; + +pub const Platform = struct { + target: Target, + pjrt_api: *const pjrt.Api, + pjrt_client: *pjrt.Client, + + // This make the pjrt struct quite fat, but is only used during compilation. + // TODO: Reconsider having it here, and maybe pass explicitly to compile, + // or create an intermediary struct: + // `const comp = platform.compiler(compile_opts); const exe = comp.compile(...);` + compilation_options: CompilationOptions = .{}, + + pub const MAX_NUM_DEVICES: u8 = if (runtimes.isEnabled(.tpu)) 32 else 8; + + pub fn init(target: Target, options: CreateOptions) !Platform { + const api = try runtimes.load(target); + + var named_values_buf: [16]pjrt.NamedValue = undefined; + const pjrt_client = try pjrt.Client.init(api, options.toNamedValues(target, &named_values_buf)); + const true_num_devices = pjrt_client.getAddressableDevices(api).len; + if (true_num_devices > MAX_NUM_DEVICES) { + log.warn("platform {} got {} devices, but ZML only support up to {} devices. Some devices won't be used.", .{ target, true_num_devices, MAX_NUM_DEVICES }); + } + return .{ + .target = target, + .pjrt_api = api, + .pjrt_client = pjrt_client, + .compilation_options = .{}, + }; + } + + pub fn getDevices(self: Platform) []const *const pjrt.Device { + const all_devices = self.pjrt_client.getAddressableDevices(self.pjrt_api); + if (all_devices.len > MAX_NUM_DEVICES) { + return all_devices[0..MAX_NUM_DEVICES]; + } + return all_devices; + } + + pub fn withCompilationOptions(self: Platform, opts: CompilationOptions) Platform { + var res = self; + res.compilation_options = opts; + return res; + } + + pub fn deinit(self: *Platform) void { + self.pjrt_client.deinit(self.pjrt_api); + } + + pub fn memoryForDevice(platform: Platform, memory: pjrt.Memory.Kind, device: *const pjrt.Device) *const pjrt.Memory { + const memory_target: pjrt.Memory.Kind = switch (memory) { + .host_unpinned => switch (platform.target) { + // Cuda doesn't have host_unpinned. + // ROCm doesn't seem to have it either. + // TODO(gwenzek): investigate why it was not forced before. + .cuda, .rocm => .host_pinned, + else => .host_unpinned, + }, + inline else => |t| t, + }; + // TODO measure the cost of this and consider caching. + const device_memories = device.addressableMemories(platform.pjrt_api); + for (device_memories) |m| { + if (memory_target == m.kind(platform.pjrt_api)) { + return m; + } + } + log.err("Platform {t} doesn't have memory {t}", .{ platform.target, memory }); + @panic("Memory kind not found"); + } + + test memoryForDevice { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const memory_fields = @typeInfo(pjrt.Memory.Kind).@"enum".fields; + inline for (memory_fields) |field| { + for (platform.getDevices()) |dev| { + _ = platform.memoryForDevice(@field(pjrt.Memory.Kind, field.name), dev); + } + } + } + + pub fn memoryStats(platform: Platform, device_id: usize) pjrt.MemoryStats { + if (platform.target == .cpu) return .zeroes; + + const device = platform.getDevices()[device_id]; + return device.memoryStats(platform.pjrt_api) catch .zeroes; + } +}; + +pub const CreateOptions = struct { + cpu: Cpu = .{ .device_count = 4 }, + + // bump memory fraction from XLA defaults of 75% to 90%. + // Even on a 8GB GPU it should leave enough space for the Cuda driver + // https://github.com/openxla/xla/blob/3e87afa11a865cf91137522492918ad18bfe5b7c/xla/pjrt/plugin/xla_gpu/xla_gpu_allocator_config.h#L25-L60 + cuda: Cuda = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.90 } } }, + rocm: struct {} = .{}, + tpu: struct {} = .{}, + neuron: struct {} = .{}, + + pub const Cpu = struct { + device_count: u32, + + fn writeNamedValues(self: Cpu, values: *std.ArrayList(pjrt.NamedValue)) void { + values.appendAssumeCapacity(pjrt.NamedValue.from("cpu_device_count", @as(i64, self.device_count))); + } + }; + + pub const Cuda = struct { + allocator: Allocator = .{ .bfc = .{} }, + // TODO support all of https://github.com/openxla/xla/blob/3d31c48c719d331d432132b3e0c2c5ce52650675/xla/pjrt/c/pjrt_c_api_gpu_internal.cc#L76-L86 + // visible_devices: []const i64 = &.{}, + // node_id + // num_nodes + // enable_mock_nccl + // mock_gpu_topology + + pub const Allocator = union(enum) { + /// "Best-Fit with Coalescing" algorithm + bfc: Options, + /// use cudaMallocAsync + async: Options, + /// use raw cuMalloc + platform, + + pub const Options = struct { + preallocate: bool = true, + memory_fraction: f32 = 0.90, + collective_memory_size_mb: u32 = 0, + }; + }; + + fn writeNamedValues(self: Cuda, values: *std.ArrayList(pjrt.NamedValue)) void { + switch (self.allocator) { + .platform => { + values.appendAssumeCapacity(.fromString("allocator", "platform")); + }, + .bfc, .async => |opt| { + values.appendAssumeCapacity(.fromString("allocator", switch (self.allocator) { + .bfc => "bfc", + .async => "cuda_async", + .platform => unreachable, + })); + values.appendAssumeCapacity(.from("preallocate", opt.preallocate)); + if (opt.memory_fraction > 0) { + values.appendAssumeCapacity(.from("memory_fraction", opt.memory_fraction)); + } + if (opt.collective_memory_size_mb > 0) { + const collective = @as(i64, opt.collective_memory_size_mb) * 1024 * 1024; + values.appendAssumeCapacity(.from("collective_memory_size", collective)); + } + }, + } + } + }; + + pub fn toNamedValues(self: CreateOptions, target: Target, out: []pjrt.NamedValue) []pjrt.NamedValue { + var values = std.ArrayList(pjrt.NamedValue).fromOwnedSlice(out); + values.shrinkRetainingCapacity(0); + switch (target) { + .cpu => self.cpu.writeNamedValues(&values), + .cuda => self.cuda.writeNamedValues(&values), + inline else => |t| { + stdx.debug.assertComptime(@hasField(CreateOptions, @tagName(t)), "zml.platform.CreateOptions doesn't list target {s}", .{@tagName(t)}); + const options = @field(self, @tagName(t)); + stdx.debug.assertComptime(@sizeOf(@TypeOf(options)) == 0, "zml.platform.CreateOptions.{s} is discarded", .{@tagName(t)}); + }, + } + return values.items; + } +}; diff --git a/zml2/posix.h b/zml2/posix.h new file mode 100644 index 00000000..0d66200c --- /dev/null +++ b/zml2/posix.h @@ -0,0 +1,2 @@ +#include +#include diff --git a/zml2/zml.zig b/zml2/zml.zig new file mode 100644 index 00000000..b1cc842c --- /dev/null +++ b/zml2/zml.zig @@ -0,0 +1,61 @@ +//! Welcome to the ZML API documentation! +//! ZML provides tools to write high level code describing a neural network, +//! compiling it for various accelerators and targets, and executing it. +//! + +// Namespaces +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const runfiles = @import("runfiles"); + +pub const platform = @import("platform.zig"); +pub const Platform = platform.Platform; +pub const Target = platform.Target; +pub const CompilationOptions = platform.CompilationOptions; +pub const floats = @import("floats.zig"); +const dtype = @import("dtype.zig"); +pub const Data = dtype.Data; +pub const DataType = dtype.DataType; +pub const pjrt = @import("pjrtx.zig"); + +var mlir_once = std.once(struct { + fn call() void { + mlir.registerPasses("Transforms"); + } +}.call); + +var runfiles_once = std.once(struct { + fn call_() !void { + if (std.process.hasEnvVarConstant("RUNFILES_MANIFEST_FILE") or std.process.hasEnvVarConstant("RUNFILES_DIR")) { + return; + } + + var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); + const allocator = arena.allocator(); + defer arena.deinit(); + + var envMap = std.process.EnvMap.init(allocator); + var r = (try runfiles.Runfiles.create(.{ .allocator = allocator })) orelse return; + try r.environment(&envMap); + + var it = envMap.iterator(); + while (it.next()) |entry| { + const keyZ = try allocator.dupeZ(u8, entry.key_ptr.*); + const valueZ = try allocator.dupeZ(u8, entry.value_ptr.*); + _ = c.setenv(keyZ.ptr, valueZ.ptr, 1); + } + } + + fn call() void { + call_() catch @panic("Unable to init runfiles env"); + } +}.call); + +pub fn init() void { + runfiles_once.call(); + mlir_once.call(); +} + +pub fn deinit() void {} From c182dda45707c09a0180e01d45ac75750723d143 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 1 Dec 2025 15:51:38 +0100 Subject: [PATCH 071/881] mlir and dialects modules rework * Complete rework of mlir that uses pointers instead of values for basic types. * Removed unused dialects. --- BUILD.bazel | 4 +- mlir/dialects/BUILD.bazel | 4 - mlir/dialects/arith.zig | 110 - mlir/dialects/dialects.zig | 6 +- mlir/dialects/math.zig | 36 - mlir/dialects/scf.zig | 61 - mlir/dialects/stablehlo/stablehlo.zig | 1578 +++++++------- mlir/dialects/tensor.zig | 36 - mlir/mlir.zig | 2880 ++++++++++++------------- 9 files changed, 2180 insertions(+), 2535 deletions(-) delete mode 100644 mlir/dialects/arith.zig delete mode 100644 mlir/dialects/math.zig delete mode 100644 mlir/dialects/scf.zig delete mode 100644 mlir/dialects/tensor.zig mode change 100755 => 100644 mlir/mlir.zig diff --git a/BUILD.bazel b/BUILD.bazel index 72cbb96b..34e18158 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -7,9 +7,9 @@ zls_completion( visibility = ["//visibility:public"], deps = [ "//async", - "//examples/llama", + #"//examples/llama", "//stdx", - "//zml", + "//zml2:zml", ], ) diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/BUILD.bazel index 101f67ff..26cca662 100644 --- a/mlir/dialects/BUILD.bazel +++ b/mlir/dialects/BUILD.bazel @@ -3,11 +3,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "dialects", srcs = [ - "arith.zig", "func.zig", - "math.zig", - "scf.zig", - "tensor.zig", ], import_name = "mlir/dialects", main = "dialects.zig", diff --git a/mlir/dialects/arith.zig b/mlir/dialects/arith.zig deleted file mode 100644 index 39cbc2a1..00000000 --- a/mlir/dialects/arith.zig +++ /dev/null @@ -1,110 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); - -pub fn constant(ctx: mlir.Context, value: mlir.Attribute, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "arith.constant", .{ - .attributes = &.{.{ "value", value }}, - .result_type_inference = true, - .location = location, - }); -} - -fn binary_fn(comptime op_name: [:0]const u8) fn (mlir.Context, mlir.Value, mlir.Value, mlir.Location) mlir.Operation { - return struct { - pub fn call(ctx: mlir.Context, lhs: mlir.Value, rhs: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, op_name, .{ - .operands = &.{ lhs, rhs }, - .result_type_inference = true, - .location = location, - }); - } - }.call; -} - -fn cast_fn(comptime op_name: [:0]const u8) fn (mlir.Context, mlir.Value, mlir.Type, mlir.Location) mlir.Operation { - return struct { - pub fn call(ctx: mlir.Context, value: mlir.Value, new_type: mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, op_name, .{ - .operands = &.{value}, - .results = &.{new_type}, - .location = location, - }); - } - }.call; -} - -pub const addi = binary_fn("arith.addi"); -pub const addf = binary_fn("arith.addf"); -pub const subi = binary_fn("arith.subi"); -pub const subf = binary_fn("arith.subf"); -pub const muli = binary_fn("arith.muli"); -pub const mulf = binary_fn("arith.mulf"); -pub const divsi = binary_fn("arith.divsi"); -pub const divui = binary_fn("arith.divui"); -pub const divf = binary_fn("arith.divf"); -pub const maxnumf = binary_fn("arith.maxnumf"); -pub const maxnumi = binary_fn("arith.maxnumi"); -pub const extsi = cast_fn("arith.extsi"); -pub const extui = cast_fn("arith.extui"); -pub const extf = cast_fn("arith.extf"); -pub const trunci = cast_fn("arith.trunci"); -pub const truncf = cast_fn("arith.truncf"); -pub const fptosi = cast_fn("arith.fptosi"); -pub const fptoui = cast_fn("arith.fptoui"); -pub const sitofp = cast_fn("arith.sitofp"); -pub const uitofp = cast_fn("arith.uitofp"); - -pub const CmpIPredicate = enum { - eq, - ne, - slt, - sle, - sgt, - sge, - ult, - ule, - ugt, - uge, -}; - -pub fn cmpi(ctx: mlir.Context, predicate: CmpIPredicate, lhs: mlir.Value, rhs: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "arith.cmpi", .{ - .operands = &.{ lhs, rhs }, - .result_type_inference = true, - .attributes = &.{ - .{ "predicate", .int(ctx, .i64, @intFromEnum(predicate)) }, - }, - .location = location, - }); -} - -pub const CmpFPredicate = enum { - false, - oeq, - ogt, - oge, - olt, - ole, - one, - ord, - ueq, - ugt, - uge, - ult, - ule, - une, - uno, - true, -}; - -pub fn cmpf(ctx: mlir.Context, predicate: CmpFPredicate, lhs: mlir.Value, rhs: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "arith.cmpf", .{ - .operands = &.{ lhs, rhs }, - .result_type_inference = true, - .attributes = &.{ - .{ "predicate", .int(ctx, .i64, @intFromEnum(predicate)) }, - }, - .location = location, - }); -} diff --git a/mlir/dialects/dialects.zig b/mlir/dialects/dialects.zig index 47f74d8e..3deac104 100644 --- a/mlir/dialects/dialects.zig +++ b/mlir/dialects/dialects.zig @@ -1,11 +1,7 @@ const std = @import("std"); -pub const arith = @import("arith.zig"); -pub const func = @import("func.zig"); -pub const math = @import("math.zig"); -pub const scf = @import("scf.zig"); -pub const tensor = @import("tensor.zig"); pub const stablehlo = @import("mlir/dialects/stablehlo"); +pub const func = @import("func.zig"); test { std.testing.refAllDecls(@This()); diff --git a/mlir/dialects/math.zig b/mlir/dialects/math.zig deleted file mode 100644 index 62ef622c..00000000 --- a/mlir/dialects/math.zig +++ /dev/null @@ -1,36 +0,0 @@ -const std = @import("std"); -const mlir = @import("mlir"); - -const namespace = "math"; - -fn unary_fn(comptime op_name: [:0]const u8) type { - return struct { - pub fn call(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, namespace ++ "." ++ op_name, .{ - .operands = &.{value}, - .results = &.{value.getType()}, - .location = location, - }); - } - }; -} - -fn binary_fn(comptime op_name: [:0]const u8) type { - return struct { - pub fn call(ctx: mlir.Context, lhs: mlir.Value, rhs: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, namespace ++ "." ++ op_name, .{ - .operands = &.{ lhs, rhs }, - .results = &.{}, - .location = location, - }); - } - }; -} - -pub const ipowi = binary_fn("ipowi").call; -pub const fpowi = binary_fn("fpowi").call; -pub const tanh = unary_fn("tanh").call; -pub const sqrt = unary_fn("sqrt").call; -pub const exp = unary_fn("exp").call; -pub const exp2 = unary_fn("exp2").call; -pub const log = unary_fn("log").call; diff --git a/mlir/dialects/scf.zig b/mlir/dialects/scf.zig deleted file mode 100644 index 61fcb189..00000000 --- a/mlir/dialects/scf.zig +++ /dev/null @@ -1,61 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); - -pub fn ForBody(ExtraArgs: type) type { - return fn (mlir.Context, mlir.Block, ExtraArgs) mlir.Operation; -} - -pub const ForRange = struct { - start: mlir.Value, - end: mlir.Value, - step: mlir.Value, -}; - -pub fn @"for"( - ExtraArgs: type, - ctx: mlir.Context, - range: ForRange, - init_values: []const mlir.Value, - body: ForBody(ExtraArgs), - extra_args: ExtraArgs, - loc: mlir.Location, -) mlir.Operation { - const n_args = init_values.len; - var init_types_buf: [32]mlir.Type = undefined; - var locs_buf: [32]mlir.Location = undefined; - - // The first block argument is the for loop induction variable, - // followed then by all the loop-carried variables. - const init_types = init_types_buf[0 .. n_args + 1]; - const locs = locs_buf[0 .. n_args + 1]; - init_types[0] = range.start.getType(); - locs[0] = loc; - for (1.., init_values) |i, val| { - init_types[i] = val.getType(); - locs[i] = loc; - } - - const block = mlir.Block.init(init_types, locs) catch unreachable; - const yield_op = @call(.auto, body, .{ ctx, block, extra_args }); - std.debug.assert(std.mem.eql(u8, "scf.yield", yield_op.name().str())); - block.appendOperationRecursive(yield_op, .open); - - const for_op = mlir.Operation.make(ctx, "scf.for", .{ - .variadic_operands = &.{ &.{ range.start, range.end, range.step }, init_values }, - .results = init_types[1..], - .blocks = &.{block}, - .location = loc, - .verify = false, - }); - return for_op; -} - -pub fn yield(ctx: mlir.Context, res: []const mlir.Value, loc: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "scf.yield", .{ - .variadic_operands = &.{res}, - .results = &.{}, - .location = loc, - .verify = false, - }); -} diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 7c7c1394..b9165367 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -2,112 +2,106 @@ const std = @import("std"); const c = @import("c"); const mlir = @import("mlir"); +const stringRef = mlir.stringRef; const stdx = @import("stdx"); -pub const abs = functors.unary_fn("stablehlo.abs").call; -pub const cosine = functors.unary_fn("stablehlo.cosine").call; -pub const sine = functors.unary_fn("stablehlo.sine").call; -pub const exponential = functors.unary_fn("stablehlo.exponential").call; -pub const exponential_minus_one = functors.unary_fn("stablehlo.exponential_minus_one").call; -pub const floor = functors.unary_fn("stablehlo.floor").call; -pub const log = functors.unary_fn("stablehlo.log").call; -pub const log_plus_one = functors.unary_fn("stablehlo.log_plus_one").call; -pub const not = functors.unary_fn("stablehlo.not").call; -pub const negate = functors.unary_fn("stablehlo.negate").call; -pub const sqrt = functors.unary_fn("stablehlo.sqrt").call; -pub const tanh = functors.unary_fn("stablehlo.tanh").call; -pub const cbrt = functors.unary_fn("stablehlo.cbrt").call; -pub const ceil = functors.unary_fn("stablehlo.ceil").call; -pub const rsqrt = functors.unary_fn("stablehlo.rsqrt").call; -pub const count_leading_zeros = functors.unary_fn("stablehlo.count_leading_zeros").call; -pub const is_finite = functors.unary_fn("stablehlo.is_finite").call; -pub const logistic = functors.unary_fn("stablehlo.logistic").call; -pub const popcnt = functors.unary_fn("stablehlo.popcnt").call; -pub const sign = functors.unary_fn("stablehlo.sign").call; -pub const real = functors.unary_fn("stablehlo.real").call; -pub const imag = functors.unary_fn("stablehlo.imag").call; +pub const abs = unaryOp("stablehlo.abs").call; +pub const cosine = unaryOp("stablehlo.cosine").call; +pub const sine = unaryOp("stablehlo.sine").call; +pub const exponential = unaryOp("stablehlo.exponential").call; +pub const exponential_minus_one = unaryOp("stablehlo.exponential_minus_one").call; +pub const floor = unaryOp("stablehlo.floor").call; +pub const log = unaryOp("stablehlo.log").call; +pub const log_plus_one = unaryOp("stablehlo.log_plus_one").call; +pub const not = unaryOp("stablehlo.not").call; +pub const negate = unaryOp("stablehlo.negate").call; +pub const sqrt = unaryOp("stablehlo.sqrt").call; +pub const tanh = unaryOp("stablehlo.tanh").call; +pub const cbrt = unaryOp("stablehlo.cbrt").call; +pub const ceil = unaryOp("stablehlo.ceil").call; +pub const rsqrt = unaryOp("stablehlo.rsqrt").call; +pub const count_leading_zeros = unaryOp("stablehlo.count_leading_zeros").call; +pub const is_finite = unaryOp("stablehlo.is_finite").call; +pub const logistic = unaryOp("stablehlo.logistic").call; +pub const popcnt = unaryOp("stablehlo.popcnt").call; +pub const sign = unaryOp("stablehlo.sign").call; +pub const real = unaryOp("stablehlo.real").call; +pub const imag = unaryOp("stablehlo.imag").call; +pub const round_nearest_afz = unaryOp("stablehlo.round_nearest_afz").call; +pub const round_nearest_even = unaryOp("stablehlo.round_nearest_even").call; +pub const tuple = unaryOp("stablehlo.tuple").call; -pub const add = functors.binary_fn("stablehlo.add").call; -pub const multiply = functors.binary_fn("stablehlo.multiply").call; -pub const divide = functors.binary_fn("stablehlo.divide").call; -pub const subtract = functors.binary_fn("stablehlo.subtract").call; -pub const or_ = functors.binary_fn("stablehlo.or").call; -pub const xor = functors.binary_fn("stablehlo.xor").call; -pub const and_ = functors.binary_fn("stablehlo.and").call; -pub const atan2 = functors.binary_fn("stablehlo.atan2").call; -pub const maximum = functors.binary_fn("stablehlo.maximum").call; -pub const minimum = functors.binary_fn("stablehlo.minimum").call; -pub const power = functors.binary_fn("stablehlo.power").call; -pub const remainder = functors.binary_fn("stablehlo.remainder").call; -pub const shift_left = functors.binary_fn("stablehlo.shift_left").call; -pub const shift_right_arithmetic = functors.binary_fn("stablehlo.shift_right_arithmetic").call; -pub const shift_right_logical = functors.binary_fn("stablehlo.shift_right_logical").call; -pub const complex = functors.binary_fn("stablehlo.complex").call; +pub const add = binaryOp("stablehlo.add").call; +pub const multiply = binaryOp("stablehlo.multiply").call; +pub const divide = binaryOp("stablehlo.divide").call; +pub const subtract = binaryOp("stablehlo.subtract").call; +pub const or_ = binaryOp("stablehlo.or").call; +pub const xor = binaryOp("stablehlo.xor").call; +pub const and_ = binaryOp("stablehlo.and").call; +pub const atan2 = binaryOp("stablehlo.atan2").call; +pub const maximum = binaryOp("stablehlo.maximum").call; +pub const minimum = binaryOp("stablehlo.minimum").call; +pub const power = binaryOp("stablehlo.power").call; +pub const remainder = binaryOp("stablehlo.remainder").call; +pub const shift_left = binaryOp("stablehlo.shift_left").call; +pub const shift_right_arithmetic = binaryOp("stablehlo.shift_right_arithmetic").call; +pub const shift_right_logical = binaryOp("stablehlo.shift_right_logical").call; +pub const complex = binaryOp("stablehlo.complex").call; -const functors = struct { - fn unary_fn(comptime op_name: [:0]const u8) type { - return struct { - pub fn call(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, op_name, .{ - .operands = &.{value}, - .result_type_inference = true, - .location = location, - }); - } - }; - } +pub const bitcast_convert = castOp("stablehlo.bitcast_convert").call; +pub const convert = castOp("stablehlo.convert").call; +pub const reshape = castOp("stablehlo.reshape").call; - pub fn binary_fn(comptime op_name: [:0]const u8) type { - return struct { - pub fn call(ctx: mlir.Context, lhs: mlir.Value, rhs: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, op_name, .{ - .operands = &.{ lhs, rhs }, - .result_type_inference = true, - .location = location, - }); - } - }; - } -}; - -pub fn return_(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.return", .{ - .variadic_operands = &.{&.{value}}, - .verify = false, - .location = location, - }); +fn castOp(comptime op_name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, value: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{value} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); + } + }; } -pub fn returns_(ctx: mlir.Context, values: []const mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.return", .{ - .variadic_operands = &.{values}, - .verify = false, - .location = location, - }); +fn unaryOp(comptime op_name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{value} }, + .result_type_inference = true, + .location = location, + }); + } + }; } -pub fn bitcast_convert(ctx: mlir.Context, value: mlir.Value, result_type: mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.bitcast_convert", .{ - .operands = &.{value}, - .results = &.{result_type}, - .location = location, - }); +pub fn binaryOp(comptime op_name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); + } + }; } -pub fn cholesky(ctx: mlir.Context, value: mlir.Value, lower: bool, location: mlir.Location) mlir.Operation { +pub fn cholesky(ctx: *mlir.Context, value: *const mlir.Value, lower: bool, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.cholesky", .{ - .operands = &.{value}, + .operands = .{ .flat = &.{value} }, .result_type_inference = true, .attributes = &.{ - .{ "lower", .i1FromBool(ctx, lower) }, + .named(ctx, "lower", mlir.integerAttribute(ctx, .i1, @intFromBool(lower))), }, .location = location, }); } -pub fn clamp(ctx: mlir.Context, min: mlir.Value, value: mlir.Value, max: mlir.Value, location: mlir.Location) mlir.Operation { +pub fn clamp(ctx: *mlir.Context, min: *const mlir.Value, value: *const mlir.Value, max: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.clamp", .{ - .operands = &.{ min, value, max }, + .operands = .{ .flat = &.{ min, value, max } }, .result_type_inference = true, .location = location, }); @@ -119,18 +113,17 @@ pub const DotPrecision = union(enum) { highest, algorithm: DotAlgorithm, - pub fn precisionAttr(self: DotPrecision, ctx: mlir.Context) mlir.Attribute { - const precision = PrecisionAttribute.init(ctx, switch (self) { + pub fn precision(self: DotPrecision) PrecisionAttribute.Precision { + return switch (self) { .fast => .DEFAULT, .high => .HIGH, .highest => .HIGHEST, // When we specify the dot algorithm, we should not specify the precision. .algorithm => .DEFAULT, - }); - return precision.asAttr(); + }; } - pub fn algorithmAttr(self: DotPrecision, ctx: mlir.Context, operand_type: mlir.RankedTensorType) ?mlir.Attribute { + pub fn algorithmAttr(self: DotPrecision, ctx: *mlir.Context, operand_type: mlir.RankedTensorType) ?mlir.Attribute { return switch (self) { .algorithm => |algo| algo.asAttr(ctx, operand_type), else => null, @@ -149,159 +142,172 @@ pub const DotAlgorithm = struct { // bf16_6x: each input is decomposed to 3 bf16 components, then 6 dot operations are done on those components, and the result is accumulated in f32. // not sure where this is available. pub const bf16_6x: DotAlgorithm = .{ - .operand = .bf16, .accumulation = .f32, .component_count = 1, .num_primitive_operations = 6, .allow_imprecise_accumulation = false, }; - - pub fn asAttr(self: DotAlgorithm, ctx: mlir.Context, tensor_type: mlir.RankedTensorType) mlir.Attribute { - const elem_type = tensor_type.getElementType(); - - return mlir.Attribute.wrap(c.stablehloDotAlgorithmGet( - ctx._inner, - elem_type._inner, - elem_type._inner, - self.accumulation.asType(ctx)._inner, - self.component_count, - self.component_count, - self.num_primitive_operations, - self.allow_imprecise_accumulation, - )); - } }; +pub fn dotAlgorithmAttribute(ctx: *mlir.Context, dot_algorithm: DotAlgorithm, element_type: *const mlir.Type) *const mlir.Attribute { + return @ptrCast(c.stablehloDotAlgorithmGet( + ctx.ptr(), + element_type.ptr(), + element_type.ptr(), + mlir.floatType(ctx, dot_algorithm.accumulation).ptr(), + dot_algorithm.component_count, + dot_algorithm.component_count, + dot_algorithm.num_primitive_operations, + dot_algorithm.allow_imprecise_accumulation, + ).ptr); +} + /// General matrix multiplication "a la Einstein sum" /// Note: stablehlo doesn't do type inference for dot_general pub fn dot_general( - ctx: mlir.Context, - lhs: mlir.Value, - rhs: mlir.Value, - result_type: mlir.Type, - location: mlir.Location, + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + result_type: *const mlir.Type, opts: struct { lhs_batching_dimensions: []const i64, rhs_batching_dimensions: []const i64, lhs_contracting_dimensions: []const i64, rhs_contracting_dimensions: []const i64, - precision: DotPrecision, + dot_precision: DotPrecision, }, -) mlir.Operation { - const precisions: [2]mlir.Attribute = @splat(opts.precision.precisionAttr(ctx)); - const attributes = [3]mlir.AttrTuple{ - .{ - "dot_dimension_numbers", DotDimensionNumbersAttribute.init(ctx, .{ + location: *const mlir.Location, +) *mlir.Operation { + const precisions: [2]*const mlir.Attribute = @splat(precisionAttribute(ctx, opts.dot_precision.precision())); + const attributes = [3]mlir.NamedAttribute{ + .named( + ctx, + "dot_dimension_numbers", + dotDimensionNumbersAttribute(ctx, .{ .lhs_batching_dimensions = opts.lhs_batching_dimensions, .rhs_batching_dimensions = opts.rhs_batching_dimensions, .lhs_contracting_dimensions = opts.lhs_contracting_dimensions, .rhs_contracting_dimensions = opts.rhs_contracting_dimensions, - }).asAttr(), - }, - .{ "precision_config", .array(ctx, &precisions) }, + }), + ), + .named(ctx, "precision_config", mlir.arrayAttribute(ctx, &precisions)), // keep algorithm as the last attribute so we can omit it when it's not set. - .{ "algorithm", opts.precision.algorithmAttr(ctx, lhs.getType().as(mlir.RankedTensorType).?) orelse undefined }, + .named(ctx, "algorithm", switch (opts.dot_precision) { + .algorithm => |v| dotAlgorithmAttribute(ctx, v, lhs.type_()), + else => undefined, + }), }; - const n_attributes = if (opts.precision == .algorithm) attributes.len else attributes.len - 1; + const n_attributes = if (opts.dot_precision == .algorithm) attributes.len else attributes.len - 1; return mlir.Operation.make(ctx, "stablehlo.dot_general", .{ - .operands = &.{ lhs, rhs }, - .results = &.{result_type}, + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, .attributes = attributes[0..n_attributes], .location = location, }); } pub fn constant( - ctx: mlir.Context, + ctx: *mlir.Context, dims: []const i64, - elem_type: mlir.DenseElementsAttributeTypes, + elem_type: *const mlir.Type, raw_bytes: []const u8, - location: mlir.Location, -) mlir.Operation { + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.constant", .{ - .operands = &.{}, - .results = &.{.tensor(dims, elem_type.mlirType(ctx))}, - .attributes = &.{.{ "value", .denseElementsFromBytes(ctx, dims, elem_type, raw_bytes) }}, + .operands = .{ .flat = &.{} }, + .results = .{ .flat = &.{mlir.rankedTensorType(dims, elem_type)} }, + .attributes = &.{ + .named(ctx, "value", mlir.denseElementsAttribute(mlir.RankedTensorType.get(dims, elem_type).shaped(), raw_bytes)), + }, .location = location, }); } -pub fn convert(ctx: mlir.Context, value: mlir.Value, result_type: mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.convert", .{ - .operands = &.{value}, - .results = &.{result_type}, - .location = location, - }); -} - -pub fn broadcast_in_dim(ctx: mlir.Context, operand: mlir.Value, dims: []const i64, result_type: mlir.Type, location: mlir.Location) mlir.Operation { +pub fn broadcast_in_dim( + ctx: *mlir.Context, + operand: *const mlir.Value, + dims: []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.broadcast_in_dim", .{ - .operands = &.{operand}, - .results = &.{result_type}, + .operands = .{ .flat = &.{operand} }, + .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .{ "broadcast_dimensions", .dense(ctx, .i64, dims) }, + .named(ctx, "broadcast_dimensions", mlir.denseArrayAttribute(ctx, .i64, dims)), }, .location = location, }); } -pub fn transpose(ctx: mlir.Context, value: mlir.Value, result_type: mlir.Type, location: mlir.Location, opts: struct { permutation: []const i64 }) mlir.Operation { +pub fn transpose( + ctx: *mlir.Context, + value: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { permutation: []const i64 }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.transpose", .{ - .operands = &.{value}, - .results = &.{result_type}, + .operands = .{ .flat = &.{value} }, + .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .{ "permutation", .dense(ctx, .i64, opts.permutation) }, + .named(ctx, "permutation", mlir.denseArrayAttribute(ctx, .i64, opts.permutation)), }, .location = location, }); } -pub fn slice(ctx: mlir.Context, operand: mlir.Value, start_indices: []const i64, limit_indices: []const i64, strides: []const i64, result_type: mlir.Type, location: mlir.Location) mlir.Operation { +pub fn slice( + ctx: *mlir.Context, + operand: *const mlir.Value, + start_indices: []const i64, + limit_indices: []const i64, + strides: []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.slice", .{ - .operands = &.{operand}, - .results = &.{result_type}, + .operands = .{ .flat = &.{operand} }, + .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .{ "start_indices", .dense(ctx, .i64, start_indices) }, - .{ "limit_indices", .dense(ctx, .i64, limit_indices) }, - .{ "strides", .dense(ctx, .i64, strides) }, + .named(ctx, "start_indices", mlir.denseArrayAttribute(ctx, .i64, start_indices)), + .named(ctx, "limit_indices", mlir.denseArrayAttribute(ctx, .i64, limit_indices)), + .named(ctx, "strides", mlir.denseArrayAttribute(ctx, .i64, strides)), }, .location = location, }); } -pub fn concatenate(ctx: mlir.Context, inputs: []const mlir.Value, dimension: i64, location: mlir.Location) mlir.Operation { +pub fn concatenate(ctx: *mlir.Context, inputs: []const *const mlir.Value, dimension: i64, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.concatenate", .{ - .operands = inputs, + .operands = .{ .flat = inputs }, .result_type_inference = true, .attributes = &.{ - .{ "dimension", .int(ctx, .i64, dimension) }, + .named(ctx, "dimension", mlir.integerAttribute(ctx, .i64, dimension)), }, .location = location, }); } -pub fn reshape(ctx: mlir.Context, value: mlir.Value, result_type: mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.reshape", .{ - .operands = &.{value}, - .results = &.{result_type}, - .location = location, - }); -} - -pub fn select(ctx: mlir.Context, condition: mlir.Value, then: mlir.Value, else_: mlir.Value, location: mlir.Location) mlir.Operation { +pub fn select( + ctx: *mlir.Context, + condition: *const mlir.Value, + then: *const mlir.Value, + else_: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.select", .{ - .operands = &.{ condition, then, else_ }, - .results = &.{then.getType()}, + .operands = .{ .flat = &.{ condition, then, else_ } }, + .results = .{ .flat = &.{then.type_()} }, .location = location, }); } pub fn gather( - ctx: mlir.Context, - value: mlir.Value, - indices: mlir.Value, + ctx: *mlir.Context, + value: *const mlir.Value, + indices: *const mlir.Value, slice_sizes: []const i64, - location: mlir.Location, args: struct { offset_dims: []const i64, collapsed_slice_dims: []const i64, @@ -311,15 +317,16 @@ pub fn gather( index_vector_dim: i64, indices_are_sorted: bool = false, }, -) mlir.Operation { + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make( ctx, "stablehlo.gather", .{ - .operands = &.{ value, indices }, + .operands = .{ .flat = &.{ value, indices } }, .result_type_inference = true, .attributes = &.{ - .{ "dimension_numbers", GatherDimensionNumbersAttribute.init( + .named(ctx, "dimension_numbers", gatherDimensionNumbersAttribute( ctx, args.offset_dims, args.collapsed_slice_dims, @@ -327,21 +334,15 @@ pub fn gather( args.start_indices_batching_dims, args.start_index_map, args.index_vector_dim, - ).asAttr() }, - .{ "slice_sizes", .dense(ctx, .i64, slice_sizes) }, - .{ "indices_are_sorted", .boolean(ctx, args.indices_are_sorted) }, + )), + .named(ctx, "slice_sizes", mlir.denseArrayAttribute(ctx, .i64, slice_sizes)), + .named(ctx, "indices_are_sorted", mlir.boolAttribute(ctx, args.indices_are_sorted)), }, .location = location, }, ); } -fn elementTypeOrSelf(typ: mlir.Type) mlir.Type { - return if (typ.as(mlir.ShapedType)) |shaped| { - return shaped.elementType(); - } else typ; -} - pub const ScatterArgs = struct { update_window_dims: []const i64, inserted_window_dims: []const i64, @@ -352,9 +353,9 @@ pub const ScatterArgs = struct { indices_are_sorted: bool = false, unique_indices: bool = false, - pub fn getScatterDimensionNumbers(self: ScatterArgs, ctx: mlir.Context) mlir.Attribute { - return .{ ._inner = c.stablehloScatterDimensionNumbersGet( - ctx._inner, + pub fn getScatterDimensionNumbers(self: ScatterArgs, ctx: *mlir.Context) *const mlir.Attribute { + return @ptrCast(c.stablehloScatterDimensionNumbersGet( + ctx.ptr(), @intCast(self.update_window_dims.len), self.update_window_dims.ptr, @intCast(self.inserted_window_dims.len), @@ -366,29 +367,29 @@ pub const ScatterArgs = struct { @intCast(self.scatter_dims_to_operand_dims.len), self.scatter_dims_to_operand_dims.ptr, self.index_vector_dim, - ) }; + ).ptr); } }; pub fn scatter( - ctx: mlir.Context, - inputs: []const mlir.Value, - scatter_indices: []const mlir.Value, - updates: []const mlir.Value, - update_block: mlir.Block, + ctx: *mlir.Context, + inputs: []const *const mlir.Value, + scatter_indices: []const *const mlir.Value, + updates: []const *const mlir.Value, + update_block: *mlir.Block, args: ScatterArgs, - location: mlir.Location, -) mlir.Operation { + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make( ctx, "stablehlo.scatter", .{ - .variadic_operands = &.{ inputs, scatter_indices, updates }, + .operands = .{ .variadic = &.{ inputs, scatter_indices, updates } }, .blocks = &.{update_block}, .attributes = &.{ - .{ "scatter_dimension_numbers", args.getScatterDimensionNumbers(ctx) }, - .{ "indices_are_sorted", .boolean(ctx, args.indices_are_sorted) }, - .{ "unique_indices", .boolean(ctx, args.unique_indices) }, + .named(ctx, "scatter_dimension_numbers", args.getScatterDimensionNumbers(ctx)), + .named(ctx, "indices_are_sorted", mlir.boolAttribute(ctx, args.indices_are_sorted)), + .named(ctx, "unique_indices", mlir.boolAttribute(ctx, args.unique_indices)), }, .result_type_inference = true, .location = location, @@ -396,163 +397,78 @@ pub fn scatter( ); } -pub fn iota(ctx: mlir.Context, dimension: i64, result_type: mlir.Type, location: mlir.Location) mlir.Operation { +pub fn iota(ctx: *mlir.Context, dimension: i64, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.iota", .{ - .operands = &.{}, - .results = &.{result_type}, + .operands = .{ .flat = &.{} }, + .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .{ "iota_dimension", .int(ctx, .i64, dimension) }, + .named(ctx, "iota_dimension", mlir.integerAttribute(ctx, .i64, dimension)), }, .location = location, }); } -pub fn reverse(ctx: mlir.Context, operand: mlir.Value, dimensions: []const i64, location: mlir.Location) mlir.Operation { - const result_type = operand.getType(); +pub fn reverse(ctx: *mlir.Context, operand: *const mlir.Value, dimensions: []const i64, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.reverse", .{ - .operands = &.{operand}, - .results = &.{result_type}, + .operands = .{ .flat = &.{operand} }, + .results = .{ .flat = &.{operand.type_()} }, .attributes = &.{ - .{ "dimensions", .dense(ctx, .i64, dimensions) }, + .named(ctx, "dimensions", mlir.denseArrayAttribute(ctx, .i64, dimensions)), }, .location = location, }); } -pub fn compare(ctx: mlir.Context, lhs: mlir.Value, rhs: mlir.Value, comparison_direction: ComparisonDirection, compare_type: CompareType, location: mlir.Location) mlir.Operation { +pub fn compare( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + comparison_direction: ComparisonDirection.Direction, + compare_type: CompareType.Type, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.compare", .{ - .operands = &.{ lhs, rhs }, + .operands = .{ .flat = &.{ lhs, rhs } }, .result_type_inference = true, .attributes = &.{ - .{ "comparison_direction", comparison_direction.asAttr() }, - .{ "compare_type", compare_type.asAttr() }, + .named(ctx, "comparison_direction", comparisonDirection(ctx, comparison_direction)), + .named(ctx, "compare_type", compareType(ctx, compare_type)), }, .location = location, }); } -pub fn reduce( - ctx: mlir.Context, - inputs: []const mlir.Value, - init_values: []const mlir.Value, - dimensions: []const i64, - blkctx: anytype, - blkfn: fn (anytype, mlir.Context, []const mlir.Value, []const mlir.Value) mlir.Operation, - location: mlir.Location, -) mlir.Operation { - const MaxBlockArguments = 32; - - const block_n_args = inputs.len + init_values.len; - const locations = ([_]mlir.Location{mlir.Location.unknown(ctx)} ** MaxBlockArguments)[0..block_n_args]; - var reduce_elem_types: [MaxBlockArguments]mlir.Type = undefined; - for (inputs, 0..) |input, i| { - const arg_type: mlir.Type = .tensor(&.{}, elementTypeOrSelf(input.getType())); - reduce_elem_types[i] = arg_type; - reduce_elem_types[inputs.len + i] = arg_type; - } - var block = mlir.Block.open(reduce_elem_types[0..block_n_args], locations) catch unreachable; - { - defer block.close(); - - var block_inputs: [MaxBlockArguments / 2]mlir.Value = undefined; - var block_accs: [MaxBlockArguments / 2]mlir.Value = undefined; - for (0..inputs.len) |i| { - block_inputs[i] = block.argument(i); - block_accs[i] = block.argument(inputs.len + i); - } - _ = blkfn(blkctx, ctx, block_inputs[0..inputs.len], block_accs[0..init_values.len]); - } - - return mlir.Operation.make(ctx, "stablehlo.reduce", .{ - .variadic_operands = &.{ inputs, init_values }, - .result_type_inference = true, - .block = block, - .attributes = &.{ - .{ "dimensions", .dense(ctx, .i64, dimensions) }, - }, - .location = location, - }); -} - -pub fn sort( - ctx: mlir.Context, - inputs: []const mlir.Value, - dimension: i64, - is_stable: bool, - blkctx: anytype, - compfn: fn (anytype, mlir.Context, []const mlir.Value) mlir.Operation, - location: mlir.Location, -) mlir.Operation { - const MaxBlockArguments = 32; - - const locations = ([_]mlir.Location{mlir.Location.unknown(ctx)} ** MaxBlockArguments)[0 .. inputs.len * 2]; - var sort_elem_types: [MaxBlockArguments]mlir.Type = undefined; - for (inputs, 0..) |input, i| { - const arg_type: mlir.Type = .tensor(&.{}, elementTypeOrSelf(input.getType())); - sort_elem_types[i * 2] = arg_type; - sort_elem_types[i * 2 + 1] = arg_type; - } - var block = mlir.Block.init(sort_elem_types[0 .. inputs.len * 2], locations) catch unreachable; - - var block_inputs: [MaxBlockArguments]mlir.Value = undefined; - for (0..inputs.len * 2) |i| { - block_inputs[i] = block.argument(i); - } - _ = compfn(blkctx, ctx, block_inputs[0 .. inputs.len * 2]); - - return mlir.Operation.make(ctx, "stablehlo.sort", .{ - .variadic_operands = &.{inputs}, - .result_type_inference = true, - .block = block, - .attributes = &.{ - .{ "dimension", .int(ctx, .i64, dimension) }, - .{ "is_stable", .boolean(ctx, is_stable) }, - }, - .location = location, - }); -} - -pub fn dynamic_slice(ctx: mlir.Context, operand: mlir.Value, new_dims: []const i64, start_indices: []const mlir.Value, location: mlir.Location) mlir.Operation { +pub fn dynamic_slice( + ctx: *mlir.Context, + operand: *const mlir.Value, + new_dims: []const i64, + start_indices: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.dynamic_slice", .{ - .variadic_operands = &.{ &.{operand}, start_indices }, + .operands = .{ .variadic = &.{ &.{operand}, start_indices } }, .result_type_inference = true, .attributes = &.{ - .{ "slice_sizes", .dense(ctx, .i64, new_dims) }, + .named(ctx, "slice_sizes", mlir.denseArrayAttribute(ctx, .i64, new_dims)), }, .location = location, }); } -pub fn round_nearest_afz(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.round_nearest_afz", .{ - .operands = &.{value}, - .result_type_inference = true, - .location = location, - }); -} - -pub fn round_nearest_even(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.round_nearest_even", .{ - .operands = &.{value}, - .result_type_inference = true, - .location = location, - }); -} - pub const PadOpts = struct { low: []const i64, high: []const i64, interior: []const i64, }; -pub fn pad(ctx: mlir.Context, value: mlir.Value, padding_value: mlir.Value, opts: PadOpts, location: mlir.Location) mlir.Operation { +pub fn pad(ctx: *mlir.Context, value: *const mlir.Value, padding_value: *const mlir.Value, location: *const mlir.Location, opts: PadOpts) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.pad", .{ - .operands = &.{ value, padding_value }, + .operands = .{ .flat = &.{ value, padding_value } }, .result_type_inference = true, .attributes = &.{ - .{ "edge_padding_low", .dense(ctx, .i64, opts.low) }, - .{ "edge_padding_high", .dense(ctx, .i64, opts.high) }, - .{ "interior_padding", .dense(ctx, .i64, opts.interior) }, + .named(ctx, "edge_padding_low", mlir.denseArrayAttribute(ctx, .i64, opts.low)), + .named(ctx, "edge_padding_high", mlir.denseArrayAttribute(ctx, .i64, opts.high)), + .named(ctx, "interior_padding", mlir.denseArrayAttribute(ctx, .i64, opts.interior)), }, .location = location, }); @@ -562,18 +478,18 @@ pub const TriangularSolveOpts = struct { left_side: bool, lower: bool, unit_diagonal: bool, - transpose_a: Transpose.Type, + transpose_a: TransposeAttribute.Type, }; -pub fn triangular_solve(ctx: mlir.Context, value: mlir.Value, other: mlir.Value, location: mlir.Location, opts: TriangularSolveOpts) mlir.Operation { +pub fn triangular_solve(ctx: *mlir.Context, value: *const mlir.Value, other: *const mlir.Value, opts: TriangularSolveOpts, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.triangular_solve", .{ - .operands = &.{ value, other }, + .operands = .{ .flat = &.{ value, other } }, .result_type_inference = true, .attributes = &.{ - .{ "left_side", .i1FromBool(ctx, opts.left_side) }, - .{ "lower", .i1FromBool(ctx, opts.lower) }, - .{ "unit_diagonal", .i1FromBool(ctx, opts.unit_diagonal) }, - .{ "transpose_a", Transpose.init(ctx, opts.transpose_a).asAttr() }, + .named(ctx, "left_side", mlir.integerAttribute(ctx, .i1, @intFromBool(opts.left_side))), + .named(ctx, "lower", mlir.integerAttribute(ctx, .i1, @intFromBool(opts.lower))), + .named(ctx, "unit_diagonal", mlir.integerAttribute(ctx, .i1, @intFromBool(opts.unit_diagonal))), + .named(ctx, "transpose_a", transposeAttribute(ctx, opts.transpose_a)), }, .location = location, }); @@ -584,74 +500,132 @@ pub const FftOpts = struct { length: []const i64, }; -pub fn fft(ctx: mlir.Context, value: mlir.Value, location: mlir.Location, opts: FftOpts) mlir.Operation { +pub fn fft(ctx: *mlir.Context, value: *const mlir.Value, opts: FftOpts, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.fft", .{ - .operands = &.{value}, + .operands = .{ .flat = &.{value} }, .result_type_inference = true, .attributes = &.{ - .{ "fft_type", FftType.init(ctx, opts.kind).asAttr() }, - .{ "fft_length", .dense(ctx, .i64, opts.length) }, + .named(ctx, "fft_type", fftType(ctx, opts.kind)), + .named(ctx, "fft_length", mlir.denseArrayAttribute(ctx, .i64, opts.length)), }, .location = location, }); } -pub fn rng(ctx: mlir.Context, a: mlir.Value, b: mlir.Value, shape: mlir.Value, rng_distribution: RngDistribution.Type, location: mlir.Location) mlir.Operation { +pub const RngDistribution = opaque { + const M = mlir.Methods(RngDistribution, c.MlirAttribute); + + pub const isAFn = c.stablehloAttributeIsARngDistributionAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const Type = enum { + UNIFORM, + NORMAL, + }; + + pub fn get(ctx: *mlir.Context, value_: Type) *const RngDistribution { + return @ptrCast(c.stablehloRngDistributionAttrGet(ctx.ptr(), stringRef(@tagName(value_))).ptr); + } + + pub fn value(self: *const RngDistribution) Type { + return std.meta.stringToEnum( + Type, + mlir.string(c.stablehloRngDistributionAttrGetValue(self.ptr())), + ) orelse unreachable; + } +}; + +pub fn rngDistribution(ctx: *mlir.Context, value: RngDistribution.Type) *const mlir.Attribute { + return @ptrCast(RngDistribution.get(ctx, value)); +} + +pub const RngAlgorithm = opaque { + const M = mlir.Methods(RngAlgorithm, c.MlirAttribute); + + pub const isAFn = c.stablehloAttributeIsARngAlgorithmAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const Type = enum { + DEFAULT, + THREE_FRY, + PHILOX, + }; + + pub fn get(ctx: *mlir.Context, value_: Type) *const RngAlgorithm { + return @ptrCast(c.stablehloRngAlgorithmAttrGet(ctx.ptr(), stringRef(@tagName(value_))).ptr); + } + + pub fn value(self: *const RngAlgorithm) Type { + return std.meta.stringToEnum( + Type, + mlir.string(c.stablehloRngAlgorithmAttrGetValue(self.ptr())), + ) orelse unreachable; + } +}; + +pub fn rngAlgorithm(ctx: *mlir.Context, value: RngAlgorithm.Type) *const mlir.Attribute { + return @ptrCast(RngAlgorithm.get(ctx, value)); +} + +pub fn rng(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, shape: *const mlir.Value, rng_distribution: RngDistribution.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.rng", .{ - .operands = &.{ a, b, shape }, + .operands = .{ .flat = &.{ a, b, shape } }, .result_type_inference = true, .attributes = &.{ - .{ "rng_distribution", RngDistribution.init(ctx, rng_distribution).asAttr() }, + .named(ctx, "rng_distribution", rngDistribution(ctx, rng_distribution)), }, .location = location, }); } -pub fn rng_bit_generator(ctx: mlir.Context, rng_algorithm: RngAlgorithm.Type, initial_state: mlir.Value, res_state_type: mlir.Type, res_type: mlir.Type, location: mlir.Location) mlir.Operation { +pub fn rng_bit_generator( + ctx: *mlir.Context, + rng_algorithm: RngAlgorithm.Type, + initial_state: *const mlir.Value, + res_state_type: *const mlir.Type, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.rng_bit_generator", .{ - .operands = &.{initial_state}, + .operands = .{ .flat = &.{initial_state} }, .results = &.{ res_state_type, res_type }, .attributes = &.{ - .{ "rng_algorithm", RngAlgorithm.init(ctx, rng_algorithm).asAttr() }, + .named(ctx, "rng_algorithm", rngAlgorithm(ctx, rng_algorithm)), }, .location = location, }); } -pub fn reduce_precision(ctx: mlir.Context, value: mlir.Value, exponent_bits: i32, mantissa_bits: i32, location: mlir.Location) mlir.Operation { +pub fn reduce_precision(ctx: *mlir.Context, value: *const mlir.Value, exponent_bits: i32, mantissa_bits: i32, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.reduce_precision", .{ - .operands = &.{value}, + .operands = .{ .flat = &.{value} }, .result_type_inference = true, .attributes = &.{ - .{ "exponent_bits", .int(ctx, .i32, exponent_bits) }, - .{ "mantissa_bits", .int(ctx, .i32, mantissa_bits) }, + .named(ctx, "exponent_bits", mlir.integerAttribute(ctx, .i32, exponent_bits)), + .named(ctx, "mantissa_bits", mlir.integerAttribute(ctx, .i32, mantissa_bits)), }, .location = location, }); } -pub fn dynamic_update_slice(ctx: mlir.Context, operand: mlir.Value, update: mlir.Value, start_indices: []const mlir.Value, location: mlir.Location) mlir.Operation { +pub fn dynamic_update_slice(ctx: *mlir.Context, operand: *const mlir.Value, update: *const mlir.Value, start_indices: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.dynamic_update_slice", .{ - .variadic_operands = &.{ &.{operand}, &.{update}, start_indices }, + .operands = .{ .variadic = &.{ &.{operand}, &.{update}, start_indices } }, .result_type_inference = true, .location = location, }); } -pub fn tuple(ctx: mlir.Context, values: []const mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.tuple", .{ - .operands = values, - .result_type_inference = true, - .location = location, - }); -} - -pub fn get_tuple_element(ctx: mlir.Context, tuple_value: mlir.Value, index: i64, location: mlir.Location) mlir.Operation { +pub fn get_tuple_element(ctx: *mlir.Context, tuple_value: *const mlir.Value, index: i64, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.get_tuple_element", .{ - .operands = &.{tuple_value}, + .operands = .{ .flat = &.{tuple_value} }, .result_type_inference = true, .attributes = &.{ - .{ "index", .int(ctx, .i32, index) }, + .named(ctx, "index", mlir.integerAttribute(ctx, .i32, index)), }, .location = location, }); @@ -679,46 +653,44 @@ pub const ConvolutionOpts = struct { }; pub fn convolution( - ctx: mlir.Context, - lhs: mlir.Value, - rhs: mlir.Value, + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + res_type: *const mlir.Type, opts: ConvolutionOpts, - res_type: mlir.Type, - location: mlir.Location, -) mlir.Operation { - var max_precisions: [2]mlir.Attribute = undefined; + location: *const mlir.Location, +) *mlir.Operation { + var max_precisions: [2]*const mlir.Attribute = undefined; for (opts.precision_config, 0..) |p, i| { - max_precisions[i] = PrecisionAttribute.init(ctx, p).asAttr(); + max_precisions[i] = precisionAttribute(ctx, p); } var window_reversal: [3]i32 = undefined; for (opts.window_reversal, 0..) |w, i| { window_reversal[i] = @intCast(@intFromBool(w)); } return mlir.Operation.make(ctx, "stablehlo.convolution", .{ - .operands = &.{ lhs, rhs }, - .results = &.{res_type}, + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{res_type} }, .attributes = &.{ - .{ "window_strides", .dense(ctx, .i64, opts.window_strides) }, - .{ "padding", .denseElements(ctx, opts.pad_shape, .i64, opts.pad_value) }, - .{ "lhs_dilation", .dense(ctx, .i64, opts.lhs_dilation) }, - .{ "rhs_dilation", .dense(ctx, .i64, opts.rhs_dilation) }, - .{ "window_reversal", .dense(ctx, .bool, window_reversal[0..opts.window_reversal.len]) }, - .{ - "dimension_numbers", ConvDimensionNumbersAttribute.init(ctx, .{ - .input_batch_dimension = opts.input_batch_dimension, - .input_feature_dimension = opts.input_feature_dimension, - .input_spatial_dimensions = opts.input_spatial_dimensions, - .kernel_input_feature_dimension = opts.kernel_input_feature_dimension, - .kernel_output_feature_dimension = opts.kernel_output_feature_dimension, - .kernel_spatial_dimensions = opts.kernel_spatial_dimensions, - .output_batch_dimension = opts.output_batch_dimension, - .output_feature_dimension = opts.output_feature_dimension, - .output_spatial_dimensions = opts.output_spatial_dimensions, - }).asAttr(), - }, - .{ "feature_group_count", .int(ctx, .i64, opts.feature_group_count) }, - .{ "batch_group_count", .int(ctx, .i64, opts.batch_group_count) }, - .{ "precision_config", .array(ctx, &max_precisions) }, + .named(ctx, "window_strides", mlir.denseArrayAttribute(ctx, .i64, opts.window_strides)), + .named(ctx, "padding", mlir.denseElementsAttribute(mlir.RankedTensorType.get(opts.pad_shape, mlir.integerType(ctx, .i64)).shaped(), opts.pad_value)), + .named(ctx, "lhs_dilation", mlir.denseArrayAttribute(ctx, .i64, opts.lhs_dilation)), + .named(ctx, "rhs_dilation", mlir.denseArrayAttribute(ctx, .i64, opts.rhs_dilation)), + .named(ctx, "window_reversal", mlir.denseArrayAttribute(ctx, .bool, window_reversal[0..opts.window_reversal.len])), + .named(ctx, "dimension_numbers", convDimensionNumbersAttribute(ctx, .{ + .input_batch_dimension = opts.input_batch_dimension, + .input_feature_dimension = opts.input_feature_dimension, + .input_spatial_dimensions = opts.input_spatial_dimensions, + .kernel_input_feature_dimension = opts.kernel_input_feature_dimension, + .kernel_output_feature_dimension = opts.kernel_output_feature_dimension, + .kernel_spatial_dimensions = opts.kernel_spatial_dimensions, + .output_batch_dimension = opts.output_batch_dimension, + .output_feature_dimension = opts.output_feature_dimension, + .output_spatial_dimensions = opts.output_spatial_dimensions, + })), + .named(ctx, "feature_group_count", mlir.integerAttribute(ctx, .i64, opts.feature_group_count)), + .named(ctx, "batch_group_count", mlir.integerAttribute(ctx, .i64, opts.batch_group_count)), + .named(ctx, "precision_config", mlir.arrayAttribute(ctx, &max_precisions)), }, .location = location, }); @@ -732,261 +704,350 @@ pub const CustomCallOpts = struct { typed_ffi = 4, }; - call_target_name: [:0]const u8, - has_side_effect: bool, - backend_config: ?mlir.Attribute, + pub const BackendConfig = union(ApiVersion) { + original: []const u8, + status_returning: []const u8, + status_returning_unified: []const u8, + typed_ffi: *const mlir.DictionaryAttribute, + }; + + call_target_name: []const u8, + has_side_effect: ?bool = null, + api_version: ?ApiVersion = null, + backend_config: ?BackendConfig = null, operand_layouts: ?[]const []const usize = null, result_layouts: ?[]const []const usize = null, - output_operand_aliases: []const i64 = &.{}, - additional_attributes: []const mlir.AttrTuple = &.{}, - api_version: ApiVersion, + output_operand_aliases: ?[]const i64 = null, + additional_attributes: []const mlir.NamedAttribute = &.{}, }; -pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCallOpts, res_types: []const mlir.Type, location: mlir.Location) mlir.Operation { +pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result_types: []const *const mlir.Type, opts: CustomCallOpts, location: *const mlir.Location) *mlir.Operation { const MAX_OPERANDS = 64; const MAX_RESULTS = 16; - - const backend_config = opts.backend_config orelse mlir.Attribute.string(ctx, ""); - if (@intFromEnum(opts.api_version) < @intFromEnum(CustomCallOpts.ApiVersion.typed_ffi)) { - stdx.debug.assert( - backend_config.isA(mlir.StringAttribute), - "API version < 4 requires a string as backend_config, got {f}", - .{backend_config}, - ); - } else { - stdx.debug.assert( - backend_config.isA(mlir.DictionaryAttribute), - "API version >= 4 requires a dictionary as backend_config, got {f}", - .{backend_config}, - ); - } - - var attrs: stdx.BoundedArray(mlir.AttrTuple, 32) = .{}; - attrs.appendSliceAssumeCapacity(&[_]mlir.AttrTuple{ - .{ "api_version", .int(ctx, .i32, @intFromEnum(opts.api_version)) }, - .{ "call_target_name", .string(ctx, opts.call_target_name) }, - .{ "has_side_effect", .boolean(ctx, opts.has_side_effect) }, - .{ "backend_config", backend_config }, - }); - - { - var output_operand_aliases: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; - for (opts.output_operand_aliases) |alias| { - output_operand_aliases.appendAssumeCapacity( - OutputOperandAliasAttribute.init(ctx, &.{}, alias, &.{}).asAttr(), - ); - } - attrs.appendAssumeCapacity(.{ "output_operand_aliases", .array(ctx, output_operand_aliases.constSlice()) }); - } - - const MINOR_TO_MAJOR = blk: { - const MAX_RANK = 8; - var ret: [MAX_RANK]usize = undefined; - for (0..MAX_RANK) |i| { - ret[i] = @intCast(MAX_RANK - i - 1); + const MINOR_TO_MAJOR = comptime blk: { + var ret: [mlir.ShapedType.MAX_RANK]usize = undefined; + for (0..ret.len) |i| { + ret[i] = @intCast(mlir.ShapedType.MAX_RANK - i - 1); } break :blk ret; }; + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 32) = .{}; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "call_target_name", mlir.stringAttribute(ctx, opts.call_target_name)), + }); + attrs.appendSliceAssumeCapacity(opts.additional_attributes); + + if (opts.has_side_effect) |has_side_effects| { + attrs.appendAssumeCapacity( + .named(ctx, "has_side_effect", mlir.boolAttribute(ctx, has_side_effects)), + ); + } + + const backend_config_api_version: ?CustomCallOpts.ApiVersion = if (opts.backend_config) |bc| bc else null; + if (opts.api_version orelse backend_config_api_version) |v| { + attrs.appendAssumeCapacity( + .named(ctx, "api_version", mlir.integerAttribute(ctx, .i32, @intFromEnum(v))), + ); + } + if (opts.backend_config) |backend_config| { + attrs.appendAssumeCapacity( + .named(ctx, "backend_config", switch (backend_config) { + .typed_ffi => |v| @ptrCast(v), + inline else => |v| mlir.stringAttribute(ctx, v), + }), + ); + } + + if (opts.output_operand_aliases) |output_operand_aliases| { + var buffer: stdx.BoundedArray(*const mlir.Attribute, MAX_RESULTS) = .{}; + for (output_operand_aliases) |alias| { + buffer.appendAssumeCapacity( + outputOperandAliasAttribute(ctx, .{ .operand_index = alias }), + ); + } + attrs.appendAssumeCapacity( + .named(ctx, "output_operand_aliases", mlir.arrayAttribute(ctx, buffer.constSlice())), + ); + } + + var layouts_buffer: stdx.BoundedArray(*const mlir.Attribute, MAX_OPERANDS) = .{}; if (opts.operand_layouts) |layouts| { - var operand_layouts: stdx.BoundedArray(mlir.Attribute, MAX_OPERANDS) = .{}; for (layouts) |ol| { - operand_layouts.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(ol.len)}, .index, ol)); + layouts_buffer.appendAssumeCapacity( + mlir.denseElementsAttribute( + mlir.RankedTensorType.get( + &.{@intCast(ol.len)}, + mlir.indexType(ctx), + null, + ).shaped(), + ol, + ), + ); } - attrs.appendAssumeCapacity(.{ "operand_layouts", .array(ctx, operand_layouts.constSlice()) }); } else { - const operand_layouts = blk: { - var ret: stdx.BoundedArray(mlir.Attribute, MAX_OPERANDS) = .{}; - for (inputs) |input| { - const ranked_type = input.getType().as(mlir.RankedTensorType).?; - const ol = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - ranked_type.getRank() ..]; - ret.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(ol.len)}, .index, ol)); - } - break :blk ret; - }; - attrs.appendAssumeCapacity(.{ "operand_layouts", .array(ctx, operand_layouts.constSlice()) }); + for (inputs) |input| { + const shaped_type = input.type_().isA(mlir.ShapedType).?; + const ol = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - shaped_type.rank() ..]; + layouts_buffer.appendAssumeCapacity( + mlir.denseElementsAttribute( + mlir.RankedTensorType.get( + &.{@intCast(ol.len)}, + mlir.indexType(ctx), + null, + ).shaped(), + ol, + ), + ); + } } + attrs.appendAssumeCapacity( + .named(ctx, "operand_layouts", mlir.arrayAttribute(ctx, layouts_buffer.constSlice())), + ); + layouts_buffer.clear(); if (opts.result_layouts) |layouts| { - var result_layouts: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; for (layouts) |rl| { - result_layouts.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(rl.len)}, .index, rl)); + layouts_buffer.appendAssumeCapacity( + mlir.denseElementsAttribute( + mlir.RankedTensorType.get( + &.{@intCast(rl.len)}, + mlir.indexType(ctx), + null, + ).shaped(), + rl, + ), + ); } - attrs.appendAssumeCapacity(.{ "result_layouts", .array(ctx, result_layouts.constSlice()) }); } else { - const result_layouts = blk: { - var ret: stdx.BoundedArray(mlir.Attribute, MAX_RESULTS) = .{}; - for (res_types) |t| { - const ranked_t = t.as(mlir.RankedTensorType).?; - const rl = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - ranked_t.getRank() ..]; - ret.appendAssumeCapacity(.denseElements(ctx, &.{@intCast(rl.len)}, .index, rl)); - } - break :blk ret; - }; - attrs.appendAssumeCapacity(.{ "result_layouts", .array(ctx, result_layouts.constSlice()) }); + for (result_types) |result_type| { + const shaped_type = result_type.isA(mlir.ShapedType).?; + const rl = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - shaped_type.rank() ..]; + layouts_buffer.appendAssumeCapacity( + mlir.denseElementsAttribute( + mlir.RankedTensorType.get( + &.{@intCast(rl.len)}, + mlir.indexType(ctx), + null, + ).shaped(), + rl, + ), + ); + } } - - attrs.appendSlice(opts.additional_attributes) catch @panic("Too many additional_attributes"); + attrs.appendAssumeCapacity( + .named(ctx, "result_layouts", mlir.arrayAttribute(ctx, layouts_buffer.constSlice())), + ); return mlir.Operation.make(ctx, "stablehlo.custom_call", .{ - .operands = inputs, - .results = res_types, + .operands = .{ .flat = inputs }, + .results = .{ .flat = result_types }, .attributes = attrs.constSlice(), .location = location, }); } -pub const DotDimensionNumbersAttribute = struct { - _inner: c.MlirAttribute, +pub fn createBuffer(ctx: *mlir.Context, value: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return custom_call(ctx, &.{}, &.{value}, .{ + .call_target_name = "CreateBuffer", + .api_version = .typed_ffi, + }, location); +} - pub const is_a_fn = c.stablehloAttributeIsADotDimensionNumbers; - const Self = DotDimensionNumbersAttribute; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); +pub fn pin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return custom_call(ctx, &.{}, &.{@ptrCast(mlir.MemRefType.fromShaped(value.type_().isA(mlir.ShapedType).?))}, .{ + .call_target_name = "Pin", + .api_version = .typed_ffi, + }, location); +} - pub fn init(ctx: mlir.Context, args: struct { +pub fn unpin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return custom_call(ctx, &.{}, &.{@ptrCast(mlir.RankedTensorType.fromShaped(value.type_().isA(mlir.ShapedType).?))}, .{ + .call_target_name = "Unpin", + .api_version = .typed_ffi, + }, location); +} + +pub fn partition_id(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.partition_id", .{ + .results = .{ .flat = &.{mlir.rankedTensorType(&.{}, mlir.integerType(ctx, .u32))} }, + .location = location, + }); +} + +pub const DotDimensionNumbersAttribute = opaque { + const M = mlir.Methods(DotDimensionNumbersAttribute, mlir.Attribute); + + pub const isAFn = c.stablehloAttributeIsADotDimensionNumbers; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const InitArgs = struct { lhs_batching_dimensions: []const i64, rhs_batching_dimensions: []const i64, lhs_contracting_dimensions: []const i64, rhs_contracting_dimensions: []const i64, - }) Self { - return .{ - ._inner = c.stablehloDotDimensionNumbersGet( - ctx._inner, - @intCast(args.lhs_batching_dimensions.len), - args.lhs_batching_dimensions.ptr, - @intCast(args.rhs_batching_dimensions.len), - args.rhs_batching_dimensions.ptr, - @intCast(args.lhs_contracting_dimensions.len), - args.lhs_contracting_dimensions.ptr, - @intCast(args.rhs_contracting_dimensions.len), - args.rhs_contracting_dimensions.ptr, - ), - }; + }; + + pub fn init(ctx: *mlir.Context, args: InitArgs) *const DotDimensionNumbersAttribute { + return @ptrCast(c.stablehloDotDimensionNumbersGet( + ctx.ptr(), + @intCast(args.lhs_batching_dimensions.len), + args.lhs_batching_dimensions.ptr, + @intCast(args.rhs_batching_dimensions.len), + args.rhs_batching_dimensions.ptr, + @intCast(args.lhs_contracting_dimensions.len), + args.lhs_contracting_dimensions.ptr, + @intCast(args.rhs_contracting_dimensions.len), + args.rhs_contracting_dimensions.ptr, + ).ptr); } - pub fn getLhsBatchingDimensionsSize(self: Self) usize { - return @intCast(c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsSize(self._inner)); + pub fn getLhsBatchingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { + return @intCast(c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsSize(self.ptr())); } - pub fn getLhsBatchingDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsElem(self._inner, @intCast(pos)); + pub fn getLhsBatchingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn getRhsBatchingDimensionsSize(self: Self) usize { - return @intCast(c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsSize(self._inner)); + pub fn getRhsBatchingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { + return @intCast(c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsSize(self.ptr())); } - pub fn getRhsBatchingDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsElem(self._inner, @intCast(pos)); + pub fn getRhsBatchingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn getLhsContractingDimensionsSize(self: Self) usize { - return @intCast(c.stablehloDotDimensionNumbersGetLhsContractingDimensionsSize(self._inner)); + pub fn getLhsContractingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { + return @intCast(c.stablehloDotDimensionNumbersGetLhsContractingDimensionsSize(self.ptr())); } - pub fn getLhsContractingDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloDotDimensionNumbersGetLhsContractingDimensionsElem(self._inner, @intCast(pos)); + pub fn getLhsContractingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloDotDimensionNumbersGetLhsContractingDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn getRhsContractingDimensionsSize(self: Self) usize { - return @intCast(c.stablehloDotDimensionNumbersGetRhsContractingDimensionsSize(self._inner)); + pub fn getRhsContractingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { + return @intCast(c.stablehloDotDimensionNumbersGetRhsContractingDimensionsSize(self.ptr())); } - pub fn getRhsContractingDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloDotDimensionNumbersGetRhsContractingDimensionsElem(self._inner, @intCast(pos)); + pub fn getRhsContractingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloDotDimensionNumbersGetRhsContractingDimensionsElem(self.ptr(), @intCast(pos)); } }; -pub const GatherDimensionNumbersAttribute = struct { - _inner: c.MlirAttribute, +pub fn dotDimensionNumbersAttribute(ctx: *mlir.Context, args: DotDimensionNumbersAttribute.InitArgs) *const mlir.Attribute { + return @ptrCast(DotDimensionNumbersAttribute.init(ctx, args)); +} - pub const is_a_fn = c.stablehloAttributeIsAGatherDimensionNumbers; - const Self = GatherDimensionNumbersAttribute; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); +pub const GatherDimensionNumbersAttribute = opaque { + const M = mlir.Methods(GatherDimensionNumbersAttribute, mlir.Attribute); + + pub const isAFn = c.stablehloAttributeIsAGatherDimensionNumbers; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.eql(c.mlirAttributePrint); pub fn init( - ctx: mlir.Context, + ctx: *mlir.Context, offset_dims: []const i64, collapsed_slice_dims: []const i64, operand_batching_dims: []const i64, start_indices_batching_dims: []const i64, start_index_map: []const i64, index_vector_dim: i64, - ) Self { - return .{ - ._inner = c.stablehloGatherDimensionNumbersGet( - ctx._inner, - @intCast(offset_dims.len), - offset_dims.ptr, - @intCast(collapsed_slice_dims.len), - collapsed_slice_dims.ptr, - @intCast(operand_batching_dims.len), - operand_batching_dims.ptr, - @intCast(start_indices_batching_dims.len), - start_indices_batching_dims.ptr, - @intCast(start_index_map.len), - start_index_map.ptr, - index_vector_dim, - ), - }; + ) *const GatherDimensionNumbersAttribute { + return @ptrCast(c.stablehloGatherDimensionNumbersGet( + ctx.ptr(), + @intCast(offset_dims.len), + offset_dims.ptr, + @intCast(collapsed_slice_dims.len), + collapsed_slice_dims.ptr, + @intCast(operand_batching_dims.len), + operand_batching_dims.ptr, + @intCast(start_indices_batching_dims.len), + start_indices_batching_dims.ptr, + @intCast(start_index_map.len), + start_index_map.ptr, + index_vector_dim, + ).ptr); } - pub fn getOffsetDimsSize(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetOffsetDimsSize(self._inner)); + pub fn getOffsetDimsSize(self: *const GatherDimensionNumbersAttribute) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetOffsetDimsSize(self.ptr())); } - pub fn getOffsetDimsElem(self: Self, pos: usize) i64 { - return c.stablehloGatherDimensionNumbersGetOffsetDimsElem(self._inner, @intCast(pos)); + pub fn getOffsetDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloGatherDimensionNumbersGetOffsetDimsElem(self.ptr(), @intCast(pos)); } - pub fn getCollapsedSliceDimsSize(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsSize(self._inner)); + pub fn getCollapsedSliceDimsSize(self: *const GatherDimensionNumbersAttribute) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsSize(self.ptr())); } - pub fn getCollapsedSliceDimsElem(self: Self, pos: usize) i64 { - return c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsElem(self._inner, @intCast(pos)); + pub fn getCollapsedSliceDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsElem(self.ptr(), @intCast(pos)); } - pub fn getStartIndexMapSize(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetStartIndexMapSize(self._inner)); + pub fn getStartIndexMapSize(self: *const GatherDimensionNumbersAttribute) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetStartIndexMapSize(self.ptr())); } - pub fn getOperandBatchingDimsSize(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetOperandBatchingDimsSize(self._inner)); + pub fn getOperandBatchingDimsSize(self: *const GatherDimensionNumbersAttribute) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetOperandBatchingDimsSize(self.ptr())); } - pub fn getOperandBatchingDimsElem(self: Self, pos: usize) i64 { - return c.stablehloGatherDimensionNumbersGetOperandBatchingDimsElem(self._inner, @intCast(pos)); + pub fn getOperandBatchingDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloGatherDimensionNumbersGetOperandBatchingDimsElem(self.ptr(), @intCast(pos)); } - pub fn getStartIndicesBatchingDimsSize(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsSize(self._inner)); + pub fn getStartIndicesBatchingDimsSize(self: *const GatherDimensionNumbersAttribute) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsSize(self.ptr())); } - pub fn getStartIndicesBatchingDimsElem(self: Self, pos: usize) i64 { - return c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsElem(self._inner, @intCast(pos)); + pub fn getStartIndicesBatchingDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsElem(self.ptr(), @intCast(pos)); } - pub fn getStartIndexMapElem(self: Self, pos: usize) i64 { - return c.stablehloGatherDimensionNumbersGetStartIndexMapElem(self._inner, @intCast(pos)); + pub fn getStartIndexMapElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloGatherDimensionNumbersGetStartIndexMapElem(self.ptr(), @intCast(pos)); } - pub fn getIndexVectorDim(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetIndexVectorDim(self._inner)); + pub fn getIndexVectorDim(self: *const GatherDimensionNumbersAttribute) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetIndexVectorDim(self.ptr())); } }; -pub const ConvDimensionNumbersAttribute = struct { - _inner: c.MlirAttribute, +pub fn gatherDimensionNumbersAttribute( + ctx: *mlir.Context, + offset_dims: []const i64, + collapsed_slice_dims: []const i64, + operand_batching_dims: []const i64, + start_indices_batching_dims: []const i64, + start_index_map: []const i64, + index_vector_dim: i64, +) *const mlir.Attribute { + return @ptrCast(GatherDimensionNumbersAttribute.init( + ctx, + offset_dims, + collapsed_slice_dims, + operand_batching_dims, + start_indices_batching_dims, + start_index_map, + index_vector_dim, + )); +} - pub const is_a_fn = c.stablehloAttributeIsAConvDimensionNumbers; - const Self = ConvDimensionNumbersAttribute; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); +pub const ConvDimensionNumbersAttribute = opaque { + const M = mlir.Methods(ConvDimensionNumbersAttribute, mlir.Attribute); - pub fn init(ctx: mlir.Context, args: struct { + pub const isAFn = c.stablehloAttributeIsAConvDimensionNumbers; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const InitArgs = struct { input_batch_dimension: i64, input_feature_dimension: i64, input_spatial_dimensions: []const i64, @@ -996,106 +1057,116 @@ pub const ConvDimensionNumbersAttribute = struct { output_batch_dimension: i64, output_feature_dimension: i64, output_spatial_dimensions: []const i64, - }) Self { - return .{ - ._inner = c.stablehloConvDimensionNumbersGet( - ctx._inner, - args.input_batch_dimension, - args.input_feature_dimension, - @intCast(args.input_spatial_dimensions.len), - args.input_spatial_dimensions.ptr, - args.kernel_input_feature_dimension, - args.kernel_output_feature_dimension, - @intCast(args.kernel_spatial_dimensions.len), - args.kernel_spatial_dimensions.ptr, - args.output_batch_dimension, - args.output_feature_dimension, - @intCast(args.output_spatial_dimensions.len), - args.output_spatial_dimensions.ptr, - ), - }; + }; + + pub fn init(ctx: *mlir.Context, args: InitArgs) *const ConvDimensionNumbersAttribute { + return @ptrCast(c.stablehloConvDimensionNumbersGet( + ctx.ptr(), + args.input_batch_dimension, + args.input_feature_dimension, + @intCast(args.input_spatial_dimensions.len), + args.input_spatial_dimensions.ptr, + args.kernel_input_feature_dimension, + args.kernel_output_feature_dimension, + @intCast(args.kernel_spatial_dimensions.len), + args.kernel_spatial_dimensions.ptr, + args.output_batch_dimension, + args.output_feature_dimension, + @intCast(args.output_spatial_dimensions.len), + args.output_spatial_dimensions.ptr, + ).ptr); } - pub fn getInputBatchDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetInputBatchDimension(self._inner); + pub fn getInputBatchDimension(self: *const ConvDimensionNumbersAttribute) i64 { + return c.stablehloConvDimensionNumbersGetInputBatchDimension(self.ptr()); } - pub fn getInputFeatureDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetInputFeatureDimension(self._inner); + pub fn getInputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { + return c.stablehloConvDimensionNumbersGetInputFeatureDimension(self.ptr()); } - pub fn getInputSpatialDimensionsSize(self: Self) usize { - return @intCast(c.stablehloConvDimensionNumbersGetInputSpatialDimensionsSize(self._inner)); + pub fn getInputSpatialDimensionsSize(self: *const ConvDimensionNumbersAttribute) usize { + return @intCast(c.stablehloConvDimensionNumbersGetInputSpatialDimensionsSize(self.ptr())); } - pub fn getInputSpatialDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloConvDimensionNumbersGetInputSpatialDimensionsElem(self._inner, @intCast(pos)); + pub fn getInputSpatialDimensionsElem(self: *const ConvDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloConvDimensionNumbersGetInputSpatialDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn getKernelInputFeatureDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetKernelInputFeatureDimension(self._inner); + pub fn getKernelInputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { + return c.stablehloConvDimensionNumbersGetKernelInputFeatureDimension(self.ptr()); } - pub fn getKernelOutputFeatureDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetKernelOutputFeatureDimension(self._inner); + pub fn getKernelOutputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { + return c.stablehloConvDimensionNumbersGetKernelOutputFeatureDimension(self.ptr()); } - pub fn getKernelSpatialDimensionsSize(self: Self) usize { - return @intCast(c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsSize(self._inner)); + pub fn getKernelSpatialDimensionsSize(self: *const ConvDimensionNumbersAttribute) usize { + return @intCast(c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsSize(self.ptr())); } - pub fn getKernelSpatialDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsElem(self._inner, @intCast(pos)); + pub fn getKernelSpatialDimensionsElem(self: *const ConvDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn getOutputBatchDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetOutputBatchDimension(self._inner); + pub fn getOutputBatchDimension(self: *const ConvDimensionNumbersAttribute) i64 { + return c.stablehloConvDimensionNumbersGetOutputBatchDimension(self.ptr()); } - pub fn getOutputFeatureDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetOutputFeatureDimension(self._inner); + pub fn getOutputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { + return c.stablehloConvDimensionNumbersGetOutputFeatureDimension(self.ptr()); } - pub fn getOutputSpatialDimensionsSize(self: Self) usize { - return @intCast(c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsSize(self._inner)); + pub fn getOutputSpatialDimensionsSize(self: *const ConvDimensionNumbersAttribute) usize { + return @intCast(c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsSize(self.ptr())); } - pub fn getOutputSpatialDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsElem(self._inner, @intCast(pos)); + pub fn getOutputSpatialDimensionsElem(self: *const ConvDimensionNumbersAttribute, pos: usize) i64 { + return c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsElem(self.ptr(), @intCast(pos)); } }; -pub const OutputOperandAliasAttribute = struct { - _inner: c.MlirAttribute, +pub fn convDimensionNumbersAttribute(ctx: *mlir.Context, args: ConvDimensionNumbersAttribute.InitArgs) *const mlir.Attribute { + return @ptrCast(ConvDimensionNumbersAttribute.init(ctx, args)); +} - pub const is_a_fn = c.stablehloAttributeIsAOutputOperandAlias; - pub const asAttr = mlir.Attribute.fromAny(OutputOperandAliasAttribute); - pub const eql = mlir.Attribute.eqlAny(OutputOperandAliasAttribute); +pub const OutputOperandAliasAttribute = opaque { + const M = mlir.Methods(OutputOperandAliasAttribute, c.MlirAttribute); - pub fn init( - ctx: mlir.Context, - output_tuple_indices: []const i64, + pub const isAFn = c.stablehloAttributeIsAOutputOperandAlias; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const InitArgs = struct { + output_tuple_indices: []const i64 = &.{}, operand_index: i64, - operand_tuple_indices: []const i64, - ) OutputOperandAliasAttribute { - return .{ ._inner = c.stablehloOutputOperandAliasGet( - ctx._inner, - @intCast(output_tuple_indices.len), - output_tuple_indices.ptr, - @intCast(operand_index), - @intCast(operand_tuple_indices.len), - operand_tuple_indices.ptr, - ) }; + operand_tuple_indices: []const i64 = &.{}, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const OutputOperandAliasAttribute { + return @ptrCast(c.stablehloOutputOperandAliasGet( + ctx.ptr(), + @intCast(args.output_tuple_indices.len), + @ptrCast(args.output_tuple_indices), + @intCast(args.operand_index), + @intCast(args.operand_tuple_indices.len), + @ptrCast(args.operand_tuple_indices), + ).ptr); } }; -pub const PrecisionAttribute = struct { - _inner: c.MlirAttribute, +pub fn outputOperandAliasAttribute(ctx: *mlir.Context, args: OutputOperandAliasAttribute.InitArgs) *const mlir.Attribute { + return @ptrCast(OutputOperandAliasAttribute.get(ctx, args)); +} - pub const is_a_fn = c.stablehloAttributeIsAPrecisionAttr; - const Self = PrecisionAttribute; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); +pub const PrecisionAttribute = opaque { + const M = mlir.Methods(PrecisionAttribute, mlir.Attribute); + + pub const isAFn = c.stablehloAttributeIsAPrecisionAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const Precision = enum { DEFAULT, @@ -1103,23 +1174,27 @@ pub const PrecisionAttribute = struct { HIGHEST, }; - pub fn init(ctx: mlir.Context, value: Precision) Self { - return .{ ._inner = c.stablehloPrecisionAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; + pub fn init(ctx: *mlir.Context, value: Precision) *const PrecisionAttribute { + return @ptrCast(c.stablehloPrecisionAttrGet(ctx.ptr(), stringRef(@tagName(value))).ptr); } - pub fn getValue(self: Self) Precision { - const value = mlir.fromStringRef(c.stablehloPrecisionAttrGetValue(self._inner)); + pub fn getValue(self: *const PrecisionAttribute) Precision { + const value = mlir.fromStringRef(c.stablehloPrecisionAttrGetValue(self.ptr())); return std.meta.stringToEnum(Precision, value) orelse unreachable; } }; -pub const ComparisonDirection = struct { - _inner: c.MlirAttribute, +pub fn precisionAttribute(ctx: *mlir.Context, precision: PrecisionAttribute.Precision) *const mlir.Attribute { + return @ptrCast(PrecisionAttribute.init(ctx, precision)); +} - pub const is_a_fn = c.stablehloAttributeIsAComparisonDirectionAttr; - const Self = ComparisonDirection; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); +pub const ComparisonDirection = opaque { + const M = mlir.Methods(ComparisonDirection, c.MlirAttribute); + + pub const isAFn = c.stablehloAttributeIsAComparisonDirectionAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const Direction = enum { EQ, @@ -1130,23 +1205,27 @@ pub const ComparisonDirection = struct { LT, }; - pub fn init(ctx: mlir.Context, value: Direction) Self { - return .{ ._inner = c.stablehloComparisonDirectionAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; + pub fn init(ctx: *mlir.Context, value: Direction) *const ComparisonDirection { + return @ptrCast(c.stablehloComparisonDirectionAttrGet(ctx.ptr(), mlir.stringRef(@tagName(value))).ptr); } - pub fn getValue(self: Self) Direction { - const value = mlir.fromStringRef(c.stablehloComparisonDirectionAttrGetValue(self._inner)); + pub fn getValue(self: *const ComparisonDirection) Direction { + const value = mlir.fromStringRef(c.stablehloComparisonDirectionAttrGetValue(self.ptr())); return std.meta.stringToEnum(Direction, value) orelse unreachable; } }; -pub const CompareType = struct { - _inner: c.MlirAttribute, +pub fn comparisonDirection(ctx: *mlir.Context, value: ComparisonDirection.Direction) *const mlir.Attribute { + return @ptrCast(ComparisonDirection.init(ctx, value)); +} - pub const is_a_fn = c.stablehloAttributeIsAComparisonTypeAttr; - const Self = CompareType; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); +pub const CompareType = opaque { + const M = mlir.Methods(CompareType, c.MlirAttribute); + + pub const isAFn = c.stablehloAttributeIsAComparisonTypeAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const Type = enum { SIGNED, @@ -1155,23 +1234,27 @@ pub const CompareType = struct { TOTALORDER, }; - pub fn init(ctx: mlir.Context, value: Type) Self { - return .{ ._inner = c.stablehloComparisonTypeAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; + pub fn init(ctx: *mlir.Context, value: Type) *const CompareType { + return @ptrCast(c.stablehloComparisonTypeAttrGet(ctx.ptr(), stringRef(@tagName(value))).ptr); } - pub fn getValue(self: Self) Type { - const value = mlir.fromStringRef(c.stablehloComparisonTypeAttrGetValue(self._inner)); + pub fn getValue(self: *const CompareType) Type { + const value = mlir.fromStringRef(c.stablehloComparisonTypeAttrGetValue(self.ptr())); return std.meta.stringToEnum(Type, value) orelse unreachable; } }; -pub const Transpose = struct { - _inner: c.MlirAttribute, +pub fn compareType(ctx: *mlir.Context, value: CompareType.Type) *const mlir.Attribute { + return @ptrCast(CompareType.init(ctx, value)); +} - pub const is_a_fn = c.stablehloAttributeIsATransposeAttr; - const Self = Transpose; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); +pub const TransposeAttribute = opaque { + const M = mlir.Methods(TransposeAttribute, mlir.Attribute); + + pub const isAFn = c.stablehloAttributeIsATransposeAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const Type = enum { NO_TRANSPOSE, @@ -1179,23 +1262,27 @@ pub const Transpose = struct { ADJOINT, }; - pub fn init(ctx: mlir.Context, value: Type) Self { - return .{ ._inner = c.stablehloTransposeAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; + pub fn init(ctx: *mlir.Context, value: Type) *const TransposeAttribute { + return @ptrCast(c.stablehloTransposeAttrGet(ctx.ptr(), mlir.stringRef(@tagName(value))).ptr); } - pub fn getValue(self: Self) Type { - const value = mlir.fromStringRef(c.stablehloTransposeAttrGetValue(self._inner)); + pub fn getValue(self: *const TransposeAttribute) Type { + const value = mlir.fromStringRef(c.stablehloTransposeAttrGetValue(self.ptr())); return std.meta.stringToEnum(Type, value) orelse unreachable; } }; -pub const FftType = struct { - _inner: c.MlirAttribute, +pub fn transposeAttribute(ctx: *mlir.Context, value: TransposeAttribute.Type) *const mlir.Attribute { + return @ptrCast(TransposeAttribute.init(ctx, value)); +} - pub const is_a_fn = c.stablehloAttributeIsAFftTypeAttr; - const Self = FftType; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); +pub const FftType = opaque { + const M = mlir.Methods(FftType, mlir.Attribute); + + pub const isAFn = c.stablehloAttributeIsAFftTypeAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const Type = enum { FFT, @@ -1204,114 +1291,45 @@ pub const FftType = struct { IRFFT, }; - pub fn init(ctx: mlir.Context, value: Type) Self { - return .{ ._inner = c.stablehloFftTypeAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; + pub fn init(ctx: *mlir.Context, value: Type) *const FftType { + return @ptrCast(c.stablehloFftTypeAttrGet(ctx.ptr(), mlir.stringRef(@tagName(value))).ptr); } - pub fn getValue(self: Self) Type { - const value = mlir.fromStringRef(c.stablehloFftTypeAttrGetValue(self._inner)); + pub fn getValue(self: *const FftType) Type { + const value = mlir.fromStringRef(c.stablehloFftTypeAttrGetValue(self.ptr())); return std.meta.stringToEnum(Type, value) orelse unreachable; } }; -pub const RngDistribution = struct { - _inner: c.MlirAttribute, - - pub const is_a_fn = c.stablehloAttributeIsARngDistributionAttr; - const Self = RngDistribution; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); - - pub const Type = enum { - UNIFORM, - NORMAL, - }; - - pub fn init(ctx: mlir.Context, value: Type) Self { - return .{ ._inner = c.stablehloRngDistributionAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; - } - - pub fn getValue(self: Self) Type { - const value = mlir.fromStringRef(c.stablehloRngDistributionAttrGetValue(self._inner)); - return std.meta.stringToEnum(Type, value) orelse unreachable; - } -}; - -pub const RngAlgorithm = struct { - _inner: c.MlirAttribute, - - pub const is_a_fn = c.stablehloAttributeIsARngAlgorithmAttr; - const Self = RngAlgorithm; - pub const asAttr = mlir.Attribute.fromAny(Self); - pub const eql = mlir.Attribute.eqlAny(Self); - - pub const Type = enum { - DEFAULT, - THREE_FRY, - PHILOX, - }; - - pub fn init(ctx: mlir.Context, value: Type) Self { - return .{ ._inner = c.stablehloRngAlgorithmAttrGet(ctx._inner, mlir.stringRef(@tagName(value))) }; - } - - pub fn getValue(self: Self) Type { - const value = mlir.fromStringRef(c.stablehloRngAlgorithmAttrGetValue(self._inner)); - return std.meta.stringToEnum(Type, value) orelse unreachable; - } -}; - -pub fn stablehloVersionFromCompatibilityRequirement(requirement: c.MlirStablehloCompatibilityRequirement) []const u8 { - const state = struct { - var buf: [32]u8 = undefined; - - fn call(req: c.MlirStablehloCompatibilityRequirement) []u8 { - var stream = std.io.fixedBufferStream(&buf); - var context = .{ .writer = stream.writer() }; - const WriterContext = @TypeOf(context); - - c.stablehloVersionFromCompatibilityRequirement(req, (struct { - pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.c) void { - const inner_ctx: *WriterContext = @ptrCast(@alignCast(userdata)); - _ = inner_ctx.writer.write(mlir.fromStringRef(mlir_str)) catch unreachable; - } - }).callback, &context); - - return buf[0..stream.pos]; - } - }; - - return state.call(requirement); +pub fn fftType(ctx: *mlir.Context, value: FftType.Type) *const mlir.Attribute { + return @ptrCast(FftType.init(ctx, value)); } -pub fn stablehloGetSmallerVersion(version1: []const u8, version2: []const u8) []const u8 { - const Cmp = struct { - v1: []const u8, - v1_is_smaller: bool = undefined, - - pub fn smallerCb(smaller_version: c.MlirStringRef, opaque_cmp: ?*anyopaque) callconv(.c) void { - var cmp: *@This() = @ptrCast(@alignCast(opaque_cmp)); - cmp.v1_is_smaller = std.mem.eql(u8, cmp.v1, smaller_version.data[0..smaller_version.length]); - } - }; - - var cmp_ctx: Cmp = .{ .v1 = version1 }; - const cmp_res = c.stablehloGetSmallerVersion(mlir.stringRef(version1), mlir.stringRef(version2), Cmp.smallerCb, &cmp_ctx); - - std.debug.assert(cmp_res.value != 0); - return if (cmp_ctx.v1_is_smaller) version1 else version2; +fn stringFromStream(buf: []u8, streamFn: anytype, args: anytype) []const u8 { + var writer = std.Io.Writer.fixed(buf); + var sctx: mlir.StringCallbackCtx = .{ .writer = &writer }; + _ = @call(.auto, streamFn, args ++ .{ mlir.stringCallback, &sctx }); + return writer.buffered(); } -pub fn getCurrentVersion() []const u8 { +pub fn versionFromCompatibilityRequirement(buffer: []u8, requirement: c.MlirStablehloCompatibilityRequirement) []const u8 { + return stringFromStream(buffer, c.stablehloVersionFromCompatibilityRequirement, .{requirement}); +} + +pub fn smallerVersion(version1: []const u8, version2: []const u8) []const u8 { + var buf: [32]u8 = undefined; + const result = stringFromStream(&buf, c.stablehloGetSmallerVersion, .{ stringRef(version1), stringRef(version2) }); + return if (std.mem.eql(u8, result, version1)) version1 else version2; +} + +pub fn currentVersion() []const u8 { const state = struct { var buf: [32]u8 = undefined; var str: []const u8 = undefined; var once = std.once(call); fn call() void { - var writer: std.Io.Writer = .fixed(&buf); - c.stablehloGetCurrentVersion(printCallbackNoFail, &writer); - str = writer.buffered(); + str = stringFromStream(&buf, c.stablehloGetCurrentVersion, .{}); } }; @@ -1319,16 +1337,14 @@ pub fn getCurrentVersion() []const u8 { return state.str; } -pub fn getMinimumVersion() []const u8 { +pub fn minimumVersion() []const u8 { const state = struct { var buf: [32]u8 = undefined; var str: []const u8 = undefined; var once = std.once(call); fn call() void { - var writer: std.Io.Writer = .fixed(&buf); - c.stablehloGetMinimumVersion(printCallbackNoFail, &writer); - str = writer.buffered(); + str = stringFromStream(&buf, c.stablehloGetMinimumVersion, .{}); } }; @@ -1336,25 +1352,51 @@ pub fn getMinimumVersion() []const u8 { return state.str; } -pub fn serializePortableArtifact( - bytecode: []const u8, - target_version: []const u8, - writer: *std.Io.Writer, -) error{ InvalidMlirBytecodeVersion, WriteFailed }!void { - var writer_err: mlir.WriterWithErr = .{ .writer = writer }; - try mlir.successOr( - c.stablehloSerializePortableArtifactFromStringRef( - mlir.stringRef(bytecode), - mlir.stringRef(target_version), - mlir.WriterWithErr.printCallback, - &writer_err, - ), - error.InvalidMlirBytecodeVersion, +pub fn serializePortableArtifact(bytecode: []const u8, target_version: []const u8, writer: *std.Io.Writer) !void { + var sctx: mlir.StringCallbackCtx = .{ .writer = writer }; + const result = c.stablehloSerializePortableArtifactFromStringRef( + stringRef(bytecode), + stringRef(target_version), + mlir.stringCallback, + &sctx, ); - return try writer_err.check(); + if (sctx.err) |err| { + return err; + } + if (result.value == 0) { + return error.InvalidMlirBytecodeVersion; + } } -fn printCallbackNoFail(mlir_str: c.MlirStringRef, opaque_writer: ?*anyopaque) callconv(.c) void { - const writer: *std.Io.Writer = @ptrCast(@alignCast(opaque_writer)); - writer.writeAll(mlir.fromStringRef(mlir_str)) catch @panic("Failed to write MLIR"); +pub fn serializePortableArtifact2(module: *mlir.Module, target_version: []const u8, writer: *std.Io.Writer) !void { + var sctx: mlir.StringCallbackCtx = .{ .writer = writer }; + const result = c.stablehloSerializePortableArtifactFromModule( + module.ptr(), + stringRef(target_version), + mlir.stringCallback, + &sctx, + true, + ); + if (sctx.err) |err| { + return err; + } + if (c.mlirLogicalResultIsFailure(result)) { + return error.InvalidMlirBytecodeVersion; + } +} + +pub fn return_(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.return", .{ + .operands = .{ .flat = &.{&.{value}} }, + .location = location, + .verify = false, + }); +} + +pub fn returns(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.return", .{ + .operands = .{ .variadic = &.{values} }, + .verify = false, + .location = location, + }); } diff --git a/mlir/dialects/tensor.zig b/mlir/dialects/tensor.zig deleted file mode 100644 index 8d7de344..00000000 --- a/mlir/dialects/tensor.zig +++ /dev/null @@ -1,36 +0,0 @@ -const std = @import("std"); -const mlir = @import("mlir"); - -pub fn empty(ctx: mlir.Context, args: struct { - result: mlir.Type, - location: mlir.Location, -}) mlir.Operation { - return mlir.Operation.make(ctx, "tensor.empty", .{ - .results = &.{args.result}, - .location = args.location, - }); -} - -pub fn splat(ctx: mlir.Context, args: struct { - value: mlir.Value, - result: mlir.Type, - location: mlir.Location, -}) mlir.Operation { - return mlir.Operation.make(ctx, "tensor.splat", .{ - .operands = &.{args.value}, - .results = &.{args.result}, - .location = args.location, - }); -} - -pub fn cast(ctx: mlir.Context, args: struct { - source: mlir.Value, - dest: mlir.Type, - location: mlir.Location, -}) mlir.Operation { - return mlir.Operation.make(ctx, "tensor.cast", .{ - .operands = &.{args.source}, - .results = &.{args.dest}, - .location = args.location, - }); -} diff --git a/mlir/mlir.zig b/mlir/mlir.zig old mode 100755 new mode 100644 index a3b9114c..3d3e4b82 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -1,5 +1,4 @@ const std = @import("std"); -const builtin = @import("builtin"); const c = @import("c"); const stdx = @import("stdx"); @@ -8,8 +7,6 @@ const log = std.log.scoped(.mlir); test { std.testing.refAllDeclsRecursive(@This()); - - _ = try Context.init(); } pub const Error = error{ @@ -19,924 +16,1190 @@ pub const Error = error{ MlirUnexpected, /// A resource/executor was not found. NotFound, - /// Bytecode version incompatibility. - InvalidMlirBytecodeVersion, - OutOfMemory, -}; +} || std.mem.Allocator.Error; pub inline fn stringRef(str: []const u8) c.MlirStringRef { return .{ .data = str.ptr, .length = str.len }; } -pub inline fn fromStringRef(str: c.MlirStringRef) []const u8 { +pub inline fn string(str: c.MlirStringRef) []const u8 { // Note: mlir.StringRef need not to be null terminated. return str.data[0..str.length]; } +pub const StringCallbackCtx = struct { + writer: *std.Io.Writer, + err: ?std.Io.Writer.Error = null, +}; + +pub fn stringCallback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { + const ctx: *StringCallbackCtx = @ptrCast(@alignCast(ctx_)); + ctx.writer.writeAll(string(str)) catch |err| { + ctx.err = err; + }; +} + +fn stringFromStream(buf: []u8, streamFn: anytype, args: anytype) []const u8 { + var writer = std.Io.Writer.fixed(buf); + var sctx: StringCallbackCtx = .{ .writer = &writer }; + _ = @call(.auto, streamFn, args ++ .{ stringCallback, &sctx }); + return writer.buffered(); +} + pub fn registerPasses(comptime passes: []const u8) void { @field(c, "mlirRegister" ++ passes ++ "Passes")(); } -pub fn successOr(res: c.MlirLogicalResult, err: anytype) @TypeOf(err)!void { - return if (res.value == 0) err else {}; +fn isPtrConst(comptime T: type) bool { + return @typeInfo(T).pointer.is_const; } -pub const Registry = struct { - _inner: c.MlirDialectRegistry, +fn CastedPtr(comptime T1: type, comptime T2: type) type { + return if (@typeInfo(T1).pointer.is_const) *const T2 else *T2; +} - pub const deinit = helpers.deinit(Registry, c.mlirDialectRegistryDestroy); +pub fn Methods(comptime T: type, comptime NativeT: type) type { + return struct { + pub const IsPtrConst = @typeInfo(@typeInfo(@FieldType(NativeT, "ptr")).optional.child).pointer.is_const; + pub const Ptr = if (IsPtrConst) *const T else *T; - pub fn init() !Registry { - const registry = c.mlirDialectRegistryCreate(); - return .{ ._inner = .{ .ptr = registry.ptr orelse return Error.MlirUnexpected } }; + pub fn ptr(self: anytype) NativeT { + return .{ .ptr = @ptrCast(@constCast(self)) }; + } + + fn format(f: anytype) fn (self: *const T, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return struct { + fn format(self: *const T, writer: *std.Io.Writer) std.Io.Writer.Error!void { + var ctx: StringCallbackCtx = .{ .writer = writer }; + f(self.ptr(), stringCallback, &ctx); + if (ctx.err) |err| { + return err; + } + } + }.format; + } + + pub fn eql(f: anytype) fn (self: *const T, other: *const T) bool { + return struct { + fn eql(self: *const T, other: *const T) bool { + return f(self.ptr(), other.ptr()); + } + }.eql; + } + + pub fn isA(self: *const T, OtherType: type) ?CastedPtr(@TypeOf(self), OtherType) { + return if (OtherType.isAFn(self.ptr())) @ptrCast(self) else null; + } + }; +} + +pub const DialectRegistry = opaque { + const M = Methods(DialectRegistry, c.MlirDialectRegistry); + + pub const ptr = M.ptr; + + pub fn init() std.mem.Allocator.Error!*DialectRegistry { + return @ptrCast(c.mlirDialectRegistryCreate().ptr orelse return std.mem.Allocator.Error.OutOfMemory); + } + + pub fn deinit(self: *DialectRegistry) void { + c.mlirDialectRegistryDestroy(self.ptr()); + } + + pub fn registerDialect(self: *DialectRegistry, comptime name: []const u8) void { + DialectHandle.fromString(name).insertDialect(self); } }; -pub const Context = struct { - _inner: c.MlirContext, - pub const deinit = helpers.deinit(Context, c.mlirContextDestroy); - pub const wrapOr = helpers.wrapOr(Context, c.mlirContextIsNull); +pub const PassManager = opaque { + const M = Methods(PassManager, c.MlirPassManager); - pub fn init() !Context { - return Context.wrapOr(c.mlirContextCreate()) orelse Error.MlirUnexpected; + pub const ptr = M.ptr; + + pub fn init(ctx: *Context) *PassManager { + return @ptrCast(c.mlirPassManagerCreate(ctx.ptr()).ptr); } - pub fn initWithRegistry(registry: Registry, threadingEnabled: bool) !Context { - return Context.wrapOr( - c.mlirContextCreateWithRegistry(registry._inner, threadingEnabled), - ) orelse Error.InvalidMlir; + pub fn deinit(self: *const PassManager) void { + c.mlirPassManagerDestroy(self.ptr()); + } + + pub fn asOpPassManager(self: *PassManager) *OpPassManager { + return @ptrCast(c.mlirPassManagerGetAsOpPassManager(self.ptr()).ptr); + } + + pub fn runOnOp(self: *const PassManager, op: *Operation) !void { + const result = c.mlirPassManagerRunOnOp(self.ptr(), op.ptr()); + if (result.value == 0) { + return error.MlirUnexpected; + } + } + + pub const EnableIRPrintingOpts = struct { + printBeforeAll: bool = false, + printAfterAll: bool = false, + printModuleScope: bool = false, + printAfterOnlyOnChange: bool = false, + printAfterOnlyOnFailure: bool = false, + flags: Operation.PrintFlags = .{}, + treePrintingPath: []const u8 = "", + }; + + pub fn enableIRPrinting(self: *PassManager, opts: EnableIRPrintingOpts) void { + const flags = opts.flags.createNative(); + defer c.mlirOpPrintingFlagsDestroy(flags); + + c.mlirPassManagerEnableIRPrinting( + self.ptr(), + opts.printBeforeAll, + opts.printAfterAll, + opts.printModuleScope, + opts.printAfterOnlyOnChange, + opts.printAfterOnlyOnFailure, + flags, + stringRef(opts.treePrintingPath), + ); + } + +}; + +pub const OpPassManager = opaque { + const M = Methods(OpPassManager, c.MlirOpPassManager); + + pub const ptr = M.ptr; + pub const format = M.format(c.mlirPrintPassPipeline); + + pub fn addPass(self: *OpPassManager, pass: *Pass) void { + c.mlirOpPassManagerAddOwnedPass(self.ptr(), pass.ptr()); + } + + pub fn addPasses(self: *OpPassManager, passes: []const *Pass) void { + for (passes) |pass| { + self.addPass(pass); + } + } + + pub fn addPipeline(self: *OpPassManager, pipeline_name: []const u8) !void { + var buffer: [1024]u8 = undefined; + const result = stringFromStream(&buffer, c.mlirOpPassManagerAddPipeline, .{ self.ptr(), stringRef(pipeline_name) }); + if (result.len != 0) { + log.err("mlir error: {s}", .{result}); + return error.MlirUnexpected; + } + } +}; + +pub const Pass = opaque { + const M = Methods(Pass, c.MlirPass); + + pub const ptr = M.ptr; + + pub fn create(comptime name: @TypeOf(.enum_literal)) *Pass { + return @ptrCast(@field(c, "mlirCreate" ++ @tagName(name))().ptr); + } +}; + +pub const Context = opaque { + const M = Methods(Context, c.MlirContext); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirContextEqual); + + pub const InitOptions = struct { + registry: ?*const DialectRegistry = null, + threading: bool = false, + }; + + pub fn init(opts: InitOptions) std.mem.Allocator.Error!*Context { + const ctx = if (opts.registry) |r| c.mlirContextCreateWithRegistry( + r.ptr(), + opts.threading, + ) else c.mlirContextCreateWithThreading(opts.threading); + return @ptrCast(ctx.ptr orelse return std.mem.Allocator.Error.OutOfMemory); + } + + pub fn deinit(self: *Context) void { + c.mlirContextDestroy(self.ptr()); } pub fn setMultiThreading(self: *Context, enabled: bool) void { - c.mlirContextEnableMultithreading(self._inner, enabled); + c.mlirContextEnableMultithreading(self.ptr(), enabled); } - pub fn appendDialectRegistry(self: *Context, registry: Registry) void { - c.mlirContextAppendDialectRegistry(self._inner, registry._inner); + pub fn appendDialectRegistry(self: *Context, registry: *const DialectRegistry) void { + c.mlirContextAppendDialectRegistry(self.ptr(), registry.ptr()); } pub fn loadAllAvailableDialects(self: *Context) void { - c.mlirContextLoadAllAvailableDialects(self._inner); + c.mlirContextLoadAllAvailableDialects(self.ptr()); } - pub fn numRegisteredDialects(self: Context) usize { - return @intCast(c.mlirContextGetNumRegisteredDialects(self._inner)); + pub fn numRegisteredDialects(self: *const Context) usize { + return @intCast(c.mlirContextGetNumRegisteredDialects(self.ptr())); } - pub fn numLoadedDialects(self: Context) usize { - return @intCast(c.mlirContextGetNumLoadedDialects(self._inner)); + pub fn numLoadedDialects(self: *const Context) usize { + return @intCast(c.mlirContextGetNumLoadedDialects(self.ptr())); } - pub fn isRegisteredOperation(self: Context, op: [:0]const u8) bool { - return c.mlirContextIsRegisteredOperation(self._inner, stringRef(op)); + pub fn numThreads(self: *const Context) usize { + return @intCast(c.mlirContextGetNumThreads(self.ptr())); } - pub fn location(self: Context, src: std.builtin.SourceLocation) Location { - return Location.fromSrc(self, src); + pub fn isRegisteredOperation(self: *const Context, op: []const u8) bool { + return c.mlirContextIsRegisteredOperation(self.ptr(), stringRef(op)); + } + + pub fn format(self: *const Context, writer: *std.Io.Writer) !void { + try writer.print("{s}{{ .numThreads = {d}, .numRegisteredDialects = {d}, .numLoadedDialects = {d} }}", .{ + @typeName(Context), + self.numThreads(), + self.numRegisteredDialects(), + self.numLoadedDialects(), + }); } }; -pub const Module = struct { - _inner: c.MlirModule, +pub const Location = opaque { + const M = Methods(Location, c.MlirLocation); - pub const deinit = helpers.deinit(Module, c.mlirModuleDestroy); - pub const wrapOr = helpers.wrapOr(Module, c.mlirModuleIsNull); + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirLocationEqual); + pub const format = M.format(c.mlirLocationPrint); - pub fn init(loc: Location) Module { - return .{ ._inner = c.mlirModuleCreateEmpty(loc._inner) }; + pub fn fromSrc(ctx: *Context, src: std.builtin.SourceLocation) *const Location { + return fileLineCol( + ctx, + src.file, + src.line, + src.column, + ); } - pub fn parse(ctx: Context, source: [:0]const u8) !Module { - return Module.wrapOr( - c.mlirModuleCreateParse(ctx._inner, stringRef(source)), - ) orelse Error.InvalidMlir; + pub fn fileLineCol(ctx: *Context, file: []const u8, line: usize, column: usize) *const Location { + return @ptrCast(c.mlirLocationFileLineColGet( + ctx.ptr(), + stringRef(file), + @intCast(line), + @intCast(column), + ).ptr); } - pub fn fromOperation(operation: Operation) Module { - return .{ ._inner = c.mlirModuleFromOperation(operation._inner) }; + pub fn callSite(callee: *const Location, caller: *const Location) *const Location { + return @ptrCast(c.mlirLocationCallSiteGet(callee.ptr(), caller.ptr()).ptr); } - pub fn context(self: Module) Context { - return .{ ._inner = c.mlirModuleGetContext(self._inner) }; + pub fn fused(ctx: *Context, locs: []const *const Location, metadata: *const Attribute) *const Location { + return @ptrCast(c.mlirLocationFusedGet( + ctx.ptr(), + @intCast(locs.len), + @ptrCast(locs), + metadata.ptr(), + ).ptr); } - pub fn getBody(self: Module) Block { - return .{ ._inner = c.mlirModuleGetBody(self._inner) }; + pub fn named(self: *const Location, ctx: *Context, loc_name: []const u8) *const Location { + return @ptrCast(c.mlirLocationNameGet(ctx.ptr(), stringRef(loc_name), self.ptr()).ptr); } - pub fn op(self: Module) Operation { - return .{ ._inner = c.mlirModuleGetOperation(self._inner) }; + pub fn namedFmt(loc: *const Location, ctx: *Context, comptime fmt: []const u8, args: anytype) *const Location { + var buf: [256]u8 = undefined; + return loc.named(loc, ctx, std.fmt.bufPrint(&buf, fmt, args) catch blk: { + buf[buf.len - 3 ..].* = "...".*; + break :blk &buf; + }); } - pub fn hash(self: Module, hasher: *std.hash.XxHash64) void { - return self.op().hash(hasher); + pub fn fromAttribute(attribute: *const Attribute) *const Location { + return @ptrCast(c.mlirLocationFromAttribute(attribute.ptr()).ptr); + } + + pub fn unknown(ctx: *Context) *const Location { + return @ptrCast(c.mlirLocationUnknownGet(ctx.ptr()).ptr); } }; -pub const PassManager = struct { - _inner: c.MlirPassManager, +pub const Type = opaque { + const M = Methods(Type, c.MlirType); - pub const deinit = helpers.deinit(PassManager, c.mlirPassManagerDestroy); - pub const wrapOr = helpers.wrapOr(PassManager, c.mlirPassManagerIsNull); + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; - pub fn init(ctx: Context) !PassManager { - return PassManager.wrapOr( - c.mlirPassManagerCreate(ctx._inner), - ) orelse Error.MlirUnexpected; - } - - pub fn initOnOperation(ctx: Context, op: [:0]const u8) !PassManager { - return PassManager.wrapOr( - c.mlirPassManagerCreateOnOperation(ctx._inner, stringRef(op)), - ) orelse Error.MlirUnexpected; - } - - pub fn asOpPassManager(self: PassManager) OpPassManager { - return .{ ._inner = c.mlirPassManagerGetAsOpPassManager(self._inner) }; - } - - // TODO mlirPassManagerEnableIRPrinting - // pub fn enableIRPrinting(self: *PassManager) void {} - - pub fn runOnOp(self: *PassManager, op: Operation) error{InvalidMlir}!void { - if (c.mlirPassManagerRunOnOp(self._inner, op._inner).value == 0) { - return Error.InvalidMlir; - } + pub fn parse(ctx: *Context, str: []const u8) Error!*const Type { + return @ptrCast(c.mlirTypeParseGet(ctx.ptr(), stringRef(str)).ptr orelse return Error.InvalidMlir); } }; -fn _mlir_passpipeline_error(err: c.MlirStringRef, ctx: ?*anyopaque) callconv(.c) void { - _ = ctx; - std.debug.print(">>ERROR: {s}\n", .{err.data}); +pub const IndexType = opaque { + const M = Methods(IndexType, c.MlirType); + + pub const isAFn = c.mlirTypeIsAIndex; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn get(ctx: *Context) *const IndexType { + return @ptrCast(c.mlirIndexTypeGet(ctx.ptr()).ptr); + } +}; + +pub fn indexType(ctx: *Context) *const Type { + return @ptrCast(IndexType.get(ctx)); } -pub const OpPassManager = struct { - _inner: c.MlirOpPassManager, - - pub fn addPipeline(self: *OpPassManager, pipeline: [:0]const u8) error{OutOfMemory}!void { - if (c.mlirOpPassManagerAddPipeline( - self._inner, - stringRef(pipeline), - &_mlir_passpipeline_error, - null, - ).value == 0) { - return Error.OutOfMemory; - } - } -}; - -pub const Identifier = struct { - _inner: c.MlirIdentifier, - - pub fn get(ctx: Context, str_: [:0]const u8) Identifier { - return .{ ._inner = c.mlirIdentifierGet(ctx._inner, stringRef(str_)) }; - } - - pub fn context(self: Identifier) Context { - return .{ ._inner = c.mlirIdentifierGetContext(self._inner) }; - } - - pub fn str(self: Identifier) []const u8 { - return fromStringRef(c.mlirIdentifierStr(self._inner)); - } - - pub fn equals(self: Identifier, other: Identifier) bool { - return c.mlirIdentifierEqual(self._inner, other._inner); - } -}; - -pub const AttrTuple = struct { [:0]const u8, Attribute }; - -pub const Attribute = struct { - _inner: c.MlirAttribute, - - pub const dump = helpers.dump(Attribute, c.mlirAttributeDump); - pub const eql = helpers.eql(Attribute, c.mlirAttributeEqual); - pub const format = helpers.format(Attribute, c.mlirAttributePrint); - pub const wrapOr = helpers.wrapOr(Attribute, c.mlirAttributeIsNull); - - pub fn wrap(c_attr: c.MlirAttribute) Attribute { - return .{ ._inner = c_attr }; - } - - pub fn parse(ctx: Context, attr: [:0]const u8) !Attribute { - return Attribute.wrapOr( - c.mlirAttributeParseGet(ctx._inner, stringRef(attr)), - ) orelse Error.InvalidMlir; - } - - pub fn fromAny(SpecificAttr: type) fn (x: SpecificAttr) Attribute { - return struct { - fn cast(x: SpecificAttr) Attribute { - return .{ ._inner = x._inner }; - } - }.cast; - } - - pub fn isA(self: Attribute, SpecificAttr: type) bool { - return SpecificAttr.is_a_fn(self._inner); - } - - // utilities function to built common attributes. - // All attributes are upcasted to the Attribute type, making it easier to chain construct, - // but losing type information. - - pub fn null_() Attribute { - return .wrap(c.mlirAttributeGetNull()); - } - - pub fn string(ctx: Context, str: []const u8) Attribute { - return StringAttribute.init(ctx, str).asAttr(); - } - - pub fn type_(t: Type) Attribute { - return TypeAttribute.init(t).asAttr(); - } - - pub fn unit(ctx: Context) Attribute { - return .wrap(c.mlirUnitAttrGet(ctx._inner)); - } - - pub fn boolean(ctx: Context, value: bool) Attribute { - return BoolAttribute.init(ctx, value).asAttr(); - } - - pub fn i1FromBool(ctx: Context, value: bool) Attribute { - return IntegerAttribute(.i1).init(ctx, @intFromBool(value)).asAttr(); - } - - pub fn int(ctx: Context, comptime int_type: IntegerTypes, value: i64) Attribute { - return IntegerAttribute(int_type).init(ctx, value).asAttr(); - } - - pub fn float(ctx: Context, comptime float_type: FloatTypes, value: f64) Attribute { - return FloatAttribute(float_type).init(ctx, value).asAttr(); - } - - pub fn array(ctx: Context, attrs: []const Attribute) Attribute { - return ArrayAttribute.init(ctx, attrs).asAttr(); - } - - pub fn dense(ctx: Context, comptime dt: DenseArrayTypes, values: []const dt.ZigType()) Attribute { - return DenseArrayAttribute(dt).init(ctx, values).asAttr(); - } - - /// Use a tensor as an attribute. - /// The tensor is specified by dims, dtype and a flat slice of values. - pub fn denseElements(ctx: Context, dims: []const i64, comptime dt: DenseElementsAttributeTypes, values: []const dt.ZigType()) Attribute { - return DenseElementsAttribute(dt).init(.tensor(dims, dt.mlirType(ctx)), values).asAttr(); - } - - pub fn denseElementsFromBytes(ctx: Context, dims: []const i64, dt: DenseElementsAttributeTypes, raw_bytes: []const u8) Attribute { - const shape: Type = .tensor(dims, dt.mlirType(ctx)); - return .{ ._inner = c.mlirDenseElementsAttrRawBufferGet( - shape._inner, - @intCast(raw_bytes.len), - raw_bytes.ptr, - ) }; - } - - pub fn symbol(ctx: Context, flat_name: [:0]const u8) Attribute { - return FlatSymbolRefAttribute.init(ctx, flat_name).asAttr(); - } - - pub fn named(attr: Attribute, ctx: Context, name: [:0]const u8) NamedAttribute { - return NamedAttribute.named(ctx, name, attr); - } - - pub fn dict(ctx: Context, named_attrs: []const AttrTuple) Attribute { - var attr_buf: [32]NamedAttribute = undefined; - stdx.debug.assert(named_attrs.len <= attr_buf.len, ".dict helper only support up to {} attribute, got {}. You will need to call mlir.DictionaryAttribute directly", .{ attr_buf.len, named_attrs.len }); - - const attrs = attr_buf[0..named_attrs.len]; - for (attrs, named_attrs) |*attr, tuple| { - attr.* = .named(ctx, tuple[0], tuple[1]); - } - - return DictionaryAttribute.init(ctx, attrs).asAttr(); - } - - pub fn eqlAny(Attr: type) fn (Attr, Attr) bool { - return struct { - fn eql(a: Attr, b: Attr) bool { - return a.asAttr().eql(b.asAttr()); - } - }.eql; - } -}; - -pub const NamedAttribute = extern struct { - _inner: c.MlirNamedAttribute, - - pub fn wrap(c_named_attribute: c.MlirNamedAttribute) NamedAttribute { - return @bitCast(c_named_attribute); - } - - pub fn named(ctx: Context, name: [:0]const u8, attr: Attribute) NamedAttribute { - return .{ ._inner = .{ - .name = c.mlirIdentifierGet(ctx._inner, stringRef(name)), - .attribute = attr._inner, - } }; - } - - pub fn init(name: Identifier, attr: Attribute) NamedAttribute { - return .{ ._inner = .{ - .name = name._inner, - .attribute = attr._inner, - } }; - } -}; - -pub const StringAttribute = struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsAString; - pub const asAttr = Attribute.fromAny(StringAttribute); - pub const eql = Attribute.eqlAny(StringAttribute); - - pub fn init(ctx: Context, str: []const u8) StringAttribute { - return .{ ._inner = c.mlirStringAttrGet(ctx._inner, stringRef(str)) }; - } - - pub fn value(self: StringAttribute) []const u8 { - return fromStringRef(c.mlirStringAttrGetValue(self._inner)); - } -}; - -pub const BoolAttribute = struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsABool; - pub const asAttr = Attribute.fromAny(BoolAttribute); - pub const eql = Attribute.eqlAny(BoolAttribute); - - pub fn init(ctx: Context, value_: bool) BoolAttribute { - return .{ ._inner = c.mlirBoolAttrGet(ctx._inner, if (value_) 1 else 0) }; - } - - pub fn value(self: BoolAttribute) bool { - return c.mlirBoolAttrGetValue(self._inner); - } -}; - -pub const TypeAttribute = struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsAType; - pub const eql = Attribute.eqlAny(TypeAttribute); - - pub fn init(type_: Type) TypeAttribute { - return .{ ._inner = c.mlirTypeAttrGet(type_._inner) }; - } - - pub fn typ(self: TypeAttribute) Type { - return .{ ._inner = c.mlirAttributeGetType(self._inner) }; - } - - pub const asAttr = Attribute.fromAny(TypeAttribute); -}; - -pub const ArrayAttribute = struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsAArray; - pub const asAttr = Attribute.fromAny(ArrayAttribute); - pub const eql = Attribute.eqlAny(ArrayAttribute); - - pub fn init(ctx: Context, attrs: []const Attribute) ArrayAttribute { - return .{ ._inner = c.mlirArrayAttrGet(ctx._inner, @intCast(attrs.len), @ptrCast(attrs.ptr)) }; - } - - pub fn size(self: ArrayAttribute) usize { - return @intCast(c.mlirArrayAttrGetNumElements(self._inner)); - } - - pub fn get(self: ArrayAttribute, index: usize) Attribute { - return .{ ._inner = c.mlirArrayAttrGetElement(self._inner, @intCast(index)) }; - } -}; - -pub fn IntegerAttribute(comptime it: IntegerTypes) type { - const ZigType, const getter = comptime switch (it) { - .i1, .i2, .i4, .i8, .i16, .i32, .i64 => .{ i64, c.mlirIntegerAttrGetValueInt }, - .si4, .si8, .si16, .si32, .si64 => .{ i64, c.mlirIntegerAttrGetValueSInt }, - .u2, .u4, .u8, .u16, .u32, .u64 => .{ u64, c.mlirIntegerAttrGetValueUInt }, - .unknown => @compileError("IntegerAttribute(unknown)"), - }; - - return struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsAInteger; - - pub const IntegerTypeType = IntegerType(it); - const IntAttr = @This(); - - pub const asAttr = Attribute.fromAny(IntAttr); - pub const eql = Attribute.eqlAny(IntAttr); - - pub fn init(ctx: Context, value: i64) IntAttr { - return .{ ._inner = c.mlirIntegerAttrGet( - IntegerType(it).init(ctx)._inner, - value, - ) }; - } - - pub fn get(value: IntAttr) ZigType { - return @intCast(getter(value._inner)); - } - }; -} - -pub fn FloatAttribute(comptime ft: FloatTypes) type { - return struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsAFloat; - const FloatAttr = @This(); - pub const asAttr = Attribute.fromAny(FloatAttr); - - pub fn init(ctx: Context, value: f64) FloatAttr { - return .{ ._inner = c.mlirFloatAttrDoubleGet( - ctx._inner, - FloatType(ft).init(ctx)._inner, - value, - ) }; - } - - pub fn get(value: FloatAttr) f64 { - return c.mlirFloatAttrGetValueDouble(value._inner); - } - }; -} - -pub const DenseArrayTypes = enum { - bool, - i8, - i16, - i32, - i64, - f32, - f64, - - pub fn ZigType(comptime dt: DenseArrayTypes) type { - return switch (dt) { - .bool => i32, - .i8 => i8, - .i16 => i16, - .i32 => i32, - .i64 => i64, - .f32 => f32, - .f64 => f64, - }; - } -}; - -pub fn DenseArrayAttribute(comptime dt: DenseArrayTypes) type { - const _is_a_fn, const get_fn, const get_element_fn = switch (dt) { - .bool => .{ c.mlirAttributeIsADenseBoolArray, c.mlirDenseBoolArrayGet, c.mlirDenseBoolArrayGetElement }, - .i8 => .{ c.mlirAttributeIsADenseI8Array, c.mlirDenseI8ArrayGet, c.mlirDenseI8ArrayGetElement }, - .i16 => .{ c.mlirAttributeIsADenseI16Array, c.mlirDenseI16ArrayGet, c.mlirDenseI16ArrayGetElement }, - .i32 => .{ c.mlirAttributeIsADenseI32Array, c.mlirDenseI32ArrayGet, c.mlirDenseI32ArrayGetElement }, - .i64 => .{ c.mlirAttributeIsADenseI64Array, c.mlirDenseI64ArrayGet, c.mlirDenseI64ArrayGetElement }, - .f32 => .{ c.mlirAttributeIsADenseF32Array, c.mlirDenseF32ArrayGet, c.mlirDenseF32ArrayGetElement }, - .f64 => .{ c.mlirAttributeIsADenseF64Array, c.mlirDenseF64ArrayGet, c.mlirDenseF64ArrayGetElement }, - }; - - return struct { - _inner: c.MlirAttribute, - const Attr = @This(); - const ElementType = dt; - const ElementTypeZig = dt.ZigType(); - - pub const asAttr = Attribute.fromAny(Attr); - pub const eql = Attribute.eqlAny(Attr); - pub const is_a_fn = _is_a_fn; - - pub fn init(ctx: Context, values: []const ElementTypeZig) Attr { - return .{ ._inner = get_fn(ctx._inner, @intCast(values.len), @ptrCast(values.ptr)) }; - } - - pub fn get(self: Attr, pos: usize) ElementTypeZig { - return get_element_fn(self._inner, @intCast(pos)); - } - - pub fn len(self: Attr) usize { - return @intCast(c.mlirDenseArrayGetNumElements(self._inner)); - } - }; -} - -pub const DenseElementsAttributeTypes = enum { - bool, +pub const IntegerTypes = enum { + i1, + i2, + i4, i8, i16, i32, i64, + i128, + u1, + u2, + u4, u8, u16, u32, u64, + u128, + si1, + si4, + si8, + si16, + si32, + si64, + si128, + + pub fn bitwidth(self: IntegerTypes) usize { + return switch (self) { + .i1, .u1, .si1 => 1, + .i2, .u2 => 2, + .i4, .u4, .si4 => 4, + .i8, .u8, .si8 => 8, + .i16, .u16, .si16 => 16, + .i32, .u32, .si32 => 32, + .i64, .u64, .si64 => 64, + .i128, .u128, .si128 => 128, + }; + } + + pub fn signedness(self: IntegerTypes) IntegerType.Signedness { + return switch (self) { + .i1, .i2, .i4, .i8, .i16, .i32, .i64, .i128 => .signless, + .si1, .si4, .si8, .si16, .si32, .si64, .si128 => .signed, + .u1, .u2, .u4, .u8, .u16, .u32, .u64, .u128 => .unsigned, + }; + } +}; + +pub const IntegerType = opaque { + pub const Signedness = enum { + signless, + signed, + unsigned, + }; + + const M = Methods(IntegerType, c.MlirType); + + pub const isAFn = c.mlirTypeIsAInteger; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + fn get(ctx: *Context, it: IntegerTypes) *const IntegerType { + return exact(ctx, it.bitwidth(), it.signedness()); + } + + fn exact(ctx: *Context, bitwidth: usize, sign: Signedness) *const IntegerType { + return @ptrCast(switch (sign) { + .signless => c.mlirIntegerTypeGet(ctx.ptr(), @intCast(bitwidth)).ptr, + .signed => c.mlirIntegerTypeSignedGet(ctx.ptr(), @intCast(bitwidth)).ptr, + .unsigned => c.mlirIntegerTypeUnsignedGet(ctx.ptr(), @intCast(bitwidth)).ptr, + }); + } +}; + +pub fn integerType(ctx: *Context, it: IntegerTypes) *const Type { + return @ptrCast(IntegerType.get(ctx, it)); +} + +pub const FloatTypes = enum { + f4e2m1fn, + f6e2m3fn, + f6e3m2fn, + f8e3m4, + f8e4m3, + f8e4m3b11fnuz, + f8e4m3fn, + f8e4m3fnuz, + f8e5m2, + f8e5m2fnuz, + f8e8m0fnu, bf16, f16, f32, f64, - index, - - pub fn ZigType(comptime dt: DenseElementsAttributeTypes) type { - return switch (dt) { - .bool => bool, - .i8 => i8, - .i16 => i16, - .i32 => i32, - .i64 => i64, - .u8 => u8, - .u16 => u16, - .u32 => u32, - .u64 => u64, - .bf16 => u16, - .f16 => f16, - .f32 => f32, - .f64 => f64, - .index => usize, - }; - } - - pub fn mlirType(dt: DenseElementsAttributeTypes, ctx: Context) Type { - return switch (dt) { - .bool => .int(ctx, .i1), - .i8 => .int(ctx, .i8), - .i16 => .int(ctx, .i16), - .i32 => .int(ctx, .i32), - .i64 => .int(ctx, .i64), - .u8 => .int(ctx, .u8), - .u16 => .int(ctx, .u16), - .u32 => .int(ctx, .u32), - .u64 => .int(ctx, .u64), - .bf16 => .float(ctx, .bf16), - .f16 => .float(ctx, .f16), - .f32 => .float(ctx, .f32), - .f64 => .float(ctx, .f64), - .index => .index(ctx), - }; - } + // f80, + // f128, + tf32, }; -pub fn DenseElementsAttribute(comptime dt: DenseElementsAttributeTypes) type { - return struct { - _inner: c.MlirAttribute, +pub fn FloatType(comptime ft: FloatTypes) type { + const _isAFn, const getter = switch (ft) { + .f4e2m1fn => .{ c.mlirTypeIsAFloat4E2M1FN, c.mlirFloat4E2M1FNTypeGet }, + .f6e2m3fn => .{ c.mlirTypeIsAFloat6E2M3FN, c.mlirFloat6E2M3FNTypeGet }, + .f6e3m2fn => .{ c.mlirTypeIsAFloat6E3M2FN, c.mlirFloat6E3M2FNTypeGet }, + .f8e3m4 => .{ c.mlirTypeIsAFloat8E3M4, c.mlirFloat8E3M4TypeGet }, + .f8e4m3 => .{ c.mlirTypeIsAFloat8E4M3, c.mlirFloat8E4M3TypeGet }, + .f8e4m3b11fnuz => .{ c.mlirTypeIsAFloat8E4M3B11FNUZ, c.mlirFloat8E4M3B11FNUZTypeGet }, + .f8e4m3fn => .{ c.mlirTypeIsAFloat8E4M3FN, c.mlirFloat8E4M3FNTypeGet }, + .f8e4m3fnuz => .{ c.mlirTypeIsAFloat8E4M3FNUZ, c.mlirFloat8E4M3FNUZTypeGet }, + .f8e5m2 => .{ c.mlirTypeIsAFloat8E5M2, c.mlirFloat8E5M2TypeGet }, + .f8e5m2fnuz => .{ c.mlirTypeIsAFloat8E5M2FNUZ, c.mlirFloat8E5M2FNUZTypeGet }, + .f8e8m0fnu => .{ c.mlirTypeIsAFloat8E8M0FNU, c.mlirFloat8E8M0FNUTypeGet }, + .bf16 => .{ c.mlirTypeIsABF16, c.mlirBF16TypeGet }, + .f16 => .{ c.mlirTypeIsAF16, c.mlirF16TypeGet }, + .f32 => .{ c.mlirTypeIsAF32, c.mlirF32TypeGet }, + .f64 => .{ c.mlirTypeIsAF64, c.mlirF64TypeGet }, + .tf32 => .{ c.mlirTypeIsATF32, c.mlirTF32TypeGet }, + }; - const Attr = @This(); + return opaque { + const Self = @This(); + const M = Methods(Self, c.MlirType); - pub const is_a_fn = c.mlirAttributeIsADenseElements; - pub const asAttr = Attribute.fromAny(Attr); - pub const eql = Attribute.eqlAny(Attr); + pub const isAFn = _isAFn; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); - pub fn init(shaped_type: Type, slice: []const dt.ZigType()) Attr { - const raw_bytes = std.mem.sliceAsBytes(slice); - const attr: Attr = .{ ._inner = c.mlirDenseElementsAttrRawBufferGet( - shaped_type._inner, - @intCast(raw_bytes.len), - @ptrCast(raw_bytes.ptr), - ) }; - std.debug.assert(attr._inner.ptr != null); - return attr; - } - - pub fn len(self: Attr) usize { - return @intCast(c.mlirElementsAttrGetNumElements(self._inner)); - } - - pub fn items(self: Attr) []const dt.ZigType() { - const raw_bytes: [*]const u8 = c.mlirDenseElementsAttrGetRawData(self._inner) orelse unreachable; - const ptr: [*]const dt.ZigType() = @ptrCast(@alignCast(raw_bytes)); - // Note the mlir API returns us the number of elements, not the number of bytes, - // that's why we track the element type at comptime to allow items to work. - return ptr[0..self.len()]; - } - - pub fn bytes(self: Attr) []const u8 { - return std.mem.sliceAsBytes(self.items()); + pub fn get(ctx: *Context) *const Self { + return @ptrCast(getter(ctx.ptr()).ptr); } }; } -pub const FlatSymbolRefAttribute = struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsAFlatSymbolRef; - const Self = FlatSymbolRefAttribute; - pub const eql = Attribute.eqlAny(Self); +pub fn floatType(ctx: *Context, ft: FloatTypes) *const Type { + return switch (ft) { + inline else => |v| @ptrCast(FloatType(v).get(ctx)), + }; +} - pub fn init(ctx: Context, str: [:0]const u8) Self { - return .{ ._inner = c.mlirFlatSymbolRefAttrGet(ctx._inner, stringRef(str)) }; +pub const ComplexTypes = enum { + c64, + c128, +}; + +pub const ComplexType = opaque { + const M = Methods(ComplexType, c.MlirType); + + pub const isAFn = c.mlirTypeIsAComplex; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn get(ctx: *Context, complex_type: ComplexTypes) *const ComplexType { + return @ptrCast(c.mlirComplexTypeGet(switch (complex_type) { + .c64 => floatType(ctx, .f32).ptr(), + .c128 => floatType(ctx, .f64).ptr(), + }).ptr); + } +}; + +pub fn complexType(ctx: *Context, complex_type: ComplexTypes) *const Type { + return @ptrCast(ComplexType.get(ctx, complex_type)); +} + +pub const Attribute = opaque { + const M = Methods(Attribute, c.MlirAttribute); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn parse(ctx: *Context, attr: []const u8) Error!*const Attribute { + return @ptrCast(c.mlirAttributeParseGet(ctx.ptr(), stringRef(attr)).ptr orelse return Error.InvalidMlir); } - pub fn value(self: Self) []const u8 { - return fromStringRef(c.mlirFlatSymbolRefAttrGetValue(self._inner)); + pub fn context(self: *const Context) *Context { + return @ptrCast(c.mlirAttributeGetContext(self.ptr()).ptr); } - pub const asAttr = Attribute.fromAny(Self); + pub fn dialect(self: *const Context) *const Dialect { + return @ptrCast(c.mlirAttributeGetDialect(self.ptr()).ptr); + } +}; + +pub const StringAttribute = opaque { + const M = Methods(StringAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAString; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, str: []const u8) *const StringAttribute { + return @ptrCast(c.mlirStringAttrGet(ctx.ptr(), stringRef(str)).ptr); + } + + pub fn value(self: *const StringAttribute) []const u8 { + return string(c.mlirStringAttrGetValue(self.ptr())); + } +}; + +pub fn stringAttribute(ctx: *Context, str: []const u8) *const Attribute { + return @ptrCast(StringAttribute.init(ctx, str)); +} + +pub const IntegerAttribute = opaque { + const M = Methods(IntegerAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAInteger; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, it: IntegerTypes, value_: anytype) *const IntegerAttribute { + return @ptrCast(c.mlirIntegerAttrGet( + IntegerType.get(ctx, it).ptr(), + @intCast(value_), + ).ptr); + } + + pub fn value(self: *const IntegerAttribute, comptime T: type) T { + const getValue = switch (@typeInfo(T).int.signedness) { + .signed => c.mlirIntegerAttrGetValueInt, + .unsigned => c.mlirIntegerAttrGetValueUInt, + }; + return @intCast(getValue(self.ptr())); + } +}; + +pub fn integerAttribute(ctx: *Context, it: IntegerTypes, value_: anytype) *const Attribute { + return @ptrCast(IntegerAttribute.init(ctx, it, value_)); +} + +pub const BoolAttribute = opaque { + const M = Methods(BoolAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsABool; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, value_: bool) *const BoolAttribute { + return @ptrCast(c.mlirBoolAttrGet(ctx.ptr(), @intFromBool(value_)).ptr); + } + + pub fn value(self: *const BoolAttribute) bool { + return c.mlirBoolAttrGetValue(self.ptr()); + } +}; + +pub fn boolAttribute(ctx: *Context, value: bool) *const Attribute { + return @ptrCast(BoolAttribute.init(ctx, value)); +} + +pub const TypeAttribute = opaque { + const M = Methods(TypeAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(type_: *const Type) *const TypeAttribute { + return @ptrCast(c.mlirTypeAttrGet(type_.ptr()).ptr); + } + + pub fn value(self: *const TypeAttribute) *const Type { + return @ptrCast(c.mlirTypeAttrGetValue(self.ptr()).ptr); + } +}; + +pub fn typeAttribute(type_: *const Type) *const Attribute { + return @ptrCast(TypeAttribute.init(type_)); +} + +pub const Dialect = struct { + const M = Methods(Dialect, c.MlirDialect); + + pub const ptr = M.ptr; + + pub fn context(self: *const Dialect) *Context { + return @ptrCast(c.mlirDialectGetContext(self.ptr()).ptr); + } + + pub fn namespace(self: *const Dialect) []const u8 { + return string(c.mlirDialectGetNamespace(self.ptr())); + } + + pub fn format(self: *const Dialect, writer: *std.Io.Writer) !void { + try writer.print("{s}{{ .namespace = \"{s}\" }}", .{ @typeName(Dialect), self.namespace() }); + } +}; + +pub const DialectHandle = opaque { + const M = Methods(DialectHandle, c.MlirDialectHandle); + + pub const ptr = M.ptr; + + pub fn format(self: *const DialectHandle, writer: *std.Io.Writer) !void { + try writer.print("{s}{{ .namespace = \"{s}\" }}", .{ @typeName(DialectHandle), self.namespace() }); + } + + pub fn namespace(self: *const DialectHandle) []const u8 { + return string(c.mlirDialectHandleGetNamespace(self.ptr())); + } + + pub fn insertDialect(self: *const DialectHandle, registry: *DialectRegistry) void { + c.mlirDialectHandleInsertDialect(self.ptr(), registry.ptr()); + } + + pub fn registerDialect(self: *const DialectHandle, ctx: *Context) void { + c.mlirDialectHandleRegisterDialect(self.ptr(), ctx.ptr()); + } + + pub fn loadDialect(self: *const DialectHandle, ctx: *Context) *Dialect { + return @ptrCast(c.mlirDialectHandleLoadDialect(self.ptr(), ctx.ptr()).ptr); + } + + pub fn fromString(comptime namespace_: []const u8) *const DialectHandle { + return @ptrCast(@field(c, "mlirGetDialectHandle__" ++ namespace_ ++ "__")().ptr); + } +}; + +pub const Identifier = opaque { + const M = Methods(Identifier, c.MlirIdentifier); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirIdentifierEqual); + + pub fn context(self: *const Identifier) *Context { + return @ptrCast(c.mlirIdentifierGetContext(self.ptr()).ptr); + } + + pub fn get(ctx: *Context, str_: []const u8) *const Identifier { + return @ptrCast(c.mlirIdentifierGet(ctx.ptr(), stringRef(str_)).ptr); + } + + pub fn str(self: *const Identifier) []const u8 { + return string(c.mlirIdentifierStr(self.ptr())); + } + + pub fn format(self: *const Identifier, writer: *std.Io.Writer) !void { + try writer.print("{s}{{ .str = \"{s}\" }}", .{ @typeName(Identifier), self.str() }); + } +}; + +pub const Module = opaque { + const M = Methods(Module, c.MlirModule); + + pub const ptr = M.ptr; + + pub fn init(loc: *const Location) *Module { + return @ptrCast(@constCast(c.mlirModuleCreateEmpty(loc.ptr()).ptr)); + } + + pub fn deinit(self: *Module) void { + c.mlirModuleDestroy(self.ptr()); + } + + pub fn format(self: *const Module, writer: *std.Io.Writer) !void { + return self.operation().format(writer); + } + + pub fn fromOperation(op: *Operation) *Module { + return @ptrCast(@constCast(c.mlirModuleFromOperation(op.ptr()).ptr)); + } + + pub fn operation(self: *const Module) *Operation { + return @ptrCast(c.mlirModuleGetOperation(self.ptr()).ptr); + } + + pub fn parse(ctx: *Context, source: []const u8) Error!*Module { + return @ptrCast(@constCast(c.mlirModuleCreateParse(ctx.ptr(), stringRef(source)).ptr orelse return Error.InvalidMlir)); + } + + pub fn context(self: *const Module) *Context { + return @ptrCast(c.mlirModuleGetContext(self.ptr()).ptr); + } + + pub fn body(self: *const Module) *Block { + return @ptrCast(c.mlirModuleGetBody(self.ptr()).ptr); + } +}; + +pub const Block = opaque { + const M = Methods(Block, c.MlirBlock); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirBlockEqual); + pub const format = M.format(c.mlirBlockPrint); + + pub fn init(args: []const *const Type, locs: []const *const Location) *Block { + return @ptrCast(c.mlirBlockCreate(@intCast(args.len), @ptrCast(args), @ptrCast(locs)).ptr); + } + + pub fn deinit(self: *Block) void { + std.debug.assert(self.parentOperation() == null); + c.mlirBlockDestroy(self.ptr()); + } + + pub fn argument(self: *const Block, index: usize) *const Value { + return @ptrCast(c.mlirBlockGetArgument(self.ptr(), @intCast(index)).ptr); + } + + pub fn numArguments(self: *const Block) usize { + return @intCast(c.mlirBlockGetNumArguments(self.ptr())); + } + + pub fn addArgument(self: *Block, type_: *const Type, loc: *const Location) *const Value { + return @ptrCast(c.mlirBlockAddArgument(self.ptr(), type_.ptr(), loc.ptr()).ptr); + } + + pub fn insertArgument(self: *Block, index: usize, type_: *const Type, loc: *const Location) *const Value { + return @ptrCast(c.mlirBlockInsertArgument(self.ptr(), @intCast(index), type_.ptr(), loc.ptr()).ptr); + } + + pub fn appendOwnedOperation(self: *Block, op: *Operation) void { + c.mlirBlockAppendOwnedOperation(self.ptr(), op.ptr()); + } + + pub fn appendOwnedOperations(self: *Block, ops: []const *Operation) void { + for (ops) |op| { + self.appendOwnedOperation(op); + } + } + + pub fn parentOperation(self: *const Block) ?*Operation { + return @ptrCast(c.mlirBlockGetParentOperation(self.ptr()).ptr); + } + + pub fn detach(self: *Block) void { + c.mlirBlockDetach(self.ptr()); + } + + pub fn terminator(self: *const Block) ?*Operation { + return @ptrCast(c.mlirBlockGetTerminator(self.ptr()).ptr); + } +}; + +pub const NamedAttribute = struct { + inner: c.MlirNamedAttribute, + + pub fn ptr(self: NamedAttribute) c.MlirNamedAttribute { + return self.inner; + } + + pub fn name(self: NamedAttribute) *const Identifier { + return @ptrCast(self.inner.name.ptr); + } + + pub fn attribute(self: NamedAttribute) *const Attribute { + return @ptrCast(self.inner.attribute.ptr); + } + + pub fn named(ctx: *Context, name_: []const u8, attr: *const Attribute) NamedAttribute { + return .{ + .inner = .{ + .name = Identifier.get(ctx, name_).ptr(), + .attribute = attr.ptr(), + }, + }; + } + + pub fn init(name_: *const Identifier, attr: *const Attribute) NamedAttribute { + return .{ + .inner = .{ + .name = name_.ptr(), + .attribute = attr.ptr(), + }, + }; + } + + pub fn format(self: NamedAttribute, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("{s}{{ .{s} = \"{f}\" }}", .{ + @typeName(NamedAttribute), + self.name().str(), + self.attribute(), + }); + } +}; + +pub const DictionaryAttribute = opaque { + const M = Methods(DictionaryAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsADictionary; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, attributes: []const NamedAttribute) *const DictionaryAttribute { + return @ptrCast(c.mlirDictionaryAttrGet( + ctx.ptr(), + @intCast(attributes.len), + @ptrCast(attributes), + ).ptr); + } + + pub fn size(self: *const DictionaryAttribute) usize { + return @intCast(c.mlirDictionaryAttrGetNumElements(self.ptr())); + } + + pub fn get(self: *const DictionaryAttribute, pos: usize) NamedAttribute { + return .{ .inner = c.mlirDictionaryAttrGetElement(self.ptr(), @intCast(pos)) }; + } + + pub fn getByName(self: *const DictionaryAttribute, name: []const u8) ?*const Attribute { + return @ptrCast(c.mlirDictionaryAttrGetElementByName(self.ptr(), stringRef(name)).ptr); + } +}; + +pub fn dictionaryAttribute(ctx: *Context, attributes: []const NamedAttribute) *const Attribute { + return @ptrCast(DictionaryAttribute.init(ctx, attributes)); +} + +pub const Value = opaque { + const M = Methods(Value, c.MlirValue); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirValueEqual); + pub const format = M.format(c.mlirValuePrint); + pub const isA = M.isA; + + pub fn type_(val: *const Value) *const Type { + return @ptrCast(c.mlirValueGetType(val.ptr()).ptr); + } + + pub fn setType(self: *Value, type__: *const Type) void { + c.mlirValueSetType(self.ptr(), type__.ptr()); + } + + pub fn firstUse(self: *const Value) *const OpOperand { + return @ptrCast(c.mlirValueGetFirstUse(self.ptr()).ptr); + } + + pub fn replaceAllUsesWith(self: *const Value, with: *const Value) void { + c.mlirValueReplaceAllUsesOfWith(self.ptr(), with.ptr()); + } +}; + +pub const OpOperand = opaque { + const M = Methods(OpOperand, c.MlirOpOperand); + + pub const ptr = M.ptr; + + pub fn owner(self: *const OpOperand) *Operation { + return @ptrCast(c.mlirOpOperandGetOwner(self.ptr()).ptr); + } + + pub fn number(self: *const OpOperand) usize { + return @intCast(c.mlirOpOperandGetOperandNumber(self.ptr())); + } + + pub fn nextUse(self: *const OpOperand) ?*const OpOperand { + return @ptrCast(c.mlirOpOperandGetNextUse(self.ptr()).ptr); + } +}; + +pub const OpResult = opaque { + const M = Methods(OpResult, c.MlirValue); + + pub const isAFn = c.mlirValueIsAOpResult; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirValueEqual); + pub const format = M.format(c.mlirValuePrint); + + pub fn owner(self: *const OpResult) *Operation { + return @ptrCast(c.mlirOpResultGetOwner(self.ptr()).ptr); + } + + pub fn resultNumber(self: *const OpResult) usize { + return @intCast(c.mlirOpResultGetResultNumber(self.ptr())); + } +}; + +pub const BlockArgument = opaque { + const M = Methods(BlockArgument, c.MlirValue); + + pub const isAFn = c.mlirValueIsABlockArgument; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirValueEqual); + pub const format = M.format(c.mlirValuePrint); + + pub fn block(arg: *const BlockArgument) *Block { + return @ptrCast(c.mlirBlockArgumentGetOwner(arg.ptr()).ptr); + } + + pub fn number(self: *const BlockArgument) usize { + return @intCast(c.mlirBlockArgumentGetArgNumber(self.ptr())); + } + + pub fn setType(self: *BlockArgument, type_: *const Type) void { + c.mlirBlockArgumentSetType(self.ptr(), type_.ptr()); + } }; pub const OperationState = struct { - _inner: c.MlirOperationState, + inner: c.MlirOperationState, - const Self = OperationState; - - pub fn init(name: [:0]const u8, loc: Location) Self { - return .{ ._inner = c.mlirOperationStateGet(stringRef(name), loc._inner) }; + pub fn init(name: []const u8, location: *const Location) OperationState { + return .{ .inner = c.mlirOperationStateGet(stringRef(name), location.ptr()) }; } - pub fn addResult(self: *Self, type_: Type) void { - c.mlirOperationStateAddResults(&self._inner, 1, &[_]c.MlirType{type_._inner}); + pub fn addResults(self: *OperationState, results: []const *const Type) void { + c.mlirOperationStateAddResults(&self.inner, @intCast(results.len), @ptrCast(results)); } - pub fn addResults(self: *Self, types: []const Type) void { - c.mlirOperationStateAddResults(&self._inner, @intCast(types.len), @ptrCast(types.ptr)); + pub fn addOperands(self: *OperationState, operands: []const *const Value) void { + c.mlirOperationStateAddOperands(&self.inner, @intCast(operands.len), @ptrCast(operands)); } - pub fn addOperand(self: *Self, value: Value) void { - c.mlirOperationStateAddOperands(&self._inner, 1, &[_]c.MlirValue{value._inner}); + pub fn addOwnedRegions(self: *OperationState, regions: []const *Region) void { + c.mlirOperationStateAddOwnedRegions(&self.inner, @intCast(regions.len), @ptrCast(regions)); } - pub fn addOperands(self: *Self, values: []const Value) void { - c.mlirOperationStateAddOperands(&self._inner, @intCast(values.len), @ptrCast(values.ptr)); + pub fn addSuccessors(self: *OperationState, successors: []*const Block) void { + c.mlirOperationStateAddSuccessors(&self.inner, @intCast(successors.len), @ptrCast(successors)); } - pub fn addRegion(self: *Self, region: *Region) void { - c.mlirOperationStateAddOwnedRegions(&self._inner, 1, &[_]c.MlirRegion{region._inner}); + pub fn addAttributes(self: *OperationState, attributes: []const NamedAttribute) void { + c.mlirOperationStateAddAttributes(&self.inner, @intCast(attributes.len), @ptrCast(attributes.ptr)); } - pub fn addRegions(self: *Self, regions: []const Region) void { - c.mlirOperationStateAddOwnedRegions(&self._inner, @intCast(regions.len), @ptrCast(regions.ptr)); - } - - pub fn addAttribute(self: *Self, ctx: Context, name: [:0]const u8, attr: Attribute) void { - c.mlirOperationStateAddAttributes(&self._inner, 1, @ptrCast(&.{ - .{ - .name = Identifier.get(ctx, name)._inner, - .attribute = attr._inner, - }, - })); - } - - pub fn addAttributeRaw(self: *Self, name: Identifier, attr: Attribute) void { - c.mlirOperationStateAddAttributes(&self._inner, 1, @ptrCast(&.{ - .{ - .name = name._inner, - .attribute = attr._inner, - }, - })); - } - - pub fn addAttributes(self: *Self, attributes: []const NamedAttribute) void { - c.mlirOperationStateAddAttributes(&self._inner, @intCast(attributes.len), @ptrCast(attributes.ptr)); - } - - pub fn resultTypeInference(self: *Self, enabled: bool) void { - self._inner.enableResultTypeInference = enabled; + pub fn enableResultTypeInference(self: *OperationState) void { + c.mlirOperationStateEnableResultTypeInference(&self.inner); } }; -pub const DictionaryAttribute = struct { - _inner: c.MlirAttribute, - pub const is_a_fn = c.mlirAttributeIsADictionary; - pub const asAttr = Attribute.fromAny(DictionaryAttribute); - pub const eql = Attribute.eqlAny(DictionaryAttribute); +pub const Operation = opaque { + const M = Methods(Operation, c.MlirOperation); - pub fn init(ctx: Context, attributes: []const NamedAttribute) DictionaryAttribute { - return .{ ._inner = c.mlirDictionaryAttrGet( - ctx._inner, - @intCast(attributes.len), - @ptrCast(attributes.ptr), - ) }; + const MAX_SEGMENTS = 32; + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirOperationEqual); + pub const format = M.format(c.mlirOperationPrint); + + pub fn init(state: *OperationState) !*Operation { + defer state.* = undefined; + return @ptrCast(c.mlirOperationCreate(&state.inner).ptr orelse return Error.InvalidMlir); } - pub fn size(self: DictionaryAttribute) usize { - return @intCast(c.mlirDictionaryAttrGetNumElements(self._inner)); + pub fn deinit(self: *Operation) void { + std.debug.assert(self.block() == null); + c.mlirOperationDestroy(self.ptr()); } - pub fn get(self: DictionaryAttribute, pos: usize) NamedAttribute { - return .wrap(c.mlirDictionaryAttrGetElement(self._inner, @bitCast(pos))); + pub fn parse(ctx: *Context, source: []const u8, name_: []const u8) Error!*Operation { + return @ptrCast(c.mlirOperationCreateParse(ctx.ptr(), stringRef(source), stringRef(name_)).ptr orelse return Error.InvalidMlir); } - pub fn getByName(self: DictionaryAttribute, name: [:0]const u8) ?Attribute { - return Attribute.wrapOr(c.mlirDictionaryAttrGetElementByName(self._inner, stringRef(name))); - } -}; - -pub const Operation = struct { - const Self = Operation; - _inner: c.MlirOperation, - - pub const dump = helpers.dump(Operation, c.mlirOperationDestroy); - pub const deinit = helpers.deinit(Operation, c.mlirOperationDestroy); - pub const wrapOr = helpers.wrapOr(Operation, c.mlirOperationIsNull); - pub const eql = helpers.eql(Operation, c.mlirOperationEqual); - - pub fn init(state: *OperationState) !Self { - return Self.wrapOr(c.mlirOperationCreate(&state._inner)) orelse Error.InvalidMlir; + pub fn name(self: *const Operation) []const u8 { + return string(c.mlirOperationGetName(self.ptr())); } - pub fn make(ctx: Context, op_name: [:0]const u8, args: struct { - operands: ?[]const Value = null, - variadic_operands: ?[]const []const Value = null, - tt_variadic_operands: ?[]const []const Value = null, - results: ?[]const Type = null, - variadic_results: ?[]const []const Type = null, - result_type_inference: ?bool = null, - n_regions: usize = 0, - attributes: ?[]const AttrTuple = null, - blocks: ?[]const Block = null, - verify: bool = true, - location: Location, - }) Self { - var state = OperationState.init(op_name, args.location); - std.debug.assert(!(args.operands != null and args.variadic_operands != null)); - if (args.operands) |operands| { - state.addOperands(operands); - } else if (args.variadic_operands) |operands_segments| { - const MAX_SEGMENTS = 32; - var segments: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; + pub fn clone(self: *const Operation) *Operation { + return @ptrCast(c.mlirOperationClone(self.ptr()).ptr); + } - for (operands_segments) |operands| { - state.addOperands(operands); - segments.appendAssumeCapacity(@intCast(operands.len)); - } - state.addAttribute(ctx, "operandSegmentSizes", .denseElements(ctx, &.{@intCast(segments.len)}, .i32, segments.constSlice())); - } else if (args.tt_variadic_operands) |operands_segments| { - // stablehlo and triton seems to disagree on the expected type of operandSegmentSizes, let's fix that. - const MAX_SEGMENTS = 32; - var segments: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; + pub fn block(self: *const Operation) ?*Block { + return @ptrCast(c.mlirOperationGetBlock(self.ptr()).ptr); + } - for (operands_segments) |operands| { - state.addOperands(operands); - segments.appendAssumeCapacity(@intCast(operands.len)); - } - state.addAttribute(ctx, "operandSegmentSizes", .dense(ctx, .i32, segments.constSlice())); - } - if (args.result_type_inference) |enable| { - state.resultTypeInference(enable); - } - std.debug.assert(!(args.results != null and args.variadic_results != null)); - if (args.results) |results| { - state.addResults(results); - } else if (args.variadic_results) |result_segments| { - for (result_segments) |results| { - state.addResults(results); - } - } - for (0..args.n_regions) |_| { - var region_ = Region.init() catch { - @panic("Failed to create MLIR region"); - }; - state.addRegion(®ion_); - } - if (args.attributes) |attrs| { - for (attrs) |attr| { - state.addAttributeRaw( - Identifier.get(ctx, attr[0]), - attr[1], - ); - } - } - if (args.blocks) |blocks_| { - for (blocks_) |block_| { - var region_ = Region.init() catch { - @panic("Failed to create MLIR region"); - }; - region_.appendBlock(block_); - state.addRegion(®ion_); - } - } + pub fn appendTo(self: *Operation, block_: *Block) *Operation { + block_.appendOwnedOperation(self); + return self; + } - const new_op = Operation.init(&state) catch { - @panic("Failed to create MLIR operation"); + pub fn parent(self: *const Operation) ?*Operation { + return @ptrCast(c.mlirOperationGetParentOperation(self.ptr()).ptr); + } + + pub fn removeFromParent(self: *Operation) void { + c.mlirOperationRemoveFromParent(self.ptr()); + } + + pub fn numOperands(self: *const Operation) usize { + return @intCast(c.mlirOperationGetNumOperands(self.ptr())); + } + + pub fn operand(self: *const Operation, index: usize) *const Value { + return @ptrCast(c.mlirOperationGetOperand(self.ptr(), @intCast(index)).ptr); + } + + pub fn setOperand(self: *Operation, index: usize, value: *const Value) void { + c.mlirOperationSetOperand(self.ptr(), @intCast(index), value.ptr()); + } + + pub fn numResults(self: *const Operation) usize { + return @intCast(c.mlirOperationGetNumResults(self.ptr())); + } + + pub fn result(self: *const Operation, index: usize) *const Value { + return @ptrCast(c.mlirOperationGetResult(self.ptr(), @intCast(index)).ptr); + } + + pub fn nextInBlock(self: *const Operation) *const Operation { + return @ptrCast(c.mlirOperationGetNextInBlock(self.ptr()).ptr); + } + + pub fn context(self: *const Operation) *Context { + return @ptrCast(c.mlirOperationGetContext(self.ptr()).ptr); + } + + pub fn setAttributeByName(self: *Operation, name_: []const u8, attribute: *const Attribute) void { + c.mlirOperationSetAttributeByName(self.ptr(), stringRef(name_), attribute.ptr()); + } + + pub const PrintFlags = struct { + elide_large_elements_attrs: ?usize = null, + debug_info: bool = false, + debug_info_pretty_form: bool = true, + print_generic_op_form: bool = false, + use_local_scope: bool = false, + assume_verified: bool = false, + + const Ctx = struct { + self: *const Operation, + flags: PrintFlags, }; + + fn createNative(self: PrintFlags) c.MlirOpPrintingFlags { + const flags = c.mlirOpPrintingFlagsCreate(); + if (self.elide_large_elements_attrs) |v| { + c.mlirOpPrintingFlagsElideLargeElementsAttrs(flags, @intCast(v)); + } + c.mlirOpPrintingFlagsEnableDebugInfo( + flags, + self.debug_info, + self.debug_info_pretty_form, + ); + if (self.print_generic_op_form) { + c.mlirOpPrintingFlagsPrintGenericOpForm(flags); + } + if (self.use_local_scope) { + c.mlirOpPrintingFlagsUseLocalScope(flags); + } + if (self.assume_verified) { + c.mlirOpPrintingFlagsAssumeVerified(flags); + } + return flags; + } + }; + + fn formatWithFlags(fctx: PrintFlags.Ctx, writer: *std.io.Writer) std.Io.Writer.Error!void { + const flags = fctx.flags.createNative(); + defer c.mlirOpPrintingFlagsDestroy(flags); + var sctx: StringCallbackCtx = .{ .writer = writer }; + c.mlirOperationPrintWithFlags(fctx.self.ptr(), flags, stringCallback, &sctx); + if (sctx.err) |err| { + return err; + } + } + + pub fn fmt(self: *const Operation, flags: PrintFlags) std.fmt.Alt(PrintFlags.Ctx, formatWithFlags) { + return .{ .data = .{ + .self = self, + .flags = flags, + } }; + } + + pub const MakeArgs = struct { + pub const AttrTuple = struct { []const u8, *const Attribute }; + + pub const Operands = union(enum) { + flat: []const *const Value, + variadic: []const []const *const Value, + }; + + pub const ResultTypes = union(enum) { + flat: []const *const Type, + variadic: []const []const *const Type, + }; + + operands: ?Operands = null, + results: ?ResultTypes = null, + result_type_inference: bool = false, + attributes: ?[]const NamedAttribute = null, + blocks: ?[]const *Block = null, + verify: bool = true, + location: ?*const Location = null, + }; + + pub fn try_make(ctx: *Context, name_: []const u8, args: MakeArgs) !*Operation { + var state: OperationState = .init(name_, args.location orelse .unknown(ctx)); + if (args.result_type_inference) { + state.enableResultTypeInference(); + } + if (args.operands) |operands| switch (operands) { + .flat => |v| state.addOperands(v), + .variadic => |segments| { + var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; + for (segments) |segment_operands| { + state.addOperands(segment_operands); + sizes.appendAssumeCapacity(@intCast(segment_operands.len)); + } + state.addAttributes(&.{ + .named(ctx, "operandSegmentSizes", denseElementsAttribute(RankedTensorType.get(&.{@intCast(sizes.len)}, integerType(ctx, .i32), null).shaped(), sizes.constSlice())), + }); + }, + }; + if (args.results) |results| switch (results) { + .flat => |v| state.addResults(v), + .variadic => |segments| { + var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; + for (segments) |segment_results| { + state.addResults(segment_results); + sizes.appendAssumeCapacity(@intCast(segment_results.len)); + } + state.addAttributes(&.{ + .named(ctx, "resultSegmentSizes", denseElementsAttribute(RankedTensorType.get(&.{@intCast(sizes.len)}, integerType(ctx, .i32), null).shaped(), sizes.constSlice())), + }); + }, + }; + if (args.attributes) |attrs| { + state.addAttributes(attrs); + } + if (args.blocks) |blocks| { + const region = Region.init(); + for (blocks) |block_| { + region.appendOwnedBlock(block_); + } + state.addOwnedRegions(&.{region}); + } + const new_op = try Operation.init(&state); + errdefer new_op.deinit(); if (args.verify and new_op.verify() == false) { - log.err("Failed to verify MLIR operation:\n{f}", .{new_op.mlirFormatter(.{ .debug_info = true })}); - @panic("Failed to verify MLIR operation"); + log.err("Failed to verify MLIR operation:\n{f}", .{ + new_op.fmt(.{ + .print_generic_op_form = true, + .debug_info = true, + }), + }); + return Error.InvalidMlir; } return new_op; } - pub fn initParse(ctx: Context, str: [:0]const u8) !Self { - return Self.wrapOr( - c.mlirOperationCreateParse(ctx._inner, stringRef(str), stringRef("pouet")), - ) orelse Error.InvalidMlir; + pub fn make(ctx: *Context, op_name: []const u8, args: MakeArgs) *Operation { + return try_make(ctx, op_name, args) catch { + @panic("Failed to create MLIR operation"); + }; } - pub fn clone(self: Self) !Self { - return Self.wrapOr( - c.mlirOperationClone(self._inner), - ) orelse Error.InvalidMlir; + pub fn verify(self: *const Operation) bool { + return c.mlirOperationVerify(self.ptr()); } - pub fn name(self: Self) Identifier { - return .{ ._inner = c.mlirOperationGetName(self._inner) }; - } - - pub fn removeFromParent(self: *Self) void { - c.mlirOperationRemoveFromParent(self._inner); - } - - pub fn numOperands(self: Self) usize { - return @intCast(c.mlirOperationGetNumOperands(self._inner)); - } - - pub fn operand(self: Self, index: usize) Value { - return .{ ._inner = c.mlirOperationGetOperand(self._inner, @intCast(index)) }; - } - - pub fn setOperand(self: *Self, index: usize, value: Value) void { - c.mlirOperationSetOperand(self._inner, @intCast(index), value._inner); - } - - pub fn numResults(self: Self) usize { - return @intCast(c.mlirOperationGetNumResults(self._inner)); - } - - pub fn result(self: Self, index: usize) Value { - return .{ ._inner = c.mlirOperationGetResult(self._inner, @intCast(index)) }; - } - - pub fn nextInBlock(self: Self) Self { - return .{ ._inner = c.mlirOperationGetNextInBlock(self._inner) }; - } - - // pub fn previousInBlock(self: Self) Self { - // return .{ ._inner = c.mlirOperationGetPrevInBlock(self._inner) }; - // } - - pub fn block(self: Self) ?Block { - return Block.wrapOr(c.mlirOperationGetBlock(self._inner)); - } - - pub fn parent(self: Self) ?Self { - return Self.wrapOr(c.mlirOperationGetParentOperation(self._inner)); - } - - pub fn region(self: Self, index: usize) Region { - return .{ ._inner = c.mlirOperationGetRegion(self._inner, @intCast(index)) }; - } - - pub fn context(self: Self) Context { - return .{ ._inner = c.mlirOperationGetContext(self._inner) }; - } - - pub fn writeBytecode(self: Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - var writer_with_err: WriterWithErr = .{ .writer = writer }; - c.mlirOperationWriteBytecode( - self._inner, - WriterWithErr.printCallback, - &writer_with_err, - ); - return writer_with_err.check(); - } - - pub fn writeBytecodeWithConfig(self: Self, writer: *std.Io.Writer, config: struct { - desiredEmitedVersion: ?i64 = null, - }) error{ InvalidMlirBytecodeVersion, WriteFailed }!void { - const cfg = c.mlirBytecodeWriterConfigCreate(); - defer c.mlirBytecodeWriterConfigDestroy(cfg); - if (config.desiredEmitedVersion) |v| { - c.mlirBytecodeWriterConfigDesiredEmitVersion(cfg, v); - } - - var writer_with_err: WriterWithErr = .{ .writer = writer }; - try successOr(c.mlirOperationWriteBytecodeWithConfig( - self._inner, - cfg, - &WriterWithErr.printCallback, - &writer_with_err, - ), error.InvalidMlirBytecodeVersion); - return writer_with_err.check(); - } - - /// Enable a full dump of the IR. - /// Usage `std.log.debug("{}", .{ module.op().mlirFormatter(.{}) }); - pub fn mlirFormatter(self: Operation, flags: OpPrintingFlags) MlirFormatter { - return .{ .op = self, .flags = flags }; - } - - const MlirFormatter = struct { - op: Operation, - flags: OpPrintingFlags, - - pub fn format(self: MlirFormatter, writer: *std.Io.Writer) !void { - try self.op.print(writer, self.flags); - } - }; - - pub fn print(self: Self, writer: *std.Io.Writer, flags: OpPrintingFlags) std.Io.Writer.Error!void { - const pflags = flags.create(); - defer c.mlirOpPrintingFlagsDestroy(pflags); - - var writer_err: WriterWithErr = .{ .writer = writer }; - c.mlirOperationPrintWithFlags(self._inner, pflags, WriterWithErr.printCallback, &writer_err); - return writer_err.check(); - } - - pub fn verify(self: Self) bool { - return c.mlirOperationVerify(self._inner); - } - - pub fn getLocation(self: Self) Location { - return .{ ._inner = c.mlirOperationGetLocation(self._inner) }; + pub fn location(self: *const Operation) *const Location { + return @ptrCast(c.mlirOperationGetLocation(self.ptr()).ptr); } pub const WalkOrder = enum(c.MlirWalkOrder) { @@ -950,7 +1213,7 @@ pub const Operation = struct { skip = c.MlirWalkResultSkip, }; - pub fn walk(self: Self, order: WalkOrder, ctx: anytype, walkfn: fn (ctx: anytype, op: Operation) WalkResult) void { + pub fn walk(self: *Operation, order: WalkOrder, ctx: anytype, walkfn: fn (ctx_: @TypeOf(ctx), op: *Operation) WalkResult) void { var inner_ctx = .{ .ctx = ctx }; const ContextType = @TypeOf(inner_ctx); @@ -959,7 +1222,7 @@ pub const Operation = struct { (struct { pub fn callback(op: c.MlirOperation, ctx_: ?*anyopaque) callconv(.c) c.MlirWalkResult { const inner_ctx_: *ContextType = @ptrCast(@alignCast(ctx_)); - return @intFromEnum(walkfn(inner_ctx_.ctx, .{ ._inner = op })); + return @intFromEnum(walkfn(inner_ctx_.ctx, @ptrCast(op.ptr))); } }).callback, &inner_ctx, @@ -967,792 +1230,383 @@ pub const Operation = struct { ); } - pub fn getAttribute(self: Self, pos: usize) NamedAttribute { - return .{ ._inner = c.mlirOperationGetAttribute(self._inner, @intCast(pos)) }; - } + pub const BytecodeWriterConfig = struct { + desired_emit_version: ?usize = null, - pub fn getAttributeByName(self: Self, name_: [:0]const u8) ?Attribute { - return Attribute.wrapOr(c.mlirOperationGetAttributeByName(self._inner, stringRef(name_))); - } - - pub fn setAttributeByName(self: Self, name_: [:0]const u8, attr: Attribute) void { - c.mlirOperationSetAttributeByName(self._inner, stringRef(name_), attr._inner); - } - - pub fn removeAttributeByName(self: Self, name_: [:0]const u8) bool { - return c.mlirOperationRemoveAttributeByName(self._inner, stringRef(name_)); - } - - /// Hash the canonicalized IR, without debug information that can change across builds. - pub fn hash(op: Operation, hasher: *std.hash.XxHash64) void { - // Note: before we where using op.writeBytecode(writer), - // but it crashes on some inputs, notably for unused variables. - // So we use the text representation of the mlir. - // See https://github.com/zml/zml/issues/97. - const flags = OpPrintingFlags.create(.{ .debug_info = false }); - defer c.mlirOpPrintingFlagsDestroy(flags); - - c.mlirOperationPrintWithFlags( - op._inner, - flags, - (struct { - pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { - const _hasher: *std.hash.XxHash64 = @ptrCast(@alignCast(ctx_)); - _hasher.update(str.data[0..str.length]); - } - }).callback, - hasher, - ); - } -}; - -pub const OpPrintingFlags = struct { - elide_large_elements_attrs: ?usize = null, - debug_info: bool = false, - debug_info_pretty_form: bool = true, - print_generic_op_form: bool = false, - use_local_scope: bool = false, - assume_verified: bool = false, - - pub fn create(self: OpPrintingFlags) c.MlirOpPrintingFlags { - const pflags = c.mlirOpPrintingFlagsCreate(); - if (self.elide_large_elements_attrs) |v| { - c.mlirOpPrintingFlagsElideLargeElementsAttrs(pflags, @intCast(v)); - } - c.mlirOpPrintingFlagsEnableDebugInfo(pflags, self.debug_info, self.debug_info_pretty_form); - if (self.print_generic_op_form) { - c.mlirOpPrintingFlagsPrintGenericOpForm(pflags); - } - if (self.use_local_scope) { - c.mlirOpPrintingFlagsUseLocalScope(pflags); - } - if (self.assume_verified) { - c.mlirOpPrintingFlagsAssumeVerified(pflags); - } - return pflags; - } -}; - -pub const OpOperand = struct { - _inner: c.MlirOpOperand, - pub const wrapOr = helpers.wrapOr(OpOperand, c.mlirOpOperandIsNull); - - pub fn owner(self: OpOperand) Operation { - return .{ ._inner = c.mlirOpOperandGetOwner(self._inner) }; - } - - pub fn number(self: OpOperand) usize { - return @intCast(c.mlirOpOperandGetOperandNumber(self._inner)); - } - - pub fn nextUse(self: OpOperand) ?OpOperand { - return wrapOr(c.mlirOpOperandGetNextUse(self._inner)); - } -}; - -pub const Region = struct { - _inner: c.MlirRegion, - - pub const eql = helpers.eql(Region, c.mlirRegionEqual); - pub const deinit = helpers.deinit(Region, c.mlirRegionDestroy); - pub const wrapOr = helpers.wrapOr(Region, c.mlirRegionIsNull); - - const Self = Region; - - pub fn init() !Self { - return Self.wrapOr(c.mlirRegionCreate()) orelse Error.InvalidMlir; - } - - pub fn appendBlock(self: *Self, block: Block) void { - c.mlirRegionAppendOwnedBlock(self._inner, block._inner); - } - - pub fn insertBlock(self: *Self, index: isize, block: Block) void { - c.mlirRegionInsertOwnedBlock(self._inner, index, block._inner); - } - - pub fn insertBlockBefore(self: *Self, reference: Block, block: Block) void { - c.mlirRegionInsertOwnedBlockBefore(self._inner, reference._inner, block._inner); - } - - pub fn insertBlockAfter(self: *Self, reference: Block, block: Block) void { - c.mlirRegionInsertOwnedBlockAfter(self._inner, reference._inner, block._inner); - } - - pub fn firstBlock(self: Self) Block { - return .{ ._inner = c.mlirRegionGetFirstBlock(self._inner) }; - } -}; - -pub const Value = struct { - _inner: c.MlirValue, - - pub const dump = helpers.dump(Value, c.mlirValueDump); - pub const eql = helpers.eql(Value, c.mlirValueEqual); - pub const format = helpers.format(Value, c.mlirValuePrint); - pub const wrapOr = helpers.wrapOr(Value, c.mlirValueIsNull); - - pub fn getType(val: Value) Type { - return .{ ._inner = c.mlirValueGetType(val._inner) }; - } - - pub fn setType(val: *Value, typ: Type) void { - c.mlirValueSetType(val._inner, typ._inner); - } - - pub fn firstUse(val: Value) OpOperand { - return .{ ._inner = c.mlirValueGetFirstUse(val._inner) }; - } - - pub fn replaceAllUsesWith(val: Value, with: Value) void { - c.mlirValueReplaceAllUsesOfWith(val._inner, with._inner); - } - - pub fn owner(val: Value) Operation { - return .{ ._inner = c.mlirOpResultGetOwner(val._inner) }; - } - - pub fn isABlockArgument(val: Value) bool { - return c.mlirValueIsABlockArgument(val._inner); - } - - pub fn isAOpResult(val: Value) bool { - return c.mlirValueIsAOpResult(val._inner); - } - - pub const Kind = union(enum) { - block_argument: BlockArgument, - op_result: Operation, - null, + const Ctx = struct { + self: *const Operation, + config: ?BytecodeWriterConfig, + }; }; - pub fn kind(val: Value) Kind { - if (val.isAOpResult()) { - return .{ .op_result = val.owner() }; - } - if (val.isABlockArgument()) { - return .{ .block_argument = .{ ._inner = val._inner } }; - } - // From MLIR docs: - // https://mlir.llvm.org/doxygen/classmlir_1_1Value.html#details - // > An SSA value is either a BlockArgument or the result of an operation. - return .null; - } -}; + const WriteBytecodeError = error{InvalidMlirBytecodeVersion} || std.Io.Writer.Error; -pub const BlockArgument = struct { - _inner: c.MlirValue, - - pub fn block(arg: BlockArgument) Block { - return .{ ._inner = c.mlirBlockArgumentGetOwner(arg._inner) }; - } - - pub fn number(arg: BlockArgument) usize { - return @bitCast(c.mlirBlockArgumentGetArgNumber(arg._inner)); - } - - pub fn format(self: BlockArgument, writer: *std.Io.Writer) !void { - const value = Value{ ._inner = self._inner }; - return value.format(writer); - } -}; - -pub const Type = struct { - _inner: c.MlirType, - - pub const dump = helpers.dump(Type, c.mlirTypeDump); - pub const eql = helpers.eql(Type, c.mlirTypeEqual); - pub const format = helpers.format(Type, c.mlirTypePrint); - pub const wrapOr = helpers.wrapOr(Type, c.mlirTypeIsNull); - - pub fn parse(ctx: Context, str: [:0]const u8) !Type { - return Type.wrapOr( - c.mlirTypeParseGet(ctx._inner, stringRef(str)), - ) orelse Error.InvalidMlir; - } - - pub fn as(generic: Type, SpecificType: type) ?SpecificType { - if (@hasDecl(SpecificType, "is_a_fn")) { - return if (SpecificType.is_a_fn(generic._inner)) - .{ ._inner = generic._inner } - else - null; - } - @compileError("Mlir subclass of type need `is_a_fn` attribute: " ++ @typeName(SpecificType)); - } - - pub fn fromAny(SpecificType: type) fn (x: SpecificType) Type { - stdx.debug.assertComptime(@hasDecl(SpecificType, "asType"), "Type.fromAny expects a type subclass, got: {}. Missing `asAttr` declaration.", .{SpecificType}); - return struct { - fn cast(x: SpecificType) Type { - return .{ ._inner = x._inner }; + pub fn writeBytecode(self: *const Operation, config: ?BytecodeWriterConfig, writer: *std.Io.Writer) WriteBytecodeError!void { + if (config) |config_| { + const bc_config = c.mlirBytecodeWriterConfigCreate(); + defer c.mlirBytecodeWriterConfigDestroy(bc_config); + if (config_.desired_emit_version) |version| { + c.mlirBytecodeWriterConfigDesiredEmitVersion(bc_config, @intCast(version)); } - }.cast; - } + var sctx: StringCallbackCtx = .{ .writer = writer }; + const logical_result = c.mlirOperationWriteBytecodeWithConfig(self.ptr(), bc_config, stringCallback, &sctx); - pub fn eqlAny(SpecificType: type) fn (SpecificType, SpecificType) bool { - return struct { - fn eql(a: SpecificType, b: SpecificType) bool { - return a.asType().eql(b.asType()); + if (logical_result.value == 0) { + return WriteBytecodeError.InvalidMlirBytecodeVersion; } - }.eql; + + if (sctx.err) |err| { + return err; + } + return; + } + + var sctx: StringCallbackCtx = .{ .writer = writer }; + c.mlirOperationWriteBytecode(self.ptr(), stringCallback, &sctx); + if (sctx.err) |err| { + return err; + } } - pub fn index(ctx: Context) Type { - return IndexType.init(ctx).asType(); + fn fmtBytecodeImpl(ctx: BytecodeWriterConfig.Ctx, writer: *std.io.Writer) std.Io.Writer.Error!void { + try ctx.self.writeBytecode(ctx.config, writer); } - pub fn int(ctx: Context, int_type: IntegerTypes) Type { - return switch (int_type) { - .unknown => @panic("Unknown integer type"), - inline else => |t| IntegerType(t).init(ctx).asType(), - }; - } - - pub fn float(ctx: Context, float_type: FloatTypes) Type { - return switch (float_type) { - inline else => |t| FloatType(t).init(ctx).asType(), - }; - } - - pub fn complex(ctx: Context, complex_type: ComplexTypes) Type { - return switch (complex_type) { - .unknown => @panic("Unknown complex type can't be created like this"), // What's the point ? - inline else => |t| ComplexType(t).init(ctx).asType(), - }; - } - - pub fn tuple(ctx: Context, types: []const Type) Type { - return (TupleType.init(ctx, types) catch unreachable).asType(); - } - - pub fn function(ctx: Context, args: []const Type, results: []const Type) Type { - return (FunctionType.init(ctx, args, results) catch unreachable).asType(); - } - - pub fn tensor(dimensions: []const i64, elem_type: Type) Type { - return RankedTensorType.init(dimensions, elem_type).asType(); + pub fn fmtBytecode(self: *const Operation, config: ?BytecodeWriterConfig) std.fmt.Alt(BytecodeWriterConfig.Ctx, fmtBytecodeImpl) { + return .{ .data = .{ + .self = self, + .config = config, + } }; } }; -pub const IndexType = struct { - _inner: c.MlirType, +pub const Region = opaque { + const M = Methods(Region, c.MlirRegion); - pub const asType = Type.fromAny(IndexType); - pub const eql = Type.eqlAny(IndexType); - pub const format = helpers.format(IndexType, c.mlirTypePrint); + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirRegionEqual); - pub fn init(ctx: Context) IndexType { - return .{ ._inner = c.mlirIndexTypeGet(ctx._inner) }; + pub fn init() *Region { + return @ptrCast(c.mlirRegionCreate().ptr); + } + + pub fn deinit(self: *Region) void { + c.mlirRegionDestroy(self.ptr()); + } + + pub fn appendOwnedBlock(self: *Region, block: *Block) void { + c.mlirRegionAppendOwnedBlock(self.ptr(), block.ptr()); + } + + pub fn insertOwnedBlock(self: *Region, index: usize, block: *Block) void { + c.mlirRegionInsertOwnedBlock(self.ptr(), @intCast(index), block.ptr()); + } + + pub fn insertOwnedBlockBefore(self: *Region, reference: *const Block, block: *Block) void { + c.mlirRegionInsertOwnedBlockBefore(self.ptr(), reference.ptr(), block.ptr()); + } + + pub fn insertOwnedBlockAfter(self: *Region, reference: *const Block, block: *Block) void { + c.mlirRegionInsertOwnedBlockAfter(self.ptr(), reference.ptr(), block.ptr()); + } + + pub fn firstBlock(self: *const Region) ?*Block { + return @ptrCast(c.mlirRegionGetFirstBlock(self.ptr()).ptr); + } + + pub fn takeBody(self: *Region, from: *Region) void { + c.mlirRegionTakeBody(self.ptr(), from.ptr()); + } + + pub fn nextInOperation(self: *const Region) *Region { + return @ptrCast(c.mlirRegionGetNextInOperation(self.ptr()).ptr); } }; -pub const IntegerTypes = enum { - i1, - i2, - i4, +pub const ShapedType = opaque { + const M = Methods(ShapedType, c.MlirType); + + pub const MAX_RANK = 16; + + pub const isAFn = c.mlirTypeIsAShaped; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn elementType(self: *const ShapedType) *const Type { + return @ptrCast(c.mlirShapedTypeGetElementType(self.ptr()).ptr); + } + + pub fn rank(self: *const ShapedType) usize { + return @intCast(c.mlirShapedTypeGetRank(self.ptr())); + } + + pub fn dimension(self: *const ShapedType, dim: usize) i64 { + return c.mlirShapedTypeGetDimSize(self.ptr(), @intCast(dim)); + } +}; + +pub const RankedTensorType = opaque { + const M = Methods(RankedTensorType, c.MlirType); + + pub const isAFn = c.mlirTypeIsARankedTensor; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn get(dimensions: []const i64, elemType: *const Type, encoding: ?*Attribute) *const RankedTensorType { + return @ptrCast(c.mlirRankedTensorTypeGet( + @intCast(dimensions.len), + @ptrCast(dimensions), + elemType.ptr(), + if (encoding) |e| e.ptr() else c.mlirAttributeGetNull(), + ).ptr); + } + + pub fn fromShaped(other: *const ShapedType) *const RankedTensorType { + var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .{}; + for (0..other.rank()) |i| { + dims.appendAssumeCapacity(other.dimension(i)); + } + return get( + dims.constSlice(), + other.elementType(), + ); + } + + pub fn elementType(self: *const RankedTensorType) *const Type { + return @ptrCast(c.mlirShapedTypeGetElementType(self.ptr()).ptr); + } + + pub fn rank(self: *const RankedTensorType) usize { + return @intCast(c.mlirShapedTypeGetRank(self.ptr())); + } + + pub fn dimension(self: *const RankedTensorType, dim: usize) i64 { + return c.mlirShapedTypeGetDimSize(self.ptr(), @intCast(dim)); + } + + pub fn shaped(self: *const RankedTensorType) *const ShapedType { + return @ptrCast(self); + } +}; + +pub fn rankedTensorType(dimensions: []const i64, elem_type: *const Type) *const Type { + return @ptrCast(RankedTensorType.get(dimensions, elem_type, null)); +} + +pub fn rankedTensorTypeWithEncoding(dimensions: []const i64, elem_type: *const Type, encoding: *Attribute) *const Type { + return @ptrCast(RankedTensorType.get(dimensions, elem_type, encoding)); +} + +pub const DenseElementsAttribute = opaque { + const M = Methods(DenseElementsAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsADenseElements; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(shaped_type: *const ShapedType, values: anytype) *const DenseElementsAttribute { + const bytes = std.mem.sliceAsBytes(values); + return @ptrCast(c.mlirDenseElementsAttrRawBufferGet( + shaped_type.ptr(), + @intCast(bytes.len), + @ptrCast(bytes), + ).ptr orelse unreachable); + } +}; + +pub fn denseElementsAttribute(shaped_type: *const ShapedType, values: anytype) *const Attribute { + return @ptrCast(DenseElementsAttribute.init(shaped_type, values)); +} + +pub const DenseArrayTypes = enum { + bool, i8, i16, i32, i64, - si4, - si8, - si16, - si32, - si64, - u2, - u4, - u8, - u16, - u32, - u64, - - unknown, -}; - -pub fn IntegerType(comptime it: IntegerTypes) type { - const Config = switch (it) { - .i1 => .{ 1, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i2 => .{ 2, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i4 => .{ 4, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i8 => .{ 8, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i16 => .{ 16, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i32 => .{ 32, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i64 => .{ 64, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .si4 => .{ 4, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, - .si8 => .{ 8, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, - .si16 => .{ 16, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, - .si32 => .{ 32, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, - .si64 => .{ 64, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, - .u2 => .{ 2, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .u4 => .{ 4, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .u8 => .{ 8, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .u16 => .{ 16, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .u32 => .{ 32, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .u64 => .{ 64, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .unknown => .{ 0, null, null }, - }; - - return struct { - _inner: c.MlirType, - - const Int = @This(); - pub const is_a_fn = switch (it) { - .unknown => c.mlirTypeIsAInteger, - else => typeIsAIntegerExact, - }; - - pub const asType = Type.fromAny(Int); - pub const eql = Type.eqlAny(Int); - pub const format = helpers.format(Int, c.mlirTypePrint); - - fn typeIsAIntegerExact(typ: c.MlirType) callconv(.c) bool { - const bit_width = Config[0]; - const is_sign = Config[2]; - return c.mlirTypeIsAInteger(typ) and (c.mlirIntegerTypeGetWidth(typ) == bit_width) and is_sign(typ); - } - - pub const BitWidth = Config[0]; - - pub const init = if (it != .unknown) struct { - pub fn init(ctx: Context) Int { - const type_get = Config[1]; - return .{ ._inner = type_get(ctx._inner, BitWidth) }; - } - }.init else {}; - }; -} - -pub const FloatTypes = enum { - f4e2m1fn, - f8e3m4, - f8e4m3, - f8e4m3b11fnuz, - f8e4m3fn, - f8e4m3fnuz, - f8e5m2, - f8e5m2fnuz, - f8e8m0fnu, - bf16, - f16, f32, f64, - pub fn asType(self: FloatTypes, ctx: Context) Type { - return switch (self) { - inline else => |ft| FloatType(ft).init(ctx).asType(), + fn ZigType(comptime ElementType: @This()) type { + return switch (ElementType) { + .bool => i32, + .i8 => i8, + .i16 => i16, + .i32 => i32, + .i64 => i64, + .f32 => f32, + .f64 => f64, }; } }; -pub fn FloatType(comptime ft: FloatTypes) type { - const Config: struct { - *const fn (c.MlirType) callconv(.c) bool, - *const fn (c.MlirContext) callconv(.c) c.MlirType, - } = switch (ft) { - .f4e2m1fn => .{ c.mlirTypeIsAFloat4E2M1FN, c.mlirFloat4E2M1FNTypeGet }, - .f8e3m4 => .{ c.mlirTypeIsAFloat8E3M4, c.mlirFloat8E3M4TypeGet }, - .f8e4m3 => .{ c.mlirTypeIsAFloat8E4M3, c.mlirFloat8E4M3TypeGet }, - .f8e4m3b11fnuz => .{ c.mlirTypeIsAFloat8E4M3B11FNUZ, c.mlirFloat8E4M3B11FNUZTypeGet }, - .f8e4m3fn => .{ c.mlirTypeIsAFloat8E4M3FN, c.mlirFloat8E4M3FNTypeGet }, - .f8e4m3fnuz => .{ c.mlirTypeIsAFloat8E4M3FNUZ, c.mlirFloat8E4M3FNUZTypeGet }, - .f8e5m2 => .{ c.mlirTypeIsAFloat8E5M2, c.mlirFloat8E5M2TypeGet }, - .f8e5m2fnuz => .{ c.mlirTypeIsAFloat8E5M2FNUZ, c.mlirFloat8E5M2FNUZTypeGet }, - .f8e8m0fnu => .{ c.mlirTypeIsAFloat8E8M0FNU, c.mlirFloat8E8M0FNUTypeGet }, - .bf16 => .{ c.mlirTypeIsABF16, c.mlirBF16TypeGet }, - .f16 => .{ c.mlirTypeIsAF16, c.mlirF16TypeGet }, - .f32 => .{ c.mlirTypeIsAF32, c.mlirF32TypeGet }, - .f64 => .{ c.mlirTypeIsAF64, c.mlirF64TypeGet }, +pub fn DenseArrayAttribute(comptime ElementType: DenseArrayTypes) type { + const ElementTypeZig = ElementType.ZigType(); + + const _isAFn, const getFn, const getElementFn = switch (ElementType) { + .bool => .{ c.mlirAttributeIsADenseBoolArray, c.mlirDenseBoolArrayGet, c.mlirDenseBoolArrayGetElement }, + .i8 => .{ c.mlirAttributeIsADenseI8Array, c.mlirDenseI8ArrayGet, c.mlirDenseI8ArrayGetElement }, + .i16 => .{ c.mlirAttributeIsADenseI16Array, c.mlirDenseI16ArrayGet, c.mlirDenseI16ArrayGetElement }, + .i32 => .{ c.mlirAttributeIsADenseI32Array, c.mlirDenseI32ArrayGet, c.mlirDenseI32ArrayGetElement }, + .i64 => .{ c.mlirAttributeIsADenseI64Array, c.mlirDenseI64ArrayGet, c.mlirDenseI64ArrayGetElement }, + .f32 => .{ c.mlirAttributeIsADenseF32Array, c.mlirDenseF32ArrayGet, c.mlirDenseF32ArrayGetElement }, + .f64 => .{ c.mlirAttributeIsADenseF64Array, c.mlirDenseF64ArrayGet, c.mlirDenseF64ArrayGetElement }, }; return struct { - _inner: c.MlirType, - const Self = @This(); - pub const is_a_fn = Config[0]; + const M = Methods(@This(), c.MlirAttribute); - pub const asType = Type.fromAny(Self); - pub const eql = Type.eqlAny(Self); - pub const format = helpers.format(Self, c.mlirTypePrint); + pub const isAFn = _isAFn; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); - pub fn init(ctx: Context) Self { - const type_get = Config[1]; - return .{ ._inner = type_get(ctx._inner) }; + pub fn init(ctx: *Context, values: []const ElementTypeZig) *const Self { + return @ptrCast(getFn(ctx.ptr(), @intCast(values.len), @ptrCast(values)).ptr); + } + + pub fn element(self: *const Self, pos: usize) ElementTypeZig { + return getElementFn(self.ptr(), @intCast(pos)); + } + + pub fn numElements(self: *const Self) usize { + return @intCast(c.mlirDenseArrayGetNumElements(self.ptr())); } }; } -pub const ComplexTypes = enum { - c64, - c128, - - unknown, -}; - -pub fn ComplexType(comptime ct: ComplexTypes) type { - return struct { - _inner: c.MlirType, - const Complex = @This(); - - fn mlirC64TypeGet(ctx: c.MlirContext) callconv(.c) c.MlirType { - return c.mlirComplexTypeGet(c.mlirF32TypeGet(ctx)); - } - - fn mlirC128TypeGet(ctx: c.MlirContext) callconv(.c) c.MlirType { - return c.mlirComplexTypeGet(c.mlirF64TypeGet(ctx)); - } - - fn mlirTypeIsAC64(typ: c.MlirType) callconv(.c) bool { - const element_type: c.MlirType = c.mlirComplexTypeGetElementType(typ); - return c.mlirTypeIsAF32(element_type); - } - - fn mlirTypeIsAC128(typ: c.MlirType) callconv(.c) bool { - const element_type: c.MlirType = c.mlirComplexTypeGetElementType(typ); - return c.mlirTypeIsAF64(element_type); - } - - const Config = switch (ct) { - .c64 => .{ mlirTypeIsAC64, mlirC64TypeGet }, - .c128 => .{ mlirTypeIsAC128, mlirC128TypeGet }, - .unknown => .{ c.mlirTypeIsAComplex, null }, - }; - - fn typeIsAUnknownComplex(typ: c.MlirType) callconv(.c) bool { - return c.mlirTypeIsAComplex(typ); - } - - pub const is_a_fn = Config[0]; - - pub const asType = Type.fromAny(Complex); - pub const eql = Type.eqlAny(Complex); - pub const format = helpers.format(Complex, c.mlirTypePrint); - pub const ComplexTypeType: ComplexTypes = ct; - - pub const init = if (ct != .unknown) struct { - pub fn init(ctx: Context) Complex { - const type_get = Config[1]; - return .{ ._inner = type_get(ctx._inner) }; - } - }.init else {}; - }; +pub fn denseArrayAttribute(ctx: *Context, comptime ElementType: DenseArrayTypes, values: []const ElementType.ZigType()) *const Attribute { + return @ptrCast(DenseArrayAttribute(ElementType).init(ctx, @ptrCast(values))); } -pub const TupleType = struct { - _inner: c.MlirType, - pub const is_a_fn = c.mlirTypeIsATuple; +pub const ArrayAttribute = opaque { + const M = Methods(ArrayAttribute, c.MlirAttribute); - pub fn init(ctx: Context, elements: []const Type) !TupleType { - const tuple_type = c.mlirTupleTypeGet( - ctx._inner, - @intCast(elements.len), - @ptrCast(elements.ptr), - ); - return .{ ._inner = .{ .ptr = tuple_type.ptr orelse return error.InvalidMlir } }; + pub const isAFn = c.mlirAttributeIsAArray; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, attrs: []const *const Attribute) *const ArrayAttribute { + return @ptrCast(c.mlirArrayAttrGet(ctx.ptr(), @intCast(attrs.len), @ptrCast(attrs)).ptr); } - pub fn getNumTypes(self: TupleType) usize { - return @intCast(c.mlirTupleTypeGetNumTypes(self._inner)); + pub fn element(self: *const ArrayAttribute, index: usize) Attribute { + return @ptrCast(c.mlirArrayAttrGetElement(self.ptr(), @intCast(index)).ptr); } - pub fn getElementType(self: TupleType, index: usize) Type { - return .{ ._inner = c.mlirTupleTypeGetType(self._inner, @intCast(index)) }; + pub fn numElements(self: *const ArrayAttribute) usize { + return @intCast(c.mlirArrayAttrGetNumElements(self.ptr())); } - - pub const asType = Type.fromAny(TupleType); }; -pub const FunctionType = struct { - _inner: c.MlirType, - pub const is_a_fn = c.mlirTypeIsAFunction; - pub const asType = Type.fromAny(FunctionType); - pub const eql = Type.eqlAny(FunctionType); +pub fn arrayAttribute(ctx: *Context, attrs: []const *const Attribute) *const Attribute { + return @ptrCast(ArrayAttribute.init(ctx, attrs)); +} - pub fn init(ctx: Context, args: []const Type, results: []const Type) !FunctionType { - const func_type = c.mlirFunctionTypeGet( - ctx._inner, +pub const FunctionType = opaque { + const M = Methods(FunctionType, c.MlirType); + pub const isAFn = c.mlirTypeIsAFunction; + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn init(ctx: *Context, args: []const *const Type, results: []const *const Type) *const FunctionType { + return @ptrCast(c.mlirFunctionTypeGet( + ctx.ptr(), @intCast(args.len), @ptrCast(args.ptr), @intCast(results.len), @ptrCast(results.ptr), - ); - return .{ ._inner = .{ .ptr = func_type.ptr orelse return error.InvalidMlir } }; + ).ptr); } }; -pub const RankedTensorType = struct { - _inner: c.MlirType, - pub const is_a_fn = c.mlirTypeIsARankedTensor; - pub const asType = Type.fromAny(RankedTensorType); - pub const eql = Type.eqlAny(RankedTensorType); - pub const format = helpers.format(RankedTensorType, c.mlirTypePrint); - - pub fn init(dimensions: []const i64, elemType: Type) RankedTensorType { - return .{ ._inner = c.mlirRankedTensorTypeGet( - @intCast(dimensions.len), - @ptrCast(dimensions.ptr), - elemType._inner, - c.mlirAttributeGetNull(), - ) }; - } - - pub fn getElementType(self: RankedTensorType) Type { - return .{ ._inner = c.mlirShapedTypeGetElementType(self._inner) }; - } - - pub fn getRank(self: RankedTensorType) usize { - return @intCast(c.mlirShapedTypeGetRank(self._inner)); - } - - pub fn getDimension(self: RankedTensorType, dim: usize) i64 { - return c.mlirShapedTypeGetDimSize(self._inner, @intCast(dim)); - } -}; - -pub const Dialect = struct { - _inner: c.MlirDialect, - - pub fn getContext(self: Dialect) Context { - return .{ ._inner = c.mlirDialectGetContext(self._inner) }; - } - - pub fn getNamespace(self: Dialect) []const u8 { - return fromStringRef(c.mlirDialectGetNamespace(self._inner)); - } -}; - -pub const DialectHandle = struct { - _inner: c.MlirDialectHandle, - - pub fn getNamespace(self: DialectHandle) []const u8 { - return fromStringRef(c.mlirDialectHandleGetNamespace(self._inner)); - } - - pub fn insertDialect(self: DialectHandle, registry: Registry) void { - c.mlirDialectHandleInsertDialect(self._inner, registry._inner); - } - - pub fn registerDialect(self: DialectHandle, ctx: Context) void { - c.mlirDialectHandleRegisterDialect(self._inner, ctx._inner); - } - - pub fn loadDialect(self: DialectHandle, ctx: Context) Dialect { - return .{ ._inner = c.mlirDialectHandleLoadDialect(self._inner, ctx._inner) }; - } - - pub fn fromString(comptime namespace: []const u8) DialectHandle { - return .{ ._inner = @field(c, "mlirGetDialectHandle__" ++ namespace ++ "__")() }; - } -}; - -pub const Location = struct { - _inner: c.MlirLocation, - - pub const eql = helpers.eql(Location, c.mlirLocationEqual); - pub const format = helpers.format(Location, c.mlirLocationPrint); - - pub fn fromSrc(ctx: Context, src: std.builtin.SourceLocation) Location { - return .{ ._inner = c.mlirLocationFileLineColGet( - ctx._inner, - stringRef(src.file), - @intCast(src.line), - @intCast(src.column), - ) }; - } - - pub fn fileLineCol(ctx: Context, file: []const u8, line: usize, column: usize) Location { - return .{ ._inner = c.mlirLocationFileLineColGet( - ctx._inner, - stringRef(file), - @intCast(line), - @intCast(column), - ) }; - } - - pub fn callSite(callee: Location, caller: Location) Location { - return .{ ._inner = c.mlirLocationCallSiteGet(callee._inner, caller._inner) }; - } - - pub fn fused(ctx: Context, locations: []const Location, metadata: Attribute) Location { - return .{ ._inner = c.mlirLocationFusedGet( - ctx._inner, - @intCast(locations.len), - @ptrCast(locations.ptr), - metadata._inner, - ) }; - } - - pub fn named(loc: Location, ctx: Context, loc_name: []const u8) Location { - return .{ ._inner = c.mlirLocationNameGet(ctx._inner, stringRef(loc_name), loc._inner) }; - } - - pub fn namedFmt(loc: Location, ctx: Context, comptime fmt: [:0]const u8, args: anytype) Location { - var buf: [256]u8 = undefined; - var writer: std.Io.Writer = .fixed(&buf); - writer.print(fmt, args) catch { - buf[256 - 3 ..].* = "...".*; - }; - return loc.named(ctx, writer.buffered()); - } - - pub fn unknown(ctx: Context) Location { - return .{ ._inner = c.mlirLocationUnknownGet(ctx._inner) }; - } -}; - -pub const Block = struct { - _inner: c.MlirBlock, - - pub const wrapOr = helpers.wrapOr(Block, c.mlirBlockIsNull); - pub const deinit = helpers.deinit(Block, c.mlirBlockDestroy); - pub const eql = helpers.eql(Block, c.mlirBlockEqual); - - pub fn init(args: []const Type, locs: []const Location) !Block { - const block = Block.wrapOr( - c.mlirBlockCreate(@intCast(args.len), @ptrCast(args.ptr), @ptrCast(locs.ptr)), - ); - return block orelse error.InvalidMlir; - } - - pub fn argument(self: Block, index: usize) Value { - return .{ ._inner = c.mlirBlockGetArgument(self._inner, @intCast(index)) }; - } - - pub fn numArguments(self: Block) usize { - return @intCast(c.mlirBlockGetNumArguments(self._inner)); - } - - pub fn addArgument(self: *Block, typ: Type, loc: Location) Value { - return .{ ._inner = c.mlirBlockAddArgument(self._inner, typ._inner, loc._inner) }; - } - - pub fn insertArgument(self: *Block, index: usize, typ: Type, loc: Location) Value { - return .{ ._inner = c.mlirBlockInsertArgument(self._inner, @intCast(index), typ._inner, loc._inner) }; - } - - pub fn equals(self: Block, other: Block) bool { - return c.mlirBlockEqual(self._inner, other._inner); - } - - pub fn appendOperation(self: Block, op: Operation) void { - c.mlirBlockAppendOwnedOperation(self._inner, op._inner); - } - - pub fn appendOperations(self: *Block, ops: []const Operation) void { - for (ops) |op| { - c.mlirBlockAppendOwnedOperation(self._inner, op._inner); - } - } - - pub const RecursiveOpts = enum { open, hermetic }; - - pub fn appendValueRecursive(self: Block, value: Value, opt: RecursiveOpts) void { - switch (value.kind()) { - .op_result => |parent_op| self.appendOperationRecursive(parent_op, opt), - .block_argument => |arg| { - // Hermetic blocks are not allowed to use arguments from other blocks. - stdx.debug.assert(opt == .open or self.eql(arg.block()), "Can't add {f} from {*} block to {*} block", .{ arg, arg.block()._inner.ptr, self._inner.ptr }); - }, - .null => @panic("InvalidMlir"), - } - } - - pub fn appendOperationRecursive(self: Block, op: Operation, opt: RecursiveOpts) void { - if (op.block()) |prev_block| { - // Hermetic blocks are not allowed to reference values from other blocks. - stdx.debug.assert(opt == .open or self.equals(prev_block), "Can't add {} from {*} block to {*} block", .{ op, prev_block._inner.ptr, self._inner.ptr }); - return; - } - for (0..op.numOperands()) |i| { - self.appendValueRecursive(op.operand(i), opt); - } - self.appendOperation(op); - } -}; - -pub const helpers = struct { - pub fn eql(T: type, equal_fn: fn (@FieldType(T, "_inner"), @FieldType(T, "_inner")) callconv(.c) bool) fn (T, T) bool { - return struct { - fn eql(a: T, b: T) bool { - return equal_fn(a._inner, b._inner); - } - }.eql; - } - - pub fn deinit(T: type, deinit_fn: fn (@FieldType(T, "_inner")) callconv(.c) void) fn (*T) void { - return struct { - fn deinit(a: *T) void { - deinit_fn(a._inner); - a.* = undefined; - } - }.deinit; - } - - pub fn dump(T: type, dump_fn: fn (@FieldType(T, "_inner")) callconv(.c) void) fn (T) void { - return struct { - fn dump(a: T) void { - return dump_fn(a._inner); - } - }.dump; - } - - pub fn isNull(T: type, is_null_fn: fn (@FieldType(T, "_inner")) callconv(.c) bool) fn (T) bool { - return struct { - fn isNull(a: T) bool { - return is_null_fn(a._inner); - } - }.isNull; - } - - pub fn format( - Any: type, - print_fn: fn (@FieldType(Any, "_inner"), ?*const MlirStrCallback, ?*anyopaque) callconv(.c) void, - ) fn (Any, *std.Io.Writer) std.Io.Writer.Error!void { - return struct { - pub fn format(self: Any, writer: *std.Io.Writer) std.Io.Writer.Error!void { - try callPrintFn(Any, self, print_fn, writer); - } - }.format; - } - - pub fn wrapOr(T: type, is_null_fn: fn (@FieldType(T, "_inner")) callconv(.c) bool) fn (@FieldType(T, "_inner")) ?T { - return struct { - fn wrapOr(inner: @FieldType(T, "_inner")) ?T { - if (is_null_fn(inner)) return null; - return .{ ._inner = inner }; - } - }.wrapOr; - } -}; - -pub const MlirStrCallback = fn (c.MlirStringRef, ?*anyopaque) callconv(.c) void; - -pub fn callPrintFn( - T: type, - value: T, - print_fn: fn (@FieldType(T, "_inner"), ?*const MlirStrCallback, ?*anyopaque) callconv(.c) void, - writer: *std.Io.Writer, -) std.Io.Writer.Error!void { - var writer_with_err: WriterWithErr = .{ .writer = writer }; - print_fn(value._inner, &WriterWithErr.printCallback, &writer_with_err); - return writer_with_err.check(); +pub fn functionType(ctx: *Context, args: []const *const Type, results: []const *const Type) *const Type { + return @ptrCast(FunctionType.init(ctx, args, results)); } -pub const WriterWithErr = struct { - writer: *std.Io.Writer, - err: ?std.Io.Writer.Error = null, +pub const FlatSymbolRefAttribute = opaque { + const M = Methods(FlatSymbolRefAttribute, c.MlirAttribute); - pub fn printCallback(mlir_str: c.MlirStringRef, opaque_ctx: ?*anyopaque) callconv(.c) void { - var ctx: *WriterWithErr = @ptrCast(@alignCast(opaque_ctx)); - if (ctx.err) |_| return; - ctx.writer.writeAll(fromStringRef(mlir_str)) catch |err| { - ctx.err = err; - return; - }; + pub const isAFn = c.mlirAttributeIsAFlatSymbolRef; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, symbol_: []const u8) *const FlatSymbolRefAttribute { + return @ptrCast(c.mlirFlatSymbolRefAttrGet(ctx.ptr(), stringRef(symbol_)).ptr); } - pub fn check(self: WriterWithErr) std.Io.Writer.Error!void { - if (self.err) |err| return err; + pub fn symbol(self: *const FlatSymbolRefAttribute) []const u8 { + return string(c.mlirFlatSymbolRefAttrGetValue(self.ptr())); } }; + +pub fn flatSymbolRefAttribute(ctx: *Context, symbol: []const u8) *const Attribute { + return @ptrCast(FlatSymbolRefAttribute.init(ctx, symbol)); +} + +pub const UnitAttribute = opaque { + const M = Methods(UnitAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAUnit; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn get(ctx: *Context) *const UnitAttribute { + return @ptrCast(c.mlirUnitAttrGet(ctx.ptr()).ptr); + } +}; + +pub fn unitAttribute(ctx: *Context) *const Attribute { + return @ptrCast(UnitAttribute.get(ctx)); +} + +pub const MemRefType = opaque { + const M = Methods(MemRefType, c.MlirType); + + pub const isAFn = c.mlirTypeIsAInteger; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn init(element_type: *const Type, shape: []const i64, layout: ?*const Attribute, memory_space: ?*const Attribute) *const MemRefType { + return @ptrCast(c.mlirMemRefTypeGet( + element_type.ptr(), + @intCast(shape.len), + @ptrCast(shape), + if (layout) |l| l.ptr() else .{}, + if (memory_space) |m| m.ptr() else .{}, + ).ptr); + } + + pub fn fromShaped(other: *const ShapedType) *const MemRefType { + var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .{}; + for (0..other.rank()) |i| { + dims.appendAssumeCapacity(other.dimension(i)); + } + return init( + other.elementType(), + dims.constSlice(), + null, + null, + ); + } + + pub fn shaped(self: *const MemRefType) *const ShapedType { + return @ptrCast(self); + } +}; + +pub fn memRefType(element_type: *const Type, shape: []const i64, layout: ?*const Attribute, memory_space: ?*const Attribute) *const Type { + return @ptrCast(MemRefType.init(element_type, shape, layout, memory_space)); +} From 897c4dc87d34cdb033bc39f6773f85ef24da6ec3 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 2 Dec 2025 18:00:48 +0100 Subject: [PATCH 072/881] Ported almost all previous Tensor operation to new mlir api * Tests are disabled for now and marked with TODOs * Missing operations are also marked with TODOs --- zml2/BUILD.bazel | 4 + zml2/dtype.zig | 290 +-- zml2/meta.zig | 776 ++++++++ zml2/mlirx.zig | 80 + zml2/shape.zig | 1097 +++++++++++ zml2/tensor.zig | 4510 ++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 6613 insertions(+), 144 deletions(-) create mode 100644 zml2/meta.zig create mode 100644 zml2/mlirx.zig create mode 100644 zml2/shape.zig create mode 100644 zml2/tensor.zig diff --git a/zml2/BUILD.bazel b/zml2/BUILD.bazel index 84eb6dda..90c95102 100644 --- a/zml2/BUILD.bazel +++ b/zml2/BUILD.bazel @@ -14,6 +14,10 @@ zig_library( "pjrtx.zig", "dtype.zig", "floats.zig", + "shape.zig", + "tensor.zig", + "mlirx.zig", + "meta.zig", ], zigopts = [ "-lc", diff --git a/zml2/dtype.zig b/zml2/dtype.zig index b14f483e..cb8af137 100644 --- a/zml2/dtype.zig +++ b/zml2/dtype.zig @@ -137,7 +137,7 @@ pub const DataType = enum(u8) { } pub fn toZigType(comptime dtype: DataType) type { - return @FieldType(Data, @tagName(dtype)); + return @FieldType(Value, @tagName(dtype)); } pub fn isSignedInt(dtype: DataType) bool { @@ -195,171 +195,171 @@ pub const DataType = enum(u8) { try std.testing.expectEqual(DataType.c64.resolvePeerType(.f32), null); } - pub fn zero(dtype: DataType) Data { - return Data.init(dtype, 0); + pub fn zero(dtype: DataType) Value { + return .init(dtype, 0); } - pub fn one(dtype: DataType) Data { - return Data.init(dtype, 1); + pub fn one(dtype: DataType) Value { + return .init(dtype, 1); } - pub fn minValue(dtype: DataType) Data { + pub fn minValue(dtype: DataType) Value { return switch (dtype) { .bool => .{ .bool = false }, - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).zero), - inline .f8e5m2, .bf16 => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).minus_inf), - inline .f16, .f32, .f64 => |tag| @unionInit(Data, @tagName(tag), -std.math.inf(@FieldType(Data, @tagName(tag)))), - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Data, @tagName(tag), std.math.minInt(@FieldType(Data, @tagName(tag)))), + inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).zero), + inline .f8e5m2, .bf16 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).minus_inf), + inline .f16, .f32, .f64 => |tag| @unionInit(Value, @tagName(tag), -std.math.inf(@FieldType(Value, @tagName(tag)))), + inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Value, @tagName(tag), std.math.minInt(@FieldType(Value, @tagName(tag)))), inline else => |tag| @panic("Unsupported type: " ++ @tagName(tag)), }; } - pub fn maxValue(dtype: DataType) Data { + pub fn maxValue(dtype: DataType) Value { return switch (dtype) { .bool => .{ .bool = true }, - inline .f4e2m1, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz, .f8e8m0 => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), - inline .f8e3m4, .f8e4m3, .f8e5m2, .bf16 => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).inf), - inline .f16, .f32, .f64 => |tag| @unionInit(Data, @tagName(tag), std.math.inf(@FieldType(Data, @tagName(tag)))), - inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Data, @tagName(tag), std.math.maxInt(@FieldType(Data, @tagName(tag)))), + inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), + inline .f8e5m2, .bf16 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).inf), + inline .f16, .f32, .f64 => |tag| @unionInit(Value, @tagName(tag), std.math.inf(@FieldType(Value, @tagName(tag)))), + inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Value, @tagName(tag), std.math.maxInt(@FieldType(Value, @tagName(tag)))), inline .c64, .c128 => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), }; } - pub fn constant(dtype: DataType, value: anytype) Data { - return Data.init(dtype, value); + pub fn constant(dtype: DataType, value: anytype) Value { + return .init(dtype, value); } -}; -pub const Data = union(DataType) { - bool: bool, - f4e2m1: floats.Float4E2M1, - f8e3m4: floats.Float8E3M4, - f8e4m3: floats.Float8E4M3, - f8e4m3b11fnuz: floats.Float8E4M3B11FNUZ, - f8e4m3fn: floats.Float8E4M3FN, - f8e4m3fnuz: floats.Float8E4M3FNUZ, - f8e5m2: floats.Float8E5M2, - f8e5m2fnuz: floats.Float8E5M2FNUZ, - f8e8m0: floats.Float8E8M0, - bf16: floats.BFloat16, - f16: f16, - f32: f32, - f64: f64, - i2: i2, - i4: i4, - i8: i8, - i16: i16, - i32: i32, - i64: i64, - u2: u2, - u4: u4, - u8: u8, - u16: u16, - u32: u32, - u64: u64, - c64: C64, - c128: C128, + pub const Value = union(DataType) { + bool: bool, + f4e2m1: floats.Float4E2M1, + f8e3m4: floats.Float8E3M4, + f8e4m3: floats.Float8E4M3, + f8e4m3b11fnuz: floats.Float8E4M3B11FNUZ, + f8e4m3fn: floats.Float8E4M3FN, + f8e4m3fnuz: floats.Float8E4M3FNUZ, + f8e5m2: floats.Float8E5M2, + f8e5m2fnuz: floats.Float8E5M2FNUZ, + f8e8m0: floats.Float8E8M0, + bf16: floats.BFloat16, + f16: f16, + f32: f32, + f64: f64, + i2: i2, + i4: i4, + i8: i8, + i16: i16, + i32: i32, + i64: i64, + u2: u2, + u4: u4, + u8: u8, + u16: u16, + u32: u32, + u64: u64, + c64: C64, + c128: C128, - /// Creates `Data` from a `value`. - /// - /// If the `dtype` and `@TypeOf(value)` are incompatible - /// or a cast from `value` to `FieldType(dtype)` would - /// be lossy, a panic occurs. - pub fn init(dtype_: DataType, value: anytype) Data { - const T = @TypeOf(value); - const Ti = @typeInfo(T); + /// Creates `Data` from a `value`. + /// + /// If the `dtype` and `@TypeOf(value)` are incompatible + /// or a cast from `value` to `FieldType(dtype)` would + /// be lossy, a panic occurs. + pub fn init(dtype_: DataType, value: anytype) Value { + const T = @TypeOf(value); + const Ti = @typeInfo(T); - return switch (dtype_) { - .bool => switch (Ti) { - .bool => .{ .bool = value }, - .comptime_int, .int, .comptime_float, .float => .{ .bool = value != 0 }, - else => @panic("Could not create Data of type bool from value of type " ++ @typeName(T)), - }, - inline .f4e2m1, .f8e3m4, .f8e4m3, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .f8e8m0, .bf16 => |tag| switch (Ti) { - .comptime_int, .int => @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).fromF32(@floatFromInt(value))), - .comptime_float, .float => @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).fromF32(@floatCast(value))), - else => @panic("Could not create Data of type bf16 from value of type " ++ @typeName(T)), - }, - inline .f16, .f32, .f64 => |tag| switch (Ti) { - .comptime_int, .int => @unionInit(Data, @tagName(tag), @floatFromInt(value)), - .comptime_float, .float => @unionInit(Data, @tagName(tag), @floatCast(value)), - else => @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), - }, - inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| switch (Ti) { - .comptime_int => blk: { - const OutT = @FieldType(Data, @tagName(tag)); - if (value >= std.math.minInt(OutT) and value <= std.math.maxInt(OutT)) { - break :blk @unionInit(Data, @tagName(tag), @intCast(value)); - } else { - @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)); - } + return switch (dtype_) { + .bool => switch (Ti) { + .bool => .{ .bool = value }, + .comptime_int, .int, .comptime_float, .float => .{ .bool = value != 0 }, + else => @panic("Could not create Value of type bool from value of type " ++ @typeName(T)), }, - .int => @unionInit(Data, @tagName(tag), @intCast(value)), - else => @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), - }, - .c64 => switch (T) { - C64 => .{ .c64 = value }, - C128 => .{ .c64 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, - else => @panic("Could not create Data of type c64 from value of type " ++ @typeName(T)), - }, - .c128 => switch (T) { - C64 => .{ .c128 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, - C128 => .{ .c128 = value }, - else => @panic("Could not create Data of type c128 from value of type " ++ @typeName(T)), - }, - }; - } - - test init { - try std.testing.expectEqual(20.0, Data.init(.f16, 20).f16); - try std.testing.expectEqual(20.5, Data.init(.f16, 20.5).f16); - try std.testing.expectEqual(20, Data.init(.f16, @as(u8, 20)).f16); - try std.testing.expectEqual(-20, Data.init(.f16, @as(i8, -20)).f16); - try std.testing.expectEqual(2000.5, Data.init(.f16, @as(f32, 2000.5)).f16); - - try std.testing.expectEqual(true, Data.init(.bool, true).bool); - - try std.testing.expectEqual(10, Data.init(.u8, 10).u8); - try std.testing.expectEqual(10, Data.init(.u8, @as(u16, 10)).u8); - - try std.testing.expectEqual(10, Data.init(.i8, 10).i8); - try std.testing.expectEqual(10, Data.init(.i8, @as(u16, 10)).i8); - try std.testing.expectEqual(-10, Data.init(.i8, -10).i8); - try std.testing.expectEqual(-10, Data.init(.i8, @as(i16, -10)).i8); - - try std.testing.expectEqual(C64.init(1, 2), Data.init(.c64, C64.init(1, 2)).c64); - try std.testing.expectEqual(C64.init(1, 2), Data.init(.c64, C128.init(1, 2)).c64); - try std.testing.expectEqual(C128.init(1, 2), Data.init(.c128, C128.init(1, 2)).c128); - try std.testing.expectEqual(C128.init(1, 2), Data.init(.c128, C64.init(1, 2)).c128); - } - - pub fn dtype(self: Data) DataType { - return std.meta.activeTag(self); - } - - pub fn constSlice(data: *const Data) []const u8 { - return switch (data.*) { - inline else => |*value| std.mem.asBytes(value), - }; - } - - pub fn as(self: Data, comptime T: type) T { - // TODO allow more lossless conversions - switch (@typeInfo(T)) { - .bool => return self.bool, - .float => switch (self) { - inline .f16, .f32, .f64 => |v| return @floatCast(v), - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16 => |v| return @floatCast(v.toF32()), - else => {}, - }, - .int => switch (self) { - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |v| return @intCast(v), - else => {}, - }, - else => {}, + inline .f4e2m1, .f8e3m4, .f8e4m3, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .f8e8m0, .bf16 => |tag| switch (Ti) { + .comptime_int, .int => @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).fromF32(@floatFromInt(value))), + .comptime_float, .float => @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).fromF32(@floatCast(value))), + else => @panic("Could not create Value of type bf16 from value of type " ++ @typeName(T)), + }, + inline .f16, .f32, .f64 => |tag| switch (Ti) { + .comptime_int, .int => @unionInit(Value, @tagName(tag), @floatFromInt(value)), + .comptime_float, .float => @unionInit(Value, @tagName(tag), @floatCast(value)), + else => @panic("Could not create Value of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), + }, + inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| switch (Ti) { + .comptime_int => blk: { + const OutT = @FieldType(Value, @tagName(tag)); + if (value >= std.math.minInt(OutT) and value <= std.math.maxInt(OutT)) { + break :blk @unionInit(Value, @tagName(tag), @intCast(value)); + } else { + @panic("Could not create Value of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)); + } + }, + .int => @unionInit(Value, @tagName(tag), @intCast(value)), + else => @panic("Could not create Value of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), + }, + .c64 => switch (T) { + C64 => .{ .c64 = value }, + C128 => .{ .c64 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, + else => @panic("Could not create Value of type c64 from value of type " ++ @typeName(T)), + }, + .c128 => switch (T) { + C64 => .{ .c128 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, + C128 => .{ .c128 = value }, + else => @panic("Could not create Value of type c128 from value of type " ++ @typeName(T)), + }, + }; } - std.debug.panic("Unsupported conversion {} -> {s}", .{ self.dtype(), @typeName(T) }); - } + + test init { + try std.testing.expectEqual(20.0, Value.init(.f16, 20).f16); + try std.testing.expectEqual(20.5, Value.init(.f16, 20.5).f16); + try std.testing.expectEqual(20, Value.init(.f16, @as(u8, 20)).f16); + try std.testing.expectEqual(-20, Value.init(.f16, @as(i8, -20)).f16); + try std.testing.expectEqual(2000.5, Value.init(.f16, @as(f32, 2000.5)).f16); + + try std.testing.expectEqual(true, Value.init(.bool, true).bool); + + try std.testing.expectEqual(10, Value.init(.u8, 10).u8); + try std.testing.expectEqual(10, Value.init(.u8, @as(u16, 10)).u8); + + try std.testing.expectEqual(10, Value.init(.i8, 10).i8); + try std.testing.expectEqual(10, Value.init(.i8, @as(u16, 10)).i8); + try std.testing.expectEqual(-10, Value.init(.i8, -10).i8); + try std.testing.expectEqual(-10, Value.init(.i8, @as(i16, -10)).i8); + + try std.testing.expectEqual(C64.init(1, 2), Value.init(.c64, C64.init(1, 2)).c64); + try std.testing.expectEqual(C64.init(1, 2), Value.init(.c64, C128.init(1, 2)).c64); + try std.testing.expectEqual(C128.init(1, 2), Value.init(.c128, C128.init(1, 2)).c128); + try std.testing.expectEqual(C128.init(1, 2), Value.init(.c128, C64.init(1, 2)).c128); + } + + pub fn dtype(self: Value) DataType { + return std.meta.activeTag(self); + } + + pub fn constSlice(data: *const Value) []const u8 { + return switch (data.*) { + inline else => |*value| std.mem.asBytes(value), + }; + } + + pub fn as(self: Value, comptime T: type) T { + // TODO allow more lossless conversions + switch (@typeInfo(T)) { + .bool => return self.bool, + .float => switch (self) { + inline .f16, .f32, .f64 => |v| return @floatCast(v), + inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16 => |v| return @floatCast(v.toF32()), + else => {}, + }, + .int => switch (self) { + inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |v| return @intCast(v), + else => {}, + }, + else => {}, + } + std.debug.panic("Unsupported conversion {} -> {s}", .{ self.dtype(), @typeName(T) }); + } + }; }; pub fn mantissaSize(dtype: DataType) usize { @@ -373,3 +373,5 @@ pub fn mantissaSize(dtype: DataType) usize { else => @panic("Can't get mantissa size for a non-float dtype"), }; } + +pub const Data = DataType.Value; diff --git a/zml2/meta.zig b/zml2/meta.zig new file mode 100644 index 00000000..891a27ec --- /dev/null +++ b/zml2/meta.zig @@ -0,0 +1,776 @@ +const std = @import("std"); +const testing = std.testing; + +const stdx = @import("stdx"); +const FnParam = stdx.meta.FnParam; +const asSlice = stdx.meta.asSlice; + +test { + std.testing.refAllDecls(@This()); +} + +/// Visit a given type `T` and replace all fields containing `From` by fields containing `To`. +pub fn MapType(From: type, To: type) type { + return struct { + pub fn map(T: type) type { + switch (T) { + To => return To, + ?To => return ?To, + From => return To, + *From => return *To, + ?From => return ?To, + else => {}, + } + + return switch (@typeInfo(T)) { + .@"struct" => |struct_infos| { + const fields = struct_infos.fields; + var same: bool = true; + var struct_fields: [fields.len]std.builtin.Type.StructField = undefined; + for (struct_fields[0..], fields) |*struct_field, field| { + if (!field.is_comptime) { + const R = map(field.type); + if (R == field.type) { + struct_field.* = field; + } else { + struct_field.* = .{ + .name = field.name, + .type = R, + .default_value_ptr = null, + .is_comptime = field.is_comptime, + .alignment = @alignOf(R), + }; + same = false; + // Handle the case `field: ?Tensor = null` + // Generic handling of default value is complicated, + // it would require to call the callback at comptime. + if (R == ?To) { + struct_field.default_value_ptr = &@as(R, null); + } + } + } else { + struct_field.* = field; + } + } + if (same) return T; + return @Type(.{ .@"struct" = .{ + .layout = .auto, + .fields = struct_fields[0..], + .decls = &.{}, + .is_tuple = struct_infos.is_tuple, + } }); + }, + .@"union" => |union_info| { + const fields = union_info.fields; + var same: bool = true; + var union_fields: [fields.len]std.builtin.Type.UnionField = undefined; + for (union_fields[0..], fields) |*union_field, field| { + const R = map(field.type); + if (R == field.type) { + union_field.* = field; + } else { + union_field.* = .{ + .name = field.name, + .type = R, + .alignment = @alignOf(R), + }; + same = false; + } + } + if (same) return T; + return @Type(.{ .@"union" = .{ + .layout = .auto, + .tag_type = union_info.tag_type, + .fields = union_fields[0..], + .decls = &.{}, + } }); + }, + .array => |arr_info| [arr_info.len]map(arr_info.child), + .pointer => |ptr_info| switch (ptr_info.size) { + .slice => if (ptr_info.is_const) + []const map(ptr_info.child) + else + []map(ptr_info.child), + .one => if (ptr_info.is_const) + *const map(ptr_info.child) + else + *map(ptr_info.child), + else => T, + }, + .optional => |opt_info| ?map(opt_info.child), + else => T, + }; + } + }; +} + +test MapType { + const A = struct { a: u32 }; + const B = struct { b: u32 }; + + const A2B = MapType(A, B); + + const StructA = struct { some: []const A, one: A, maybe: ?A, other: u32 }; + const struct_b = A2B.map(StructA){ + .some = &[2]B{ .{ .b = 0 }, .{ .b = 1 } }, + .maybe = null, + .one = .{ .b = 2 }, + .other = 43, + }; + _ = struct_b; + + const UnionA = union { some: []const A, one: A, maybe: ?A, other: u32 }; + const union_b = [_]A2B.map(UnionA){ + .{ .some = &[2]B{ .{ .b = 0 }, .{ .b = 1 } } }, + .{ .one = .{ .b = 2 } }, + .{ .maybe = null }, + .{ .other = 43 }, + }; + _ = union_b; +} + +/// Given a callback: `fn(Ctx, From) To`, recursively visits the given `from` struct +/// and calls the callback when it finds a `From` element, and writes it to the `to` struct. +/// The `to` parameter must be passed with mutable pointer, and tensor data need to be mutable if callback needs it. +/// `mapAlloc` tries as much as possible to respect the conversions made by Zig itself. +/// For example it can convert from a comptime array to a runtime slice. +/// `mapAlloc` can allocate new slices to write the result if the result struct requires it. +/// The caller is owning said allocations, using an `ArenaAllocator` might help tracking them. +/// +/// Note: to avoid infinite loop, mapAlloc doesn't look for `From` fields inside `To` struct. +/// Any `To` struct inside `from` will be copied over to the target. +pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam(cb, 0), from: anytype, to: anytype) !void { + // TODO: handle tuple to slice conversion + // TODO: handle error bubbling up + const From = FnParam(cb, 1); + const To = stdx.meta.FnResult(cb); + const FromStruct = @TypeOf(from); + const type_info_to_ptr = @typeInfo(@TypeOf(to)); + if (type_info_to_ptr != .pointer) { + stdx.debug.compileError("convertType is expecting a mutable `to` argument but received: {}", .{@TypeOf(to)}); + } + const ToStruct = type_info_to_ptr.pointer.child; + const type_info_to = @typeInfo(ToStruct); + + if (FromStruct == From) { + // We have an issues with `Tensor` -> `Shape` -> `Tensor` conversion when compiling ZML functions where one argument is a Shape itself. + // Normally we should call `cb` on all `Shape`. + // But the "ShapeOf" struct will have more Shape than need on the output. + // So here we take a hint from the receiving object. + // If the target is indeed a Tensor, use the callback, but if the target is `Shape` just copy it over. + if (ToStruct != To and FromStruct == ToStruct) { + to.* = from; + } else { + to.* = @call(.auto, cb, .{ ctx, from }); + } + return; + } + + if (FromStruct == To) { + to.* = from; + return; + } + + if (@sizeOf(ToStruct) == 0) return; + + switch (type_info_to) { + .@"struct" => |info| inline for (info.fields) |field| { + if (field.is_comptime or @sizeOf(field.type) == 0) continue; + const field_type_info = @typeInfo(field.type); + // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. + switch (field_type_info) { + // .pointer => try convertType(From, To, allocator, @field(from, field.name), @field(to, field.name), Ctx, ctx, cb), + .array, .optional, .@"union", .@"struct", .pointer => if (@hasField(FromStruct, field.name)) { + try mapAlloc( + cb, + allocator, + ctx, + @field(from, field.name), + &@field(to, field.name), + ); + } else if (field.default_value_ptr) |_| { + @field(to, field.name) = null; + } else { + stdx.debug.compileError("Mapping {} -> {} inside {} failed. Missing field {s} in {}", .{ From, To, FromStruct, field.name, ToStruct }); + }, + else => @field(to, field.name) = @field(from, field.name), + } + }, + .@"union" => { + switch (from) { + inline else => |_, tag| { + to.* = @unionInit(ToStruct, @tagName(tag), undefined); + try mapAlloc(cb, allocator, ctx, @field(from, @tagName(tag)), &@field(to, @tagName(tag))); + }, + } + }, + .array => for (from, to) |f, *t| { + try mapAlloc(cb, allocator, ctx, f, t); + }, + .pointer => |ptr_info| switch (ptr_info.size) { + .one => switch (type_info_to_ptr.pointer.size) { + // pointer to array -> slice promotion + .slice => { + const items = try allocator.alloc(type_info_to_ptr.pointer.child, from.len); + for (from, items) |f, *t| { + try mapAlloc(cb, allocator, ctx, f, t); + } + to.* = items; + }, + else => try mapAlloc(cb, allocator, ctx, from.*, to.*), + }, + .slice => { + const items = try allocator.alloc(@typeInfo(ToStruct).pointer.child, from.len); + for (from, items) |f, *t| { + try mapAlloc(cb, allocator, ctx, f, t); + } + to.* = items; + }, + else => stdx.debug.compileError("zml.meta.mapAlloc doesn't support: {}", .{FromStruct}), + }, + .optional => if (from) |f| { + to.* = @as(@typeInfo(type_info_to_ptr.pointer.child).optional.child, undefined); + try mapAlloc(cb, allocator, ctx, f, &(to.*.?)); + } else { + to.* = null; + }, + .int, .float, .@"enum" => to.* = from, + else => stdx.debug.compileError("zml.meta.mapAlloc doesn't support: {}", .{FromStruct}), + } +} + +test mapAlloc { + const B = struct { b: u8 }; + const A = struct { + a: u8, + pub fn convert(_: void, a: @This()) B { + return .{ .b = a.a }; + } + }; + + const Empty = struct {}; + + const AA = struct { + field: A, + array: [2]A, + slice: []const A, + other: u8, + // We want to allow conversion from comptime to runtime, because Zig type inference works like this. + comptime static_val: u8 = 8, + comptime static_slice: [2]A = .{ .{ .a = 11 }, .{ .a = 12 } }, + field_with_empty: struct { A, Empty }, + }; + const BB = struct { + field: B, + array: [2]B, + slice: []const B, + other: u8, + static_val: u8, + static_slice: []B, + field_with_empty: struct { B, Empty }, + }; + + const aa: AA = .{ + .field = .{ .a = 4 }, + .array = .{ .{ .a = 5 }, .{ .a = 6 } }, + .other = 7, + .slice = &.{ .{ .a = 9 }, .{ .a = 10 } }, + .field_with_empty = .{ .{ .a = 9 }, .{} }, + }; + var bb: BB = undefined; + + try mapAlloc(A.convert, testing.allocator, {}, aa, &bb); + defer testing.allocator.free(bb.slice); + defer testing.allocator.free(bb.static_slice); + + try testing.expectEqual(4, bb.field.b); + try testing.expectEqual(5, bb.array[0].b); + try testing.expectEqual(6, bb.array[1].b); + try testing.expectEqual(7, bb.other); + try testing.expectEqual(8, bb.static_val); + try testing.expectEqual(9, bb.slice[0].b); + try testing.expectEqual(10, bb.slice[1].b); + try testing.expectEqual(11, bb.static_slice[0].b); + try testing.expectEqual(12, bb.static_slice[1].b); +} + +/// Visit a given type `T` and: +/// * replace all fields containing `From` by fields containing `To` +/// * drop all fields not containing any `From`. +/// The returned type will contains only `To` making it easy for the compiler to produce compact layout. +/// Used by `zml.Bufferized` to strip compile time arguments from a model struct. +pub fn MapRestrict(From: type, To: type) type { + return struct { + pub fn map(T: type) type { + switch (T) { + From => return To, + ?From => return ?To, + *From => return *To, + *const From => return *const To, + []From => return []To, + []const From => return []const To, + else => {}, + } + + if (!Contains(T, From)) return void; + + return switch (@typeInfo(T)) { + .@"struct" => |struct_infos| { + // We know that at least one of the struct field contains a From. + // We map each field individually. Fields without From and comptime fields are removed. + const fields = struct_infos.fields; + var num_fields: usize = 0; + + var struct_fields: [fields.len]std.builtin.Type.StructField = undefined; + for (fields) |field| { + if (!field.is_comptime and Contains(field.type, From)) { + const R = map(field.type); + if (R == field.type) { + struct_fields[num_fields] = field; + } else { + const name = if (struct_infos.is_tuple) struct_infos.fields[num_fields].name else field.name; + struct_fields[num_fields] = .{ + .name = name, + .type = R, + .default_value_ptr = null, + .is_comptime = false, + .alignment = @alignOf(R), + }; + // Handle the case `field: ?Tensor = null` + // Generic handling of default value is not possible. + if (R == ?To) { + struct_fields[num_fields].default_value_ptr = &@as(R, null); + } + } + num_fields += 1; + } + } + if (num_fields == 0) return void; + return @Type(.{ .@"struct" = .{ + .layout = .auto, + .fields = struct_fields[0..num_fields], + .decls = &.{}, + .is_tuple = struct_infos.is_tuple, + } }); + }, + .@"union" => |union_info| { + // We know that at least one of the union field contains a From. + // We map each field individually. Fields without From, are replaced by "void". + const fields = union_info.fields; + var union_fields: [fields.len]std.builtin.Type.UnionField = undefined; + for (0.., fields) |i, field| { + const FT = map(field.type); + union_fields[i] = .{ + .name = field.name, + .type = FT, + .alignment = @alignOf(FT), + }; + } + return @Type(.{ .@"union" = .{ + .layout = .auto, + .tag_type = union_info.tag_type, + .fields = union_fields[0..], + .decls = &.{}, + } }); + }, + .array => |arr_info| [arr_info.len]map(arr_info.child), + .pointer => |ptr_info| switch (ptr_info.size) { + .slice => if (ptr_info.is_const) + []const map(ptr_info.child) + else + []map(ptr_info.child), + .one => if (ptr_info.is_const) + *const map(ptr_info.child) + else + *map(ptr_info.child), + .many => if (ptr_info.is_const) + [*]const map(ptr_info.child) + else + [*]map(ptr_info.child), + .c => if (ptr_info.is_const) + [*c]map(ptr_info.child) + else + [*c]map(ptr_info.child), + }, + .optional => |opt_info| ?map(opt_info.child), + else => T, + }; + } + }; +} + +test MapRestrict { + const A = struct { a: u32 }; + const B = struct { b: u32 }; + + const A2B = MapRestrict(A, B); + + const StructA = struct { some: []const A, one: A, maybe: ?A, other: u32 }; + const struct_b = A2B.map(StructA){ + .some = &[2]B{ .{ .b = 0 }, .{ .b = 1 } }, + .maybe = null, + .one = .{ .b = 2 }, + // Note how struct_b doesn't even have a .other field now. + }; + _ = struct_b; + + const UnionA = union { some: []const A, one: A, maybe: ?A, other: u32 }; + const union_b = [_]A2B.map(UnionA){ + .{ .some = &[2]B{ .{ .b = 0 }, .{ .b = 1 } } }, + .{ .one = .{ .b = 2 } }, + .{ .maybe = null }, + // Note how union_b.other is void now. + .{ .other = {} }, + }; + _ = union_b; +} + +/// Recursively visit the given struct and calls the callback for each K found. +/// The `v` parameter must me a pointer, and tensor data need to be mutable if callbacks needs it. +pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) void { + const Callback = @TypeOf(cb); + const Ptr = @TypeOf(v); + const type_info_v = @typeInfo(Ptr); + if (type_info_v != .pointer) { + stdx.debug.compileError("zml.meta.visit({}) is expecting a pointer/slice input, but received: {}", .{ Callback, Ptr }); + } + const ptr_info = type_info_v.pointer; + const Child = ptr_info.child; + + const K, const mutating_cb = switch (@typeInfo(FnParam(cb, 1))) { + .pointer => |info| .{ info.child, !info.is_const }, + else => stdx.debug.compileError("zml.meta.visit is expecting a callback with a pointer as second argument but found {}", .{FnParam(cb, 1)}), + }; + // Abort if v doesnt' contain any K. + if (comptime !Contains(Ptr, K)) return; + + // Handle simple cases. + switch (Ptr) { + *const K, *K => return cb(ctx, v), + *const ?K, *?K => return if (v.*) |*val| cb(ctx, val) else {}, + []const K, []K => { + for (v) |*v_elem| cb(ctx, v_elem); + return; + }, + else => {}, + } + + // Handle stdx.BoundedArray that contains uninitalized data. + if (@typeInfo(Child) == .@"struct" and @hasDecl(Child, "constSlice") and @hasDecl(Child, "slice")) { + return visit(cb, ctx, if (mutating_cb) v.slice() else v.constSlice()); + } + + // Recursively visit fields of v. + switch (ptr_info.size) { + .one => switch (@typeInfo(Child)) { + .@"struct" => |s| inline for (s.fields) |field| { + if (field.is_comptime or comptime !Contains(field.type, K)) continue; + const field_type_info = @typeInfo(field.type); + // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. + switch (field_type_info) { + .pointer => visit(cb, ctx, @field(v, field.name)), + .array, .optional, .@"union", .@"struct" => visit(cb, ctx, &@field(v, field.name)), + else => {}, + } + }, + .array => |_| for (v) |*elem| visit(cb, ctx, elem), + .optional => if (v.* != null) visit(cb, ctx, &v.*.?), + .@"union" => switch (v.*) { + inline else => |*v_field| visit(cb, ctx, v_field), + }, + else => stdx.debug.compileError("zml.meta.visit({}) doesn't support fields of type: {}", .{ Callback, Child }), + }, + .slice => { + for (v) |*v_elem| { + switch (@typeInfo(Child)) { + .@"struct" => |s| inline for (s.fields) |field| { + if (field.is_comptime or comptime !Contains(field.type, K)) continue; + const field_type_info = @typeInfo(field.type); + // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. + if (field_type_info == .pointer) { + visit(cb, ctx, @field(v_elem, field.name)); + } else { + visit(cb, ctx, &@field(v_elem, field.name)); + } + }, + .array => |_| for (v) |*elem| visit(cb, ctx, elem), + .optional => if (v.* != null) visit(cb, ctx, &v.*.?), + .@"union" => switch (v_elem.*) { + inline else => |*v_field| visit(cb, ctx, v_field), + }, + else => stdx.debug.compileError("zml.meta.visit({}) doesn't support fields of type: {}", .{ Callback, Child }), + } + } + }, + .many, .c => stdx.debug.compileError("zml.meta.visit({}) doesn't support [*] style pointers, got: {}", .{ Callback, Ptr }), + } +} + +test visit { + const Attr = struct { data: usize }; + const OtherAttr = struct { other: []const u8 }; + const NestedAttr = struct { nested: Attr }; + const NestedAttrOptional = struct { nested: ?Attr }; + const SimpleStruct = struct { prop: Attr }; + const MultipleTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: ?Attr }; + const NestedTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: NestedAttr, prop4: NestedAttrOptional, prop5: stdx.BoundedArray(Attr, 8) }; + + const LocalContext = struct { result: usize }; + + { + var context: LocalContext = .{ .result = 0 }; + const container: SimpleStruct = .{ .prop = .{ .data = 1 } }; + + visit((struct { + fn cb(ctx: *LocalContext, attr: *const Attr) void { + ctx.result += attr.data; + } + }).cb, &context, &container); + + try std.testing.expectEqual(1, context.result); + } + { + var context: LocalContext = .{ .result = 0 }; + var container: SimpleStruct = .{ .prop = .{ .data = 1 } }; + + visit((struct { + fn cb(ctx: *LocalContext, attr: *Attr) void { + ctx.result += attr.data; + } + }).cb, &context, &container); + + try std.testing.expectEqual(1, context.result); + } + { + var context: LocalContext = .{ .result = 0 }; + var container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = null }; + + visit((struct { + fn cb(ctx: *LocalContext, attr: *Attr) void { + ctx.result += attr.data; + } + }).cb, &context, &container); + + try std.testing.expectEqual(1, context.result); + } + { + var context: LocalContext = .{ .result = 0 }; + const container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = .{ .data = 2 } }; + + visit((struct { + fn cb(ctx: *LocalContext, attr: *const Attr) void { + ctx.result += attr.data; + } + }).cb, &context, &container); + + try std.testing.expectEqual(3, context.result); + } + { + var context: LocalContext = .{ .result = 0 }; + const prop5: stdx.BoundedArray(Attr, 8) = .{ + .buffer = @splat(.{ .data = 4 }), + .len = 2, + }; + const container: NestedTypesStruct = .{ + .prop1 = .{ .data = 1 }, + .prop2 = .{ .other = "hello" }, + .prop3 = .{ .nested = .{ .data = 2 } }, + .prop4 = .{ .nested = .{ .data = 3 } }, + .prop5 = prop5, // 4 will be counted twice. + }; + + visit((struct { + fn cb(ctx: *LocalContext, attr: *const Attr) void { + ctx.result += attr.data; + } + }).cb, &context, &container); + + try std.testing.expectEqual(14, context.result); + } +} + +pub fn count(T: type, value: anytype) u32 { + var counter: u32 = 0; + visit(struct { + pub fn cb(res: *u32, _: *const T) void { + res.* += 1; + } + }.cb, &counter, value); + return counter; +} + +pub fn first(T: type, value: anytype) T { + var res: ?T = null; + visit(struct { + pub fn cb(res_ptr: *?T, x: *const T) void { + if (res_ptr.* == null) res_ptr.* = x.*; + } + }.cb, &res, &value); + return res.?; +} + +/// Given a `fn([]const T, Args) T` and a slice of values, +/// will combine all values in one value. +/// Only T elements of values will be looked at. +/// This only works for simple types, in particular `zip` doesn't follow pointers. +/// Which means that zip only allocate temp memory, and nothing need to be freed after the call. +pub fn zip(comptime func: anytype, allocator: std.mem.Allocator, values: anytype, args: anytype) error{OutOfMemory}!asSlice(@TypeOf(values)) { + const sliceT = @typeInfo(FnParam(func, 0)); + const T = sliceT.pointer.child; + const V = asSlice(@TypeOf(values)); + if (V == T) { + return @call(.auto, func, .{values} ++ args); + } + // const fn_args + + return switch (@typeInfo(V)) { + .pointer => stdx.debug.compileError("zip only accept by value arguments. Received: {}", .{V}), + .@"struct" => |struct_info| { + var out: V = values[0]; + inline for (struct_info.fields) |f| { + if (f.is_comptime) continue; + if (@typeInfo(f.type) == .pointer) { + stdx.debug.compileError("zip doesn't follow pointers and don't accept struct containing them. Received: {}", .{V}); + } + var fields = try allocator.alloc(f.type, values.len); + defer allocator.free(fields); + for (values, 0..) |val, i| { + fields[i] = @field(val, f.name); + } + @field(out, f.name) = try zip(func, allocator, fields, args); + } + return out; + }, + .array => |arr_info| { + if (@typeInfo(arr_info.child) == .pointer) { + stdx.debug.compileError("zip doesn't follow pointers and don't accept struct containing them. Received: {}", .{V}); + } + var out: V = undefined; + var slice = try allocator.alloc(arr_info.child, values.len); + defer allocator.free(slice); + for (&out, 0..) |*o, j| { + for (values, 0..) |val, i| { + slice[i] = val[j]; + } + o.* = try zip(func, allocator, slice, args); + } + return out; + }, + .@"union", .optional => stdx.debug.compileError("zip doesn't yet support {}", .{V}), + else => values[0], + }; +} + +test zip { + const A = struct { a: u8, b: [2]u8 }; + const a0: A = .{ .a = 1, .b = .{ 2, 3 } }; + const a1: A = .{ .a = 4, .b = .{ 5, 6 } }; + + const Sum = struct { + pub fn call(x: []const u8) u8 { + var res: u8 = 0; + for (x) |xx| res += xx; + return res; + } + }; + const a_sum: A = try zip(Sum.call, testing.allocator, &[_]A{ a0, a1 }, .{}); + try testing.expectEqual(A{ .a = 5, .b = .{ 7, 9 } }, a_sum); +} + +/// Given a func(X) -> Y or a func(Ctx, X) -> Y, +/// finds all X in the given object, and write the result of func(X) into an arraylist. +pub fn collectAlloc( + func: anytype, + func_ctx: _CollectCtx(func), + allocator: std.mem.Allocator, + obj: anytype, +) std.mem.Allocator.Error![]stdx.meta.FnSignature(func, null).ReturnT { + stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".params.len <= 2, "zml.meta.collect expects a func with two arguments, got: {}", .{@TypeOf(func)}); + + const CollectAllocCtx = struct { + func_ctx: _CollectCtx(func), + allocator: std.mem.Allocator, + out: std.ArrayList(stdx.meta.FnSignature(func, null).ReturnT) = .empty, + oom: bool = false, + + fn cb(ctx: *@This(), val: *const _CollectArg(func)) void { + if (ctx.oom) return; + const res = if (_CollectCtx(func) == void) func(val.*) else func(ctx.func_ctx, val.*); + ctx.out.append(ctx.allocator, res) catch { + ctx.oom = true; + }; + } + }; + var context = CollectAllocCtx{ .func_ctx = func_ctx, .allocator = allocator }; + visit(CollectAllocCtx.cb, &context, obj); + if (context.oom) return error.OutOfMemory; + + return context.out.toOwnedSlice(allocator); +} + +/// Given a func(X) -> Y or a func(Ctx, X) -> Y, +/// finds all X in the given object, and write the result of func(X) into a given slice. +/// Asserts that the number of X found is equal to the slice len. +pub fn collectBuf(func: anytype, func_ctx: _CollectCtx(func), obj: anytype, out: []stdx.meta.FnResult(func)) void { + stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".params.len <= 2, "zml.meta.collectBuf expects a func with one or two arguments, got: {}", .{@TypeOf(func)}); + const LocalContext = struct { + func_ctx: _CollectCtx(func), + out: @TypeOf(out), + idx: usize = 0, + }; + var context = LocalContext{ .func_ctx = func_ctx, .out = out }; + visit((struct { + fn cb(ctx: *LocalContext, val: *const _CollectArg(func)) void { + if (ctx.idx >= ctx.out.len) return; + + const res = if (_CollectCtx(func) == void) func(val.*) else func(ctx.func_ctx, val.*); + ctx.out[ctx.idx] = res; + ctx.idx += 1; + } + }).cb, &context, obj); + std.debug.assert(context.idx == context.out.len); +} + +fn _CollectCtx(func: anytype) type { + const params = @typeInfo(@TypeOf(func)).@"fn".params; + if (params.len == 1) return void; + return params[0].type orelse @compileError("anytype not supported in collect"); +} + +fn _CollectArg(func: anytype) type { + const params = @typeInfo(@TypeOf(func)).@"fn".params; + return params[params.len - 1].type orelse @compileError("anytype not supported in collect"); +} + +pub fn Contains(Haystack: type, T: type) bool { + switch (Haystack) { + T, ?T => return true, + *T, ?*T => return true, + *const T, ?*const T => return true, + []const T, ?[]const T => return true, + anyopaque => return false, + else => {}, + } + + return switch (@typeInfo(Haystack)) { + .@"struct" => |info| { + inline for (info.fields) |field| { + if (!field.is_comptime and Contains(field.type, T)) + return true; + } + return false; + }, + .@"union" => |info| { + inline for (info.fields) |field| { + if (Contains(field.type, T)) + return true; + } + return false; + }, + .array => |info| Contains(info.child, T), + .pointer => |info| Contains(info.child, T), + .optional => |info| Contains(info.child, T), + .vector => |info| Contains(info.child, T), + else => false, + }; +} diff --git a/zml2/mlirx.zig b/zml2/mlirx.zig new file mode 100644 index 00000000..2224b88d --- /dev/null +++ b/zml2/mlirx.zig @@ -0,0 +1,80 @@ +const std = @import("std"); + +const mlir = @import("mlir"); + +const dtype = @import("dtype.zig"); +const Shape = @import("shape.zig").Shape; + +const mlirx = @This(); + +pub const Type = struct { + pub fn fromDType(ctx: *mlir.Context, dt: dtype.DataType) *const mlir.Type { + return switch (dt) { + .bool => mlir.integerType(ctx, .i1), + .f4e2m1 => mlir.floatType(ctx, .f4e2m1fn), + .f8e4m3b11fnuz => mlir.floatType(ctx, .f8e4m3b11fnuz), + .f8e4m3fn => mlir.floatType(ctx, .f8e4m3fn), + .f8e4m3fnuz => mlir.floatType(ctx, .f8e4m3fnuz), + .f8e5m2 => mlir.floatType(ctx, .f8e5m2), + .f8e5m2fnuz => mlir.floatType(ctx, .f8e5m2fnuz), + .bf16 => mlir.floatType(ctx, .bf16), + .f16 => mlir.floatType(ctx, .f16), + .f32 => mlir.floatType(ctx, .f32), + .f64 => mlir.floatType(ctx, .f64), + .i4 => mlir.integerType(ctx, .i4), + .i8 => mlir.integerType(ctx, .i8), + .i16 => mlir.integerType(ctx, .i16), + .i32 => mlir.integerType(ctx, .i32), + .i64 => mlir.integerType(ctx, .i64), + .u4 => mlir.integerType(ctx, .u4), + .u8 => mlir.integerType(ctx, .u8), + .u16 => mlir.integerType(ctx, .u16), + .u32 => mlir.integerType(ctx, .u32), + .u64 => mlir.integerType(ctx, .u64), + .c64 => mlir.complexType(ctx, .c64), + .c128 => mlir.complexType(ctx, .c128), + }; + } + + // TODO(Corentin): Maybe remove that as it now requires a mlir context. + // It's weird to have to provide a mlir context to get a zml.DataType from a mlir.Type. + pub fn toDType(mlir_type: *const mlir.Type) dtype.DataType { + const mapping = .{ + .{ .bool, mlir.IntegerType(.i1) }, + + .{ .f8e4m3b11fnuz, mlir.FloatType(.f8e4m3b11fnuz) }, + .{ .f8e4m3fn, mlir.FloatType(.f8e4m3fn) }, + .{ .f8e4m3fnuz, mlir.FloatType(.f8e4m3fnuz) }, + .{ .f8e5m2, mlir.FloatType(.f8e5m2) }, + .{ .f8e5m2fnuz, mlir.FloatType(.f8e5m2fnuz) }, + .{ .bf16, mlir.FloatType(.bf16) }, + .{ .f16, mlir.FloatType(.f16) }, + .{ .f32, mlir.FloatType(.f32) }, + .{ .f64, mlir.FloatType(.f64) }, + + .{ .i4, mlir.IntegerType(.i4) }, + .{ .i8, mlir.IntegerType(.i8) }, + .{ .i16, mlir.IntegerType(.i16) }, + .{ .i32, mlir.IntegerType(.i32) }, + .{ .i64, mlir.IntegerType(.i64) }, + + .{ .u4, mlir.IntegerType(.u4) }, + .{ .u8, mlir.IntegerType(.u8) }, + .{ .u16, mlir.IntegerType(.u16) }, + .{ .u32, mlir.IntegerType(.u32) }, + .{ .u64, mlir.IntegerType(.u64) }, + + .{ .c64, mlir.ComplexType(.c64) }, + .{ .c128, mlir.ComplexType(.c128) }, + }; + + inline for (mapping) |entry| { + const dt, const mlirT = entry; + if (mlirT.isAFn(mlir_type._inner)) { + return dt; + } + } + + std.debug.panic("Could not convert mlir.Type to DataType: {}", .{mlir_type}); + } +}; diff --git a/zml2/shape.zig b/zml2/shape.zig new file mode 100644 index 00000000..564f33b5 --- /dev/null +++ b/zml2/shape.zig @@ -0,0 +1,1097 @@ +const std = @import("std"); +const testing = std.testing; +const builtin = @import("builtin"); + +const stdx = @import("stdx"); + +const DataType = @import("dtype.zig").DataType; + +const EnumLiteral = @TypeOf(.enum_literal); + +const log = std.log.scoped(.shape); + +test { + std.testing.refAllDecls(Shape); +} + +/// Represent the shape of a tensor. +pub const Shape = struct { + pub const MAX_RANK: u8 = 8; + + pub const Tag = [*:0]const u8; + pub const TagUnknown = "_".ptr; + const TagLast = "last".ptr; + + pub const DimsArray = stdx.BoundedArray(i64, MAX_RANK); + pub const TagsArray = stdx.BoundedArray(Tag, MAX_RANK); + pub const AxesArray = stdx.BoundedArray(u3, MAX_RANK); + pub const ShardingInfo = @Vector(MAX_RANK, bool); + + const UnknownTags: TagsArray = .{ .len = 0, .buffer = [_]Tag{TagUnknown} ** MAX_RANK }; + + _dtype: DataType, + _dims: DimsArray = .{}, + _tags: TagsArray = UnknownTags, + _sharding_info: ShardingInfo = @splat(false), + + pub fn parseDimensions(v: anytype) struct { DimsArray, TagsArray } { + const T = @TypeOf(v); + + if (T == Shape) { + return .{ v._dims, v._tags }; + } + + if (comptime stdx.meta.isSliceOfAny(T, stdx.meta.isInteger)) { + var dims_ = DimsArray.init(0) catch unreachable; + var tags_ = TagsArray.init(0) catch unreachable; + for (v) |d| { + dims_.appendAssumeCapacity(@intCast(d)); + tags_.appendAssumeCapacity(TagUnknown); + } + return .{ dims_, tags_ }; + } + + if (comptime stdx.meta.isStruct(T)) { + var dims_: DimsArray = .{}; + var tags_: TagsArray = .{}; + inline for (std.meta.fields(T)) |field| { + const fv = @field(v, field.name); + if (comptime stdx.meta.isInteger(field.type)) { + dims_.appendAssumeCapacity(@intCast(fv)); + } else if (@TypeOf(fv) == EnumLiteral and comptime isAutoDim(fv)) { + dims_.appendAssumeCapacity(-1); + } else { + stdx.debug.compileError("Field {s} should be an integer or an auto dimension, got {}", .{ field.name, field.type }); + } + if (comptime stdx.meta.isTuple(T)) { + tags_.appendAssumeCapacity(TagUnknown); + } else { + tags_.appendAssumeCapacity(toTag(field)); + } + } + + return .{ dims_, tags_ }; + } + + stdx.debug.compileError("expected a dimension tuple eg '.{{ .a = 10, .b = 20}}' or '.{{ 10, 20 }}', got {}", .{T}); + } + + test parseDimensions { + const ref = Shape.init(.{ .a = 0, .b = 1 }, .f32); + const dims_, const tags_ = parseDimensions(.{ .a = 0, .b = 1 }); + + try testing.expectEqualSlices(i64, ref.dims(), dims_.constSlice()); + try testing.expectEqualSlices(Tag, ref.tags(), tags_.constSlice()); + } + + pub fn parseAxes(self: Shape, v: anytype) struct { AxesArray, TagsArray } { + const T = @TypeOf(v); + + if (T == Shape) { + return self.parseAxes(self.tags()); + } + + var axes_ = AxesArray.init(0) catch unreachable; + var tags_ = TagsArray.init(0) catch unreachable; + + if (comptime stdx.meta.isSliceOfAny(T, isAxisConvertible)) { + for (v) |d| { + axes_.appendAssumeCapacity(self.axis(d)); + tags_.appendAssumeCapacity(self.tag(d)); + } + return .{ axes_, tags_ }; + } + + if (comptime stdx.meta.isTupleOfAny(T, isAxisConvertible)) { + inline for (std.meta.fields(T)) |field| { + axes_.appendAssumeCapacity(self.axis(@field(v, field.name))); + tags_.appendAssumeCapacity(self.tag(@field(v, field.name))); + } + return .{ axes_, tags_ }; + } + + stdx.debug.compileError("Wrong type, got {}. Expected .{{.a, .b}}", .{T}); + } + + pub fn parseTags(v: anytype) TagsArray { + const T = @TypeOf(v); + stdx.debug.assertComptime(stdx.meta.isTupleOf(T, EnumLiteral), "Wrong type, got {}. Expected .{{ .a, .b }}", .{T}); + var tags_ = TagsArray.init(0) catch unreachable; + inline for (v) |field| { + tags_.appendAssumeCapacity(toTag(field)); + } + return tags_; + } + + /// Create a shape from a struct literal, eg: + /// Shape.init(.{ .h = 1024, .w = 512, .c = 3 }); + /// Shape.init(.{ 1024, 512, 3 }); + pub fn init(dimz: anytype, dt: DataType) Shape { + var res: Shape = .{ ._dtype = dt }; + res._dims, res._tags = parseDimensions(dimz); + return res; + } + + pub fn scalar(dt: DataType) Shape { + return .{ ._dtype = dt }; + } + + /// Creates a Shape with dims set to `.{0, 1, 2, ..., rank-1}`. + pub fn range(rank_: usize, dt: DataType) Shape { + var res: Shape = .{ ._dtype = dt }; + for (0..rank_) |i| { + res._dims.append(@intCast(i)) catch { + stdx.debug.panic("Too many dimensions! Max: {d}, passed: {d}", .{ res._dims.capacity(), rank_ }); + }; + res._tags.append(TagUnknown) catch unreachable; + } + return res; + } + + pub fn dtype(self: Shape) DataType { + return self._dtype; + } + + pub fn rank(self: Shape) u4 { + self.ensureDimsAndTagsAreSync(); + return @intCast(self._dims.len); + } + + pub fn dim(self: Shape, ax: anytype) i64 { + self.ensureDimsAndTagsAreSync(); + return self._dims.get(self.axis(ax)); + } + + pub fn dims(self: *const Shape) []const i64 { + self.ensureDimsAndTagsAreSync(); + return self._dims.constSlice(); + } + + fn isAxisConvertible(comptime T: type) bool { + return stdx.meta.isInteger(T) or isTagConvertible(T); + } + + fn isTagConvertible(comptime T: type) bool { + return switch (T) { + EnumLiteral => true, + std.builtin.Type.StructField => true, + Tag => true, + else => false, + }; + } + + fn toTag(v: anytype) Tag { + const T = @TypeOf(v); + return switch (T) { + EnumLiteral => @tagName(v).ptr, + std.builtin.Type.StructField => v.name.ptr, + Tag => v, + else => stdx.debug.compileError("Shape tag should be an EnumLiteral, a Shape.Tag or a StructField, got {}", .{T}), + }; + } + + inline fn ensureDimsAndTagsAreSync(self: Shape) void { + stdx.debug.assert(self._dims.len == self._tags.len, "Tags and dims have diverged! dims={d} tags={d}", .{ self._dims.len, self._tags.len }); + } + + pub fn tag(self: Shape, ax: anytype) Tag { + self.ensureDimsAndTagsAreSync(); + return self._tags.get(self.axis(ax)); + } + + /// Returns a printable name for a given axis. + /// Either the tag itself, or a digit if it's not tagged. + pub fn debugTag(self: Shape, ax: usize) []const u8 { + const t = self.tag(ax); + if (t != TagUnknown) return std.mem.span(t); + + return "01234567"[ax .. ax + 1]; + } + + pub fn setTag(self: Shape, ax: anytype, tag_: anytype) Shape { + var res = self; + res._tags.set(self.axis(ax), toTag(tag_)); + return res; + } + + pub fn tags(self: *const Shape) []const Tag { + self.ensureDimsAndTagsAreSync(); + return self._tags.constSlice(); + } + + pub fn hasTag(self: Shape, tag_: anytype) ?u3 { + return self.axisFromTagMaybe(toTag(tag_)); + } + + pub fn hasTags(self: Shape, tagz: anytype) bool { + const T = @TypeOf(tagz); + + if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, EnumLiteral)) { + for (tagz) |t| { + if (self.hasTag(t) == null) { + return false; + } + } + return true; + } + + if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, EnumLiteral)) { + inline for (tagz) |t| { + if (self.hasTag(t) == null) { + return false; + } + } + return true; + } + + stdx.debug.compileError("Expected tuple of tags, got {any}", .{T}); + } + + pub fn isFullyTagged(self: Shape) bool { + for (self._tags.constSlice()) |t| { + if (t == TagUnknown) return false; + } + return true; + } + + pub fn axis(self: Shape, axis_: anytype) u3 { + self.ensureDimsAndTagsAreSync(); + + const T = @TypeOf(axis_); + if (comptime stdx.meta.isInteger(T)) { + return self.axisFromInt(@intCast(axis_)); + } + + if (comptime isTagConvertible(T)) { + return self.axisFromTag(toTag(axis_)); + } + + stdx.debug.compileError("Wrong axis type, expected .literal, or an integer, got: {any}", .{T}); + } + + pub fn axes(self: Shape, axes_: anytype) AxesArray { + self.ensureDimsAndTagsAreSync(); + + const T = @TypeOf(axes_); + + if (T == Shape) { + return self.axes(axes_.tags()); + } + + var res = AxesArray.init(0) catch unreachable; + + if (comptime stdx.meta.isSliceOfAny(T, stdx.meta.isInteger) or stdx.meta.isSliceOf(T, Tag)) { + for (axes_) |ax| { + res.appendAssumeCapacity(self.axis(ax)); + } + return res; + } + + if (comptime stdx.meta.isStruct(T)) { + inline for (std.meta.fields(T)) |field| { + res.appendAssumeCapacity(self.axis(@field(axes_, field.name))); + } + return res; + } + + stdx.debug.compileError("axes expects an int-tuple or a tuple of enum literal, got {}", .{T}); + } + + fn axisFromInt(self: Shape, a: isize) u3 { + const rk: i8 = self.rank(); + if (a < -rk or a > rk) { + stdx.debug.panic("Tensor {f} doesn't have dimension: {d}", .{ self, a }); + } + return if (a < 0) + @intCast(a + rk) + else + @intCast(a); + } + + fn axisFromTagMaybe(self: Shape, t: Tag) ?u3 { + if (t == TagUnknown) return null; + + if (axisFromLiteralInt(t)) |ax| return ax; + + if (@inComptime()) { + // At comptime two duplicated strings may have two different representations + const t_bytes: []const u8 = std.mem.span(t); + for (self.tags(), 0..) |self_tag, ax| { + if (std.mem.eql(u8, t_bytes, std.mem.span(self_tag))) { + return @truncate(ax); + } + } + return null; + } + + // But at runtime the comptime strings have been deduplicated and ptr match is enough. + if (std.mem.indexOfScalar(Tag, self.tags(), t)) |ax| { + return @truncate(ax); + } + return null; + } + + /// Handle .{ ._0 = x } syntax. + fn axisFromLiteralInt(t: Tag) ?u3 { + // match .{ '_', '0-9', null } + if (t[0] == '_' and t[1] >= '0' and t[1] < '8' and t[2] == 0) { + return @intCast(t[1] - '0'); + } + return null; + } + + fn axisFromTag(self: Shape, d: Tag) u3 { + stdx.debug.assert(d != TagUnknown, "The unknown tag .{s} can't be used to fetch axis in {f}", .{ d, self }); + return self.axisFromTagMaybe(d) orelse { + stdx.debug.panic("Tensor {f} doesn't have dimension with tag: {s}", .{ self, d }); + }; + } + + test axis { + try testing.expectEqual(1, Shape.init(.{ 5, 2 }, .f32).axis(1)); + try testing.expectEqual(1, Shape.init(.{ 5, 2 }, .f32).axis(-1)); + try testing.expectEqual(1, Shape.init(.{ .a = 5, .b = 2 }, .f32).axis(.b)); + } + + /// The number of element inside a Tensor described by this shape. + pub fn count(self: Shape) usize { + var res: i64 = 1; + for (self.dims()) |d| { + stdx.debug.assert(d >= 0, "Can't count elements in shape with negative dimension: {f}", .{self}); + res *= d; + } + return @intCast(res); + } + + /// Total size in bytes needed to represent this shape. + pub fn byteSize(self: Shape) usize { + return self.dtype().sizeOf() * self.count(); + } + + /// Compares the two shapes described, ignoring tagging. + pub fn eql(self: Shape, other: Shape) bool { + return std.mem.eql(i64, self.dims(), other.dims()) and self.dtype() == other.dtype(); + } + + /// Compares the two shapes described, ignoring tagging and dtype. + pub fn eqlDims(self: Shape, other: Shape) bool { + return std.mem.eql(i64, self.dims(), other.dims()); + } + + /// Compares the two shapes described including tags. + pub fn eqlWithTags(self: Shape, other: Shape) bool { + return self.eql(other) and std.mem.eql(Tag, self.tags(), other.tags()) and self.dtype() == other.dtype(); + } + + /// Format the shape. + /// Default format: "Shape({.a=10, .b=20}, dtype=.f32)" + /// Bare format {_}: "{.a=10, .b=20}, dtype=.f32" + pub fn format(self: Shape, writer: *std.Io.Writer) !void { + _ = try writer.writeByte('{'); + + var need_comma = false; + for (self.dims(), 0..) |d, i| { + if (need_comma) try writer.writeByte(','); + const t = self.tag(i); + if (t != TagUnknown) { + try writer.print("{s}={d}", .{ t, d }); + } else { + try writer.print("{d}", .{d}); + } + if (self._sharding_info[i]) { + try writer.writeByte('!'); + } + need_comma = true; + } + if (need_comma) try writer.writeByte(','); + _ = try writer.write(@tagName(self.dtype())); + _ = try writer.writeByte('}'); + } + + /// Broadcasts a Tensor to the given shape, extending dimensions if needed. + pub fn canBroadcastTo(self: Shape, other: Shape) bool { + // Already the right shape + if (std.mem.eql(i64, self.dims(), other.dims())) return true; + + if (std.mem.eql(Tag, self.tags(), other.tags())) { + for (0..self.rank()) |i| { + if (self.dim(i) != 1 and self.dim(i) != other.dim(i)) return false; + } + return true; + } + + // Non ambiguous broadcasting + // TODO: broad is error prone because of this: + // it will happily broadcast .{ .a = 10, .b = 1 } to .{ .b = 10, .a = 5 } + if (self.rank() == 0 or self.rank() == other.rank()) { + for (0..self.rank()) |i| { + if (self.dim(i) != 1 and self.dim(i) != other.dim(i)) return false; + } + return true; + } + + for (self.dims(), self.tags()) |d, t| { + // TODO this is also wrong when the axes are in different order in the two shapes. + const other_ax = other.hasTag(t) orelse return false; + if (d != 1 and d != other.dim(other_ax)) return false; + } + return true; + } + + pub fn reshape(self: Shape, new_shape_: anytype) Shape { + var new_shape: Shape = .{ ._dtype = self.dtype() }; + new_shape._dims, new_shape._tags = parseDimensions(new_shape_); + new_shape.inferMissingAxis(self.count()); + stdx.debug.assert(self.count() == new_shape.count(), "Can't reshape {any} to {any}", .{ self.dims(), new_shape.dims() }); + return new_shape; + } + + fn inferMissingAxis(self: *Shape, n_: usize) void { + stdx.debug.assert(std.mem.count(i64, self.dims(), &.{-1}) < 2, "Cannot infer multiple dimensions when reshaping to: {f}", .{self.*}); + + const inferred_ax = std.mem.indexOfScalar(i64, self.dims(), -1) orelse return; + // We can't use `self.count()` yet cause we have negative dims. + var tmp_count: i64 = 1; + for (self.dims()) |d| { + if (d > 0) { + tmp_count *= d; + } + } + const n: i64 = @intCast(n_); + // Abort, `reshape` will panic with more context. + if (@mod(n, tmp_count) != 0) { + return; + } + self._dims.set(inferred_ax, @divExact(n, tmp_count)); + } + + test reshape { + const x = Shape.init(.{ 2, 5, 3 }, .f32); + { + const res = x.reshape(.{ .auto, 3 }); + try testing.expectEqualSlices(i64, &.{ 10, 3 }, res.dims()); + } + { + const res = x.reshape(.{ 10, -1 }); + try testing.expectEqualSlices(i64, &.{ 10, 3 }, res.dims()); + } + { + const res = x.reshape(.{-1}); + try testing.expectEqualSlices(i64, &.{30}, res.dims()); + } + } + + pub fn setDim(self: Shape, ax: anytype, d: i64) Shape { + var res = self; + res._dims.set(self.axis(ax), d); + return res; + } + + pub const set = setDim; + + fn isAutoDim(v: anytype) bool { + return toTag(v) == toTag(.auto); + } + + fn isDynDim(v: anytype) bool { + return toTag(v) == toTag(.dyn); + } + + /// Inserts one ore more axes with the given dimensions, before the given axis. + /// Negative axis is interpreted wrt the current shape. + /// `.last` axis can be used to insert at the end (ie to append). + /// ``` + /// .{10, 11, 12}.insert(1, 2) -> .{10, 2, 11, 12} + /// .{10, 11, 12}.insert(-1, 2) -> .{10, 11, 2, 12} + /// .{10, 11, 12}.insert(.last, 2) -> .{10, 11, 12, 2} + /// ``` + pub fn insert(self: Shape, axis_: anytype, dimz: anytype) Shape { + const dims_, const tags_ = parseDimensions(dimz); + const ax = if (@TypeOf(axis_) == EnumLiteral and axis_ == .last) + self.rank() + else + self.axis(axis_); + + var res = self; + res._dims.insertSlice(ax, dims_.constSlice()) catch unreachable; + res._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; + return res; + } + + test insert { + try testing.expectEqualSlices(i64, &.{ 10, 1, 11, 12 }, Shape.init(.{ 10, 11, 12 }, .f32).insert(1, .{1}).dims()); + try testing.expectEqualSlices(i64, &.{ 10, 11, 12, 1, 13 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).insert(-1, .{1}).dims()); + try testing.expectEqualSlices(i64, &.{ 10, 11, 12, 13, 1 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).insert(.last, .{1}).dims()); + } + + pub fn insertTag(self: Shape, axis_: anytype, d: i64, tag_: anytype) Shape { + stdx.debug.assert(self.rank() < MAX_RANK - 1, "Can't insert new axis in {f}, it's already at max rank.", .{self}); + + const ax = if (@TypeOf(axis_) == EnumLiteral and axis_ == .last) + self.rank() + else + self.axis(axis_); + + var res = self; + res._dims.insert(ax, d) catch unreachable; + res._tags.insert(ax, toTag(tag_)) catch unreachable; + return res; + } + + pub fn append(self: Shape, v: anytype) Shape { + var res = self; + const dims_, const tags_ = parseDimensions(v); + res._dims.appendSliceAssumeCapacity(dims_.constSlice()); + res._tags.appendSliceAssumeCapacity(tags_.constSlice()); + return res; + } + + test append { + try testing.expectEqualSlices( + i64, + &.{ 10, 11, 12, 1 }, + Shape.init(.{ 10, 11, 12 }, .f32).append(.{1}).dims(), + ); + + try testing.expect( + Shape.init(.{ .a = 10, .b = 11, .c = 12 }, .f32).eqlWithTags( + Shape.init(.{ .a = 10, .b = 11 }, .f32).append(.{ .c = 12 }), + ), + ); + } + + pub fn appendDim(self: Shape, d: i64, tag_: ?Tag) Shape { + var res = self; + res._dims.appendAssumeCapacity(d); + res._tags.appendAssumeCapacity(if (tag_) |t| t else TagUnknown); + return res; + } + + pub fn remove(self: Shape, axis_: anytype) Shape { + var res = self; + const a = self.axis(axis_); + _ = res._dims.orderedRemove(a); + _ = res._tags.orderedRemove(a); + return res; + } + + pub const drop = remove; + + test remove { + try std.testing.expectEqualSlices(i64, &.{ 10, 12 }, Shape.init(.{ 10, 11, 12 }, .f32).remove(1).dims()); + try std.testing.expectEqualSlices(i64, &.{ 10, 11, 12 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).remove(-1).dims()); + } + + pub fn removeMany(self: Shape, axes_: anytype) Shape { + var to_remove = self.axes(axes_); + if (to_remove.len == 0) return self; + std.mem.sort(u3, to_remove.slice(), {}, std.sort.asc(u3)); + + var sh: Shape = self; + const rk = self.rank(); + var res_ax: u32 = 0; + for (0..rk) |ax| { + if (std.mem.indexOfScalar(u3, to_remove.constSlice(), @intCast(ax))) |_| { + continue; + } + + sh._dims.buffer[res_ax] = self._dims.buffer[ax]; + sh._tags.buffer[res_ax] = self._tags.buffer[ax]; + res_ax += 1; + } + sh._dims.len = rk - to_remove.len; + sh._tags.len = rk - to_remove.len; + return sh; + } + + test removeMany { + try std.testing.expectEqualSlices( + i64, + &.{12}, + Shape.init(.{ 10, 11, 12 }, .f32).removeMany(.{ 0, 1 }).dims(), + ); + try std.testing.expectEqualSlices( + i64, + &.{ 10, 11 }, + Shape.init(.{ 10, 11, 12, 13 }, .f32).removeMany(.{ -1, -2 }).dims(), + ); + } + + pub fn transpose(self: Shape, permutations: anytype) Shape { + std.debug.assert(self.rank() == permutations.len); + const permutations_ = self.axes(permutations); + var res = self; + for (permutations_.constSlice(), 0..) |permutation, i| { + res._dims.set(i, self.dim(permutation)); + res._tags.set(i, self.tag(permutation)); + } + return res; + } + + test transpose { + try testing.expect( + Shape.init(.{ 12, 11, 10 }, .f32).eql( + Shape.init(.{ 10, 11, 12 }, .f32).transpose(.{ 2, 1, 0 }), + ), + ); + + try testing.expect( + Shape.init(.{ .a = 10, .c = 12, .b = 11, .d = 13 }, .f32).eqlWithTags( + Shape.init(.{ .a = 10, .b = 11, .c = 12, .d = 13 }, .f32).transpose(.{ 0, 2, 1, 3 }), + ), + ); + } + + /// Tag each ax of this shape with tags from a tuple. + pub fn withTags(self: Shape, tagz: anytype) Shape { + const T = @TypeOf(tagz); + + if (T == Shape) { + return self.withTags(tagz.tags()); + } + + var res = self; + + if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, EnumLiteral)) { + stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {any}", .{ self, tagz }); + for (tagz, 0..) |tag_, i| { + res._tags.set(i, toTag(tag_)); + } + return res; + } + + if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, EnumLiteral)) { + stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {}", .{ self, tagz }); + inline for (tagz, 0..) |tag_, i| { + res._tags.set(i, toTag(tag_)); + } + return res; + } + + stdx.debug.compileError("Expected a tuple of enum literals eg: .{ .a, .b, .c } got: {any}", .{@TypeOf(tagz)}); + } + + test withTags { + { + const tagged = Shape.init(.{ 0, 1 }, .f32).withTags(.{ .a, .b }); + try testing.expectEqual(0, tagged.axis(.a)); + try testing.expectEqual(1, tagged.axis(.b)); + } + { + const tagged = Shape.init(.{ 0, 1, 2 }, .f32).withTags(.{ ._, .a, .b }); + try testing.expectEqual(1, tagged.axis(.a)); + try testing.expectEqual(2, tagged.axis(.b)); + } + { + const tagged = Shape.init(.{ 0, 1, 2, 3 }, .f32).withTags(.{ ._, ._, .a, .b }); + try testing.expectEqual(2, tagged.axis(.a)); + try testing.expectEqual(3, tagged.axis(.b)); + } + } + + /// Tag the last axes of this shape with tags from a tuple. + pub fn withPartialTags(self: Shape, tagz: anytype) Shape { + const T = @TypeOf(tagz); + + if (T == Shape) { + return self.withPartialTags(tagz.tags()); + } + + var res = self; + + if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, EnumLiteral)) { + stdx.debug.assert(tagz.len <= self.rank(), "Too many tags for shape {f}, got {any}", .{ self, tagz }); + for (tagz, self.rank() - tagz.len..) |tag_, i| { + res._tags.set(i, toTag(tag_)); + } + return res; + } + + if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, EnumLiteral)) { + stdx.debug.assert(tagz.len <= self.rank(), "Too many tags for shape {f}, got {}", .{ self, tagz }); + inline for (tagz, self.rank() - tagz.len..) |tag_, i| { + res._tags.set(i, toTag(tag_)); + } + return res; + } + + stdx.debug.compileError("Expected a tuple of enum literals eg: .{ .a, .b, .c } got: {any}", .{@TypeOf(tagz)}); + } + + test withPartialTags { + { + const tagged = Shape.init(.{ 0, 1 }, .f32).withPartialTags(.{ .a, .b }); + try testing.expectEqual(0, tagged.axis(.a)); + try testing.expectEqual(1, tagged.axis(.b)); + } + { + const tagged = Shape.init(.{ 0, 1, 2 }, .f32).withPartialTags(.{ .a, .b }); + try testing.expectEqual(1, tagged.axis(.a)); + try testing.expectEqual(2, tagged.axis(.b)); + } + { + const tagged = Shape.init(.{ 0, 1, 2, 3, 4 }, .f32).withPartialTags(.{ .a, .b }); + try testing.expectEqual(3, tagged.axis(.a)); + try testing.expectEqual(4, tagged.axis(.b)); + } + { + const tagged = Shape.init(.{ 0, 1, 2, 3, 4, 5, 6 }, .f32).withPartialTags(.{ .a, .b, .c }); + try testing.expectEqual(4, tagged.axis(.a)); + try testing.expectEqual(5, tagged.axis(.b)); + try testing.expectEqual(6, tagged.axis(.c)); + } + } + + pub fn withDtype(self: Shape, dt: DataType) Shape { + var res = self; + res._dtype = dt; + return res; + } + + pub fn withSharding(self: Shape, axes_: anytype) Shape { + var res = self; + // Reset sharding. + res._sharding_info = @splat(false); + for (self.axes(axes_).constSlice()) |ax| { + res._sharding_info[ax] = true; + } + return res; + } + + /// Renames some of the tags in this shape. + /// Shape.init(.{ .a = 10, .b = 20 }).rename(.{ .b = .batch }); // .{ .a = 10, .batch = 20 }; + pub fn rename(self: Shape, renames: anytype) Shape { + const T = @TypeOf(renames); + stdx.debug.assertComptime(stdx.meta.isStructOfAny(T, isAxisConvertible), "Must pass a struct of enum literals. Passed: {any}", .{T}); + var res = self; + inline for (std.meta.fields(T)) |field| { + const new_field = @field(renames, field.name); + stdx.debug.assert(self.hasTag(new_field) == null, "{f}.rename({any}) failed because of duplicated axis {}", .{ self, renames, new_field }); + res._tags.set(self.axis(field), toTag(new_field)); + } + return res; + } + + test rename { + { + const tagged = Shape.init(.{ .a = 0, .b = 1 }, .f32).rename(.{ .a = .x, .b = .y }); + try testing.expectEqual(0, tagged.dim(.x)); + try testing.expectEqual(1, tagged.dim(.y)); + } + { + const tagged = Shape.init(.{ .a = 0, .b = 1, .c = 2 }, .f32).rename(.{ .a = .x, .c = .z }); + try testing.expectEqual(0, tagged.dim(.x)); + try testing.expectEqual(1, tagged.dim(.b)); + try testing.expectEqual(2, tagged.dim(.z)); + } + } + + pub fn computeStrides(self: Shape) stdx.BoundedArray(i64, MAX_RANK) { + const rk = self.rank(); + var strides: stdx.BoundedArray(i64, MAX_RANK) = .{ .len = rk }; + if (rk == 0) return strides; + + const V = @Vector(MAX_RANK, i64); + const rank_mask = std.simd.iota(u8, MAX_RANK) < @as(@Vector(MAX_RANK, u8), @splat(rk)); + // For each axis compute the product of all following dimensions + // and the element size in bytes. + var d: V = @bitCast(self._dims.buffer); + d = @select(i64, rank_mask, d, @as(V, @splat(1))); + d = std.simd.shiftElementsLeft(d, 1, self.dtype().sizeOf()); + d = std.simd.prefixScan(.Mul, -1, d); + + strides.buffer = @bitCast(d); + return strides; + } + + /// Returns the permutation needed to transpose this shape + /// so that the given axes are contiguous. + pub fn contiguousPerm(self: Shape, axes_: anytype) AxesArray { + const axes__, _ = self.parseAxes(axes_); + var perms = AxesArray.init(0) catch unreachable; + for (0..self.rank()) |i| { + if (std.mem.indexOfScalar(u3, axes__.constSlice(), @intCast(i))) |_| { + continue; + } + perms.appendAssumeCapacity(@intCast(i)); + } + perms.appendSliceAssumeCapacity(axes__.constSlice()); + return perms; + } + + test contiguousPerm { + const abc = Shape.init(.{ .a = 10, .b = 11, .c = 12 }, .f32); + try testing.expect( + Shape.init(.{ .b = 11, .c = 12, .a = 10 }, .f32).eqlWithTags( + abc.transpose(abc.contiguousPerm(.{.a}).constSlice()), + ), + ); + try testing.expect( + Shape.init(.{ .c = 12, .b = 11, .a = 10 }, .f32).eqlWithTags( + abc.transpose(abc.contiguousPerm(.{ .b, .a }).constSlice()), + ), + ); + const abcd = Shape.init(.{ .a = 10, .b = 11, .c = 12, .d = 13 }, .f32); + try testing.expect( + Shape.init(.{ .a = 10, .c = 12, .b = 11, .d = 13 }, .f32).eqlWithTags( + abcd.transpose(abcd.contiguousPerm(.{ .b, .d }).constSlice()), + ), + ); + + const abcde = Shape.init(.{ .a = 10, .b = 11, .c = 12, .d = 13, .e = 14 }, .f32); + try testing.expect( + Shape.init(.{ .a = 10, .b = 11, .d = 13, .c = 12, .e = 14 }, .f32).eqlWithTags( + abcde.transpose(abcde.contiguousPerm(.{ .b, .d, .c, .e }).constSlice()), + ), + ); + } + + /// Splits the given axis in several axes. + /// eg: `Shape.init(.{ .a = 10, .b = 3 }).split(.a, .{.a1 = 5, .a2 = 2}); -> .{.a1 = 5, .a2 = 2, .b = 3}` + /// The number of elements in the split shape must match the number of element + /// in the target axis. + pub fn splitAxis(self: Shape, axis_: anytype, split_shape_: anytype) Shape { + const ax = self.axis(axis_); + const dims_, const tags_ = parseDimensions(split_shape_); + var new_shape = self; + new_shape._dims.replaceRange(ax, 1, dims_.constSlice()) catch unreachable; + new_shape._tags.replaceRange(ax, 1, tags_.constSlice()) catch unreachable; + new_shape.inferMissingAxis(self.count()); + return new_shape; + } + + test splitAxis { + try testing.expect( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).eql( + Shape.init(.{ .a = 10, .b = 3 }, .f32).splitAxis(.a, .{ .a1 = 5, .a2 = 2 }), + ), + ); + try testing.expect( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).eql( + Shape.init(.{ .a = 10, .b = 3 }, .f32).splitAxis(.a, .{ .a1 = .auto, .a2 = 2 }), + ), + ); + } + + pub fn splitAxes(self: Shape, axes_: anytype) Shape { + const T = @TypeOf(axes_); + stdx.debug.assertComptime(stdx.meta.isStruct(T), "Must pass struct of enum literals like .{ .a = .{ .a1, .a2 } }. Passed: {any}", .{T}); + + var res = self; + inline for (std.meta.fields(T)) |field| { + res = res.splitAxis(field, @field(axes_, field.name)); + } + return res; + } + + test splitAxes { + try testing.expect( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).eql( + Shape.init(.{ .a = 10, .b = 3 }, .f32).splitAxes(.{ .a = .{ .a1 = 5, .a2 = .auto } }), + ), + ); + try testing.expect( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).eql( + Shape.init(.{ .a = 10, .b = 3 }, .f32).splitAxes(.{ .a = .{ .a1 = 5, .a2 = .auto } }), + ), + ); + } + + /// Merge the given axes into one axis. + /// eg: `Shape.init(.{.a1 = 5, .a2 = 2, .b = 3}).merge(.{ .a = .{ .a1, .a2 }); -> .{ .a = 10, .b = 3 }` + pub fn mergeAxis(self: Shape, axis_: anytype, axes_: anytype) Shape { + const axes__ = self.axes(axes_); + + const first_axis = axes__.get(0); + const last_axis = axes__.get(axes__.len - 1); + + var new_dim: i64 = 1; + for (axes__.constSlice(), first_axis..) |ax, counter| { + new_dim *= self.dim(ax); + stdx.debug.assert(ax == counter, "Can't merge shape {f} along non-contiguous axes {any}", .{ self, axes_ }); + } + + var new_shape = self; + new_shape._dims.set(first_axis, new_dim); + new_shape._dims.replaceRange(first_axis + 1, self.dims()[first_axis + 1 ..].len, self.dims()[last_axis + 1 ..]) catch unreachable; + new_shape._tags.set(first_axis, if (comptime isTagConvertible(@TypeOf(axis_))) toTag(axis_) else TagUnknown); + new_shape._tags.replaceRange(first_axis + 1, self.dims()[first_axis + 1 ..].len, self.tags()[last_axis + 1 ..]) catch unreachable; + return new_shape; + } + + test mergeAxis { + try testing.expect( + Shape.init(.{ .a = 10, .b = 3, .c = 4 }, .f32).eqlWithTags( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3, .c = 4 }, .f32).mergeAxis(.a, .{ .a1, .a2 }), + ), + ); + try testing.expect( + Shape.init(.{ .a = 5, .c = 6 }, .f32).eqlWithTags( + Shape.init(.{ .a = 5, .b1 = 2, .b2 = 3 }, .f32).mergeAxis(.c, .{ .b1, .b2 }), + ), + ); + try testing.expect( + Shape.init(.{ .a = 10, .b = 3 }, .f32).eqlWithTags( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).mergeAxis(.a, .{ toTag(.a1), toTag(.a2) }), + ), + ); + try testing.expect( + Shape.init(.{ .a = 10, .b = 3 }, .f32).eqlWithTags( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).mergeAxis(toTag(.a), @as([]const Tag, &.{ toTag(.a1), toTag(.a2) })), + ), + ); + try testing.expect( + Shape.init(.{ .a = 10, .b = 3 }, .f32).eqlWithTags( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).mergeAxis(.a, @as([]const usize, &.{ 0, 1 })), + ), + ); + } + + pub fn mergeAxes(self: Shape, axes_: anytype) Shape { + const T = @TypeOf(axes_); + stdx.debug.assertComptime(stdx.meta.isStruct(T), "Must pass struct of enum literals like .{ .a = .{ .a1, .a2 } }. Passed: {any}", .{T}); + + var res = self; + inline for (std.meta.fields(T)) |field| { + stdx.debug.assertComptime(stdx.meta.isTupleOfAny(field.type, isAxisConvertible) or stdx.meta.isSliceOfAny(field.type, isAxisConvertible), "Must pass struct of axes. Passed: {any}", .{field.type}); + res = res.mergeAxis(field, @field(axes_, field.name)); + } + return res; + } + + test mergeAxes { + try testing.expect( + Shape.init(.{ .a = 10, .b = 3 }, .f32).eqlWithTags( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).mergeAxes(.{ .a = .{ .a1, .a2 } }), + ), + ); + try testing.expect( + Shape.init(.{ .a = 10, .b = 3 }, .f32).eqlWithTags( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).mergeAxes(.{ .a = .{ toTag(.a1), toTag(.a2) } }), + ), + ); + try testing.expect( + Shape.init(.{ .a = 10, .b = 3 }, .f32).eqlWithTags( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).mergeAxes(.{ .a = .{ 0, 1 } }), + ), + ); + try testing.expect( + Shape.init(.{ .a = 10, .b = 3 }, .f32).eqlWithTags( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).mergeAxes(.{ .a = @as([]const usize, &.{ 0, 1 }) }), + ), + ); + } + + fn intersectTags(a: []const Tag, b: []const Tag) TagsArray { + var res = TagsArray.init(0) catch unreachable; + for (a) |tag_| { + if (std.mem.indexOfScalar(Tag, b, tag_)) { + res.appendAssumeCapacity(tag_); + } + } + return res; + } + + pub fn parseStruct(T: type, v: anytype) struct { stdx.BoundedArray(T, MAX_RANK), TagsArray } { + const V = @TypeOf(v); + + var vals_: stdx.BoundedArray(T, MAX_RANK) = .{}; + var tags_: TagsArray = .{}; + + if (comptime stdx.meta.isSliceOf(V, T)) { + for (v) |d| { + vals_.appendAssumeCapacity(d); + } + return .{ vals_, tags_ }; + } + + if (comptime stdx.meta.isStruct(V)) { + const fields = std.meta.fields(V); + stdx.debug.assertComptime(fields.len <= MAX_RANK, "Too many fields in struct {} ({d}). Max supported is {d}.", .{ V, fields.len, MAX_RANK }); + inline for (fields) |field| { + const fv = @field(v, field.name); + vals_.appendAssumeCapacity(fv); + + if (!comptime stdx.meta.isTuple(V)) { + tags_.appendAssumeCapacity(toTag(field)); + } + } + return .{ vals_, tags_ }; + } + + stdx.debug.compileError("parseStruct expects struct or tuple, got {}", .{V}); + } + + test parseStruct { + const vals_, const tags_ = parseStruct(f32, .{ .a = 0.1, .b = 1.2 }); + + try testing.expectEqualSlices(f32, &.{ 0.1, 1.2 }, vals_.constSlice()); + try testing.expectEqualSlices(Tag, &.{ "a".ptr, "b".ptr }, tags_.constSlice()); + } + + /// Parses a struct literal into a list of options for each axes. + pub fn parseAxesOptions(self: Shape, T: type, options: anytype, default: T) stdx.BoundedArray(T, MAX_RANK) { + const V = @TypeOf(options); + + var res: stdx.BoundedArray(T, MAX_RANK) = .{}; + if (comptime stdx.meta.isSliceOf(V, T)) { + stdx.debug.assert(options.len == self.rank(), "expects exactly {} options in slice, for {} got {}", .{ self.rank(), self, options.len }); + for (options) |d| { + res.appendAssumeCapacity(d); + } + } + + if (comptime stdx.meta.isStruct(V)) { + for (0..self.rank()) |_| res.appendAssumeCapacity(default); + const fields = std.meta.fields(V); + stdx.debug.assertComptime(fields.len <= MAX_RANK, "expects up to {} options struct literal, got {}", .{ V, MAX_RANK, fields.len }); + inline for (fields) |field| { + const a = self.axis(field); + res.buffer[a] = @field(options, field.name); + } + return res; + } + + stdx.debug.compileError("parseStruct expects struct or tuple, got {}", .{V}); + } + + test parseAxesOptions { + const shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .u8); + const scaling = shape.parseAxesOptions(f32, .{ .b = 1.2, .a = 0.1 }, 1.0); + + try testing.expectEqualSlices(f32, &.{ 0.1, 1.2, 1.0 }, scaling.constSlice()); + } + + test "comptimeShape" { + comptime { + const s = Shape.init(.{ .a = 5, .b = 6, .c = 7 }, .f32); + try std.testing.expectEqual(3, s.rank()); + try std.testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); + try std.testing.expectEqual(1, s.axis(.b)); + } + + // comptime only the shape + { + const s = comptime Shape.init(.{ .a = 5, .b = 6, .c = 7 }, .f32); + try std.testing.expectEqual(3, s.rank()); + try std.testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); + try std.testing.expectEqual(1, s.axis(.b)); + } + } + + pub fn outer(self: Shape, other: Shape) Shape { + var res_shape = self; + var batching_axes: u8 = 0; + for (0..other.rank()) |ax| { + if (other.tag(ax) != Shape.TagUnknown) { + if (self.hasTag(other.tag(ax))) |batching_ax| { + stdx.debug.assert(batching_ax == batching_axes and batching_ax == ax, "outer expects batching dims to be the first dims in both tensors, got outer({f}, {f})", .{ self, other }); + batching_axes += 1; + } + } + + res_shape = res_shape.appendDim(other.dim(ax), other.tag(ax)); + } + return res_shape; + } +}; diff --git a/zml2/tensor.zig b/zml2/tensor.zig new file mode 100644 index 00000000..c74955c0 --- /dev/null +++ b/zml2/tensor.zig @@ -0,0 +1,4510 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const stdx = @import("stdx"); +const meta = @import("meta.zig"); + +const Shape = @import("shape.zig").Shape; +const mlir = @import("mlir"); +const mlirx = @import("mlirx.zig"); +const dialects = @import("mlir/dialects"); + +const DataType = @import("dtype.zig").DataType; +const CompilationContext = struct {}; + +pub const Tensor = struct { + var current_id: std.atomic.Value(usize) = .{ .raw = 1 }; + pub const MAX_RANK = Shape.MAX_RANK; + + id: usize, + auto_broadcast: bool = false, + _shape: Shape, + _value: ?*const mlir.Value = null, + + pub fn format(self: Tensor, writer: *std.Io.Writer) !void { + try writer.print("Tensor({f})", .{self._shape}); + } + + /// Returns the shape of a Tensor. + pub fn shape(self: Tensor) Shape { + return self._shape; + } + + /// Returns the datatype of a Tensor. + pub fn dtype(self: Tensor) DataType { + return self._shape.dtype(); + } + + /// Returns the rank of a Tensor. + pub fn rank(self: Tensor) u4 { + return self._shape.rank(); + } + + /// Returns the number of element of a Tensor. + pub fn count(self: Tensor) usize { + return self._shape.count(); + } + + /// Returns the size in bytes of a Tensor. + pub fn byteSize(self: Tensor) usize { + return self._shape.byteSize(); + } + + /// Internal use + /// + /// Creates a tensor from a Shape and an mlir.Value. + pub fn _result(sh: Shape, val: *const mlir.Value) Tensor { + const res: Tensor = .{ + ._shape = sh, + ._value = val, + .id = Tensor.current_id.fetchAdd(1, .seq_cst), + }; + + if (builtin.mode == .Debug) { + // Check that the MLIR value actually have the same shape. + const other = fromMlirValue(val); + stdx.debug.internalAssert(sh.eql(other._shape), "Created a {f} from Mlir value but expected {f}", .{ other._shape, res._shape }); + } + + return res; + } + + /// Creates a Tensor from a mlir.Value + /// + /// The shape is derived from the type of the mlir.Value. + pub fn fromMlirValue(val: mlir.Value) Tensor { + const ranked_tensor = val.getType().as(mlir.RankedTensorType).?; + const n = ranked_tensor.getRank(); + + stdx.debug.assert(n <= MAX_RANK, "Can't represent MLIR tensor of rank {}, max supported rank is {}.", .{ n, MAX_RANK }); + + var sh: Shape = .{ ._dtype = mlirx.Type.toDType(ranked_tensor.getElementType()) }; + for (0..n) |i| { + sh._dims.appendAssumeCapacity(ranked_tensor.getDimension(i)); + } + sh._tags.resize(n) catch unreachable; + + return .{ ._shape = sh, ._value = val, .id = Tensor.current_id.fetchAdd(1, .seq_cst) }; + } + + /// Returns the dimension of axis 'axis_'. + /// + /// 'axis_' can be an integer or a tag. + pub fn dim(self: Tensor, axis_: anytype) i64 { + return self._shape.dim(axis_); + } + + /// Returns the dimensions of a Tensor as a slice. + pub fn dims(self: *const Tensor) []const i64 { + return self._shape.dims(); + } + + /// Returns the index of axis 'axis_'. + /// + /// 'axis_' can be an integer or a tag. + pub fn axis(self: Tensor, axis_: anytype) u3 { + return self._shape.axis(axis_); + } + + /// Returns the indices of each of the given axes. + /// + /// 'axis_' can be an integer or a tag. + pub fn axes(self: Tensor, axes_: anytype) stdx.BoundedArray(u3, Tensor.MAX_RANK) { + return self._shape.axes(axes_); + } + + /// Returns a Tensor tagged with the tags in 'tagz'. + pub fn withTags(self: Tensor, tagz: anytype) Tensor { + var res = self; + res._shape = self._shape.withTags(tagz); + return res; + } + + /// Returns a Tensor tagged partially with the tags in 'tagz'. + /// + /// If 'tagz' is of length n, the n last dimensions of the Tensor will be tagged. + pub fn withPartialTags(self: Tensor, tagz: anytype) Tensor { + var res = self; + res._shape = self._shape.withPartialTags(tagz); + return res; + } + + // TODO(Corentin) + pub fn withSharding(self: Tensor, axes_: anytype) Tensor { + _ = self; // autofix + _ = axes_; // autofix + @panic("TODO"); + } + + // TODO(Corentin) + //pub fn toMemory(self: Tensor, kind: Memory) Tensor { + // @panic("TODO"); + //} + + /// Returns a Tensor with new tag names. + pub fn rename(self: Tensor, renames: anytype) Tensor { + var res = self; + res._shape = self._shape.rename(renames); + return res; + } + + pub fn renameAxis(self: Tensor, ax: i8, name: @TypeOf(.enum_literal)) Tensor { + var res = self; + res._shape._tags.set(self.axis(ax), @tagName(name).ptr); + return res; + } + + /// Returns the mlir.Value associated with the Tensor. + /// + /// This will fail if used outside of a compilation context. + pub fn value(self: Tensor) *const mlir.Value { + if (self.getContext().currentScope().id_to_argument.get(self.id)) |argument_index| { + return self.getContext().currentScope().block.argument(argument_index); + } else if (self._value) |v| { + return v; + } else @panic("Something went really wrong, tensor is not an argument nor has an mlir.Value"); + } + + /// Tell PJRT compiler that memory should be reuse between the two tensors. + /// The compiler is already aggressively reusing tensors for intermediate results, + /// but this API allows to reuse buffer between input and output arguments + /// of a given function. + /// Note this is visible from the outside. The caller of a function with donations + /// is not allowed to reuse the donated input buffer after the call. + /// For `reuseBuffer` to be effective, it needs to propagate all the way through the output. + pub fn reuseBuffer(self: Tensor, origin: Tensor) Tensor { + const compilation_context = CompilationContext.current(); + const scope = compilation_context.currentScope(); + if (scope.id_to_argument.get(origin.id)) |argument_index| { + const gop = scope.id_to_donation.getOrPut(scope.arena.allocator(), self.id) catch unreachable; + gop.value_ptr.* = argument_index; + } else if (scope.id_to_donation.get(origin.id)) |origin_donation| { + const gop = scope.id_to_donation.getOrPut(scope.arena.allocator(), self.id) catch unreachable; + gop.value_ptr.* = origin_donation; + } else @panic("Something went really wrong, tensor is not an argument nor has an mlir.Value"); + return self; + } + + /// Returns a Tensor containing the absolute value of each element of the input Tensor. + pub fn abs(self: Tensor) Tensor { + const op = dialects.stablehlo.abs(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const dt = switch (self.dtype()) { + .c64 => .f32, + .c128 => .f64, + else => self.dtype(), + }; + + return _result(self._shape.withDtype(dt), op.result(0)); + } + + /// Returns a Tensor whose elements have been bitcast to a target datatype. + /// + /// The Tensor shape needs to be compatible with the target datatype. + pub fn bitCast(self: Tensor, dt: DataType) Tensor { + const src_bit_size = self.dtype().bitSizeOf(); + const tgt_bit_size = dt.bitSizeOf(); + var res_shape = if (src_bit_size == tgt_bit_size) + self._shape + else if (src_bit_size > tgt_bit_size) gt: { + const new_dim = std.math.divExact(u16, src_bit_size, tgt_bit_size) catch std.debug.panic("bitcast expects target datatype to be a multiple of source datatype when upcasting, got {} (bitsize of {}) and {} (bitsize of {})", .{ self.dtype(), src_bit_size, dt, tgt_bit_size }); + var res = self._shape; + res = res.append(.{ .bitcast = new_dim }); + break :gt res; + } else lt: { + // several contiguous elements of self maps to one element of the result + stdx.debug.assert(self.dim(-1) * src_bit_size == tgt_bit_size, "bitcast expects elements of the input tensor last dimension to map to one element of the target datatype, got {0} elements (bitsize of {0}x{1}={2}) and {3} (bitsize of {4})", .{ self.dim(-1), src_bit_size, self.dim(-1) * src_bit_size, dt, tgt_bit_size }); + break :lt self._shape.remove(-1); + }; + + res_shape = res_shape.withDtype(dt); + + const op = dialects.stablehlo.bitcast_convert( + self.getContext().mlirCtx(), + self.value(), + mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + return _result(res_shape, op.result(0)); + } + + /// Returns the given tensor as one contiguous buffer of bytes. + pub fn bytes(self: Tensor) Tensor { + return self.bitCast(.u8).flatten().withTags(.{.bytes}); + } + + /// Returns a Tensor containing the element-wise number of leading 0 bits in the input Tensor. + pub fn countLeadingZeros(self: Tensor) Tensor { + const op = dialects.stablehlo.count_leading_zeros(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + /// Returns a Tensor containing booleans indicating if each element of the input Tensor is finite. + pub fn isFinite(self: Tensor) Tensor { + const op = dialects.stablehlo.is_finite(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape.withDtype(.bool), op.result(0)); + } + + /// Returns a Tensor containing the element-wise number of bits set in the input Tensor. + pub fn popcnt(self: Tensor) Tensor { + stdx.debug.assert(self.dtype().isInteger(), "popcnt expects tensor type to be an integer, got {}", .{self.dtype()}); + const op = dialects.stablehlo.popcnt(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + /// Returns a Tensor containing the sign of the input Tensor element-wise. + pub fn sign(self: Tensor) Tensor { + const op = dialects.stablehlo.sign(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + /// Returns a Tensor containing the element-wise remainder of dividend 'self' and divisor 'other'. + /// + /// See https://pytorch.org/docs/stable/generated/torch.fmod.html for more details. + pub fn fmod(self: Tensor, divisor: f32) Tensor { + return self.remainder(scalar(divisor, .f32).broadcast(self._shape, &.{})); + } + + // TODO(Corentin) + //test fmod { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const inputs: [2][6]f32 = .{ .{ -3.0, -2, -1, 1, 2, 3 }, .{ 1, 2, 3, 4, 5, -5 } }; + // const expectations: [2][6]f32 = .{ .{ -1.0, -0.0, -1.0, 1.0, 0.0, 1.0 }, .{ 1.0000, 0.5000, 0.0000, 1.0000, 0.5000, -0.5000 } }; + // const divisors: [2]f32 = .{ 2, -1.5 }; + + // inline for (inputs, expectations, divisors) |i, e, d| { + // const input = try zml.Buffer.fromSlice(platform, .{6}, &i); + // const output = try zml.testing.compileAndCall(platform, Tensor.fmod, .{ input, d }); + + // try zml.testing.expectClose(zml.HostBuffer.fromSlice(.{6}, &e), output, 1e-4); + // } + //} + + /// Returns a Tensor containing the element-wise left-shift operation of 'self' by 'other'. + pub fn shiftLeft(self: Tensor, other: Tensor) Tensor { + return binaryOp("shiftLeft", dialects.stablehlo.shift_left)(self, other); + } + + /// Returns a Tensor containing the element-wise arithmetic right-shift operation of 'self' by 'other'. + pub fn shiftRightArithmetic(self: Tensor, other: Tensor) Tensor { + return binaryOp("shiftRightArithmetic", dialects.stablehlo.shift_right_arithmetic)(self, other); + } + + /// Returns a Tensor containing the element-wise logical right-shift operation of 'self' by 'other'. + pub fn shiftRightLogical(self: Tensor, other: Tensor) Tensor { + return binaryOp("shiftRightLogical", dialects.stablehlo.shift_right_logical)(self, other); + } + + /// Returns the Cholesky decomposition of the input Tensor. + /// + /// 'lower' controls the form of the output Tensor. The output will be lower-triangular if 'lower' is true + /// and upper-triangular otherwise. + pub fn cholesky(self: Tensor, lower: bool) Tensor { + stdx.debug.assert(self.rank() <= 2, "cholesky expects tensor rank to be <= 2, got {}", .{self.rank()}); + + const op = dialects.stablehlo.cholesky(mlirCtx(), self.value(), lower, .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + /// Solves the system of linear equations formed by the input tensors. + pub fn triangularSolve(self: Tensor, other: Tensor, opts: dialects.stablehlo.TriangularSolveOpts) Tensor { + stdx.debug.assert(self.dtype() == other.dtype(), "triangularSolve expects tensors to be of the same type, got {} and {}", .{ self.dtype(), other.dtype() }); + stdx.debug.assert(self.rank() <= 2 and self.rank() == other.rank(), "triangularSolve expects tensors to have the same rank and be <= 2, got {} and {}", .{ self.rank(), other.rank() }); + + const op = dialects.stablehlo.triangular_solve(mlirCtx(), self.value(), other.value(), opts, .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + /// Returns a Tensor containing the element-wise rounding towards the nearest integer, breaking ties away from zero, of the input Tensor. + pub fn roundNearestAfz(self: Tensor) Tensor { + stdx.debug.assert(self.dtype().isFloat(), "roundNearestAfz expects tensor type to be a float, got {}", .{self.dtype()}); + + const op = dialects.stablehlo.round_nearest_afz(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + /// Returns a Tensor containing the element-wise rounding towards the nearest integer, breaking ties towards the even integer, of the input Tensor. + pub fn roundNearestEven(self: Tensor) Tensor { + stdx.debug.assert(self.dtype().isFloat(), "roundNearestEven expects tensor type to be a float, got {}", .{self.dtype()}); + + const op = dialects.stablehlo.round_nearest_even(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + /// Returns a Tensor of complex number converted from a pair of real and imaginary Tensors. + pub fn complex(re: Tensor, im: Tensor) Tensor { + stdx.debug.assert(re._shape.eql(im._shape), "complex expects tensor shapes to match, got {f} and {f}", .{ re._shape, im._shape }); + stdx.debug.assert(re.dtype() == .f32 or re.dtype() == .f64, "complex expects tensors type to be f32 or f64, got {}", .{re.dtype()}); + + const op = dialects.stablehlo.complex(mlirCtx(), re.value(), im.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const dt: DataType = if (re.dtype() == .f32) .c64 else .c128; + return _result(re._shape.withDtype(dt), op.result(0)); + } + + /// Returns a Tensor containing the element-wise real part of the input Tensor. + /// + /// Tensor type can float or complex. + pub fn real(self: Tensor) Tensor { + stdx.debug.assert(self.dtype().isComplex() or self.dtype().isFloat(), "real expects tensor type to be a float or a complex, got {}", .{self.dtype()}); + + if (self.dtype().isFloat()) { + return self; + } + + const dt: DataType = switch (self.dtype()) { + .c64 => .f32, + .c128 => .f64, + else => unreachable, + }; + const op = dialects.stablehlo.real(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape.withDtype(dt), op.result(0)); + } + + /// Returns a Tensor containing the element-wise imaginary part of the input Tensor. + /// + /// Tensor type can float or complex. + pub fn imag(self: Tensor) Tensor { + stdx.debug.assert(self.dtype().isFloat() or self.dtype().isComplex(), "imag expects tensor type to be a float or a complex, got {}", .{self.dtype()}); + + // Real tensors don't have imaginary part. + if (self.dtype().isFloat()) { + return Tensor.constant(self._shape, self.dtype().zero()); + } + + const dt: DataType = switch (self.dtype()) { + .bf16, .f16, .f32, .f64 => self.dtype(), + .c64 => .f32, + .c128 => .f64, + else => unreachable, + }; + const op = dialects.stablehlo.imag(mlirCtx(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape.withDtype(dt), op.result(0)); + } + + /// Returns the Fast Fourier Transform (FFT) of the input Tensor. + pub fn fft(self: Tensor, opts: dialects.stablehlo.FftOpts) Tensor { + // TODO: support tagged API. + + stdx.debug.assert(1 <= opts.length.len and opts.length.len <= 3, "fft expects 'opts.length' length to be between 1 and 3 (inclusive), got {}", .{opts.length.len}); + stdx.debug.assert(opts.length.len <= self.rank(), "fft expects 'opts.length' length to be less than tensor rank, got {} and {}", .{ opts.length.len, self.rank() }); + + const sh = switch (opts.kind) { + .FFT, .IFFT => blk: { + stdx.debug.assert(self.dtype().isComplex(), "fft({any}) expects tensor type to be complex, got {}", .{ opts, self.dtype() }); + + break :blk self._shape; + }, + .RFFT => blk: { + stdx.debug.assert(self.dtype() == .f32 or self.dtype() == .f64, "fft({}) expects tensor type to be f32 or f64, got {}", .{ opts, self.dtype() }); + stdx.debug.assert(std.mem.eql(i64, self.dims()[self.rank() - opts.length.len ..], opts.length), "fft({}) expects tensor last dimensions to match given lengths, got {} and {}", .{ opts, self.dims()[self.rank() - opts.length.len ..].len, opts.length.len }); + + const dt: DataType = switch (self.dtype()) { + .f32 => .c64, + else => .c128, + }; + const shape_ = self._shape.setDim(-1, @divExact(self.dim(-1), 2) + 1); + break :blk shape_.withDtype(dt); + }, + .IRFFT => blk: { + stdx.debug.assert(self.dtype().isComplex(), "fft({any}) expects tensor type to be complex, got {}", .{ opts, self.dtype() }); + stdx.debug.assert(std.mem.eql(i64, self.dims()[self.rank() - opts.length.len ..], opts.length), "fft({any}) expects tensor last dimensions to match given lengths, got {} and {}", .{ opts, self.dims()[self.rank() - opts.length.len ..].len, opts.length.len }); + + const dt: DataType = switch (self.dtype()) { + .c64 => .f32, + else => .f64, + }; + const shape_ = self._shape.setDim(-1, @divExact(self.dim(-1) - 1, 2)); + break :blk shape_.withDtype(dt); + }, + }; + + const op = dialects.stablehlo.fft(mlirCtx(), self.value(), opts, .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(sh, op.result(0)); + } + + pub const Rng = struct { + _state: Tensor, + algorithm: dialects.stablehlo.RngAlgorithm.Type = .DEFAULT, + + pub fn init() Rng { + return .{ ._state = .init(Shape.init(.{2}, .u64)) }; + } + + // TODO(Corentin) + //pub fn initBuffer(platform: Platform, seed: u128) !Bufferized(Rng) { + // return .{ + // ._state = try zml.Buffer.fromBytes(platform, Shape.init(.{2}, .u64), std.mem.asBytes(&seed)), + // }; + //} + + /// Returns a Tensor of the given shape, filled with uniform random bits, and a new Rng state. + /// + /// The given Rng state should not be used anymore (or you'll get the same numbers again). + /// The output is guaranteed to be deterministic function of `self` Rng state, + /// but it is not guaranteed to be deterministic between implementations. + pub fn bitGenerator(self: Rng, sh: Shape) struct { Rng, Tensor } { + const op = dialects.stablehlo.rng_bit_generator( + mlirCtx(), + self.algorithm, + self._state.value(), + mlir.rankedTensorType(self._state.dims(), mlirx.Type.fromDType(mlirCtx(), self._state.dtype())), + mlir.rankedTensorType(sh.dims(), mlirx.Type.fromDType(mlirCtx(), sh.dtype())), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return .{ self.update(op.result(0)), _result(sh, op.result(1)) }; + } + + fn update(self: Rng, new_state: mlir.Value) Rng { + return .{ + ._state = _result(self._state._shape, new_state).reuseBuffer(self._state), + .algorithm = self.algorithm, + }; + } + + /// Returns a Tensor of the given shape, filled with uniformly sampled floating point numbers from an interval, + /// and a new Rng state. + /// + /// https://en.wikipedia.org/wiki/Continuous_uniform_distribution + pub fn uniform( + self: Rng, + shape_: Shape, + opts: struct { min: f64 = 0, max: f64 = 1 }, + ) struct { Rng, Tensor } { + const dt = if (shape_.dtype().isFloat()) shape_.dtype() else .f32; + + const mantissa_bit_count = @import("dtype.zig").mantissaSize(dt); + const bit_count: usize = dt.bitSizeOf(); + const rng_bit_count = if (mantissa_bit_count < 8) 8 else bit_count; + const uint_dtype: DataType = switch (bit_count) { + 8 => .u8, + 16 => .u16, + 32 => .u32, + 64 => .u64, + else => stdx.debug.panic("uniform don't support non-byte aligned dtype. Got: {f}", .{shape_}), + }; + + const rng, const bits = self.bitGenerator(shape_.withDtype(uint_dtype)); + + // Erase bits outside of mantissa. + var float_bits = bits.shiftRightLogical(scalar(rng_bit_count - mantissa_bit_count, uint_dtype)); + + // Set exponent bits to represent e^0 (eg 127 for f32). + float_bits = float_bits.logical(.OR, scalar(1, dt).bitCast(uint_dtype)); + + // float_bits now uniformly represents number in [1, 2[ range. + // Let's convert to floats, and subtract one to go to [0, 1[ range. + var floats = float_bits.bitCast(dt).sub(scalar(1, dt)); + floats = floats.mul(scalar(opts.max - opts.min, dt)).addConstant(opts.min); + + // Convert back to integer if needed. + return .{ rng, floats.convert(shape_.dtype()) }; + } + + //test uniform { + // const zml = @import("zml.zig"); + // const Stats = struct { + // const Stats = @This(); + + // mean: Tensor, + // variance: Tensor, + // min: Tensor, + // max: Tensor, + + // pub fn uniformStats( + // rand: Rng, + // shape_: Shape, + // opts: struct { min: f64, max: f64 }, + // ) struct { Rng, Stats } { + // const rng, const data = rand.uniform(shape_, .{ .min = opts.min, .max = opts.max }); + // const mean_ = data.mean(0); + // const variance = data.sub(mean_.broad(data.shape())).pow(Tensor.scalar(2, .f32)).mean(0); + // return .{ rng, .{ + // .mean = mean_, + // .variance = variance, + // .min = data.min(0), + // .max = data.max(0), + // } }; + // } + // }; + + // const platform = zml.testing.env(); + // // Compute stats over a uniform distribution on [-2, 10]. + // const rand, const stats = try zml.testing.compileAndCallWithTensors( + // platform, + // Stats.uniformStats, + // .{ Rng.shape(), zml.Shape.init(.{1024}, .f32), .{ .min = -2, .max = 10 } }, + // .{try Rng.init(platform, 1234)}, + // ); + + // // Check the Rng state has been modified. + // try std.testing.expect(try rand._state.getValue(u128) != 1234); + + // // Check the mean and variance are close to theoritical values. + // const mean_ = try stats.mean.getValue(f32); + // try std.testing.expectApproxEqAbs(4, mean_, 0.03); + + // const variance = try stats.variance.getValue(f32); + // try std.testing.expectApproxEqAbs(12.0 * 12.0 / 12.0, variance, 0.01); + + // // Check that no value is outside of the interval + // // and we have samples close to the edges. + // const min_ = try stats.min.getValue(f32); + // try std.testing.expect(min_ >= -2); + // try std.testing.expectApproxEqAbs(-2, min_, 0.05); + + // const max_ = try stats.max.getValue(f32); + // try std.testing.expect(max_ < 10); + // try std.testing.expectApproxEqAbs(10, max_, 0.05); + //} + + /// Returns a Tensor of the given shape, filled with floating point numbers sampled from a normal distribution. + /// + /// Note: this uses stablehlo.rng which is deprecated. + /// https://github.com/openxla/stablehlo/blob/main/rfcs/20240503-opset-deprecations.md + pub fn normal(sh: Shape, opts: struct { mean: f64 = 0, stddev: f64 = 1 }) Tensor { + stdx.debug.assert(sh.dtype().isFloat(), "normal expects tensor type to be a float, got {}", .{sh.dtype()}); + + const a = Tensor.constant(.{}, DataType.Value.init(sh.dtype(), opts.mean)); + const b = Tensor.constant(.{}, DataType.Value.init(sh.dtype(), opts.stddev)); + const res_shape = Tensor.constantTensor(HostBuffer.fromSlice(.{sh.rank()}, sh.dims())); + const op = dialects.stablehlo.rng(mlirCtx(), a.value(), b.value(), res_shape.value(), .NORMAL, .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(sh, op.result(0)); + } + + /// Returns a Tensor of the given shape, filled with floating point numbers sampled from a Gumbel distribution, and a new Rng state. + /// + /// Often used in ML because of the reparametrization tricks. + /// Sampling from a gumbel distribution is equivalent to sample + /// from a softmax distribution, but doesn't require to compute the sum of exponentials. + /// https://en.wikipedia.org/wiki/Gumbel_distribution#Gumbel_reparametrization_tricks + /// See `sampleTokens` for a practical use case. + /// Note: we only implement the μ=0, β=1 version. + pub fn gumbel(self: Rng, shape_: Shape) struct { Rng, Tensor } { + const rand, const u = self.uniform( + // Always use .f32 to have a big enough mantissa. + shape_.withDtype(.f32), + // We don't want 0 to be sampled otherwise `log` will return -inf. + .{ .min = std.math.floatEps(f32), .max = 1 }, + ); + return .{ rand, u.log().scale(-1).log().scale(-1).convert(shape_.dtype()) }; + } + + //test gumbel { + // const zml = @import("zml.zig"); + // const Stats = struct { + // const Stats = @This(); + + // mean: Tensor, + // variance: Tensor, + // actual_dist: Tensor, + + // pub fn gumbelStats(rand: Rng, target_dist: Tensor) struct { Rng, Stats } { + // const s = Shape.init(.{ .n = 1024, .d = 4 }, .f32); + // const rng, const data = rand.gumbel(s); + // const flat = data.flattenAll(); + // const mean_ = flat.mean(0); + // const variance = flat.sub(mean_.broad(flat.shape())).pow(Tensor.scalar(2, .f32)).mean(0); + + // // Test out the gumbel reparametrization trick + // var x = target_dist.log().withTags(.{.d}).broad(s); + // x = x.add(data); + // const samples = x.argMax(.d).indices.squeeze(.d); + + // // count 0, 1, 2 and 3 in samples: + // // - map 0 to 1, 1 to 2**16, 2 to 2**32, 3 to N**58 + // // - sum in u64 + // // - split to [4]u16 + // const powers = blk: { + // var powers: [4]u64 = undefined; + // for (&powers, 0..) |*p, i| p.* = std.math.pow(u64, 2, i * 16); + // break :blk powers; + // }; + // const values = Tensor.constantTensor(HostBuffer.fromArray(&powers)).withTags(.{.d}); + // const counts = values.gatherValues(.d, samples, .{}).sum(.n).bitCast(.u16); + // const actual_dist = counts.reshape(target_dist.shape()).convert(target_dist.dtype()).divByConst(s.dim(.n)); + // return .{ rng, .{ .mean = mean_, .variance = variance, .actual_dist = actual_dist } }; + // } + // }; + + // const platform = zml.testing.env(); + // const tgt_dist = [_]f32{ 2.0, 1.0, 4.0, 3.0 }; + // const rand, const stats = try zml.testing.compileAndCallWithTensors( + // platform, + // Stats.gumbelStats, + // .{ Rng.shape(), zml.Shape.init(.{tgt_dist.len}, .f32) }, + // .{ try Rng.init(platform, 1234), try .fromArray(platform, tgt_dist) }, + // ); + // // Check the Rng state has been modified. + // try std.testing.expect(try rand._state.getValue(i128) != 1234); + + // // Check the mean and variance are close to theoritical values. + // const mean_ = try stats.mean.getValue(f32); + // try std.testing.expectApproxEqAbs(0.5772, mean_, 0.02); + + // const variance = try stats.variance.getValue(f32); + // const pi = std.math.pi; + // try std.testing.expectApproxEqAbs(pi * pi / 6.0, variance, 0.03); + + // // Check the distribution obtained with the gumbel trick matches the target distribution. + // const actual_dist = try stats.actual_dist.getValue([4]f32); + // scoped_log.debug("tgt_dist: {d}, actual_dist: {d}", .{ tgt_dist, actual_dist }); + // for (tgt_dist, actual_dist) |tgt, actual| { + // // We normalize tgt_dist to make it a well formed distribution. + // // We didn't do it before calling gumbel, because the gumbel trick + // // doesn't require normalized distributions as input. + // try std.testing.expectApproxEqAbs(tgt / 10.0, actual, 0.05); + // } + //} + }; + + /// Returns a Tensor containing the element-wise conversion to another floating point type. + pub fn reducePrecision(self: Tensor, exponent_bits: i32, mantissa_bits: i32) Tensor { + stdx.debug.assert(self.dtype().isFloat(), "reducePrecision expects tensor type to be a float, got {}", .{self.dtype()}); + stdx.debug.assert(1 <= exponent_bits, "reducePrecision expects 'exponent_bits' to be >= 1, got {}", .{exponent_bits}); + stdx.debug.assert(0 <= mantissa_bits, "reducePrecision expects 'mantissa_bits' to be positive, got {}", .{mantissa_bits}); + + const op = dialects.stablehlo.reduce_precision(mlirCtx(), self.value(), exponent_bits, mantissa_bits, .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + fn convolution(self: Tensor, other: Tensor, opts: dialects.stablehlo.ConvolutionOpts, loc: mlir.Location) Tensor { + _ = loc; // autofix + stdx.debug.assert(self.rank() == other.rank(), "convolution expects tensor ranks to match, got {} and {}", .{ self.rank(), other.rank() }); + const N = self.rank(); + stdx.debug.guard(opts.window_strides.len == N - 2, @src()); + for (opts.window_strides) |s| stdx.debug.guard(0 < s, @src()); + stdx.debug.guard(opts.lhs_dilation.len == N - 2, @src()); + for (opts.lhs_dilation) |d| stdx.debug.guard(0 < d, @src()); + stdx.debug.guard(opts.rhs_dilation.len == N - 2, @src()); + for (opts.rhs_dilation) |d| stdx.debug.guard(0 < d, @src()); + stdx.debug.guard(opts.window_reversal.len == N - 2, @src()); + stdx.debug.guard(@rem(self.dim(opts.input_batch_dimension), opts.batch_group_count) == 0, @src()); + stdx.debug.guard(@rem(self.dim(opts.input_feature_dimension), opts.feature_group_count) == 0, @src()); + stdx.debug.guard(opts.input_spatial_dimensions.len == N - 2, @src()); + stdx.debug.guard(opts.input_batch_dimension != opts.input_feature_dimension, @src()); + stdx.debug.guard(0 <= opts.input_batch_dimension and opts.input_batch_dimension < N, @src()); + stdx.debug.guard(0 <= opts.input_feature_dimension and opts.input_feature_dimension < N, @src()); + for (opts.input_spatial_dimensions, 0..) |d, i| { + stdx.debug.guard(d != opts.input_batch_dimension, @src()); + stdx.debug.guard(d != opts.input_feature_dimension, @src()); + stdx.debug.guard(0 <= d and d < N, @src()); + if (i < opts.input_spatial_dimensions.len - 1) continue; + stdx.debug.guard(std.mem.indexOfScalar(i64, opts.input_spatial_dimensions[i + 1 ..], d) == null, @src()); + } + stdx.debug.guard(other.dim(opts.kernel_input_feature_dimension) == @divTrunc(self.dim(opts.input_feature_dimension), opts.feature_group_count), @src()); + stdx.debug.guard(@rem(other.dim(opts.kernel_output_feature_dimension), opts.batch_group_count) == 0, @src()); + stdx.debug.guard(@rem(other.dim(opts.kernel_output_feature_dimension), opts.feature_group_count) == 0, @src()); + stdx.debug.guard(opts.kernel_spatial_dimensions.len == N - 2, @src()); + stdx.debug.guard(opts.kernel_input_feature_dimension != opts.kernel_output_feature_dimension, @src()); + stdx.debug.guard(0 <= opts.kernel_input_feature_dimension and opts.kernel_input_feature_dimension < N, @src()); + stdx.debug.guard(0 <= opts.kernel_output_feature_dimension and opts.kernel_output_feature_dimension < N, @src()); + for (opts.kernel_spatial_dimensions, 0..) |d, i| { + stdx.debug.guard(d != opts.kernel_input_feature_dimension, @src()); + stdx.debug.guard(d != opts.kernel_output_feature_dimension, @src()); + stdx.debug.guard(0 <= d and d < N, @src()); + if (i < opts.kernel_spatial_dimensions.len - 1) continue; + stdx.debug.guard(std.mem.indexOfScalar(i64, opts.kernel_spatial_dimensions[i + 1 ..], d) == null, @src()); + } + stdx.debug.guard(opts.output_spatial_dimensions.len == N - 2, @src()); + stdx.debug.guard(opts.output_batch_dimension != opts.output_feature_dimension, @src()); + stdx.debug.guard(0 <= opts.output_batch_dimension and opts.output_batch_dimension < N, @src()); + stdx.debug.guard(0 <= opts.output_feature_dimension and opts.output_feature_dimension < N, @src()); + for (opts.output_spatial_dimensions, 0..) |d, i| { + stdx.debug.guard(d != opts.output_batch_dimension, @src()); + stdx.debug.guard(d != opts.output_feature_dimension, @src()); + stdx.debug.guard(0 <= d and d < N, @src()); + if (i < opts.output_spatial_dimensions.len - 1) continue; + stdx.debug.guard(std.mem.indexOfScalar(i64, opts.output_spatial_dimensions[i + 1 ..], d) == null, @src()); + } + stdx.debug.guard(0 < opts.feature_group_count, @src()); + stdx.debug.guard(0 < opts.batch_group_count, @src()); + stdx.debug.guard(opts.feature_group_count == 1 or opts.batch_group_count == 1, @src()); + var used_opts = opts; + used_opts.pad_shape = &.{ @intCast(N - 2), 2 }; + used_opts.precision_config = &.{ .DEFAULT, .DEFAULT }; + + var new_shape = self._shape; + var res_dim: i64 = undefined; + + for (0..N) |i| { + if (i == @as(usize, @intCast(opts.output_batch_dimension))) { + res_dim = @divTrunc(self.dim(opts.input_batch_dimension), opts.batch_group_count); + } else if (i == @as(usize, @intCast(opts.output_feature_dimension))) { + res_dim = other.dim(opts.kernel_output_feature_dimension); + } else { + // calculate spatial dimension value + const spatial_dim: usize = std.mem.indexOfScalar(i64, opts.output_spatial_dimensions, @as(i64, @intCast(i))).?; + const lhs_dim = opts.input_spatial_dimensions[spatial_dim]; + const rhs_dim = opts.kernel_spatial_dimensions[spatial_dim]; + const dilated_input_shape_lhs_dim: i64 = if (self.dim(lhs_dim) == 0) 0 else (self.dim(lhs_dim) - 1) * opts.lhs_dilation[spatial_dim] + 1; + const left_pad_value, const right_pad_value = if (opts.pad_value.len == 1) + .{ opts.pad_value[0], opts.pad_value[0] } + else + .{ opts.pad_value[2 * spatial_dim], opts.pad_value[2 * spatial_dim + 1] }; + const padded_input_shape_lhs_dim = left_pad_value + dilated_input_shape_lhs_dim + right_pad_value; + const dilated_window_shape_lhs_dim: i64 = if (other.dim(rhs_dim) == 0) 0 else (other.dim(rhs_dim) - 1) * opts.rhs_dilation[spatial_dim] + 1; + const is_empty_window_lhs_dim = padded_input_shape_lhs_dim == 0 or dilated_window_shape_lhs_dim > padded_input_shape_lhs_dim; + res_dim = if (is_empty_window_lhs_dim) 0 else @divTrunc(padded_input_shape_lhs_dim - dilated_window_shape_lhs_dim, opts.window_strides[spatial_dim]) + 1; + } + + new_shape = new_shape.set(i, res_dim); + } + + // inferred shape '[1, 256, 1, 12008]' is incompatible with return type of operation '[1, 256, 1, 11978]' + const op = dialects.stablehlo.convolution( + mlirCtx(), + self.value(), + other.value(), + used_opts, + mlir.rankedTensorType(new_shape.dims(), mlirx.Type.fromDType(mlirCtx(), new_shape.dtype())), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + return _result(new_shape, op.result(0)); + } + + /// Returns a Tensor containing the result of the 1D convolution of 'input' by 'kernel'. + pub fn conv1d( + input: Tensor, + kernel: Tensor, + opts: struct { + window_strides: i64 = 1, + padding: []const i64 = &.{ 0, 0 }, + lhs_dilation: i64 = 1, + rhs_dilation: i64 = 1, + window_reversal: bool = false, + input_batch_dimension: i64 = 0, + input_feature_dimension: i64 = 1, + input_spatial_dimensions: i64 = 2, + kernel_output_feature_dimension: i64 = 0, + kernel_input_feature_dimension: i64 = 1, + kernel_spatial_dimensions: i64 = 2, + output_batch_dimension: i64 = 0, + output_feature_dimension: i64 = 1, + output_spatial_dimensions: i64 = 2, + feature_group_count: i64 = 1, + batch_group_count: i64 = 1, + }, + ) Tensor { + return input.convolution(kernel, .{ + .window_strides = &.{opts.window_strides}, + .pad_value = opts.padding, + .lhs_dilation = &.{opts.lhs_dilation}, + .rhs_dilation = &.{opts.rhs_dilation}, + .window_reversal = &.{opts.window_reversal}, + .input_batch_dimension = opts.input_batch_dimension, + .input_feature_dimension = opts.input_feature_dimension, + .input_spatial_dimensions = &.{opts.input_spatial_dimensions}, + .kernel_input_feature_dimension = opts.kernel_input_feature_dimension, + .kernel_output_feature_dimension = opts.kernel_output_feature_dimension, + .kernel_spatial_dimensions = &.{opts.kernel_spatial_dimensions}, + .output_batch_dimension = opts.output_batch_dimension, + .output_feature_dimension = opts.output_feature_dimension, + .output_spatial_dimensions = &.{opts.output_spatial_dimensions}, + .feature_group_count = opts.feature_group_count, + .batch_group_count = opts.batch_group_count, + }, .unknown(mlirCtx())); + } + + /// Returns a Tensor containing the result of the 2D convolution of 'input' by 'kernel'. + /// Defaults values correspond to a (B, C_in, W, H) image, (C_out, C_in, W, H) kernel weights and (B, C_out, W, H) output. + pub fn conv2d( + input: Tensor, + kernel: Tensor, + opts: struct { + window_strides: []const i64 = &.{ 1, 1 }, + padding: []const i64 = &.{ 0, 0, 0, 0 }, + lhs_dilation: []const i64 = &.{ 1, 1 }, + rhs_dilation: []const i64 = &.{ 1, 1 }, + window_reversal: []const bool = &.{ false, false }, + input_batch_dimension: i64 = 0, + input_feature_dimension: i64 = 1, + input_spatial_dimensions: []const i64 = &.{ 2, 3 }, + kernel_input_feature_dimension: i64 = 1, + kernel_output_feature_dimension: i64 = 0, + kernel_spatial_dimensions: []const i64 = &.{ 2, 3 }, + output_batch_dimension: i64 = 0, + output_feature_dimension: i64 = 1, + output_spatial_dimensions: []const i64 = &.{ 2, 3 }, + feature_group_count: i64 = 1, + batch_group_count: i64 = 1, + }, + ) Tensor { + return input.convolution(kernel, .{ + .window_strides = opts.window_strides, + .pad_value = opts.padding, + .lhs_dilation = opts.lhs_dilation, + .rhs_dilation = opts.rhs_dilation, + .window_reversal = opts.window_reversal, + .input_batch_dimension = opts.input_batch_dimension, + .input_feature_dimension = opts.input_feature_dimension, + .input_spatial_dimensions = opts.input_spatial_dimensions, + .kernel_input_feature_dimension = opts.kernel_input_feature_dimension, + .kernel_output_feature_dimension = opts.kernel_output_feature_dimension, + .kernel_spatial_dimensions = opts.kernel_spatial_dimensions, + .output_batch_dimension = opts.output_batch_dimension, + .output_feature_dimension = opts.output_feature_dimension, + .output_spatial_dimensions = opts.output_spatial_dimensions, + .feature_group_count = opts.feature_group_count, + .batch_group_count = opts.batch_group_count, + }, .unknown(mlirCtx())); + } + + /// Returns a Tensor containing the element-wise addition of the input Tensors. + pub fn add(self: Tensor, other: Tensor) Tensor { + return binaryOp("add", dialects.stablehlo.add)(self, other); + } + + /// Returns a Tensor containing the element-wise subtraction of the input Tensors. + pub fn sub(self: Tensor, other: Tensor) Tensor { + return binaryOp("subtract", dialects.stablehlo.subtract)(self, other); + } + + /// Returns a Tensor containing the element-wise multiplication of the input Tensors. + pub fn mul(self: Tensor, other: Tensor) Tensor { + return binaryOp("mul", dialects.stablehlo.multiply)(self, other); + } + + /// Returns a Tensor containing the element-wise division of the input Tensors. + pub fn div(self: Tensor, other: Tensor) Tensor { + return binaryOp("div", dialects.stablehlo.divide)(self, other); + } + + /// Returns a Tensor containing the element-wise exponentiation of the input Tensors. + pub fn pow(self: Tensor, other: Tensor) Tensor { + return binaryOp("pow", dialects.stablehlo.power)(self, other); + } + + /// Returns a Tensor containing the element-wise maximum operation of the input Tensors. + pub fn maximum(self: Tensor, other: Tensor) Tensor { + return binaryOp("maximum", dialects.stablehlo.maximum)(self, other); + } + + /// Returns a Tensor containing the element-wise minimum operation of the input Tensors. + pub fn minimum(self: Tensor, other: Tensor) Tensor { + return binaryOp("minimum", dialects.stablehlo.minimum)(self, other); + } + + /// Returns a Tensor containing the element-wise remainder of dividend 'self' and divisor 'other'. + pub fn remainder(self: Tensor, other: Tensor) Tensor { + return binaryOp("remainder", dialects.stablehlo.remainder)(self, other); + } + + /// Returns a Tensor containing the element-wise addition of the input Tensor with a constant. + pub fn addConstant(self: Tensor, b: anytype) Tensor { + return self.add(.scalar(b, self.dtype())); + } + + /// Returns a Tensor containing the element-wise division of the input Tensor by a constant. + pub fn divByConst(self: Tensor, b: anytype) Tensor { + return self.div(.scalar(b, self.dtype())); + } + + /// Returns a Tensor containing the element-wise power of the input Tensor by a constant. + pub fn powByConst(self: Tensor, b: anytype) Tensor { + return self.pow(.scalar(b, self.dtype())); + } + + /// Returns a Tensor containing the element-wise multiplication of the input Tensor by a constant. + pub fn scale(self: Tensor, val: anytype) Tensor { + return self.mul(.scalar(val, self.dtype())); + } + + pub const LogicalOp = enum { OR, XOR, AND }; + + /// Returns a Tensor containing the element-wise logical operation of the input Tensors. + pub fn logical(self: Tensor, comptime logical_op: LogicalOp, other: Tensor) Tensor { + return switch (logical_op) { + .OR => binaryOp("or", dialects.stablehlo.or_)(self, other), + .XOR => binaryOp("xor", dialects.stablehlo.xor)(self, other), + .AND => binaryOp("and", dialects.stablehlo.and_)(self, other), + }; + } + + /// Returns a Tensor containing the element-wise floor operation of the input Tensor. + pub fn floor(self: Tensor) Tensor { + return _result(self._shape, dialects.stablehlo.floor(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()).result(0)); + } + + /// Returns a Tensor containing the element-wise ceil operation of the input Tensor. + pub fn ceil(self: Tensor) Tensor { + return _result(self._shape, dialects.stablehlo.ceil(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()).result(0)); + } + + /// Returns a Tensor containing the element-wise conversion to another type. + pub fn convert(self: Tensor, to: DataType) Tensor { + if (to == self.dtype()) { + return self; + } + + const res_type = mlir.rankedTensorType(self.shape().dims(), mlirx.Type.fromDType(mlirCtx(), to)); + const op = dialects.stablehlo.convert(mlirCtx(), self.value(), res_type, .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape.withDtype(to), op.result(0)); + } + + // TODO(Corentin) + //test convert { + // const floats = @import("floats.zig"); + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // // f4e2m1 + // { + // const x = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; + // var x_f4: [x.len]floats.Float4E2M1 = undefined; + // for (&x_f4, &x) |*xi_f4, xi| xi_f4.* = .fromF32(xi); + + // const x_d = try zml.Buffer.fromArray(platform, x); + // const x_f4_xla_d = try zml.testing.compileAndCall(platform, Tensor.convert, .{ x_d, .f4e2m1 }); + + // const x_f4_xla = x_f4_xla_d.getValue(@TypeOf(x_f4)); + // errdefer std.log.warn("convert(.f4e2m1) failed !\ninput f32:\n{e}\nzml.floats computed:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x), x_f4, x_f4_xla }); + // try std.testing.expectEqualDeep(x_f4, x_f4_xla); + // } + + // // f8e3m4 + // { + // const x = [_]f32{ 1.1 / 4.0, 1.1 / 8.0, 1.1 / 16.0, 1.1 / 32.0, 1.1 / 64.0, 1.1 / 128.0 }; + // var x_f8e3: [x.len]floats.Float8E3M4 = undefined; + // for (&x_f8e3, &x) |*xi_f8e3, xi| xi_f8e3.* = .fromF32(xi); + + // const x_d = try zml.Buffer.fromArray(platform, x); + // const x_f8e3_xla_d = try zml.testing.compileAndCall(platform, Tensor.convert, .{ x_d, .f8e3m4 }); + + // const x_f8e3_xla = x_f8e3_xla_d.getValue(@TypeOf(x_f8e3)); + // errdefer std.log.warn("convert(.f8e3m4) failed !\ninput f32:\n{e}\nzml.floats computed:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x), x_f8e3, x_f8e3_xla }); + // try std.testing.expectEqualDeep(x_f8e3, x_f8e3_xla); + // } + //} + + /// Returns a Tensor containing the element-wise rounding operation of the input Tensor. + pub fn round(self: Tensor) Tensor { + const round_op = dialects.stablehlo.round_nearest_even(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, round_op.result(0)); + } + + /// Returns a Tensor containing the element-wise clamping operation of the input Tensor. + pub fn clamp(self: Tensor, min_: Tensor, max_: Tensor) Tensor { + const op = dialects.stablehlo.clamp(mlirCtx(), min_.value(), self.value(), max_.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + // TODO(Corentin) + /// See torch.matmul + //pub fn matmul(lhs: Tensor, rhs: Tensor) Tensor { + // return @import("torch.zig").matmul(lhs, rhs); + //} + + /// Matrix multiplication, where contracting axes are specified using their tags. + /// eg dot(.{ .a, .b, .c }, .{ .a, .c, .d }, .c) -> .{ .a, .b, .d } + /// Axes with the same tag on both sides, and which aren't contracting, + /// are considered "batching axes". + pub fn dot(lhs: Tensor, rhs: Tensor, args: anytype) Tensor { + const lhs_contracting_dim: i8 = @intCast(lhs.shape().hasTag(args).?); + const rhs_contracting_dim: i8 = @intCast(rhs.shape().hasTag(args).?); + + var batching_axes: stdx.BoundedArray([2]i8, Shape.MAX_RANK) = .{}; + for (0..lhs.rank()) |lhs_tag_index| { + const lhs_tag = lhs.shape().tag(lhs_tag_index); + if (lhs_tag == Shape.toTag(args)) continue; + if (rhs.shape().hasTag(lhs_tag)) |rhs_tag_index| { + batching_axes.appendAssumeCapacity(.{ @intCast(lhs_tag_index), @intCast(rhs_tag_index) }); + } + } + return lhs.dotGeneral(rhs, &.{.{ lhs_contracting_dim, rhs_contracting_dim }}, batching_axes.slice()); + } + + // TODO(Corentin) + //test dot { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + // defer comp.deinit(); + + // comp.activate(); + // defer comp.deactivate(); + + // inline for (.{ + // .{ .{ .c = 20 }, .{ .c = 20 }, .{.c}, .{} }, + // .{ + // .{ .a = 20, .b = 21, .c = 22 }, + // .{ .a = 20, .d = 23, .c = 22 }, + // .{.c}, + // .{ .a = 20, .b = 21, .d = 23 }, + // }, + // .{ + // .{ .a = 20, .b = 21, .c = 22 }, + // .{ .c = 22, .d = 23, .e = 24 }, + // .{.c}, + // .{ .a = 20, .b = 21, .d = 23, .e = 24 }, + // }, + // .{ + // .{ .a = 20, .b = 21, .c = 22 }, + // .{ .c = 22, .d = 23, .a = 20 }, + // .{ .c, .a }, + // .{ .b = 21, .d = 23 }, + // }, + // }) |testcase| { + // const x_shape, const y_shape, const ctr, const z_shape = testcase; + // const x = Tensor.constant(x_shape, .{ .f32 = 0.0 }); + // const y = Tensor.constant(y_shape, .{ .f32 = 0.0 }); + // const z = x.dot(y, ctr); + + // try zml.testing.expectEqualShapes(Shape.init(z_shape, .f32), z.shape()); + // } + //} + + /// Generalized matrix multiplication of two tensors along the specified axes. + /// In this version batching dimensions need to be explicitly specified. + /// The result shape is made of (batching_axes ++ lhs_result_axes ++ rhs_result_axes. + /// Where "result axes" are non-contracting, non-batching axes of each input tensor. + pub fn dotGeneral( + lhs: Tensor, + rhs: Tensor, + contracting_axes: []const [2]i8, + batching_axes: []const [2]i8, + ) Tensor { + stdx.debug.assert(lhs.dtype() == rhs.dtype(), "dotGeneral expects tensors to be of the same type, got {} and {}", .{ lhs.dtype(), rhs.dtype() }); + + const Axes = stdx.BoundedArray(i64, MAX_RANK); + + var res_shape: Shape = .{ ._dtype = lhs.dtype() }; + // Validate batching axes + var lhs_batching_axes: Axes = .{}; + var rhs_batching_axes: Axes = .{}; + for (batching_axes) |b_axes| { + const l, const r = b_axes; + stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects batching dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); + var t = lhs._shape.tag(l); + if (t == Shape.TagUnknown) t = rhs._shape.tag(r); + res_shape = res_shape.appendDim(lhs._shape.dim(l), t); + lhs_batching_axes.appendAssumeCapacity(lhs._shape.axis(l)); + rhs_batching_axes.appendAssumeCapacity(rhs._shape.axis(r)); + } + + // Validate contracting axes + var lhs_contracting_axes: Axes = .{}; + var rhs_contracting_axes: Axes = .{}; + for (contracting_axes) |c_axes| { + const l, const r = c_axes; + stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects contracting dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); + lhs_contracting_axes.appendAssumeCapacity(lhs._shape.axis(l)); + rhs_contracting_axes.appendAssumeCapacity(rhs._shape.axis(r)); + } + + // Result shape is obtained by concatenating batching dimensions, (already done) + // then dimensions from lhs axes that aren't contracting nor batching, + // then dimensions from rhs axes that aren't contracting nor batching. + for (0..lhs.rank()) |l| { + if (std.mem.indexOfScalar(i64, lhs_contracting_axes.constSlice(), @intCast(l))) |_| { + continue; + } + if (std.mem.indexOfScalar(i64, lhs_batching_axes.constSlice(), @intCast(l))) |_| { + continue; + } + res_shape = res_shape.appendDim(lhs._shape.dim(l), lhs._shape.tag(l)); + } + for (0..rhs.rank()) |r| { + if (std.mem.indexOfScalar(i64, rhs_contracting_axes.constSlice(), @intCast(r))) |_| { + continue; + } + if (std.mem.indexOfScalar(i64, rhs_batching_axes.constSlice(), @intCast(r))) |_| { + continue; + } + res_shape = res_shape.appendDim(rhs._shape.dim(r), rhs._shape.tag(r)); + } + + const mlir_ctx = lhs.getContext().mlir_ctx; + const loc = mlir.Location.unknown(mlirCtx()); + const op = dialects.stablehlo.dot_general( + mlir_ctx, + lhs.value(), + rhs.value(), + mlir.rankedTensorType(res_shape.dims(), mlir.Type.fromDType(mlir_ctx, res_shape.dtype())), + .{ + .lhs_batching_dimensions = lhs_batching_axes.constSlice(), + .rhs_batching_dimensions = rhs_batching_axes.constSlice(), + .lhs_contracting_dimensions = lhs_contracting_axes.constSlice(), + .rhs_contracting_dimensions = rhs_contracting_axes.constSlice(), + .dot_precision = .fast, + }, + loc, + ).appendTo(currentBlock()); + return _result(res_shape, op.result(0)); + } + + /// Returns a Tensor containing the sigmoid function applied to each element of the input Tensor. + pub fn sigmoid(self: Tensor) Tensor { + const op = dialects.stablehlo.logistic(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + pub const logistic = sigmoid; + + /// Returns a Tensor containing the ReLU activation function applied to each element of the input Tensor. + pub fn relu(self: Tensor) Tensor { + return self.maximum(.constant(self.dims(), self.dtype().zero())); + } + + /// Returns a Tensor containing the leaky-ReLU activation function applied to each element of the input Tensor. + /// + /// LeakyReLU(x) = max(0,x) + negative_slope * min(0,x) + /// ref: https://paperswithcode.com/method/leaky-relu + pub fn leakyReLU(self: Tensor, negative_slope: f32) Tensor { + const below_zero = self.scale(negative_slope).minimum(.scalar(0, self.dtype())); + return self.relu().add(below_zero); + } + + // TODO(Corentin) + //test leakyReLU { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const input = try zml.Buffer.fromSlice(platform, .{2}, &[_]f32{ -0.6884, 1.6795 }); + // const res = try zml.testing.compileAndCall(platform, leakyReLU, .{ input, 0.1 }); + + // const expectation = zml.HostBuffer.fromArray(&[2]f32{ -0.0688, 1.6795 }); + // try zml.testing.expectClose(expectation, res, 1e-4); + //} + + /// Returns a Tensor containing the SwiGLU activation function applied to the input Tensor. + pub fn swiglu(self: Tensor, beta: f32, w: Tensor, b: Tensor) Tensor { + const sigmoid_tensor = self.mul(.constant(self._shape, DataType.Value.init(self.dtype(), beta))).sigmoid(); + const one_minus_sigmoid_tensor: Tensor = .constant(self._shape, .init(self.dtype(), 1)).sub(sigmoid_tensor); + + return self.mul(sigmoid_tensor).add(one_minus_sigmoid_tensor.mul(w.matmul(self).add(b))); + } + + /// Returns a Tensor containing the Gaussian Error Linear Units (GeLU) activation function applied to each element of the input Tensor. + /// + /// We use an approximation of the erf function using tanh: + /// gelu(x) ≃ 0.5 * x * (1 + tanh(sqrt(2 / pi) * (x + 0.044715 * x^3))) + /// see: https://paperswithcode.com/method/gelu + pub fn gelu(x: Tensor) Tensor { + const scaled_x_cube = x.mul(x).mul(x).scale(0.044715); + const beta = std.math.sqrt(2.0 / std.math.pi); + const tanh_ = x.add(scaled_x_cube).scale(beta).tanh(); + return tanh_.addConstant(1).mul(x).scale(0.5); + } + + /// Returns a Tensor containing an approximation of the Gaussian Error Linear Units (GeLU) activation function applied to each element of the input Tensor. + /// + /// It's an even more crude approximation than gelu. + pub fn quickGelu(x: Tensor) Tensor { + return x.scale(1.702).sigmoid().mul(x); + } + + /// Returns a Tensor containing the Sigmoid Linear Unit (SiLU) activation function applied to each element of the input Tensor. + /// + /// silu(x) = x σ(x) + pub fn silu(x: Tensor) Tensor { + return x.mul(.sigmoid(x)); + } + + /// Computes softmax along the given axis. + /// y[i] = exp(x[i]) / ( Σ_k exp(x[k]) + bias ) + pub fn softmax(self: Tensor, axis_: anytype) Tensor { + const a = self.axis(axis_); + const max_val = self.max(a); + const row_mask = max_val.cmp(.GT, .scalar(-std.math.inf(f64), self.dtype())); + + const exp_diff_max = self.sub(max_val).convert(.f32).exp(); + const res = exp_diff_max.div(exp_diff_max.sum(a)).convert(self.dtype()); + + // If a row is full -inf return full 0 instead of full nan, + // this fix attention when mask hides a full row. + return row_mask.broad(self.shape()).select(res, .scalar(0, self.dtype())); + } + + /// Computes softmax, but adds a bias to the sum of exponentiel. + /// y[i] = exp(x[i]) / ( Σ_k exp(x[k]) + bias ) + pub fn softmaxBiased(self: Tensor, axis_: anytype, bias: ?Tensor) Tensor { + const a = self.axis(axis_); + + if (bias == null) return self.softmax(axis_); + const b = bias.?.convert(self.dtype()).broad(self.shape().setDim(a, 1)); + const max_val: Tensor = maximum(self.max(a), b); + const exp_diff_max = self.sub(max_val).exp(); + const bias_diff_max = b.sub(max_val).exp(); + const res = exp_diff_max.div(exp_diff_max.sum(a).add(bias_diff_max)); + + // The bias means that denominator won't be 0, we don't need to handle that case. + return res; + } + + /// Returns a Tensor containing the log of the sum of exponential over the given axis. + pub fn logSumExp(self: Tensor, axis_: anytype) Tensor { + const a = self.axis(axis_); + // stabilization: shift `self` by it's max value before passing to exponent. + const max_ = self.max(a); + const log_sum_exp = log(sum(exp(self.sub(max_.broad(self._shape))), a)); + // restore the shift again + return max_.add(log_sum_exp); + } + + // TODO(Corentin) + /// Returns a Tensor containing the sum of elements over the given axis. + /// Output shape is the input shape with the axis_ dim set to 1. + //pub fn sum(self: Tensor, axis_: anytype) Tensor { + // const a = self.axis(axis_); + // return ops.reduce( + // struct { + // pub fn acc(x: Tensor, res: Tensor) Tensor { + // return res.add(x.convert(res.dtype())); + // } + // }.acc, + // self, + // Tensor.scalar(0, self.dtype()), + // &.{a}, + // ); + //} + + /// Returns a Tensor containing the mean of elements over the given axis. + /// Output shape is the input shape with the axis_ dim set to 1. + pub fn mean(self: Tensor, axis_: anytype) Tensor { + return self.sum(axis_).divByConst(self.dim(axis_)); + } + + // TODO(Corentin) + /// Returns a Tensor containing the cumulative sum of elements over the given axis. + /// Output shape is the same as input shape. + /// [0, 1, 0, 1, 0, 0, 1, 1].cumulativeSum(0) -> [0, 1, 1, 2, 2, 2, 3, 4] + /// The last value contains the sum of all element in the array. + //pub fn cumulativeSum(self: Tensor, axis_: anytype) Tensor { + // const rk = self.rank(); + // const a = self.axis(axis_); + + // const ones = [_]i64{1} ** MAX_RANK; + // var window_dimensions = ones; + // window_dimensions[a] = self.dim(a); + // var padding = [_][2]i64{.{ 0, 0 }} ** MAX_RANK; + // padding[a] = .{ self.dim(a) - 1, 0 }; + + // return ops.reduceWindow( + // Tensor.add, + // self, + // Tensor.scalar(0, self.dtype()), + // .{ + // .base_dilations = ones[0..rk], + // .window_dilations = ones[0..rk], + // .window_strides = ones[0..rk], + // .window_dimensions = window_dimensions[0..rk], + // .padding = padding[0..rk], + // }, + // ); + //} + + //test cumulativeSum { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const Local = struct { + // pub fn _cumsum(input: Tensor) Tensor { + // const x = input.withPartialTags(.{.n}); + // const y = x.cumulativeSum(.n); + // // Check that tags are propagated + // std.debug.assert(y.shape().eqlWithTags(x.shape())); + // return y; + // } + // }; + + // const x = try zml.Buffer.fromArray( + // platform, + // [2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } }, + // ); + // const res = try zml.testing.compileAndCall(platform, Local._cumsum, .{x}); + // try std.testing.expectEqual( + // [2][5]f32{ .{ 0, 1, 2, 2, 3 }, .{ 3, 4, 4, 6, 7 } }, + // try res.getValue([2][5]f32), + // ); + //} + + /// Returns a transposed Tensor computed using the given axes. + pub fn transpose(self: Tensor, axes_: anytype) Tensor { + const axes__ = self.axes(axes_).constSlice(); + const default_perm = [MAX_RANK]i64{ 7, 6, 5, 4, 3, 2, 1, 0 }; + const no_op = [MAX_RANK]i64{ 0, 1, 2, 3, 4, 5, 6, 7 }; + + const permutation: []const i64 = if (axes__.len == 0) + default_perm[MAX_RANK - self.rank() ..] + else + toI64(axes__); + + stdx.debug.assert(permutation.len == self.rank(), "transpose expects input tensor rank and 'axes_' length to be equal, got {f} and {any}", .{ self, permutation[0..@min(permutation.len, MAX_RANK + 2)] }); + + if (std.mem.eql(i64, permutation, no_op[0..self.rank()])) { + return self; + } + + const res_shape = self._shape.transpose(permutation); + if (transposeIsJustAReshape(self.shape(), permutation)) { + return self.reshape(res_shape); + } + + const op = dialects.stablehlo.transpose( + mlirCtx(), + self.value(), + mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), + .{ .permutation = toI64(permutation) }, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(res_shape, op.result(0)); + } + + pub fn swapAxes(self: Tensor, a: anytype, b: anytype) Tensor { + if (self.axis(a) == self.axis(b)) return self; + var perm: Shape.AxesArray = .{}; + for (0..self.rank()) |i| { + perm.appendAssumeCapacity(@intCast(i)); + } + perm.set(self.axis(a), self.axis(b)); + perm.set(self.axis(b), self.axis(a)); + return self.transpose(perm.constSlice()); + } + + /// Returns a Tensor with the given axis unflattened. + /// + /// unflatten((d0, d1, axis_m, d3), 2, n) -> (d0, d1, n, d2_m, d3) + pub fn unflatten(self: Tensor, axis_: i8, n: i64) Tensor { + // TODO: move to torch.zig, this equivalent to `spitAxis` + stdx.debug.assert(self.rank() < Tensor.MAX_RANK, "unflatten expects input tensor rank to be less than {}, got {}", .{ Tensor.MAX_RANK, self.rank() }); + + const a = if (axis_ >= 0) self.axis(axis_) else self.axis(axis_) + 1; + const new_dim = std.math.divExact(i64, self.dim(a), n) catch std.debug.panic("unflatten expects chosen dimension to be divisible by 'n' but {} is not divisible by {}", .{ self.dim(a), n }); + const new_shape = self._shape.set(a, n).insert(a + 1, .{ ._ = new_dim }); + + const reshaped_val = dialects.stablehlo.reshape( + mlirCtx(), + self.value(), + mlir.rankedTensorType(new_shape.dims(), mlirx.Type.fromDType(mlirCtx(), new_shape.dtype())), + mlirx.tensorType(self.getContext().mlirCtx(), new_shape), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(new_shape, reshaped_val.result(0)); + } + + /// Splits the given axis in several axes. + /// eg: `Tensor.init(.{ .a = 10, .b = 3 }).split(.a, .{.a1 = 5, .a2 = 2});` + /// The number of elements in the split shape must match the number of element + /// in the target axis. + pub fn splitAxis(self: Tensor, ax: anytype, split_shape: anytype) Tensor { + const new_shape = self._shape.splitAxis(ax, split_shape); + + const reshaped_val = dialects.stablehlo.reshape( + mlirCtx(), + self.value(), + mlir.rankedTensorType(new_shape.dims(), mlirx.Type.fromDType(mlirCtx(), new_shape.dtype())), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(new_shape, reshaped_val.result(0)); + } + + /// Merges two or more contiguous axes into one axis. + pub fn merge(self: Tensor, merges_: anytype) Tensor { + return self.reshape(self._shape.mergeAxes(merges_)); + } + + /// Merges two or more non-contiguous axes into one axis. + /// Will make a transpose if needed. + /// .{ .a, .b, .c }.mergeTranspose(.{ .a, .c }, .ac) -> .{ .b, .ac } + pub fn mergeTranspose(self: Tensor, axes_: anytype, merged: @TypeOf(.enum_literal)) Tensor { + const cont = self.contiguous(axes_); + return cont.reshape(cont._shape.mergeAxis(merged, axes_)); + } + + /// Transposes the input Tensor, such has the given axes end up in contiguous position. + /// .{ .a, .b, .c, .d }.contiguous(.{ .c, .a }) -> .{ .b, .d, .c, .a } + pub fn contiguous(self: Tensor, axes_: anytype) Tensor { + const perm = self._shape.contiguousPerm(axes_); + return self.transpose(perm.constSlice()); + } + + pub fn flatten(self: Tensor) Tensor { + return self.reshape(.{self.count()}); + } + + pub const Slice = struct { + start: i64 = 0, + end: i64 = to_the_end, + step: i32 = 1, + singleton: bool = false, + + const full = .{ .start = 0, .end = to_the_end, .step = 1 }; + + pub fn single(offset: i64) Slice { + return .{ .start = offset, .end = offset + 1, .singleton = true }; + } + + pub fn absolute(s: Slice, d: i64) Slice { + const start = if (s.start < 0) d + s.start else s.start; + const end = if (s.end == to_the_end) d else if (s.end < 0) d + s.end else s.end; + const res: Slice = .{ .start = start, .end = end, .step = s.step, .singleton = s.singleton }; + stdx.debug.assert(start < end, "Slice {f} is invalid for axis of dimension {d} (resolved to {f}", .{ s, d, res }); + return res; + } + + const to_the_end = std.math.maxInt(i64); + + pub fn format(self: Slice, writer: *std.Io.Writer) !void { + if (self.singleton) { + try writer.print("[{d}]", .{self.start}); + } else if (self.end == to_the_end and self.step == 1) { + try writer.print("[{d}..]", .{self.start}); + } else if (self.step == 1) { + try writer.print("[{d}..{d}]", .{ self.start, self.end }); + } else { + try writer.print("[{d}..{d}:{d}]", .{ self.start, self.end, self.step }); + } + } + }; + + /// Slices the input Tensor over the given axis using the given parameters. + pub fn slice1d(self: Tensor, axis_: anytype, s: Slice) Tensor { + var slices = [_]Slice{.{}} ** MAX_RANK; + slices[self.axis(axis_)] = s; + return self.slice(slices[0..self.rank()]); + } + + /// Slices the input Tensor using the given parameters. + pub fn slice(self: Tensor, slices: []const Slice) Tensor { + var start_indices: [MAX_RANK]i64 = undefined; + var strides: [MAX_RANK]i64 = undefined; + var limit_indices: [MAX_RANK]i64 = undefined; + var res_shape: Shape = self._shape; + + for (slices, 0..) |s, a| { + stdx.debug.assert(s.step > 0, "slice expects 'step' to be positive, got {} at index {}", .{ s.step, a }); + + const args: Slice = s.absolute(self.dim(a)); + start_indices[a] = args.start; + limit_indices[a] = args.end; + strides[a] = args.step; + res_shape = res_shape.setDim(a, std.math.divCeil(i64, args.end - args.start, args.step) catch unreachable); + } + + const result_type = mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())); + const slice_op = dialects.stablehlo.slice( + mlirCtx(), + self.value(), + start_indices[0..self.rank()], + limit_indices[0..self.rank()], + strides[0..self.rank()], + result_type, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + var res = _result(res_shape, slice_op.result(0)); + var to_remove: Shape.AxesArray = .{}; + for (slices, 0..) |s, a| { + if (s.singleton) to_remove.appendAssumeCapacity(@intCast(a)); + } + return res.reshape(res_shape.removeMany(to_remove.constSlice())); + } + + // TODO(Corentin) + //test slice { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const x = try zml.Buffer.fromSlice(platform, .{ 2, 5 }, &[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); + + // // Wrap slice1d to hide the anytype in the signature. + // const Local = struct { + // pub fn _slice1dAxis(input: Tensor, ax: i8, slice_: Tensor.Slice) Tensor { + // return input.slice1d(ax, slice_); + // } + // }; + + // { + // const res = try zml.testing.compileAndCall(platform, Local._slice1dAxis, .{ x, 0, .{ .end = 1 } }); + // try std.testing.expectEqual([5]f32{ 0, 1, 2, 3, 4 }, try res.getValue([5]f32)); + // } + // { + // const res = try zml.testing.compileAndCall(platform, Local._slice1dAxis, .{ x, 1, .{ .start = 1, .step = 2 } }); + // try std.testing.expectEqual([4]f32{ 1, 3, 6, 8 }, try res.getValue([4]f32)); + // } + // { + // const res = try zml.testing.compileAndCall(platform, Local._slice1dAxis, .{ x, -1, .{ .start = -2 } }); + // try std.testing.expectEqual([4]f32{ 3, 4, 8, 9 }, try res.getValue([4]f32)); + // } + //} + + inline fn wrapIndex(self: Tensor, axis_: usize, idx: i64) i64 { + return if (idx < 0) self.dim(axis_) + idx else idx; + } + + pub fn choose1d(self: Tensor, axis_: anytype, i: i64) Tensor { + return self.slice1d(axis_, .single(i)); + } + + pub fn choose(self: Tensor, offsets: anytype) Tensor { + const off, const tags = Shape.parseDimensions(offsets); + var slices = [_]Slice{.{}} ** MAX_RANK; + for (off.constSlice(), tags.constSlice()) |o, t| { + const ax = self.axis(t); + slices[ax] = .single(o); + } + return self.slice(slices[0..self.rank()]); + } + + /// Concatenates the input Tensors along the given axis. + pub fn concatenate(tensors: []const Tensor, axis_: anytype) Tensor { + if (tensors.len == 1) return tensors[0]; + stdx.debug.assert(tensors.len <= 32, "concatenate only supports up to 32 tensors, got {}", .{tensors.len}); + var buffer: [32]mlir.Value = undefined; + std.debug.assert(tensors.len <= buffer.len); + std.debug.assert(tensors.len > 0); + const a = tensors[0].axis(axis_); + // TODO(Corendos): Check that tensor axes match. + + var concatenated_dim: i64 = 0; + for (tensors, 0..) |t, i| { + buffer[i] = t.value(); + concatenated_dim += t.dim(a); + } + + const res_shape = tensors[0]._shape.set(a, concatenated_dim); + const op = dialects.stablehlo.concatenate(mlirCtx(), buffer[0..tensors.len], a, .unknown(mlirCtx())).appendTo(currentBlock()); + // log.debug("concatenate({}, {}, {d}) -> {d}", .{ tensors[0], tensors[1], a, res_shape }); + return _result(res_shape, op.result(0)); + } + + /// Concatenates the input Tensors along a new axis. The Tensors must have the same shape. + /// For x, y, z of shape .{ .a = 10, .b = 11, .c = 12 }: + /// - Tensor.stack(&.{x, y, z}, .b, .layers) -> .{ .a, .layers, .b, .c } + /// - Tensor.stack(&.{x, y, z}, 1, .layers) -> .{ .a, .layers, .b, .c } + /// - Tensor.stack(&.{x, y, z}, .last, .layers) -> .{ .a, .b, .c, .layers } + pub fn stack(tensors: []const Tensor, axis_: anytype, tag: anytype) Tensor { + // Note: we could ask the compilation context for some memory instead of stack allocating + stdx.debug.assert(tensors.len <= 32, "stack only supports up to 32 tensors, got {}", .{tensors.len}); + + const shape0 = tensors[0]._shape; + const res_shape = shape0.insertTag(axis_, 1, tag); + + for (tensors[1..]) |tensor| { + stdx.debug.assert(shape0.eqlWithTags(tensor._shape), "stack expects tensor shapes to match, got {f} and {f}", .{ shape0, tensor._shape }); + } + + var reshaped: [32]Tensor = undefined; + for (tensors, 0..) |tensor, i| { + reshaped[i] = tensor.reshape(res_shape); + } + + // Be careful here: we need to resolve ax before calling concatenate, + // because we added an axis, so all + const ax = if (@TypeOf(axis_) == @TypeOf(.enum_literal) and axis_ == .last) + shape0.rank() + else + shape0.axis(axis_); + + return Tensor.concatenate(reshaped[0..tensors.len], ax); + } + + /// Repeats a Tensor several times along the given axis. + /// + /// * repeat1d(x, axis, 4) = concat(&.{x, x, x, x}, axis); + /// * repeat1d([0, 1, 2, 3], 0, 2) = [0, 1, 2, 3, 0, 1, 2, 3] + pub fn repeat1d(self: Tensor, axis_: anytype, n_rep: u63) Tensor { + if (n_rep == 1) { + return self; + } + + const a = self.axis(axis_); + const res_shape = self._shape.setDim(a, self.dim(a) * n_rep); + + const broadshape = self._shape.insert(a, .{n_rep}); + const repeat_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a); + + return self.broadcast(broadshape, repeat_dims.dims()).reshape(res_shape); + } + + /// Repeats a Tensor several times along the given axes. + pub fn repeat(self: Tensor, n_reps: []const u63) Tensor { + // TODO: this should support the tagged syntax: x.repeat(.{ .a = 3, .b = 2}); + stdx.debug.assert(n_reps.len == self.rank(), "repeat expects tensor rank and 'n_reps' length to be equal, got {} and {}", .{ self.rank(), n_reps.len }); + + var res = self; + for (n_reps, 0..) |n_rep, a| { + if (n_rep == 1) continue; + + res = res.repeat1d(a, n_rep); + } + return res; + } + + // TODO(Corentin) + //test repeat1d { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const Local = struct { + // fn repeat1d(x: Tensor, axis_: u3, n_reps: u32) Tensor { + // return x.repeat1d(axis_, n_reps); + // } + // }; + + // { + // const inputs: [3]u8 = .{ 1, 2, 3 }; + // const expectations: [6]u8 = .{ 1, 2, 3, 1, 2, 3 }; + + // const input = try zml.Buffer.fromArray(platform, inputs); + // const output = try zml.testing.compileAndCall(platform, Local.repeat1d, .{ input, 0, 2 }); + + // try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + // } + // { + // const inputs: [2][3]u8 = .{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }; + // const expectations: [2][6]u8 = .{ .{ 1, 2, 3, 1, 2, 3 }, .{ 4, 5, 6, 4, 5, 6 } }; + + // const input = try zml.Buffer.fromArray(platform, inputs); + // const output = try zml.testing.compileAndCall(platform, Local.repeat1d, .{ input, 1, 2 }); + + // try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + // } + //} + + /// Repeats in line each value along the given axis. + /// + /// * stutter1d([0, 1, 2, 3], -1, 2) = [0, 0, 1, 1, 2, 2, 3, 3] + /// This is equivalent to repeat(ax+1) unless ax is the last axis. + pub fn stutter1d(self: Tensor, axis_: i8, n_rep: u63) Tensor { + const a = self.axis(axis_); + const broadshape = self._shape.insert(a + 1, .{n_rep}); + const res_shape = self._shape.setDim(a, self.dim(a) * n_rep); + + const stutter_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a + 1); + return self.broadcast(broadshape, stutter_dims.dims()).reshape(res_shape); + } + + /// Repeats in line each value along the given axes. + pub fn stutter(self: Tensor, n_reps: []const u63) Tensor { + // TODO: this should support the tagged syntax: x.repeat(.{ .a = 3, .b = 2}); + stdx.debug.assert(n_reps.len == self.rank(), "stutter expects tensor rank and 'n_reps' length to be equal, got {} and {}", .{ self.rank(), n_reps.len }); + + var res = self; + for (n_reps, 0..) |n_rep, a| { + if (n_rep == 1) continue; + res = res.stutter1d(@intCast(a), n_rep); + } + return res; + } + + // TODO(Corentin) + //test stutter1d { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const Local = struct { + // fn stutter1d(x: Tensor, axis_: u3, n_reps: u32) Tensor { + // return x.stutter1d(axis_, n_reps); + // } + // }; + + // { + // const inputs: [3]u8 = .{ 1, 2, 3 }; + // const expectations: [6]u8 = .{ 1, 1, 2, 2, 3, 3 }; + + // const input = try zml.Buffer.fromArray(platform, inputs); + // const output = try zml.testing.compileAndCall(platform, Local.stutter1d, .{ input, 0, 2 }); + + // try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + // } + // { + // const inputs: [2][3]u8 = .{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }; + // const expectations: [2][6]u8 = .{ .{ 1, 1, 2, 2, 3, 3 }, .{ 4, 4, 5, 5, 6, 6 } }; + + // const input = try zml.Buffer.fromArray(platform, inputs); + // const output = try zml.testing.compileAndCall(platform, Local.stutter1d, .{ input, 1, 2 }); + + // try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + // } + // { + // const inputs: [2][3]u8 = .{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }; + // const expectations: [2][6]u8 = .{ .{ 1, 2, 3, 1, 2, 3 }, .{ 4, 5, 6, 4, 5, 6 } }; + + // const input = try zml.Buffer.fromArray(platform, inputs); + // const output = try zml.testing.compileAndCall(platform, Local.stutter1d, .{ input, 0, 2 }); + + // try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + // } + //} + + /// Returns a Tensor containing the element-wise negation of the input Tensor. + pub fn negate(self: Tensor) Tensor { + const negate_op = dialects.stablehlo.negate(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, negate_op.result(0)); + } + + /// Returns a Tensor containing the element-wise cosine of the input Tensor. + pub fn cos(self: Tensor) Tensor { + const cosine_op = dialects.stablehlo.cosine(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, cosine_op.result(0)); + } + + /// Returns a Tensor containing the element-wise sine of the input Tensor. + pub fn sin(self: Tensor) Tensor { + const sine_op = dialects.stablehlo.sine(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, sine_op.result(0)); + } + + /// Returns a Tensor containing the element-wise exponential operation of the input Tensor. + pub fn exp(self: Tensor) Tensor { + const op = dialects.stablehlo.exponential(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + /// Returns a Tensor containing the element-wise logarithm operation of the input Tensor. + pub fn log(self: Tensor) Tensor { + const op = dialects.stablehlo.log(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + /// Returns a Tensor containing the element-wise square-root of the input Tensor. + pub fn sqrt(self: Tensor) Tensor { + const sqrt_op = dialects.stablehlo.sqrt(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, sqrt_op.result(0)); + } + + /// Returns a Tensor containing the element-wise reverse square-root of the input Tensor. + pub fn rsqrt(self: Tensor) Tensor { + const rsqrt_op = dialects.stablehlo.rsqrt(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, rsqrt_op.result(0)); + } + + /// Returns a Tensor containing the element-wise hyperbolic tangent of the input Tensor. + pub fn tanh(self: Tensor) Tensor { + const tanh_op = dialects.stablehlo.tanh(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, tanh_op.result(0)); + } + + /// Returns a Tensor containing the element-wise exponential minus one operation of the input Tensor. + pub fn exponentialMinusOne(self: Tensor) Tensor { + const expm1_op = dialects.stablehlo.exponential_minus_one(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, expm1_op.result(0)); + } + + pub const ArangeArgs = struct { + start: i64 = 0, + end: i64, + step: i64 = 1, + }; + /// + /// Returns a Tensor containing evenly spaced values within a given interval. + pub fn arange(args: ArangeArgs, dt: DataType) Tensor { + stdx.debug.assert(args.start <= args.end, "arange expects 'args.start' to be less than 'args.end', got {} and {}", .{ args.start, args.end }); + stdx.debug.assert(args.step > 0, "arange expects 'args.step' to be positive, got {}", .{args.step}); + + const n_steps = std.math.divCeil(i64, args.end - args.start, args.step) catch unreachable; + const sh = Shape.init(.{n_steps}, dt); + var op = dialects.stablehlo.iota( + mlirCtx(), + 0, + mlir.rankedTensorType(sh.dims(), mlirx.Type.fromDType(mlirCtx(), sh.dtype())), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + var res = _result(sh, op.result(0)); + + if (args.step != 1) { + res = res.scale(args.step); + } + + if (args.start != 0) { + res = res.addConstant(args.start); + } + + return res; + } + + /// Returns a Tensor containing values in increasing order starting from 0 along the given axis. + /// + /// The output dtype will be `.i32`, unless the given axis has a too big dimension, in that case we use `.i64`. + /// In most program this shouldn't matter, because typically this will be used in a comparison, + /// or explicitly converted by the user to do floating point arithmetic. + pub fn iota(sh: Shape, axis_: anytype) Tensor { + const a = sh.axis(axis_); + const dt: DataType = if (sh.dim(a) <= std.math.maxInt(i32)) .i32 else .i64; + const res_shape = sh.withDtype(dt); + + var op = dialects.stablehlo.iota( + mlirCtx(), + a, + mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(res_shape, op.result(0)); + } + + pub const LinspaceArgs = struct { + start: f64, + end: f64, + steps: i64, + }; + + /// Returns a Tensor containing 'args.steps' values evenly spaced from 'args.start' to 'args.end', inclusive. + pub fn linspace(args: LinspaceArgs, dt: DataType) Tensor { + stdx.debug.assert(args.start < args.end, "linspace expects 'args.start' to be less than 'args.end', got {} and {}", .{ args.start, args.end }); + stdx.debug.assert(args.steps > 0, "linspace expects 'args.steps' to be positive, got {}", .{args.steps}); + stdx.debug.assert(dt.isFloat(), "linspace expects type to be a float, got {} (hint: use arange instead)", .{dt}); + + const sh = Shape.init(.{args.steps}, dt); + var iota_op = dialects.stablehlo.iota( + mlirCtx(), + 0, + mlir.rankedTensorType(sh.dims(), mlirx.Type.fromDType(mlirCtx(), sh.dtype())), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + var res = _result(sh, iota_op.result(0)); + + if (args.steps != 1) { + res = res.scale(args.steps); + } + + if (args.start != 0) { + res = res.addConstant(args.start); + } + + return res; + } + + /// Returns a 0-rank Tensor with the given value. + pub fn scalar(val: anytype, dt: DataType) Tensor { + const data = DataType.Value.init(dt, val); + switch (dt.class()) { + .float => stdx.debug.assert(!std.math.isNan(val), "scalar(NaN) is probably due to compiling a model with an uninitialized field", .{}), + else => {}, + } + return Tensor.constant(.{}, data); + } + + // TODO(Corentin) + //test scalar { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const Local = struct { + // pub fn _fwd() [6]Tensor { + // var res: [6]Tensor = undefined; + // const dtypes = .{ .bool, .u8, .i32, .f32, .bf16, .u64 }; + // inline for (0..6) |i| res[i] = scalar(0, dtypes[i]); + // return res; + // } + // }; + + // _ = try zml.testing.compileAndCall(platform, Local._fwd, .{}); + //} + + /// Returns a constant Tensor with the given value. + pub fn constant(val: DataType.Value) Tensor { + const op = dialects.stablehlo.constant( + mlirCtx(), + &.{}, + mlirx.Type.fromDType(mlirCtx(), val.dtype()), + val.constSlice(), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(.init(&.{}, val.dtype()), op.result(0)).autoBroadcast(); + } + + // TODO(Corentin) + pub fn zeroes(sh: Shape) Tensor { + return .constant(sh.dtype().zero()); + } + + // TODO(Corentin) + /// Embeds a buffer with concrete values into an Mlir program. + pub fn constantTensor(val: HostBuffer) Tensor { + const ctx = CompilationContext.current().mlirCtx(); + const loc = ctx.location(@src()); + const elem_type = mlirx.denseElementAttrType(val.dtype()) orelse std.debug.panic("constantTensor expects a dtype that can be serialized to MLIR, like f32 or i32, got {f}", .{val.shape()}); + const constant_op = dialects.stablehlo.constant(ctx, val.shape().dims(), elem_type, val.bytes(), loc); + return _result(val.shape(), constant_op.result(0)); + } + + /// Returns a Tensor containing the result of the outer product between the input Tensors. + pub fn outer(self: Tensor, other: Tensor) Tensor { + if (self.rank() + other.rank() == 1) { + return self.mul(other); + } + + const res_shape = self.shape().outer(other.shape()); + return self.broad(res_shape).mul(other.broad(res_shape)); + } + + /// Given a tensor and a shape of the same rank, + /// will "broadcast" the given axes, so that `self` has the given shape. + /// This happens by virtually repeating the data several time along each give axes. + /// Note: most of the time the optimizer will make it so that the broadcast doesn't trigger a copy. + /// Note: the tags of the return tensor will be from the `output_shape`. + /// This means if you use and un-tagged broadcast on a tagged tensor, + /// you will lose the tags. + /// To avoid use favorise `.broad(shape)` when working with tagged tensors. + pub fn broadcast(self: Tensor, output_shape: Shape, axes_: []const i64) Tensor { + stdx.debug.assert(axes_.len == self.rank(), "broadcast expects axes_ to map all axes from self to axes of the output shape, got broadcast({f}, {f}, {any})", .{ self, output_shape, axes_ }); + for (0.., axes_) |self_ax, other_ax| { + const d = self.dim(self_ax); + stdx.debug.assert(d == 1 or d == output_shape.dim(other_ax), "broadcast expects shape axes to either be 1-sized or to match the target size. got broadcast({f}, {f}, {any}), error on self axis {d} mapping to other axis {d}", .{ self, output_shape, axes_, self_ax, other_ax }); + } + + const res_shape = output_shape.withDtype(self.dtype()); + if (std.mem.eql(i64, self.dims(), output_shape.dims())) { + // No broadcast needed. We don't emit a new stablehlo value + // but we propagate output_shape tags. + return _result(res_shape, self.value()); + } + const result_type = mlir.rankedTensorType( + res_shape.dims(), + mlirx.Type.fromDType(mlirCtx(), res_shape.dtype()), + ); + const broadcast_op = dialects.stablehlo.broadcast_in_dim(mlirCtx(), self.value(), axes_, result_type, .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(res_shape, broadcast_op.result(0)); + } + + /// Broadcasts a Tensor to the given shape, adding axes at the beginning. + pub fn broadcastLeft(self: Tensor, output_shape: Shape) Tensor { + stdx.debug.assert(self.rank() <= output_shape.rank(), "broadcastLeft expects tensor rank to be less than output tensor rank, got {d} and {d}", .{ self.rank(), output_shape.rank() }); + + const a = output_shape.rank() - self.rank(); + if (self.rank() == output_shape.rank() and std.mem.eql(i64, self.dims(), output_shape.dims())) { + return self; + } + + return self.broadcast(output_shape, Shape.range(output_shape.rank(), output_shape.dtype()).dims()[a..]); + } + + /// Broadcasts a Tensor to the given shape, adding axes at the end. + pub fn broadcastRight(self: Tensor, output_shape: Shape) Tensor { + stdx.debug.assert(self.rank() <= output_shape.rank(), "broadcastRight expects tensor rank to be less than output tensor rank, got {d} and {d}", .{ self.rank(), output_shape.rank() }); + + if (self.rank() == output_shape.rank() and self._shape.eql(output_shape)) { + return self; + } + + return self.broadcast(output_shape, Shape.range(self.rank(), output_shape.dtype()).dims()); + } + + /// Broadcasts a Tensor to the given shape, extending dimensions if needed. + pub fn broad(self: Tensor, other: Shape) Tensor { + // TODO: broad is too restrictive because sometime you only want to specify one specific axis + // Note: if you code below, make sure to update Shape.canBroadcastTo. + stdx.debug.assert(self._shape.canBroadcastTo(other), "Can't broadcast {f} to {f}", .{ self, other }); + + // Already the right shape + if (std.mem.eql(i64, self.dims(), other.dims())) return self; + + // Non ambiguous broadcasting + // TODO: broad is error prone because of this: + // it will happily broadcast .{ .a = 10, .b = 1 } to .{ .b = 10, .a = 5 } + if (self._shape.rank() == 0 or self._shape.rank() == other.rank()) { + const all_axes = [MAX_RANK]i64{ 0, 1, 2, 3, 4, 5, 6, 7 }; + return self.broadcast(other, all_axes[0..self.rank()]); + } + + // check that each axis of self maps to an axis of other + var axes_: stdx.BoundedArray(i64, MAX_RANK) = .{}; + for (self._shape.tags()) |t| { + axes_.appendAssumeCapacity(@intCast(other.axis(t))); + } + return self.broadcast(other, axes_.constSlice()); + } + + /// Reshapes the input Tensor with the given shape. + pub fn reshape(self: Tensor, output_shape_: anytype) Tensor { + const output_shape = self._shape.reshape(output_shape_); + const tensor_type = mlir.rankedTensorType(output_shape.dims(), mlirx.Type.fromDType(mlirCtx(), output_shape.dtype())); + const reshape_value = dialects.stablehlo.reshape(mlirCtx(), self.value(), tensor_type, .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(output_shape, reshape_value.result(0)); + } + + /// Converts the given 1 element Tensor into a 0-rank Tensor. + pub fn asScalar(self: Tensor) Tensor { + stdx.debug.assert(self.count() == 1, "Tensor.asScalar expects an input with exactly 1-element got {f}", .{self}); + return if (self.rank() == 0) self else self.reshape(.{}); + } + + pub const Pad = struct { + low: i64 = 0, + high: i64 = 0, + interior: i64 = 0, + }; + + /// Pads the input Tensor with the given values. + /// Usage: x.pad(0, .{ .a = .{ .low = 1, .high = 1 }}); + pub fn pad(self: Tensor, padding_value: anytype, paddings: anytype) Tensor { + const _paddings = self.shape().parseAxesOptions(Pad, paddings, .{}); + + const ZEROS = [_]i64{0} ** MAX_RANK; + var low = ZEROS; + var high = ZEROS; + var interior = ZEROS; + + var res_shape = self._shape; + for (_paddings.constSlice(), 0..) |padding, i| { + low[i] = padding.low; + high[i] = padding.high; + interior[i] = padding.interior; + + var d: i64 = self.dim(i); + d += low[i] + (@max(d - 1, 0) * interior[i]) + high[i]; + res_shape._dims.set(i, d); + } + + const rk = self.rank(); + const pad_op = dialects.stablehlo.pad( + mlirCtx(), + self.value(), + Tensor.scalar(padding_value, self.dtype()).value(), + .{ .low = low[0..rk], .high = high[0..rk], .interior = interior[0..rk] }, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + return _result(res_shape, pad_op.result(0)); + } + + /// Inserts 1-dim axes at the given position, with the given tags. + /// `.{.a = 5, .b = 4}.insert(.b, .{ .c, .d }) -> .{ .a = 5, .c = 1, .d = 1, .b = 4 }` + pub fn insertAxes(self: Tensor, axis_: anytype, tags: anytype) Tensor { + const tags_ = Shape.parseTags(tags); + const ax = if (@TypeOf(axis_) == @TypeOf(.enum_literal) and axis_ == .last) + self.rank() + else + self.axis(axis_); + + var res_shape = self._shape; + const ones = [_]i64{1} ** MAX_RANK; + res_shape._dims.insertSlice(ax, ones[0..tags_.len]) catch unreachable; + res_shape._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; + + return self.reshape(res_shape); + } + + /// Appends a 1-dim axis, with the given tag. + pub fn appendAxes(self: Tensor, t: anytype) Tensor { + // stdx.debug.assert(self.rank() < Tensor.MAX_RANK - t.len, "appendAxis expects tensor rank to be small enough in order to extend it, got {} and {} (max is {})", .{ self.rank(), t.len, Tensor.MAX_RANK }); + + return self.insertAxes(.last, t); + } + + /// Drops a 1-dim axis at the given index + pub fn squeeze(self: Tensor, axis_: anytype) Tensor { + const a = self.axis(axis_); + stdx.debug.assert(self.dim(a) == 1, "squeeze expects axis to be squeezed to have a dimension of 1, got {}", .{self.dim(a)}); + + const new_shape = self._shape.remove(a); + // log.debug("squeeze({}, {d}={d}) -> ({})", .{ self, axis, a, new_shape }); + + return _result(new_shape, self.reshape(new_shape).value()); + } + + /// Returns a Tensor with the given axes reversed. + pub fn reverse(self: Tensor, axes_: anytype) Tensor { + const actual_axes = self._shape.axes(axes_); + + const reverse_op = dialects.stablehlo.reverse(mlirCtx(), self.value(), toI64(actual_axes.constSlice()), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, reverse_op.result(0)); + } + + pub const GatherOpts = struct { indices_are_sorted: bool = false }; + + /// `gather` extracts slices from the given tensor at the specified offsets. + /// example: `values.gather(.{ .a = idx }, .{})` + /// + /// * indices is a named list of integer tensors: eg `.{ .a = idx }`. + /// Each names specify a gathering axis, it must refer to an axis of self, + /// and the corresponding idx Tensor must contains valid indices into axis .a. + /// All indices must have the same shape or broadcast to the same shape. + /// + /// * result is a tensor whose shape is similar to the input shape + /// where the gathered axes have been replaced by axes from 'indices'. + /// + /// Some example input for the base case where we work on one axis: + /// - gather(f:[a], .{ .a = idx:[n]})[n] == f[idx[n]] + /// - gather(f:[a, b], .a, idx:[n])[n, b] == f[idx[n], b] + /// - gather(f:[a,b,c], .{.b = idx:[n,m]})[a, n, m, c] == f[a, idx[n, m], c] + /// + /// If an axis in common between `self` and `indices`, + /// it is treated as a "batching" axis, meaning that semantically + /// the operator is doing a gather one time per dimension of this axis: + /// - gather(f: [a,b,c], .{.b=idx: [a,n]})[a, n] == f[a, idx[a, n]] + /// + /// It's possible to pass several indices: + /// - gather(f: [a,b,c], .{.b=idx_b[n], .c=idx_c[n]})[a, n] == f[a, idx_b[n], idx_c[n]] + /// - gather(f: [a,b,c,d], .{.b=idx_b[a,n], .c=idx_c[a, n]})[a, n, d] == f[a, idx_b[a, n], idx_c[a, n], d] + /// + /// If `self` isn't tagged, you can use `gather_` to specify gathered axis by their position but batching won't be available. + /// + /// For performance it's better to have batching and gathering axes of `self` be the first one, + /// so that gather can + pub fn gather(self: Tensor, _indices: anytype, opts: GatherOpts) Tensor { + const idx_per_axis, const idx_tags = Shape.parseStruct(Tensor, _indices); + var idx_axes: Shape.AxesArray = .{}; + for (idx_tags.slice()) |t| { + idx_axes.appendAssumeCapacity(self.axis(t)); + } + + // TODO: sort indices following self.shape instead of asking the user to do it. + return self.gather_(idx_axes.slice(), idx_per_axis.slice(), opts); + } + + pub fn gather_(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, opts: GatherOpts) Tensor { + stdx.debug.assert(idx_axes.len > 0, "gather expects 1 or more axes to operate one, received none. Example: `x.gather(.a, indices, .{{}})`", .{}); + for (idx_axes, 0..) |a, i| { + if (i > 0) { + stdx.debug.assert(a == idx_axes[i - 1] + 1, "gather expects 'idx_axes' to be sequential. But {any} aren't sequential in {f}", .{ idx_axes, self }); + } + } + var indices_shape = idx_per_axis[0].shape(); + for (idx_per_axis[1..]) |idx| { + if (idx.rank() > indices_shape.rank()) { + indices_shape = idx.shape(); + } + } + for (idx_per_axis) |idx| { + stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "gather indices can't be broadcasted together {any}", .{idx_per_axis}); + } + + var idx_batch_axes: Shape.DimsArray = .{}; + + const AxisKind = enum { batching, offset, collapsed, indices }; + var self_kind: stdx.BoundedArray(AxisKind, MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; + + for (self._shape.tags(), 0..self.rank()) |t, self_ax| { + const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, 1, @intCast(self_ax)); + if (indices_shape.hasTag(t)) |id_ax| { + // tag is both in self and indices -> it's a batching dim + // Note: tags are required for batching. + self_kind.buffer[self_ax] = .batching; + idx_batch_axes.appendAssumeCapacity(id_ax); + stdx.debug.assert(!is_gather_axis, "gather expects axes to appear at most twice. Axis {s} has been found both in 'self={f}', in 'idx_axes={any}' and in 'indices={f}'", .{ t, self, idx_axes, indices_shape }); + } else if (is_gather_axis) { + // we collapsed all gathered axes + self_kind.buffer[self_ax] = .collapsed; + // idx_kind.buffer[id_ax] = .indices; + } else { + self_kind.buffer[self_ax] = .offset; + } + } + + // compute res shape + var res_shape = Shape.init(.{}, self.dtype()); + var res_kind: stdx.BoundedArray(AxisKind, MAX_RANK) = .{}; + for (self_kind.slice(), 0..) |kind, ax_usize| { + const ax: u3 = @intCast(ax_usize); + if (ax == idx_axes[0]) { + // The first val_ax is special cause this is the place where we insert indices axes. + for (0.., indices_shape.tags(), indices_shape.dims()) |id_axis_order, id_axis, id_inserted_dim| { + const is_batching_axis = std.mem.containsAtLeastScalar(i64, idx_batch_axes.constSlice(), 1, @intCast(id_axis_order)); + // Batching axis is already in self. + if (is_batching_axis) continue; + + res_shape = res_shape.appendDim(id_inserted_dim, id_axis); + res_kind.appendAssumeCapacity(.indices); + } + } + switch (kind) { + .collapsed => continue, + else => { + res_shape = res_shape.appendDim(self.dim(ax), self._shape.tag(ax)); + res_kind.appendAssumeCapacity(kind); + }, + } + } + + // This is not a gather, but a dynamicSlice. + // Sometimes the backend recognize this pattern, but not always. + // So let us handle that. + if (indices_shape.count() == 1 and idx_axes.len == 1) { + return self.dynamicSlice1d(idx_axes[0], .{ .start = idx_per_axis[0].asScalar(), .len = 1 }).reshape(res_shape); + } + + var slice_dims: Shape.DimsArray = .{}; + for (self_kind.slice(), self.dims()) |k, d| { + slice_dims.appendAssumeCapacity(switch (k) { + .batching, .collapsed => 1, + .offset => d, + .indices => unreachable, + }); + } + + // TODO: try changing .last by other axis and see the perf impact. + const indices = Tensor.stack(idx_per_axis, .last, .coord); + // scoped_log.debug("gather --> {} {any}", .{ res_shape, res_kind.constSlice() }); + const gather_op = dialects.stablehlo.gather( + mlirCtx(), + self.value(), + indices.value(), + slice_dims.constSlice(), + .{ + .offset_dims = _collectAxes(AxisKind, res_kind, .offset).constSlice(), + .collapsed_slice_dims = _collectAxes(AxisKind, self_kind, .collapsed).constSlice(), + .operand_batching_dims = _collectAxes(AxisKind, self_kind, .batching).constSlice(), + .start_indices_batching_dims = idx_batch_axes.constSlice(), + .start_index_map = _collectAxes(AxisKind, self_kind, .collapsed).constSlice(), + .index_vector_dim = indices.axis(.coord), + .indices_are_sorted = opts.indices_are_sorted, + }, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + const mlir_shape = fromMlirValue(gather_op.result(0)).shape(); + stdx.debug.assert(mlir_shape.eql(res_shape), "gather expects that batching indices appear in the same order in 'self' and 'indices', got: self={f}, indices={f}. You should transpose one or the other.", .{ self, indices }); + return _result(res_shape, gather_op.result(0)); + } + + // TODO(Corentin) + //test gather { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const Local = struct { + // pub fn _idx(idx_shape: anytype) Tensor { + // return Tensor.constant(idx_shape, .{ .i32 = 0 }); + // } + // }; + + // const idx = Local._idx; + + // { + // // Only test shapes + // var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + // defer comp.deinit(); + // comp.activate(); + // defer comp.deactivate(); + + // inline for (.{ + // .{ .{ .a = 10 }, .{ .a = idx(.{}) }, .{} }, + // .{ .{ .a = 10 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{}) }, .{ .b = 20 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8, .b = 20 } }, + // // .{ .{ .a = 10, .b = 20 }, 0, idx(.{ .n = 8 }), .{ .n = 8, .b = 20 } }, + // // Favor val shape, instead of indices shape. + // .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .n = 8 }) }, .{ .a = 10, .n = 8 } }, + // .{ .{ .a = 10, .b = 20, .c = 30 }, .{ .b = idx(.{ .n = 8 }) }, .{ .a = 10, .n = 8, .c = 30 } }, + // // batching axes are implicits. + // .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10 }) }, .{ .a = 10 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .b = 20 }) }, .{ .b = 20 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .n = 8 } }, + // // stablehlo.gather is biased toward indices shape (like gatherSlice). + // // This make it awkward to use when you have both batching dimension and new indices dimensions. + // // For now we reject those, and let user explicitly transpose self or indices if needed. + // // .{ .{ .a = 10, .b = 20 }, .{.b = idx(.{ .n = 8, .a = 10 })}, .{ .a = 10, .n = 8 } }, + // // Also handle tuples + // .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }), .b = idx(.{ .n = 8 }) }, .{ .n = 8 } }, + // }) |testcase| { + // const x_shape, const indices, const res_shape = testcase; + // const x = Tensor.constant(x_shape, .{ .f16 = 0 }); + // const y = gather(x, indices, .{}); + // try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); + // try std.testing.expect(y.value().owner().verify()); + // } + + // inline for (.{ + // .{ .{ 10, 20 }, &[_]u3{ 0, 1 }, &[_]Tensor{ idx(.{8}), idx(.{8}) }, .{8} }, + // }) |testcase| { + // const x_shape, const idx_axes, const idx_per_axis, const res_shape = testcase; + // const x = Tensor.constant(x_shape, .{ .f16 = 0 }); + // const y = gather_(x, idx_axes, idx_per_axis, .{}); + // try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); + // try std.testing.expect(y.value().owner().verify()); + // } + // } + //} + + /// Gathers slices along the given axes with runtime indices. + /// * slice_shape represents the shape of the slices to extract, + /// it must be smaller than original shape. + /// It must use a subset of self axes. + /// If slice_shape is **not** tagged, then it must have the same rank than self. + /// * `indices` represents a set of coordinates. + /// The coordinates are read from the `.coord` axis, or last axis if `.coord` is not found. + /// The coordinate axis must have `slice_shape.rank()` dims. + /// The coordinates represent the "top-left" corner of the slice to extract. + /// * the output tensor starts with axes from `indices`. + /// * if the input tensor has tagged axes, matching `indices` axes, + /// they will be considered "batching" axes. + /// + /// Sample input/output shapes: + /// * gatherSlices([A, B, C, D], .{.b=B', .c=C'}, [N, 2]) -> [N, A, B', C', D] + /// * gatherSlices(x(a,b,c,d), .{.b=B', .c=C'}, g(n,m)) = z(n, a, b', c', d) = x(a, g(n, 0) + b', g(n, 1) + c', d) + /// + /// Note: the axis order of the result is different from gatherValues. + /// This is because gatherSlices, favorizes contiguous copy of the extracted slices, + /// while gatherValues, always copy values one by one, and as such don't have the same issues. + /// In our example the contiguous dimension .d is not sliced + /// and gatherSlices can copy data by group of C'*D elements. + pub fn gatherSlices(self: Tensor, slice_shape_: anytype, indices: Tensor, opts: GatherOpts) Tensor { + const slice_shape = if (@TypeOf(slice_shape_) == Shape) slice_shape_ else Shape.init(slice_shape_, .i32); + // scoped_log.debug("gatherSlice({}, {f}, {})", .{ self, slice_shape, indices }); + + const tagged_api = slice_shape.isFullyTagged(); + if (tagged_api) { + for (slice_shape.tags()) |t| { + stdx.debug.assert(self._shape.hasTag(t) != null, "gatherSlices expects `slices_shape` to only use tags from `self`. But {s} wasn't found in {f}", .{ t, self }); + } + } else { + // For untagged api, we require all slices to be specified. + // Note: we could relax this and right align the slice. + stdx.debug.assert(slice_shape.rank() == self.rank(), "gatherSlices expects `slice_shape.rank()` to match `self.rank()`. Got: gatherSlices({f}, slice={f}). To avoid specifying all axes in `slice_shape`, you can use tags.", .{ self, slice_shape }); + } + + const index_coord_axis = indices._shape.hasTag(.coord) orelse indices._shape.axis(-1); + stdx.debug.assert(indices.dim(index_coord_axis) == slice_shape.rank(), "gatherSlices({f}, slice={f}, indices) expects 'indices' to be a tensor [..., {}], got {f}", .{ self, slice_shape, slice_shape.rank(), indices }); + + // Compute result shape + var res_shape = indices._shape.remove(index_coord_axis).withDtype(self.dtype()); + var slice_dims = self._shape._dims; + var self_batch_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; + var indices_batch_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; + var start_index_map: stdx.BoundedArray(i64, MAX_RANK) = .{}; + var self_offset_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; + for (self._shape.tags(), 0..self.rank()) |t, self_ax| { + const maybe_slice_ax: ?u3 = if (tagged_api) slice_shape.hasTag(t) else @intCast(self_ax); + + if (tagged_api and indices._shape.hasTag(t) != null) { + // tag is both in self and indices -> it's a batching dim + // Note: tags are required for batching. + self_batch_axes.appendAssumeCapacity(@intCast(self_ax)); + indices_batch_axes.appendAssumeCapacity(indices._shape.axis(t)); + slice_dims.set(self_ax, 1); + stdx.debug.assert(slice_shape.hasTag(t) == null, "gatherSlices expect axes to be either batches or slices axes. Axis {s} has been found both in `slices={f}` and `indices={f}`", .{ t, slice_shape, indices }); + } else if (maybe_slice_ax) |slice_ax| { + // Specified axes contains the start offset of the slices, + // and are collected in `start_index_map`. + const slice_dim = slice_shape.dim(slice_ax); + stdx.debug.assert(slice_dim <= self._shape.dim(self_ax), "gatherSlices expects `slice_shape` to be smaller than `self.shape()`. On axis {s}, got {f} > {f}.", .{ t, slice_shape, self._shape }); + slice_dims.set(self_ax, slice_dim); + res_shape = res_shape.appendDim(slice_dim, t); + start_index_map.appendAssumeCapacity(@intCast(self_ax)); + self_offset_axes.appendAssumeCapacity(res_shape.rank() - 1); + } else { + // non-batching, non-indexed axes + res_shape = res_shape.appendDim(self.dim(self_ax), t); + self_offset_axes.appendAssumeCapacity(res_shape.rank() - 1); + } + } + + const gather_op = dialects.stablehlo.gather( + mlirCtx(), + self.value(), + indices.value(), + slice_dims.constSlice(), + .{ + .offset_dims = self_offset_axes.constSlice(), + .collapsed_slice_dims = &.{}, + .operand_batching_dims = self_batch_axes.constSlice(), + .start_indices_batching_dims = indices_batch_axes.constSlice(), + .start_index_map = start_index_map.constSlice(), + .index_vector_dim = index_coord_axis, + .indices_are_sorted = opts.indices_are_sorted, + }, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(res_shape, gather_op.result(0)); + } + + // TODO(Corentin) + //test gatherSlices { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const Local = struct { + // pub fn _gatherSlices(self: Tensor, slice_shape: Shape, indices: Tensor, opts: GatherOpts) Tensor { + // return self.gatherSlices(slice_shape, indices, opts); + // } + // }; + + // { + // // Only test shapes + // var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + // defer comp.deinit(); + // comp.activate(); + // defer comp.deactivate(); + + // inline for (.{ + // .{ .{ .a = 10 }, .{}, .{ ._ = 0 }, .{ .a = 10 } }, + // .{ .{ .a = 10 }, .{ .a = 7 }, .{ ._ = 1 }, .{ .a = 7 } }, + // .{ .{ .a = 10 }, .{ .a = 7 }, .{ .n = 8, ._ = 1 }, .{ .n = 8, .a = 7 } }, + // .{ .{ .a = 10 }, .{ .a = 7 }, .{ .coord = 1, .n = 8 }, .{ .n = 8, .a = 7 } }, + // // tags aren't required. + // .{ .{10}, .{7}, .{ .n = 8, ._ = 1 }, .{ .n = 8, ._ = 7 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .a = 7 }, .{ ._ = 1 }, .{ .a = 7, .b = 20 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .a = 7 }, .{ .n = 8, ._ = 1 }, .{ .n = 8, .a = 7, .b = 20 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .a = 7 }, .{ .n = 8, .coord = 1, .m = 9 }, .{ .n = 8, .m = 9, .a = 7, .b = 20 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .b = 17 }, .{ .n = 8, ._ = 1 }, .{ .n = 8, .a = 10, .b = 17 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .a = 7, .b = 17 }, .{ .n = 8, ._ = 2 }, .{ .n = 8, .a = 7, .b = 17 } }, + // // Note: currently the order of the axes in the slice is not used. + // .{ .{ .a = 10, .b = 20 }, .{ .b = 17, .a = 7 }, .{ .n = 8, ._ = 2 }, .{ .n = 8, .a = 7, .b = 17 } }, + // .{ .{ .a = 10, .b = 20, .c = 20 }, .{ .b = 17 }, .{ .n = 8, ._ = 1 }, .{ .n = 8, .a = 10, .b = 17, .c = 20 } }, + // // batching dims + // .{ .{ .a = 10, .b = 20 }, .{ .b = 17 }, .{ .a = 10, ._ = 1 }, .{ .a = 10, .b = 17 } }, + // .{ .{ .b = 200, .a = 100, .c = 300 }, .{ .c = 300 }, .{ .a = 100, .b = 200, ._ = 1 }, .{ .a = 100, .b = 200, .c = 300 } }, + // }) |testcase| { + // const x_shape, const slice_dims, const idx_shape, const res_shape = testcase; + // const x = Tensor.constant(x_shape, .{ .f16 = 0 }); + // const slice_shape = Shape.init(slice_dims, .u16); + // const idx = Tensor.constant(idx_shape, .{ .i32 = 0 }); + // const y = gatherSlices(x, slice_shape, idx, .{}); + // try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); + // try std.testing.expect(y.value().owner().verify()); + + // const mod = try zml.compileFn( + // std.testing.allocator, + // Local._gatherSlices, + // .{ x.shape(), slice_shape, idx.shape(), .{ .indices_are_sorted = true } }, + // platform, + // ); + // defer mod.deinit(); + // } + // } + + // // Test with actual values. + // const range = try zml.HostBuffer.arange(std.testing.allocator, .{ .end = 2 * 4 * 6 }, .u16); + // defer range.deinit(std.testing.allocator); + // const operand = try range.reshape(.{ .a = 2, .b = 4, .c = 6 }).toDevice(platform); + // defer operand.deinit(); + // const start_indices = (try zml.Buffer.fromArray(platform, [2][2]i32{ .{ 2, 1 }, .{ 0, 3 } })).withTags(.{ .n, ._ }); + // defer start_indices.deinit(); + + // const result = try zml.testing.compileAndCall(platform, Local._gatherSlices, .{ operand, Shape.init(.{ .b = 2, .c = 3 }, .u16), start_indices, .{} }); + + // const expected = zml.HostBuffer.fromArray(&[2][2][2][3]u16{ + // .{ + // .{ .{ 13, 14, 15 }, .{ 19, 20, 21 } }, + // .{ .{ 37, 38, 39 }, .{ 43, 44, 45 } }, + // }, + // .{ + // .{ .{ 3, 4, 5 }, .{ 9, 10, 11 } }, + // .{ .{ 27, 28, 29 }, .{ 33, 34, 35 } }, + // }, + // }); + // try zml.testing.expectClose(expected, result, 0); + //} + + pub const ScatterArgs = struct { + input: Tensor, + update: Tensor, + }; + + pub const ScatterOpts = struct { + /// Promise scatter that all coordinates in `indices` are sorted, wrt to the final offset in `self` + /// Result is undefined if the promise is violated. + indices_are_sorted: bool = false, + + /// Promise scatter that slices don't overlap. + /// Result is undefined if the promise is violated. + /// This allows for better code generation, because it means that updates can be applied in parallel. + indices_are_unique: bool = false, + + /// Function used to update previous value in `self` with values from `updates`. + /// If `update_fn` is not associative (ie the order of execution matters), + /// then you should make sure the slices don't overlap, + /// otherwise the result will depend on the runtime scheduling + /// of the operator which is backend specific. + update_fn: *const fn (ScatterArgs) struct { Tensor } = increment, + + pub fn increment(values: ScatterArgs) struct { Tensor } { + return .{values.input.add(values.update)}; + } + + pub fn override(values: ScatterArgs) struct { Tensor } { + return .{values.update}; + } + }; + + /// Generalized version of scatter to many inputs. + /// See `zml.Tensor.scatterSlices` for documentation on scatter. + /// + /// This allows to use the same indices to update several tensors at once, + /// and where the update function is allow to look at elements from the different tensors + /// to compute the final value. + /// + /// This sounds nice but in practice XLA doesn't support this well on GPU, + /// and will generate slow code. In practice stick with `zml.Tensor.scatterSlices`. + pub fn scatter( + inputs: anytype, + index_tensors: anytype, + updates: anytype, + comptime func: anytype, + context: anytype, + opts: Tensor.ScatterOpts, + ) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const update_block, var result = b: { + const ArgsType = std.meta.Tuple(&[1]type{ScatterArgs} ** inputs.len); + var args: ArgsType = undefined; + var block_types: [2 * inputs.len]*const mlir.Type = undefined; + + inline for (0..inputs.len) |i| { + args[i].input = Tensor.init(Shape.init(.{}, inputs[i].dtype())); + args[i].update = Tensor.init(Shape.init(.{}, inputs[i].dtype())); + + block_types[i] = mlir.rankedTensorType(args[i].input.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].input.dtype())); + block_types[i + inputs.len] = mlir.rankedTensorType(args[i].update.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].update.dtype())); + } + + const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlirCtx())); + const update_block = mlir.Block.init(&block_types, &block_locs); + errdefer update_block.deinit(); + + CompilationContext.current().pushBlock(update_block); + defer CompilationContext.current().popBlock(); + + const scope = CompilationContext.current().currentScope(); + inline for (0..inputs.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), args[i].input.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].update.id, i + inputs.len) catch unreachable; + } + + var result = @call(.auto, func, args ++ context); + + var result_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + result_values[i] = result[i].value(); + } + + _ = dialects.stablehlo.returns(mlirCtx(), &result_values, .unknown(mlirCtx())).appendTo(update_block); + break :b .{ update_block, result }; + }; + + // Note: I was a bit lazy here, and I only look at tags on the first tensor. + // we probably should check all of them. + const self = meta.first(Tensor, inputs); + const update = meta.first(Tensor, updates); + var indices_per_axis, var indices_axes = Shape.parseStruct(Tensor, index_tensors); + + if (indices_per_axis.len == 0) return inputs; + + // validate coord axes: all coord_axes should exist inside self + for (indices_axes.constSlice()) |t| { + stdx.debug.assert(self._shape.hasTag(t) != null, "zml.ops.scatter expects axes of indices to be axes of inputs, got input={f} and indices={any}", .{ self, indices_axes.constSlice() }); + } + + // Handle scalar indices by broadcasting them to the indices with the highest rank. + const indices_shape = blk: { + var higher_rank = indices_per_axis.get(0).shape(); + for (indices_per_axis.constSlice()[1..]) |indices| { + if (indices.rank() > higher_rank.rank()) { + higher_rank = indices.shape(); + } + } + break :blk higher_rank; + }; + for (indices_per_axis.slice()) |*idx| { + stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "zml.ops.scatter expects all indices tensor to have the same shape, got {any}", .{indices_per_axis.slice()}); + stdx.debug.assert(idx.dtype() == indices_shape.dtype(), "zml.ops.scatter expects all indices tensor to have the same dtype, got {any}", .{indices_per_axis.slice()}); + idx.* = idx.broad(indices_shape); + } + + // rewrite simple scatters to dynamicUpdateSlice. + if (@TypeOf(inputs) == struct { Tensor } and indices_shape.rank() == 0) { + return .{self.dynamicUpdateSlice(index_tensors, update)}; + } + + // TODO: ideally we should catch all possible scatter errors and provide nice error messages. + var config = scatterConfig(self.shape(), update.shape(), indices_per_axis, indices_axes); + const indices = scatterPrepareIndices(&config, self.shape(), update.shape(), &indices_per_axis, &indices_axes); + // const n_indices_axes = update.rank() - _collectAxes(AxisKind, up_kind, .update_window).len; + // stdx.debug.assert(n_indices_axe == indices_axes.len, "scatter({f}, {any}) expects 'updates' to contain all axes from 'indices', got indices={s}, updates={f}", .{ self, index_tensors, indices_axes.constSlice(), update }); + + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); + } + + var updates_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..updates.len) |i| updates_values[i] = updates[i].value(); + + const op = dialects.stablehlo.scatter( + mlirCtx(), + &input_values, + &.{indices.value()}, + &updates_values, + update_block, + .{ + .update_window_dims = _collectAxes(AxisKind, config.up_kind, .update_window).constSlice(), + .inserted_window_dims = _collectAxes(AxisKind, config.op_kind, .inserted_window).constSlice(), + .input_batching_dims = _collectAxes(AxisKind, config.op_kind, .batching).constSlice(), + .scatter_indices_batching_dims = config.indices_batch_axes.constSlice(), + .scatter_dims_to_operand_dims = config.scatter_to_operand_axes.constSlice(), + .index_vector_dim = indices.rank() - 1, + .indices_are_sorted = opts.indices_are_sorted, + .unique_indices = opts.indices_are_unique, + }, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + inline for (0..result.len) |i| { + result[i] = Tensor._result(inputs[i].shape(), op.result(i)); + } + + return result; + } + + const ScatterConfig = struct { + op_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}, + up_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}, + indices_batch_axes: Shape.DimsArray = .{}, + scatter_to_operand_axes: Shape.DimsArray = .{}, + updates_transpose: Shape.AxesArray = .{}, + }; + + const AxisKind = enum { batching, update_window, inserted_window, window_id }; + + fn scatterConfig( + op: Shape, + update: Shape, + indices_per_axis: stdx.BoundedArray(Tensor, Tensor.MAX_RANK), + indices_axes: Shape.TagsArray, + ) ScatterConfig { + var op_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}; + var up_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}; + var indices_batch_axes: Shape.DimsArray = .{}; + var scatter_to_operand_axes: Shape.DimsArray = .{}; + var updates_transpose: Shape.AxesArray = .{}; + + const tagged_api = indices_axes.len > 0; + const indices = indices_per_axis.get(0).shape(); + + if (tagged_api) { + for (indices_axes.constSlice()) |t| { + scatter_to_operand_axes.appendAssumeCapacity(op.axis(t)); + } + for (indices.tags()) |t| { + stdx.debug.assert(update.hasTag(t) != null, "scatter expects 'updates' to have all axes of 'indices', got self={f}, updates={f} and indices={f}", .{ op, update, indices }); + updates_transpose.appendAssumeCapacity(update.axis(t)); + } + + for (op.tags()) |t| { + if (update.hasTag(t)) |up_ax| { + updates_transpose.appendAssumeCapacity(up_ax); + + if (indices.hasTag(t)) |id_ax| { + if (std.mem.indexOfScalar(Shape.Tag, indices_axes.constSlice(), t) != null) { + // tag is in indices AND in coords -> it's a batching dim that has been rewritten to a regular insertion dim + op_kind.appendAssumeCapacity(.inserted_window); + } else { + // tag is in op, indices and updates -> it's a batching dim + op_kind.appendAssumeCapacity(.batching); + indices_batch_axes.appendAssumeCapacity(@intCast(id_ax)); + } + } else { + op_kind.appendAssumeCapacity(.update_window); + } + } else { + op_kind.appendAssumeCapacity(.inserted_window); + } + } + + for (update.tags(), 0..) |t, up_ax| { + // Handle batch axes right away. + if (op.hasTag(t)) |self_ax| { + if (op_kind.get(self_ax) == .batching) { + up_kind.appendAssumeCapacity(.batching); + continue; + } + } + if (indices.hasTag(t) != null) { + up_kind.appendAssumeCapacity(.window_id); + } else if (op.hasTag(t)) |self_ax| { + stdx.debug.assert(update.dim(up_ax) <= op.dim(self_ax), "scatter expects the slices described in 'updates' to fit inside 'op', but along axis .{s} it doesn't. Got op={f}, updates={f}.", .{ t, op, update }); + up_kind.appendAssumeCapacity(.update_window); + } else { + // TODO: consider accepting untagged update here. + std.debug.panic("scatter expects 'updates' to be made of axes from op={f} and from indices={any}, got unknown tag {s} in {f}", .{ op, indices_axes.constSlice(), std.mem.sliceTo(t, 0), update }); + } + } + } else { + for (0..indices_per_axis.len) |i| { + op_kind.appendAssumeCapacity(.inserted_window); + scatter_to_operand_axes.appendAssumeCapacity(@intCast(i)); + up_kind.appendAssumeCapacity(.window_id); + } + for (indices_per_axis.len..op.rank()) |_| { + op_kind.appendAssumeCapacity(.update_window); + } + for (indices_per_axis.len..update.rank()) |_| { + up_kind.appendAssumeCapacity(.update_window); + } + for (0..update.rank()) |i| { + updates_transpose.appendAssumeCapacity(@intCast(i)); + } + } + + return .{ + .op_kind = op_kind, + .up_kind = up_kind, + .indices_batch_axes = indices_batch_axes, + .scatter_to_operand_axes = scatter_to_operand_axes, + .updates_transpose = updates_transpose, + }; + } + + // TODO(Corentin) + //test scatterConfig { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + // defer comp.deinit(); + // comp.activate(); + // defer comp.deactivate(); + + // const Local = struct { + // pub fn _idx(idx_shape: anytype) Tensor { + // return Tensor.constant(idx_shape, .{ .i32 = 0 }); + // } + // }; + + // const idx = Local._idx; + // const op = Shape.init(.{ .a = 10, .b = 20 }, .f32); + + // // Use .a as a batching axis with .a=10 x .n=8 updates of 2 elements of .b + // { + // const indices, const coords_tags = Shape.parseStruct(Tensor, .{ .b = idx(.{ .a = 10, .n = 8 }) }); + // const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); + + // const cfg = scatterConfig(op, update, indices, coords_tags); + // try std.testing.expectEqualSlices(AxisKind, &.{ .batching, .update_window }, cfg.op_kind.constSlice()); + // try std.testing.expectEqualSlices(AxisKind, &.{ .batching, .window_id, .update_window }, cfg.up_kind.constSlice()); + // } + + // // similar, but use the normalized form where .a is no longer an explicit batching axis. + // { + // const indices, const coords_tags = Shape.parseStruct(Tensor, .{ .a = idx(.{ .a = 10, .n = 8 }), .b = idx(.{ .a = 10, .n = 8 }) }); + // const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); + + // const cfg = scatterConfig(op, update, indices, coords_tags); + // try std.testing.expectEqualSlices(AxisKind, &.{ .inserted_window, .update_window }, cfg.op_kind.constSlice()); + // try std.testing.expectEqualSlices(AxisKind, &.{ .window_id, .window_id, .update_window }, cfg.up_kind.constSlice()); + // } + //} + + /// Concatenate all indices tensor in one tensor. + /// + /// Is allowed to reorder stuff to simplify the job of the backend, + /// and to expand the batching dims. + fn scatterPrepareIndices( + cfg: *ScatterConfig, + op: Shape, + update: Shape, + indices_per_axis: *stdx.BoundedArray(Tensor, Tensor.MAX_RANK), + indices_axes: *Shape.TagsArray, + ) Tensor { + var old_scatter_to_op_axes = cfg.scatter_to_operand_axes; + const batching = _collectAxes(AxisKind, cfg.op_kind, .batching); + for (batching.constSlice()) |batch_ax| { + const id_shape = indices_per_axis.get(0).shape(); + // batching requires tagging, so we're sure to have a tag here. + const batch_tag = op.tag(batch_ax); + indices_axes.appendAssumeCapacity(batch_tag); + const batch_id = Tensor.iota(id_shape, batch_tag).convert(id_shape.dtype()); + indices_per_axis.appendAssumeCapacity(batch_id); + cfg.op_kind.buffer[@intCast(batch_ax)] = .inserted_window; + cfg.up_kind.buffer[update.axis(batch_tag)] = .window_id; + old_scatter_to_op_axes.appendAssumeCapacity(batch_ax); + } + cfg.indices_batch_axes = .{}; + + // Reorder the axes so that in indices_per_axis is ordered like in op if possible. + // TODO: transpose updates if needed + var indices: stdx.BoundedArray(Tensor, Tensor.MAX_RANK) = .{}; + var scatter_to_op_axes: Shape.DimsArray = .{}; + + while (old_scatter_to_op_axes.len > 0) { + const scatter_ax = std.sort.argMin(i64, old_scatter_to_op_axes.constSlice(), {}, std.sort.asc(i64)).?; + const op_ax = old_scatter_to_op_axes.orderedRemove(scatter_ax); + const scatter_idx = indices_per_axis.orderedRemove(scatter_ax); + + scatter_to_op_axes.appendAssumeCapacity(op_ax); + indices.appendAssumeCapacity(scatter_idx); + } + cfg.scatter_to_operand_axes = scatter_to_op_axes; + + for (scatter_to_op_axes.constSlice(), 0..) |sc_ax, i| { + if (i != sc_ax) { + //log.warn("Found a slow scatter pattern, which is going to generate a while loop: scatter({f}, {any}, {f}). Because the index axes aren't the major ones in the input tensor.", .{ op, scatter_to_op_axes.constSlice(), update }); + break; + } + } + return Tensor.stack(indices.constSlice(), .last, .coord); + } + + /// Update the given tensor, by copying `values` into slice by slice into `self`. + /// The slices are chosen at runtime by interpreting indices as coordinates into `self`. + /// This is a generalized version of `dynamicUpdateSlice` where more than one offset can be specified at a time. + /// + /// ### Arguments + /// + /// - Return a tensor with same shape than `self`, with updated content. + /// - `indices` is a set of Tensor (typically rank 1), representing coordinates into `self`. + /// all indices must have the same shape, but scalars are accepted. + /// - each `indices` entry contains offset along an axes into `self`. + /// Typically axes are identified by their tags, but in the absence of tags on `indices`, + /// The entry in indices will be assigned to axes of `self` from major to minor axis. + /// It is recommended to have indices referencing only major axes of `self` for better performance. + /// - `values` shape is obtained by concatenating the shape of `indices` with the shape of the slices to be extracted. + /// - `opts`: `zml.Tensor.ScatterOpts` des + /// + /// ### Sample input/output shapes with corresponding pseudo-code. + /// + /// Basic `scatterSlices` with the first two axes (.a, .b) being indexed, and full (.c, .d) slice copies: + /// + /// ``` + /// fn scatterSlices(x[A, B, C, D], .{.a=off_a[N], .b=off_b[N]}, y[N, C, D]) [A, B, C, D] { + /// var z = x; + /// for (0..N) |n| { + /// for (0..C) |c| for (0..D) |d| {{ + /// z[off_a[n],off_b[n],c,d] += y[n, c, d]; + /// }} + /// } + /// return z; + /// } + /// ``` + /// + /// `scatterSlices` with the first three axes (.a, .b, .c) being indexed, and a partial copy of (.c, .d). + /// Note that .c axis is present both in the indices and updates, and `updates.dim(.c) < self.dim(.c)`. + /// + /// ``` + /// fn scatterSlices(x[A, B, C, D], .{.a=off_a[N], .b=off_b[N], .c=off_c[N]}, y[N, C', D]) [A, B, C, D] { + /// var z = x; + /// for (0..N) |n| { + /// for (0..C') |c| for (0..D) |d| {{ + /// z[off_a[n],off_b[n],off_c[n]+c,d] += y[n, c, d]; + /// }} + /// } + /// return z; + /// } + /// ``` + /// + /// `scatterSlices` with the first axis .a being indexed, and where .b is used as a batching axis. + /// Note that here .b axis is present in `self`, `off_a`, and `updates`, + /// and is not mentionned in the axes of indices. + /// + /// ``` + /// fn scatterSlices(x[A, B, C, D], .{.a=off_a[B,N]}, y[N, B, C, D]) [A, B, C, D] { + /// var z = x; + /// for (0..B) |b| { + /// for (0..N) |n| { + /// for (0..C) |c| for (0..D) |d| {{ + /// z[off_a[b,n],b,c,d] += y[n, b, c, d]; + /// }} + /// } + /// } + /// return z; + /// } + /// ``` + /// + /// ### Warnings + /// + /// - if `opts.update_fn` is not associative not all calls to `scatterSlices` are sound. + /// In particular if you scatter overlapping slices, with `zml.Tensor.ScatterOpts.override`, + /// then the result will depend on the execution order that you don't control. + /// - `scatterSlices` is a very expressive operator, and can lead to complicated code generation + /// that requires host<->device synchronization. + /// ZML tries to generate the easiest to optimize IR, and will warn you if it generates known problematic IR. + pub fn scatterSlices(self: Tensor, indices: anytype, updates: Tensor, opts: ScatterOpts) Tensor { + //scoped_log.debug("scatterSlices({}, {any}, {})", .{ self, indices, updates }); + + const UpdateType = @TypeOf(ScatterOpts.increment); + + const Custom = struct { + pub fn inc(values: ScatterArgs, custom: *const UpdateType) struct { Tensor } { + return @call(.auto, custom, .{values}); + } + }; + + return scatter(.{self}, indices, .{updates}, Custom.inc, .{opts.update_fn}, opts)[0]; + } + + // TODO(Corentin) + //test scatterSlices { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const Local = struct { + // pub fn _scatter(self: Tensor, indices: []const Tensor, updates: Tensor) Tensor { + // return self.scatterSlices( + // indices, + // updates, + // .{ .update_fn = ScatterOpts.increment }, + // ); + // } + + // pub fn _scatterCB(self: Tensor, coords: Tensor, updates: Tensor) Tensor { + // return self.scatterSlices( + // .{ .c = coords.choose1d(.coord, 0), .b = coords.choose1d(.coord, 1) }, + // updates, + // .{ .update_fn = ScatterOpts.increment }, + // ); + // } + + // pub fn _idx(idx_shape: anytype) Tensor { + // return Tensor.constant(idx_shape, .{ .i32 = 0 }); + // } + // }; + + // { + // // Only test shapes + // var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + // defer comp.deinit(); + // comp.activate(); + // defer comp.deactivate(); + // const idx = Local._idx; + + // inline for (.{ + // // This is equivalent to a dynamic update slice, update 3 values at given offset of axis .a: + // .{ .{ .a = 10 }, .{ .a = idx(.{}) }, .{ .a = 3 } }, + // // Use .a as a batching axis with .a=10 x .n=8 updates of 2 elements of .b + // .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .n = 8, .b = 2 } }, + // // Same but with update transposed + // .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .b = 2, .n = 8 } }, + // // similar, but use the normalized form where a is no longer an explicit batching axis. + // .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .a2 = 10, .n = 8 }), .b = idx(.{ .a2 = 10, .n = 8 }) }, .{ .a2 = 10, .n = 8, .b = 2 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .a = 10, .n = 8 }), .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .n = 8, .b = 2 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8, .a = 2 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .n = 8 }), .a = idx(.{ .n = 8 }) }, .{ .n = 8, .a = 3, .b = 2 } }, + // .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }), .b = idx(.{ .n = 8 }) }, .{ .a = 3, .n = 8, .b = 2 } }, + // }) |testcase| { + // const x_shape, const indices, const updates_shapes = testcase; + // const x = Tensor.constant(x_shape, .{ .f16 = 0 }); + // const updates = Tensor.constant(updates_shapes, .{ .f16 = 0 }); + + // const y = scatterSlices(x, indices, updates, .{}); + // // Shape doesn't change with scatterSlices + // try zml.testing.expectEqualShapes(x.shape(), y.shape()); + // try std.testing.expect(y.value().owner().verify()); + // } + // } + // // Test with actual values, no batching. + // { + // const a_host = try zml.HostBuffer.arange(std.testing.allocator, .{ .end = 9 }, .i32); + // const a = (try zml.Buffer.from(platform, a_host.reshape(.{ 3, 3 }), .{})).withTags(.{ .a, .b }); + // defer a.deinit(); + // a_host.deinit(std.testing.allocator); + + // const scatter_indices = try zml.Buffer.fromArray(platform, [2]i32{ 0, 2 }); + // const updates = try zml.Buffer.fromArray(platform, [2][3]i32{ .{ 10, 20, 30 }, .{ 70, 80, 90 } }); + + // const expected = [3][3]i32{ .{ 10, 21, 32 }, .{ 3, 4, 5 }, .{ 76, 87, 98 } }; + // const result = try zml.testing.compileAndCall(platform, Local._scatter, .{ + // a, + // &.{scatter_indices.withTags(.{.n})}, + // updates.withTags(.{ .n, .b }), + // }); + // try std.testing.expect(a.shape().eql(result.shape())); + // try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected))); + // } + // // Test with setting individual values (no batching) + // { + // const a_host = try zml.HostBuffer.arange(std.testing.allocator, .{ .end = 9 }, .i32); + // const a = try zml.Buffer.from(platform, a_host, .{}); + // defer a.deinit(); + // a_host.deinit(std.testing.allocator); + + // const scatter_indices = try zml.Buffer.fromArray(platform, [2]i32{ 2, 7 }); + // const updates = try zml.Buffer.fromArray(platform, [2]i32{ 20, 70 }); + + // const expected = [9]i32{ 0, 1, 22, 3, 4, 5, 6, 77, 8 }; + // const result = try zml.testing.compileAndCall(platform, Local._scatter, .{ + // a, + // &.{scatter_indices.withTags(.{.n})}, + // updates.withTags(.{.n}), + // }); + // try std.testing.expect(a.shape().eql(result.shape())); + // try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected))); + // } + // { + // // Test with actual values and batching along axis .a + // const operand = try zml.Buffer.constant(platform, Shape.init(.{ .a = 2, .b = 3, .c = 4, .d = 2 }, .u16), 0, .{}); + // defer operand.deinit(); + // const start_indices = (try zml.Buffer.fromArray( + // platform, + // [2][2][3][2]i32{ + // .{ + // .{ .{ 0, 0 }, .{ 1, 0 }, .{ 2, 1 } }, + // .{ .{ 0, 1 }, .{ 1, 1 }, .{ 0, 9 } }, + // }, + // .{ + // .{ .{ 0, 0 }, .{ 2, 1 }, .{ 2, 2 } }, + // .{ .{ 1, 2 }, .{ 0, 1 }, .{ 1, 0 } }, + // }, + // }, + // )).withTags(.{ .n, .a, .m, .coord }); + // defer start_indices.deinit(); + + // const values = try zml.Buffer.constant( + // platform, + // Shape.init(.{ .n = 2, .a = 2, .m = 3, .c = 2, .d = 2 }, .u16), + // 1, + // .{}, + // ); + // defer values.deinit(); + + // const result = try zml.testing.compileAndCall(platform, Local._scatterCB, .{ operand, start_indices, values }); + + // const expected = [2][3][4][2]u16{ + // .{ + // .{ .{ 2, 2 }, .{ 3, 3 }, .{ 1, 1 }, .{ 0, 0 } }, + // .{ .{ 0, 0 }, .{ 0, 0 }, .{ 2, 2 }, .{ 2, 2 } }, + // .{ .{ 0, 0 }, .{ 0, 0 }, .{ 1, 1 }, .{ 1, 1 } }, + // }, + // .{ + // .{ .{ 0, 0 }, .{ 1, 1 }, .{ 1, 1 }, .{ 0, 0 } }, + // .{ .{ 2, 2 }, .{ 3, 3 }, .{ 1, 1 }, .{ 0, 0 } }, + // .{ .{ 0, 0 }, .{ 1, 1 }, .{ 1, 1 }, .{ 0, 0 } }, + // }, + // }; + // try std.testing.expect(operand.shape().eql(result.shape())); + // try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected))); + // } + //} + + /// Returns a Tensor containing the maximum over a given axis. + pub fn max(self: Tensor, axis_: anytype) Tensor { + const a = self.axis(axis_); + return reduce( + .{self}, + .{Tensor.constant(self.dtype().minValue())}, + &.{a}, + struct { + pub fn cmp(args: ReduceArgs) struct { Tensor } { + return .{args.left.maximum(args.right)}; + } + }, + .{}, + )[0]; + } + + /// Returns a Tensor containing the minimum over a given axis. + pub fn min(self: Tensor, axis_: anytype) Tensor { + const a = self.axis(axis_); + return reduce( + .{self}, + .{Tensor.constant(self.dtype().maxValue())}, + &.{a}, + struct { + pub fn cmp(args: ReduceArgs) struct { Tensor } { + return .{args.left.minimum(args.right)}; + } + }, + .{}, + )[0]; + } + + pub const ArgMaxRes = struct { + values: Tensor, + indices: Tensor, + }; + + pub fn argMax(x: Tensor, axis_: anytype) ArgMaxRes { + const a = x.axis(axis_); + const dt: DataType = if (x.dim(a) <= std.math.maxInt(i32)) .i32 else .i64; + + const values, const indices = reduce( + .{ x, Tensor.arange(.{ .end = x.dim(axis_) }, dt).broadcast(x.shape(), &.{a}) }, + .{ Tensor.constant(x.dtype().minValue()), Tensor.constant(dt.zero()) }, + &.{a}, + struct { + fn cmp(values: ReduceArgs, indices: ReduceArgs) struct { Tensor, Tensor } { + const left_gt_right = values.left.cmp(.GT, values.right); + const is_nan = values.left.cmp(.NE, values.left); + const left_gt_or_nan = left_gt_right.logical(.OR, is_nan); + // we are bubbling up Nan. + const max_val = left_gt_or_nan.select(values.left, values.right); + + // If values.left == values.right: keep the smallest idx. + const is_same = values.left.cmp(.EQ, values.right); + const is_first = indices.left.cmp(.LT, indices.right); + const is_same_but_first = is_same.logical(.AND, is_first); + const keep_left_idx = left_gt_or_nan.logical(.OR, is_same_but_first); + const max_idx = keep_left_idx.select(indices.left, indices.right); + + return .{ max_val, max_idx }; + } + }.cmp, + .{}, + ); + return .{ .values = values, .indices = indices }; + } + + // TODO(Corentin) + //test argMax { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + // const allocator = std.testing.allocator; + // const ArgMaxTest = struct { + // pub fn _fwd(x: Tensor) Tensor.ArgMaxRes { + // return x.argMax(1); + // } + // }; + + // const argmax = try zml.compileFn(allocator, ArgMaxTest._fwd, .{Shape.init(.{ 1, 5 }, .f32)}, platform); + // defer argmax.deinit(); + // // Test with tie + // { + // const x = try zml.Buffer.fromArray(platform, [1][5]f32{.{ 5.0, 4.1, 7.9, 0, 7.9 }}); + // const res = argmax.call(.{x}); + // const max_ = res.values.getValue(f32); + // const max_idx = res.indices.getValue(i32); + // try std.testing.expectEqual(max_, 7.9); + // // We should always return the first max found. + // try std.testing.expectEqual(max_idx, 2); + // } + + // // Test with Nan + // { + // const x = try zml.Buffer.fromArray(platform, [1][5]f32{.{ 5.0, std.math.nan(f32), 7.9, 0, 7.9 }}); + // const res = argmax.call(.{x}); + // const max_ = try res.values.getValue(f32); + // const max_idx = try res.indices.getValue(i32); + // try std.testing.expect(std.math.isNan(max_)); + // try std.testing.expectEqual(max_idx, 1); + // } + //} + + pub const ReduceArgs = struct { + left: Tensor, + right: Tensor, + }; + + fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const reduce_block, var result = b: { + const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inits.len); + var args: ArgsType = undefined; + var block_types: [2 * inits.len]*const mlir.Type = undefined; + + inline for (0..inits.len) |i| { + args[i].left = Tensor.init(inits[i].shape()); + args[i].right = Tensor.init(inits[i].shape()); + + block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].left.dtype())); + block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].right.dtype())); + } + + const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlirCtx())); + const reduce_block = mlir.Block.init(&block_types, &block_locs); + errdefer reduce_block.deinit(); + + CompilationContext.current().pushBlock(reduce_block); + defer CompilationContext.current().popBlock(); + + const scope = CompilationContext.current().currentScope(); + inline for (0..inits.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inits.len) catch unreachable; + } + + var result = @call(.auto, func, args ++ context); + + var result_values: [inits.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| { + result_values[i] = result[i].value(); + } + + _ = dialects.stablehlo.returns(mlirCtx(), &result_values, .unknown(mlirCtx())).appendTo(reduce_block); + break :b .{ reduce_block, result }; + }; + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); + } + + var init_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| init_values[i] = inits[i].value(); + + const reduce_op = mlir.Operation.make(mlirCtx(), "stablehlo.reduce", .{ + .operands = .{ .variadic = &.{ &input_values, &init_values } }, + .result_type_inference = true, + .blocks = &.{reduce_block}, + .attributes = &.{ + .named(mlirCtx(), "dimensions", mlir.denseArrayAttribute(mlirCtx(), .i64, axes_)), + }, + .verify = true, + .location = .unknown(mlirCtx()), + }).appendTo(currentBlock()); + + // `stablehlo.reduce` drops axes. We want to avoid that to propagate tags. + // So we need to broadcast the output of `stablehlo.reduce` to the input shapes. + // To that order, we initialize `result` to `inputs`, then we use stdx.meta.visit, + // to find the correct mlir.Value, but we first broadcast before creating the final + // Tensor struct. + var broadcasting_axes: stdx.BoundedArray(i64, Tensor.MAX_RANK) = .{}; + for (0..Tensor.MAX_RANK) |i| { + if (std.mem.indexOfScalar(i64, axes_, @intCast(i)) == null) { + broadcasting_axes.append(@intCast(i)) catch unreachable; + } + } + + inline for (0..result.len) |i| { + var reduced_shape: Shape = inputs[i].shape(); + for (axes_) |a| { + reduced_shape = reduced_shape.setDim(a, 1); + } + + const tensor_type = mlir.rankedTensorType(reduced_shape.dims(), mlirx.Type.fromDType(mlirCtx(), reduced_shape.dtype())); + const broad_op = dialects.stablehlo.broadcast_in_dim( + mlirCtx(), + reduce_op.result(i), + broadcasting_axes.slice()[0 .. reduced_shape.rank() - axes_.len], + tensor_type, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + result[i] = Tensor._result(reduced_shape, broad_op.result(0)); + } + + return result; + } + + pub const SortRes = ArgMaxRes; + + pub const SortArgs = struct { + left: Tensor, + right: Tensor, + }; + + pub const SortOpts = struct { descending: bool = false }; + + /// Returns two Tensors. The first contains the sorted values and the second one contains the sorted indices. + pub fn sort(self: Tensor, axis_: anytype, opts: SortOpts) SortRes { + const a = self.axis(axis_); + const indices = Tensor.arange(.{ .end = self.dim(a) }, .i32).broadcast(self._shape, &.{a}); + const res = sort_(.{ self, indices }, a, struct { + fn call(direction: dialects.stablehlo.ComparisonDirection.Direction, values: SortArgs, _: SortArgs) Tensor { + return values.left.cmp(direction, values.right); + } + }.call, .{if (opts.descending) .GT else .LT}, true); + return .{ .values = res[0], .indices = res[1] }; + } + + fn sort_(inputs: anytype, axis_: i64, comptime func: anytype, context: anytype, is_stable: bool) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const sort_block, var result = b: { + const ArgsType = std.meta.Tuple(&[1]type{SortArgs} ** inputs.len); + var args: ArgsType = undefined; + var block_types: [2 * inputs.len]*const mlir.Type = undefined; + + inline for (0..inputs.len) |i| { + args[i].left = Tensor.init(Shape.init(.{}, inputs[i].shape().dtype())); + args[i].right = Tensor.init(Shape.init(.{}, inputs[i].shape().dtype())); + + block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].left.dtype())); + block_types[i + inputs.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].right.dtype())); + } + + const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlirCtx())); + const sort_block = mlir.Block.init(&block_types, &block_locs); + errdefer sort_block.deinit(); + + CompilationContext.current().pushBlock(sort_block); + defer CompilationContext.current().popBlock(); + + const scope = CompilationContext.current().currentScope(); + inline for (0..inputs.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inputs.len) catch unreachable; + } + + var result = @call(.auto, func, args ++ context); + + var result_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + result_values[i] = result[i].value(); + } + + _ = dialects.stablehlo.returns(mlirCtx(), &result_values, .unknown(mlirCtx())).appendTo(sort_block); + break :b .{ sort_block, result }; + }; + + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); + } + + const sort_op = mlir.Operation.make(mlirCtx(), "stablehlo.sort", .{ + .operands = .{ .variadic = &.{&input_values} }, + .result_type_inference = true, + .blocks = &.{sort_block}, + .attributes = &.{ + .named(mlirCtx(), "dimension", mlir.integerAttribute(mlirCtx(), .i64, axis_)), + .named(mlirCtx(), "is_stable", mlir.boolAttribute(mlirCtx(), is_stable)), + }, + .verify = true, + .location = .unknown(mlirCtx()), + }).appendTo(currentBlock()); + + inline for (0..result.len) |i| { + result[i] = Tensor._result(inputs[i].shape(), sort_op.result(i)); + } + + return result; + } + + pub const ArgSortOpts = SortOpts; + + /// Returns a Tensor containing the indices corresponding to the sorted values over the given axis. + pub fn argsort(self: Tensor, axis_: anytype, opts: ArgSortOpts) Tensor { + return self.sort(axis_, .{ .descending = opts.descending }).indices; + } + + // TODO(Corentin) + //test argsort { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const Local = struct { + // pub fn _argsort(x: Tensor, axis_: u3, opts: ArgSortOpts) Tensor { + // return x.argsort(axis_, opts); + // } + // }; + + // var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + // defer arena_state.deinit(); + // const allocator = arena_state.allocator(); + // // 2D Tensor - dim = 1, ascending + // { + // const x = try zml.Buffer.fromSlice(platform, .{ 2, 5 }, &[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 }); + // const res = try zml.testing.compileAndCall(platform, Local._argsort, .{ x, 1, .{} }); + // const res_cpu = try res.toHostAlloc(allocator); + // try std.testing.expectEqualSlices(i32, &.{ 0, 3, 1, 2, 4, 1, 4, 3, 0, 2 }, res_cpu.items(i32)); + // } + // // 3D Tensor, dim = 1, descending + // { + // const x = try zml.Buffer.fromSlice(platform, .{ 1, 5, 10 }, &[_]f16{ + // -0.2505, 1.2520, -0.7041, 0.1066, 1.2773, -1.7246, 0.8389, 1.1094, 0.0601, 1.0684, + // 0.9619, 1.3916, 1.2246, -0.1406, 0.3674, -1.2480, -1.7051, -0.0934, 0.3435, 0.4373, + // 1.3809, 0.5444, -0.6079, 1.2031, -0.6880, 1.2979, -0.1869, 0.2991, 0.0156, 0.1847, + // 0.6626, -0.3040, -0.8726, -1.4805, -1.6943, 1.1055, -2.0078, -0.5288, 0.8813, 0.8008, + // 2.0527, 1.1230, 0.5430, 0.2494, -0.9434, 0.7876, 0.1818, 0.9258, -2.4902, 1.5918, + // }); + // const res_dev = try zml.testing.compileAndCall(platform, Local._argsort, .{ x, 1, .{ .descending = true } }); + // const res = try res_dev.toHostAlloc(allocator); + // try std.testing.expectEqualSlices(i32, &.{ + // 4, 1, 1, 2, 0, 2, 0, 0, 3, 4, + // 2, 0, 4, 4, 1, 3, 4, 4, 1, 0, + // 1, 4, 2, 0, 2, 4, 2, 2, 0, 3, + // 3, 2, 0, 1, 4, 1, 1, 1, 2, 1, + // 0, 3, 3, 3, 3, 0, 3, 3, 4, 2, + // }, res.items(i32)); + // } + // // 4D Tensor, dim = 3, ascending + // { + // const x = try zml.Buffer.fromSlice(platform, .{ 4, 2, 1, 4 }, &[_]i32{ + // 89, 31, 22, 42, + // 64, 39, 0, 30, + // 64, 71, 46, 31, + // 89, 82, 78, 86, + // 55, 32, 43, 19, + // 93, 24, 45, 72, + // 64, 86, 62, 88, + // 57, 21, 19, 12, + // }); + // const res_dev = try zml.testing.compileAndCall(platform, Local._argsort, .{ x, 3, .{} }); + // const res = try res_dev.toHostAlloc(allocator); + // try std.testing.expectEqualSlices(i32, &.{ + // 2, 1, 3, 0, + // 2, 3, 1, 0, + // 3, 2, 0, 1, + // 2, 1, 3, 0, + // 3, 1, 2, 0, + // 1, 2, 3, 0, + // 2, 0, 1, 3, + // 3, 2, 1, 0, + // }, res.items(i32)); + // } + //} + + pub const TopKOpts = SortOpts; + + /// Returns a Tensor representing the result of Top-K over the given axis. + pub fn topK(self: Tensor, named_axis_: anytype, k: u32, opts: TopKOpts) SortRes { + stdx.debug.assert(k > 0, "topK expects a k > 0, got 0", .{}); + const err_msg = "topK named axis should be an integer or a named axis, eg `x.topK(.{{ .best_token = .token }}, 16)` or `x.topK(-1, 16)`"; + const has_name: ?[:0]const u8, const a = switch (@typeInfo(@TypeOf(named_axis_))) { + .int, .comptime_int => .{ null, self.axis(@as(i64, @intCast(named_axis_))) }, + .@"struct" => |info| blk: { + stdx.debug.assertComptime(info.fields.len == 1, err_msg, .{}); + break :blk .{ info.fields[0].name, self.axis(@field(named_axis_, info.fields[0].name)) }; + }, + else => stdx.debug.compileError(err_msg, .{}), + }; + var result = self.sort(a, opts); + result.values = result.values.slice1d(a, .{ .end = k }); + result.indices = result.indices.slice1d(a, .{ .end = k }); + if (has_name) |new_name| { + result.values._shape._tags.set(a, new_name.ptr); + result.indices._shape._tags.set(a, new_name.ptr); + } + return result; + } + + // TODO(Corentin) + // + //pub const MaxPoolRes = ArgMaxRes; + /// Computes the 1d maxPool operation on the input Tensor. + //pub fn maxPool1d(self: Tensor, opts: struct { + // window_dimensions: i64, + // window_strides: ?i64, + // base_dilations: i64 = 1, + // window_dilations: i64 = 1, + // padding: [2]i64 = .{ 0, 0 }, + //}) MaxPoolRes { + // // TODO migrate to the following syntax. + // // maxPool(.{.a = .{ .stride = 5, .dilation = 2, .padding = .{0, 1} }, + // // .b = .{ .stride = 8, .dilation = 2, .padding = .{0, 1} }), + // // maxPool(.{ + // // .stride = .{ .a = 5, .b = 8 }, + // // .dilation = .{ .a = 2, .b = 2 }, + // // .padding = .{ .a = .{ 0, 2 }, .b = .{0, 2} + // // }) + + // // TODO: support maxPool on non last axis + // const a = self.axis(-1); + // const ones = [_]i64{1} ** Tensor.MAX_RANK; + // var window_dimensions = ones; + // window_dimensions[a] = opts.window_dimensions; + // var window_strides = window_dimensions; + // if (opts.window_strides) |stride| window_strides[a] = stride; + + // var base_dilations = ones; + // base_dilations[a] = opts.base_dilations; + // var window_dilations = ones; + // window_dilations[a] = opts.window_dilations; + + // var padding = [_][2]i64{.{ 0, 0 }} ** Tensor.MAX_RANK; + // padding[a] = opts.padding; + + // return ops.reduceWindow( + // MaxPoolRes.cmp, + // .{ .values = self, .indices = iota(self._shape, a) }, + // .{ .values = Tensor.constant(.{}, self.dtype().minValue()), .indices = Tensor.scalar(0, .i32) }, + // .{ + // .window_dimensions = window_dimensions[0..self.rank()], + // .window_strides = window_strides[0..self.rank()], + // .base_dilations = base_dilations[0..self.rank()], + // .window_dilations = window_dilations[0..self.rank()], + // .padding = padding[0..self.rank()], + // }, + // ); + //} + + ///// Computes the 2d maxPool operation on the input Tensor. + //pub fn maxPool2d(self: Tensor, opts: struct { + // window_dimensions: [2]i64, + // window_strides: ?[2]i64 = null, + // base_dilations: [2]i64 = .{ 1, 1 }, + // window_dilations: [2]i64 = .{ 1, 1 }, + // padding: [2][2]i64 = .{ .{ 0, 0 }, .{ 0, 0 } }, + //}) MaxPoolRes { + // // TODO: rewrite using modern ZML + // stdx.debug.guard(self.rank() == 3 or self.rank() == 4, @src()); + + // // TODO: support maxPool on non last axis + // // Note: the problem is initPoolArg assuming last axis + // const a = self.axis(-1); + + // const window_dimensions = initPoolArg(self.rank(), &opts.window_dimensions); + // const window_strides = if (opts.window_strides) |ws| initPoolArg(self.rank(), &ws) else window_dimensions; + // const base_dilation = initPoolArg(self.rank(), &opts.base_dilations); + // const window_dilations = initPoolArg(self.rank(), &opts.window_dilations); + + // var padding = [_][2]i64{.{ 0, 0 }} ** Tensor.MAX_RANK; + // padding[a - 1] = opts.padding[0]; + // padding[a] = opts.padding[1]; + + // return ops.reduceWindow( + // MaxPoolRes.cmp, + // .{ .values = self, .indices = iota(self._shape, a) }, + // .{ .values = Tensor.constant(.{}, self.dtype().minValue()), .indices = Tensor.scalar(0, .i32) }, + // .{ + // .window_dimensions = window_dimensions[0..self.rank()], + // .window_strides = window_strides[0..self.rank()], + // .base_dilations = base_dilation[0..self.rank()], + // .window_dilations = window_dilations[0..self.rank()], + // .padding = padding[0..self.rank()], + // }, + // ); + //} + + /// Chunk a given tensor into exactly n parts of equal shape. + /// `self.dim(axis_)` must be divisible by n_chunks. + pub fn chunkExact(self: Tensor, axis_: anytype, n_chunks: comptime_int) [n_chunks]Tensor { + const a = self.axis(axis_); + const d = self.dim(a); + const chunk_size = @divExact(d, n_chunks); + var chunks: [n_chunks]Tensor = undefined; + for (0..n_chunks) |i| { + const start: i64 = @as(i64, @intCast(i)) * chunk_size; + chunks[i] = self.slice1d(a, .{ .start = start, .end = start + chunk_size }); + } + return chunks; + } + + // TODO(Corentin) + //test chunkExact { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // // Only test shapes + // var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + // defer comp.deinit(); + // comp.activate(); + // defer comp.deactivate(); + + // inline for (.{ + // .{ .{ .a = 12 }, .a, 3, .{ .a = 4 } }, + // .{ .{ .a = 12, .b = 2 }, .a, 3, .{ .a = 4, .b = 2 } }, + // .{ .{ 12, 2 }, 0, 3, .{ 4, 2 } }, + // }) |testcase| { + // const x_shape, const ax, const n_chunks, const res = testcase; + // const x = Tensor.constant(x_shape, .{ .f16 = 0 }); + // const chunks = x.chunkExact(ax, n_chunks); + + // const res_shape = Shape.init(res, .f16); + // for (chunks) |chk| { + // try zml.testing.expectEqualShapes(res_shape, chk.shape()); + // } + // } + //} + + /// Chunk a given tensor into n parts of equal shape, and one part with the remaining items. + /// When `self.dim(axis_)` is divisible by `n_chunks` it will return exactly `n_chunks`. + pub fn chunkAllowTrailing( + self: Tensor, + axis_: i64, + n_chunks: comptime_int, + ) []Tensor { + const a = self.axis(axis_); + const d = self.dim(a); + const chunk_size: i64 = @divFloor(d, n_chunks); + const tail_chunk_size: i64 = @rem(d, chunk_size); + + const allocator = self.getContext().allocator(); + var chunks = std.ArrayList(Tensor).initCapacity(allocator, n_chunks + 1) catch @panic("OOM"); + for (0..n_chunks) |i| { + const start: i64 = @as(i64, @intCast(i)) * chunk_size; + chunks.appendAssumeCapacity( + self.slice1d(a, .{ .start = start, .end = start + chunk_size }), + ); + } + if (tail_chunk_size != 0) { + const start: i64 = n_chunks * chunk_size; + chunks.appendAssumeCapacity(self.slice1d(a, .{ .start = start })); + } + return chunks.items; + } + + // TODO(Corentin) + //test chunkAllowTrailing { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // // Only test shapes + // var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + // defer comp.deinit(); + // comp.activate(); + // defer comp.deactivate(); + + // inline for (.{ + // .{ .{ .a = 10 }, .a, 3, .{ .a = 3 }, .{ .a = 1 } }, + // .{ .{ .a = 10, .b = 2 }, .a, 3, .{ .a = 3, .b = 2 }, .{ .a = 1, .b = 2 } }, + // .{ .{ 10, 2 }, 0, 3, .{ 3, 2 }, .{ 1, 2 } }, + // .{ .{ 12, 2 }, 0, 3, .{ 4, 2 }, .{} }, + // }) |testcase| { + // const x_shape, const ax, const n_chunks, const res, const trailing = testcase; + // const x = Tensor.constant(x_shape, .{ .f16 = 0 }); + // const chunks = x.chunkAllowTrailing(x.axis(ax), n_chunks); + + // const res_shape = Shape.init(res, .f16); + // for (chunks[0..n_chunks]) |chk| { + // try zml.testing.expectEqualShapes(res_shape, chk.shape()); + // } + // const trailing_shape = Shape.init(trailing, .f16); + // if (trailing_shape.rank() > 0) { + // try std.testing.expectEqual(n_chunks + 1, chunks.len); + // try zml.testing.expectEqualShapes(trailing_shape, chunks[n_chunks].shape()); + // } else { + // try std.testing.expectEqual(n_chunks, chunks.len); + // } + // } + //} + + pub fn split(self: Tensor, axis_: anytype, split_sizes: []const i64) []Tensor { + stdx.debug.assert(split_sizes.len > 0, "split expects at least one 'split_sizes', got 0", .{}); + + const a = self.axis(axis_); + const d = self.dim(a); + var split_sum: i64 = 0; + for (split_sizes) |n| split_sum += n; + stdx.debug.assert(split_sum == d, "split expects sum of 'split_sizes' values and axis dimension to be equal, got {} and {}", .{ split_sum, d }); + + const allocator = self.getContext().allocator(); + const res = allocator.alloc(Tensor, split_sizes.len) catch @panic("OOM"); + errdefer allocator.dealloc(res); + + var start: i64 = 0; + for (split_sizes, 0..) |n, i| { + res[i] = self.slice1d(a, .{ .start = start, .end = start + n }); + start += n; + } + return res; + } + + /// Slices the input Tensor along a specific axis, with a start offset known at runtime. + /// Note: this doesn't support tagging, if you have tags, + /// you should use `dynamicSlice` directly. + pub fn dynamicSlice1d(self: Tensor, axis_: i8, slice_: DynSlice) Tensor { + stdx.debug.assert(slice_.start.rank() == 0, "dynamicSlice1d expects 'slice_.start' tensor rank to be a scalar, got {f}", .{slice_.start}); + + const a = self.axis(axis_); + const new_shape = self._shape.set(a, slice_.len); + + var start_indices = [_]*const mlir.Value{constant(slice_.start.dtype().zero()).value()} ** MAX_RANK; + start_indices[a] = slice_.start.value(); + + const op = dialects.stablehlo.dynamic_slice( + mlirCtx(), + self.value(), + new_shape.dims(), + start_indices[0..self.rank()], + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + return _result(new_shape, op.result(0)); + } + + pub const DynSlice = struct { start: Tensor, len: i64 }; + + /// Slices a Tensor across many axes, with runtime known offsets. + /// + /// Due to the nature of stablehlo, the length of the slices need to be known when compiling the IR. + /// When using the tagged API it is allowed to not specify some axes. + /// But with the non-tagged API all slices need to be specified. + /// Examples: + /// ``` + /// Tensor(.{.a=20,.b=30,.c=40 }).dynamicSlice(.{ .a = .{ .start = a_off, .len = 11}); + /// Tensor(.{.a=20,.b=30,.c=40 }).dynamicSlice(.{ + /// .a = .{ .start = a_off, .len = 11 }, + /// .b = .{ .start = b_off, .len = 12 }, + /// }); + /// Tensor(.{ 20,30,40}).dynamicSlice(.{.{ .start = scalar(0, .i32), .len = 20 }, .{ .start = b_off, .len = 12 }, .{ .start = scalar(0, .i32), .len = 40 }}); + /// ``` + pub fn dynamicSlice(self: Tensor, slices_: anytype) Tensor { + // TODO: the untagged api is a bit verbose. Should I allow: `Tensor(.{ 20,30,40}).dynamicSlice(.{.{}, .{ .start = b_off, .len = 12 }, .{}});` ?? + // + const slices, const slices_tags = Shape.parseStruct(DynSlice, slices_); + + // TODO use slices and slices_tags for the format. + // Currently this prints: "dynSlice(struct{q: struct{start: tensor.Tensor, comptime len: comptime_int = 1}}{ .q = struct{start: tensor.Tensor, comptime len: comptime_int = 1}{ .start = Tensor({1,10}, dtype=.i64), .len = 1 } })" + // which is kinda ugly. + + const idx_dtype = if (slices.len > 0) slices.get(0).start.dtype() else .i32; + const zero = Tensor.scalar(0, idx_dtype).value(); + var offset_values = [_]*const mlir.Value{zero} ** MAX_RANK; + var res_shape = self._shape; + for (slices.constSlice(), 0..) |slice_, i| { + const offset = slice_.start; + const len = slice_.len; + if (slices_tags.len == 0) { + stdx.debug.assert(self.rank() == slices.len, "dynamicSlice expects tensor rank and 'slices_' length to be equal, got {d} and {d}", .{ self.rank(), slices.len }); + + offset_values[i] = offset.value(); + res_shape._dims.set(i, len); + + stdx.debug.assert(len <= self.dim(i), "dynamicSlice expects slices 'len' to be less than or equal to their corresponding dimension in input tensor, got {d} and {d} for index {d}", .{ len, self.dim(i), i }); + } else { + const t = slices_tags.get(i); + const a = res_shape.hasTag(t) orelse stdx.debug.panic("dynamicSlice expects input tensor to have tags used in 'slices_' but {s} is missing (input shape is {f})", .{ t, self._shape }); + + stdx.debug.assert(len <= self.dim(a), "dynamicSlice expects slices 'len' to be less than their corresponding dimension in input tensor, got {d} and {d} for axis {s}", .{ len, self.dim(a), t }); + + offset_values[a] = offset.value(); + res_shape._dims.set(a, len); + } + } + const op = dialects.stablehlo.dynamic_slice(mlirCtx(), self.value(), res_shape.dims(), offset_values[0..self.rank()], .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(res_shape, op.result(0)); + } + + // TODO(Corentin) + //test dynamicSlice { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + // const T = f32; + + // { + // const x = try zml.Buffer.fromArray(platform, [10]T{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); + // const z = try zml.Buffer.scalar(platform, 4, .i32); + // const res = try zml.testing.compileAndCall(platform, Tensor.dynamicSlice1d, .{ x, 0, .{ .len = 2, .start = z } }); + + // try std.testing.expectEqual([2]T{ 4, 5 }, try res.getValue([2]T)); + // } + + // { + // // Strided + // const x = try zml.Buffer.fromArray(platform, [2][5]T{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }); + // const z = try zml.Buffer.scalar(platform, 3, .i32); + + // const res = try zml.testing.compileAndCall(platform, Tensor.dynamicSlice1d, .{ x, 1, .{ .len = 2, .start = z } }); + // try std.testing.expectEqual([4]T{ 3, 4, 8, 9 }, res.getValue([4]T)); + // } + //} + + /// Updates a slice of the input Tensor along a specific axis using the given 'update' Tensor, with a start offset known at runtime. + /// Note this is the untagged api, if you have tags, you should use dynamicUpdateSlice directly. + pub fn dynamicUpdateSlice1d(self: Tensor, update: Tensor, axis_: i64, offset: Tensor) Tensor { + const placeholder = Tensor.scalar(0, .i32); + var start_indices = [_]Tensor{placeholder} ** MAX_RANK; + start_indices[self.axis(axis_)] = offset; + return self.dynamicUpdateSlice(start_indices[0..self.rank()], update); + } + + /// Updates a part of the input Tensor using the given 'update' Tensor, with runtime known offsets. + /// + /// The offsets are specified similarly to the dynamicSlice api. + /// It's semantically equivalent to: + /// self.dynamicSlice(offsets_) := update + /// Examples: + /// ``` + /// Tensor(.{ .a = 2, .b = 5 }).dynamicUpdateSlice(.{ .a = scalar(1, .i32) }, Tensor(.{ .b = 5 })); + /// ``` + pub fn dynamicUpdateSlice(self: Tensor, offset_: anytype, update_: Tensor) Tensor { + // TODO: add updateSlice for when the offset isn't dynamic + stdx.debug.assert(self.dtype() == update_.dtype(), "dynamicUpdateSlice expects input and 'update_' tensors to be of the same type, got {} and {}", .{ self.dtype(), update_.dtype() }); + + const offset, const offset_tags = Shape.parseStruct(Tensor, offset_); + // log.debug("offset: {any}, offset_tags: {any}", .{ offset, offset_tags }); + for (offset.constSlice(), 0..) |start_idx, i| { + stdx.debug.assert(start_idx.rank() == 0, "dynamicUpdateSlice expects 'offset_' tensor ranks to be equal to 0, got {} at index {}", .{ start_idx.rank(), i }); + } + + const tagged_api = update_._shape.isFullyTagged() and self._shape.isFullyTagged() and offset_tags.len > 0; + // When using tags, we can safely insert axis with a 1-dim. + // the offset into the inserted axis will need to be specified through indices. + var update = update_; + if (tagged_api) { + // Check that all update tags are known. + for (update._shape._tags.constSlice()) |t| { + stdx.debug.assert(self._shape.hasTag(t) != null, "dynamicUpdateSlice expects 'update_' tensor tags to be a subset of input tensor tags but {s} is missing (input shape is {f})", .{ t, self._shape }); + } + + var update_shape = self._shape; + var prev_ax: i8 = -1; + for (self._shape.tags(), 0..) |t, self_ax| { + if (update._shape.hasTag(t)) |up_ax| { + stdx.debug.assert(up_ax == prev_ax + 1, "dynamicUpdateSlice expects 'update_' and input tensor axis to have the same order, got {f} and {f}. (hint: you need to explicitly transpose 'update_')", .{ update_, self }); + + update_shape._dims.set(self_ax, update.dim(up_ax)); + prev_ax = up_ax; + } else { + update_shape._dims.set(self_ax, 1); + } + } + update = update.reshape(update_shape); + } + + stdx.debug.assert(self.rank() == update.rank(), "dynamicUpdateSlice expects input and computed update tensors to have the same rank, got {f} and {f}", .{ self, update }); + + for (self.dims(), update.dims(), 0..) |self_d, up_d, ax| { + const t = self._shape.debugTag(ax); + stdx.debug.assert(up_d <= self_d, "dynamicUpdateSlice expects 'update_' dimensions to be less than or equal to their corresponding dimension in input tensor, got {} and {} for axis .{s}", .{ up_d, self_d, t }); + + if (tagged_api and up_d < self_d) { + const axis_has_offset = std.mem.indexOfScalar(Shape.Tag, offset_tags.constSlice(), self._shape._tags.get(ax)) != null; + + stdx.debug.assert(axis_has_offset, "dynamicUpdateSlice expects 'update_' dimensions to be equal to their corresponding dimension in input tensor, got {} and {} for axis .{s} (hint: you need to provide an offset)", .{ up_d, self_d, t }); + } + } + + const idx_dtype = if (offset.len > 0) offset.get(0).dtype() else .i32; + const zero = Tensor.scalar(0, idx_dtype).value(); + var offset_values: [MAX_RANK]*const mlir.Value = undefined; + if (offset_tags.len == 0) { + // Without offset tags we need the same number of offset than rank. + stdx.debug.assert(self.rank() == offset.len, "dynamicUpdateSlice expects input tensor rank and 'offset_' length to be equal, got {} and {}", .{ self.rank(), offset.len }); + + for (offset.constSlice(), 0..) |idx, i| { + offset_values[i] = idx.value(); + } + } else { + // If an axis isn't specified, update the full slice. + // This is only allowed when using tagged sliced. + offset_values = .{zero} ** MAX_RANK; + for (offset.constSlice(), offset_tags.constSlice()) |start, t| { + const a = self._shape.hasTag(t) orelse stdx.debug.panic("dynamicUpdateSlice expects input tensor to have tags used in 'offset_' but {s} is missing (input shape is {f})", .{ t, self._shape }); + offset_values[a] = start.value(); + } + } + + const op = dialects.stablehlo.dynamic_update_slice( + mlirCtx(), + self.value(), + update.value(), + offset_values[0..self.rank()], + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + // TODO(Corentin) + //test dynamicUpdateSlice { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // { + // const x = try zml.Buffer.fromArray(platform, [10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); + // const y = try zml.Buffer.fromArray(platform, [2]f32{ -1, -1 }); + // const idx = try zml.Buffer.scalar(platform, 4, .i32); + // const res = try zml.testing.compileAndCall( + // platform, + // struct { + // pub fn _fwd(x_: Tensor, idx_: struct { a: Tensor }, y_: Tensor) Tensor { + // return x_.dynamicUpdateSlice(idx_, y_); + // } + // }._fwd, + // .{ x.withTags(.{.a}), .{ .a = idx }, y.withTags(.{.a}) }, + // ); + // try std.testing.expectEqual([10]f32{ 0, 1, 2, 3, -1, -1, 6, 7, 8, 9 }, try res.getValue([10]f32)); + // } + + // { + // // Updates 2D, tagged api + // const x = try zml.Buffer.fromArray(platform, [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }); + // const y = try zml.Buffer.fromArray(platform, [2]f32{ -1, -1 }); + // const idx = try zml.Buffer.scalar(platform, 3, .i32); + + // const res = try zml.testing.compileAndCall( + // platform, + // struct { + // pub fn _fwd(x_: Tensor, idx_: Tensor, y_: Tensor) Tensor { + // return x_.dynamicUpdateSlice(.{ .b = idx_ }, y_); + // } + // }._fwd, + // .{ x.withTags(.{ .a, .b }), idx, y.withTags(.{.a}) }, + // ); + // try std.testing.expectEqualDeep( + // [2][5]f32{ .{ 0, 1, 2, -1, 4 }, .{ 5, 6, 7, -1, 9 } }, + // try res.getValue([2][5]f32), + // ); + // } + + // { + // // Updates 2D slice, un-tagged api. Note that `y` needs to have a 1 dimension axis. + // const x = try zml.Buffer.fromArray(platform, [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }); + // const y = try zml.Buffer.fromArray(platform, [2][1]f32{ .{-1}, .{-1} }); + // const idx = try zml.Buffer.scalar(platform, 3, .i32); + // const res = try zml.testing.compileAndCall( + // platform, + // struct { + // pub fn _fwd(x_: Tensor, idx_: Tensor, y_: Tensor) Tensor { + // return x_.dynamicUpdateSlice(.{ zml.Tensor.scalar(0, .i32), idx_ }, y_); + // } + // }._fwd, + // .{ x, idx, y }, + // ); + // try std.testing.expectEqualDeep( + // [2][5]f32{ .{ 0, 1, 2, -1, 4 }, .{ 5, 6, 7, -1, 9 } }, + // res.getValue([2][5]f32), + // ); + // } + + // { + // // Updates 2D, partial update + // const x = try zml.Buffer.fromArray(platform, [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }); + // const y = try zml.Buffer.fromArray(platform, [1]f32{-1}); + // const idx_a = try zml.Buffer.scalar(platform, 1, .i32); + // const idx_b = try zml.Buffer.scalar(platform, 3, .i32); + // const res = try zml.testing.compileAndCall( + // platform, + // struct { + // pub fn _fwd(x_: Tensor, idx_: struct { a: Tensor, b: Tensor }, y_: Tensor) Tensor { + // return x_.dynamicUpdateSlice(idx_, y_); + // } + // }._fwd, + // .{ x.withTags(.{ .a, .b }), .{ .a = idx_a, .b = idx_b }, y.withTags(.{.a}) }, + // ); + // try std.testing.expectEqualDeep( + // [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, -1, 9 } }, + // res.getValue([2][5]f32), + // ); + // } + + // { + // // Updates 2D, partial update, un-tagged api. + // const x = try zml.Buffer.fromArray(platform, [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }); + // const y = try zml.Buffer.fromArray(platform, [1][1]f32{.{-1}}); + // const idx_a = try zml.Buffer.scalar(platform, 1, .i32); + // const idx_b = try zml.Buffer.scalar(platform, 3, .i32); + // const A = struct { + // pub fn _fwd(x_: Tensor, idx_: [2]Tensor, y_: Tensor) Tensor { + // return x_.dynamicUpdateSlice(&idx_, y_); + // } + // }; + // const res = try zml.testing.compileAndCall(platform, A._fwd, .{ x, .{ idx_a, idx_b }, y }); + // try std.testing.expectEqualDeep( + // [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, -1, 9 } }, + // res.getValue([2][5]f32), + // ); + // } + //} + + /// Returns a Tensor containing the element-wise result of the given 'cmp' comparison between the two input Tensors. + pub fn cmp(self: Tensor, direction: dialects.stablehlo.ComparisonDirection.Direction, other: Tensor) Tensor { + stdx.debug.assert(self.dtype() == other.dtype(), "cmp expects input tensors to be of the same type, got {t} and {t}", .{ self.dtype(), other.dtype() }); + + if (self.rank() == 0 and other.rank() != 0) return self.broadcast(other._shape, &.{}).cmp(direction, other); + if (self.rank() != 0 and other.rank() == 0) return self.cmp(direction, other.broadcast(self._shape, &.{})); + + stdx.debug.assert(self._shape.eql(other._shape), "cmp expects input tensor shapes to match, got {f} and {f}", .{ self._shape, other._shape }); + + const op = dialects.stablehlo.compare( + mlirCtx(), + self.value(), + other.value(), + dialects.stablehlo.ComparisonDirection.init(self.getContext().mlirCtx(), direction), + getComparisonType(mlirCtx(), self.dtype()), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + return _result(self._shape.withDtype(.bool), op.result(0)); + } + + /// For each vector in the input tensor, + /// creates a diagonal-matrix where diagonal values are set to the vector values. + pub fn toDiagonal(self: Tensor, axis_: anytype, new_tags: [2]@TypeOf(.enum_literal)) Tensor { + stdx.debug.assert(self.rank() < MAX_RANK - 1, "toDiagonal expects input up to {d} rank, got {f}", .{ MAX_RANK - 1, self }); + const a = self.axis(axis_); + const d = self.dim(a); + var res_shape = self._shape; + res_shape._dims.replaceRange(a, 1, &.{ d, d }) catch unreachable; + res_shape._tags.replaceRange(a, 1, &.{ @tagName(new_tags[0]), @tagName(new_tags[1]) }) catch unreachable; + + const values = self.insertAxes(a + 1, .{new_tags[1]}).broad(res_shape); + const zeros = Tensor.constant(res_shape, self.dtype().zero()); + + const x = Tensor.iota(res_shape, a); + const y = Tensor.iota(res_shape, a + 1); + var res = x.cmp(.EQ, y).select(values, zeros); + res._shape = res_shape; + return res; + } + + // TODO(Corentin) + //test toDiagonal { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const Local = struct { + // pub fn _toDiag(input: Tensor) Tensor { + // const res = input.toDiagonal(-1, .{ .x, .y }); + // std.debug.assert(res.dim(.x) == input.dim(-1)); + // std.debug.assert(res.dim(.y) == input.dim(-1)); + // return res; + // } + // }; + + // const x = try zml.Buffer.fromArray(platform, [2][2]u8{ .{ 1, 2 }, .{ 3, 4 } }); + // { + // const res = try zml.testing.compileAndCall(platform, Local._toDiag, .{x}); + // try std.testing.expectEqual( + // [2][2][2]u8{ .{ + // .{ 1, 0 }, + // .{ 0, 2 }, + // }, .{ + // .{ 3, 0 }, + // .{ 0, 4 }, + // } }, + // try res.getValue([2][2][2]u8), + // ); + // } + //} + + /// For each matrix specified by the two axes, returns the lower triangular part of it. + /// The other elements are set to 0. + /// Usage: `.{ .b = 32, .w = 20, .h = 20 }.triangular(.{ .w, .h}, 0);` + /// + /// * if `num_diagonals` is set to 0, the diagonal is not modified. + /// * if set to -1, the diagonal is set to 0 + /// * if set to n, the n "quasi diagonal" above the diagonal are conserved. + /// + /// To get the upper triangular part, swap the order of axes: + /// `.{ .b = 32, .w = 20, .h = 20 }.triangular(.{ .h, .w }, 0);` + pub fn triangular(self: Tensor, axes_: anytype, num_diagonals: i32) Tensor { + stdx.debug.assertComptime(stdx.meta.isTuple(@TypeOf(axes_)) and axes_.len == 2, "triangular expects exactly two axes to work on.", .{}); + const _axes = self.axes(axes_); + + const x = Tensor.iota(self.shape(), _axes.get(0)); + const y = Tensor.iota(self.shape(), _axes.get(1)); + + const zeros = Tensor.constant(self.shape(), self.dtype().zero()); + return x.addConstant(num_diagonals).cmp(.GE, y).select(self, zeros); + } + + // TODO(Corentin) + //test triangular { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + + // const Local = struct { + // pub fn _tri(input: Tensor, num_diagonals: i32) Tensor { + // return input.triangular(.{ -2, -1 }, num_diagonals); + // } + // }; + + // const x = try zml.Buffer.fromArray(platform, [3][3]u8{ + // .{ 1, 1, 1 }, + // .{ 1, 1, 1 }, + // .{ 1, 1, 1 }, + // }); + // { + // const res = try zml.testing.compileAndCall(platform, Local._tri, .{ x, 0 }); + // try std.testing.expectEqual( + // [3][3]u8{ + // .{ 1, 0, 0 }, + // .{ 1, 1, 0 }, + // .{ 1, 1, 1 }, + // }, + // try res.getValue([3][3]u8), + // ); + // } + // { + // const res = try zml.testing.compileAndCall(platform, Local._tri, .{ x, 1 }); + // try std.testing.expectEqual( + // [3][3]u8{ + // .{ 1, 1, 0 }, + // .{ 1, 1, 1 }, + // .{ 1, 1, 1 }, + // }, + // try res.getValue([3][3]u8), + // ); + // } + // { + // const res = try zml.testing.compileAndCall(platform, Local._tri, .{ x, -1 }); + // try std.testing.expectEqual( + // [3][3]u8{ + // .{ 0, 0, 0 }, + // .{ 1, 0, 0 }, + // .{ 1, 1, 0 }, + // }, + // try res.getValue([3][3]u8), + // ); + // } + //} + + /// For each element at index `i`, if `bool_tensor[i] == true`, `output[i] = on_true[i]` + /// otherwise, if `bool_tensor[i] == false`, `output[i] = on_false[i]` + pub fn select(bool_tensor: Tensor, on_true: Tensor, on_false: Tensor) Tensor { + stdx.debug.assert(bool_tensor.dtype() == .bool, "select expects input tensor type to be a boolean, got {}", .{bool_tensor.dtype()}); + stdx.debug.assert(on_true.dtype() == on_false.dtype(), "select expects 'on_true' and 'on_false' tensor types to be equal, got {} and {}", .{ on_true.dtype(), on_false.dtype() }); + + if (bool_tensor.rank() != 0 and on_true.rank() == 0) { + return bool_tensor.select(on_true.broad(bool_tensor.shape()), on_false); + } + if (bool_tensor.rank() != 0 and on_false.rank() == 0) { + return bool_tensor.select(on_true, on_false.broad(bool_tensor.shape())); + } + + stdx.debug.assert(bool_tensor._shape.eqlDims(on_true._shape), "select expects input tensor and 'on_true' tensor dimensions to match, got {f} and {f}", .{ bool_tensor._shape, on_true._shape }); + stdx.debug.assert(bool_tensor._shape.eqlDims(on_false._shape), "select expects input tensor and 'on_false' tensor dimensions to match, got {f} and {f}", .{ bool_tensor._shape, on_false._shape }); + + const op = dialects.stablehlo.select( + mlirCtx(), + bool_tensor.value(), + on_true.value(), + on_false.value(), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + return _result(on_true._shape, op.result(0)); + } + + /// Returns a Tensor containing the element-wise not logical operation of the input Tensor. + pub fn not(self: Tensor) Tensor { + const op = dialects.stablehlo.not(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); + } + + /// Returns a Tensor containing boolean indicating if there is a non-zero value over the given axis. + pub fn any(self: Tensor, axis_: anytype) Tensor { + const pred = self.cmp(.NE, Tensor.constant(self.dims(), self.dtype().zero())); + return reduce( + .{pred}, + .{Tensor.scalar(false, .bool)}, + &.{self.axis(axis_)}, + struct { + pub fn acc(args: ReduceArgs) Tensor { + return args.left.logical(.OR, args.right); + } + }.acc, + .{}, + )[0]; + } + + /// Returns a Tensor containing boolean indicating if there is a non-zero value over the given axis. + pub fn all(self: Tensor, axis_: anytype) Tensor { + const pred = if (self.dtype() == .bool) self else self.cmp(.NE, Tensor.scalar(0, self.dtype())); + return reduce( + .{pred}, + .{Tensor.scalar(true, .bool)}, + &.{self.axis(axis_)}, + struct { + pub fn acc(args: ReduceArgs) Tensor { + return args.left.logical(.AND, args.right); + } + }.acc, + .{}, + )[0]; + } + + /// Given a set of N vectors of lengths A, B, C, D, + /// returns N tensors of rank N, and shape (A, B, C, D). + /// For any coordinate (a, b, c, d), we have: + /// + /// - res[0][a, b, c, d] == A[a] + /// - res[1][a, b, c, d] == B[b] + /// - res[2][a, b, c, d] == C[c] + /// - res[3][a, b, c, d] == D[d] + /// + /// This is implemented with broadcasting, so typically it won't copy. + /// In Pytorch/Numpy this is know as `meshgrid` with "ij" mode. + /// See `zml.torch.meshgrid` for the "xy" mode. + pub fn cartesianProduct(comptime N: u3, vectors: [N]Tensor) [N]Tensor { + var out: @TypeOf(vectors) = undefined; + _cartesianProduct(&vectors, &out); + return out; + } + + fn _cartesianProduct(vectors: []const Tensor, out: []Tensor) void { + stdx.debug.assert(vectors.len >= 1, "cartesianProduct expects at least one input.", .{}); + stdx.debug.assert(vectors.len < Tensor.MAX_RANK, "cartesianProduct expects at most {} input vectors, received {} !", .{ Tensor.MAX_RANK - 1, vectors.len }); + for (vectors) |x| { + stdx.debug.assert(x.rank() <= 1, "cartesianProduct expects 0 or 1 rank input vectors. Got: {any}", .{vectors}); + stdx.debug.assert(vectors[0].dtype() == x.dtype(), "cartesianProduct expects input vectors to have all the same dtype. Got: {any}", .{vectors}); + } + + var res_shape = Shape.init(.{}, vectors[0].dtype()); + for (vectors) |x| { + if (x.rank() == 0) { + res_shape = res_shape.appendDim(1, null); + } else { + res_shape = res_shape.appendDim(x.dim(0), x.shape().tag(0)); + } + } + + for (out, vectors, 0..) |*o, x, i| { + o.* = x.broadcast(res_shape, &[1]i64{@intCast(i)}); + } + } + + // TODO(Corentin) + //test cartesianProduct { + // const zml = @import("zml.zig"); + // const client = zml.testing.env(); + + // const x = try zml.Buffer.fromSlice(client, .{6}, &[_]i32{ 0, 1, 2, 3, 4, 5 }); + // const y = try zml.Buffer.fromSlice(client, .{4}, &[_]i32{ 0, 1, 2, 3 }); + + // const Local = struct { + // pub fn _cartesianProduct2(a: Tensor, b: Tensor) [2]Tensor { + // return cartesianProduct(2, .{ a, b }); + // } + // }; + + // { + // const xs, const ys = try zml.testing.compileAndCall(client, Local._cartesianProduct2, .{ x, y }); + // try std.testing.expectEqualSlices(i64, &.{ 6, 4 }, xs.shape().dims()); + // try std.testing.expectEqualSlices(i64, &.{ 6, 4 }, ys.shape().dims()); + // try std.testing.expectEqualDeep( + // [6][4]i32{ + // .{ 0, 0, 0, 0 }, + // .{ 1, 1, 1, 1 }, + // .{ 2, 2, 2, 2 }, + // .{ 3, 3, 3, 3 }, + // .{ 4, 4, 4, 4 }, + // .{ 5, 5, 5, 5 }, + // }, + // try xs.getValue([6][4]i32), + // ); + // try std.testing.expectEqualDeep( + // [6][4]i32{ + // .{ 0, 1, 2, 3 }, + // .{ 0, 1, 2, 3 }, + // .{ 0, 1, 2, 3 }, + // .{ 0, 1, 2, 3 }, + // .{ 0, 1, 2, 3 }, + // .{ 0, 1, 2, 3 }, + // }, + // try ys.getValue([6][4]i32), + // ); + // } + //} + + /// Given a set of N vectors of lengths A, B, C, D, + /// returns 1 tensors of rank N+1, and shape (A, B, C, D, N). + /// For any coordinate (a, b, c, d), we have: + /// + /// - res[a, b, c, d] == (A[a], B[b], C[c], D[d]) + pub fn cartesianProductStacked(vectors: []const Tensor) Tensor { + var out = stdx.BoundedArray(Tensor, Tensor.MAX_RANK).init(vectors.len) catch unreachable; + _cartesianProduct(vectors, out.slice()); + + return Tensor.stack(out.constSlice(), .last, .coord); + } + + // TODO(Corentin) + //test cartesianProductStacked { + // const zml = @import("zml.zig"); + // const platform = zml.testing.env(); + // const x = try zml.Buffer.fromSlice(platform, .{6}, &[_]i32{ 0, 1, 2, 3, 4, 5 }); + // const y = try zml.Buffer.fromSlice(platform, .{4}, &[_]i32{ 0, 1, 2, 3 }); + + // const Local = struct { + // pub fn _fwd(a: Tensor, b: Tensor) Tensor { + // return cartesianProductStacked(&.{ a, b }); + // } + // }; + + // const z = try zml.testing.compileAndCall(platform, Local._fwd, .{ x, y }); + // try std.testing.expectEqualDeep( + // [6][4][2]i32{ + // .{ .{ 0, 0 }, .{ 0, 1 }, .{ 0, 2 }, .{ 0, 3 } }, + // .{ .{ 1, 0 }, .{ 1, 1 }, .{ 1, 2 }, .{ 1, 3 } }, + // .{ .{ 2, 0 }, .{ 2, 1 }, .{ 2, 2 }, .{ 2, 3 } }, + // .{ .{ 3, 0 }, .{ 3, 1 }, .{ 3, 2 }, .{ 3, 3 } }, + // .{ .{ 4, 0 }, .{ 4, 1 }, .{ 4, 2 }, .{ 4, 3 } }, + // .{ .{ 5, 0 }, .{ 5, 1 }, .{ 5, 2 }, .{ 5, 3 } }, + // }, + // try z.getValue([6][4][2]i32), + // ); + //} + + fn binaryOp( + op_name: []const u8, + op_fn: fn (mlir.Context, mlir.Value, mlir.Value, mlir.Location) mlir.Operation, + ) fn (Tensor, Tensor) Tensor { + return struct { + pub fn binaryOpHelper(self: Tensor, other: Tensor) Tensor { + stdx.debug.assert(self.dtype() == other.dtype(), "{s} expects tensor to be of same type, got {f} and {f}", .{ op_name, self, other }); + + if (self.rank() == 0 and other.rank() != 0) { + return binaryOpHelper(self.broad(other._shape), other); + } + + if (self.rank() != 0 and other.rank() == 0) { + return binaryOpHelper(self, other.broad(self._shape)); + } + + var other_ = other; + var same_shape = self._shape.eql(other._shape); + if (!same_shape and std.mem.eql(Shape.Tag, self._shape.tags(), other._shape.tags()) and other._shape.canBroadcastTo(self._shape)) { + // Only a restrictive version of broadcasting is allowed here, where all the tags matches already. + // Typical use case: `x.div(x.sum(.a))` + same_shape = true; + other_ = other.broad(self._shape); + } + + stdx.debug.assert(same_shape, "{s} expects tensor shapes to match, got {f} and {f}", .{ op_name, self._shape, other._shape }); + + const ret = @call(.auto, op_fn, .{ mlirCtx(), self.value(), other_.value(), .unknown(mlirCtx()) }).appendTo(currentBlock()); + return _result(self._shape, ret.result(0)); + } + }.binaryOpHelper; + } + + /// Insert code that will print the content of the given buffer at runtime. + /// Only for debug purpose, it inserts device to host synchronization + /// so it will slow down the program execution. + pub fn print(input: Tensor) Tensor { + _ = input; // autofix + // TODO(Corentin) + //return callback.call(callback.Print, .{input}, &.{input.shape()})[0]; + } + + fn mlirCtx() *mlir.Context { + return CompilationContext.current().mlir_ctx; + } + + fn currentBlock() *mlir.Block { + return CompilationContext.current().currentScope().block; + } +}; + +fn initPoolArg(rank: usize, data: []const i64) [Tensor.MAX_RANK]i64 { + // TODO use shape + var result = [_]i64{1} ** Tensor.MAX_RANK; + const start = rank - data.len; + @memcpy(result[start .. start + data.len], data); + return result; +} + +fn getPoolResDims(dt: DataType, in_dims: []const i64, base_dilations: @Vector(Tensor.MAX_RANK, i64), padding: []const i64, window_dimensions: @Vector(Tensor.MAX_RANK, i64), window_dilations: @Vector(Tensor.MAX_RANK, i64), window_strides: @Vector(Tensor.MAX_RANK, i64)) Shape { + // TODO use shape + var input_dims = [_]i64{1} ** Tensor.MAX_RANK; + @memcpy(input_dims[0..in_dims.len], in_dims); + + const input_dims_: @Vector(Tensor.MAX_RANK, i64) = input_dims; + const splat_one: @Vector(Tensor.MAX_RANK, i64) = @splat(1); + const dilated_input_shape: @Vector(Tensor.MAX_RANK, i64) = (input_dims_ - splat_one) * base_dilations + splat_one; + var pad_slice0: @Vector(Tensor.MAX_RANK, i64) = @splat(padding[0]); + var pad_slice1: @Vector(Tensor.MAX_RANK, i64) = @splat(padding[0]); + if (padding.len > 1) { + var idx: usize = 0; + while (idx < in_dims.len * 2) : (idx += 2) { + pad_slice0[idx / 2] = padding[idx]; + pad_slice1[idx / 2] = padding[idx + 1]; + } + } + const padded_input_shape: @Vector(Tensor.MAX_RANK, i64) = pad_slice0 + dilated_input_shape + pad_slice1; + const dilated_window_shape = (window_dimensions - splat_one) * window_dilations + splat_one; + const dims = @divFloor(padded_input_shape - dilated_window_shape, window_strides) + splat_one; + const dims_arr: [Tensor.MAX_RANK]i64 = @bitCast(dims); + return Shape.init(dims_arr[0..in_dims.len], dt); +} + +fn getComparisonType(ctx: mlir.Context, dtype: DataType) dialects.stablehlo.CompareType { + return dialects.stablehlo.CompareType.init(ctx, switch (dtype.class()) { + .bool => .UNSIGNED, + .integer => if (dtype.isSignedInt()) .SIGNED else .UNSIGNED, + .float => .FLOAT, + .complex => @panic("Can't compare complex numbers"), + }); +} + +// TODO(Corentin) +//test "Tensor.maxPool1d" { +// const zml = @import("zml.zig"); +// const platform = zml.testing.env(); +// +// const MaxPool = struct { +// pub fn _fwd(x: zml.Tensor) Tensor.ArgMaxRes { +// return x.maxPool1d(.{ +// .window_dimensions = 3, +// .window_strides = 2, +// }); +// } +// }; +// +// var data: [20]f32 = undefined; +// for (&data, 0..) |*v, i| v.* = @floatFromInt(i); +// +// const x = try zml.Buffer.fromSlice(platform, .{ 2, 2, 5 }, &data); +// const result = try zml.testing.compileAndCall(platform, MaxPool._fwd, .{x}); +// try zml.testing.expectEqualShapes(.init(.{ 2, 2, 2 }, .f32), result.values.shape()); +// try zml.testing.expectEqualShapes(.init(.{ 2, 2, 2 }, .i32), result.indices.shape()); +// try std.testing.expectEqualDeep( +// [2][2][2]f32{ +// .{ .{ 2, 4 }, .{ 7, 9 } }, +// .{ .{ 12, 14 }, .{ 17, 19 } }, +// }, +// result.values.getValue([2][2][2]f32), +// ); +//} +// +//test "Tensor.maxPool2d" { +// const zml = @import("zml.zig"); +// const platform = zml.testing.env(); +// +// const MaxPool = struct { +// pub fn _fwd(x: Tensor) Tensor.ArgMaxRes { +// return x.maxPool2d(.{ +// .window_dimensions = .{ 3, 2 }, +// .window_strides = .{ 2, 1 }, +// }); +// } +// }; +// +// var data: [100]f32 = undefined; +// for (&data, 0..) |*v, i| v.* = @floatFromInt(i); +// const x = try zml.Buffer.fromSlice(platform, .{ 2, 2, 5, 5 }, &data); +// +// const result = try zml.testing.compileAndCall(platform, MaxPool._fwd, .{x}); +// try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2, 4 }, .f32), result.values.shape()); +// try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2, 4 }, .i32), result.indices.shape()); +// var buffer: [2][2][2][4]f32 = undefined; +// _ = try result.values.toHost(std.mem.asBytes(&buffer)); +// try std.testing.expectEqualDeep( +// [2][2][2][4]f32{ +// .{ +// .{ .{ 11, 12, 13, 14 }, .{ 21, 22, 23, 24 } }, +// .{ .{ 36, 37, 38, 39 }, .{ 46, 47, 48, 49 } }, +// }, +// .{ +// .{ .{ 61, 62, 63, 64 }, .{ 71, 72, 73, 74 } }, +// .{ .{ 86, 87, 88, 89 }, .{ 96, 97, 98, 99 } }, +// }, +// }, +// buffer, +// ); +//} + +fn toI64(values: anytype) []i64 { + var res: [Tensor.MAX_RANK]i64 = undefined; + for (values, 0..) |val, i| res[i] = @intCast(val); + return res[0..values.len]; +} + +fn transposeIsJustAReshape(x: Shape, permutation: []const i64) bool { + var perm: stdx.BoundedArray(struct { u8, bool }, Tensor.MAX_RANK) = .{}; + // Don't rewrite on invalid inputs. + if (permutation.len > x.rank()) return false; + for (permutation) |ax| { + const squeezable = x.dim(ax) == 1; + perm.appendAssumeCapacity(.{ @intCast(ax), squeezable }); + } + + var effective_ax: u8 = 0; + for (0..perm.len) |i| { + const ax, const squeezable = perm.get(i); + if (squeezable) { + // Effectively squeeze this axis by decrementing axes coming after by 1. + for (i..perm.len) |j| { + if (perm.buffer[j][0] > ax) { + perm.buffer[j][0] -= 1; + } + } + continue; + } + + if (ax != effective_ax) return false; + effective_ax += 1; + } + + return true; +} + +test transposeIsJustAReshape { + try std.testing.expect(transposeIsJustAReshape(Shape.init(.{ 5, 1, 3 }, .i32), &.{ 0, 1, 2 })); + try std.testing.expect(transposeIsJustAReshape(Shape.init(.{ 5, 1, 3 }, .i32), &.{ 1, 0, 2 })); + try std.testing.expect(!transposeIsJustAReshape(Shape.init(.{ 5, 1, 3 }, .i32), &.{ 2, 1, 0 })); + try std.testing.expect(transposeIsJustAReshape(Shape.init(.{ 64, 8, 1, 128 }, .bf16), &.{ 0, 2, 1, 3 })); + try std.testing.expect(!transposeIsJustAReshape(Shape.init(.{ 64, 8, 155, 128 }, .bf16), &.{ 0, 2, 1, 3 })); + try std.testing.expect(transposeIsJustAReshape(Shape.init(.{ 64, 1, 1, 128 }, .bf16), &.{ 1, 2, 0, 3 })); + try std.testing.expect(!transposeIsJustAReshape(Shape.init(.{ .b = 1, .h = 10, .q = 155, .hd = 1 }, .f32), &.{ 0, 2, 1, 3 })); + try std.testing.expect(!transposeIsJustAReshape(Shape.init(.{ 1, 10, 155, 1 }, .f32), &.{ 0, 2, 3, 1 })); + try std.testing.expect(transposeIsJustAReshape(Shape.init(.{ 1, 10, 155, 1 }, .f32), &.{ 0, 1, 3, 2 })); +} + +test "unused tensor" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const Local = struct { + pub fn _fwd(x: Tensor) Tensor { + const y = x.addConstant(1); + _ = y; + return x; + } + }; + + const mod = try zml.compileFn(std.testing.allocator, Local._fwd, .{Shape.init(.{10}, .f32)}, platform); + defer mod.deinit(); +} From 216c3bcab5f40a4bea0fa1fdd0f370aaf65881a7 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 3 Dec 2025 11:15:10 +0100 Subject: [PATCH 073/881] Tackled some TODOs on operations --- zml2/tensor.zig | 298 ++++++++++++++++++++++++++++++------------------ 1 file changed, 187 insertions(+), 111 deletions(-) diff --git a/zml2/tensor.zig b/zml2/tensor.zig index c74955c0..ed9895bc 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -1245,22 +1245,16 @@ pub const Tensor = struct { return max_.add(log_sum_exp); } - // TODO(Corentin) /// Returns a Tensor containing the sum of elements over the given axis. /// Output shape is the input shape with the axis_ dim set to 1. - //pub fn sum(self: Tensor, axis_: anytype) Tensor { - // const a = self.axis(axis_); - // return ops.reduce( - // struct { - // pub fn acc(x: Tensor, res: Tensor) Tensor { - // return res.add(x.convert(res.dtype())); - // } - // }.acc, - // self, - // Tensor.scalar(0, self.dtype()), - // &.{a}, - // ); - //} + pub fn sum(self: Tensor, axis_: anytype) Tensor { + const a = self.axis(axis_); + return reduce(.{self}, .{Tensor.constant(self.dtype().zero())}, &.{a}, struct { + pub fn acc(args: ReduceArgs) Tensor { + return args.right.add(args.left.convert(args.right.dtype())); + } + }.acc, .{})[0]; + } /// Returns a Tensor containing the mean of elements over the given axis. /// Output shape is the input shape with the axis_ dim set to 1. @@ -1908,9 +1902,8 @@ pub const Tensor = struct { return _result(.init(&.{}, val.dtype()), op.result(0)).autoBroadcast(); } - // TODO(Corentin) pub fn zeroes(sh: Shape) Tensor { - return .constant(sh.dtype().zero()); + return .constant(sh.dtype().zero()).broad(sh); } // TODO(Corentin) @@ -3098,6 +3091,23 @@ pub const Tensor = struct { pub const ArgMaxRes = struct { values: Tensor, indices: Tensor, + + fn cmp(values: ReduceArgs, indices: ReduceArgs) struct { Tensor, Tensor } { + const left_gt_right = values.left.cmp(.GT, values.right); + const is_nan = values.left.cmp(.NE, values.left); + const left_gt_or_nan = left_gt_right.logical(.OR, is_nan); + // we are bubbling up Nan. + const max_val = left_gt_or_nan.select(values.left, values.right); + + // If values.left == values.right: keep the smallest idx. + const is_same = values.left.cmp(.EQ, values.right); + const is_first = indices.left.cmp(.LT, indices.right); + const is_same_but_first = is_same.logical(.AND, is_first); + const keep_left_idx = left_gt_or_nan.logical(.OR, is_same_but_first); + const max_idx = keep_left_idx.select(indices.left, indices.right); + + return .{ max_val, max_idx }; + } }; pub fn argMax(x: Tensor, axis_: anytype) ArgMaxRes { @@ -3108,24 +3118,7 @@ pub const Tensor = struct { .{ x, Tensor.arange(.{ .end = x.dim(axis_) }, dt).broadcast(x.shape(), &.{a}) }, .{ Tensor.constant(x.dtype().minValue()), Tensor.constant(dt.zero()) }, &.{a}, - struct { - fn cmp(values: ReduceArgs, indices: ReduceArgs) struct { Tensor, Tensor } { - const left_gt_right = values.left.cmp(.GT, values.right); - const is_nan = values.left.cmp(.NE, values.left); - const left_gt_or_nan = left_gt_right.logical(.OR, is_nan); - // we are bubbling up Nan. - const max_val = left_gt_or_nan.select(values.left, values.right); - - // If values.left == values.right: keep the smallest idx. - const is_same = values.left.cmp(.EQ, values.right); - const is_first = indices.left.cmp(.LT, indices.right); - const is_same_but_first = is_same.logical(.AND, is_first); - const keep_left_idx = left_gt_or_nan.logical(.OR, is_same_but_first); - const max_idx = keep_left_idx.select(indices.left, indices.right); - - return .{ max_val, max_idx }; - } - }.cmp, + ArgMaxRes.cmp, .{}, ); return .{ .values = values, .indices = indices }; @@ -3447,93 +3440,176 @@ pub const Tensor = struct { return result; } - // TODO(Corentin) - // - //pub const MaxPoolRes = ArgMaxRes; + pub const ReduceWindowOpts = struct { + // TODO replace with Shape + window_dimensions: []const i64, + window_strides: []const i64, + base_dilations: []const i64, + window_dilations: []const i64, + padding: []const [2]i64, + }; + + fn reduceWindow(inputs: anytype, inits: anytype, opts: ReduceWindowOpts, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const reduce_block, var result = b: { + const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inits.len); + var args: ArgsType = undefined; + var block_types: [2 * inits.len]*const mlir.Type = undefined; + + inline for (0..inits.len) |i| { + args[i].left = Tensor.init(inits[i].shape()); + args[i].right = Tensor.init(inits[i].shape()); + + block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].left.dtype())); + block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].right.dtype())); + } + + const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlirCtx())); + const reduce_block = mlir.Block.init(&block_types, &block_locs); + errdefer reduce_block.deinit(); + + CompilationContext.current().pushBlock(reduce_block); + defer CompilationContext.current().popBlock(); + + const scope = CompilationContext.current().currentScope(); + inline for (0..inits.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inits.len) catch unreachable; + } + + var result = @call(.auto, func, args ++ context); + + var result_values: [inits.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| { + result_values[i] = result[i].value(); + } + + _ = dialects.stablehlo.returns(mlirCtx(), &result_values, .unknown(mlirCtx())).appendTo(reduce_block); + break :b .{ reduce_block, result }; + }; + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); + } + + var init_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| init_values[i] = inits[i].value(); + + const reduce_op = mlir.Operation.make(mlirCtx(), "stablehlo.reduce_window", .{ + .operands = .{ .variadic = &.{ &input_values, &init_values } }, + .result_type_inference = true, + .blocks = &.{reduce_block}, + .attributes = &.{ + .named(mlirCtx(), "window_dimensions", mlir.denseArrayAttribute(mlirCtx(), .i64, opts.window_dimensions)), + .named(mlirCtx(), "window_strides", mlir.denseArrayAttribute(mlirCtx(), .i64, opts.window_strides)), + .named(mlirCtx(), "base_dilations", mlir.denseArrayAttribute(mlirCtx(), .i64, opts.base_dilations)), + .named(mlirCtx(), "window_dilations", mlir.denseArrayAttribute(mlirCtx(), .i64, opts.window_dilations)), + // Cast the [][2]i64 to []i64 (safe) + .named(mlirCtx(), "padding", mlir.denseElementsAttribute(mlir.RankedTensorType.get(&.{}, mlir.integerType(mlirCtx(), .i64), null).shaped(), @as([]const i64, @ptrCast(opts.padding)))), + }, + .verify = true, + .location = .unknown(mlirCtx()), + }).appendTo(currentBlock()); + + inline for (0..result.len) |i| { + result[i] = Tensor._result(inputs[i].shape(), reduce_op.result(i)); + } + + return result; + } + + pub const MaxPoolRes = ArgMaxRes; + /// Computes the 1d maxPool operation on the input Tensor. - //pub fn maxPool1d(self: Tensor, opts: struct { - // window_dimensions: i64, - // window_strides: ?i64, - // base_dilations: i64 = 1, - // window_dilations: i64 = 1, - // padding: [2]i64 = .{ 0, 0 }, - //}) MaxPoolRes { - // // TODO migrate to the following syntax. - // // maxPool(.{.a = .{ .stride = 5, .dilation = 2, .padding = .{0, 1} }, - // // .b = .{ .stride = 8, .dilation = 2, .padding = .{0, 1} }), - // // maxPool(.{ - // // .stride = .{ .a = 5, .b = 8 }, - // // .dilation = .{ .a = 2, .b = 2 }, - // // .padding = .{ .a = .{ 0, 2 }, .b = .{0, 2} - // // }) + pub fn maxPool1d(self: Tensor, opts: struct { + window_dimensions: i64, + window_strides: ?i64, + base_dilations: i64 = 1, + window_dilations: i64 = 1, + padding: [2]i64 = .{ 0, 0 }, + }) MaxPoolRes { + // TODO migrate to the following syntax. + // maxPool(.{.a = .{ .stride = 5, .dilation = 2, .padding = .{0, 1} }, + // .b = .{ .stride = 8, .dilation = 2, .padding = .{0, 1} }), + // maxPool(.{ + // .stride = .{ .a = 5, .b = 8 }, + // .dilation = .{ .a = 2, .b = 2 }, + // .padding = .{ .a = .{ 0, 2 }, .b = .{0, 2} + // }) - // // TODO: support maxPool on non last axis - // const a = self.axis(-1); - // const ones = [_]i64{1} ** Tensor.MAX_RANK; - // var window_dimensions = ones; - // window_dimensions[a] = opts.window_dimensions; - // var window_strides = window_dimensions; - // if (opts.window_strides) |stride| window_strides[a] = stride; + // TODO: support maxPool on non last axis + const a = self.axis(-1); + const ones = [_]i64{1} ** Tensor.MAX_RANK; + var window_dimensions = ones; + window_dimensions[a] = opts.window_dimensions; + var window_strides = window_dimensions; + if (opts.window_strides) |stride| window_strides[a] = stride; - // var base_dilations = ones; - // base_dilations[a] = opts.base_dilations; - // var window_dilations = ones; - // window_dilations[a] = opts.window_dilations; + var base_dilations = ones; + base_dilations[a] = opts.base_dilations; + var window_dilations = ones; + window_dilations[a] = opts.window_dilations; - // var padding = [_][2]i64{.{ 0, 0 }} ** Tensor.MAX_RANK; - // padding[a] = opts.padding; + var padding = [_][2]i64{.{ 0, 0 }} ** Tensor.MAX_RANK; + padding[a] = opts.padding; - // return ops.reduceWindow( - // MaxPoolRes.cmp, - // .{ .values = self, .indices = iota(self._shape, a) }, - // .{ .values = Tensor.constant(.{}, self.dtype().minValue()), .indices = Tensor.scalar(0, .i32) }, - // .{ - // .window_dimensions = window_dimensions[0..self.rank()], - // .window_strides = window_strides[0..self.rank()], - // .base_dilations = base_dilations[0..self.rank()], - // .window_dilations = window_dilations[0..self.rank()], - // .padding = padding[0..self.rank()], - // }, - // ); - //} + const result = reduceWindow( + .{ self, iota(self._shape, a) }, + .{ Tensor.constant(self.dtype().minValue()), Tensor.scalar(0, .i32) }, + .{ + .window_dimensions = window_dimensions[0..self.rank()], + .window_strides = window_strides[0..self.rank()], + .base_dilations = base_dilations[0..self.rank()], + .window_dilations = window_dilations[0..self.rank()], + .padding = padding[0..self.rank()], + }, + MaxPoolRes.cmp, + .{}, + ); + return .{ .values = result[0], .indices = result[1] }; + } - ///// Computes the 2d maxPool operation on the input Tensor. - //pub fn maxPool2d(self: Tensor, opts: struct { - // window_dimensions: [2]i64, - // window_strides: ?[2]i64 = null, - // base_dilations: [2]i64 = .{ 1, 1 }, - // window_dilations: [2]i64 = .{ 1, 1 }, - // padding: [2][2]i64 = .{ .{ 0, 0 }, .{ 0, 0 } }, - //}) MaxPoolRes { - // // TODO: rewrite using modern ZML - // stdx.debug.guard(self.rank() == 3 or self.rank() == 4, @src()); + /// Computes the 2d maxPool operation on the input Tensor. + pub fn maxPool2d(self: Tensor, opts: struct { + window_dimensions: [2]i64, + window_strides: ?[2]i64 = null, + base_dilations: [2]i64 = .{ 1, 1 }, + window_dilations: [2]i64 = .{ 1, 1 }, + padding: [2][2]i64 = .{ .{ 0, 0 }, .{ 0, 0 } }, + }) MaxPoolRes { + // TODO: rewrite using modern ZML + stdx.debug.guard(self.rank() == 3 or self.rank() == 4, @src()); - // // TODO: support maxPool on non last axis - // // Note: the problem is initPoolArg assuming last axis - // const a = self.axis(-1); + // TODO: support maxPool on non last axis + // Note: the problem is initPoolArg assuming last axis + const a = self.axis(-1); - // const window_dimensions = initPoolArg(self.rank(), &opts.window_dimensions); - // const window_strides = if (opts.window_strides) |ws| initPoolArg(self.rank(), &ws) else window_dimensions; - // const base_dilation = initPoolArg(self.rank(), &opts.base_dilations); - // const window_dilations = initPoolArg(self.rank(), &opts.window_dilations); + const window_dimensions = initPoolArg(self.rank(), &opts.window_dimensions); + const window_strides = if (opts.window_strides) |ws| initPoolArg(self.rank(), &ws) else window_dimensions; + const base_dilation = initPoolArg(self.rank(), &opts.base_dilations); + const window_dilations = initPoolArg(self.rank(), &opts.window_dilations); - // var padding = [_][2]i64{.{ 0, 0 }} ** Tensor.MAX_RANK; - // padding[a - 1] = opts.padding[0]; - // padding[a] = opts.padding[1]; + var padding = [_][2]i64{.{ 0, 0 }} ** Tensor.MAX_RANK; + padding[a - 1] = opts.padding[0]; + padding[a] = opts.padding[1]; - // return ops.reduceWindow( - // MaxPoolRes.cmp, - // .{ .values = self, .indices = iota(self._shape, a) }, - // .{ .values = Tensor.constant(.{}, self.dtype().minValue()), .indices = Tensor.scalar(0, .i32) }, - // .{ - // .window_dimensions = window_dimensions[0..self.rank()], - // .window_strides = window_strides[0..self.rank()], - // .base_dilations = base_dilation[0..self.rank()], - // .window_dilations = window_dilations[0..self.rank()], - // .padding = padding[0..self.rank()], - // }, - // ); - //} + const result = reduceWindow( + .{ self, iota(self._shape, a) }, + .{ Tensor.constant(self.dtype().minValue()), Tensor.scalar(0, .i32) }, + .{ + .window_dimensions = window_dimensions[0..self.rank()], + .window_strides = window_strides[0..self.rank()], + .base_dilations = base_dilation[0..self.rank()], + .window_dilations = window_dilations[0..self.rank()], + .padding = padding[0..self.rank()], + }, + MaxPoolRes.cmp, + .{}, + ); + return .{ .values = result[0], .indices = result[1] }; + } /// Chunk a given tensor into exactly n parts of equal shape. /// `self.dim(axis_)` must be divisible by n_chunks. From d1fae650a49a3a7d51db37b90bff2a2d090b7c8f Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 3 Dec 2025 17:27:55 +0100 Subject: [PATCH 074/881] Basic compilation support --- mlir/dialects/func.zig | 101 ++++++--- zml2/BUILD.bazel | 37 +++- zml2/exe.zig | 226 +++++++++++++++++++ zml2/main.zig | 31 +++ zml2/meta.zig | 30 ++- zml2/mlirx.zig | 63 +++--- zml2/module.zig | 487 +++++++++++++++++++++++++++++++++++++++++ zml2/pjrtx.zig | 12 +- zml2/platform.zig | 15 ++ zml2/shape.zig | 2 +- zml2/tensor.zig | 133 +++++++---- zml2/zml.zig | 19 +- 12 files changed, 1015 insertions(+), 141 deletions(-) create mode 100644 zml2/exe.zig create mode 100644 zml2/module.zig diff --git a/mlir/dialects/func.zig b/mlir/dialects/func.zig index 868ff3bc..b9f8b7f5 100644 --- a/mlir/dialects/func.zig +++ b/mlir/dialects/func.zig @@ -3,50 +3,91 @@ const std = @import("std"); const mlir = @import("mlir"); const stdx = @import("stdx"); -pub fn func( - ctx: mlir.Context, - args: struct { - sym_name: []const u8, - args: []const mlir.Type, - arg_attrs: []const mlir.Attribute = &.{}, - results: []const mlir.Type, - res_attrs: []const mlir.Attribute = &.{}, - block: mlir.Block, - location: mlir.Location, - }, -) mlir.Operation { - var attrs_tuple_buffer = stdx.BoundedArray(mlir.AttrTuple, 4){}; - attrs_tuple_buffer.appendAssumeCapacity(.{ "sym_name", .string(ctx, args.sym_name) }); - attrs_tuple_buffer.appendAssumeCapacity(.{ "function_type", .type_(.function(ctx, args.args, args.results)) }); - if (args.arg_attrs.len > 0) { - attrs_tuple_buffer.appendAssumeCapacity(.{ "arg_attrs", .array(ctx, args.arg_attrs) }); - } +const FuncOpArgs = struct { + pub const Visibility = enum { + public, + private, + }; - if (args.res_attrs.len > 0) { - attrs_tuple_buffer.appendAssumeCapacity(.{ "res_attrs", .array(ctx, args.res_attrs) }); + name: []const u8, + args: ?[]const *const mlir.Type = null, + args_attributes: ?[]const *const mlir.Attribute = null, + results: ?[]const *const mlir.Type = null, + results_attributes: ?[]const *const mlir.Attribute = null, + block: *mlir.Block, + location: *const mlir.Location, + no_inline: bool = false, + visibility: Visibility = .public, +}; + +pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { + var args_buffer: stdx.BoundedArray(*const mlir.Type, 1024) = .{}; + var results_buffer: stdx.BoundedArray(*const mlir.Type, 32) = .{}; + + var attr_tuples_buffer: stdx.BoundedArray(mlir.NamedAttribute, 16) = .{}; + attr_tuples_buffer.appendSliceAssumeCapacity(&.{ + .named(ctx, "sym_name", mlir.stringAttribute(ctx, args.name)), + .named(ctx, "sym_visibility", mlir.stringAttribute(ctx, @tagName(args.visibility))), + .named(ctx, "function_type", mlir.typeAttribute(mlir.functionType( + ctx, + args.args orelse args: { + for (0..args.block.numArguments()) |i| { + args_buffer.appendAssumeCapacity(args.block.argument(i).type_()); + } + break :args args_buffer.constSlice(); + }, + args.results orelse results: { + const terminator = args.block.terminator() orelse { + @panic("block has no terminator"); + }; + for (0..terminator.numOperands()) |i| { + results_buffer.appendAssumeCapacity(terminator.operand(i).type_()); + } + break :results results_buffer.constSlice(); + }, + ))), + }); + + if (args.args_attributes) |args_attributes| { + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "arg_attrs", mlir.arrayAttribute(ctx, args_attributes))); + } + if (args.results_attributes) |results_attributes| { + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "res_attrs", mlir.arrayAttribute(ctx, results_attributes))); + } + if (args.no_inline) { + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "no_inline", mlir.unitAttribute(ctx))); } return mlir.Operation.make(ctx, "func.func", .{ .blocks = &.{args.block}, - .attributes = attrs_tuple_buffer.constSlice(), + .attributes = attr_tuples_buffer.constSlice(), .location = args.location, }); } -pub fn call(ctx: mlir.Context, name: [:0]const u8, values: []const mlir.Value, results: []const mlir.Type, loc: mlir.Location) mlir.Operation { +pub fn call(ctx: *mlir.Context, name: []const u8, values: []const *const mlir.Value, results: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "func.call", .{ - .variadic_operands = &.{values}, - .results = results, - .verify = true, - .attributes = &.{.{ "callee", .symbol(ctx, name) }}, - .location = loc, + .operands = .{ .variadic = &.{values} }, + .results = .{ .flat = results }, + .attributes = &.{ + .named(ctx, "callee", mlir.flatSymbolRefAttribute(ctx, name)), + }, + .location = location, }); } -pub fn return_(ctx: mlir.Context, values: []const mlir.Value, loc: mlir.Location) mlir.Operation { +pub fn return_(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "func.return", .{ - .operands = values, + .operands = .{ .flat = &.{value} }, + .location = location, .verify = false, - .location = loc, + }); +} + +pub fn returns(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "func.return", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, }); } diff --git a/zml2/BUILD.bazel b/zml2/BUILD.bazel index 90c95102..277e5169 100644 --- a/zml2/BUILD.bazel +++ b/zml2/BUILD.bazel @@ -1,5 +1,16 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_binary") +load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_c_proto_library") load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") + +upb_c_proto_library( + name = "xla_data_upb", + deps = ["@xla//xla:xla_data_proto"], +) + +upb_c_proto_library( + name = "xla_compile_options_upb", + deps = ["@xla//xla/pjrt/proto:compile_options_proto"], +) cc_library( name = "posix", @@ -9,30 +20,36 @@ cc_library( zig_library( name = "zml", srcs = [ - "zml.zig", - "platform.zig", - "pjrtx.zig", "dtype.zig", + "exe.zig", "floats.zig", + "meta.zig", + "mlirx.zig", + "module.zig", + "pjrtx.zig", + "platform.zig", "shape.zig", "tensor.zig", - "mlirx.zig", - "meta.zig", + "zml.zig", ], + main = "zml.zig", + visibility = ["//visibility:public"], zigopts = [ "-lc", "-freference-trace=20", ], - main = "zml.zig", - visibility = ["//visibility:public"], deps = [ - "//runtimes", + ":posix", + ":xla_compile_options_upb", + ":xla_data_upb", + "//async", "//mlir", "//mlir/dialects", "//pjrt", + "//runtimes", "//stdx", + "//upb", "@rules_zig//zig/runfiles", - ":posix", ], ) diff --git a/zml2/exe.zig b/zml2/exe.zig new file mode 100644 index 00000000..23dffafa --- /dev/null +++ b/zml2/exe.zig @@ -0,0 +1,226 @@ +const std = @import("std"); + +const stdx = @import("stdx"); +const pjrt = @import("pjrtx.zig"); +const meta = @import("meta.zig"); +const Shape = @import("shape.zig").Shape; + +const Platform = @import("platform.zig").Platform; + +// TODO(Corentin) +const Buffer = struct {}; + +pub const Exe = struct { + platform: Platform, + exe: *pjrt.LoadedExecutable, + + context: ?*pjrt.ExecuteContext = null, + + input_shapes: []const Shape, + output_shapes: []const Shape, + + num_devices: u8, + + arena: std.heap.ArenaAllocator, + + pub fn init( + allocator: std.mem.Allocator, + platform: Platform, + exe: *pjrt.LoadedExecutable, + input_shapes: []const Shape, + output_shapes: []const Shape, + num_devices: u8, + ) !Exe { + var arena = std.heap.ArenaAllocator.init(allocator); + errdefer arena.deinit(); + + const input_shapes_copy = try arena.allocator().dupe(Shape, input_shapes); + const output_shapes_copy = try arena.allocator().dupe(Shape, output_shapes); + + return .{ + .platform = platform, + .exe = exe, + .input_shapes = input_shapes_copy, + .output_shapes = output_shapes_copy, + .num_devices = num_devices, + .arena = arena, + }; + } + + pub fn deinit(self: *const Exe) void { + self.arena.deinit(); + } + + pub fn args(self: *const Exe, allocator: std.mem.Allocator) !Arguments { + return Arguments.init(allocator, self.input_shapes, self.num_devices); + } + + pub fn results(self: *const Exe, allocator: std.mem.Allocator) !Results { + return Results.init(allocator, self.output_shapes, self.num_devices, self.platform); + } + + pub const FlatBuffers = struct { + buffers: []const [*]*pjrt.Buffer, + raw_buffers: []const *pjrt.Buffer, + + num_devices: usize, + + pub fn init(allocator: std.mem.Allocator, count: usize, num_devices: usize) !FlatBuffers { + const raw_buffers = try allocator.alloc(*pjrt.Buffer, num_devices * count); + errdefer allocator.free(raw_buffers); + + const buffers = try allocator.alloc([*]*pjrt.Buffer, num_devices); + errdefer allocator.free(buffers); + + for (0..num_devices) |i| { + buffers[i] = raw_buffers[i * count ..].ptr; + } + + return .{ + .buffers = buffers, + .raw_buffers = raw_buffers, + .num_devices = num_devices, + }; + } + + pub fn deinit(self: *const FlatBuffers, allocator: std.mem.Allocator) void { + allocator.free(self.buffers); + allocator.free(self.raw_buffers); + } + }; + + pub const Arguments = struct { + flat_buffers: FlatBuffers, + expected_shapes: []const Shape, + + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize) !Arguments { + const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); + errdefer flat_buffers.deinit(allocator); + + const expected_shapes = try allocator.dupe(Shape, shapes); + errdefer allocator.free(expected_shapes); + + return .{ + .flat_buffers = flat_buffers, + .expected_shapes = expected_shapes, + }; + } + + pub fn deinit(self: *const Arguments, allocator: std.mem.Allocator) void { + allocator.free(self.expected_shapes); + self.flat_buffers.deinit(allocator); + } + + pub fn set(self: *Arguments, v: anytype) void { + return self.setPartial(v, 0); + } + + pub fn setPartial(self: *Arguments, v: anytype, offset: usize) void { + const LocalContext = struct { + self: *Arguments, + current_index: usize = 0, + }; + var context: LocalContext = .{ .self = self, .current_index = offset }; + meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *const Buffer) void { + stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected argument {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); + for (0..context_.self.flat_buffers.num_devices) |device_index| { + context_.self.flat_buffers.buffers[device_index][context_.current_index] = buffer._shards.get(device_index); + } + + context_.current_index += 1; + } + }.cb, &context, &v); + } + }; + + pub const Results = struct { + platform: Platform, + flat_buffers: FlatBuffers, + + expected_shapes: []const Shape, + + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize, platform: Platform) !Results { + const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); + errdefer flat_buffers.deinit(allocator); + + const expected_shapes = try allocator.dupe(Shape, shapes); + errdefer allocator.free(expected_shapes); + + return .{ + .platform = platform, + .flat_buffers = flat_buffers, + .expected_shapes = expected_shapes, + }; + } + + pub fn deinit(self: *const Results, allocator: std.mem.Allocator) void { + allocator.free(self.expected_shapes); + self.flat_buffers.deinit(allocator); + } + + pub fn get(self: *Results, comptime T: type) T { + var result: T = undefined; + const LocalContext = struct { + self: *Results, + current_index: usize = 0, + }; + var context: LocalContext = .{ .self = self, .current_index = 0 }; + meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *Buffer) void { + var shards: Buffer.Shards = .{}; + for (0..context_.self.flat_buffers.num_devices) |device_index| { + shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); + } + buffer.* = Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], shards.constSlice()); + context_.current_index += 1; + } + }.cb, &context, &result); + return result; + } + + pub fn fill(self: *Results, v: anytype) void { + const LocalContext = struct { + self: *Results, + current_index: usize = 0, + }; + var context: LocalContext = .{ .self = self, .current_index = 0 }; + meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *Buffer) void { + //stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected result {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); + var shards: Buffer.Shards = .{}; + for (0..context_.self.flat_buffers.num_devices) |device_index| { + shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); + } + buffer.* = Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], shards.constSlice()); + context_.current_index += 1; + } + }.cb, &context, v); + } + }; + + pub fn call(self: *const Exe, arguments: Arguments, results_: *Results) void { + var events = [_]?*pjrt.Event{null} ** Platform.MAX_NUM_DEVICES; + const sharding = self.platform.sharding(); + + self.exe.execute(self.platform.pjrt_api, .{ + .arguments = arguments.flat_buffers.buffers, + .num_args = arguments.expected_shapes.len, + .results = results_.flat_buffers.buffers, + .events = events[0..sharding.num_partitions], + // this allows to tell a specific buffer shouldn't be donated, + // even if it has been marked as "can be donated" during compilation. + // TODO: expose it ? + .non_donatable_input_indices = &.{}, + .context = self.context, + }) catch |err| { + std.debug.panic("PJRT_LoadedExecutable_Execute failed with: {}", .{err}); + }; + + for (events[0..sharding.num_partitions]) |e| { + if (e) |ev| { + ev.await_(self.platform.pjrt_api) catch unreachable; + } + } + } +}; diff --git a/zml2/main.zig b/zml2/main.zig index 66d7a6cd..41ca7d1a 100644 --- a/zml2/main.zig +++ b/zml2/main.zig @@ -2,6 +2,26 @@ const std = @import("std"); const async = @import("async"); const zml = @import("zml"); +pub const Model = struct { + weight: zml.Tensor, + bias: zml.Tensor, + + pub fn init() !Model { + return .{ + .weight = .init(zml.Shape.init(.{ .m = 4096, .n = 4096 }, .f32)), + .bias = .init(zml.Shape.init(.{ .m = 4096 }, .f32)), + }; + } + + pub fn deinit(self: *Model) void { + _ = self; + } + + pub fn forward(self: Model, input: zml.Tensor) zml.Tensor { + return self.weight.dot(input, .n).add(self.bias); + } +}; + pub fn main() !void { try async.AsyncThread.main(std.heap.c_allocator, asyncMain); } @@ -10,6 +30,17 @@ pub fn asyncMain() !void { zml.init(); defer zml.deinit(); + var gpa: std.heap.DebugAllocator(.{}) = .init; + defer _ = gpa.deinit(); + + const allocator = gpa.allocator(); + var platform = try zml.Platform.init(.cpu, .{}); defer platform.deinit(); + + var model: Model = try .init(); + defer model.deinit(); + + var exe = try zml.module.compileModel(allocator, Model.forward, model, .{zml.Tensor.init(zml.Shape.init(.{ .n = 4096 }, .f32))}, platform); + defer exe.deinit(); } diff --git a/zml2/meta.zig b/zml2/meta.zig index 891a27ec..9f630742 100644 --- a/zml2/meta.zig +++ b/zml2/meta.zig @@ -427,7 +427,14 @@ test MapRestrict { /// Recursively visit the given struct and calls the callback for each K found. /// The `v` parameter must me a pointer, and tensor data need to be mutable if callbacks needs it. -pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) void { +pub fn VisitReturn(comptime cb: anytype) type { + return if (stdx.meta.FnSignature(cb, null).ReturnErrorSet) |error_set| + @Type(std.builtin.Type{ .error_union = .{ .error_set = error_set, .payload = void } }) + else + void; +} + +pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) VisitReturn(cb) { const Callback = @TypeOf(cb); const Ptr = @TypeOf(v); const type_info_v = @typeInfo(Ptr); @@ -436,6 +443,7 @@ pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) void { } const ptr_info = type_info_v.pointer; const Child = ptr_info.child; + const can_error = stdx.meta.FnSignature(cb, null).ReturnErrorSet != null; const K, const mutating_cb = switch (@typeInfo(FnParam(cb, 1))) { .pointer => |info| .{ info.child, !info.is_const }, @@ -468,15 +476,15 @@ pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) void { const field_type_info = @typeInfo(field.type); // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. switch (field_type_info) { - .pointer => visit(cb, ctx, @field(v, field.name)), - .array, .optional, .@"union", .@"struct" => visit(cb, ctx, &@field(v, field.name)), + .pointer => if (can_error) try visit(cb, ctx, @field(v, field.name)) else visit(cb, ctx, @field(v, field.name)), + .array, .optional, .@"union", .@"struct" => if (can_error) try visit(cb, ctx, &@field(v, field.name)) else visit(cb, ctx, &@field(v, field.name)), else => {}, } }, - .array => |_| for (v) |*elem| visit(cb, ctx, elem), - .optional => if (v.* != null) visit(cb, ctx, &v.*.?), + .array => |_| for (v) |*elem| if (can_error) try visit(cb, ctx, elem) else visit(cb, ctx, elem), + .optional => if (v.* != null) if (can_error) try visit(cb, ctx, &v.*.?) else visit(cb, ctx, &v.*.?), .@"union" => switch (v.*) { - inline else => |*v_field| visit(cb, ctx, v_field), + inline else => |*v_field| if (can_error) try visit(cb, ctx, v_field) else visit(cb, ctx, v_field), }, else => stdx.debug.compileError("zml.meta.visit({}) doesn't support fields of type: {}", .{ Callback, Child }), }, @@ -488,15 +496,15 @@ pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) void { const field_type_info = @typeInfo(field.type); // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. if (field_type_info == .pointer) { - visit(cb, ctx, @field(v_elem, field.name)); + if (can_error) try visit(cb, ctx, @field(v_elem, field.name)) else visit(cb, ctx, @field(v_elem, field.name)); } else { - visit(cb, ctx, &@field(v_elem, field.name)); + if (can_error) try visit(cb, ctx, &@field(v_elem, field.name)) else visit(cb, ctx, &@field(v_elem, field.name)); } }, - .array => |_| for (v) |*elem| visit(cb, ctx, elem), - .optional => if (v.* != null) visit(cb, ctx, &v.*.?), + .array => |_| for (v) |*elem| if (can_error) try visit(cb, ctx, elem) else visit(cb, ctx, elem), + .optional => if (v.* != null) if (can_error) try visit(cb, ctx, &v.*.?) else visit(cb, ctx, &v.*.?), .@"union" => switch (v_elem.*) { - inline else => |*v_field| visit(cb, ctx, v_field), + inline else => |*v_field| if (can_error) try visit(cb, ctx, v_field) else visit(cb, ctx, v_field), }, else => stdx.debug.compileError("zml.meta.visit({}) doesn't support fields of type: {}", .{ Callback, Child }), } diff --git a/zml2/mlirx.zig b/zml2/mlirx.zig index 2224b88d..3b13b2d8 100644 --- a/zml2/mlirx.zig +++ b/zml2/mlirx.zig @@ -12,20 +12,25 @@ pub const Type = struct { return switch (dt) { .bool => mlir.integerType(ctx, .i1), .f4e2m1 => mlir.floatType(ctx, .f4e2m1fn), - .f8e4m3b11fnuz => mlir.floatType(ctx, .f8e4m3b11fnuz), + .f8e3m4 => mlir.floatType(ctx, .f8e3m4), + .f8e4m3 => mlir.floatType(ctx, .f8e4m3), .f8e4m3fn => mlir.floatType(ctx, .f8e4m3fn), .f8e4m3fnuz => mlir.floatType(ctx, .f8e4m3fnuz), + .f8e4m3b11fnuz => mlir.floatType(ctx, .f8e4m3b11fnuz), .f8e5m2 => mlir.floatType(ctx, .f8e5m2), .f8e5m2fnuz => mlir.floatType(ctx, .f8e5m2fnuz), + .f8e8m0 => @panic("Unsupported type"), .bf16 => mlir.floatType(ctx, .bf16), .f16 => mlir.floatType(ctx, .f16), .f32 => mlir.floatType(ctx, .f32), .f64 => mlir.floatType(ctx, .f64), + .i2 => mlir.integerType(ctx, .i2), .i4 => mlir.integerType(ctx, .i4), .i8 => mlir.integerType(ctx, .i8), .i16 => mlir.integerType(ctx, .i16), .i32 => mlir.integerType(ctx, .i32), .i64 => mlir.integerType(ctx, .i64), + .u2 => mlir.integerType(ctx, .u2), .u4 => mlir.integerType(ctx, .u4), .u8 => mlir.integerType(ctx, .u8), .u16 => mlir.integerType(ctx, .u16), @@ -38,43 +43,49 @@ pub const Type = struct { // TODO(Corentin): Maybe remove that as it now requires a mlir context. // It's weird to have to provide a mlir context to get a zml.DataType from a mlir.Type. - pub fn toDType(mlir_type: *const mlir.Type) dtype.DataType { + pub fn toDType(ctx: *mlir.Context, mlir_type: *const mlir.Type) dtype.DataType { const mapping = .{ - .{ .bool, mlir.IntegerType(.i1) }, + .{ .bool, mlir.integerType(ctx, .i1) }, - .{ .f8e4m3b11fnuz, mlir.FloatType(.f8e4m3b11fnuz) }, - .{ .f8e4m3fn, mlir.FloatType(.f8e4m3fn) }, - .{ .f8e4m3fnuz, mlir.FloatType(.f8e4m3fnuz) }, - .{ .f8e5m2, mlir.FloatType(.f8e5m2) }, - .{ .f8e5m2fnuz, mlir.FloatType(.f8e5m2fnuz) }, - .{ .bf16, mlir.FloatType(.bf16) }, - .{ .f16, mlir.FloatType(.f16) }, - .{ .f32, mlir.FloatType(.f32) }, - .{ .f64, mlir.FloatType(.f64) }, + .{ .f8e4m3b11fnuz, mlir.floatType(ctx, .f8e4m3b11fnuz) }, + .{ .f8e4m3fn, mlir.floatType(ctx, .f8e4m3fn) }, + .{ .f8e4m3fnuz, mlir.floatType(ctx, .f8e4m3fnuz) }, + .{ .f8e5m2, mlir.floatType(ctx, .f8e5m2) }, + .{ .f8e5m2fnuz, mlir.floatType(ctx, .f8e5m2fnuz) }, + .{ .bf16, mlir.floatType(ctx, .bf16) }, + .{ .f16, mlir.floatType(ctx, .f16) }, + .{ .f32, mlir.floatType(ctx, .f32) }, + .{ .f64, mlir.floatType(ctx, .f64) }, - .{ .i4, mlir.IntegerType(.i4) }, - .{ .i8, mlir.IntegerType(.i8) }, - .{ .i16, mlir.IntegerType(.i16) }, - .{ .i32, mlir.IntegerType(.i32) }, - .{ .i64, mlir.IntegerType(.i64) }, + .{ .i4, mlir.integerType(ctx, .i4) }, + .{ .i8, mlir.integerType(ctx, .i8) }, + .{ .i16, mlir.integerType(ctx, .i16) }, + .{ .i32, mlir.integerType(ctx, .i32) }, + .{ .i64, mlir.integerType(ctx, .i64) }, - .{ .u4, mlir.IntegerType(.u4) }, - .{ .u8, mlir.IntegerType(.u8) }, - .{ .u16, mlir.IntegerType(.u16) }, - .{ .u32, mlir.IntegerType(.u32) }, - .{ .u64, mlir.IntegerType(.u64) }, + .{ .u4, mlir.integerType(ctx, .u4) }, + .{ .u8, mlir.integerType(ctx, .u8) }, + .{ .u16, mlir.integerType(ctx, .u16) }, + .{ .u32, mlir.integerType(ctx, .u32) }, + .{ .u64, mlir.integerType(ctx, .u64) }, - .{ .c64, mlir.ComplexType(.c64) }, - .{ .c128, mlir.ComplexType(.c128) }, + .{ .c64, mlir.complexType(ctx, .c64) }, + .{ .c128, mlir.complexType(ctx, .c128) }, }; inline for (mapping) |entry| { const dt, const mlirT = entry; - if (mlirT.isAFn(mlir_type._inner)) { + if (mlirT.eql(mlir_type)) { return dt; } } - std.debug.panic("Could not convert mlir.Type to DataType: {}", .{mlir_type}); + std.debug.panic("Could not convert mlir.Type to DataType: {f}", .{mlir_type}); } }; + +pub var once = std.once(struct { + fn call() void { + mlir.registerPasses("Transforms"); + } +}.call); diff --git a/zml2/module.zig b/zml2/module.zig new file mode 100644 index 00000000..f9f68daa --- /dev/null +++ b/zml2/module.zig @@ -0,0 +1,487 @@ +const std = @import("std"); + +const stdx = @import("stdx"); +const pjrt = @import("pjrtx.zig"); +const mlir = @import("mlir"); +const c = @import("c"); +const upb = @import("upb"); + +const meta = @import("meta.zig"); +const dialects = @import("mlir/dialects"); +const mlirx = @import("mlirx.zig"); + +const Platform = @import("platform.zig").Platform; +const Shape = @import("shape.zig").Shape; +const Exe = @import("exe.zig").Exe; +const Tensor = @import("tensor.zig").Tensor; + +const log = std.log.scoped(.@"zml/module"); + +const AttributeList = stdx.BoundedArray(mlir.NamedAttribute, 3); + +pub const CompilationContext = struct { + pub const Scope = struct { + block: *mlir.Block, + id_to_argument: std.AutoArrayHashMapUnmanaged(usize, usize), + id_to_donation: std.AutoArrayHashMapUnmanaged(usize, usize), + arena: std.heap.ArenaAllocator, + + pub fn initFromBlock(allocator: std.mem.Allocator, block: *mlir.Block) Scope { + const arena: std.heap.ArenaAllocator = .init(allocator); + return .{ + .block = block, + .id_to_argument = .empty, + .id_to_donation = .empty, + .arena = arena, + }; + } + + pub fn deinit(self: *Scope) void { + self.arena.deinit(); + } + }; + + allocator: std.mem.Allocator, + + mlir_registry: *mlir.DialectRegistry, + mlir_ctx: *mlir.Context, + mlir_pass_manager: *mlir.PassManager, + //mlir_op_pass_manager: *mlir.OpPassManager, + module: *mlir.Module, + + scopes: stdx.BoundedArray(Scope, 16) = .{}, + + threadlocal var _current: ?*CompilationContext = null; + + pub fn init(allocator: std.mem.Allocator) CompilationContext { + mlir.registerPasses("Transforms"); + const mlir_registry = mlir.DialectRegistry.init() catch unreachable; + inline for (.{ "func", "stablehlo" }) |d| { + mlir.DialectHandle.fromString(d).insertDialect(mlir_registry); + } + var mlir_ctx = mlir.Context.init(.{ .registry = mlir_registry, .threading = false }) catch unreachable; + mlir_ctx.loadAllAvailableDialects(); + + //const loc = mlir.Location.fromSrc(mlir_ctx, @src()).named(mlir_ctx, "main"); + const module = mlir.Module.init(.unknown(mlir_ctx)); + module.operation().setAttributeByName("sym_name", mlir.stringAttribute(mlir_ctx, "zml")); + + const pass_manager = mlir.PassManager.init(mlir_ctx); + { + var opm = pass_manager.asOpPassManager(); + const passes: []const []const u8 = &.{ + "canonicalize", + "cse", + "canonicalize", + }; + for (passes) |pass| { + opm.addPipeline(pass) catch unreachable; + } + } + + return .{ + .allocator = allocator, + .mlir_registry = mlir_registry, + .mlir_ctx = mlir_ctx, + .mlir_pass_manager = pass_manager, + .module = module, + }; + } + + pub fn deinit(self: *CompilationContext) void { + self.mlir_pass_manager.deinit(); + self.module.deinit(); + self.mlir_ctx.deinit(); + self.mlir_registry.deinit(); + } + + pub fn activate(self: *CompilationContext) void { + std.debug.assert(_current == null); + _current = self; + } + + pub fn deactivate(self: *CompilationContext) void { + _ = self; + _current = null; + } + + pub fn current() *CompilationContext { + return _current.?; + } + + pub fn currentScope(self: *CompilationContext) *Scope { + return &self.scopes.slice()[self.scopes.len - 1]; + } + + pub fn pushBlock(self: *CompilationContext, block: *mlir.Block) void { + const scope = Scope.initFromBlock(self.allocator, block); + self.scopes.appendAssumeCapacity(scope); + } + + pub fn popBlock(self: *CompilationContext) void { + var maybe_popped_scope = self.scopes.pop(); + if (maybe_popped_scope) |*popped| { + popped.deinit(); + } + } + + //pub fn addShardingAttributes( + // self: *const CompilationContext, + // mesh: Mesh, + // arg_attrs: []AttributeList, + // res_attrs: []AttributeList, + // input_shapes: []const Shape, + // input_meshes: []const ?Mesh, + // output_info: OutputInfo, + //) void { + // if (mesh.isSinglePartition()) return; // No sharding attributes for single partition. + + // const default_layout = mlir.NamedAttribute.named(self.mlir_ctx, "mhlo.layout_mode", mlir.stringAttribute(self.mlir_ctx, "default")); + // for (arg_attrs, input_shapes, input_meshes) |*attr, shape, input_mesh| { + // attr.appendAssumeCapacity(default_layout); + // if (input_mesh) |m| { + // // log.warn("addShardingAttributes for {} with input mesh: {}", .{ shape, m }); + // attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(m, shape)))); + // } else { + // // log.warn("addShardingAttributes for {} with progran mesh: {}", .{ shape, mesh }); + // attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(mesh, shape)))); + // } + // } + + // for (res_attrs, output_info.shapes) |*attr, shape| { + // attr.appendAssumeCapacity(default_layout); + // attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(mesh, shape)))); + // } + //} + + //pub fn getShardingAttr(self: *const CompilationContext, sharding: Sharding) *const mlir.Attribute { + // var sharding_str: stdx.BoundedArray(u8, 128) = .{}; + // sharding.writeShardingRepresentation(sharding_str.writer()) catch unreachable; + // return mlir.stringAttribute(self.mlir_ctx, sharding_str.constSlice()); + //} +}; + +pub fn compileModel(allocator: std.mem.Allocator, comptime func: anytype, model: stdx.meta.Head(stdx.meta.FnArgs(func)), args: stdx.meta.Tail(stdx.meta.FnArgs(func)), platform: Platform) !Exe { + return compile(allocator, func, .{model} ++ args, platform); +} + +pub fn compile(allocator: std.mem.Allocator, comptime func: anytype, args: stdx.meta.FnArgs(func), platform: Platform) !Exe { + var compilation_context: CompilationContext = .init(allocator); + defer compilation_context.deinit(); + + const result = emitMlir(&compilation_context, func, args) catch unreachable; + defer result.output_info.deinit(compilation_context.allocator); + defer result.input_info.deinit(compilation_context.allocator); + + _ = result.func.appendTo(compilation_context.module.body()); + const sharding = platform.sharding(); + compilation_context.module.operation().setAttributeByName("mhlo.num_replicas", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, sharding.num_replicas)); + compilation_context.module.operation().setAttributeByName("mhlo.num_partitions", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, sharding.num_partitions)); + + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), platform, compilation_context.module, null) catch unreachable; + + log.warn("******** ZML generated MLIR ********", .{}); + log.warn("{f}", .{compilation_context.module.operation()}); + + const num_devices = sharding.num_partitions * sharding.num_replicas; + const exe = try Exe.init(allocator, platform, loaded_executable, result.input_info.shapes, result.output_info.shapes, num_devices); + errdefer exe.deinit(); + + return exe; +} + +fn collectShapes(allocator: std.mem.Allocator, v: anytype) ![]Shape { + const LocalContext = struct { + list: *std.array_list.Managed(Shape), + }; + var list = std.array_list.Managed(Shape).init(allocator); + errdefer list.deinit(); + + var context: LocalContext = .{ .list = &list }; + try meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { + try ctx_.list.append(tensor.shape()); + } + }.cb, &context, v); + + return try list.toOwnedSlice(); +} + +pub const OutputInfo = struct { + shapes: []Shape, + values: []*const mlir.Value, + donations: []?usize, + + pub fn deinit(self: OutputInfo, allocator: std.mem.Allocator) void { + allocator.free(self.shapes); + allocator.free(self.values); + allocator.free(self.donations); + } +}; + +fn collectOutputInfo(allocator: std.mem.Allocator, v: anytype) !OutputInfo { + const LocalContext = struct { + shape_list: *std.array_list.Managed(Shape), + value_list: *std.array_list.Managed(*const mlir.Value), + donation_list: *std.array_list.Managed(?usize), + }; + + var shape_list = std.array_list.Managed(Shape).init(allocator); + errdefer shape_list.deinit(); + var value_list = std.array_list.Managed(*const mlir.Value).init(allocator); + errdefer value_list.deinit(); + var donation_list = std.array_list.Managed(?usize).init(allocator); + errdefer donation_list.deinit(); + + var context: LocalContext = .{ .shape_list = &shape_list, .value_list = &value_list, .donation_list = &donation_list }; + + try meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { + try ctx_.shape_list.append(tensor.shape()); + try ctx_.value_list.append(tensor.value()); + try ctx_.donation_list.append(tensor.donation()); + } + }.cb, &context, v); + + return .{ + .shapes = try shape_list.toOwnedSlice(), + .values = try value_list.toOwnedSlice(), + .donations = try donation_list.toOwnedSlice(), + }; +} + +pub const InputInfo = struct { + shapes: []Shape, + + pub fn deinit(self: InputInfo, allocator: std.mem.Allocator) void { + allocator.free(self.shapes); + } +}; + +fn collectInputInfo(allocator: std.mem.Allocator, v: anytype) !InputInfo { + const LocalContext = struct { + shape_list: *std.array_list.Managed(Shape), + }; + + var shape_list = std.array_list.Managed(Shape).init(allocator); + errdefer shape_list.deinit(); + + var context: LocalContext = .{ .shape_list = &shape_list }; + + try meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { + try ctx_.shape_list.append(tensor.shape()); + } + }.cb, &context, v); + + return .{ + .shapes = try shape_list.toOwnedSlice(), + }; +} + +const EmitMlirResult = struct { + func: *mlir.Operation, + input_info: InputInfo, + output_info: OutputInfo, +}; + +fn finalizeAttributeList(allocator_: std.mem.Allocator, mlir_ctx: *mlir.Context, attributes: []AttributeList) ![]*const mlir.Attribute { + const res = try allocator_.alloc(*const mlir.Attribute, attributes.len); + for (res, attributes) |*r, attr| { + r.* = mlir.dictionaryAttribute(mlir_ctx, attr.constSlice()); + } + return res; +} + +fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, args: stdx.meta.FnArgs(func)) !EmitMlirResult { + var arena = std.heap.ArenaAllocator.init(compilation_context.allocator); + defer arena.deinit(); + + const module = mlir.Module.init(.unknown(compilation_context.mlir_ctx)); + errdefer module.deinit(); + + const block = mlir.Block.init(&.{}, &.{}); + errdefer block.deinit(); + + compilation_context.pushBlock(block); + defer compilation_context.popBlock(); + + const LocalContext = struct { + compilation_context: *CompilationContext, + current_argument_id: usize = 0, + }; + var context: LocalContext = .{ + .compilation_context = compilation_context, + }; + meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { + std.log.debug("Argument {} - shape: {f}", .{ ctx_.current_argument_id, tensor.shape() }); + const mlir_type = mlir.rankedTensorType( + tensor.dims(), + mlirx.Type.fromDType(ctx_.compilation_context.mlir_ctx, tensor.dtype()), + ); + _ = ctx_.compilation_context.currentScope().block.addArgument(mlir_type, .unknown(ctx_.compilation_context.mlir_ctx)); + ctx_.compilation_context.currentScope().id_to_argument.put(ctx_.compilation_context.currentScope().arena.allocator(), tensor.id, ctx_.current_argument_id) catch unreachable; + ctx_.current_argument_id += 1; + } + }.cb, &context, &args); + + const input_info = try collectInputInfo(compilation_context.allocator, &args); + errdefer input_info.deinit(compilation_context.allocator); + + const input_attributes = try arena.allocator().alloc(AttributeList, input_info.shapes.len); + @memset(input_attributes, .{}); + + const output_info = b: { + compilation_context.activate(); + defer compilation_context.deactivate(); + + const result = @call(.auto, func, args); + + const output_info = try collectOutputInfo(compilation_context.allocator, &result); + errdefer output_info.deinit(compilation_context.allocator); + + break :b output_info; + }; + errdefer output_info.deinit(compilation_context.allocator); + + const output_attributes = try arena.allocator().alloc(AttributeList, output_info.shapes.len); + @memset(output_attributes, .{}); + + for (output_info.donations, 0..) |donation, index| if (donation) |argument_index| { + input_attributes[argument_index].appendAssumeCapacity(.named(compilation_context.mlir_ctx, "tf.aliasing_output", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, index))); + }; + _ = dialects.func.returns(compilation_context.mlir_ctx, output_info.values, .unknown(compilation_context.mlir_ctx)).appendTo(compilation_context.currentScope().block); + + //self.add + + const mlir_func = dialects.func.func(compilation_context.mlir_ctx, .{ + .name = "main", + .block = compilation_context.currentScope().block, + .location = .unknown(compilation_context.mlir_ctx), + .args_attributes = try finalizeAttributeList(arena.allocator(), compilation_context.mlir_ctx, input_attributes), + }); + + compilation_context.mlir_pass_manager.runOnOp(mlir_func) catch |err| switch (err) { + error.MlirUnexpected => { + std.log.err("Failed to canonicalize invalid mlir: {f}", .{mlir_func}); + // user errors should have triggered a panic before we reach this. + @panic("ZML generated invalid mlir. Please open a bug report"); + }, + }; + + return .{ + .func = mlir_func, + .input_info = input_info, + .output_info = output_info, + }; +} + +fn setXlaOverrideFlag(map: *c.upb_Map, flag: []const u8, value: anytype, upb_arena: *c.upb_Arena) !void { + const result = c.upb_Map_Set( + map, + .{ .str_val = upb.stringView(flag) }, + .{ .msg_val = blk: { + const field = try upb.new(c.xla_OptionOverrideProto, upb_arena); + switch (@typeInfo(@TypeOf(value))) { + .bool => c.xla_OptionOverrideProto_set_bool_field(field, value), + .comptime_int, .int => c.xla_OptionOverrideProto_set_int_field(field, @intCast(value)), + .comptime_float, .float => c.xla_OptionOverrideProto_set_double_field(field, @floatCast(value)), + else => c.xla_OptionOverrideProto_set_string_field(field, upb.stringView(value)), + } + break :blk @ptrCast(field); + } }, + upb_arena, + ); + + if (result == false) { + return std.mem.Allocator.Error.OutOfMemory; + } +} + +fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, module: *const mlir.Module, xla_dump_to_: ?[]const u8) !*pjrt.LoadedExecutable { + //const tracer = Tracer.init("ai.zml.compilation"); + //const compile_frame = tracer.frameStart("pjrt compilation"); + //defer tracer.frameEnd(compile_frame, "pjrt compilation"); + + const sharding = platform.sharding(); + + var upb_alloc: upb.Allocator = .init(arena); + const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); + defer c.upb_Arena_Free(upb_arena); + + const options = blk: { + const options = try upb.new(c.xla_CompileOptionsProto, upb_arena); + c.xla_CompileOptionsProto_set_executable_build_options(options, executable_build_options_blk: { + const exec_build_options = try upb.new(c.xla_ExecutableBuildOptionsProto, upb_arena); + + c.xla_ExecutableBuildOptionsProto_set_device_ordinal(exec_build_options, -1); + c.xla_ExecutableBuildOptionsProto_set_num_replicas(exec_build_options, sharding.num_replicas); + c.xla_ExecutableBuildOptionsProto_set_num_partitions(exec_build_options, sharding.num_partitions); + c.xla_ExecutableBuildOptionsProto_set_use_spmd_partitioning(exec_build_options, sharding.num_partitions > 1 or sharding.num_replicas > 1); + + c.xla_ExecutableBuildOptionsProto_set_device_assignment(exec_build_options, device_assignment_blk: { + const device_assignment = try upb.new(c.xla_DeviceAssignmentProto, upb_arena); + + c.xla_DeviceAssignmentProto_set_replica_count(device_assignment, sharding.num_replicas); + c.xla_DeviceAssignmentProto_set_computation_count(device_assignment, sharding.num_partitions); + + const computation_devices = c.xla_DeviceAssignmentProto_resize_computation_devices(device_assignment, sharding.num_partitions, upb_arena); + for (computation_devices[0..sharding.num_partitions], 0..) |*computation_device, i| { + computation_device.* = try upb.new(c.xla_DeviceAssignmentProto_ComputationDevice, upb_arena); + _ = c.xla_DeviceAssignmentProto_ComputationDevice_add_replica_device_ids(computation_device.*, @intCast(i), upb_arena); + } + break :device_assignment_blk device_assignment; + }); + + break :executable_build_options_blk exec_build_options; + }); + + const overrides_map = c._xla_CompileOptionsProto_env_option_overrides_mutable_upb_map(options, upb_arena); + switch (platform.target) { + .cuda => { + // NVIDIA recommends these settings + // https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_triton_gemm", false, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_latency_hiding_scheduler", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_llvm_module_compilation_parallelism", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_libnvptxcompiler", true, upb_arena); + }, + .rocm => { + // Disable Triton GEMM on ROCM. For some reason it's much, much slower when + // enabled on CDNA and it's used on RDNA. Disable it altogether. + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_triton_gemm", false, upb_arena); + // Use lld from libllvm instead of invoking the ld.lld binary. + // This saves us from having to sandbox it. + try setXlaOverrideFlag(overrides_map, "xla_gpu_use_inprocess_lld", true, upb_arena); + }, + else => {}, + } + + if (xla_dump_to_ orelse platform.compilation_options.xla_dump_to) |xla_dump_to| { + try setXlaOverrideFlag(overrides_map, "xla_dump_to", xla_dump_to, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_dump_hlo_as_proto", true, upb_arena); + if (platform.compilation_options.xla_dump_fusion_visualization) { + try setXlaOverrideFlag(overrides_map, "xla_dump_fusion_visualization", true, upb_arena); + } + if (platform.compilation_options.xla_dump_hlo_pass_re) |re| { + try setXlaOverrideFlag(overrides_map, "xla_dump_hlo_pass_re", re, upb_arena); + } + } + + break :blk options; + }; + + const loaded_executable = try platform.pjrt_client.compile( + platform.pjrt_api, + arena, + module, + try upb.serialize(options, upb_arena), + ); + errdefer loaded_executable.deinit(); + + return loaded_executable; +} diff --git a/zml2/pjrtx.zig b/zml2/pjrtx.zig index f09ffa1e..207b5841 100644 --- a/zml2/pjrtx.zig +++ b/zml2/pjrtx.zig @@ -79,13 +79,13 @@ pub const Client = opaque { return @ptrCast(try self.inner().createViewOfDeviceBuffer(api, args)); } - fn compileSync(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { + fn compileSync(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { var bytecode: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); defer bytecode.deinit(); - module.op().writeBytecodeWithConfig(&bytecode.writer, .{ .desiredEmitedVersion = 1 }) catch |err| { + module.operation().writeBytecode(.{ .desired_emit_version = 1 }, &bytecode.writer) catch |err| { log.err("failed to write module bytecode: {}", .{err}); return switch (err) { - std.Io.Writer.Error.WriteFailed => error.OutOfMemory, + error.WriteFailed => error.OutOfMemory, else => |e| e, }; }; @@ -95,9 +95,9 @@ pub const Client = opaque { const stablehlo_version = blk: { if (api.stablehloCurrentVersion()) |requested_version| { - break :blk dialects.stablehlo.stablehloGetSmallerVersion(requested_version, dialects.stablehlo.getCurrentVersion()); + break :blk dialects.stablehlo.smallerVersion(requested_version, dialects.stablehlo.currentVersion()); } - break :blk dialects.stablehlo.getMinimumVersion(); + break :blk dialects.stablehlo.minimumVersion(); }; dialects.stablehlo.serializePortableArtifact(bytecode.written(), stablehlo_version, &serialized_buffer.writer) catch |err| { @@ -115,7 +115,7 @@ pub const Client = opaque { })); } - pub fn compile(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { + pub fn compile(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { return try async.callBlocking(compileSync, .{ self, api, allocator, module, compile_options_pb }); } diff --git a/zml2/platform.zig b/zml2/platform.zig index d942290b..e67e7128 100644 --- a/zml2/platform.zig +++ b/zml2/platform.zig @@ -55,6 +55,21 @@ pub const Platform = struct { return all_devices; } + pub const Sharding = struct { num_replicas: u8, num_partitions: u8 }; + + pub fn sharding(self: Platform) Sharding { + // replicas run the same function but with different inputs, + // while partitions contribute to one evaluation over a shared input. + // Inside an inference process, we generally don't want replicas, + // as it's best to fully isolate replicas on different processes. + // For now we hardcode num_replicas = 1. + const num_devices: u8 = @intCast(self.getDevices().len); + return if (self.compilation_options.sharding_enabled) + .{ .num_replicas = 1, .num_partitions = num_devices } + else + .{ .num_replicas = 1, .num_partitions = 1 }; + } + pub fn withCompilationOptions(self: Platform, opts: CompilationOptions) Platform { var res = self; res.compilation_options = opts; diff --git a/zml2/shape.zig b/zml2/shape.zig index 564f33b5..e45313c9 100644 --- a/zml2/shape.zig +++ b/zml2/shape.zig @@ -180,7 +180,7 @@ pub const Shape = struct { }; } - fn toTag(v: anytype) Tag { + pub fn toTag(v: anytype) Tag { const T = @TypeOf(v); return switch (T) { EnumLiteral => @tagName(v).ptr, diff --git a/zml2/tensor.zig b/zml2/tensor.zig index ed9895bc..9ed4ace2 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -10,7 +10,7 @@ const mlirx = @import("mlirx.zig"); const dialects = @import("mlir/dialects"); const DataType = @import("dtype.zig").DataType; -const CompilationContext = struct {}; +const CompilationContext = @import("module.zig").CompilationContext; pub const Tensor = struct { var current_id: std.atomic.Value(usize) = .{ .raw = 1 }; @@ -21,6 +21,10 @@ pub const Tensor = struct { _shape: Shape, _value: ?*const mlir.Value = null, + pub fn init(shape_: Shape) Tensor { + return .{ .id = Tensor.current_id.fetchAdd(1, .seq_cst), ._shape = shape_ }; + } + pub fn format(self: Tensor, writer: *std.Io.Writer) !void { try writer.print("Tensor({f})", .{self._shape}); } @@ -72,15 +76,15 @@ pub const Tensor = struct { /// Creates a Tensor from a mlir.Value /// /// The shape is derived from the type of the mlir.Value. - pub fn fromMlirValue(val: mlir.Value) Tensor { - const ranked_tensor = val.getType().as(mlir.RankedTensorType).?; - const n = ranked_tensor.getRank(); + pub fn fromMlirValue(val: *const mlir.Value) Tensor { + const ranked_tensor = val.type_().isA(mlir.RankedTensorType).?; + const n = ranked_tensor.rank(); stdx.debug.assert(n <= MAX_RANK, "Can't represent MLIR tensor of rank {}, max supported rank is {}.", .{ n, MAX_RANK }); - var sh: Shape = .{ ._dtype = mlirx.Type.toDType(ranked_tensor.getElementType()) }; + var sh: Shape = .{ ._dtype = mlirx.Type.toDType(mlirCtx(), ranked_tensor.elementType()) }; for (0..n) |i| { - sh._dims.appendAssumeCapacity(ranked_tensor.getDimension(i)); + sh._dims.appendAssumeCapacity(ranked_tensor.dimension(i)); } sh._tags.resize(n) catch unreachable; @@ -158,8 +162,8 @@ pub const Tensor = struct { /// /// This will fail if used outside of a compilation context. pub fn value(self: Tensor) *const mlir.Value { - if (self.getContext().currentScope().id_to_argument.get(self.id)) |argument_index| { - return self.getContext().currentScope().block.argument(argument_index); + if (CompilationContext.current().currentScope().id_to_argument.get(self.id)) |argument_index| { + return CompilationContext.current().currentScope().block.argument(argument_index); } else if (self._value) |v| { return v; } else @panic("Something went really wrong, tensor is not an argument nor has an mlir.Value"); @@ -219,7 +223,7 @@ pub const Tensor = struct { res_shape = res_shape.withDtype(dt); const op = dialects.stablehlo.bitcast_convert( - self.getContext().mlirCtx(), + mlirCtx(), self.value(), mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), .unknown(mlirCtx()), @@ -567,10 +571,13 @@ pub const Tensor = struct { stdx.debug.assert(sh.dtype().isFloat(), "normal expects tensor type to be a float, got {}", .{sh.dtype()}); const a = Tensor.constant(.{}, DataType.Value.init(sh.dtype(), opts.mean)); + _ = a; // autofix const b = Tensor.constant(.{}, DataType.Value.init(sh.dtype(), opts.stddev)); - const res_shape = Tensor.constantTensor(HostBuffer.fromSlice(.{sh.rank()}, sh.dims())); - const op = dialects.stablehlo.rng(mlirCtx(), a.value(), b.value(), res_shape.value(), .NORMAL, .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(sh, op.result(0)); + _ = b; // autofix + unreachable; + //const res_shape = Tensor.constantTensor(HostBuffer.fromSlice(.{sh.rank()}, sh.dims())); + //const op = dialects.stablehlo.rng(mlirCtx(), a.value(), b.value(), res_shape.value(), .NORMAL, .unknown(mlirCtx())).appendTo(currentBlock()); + //return _result(sh, op.result(0)); } /// Returns a Tensor of the given shape, filled with floating point numbers sampled from a Gumbel distribution, and a new Rng state. @@ -1117,13 +1124,11 @@ pub const Tensor = struct { res_shape = res_shape.appendDim(rhs._shape.dim(r), rhs._shape.tag(r)); } - const mlir_ctx = lhs.getContext().mlir_ctx; - const loc = mlir.Location.unknown(mlirCtx()); const op = dialects.stablehlo.dot_general( - mlir_ctx, + mlirCtx(), lhs.value(), rhs.value(), - mlir.rankedTensorType(res_shape.dims(), mlir.Type.fromDType(mlir_ctx, res_shape.dtype())), + mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), .{ .lhs_batching_dimensions = lhs_batching_axes.constSlice(), .rhs_batching_dimensions = rhs_batching_axes.constSlice(), @@ -1131,7 +1136,7 @@ pub const Tensor = struct { .rhs_contracting_dimensions = rhs_contracting_axes.constSlice(), .dot_precision = .fast, }, - loc, + .unknown(mlirCtx()), ).appendTo(currentBlock()); return _result(res_shape, op.result(0)); } @@ -1374,7 +1379,7 @@ pub const Tensor = struct { mlirCtx(), self.value(), mlir.rankedTensorType(new_shape.dims(), mlirx.Type.fromDType(mlirCtx(), new_shape.dtype())), - mlirx.tensorType(self.getContext().mlirCtx(), new_shape), + mlirx.tensorType(mlirCtx(), new_shape), .unknown(mlirCtx()), ).appendTo(currentBlock()); return _result(new_shape, reshaped_val.result(0)); @@ -1908,12 +1913,13 @@ pub const Tensor = struct { // TODO(Corentin) /// Embeds a buffer with concrete values into an Mlir program. - pub fn constantTensor(val: HostBuffer) Tensor { - const ctx = CompilationContext.current().mlirCtx(); - const loc = ctx.location(@src()); - const elem_type = mlirx.denseElementAttrType(val.dtype()) orelse std.debug.panic("constantTensor expects a dtype that can be serialized to MLIR, like f32 or i32, got {f}", .{val.shape()}); - const constant_op = dialects.stablehlo.constant(ctx, val.shape().dims(), elem_type, val.bytes(), loc); - return _result(val.shape(), constant_op.result(0)); + pub fn constantTensor() Tensor { + @panic("unimplemented"); + //const ctx = CompilationContext.current().mlirCtx(); + //const loc = ctx.location(@src()); + //const elem_type = mlirx.denseElementAttrType(val.dtype()) orelse std.debug.panic("constantTensor expects a dtype that can be serialized to MLIR, like f32 or i32, got {f}", .{val.shape()}); + //const constant_op = dialects.stablehlo.constant(ctx, val.shape().dims(), elem_type, val.bytes(), loc); + //return _result(val.shape(), constant_op.result(0)); } /// Returns a Tensor containing the result of the outer product between the input Tensors. @@ -2141,6 +2147,8 @@ pub const Tensor = struct { return self.gather_(idx_axes.slice(), idx_per_axis.slice(), opts); } + const GatherAxisKind = enum { batching, offset, collapsed, indices }; + pub fn gather_(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, opts: GatherOpts) Tensor { stdx.debug.assert(idx_axes.len > 0, "gather expects 1 or more axes to operate one, received none. Example: `x.gather(.a, indices, .{{}})`", .{}); for (idx_axes, 0..) |a, i| { @@ -2160,8 +2168,7 @@ pub const Tensor = struct { var idx_batch_axes: Shape.DimsArray = .{}; - const AxisKind = enum { batching, offset, collapsed, indices }; - var self_kind: stdx.BoundedArray(AxisKind, MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; + var self_kind: stdx.BoundedArray(GatherAxisKind, MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; for (self._shape.tags(), 0..self.rank()) |t, self_ax| { const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, 1, @intCast(self_ax)); @@ -2182,7 +2189,7 @@ pub const Tensor = struct { // compute res shape var res_shape = Shape.init(.{}, self.dtype()); - var res_kind: stdx.BoundedArray(AxisKind, MAX_RANK) = .{}; + var res_kind: stdx.BoundedArray(GatherAxisKind, MAX_RANK) = .{}; for (self_kind.slice(), 0..) |kind, ax_usize| { const ax: u3 = @intCast(ax_usize); if (ax == idx_axes[0]) { @@ -2230,11 +2237,11 @@ pub const Tensor = struct { indices.value(), slice_dims.constSlice(), .{ - .offset_dims = _collectAxes(AxisKind, res_kind, .offset).constSlice(), - .collapsed_slice_dims = _collectAxes(AxisKind, self_kind, .collapsed).constSlice(), - .operand_batching_dims = _collectAxes(AxisKind, self_kind, .batching).constSlice(), + .offset_dims = _collectAxes(GatherAxisKind, res_kind, .offset).constSlice(), + .collapsed_slice_dims = _collectAxes(GatherAxisKind, self_kind, .collapsed).constSlice(), + .operand_batching_dims = _collectAxes(GatherAxisKind, self_kind, .batching).constSlice(), .start_indices_batching_dims = idx_batch_axes.constSlice(), - .start_index_map = _collectAxes(AxisKind, self_kind, .collapsed).constSlice(), + .start_index_map = _collectAxes(GatherAxisKind, self_kind, .collapsed).constSlice(), .index_vector_dim = indices.axis(.coord), .indices_are_sorted = opts.indices_are_sorted, }, @@ -2618,9 +2625,9 @@ pub const Tensor = struct { &updates_values, update_block, .{ - .update_window_dims = _collectAxes(AxisKind, config.up_kind, .update_window).constSlice(), - .inserted_window_dims = _collectAxes(AxisKind, config.op_kind, .inserted_window).constSlice(), - .input_batching_dims = _collectAxes(AxisKind, config.op_kind, .batching).constSlice(), + .update_window_dims = _collectAxes(ScatterAxisKind, config.up_kind, .update_window).constSlice(), + .inserted_window_dims = _collectAxes(ScatterAxisKind, config.op_kind, .inserted_window).constSlice(), + .input_batching_dims = _collectAxes(ScatterAxisKind, config.op_kind, .batching).constSlice(), .scatter_indices_batching_dims = config.indices_batch_axes.constSlice(), .scatter_dims_to_operand_dims = config.scatter_to_operand_axes.constSlice(), .index_vector_dim = indices.rank() - 1, @@ -2638,14 +2645,14 @@ pub const Tensor = struct { } const ScatterConfig = struct { - op_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}, - up_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}, + op_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}, + up_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}, indices_batch_axes: Shape.DimsArray = .{}, scatter_to_operand_axes: Shape.DimsArray = .{}, updates_transpose: Shape.AxesArray = .{}, }; - const AxisKind = enum { batching, update_window, inserted_window, window_id }; + const ScatterAxisKind = enum { batching, update_window, inserted_window, window_id }; fn scatterConfig( op: Shape, @@ -2653,8 +2660,8 @@ pub const Tensor = struct { indices_per_axis: stdx.BoundedArray(Tensor, Tensor.MAX_RANK), indices_axes: Shape.TagsArray, ) ScatterConfig { - var op_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}; - var up_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}; + var op_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}; + var up_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}; var indices_batch_axes: Shape.DimsArray = .{}; var scatter_to_operand_axes: Shape.DimsArray = .{}; var updates_transpose: Shape.AxesArray = .{}; @@ -2788,7 +2795,7 @@ pub const Tensor = struct { indices_axes: *Shape.TagsArray, ) Tensor { var old_scatter_to_op_axes = cfg.scatter_to_operand_axes; - const batching = _collectAxes(AxisKind, cfg.op_kind, .batching); + const batching = _collectAxes(ScatterAxisKind, cfg.op_kind, .batching); for (batching.constSlice()) |batch_ax| { const id_shape = indices_per_axis.get(0).shape(); // batching requires tagging, so we're sure to have a tag here. @@ -3664,8 +3671,10 @@ pub const Tensor = struct { const chunk_size: i64 = @divFloor(d, n_chunks); const tail_chunk_size: i64 = @rem(d, chunk_size); - const allocator = self.getContext().allocator(); - var chunks = std.ArrayList(Tensor).initCapacity(allocator, n_chunks + 1) catch @panic("OOM"); + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + var chunks = std.ArrayList(Tensor).initCapacity(arena.allocator(), n_chunks + 1) catch @panic("OOM"); for (0..n_chunks) |i| { const start: i64 = @as(i64, @intCast(i)) * chunk_size; chunks.appendAssumeCapacity( @@ -3723,9 +3732,10 @@ pub const Tensor = struct { for (split_sizes) |n| split_sum += n; stdx.debug.assert(split_sum == d, "split expects sum of 'split_sizes' values and axis dimension to be equal, got {} and {}", .{ split_sum, d }); - const allocator = self.getContext().allocator(); - const res = allocator.alloc(Tensor, split_sizes.len) catch @panic("OOM"); - errdefer allocator.dealloc(res); + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const res = arena.allocator().alloc(Tensor, split_sizes.len) catch @panic("OOM"); var start: i64 = 0; for (split_sizes, 0..) |n, i| { @@ -4046,7 +4056,7 @@ pub const Tensor = struct { mlirCtx(), self.value(), other.value(), - dialects.stablehlo.ComparisonDirection.init(self.getContext().mlirCtx(), direction), + dialects.stablehlo.ComparisonDirection.init(mlirCtx(), direction), getComparisonType(mlirCtx(), self.dtype()), .unknown(mlirCtx()), ).appendTo(currentBlock()); @@ -4365,7 +4375,7 @@ pub const Tensor = struct { fn binaryOp( op_name: []const u8, - op_fn: fn (mlir.Context, mlir.Value, mlir.Value, mlir.Location) mlir.Operation, + op_fn: fn (*mlir.Context, *const mlir.Value, *const mlir.Value, *const mlir.Location) *mlir.Operation, ) fn (Tensor, Tensor) Tensor { return struct { pub fn binaryOpHelper(self: Tensor, other: Tensor) Tensor { @@ -4390,7 +4400,7 @@ pub const Tensor = struct { stdx.debug.assert(same_shape, "{s} expects tensor shapes to match, got {f} and {f}", .{ op_name, self._shape, other._shape }); - const ret = @call(.auto, op_fn, .{ mlirCtx(), self.value(), other_.value(), .unknown(mlirCtx()) }).appendTo(currentBlock()); + const ret = @call(.auto, op_fn, .{ mlirCtx(), self.value(), other_.value(), mlir.Location.unknown(mlirCtx()) }).appendTo(currentBlock()); return _result(self._shape, ret.result(0)); } }.binaryOpHelper; @@ -4412,6 +4422,11 @@ pub const Tensor = struct { fn currentBlock() *mlir.Block { return CompilationContext.current().currentScope().block; } + + /// Returns the donation data of the tensor. + pub fn donation(self: Tensor) ?usize { + return CompilationContext.current().currentScope().id_to_donation.get(self.id); + } }; fn initPoolArg(rank: usize, data: []const i64) [Tensor.MAX_RANK]i64 { @@ -4522,6 +4537,28 @@ fn getComparisonType(ctx: mlir.Context, dtype: DataType) dialects.stablehlo.Comp // ); //} +pub fn _collectAxes(T: type, bounded_array: stdx.BoundedArray(T, Tensor.MAX_RANK), value: T) stdx.BoundedArray(i64, Tensor.MAX_RANK) { + var res: stdx.BoundedArray(i64, Tensor.MAX_RANK) = .{}; + for (bounded_array.constSlice(), 0..) |v, ax| { + if (v == value) { + res.appendAssumeCapacity(@intCast(ax)); + } + } + return res; +} + +fn _parseGatherCoord(self: Tensor, axes_: anytype) struct { bool, stdx.BoundedArray(u3, Tensor.MAX_RANK) } { + const AxesT = @TypeOf(axes_); + const axes_is_scalar = AxesT == @TypeOf(.enum_literal) or AxesT == comptime_int or @typeInfo(AxesT) == .int; + + const coord_axes = if (axes_is_scalar) + stdx.BoundedArray(u3, Tensor.MAX_RANK).fromSlice(&.{self.axis(axes_)}) catch unreachable + else + self.axes(axes_); + + return .{ axes_is_scalar, coord_axes }; +} + fn toI64(values: anytype) []i64 { var res: [Tensor.MAX_RANK]i64 = undefined; for (values, 0..) |val, i| res[i] = @intCast(val); diff --git a/zml2/zml.zig b/zml2/zml.zig index b1cc842c..03769e68 100644 --- a/zml2/zml.zig +++ b/zml2/zml.zig @@ -7,7 +7,6 @@ const std = @import("std"); const c = @import("c"); -const mlir = @import("mlir"); const runfiles = @import("runfiles"); pub const platform = @import("platform.zig"); @@ -15,16 +14,18 @@ pub const Platform = platform.Platform; pub const Target = platform.Target; pub const CompilationOptions = platform.CompilationOptions; pub const floats = @import("floats.zig"); -const dtype = @import("dtype.zig"); +pub const dtype = @import("dtype.zig"); pub const Data = dtype.Data; pub const DataType = dtype.DataType; pub const pjrt = @import("pjrtx.zig"); - -var mlir_once = std.once(struct { - fn call() void { - mlir.registerPasses("Transforms"); - } -}.call); +pub const module = @import("module.zig"); +pub const exe = @import("exe.zig"); +pub const meta = @import("meta.zig"); +pub const mlir = @import("mlirx.zig"); +pub const shape = @import("shape.zig"); +pub const Shape = shape.Shape; +pub const tensor = @import("tensor.zig"); +pub const Tensor = tensor.Tensor; var runfiles_once = std.once(struct { fn call_() !void { @@ -55,7 +56,7 @@ var runfiles_once = std.once(struct { pub fn init() void { runfiles_once.call(); - mlir_once.call(); + mlir.once.call(); } pub fn deinit() void {} From fa8f75b9a17e17308c5a24d948f86e071489bc32 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 4 Dec 2025 14:20:57 +0100 Subject: [PATCH 075/881] Move some operations from tensor.zig to ops.zig --- zml2/BUILD.bazel | 1 + zml2/ops.zig | 711 +++++++++++++++++++++++++++++++++++++++++++++ zml2/tensor.zig | 735 ++--------------------------------------------- 3 files changed, 736 insertions(+), 711 deletions(-) create mode 100644 zml2/ops.zig diff --git a/zml2/BUILD.bazel b/zml2/BUILD.bazel index 277e5169..ff846ad2 100644 --- a/zml2/BUILD.bazel +++ b/zml2/BUILD.bazel @@ -26,6 +26,7 @@ zig_library( "meta.zig", "mlirx.zig", "module.zig", + "ops.zig", "pjrtx.zig", "platform.zig", "shape.zig", diff --git a/zml2/ops.zig b/zml2/ops.zig new file mode 100644 index 00000000..fc99d38c --- /dev/null +++ b/zml2/ops.zig @@ -0,0 +1,711 @@ +const std = @import("std"); + +const stdx = @import("stdx"); +const meta = @import("meta.zig"); +const mlir = @import("mlir"); +const mlirx = @import("mlirx.zig"); +const dialects = @import("mlir/dialects"); + +const Tensor = @import("tensor.zig").Tensor; +const Shape = @import("shape.zig").Shape; +const CompilationContext = @import("module.zig").CompilationContext; + +pub const ReduceArgs = struct { + left: Tensor, + right: Tensor, +}; + +pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const mlir_ctx = CompilationContext.current().mlir_ctx; + + const reduce_block, var result = b: { + const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inits.len); + var args: ArgsType = undefined; + var block_types: [2 * inits.len]*const mlir.Type = undefined; + + inline for (0..inits.len) |i| { + args[i].left = Tensor.init(inits[i].shape()); + args[i].right = Tensor.init(inits[i].shape()); + + block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); + block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); + } + + const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); + const reduce_block = mlir.Block.init(&block_types, &block_locs); + errdefer reduce_block.deinit(); + + CompilationContext.current().pushBlock(reduce_block); + defer CompilationContext.current().popBlock(); + + const scope = CompilationContext.current().currentScope(); + inline for (0..inits.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inits.len) catch unreachable; + } + + var result = @call(.auto, func, args ++ context); + + var result_values: [inits.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| { + result_values[i] = result[i].value(); + } + + _ = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)).appendTo(reduce_block); + break :b .{ reduce_block, result }; + }; + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); + } + + var init_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| init_values[i] = inits[i].value(); + + const reduce_op = mlir.Operation.make(mlir_ctx, "stablehlo.reduce", .{ + .operands = .{ .variadic = &.{ &input_values, &init_values } }, + .result_type_inference = true, + .blocks = &.{reduce_block}, + .attributes = &.{ + .named(mlir_ctx, "dimensions", mlir.denseArrayAttribute(mlir_ctx, .i64, axes_)), + }, + .verify = true, + .location = .unknown(mlir_ctx), + }).appendTo(CompilationContext.current().currentScope().block); + + // `stablehlo.reduce` drops axes. We want to avoid that to propagate tags. + // So we need to broadcast the output of `stablehlo.reduce` to the input shapes. + // To that order, we initialize `result` to `inputs`, then we use stdx.meta.visit, + // to find the correct mlir.Value, but we first broadcast before creating the final + // Tensor struct. + var broadcasting_axes: stdx.BoundedArray(i64, Tensor.MAX_RANK) = .{}; + for (0..Tensor.MAX_RANK) |i| { + if (std.mem.indexOfScalar(i64, axes_, @intCast(i)) == null) { + broadcasting_axes.append(@intCast(i)) catch unreachable; + } + } + + inline for (0..result.len) |i| { + var reduced_shape: Shape = inputs[i].shape(); + for (axes_) |a| { + reduced_shape = reduced_shape.setDim(a, 1); + } + + const tensor_type = mlir.rankedTensorType(reduced_shape.dims(), mlirx.Type.fromDType(mlir_ctx, reduced_shape.dtype())); + const broad_op = dialects.stablehlo.broadcast_in_dim( + mlir_ctx, + reduce_op.result(i), + broadcasting_axes.slice()[0 .. reduced_shape.rank() - axes_.len], + tensor_type, + .unknown(mlir_ctx), + ).appendTo(CompilationContext.current().currentScope().block); + + result[i] = Tensor._result(reduced_shape, broad_op.result(0)); + } + + return result; +} + +pub const ReduceWindowOpts = struct { + // TODO replace with Shape + window_dimensions: []const i64, + window_strides: []const i64, + base_dilations: []const i64, + window_dilations: []const i64, + padding: []const [2]i64, +}; + +pub fn reduceWindow(inputs: anytype, inits: anytype, opts: ReduceWindowOpts, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const mlir_ctx = CompilationContext.current().mlir_ctx; + + const reduce_block, var result = b: { + const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inits.len); + var args: ArgsType = undefined; + var block_types: [2 * inits.len]*const mlir.Type = undefined; + + inline for (0..inits.len) |i| { + args[i].left = Tensor.init(inits[i].shape()); + args[i].right = Tensor.init(inits[i].shape()); + + block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); + block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); + } + + const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); + const reduce_block = mlir.Block.init(&block_types, &block_locs); + errdefer reduce_block.deinit(); + + CompilationContext.current().pushBlock(reduce_block); + defer CompilationContext.current().popBlock(); + + const scope = CompilationContext.current().currentScope(); + inline for (0..inits.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inits.len) catch unreachable; + } + + var result = @call(.auto, func, args ++ context); + + var result_values: [inits.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| { + result_values[i] = result[i].value(); + } + + _ = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)).appendTo(reduce_block); + break :b .{ reduce_block, result }; + }; + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); + } + + var init_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| init_values[i] = inits[i].value(); + + const reduce_op = mlir.Operation.make(mlir_ctx, "stablehlo.reduce_window", .{ + .operands = .{ .variadic = &.{ &input_values, &init_values } }, + .result_type_inference = true, + .blocks = &.{reduce_block}, + .attributes = &.{ + .named(mlir_ctx, "window_dimensions", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.window_dimensions)), + .named(mlir_ctx, "window_strides", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.window_strides)), + .named(mlir_ctx, "base_dilations", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.base_dilations)), + .named(mlir_ctx, "window_dilations", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.window_dilations)), + // Cast the [][2]i64 to []i64 (safe) + .named(mlir_ctx, "padding", mlir.denseElementsAttribute(mlir.RankedTensorType.get(&.{}, mlir.integerType(mlir_ctx, .i64), null).shaped(), @as([]const i64, @ptrCast(opts.padding)))), + }, + .verify = true, + .location = .unknown(mlir_ctx), + }).appendTo(CompilationContext.current().currentScope().block); + + inline for (0..result.len) |i| { + result[i] = Tensor._result(inputs[i].shape(), reduce_op.result(i)); + } + + return result; +} + +pub const SortArgs = struct { + left: Tensor, + right: Tensor, +}; + +pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytype, is_stable: bool) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const mlir_ctx = CompilationContext.current().mlir_ctx; + + const sort_block, var result = b: { + const ArgsType = std.meta.Tuple(&[1]type{SortArgs} ** inputs.len); + var args: ArgsType = undefined; + var block_types: [2 * inputs.len]*const mlir.Type = undefined; + + inline for (0..inputs.len) |i| { + args[i].left = Tensor.init(Shape.init(.{}, inputs[i].shape().dtype())); + args[i].right = Tensor.init(Shape.init(.{}, inputs[i].shape().dtype())); + + block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); + block_types[i + inputs.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); + } + + const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); + const sort_block = mlir.Block.init(&block_types, &block_locs); + errdefer sort_block.deinit(); + + CompilationContext.current().pushBlock(sort_block); + defer CompilationContext.current().popBlock(); + + const scope = CompilationContext.current().currentScope(); + inline for (0..inputs.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inputs.len) catch unreachable; + } + + var result = @call(.auto, func, args ++ context); + + var result_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + result_values[i] = result[i].value(); + } + + _ = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)).appendTo(sort_block); + break :b .{ sort_block, result }; + }; + + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); + } + + const sort_op = mlir.Operation.make(mlir_ctx, "stablehlo.sort", .{ + .operands = .{ .variadic = &.{&input_values} }, + .result_type_inference = true, + .blocks = &.{sort_block}, + .attributes = &.{ + .named(mlir_ctx, "dimension", mlir.integerAttribute(mlir_ctx, .i64, axis_)), + .named(mlir_ctx, "is_stable", mlir.boolAttribute(mlir_ctx, is_stable)), + }, + .verify = true, + .location = .unknown(mlir_ctx), + }).appendTo(CompilationContext.current().currentScope().block); + + inline for (0..result.len) |i| { + result[i] = Tensor._result(inputs[i].shape(), sort_op.result(i)); + } + + return result; +} + +pub const ScatterArgs = struct { + input: Tensor, + update: Tensor, +}; + +/// Generalized version of scatter to many inputs. +/// See `zml.Tensor.scatterSlices` for documentation on scatter. +/// +/// This allows to use the same indices to update several tensors at once, +/// and where the update function is allow to look at elements from the different tensors +/// to compute the final value. +/// +/// This sounds nice but in practice XLA doesn't support this well on GPU, +/// and will generate slow code. In practice stick with `zml.Tensor.scatterSlices`. +pub fn scatter( + inputs: anytype, + index_tensors: anytype, + updates: anytype, + comptime func: anytype, + context: anytype, + opts: Tensor.ScatterOpts, +) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const mlir_ctx = CompilationContext.current().mlir_ctx; + + const update_block, var result = b: { + const ArgsType = std.meta.Tuple(&[1]type{ScatterArgs} ** inputs.len); + var args: ArgsType = undefined; + var block_types: [2 * inputs.len]*const mlir.Type = undefined; + + inline for (0..inputs.len) |i| { + args[i].input = Tensor.init(Shape.init(.{}, inputs[i].dtype())); + args[i].update = Tensor.init(Shape.init(.{}, inputs[i].dtype())); + + block_types[i] = mlir.rankedTensorType(args[i].input.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].input.dtype())); + block_types[i + inputs.len] = mlir.rankedTensorType(args[i].update.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].update.dtype())); + } + + const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); + const update_block = mlir.Block.init(&block_types, &block_locs); + errdefer update_block.deinit(); + + CompilationContext.current().pushBlock(update_block); + defer CompilationContext.current().popBlock(); + + const scope = CompilationContext.current().currentScope(); + inline for (0..inputs.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), args[i].input.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].update.id, i + inputs.len) catch unreachable; + } + + var result = @call(.auto, func, args ++ context); + + var result_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + result_values[i] = result[i].value(); + } + + _ = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)).appendTo(update_block); + break :b .{ update_block, result }; + }; + + // Note: I was a bit lazy here, and I only look at tags on the first tensor. + // we probably should check all of them. + const self = meta.first(Tensor, inputs); + const update = meta.first(Tensor, updates); + var indices_per_axis, var indices_axes = Shape.parseStruct(Tensor, index_tensors); + + if (indices_per_axis.len == 0) return inputs; + + // validate coord axes: all coord_axes should exist inside self + for (indices_axes.constSlice()) |t| { + stdx.debug.assert(self._shape.hasTag(t) != null, "zml.ops.scatter expects axes of indices to be axes of inputs, got input={f} and indices={any}", .{ self, indices_axes.constSlice() }); + } + + // Handle scalar indices by broadcasting them to the indices with the highest rank. + const indices_shape = blk: { + var higher_rank = indices_per_axis.get(0).shape(); + for (indices_per_axis.constSlice()[1..]) |indices| { + if (indices.rank() > higher_rank.rank()) { + higher_rank = indices.shape(); + } + } + break :blk higher_rank; + }; + for (indices_per_axis.slice()) |*idx| { + stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "zml.ops.scatter expects all indices tensor to have the same shape, got {any}", .{indices_per_axis.slice()}); + stdx.debug.assert(idx.dtype() == indices_shape.dtype(), "zml.ops.scatter expects all indices tensor to have the same dtype, got {any}", .{indices_per_axis.slice()}); + idx.* = idx.broad(indices_shape); + } + + // rewrite simple scatters to dynamicUpdateSlice. + if (@TypeOf(inputs) == struct { Tensor } and indices_shape.rank() == 0) { + return .{self.dynamicUpdateSlice(index_tensors, update)}; + } + + // TODO: ideally we should catch all possible scatter errors and provide nice error messages. + var config = scatterConfig(self.shape(), update.shape(), indices_per_axis, indices_axes); + const indices = scatterPrepareIndices(&config, self.shape(), update.shape(), &indices_per_axis, &indices_axes); + // const n_indices_axes = update.rank() - _collectAxes(AxisKind, up_kind, .update_window).len; + // stdx.debug.assert(n_indices_axe == indices_axes.len, "scatter({f}, {any}) expects 'updates' to contain all axes from 'indices', got indices={s}, updates={f}", .{ self, index_tensors, indices_axes.constSlice(), update }); + + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); + } + + var updates_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..updates.len) |i| updates_values[i] = updates[i].value(); + + const op = dialects.stablehlo.scatter( + mlir_ctx, + &input_values, + &.{indices.value()}, + &updates_values, + update_block, + .{ + .update_window_dims = _collectAxes(ScatterAxisKind, config.up_kind, .update_window).constSlice(), + .inserted_window_dims = _collectAxes(ScatterAxisKind, config.op_kind, .inserted_window).constSlice(), + .input_batching_dims = _collectAxes(ScatterAxisKind, config.op_kind, .batching).constSlice(), + .scatter_indices_batching_dims = config.indices_batch_axes.constSlice(), + .scatter_dims_to_operand_dims = config.scatter_to_operand_axes.constSlice(), + .index_vector_dim = indices.rank() - 1, + .indices_are_sorted = opts.indices_are_sorted, + .unique_indices = opts.indices_are_unique, + }, + .unknown(mlir_ctx), + ).appendTo(CompilationContext.current().currentScope().block); + + inline for (0..result.len) |i| { + result[i] = Tensor._result(inputs[i].shape(), op.result(i)); + } + + return result; +} + +const ScatterConfig = struct { + op_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}, + up_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}, + indices_batch_axes: Shape.DimsArray = .{}, + scatter_to_operand_axes: Shape.DimsArray = .{}, + updates_transpose: Shape.AxesArray = .{}, +}; + +const ScatterAxisKind = enum { batching, update_window, inserted_window, window_id }; + +fn scatterConfig( + op: Shape, + update: Shape, + indices_per_axis: stdx.BoundedArray(Tensor, Tensor.MAX_RANK), + indices_axes: Shape.TagsArray, +) ScatterConfig { + var op_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}; + var up_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}; + var indices_batch_axes: Shape.DimsArray = .{}; + var scatter_to_operand_axes: Shape.DimsArray = .{}; + var updates_transpose: Shape.AxesArray = .{}; + + const tagged_api = indices_axes.len > 0; + const indices = indices_per_axis.get(0).shape(); + + if (tagged_api) { + for (indices_axes.constSlice()) |t| { + scatter_to_operand_axes.appendAssumeCapacity(op.axis(t)); + } + for (indices.tags()) |t| { + stdx.debug.assert(update.hasTag(t) != null, "scatter expects 'updates' to have all axes of 'indices', got self={f}, updates={f} and indices={f}", .{ op, update, indices }); + updates_transpose.appendAssumeCapacity(update.axis(t)); + } + + for (op.tags()) |t| { + if (update.hasTag(t)) |up_ax| { + updates_transpose.appendAssumeCapacity(up_ax); + + if (indices.hasTag(t)) |id_ax| { + if (std.mem.indexOfScalar(Shape.Tag, indices_axes.constSlice(), t) != null) { + // tag is in indices AND in coords -> it's a batching dim that has been rewritten to a regular insertion dim + op_kind.appendAssumeCapacity(.inserted_window); + } else { + // tag is in op, indices and updates -> it's a batching dim + op_kind.appendAssumeCapacity(.batching); + indices_batch_axes.appendAssumeCapacity(@intCast(id_ax)); + } + } else { + op_kind.appendAssumeCapacity(.update_window); + } + } else { + op_kind.appendAssumeCapacity(.inserted_window); + } + } + + for (update.tags(), 0..) |t, up_ax| { + // Handle batch axes right away. + if (op.hasTag(t)) |self_ax| { + if (op_kind.get(self_ax) == .batching) { + up_kind.appendAssumeCapacity(.batching); + continue; + } + } + if (indices.hasTag(t) != null) { + up_kind.appendAssumeCapacity(.window_id); + } else if (op.hasTag(t)) |self_ax| { + stdx.debug.assert(update.dim(up_ax) <= op.dim(self_ax), "scatter expects the slices described in 'updates' to fit inside 'op', but along axis .{s} it doesn't. Got op={f}, updates={f}.", .{ t, op, update }); + up_kind.appendAssumeCapacity(.update_window); + } else { + // TODO: consider accepting untagged update here. + std.debug.panic("scatter expects 'updates' to be made of axes from op={f} and from indices={any}, got unknown tag {s} in {f}", .{ op, indices_axes.constSlice(), std.mem.sliceTo(t, 0), update }); + } + } + } else { + for (0..indices_per_axis.len) |i| { + op_kind.appendAssumeCapacity(.inserted_window); + scatter_to_operand_axes.appendAssumeCapacity(@intCast(i)); + up_kind.appendAssumeCapacity(.window_id); + } + for (indices_per_axis.len..op.rank()) |_| { + op_kind.appendAssumeCapacity(.update_window); + } + for (indices_per_axis.len..update.rank()) |_| { + up_kind.appendAssumeCapacity(.update_window); + } + for (0..update.rank()) |i| { + updates_transpose.appendAssumeCapacity(@intCast(i)); + } + } + + return .{ + .op_kind = op_kind, + .up_kind = up_kind, + .indices_batch_axes = indices_batch_axes, + .scatter_to_operand_axes = scatter_to_operand_axes, + .updates_transpose = updates_transpose, + }; +} + +// TODO(Corentin) +//test scatterConfig { +// const zml = @import("zml.zig"); +// const platform = zml.testing.env(); + +// var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); +// defer comp.deinit(); +// comp.activate(); +// defer comp.deactivate(); + +// const Local = struct { +// pub fn _idx(idx_shape: anytype) Tensor { +// return Tensor.constant(idx_shape, .{ .i32 = 0 }); +// } +// }; + +// const idx = Local._idx; +// const op = Shape.init(.{ .a = 10, .b = 20 }, .f32); + +// // Use .a as a batching axis with .a=10 x .n=8 updates of 2 elements of .b +// { +// const indices, const coords_tags = Shape.parseStruct(Tensor, .{ .b = idx(.{ .a = 10, .n = 8 }) }); +// const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); + +// const cfg = scatterConfig(op, update, indices, coords_tags); +// try std.testing.expectEqualSlices(AxisKind, &.{ .batching, .update_window }, cfg.op_kind.constSlice()); +// try std.testing.expectEqualSlices(AxisKind, &.{ .batching, .window_id, .update_window }, cfg.up_kind.constSlice()); +// } + +// // similar, but use the normalized form where .a is no longer an explicit batching axis. +// { +// const indices, const coords_tags = Shape.parseStruct(Tensor, .{ .a = idx(.{ .a = 10, .n = 8 }), .b = idx(.{ .a = 10, .n = 8 }) }); +// const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); + +// const cfg = scatterConfig(op, update, indices, coords_tags); +// try std.testing.expectEqualSlices(AxisKind, &.{ .inserted_window, .update_window }, cfg.op_kind.constSlice()); +// try std.testing.expectEqualSlices(AxisKind, &.{ .window_id, .window_id, .update_window }, cfg.up_kind.constSlice()); +// } +//} + +/// Concatenate all indices tensor in one tensor. +/// +/// Is allowed to reorder stuff to simplify the job of the backend, +/// and to expand the batching dims. +fn scatterPrepareIndices( + cfg: *ScatterConfig, + op: Shape, + update: Shape, + indices_per_axis: *stdx.BoundedArray(Tensor, Tensor.MAX_RANK), + indices_axes: *Shape.TagsArray, +) Tensor { + var old_scatter_to_op_axes = cfg.scatter_to_operand_axes; + const batching = _collectAxes(ScatterAxisKind, cfg.op_kind, .batching); + for (batching.constSlice()) |batch_ax| { + const id_shape = indices_per_axis.get(0).shape(); + // batching requires tagging, so we're sure to have a tag here. + const batch_tag = op.tag(batch_ax); + indices_axes.appendAssumeCapacity(batch_tag); + const batch_id = Tensor.iota(id_shape, batch_tag).convert(id_shape.dtype()); + indices_per_axis.appendAssumeCapacity(batch_id); + cfg.op_kind.buffer[@intCast(batch_ax)] = .inserted_window; + cfg.up_kind.buffer[update.axis(batch_tag)] = .window_id; + old_scatter_to_op_axes.appendAssumeCapacity(batch_ax); + } + cfg.indices_batch_axes = .{}; + + // Reorder the axes so that in indices_per_axis is ordered like in op if possible. + // TODO: transpose updates if needed + var indices: stdx.BoundedArray(Tensor, Tensor.MAX_RANK) = .{}; + var scatter_to_op_axes: Shape.DimsArray = .{}; + + while (old_scatter_to_op_axes.len > 0) { + const scatter_ax = std.sort.argMin(i64, old_scatter_to_op_axes.constSlice(), {}, std.sort.asc(i64)).?; + const op_ax = old_scatter_to_op_axes.orderedRemove(scatter_ax); + const scatter_idx = indices_per_axis.orderedRemove(scatter_ax); + + scatter_to_op_axes.appendAssumeCapacity(op_ax); + indices.appendAssumeCapacity(scatter_idx); + } + cfg.scatter_to_operand_axes = scatter_to_op_axes; + + for (scatter_to_op_axes.constSlice(), 0..) |sc_ax, i| { + if (i != sc_ax) { + //log.warn("Found a slow scatter pattern, which is going to generate a while loop: scatter({f}, {any}, {f}). Because the index axes aren't the major ones in the input tensor.", .{ op, scatter_to_op_axes.constSlice(), update }); + break; + } + } + return Tensor.stack(indices.constSlice(), .last, .coord); +} + +pub const GatherAxisKind = enum { batching, offset, collapsed, indices }; + +pub const GatherOpts = struct { indices_are_sorted: bool = false }; + +pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, opts: GatherOpts) Tensor { + const mlir_ctx = CompilationContext.current().mlir_ctx; + + stdx.debug.assert(idx_axes.len > 0, "gather expects 1 or more axes to operate one, received none. Example: `x.gather(.a, indices, .{{}})`", .{}); + for (idx_axes, 0..) |a, i| { + if (i > 0) { + stdx.debug.assert(a == idx_axes[i - 1] + 1, "gather expects 'idx_axes' to be sequential. But {any} aren't sequential in {f}", .{ idx_axes, self }); + } + } + var indices_shape = idx_per_axis[0].shape(); + for (idx_per_axis[1..]) |idx| { + if (idx.rank() > indices_shape.rank()) { + indices_shape = idx.shape(); + } + } + for (idx_per_axis) |idx| { + stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "gather indices can't be broadcasted together {any}", .{idx_per_axis}); + } + + var idx_batch_axes: Shape.DimsArray = .{}; + + var self_kind: stdx.BoundedArray(GatherAxisKind, Shape.MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; + + for (self._shape.tags(), 0..self.rank()) |t, self_ax| { + const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, 1, @intCast(self_ax)); + if (indices_shape.hasTag(t)) |id_ax| { + // tag is both in self and indices -> it's a batching dim + // Note: tags are required for batching. + self_kind.buffer[self_ax] = .batching; + idx_batch_axes.appendAssumeCapacity(id_ax); + stdx.debug.assert(!is_gather_axis, "gather expects axes to appear at most twice. Axis {s} has been found both in 'self={f}', in 'idx_axes={any}' and in 'indices={f}'", .{ t, self, idx_axes, indices_shape }); + } else if (is_gather_axis) { + // we collapsed all gathered axes + self_kind.buffer[self_ax] = .collapsed; + // idx_kind.buffer[id_ax] = .indices; + } else { + self_kind.buffer[self_ax] = .offset; + } + } + + // compute res shape + var res_shape = Shape.init(.{}, self.dtype()); + var res_kind: stdx.BoundedArray(GatherAxisKind, Shape.MAX_RANK) = .{}; + for (self_kind.slice(), 0..) |kind, ax_usize| { + const ax: u3 = @intCast(ax_usize); + if (ax == idx_axes[0]) { + // The first val_ax is special cause this is the place where we insert indices axes. + for (0.., indices_shape.tags(), indices_shape.dims()) |id_axis_order, id_axis, id_inserted_dim| { + const is_batching_axis = std.mem.containsAtLeastScalar(i64, idx_batch_axes.constSlice(), 1, @intCast(id_axis_order)); + // Batching axis is already in self. + if (is_batching_axis) continue; + + res_shape = res_shape.appendDim(id_inserted_dim, id_axis); + res_kind.appendAssumeCapacity(.indices); + } + } + switch (kind) { + .collapsed => continue, + else => { + res_shape = res_shape.appendDim(self.dim(ax), self._shape.tag(ax)); + res_kind.appendAssumeCapacity(kind); + }, + } + } + + // This is not a gather, but a dynamicSlice. + // Sometimes the backend recognize this pattern, but not always. + // So let us handle that. + if (indices_shape.count() == 1 and idx_axes.len == 1) { + return self.dynamicSlice1d(idx_axes[0], .{ .start = idx_per_axis[0].asScalar(), .len = 1 }).reshape(res_shape); + } + + var slice_dims: Shape.DimsArray = .{}; + for (self_kind.slice(), self.dims()) |k, d| { + slice_dims.appendAssumeCapacity(switch (k) { + .batching, .collapsed => 1, + .offset => d, + .indices => unreachable, + }); + } + + // TODO: try changing .last by other axis and see the perf impact. + const indices = Tensor.stack(idx_per_axis, .last, .coord); + // scoped_log.debug("gather --> {} {any}", .{ res_shape, res_kind.constSlice() }); + const gather_op = dialects.stablehlo.gather( + mlir_ctx, + self.value(), + indices.value(), + slice_dims.constSlice(), + .{ + .offset_dims = _collectAxes(GatherAxisKind, res_kind, .offset).constSlice(), + .collapsed_slice_dims = _collectAxes(GatherAxisKind, self_kind, .collapsed).constSlice(), + .operand_batching_dims = _collectAxes(GatherAxisKind, self_kind, .batching).constSlice(), + .start_indices_batching_dims = idx_batch_axes.constSlice(), + .start_index_map = _collectAxes(GatherAxisKind, self_kind, .collapsed).constSlice(), + .index_vector_dim = indices.axis(.coord), + .indices_are_sorted = opts.indices_are_sorted, + }, + .unknown(mlir_ctx), + ).appendTo(CompilationContext.current().currentScope().block); + + const mlir_shape = Tensor.fromMlirValue(gather_op.result(0)).shape(); + stdx.debug.assert(mlir_shape.eql(res_shape), "gather expects that batching indices appear in the same order in 'self' and 'indices', got: self={f}, indices={f}. You should transpose one or the other.", .{ self, indices }); + return Tensor._result(res_shape, gather_op.result(0)); +} + +pub fn _collectAxes(T: type, bounded_array: stdx.BoundedArray(T, Tensor.MAX_RANK), value: T) stdx.BoundedArray(i64, Tensor.MAX_RANK) { + var res: stdx.BoundedArray(i64, Tensor.MAX_RANK) = .{}; + for (bounded_array.constSlice(), 0..) |v, ax| { + if (v == value) { + res.appendAssumeCapacity(@intCast(ax)); + } + } + return res; +} diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 9ed4ace2..f3c4515e 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -8,6 +8,7 @@ const Shape = @import("shape.zig").Shape; const mlir = @import("mlir"); const mlirx = @import("mlirx.zig"); const dialects = @import("mlir/dialects"); +const ops = @import("ops.zig"); const DataType = @import("dtype.zig").DataType; const CompilationContext = @import("module.zig").CompilationContext; @@ -1254,8 +1255,8 @@ pub const Tensor = struct { /// Output shape is the input shape with the axis_ dim set to 1. pub fn sum(self: Tensor, axis_: anytype) Tensor { const a = self.axis(axis_); - return reduce(.{self}, .{Tensor.constant(self.dtype().zero())}, &.{a}, struct { - pub fn acc(args: ReduceArgs) Tensor { + return ops.reduce(.{self}, .{Tensor.constant(self.dtype().zero())}, &.{a}, struct { + pub fn acc(args: ops.ReduceArgs) Tensor { return args.right.add(args.left.convert(args.right.dtype())); } }.acc, .{})[0]; @@ -2105,7 +2106,7 @@ pub const Tensor = struct { return _result(self._shape, reverse_op.result(0)); } - pub const GatherOpts = struct { indices_are_sorted: bool = false }; + pub const GatherOpts = ops.GatherOpts; /// `gather` extracts slices from the given tensor at the specified offsets. /// example: `values.gather(.{ .a = idx }, .{})` @@ -2144,113 +2145,7 @@ pub const Tensor = struct { } // TODO: sort indices following self.shape instead of asking the user to do it. - return self.gather_(idx_axes.slice(), idx_per_axis.slice(), opts); - } - - const GatherAxisKind = enum { batching, offset, collapsed, indices }; - - pub fn gather_(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, opts: GatherOpts) Tensor { - stdx.debug.assert(idx_axes.len > 0, "gather expects 1 or more axes to operate one, received none. Example: `x.gather(.a, indices, .{{}})`", .{}); - for (idx_axes, 0..) |a, i| { - if (i > 0) { - stdx.debug.assert(a == idx_axes[i - 1] + 1, "gather expects 'idx_axes' to be sequential. But {any} aren't sequential in {f}", .{ idx_axes, self }); - } - } - var indices_shape = idx_per_axis[0].shape(); - for (idx_per_axis[1..]) |idx| { - if (idx.rank() > indices_shape.rank()) { - indices_shape = idx.shape(); - } - } - for (idx_per_axis) |idx| { - stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "gather indices can't be broadcasted together {any}", .{idx_per_axis}); - } - - var idx_batch_axes: Shape.DimsArray = .{}; - - var self_kind: stdx.BoundedArray(GatherAxisKind, MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; - - for (self._shape.tags(), 0..self.rank()) |t, self_ax| { - const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, 1, @intCast(self_ax)); - if (indices_shape.hasTag(t)) |id_ax| { - // tag is both in self and indices -> it's a batching dim - // Note: tags are required for batching. - self_kind.buffer[self_ax] = .batching; - idx_batch_axes.appendAssumeCapacity(id_ax); - stdx.debug.assert(!is_gather_axis, "gather expects axes to appear at most twice. Axis {s} has been found both in 'self={f}', in 'idx_axes={any}' and in 'indices={f}'", .{ t, self, idx_axes, indices_shape }); - } else if (is_gather_axis) { - // we collapsed all gathered axes - self_kind.buffer[self_ax] = .collapsed; - // idx_kind.buffer[id_ax] = .indices; - } else { - self_kind.buffer[self_ax] = .offset; - } - } - - // compute res shape - var res_shape = Shape.init(.{}, self.dtype()); - var res_kind: stdx.BoundedArray(GatherAxisKind, MAX_RANK) = .{}; - for (self_kind.slice(), 0..) |kind, ax_usize| { - const ax: u3 = @intCast(ax_usize); - if (ax == idx_axes[0]) { - // The first val_ax is special cause this is the place where we insert indices axes. - for (0.., indices_shape.tags(), indices_shape.dims()) |id_axis_order, id_axis, id_inserted_dim| { - const is_batching_axis = std.mem.containsAtLeastScalar(i64, idx_batch_axes.constSlice(), 1, @intCast(id_axis_order)); - // Batching axis is already in self. - if (is_batching_axis) continue; - - res_shape = res_shape.appendDim(id_inserted_dim, id_axis); - res_kind.appendAssumeCapacity(.indices); - } - } - switch (kind) { - .collapsed => continue, - else => { - res_shape = res_shape.appendDim(self.dim(ax), self._shape.tag(ax)); - res_kind.appendAssumeCapacity(kind); - }, - } - } - - // This is not a gather, but a dynamicSlice. - // Sometimes the backend recognize this pattern, but not always. - // So let us handle that. - if (indices_shape.count() == 1 and idx_axes.len == 1) { - return self.dynamicSlice1d(idx_axes[0], .{ .start = idx_per_axis[0].asScalar(), .len = 1 }).reshape(res_shape); - } - - var slice_dims: Shape.DimsArray = .{}; - for (self_kind.slice(), self.dims()) |k, d| { - slice_dims.appendAssumeCapacity(switch (k) { - .batching, .collapsed => 1, - .offset => d, - .indices => unreachable, - }); - } - - // TODO: try changing .last by other axis and see the perf impact. - const indices = Tensor.stack(idx_per_axis, .last, .coord); - // scoped_log.debug("gather --> {} {any}", .{ res_shape, res_kind.constSlice() }); - const gather_op = dialects.stablehlo.gather( - mlirCtx(), - self.value(), - indices.value(), - slice_dims.constSlice(), - .{ - .offset_dims = _collectAxes(GatherAxisKind, res_kind, .offset).constSlice(), - .collapsed_slice_dims = _collectAxes(GatherAxisKind, self_kind, .collapsed).constSlice(), - .operand_batching_dims = _collectAxes(GatherAxisKind, self_kind, .batching).constSlice(), - .start_indices_batching_dims = idx_batch_axes.constSlice(), - .start_index_map = _collectAxes(GatherAxisKind, self_kind, .collapsed).constSlice(), - .index_vector_dim = indices.axis(.coord), - .indices_are_sorted = opts.indices_are_sorted, - }, - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - - const mlir_shape = fromMlirValue(gather_op.result(0)).shape(); - stdx.debug.assert(mlir_shape.eql(res_shape), "gather expects that batching indices appear in the same order in 'self' and 'indices', got: self={f}, indices={f}. You should transpose one or the other.", .{ self, indices }); - return _result(res_shape, gather_op.result(0)); + return ops.gather_(self, idx_axes.slice(), idx_per_axis.slice(), opts); } // TODO(Corentin) @@ -2482,11 +2377,6 @@ pub const Tensor = struct { // try zml.testing.expectClose(expected, result, 0); //} - pub const ScatterArgs = struct { - input: Tensor, - update: Tensor, - }; - pub const ScatterOpts = struct { /// Promise scatter that all coordinates in `indices` are sorted, wrt to the final offset in `self` /// Result is undefined if the promise is violated. @@ -2502,337 +2392,17 @@ pub const Tensor = struct { /// then you should make sure the slices don't overlap, /// otherwise the result will depend on the runtime scheduling /// of the operator which is backend specific. - update_fn: *const fn (ScatterArgs) struct { Tensor } = increment, + update_fn: *const fn (ops.ScatterArgs) struct { Tensor } = increment, - pub fn increment(values: ScatterArgs) struct { Tensor } { + pub fn increment(values: ops.ScatterArgs) struct { Tensor } { return .{values.input.add(values.update)}; } - pub fn override(values: ScatterArgs) struct { Tensor } { + pub fn override(values: ops.ScatterArgs) struct { Tensor } { return .{values.update}; } }; - /// Generalized version of scatter to many inputs. - /// See `zml.Tensor.scatterSlices` for documentation on scatter. - /// - /// This allows to use the same indices to update several tensors at once, - /// and where the update function is allow to look at elements from the different tensors - /// to compute the final value. - /// - /// This sounds nice but in practice XLA doesn't support this well on GPU, - /// and will generate slow code. In practice stick with `zml.Tensor.scatterSlices`. - pub fn scatter( - inputs: anytype, - index_tensors: anytype, - updates: anytype, - comptime func: anytype, - context: anytype, - opts: Tensor.ScatterOpts, - ) stdx.meta.FnResult(func) { - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); - defer arena.deinit(); - - const update_block, var result = b: { - const ArgsType = std.meta.Tuple(&[1]type{ScatterArgs} ** inputs.len); - var args: ArgsType = undefined; - var block_types: [2 * inputs.len]*const mlir.Type = undefined; - - inline for (0..inputs.len) |i| { - args[i].input = Tensor.init(Shape.init(.{}, inputs[i].dtype())); - args[i].update = Tensor.init(Shape.init(.{}, inputs[i].dtype())); - - block_types[i] = mlir.rankedTensorType(args[i].input.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].input.dtype())); - block_types[i + inputs.len] = mlir.rankedTensorType(args[i].update.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].update.dtype())); - } - - const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlirCtx())); - const update_block = mlir.Block.init(&block_types, &block_locs); - errdefer update_block.deinit(); - - CompilationContext.current().pushBlock(update_block); - defer CompilationContext.current().popBlock(); - - const scope = CompilationContext.current().currentScope(); - inline for (0..inputs.len) |i| { - scope.id_to_argument.put(scope.arena.allocator(), args[i].input.id, i) catch unreachable; - scope.id_to_argument.put(scope.arena.allocator(), args[i].update.id, i + inputs.len) catch unreachable; - } - - var result = @call(.auto, func, args ++ context); - - var result_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| { - result_values[i] = result[i].value(); - } - - _ = dialects.stablehlo.returns(mlirCtx(), &result_values, .unknown(mlirCtx())).appendTo(update_block); - break :b .{ update_block, result }; - }; - - // Note: I was a bit lazy here, and I only look at tags on the first tensor. - // we probably should check all of them. - const self = meta.first(Tensor, inputs); - const update = meta.first(Tensor, updates); - var indices_per_axis, var indices_axes = Shape.parseStruct(Tensor, index_tensors); - - if (indices_per_axis.len == 0) return inputs; - - // validate coord axes: all coord_axes should exist inside self - for (indices_axes.constSlice()) |t| { - stdx.debug.assert(self._shape.hasTag(t) != null, "zml.ops.scatter expects axes of indices to be axes of inputs, got input={f} and indices={any}", .{ self, indices_axes.constSlice() }); - } - - // Handle scalar indices by broadcasting them to the indices with the highest rank. - const indices_shape = blk: { - var higher_rank = indices_per_axis.get(0).shape(); - for (indices_per_axis.constSlice()[1..]) |indices| { - if (indices.rank() > higher_rank.rank()) { - higher_rank = indices.shape(); - } - } - break :blk higher_rank; - }; - for (indices_per_axis.slice()) |*idx| { - stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "zml.ops.scatter expects all indices tensor to have the same shape, got {any}", .{indices_per_axis.slice()}); - stdx.debug.assert(idx.dtype() == indices_shape.dtype(), "zml.ops.scatter expects all indices tensor to have the same dtype, got {any}", .{indices_per_axis.slice()}); - idx.* = idx.broad(indices_shape); - } - - // rewrite simple scatters to dynamicUpdateSlice. - if (@TypeOf(inputs) == struct { Tensor } and indices_shape.rank() == 0) { - return .{self.dynamicUpdateSlice(index_tensors, update)}; - } - - // TODO: ideally we should catch all possible scatter errors and provide nice error messages. - var config = scatterConfig(self.shape(), update.shape(), indices_per_axis, indices_axes); - const indices = scatterPrepareIndices(&config, self.shape(), update.shape(), &indices_per_axis, &indices_axes); - // const n_indices_axes = update.rank() - _collectAxes(AxisKind, up_kind, .update_window).len; - // stdx.debug.assert(n_indices_axe == indices_axes.len, "scatter({f}, {any}) expects 'updates' to contain all axes from 'indices', got indices={s}, updates={f}", .{ self, index_tensors, indices_axes.constSlice(), update }); - - var input_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| { - input_values[i] = inputs[i].value(); - } - - var updates_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..updates.len) |i| updates_values[i] = updates[i].value(); - - const op = dialects.stablehlo.scatter( - mlirCtx(), - &input_values, - &.{indices.value()}, - &updates_values, - update_block, - .{ - .update_window_dims = _collectAxes(ScatterAxisKind, config.up_kind, .update_window).constSlice(), - .inserted_window_dims = _collectAxes(ScatterAxisKind, config.op_kind, .inserted_window).constSlice(), - .input_batching_dims = _collectAxes(ScatterAxisKind, config.op_kind, .batching).constSlice(), - .scatter_indices_batching_dims = config.indices_batch_axes.constSlice(), - .scatter_dims_to_operand_dims = config.scatter_to_operand_axes.constSlice(), - .index_vector_dim = indices.rank() - 1, - .indices_are_sorted = opts.indices_are_sorted, - .unique_indices = opts.indices_are_unique, - }, - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - - inline for (0..result.len) |i| { - result[i] = Tensor._result(inputs[i].shape(), op.result(i)); - } - - return result; - } - - const ScatterConfig = struct { - op_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}, - up_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}, - indices_batch_axes: Shape.DimsArray = .{}, - scatter_to_operand_axes: Shape.DimsArray = .{}, - updates_transpose: Shape.AxesArray = .{}, - }; - - const ScatterAxisKind = enum { batching, update_window, inserted_window, window_id }; - - fn scatterConfig( - op: Shape, - update: Shape, - indices_per_axis: stdx.BoundedArray(Tensor, Tensor.MAX_RANK), - indices_axes: Shape.TagsArray, - ) ScatterConfig { - var op_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}; - var up_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}; - var indices_batch_axes: Shape.DimsArray = .{}; - var scatter_to_operand_axes: Shape.DimsArray = .{}; - var updates_transpose: Shape.AxesArray = .{}; - - const tagged_api = indices_axes.len > 0; - const indices = indices_per_axis.get(0).shape(); - - if (tagged_api) { - for (indices_axes.constSlice()) |t| { - scatter_to_operand_axes.appendAssumeCapacity(op.axis(t)); - } - for (indices.tags()) |t| { - stdx.debug.assert(update.hasTag(t) != null, "scatter expects 'updates' to have all axes of 'indices', got self={f}, updates={f} and indices={f}", .{ op, update, indices }); - updates_transpose.appendAssumeCapacity(update.axis(t)); - } - - for (op.tags()) |t| { - if (update.hasTag(t)) |up_ax| { - updates_transpose.appendAssumeCapacity(up_ax); - - if (indices.hasTag(t)) |id_ax| { - if (std.mem.indexOfScalar(Shape.Tag, indices_axes.constSlice(), t) != null) { - // tag is in indices AND in coords -> it's a batching dim that has been rewritten to a regular insertion dim - op_kind.appendAssumeCapacity(.inserted_window); - } else { - // tag is in op, indices and updates -> it's a batching dim - op_kind.appendAssumeCapacity(.batching); - indices_batch_axes.appendAssumeCapacity(@intCast(id_ax)); - } - } else { - op_kind.appendAssumeCapacity(.update_window); - } - } else { - op_kind.appendAssumeCapacity(.inserted_window); - } - } - - for (update.tags(), 0..) |t, up_ax| { - // Handle batch axes right away. - if (op.hasTag(t)) |self_ax| { - if (op_kind.get(self_ax) == .batching) { - up_kind.appendAssumeCapacity(.batching); - continue; - } - } - if (indices.hasTag(t) != null) { - up_kind.appendAssumeCapacity(.window_id); - } else if (op.hasTag(t)) |self_ax| { - stdx.debug.assert(update.dim(up_ax) <= op.dim(self_ax), "scatter expects the slices described in 'updates' to fit inside 'op', but along axis .{s} it doesn't. Got op={f}, updates={f}.", .{ t, op, update }); - up_kind.appendAssumeCapacity(.update_window); - } else { - // TODO: consider accepting untagged update here. - std.debug.panic("scatter expects 'updates' to be made of axes from op={f} and from indices={any}, got unknown tag {s} in {f}", .{ op, indices_axes.constSlice(), std.mem.sliceTo(t, 0), update }); - } - } - } else { - for (0..indices_per_axis.len) |i| { - op_kind.appendAssumeCapacity(.inserted_window); - scatter_to_operand_axes.appendAssumeCapacity(@intCast(i)); - up_kind.appendAssumeCapacity(.window_id); - } - for (indices_per_axis.len..op.rank()) |_| { - op_kind.appendAssumeCapacity(.update_window); - } - for (indices_per_axis.len..update.rank()) |_| { - up_kind.appendAssumeCapacity(.update_window); - } - for (0..update.rank()) |i| { - updates_transpose.appendAssumeCapacity(@intCast(i)); - } - } - - return .{ - .op_kind = op_kind, - .up_kind = up_kind, - .indices_batch_axes = indices_batch_axes, - .scatter_to_operand_axes = scatter_to_operand_axes, - .updates_transpose = updates_transpose, - }; - } - - // TODO(Corentin) - //test scatterConfig { - // const zml = @import("zml.zig"); - // const platform = zml.testing.env(); - - // var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); - // defer comp.deinit(); - // comp.activate(); - // defer comp.deactivate(); - - // const Local = struct { - // pub fn _idx(idx_shape: anytype) Tensor { - // return Tensor.constant(idx_shape, .{ .i32 = 0 }); - // } - // }; - - // const idx = Local._idx; - // const op = Shape.init(.{ .a = 10, .b = 20 }, .f32); - - // // Use .a as a batching axis with .a=10 x .n=8 updates of 2 elements of .b - // { - // const indices, const coords_tags = Shape.parseStruct(Tensor, .{ .b = idx(.{ .a = 10, .n = 8 }) }); - // const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); - - // const cfg = scatterConfig(op, update, indices, coords_tags); - // try std.testing.expectEqualSlices(AxisKind, &.{ .batching, .update_window }, cfg.op_kind.constSlice()); - // try std.testing.expectEqualSlices(AxisKind, &.{ .batching, .window_id, .update_window }, cfg.up_kind.constSlice()); - // } - - // // similar, but use the normalized form where .a is no longer an explicit batching axis. - // { - // const indices, const coords_tags = Shape.parseStruct(Tensor, .{ .a = idx(.{ .a = 10, .n = 8 }), .b = idx(.{ .a = 10, .n = 8 }) }); - // const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); - - // const cfg = scatterConfig(op, update, indices, coords_tags); - // try std.testing.expectEqualSlices(AxisKind, &.{ .inserted_window, .update_window }, cfg.op_kind.constSlice()); - // try std.testing.expectEqualSlices(AxisKind, &.{ .window_id, .window_id, .update_window }, cfg.up_kind.constSlice()); - // } - //} - - /// Concatenate all indices tensor in one tensor. - /// - /// Is allowed to reorder stuff to simplify the job of the backend, - /// and to expand the batching dims. - fn scatterPrepareIndices( - cfg: *ScatterConfig, - op: Shape, - update: Shape, - indices_per_axis: *stdx.BoundedArray(Tensor, Tensor.MAX_RANK), - indices_axes: *Shape.TagsArray, - ) Tensor { - var old_scatter_to_op_axes = cfg.scatter_to_operand_axes; - const batching = _collectAxes(ScatterAxisKind, cfg.op_kind, .batching); - for (batching.constSlice()) |batch_ax| { - const id_shape = indices_per_axis.get(0).shape(); - // batching requires tagging, so we're sure to have a tag here. - const batch_tag = op.tag(batch_ax); - indices_axes.appendAssumeCapacity(batch_tag); - const batch_id = Tensor.iota(id_shape, batch_tag).convert(id_shape.dtype()); - indices_per_axis.appendAssumeCapacity(batch_id); - cfg.op_kind.buffer[@intCast(batch_ax)] = .inserted_window; - cfg.up_kind.buffer[update.axis(batch_tag)] = .window_id; - old_scatter_to_op_axes.appendAssumeCapacity(batch_ax); - } - cfg.indices_batch_axes = .{}; - - // Reorder the axes so that in indices_per_axis is ordered like in op if possible. - // TODO: transpose updates if needed - var indices: stdx.BoundedArray(Tensor, Tensor.MAX_RANK) = .{}; - var scatter_to_op_axes: Shape.DimsArray = .{}; - - while (old_scatter_to_op_axes.len > 0) { - const scatter_ax = std.sort.argMin(i64, old_scatter_to_op_axes.constSlice(), {}, std.sort.asc(i64)).?; - const op_ax = old_scatter_to_op_axes.orderedRemove(scatter_ax); - const scatter_idx = indices_per_axis.orderedRemove(scatter_ax); - - scatter_to_op_axes.appendAssumeCapacity(op_ax); - indices.appendAssumeCapacity(scatter_idx); - } - cfg.scatter_to_operand_axes = scatter_to_op_axes; - - for (scatter_to_op_axes.constSlice(), 0..) |sc_ax, i| { - if (i != sc_ax) { - //log.warn("Found a slow scatter pattern, which is going to generate a while loop: scatter({f}, {any}, {f}). Because the index axes aren't the major ones in the input tensor.", .{ op, scatter_to_op_axes.constSlice(), update }); - break; - } - } - return Tensor.stack(indices.constSlice(), .last, .coord); - } - /// Update the given tensor, by copying `values` into slice by slice into `self`. /// The slices are chosen at runtime by interpreting indices as coordinates into `self`. /// This is a generalized version of `dynamicUpdateSlice` where more than one offset can be specified at a time. @@ -2912,12 +2482,12 @@ pub const Tensor = struct { const UpdateType = @TypeOf(ScatterOpts.increment); const Custom = struct { - pub fn inc(values: ScatterArgs, custom: *const UpdateType) struct { Tensor } { + pub fn inc(values: ops.ScatterArgs, custom: *const UpdateType) struct { Tensor } { return @call(.auto, custom, .{values}); } }; - return scatter(.{self}, indices, .{updates}, Custom.inc, .{opts.update_fn}, opts)[0]; + return ops.scatter(.{self}, indices, .{updates}, Custom.inc, .{opts.update_fn}, opts)[0]; } // TODO(Corentin) @@ -3066,12 +2636,12 @@ pub const Tensor = struct { /// Returns a Tensor containing the maximum over a given axis. pub fn max(self: Tensor, axis_: anytype) Tensor { const a = self.axis(axis_); - return reduce( + return ops.reduce( .{self}, .{Tensor.constant(self.dtype().minValue())}, &.{a}, struct { - pub fn cmp(args: ReduceArgs) struct { Tensor } { + pub fn cmp(args: ops.ReduceArgs) struct { Tensor } { return .{args.left.maximum(args.right)}; } }, @@ -3082,12 +2652,12 @@ pub const Tensor = struct { /// Returns a Tensor containing the minimum over a given axis. pub fn min(self: Tensor, axis_: anytype) Tensor { const a = self.axis(axis_); - return reduce( + return ops.reduce( .{self}, .{Tensor.constant(self.dtype().maxValue())}, &.{a}, struct { - pub fn cmp(args: ReduceArgs) struct { Tensor } { + pub fn cmp(args: ops.ReduceArgs) struct { Tensor } { return .{args.left.minimum(args.right)}; } }, @@ -3099,7 +2669,7 @@ pub const Tensor = struct { values: Tensor, indices: Tensor, - fn cmp(values: ReduceArgs, indices: ReduceArgs) struct { Tensor, Tensor } { + fn cmp(values: ops.ReduceArgs, indices: ops.ReduceArgs) struct { Tensor, Tensor } { const left_gt_right = values.left.cmp(.GT, values.right); const is_nan = values.left.cmp(.NE, values.left); const left_gt_or_nan = left_gt_right.logical(.OR, is_nan); @@ -3121,7 +2691,7 @@ pub const Tensor = struct { const a = x.axis(axis_); const dt: DataType = if (x.dim(a) <= std.math.maxInt(i32)) .i32 else .i64; - const values, const indices = reduce( + const values, const indices = ops.reduce( .{ x, Tensor.arange(.{ .end = x.dim(axis_) }, dt).broadcast(x.shape(), &.{a}) }, .{ Tensor.constant(x.dtype().minValue()), Tensor.constant(dt.zero()) }, &.{a}, @@ -3166,189 +2736,22 @@ pub const Tensor = struct { // } //} - pub const ReduceArgs = struct { - left: Tensor, - right: Tensor, - }; - - fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); - defer arena.deinit(); - - const reduce_block, var result = b: { - const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inits.len); - var args: ArgsType = undefined; - var block_types: [2 * inits.len]*const mlir.Type = undefined; - - inline for (0..inits.len) |i| { - args[i].left = Tensor.init(inits[i].shape()); - args[i].right = Tensor.init(inits[i].shape()); - - block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].left.dtype())); - block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].right.dtype())); - } - - const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlirCtx())); - const reduce_block = mlir.Block.init(&block_types, &block_locs); - errdefer reduce_block.deinit(); - - CompilationContext.current().pushBlock(reduce_block); - defer CompilationContext.current().popBlock(); - - const scope = CompilationContext.current().currentScope(); - inline for (0..inits.len) |i| { - scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; - scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inits.len) catch unreachable; - } - - var result = @call(.auto, func, args ++ context); - - var result_values: [inits.len]*const mlir.Value = undefined; - inline for (0..inits.len) |i| { - result_values[i] = result[i].value(); - } - - _ = dialects.stablehlo.returns(mlirCtx(), &result_values, .unknown(mlirCtx())).appendTo(reduce_block); - break :b .{ reduce_block, result }; - }; - var input_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| { - input_values[i] = inputs[i].value(); - } - - var init_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inits.len) |i| init_values[i] = inits[i].value(); - - const reduce_op = mlir.Operation.make(mlirCtx(), "stablehlo.reduce", .{ - .operands = .{ .variadic = &.{ &input_values, &init_values } }, - .result_type_inference = true, - .blocks = &.{reduce_block}, - .attributes = &.{ - .named(mlirCtx(), "dimensions", mlir.denseArrayAttribute(mlirCtx(), .i64, axes_)), - }, - .verify = true, - .location = .unknown(mlirCtx()), - }).appendTo(currentBlock()); - - // `stablehlo.reduce` drops axes. We want to avoid that to propagate tags. - // So we need to broadcast the output of `stablehlo.reduce` to the input shapes. - // To that order, we initialize `result` to `inputs`, then we use stdx.meta.visit, - // to find the correct mlir.Value, but we first broadcast before creating the final - // Tensor struct. - var broadcasting_axes: stdx.BoundedArray(i64, Tensor.MAX_RANK) = .{}; - for (0..Tensor.MAX_RANK) |i| { - if (std.mem.indexOfScalar(i64, axes_, @intCast(i)) == null) { - broadcasting_axes.append(@intCast(i)) catch unreachable; - } - } - - inline for (0..result.len) |i| { - var reduced_shape: Shape = inputs[i].shape(); - for (axes_) |a| { - reduced_shape = reduced_shape.setDim(a, 1); - } - - const tensor_type = mlir.rankedTensorType(reduced_shape.dims(), mlirx.Type.fromDType(mlirCtx(), reduced_shape.dtype())); - const broad_op = dialects.stablehlo.broadcast_in_dim( - mlirCtx(), - reduce_op.result(i), - broadcasting_axes.slice()[0 .. reduced_shape.rank() - axes_.len], - tensor_type, - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - - result[i] = Tensor._result(reduced_shape, broad_op.result(0)); - } - - return result; - } - pub const SortRes = ArgMaxRes; - pub const SortArgs = struct { - left: Tensor, - right: Tensor, - }; - pub const SortOpts = struct { descending: bool = false }; /// Returns two Tensors. The first contains the sorted values and the second one contains the sorted indices. pub fn sort(self: Tensor, axis_: anytype, opts: SortOpts) SortRes { const a = self.axis(axis_); const indices = Tensor.arange(.{ .end = self.dim(a) }, .i32).broadcast(self._shape, &.{a}); - const res = sort_(.{ self, indices }, a, struct { - fn call(direction: dialects.stablehlo.ComparisonDirection.Direction, values: SortArgs, _: SortArgs) Tensor { + const res = ops.sort(.{ self, indices }, a, struct { + fn call(direction: dialects.stablehlo.ComparisonDirection.Direction, values: ops.SortArgs, _: ops.SortArgs) Tensor { return values.left.cmp(direction, values.right); } }.call, .{if (opts.descending) .GT else .LT}, true); return .{ .values = res[0], .indices = res[1] }; } - fn sort_(inputs: anytype, axis_: i64, comptime func: anytype, context: anytype, is_stable: bool) stdx.meta.FnResult(func) { - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); - defer arena.deinit(); - - const sort_block, var result = b: { - const ArgsType = std.meta.Tuple(&[1]type{SortArgs} ** inputs.len); - var args: ArgsType = undefined; - var block_types: [2 * inputs.len]*const mlir.Type = undefined; - - inline for (0..inputs.len) |i| { - args[i].left = Tensor.init(Shape.init(.{}, inputs[i].shape().dtype())); - args[i].right = Tensor.init(Shape.init(.{}, inputs[i].shape().dtype())); - - block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].left.dtype())); - block_types[i + inputs.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].right.dtype())); - } - - const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlirCtx())); - const sort_block = mlir.Block.init(&block_types, &block_locs); - errdefer sort_block.deinit(); - - CompilationContext.current().pushBlock(sort_block); - defer CompilationContext.current().popBlock(); - - const scope = CompilationContext.current().currentScope(); - inline for (0..inputs.len) |i| { - scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; - scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inputs.len) catch unreachable; - } - - var result = @call(.auto, func, args ++ context); - - var result_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| { - result_values[i] = result[i].value(); - } - - _ = dialects.stablehlo.returns(mlirCtx(), &result_values, .unknown(mlirCtx())).appendTo(sort_block); - break :b .{ sort_block, result }; - }; - - var input_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| { - input_values[i] = inputs[i].value(); - } - - const sort_op = mlir.Operation.make(mlirCtx(), "stablehlo.sort", .{ - .operands = .{ .variadic = &.{&input_values} }, - .result_type_inference = true, - .blocks = &.{sort_block}, - .attributes = &.{ - .named(mlirCtx(), "dimension", mlir.integerAttribute(mlirCtx(), .i64, axis_)), - .named(mlirCtx(), "is_stable", mlir.boolAttribute(mlirCtx(), is_stable)), - }, - .verify = true, - .location = .unknown(mlirCtx()), - }).appendTo(currentBlock()); - - inline for (0..result.len) |i| { - result[i] = Tensor._result(inputs[i].shape(), sort_op.result(i)); - } - - return result; - } - pub const ArgSortOpts = SortOpts; /// Returns a Tensor containing the indices corresponding to the sorted values over the given axis. @@ -3447,86 +2850,6 @@ pub const Tensor = struct { return result; } - pub const ReduceWindowOpts = struct { - // TODO replace with Shape - window_dimensions: []const i64, - window_strides: []const i64, - base_dilations: []const i64, - window_dilations: []const i64, - padding: []const [2]i64, - }; - - fn reduceWindow(inputs: anytype, inits: anytype, opts: ReduceWindowOpts, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); - defer arena.deinit(); - - const reduce_block, var result = b: { - const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inits.len); - var args: ArgsType = undefined; - var block_types: [2 * inits.len]*const mlir.Type = undefined; - - inline for (0..inits.len) |i| { - args[i].left = Tensor.init(inits[i].shape()); - args[i].right = Tensor.init(inits[i].shape()); - - block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].left.dtype())); - block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlirCtx(), args[i].right.dtype())); - } - - const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlirCtx())); - const reduce_block = mlir.Block.init(&block_types, &block_locs); - errdefer reduce_block.deinit(); - - CompilationContext.current().pushBlock(reduce_block); - defer CompilationContext.current().popBlock(); - - const scope = CompilationContext.current().currentScope(); - inline for (0..inits.len) |i| { - scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; - scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inits.len) catch unreachable; - } - - var result = @call(.auto, func, args ++ context); - - var result_values: [inits.len]*const mlir.Value = undefined; - inline for (0..inits.len) |i| { - result_values[i] = result[i].value(); - } - - _ = dialects.stablehlo.returns(mlirCtx(), &result_values, .unknown(mlirCtx())).appendTo(reduce_block); - break :b .{ reduce_block, result }; - }; - var input_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| { - input_values[i] = inputs[i].value(); - } - - var init_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inits.len) |i| init_values[i] = inits[i].value(); - - const reduce_op = mlir.Operation.make(mlirCtx(), "stablehlo.reduce_window", .{ - .operands = .{ .variadic = &.{ &input_values, &init_values } }, - .result_type_inference = true, - .blocks = &.{reduce_block}, - .attributes = &.{ - .named(mlirCtx(), "window_dimensions", mlir.denseArrayAttribute(mlirCtx(), .i64, opts.window_dimensions)), - .named(mlirCtx(), "window_strides", mlir.denseArrayAttribute(mlirCtx(), .i64, opts.window_strides)), - .named(mlirCtx(), "base_dilations", mlir.denseArrayAttribute(mlirCtx(), .i64, opts.base_dilations)), - .named(mlirCtx(), "window_dilations", mlir.denseArrayAttribute(mlirCtx(), .i64, opts.window_dilations)), - // Cast the [][2]i64 to []i64 (safe) - .named(mlirCtx(), "padding", mlir.denseElementsAttribute(mlir.RankedTensorType.get(&.{}, mlir.integerType(mlirCtx(), .i64), null).shaped(), @as([]const i64, @ptrCast(opts.padding)))), - }, - .verify = true, - .location = .unknown(mlirCtx()), - }).appendTo(currentBlock()); - - inline for (0..result.len) |i| { - result[i] = Tensor._result(inputs[i].shape(), reduce_op.result(i)); - } - - return result; - } - pub const MaxPoolRes = ArgMaxRes; /// Computes the 1d maxPool operation on the input Tensor. @@ -3562,7 +2885,7 @@ pub const Tensor = struct { var padding = [_][2]i64{.{ 0, 0 }} ** Tensor.MAX_RANK; padding[a] = opts.padding; - const result = reduceWindow( + const result = ops.reduceWindow( .{ self, iota(self._shape, a) }, .{ Tensor.constant(self.dtype().minValue()), Tensor.scalar(0, .i32) }, .{ @@ -3602,7 +2925,7 @@ pub const Tensor = struct { padding[a - 1] = opts.padding[0]; padding[a] = opts.padding[1]; - const result = reduceWindow( + const result = ops.reduceWindow( .{ self, iota(self._shape, a) }, .{ Tensor.constant(self.dtype().minValue()), Tensor.scalar(0, .i32) }, .{ @@ -4222,12 +3545,12 @@ pub const Tensor = struct { /// Returns a Tensor containing boolean indicating if there is a non-zero value over the given axis. pub fn any(self: Tensor, axis_: anytype) Tensor { const pred = self.cmp(.NE, Tensor.constant(self.dims(), self.dtype().zero())); - return reduce( + return ops.reduce( .{pred}, .{Tensor.scalar(false, .bool)}, &.{self.axis(axis_)}, struct { - pub fn acc(args: ReduceArgs) Tensor { + pub fn acc(args: ops.ReduceArgs) Tensor { return args.left.logical(.OR, args.right); } }.acc, @@ -4238,12 +3561,12 @@ pub const Tensor = struct { /// Returns a Tensor containing boolean indicating if there is a non-zero value over the given axis. pub fn all(self: Tensor, axis_: anytype) Tensor { const pred = if (self.dtype() == .bool) self else self.cmp(.NE, Tensor.scalar(0, self.dtype())); - return reduce( + return ops.reduce( .{pred}, .{Tensor.scalar(true, .bool)}, &.{self.axis(axis_)}, struct { - pub fn acc(args: ReduceArgs) Tensor { + pub fn acc(args: ops.ReduceArgs) Tensor { return args.left.logical(.AND, args.right); } }.acc, @@ -4537,16 +3860,6 @@ fn getComparisonType(ctx: mlir.Context, dtype: DataType) dialects.stablehlo.Comp // ); //} -pub fn _collectAxes(T: type, bounded_array: stdx.BoundedArray(T, Tensor.MAX_RANK), value: T) stdx.BoundedArray(i64, Tensor.MAX_RANK) { - var res: stdx.BoundedArray(i64, Tensor.MAX_RANK) = .{}; - for (bounded_array.constSlice(), 0..) |v, ax| { - if (v == value) { - res.appendAssumeCapacity(@intCast(ax)); - } - } - return res; -} - fn _parseGatherCoord(self: Tensor, axes_: anytype) struct { bool, stdx.BoundedArray(u3, Tensor.MAX_RANK) } { const AxesT = @TypeOf(axes_); const axes_is_scalar = AxesT == @TypeOf(.enum_literal) or AxesT == comptime_int or @typeInfo(AxesT) == .int; From 8e0e9879685718688ef6df449dd05cb69cb2786d Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 5 Dec 2025 14:11:24 +0100 Subject: [PATCH 076/881] Made it working on Zig 0.16 --- MODULE.bazel | 10 +--- mlir/mlir.zig | 4 +- pjrt/pjrt.zig | 90 +++++++++++++++++------------ runtimes/cpu/BUILD.bazel | 1 - runtimes/cpu/cpu.zig | 8 ++- stdx/flags.zig | 23 ++++---- stdx/meta.zig | 23 ++++---- stdx/signature.zig | 27 ++------- upb/upb.zig | 2 +- zml2/BUILD.bazel | 2 - zml2/main.zig | 5 -- zml2/meta.zig | 121 +++++++++++++++++++++------------------ zml2/pjrtx.zig | 54 +++++++++-------- zml2/shape.zig | 2 +- zml2/zml.zig | 5 +- 15 files changed, 191 insertions(+), 186 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 19a3f7ff..1f94b406 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -225,14 +225,10 @@ huggingface.model( use_repo(huggingface, "ModernBERT-base", "ModernBERT-large") ################################################################################ -# git_override( -# module_name = "rules_zig", -# commit = "2c91b54305772d7b43445168e7cf0037b5cbc9e2", -# remote = "https://github.com/zml/rules_zig.git", -# ) -local_path_override( +git_override( module_name = "rules_zig", - path = "../rules_zig", + commit = "566daebc8655bbf3ef18f187556daa83928afc84", + remote = "https://github.com/zml/rules_zig.git", ) git_override( diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 3d3e4b82..c5dcff60 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -1094,7 +1094,7 @@ pub const Operation = opaque { } }; - fn formatWithFlags(fctx: PrintFlags.Ctx, writer: *std.io.Writer) std.Io.Writer.Error!void { + fn formatWithFlags(fctx: PrintFlags.Ctx, writer: *std.Io.Writer) std.Io.Writer.Error!void { const flags = fctx.flags.createNative(); defer c.mlirOpPrintingFlagsDestroy(flags); var sctx: StringCallbackCtx = .{ .writer = writer }; @@ -1268,7 +1268,7 @@ pub const Operation = opaque { } } - fn fmtBytecodeImpl(ctx: BytecodeWriterConfig.Ctx, writer: *std.io.Writer) std.Io.Writer.Error!void { + fn fmtBytecodeImpl(ctx: BytecodeWriterConfig.Ctx, writer: *std.Io.Writer) std.Io.Writer.Error!void { try ctx.self.writeBytecode(ctx.config, writer); } diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index f9f30bda..cdbb7cd3 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -36,6 +36,12 @@ pub inline fn pjrtStruct(v: anytype) @TypeOf(v) { return ret; } +pub fn pjrtStruct2(T: type, v: anytype) T { + var ret = std.mem.zeroInit(T, v); + ret.struct_size = pjrtStructSize(T); + return ret; +} + pub const ApiError = error{ Cancelled, Unknown, @@ -94,7 +100,7 @@ pub const Api = struct { const api = DynGetPjrtApi(); log.info("Loaded library: {s}", .{library}); - _ = api.call(.PJRT_Plugin_Initialize, .{}) catch unreachable; + _ = api.call(.PJRT_Plugin_Initialize, pjrtStruct2(c.PJRT_Plugin_Initialize_Args, .{})) catch unreachable; return api; } @@ -107,8 +113,8 @@ pub const Api = struct { return arg_array_type_info.pointer.child; } - inline fn call(self: *const Api, comptime method: Funcs, arg: CallFnArgType(method)) ApiError!@TypeOf(arg) { - var ret = pjrtStruct(arg); + inline fn call(self: *const Api, comptime method: Funcs, arg: anytype) ApiError!CallFnArgType(method) { + var ret = pjrtStruct2(CallFnArgType(method), arg); const fn_ptr = @field(&self.inner, @tagName(method)).?; const result = fn_ptr(&ret); if (@TypeOf(result) == void) { @@ -242,14 +248,14 @@ pub const Error = opaque { pub fn getCode(self: *Error, api: *const Api) ErrorCode { const ret = api.call(.PJRT_Error_GetCode, .{ - .@"error" = @ptrCast(self), + .@"error" = @as(*c.PJRT_Error, @ptrCast(self)), }) catch unreachable; return @enumFromInt(ret.code); } pub fn getMessage(self: *Error, api: *const Api) []const u8 { const ret = api.call(.PJRT_Error_Message, .{ - .@"error" = @ptrCast(self), + .@"error" = @as(*c.PJRT_Error, @ptrCast(self)), }) catch unreachable; return ret.message[0..ret.message_size]; } @@ -266,7 +272,7 @@ pub const ShapeSpec = extern struct { pub fn init(dims_: []const usize, bt: BufferType) ShapeSpec { return .{ - .inner = pjrtStruct(c.PJRT_ShapeSpec{ + .inner = pjrtStruct2(c.PJRT_ShapeSpec, .{ .dims = @ptrCast(@constCast(dims_.ptr)), .num_dims = dims.len, .buffer_type = @intFromEnum(bt), @@ -296,7 +302,7 @@ pub const Client = opaque { pub fn init(api: *const Api, create_options: []const NamedValue) ClientInitError!*Client { // log.info("Loaded PJRT runtime plugin: {s}", .{api.Platform}); const ret = try api.call(.PJRT_Client_Create, .{ - .create_options = @ptrCast(create_options.ptr), + .create_options = @as([*c]const c.PJRT_NamedValue, @ptrCast(create_options.ptr)), .num_options = create_options.len, .kv_get_callback = null, .kv_put_callback = null, @@ -342,13 +348,13 @@ pub const Client = opaque { pub fn compile(self: *const Client, api: *const Api, args: CompileArgs) ApiError!*LoadedExecutable { const bytecode_format_ = @tagName(args.bytecode_format); const ret = try api.call(.PJRT_Client_Compile, .{ - .program = &pjrtStruct(c.PJRT_Program{ - .code = @ptrCast(@constCast(args.bytecode.ptr)), + .program = &pjrtStruct2(c.PJRT_Program, .{ + .code = @as([*c]u8, @ptrCast(@constCast(args.bytecode.ptr))), .code_size = args.bytecode.len, - .format = @ptrCast(@constCast(bytecode_format_.ptr)), + .format = @as([*c]const u8, @ptrCast(@constCast(bytecode_format_.ptr))), .format_size = bytecode_format_.len, }), - .compile_options = @ptrCast(@constCast(args.compile_options_pb.ptr)), + .compile_options = @as([*c]const u8, @ptrCast(@constCast(args.compile_options_pb.ptr))), .compile_options_size = args.compile_options_pb.len, .client = self.inner(), }); @@ -767,7 +773,7 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ApiError!void { - var options = pjrtStruct(c.PJRT_ExecuteOptions{ + var options = pjrtStruct2(c.PJRT_ExecuteOptions, .{ .send_callbacks = null, .recv_callbacks = null, .num_send_ops = 0, @@ -851,8 +857,8 @@ pub const MemoryLayout = union(MemoryLayoutType) { strides: Strides, fn toCStruct(self: MemoryLayout) c.PJRT_Buffer_MemoryLayout { - return pjrtStruct(switch (self) { - .tiled => |v| c.PJRT_Buffer_MemoryLayout{ + return switch (self) { + .tiled => |v| pjrtStruct2(c.PJRT_Buffer_MemoryLayout, .{ .type = c.PJRT_Buffer_MemoryLayout_Type_Tiled, .unnamed_0 = .{ .tiled = c.PJRT_Buffer_MemoryLayout_Tiled{ @@ -863,8 +869,8 @@ pub const MemoryLayout = union(MemoryLayoutType) { .num_tiles = v.tile_dims_sizes.len, }, }, - }, - .strides => |v| c.PJRT_Buffer_MemoryLayout{ + }), + .strides => |v| pjrtStruct2(c.PJRT_Buffer_MemoryLayout, .{ .type = c.PJRT_Buffer_MemoryLayout_Type_Strides, .unnamed_0 = .{ .strides = c.PJRT_Buffer_MemoryLayout_Strides{ @@ -872,8 +878,8 @@ pub const MemoryLayout = union(MemoryLayoutType) { .num_byte_strides = v.byte_strides.len, }, }, - }, - }); + }), + }; } }; @@ -1038,7 +1044,7 @@ pub const Event = opaque { pub fn getEventError(self: *const Event, api: *const Api) ?*Error { var args: Api.CallFnArgType(.PJRT_Event_Error) = .{ .event = self.inner() }; - args = pjrtStruct(args); + args = pjrtStruct2(@TypeOf(args), args); const result: ?*c.PJRT_Error = api.inner.PJRT_Event_Error.?(&args); return @ptrCast(result); } @@ -1233,53 +1239,63 @@ pub const NamedValue = extern struct { } pub fn fromString(name_: []const u8, value: []const u8) NamedValue { - return .{ .inner = pjrtStruct(c.PJRT_NamedValue{ - .name = @ptrCast(@constCast(name_.ptr)), + return .{ .inner = c.PJRT_NamedValue{ + .struct_size = c.PJRT_NamedValue_STRUCT_SIZE, + .extension_start = null, + .name = @as([*c]const u8, @ptrCast(@constCast(name_.ptr))), .name_size = name_.len, .type = c.PJRT_NamedValue_kString, .unnamed_0 = .{ .string_value = @ptrCast(@constCast(value.ptr)) }, .value_size = value.len, - }) }; + } }; } pub fn fromInt64(name_: []const u8, value: i64) NamedValue { - return .{ .inner = pjrtStruct(c.PJRT_NamedValue{ - .name = @ptrCast(@constCast(name_.ptr)), + return .{ .inner = c.PJRT_NamedValue{ + .struct_size = c.PJRT_NamedValue_STRUCT_SIZE, + .extension_start = null, + .name = @as([*c]const u8, @ptrCast(@constCast(name_.ptr))), .name_size = name_.len, .type = c.PJRT_NamedValue_kInt64, .unnamed_0 = .{ .int64_value = value }, .value_size = 1, - }) }; + } }; } pub fn fromInt64List(name_: []const u8, value: []const i64) NamedValue { - return .{ .inner = pjrtStruct(c.PJRT_NamedValue{ - .name = @ptrCast(@constCast(name_.ptr)), + return .{ .inner = c.PJRT_NamedValue{ + .struct_size = c.PJRT_NamedValue_STRUCT_SIZE, + .extension_start = null, + .name = @as([*c]const u8, @ptrCast(@constCast(name_.ptr))), .name_size = name_.len, .type = c.PJRT_NamedValue_kInt64List, .unnamed_0 = .{ .int64_array_value = @ptrCast(@constCast(value.ptr)) }, .value_size = value.len, - }) }; + } }; } pub fn fromFloat(name_: []const u8, value: f32) NamedValue { - return .{ .inner = pjrtStruct(c.PJRT_NamedValue{ - .name = @ptrCast(@constCast(name_.ptr)), + return .{ .inner = c.PJRT_NamedValue{ + .struct_size = c.PJRT_NamedValue_STRUCT_SIZE, + .extension_start = null, + .name = @as([*c]const u8, @ptrCast(@constCast(name_.ptr))), .name_size = name_.len, .type = c.PJRT_NamedValue_kFloat, .unnamed_0 = .{ .float_value = value }, .value_size = 1, - }) }; + } }; } pub fn fromBool(name_: []const u8, value: bool) NamedValue { - return .{ .inner = pjrtStruct(c.PJRT_NamedValue{ - .name = @ptrCast(@constCast(name_.ptr)), + return .{ .inner = c.PJRT_NamedValue{ + .struct_size = c.PJRT_NamedValue_STRUCT_SIZE, + .extension_start = null, + .name = @as([*c]const u8, @ptrCast(@constCast(name_.ptr))), .name_size = name_.len, .type = c.PJRT_NamedValue_kBool, .unnamed_0 = .{ .bool_value = value }, .value_size = 1, - }) }; + } }; } pub fn format(self: NamedValue, writer: *std.Io.Writer) !void { @@ -1335,7 +1351,7 @@ pub const Ffi = extern struct { func: *const ffi.Handler, traits: ffi.HandlerTraits, ) ApiError!void { - var ret = pjrtStruct(c.PJRT_FFI_Register_Handler_Args{ + var ret = pjrtStruct2(c.PJRT_FFI_Register_Handler_Args, .{ .target_name = target_name.ptr, .target_name_size = target_name.len, .handler = @ptrCast(@constCast(func)), @@ -1352,7 +1368,7 @@ pub const Ffi = extern struct { } pub fn registerTypeId(self: *const Ffi, api: *const Api, type_name: []const u8, type_info: ?*const c.PJRT_FFI_Type_Info) ApiError!ffi.TypeId { - var ret = pjrtStruct(c.PJRT_FFI_Type_Register_Args{ + var ret = pjrtStruct2(c.PJRT_FFI_Type_Register_Args, .{ .type_name = type_name.ptr, .type_name_size = type_name.len, .type_id = 0, // let the plugin assign a unique type ID @@ -1368,7 +1384,7 @@ pub const Ffi = extern struct { } pub fn addUserData(self: *const Ffi, api: *const Api, context: *ExecuteContext, user_data: UserData) ApiError!void { - var ret = pjrtStruct(c.PJRT_FFI_UserData_Add_Args{ + var ret = pjrtStruct2(c.PJRT_FFI_UserData_Add_Args, .{ .context = @ptrCast(context), .user_data = user_data.toCStruct(), }); diff --git a/runtimes/cpu/BUILD.bazel b/runtimes/cpu/BUILD.bazel index 04156daf..0c016593 100644 --- a/runtimes/cpu/BUILD.bazel +++ b/runtimes/cpu/BUILD.bazel @@ -50,7 +50,6 @@ zig_library( ] + select({ "//runtimes:cpu.enabled": [ ":libpjrt_cpu", - "//async", "//stdx", "@rules_zig//zig/runfiles", ], diff --git a/runtimes/cpu/cpu.zig b/runtimes/cpu/cpu.zig index a3b8cde0..5ec12bbe 100644 --- a/runtimes/cpu/cpu.zig +++ b/runtimes/cpu/cpu.zig @@ -1,6 +1,5 @@ const builtin = @import("builtin"); -const async = @import("async"); const c = @import("c"); const pjrt = @import("pjrt"); const bazel_builtin = @import("bazel_builtin"); @@ -19,10 +18,13 @@ pub fn load() !*const pjrt.Api { return error.Unavailable; } + var io: std.Io.Threaded = .init_single_threaded; + defer io.deinit(); + var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); defer arena.deinit(); - var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator() }) orelse { + var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator(), .io = io.io() }) orelse { stdx.debug.panic("Unable to find runfiles", .{}); }; @@ -44,6 +46,6 @@ pub fn load() !*const pjrt.Api { var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; const path = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "libpjrt_cpu" ++ ext }); - break :blk async.callBlocking(pjrt.Api.loadFrom, .{path}); + break :blk pjrt.Api.loadFrom(path); }; } diff --git a/stdx/flags.zig b/stdx/flags.zig index 667814f2..3266c45d 100644 --- a/stdx/flags.zig +++ b/stdx/flags.zig @@ -182,18 +182,19 @@ fn parse_flags(args: *std.process.ArgIterator, comptime Flags: type) Flags { var result: Flags = undefined; // Would use std.enums.EnumFieldStruct(Flags, u32, 0) here but Flags is a struct not an Enum. var counts = comptime blk: { - var count_fields = std.meta.fields(Flags)[0..std.meta.fields(Flags).len].*; - for (&count_fields) |*field| { - field.type = u32; - field.alignment = @alignOf(u32); - field.default_value_ptr = @ptrCast(&@as(u32, 0)); + const f = std.meta.fields(Flags); + var count_field_names: [fields.len][]const u8 = undefined; + var count_field_types: [fields.len]type = undefined; + var count_field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; + for (f, &count_field_names, &count_field_types, &count_field_attrs) |field, *field_name, *field_type, *field_attr| { + field_name.* = field.name; + field_type.* = u32; + field_attr.* = .{ + .@"align" = @alignOf(u32), + .default_value_ptr = @ptrCast(&@as(u32, 0)), + }; } - break :blk @Type(.{ .@"struct" = .{ - .layout = .auto, - .fields = &count_fields, - .decls = &.{}, - .is_tuple = false, - } }){}; + break :blk @Struct(.auto, null, count_field_names, count_field_types, count_field_attrs){}; }; // When parsing arguments, we must consider longer arguments first, such that `--foo-bar=92` is diff --git a/stdx/meta.zig b/stdx/meta.zig index 0441d689..0d724a56 100644 --- a/stdx/meta.zig +++ b/stdx/meta.zig @@ -128,7 +128,11 @@ pub fn TupleRange(comptime T: type, comptime start: ?usize, comptime end: ?usize pub fn TupleRangeX(comptime T: type, comptime start: usize, comptime end: usize) type { const fields = std.meta.fields(T); - var new_fields: [end - start]std.builtin.Type.StructField = undefined; + + var field_names: [end - start][]const u8 = undefined; + var field_types: [end - start]type = undefined; + var field_attrs: [end - start]std.builtin.Type.StructField.Attributes = undefined; + inline for (start..end, 0..) |i, j| { var new_field = fields[i]; var num_buf: [32]u8 = undefined; @@ -137,16 +141,15 @@ pub fn TupleRangeX(comptime T: type, comptime start: usize, comptime end: usize) num_buf[s] = 0; break :blk num_buf[0..s :0]; }; - new_fields[j] = new_field; + field_names[j] = new_field.name; + field_types[j] = new_field.type; + field_attrs[j] = .{ + .@"comptime" = new_field.is_comptime, + .@"align" = new_field.alignment, + .default_value_ptr = new_field.default_value_ptr, + }; } - return @Type(.{ - .@"struct" = .{ - .is_tuple = true, - .layout = .auto, - .decls = &.{}, - .fields = &new_fields, - }, - }); + return @Struct(.auto, null, &field_names, &field_types, &field_attrs); } pub fn FnParam(comptime func: anytype, comptime n: comptime_int) type { diff --git a/stdx/signature.zig b/stdx/signature.zig index 2f042ce8..955a1887 100644 --- a/stdx/signature.zig +++ b/stdx/signature.zig @@ -13,38 +13,19 @@ pub fn ArgsTuple(comptime funcT: anytype, comptime ArgsT: ?type) type { } const args = std.meta.fields(ArgsT orelse @compileError("generic function requires an explicit ArgsTuple")); - var tuple_fields: [params.len]std.builtin.Type.StructField = undefined; + var tuple_types: [params.len]type = undefined; if (params.len != args.len) { compileError("function {} expected {} args, got {}", .{ funcT, params.len, args.len }); } inline for (params, args, 0..) |param, arg, i| { if (param.type == null) { - tuple_fields[i] = arg; + tuple_types[i] = arg.type; continue; } - const T = param.type.?; - var num_buf: [8]u8 = undefined; - tuple_fields[i] = .{ - .name = blk: { - const s = std.fmt.printInt(&num_buf, i, 10, .lower, .{}); - num_buf[s] = 0; - break :blk num_buf[0..s :0]; - }, - .type = T, - .default_value_ptr = null, - .is_comptime = false, - .alignment = if (@sizeOf(T) > 0) @alignOf(T) else 0, - }; + tuple_types[i] = param.type.?; } - return @Type(.{ - .@"struct" = .{ - .is_tuple = true, - .layout = .auto, - .decls = &.{}, - .fields = &tuple_fields, - }, - }); + return @Tuple(tuple_types); } pub const Signature = struct { diff --git a/upb/upb.zig b/upb/upb.zig index 2a9b1d52..b58252d7 100644 --- a/upb/upb.zig +++ b/upb/upb.zig @@ -47,7 +47,7 @@ pub fn shallowClone(comptime UpbType: type, arena: *c.upb_Arena, msg: *const Upb } pub fn stringView(data: ?[]const u8) c.upb_StringView { - return if (data) |d| c.upb_StringView_FromDataAndSize(d.ptr, d.len) else .{}; + return if (data) |d| c.upb_StringView_FromDataAndSize(d.ptr, d.len) else .{ .data = null, .size = 0 }; } pub fn slice(sv: c.upb_StringView) ?[]const u8 { diff --git a/zml2/BUILD.bazel b/zml2/BUILD.bazel index ff846ad2..311ee65a 100644 --- a/zml2/BUILD.bazel +++ b/zml2/BUILD.bazel @@ -43,7 +43,6 @@ zig_library( ":posix", ":xla_compile_options_upb", ":xla_data_upb", - "//async", "//mlir", "//mlir/dialects", "//pjrt", @@ -60,7 +59,6 @@ zig_binary( data = [], main = "main.zig", deps = [ - "//async", "//zml2:zml", ], ) diff --git a/zml2/main.zig b/zml2/main.zig index 41ca7d1a..81a0b054 100644 --- a/zml2/main.zig +++ b/zml2/main.zig @@ -1,5 +1,4 @@ const std = @import("std"); -const async = @import("async"); const zml = @import("zml"); pub const Model = struct { @@ -23,10 +22,6 @@ pub const Model = struct { }; pub fn main() !void { - try async.AsyncThread.main(std.heap.c_allocator, asyncMain); -} - -pub fn asyncMain() !void { zml.init(); defer zml.deinit(); diff --git a/zml2/meta.zig b/zml2/meta.zig index 9f630742..e22f7f9e 100644 --- a/zml2/meta.zig +++ b/zml2/meta.zig @@ -26,64 +26,73 @@ pub fn MapType(From: type, To: type) type { .@"struct" => |struct_infos| { const fields = struct_infos.fields; var same: bool = true; - var struct_fields: [fields.len]std.builtin.Type.StructField = undefined; - for (struct_fields[0..], fields) |*struct_field, field| { + var struct_field_names: [fields.len][]const u8 = undefined; + var struct_field_types: [fields.len]type = undefined; + var struct_field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; + for (struct_field_names[0..], struct_field_types[0..], struct_field_attrs[0..], fields) |*field_name, *field_type, *field_attr, field| { if (!field.is_comptime) { const R = map(field.type); if (R == field.type) { - struct_field.* = field; + field_name.* = field.name; + field_type.* = field.type; + field_attr.* = .{ + .@"comptime" = field.is_comptime, + .@"align" = field.alignment, + .default_value_ptr = field.default_value_ptr, + }; } else { - struct_field.* = .{ - .name = field.name, - .type = R, - .default_value_ptr = null, - .is_comptime = field.is_comptime, - .alignment = @alignOf(R), + field_name.* = field.name; + field_type.* = R; + field_attr.* = .{ + .@"comptime" = field.is_comptime, + .@"align" = @alignOf(R), }; same = false; // Handle the case `field: ?Tensor = null` // Generic handling of default value is complicated, // it would require to call the callback at comptime. if (R == ?To) { - struct_field.default_value_ptr = &@as(R, null); + field_attr.default_value_ptr = &@as(R, null); } } } else { - struct_field.* = field; + field_name.* = field.name; + field_type.* = field.type; + field_attr.* = .{ + .@"comptime" = field.is_comptime, + .@"align" = field.alignment, + .default_value_ptr = field.default_value_ptr, + }; } } if (same) return T; - return @Type(.{ .@"struct" = .{ - .layout = .auto, - .fields = struct_fields[0..], - .decls = &.{}, - .is_tuple = struct_infos.is_tuple, - } }); + return if (struct_infos.is_tuple) @Tuple(struct_field_types) else @Struct(.auto, null, struct_field_names, struct_field_types, struct_field_attrs); }, .@"union" => |union_info| { const fields = union_info.fields; var same: bool = true; - var union_fields: [fields.len]std.builtin.Type.UnionField = undefined; - for (union_fields[0..], fields) |*union_field, field| { + var union_field_names: [fields.len][]const u8 = undefined; + var union_field_types: [fields.len]type = undefined; + var union_field_attrs: [fields.len]std.builtin.Type.UnionField.Attributes = undefined; + for (union_field_names[0..], union_field_types[0..], union_field_attrs[0..], fields) |*field_name, *field_type, *field_attr, field| { const R = map(field.type); if (R == field.type) { - union_field.* = field; + field_name.* = field.name; + field_type.* = field.type; + field_attr.* = .{ + .@"align" = field.alignment, + }; } else { - union_field.* = .{ - .name = field.name, - .type = R, - .alignment = @alignOf(R), + field_name.* = field.name; + field_type.* = R; + field_attr.* = .{ + .@"align" = @alignOf(R), }; same = false; } } if (same) return T; - return @Type(.{ .@"union" = .{ - .layout = .auto, - .tag_type = union_info.tag_type, - .fields = union_fields[0..], - .decls = &.{}, - } }); + return @Union(.auto, union_info.tag_type, union_field_names, union_field_types, union_field_attrs); }, .array => |arr_info| [arr_info.len]map(arr_info.child), .pointer => |ptr_info| switch (ptr_info.size) { @@ -321,57 +330,55 @@ pub fn MapRestrict(From: type, To: type) type { const fields = struct_infos.fields; var num_fields: usize = 0; - var struct_fields: [fields.len]std.builtin.Type.StructField = undefined; + var struct_field_names: [fields.len][]const u8 = undefined; + var struct_field_types: [fields.len]type = undefined; + var struct_field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; for (fields) |field| { if (!field.is_comptime and Contains(field.type, From)) { const R = map(field.type); if (R == field.type) { - struct_fields[num_fields] = field; + struct_field_names[num_fields] = field.name; + struct_field_types[num_fields] = field.type; + struct_field_attrs[num_fields] = .{ + .@"comptime" = field.is_comptime, + .@"align" = field.alignment, + .default_value_ptr = field.default_value_ptr, + }; } else { const name = if (struct_infos.is_tuple) struct_infos.fields[num_fields].name else field.name; - struct_fields[num_fields] = .{ - .name = name, - .type = R, - .default_value_ptr = null, - .is_comptime = false, - .alignment = @alignOf(R), + struct_field_names[num_fields] = name; + struct_field_types[num_fields] = R; + struct_field_attrs[num_fields] = .{ + .@"align" = @alignOf(R), }; // Handle the case `field: ?Tensor = null` // Generic handling of default value is not possible. if (R == ?To) { - struct_fields[num_fields].default_value_ptr = &@as(R, null); + struct_field_attrs[num_fields].default_value_ptr = &@as(R, null); } } num_fields += 1; } } if (num_fields == 0) return void; - return @Type(.{ .@"struct" = .{ - .layout = .auto, - .fields = struct_fields[0..num_fields], - .decls = &.{}, - .is_tuple = struct_infos.is_tuple, - } }); + return if (struct_infos.is_tuple) @Tuple(struct_field_types) else @Struct(.auto, null, struct_field_names, struct_field_types, struct_field_attrs); }, .@"union" => |union_info| { // We know that at least one of the union field contains a From. // We map each field individually. Fields without From, are replaced by "void". const fields = union_info.fields; - var union_fields: [fields.len]std.builtin.Type.UnionField = undefined; + var union_field_names: [fields.len][]const u8 = undefined; + var union_field_types: [fields.len]type = undefined; + var union_field_attrs: [fields.len]std.builtin.Type.UnionField.Attributes = undefined; for (0.., fields) |i, field| { const FT = map(field.type); - union_fields[i] = .{ - .name = field.name, - .type = FT, - .alignment = @alignOf(FT), + union_field_names[i] = field.name; + union_field_types[i] = FT; + union_field_attrs[i] = .{ + .@"align" = @alignOf(FT), }; } - return @Type(.{ .@"union" = .{ - .layout = .auto, - .tag_type = union_info.tag_type, - .fields = union_fields[0..], - .decls = &.{}, - } }); + return @Union(.auto, union_info.tag_type, union_field_names, union_field_types, union_field_attrs); }, .array => |arr_info| [arr_info.len]map(arr_info.child), .pointer => |ptr_info| switch (ptr_info.size) { @@ -429,7 +436,7 @@ test MapRestrict { /// The `v` parameter must me a pointer, and tensor data need to be mutable if callbacks needs it. pub fn VisitReturn(comptime cb: anytype) type { return if (stdx.meta.FnSignature(cb, null).ReturnErrorSet) |error_set| - @Type(std.builtin.Type{ .error_union = .{ .error_set = error_set, .payload = void } }) + error_set!void else void; } diff --git a/zml2/pjrtx.zig b/zml2/pjrtx.zig index 207b5841..104c42db 100644 --- a/zml2/pjrtx.zig +++ b/zml2/pjrtx.zig @@ -1,6 +1,5 @@ const std = @import("std"); -const async = @import("async"); const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); @@ -71,7 +70,8 @@ pub const Client = opaque { } pub fn deserializeAndLoad(self: *const Client, api: *const Api, bytes: []const u8) ApiError!*LoadedExecutable { - return @ptrCast(try async.callBlocking(pjrt.Client.deserializeAndLoad, .{ self.inner(), api, bytes })); + //return @ptrCast(try async.callBlocking(pjrt.Client.deserializeAndLoad, .{ self.inner(), api, bytes })); + return @ptrCast(pjrt.Client.deserializeAndLoad(self.inner(), api, bytes)); } pub const CreateViewOfDeviceBufferArgs = pjrt.Client.CreateViewOfDeviceBufferArgs; @@ -116,7 +116,8 @@ pub const Client = opaque { } pub fn compile(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { - return try async.callBlocking(compileSync, .{ self, api, allocator, module, compile_options_pb }); + //return try async.callBlocking(compileSync, .{ self, api, allocator, module, compile_options_pb }); + return try compileSync(self, api, allocator, module, compile_options_pb); } pub fn addressableMemories(self: *const Client, api: *const Api) []*const Memory { @@ -226,32 +227,35 @@ pub const Event = opaque { } pub fn await(self: *Event, api: *const Api) ApiError!void { - defer self.deinit(api); + _ = self; // autofix + _ = api; // autofix + @panic("Uh oh"); + //defer self.deinit(api); - if (self.isReady(api)) { - return; - } + //if (self.isReady(api)) { + // return; + //} - var ctx = struct { - err: ?*pjrt.Error = null, - event: async.threading.ResetEventSingle = .{}, - }{}; + //var ctx = struct { + // err: ?*pjrt.Error = null, + // event: async.threading.ResetEventSingle = .{}, + //}{}; - try self.inner().onReady(api, &(struct { - fn call(err: ?*pjrt.Error, user_arg: ?*anyopaque) callconv(.c) void { - const ctx_: *@TypeOf(ctx) = @ptrCast(@alignCast(user_arg.?)); - ctx_.err = err; - ctx_.event.set(); - } - }.call), &ctx); - ctx.event.wait(); + //try self.inner().onReady(api, &(struct { + // fn call(err: ?*pjrt.Error, user_arg: ?*anyopaque) callconv(.c) void { + // const ctx_: *@TypeOf(ctx) = @ptrCast(@alignCast(user_arg.?)); + // ctx_.err = err; + // ctx_.event.set(); + // } + //}.call), &ctx); + //ctx.event.wait(); - if (ctx.err) |e| { - defer e.deinit(api); - const err_code = e.getCode(api).toApiError(); - log.err("{t} {s}", .{ err_code, e.getMessage(api) }); - return err_code; - } + //if (ctx.err) |e| { + // defer e.deinit(api); + // const err_code = e.getCode(api).toApiError(); + // log.err("{t} {s}", .{ err_code, e.getMessage(api) }); + // return err_code; + //} } }; diff --git a/zml2/shape.zig b/zml2/shape.zig index e45313c9..c34641b4 100644 --- a/zml2/shape.zig +++ b/zml2/shape.zig @@ -398,7 +398,7 @@ pub const Shape = struct { } else { try writer.print("{d}", .{d}); } - if (self._sharding_info[i]) { + if (@as([MAX_RANK]bool, self._sharding_info)[i]) { try writer.writeByte('!'); } need_comma = true; diff --git a/zml2/zml.zig b/zml2/zml.zig index 03769e68..7a66c137 100644 --- a/zml2/zml.zig +++ b/zml2/zml.zig @@ -33,12 +33,15 @@ var runfiles_once = std.once(struct { return; } + var io: std.Io.Threaded = .init_single_threaded; + defer io.deinit(); + var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); const allocator = arena.allocator(); defer arena.deinit(); var envMap = std.process.EnvMap.init(allocator); - var r = (try runfiles.Runfiles.create(.{ .allocator = allocator })) orelse return; + var r = (try runfiles.Runfiles.create(.{ .allocator = allocator, .io = io.io() })) orelse return; try r.environment(&envMap); var it = envMap.iterator(); From 2178b50c935655ea03a435ffaad4581d58fcd933 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 5 Dec 2025 17:29:48 +0100 Subject: [PATCH 077/881] Moved MAX_RANK from shape.zig to constants.zig This will be the single source of truth for constants throughout the codebase. --- zml2/BUILD.bazel | 1 + zml2/constants.zig | 6 +++ zml2/ops.zig | 23 +++++------ zml2/shape.zig | 37 +++++++++-------- zml2/tensor.zig | 99 +++++++++++++++++++++++----------------------- zml2/zml.zig | 1 + 6 files changed, 87 insertions(+), 80 deletions(-) create mode 100644 zml2/constants.zig diff --git a/zml2/BUILD.bazel b/zml2/BUILD.bazel index 311ee65a..c970566b 100644 --- a/zml2/BUILD.bazel +++ b/zml2/BUILD.bazel @@ -20,6 +20,7 @@ cc_library( zig_library( name = "zml", srcs = [ + "constants.zig", "dtype.zig", "exe.zig", "floats.zig", diff --git a/zml2/constants.zig b/zml2/constants.zig new file mode 100644 index 00000000..c700cb40 --- /dev/null +++ b/zml2/constants.zig @@ -0,0 +1,6 @@ +pub const MAX_RANK: u8 = 8; +pub const AXES_IOTA: [MAX_RANK]i64 = b: { + var values: [MAX_RANK]i64 = undefined; + for (0..MAX_RANK) |i| values[i] = i; + break :b values; +}; diff --git a/zml2/ops.zig b/zml2/ops.zig index fc99d38c..254afcb4 100644 --- a/zml2/ops.zig +++ b/zml2/ops.zig @@ -5,6 +5,7 @@ const meta = @import("meta.zig"); const mlir = @import("mlir"); const mlirx = @import("mlirx.zig"); const dialects = @import("mlir/dialects"); +const constants = @import("constants.zig"); const Tensor = @import("tensor.zig").Tensor; const Shape = @import("shape.zig").Shape; @@ -81,8 +82,8 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func // To that order, we initialize `result` to `inputs`, then we use stdx.meta.visit, // to find the correct mlir.Value, but we first broadcast before creating the final // Tensor struct. - var broadcasting_axes: stdx.BoundedArray(i64, Tensor.MAX_RANK) = .{}; - for (0..Tensor.MAX_RANK) |i| { + var broadcasting_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; + for (0..constants.MAX_RANK) |i| { if (std.mem.indexOfScalar(i64, axes_, @intCast(i)) == null) { broadcasting_axes.append(@intCast(i)) catch unreachable; } @@ -402,8 +403,8 @@ pub fn scatter( } const ScatterConfig = struct { - op_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}, - up_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}, + op_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .{}, + up_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .{}, indices_batch_axes: Shape.DimsArray = .{}, scatter_to_operand_axes: Shape.DimsArray = .{}, updates_transpose: Shape.AxesArray = .{}, @@ -414,11 +415,11 @@ const ScatterAxisKind = enum { batching, update_window, inserted_window, window_ fn scatterConfig( op: Shape, update: Shape, - indices_per_axis: stdx.BoundedArray(Tensor, Tensor.MAX_RANK), + indices_per_axis: stdx.BoundedArray(Tensor, constants.MAX_RANK), indices_axes: Shape.TagsArray, ) ScatterConfig { - var op_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}; - var up_kind: stdx.BoundedArray(ScatterAxisKind, Tensor.MAX_RANK) = .{}; + var op_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .{}; + var up_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .{}; var indices_batch_axes: Shape.DimsArray = .{}; var scatter_to_operand_axes: Shape.DimsArray = .{}; var updates_transpose: Shape.AxesArray = .{}; @@ -548,7 +549,7 @@ fn scatterPrepareIndices( cfg: *ScatterConfig, op: Shape, update: Shape, - indices_per_axis: *stdx.BoundedArray(Tensor, Tensor.MAX_RANK), + indices_per_axis: *stdx.BoundedArray(Tensor, constants.MAX_RANK), indices_axes: *Shape.TagsArray, ) Tensor { var old_scatter_to_op_axes = cfg.scatter_to_operand_axes; @@ -568,7 +569,7 @@ fn scatterPrepareIndices( // Reorder the axes so that in indices_per_axis is ordered like in op if possible. // TODO: transpose updates if needed - var indices: stdx.BoundedArray(Tensor, Tensor.MAX_RANK) = .{}; + var indices: stdx.BoundedArray(Tensor, constants.MAX_RANK) = .{}; var scatter_to_op_axes: Shape.DimsArray = .{}; while (old_scatter_to_op_axes.len > 0) { @@ -700,8 +701,8 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, return Tensor._result(res_shape, gather_op.result(0)); } -pub fn _collectAxes(T: type, bounded_array: stdx.BoundedArray(T, Tensor.MAX_RANK), value: T) stdx.BoundedArray(i64, Tensor.MAX_RANK) { - var res: stdx.BoundedArray(i64, Tensor.MAX_RANK) = .{}; +pub fn _collectAxes(T: type, bounded_array: stdx.BoundedArray(T, constants.MAX_RANK), value: T) stdx.BoundedArray(i64, constants.MAX_RANK) { + var res: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; for (bounded_array.constSlice(), 0..) |v, ax| { if (v == value) { res.appendAssumeCapacity(@intCast(ax)); diff --git a/zml2/shape.zig b/zml2/shape.zig index c34641b4..7435c1da 100644 --- a/zml2/shape.zig +++ b/zml2/shape.zig @@ -3,6 +3,7 @@ const testing = std.testing; const builtin = @import("builtin"); const stdx = @import("stdx"); +const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; @@ -16,18 +17,16 @@ test { /// Represent the shape of a tensor. pub const Shape = struct { - pub const MAX_RANK: u8 = 8; - pub const Tag = [*:0]const u8; pub const TagUnknown = "_".ptr; const TagLast = "last".ptr; - pub const DimsArray = stdx.BoundedArray(i64, MAX_RANK); - pub const TagsArray = stdx.BoundedArray(Tag, MAX_RANK); - pub const AxesArray = stdx.BoundedArray(u3, MAX_RANK); - pub const ShardingInfo = @Vector(MAX_RANK, bool); + pub const DimsArray = stdx.BoundedArray(i64, constants.MAX_RANK); + pub const TagsArray = stdx.BoundedArray(Tag, constants.MAX_RANK); + pub const AxesArray = stdx.BoundedArray(u3, constants.MAX_RANK); + pub const ShardingInfo = @Vector(constants.MAX_RANK, bool); - const UnknownTags: TagsArray = .{ .len = 0, .buffer = [_]Tag{TagUnknown} ** MAX_RANK }; + const UnknownTags: TagsArray = .{ .len = 0, .buffer = [_]Tag{TagUnknown} ** constants.MAX_RANK }; _dtype: DataType, _dims: DimsArray = .{}, @@ -398,7 +397,7 @@ pub const Shape = struct { } else { try writer.print("{d}", .{d}); } - if (@as([MAX_RANK]bool, self._sharding_info)[i]) { + if (@as([constants.MAX_RANK]bool, self._sharding_info)[i]) { try writer.writeByte('!'); } need_comma = true; @@ -525,7 +524,7 @@ pub const Shape = struct { } pub fn insertTag(self: Shape, axis_: anytype, d: i64, tag_: anytype) Shape { - stdx.debug.assert(self.rank() < MAX_RANK - 1, "Can't insert new axis in {f}, it's already at max rank.", .{self}); + stdx.debug.assert(self.rank() < constants.MAX_RANK - 1, "Can't insert new axis in {f}, it's already at max rank.", .{self}); const ax = if (@TypeOf(axis_) == EnumLiteral and axis_ == .last) self.rank() @@ -786,13 +785,13 @@ pub const Shape = struct { } } - pub fn computeStrides(self: Shape) stdx.BoundedArray(i64, MAX_RANK) { + pub fn computeStrides(self: Shape) stdx.BoundedArray(i64, constants.MAX_RANK) { const rk = self.rank(); - var strides: stdx.BoundedArray(i64, MAX_RANK) = .{ .len = rk }; + var strides: stdx.BoundedArray(i64, constants.MAX_RANK) = .{ .len = rk }; if (rk == 0) return strides; - const V = @Vector(MAX_RANK, i64); - const rank_mask = std.simd.iota(u8, MAX_RANK) < @as(@Vector(MAX_RANK, u8), @splat(rk)); + const V = @Vector(constants.MAX_RANK, i64); + const rank_mask = std.simd.iota(u8, constants.MAX_RANK) < @as(@Vector(constants.MAX_RANK, u8), @splat(rk)); // For each axis compute the product of all following dimensions // and the element size in bytes. var d: V = @bitCast(self._dims.buffer); @@ -992,10 +991,10 @@ pub const Shape = struct { return res; } - pub fn parseStruct(T: type, v: anytype) struct { stdx.BoundedArray(T, MAX_RANK), TagsArray } { + pub fn parseStruct(T: type, v: anytype) struct { stdx.BoundedArray(T, constants.MAX_RANK), TagsArray } { const V = @TypeOf(v); - var vals_: stdx.BoundedArray(T, MAX_RANK) = .{}; + var vals_: stdx.BoundedArray(T, constants.MAX_RANK) = .{}; var tags_: TagsArray = .{}; if (comptime stdx.meta.isSliceOf(V, T)) { @@ -1007,7 +1006,7 @@ pub const Shape = struct { if (comptime stdx.meta.isStruct(V)) { const fields = std.meta.fields(V); - stdx.debug.assertComptime(fields.len <= MAX_RANK, "Too many fields in struct {} ({d}). Max supported is {d}.", .{ V, fields.len, MAX_RANK }); + stdx.debug.assertComptime(fields.len <= constants.MAX_RANK, "Too many fields in struct {} ({d}). Max supported is {d}.", .{ V, fields.len, constants.MAX_RANK }); inline for (fields) |field| { const fv = @field(v, field.name); vals_.appendAssumeCapacity(fv); @@ -1030,10 +1029,10 @@ pub const Shape = struct { } /// Parses a struct literal into a list of options for each axes. - pub fn parseAxesOptions(self: Shape, T: type, options: anytype, default: T) stdx.BoundedArray(T, MAX_RANK) { + pub fn parseAxesOptions(self: Shape, T: type, options: anytype, default: T) stdx.BoundedArray(T, constants.MAX_RANK) { const V = @TypeOf(options); - var res: stdx.BoundedArray(T, MAX_RANK) = .{}; + var res: stdx.BoundedArray(T, constants.MAX_RANK) = .{}; if (comptime stdx.meta.isSliceOf(V, T)) { stdx.debug.assert(options.len == self.rank(), "expects exactly {} options in slice, for {} got {}", .{ self.rank(), self, options.len }); for (options) |d| { @@ -1044,7 +1043,7 @@ pub const Shape = struct { if (comptime stdx.meta.isStruct(V)) { for (0..self.rank()) |_| res.appendAssumeCapacity(default); const fields = std.meta.fields(V); - stdx.debug.assertComptime(fields.len <= MAX_RANK, "expects up to {} options struct literal, got {}", .{ V, MAX_RANK, fields.len }); + stdx.debug.assertComptime(fields.len <= constants.MAX_RANK, "expects up to {} options struct literal, got {}", .{ V, constants.MAX_RANK, fields.len }); inline for (fields) |field| { const a = self.axis(field); res.buffer[a] = @field(options, field.name); diff --git a/zml2/tensor.zig b/zml2/tensor.zig index f3c4515e..44ed74e9 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -9,13 +9,13 @@ const mlir = @import("mlir"); const mlirx = @import("mlirx.zig"); const dialects = @import("mlir/dialects"); const ops = @import("ops.zig"); +const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const CompilationContext = @import("module.zig").CompilationContext; pub const Tensor = struct { var current_id: std.atomic.Value(usize) = .{ .raw = 1 }; - pub const MAX_RANK = Shape.MAX_RANK; id: usize, auto_broadcast: bool = false, @@ -81,7 +81,7 @@ pub const Tensor = struct { const ranked_tensor = val.type_().isA(mlir.RankedTensorType).?; const n = ranked_tensor.rank(); - stdx.debug.assert(n <= MAX_RANK, "Can't represent MLIR tensor of rank {}, max supported rank is {}.", .{ n, MAX_RANK }); + stdx.debug.assert(n <= constants.MAX_RANK, "Can't represent MLIR tensor of rank {}, max supported rank is {}.", .{ n, constants.MAX_RANK }); var sh: Shape = .{ ._dtype = mlirx.Type.toDType(mlirCtx(), ranked_tensor.elementType()) }; for (0..n) |i| { @@ -114,7 +114,7 @@ pub const Tensor = struct { /// Returns the indices of each of the given axes. /// /// 'axis_' can be an integer or a tag. - pub fn axes(self: Tensor, axes_: anytype) stdx.BoundedArray(u3, Tensor.MAX_RANK) { + pub fn axes(self: Tensor, axes_: anytype) stdx.BoundedArray(u3, constants.MAX_RANK) { return self._shape.axes(axes_); } @@ -1013,7 +1013,7 @@ pub const Tensor = struct { const lhs_contracting_dim: i8 = @intCast(lhs.shape().hasTag(args).?); const rhs_contracting_dim: i8 = @intCast(rhs.shape().hasTag(args).?); - var batching_axes: stdx.BoundedArray([2]i8, Shape.MAX_RANK) = .{}; + var batching_axes: stdx.BoundedArray([2]i8, constants.MAX_RANK) = .{}; for (0..lhs.rank()) |lhs_tag_index| { const lhs_tag = lhs.shape().tag(lhs_tag_index); if (lhs_tag == Shape.toTag(args)) continue; @@ -1077,7 +1077,7 @@ pub const Tensor = struct { ) Tensor { stdx.debug.assert(lhs.dtype() == rhs.dtype(), "dotGeneral expects tensors to be of the same type, got {} and {}", .{ lhs.dtype(), rhs.dtype() }); - const Axes = stdx.BoundedArray(i64, MAX_RANK); + const Axes = stdx.BoundedArray(i64, constants.MAX_RANK); var res_shape: Shape = .{ ._dtype = lhs.dtype() }; // Validate batching axes @@ -1325,15 +1325,15 @@ pub const Tensor = struct { /// Returns a transposed Tensor computed using the given axes. pub fn transpose(self: Tensor, axes_: anytype) Tensor { const axes__ = self.axes(axes_).constSlice(); - const default_perm = [MAX_RANK]i64{ 7, 6, 5, 4, 3, 2, 1, 0 }; - const no_op = [MAX_RANK]i64{ 0, 1, 2, 3, 4, 5, 6, 7 }; + const default_perm = [constants.MAX_RANK]i64{ 7, 6, 5, 4, 3, 2, 1, 0 }; + const no_op = [constants.MAX_RANK]i64{ 0, 1, 2, 3, 4, 5, 6, 7 }; const permutation: []const i64 = if (axes__.len == 0) - default_perm[MAX_RANK - self.rank() ..] + default_perm[constants.MAX_RANK - self.rank() ..] else toI64(axes__); - stdx.debug.assert(permutation.len == self.rank(), "transpose expects input tensor rank and 'axes_' length to be equal, got {f} and {any}", .{ self, permutation[0..@min(permutation.len, MAX_RANK + 2)] }); + stdx.debug.assert(permutation.len == self.rank(), "transpose expects input tensor rank and 'axes_' length to be equal, got {f} and {any}", .{ self, permutation[0..@min(permutation.len, constants.MAX_RANK + 2)] }); if (std.mem.eql(i64, permutation, no_op[0..self.rank()])) { return self; @@ -1370,7 +1370,7 @@ pub const Tensor = struct { /// unflatten((d0, d1, axis_m, d3), 2, n) -> (d0, d1, n, d2_m, d3) pub fn unflatten(self: Tensor, axis_: i8, n: i64) Tensor { // TODO: move to torch.zig, this equivalent to `spitAxis` - stdx.debug.assert(self.rank() < Tensor.MAX_RANK, "unflatten expects input tensor rank to be less than {}, got {}", .{ Tensor.MAX_RANK, self.rank() }); + stdx.debug.assert(self.rank() < constants.MAX_RANK, "unflatten expects input tensor rank to be less than {}, got {}", .{ constants.MAX_RANK, self.rank() }); const a = if (axis_ >= 0) self.axis(axis_) else self.axis(axis_) + 1; const new_dim = std.math.divExact(i64, self.dim(a), n) catch std.debug.panic("unflatten expects chosen dimension to be divisible by 'n' but {} is not divisible by {}", .{ self.dim(a), n }); @@ -1463,16 +1463,16 @@ pub const Tensor = struct { /// Slices the input Tensor over the given axis using the given parameters. pub fn slice1d(self: Tensor, axis_: anytype, s: Slice) Tensor { - var slices = [_]Slice{.{}} ** MAX_RANK; + var slices = [_]Slice{.{}} ** constants.MAX_RANK; slices[self.axis(axis_)] = s; return self.slice(slices[0..self.rank()]); } /// Slices the input Tensor using the given parameters. pub fn slice(self: Tensor, slices: []const Slice) Tensor { - var start_indices: [MAX_RANK]i64 = undefined; - var strides: [MAX_RANK]i64 = undefined; - var limit_indices: [MAX_RANK]i64 = undefined; + var start_indices: [constants.MAX_RANK]i64 = undefined; + var strides: [constants.MAX_RANK]i64 = undefined; + var limit_indices: [constants.MAX_RANK]i64 = undefined; var res_shape: Shape = self._shape; for (slices, 0..) |s, a| { @@ -1542,7 +1542,7 @@ pub const Tensor = struct { pub fn choose(self: Tensor, offsets: anytype) Tensor { const off, const tags = Shape.parseDimensions(offsets); - var slices = [_]Slice{.{}} ** MAX_RANK; + var slices = [_]Slice{.{}} ** constants.MAX_RANK; for (off.constSlice(), tags.constSlice()) |o, t| { const ax = self.axis(t); slices[ax] = .single(o); @@ -1998,12 +1998,11 @@ pub const Tensor = struct { // TODO: broad is error prone because of this: // it will happily broadcast .{ .a = 10, .b = 1 } to .{ .b = 10, .a = 5 } if (self._shape.rank() == 0 or self._shape.rank() == other.rank()) { - const all_axes = [MAX_RANK]i64{ 0, 1, 2, 3, 4, 5, 6, 7 }; - return self.broadcast(other, all_axes[0..self.rank()]); + return self.broadcast(other, constants.AXES_IOTA[0..self.rank()]); } // check that each axis of self maps to an axis of other - var axes_: stdx.BoundedArray(i64, MAX_RANK) = .{}; + var axes_: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; for (self._shape.tags()) |t| { axes_.appendAssumeCapacity(@intCast(other.axis(t))); } @@ -2035,7 +2034,7 @@ pub const Tensor = struct { pub fn pad(self: Tensor, padding_value: anytype, paddings: anytype) Tensor { const _paddings = self.shape().parseAxesOptions(Pad, paddings, .{}); - const ZEROS = [_]i64{0} ** MAX_RANK; + const ZEROS = [_]i64{0} ** constants.MAX_RANK; var low = ZEROS; var high = ZEROS; var interior = ZEROS; @@ -2073,7 +2072,7 @@ pub const Tensor = struct { self.axis(axis_); var res_shape = self._shape; - const ones = [_]i64{1} ** MAX_RANK; + const ones = [_]i64{1} ** constants.MAX_RANK; res_shape._dims.insertSlice(ax, ones[0..tags_.len]) catch unreachable; res_shape._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; @@ -2250,10 +2249,10 @@ pub const Tensor = struct { // Compute result shape var res_shape = indices._shape.remove(index_coord_axis).withDtype(self.dtype()); var slice_dims = self._shape._dims; - var self_batch_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; - var indices_batch_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; - var start_index_map: stdx.BoundedArray(i64, MAX_RANK) = .{}; - var self_offset_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; + var self_batch_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; + var indices_batch_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; + var start_index_map: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; + var self_offset_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; for (self._shape.tags(), 0..self.rank()) |t, self_ax| { const maybe_slice_ax: ?u3 = if (tagged_api) slice_shape.hasTag(t) else @intCast(self_ax); @@ -2871,7 +2870,7 @@ pub const Tensor = struct { // TODO: support maxPool on non last axis const a = self.axis(-1); - const ones = [_]i64{1} ** Tensor.MAX_RANK; + const ones = [_]i64{1} ** constants.MAX_RANK; var window_dimensions = ones; window_dimensions[a] = opts.window_dimensions; var window_strides = window_dimensions; @@ -2882,7 +2881,7 @@ pub const Tensor = struct { var window_dilations = ones; window_dilations[a] = opts.window_dilations; - var padding = [_][2]i64{.{ 0, 0 }} ** Tensor.MAX_RANK; + var padding = [_][2]i64{.{ 0, 0 }} ** constants.MAX_RANK; padding[a] = opts.padding; const result = ops.reduceWindow( @@ -2921,7 +2920,7 @@ pub const Tensor = struct { const base_dilation = initPoolArg(self.rank(), &opts.base_dilations); const window_dilations = initPoolArg(self.rank(), &opts.window_dilations); - var padding = [_][2]i64{.{ 0, 0 }} ** Tensor.MAX_RANK; + var padding = [_][2]i64{.{ 0, 0 }} ** constants.MAX_RANK; padding[a - 1] = opts.padding[0]; padding[a] = opts.padding[1]; @@ -3077,7 +3076,7 @@ pub const Tensor = struct { const a = self.axis(axis_); const new_shape = self._shape.set(a, slice_.len); - var start_indices = [_]*const mlir.Value{constant(slice_.start.dtype().zero()).value()} ** MAX_RANK; + var start_indices = [_]*const mlir.Value{constant(slice_.start.dtype().zero()).value()} ** constants.MAX_RANK; start_indices[a] = slice_.start.value(); const op = dialects.stablehlo.dynamic_slice( @@ -3118,7 +3117,7 @@ pub const Tensor = struct { const idx_dtype = if (slices.len > 0) slices.get(0).start.dtype() else .i32; const zero = Tensor.scalar(0, idx_dtype).value(); - var offset_values = [_]*const mlir.Value{zero} ** MAX_RANK; + var offset_values = [_]*const mlir.Value{zero} ** constants.MAX_RANK; var res_shape = self._shape; for (slices.constSlice(), 0..) |slice_, i| { const offset = slice_.start; @@ -3172,7 +3171,7 @@ pub const Tensor = struct { /// Note this is the untagged api, if you have tags, you should use dynamicUpdateSlice directly. pub fn dynamicUpdateSlice1d(self: Tensor, update: Tensor, axis_: i64, offset: Tensor) Tensor { const placeholder = Tensor.scalar(0, .i32); - var start_indices = [_]Tensor{placeholder} ** MAX_RANK; + var start_indices = [_]Tensor{placeholder} ** constants.MAX_RANK; start_indices[self.axis(axis_)] = offset; return self.dynamicUpdateSlice(start_indices[0..self.rank()], update); } @@ -3236,7 +3235,7 @@ pub const Tensor = struct { const idx_dtype = if (offset.len > 0) offset.get(0).dtype() else .i32; const zero = Tensor.scalar(0, idx_dtype).value(); - var offset_values: [MAX_RANK]*const mlir.Value = undefined; + var offset_values: [constants.MAX_RANK]*const mlir.Value = undefined; if (offset_tags.len == 0) { // Without offset tags we need the same number of offset than rank. stdx.debug.assert(self.rank() == offset.len, "dynamicUpdateSlice expects input tensor rank and 'offset_' length to be equal, got {} and {}", .{ self.rank(), offset.len }); @@ -3247,7 +3246,7 @@ pub const Tensor = struct { } else { // If an axis isn't specified, update the full slice. // This is only allowed when using tagged sliced. - offset_values = .{zero} ** MAX_RANK; + offset_values = .{zero} ** constants.MAX_RANK; for (offset.constSlice(), offset_tags.constSlice()) |start, t| { const a = self._shape.hasTag(t) orelse stdx.debug.panic("dynamicUpdateSlice expects input tensor to have tags used in 'offset_' but {s} is missing (input shape is {f})", .{ t, self._shape }); offset_values[a] = start.value(); @@ -3390,7 +3389,7 @@ pub const Tensor = struct { /// For each vector in the input tensor, /// creates a diagonal-matrix where diagonal values are set to the vector values. pub fn toDiagonal(self: Tensor, axis_: anytype, new_tags: [2]@TypeOf(.enum_literal)) Tensor { - stdx.debug.assert(self.rank() < MAX_RANK - 1, "toDiagonal expects input up to {d} rank, got {f}", .{ MAX_RANK - 1, self }); + stdx.debug.assert(self.rank() < constants.MAX_RANK - 1, "toDiagonal expects input up to {d} rank, got {f}", .{ constants.MAX_RANK - 1, self }); const a = self.axis(axis_); const d = self.dim(a); var res_shape = self._shape; @@ -3594,7 +3593,7 @@ pub const Tensor = struct { fn _cartesianProduct(vectors: []const Tensor, out: []Tensor) void { stdx.debug.assert(vectors.len >= 1, "cartesianProduct expects at least one input.", .{}); - stdx.debug.assert(vectors.len < Tensor.MAX_RANK, "cartesianProduct expects at most {} input vectors, received {} !", .{ Tensor.MAX_RANK - 1, vectors.len }); + stdx.debug.assert(vectors.len < constants.MAX_RANK, "cartesianProduct expects at most {} input vectors, received {} !", .{ constants.MAX_RANK - 1, vectors.len }); for (vectors) |x| { stdx.debug.assert(x.rank() <= 1, "cartesianProduct expects 0 or 1 rank input vectors. Got: {any}", .{vectors}); stdx.debug.assert(vectors[0].dtype() == x.dtype(), "cartesianProduct expects input vectors to have all the same dtype. Got: {any}", .{vectors}); @@ -3663,7 +3662,7 @@ pub const Tensor = struct { /// /// - res[a, b, c, d] == (A[a], B[b], C[c], D[d]) pub fn cartesianProductStacked(vectors: []const Tensor) Tensor { - var out = stdx.BoundedArray(Tensor, Tensor.MAX_RANK).init(vectors.len) catch unreachable; + var out = stdx.BoundedArray(Tensor, constants.MAX_RANK).init(vectors.len) catch unreachable; _cartesianProduct(vectors, out.slice()); return Tensor.stack(out.constSlice(), .last, .coord); @@ -3752,24 +3751,24 @@ pub const Tensor = struct { } }; -fn initPoolArg(rank: usize, data: []const i64) [Tensor.MAX_RANK]i64 { +fn initPoolArg(rank: usize, data: []const i64) [constants.MAX_RANK]i64 { // TODO use shape - var result = [_]i64{1} ** Tensor.MAX_RANK; + var result = [_]i64{1} ** constants.MAX_RANK; const start = rank - data.len; @memcpy(result[start .. start + data.len], data); return result; } -fn getPoolResDims(dt: DataType, in_dims: []const i64, base_dilations: @Vector(Tensor.MAX_RANK, i64), padding: []const i64, window_dimensions: @Vector(Tensor.MAX_RANK, i64), window_dilations: @Vector(Tensor.MAX_RANK, i64), window_strides: @Vector(Tensor.MAX_RANK, i64)) Shape { +fn getPoolResDims(dt: DataType, in_dims: []const i64, base_dilations: @Vector(constants.MAX_RANK, i64), padding: []const i64, window_dimensions: @Vector(constants.MAX_RANK, i64), window_dilations: @Vector(constants.MAX_RANK, i64), window_strides: @Vector(constants.MAX_RANK, i64)) Shape { // TODO use shape - var input_dims = [_]i64{1} ** Tensor.MAX_RANK; + var input_dims = [_]i64{1} ** constants.MAX_RANK; @memcpy(input_dims[0..in_dims.len], in_dims); - const input_dims_: @Vector(Tensor.MAX_RANK, i64) = input_dims; - const splat_one: @Vector(Tensor.MAX_RANK, i64) = @splat(1); - const dilated_input_shape: @Vector(Tensor.MAX_RANK, i64) = (input_dims_ - splat_one) * base_dilations + splat_one; - var pad_slice0: @Vector(Tensor.MAX_RANK, i64) = @splat(padding[0]); - var pad_slice1: @Vector(Tensor.MAX_RANK, i64) = @splat(padding[0]); + const input_dims_: @Vector(constants.MAX_RANK, i64) = input_dims; + const splat_one: @Vector(constants.MAX_RANK, i64) = @splat(1); + const dilated_input_shape: @Vector(constants.MAX_RANK, i64) = (input_dims_ - splat_one) * base_dilations + splat_one; + var pad_slice0: @Vector(constants.MAX_RANK, i64) = @splat(padding[0]); + var pad_slice1: @Vector(constants.MAX_RANK, i64) = @splat(padding[0]); if (padding.len > 1) { var idx: usize = 0; while (idx < in_dims.len * 2) : (idx += 2) { @@ -3777,10 +3776,10 @@ fn getPoolResDims(dt: DataType, in_dims: []const i64, base_dilations: @Vector(Te pad_slice1[idx / 2] = padding[idx + 1]; } } - const padded_input_shape: @Vector(Tensor.MAX_RANK, i64) = pad_slice0 + dilated_input_shape + pad_slice1; + const padded_input_shape: @Vector(constants.MAX_RANK, i64) = pad_slice0 + dilated_input_shape + pad_slice1; const dilated_window_shape = (window_dimensions - splat_one) * window_dilations + splat_one; const dims = @divFloor(padded_input_shape - dilated_window_shape, window_strides) + splat_one; - const dims_arr: [Tensor.MAX_RANK]i64 = @bitCast(dims); + const dims_arr: [constants.MAX_RANK]i64 = @bitCast(dims); return Shape.init(dims_arr[0..in_dims.len], dt); } @@ -3860,12 +3859,12 @@ fn getComparisonType(ctx: mlir.Context, dtype: DataType) dialects.stablehlo.Comp // ); //} -fn _parseGatherCoord(self: Tensor, axes_: anytype) struct { bool, stdx.BoundedArray(u3, Tensor.MAX_RANK) } { +fn _parseGatherCoord(self: Tensor, axes_: anytype) struct { bool, stdx.BoundedArray(u3, constants.MAX_RANK) } { const AxesT = @TypeOf(axes_); const axes_is_scalar = AxesT == @TypeOf(.enum_literal) or AxesT == comptime_int or @typeInfo(AxesT) == .int; const coord_axes = if (axes_is_scalar) - stdx.BoundedArray(u3, Tensor.MAX_RANK).fromSlice(&.{self.axis(axes_)}) catch unreachable + stdx.BoundedArray(u3, constants.MAX_RANK).fromSlice(&.{self.axis(axes_)}) catch unreachable else self.axes(axes_); @@ -3873,13 +3872,13 @@ fn _parseGatherCoord(self: Tensor, axes_: anytype) struct { bool, stdx.BoundedAr } fn toI64(values: anytype) []i64 { - var res: [Tensor.MAX_RANK]i64 = undefined; + var res: [constants.MAX_RANK]i64 = undefined; for (values, 0..) |val, i| res[i] = @intCast(val); return res[0..values.len]; } fn transposeIsJustAReshape(x: Shape, permutation: []const i64) bool { - var perm: stdx.BoundedArray(struct { u8, bool }, Tensor.MAX_RANK) = .{}; + var perm: stdx.BoundedArray(struct { u8, bool }, constants.MAX_RANK) = .{}; // Don't rewrite on invalid inputs. if (permutation.len > x.rank()) return false; for (permutation) |ax| { diff --git a/zml2/zml.zig b/zml2/zml.zig index 7a66c137..0a9edf02 100644 --- a/zml2/zml.zig +++ b/zml2/zml.zig @@ -26,6 +26,7 @@ pub const shape = @import("shape.zig"); pub const Shape = shape.Shape; pub const tensor = @import("tensor.zig"); pub const Tensor = tensor.Tensor; +pub const constants = @import("constants.zig"); var runfiles_once = std.once(struct { fn call_() !void { From 44fa0bf1257698ca9713f7a5cbbb989b53a11ddc Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 5 Dec 2025 18:55:50 +0100 Subject: [PATCH 078/881] Added std.Io to compile functions --- zml2/main.zig | 11 +++++++---- zml2/module.zig | 11 ++++++----- zml2/pjrtx.zig | 12 ++++++------ 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/zml2/main.zig b/zml2/main.zig index 81a0b054..a63d76ae 100644 --- a/zml2/main.zig +++ b/zml2/main.zig @@ -22,20 +22,23 @@ pub const Model = struct { }; pub fn main() !void { - zml.init(); - defer zml.deinit(); - var gpa: std.heap.DebugAllocator(.{}) = .init; defer _ = gpa.deinit(); const allocator = gpa.allocator(); + var io: std.Io.Threaded = .init(allocator); + defer io.deinit(); + + zml.init(); + defer zml.deinit(); + var platform = try zml.Platform.init(.cpu, .{}); defer platform.deinit(); var model: Model = try .init(); defer model.deinit(); - var exe = try zml.module.compileModel(allocator, Model.forward, model, .{zml.Tensor.init(zml.Shape.init(.{ .n = 4096 }, .f32))}, platform); + var exe = try zml.module.compileModel(allocator, io.io(), Model.forward, model, .{zml.Tensor.init(zml.Shape.init(.{ .n = 4096 }, .f32))}, platform); defer exe.deinit(); } diff --git a/zml2/module.zig b/zml2/module.zig index f9f68daa..678f0c7d 100644 --- a/zml2/module.zig +++ b/zml2/module.zig @@ -161,11 +161,11 @@ pub const CompilationContext = struct { //} }; -pub fn compileModel(allocator: std.mem.Allocator, comptime func: anytype, model: stdx.meta.Head(stdx.meta.FnArgs(func)), args: stdx.meta.Tail(stdx.meta.FnArgs(func)), platform: Platform) !Exe { - return compile(allocator, func, .{model} ++ args, platform); +pub fn compileModel(allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, model: stdx.meta.Head(stdx.meta.FnArgs(func)), args: stdx.meta.Tail(stdx.meta.FnArgs(func)), platform: Platform) !Exe { + return compile(allocator, io, func, .{model} ++ args, platform); } -pub fn compile(allocator: std.mem.Allocator, comptime func: anytype, args: stdx.meta.FnArgs(func), platform: Platform) !Exe { +pub fn compile(allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, args: stdx.meta.FnArgs(func), platform: Platform) !Exe { var compilation_context: CompilationContext = .init(allocator); defer compilation_context.deinit(); @@ -181,7 +181,7 @@ pub fn compile(allocator: std.mem.Allocator, comptime func: anytype, args: stdx. var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); - const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), platform, compilation_context.module, null) catch unreachable; + const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), io, platform, compilation_context.module, null) catch unreachable; log.warn("******** ZML generated MLIR ********", .{}); log.warn("{f}", .{compilation_context.module.operation()}); @@ -402,7 +402,7 @@ fn setXlaOverrideFlag(map: *c.upb_Map, flag: []const u8, value: anytype, upb_are } } -fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, module: *const mlir.Module, xla_dump_to_: ?[]const u8) !*pjrt.LoadedExecutable { +fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: Platform, module: *const mlir.Module, xla_dump_to_: ?[]const u8) !*pjrt.LoadedExecutable { //const tracer = Tracer.init("ai.zml.compilation"); //const compile_frame = tracer.frameStart("pjrt compilation"); //defer tracer.frameEnd(compile_frame, "pjrt compilation"); @@ -478,6 +478,7 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m const loaded_executable = try platform.pjrt_client.compile( platform.pjrt_api, arena, + io, module, try upb.serialize(options, upb_arena), ); diff --git a/zml2/pjrtx.zig b/zml2/pjrtx.zig index 104c42db..5efb9ff0 100644 --- a/zml2/pjrtx.zig +++ b/zml2/pjrtx.zig @@ -69,9 +69,9 @@ pub const Client = opaque { return .{ @ptrCast(buffer), @ptrCast(event_) }; } - pub fn deserializeAndLoad(self: *const Client, api: *const Api, bytes: []const u8) ApiError!*LoadedExecutable { - //return @ptrCast(try async.callBlocking(pjrt.Client.deserializeAndLoad, .{ self.inner(), api, bytes })); - return @ptrCast(pjrt.Client.deserializeAndLoad(self.inner(), api, bytes)); + pub fn deserializeAndLoad(self: *const Client, api: *const Api, io: std.Io, bytes: []const u8) ApiError!*LoadedExecutable { + var future = io.async(pjrt.Client.deserializeAndLoad, .{ self.inner(), api, bytes }); + return @ptrCast(future.await(io)); } pub const CreateViewOfDeviceBufferArgs = pjrt.Client.CreateViewOfDeviceBufferArgs; @@ -115,9 +115,9 @@ pub const Client = opaque { })); } - pub fn compile(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { - //return try async.callBlocking(compileSync, .{ self, api, allocator, module, compile_options_pb }); - return try compileSync(self, api, allocator, module, compile_options_pb); + pub fn compile(self: *const Client, api: *const Api, allocator: std.mem.Allocator, io: std.Io, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { + var future = io.async(compileSync, .{ self, api, allocator, module, compile_options_pb }); + return future.await(io); } pub fn addressableMemories(self: *const Client, api: *const Api) []*const Memory { From 9cd56326e36e63c9c0050b5d664053dfe37acdb3 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 8 Dec 2025 11:59:46 +0100 Subject: [PATCH 079/881] Basic support of zml.Buffer --- pjrt/pjrt.zig | 12 ++--- zml2/BUILD.bazel | 1 + zml2/buffer.zig | 119 +++++++++++++++++++++++++++++++++++++++++++++++ zml2/pjrtx.zig | 69 ++++++++++++++++----------- zml2/zml.zig | 1 + 5 files changed, 168 insertions(+), 34 deletions(-) create mode 100644 zml2/buffer.zig diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index cdbb7cd3..27834290 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -242,7 +242,7 @@ pub const ErrorCode = enum(c.PJRT_Error_Code) { pub const Error = opaque { pub fn deinit(self: *Error, api: *const Api) void { _ = api.call(.PJRT_Error_Destroy, .{ - .@"error" = @ptrCast(self), + .@"error" = @as(*c.PJRT_Error, @ptrCast(self)), }) catch unreachable; } @@ -378,13 +378,13 @@ pub const Client = opaque { .client = self.inner(), .data = @constCast(args.data), .type = @intFromEnum(args.buffer_type), - .dims = @ptrCast(@constCast(args.dims.ptr)), + .dims = @as([*c]const i64, @ptrCast(@constCast(args.dims.ptr))), .num_dims = args.dims.len, - .byte_strides = if (args.byte_strides) |bs| @ptrCast(@constCast(bs.ptr)) else null, + .byte_strides = @as([*c]const i64, if (args.byte_strides) |bs| @ptrCast(@constCast(bs.ptr)) else null), .num_byte_strides = if (args.byte_strides) |bs| bs.len else 0, .host_buffer_semantics = @intFromEnum(args.host_buffer_semantics), - .device = if (args.dst == .device) @ptrCast(@constCast(args.dst.device)) else null, - .memory = if (args.dst == .memory) @ptrCast(@constCast(args.dst.memory)) else null, + .device = @as(?*c.PJRT_Device, if (args.dst == .device) @ptrCast(@constCast(args.dst.device)) else null), + .memory = @as(?*c.PJRT_Memory, if (args.dst == .memory) @ptrCast(@constCast(args.dst.memory)) else null), .device_layout = null, // TODO .done_with_host_buffer = null, // out .buffer = null, // out @@ -1058,7 +1058,7 @@ pub const Event = opaque { pub fn onReady(self: *Event, api: *const Api, func: *const fn (err: ?*Error, user_arg: ?*anyopaque) callconv(.c) void, user_arg: ?*anyopaque) ApiError!void { _ = try api.call(.PJRT_Event_OnReady, .{ .event = self.inner(), - .callback = @ptrCast(func), + .callback = @as(c.PJRT_Event_OnReadyCallback, @ptrCast(func)), .user_arg = user_arg, }); } diff --git a/zml2/BUILD.bazel b/zml2/BUILD.bazel index c970566b..8ef9a5a6 100644 --- a/zml2/BUILD.bazel +++ b/zml2/BUILD.bazel @@ -20,6 +20,7 @@ cc_library( zig_library( name = "zml", srcs = [ + "buffer.zig", "constants.zig", "dtype.zig", "exe.zig", diff --git a/zml2/buffer.zig b/zml2/buffer.zig new file mode 100644 index 00000000..3449333c --- /dev/null +++ b/zml2/buffer.zig @@ -0,0 +1,119 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const pjrt = @import("pjrtx.zig"); + +const Shape = @import("shape.zig").Shape; +const Platform = @import("platform.zig").Platform; +const Target = @import("platform.zig").Target; + +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(Buffer); +} + +const log = std.log.scoped(.zml); + +/// Buffer is a multi-dimension array, whose memory is allocated on an accelerator. +/// +/// * contains a handle that the ZML runtime can use to convert into a physical address, but there is no guarantee this address is visible from the CPU. +/// * loading weights from disk directly to the `device zml.aio.loadBuffers` +/// * can be created by calling `HostBuffer.toDevice(platform)`. +pub const Buffer = struct { + _shape: Shape, + _api: *const pjrt.Api, + _target: Target, + _shards: Shards, + + pub const MAX_NUM_SHARDS: u8 = Platform.MAX_NUM_DEVICES; + pub const Shards = stdx.BoundedArray(*pjrt.Buffer, MAX_NUM_SHARDS); + + pub const Memory = pjrt.Memory.Kind; + pub const FromOptions = struct { wait: bool = true, memory: Memory = .device }; + + /// Frees the accelerator memory. + /// Depending on the platform, the memory is typically not released to the OS + /// but just marked as available in the memory pool. + pub fn deinit(self: *const Buffer) void { + // log.warn("Unloading {f} {d} bytes", .{ self._shape, self._shape.byteSize() }); + for (self._shards.constSlice()) |buffer| { + buffer.deinit(self._api); + } + } + + /// Copies the content of the given buffer from host memory to the accelerator memory. + pub fn from(platform: Platform, shape_: Shape, data_: []const u8, io: std.Io, opts: FromOptions) !Buffer { + var res: Buffer = .{ + ._api = platform.pjrt_api, + ._target = platform.target, + ._shape = shape_, + ._shards = .{}, + }; + + // We shard only on the first axis so that the chunks are still contiguous. + // TODO: support more advanced sharding specs + stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); + const sharding_ax: ?u3 = std.simd.firstTrue(shape_._sharding_info); + const n_partitions = platform.sharding().num_partitions; + const chunk_size = if (sharding_ax) |ax| cs: { + // This kind of sharding error should be detected earlier on. + stdx.debug.assert(@rem(shape_.dim(ax), n_partitions) == 0, "Buffer.from({f}) expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ shape_, ax, n_partitions }); + break :cs @divExact(shape_.dim(ax), n_partitions); + } else 0; + _ = chunk_size; // autofix + + const buffer_type = pjrt.bufferTypeFromDtype(shape_.dtype()); + const byte_strides = shape_.computeStrides(); + + const devices = platform.getDevices(); + for (0..n_partitions) |i| { + // If no sharding if found, the given buffer is replicated on all devices. + //const buf = if (sharding_ax) |ax| buf: { + // @panic("sharding not implemented"); + // //const start: i64 = @as(i64, @intCast(i)) * chunk_size; + // //break :buf host_buffer.slice1d(ax, .{ .start = start, .end = start + chunk_size }); + //} else host_buffer; + + const args = pjrt.Client.BufferFromHostBufferArgs{ + .data = data_.ptr, + .buffer_type = buffer_type, + .dims = shape_.dims(), + .byte_strides = byte_strides.slice(), + .host_buffer_semantics = .ImmutableUntilTransferCompletes, + // CPU has no distinctions between memories. + .dst = if (platform.target == .cpu or opts.memory == .device) + .{ .device = devices[i] } + else + .{ .memory = platform.memoryForDevice(opts.memory, devices[i]) }, + }; + + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); + + if (event) |ev| ev.deinit(platform.pjrt_api); + res._shards.appendAssumeCapacity(pjrt_buffer); + } + + if (opts.wait) { + res = try res.await(io); + } + + return res; + } + + /// Copies the given Zig slice to the accelerator memory and + /// return a Buffer with the given dimensions. + pub fn fromBytes(platform: Platform, sh: Shape, data: []const u8, io: std.Io) !Buffer { + return from(platform, sh, data, io, .{}); + } + + pub fn await(self: Buffer, io: std.Io) !Buffer { + for (self._shards.constSlice()) |buffer| { + if (buffer.getReadyEvent(self._api)) |ev| { + try ev.await(self._api, io); + } + } + + return self; + } +}; diff --git a/zml2/pjrtx.zig b/zml2/pjrtx.zig index 5efb9ff0..ea1474a5 100644 --- a/zml2/pjrtx.zig +++ b/zml2/pjrtx.zig @@ -219,43 +219,56 @@ pub const Event = opaque { return self.inner().getEventError(api); } - pub fn awaitBlocking(self: *Event, api: *const Api) ApiError!void { + pub fn awaitBlocking(self: *Event, api: *const Api, io: std.Io) ApiError!void { if (self.isReady(api)) { return; } - try self.inner().await(api); + try self.inner().await(api, io); } - pub fn await(self: *Event, api: *const Api) ApiError!void { - _ = self; // autofix - _ = api; // autofix - @panic("Uh oh"); - //defer self.deinit(api); + pub fn await(self: *Event, api: *const Api, io: std.Io) ApiError!void { + defer self.deinit(api); - //if (self.isReady(api)) { - // return; - //} + if (self.isReady(api)) { + return; + } - //var ctx = struct { - // err: ?*pjrt.Error = null, - // event: async.threading.ResetEventSingle = .{}, - //}{}; + var mutex: std.Io.Mutex = .init; + var condvar: std.Io.Condition = .{}; + var done: bool = false; - //try self.inner().onReady(api, &(struct { - // fn call(err: ?*pjrt.Error, user_arg: ?*anyopaque) callconv(.c) void { - // const ctx_: *@TypeOf(ctx) = @ptrCast(@alignCast(user_arg.?)); - // ctx_.err = err; - // ctx_.event.set(); - // } - //}.call), &ctx); - //ctx.event.wait(); + var ctx = struct { + err: ?*pjrt.Error = null, + mutex: *std.Io.Mutex, + condvar: *std.Io.Condition, + done: *bool, + io: std.Io, + }{ .mutex = &mutex, .condvar = &condvar, .done = &done, .io = io }; - //if (ctx.err) |e| { - // defer e.deinit(api); - // const err_code = e.getCode(api).toApiError(); - // log.err("{t} {s}", .{ err_code, e.getMessage(api) }); - // return err_code; - //} + try self.inner().onReady(api, &(struct { + fn call(err: ?*pjrt.Error, user_arg: ?*anyopaque) callconv(.c) void { + const ctx_: *@TypeOf(ctx) = @ptrCast(@alignCast(user_arg.?)); + ctx_.err = err; + { + ctx_.mutex.lock(ctx_.io) catch unreachable; + defer ctx_.mutex.unlock(ctx_.io); + ctx_.done.* = true; + } + + ctx_.condvar.signal(ctx_.io); + } + }.call), &ctx); + mutex.lock(io) catch unreachable; + while (!done) { + ctx.condvar.wait(io, &mutex) catch unreachable; + } + + if (ctx.err) |e| { + defer e.deinit(api); + const err_code = e.getCode(api).toApiError(); + log.err("{t} {s}", .{ err_code, e.getMessage(api) }); + return err_code; + } } }; diff --git a/zml2/zml.zig b/zml2/zml.zig index 0a9edf02..71b87330 100644 --- a/zml2/zml.zig +++ b/zml2/zml.zig @@ -27,6 +27,7 @@ pub const Shape = shape.Shape; pub const tensor = @import("tensor.zig"); pub const Tensor = tensor.Tensor; pub const constants = @import("constants.zig"); +pub const Buffer = @import("buffer.zig").Buffer; var runfiles_once = std.once(struct { fn call_() !void { From 45c94dd5b5ac892ab75f4802d5347c08fdeffbd7 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 8 Dec 2025 17:45:18 +0100 Subject: [PATCH 080/881] Small fix in stablehlo.zig --- mlir/dialects/stablehlo/stablehlo.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index b9165367..0cad2e65 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -217,7 +217,7 @@ pub fn constant( .operands = .{ .flat = &.{} }, .results = .{ .flat = &.{mlir.rankedTensorType(dims, elem_type)} }, .attributes = &.{ - .named(ctx, "value", mlir.denseElementsAttribute(mlir.RankedTensorType.get(dims, elem_type).shaped(), raw_bytes)), + .named(ctx, "value", mlir.denseElementsAttribute(mlir.RankedTensorType.get(dims, elem_type, null).shaped(), raw_bytes)), }, .location = location, }); From 86b5bb84dfb4af00dce310345c02e9528a77ee71 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 8 Dec 2025 17:46:17 +0100 Subject: [PATCH 081/881] Various small fixes --- pjrt/pjrt.zig | 16 ++++++++-------- stdx/fmt.zig | 12 ++++++------ zml2/floats.zig | 2 +- zml2/meta.zig | 14 ++++++++++---- 4 files changed, 25 insertions(+), 19 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 27834290..05aee495 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -779,18 +779,18 @@ pub const LoadedExecutable = opaque { .num_send_ops = 0, .num_recv_ops = 0, .launch_id = 0, - .non_donatable_input_indices = @ptrCast(args.non_donatable_input_indices.ptr), + .non_donatable_input_indices = @as([*c]const i64, @ptrCast(args.non_donatable_input_indices.ptr)), .num_non_donatable_input_indices = args.non_donatable_input_indices.len, - .context = @ptrCast(args.context), + .context = @as(?*c.PJRT_ExecuteContext, @ptrCast(args.context)), }); _ = try api.call(.PJRT_LoadedExecutable_Execute, .{ .executable = self.inner(), - .options = @ptrCast(&options), - .argument_lists = @ptrCast(args.arguments.ptr), - .num_devices = @intCast(args.arguments.len), + .options = @as(*c.PJRT_ExecuteOptions, @ptrCast(&options)), + .argument_lists = @as([*c]const [*c]const ?*c.PJRT_Buffer, @ptrCast(args.arguments.ptr)), + .num_devices = @as(usize, @intCast(args.arguments.len)), .num_args = args.num_args, - .output_lists = @ptrCast(args.results.ptr), - .device_complete_events = @ptrCast(args.events.ptr), + .output_lists = @as([*c]const [*c]?*c.PJRT_Buffer, @ptrCast(args.results.ptr)), + .device_complete_events = @as([*c]?*c.PJRT_Event, @ptrCast(args.events.ptr)), .execute_device = null, }); } @@ -929,7 +929,7 @@ pub const Buffer = opaque { pub fn toHostBuffer(self: *const Buffer, api: *const Api, dst: []u8) ApiError!?*Event { const ret = try api.call(.PJRT_Buffer_ToHostBuffer, .{ .src = self.inner(), - .dst = @ptrCast(dst.ptr), + .dst = dst.ptr, .dst_size = dst.len, }); return @ptrCast(ret.event); diff --git a/stdx/fmt.zig b/stdx/fmt.zig index e9df9131..16dd843a 100644 --- a/stdx/fmt.zig +++ b/stdx/fmt.zig @@ -9,11 +9,11 @@ fn FmtSlice(T: type) type { return struct { slice: []const T, - pub fn format(f: @This(), writer: *std.io.Writer) std.io.Writer.Error!void { + pub fn format(f: @This(), writer: *std.Io.Writer) std.Io.Writer.Error!void { return try formatSliceAny(f.slice, .{}, 1, writer); } - pub fn formatNumber(f: @This(), writer: *std.io.Writer, n: std.fmt.Number) std.io.Writer.Error!void { + pub fn formatNumber(f: @This(), writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { return switch (@typeInfo(T)) { .comptime_float, .float => try formatFloatSlice(f.slice, n, 1, writer), .comptime_int, .int => try formatIntSlice(f.slice, n, 1, writer), @@ -132,19 +132,19 @@ fn FmtAny(Data: type) type { return struct { data: Data, - pub inline fn format(self: @This(), writer: *std.io.Writer) std.io.Writer.Error!void { + pub inline fn format(self: @This(), writer: *std.Io.Writer) std.Io.Writer.Error!void { try printValue(writer, .{}, self.data, std.options.fmt_max_depth); } }; } -/// Fix up of std.io.Writer.printValue that uses `format` method of subfields when possible +/// Fix up of std.Io.Writer.printValue that uses `format` method of subfields when possible fn printValue( - w: *std.io.Writer, + w: *std.Io.Writer, options: std.fmt.Options, value: anytype, max_depth: usize, -) std.io.Writer.Error!void { +) std.Io.Writer.Error!void { const T = @TypeOf(value); if (std.meta.hasMethod(T, "format")) { return try value.format(w); diff --git a/zml2/floats.zig b/zml2/floats.zig index 54240485..8a60bc0b 100644 --- a/zml2/floats.zig +++ b/zml2/floats.zig @@ -107,7 +107,7 @@ fn FloatHelpers(Float: type) type { return @as(T, x.mantissa) << f32_mantissa_bits - mantissa_bits; } - pub fn formatNumber(x: Float, writer: *std.io.Writer, n: std.fmt.Number) std.io.Writer.Error!void { + pub fn formatNumber(x: Float, writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { switch (n.mode) { .binary, .octal, .hex => try writer.print("{{ .sign={}, .exp={}, .mantissa={} }}", .{ x.sign, x.exponent, x.mantissa }), else => try writer.printFloat(x.toF32(), n), diff --git a/zml2/meta.zig b/zml2/meta.zig index e22f7f9e..6933479f 100644 --- a/zml2/meta.zig +++ b/zml2/meta.zig @@ -66,7 +66,7 @@ pub fn MapType(From: type, To: type) type { } } if (same) return T; - return if (struct_infos.is_tuple) @Tuple(struct_field_types) else @Struct(.auto, null, struct_field_names, struct_field_types, struct_field_attrs); + return if (struct_infos.is_tuple) @Tuple(&struct_field_types) else @Struct(.auto, null, &struct_field_names, &struct_field_types, &struct_field_attrs); }, .@"union" => |union_info| { const fields = union_info.fields; @@ -92,7 +92,7 @@ pub fn MapType(From: type, To: type) type { } } if (same) return T; - return @Union(.auto, union_info.tag_type, union_field_names, union_field_types, union_field_attrs); + return @Union(.auto, union_info.tag_type, &union_field_names, &union_field_types, &union_field_attrs); }, .array => |arr_info| [arr_info.len]map(arr_info.child), .pointer => |ptr_info| switch (ptr_info.size) { @@ -361,7 +361,13 @@ pub fn MapRestrict(From: type, To: type) type { } } if (num_fields == 0) return void; - return if (struct_infos.is_tuple) @Tuple(struct_field_types) else @Struct(.auto, null, struct_field_names, struct_field_types, struct_field_attrs); + return if (struct_infos.is_tuple) @Tuple(struct_field_types[0..num_fields]) else @Struct( + .auto, + null, + struct_field_names[0..num_fields], + struct_field_types[0..num_fields], + struct_field_attrs[0..num_fields], + ); }, .@"union" => |union_info| { // We know that at least one of the union field contains a From. @@ -378,7 +384,7 @@ pub fn MapRestrict(From: type, To: type) type { .@"align" = @alignOf(FT), }; } - return @Union(.auto, union_info.tag_type, union_field_names, union_field_types, union_field_attrs); + return @Union(.auto, union_info.tag_type, &union_field_names, &union_field_types, &union_field_attrs); }, .array => |arr_info| [arr_info.len]map(arr_info.child), .pointer => |ptr_info| switch (ptr_info.size) { From 054642f5b21d1f1e2cd9d2ce47286db9b04b9ee4 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 8 Dec 2025 17:46:50 +0100 Subject: [PATCH 082/881] Restored basic test * Only one Tensor test has been restored for now, Tensor.fmod * Re-introduced helpers for testing --- MODULE.bazel | 2 +- zml2/BUILD.bazel | 21 +++- zml2/buffer.zig | 35 +++++++ zml2/exe.zig | 8 +- zml2/tensor.zig | 67 +++++++------ zml2/test_runner.zig | 111 +++++++++++++++++++++ zml2/testing.zig | 225 +++++++++++++++++++++++++++++++++++++++++++ zml2/zml.zig | 12 +++ 8 files changed, 444 insertions(+), 37 deletions(-) create mode 100644 zml2/test_runner.zig create mode 100644 zml2/testing.zig diff --git a/MODULE.bazel b/MODULE.bazel index 1f94b406..176b7ffc 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -227,7 +227,7 @@ use_repo(huggingface, "ModernBERT-base", "ModernBERT-large") git_override( module_name = "rules_zig", - commit = "566daebc8655bbf3ef18f187556daa83928afc84", + commit = "15cf8dfd3007025a607fe228d16cdc106a307aed", remote = "https://github.com/zml/rules_zig.git", ) diff --git a/zml2/BUILD.bazel b/zml2/BUILD.bazel index 8ef9a5a6..44e2350a 100644 --- a/zml2/BUILD.bazel +++ b/zml2/BUILD.bazel @@ -1,6 +1,6 @@ load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_c_proto_library") load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_test") upb_c_proto_library( name = "xla_data_upb", @@ -33,6 +33,7 @@ zig_library( "platform.zig", "shape.zig", "tensor.zig", + "testing.zig", "zml.zig", ], main = "zml.zig", @@ -64,3 +65,21 @@ zig_binary( "//zml2:zml", ], ) + +# All ZML Tests + +zig_test( + name = "test", + data = [ + #"aio/torch/simple.pt", + #"aio/torch/simple_test_4.pickle", + ], + test_runner = ":test_runner", + deps = ["//zml2:zml"], +) + +filegroup( + name = "test_runner", + srcs = ["test_runner.zig"], + visibility = ["//visibility:public"], +) diff --git a/zml2/buffer.zig b/zml2/buffer.zig index 3449333c..b23abc51 100644 --- a/zml2/buffer.zig +++ b/zml2/buffer.zig @@ -42,6 +42,11 @@ pub const Buffer = struct { } } + /// This Buffer shape. + pub fn shape(self: Buffer) Shape { + return self._shape; + } + /// Copies the content of the given buffer from host memory to the accelerator memory. pub fn from(platform: Platform, shape_: Shape, data_: []const u8, io: std.Io, opts: FromOptions) !Buffer { var res: Buffer = .{ @@ -116,4 +121,34 @@ pub const Buffer = struct { return self; } + + /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. + pub fn fromPjrtBuffers(platform: Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { + stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); + stdx.debug.assert(pjrt_buffers.len > 0, "fromPjrtBuffers expects at least one buffer, got 0.", .{}); + var shards: Shards = .{}; + shards.appendSliceAssumeCapacity(pjrt_buffers); + return .{ + ._api = platform.pjrt_api, + ._target = platform.target, + ._shape = shape_, + ._shards = shards, + }; + } + + /// Copies the content of the Buffer to the host. + /// The returned `HostBuffer` does own the memory. + pub fn toHostAlloc(self: Buffer, allocator: std.mem.Allocator, io: std.Io) ![]u8 { + //const output = try allocator.alignedAlloc(u8, .fromByteUnits(self._shape.dtype().alignOf()), self._shape.byteSize()); + //const output = try allocator.alignedAlloc(u8, .@"64", self._shape.byteSize()); + const output = try allocator.alloc(u8, self._shape.byteSize()); + errdefer allocator.free(output); + + //stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); + const maybe_event = try self._shards.get(0).toHostBuffer(self._api, output); + if (maybe_event) |event| { + try event.await(self._api, io); + } + return output; + } }; diff --git a/zml2/exe.zig b/zml2/exe.zig index 23dffafa..d31b9581 100644 --- a/zml2/exe.zig +++ b/zml2/exe.zig @@ -6,9 +6,7 @@ const meta = @import("meta.zig"); const Shape = @import("shape.zig").Shape; const Platform = @import("platform.zig").Platform; - -// TODO(Corentin) -const Buffer = struct {}; +const Buffer = @import("buffer.zig").Buffer; pub const Exe = struct { platform: Platform, @@ -199,7 +197,7 @@ pub const Exe = struct { } }; - pub fn call(self: *const Exe, arguments: Arguments, results_: *Results) void { + pub fn call(self: *const Exe, arguments: Arguments, results_: *Results, io: std.Io) void { var events = [_]?*pjrt.Event{null} ** Platform.MAX_NUM_DEVICES; const sharding = self.platform.sharding(); @@ -219,7 +217,7 @@ pub const Exe = struct { for (events[0..sharding.num_partitions]) |e| { if (e) |ev| { - ev.await_(self.platform.pjrt_api) catch unreachable; + ev.await(self.platform.pjrt_api, io) catch unreachable; } } } diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 44ed74e9..74b8b60c 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -270,22 +270,28 @@ pub const Tensor = struct { return self.remainder(scalar(divisor, .f32).broadcast(self._shape, &.{})); } - // TODO(Corentin) - //test fmod { - // const zml = @import("zml.zig"); - // const platform = zml.testing.env(); + test fmod { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); - // const inputs: [2][6]f32 = .{ .{ -3.0, -2, -1, 1, 2, 3 }, .{ 1, 2, 3, 4, 5, -5 } }; - // const expectations: [2][6]f32 = .{ .{ -1.0, -0.0, -1.0, 1.0, 0.0, 1.0 }, .{ 1.0000, 0.5000, 0.0000, 1.0000, 0.5000, -0.5000 } }; - // const divisors: [2]f32 = .{ 2, -1.5 }; + const inputs: [2][6]f32 = .{ .{ -3.0, -2, -1, 1, 2, 3 }, .{ 1, 2, 3, 4, 5, -5 } }; + const expectations: [2][6]f32 = .{ .{ -1.0, -0.0, -1.0, 1.0, 0.0, 1.0 }, .{ 1.0000, 0.5000, 0.0000, 1.0000, 0.5000, -0.5000 } }; + const divisors: [2]f32 = .{ 2, -1.5 }; - // inline for (inputs, expectations, divisors) |i, e, d| { - // const input = try zml.Buffer.fromSlice(platform, .{6}, &i); - // const output = try zml.testing.compileAndCall(platform, Tensor.fmod, .{ input, d }); + inline for (inputs, expectations, divisors) |i, e, d| { + const input: zml.Tensor = .init(Shape.init(.{6}, .f32)); - // try zml.testing.expectClose(zml.HostBuffer.fromSlice(.{6}, &e), output, 1e-4); - // } - //} + const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.fmod, .{ input, d }, platform); + defer exe.deinit(); + + const input_buffer = try zml.Buffer.fromBytes(platform, input.shape(), std.mem.sliceAsBytes(&i), std.testing.io); + defer input_buffer.deinit(); + + const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.fmod, .{input_buffer}); + + try zml.testing.expectClose(std.testing.io, zml.testing.Slice.init(zml.Shape.init(.{6}, .f32), std.mem.sliceAsBytes(&e)), output, 1e-4); + } + } /// Returns a Tensor containing the element-wise left-shift operation of 'self' by 'other'. pub fn shiftLeft(self: Tensor, other: Tensor) Tensor { @@ -1876,7 +1882,7 @@ pub const Tensor = struct { .float => stdx.debug.assert(!std.math.isNan(val), "scalar(NaN) is probably due to compiling a model with an uninitialized field", .{}), else => {}, } - return Tensor.constant(.{}, data); + return Tensor.constant(data); } // TODO(Corentin) @@ -1905,7 +1911,7 @@ pub const Tensor = struct { val.constSlice(), .unknown(mlirCtx()), ).appendTo(currentBlock()); - return _result(.init(&.{}, val.dtype()), op.result(0)).autoBroadcast(); + return _result(.init(&.{}, val.dtype()), op.result(0)); } pub fn zeroes(sh: Shape) Tensor { @@ -3918,18 +3924,19 @@ test transposeIsJustAReshape { try std.testing.expect(transposeIsJustAReshape(Shape.init(.{ 1, 10, 155, 1 }, .f32), &.{ 0, 1, 3, 2 })); } -test "unused tensor" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const Local = struct { - pub fn _fwd(x: Tensor) Tensor { - const y = x.addConstant(1); - _ = y; - return x; - } - }; - - const mod = try zml.compileFn(std.testing.allocator, Local._fwd, .{Shape.init(.{10}, .f32)}, platform); - defer mod.deinit(); -} +// TODO(Corentin) +//test "unused tensor" { +// const zml = @import("zml.zig"); +// const platform = zml.testing.env(); +// +// const Local = struct { +// pub fn _fwd(x: Tensor) Tensor { +// const y = x.addConstant(1); +// _ = y; +// return x; +// } +// }; +// +// const mod = try zml.compileFn(std.testing.allocator, Local._fwd, .{Shape.init(.{10}, .f32)}, platform); +// defer mod.deinit(); +// diff --git a/zml2/test_runner.zig b/zml2/test_runner.zig new file mode 100644 index 00000000..e3c801f3 --- /dev/null +++ b/zml2/test_runner.zig @@ -0,0 +1,111 @@ +//! Test runner for unit test based on https://github.com/ziglang/zig/blob/master/lib/compiler/test_runner.zig with async +const std = @import("std"); +const testing = std.testing; +const builtin = @import("builtin"); + +// note: std_options.log_level does not respect testing.log_level +// ref: https://github.com/ziglang/zig/issues/5738 +const log_level: std.log.Level = .warn; + +pub const std_options: std.Options = .{ + .log_level = log_level, +}; + +var log_err_count: usize = 0; +var fba_buffer: [8192]u8 = undefined; +var fba = std.heap.FixedBufferAllocator.init(&fba_buffer); + +pub fn main() !void { + testing.log_level = log_level; + + const test_fn_list: []const std.builtin.TestFn = builtin.test_functions; + var ok_count: usize = 0; + var skip_count: usize = 0; + var fail_count: usize = 0; + const root_node = std.Progress.start(.{ + .root_name = "Test", + .estimated_total_items = test_fn_list.len, + }); + const have_tty = std.fs.File.stderr().isTty(); + + var args = std.process.args(); + // Skip executable path + _ = args.next().?; + + const identifier_query = if (args.next()) |arg| blk: { + std.debug.print("Only tests with identifiers that includes `{s}` will be run\n", .{arg}); + break :blk arg; + } else blk: { + break :blk ""; + }; + + var leaks: usize = 0; + + for (test_fn_list, 0..) |test_fn, i| { + if (std.mem.indexOf(u8, test_fn.name, identifier_query) == null) { + continue; + } + + testing.allocator_instance = .{}; + defer { + if (testing.allocator_instance.deinit() == .leak) { + leaks += 1; + } + } + + testing.io_instance = .init(testing.allocator_instance.allocator()); + defer testing.io_instance.deinit(); + + const test_node = root_node.start(test_fn.name, 0); + if (!have_tty) { + std.debug.print("{d}/{d} {s}...", .{ i + 1, test_fn_list.len, test_fn.name }); + } + if (test_fn.func()) |_| { + ok_count += 1; + test_node.end(); + if (!have_tty) std.debug.print("OK\n", .{}); + } else |err| switch (err) { + error.SkipZigTest => { + skip_count += 1; + if (have_tty) { + std.debug.print("{d}/{d} {s}...SKIP\n", .{ i + 1, test_fn_list.len, test_fn.name }); + } else { + std.debug.print("SKIP\n", .{}); + } + test_node.end(); + }, + else => { + fail_count += 1; + if (have_tty) { + std.debug.print("{d}/{d} {s}...FAIL ({s})\n", .{ + i + 1, test_fn_list.len, test_fn.name, @errorName(err), + }); + } else { + std.debug.print("FAIL ({s})\n", .{@errorName(err)}); + } + if (@errorReturnTrace()) |trace| { + std.debug.dumpStackTrace(trace); + } + test_node.end(); + }, + } + } + root_node.end(); + if (ok_count == test_fn_list.len) { + std.debug.print("All {d} tests passed.\n", .{ok_count}); + } else { + std.debug.print("{d} passed; {d} skipped; {d} failed.\n", .{ ok_count, skip_count, fail_count }); + } + if (log_err_count != 0) { + std.debug.print("{d} errors were logged.\n", .{log_err_count}); + } + if (leaks != 0) { + std.debug.print("{d} tests leaked memory.\n", .{leaks}); + } + if (leaks != 0 or log_err_count != 0 or fail_count != 0) { + std.process.exit(1); + } + + // Explicit exit 0 to exit from async main thread and main properly + std.process.exit(0); +} diff --git a/zml2/testing.zig b/zml2/testing.zig new file mode 100644 index 00000000..2d69734e --- /dev/null +++ b/zml2/testing.zig @@ -0,0 +1,225 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const stdx = @import("stdx"); + +const Platform = @import("platform.zig").Platform; +const zml = @import("zml.zig"); + +const log = std.log.scoped(.@"zml/testing"); + +var _platform: ?Platform = null; + +pub fn env() Platform { + if (!builtin.is_test) @compileError("Cannot use zml.testing.env outside of a test block"); + if (_platform == null) { + zml.init(); + + _platform = Platform.init(.cpu, .{}) catch unreachable; + } + + return _platform.?; +} + +pub const Slice = struct { + data: []const u8, + shape: zml.Shape, + + pub fn init(shape: zml.Shape, data: []const u8) Slice { + return .{ .data = data, .shape = shape }; + } + + pub fn dtype(self: Slice) zml.DataType { + return self.shape.dtype(); + } + + pub fn items(self: Slice, comptime T: type) []const T { + return @alignCast(std.mem.bytesAsSlice(T, self.data)); + } + + pub fn format( + self: @This(), + writer: *std.Io.Writer, + ) std.Io.Writer.Error!void { + return writer.print("{any}", .{self}); + } + + pub fn formatNumber(self: @This(), writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + _ = self; // autofix + _ = n; // autofix + return writer.print("TODO", .{}); + } +}; + +/// In neural network we generally care about the relative precision, +/// but on a given dimension, if the output is close to 0, then the precision +/// don't matter as much. +pub fn approxEq(comptime Float: type, l: Float, r: Float, tolerance: Float) bool { + const closeRel = std.math.approxEqRel(Float, l, r, @floatCast(tolerance)); + const closeAbs = std.math.approxEqAbs(Float, l, r, @floatCast(tolerance / 2)); + return closeRel or closeAbs; +} + +/// Testing utility. Accepts both zml.Buffer and zml.HostBuffer but zml.Buffer will be copied to the +/// host for comparison ! +pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, tolerance: f32) !void { + const allocator = if (builtin.is_test) std.testing.allocator else std.heap.smp_allocator; + var left: Slice, const should_free_left = if (@TypeOf(left_) == zml.Buffer) b: { + const bytes = try left_.toHostAlloc(allocator, io); + break :b .{ .init(left_.shape(), bytes), true }; + } else .{ left_, false }; + + var right: Slice, const should_free_right = if (@TypeOf(right_) == zml.Buffer) b: { + const bytes = try right_.toHostAlloc(allocator, io); + break :b .{ .init(right_.shape(), bytes), true }; + } else .{ right_, false }; + + defer { + if (should_free_left) allocator.free(left.data); + if (should_free_right) allocator.free(right.data); + } + errdefer log.err("\n--> Left: {0f}{0d:24.3}\n--> Right: {1f}{1d:24.3}", .{ left, right }); + if (!std.mem.eql(i64, left.shape.dims(), right.shape.dims())) { + log.err("left.shape() {f} != right.shape() {f}", .{ left.shape, right.shape }); + return error.TestUnexpectedResult; + } + if (left.dtype() != right.dtype() and !(left.dtype() == .f16 and right.dtype() == .bf16)) { + log.err("left.dtype ({}) != right.dtype ({})", .{ left.dtype(), right.dtype() }); + return error.TestUnexpectedResult; + } + + switch (left.dtype()) { + inline .bf16, + .f16, + .f32, + .f64, + .f4e2m1, + .f8e3m4, + .f8e4m3, + .f8e4m3b11fnuz, + .f8e4m3fn, + .f8e4m3fnuz, + .f8e5m2, + .f8e5m2fnuz, + .f8e8m0, + => |t| { + const L = t.toZigType(); + const left_data = left.items(L); + switch (right.dtype()) { + inline .bf16, + .f16, + .f32, + .f64, + .f8e4m3b11fnuz, + .f8e4m3fn, + .f8e4m3fnuz, + .f8e5m2, + .f8e5m2fnuz, + => |rt| { + const R = rt.toZigType(); + const right_data = right.items(R); + for (left_data, right_data, 0..) |l, r, i| { + if (!approxEq(f32, zml.floats.floatCast(f32, l), zml.floats.floatCast(f32, r), tolerance)) { + log.err("left.data != right_data.\n < {d:40.3} \n > {d:40.3}\n error at idx {d}: {d:.3} != {d:.3}", .{ stdx.fmt.slice(center(left_data, i)), stdx.fmt.slice(center(right_data, i)), i, left_data[i], right_data[i] }); + return error.TestUnexpectedResult; + } + } + }, + else => unreachable, + } + }, + inline .bool, .u2, .u4, .u8, .u16, .u32, .u64, .i2, .i4, .i8, .i16, .i32, .i64 => |t| { + const T = t.toZigType(); + return std.testing.expectEqualSlices(T, left.items(T), right.items(T)); + }, + .c64, .c128 => @panic("TODO: support comparison of complex"), + } +} + +fn BufferizedWithArgs(comptime T: type) type { + return zml.meta.MapType(zml.Tensor, zml.Buffer).map(T); +} + +/// Compile a function and immediatly call it with the given buffers. +/// The compiled module is discarded after the call. +/// Useful during testing when a module is typically called only once. +/// +/// Note: `func` needs explicit types on all parameters. +/// To test a function with `anytype` (typically for tagged API), you need to create a specialized version of it with specific types. +pub fn compileAndCall( + io: std.Io, + platform: zml.Platform, + func: anytype, + buffer_and_args: BufferizedWithArgs(stdx.meta.FnArgs(func)), +) !zml.Bufferized(stdx.meta.FnResult(func)) { + // This simplify test API and also ensure this fn isn't used outside of tests. + const allocator = std.testing.allocator; + + var shape_list: std.array_list.Managed(zml.Shape) = .init(allocator); + defer shape_list.deinit(); + const LocalContext1 = struct { + shape_list: *std.array_list.Managed(zml.Shape), + }; + var context1: LocalContext1 = .{ .shape_list = &shape_list }; + try zml.meta.visit(struct { + fn cb(context_: *LocalContext1, buffer: *const zml.Buffer) !void { + try context_.shape_list.append(buffer.shape()); + } + }.cb, &context1, &buffer_and_args); + + var func_args: stdx.meta.FnArgs(func) = undefined; + const LocalContext2 = struct { + shape_list: *std.array_list.Managed(zml.Shape), + index: usize = 0, + }; + var context2: LocalContext2 = .{ .shape_list = &shape_list }; + try zml.meta.visit(struct { + fn cb(context_: *LocalContext2, tensor: *zml.Tensor) !void { + tensor.* = zml.Tensor.init(context_.shape_list.items[context_.index]); + context_.index += 1; + } + }.cb, &context2, &func_args); + + var exe = try zml.module.compile(allocator, io, func, func_args, platform); + defer exe.deinit(); + + var args = try exe.args(allocator); + defer args.deinit(allocator); + + args.set(buffer_and_args); + + var results = try exe.results(allocator); + defer results.deinit(allocator); + + exe.call(args, &results, io); + + var result: zml.Bufferized(stdx.meta.FnResult(func)) = undefined; + results.fill(&result); + return result; +} + +/// Automatically calls the executable with the given arguments, taking care of arguments and results allocation. +/// This helper can be used in tests to make the code a bit less verbose. +/// It doesn't handle pointers in inputs/outputs structs. +pub fn autoCall(allocator: std.mem.Allocator, io: std.Io, exe: *const zml.exe.Exe, func: anytype, inputs: zml.Bufferized(stdx.meta.FnArgs(func))) !zml.Bufferized(stdx.meta.FnResult(func)) { + var args = try exe.args(allocator); + defer args.deinit(allocator); + + var results = try exe.results(allocator); + defer results.deinit(allocator); + + args.set(inputs); + exe.call(args, &results, io); + + var output: zml.Bufferized(stdx.meta.FnResult(func)) = undefined; + results.fill(&output); + + return output; +} + +fn center(slice: anytype, i: usize) @TypeOf(slice) { + const c = 20; + const start = if (i < c) 0 else i - c; + const end = @min(start + 2 * c, slice.len); + return slice[start..end]; +} diff --git a/zml2/zml.zig b/zml2/zml.zig index 71b87330..2db5f1e2 100644 --- a/zml2/zml.zig +++ b/zml2/zml.zig @@ -28,6 +28,7 @@ pub const tensor = @import("tensor.zig"); pub const Tensor = tensor.Tensor; pub const constants = @import("constants.zig"); pub const Buffer = @import("buffer.zig").Buffer; +pub const testing = @import("testing.zig"); var runfiles_once = std.once(struct { fn call_() !void { @@ -65,3 +66,14 @@ pub fn init() void { } pub fn deinit() void {} + +/// Return a clone of a type with Tensors replaced by Buffer. +/// Non-Tensor metadata is stripped out of the resulting struct. +/// Recursively descends into the type. +pub fn Bufferized(comptime T: type) type { + return meta.MapRestrict(Tensor, Buffer).map(T); +} + +test "zml" { + _ = Tensor; +} From 6391597f82089699d594d6613d67cae9da113a47 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 14:58:05 +0100 Subject: [PATCH 083/881] Cleanup pass on floats.zig --- zml2/floats.zig | 207 +++++++++++++++++++++++++++++------------------- 1 file changed, 125 insertions(+), 82 deletions(-) diff --git a/zml2/floats.zig b/zml2/floats.zig index 8a60bc0b..fe3dce87 100644 --- a/zml2/floats.zig +++ b/zml2/floats.zig @@ -1,11 +1,11 @@ ///! Conversion utilities between different Floating point formats. const std = @import("std"); -test { - std.testing.refAllDecls(@This()); -} +//test { +// std.testing.refAllDecls(@This()); +//} -fn FloatHelpers(Float: type) type { +fn FloatConversionHelpers(Float: type) type { const info = @typeInfo(Float); const err_msg = "FloatHelpers expect a packed struct { mantissa: uXX, exponent: uXX, sign: u1}"; if (info != .@"struct" or info.@"struct".backing_integer == null) { @@ -18,22 +18,14 @@ fn FloatHelpers(Float: type) type { } } + const zero = FloatHelpers(Float).zero; + return struct { const mantissa_bits: u8 = @typeInfo(@FieldType(Float, "mantissa")).int.bits; const exponent_bits: u8 = @typeInfo(@FieldType(Float, "exponent")).int.bits; const f32_mantissa_bits: u8 = @typeInfo(@FieldType(Float32, "mantissa")).int.bits; const exp_bias: i16 = std.math.maxInt(std.meta.Int(.unsigned, exponent_bits - 1)); - const exp_off: u8 = FloatHelpers(Float32).exp_bias - exp_bias; - - pub const zero: Float = .{ .sign = 0, .exponent = 0, .mantissa = 0 }; - - pub fn neg(x: Float) Float { - return .{ - .sign = x.sign ^ 1, - .exponent = x.exponent, - .mantissa = x.mantissa, - }; - } + const exp_off: u8 = FloatConversionHelpers(Float32).exp_bias - exp_bias; /// Lossy conversion from f32, similar to @floatCast pub fn fromF32(f: f32) Float { @@ -46,8 +38,11 @@ fn FloatHelpers(Float: type) type { @branchHint(.unlikely); return if (@hasDecl(Float, "inf")) if (vf32.sign == 0) Float.inf else Float.minus_inf - else - Float.nan; + else if (@hasDecl(Float, "nan")) + Float.nan + else b: { + break :b if (vf32.sign == 0) Float.max else Float.min; + }; } return if (exponent <= 0) @@ -106,6 +101,32 @@ fn FloatHelpers(Float: type) type { const T = @FieldType(Float32, "mantissa"); return @as(T, x.mantissa) << f32_mantissa_bits - mantissa_bits; } + }; +} + +fn FloatHelpers(Float: type) type { + const info = @typeInfo(Float); + const err_msg = "FloatHelpers expect a packed struct { mantissa: uXX, exponent: uXX, sign: u1}"; + if (info != .@"struct" or info.@"struct".backing_integer == null) { + @compileError(err_msg); + } + comptime { + for (info.@"struct".fields, &.{ "mantissa", "exponent", "sign" }) |field, expected_name| { + if (!std.mem.eql(u8, field.name, expected_name)) + @compileError(err_msg); + } + } + + return struct { + pub const zero: Float = .{ .sign = 0, .exponent = 0, .mantissa = 0 }; + + pub fn neg(x: Float) Float { + return .{ + .sign = x.sign ^ 1, + .exponent = x.exponent, + .mantissa = x.mantissa, + }; + } pub fn formatNumber(x: Float, writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { switch (n.mode) { @@ -121,14 +142,28 @@ pub const Float32 = packed struct(u32) { exponent: u8, sign: u1, + pub const nan: Float32 = .{ .sign = 0, .exponent = std.math.maxInt(u8), .mantissa = 1 }; + pub const max: Float32 = .{ .sign = 0, .exponent = std.math.maxInt(u8) - 1, .mantissa = std.math.maxInt(u23) }; + pub const min: Float32 = neg(max); pub const inf: Float32 = .{ .sign = 0, .exponent = std.math.maxInt(u8), .mantissa = 0 }; pub const minus_inf = neg(inf); + pub fn isNan(self: Float32) bool { + return self.exponent == nan.exponent and self.mantissa != 0; + } + const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; pub const neg = Helpers.neg; - pub const fromF32 = Helpers.fromF32; - pub const toF32 = Helpers.toF32; + + pub fn fromF32(f: f32) Float32 { + return @bitCast(f); + } + + pub fn toF32(x: Float32) f32 { + return @bitCast(x); + } + pub const formatNumber = Helpers.formatNumber; }; @@ -139,11 +174,27 @@ pub const Float64 = packed struct(u64) { exponent: u11, sign: u1, + pub const nan: Float64 = .{ .sign = 0, .exponent = std.math.maxInt(u11), .mantissa = 1 }; + pub const max: Float64 = .{ .sign = 0, .exponent = std.math.maxInt(u11) - 1, .mantissa = std.math.maxInt(u32) }; + pub const min: Float64 = neg(max); + pub const inf: Float64 = .{ .sign = 0, .exponent = std.math.maxInt(u11), .mantissa = 0 }; + pub const minus_inf = neg(inf); + + pub fn isNan(self: Float64) bool { + return self.exponent == nan.exponent and self.mantissa != 0; + } + const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; pub const neg = Helpers.neg; - pub const fromF32 = Helpers.fromF32; - pub const toF32 = Helpers.toF32; + pub fn fromF32(f: f32) Float64 { + return @bitCast(@as(f64, @floatCast(f))); + } + + pub fn toF32(x: Float64) f32 { + return @floatCast(@as(f64, @bitCast(x))); + } + pub const formatNumber = Helpers.formatNumber; }; @@ -153,16 +204,18 @@ pub const Float8E4M3B11FNUZ = packed struct(u8) { sign: u1, pub const nan: Float8E4M3B11FNUZ = .{ .sign = 1, .exponent = 0, .mantissa = 0 }; + pub const max: Float8E4M3B11FNUZ = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = std.math.maxInt(u3) }; + pub const min: Float8E4M3B11FNUZ = neg(max); pub fn isNan(self: Float8E4M3B11FNUZ) bool { - return self.sign == 1 and self.exponent == 0 and self.mantissa == 0; + return self == nan; } const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; pub const neg = Helpers.neg; - pub const fromF32 = Helpers.fromF32; - pub const toF32 = Helpers.toF32; + pub const fromF32 = FloatConversionHelpers(@This()).fromF32; + pub const toF32 = FloatConversionHelpers(@This()).toF32; pub const formatNumber = Helpers.formatNumber; }; @@ -172,15 +225,18 @@ pub const Float8E4M3FN = packed struct(u8) { sign: u1, pub const nan: Float8E4M3FN = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = std.math.maxInt(u3) }; + pub const max: Float8E4M3FN = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = 0b110 }; + pub const min: Float8E4M3FN = neg(max); pub fn isNan(self: Float8E4M3FN) bool { return self.exponent == nan.exponent and self.mantissa == nan.mantissa; } + const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; pub const neg = Helpers.neg; - pub const fromF32 = Helpers.fromF32; - pub const toF32 = Helpers.toF32; + pub const fromF32 = FloatConversionHelpers(@This()).fromF32; + pub const toF32 = FloatConversionHelpers(@This()).toF32; pub const formatNumber = Helpers.formatNumber; }; @@ -190,16 +246,18 @@ pub const Float8E4M3FNUZ = packed struct(u8) { sign: u1, pub const nan: Float8E4M3FNUZ = .{ .sign = 1, .exponent = 0, .mantissa = 0 }; + pub const max: Float8E4M3FNUZ = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = std.math.maxInt(u3) }; + pub const min: Float8E4M3FNUZ = neg(max); pub fn isNan(self: Float8E4M3FNUZ) bool { - return self.sign == 1 and self.exponent == 0 and self.mantissa == 0; + return self == nan; } const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; pub const neg = Helpers.neg; - pub const fromF32 = Helpers.fromF32; - pub const toF32 = Helpers.toF32; + pub const fromF32 = FloatConversionHelpers(@This()).fromF32; + pub const toF32 = FloatConversionHelpers(@This()).toF32; pub const formatNumber = Helpers.formatNumber; }; @@ -229,25 +287,21 @@ pub const Float8E5M2 = packed struct(u8) { exponent: u5, sign: u1, - pub const nan: Float8E5M2 = .{ .sign = 0, .exponent = std.math.maxInt(u5), .mantissa = 1 }; + pub const nan: Float8E5M2 = .{ .sign = 0, .exponent = std.math.maxInt(u5), .mantissa = 0b01 }; + pub const max: Float8E5M2 = .{ .sign = 0, .exponent = 0b11110, .mantissa = std.math.maxInt(u2) }; + pub const min: Float8E5M2 = neg(max); + pub const inf: Float8E5M2 = .{ .sign = 0, .exponent = std.math.maxInt(u5), .mantissa = 0 }; + pub const minus_inf: Float8E5M2 = .neg(inf); pub fn isNan(self: Float8E5M2) bool { return self.exponent == nan.exponent and self.mantissa != 0; } - pub const inf: Float8E5M2 = .{ - .sign = 0, - .exponent = std.math.maxInt(u5), - .mantissa = 0, - }; - - pub const minus_inf: Float8E5M2 = .neg(inf); - const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; pub const neg = Helpers.neg; - pub const fromF32 = Helpers.fromF32; - pub const toF32 = Helpers.toF32; + pub const fromF32 = FloatConversionHelpers(@This()).fromF32; + pub const toF32 = FloatConversionHelpers(@This()).toF32; pub const formatNumber = Helpers.formatNumber; }; @@ -257,16 +311,18 @@ pub const Float8E5M2FNUZ = packed struct(u8) { sign: u1, pub const nan: Float8E5M2FNUZ = .{ .sign = 1, .exponent = 0, .mantissa = 0 }; + pub const max: Float8E5M2FNUZ = .{ .sign = 0, .exponent = std.math.maxInt(u5), .mantissa = std.math.maxInt(u2) }; + pub const min: Float8E5M2FNUZ = neg(max); pub fn isNan(self: Float8E5M2FNUZ) bool { - return self.sign == 1 and self.exponent == 0 and self.mantissa == 0; + return self == nan; } const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; pub const neg = Helpers.neg; - pub const fromF32 = Helpers.fromF32; - pub const toF32 = Helpers.toF32; + pub const fromF32 = FloatConversionHelpers(@This()).fromF32; + pub const toF32 = FloatConversionHelpers(@This()).toF32; pub const formatNumber = Helpers.formatNumber; }; @@ -286,19 +342,15 @@ pub const BFloat16 = packed struct(u16) { sign: u1, pub const nan: BFloat16 = .{ .sign = 0, .exponent = std.math.maxInt(u8), .mantissa = 1 }; + pub const max: BFloat16 = .{ .sign = 0, .exponent = 0b11111110, .mantissa = std.math.maxInt(u7) }; + pub const min: BFloat16 = neg(max); + pub const inf: BFloat16 = .{ .sign = 0, .exponent = std.math.maxInt(u8), .mantissa = 0 }; + pub const minus_inf: BFloat16 = .neg(inf); pub fn isNan(self: BFloat16) bool { - return allBitsOne(self.exponent) and self.mantissa != 0; + return self.exponent == nan.exponent and self.mantissa != 0; } - pub const inf: BFloat16 = .{ - .sign = 0, - .exponent = std.math.maxInt(u8), - .mantissa = 0, - }; - - pub const minus_inf: BFloat16 = .neg(inf); - // Specialized versions of to/from F32. Since BFloat16 has the same exponent range than F32, // no overflow/underflow can happen, simplifiying conversion logic. pub fn toF32(self: BFloat16) f32 { @@ -340,25 +392,21 @@ pub const Float8E4M3 = packed struct(u8) { exponent: u4, sign: u1, - pub const nan: Float8E4M3 = @bitCast(0xFF); + pub const nan: Float8E4M3 = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = 1 }; + pub const max: Float8E4M3 = .{ .sign = 0, .exponent = 0b1110, .mantissa = std.math.maxInt(u3) }; + pub const min: Float8E4M3 = neg(max); + pub const inf: Float8E4M3 = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = 0 }; + pub const minus_inf = neg(inf); pub fn isNan(self: Float8E4M3) bool { - return self == nan or self == comptime nan.neg(); + return self.exponent == nan.exponent and self.mantissa != 0; } - pub const inf: Float8E4M3 = .{ - .sign = 0, - .exponent = std.math.maxInt(u4), - .mantissa = 0, - }; - - pub const minus_inf = neg(inf); - const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; pub const neg = Helpers.neg; - pub const fromF32 = Helpers.fromF32; - pub const toF32 = Helpers.toF32; + pub const fromF32 = FloatConversionHelpers(@This()).fromF32; + pub const toF32 = FloatConversionHelpers(@This()).toF32; pub const formatNumber = Helpers.formatNumber; }; @@ -367,25 +415,21 @@ pub const Float8E3M4 = packed struct(u8) { exponent: u3, sign: u1, - pub const nan: Float8E3M4 = @bitCast(0xFF); + pub const nan: Float8E3M4 = .{ .sign = 0, .exponent = std.math.maxInt(u3), .mantissa = 1 }; + pub const max: Float8E3M4 = .{ .sign = 0, .exponent = 0b110, .mantissa = std.math.maxInt(u4) }; + pub const min: Float8E3M4 = neg(max); + pub const inf: Float8E3M4 = .{ .sign = 0, .exponent = std.math.maxInt(u3), .mantissa = 0 }; + pub const minus_inf = neg(inf); pub fn isNan(self: Float8E3M4) bool { - return self == nan or self == comptime nan.neg(); + return self.exponent == nan.exponent and self.mantissa != 0; } - pub const inf: Float8E3M4 = .{ - .sign = 0, - .exponent = std.math.maxInt(u3), - .mantissa = 0, - }; - - pub const minus_inf = neg(inf); - const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; pub const neg = Helpers.neg; - pub const fromF32 = Helpers.fromF32; - pub const toF32 = Helpers.toF32; + pub const fromF32 = FloatConversionHelpers(@This()).fromF32; + pub const toF32 = FloatConversionHelpers(@This()).toF32; pub const formatNumber = Helpers.formatNumber; }; @@ -394,6 +438,8 @@ pub const Float8E8M0 = packed struct(u8) { exponent: u8, sign: u0 = 0, + pub const max: Float8E8M0 = .{ .exponent = std.math.maxInt(u8) }; + pub const min: Float8E8M0 = .{ .exponent = 0 }; pub const min_scale: f32 = @bitCast(Float32{ .sign = 0, .exponent = 0, .mantissa = 0b1 << 22 }); /// Lossy conversion from f32, similar to @floatCast @@ -432,11 +478,12 @@ pub const Float4E2M1 = packed struct(u4) { exponent: u2, sign: u1, - pub const nan: Float4E2M1 = @bitCast(@as(u4, 0xF)); + pub const max: Float4E2M1 = .{ .sign = 0, .exponent = std.math.maxInt(u2), .mantissa = std.math.maxInt(u1) }; + pub const min: Float4E2M1 = neg(max); const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; pub const neg = Helpers.neg; - pub const fromF32 = Helpers.fromF32; + pub const fromF32 = FloatConversionHelpers(@This()).fromF32; pub const formatNumber = Helpers.formatNumber; pub const values = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; @@ -490,10 +537,6 @@ pub fn isInf(x: anytype) bool { return @as(FBits, @bitCast(x)) & remove_sign == @as(FBits, @bitCast(Float.inf)); } -fn allBitsOne(v: anytype) bool { - return v == std.math.maxInt(@TypeOf(v)); -} - const TestCase = struct { lossless: []const f32, lossy: []const f32, From 32d995ae10f8ba4ac8bcd251e624952f5edbbf24 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 14:58:30 +0100 Subject: [PATCH 084/881] Fixed DataType.minValue() and .maxValue() --- zml2/dtype.zig | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zml2/dtype.zig b/zml2/dtype.zig index cb8af137..78fa853c 100644 --- a/zml2/dtype.zig +++ b/zml2/dtype.zig @@ -206,19 +206,19 @@ pub const DataType = enum(u8) { pub fn minValue(dtype: DataType) Value { return switch (dtype) { .bool => .{ .bool = false }, - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).zero), - inline .f8e5m2, .bf16 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).minus_inf), + inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz, .f8e8m0, .f4e2m1 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).min), + inline .f8e5m2, .f8e3m4, .f8e4m3, .bf16 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).minus_inf), inline .f16, .f32, .f64 => |tag| @unionInit(Value, @tagName(tag), -std.math.inf(@FieldType(Value, @tagName(tag)))), inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Value, @tagName(tag), std.math.minInt(@FieldType(Value, @tagName(tag)))), - inline else => |tag| @panic("Unsupported type: " ++ @tagName(tag)), + inline .c64, .c128 => |tag| @panic("DataType doesn't have a min value: " ++ @tagName(tag)), }; } pub fn maxValue(dtype: DataType) Value { return switch (dtype) { .bool => .{ .bool = true }, - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), - inline .f8e5m2, .bf16 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).inf), + inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz, .f8e8m0, .f4e2m1 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).max), + inline .f8e5m2, .f8e3m4, .f8e4m3, .bf16 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).inf), inline .f16, .f32, .f64 => |tag| @unionInit(Value, @tagName(tag), std.math.inf(@FieldType(Value, @tagName(tag)))), inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Value, @tagName(tag), std.math.maxInt(@FieldType(Value, @tagName(tag)))), inline .c64, .c128 => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), From 0916e7d7deaa297d5869795b4822fa4add4873db Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 14:58:56 +0100 Subject: [PATCH 085/881] Various PJRT fixes in bindings --- pjrt/ffi.zig | 28 +++++++++---------- pjrt/pjrt.zig | 75 ++++++++++++++++++++++++++++---------------------- zml2/pjrtx.zig | 27 +++++++++--------- 3 files changed, 69 insertions(+), 61 deletions(-) diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index 6a49294e..8d28bd66 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -82,7 +82,7 @@ pub const Api = opaque { pub const inner = TransmuteMixin(Api, c.XLA_FFI_Api).to; pub fn stream(self: *const Api, context: *const ExecutionContext) *pjrt.Stream { - var ret = pjrt.pjrtStruct(c.XLA_FFI_Stream_Get_Args{ + var ret = pjrt.pjrtStruct2(c.XLA_FFI_Stream_Get_Args, .{ .ctx = @constCast(context.inner()), }); const result = self.inner().XLA_FFI_Stream_Get.?(&ret); @@ -98,7 +98,7 @@ pub const Api = opaque { } pub fn allocateDeviceMemory(self: *const Api, context: *const ExecutionContext, size: usize, alignment: usize) pjrt.ApiError!*anyopaque { - var ret = pjrt.pjrtStruct(c.XLA_FFI_DeviceMemory_Allocate_Args{ + var ret = pjrt.pjrtStruct2(c.XLA_FFI_DeviceMemory_Allocate_Args, .{ .ctx = @constCast(context.inner()), .size = size, .alignment = alignment, @@ -118,7 +118,7 @@ pub const Api = opaque { } pub fn freeDeviceMemory(self: *const Api, context: *const ExecutionContext, data: *anyopaque, size: usize) pjrt.ApiError!void { - var ret = pjrt.pjrtStruct(c.XLA_FFI_DeviceMemory_Free_Args{ + var ret = pjrt.pjrtStruct2(c.XLA_FFI_DeviceMemory_Free_Args, .{ .ctx = @constCast(context.inner()), .size = size, .data = data, @@ -177,7 +177,7 @@ pub const ExecutionContext = opaque { } pub fn getDeviceOrdinal(self: *const ExecutionContext, api: *const Api) pjrt.ApiError!i32 { - var ret = pjrt.pjrtStruct(c.XLA_FFI_DeviceOrdinal_Get_Args{ + var ret = pjrt.pjrtStruct2(c.XLA_FFI_DeviceOrdinal_Get_Args, .{ .ctx = @constCast(self.inner()), }); const result = api.inner().XLA_FFI_DeviceOrdinal_Get.?(&ret); @@ -197,10 +197,10 @@ pub const ExecutionContext = opaque { const Task = fn (*anyopaque) void; pub fn scheduleTask(self: *const ExecutionContext, api: *const Api, task: *const Task, data: *anyopaque) pjrt.ApiError!void { - var ret = pjrt.pjrtStruct(c.XLA_FFI_ThreadPool_Schedule_Args{ + var ret = pjrt.pjrtStruct2(c.XLA_FFI_ThreadPool_Schedule_Args, .{ .ctx = @constCast(self.inner()), - .task = @ptrCast(@alignCast(task)), - .data = @ptrCast(@alignCast(data)), + .task = @as(?*const c.XLA_FFI_Task, @ptrCast(@alignCast(task))), + .data = data, }); const result = api.inner().XLA_FFI_ThreadPool_Schedule.?(&ret); @@ -422,7 +422,7 @@ pub const ErrorCode = enum(c.XLA_FFI_Error_Code) { .unknown => error.Unknown, .invalid_argument => error.InvalidArgument, .deadline_exceeded => error.DeadlineExceeded, - .not_found => error.FfiNotFound, + .not_found => error.NotFound, .already_exists => error.AlreadyExists, .permission_denied => error.PermissionDenied, .resource_exhausted => error.ResourceExhausted, @@ -445,7 +445,7 @@ pub const Error = opaque { pub const ok: ?*Error = null; pub fn create(api: *const Api, error_code: ErrorCode, message: []const u8) *Error { - var ret = pjrt.pjrtStruct(c.XLA_FFI_Error_Create_Args{ + var ret = pjrt.pjrtStruct2(c.XLA_FFI_Error_Create_Args, .{ .message = message.ptr, .errc = @intFromEnum(error_code), }); @@ -453,12 +453,12 @@ pub const Error = opaque { } pub fn destroy(err: *Error, api: *const Api) void { - var ret = pjrt.pjrtStruct(c.XLA_FFI_Error_Destroy_Args{ .@"error" = err.inner() }); + var ret = pjrt.pjrtStruct2(c.XLA_FFI_Error_Destroy_Args, .{ .@"error" = err.inner() }); api.inner().XLA_FFI_Error_Destroy.?(&ret); } pub fn getMessage(err: *Error, api: *const Api) [:0]const u8 { - var ret = pjrt.pjrtStruct(c.XLA_FFI_Error_GetMessage_Args{ + var ret = pjrt.pjrtStruct2(c.XLA_FFI_Error_GetMessage_Args, .{ .@"error" = err.inner(), }); api.inner().XLA_FFI_Error_GetMessage.?(&ret); @@ -471,7 +471,7 @@ pub const Future = opaque { pub const fromInner = TransmuteMixin(Future, c.XLA_FFI_Future).from; pub fn create(api: *const Api) pjrt.ApiError!*Future { - var ret = pjrt.pjrtStruct(c.XLA_FFI_Future_Create_Args{}); + var ret = pjrt.pjrtStruct2(c.XLA_FFI_Future_Create_Args, .{}); const result = api.inner().XLA_FFI_Future_Create.?(&ret); if (result) |ffi_error| { @@ -487,7 +487,7 @@ pub const Future = opaque { } pub fn setAvailable(self: *Future, api: *const Api) pjrt.ApiError!void { - var ret = pjrt.pjrtStruct(c.XLA_FFI_Future_SetAvailable_Args{ + var ret = pjrt.pjrtStruct2(c.XLA_FFI_Future_SetAvailable_Args, .{ .future = self.inner(), }); @@ -504,7 +504,7 @@ pub const Future = opaque { } pub fn setError(self: *Future, api: *const Api, err: *Error) pjrt.ApiError!void { - var ret = pjrt.pjrtStruct(c.XLA_FFI_Future_SetError_Args{ + var ret = pjrt.pjrtStruct2(c.XLA_FFI_Future_SetError_Args, .{ .future = self.inner(), .@"error" = err.inner(), }); diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 05aee495..b9bc3e93 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -422,15 +422,15 @@ pub const Client = opaque { const layout = args.layout.toCStruct(); const ret = try api.call(.PJRT_Client_CreateViewOfDeviceBuffer, .{ .client = self.inner(), - .device_buffer_ptr = @ptrCast(@constCast(args.data)), + .device_buffer_ptr = @constCast(args.data), .dims = args.dims.ptr, .num_dims = args.dims.len, .element_type = @intFromEnum(args.element_type), - .layout = @ptrCast(@constCast(&layout)), - .device = @ptrCast(@constCast(args.device)), + .layout = @as([*c]c.PJRT_Buffer_MemoryLayout, @ptrCast(@constCast(&layout))), + .device = @as(?*c.PJRT_Device, @ptrCast(@constCast(args.device))), .on_delete_callback = args.on_delete_callback, .on_delete_callback_arg = args.on_delete_callback_arg, - .stream = @bitCast(@intFromPtr(args.stream)), + .stream = @as(isize, @bitCast(@intFromPtr(args.stream))), }); return @ptrCast(ret.buffer.?); } @@ -493,11 +493,11 @@ pub const Client = opaque { const ret = try api.call(.PJRT_Client_CreateUninitializedBuffer, .{ .client = self.inner(), .shape_dims = args.dims.ptr, - .shape_num_dims = @intCast(args.dims.len), + .shape_num_dims = @as(usize, @intCast(args.dims.len)), .shape_element_type = @intFromEnum(args.element_type), - .shape_layout = @ptrCast(&layout), - .device = if (args.dst == .device) @ptrCast(@constCast(args.dst.device)) else null, - .memory = if (args.dst == .memory) @ptrCast(@constCast(args.dst.memory)) else null, + .shape_layout = @as([*c]c.PJRT_Buffer_MemoryLayout, @ptrCast(&layout)), + .device = if (args.dst == .device) @as(?*c.PJRT_Device, @ptrCast(@constCast(args.dst.device))) else null, + .memory = if (args.dst == .memory) @as(?*c.PJRT_Memory, @ptrCast(@constCast(args.dst.memory))) else null, }); return @ptrCast(ret.buffer.?); } @@ -662,7 +662,7 @@ pub const SerializeResult = struct { pub const ExecuteContext = opaque { pub fn deinit(self: *ExecuteContext, api: *const Api) void { _ = api.call(.PJRT_ExecuteContext_Destroy, .{ - .context = @ptrCast(self), + .context = @as(?*c.PJRT_ExecuteContext, @ptrCast(self)), }) catch {}; } }; @@ -676,11 +676,11 @@ pub const Executable = opaque { }) catch unreachable; } - pub fn getCostAnalysis(self: *const Executable, api: *const Api) GetCostAnalysisError![]*const NamedValue { + pub fn getCostAnalysis(self: *const Executable, api: *const Api) GetCostAnalysisError![]const NamedValue { const ret = try api.call(.PJRT_Executable_GetCostAnalysis, .{ .executable = self.inner(), }); - const values: [*]*const NamedValue = @ptrCast(ret.properties); + const values: [*]const NamedValue = @ptrCast(ret.properties); return values[0..ret.num_properties]; } @@ -756,11 +756,11 @@ pub const LoadedExecutable = opaque { return ret.is_deleted; } - pub fn getAddressableDevices(self: *const LoadedExecutable, api: *const Api) []Device { + pub fn getAddressableDevices(self: *const LoadedExecutable, api: *const Api) []const *Device { const ret = api.call(.PJRT_LoadedExecutable_AddressableDevices, .{ .executable = self.inner(), }) catch unreachable; - return @ptrCast(ret.addressable_devices); + return @ptrCast(ret.addressable_devices[0..ret.num_addressable_devices]); } pub const ExecuteArgs = struct { @@ -858,10 +858,14 @@ pub const MemoryLayout = union(MemoryLayoutType) { fn toCStruct(self: MemoryLayout) c.PJRT_Buffer_MemoryLayout { return switch (self) { - .tiled => |v| pjrtStruct2(c.PJRT_Buffer_MemoryLayout, .{ + .tiled => |v| c.PJRT_Buffer_MemoryLayout{ + .struct_size = c.PJRT_Buffer_MemoryLayout_STRUCT_SIZE, + .extension_start = null, .type = c.PJRT_Buffer_MemoryLayout_Type_Tiled, .unnamed_0 = .{ .tiled = c.PJRT_Buffer_MemoryLayout_Tiled{ + .struct_size = c.PJRT_Buffer_MemoryLayout_Tiled_STRUCT_SIZE, + .extension_start = null, .minor_to_major = v.minor_to_major.ptr, .minor_to_major_size = v.minor_to_major.len, .tile_dims = v.tile_dims.ptr, @@ -869,16 +873,20 @@ pub const MemoryLayout = union(MemoryLayoutType) { .num_tiles = v.tile_dims_sizes.len, }, }, - }), - .strides => |v| pjrtStruct2(c.PJRT_Buffer_MemoryLayout, .{ + }, + .strides => |v| c.PJRT_Buffer_MemoryLayout{ + .struct_size = c.PJRT_Buffer_MemoryLayout_STRUCT_SIZE, + .extension_start = null, .type = c.PJRT_Buffer_MemoryLayout_Type_Strides, .unnamed_0 = .{ .strides = c.PJRT_Buffer_MemoryLayout_Strides{ + .struct_size = c.PJRT_Buffer_MemoryLayout_Strides_STRUCT_SIZE, + .extension_start = null, .byte_strides = v.byte_strides.ptr, .num_byte_strides = v.byte_strides.len, }, }, - }), + }, }; } }; @@ -956,7 +964,7 @@ pub const Buffer = opaque { const ret = try api.call(.PJRT_Buffer_UnpaddedDimensions, .{ .buffer = self.inner(), }); - return ret.dims[0..ret.num_dims]; + return ret.unpadded_dims[0..ret.num_dims]; } pub fn getOnDeviceSizeInBytes(self: *const Buffer, api: *const Api) ApiError!usize { @@ -966,10 +974,10 @@ pub const Buffer = opaque { return ret.on_device_size_in_bytes; } - pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!*Buffer { + pub fn copyToDevice(self: *const Buffer, api: *const Api, device: *Device) ApiError!*Buffer { const ret = try api.call(.PJRT_Buffer_CopyToDevice, .{ .buffer = self.inner(), - .dst_device = device.inner, + .dst_device = device.inner(), }); return @ptrCast(ret.dst_buffer.?); } @@ -1001,7 +1009,7 @@ pub const Buffer = opaque { pub fn copyToMemory(self: *const Buffer, api: *const Api, dst_memory: *const Memory) ApiError!*Buffer { const ret = try api.call(.PJRT_Buffer_CopyToMemory, .{ .buffer = self.inner(), - .dst_memory = @ptrCast(@constCast(dst_memory)), + .dst_memory = @as(?*c.PJRT_Memory, @ptrCast(@constCast(dst_memory))), }); return @ptrCast(ret.dst_buffer); } @@ -1043,8 +1051,9 @@ pub const Event = opaque { } pub fn getEventError(self: *const Event, api: *const Api) ?*Error { - var args: Api.CallFnArgType(.PJRT_Event_Error) = .{ .event = self.inner() }; - args = pjrtStruct2(@TypeOf(args), args); + var args = pjrtStruct2(c.PJRT_Event_Error_Args, .{ + .event = self.inner(), + }); const result: ?*c.PJRT_Error = api.inner.PJRT_Event_Error.?(&args); return @ptrCast(result); } @@ -1123,12 +1132,12 @@ pub const Memory = opaque { return &.{}; } - pub fn addressableByDevices(self: *const Memory, api: *const Api) []*Device { + pub fn addressableByDevices(self: *const Memory, api: *const Api) []const *Device { const ret = api.call(.PJRT_Memory_AddressableByDevices, .{ - .event = self.inner(), + .memory = self.inner(), }) catch unreachable; if (ret.devices) |devices| { - return devices[0..ret.num_devices]; + return @ptrCast(devices[0..ret.num_devices]); } return &.{}; } @@ -1146,10 +1155,10 @@ pub const AsyncHostToDeviceTransferManager = opaque { pub fn transferData(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize, data: []const u8, offset: i64, is_last_transfer: bool) ApiError!*Event { const ret = try api.call(.PJRT_AsyncHostToDeviceTransferManager_TransferData, .{ .transfer_manager = self.inner(), - .buffer_index = @intCast(buffer_index), + .buffer_index = @as(c_int, @intCast(buffer_index)), .data = data.ptr, .offset = offset, - .transfer_size = @intCast(data.len), + .transfer_size = @as(i64, @intCast(data.len)), .is_last_transfer = is_last_transfer, }); return @ptrCast(ret.done_with_h2d_transfer.?); @@ -1158,7 +1167,7 @@ pub const AsyncHostToDeviceTransferManager = opaque { pub fn retrieveBuffer(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize) ApiError!*Buffer { const ret = try api.call(.PJRT_AsyncHostToDeviceTransferManager_RetrieveBuffer, .{ .transfer_manager = self.inner(), - .buffer_index = @intCast(buffer_index), + .buffer_index = @as(c_int, @intCast(buffer_index)), }); return @ptrCast(ret.buffer_out.?); } @@ -1180,7 +1189,7 @@ pub const AsyncHostToDeviceTransferManager = opaque { pub fn bufferSize(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize) ApiError!usize { const ret = try api.call(.PJRT_AsyncHostToDeviceTransferManager_BufferSize, .{ .transfer_manager = self.inner(), - .buffer_index = @intCast(buffer_index), + .buffer_index = @as(c_int, @intCast(buffer_index)), }); return ret.buffer_size; } @@ -1188,7 +1197,7 @@ pub const AsyncHostToDeviceTransferManager = opaque { pub fn setBufferError(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize, error_code: c.PJRT_Error_Code, error_message: []const u8) ApiError!void { _ = try api.call(.PJRT_AsyncHostToDeviceTransferManager_SetBufferError, .{ .transfer_manager = self.inner(), - .buffer_index = @intCast(buffer_index), + .buffer_index = @as(c_int, @intCast(buffer_index)), .error_code = error_code, .error_message = error_message.ptr, .error_message_size = error_message.len, @@ -1198,7 +1207,7 @@ pub const AsyncHostToDeviceTransferManager = opaque { pub fn addMetadata(self: *AsyncHostToDeviceTransferManager, api: *const Api, transfer_metadata: []const NamedValue) ApiError!void { _ = try api.call(.PJRT_AsyncHostToDeviceTransferManager_AddMetadata, .{ .transfer_manager = self.inner(), - .transfer_metadata = transfer_metadata.ptr, + .transfer_metadata = @as([*c]const c.PJRT_NamedValue, @ptrCast(transfer_metadata.ptr)), .num_metadata = transfer_metadata.len, }); } @@ -1304,7 +1313,7 @@ pub const NamedValue = extern struct { switch (self.kind()) { .string => try writer.print(" .string = {s} ", .{u.string_value[0..self.inner.value_size]}), .int64 => try writer.print(" .int64 = {d} ", .{u.int64_value}), - .int64list => try writer.print(" .int64list = {d} ", .{u.int64_array_value[0..self.inner.value_size]}), + .int64list => try writer.print(" .int64list = {any} ", .{u.int64_array_value[0..self.inner.value_size]}), .float => try writer.print(" .float = {d} ", .{u.float_value}), .bool => try writer.print(" .bool = {} ", .{u.bool_value}), } diff --git a/zml2/pjrtx.zig b/zml2/pjrtx.zig index ea1474a5..ac37b95f 100644 --- a/zml2/pjrtx.zig +++ b/zml2/pjrtx.zig @@ -20,7 +20,6 @@ pub const GetCostAnalysisError = pjrt.GetCostAnalysisError; pub const Memory = pjrt.Memory; pub const MemoryStats = pjrt.MemoryStats; pub const NamedValue = pjrt.NamedValue; -pub const Profiler = pjrt.Profiler; pub const SerializeResult = pjrt.SerializeResult; pub const Stream = pjrt.Stream; @@ -71,7 +70,7 @@ pub const Client = opaque { pub fn deserializeAndLoad(self: *const Client, api: *const Api, io: std.Io, bytes: []const u8) ApiError!*LoadedExecutable { var future = io.async(pjrt.Client.deserializeAndLoad, .{ self.inner(), api, bytes }); - return @ptrCast(future.await(io)); + return @ptrCast(try future.await(io)); } pub const CreateViewOfDeviceBufferArgs = pjrt.Client.CreateViewOfDeviceBufferArgs; @@ -187,8 +186,8 @@ pub const Buffer = opaque { return try self.inner().getOnDeviceSizeInBytes(api); } - pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!*Buffer { - return @ptrCast(self.inner().copyToDevice(api, device)); + pub fn copyToDevice(self: *const Buffer, api: *const Api, device: *Device) ApiError!*Buffer { + return @ptrCast(try self.inner().copyToDevice(api, device)); } pub fn copyToMemory(self: *const Buffer, api: *const Api, memory_: *const Memory) ApiError!*Buffer { @@ -219,11 +218,11 @@ pub const Event = opaque { return self.inner().getEventError(api); } - pub fn awaitBlocking(self: *Event, api: *const Api, io: std.Io) ApiError!void { + pub fn awaitBlocking(self: *Event, api: *const Api) ApiError!void { if (self.isReady(api)) { return; } - try self.inner().await(api, io); + try self.inner().await(api); } pub fn await(self: *Event, api: *const Api, io: std.Io) ApiError!void { @@ -287,7 +286,7 @@ pub const LoadedExecutable = opaque { return self.inner().isDeleted(api); } - pub fn getAddressableDevices(self: *const LoadedExecutable, api: *const Api) []*const Device { + pub fn getAddressableDevices(self: *const LoadedExecutable, api: *const Api) []const *Device { return self.inner().getAddressableDevices(api); } @@ -331,23 +330,23 @@ pub const AsyncHostToDeviceTransferManager = opaque { return @ptrCast(try self.inner().retrieveBuffer(api, buffer_index)); } - pub fn device(self: *AsyncHostToDeviceTransferManager, api: *const Api) *Device { - return @ptrCast(self.inner().device(api)); + pub fn device(self: *AsyncHostToDeviceTransferManager, api: *const Api) ApiError!*Device { + return @ptrCast(try self.inner().device(api)); } - pub fn bufferCount(self: *AsyncHostToDeviceTransferManager, api: *const Api) usize { + pub fn bufferCount(self: *AsyncHostToDeviceTransferManager, api: *const Api) ApiError!usize { return self.inner().bufferCount(api); } - pub fn bufferSize(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize) usize { + pub fn bufferSize(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize) ApiError!usize { return self.inner().bufferSize(api, buffer_index); } - pub fn setBufferError(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize, error_code: ErrorCode, error_message: []const u8) void { - self.inner().setBufferError(api, buffer_index, error_code, error_message); + pub fn setBufferError(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize, error_code: ErrorCode, error_message: []const u8) ApiError!void { + return self.inner().setBufferError(api, buffer_index, @intFromEnum(error_code), error_message); } - pub fn addMetadata(self: *AsyncHostToDeviceTransferManager, api: *const Api, transfer_metadata: []const NamedValue) void { + pub fn addMetadata(self: *AsyncHostToDeviceTransferManager, api: *const Api, transfer_metadata: []const NamedValue) ApiError!void { return self.inner().addMetadata(api, transfer_metadata); } }; From 715b0f745a03508a82e0a897961bb18b18c9758d Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 14:59:14 +0100 Subject: [PATCH 086/881] Commented Writer in stdx.BoundedArray It was causing compile issue in tests --- stdx/bounded_array.zig | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/stdx/bounded_array.zig b/stdx/bounded_array.zig index 670201a4..bdee5bee 100644 --- a/stdx/bounded_array.zig +++ b/stdx/bounded_array.zig @@ -273,16 +273,16 @@ pub fn BoundedArrayAligned( @memset(self.slice()[old_len..self.len], value); } - pub const Writer = if (T != u8) - @compileError("The Writer interface is only defined for BoundedArray(u8, ...) " ++ - "but the given type is BoundedArray(" ++ @typeName(T) ++ ", ...)") - else - std.io.GenericWriter(*Self, error{Overflow}, appendWrite); + //pub const Writer = if (T != u8) + // @compileError("The Writer interface is only defined for BoundedArray(u8, ...) " ++ + // "but the given type is BoundedArray(" ++ @typeName(T) ++ ", ...)") + //else + // std.io.GenericWriter(*Self, error{Overflow}, appendWrite); - /// Initializes a writer which will write into the array. - pub fn writer(self: *Self) Writer { - return .{ .context = self }; - } + ///// Initializes a writer which will write into the array. + //pub fn writer(self: *Self) Writer { + // return .{ .context = self }; + //} /// Same as `appendSlice` except it returns the number of bytes written, which is always the same /// as `m.len`. The purpose of this function existing is to match `std.io.GenericWriter` API. From edc451fcf32a126391446fe9476efd3cebb805f0 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 14:59:47 +0100 Subject: [PATCH 087/881] Various fixes in stablehlo bindings --- mlir/dialects/stablehlo/stablehlo.zig | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 0cad2e65..b3404e78 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -592,7 +592,7 @@ pub fn rng_bit_generator( ) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.rng_bit_generator", .{ .operands = .{ .flat = &.{initial_state} }, - .results = &.{ res_state_type, res_type }, + .results = .{ .flat = &.{ res_state_type, res_type } }, .attributes = &.{ .named(ctx, "rng_algorithm", rngAlgorithm(ctx, rng_algorithm)), }, @@ -673,7 +673,7 @@ pub fn convolution( .results = .{ .flat = &.{res_type} }, .attributes = &.{ .named(ctx, "window_strides", mlir.denseArrayAttribute(ctx, .i64, opts.window_strides)), - .named(ctx, "padding", mlir.denseElementsAttribute(mlir.RankedTensorType.get(opts.pad_shape, mlir.integerType(ctx, .i64)).shaped(), opts.pad_value)), + .named(ctx, "padding", mlir.denseElementsAttribute(mlir.RankedTensorType.get(opts.pad_shape, mlir.integerType(ctx, .i64), null).shaped(), opts.pad_value)), .named(ctx, "lhs_dilation", mlir.denseArrayAttribute(ctx, .i64, opts.lhs_dilation)), .named(ctx, "rhs_dilation", mlir.denseArrayAttribute(ctx, .i64, opts.rhs_dilation)), .named(ctx, "window_reversal", mlir.denseArrayAttribute(ctx, .bool, window_reversal[0..opts.window_reversal.len])), @@ -1179,7 +1179,7 @@ pub const PrecisionAttribute = opaque { } pub fn getValue(self: *const PrecisionAttribute) Precision { - const value = mlir.fromStringRef(c.stablehloPrecisionAttrGetValue(self.ptr())); + const value = mlir.string(c.stablehloPrecisionAttrGetValue(self.ptr())); return std.meta.stringToEnum(Precision, value) orelse unreachable; } }; @@ -1210,7 +1210,7 @@ pub const ComparisonDirection = opaque { } pub fn getValue(self: *const ComparisonDirection) Direction { - const value = mlir.fromStringRef(c.stablehloComparisonDirectionAttrGetValue(self.ptr())); + const value = mlir.string(c.stablehloComparisonDirectionAttrGetValue(self.ptr())); return std.meta.stringToEnum(Direction, value) orelse unreachable; } }; @@ -1239,7 +1239,7 @@ pub const CompareType = opaque { } pub fn getValue(self: *const CompareType) Type { - const value = mlir.fromStringRef(c.stablehloComparisonTypeAttrGetValue(self.ptr())); + const value = mlir.string(c.stablehloComparisonTypeAttrGetValue(self.ptr())); return std.meta.stringToEnum(Type, value) orelse unreachable; } }; @@ -1267,7 +1267,7 @@ pub const TransposeAttribute = opaque { } pub fn getValue(self: *const TransposeAttribute) Type { - const value = mlir.fromStringRef(c.stablehloTransposeAttrGetValue(self.ptr())); + const value = mlir.string(c.stablehloTransposeAttrGetValue(self.ptr())); return std.meta.stringToEnum(Type, value) orelse unreachable; } }; @@ -1296,7 +1296,7 @@ pub const FftType = opaque { } pub fn getValue(self: *const FftType) Type { - const value = mlir.fromStringRef(c.stablehloFftTypeAttrGetValue(self.ptr())); + const value = mlir.string(c.stablehloFftTypeAttrGetValue(self.ptr())); return std.meta.stringToEnum(Type, value) orelse unreachable; } }; From 6de7409a81a816006f28bf7c5062d278f2359dd1 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 15:00:09 +0100 Subject: [PATCH 088/881] Reenabled all tests using refAllDecls --- zml2/shape.zig | 4 ---- zml2/tensor.zig | 36 ++++++++++++++++++------------------ zml2/zml.zig | 2 +- 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/zml2/shape.zig b/zml2/shape.zig index 7435c1da..034a4c5f 100644 --- a/zml2/shape.zig +++ b/zml2/shape.zig @@ -11,10 +11,6 @@ const EnumLiteral = @TypeOf(.enum_literal); const log = std.log.scoped(.shape); -test { - std.testing.refAllDecls(Shape); -} - /// Represent the shape of a tensor. pub const Shape = struct { pub const Tag = [*:0]const u8; diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 74b8b60c..706e5ea6 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -381,7 +381,7 @@ pub const Tensor = struct { // Real tensors don't have imaginary part. if (self.dtype().isFloat()) { - return Tensor.constant(self._shape, self.dtype().zero()); + return Tensor.constant(self.dtype().zero()).broad(self._shape); } const dt: DataType = switch (self.dtype()) { @@ -390,7 +390,7 @@ pub const Tensor = struct { .c128 => .f64, else => unreachable, }; - const op = dialects.stablehlo.imag(mlirCtx(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = dialects.stablehlo.imag(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape.withDtype(dt), op.result(0)); } @@ -467,7 +467,7 @@ pub const Tensor = struct { return .{ self.update(op.result(0)), _result(sh, op.result(1)) }; } - fn update(self: Rng, new_state: mlir.Value) Rng { + fn update(self: Rng, new_state: *const mlir.Value) Rng { return .{ ._state = _result(self._state._shape, new_state).reuseBuffer(self._state), .algorithm = self.algorithm, @@ -577,9 +577,9 @@ pub const Tensor = struct { pub fn normal(sh: Shape, opts: struct { mean: f64 = 0, stddev: f64 = 1 }) Tensor { stdx.debug.assert(sh.dtype().isFloat(), "normal expects tensor type to be a float, got {}", .{sh.dtype()}); - const a = Tensor.constant(.{}, DataType.Value.init(sh.dtype(), opts.mean)); + const a = Tensor.constant(DataType.Value.init(sh.dtype(), opts.mean)); _ = a; // autofix - const b = Tensor.constant(.{}, DataType.Value.init(sh.dtype(), opts.stddev)); + const b = Tensor.constant(DataType.Value.init(sh.dtype(), opts.stddev)); _ = b; // autofix unreachable; //const res_shape = Tensor.constantTensor(HostBuffer.fromSlice(.{sh.rank()}, sh.dims())); @@ -683,7 +683,7 @@ pub const Tensor = struct { return _result(self._shape, op.result(0)); } - fn convolution(self: Tensor, other: Tensor, opts: dialects.stablehlo.ConvolutionOpts, loc: mlir.Location) Tensor { + fn convolution(self: Tensor, other: Tensor, opts: dialects.stablehlo.ConvolutionOpts, loc: *const mlir.Location) Tensor { _ = loc; // autofix stdx.debug.assert(self.rank() == other.rank(), "convolution expects tensor ranks to match, got {} and {}", .{ self.rank(), other.rank() }); const N = self.rank(); @@ -771,8 +771,8 @@ pub const Tensor = struct { mlirCtx(), self.value(), other.value(), - used_opts, mlir.rankedTensorType(new_shape.dims(), mlirx.Type.fromDType(mlirCtx(), new_shape.dtype())), + used_opts, .unknown(mlirCtx()), ).appendTo(currentBlock()); @@ -1158,7 +1158,7 @@ pub const Tensor = struct { /// Returns a Tensor containing the ReLU activation function applied to each element of the input Tensor. pub fn relu(self: Tensor) Tensor { - return self.maximum(.constant(self.dims(), self.dtype().zero())); + return self.maximum(Tensor.constant(self.dtype().zero()).broad(.init(self.dims(), self.dtype()))); } /// Returns a Tensor containing the leaky-ReLU activation function applied to each element of the input Tensor. @@ -1183,11 +1183,11 @@ pub const Tensor = struct { //} /// Returns a Tensor containing the SwiGLU activation function applied to the input Tensor. - pub fn swiglu(self: Tensor, beta: f32, w: Tensor, b: Tensor) Tensor { - const sigmoid_tensor = self.mul(.constant(self._shape, DataType.Value.init(self.dtype(), beta))).sigmoid(); - const one_minus_sigmoid_tensor: Tensor = .constant(self._shape, .init(self.dtype(), 1)).sub(sigmoid_tensor); + pub fn swiglu(self: Tensor, beta: f32, w: Tensor, b: Tensor, tag: Shape.Tag) Tensor { + const sigmoid_tensor = self.mul(Tensor.constant(DataType.Value.init(self.dtype(), beta)).broad(.init(self._shape, self.dtype()))).sigmoid(); + const one_minus_sigmoid_tensor = Tensor.constant(.init(self.dtype(), 1)).broad(.init(self._shape, self.dtype())).sub(sigmoid_tensor); - return self.mul(sigmoid_tensor).add(one_minus_sigmoid_tensor.mul(w.matmul(self).add(b))); + return self.mul(sigmoid_tensor).add(one_minus_sigmoid_tensor.mul(w.dot(self, tag).add(b))); } /// Returns a Tensor containing the Gaussian Error Linear Units (GeLU) activation function applied to each element of the input Tensor. @@ -1386,7 +1386,6 @@ pub const Tensor = struct { mlirCtx(), self.value(), mlir.rankedTensorType(new_shape.dims(), mlirx.Type.fromDType(mlirCtx(), new_shape.dtype())), - mlirx.tensorType(mlirCtx(), new_shape), .unknown(mlirCtx()), ).appendTo(currentBlock()); return _result(new_shape, reshaped_val.result(0)); @@ -1560,7 +1559,7 @@ pub const Tensor = struct { pub fn concatenate(tensors: []const Tensor, axis_: anytype) Tensor { if (tensors.len == 1) return tensors[0]; stdx.debug.assert(tensors.len <= 32, "concatenate only supports up to 32 tensors, got {}", .{tensors.len}); - var buffer: [32]mlir.Value = undefined; + var buffer: [32]*const mlir.Value = undefined; std.debug.assert(tensors.len <= buffer.len); std.debug.assert(tensors.len > 0); const a = tensors[0].axis(axis_); @@ -1915,7 +1914,7 @@ pub const Tensor = struct { } pub fn zeroes(sh: Shape) Tensor { - return .constant(sh.dtype().zero()).broad(sh); + return Tensor.constant(sh.dtype().zero()).broad(sh); } // TODO(Corentin) @@ -3384,8 +3383,8 @@ pub const Tensor = struct { mlirCtx(), self.value(), other.value(), - dialects.stablehlo.ComparisonDirection.init(mlirCtx(), direction), - getComparisonType(mlirCtx(), self.dtype()), + dialects.stablehlo.ComparisonDirection.init(mlirCtx(), direction).getValue(), + getComparisonType(mlirCtx(), self.dtype()).getValue(), .unknown(mlirCtx()), ).appendTo(currentBlock()); @@ -3739,6 +3738,7 @@ pub const Tensor = struct { /// so it will slow down the program execution. pub fn print(input: Tensor) Tensor { _ = input; // autofix + unreachable; // TODO(Corentin) //return callback.call(callback.Print, .{input}, &.{input.shape()})[0]; } @@ -3789,7 +3789,7 @@ fn getPoolResDims(dt: DataType, in_dims: []const i64, base_dilations: @Vector(co return Shape.init(dims_arr[0..in_dims.len], dt); } -fn getComparisonType(ctx: mlir.Context, dtype: DataType) dialects.stablehlo.CompareType { +fn getComparisonType(ctx: *mlir.Context, dtype: DataType) *const dialects.stablehlo.CompareType { return dialects.stablehlo.CompareType.init(ctx, switch (dtype.class()) { .bool => .UNSIGNED, .integer => if (dtype.isSignedInt()) .SIGNED else .UNSIGNED, diff --git a/zml2/zml.zig b/zml2/zml.zig index 2db5f1e2..52d048e7 100644 --- a/zml2/zml.zig +++ b/zml2/zml.zig @@ -75,5 +75,5 @@ pub fn Bufferized(comptime T: type) type { } test "zml" { - _ = Tensor; + std.testing.refAllDeclsRecursive(@This()); } From 25df30598ee58ffc634c1e3fb188a25cbf813845 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 15:13:58 +0100 Subject: [PATCH 089/881] Re-enabled convert tests --- zml2/mlirx.zig | 2 ++ zml2/tensor.zig | 73 ++++++++++++++++++++++++++++++------------------- 2 files changed, 47 insertions(+), 28 deletions(-) diff --git a/zml2/mlirx.zig b/zml2/mlirx.zig index 3b13b2d8..2c3ddb17 100644 --- a/zml2/mlirx.zig +++ b/zml2/mlirx.zig @@ -50,8 +50,10 @@ pub const Type = struct { .{ .f8e4m3b11fnuz, mlir.floatType(ctx, .f8e4m3b11fnuz) }, .{ .f8e4m3fn, mlir.floatType(ctx, .f8e4m3fn) }, .{ .f8e4m3fnuz, mlir.floatType(ctx, .f8e4m3fnuz) }, + .{ .f8e3m4, mlir.floatType(ctx, .f8e3m4) }, .{ .f8e5m2, mlir.floatType(ctx, .f8e5m2) }, .{ .f8e5m2fnuz, mlir.floatType(ctx, .f8e5m2fnuz) }, + .{ .f4e2m1, mlir.floatType(ctx, .f4e2m1fn) }, .{ .bf16, mlir.floatType(ctx, .bf16) }, .{ .f16, mlir.floatType(ctx, .f16) }, .{ .f32, mlir.floatType(ctx, .f32) }, diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 706e5ea6..432baa77 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -958,40 +958,57 @@ pub const Tensor = struct { return _result(self._shape.withDtype(to), op.result(0)); } - // TODO(Corentin) - //test convert { - // const floats = @import("floats.zig"); - // const zml = @import("zml.zig"); - // const platform = zml.testing.env(); + test convert { + const floats = @import("floats.zig"); + const zml = @import("zml.zig"); + const platform = zml.testing.env(); - // // f4e2m1 - // { - // const x = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; - // var x_f4: [x.len]floats.Float4E2M1 = undefined; - // for (&x_f4, &x) |*xi_f4, xi| xi_f4.* = .fromF32(xi); + // f4e2m1 + { + const x = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; + var x_f4: [x.len]floats.Float4E2M1 = undefined; + for (&x_f4, &x) |*xi_f4, xi| xi_f4.* = .fromF32(xi); - // const x_d = try zml.Buffer.fromArray(platform, x); - // const x_f4_xla_d = try zml.testing.compileAndCall(platform, Tensor.convert, .{ x_d, .f4e2m1 }); + const x_d: Tensor = .init(Shape.init(.{x.len}, .f32)); + const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.convert, .{ x_d, .f4e2m1 }, platform); + defer exe.deinit(); - // const x_f4_xla = x_f4_xla_d.getValue(@TypeOf(x_f4)); - // errdefer std.log.warn("convert(.f4e2m1) failed !\ninput f32:\n{e}\nzml.floats computed:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x), x_f4, x_f4_xla }); - // try std.testing.expectEqualDeep(x_f4, x_f4_xla); - // } + const x_d_buffer: zml.Buffer = try .fromBytes(platform, x_d.shape(), std.mem.sliceAsBytes(&x), std.testing.io); + defer x_d_buffer.deinit(); - // // f8e3m4 - // { - // const x = [_]f32{ 1.1 / 4.0, 1.1 / 8.0, 1.1 / 16.0, 1.1 / 32.0, 1.1 / 64.0, 1.1 / 128.0 }; - // var x_f8e3: [x.len]floats.Float8E3M4 = undefined; - // for (&x_f8e3, &x) |*xi_f8e3, xi| xi_f8e3.* = .fromF32(xi); + const x_f4_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); + const x_f4_xla_bytes = try x_f4_xla_d.toHostAlloc(std.testing.allocator, std.testing.io); + defer std.testing.allocator.free(x_f4_xla_bytes); - // const x_d = try zml.Buffer.fromArray(platform, x); - // const x_f8e3_xla_d = try zml.testing.compileAndCall(platform, Tensor.convert, .{ x_d, .f8e3m4 }); + const x_f4_xla = std.mem.bytesAsSlice(floats.Float4E2M1, x_f4_xla_bytes); - // const x_f8e3_xla = x_f8e3_xla_d.getValue(@TypeOf(x_f8e3)); - // errdefer std.log.warn("convert(.f8e3m4) failed !\ninput f32:\n{e}\nzml.floats computed:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x), x_f8e3, x_f8e3_xla }); - // try std.testing.expectEqualDeep(x_f8e3, x_f8e3_xla); - // } - //} + errdefer std.log.warn("convert(.f4e2m1) failed !\ninput f32:\n{e}\nzml.floats computed:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x), x_f4, x_f4_xla }); + try std.testing.expectEqualDeep(&x_f4, x_f4_xla); + } + + // f8e3m4 + { + const x = [_]f32{ 1.1 / 4.0, 1.1 / 8.0, 1.1 / 16.0, 1.1 / 32.0, 1.1 / 64.0, 1.1 / 128.0 }; + var x_f8e3: [x.len]floats.Float8E3M4 = undefined; + for (&x_f8e3, &x) |*xi_f8e3, xi| xi_f8e3.* = .fromF32(xi); + + const x_d: Tensor = .init(Shape.init(.{x.len}, .f32)); + const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.convert, .{ x_d, .f8e3m4 }, platform); + defer exe.deinit(); + + const x_d_buffer: zml.Buffer = try .fromBytes(platform, x_d.shape(), std.mem.sliceAsBytes(&x), std.testing.io); + defer x_d_buffer.deinit(); + + const x_f8e3_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); + const x_f8e3_xla_bytes = try x_f8e3_xla_d.toHostAlloc(std.testing.allocator, std.testing.io); + defer std.testing.allocator.free(x_f8e3_xla_bytes); + + const x_f8e3_xla = std.mem.bytesAsSlice(floats.Float8E3M4, x_f8e3_xla_bytes); + + errdefer std.log.warn("convert(.f8e3m4) failed !\ninput f32:\n{e}\nzml.floats computed:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x), x_f8e3, x_f8e3_xla }); + try std.testing.expectEqualDeep(&x_f8e3, x_f8e3_xla); + } + } /// Returns a Tensor containing the element-wise rounding operation of the input Tensor. pub fn round(self: Tensor) Tensor { From e50db03c21f615c8c356111a36782bd0098c4389 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 15:31:23 +0100 Subject: [PATCH 090/881] Partially re-enabled Tensor.dot tests --- zml2/tensor.zig | 76 +++++++++++++++++++++++++----------------------- zml2/testing.zig | 7 +++++ 2 files changed, 46 insertions(+), 37 deletions(-) diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 432baa77..8f58d689 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -1047,46 +1047,48 @@ pub const Tensor = struct { return lhs.dotGeneral(rhs, &.{.{ lhs_contracting_dim, rhs_contracting_dim }}, batching_axes.slice()); } - // TODO(Corentin) - //test dot { - // const zml = @import("zml.zig"); - // const platform = zml.testing.env(); + test dot { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); - // var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); - // defer comp.deinit(); + inline for (.{ + .{ .{ .c = 20 }, .{ .c = 20 }, .c, .{} }, + .{ + .{ .a = 20, .b = 21, .c = 22 }, + .{ .a = 20, .d = 23, .c = 22 }, + .c, + .{ .a = 20, .b = 21, .d = 23 }, + }, + .{ + .{ .a = 20, .b = 21, .c = 22 }, + .{ .c = 22, .d = 23, .e = 24 }, + .c, + .{ .a = 20, .b = 21, .d = 23, .e = 24 }, + }, + // TODO(Corentin): Re-enable that + //.{ + // .{ .a = 20, .b = 21, .c = 22 }, + // .{ .c = 22, .d = 23, .a = 20 }, + // .{ .c, .a }, + // .{ .b = 21, .d = 23 }, + //}, + }) |testcase| { + const x: Tensor = .init(Shape.init(testcase[0], .f32)); + const y: Tensor = .init(Shape.init(testcase[1], .f32)); + const ctr = Shape.toTag(testcase[2]); + const z_shape: Shape = .init(testcase[3], .f32); + const forward = struct { + fn forward(x_: Tensor, y_: Tensor, tag: Shape.Tag) Tensor { + return x_.dot(y_, tag); + } + }.forward; - // comp.activate(); - // defer comp.deactivate(); + const exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, y, ctr }, platform); + defer exe.deinit(); - // inline for (.{ - // .{ .{ .c = 20 }, .{ .c = 20 }, .{.c}, .{} }, - // .{ - // .{ .a = 20, .b = 21, .c = 22 }, - // .{ .a = 20, .d = 23, .c = 22 }, - // .{.c}, - // .{ .a = 20, .b = 21, .d = 23 }, - // }, - // .{ - // .{ .a = 20, .b = 21, .c = 22 }, - // .{ .c = 22, .d = 23, .e = 24 }, - // .{.c}, - // .{ .a = 20, .b = 21, .d = 23, .e = 24 }, - // }, - // .{ - // .{ .a = 20, .b = 21, .c = 22 }, - // .{ .c = 22, .d = 23, .a = 20 }, - // .{ .c, .a }, - // .{ .b = 21, .d = 23 }, - // }, - // }) |testcase| { - // const x_shape, const y_shape, const ctr, const z_shape = testcase; - // const x = Tensor.constant(x_shape, .{ .f32 = 0.0 }); - // const y = Tensor.constant(y_shape, .{ .f32 = 0.0 }); - // const z = x.dot(y, ctr); - - // try zml.testing.expectEqualShapes(Shape.init(z_shape, .f32), z.shape()); - // } - //} + try zml.testing.expectEqualShapes(Shape.init(z_shape, .f32), exe.output_shapes[0]); + } + } /// Generalized matrix multiplication of two tensors along the specified axes. /// In this version batching dimensions need to be explicitly specified. diff --git a/zml2/testing.zig b/zml2/testing.zig index 2d69734e..abd2fff2 100644 --- a/zml2/testing.zig +++ b/zml2/testing.zig @@ -136,6 +136,13 @@ pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, tolerance: f32) } } +pub fn expectEqualShapes(expected: zml.Shape, actual: zml.Shape) error{TestExpectedEqual}!void { + if (expected.eqlWithTags(actual)) return; + + std.debug.print("Expected {f}, got {f}", .{ expected, actual }); + return error.TestExpectedEqual; +} + fn BufferizedWithArgs(comptime T: type) type { return zml.meta.MapType(zml.Tensor, zml.Buffer).map(T); } From 0f05158a375d4b97cd2882be9ece4966e78f17f5 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 15:38:30 +0100 Subject: [PATCH 091/881] Re-enabled leakyReLU tests --- zml2/tensor.zig | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 8f58d689..d2a01678 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -1189,17 +1189,23 @@ pub const Tensor = struct { return self.relu().add(below_zero); } - // TODO(Corentin) - //test leakyReLU { - // const zml = @import("zml.zig"); - // const platform = zml.testing.env(); + test leakyReLU { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); - // const input = try zml.Buffer.fromSlice(platform, .{2}, &[_]f32{ -0.6884, 1.6795 }); - // const res = try zml.testing.compileAndCall(platform, leakyReLU, .{ input, 0.1 }); + const input: Tensor = .init(Shape.init(.{2}, .f32)); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.leakyReLU, .{ input, 0.1 }, platform); + defer exe.deinit(); - // const expectation = zml.HostBuffer.fromArray(&[2]f32{ -0.0688, 1.6795 }); - // try zml.testing.expectClose(expectation, res, 1e-4); - //} + const input_buffer: zml.Buffer = try .fromBytes(platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6884, 1.6795 }), std.testing.io); + defer input_buffer.deinit(); + + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.leakyReLU, .{input_buffer}); + defer res.deinit(); + + const expectation: zml.testing.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[2]f32{ -0.0688, 1.6795 })); + try zml.testing.expectClose(std.testing.io, expectation, res, 1e-4); + } /// Returns a Tensor containing the SwiGLU activation function applied to the input Tensor. pub fn swiglu(self: Tensor, beta: f32, w: Tensor, b: Tensor, tag: Shape.Tag) Tensor { From 5b82edbc63d7164644d19d4c2b9f2474d486661b Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 15:59:28 +0100 Subject: [PATCH 092/881] Added getValue in zml.Buffer --- zml2/buffer.zig | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/zml2/buffer.zig b/zml2/buffer.zig index b23abc51..8ee19fb1 100644 --- a/zml2/buffer.zig +++ b/zml2/buffer.zig @@ -47,6 +47,10 @@ pub const Buffer = struct { return self._shape; } + pub fn format(self: Buffer, writer: *std.Io.Writer) !void { + try writer.print("Buffer({f})@{x}", .{ self._shape, @intFromPtr(self.devicePtr()) }); + } + /// Copies the content of the given buffer from host memory to the accelerator memory. pub fn from(platform: Platform, shape_: Shape, data_: []const u8, io: std.Io, opts: FromOptions) !Buffer { var res: Buffer = .{ @@ -136,6 +140,29 @@ pub const Buffer = struct { }; } + pub fn devicePtr(self: Buffer) *anyopaque { + //stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer", .{}); + return self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable; + } + + /// Fetches the content of the given buffer into a stack variable of the given type. + pub fn getValue(self: Buffer, T: type, io: std.Io) !T { + stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {f} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); + var res: T = undefined; + + try self.toHost(std.mem.asBytes(&res), io); + + return res; + } + + pub fn toHost(self: Buffer, buf: []u8, io: std.Io) !void { + //stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); + const maybe_event = try self._shards.get(0).toHostBuffer(self._api, buf); + if (maybe_event) |event| { + try event.await(self._api, io); + } + } + /// Copies the content of the Buffer to the host. /// The returned `HostBuffer` does own the memory. pub fn toHostAlloc(self: Buffer, allocator: std.mem.Allocator, io: std.Io) ![]u8 { @@ -144,11 +171,8 @@ pub const Buffer = struct { const output = try allocator.alloc(u8, self._shape.byteSize()); errdefer allocator.free(output); - //stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, output); - if (maybe_event) |event| { - try event.await(self._api, io); - } + try self.toHost(output, io); + return output; } }; From 9339b8d470c546db8c7942ca427497d4a4341333 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 15:59:40 +0100 Subject: [PATCH 093/881] Re-enabled cumsum tests --- zml2/ops.zig | 2 +- zml2/tensor.zig | 97 +++++++++++++++++++++++++++---------------------- 2 files changed, 54 insertions(+), 45 deletions(-) diff --git a/zml2/ops.zig b/zml2/ops.zig index 254afcb4..956b9d8c 100644 --- a/zml2/ops.zig +++ b/zml2/ops.zig @@ -179,7 +179,7 @@ pub fn reduceWindow(inputs: anytype, inits: anytype, opts: ReduceWindowOpts, com .named(mlir_ctx, "base_dilations", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.base_dilations)), .named(mlir_ctx, "window_dilations", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.window_dilations)), // Cast the [][2]i64 to []i64 (safe) - .named(mlir_ctx, "padding", mlir.denseElementsAttribute(mlir.RankedTensorType.get(&.{}, mlir.integerType(mlir_ctx, .i64), null).shaped(), @as([]const i64, @ptrCast(opts.padding)))), + .named(mlir_ctx, "padding", mlir.denseElementsAttribute(mlir.RankedTensorType.get(&.{ @intCast(opts.padding.len), 2 }, mlir.integerType(mlir_ctx, .i64), null).shaped(), @as([]const i64, @ptrCast(opts.padding)))), }, .verify = true, .location = .unknown(mlir_ctx), diff --git a/zml2/tensor.zig b/zml2/tensor.zig index d2a01678..00e6814c 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -1299,59 +1299,68 @@ pub const Tensor = struct { return self.sum(axis_).divByConst(self.dim(axis_)); } - // TODO(Corentin) /// Returns a Tensor containing the cumulative sum of elements over the given axis. /// Output shape is the same as input shape. /// [0, 1, 0, 1, 0, 0, 1, 1].cumulativeSum(0) -> [0, 1, 1, 2, 2, 2, 3, 4] /// The last value contains the sum of all element in the array. - //pub fn cumulativeSum(self: Tensor, axis_: anytype) Tensor { - // const rk = self.rank(); - // const a = self.axis(axis_); + pub fn cumulativeSum(self: Tensor, axis_: anytype) Tensor { + const rk = self.rank(); + const a = self.axis(axis_); - // const ones = [_]i64{1} ** MAX_RANK; - // var window_dimensions = ones; - // window_dimensions[a] = self.dim(a); - // var padding = [_][2]i64{.{ 0, 0 }} ** MAX_RANK; - // padding[a] = .{ self.dim(a) - 1, 0 }; + const ones = [_]i64{1} ** constants.MAX_RANK; + var window_dimensions = ones; + window_dimensions[a] = self.dim(a); + var padding = [_][2]i64{.{ 0, 0 }} ** constants.MAX_RANK; + padding[a] = .{ self.dim(a) - 1, 0 }; - // return ops.reduceWindow( - // Tensor.add, - // self, - // Tensor.scalar(0, self.dtype()), - // .{ - // .base_dilations = ones[0..rk], - // .window_dilations = ones[0..rk], - // .window_strides = ones[0..rk], - // .window_dimensions = window_dimensions[0..rk], - // .padding = padding[0..rk], - // }, - // ); - //} + const result = ops.reduceWindow( + .{self}, + .{Tensor.scalar(0, self.dtype())}, + .{ + .base_dilations = ones[0..rk], + .window_dilations = ones[0..rk], + .window_strides = ones[0..rk], + .window_dimensions = window_dimensions[0..rk], + .padding = padding[0..rk], + }, + struct { + fn add(values: ops.ReduceArgs) struct { Tensor } { + return .{values.left.add(values.right)}; + } + }.add, + .{}, + ); - //test cumulativeSum { - // const zml = @import("zml.zig"); - // const platform = zml.testing.env(); + return result[0]; + } - // const Local = struct { - // pub fn _cumsum(input: Tensor) Tensor { - // const x = input.withPartialTags(.{.n}); - // const y = x.cumulativeSum(.n); - // // Check that tags are propagated - // std.debug.assert(y.shape().eqlWithTags(x.shape())); - // return y; - // } - // }; + test cumulativeSum { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); - // const x = try zml.Buffer.fromArray( - // platform, - // [2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } }, - // ); - // const res = try zml.testing.compileAndCall(platform, Local._cumsum, .{x}); - // try std.testing.expectEqual( - // [2][5]f32{ .{ 0, 1, 2, 2, 3 }, .{ 3, 4, 4, 6, 7 } }, - // try res.getValue([2][5]f32), - // ); - //} + const Local = struct { + pub fn _cumsum(input: Tensor) Tensor { + const x = input.withPartialTags(.{.n}); + const y = x.cumulativeSum(.n); + // Check that tags are propagated + std.debug.assert(y.shape().eqlWithTags(x.shape())); + return y; + } + }; + + const x: Tensor = .init(Shape.init(.{ 2, 5 }, .f32)); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cumsum, .{x}, platform); + defer exe.deinit(); + + const x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } }), std.testing.io); + defer x_buffer.deinit(); + + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._cumsum, .{x_buffer}); + defer res.deinit(); + + try std.testing.expectEqual([2][5]f32{ .{ 0, 1, 2, 2, 3 }, .{ 3, 4, 4, 6, 7 } }, try res.getValue([2][5]f32, std.testing.io)); + } /// Returns a transposed Tensor computed using the given axes. pub fn transpose(self: Tensor, axes_: anytype) Tensor { From 3772d657f0cc5a3b3a244f386f5a6b488a683b5b Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 16:10:10 +0100 Subject: [PATCH 094/881] Re-enabled slice tests --- zml2/tensor.zig | 52 ++++++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 00e6814c..f3d1afe2 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -1543,33 +1543,37 @@ pub const Tensor = struct { return res.reshape(res_shape.removeMany(to_remove.constSlice())); } - // TODO(Corentin) - //test slice { - // const zml = @import("zml.zig"); - // const platform = zml.testing.env(); + test slice { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); - // const x = try zml.Buffer.fromSlice(platform, .{ 2, 5 }, &[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); + const x: Tensor = .init(Shape.init(.{ 2, 5 }, .f32)); - // // Wrap slice1d to hide the anytype in the signature. - // const Local = struct { - // pub fn _slice1dAxis(input: Tensor, ax: i8, slice_: Tensor.Slice) Tensor { - // return input.slice1d(ax, slice_); - // } - // }; + const x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }), std.testing.io); + defer x_buffer.deinit(); - // { - // const res = try zml.testing.compileAndCall(platform, Local._slice1dAxis, .{ x, 0, .{ .end = 1 } }); - // try std.testing.expectEqual([5]f32{ 0, 1, 2, 3, 4 }, try res.getValue([5]f32)); - // } - // { - // const res = try zml.testing.compileAndCall(platform, Local._slice1dAxis, .{ x, 1, .{ .start = 1, .step = 2 } }); - // try std.testing.expectEqual([4]f32{ 1, 3, 6, 8 }, try res.getValue([4]f32)); - // } - // { - // const res = try zml.testing.compileAndCall(platform, Local._slice1dAxis, .{ x, -1, .{ .start = -2 } }); - // try std.testing.expectEqual([4]f32{ 3, 4, 8, 9 }, try res.getValue([4]f32)); - // } - //} + // Wrap slice1d to hide the anytype in the signature. + const Local = struct { + pub fn _slice1dAxis(input: Tensor, ax: i8, slice_: Tensor.Slice) Tensor { + return input.slice1d(ax, slice_); + } + }; + + inline for (.{ + .{ 0, Tensor.Slice{ .end = 1 }, [5]f32{ 0, 1, 2, 3, 4 } }, + .{ 1, Tensor.Slice{ .start = 1, .step = 2 }, [4]f32{ 1, 3, 6, 8 } }, + .{ -1, Tensor.Slice{ .start = -2 }, [4]f32{ 3, 4, 8, 9 } }, + }) |testcase| { + const ax, const slice_, const expectation = testcase; + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._slice1dAxis, .{ x, ax, slice_ }, platform); + defer exe.deinit(); + + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._slice1dAxis, .{x_buffer}); + defer res.deinit(); + + try std.testing.expectEqual(expectation, try res.getValue(@TypeOf(expectation), std.testing.io)); + } + } inline fn wrapIndex(self: Tensor, axis_: usize, idx: i64) i64 { return if (idx < 0) self.dim(axis_) + idx else idx; From 4a71b381d661a769d9c24cef711417d36f6e605e Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 16:30:10 +0100 Subject: [PATCH 095/881] Re-enabled repeat1d tests --- zml2/tensor.zig | 47 ++++++++++++++++++++++------------------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/zml2/tensor.zig b/zml2/tensor.zig index f3d1afe2..ddb8d146 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -1678,36 +1678,33 @@ pub const Tensor = struct { return res; } - // TODO(Corentin) - //test repeat1d { - // const zml = @import("zml.zig"); - // const platform = zml.testing.env(); + test repeat1d { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); - // const Local = struct { - // fn repeat1d(x: Tensor, axis_: u3, n_reps: u32) Tensor { - // return x.repeat1d(axis_, n_reps); - // } - // }; + const Local = struct { + fn repeat1d(x: Tensor, axis_: u3, n_reps: u32) Tensor { + return x.repeat1d(axis_, n_reps); + } + }; - // { - // const inputs: [3]u8 = .{ 1, 2, 3 }; - // const expectations: [6]u8 = .{ 1, 2, 3, 1, 2, 3 }; + inline for (.{ + .{ [3]u8{ 1, 2, 3 }, Shape.init(.{3}, .u8), [6]u8{ 1, 2, 3, 1, 2, 3 }, 0, 2 }, + .{ [2][3]u8{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }, Shape.init(.{ 2, 3 }, .u8), [2][6]u8{ .{ 1, 2, 3, 1, 2, 3 }, .{ 4, 5, 6, 4, 5, 6 } }, 1, 2 }, + }) |testcase| { + const input_data, const shape_, const expectation, const ax, const reps = testcase; + const input: Tensor = .init(shape_); - // const input = try zml.Buffer.fromArray(platform, inputs); - // const output = try zml.testing.compileAndCall(platform, Local.repeat1d, .{ input, 0, 2 }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.repeat1d, .{ input, ax, reps }, platform); + defer exe.deinit(); - // try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); - // } - // { - // const inputs: [2][3]u8 = .{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }; - // const expectations: [2][6]u8 = .{ .{ 1, 2, 3, 1, 2, 3 }, .{ 4, 5, 6, 4, 5, 6 } }; + var input_buffer: zml.Buffer = try .fromBytes(platform, input.shape(), std.mem.sliceAsBytes(&input_data), std.testing.io); + defer input_buffer.deinit(); - // const input = try zml.Buffer.fromArray(platform, inputs); - // const output = try zml.testing.compileAndCall(platform, Local.repeat1d, .{ input, 1, 2 }); - - // try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); - // } - //} + const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local.repeat1d, .{input_buffer}); + try std.testing.expectEqual(expectation, try output.getValue(@TypeOf(expectation), std.testing.io)); + } + } /// Repeats in line each value along the given axis. /// From ca49f06610f6f891b08cc060d37ea6098e53b907 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 16:32:24 +0100 Subject: [PATCH 096/881] Re-enabled stutter1d tests --- zml2/tensor.zig | 57 ++++++++++++++++++++----------------------------- 1 file changed, 23 insertions(+), 34 deletions(-) diff --git a/zml2/tensor.zig b/zml2/tensor.zig index ddb8d146..4b8bfbe5 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -1732,45 +1732,34 @@ pub const Tensor = struct { return res; } - // TODO(Corentin) - //test stutter1d { - // const zml = @import("zml.zig"); - // const platform = zml.testing.env(); + test stutter1d { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); - // const Local = struct { - // fn stutter1d(x: Tensor, axis_: u3, n_reps: u32) Tensor { - // return x.stutter1d(axis_, n_reps); - // } - // }; + const Local = struct { + fn stutter1d(x: Tensor, axis_: u3, n_reps: u32) Tensor { + return x.stutter1d(axis_, n_reps); + } + }; - // { - // const inputs: [3]u8 = .{ 1, 2, 3 }; - // const expectations: [6]u8 = .{ 1, 1, 2, 2, 3, 3 }; + inline for (.{ + .{ [3]u8{ 1, 2, 3 }, Shape.init(.{3}, .u8), [6]u8{ 1, 1, 2, 2, 3, 3 }, 0, 2 }, + .{ [2][3]u8{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }, Shape.init(.{ 2, 3 }, .u8), [2][6]u8{ .{ 1, 1, 2, 2, 3, 3 }, .{ 4, 4, 5, 5, 6, 6 } }, 1, 2 }, + .{ [2][3]u8{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }, Shape.init(.{ 2, 3 }, .u8), [2][6]u8{ .{ 1, 2, 3, 1, 2, 3 }, .{ 4, 5, 6, 4, 5, 6 } }, 0, 2 }, + }) |testcase| { + const input_data, const shape_, const expectation, const ax, const reps = testcase; + const input: Tensor = .init(shape_); - // const input = try zml.Buffer.fromArray(platform, inputs); - // const output = try zml.testing.compileAndCall(platform, Local.stutter1d, .{ input, 0, 2 }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.stutter1d, .{ input, ax, reps }, platform); + defer exe.deinit(); - // try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); - // } - // { - // const inputs: [2][3]u8 = .{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }; - // const expectations: [2][6]u8 = .{ .{ 1, 1, 2, 2, 3, 3 }, .{ 4, 4, 5, 5, 6, 6 } }; + var input_buffer: zml.Buffer = try .fromBytes(platform, input.shape(), std.mem.sliceAsBytes(&input_data), std.testing.io); + defer input_buffer.deinit(); - // const input = try zml.Buffer.fromArray(platform, inputs); - // const output = try zml.testing.compileAndCall(platform, Local.stutter1d, .{ input, 1, 2 }); - - // try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); - // } - // { - // const inputs: [2][3]u8 = .{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }; - // const expectations: [2][6]u8 = .{ .{ 1, 2, 3, 1, 2, 3 }, .{ 4, 5, 6, 4, 5, 6 } }; - - // const input = try zml.Buffer.fromArray(platform, inputs); - // const output = try zml.testing.compileAndCall(platform, Local.stutter1d, .{ input, 0, 2 }); - - // try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); - // } - //} + const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local.stutter1d, .{input_buffer}); + try std.testing.expectEqual(expectation, try output.getValue(@TypeOf(expectation), std.testing.io)); + } + } /// Returns a Tensor containing the element-wise negation of the input Tensor. pub fn negate(self: Tensor) Tensor { From 0c90a0ff2d03b219e9bae5c006365ac0cc7173a3 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 16:33:59 +0100 Subject: [PATCH 097/881] Re-enabled scalar tests --- zml2/tensor.zig | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 4b8bfbe5..fbac28f1 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -1908,22 +1908,22 @@ pub const Tensor = struct { return Tensor.constant(data); } - // TODO(Corentin) - //test scalar { - // const zml = @import("zml.zig"); - // const platform = zml.testing.env(); + test scalar { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); - // const Local = struct { - // pub fn _fwd() [6]Tensor { - // var res: [6]Tensor = undefined; - // const dtypes = .{ .bool, .u8, .i32, .f32, .bf16, .u64 }; - // inline for (0..6) |i| res[i] = scalar(0, dtypes[i]); - // return res; - // } - // }; + const Local = struct { + pub fn _fwd() [6]Tensor { + var res: [6]Tensor = undefined; + const dtypes = .{ .bool, .u8, .i32, .f32, .bf16, .u64 }; + inline for (0..6) |i| res[i] = scalar(0, dtypes[i]); + return res; + } + }; - // _ = try zml.testing.compileAndCall(platform, Local._fwd, .{}); - //} + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{}, platform); + defer exe.deinit(); + } /// Returns a constant Tensor with the given value. pub fn constant(val: DataType.Value) Tensor { From 09e3ddf55d61b971782e9696334eeffabc05ec01 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 16:56:29 +0100 Subject: [PATCH 098/881] Re-enabled gather tests --- mlir/mlir.zig | 5 ++- zml2/ops.zig | 4 +- zml2/tensor.zig | 112 +++++++++++++++++++++++++----------------------- 3 files changed, 64 insertions(+), 57 deletions(-) diff --git a/mlir/mlir.zig b/mlir/mlir.zig index c5dcff60..e9b6f2d7 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -160,7 +160,6 @@ pub const PassManager = opaque { stringRef(opts.treePrintingPath), ); } - }; pub const OpPassManager = opaque { @@ -888,6 +887,10 @@ pub const Value = opaque { pub fn replaceAllUsesWith(self: *const Value, with: *const Value) void { c.mlirValueReplaceAllUsesOfWith(self.ptr(), with.ptr()); } + + pub fn owner(self: *const Value) *Operation { + return @ptrCast(c.mlirOpResultGetOwner(self.ptr()).ptr); + } }; pub const OpOperand = opaque { diff --git a/zml2/ops.zig b/zml2/ops.zig index 956b9d8c..b1165ee0 100644 --- a/zml2/ops.zig +++ b/zml2/ops.zig @@ -616,7 +616,7 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, var idx_batch_axes: Shape.DimsArray = .{}; - var self_kind: stdx.BoundedArray(GatherAxisKind, Shape.MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; + var self_kind: stdx.BoundedArray(GatherAxisKind, constants.MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; for (self._shape.tags(), 0..self.rank()) |t, self_ax| { const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, 1, @intCast(self_ax)); @@ -637,7 +637,7 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, // compute res shape var res_shape = Shape.init(.{}, self.dtype()); - var res_kind: stdx.BoundedArray(GatherAxisKind, Shape.MAX_RANK) = .{}; + var res_kind: stdx.BoundedArray(GatherAxisKind, constants.MAX_RANK) = .{}; for (self_kind.slice(), 0..) |kind, ax_usize| { const ax: u3 = @intCast(ax_usize); if (ax == idx_axes[0]) { diff --git a/zml2/tensor.zig b/zml2/tensor.zig index fbac28f1..4fd0235d 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -2173,67 +2173,71 @@ pub const Tensor = struct { } // TODO: sort indices following self.shape instead of asking the user to do it. - return ops.gather_(self, idx_axes.slice(), idx_per_axis.slice(), opts); + return ops.gather(self, idx_axes.slice(), idx_per_axis.slice(), opts); } - // TODO(Corentin) - //test gather { - // const zml = @import("zml.zig"); - // const platform = zml.testing.env(); + test gather { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + _ = platform; // autofix - // const Local = struct { - // pub fn _idx(idx_shape: anytype) Tensor { - // return Tensor.constant(idx_shape, .{ .i32 = 0 }); - // } - // }; + const Local = struct { + pub fn _idx(idx_shape: anytype) Tensor { + return Tensor.constant(.{ .i32 = 0 }).broad(Shape.init(idx_shape, .i64)); + } + }; - // const idx = Local._idx; + const idx = Local._idx; - // { - // // Only test shapes - // var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); - // defer comp.deinit(); - // comp.activate(); - // defer comp.deactivate(); + { + // Only test shapes + var comp = zml.module.CompilationContext.init(std.testing.allocator); + defer comp.deinit(); + comp.activate(); + defer comp.deactivate(); - // inline for (.{ - // .{ .{ .a = 10 }, .{ .a = idx(.{}) }, .{} }, - // .{ .{ .a = 10 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8 } }, - // .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{}) }, .{ .b = 20 } }, - // .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8, .b = 20 } }, - // // .{ .{ .a = 10, .b = 20 }, 0, idx(.{ .n = 8 }), .{ .n = 8, .b = 20 } }, - // // Favor val shape, instead of indices shape. - // .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .n = 8 }) }, .{ .a = 10, .n = 8 } }, - // .{ .{ .a = 10, .b = 20, .c = 30 }, .{ .b = idx(.{ .n = 8 }) }, .{ .a = 10, .n = 8, .c = 30 } }, - // // batching axes are implicits. - // .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10 }) }, .{ .a = 10 } }, - // .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .b = 20 }) }, .{ .b = 20 } }, - // .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .n = 8 } }, - // // stablehlo.gather is biased toward indices shape (like gatherSlice). - // // This make it awkward to use when you have both batching dimension and new indices dimensions. - // // For now we reject those, and let user explicitly transpose self or indices if needed. - // // .{ .{ .a = 10, .b = 20 }, .{.b = idx(.{ .n = 8, .a = 10 })}, .{ .a = 10, .n = 8 } }, - // // Also handle tuples - // .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }), .b = idx(.{ .n = 8 }) }, .{ .n = 8 } }, - // }) |testcase| { - // const x_shape, const indices, const res_shape = testcase; - // const x = Tensor.constant(x_shape, .{ .f16 = 0 }); - // const y = gather(x, indices, .{}); - // try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); - // try std.testing.expect(y.value().owner().verify()); - // } + const block = mlir.Block.init(&.{}, &.{}); + comp.pushBlock(block); + defer comp.popBlock(); - // inline for (.{ - // .{ .{ 10, 20 }, &[_]u3{ 0, 1 }, &[_]Tensor{ idx(.{8}), idx(.{8}) }, .{8} }, - // }) |testcase| { - // const x_shape, const idx_axes, const idx_per_axis, const res_shape = testcase; - // const x = Tensor.constant(x_shape, .{ .f16 = 0 }); - // const y = gather_(x, idx_axes, idx_per_axis, .{}); - // try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); - // try std.testing.expect(y.value().owner().verify()); - // } - // } - //} + inline for (.{ + .{ .{ .a = 10 }, .{ .a = idx(.{}) }, .{} }, + .{ .{ .a = 10 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{}) }, .{ .b = 20 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8, .b = 20 } }, + // .{ .{ .a = 10, .b = 20 }, 0, idx(.{ .n = 8 }), .{ .n = 8, .b = 20 } }, + // Favor val shape, instead of indices shape. + .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .n = 8 }) }, .{ .a = 10, .n = 8 } }, + .{ .{ .a = 10, .b = 20, .c = 30 }, .{ .b = idx(.{ .n = 8 }) }, .{ .a = 10, .n = 8, .c = 30 } }, + // batching axes are implicits. + .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10 }) }, .{ .a = 10 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .b = 20 }) }, .{ .b = 20 } }, + .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .n = 8 } }, + // stablehlo.gather is biased toward indices shape (like gatherSlice). + // This make it awkward to use when you have both batching dimension and new indices dimensions. + // For now we reject those, and let user explicitly transpose self or indices if needed. + // .{ .{ .a = 10, .b = 20 }, .{.b = idx(.{ .n = 8, .a = 10 })}, .{ .a = 10, .n = 8 } }, + // Also handle tuples + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }), .b = idx(.{ .n = 8 }) }, .{ .n = 8 } }, + }) |testcase| { + const x_shape, const indices, const res_shape = testcase; + const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .i64)); + const y = gather(x, indices, .{}); + try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); + try std.testing.expect(y.value().owner().verify()); + } + + inline for (.{ + .{ .{ 10, 20 }, &[_]u3{ 0, 1 }, &[_]Tensor{ idx(.{8}), idx(.{8}) }, .{8} }, + }) |testcase| { + const x_shape, const idx_axes, const idx_per_axis, const res_shape = testcase; + const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .i64)); + const y = ops.gather(x, idx_axes, idx_per_axis, .{}); + try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); + try std.testing.expect(y.value().owner().verify()); + } + } + } /// Gathers slices along the given axes with runtime indices. /// * slice_shape represents the shape of the slices to extract, From fe91568a2cbf1937c2accaa58a7e4e93068eda5c Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 17:12:24 +0100 Subject: [PATCH 099/881] Re-enabled gatherSlices tests --- zml2/tensor.zig | 151 ++++++++++++++++++++++++++---------------------- 1 file changed, 81 insertions(+), 70 deletions(-) diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 4fd0235d..2152c52f 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -2331,83 +2331,94 @@ pub const Tensor = struct { return _result(res_shape, gather_op.result(0)); } - // TODO(Corentin) - //test gatherSlices { - // const zml = @import("zml.zig"); - // const platform = zml.testing.env(); + test gatherSlices { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); - // const Local = struct { - // pub fn _gatherSlices(self: Tensor, slice_shape: Shape, indices: Tensor, opts: GatherOpts) Tensor { - // return self.gatherSlices(slice_shape, indices, opts); - // } - // }; + const Local = struct { + pub fn _gatherSlices(self: Tensor, slice_shape: Shape, indices: Tensor, opts: GatherOpts) Tensor { + return self.gatherSlices(slice_shape, indices, opts); + } + }; - // { - // // Only test shapes - // var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); - // defer comp.deinit(); - // comp.activate(); - // defer comp.deactivate(); + { + // Only test shapes + var comp = zml.module.CompilationContext.init(std.testing.allocator); + defer comp.deinit(); + comp.activate(); + defer comp.deactivate(); - // inline for (.{ - // .{ .{ .a = 10 }, .{}, .{ ._ = 0 }, .{ .a = 10 } }, - // .{ .{ .a = 10 }, .{ .a = 7 }, .{ ._ = 1 }, .{ .a = 7 } }, - // .{ .{ .a = 10 }, .{ .a = 7 }, .{ .n = 8, ._ = 1 }, .{ .n = 8, .a = 7 } }, - // .{ .{ .a = 10 }, .{ .a = 7 }, .{ .coord = 1, .n = 8 }, .{ .n = 8, .a = 7 } }, - // // tags aren't required. - // .{ .{10}, .{7}, .{ .n = 8, ._ = 1 }, .{ .n = 8, ._ = 7 } }, - // .{ .{ .a = 10, .b = 20 }, .{ .a = 7 }, .{ ._ = 1 }, .{ .a = 7, .b = 20 } }, - // .{ .{ .a = 10, .b = 20 }, .{ .a = 7 }, .{ .n = 8, ._ = 1 }, .{ .n = 8, .a = 7, .b = 20 } }, - // .{ .{ .a = 10, .b = 20 }, .{ .a = 7 }, .{ .n = 8, .coord = 1, .m = 9 }, .{ .n = 8, .m = 9, .a = 7, .b = 20 } }, - // .{ .{ .a = 10, .b = 20 }, .{ .b = 17 }, .{ .n = 8, ._ = 1 }, .{ .n = 8, .a = 10, .b = 17 } }, - // .{ .{ .a = 10, .b = 20 }, .{ .a = 7, .b = 17 }, .{ .n = 8, ._ = 2 }, .{ .n = 8, .a = 7, .b = 17 } }, - // // Note: currently the order of the axes in the slice is not used. - // .{ .{ .a = 10, .b = 20 }, .{ .b = 17, .a = 7 }, .{ .n = 8, ._ = 2 }, .{ .n = 8, .a = 7, .b = 17 } }, - // .{ .{ .a = 10, .b = 20, .c = 20 }, .{ .b = 17 }, .{ .n = 8, ._ = 1 }, .{ .n = 8, .a = 10, .b = 17, .c = 20 } }, - // // batching dims - // .{ .{ .a = 10, .b = 20 }, .{ .b = 17 }, .{ .a = 10, ._ = 1 }, .{ .a = 10, .b = 17 } }, - // .{ .{ .b = 200, .a = 100, .c = 300 }, .{ .c = 300 }, .{ .a = 100, .b = 200, ._ = 1 }, .{ .a = 100, .b = 200, .c = 300 } }, - // }) |testcase| { - // const x_shape, const slice_dims, const idx_shape, const res_shape = testcase; - // const x = Tensor.constant(x_shape, .{ .f16 = 0 }); - // const slice_shape = Shape.init(slice_dims, .u16); - // const idx = Tensor.constant(idx_shape, .{ .i32 = 0 }); - // const y = gatherSlices(x, slice_shape, idx, .{}); - // try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); - // try std.testing.expect(y.value().owner().verify()); + const block = mlir.Block.init(&.{}, &.{}); + comp.pushBlock(block); + defer comp.popBlock(); - // const mod = try zml.compileFn( - // std.testing.allocator, - // Local._gatherSlices, - // .{ x.shape(), slice_shape, idx.shape(), .{ .indices_are_sorted = true } }, - // platform, - // ); - // defer mod.deinit(); - // } - // } + inline for (.{ + .{ .{ .a = 10 }, .{}, .{ ._ = 0 }, .{ .a = 10 } }, + .{ .{ .a = 10 }, .{ .a = 7 }, .{ ._ = 1 }, .{ .a = 7 } }, + .{ .{ .a = 10 }, .{ .a = 7 }, .{ .n = 8, ._ = 1 }, .{ .n = 8, .a = 7 } }, + .{ .{ .a = 10 }, .{ .a = 7 }, .{ .coord = 1, .n = 8 }, .{ .n = 8, .a = 7 } }, + // tags aren't required. + .{ .{10}, .{7}, .{ .n = 8, ._ = 1 }, .{ .n = 8, ._ = 7 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = 7 }, .{ ._ = 1 }, .{ .a = 7, .b = 20 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = 7 }, .{ .n = 8, ._ = 1 }, .{ .n = 8, .a = 7, .b = 20 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = 7 }, .{ .n = 8, .coord = 1, .m = 9 }, .{ .n = 8, .m = 9, .a = 7, .b = 20 } }, + .{ .{ .a = 10, .b = 20 }, .{ .b = 17 }, .{ .n = 8, ._ = 1 }, .{ .n = 8, .a = 10, .b = 17 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = 7, .b = 17 }, .{ .n = 8, ._ = 2 }, .{ .n = 8, .a = 7, .b = 17 } }, + // Note: currently the order of the axes in the slice is not used. + .{ .{ .a = 10, .b = 20 }, .{ .b = 17, .a = 7 }, .{ .n = 8, ._ = 2 }, .{ .n = 8, .a = 7, .b = 17 } }, + .{ .{ .a = 10, .b = 20, .c = 20 }, .{ .b = 17 }, .{ .n = 8, ._ = 1 }, .{ .n = 8, .a = 10, .b = 17, .c = 20 } }, + // batching dims + .{ .{ .a = 10, .b = 20 }, .{ .b = 17 }, .{ .a = 10, ._ = 1 }, .{ .a = 10, .b = 17 } }, + .{ .{ .b = 200, .a = 100, .c = 300 }, .{ .c = 300 }, .{ .a = 100, .b = 200, ._ = 1 }, .{ .a = 100, .b = 200, .c = 300 } }, + }) |testcase| { + const x_shape, const slice_dims, const idx_shape, const res_shape = testcase; + const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); + const slice_shape = Shape.init(slice_dims, .u16); + const idx = Tensor.constant(.{ .i32 = 0 }).broad(Shape.init(idx_shape, .i32)); + const y = gatherSlices(x, slice_shape, idx, .{}); + try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); + try std.testing.expect(y.value().owner().verify()); - // // Test with actual values. - // const range = try zml.HostBuffer.arange(std.testing.allocator, .{ .end = 2 * 4 * 6 }, .u16); - // defer range.deinit(std.testing.allocator); - // const operand = try range.reshape(.{ .a = 2, .b = 4, .c = 6 }).toDevice(platform); - // defer operand.deinit(); - // const start_indices = (try zml.Buffer.fromArray(platform, [2][2]i32{ .{ 2, 1 }, .{ 0, 3 } })).withTags(.{ .n, ._ }); - // defer start_indices.deinit(); + _ = Local._gatherSlices(x, slice_shape, idx, .{ .indices_are_sorted = true }); + } + } - // const result = try zml.testing.compileAndCall(platform, Local._gatherSlices, .{ operand, Shape.init(.{ .b = 2, .c = 3 }, .u16), start_indices, .{} }); + // Test with actual values. + const operand: Tensor = .init(Shape.init(.{ .a = 2, .b = 4, .c = 6 }, .u16)); + const indices: Tensor = .init(Shape.init(.{ .n = 2, ._ = 2 }, .i32)); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._gatherSlices, .{ operand, Shape.init(.{ .b = 2, .c = 3 }, .u16), indices, .{} }, platform); + defer exe.deinit(); - // const expected = zml.HostBuffer.fromArray(&[2][2][2][3]u16{ - // .{ - // .{ .{ 13, 14, 15 }, .{ 19, 20, 21 } }, - // .{ .{ 37, 38, 39 }, .{ 43, 44, 45 } }, - // }, - // .{ - // .{ .{ 3, 4, 5 }, .{ 9, 10, 11 } }, - // .{ .{ 27, 28, 29 }, .{ 33, 34, 35 } }, - // }, - // }); - // try zml.testing.expectClose(expected, result, 0); - //} + const indices_buffer: zml.Buffer = try .fromBytes(platform, indices.shape(), std.mem.sliceAsBytes(&[2][2]i32{ .{ 2, 1 }, .{ 0, 3 } }), std.testing.io); + defer indices_buffer.deinit(); + const operand_buffer: zml.Buffer = b: { + const temp_func = struct { + fn forward() Tensor { + return Tensor.arange(.{ .end = 2 * 4 * 6 }, .u16).reshape(.{ .a = 2, .b = 4, .c = 6 }); + } + }.forward; + var temp_exe = try zml.module.compile(std.testing.allocator, std.testing.io, temp_func, .{}, platform); + defer temp_exe.deinit(); + + const buffer = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &temp_exe, temp_func, {}); + break :b buffer; + }; + defer operand_buffer.deinit(); + const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._gatherSlices, .{ operand_buffer, indices_buffer }); + defer result.deinit(); + + const expected: zml.testing.Slice = .init(Shape.init(.{ 2, 2, 2, 3 }, .u16), std.mem.sliceAsBytes(&[2][2][2][3]u16{ + .{ + .{ .{ 13, 14, 15 }, .{ 19, 20, 21 } }, + .{ .{ 37, 38, 39 }, .{ 43, 44, 45 } }, + }, + .{ + .{ .{ 3, 4, 5 }, .{ 9, 10, 11 } }, + .{ .{ 27, 28, 29 }, .{ 33, 34, 35 } }, + }, + })); + try zml.testing.expectClose(std.testing.io, expected, result, 0); + } pub const ScatterOpts = struct { /// Promise scatter that all coordinates in `indices` are sorted, wrt to the final offset in `self` From a56fe39534f1c0a2bab3c67df1580d239b63e85d Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 17:36:21 +0100 Subject: [PATCH 100/881] Re-enabled argMax tests --- zml2/tensor.zig | 71 +++++++++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 2152c52f..946d83f9 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -2744,40 +2744,47 @@ pub const Tensor = struct { return .{ .values = values, .indices = indices }; } - // TODO(Corentin) - //test argMax { - // const zml = @import("zml.zig"); - // const platform = zml.testing.env(); - // const allocator = std.testing.allocator; - // const ArgMaxTest = struct { - // pub fn _fwd(x: Tensor) Tensor.ArgMaxRes { - // return x.argMax(1); - // } - // }; + test argMax { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const allocator = std.testing.allocator; + const ArgMaxTest = struct { + pub fn _fwd(x: Tensor) Tensor.ArgMaxRes { + return x.argMax(1); + } + }; - // const argmax = try zml.compileFn(allocator, ArgMaxTest._fwd, .{Shape.init(.{ 1, 5 }, .f32)}, platform); - // defer argmax.deinit(); - // // Test with tie - // { - // const x = try zml.Buffer.fromArray(platform, [1][5]f32{.{ 5.0, 4.1, 7.9, 0, 7.9 }}); - // const res = argmax.call(.{x}); - // const max_ = res.values.getValue(f32); - // const max_idx = res.indices.getValue(i32); - // try std.testing.expectEqual(max_, 7.9); - // // We should always return the first max found. - // try std.testing.expectEqual(max_idx, 2); - // } + const x: Tensor = .init(Shape.init(.{ 1, 5 }, .f32)); + var exe = try zml.module.compile(allocator, std.testing.io, ArgMaxTest._fwd, .{x}, platform); + defer exe.deinit(); - // // Test with Nan - // { - // const x = try zml.Buffer.fromArray(platform, [1][5]f32{.{ 5.0, std.math.nan(f32), 7.9, 0, 7.9 }}); - // const res = argmax.call(.{x}); - // const max_ = try res.values.getValue(f32); - // const max_idx = try res.indices.getValue(i32); - // try std.testing.expect(std.math.isNan(max_)); - // try std.testing.expectEqual(max_idx, 1); - // } - //} + { + var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, 4.1, 7.9, 0, 7.9 }}), std.testing.io); + defer x_buffer.deinit(); + + const res = try zml.testing.autoCall(allocator, std.testing.io, &exe, ArgMaxTest._fwd, .{x_buffer}); + defer res.values.deinit(); + defer res.indices.deinit(); + const max_ = res.values.getValue(f32, std.testing.io); + const max_idx = res.indices.getValue(i32, std.testing.io); + try std.testing.expectEqual(max_, 7.9); + // We should always return the first max found. + try std.testing.expectEqual(max_idx, 2); + } + + { + var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, std.math.nan(f32), 7.9, 0, 7.9 }}), std.testing.io); + defer x_buffer.deinit(); + + const res = try zml.testing.autoCall(allocator, std.testing.io, &exe, ArgMaxTest._fwd, .{x_buffer}); + defer res.values.deinit(); + defer res.indices.deinit(); + const max_ = try res.values.getValue(f32, std.testing.io); + const max_idx = try res.indices.getValue(i32, std.testing.io); + try std.testing.expect(std.math.isNan(max_)); + try std.testing.expectEqual(max_idx, 1); + } + } pub const SortRes = ArgMaxRes; From 145a8024917de58bb7973ed893215cbb1fcab468 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 18:03:14 +0100 Subject: [PATCH 101/881] Re-enabled argsort tests --- mlir/dialects/stablehlo/stablehlo.zig | 2 +- zml2/ops.zig | 24 ++-- zml2/tensor.zig | 166 +++++++++++++++----------- 3 files changed, 110 insertions(+), 82 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index b3404e78..2ad416e7 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -1387,7 +1387,7 @@ pub fn serializePortableArtifact2(module: *mlir.Module, target_version: []const pub fn return_(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.return", .{ - .operands = .{ .flat = &.{&.{value}} }, + .operands = .{ .flat = &.{value} }, .location = location, .verify = false, }); diff --git a/zml2/ops.zig b/zml2/ops.zig index b1165ee0..f94cc79a 100644 --- a/zml2/ops.zig +++ b/zml2/ops.zig @@ -197,13 +197,13 @@ pub const SortArgs = struct { right: Tensor, }; -pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytype, is_stable: bool) stdx.meta.FnResult(func) { +pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytype, is_stable: bool) [inputs.len]Tensor { var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); defer arena.deinit(); const mlir_ctx = CompilationContext.current().mlir_ctx; - const sort_block, var result = b: { + const sort_block = b: { const ArgsType = std.meta.Tuple(&[1]type{SortArgs} ** inputs.len); var args: ArgsType = undefined; var block_types: [2 * inputs.len]*const mlir.Type = undefined; @@ -212,8 +212,8 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytyp args[i].left = Tensor.init(Shape.init(.{}, inputs[i].shape().dtype())); args[i].right = Tensor.init(Shape.init(.{}, inputs[i].shape().dtype())); - block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); - block_types[i + inputs.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); + block_types[2 * i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); + block_types[2 * i + 1] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); } const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); @@ -225,19 +225,14 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytyp const scope = CompilationContext.current().currentScope(); inline for (0..inputs.len) |i| { - scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; - scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inputs.len) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, 2 * i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, 2 * i + 1) catch unreachable; } var result = @call(.auto, func, args ++ context); - var result_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| { - result_values[i] = result[i].value(); - } - - _ = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)).appendTo(sort_block); - break :b .{ sort_block, result }; + _ = dialects.stablehlo.return_(mlir_ctx, result.value(), .unknown(mlir_ctx)).appendTo(sort_block); + break :b sort_block; }; var input_values: [inputs.len]*const mlir.Value = undefined; @@ -257,7 +252,8 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytyp .location = .unknown(mlir_ctx), }).appendTo(CompilationContext.current().currentScope().block); - inline for (0..result.len) |i| { + var result: [inputs.len]Tensor = undefined; + inline for (0..inputs.len) |i| { result[i] = Tensor._result(inputs[i].shape(), sort_op.result(i)); } diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 946d83f9..e746d9e5 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -2794,11 +2794,12 @@ pub const Tensor = struct { pub fn sort(self: Tensor, axis_: anytype, opts: SortOpts) SortRes { const a = self.axis(axis_); const indices = Tensor.arange(.{ .end = self.dim(a) }, .i32).broadcast(self._shape, &.{a}); + const direction: dialects.stablehlo.ComparisonDirection.Direction = if (opts.descending) .GT else .LT; const res = ops.sort(.{ self, indices }, a, struct { - fn call(direction: dialects.stablehlo.ComparisonDirection.Direction, values: ops.SortArgs, _: ops.SortArgs) Tensor { - return values.left.cmp(direction, values.right); + fn call(values: ops.SortArgs, _: ops.SortArgs, direction_: dialects.stablehlo.ComparisonDirection.Direction) Tensor { + return values.left.cmp(direction_, values.right); } - }.call, .{if (opts.descending) .GT else .LT}, true); + }.call, .{direction}, true); return .{ .values = res[0], .indices = res[1] }; } @@ -2809,72 +2810,103 @@ pub const Tensor = struct { return self.sort(axis_, .{ .descending = opts.descending }).indices; } - // TODO(Corentin) - //test argsort { - // const zml = @import("zml.zig"); - // const platform = zml.testing.env(); + test argsort { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); - // const Local = struct { - // pub fn _argsort(x: Tensor, axis_: u3, opts: ArgSortOpts) Tensor { - // return x.argsort(axis_, opts); - // } - // }; + const Local = struct { + pub fn _argsort(x: Tensor, axis_: u3, opts: ArgSortOpts) Tensor { + return x.argsort(axis_, opts); + } + }; - // var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); - // defer arena_state.deinit(); - // const allocator = arena_state.allocator(); - // // 2D Tensor - dim = 1, ascending - // { - // const x = try zml.Buffer.fromSlice(platform, .{ 2, 5 }, &[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 }); - // const res = try zml.testing.compileAndCall(platform, Local._argsort, .{ x, 1, .{} }); - // const res_cpu = try res.toHostAlloc(allocator); - // try std.testing.expectEqualSlices(i32, &.{ 0, 3, 1, 2, 4, 1, 4, 3, 0, 2 }, res_cpu.items(i32)); - // } - // // 3D Tensor, dim = 1, descending - // { - // const x = try zml.Buffer.fromSlice(platform, .{ 1, 5, 10 }, &[_]f16{ - // -0.2505, 1.2520, -0.7041, 0.1066, 1.2773, -1.7246, 0.8389, 1.1094, 0.0601, 1.0684, - // 0.9619, 1.3916, 1.2246, -0.1406, 0.3674, -1.2480, -1.7051, -0.0934, 0.3435, 0.4373, - // 1.3809, 0.5444, -0.6079, 1.2031, -0.6880, 1.2979, -0.1869, 0.2991, 0.0156, 0.1847, - // 0.6626, -0.3040, -0.8726, -1.4805, -1.6943, 1.1055, -2.0078, -0.5288, 0.8813, 0.8008, - // 2.0527, 1.1230, 0.5430, 0.2494, -0.9434, 0.7876, 0.1818, 0.9258, -2.4902, 1.5918, - // }); - // const res_dev = try zml.testing.compileAndCall(platform, Local._argsort, .{ x, 1, .{ .descending = true } }); - // const res = try res_dev.toHostAlloc(allocator); - // try std.testing.expectEqualSlices(i32, &.{ - // 4, 1, 1, 2, 0, 2, 0, 0, 3, 4, - // 2, 0, 4, 4, 1, 3, 4, 4, 1, 0, - // 1, 4, 2, 0, 2, 4, 2, 2, 0, 3, - // 3, 2, 0, 1, 4, 1, 1, 1, 2, 1, - // 0, 3, 3, 3, 3, 0, 3, 3, 4, 2, - // }, res.items(i32)); - // } - // // 4D Tensor, dim = 3, ascending - // { - // const x = try zml.Buffer.fromSlice(platform, .{ 4, 2, 1, 4 }, &[_]i32{ - // 89, 31, 22, 42, - // 64, 39, 0, 30, - // 64, 71, 46, 31, - // 89, 82, 78, 86, - // 55, 32, 43, 19, - // 93, 24, 45, 72, - // 64, 86, 62, 88, - // 57, 21, 19, 12, - // }); - // const res_dev = try zml.testing.compileAndCall(platform, Local._argsort, .{ x, 3, .{} }); - // const res = try res_dev.toHostAlloc(allocator); - // try std.testing.expectEqualSlices(i32, &.{ - // 2, 1, 3, 0, - // 2, 3, 1, 0, - // 3, 2, 0, 1, - // 2, 1, 3, 0, - // 3, 1, 2, 0, - // 1, 2, 3, 0, - // 2, 0, 1, 3, - // 3, 2, 1, 0, - // }, res.items(i32)); - // } - //} + // 2D Tensor - dim = 1, ascending + { + const x: Tensor = .init(Shape.init(.{ 2, 5 }, .f32)); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{} }, platform); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 }), std.testing.io); + defer x_buffer.deinit(); + + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); + defer res.deinit(); + + const res_cpu = try res.toHostAlloc(std.testing.allocator, std.testing.io); + defer std.testing.allocator.free(res_cpu); + + try std.testing.expectEqualSlices(i32, &.{ 0, 3, 1, 2, 4, 1, 4, 3, 0, 2 }, @alignCast(std.mem.bytesAsSlice(i32, res_cpu))); + } + + // 3D Tensor, dim = 1, descending + { + const x: Tensor = .init(Shape.init(.{ 1, 5, 10 }, .f16)); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{ .descending = true } }, platform); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[_]f16{ + -0.2505, 1.2520, -0.7041, 0.1066, 1.2773, -1.7246, 0.8389, 1.1094, 0.0601, 1.0684, + 0.9619, 1.3916, 1.2246, -0.1406, 0.3674, -1.2480, -1.7051, -0.0934, 0.3435, 0.4373, + 1.3809, 0.5444, -0.6079, 1.2031, -0.6880, 1.2979, -0.1869, 0.2991, 0.0156, 0.1847, + 0.6626, -0.3040, -0.8726, -1.4805, -1.6943, 1.1055, -2.0078, -0.5288, 0.8813, 0.8008, + 2.0527, 1.1230, 0.5430, 0.2494, -0.9434, 0.7876, 0.1818, 0.9258, -2.4902, 1.5918, + }), std.testing.io); + defer x_buffer.deinit(); + + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); + defer res.deinit(); + + const res_cpu = try res.toHostAlloc(std.testing.allocator, std.testing.io); + defer std.testing.allocator.free(res_cpu); + + try std.testing.expectEqualSlices(i32, &.{ + 4, 1, 1, 2, 0, 2, 0, 0, 3, 4, + 2, 0, 4, 4, 1, 3, 4, 4, 1, 0, + 1, 4, 2, 0, 2, 4, 2, 2, 0, 3, + 3, 2, 0, 1, 4, 1, 1, 1, 2, 1, + 0, 3, 3, 3, 3, 0, 3, 3, 4, 2, + }, @alignCast(std.mem.bytesAsSlice(i32, res_cpu))); + } + + // 4D Tensor, dim = 3, ascending + { + const x: Tensor = .init(Shape.init(.{ 4, 2, 1, 4 }, .i32)); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 3, .{} }, platform); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ + 89, 31, 22, 42, + 64, 39, 0, 30, + 64, 71, 46, 31, + 89, 82, 78, 86, + 55, 32, 43, 19, + 93, 24, 45, 72, + 64, 86, 62, 88, + 57, 21, 19, 12, + }), std.testing.io); + defer x_buffer.deinit(); + + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); + defer res.deinit(); + + const res_cpu = try res.toHostAlloc(std.testing.allocator, std.testing.io); + defer std.testing.allocator.free(res_cpu); + + try std.testing.expectEqualSlices(i32, &.{ + 2, 1, 3, 0, + 2, 3, 1, 0, + 3, 2, 0, 1, + 2, 1, 3, 0, + 3, 1, 2, 0, + 1, 2, 3, 0, + 2, 0, 1, 3, + 3, 2, 1, 0, + }, @alignCast(std.mem.bytesAsSlice(i32, res_cpu))); + } + } pub const TopKOpts = SortOpts; From f59d5c5b19de55062922428984641c2e3c0be9fc Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 18:08:37 +0100 Subject: [PATCH 102/881] Re-enabled chunkExact tests --- zml2/tensor.zig | 49 ++++++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/zml2/tensor.zig b/zml2/tensor.zig index e746d9e5..621f83e1 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -3037,32 +3037,35 @@ pub const Tensor = struct { return chunks; } - // TODO(Corentin) - //test chunkExact { - // const zml = @import("zml.zig"); - // const platform = zml.testing.env(); + test chunkExact { + const zml = @import("zml.zig"); - // // Only test shapes - // var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); - // defer comp.deinit(); - // comp.activate(); - // defer comp.deactivate(); + // Only test shapes + var comp = zml.module.CompilationContext.init(std.testing.allocator); + defer comp.deinit(); + comp.activate(); + defer comp.deactivate(); - // inline for (.{ - // .{ .{ .a = 12 }, .a, 3, .{ .a = 4 } }, - // .{ .{ .a = 12, .b = 2 }, .a, 3, .{ .a = 4, .b = 2 } }, - // .{ .{ 12, 2 }, 0, 3, .{ 4, 2 } }, - // }) |testcase| { - // const x_shape, const ax, const n_chunks, const res = testcase; - // const x = Tensor.constant(x_shape, .{ .f16 = 0 }); - // const chunks = x.chunkExact(ax, n_chunks); + const block = mlir.Block.init(&.{}, &.{}); + defer block.deinit(); + comp.pushBlock(block); + defer comp.popBlock(); - // const res_shape = Shape.init(res, .f16); - // for (chunks) |chk| { - // try zml.testing.expectEqualShapes(res_shape, chk.shape()); - // } - // } - //} + inline for (.{ + .{ .{ .a = 12 }, .a, 3, .{ .a = 4 } }, + .{ .{ .a = 12, .b = 2 }, .a, 3, .{ .a = 4, .b = 2 } }, + .{ .{ 12, 2 }, 0, 3, .{ 4, 2 } }, + }) |testcase| { + const x_shape, const ax, const n_chunks, const res = testcase; + const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); + const chunks = x.chunkExact(ax, n_chunks); + + const res_shape = Shape.init(res, .f16); + for (chunks) |chk| { + try zml.testing.expectEqualShapes(res_shape, chk.shape()); + } + } + } /// Chunk a given tensor into n parts of equal shape, and one part with the remaining items. /// When `self.dim(axis_)` is divisible by `n_chunks` it will return exactly `n_chunks`. From b0110a2b25e27b6e18bf90c05e3fd67f41c4ddf5 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 18:19:13 +0100 Subject: [PATCH 103/881] Re-enableds chunkAllowTrailing tests I had to make choice about allocation to avoid memory leaks --- zml2/tensor.zig | 76 ++++++++++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 621f83e1..354076ba 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -3071,18 +3071,18 @@ pub const Tensor = struct { /// When `self.dim(axis_)` is divisible by `n_chunks` it will return exactly `n_chunks`. pub fn chunkAllowTrailing( self: Tensor, + allocator: std.mem.Allocator, axis_: i64, n_chunks: comptime_int, - ) []Tensor { + ) ![]Tensor { const a = self.axis(axis_); const d = self.dim(a); const chunk_size: i64 = @divFloor(d, n_chunks); const tail_chunk_size: i64 = @rem(d, chunk_size); - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); - defer arena.deinit(); + var chunks = std.ArrayList(Tensor).initCapacity(allocator, n_chunks + 1) catch @panic("OOM"); + defer chunks.deinit(allocator); - var chunks = std.ArrayList(Tensor).initCapacity(arena.allocator(), n_chunks + 1) catch @panic("OOM"); for (0..n_chunks) |i| { const start: i64 = @as(i64, @intCast(i)) * chunk_size; chunks.appendAssumeCapacity( @@ -3093,43 +3093,47 @@ pub const Tensor = struct { const start: i64 = n_chunks * chunk_size; chunks.appendAssumeCapacity(self.slice1d(a, .{ .start = start })); } - return chunks.items; + return chunks.toOwnedSlice(allocator); } - // TODO(Corentin) - //test chunkAllowTrailing { - // const zml = @import("zml.zig"); - // const platform = zml.testing.env(); + test chunkAllowTrailing { + const zml = @import("zml.zig"); - // // Only test shapes - // var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); - // defer comp.deinit(); - // comp.activate(); - // defer comp.deactivate(); + // Only test shapes + var comp = zml.module.CompilationContext.init(std.testing.allocator); + defer comp.deinit(); + comp.activate(); + defer comp.deactivate(); - // inline for (.{ - // .{ .{ .a = 10 }, .a, 3, .{ .a = 3 }, .{ .a = 1 } }, - // .{ .{ .a = 10, .b = 2 }, .a, 3, .{ .a = 3, .b = 2 }, .{ .a = 1, .b = 2 } }, - // .{ .{ 10, 2 }, 0, 3, .{ 3, 2 }, .{ 1, 2 } }, - // .{ .{ 12, 2 }, 0, 3, .{ 4, 2 }, .{} }, - // }) |testcase| { - // const x_shape, const ax, const n_chunks, const res, const trailing = testcase; - // const x = Tensor.constant(x_shape, .{ .f16 = 0 }); - // const chunks = x.chunkAllowTrailing(x.axis(ax), n_chunks); + const block = mlir.Block.init(&.{}, &.{}); + defer block.deinit(); + comp.pushBlock(block); + defer comp.popBlock(); - // const res_shape = Shape.init(res, .f16); - // for (chunks[0..n_chunks]) |chk| { - // try zml.testing.expectEqualShapes(res_shape, chk.shape()); - // } - // const trailing_shape = Shape.init(trailing, .f16); - // if (trailing_shape.rank() > 0) { - // try std.testing.expectEqual(n_chunks + 1, chunks.len); - // try zml.testing.expectEqualShapes(trailing_shape, chunks[n_chunks].shape()); - // } else { - // try std.testing.expectEqual(n_chunks, chunks.len); - // } - // } - //} + inline for (.{ + .{ .{ .a = 10 }, .a, 3, .{ .a = 3 }, .{ .a = 1 } }, + .{ .{ .a = 10, .b = 2 }, .a, 3, .{ .a = 3, .b = 2 }, .{ .a = 1, .b = 2 } }, + .{ .{ 10, 2 }, 0, 3, .{ 3, 2 }, .{ 1, 2 } }, + .{ .{ 12, 2 }, 0, 3, .{ 4, 2 }, .{} }, + }) |testcase| { + const x_shape, const ax, const n_chunks, const res, const trailing = testcase; + const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); + const chunks = try x.chunkAllowTrailing(std.testing.allocator, x.axis(ax), n_chunks); + defer std.testing.allocator.free(chunks); + + const res_shape = Shape.init(res, .f16); + for (chunks[0..n_chunks]) |chk| { + try zml.testing.expectEqualShapes(res_shape, chk.shape()); + } + const trailing_shape = Shape.init(trailing, .f16); + if (trailing_shape.rank() > 0) { + try std.testing.expectEqual(n_chunks + 1, chunks.len); + try zml.testing.expectEqualShapes(trailing_shape, chunks[n_chunks].shape()); + } else { + try std.testing.expectEqual(n_chunks, chunks.len); + } + } + } pub fn split(self: Tensor, axis_: anytype, split_sizes: []const i64) []Tensor { stdx.debug.assert(split_sizes.len > 0, "split expects at least one 'split_sizes', got 0", .{}); From 5db4a7ed298509bb4784fa2d84ec9bd5badff539 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 18:30:01 +0100 Subject: [PATCH 104/881] Re-enabled dynamicSlice tests --- zml2/tensor.zig | 41 ++++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 354076ba..ef2b122d 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -3233,29 +3233,32 @@ pub const Tensor = struct { return _result(res_shape, op.result(0)); } - // TODO(Corentin) - //test dynamicSlice { - // const zml = @import("zml.zig"); - // const platform = zml.testing.env(); - // const T = f32; + test dynamicSlice { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); - // { - // const x = try zml.Buffer.fromArray(platform, [10]T{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); - // const z = try zml.Buffer.scalar(platform, 4, .i32); - // const res = try zml.testing.compileAndCall(platform, Tensor.dynamicSlice1d, .{ x, 0, .{ .len = 2, .start = z } }); + inline for (.{ + .{ [10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, Shape.init(.{10}, .f32), [2]f32{ 4, 5 }, 4, 0 }, + .{ [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }, Shape.init(.{ 2, 5 }, .f32), [4]f32{ 3, 4, 8, 9 }, 3, 1 }, + }) |testcase| { + const x_data, const x_shape, const expectation, const z_value: i32, const ax = testcase; + const x: Tensor = .init(x_shape); + const z: Tensor = .init(Shape.init(.{}, .i32)); - // try std.testing.expectEqual([2]T{ 4, 5 }, try res.getValue([2]T)); - // } + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.dynamicSlice1d, .{ x, ax, .{ .len = 2, .start = z } }, platform); + defer exe.deinit(); - // { - // // Strided - // const x = try zml.Buffer.fromArray(platform, [2][5]T{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }); - // const z = try zml.Buffer.scalar(platform, 3, .i32); + var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&x_data), std.testing.io); + defer x_buffer.deinit(); + var z_buffer: zml.Buffer = try .fromBytes(platform, z.shape(), std.mem.asBytes(&z_value), std.testing.io); + defer z_buffer.deinit(); - // const res = try zml.testing.compileAndCall(platform, Tensor.dynamicSlice1d, .{ x, 1, .{ .len = 2, .start = z } }); - // try std.testing.expectEqual([4]T{ 3, 4, 8, 9 }, res.getValue([4]T)); - // } - //} + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.dynamicSlice1d, .{ x_buffer, .{ .start = z_buffer } }); + defer res.deinit(); + + try std.testing.expectEqual(expectation, try res.getValue(@TypeOf(expectation), std.testing.io)); + } + } /// Updates a slice of the input Tensor along a specific axis using the given 'update' Tensor, with a start offset known at runtime. /// Note this is the untagged api, if you have tags, you should use dynamicUpdateSlice directly. From f10a38c129a29ca06c3c523b058fe15c3f86acf1 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 18:35:12 +0100 Subject: [PATCH 105/881] Re-enabled toDiagonal tests --- zml2/tensor.zig | 61 ++++++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/zml2/tensor.zig b/zml2/tensor.zig index ef2b122d..e5de453a 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -3490,7 +3490,7 @@ pub const Tensor = struct { res_shape._tags.replaceRange(a, 1, &.{ @tagName(new_tags[0]), @tagName(new_tags[1]) }) catch unreachable; const values = self.insertAxes(a + 1, .{new_tags[1]}).broad(res_shape); - const zeros = Tensor.constant(res_shape, self.dtype().zero()); + const zeros = Tensor.constant(self.dtype().zero()).broad(res_shape); const x = Tensor.iota(res_shape, a); const y = Tensor.iota(res_shape, a + 1); @@ -3499,35 +3499,40 @@ pub const Tensor = struct { return res; } - // TODO(Corentin) - //test toDiagonal { - // const zml = @import("zml.zig"); - // const platform = zml.testing.env(); + test toDiagonal { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); - // const Local = struct { - // pub fn _toDiag(input: Tensor) Tensor { - // const res = input.toDiagonal(-1, .{ .x, .y }); - // std.debug.assert(res.dim(.x) == input.dim(-1)); - // std.debug.assert(res.dim(.y) == input.dim(-1)); - // return res; - // } - // }; + const Local = struct { + pub fn _toDiag(input: Tensor) Tensor { + const res = input.toDiagonal(-1, .{ .x, .y }); + std.debug.assert(res.dim(.x) == input.dim(-1)); + std.debug.assert(res.dim(.y) == input.dim(-1)); + return res; + } + }; - // const x = try zml.Buffer.fromArray(platform, [2][2]u8{ .{ 1, 2 }, .{ 3, 4 } }); - // { - // const res = try zml.testing.compileAndCall(platform, Local._toDiag, .{x}); - // try std.testing.expectEqual( - // [2][2][2]u8{ .{ - // .{ 1, 0 }, - // .{ 0, 2 }, - // }, .{ - // .{ 3, 0 }, - // .{ 0, 4 }, - // } }, - // try res.getValue([2][2][2]u8), - // ); - // } - //} + const x: Tensor = .init(Shape.init(.{ 2, 2 }, .u8)); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._toDiag, .{x}, platform); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[2][2]u8{ .{ 1, 2 }, .{ 3, 4 } }), std.testing.io); + defer x_buffer.deinit(); + + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._toDiag, .{x_buffer}); + defer res.deinit(); + try std.testing.expectEqual( + [2][2][2]u8{ .{ + .{ 1, 0 }, + .{ 0, 2 }, + }, .{ + .{ 3, 0 }, + .{ 0, 4 }, + } }, + try res.getValue([2][2][2]u8, std.testing.io), + ); + } /// For each matrix specified by the two axes, returns the lower triangular part of it. /// The other elements are set to 0. From d0da1c502f6823c34c1b1769a85a9deeb4a5a463 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 18:39:27 +0100 Subject: [PATCH 106/881] Re-enabled triangular tests --- zml2/tensor.zig | 116 ++++++++++++++++++++++++++++-------------------- 1 file changed, 67 insertions(+), 49 deletions(-) diff --git a/zml2/tensor.zig b/zml2/tensor.zig index e5de453a..256fd114 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -3551,60 +3551,78 @@ pub const Tensor = struct { const x = Tensor.iota(self.shape(), _axes.get(0)); const y = Tensor.iota(self.shape(), _axes.get(1)); - const zeros = Tensor.constant(self.shape(), self.dtype().zero()); + const zeros = Tensor.constant(self.dtype().zero()).broad(self.shape()); return x.addConstant(num_diagonals).cmp(.GE, y).select(self, zeros); } - // TODO(Corentin) - //test triangular { - // const zml = @import("zml.zig"); - // const platform = zml.testing.env(); + test triangular { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); - // const Local = struct { - // pub fn _tri(input: Tensor, num_diagonals: i32) Tensor { - // return input.triangular(.{ -2, -1 }, num_diagonals); - // } - // }; + const Local = struct { + pub fn _tri(input: Tensor, num_diagonals: i32) Tensor { + return input.triangular(.{ -2, -1 }, num_diagonals); + } + }; - // const x = try zml.Buffer.fromArray(platform, [3][3]u8{ - // .{ 1, 1, 1 }, - // .{ 1, 1, 1 }, - // .{ 1, 1, 1 }, - // }); - // { - // const res = try zml.testing.compileAndCall(platform, Local._tri, .{ x, 0 }); - // try std.testing.expectEqual( - // [3][3]u8{ - // .{ 1, 0, 0 }, - // .{ 1, 1, 0 }, - // .{ 1, 1, 1 }, - // }, - // try res.getValue([3][3]u8), - // ); - // } - // { - // const res = try zml.testing.compileAndCall(platform, Local._tri, .{ x, 1 }); - // try std.testing.expectEqual( - // [3][3]u8{ - // .{ 1, 1, 0 }, - // .{ 1, 1, 1 }, - // .{ 1, 1, 1 }, - // }, - // try res.getValue([3][3]u8), - // ); - // } - // { - // const res = try zml.testing.compileAndCall(platform, Local._tri, .{ x, -1 }); - // try std.testing.expectEqual( - // [3][3]u8{ - // .{ 0, 0, 0 }, - // .{ 1, 0, 0 }, - // .{ 1, 1, 0 }, - // }, - // try res.getValue([3][3]u8), - // ); - // } - //} + const x: Tensor = .init(Shape.init(.{ 3, 3 }, .u8)); + + var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[3][3]u8{ + .{ 1, 1, 1 }, + .{ 1, 1, 1 }, + .{ 1, 1, 1 }, + }), std.testing.io); + defer x_buffer.deinit(); + + { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 0 }, platform); + defer exe.deinit(); + + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); + defer res.deinit(); + + try std.testing.expectEqual( + [3][3]u8{ + .{ 1, 0, 0 }, + .{ 1, 1, 0 }, + .{ 1, 1, 1 }, + }, + try res.getValue([3][3]u8, std.testing.io), + ); + } + { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 1 }, platform); + defer exe.deinit(); + + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); + defer res.deinit(); + + try std.testing.expectEqual( + [3][3]u8{ + .{ 1, 1, 0 }, + .{ 1, 1, 1 }, + .{ 1, 1, 1 }, + }, + try res.getValue([3][3]u8, std.testing.io), + ); + } + { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, -1 }, platform); + defer exe.deinit(); + + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); + defer res.deinit(); + + try std.testing.expectEqual( + [3][3]u8{ + .{ 0, 0, 0 }, + .{ 1, 0, 0 }, + .{ 1, 1, 0 }, + }, + try res.getValue([3][3]u8, std.testing.io), + ); + } + } /// For each element at index `i`, if `bool_tensor[i] == true`, `output[i] = on_true[i]` /// otherwise, if `bool_tensor[i] == false`, `output[i] = on_false[i]` From 0e4c4d986409d54b5f3661a14e5d5f634c264569 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 18:45:37 +0100 Subject: [PATCH 107/881] Re-enabled cartesianProduct tests --- zml2/tensor.zig | 88 +++++++++++++++++++++++++++---------------------- 1 file changed, 49 insertions(+), 39 deletions(-) diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 256fd114..01157010 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -3729,48 +3729,58 @@ pub const Tensor = struct { } } - // TODO(Corentin) - //test cartesianProduct { - // const zml = @import("zml.zig"); - // const client = zml.testing.env(); + test cartesianProduct { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); - // const x = try zml.Buffer.fromSlice(client, .{6}, &[_]i32{ 0, 1, 2, 3, 4, 5 }); - // const y = try zml.Buffer.fromSlice(client, .{4}, &[_]i32{ 0, 1, 2, 3 }); + const x: Tensor = .init(Shape.init(.{6}, .i32)); + const y: Tensor = .init(Shape.init(.{4}, .i32)); - // const Local = struct { - // pub fn _cartesianProduct2(a: Tensor, b: Tensor) [2]Tensor { - // return cartesianProduct(2, .{ a, b }); - // } - // }; + const Local = struct { + pub fn _cartesianProduct2(a: Tensor, b: Tensor) [2]Tensor { + return cartesianProduct(2, .{ a, b }); + } + }; - // { - // const xs, const ys = try zml.testing.compileAndCall(client, Local._cartesianProduct2, .{ x, y }); - // try std.testing.expectEqualSlices(i64, &.{ 6, 4 }, xs.shape().dims()); - // try std.testing.expectEqualSlices(i64, &.{ 6, 4 }, ys.shape().dims()); - // try std.testing.expectEqualDeep( - // [6][4]i32{ - // .{ 0, 0, 0, 0 }, - // .{ 1, 1, 1, 1 }, - // .{ 2, 2, 2, 2 }, - // .{ 3, 3, 3, 3 }, - // .{ 4, 4, 4, 4 }, - // .{ 5, 5, 5, 5 }, - // }, - // try xs.getValue([6][4]i32), - // ); - // try std.testing.expectEqualDeep( - // [6][4]i32{ - // .{ 0, 1, 2, 3 }, - // .{ 0, 1, 2, 3 }, - // .{ 0, 1, 2, 3 }, - // .{ 0, 1, 2, 3 }, - // .{ 0, 1, 2, 3 }, - // .{ 0, 1, 2, 3 }, - // }, - // try ys.getValue([6][4]i32), - // ); - // } - //} + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cartesianProduct2, .{ x, y }, platform); + defer exe.deinit(); + + { + var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 }), std.testing.io); + defer x_buffer.deinit(); + var y_buffer: zml.Buffer = try .fromBytes(platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 }), std.testing.io); + defer y_buffer.deinit(); + + var xs, var ys = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._cartesianProduct2, .{ x_buffer, y_buffer }); + defer xs.deinit(); + defer ys.deinit(); + + try std.testing.expectEqualSlices(i64, &.{ 6, 4 }, xs.shape().dims()); + try std.testing.expectEqualSlices(i64, &.{ 6, 4 }, ys.shape().dims()); + try std.testing.expectEqualDeep( + [6][4]i32{ + .{ 0, 0, 0, 0 }, + .{ 1, 1, 1, 1 }, + .{ 2, 2, 2, 2 }, + .{ 3, 3, 3, 3 }, + .{ 4, 4, 4, 4 }, + .{ 5, 5, 5, 5 }, + }, + try xs.getValue([6][4]i32, std.testing.io), + ); + try std.testing.expectEqualDeep( + [6][4]i32{ + .{ 0, 1, 2, 3 }, + .{ 0, 1, 2, 3 }, + .{ 0, 1, 2, 3 }, + .{ 0, 1, 2, 3 }, + .{ 0, 1, 2, 3 }, + .{ 0, 1, 2, 3 }, + }, + try ys.getValue([6][4]i32, std.testing.io), + ); + } + } /// Given a set of N vectors of lengths A, B, C, D, /// returns 1 tensors of rank N+1, and shape (A, B, C, D, N). From eb1fad99e8194e0a0f0e459f19c1f2ebf89e32f3 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 18:47:37 +0100 Subject: [PATCH 108/881] Re-enabled cartesianProductStacked tests --- zml2/tensor.zig | 121 +++++++++++++++++++++++++----------------------- 1 file changed, 64 insertions(+), 57 deletions(-) diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 01157010..7f4e5ed5 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -3745,41 +3745,39 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cartesianProduct2, .{ x, y }, platform); defer exe.deinit(); - { - var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 }), std.testing.io); - defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 }), std.testing.io); - defer y_buffer.deinit(); + var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 }), std.testing.io); + defer x_buffer.deinit(); + var y_buffer: zml.Buffer = try .fromBytes(platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 }), std.testing.io); + defer y_buffer.deinit(); - var xs, var ys = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._cartesianProduct2, .{ x_buffer, y_buffer }); - defer xs.deinit(); - defer ys.deinit(); + var xs, var ys = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._cartesianProduct2, .{ x_buffer, y_buffer }); + defer xs.deinit(); + defer ys.deinit(); - try std.testing.expectEqualSlices(i64, &.{ 6, 4 }, xs.shape().dims()); - try std.testing.expectEqualSlices(i64, &.{ 6, 4 }, ys.shape().dims()); - try std.testing.expectEqualDeep( - [6][4]i32{ - .{ 0, 0, 0, 0 }, - .{ 1, 1, 1, 1 }, - .{ 2, 2, 2, 2 }, - .{ 3, 3, 3, 3 }, - .{ 4, 4, 4, 4 }, - .{ 5, 5, 5, 5 }, - }, - try xs.getValue([6][4]i32, std.testing.io), - ); - try std.testing.expectEqualDeep( - [6][4]i32{ - .{ 0, 1, 2, 3 }, - .{ 0, 1, 2, 3 }, - .{ 0, 1, 2, 3 }, - .{ 0, 1, 2, 3 }, - .{ 0, 1, 2, 3 }, - .{ 0, 1, 2, 3 }, - }, - try ys.getValue([6][4]i32, std.testing.io), - ); - } + try std.testing.expectEqualSlices(i64, &.{ 6, 4 }, xs.shape().dims()); + try std.testing.expectEqualSlices(i64, &.{ 6, 4 }, ys.shape().dims()); + try std.testing.expectEqualDeep( + [6][4]i32{ + .{ 0, 0, 0, 0 }, + .{ 1, 1, 1, 1 }, + .{ 2, 2, 2, 2 }, + .{ 3, 3, 3, 3 }, + .{ 4, 4, 4, 4 }, + .{ 5, 5, 5, 5 }, + }, + try xs.getValue([6][4]i32, std.testing.io), + ); + try std.testing.expectEqualDeep( + [6][4]i32{ + .{ 0, 1, 2, 3 }, + .{ 0, 1, 2, 3 }, + .{ 0, 1, 2, 3 }, + .{ 0, 1, 2, 3 }, + .{ 0, 1, 2, 3 }, + .{ 0, 1, 2, 3 }, + }, + try ys.getValue([6][4]i32, std.testing.io), + ); } /// Given a set of N vectors of lengths A, B, C, D, @@ -3794,32 +3792,41 @@ pub const Tensor = struct { return Tensor.stack(out.constSlice(), .last, .coord); } - // TODO(Corentin) - //test cartesianProductStacked { - // const zml = @import("zml.zig"); - // const platform = zml.testing.env(); - // const x = try zml.Buffer.fromSlice(platform, .{6}, &[_]i32{ 0, 1, 2, 3, 4, 5 }); - // const y = try zml.Buffer.fromSlice(platform, .{4}, &[_]i32{ 0, 1, 2, 3 }); + test cartesianProductStacked { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const x: Tensor = .init(Shape.init(.{6}, .i32)); + const y: Tensor = .init(Shape.init(.{4}, .i32)); - // const Local = struct { - // pub fn _fwd(a: Tensor, b: Tensor) Tensor { - // return cartesianProductStacked(&.{ a, b }); - // } - // }; + const Local = struct { + pub fn _fwd(a: Tensor, b: Tensor) Tensor { + return cartesianProductStacked(&.{ a, b }); + } + }; - // const z = try zml.testing.compileAndCall(platform, Local._fwd, .{ x, y }); - // try std.testing.expectEqualDeep( - // [6][4][2]i32{ - // .{ .{ 0, 0 }, .{ 0, 1 }, .{ 0, 2 }, .{ 0, 3 } }, - // .{ .{ 1, 0 }, .{ 1, 1 }, .{ 1, 2 }, .{ 1, 3 } }, - // .{ .{ 2, 0 }, .{ 2, 1 }, .{ 2, 2 }, .{ 2, 3 } }, - // .{ .{ 3, 0 }, .{ 3, 1 }, .{ 3, 2 }, .{ 3, 3 } }, - // .{ .{ 4, 0 }, .{ 4, 1 }, .{ 4, 2 }, .{ 4, 3 } }, - // .{ .{ 5, 0 }, .{ 5, 1 }, .{ 5, 2 }, .{ 5, 3 } }, - // }, - // try z.getValue([6][4][2]i32), - // ); - //} + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, y }, platform); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 }), std.testing.io); + defer x_buffer.deinit(); + var y_buffer: zml.Buffer = try .fromBytes(platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 }), std.testing.io); + defer y_buffer.deinit(); + + var z = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._fwd, .{ x_buffer, y_buffer }); + defer z.deinit(); + + try std.testing.expectEqualDeep( + [6][4][2]i32{ + .{ .{ 0, 0 }, .{ 0, 1 }, .{ 0, 2 }, .{ 0, 3 } }, + .{ .{ 1, 0 }, .{ 1, 1 }, .{ 1, 2 }, .{ 1, 3 } }, + .{ .{ 2, 0 }, .{ 2, 1 }, .{ 2, 2 }, .{ 2, 3 } }, + .{ .{ 3, 0 }, .{ 3, 1 }, .{ 3, 2 }, .{ 3, 3 } }, + .{ .{ 4, 0 }, .{ 4, 1 }, .{ 4, 2 }, .{ 4, 3 } }, + .{ .{ 5, 0 }, .{ 5, 1 }, .{ 5, 2 }, .{ 5, 3 } }, + }, + try z.getValue([6][4][2]i32, std.testing.io), + ); + } fn binaryOp( op_name: []const u8, From b4fddca0d6f1ebc4fbee1f34e3b66d4ae545a76e Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 18:49:50 +0100 Subject: [PATCH 109/881] Re-enabled unused tensor test --- zml2/tensor.zig | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 7f4e5ed5..0c9fa6b1 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -4052,19 +4052,18 @@ test transposeIsJustAReshape { try std.testing.expect(transposeIsJustAReshape(Shape.init(.{ 1, 10, 155, 1 }, .f32), &.{ 0, 1, 3, 2 })); } -// TODO(Corentin) -//test "unused tensor" { -// const zml = @import("zml.zig"); -// const platform = zml.testing.env(); -// -// const Local = struct { -// pub fn _fwd(x: Tensor) Tensor { -// const y = x.addConstant(1); -// _ = y; -// return x; -// } -// }; -// -// const mod = try zml.compileFn(std.testing.allocator, Local._fwd, .{Shape.init(.{10}, .f32)}, platform); -// defer mod.deinit(); -// +test "unused tensor" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const Local = struct { + pub fn _fwd(x: Tensor) Tensor { + const y = x.addConstant(1); + _ = y; + return x; + } + }; + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{Tensor.init(Shape.init(.{10}, .f32))}, platform); + defer exe.deinit(); +} From 1cfc13cd82f3d553d69437f6368d05730f3b2dd3 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 19:03:38 +0100 Subject: [PATCH 110/881] Re-enabled maxPool tests --- zml2/ops.zig | 2 +- zml2/tensor.zig | 151 +++++++++++++++++++++++++++--------------------- 2 files changed, 86 insertions(+), 67 deletions(-) diff --git a/zml2/ops.zig b/zml2/ops.zig index f94cc79a..e6a653a4 100644 --- a/zml2/ops.zig +++ b/zml2/ops.zig @@ -186,7 +186,7 @@ pub fn reduceWindow(inputs: anytype, inits: anytype, opts: ReduceWindowOpts, com }).appendTo(CompilationContext.current().currentScope().block); inline for (0..result.len) |i| { - result[i] = Tensor._result(inputs[i].shape(), reduce_op.result(i)); + result[i] = Tensor.fromMlirValue(reduce_op.result(i)).withTags(inputs[i].shape()); } return result; diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 0c9fa6b1..10f0a83e 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -3926,72 +3926,91 @@ fn getComparisonType(ctx: *mlir.Context, dtype: DataType) *const dialects.stable }); } -// TODO(Corentin) -//test "Tensor.maxPool1d" { -// const zml = @import("zml.zig"); -// const platform = zml.testing.env(); -// -// const MaxPool = struct { -// pub fn _fwd(x: zml.Tensor) Tensor.ArgMaxRes { -// return x.maxPool1d(.{ -// .window_dimensions = 3, -// .window_strides = 2, -// }); -// } -// }; -// -// var data: [20]f32 = undefined; -// for (&data, 0..) |*v, i| v.* = @floatFromInt(i); -// -// const x = try zml.Buffer.fromSlice(platform, .{ 2, 2, 5 }, &data); -// const result = try zml.testing.compileAndCall(platform, MaxPool._fwd, .{x}); -// try zml.testing.expectEqualShapes(.init(.{ 2, 2, 2 }, .f32), result.values.shape()); -// try zml.testing.expectEqualShapes(.init(.{ 2, 2, 2 }, .i32), result.indices.shape()); -// try std.testing.expectEqualDeep( -// [2][2][2]f32{ -// .{ .{ 2, 4 }, .{ 7, 9 } }, -// .{ .{ 12, 14 }, .{ 17, 19 } }, -// }, -// result.values.getValue([2][2][2]f32), -// ); -//} -// -//test "Tensor.maxPool2d" { -// const zml = @import("zml.zig"); -// const platform = zml.testing.env(); -// -// const MaxPool = struct { -// pub fn _fwd(x: Tensor) Tensor.ArgMaxRes { -// return x.maxPool2d(.{ -// .window_dimensions = .{ 3, 2 }, -// .window_strides = .{ 2, 1 }, -// }); -// } -// }; -// -// var data: [100]f32 = undefined; -// for (&data, 0..) |*v, i| v.* = @floatFromInt(i); -// const x = try zml.Buffer.fromSlice(platform, .{ 2, 2, 5, 5 }, &data); -// -// const result = try zml.testing.compileAndCall(platform, MaxPool._fwd, .{x}); -// try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2, 4 }, .f32), result.values.shape()); -// try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2, 4 }, .i32), result.indices.shape()); -// var buffer: [2][2][2][4]f32 = undefined; -// _ = try result.values.toHost(std.mem.asBytes(&buffer)); -// try std.testing.expectEqualDeep( -// [2][2][2][4]f32{ -// .{ -// .{ .{ 11, 12, 13, 14 }, .{ 21, 22, 23, 24 } }, -// .{ .{ 36, 37, 38, 39 }, .{ 46, 47, 48, 49 } }, -// }, -// .{ -// .{ .{ 61, 62, 63, 64 }, .{ 71, 72, 73, 74 } }, -// .{ .{ 86, 87, 88, 89 }, .{ 96, 97, 98, 99 } }, -// }, -// }, -// buffer, -// ); -//} +test "Tensor.maxPool1d" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const MaxPool = struct { + pub fn _fwd(x: zml.Tensor) Tensor.ArgMaxRes { + return x.maxPool1d(.{ + .window_dimensions = 3, + .window_strides = 2, + }); + } + }; + + var data: [20]f32 = undefined; + for (&data, 0..) |*v, i| v.* = @floatFromInt(i); + + const x: Tensor = .init(Shape.init(.{ 2, 2, 5 }, .f32)); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&data), std.testing.io); + defer x_buffer.deinit(); + + const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, MaxPool._fwd, .{x_buffer}); + defer result.values.deinit(); + defer result.indices.deinit(); + + try zml.testing.expectEqualShapes(.init(.{ 2, 2, 2 }, .f32), result.values.shape()); + try zml.testing.expectEqualShapes(.init(.{ 2, 2, 2 }, .i32), result.indices.shape()); + try std.testing.expectEqualDeep( + [2][2][2]f32{ + .{ .{ 2, 4 }, .{ 7, 9 } }, + .{ .{ 12, 14 }, .{ 17, 19 } }, + }, + result.values.getValue([2][2][2]f32, std.testing.io), + ); +} + +test "Tensor.maxPool2d" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const MaxPool = struct { + pub fn _fwd(x: Tensor) Tensor.ArgMaxRes { + return x.maxPool2d(.{ + .window_dimensions = .{ 3, 2 }, + .window_strides = .{ 2, 1 }, + }); + } + }; + + var data: [100]f32 = undefined; + for (&data, 0..) |*v, i| v.* = @floatFromInt(i); + + const x: Tensor = .init(Shape.init(.{ 2, 2, 5, 5 }, .f32)); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&data), std.testing.io); + defer x_buffer.deinit(); + + const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, MaxPool._fwd, .{x_buffer}); + defer result.values.deinit(); + defer result.indices.deinit(); + + try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2, 4 }, .f32), result.values.shape()); + try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2, 4 }, .i32), result.indices.shape()); + var buffer: [2][2][2][4]f32 = undefined; + _ = try result.values.toHost(std.mem.asBytes(&buffer), std.testing.io); + try std.testing.expectEqualDeep( + [2][2][2][4]f32{ + .{ + .{ .{ 11, 12, 13, 14 }, .{ 21, 22, 23, 24 } }, + .{ .{ 36, 37, 38, 39 }, .{ 46, 47, 48, 49 } }, + }, + .{ + .{ .{ 61, 62, 63, 64 }, .{ 71, 72, 73, 74 } }, + .{ .{ 86, 87, 88, 89 }, .{ 96, 97, 98, 99 } }, + }, + }, + buffer, + ); +} fn _parseGatherCoord(self: Tensor, axes_: anytype) struct { bool, stdx.BoundedArray(u3, constants.MAX_RANK) } { const AxesT = @TypeOf(axes_); From e0cba48ed7ca2984d53d09feb677ff780842352a Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 19:34:15 +0100 Subject: [PATCH 111/881] Re-enabled scatterSlices tests --- zml2/meta.zig | 2 +- zml2/tensor.zig | 272 +++++++++++++++++++++++++----------------------- 2 files changed, 144 insertions(+), 130 deletions(-) diff --git a/zml2/meta.zig b/zml2/meta.zig index 6933479f..be79a8a2 100644 --- a/zml2/meta.zig +++ b/zml2/meta.zig @@ -470,7 +470,7 @@ pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) VisitReturn( *const K, *K => return cb(ctx, v), *const ?K, *?K => return if (v.*) |*val| cb(ctx, val) else {}, []const K, []K => { - for (v) |*v_elem| cb(ctx, v_elem); + for (v) |*v_elem| if (can_error) try cb(ctx, v_elem) else cb(ctx, v_elem); return; }, else => {}, diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 10f0a83e..7521ab4f 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -2533,148 +2533,162 @@ pub const Tensor = struct { return ops.scatter(.{self}, indices, .{updates}, Custom.inc, .{opts.update_fn}, opts)[0]; } - // TODO(Corentin) - //test scatterSlices { - // const zml = @import("zml.zig"); - // const platform = zml.testing.env(); + test scatterSlices { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); - // const Local = struct { - // pub fn _scatter(self: Tensor, indices: []const Tensor, updates: Tensor) Tensor { - // return self.scatterSlices( - // indices, - // updates, - // .{ .update_fn = ScatterOpts.increment }, - // ); - // } + const Local = struct { + pub fn _scatter(self: Tensor, indices: []const Tensor, updates: Tensor) Tensor { + return self.scatterSlices( + indices, + updates, + .{ .update_fn = ScatterOpts.increment }, + ); + } - // pub fn _scatterCB(self: Tensor, coords: Tensor, updates: Tensor) Tensor { - // return self.scatterSlices( - // .{ .c = coords.choose1d(.coord, 0), .b = coords.choose1d(.coord, 1) }, - // updates, - // .{ .update_fn = ScatterOpts.increment }, - // ); - // } + pub fn _scatterCB(self: Tensor, coords: Tensor, updates: Tensor) Tensor { + return self.scatterSlices( + .{ .c = coords.choose1d(.coord, 0), .b = coords.choose1d(.coord, 1) }, + updates, + .{ .update_fn = ScatterOpts.increment }, + ); + } - // pub fn _idx(idx_shape: anytype) Tensor { - // return Tensor.constant(idx_shape, .{ .i32 = 0 }); - // } - // }; + pub fn _idx(idx_shape: anytype) Tensor { + return Tensor.constant(.{ .i32 = 0 }).broad(Shape.init(idx_shape, .i32)); + } + }; - // { - // // Only test shapes - // var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); - // defer comp.deinit(); - // comp.activate(); - // defer comp.deactivate(); - // const idx = Local._idx; + { + // Only test shapes + var comp = zml.module.CompilationContext.init(std.testing.allocator); + defer comp.deinit(); + comp.activate(); + defer comp.deactivate(); - // inline for (.{ - // // This is equivalent to a dynamic update slice, update 3 values at given offset of axis .a: - // .{ .{ .a = 10 }, .{ .a = idx(.{}) }, .{ .a = 3 } }, - // // Use .a as a batching axis with .a=10 x .n=8 updates of 2 elements of .b - // .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .n = 8, .b = 2 } }, - // // Same but with update transposed - // .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .b = 2, .n = 8 } }, - // // similar, but use the normalized form where a is no longer an explicit batching axis. - // .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .a2 = 10, .n = 8 }), .b = idx(.{ .a2 = 10, .n = 8 }) }, .{ .a2 = 10, .n = 8, .b = 2 } }, - // .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .a = 10, .n = 8 }), .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .n = 8, .b = 2 } }, - // .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8, .a = 2 } }, - // .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .n = 8 }), .a = idx(.{ .n = 8 }) }, .{ .n = 8, .a = 3, .b = 2 } }, - // .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }), .b = idx(.{ .n = 8 }) }, .{ .a = 3, .n = 8, .b = 2 } }, - // }) |testcase| { - // const x_shape, const indices, const updates_shapes = testcase; - // const x = Tensor.constant(x_shape, .{ .f16 = 0 }); - // const updates = Tensor.constant(updates_shapes, .{ .f16 = 0 }); + const block = mlir.Block.init(&.{}, &.{}); + defer block.deinit(); + comp.pushBlock(block); + defer comp.popBlock(); - // const y = scatterSlices(x, indices, updates, .{}); - // // Shape doesn't change with scatterSlices - // try zml.testing.expectEqualShapes(x.shape(), y.shape()); - // try std.testing.expect(y.value().owner().verify()); - // } - // } - // // Test with actual values, no batching. - // { - // const a_host = try zml.HostBuffer.arange(std.testing.allocator, .{ .end = 9 }, .i32); - // const a = (try zml.Buffer.from(platform, a_host.reshape(.{ 3, 3 }), .{})).withTags(.{ .a, .b }); - // defer a.deinit(); - // a_host.deinit(std.testing.allocator); + const idx = Local._idx; - // const scatter_indices = try zml.Buffer.fromArray(platform, [2]i32{ 0, 2 }); - // const updates = try zml.Buffer.fromArray(platform, [2][3]i32{ .{ 10, 20, 30 }, .{ 70, 80, 90 } }); + inline for (.{ + // This is equivalent to a dynamic update slice, update 3 values at given offset of axis .a: + .{ .{ .a = 10 }, .{ .a = idx(.{}) }, .{ .a = 3 } }, + // Use .a as a batching axis with .a=10 x .n=8 updates of 2 elements of .b + .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .n = 8, .b = 2 } }, + // Same but with update transposed + .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .b = 2, .n = 8 } }, + // similar, but use the normalized form where a is no longer an explicit batching axis. + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .a2 = 10, .n = 8 }), .b = idx(.{ .a2 = 10, .n = 8 }) }, .{ .a2 = 10, .n = 8, .b = 2 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .a = 10, .n = 8 }), .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .n = 8, .b = 2 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8, .a = 2 } }, + .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .n = 8 }), .a = idx(.{ .n = 8 }) }, .{ .n = 8, .a = 3, .b = 2 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }), .b = idx(.{ .n = 8 }) }, .{ .a = 3, .n = 8, .b = 2 } }, + }) |testcase| { + const x_shape, const indices, const updates_shapes = testcase; + const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); + const updates = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(updates_shapes, .f16)); - // const expected = [3][3]i32{ .{ 10, 21, 32 }, .{ 3, 4, 5 }, .{ 76, 87, 98 } }; - // const result = try zml.testing.compileAndCall(platform, Local._scatter, .{ - // a, - // &.{scatter_indices.withTags(.{.n})}, - // updates.withTags(.{ .n, .b }), - // }); - // try std.testing.expect(a.shape().eql(result.shape())); - // try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected))); - // } - // // Test with setting individual values (no batching) - // { - // const a_host = try zml.HostBuffer.arange(std.testing.allocator, .{ .end = 9 }, .i32); - // const a = try zml.Buffer.from(platform, a_host, .{}); - // defer a.deinit(); - // a_host.deinit(std.testing.allocator); + const y = scatterSlices(x, indices, updates, .{}); + // Shape doesn't change with scatterSlices + try zml.testing.expectEqualShapes(x.shape(), y.shape()); + try std.testing.expect(y.value().owner().verify()); + } + } + // Test with actual values, no batching. + { + const a: Tensor = .init(Shape.init(.{ 3, 3 }, .i32)); - // const scatter_indices = try zml.Buffer.fromArray(platform, [2]i32{ 2, 7 }); - // const updates = try zml.Buffer.fromArray(platform, [2]i32{ 20, 70 }); + const scatter_indices = Tensor.init(Shape.init(.{2}, .i32)).withTags(.{.n}); + const updates = Tensor.init(Shape.init(.{ 2, 3 }, .i32)).withTags(.{ .n, .b }); - // const expected = [9]i32{ 0, 1, 22, 3, 4, 5, 6, 77, 8 }; - // const result = try zml.testing.compileAndCall(platform, Local._scatter, .{ - // a, - // &.{scatter_indices.withTags(.{.n})}, - // updates.withTags(.{.n}), - // }); - // try std.testing.expect(a.shape().eql(result.shape())); - // try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected))); - // } - // { - // // Test with actual values and batching along axis .a - // const operand = try zml.Buffer.constant(platform, Shape.init(.{ .a = 2, .b = 3, .c = 4, .d = 2 }, .u16), 0, .{}); - // defer operand.deinit(); - // const start_indices = (try zml.Buffer.fromArray( - // platform, - // [2][2][3][2]i32{ - // .{ - // .{ .{ 0, 0 }, .{ 1, 0 }, .{ 2, 1 } }, - // .{ .{ 0, 1 }, .{ 1, 1 }, .{ 0, 9 } }, - // }, - // .{ - // .{ .{ 0, 0 }, .{ 2, 1 }, .{ 2, 2 } }, - // .{ .{ 1, 2 }, .{ 0, 1 }, .{ 1, 0 } }, - // }, - // }, - // )).withTags(.{ .n, .a, .m, .coord }); - // defer start_indices.deinit(); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform); + defer exe.deinit(); - // const values = try zml.Buffer.constant( - // platform, - // Shape.init(.{ .n = 2, .a = 2, .m = 3, .c = 2, .d = 2 }, .u16), - // 1, - // .{}, - // ); - // defer values.deinit(); + var a_buffer: zml.Buffer = try .fromBytes(platform, a.shape(), std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 }), std.testing.io); + defer a_buffer.deinit(); + var scatter_indices_buffer: zml.Buffer = try .fromBytes(platform, scatter_indices.shape(), std.mem.sliceAsBytes(&[2]i32{ 0, 2 }), std.testing.io); + defer scatter_indices_buffer.deinit(); + var updates_buffer: zml.Buffer = try .fromBytes(platform, updates.shape(), std.mem.sliceAsBytes(&[2][3]i32{ .{ 10, 20, 30 }, .{ 70, 80, 90 } }), std.testing.io); + defer updates_buffer.deinit(); - // const result = try zml.testing.compileAndCall(platform, Local._scatterCB, .{ operand, start_indices, values }); + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatter, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); + defer result.deinit(); - // const expected = [2][3][4][2]u16{ - // .{ - // .{ .{ 2, 2 }, .{ 3, 3 }, .{ 1, 1 }, .{ 0, 0 } }, - // .{ .{ 0, 0 }, .{ 0, 0 }, .{ 2, 2 }, .{ 2, 2 } }, - // .{ .{ 0, 0 }, .{ 0, 0 }, .{ 1, 1 }, .{ 1, 1 } }, - // }, - // .{ - // .{ .{ 0, 0 }, .{ 1, 1 }, .{ 1, 1 }, .{ 0, 0 } }, - // .{ .{ 2, 2 }, .{ 3, 3 }, .{ 1, 1 }, .{ 0, 0 } }, - // .{ .{ 0, 0 }, .{ 1, 1 }, .{ 1, 1 }, .{ 0, 0 } }, - // }, - // }; - // try std.testing.expect(operand.shape().eql(result.shape())); - // try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected))); - // } - //} + const expected = [3][3]i32{ .{ 10, 21, 32 }, .{ 3, 4, 5 }, .{ 76, 87, 98 } }; + try std.testing.expect(a.shape().eql(result.shape())); + try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected), std.testing.io)); + } + // Test with setting individual values (no batching) + { + const a: Tensor = .init(Shape.init(.{9}, .i32)); + + const scatter_indices = Tensor.init(Shape.init(.{2}, .i32)).withTags(.{.n}); + const updates = Tensor.init(Shape.init(.{2}, .i32)).withTags(.{.n}); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform); + defer exe.deinit(); + + var a_buffer: zml.Buffer = try .fromBytes(platform, a.shape(), std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 }), std.testing.io); + defer a_buffer.deinit(); + var scatter_indices_buffer: zml.Buffer = try .fromBytes(platform, scatter_indices.shape(), std.mem.sliceAsBytes(&[2]i32{ 2, 7 }), std.testing.io); + defer scatter_indices_buffer.deinit(); + var updates_buffer: zml.Buffer = try .fromBytes(platform, updates.shape(), std.mem.sliceAsBytes(&[2]i32{ 20, 70 }), std.testing.io); + defer updates_buffer.deinit(); + + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatter, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); + defer result.deinit(); + + const expected = [9]i32{ 0, 1, 22, 3, 4, 5, 6, 77, 8 }; + try std.testing.expect(a.shape().eql(result.shape())); + try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected), std.testing.io)); + } + { + // Test with actual values and batching along axis .a + const operand: Tensor = .init(Shape.init(.{ .a = 2, .b = 3, .c = 4, .d = 2 }, .u16)); + const start_indices: Tensor = .init(Shape.init(.{ .n = 2, .a = 2, .m = 3, .coord = 2 }, .i32)); + const values: Tensor = .init((Shape.init(.{ .n = 2, .a = 2, .m = 3, .c = 2, .d = 2 }, .u16))); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatterCB, .{ operand, start_indices, values }, platform); + defer exe.deinit(); + + var operand_buffer: zml.Buffer = try .fromBytes(platform, operand.shape(), std.mem.sliceAsBytes(&@as([2 * 3 * 4 * 2]u16, @splat(0))), std.testing.io); + defer operand_buffer.deinit(); + var start_indices_buffer: zml.Buffer = try .fromBytes(platform, start_indices.shape(), std.mem.sliceAsBytes(&[2][2][3][2]i32{ + .{ + .{ .{ 0, 0 }, .{ 1, 0 }, .{ 2, 1 } }, + .{ .{ 0, 1 }, .{ 1, 1 }, .{ 0, 9 } }, + }, + .{ + .{ .{ 0, 0 }, .{ 2, 1 }, .{ 2, 2 } }, + .{ .{ 1, 2 }, .{ 0, 1 }, .{ 1, 0 } }, + }, + }), std.testing.io); + defer start_indices_buffer.deinit(); + var values_buffer: zml.Buffer = try .fromBytes(platform, values.shape(), std.mem.sliceAsBytes(&@as([2 * 2 * 3 * 2 * 2]u16, @splat(1))), std.testing.io); + defer values_buffer.deinit(); + + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatterCB, .{ operand_buffer, start_indices_buffer, values_buffer }); + defer result.deinit(); + + const expected = [2][3][4][2]u16{ + .{ + .{ .{ 2, 2 }, .{ 3, 3 }, .{ 1, 1 }, .{ 0, 0 } }, + .{ .{ 0, 0 }, .{ 0, 0 }, .{ 2, 2 }, .{ 2, 2 } }, + .{ .{ 0, 0 }, .{ 0, 0 }, .{ 1, 1 }, .{ 1, 1 } }, + }, + .{ + .{ .{ 0, 0 }, .{ 1, 1 }, .{ 1, 1 }, .{ 0, 0 } }, + .{ .{ 2, 2 }, .{ 3, 3 }, .{ 1, 1 }, .{ 0, 0 } }, + .{ .{ 0, 0 }, .{ 1, 1 }, .{ 1, 1 }, .{ 0, 0 } }, + }, + }; + try std.testing.expect(operand.shape().eql(result.shape())); + try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected), std.testing.io)); + } + } /// Returns a Tensor containing the maximum over a given axis. pub fn max(self: Tensor, axis_: anytype) Tensor { From 34b9283a6eec2b96ae36be4fa1ee6d39e9bcb755 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 9 Dec 2025 19:49:31 +0100 Subject: [PATCH 112/881] Re-enabled dynamicUpdateSlice tests --- zml2/tensor.zig | 240 +++++++++++++++++++++++++++++------------------- 1 file changed, 145 insertions(+), 95 deletions(-) diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 7521ab4f..9e707e81 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -3370,107 +3370,157 @@ pub const Tensor = struct { return _result(self._shape, op.result(0)); } - // TODO(Corentin) - //test dynamicUpdateSlice { - // const zml = @import("zml.zig"); - // const platform = zml.testing.env(); + test dynamicUpdateSlice { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); - // { - // const x = try zml.Buffer.fromArray(platform, [10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); - // const y = try zml.Buffer.fromArray(platform, [2]f32{ -1, -1 }); - // const idx = try zml.Buffer.scalar(platform, 4, .i32); - // const res = try zml.testing.compileAndCall( - // platform, - // struct { - // pub fn _fwd(x_: Tensor, idx_: struct { a: Tensor }, y_: Tensor) Tensor { - // return x_.dynamicUpdateSlice(idx_, y_); - // } - // }._fwd, - // .{ x.withTags(.{.a}), .{ .a = idx }, y.withTags(.{.a}) }, - // ); - // try std.testing.expectEqual([10]f32{ 0, 1, 2, 3, -1, -1, 6, 7, 8, 9 }, try res.getValue([10]f32)); - // } + { + const x = Tensor.init(Shape.init(.{10}, .f32)).withTags(.{.a}); + const y = Tensor.init(Shape.init(.{2}, .f32)).withTags(.{.a}); + const ids: Tensor = .init(Shape.init(.{}, .i32)); + const forward = struct { + pub fn _fwd(x_: Tensor, idx_: struct { a: Tensor }, y_: Tensor) Tensor { + return x_.dynamicUpdateSlice(idx_, y_); + } + }._fwd; + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = ids }, y }, platform); + defer exe.deinit(); - // { - // // Updates 2D, tagged api - // const x = try zml.Buffer.fromArray(platform, [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }); - // const y = try zml.Buffer.fromArray(platform, [2]f32{ -1, -1 }); - // const idx = try zml.Buffer.scalar(platform, 3, .i32); + var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }), std.testing.io); + defer x_buffer.deinit(); + var y_buffer: zml.Buffer = try .fromBytes(platform, y.shape(), std.mem.sliceAsBytes(&[2]f32{ -1, -1 }), std.testing.io); + defer y_buffer.deinit(); + var ids_buffer: zml.Buffer = try .fromBytes(platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{4}), std.testing.io); + defer ids_buffer.deinit(); - // const res = try zml.testing.compileAndCall( - // platform, - // struct { - // pub fn _fwd(x_: Tensor, idx_: Tensor, y_: Tensor) Tensor { - // return x_.dynamicUpdateSlice(.{ .b = idx_ }, y_); - // } - // }._fwd, - // .{ x.withTags(.{ .a, .b }), idx, y.withTags(.{.a}) }, - // ); - // try std.testing.expectEqualDeep( - // [2][5]f32{ .{ 0, 1, 2, -1, 4 }, .{ 5, 6, 7, -1, 9 } }, - // try res.getValue([2][5]f32), - // ); - // } + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ .a = ids_buffer }, y_buffer }); + defer res.deinit(); - // { - // // Updates 2D slice, un-tagged api. Note that `y` needs to have a 1 dimension axis. - // const x = try zml.Buffer.fromArray(platform, [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }); - // const y = try zml.Buffer.fromArray(platform, [2][1]f32{ .{-1}, .{-1} }); - // const idx = try zml.Buffer.scalar(platform, 3, .i32); - // const res = try zml.testing.compileAndCall( - // platform, - // struct { - // pub fn _fwd(x_: Tensor, idx_: Tensor, y_: Tensor) Tensor { - // return x_.dynamicUpdateSlice(.{ zml.Tensor.scalar(0, .i32), idx_ }, y_); - // } - // }._fwd, - // .{ x, idx, y }, - // ); - // try std.testing.expectEqualDeep( - // [2][5]f32{ .{ 0, 1, 2, -1, 4 }, .{ 5, 6, 7, -1, 9 } }, - // res.getValue([2][5]f32), - // ); - // } + try std.testing.expectEqual([10]f32{ 0, 1, 2, 3, -1, -1, 6, 7, 8, 9 }, try res.getValue([10]f32, std.testing.io)); + } - // { - // // Updates 2D, partial update - // const x = try zml.Buffer.fromArray(platform, [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }); - // const y = try zml.Buffer.fromArray(platform, [1]f32{-1}); - // const idx_a = try zml.Buffer.scalar(platform, 1, .i32); - // const idx_b = try zml.Buffer.scalar(platform, 3, .i32); - // const res = try zml.testing.compileAndCall( - // platform, - // struct { - // pub fn _fwd(x_: Tensor, idx_: struct { a: Tensor, b: Tensor }, y_: Tensor) Tensor { - // return x_.dynamicUpdateSlice(idx_, y_); - // } - // }._fwd, - // .{ x.withTags(.{ .a, .b }), .{ .a = idx_a, .b = idx_b }, y.withTags(.{.a}) }, - // ); - // try std.testing.expectEqualDeep( - // [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, -1, 9 } }, - // res.getValue([2][5]f32), - // ); - // } + { + // Updates 2D, tagged api + const x = Tensor.init(Shape.init(.{ 2, 5 }, .f32)).withTags(.{ .a, .b }); + const y = Tensor.init(Shape.init(.{2}, .f32)).withTags(.{.a}); + const ids: Tensor = .init(Shape.init(.{}, .i32)); + const forward = struct { + pub fn _fwd(x_: Tensor, idx_: Tensor, y_: Tensor) Tensor { + return x_.dynamicUpdateSlice(.{ .b = idx_ }, y_); + } + }._fwd; + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform); + defer exe.deinit(); - // { - // // Updates 2D, partial update, un-tagged api. - // const x = try zml.Buffer.fromArray(platform, [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }); - // const y = try zml.Buffer.fromArray(platform, [1][1]f32{.{-1}}); - // const idx_a = try zml.Buffer.scalar(platform, 1, .i32); - // const idx_b = try zml.Buffer.scalar(platform, 3, .i32); - // const A = struct { - // pub fn _fwd(x_: Tensor, idx_: [2]Tensor, y_: Tensor) Tensor { - // return x_.dynamicUpdateSlice(&idx_, y_); - // } - // }; - // const res = try zml.testing.compileAndCall(platform, A._fwd, .{ x, .{ idx_a, idx_b }, y }); - // try std.testing.expectEqualDeep( - // [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, -1, 9 } }, - // res.getValue([2][5]f32), - // ); - // } - //} + var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }), std.testing.io); + defer x_buffer.deinit(); + var y_buffer: zml.Buffer = try .fromBytes(platform, y.shape(), std.mem.sliceAsBytes(&[2]f32{ -1, -1 }), std.testing.io); + defer y_buffer.deinit(); + var ids_buffer: zml.Buffer = try .fromBytes(platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3}), std.testing.io); + defer ids_buffer.deinit(); + + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, ids_buffer, y_buffer }); + defer res.deinit(); + + try std.testing.expectEqualDeep( + [2][5]f32{ .{ 0, 1, 2, -1, 4 }, .{ 5, 6, 7, -1, 9 } }, + try res.getValue([2][5]f32, std.testing.io), + ); + } + + { + // Updates 2D slice, un-tagged api. Note that `y` needs to have a 1 dimension axis. + const x: Tensor = .init(Shape.init(.{ 2, 5 }, .f32)); + const y: Tensor = .init(Shape.init(.{ 2, 1 }, .f32)); + const ids: Tensor = .init(Shape.init(.{}, .i32)); + const forward = struct { + pub fn _fwd(x_: Tensor, idx_: Tensor, y_: Tensor) Tensor { + return x_.dynamicUpdateSlice(.{ zml.Tensor.scalar(0, .i32), idx_ }, y_); + } + }._fwd; + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }), std.testing.io); + defer x_buffer.deinit(); + var y_buffer: zml.Buffer = try .fromBytes(platform, y.shape(), std.mem.sliceAsBytes(&[2][1]f32{ .{-1}, .{-1} }), std.testing.io); + defer y_buffer.deinit(); + var ids_buffer: zml.Buffer = try .fromBytes(platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3}), std.testing.io); + defer ids_buffer.deinit(); + + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, ids_buffer, y_buffer }); + defer res.deinit(); + + try std.testing.expectEqualDeep( + [2][5]f32{ .{ 0, 1, 2, -1, 4 }, .{ 5, 6, 7, -1, 9 } }, + res.getValue([2][5]f32, std.testing.io), + ); + } + + { + // Updates 2D, partial update + const x = Tensor.init(Shape.init(.{ 2, 5 }, .f32)).withTags(.{ .a, .b }); + const y = Tensor.init(Shape.init(.{1}, .f32)).withTags(.{.a}); + const idx_a: Tensor = .init(Shape.init(.{}, .i32)); + const idx_b: Tensor = .init(Shape.init(.{}, .i32)); + const forward = struct { + pub fn _fwd(x_: Tensor, idx_: struct { a: Tensor, b: Tensor }, y_: Tensor) Tensor { + return x_.dynamicUpdateSlice(idx_, y_); + } + }._fwd; + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = idx_a, .b = idx_b }, y }, platform); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }), std.testing.io); + defer x_buffer.deinit(); + var y_buffer: zml.Buffer = try .fromBytes(platform, y.shape(), std.mem.sliceAsBytes(&[1]f32{-1}), std.testing.io); + defer y_buffer.deinit(); + var idx_a_buffer: zml.Buffer = try .fromBytes(platform, idx_a.shape(), std.mem.sliceAsBytes(&[1]i32{1}), std.testing.io); + defer idx_a_buffer.deinit(); + var idx_b_buffer: zml.Buffer = try .fromBytes(platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3}), std.testing.io); + defer idx_b_buffer.deinit(); + + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ .a = idx_a_buffer, .b = idx_b_buffer }, y_buffer }); + defer res.deinit(); + + try std.testing.expectEqualDeep( + [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, -1, 9 } }, + res.getValue([2][5]f32, std.testing.io), + ); + } + + { + // Updates 2D, partial update, un-tagged api. + const x: Tensor = .init(Shape.init(.{ 2, 5 }, .f32)); + const y: Tensor = .init(Shape.init(.{ 1, 1 }, .f32)); + const idx_a: Tensor = .init(Shape.init(.{}, .i32)); + const idx_b: Tensor = .init(Shape.init(.{}, .i32)); + const forward = struct { + pub fn _fwd(x_: Tensor, idx_: [2]Tensor, y_: Tensor) Tensor { + return x_.dynamicUpdateSlice(&idx_, y_); + } + }._fwd; + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ idx_a, idx_b }, y }, platform); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }), std.testing.io); + defer x_buffer.deinit(); + var y_buffer: zml.Buffer = try .fromBytes(platform, y.shape(), std.mem.sliceAsBytes(&[1][1]f32{.{-1}}), std.testing.io); + defer y_buffer.deinit(); + var idx_a_buffer: zml.Buffer = try .fromBytes(platform, idx_a.shape(), std.mem.sliceAsBytes(&[1]i32{1}), std.testing.io); + defer idx_a_buffer.deinit(); + var idx_b_buffer: zml.Buffer = try .fromBytes(platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3}), std.testing.io); + defer idx_b_buffer.deinit(); + + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ idx_a_buffer, idx_b_buffer }, y_buffer }); + defer res.deinit(); + + try std.testing.expectEqualDeep( + [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, -1, 9 } }, + res.getValue([2][5]f32, std.testing.io), + ); + } + } /// Returns a Tensor containing the element-wise result of the given 'cmp' comparison between the two input Tensors. pub fn cmp(self: Tensor, direction: dialects.stablehlo.ComparisonDirection.Direction, other: Tensor) Tensor { From b0555085b38ff41b828c14b79aff7c34eadb4c69 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 10 Dec 2025 12:26:40 +0100 Subject: [PATCH 113/881] Get new shape.zig from hugomano/explo-partitioning --- zml2/buffer.zig | 16 +- zml2/shape.zig | 750 ++++++++++++++++++++++++++++++++++++++++++++---- zml2/tensor.zig | 1 + 3 files changed, 708 insertions(+), 59 deletions(-) diff --git a/zml2/buffer.zig b/zml2/buffer.zig index 8ee19fb1..5dd13902 100644 --- a/zml2/buffer.zig +++ b/zml2/buffer.zig @@ -63,17 +63,17 @@ pub const Buffer = struct { // We shard only on the first axis so that the chunks are still contiguous. // TODO: support more advanced sharding specs stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); - const sharding_ax: ?u3 = std.simd.firstTrue(shape_._sharding_info); + //const sharding_ax: ?u3 = std.simd.firstTrue(shape_._sharding_info); const n_partitions = platform.sharding().num_partitions; - const chunk_size = if (sharding_ax) |ax| cs: { - // This kind of sharding error should be detected earlier on. - stdx.debug.assert(@rem(shape_.dim(ax), n_partitions) == 0, "Buffer.from({f}) expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ shape_, ax, n_partitions }); - break :cs @divExact(shape_.dim(ax), n_partitions); - } else 0; - _ = chunk_size; // autofix + //const chunk_size = if (sharding_ax) |ax| cs: { + // // This kind of sharding error should be detected earlier on. + // stdx.debug.assert(@rem(shape_.dim(ax), n_partitions) == 0, "Buffer.from({f}) expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ shape_, ax, n_partitions }); + // break :cs @divExact(shape_.dim(ax), n_partitions); + //} else 0; + //_ = chunk_size; // autofix const buffer_type = pjrt.bufferTypeFromDtype(shape_.dtype()); - const byte_strides = shape_.computeStrides(); + const byte_strides = shape_.computeByteStrides(); const devices = platform.getDevices(); for (0..n_partitions) |i| { diff --git a/zml2/shape.zig b/zml2/shape.zig index 034a4c5f..661e634e 100644 --- a/zml2/shape.zig +++ b/zml2/shape.zig @@ -13,21 +13,154 @@ const log = std.log.scoped(.shape); /// Represent the shape of a tensor. pub const Shape = struct { + pub const MAX_RANK: u8 = 8; + + pub const PartitionSpec = union(enum) { + axis: Tag, + open, + replicated, + unknown, + + pub fn init(comptime value: anytype) PartitionSpec { + const T = @TypeOf(value); + + if (T == PartitionSpec) { + return value; + } + + if (comptime T == EnumLiteral) { + const name = @tagName(value); + inline for (std.meta.fields(PartitionSpec)) |field| { + if (field.type == void and std.mem.eql(u8, field.name, name)) { + return @field(PartitionSpec, field.name); + } + } + } + + if (comptime isTagConvertible(T)) { + const tag_ = Shape.toTag(value); + + return .{ .axis = tag_ }; + } + + stdx.debug.compileError("PartitionSpec.init expected a tag, got: {any}", .{T}); + } + + test "PartitionSpec.init" { + const spec_axis: PartitionSpec = .{ .axis = Shape.toTag(.a) }; + try testing.expectEqual(spec_axis, PartitionSpec.init(.a)); + + const spec_axis_open: PartitionSpec = .open; + try testing.expect(spec_axis_open.eql(.init(.open))); + } + + pub fn eql(self: PartitionSpec, other: PartitionSpec) bool { + if (@as(u4, @intFromEnum(self)) != @as(u4, @intFromEnum(other))) { + return false; + } + + switch (self) { + .axis => |t1| { + const t2 = other.axis; + return std.mem.eql(u8, std.mem.span(t1), std.mem.span(t2)); + }, + else => return true, + } + } + + test "PartitionSpec.eql" { + const spec_axis: PartitionSpec = .init(.a); + const spec_axis_b: PartitionSpec = .init(.b); + const spec_open: PartitionSpec = .open; + const spec_replicated: PartitionSpec = .replicated; + const spec_unknown: PartitionSpec = .unknown; + + try testing.expect(spec_axis.eql(.{ .axis = Shape.toTag(.a) })); + try testing.expect(!spec_axis.eql(spec_axis_b)); + try testing.expect(!spec_axis.eql(.open)); + try testing.expect(!spec_axis.eql(.replicated)); + try testing.expect(!spec_axis.eql(.unknown)); + + try testing.expect(spec_open.eql(.open)); + try testing.expect(!spec_open.eql(spec_axis)); + try testing.expect(!spec_open.eql(.replicated)); + try testing.expect(!spec_open.eql(.unknown)); + + try testing.expect(spec_replicated.eql(.replicated)); + try testing.expect(!spec_replicated.eql(spec_axis)); + try testing.expect(!spec_replicated.eql(.open)); + try testing.expect(!spec_replicated.eql(.unknown)); + + try testing.expect(spec_unknown.eql(.unknown)); + try testing.expect(!spec_unknown.eql(spec_axis)); + try testing.expect(!spec_unknown.eql(.open)); + try testing.expect(!spec_unknown.eql(.replicated)); + } + + pub fn isClosed(self: PartitionSpec) bool { + return switch (self) { + .axis => true, + .open => false, + .replicated => true, + .unknown => false, + }; + } + + test isClosed { + const spec_axis: PartitionSpec = .init(.a); + try testing.expect(spec_axis.isClosed()); + + const spec_open: PartitionSpec = .open; + try testing.expect(!spec_open.isClosed()); + + const spec_replicated: PartitionSpec = .replicated; + try testing.expect(spec_replicated.isClosed()); + + const spec_unknown: PartitionSpec = .unknown; + try testing.expect(!spec_unknown.isClosed()); + } + + pub fn toTag(self: PartitionSpec) Tag { + return switch (self) { + .axis => |t| return t, + .open => TagOpen, + .replicated => TagReplicated, + .unknown => TagUnknown, + }; + } + + test "PartitionSpec.toTag" { + const spec_axis: PartitionSpec = .init(.a); + try testing.expect(spec_axis.toTag() == Shape.toTag(.a)); + + const spec_open: PartitionSpec = .open; + try testing.expect(spec_open.toTag() == TagOpen); + + const spec_replicated: PartitionSpec = .replicated; + try testing.expect(spec_replicated.toTag() == TagReplicated); + + const spec_unknown: PartitionSpec = .unknown; + try testing.expect(spec_unknown.toTag() == TagUnknown); + } + }; + pub const Tag = [*:0]const u8; pub const TagUnknown = "_".ptr; const TagLast = "last".ptr; + const TagReplicated = "replicated".ptr; + const TagOpen = "open".ptr; pub const DimsArray = stdx.BoundedArray(i64, constants.MAX_RANK); pub const TagsArray = stdx.BoundedArray(Tag, constants.MAX_RANK); pub const AxesArray = stdx.BoundedArray(u3, constants.MAX_RANK); - pub const ShardingInfo = @Vector(constants.MAX_RANK, bool); + pub const PartitionArray = stdx.BoundedArray(PartitionSpec, constants.MAX_RANK); const UnknownTags: TagsArray = .{ .len = 0, .buffer = [_]Tag{TagUnknown} ** constants.MAX_RANK }; _dtype: DataType, _dims: DimsArray = .{}, _tags: TagsArray = UnknownTags, - _sharding_info: ShardingInfo = @splat(false), + _partitioning: PartitionArray = .{}, pub fn parseDimensions(v: anytype) struct { DimsArray, TagsArray } { const T = @TypeOf(v); @@ -124,6 +257,8 @@ pub const Shape = struct { pub fn init(dimz: anytype, dt: DataType) Shape { var res: Shape = .{ ._dtype = dt }; res._dims, res._tags = parseDimensions(dimz); + res._partitioning.appendNTimes(.unknown, res._dims.len) catch @panic("Rank too large"); + return res; } @@ -140,6 +275,7 @@ pub const Shape = struct { }; res._tags.append(TagUnknown) catch unreachable; } + res._partitioning.appendNTimes(.unknown, rank_) catch @panic("Rank too large"); return res; } @@ -148,17 +284,17 @@ pub const Shape = struct { } pub fn rank(self: Shape) u4 { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return @intCast(self._dims.len); } pub fn dim(self: Shape, ax: anytype) i64 { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return self._dims.get(self.axis(ax)); } pub fn dims(self: *const Shape) []const i64 { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return self._dims.constSlice(); } @@ -185,12 +321,12 @@ pub const Shape = struct { }; } - inline fn ensureDimsAndTagsAreSync(self: Shape) void { - stdx.debug.assert(self._dims.len == self._tags.len, "Tags and dims have diverged! dims={d} tags={d}", .{ self._dims.len, self._tags.len }); + inline fn ensureAttributesAreSync(self: Shape) void { + stdx.debug.assert(self._dims.len == self._tags.len and self._dims.len == self._partitioning.len, "Tags, dims and partitioning have diverged! dims={d} tags={d} partitioning={d}", .{ self._dims.len, self._tags.len, self._partitioning.len }); } pub fn tag(self: Shape, ax: anytype) Tag { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return self._tags.get(self.axis(ax)); } @@ -210,7 +346,7 @@ pub const Shape = struct { } pub fn tags(self: *const Shape) []const Tag { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return self._tags.constSlice(); } @@ -250,7 +386,7 @@ pub const Shape = struct { } pub fn axis(self: Shape, axis_: anytype) u3 { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); const T = @TypeOf(axis_); if (comptime stdx.meta.isInteger(T)) { @@ -265,7 +401,7 @@ pub const Shape = struct { } pub fn axes(self: Shape, axes_: anytype) AxesArray { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); const T = @TypeOf(axes_); @@ -393,9 +529,12 @@ pub const Shape = struct { } else { try writer.print("{d}", .{d}); } - if (@as([constants.MAX_RANK]bool, self._sharding_info)[i]) { - try writer.writeByte('!'); + + const part = self._partitioning.get(i); + if (part.toTag() != TagUnknown) { + try writer.print("/{s}", .{part.toTag()}); } + need_comma = true; } if (need_comma) try writer.writeByte(','); @@ -408,13 +547,6 @@ pub const Shape = struct { // Already the right shape if (std.mem.eql(i64, self.dims(), other.dims())) return true; - if (std.mem.eql(Tag, self.tags(), other.tags())) { - for (0..self.rank()) |i| { - if (self.dim(i) != 1 and self.dim(i) != other.dim(i)) return false; - } - return true; - } - // Non ambiguous broadcasting // TODO: broad is error prone because of this: // it will happily broadcast .{ .a = 10, .b = 1 } to .{ .b = 10, .a = 5 } @@ -436,6 +568,7 @@ pub const Shape = struct { pub fn reshape(self: Shape, new_shape_: anytype) Shape { var new_shape: Shape = .{ ._dtype = self.dtype() }; new_shape._dims, new_shape._tags = parseDimensions(new_shape_); + new_shape._partitioning.appendNTimes(.unknown, new_shape._dims.len) catch @panic("Rank too large"); new_shape.inferMissingAxis(self.count()); stdx.debug.assert(self.count() == new_shape.count(), "Can't reshape {any} to {any}", .{ self.dims(), new_shape.dims() }); return new_shape; @@ -510,6 +643,11 @@ pub const Shape = struct { var res = self; res._dims.insertSlice(ax, dims_.constSlice()) catch unreachable; res._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; + + for (0..dims_.len) |_| { + res._partitioning.insert(ax, .unknown) catch unreachable; + } + return res; } @@ -530,6 +668,8 @@ pub const Shape = struct { var res = self; res._dims.insert(ax, d) catch unreachable; res._tags.insert(ax, toTag(tag_)) catch unreachable; + res._partitioning.insert(ax, .unknown) catch unreachable; + return res; } @@ -538,6 +678,7 @@ pub const Shape = struct { const dims_, const tags_ = parseDimensions(v); res._dims.appendSliceAssumeCapacity(dims_.constSlice()); res._tags.appendSliceAssumeCapacity(tags_.constSlice()); + res._partitioning.appendNTimesAssumeCapacity(.unknown, dims_.len); return res; } @@ -559,6 +700,7 @@ pub const Shape = struct { var res = self; res._dims.appendAssumeCapacity(d); res._tags.appendAssumeCapacity(if (tag_) |t| t else TagUnknown); + res._partitioning.appendAssumeCapacity(.unknown); return res; } @@ -567,14 +709,15 @@ pub const Shape = struct { const a = self.axis(axis_); _ = res._dims.orderedRemove(a); _ = res._tags.orderedRemove(a); + _ = res._partitioning.orderedRemove(a); return res; } pub const drop = remove; test remove { - try std.testing.expectEqualSlices(i64, &.{ 10, 12 }, Shape.init(.{ 10, 11, 12 }, .f32).remove(1).dims()); - try std.testing.expectEqualSlices(i64, &.{ 10, 11, 12 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).remove(-1).dims()); + try testing.expectEqualSlices(i64, &.{ 10, 12 }, Shape.init(.{ 10, 11, 12 }, .f32).remove(1).dims()); + try testing.expectEqualSlices(i64, &.{ 10, 11, 12 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).remove(-1).dims()); } pub fn removeMany(self: Shape, axes_: anytype) Shape { @@ -592,20 +735,22 @@ pub const Shape = struct { sh._dims.buffer[res_ax] = self._dims.buffer[ax]; sh._tags.buffer[res_ax] = self._tags.buffer[ax]; + sh._partitioning.buffer[res_ax] = self._partitioning.buffer[ax]; res_ax += 1; } sh._dims.len = rk - to_remove.len; sh._tags.len = rk - to_remove.len; + sh._partitioning.len = rk - to_remove.len; return sh; } test removeMany { - try std.testing.expectEqualSlices( + try testing.expectEqualSlices( i64, &.{12}, Shape.init(.{ 10, 11, 12 }, .f32).removeMany(.{ 0, 1 }).dims(), ); - try std.testing.expectEqualSlices( + try testing.expectEqualSlices( i64, &.{ 10, 11 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).removeMany(.{ -1, -2 }).dims(), @@ -616,10 +761,16 @@ pub const Shape = struct { std.debug.assert(self.rank() == permutations.len); const permutations_ = self.axes(permutations); var res = self; + const original_dims = self._dims; + const original_tags = self._tags; + const original_parts = self._partitioning; + for (permutations_.constSlice(), 0..) |permutation, i| { - res._dims.set(i, self.dim(permutation)); - res._tags.set(i, self.tag(permutation)); + res._dims.set(i, original_dims.get(permutation)); + res._tags.set(i, original_tags.get(permutation)); + res._partitioning.set(i, original_parts.get(permutation)); } + return res; } @@ -635,6 +786,10 @@ pub const Shape = struct { Shape.init(.{ .a = 10, .b = 11, .c = 12, .d = 13 }, .f32).transpose(.{ 0, 2, 1, 3 }), ), ); + + const shape = Shape.init(.{ .a = 10, .c = 12, .b = 11 }, .f32).withPartitioning(.{ .a = .batch, .c = .colors, .b = .open }).transpose(.{ 0, 2, 1 }); + try testing.expect(shape.eqlWithTags(Shape.init(.{ .a = 10, .b = 11, .c = 12 }, .f32))); + try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .open, .init(.colors) }, shape._partitioning.constSlice()); } /// Tag each ax of this shape with tags from a tuple. @@ -743,16 +898,233 @@ pub const Shape = struct { return res; } - pub fn withSharding(self: Shape, axes_: anytype) Shape { + pub fn withPartitioning(self: Shape, specs: anytype) Shape { + const T = @TypeOf(specs); + + var res = self.withDefaultPartitioning(); // todo add test for this new change + + if (stdx.meta.isStruct(T)) { + inline for (std.meta.fields(T)) |field| { + const partition_axis = @field(specs, field.name); + const axis_ = res.axisFromTagMaybe(toTag(field)); + + if (axis_) |ax| { + res._partitioning.set(ax, PartitionSpec.init(partition_axis)); + } else { + stdx.debug.panic("Partitioning axis {s} not found", .{field.name}); + } + } + } else { + stdx.debug.panic("Expected a struct of enum literals, got: {any}", .{T}); + } + + // Check that no mesh axis is used to partition multiple tensor dimensions. + var used_mesh_axes: stdx.BoundedArray(Tag, constants.MAX_RANK) = .{ .len = 0 }; + + for (res._partitioning.constSlice()) |spec| { + const axis_tag = spec.toTag(); + + if (std.mem.indexOfScalar(Tag, used_mesh_axes.constSlice(), axis_tag)) |_| { + stdx.debug.panic("Mesh axis '{s}' used to partition multiple tensor dimensions. This is not allowed.", .{axis_tag}); + } + + switch (spec) { + .unknown => {}, + .open => {}, + .replicated => {}, + .axis => used_mesh_axes.appendAssumeCapacity(axis_tag), + } + } + + return res; + } + + test withPartitioning { + var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(.{ .c = .feature, .a = .batch }); + try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .unknown, .init(.feature) }, shape._partitioning.constSlice()); + + shape = shape.withDefaultPartitioning(); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + + shape = shape.withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); + try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .open, .init(.feature) }, shape._partitioning.constSlice()); + } + + pub fn mapPartitioningAxes(self: Shape, mapping: anytype) Shape { + const T = @TypeOf(mapping); + stdx.debug.assertComptime(stdx.meta.isStruct(T), "Mapping must be a struct, e.g., .{ .old_axis = .new_axis }", .{}); + var res = self; - // Reset sharding. - res._sharding_info = @splat(false); - for (self.axes(axes_).constSlice()) |ax| { - res._sharding_info[ax] = true; + for (res._partitioning.slice()) |*spec| { + switch (spec.*) { + .unknown => {}, + .open => {}, + .replicated => {}, + .axis => |*old_axis_tag| { + inline for (std.meta.fields(T)) |field| { + const logical_axis_in_map = toTag(field); + + if (std.mem.eql(u8, std.mem.span(old_axis_tag.*), std.mem.span(logical_axis_in_map))) { + const physical_axis_from_map = @field(mapping, field.name); + spec.* = PartitionSpec.init(physical_axis_from_map); + break; + } + } + }, + } } return res; } + test mapPartitioningAxes { + const logical_shape = Shape.init(.{ .batch = 64, .seq = 128, .features = 512 }, .f32).withPartitioning(.{ + .batch = .data, + .seq = .replicated, + .features = .model, + }); + + const physical_shape = logical_shape.mapPartitioningAxes(.{ .data = .x, .model = .y }); + + try testing.expect(physical_shape.partition(.batch).eql(.init(.x))); + try testing.expect(physical_shape.partition(.features).eql(.init(.y))); + try testing.expect(physical_shape.partition(.seq).eql(.replicated)); + + const partial_mapped_shape = logical_shape.mapPartitioningAxes(.{ .data = .z }); + try testing.expect(partial_mapped_shape.partition(.batch).eql(.init(.z))); + try testing.expect(partial_mapped_shape.partition(.features).eql(.init(.model))); + } + + pub fn withDefaultPartitioning(self: Shape) Shape { + var res = self; + res._partitioning.clear(); + res._partitioning.appendNTimes(.unknown, self._dims.len) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); + return res; + } + + test withDefaultPartitioning { + var shape = Shape.init(.{ 10, 20, 30 }, .f32); + try testing.expectEqual(3, shape._partitioning.len); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + + shape = shape.withPartitioning(.{ ._1 = .batch }); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .init(.batch), .unknown }, shape._partitioning.constSlice()); + + shape = shape.withDefaultPartitioning(); + try testing.expectEqual(3, shape._partitioning.len); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + } + + pub fn withReplicatedPartitioning(self: Shape) Shape { + var res = self; + res._partitioning.clear(); + res._partitioning.appendNTimes(.replicated, self._dims.len) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); + return res; + } + + test withReplicatedPartitioning { + var shape = Shape.init(.{ 10, 20, 30 }, .f32); + try testing.expectEqual(3, shape._partitioning.len); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + + shape = shape.withPartitioning(.{ ._1 = .batch }); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .init(.batch), .unknown }, shape._partitioning.constSlice()); + + shape = shape.withReplicatedPartitioning(); + try testing.expectEqual(3, shape._partitioning.len); + try testing.expectEqualSlices(PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._partitioning.constSlice()); + } + + pub fn hasAtLeastOnePartitionedAxis(self: Shape) bool { + for (self._partitioning.constSlice()) |spec| { + if (spec == .axis) return true; + } + return false; + } + + test hasAtLeastOnePartitionedAxis { + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .x, .b = .replicated }); + try testing.expect(shape.hasAtLeastOnePartitionedAxis()); + + shape = shape.withPartitioning(.{ .a = .replicated, .b = .replicated }); + try testing.expect(!shape.hasAtLeastOnePartitionedAxis()); + + shape = shape.withDefaultPartitioning(); + try testing.expect(!shape.hasAtLeastOnePartitionedAxis()); + } + + pub fn isFullyPartitioned(self: Shape) bool { + for (self._partitioning.constSlice()) |spec| { + switch (spec) { + .replicated, .open, .unknown => return false, + .axis => {}, + } + } + return true; + } + + test isFullyPartitioned { + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .x, .b = .y }); + try testing.expect(shape.isFullyPartitioned()); + + shape = shape.withPartitioning(.{ .a = .x, .b = .replicated }); + try testing.expect(!shape.isFullyPartitioned()); + + shape = shape.withDefaultPartitioning(); + try testing.expect(!shape.isFullyPartitioned()); + } + + pub fn isFullyReplicated(self: Shape) bool { + for (self._partitioning.constSlice()) |spec| { + if (spec != .replicated and spec != .unknown) { + return false; + } + } + return true; + } + + test isFullyReplicated { + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .replicated, .b = .replicated }); + try testing.expect(shape.isFullyReplicated()); + + shape = shape.withPartitioning(.{ .a = .replicated, .b = .unknown }); + try testing.expect(shape.isFullyReplicated()); + + shape = shape.withPartitioning(.{ .a = .x, .b = .replicated }); + try testing.expect(!shape.isFullyReplicated()); + } + + pub fn partition(self: Shape, ax: anytype) PartitionSpec { + return self._partitioning.get(self.axis(ax)); + } + + test partition { + const shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); + try testing.expectEqual(PartitionSpec.init(.batch), shape.partition(.a)); + try testing.expectEqual(.open, shape.partition(.b)); + try testing.expectEqual(PartitionSpec.init(.feature), shape.partition(.c)); + } + + pub fn containsPartitionSpec(self: Shape, part: PartitionSpec) bool { + for (self._partitioning.constSlice()[0..self.rank()]) |dim_part| { + if (dim_part.eql(part)) return true; + } + return false; + } + + test containsPartitionSpec { + var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30, .d = 40 }, .f32).withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); + try testing.expect(shape.containsPartitionSpec(.init(.batch))); + try testing.expect(shape.containsPartitionSpec(.open)); + try testing.expect(!shape.containsPartitionSpec(.replicated)); + try testing.expect(shape.containsPartitionSpec(.unknown)); + + shape = shape.withDefaultPartitioning(); + try testing.expect(!shape.containsPartitionSpec(.init(.batch))); + try testing.expect(shape.containsPartitionSpec(.unknown)); + try testing.expect(!shape.containsPartitionSpec(.replicated)); + try testing.expect(!shape.containsPartitionSpec(.open)); + } + /// Renames some of the tags in this shape. /// Shape.init(.{ .a = 10, .b = 20 }).rename(.{ .b = .batch }); // .{ .a = 10, .batch = 20 }; pub fn rename(self: Shape, renames: anytype) Shape { @@ -761,7 +1133,7 @@ pub const Shape = struct { var res = self; inline for (std.meta.fields(T)) |field| { const new_field = @field(renames, field.name); - stdx.debug.assert(self.hasTag(new_field) == null, "{f}.rename({any}) failed because of duplicated axis {}", .{ self, renames, new_field }); + stdx.debug.assert(self.hasTag(new_field) == null, "{f}.rename({any}) failed because of duplicated axis {any}", .{ self, renames, new_field }); res._tags.set(self.axis(field), toTag(new_field)); } return res; @@ -781,7 +1153,103 @@ pub const Shape = struct { } } - pub fn computeStrides(self: Shape) stdx.BoundedArray(i64, constants.MAX_RANK) { + /// Computes the strides of the shape in units of elements. + /// For a shape {d0, d1, d2}, the strides would be {d1*d2, d2, 1}. + pub fn computeElementStrides(self: Shape) stdx.BoundedArray(i64, constants.MAX_RANK) { + return self.computeStrides(1); + } + + test computeElementStrides { + { + const s = Shape.init(.{ 4, 1024 }, .bf16); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 1024, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 2, 3, 4 }, .f32); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 12, 4, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 5, 6 }, .i16); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 6, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{10}, .u8); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{1}, strides.constSlice()); + } + + { + const s = Shape.scalar(.f64); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{}, strides.constSlice()); + } + + { + const s = Shape.init(.{ 4, 1, 5 }, .i32); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 5, 5, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ .n = 8, .c = 3, .h = 224, .w = 224 }, .f32); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 3 * 224 * 224, 224 * 224, 224, 1 }, strides.constSlice()); + } + } + + /// Computes the strides of the shape in units of bytes. + /// For a shape {d0, d1, d2} of type T, the strides would be + /// {(d1*d2)*sizeof(T), d2*sizeof(T), 1*sizeof(T)}. + pub fn computeByteStrides(self: Shape) stdx.BoundedArray(i64, constants.MAX_RANK) { + return self.computeStrides(self.dtype().sizeOf()); + } + + test computeByteStrides { + { + const s = Shape.init(.{ 2, 3, 4 }, .f16); // 2 bytes + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 12 * 2, 4 * 2, 1 * 2 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 4, 1024 }, .bf16); // 2 bytes + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 1024 * 2, 1 * 2 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 5, 6 }, .c64); // 8 bytes (f32 + f32) + const strides = s.computeByteStrides(); + try std.testing.expectEqual(@sizeOf(std.math.Complex(f32)), 8); + try std.testing.expectEqualSlices(i64, &.{ 6 * 8, 1 * 8 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 128, 512 }, .i8); // 1 byte + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 512, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 1, 4096 }, .u32); // 4 bytes + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 4096 * 4, 1 * 4 }, strides.constSlice()); + } + + { + const s = Shape.scalar(.f64); // 8 bytes + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{}, strides.constSlice()); + } + } + + fn computeStrides(self: Shape, element_size: i64) stdx.BoundedArray(i64, constants.MAX_RANK) { const rk = self.rank(); var strides: stdx.BoundedArray(i64, constants.MAX_RANK) = .{ .len = rk }; if (rk == 0) return strides; @@ -792,7 +1260,7 @@ pub const Shape = struct { // and the element size in bytes. var d: V = @bitCast(self._dims.buffer); d = @select(i64, rank_mask, d, @as(V, @splat(1))); - d = std.simd.shiftElementsLeft(d, 1, self.dtype().sizeOf()); + d = std.simd.shiftElementsLeft(d, 1, element_size); d = std.simd.prefixScan(.Mul, -1, d); strides.buffer = @bitCast(d); @@ -851,6 +1319,12 @@ pub const Shape = struct { var new_shape = self; new_shape._dims.replaceRange(ax, 1, dims_.constSlice()) catch unreachable; new_shape._tags.replaceRange(ax, 1, tags_.constSlice()) catch unreachable; + _ = new_shape._partitioning.orderedRemove(ax); + + for (0..dims_.len) |_| { + new_shape._partitioning.insert(ax, .unknown) catch unreachable; + } + new_shape.inferMissingAxis(self.count()); return new_shape; } @@ -895,22 +1369,43 @@ pub const Shape = struct { /// Merge the given axes into one axis. /// eg: `Shape.init(.{.a1 = 5, .a2 = 2, .b = 3}).merge(.{ .a = .{ .a1, .a2 }); -> .{ .a = 10, .b = 3 }` pub fn mergeAxis(self: Shape, axis_: anytype, axes_: anytype) Shape { - const axes__ = self.axes(axes_); - - const first_axis = axes__.get(0); - const last_axis = axes__.get(axes__.len - 1); + const axes_to_merge = self.axes(axes_); + stdx.debug.assert(axes_to_merge.len > 1, "Must merge at least two axes, got {any}", .{axes_}); var new_dim: i64 = 1; - for (axes__.constSlice(), first_axis..) |ax, counter| { - new_dim *= self.dim(ax); - stdx.debug.assert(ax == counter, "Can't merge shape {f} along non-contiguous axes {any}", .{ self, axes_ }); + // Merging partitioned axes is complex. If ANY axis being merged is partitioned, + // the resulting sharding is ambiguous. A propagation pass must resolve this. + // Marking it 'open' is the correct signal. If all are replicated/unknown, the + // result is also replicated/unknown. + // rule: .replicated wins over .unknown. .open wins over everything. + var resulting_partition_spec: PartitionSpec = .unknown; + + for (0..axes_to_merge.len) |i| { + const current_axis = axes_to_merge.get(i); + new_dim *= self.dim(current_axis); + + if (i > 0) { + stdx.debug.assert(current_axis == axes_to_merge.get(i - 1) + 1, "Can't merge shape {f} along non-contiguous axes {any}", .{ self, axes_ }); + } + + switch (self.partition(current_axis)) { + .axis, .open => resulting_partition_spec = .open, + .replicated => if (resulting_partition_spec != .open) { + resulting_partition_spec = .replicated; + }, + .unknown => {}, + } } + const first_axis_to_merge = axes_to_merge.get(0); + const num_axes_to_merge = axes_to_merge.len; + var new_shape = self; - new_shape._dims.set(first_axis, new_dim); - new_shape._dims.replaceRange(first_axis + 1, self.dims()[first_axis + 1 ..].len, self.dims()[last_axis + 1 ..]) catch unreachable; - new_shape._tags.set(first_axis, if (comptime isTagConvertible(@TypeOf(axis_))) toTag(axis_) else TagUnknown); - new_shape._tags.replaceRange(first_axis + 1, self.dims()[first_axis + 1 ..].len, self.tags()[last_axis + 1 ..]) catch unreachable; + + new_shape._dims.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{new_dim}) catch @panic("mergeAxis failed on dims"); + new_shape._tags.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{toTag(axis_)}) catch @panic("mergeAxis failed on tags"); + new_shape._partitioning.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{resulting_partition_spec}) catch @panic("mergeAxis failed on partitioning"); + return new_shape; } @@ -942,6 +1437,28 @@ pub const Shape = struct { ); } + test "mergeAxis partitioning" { + // Merging a partitioned axis with a replicated one -> open + const shape1 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32) + .withPartitioning(.{ .a = .replicated, .b = .x, .c = .replicated }); + const merged1 = shape1.mergeAxis(.ab, .{ .a, .b }); + try testing.expect(merged1.partition(.ab).eql(.open)); + try testing.expect(merged1.dim(.ab) == 10); + try testing.expect(merged1.rank() == 2); + + // Merging two replicated axes -> replicated + const shape2 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32) + .withPartitioning(.{ .a = .replicated, .b = .replicated, .c = .y }); + const merged2 = shape2.mergeAxis(.ab, .{ .a, .b }); + try testing.expect(merged2.partition(.ab).eql(.replicated)); + try testing.expect(merged2.partition(.c).eql(.init(.y))); + + // Merging two unknown axes -> unknown + const shape3 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32); // default is .unknown + const merged3 = shape3.mergeAxis(.ab, .{ .a, .b }); + try testing.expect(merged3.partition(.ab).eql(.unknown)); + } + pub fn mergeAxes(self: Shape, axes_: anytype) Shape { const T = @TypeOf(axes_); stdx.debug.assertComptime(stdx.meta.isStruct(T), "Must pass struct of enum literals like .{ .a = .{ .a1, .a2 } }. Passed: {any}", .{T}); @@ -1060,17 +1577,17 @@ pub const Shape = struct { test "comptimeShape" { comptime { const s = Shape.init(.{ .a = 5, .b = 6, .c = 7 }, .f32); - try std.testing.expectEqual(3, s.rank()); - try std.testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); - try std.testing.expectEqual(1, s.axis(.b)); + try testing.expectEqual(3, s.rank()); + try testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); + try testing.expectEqual(1, s.axis(.b)); } // comptime only the shape { const s = comptime Shape.init(.{ .a = 5, .b = 6, .c = 7 }, .f32); - try std.testing.expectEqual(3, s.rank()); - try std.testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); - try std.testing.expectEqual(1, s.axis(.b)); + try testing.expectEqual(3, s.rank()); + try testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); + try testing.expectEqual(1, s.axis(.b)); } } @@ -1089,4 +1606,135 @@ pub const Shape = struct { } return res_shape; } + + pub fn iterator(self: Shape) MultiDimIterator { + return .{ + .shape = self, + .current_coords = .{0} ** Shape.MAX_RANK, + .flat_index = 0, + }; + } }; + +/// Iterates over all coordinates of a shape, yielding the multi-dimensional +/// coordinates and the corresponding flat index for each element. +pub const MultiDimIterator = struct { + shape: Shape, + current_coords: [Shape.MAX_RANK]i64, + flat_index: usize, + + pub const Item = struct { + coords: [Shape.MAX_RANK]i64, + rank: u4, + flat_index: usize, + }; + + pub fn next(self: *MultiDimIterator) ?Item { + if (self.flat_index >= self.shape.count()) { + return null; + } + + const rank = self.shape.rank(); + const item = Item{ + .coords = self.current_coords, + .rank = rank, + .flat_index = self.flat_index, + }; + + self.flat_index += 1; + + var i: usize = rank; + while (i > 0) { + i -= 1; + self.current_coords[i] += 1; + if (self.current_coords[i] < self.shape.dim(i)) { + return item; + } + self.current_coords[i] = 0; + } + + return item; + } +}; + +test "MultiDimIterator / Scalar" { + const shape = Shape.scalar(.f32); + var iter = shape.iterator(); + const item = iter.next().?; + + try std.testing.expectEqual(0, item.rank); + try std.testing.expectEqual(0, item.flat_index); + try std.testing.expect(iter.next() == null); +} + +test "MultiDimIterator / Empty" { + const shape = Shape.init(.{ .x = 5, .y = 0, .z = 2 }, .f32); + var iter = shape.iterator(); + + try std.testing.expect(iter.next() == null); +} + +test "MultiDimIterator / 1D shape" { + const shape = Shape.init(.{5}, .f32); + var iter = shape.iterator(); + var count: i64 = 0; + + while (iter.next()) |item| : (count += 1) { + try std.testing.expectEqual(count, item.coords[0]); + } + try std.testing.expectEqual(5, count); +} + +test "MultiDimIterator / 2D shape" { + const shape = Shape.init(.{ .rows = 2, .cols = 3 }, .f32); + var iter = shape.iterator(); + var items: std.ArrayList(MultiDimIterator.Item) = .empty; + defer items.deinit(std.testing.allocator); + + while (iter.next()) |item| { + try items.append(std.testing.allocator, item); + } + + try std.testing.expectEqual(6, items.items.len); + try std.testing.expectEqualSlices(i64, &.{ 0, 0 }, items.items[0].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 0, 1 }, items.items[1].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 0, 2 }, items.items[2].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 1, 0 }, items.items[3].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 1, 1 }, items.items[4].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 1, 2 }, items.items[5].coords[0..2]); + try std.testing.expectEqual(5, items.items[5].flat_index); +} + +test "MultiDimIterator / 3D shape" { + const shape = Shape.init(.{ .d = 2, .h = 2, .w = 2 }, .f32); + var iter = shape.iterator(); + const expected_coords = [_][3]i64{ + .{ 0, 0, 0 }, + .{ 0, 0, 1 }, + .{ 0, 1, 0 }, + .{ 0, 1, 1 }, + .{ 1, 0, 0 }, + .{ 1, 0, 1 }, + .{ 1, 1, 0 }, + .{ 1, 1, 1 }, + }; + var count: usize = 0; + + while (iter.next()) |item| : (count += 1) { + try std.testing.expectEqual(count, item.flat_index); + try std.testing.expectEqualSlices(i64, &expected_coords[count], item.coords[0..3]); + } + + try std.testing.expectEqual(8, count); +} + +test "MultiDimIterator / Shape with dimension of size 1" { + const shape = Shape.init(.{ .a = 2, .b = 1, .c = 3 }, .f32); + var iter = shape.iterator(); + var count: usize = 0; + + while (iter.next()) |item| : (count += 1) { + try std.testing.expectEqual(0, item.coords[1]); + } + try std.testing.expectEqual(6, count); +} diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 9e707e81..28c6cdc0 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -88,6 +88,7 @@ pub const Tensor = struct { sh._dims.appendAssumeCapacity(ranked_tensor.dimension(i)); } sh._tags.resize(n) catch unreachable; + sh._partitioning.resize(n) catch unreachable; return .{ ._shape = sh, ._value = val, .id = Tensor.current_id.fetchAdd(1, .seq_cst) }; } From b4b8c2739b8e8a3b8b9cca766dfe5dc8d64b9332 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 10 Dec 2025 18:02:28 +0100 Subject: [PATCH 114/881] Various fixes in tensor.zig + Rng tests --- zml2/tensor.zig | 288 +++++++++++++++++++++++++----------------------- 1 file changed, 150 insertions(+), 138 deletions(-) diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 28c6cdc0..6aa20303 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -444,12 +444,11 @@ pub const Tensor = struct { return .{ ._state = .init(Shape.init(.{2}, .u64)) }; } - // TODO(Corentin) - //pub fn initBuffer(platform: Platform, seed: u128) !Bufferized(Rng) { - // return .{ - // ._state = try zml.Buffer.fromBytes(platform, Shape.init(.{2}, .u64), std.mem.asBytes(&seed)), - // }; - //} + pub fn initBuffer(platform: Platform, seed: u128, io: std.Io) !Bufferized(Rng) { + return .{ + ._state = try .fromBytes(platform, Shape.init(.{2}, .u64), std.mem.asBytes(&seed), io), + }; + } /// Returns a Tensor of the given shape, filled with uniform random bits, and a new Rng state. /// @@ -514,62 +513,68 @@ pub const Tensor = struct { return .{ rng, floats.convert(shape_.dtype()) }; } - //test uniform { - // const zml = @import("zml.zig"); - // const Stats = struct { - // const Stats = @This(); + test uniform { + const zml = @import("zml.zig"); + const Stats = struct { + const Stats = @This(); - // mean: Tensor, - // variance: Tensor, - // min: Tensor, - // max: Tensor, + mean: Tensor, + variance: Tensor, + min: Tensor, + max: Tensor, - // pub fn uniformStats( - // rand: Rng, - // shape_: Shape, - // opts: struct { min: f64, max: f64 }, - // ) struct { Rng, Stats } { - // const rng, const data = rand.uniform(shape_, .{ .min = opts.min, .max = opts.max }); - // const mean_ = data.mean(0); - // const variance = data.sub(mean_.broad(data.shape())).pow(Tensor.scalar(2, .f32)).mean(0); - // return .{ rng, .{ - // .mean = mean_, - // .variance = variance, - // .min = data.min(0), - // .max = data.max(0), - // } }; - // } - // }; + pub fn uniformStats( + rand: Rng, + shape_: Shape, + opts: struct { min: f64, max: f64 }, + ) struct { Rng, Stats } { + const rng, const data = rand.uniform(shape_, .{ .min = opts.min, .max = opts.max }); + const mean_ = data.mean(0); + const variance = data.sub(mean_.broad(data.shape())).pow(Tensor.scalar(2, .f32)).mean(0); + return .{ rng, .{ + .mean = mean_, + .variance = variance, + .min = data.min(0), + .max = data.max(0), + } }; + } + }; - // const platform = zml.testing.env(); - // // Compute stats over a uniform distribution on [-2, 10]. - // const rand, const stats = try zml.testing.compileAndCallWithTensors( - // platform, - // Stats.uniformStats, - // .{ Rng.shape(), zml.Shape.init(.{1024}, .f32), .{ .min = -2, .max = 10 } }, - // .{try Rng.init(platform, 1234)}, - // ); + const platform = zml.testing.env(); + // Compute stats over a uniform distribution on [-2, 10]. + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.uniformStats, .{ Rng.init(), Shape.init(.{1024}, .f32), .{ .min = -2, .max = 10 } }, platform); + defer exe.deinit(); - // // Check the Rng state has been modified. - // try std.testing.expect(try rand._state.getValue(u128) != 1234); + const rng_buffer = try Rng.initBuffer(platform, 1234, std.testing.io); + defer rng_buffer._state.deinit(); - // // Check the mean and variance are close to theoritical values. - // const mean_ = try stats.mean.getValue(f32); - // try std.testing.expectApproxEqAbs(4, mean_, 0.03); + const rand, const stats = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Stats.uniformStats, .{rng_buffer}); + defer rand._state.deinit(); + defer stats.mean.deinit(); + defer stats.variance.deinit(); + defer stats.min.deinit(); + defer stats.max.deinit(); - // const variance = try stats.variance.getValue(f32); - // try std.testing.expectApproxEqAbs(12.0 * 12.0 / 12.0, variance, 0.01); + // Check the Rng state has been modified. + try std.testing.expect(try rand._state.getValue(u128, std.testing.io) != 1234); - // // Check that no value is outside of the interval - // // and we have samples close to the edges. - // const min_ = try stats.min.getValue(f32); - // try std.testing.expect(min_ >= -2); - // try std.testing.expectApproxEqAbs(-2, min_, 0.05); + // Check the mean and variance are close to theoritical values. + const mean_ = try stats.mean.getValue(f32, std.testing.io); + try std.testing.expectApproxEqAbs(4, mean_, 0.03); - // const max_ = try stats.max.getValue(f32); - // try std.testing.expect(max_ < 10); - // try std.testing.expectApproxEqAbs(10, max_, 0.05); - //} + const variance = try stats.variance.getValue(f32, std.testing.io); + try std.testing.expectApproxEqAbs(12.0 * 12.0 / 12.0, variance, 0.01); + + // Check that no value is outside of the interval + // and we have samples close to the edges. + const min_ = try stats.min.getValue(f32, std.testing.io); + try std.testing.expect(min_ >= -2); + try std.testing.expectApproxEqAbs(-2, min_, 0.05); + + const max_ = try stats.max.getValue(f32, std.testing.io); + try std.testing.expect(max_ < 10); + try std.testing.expectApproxEqAbs(10, max_, 0.05); + } /// Returns a Tensor of the given shape, filled with floating point numbers sampled from a normal distribution. /// @@ -579,13 +584,10 @@ pub const Tensor = struct { stdx.debug.assert(sh.dtype().isFloat(), "normal expects tensor type to be a float, got {}", .{sh.dtype()}); const a = Tensor.constant(DataType.Value.init(sh.dtype(), opts.mean)); - _ = a; // autofix const b = Tensor.constant(DataType.Value.init(sh.dtype(), opts.stddev)); - _ = b; // autofix - unreachable; - //const res_shape = Tensor.constantTensor(HostBuffer.fromSlice(.{sh.rank()}, sh.dims())); - //const op = dialects.stablehlo.rng(mlirCtx(), a.value(), b.value(), res_shape.value(), .NORMAL, .unknown(mlirCtx())).appendTo(currentBlock()); - //return _result(sh, op.result(0)); + const res_tensor_shape = Tensor.constantTensor(Shape.init(.{sh.rank()}, .i64), std.mem.sliceAsBytes(sh.dims())); + const op = dialects.stablehlo.rng(mlirCtx(), a.value(), b.value(), res_tensor_shape.value(), .NORMAL, .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(sh, op.result(0)); } /// Returns a Tensor of the given shape, filled with floating point numbers sampled from a Gumbel distribution, and a new Rng state. @@ -606,72 +608,81 @@ pub const Tensor = struct { return .{ rand, u.log().scale(-1).log().scale(-1).convert(shape_.dtype()) }; } - //test gumbel { - // const zml = @import("zml.zig"); - // const Stats = struct { - // const Stats = @This(); + test gumbel { + const zml = @import("zml.zig"); + const Stats = struct { + const Stats = @This(); - // mean: Tensor, - // variance: Tensor, - // actual_dist: Tensor, + mean: Tensor, + variance: Tensor, + actual_dist: Tensor, - // pub fn gumbelStats(rand: Rng, target_dist: Tensor) struct { Rng, Stats } { - // const s = Shape.init(.{ .n = 1024, .d = 4 }, .f32); - // const rng, const data = rand.gumbel(s); - // const flat = data.flattenAll(); - // const mean_ = flat.mean(0); - // const variance = flat.sub(mean_.broad(flat.shape())).pow(Tensor.scalar(2, .f32)).mean(0); + pub fn gumbelStats(rand: Rng, target_dist: Tensor) struct { Rng, Stats } { + const s = Shape.init(.{ .n = 1024, .d = 4 }, .f32); + const rng, const data = rand.gumbel(s); + const flat = data.flatten(); + const mean_ = flat.mean(0); + const variance = flat.sub(mean_.broad(flat.shape())).pow(Tensor.scalar(2, .f32)).mean(0); - // // Test out the gumbel reparametrization trick - // var x = target_dist.log().withTags(.{.d}).broad(s); - // x = x.add(data); - // const samples = x.argMax(.d).indices.squeeze(.d); + // Test out the gumbel reparametrization trick + var x = target_dist.log().withTags(.{.d}).broad(s); + x = x.add(data); + const samples = x.argMax(.d).indices.squeeze(.d); - // // count 0, 1, 2 and 3 in samples: - // // - map 0 to 1, 1 to 2**16, 2 to 2**32, 3 to N**58 - // // - sum in u64 - // // - split to [4]u16 - // const powers = blk: { - // var powers: [4]u64 = undefined; - // for (&powers, 0..) |*p, i| p.* = std.math.pow(u64, 2, i * 16); - // break :blk powers; - // }; - // const values = Tensor.constantTensor(HostBuffer.fromArray(&powers)).withTags(.{.d}); - // const counts = values.gatherValues(.d, samples, .{}).sum(.n).bitCast(.u16); - // const actual_dist = counts.reshape(target_dist.shape()).convert(target_dist.dtype()).divByConst(s.dim(.n)); - // return .{ rng, .{ .mean = mean_, .variance = variance, .actual_dist = actual_dist } }; - // } - // }; + // count 0, 1, 2 and 3 in samples: + // - map 0 to 1, 1 to 2**16, 2 to 2**32, 3 to N**58 + // - sum in u64 + // - split to [4]u16 + const powers = blk: { + var powers: [4]u64 = undefined; + for (&powers, 0..) |*p, i| p.* = std.math.pow(u64, 2, i * 16); + break :blk powers; + }; + const values = Tensor.constantTensor(Shape.init(.{4}, .u64), std.mem.sliceAsBytes(&powers)).withTags(.{.d}); + const counts = values.gather(.{ .d = samples }, .{}).sum(.n).bitCast(.u16); + const actual_dist = counts.reshape(target_dist.shape()).convert(target_dist.dtype()).divByConst(s.dim(.n)); + return .{ rng, .{ .mean = mean_, .variance = variance, .actual_dist = actual_dist } }; + } + }; - // const platform = zml.testing.env(); - // const tgt_dist = [_]f32{ 2.0, 1.0, 4.0, 3.0 }; - // const rand, const stats = try zml.testing.compileAndCallWithTensors( - // platform, - // Stats.gumbelStats, - // .{ Rng.shape(), zml.Shape.init(.{tgt_dist.len}, .f32) }, - // .{ try Rng.init(platform, 1234), try .fromArray(platform, tgt_dist) }, - // ); - // // Check the Rng state has been modified. - // try std.testing.expect(try rand._state.getValue(i128) != 1234); + const platform = zml.testing.env(); + const tgt_dist_data = [_]f32{ 2.0, 1.0, 4.0, 3.0 }; + const tgt_dist: Tensor = .init(Shape.init(.{tgt_dist_data.len}, .f32)); - // // Check the mean and variance are close to theoritical values. - // const mean_ = try stats.mean.getValue(f32); - // try std.testing.expectApproxEqAbs(0.5772, mean_, 0.02); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.gumbelStats, .{ Rng.init(), tgt_dist }, platform); + defer exe.deinit(); - // const variance = try stats.variance.getValue(f32); - // const pi = std.math.pi; - // try std.testing.expectApproxEqAbs(pi * pi / 6.0, variance, 0.03); + const rng_buffer = try Rng.initBuffer(platform, 1234, std.testing.io); + defer rng_buffer._state.deinit(); + const tgt_dist_buffer: Buffer = try .fromBytes(platform, tgt_dist.shape(), std.mem.sliceAsBytes(&tgt_dist_data), std.testing.io); + defer tgt_dist_buffer.deinit(); - // // Check the distribution obtained with the gumbel trick matches the target distribution. - // const actual_dist = try stats.actual_dist.getValue([4]f32); - // scoped_log.debug("tgt_dist: {d}, actual_dist: {d}", .{ tgt_dist, actual_dist }); - // for (tgt_dist, actual_dist) |tgt, actual| { - // // We normalize tgt_dist to make it a well formed distribution. - // // We didn't do it before calling gumbel, because the gumbel trick - // // doesn't require normalized distributions as input. - // try std.testing.expectApproxEqAbs(tgt / 10.0, actual, 0.05); - // } - //} + const rand, const stats = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Stats.gumbelStats, .{ rng_buffer, tgt_dist_buffer }); + defer rand._state.deinit(); + defer stats.mean.deinit(); + defer stats.variance.deinit(); + defer stats.actual_dist.deinit(); + + // Check the Rng state has been modified. + try std.testing.expect(try rand._state.getValue(i128, std.testing.io) != 1234); + + // Check the mean and variance are close to theoritical values. + const mean_ = try stats.mean.getValue(f32, std.testing.io); + try std.testing.expectApproxEqAbs(0.5772, mean_, 0.02); + + const variance = try stats.variance.getValue(f32, std.testing.io); + const pi = std.math.pi; + try std.testing.expectApproxEqAbs(pi * pi / 6.0, variance, 0.03); + + // Check the distribution obtained with the gumbel trick matches the target distribution. + const actual_dist = try stats.actual_dist.getValue([4]f32, std.testing.io); + for (tgt_dist_data, actual_dist) |tgt, actual| { + // We normalize tgt_dist to make it a well formed distribution. + // We didn't do it before calling gumbel, because the gumbel trick + // doesn't require normalized distributions as input. + try std.testing.expectApproxEqAbs(tgt / 10.0, actual, 0.05); + } + } }; /// Returns a Tensor containing the element-wise conversion to another floating point type. @@ -1288,8 +1299,8 @@ pub const Tensor = struct { pub fn sum(self: Tensor, axis_: anytype) Tensor { const a = self.axis(axis_); return ops.reduce(.{self}, .{Tensor.constant(self.dtype().zero())}, &.{a}, struct { - pub fn acc(args: ops.ReduceArgs) Tensor { - return args.right.add(args.left.convert(args.right.dtype())); + pub fn acc(args: ops.ReduceArgs) struct { Tensor } { + return .{args.right.add(args.left.convert(args.right.dtype()))}; } }.acc, .{})[0]; } @@ -1366,13 +1377,17 @@ pub const Tensor = struct { /// Returns a transposed Tensor computed using the given axes. pub fn transpose(self: Tensor, axes_: anytype) Tensor { const axes__ = self.axes(axes_).constSlice(); - const default_perm = [constants.MAX_RANK]i64{ 7, 6, 5, 4, 3, 2, 1, 0 }; - const no_op = [constants.MAX_RANK]i64{ 0, 1, 2, 3, 4, 5, 6, 7 }; + const no_op = constants.AXES_IOTA; + const default_perm = b: { + var buf = constants.AXES_IOTA; + std.mem.reverse(i64, &buf); + break :b buf; + }; const permutation: []const i64 = if (axes__.len == 0) default_perm[constants.MAX_RANK - self.rank() ..] else - toI64(axes__); + toI64(axes__).constSlice(); stdx.debug.assert(permutation.len == self.rank(), "transpose expects input tensor rank and 'axes_' length to be equal, got {f} and {any}", .{ self, permutation[0..@min(permutation.len, constants.MAX_RANK + 2)] }); @@ -1389,7 +1404,7 @@ pub const Tensor = struct { mlirCtx(), self.value(), mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), - .{ .permutation = toI64(permutation) }, + .{ .permutation = toI64(permutation).constSlice() }, .unknown(mlirCtx()), ).appendTo(currentBlock()); return _result(res_shape, op.result(0)); @@ -1942,15 +1957,12 @@ pub const Tensor = struct { return Tensor.constant(sh.dtype().zero()).broad(sh); } - // TODO(Corentin) /// Embeds a buffer with concrete values into an Mlir program. - pub fn constantTensor() Tensor { - @panic("unimplemented"); - //const ctx = CompilationContext.current().mlirCtx(); - //const loc = ctx.location(@src()); + pub fn constantTensor(sh: Shape, bytes_: []const u8) Tensor { + const elem_type = mlirx.Type.fromDType(mlirCtx(), sh.dtype()); //const elem_type = mlirx.denseElementAttrType(val.dtype()) orelse std.debug.panic("constantTensor expects a dtype that can be serialized to MLIR, like f32 or i32, got {f}", .{val.shape()}); - //const constant_op = dialects.stablehlo.constant(ctx, val.shape().dims(), elem_type, val.bytes(), loc); - //return _result(val.shape(), constant_op.result(0)); + const constant_op = dialects.stablehlo.constant(mlirCtx(), sh.dims(), elem_type, bytes_, .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(sh, constant_op.result(0)); } /// Returns a Tensor containing the result of the outer product between the input Tensors. @@ -2131,7 +2143,7 @@ pub const Tensor = struct { pub fn reverse(self: Tensor, axes_: anytype) Tensor { const actual_axes = self._shape.axes(axes_); - const reverse_op = dialects.stablehlo.reverse(mlirCtx(), self.value(), toI64(actual_axes.constSlice()), .unknown(mlirCtx())).appendTo(currentBlock()); + const reverse_op = dialects.stablehlo.reverse(mlirCtx(), self.value(), toI64(actual_axes.constSlice()).constSlice(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, reverse_op.result(0)); } @@ -2702,7 +2714,7 @@ pub const Tensor = struct { pub fn cmp(args: ops.ReduceArgs) struct { Tensor } { return .{args.left.maximum(args.right)}; } - }, + }.cmp, .{}, )[0]; } @@ -2718,7 +2730,7 @@ pub const Tensor = struct { pub fn cmp(args: ops.ReduceArgs) struct { Tensor } { return .{args.left.minimum(args.right)}; } - }, + }.cmp, .{}, )[0]; } @@ -4089,10 +4101,10 @@ fn _parseGatherCoord(self: Tensor, axes_: anytype) struct { bool, stdx.BoundedAr return .{ axes_is_scalar, coord_axes }; } -fn toI64(values: anytype) []i64 { - var res: [constants.MAX_RANK]i64 = undefined; - for (values, 0..) |val, i| res[i] = @intCast(val); - return res[0..values.len]; +fn toI64(values: anytype) stdx.BoundedArray(i64, constants.MAX_RANK) { + var res: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; + for (values) |val| res.appendAssumeCapacity(@intCast(val)); + return res; } fn transposeIsJustAReshape(x: Shape, permutation: []const i64) bool { From 858b5bdd2d4bc3771f94511a170d513626719f9f Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 11 Dec 2025 10:52:37 +0100 Subject: [PATCH 115/881] Fix partitioning for toDiagonal --- zml2/tensor.zig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 6aa20303..95ee391a 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -3562,9 +3562,12 @@ pub const Tensor = struct { stdx.debug.assert(self.rank() < constants.MAX_RANK - 1, "toDiagonal expects input up to {d} rank, got {f}", .{ constants.MAX_RANK - 1, self }); const a = self.axis(axis_); const d = self.dim(a); + const p = self.shape()._partitioning.get(a); var res_shape = self._shape; res_shape._dims.replaceRange(a, 1, &.{ d, d }) catch unreachable; res_shape._tags.replaceRange(a, 1, &.{ @tagName(new_tags[0]), @tagName(new_tags[1]) }) catch unreachable; + // TODO(Corentin): Not sure about that + res_shape._partitioning.replaceRange(a, 1, &.{ p, p }) catch unreachable; const values = self.insertAxes(a + 1, .{new_tags[1]}).broad(res_shape); const zeros = Tensor.constant(self.dtype().zero()).broad(res_shape); From 1e9797f46bd8043858bb39cd6e04d7f3f36280f6 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 11 Dec 2025 13:09:23 +0100 Subject: [PATCH 116/881] Re-introduced nn.zig + tests --- zml2/BUILD.bazel | 1 + zml2/buffer.zig | 7 + zml2/dtype.zig | 2 +- zml2/nn.zig | 1375 ++++++++++++++++++++++++++++++++++++++++++++++ zml2/tensor.zig | 4 +- zml2/testing.zig | 4 + zml2/zml.zig | 1 + 7 files changed, 1392 insertions(+), 2 deletions(-) create mode 100644 zml2/nn.zig diff --git a/zml2/BUILD.bazel b/zml2/BUILD.bazel index 44e2350a..0c50c0fa 100644 --- a/zml2/BUILD.bazel +++ b/zml2/BUILD.bazel @@ -28,6 +28,7 @@ zig_library( "meta.zig", "mlirx.zig", "module.zig", + "nn.zig", "ops.zig", "pjrtx.zig", "platform.zig", diff --git a/zml2/buffer.zig b/zml2/buffer.zig index 5dd13902..906b740e 100644 --- a/zml2/buffer.zig +++ b/zml2/buffer.zig @@ -5,6 +5,7 @@ const stdx = @import("stdx"); const pjrt = @import("pjrtx.zig"); const Shape = @import("shape.zig").Shape; +const DataType = @import("dtype.zig").DataType; const Platform = @import("platform.zig").Platform; const Target = @import("platform.zig").Target; @@ -116,6 +117,12 @@ pub const Buffer = struct { return from(platform, sh, data, io, .{}); } + /// Creates a Buffer with a single element. + pub fn scalar(platform: Platform, val: anytype, dtype_: DataType, io: std.Io) !Buffer { + const x = dtype_.constant(val); + return fromBytes(platform, Shape.init(.{}, dtype_), x.asBytes(), io); + } + pub fn await(self: Buffer, io: std.Io) !Buffer { for (self._shards.constSlice()) |buffer| { if (buffer.getReadyEvent(self._api)) |ev| { diff --git a/zml2/dtype.zig b/zml2/dtype.zig index 78fa853c..fe80b711 100644 --- a/zml2/dtype.zig +++ b/zml2/dtype.zig @@ -336,7 +336,7 @@ pub const DataType = enum(u8) { return std.meta.activeTag(self); } - pub fn constSlice(data: *const Value) []const u8 { + pub fn asBytes(data: *const Value) []const u8 { return switch (data.*) { inline else => |*value| std.mem.asBytes(value), }; diff --git a/zml2/nn.zig b/zml2/nn.zig new file mode 100644 index 00000000..9db31b98 --- /dev/null +++ b/zml2/nn.zig @@ -0,0 +1,1375 @@ +//! Common layer definition and functions for Neural Networks (NN) +const std = @import("std"); + +const stdx = @import("stdx"); + +const DataType = @import("dtype.zig").DataType; +const meta = @import("meta.zig"); +const ops = @import("ops.zig"); +const Shape = @import("shape.zig").Shape; +const Tensor = @import("tensor.zig").Tensor; +const zml = @import("zml.zig"); + +const log = std.log.scoped(.@"zml/nn"); + +pub const Linear = struct { + weight: Tensor, + bias: ?Tensor = null, + + pub fn forward(self: Linear, x: Tensor) Tensor { + var y = x.dotGeneral(self.weight.convert(x.dtype()), &.{.{ -1, -1 }}, &.{}); + // If self.weight doesn't have tags, preserve tags from x. + if (y.shape().tag(-1) == Shape.TagUnknown) { + y._shape._tags.set(y.rank() - 1, x.shape().tag(-1)); + } + + // log.debug("Linear({*}): {d} -> {d} -> {d}", .{ self, x.dims(), y.dims(), if (self.bias) |bias| y.add(bias).dims() else y.dims() }); + return if (self.bias) |bias| y.add(bias.broadcast(y.shape(), &.{y.axis(-1)})) else y; + } +}; + +pub const TokenEmbedding = struct { + weight: Tensor, + + pub fn forward(self: TokenEmbedding, idx: Tensor) Tensor { + stdx.debug.assert(idx.dtype().isInteger(), "TokenEmbedding expects an integer input, received: {f}", .{idx}); + stdx.debug.assert(self.weight.rank() == 2, "TokenEmbedding expects it's weight Tensor to be a 2D matrix, got {f}", .{self.weight}); + return self.weight.withTags(.{ .voc, .d }).gather(.{ .voc = idx }, .{}); + } +}; + +pub const Activation = union(enum) { + sigmoid, + tanh, + relu, + leakyReLU: f32, + elu: f32, + silu, + gelu, + quick_gelu, + + pub fn forward(self: Activation, x: Tensor) Tensor { + return switch (self) { + .sigmoid => x.sigmoid(), + .tanh => x.tanh(), + .relu => x.relu(), + .silu => x.silu(), + .gelu => x.gelu(), + .elu => |alpha| elu(x, alpha), + .quick_gelu => x.quickGelu(), + .leakyReLU => |slope| x.leakyReLU(slope), + }; + } +}; + +pub fn elu(x: Tensor, alpha: f32) Tensor { + return x.cmp(.GE, Tensor.scalar(0, x.dtype())).select( + x, + x.exp().addConstant(-1).scale(alpha), + ); +} + +/// Layer Normalization +pub const LayerNorm = struct { + weight: Tensor, + bias: ?Tensor = null, + eps: f32 = 1e-5, + + pub fn forward(self: LayerNorm, x: Tensor) Tensor { + const normed = normalizeVariance(x, self.eps); + const ax = x.axis(-1); + var out = normed.mul(self.weight.broadcast(x.shape(), &.{ax})); + if (self.bias) |bias| out = out.add(bias.broadcast(x.shape(), &.{ax})); + + return out; + } +}; + +pub fn rmsNorm(x: Tensor, axis: anytype, eps: f32) Tensor { + const ax = x.axis(axis); + // upcast to improve precision + const variance = x.convert(.f32).powByConst(2).mean(ax); + const rsqrt = Tensor.rsqrt(variance.addConstant(eps)).convert(x.dtype()); + return x.mul(rsqrt.broad(x.shape())); +} + +/// Center and scale by the variance. +/// normalize(x, eps) = (x - mean(x)) / sqrt(var(x) + eps) +/// Work on the last axis. +pub fn normalizeVariance(x: Tensor, eps: f32) Tensor { + const N: f32 = @floatFromInt(x.dim(-1)); + + // Upcast to improve precision + const xf32 = x.convert(.f32); + const mean = xf32.sum(-1).scale(1.0 / N); + const mean_dev = xf32.sub(mean); + const variance = mean_dev.mul(mean_dev).sum(-1).divByConst(N); + const rsqrt = Tensor.rsqrt(variance.addConstant(eps)); + + return mean_dev.mul(rsqrt).convert(x.dtype()); +} + +// ref: https://pytorch.org/docs/stable/generated/torch.nn.functional.normalize.html +// Implementation equivalent to `nn.functional.normalize(tensor, dim=-1)` call +pub fn normalizeL2(input: Tensor, eps: f32) Tensor { + const inv_norm = input.powByConst(2).sum(-1).addConstant(eps).rsqrt(); + return input.mul(inv_norm); +} + +test normalizeL2 { + const platform = zml.testing.env(); + + const input: zml.Tensor = .init(Shape.init(.{ 2, 2 }, .f32)); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, normalizeL2, .{ input, 1e-12 }, platform); + defer exe.deinit(); + + var input_buffer: zml.Buffer = try .fromBytes(platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.9686, -1.0058, -1.7808, 0.6698 }), std.testing.io); + defer input_buffer.deinit(); + + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, normalizeL2, .{input_buffer}); + defer res.deinit(); + + const expectation = zml.testing.Slice.init(input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6937, -0.7203, -0.9360, 0.3520 })); + try zml.testing.expectClose(std.testing.io, expectation, res, 1e-4); +} + +pub const RopeOpts = struct { + layout: Layout = .sequential, + freq_base: f32 = 10_000, + scaling: Scaling = .default, + + /// There are two layouts corresponding to how to split `x` in real/imag parts. + /// * Interleaved means that the real/imag of each scalar is contiguous. + /// * Sequential means that you first read all real values then all imag values. + /// Typically HF models use sequential, while GGUF use interleaved. + pub const Layout = enum { interleaved, sequential }; + + /// There are several ways to init the scaling aka "inv_freq" + pub const Scaling = union(enum) { + default: void, + custom: []const f32, + llama3: Llama3, + yarn: Yarn, + + pub const Llama3 = struct { + factor: f32, + high_freq_factor: f32, + low_freq_factor: f32, + original_max_position_embeddings: u32, + truncate: bool = true, + }; + + pub const Yarn = struct { + beta_fast: f32 = 32.0, + beta_slow: f32 = 1.0, + factor: f32, + truncate: bool = true, + original_max_position_embeddings: u32, + }; + + /// Read a Rope scaling config from HF config.json format. + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !Scaling { + const content = try std.json.Value.jsonParse(allocator, source, options); + if (content == .null) return .default; + + if (content != .object) return error.InvalidEnumTag; + + const obj = content.object; + const impl = obj.get("rope_type") orelse return error.MissingField; + if (impl != .string) return error.InvalidEnumTag; + if (std.mem.eql(u8, impl.string, "llama3")) { + // Note: leaky is fine here cause Llama3 struct don't need to allocate memory. + return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, stdx.noalloc, content, .{ .ignore_unknown_fields = true }) }; + } else if (std.mem.eql(u8, impl.string, "yarn")) { + return .{ .yarn = try std.json.parseFromValueLeaky(Yarn, stdx.noalloc, content, .{ .ignore_unknown_fields = true }) }; + } else { + log.warn("Unsupported Rope implementation: {s}, will use the default one which will produce altered results", .{impl.string}); + return .{ .default = {} }; + } + } + + pub fn attentionScaling(scaling: Scaling) f32 { + return switch (scaling) { + .yarn => |yarn| 0.1 * @log(yarn.factor) + 1.0, + else => 1.0, + }; + } + }; +}; + +/// Rotary position embedding modify queries and keys tensor before compute Q * K in self attention. +/// This biases a token to look at token near him. +/// The nice thing with rope is that you can cache the modified queries and keys directly. +/// See: https://paperswithcode.com/method/rope +/// +/// Expected shapes of tensor: +/// - x: .{ .s, .hd } where .s is the sequence length and .hd the head dimension +/// - pos_idx: optional tensor which indicates which positions are needed. +/// When not set `rope` return all positions from 0 to x.dim(.s) which is the max seq len. +pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { + stdx.debug.assert(@mod(x.dim(.hd), 2) == 0, "rope expects a even head dim (.hd), got {f}", .{x}); + + const idx = if (pos_idx) |idx| blk: { + stdx.debug.assert(x.shape().hasTags(.{.hd}), "rope expects x argument to have .hd axes got: rope(x={f}, idx={f})", .{ x, idx }); + break :blk idx; + } else blk: { + stdx.debug.assert(x.shape().hasTags(.{ .s, .hd }), "rope expects x argument to have both .s and .hd axes got: rope(x={f})", .{x}); + break :blk Tensor.arange(.{ .end = x.dim(.s) }, .f32).withTags(.{.s}); + }; + const x_real, const x_imag = splitRealImg(x, opts.layout); + + // compute sin and cos in f32 before downcasting to x type. + const inv_freq = invFreq(x.dim(.hd), opts).withTags(.{.hd}); + const inv_freq_pos = Tensor.outer(idx.convert(.f32), inv_freq); + const scaling = opts.scaling.attentionScaling(); + const cos = inv_freq_pos.cos().scale(scaling).convert(x.dtype()).broad(x_real.shape()); + const sin = inv_freq_pos.sin().scale(scaling).convert(x.dtype()).broad(x_real.shape()); + + // apply rotation + const y_real = x_real.mul(cos).sub(x_imag.mul(sin)); + const y_imag = x_real.mul(sin).add(x_imag.mul(cos)); + + // flatten last dimensions + return mergeRealImg(y_real, y_imag, opts.layout); +} + +pub fn splitRealImg(x: Tensor, layout: RopeOpts.Layout) [2]Tensor { + const n = x.dim(-1); + + return switch (layout) { + .sequential => .{ + x.slice1d(-1, .{ .end = @divExact(n, 2) }), + x.slice1d(-1, .{ .start = @divExact(n, 2), .end = n }), + }, + .interleaved => .{ + x.slice1d(-1, .{ .start = 0, .step = 2 }), + x.slice1d(-1, .{ .start = 1, .step = 2 }), + }, + }; +} + +pub fn mergeRealImg(x_real: Tensor, x_imag: Tensor, layout: RopeOpts.Layout) Tensor { + return switch (layout) { + .sequential => Tensor.concatenate(&.{ x_real, x_imag }, -1), + .interleaved => Tensor.concatenate(&.{ + x_real.appendAxes(.{.interleaved_real_img}), + x_imag.appendAxes(.{.interleaved_real_img}), + }, -1).reshape(x_imag.shape().setDim(-1, -1)), + }; +} + +/// {exp( - n * ln(10_000) / N ) | n in [0..N] } +fn invFreq(N: i64, opts: RopeOpts) Tensor { + const allocator = zml.module.CompilationContext.current().allocator; + const N_half: usize = @intCast(@divExact(N, 2)); + const inv_freq = allocator.alloc(f32, N_half) catch @panic("OOM"); + defer allocator.free(inv_freq); + + _invFreq(opts, inv_freq); + return zml.Tensor.constantTensor(zml.Shape.init(.{@divExact(N, 2)}, .f32), std.mem.sliceAsBytes(inv_freq)); +} + +fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { + const N = inv_freq.len; + // Default frequencies + for (0.., inv_freq) |n, *f| { + f.* = @exp(-@log(opts.freq_base) * stdx.math.divFloat(f32, n, N)); + } + + switch (opts.scaling) { + .default => {}, + .custom => { + stdx.debug.assert(opts.scaling.custom.len == N, "rope expected custom inv_freq to match half head dimension {d}, got {d}", .{ N, opts.scaling.custom.len }); + @memcpy(inv_freq, opts.scaling.custom); + }, + .llama3 => |s| { + // https://arxiv.org/pdf/2309.16039 + // After Llama2 they observed that the rope frequencies where too sharp and hurting long distance attention. + // In Llama3 they used a higher base freq and also downscaled low frequencies. + std.debug.assert(s.low_freq_factor < s.high_freq_factor); + const M: f64 = @floatFromInt(s.original_max_position_embeddings); + const f_high = s.high_freq_factor * (2 * std.math.pi) / M; + const f_low = s.low_freq_factor * (2 * std.math.pi) / M; + const downscaling = 1.0 / s.factor; + + for (0..N, inv_freq) |n, f| { + if (f > f_high) { + // High freq match default implem + } else if (f < f_low) { + // Downscaling for low freq + inv_freq[n] *= downscaling; + } else { + // Linear interpolation for middle freq + const lerp: f64 = (inv_freq[n] - f_low) / (f_high - f_low); + inv_freq[n] *= @floatCast(lerp + (1 - lerp) * downscaling); + } + } + }, + .yarn => |s| { + const N_f: f64 = @floatFromInt(inv_freq.len); + const M: f64 = @floatFromInt(s.original_max_position_embeddings); + const f_high = s.beta_fast * (2 * std.math.pi) / M; + const f_low = s.beta_slow * (2 * std.math.pi) / M; + const downscaling = 1.0 / s.factor; + + // This isn't a typo: low n have a high frequency, high n have a low frequency. + var n_low: f64 = -@log(f_high) / @log(opts.freq_base) * N_f; + var n_high: f64 = -@log(f_low) / @log(opts.freq_base) * N_f; + if (s.truncate) { + n_high = std.math.ceil(n_high); + n_low = std.math.floor(n_low); + } + std.debug.assert(n_high > n_low); + for (0..N, inv_freq) |n, f| { + if (f > f_high) { + // High freq match default implem + } else if (f < f_low) { + // Downscaling for low freq + inv_freq[n] *= downscaling; + } else { + // Yarn use lerp too but not in the frequency space, in the time space. + const lerp: f64 = (n_high - @as(f64, @floatFromInt(n))) / (n_high - n_low); + inv_freq[n] *= @floatCast(lerp + (1 - lerp) * downscaling); + } + } + }, + } +} + +test "invFreq Llama3" { + // Llama 3.2-1B config + const llama_conf: RopeOpts = .{ + .freq_base = 500_000, + .scaling = .{ .llama3 = .{ + .factor = 32, + .high_freq_factor = 4, + .low_freq_factor = 1, + .original_max_position_embeddings = 8192, + } }, + }; + const llama_freq = [_]f32{ 1.000000000000e+00, 6.636012792587e-01, 4.403666257858e-01, 2.922278344631e-01, 1.939227581024e-01, 1.286873817444e-01, 8.539710193872e-02, 5.666961893439e-02, 3.760603070259e-02, 2.495540864766e-02, 1.656044088304e-02, 1.098952908069e-02, 7.292665075511e-03, 4.839421249926e-03, 3.211446106434e-03, 1.290548010729e-03, 4.295567050576e-04, 9.708286233945e-05, 1.946163865796e-05, 1.291476746701e-05, 8.570255886298e-06, 5.687232260243e-06, 3.774054448513e-06, 2.504467147446e-06, 1.661967417022e-06, 1.102883629756e-06, 7.318749339902e-07, 4.856731266045e-07, 3.222932889457e-07, 2.138742303259e-07, 1.419272024350e-07, 9.418306490261e-08 }; + + var inv_freq: @TypeOf(llama_freq) = undefined; + _invFreq(llama_conf, &inv_freq); + for (llama_freq, inv_freq, 0..) |expected, actual, i| { + errdefer log.err("Mismatch at position {d}.\nExpected: {any}\nActual: {any}", .{ i, llama_freq, inv_freq }); + try std.testing.expectApproxEqRel(expected, actual, 1e-5); + } +} + +test "invFreq Yarn" { + const yarn_conf: RopeOpts = .{ + .freq_base = 150_000, + .scaling = .{ .yarn = .{ + .factor = 32.0, + .beta_fast = 32.0, + .beta_slow = 1.0, + .original_max_position_embeddings = 4096, + .truncate = true, + } }, + }; + const yarn_freq = [_]f32{ 1.000000000000e+00, 6.890442967415e-01, 4.747820496559e-01, 3.271458745003e-01, 2.254180014133e-01, 1.553229838610e-01, 1.070244237781e-01, 7.374456524849e-02, 5.081327259541e-02, 3.162075206637e-02, 1.945096626878e-02, 1.179219130427e-02, 7.015713956207e-03, 4.069554619491e-03, 2.277272054926e-03, 1.206130953506e-03, 5.809474969283e-04, 2.279478358105e-04, 3.830881178146e-05, 2.639646845637e-05, 1.818833698053e-05, 1.253256959899e-05, 8.635495760245e-06, 5.950239483354e-06, 4.099978468730e-06, 2.825066758305e-06, 1.946596285052e-06, 1.341290953860e-06, 9.242089618056e-07, 6.368209142238e-07, 4.387978549403e-07, 3.023511396805e-07 }; + + var inv_freq: @TypeOf(yarn_freq) = undefined; + _invFreq(yarn_conf, &inv_freq); + for (yarn_freq, inv_freq, 0..) |expected, actual, i| { + errdefer log.err("Mismatch at position {d}.\nExpected: {d}\nActual: {d}", .{ i, stdx.fmt.slice(&yarn_freq), stdx.fmt.slice(&inv_freq) }); + try std.testing.expectApproxEqRel(expected, actual, 1e-5); + } + try std.testing.expectApproxEqRel(1.3465735902799727, yarn_conf.scaling.attentionScaling(), 1e-5); +} + +test "real/img" { + const platform = zml.testing.env(); + + const Fns = struct { + fn testSplitMergeIsId(layout: RopeOpts.Layout) Tensor { + const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ 5, 4 }); + const real, const imag = splitRealImg(x, layout); + const y = mergeRealImg(real, imag, layout); + const real2, const imag2 = splitRealImg(y, layout); + return real.cmp(.EQ, real2).flatten().convert(.i32).sum(-1).add( + imag.cmp(.EQ, imag2).flatten().convert(.i32).sum(-1), + ); + } + + fn testSplitSeqVoid(_: void) Tensor { + const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ 5, 4 }); + const real, const imag = splitRealImg(x, .sequential); + const x_real = Tensor.concatenate(&.{ + Tensor.arange(.{ .start = 0, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), + Tensor.arange(.{ .start = 1, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), + }, 1); + const x_imag = Tensor.concatenate(&.{ + Tensor.arange(.{ .start = 2, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), + Tensor.arange(.{ .start = 3, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), + }, 1); + + return real.cmp(.EQ, x_real).flatten().convert(.i32).sum(-1).add( + imag.cmp(.EQ, x_imag).flatten().convert(.i32).sum(-1), + ); + } + + fn testSplitSeq() Tensor { + const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ 5, 4 }); + const real, const imag = splitRealImg(x, .sequential); + const x_real = Tensor.concatenate(&.{ + Tensor.arange(.{ .start = 0, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), + Tensor.arange(.{ .start = 1, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), + }, 1); + const x_imag = Tensor.concatenate(&.{ + Tensor.arange(.{ .start = 2, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), + Tensor.arange(.{ .start = 3, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), + }, 1); + + return real.cmp(.EQ, x_real).flatten().convert(.i32).sum(-1).add( + imag.cmp(.EQ, x_imag).flatten().convert(.i32).sum(-1), + ); + } + + fn testSplitInterleaved() Tensor { + const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ 5, 4 }); + const real, const imag = splitRealImg(x, .interleaved); + const x_real = Tensor.arange(.{ .start = 0, .end = 20, .step = 2 }, .f32).reshape(.{ 5, 2 }); + const x_imag = Tensor.arange(.{ .start = 1, .end = 20, .step = 2 }, .f32).reshape(.{ 5, 2 }); + + return real.cmp(.EQ, x_real).flatten().convert(.i32).sum(-1).add( + imag.cmp(.EQ, x_imag).flatten().convert(.i32).sum(-1), + ); + } + }; + { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.interleaved}, platform); + defer exe.deinit(); + + var d_interleaved = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitMergeIsId, {}); + defer d_interleaved.deinit(); + try std.testing.expectEqual(20, try d_interleaved.getValue(i32, std.testing.io)); + } + { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.sequential}, platform); + defer exe.deinit(); + + var d_sequential = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitMergeIsId, {}); + defer d_sequential.deinit(); + try std.testing.expectEqual(20, try d_sequential.getValue(i32, std.testing.io)); + } + + // test the function that accepts 1 void argument + { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitSeqVoid, .{{}}, platform); + defer exe.deinit(); + + var d_split_seq_void = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitSeqVoid, {}); + defer d_split_seq_void.deinit(); + try std.testing.expectEqual(20, try d_split_seq_void.getValue(i32, std.testing.io)); + } + + // test the function that takes NO arguments + { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitSeq, .{}, platform); + defer exe.deinit(); + + var d_split_seq = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitSeq, {}); + defer d_split_seq.deinit(); + try std.testing.expectEqual(20, try d_split_seq.getValue(i32, std.testing.io)); + } + + { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitInterleaved, .{}, platform); + defer exe.deinit(); + + var d_split_seq = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitInterleaved, {}); + defer d_split_seq.deinit(); + try std.testing.expectEqual(20, try d_split_seq.getValue(i32, std.testing.io)); + } +} + +test rope { + const platform = zml.testing.env(); + + const Local = struct { + fn _fwd(x: Tensor, opts: RopeOpts) Tensor { + var input = x; + { + // Convert input to the requested format + const real, const imag = splitRealImg(input, .sequential); + input = mergeRealImg(real, imag, opts.layout); + } + var res = rope(input, null, opts).squeeze(0); + + { + // Convert back to sequential + const real, const imag = splitRealImg(res, opts.layout); + res = mergeRealImg(real, imag, .sequential); + } + return res; + } + }; + + // x is made such as the interleaved and sequential reps are the same. + // So the two implementations should give the same results. + const x: zml.Tensor = .init(Shape.init(.{ .b = 1, .s = 5, .hd = 4 }, .f32)); + var exe_interleaved = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .interleaved } }, platform); + defer exe_interleaved.deinit(); + + var exe_sequential = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .sequential } }, platform); + defer exe_sequential.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ 1.0, 0.1, -1.0, -0.5 } ** 5), std.testing.io); + defer x_buffer.deinit(); + + var res1 = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_interleaved, Local._fwd, .{x_buffer}); + defer res1.deinit(); + var res2 = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_sequential, Local._fwd, .{x_buffer}); + defer res2.deinit(); + + try zml.testing.expectClose(std.testing.io, res1, res2, 1e-4); +} + +pub const UpsampleMode = enum { + nearest, + // TODO: Linear, + // TODO: Bilinear, + // TODO: Bicubic, + // TODO: Trilinear, +}; + +/// Upsample +pub fn upsample( + input: Tensor, + opts: struct { mode: UpsampleMode, scale_factor: []const f64 }, +) Tensor { + // TODO(james): make `nearest` compatible with resizeBilinear and resizeBicubic, and wrap them here. + // resize* have API which are more explicit, this assume you want to scale the N-2 last axes. + stdx.debug.assert(3 <= input.rank() and input.rank() <= 5, "upsample is only implemented for (3,4,5)-D tensors, received {f}", .{input}); + stdx.debug.assert(opts.scale_factor.len == 1 or opts.scale_factor.len == input.rank() - 2, "scale factors", .{}); + return switch (opts.mode) { + .nearest => { + var scale_factors: [3]f64 = undefined; + switch (opts.scale_factor.len) { + 1 => { + for (0..input.rank() - 2) |i| scale_factors[i] = opts.scale_factor[0]; + }, + else => @memcpy(scale_factors[0..opts.scale_factor.len], opts.scale_factor), + } + return nearest(input, scale_factors[0 .. input.rank() - 2]); + }, + }; +} + +pub fn nearest(input: Tensor, scale_factor: []const f64) Tensor { + var out_shape = input.shape(); + for (scale_factor, 0..) |sf, i| { + out_shape._dims.set(i + 2, @intFromFloat(@floor(@as(f64, @floatFromInt(out_shape.dim(i + 2))) * sf))); + } + // TODO(james): remove this implicit two batching dims + var sd: [3]u3 = undefined; + var len_sd: usize = 0; + for (2..input.rank()) |i| { + if (input.dim(i) != out_shape.dim(i)) { + sd[len_sd] = @intCast(i); + len_sd += 1; + } + } + const spatial_axes = sd[0..len_sd]; + var res = input; + for (spatial_axes) |ax| { + const n = out_shape.dim(ax); + const ratio = stdx.math.divFloat(f32, input.dim(ax), n); + const offsets = Tensor.arange(.{ .end = n }, .f32).addConstant(0.5).scale(ratio).floor().convert(.i32); + res = ops.gather(res, &.{ax}, &.{offsets}, .{ .indices_are_sorted = true }); + } + return res; +} + +test nearest { + const platform = zml.testing.env(); + + // 3D Tensor (basic) + { + const input_3d_basic: zml.Tensor = .init(Shape.init(.{ 1, 1, 2 }, .i32)); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_3d_basic, .{ .scale_factor = &.{3}, .mode = .nearest } }, platform); + defer exe.deinit(); + + var input_3d_basic_buffer: zml.Buffer = try .fromBytes(platform, input_3d_basic.shape(), std.mem.sliceAsBytes(&[1][1][2]i32{.{.{ 1, 2 }}}), std.testing.io); + defer input_3d_basic_buffer.deinit(); + + const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_3d_basic_buffer}); + defer result.deinit(); + + try std.testing.expectEqualSlices(i64, &.{ 1, 1, 6 }, result.shape().dims()); + const expected: zml.testing.Slice = .init(Shape.init(.{ 1, 1, 6 }, .i32), std.mem.sliceAsBytes(&[1][1][6]i32{.{.{ 1, 1, 1, 2, 2, 2 }}})); + try zml.testing.expectClose(std.testing.io, expected, result, 0); + } + + // 3D Tensor (advanced) + { + const input_3d_advanced: zml.Tensor = .init(Shape.init(.{ 2, 3, 4 }, .i32)); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_3d_advanced, .{ .scale_factor = &.{2}, .mode = .nearest } }, platform); + defer exe.deinit(); + + var input_3d_advanced_buffer: zml.Buffer = try .fromBytes(platform, input_3d_advanced.shape(), std.mem.sliceAsBytes(&[2][3][4]i32{ + .{ .{ 1, 2, 3, 4 }, .{ 5, 6, 7, 8 }, .{ 9, 10, 11, 12 } }, + .{ .{ 13, 14, 15, 16 }, .{ 17, 18, 19, 20 }, .{ 21, 22, 23, 24 } }, + }), std.testing.io); + defer input_3d_advanced_buffer.deinit(); + + const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_3d_advanced_buffer}); + defer result.deinit(); + + try std.testing.expectEqualSlices(i64, &.{ 2, 3, 8 }, result.shape().dims()); + const expected: zml.testing.Slice = .init(Shape.init(.{ 2, 3, 8 }, .i32), std.mem.sliceAsBytes(&[2][3][8]i32{ + .{ + .{ 1, 1, 2, 2, 3, 3, 4, 4 }, + .{ 5, 5, 6, 6, 7, 7, 8, 8 }, + .{ 9, 9, 10, 10, 11, 11, 12, 12 }, + }, + .{ + .{ 13, 13, 14, 14, 15, 15, 16, 16 }, + .{ 17, 17, 18, 18, 19, 19, 20, 20 }, + .{ 21, 21, 22, 22, 23, 23, 24, 24 }, + }, + })); + try zml.testing.expectClose(std.testing.io, expected, result, 0); + } + + // 4D Tensor (basic) + { + const input_4d_basic: zml.Tensor = .init(Shape.init(.{ 1, 1, 2, 2 }, .i32)); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_4d_basic, .{ .scale_factor = &.{ 3, 3 }, .mode = .nearest } }, platform); + defer exe.deinit(); + + var input_4d_basic_buffer: zml.Buffer = try .fromBytes(platform, input_4d_basic.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 }), std.testing.io); + defer input_4d_basic_buffer.deinit(); + + const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_4d_basic_buffer}); + defer result.deinit(); + + try std.testing.expectEqualSlices(i64, &.{ 1, 1, 6, 6 }, result.shape().dims()); + const expected: zml.testing.Slice = .init(Shape.init(.{ 1, 1, 6, 6 }, .i32), std.mem.sliceAsBytes(&[1][1][6][6]i32{.{.{ + .{ 1, 1, 1, 2, 2, 2 }, + .{ 1, 1, 1, 2, 2, 2 }, + .{ 1, 1, 1, 2, 2, 2 }, + .{ 3, 3, 3, 4, 4, 4 }, + .{ 3, 3, 3, 4, 4, 4 }, + .{ 3, 3, 3, 4, 4, 4 }, + }}})); + try zml.testing.expectClose(std.testing.io, expected, result, 0); + } + // 4D Tensor (advanced) + { + const input_4d_advanced: zml.Tensor = .init(Shape.init(.{ 2, 2, 2, 2 }, .i32)); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_4d_advanced, .{ .scale_factor = &.{ 2, 2 }, .mode = .nearest } }, platform); + defer exe.deinit(); + + var input_4d_advanced_buffer: zml.Buffer = try .fromBytes(platform, input_4d_advanced.shape(), std.mem.sliceAsBytes(&[2][2][2][2]i32{ .{ + .{ .{ 1, 2 }, .{ 3, 4 } }, + .{ .{ 5, 6 }, .{ 7, 8 } }, + }, .{ + .{ .{ 9, 10 }, .{ 11, 12 } }, + .{ .{ 13, 14 }, .{ 15, 16 } }, + } }), std.testing.io); + defer input_4d_advanced_buffer.deinit(); + + const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_4d_advanced_buffer}); + defer result.deinit(); + + try std.testing.expectEqualSlices(i64, &.{ 2, 2, 4, 4 }, result.shape().dims()); + const expected: zml.testing.Slice = .init(Shape.init(.{ 2, 2, 4, 4 }, .i32), std.mem.sliceAsBytes(&[2][2][4][4]i32{ + .{ + .{ + .{ 1, 1, 2, 2 }, + .{ 1, 1, 2, 2 }, + .{ 3, 3, 4, 4 }, + .{ 3, 3, 4, 4 }, + }, + .{ + .{ 5, 5, 6, 6 }, + .{ 5, 5, 6, 6 }, + .{ 7, 7, 8, 8 }, + .{ 7, 7, 8, 8 }, + }, + }, + .{ + .{ + .{ 9, 9, 10, 10 }, + .{ 9, 9, 10, 10 }, + .{ 11, 11, 12, 12 }, + .{ 11, 11, 12, 12 }, + }, + .{ + .{ 13, 13, 14, 14 }, + .{ 13, 13, 14, 14 }, + .{ 15, 15, 16, 16 }, + .{ 15, 15, 16, 16 }, + }, + }, + })); + try zml.testing.expectClose(std.testing.io, expected, result, 0); + } + // 5D Tensor (basic) + { + const input_5d: zml.Tensor = .init(Shape.init(.{ 1, 1, 1, 2, 2 }, .i32)); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_5d, .{ .scale_factor = &.{2}, .mode = .nearest } }, platform); + defer exe.deinit(); + + var input_5d_buffer: zml.Buffer = try .fromBytes(platform, input_5d.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 }), std.testing.io); + defer input_5d_buffer.deinit(); + + const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_5d_buffer}); + defer result.deinit(); + + try std.testing.expectEqualSlices(i64, &.{ 1, 1, 2, 4, 4 }, result.shape().dims()); + const expected: zml.testing.Slice = .init(Shape.init(.{ 1, 1, 2, 4, 4 }, .i32), std.mem.sliceAsBytes(&[1][1][2][4][4]i32{ + .{ + .{ + .{ + .{ 1, 1, 2, 2 }, + .{ 1, 1, 2, 2 }, + .{ 3, 3, 4, 4 }, + .{ 3, 3, 4, 4 }, + }, + .{ + .{ 1, 1, 2, 2 }, + .{ 1, 1, 2, 2 }, + .{ 3, 3, 4, 4 }, + .{ 3, 3, 4, 4 }, + }, + }, + }, + })); + try zml.testing.expectClose(std.testing.io, expected, result, 0); + } +} + +pub const ResizeOpts = struct { + /// scalar tensor containing the original dimension of the image. + /// It can be different from the image shape, + /// if the image has been padded. + /// This allows to compile one module that handle different input image sizes. + original_len: ?Tensor = null, + + /// Internal precision to do the interpolation. + /// Result will always use the same dtype than the original. + /// If not set, will use the image dtype, unless it's an integer type, in which case f32 will be used. + precision: ?zml.DataType = null, +}; + +pub fn resizeBilinear(image: Tensor, resized_axes: anytype, opt: ResizeOpts) Tensor { + const new_size, const tags_ = Shape.parseStruct(u63, resized_axes); + var out = image; + for (new_size.constSlice(), tags_.constSlice()) |d, t| { + const ax = image.shape().axis(t); + const child_opt: ResizeOpts = .{ + .original_len = if (opt.original_len) |o| o.choose1d(0, ax) else null, + }; + out = resizeLinear1d(out, ax, d, child_opt); + } + return out; +} + +test resizeBilinear { + // Only test shapes + var comp = zml.module.CompilationContext.init(std.testing.allocator); + defer comp.deinit(); + comp.activate(); + defer comp.deactivate(); + + const block = @import("mlir").Block.init(&.{}, &.{}); + comp.pushBlock(block); + defer comp.popBlock(); + + inline for (.{ + .{ .{ .a = 10, .b = 10 }, .{ .a = 20 }, .{ .a = 20, .b = 10 } }, + .{ .{ .a = 10, .b = 10 }, .{ .b = 5 }, .{ .a = 10, .b = 5 } }, + .{ .{ .a = 10, .b = 10 }, .{ .a = 20, .b = 5 }, .{ .a = 20, .b = 5 } }, + }) |testcase| { + const x_shape, const resizing, const res_shape = testcase; + const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); + const y = resizeBilinear(x, resizing, .{}); + try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); + try std.testing.expect(y.value().owner().verify()); + } +} + +pub fn resizeLinear1d(image: Tensor, axis: i8, new_len: u63, opt: ResizeOpts) Tensor { + const ax = image.axis(axis); + const res_shape = image.shape().set(ax, new_len); + + const dtype = opt.precision orelse if (image.dtype().class() == .integer) .f32 else image.dtype(); + const og_len = opt.original_len orelse Tensor.scalar(image.dim(ax), dtype); + const ratio = og_len.convert(dtype).scale(stdx.math.divFloat(f32, 1, new_len)); + const scaled = Tensor.arange(.{ .end = new_len }, dtype).mul(ratio); + const left = scaled.floor(); + const right = left.addConstant(1); + + // TODO: check that two gather isn't too bad perf wise. + // Normally we should use gatherSlices to collect the values 2 by 2, + // but gatherSlices messes up with the order of axes. + const left_val = ops.gather(image, &.{ax}, &.{left.convert(.i32)}, .{ .indices_are_sorted = true }).convert(dtype); + const right_val = ops.gather(image, &.{ax}, &.{right.convert(.i32)}, .{ .indices_are_sorted = true }).convert(dtype); + + const left_weight = right.sub(scaled).broadcast(res_shape, &.{ax}); + const right_weight = scaled.sub(left).broadcast(res_shape, &.{ax}); + + const res = left_val.mul(left_weight).add(right_val.mul(right_weight)); + return res.convert(image.dtype()).withTags(image.shape().tags()); +} + +/// Bicubic interpolation of the given image. +/// Warning as of May 2024 the cpu backend don't optimize this very well +/// and is not able to merge the weighting with the gather, +/// leading to 20x slow down compared to STB implementation. +pub fn resizeBicubic(image: Tensor, resized_axes: anytype, opt: ResizeOpts) Tensor { + const new_size, const tags_ = Shape.parseStruct(u63, resized_axes); + var out = image; + for (new_size.constSlice(), tags_.constSlice()) |d, t| { + const ax = image.shape().axis(t); + const child_opt: ResizeOpts = .{ + .original_len = if (opt.original_len) |o| o.choose1d(0, ax) else null, + }; + out = resizeCubic1d(out, ax, d, child_opt); + } + return out; +} + +test resizeBicubic { + // Only test shapes + var comp = zml.module.CompilationContext.init(std.testing.allocator); + defer comp.deinit(); + comp.activate(); + defer comp.deactivate(); + + const block = @import("mlir").Block.init(&.{}, &.{}); + comp.pushBlock(block); + defer comp.popBlock(); + + inline for (.{ + .{ .{ .a = 10, .b = 10 }, .{ .a = 20 }, .{ .a = 20, .b = 10 } }, + .{ .{ .a = 10, .b = 10 }, .{ .b = 5 }, .{ .a = 10, .b = 5 } }, + .{ .{ .a = 10, .b = 10 }, .{ .a = 20, .b = 5 }, .{ .a = 20, .b = 5 } }, + }) |testcase| { + const x_shape, const resizing, const res_shape = testcase; + const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); + const y = resizeBicubic(x, resizing, .{}); + try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); + try std.testing.expect(y.value().owner().verify()); + } +} + +pub fn resizeCubic1d(image: Tensor, axis: i8, new_len: u63, opt: ResizeOpts) Tensor { + // Extract neighboring pixels from the image. + const dtype = opt.precision orelse if (image.dtype().class() == .integer) .f32 else image.dtype(); + const og_len = opt.original_len orelse Tensor.scalar(image.dim(axis), dtype); + + const ratio = og_len.convert(dtype).scale(stdx.math.divFloat(f32, 1, new_len)); + const scaled = Tensor.arange(.{ .end = new_len }, dtype).mul(ratio); + const t = scaled.sub(scaled.floor()); + const pos = Tensor.stack(&.{ + Tensor.constant(dtype.one()).broad(t.shape()), + t, + t.mul(t), + t.powByConst(3), + }, .last, ._interpolated); + + std.debug.assert(pos.dim(0) == new_len); + std.debug.assert(pos.dim(1) == 4); + + const neighbors = scaled.floor().addConstant(-1).convert(.i32).maximum(Tensor.scalar(0, .i32)); + + const values = image.renameAxis(axis, ._neighbors).gatherSlices( + Shape.init(.{ ._neighbors = 4 }, image.dtype()), + neighbors.appendAxes(.{.coord}), + .{ .indices_are_sorted = true }, + ).convert(dtype); + + const weights_: [4][4]f32 = .{ + .{ 0, 1, 0, 0 }, + .{ -0.5, 0, 0.5, 0 }, + .{ 1, -2.5, 2, -0.5 }, + .{ -0.5, 1.5, -1.5, 0.5 }, + }; + const weights = zml.Tensor.constantTensor(Shape.init(.{ 4, 4 }, .f32), std.mem.sliceAsBytes(&weights_)).convert(dtype).withTags(.{ ._interpolated, ._neighbors }); + + // actually do the interpolation. + // Note: ideally this matmul should be inlined with the gather, but that's currently not the case. + // TODO: not being able to use dot here is a bit annoying. + var res = values.dotGeneral(weights, &.{.{ values.axis(._neighbors), weights.axis(._neighbors) }}, &.{}); + res = pos.dotGeneral(res, &.{.{ pos.axis(._interpolated), res.axis(._interpolated) }}, &.{.{ 0, 0 }}); + + // the current axis is outputted in first position because it's a batching dim, put it back in place. + if (axis != 0) { + res = res.swapAxes(0, axis); + } + + // verify the shape + const res_shape = image.shape().set(axis, new_len); + // log.debug("resizeCubic1d: ({}, {}, {}, {}) -> {}", .{ image, axis, new_len, opt, res }); + std.debug.assert(std.mem.eql(i64, res_shape.dims(), res.dims())); + return res.convert(image.dtype()).withTags(image.shape()); +} + +/// Return causal attention masks for the given shape. +/// The last dimensions are +pub fn causalAttnMask( + attn_shape_: anytype, + dtype: DataType, + attn_window_len: ?u32, +) Tensor { + const attn_shape = Shape.init(attn_shape_, dtype); + stdx.debug.assert(attn_shape.rank() == 2, "causalAttnMask({f}) shape need to be exactly 2 axes", .{attn_shape}); + const qlen = attn_shape.dim(-2); + const q_idx = Tensor.iota(attn_shape, -2); + const klen = attn_shape.dim(-1); + const k_idx = Tensor.iota(attn_shape, -1); + + // all elements > main diagonal must be 0 + // (q_idx - window_len < k_idx <= q_idx) + var mask = k_idx.cmp(.LE, q_idx); + if (attn_window_len) |window_len| { + if (qlen >= window_len or klen >= window_len) { + const window_mask = q_idx.cmp(.LT, k_idx.addConstant(window_len)); + mask = mask.logical(.AND, window_mask); + } + } + + if (dtype.isFloat()) { + const zeros = Tensor.constant(mask.shape(), dtype.zero()); + const minus_inf = Tensor.constant(mask.shape(), dtype.minValue()); + mask = Tensor.select(mask, zeros, minus_inf); + } else { + mask = mask.convert(dtype); + } + + return mask; +} + +pub const SdpaOpts = struct { + attn_mask: ?Tensor = null, + scale: ?Tensor = null, + softmax_bias: ?Tensor = null, + allow_cudnn: bool = true, + // TODO: put a callback instead of all this field, + // so that +}; + +/// Scaled dot product attention. +/// +/// **Shapes**: +/// - q, result: .{ .h, .q, .hd } +/// - k, v: .{ .h, .k, .hd } +/// +/// Where: +/// - .h is the number of head +/// - .q is the number of queries +/// - .k is the number of keys +/// - .hd is the head dimension +/// +/// .h is allowed to differ from queries and keys as long as the key heads +/// can be repeated to match query heads. +pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { + var q, var k, var v = .{ q_, k_, v_ }; + + const err_template = "sdpa(q: {f}, k: {f}, v: {f}, attn: {?f}) is invalid ! "; + const err_args = .{ q, k, v, opts.attn_mask }; + stdx.debug.assert(q.shape().hasTags(.{ .h, .q, .hd }), err_template ++ "q is missing tags {{.h, .q, .hd}}", err_args); + stdx.debug.assert(k.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "k is missing tags {{.h, .k, .hd}}", err_args); + stdx.debug.assert(v.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "v is missing tags {{.h, .k, .hd}}", err_args); + + // TODO(Corentin): Re-enable that + //if (opts.allow_cudnn and cuda.canUseCudnnSdpa(q.shape()) and opts.softmax_bias == null) { + // return cuda.sdpa(q, k, v, opts); + //} + + // Handle different numbers of head by splitting q heads. + // This is a bit error prone in the sense that it depends of the layout of q heads. + // This is the Llama convention though. + q = q.splitAxis(.h, .{ .h = k.dim(.h), .hq = .auto }); + const attn_mask = if (opts.attn_mask) |m| m else null; + + const dims = collectDims(.{ .h, .q, .k, .hd }, &.{ q, k, v, attn_mask }, .strict) catch { + stdx.debug.panic(err_template ++ "Inputs have incompatible shapes.", err_args); + }; + const sqrtHeadDim: f32 = 1.0 / std.math.sqrt(@as(f32, @floatFromInt(dims.hd))); + const head_scaling = if (opts.scale) |s| s else Tensor.scalar(sqrtHeadDim, k.dtype()); + k = k.mul(head_scaling.convert(k.dtype())); + + var attn_weights = q.dot(k, .hd); + // log.debug("attn_weights : {f}, attn_mask : {?f}", .{ attn_weights, attn_mask }); + if (attn_mask) |mask| attn_weights = attn_weights.add(mask.broad(attn_weights.shape())); + attn_weights = attn_weights.convert(.f32); + attn_weights = if (opts.softmax_bias) |softmax_bias| attn: { + // The split is needed because we also split q ourselves. + // TODO: consider letting the user do that. + const bias = softmax_bias.splitAxis(.h, .{ .h = k.dim(.h), .hq = .auto }); + break :attn attn_weights.convert(.f32).softmaxBiased(.k, bias).convert(q.dtype()); + } else attn_weights.convert(.f32).softmax(.k).convert(q.dtype()); + + var attn = attn_weights.dot(v, .k); + return attn.transpose(q.shape()).merge(.{ .h = .{ .h, .hq } }); +} + +/// Options controlling generation. The default values correspond to greedy decoding. +pub const SamplingStrategy = struct { + topk: u32 = 1, + temperature: f32 = 1.0, +}; + +/// Given the output of the last layer of a LM with a `.voc` axis, +/// Compute indices for the next tokens, following the given sampling strategy. +/// Returns an integer tensor with a shape similar to the input, but without the .voc axis. +pub fn sampleTokens(activations: Tensor, opts: SamplingStrategy, rng: Tensor.Rng) struct { Tensor, Tensor.Rng } { + if (opts.topk <= 1) { + const next_tokens = activations.argMax(.voc).indices.squeeze(.voc); + return .{ next_tokens, rng }; + } + + const topk = activations.topK(.{ .topk = .voc }, opts.topk, .{}); + // After the topk, we don't have .voc values, anymore, only .topk. + var x = topk.values; + if (opts.temperature != 1.0) { + x = x.scale(1 / opts.temperature); + } + + // Gumbel reparametrization trick: + // Adding gumbel noise and taking the argmax is equivalent + // to sampling from the categorical distribution produced by the softmax. + // https://en.wikipedia.org/wiki/Gumbel_distribution#Gumbel_reparametrization_tricks + const next_rng, const gumbel_noise = rng.gumbel(x.shape()); + x = x.add(gumbel_noise); + const topk_idx = x.argMax(.topk).indices; + + // topk_idx is indices into topk.values ! so in the range [0, topk] + // Convert for the original indices from the full [0, voc] range. + const next_tokens = topk.indices.gather(.{ .topk = topk_idx.squeeze(.topk) }, .{}); + // log.debug("sampleTokens({}) -> {} -> {} -> {}", .{ activations, topk.indices, topk_idx, next_tokens }); + return .{ next_tokens, next_rng }; +} + +test sampleTokens { + const platform = zml.testing.env(); + + const rng: zml.Tensor.Rng = .init(); + const activations: zml.Tensor = .init(Shape.init(.{ .voc = 4 }, .f32)); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, sampleTokens, .{ activations, .{ .topk = 4, .temperature = 2.0 }, rng }, platform); + defer exe.deinit(); + + var rng_buffer = try zml.Tensor.Rng.initBuffer(platform, 0xdeadbeef, std.testing.io); + defer rng_buffer._state.deinit(); + + const inf = std.math.inf(f32); + + inline for (.{ + .{ [_]f32{ inf, 3.0, 2.0, 1.0 }, 0 }, + .{ [_]f32{ -inf, 3.0, -inf, -inf }, 1 }, + .{ [_]f32{ 3.0, 2, inf, inf }, 2 }, + }) |logits_expected| { + const logits, const expected: i32 = logits_expected; + + const activations_buffer: zml.Buffer = try .fromBytes(platform, activations.shape(), std.mem.sliceAsBytes(&logits), std.testing.io); + defer activations_buffer.deinit(); + + var sampled, rng_buffer = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, sampleTokens, .{ activations_buffer, rng_buffer }); + defer sampled.deinit(); + + try zml.testing.expectEqual(expected, try sampled.getValue(i32, std.testing.io)); + } +} + +pub const DynamicSamplingStrategy = struct { + max_top_k: u32, + top_k: Tensor, + temperature: Tensor, + top_p: Tensor, + min_p: Tensor, + + pub const Opts = struct { + top_k: u32, + temperature: f32 = 1.0, + top_p: f32 = 1.0, + min_p: f32 = 0.0, + }; + + pub fn init(dtype: DataType, max_top_k: u32) DynamicSamplingStrategy { + const scalar_float_shape = Shape.init(.{}, dtype); + const scalar_i32_shape = Shape.init(.{}, .i32); + return .{ + .max_top_k = max_top_k, + .top_k = .init(scalar_i32_shape), + .temperature = .init(scalar_float_shape), + .top_p = .init(scalar_float_shape), + .min_p = .init(scalar_float_shape), + }; + } + + pub fn makeBuffers( + platform: zml.Platform, + io: std.Io, + dtype: zml.DataType, + opts: Opts, + ) !zml.Bufferized(DynamicSamplingStrategy) { + return .{ + .top_k = try zml.Buffer.scalar(platform, opts.top_k, .i32, io), + .temperature = try zml.Buffer.scalar(platform, opts.temperature, dtype, io), + .top_p = try zml.Buffer.scalar(platform, opts.top_p, dtype, io), + .min_p = try zml.Buffer.scalar(platform, opts.min_p, dtype, io), + }; + } + + pub fn deinitBuffers(self: *zml.Bufferized(DynamicSamplingStrategy)) void { + self.top_k.deinit(); + self.temperature.deinit(); + self.top_p.deinit(); + self.min_p.deinit(); + } +}; + +/// Given the output of the last layer of a LM with a `.voc` axis, +/// Compute indices for the next tokens, following the given sampling strategy. +/// The dynamic sampling strategy is more expressive but top_p requires computing the softmax. +/// +/// Options are: +/// +/// * top_k: only sample among the k top scoring tokens, +/// * max_top_k: limit a compilation time what is the max possible runtime value for top_k, saving memory and compute by not having to fully sort the tokens. +/// * top_p: only sample among top scoring tokens whose probabilities sum up to top_p +/// * min_p: drop tokens whose probabilities are lower than a ratio of the most likely token +pub fn sampleTokensDynamic(logits: Tensor, opts: DynamicSamplingStrategy, rng: Tensor.Rng) struct { Tensor, Tensor.Rng } { + var x, const topk_indices = fixupLogits(logits, opts); + + // the rest is similar to sampleTokens + const next_rng, const gumbel_noise = rng.gumbel(x.shape()); + x = x.add(gumbel_noise); + + const topk_idx = x.argMax(.topk).indices; + const next_tokens = topk_indices.gather(.{ .voc = topk_idx.squeeze(.topk) }, .{}); + return .{ next_tokens, next_rng }; +} + +fn fixupLogits(logits: Tensor, opts: DynamicSamplingStrategy) [2]Tensor { + const min_inf = Tensor.constant(logits.dtype().minValue()); + + // First reduce the vocab size to a reasonable sub set of candidate. + const full_topk = if (opts.max_top_k > 0) + logits.topK(.{ .voc = .voc }, opts.max_top_k, .{ .descending = true }) + else + logits.sort(.voc, .{ .descending = true }); + + // After the topk, we don't have .voc indices, anymore, only topk. + var x = full_topk.values.rename(.{ .voc = .topk }); + // mask values above the dynamic top_k + x = Tensor.iota(x.shape(), .topk).cmp(.GE, opts.top_k).select(min_inf, x); + x = x.mul(opts.temperature); + + // if there are high values in x, softmax can overflow and will create nans in full probs + // this propagate to probs_sum and probs_max. + const probs = x.softmax(.topk); + const probs_sum = probs.cumulativeSum(.topk); + const probs_max = probs.slice1d(.topk, .{ .start = 0, .end = 1 }); + + const top_p = opts.top_p.broad(x.shape()); + const min_p = probs_max.mul(opts.min_p).broad(x.shape()); + + // * if first candidate has very high prob, then probs_sum is always greater than top_p and candidate is full false + // * if first candidate score is even bigger, the probs become Nan because of the softmax, + // then cmp is is full false, and candidate is full false too. + const candidate = probs_sum.cmp(.LE, top_p).logical(.AND, probs.cmp(.GE, min_p)); + // * so we explicitly always accept first candidate. + const first_token = Tensor.iota(x.shape(), .topk).cmp(.EQ, Tensor.scalar(0, .i32)); + x = candidate.logical(.OR, first_token).select(x, min_inf); + + return .{ x, full_topk.indices }; +} + +test sampleTokensDynamic { + const platform = zml.testing.env(); + + const ___ = -std.math.inf(f32); + const logits_data = [_]f32{ @log(2.0), @log(1.0), @log(4.0), @log(3.0) }; + const top_k_indices = [_]i32{ 2, 3, 0, 1 }; + + const logits: zml.Tensor = .init(zml.Shape.init(.{ .voc = logits_data.len }, .f32)); + const dynamic_sampling_strategy = DynamicSamplingStrategy.init(.f32, 0); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, fixupLogits, .{ logits, dynamic_sampling_strategy }, platform); + defer exe.deinit(); + + var logits_buffer: zml.Buffer = try .fromBytes(platform, logits.shape(), std.mem.sliceAsBytes(&logits_data), std.testing.io); + defer logits_buffer.deinit(); + + const Args = struct { DynamicSamplingStrategy.Opts, [4]f32 }; + inline for ([_]Args{ + // top_k == logits.len -> just sort the input + .{ .{ .top_k = 4 }, [_]f32{ @log(4.0), @log(3.0), @log(2.0), @log(1.0) } }, + .{ .{ .top_k = 2 }, [_]f32{ @log(4.0), @log(3.0), ___, ___ } }, + .{ .{ .top_k = 2, .temperature = 0.1 }, [_]f32{ @log(4.0) * 0.1, @log(3.0) * 0.1, ___, ___ } }, + // top_k == logits.len and small top_p -> make sure at least one is returned + .{ .{ .top_k = 4, .top_p = 0.1 }, [_]f32{ @log(4.0), ___, ___, ___ } }, + .{ .{ .top_k = 4, .top_p = 0.701 }, [_]f32{ @log(4.0), @log(3.0), ___, ___ } }, + .{ .{ .top_k = 4, .top_p = 0.901 }, [_]f32{ @log(4.0), @log(3.0), @log(2.0), ___ } }, + // Here top_p is computed on the top 3 items, so 0.701 isn't enougth anymore to allow @log(3.0) + .{ .{ .top_k = 3, .top_p = 0.701 }, [_]f32{ @log(4.0), ___, ___, ___ } }, + // Here top_p allows the first 3 results, but min_p only accepts the first two. + .{ .{ .top_k = 4, .top_p = 0.901, .min_p = 0.6 }, [_]f32{ @log(4.0), @log(3.0), ___, ___ } }, + }) |args_expected| { + const args, const expected = args_expected; + var dynamic_sampling_strategy_buffers = try DynamicSamplingStrategy.makeBuffers(platform, std.testing.io, .f32, args); + defer DynamicSamplingStrategy.deinitBuffers(&dynamic_sampling_strategy_buffers); + var new_logits, var indices = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, fixupLogits, .{ logits_buffer, dynamic_sampling_strategy_buffers }); + defer new_logits.deinit(); + defer indices.deinit(); + try std.testing.expectEqual(top_k_indices, try indices.getValue(@TypeOf(top_k_indices), std.testing.io)); + try zml.testing.expectEqual(expected, try new_logits.getValue(@TypeOf(expected), std.testing.io)); + } + + { + // Similar but use bf16, and uses infinity to trigger nans after the softmax. + const bf16 = zml.floats.BFloat16; + + const logits_bf16: zml.Tensor = .init(Shape.init(.{ .voc = logits_data.len }, .bf16)); + const dynamic_sampling_strategy_bf16 = DynamicSamplingStrategy.init(.bf16, 0); + var exe_bf16 = try zml.module.compile(std.testing.allocator, std.testing.io, fixupLogits, .{ logits_bf16, dynamic_sampling_strategy_bf16 }, platform); + defer exe_bf16.deinit(); + + const boost = bf16.inf; + const nerf = bf16.minus_inf; + + var logits_bf16_buffer: zml.Buffer = try .fromBytes(platform, logits_bf16.shape(), std.mem.sliceAsBytes(&[4]bf16{ boost, boost, bf16.fromF32(2), nerf }), std.testing.io); + defer logits_bf16_buffer.deinit(); + var dynamic_sampling_strategy_bf16_buffers = try DynamicSamplingStrategy.makeBuffers(platform, std.testing.io, .bf16, .{ .top_k = 4, .top_p = 0.9, .min_p = 0.1 }); + defer DynamicSamplingStrategy.deinitBuffers(&dynamic_sampling_strategy_bf16_buffers); + + var new_logits, var indices = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_bf16, fixupLogits, .{ logits_bf16_buffer, dynamic_sampling_strategy_bf16_buffers }); + + try std.testing.expectEqual([_]i32{ 0, 1, 2, 3 }, try indices.getValue([4]i32, std.testing.io)); + try zml.testing.expectEqual([_]bf16{ boost, nerf, nerf, nerf }, try new_logits.getValue([4]bf16, std.testing.io)); + } +} + +const ShapeError = error{ DimMismatch, NotFound }; +const NOT_SET: i64 = -2; +const DIM_MISMATCH: i64 = -1; + +/// Collect the given dimensions inside a struct containing tagged tensors. +pub fn collectDims( + comptime dims: anytype, + v: anytype, + comptime mode: enum { strict, allow_extra_dims, ignore_errors }, +) ShapeError!ShapeStruct(dims) { + const LocalContext = struct { + res: ShapeStruct(dims), + mode: @TypeOf(mode), + }; + + var context = LocalContext{ + .res = undefined, + .mode = mode, + }; + @memset(std.mem.bytesAsSlice(i64, std.mem.asBytes(&context.res)), NOT_SET); + + meta.visit((struct { + fn cb(ctx: *LocalContext, shape: *const Shape) void { + inline for (dims) |a| { + if (shape.hasTag(a)) |axis| { + const dim = shape.dim(axis); + + const expected_dim = &@field(ctx.res, @tagName(a)); + if (expected_dim.* == NOT_SET) { + expected_dim.* = dim; + } else if (expected_dim.* == DIM_MISMATCH) { + // this axis has already been reported as invalid. + } else if (dim != expected_dim.*) { + if (mode != .ignore_errors) { + log.warn("Dim mismatch ! Axis {0s}={1d} but received a new tensor where {0s}={2d}", .{ @tagName(a), expected_dim.*, dim }); + } + expected_dim.* = DIM_MISMATCH; + } + } + } + } + }).cb, &context, v); + + if (context.mode != .ignore_errors) { + inline for (shapeToDims(context.res), dims) |dim, dim_tag| { + if (dim == NOT_SET) { + log.warn("Axis not found: {s}", .{@tagName(dim_tag)}); + return error.NotFound; + } + if (dim == DIM_MISMATCH) return error.DimMismatch; + } + } + return context.res; +} + +fn shapeToDims(shape: anytype) [@divExact(@sizeOf(@TypeOf(shape)), @sizeOf(i64))]i64 { + return @bitCast(shape); +} + +test collectDims { + const Model = struct { + x: Shape, + y: Shape, + bias: Shape, + }; + + { + var model: Model = .{ + .x = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), + .y = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), + .bias = Shape.init(.{5}, .f32).withTags(.{.d}), + }; + try zml.testing.expectEqual(collectDims(.{ .b, .d }, &model, .strict), .{ .b = 2, .d = 5 }); + } + { + var model: Model = .{ + .x = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), + .y = Shape.init(.{ 3, 5 }, .f32).withTags(.{ .b, .d }), + .bias = Shape.init(.{5}, .f32).withTags(.{.d}), + }; + try std.testing.expectEqual( + collectDims(.{ .b, .d }, &model, .strict), + error.DimMismatch, + ); + try zml.testing.expectEqual(collectDims(.{ .b, .d }, &model, .ignore_errors), .{ .b = DIM_MISMATCH, .d = 5 }); + } + { + var model: Model = .{ + .x = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), + .y = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), + .bias = Shape.init(.{5}, .f32).withTags(.{.d}), + }; + try std.testing.expectEqual(collectDims(.{ .b, .d, .c }, &model, .strict), error.NotFound); + try zml.testing.expectEqual(collectDims(.{ .b, .d, .c }, &model, .ignore_errors), .{ .b = 2, .d = 5, .c = NOT_SET }); + } + { + var model: Model = .{ + .x = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), + .y = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), + .bias = Shape.init(.{7}, .f32).withTags(.{.d}), + }; + try std.testing.expectEqual(collectDims(.{ .b, .d }, &model, .strict), error.DimMismatch); + try zml.testing.expectEqual(collectDims(.{ .b, .d }, &model, .ignore_errors), .{ .b = 2, .d = DIM_MISMATCH }); + } +} + +fn ShapeStruct(comptime dims: anytype) type { + const rank = dims.len; + @setEvalBranchQuota(rank + 5); + var struct_field_names: [rank][]const u8 = undefined; + var struct_field_types: [rank]type = undefined; + var struct_field_attrs: [rank]std.builtin.Type.StructField.Attributes = undefined; + const default: i64 = NOT_SET; + for (&struct_field_names, &struct_field_types, &struct_field_attrs, dims) |*field_name, *field_type, *field_attr, axis| { + field_name.* = @tagName(axis); + field_type.* = i64; + field_attr.* = .{ + .@"align" = @alignOf(i64), + .default_value_ptr = &default, + }; + } + + return @Struct(.@"extern", null, &struct_field_names, &struct_field_types, &struct_field_attrs); +} diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 95ee391a..041af3ef 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -13,6 +13,8 @@ const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const CompilationContext = @import("module.zig").CompilationContext; +const Bufferized = @import("zml.zig").Bufferized; +const Buffer = @import("buffer.zig").Buffer; pub const Tensor = struct { var current_id: std.atomic.Value(usize) = .{ .raw = 1 }; @@ -1947,7 +1949,7 @@ pub const Tensor = struct { mlirCtx(), &.{}, mlirx.Type.fromDType(mlirCtx(), val.dtype()), - val.constSlice(), + val.asBytes(), .unknown(mlirCtx()), ).appendTo(currentBlock()); return _result(.init(&.{}, val.dtype()), op.result(0)); diff --git a/zml2/testing.zig b/zml2/testing.zig index abd2fff2..909b36f7 100644 --- a/zml2/testing.zig +++ b/zml2/testing.zig @@ -230,3 +230,7 @@ fn center(slice: anytype, i: usize) @TypeOf(slice) { const end = @min(start + 2 * c, slice.len); return slice[start..end]; } + +pub inline fn expectEqual(expected: anytype, actual: @TypeOf(expected)) !void { + return std.testing.expectEqual(expected, actual); +} diff --git a/zml2/zml.zig b/zml2/zml.zig index 52d048e7..edcf14f1 100644 --- a/zml2/zml.zig +++ b/zml2/zml.zig @@ -22,6 +22,7 @@ pub const module = @import("module.zig"); pub const exe = @import("exe.zig"); pub const meta = @import("meta.zig"); pub const mlir = @import("mlirx.zig"); +pub const nn = @import("nn.zig"); pub const shape = @import("shape.zig"); pub const Shape = shape.Shape; pub const tensor = @import("tensor.zig"); From 1574931db2c76173a5a518581e4f3d33327e6bf1 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 11 Dec 2025 13:41:59 +0100 Subject: [PATCH 117/881] Removed useless refAllDecls + reordered imports --- zml2/buffer.zig | 10 ++-------- zml2/dtype.zig | 4 ---- zml2/exe.zig | 8 ++++---- zml2/floats.zig | 4 ---- zml2/meta.zig | 4 ---- zml2/module.zig | 11 +++++------ zml2/ops.zig | 14 +++++++------- zml2/shape.zig | 2 +- zml2/tensor.zig | 22 +++++++++++----------- zml2/zml.zig | 24 ++++++++++++------------ 10 files changed, 42 insertions(+), 61 deletions(-) diff --git a/zml2/buffer.zig b/zml2/buffer.zig index 906b740e..55627b3f 100644 --- a/zml2/buffer.zig +++ b/zml2/buffer.zig @@ -2,18 +2,12 @@ const std = @import("std"); const stdx = @import("stdx"); -const pjrt = @import("pjrtx.zig"); - -const Shape = @import("shape.zig").Shape; const DataType = @import("dtype.zig").DataType; +const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; +const Shape = @import("shape.zig").Shape; const Target = @import("platform.zig").Target; -test { - std.testing.refAllDecls(@This()); - std.testing.refAllDecls(Buffer); -} - const log = std.log.scoped(.zml); /// Buffer is a multi-dimension array, whose memory is allocated on an accelerator. diff --git a/zml2/dtype.zig b/zml2/dtype.zig index fe80b711..8ec73c3d 100644 --- a/zml2/dtype.zig +++ b/zml2/dtype.zig @@ -5,10 +5,6 @@ const floats = @import("floats.zig"); const C64 = std.math.Complex(f32); const C128 = std.math.Complex(f64); -test { - std.testing.refAllDecls(@This()); -} - pub const DataType = enum(u8) { bool, // Note: the support of the float8 is a bit spotty, f8e4m3b11fnuz seems to be the most supported one on Cuda. diff --git a/zml2/exe.zig b/zml2/exe.zig index d31b9581..735b3213 100644 --- a/zml2/exe.zig +++ b/zml2/exe.zig @@ -1,12 +1,12 @@ const std = @import("std"); const stdx = @import("stdx"); -const pjrt = @import("pjrtx.zig"); -const meta = @import("meta.zig"); -const Shape = @import("shape.zig").Shape; -const Platform = @import("platform.zig").Platform; const Buffer = @import("buffer.zig").Buffer; +const meta = @import("meta.zig"); +const pjrt = @import("pjrtx.zig"); +const Platform = @import("platform.zig").Platform; +const Shape = @import("shape.zig").Shape; pub const Exe = struct { platform: Platform, diff --git a/zml2/floats.zig b/zml2/floats.zig index fe3dce87..538722cd 100644 --- a/zml2/floats.zig +++ b/zml2/floats.zig @@ -1,10 +1,6 @@ ///! Conversion utilities between different Floating point formats. const std = @import("std"); -//test { -// std.testing.refAllDecls(@This()); -//} - fn FloatConversionHelpers(Float: type) type { const info = @typeInfo(Float); const err_msg = "FloatHelpers expect a packed struct { mantissa: uXX, exponent: uXX, sign: u1}"; diff --git a/zml2/meta.zig b/zml2/meta.zig index be79a8a2..e26fa014 100644 --- a/zml2/meta.zig +++ b/zml2/meta.zig @@ -5,10 +5,6 @@ const stdx = @import("stdx"); const FnParam = stdx.meta.FnParam; const asSlice = stdx.meta.asSlice; -test { - std.testing.refAllDecls(@This()); -} - /// Visit a given type `T` and replace all fields containing `From` by fields containing `To`. pub fn MapType(From: type, To: type) type { return struct { diff --git a/zml2/module.zig b/zml2/module.zig index 678f0c7d..c93ed0c7 100644 --- a/zml2/module.zig +++ b/zml2/module.zig @@ -1,18 +1,17 @@ const std = @import("std"); -const stdx = @import("stdx"); -const pjrt = @import("pjrtx.zig"); -const mlir = @import("mlir"); const c = @import("c"); +const dialects = @import("mlir/dialects"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); const upb = @import("upb"); +const Exe = @import("exe.zig").Exe; const meta = @import("meta.zig"); -const dialects = @import("mlir/dialects"); const mlirx = @import("mlirx.zig"); - +const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; -const Exe = @import("exe.zig").Exe; const Tensor = @import("tensor.zig").Tensor; const log = std.log.scoped(.@"zml/module"); diff --git a/zml2/ops.zig b/zml2/ops.zig index e6a653a4..68ab154e 100644 --- a/zml2/ops.zig +++ b/zml2/ops.zig @@ -1,15 +1,15 @@ const std = @import("std"); -const stdx = @import("stdx"); -const meta = @import("meta.zig"); -const mlir = @import("mlir"); -const mlirx = @import("mlirx.zig"); const dialects = @import("mlir/dialects"); -const constants = @import("constants.zig"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); -const Tensor = @import("tensor.zig").Tensor; -const Shape = @import("shape.zig").Shape; const CompilationContext = @import("module.zig").CompilationContext; +const constants = @import("constants.zig"); +const meta = @import("meta.zig"); +const mlirx = @import("mlirx.zig"); +const Shape = @import("shape.zig").Shape; +const Tensor = @import("tensor.zig").Tensor; pub const ReduceArgs = struct { left: Tensor, diff --git a/zml2/shape.zig b/zml2/shape.zig index 661e634e..71968b9e 100644 --- a/zml2/shape.zig +++ b/zml2/shape.zig @@ -3,8 +3,8 @@ const testing = std.testing; const builtin = @import("builtin"); const stdx = @import("stdx"); -const constants = @import("constants.zig"); +const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const EnumLiteral = @TypeOf(.enum_literal); diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 041af3ef..c6b2f923 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -1,20 +1,20 @@ const std = @import("std"); const builtin = @import("builtin"); -const stdx = @import("stdx"); -const meta = @import("meta.zig"); - -const Shape = @import("shape.zig").Shape; -const mlir = @import("mlir"); -const mlirx = @import("mlirx.zig"); const dialects = @import("mlir/dialects"); -const ops = @import("ops.zig"); -const constants = @import("constants.zig"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); -const DataType = @import("dtype.zig").DataType; -const CompilationContext = @import("module.zig").CompilationContext; -const Bufferized = @import("zml.zig").Bufferized; const Buffer = @import("buffer.zig").Buffer; +const Bufferized = @import("zml.zig").Bufferized; +const CompilationContext = @import("module.zig").CompilationContext; +const constants = @import("constants.zig"); +const DataType = @import("dtype.zig").DataType; +const meta = @import("meta.zig"); +const mlirx = @import("mlirx.zig"); +const ops = @import("ops.zig"); +const Platform = @import("platform.zig").Platform; +const Shape = @import("shape.zig").Shape; pub const Tensor = struct { var current_id: std.atomic.Value(usize) = .{ .raw = 1 }; diff --git a/zml2/zml.zig b/zml2/zml.zig index edcf14f1..13998886 100644 --- a/zml2/zml.zig +++ b/zml2/zml.zig @@ -9,26 +9,26 @@ const std = @import("std"); const c = @import("c"); const runfiles = @import("runfiles"); +pub const Buffer = @import("buffer.zig").Buffer; +pub const constants = @import("constants.zig"); +pub const dtype = @import("dtype.zig"); +pub const Data = dtype.Data; +pub const DataType = dtype.DataType; +pub const exe = @import("exe.zig"); +pub const floats = @import("floats.zig"); +pub const meta = @import("meta.zig"); +pub const mlir = @import("mlirx.zig"); +pub const module = @import("module.zig"); +pub const nn = @import("nn.zig"); +pub const pjrt = @import("pjrtx.zig"); pub const platform = @import("platform.zig"); pub const Platform = platform.Platform; pub const Target = platform.Target; pub const CompilationOptions = platform.CompilationOptions; -pub const floats = @import("floats.zig"); -pub const dtype = @import("dtype.zig"); -pub const Data = dtype.Data; -pub const DataType = dtype.DataType; -pub const pjrt = @import("pjrtx.zig"); -pub const module = @import("module.zig"); -pub const exe = @import("exe.zig"); -pub const meta = @import("meta.zig"); -pub const mlir = @import("mlirx.zig"); -pub const nn = @import("nn.zig"); pub const shape = @import("shape.zig"); pub const Shape = shape.Shape; pub const tensor = @import("tensor.zig"); pub const Tensor = tensor.Tensor; -pub const constants = @import("constants.zig"); -pub const Buffer = @import("buffer.zig").Buffer; pub const testing = @import("testing.zig"); var runfiles_once = std.once(struct { From 742a8f120261809947c530aab993a294acffef99 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 11 Dec 2025 15:12:23 +0100 Subject: [PATCH 118/881] Introduced slice.zig with Slice and ConstSlice This allows to easily manipulate shaped slice on the host. --- zml2/BUILD.bazel | 1 + zml2/buffer.zig | 25 ++++++------ zml2/nn.zig | 13 +++--- zml2/slice.zig | 103 +++++++++++++++++++++++++++++++++++++++++++++++ zml2/tensor.zig | 43 +++++++++----------- zml2/testing.zig | 47 +++++---------------- zml2/zml.zig | 3 ++ 7 files changed, 156 insertions(+), 79 deletions(-) create mode 100644 zml2/slice.zig diff --git a/zml2/BUILD.bazel b/zml2/BUILD.bazel index 0c50c0fa..4731acb9 100644 --- a/zml2/BUILD.bazel +++ b/zml2/BUILD.bazel @@ -33,6 +33,7 @@ zig_library( "pjrtx.zig", "platform.zig", "shape.zig", + "slice.zig", "tensor.zig", "testing.zig", "zml.zig", diff --git a/zml2/buffer.zig b/zml2/buffer.zig index 55627b3f..bf85d1da 100644 --- a/zml2/buffer.zig +++ b/zml2/buffer.zig @@ -6,7 +6,9 @@ const DataType = @import("dtype.zig").DataType; const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; +const Slice = @import("slice.zig").Slice; const Target = @import("platform.zig").Target; +const testing = @import("testing.zig"); const log = std.log.scoped(.zml); @@ -151,29 +153,28 @@ pub const Buffer = struct { stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {f} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); var res: T = undefined; - try self.toHost(std.mem.asBytes(&res), io); + try self.toSlice(.init(self.shape(), std.mem.asBytes(&res)), io); return res; } - pub fn toHost(self: Buffer, buf: []u8, io: std.Io) !void { + /// Copies the content of the Buffer to the provided slice. + pub fn toSlice(self: Buffer, slice: Slice, io: std.Io) !void { //stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, buf); + const maybe_event = try self._shards.get(0).toHostBuffer(self._api, slice.data); if (maybe_event) |event| { try event.await(self._api, io); } } - /// Copies the content of the Buffer to the host. - /// The returned `HostBuffer` does own the memory. - pub fn toHostAlloc(self: Buffer, allocator: std.mem.Allocator, io: std.Io) ![]u8 { - //const output = try allocator.alignedAlloc(u8, .fromByteUnits(self._shape.dtype().alignOf()), self._shape.byteSize()); - //const output = try allocator.alignedAlloc(u8, .@"64", self._shape.byteSize()); - const output = try allocator.alloc(u8, self._shape.byteSize()); - errdefer allocator.free(output); + /// Copies the content of the Buffer to the provided slice. + /// The returned slice owns the memory. + pub fn toSliceAlloc(self: Buffer, allocator: std.mem.Allocator, io: std.Io) !Slice { + const slice = try Slice.alloc(allocator, self.shape()); + errdefer slice.free(allocator); - try self.toHost(output, io); + try self.toSlice(slice, io); - return output; + return slice; } }; diff --git a/zml2/nn.zig b/zml2/nn.zig index 9db31b98..184f7930 100644 --- a/zml2/nn.zig +++ b/zml2/nn.zig @@ -3,6 +3,7 @@ const std = @import("std"); const stdx = @import("stdx"); +const ConstSlice = @import("slice.zig").ConstSlice; const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); const ops = @import("ops.zig"); @@ -130,7 +131,7 @@ test normalizeL2 { var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, normalizeL2, .{input_buffer}); defer res.deinit(); - const expectation = zml.testing.Slice.init(input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6937, -0.7203, -0.9360, 0.3520 })); + const expectation: ConstSlice = .init(input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6937, -0.7203, -0.9360, 0.3520 })); try zml.testing.expectClose(std.testing.io, expectation, res, 1e-4); } @@ -600,7 +601,7 @@ test nearest { defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 1, 1, 6 }, result.shape().dims()); - const expected: zml.testing.Slice = .init(Shape.init(.{ 1, 1, 6 }, .i32), std.mem.sliceAsBytes(&[1][1][6]i32{.{.{ 1, 1, 1, 2, 2, 2 }}})); + const expected: ConstSlice = .init(Shape.init(.{ 1, 1, 6 }, .i32), std.mem.sliceAsBytes(&[1][1][6]i32{.{.{ 1, 1, 1, 2, 2, 2 }}})); try zml.testing.expectClose(std.testing.io, expected, result, 0); } @@ -620,7 +621,7 @@ test nearest { defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 2, 3, 8 }, result.shape().dims()); - const expected: zml.testing.Slice = .init(Shape.init(.{ 2, 3, 8 }, .i32), std.mem.sliceAsBytes(&[2][3][8]i32{ + const expected: ConstSlice = .init(Shape.init(.{ 2, 3, 8 }, .i32), std.mem.sliceAsBytes(&[2][3][8]i32{ .{ .{ 1, 1, 2, 2, 3, 3, 4, 4 }, .{ 5, 5, 6, 6, 7, 7, 8, 8 }, @@ -648,7 +649,7 @@ test nearest { defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 1, 1, 6, 6 }, result.shape().dims()); - const expected: zml.testing.Slice = .init(Shape.init(.{ 1, 1, 6, 6 }, .i32), std.mem.sliceAsBytes(&[1][1][6][6]i32{.{.{ + const expected: ConstSlice = .init(Shape.init(.{ 1, 1, 6, 6 }, .i32), std.mem.sliceAsBytes(&[1][1][6][6]i32{.{.{ .{ 1, 1, 1, 2, 2, 2 }, .{ 1, 1, 1, 2, 2, 2 }, .{ 1, 1, 1, 2, 2, 2 }, @@ -677,7 +678,7 @@ test nearest { defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 2, 2, 4, 4 }, result.shape().dims()); - const expected: zml.testing.Slice = .init(Shape.init(.{ 2, 2, 4, 4 }, .i32), std.mem.sliceAsBytes(&[2][2][4][4]i32{ + const expected: ConstSlice = .init(Shape.init(.{ 2, 2, 4, 4 }, .i32), std.mem.sliceAsBytes(&[2][2][4][4]i32{ .{ .{ .{ 1, 1, 2, 2 }, @@ -722,7 +723,7 @@ test nearest { defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 1, 1, 2, 4, 4 }, result.shape().dims()); - const expected: zml.testing.Slice = .init(Shape.init(.{ 1, 1, 2, 4, 4 }, .i32), std.mem.sliceAsBytes(&[1][1][2][4][4]i32{ + const expected: ConstSlice = .init(Shape.init(.{ 1, 1, 2, 4, 4 }, .i32), std.mem.sliceAsBytes(&[1][1][2][4][4]i32{ .{ .{ .{ diff --git a/zml2/slice.zig b/zml2/slice.zig new file mode 100644 index 00000000..4acafb7d --- /dev/null +++ b/zml2/slice.zig @@ -0,0 +1,103 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const DataType = @import("dtype.zig").DataType; +const Shape = @import("shape.zig").Shape; + +pub const Slice = struct { + data: []u8, + shape: Shape, + + pub fn init(shape: Shape, data: []u8) Slice { + return .{ .data = data, .shape = shape }; + } + + pub fn alloc(allocator: std.mem.Allocator, shape_: Shape) !Slice { + const size = shape_.byteSize(); + const bytes: []u8 = switch (shape_.dtype().alignOf()) { + 1 => try allocator.alignedAlloc(u8, .@"1", size), + 2 => try allocator.alignedAlloc(u8, .@"2", size), + 4 => try allocator.alignedAlloc(u8, .@"4", size), + 8 => try allocator.alignedAlloc(u8, .@"8", size), + 16 => try allocator.alignedAlloc(u8, .@"16", size), + 32 => try allocator.alignedAlloc(u8, .@"32", size), + 64 => try allocator.alignedAlloc(u8, .@"64", size), + else => |v| stdx.debug.panic("Unsupported alignment: {}", .{v}), + }; + + return .{ .data = bytes, .shape = shape_ }; + } + + pub fn free(slice: Slice, allocator: std.mem.Allocator) void { + slice.constSlice().free(allocator); + } + + pub fn constSlice(self: Slice) ConstSlice { + return .{ .data = self.data, .shape = self.shape }; + } + + pub fn dtype(self: Slice) DataType { + return self.shape.dtype(); + } + + pub fn items(self: Slice, comptime T: type) []T { + return @alignCast(std.mem.bytesAsSlice(T, self.data)); + } + + pub fn format( + self: @This(), + writer: *std.Io.Writer, + ) std.Io.Writer.Error!void { + return writer.print("{any}", .{self}); + } + + pub fn formatNumber(self: @This(), writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + _ = self; // autofix + _ = n; // autofix + return writer.print("TODO", .{}); + } +}; + +pub const ConstSlice = struct { + data: []const u8, + shape: Shape, + + pub fn init(shape: Shape, data: []const u8) ConstSlice { + return .{ .data = data, .shape = shape }; + } + + pub fn free(slice: ConstSlice, allocator: std.mem.Allocator) void { + switch (slice.shape.dtype().alignOf()) { + 1 => allocator.free(@as([]align(1) const u8, @alignCast(slice.data))), + 2 => allocator.free(@as([]align(2) const u8, @alignCast(slice.data))), + 4 => allocator.free(@as([]align(4) const u8, @alignCast(slice.data))), + 8 => allocator.free(@as([]align(8) const u8, @alignCast(slice.data))), + 16 => allocator.free(@as([]align(16) const u8, @alignCast(slice.data))), + 32 => allocator.free(@as([]align(32) const u8, @alignCast(slice.data))), + 64 => allocator.free(@as([]align(64) const u8, @alignCast(slice.data))), + else => |v| stdx.debug.panic("Unsupported alignment: {}", .{v}), + } + } + + pub fn dtype(self: ConstSlice) DataType { + return self.shape.dtype(); + } + + pub fn items(self: ConstSlice, comptime T: type) []const T { + return @alignCast(std.mem.bytesAsSlice(T, self.data)); + } + + pub fn format( + self: @This(), + writer: *std.Io.Writer, + ) std.Io.Writer.Error!void { + return writer.print("{any}", .{self}); + } + + pub fn formatNumber(self: @This(), writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + _ = self; // autofix + _ = n; // autofix + return writer.print("TODO", .{}); + } +}; diff --git a/zml2/tensor.zig b/zml2/tensor.zig index c6b2f923..64fc0000 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -9,6 +9,7 @@ const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("zml.zig").Bufferized; const CompilationContext = @import("module.zig").CompilationContext; const constants = @import("constants.zig"); +const ConstSlice = @import("slice.zig").ConstSlice; const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); @@ -292,7 +293,7 @@ pub const Tensor = struct { const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.fmod, .{input_buffer}); - try zml.testing.expectClose(std.testing.io, zml.testing.Slice.init(zml.Shape.init(.{6}, .f32), std.mem.sliceAsBytes(&e)), output, 1e-4); + try zml.testing.expectClose(std.testing.io, ConstSlice.init(zml.Shape.init(.{6}, .f32), std.mem.sliceAsBytes(&e)), output, 1e-4); } } @@ -991,13 +992,11 @@ pub const Tensor = struct { defer x_d_buffer.deinit(); const x_f4_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); - const x_f4_xla_bytes = try x_f4_xla_d.toHostAlloc(std.testing.allocator, std.testing.io); - defer std.testing.allocator.free(x_f4_xla_bytes); - - const x_f4_xla = std.mem.bytesAsSlice(floats.Float4E2M1, x_f4_xla_bytes); + const x_f4_xla = try x_f4_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); + defer x_f4_xla.free(std.testing.allocator); errdefer std.log.warn("convert(.f4e2m1) failed !\ninput f32:\n{e}\nzml.floats computed:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x), x_f4, x_f4_xla }); - try std.testing.expectEqualDeep(&x_f4, x_f4_xla); + try std.testing.expectEqualDeep(&x_f4, x_f4_xla.items(floats.Float4E2M1)); } // f8e3m4 @@ -1014,13 +1013,11 @@ pub const Tensor = struct { defer x_d_buffer.deinit(); const x_f8e3_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); - const x_f8e3_xla_bytes = try x_f8e3_xla_d.toHostAlloc(std.testing.allocator, std.testing.io); - defer std.testing.allocator.free(x_f8e3_xla_bytes); - - const x_f8e3_xla = std.mem.bytesAsSlice(floats.Float8E3M4, x_f8e3_xla_bytes); + const x_f8e3_xla = try x_f8e3_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); + defer x_f8e3_xla.free(std.testing.allocator); errdefer std.log.warn("convert(.f8e3m4) failed !\ninput f32:\n{e}\nzml.floats computed:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x), x_f8e3, x_f8e3_xla }); - try std.testing.expectEqualDeep(&x_f8e3, x_f8e3_xla); + try std.testing.expectEqualDeep(&x_f8e3, x_f8e3_xla.items(floats.Float8E3M4)); } } @@ -1217,7 +1214,7 @@ pub const Tensor = struct { const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.leakyReLU, .{input_buffer}); defer res.deinit(); - const expectation: zml.testing.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[2]f32{ -0.0688, 1.6795 })); + const expectation: ConstSlice = .init(input.shape(), std.mem.sliceAsBytes(&[2]f32{ -0.0688, 1.6795 })); try zml.testing.expectClose(std.testing.io, expectation, res, 1e-4); } @@ -2422,7 +2419,7 @@ pub const Tensor = struct { const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._gatherSlices, .{ operand_buffer, indices_buffer }); defer result.deinit(); - const expected: zml.testing.Slice = .init(Shape.init(.{ 2, 2, 2, 3 }, .u16), std.mem.sliceAsBytes(&[2][2][2][3]u16{ + const expected: ConstSlice = .init(Shape.init(.{ 2, 2, 2, 3 }, .u16), std.mem.sliceAsBytes(&[2][2][2][3]u16{ .{ .{ .{ 13, 14, 15 }, .{ 19, 20, 21 } }, .{ .{ 37, 38, 39 }, .{ 43, 44, 45 } }, @@ -2862,10 +2859,10 @@ pub const Tensor = struct { var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); defer res.deinit(); - const res_cpu = try res.toHostAlloc(std.testing.allocator, std.testing.io); - defer std.testing.allocator.free(res_cpu); + const res_cpu = try res.toSliceAlloc(std.testing.allocator, std.testing.io); + defer res_cpu.free(std.testing.allocator); - try std.testing.expectEqualSlices(i32, &.{ 0, 3, 1, 2, 4, 1, 4, 3, 0, 2 }, @alignCast(std.mem.bytesAsSlice(i32, res_cpu))); + try std.testing.expectEqualSlices(i32, &.{ 0, 3, 1, 2, 4, 1, 4, 3, 0, 2 }, res_cpu.items(i32)); } // 3D Tensor, dim = 1, descending @@ -2887,8 +2884,8 @@ pub const Tensor = struct { var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); defer res.deinit(); - const res_cpu = try res.toHostAlloc(std.testing.allocator, std.testing.io); - defer std.testing.allocator.free(res_cpu); + const res_cpu = try res.toSliceAlloc(std.testing.allocator, std.testing.io); + defer res_cpu.free(std.testing.allocator); try std.testing.expectEqualSlices(i32, &.{ 4, 1, 1, 2, 0, 2, 0, 0, 3, 4, @@ -2896,7 +2893,7 @@ pub const Tensor = struct { 1, 4, 2, 0, 2, 4, 2, 2, 0, 3, 3, 2, 0, 1, 4, 1, 1, 1, 2, 1, 0, 3, 3, 3, 3, 0, 3, 3, 4, 2, - }, @alignCast(std.mem.bytesAsSlice(i32, res_cpu))); + }, res_cpu.items(i32)); } // 4D Tensor, dim = 3, ascending @@ -2921,8 +2918,8 @@ pub const Tensor = struct { var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); defer res.deinit(); - const res_cpu = try res.toHostAlloc(std.testing.allocator, std.testing.io); - defer std.testing.allocator.free(res_cpu); + const res_cpu = try res.toSliceAlloc(std.testing.allocator, std.testing.io); + defer res_cpu.free(std.testing.allocator); try std.testing.expectEqualSlices(i32, &.{ 2, 1, 3, 0, @@ -2933,7 +2930,7 @@ pub const Tensor = struct { 1, 2, 3, 0, 2, 0, 1, 3, 3, 2, 1, 0, - }, @alignCast(std.mem.bytesAsSlice(i32, res_cpu))); + }, res_cpu.items(i32)); } } @@ -4078,7 +4075,7 @@ test "Tensor.maxPool2d" { try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2, 4 }, .f32), result.values.shape()); try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2, 4 }, .i32), result.indices.shape()); var buffer: [2][2][2][4]f32 = undefined; - _ = try result.values.toHost(std.mem.asBytes(&buffer), std.testing.io); + _ = try result.values.toSlice(.init(result.values.shape(), std.mem.asBytes(&buffer)), std.testing.io); try std.testing.expectEqualDeep( [2][2][2][4]f32{ .{ diff --git a/zml2/testing.zig b/zml2/testing.zig index 909b36f7..d1af5402 100644 --- a/zml2/testing.zig +++ b/zml2/testing.zig @@ -3,6 +3,7 @@ const builtin = @import("builtin"); const stdx = @import("stdx"); +const ConstSlice = @import("slice.zig").ConstSlice; const Platform = @import("platform.zig").Platform; const zml = @import("zml.zig"); @@ -21,36 +22,6 @@ pub fn env() Platform { return _platform.?; } -pub const Slice = struct { - data: []const u8, - shape: zml.Shape, - - pub fn init(shape: zml.Shape, data: []const u8) Slice { - return .{ .data = data, .shape = shape }; - } - - pub fn dtype(self: Slice) zml.DataType { - return self.shape.dtype(); - } - - pub fn items(self: Slice, comptime T: type) []const T { - return @alignCast(std.mem.bytesAsSlice(T, self.data)); - } - - pub fn format( - self: @This(), - writer: *std.Io.Writer, - ) std.Io.Writer.Error!void { - return writer.print("{any}", .{self}); - } - - pub fn formatNumber(self: @This(), writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { - _ = self; // autofix - _ = n; // autofix - return writer.print("TODO", .{}); - } -}; - /// In neural network we generally care about the relative precision, /// but on a given dimension, if the output is close to 0, then the precision /// don't matter as much. @@ -64,19 +35,19 @@ pub fn approxEq(comptime Float: type, l: Float, r: Float, tolerance: Float) bool /// host for comparison ! pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, tolerance: f32) !void { const allocator = if (builtin.is_test) std.testing.allocator else std.heap.smp_allocator; - var left: Slice, const should_free_left = if (@TypeOf(left_) == zml.Buffer) b: { - const bytes = try left_.toHostAlloc(allocator, io); - break :b .{ .init(left_.shape(), bytes), true }; + var left: ConstSlice, const should_free_left = if (@TypeOf(left_) == zml.Buffer) b: { + const slice = try left_.toSliceAlloc(allocator, io); + break :b .{ slice.constSlice(), true }; } else .{ left_, false }; - var right: Slice, const should_free_right = if (@TypeOf(right_) == zml.Buffer) b: { - const bytes = try right_.toHostAlloc(allocator, io); - break :b .{ .init(right_.shape(), bytes), true }; + var right: ConstSlice, const should_free_right = if (@TypeOf(right_) == zml.Buffer) b: { + const slice = try right_.toSliceAlloc(allocator, io); + break :b .{ slice.constSlice(), true }; } else .{ right_, false }; defer { - if (should_free_left) allocator.free(left.data); - if (should_free_right) allocator.free(right.data); + if (should_free_left) left.free(allocator); + if (should_free_right) right.free(allocator); } errdefer log.err("\n--> Left: {0f}{0d:24.3}\n--> Right: {1f}{1d:24.3}", .{ left, right }); if (!std.mem.eql(i64, left.shape.dims(), right.shape.dims())) { diff --git a/zml2/zml.zig b/zml2/zml.zig index 13998886..d9db13b3 100644 --- a/zml2/zml.zig +++ b/zml2/zml.zig @@ -27,6 +27,9 @@ pub const Target = platform.Target; pub const CompilationOptions = platform.CompilationOptions; pub const shape = @import("shape.zig"); pub const Shape = shape.Shape; +pub const slice = @import("slice.zig"); +pub const Slice = slice.Slice; +pub const ConstSlice = slice.ConstSlice; pub const tensor = @import("tensor.zig"); pub const Tensor = tensor.Tensor; pub const testing = @import("testing.zig"); From 33a5371dba4f0269e3142785b4c77cbdc1dba234 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 11 Dec 2025 15:48:06 +0100 Subject: [PATCH 119/881] Plug std.Io in runtimes --- runtimes/cpu/cpu.zig | 14 ++++++++------ runtimes/cuda/BUILD.bazel | 1 - runtimes/cuda/cuda.zig | 19 ++++++++++++------- runtimes/neuron/BUILD.bazel | 9 ++++----- runtimes/neuron/neuron.zig | 26 ++++++++++++++++---------- runtimes/rocm/BUILD.bazel | 1 - runtimes/rocm/rocm.zig | 12 ++++++------ runtimes/runtimes.zig | 14 ++++++++------ runtimes/tpu/BUILD.bazel | 1 - runtimes/tpu/tpu.zig | 17 +++++++++++------ zml2/platform.zig | 4 ++-- 11 files changed, 67 insertions(+), 51 deletions(-) diff --git a/runtimes/cpu/cpu.zig b/runtimes/cpu/cpu.zig index 5ec12bbe..e5c05e68 100644 --- a/runtimes/cpu/cpu.zig +++ b/runtimes/cpu/cpu.zig @@ -13,18 +13,15 @@ pub fn isEnabled() bool { return @hasDecl(c, "ZML_RUNTIME_CPU"); } -pub fn load() !*const pjrt.Api { +pub fn load(io: std.Io) !*const pjrt.Api { if (comptime !isEnabled()) { return error.Unavailable; } - var io: std.Io.Threaded = .init_single_threaded; - defer io.deinit(); - var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); defer arena.deinit(); - var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator(), .io = io.io() }) orelse { + var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator(), .io = io }) orelse { stdx.debug.panic("Unable to find runfiles", .{}); }; @@ -46,6 +43,11 @@ pub fn load() !*const pjrt.Api { var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; const path = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "libpjrt_cpu" ++ ext }); - break :blk pjrt.Api.loadFrom(path); + var future = io.async(struct { + fn call(path_: [:0]const u8) !*const pjrt.Api { + return pjrt.Api.loadFrom(path_); + } + }.call, .{path}); + break :blk future.await(io); }; } diff --git a/runtimes/cuda/BUILD.bazel b/runtimes/cuda/BUILD.bazel index cf266d7b..6009eb72 100644 --- a/runtimes/cuda/BUILD.bazel +++ b/runtimes/cuda/BUILD.bazel @@ -34,7 +34,6 @@ zig_library( ] + select({ "//runtimes:cuda.enabled": [ ":libpjrt_cuda", - "//async", "//stdx", "@rules_zig//zig/runfiles", ], diff --git a/runtimes/cuda/cuda.zig b/runtimes/cuda/cuda.zig index bf52dc07..30148e9b 100644 --- a/runtimes/cuda/cuda.zig +++ b/runtimes/cuda/cuda.zig @@ -1,7 +1,6 @@ const std = @import("std"); const builtin = @import("builtin"); -const async = @import("async"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); @@ -16,8 +15,9 @@ pub fn isEnabled() bool { return @hasDecl(c, "ZML_RUNTIME_CUDA"); } -fn hasNvidiaDevice() bool { - async.File.access("/dev/nvidiactl", .{ .mode = .read_only }) catch return false; +fn hasNvidiaDevice(io: std.Io) bool { + _ = io; // autofix + std.fs.accessAbsolute("/dev/nvidiactl", .{ .read = true }) catch return false; return true; } @@ -33,14 +33,14 @@ fn setupXlaGpuCudaDirFlag(allocator: std.mem.Allocator, sandbox: []const u8) !vo _ = c.setenv("XLA_FLAGS", new_xla_flagsZ, 1); } -pub fn load() !*const pjrt.Api { +pub fn load(io: std.Io) !*const pjrt.Api { if (comptime !isEnabled()) { return error.Unavailable; } if (comptime builtin.os.tag != .linux) { return error.Unavailable; } - if (!hasNvidiaDevice()) { + if (!hasNvidiaDevice(io)) { return error.Unavailable; } if (hasCudaPathInLDPath()) { @@ -50,7 +50,7 @@ pub fn load() !*const pjrt.Api { var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); defer arena.deinit(); - var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator() }) orelse { + var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator(), .io = io }) orelse { stdx.debug.panic("Unable to find runfiles", .{}); }; @@ -79,6 +79,11 @@ pub fn load() !*const pjrt.Api { return blk: { var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; const path = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_cuda.so" }); - break :blk async.callBlocking(pjrt.Api.loadFrom, .{path}); + var future = io.async(struct { + fn call(path_: [:0]const u8) !*const pjrt.Api { + return pjrt.Api.loadFrom(path_); + } + }.call, .{path}); + break :blk future.await(io); }; } diff --git a/runtimes/neuron/BUILD.bazel b/runtimes/neuron/BUILD.bazel index 1d6c0280..993ed5e5 100644 --- a/runtimes/neuron/BUILD.bazel +++ b/runtimes/neuron/BUILD.bazel @@ -11,9 +11,9 @@ load("@zml//bazel:runfiles.bzl", "runfiles_to_default") # of neuronx-cc (see add_needed of the patchelf target below). zig_shared_library( name = "libpjrt_neuron", - zigopts = ["-fno-stack-check"], main = "libpjrt_neuron.zig", visibility = ["@libpjrt_neuron//:__subpackages__"], + zigopts = ["-fno-stack-check"], deps = [ ":libpython", "//stdx", @@ -80,13 +80,13 @@ upb_c_proto_library( zig_shared_library( name = "libneuronxla", + main = "libneuronxla.zig", + shared_lib_name = "libneuronxla.so", + visibility = ["@libpjrt_neuron//:__subpackages__"], zigopts = [ "-fno-stack-check", "-fPIC", ], - main = "libneuronxla.zig", - shared_lib_name = "libneuronxla.so", - visibility = ["@libpjrt_neuron//:__subpackages__"], deps = [ ":hlo_proto_upb", ":libpython", @@ -122,7 +122,6 @@ zig_library( ":libnrt_headers", ":libpython", ":zmlxneuron", - "//async", "//stdx", "@libpjrt_neuron", "@rules_zig//zig/runfiles", diff --git a/runtimes/neuron/neuron.zig b/runtimes/neuron/neuron.zig index d40106f0..3638e27e 100644 --- a/runtimes/neuron/neuron.zig +++ b/runtimes/neuron/neuron.zig @@ -1,7 +1,6 @@ const std = @import("std"); const builtin = @import("builtin"); -const async = @import("async"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); @@ -14,15 +13,17 @@ pub fn isEnabled() bool { return @hasDecl(c, "ZML_RUNTIME_NEURON"); } -fn hasNeuronDevice() bool { - async.File.access("/dev/neuron0", .{ .mode = .read_only }) catch return false; +fn hasNeuronDevice(io: std.Io) bool { + _ = io; // autofix + std.fs.accessAbsolute("/dev/neuron0", .{ .read = true }) catch return false; return true; } -fn isRunningOnEC2() !bool { +fn isRunningOnEC2(io: std.Io) !bool { + _ = io; // autofix const AmazonEC2 = "Amazon EC2"; - var f = try async.File.open("/sys/devices/virtual/dmi/id/sys_vendor", .{ .mode = .read_only }); + var f = try std.fs.openFileAbsolute("/sys/devices/virtual/dmi/id/sys_vendor", .{ .read = true }); defer f.close() catch {}; var content: [AmazonEC2.len]u8 = undefined; @@ -31,24 +32,24 @@ fn isRunningOnEC2() !bool { return std.mem.eql(u8, content[0..n_read], AmazonEC2); } -pub fn load() !*const pjrt.Api { +pub fn load(io: std.Io) !*const pjrt.Api { if (comptime !isEnabled()) { return error.Unavailable; } if (comptime builtin.os.tag != .linux) { return error.Unavailable; } - if (!(isRunningOnEC2() catch false)) { + if (!(isRunningOnEC2(io) catch false)) { return error.Unavailable; } - if (!hasNeuronDevice()) { + if (!hasNeuronDevice(io)) { return error.Unavailable; } var arena = std.heap.ArenaAllocator.init(std.heap.smp_allocator); defer arena.deinit(); - var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator() }) orelse { + var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator(), .io = io }) orelse { stdx.debug.panic("Unable to find runfiles", .{}); }; @@ -64,6 +65,11 @@ pub fn load() !*const pjrt.Api { return blk: { var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; const path = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_neuron.so" }); - break :blk async.callBlocking(pjrt.Api.loadFrom, .{path}); + var future = io.async(struct { + fn call(path_: [:0]const u8) !*const pjrt.Api { + return pjrt.Api.loadFrom(path_); + } + }.call, .{path}); + break :blk future.await(io); }; } diff --git a/runtimes/rocm/BUILD.bazel b/runtimes/rocm/BUILD.bazel index 7f9be788..6e0cd373 100644 --- a/runtimes/rocm/BUILD.bazel +++ b/runtimes/rocm/BUILD.bazel @@ -43,7 +43,6 @@ zig_library( ] + select({ "//runtimes:rocm.enabled": [ ":libpjrt_rocm", - "//async", "//stdx", "@rules_zig//zig/runfiles", ], diff --git a/runtimes/rocm/rocm.zig b/runtimes/rocm/rocm.zig index c5c12285..95e85fb4 100644 --- a/runtimes/rocm/rocm.zig +++ b/runtimes/rocm/rocm.zig @@ -1,7 +1,6 @@ const std = @import("std"); const builtin = @import("builtin"); -const async = @import("async"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); @@ -14,9 +13,10 @@ pub fn isEnabled() bool { return @hasDecl(c, "ZML_RUNTIME_ROCM"); } -fn hasRocmDevices() bool { +fn hasRocmDevices(io: std.Io) bool { + _ = io; // autofix inline for (&.{ "/dev/kfd", "/dev/dri" }) |path| { - async.File.access(path, .{ .mode = .read_only }) catch return false; + std.fs.accessAbsolute(path, .{ .read = true }) catch return false; } return true; } @@ -26,21 +26,21 @@ fn setupRocmEnv(rocm_data_dir: []const u8) !void { _ = c.setenv("ROCM_PATH", try stdx.fs.path.bufJoinZ(&buf, &.{rocm_data_dir}), 1); // must be zero terminated } -pub fn load() !*const pjrt.Api { +pub fn load(io: std.Io) !*const pjrt.Api { if (comptime !isEnabled()) { return error.Unavailable; } if (comptime builtin.os.tag != .linux) { return error.Unavailable; } - if (!hasRocmDevices()) { + if (!hasRocmDevices(io)) { return error.Unavailable; } var arena = std.heap.ArenaAllocator.init(std.heap.smp_allocator); defer arena.deinit(); - var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator() }) orelse { + var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator(), .io = io }) orelse { stdx.debug.panic("Unable to find runfiles", .{}); }; diff --git a/runtimes/runtimes.zig b/runtimes/runtimes.zig index 7319da76..76311ee6 100644 --- a/runtimes/runtimes.zig +++ b/runtimes/runtimes.zig @@ -1,3 +1,5 @@ +const std = @import("std"); + const pjrt = @import("pjrt"); const cpu = @import("runtimes/cpu"); const cuda = @import("runtimes/cuda"); @@ -13,13 +15,13 @@ pub const Platform = enum { neuron, }; -pub fn load(tag: Platform) !*const pjrt.Api { +pub fn load(tag: Platform, io: std.Io) !*const pjrt.Api { return switch (tag) { - .cpu => try cpu.load(), - .cuda => try cuda.load(), - .rocm => try rocm.load(), - .tpu => try tpu.load(), - .neuron => try neuron.load(), + .cpu => try cpu.load(io), + .cuda => try cuda.load(io), + .rocm => try rocm.load(io), + .tpu => try tpu.load(io), + .neuron => try neuron.load(io), }; } diff --git a/runtimes/tpu/BUILD.bazel b/runtimes/tpu/BUILD.bazel index fe513d3c..bc1e3a99 100644 --- a/runtimes/tpu/BUILD.bazel +++ b/runtimes/tpu/BUILD.bazel @@ -22,7 +22,6 @@ zig_library( ] + select({ "//runtimes:tpu.enabled": [ ":libpjrt_tpu", - "//async", "//stdx", "@rules_zig//zig/runfiles", ], diff --git a/runtimes/tpu/tpu.zig b/runtimes/tpu/tpu.zig index fd972c8a..adf17923 100644 --- a/runtimes/tpu/tpu.zig +++ b/runtimes/tpu/tpu.zig @@ -1,7 +1,6 @@ const std = @import("std"); const builtin = @import("builtin"); -const async = @import("async"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); @@ -18,11 +17,12 @@ pub fn isEnabled() bool { /// metadata server, hanging the process. So only do it on GCP. /// Do it using the official method at: /// https://cloud.google.com/compute/docs/instances/detect-compute-engine?hl=en#use_operating_system_tools_to_detect_if_a_vm_is_running_in -fn isOnGCP() !bool { +fn isOnGCP(io: std.Io) !bool { + _ = io; // autofix // TODO: abstract that in the client and fail init const GoogleComputeEngine = "Google Compute Engine"; - var f = try async.File.open("/sys/devices/virtual/dmi/id/product_name", .{ .mode = .read_only }); + var f = try std.fs.openFileAbsolute("/sys/devices/virtual/dmi/id/product_name", .{ .read = true }); defer f.close() catch {}; var content: [GoogleComputeEngine.len]u8 = undefined; @@ -31,14 +31,14 @@ fn isOnGCP() !bool { return std.mem.eql(u8, content[0..n_read], GoogleComputeEngine); } -pub fn load() !*const pjrt.Api { +pub fn load(io: std.Io) !*const pjrt.Api { if (comptime !isEnabled()) { return error.Unavailable; } if (comptime builtin.os.tag != .linux) { return error.Unavailable; } - if (!(isOnGCP() catch false)) { + if (!(isOnGCP(io) catch false)) { return error.Unavailable; } @@ -61,6 +61,11 @@ pub fn load() !*const pjrt.Api { return blk: { var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; const path = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_tpu.so" }); - break :blk async.callBlocking(pjrt.Api.loadFrom, .{path}); + var future = io.async(struct { + fn call(path_: [:0]const u8) !*const pjrt.Api { + return pjrt.Api.loadFrom(path_); + } + }.call, .{path}); + break :blk future.await(io); }; } diff --git a/zml2/platform.zig b/zml2/platform.zig index e67e7128..134ff487 100644 --- a/zml2/platform.zig +++ b/zml2/platform.zig @@ -30,8 +30,8 @@ pub const Platform = struct { pub const MAX_NUM_DEVICES: u8 = if (runtimes.isEnabled(.tpu)) 32 else 8; - pub fn init(target: Target, options: CreateOptions) !Platform { - const api = try runtimes.load(target); + pub fn init(target: Target, io: std.Io, options: CreateOptions) !Platform { + const api = try runtimes.load(target, io); var named_values_buf: [16]pjrt.NamedValue = undefined; const pjrt_client = try pjrt.Client.init(api, options.toNamedValues(target, &named_values_buf)); From 447fc8c59bbf433e54422a37fb79b66887b70760 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 11 Dec 2025 16:02:28 +0100 Subject: [PATCH 120/881] Made a simple main to test on other platforms --- zml2/main.zig | 60 +++++++++++++++++++++++++++++++++++++++++++++---- zml2/module.zig | 1 - 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/zml2/main.zig b/zml2/main.zig index a63d76ae..395051ec 100644 --- a/zml2/main.zig +++ b/zml2/main.zig @@ -16,6 +16,28 @@ pub const Model = struct { _ = self; } + pub fn loadBuffers(model: Model, allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform) !zml.Bufferized(Model) { + const weight_slice: zml.Slice = try .alloc(allocator, model.weight.shape()); + defer weight_slice.free(allocator); + @memset(weight_slice.items(f32), 1); + + const bias_slice: zml.Slice = try .alloc(allocator, model.bias.shape()); + defer bias_slice.free(allocator); + @memset(bias_slice.items(f32), 2); + + const weight_buffer: zml.Buffer = try .fromBytes(platform, weight_slice.shape, weight_slice.data, io); + errdefer weight_buffer.deinit(); + const bias_buffer: zml.Buffer = try .fromBytes(platform, bias_slice.shape, bias_slice.data, io); + errdefer bias_buffer.deinit(); + + return .{ .weight = weight_buffer, .bias = bias_buffer }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Model)) void { + self.weight.deinit(); + self.bias.deinit(); + } + pub fn forward(self: Model, input: zml.Tensor) zml.Tensor { return self.weight.dot(input, .n).add(self.bias); } @@ -27,18 +49,48 @@ pub fn main() !void { const allocator = gpa.allocator(); - var io: std.Io.Threaded = .init(allocator); - defer io.deinit(); + var threaded: std.Io.Threaded = .init(allocator); + defer threaded.deinit(); + + const io = threaded.io(); zml.init(); defer zml.deinit(); - var platform = try zml.Platform.init(.cpu, .{}); + var platform = try zml.Platform.init(.cpu, io, .{}); defer platform.deinit(); var model: Model = try .init(); defer model.deinit(); - var exe = try zml.module.compileModel(allocator, io.io(), Model.forward, model, .{zml.Tensor.init(zml.Shape.init(.{ .n = 4096 }, .f32))}, platform); + const input: zml.Tensor = .init(zml.Shape.init(.{ .n = 4096 }, .f32)); + var exe = try zml.module.compileModel(allocator, io, Model.forward, model, .{input}, platform); defer exe.deinit(); + + const slice: zml.Slice = try .alloc(allocator, input.shape()); + defer slice.free(allocator); + @memset(slice.items(f32), 1); + + var model_buffers = try model.loadBuffers(allocator, io, platform); + defer Model.unloadBuffers(&model_buffers); + + const input_buffer: zml.Buffer = try .fromBytes(platform, slice.shape, slice.data, io); + defer input_buffer.deinit(); + + var args = try exe.args(allocator); + defer args.deinit(allocator); + + var results = try exe.results(allocator); + defer results.deinit(allocator); + + args.set(.{ model_buffers, input_buffer }); + exe.call(args, &results, io); + + const output = results.get(zml.Buffer); + defer output.deinit(); + + const output_slice = try output.toSliceAlloc(allocator, io); + defer output_slice.free(allocator); + + std.log.info("Output: {any}", .{output_slice.items(f32)[0..10]}); } diff --git a/zml2/module.zig b/zml2/module.zig index c93ed0c7..c6873363 100644 --- a/zml2/module.zig +++ b/zml2/module.zig @@ -317,7 +317,6 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar }; meta.visit(struct { fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { - std.log.debug("Argument {} - shape: {f}", .{ ctx_.current_argument_id, tensor.shape() }); const mlir_type = mlir.rankedTensorType( tensor.dims(), mlirx.Type.fromDType(ctx_.compilation_context.mlir_ctx, tensor.dtype()), From f7aa2f3166c0b3cf4510348a679a58d67f0f0791 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 11 Dec 2025 16:04:23 +0000 Subject: [PATCH 121/881] Some minor fixes * Use std.testing.io to get test env * Disable libnvptx when compiling --- zml2/module.zig | 1 - zml2/testing.zig | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/zml2/module.zig b/zml2/module.zig index c6873363..51869d22 100644 --- a/zml2/module.zig +++ b/zml2/module.zig @@ -446,7 +446,6 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_triton_gemm", false, upb_arena); try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_latency_hiding_scheduler", true, upb_arena); try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_llvm_module_compilation_parallelism", true, upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_libnvptxcompiler", true, upb_arena); }, .rocm => { // Disable Triton GEMM on ROCM. For some reason it's much, much slower when diff --git a/zml2/testing.zig b/zml2/testing.zig index d1af5402..e7d8df8b 100644 --- a/zml2/testing.zig +++ b/zml2/testing.zig @@ -16,7 +16,7 @@ pub fn env() Platform { if (_platform == null) { zml.init(); - _platform = Platform.init(.cpu, .{}) catch unreachable; + _platform = Platform.init(.cpu, std.testing.io, .{}) catch unreachable; } return _platform.?; From ee18a2745b606a36b8d9bbb2e74ec7f6ab92ef39 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 11 Dec 2025 17:41:19 +0100 Subject: [PATCH 122/881] zml/io (#361) --- zml2/BUILD.bazel | 12 + zml2/io.zig | 6 + zml2/io/BUILD.bazel | 14 + zml2/io/io.zig | 1 + zml2/io/vfs.zig | 581 +++++++++++++++++++++++++++++ zml2/io/vfs/BUILD.bazel | 10 + zml2/io/vfs/http.zig | 788 ++++++++++++++++++++++++++++++++++++++++ zml2/io/vfs/index.zig | 5 + zml2/main_io.zig | 208 +++++++++++ zml2/zml.zig | 7 +- 10 files changed, 1629 insertions(+), 3 deletions(-) create mode 100644 zml2/io.zig create mode 100644 zml2/io/BUILD.bazel create mode 100644 zml2/io/io.zig create mode 100644 zml2/io/vfs.zig create mode 100644 zml2/io/vfs/BUILD.bazel create mode 100644 zml2/io/vfs/http.zig create mode 100644 zml2/io/vfs/index.zig create mode 100644 zml2/main_io.zig diff --git a/zml2/BUILD.bazel b/zml2/BUILD.bazel index 4731acb9..cabb6606 100644 --- a/zml2/BUILD.bazel +++ b/zml2/BUILD.bazel @@ -25,6 +25,7 @@ zig_library( "dtype.zig", "exe.zig", "floats.zig", + "io.zig", "meta.zig", "mlirx.zig", "module.zig", @@ -54,6 +55,7 @@ zig_library( "//runtimes", "//stdx", "//upb", + "//zml2/io", "@rules_zig//zig/runfiles", ], ) @@ -68,6 +70,16 @@ zig_binary( ], ) +zig_binary( + name = "main_io", + args = [], + data = [], + main = "main_io.zig", + deps = [ + "//zml2:zml", + ], +) + # All ZML Tests zig_test( diff --git a/zml2/io.zig b/zml2/io.zig new file mode 100644 index 00000000..67f54a6c --- /dev/null +++ b/zml2/io.zig @@ -0,0 +1,6 @@ +const builtin = @import("builtin"); +const std = @import("std"); + +pub const VFS = @import("io").VFS; + +const log = std.log.scoped(.@"zml/io"); diff --git a/zml2/io/BUILD.bazel b/zml2/io/BUILD.bazel new file mode 100644 index 00000000..6d5241ca --- /dev/null +++ b/zml2/io/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "io", + srcs = [ + "vfs.zig", + ], + main = "io.zig", + visibility = ["//visibility:public"], + deps = [ + "//stdx", + "//zml2/io/vfs", + ], +) diff --git a/zml2/io/io.zig b/zml2/io/io.zig new file mode 100644 index 00000000..78c59e21 --- /dev/null +++ b/zml2/io/io.zig @@ -0,0 +1 @@ +pub const VFS = @import("vfs.zig").VFS; diff --git a/zml2/io/vfs.zig b/zml2/io/vfs.zig new file mode 100644 index 00000000..9ec67f95 --- /dev/null +++ b/zml2/io/vfs.zig @@ -0,0 +1,581 @@ +const std = @import("std"); + +const log = std.log.scoped(.@"zml/io/vfs"); + +pub const VFS = struct { + pub const Error = FileEntryError || DirEntryError || ResolveAbsolutePathError; + + pub const FileEntryError = error{ + MissingVirtualFileHandle, + } || ResolveAbsolutePathError; + + pub const DirEntryError = error{ + MissingVirtualDirHandle, + } || ResolveAbsolutePathError; + + pub const ResolveAbsolutePathError = error{ + SchemeNotRegistered, + InvalidURI, + }; + + pub const HTTP = @import("vfs").HTTP; + + const FsFile = struct { + backend: std.Io, + handle: std.Io.File.Handle, + }; + + const FsDir = struct { + backend: std.Io, + handle: std.Io.Dir.Handle, + }; + + const ROOT_DIR_HANDLE: std.Io.Dir.Handle = 0; + + allocator: std.mem.Allocator, + mutex: std.Thread.Mutex, + + backends: std.StringHashMapUnmanaged(std.Io), + // Maps virtual file and dir handles to backend handles + files: std.AutoHashMapUnmanaged(std.Io.File.Handle, *FsFile), + dirs: std.AutoHashMapUnmanaged(std.Io.Dir.Handle, *FsDir), + next_file_handle: std.Io.File.Handle, + next_dir_handle: std.Io.Dir.Handle, + + vtable: std.Io.VTable, + + pub fn init(allocator: std.mem.Allocator, base_io: std.Io) VFS { + var vtable = base_io.vtable.*; + vtable.dirMake = dirMake; + vtable.dirMakePath = dirMakePath; + vtable.dirMakeOpenPath = dirMakeOpenPath; + vtable.dirStat = dirStat; + vtable.dirStatPath = dirStatPath; + vtable.dirAccess = dirAccess; + vtable.dirCreateFile = dirCreateFile; + vtable.dirOpenFile = dirOpenFile; + vtable.dirOpenDir = dirOpenDir; + vtable.dirClose = dirClose; + vtable.fileStat = fileStat; + vtable.fileClose = fileClose; + vtable.fileWriteStreaming = fileWriteStreaming; + vtable.fileWritePositional = fileWritePositional; + vtable.fileReadStreaming = fileReadStreaming; + vtable.fileReadPositional = fileReadPositional; + vtable.fileSeekBy = fileSeekBy; + vtable.fileSeekTo = fileSeekTo; + vtable.openSelfExe = openSelfExe; + + return .{ + .allocator = allocator, + .mutex = .{}, + .backends = .{}, + .files = .{}, + .dirs = .{}, + .next_file_handle = 0, + .next_dir_handle = 1, // 0 reserved for ROOT_DIR_HANDLE + .vtable = vtable, + }; + } + + pub fn deinit(self: *VFS) void { + var file_it = self.files.valueIterator(); + while (file_it.next()) |entry_ptr| { + self.allocator.destroy(entry_ptr.*); + } + self.files.deinit(self.allocator); + + var dir_it = self.dirs.valueIterator(); + while (dir_it.next()) |entry_ptr| { + self.allocator.destroy(entry_ptr.*); + } + self.dirs.deinit(self.allocator); + + self.backends.deinit(self.allocator); + } + + pub fn register(self: *VFS, scheme: []const u8, backend: std.Io) std.mem.Allocator.Error!void { + self.mutex.lock(); + defer self.mutex.unlock(); + + try self.backends.put(self.allocator, scheme, backend); + } + + pub fn unregister(self: *VFS, scheme: []const u8) bool { + self.mutex.lock(); + defer self.mutex.unlock(); + + return self.backends.remove(scheme); + } + + pub fn io(self: *VFS) std.Io { + return .{ + .vtable = &self.vtable, + .userdata = self, + }; + } + + fn backendFromAbsolutePath(self: *VFS, path: []const u8) ResolveAbsolutePathError!std.Io { + const uri = std.Uri.parse(path) catch return ResolveAbsolutePathError.InvalidURI; + return self.backends.get(uri.scheme) orelse return ResolveAbsolutePathError.SchemeNotRegistered; + } + + fn registerFile(self: *VFS, backend: std.Io, handle: std.Io.File.Handle) std.mem.Allocator.Error!std.Io.File.Handle { + const entry = try self.allocator.create(FsFile); + errdefer self.allocator.destroy(entry); + + entry.* = .{ + .backend = backend, + .handle = handle, + }; + + self.mutex.lock(); + defer self.mutex.unlock(); + + const virtual_handle = self.next_file_handle; + self.next_file_handle += 1; + + try self.files.put(self.allocator, virtual_handle, entry); + + return virtual_handle; + } + + fn closeFile(self: *VFS, virtual_handle: std.Io.File.Handle) void { + self.mutex.lock(); + const entry = self.files.fetchRemove(virtual_handle); + self.mutex.unlock(); + + if (entry) |kv| { + const fs_file = kv.value; + fs_file.backend.vtable.fileClose(fs_file.backend.userdata, .{ .handle = fs_file.handle }); + self.allocator.destroy(fs_file); + } + } + + fn fsFile(self: *VFS, virtual_handle: std.Io.File.Handle) FileEntryError!FsFile { + self.mutex.lock(); + defer self.mutex.unlock(); + + const entry = self.files.get(virtual_handle) orelse return FileEntryError.MissingVirtualFileHandle; + return entry.*; + } + + fn registerDir(self: *VFS, backend: std.Io, handle: std.Io.Dir.Handle) std.mem.Allocator.Error!std.Io.Dir.Handle { + const entry = try self.allocator.create(FsDir); + errdefer self.allocator.destroy(entry); + + entry.* = .{ + .backend = backend, + .handle = handle, + }; + + self.mutex.lock(); + defer self.mutex.unlock(); + + const virtual_handle = self.next_dir_handle; + self.next_dir_handle += 1; + + try self.dirs.put(self.allocator, virtual_handle, entry); + return virtual_handle; + } + + fn closeDir(self: *VFS, virtual_handle: std.Io.Dir.Handle) void { + if (virtual_handle == ROOT_DIR_HANDLE) return; + + self.mutex.lock(); + const entry = self.dirs.fetchRemove(virtual_handle); + self.mutex.unlock(); + + if (entry) |kv| { + const fs_dir = kv.value; + fs_dir.backend.vtable.dirClose(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }); + self.allocator.destroy(fs_dir); + } + } + + fn fsDir(self: *VFS, virtual_handle: std.Io.Dir.Handle, sub_path: ?[]const u8) DirEntryError!FsDir { + if (virtual_handle == ROOT_DIR_HANDLE) { + std.debug.assert(sub_path != null); + const backend = try self.backendFromAbsolutePath(sub_path.?); + return .{ .backend = backend, .handle = ROOT_DIR_HANDLE }; + } + + self.mutex.lock(); + defer self.mutex.unlock(); + + const entry = self.dirs.get(virtual_handle) orelse return DirEntryError.MissingVirtualDirHandle; + return entry.*; + } + + pub fn rootDir(self: *VFS) std.Io.Dir { + _ = self; + return .{ .handle = ROOT_DIR_HANDLE }; + } + + pub fn openAbsoluteDir(self: *VFS, vfs_io: std.Io, path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const backend_dir = try vfs_io.vtable.dirOpenDir(vfs_io.userdata, .{ .handle = ROOT_DIR_HANDLE }, path, options); + const virtual_handle = self.registerDir(vfs_io, backend_dir.handle) catch return std.Io.Dir.OpenError.Unexpected; + + return .{ .handle = virtual_handle }; + } + + pub fn openAbsoluteFile(self: *VFS, vfs_io: std.Io, path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + const file = try vfs_io.vtable.dirOpenFile(vfs_io.userdata, .{ .handle = ROOT_DIR_HANDLE }, path, flags); + const virtual_handle = self.registerFile(vfs_io, file.handle) catch return std.Io.File.OpenError.SystemResources; + return .{ .handle = virtual_handle }; + } + + // VTable implementations + + fn dirMake( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + mode: std.Io.Dir.Mode, + ) std.Io.Dir.MakeError!void { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + + const fs_dir = self.fsDir(dir.handle, sub_path) catch |err| { + return switch (err) { + DirEntryError.SchemeNotRegistered => std.Io.Dir.MakeError.BadPathName, + DirEntryError.InvalidURI => std.Io.Dir.MakeError.BadPathName, + DirEntryError.MissingVirtualDirHandle => std.Io.Dir.MakeError.FileNotFound, + }; + }; + + return fs_dir.backend.vtable.dirMake(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }, sub_path, mode); + } + + fn dirMakePath( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + mode: std.Io.Dir.Mode, + ) std.Io.Dir.MakeError!void { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + + const fs_dir = self.fsDir(dir.handle, sub_path) catch |err| { + return switch (err) { + DirEntryError.SchemeNotRegistered => std.Io.Dir.MakeError.BadPathName, + DirEntryError.InvalidURI => std.Io.Dir.MakeError.BadPathName, + DirEntryError.MissingVirtualDirHandle => std.Io.Dir.MakeError.FileNotFound, + }; + }; + + return fs_dir.backend.vtable.dirMakePath(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }, sub_path, mode); + } + + fn dirMakeOpenPath( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + options: std.Io.Dir.OpenOptions, + ) std.Io.Dir.MakeOpenPathError!std.Io.Dir { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + + const fs_dir = self.fsDir(dir.handle, sub_path) catch |err| { + return switch (err) { + DirEntryError.SchemeNotRegistered => std.Io.Dir.MakeOpenPathError.BadPathName, + DirEntryError.InvalidURI => std.Io.Dir.MakeOpenPathError.BadPathName, + DirEntryError.MissingVirtualDirHandle => std.Io.Dir.MakeOpenPathError.FileNotFound, + }; + }; + + const fs_open_dir = try fs_dir.backend.vtable.dirMakeOpenPath(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }, sub_path, options); + + const vfs_handle = self.registerDir(fs_dir.backend, fs_open_dir.handle) catch |err| { + return switch (err) { + std.mem.Allocator.Error.OutOfMemory => std.Io.Dir.OpenError.SystemFdQuotaExceeded, + }; + }; + + return .{ .handle = vfs_handle }; + } + + fn dirStat( + userdata: ?*anyopaque, + dir: std.Io.Dir, + ) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + + if (dir.handle == ROOT_DIR_HANDLE) return std.Io.Dir.StatError.Unexpected; + + const fs_dir = self.fsDir(dir.handle, null) catch |err| { + return switch (err) { + DirEntryError.SchemeNotRegistered => std.Io.Dir.StatError.Unexpected, + DirEntryError.InvalidURI => std.Io.Dir.StatError.Unexpected, + DirEntryError.MissingVirtualDirHandle => std.Io.Dir.StatError.Unexpected, + }; + }; + + return fs_dir.backend.vtable.dirStat(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }); + } + + fn dirStatPath( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + options: std.Io.Dir.StatPathOptions, + ) std.Io.Dir.StatPathError!std.Io.Dir.Stat { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + + const fs_dir = self.fsDir(dir.handle, sub_path) catch |err| { + return switch (err) { + DirEntryError.SchemeNotRegistered => std.Io.Dir.StatPathError.BadPathName, + DirEntryError.InvalidURI => std.Io.Dir.StatPathError.BadPathName, + DirEntryError.MissingVirtualDirHandle => std.Io.Dir.StatPathError.FileNotFound, + }; + }; + + return fs_dir.backend.vtable.dirStatPath(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }, sub_path, options); + } + + fn dirAccess( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + options: std.Io.Dir.AccessOptions, + ) std.Io.Dir.AccessError!void { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + + const fs_dir = self.fsDir(dir.handle, sub_path) catch |err| { + return switch (err) { + DirEntryError.SchemeNotRegistered => std.Io.Dir.AccessError.BadPathName, + DirEntryError.InvalidURI => std.Io.Dir.AccessError.BadPathName, + DirEntryError.MissingVirtualDirHandle => std.Io.Dir.AccessError.FileNotFound, + }; + }; + + return fs_dir.backend.vtable.dirAccess(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }, sub_path, options); + } + + fn dirCreateFile( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + flags: std.Io.File.CreateFlags, + ) std.Io.File.OpenError!std.Io.File { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + + const fs_dir = self.fsDir(dir.handle, sub_path) catch |err| { + return switch (err) { + DirEntryError.SchemeNotRegistered => std.Io.File.OpenError.BadPathName, + DirEntryError.InvalidURI => std.Io.File.OpenError.BadPathName, + DirEntryError.MissingVirtualDirHandle => std.Io.File.OpenError.FileNotFound, + }; + }; + + const fs_file = try fs_dir.backend.vtable.dirCreateFile(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }, sub_path, flags); + + const vfs_handle = self.registerFile(fs_dir.backend, fs_file.handle) catch |err| { + return switch (err) { + std.mem.Allocator.Error.OutOfMemory => std.Io.File.OpenError.SystemFdQuotaExceeded, + }; + }; + + return .{ .handle = vfs_handle }; + } + + fn dirOpenFile( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + flags: std.Io.File.OpenFlags, + ) std.Io.File.OpenError!std.Io.File { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + + const fs_dir = self.fsDir(dir.handle, sub_path) catch |err| { + return switch (err) { + DirEntryError.SchemeNotRegistered => std.Io.File.OpenError.BadPathName, + DirEntryError.InvalidURI => std.Io.File.OpenError.BadPathName, + DirEntryError.MissingVirtualDirHandle => std.Io.File.OpenError.FileNotFound, + }; + }; + + const fs_file = try fs_dir.backend.vtable.dirOpenFile(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }, sub_path, flags); + + const vfs_handle = self.registerFile(fs_dir.backend, fs_file.handle) catch |err| { + return switch (err) { + std.mem.Allocator.Error.OutOfMemory => std.Io.File.OpenError.SystemFdQuotaExceeded, + }; + }; + + return .{ .handle = vfs_handle }; + } + + fn dirOpenDir( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + options: std.Io.Dir.OpenOptions, + ) std.Io.Dir.OpenError!std.Io.Dir { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + + const fs_dir = self.fsDir(dir.handle, sub_path) catch |err| { + return switch (err) { + DirEntryError.SchemeNotRegistered => std.Io.Dir.OpenError.BadPathName, + DirEntryError.InvalidURI => std.Io.Dir.OpenError.BadPathName, + DirEntryError.MissingVirtualDirHandle => std.Io.Dir.OpenError.FileNotFound, + }; + }; + + const fs_sub_dir = try fs_dir.backend.vtable.dirOpenDir(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }, sub_path, options); + + const vfs_handle = self.registerDir(fs_dir.backend, fs_sub_dir.handle) catch |err| { + return switch (err) { + std.mem.Allocator.Error.OutOfMemory => std.Io.Dir.OpenError.SystemFdQuotaExceeded, + }; + }; + + return .{ .handle = vfs_handle }; + } + + fn dirClose( + userdata: ?*anyopaque, + dir: std.Io.Dir, + ) void { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + self.closeDir(dir.handle); + } + + fn fileStat( + userdata: ?*anyopaque, + file: std.Io.File, + ) std.Io.File.StatError!std.Io.File.Stat { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + + const fs_file = self.fsFile(file.handle) catch |err| { + return switch (err) { + FileEntryError.SchemeNotRegistered => std.Io.File.StatError.Unexpected, + FileEntryError.InvalidURI => std.Io.File.StatError.Unexpected, + FileEntryError.MissingVirtualFileHandle => std.Io.File.StatError.Unexpected, + }; + }; + + return fs_file.backend.vtable.fileStat(fs_file.backend.userdata, .{ .handle = fs_file.handle }); + } + + fn fileClose( + userdata: ?*anyopaque, + file: std.Io.File, + ) void { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + self.closeFile(file.handle); + } + + fn fileWriteStreaming( + userdata: ?*anyopaque, + file: std.Io.File, + buffer: [][]const u8, + ) std.Io.File.WriteStreamingError!usize { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + + const fs_file = self.fsFile(file.handle) catch |err| { + return switch (err) { + FileEntryError.SchemeNotRegistered => std.Io.File.WriteStreamingError.Unexpected, + FileEntryError.InvalidURI => std.Io.File.WriteStreamingError.Unexpected, + FileEntryError.MissingVirtualFileHandle => std.Io.File.WriteStreamingError.Unexpected, + }; + }; + + return fs_file.backend.vtable.fileWriteStreaming(fs_file.backend.userdata, .{ .handle = fs_file.handle }, buffer); + } + + fn fileWritePositional( + userdata: ?*anyopaque, + file: std.Io.File, + buffer: [][]const u8, + offset: u64, + ) std.Io.File.WritePositionalError!usize { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + + const fs_file = self.fsFile(file.handle) catch |err| { + return switch (err) { + FileEntryError.SchemeNotRegistered => std.Io.File.WritePositionalError.Unexpected, + FileEntryError.InvalidURI => std.Io.File.WritePositionalError.Unexpected, + FileEntryError.MissingVirtualFileHandle => std.Io.File.WritePositionalError.Unexpected, + }; + }; + + return fs_file.backend.vtable.fileWritePositional(fs_file.backend.userdata, .{ .handle = fs_file.handle }, buffer, offset); + } + + fn fileReadStreaming( + userdata: ?*anyopaque, + file: std.Io.File, + data: [][]u8, + ) std.Io.File.Reader.Error!usize { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + + const fs_file = self.fsFile(file.handle) catch |err| { + return switch (err) { + FileEntryError.SchemeNotRegistered => std.Io.File.Reader.Error.Unexpected, + FileEntryError.InvalidURI => std.Io.File.Reader.Error.Unexpected, + FileEntryError.MissingVirtualFileHandle => std.Io.File.Reader.Error.Unexpected, + }; + }; + + return fs_file.backend.vtable.fileReadStreaming(fs_file.backend.userdata, .{ .handle = fs_file.handle }, data); + } + + fn fileReadPositional( + userdata: ?*anyopaque, + file: std.Io.File, + data: [][]u8, + offset: u64, + ) std.Io.File.ReadPositionalError!usize { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + + const fs_file = self.fsFile(file.handle) catch |err| { + return switch (err) { + FileEntryError.SchemeNotRegistered => std.Io.File.ReadPositionalError.Unexpected, + FileEntryError.InvalidURI => std.Io.File.ReadPositionalError.Unexpected, + FileEntryError.MissingVirtualFileHandle => std.Io.File.ReadPositionalError.Unexpected, + }; + }; + + return fs_file.backend.vtable.fileReadPositional(fs_file.backend.userdata, .{ .handle = fs_file.handle }, data, offset); + } + + fn fileSeekBy( + userdata: ?*anyopaque, + file: std.Io.File, + relative_offset: i64, + ) std.Io.File.SeekError!void { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + + const fs_file = self.fsFile(file.handle) catch |err| { + return switch (err) { + FileEntryError.SchemeNotRegistered => std.Io.File.SeekError.Unexpected, + FileEntryError.InvalidURI => std.Io.File.SeekError.Unexpected, + FileEntryError.MissingVirtualFileHandle => std.Io.File.SeekError.Unexpected, + }; + }; + + return fs_file.backend.vtable.fileSeekBy(fs_file.backend.userdata, .{ .handle = fs_file.handle }, relative_offset); + } + + fn fileSeekTo( + userdata: ?*anyopaque, + file: std.Io.File, + absolute_offset: u64, + ) std.Io.File.SeekError!void { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + + const fs_file = self.fsFile(file.handle) catch |err| { + return switch (err) { + FileEntryError.SchemeNotRegistered => std.Io.File.SeekError.Unexpected, + FileEntryError.InvalidURI => std.Io.File.SeekError.Unexpected, + FileEntryError.MissingVirtualFileHandle => std.Io.File.SeekError.Unexpected, + }; + }; + + return fs_file.backend.vtable.fileSeekTo(fs_file.backend.userdata, .{ .handle = fs_file.handle }, absolute_offset); + } + + fn openSelfExe(userdata: ?*anyopaque, flags: std.Io.File.OpenFlags) std.Io.File.OpenSelfExeError!std.Io.File { + _ = userdata; + _ = flags; + return std.Io.File.OpenSelfExeError.NotSupported; + } +}; diff --git a/zml2/io/vfs/BUILD.bazel b/zml2/io/vfs/BUILD.bazel new file mode 100644 index 00000000..4f1978c4 --- /dev/null +++ b/zml2/io/vfs/BUILD.bazel @@ -0,0 +1,10 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "vfs", + srcs = [ + "http.zig", + ], + main = "index.zig", + visibility = ["//visibility:public"], +) diff --git a/zml2/io/vfs/http.zig b/zml2/io/vfs/http.zig new file mode 100644 index 00000000..85a55d15 --- /dev/null +++ b/zml2/io/vfs/http.zig @@ -0,0 +1,788 @@ +const std = @import("std"); + +const log = std.log.scoped(.@"zml/io/vfs/http"); + +pub const HTTP = struct { + pub const HandleId = i32; + + pub const FileHandle = struct { + allocator: std.mem.Allocator, + + url: []const u8, + pos: u64, + size: ?u64, + redirect_buffer: [1024 * 1024]u8, + request: ?*std.http.Client.Request, + response: ?std.http.Client.Response, + body_reader: ?*std.Io.Reader, + + total_requests: u64 = 0, + total_latency_ns: u64 = 0, + min_latency_ns: u64 = std.math.maxInt(u64), + max_latency_ns: u64 = 0, + + pub fn init(allocator: std.mem.Allocator, parent_path: []const u8, sub_path: []const u8) std.mem.Allocator.Error!*FileHandle { + const handle = try allocator.create(FileHandle); + errdefer allocator.destroy(handle); + + handle.* = .{ + .allocator = allocator, + .url = try resolveUri(allocator, parent_path, sub_path), + .pos = 0, + .size = null, + .redirect_buffer = undefined, + .request = null, + .response = null, + .body_reader = null, + }; + + return handle; + } + + pub fn deinit(self: *FileHandle) void { + self.releaseRequest(); + self.allocator.free(self.url); + self.allocator.destroy(self); + } + + pub fn releaseRequest(self: *FileHandle) void { + if (self.request) |req| { + req.deinit(); + self.allocator.destroy(req); + self.request = null; + self.response = null; + self.body_reader = null; + } + } + + pub fn recordLatency(self: *FileHandle, latency_ns: u64) void { + self.total_requests += 1; + self.total_latency_ns += latency_ns; + self.min_latency_ns = @min(self.min_latency_ns, latency_ns); + self.max_latency_ns = @max(self.max_latency_ns, latency_ns); + } + + pub fn avgLatencyMs(self: *FileHandle) u64 { + if (self.total_requests == 0) return 0; + return (self.total_latency_ns / self.total_requests) / std.time.ns_per_ms; + } + + fn resolveUri(allocator: std.mem.Allocator, parent_path: []const u8, sub_path: []const u8) std.mem.Allocator.Error![]const u8 { + // todo: check uri + if (std.mem.startsWith(u8, sub_path, "http://") or + std.mem.startsWith(u8, sub_path, "https://")) + { + return try allocator.dupe(u8, sub_path); + } + + return std.fs.path.join(allocator, &.{ parent_path, sub_path }); + } + }; + + pub const DirHandle = struct { + allocator: std.mem.Allocator, + url: []const u8, + + pub fn init(allocator: std.mem.Allocator, parent_path: []const u8, sub_path: []const u8) std.mem.Allocator.Error!*DirHandle { + const self = try allocator.create(DirHandle); + errdefer allocator.destroy(self); + + const url = try std.fs.path.join(allocator, &.{ parent_path, sub_path }); + errdefer allocator.free(url); + + self.* = .{ + .allocator = allocator, + .url = url, + }; + + return self; + } + + pub fn deinit(self: *DirHandle) void { + self.allocator.free(self.url); + self.allocator.destroy(self); + } + }; + + pub const Config = struct { + max_fetch_attempts: u32 = 10, + request_range_size: u64 = 128 * 1024 * 1024, + }; + + allocator: std.mem.Allocator, + mutex: std.Thread.Mutex, + + file_handles: std.AutoHashMapUnmanaged(HandleId, *FileHandle), + dir_handles: std.AutoHashMapUnmanaged(HandleId, *DirHandle), + next_file_handle_id: HandleId, + next_dir_handle_id: HandleId, + + client: *std.http.Client, + config: Config, + + vtable: std.Io.VTable, + + pub fn init(allocator: std.mem.Allocator, base_io: std.Io, http_client: *std.http.Client, config: Config) !HTTP { + var self: HTTP = .{ + .allocator = allocator, + .mutex = .{}, + .file_handles = .{}, + .dir_handles = .{}, + .next_file_handle_id = 0, + .next_dir_handle_id = 0, + .client = http_client, + .config = config, + .vtable = makeVTable(base_io), + }; + + // Used by VFS + const root_dir_handle = DirHandle.init(allocator, "", "") catch { + log.err("Failed to create root DirHandle", .{}); + return self; + }; + errdefer root_dir_handle.deinit(); + + _ = try self.registerDirHandle(root_dir_handle); + + return self; + } + + pub fn deinit(self: *HTTP) void { + var file_it = self.file_handles.valueIterator(); + while (file_it.next()) |handle_ptr| { + handle_ptr.*.deinit(); + } + self.file_handles.deinit(self.allocator); + + var dir_it = self.dir_handles.valueIterator(); + while (dir_it.next()) |handle_ptr| { + handle_ptr.*.deinit(); + } + self.dir_handles.deinit(self.allocator); + } + + pub fn io(self: *HTTP) std.Io { + return .{ + .vtable = &self.vtable, + .userdata = self, + }; + } + + fn registerFileHandle(self: *HTTP, handle: *FileHandle) std.mem.Allocator.Error!HandleId { + self.mutex.lock(); + defer self.mutex.unlock(); + + const handle_id = self.next_file_handle_id; + self.next_file_handle_id += 1; + + try self.file_handles.put(self.allocator, handle_id, handle); + + return handle_id; + } + + fn fileHandle(self: *HTTP, file: std.Io.File) ?*FileHandle { + self.mutex.lock(); + defer self.mutex.unlock(); + + return self.file_handles.get(file.handle); + } + + fn registerDirHandle(self: *HTTP, handle: *DirHandle) std.mem.Allocator.Error!HandleId { + self.mutex.lock(); + defer self.mutex.unlock(); + + const handle_id = self.next_dir_handle_id; + self.next_dir_handle_id += 1; + + try self.dir_handles.put(self.allocator, handle_id, handle); + + return handle_id; + } + + fn dirHandle(self: *HTTP, dir: std.Io.Dir) ?*DirHandle { + self.mutex.lock(); + defer self.mutex.unlock(); + + return self.dir_handles.get(dir.handle); + } + + fn openFile( + self: *HTTP, + dir: std.Io.Dir, + sub_path: []const u8, + ) std.Io.File.OpenError!std.Io.File { + const dir_handle = self.dirHandle(dir) orelse { + log.err("Directory handle not found for dirOpenFile", .{}); + return std.Io.File.OpenError.FileNotFound; + }; + + const handle = FileHandle.init(self.allocator, dir_handle.url, sub_path) catch { + log.err("Failed to create FileHandle in dirOpenFile", .{}); + return std.Io.File.OpenError.SystemResources; + }; + errdefer handle.deinit(); + + const handle_id = self.registerFileHandle(handle) catch return error.SystemResources; + + return .{ .handle = @intCast(handle_id) }; + } + + fn openDir( + self: *HTTP, + dir: std.Io.Dir, + sub_path: []const u8, + ) std.Io.Dir.OpenError!std.Io.Dir { + const dir_handle = self.dirHandle(dir) orelse { + log.err("Directory handle not found for dirOpenFile", .{}); + return std.Io.File.OpenError.FileNotFound; + }; + + const handle = DirHandle.init(self.allocator, dir_handle.url, sub_path) catch { + log.err("Failed to create DirHandle in dirOpenFile", .{}); + return std.Io.Dir.OpenError.SystemResources; + }; + errdefer handle.deinit(); + + const handle_id = self.registerDirHandle(handle) catch return error.SystemResources; + + return .{ .handle = @intCast(handle_id) }; + } + + fn closeDir(self: *HTTP, dir: std.Io.Dir) void { + self.mutex.lock(); + defer self.mutex.unlock(); + + const handle_kv = self.dir_handles.fetchRemove(dir.handle) orelse { + log.warn("Attempted to close non-existent dir handle: {d}", .{dir.handle}); + return; + }; + + handle_kv.value.deinit(); + } + + fn closeFile(self: *HTTP, file: std.Io.File) void { + self.mutex.lock(); + defer self.mutex.unlock(); + + const handle_kv = self.file_handles.fetchRemove(file.handle) orelse { + log.warn("Attempted to close non-existent file handle: {d}", .{file.handle}); + return; + }; + + handle_kv.value.deinit(); + } + + fn setAbsolutePos(self: *HTTP, file: std.Io.File, pos: u64) std.Io.File.SeekError!void { + const handle = self.fileHandle(file) orelse return error.Unexpected; + handle.releaseRequest(); + handle.pos = pos; + } + + fn setRelativeOffset(self: *HTTP, file: std.Io.File, offset: i64) std.Io.File.SeekError!void { + const handle = self.fileHandle(file) orelse return error.Unexpected; + handle.releaseRequest(); + + const new_pos = @as(i64, @intCast(handle.pos)) + offset; + handle.pos = @intCast(new_pos); + } + + fn performPositionalRead(self: *HTTP, file: std.Io.File, data: [][]u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const handle = self.fileHandle(file) orelse return std.Io.File.ReadPositionalError.Unexpected; + + const saved_pos = handle.pos; + + if (handle.pos != offset) { + handle.releaseRequest(); + handle.pos = offset; + } + + const bytes_read = try self.performRead(file, data); + + handle.pos = saved_pos; + handle.releaseRequest(); + + return bytes_read; + } + + fn performRead(self: *HTTP, file: std.Io.File, data: [][]u8) std.Io.File.Reader.Error!usize { + const handle = self.fileHandle(file) orelse { + log.err("Invalid file handle in HTTP VFS read", .{}); + return std.Io.File.Reader.Error.Unexpected; + }; + + if (data.len == 0) return 0; + + var total_read: usize = 0; + + for (data) |buf| { + var offset: usize = 0; + + while (offset < buf.len) { + // Check EOF against known file size + if (handle.size) |size| { + if (handle.pos >= size) { + return total_read; + } + } + + try self.requestRead(file); + const reader = handle.body_reader orelse { + return total_read; // todo: handle this properly + }; + + const dest = buf[offset..]; + var iov = [_][]u8{dest}; + + const n = reader.readVec(&iov) catch |err| switch (err) { + error.EndOfStream => { + // Current HTTP range exhausted, need new request + handle.releaseRequest(); + continue; + }, + else => { + return std.Io.File.Reader.Error.Unexpected; + }, + }; + + // Data not ready yet, just retry same read + if (n == 0) { + continue; + } + + offset += n; + handle.pos += n; + total_read += n; + } + } + + return total_read; + } + + fn requestRead(self: *HTTP, file: std.Io.File) std.Io.File.Reader.Error!void { + const handle = self.fileHandle(file) orelse return error.Unexpected; + + if (handle.body_reader != null) return; + + if (handle.size == null) { + const stat = self.fetchFileStat(file) catch |err| { + log.err("Failed to fetch file stat during read: {}", .{err}); + return std.Io.File.Reader.Error.SystemResources; + }; + handle.size = stat.size; + } + + const size = handle.size.?; + + if (handle.pos >= size) return; + + const start = handle.pos; + const end = @min(start + self.config.request_range_size, size) - 1; + + var range_buf: [64]u8 = undefined; + const range_header = std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ start, end }) catch unreachable; + + var timer = std.time.Timer.start() catch { + log.err("Failed to start timer for HTTP request", .{}); + return std.Io.File.Reader.Error.SystemResources; + }; + + const request = self.allocator.create(std.http.Client.Request) catch { + log.err("Failed to allocate HTTP request", .{}); + return std.Io.File.Reader.Error.SystemResources; + }; + errdefer self.allocator.destroy(request); + + const uri = std.Uri.parse(handle.url) catch |err| { + log.err("Failed to parse URL during read request: {}", .{err}); + return std.Io.File.Reader.Error.Unexpected; + }; + + request.* = self.client.request(.GET, uri, .{ + .headers = .{ .accept_encoding = .{ .override = "identity" } }, + .extra_headers = &.{.{ .name = "Range", .value = range_header }}, + }) catch |err| { + log.err("Failed to create HTTP GET request: {}", .{err}); + return std.Io.File.Reader.Error.SystemResources; + }; + errdefer request.deinit(); + + request.sendBodiless() catch |err| { + log.err("Failed to send HTTP GET request: {}", .{err}); + return std.Io.File.Reader.Error.SystemResources; + }; + + const response = request.receiveHead(&handle.redirect_buffer) catch |err| { + log.err("Failed to receive HTTP response head: {}", .{err}); + return std.Io.File.Reader.Error.Unexpected; + }; + + const latency = timer.read(); + handle.recordLatency(latency); + + log.debug("HTTP request: range={s} latency={d}ms avg={d}ms min={d}ms max={d}ms for {s}", .{ + range_header, + latency / std.time.ns_per_ms, + handle.avgLatencyMs(), + handle.min_latency_ns / std.time.ns_per_ms, + handle.max_latency_ns / std.time.ns_per_ms, + handle.url, + }); + + if (response.head.status != .partial_content and response.head.status != .ok) { + request.deinit(); + self.allocator.destroy(request); + return error.Unexpected; + } + + handle.request = request; + handle.response = response; + handle.body_reader = handle.response.?.reader(&.{}); + } + + pub const FileStat = struct { + inode: u64, + size: u64, + }; + + fn fetchFileStat(self: *HTTP, file: std.Io.File) std.Io.File.StatError!FileStat { + const handle = self.fileHandle(file) orelse return error.Unexpected; + + var attempt: u32 = 0; + while (attempt < self.config.max_fetch_attempts) : (attempt += 1) { + const req = self.allocator.create(std.http.Client.Request) catch continue; + errdefer self.allocator.destroy(req); + + req.* = self.client.request(.HEAD, std.Uri.parse(handle.url) catch return error.Unexpected, .{ + .headers = .{ .accept_encoding = .{ .override = "identity" } }, + }) catch { + self.allocator.destroy(req); + continue; + }; + + req.sendBodiless() catch { + req.deinit(); + self.allocator.destroy(req); + continue; + }; + + var response = req.receiveHead(&handle.redirect_buffer) catch { + req.deinit(); + self.allocator.destroy(req); + continue; + }; + + defer { + req.deinit(); + self.allocator.destroy(req); + } + + switch (response.head.status) { + .moved_permanently, .found, .see_other, .temporary_redirect, .permanent_redirect => { + if (response.head.location) |location| { + var buf: [std.fs.max_path_bytes]u8 = undefined; + + if (location.len > buf.len) return std.Io.File.StatError.Unexpected; + @memcpy(buf[0..location.len], location); + + var aux_buf: []u8 = buf[0..]; + + const uri = std.Uri.parse(handle.url) catch |err| { + log.err("Failed to parse URL during redirect: {}", .{err}); + return std.Io.File.StatError.Unexpected; + }; + + const resolved = std.Uri.resolveInPlace(uri, location.len, &aux_buf) catch |err| { + log.err("Failed to resolve redirect URL: {}", .{err}); + return std.Io.File.StatError.Unexpected; + }; + const new_url = std.fmt.allocPrint(self.allocator, "{f}", .{resolved.fmt(.{ + .scheme = true, + .authority = true, + .path = true, + .query = true, + .fragment = true, + })}) catch |err| { + log.err("Failed to allocate new URL during redirect: {}", .{err}); + return std.Io.File.StatError.SystemResources; + }; + + self.allocator.free(handle.url); + handle.url = new_url; + + continue; + } + }, + .ok, .partial_content => { + const size = response.head.content_length orelse { + log.err("HTTP stat response missing Content-Length", .{}); + return std.Io.File.StatError.SystemResources; + }; + handle.size = size; + + return .{ + .inode = @intCast(file.handle), + .size = size, + }; + }, + .internal_server_error, .bad_gateway, .service_unavailable, .gateway_timeout => { + std.Io.sleep(self.io(), .{ .nanoseconds = 50 * std.time.ns_per_ms }, .real) catch |err| { + log.err("Sleep failed during HTTP stat retry: {}", .{err}); + }; + continue; + }, + else => { + log.err("HTTP stat failed with status: {s}", .{@tagName(response.head.status)}); + return std.Io.File.StatError.Unexpected; + }, + } + } + + log.err("HTTP stat failed after {d} attempts", .{self.config.max_fetch_attempts}); + return std.Io.File.StatError.Unexpected; + } + + fn makeVTable(base_io: std.Io) std.Io.VTable { + var vtable = base_io.vtable.*; + vtable.dirMake = dirMake; + vtable.dirMakePath = dirMakePath; + vtable.dirMakeOpenPath = dirMakeOpenPath; + vtable.dirStat = dirStat; + vtable.dirStatPath = dirStatPath; + vtable.dirAccess = dirAccess; + vtable.dirCreateFile = dirCreateFile; + vtable.dirOpenFile = dirOpenFile; + vtable.dirOpenDir = dirOpenDir; + vtable.dirClose = dirClose; + vtable.fileStat = fileStat; + vtable.fileClose = fileClose; + vtable.fileWriteStreaming = fileWriteStreaming; + vtable.fileWritePositional = fileWritePositional; + vtable.fileReadStreaming = fileReadStreaming; + vtable.fileReadPositional = fileReadPositional; + vtable.fileSeekBy = fileSeekBy; + vtable.fileSeekTo = fileSeekTo; + vtable.openSelfExe = openSelfExe; + + return vtable; + } + + fn dirMake( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + mode: std.Io.Dir.Mode, + ) std.Io.Dir.MakeError!void { + _ = userdata; + _ = dir; + _ = sub_path; + _ = mode; + log.err("HTTP VFS is read-only, dirMake not supported", .{}); + return std.Io.Dir.MakeError.ReadOnlyFileSystem; + } + + fn dirMakePath( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + mode: std.Io.Dir.Mode, + ) std.Io.Dir.MakeError!void { + _ = userdata; + _ = dir; + _ = sub_path; + _ = mode; + log.err("HTTP VFS is read-only, dirMakePath not supported", .{}); + return std.Io.Dir.MakeError.ReadOnlyFileSystem; + } + + fn dirMakeOpenPath( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + options: std.Io.Dir.OpenOptions, + ) std.Io.Dir.MakeOpenPathError!std.Io.Dir { + _ = userdata; + _ = dir; + _ = sub_path; + _ = options; + log.err("HTTP VFS is read-only, dirMakeOpenPath not supported", .{}); + return error.Unexpected; + } + + fn dirStat( + userdata: ?*anyopaque, + dir: std.Io.Dir, + ) std.Io.Dir.StatError!std.Io.Dir.Stat { + _ = userdata; + _ = dir; + log.err("HTTP VFS does not support dirStat", .{}); + return std.Io.Dir.StatError.AccessDenied; + } + + fn dirStatPath( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + options: std.Io.Dir.StatPathOptions, + ) std.Io.Dir.StatPathError!std.Io.File.Stat { + _ = userdata; + _ = dir; + _ = sub_path; + _ = options; + @panic("TODO: implement HTTP VFS dirStatPath"); + } + + fn dirAccess( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + options: std.Io.Dir.AccessOptions, + ) std.Io.Dir.AccessError!void { + _ = userdata; + _ = dir; + _ = sub_path; + + if (options.write) { + return std.Io.Dir.AccessError.ReadOnlyFileSystem; + } + + log.err("HTTP VFS does not support dirAccess", .{}); + return std.Io.Dir.AccessError.AccessDenied; + } + + fn dirCreateFile( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + flags: std.Io.File.CreateFlags, + ) std.Io.File.OpenError!std.Io.File { + _ = userdata; + _ = dir; + _ = sub_path; + _ = flags; + log.err("HTTP VFS is read-only, dirCreateFile not supported", .{}); + return std.Io.File.OpenError.Unexpected; + } + + fn dirOpenFile( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + flags: std.Io.File.OpenFlags, + ) std.Io.File.OpenError!std.Io.File { + _ = flags; + const self: *HTTP = @ptrCast(@alignCast(userdata orelse unreachable)); + return try self.openFile(dir, sub_path); + } + + fn dirOpenDir( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + options: std.Io.Dir.OpenOptions, + ) std.Io.Dir.OpenError!std.Io.Dir { + _ = options; + const self: *HTTP = @ptrCast(@alignCast(userdata orelse unreachable)); + return try self.openDir(dir, sub_path); + } + + fn dirClose(userdata: ?*anyopaque, dir: std.Io.Dir) void { + const self: *HTTP = @ptrCast(@alignCast(userdata orelse unreachable)); + self.closeDir(dir); + } + + fn fileStat( + userdata: ?*anyopaque, + file: std.Io.File, + ) std.Io.File.StatError!std.Io.File.Stat { + const self: *HTTP = @ptrCast(@alignCast(userdata orelse unreachable)); + const stat = try self.fetchFileStat(file); + + return .{ + .inode = stat.inode, + .kind = .file, + .size = stat.size, + .mode = 0o444, + .atime = .{ .nanoseconds = 0 }, + .mtime = .{ .nanoseconds = 0 }, + .ctime = .{ .nanoseconds = 0 }, + }; + } + + fn fileClose(userdata: ?*anyopaque, file: std.Io.File) void { + const self: *HTTP = @ptrCast(@alignCast(userdata orelse unreachable)); + self.closeFile(file); + } + + fn fileWriteStreaming( + userdata: ?*anyopaque, + file: std.Io.File, + buffer: [][]const u8, + ) std.Io.File.WriteStreamingError!usize { + _ = userdata; + _ = file; + _ = buffer; + log.err("HTTP VFS is read-only, fileWriteStreaming not supported", .{}); + return std.Io.File.WriteStreamingError.Unexpected; + } + + fn fileWritePositional( + userdata: ?*anyopaque, + file: std.Io.File, + buffer: [][]const u8, + offset: u64, + ) std.Io.File.WritePositionalError!usize { + _ = userdata; + _ = file; + _ = buffer; + _ = offset; + log.err("HTTP VFS is read-only, fileWritePositional not supported", .{}); + return std.Io.File.WritePositionalError.Unexpected; + } + + fn fileReadStreaming( + userdata: ?*anyopaque, + file: std.Io.File, + data: [][]u8, + ) std.Io.File.Reader.Error!usize { + const self: *HTTP = @ptrCast(@alignCast(userdata orelse unreachable)); + return try self.performRead(file, data); + } + + fn fileReadPositional( + userdata: ?*anyopaque, + file: std.Io.File, + data: [][]u8, + offset: u64, + ) std.Io.File.ReadPositionalError!usize { + const self: *HTTP = @ptrCast(@alignCast(userdata orelse unreachable)); + return try self.performPositionalRead(file, data, offset); + } + + fn fileSeekBy( + userdata: ?*anyopaque, + file: std.Io.File, + relative_offset: i64, + ) std.Io.File.SeekError!void { + const self: *HTTP = @ptrCast(@alignCast(userdata orelse unreachable)); + try self.setRelativeOffset(file, relative_offset); + } + + fn fileSeekTo( + userdata: ?*anyopaque, + file: std.Io.File, + absolute_offset: u64, + ) std.Io.File.SeekError!void { + const self: *HTTP = @ptrCast(@alignCast(userdata orelse unreachable)); + try self.setAbsolutePos(file, absolute_offset); + } + + fn openSelfExe( + userdata: ?*anyopaque, + flags: std.Io.File.OpenFlags, + ) std.Io.File.OpenSelfExeError!std.Io.File { + _ = userdata; + _ = flags; + log.err("HTTP VFS does not support openSelfExe", .{}); + return std.Io.File.OpenSelfExeError.NotSupported; + } +}; diff --git a/zml2/io/vfs/index.zig b/zml2/io/vfs/index.zig new file mode 100644 index 00000000..621077a5 --- /dev/null +++ b/zml2/io/vfs/index.zig @@ -0,0 +1,5 @@ +const std = @import("std"); + +const log = std.log.scoped(.@"zml/io/vfs/vfs"); + +pub const HTTP = @import("http.zig").HTTP; diff --git a/zml2/main_io.zig b/zml2/main_io.zig new file mode 100644 index 00000000..d4a94a2f --- /dev/null +++ b/zml2/main_io.zig @@ -0,0 +1,208 @@ +const builtin = @import("builtin"); +const std = @import("std"); + +const zml = @import("zml"); + +const log = std.log.scoped(.@"zml/main_io"); + +pub fn main() !void { + var debug_allocator: std.heap.DebugAllocator(.{}) = .init; + defer std.debug.assert(debug_allocator.deinit() == .ok); + + const allocator = debug_allocator.allocator(); + + var threaded: std.Io.Threaded = .init(allocator); + defer threaded.deinit(); + + if (try std.process.hasEnvVar(allocator, "ASYNC_LIMIT")) { + const limit = try std.process.getEnvVarOwned(allocator, "ASYNC_LIMIT"); + defer allocator.free(limit); + + threaded.async_limit = .limited(try std.fmt.parseInt(usize, limit, 10)); + } + + log.debug("Running with threaded async limit: {?d}", .{threaded.async_limit.toInt()}); + + var http_client: std.http.Client = .{ + .allocator = allocator, + .io = threaded.io(), + .connection_pool = .{ + .free_size = threaded.async_limit.toInt() orelse 16, + }, + }; + + try http_client.initDefaultProxies(allocator); + defer http_client.deinit(); + + var vfs_http: zml.io.VFS.HTTP = try .init(allocator, threaded.io(), &http_client, .{}); + defer vfs_http.deinit(); + + var vfs: zml.io.VFS = .init(allocator, threaded.io()); + defer vfs.deinit(); + + try vfs.register("http", vfs_http.io()); + try vfs.register("https", vfs_http.io()); + + const io = vfs.io(); + + const repo_uri = "https://storage.googleapis.com/zig-vfs/Llama-3.1-8B-Instruct/"; + // const repo_uri = "http://9960x-5090x2:8003/"; + + { + const root_dir = try vfs.openAbsoluteDir(io, "https://storage.googleapis.com", .{}); + defer root_dir.close(io); + + const buckets_dir = try root_dir.openDir(io, "zig-vfs", .{}); + defer buckets_dir.close(io); + + const model_dir = try buckets_dir.openDir(io, "Llama-3.1-8B-Instruct", .{}); + defer model_dir.close(io); + + const file = try model_dir.openFile(io, "model.safetensors.index.json", .{}); + defer file.close(io); + + const stat = try file.stat(io); + log.info("Opened remote file with size: {d} bytes by navigating dirs", .{stat.size}); + } + + { + const path = try std.fs.path.join(allocator, &.{ repo_uri, "model-00004-of-00004.safetensors" }); + defer allocator.free(path); + + const file = try vfs.openAbsoluteFile(io, path, .{}); + defer file.close(io); + + const stat = try file.stat(io); + + var group: std.Io.Group = .init; + defer group.cancel(io); + + const chunk_size: usize = 32 * 1024 * 1024; + const num_chunks = (stat.size + chunk_size - 1) / chunk_size; + + for (0..num_chunks) |i| { + const offset = i * chunk_size; + const size = if (offset + chunk_size > stat.size) stat.size - offset else chunk_size; + + group.async(threaded.io(), readData, .{ allocator, io, &vfs, path, offset, size, i }); + } + + group.wait(io); + } + + { + const path = try std.fs.path.join(allocator, &.{ repo_uri, "model.safetensors.index.json" }); + defer allocator.free(path); + + const file = try vfs.openAbsoluteFile(io, path, .{}); + defer file.close(io); + + const stat = try file.stat(io); + log.info("Opened remote file with size: {d} bytes with openAbsoluteFile", .{stat.size}); + } + + { + const model_dir = try vfs.openAbsoluteDir(io, repo_uri, .{}); + + const safetensors_index = try model_dir.openFile(io, "model.safetensors.index.json", .{}); + defer safetensors_index.close(io); + + const stat = try safetensors_index.stat(io); + log.info("Opened remote index file with size: {d} bytes with openAbsoluteDir + openFile", .{stat.size}); + } + + { + const path = try std.fs.path.join(allocator, &.{ repo_uri, "model-00004-of-00004.safetensors" }); + defer allocator.free(path); + + const file = try vfs.openAbsoluteFile(io, path, .{}); + defer file.close(io); + + var reader = file.reader(io, &.{}); + + const read_size = 16 * 1024 * 1024; + + var allocating_writer: std.Io.Writer.Allocating = try .initCapacity(allocator, read_size); + defer allocating_writer.deinit(); + + const read = try reader.interface.stream(&allocating_writer.writer, .limited(read_size)); + std.debug.assert(read == read_size); + } + + { + const path = try std.fs.path.join(allocator, &.{ repo_uri, "model-00004-of-00004.safetensors" }); // SHA256: 92ecfe1a2414458b4821ac8c13cf8cb70aed66b5eea8dc5ad9eeb4ff309d6d7b + defer allocator.free(path); + + const file = try vfs.openAbsoluteFile(io, path, .{}); + defer file.close(io); + + const stat = try file.stat(io); + + const buf_size = 64 * 1024 * 1024; + const buf = try allocator.alloc(u8, buf_size); + defer allocator.free(buf); + + var sha256: std.crypto.hash.sha2.Sha256 = .init(.{}); + const compute_sha = true; + + // Read directly without extra buffering layers + var total_read: usize = 0; + var bufs = [_][]u8{buf}; + + while (true) { + const n = try io.vtable.fileReadStreaming(io.userdata, file, &bufs); + if (n == 0) break; + if (compute_sha) sha256.update(buf[0..n]); + total_read += n; + } + + if (compute_sha) { + var hash: [32]u8 = undefined; + sha256.final(&hash); + log.info("SHA256: {x}", .{hash}); + } + + std.debug.assert(total_read == stat.size); + } +} + +fn readData( + allocator: std.mem.Allocator, + io: std.Io, + vfs: *zml.io.VFS, + uri: []const u8, + offset: usize, + size: usize, + chunk_index: usize, +) void { + const buffer = allocator.alloc(u8, size) catch { + log.err("Failed to allocate buffer of size {d} bytes for chunk {d}", .{ size, chunk_index }); + return; + }; + defer allocator.free(buffer); + + var file = vfs.openAbsoluteFile(io, uri, .{}) catch { + log.err("Failed to open file at {s}", .{uri}); + return; + }; + + var reader = file.reader(io, &.{}); + + reader.seekTo(@intCast(offset)) catch { + log.err("Failed to seek to offset {d} for chunk {d}", .{ offset, chunk_index }); + return; + }; + + var total_read: usize = 0; + while (total_read < size) { + const n = reader.interface.readSliceShort(buffer[total_read..]) catch { + log.err("Failed to read data for chunk {d} at offset {d}", .{ chunk_index, offset + total_read }); + return; + }; + if (n == 0) break; + total_read += n; + } + + std.debug.assert(total_read == size); + log.info("Read chunk {d} of size {d} bytes at offset {d}", .{ chunk_index, size, offset }); +} diff --git a/zml2/zml.zig b/zml2/zml.zig index d9db13b3..ed8f8ddc 100644 --- a/zml2/zml.zig +++ b/zml2/zml.zig @@ -21,6 +21,7 @@ pub const mlir = @import("mlirx.zig"); pub const module = @import("module.zig"); pub const nn = @import("nn.zig"); pub const pjrt = @import("pjrtx.zig"); +pub const io = @import("io.zig"); pub const platform = @import("platform.zig"); pub const Platform = platform.Platform; pub const Target = platform.Target; @@ -40,15 +41,15 @@ var runfiles_once = std.once(struct { return; } - var io: std.Io.Threaded = .init_single_threaded; - defer io.deinit(); + var io_: std.Io.Threaded = .init_single_threaded; + defer io_.deinit(); var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); const allocator = arena.allocator(); defer arena.deinit(); var envMap = std.process.EnvMap.init(allocator); - var r = (try runfiles.Runfiles.create(.{ .allocator = allocator, .io = io.io() })) orelse return; + var r = (try runfiles.Runfiles.create(.{ .allocator = allocator, .io = io_.io() })) orelse return; try r.environment(&envMap); var it = envMap.iterator(); From cac4f38d03a35420e43028b4937f7cb047887955 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 11 Dec 2025 17:44:04 +0100 Subject: [PATCH 123/881] Remove async package --- BUILD.bazel | 1 - async/BUILD.bazel | 27 -- async/asm/coro_aarch64.s | 49 --- async/asm/coro_riscv64.s | 77 ---- async/asm/coro_x86_64.s | 30 -- async/asm/coro_x86_64_windows.s | 65 --- async/async.zig | 713 -------------------------------- async/asyncio.zig | 659 ----------------------------- async/channel.zig | 75 ---- async/coro.zig | 529 ------------------------ async/coro_base.zig | 78 ---- async/executor.zig | 182 -------- async/stack.zig | 83 ---- 13 files changed, 2568 deletions(-) delete mode 100644 async/BUILD.bazel delete mode 100644 async/asm/coro_aarch64.s delete mode 100644 async/asm/coro_riscv64.s delete mode 100644 async/asm/coro_x86_64.s delete mode 100644 async/asm/coro_x86_64_windows.s delete mode 100644 async/async.zig delete mode 100644 async/asyncio.zig delete mode 100644 async/channel.zig delete mode 100644 async/coro.zig delete mode 100644 async/coro_base.zig delete mode 100644 async/executor.zig delete mode 100644 async/stack.zig diff --git a/BUILD.bazel b/BUILD.bazel index 34e18158..faf47f30 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -6,7 +6,6 @@ zls_completion( name = "completion", visibility = ["//visibility:public"], deps = [ - "//async", #"//examples/llama", "//stdx", "//zml2:zml", diff --git a/async/BUILD.bazel b/async/BUILD.bazel deleted file mode 100644 index 647ac4d4..00000000 --- a/async/BUILD.bazel +++ /dev/null @@ -1,27 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "async", - srcs = [ - "asyncio.zig", - "channel.zig", - "coro.zig", - "coro_base.zig", - "executor.zig", - "stack.zig", - ], - zigopts = ["-lc"], - extra_srcs = glob(["asm/*.s"]), - main = "async.zig", - visibility = ["//visibility:public"], - deps = [ - "//stdx", - "@libxev", - ], -) - -zig_test( - name = "test", - testonly = False, - deps = [":async"], -) diff --git a/async/asm/coro_aarch64.s b/async/asm/coro_aarch64.s deleted file mode 100644 index 99485f80..00000000 --- a/async/asm/coro_aarch64.s +++ /dev/null @@ -1,49 +0,0 @@ -# See arm64 procedure call standard -# https://github.com/ARM-software/abi-aa/releases -.global _libcoro_stack_swap -_libcoro_stack_swap: -.global libcoro_stack_swap -libcoro_stack_swap: - -# Store caller registers on the current stack -# Each register requires 8 bytes, there are 20 registers to save -sub sp, sp, 0xa0 -# d* are the 128-bit floating point registers, the lower 64 bits are preserved -stp d8, d9, [sp, 0x00] -stp d10, d11, [sp, 0x10] -stp d12, d13, [sp, 0x20] -stp d14, d15, [sp, 0x30] -# x* are the scratch registers -stp x19, x20, [sp, 0x40] -stp x21, x22, [sp, 0x50] -stp x23, x24, [sp, 0x60] -stp x25, x26, [sp, 0x70] -stp x27, x28, [sp, 0x80] -# fp=frame pointer, lr=link register -stp fp, lr, [sp, 0x90] - -# Modify stack pointer of current coroutine (x0, first argument) -mov x2, sp -str x2, [x0, 0] - -# Load stack pointer from target coroutine (x1, second argument) -ldr x9, [x1, 0] -mov sp, x9 - -# Restore target registers -ldp d8, d9, [sp, 0x00] -ldp d10, d11, [sp, -0x10] -ldp d12, d13, [sp, 0x20] -ldp d14, d15, [sp, 0x30] -ldp x19, x20, [sp, 0x40] -ldp x21, x22, [sp, 0x50] -ldp x23, x24, [sp, 0x60] -ldp x25, x26, [sp, 0x70] -ldp x27, x28, [sp, 0x80] -ldp fp, lr, [sp, 0x90] - -# Pop stack frame -add sp, sp, 0xa0 - -# jump to lr -ret diff --git a/async/asm/coro_riscv64.s b/async/asm/coro_riscv64.s deleted file mode 100644 index 49cce751..00000000 --- a/async/asm/coro_riscv64.s +++ /dev/null @@ -1,77 +0,0 @@ -# See riscv procedure calling convention -# https://github.com/riscv-non-isa/riscv-elf-psabi-doc -.global libcoro_stack_swap -libcoro_stack_swap: - -# Store caller registers on the current stack -# Each register requires 8 bytes, there are 25 registers to save -addi sp, sp, -0xc8 -# s* are integer callee-saved registers -sd s0, 0x00(sp) -sd s1, 0x08(sp) -sd s2, 0x10(sp) -sd s3, 0x18(sp) -sd s4, 0x20(sp) -sd s5, 0x28(sp) -sd s6, 0x30(sp) -sd s7, 0x38(sp) -sd s8, 0x40(sp) -sd s9, 0x48(sp) -sd s10, 0x50(sp) -sd s11, 0x58(sp) -# fs* are float callee-saved registers -fsd fs0, 0x60(sp) -fsd fs1, 0x68(sp) -fsd fs2, 0x70(sp) -fsd fs3, 0x78(sp) -fsd fs4, 0x80(sp) -fsd fs5, 0x88(sp) -fsd fs6, 0x90(sp) -fsd fs7, 0x98(sp) -fsd fs8, 0xa0(sp) -fsd fs9, 0xa8(sp) -fsd fs10, 0xb0(sp) -fsd fs11, 0xb8(sp) -# ra=return address -sd ra, 0xc0(sp) - -# Modify stack pointer of current coroutine (a0, first argument) -mv t0, sp -sd t0, 0(a0) - -# Load stack pointer from target coroutine (a1, second argument) -ld t1, 0(a1) -mv sp, t1 - -# Restore -ld s0, 0x00(sp) -ld s1, 0x08(sp) -ld s2, 0x10(sp) -ld s3, 0x18(sp) -ld s4, 0x20(sp) -ld s5, 0x28(sp) -ld s6, 0x30(sp) -ld s7, 0x38(sp) -ld s8, 0x40(sp) -ld s9, 0x48(sp) -ld s10, 0x50(sp) -ld s11, 0x58(sp) -fld fs0, 0x60(sp) -fld fs1, 0x68(sp) -fld fs2, 0x70(sp) -fld fs3, 0x78(sp) -fld fs4, 0x80(sp) -fld fs5, 0x88(sp) -fld fs6, 0x90(sp) -fld fs7, 0x98(sp) -fld fs8, 0xa0(sp) -fld fs9, 0xa8(sp) -fld fs10, 0xb0(sp) -fld fs11, 0xb8(sp) -ld ra, 0xc0(sp) - -# Pop stack frame -addi sp, sp, 0xc8 - -# jump to ra -ret diff --git a/async/asm/coro_x86_64.s b/async/asm/coro_x86_64.s deleted file mode 100644 index d7cd3e89..00000000 --- a/async/asm/coro_x86_64.s +++ /dev/null @@ -1,30 +0,0 @@ -# See System V x86-64 calling convention -# https://gitlab.com/x86-psABIs/x86-64-ABI -.global _libcoro_stack_swap -_libcoro_stack_swap: -.global libcoro_stack_swap -libcoro_stack_swap: -# Store caller registers on the current stack -pushq %rbp -pushq %rbx -pushq %r12 -pushq %r13 -pushq %r14 -pushq %r15 - -# Modify stack pointer of current coroutine (rdi, first argument) -movq %rsp, (%rdi) - -# Load stack pointer from target coroutine (rsi, second argument) -movq (%rsi), %rsp - -# Restore target registers -popq %r15 -popq %r14 -popq %r13 -popq %r12 -popq %rbx -popq %rbp - -# jump -retq diff --git a/async/asm/coro_x86_64_windows.s b/async/asm/coro_x86_64_windows.s deleted file mode 100644 index d38e6e69..00000000 --- a/async/asm/coro_x86_64_windows.s +++ /dev/null @@ -1,65 +0,0 @@ -# See Microsoft x86-64 calling convention -# https://learn.microsoft.com/en-us/cpp/build/x64-calling-convention -.global libcoro_stack_swap -libcoro_stack_swap: -# Store Windows stack information -pushq %gs:0x10 -pushq %gs:0x08 - -# Store caller registers -pushq %rbp -pushq %rbx -pushq %rdi -pushq %rsi -pushq %r12 -pushq %r13 -pushq %r14 -pushq %r15 - -# Store caller simd/float registers -subq $0xa0, %rsp -movups %xmm6, 0x00(%rsp) -movups %xmm7, 0x10(%rsp) -movups %xmm8, 0x20(%rsp) -movups %xmm9, 0x30(%rsp) -movups %xmm10, 0x40(%rsp) -movups %xmm11, 0x50(%rsp) -movups %xmm12, 0x60(%rsp) -movups %xmm13, 0x70(%rsp) -movups %xmm14, 0x80(%rsp) -movups %xmm15, 0x90(%rsp) - -# Modify stack pointer of current coroutine (rcx, first argument) -movq %rsp, (%rcx) - -# Load stack pointer from target coroutine (rdx, second argument) -movq (%rdx), %rsp - -# Restore target simd/float registers -movups 0x00(%rsp), %xmm6 -movups 0x10(%rsp), %xmm7 -movups 0x20(%rsp), %xmm8 -movups 0x30(%rsp), %xmm9 -movups 0x40(%rsp), %xmm10 -movups 0x50(%rsp), %xmm11 -movups 0x60(%rsp), %xmm12 -movups 0x70(%rsp), %xmm13 -movups 0x80(%rsp), %xmm14 -movups 0x90(%rsp), %xmm15 -addq $0xa0, %rsp - -# Restore target registers -popq %r15 -popq %r14 -popq %r13 -popq %r12 -popq %rsi -popq %rdi -popq %rbx -popq %rbp - -# Restore Windows stack information -popq %gs:0x08 -popq %gs:0x10 - -retq diff --git a/async/async.zig b/async/async.zig deleted file mode 100644 index 08e56516..00000000 --- a/async/async.zig +++ /dev/null @@ -1,713 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); -const xev = @import("xev").Dynamic; -const XevThreadPool = @import("xev").ThreadPool; - -const aio = @import("asyncio.zig"); -const channel_mod = @import("channel.zig"); -const coro = @import("coro.zig"); -const executor = @import("executor.zig"); -const stack = @import("stack.zig"); - -test { - std.testing.refAllDecls(@This()); - std.testing.refAllDecls(coro); -} - -pub const Condition = struct { - inner: executor.Condition, - - pub fn init() Condition { - return .{ .inner = executor.Condition.init(&AsyncThread.current.executor.exec) }; - } - - pub fn broadcast(self: *Condition) void { - self.inner.broadcast(); - } - - pub fn signal(self: *Condition) void { - self.inner.signal(); - } - - pub fn wait(self: *Condition) void { - self.inner.wait(); - } -}; - -pub fn Frame(comptime func: anytype) type { - const Signature = stdx.meta.FnSignature(func, null); - return FrameExx(func, Signature.ArgsT, Signature.ReturnT); -} - -pub fn FrameEx(comptime func: anytype, comptime argsT: type) type { - const Signature = stdx.meta.FnSignature(func, argsT); - return FrameExx(func, Signature.ArgsT, Signature.ReturnT); -} - -fn FrameExx(comptime func: anytype, comptime argsT: type, comptime returnT: type) type { - return struct { - const Self = @This(); - const FrameT = coro.FrameT(func, argsT); - - inner: FrameT, - - pub fn await(self: *Self) returnT { - defer { - self.inner.deinit(); - AsyncThread.current.stack_allocator.destroy(&self.inner._frame.stack); - self.* = undefined; - } - return coro.xawait(self.inner); - } - }; -} - -pub fn async(comptime func: anytype, args: anytype) !FrameEx(func, @TypeOf(args)) { - const Signature = stdx.meta.FnSignature(func, @TypeOf(args)); - const new_stack = try AsyncThread.current.stack_allocator.create(); - return .{ - .inner = try aio.xasync(func, @as(Signature.ArgsT, args), new_stack), - }; -} - -pub fn callBlocking(comptime func: anytype, args: anytype) stdx.meta.FnSignature(func, @TypeOf(args)).ReturnT { - const Signature = stdx.meta.FnSignature(func, @TypeOf(args)); - - const TaskT = struct { - const Self = @This(); - - _task: XevThreadPool.Task = .{ .callback = &Self.run }, - - event: threading.ResetEventSingle = .{}, - args: Signature.ArgsT, - result: Signature.ReturnT = undefined, - - pub fn run(task_: *XevThreadPool.Task) void { - const task: *Self = @alignCast(@fieldParentPtr("_task", task_)); - task.result = @call(.auto, func, task.args); - task.event.set(); - } - }; - - var newtask: TaskT = .{ - .args = args, - }; - AsyncThread.current.thread_pool.schedule(XevThreadPool.Batch.from(&newtask._task)); - newtask.event.wait(); - - return newtask.result; -} - -pub fn sleep(ms: u64) !void { - try aio.sleep(AsyncThread.current.executor, ms); -} - -pub const threading = struct { - const Waiter = struct { - frame: coro.Frame, - thread: *const AsyncThread, - next: ?*Waiter = null, - }; - - const WaiterQueue = stdx.queue.MPSC(Waiter); - - pub const ResetEventSingle = struct { - const State = union(enum) { - unset, - waiting: *Waiter, - set, - - const unset_state: State = .unset; - const set_state: State = .set; - }; - - waiter: std.atomic.Value(*const State) = std.atomic.Value(*const State).init(&State.unset_state), - - pub fn isSet(self: *ResetEventSingle) bool { - return self.waiter.load(.seq_cst) == &State.set_state; - } - - pub fn reset(self: *ResetEventSingle) void { - self.waiter.store(&State.unset_state, .seq_cst); - } - - pub fn set(self: *ResetEventSingle) void { - switch (self.waiter.swap(&State.set_state, .seq_cst).*) { - .waiting => |waiter| { - waiter.thread.waiters_queue.push(waiter); - waiter.thread.wake(); - }, - else => {}, - } - } - - pub fn wait(self: *ResetEventSingle) void { - var waiter: Waiter = .{ - .frame = coro.xframe(), - .thread = AsyncThread.current, - }; - var new_state: State = .{ - .waiting = &waiter, - }; - if (self.waiter.cmpxchgStrong(&State.unset_state, &new_state, .seq_cst, .seq_cst) == null) { - while (self.isSet() == false) { - coro.xsuspend(); - } - } - } - }; -}; - -pub const AsyncThread = struct { - threadlocal var current: *AsyncThread = undefined; - - executor: *aio.Executor, - stack_allocator: *stack.StackAllocator, - loop: *xev.Loop, - thread_pool: *XevThreadPool, - async_notifier: *xev.Async, - waiters_queue: *threading.WaiterQueue, - - pub fn wake(self: *const AsyncThread) void { - self.async_notifier.notify() catch {}; - } - - fn wakerCallback(self: ?*AsyncThread, _: *xev.Loop, _: *xev.Completion, _: xev.Async.WaitError!void) xev.CallbackAction { - while (self.?.waiters_queue.pop()) |waiter| { - coro.xresume(waiter.frame); - } - return .rearm; - } - - pub fn main(allocator: std.mem.Allocator, comptime mainFunc: fn () anyerror!void) !void { - if (xev.dynamic) try xev.detect(); - var thread_pool = XevThreadPool.init(.{}); - defer { - thread_pool.shutdown(); - thread_pool.deinit(); - } - - var loop = try xev.Loop.init(.{ - .thread_pool = &thread_pool, - }); - defer loop.deinit(); - - var executor_ = aio.Executor.init(&loop); - - var async_notifier = try xev.Async.init(); - defer async_notifier.deinit(); - - var waiters_queue: threading.WaiterQueue = undefined; - waiters_queue.init(); - - var stack_allocator = stack.StackAllocator.init(allocator); - defer stack_allocator.deinit(); - - var asyncThread: AsyncThread = .{ - .executor = &executor_, - .stack_allocator = &stack_allocator, - .loop = &loop, - .thread_pool = &thread_pool, - .async_notifier = &async_notifier, - .waiters_queue = &waiters_queue, - }; - AsyncThread.current = &asyncThread; - - var c2: xev.Completion = undefined; - async_notifier.wait(AsyncThread.current.loop, &c2, AsyncThread, AsyncThread.current, &AsyncThread.wakerCallback); - - // allocate the main coroutine stack, on the current thread's stack! - var mainStackData: stack.Stack.Data = undefined; - const mainStack = stack.Stack.init(&mainStackData); - - return try aio.run(&executor_, mainFunc, .{}, mainStack); - } -}; - -pub fn getStdIn() !File { - return File.initStreaming(std.fs.File.stdin()) catch @panic("Unable to open stdin"); -} - -pub fn getStdOut() File { - return File.initStreaming(std.fs.File.stdout()) catch @panic("Unable to open stdout"); -} - -pub fn getStdErr() File { - return File.initStreaming(std.fs.File.stderr()) catch @panic("Unable to open stderr"); -} - -pub const File = struct { - pub const Reader = struct { - interface: std.Io.Reader, - file: File, - mode: std.fs.File.Reader.Mode = .positional, - pos: u64 = 0, - - fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { - const self: *Reader = @alignCast(@fieldParentPtr("interface", r)); - const dest = limit.slice(try w.writableSliceGreedy(1)); - const n = switch (self.mode) { - .streaming => self.file.read(dest), - .positional => self.file.pread(dest, self.pos), - else => @panic("UNSUPPORTED"), - } catch { - return std.Io.Reader.StreamError.ReadFailed; - }; - if (n == 0) { - return std.Io.Reader.StreamError.EndOfStream; - } - self.pos += n; - w.advance(n); - return n; - } - }; - - pub const Writer = struct { - interface: std.Io.Writer, - file: File, - mode: std.fs.File.Writer.Mode = .positional, - pos: u64 = 0, - - fn write(self: *Writer, buf: []const u8) !usize { - const n = switch (self.mode) { - .streaming => self.file.write(buf), - .positional => self.file.pwrite(buf, self.pos), - else => unreachable, - } catch { - return std.Io.Writer.Error.WriteFailed; - }; - self.pos += n; - return n; - } - - fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { - // TODO: implement splat - _ = splat; - const self: *Writer = @alignCast(@fieldParentPtr("interface", w)); - var total: usize = 0; - if (w.buffered().len > 0) { - total += w.consume(try self.write(w.buffered())); - } - for (data) |d| { - const n = try self.write(d); - total += n; - if (n < d.len) { - return total; - } - } - return total; - } - }; - - pub const SeekError = stdx.meta.FnSignature(File.seekTo, null).ReturnErrorSet.? || stdx.meta.FnSignature(File.seekBy, null).ReturnErrorSet.?; - pub const GetSeekPosError = SeekError || stdx.meta.FnSignature(File.stat, null).ReturnErrorSet.?; - - _handle: std.fs.File.Handle, - inner: aio.File, - is_streaming: bool = false, - - pub fn asFile(self: File) std.fs.File { - return .{ .handle = self._handle }; - } - - pub fn handle(self: File) std.fs.File.Handle { - return self._handle; - } - - pub fn init(file_: std.fs.File) !File { - return .{ - ._handle = file_.handle, - .inner = aio.File.init(AsyncThread.current.executor, try xev.File.init(file_)), - }; - } - - pub fn initStreaming(file_: std.fs.File) !File { - return .{ - ._handle = file_.handle, - .inner = aio.File.init(AsyncThread.current.executor, try xev.File.init(file_)), - .is_streaming = true, - }; - } - - pub fn open(path: []const u8, flags: std.fs.File.OpenFlags) !File { - return init(try callBlocking(std.fs.Dir.openFile, .{ std.fs.cwd(), path, flags })); - } - - pub fn access(path: []const u8, flags: std.fs.File.OpenFlags) !void { - return try callBlocking(std.fs.Dir.access, .{ std.fs.cwd(), path, flags }); - } - - pub fn reader(self: File, buffer: []u8) Reader { - return .{ - .file = self, - .interface = .{ - .vtable = &.{ - .stream = Reader.stream, - }, - .buffer = buffer, - .seek = 0, - .end = 0, - }, - .mode = if (self.is_streaming) .streaming else .positional, - }; - } - - pub fn read(self: File, buf: []u8) !usize { - // NOTE(Corentin): Early return is required to avoid error with xev on Linux with io_uring backend. - if (buf.len == 0) { - return 0; - } - - return self.inner.read(.{ .slice = buf }) catch |err| switch (err) { - // NOTE(Corentin): read shouldn't return an error on EOF, but a read length of 0 instead. This is to be iso with std.fs.File. - error.EOF => 0, - else => err, - }; - } - - pub fn pread(self: File, buf: []u8, offset: u64) !usize { - // NOTE(Corentin): Early return is required to avoid error with xev on Linux with io_uring backend. - if (buf.len == 0) { - return 0; - } - - return self.inner.pread(.{ .slice = buf }, offset) catch |err| switch (err) { - // NOTE(Corentin): pread shouldn't return an error on EOF, but a read length of 0 instead. This is to be iso with std.fs.File. - error.EOF => 0, - else => err, - }; - } - - pub fn write(self: File, buf: []const u8) !usize { - return self.inner.write(.{ .slice = buf }); - } - - pub fn writer(self: File, buffer: []u8) Writer { - return .{ - .file = self, - .interface = .{ - .vtable = &.{ - .drain = Writer.drain, - }, - .buffer = buffer, - }, - .mode = if (self.is_streaming) .streaming else .positional, - }; - } - - pub fn pwrite(self: File, buf: []const u8, offset: u64) !usize { - return self.inner.pwrite(.{ .slice = buf }, offset); - } - - pub fn close(self: File) !void { - return self.inner.close(); - } - - pub fn stat(self: File) !std.fs.File.Stat { - return try callBlocking(std.fs.File.stat, .{self.asFile()}); - } - - pub fn seekBy(self: File, offset: i64) !void { - try callBlocking(std.fs.File.seekBy, .{ self.asFile(), offset }); - } - - pub fn seekTo(self: File, offset: u64) !void { - try callBlocking(std.fs.File.seekTo, .{ self.asFile(), offset }); - } - - pub fn getPos(self: File) !u64 { - return try callBlocking(std.fs.File.getPos, .{self.asFile()}); - } - - pub fn getEndPos(self: File) !u64 { - return try callBlocking(std.fs.File.getEndPos, .{self.asFile()}); - } -}; - -pub const Socket = struct { - pub fn Listener(comptime T: type) type { - return struct { - const Self = @This(); - - inner: T.Inner, - - pub fn accept(self: *Self) !T { - return .{ .inner = try self.inner.accept() }; - } - - pub fn close(self: *Self) !void { - return self.inner.close(); - } - - pub fn deinit(self: *Self) !void { - self.inner.shutdown(); - } - }; - } - - pub const TCP = struct { - const Inner = aio.TCP; - - pub const Reader = struct { - interface: std.Io.Reader, - socket: TCP, - - fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { - const self: *Reader = @alignCast(@fieldParentPtr("interface", r)); - const dest = limit.slice(try w.writableSliceGreedy(1)); - const n = self.socket.read(dest) catch { - return std.Io.Reader.StreamError.ReadFailed; - }; - w.advance(n); - return n; - } - }; - - pub const Writer = struct { - interface: std.Io.Writer, - socket: TCP, - - fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { - // TODO: implement splat - _ = splat; - const self: *Writer = @alignCast(@fieldParentPtr("interface", w)); - var total: usize = 0; - if (w.buffered().len >= 0) { - total += w.consume(self.socket.write(w.buffered()) catch { - return std.Io.Writer.Error.WriteFailed; - }); - } - for (data) |d| { - const n = self.socket.write(d) catch { - return std.Io.Writer.Error.WriteFailed; - }; - total += n; - if (n < d.len) { - return total; - } - } - return total; - } - }; - - inner: aio.TCP, - - pub fn listen(addr: std.net.Address) !Listener(TCP) { - var self: Listener(TCP) = .{ - .inner = aio.TCP.init(AsyncThread.current.executor, try xev.TCP.init(addr)), - }; - try self.inner.tcp.bind(addr); - try self.inner.tcp.listen(1024); - return self; - } - - pub fn deinit(self: *TCP) void { - self.inner.shutdown(); - } - - pub fn accept(self: *TCP) !TCP { - return .{ .inner = try self.inner.accept() }; - } - - pub fn connect(addr: std.net.Address) !TCP { - var self: TCP = .{ - .inner = aio.TCP.init(AsyncThread.current.executor, try xev.TCP.init(addr)), - }; - try self.inner.connect(addr); - return self; - } - - pub fn read(self: TCP, buf: []u8) !usize { - return self.inner.read(.{ .slice = buf }); - } - - pub fn write(self: TCP, buf: []const u8) !usize { - return self.inner.write(.{ .slice = buf }); - } - - pub fn close(self: TCP) !void { - // defer self.* = undefined; - return self.inner.close(); - } - - pub fn reader(self: TCP, buffer: []u8) Reader { - return .{ - .socket = self, - .interface = .{ - .vtable = &.{ - .stream = Reader.stream, - }, - .buffer = buffer, - .seek = 0, - .end = 0, - }, - }; - } - - pub fn writer(self: TCP, buffer: []u8) Writer { - return .{ - .socket = self, - .interface = .{ - .vtable = &.{ - .drain = Writer.drain, - }, - .buffer = buffer, - }, - }; - } - }; - - pub const UDP = struct { - const Inner = aio.TCP; - - pub const Reader = std.io.GenericReader(UDP, stdx.meta.FnSignature(UDP.read, null).ReturnErrorSet.?, UDP.read); - pub const WriterContext = struct { - file: UDP, - addr: std.net.Address, - }; - pub const Writer = std.io.GenericWriter(WriterContext, stdx.meta.FnSignature(UDP.write, null).ReturnErrorSet.?, struct { - fn callBlocking(self: WriterContext, buf: []const u8) !usize { - return self.file.write(self.addr, buf); - } - }.call); - - inner: aio.UDP, - - pub fn listen(addr: std.net.Address) !Listener(UDP) { - var self: Listener(UDP) = .{ - .inner = aio.UDP.init(AsyncThread.current.executor, try xev.UDP.init(addr)), - }; - try self.inner.udp.bind(addr); - try self.inner.udp.listen(1024); - return self; - } - - pub fn read(self: UDP, buf: []u8) !usize { - return self.inner.read(.{ .slice = buf }); - } - - pub fn write(self: UDP, addr: std.net.Address, buf: []const u8) !usize { - return self.inner.write(addr, .{ .slice = buf }); - } - - pub fn close(self: *UDP) !void { - defer self.* = undefined; - return self.inner.close(); - } - - pub fn reader(self: File) Reader { - return .{ .context = self }; - } - - pub fn writer(self: File, addr: std.net.Address) Writer { - return .{ - .context = .{ - .file = self, - .addr = addr, - }, - }; - } - }; -}; - -pub fn Channel(comptime T: type, capacity: usize) type { - return struct { - const Self = @This(); - const Inner = channel_mod.Channel(T, capacity); - - inner: Inner, - - pub fn init() Self { - return .{ .inner = Inner.init(&AsyncThread.current.executor.exec) }; - } - - pub fn initWithLen(len: usize) Self { - return .{ .inner = Inner.initWithLen(&AsyncThread.current.executor.exec, len) }; - } - - pub fn close(self: *Self) void { - self.inner.close(); - } - - pub fn try_send(self: *Self, val: T) bool { - return self.inner.try_send(val); - } - - pub fn send(self: *Self, val: T) void { - self.inner.send(val); - } - - pub fn recv(self: *Self) ?T { - return self.inner.recv(); - } - - pub fn try_recv(self: *Self) ?T { - return self.inner.try_recv(); - } - }; -} - -pub fn channel(comptime T: type, len: usize, comptime capacity: usize) Channel(T, capacity) { - return Channel(T, capacity).initWithLen(len); -} - -pub const Mutex = struct { - const VoidChannel = Channel(void, 1); - - inner: VoidChannel, - - pub fn init() Mutex { - return .{ .inner = VoidChannel.init() }; - } - - pub fn lock(self: *Mutex) void { - self.inner.send({}); - } - - pub fn unlock(self: *Mutex) void { - _ = self.inner.recv(); - } -}; - -pub const LogFn = fn ( - comptime message_level: std.log.Level, - comptime scope: @TypeOf(.enum_literal), - comptime format: []const u8, - args: anytype, -) void; - -pub fn logFn(comptime fallbackLogFn: LogFn) LogFn { - return struct { - const Self = @This(); - - var mu: ?Mutex = null; - - pub fn call( - comptime message_level: std.log.Level, - comptime scope: @TypeOf(.enum_literal), - comptime format: []const u8, - args: anytype, - ) void { - if (coro.inCoro() == false) { - return fallbackLogFn(message_level, scope, format, args); - } - - const level_txt = comptime message_level.asText(); - const prefix2 = if (scope == .default) ": " else "(" ++ @tagName(scope) ++ "): "; - var buffer: [1024]u8 = undefined; - var stderr = getStdErr().writer(&buffer); - - var mutex = Self.mu orelse blk: { - Self.mu = Mutex.init(); - break :blk Self.mu.?; - }; - mutex.lock(); - defer mutex.unlock(); - nosuspend { - stderr.interface.print(level_txt ++ prefix2 ++ format ++ "\n", args) catch unreachable; - stderr.interface.flush() catch unreachable; - } - } - }.call; -} diff --git a/async/asyncio.zig b/async/asyncio.zig deleted file mode 100644 index 7fe05903..00000000 --- a/async/asyncio.zig +++ /dev/null @@ -1,659 +0,0 @@ -const std = @import("std"); -const stdx = @import("stdx"); -const xev = @import("xev").Dynamic; -const libcoro = @import("coro.zig"); -const CoroExecutor = @import("executor.zig").Executor; - -pub const xasync = libcoro.xasync; -pub const xawait = libcoro.xawait; - -const Frame = libcoro.Frame; - -const Env = struct { - exec: ?*Executor = null, -}; - -pub const EnvArg = struct { - executor: ?*Executor = null, - stack_allocator: ?std.mem.Allocator = null, - default_stack_size: ?usize = null, -}; - -threadlocal var env: Env = .{}; - -pub fn initEnv(e: EnvArg) void { - env = .{ .exec = e.executor }; - libcoro.initEnv(.{ - .stack_allocator = e.stack_allocator, - .default_stack_size = e.default_stack_size, - .executor = if (e.executor) |ex| &ex.exec else null, - }); -} - -pub const Executor = struct { - loop: *xev.Loop, - exec: CoroExecutor = .{}, - - pub fn init(loop: *xev.Loop) Executor { - return .{ .loop = loop }; - } - - fn tick(self: *Executor) !void { - try self.loop.run(.once); - _ = self.exec.tick(); - } -}; - -/// Run a coroutine to completion. -/// Must be called from "root", outside of any created coroutine. -pub fn run( - exec: *Executor, - comptime func: anytype, - args: anytype, - stack: anytype, -) !stdx.meta.FnSignature(func, @TypeOf(args)).ReturnPayloadT { - stdx.debug.assert(libcoro.inCoro() == false, "Not in a coroutine", .{}); - const frame = try xasync(func, args, stack); - defer frame.deinit(); - try runCoro(exec, frame); - return xawait(frame); -} - -/// Run a coroutine to completion. -/// Must be called from "root", outside of any created coroutine. -fn runCoro(exec: *Executor, frame: anytype) !void { - const f = frame.frame(); - if (f.status == .Start) { - libcoro.xresume(f); - } - while (f.status != .Done) { - try exec.tick(); - } -} - -const SleepResult = xev.Timer.RunError!void; - -pub fn sleep(exec: *Executor, ms: u64) !void { - const loop = exec.loop; - const Data = XCallback(SleepResult); - - var data = Data.init(); - var w = try xev.Timer.init(); - defer w.deinit(); - var c: xev.Completion = .{}; - w.run(loop, &c, ms, Data, &data, &Data.callback); - - try waitForCompletion(exec, &c); - - return data.result; -} - -pub fn waitForCompletionOutsideCoro(exec: *Executor, c: *xev.Completion) !void { - @branchHint(.unlikely); - while (c.state() != .dead) { - try exec.tick(); - } -} - -pub fn waitForCompletionInCoro(c: *xev.Completion) void { - while (c.state() != .dead) { - libcoro.xsuspend(); - } -} - -pub fn waitForCompletion(exec: *Executor, c: *xev.Completion) !void { - if (libcoro.inCoro()) { - waitForCompletionInCoro(c); - } else { - try waitForCompletionOutsideCoro(exec, c); - } -} - -pub const TCP = struct { - const Self = @This(); - - exec: *Executor, - tcp: xev.TCP, - - pub const close = Closeable(Self, xev.TCP); - pub const poll = Pollable(Self, xev.TCP); - pub const read = Readable(Self, xev.TCP); - pub const write = Writeable(Self, xev.TCP); - - pub fn init(exec: *Executor, tcp: xev.TCP) Self { - return .{ .exec = exec, .tcp = tcp }; - } - - fn stream(self: Self) xev.TCP { - return self.tcp; - } - - pub fn accept(self: Self) !Self { - const AcceptResult = xev.AcceptError!xev.TCP; - const Data = XCallback(AcceptResult); - - const loop = self.exec.loop; - - var data = Data.init(); - var c: xev.Completion = .{}; - self.tcp.accept(loop, &c, Data, &data, &Data.callback); - - try waitForCompletion(self.exec, &c); - - const result = try data.result; - return .{ .exec = self.exec, .tcp = result }; - } - - const ConnectResult = xev.ConnectError!void; - - pub fn connect(self: Self, addr: std.net.Address) !void { - const ResultT = ConnectResult; - const Data = struct { - result: ResultT = undefined, - frame: ?Frame = null, - - fn callback( - userdata: ?*@This(), - l: *xev.Loop, - c: *xev.Completion, - s: xev.TCP, - result: ResultT, - ) xev.CallbackAction { - _ = l; - _ = c; - _ = s; - const data = userdata.?; - data.result = result; - if (data.frame != null) libcoro.xresume(data.frame.?); - return .disarm; - } - }; - - var data: Data = .{ .frame = libcoro.xframe() }; - const loop = self.exec.loop; - var c: xev.Completion = .{}; - self.tcp.connect(loop, &c, addr, Data, &data, &Data.callback); - - try waitForCompletion(self.exec, &c); - - return data.result; - } - - const ShutdownResult = xev.TCP.ShutdownError!void; - pub fn shutdown(self: Self) ShutdownResult { - const ResultT = ShutdownResult; - const Data = struct { - result: ResultT = undefined, - frame: ?Frame = null, - - fn callback( - userdata: ?*@This(), - l: *xev.Loop, - c: *xev.Completion, - s: xev.TCP, - result: ResultT, - ) xev.CallbackAction { - _ = l; - _ = c; - _ = s; - const data = userdata.?; - data.result = result; - if (data.frame != null) libcoro.xresume(data.frame.?); - return .disarm; - } - }; - - var data: Data = .{ .frame = libcoro.xframe() }; - const loop = self.exec.loop; - var c: xev.Completion = .{}; - self.tcp.shutdown(loop, &c, Data, &data, &Data.callback); - - try waitForCompletion(self.exec, &c); - - return data.result; - } -}; - -pub const AsyncError = error{ - AccessDenied, - EventNotFound, - SystemResources, - ProcessNotFound, - Overflow, - NestedRunsNotAllowed, -}; - -fn Closeable(comptime T: type, comptime StreamT: type) fn (T) anyerror!void { - return struct { - const Self = T; - const CloseResult = xev.CloseError!void; - pub fn close(self: Self) anyerror!void { - const ResultT = CloseResult; - const Data = struct { - result: ResultT = undefined, - frame: ?Frame = null, - - fn callback( - userdata: ?*@This(), - l: *xev.Loop, - c: *xev.Completion, - s: StreamT, - result: ResultT, - ) xev.CallbackAction { - _ = l; - _ = c; - _ = s; - const data = userdata.?; - data.result = result; - if (data.frame != null) libcoro.xresume(data.frame.?); - return .disarm; - } - }; - - var data: Data = .{ .frame = libcoro.xframe() }; - - const loop = self.exec.loop; - var c: xev.Completion = .{}; - self.stream().close(loop, &c, Data, &data, &Data.callback); - - try waitForCompletion(self.exec, &c); - - return data.result; - } - }.close; -} - -fn Pollable(comptime T: type, comptime StreamT: type) fn (T) anyerror!void { - return struct { - const PollResult = xev.PollError!void; - pub fn poll(self: T) anyerror!void { - const ResultT = PollResult; - const Data = struct { - result: ResultT = undefined, - frame: ?Frame = null, - - fn callback( - userdata: ?*@This(), - l: *xev.Loop, - c: *xev.Completion, - s: StreamT, - result: ResultT, - ) xev.CallbackAction { - _ = l; - _ = c; - _ = s; - const data = userdata.?; - data.result = result; - if (data.frame != null) libcoro.xresume(data.frame.?); - return .disarm; - } - }; - - var data: Data = .{ .frame = libcoro.xframe() }; - - const loop = self.exec.loop; - var c: xev.Completion = .{}; - self.stream().poll(loop, &c, Data, &data, &Data.callback); - - try waitForCompletion(self.exec, &c); - - return data.result; - } - }.poll; -} - -fn Readable(comptime T: type, comptime StreamT: type) fn (T, xev.ReadBuffer) anyerror!usize { - return struct { - const Self = T; - const ReadResult = xev.ReadError!usize; - pub fn read(self: Self, buf: xev.ReadBuffer) anyerror!usize { - const ResultT = ReadResult; - const Data = struct { - result: ResultT = undefined, - frame: ?Frame = null, - - fn callback( - userdata: ?*@This(), - l: *xev.Loop, - c: *xev.Completion, - s: StreamT, - b: xev.ReadBuffer, - result: ResultT, - ) xev.CallbackAction { - _ = l; - _ = c; - _ = s; - _ = b; - const data = userdata.?; - data.result = result; - if (data.frame != null) libcoro.xresume(data.frame.?); - return .disarm; - } - }; - - var data: Data = .{ .frame = libcoro.xframe() }; - - const loop = self.exec.loop; - var c: xev.Completion = .{}; - self.stream().read(loop, &c, buf, Data, &data, &Data.callback); - - try waitForCompletion(self.exec, &c); - - return data.result; - } - }.read; -} - -fn Writeable(comptime T: type, comptime StreamT: type) fn (T, xev.WriteBuffer) anyerror!usize { - return struct { - const Self = T; - const WriteResult = xev.WriteError!usize; - pub fn write(self: Self, buf: xev.WriteBuffer) !usize { - const ResultT = WriteResult; - const Data = struct { - result: ResultT = undefined, - frame: ?Frame = null, - - fn callback( - userdata: ?*@This(), - l: *xev.Loop, - c: *xev.Completion, - s: StreamT, - b: xev.WriteBuffer, - result: ResultT, - ) xev.CallbackAction { - _ = l; - _ = c; - _ = s; - _ = b; - const data = userdata.?; - data.result = result; - if (data.frame != null) libcoro.xresume(data.frame.?); - return .disarm; - } - }; - - var data: Data = .{ .frame = libcoro.xframe() }; - - const loop = self.exec.loop; - var c: xev.Completion = .{}; - self.stream().write(loop, &c, buf, Data, &data, &Data.callback); - - try waitForCompletion(self.exec, &c); - return data.result; - } - }.write; -} - -pub const File = struct { - const Self = @This(); - - exec: *Executor, - file: xev.File, - - pub const close = Closeable(Self, xev.File); - pub const poll = Pollable(Self, xev.File); - pub const read = Readable(Self, xev.File); - pub const write = Writeable(Self, xev.File); - - pub fn init(exec: *Executor, file: xev.File) Self { - return .{ .exec = exec, .file = file }; - } - - fn stream(self: Self) xev.File { - return self.file; - } - - const PReadResult = xev.ReadError!usize; - pub fn pread(self: Self, buf: xev.ReadBuffer, offset: u64) !usize { - const ResultT = PReadResult; - const Data = struct { - result: ResultT = undefined, - frame: ?Frame = null, - - fn callback( - userdata: ?*@This(), - l: *xev.Loop, - c: *xev.Completion, - s: xev.File, - b: xev.ReadBuffer, - result: ResultT, - ) xev.CallbackAction { - _ = l; - _ = c; - _ = s; - _ = b; - const data = userdata.?; - data.result = result; - if (data.frame != null) libcoro.xresume(data.frame.?); - return .disarm; - } - }; - - var data: Data = .{ .frame = libcoro.xframe() }; - - const loop = self.exec.loop; - var c: xev.Completion = .{}; - self.file.pread(loop, &c, buf, offset, Data, &data, &Data.callback); - - try waitForCompletion(self.exec, &c); - - return data.result; - } - - const PWriteResult = xev.WriteError!usize; - pub fn pwrite(self: Self, buf: xev.WriteBuffer, offset: u64) !usize { - const ResultT = PWriteResult; - const Data = struct { - result: ResultT = undefined, - frame: ?Frame = null, - - fn callback( - userdata: ?*@This(), - l: *xev.Loop, - c: *xev.Completion, - s: xev.File, - b: xev.WriteBuffer, - result: ResultT, - ) xev.CallbackAction { - _ = l; - _ = c; - _ = s; - _ = b; - const data = userdata.?; - data.result = result; - if (data.frame != null) libcoro.xresume(data.frame.?); - return .disarm; - } - }; - - var data: Data = .{ .frame = libcoro.xframe() }; - - const loop = self.exec.loop; - var c: xev.Completion = .{}; - self.file.pwrite(loop, &c, buf, offset, Data, &data, &Data.callback); - - try waitForCompletion(self.exec, &c); - - return data.result; - } -}; - -pub const Process = struct { - const Self = @This(); - - exec: *Executor, - p: xev.Process, - - pub fn init(exec: *Executor, p: xev.Process) Self { - return .{ .exec = exec, .p = p }; - } - - const WaitResult = xev.Process.WaitError!u32; - pub fn wait(self: Self) !u32 { - const Data = XCallback(WaitResult); - var c: xev.Completion = .{}; - var data = Data.init(); - const loop = self.exec.loop; - self.p.wait(loop, &c, Data, &data, &Data.callback); - - try waitForCompletion(self.exec, &c); - - return data.result; - } -}; - -pub const AsyncNotification = struct { - const Self = @This(); - - exec: *Executor, - notif: xev.Async, - - pub fn init(exec: *Executor, notif: xev.Async) Self { - return .{ .exec = exec, .notif = notif }; - } - - const WaitResult = xev.Async.WaitError!void; - pub fn wait(self: Self) !void { - const Data = XCallback(WaitResult); - - const loop = self.exec.loop; - var c: xev.Completion = .{}; - var data = Data.init(); - - self.notif.wait(loop, &c, Data, &data, &Data.callback); - - try waitForCompletion(self.exec, &c); - - return data.result; - } -}; - -pub const UDP = struct { - const Self = @This(); - - exec: *Executor, - udp: xev.UDP, - - pub const close = Closeable(Self, xev.UDP); - pub const poll = Pollable(Self, xev.UDP); - - pub fn init(exec: *Executor, udp: xev.UDP) Self { - return .{ .exec = exec, .udp = udp }; - } - - pub fn stream(self: Self) xev.UDP { - return self.udp; - } - - const ReadResult = xev.ReadError!usize; - pub fn read(self: Self, buf: xev.ReadBuffer) anyerror!usize { - const ResultT = ReadResult; - const Data = struct { - result: ResultT = undefined, - frame: ?Frame = null, - - fn callback( - userdata: ?*@This(), - l: *xev.Loop, - c: *xev.Completion, - s: *xev.UDP.State, - addr: std.net.Address, - udp: xev.UDP, - b: xev.ReadBuffer, - result: ResultT, - ) xev.CallbackAction { - _ = l; - _ = c; - _ = s; - _ = addr; - _ = udp; - _ = b; - const data = userdata.?; - data.result = result; - if (data.frame != null) libcoro.xresume(data.frame.?); - return .disarm; - } - }; - - const loop = self.exec.loop; - var s: xev.UDP.State = undefined; - var c: xev.Completion = .{}; - var data: Data = .{ .frame = libcoro.xframe() }; - self.udp.read(loop, &c, &s, buf, Data, &data, &Data.callback); - - try waitForCompletion(self.exec, &c); - - return data.result; - } - - const WriteResult = xev.WriteError!usize; - pub fn write(self: Self, addr: std.net.Address, buf: xev.WriteBuffer) anyerror!usize { - const ResultT = WriteResult; - const Data = struct { - result: ResultT = undefined, - frame: ?Frame = null, - - fn callback( - userdata: ?*@This(), - l: *xev.Loop, - c: *xev.Completion, - s: *xev.UDP.State, - udp: xev.UDP, - b: xev.WriteBuffer, - result: ResultT, - ) xev.CallbackAction { - _ = l; - _ = c; - _ = s; - _ = udp; - _ = b; - const data = userdata.?; - data.result = result; - if (data.frame != null) libcoro.xresume(data.frame.?); - return .disarm; - } - }; - - const loop = self.exec.loop; - var s: xev.UDP.State = undefined; - var c: xev.Completion = .{}; - var data: Data = .{ .frame = libcoro.xframe() }; - self.udp.write(loop, &c, &s, addr, buf, Data, &data, &Data.callback); - - try waitForCompletion(self.exec, &c); - - return data.result; - } -}; - -fn RunT(comptime Func: anytype) type { - const T = @typeInfo(@TypeOf(Func)).Fn.return_type.?; - return switch (@typeInfo(T)) { - .ErrorUnion => |E| E.payload, - else => T, - }; -} - -fn XCallback(comptime ResultT: type) type { - return struct { - frame: ?Frame = null, - result: ResultT = undefined, - - fn init() @This() { - return .{ .frame = libcoro.xframe() }; - } - - fn callback( - userdata: ?*@This(), - _: *xev.Loop, - _: *xev.Completion, - result: ResultT, - ) xev.CallbackAction { - const data = userdata.?; - data.result = result; - if (data.frame != null) libcoro.xresume(data.frame.?); - return .disarm; - } - }; -} diff --git a/async/channel.zig b/async/channel.zig deleted file mode 100644 index 2904d0f0..00000000 --- a/async/channel.zig +++ /dev/null @@ -1,75 +0,0 @@ -const std = @import("std"); -const stdx = @import("stdx"); -const executor = @import("executor.zig"); - -pub fn Channel(comptime T: type, comptime capacity: usize) type { - const Storage = stdx.queue.ArrayQueue(T, capacity); - - return struct { - const Self = @This(); - - q: Storage = .{}, - closed: bool = false, - len: usize = capacity, - space_notif: executor.Condition, - value_notif: executor.Condition, - - pub fn init(exec: *executor.Executor) Self { - return initWithLen(exec, capacity); - } - - pub fn initWithLen(exec: *executor.Executor, len: usize) Self { - return .{ - .len = len, - .space_notif = executor.Condition.init(exec), - .value_notif = executor.Condition.init(exec), - }; - } - - pub fn close(self: *Self) void { - self.closed = true; - self.value_notif.signal(); - } - - pub fn try_send(self: *Self, val: T) bool { - stdx.debug.assert(self.closed == false, "cannot send on closed Channel", .{}); - if (self.q.len() >= self.len) { - return false; - } - self.q.push(val) catch stdx.debug.panic("tried to send on full Channel. This shouldn't happen.", .{}); - self.value_notif.signal(); - return true; - } - - pub fn send(self: *Self, val: T) void { - stdx.debug.assert(self.closed == false, "cannot send on closed Channel", .{}); - while (self.q.len() >= self.len) { - self.space_notif.wait(); - } - self.q.push(val) catch stdx.debug.panic("tried to send on full Channel. This shouldn't happen.", .{}); - self.value_notif.signal(); - } - - pub fn recv(self: *Self) ?T { - while (self.closed == false or self.q.len() > 0) { - if (self.q.pop()) |val| { - self.space_notif.signal(); - return val; - } - self.value_notif.wait(); - } - return null; - } - - pub fn try_recv(self: *Self) ?T { - if (self.closed == true) { - return null; - } - if (self.q.pop()) |val| { - self.space_notif.signal(); - return val; - } - return null; - } - }; -} diff --git a/async/coro.zig b/async/coro.zig deleted file mode 100644 index 67042967..00000000 --- a/async/coro.zig +++ /dev/null @@ -1,529 +0,0 @@ -//! libcoro mutable state: -//! * ThreadState -//! * current_coro: set in ThreadState.switchTo -//! * next_coro_id: set in ThreadState.nextCoroId -//! * suspend_block: set in xsuspendBlock, cleared in ThreadState.switchIn -//! * Coro -//! * resumer: set in ThreadState.switchTo -//! * status: -//! * Active, Suspended: set in ThreadState.switchTo -//! * Done: set in runcoro -//! * id.invocation: incremented in ThreadState.switchTo -const std = @import("std"); -const builtin = @import("builtin"); - -const stdx = @import("stdx"); - -const base = @import("coro_base.zig"); -const Executor = @import("executor.zig").Executor; -const stack = @import("stack.zig"); -pub const StackT = stack.Stack; - -// Public API -// ============================================================================ -pub const Error = error{ - StackTooSmall, - StackOverflow, - SuspendFromMain, -}; -pub const Frame = *Coro; - -/// Await the coroutine(s). -/// frame: FrameT: runs the coroutine until done and returns its return value. -pub fn xawait(frame: anytype) @TypeOf(frame).Signature.ReturnT { - const f = frame.frame(); - while (f.status != .Done) xsuspend(); - std.debug.assert(f.status == .Done); - return frame.xreturned(); -} - -/// Create a coroutine and start it -/// stack is {null, usize, StackT}. If null or usize, initEnv must have been -/// called with a default stack allocator. -pub fn xasync(func: anytype, args: anytype, stack_: stack.Stack) !FrameT(func, @TypeOf(args)) { - const FrameType = CoroT.fromFunc(func, @TypeOf(args)); - const framet = try FrameType.init(args, stack_); - const frame = framet.frame(); - xresume(frame); - return FrameType.wrap(frame); -} - -pub const FrameT = CoroT.fromFunc; - -/// True if within a coroutine, false if at top-level. -pub fn inCoro() bool { - return thread_state.inCoro(); -} - -/// Returns the currently running coroutine -pub fn xframe() Frame { - return thread_state.current(); -} - -/// Resume the passed coroutine, suspending the current coroutine. -/// When the resumed coroutine suspends, this call will return. -/// Note: When the resumed coroutine returns, control will switch to its parent -/// (i.e. its original resumer). -/// frame: Frame or FrameT -pub fn xresume(frame: anytype) void { - const f = frame.frame(); - thread_state.switchIn(f); -} - -/// Suspend the current coroutine, yielding control back to its -/// resumer. Returns when the coroutine is resumed. -/// Must be called from within a coroutine (i.e. not the top level). -pub fn xsuspend() void { - xsuspendSafe() catch |e| { - log(.err, "{any}", .{e}); - @panic("xsuspend"); - }; -} - -pub fn xsuspendBlock(comptime func: anytype, args: anytype) void { - const Signature = stdx.meta.FnSignature(func, @TypeOf(args)); - const Callback = struct { - func: *const Signature.FuncT, - args: Signature.ArgsT, - fn cb(ud: ?*anyopaque) void { - const self: *@This() = @ptrCast(@alignCast(ud)); - @call(.auto, self.func, self.args); - } - }; - var cb = Callback{ .func = func, .args = args }; - thread_state.suspend_block = .{ .func = Callback.cb, .data = @ptrCast(&cb) }; - xsuspend(); -} - -pub fn xsuspendSafe() Error!void { - if (thread_state.current_coro == null) { - return Error.SuspendFromMain; - } - const coro = thread_state.current_coro.?; - try StackOverflow.check(coro); - thread_state.switchOut(coro.resumer); -} - -const Coro = struct { - /// Coroutine status - const Status = enum { - Start, - Suspended, - Active, - Done, - }; - const Signature = VoidSignature; - - /// Function to run in the coroutine - func: *const fn () void, - /// Coroutine stack - stack: stack.Stack, - /// Architecture-specific implementation - impl: base.Coro, - /// The coroutine that will be yielded to upon suspend - resumer: *Coro = undefined, - /// Current status - status: Status = .Start, - /// Coro id, {thread, coro id, invocation id} - id: CoroId.InvocationId, - /// Caller-specified coro-local storage - storage: ?*anyopaque = null, - - fn init(func: *const fn () void, stack_: stack.Stack, storage: ?*anyopaque) !Frame { - return initFromStack(func, stack_, storage); - } - - pub fn deinit(_: Coro) void { - // empty - } - - fn initFromStack(func: *const fn () void, stack_: stack.Stack, storage: ?*anyopaque) !Frame { - // try StackOverflow.setMagicNumber(stack.full); - var stack__ = stack_; - const coro = try stack__.push(Coro); - const base_coro = try base.Coro.init(&runcoro, stack__.remaining()); - coro.* = .{ - .func = func, - .impl = base_coro, - .stack = stack__, - .storage = storage, - .id = thread_state.newCoroId(), - }; - return coro; - } - - pub fn frame(self: *Coro) Frame { - return self; - } - - fn runcoro(from: *base.Coro, this: *base.Coro) callconv(.c) noreturn { - const from_coro: *Coro = @fieldParentPtr("impl", from); - const this_coro: *Coro = @fieldParentPtr("impl", this); - log(.debug, "coro start {f}", .{this_coro.*}); - @call(.auto, this_coro.func, .{}); - this_coro.status = .Done; - thread_state.switchOut(from_coro); - - // Never returns - stdx.debug.panic("Cannot resume an already completed coroutine {f}", .{this_coro.*}); - } - - pub fn getStorage(self: Coro, comptime T: type) *T { - return @ptrCast(@alignCast(self.storage)); - } - - pub fn format(self: Coro, writer: *std.Io.Writer) !void { - try writer.print("Coro{{.id = {any}, .status = {t}}}", .{ self.id, self.status }); - } -}; - -const VoidSignature = CoroT.Signature.init((struct { - fn func() void {} -}).func, .{}); - -const CoroT = struct { - fn fromFunc(comptime Func: anytype, comptime ArgsT: ?type) type { - return fromSig(stdx.meta.FnSignature(Func, ArgsT)); - } - - fn fromSig(comptime Sig: stdx.meta.Signature) type { - // Stored in the coro stack - const InnerStorage = struct { - args: Sig.ArgsT, - /// Values that are produced during coroutine execution - retval: Sig.ReturnT = undefined, - }; - - return struct { - const Self = @This(); - pub const Signature = Sig; - - _frame: Frame, - - /// Create a Coro - /// self and stack pointers must remain stable for the lifetime of - /// the coroutine. - fn init(args: Sig.ArgsT, stack_: StackT) !Self { - var coro_stack = stack_; - const inner = try coro_stack.push(InnerStorage); - inner.* = .{ - .args = args, - }; - return .{ ._frame = try Coro.initFromStack(wrapfn, coro_stack, inner) }; - } - - pub fn wrap(_frame: Frame) Self { - return .{ ._frame = _frame }; - } - - pub fn deinit(self: Self) void { - self._frame.deinit(); - } - - pub fn status(self: Self) Coro.Status { - return self._frame.status; - } - - pub fn frame(self: Self) Frame { - return self._frame; - } - - // Coroutine functions. - // - // When considering basic coroutine execution, the coroutine state - // machine is: - // * Start - // * Start->xresume->Active - // * Active->xsuspend->Suspended - // * Active->(fn returns)->Done - // * Suspended->xresume->Active - // - // Note that actions in the Active* states are taken from within the - // coroutine. All other actions act upon the coroutine from the - // outside. - - /// Returns the value the coroutine returned - pub fn xreturned(self: Self) Sig.ReturnT { - const storage = self._frame.getStorage(InnerStorage); - return storage.retval; - } - - fn wrapfn() void { - const storage = thread_state.currentStorage(InnerStorage); - storage.retval = @call( - .auto, - Sig.Func.Value, - storage.args, - ); - } - }; - } -}; - -// ============================================================================ - -/// Thread-local coroutine runtime -threadlocal var thread_state: ThreadState = .{}; - -const ThreadState = struct { - root_coro: Coro = .{ - .func = undefined, - .stack = undefined, - .impl = undefined, - .id = CoroId.InvocationId.root(), - }, - current_coro: ?Frame = null, - next_coro_id: usize = 1, - suspend_block: ?SuspendBlock = null, - - const SuspendBlock = struct { - func: *const fn (?*anyopaque) void, - data: ?*anyopaque, - - fn run(self: SuspendBlock) void { - @call(.auto, self.func, .{self.data}); - } - }; - - /// Called from resume - fn switchIn(self: *ThreadState, target: Frame) void { - log(.debug, "coro resume {f} from {f}", .{ target.id, self.current().id }); - - // Switch to target, setting this coro as the resumer. - self.switchTo(target, true); - - // Suspend within target brings control back here - // If a suspend block has been set, pop and run it. - if (self.suspend_block) |block| { - self.suspend_block = null; - block.run(); - } - } - - /// Called from suspend - fn switchOut(self: *ThreadState, target: Frame) void { - log(.debug, "coro suspend {f} to {f}", .{ self.current().id, target.id }); - self.switchTo(target, false); - } - - fn switchTo(self: *ThreadState, target: Frame, set_resumer: bool) void { - const suspender = self.current(); - if (suspender == target) { - return; - } - if (suspender.status != .Done) { - suspender.status = .Suspended; - } - if (set_resumer) { - target.resumer = suspender; - } - target.status = .Active; - target.id.incr(); - self.current_coro = target; - target.impl.resumeFrom(&suspender.impl); - } - - fn newCoroId(self: *ThreadState) CoroId.InvocationId { - const out = CoroId.InvocationId.init(.{ - .coro = self.next_coro_id, - }); - self.next_coro_id += 1; - return out; - } - - fn current(self: *ThreadState) Frame { - return self.current_coro orelse &self.root_coro; - } - - fn inCoro(self: *ThreadState) bool { - return self.current() != &self.root_coro; - } - - /// Returns the storage of the currently running coroutine - fn currentStorage(self: *ThreadState, comptime T: type) *T { - return self.current_coro.?.getStorage(T); - } -}; - -const CoroId = struct { - coro: usize, - - pub const InvocationId = if (builtin.mode == .Debug) DebugInvocationId else DummyInvocationId; - - const DummyInvocationId = struct { - fn init(id: CoroId) @This() { - _ = id; - return .{}; - } - fn root() @This() { - return .{}; - } - fn incr(self: *@This()) void { - _ = self; - } - }; - - const DebugInvocationId = struct { - id: CoroId, - invocation: i64 = -1, - - fn init(id: CoroId) @This() { - return .{ .id = id }; - } - - fn root() @This() { - return .{ .id = .{ .coro = 0 } }; - } - - fn incr(self: *@This()) void { - self.invocation += 1; - } - - pub fn format(self: @This(), writer: *std.Io.Writer) !void { - try writer.print("CoroId{{.cid={d}, .i={d}}}", .{ self.id.coro, self.invocation }); - } - }; -}; - -const StackOverflow = struct { - const magic_number: usize = 0x5E574D6D; - - fn check(_: Frame) !void { - // const stack = coro.stack.ptr; - // const sp = coro.impl.stack_pointer; - // const magic_number_ptr: *usize = @ptrCast(stack); - // if (magic_number_ptr.* != magic_number or // - // @intFromPtr(sp) < @intFromPtr(stack)) - // { - // return Error.StackOverflow; - // } - } - - fn setMagicNumber(_: stack.Stack) !void { - // if (stack.len <= @sizeOf(usize)) { - // return Error.StackTooSmall; - // } - // const magic_number_ptr: *usize = @ptrCast(stack.ptr); - // magic_number_ptr.* = magic_number; - } -}; - -var test_idx: usize = 0; -var test_steps = [_]usize{0} ** 8; - -fn testSetIdx(val: usize) void { - test_steps[test_idx] = val; - test_idx += 1; -} - -fn testFn() void { - // Check if the stack was already overflowed - const current = xframe(); - std.debug.assert(current.stack.remaining().len > 2048); - testSetIdx(2); - xsuspend(); - testSetIdx(4); - xsuspend(); - testSetIdx(6); -} - -test "basic suspend and resume" { - // const allocator = std.testing.allocator; - - // const stack_size: usize = 1024 * 4; - // const stack_ = try stackAlloc(allocator, stack_size); - // defer allocator.free(stack); - - // testSetIdx(0); - // const test_coro = try Coro.init(testFn, stack, false, null); - - // testSetIdx(1); - // try std.testing.expectEqual(test_coro.status, .Start); - // xresume(test_coro); - // testSetIdx(3); - // try std.testing.expectEqual(test_coro.status, .Suspended); - // xresume(test_coro); - // try std.testing.expectEqual(test_coro.status, .Suspended); - // testSetIdx(5); - // xresume(test_coro); - // testSetIdx(7); - - // try std.testing.expectEqual(test_coro.status, .Done); - - // for (0..test_steps.len) |i| { - // try std.testing.expectEqual(i, test_steps[i]); - // } -} - -test "with values" { - // const Test = struct { - // const Storage = struct { - // x: *usize, - // }; - // fn coroInner(x: *usize) void { - // x.* += 1; - // xsuspend(); - // x.* += 3; - // } - // fn coroWrap() void { - // const storage = xframe().getStorage(Storage); - // const x = storage.x; - // coroInner(x); - // } - // }; - // var x: usize = 0; - // var storage = Test.Storage{ .x = &x }; - - // const allocator = std.testing.allocator; - // const stack = try stackAlloc(allocator, null); - // defer allocator.free(stack); - // const coro = try Coro.init(Test.coroWrap, stack, false, @ptrCast(&storage)); - - // try std.testing.expectEqual(storage.x.*, 0); - // xresume(coro); - // try std.testing.expectEqual(storage.x.*, 1); - // xresume(coro); - // try std.testing.expectEqual(storage.x.*, 4); -} - -fn testCoroFnImpl(x: *usize) usize { - x.* += 1; - xsuspend(); - x.* += 3; - xsuspend(); - return x.* + 10; -} - -test "with CoroT" { - // const allocator = std.testing.allocator; - // const stack = try stackAlloc(allocator, null); - // defer allocator.free(stack); - - // var x: usize = 0; - - // const CoroFn = CoroT.fromFunc(testCoroFnImpl, .{}); - // var coro = try CoroFn.init(.{&x}, stack, false); - - // try std.testing.expectEqual(x, 0); - // xresume(coro); - // try std.testing.expectEqual(x, 1); - // xresume(coro); - // try std.testing.expectEqual(x, 4); - // xresume(coro); - // try std.testing.expectEqual(x, 4); - // try std.testing.expectEqual(coro.status(), .Done); - // try std.testing.expectEqual(CoroFn.xreturned(coro), 14); - // comptime try std.testing.expectEqual(CoroFn.Signature.ReturnT(), usize); -} - -test "resume self" { - xresume(xframe()); - try std.testing.expectEqual(1, 1); -} - -pub fn log(comptime level: std.log.Level, comptime fmt: []const u8, args: anytype) void { - if (comptime !std.log.logEnabled(level, .@"zml/async")) return; - - // Since this logs are to debug the async runtime, we want it to happen synchronously. - std.log.defaultLog(level, .@"zml/async", fmt, args); -} diff --git a/async/coro_base.zig b/async/coro_base.zig deleted file mode 100644 index e1a4d6c1..00000000 --- a/async/coro_base.zig +++ /dev/null @@ -1,78 +0,0 @@ -const std = @import("std"); -const stdx = @import("stdx"); -const builtin = @import("builtin"); -const Error = @import("coro.zig").Error; - -const ArchInfo = struct { - num_registers: usize, - jump_idx: usize, - assembly: []const u8, -}; - -const arch_info: ArchInfo = switch (builtin.cpu.arch) { - .aarch64 => .{ - .num_registers = 20, - .jump_idx = 19, - .assembly = @embedFile("asm/coro_aarch64.s"), - }, - .x86_64 => switch (builtin.os.tag) { - .windows => .{ - .num_registers = 32, - .jump_idx = 30, - .assembly = @embedFile("asm/coro_x86_64_windows.s"), - }, - else => .{ - .num_registers = 8, - .jump_idx = 6, - .assembly = @embedFile("asm/coro_x86_64.s"), - }, - }, - .riscv64 => .{ - .num_registers = 25, - .jump_idx = 24, - .assembly = @embedFile("asm/coro_riscv64.s"), - }, - else => @compileError("Unsupported cpu architecture"), -}; - -pub const stack_alignment = 16; - -extern fn libcoro_stack_swap(current: *Coro, target: *Coro) void; -comptime { - asm (arch_info.assembly); -} - -pub const Coro = packed struct { - stack_pointer: [*]u8, - - const Self = @This(); - const Func = *const fn ( - from: *Coro, - self: *Coro, - ) callconv(.c) noreturn; - - pub fn init(func: Func, stack: []align(stack_alignment) u8) !Self { - stdx.debug.assertComptime(@sizeOf(usize) == 8, "usize expected to take 8 bytes", .{}); - stdx.debug.assertComptime(@sizeOf(*Func) == 8, "function pointer expected to take 8 bytes", .{}); - - const register_bytes = arch_info.num_registers * 8; - if (register_bytes > stack.len) { - return Error.StackTooSmall; - } - const register_space = stack[stack.len - register_bytes ..]; - - // Zero out the register space so that the initial stack swap - // of new coroutines doesn't poison ebp. - // - // A better solution would be to prepare the initial stack so that the - // stack is valid up to the caller. - @memset(register_space, 0); - const jump_ptr: *Func = @ptrCast(@alignCast(®ister_space[arch_info.jump_idx * 8])); - jump_ptr.* = func; - return .{ .stack_pointer = register_space.ptr }; - } - - pub inline fn resumeFrom(self: *Self, from: *Self) void { - libcoro_stack_swap(from, self); - } -}; diff --git a/async/executor.zig b/async/executor.zig deleted file mode 100644 index 6f87311b..00000000 --- a/async/executor.zig +++ /dev/null @@ -1,182 +0,0 @@ -const std = @import("std"); - -const libcoro_options = @import("libcoro_options"); -const stdx = @import("stdx"); - -const libcoro = @import("coro.zig"); - -pub const Executor = struct { - const Self = @This(); - - pub const Func = struct { - const FuncFn = *const fn (userdata: ?*anyopaque) void; - - func: FuncFn, - userdata: ?*anyopaque = null, - next: ?*Func = null, - - pub fn init(func: FuncFn, userdata: ?*anyopaque) Func { - return .{ .func = func, .userdata = userdata }; - } - - fn run(self: Func) void { - @call(.auto, self.func, .{self.userdata}); - } - }; - - readyq: stdx.queue.SPSC(Func) = .{}, - - pub fn init() Self { - return .{}; - } - - pub fn runSoon(self: *Self, func: *Func) void { - self.readyq.push(func); - } - - pub fn runAllSoon(self: *Self, funcs: stdx.queue.SPSC(Func)) void { - self.readyq.pushAll(funcs); - } - - pub fn tick(self: *Self) bool { - // Reset readyq so that adds run on next tick. - var now = self.readyq; - self.readyq = .{}; - - libcoro.log(.debug, "Executor.tick readyq.len={d}", .{now.len()}); - - var count: usize = 0; - while (now.pop()) |func| : (count += 1) { - func.run(); - } - - libcoro.log(.debug, "Executor.tick done", .{}); - - return count > 0; - } -}; - -pub const Condition = struct { - exec: *Executor, - waiters: stdx.queue.SPSC(Executor.Func) = .{}, - - pub fn init(exec: *Executor) Condition { - return .{ .exec = exec }; - } - - pub fn broadcast(self: *Condition) void { - self.exec.runAllSoon(self.waiters); - } - - pub fn signal(self: *Condition) void { - if (self.waiters.pop()) |waiter_func| { - self.exec.runSoon(waiter_func); - } - } - - pub fn wait(self: *Condition) void { - var cr = CoroResume.init(); - var cb = cr.func(); - self.waiters.push(&cb); - libcoro.xsuspend(); - } -}; - -pub const CoroResume = struct { - const Self = @This(); - - coro: libcoro.Frame, - - pub fn init() Self { - return .{ .coro = libcoro.xframe() }; - } - - pub fn func(self: *Self) Executor.Func { - return .{ .func = Self.cb, .userdata = self }; - } - - fn cb(ud: ?*anyopaque) void { - const self: *Self = @ptrCast(@alignCast(ud)); - libcoro.xresume(self.coro); - } -}; - -pub fn getExec(exec: ?*Executor) *Executor { - if (exec != null) return exec.?; - if (libcoro.getEnv().executor) |x| return x; - @panic("No explicit Executor passed and no default Executor available"); -} - -pub fn ArrayQueue(comptime T: type, comptime size: usize) type { - return struct { - const Self = @This(); - - vals: [size]T = undefined, - head: ?usize = null, - tail: ?usize = null, - - fn init() Self { - return .{}; - } - - fn len(self: Self) usize { - switch (self.state()) { - .empty => return 0, - .one => return 1, - .many => { - const head = self.head.?; - const tail = self.tail.?; - if (tail > head) { - return tail - head + 1; - } - return size - head + tail + 1; - }, - } - } - - fn space(self: Self) usize { - return size - self.len(); - } - - fn push(self: *@This(), val: T) !void { - if (self.space() < 1) return error.QueueFull; - switch (self.state()) { - .empty => { - self.head = 0; - self.tail = 0; - self.vals[0] = val; - }, - .one, .many => { - const tail = self.tail.?; - const new_tail = (tail + 1) % size; - self.vals[new_tail] = val; - self.tail = new_tail; - }, - } - } - - fn pop(self: *Self) ?T { - switch (self.state()) { - .empty => return null, - .one => { - const out = self.vals[self.head.?]; - self.head = null; - self.tail = null; - return out; - }, - .many => { - const out = self.vals[self.head.?]; - self.head = (self.head.? + 1) % size; - return out; - }, - } - } - - const State = enum { empty, one, many }; - inline fn state(self: Self) State { - if (self.head == null) return .empty; - if (self.head.? == self.tail.?) return .one; - return .many; - } - }; -} diff --git a/async/stack.zig b/async/stack.zig deleted file mode 100644 index 1e08c8fb..00000000 --- a/async/stack.zig +++ /dev/null @@ -1,83 +0,0 @@ -const std = @import("std"); -const coro_base = @import("coro_base.zig"); - -pub const stack_size = 1 * 1024 * 1024; - -pub const Stack = struct { - pub const Data = struct { - data: [stack_size]u8 align(coro_base.stack_alignment) = undefined, - - pub fn ptr(self: *const Data) [*]u8 { - return &self.data; - } - }; - - full: *Data, - used: []u8, - - pub fn init(full: *Data) Stack { - return .{ - .full = full, - .used = full.data[full.data.len..], - }; - } - - pub fn remaining(self: Stack) []align(coro_base.stack_alignment) u8 { - return self.full.data[0 .. self.full.data.len - self.used.len]; - } - - pub fn push(self: *Stack, comptime T: type) !*T { - const ptr_i = std.mem.alignBackward( - usize, - @intFromPtr(self.used.ptr - @sizeOf(T)), - coro_base.stack_alignment, - ); - if (ptr_i <= @intFromPtr(&self.full.data[0])) { - return error.StackTooSmall; - } - self.used = self.full.data[ptr_i - @intFromPtr(&self.full.data[0]) ..]; - return @ptrFromInt(ptr_i); - } -}; - -pub const PooledStackAllocator = struct { - const Pool = std.heap.MemoryPoolAligned(Stack.Data, coro_base.stack_alignment); - - pool: Pool, - - pub fn init(allocator: std.mem.Allocator) PooledStackAllocator { - return .{ .pool = Pool.init(allocator) }; - } - - pub fn deinit(self: *PooledStackAllocator) void { - self.pool.deinit(); - } - - pub fn create(self: *PooledStackAllocator) !Stack { - return Stack.init(try self.pool.create()); - } - - pub fn destroy(self: *PooledStackAllocator, stack: *Stack) void { - self.pool.destroy(stack.full); - } -}; - -pub const StackAllocator = struct { - allocator: std.mem.Allocator, - - pub fn init(allocator: std.mem.Allocator) StackAllocator { - return .{ .allocator = allocator }; - } - - pub fn deinit(_: *StackAllocator) void { - // empty - } - - pub fn create(self: *StackAllocator) !Stack { - return Stack.init(try self.allocator.create(Stack.Data)); - } - - pub fn destroy(self: *StackAllocator, stack: *Stack) void { - self.allocator.destroy(stack.full); - } -}; From 920fcbe9b2e13a35bc9d1a14edd0f3e8b6e0387e Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 11 Dec 2025 17:44:15 +0100 Subject: [PATCH 124/881] Cleanup root main.zig --- BUILD.bazel | 20 -------------------- main.zig | 51 --------------------------------------------------- 2 files changed, 71 deletions(-) delete mode 100644 main.zig diff --git a/BUILD.bazel b/BUILD.bazel index faf47f30..5d775990 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -11,23 +11,3 @@ zls_completion( "//zml2:zml", ], ) - -zig_binary( - name = "main", - main = "main.zig", - deps = [ - ":xla_compile_options_upb", - ":xla_data_upb", - "//upb", - ], -) - -upb_c_proto_library( - name = "xla_data_upb", - deps = ["@xla//xla:xla_data_proto"], -) - -upb_c_proto_library( - name = "xla_compile_options_upb", - deps = ["@xla//xla/pjrt/proto:compile_options_proto"], -) diff --git a/main.zig b/main.zig deleted file mode 100644 index 83d4cd52..00000000 --- a/main.zig +++ /dev/null @@ -1,51 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const upb = @import("upb"); - -pub fn main() !void { - var arena = std.heap.ArenaAllocator.init(std.heap.smp_allocator); - defer arena.deinit(); - - var upb_alloc: upb.Allocator = .init(arena.allocator()); - const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); - defer c.upb_Arena_Free(upb_arena); - - const sharding = struct { - num_replicas: i32 = 2, - num_partitions: i32 = 4, - }{}; - - const options = blk: { - const options = try upb.new(c.xla_CompileOptionsProto, upb_arena); - c.xla_CompileOptionsProto_set_executable_build_options(options, executable_build_options_blk: { - const exec_build_options = try upb.new(c.xla_ExecutableBuildOptionsProto, upb_arena); - - c.xla_ExecutableBuildOptionsProto_set_device_ordinal(exec_build_options, -1); - c.xla_ExecutableBuildOptionsProto_set_num_replicas(exec_build_options, sharding.num_replicas); - c.xla_ExecutableBuildOptionsProto_set_num_partitions(exec_build_options, sharding.num_partitions); - c.xla_ExecutableBuildOptionsProto_set_use_spmd_partitioning(exec_build_options, sharding.num_partitions > 1 or sharding.num_replicas > 1); - - c.xla_ExecutableBuildOptionsProto_set_device_assignment(exec_build_options, device_assignment_blk: { - const device_assignment = try upb.new(c.xla_DeviceAssignmentProto, upb_arena); - - c.xla_DeviceAssignmentProto_set_replica_count(device_assignment, sharding.num_replicas); - c.xla_DeviceAssignmentProto_set_computation_count(device_assignment, sharding.num_partitions); - - const computation_devices = c.xla_DeviceAssignmentProto_resize_computation_devices(device_assignment, sharding.num_partitions, upb_arena); - for (computation_devices[0..sharding.num_partitions], 0..) |*computation_device, i| { - computation_device.* = try upb.new(c.xla_DeviceAssignmentProto_ComputationDevice, upb_arena); - _ = c.xla_DeviceAssignmentProto_ComputationDevice_add_replica_device_ids(computation_device.*, @intCast(i), upb_arena); - } - break :device_assignment_blk device_assignment; - }); - break :executable_build_options_blk exec_build_options; - }); - - // const overrides_map = c._xla_CompileOptionsProto_env_option_overrides_mutable_upb_map(options, upb_arena); - - break :blk options; - }; - - _ = options; -} From 51ec792d878bc6dd2b3b93cdb588062e1f36032f Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 11 Dec 2025 17:44:27 +0100 Subject: [PATCH 125/881] Migrate to @EnumLiteral() --- mlir/mlir.zig | 2 +- zml2/shape.zig | 30 ++++++++++++++---------------- zml2/tensor.zig | 12 ++++++------ 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/mlir/mlir.zig b/mlir/mlir.zig index e9b6f2d7..b90e3f3e 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -193,7 +193,7 @@ pub const Pass = opaque { pub const ptr = M.ptr; - pub fn create(comptime name: @TypeOf(.enum_literal)) *Pass { + pub fn create(comptime name: @EnumLiteral()) *Pass { return @ptrCast(@field(c, "mlirCreate" ++ @tagName(name))().ptr); } }; diff --git a/zml2/shape.zig b/zml2/shape.zig index 71968b9e..56c724e0 100644 --- a/zml2/shape.zig +++ b/zml2/shape.zig @@ -7,8 +7,6 @@ const stdx = @import("stdx"); const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; -const EnumLiteral = @TypeOf(.enum_literal); - const log = std.log.scoped(.shape); /// Represent the shape of a tensor. @@ -28,7 +26,7 @@ pub const Shape = struct { return value; } - if (comptime T == EnumLiteral) { + if (comptime T == @EnumLiteral()) { const name = @tagName(value); inline for (std.meta.fields(PartitionSpec)) |field| { if (field.type == void and std.mem.eql(u8, field.name, name)) { @@ -186,7 +184,7 @@ pub const Shape = struct { const fv = @field(v, field.name); if (comptime stdx.meta.isInteger(field.type)) { dims_.appendAssumeCapacity(@intCast(fv)); - } else if (@TypeOf(fv) == EnumLiteral and comptime isAutoDim(fv)) { + } else if (@TypeOf(fv) == @EnumLiteral() and comptime isAutoDim(fv)) { dims_.appendAssumeCapacity(-1); } else { stdx.debug.compileError("Field {s} should be an integer or an auto dimension, got {}", .{ field.name, field.type }); @@ -243,7 +241,7 @@ pub const Shape = struct { pub fn parseTags(v: anytype) TagsArray { const T = @TypeOf(v); - stdx.debug.assertComptime(stdx.meta.isTupleOf(T, EnumLiteral), "Wrong type, got {}. Expected .{{ .a, .b }}", .{T}); + stdx.debug.assertComptime(stdx.meta.isTupleOf(T, @EnumLiteral()), "Wrong type, got {}. Expected .{{ .a, .b }}", .{T}); var tags_ = TagsArray.init(0) catch unreachable; inline for (v) |field| { tags_.appendAssumeCapacity(toTag(field)); @@ -304,7 +302,7 @@ pub const Shape = struct { fn isTagConvertible(comptime T: type) bool { return switch (T) { - EnumLiteral => true, + @EnumLiteral() => true, std.builtin.Type.StructField => true, Tag => true, else => false, @@ -314,10 +312,10 @@ pub const Shape = struct { pub fn toTag(v: anytype) Tag { const T = @TypeOf(v); return switch (T) { - EnumLiteral => @tagName(v).ptr, + @EnumLiteral() => @tagName(v).ptr, std.builtin.Type.StructField => v.name.ptr, Tag => v, - else => stdx.debug.compileError("Shape tag should be an EnumLiteral, a Shape.Tag or a StructField, got {}", .{T}), + else => stdx.debug.compileError("Shape tag should be an @EnumLiteral(), a Shape.Tag or a StructField, got {}", .{T}), }; } @@ -357,7 +355,7 @@ pub const Shape = struct { pub fn hasTags(self: Shape, tagz: anytype) bool { const T = @TypeOf(tagz); - if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, EnumLiteral)) { + if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, @EnumLiteral())) { for (tagz) |t| { if (self.hasTag(t) == null) { return false; @@ -366,7 +364,7 @@ pub const Shape = struct { return true; } - if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, EnumLiteral)) { + if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, @EnumLiteral())) { inline for (tagz) |t| { if (self.hasTag(t) == null) { return false; @@ -635,7 +633,7 @@ pub const Shape = struct { /// ``` pub fn insert(self: Shape, axis_: anytype, dimz: anytype) Shape { const dims_, const tags_ = parseDimensions(dimz); - const ax = if (@TypeOf(axis_) == EnumLiteral and axis_ == .last) + const ax = if (@TypeOf(axis_) == @EnumLiteral() and axis_ == .last) self.rank() else self.axis(axis_); @@ -660,7 +658,7 @@ pub const Shape = struct { pub fn insertTag(self: Shape, axis_: anytype, d: i64, tag_: anytype) Shape { stdx.debug.assert(self.rank() < constants.MAX_RANK - 1, "Can't insert new axis in {f}, it's already at max rank.", .{self}); - const ax = if (@TypeOf(axis_) == EnumLiteral and axis_ == .last) + const ax = if (@TypeOf(axis_) == @EnumLiteral() and axis_ == .last) self.rank() else self.axis(axis_); @@ -802,7 +800,7 @@ pub const Shape = struct { var res = self; - if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, EnumLiteral)) { + if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, @EnumLiteral())) { stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {any}", .{ self, tagz }); for (tagz, 0..) |tag_, i| { res._tags.set(i, toTag(tag_)); @@ -810,7 +808,7 @@ pub const Shape = struct { return res; } - if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, EnumLiteral)) { + if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, @EnumLiteral())) { stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {}", .{ self, tagz }); inline for (tagz, 0..) |tag_, i| { res._tags.set(i, toTag(tag_)); @@ -849,7 +847,7 @@ pub const Shape = struct { var res = self; - if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, EnumLiteral)) { + if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, @EnumLiteral())) { stdx.debug.assert(tagz.len <= self.rank(), "Too many tags for shape {f}, got {any}", .{ self, tagz }); for (tagz, self.rank() - tagz.len..) |tag_, i| { res._tags.set(i, toTag(tag_)); @@ -857,7 +855,7 @@ pub const Shape = struct { return res; } - if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, EnumLiteral)) { + if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, @EnumLiteral())) { stdx.debug.assert(tagz.len <= self.rank(), "Too many tags for shape {f}, got {}", .{ self, tagz }); inline for (tagz, self.rank() - tagz.len..) |tag_, i| { res._tags.set(i, toTag(tag_)); diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 64fc0000..c0f220d7 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -157,7 +157,7 @@ pub const Tensor = struct { return res; } - pub fn renameAxis(self: Tensor, ax: i8, name: @TypeOf(.enum_literal)) Tensor { + pub fn renameAxis(self: Tensor, ax: i8, name: @EnumLiteral()) Tensor { var res = self; res._shape._tags.set(self.axis(ax), @tagName(name).ptr); return res; @@ -1464,7 +1464,7 @@ pub const Tensor = struct { /// Merges two or more non-contiguous axes into one axis. /// Will make a transpose if needed. /// .{ .a, .b, .c }.mergeTranspose(.{ .a, .c }, .ac) -> .{ .b, .ac } - pub fn mergeTranspose(self: Tensor, axes_: anytype, merged: @TypeOf(.enum_literal)) Tensor { + pub fn mergeTranspose(self: Tensor, axes_: anytype, merged: @EnumLiteral()) Tensor { const cont = self.contiguous(axes_); return cont.reshape(cont._shape.mergeAxis(merged, axes_)); } @@ -1653,7 +1653,7 @@ pub const Tensor = struct { // Be careful here: we need to resolve ax before calling concatenate, // because we added an axis, so all - const ax = if (@TypeOf(axis_) == @TypeOf(.enum_literal) and axis_ == .last) + const ax = if (@TypeOf(axis_) == @EnumLiteral() and axis_ == .last) shape0.rank() else shape0.axis(axis_); @@ -2107,7 +2107,7 @@ pub const Tensor = struct { /// `.{.a = 5, .b = 4}.insert(.b, .{ .c, .d }) -> .{ .a = 5, .c = 1, .d = 1, .b = 4 }` pub fn insertAxes(self: Tensor, axis_: anytype, tags: anytype) Tensor { const tags_ = Shape.parseTags(tags); - const ax = if (@TypeOf(axis_) == @TypeOf(.enum_literal) and axis_ == .last) + const ax = if (@TypeOf(axis_) == @EnumLiteral() and axis_ == .last) self.rank() else self.axis(axis_); @@ -3557,7 +3557,7 @@ pub const Tensor = struct { /// For each vector in the input tensor, /// creates a diagonal-matrix where diagonal values are set to the vector values. - pub fn toDiagonal(self: Tensor, axis_: anytype, new_tags: [2]@TypeOf(.enum_literal)) Tensor { + pub fn toDiagonal(self: Tensor, axis_: anytype, new_tags: [2]@EnumLiteral()) Tensor { stdx.debug.assert(self.rank() < constants.MAX_RANK - 1, "toDiagonal expects input up to {d} rank, got {f}", .{ constants.MAX_RANK - 1, self }); const a = self.axis(axis_); const d = self.dim(a); @@ -4093,7 +4093,7 @@ test "Tensor.maxPool2d" { fn _parseGatherCoord(self: Tensor, axes_: anytype) struct { bool, stdx.BoundedArray(u3, constants.MAX_RANK) } { const AxesT = @TypeOf(axes_); - const axes_is_scalar = AxesT == @TypeOf(.enum_literal) or AxesT == comptime_int or @typeInfo(AxesT) == .int; + const axes_is_scalar = AxesT == @EnumLiteral() or AxesT == comptime_int or @typeInfo(AxesT) == .int; const coord_axes = if (axes_is_scalar) stdx.BoundedArray(u3, constants.MAX_RANK).fromSlice(&.{self.axis(axes_)}) catch unreachable From dc00398d1c85e4d6052b8b81970bf89220e06d18 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 11 Dec 2025 18:23:11 +0100 Subject: [PATCH 126/881] Restored ops.triton --- mlir/dialects/stablehlo/stablehlo.zig | 2 +- zml2/constants.zig | 9 ++ zml2/ops.zig | 136 ++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 1 deletion(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 2ad416e7..a3b4b5d2 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -708,7 +708,7 @@ pub const CustomCallOpts = struct { original: []const u8, status_returning: []const u8, status_returning_unified: []const u8, - typed_ffi: *const mlir.DictionaryAttribute, + typed_ffi: *const mlir.Attribute, }; call_target_name: []const u8, diff --git a/zml2/constants.zig b/zml2/constants.zig index c700cb40..ad8ff445 100644 --- a/zml2/constants.zig +++ b/zml2/constants.zig @@ -4,3 +4,12 @@ pub const AXES_IOTA: [MAX_RANK]i64 = b: { for (0..MAX_RANK) |i| values[i] = i; break :b values; }; +pub const MINOR_TO_MAJOR: [MAX_RANK]i64 = b: { + var values: [MAX_RANK]i64 = undefined; + for (0..MAX_RANK) |i| values[i] = MAX_RANK - i - 1; + break :b values; +}; + +pub fn minorToMajor(rank: u8) []const i64 { + return MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - rank ..]; +} diff --git a/zml2/ops.zig b/zml2/ops.zig index 68ab154e..7a0eac91 100644 --- a/zml2/ops.zig +++ b/zml2/ops.zig @@ -260,6 +260,136 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytyp return result; } +pub const TritonOps = struct { + debug: bool = false, + name: [:0]const u8, + ir: [:0]const u8, + grid: [3]i32, + num_stages: i32, + num_warps: i32, + output_operand_aliases: []const i64 = &.{}, +}; + +/// Generate an MLIR call to the given member function with the given tensors. +pub fn triton(inputs: anytype, outputs: anytype, opts: TritonOps) [outputs.len]Tensor { + const mlir_ctx = CompilationContext.current().mlir_ctx; + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + var values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + values[i] = inputs[i].value(); + } + + var res_types: [outputs.len]*const mlir.Type = undefined; + inline for (outputs, 0..) |output, i| { + res_types[i] = mlir.rankedTensorType(output.dims(), mlirx.Type.fromDType(mlir_ctx, output.dtype())); + } + + const backend_config = mlir.dictionaryAttribute(mlir_ctx, &.{ + mlir.NamedAttribute.named(mlir_ctx, "name", mlir.stringAttribute(mlir_ctx, opts.name)), + mlir.NamedAttribute.named(mlir_ctx, "ir", mlir.stringAttribute(mlir_ctx, opts.ir)), + mlir.NamedAttribute.named(mlir_ctx, "grid_x", mlir.integerAttribute(mlir_ctx, .i32, opts.grid[0])), + mlir.NamedAttribute.named(mlir_ctx, "grid_y", mlir.integerAttribute(mlir_ctx, .i32, opts.grid[1])), + mlir.NamedAttribute.named(mlir_ctx, "grid_z", mlir.integerAttribute(mlir_ctx, .i32, opts.grid[2])), + mlir.NamedAttribute.named(mlir_ctx, "num_stages", mlir.integerAttribute(mlir_ctx, .i32, opts.num_stages)), + mlir.NamedAttribute.named(mlir_ctx, "num_warps", mlir.integerAttribute(mlir_ctx, .i32, opts.num_warps)), + }); + + var operands_layouts: [inputs.len][]const usize = undefined; + inline for (inputs, 0..) |input, i| { + operands_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(input.rank())).constSlice()) catch unreachable; + } + + var results_layouts: [outputs.len][]const usize = undefined; + inline for (outputs, 0..) |output, i| { + results_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(output.rank())).constSlice()) catch unreachable; + } + + const op = dialects.stablehlo.custom_call( + mlir_ctx, + &values, + &res_types, + .{ + .call_target_name = "__gpu$xla.gpu.triton", + .backend_config = .{ .typed_ffi = backend_config }, + .has_side_effect = false, + .operand_layouts = &operands_layouts, + .result_layouts = &results_layouts, + .output_operand_aliases = opts.output_operand_aliases, + }, + .unknown(mlir_ctx), + ).appendTo(CompilationContext.current().currentScope().block); + + var outputs_: [outputs.len]Tensor = undefined; + inline for (outputs, 0..) |output, i| { + outputs_[i] = Tensor._result(output, op.result(i)); + } + + return outputs_; +} + +test "triton" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + if (platform.target != .cuda and platform.target != .rocm) return error.SkipZigTest; + + const ir = + \\ module { + \\ tt.func public @add_one(%arg0: !tt.ptr {tt.divisibility = 32 : i32}, %arg1: !tt.ptr {tt.divisibility = 32 : i32}, %arg2: !tt.ptr {tt.divisibility = 32 : i32}, %arg3: !tt.ptr {tt.divisibility = 32 : i32}) { + \\ %0 = tt.get_program_id x : i32 + \\ %1 = tt.load %arg0 {cache = 1 : i32, evict = 1 : i32, isVolatile = false} : !tt.ptr + \\ %2 = tt.load %arg1 {cache = 1 : i32, evict = 1 : i32, isVolatile = false} : !tt.ptr + \\ %cst = arith.constant 1.000000e+00 : f32 + \\ %3 = arith.addf %1, %cst : f32 + \\ tt.store %arg2, %3 {cache = 1 : i32, evict = 1 : i32} : !tt.ptr + \\ tt.store %arg3, %2 {cache = 1 : i32, evict = 1 : i32} : !tt.ptr + \\ tt.return + \\ } + \\ } + ; + + const TritonMod = struct { + pub fn forward(a: Tensor, b: Tensor) [2]Tensor { + return triton(.{ a, b }, .{ a.shape(), b.shape() }, .{ + .debug = false, + .name = "add_one", + .ir = ir, + .grid = .{ 1, 1, 1 }, + .num_stages = 1, + .num_warps = 1, + }); + } + }; + + const a: zml.Tensor = .init(Shape.init(.{}, .f32)); + const b: zml.Tensor = .init(Shape.init(.{}, .f32)); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, TritonMod.forward, .{ a, b }, platform); + defer exe.deinit(); + + var a_buffer: zml.Buffer = try .fromBytes(platform, a.shape(), std.mem.sliceAsBytes(&[1]f32{1}), std.testing.io); + defer a_buffer.deinit(); + var b_buffer: zml.Buffer = try .fromBytes(platform, b.shape(), std.mem.sliceAsBytes(&[1]f32{3}), std.testing.io); + defer b_buffer.deinit(); + + const results = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, TritonMod.forward, .{ a_buffer, b_buffer }); + defer results[0].deinit(); + defer results[1].deinit(); + + var cpu_result_0 = try results[0].toSliceAlloc(std.testing.allocator, std.testing.io); + defer cpu_result_0.free(std.testing.allocator); + var cpu_result_1 = try results[1].toSliceAlloc(std.testing.allocator, std.testing.io); + defer cpu_result_1.free(std.testing.allocator); + + const expected_result_a: f32 = 2.0; + const expected_result_b: f32 = 3.0; + + try std.testing.expectEqual(expected_result_a, cpu_result_0.items(f32)[0]); + try std.testing.expectEqual(expected_result_b, cpu_result_1.items(f32)[0]); +} + pub const ScatterArgs = struct { input: Tensor, update: Tensor, @@ -706,3 +836,9 @@ pub fn _collectAxes(T: type, bounded_array: stdx.BoundedArray(T, constants.MAX_R } return res; } + +fn toUsize(values: anytype) stdx.BoundedArray(usize, constants.MAX_RANK) { + var res: stdx.BoundedArray(usize, constants.MAX_RANK) = .{}; + for (values) |val| res.appendAssumeCapacity(@intCast(val)); + return res; +} From 34dd16167e16dd36ee6608ec820b333103af2169 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 11 Dec 2025 18:23:50 +0100 Subject: [PATCH 127/881] io/file (#362) --- zml2/io/vfs.zig | 1 + zml2/io/vfs/BUILD.bazel | 1 + zml2/io/vfs/file.zig | 117 ++++++++++++++++++++++++++++++++++++++++ zml2/io/vfs/index.zig | 1 + zml2/main_io.zig | 6 ++- 5 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 zml2/io/vfs/file.zig diff --git a/zml2/io/vfs.zig b/zml2/io/vfs.zig index 9ec67f95..9f92b3b5 100644 --- a/zml2/io/vfs.zig +++ b/zml2/io/vfs.zig @@ -19,6 +19,7 @@ pub const VFS = struct { }; pub const HTTP = @import("vfs").HTTP; + pub const File = @import("vfs").File; const FsFile = struct { backend: std.Io, diff --git a/zml2/io/vfs/BUILD.bazel b/zml2/io/vfs/BUILD.bazel index 4f1978c4..591e869d 100644 --- a/zml2/io/vfs/BUILD.bazel +++ b/zml2/io/vfs/BUILD.bazel @@ -3,6 +3,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library") zig_library( name = "vfs", srcs = [ + "file.zig", "http.zig", ], main = "index.zig", diff --git a/zml2/io/vfs/file.zig b/zml2/io/vfs/file.zig new file mode 100644 index 00000000..b6e6ebdb --- /dev/null +++ b/zml2/io/vfs/file.zig @@ -0,0 +1,117 @@ +const std = @import("std"); + +pub const File = struct { + inner: std.Io, + vtable: std.Io.VTable, + + pub fn init(inner: std.Io) File { + var vtable = inner.vtable.*; + vtable.dirMake = dirMake; + vtable.dirMakePath = dirMakePath; + vtable.dirMakeOpenPath = dirMakeOpenPath; + vtable.dirStatPath = dirStatPath; + vtable.dirAccess = dirAccess; + vtable.dirCreateFile = dirCreateFile; + vtable.dirOpenFile = dirOpenFile; + vtable.dirOpenDir = dirOpenDir; + + return .{ + .inner = inner, + .vtable = vtable, + }; + } + + pub fn io(self: *File) std.Io { + return .{ + .userdata = self, + .vtable = &self.vtable, + }; + } + + fn stripScheme(path: []const u8) []const u8 { + if (std.mem.startsWith(u8, path, "file://")) { + return path[7..]; + } + return path; + } + + fn dirMake( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + mode: std.Io.Dir.Mode, + ) std.Io.Dir.MakeError!void { + const self: *File = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirMake(self.inner.userdata, dir, stripScheme(sub_path), mode); + } + + fn dirMakePath( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + mode: std.Io.Dir.Mode, + ) std.Io.Dir.MakeError!void { + const self: *File = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirMakePath(self.inner.userdata, dir, stripScheme(sub_path), mode); + } + + fn dirMakeOpenPath( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + options: std.Io.Dir.OpenOptions, + ) std.Io.Dir.MakeOpenPathError!std.Io.Dir { + const self: *File = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirMakeOpenPath(self.inner.userdata, dir, stripScheme(sub_path), options); + } + + fn dirStatPath( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + options: std.Io.Dir.StatPathOptions, + ) std.Io.Dir.StatPathError!std.Io.File.Stat { + const self: *File = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirStatPath(self.inner.userdata, dir, stripScheme(sub_path), options); + } + + fn dirAccess( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + options: std.Io.Dir.AccessOptions, + ) std.Io.Dir.AccessError!void { + const self: *File = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirAccess(self.inner.userdata, dir, stripScheme(sub_path), options); + } + + fn dirCreateFile( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + flags: std.Io.File.CreateFlags, + ) std.Io.File.OpenError!std.Io.File { + const self: *File = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirCreateFile(self.inner.userdata, dir, stripScheme(sub_path), flags); + } + + fn dirOpenFile( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + flags: std.Io.File.OpenFlags, + ) std.Io.File.OpenError!std.Io.File { + const self: *File = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirOpenFile(self.inner.userdata, dir, stripScheme(sub_path), flags); + } + + fn dirOpenDir( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + options: std.Io.Dir.OpenOptions, + ) std.Io.Dir.OpenError!std.Io.Dir { + const self: *File = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirOpenDir(self.inner.userdata, dir, stripScheme(sub_path), options); + } +}; diff --git a/zml2/io/vfs/index.zig b/zml2/io/vfs/index.zig index 621077a5..38b1c6e5 100644 --- a/zml2/io/vfs/index.zig +++ b/zml2/io/vfs/index.zig @@ -2,4 +2,5 @@ const std = @import("std"); const log = std.log.scoped(.@"zml/io/vfs/vfs"); +pub const File = @import("file.zig").File; pub const HTTP = @import("http.zig").HTTP; diff --git a/zml2/main_io.zig b/zml2/main_io.zig index d4a94a2f..13f2b6bb 100644 --- a/zml2/main_io.zig +++ b/zml2/main_io.zig @@ -34,18 +34,22 @@ pub fn main() !void { try http_client.initDefaultProxies(allocator); defer http_client.deinit(); + var vfs_file: zml.io.VFS.File = .init(threaded.io()); + var vfs_http: zml.io.VFS.HTTP = try .init(allocator, threaded.io(), &http_client, .{}); defer vfs_http.deinit(); var vfs: zml.io.VFS = .init(allocator, threaded.io()); defer vfs.deinit(); + try vfs.register("file", vfs_file.io()); try vfs.register("http", vfs_http.io()); try vfs.register("https", vfs_http.io()); const io = vfs.io(); - const repo_uri = "https://storage.googleapis.com/zig-vfs/Llama-3.1-8B-Instruct/"; + const repo_uri = "file:///Users/hugo/Developer/Llama-3.1-8B-Instruct"; + // const repo_uri = "https://storage.googleapis.com/zig-vfs/Llama-3.1-8B-Instruct/"; // const repo_uri = "http://9960x-5090x2:8003/"; { From 13d2753c3b5530c16c6b2826af3cd73a2033e7d6 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 11 Dec 2025 18:26:44 +0100 Subject: [PATCH 128/881] Re-enabled scatterConfig test --- zml2/ops.zig | 66 +++++++++++++++++++++++++++------------------------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/zml2/ops.zig b/zml2/ops.zig index 7a0eac91..3abfc80f 100644 --- a/zml2/ops.zig +++ b/zml2/ops.zig @@ -627,45 +627,47 @@ fn scatterConfig( }; } -// TODO(Corentin) -//test scatterConfig { -// const zml = @import("zml.zig"); -// const platform = zml.testing.env(); +test scatterConfig { + const zml = @import("zml.zig"); -// var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); -// defer comp.deinit(); -// comp.activate(); -// defer comp.deactivate(); + var comp = zml.module.CompilationContext.init(std.testing.allocator); + defer comp.deinit(); + comp.activate(); + defer comp.deactivate(); -// const Local = struct { -// pub fn _idx(idx_shape: anytype) Tensor { -// return Tensor.constant(idx_shape, .{ .i32 = 0 }); -// } -// }; + const block = mlir.Block.init(&.{}, &.{}); + comp.pushBlock(block); + defer comp.popBlock(); -// const idx = Local._idx; -// const op = Shape.init(.{ .a = 10, .b = 20 }, .f32); + const Local = struct { + pub fn _idx(idx_shape: anytype) Tensor { + return Tensor.constant(.{ .i32 = 0 }).broad(Shape.init(idx_shape, .i32)); + } + }; -// // Use .a as a batching axis with .a=10 x .n=8 updates of 2 elements of .b -// { -// const indices, const coords_tags = Shape.parseStruct(Tensor, .{ .b = idx(.{ .a = 10, .n = 8 }) }); -// const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); + const idx = Local._idx; + const op = Shape.init(.{ .a = 10, .b = 20 }, .f32); -// const cfg = scatterConfig(op, update, indices, coords_tags); -// try std.testing.expectEqualSlices(AxisKind, &.{ .batching, .update_window }, cfg.op_kind.constSlice()); -// try std.testing.expectEqualSlices(AxisKind, &.{ .batching, .window_id, .update_window }, cfg.up_kind.constSlice()); -// } + // Use .a as a batching axis with .a=10 x .n=8 updates of 2 elements of .b + { + const indices, const coords_tags = Shape.parseStruct(Tensor, .{ .b = idx(.{ .a = 10, .n = 8 }) }); + const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); -// // similar, but use the normalized form where .a is no longer an explicit batching axis. -// { -// const indices, const coords_tags = Shape.parseStruct(Tensor, .{ .a = idx(.{ .a = 10, .n = 8 }), .b = idx(.{ .a = 10, .n = 8 }) }); -// const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); + const cfg = scatterConfig(op, update, indices, coords_tags); + try std.testing.expectEqualSlices(ScatterAxisKind, &.{ .batching, .update_window }, cfg.op_kind.constSlice()); + try std.testing.expectEqualSlices(ScatterAxisKind, &.{ .batching, .window_id, .update_window }, cfg.up_kind.constSlice()); + } -// const cfg = scatterConfig(op, update, indices, coords_tags); -// try std.testing.expectEqualSlices(AxisKind, &.{ .inserted_window, .update_window }, cfg.op_kind.constSlice()); -// try std.testing.expectEqualSlices(AxisKind, &.{ .window_id, .window_id, .update_window }, cfg.up_kind.constSlice()); -// } -//} + // similar, but use the normalized form where .a is no longer an explicit batching axis. + { + const indices, const coords_tags = Shape.parseStruct(Tensor, .{ .a = idx(.{ .a = 10, .n = 8 }), .b = idx(.{ .a = 10, .n = 8 }) }); + const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); + + const cfg = scatterConfig(op, update, indices, coords_tags); + try std.testing.expectEqualSlices(ScatterAxisKind, &.{ .inserted_window, .update_window }, cfg.op_kind.constSlice()); + try std.testing.expectEqualSlices(ScatterAxisKind, &.{ .window_id, .window_id, .update_window }, cfg.up_kind.constSlice()); + } +} /// Concatenate all indices tensor in one tensor. /// From 92d42bde0aa5bb4365da30cfe8a658ca450b3af9 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 11 Dec 2025 19:02:49 +0100 Subject: [PATCH 129/881] Added platform and an arena to CompilationContext --- zml2/module.zig | 9 +++++++-- zml2/nn.zig | 6 ++++-- zml2/ops.zig | 3 ++- zml2/tensor.zig | 13 +++++++------ 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/zml2/module.zig b/zml2/module.zig index 51869d22..2fdf03bd 100644 --- a/zml2/module.zig +++ b/zml2/module.zig @@ -41,18 +41,20 @@ pub const CompilationContext = struct { }; allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, mlir_registry: *mlir.DialectRegistry, mlir_ctx: *mlir.Context, mlir_pass_manager: *mlir.PassManager, //mlir_op_pass_manager: *mlir.OpPassManager, module: *mlir.Module, + platform: Platform, scopes: stdx.BoundedArray(Scope, 16) = .{}, threadlocal var _current: ?*CompilationContext = null; - pub fn init(allocator: std.mem.Allocator) CompilationContext { + pub fn init(allocator: std.mem.Allocator, platform: Platform) CompilationContext { mlir.registerPasses("Transforms"); const mlir_registry = mlir.DialectRegistry.init() catch unreachable; inline for (.{ "func", "stablehlo" }) |d| { @@ -80,10 +82,12 @@ pub const CompilationContext = struct { return .{ .allocator = allocator, + .arena = std.heap.ArenaAllocator.init(allocator), .mlir_registry = mlir_registry, .mlir_ctx = mlir_ctx, .mlir_pass_manager = pass_manager, .module = module, + .platform = platform, }; } @@ -92,6 +96,7 @@ pub const CompilationContext = struct { self.module.deinit(); self.mlir_ctx.deinit(); self.mlir_registry.deinit(); + self.arena.deinit(); } pub fn activate(self: *CompilationContext) void { @@ -165,7 +170,7 @@ pub fn compileModel(allocator: std.mem.Allocator, io: std.Io, comptime func: any } pub fn compile(allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, args: stdx.meta.FnArgs(func), platform: Platform) !Exe { - var compilation_context: CompilationContext = .init(allocator); + var compilation_context: CompilationContext = .init(allocator, platform); defer compilation_context.deinit(); const result = emitMlir(&compilation_context, func, args) catch unreachable; diff --git a/zml2/nn.zig b/zml2/nn.zig index 184f7930..bef1e452 100644 --- a/zml2/nn.zig +++ b/zml2/nn.zig @@ -772,8 +772,9 @@ pub fn resizeBilinear(image: Tensor, resized_axes: anytype, opt: ResizeOpts) Ten } test resizeBilinear { + const platform = zml.testing.env(); // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator); + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -837,8 +838,9 @@ pub fn resizeBicubic(image: Tensor, resized_axes: anytype, opt: ResizeOpts) Tens } test resizeBicubic { + const platform = zml.testing.env(); // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator); + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); diff --git a/zml2/ops.zig b/zml2/ops.zig index 3abfc80f..b900df09 100644 --- a/zml2/ops.zig +++ b/zml2/ops.zig @@ -629,8 +629,9 @@ fn scatterConfig( test scatterConfig { const zml = @import("zml.zig"); + const platform = zml.testing.env(); - var comp = zml.module.CompilationContext.init(std.testing.allocator); + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); diff --git a/zml2/tensor.zig b/zml2/tensor.zig index c0f220d7..5b2cf569 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -2191,7 +2191,6 @@ pub const Tensor = struct { test gather { const zml = @import("zml.zig"); const platform = zml.testing.env(); - _ = platform; // autofix const Local = struct { pub fn _idx(idx_shape: anytype) Tensor { @@ -2203,7 +2202,7 @@ pub const Tensor = struct { { // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator); + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -2355,7 +2354,7 @@ pub const Tensor = struct { { // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator); + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -2573,7 +2572,7 @@ pub const Tensor = struct { { // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator); + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -3065,9 +3064,10 @@ pub const Tensor = struct { test chunkExact { const zml = @import("zml.zig"); + const platform = zml.testing.env(); // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator); + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -3124,9 +3124,10 @@ pub const Tensor = struct { test chunkAllowTrailing { const zml = @import("zml.zig"); + const platform = zml.testing.env(); // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator); + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); From 9d5e54fddc36235c42614033bb8c12d473598908 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 11 Dec 2025 19:03:07 +0100 Subject: [PATCH 130/881] Re-introduced ops.customCall + test --- zml2/ops.zig | 188 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 188 insertions(+) diff --git a/zml2/ops.zig b/zml2/ops.zig index b900df09..d82a218e 100644 --- a/zml2/ops.zig +++ b/zml2/ops.zig @@ -840,6 +840,194 @@ pub fn _collectAxes(T: type, bounded_array: stdx.BoundedArray(T, constants.MAX_R return res; } +fn TensorOrTensorArray(comptime T: type) type { + const type_info = @typeInfo(T); + return switch (type_info) { + .@"struct" => |struct_info| b: { + if (T == Tensor) break :b Tensor; + if (!struct_info.is_tuple) @compileError("Expected tuple"); + break :b if (struct_info.fields.len == 1) + Tensor + else + [struct_info.fields.len]Tensor; + }, + .array => |array_info| b: { + break :b if (array_info.len == 1) + Tensor + else + [array_info.len]Tensor; + }, + .pointer => |pointer_info| b: { + if (pointer_info.size != .slice) @compileError("Expected slice"); + break :b []Tensor; + }, + else => @compileError("Unsupported type: " ++ @typeName(T)), + }; +} + +pub const CustomCallOptions = struct { + has_side_effect: bool, + output_operand_aliases: ?[]const i64 = null, +}; + +pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, metadata: anytype, opts: CustomCallOptions) TensorOrTensorArray(@TypeOf(outputs)) { + // Transform generic inputs to flat slice. + const inputs_: []const Tensor = switch (@typeInfo(@TypeOf(inputs))) { + .@"struct" => |struct_info| b: { + if (@TypeOf(inputs) == Tensor) { + break :b &[1]Tensor{inputs}; + } + if (!struct_info.is_tuple) @compileError("Expected tuple"); + var inputs_: [struct_info.fields.len]Tensor = undefined; + meta.collectBuf((struct { + pub fn func(t: Tensor) Tensor { + return t; + } + }).func, {}, &inputs, &inputs_); + break :b &inputs_; + }, + .array => &inputs, + .pointer => |pointer_info| b: { + if (pointer_info.size != .slice) @compileError("Expected slice"); + break :b inputs; + }, + else => @compileError("Unsupported type: " ++ @typeName(@TypeOf(inputs))), + }; + + // Transform generic outputs to flat slice. + const output_shapes: []const Shape = switch (@typeInfo(@TypeOf(outputs))) { + .@"struct" => |struct_info| b: { + if (@TypeOf(outputs) == Shape) { + break :b &[1]Shape{outputs}; + } + if (!struct_info.is_tuple) @compileError("Expected tuple"); + var output_shapes: [struct_info.fields.len]Shape = undefined; + meta.collectBuf((struct { + pub fn func(t: Shape) Shape { + return t; + } + }).func, {}, &outputs, &output_shapes); + break :b &output_shapes; + }, + .array => &outputs, + .pointer => |pointer_info| b: { + if (pointer_info.size != .slice) @compileError("Expected slice"); + break :b outputs; + }, + else => @compileError("Unsupported type: " ++ @typeName(@TypeOf(outputs))), + }; + + const outputs_flat = customCallInternal(target_name, inputs_, output_shapes, metadata, opts); + + // Transform flat slice to generic outputs. + return switch (@typeInfo(@TypeOf(outputs))) { + .@"struct" => |struct_info| b: { + if (@TypeOf(outputs) == Shape) break :b outputs_flat[0]; + if (!struct_info.is_tuple) @compileError("Expected tuple"); + if (struct_info.fields.len == 1) break :b outputs_flat[0]; + var outputs_: [struct_info.fields.len]Tensor = undefined; + @memcpy(&outputs_, outputs_flat); + break :b outputs_; + }, + .array => |array_info| b: { + if (array_info.len == 1) break :b outputs_flat[0]; + var outputs_: [array_info.fields.len]Tensor = undefined; + @memcpy(&outputs_, outputs_flat); + break :b outputs_; + }, + .pointer => |pointer_info| b: { + if (pointer_info.size != .slice) @compileError("Expected slice"); + break :b outputs_flat; + }, + else => @compileError("Unsupported type: " ++ @typeName(@TypeOf(outputs))), + }; +} + +fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs: []const Shape, metadata: anytype, opts: CustomCallOptions) []Tensor { + const ctx = CompilationContext.current(); + var arena = std.heap.ArenaAllocator.init(ctx.allocator); + defer arena.deinit(); + + const values = arena.allocator().alloc(*const mlir.Value, inputs.len) catch unreachable; + for (0..inputs.len) |i| { + values[i] = inputs[i].value(); + } + + const res_types = arena.allocator().alloc(*const mlir.Type, outputs.len) catch unreachable; + for (outputs, 0..) |output, i| { + res_types[i] = mlir.rankedTensorType(output.dims(), mlirx.Type.fromDType(ctx.mlir_ctx, output.dtype())); + } + + const metadata_type_info = @typeInfo(@TypeOf(metadata)); + var metadata_attributes: [metadata_type_info.@"struct".fields.len]mlir.NamedAttribute = undefined; + inline for (metadata_type_info.@"struct".fields, 0..) |field, i| { + const attribute: *const mlir.Attribute = switch (@typeInfo(field.type)) { + .int, .comptime_int => mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@field(metadata, field.name)))), + else => @compileError("Unsupported metadata type: " ++ @typeName(field.type)), + }; + metadata_attributes[i] = mlir.NamedAttribute.named(ctx.mlir_ctx, field.name, attribute); + } + + const backend_config = mlir.dictionaryAttribute(ctx.mlir_ctx, &(metadata_attributes ++ [_]mlir.NamedAttribute{ + .named(ctx.mlir_ctx, "pjrt_api", mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_api))))), + .named(ctx.mlir_ctx, "pjrt_client", mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_client))))), + })); + + const operands_layouts = arena.allocator().alloc([]const usize, inputs.len) catch unreachable; + for (inputs, 0..) |input, i| { + operands_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(input.rank())).constSlice()) catch unreachable; + } + + const results_layouts = arena.allocator().alloc([]const usize, outputs.len) catch unreachable; + for (outputs, 0..) |output, i| { + results_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(output.rank())).constSlice()) catch unreachable; + } + + const op = dialects.stablehlo.custom_call( + ctx.mlir_ctx, + values, + res_types, + .{ + .call_target_name = target_name, + .backend_config = .{ .typed_ffi = backend_config }, + .has_side_effect = opts.has_side_effect, + .operand_layouts = operands_layouts, + .result_layouts = results_layouts, + .output_operand_aliases = opts.output_operand_aliases orelse &.{}, + }, + .unknown(ctx.mlir_ctx), + ); + + const outputs_ = ctx.arena.allocator().alloc(Tensor, outputs.len) catch unreachable; + for (outputs, 0..) |output, i| { + outputs_[i] = Tensor._result(output, op.result(i)); + } + + return outputs_; +} + +test customCall { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); + defer comp.deinit(); + comp.activate(); + defer comp.deactivate(); + + const block = mlir.Block.init(&.{}, &.{}); + comp.pushBlock(block); + defer comp.popBlock(); + + const input = Tensor.constant(zml.DataType.bf16.constant(0)).broad(Shape.init(.{128}, .bf16)); + const output = customCall("my_custom_call", .{input}, .{zml.Shape.init(.{128}, .bf16)}, .{}, .{ + .has_side_effect = false, + .output_operand_aliases = &.{0}, + }); + + try zml.testing.expectEqualShapes(input.shape(), output.shape()); +} + fn toUsize(values: anytype) stdx.BoundedArray(usize, constants.MAX_RANK) { var res: stdx.BoundedArray(usize, constants.MAX_RANK) = .{}; for (values) |val| res.appendAssumeCapacity(@intCast(val)); From ed4bc7183d30b682ae5dfefc755900bb3236a62d Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 11 Dec 2025 19:06:45 +0100 Subject: [PATCH 131/881] Use CompilationContext arena for chunkAllowTrailing --- zml2/tensor.zig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 5b2cf569..6f060ed3 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -3097,7 +3097,6 @@ pub const Tensor = struct { /// When `self.dim(axis_)` is divisible by `n_chunks` it will return exactly `n_chunks`. pub fn chunkAllowTrailing( self: Tensor, - allocator: std.mem.Allocator, axis_: i64, n_chunks: comptime_int, ) ![]Tensor { @@ -3106,6 +3105,8 @@ pub const Tensor = struct { const chunk_size: i64 = @divFloor(d, n_chunks); const tail_chunk_size: i64 = @rem(d, chunk_size); + const allocator = CompilationContext.current().arena.allocator(); + var chunks = std.ArrayList(Tensor).initCapacity(allocator, n_chunks + 1) catch @panic("OOM"); defer chunks.deinit(allocator); @@ -3145,8 +3146,7 @@ pub const Tensor = struct { }) |testcase| { const x_shape, const ax, const n_chunks, const res, const trailing = testcase; const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); - const chunks = try x.chunkAllowTrailing(std.testing.allocator, x.axis(ax), n_chunks); - defer std.testing.allocator.free(chunks); + const chunks = try x.chunkAllowTrailing(x.axis(ax), n_chunks); const res_shape = Shape.init(res, .f16); for (chunks[0..n_chunks]) |chk| { From b0f7f01a38f009637de770cdfcbcd51377cd29c2 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 11 Dec 2025 20:31:19 +0100 Subject: [PATCH 132/881] Simplified Tensor.init signature --- zml2/main.zig | 6 +-- zml2/nn.zig | 28 ++++++------ zml2/ops.zig | 20 ++++---- zml2/tensor.zig | 116 ++++++++++++++++++++++++----------------------- zml2/testing.zig | 2 +- 5 files changed, 88 insertions(+), 84 deletions(-) diff --git a/zml2/main.zig b/zml2/main.zig index 395051ec..9fdb52eb 100644 --- a/zml2/main.zig +++ b/zml2/main.zig @@ -7,8 +7,8 @@ pub const Model = struct { pub fn init() !Model { return .{ - .weight = .init(zml.Shape.init(.{ .m = 4096, .n = 4096 }, .f32)), - .bias = .init(zml.Shape.init(.{ .m = 4096 }, .f32)), + .weight = .init(.{ .m = 4096, .n = 4096 }, .f32), + .bias = .init(.{ .m = 4096 }, .f32), }; } @@ -63,7 +63,7 @@ pub fn main() !void { var model: Model = try .init(); defer model.deinit(); - const input: zml.Tensor = .init(zml.Shape.init(.{ .n = 4096 }, .f32)); + const input: zml.Tensor = .init(.{ .n = 4096 }, .f32); var exe = try zml.module.compileModel(allocator, io, Model.forward, model, .{input}, platform); defer exe.deinit(); diff --git a/zml2/nn.zig b/zml2/nn.zig index bef1e452..3317179c 100644 --- a/zml2/nn.zig +++ b/zml2/nn.zig @@ -120,7 +120,7 @@ pub fn normalizeL2(input: Tensor, eps: f32) Tensor { test normalizeL2 { const platform = zml.testing.env(); - const input: zml.Tensor = .init(Shape.init(.{ 2, 2 }, .f32)); + const input: zml.Tensor = .init(.{ 2, 2 }, .f32); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, normalizeL2, .{ input, 1e-12 }, platform); defer exe.deinit(); @@ -511,7 +511,7 @@ test rope { // x is made such as the interleaved and sequential reps are the same. // So the two implementations should give the same results. - const x: zml.Tensor = .init(Shape.init(.{ .b = 1, .s = 5, .hd = 4 }, .f32)); + const x: zml.Tensor = .init(.{ .b = 1, .s = 5, .hd = 4 }, .f32); var exe_interleaved = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .interleaved } }, platform); defer exe_interleaved.deinit(); @@ -590,7 +590,7 @@ test nearest { // 3D Tensor (basic) { - const input_3d_basic: zml.Tensor = .init(Shape.init(.{ 1, 1, 2 }, .i32)); + const input_3d_basic: zml.Tensor = .init(.{ 1, 1, 2 }, .i32); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_3d_basic, .{ .scale_factor = &.{3}, .mode = .nearest } }, platform); defer exe.deinit(); @@ -607,7 +607,7 @@ test nearest { // 3D Tensor (advanced) { - const input_3d_advanced: zml.Tensor = .init(Shape.init(.{ 2, 3, 4 }, .i32)); + const input_3d_advanced: zml.Tensor = .init(.{ 2, 3, 4 }, .i32); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_3d_advanced, .{ .scale_factor = &.{2}, .mode = .nearest } }, platform); defer exe.deinit(); @@ -638,7 +638,7 @@ test nearest { // 4D Tensor (basic) { - const input_4d_basic: zml.Tensor = .init(Shape.init(.{ 1, 1, 2, 2 }, .i32)); + const input_4d_basic: zml.Tensor = .init(.{ 1, 1, 2, 2 }, .i32); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_4d_basic, .{ .scale_factor = &.{ 3, 3 }, .mode = .nearest } }, platform); defer exe.deinit(); @@ -661,7 +661,7 @@ test nearest { } // 4D Tensor (advanced) { - const input_4d_advanced: zml.Tensor = .init(Shape.init(.{ 2, 2, 2, 2 }, .i32)); + const input_4d_advanced: zml.Tensor = .init(.{ 2, 2, 2, 2 }, .i32); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_4d_advanced, .{ .scale_factor = &.{ 2, 2 }, .mode = .nearest } }, platform); defer exe.deinit(); @@ -712,7 +712,7 @@ test nearest { } // 5D Tensor (basic) { - const input_5d: zml.Tensor = .init(Shape.init(.{ 1, 1, 1, 2, 2 }, .i32)); + const input_5d: zml.Tensor = .init(.{ 1, 1, 1, 2, 2 }, .i32); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_5d, .{ .scale_factor = &.{2}, .mode = .nearest } }, platform); defer exe.deinit(); @@ -1055,7 +1055,7 @@ test sampleTokens { const platform = zml.testing.env(); const rng: zml.Tensor.Rng = .init(); - const activations: zml.Tensor = .init(Shape.init(.{ .voc = 4 }, .f32)); + const activations: zml.Tensor = .init(.{ .voc = 4 }, .f32); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, sampleTokens, .{ activations, .{ .topk = 4, .temperature = 2.0 }, rng }, platform); defer exe.deinit(); @@ -1101,10 +1101,10 @@ pub const DynamicSamplingStrategy = struct { const scalar_i32_shape = Shape.init(.{}, .i32); return .{ .max_top_k = max_top_k, - .top_k = .init(scalar_i32_shape), - .temperature = .init(scalar_float_shape), - .top_p = .init(scalar_float_shape), - .min_p = .init(scalar_float_shape), + .top_k = .fromShape(scalar_i32_shape), + .temperature = .fromShape(scalar_float_shape), + .top_p = .fromShape(scalar_float_shape), + .min_p = .fromShape(scalar_float_shape), }; } @@ -1194,7 +1194,7 @@ test sampleTokensDynamic { const logits_data = [_]f32{ @log(2.0), @log(1.0), @log(4.0), @log(3.0) }; const top_k_indices = [_]i32{ 2, 3, 0, 1 }; - const logits: zml.Tensor = .init(zml.Shape.init(.{ .voc = logits_data.len }, .f32)); + const logits: zml.Tensor = .init(.{ .voc = logits_data.len }, .f32); const dynamic_sampling_strategy = DynamicSamplingStrategy.init(.f32, 0); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, fixupLogits, .{ logits, dynamic_sampling_strategy }, platform); @@ -1232,7 +1232,7 @@ test sampleTokensDynamic { // Similar but use bf16, and uses infinity to trigger nans after the softmax. const bf16 = zml.floats.BFloat16; - const logits_bf16: zml.Tensor = .init(Shape.init(.{ .voc = logits_data.len }, .bf16)); + const logits_bf16: zml.Tensor = .init(.{ .voc = logits_data.len }, .bf16); const dynamic_sampling_strategy_bf16 = DynamicSamplingStrategy.init(.bf16, 0); var exe_bf16 = try zml.module.compile(std.testing.allocator, std.testing.io, fixupLogits, .{ logits_bf16, dynamic_sampling_strategy_bf16 }, platform); defer exe_bf16.deinit(); diff --git a/zml2/ops.zig b/zml2/ops.zig index d82a218e..fadc9065 100644 --- a/zml2/ops.zig +++ b/zml2/ops.zig @@ -28,8 +28,8 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func var block_types: [2 * inits.len]*const mlir.Type = undefined; inline for (0..inits.len) |i| { - args[i].left = Tensor.init(inits[i].shape()); - args[i].right = Tensor.init(inits[i].shape()); + args[i].left = .fromShape(inits[i].shape()); + args[i].right = .fromShape(inits[i].shape()); block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); @@ -131,8 +131,8 @@ pub fn reduceWindow(inputs: anytype, inits: anytype, opts: ReduceWindowOpts, com var block_types: [2 * inits.len]*const mlir.Type = undefined; inline for (0..inits.len) |i| { - args[i].left = Tensor.init(inits[i].shape()); - args[i].right = Tensor.init(inits[i].shape()); + args[i].left = .fromShape(inits[i].shape()); + args[i].right = .fromShape(inits[i].shape()); block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); @@ -209,8 +209,8 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytyp var block_types: [2 * inputs.len]*const mlir.Type = undefined; inline for (0..inputs.len) |i| { - args[i].left = Tensor.init(Shape.init(.{}, inputs[i].shape().dtype())); - args[i].right = Tensor.init(Shape.init(.{}, inputs[i].shape().dtype())); + args[i].left = Tensor.init(.{}, inputs[i].shape().dtype()); + args[i].right = Tensor.init(.{}, inputs[i].shape().dtype()); block_types[2 * i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); block_types[2 * i + 1] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); @@ -363,8 +363,8 @@ test "triton" { } }; - const a: zml.Tensor = .init(Shape.init(.{}, .f32)); - const b: zml.Tensor = .init(Shape.init(.{}, .f32)); + const a: zml.Tensor = .init(.{}, .f32); + const b: zml.Tensor = .init(.{}, .f32); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, TritonMod.forward, .{ a, b }, platform); defer exe.deinit(); @@ -423,8 +423,8 @@ pub fn scatter( var block_types: [2 * inputs.len]*const mlir.Type = undefined; inline for (0..inputs.len) |i| { - args[i].input = Tensor.init(Shape.init(.{}, inputs[i].dtype())); - args[i].update = Tensor.init(Shape.init(.{}, inputs[i].dtype())); + args[i].input = Tensor.init(.{}, inputs[i].dtype()); + args[i].update = Tensor.init(.{}, inputs[i].dtype()); block_types[i] = mlir.rankedTensorType(args[i].input.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].input.dtype())); block_types[i + inputs.len] = mlir.rankedTensorType(args[i].update.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].update.dtype())); diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 6f060ed3..3f3f6230 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -25,7 +25,11 @@ pub const Tensor = struct { _shape: Shape, _value: ?*const mlir.Value = null, - pub fn init(shape_: Shape) Tensor { + pub fn init(shape_like: anytype, dt: DataType) Tensor { + return .fromShape(.init(shape_like, dt)); + } + + pub fn fromShape(shape_: Shape) Tensor { return .{ .id = Tensor.current_id.fetchAdd(1, .seq_cst), ._shape = shape_ }; } @@ -283,7 +287,7 @@ pub const Tensor = struct { const divisors: [2]f32 = .{ 2, -1.5 }; inline for (inputs, expectations, divisors) |i, e, d| { - const input: zml.Tensor = .init(Shape.init(.{6}, .f32)); + const input: zml.Tensor = .init(.{6}, .f32); const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.fmod, .{ input, d }, platform); defer exe.deinit(); @@ -444,7 +448,7 @@ pub const Tensor = struct { algorithm: dialects.stablehlo.RngAlgorithm.Type = .DEFAULT, pub fn init() Rng { - return .{ ._state = .init(Shape.init(.{2}, .u64)) }; + return .{ ._state = .init(.{2}, .u64) }; } pub fn initBuffer(platform: Platform, seed: u128, io: std.Io) !Bufferized(Rng) { @@ -650,7 +654,7 @@ pub const Tensor = struct { const platform = zml.testing.env(); const tgt_dist_data = [_]f32{ 2.0, 1.0, 4.0, 3.0 }; - const tgt_dist: Tensor = .init(Shape.init(.{tgt_dist_data.len}, .f32)); + const tgt_dist: Tensor = .init(.{tgt_dist_data.len}, .f32); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.gumbelStats, .{ Rng.init(), tgt_dist }, platform); defer exe.deinit(); @@ -984,7 +988,7 @@ pub const Tensor = struct { var x_f4: [x.len]floats.Float4E2M1 = undefined; for (&x_f4, &x) |*xi_f4, xi| xi_f4.* = .fromF32(xi); - const x_d: Tensor = .init(Shape.init(.{x.len}, .f32)); + const x_d: Tensor = .init(.{x.len}, .f32); const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.convert, .{ x_d, .f4e2m1 }, platform); defer exe.deinit(); @@ -1005,7 +1009,7 @@ pub const Tensor = struct { var x_f8e3: [x.len]floats.Float8E3M4 = undefined; for (&x_f8e3, &x) |*xi_f8e3, xi| xi_f8e3.* = .fromF32(xi); - const x_d: Tensor = .init(Shape.init(.{x.len}, .f32)); + const x_d: Tensor = .init(.{x.len}, .f32); const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.convert, .{ x_d, .f8e3m4 }, platform); defer exe.deinit(); @@ -1084,8 +1088,8 @@ pub const Tensor = struct { // .{ .b = 21, .d = 23 }, //}, }) |testcase| { - const x: Tensor = .init(Shape.init(testcase[0], .f32)); - const y: Tensor = .init(Shape.init(testcase[1], .f32)); + const x: Tensor = .init(testcase[0], .f32); + const y: Tensor = .init(testcase[1], .f32); const ctr = Shape.toTag(testcase[2]); const z_shape: Shape = .init(testcase[3], .f32); const forward = struct { @@ -1204,7 +1208,7 @@ pub const Tensor = struct { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const input: Tensor = .init(Shape.init(.{2}, .f32)); + const input: Tensor = .init(.{2}, .f32); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.leakyReLU, .{ input, 0.1 }, platform); defer exe.deinit(); @@ -1359,7 +1363,7 @@ pub const Tensor = struct { } }; - const x: Tensor = .init(Shape.init(.{ 2, 5 }, .f32)); + const x: Tensor = .init(.{ 2, 5 }, .f32); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cumsum, .{x}, platform); defer exe.deinit(); @@ -1562,7 +1566,7 @@ pub const Tensor = struct { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const x: Tensor = .init(Shape.init(.{ 2, 5 }, .f32)); + const x: Tensor = .init(.{ 2, 5 }, .f32); const x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }), std.testing.io); defer x_buffer.deinit(); @@ -1708,7 +1712,7 @@ pub const Tensor = struct { .{ [2][3]u8{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }, Shape.init(.{ 2, 3 }, .u8), [2][6]u8{ .{ 1, 2, 3, 1, 2, 3 }, .{ 4, 5, 6, 4, 5, 6 } }, 1, 2 }, }) |testcase| { const input_data, const shape_, const expectation, const ax, const reps = testcase; - const input: Tensor = .init(shape_); + const input: Tensor = .fromShape(shape_); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.repeat1d, .{ input, ax, reps }, platform); defer exe.deinit(); @@ -1763,7 +1767,7 @@ pub const Tensor = struct { .{ [2][3]u8{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }, Shape.init(.{ 2, 3 }, .u8), [2][6]u8{ .{ 1, 2, 3, 1, 2, 3 }, .{ 4, 5, 6, 4, 5, 6 } }, 0, 2 }, }) |testcase| { const input_data, const shape_, const expectation, const ax, const reps = testcase; - const input: Tensor = .init(shape_); + const input: Tensor = .fromShape(shape_); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.stutter1d, .{ input, ax, reps }, platform); defer exe.deinit(); @@ -2395,8 +2399,8 @@ pub const Tensor = struct { } // Test with actual values. - const operand: Tensor = .init(Shape.init(.{ .a = 2, .b = 4, .c = 6 }, .u16)); - const indices: Tensor = .init(Shape.init(.{ .n = 2, ._ = 2 }, .i32)); + const operand: Tensor = .init(.{ .a = 2, .b = 4, .c = 6 }, .u16); + const indices: Tensor = .init(.{ .n = 2, ._ = 2 }, .i32); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._gatherSlices, .{ operand, Shape.init(.{ .b = 2, .c = 3 }, .u16), indices, .{} }, platform); defer exe.deinit(); @@ -2610,10 +2614,10 @@ pub const Tensor = struct { } // Test with actual values, no batching. { - const a: Tensor = .init(Shape.init(.{ 3, 3 }, .i32)); + const a: Tensor = .init(.{ 3, 3 }, .i32); - const scatter_indices = Tensor.init(Shape.init(.{2}, .i32)).withTags(.{.n}); - const updates = Tensor.init(Shape.init(.{ 2, 3 }, .i32)).withTags(.{ .n, .b }); + const scatter_indices = Tensor.init(.{2}, .i32).withTags(.{.n}); + const updates = Tensor.init(.{ 2, 3 }, .i32).withTags(.{ .n, .b }); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform); defer exe.deinit(); @@ -2634,10 +2638,10 @@ pub const Tensor = struct { } // Test with setting individual values (no batching) { - const a: Tensor = .init(Shape.init(.{9}, .i32)); + const a: Tensor = .init(.{9}, .i32); - const scatter_indices = Tensor.init(Shape.init(.{2}, .i32)).withTags(.{.n}); - const updates = Tensor.init(Shape.init(.{2}, .i32)).withTags(.{.n}); + const scatter_indices = Tensor.init(.{2}, .i32).withTags(.{.n}); + const updates = Tensor.init(.{2}, .i32).withTags(.{.n}); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform); defer exe.deinit(); @@ -2658,9 +2662,9 @@ pub const Tensor = struct { } { // Test with actual values and batching along axis .a - const operand: Tensor = .init(Shape.init(.{ .a = 2, .b = 3, .c = 4, .d = 2 }, .u16)); - const start_indices: Tensor = .init(Shape.init(.{ .n = 2, .a = 2, .m = 3, .coord = 2 }, .i32)); - const values: Tensor = .init((Shape.init(.{ .n = 2, .a = 2, .m = 3, .c = 2, .d = 2 }, .u16))); + const operand: Tensor = .init(.{ .a = 2, .b = 3, .c = 4, .d = 2 }, .u16); + const start_indices: Tensor = .init(.{ .n = 2, .a = 2, .m = 3, .coord = 2 }, .i32); + const values: Tensor = .init(.{ .n = 2, .a = 2, .m = 3, .c = 2, .d = 2 }, .u16); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatterCB, .{ operand, start_indices, values }, platform); defer exe.deinit(); @@ -2779,7 +2783,7 @@ pub const Tensor = struct { } }; - const x: Tensor = .init(Shape.init(.{ 1, 5 }, .f32)); + const x: Tensor = .init(.{ 1, 5 }, .f32); var exe = try zml.module.compile(allocator, std.testing.io, ArgMaxTest._fwd, .{x}, platform); defer exe.deinit(); @@ -2847,7 +2851,7 @@ pub const Tensor = struct { // 2D Tensor - dim = 1, ascending { - const x: Tensor = .init(Shape.init(.{ 2, 5 }, .f32)); + const x: Tensor = .init(.{ 2, 5 }, .f32); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{} }, platform); defer exe.deinit(); @@ -2866,7 +2870,7 @@ pub const Tensor = struct { // 3D Tensor, dim = 1, descending { - const x: Tensor = .init(Shape.init(.{ 1, 5, 10 }, .f16)); + const x: Tensor = .init(.{ 1, 5, 10 }, .f16); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{ .descending = true } }, platform); defer exe.deinit(); @@ -2897,7 +2901,7 @@ pub const Tensor = struct { // 4D Tensor, dim = 3, ascending { - const x: Tensor = .init(Shape.init(.{ 4, 2, 1, 4 }, .i32)); + const x: Tensor = .init(.{ 4, 2, 1, 4 }, .i32); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 3, .{} }, platform); defer exe.deinit(); @@ -3269,8 +3273,8 @@ pub const Tensor = struct { .{ [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }, Shape.init(.{ 2, 5 }, .f32), [4]f32{ 3, 4, 8, 9 }, 3, 1 }, }) |testcase| { const x_data, const x_shape, const expectation, const z_value: i32, const ax = testcase; - const x: Tensor = .init(x_shape); - const z: Tensor = .init(Shape.init(.{}, .i32)); + const x: Tensor = .fromShape(x_shape); + const z: Tensor = .init(.{}, .i32); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.dynamicSlice1d, .{ x, ax, .{ .len = 2, .start = z } }, platform); defer exe.deinit(); @@ -3388,9 +3392,9 @@ pub const Tensor = struct { const platform = zml.testing.env(); { - const x = Tensor.init(Shape.init(.{10}, .f32)).withTags(.{.a}); - const y = Tensor.init(Shape.init(.{2}, .f32)).withTags(.{.a}); - const ids: Tensor = .init(Shape.init(.{}, .i32)); + const x = Tensor.init(.{10}, .f32).withTags(.{.a}); + const y = Tensor.init(.{2}, .f32).withTags(.{.a}); + const ids: Tensor = .init(.{}, .i32); const forward = struct { pub fn _fwd(x_: Tensor, idx_: struct { a: Tensor }, y_: Tensor) Tensor { return x_.dynamicUpdateSlice(idx_, y_); @@ -3414,9 +3418,9 @@ pub const Tensor = struct { { // Updates 2D, tagged api - const x = Tensor.init(Shape.init(.{ 2, 5 }, .f32)).withTags(.{ .a, .b }); - const y = Tensor.init(Shape.init(.{2}, .f32)).withTags(.{.a}); - const ids: Tensor = .init(Shape.init(.{}, .i32)); + const x = Tensor.init(.{ 2, 5 }, .f32).withTags(.{ .a, .b }); + const y = Tensor.init(.{2}, .f32).withTags(.{.a}); + const ids: Tensor = .init(.{}, .i32); const forward = struct { pub fn _fwd(x_: Tensor, idx_: Tensor, y_: Tensor) Tensor { return x_.dynamicUpdateSlice(.{ .b = idx_ }, y_); @@ -3443,9 +3447,9 @@ pub const Tensor = struct { { // Updates 2D slice, un-tagged api. Note that `y` needs to have a 1 dimension axis. - const x: Tensor = .init(Shape.init(.{ 2, 5 }, .f32)); - const y: Tensor = .init(Shape.init(.{ 2, 1 }, .f32)); - const ids: Tensor = .init(Shape.init(.{}, .i32)); + const x: Tensor = .init(.{ 2, 5 }, .f32); + const y: Tensor = .init(.{ 2, 1 }, .f32); + const ids: Tensor = .init(.{}, .i32); const forward = struct { pub fn _fwd(x_: Tensor, idx_: Tensor, y_: Tensor) Tensor { return x_.dynamicUpdateSlice(.{ zml.Tensor.scalar(0, .i32), idx_ }, y_); @@ -3472,10 +3476,10 @@ pub const Tensor = struct { { // Updates 2D, partial update - const x = Tensor.init(Shape.init(.{ 2, 5 }, .f32)).withTags(.{ .a, .b }); - const y = Tensor.init(Shape.init(.{1}, .f32)).withTags(.{.a}); - const idx_a: Tensor = .init(Shape.init(.{}, .i32)); - const idx_b: Tensor = .init(Shape.init(.{}, .i32)); + const x = Tensor.init(.{ 2, 5 }, .f32).withTags(.{ .a, .b }); + const y = Tensor.init(.{1}, .f32).withTags(.{.a}); + const idx_a: Tensor = .init(.{}, .i32); + const idx_b: Tensor = .init(.{}, .i32); const forward = struct { pub fn _fwd(x_: Tensor, idx_: struct { a: Tensor, b: Tensor }, y_: Tensor) Tensor { return x_.dynamicUpdateSlice(idx_, y_); @@ -3504,10 +3508,10 @@ pub const Tensor = struct { { // Updates 2D, partial update, un-tagged api. - const x: Tensor = .init(Shape.init(.{ 2, 5 }, .f32)); - const y: Tensor = .init(Shape.init(.{ 1, 1 }, .f32)); - const idx_a: Tensor = .init(Shape.init(.{}, .i32)); - const idx_b: Tensor = .init(Shape.init(.{}, .i32)); + const x: Tensor = .init(.{ 2, 5 }, .f32); + const y: Tensor = .init(.{ 1, 1 }, .f32); + const idx_a: Tensor = .init(.{}, .i32); + const idx_b: Tensor = .init(.{}, .i32); const forward = struct { pub fn _fwd(x_: Tensor, idx_: [2]Tensor, y_: Tensor) Tensor { return x_.dynamicUpdateSlice(&idx_, y_); @@ -3592,7 +3596,7 @@ pub const Tensor = struct { } }; - const x: Tensor = .init(Shape.init(.{ 2, 2 }, .u8)); + const x: Tensor = .init(.{ 2, 2 }, .u8); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._toDiag, .{x}, platform); defer exe.deinit(); @@ -3645,7 +3649,7 @@ pub const Tensor = struct { } }; - const x: Tensor = .init(Shape.init(.{ 3, 3 }, .u8)); + const x: Tensor = .init(.{ 3, 3 }, .u8); var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[3][3]u8{ .{ 1, 1, 1 }, @@ -3813,8 +3817,8 @@ pub const Tensor = struct { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const x: Tensor = .init(Shape.init(.{6}, .i32)); - const y: Tensor = .init(Shape.init(.{4}, .i32)); + const x: Tensor = .init(.{6}, .i32); + const y: Tensor = .init(.{4}, .i32); const Local = struct { pub fn _cartesianProduct2(a: Tensor, b: Tensor) [2]Tensor { @@ -3875,8 +3879,8 @@ pub const Tensor = struct { test cartesianProductStacked { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const x: Tensor = .init(Shape.init(.{6}, .i32)); - const y: Tensor = .init(Shape.init(.{4}, .i32)); + const x: Tensor = .init(.{6}, .i32); + const y: Tensor = .init(.{4}, .i32); const Local = struct { pub fn _fwd(a: Tensor, b: Tensor) Tensor { @@ -4022,7 +4026,7 @@ test "Tensor.maxPool1d" { var data: [20]f32 = undefined; for (&data, 0..) |*v, i| v.* = @floatFromInt(i); - const x: Tensor = .init(Shape.init(.{ 2, 2, 5 }, .f32)); + const x: Tensor = .init(.{ 2, 2, 5 }, .f32); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform); defer exe.deinit(); @@ -4061,7 +4065,7 @@ test "Tensor.maxPool2d" { var data: [100]f32 = undefined; for (&data, 0..) |*v, i| v.* = @floatFromInt(i); - const x: Tensor = .init(Shape.init(.{ 2, 2, 5, 5 }, .f32)); + const x: Tensor = .init(.{ 2, 2, 5, 5 }, .f32); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform); defer exe.deinit(); @@ -4163,6 +4167,6 @@ test "unused tensor" { } }; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{Tensor.init(Shape.init(.{10}, .f32))}, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{Tensor.init(.{10}, .f32)}, platform); defer exe.deinit(); } diff --git a/zml2/testing.zig b/zml2/testing.zig index e7d8df8b..551b6ed3 100644 --- a/zml2/testing.zig +++ b/zml2/testing.zig @@ -153,7 +153,7 @@ pub fn compileAndCall( var context2: LocalContext2 = .{ .shape_list = &shape_list }; try zml.meta.visit(struct { fn cb(context_: *LocalContext2, tensor: *zml.Tensor) !void { - tensor.* = zml.Tensor.init(context_.shape_list.items[context_.index]); + tensor.* = .fromShape(context_.shape_list.items[context_.index]); context_.index += 1; } }.cb, &context2, &func_args); From 1e9fe65c337c1ee1d11214aefe818e94aa8d9c0f Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 12 Dec 2025 14:00:21 +0100 Subject: [PATCH 133/881] Some work on Platform * Added a way to automatically select the best available platform * Added a way to print available platforms --- zml2/main.zig | 7 ++++- zml2/platform.zig | 68 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/zml2/main.zig b/zml2/main.zig index 9fdb52eb..5e66eeaa 100644 --- a/zml2/main.zig +++ b/zml2/main.zig @@ -57,9 +57,14 @@ pub fn main() !void { zml.init(); defer zml.deinit(); - var platform = try zml.Platform.init(.cpu, io, .{}); + const available = zml.Platform.availablePlatforms(io); + std.log.info("Available platforms: {any}", .{available}); + + var platform = try zml.Platform.auto(io, .{}); defer platform.deinit(); + std.log.info("Selected platform: {f}", .{platform}); + var model: Model = try .init(); defer model.deinit(); diff --git a/zml2/platform.zig b/zml2/platform.zig index 134ff487..d67814e8 100644 --- a/zml2/platform.zig +++ b/zml2/platform.zig @@ -17,6 +17,29 @@ pub const CompilationOptions = struct { device_memory_size: u64 = 0, }; +fn StaticPlatformMap(comptime E: type, comptime T: type) type { + const tag_count = @typeInfo(E).@"enum".fields.len; + var struct_field_names: [tag_count][]const u8 = undefined; + var struct_field_types: [tag_count]type = @splat(T); + const is_optional = @typeInfo(T) == .optional; + const default_value_ptr: ?*const anyopaque = if (is_optional) @ptrCast(&@as(T, null)) else null; + var struct_field_attrs: [tag_count]std.builtin.Type.StructField.Attributes = @splat(.{ .default_value_ptr = default_value_ptr }); + inline for (@typeInfo(E).@"enum".fields, 0..) |f, i| struct_field_names[i] = f.name; + return @Struct(.auto, null, &struct_field_names, &struct_field_types, &struct_field_attrs); +} + +var api_map: StaticPlatformMap(Target, ?*const pjrt.Api) = .{}; + +fn loadOrGetApi(target: Target, io: std.Io) !*const pjrt.Api { + return switch (target) { + inline else => |tag| @field(api_map, @tagName(tag)) orelse b: { + const api = try runtimes.load(tag, io); + @field(api_map, @tagName(tag)) = api; + break :b api; + }, + }; +} + pub const Platform = struct { target: Target, pjrt_api: *const pjrt.Api, @@ -31,7 +54,7 @@ pub const Platform = struct { pub const MAX_NUM_DEVICES: u8 = if (runtimes.isEnabled(.tpu)) 32 else 8; pub fn init(target: Target, io: std.Io, options: CreateOptions) !Platform { - const api = try runtimes.load(target, io); + const api = try loadOrGetApi(target, io); var named_values_buf: [16]pjrt.NamedValue = undefined; const pjrt_client = try pjrt.Client.init(api, options.toNamedValues(target, &named_values_buf)); @@ -47,6 +70,38 @@ pub const Platform = struct { }; } + pub fn auto(io: std.Io, options: CreateOptions) !Platform { + const ordered_targets = [_]Target{ .tpu, .neuron, .cuda, .rocm, .cpu }; + return for (ordered_targets) |target| { + break init(target, io, options) catch continue; + } else error.Unavailable; + } + + pub fn availablePlatforms(io: std.Io) StaticPlatformMap(Target, bool) { + var result: StaticPlatformMap(Target, bool) = undefined; + inline for (@typeInfo(Target).@"enum".fields) |field| { + if (@field(api_map, field.name) != null) { + @field(result, field.name) = true; + } else { + if (loadOrGetApi(@enumFromInt(field.value), io)) |_| { + @field(result, field.name) = true; + } else |_| { + @field(result, field.name) = false; + } + } + } + return result; + } + + pub fn printAvailablePlatforms(io: std.Io) void { + const available_platforms = availablePlatforms(io); + inline for (@typeInfo(Target).@"enum".fields) |field| { + if (@field(available_platforms, field.name)) { + std.log.info("{f}", .{@field(api_map, field.name).?}); + } + } + } + pub fn getDevices(self: Platform) []const *const pjrt.Device { const all_devices = self.pjrt_client.getAddressableDevices(self.pjrt_api); if (all_devices.len > MAX_NUM_DEVICES) { @@ -119,6 +174,17 @@ pub const Platform = struct { const device = platform.getDevices()[device_id]; return device.memoryStats(platform.pjrt_api) catch .zeroes; } + + pub fn format(self: @This(), writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("{s} {{ ", .{@tagName(self.target)}); + const devices = self.getDevices(); + for (0..devices.len) |i| { + const description = devices[i].getDescription(self.pjrt_api); + try writer.print("{s}(\"{s}\")", .{ description.toString(self.pjrt_api), description.getKind(self.pjrt_api) }); + if (i < devices.len - 1) try writer.writeAll(", "); + } + try writer.writeAll(" }"); + } }; pub const CreateOptions = struct { From 1c51e65f9094d76d1b1d610d16dcbbc5e65365e0 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 12 Dec 2025 14:23:59 +0100 Subject: [PATCH 134/881] Changed order of parameters for Buffer.from/fromBytes std.Io should be put at the beginning (similar to std.mem.Allocator) --- zml2/buffer.zig | 11 ++--- zml2/nn.zig | 38 ++++++++-------- zml2/ops.zig | 4 +- zml2/tensor.zig | 112 ++++++++++++++++++++++++------------------------ 4 files changed, 83 insertions(+), 82 deletions(-) diff --git a/zml2/buffer.zig b/zml2/buffer.zig index bf85d1da..f3bb8bc0 100644 --- a/zml2/buffer.zig +++ b/zml2/buffer.zig @@ -49,7 +49,7 @@ pub const Buffer = struct { } /// Copies the content of the given buffer from host memory to the accelerator memory. - pub fn from(platform: Platform, shape_: Shape, data_: []const u8, io: std.Io, opts: FromOptions) !Buffer { + pub fn from(io: std.Io, platform: Platform, shape_: Shape, data_: []const u8, opts: FromOptions) !Buffer { var res: Buffer = .{ ._api = platform.pjrt_api, ._target = platform.target, @@ -109,14 +109,15 @@ pub const Buffer = struct { /// Copies the given Zig slice to the accelerator memory and /// return a Buffer with the given dimensions. - pub fn fromBytes(platform: Platform, sh: Shape, data: []const u8, io: std.Io) !Buffer { - return from(platform, sh, data, io, .{}); + pub fn fromBytes(io: std.Io, platform: Platform, sh: Shape, data: []const u8) !Buffer { + return from(io, platform, sh, data, .{}); + } } /// Creates a Buffer with a single element. - pub fn scalar(platform: Platform, val: anytype, dtype_: DataType, io: std.Io) !Buffer { + pub fn scalar(io: std.Io, platform: Platform, val: anytype, dtype_: DataType) !Buffer { const x = dtype_.constant(val); - return fromBytes(platform, Shape.init(.{}, dtype_), x.asBytes(), io); + return fromBytes(io, platform, Shape.init(.{}, dtype_), x.asBytes()); } pub fn await(self: Buffer, io: std.Io) !Buffer { diff --git a/zml2/nn.zig b/zml2/nn.zig index 3317179c..32dd36a7 100644 --- a/zml2/nn.zig +++ b/zml2/nn.zig @@ -125,7 +125,7 @@ test normalizeL2 { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, normalizeL2, .{ input, 1e-12 }, platform); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.9686, -1.0058, -1.7808, 0.6698 }), std.testing.io); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.9686, -1.0058, -1.7808, 0.6698 })); defer input_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, normalizeL2, .{input_buffer}); @@ -518,7 +518,7 @@ test rope { var exe_sequential = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .sequential } }, platform); defer exe_sequential.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ 1.0, 0.1, -1.0, -0.5 } ** 5), std.testing.io); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ 1.0, 0.1, -1.0, -0.5 } ** 5)); defer x_buffer.deinit(); var res1 = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_interleaved, Local._fwd, .{x_buffer}); @@ -594,7 +594,7 @@ test nearest { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_3d_basic, .{ .scale_factor = &.{3}, .mode = .nearest } }, platform); defer exe.deinit(); - var input_3d_basic_buffer: zml.Buffer = try .fromBytes(platform, input_3d_basic.shape(), std.mem.sliceAsBytes(&[1][1][2]i32{.{.{ 1, 2 }}}), std.testing.io); + var input_3d_basic_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_3d_basic.shape(), std.mem.sliceAsBytes(&[1][1][2]i32{.{.{ 1, 2 }}})); defer input_3d_basic_buffer.deinit(); const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_3d_basic_buffer}); @@ -611,10 +611,10 @@ test nearest { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_3d_advanced, .{ .scale_factor = &.{2}, .mode = .nearest } }, platform); defer exe.deinit(); - var input_3d_advanced_buffer: zml.Buffer = try .fromBytes(platform, input_3d_advanced.shape(), std.mem.sliceAsBytes(&[2][3][4]i32{ + var input_3d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_3d_advanced.shape(), std.mem.sliceAsBytes(&[2][3][4]i32{ .{ .{ 1, 2, 3, 4 }, .{ 5, 6, 7, 8 }, .{ 9, 10, 11, 12 } }, .{ .{ 13, 14, 15, 16 }, .{ 17, 18, 19, 20 }, .{ 21, 22, 23, 24 } }, - }), std.testing.io); + })); defer input_3d_advanced_buffer.deinit(); const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_3d_advanced_buffer}); @@ -642,7 +642,7 @@ test nearest { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_4d_basic, .{ .scale_factor = &.{ 3, 3 }, .mode = .nearest } }, platform); defer exe.deinit(); - var input_4d_basic_buffer: zml.Buffer = try .fromBytes(platform, input_4d_basic.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 }), std.testing.io); + var input_4d_basic_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_basic.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); defer input_4d_basic_buffer.deinit(); const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_4d_basic_buffer}); @@ -665,13 +665,13 @@ test nearest { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_4d_advanced, .{ .scale_factor = &.{ 2, 2 }, .mode = .nearest } }, platform); defer exe.deinit(); - var input_4d_advanced_buffer: zml.Buffer = try .fromBytes(platform, input_4d_advanced.shape(), std.mem.sliceAsBytes(&[2][2][2][2]i32{ .{ + var input_4d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_advanced.shape(), std.mem.sliceAsBytes(&[2][2][2][2]i32{ .{ .{ .{ 1, 2 }, .{ 3, 4 } }, .{ .{ 5, 6 }, .{ 7, 8 } }, }, .{ .{ .{ 9, 10 }, .{ 11, 12 } }, .{ .{ 13, 14 }, .{ 15, 16 } }, - } }), std.testing.io); + } })); defer input_4d_advanced_buffer.deinit(); const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_4d_advanced_buffer}); @@ -716,7 +716,7 @@ test nearest { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_5d, .{ .scale_factor = &.{2}, .mode = .nearest } }, platform); defer exe.deinit(); - var input_5d_buffer: zml.Buffer = try .fromBytes(platform, input_5d.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 }), std.testing.io); + var input_5d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_5d.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); defer input_5d_buffer.deinit(); const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_5d_buffer}); @@ -1072,7 +1072,7 @@ test sampleTokens { }) |logits_expected| { const logits, const expected: i32 = logits_expected; - const activations_buffer: zml.Buffer = try .fromBytes(platform, activations.shape(), std.mem.sliceAsBytes(&logits), std.testing.io); + const activations_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, activations.shape(), std.mem.sliceAsBytes(&logits)); defer activations_buffer.deinit(); var sampled, rng_buffer = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, sampleTokens, .{ activations_buffer, rng_buffer }); @@ -1109,16 +1109,16 @@ pub const DynamicSamplingStrategy = struct { } pub fn makeBuffers( - platform: zml.Platform, io: std.Io, + platform: zml.Platform, dtype: zml.DataType, opts: Opts, ) !zml.Bufferized(DynamicSamplingStrategy) { return .{ - .top_k = try zml.Buffer.scalar(platform, opts.top_k, .i32, io), - .temperature = try zml.Buffer.scalar(platform, opts.temperature, dtype, io), - .top_p = try zml.Buffer.scalar(platform, opts.top_p, dtype, io), - .min_p = try zml.Buffer.scalar(platform, opts.min_p, dtype, io), + .top_k = try zml.Buffer.scalar(io, platform, opts.top_k, .i32), + .temperature = try zml.Buffer.scalar(io, platform, opts.temperature, dtype), + .top_p = try zml.Buffer.scalar(io, platform, opts.top_p, dtype), + .min_p = try zml.Buffer.scalar(io, platform, opts.min_p, dtype), }; } @@ -1200,7 +1200,7 @@ test sampleTokensDynamic { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, fixupLogits, .{ logits, dynamic_sampling_strategy }, platform); defer exe.deinit(); - var logits_buffer: zml.Buffer = try .fromBytes(platform, logits.shape(), std.mem.sliceAsBytes(&logits_data), std.testing.io); + var logits_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits.shape(), std.mem.sliceAsBytes(&logits_data)); defer logits_buffer.deinit(); const Args = struct { DynamicSamplingStrategy.Opts, [4]f32 }; @@ -1219,7 +1219,7 @@ test sampleTokensDynamic { .{ .{ .top_k = 4, .top_p = 0.901, .min_p = 0.6 }, [_]f32{ @log(4.0), @log(3.0), ___, ___ } }, }) |args_expected| { const args, const expected = args_expected; - var dynamic_sampling_strategy_buffers = try DynamicSamplingStrategy.makeBuffers(platform, std.testing.io, .f32, args); + var dynamic_sampling_strategy_buffers = try DynamicSamplingStrategy.makeBuffers(std.testing.io, platform, .f32, args); defer DynamicSamplingStrategy.deinitBuffers(&dynamic_sampling_strategy_buffers); var new_logits, var indices = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, fixupLogits, .{ logits_buffer, dynamic_sampling_strategy_buffers }); defer new_logits.deinit(); @@ -1240,9 +1240,9 @@ test sampleTokensDynamic { const boost = bf16.inf; const nerf = bf16.minus_inf; - var logits_bf16_buffer: zml.Buffer = try .fromBytes(platform, logits_bf16.shape(), std.mem.sliceAsBytes(&[4]bf16{ boost, boost, bf16.fromF32(2), nerf }), std.testing.io); + var logits_bf16_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits_bf16.shape(), std.mem.sliceAsBytes(&[4]bf16{ boost, boost, bf16.fromF32(2), nerf })); defer logits_bf16_buffer.deinit(); - var dynamic_sampling_strategy_bf16_buffers = try DynamicSamplingStrategy.makeBuffers(platform, std.testing.io, .bf16, .{ .top_k = 4, .top_p = 0.9, .min_p = 0.1 }); + var dynamic_sampling_strategy_bf16_buffers = try DynamicSamplingStrategy.makeBuffers(std.testing.io, platform, .bf16, .{ .top_k = 4, .top_p = 0.9, .min_p = 0.1 }); defer DynamicSamplingStrategy.deinitBuffers(&dynamic_sampling_strategy_bf16_buffers); var new_logits, var indices = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_bf16, fixupLogits, .{ logits_bf16_buffer, dynamic_sampling_strategy_bf16_buffers }); diff --git a/zml2/ops.zig b/zml2/ops.zig index fadc9065..637e0947 100644 --- a/zml2/ops.zig +++ b/zml2/ops.zig @@ -369,9 +369,9 @@ test "triton" { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, TritonMod.forward, .{ a, b }, platform); defer exe.deinit(); - var a_buffer: zml.Buffer = try .fromBytes(platform, a.shape(), std.mem.sliceAsBytes(&[1]f32{1}), std.testing.io); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[1]f32{1})); defer a_buffer.deinit(); - var b_buffer: zml.Buffer = try .fromBytes(platform, b.shape(), std.mem.sliceAsBytes(&[1]f32{3}), std.testing.io); + var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), std.mem.sliceAsBytes(&[1]f32{3})); defer b_buffer.deinit(); const results = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, TritonMod.forward, .{ a_buffer, b_buffer }); diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 3f3f6230..6ffcdc46 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -292,7 +292,7 @@ pub const Tensor = struct { const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.fmod, .{ input, d }, platform); defer exe.deinit(); - const input_buffer = try zml.Buffer.fromBytes(platform, input.shape(), std.mem.sliceAsBytes(&i), std.testing.io); + const input_buffer = try zml.Buffer.fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&i)); defer input_buffer.deinit(); const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.fmod, .{input_buffer}); @@ -453,7 +453,7 @@ pub const Tensor = struct { pub fn initBuffer(platform: Platform, seed: u128, io: std.Io) !Bufferized(Rng) { return .{ - ._state = try .fromBytes(platform, Shape.init(.{2}, .u64), std.mem.asBytes(&seed), io), + ._state = try .fromBytes(io, platform, Shape.init(.{2}, .u64), std.mem.asBytes(&seed)), }; } @@ -661,7 +661,7 @@ pub const Tensor = struct { const rng_buffer = try Rng.initBuffer(platform, 1234, std.testing.io); defer rng_buffer._state.deinit(); - const tgt_dist_buffer: Buffer = try .fromBytes(platform, tgt_dist.shape(), std.mem.sliceAsBytes(&tgt_dist_data), std.testing.io); + const tgt_dist_buffer: Buffer = try .fromBytes(std.testing.io, platform, tgt_dist.shape(), std.mem.sliceAsBytes(&tgt_dist_data)); defer tgt_dist_buffer.deinit(); const rand, const stats = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Stats.gumbelStats, .{ rng_buffer, tgt_dist_buffer }); @@ -992,7 +992,7 @@ pub const Tensor = struct { const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.convert, .{ x_d, .f4e2m1 }, platform); defer exe.deinit(); - const x_d_buffer: zml.Buffer = try .fromBytes(platform, x_d.shape(), std.mem.sliceAsBytes(&x), std.testing.io); + const x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), std.mem.sliceAsBytes(&x)); defer x_d_buffer.deinit(); const x_f4_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); @@ -1013,7 +1013,7 @@ pub const Tensor = struct { const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.convert, .{ x_d, .f8e3m4 }, platform); defer exe.deinit(); - const x_d_buffer: zml.Buffer = try .fromBytes(platform, x_d.shape(), std.mem.sliceAsBytes(&x), std.testing.io); + const x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), std.mem.sliceAsBytes(&x)); defer x_d_buffer.deinit(); const x_f8e3_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); @@ -1212,7 +1212,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.leakyReLU, .{ input, 0.1 }, platform); defer exe.deinit(); - const input_buffer: zml.Buffer = try .fromBytes(platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6884, 1.6795 }), std.testing.io); + const input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6884, 1.6795 })); defer input_buffer.deinit(); const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.leakyReLU, .{input_buffer}); @@ -1368,7 +1368,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cumsum, .{x}, platform); defer exe.deinit(); - const x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } }), std.testing.io); + const x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } })); defer x_buffer.deinit(); const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._cumsum, .{x_buffer}); @@ -1568,7 +1568,7 @@ pub const Tensor = struct { const x: Tensor = .init(.{ 2, 5 }, .f32); - const x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }), std.testing.io); + const x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); defer x_buffer.deinit(); // Wrap slice1d to hide the anytype in the signature. @@ -1717,7 +1717,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.repeat1d, .{ input, ax, reps }, platform); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(platform, input.shape(), std.mem.sliceAsBytes(&input_data), std.testing.io); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&input_data)); defer input_buffer.deinit(); const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local.repeat1d, .{input_buffer}); @@ -1772,7 +1772,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.stutter1d, .{ input, ax, reps }, platform); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(platform, input.shape(), std.mem.sliceAsBytes(&input_data), std.testing.io); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&input_data)); defer input_buffer.deinit(); const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local.stutter1d, .{input_buffer}); @@ -2404,7 +2404,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._gatherSlices, .{ operand, Shape.init(.{ .b = 2, .c = 3 }, .u16), indices, .{} }, platform); defer exe.deinit(); - const indices_buffer: zml.Buffer = try .fromBytes(platform, indices.shape(), std.mem.sliceAsBytes(&[2][2]i32{ .{ 2, 1 }, .{ 0, 3 } }), std.testing.io); + const indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, indices.shape(), std.mem.sliceAsBytes(&[2][2]i32{ .{ 2, 1 }, .{ 0, 3 } })); defer indices_buffer.deinit(); const operand_buffer: zml.Buffer = b: { const temp_func = struct { @@ -2622,11 +2622,11 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform); defer exe.deinit(); - var a_buffer: zml.Buffer = try .fromBytes(platform, a.shape(), std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 }), std.testing.io); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); defer a_buffer.deinit(); - var scatter_indices_buffer: zml.Buffer = try .fromBytes(platform, scatter_indices.shape(), std.mem.sliceAsBytes(&[2]i32{ 0, 2 }), std.testing.io); + var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), std.mem.sliceAsBytes(&[2]i32{ 0, 2 })); defer scatter_indices_buffer.deinit(); - var updates_buffer: zml.Buffer = try .fromBytes(platform, updates.shape(), std.mem.sliceAsBytes(&[2][3]i32{ .{ 10, 20, 30 }, .{ 70, 80, 90 } }), std.testing.io); + var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), std.mem.sliceAsBytes(&[2][3]i32{ .{ 10, 20, 30 }, .{ 70, 80, 90 } })); defer updates_buffer.deinit(); var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatter, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); @@ -2646,11 +2646,11 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform); defer exe.deinit(); - var a_buffer: zml.Buffer = try .fromBytes(platform, a.shape(), std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 }), std.testing.io); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); defer a_buffer.deinit(); - var scatter_indices_buffer: zml.Buffer = try .fromBytes(platform, scatter_indices.shape(), std.mem.sliceAsBytes(&[2]i32{ 2, 7 }), std.testing.io); + var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), std.mem.sliceAsBytes(&[2]i32{ 2, 7 })); defer scatter_indices_buffer.deinit(); - var updates_buffer: zml.Buffer = try .fromBytes(platform, updates.shape(), std.mem.sliceAsBytes(&[2]i32{ 20, 70 }), std.testing.io); + var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), std.mem.sliceAsBytes(&[2]i32{ 20, 70 })); defer updates_buffer.deinit(); var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatter, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); @@ -2669,9 +2669,9 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatterCB, .{ operand, start_indices, values }, platform); defer exe.deinit(); - var operand_buffer: zml.Buffer = try .fromBytes(platform, operand.shape(), std.mem.sliceAsBytes(&@as([2 * 3 * 4 * 2]u16, @splat(0))), std.testing.io); + var operand_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, operand.shape(), std.mem.sliceAsBytes(&@as([2 * 3 * 4 * 2]u16, @splat(0)))); defer operand_buffer.deinit(); - var start_indices_buffer: zml.Buffer = try .fromBytes(platform, start_indices.shape(), std.mem.sliceAsBytes(&[2][2][3][2]i32{ + var start_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, start_indices.shape(), std.mem.sliceAsBytes(&[2][2][3][2]i32{ .{ .{ .{ 0, 0 }, .{ 1, 0 }, .{ 2, 1 } }, .{ .{ 0, 1 }, .{ 1, 1 }, .{ 0, 9 } }, @@ -2680,9 +2680,9 @@ pub const Tensor = struct { .{ .{ 0, 0 }, .{ 2, 1 }, .{ 2, 2 } }, .{ .{ 1, 2 }, .{ 0, 1 }, .{ 1, 0 } }, }, - }), std.testing.io); + })); defer start_indices_buffer.deinit(); - var values_buffer: zml.Buffer = try .fromBytes(platform, values.shape(), std.mem.sliceAsBytes(&@as([2 * 2 * 3 * 2 * 2]u16, @splat(1))), std.testing.io); + var values_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, values.shape(), std.mem.sliceAsBytes(&@as([2 * 2 * 3 * 2 * 2]u16, @splat(1)))); defer values_buffer.deinit(); var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatterCB, .{ operand_buffer, start_indices_buffer, values_buffer }); @@ -2788,7 +2788,7 @@ pub const Tensor = struct { defer exe.deinit(); { - var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, 4.1, 7.9, 0, 7.9 }}), std.testing.io); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, 4.1, 7.9, 0, 7.9 }})); defer x_buffer.deinit(); const res = try zml.testing.autoCall(allocator, std.testing.io, &exe, ArgMaxTest._fwd, .{x_buffer}); @@ -2802,7 +2802,7 @@ pub const Tensor = struct { } { - var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, std.math.nan(f32), 7.9, 0, 7.9 }}), std.testing.io); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, std.math.nan(f32), 7.9, 0, 7.9 }})); defer x_buffer.deinit(); const res = try zml.testing.autoCall(allocator, std.testing.io, &exe, ArgMaxTest._fwd, .{x_buffer}); @@ -2856,7 +2856,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{} }, platform); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 }), std.testing.io); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 })); defer x_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); @@ -2875,13 +2875,13 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{ .descending = true } }, platform); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[_]f16{ + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f16{ -0.2505, 1.2520, -0.7041, 0.1066, 1.2773, -1.7246, 0.8389, 1.1094, 0.0601, 1.0684, 0.9619, 1.3916, 1.2246, -0.1406, 0.3674, -1.2480, -1.7051, -0.0934, 0.3435, 0.4373, 1.3809, 0.5444, -0.6079, 1.2031, -0.6880, 1.2979, -0.1869, 0.2991, 0.0156, 0.1847, 0.6626, -0.3040, -0.8726, -1.4805, -1.6943, 1.1055, -2.0078, -0.5288, 0.8813, 0.8008, 2.0527, 1.1230, 0.5430, 0.2494, -0.9434, 0.7876, 0.1818, 0.9258, -2.4902, 1.5918, - }), std.testing.io); + })); defer x_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); @@ -2906,7 +2906,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 3, .{} }, platform); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 89, 31, 22, 42, 64, 39, 0, 30, 64, 71, 46, 31, @@ -2915,7 +2915,7 @@ pub const Tensor = struct { 93, 24, 45, 72, 64, 86, 62, 88, 57, 21, 19, 12, - }), std.testing.io); + })); defer x_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); @@ -3279,9 +3279,9 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.dynamicSlice1d, .{ x, ax, .{ .len = 2, .start = z } }, platform); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&x_data), std.testing.io); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&x_data)); defer x_buffer.deinit(); - var z_buffer: zml.Buffer = try .fromBytes(platform, z.shape(), std.mem.asBytes(&z_value), std.testing.io); + var z_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, z.shape(), std.mem.asBytes(&z_value)); defer z_buffer.deinit(); const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.dynamicSlice1d, .{ x_buffer, .{ .start = z_buffer } }); @@ -3403,11 +3403,11 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = ids }, y }, platform); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }), std.testing.io); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(platform, y.shape(), std.mem.sliceAsBytes(&[2]f32{ -1, -1 }), std.testing.io); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); defer y_buffer.deinit(); - var ids_buffer: zml.Buffer = try .fromBytes(platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{4}), std.testing.io); + var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{4})); defer ids_buffer.deinit(); const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ .a = ids_buffer }, y_buffer }); @@ -3429,11 +3429,11 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }), std.testing.io); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(platform, y.shape(), std.mem.sliceAsBytes(&[2]f32{ -1, -1 }), std.testing.io); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); defer y_buffer.deinit(); - var ids_buffer: zml.Buffer = try .fromBytes(platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3}), std.testing.io); + var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer ids_buffer.deinit(); const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, ids_buffer, y_buffer }); @@ -3458,11 +3458,11 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }), std.testing.io); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(platform, y.shape(), std.mem.sliceAsBytes(&[2][1]f32{ .{-1}, .{-1} }), std.testing.io); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[2][1]f32{ .{-1}, .{-1} })); defer y_buffer.deinit(); - var ids_buffer: zml.Buffer = try .fromBytes(platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3}), std.testing.io); + var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer ids_buffer.deinit(); const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, ids_buffer, y_buffer }); @@ -3488,13 +3488,13 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = idx_a, .b = idx_b }, y }, platform); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }), std.testing.io); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(platform, y.shape(), std.mem.sliceAsBytes(&[1]f32{-1}), std.testing.io); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[1]f32{-1})); defer y_buffer.deinit(); - var idx_a_buffer: zml.Buffer = try .fromBytes(platform, idx_a.shape(), std.mem.sliceAsBytes(&[1]i32{1}), std.testing.io); + var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), std.mem.sliceAsBytes(&[1]i32{1})); defer idx_a_buffer.deinit(); - var idx_b_buffer: zml.Buffer = try .fromBytes(platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3}), std.testing.io); + var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer idx_b_buffer.deinit(); const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ .a = idx_a_buffer, .b = idx_b_buffer }, y_buffer }); @@ -3520,13 +3520,13 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ idx_a, idx_b }, y }, platform); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }), std.testing.io); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(platform, y.shape(), std.mem.sliceAsBytes(&[1][1]f32{.{-1}}), std.testing.io); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[1][1]f32{.{-1}})); defer y_buffer.deinit(); - var idx_a_buffer: zml.Buffer = try .fromBytes(platform, idx_a.shape(), std.mem.sliceAsBytes(&[1]i32{1}), std.testing.io); + var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), std.mem.sliceAsBytes(&[1]i32{1})); defer idx_a_buffer.deinit(); - var idx_b_buffer: zml.Buffer = try .fromBytes(platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3}), std.testing.io); + var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer idx_b_buffer.deinit(); const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ idx_a_buffer, idx_b_buffer }, y_buffer }); @@ -3601,7 +3601,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._toDiag, .{x}, platform); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[2][2]u8{ .{ 1, 2 }, .{ 3, 4 } }), std.testing.io); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][2]u8{ .{ 1, 2 }, .{ 3, 4 } })); defer x_buffer.deinit(); const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._toDiag, .{x_buffer}); @@ -3651,11 +3651,11 @@ pub const Tensor = struct { const x: Tensor = .init(.{ 3, 3 }, .u8); - var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[3][3]u8{ + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[3][3]u8{ .{ 1, 1, 1 }, .{ 1, 1, 1 }, .{ 1, 1, 1 }, - }), std.testing.io); + })); defer x_buffer.deinit(); { @@ -3829,9 +3829,9 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cartesianProduct2, .{ x, y }, platform); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 }), std.testing.io); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 }), std.testing.io); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); defer y_buffer.deinit(); var xs, var ys = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._cartesianProduct2, .{ x_buffer, y_buffer }); @@ -3891,9 +3891,9 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, y }, platform); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 }), std.testing.io); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 }), std.testing.io); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); defer y_buffer.deinit(); var z = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._fwd, .{ x_buffer, y_buffer }); @@ -4031,7 +4031,7 @@ test "Tensor.maxPool1d" { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&data), std.testing.io); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&data)); defer x_buffer.deinit(); const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, MaxPool._fwd, .{x_buffer}); @@ -4070,7 +4070,7 @@ test "Tensor.maxPool2d" { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(platform, x.shape(), std.mem.sliceAsBytes(&data), std.testing.io); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&data)); defer x_buffer.deinit(); const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, MaxPool._fwd, .{x_buffer}); From 11a9ef2d58ac35f93a4d6d21359512d1ee5e1258 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 12 Dec 2025 14:27:16 +0100 Subject: [PATCH 135/881] Introduced Buffer.fromSlice --- zml2/buffer.zig | 10 +++++++++- zml2/main.zig | 6 +++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/zml2/buffer.zig b/zml2/buffer.zig index f3bb8bc0..b47d7942 100644 --- a/zml2/buffer.zig +++ b/zml2/buffer.zig @@ -2,6 +2,7 @@ const std = @import("std"); const stdx = @import("stdx"); +const ConstSlice = @import("slice.zig").ConstSlice; const DataType = @import("dtype.zig").DataType; const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; @@ -107,11 +108,18 @@ pub const Buffer = struct { return res; } - /// Copies the given Zig slice to the accelerator memory and + /// Copies the given Zig bytes to the accelerator memory and /// return a Buffer with the given dimensions. pub fn fromBytes(io: std.Io, platform: Platform, sh: Shape, data: []const u8) !Buffer { return from(io, platform, sh, data, .{}); } + + /// Copies the given zml.Slice or zml.ConstSlice to the accelerator memory and + /// return a Buffer. + pub fn fromSlice(io: std.Io, platform: Platform, slice: anytype) !Buffer { + if (@TypeOf(slice) != Slice and @TypeOf(slice) != ConstSlice) @compileError("Buffer.fromSlice expects a Slice or ConstSlice, got " ++ @typeName(@TypeOf(slice))); + const real_slice: ConstSlice = if (@TypeOf(slice) == Slice) slice.constSlice() else slice; + return from(io, platform, real_slice.shape, std.mem.sliceAsBytes(real_slice.data), .{}); } /// Creates a Buffer with a single element. diff --git a/zml2/main.zig b/zml2/main.zig index 5e66eeaa..d4254655 100644 --- a/zml2/main.zig +++ b/zml2/main.zig @@ -25,9 +25,9 @@ pub const Model = struct { defer bias_slice.free(allocator); @memset(bias_slice.items(f32), 2); - const weight_buffer: zml.Buffer = try .fromBytes(platform, weight_slice.shape, weight_slice.data, io); + const weight_buffer: zml.Buffer = try .fromSlice(io, platform, weight_slice); errdefer weight_buffer.deinit(); - const bias_buffer: zml.Buffer = try .fromBytes(platform, bias_slice.shape, bias_slice.data, io); + const bias_buffer: zml.Buffer = try .fromSlice(io, platform, bias_slice); errdefer bias_buffer.deinit(); return .{ .weight = weight_buffer, .bias = bias_buffer }; @@ -79,7 +79,7 @@ pub fn main() !void { var model_buffers = try model.loadBuffers(allocator, io, platform); defer Model.unloadBuffers(&model_buffers); - const input_buffer: zml.Buffer = try .fromBytes(platform, slice.shape, slice.data, io); + const input_buffer: zml.Buffer = try .fromSlice(io, platform, slice); defer input_buffer.deinit(); var args = try exe.args(allocator); From 4294a19dd8e283b4bb55f5fcc17a17e794946150 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 12 Dec 2025 14:54:02 +0100 Subject: [PATCH 136/881] Expose compile and compileModel through Platform --- zml2/main.zig | 2 +- zml2/module.zig | 4 ---- zml2/platform.zig | 11 ++++++++++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/zml2/main.zig b/zml2/main.zig index d4254655..3ace2ce3 100644 --- a/zml2/main.zig +++ b/zml2/main.zig @@ -69,7 +69,7 @@ pub fn main() !void { defer model.deinit(); const input: zml.Tensor = .init(.{ .n = 4096 }, .f32); - var exe = try zml.module.compileModel(allocator, io, Model.forward, model, .{input}, platform); + var exe = try platform.compileModel(allocator, io, Model.forward, model, .{input}); defer exe.deinit(); const slice: zml.Slice = try .alloc(allocator, input.shape()); diff --git a/zml2/module.zig b/zml2/module.zig index 2fdf03bd..dddfb7fc 100644 --- a/zml2/module.zig +++ b/zml2/module.zig @@ -165,10 +165,6 @@ pub const CompilationContext = struct { //} }; -pub fn compileModel(allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, model: stdx.meta.Head(stdx.meta.FnArgs(func)), args: stdx.meta.Tail(stdx.meta.FnArgs(func)), platform: Platform) !Exe { - return compile(allocator, io, func, .{model} ++ args, platform); -} - pub fn compile(allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, args: stdx.meta.FnArgs(func), platform: Platform) !Exe { var compilation_context: CompilationContext = .init(allocator, platform); defer compilation_context.deinit(); diff --git a/zml2/platform.zig b/zml2/platform.zig index d67814e8..df212333 100644 --- a/zml2/platform.zig +++ b/zml2/platform.zig @@ -4,7 +4,9 @@ const runtimes = @import("runtimes"); pub const Target = runtimes.Platform; const stdx = @import("stdx"); +const Exe = @import("exe.zig").Exe; const pjrt = @import("pjrtx.zig"); +const zml = @import("zml.zig"); const log = std.log.scoped(.zml); @@ -158,7 +160,6 @@ pub const Platform = struct { } test memoryForDevice { - const zml = @import("zml.zig"); const platform = zml.testing.env(); const memory_fields = @typeInfo(pjrt.Memory.Kind).@"enum".fields; inline for (memory_fields) |field| { @@ -175,6 +176,14 @@ pub const Platform = struct { return device.memoryStats(platform.pjrt_api) catch .zeroes; } + pub fn compileModel(self: Platform, allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, model: stdx.meta.Head(stdx.meta.FnArgs(func)), args: stdx.meta.Tail(stdx.meta.FnArgs(func))) !Exe { + return self.compile(allocator, io, func, .{model} ++ args); + } + + pub fn compile(self: Platform, allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, args: stdx.meta.FnArgs(func)) !Exe { + return zml.module.compile(allocator, io, func, args, self); + } + pub fn format(self: @This(), writer: *std.Io.Writer) std.Io.Writer.Error!void { try writer.print("{s} {{ ", .{@tagName(self.target)}); const devices = self.getDevices(); From 03b0b5726d18cdd30d62d7153d662be2f1b9ebac Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 12 Dec 2025 15:25:01 +0100 Subject: [PATCH 137/881] Ported benchmark examples to v2 --- examples/benchmark/BUILD.bazel | 3 +- examples/benchmark/main.zig | 84 +++++++++++++++++----------------- stdx/flags.zig | 2 +- 3 files changed, 44 insertions(+), 45 deletions(-) diff --git a/examples/benchmark/BUILD.bazel b/examples/benchmark/BUILD.bazel index ff55ed6b..ab10bd77 100644 --- a/examples/benchmark/BUILD.bazel +++ b/examples/benchmark/BUILD.bazel @@ -5,9 +5,8 @@ zig_binary( name = "benchmark", main = "main.zig", deps = [ - "//async", "//stdx", - "//zml", + "//zml2:zml", ], ) diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index fb6bccb8..f0321e88 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -1,24 +1,18 @@ const std = @import("std"); const zml = @import("zml"); const stdx = @import("stdx"); -const async = @import("async"); const flags = stdx.flags; // set log level to debug to print the generated IR pub const std_options: std.Options = .{ .log_level = .warn, - .logFn = async.logFn(std.log.defaultLog), }; pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { - return a.withSharding(.{.k}).dot(b.withSharding(.{.k}), .{.k}).withSharding(.{.m}); + return a.dot(b, .k); } pub fn main() !void { - try async.AsyncThread.main(std.heap.c_allocator, asyncMain); -} - -pub fn asyncMain() !void { const CliArgs = struct { pub const help = \\ benchmark --size=4096 --dtype=f16 @@ -30,34 +24,35 @@ pub fn asyncMain() !void { // Short lived allocations var gpa = std.heap.GeneralPurposeAllocator(.{}){}; defer _ = gpa.deinit(); + const allocator = gpa.allocator(); - var context = try zml.Context.init(); - defer context.deinit(); + var threaded: std.Io.Threaded = .init(allocator); + defer threaded.deinit(); + + const io = threaded.io(); + + zml.init(); + defer zml.deinit(); // Auto-select platform - const platform = context.autoPlatform(.{}).withCompilationOptions(.{ - .sharding_enabled = true, - }); - context.printAvailablePlatforms(platform); + const platform: zml.Platform = try .auto(io, .{}); var args = std.process.args(); const cli_args = flags.parse(&args, CliArgs); - const a_shape = zml.Shape.init(.{ cli_args.size, cli_args.size }, cli_args.dtype).withTags(.{ .m, .k }).withSharding(.{.k}); - const b_shape = a_shape.withTags(.{ .k, .n }).withSharding(.{.k}); + const a = zml.Tensor.init(.{ cli_args.size, cli_args.size }, cli_args.dtype).withTags(.{ .m, .k }); + const b = zml.Tensor.init(.{ cli_args.size, cli_args.size }, cli_args.dtype).withTags(.{ .k, .n }); var timer = try std.time.Timer.start(); std.debug.print("\nCompiling model to MLIR....\n", .{}); std.debug.print("-" ** 160 ++ "\n", .{}); - // Start compiling. - // The shape of the input tensor, we have to pass in manually. - timer.reset(); - var compilation = try async.async(zml.compileFn, .{ allocator, benchmark, .{ a_shape, b_shape }, platform }); - // Wait for compilation to finish - const executable = try compilation.await(); - defer executable.deinit(); + timer.reset(); + + var exe = try platform.compile(allocator, io, benchmark, .{ a, b }); + defer exe.deinit(); + const compilation_elapsed = timer.lap() / std.time.ns_per_ms; std.debug.print("-" ** 160 ++ "\n\n", .{}); std.debug.print("✅ Compiled Benchmark model in {d} milliseconds! \n", .{compilation_elapsed}); @@ -65,22 +60,32 @@ pub fn asyncMain() !void { var rng = std.Random.DefaultPrng.init(0); const random = rng.random(); - var a_buffer = try createRandomBuffer(allocator, platform, a_shape, random); + var a_buffer = try createRandomBuffer(allocator, io, platform, a.shape(), random); defer a_buffer.deinit(); - var b_buffer = try createRandomBuffer(allocator, platform, b_shape, random); + var b_buffer = try createRandomBuffer(allocator, io, platform, b.shape(), random); defer b_buffer.deinit(); + var exe_args = try exe.args(allocator); + defer exe_args.deinit(allocator); + + var exe_results = try exe.results(allocator); + defer exe_results.deinit(allocator); + + exe_args.set(.{ a_buffer, b_buffer }); + std.debug.print("\nRunning benchmark....\n", .{}); // Ignore first run { - var result: zml.Buffer = executable.call(.{ a_buffer, b_buffer }); + exe.call(exe_args, &exe_results, io); + var result = exe_results.get(zml.Buffer); defer result.deinit(); } // call our executable module timer.reset(); - var result: zml.Buffer = executable.call(.{ a_buffer, b_buffer }); + exe.call(exe_args, &exe_results, io); + var result = exe_results.get(zml.Buffer); defer result.deinit(); const elapsed_ns = timer.lap(); const elapsed_ms = @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_ms; @@ -93,9 +98,9 @@ pub fn asyncMain() !void { std.debug.print("Dot product size: {d}x{d} - Datatype: {s} - Elapsed: {d:.3}ms - {d:.3} GFLOP/s\n\n", .{ cli_args.size, cli_args.size, @tagName(cli_args.dtype), elapsed_ms, flops / 1_000_000_000 }); } -fn createRandomBuffer(allocator: std.mem.Allocator, platform: zml.Platform, shape: zml.Shape, random: std.Random) !zml.Buffer { - const data = try allocator.alloc(u8, shape.byteSize()); - defer allocator.free(data); +fn createRandomBuffer(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform, shape: zml.Shape, random: std.Random) !zml.Buffer { + const slice = try zml.Slice.alloc(allocator, shape); + defer slice.free(allocator); switch (shape.dtype()) { inline else => |v| { @@ -103,25 +108,20 @@ fn createRandomBuffer(allocator: std.mem.Allocator, platform: zml.Platform, shap switch (comptime v.class()) { .bool => unreachable, .integer => { - for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = random.int(ZigType); + for (slice.items(ZigType)) |*e| e.* = random.int(ZigType); }, .float => { - const value = random.float(f64); - for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = if (ZigType == f64) - value - else if (ZigType == f32) - @floatCast(value) - else if (ZigType == f16) - @floatCast(value) - else - @bitCast(random.int(std.meta.Int(.unsigned, @bitSizeOf(ZigType)))); + const value = random.float(f32); + for (slice.items(ZigType)) |*e| e.* = switch (ZigType) { + f64, f32 => value, + f16 => @floatCast(value), + inline else => |T| if (@hasDecl(T, "fromF32")) T.fromF32(value) else unreachable, + }; }, .complex => unreachable, } }, } - var host_buffer = zml.HostBuffer.fromBytes(shape, data); - errdefer host_buffer.deinit(allocator); - return zml.Buffer.from(platform, host_buffer, .{}); + return .fromSlice(io, platform, slice); } diff --git a/stdx/flags.zig b/stdx/flags.zig index 3266c45d..3c5213ee 100644 --- a/stdx/flags.zig +++ b/stdx/flags.zig @@ -194,7 +194,7 @@ fn parse_flags(args: *std.process.ArgIterator, comptime Flags: type) Flags { .default_value_ptr = @ptrCast(&@as(u32, 0)), }; } - break :blk @Struct(.auto, null, count_field_names, count_field_types, count_field_attrs){}; + break :blk @Struct(.auto, null, &count_field_names, &count_field_types, &count_field_attrs){}; }; // When parsing arguments, we must consider longer arguments first, such that `--foo-bar=92` is From 39911d6de7aae60eb402b8aa07771612dd8f38a8 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Mon, 15 Dec 2025 13:39:48 +0100 Subject: [PATCH 138/881] io: hf + zml: safentensors (#363) --- MODULE.bazel.lock | 4 +- zml2/BUILD.bazel | 1 + zml2/io/vfs.zig | 341 +++++++++++++- zml2/io/vfs/BUILD.bazel | 1 + zml2/io/vfs/file.zig | 2 + zml2/io/vfs/hf.zig | 981 ++++++++++++++++++++++++++++++++++++++++ zml2/io/vfs/http.zig | 77 ++-- zml2/io/vfs/index.zig | 1 + zml2/main_io.zig | 361 ++++++++++----- zml2/safetensors.zig | 864 +++++++++++++++++++++++++++++++++++ zml2/zml.zig | 1 + 11 files changed, 2480 insertions(+), 154 deletions(-) create mode 100644 zml2/io/vfs/hf.zig create mode 100644 zml2/safetensors.zig diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index aae72b5a..06aa29e2 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -645,7 +645,7 @@ }, "@@rules_apple+//apple:apple.bzl%provisioning_profile_repository_extension": { "general": { - "bzlTransitiveDigest": "Oa+WNwUQ6db3WoQOwZ8qU+aRE5kJufCF45z4ZCs6298=", + "bzlTransitiveDigest": "asHlWWwDeYTUAyZk9zlK+dm0Ezyb97q8UBEfsY8p83k=", "usagesDigest": "vsJl8Rw5NL+5Ag2wdUDoTeRF/5klkXO8545Iy7U1Q08=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -2531,7 +2531,7 @@ }, "@@rules_rust+//crate_universe/private:internal_extensions.bzl%cu_nr": { "general": { - "bzlTransitiveDigest": "XkpmtSAoMsVR5D0IWKWoGVKSPjnIiH3NEyMQqyFvqm8=", + "bzlTransitiveDigest": "ANGVT5r/yDVCawqIISn+1Gh62Yw8VCMEqxi/JAAJCzw=", "usagesDigest": "3vKI8uvqTpJCf+t8aU6UD5d5cUWinWhtMjKkRpCLR+A=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, diff --git a/zml2/BUILD.bazel b/zml2/BUILD.bazel index cabb6606..4abf9ad8 100644 --- a/zml2/BUILD.bazel +++ b/zml2/BUILD.bazel @@ -33,6 +33,7 @@ zig_library( "ops.zig", "pjrtx.zig", "platform.zig", + "safetensors.zig", "shape.zig", "slice.zig", "tensor.zig", diff --git a/zml2/io/vfs.zig b/zml2/io/vfs.zig index 9f92b3b5..f982e7f0 100644 --- a/zml2/io/vfs.zig +++ b/zml2/io/vfs.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const vfs = @import("vfs"); const log = std.log.scoped(.@"zml/io/vfs"); @@ -18,8 +19,9 @@ pub const VFS = struct { InvalidURI, }; - pub const HTTP = @import("vfs").HTTP; - pub const File = @import("vfs").File; + pub const File = vfs.File; + pub const HTTP = vfs.HTTP; + pub const HF = vfs.HF; const FsFile = struct { backend: std.Io, @@ -44,28 +46,62 @@ pub const VFS = struct { next_dir_handle: std.Io.Dir.Handle, vtable: std.Io.VTable, + base_io: std.Io, pub fn init(allocator: std.mem.Allocator, base_io: std.Io) VFS { - var vtable = base_io.vtable.*; - vtable.dirMake = dirMake; - vtable.dirMakePath = dirMakePath; - vtable.dirMakeOpenPath = dirMakeOpenPath; - vtable.dirStat = dirStat; - vtable.dirStatPath = dirStatPath; - vtable.dirAccess = dirAccess; - vtable.dirCreateFile = dirCreateFile; - vtable.dirOpenFile = dirOpenFile; - vtable.dirOpenDir = dirOpenDir; - vtable.dirClose = dirClose; - vtable.fileStat = fileStat; - vtable.fileClose = fileClose; - vtable.fileWriteStreaming = fileWriteStreaming; - vtable.fileWritePositional = fileWritePositional; - vtable.fileReadStreaming = fileReadStreaming; - vtable.fileReadPositional = fileReadPositional; - vtable.fileSeekBy = fileSeekBy; - vtable.fileSeekTo = fileSeekTo; - vtable.openSelfExe = openSelfExe; + const vtable: std.Io.VTable = .{ + .async = async, + .concurrent = concurrent, + .await = await, + .cancel = cancel, + .cancelRequested = cancelRequested, + .groupAsync = groupAsync, + .groupConcurrent = groupConcurrent, + .groupWait = groupWait, + .groupCancel = groupCancel, + .select = select, + .mutexLock = mutexLock, + .mutexLockUncancelable = mutexLockUncancelable, + .mutexUnlock = mutexUnlock, + .conditionWait = conditionWait, + .conditionWaitUncancelable = conditionWaitUncancelable, + .conditionWake = conditionWake, + .now = now, + .sleep = sleep, + .netListenIp = netListenIp, + .netAccept = netAccept, + .netBindIp = netBindIp, + .netConnectIp = netConnectIp, + .netListenUnix = netListenUnix, + .netConnectUnix = netConnectUnix, + .netSend = netSend, + .netReceive = netReceive, + .netRead = netRead, + .netWrite = netWrite, + .netClose = netClose, + .netInterfaceNameResolve = netInterfaceNameResolve, + .netInterfaceName = netInterfaceName, + .netLookup = netLookup, + .dirMake = dirMake, + .dirMakePath = dirMakePath, + .dirMakeOpenPath = dirMakeOpenPath, + .dirStat = dirStat, + .dirStatPath = dirStatPath, + .dirAccess = dirAccess, + .dirCreateFile = dirCreateFile, + .dirOpenFile = dirOpenFile, + .dirOpenDir = dirOpenDir, + .dirClose = dirClose, + .fileStat = fileStat, + .fileClose = fileClose, + .fileWriteStreaming = fileWriteStreaming, + .fileWritePositional = fileWritePositional, + .fileReadStreaming = fileReadStreaming, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .openSelfExe = openSelfExe, + }; return .{ .allocator = allocator, @@ -76,6 +112,7 @@ pub const VFS = struct { .next_file_handle = 0, .next_dir_handle = 1, // 0 reserved for ROOT_DIR_HANDLE .vtable = vtable, + .base_io = base_io, }; } @@ -579,4 +616,264 @@ pub const VFS = struct { _ = flags; return std.Io.File.OpenSelfExeError.NotSupported; } + + // Forwarding wrappers + + fn async( + userdata: ?*anyopaque, + result: []u8, + result_alignment: std.mem.Alignment, + context: []const u8, + context_alignment: std.mem.Alignment, + start: *const fn (context: *const anyopaque, result: *anyopaque) void, + ) ?*std.Io.AnyFuture { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + const f = self.base_io.vtable.async; + return f(self.base_io.userdata, result, result_alignment, context, context_alignment, start); + } + + fn concurrent( + userdata: ?*anyopaque, + result_len: usize, + result_alignment: std.mem.Alignment, + context: []const u8, + context_alignment: std.mem.Alignment, + start: *const fn (context: *const anyopaque, result: *anyopaque) void, + ) std.Io.ConcurrentError!*std.Io.AnyFuture { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + const f = self.base_io.vtable.concurrent; + return f(self.base_io.userdata, result_len, result_alignment, context, context_alignment, start); + } + + fn await( + userdata: ?*anyopaque, + any_future: *std.Io.AnyFuture, + result: []u8, + result_alignment: std.mem.Alignment, + ) void { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + const f = self.base_io.vtable.await; + f(self.base_io.userdata, any_future, result, result_alignment); + } + + fn cancel( + userdata: ?*anyopaque, + any_future: *std.Io.AnyFuture, + result: []u8, + result_alignment: std.mem.Alignment, + ) void { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + const f = self.base_io.vtable.cancel; + f(self.base_io.userdata, any_future, result, result_alignment); + } + + fn cancelRequested(userdata: ?*anyopaque) bool { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + const f = self.base_io.vtable.cancelRequested; + return f(self.base_io.userdata); + } + + fn groupAsync( + userdata: ?*anyopaque, + group: *std.Io.Group, + context: []const u8, + context_alignment: std.mem.Alignment, + start: *const fn (*std.Io.Group, context: *const anyopaque) void, + ) void { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + const f = self.base_io.vtable.groupAsync; + f(self.base_io.userdata, group, context, context_alignment, start); + } + + fn groupConcurrent( + userdata: ?*anyopaque, + group: *std.Io.Group, + context: []const u8, + context_alignment: std.mem.Alignment, + start: *const fn (*std.Io.Group, context: *const anyopaque) void, + ) std.Io.ConcurrentError!void { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.base_io.vtable.groupConcurrent(self.base_io.userdata, group, context, context_alignment, start); + } + + fn groupWait(userdata: ?*anyopaque, group: *std.Io.Group, token: *anyopaque) void { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + const f = self.base_io.vtable.groupWait; + f(self.base_io.userdata, group, token); + } + + fn groupCancel(userdata: ?*anyopaque, group: *std.Io.Group, token: *anyopaque) void { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + const f = self.base_io.vtable.groupCancel; + f(self.base_io.userdata, group, token); + } + + fn select(userdata: ?*anyopaque, futures: []const *std.Io.AnyFuture) std.Io.Cancelable!usize { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.base_io.vtable.select(self.base_io.userdata, futures); + } + + fn mutexLock(userdata: ?*anyopaque, prev_state: std.Io.Mutex.State, mutex: *std.Io.Mutex) std.Io.Cancelable!void { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.base_io.vtable.mutexLock(self.base_io.userdata, prev_state, mutex); + } + + fn mutexLockUncancelable(userdata: ?*anyopaque, prev_state: std.Io.Mutex.State, mutex: *std.Io.Mutex) void { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + const f = self.base_io.vtable.mutexLockUncancelable; + f(self.base_io.userdata, prev_state, mutex); + } + + fn mutexUnlock(userdata: ?*anyopaque, prev_state: std.Io.Mutex.State, mutex: *std.Io.Mutex) void { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + const f = self.base_io.vtable.mutexUnlock; + f(self.base_io.userdata, prev_state, mutex); + } + + fn conditionWait(userdata: ?*anyopaque, cond: *std.Io.Condition, mutex: *std.Io.Mutex) std.Io.Cancelable!void { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.base_io.vtable.conditionWait(self.base_io.userdata, cond, mutex); + } + + fn conditionWaitUncancelable(userdata: ?*anyopaque, cond: *std.Io.Condition, mutex: *std.Io.Mutex) void { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + const f = self.base_io.vtable.conditionWaitUncancelable; + f(self.base_io.userdata, cond, mutex); + } + + fn conditionWake(userdata: ?*anyopaque, cond: *std.Io.Condition, wake: std.Io.Condition.Wake) void { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + const f = self.base_io.vtable.conditionWake; + f(self.base_io.userdata, cond, wake); + } + + fn now(userdata: ?*anyopaque, clock: std.Io.Clock) std.Io.Clock.Error!std.Io.Timestamp { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.base_io.vtable.now(self.base_io.userdata, clock); + } + + fn sleep(userdata: ?*anyopaque, timeout: std.Io.Timeout) std.Io.SleepError!void { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.base_io.vtable.sleep(self.base_io.userdata, timeout); + } + + fn netListenIp( + userdata: ?*anyopaque, + address: std.Io.net.IpAddress, + options: std.Io.net.IpAddress.ListenOptions, + ) std.Io.net.IpAddress.ListenError!std.Io.net.Server { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.base_io.vtable.netListenIp(self.base_io.userdata, address, options); + } + + fn netAccept(userdata: ?*anyopaque, server: std.Io.net.Socket.Handle) std.Io.net.Server.AcceptError!std.Io.net.Stream { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.base_io.vtable.netAccept(self.base_io.userdata, server); + } + + fn netBindIp( + userdata: ?*anyopaque, + address: *const std.Io.net.IpAddress, + options: std.Io.net.IpAddress.BindOptions, + ) std.Io.net.IpAddress.BindError!std.Io.net.Socket { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.base_io.vtable.netBindIp(self.base_io.userdata, address, options); + } + + fn netConnectIp( + userdata: ?*anyopaque, + address: *const std.Io.net.IpAddress, + options: std.Io.net.IpAddress.ConnectOptions, + ) std.Io.net.IpAddress.ConnectError!std.Io.net.Stream { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.base_io.vtable.netConnectIp(self.base_io.userdata, address, options); + } + + fn netListenUnix( + userdata: ?*anyopaque, + address: *const std.Io.net.UnixAddress, + options: std.Io.net.UnixAddress.ListenOptions, + ) std.Io.net.UnixAddress.ListenError!std.Io.net.Socket.Handle { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.base_io.vtable.netListenUnix(self.base_io.userdata, address, options); + } + + fn netConnectUnix( + userdata: ?*anyopaque, + address: *const std.Io.net.UnixAddress, + ) std.Io.net.UnixAddress.ConnectError!std.Io.net.Socket.Handle { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.base_io.vtable.netConnectUnix(self.base_io.userdata, address); + } + + fn netSend( + userdata: ?*anyopaque, + handle: std.Io.net.Socket.Handle, + msgs: []std.Io.net.OutgoingMessage, + flags: std.Io.net.SendFlags, + ) struct { ?std.Io.net.Socket.SendError, usize } { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.base_io.vtable.netSend(self.base_io.userdata, handle, msgs, flags); + } + + fn netReceive( + userdata: ?*anyopaque, + handle: std.Io.net.Socket.Handle, + message_buffer: []std.Io.net.IncomingMessage, + data_buffer: []u8, + flags: std.Io.net.ReceiveFlags, + timeout: std.Io.Timeout, + ) struct { ?std.Io.net.Socket.ReceiveTimeoutError, usize } { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.base_io.vtable.netReceive(self.base_io.userdata, handle, message_buffer, data_buffer, flags, timeout); + } + + fn netRead(userdata: ?*anyopaque, src: std.Io.net.Socket.Handle, data: [][]u8) std.Io.net.Stream.Reader.Error!usize { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.base_io.vtable.netRead(self.base_io.userdata, src, data); + } + + fn netWrite( + userdata: ?*anyopaque, + dest: std.Io.net.Socket.Handle, + header: []const u8, + data: []const []const u8, + splat: usize, + ) std.Io.net.Stream.Writer.Error!usize { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.base_io.vtable.netWrite(self.base_io.userdata, dest, header, data, splat); + } + + fn netClose(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle) void { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + const f = self.base_io.vtable.netClose; + f(self.base_io.userdata, handle); + } + + fn netInterfaceNameResolve( + userdata: ?*anyopaque, + name: *const std.Io.net.Interface.Name, + ) std.Io.net.Interface.Name.ResolveError!std.Io.net.Interface { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.base_io.vtable.netInterfaceNameResolve(self.base_io.userdata, name); + } + + fn netInterfaceName( + userdata: ?*anyopaque, + iface: std.Io.net.Interface, + ) std.Io.net.Interface.NameError!std.Io.net.Interface.Name { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.base_io.vtable.netInterfaceName(self.base_io.userdata, iface); + } + + fn netLookup( + userdata: ?*anyopaque, + host: std.Io.net.HostName, + q: *std.Io.Queue(std.Io.net.HostName.LookupResult), + options: std.Io.net.HostName.LookupOptions, + ) void { + const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + const f = self.base_io.vtable.netLookup; + f(self.base_io.userdata, host, q, options); + } }; diff --git a/zml2/io/vfs/BUILD.bazel b/zml2/io/vfs/BUILD.bazel index 591e869d..c60e0dce 100644 --- a/zml2/io/vfs/BUILD.bazel +++ b/zml2/io/vfs/BUILD.bazel @@ -4,6 +4,7 @@ zig_library( name = "vfs", srcs = [ "file.zig", + "hf.zig", "http.zig", ], main = "index.zig", diff --git a/zml2/io/vfs/file.zig b/zml2/io/vfs/file.zig index b6e6ebdb..0dfbbeaa 100644 --- a/zml2/io/vfs/file.zig +++ b/zml2/io/vfs/file.zig @@ -1,5 +1,7 @@ const std = @import("std"); +const log = std.log.scoped(.@"zml/io/vfs/file"); + pub const File = struct { inner: std.Io, vtable: std.Io.VTable, diff --git a/zml2/io/vfs/hf.zig b/zml2/io/vfs/hf.zig new file mode 100644 index 00000000..835fac8e --- /dev/null +++ b/zml2/io/vfs/hf.zig @@ -0,0 +1,981 @@ +const std = @import("std"); + +const log = std.log.scoped(.@"zml/io/vfs/hf"); + +// todo : refactor this code to align with http vfs + +pub const HF = struct { + pub const Auth = union(enum) { + none, + hf_token: []const u8, + + pub const AuthError = error{ + MissingHFToken, + InvalidHFToken, + MissingHomePath, + MissingConfigFile, + } || std.process.GetEnvVarOwnedError || std.fs.File.OpenError || std.fs.File.Reader.Error || std.Io.Reader.ReadAllocError || std.mem.Allocator.Error; + + pub fn deinit(self: *Auth, allocator: std.mem.Allocator) void { + switch (self.*) { + .none => {}, + .hf_token => |token| { + allocator.free(token); + }, + } + } + + pub fn auto(allocator: std.mem.Allocator, io_: std.Io) AuthError!Auth { + return fromHomeConfig(allocator, io_) catch |err| switch (err) { + AuthError.MissingHomePath => return fromEnv(allocator), + AuthError.MissingConfigFile => return fromEnv(allocator), + else => |e| return e, + }; + } + + pub fn fromEnv(allocator: std.mem.Allocator) AuthError!Auth { + const token = std.process.getEnvVarOwned(allocator, "HF_TOKEN") catch |err| switch (err) { + error.EnvironmentVariableNotFound => return AuthError.MissingHFToken, + else => |e| return e, + }; + defer allocator.free(token); + + return .{ .hf_token = try std.fmt.allocPrint(allocator, "Bearer {s}", .{token}) }; + } + + pub fn fromHomeConfig(allocator: std.mem.Allocator, io_: std.Io) AuthError!Auth { + const home_path = std.process.getEnvVarOwned(allocator, "HOME") catch |err| switch (err) { + error.EnvironmentVariableNotFound => return AuthError.MissingHomePath, + else => |e| return e, + }; + defer allocator.free(home_path); + + const token_path = try std.fmt.allocPrint(allocator, "{s}/.cache/huggingface/token", .{home_path}); + defer allocator.free(token_path); + + const file = std.Io.File.openAbsolute(io_, token_path, .{ .mode = .read_only }) catch |err| switch (err) { + error.FileNotFound => return AuthError.MissingConfigFile, + else => |e| return e, + }; + + var reader = file.reader(io_, &.{}); + const size = reader.getSize() catch unreachable; + + const token = try reader.interface.readAlloc(allocator, size); + defer allocator.free(token); + + return .{ .hf_token = try std.fmt.allocPrint(allocator, "Bearer {s}", .{token}) }; + } + }; + + const Config = struct { + auth: Auth = .none, + range_size: u64 = 128 * 1024 * 1024, // todo: rename field + }; + + allocator: std.mem.Allocator, + client: *std.http.Client, // todo: rename http_client + http_headers: []std.http.Header, + config: Config, + + file_handles: std.AutoHashMapUnmanaged(HandleId, *FileHandle), + dir_handles: std.AutoHashMapUnmanaged(HandleId, *DirHandle), + next_handle_id: HandleId, + mutex: std.Thread.Mutex, + vtable: std.Io.VTable, + + pub const HandleId = i32; + const max_retries = 10; + + pub const FileHandle = struct { + id: HandleId, + allocator: std.mem.Allocator, + hf_path: []const u8, + pos: u64, + built_url: []const u8, + size: ?u64, + redirect_buffer: [1024 * 1024]u8, + request: ?*std.http.Client.Request, + response: ?std.http.Client.Response, + body_reader: ?*std.Io.Reader, + response_bytes_remaining: u64, + + pub fn init(allocator: std.mem.Allocator, id: HandleId, hf_path: []const u8) !*FileHandle { + const handle = try allocator.create(FileHandle); + handle.* = .{ + .id = id, + .allocator = allocator, + .hf_path = try allocator.dupe(u8, hf_path), + .pos = 0, + .built_url = "", + .size = null, + .redirect_buffer = undefined, + .request = null, + .response = null, + .body_reader = null, + .response_bytes_remaining = 0, + }; + return handle; + } + + pub fn deinit(self: *FileHandle) void { + self.releaseRequest(); + self.allocator.free(self.hf_path); + self.allocator.free(self.built_url); + self.allocator.destroy(self); + } + + pub fn releaseRequest(self: *FileHandle) void { + if (self.request) |req| { + req.deinit(); + self.allocator.destroy(req); + self.request = null; + self.response = null; + self.body_reader = null; + self.response_bytes_remaining = 0; + } + } + }; + + pub const DirHandle = struct { + id: HandleId, + allocator: std.mem.Allocator, + hf_path: []const u8, + built_url: []const u8, + redirect_buffer: []u8, + request: ?*std.http.Client.Request, + response: ?std.http.Client.Response, + body_reader: ?*std.Io.Reader, + + pub fn init(allocator: std.mem.Allocator, id: HandleId, url: []const u8) std.mem.Allocator.Error!*DirHandle { + const self = try allocator.create(DirHandle); + errdefer allocator.destroy(self); + + const redirect_buffer = try allocator.alloc(u8, 1024 * 1024); + errdefer allocator.free(redirect_buffer); + + const owned_path = try allocator.dupe(u8, url); + errdefer allocator.free(owned_path); + + self.* = .{ + .id = id, + .allocator = allocator, + .hf_path = owned_path, + .built_url = "", + .redirect_buffer = redirect_buffer, + .request = null, + .response = null, + .body_reader = null, + }; + + return self; + } + + pub fn deinit(self: *DirHandle) void { + self.allocator.free(self.hf_path); + self.allocator.free(self.built_url); + self.allocator.free(self.redirect_buffer); + self.allocator.destroy(self); + } + }; + + pub const InitArgs = struct { + allocator: std.mem.Allocator, + io: std.Io, + http_client: *std.http.Client, + config: Config, + }; + + pub fn init(args: InitArgs) !HF { + var self: HF = .{ + .allocator = args.allocator, + .client = args.http_client, + .http_headers = try args.allocator.alloc(std.http.Header, 2), + .config = args.config, + .file_handles = .{}, + .dir_handles = .{}, + .next_handle_id = 1, + .mutex = .{}, + .vtable = makeVTable(args.io), + }; + + switch (self.config.auth) { + .hf_token => |token| { + self.http_headers[0] = .{ .name = "Authorization", .value = token }; + }, + else => {}, + } + + return self; + } + + pub fn deinit(self: *HF) void { + var file_it = self.file_handles.valueIterator(); + while (file_it.next()) |handle_ptr| { + handle_ptr.*.deinit(); + } + self.file_handles.deinit(self.allocator); + + var dir_it = self.dir_handles.valueIterator(); + while (dir_it.next()) |handle_ptr| { + handle_ptr.*.deinit(); + } + self.dir_handles.deinit(self.allocator); + self.allocator.free(self.http_headers); + } + + pub fn io(self: *HF) std.Io { + return .{ + .vtable = &self.vtable, + .userdata = self, + }; + } + + fn allocateHandleId(self: *HF) HandleId { + self.mutex.lock(); + defer self.mutex.unlock(); + + const id = self.next_handle_id; + self.next_handle_id += 1; + return id; + } + + fn registerFileHandle(self: *HF, handle: *FileHandle) std.mem.Allocator.Error!void { + self.mutex.lock(); + defer self.mutex.unlock(); + + try self.file_handles.put(self.allocator, handle.id, handle); + } + + fn unregisterFileHandle(self: *HF, id: HandleId) ?*FileHandle { + self.mutex.lock(); + defer self.mutex.unlock(); + + const kv = self.file_handles.fetchRemove(id) orelse return null; + return kv.value; + } + + fn getFileHandle(self: *HF, id: HandleId) ?*FileHandle { + self.mutex.lock(); + defer self.mutex.unlock(); + + return self.file_handles.get(id); + } + + fn registerDirHandle(self: *HF, handle: *DirHandle) std.mem.Allocator.Error!void { + self.mutex.lock(); + defer self.mutex.unlock(); + + try self.dir_handles.put(self.allocator, handle.id, handle); + } + + fn unregisterDirHandle(self: *HF, id: HandleId) ?*DirHandle { + self.mutex.lock(); + defer self.mutex.unlock(); + + const kv = self.dir_handles.fetchRemove(id) orelse return null; + return kv.value; + } + + fn getDirHandle(self: *HF, id: HandleId) ?*DirHandle { + self.mutex.lock(); + defer self.mutex.unlock(); + return self.dir_handles.get(id); + } + + fn resolvePath(self: *HF, dir_handle: std.Io.Dir.Handle, sub_path: []const u8) ![]const u8 { + if (std.mem.startsWith(u8, sub_path, "hf://")) { + return try self.allocator.dupe(u8, sub_path); + } + + const dir = self.getDirHandle(dir_handle) orelse { + return error.DirectoryNotfound; + }; + + return joinPath(self.allocator, dir.hf_path, sub_path); + } + + pub fn joinPath(allocator: std.mem.Allocator, base: []const u8, path: []const u8) ![]const u8 { + if (base.len == 0) { + return try allocator.dupe(u8, path); + } + if (path.len == 0) { + return try allocator.dupe(u8, base); + } + + const base_has_slash = base[base.len - 1] == '/'; + const path_has_slash = path[0] == '/'; + + if (base_has_slash and path_has_slash) { + return try std.fmt.allocPrint(allocator, "{s}{s}", .{ base, path[1..] }); + } else if (base_has_slash or path_has_slash) { + return try std.fmt.allocPrint(allocator, "{s}{s}", .{ base, path }); + } else { + return try std.fmt.allocPrint(allocator, "{s}/{s}", .{ base, path }); + } + } + + fn makeVTable(base_io: std.Io) std.Io.VTable { + var vtable = base_io.vtable.*; + vtable.dirMake = dirMake; + vtable.dirMakePath = dirMakePath; + vtable.dirMakeOpenPath = dirMakeOpenPath; + vtable.dirStat = dirStat; + vtable.dirStatPath = dirStatPath; + vtable.dirAccess = dirAccess; + vtable.dirCreateFile = dirCreateFile; + vtable.dirOpenFile = dirOpenFile; + vtable.dirOpenDir = dirOpenDir; + vtable.dirClose = dirClose; + vtable.fileStat = fileStat; + vtable.fileClose = fileClose; + vtable.fileWriteStreaming = fileWriteStreaming; + vtable.fileWritePositional = fileWritePositional; + vtable.fileReadStreaming = fileReadStreaming; + vtable.fileReadPositional = fileReadPositional; + vtable.fileSeekBy = fileSeekBy; + vtable.fileSeekTo = fileSeekTo; + vtable.openSelfExe = openSelfExe; + + return vtable; + } + + fn dirMake( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + mode: std.Io.Dir.Mode, + ) std.Io.Dir.MakeError!void { + _ = userdata; + _ = dir; + _ = sub_path; + _ = mode; + return error.Unexpected; + } + + fn dirMakePath( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + mode: std.Io.Dir.Mode, + ) std.Io.Dir.MakeError!void { + _ = userdata; + _ = dir; + _ = sub_path; + _ = mode; + return error.Unexpected; + } + + fn dirMakeOpenPath( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + options: std.Io.Dir.OpenOptions, + ) std.Io.Dir.MakeOpenPathError!std.Io.Dir { + _ = userdata; + _ = dir; + _ = sub_path; + _ = options; + return error.Unexpected; + } + + pub fn updateBuiltUrl(self: *HF, url_ptr: *[]const u8, new_url: []const u8) !void { + const owned_url = try self.allocator.dupe(u8, new_url); + errdefer self.allocator.free(owned_url); + + self.allocator.free(url_ptr.*); + url_ptr.* = owned_url; + } + + fn dirStat( + userdata: ?*anyopaque, + dir: std.Io.Dir, + ) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); + const handle = self.getDirHandle(@intCast(dir.handle)) orelse return error.Unexpected; + const stat_result = self.statDir(handle) catch return error.Unexpected; + + return .{ + .inode = @intCast(handle.id), + .kind = .directory, + .size = stat_result.size, + .mode = 0o555, + .atime = .{ .nanoseconds = 0 }, + .mtime = .{ .nanoseconds = 0 }, + .ctime = .{ .nanoseconds = 0 }, + }; + } + + fn statDir(self: *HF, handle: *DirHandle) !StatResult { + const path = handle.hf_path[5..]; + var path_it = std.mem.splitSequence(u8, path, "/"); + const namespace = path_it.next() orelse return error.Unexpected; + const repo = path_it.next() orelse return error.Unexpected; + const rev = "main"; + // Build API URL + var url_buf: [512]u8 = undefined; + const url = std.fmt.bufPrint( + &url_buf, + "https://huggingface.co/api/models/{s}/{s}/treesize/{s}/", + .{ namespace, repo, rev }, + ) catch return error.SystemResources; + + self.updateBuiltUrl(&handle.built_url, url) catch return error.SystemResources; + + return try self.fetchStatDir(handle); + } + + pub const StatResult = struct { + size: u64, + }; + + fn fetchStatDir(self: *HF, handle: *DirHandle) !StatResult { + var attempt: u32 = 0; + while (attempt < max_retries) : (attempt += 1) { + const req = self.allocator.create(std.http.Client.Request) catch continue; + + req.* = self.client.request(.GET, std.Uri.parse(handle.built_url) catch return error.Unexpected, .{ + .headers = .{ .accept_encoding = .{ .override = "identity" } }, + .extra_headers = self.http_headers[0..1], + }) catch { + self.allocator.destroy(req); + continue; + }; + + req.sendBodiless() catch { + req.deinit(); + self.allocator.destroy(req); + continue; + }; + + var response = req.receiveHead(handle.redirect_buffer) catch { + req.deinit(); + self.allocator.destroy(req); + continue; + }; + + defer { + req.deinit(); + self.allocator.destroy(req); + } + + switch (response.head.status) { + .moved_permanently, .found, .see_other, .temporary_redirect, .permanent_redirect => { + if (response.head.location) |location| { + var buf: [4096]u8 = undefined; + if (location.len > buf.len) return error.InvalidRedirect; + @memcpy(buf[0..location.len], location); + + var aux_buf: []u8 = buf[0..]; + const resolved = try std.Uri.resolveInPlace(try std.Uri.parse(handle.built_url), location.len, &aux_buf); + const new_url = try std.fmt.allocPrint(self.allocator, "{f}", .{resolved.fmt(.{ + .scheme = true, + .authority = true, + .path = true, + .query = true, + .fragment = true, + })}); + self.allocator.free(handle.built_url); + handle.built_url = new_url; + + continue; + } + }, + .ok => { + var body_buf: [256]u8 = undefined; + var body_reader = response.reader(&.{}); + var iov = [_][]u8{&body_buf}; + const body_len = try body_reader.readVec(&iov); + const body = body_buf[0..body_len]; + + var parsed = try std.json.parseFromSlice(std.json.Value, self.allocator, body, .{}); + defer parsed.deinit(); + + const size_value = parsed.value.object.get("size") orelse return error.InvalidFormat; + const size = size_value.integer; + + return .{ .size = @intCast(size) }; + }, + .internal_server_error, .bad_gateway, .service_unavailable, .gateway_timeout => { + continue; + }, + else => { + log.err("HTTP stat dir failed with status: {s} for {s}", .{ @tagName(response.head.status), handle.built_url }); + return error.HttpRequestFailed; + }, + } + } + + return error.Unexpected; + } + + fn dirStatPath( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + options: std.Io.Dir.StatPathOptions, + ) std.Io.Dir.StatPathError!std.Io.File.Stat { + const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); + _ = options; + + const file = dirOpenFile(userdata, dir, sub_path, .{}) catch return std.Io.Dir.StatPathError.BadPathName; + + const handle = self.getFileHandle(@intCast(file.handle)) orelse return error.Unexpected; + defer fileClose(userdata, file); + + const stat_result = self.fetchStat(handle) catch return error.Unexpected; + + return .{ + .inode = @intCast(file.handle), + .kind = .file, + .size = stat_result.size, + .mode = 0o444, + .atime = .{ .nanoseconds = 0 }, + .mtime = .{ .nanoseconds = 0 }, + .ctime = .{ .nanoseconds = 0 }, + }; + } + + fn dirAccess( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + options: std.Io.Dir.AccessOptions, + ) std.Io.Dir.AccessError!void { + _ = userdata; + _ = dir; + _ = sub_path; + if (options.write) { + return std.Io.Dir.AccessError.ReadOnlyFileSystem; + } + + // todo: check + } + + fn dirCreateFile( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + flags: std.Io.File.CreateFlags, + ) std.Io.File.OpenError!std.Io.File { + _ = userdata; + _ = dir; + _ = sub_path; + _ = flags; + return std.Io.File.OpenError.Unexpected; + } + + fn dirOpenFile( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + flags: std.Io.File.OpenFlags, + ) std.Io.File.OpenError!std.Io.File { + const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); + _ = flags; + + const path = self.resolvePath(@intCast(dir.handle), sub_path) catch return error.SystemResources; + defer self.allocator.free(path); + + return try self.openFile(path); + } + + fn openFile(self: *HF, path: []const u8) std.Io.File.OpenError!std.Io.File { + const url = hfPathToHttps(self.allocator, path) catch return error.SystemResources; + defer self.allocator.free(url); + const id = self.allocateHandleId(); + + // Check if it's a valid file + var handle = FileHandle.init(self.allocator, id, path) catch return error.SystemResources; + errdefer handle.deinit(); + self.updateBuiltUrl(&handle.built_url, url) catch return error.SystemResources; + + _ = self.fetchStat(handle) catch return error.FileNotFound; + + // URL ownership transferred to handle + + self.registerFileHandle(handle) catch return error.SystemResources; + + return .{ .handle = @intCast(id) }; + } + + fn dirOpenDir( + userdata: ?*anyopaque, + dir: std.Io.Dir, + sub_path: []const u8, + options: std.Io.Dir.OpenOptions, + ) std.Io.Dir.OpenError!std.Io.Dir { + const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); + _ = options; + const path = self.resolvePath(@intCast(dir.handle), sub_path) catch return error.Unexpected; + defer self.allocator.free(path); + + return try self.openDir(path); + } + + fn openDir(self: *HF, path: []const u8) std.Io.Dir.OpenError!std.Io.Dir { + const url = hfPathToHttps(self.allocator, path) catch return error.SystemResources; + defer self.allocator.free(url); + // Ensure URL ends with / + const dir_url = if (url.len > 0 and url[url.len - 1] != '/') blk: { + const new_url = std.fmt.allocPrint(self.allocator, "{s}/", .{url}) catch return error.Unexpected; + self.allocator.free(url); + break :blk new_url; + } else url; + errdefer if (dir_url.ptr != url.ptr) self.allocator.free(dir_url); + + const id = self.allocateHandleId(); + const handle = DirHandle.init(self.allocator, id, path) catch return error.Unexpected; + + // Ensure it's a correct directory + const stat = self.statDir(handle) catch return error.NotDir; + _ = stat; + + errdefer handle.deinit(); + + self.registerDirHandle(handle) catch return error.Unexpected; + + return .{ .handle = @intCast(id) }; + } + + fn dirClose(userdata: ?*anyopaque, dir: std.Io.Dir) void { + const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); + const handle = self.unregisterDirHandle(@intCast(dir.handle)) orelse return; + handle.deinit(); + } + + fn fileStat( + userdata: ?*anyopaque, + file: std.Io.File, + ) std.Io.File.StatError!std.Io.File.Stat { + const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); + const handle = self.getFileHandle(@intCast(file.handle)) orelse return error.Unexpected; + + // call self.httpengine.fetchStat(handle) + // log.debug("Fetching stat for file handle {d}", handle.id); + const stat_result = self.fetchStat(handle) catch return error.Unexpected; + + return .{ + .inode = @intCast(file.handle), + .kind = .file, + .size = stat_result.size, + .mode = 0o444, + .atime = .{ .nanoseconds = 0 }, + .mtime = .{ .nanoseconds = 0 }, + .ctime = .{ .nanoseconds = 0 }, + }; + } + + pub fn fetchStat(self: *HF, handle: *FileHandle) !StatResult { + var attempt: u32 = 0; + while (attempt < max_retries) : (attempt += 1) { + const req = self.allocator.create(std.http.Client.Request) catch continue; + + req.* = self.client.request(.HEAD, std.Uri.parse(handle.built_url) catch return error.Unexpected, .{ + .headers = .{ .accept_encoding = .{ .override = "identity" } }, + .extra_headers = self.http_headers[0..1], + }) catch { + self.allocator.destroy(req); + continue; + }; + + req.sendBodiless() catch { + req.deinit(); + self.allocator.destroy(req); + continue; + }; + + var response = req.receiveHead(&handle.redirect_buffer) catch { + req.deinit(); + self.allocator.destroy(req); + continue; + }; + + defer { + req.deinit(); + self.allocator.destroy(req); + } + + switch (response.head.status) { + .moved_permanently, .found, .see_other, .temporary_redirect, .permanent_redirect => { + if (response.head.location) |location| { + var buf: [4096]u8 = undefined; + if (location.len > buf.len) return error.InvalidRedirect; + @memcpy(buf[0..location.len], location); + + var aux_buf: []u8 = buf[0..]; + const resolved = try std.Uri.resolveInPlace(try std.Uri.parse(handle.built_url), location.len, &aux_buf); + const new_url = try std.fmt.allocPrint(self.allocator, "{f}", .{resolved.fmt(.{ + .scheme = true, + .authority = true, + .path = true, + .query = true, + .fragment = true, + })}); + self.allocator.free(handle.built_url); + handle.built_url = new_url; + + continue; + } + }, + .ok, .partial_content => { + const size = response.head.content_length orelse return error.MissingContentLength; + handle.size = size; + + return .{ .size = size }; + }, + .internal_server_error, .bad_gateway, .service_unavailable, .gateway_timeout => { + continue; + }, + else => { + log.err("HTTP stat failed with status: {s} - {s}", .{ @tagName(response.head.status), handle.built_url }); + return error.HttpRequestFailed; + }, + } + } + + return error.Unexpected; + } + + fn fileClose(userdata: ?*anyopaque, file: std.Io.File) void { + const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); + const handle = self.unregisterFileHandle(@intCast(file.handle)) orelse return; + handle.deinit(); + } + + fn fileWriteStreaming( + userdata: ?*anyopaque, + file: std.Io.File, + buffer: [][]const u8, + ) std.Io.File.WriteStreamingError!usize { + _ = userdata; + _ = file; + _ = buffer; + return error.Unexpected; + } + + fn fileWritePositional( + userdata: ?*anyopaque, + file: std.Io.File, + buffer: [][]const u8, + offset: u64, + ) std.Io.File.WritePositionalError!usize { + _ = userdata; + _ = file; + _ = buffer; + _ = offset; + return error.Unexpected; + } + + fn fileReadStreaming( + userdata: ?*anyopaque, + file: std.Io.File, + data: [][]u8, + ) std.Io.File.Reader.Error!usize { + const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); + const handle = self.getFileHandle(@intCast(file.handle)) orelse return error.Unexpected; + return self.performRead(handle, data, null) catch return error.Unexpected; + } + + fn fileReadPositional( + userdata: ?*anyopaque, + file: std.Io.File, + data: [][]u8, + offset: u64, + ) std.Io.File.ReadPositionalError!usize { + const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); + const handle = self.getFileHandle(@intCast(file.handle)) orelse return error.Unexpected; + return self.performRead(handle, data, offset) catch return error.Unexpected; + } + + fn fileSeekBy( + userdata: ?*anyopaque, + file: std.Io.File, + relative_offset: i64, + ) std.Io.File.SeekError!void { + const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); + const handle = self.getFileHandle(@intCast(file.handle)) orelse return error.Unexpected; + + seekBy(handle, relative_offset) catch return error.Unexpected; + } + + fn fileSeekTo( + userdata: ?*anyopaque, + file: std.Io.File, + absolute_offset: u64, + ) std.Io.File.SeekError!void { + const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); + const handle = self.getFileHandle(@intCast(file.handle)) orelse return error.Unexpected; + + seekTo(handle, absolute_offset) catch return error.Unexpected; + } + + pub fn seekBy(handle: *FileHandle, relative_offset: i64) !void { + handle.releaseRequest(); + if (relative_offset >= 0) { + handle.pos +|= @intCast(relative_offset); + } else { + const abs_offset: u64 = @intCast(-relative_offset); + if (abs_offset > handle.pos) { + handle.pos = 0; + } else { + handle.pos -= abs_offset; + } + } + } + + // Go back to VFS + pub fn seekTo(handle: *FileHandle, absolute_offset: u64) !void { + handle.releaseRequest(); + handle.pos = absolute_offset; + } + + fn openSelfExe( + userdata: ?*anyopaque, + flags: std.Io.File.OpenFlags, + ) std.Io.File.OpenSelfExeError!std.Io.File { + _ = userdata; + _ = flags; + return error.NotSupported; + } + + fn hfPathToHttps(allocator: std.mem.Allocator, hf_path: []const u8) ![]const u8 { + // Remove the "hf://" prefix + const path = hf_path[5..hf_path.len]; + var path_it = std.mem.splitSequence(u8, path, "/"); + const namespace = path_it.next() orelse return error.InvalidPath; + const repo = path_it.next() orelse return error.InvalidPath; + const rev = "main"; + var file: []const u8 = ""; + if (path_it.next()) |file_path| { + file = file_path; + } + + var final_url_buf: [std.fs.max_path_bytes]u8 = undefined; + + const final_url = try std.fmt.bufPrint( + &final_url_buf, + "https://huggingface.co/{s}/{s}/resolve/{s}/{s}", + .{ namespace, repo, rev, file }, + ); + + return try allocator.dupe(u8, final_url); + } + + fn ensureConnection(self: *HF, handle: *FileHandle) !void { + if (handle.body_reader != null) return; + + if (handle.size == null) { + const stat = try self.fetchStat(handle); + handle.size = stat.size; + } + + const size = handle.size.?; + + if (handle.pos >= size) return; + + const start = handle.pos; + const end = @min(start + self.config.range_size, size) - 1; + + var range_buf: [64]u8 = undefined; + const range_header = std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ start, end }) catch unreachable; + + const request = try self.allocator.create(std.http.Client.Request); + errdefer self.allocator.destroy(request); + + const uri = try std.Uri.parse(handle.built_url); + + self.http_headers[1] = .{ .name = "Range", .value = range_header }; + + request.* = try self.client.request(.GET, uri, .{ + .headers = .{ .accept_encoding = .{ .override = "identity" } }, + .extra_headers = self.http_headers[0..2], + }); + // errdefer request.deinit(); + + try request.sendBodiless(); + + const response = try request.receiveHead(&handle.redirect_buffer); + + if (response.head.status != .partial_content and response.head.status != .ok) { + request.deinit(); + self.allocator.destroy(request); + return error.Unexpected; + } + + handle.request = request; + handle.response = response; + handle.body_reader = handle.response.?.reader(&.{}); + } + + fn doRead(self: *HF, handle: *FileHandle, data: [][]u8) !usize { + if (data.len == 0) return 0; + + var total_read: usize = 0; + + for (data) |buf| { + var offset: usize = 0; + + while (offset < buf.len) { + // Check EOF against known file size + if (handle.size) |size| { + if (handle.pos >= size) { + return total_read; + } + } + + try self.ensureConnection(handle); + const reader = handle.body_reader orelse { + return total_read; + }; + + // Read into remaining buffer + const dest = buf[offset..]; + var iov = [_][]u8{dest}; + + const n = reader.readVec(&iov) catch |err| switch (err) { + error.EndOfStream => { + // Current HTTP range exhausted, need new request + handle.releaseRequest(); + continue; + }, + else => { + return err; + }, + }; + + // Data not ready yet, just retry same read + if (n == 0) { + continue; + } + + offset += n; + handle.pos += n; + total_read += n; + } + } + + return total_read; + } + + fn performRead(self: *HF, handle: *FileHandle, data: [][]u8, offset: ?u64) !usize { + const saved_pos = handle.pos; + const is_positional = offset != null; + + if (offset) |o| { + if (handle.pos != o) { + handle.releaseRequest(); + handle.pos = o; + } + } + + const bytes_read = try self.doRead(handle, data); + + if (is_positional) { + handle.pos = saved_pos; + handle.releaseRequest(); + } + + return bytes_read; + } +}; diff --git a/zml2/io/vfs/http.zig b/zml2/io/vfs/http.zig index 85a55d15..f459b484 100644 --- a/zml2/io/vfs/http.zig +++ b/zml2/io/vfs/http.zig @@ -3,7 +3,7 @@ const std = @import("std"); const log = std.log.scoped(.@"zml/io/vfs/http"); pub const HTTP = struct { - pub const HandleId = i32; + const HandleId = i32; pub const FileHandle = struct { allocator: std.mem.Allocator, @@ -11,7 +11,7 @@ pub const HTTP = struct { url: []const u8, pos: u64, size: ?u64, - redirect_buffer: [1024 * 1024]u8, + redirect_buffer: [std.fs.max_path_bytes]u8, request: ?*std.http.Client.Request, response: ?std.http.Client.Response, body_reader: ?*std.Io.Reader, @@ -106,7 +106,8 @@ pub const HTTP = struct { pub const Config = struct { max_fetch_attempts: u32 = 10, - request_range_size: u64 = 128 * 1024 * 1024, + request_range_min: u64 = 8 * 1024, + request_range_max: u64 = 128 * 1024 * 1024, }; allocator: std.mem.Allocator, @@ -122,7 +123,7 @@ pub const HTTP = struct { vtable: std.Io.VTable, - pub fn init(allocator: std.mem.Allocator, base_io: std.Io, http_client: *std.http.Client, config: Config) !HTTP { + pub fn init(allocator: std.mem.Allocator, base_io: std.Io, http_client: *std.http.Client, config: Config) std.mem.Allocator.Error!HTTP { var self: HTTP = .{ .allocator = allocator, .mutex = .{}, @@ -212,17 +213,20 @@ pub const HTTP = struct { sub_path: []const u8, ) std.Io.File.OpenError!std.Io.File { const dir_handle = self.dirHandle(dir) orelse { - log.err("Directory handle not found for dirOpenFile", .{}); + log.err("Directory handle not found for dirOpenFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); return std.Io.File.OpenError.FileNotFound; }; const handle = FileHandle.init(self.allocator, dir_handle.url, sub_path) catch { - log.err("Failed to create FileHandle in dirOpenFile", .{}); + log.err("Failed to create FileHandle in openFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); return std.Io.File.OpenError.SystemResources; }; errdefer handle.deinit(); - const handle_id = self.registerFileHandle(handle) catch return error.SystemResources; + const handle_id = self.registerFileHandle(handle) catch { + log.err("Failed to register FileHandle in openFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); + return std.Io.File.OpenError.SystemResources; + }; return .{ .handle = @intCast(handle_id) }; } @@ -243,7 +247,7 @@ pub const HTTP = struct { }; errdefer handle.deinit(); - const handle_id = self.registerDirHandle(handle) catch return error.SystemResources; + const handle_id = self.registerDirHandle(handle) catch return std.Io.Dir.OpenError.SystemResources; return .{ .handle = @intCast(handle_id) }; } @@ -273,13 +277,13 @@ pub const HTTP = struct { } fn setAbsolutePos(self: *HTTP, file: std.Io.File, pos: u64) std.Io.File.SeekError!void { - const handle = self.fileHandle(file) orelse return error.Unexpected; + const handle = self.fileHandle(file) orelse return std.Io.File.SeekError.Unexpected; handle.releaseRequest(); handle.pos = pos; } fn setRelativeOffset(self: *HTTP, file: std.Io.File, offset: i64) std.Io.File.SeekError!void { - const handle = self.fileHandle(file) orelse return error.Unexpected; + const handle = self.fileHandle(file) orelse return std.Io.File.SeekError.Unexpected; handle.releaseRequest(); const new_pos = @as(i64, @intCast(handle.pos)) + offset; @@ -314,7 +318,13 @@ pub const HTTP = struct { var total_read: usize = 0; + log.debug("Starting HTTP read handle={d} pos={d} buffers={d} url={s}", .{ file.handle, handle.pos, data.len, handle.url }); + for (data) |buf| { + log.debug(" buffer len={d}", .{buf.len}); + } + + for (data, 0..) |buf, buf_index| { var offset: usize = 0; while (offset < buf.len) { @@ -325,17 +335,22 @@ pub const HTTP = struct { } } - try self.requestRead(file); - const reader = handle.body_reader orelse { - return total_read; // todo: handle this properly - }; + var desired: u64 = 0; + for (data[buf_index..]) |b| desired += b.len; + desired -= offset; + + if (desired < self.config.request_range_min) desired = self.config.request_range_min; + if (desired > self.config.request_range_max) desired = self.config.request_range_max; + + try self.requestRead(file, desired); + const reader = handle.body_reader.?; const dest = buf[offset..]; var iov = [_][]u8{dest}; const n = reader.readVec(&iov) catch |err| switch (err) { error.EndOfStream => { - // Current HTTP range exhausted, need new request + log.debug("Reached EndOfStream for handle={d} pos={d}, will re-request remaining", .{ file.handle, handle.pos }); handle.releaseRequest(); continue; }, @@ -344,11 +359,13 @@ pub const HTTP = struct { }, }; - // Data not ready yet, just retry same read if (n == 0) { + // log.debug("readVec returned 0 (no progress) handle={d} pos={d} buf_index={d} offset={d}", .{ file.handle, handle.pos, buf_index, offset }); continue; } + // log.debug("readVec wrote {d} bytes for handle={d} pos(before)={d} buf_index={d} offset(before)={d}", .{ n, file.handle, handle.pos, buf_index, offset }); + offset += n; handle.pos += n; total_read += n; @@ -358,11 +375,13 @@ pub const HTTP = struct { return total_read; } - fn requestRead(self: *HTTP, file: std.Io.File) std.Io.File.Reader.Error!void { - const handle = self.fileHandle(file) orelse return error.Unexpected; + fn requestRead(self: *HTTP, file: std.Io.File, desired: u64) std.Io.File.Reader.Error!void { + const handle = self.fileHandle(file) orelse return std.Io.File.Reader.Error.Unexpected; if (handle.body_reader != null) return; + log.debug("Starting HTTP request lifecycle handle={d} pos={d} desired={d} url={s}", .{ file.handle, handle.pos, desired, handle.url }); + if (handle.size == null) { const stat = self.fetchFileStat(file) catch |err| { log.err("Failed to fetch file stat during read: {}", .{err}); @@ -376,7 +395,7 @@ pub const HTTP = struct { if (handle.pos >= size) return; const start = handle.pos; - const end = @min(start + self.config.request_range_size, size) - 1; + const end = @min(start + desired, size) - 1; var range_buf: [64]u8 = undefined; const range_header = std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ start, end }) catch unreachable; @@ -412,15 +431,20 @@ pub const HTTP = struct { }; const response = request.receiveHead(&handle.redirect_buffer) catch |err| { - log.err("Failed to receive HTTP response head: {}", .{err}); + log.err("Failed to receive HTTP response head for handle={d} url={s}: {}", .{ file.handle, handle.url, err }); return std.Io.File.Reader.Error.Unexpected; }; const latency = timer.read(); handle.recordLatency(latency); - log.debug("HTTP request: range={s} latency={d}ms avg={d}ms min={d}ms max={d}ms for {s}", .{ - range_header, + log.debug("HTTP call: handle={d} status={s} content_length={?} range={d}-{d} ({d}) latency={d}ms avg={d}ms min={d}ms max={d}ms for {s}", .{ + file.handle, + @tagName(response.head.status), + response.head.content_length, + start, + end, + end - start + 1, latency / std.time.ns_per_ms, handle.avgLatencyMs(), handle.min_latency_ns / std.time.ns_per_ms, @@ -431,12 +455,13 @@ pub const HTTP = struct { if (response.head.status != .partial_content and response.head.status != .ok) { request.deinit(); self.allocator.destroy(request); - return error.Unexpected; + return std.Io.File.Reader.Error.Unexpected; } handle.request = request; handle.response = response; handle.body_reader = handle.response.?.reader(&.{}); + log.debug("Initialized body_reader len={d} seek={d} for handle={s}", .{ handle.body_reader.?.buffer.len, handle.body_reader.?.seek, handle.url }); } pub const FileStat = struct { @@ -445,14 +470,14 @@ pub const HTTP = struct { }; fn fetchFileStat(self: *HTTP, file: std.Io.File) std.Io.File.StatError!FileStat { - const handle = self.fileHandle(file) orelse return error.Unexpected; + const handle = self.fileHandle(file) orelse return std.Io.File.StatError.Unexpected; var attempt: u32 = 0; while (attempt < self.config.max_fetch_attempts) : (attempt += 1) { const req = self.allocator.create(std.http.Client.Request) catch continue; errdefer self.allocator.destroy(req); - req.* = self.client.request(.HEAD, std.Uri.parse(handle.url) catch return error.Unexpected, .{ + req.* = self.client.request(.HEAD, std.Uri.parse(handle.url) catch return std.Io.File.StatError.Unexpected, .{ .headers = .{ .accept_encoding = .{ .override = "identity" } }, }) catch { self.allocator.destroy(req); @@ -605,7 +630,7 @@ pub const HTTP = struct { _ = sub_path; _ = options; log.err("HTTP VFS is read-only, dirMakeOpenPath not supported", .{}); - return error.Unexpected; + return std.Io.Dir.MakeOpenPathError.Unexpected; } fn dirStat( diff --git a/zml2/io/vfs/index.zig b/zml2/io/vfs/index.zig index 38b1c6e5..fc44f6b6 100644 --- a/zml2/io/vfs/index.zig +++ b/zml2/io/vfs/index.zig @@ -4,3 +4,4 @@ const log = std.log.scoped(.@"zml/io/vfs/vfs"); pub const File = @import("file.zig").File; pub const HTTP = @import("http.zig").HTTP; +pub const HF = @import("hf.zig").HF; diff --git a/zml2/main_io.zig b/zml2/main_io.zig index 13f2b6bb..a805c257 100644 --- a/zml2/main_io.zig +++ b/zml2/main_io.zig @@ -5,6 +5,14 @@ const zml = @import("zml"); const log = std.log.scoped(.@"zml/main_io"); +pub const std_options: std.Options = .{ + .log_level = .debug, + .log_scope_levels = &[_]std.log.ScopeLevel{ + .{ .scope = .@"zml/safetensors", .level = .debug }, + .{ .scope = .@"zml/io/vfs/http", .level = .info }, + }, +}; + pub fn main() !void { var debug_allocator: std.heap.DebugAllocator(.{}) = .init; defer std.debug.assert(debug_allocator.deinit() == .ok); @@ -39,135 +47,280 @@ pub fn main() !void { var vfs_http: zml.io.VFS.HTTP = try .init(allocator, threaded.io(), &http_client, .{}); defer vfs_http.deinit(); + var hf_auth: zml.io.VFS.HF.Auth = try .auto(allocator, threaded.io()); + defer hf_auth.deinit(allocator); + + var hf_vfs: zml.io.VFS.HF = try .init(.{ + .allocator = allocator, + .io = threaded.io(), + .http_client = &http_client, + .config = .{ + .auth = hf_auth, + }, + }); + defer hf_vfs.deinit(); + var vfs: zml.io.VFS = .init(allocator, threaded.io()); defer vfs.deinit(); try vfs.register("file", vfs_file.io()); try vfs.register("http", vfs_http.io()); try vfs.register("https", vfs_http.io()); + try vfs.register("hf", hf_vfs.io()); const io = vfs.io(); - const repo_uri = "file:///Users/hugo/Developer/Llama-3.1-8B-Instruct"; - // const repo_uri = "https://storage.googleapis.com/zig-vfs/Llama-3.1-8B-Instruct/"; - // const repo_uri = "http://9960x-5090x2:8003/"; + const default_uri = "hf://Qwen/Qwen3-8B"; + // const default_uri = "file:///Users/hugo/Developer/Llama-3.1-8B-Instruct"; + // const default_uri = "https://storage.googleapis.com/zig-vfs/Llama-3.1-8B-Instruct/"; + // const default_uri = "http://9960x-5090x2:8003/"; + + var args = std.process.args(); + _ = args.next(); // skip program name + + const uri = if (args.next()) |uri| blk: { + log.info("Using repo URI from args: {s}", .{uri}); + break :blk uri; + } else blk: { + log.info("Using default repo URI: {s}", .{default_uri}); + break :blk default_uri; + }; { - const root_dir = try vfs.openAbsoluteDir(io, "https://storage.googleapis.com", .{}); - defer root_dir.close(io); - - const buckets_dir = try root_dir.openDir(io, "zig-vfs", .{}); - defer buckets_dir.close(io); - - const model_dir = try buckets_dir.openDir(io, "Llama-3.1-8B-Instruct", .{}); - defer model_dir.close(io); - - const file = try model_dir.openFile(io, "model.safetensors.index.json", .{}); - defer file.close(io); - - const stat = try file.stat(io); - log.info("Opened remote file with size: {d} bytes by navigating dirs", .{stat.size}); - } - - { - const path = try std.fs.path.join(allocator, &.{ repo_uri, "model-00004-of-00004.safetensors" }); - defer allocator.free(path); - - const file = try vfs.openAbsoluteFile(io, path, .{}); - defer file.close(io); - - const stat = try file.stat(io); - - var group: std.Io.Group = .init; - defer group.cancel(io); - - const chunk_size: usize = 32 * 1024 * 1024; - const num_chunks = (stat.size + chunk_size - 1) / chunk_size; - - for (0..num_chunks) |i| { - const offset = i * chunk_size; - const size = if (offset + chunk_size > stat.size) stat.size - offset else chunk_size; - - group.async(threaded.io(), readData, .{ allocator, io, &vfs, path, offset, size, i }); + var timer: std.time.Timer = try .start(); + defer { + const elapsed = timer.read(); + log.info("Completed in {d} ms", .{elapsed / std.time.ns_per_ms}); } - group.wait(io); - } + var registry = try zml.safetensors.parseFromPath(allocator, io, &vfs, uri); + defer registry.deinit(); - { - const path = try std.fs.path.join(allocator, &.{ repo_uri, "model.safetensors.index.json" }); - defer allocator.free(path); + // var it = registry.iterator(); + // while (it.next()) |entry| { + // const tensor = entry.value_ptr.*; + // log.info("{f}", .{tensor}); + // } - const file = try vfs.openAbsoluteFile(io, path, .{}); - defer file.close(io); + log.info("Parsed {d} tensors", .{registry.tensors.count()}); - const stat = try file.stat(io); - log.info("Opened remote file with size: {d} bytes with openAbsoluteFile", .{stat.size}); - } + { + const tensor_name = "model.layers.31.mlp.down_proj.weight"; + // const tensor_name = "model.layers.31.input_layernorm.weight"; + const tensor = registry.tensors.get(tensor_name) orelse return error.TensorNotFound; + log.info("Reading {f}...", .{tensor}); - { - const model_dir = try vfs.openAbsoluteDir(io, repo_uri, .{}); + const tensor_reader_buf = try allocator.alloc(u8, tensor.byteSize()); + defer allocator.free(tensor_reader_buf); - const safetensors_index = try model_dir.openFile(io, "model.safetensors.index.json", .{}); - defer safetensors_index.close(io); + const writer_buffer = try allocator.alloc(u8, tensor.byteSize()); + defer allocator.free(writer_buffer); - const stat = try safetensors_index.stat(io); - log.info("Opened remote index file with size: {d} bytes with openAbsoluteDir + openFile", .{stat.size}); - } + var tensor_reader = try registry.reader(io, &vfs, tensor_name, tensor_reader_buf); + defer tensor_reader.deinit(); - { - const path = try std.fs.path.join(allocator, &.{ repo_uri, "model-00004-of-00004.safetensors" }); - defer allocator.free(path); + var writer: std.Io.Writer = .fixed(writer_buffer); + const read = try tensor_reader.interface.streamRemaining(&writer); - const file = try vfs.openAbsoluteFile(io, path, .{}); - defer file.close(io); - - var reader = file.reader(io, &.{}); - - const read_size = 16 * 1024 * 1024; - - var allocating_writer: std.Io.Writer.Allocating = try .initCapacity(allocator, read_size); - defer allocating_writer.deinit(); - - const read = try reader.interface.stream(&allocating_writer.writer, .limited(read_size)); - std.debug.assert(read == read_size); - } - - { - const path = try std.fs.path.join(allocator, &.{ repo_uri, "model-00004-of-00004.safetensors" }); // SHA256: 92ecfe1a2414458b4821ac8c13cf8cb70aed66b5eea8dc5ad9eeb4ff309d6d7b - defer allocator.free(path); - - const file = try vfs.openAbsoluteFile(io, path, .{}); - defer file.close(io); - - const stat = try file.stat(io); - - const buf_size = 64 * 1024 * 1024; - const buf = try allocator.alloc(u8, buf_size); - defer allocator.free(buf); - - var sha256: std.crypto.hash.sha2.Sha256 = .init(.{}); - const compute_sha = true; - - // Read directly without extra buffering layers - var total_read: usize = 0; - var bufs = [_][]u8{buf}; - - while (true) { - const n = try io.vtable.fileReadStreaming(io.userdata, file, &bufs); - if (n == 0) break; - if (compute_sha) sha256.update(buf[0..n]); - total_read += n; + log.info("Read tensor data: {d} bytes", .{read}); } - if (compute_sha) { - var hash: [32]u8 = undefined; - sha256.final(&hash); - log.info("SHA256: {x}", .{hash}); - } + { + var err: ?anyerror = null; - std.debug.assert(total_read == stat.size); + var group: std.Io.Group = .init; + defer group.cancel(io); + + var timer_async_read: std.time.Timer = try .start(); + var it = registry.iterator(); + while (it.next()) |entry| { + const tensor_name = entry.key_ptr.*; + group.async(threaded.io(), readTensor, .{ allocator, io, &vfs, ®istry, tensor_name, &err }); + } + + group.wait(io); + + if (err) |e| { + log.err("Error reading tensor: {any}", .{e}); + return e; + } + + const elapsed = timer_async_read.read(); + const read_mb = @as(f64, @floatFromInt(registry.totalBytes())) / (1024.0 * 1024.0); + const read_time_s = @as(f64, @floatFromInt(elapsed)) / @as(f64, std.time.ns_per_s); + log.info("Throughput: {d:.2} MB in {d:.2} s = {d:.2} MB/s", .{ read_mb, read_time_s, read_mb / read_time_s }); + } } + + // { + // const root_dir = try vfs.openAbsoluteDir(io, "https://storage.googleapis.com", .{}); + // defer root_dir.close(io); + + // const buckets_dir = try root_dir.openDir(io, "zig-vfs", .{}); + // defer buckets_dir.close(io); + + // const model_dir = try buckets_dir.openDir(io, "Llama-3.1-8B-Instruct", .{}); + // defer model_dir.close(io); + + // const file = try model_dir.openFile(io, "model.safetensors.index.json", .{}); + // defer file.close(io); + + // const stat = try file.stat(io); + // log.info("Opened remote file with size: {d} bytes by navigating dirs", .{stat.size}); + // } + + // { + // const path = try std.fs.path.join(allocator, &.{ default_uri, "model-00004-of-00005.safetensors" }); + // defer allocator.free(path); + + // const file = try vfs.openAbsoluteFile(io, path, .{}); + // defer file.close(io); + + // const stat = try file.stat(io); + + // var group: std.Io.Group = .init; + // defer group.cancel(io); + + // const chunk_size: usize = 32 * 1024 * 1024; + // const num_chunks = (stat.size + chunk_size - 1) / chunk_size; + + // for (0..num_chunks) |i| { + // const offset = i * chunk_size; + // const size = if (offset + chunk_size > stat.size) stat.size - offset else chunk_size; + + // group.async(threaded.io(), readData, .{ allocator, io, &vfs, path, offset, size, i }); + // } + + // group.wait(io); + // } + + // { + // const path = try std.fs.path.join(allocator, &.{ repo_uri, "model.safetensors.index.json" }); + // defer allocator.free(path); + + // const file = try vfs.openAbsoluteFile(io, path, .{}); + // defer file.close(io); + + // const stat = try file.stat(io); + // log.info("Opened remote file with size: {d} bytes with openAbsoluteFile", .{stat.size}); + // } + + // { + // const model_dir = try vfs.openAbsoluteDir(io, repo_uri, .{}); + + // const safetensors_index = try model_dir.openFile(io, "model.safetensors.index.json", .{}); + // defer safetensors_index.close(io); + + // const stat = try safetensors_index.stat(io); + // log.info("Opened remote index file with size: {d} bytes with openAbsoluteDir + openFile", .{stat.size}); + // } + + // { + // const path = try std.fs.path.join(allocator, &.{ repo_uri, "model-00004-of-00005.safetensors" }); + // defer allocator.free(path); + + // const file = try vfs.openAbsoluteFile(io, path, .{}); + // defer file.close(io); + + // var reader = file.reader(io, &.{}); + + // const read_size = 16 * 1024 * 1024; + + // var allocating_writer: std.Io.Writer.Allocating = try .initCapacity(allocator, read_size); + // defer allocating_writer.deinit(); + + // const read = try reader.interface.stream(&allocating_writer.writer, .limited(read_size)); + // std.debug.assert(read == read_size); + // } + + // { + // const path = try std.fs.path.join(allocator, &.{ default_uri, "model-00004-of-00005.safetensors" }); // SHA256: 92ecfe1a2414458b4821ac8c13cf8cb70aed66b5eea8dc5ad9eeb4ff309d6d7b + // defer allocator.free(path); + + // const file = try vfs.openAbsoluteFile(io, path, .{}); + // defer file.close(io); + + // const stat = try file.stat(io); + + // const buf_size = 64 * 1024 * 1024; + // const buf = try allocator.alloc(u8, buf_size); + // defer allocator.free(buf); + + // var sha256: std.crypto.hash.sha2.Sha256 = .init(.{}); + // const compute_sha = true; + + // // Read directly without extra buffering layers + // var total_read: usize = 0; + // var bufs = [_][]u8{buf}; + + // while (true) { + // const n = try io.vtable.fileReadStreaming(io.userdata, file, &bufs); + // if (n == 0) break; + // if (compute_sha) sha256.update(buf[0..n]); + // total_read += n; + // } + + // if (compute_sha) { + // var hash: [32]u8 = undefined; + // sha256.final(&hash); + // log.info("SHA256: {x}", .{hash}); + // } + + // std.debug.assert(total_read == stat.size); + // } +} + +fn readTensor( + allocator: std.mem.Allocator, + io: std.Io, + vfs: *zml.io.VFS, + registry: *zml.safetensors.TensorRegistry, + tensor_name: []const u8, + err_ptr: *?anyerror, +) void { + readTensorImpl(allocator, io, vfs, registry, tensor_name) catch |err| { + err_ptr.* = err; + log.err("Failed to read tensor {s}: {any}", .{ tensor_name, err }); + }; +} + +pub fn readTensorImpl( + allocator: std.mem.Allocator, + io: std.Io, + vfs: *zml.io.VFS, + registry: *zml.safetensors.TensorRegistry, + tensor_name: []const u8, +) !void { + var total_timer = try std.time.Timer.start(); + + const tensor = registry.tensors.get(tensor_name) orelse return error.TensorNotFound; + + const buff = try allocator.alloc(u8, tensor.byteSize() * 2); + defer allocator.free(buff); + + var tensor_reader = try registry.reader(io, vfs, tensor_name, buff[0..tensor.byteSize()]); + defer tensor_reader.deinit(); + + var writer: std.Io.Writer = .fixed(buff[tensor.byteSize()..]); + + var read_timer = try std.time.Timer.start(); + const read = try tensor_reader.interface.streamRemaining(&writer); + + const read_time = read_timer.read(); + const total_time = total_timer.read(); + const read_mb = @as(f64, @floatFromInt(read)) / (1024.0 * 1024.0); + const read_time_s = @as(f64, @floatFromInt(read_time)) / @as(f64, std.time.ns_per_s); + const throughput = if (read_time_s > 0) read_mb / read_time_s else 0; + + log.info("Tensor {s} read {d}/{d} bytes at offset {d} | read={d}ms total={d}ms | {d:.2} MB/s", .{ + tensor.name, + read, + tensor.byteSize(), + tensor.offset, + read_time / std.time.ns_per_ms, + total_time / std.time.ns_per_ms, + throughput, + }); } fn readData( diff --git a/zml2/safetensors.zig b/zml2/safetensors.zig new file mode 100644 index 00000000..dc09c631 --- /dev/null +++ b/zml2/safetensors.zig @@ -0,0 +1,864 @@ +const std = @import("std"); + +const Shape = @import("shape.zig").Shape; +const DataType = @import("dtype.zig").DataType; +const VFS = @import("io").VFS; + +const Dims = Shape.DimsArray; +const StringBuilder = std.ArrayListUnmanaged(u8); + +pub const Tensors = std.StringArrayHashMapUnmanaged(Tensor); +pub const Metadatas = std.StringArrayHashMapUnmanaged(Metadata); + +const log = std.log.scoped(.@"zml/safetensors"); + +const BYTES_HEADER = 8; + +pub fn parseFromPath( + allocator: std.mem.Allocator, + io: std.Io, + vfs: *VFS, // todo: remove pointer when std.Io.Dir/File ready + path: []const u8, +) !TensorRegistry { + const file_path = try resolveModelPathEntrypoint(allocator, io, vfs, path); + defer allocator.free(file_path); + log.debug("Resolved model path entrypoint for '{s}': {s}", .{ path, file_path }); + + const repo_path = try resolveModelRepoPath(allocator, io, vfs, path); + defer allocator.free(repo_path); + log.info("Resolved model repo path for '{s}': {s}", .{ path, repo_path }); + + const repo = try vfs.openAbsoluteDir(io, repo_path, .{}); + defer repo.close(io); + + const file_type = resolveFiletype(file_path); + log.info("Resolved file type for '{s}': {any}", .{ path, file_type }); + + return switch (file_type) { + .index => blk: { + const index_file = try vfs.openAbsoluteFile(io, file_path, .{}); + defer index_file.close(io); + + const file_reader_buf = try allocator.alloc(u8, 64 * 1024); + defer allocator.free(file_reader_buf); + + var index_reader = index_file.reader(io, file_reader_buf); + + var safetensors_index = try parseSafetensorsIndex( + allocator, + &index_reader.interface, + ); + defer safetensors_index.deinit(); + + var registry: TensorRegistry = try .initWithMetadata(allocator, safetensors_index.metadata); + + try parseSafetensorsIndexFiles( + allocator, + io, + safetensors_index, + ®istry, + repo, + repo_path, + ); + + break :blk registry; + }, + .safetensors => blk: { + var registry: TensorRegistry = .init(allocator); + + log.info("Parsing single safetensors file '{s}'", .{file_path}); + const file = try repo.openFile(io, file_path, .{}); + defer file.close(io); + + log.info("Opened safetensors file '{s}'", .{file_path}); + + const header_buf = try allocator.alloc(u8, 16 * 1024); + defer allocator.free(header_buf); + + var reader = file.reader(io, header_buf); + + try parseSafetensors(allocator, file_path, &reader.interface, ®istry); + + break :blk registry; + }, + else => return error.InvalidPath, + }; +} + +pub const FileType = enum { + index, + safetensors, + unknown, +}; + +pub fn resolveFiletype(path: []const u8) FileType { + if (std.mem.endsWith(u8, path, ".safetensors.index.json")) { + return .index; + } else if (std.mem.endsWith(u8, path, ".safetensors")) { + return .safetensors; + } else { + return .unknown; + } +} + +pub const ModelPathResolutionError = error{ + FileNotFound, + InvalidPath, +} || std.mem.Allocator.Error; + +pub fn resolveModelRepoPath(allocator: std.mem.Allocator, io: std.Io, vfs: *VFS, path: []const u8) ModelPathResolutionError![]const u8 { + const resolved_path = if (std.mem.startsWith(u8, path, "/")) + try std.fmt.allocPrint(allocator, "file://{s}", .{path}) + else + try allocator.dupe(u8, path); + defer allocator.free(resolved_path); + + var supported_schemes_it = vfs.backends.keyIterator(); + + while (supported_schemes_it.next()) |scheme| { + if (std.mem.startsWith(u8, resolved_path, scheme.*)) { + break; + } + } else { + log.err("Unsupported URI scheme in path: {s}", .{resolved_path}); + return ModelPathResolutionError.InvalidPath; + } + + if (std.mem.endsWith(u8, resolved_path, ".safetensors.index.json") or + std.mem.endsWith(u8, resolved_path, ".safetensors")) + { + const dir_path = std.fs.path.dirname(resolved_path) orelse return ModelPathResolutionError.InvalidPath; + return try allocator.dupe(u8, dir_path); + } + + if (vfs.openAbsoluteDir(io, resolved_path, .{})) |dir| { + defer dir.close(io); + return try allocator.dupe(u8, resolved_path); + } else |_| {} + + return ModelPathResolutionError.FileNotFound; +} + +pub fn resolveModelPathEntrypoint(allocator: std.mem.Allocator, io: std.Io, vfs: *VFS, path: []const u8) ModelPathResolutionError![]const u8 { + const resolved_path = if (std.mem.startsWith(u8, path, "/")) + try std.fmt.allocPrint(allocator, "file://{s}", .{path}) + else + try allocator.dupe(u8, path); + defer allocator.free(resolved_path); + + var supported_schemes_it = vfs.backends.keyIterator(); + + while (supported_schemes_it.next()) |scheme| { + if (std.mem.startsWith(u8, resolved_path, scheme.*)) { + break; + } + } else { + log.err("Unsupported URI scheme in path: {s}", .{resolved_path}); + return ModelPathResolutionError.InvalidPath; + } + + if (std.mem.endsWith(u8, resolved_path, ".safetensors.index.json") or + std.mem.endsWith(u8, resolved_path, ".safetensors")) + { + if (vfs.openAbsoluteFile(io, resolved_path, .{})) |file| { + defer file.close(io); + if (file.stat(io)) |_| { + return try allocator.dupe(u8, resolved_path); + } else |_| {} + } else |_| {} + return ModelPathResolutionError.FileNotFound; + } + + const index_path = try std.fs.path.join(allocator, &[_][]const u8{ resolved_path, "model.safetensors.index.json" }); + + if (vfs.openAbsoluteFile(io, index_path, .{})) |index_file| { + defer index_file.close(io); + if (index_file.stat(io)) |_| { + return index_path; + } else |_| {} + } else |_| {} + + allocator.free(index_path); + + const model_path = try std.fs.path.join(allocator, &[_][]const u8{ resolved_path, "model.safetensors" }); + + if (vfs.openAbsoluteFile(io, model_path, .{})) |model_file| { + defer model_file.close(io); + if (model_file.stat(io)) |_| { + return model_path; + } else |_| {} + } else |_| {} + + allocator.free(model_path); + + return ModelPathResolutionError.FileNotFound; +} + +pub const TensorReader = struct { + file: std.Io.File, + file_reader: std.Io.File.Reader, + remaining: u64, + io: std.Io, + interface: std.Io.Reader, + + pub const Error = error{TensorNotFound} || std.Io.File.OpenError || std.Io.File.Reader.SeekError || std.mem.Allocator.Error; + + pub fn init( + io: std.Io, + vfs: *VFS, + tensor: Tensor, + buffer: []u8, + ) Error!TensorReader { + const file = try vfs.openAbsoluteFile(io, tensor.file_uri, .{}); + errdefer file.close(io); + + var file_reader = file.reader(io, buffer); + try file_reader.seekTo(tensor.offset); + + return .{ + .file = file, + .file_reader = file_reader, + .remaining = tensor.byteSize(), + .io = io, + .interface = .{ + .vtable = &.{ + .stream = stream, + .discard = discard, + }, + .buffer = buffer, + .seek = 0, + .end = 0, + }, + }; + } + + pub fn deinit(self: *TensorReader) void { + self.file.close(self.io); + } + + fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { + const self: *TensorReader = @fieldParentPtr("interface", r); + if (self.remaining == 0) return error.EndOfStream; + + const combined_limit = limit.min(.limited64(self.remaining)); + const n = try self.file_reader.interface.stream(w, combined_limit); + self.remaining -= n; + return n; + } + + fn discard(r: *std.Io.Reader, limit: std.Io.Limit) std.Io.Reader.Error!usize { + const self: *TensorReader = @fieldParentPtr("interface", r); + if (self.remaining == 0) return error.EndOfStream; + + const combined_limit = limit.min(.limited64(self.remaining)); + const n = try self.file_reader.interface.discard(combined_limit); + self.remaining -= n; + return n; + } +}; + +pub const Tensor = struct { + file_uri: []const u8, + name: []const u8, + shape: Shape, + offset: u64, + + pub fn byteSize(self: Tensor) u64 { + return self.shape.byteSize(); + } + + pub fn format(self: Tensor, writer: *std.Io.Writer) !void { + try writer.print("Tensor(name={s} shape={f} size={d}, offset={d}, file_uri={s})", .{ + self.name, + self.shape, + self.byteSize(), + self.offset, + self.file_uri, + }); + } +}; + +pub const TensorRegistry = struct { + arena: std.heap.ArenaAllocator, + + tensors: Tensors, + metadata: Metadatas, + + mutex: std.Thread.Mutex, + + pub fn init(allocator: std.mem.Allocator) TensorRegistry { + return .{ + .arena = std.heap.ArenaAllocator.init(allocator), + .tensors = .{}, + .metadata = .{}, + .mutex = .{}, + }; + } + + pub fn initWithMetadata( + allocator: std.mem.Allocator, + metadata: Metadatas, + ) !TensorRegistry { + var self: TensorRegistry = .{ + .arena = std.heap.ArenaAllocator.init(allocator), + .tensors = .{}, + .metadata = .empty, + .mutex = .{}, + }; + + try self.mergeMetadata(metadata); + + return self; + } + + pub fn deinit(self: *TensorRegistry) void { + const allocator = self.arena.allocator(); + self.tensors.deinit(allocator); + self.metadata.deinit(allocator); + self.arena.deinit(); + } + + pub fn mergeMetadata( + self: *TensorRegistry, + other: Metadatas, + ) !void { + self.mutex.lock(); + defer self.mutex.unlock(); + + const allocator = self.arena.allocator(); + + var it = other.iterator(); + while (it.next()) |entry| { + const key = try allocator.dupe(u8, entry.key_ptr.*); + const value = try entry.value_ptr.*.clone(allocator); + + const gop = try self.metadata.getOrPut(allocator, key); + if (gop.found_existing) { + gop.value_ptr.*.deinit(allocator); + gop.value_ptr.* = value; + log.warn("Overwrote existing metadata key={s} with value={f}", .{ key, value }); + } else { + gop.value_ptr.* = value; + log.debug("Added new metadata key={s} with value={f}", .{ key, value }); + } + } + } + + pub fn registerTensor( + self: *TensorRegistry, + tensor: Tensor, + ) !void { + const allocator = self.arena.allocator(); + + self.mutex.lock(); + defer self.mutex.unlock(); + + var tensor_copy = tensor; + + tensor_copy.name = try allocator.dupe(u8, tensor.name); + tensor_copy.file_uri = try allocator.dupe(u8, tensor.file_uri); + + try self.tensors.put(allocator, tensor_copy.name, tensor_copy); + } + + pub fn reader( + self: *TensorRegistry, + io: std.Io, + vfs: *VFS, + tensor_name: []const u8, + buffer: []u8, + ) TensorReader.Error!TensorReader { + const tensor = self.tensors.get(tensor_name) orelse { + log.err("Tensor {s} not found in registry", .{tensor_name}); + return TensorReader.Error.TensorNotFound; + }; + + return try .init(io, vfs, tensor, buffer); + } + + pub fn iterator(self: *TensorRegistry) Tensors.Iterator { + return self.tensors.iterator(); + } + + pub fn totalBytes(self: *TensorRegistry) u64 { + var total: u64 = 0; + + var it = self.tensors.iterator(); + while (it.next()) |entry| { + total += entry.value_ptr.byteSize(); + } + + return total; + } +}; + +pub fn parseSafetensors( + allocator: std.mem.Allocator, + file_uri: []const u8, + reader: *std.Io.Reader, + registry: *TensorRegistry, +) !void { + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + const json_header_length: u64 = try reader.takeInt(u64, .little); + + const data_start_offset = BYTES_HEADER + json_header_length; + + const json_data = try arena.allocator().alloc(u8, @intCast(json_header_length)); + + try reader.readSliceAll(json_data); + + const headers: std.json.Value = try std.json.parseFromSliceLeaky( + std.json.Value, + arena.allocator(), + json_data, + .{}, + ); + + var it = headers.object.iterator(); + while (it.next()) |entry| { + const key = entry.key_ptr.*; + const value = entry.value_ptr.*; + + if (std.mem.eql(u8, key, "__metadata__")) { + // Parse metadata into a local arena to avoid concurrent allocations + // against registry.arena (which is not thread-safe). + var local_arena = std.heap.ArenaAllocator.init(allocator); + defer local_arena.deinit(); + + const metas = try parseMetadata(local_arena.allocator(), value); + try registry.mergeMetadata(metas); + + continue; + } + + const shape_field = value.object.get("shape").?.array; + + if (shape_field.items.len > Shape.MAX_RANK) { + log.warn("Can't load tensor {s}, too many dims: {}", .{ key, shape_field.items.len }); + return error.TooManyDimensions; + } + + const offset_field = value.object.get("data_offsets").?; + const start: u64 = @intCast(offset_field.array.items[0].integer); + const dtype = try stringToDtype(value.object.get("dtype").?.string); + + var dims: Dims = .{}; + for (shape_field.items) |d| { + dims.appendAssumeCapacity(d.integer); + } + + const tensor: Tensor = .{ + .file_uri = file_uri, + .name = key, + .shape = .init(dims.slice(), dtype), + .offset = data_start_offset + start, + }; + + try registry.registerTensor(tensor); + } +} + +pub const SafetensorsIndex = struct { + pub const Map = std.StringArrayHashMapUnmanaged(std.ArrayList([]const u8)); + + arena: std.heap.ArenaAllocator, + + map: Map, + metadata: Metadatas, + + pub fn init(allocator: std.mem.Allocator) SafetensorsIndex { + return .{ + .arena = std.heap.ArenaAllocator.init(allocator), + .map = .empty, + .metadata = .empty, + }; + } + + pub fn iterator(self: *SafetensorsIndex) Map.Iterator { + return self.map.iterator(); + } + + pub fn deinit(self: *SafetensorsIndex) void { + const allocator = self.arena.allocator(); + self.map.deinit(allocator); + self.metadata.deinit(allocator); + self.arena.deinit(); + } +}; + +pub fn parseSafetensorsIndex( + allocator: std.mem.Allocator, + reader: *std.Io.Reader, +) !SafetensorsIndex { + var safetensors_index: SafetensorsIndex = .init(allocator); + errdefer safetensors_index.deinit(); + + const arena_allocator = safetensors_index.arena.allocator(); + var json_reader: std.json.Reader = .init(arena_allocator, reader); + + const index = try std.json.parseFromTokenSourceLeaky( + std.json.Value, + arena_allocator, + &json_reader, + .{ .allocate = .alloc_if_needed }, + ); + + const weight_map = index.object.get("weight_map"); + + if (weight_map) |wm| { + const num_tensors = wm.object.count(); + log.debug("Found {} tensors in weight_map", .{num_tensors}); + + var it = wm.object.iterator(); + + while (it.next()) |entry| { + const weight_name = entry.key_ptr.*; + const filename = entry.value_ptr.string; + + const map_entry = try safetensors_index.map.getOrPut(arena_allocator, filename); + + if (!map_entry.found_existing) { + map_entry.value_ptr.* = .{}; + } + + try map_entry.value_ptr.append(arena_allocator, try arena_allocator.dupe(u8, weight_name)); + } + } else { + log.warn("No weight_map attribute found in index", .{}); + } + + if (index.object.get("metadata")) |metadata_val| { + safetensors_index.metadata = try parseMetadata(arena_allocator, metadata_val); + } + + return safetensors_index; +} + +pub fn parseSafetensorsIndexFiles( + allocator: std.mem.Allocator, + io: std.Io, + safetensors_index: SafetensorsIndex, + registry: *TensorRegistry, + repo: std.Io.Dir, + repo_path: []const u8, +) !void { + var group: std.Io.Group = .init; + defer group.cancel(io); + + var err: ?anyerror = null; + + const filenames = safetensors_index.map.keys(); + + for (filenames) |filename| { + group.async(io, AsyncParseSafetensorsIndexFile.run, .{ + allocator, + io, + registry, + repo, + repo_path, + filename, + &err, + }); + } + + group.wait(io); + + if (err) |e| { + log.err("Error parsing safetensors index files: {any}", .{e}); + return e; + } +} + +const AsyncParseSafetensorsIndexFile = struct { + pub fn run( + allocator: std.mem.Allocator, + io: std.Io, + registry: *TensorRegistry, + repo: std.Io.Dir, + repo_path: []const u8, // todo: Dir.realPath when ready + filename: []const u8, + err_ptr: *?anyerror, + ) void { + parse(allocator, io, registry, repo, repo_path, filename) catch |err| { + err_ptr.* = err; + }; + } + + fn parse( + allocator: std.mem.Allocator, + io: std.Io, + registry: *TensorRegistry, + repo: std.Io.Dir, + repo_path: []const u8, // todo: Dir.realPath when ready + filename: []const u8, + ) !void { + const file_uri = try std.fs.path.join(allocator, &.{ repo_path, filename }); + defer allocator.free(file_uri); + + const file = try repo.openFile(io, filename, .{}); + defer file.close(io); + + const header_buf = try allocator.alloc(u8, 64 * 1024); + defer allocator.free(header_buf); + + var reader = file.reader(io, header_buf); + + try parseSafetensors(allocator, file_uri, &reader.interface, registry); + } +}; + +pub const Metadata = union(enum) { + null: void, + int: i64, + float: f64, + bool: bool, + string: []const u8, + + array_bool: []const bool, + array_int: []const i64, + array_float: []const f64, + array_string: []const []const u8, + + pub const ItemType = enum { + int, + float, + bool, + string, + + pub fn toZigType(comptime kind: ItemType) type { + return switch (kind) { + .int => i64, + .float => f64, + .bool => bool, + .string => []const u8, + }; + } + }; + + pub fn wrap(x: anytype) Metadata { + return switch (@TypeOf(x)) { + inline u8, i8, u16, i16, u32, i32, u64, i64 => .{ .int = @intCast(x) }, + inline f16, f32, f64 => .{ .float = @floatCast(x) }, + bool => .{ .bool = x }, + []const u8 => .{ .string = x }, + else => @panic("Unsupported type for Value: " ++ @typeName(@TypeOf(x))), + }; + } + + pub fn copySlice(allocator: std.mem.Allocator, any_slice: anytype) !Metadata { + return switch (@TypeOf(any_slice[0])) { + inline u8, i8, u16, i16, u32, i32, u64, i64 => { + const res = try allocator.alloc(i64, any_slice.len); + for (res, any_slice) |*r, val| r.* = @intCast(val); + return .{ .array_int = res }; + }, + inline f16, f32, f64 => { + const res = try allocator.alloc(f64, any_slice.len); + for (res, any_slice) |*r, val| r.* = @floatCast(val); + return .{ .array_float = res }; + }, + bool => .{ .array_bool = try allocator.dupe(bool, any_slice) }, + []const u8 => .{ .array_string = try allocator.dupe([]const u8, @alignCast(any_slice)) }, + else => @panic("Unsupported type for Value: " ++ @typeName(@TypeOf(any_slice))), + }; + } + + pub fn format( + self: Metadata, + writer: *std.Io.Writer, + ) !void { + switch (self) { + .null => _ = try writer.write("null"), + .string => |s| try writer.print("{s}", .{s}), + .bool => |b| try writer.print("{}", .{b}), + .int => |i| try writer.print("{d}", .{i}), + .float => |f| try writer.print("{d}", .{f}), + .array_bool => |arr| { + try writer.writeByte('['); + for (arr, 0..) |v, i| { + if (i > 0) try writer.writeAll(", "); + try writer.print("{}", .{v}); + } + try writer.writeByte(']'); + }, + .array_int => |arr| { + try writer.writeByte('['); + for (arr, 0..) |v, i| { + if (i > 0) try writer.writeAll(", "); + try writer.print("{d}", .{v}); + } + try writer.writeByte(']'); + }, + .array_float => |arr| { + try writer.writeByte('['); + for (arr, 0..) |v, i| { + if (i > 0) try writer.writeAll(", "); + try writer.print("{d}", .{v}); + } + try writer.writeByte(']'); + }, + .array_string => |arr| { + try writer.writeByte('['); + for (arr, 0..) |v, i| { + if (i > 0) try writer.writeAll(", "); + try writer.print("\"{s}\"", .{v}); + } + try writer.writeByte(']'); + }, + } + } + + pub fn deinit(self: Metadata, allocator: std.mem.Allocator) void { + switch (self) { + .string => |s| allocator.free(s), + .array_bool => |s| allocator.free(s), + .array_int => |s| allocator.free(s), + .array_float => |s| allocator.free(s), + .array_string => |s| { + for (s) |str| allocator.free(str); + allocator.free(s); + }, + else => {}, + } + } + + pub fn clone(self: Metadata, allocator: std.mem.Allocator) !Metadata { + return switch (self) { + .null => .null, + .int => |v| .{ .int = v }, + .float => |v| .{ .float = v }, + .bool => |v| .{ .bool = v }, + .string => |s| .{ .string = try allocator.dupe(u8, s) }, + .array_bool => |s| .{ .array_bool = try allocator.dupe(bool, s) }, + .array_int => |s| .{ .array_int = try allocator.dupe(i64, s) }, + .array_float => |s| .{ .array_float = try allocator.dupe(f64, s) }, + .array_string => |s| blk: { + const new_slice = try allocator.alloc([]const u8, s.len); + errdefer allocator.free(new_slice); + for (s, 0..) |str, i| { + new_slice[i] = try allocator.dupe(u8, str); + } + break :blk .{ .array_string = new_slice }; + }, + }; + } +}; + +pub fn stringToDtype(safetensor_type: []const u8) !DataType { + const map = std.StaticStringMap(DataType).initComptime(.{ + .{ "F64", .f64 }, + .{ "F32", .f32 }, + .{ "F16", .f16 }, + .{ "BF16", .bf16 }, + .{ "F8_E4M3", .f8e4m3fn }, + .{ "I64", .i64 }, + .{ "I32", .i32 }, + .{ "I16", .i16 }, + .{ "I8", .i8 }, + .{ "U64", .u64 }, + .{ "U32", .u32 }, + .{ "U16", .u16 }, + .{ "U8", .u8 }, + .{ "BOOL", .bool }, + }); + + return map.get(safetensor_type) orelse { + log.err("Unsupported safetensor data type: {s}", .{safetensor_type}); + return error.UnsupportedDataType; + }; +} + +fn parseMetadata(allocator: std.mem.Allocator, val: std.json.Value) !Metadatas { + var metadatas: Metadatas = .{}; + var prefix_buf: [1024]u8 = undefined; + var prefix = StringBuilder.initBuffer(&prefix_buf); + + try populateMetadata(allocator, &prefix, val, &metadatas); + + return metadatas; +} + +fn populateMetadata(allocator: std.mem.Allocator, prefix: *StringBuilder, val: std.json.Value, metadatas: *Metadatas) !void { + const key = prefix.items; + return switch (val) { + .null => try metadatas.put(allocator, try allocator.dupe(u8, key), .null), + .bool => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .bool = v }), + .integer => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .int = v }), + .float => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .float = v }), + .number_string, .string => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .string = try allocator.dupe(u8, v) }), + .array => |v| { + if (v.items.len == 0) return; + if (validSlice(v)) |item_type| { + const data: Metadata = switch (item_type) { + .bool => blk: { + const values = try allocator.alloc(bool, v.items.len); + for (v.items, 0..) |item, i| values[i] = item.bool; + break :blk .{ .array_bool = values }; + }, + .integer => blk: { + const values = try allocator.alloc(i64, v.items.len); + for (v.items, 0..) |item, i| values[i] = item.integer; + break :blk .{ .array_int = values }; + }, + .float => blk: { + const values = try allocator.alloc(f64, v.items.len); + for (v.items, 0..) |item, i| values[i] = item.float; + break :blk .{ .array_float = values }; + }, + inline .string, .number_string => |tag| blk: { + const values = try allocator.alloc([]const u8, v.items.len); + for (v.items, 0..) |item, i| { + values[i] = try allocator.dupe(u8, @field(item, @tagName(tag))); + } + break :blk .{ .array_string = values }; + }, + .null, .array, .object => unreachable, + }; + try metadatas.put(allocator, try allocator.dupe(u8, key), data); + } else { + for (v.items, 0..) |item, i| { + const old_len = prefix.items.len; + if (prefix.items.len > 0) { + prefix.appendAssumeCapacity('.'); + } + prefix.items.len += std.fmt.printInt(prefix.unusedCapacitySlice(), i, 10, .lower, .{}); + try populateMetadata(allocator, prefix, item, metadatas); + prefix.items.len = old_len; + } + } + }, + .object => |v| { + var obj_iter = v.iterator(); + while (obj_iter.next()) |entry| { + const old_len = prefix.items.len; + if (prefix.items.len > 0) { + prefix.appendAssumeCapacity('.'); + } + prefix.appendSliceAssumeCapacity(entry.key_ptr.*); + try populateMetadata(allocator, prefix, entry.value_ptr.*, metadatas); + prefix.items.len = old_len; + } + }, + }; +} + +/// We can only create a Zig slice out of json array, if all values +/// in the array have the same type. +fn validSlice(v: std.json.Array) ?std.meta.Tag(std.json.Value) { + if (v.items.len == 0) return null; + + const item_type: std.meta.Tag(std.json.Value) = v.items[0]; + switch (item_type) { + .null, .array, .object => return null, + else => {}, + } + + for (v.items[1..]) |item| { + if (item != item_type) + return null; + } + + return item_type; +} diff --git a/zml2/zml.zig b/zml2/zml.zig index ed8f8ddc..f6b6e6a7 100644 --- a/zml2/zml.zig +++ b/zml2/zml.zig @@ -28,6 +28,7 @@ pub const Target = platform.Target; pub const CompilationOptions = platform.CompilationOptions; pub const shape = @import("shape.zig"); pub const Shape = shape.Shape; +pub const safetensors = @import("safetensors.zig"); pub const slice = @import("slice.zig"); pub const Slice = slice.Slice; pub const ConstSlice = slice.ConstSlice; From ed1e69057821e3d0e7055fd79b5affa03594523e Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 15 Dec 2025 16:10:37 +0100 Subject: [PATCH 139/881] Small fix in stdx --- stdx/time.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdx/time.zig b/stdx/time.zig index 48d38fdf..4d2fde10 100644 --- a/stdx/time.zig +++ b/stdx/time.zig @@ -11,7 +11,7 @@ pub const Duration = struct { return (1 * std.time.ns_per_s) / self.ns; } - pub fn formatDuration(duration: Duration, writer: *std.io.Writer) std.io.Writer.Error!void { + pub fn formatDuration(duration: Duration, writer: *std.Io.Writer) std.Io.Writer.Error!void { try writer.printDuration(duration.ns, .{}); } From 6df392bbe2651844946855425dac0f727eedf7af Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 15 Dec 2025 16:10:50 +0100 Subject: [PATCH 140/881] First port of Mnist to new API Everything is done by hand so that's not the final version, but at least it works! --- examples/mnist/BUILD.bazel | 8 +- examples/mnist/mnist.zig | 199 ++++++++++++++++++++++++++++--------- 2 files changed, 157 insertions(+), 50 deletions(-) diff --git a/examples/mnist/BUILD.bazel b/examples/mnist/BUILD.bazel index 11ac0c66..f76bfc48 100644 --- a/examples/mnist/BUILD.bazel +++ b/examples/mnist/BUILD.bazel @@ -4,8 +4,8 @@ load("@rules_zig//zig:defs.bzl", "zig_binary") zig_binary( name = "mnist", args = [ - "$(location @mnist//:mnist.safetensors)", - "$(location @mnist//:t10k-images.idx3-ubyte)", + #"$(location @mnist//:mnist.safetensors)", + #"$(location @mnist//:t10k-images.idx3-ubyte)", ], data = [ "@mnist//:mnist.safetensors", @@ -13,8 +13,8 @@ zig_binary( ], main = "mnist.zig", deps = [ - "//async", - "//zml", + "//stdx", + "//zml2:zml", ], ) diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 4130bf7a..2c0c79d0 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -1,13 +1,12 @@ const std = @import("std"); +const stdx = @import("stdx"); -const async = @import("async"); const zml = @import("zml"); const log = std.log.scoped(.mnist); pub const std_options: std.Options = .{ .log_level = .info, - .logFn = async.logFn(std.log.defaultLog), }; /// Model definition @@ -20,14 +19,108 @@ const Mnist = struct { bias: zml.Tensor, pub fn forward(self: Layer, input: zml.Tensor) zml.Tensor { - return self.weight.matmul(input).add(self.bias).relu(); + return self.weight.dot(input, .d).add(self.bias).relu().withTags(.{.d}); } }; + pub fn init(registry: *const zml.safetensors.TensorRegistry) Mnist { + const fc1_weight = b: { + const desc = registry.tensors.get("fc1.weight").?; + break :b zml.Tensor.fromShape(desc.shape).withTags(.{ .d_out, .d }); + }; + const fc1_bias: zml.Tensor = b: { + const desc = registry.tensors.get("fc1.bias").?; + break :b zml.Tensor.fromShape(desc.shape).withTags(.{.d_out}); + }; + const fc2_weight = b: { + const desc = registry.tensors.get("fc2.weight").?; + break :b zml.Tensor.fromShape(desc.shape).withTags(.{ .d_out, .d }); + }; + const fc2_bias: zml.Tensor = b: { + const desc = registry.tensors.get("fc2.bias").?; + break :b zml.Tensor.fromShape(desc.shape).withTags(.{.d_out}); + }; + return .{ + .fc1 = .{ .weight = fc1_weight, .bias = fc1_bias }, + .fc2 = .{ .weight = fc2_weight, .bias = fc2_bias }, + }; + } + + pub fn deinit(self: *const Mnist) void { + _ = self; // autofix + } + + pub fn loadBuffers(self: Mnist, allocator: std.mem.Allocator, io: std.Io, vfs: *zml.io.VFS, registry: *zml.safetensors.TensorRegistry, platform: zml.Platform) !zml.Bufferized(Mnist) { + _ = self; // autofix + const read_buffer = try allocator.alloc(u8, 4096); + defer allocator.free(read_buffer); + + const fc1_weight: zml.Buffer = b: { + const desc = registry.tensors.get("fc1.weight").?; + const slice = try zml.Slice.alloc(allocator, desc.shape); + defer slice.free(allocator); + + var writer = std.Io.Writer.fixed(slice.data); + var reader = try registry.reader(io, vfs, "fc1.weight", read_buffer); + + _ = try reader.interface.streamRemaining(&writer); + break :b try .fromSlice(io, platform, slice); + }; + errdefer fc1_weight.deinit(); + const fc1_bias: zml.Buffer = b: { + const desc = registry.tensors.get("fc1.bias").?; + const slice = try zml.Slice.alloc(allocator, desc.shape); + defer slice.free(allocator); + + var writer = std.Io.Writer.fixed(slice.data); + var reader = try registry.reader(io, vfs, "fc1.bias", read_buffer); + + _ = try reader.interface.streamRemaining(&writer); + break :b try .fromSlice(io, platform, slice); + }; + errdefer fc1_bias.deinit(); + const fc2_weight: zml.Buffer = b: { + const desc = registry.tensors.get("fc2.weight").?; + const slice = try zml.Slice.alloc(allocator, desc.shape); + defer slice.free(allocator); + + var writer = std.Io.Writer.fixed(slice.data); + var reader = try registry.reader(io, vfs, "fc2.weight", read_buffer); + + _ = try reader.interface.streamRemaining(&writer); + break :b try .fromSlice(io, platform, slice); + }; + errdefer fc2_weight.deinit(); + const fc2_bias: zml.Buffer = b: { + const desc = registry.tensors.get("fc2.bias").?; + const slice = try zml.Slice.alloc(allocator, desc.shape); + defer slice.free(allocator); + + var writer = std.Io.Writer.fixed(slice.data); + var reader = try registry.reader(io, vfs, "fc2.bias", read_buffer); + + _ = try reader.interface.streamRemaining(&writer); + break :b try .fromSlice(io, platform, slice); + }; + errdefer fc2_bias.deinit(); + + return .{ + .fc1 = .{ .weight = fc1_weight, .bias = fc1_bias }, + .fc2 = .{ .weight = fc2_weight, .bias = fc2_bias }, + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Mnist)) void { + self.fc1.weight.deinit(); + self.fc1.bias.deinit(); + self.fc2.weight.deinit(); + self.fc2.bias.deinit(); + } + /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { // std.log.info("Compiling for target: {s}", .{@tagName(input.getContext().target())}); - var x = input.flatten().convert(.f32); + var x = input.flatten().convert(.f32).withTags(.{.d}); const layers: []const Layer = &.{ self.fc1, self.fc2 }; for (layers) |layer| { x = layer.forward(x); @@ -37,75 +130,89 @@ const Mnist = struct { }; pub fn main() !void { - try async.AsyncThread.main(std.heap.c_allocator, asyncMain); -} + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); -pub fn asyncMain() !void { - const allocator = std.heap.c_allocator; + const allocator = gpa.allocator(); - // // Create ZML context - var context = try zml.Context.init(); - defer context.deinit(); + var threaded: std.Io.Threaded = .init(allocator); + defer threaded.deinit(); - // log.info("\n===========================\n== ZML MNIST Example ==\n===========================\n\n", .{}); + var vfs_file: zml.io.VFS.File = .init(threaded.io()); - // // Auto-select platform - const platform = context.autoPlatform(.{}); - context.printAvailablePlatforms(platform); + var vfs: zml.io.VFS = .init(allocator, threaded.io()); + defer vfs.deinit(); + + try vfs.register("file", vfs_file.io()); + + const io = vfs.io(); + + zml.init(); + defer zml.deinit(); + + // Auto-select platform + const platform: zml.Platform = try .auto(threaded.io(), .{}); // Parse program args const process_args = try std.process.argsAlloc(allocator); defer std.process.argsFree(allocator, process_args); - const pt_model = process_args[1]; + const model_path = process_args[1]; const t10kfilename = process_args[2]; - // Memory arena dedicated to model shapes and weights - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - // Read model shapes. - // Note this works because Mnist struct uses the same layer names as the pytorch model - var buffer_store = try zml.aio.detectFormatAndOpen(allocator, pt_model); - defer buffer_store.deinit(); + var registry = try zml.safetensors.parseFromPath(allocator, io, &vfs, model_path); + defer registry.deinit(); - const mnist_model = try zml.aio.populateModel(Mnist, allocator, buffer_store); - log.info("Reading model shapes from PyTorch file {s}...", .{pt_model}); + // Init model + const mnist_model = Mnist.init(®istry); + defer mnist_model.deinit(); - // Start compiling + // Compile model log.info("Compiling model to MLIR....", .{}); - var start_time = try std.time.Timer.start(); - var compilation = try async.async(zml.compile, .{ allocator, Mnist.forward, .{}, .{zml.Shape.init(.{ 28, 28 }, .u8)}, buffer_store, platform }); + var timer = try stdx.time.Timer.start(); + const input: zml.Tensor = .init(.{ 28, 28 }, .u8); + var exe = try platform.compileModel(allocator, io, Mnist.forward, mnist_model, .{input}); + defer exe.deinit(); - // While compiling, start loading weights on the platform - var model_weights = try zml.aio.loadModelBuffers(Mnist, mnist_model, buffer_store, arena, platform); - defer zml.aio.unloadBuffers(&model_weights); + log.info("✅ Compiled model in {f}", .{timer.read()}); - // Wait for end of compilation and end of weights loading. - const compiled_mnist = try compilation.await(); - log.info("✅ Compiled model in {d}ms", .{start_time.read() / std.time.ns_per_ms}); + var args = try exe.args(allocator); + defer args.deinit(allocator); - const mnist = compiled_mnist.prepare(model_weights); - defer mnist.deinit(); - log.info("✅ Weights transferred in {d}ms", .{start_time.read() / std.time.ns_per_ms}); + var results = try exe.results(allocator); + defer results.deinit(allocator); + + // Load buffers + var mnist_buffers = try mnist_model.loadBuffers(allocator, io, &vfs, ®istry, platform); + defer Mnist.unloadBuffers(&mnist_buffers); + log.info("✅ Weights transferred in {f}", .{timer.read()}); log.info("Starting inference...", .{}); // Load a random digit image from the dataset. - const dataset = try async.File.open(t10kfilename, .{ .mode = .read_only }); - defer dataset.close() catch unreachable; - var rng = std.Random.Xoshiro256.init(@intCast(std.time.timestamp())); + const dataset = try vfs.openAbsoluteFile(io, t10kfilename, .{ .mode = .read_only }); + defer dataset.close(io); + + const now = std.Io.Clock.now(.awake, io) catch unreachable; + var rng = std.Random.Xoshiro256.init(@intCast(now.toMilliseconds())); // inference - can be looped { const idx = rng.random().intRangeAtMost(u64, 0, 10000 - 1); var sample: [28 * 28]u8 align(16) = undefined; - _ = try dataset.pread(&sample, 16 + (idx * 28 * 28)); - var input = try zml.Buffer.from(platform, zml.HostBuffer.fromBytes(zml.Shape.init(.{ 28, 28 }, .u8), &sample), .{}); - defer input.deinit(); + var reader = dataset.reader(io, &.{}); + try reader.seekTo(16 + (idx * 28 * 28)); + _ = try reader.interface.readSliceShort(&sample); + + var input_buffer: zml.Buffer = try .fromSlice(io, platform, zml.Slice.init(input.shape(), &sample)); + defer input_buffer.deinit(); + + args.set(.{ mnist_buffers, input_buffer }); printDigit(sample); - var result: zml.Buffer = mnist.call(.{input}); + exe.call(args, &results, io); + + var result: zml.Buffer = results.get(zml.Buffer); defer result.deinit(); log.info( @@ -114,7 +221,7 @@ pub fn asyncMain() !void { \\{s} \\ +-------------+ \\ - , .{digits[try result.getValue(u8)]}); + , .{digits[try result.getValue(u8, io)]}); } } From f680e17178f6e519839a9f2909de003f7d950600 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 15 Dec 2025 18:55:43 +0100 Subject: [PATCH 141/881] First clean version of Mnist --- examples/mnist/mnist.zig | 478 ++++++++++++++++++++++++++++++++------- pjrt/pjrt.zig | 16 +- zml2/main.zig | 16 +- zml2/pjrtx.zig | 7 + 4 files changed, 419 insertions(+), 98 deletions(-) diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 2c0c79d0..f7a22ff0 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -18,96 +18,27 @@ const Mnist = struct { weight: zml.Tensor, bias: zml.Tensor, + pub fn init(store: TensorStore.View) Layer { + return .{ + .weight = store.createTensorWithTags("weight", .{ .d_out, .d }), + .bias = store.createTensorWithTags("bias", .{.d_out}), + }; + } + pub fn forward(self: Layer, input: zml.Tensor) zml.Tensor { return self.weight.dot(input, .d).add(self.bias).relu().withTags(.{.d}); } }; - pub fn init(registry: *const zml.safetensors.TensorRegistry) Mnist { - const fc1_weight = b: { - const desc = registry.tensors.get("fc1.weight").?; - break :b zml.Tensor.fromShape(desc.shape).withTags(.{ .d_out, .d }); - }; - const fc1_bias: zml.Tensor = b: { - const desc = registry.tensors.get("fc1.bias").?; - break :b zml.Tensor.fromShape(desc.shape).withTags(.{.d_out}); - }; - const fc2_weight = b: { - const desc = registry.tensors.get("fc2.weight").?; - break :b zml.Tensor.fromShape(desc.shape).withTags(.{ .d_out, .d }); - }; - const fc2_bias: zml.Tensor = b: { - const desc = registry.tensors.get("fc2.bias").?; - break :b zml.Tensor.fromShape(desc.shape).withTags(.{.d_out}); - }; + pub fn init(store: TensorStore.View) Mnist { return .{ - .fc1 = .{ .weight = fc1_weight, .bias = fc1_bias }, - .fc2 = .{ .weight = fc2_weight, .bias = fc2_bias }, + .fc1 = .init(store.withPrefix("fc1")), + .fc2 = .init(store.withPrefix("fc2")), }; } - pub fn deinit(self: *const Mnist) void { - _ = self; // autofix - } - - pub fn loadBuffers(self: Mnist, allocator: std.mem.Allocator, io: std.Io, vfs: *zml.io.VFS, registry: *zml.safetensors.TensorRegistry, platform: zml.Platform) !zml.Bufferized(Mnist) { - _ = self; // autofix - const read_buffer = try allocator.alloc(u8, 4096); - defer allocator.free(read_buffer); - - const fc1_weight: zml.Buffer = b: { - const desc = registry.tensors.get("fc1.weight").?; - const slice = try zml.Slice.alloc(allocator, desc.shape); - defer slice.free(allocator); - - var writer = std.Io.Writer.fixed(slice.data); - var reader = try registry.reader(io, vfs, "fc1.weight", read_buffer); - - _ = try reader.interface.streamRemaining(&writer); - break :b try .fromSlice(io, platform, slice); - }; - errdefer fc1_weight.deinit(); - const fc1_bias: zml.Buffer = b: { - const desc = registry.tensors.get("fc1.bias").?; - const slice = try zml.Slice.alloc(allocator, desc.shape); - defer slice.free(allocator); - - var writer = std.Io.Writer.fixed(slice.data); - var reader = try registry.reader(io, vfs, "fc1.bias", read_buffer); - - _ = try reader.interface.streamRemaining(&writer); - break :b try .fromSlice(io, platform, slice); - }; - errdefer fc1_bias.deinit(); - const fc2_weight: zml.Buffer = b: { - const desc = registry.tensors.get("fc2.weight").?; - const slice = try zml.Slice.alloc(allocator, desc.shape); - defer slice.free(allocator); - - var writer = std.Io.Writer.fixed(slice.data); - var reader = try registry.reader(io, vfs, "fc2.weight", read_buffer); - - _ = try reader.interface.streamRemaining(&writer); - break :b try .fromSlice(io, platform, slice); - }; - errdefer fc2_weight.deinit(); - const fc2_bias: zml.Buffer = b: { - const desc = registry.tensors.get("fc2.bias").?; - const slice = try zml.Slice.alloc(allocator, desc.shape); - defer slice.free(allocator); - - var writer = std.Io.Writer.fixed(slice.data); - var reader = try registry.reader(io, vfs, "fc2.bias", read_buffer); - - _ = try reader.interface.streamRemaining(&writer); - break :b try .fromSlice(io, platform, slice); - }; - errdefer fc2_bias.deinit(); - - return .{ - .fc1 = .{ .weight = fc1_weight, .bias = fc1_bias }, - .fc2 = .{ .weight = fc2_weight, .bias = fc2_bias }, - }; + pub fn loadBuffers(self: Mnist, allocator: std.mem.Allocator, io: std.Io, vfs: *zml.io.VFS, store: TensorStore.View, platform: zml.Platform) !zml.Bufferized(Mnist) { + return loadBuffersFromId(allocator, io, vfs, self, store, platform); } pub fn unloadBuffers(self: *zml.Bufferized(Mnist)) void { @@ -130,10 +61,11 @@ const Mnist = struct { }; pub fn main() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); + const allocator = std.heap.c_allocator; + //var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + //defer _ = gpa.deinit(); - const allocator = gpa.allocator(); + //const allocator = gpa.allocator(); var threaded: std.Io.Threaded = .init(allocator); defer threaded.deinit(); @@ -164,8 +96,9 @@ pub fn main() !void { defer registry.deinit(); // Init model - const mnist_model = Mnist.init(®istry); - defer mnist_model.deinit(); + var store: TensorStore = .fromRegistry(allocator, ®istry); + defer store.deinit(); + const mnist_model = Mnist.init(store.view()); // Compile model log.info("Compiling model to MLIR....", .{}); @@ -183,7 +116,7 @@ pub fn main() !void { defer results.deinit(allocator); // Load buffers - var mnist_buffers = try mnist_model.loadBuffers(allocator, io, &vfs, ®istry, platform); + var mnist_buffers = try mnist_model.loadBuffers(allocator, io, &vfs, store.view(), platform); defer Mnist.unloadBuffers(&mnist_buffers); log.info("✅ Weights transferred in {f}", .{timer.read()}); @@ -333,3 +266,374 @@ const digits = [_][]const u8{ \\ | ##### | , }; + +pub const TensorStore = struct { + registry: *zml.safetensors.TensorRegistry, + id_map: std.AutoHashMapUnmanaged(usize, *zml.safetensors.Tensor), + allocator: std.mem.Allocator, + + pub fn fromRegistry(allocator: std.mem.Allocator, registry: *zml.safetensors.TensorRegistry) TensorStore { + return .{ + .registry = registry, + .id_map = .empty, + .allocator = allocator, + }; + } + + pub fn deinit(self: *TensorStore) void { + self.id_map.deinit(self.allocator); + } + + fn bindIdToKey(self: *TensorStore, key: []const u8, id: usize) !void { + const tensor_desc_ptr = self.registry.tensors.getPtr(key).?; + + const gop = try self.id_map.getOrPut(self.allocator, id); + if (gop.found_existing) { + stdx.debug.panic("Key {s} already has an associated tensor (id: {})", .{ key, gop.key_ptr.* }); + } + errdefer self.id_map.removeByPtr(gop.key_ptr); + + gop.value_ptr.* = tensor_desc_ptr; + } + + fn getPtrFromKey(self: *const TensorStore, key: []const u8) ?*zml.safetensors.Tensor { + const tensor_desc_ptr = self.registry.tensors.getPtr(key) orelse return null; + return tensor_desc_ptr; + } + + fn getPtrFromId(self: *const TensorStore, id: usize) ?*zml.safetensors.Tensor { + const tensor_desc_ptr = self.id_map.get(id) orelse return null; + return tensor_desc_ptr; + } + + pub fn getReader(self: *const TensorStore, key: []const u8, io: std.Io, vfs: *zml.io.VFS, buffer: []u8) !zml.safetensors.TensorReader { + return self.registry.reader(io, vfs, key, buffer); + } + + pub fn getReaderById(self: *const TensorStore, id: usize, io: std.Io, vfs: *zml.io.VFS, buffer: []u8) !zml.safetensors.TensorReader { + const tensor_desc = self.id_map.get(id) orelse return error.NotFound; + + return zml.safetensors.TensorReader.init(io, vfs, tensor_desc.*, buffer); + } + + pub fn view(self: *TensorStore) View { + return .{ .store = self }; + } + + pub const View = struct { + store: *TensorStore, + + prefix_buffer: [256]u8 = undefined, + prefix_length: usize = 0, + + pub fn root(self: *const View) View { + return .{ + .store = self.store, + }; + } + + pub fn parent(self: *const View) View { + const slice = self.prefix() orelse unreachable; + const index = std.mem.lastIndexOfScalar(u8, slice[0 .. slice.len - 1], '.') orelse return self.root(); + var buffer: [256]u8 = undefined; + @memcpy(buffer[0 .. index + 1], slice[0 .. index + 1]); + return .{ + .store = self.store, + .prefix_buffer = buffer, + .prefix_length = index + 1, + }; + } + + pub fn withPrefix(self: *const View, prefix_: []const u8) View { + var buffer: [256]u8 = undefined; + const new_prefix = std.fmt.bufPrint(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }) catch unreachable; + + return .{ + .store = self.store, + .prefix_buffer = buffer, + .prefix_length = new_prefix.len, + }; + } + + fn prefix(self: *const View) ?[]const u8 { + return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; + } + + pub fn maybeCreateTensor(self: View, subkey: []const u8) ?zml.Tensor { + var buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + + const ptr = self.store.getPtrFromKey(key) orelse return null; + + const tensor: zml.Tensor = .init(ptr.shape); + self.store.bindIdToKey(key, tensor.id) catch unreachable; + + return tensor; + } + + pub fn createTensor(self: View, subkey: []const u8) zml.Tensor { + return self.maybeCreateTensor(subkey).?; + } + + pub fn maybeCreateTensorWithTags(self: View, subkey: []const u8, tagz: anytype) ?zml.Tensor { + var buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + + const ptr = self.store.getPtrFromKey(key) orelse return null; + ptr.shape = ptr.shape.withTags(tagz); + + const tensor: zml.Tensor = .fromShape(ptr.shape); + self.store.bindIdToKey(key, tensor.id) catch unreachable; + + return tensor; + } + + pub fn createTensorWithTags(self: View, subkey: []const u8, tagz: anytype) zml.Tensor { + return self.maybeCreateTensorWithTags(subkey, tagz).?; + } + + pub fn getShape(self: View, subkey: []const u8) ?zml.Shape { + var buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + const entry_ptr = self.store.getPtrFromKey(key) orelse return null; + return entry_ptr.shape; + } + + pub fn getShapeOpts(self: View, subkey: []const u8, opts: struct { no_prefix: bool = false }) ?zml.Shape { + var buffer: [256]u8 = undefined; + const key = if (opts.no_prefix) + subkey + else b: { + break :b std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + }; + const entry_ptr = self.store.getPtrFromKey(key) orelse return null; + return entry_ptr.shape; + } + + pub fn getReader(self: View, subkey: []const u8, io: std.Io, vfs: *zml.io.VFS, buffer: []u8) !zml.safetensors.TensorReader { + var key_buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&key_buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + return self.store.getReader(key, io, vfs, buffer); + } + }; +}; + +fn collectShapes(allocator: std.mem.Allocator, v: anytype) ![]zml.Shape { + const LocalContext = struct { + list: *std.array_list.Managed(zml.Shape), + }; + var list = std.array_list.Managed(zml.Shape).init(allocator); + errdefer list.deinit(); + + var context: LocalContext = .{ .list = &list }; + try zml.meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const zml.Tensor) !void { + try ctx_.list.append(tensor.shape()); + } + }.cb, &context, v); + + return try list.toOwnedSlice(); +} + +fn collectTensorDesc(allocator: std.mem.Allocator, store: TensorStore.View, v: anytype) ![]zml.safetensors.Tensor { + const LocalContext = struct { + list: *std.array_list.Managed(zml.safetensors.Tensor), + store: TensorStore.View, + }; + var list = std.array_list.Managed(zml.safetensors.Tensor).init(allocator); + var context: LocalContext = .{ .list = &list, .store = store }; + zml.meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const zml.Tensor) void { + const tensor_desc = ctx_.store.store.getPtrFromId(tensor.id).?.*; + ctx_.list.append(tensor_desc) catch unreachable; + } + }.cb, &context, v); + + return try list.toOwnedSlice(); +} + +pub fn loadBuffersFromId(allocator: std.mem.Allocator, io: std.Io, vfs: *zml.io.VFS, model: anytype, store: TensorStore.View, platform: zml.Platform) !zml.Bufferized(@TypeOf(model)) { + const Model = @TypeOf(model); + var result: zml.Bufferized(Model) = undefined; + initBufferizedFrom(model, &result); + + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + + const shapes = try collectShapes(arena.allocator(), &model); + + var transfer: Transfer = try .init(arena.allocator(), shapes, platform); + defer transfer.deinit(platform); + + const tensor_descs = try collectTensorDesc(arena.allocator(), store, &model); + + // TODO(Corentin): Find a way to inject that + const buffer_reader = try allocator.alloc(u8, 16 * 1024 * 1024); + defer allocator.free(buffer_reader); + const buffer_writer = try allocator.alloc(u8, 16 * 1024 * 1024); + defer allocator.free(buffer_writer); + + const LocalContext = struct { + tensor_descs: []zml.safetensors.Tensor, + shapes: []const zml.Shape, + platform: zml.Platform, + transfer: *Transfer, + index: usize = 0, + buffer_reader: []u8, + buffer_writer: []u8, + store: TensorStore.View, + allocator: std.mem.Allocator, + io: std.Io, + vfs: *zml.io.VFS, + }; + var context: LocalContext = .{ + .tensor_descs = tensor_descs, + .shapes = shapes, + .platform = platform, + .transfer = &transfer, + .buffer_reader = buffer_reader, + .buffer_writer = buffer_writer, + .store = store, + .allocator = allocator, + .io = io, + .vfs = vfs, + }; + try zml.meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *zml.Buffer) !void { + const tensor_desc = context_.tensor_descs[context_.index]; + + var reader = try zml.safetensors.TensorReader.init(context_.io, context_.vfs, tensor_desc, context_.buffer_reader); + defer reader.deinit(); + + var writer = try context_.transfer.getWriter(context_.index, context_.buffer_writer); + + _ = try reader.interface.streamRemaining(&writer.interface); + try writer.interface.flush(); + + buffer.* = try context_.transfer.getBuffer(context_.index); + context_.index += 1; + } + }.cb, &context, &result); + + return result; +} + +pub fn initBufferizedFrom(model: anytype, bufferized_: *zml.Bufferized(@TypeOf(model))) void { + const Model = @TypeOf(model); + const type_info = @typeInfo(zml.Bufferized(Model)); + switch (type_info) { + .@"struct" => |struct_type_info| { + if (zml.Bufferized(Model) == zml.Buffer) return; + inline for (struct_type_info.fields) |field| { + initBufferizedFrom(@field(model, field.name), &@field(bufferized_, field.name)); + } + }, + .@"union" => { + switch (model) { + inline else => |v, tag| { + bufferized_.* = @unionInit(zml.Bufferized(Model), @tagName(tag), undefined); + initBufferizedFrom(v, @field(bufferized_, @tagName(tag))); + }, + } + }, + .optional => { + if (model == null) { + bufferized_.* = null; + } else { + bufferized_.* = undefined; + initBufferizedFrom(model.?, &bufferized_.*.?); + } + }, + .void, .int, .@"enum", .bool, .enum_literal, .float, .pointer, .vector => {}, + else => unreachable, + } +} + +const SimpleWriter = struct { + offset: usize = 0, + transfer_manager: *zml.pjrt.AsyncHostToDeviceTransferManager, + shape: zml.Shape, + buffer_index: usize, + platform: zml.Platform, + interface: std.Io.Writer, + + pub fn init(buffer: []u8, transfer_manager: *zml.pjrt.AsyncHostToDeviceTransferManager, shape: zml.Shape, buffer_index: usize, platform: zml.Platform) SimpleWriter { + return .{ + .transfer_manager = transfer_manager, + .shape = shape, + .buffer_index = buffer_index, + .platform = platform, + .interface = .{ + .buffer = buffer, + .end = 0, + .vtable = &.{ + .drain = drain, + }, + }, + }; + } + + pub fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + _ = data; // autofix + _ = splat; // autofix + const writer: *SimpleWriter = @alignCast(@fieldParentPtr("interface", w)); + stdx.debug.assert(writer.offset + w.end <= writer.shape.byteSize(), "Can't write more data than required", .{}); + const is_last_transfer = writer.offset + w.end >= writer.shape.byteSize(); + log.debug("Writing {} bytes", .{w.end}); + _ = writer.transfer_manager.transferData(writer.platform.pjrt_api, writer.buffer_index, w.buffer[0..w.end], @intCast(writer.offset), is_last_transfer) catch return error.WriteFailed; + const written = w.end; + writer.offset += written; + w.end = 0; + return 0; + } +}; + +const Transfer = struct { + shapes: []zml.Shape, + transfer_manager: *zml.pjrt.AsyncHostToDeviceTransferManager, + arena: std.heap.ArenaAllocator, + platform: zml.Platform, + + pub fn init(allocator: std.mem.Allocator, shapes: []const zml.Shape, platform: zml.Platform) !Transfer { + const shape_specs = try allocator.alloc(zml.pjrt.ShapeSpec, shapes.len); + defer allocator.free(shape_specs); + + var temp_arena = std.heap.ArenaAllocator.init(allocator); + defer temp_arena.deinit(); + + var arena = std.heap.ArenaAllocator.init(allocator); + errdefer arena.deinit(); + + for (shape_specs, shapes) |*spec, shape| { + const dims = try temp_arena.allocator().dupe(i64, shape.dims()); + spec.* = zml.pjrt.ShapeSpec.init(dims, zml.pjrt.bufferTypeFromDtype(shape.dtype())); + } + + const memory = platform.pjrt_client.memoryByKind(platform.pjrt_api, .device).?; + + const transfer_manager = try platform.pjrt_client.createBuffersForAsyncHostToDevice(platform.pjrt_api, .{ .shape_specs = shape_specs, .memory = memory }); + errdefer transfer_manager.deinit(platform.pjrt_api); + + return .{ + .shapes = try arena.allocator().dupe(zml.Shape, shapes), + .transfer_manager = transfer_manager, + .arena = arena, + .platform = platform, + }; + } + + pub fn deinit(self: Transfer, platform: zml.Platform) void { + self.arena.deinit(); + self.transfer_manager.deinit(platform.pjrt_api); + } + + pub fn getBuffer(self: *const Transfer, index: usize) !zml.Buffer { + const pjrt_buffer = self.transfer_manager.retrieveBuffer(self.platform.pjrt_api, index) catch return error.NotFound; + return .fromPjrtBuffers(self.platform, self.shapes[index], &.{pjrt_buffer}); + } + + pub fn getWriter(self: *const Transfer, index: usize, buffer: []u8) !SimpleWriter { + const writer: SimpleWriter = .init(buffer, self.transfer_manager, self.shapes[index], index, self.platform); + return writer; + } +}; diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index b9bc3e93..0093681c 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -270,12 +270,12 @@ pub const ShapeSpec = extern struct { inner: c.PJRT_ShapeSpec, - pub fn init(dims_: []const usize, bt: BufferType) ShapeSpec { + pub fn init(dims_: []const i64, bt: BufferType) ShapeSpec { return .{ .inner = pjrtStruct2(c.PJRT_ShapeSpec, .{ - .dims = @ptrCast(@constCast(dims_.ptr)), - .num_dims = dims.len, - .buffer_type = @intFromEnum(bt), + .dims = @as([*c]const i64, @ptrCast(@constCast(dims_.ptr))), + .num_dims = dims_.len, + .element_type = @intFromEnum(bt), }), }; } @@ -469,11 +469,11 @@ pub const Client = opaque { pub fn createBuffersForAsyncHostToDevice(self: *const Client, api: *const Api, args: CreateBuffersForAsyncHostToDeviceArgs) ApiError!*AsyncHostToDeviceTransferManager { const ret = try api.call(.PJRT_Client_CreateBuffersForAsyncHostToDevice, .{ .client = self.inner(), - .shape_specs = @ptrCast(args.shape_specs.ptr), + .shape_specs = @as([*c]c.PJRT_ShapeSpec, @ptrCast(@constCast(args.shape_specs.ptr))), .num_shape_specs = args.shape_specs.len, - .device_layouts = if (args.device_layouts) |layouts| @ptrCast(@constCast(layouts.ptr)) else null, - .num_device_layouts = if (args.device_layouts) |layouts| @intCast(layouts.len) else 0, - .memory = @ptrCast(@constCast(args.memory)), + .device_layouts = @as([*c][*c]c.PJRT_Buffer_MemoryLayout, if (args.device_layouts) |layouts| @ptrCast(@constCast(layouts.ptr)) else null), + .num_device_layouts = @as(usize, if (args.device_layouts) |layouts| @intCast(layouts.len) else 0), + .memory = @as(?*c.PJRT_Memory, @ptrCast(@constCast(args.memory))), }); return @ptrCast(ret.transfer_manager.?); } diff --git a/zml2/main.zig b/zml2/main.zig index 3ace2ce3..c3f2f897 100644 --- a/zml2/main.zig +++ b/zml2/main.zig @@ -52,15 +52,25 @@ pub fn main() !void { var threaded: std.Io.Threaded = .init(allocator); defer threaded.deinit(); - const io = threaded.io(); + var vfs_file: zml.io.VFS.File = .init(threaded.io()); + + var vfs: zml.io.VFS = .init(allocator, threaded.io()); + defer vfs.deinit(); + + try vfs.register("file", vfs_file.io()); + + const io = vfs.io(); + + const file = try vfs.openAbsoluteFile(io, "//private/var/tmp/_bazel_corendos/c1731bd4fcb7bdb2687c1972ca03d70f/external/+non_module_deps+mnist/mnist.safetensors", .{ .mode = .read_only }); + defer file.close(io); zml.init(); defer zml.deinit(); - const available = zml.Platform.availablePlatforms(io); + const available = zml.Platform.availablePlatforms(threaded.io()); std.log.info("Available platforms: {any}", .{available}); - var platform = try zml.Platform.auto(io, .{}); + var platform = try zml.Platform.auto(threaded.io(), .{}); defer platform.deinit(); std.log.info("Selected platform: {f}", .{platform}); diff --git a/zml2/pjrtx.zig b/zml2/pjrtx.zig index ac37b95f..db672b8f 100644 --- a/zml2/pjrtx.zig +++ b/zml2/pjrtx.zig @@ -22,6 +22,7 @@ pub const MemoryStats = pjrt.MemoryStats; pub const NamedValue = pjrt.NamedValue; pub const SerializeResult = pjrt.SerializeResult; pub const Stream = pjrt.Stream; +pub const ShapeSpec = pjrt.ShapeSpec; const zml = struct { pub const DataType = @import("dtype.zig").DataType; @@ -137,6 +138,12 @@ pub const Client = opaque { pub fn createUnitializedBuffer(self: *const Client, api: *const Api, args: CreateUninitializedBufferArgs) ApiError!*Buffer { return @ptrCast(try self.inner().createUninitializedBuffer(api, args)); } + + pub const CreateBuffersForAsyncHostToDeviceArgs = pjrt.Client.CreateBuffersForAsyncHostToDeviceArgs; + + pub fn createBuffersForAsyncHostToDevice(self: *const Client, api: *const Api, args: CreateBuffersForAsyncHostToDeviceArgs) ApiError!*AsyncHostToDeviceTransferManager { + return @ptrCast(try self.inner().createBuffersForAsyncHostToDevice(api, args)); + } }; pub const Buffer = opaque { From 56f16dadc781d3f87b5207d414edbd84319dc380 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 16 Dec 2025 11:40:49 +0100 Subject: [PATCH 142/881] Add output Memory Kind support --- zml2/module.zig | 24 +++++++++++++++++++++--- zml2/tensor.zig | 37 +++++++++++++++++++++++++++++++++---- 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/zml2/module.zig b/zml2/module.zig index dddfb7fc..337d5d03 100644 --- a/zml2/module.zig +++ b/zml2/module.zig @@ -13,6 +13,7 @@ const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const Tensor = @import("tensor.zig").Tensor; +const Buffer = @import("buffer.zig").Buffer; const log = std.log.scoped(.@"zml/module"); @@ -23,6 +24,7 @@ pub const CompilationContext = struct { block: *mlir.Block, id_to_argument: std.AutoArrayHashMapUnmanaged(usize, usize), id_to_donation: std.AutoArrayHashMapUnmanaged(usize, usize), + id_to_output_memory_kind: std.AutoArrayHashMapUnmanaged(usize, Buffer.Memory), arena: std.heap.ArenaAllocator, pub fn initFromBlock(allocator: std.mem.Allocator, block: *mlir.Block) Scope { @@ -31,6 +33,7 @@ pub const CompilationContext = struct { .block = block, .id_to_argument = .empty, .id_to_donation = .empty, + .id_to_output_memory_kind = .empty, .arena = arena, }; } @@ -214,11 +217,13 @@ pub const OutputInfo = struct { shapes: []Shape, values: []*const mlir.Value, donations: []?usize, + output_memory_kinds: []Buffer.Memory, pub fn deinit(self: OutputInfo, allocator: std.mem.Allocator) void { allocator.free(self.shapes); allocator.free(self.values); allocator.free(self.donations); + allocator.free(self.output_memory_kinds); } }; @@ -227,6 +232,7 @@ fn collectOutputInfo(allocator: std.mem.Allocator, v: anytype) !OutputInfo { shape_list: *std.array_list.Managed(Shape), value_list: *std.array_list.Managed(*const mlir.Value), donation_list: *std.array_list.Managed(?usize), + output_memory_kind_list: *std.array_list.Managed(Buffer.Memory), }; var shape_list = std.array_list.Managed(Shape).init(allocator); @@ -235,14 +241,22 @@ fn collectOutputInfo(allocator: std.mem.Allocator, v: anytype) !OutputInfo { errdefer value_list.deinit(); var donation_list = std.array_list.Managed(?usize).init(allocator); errdefer donation_list.deinit(); + var output_memory_kind_list = std.array_list.Managed(Buffer.Memory).init(allocator); + errdefer output_memory_kind_list.deinit(); - var context: LocalContext = .{ .shape_list = &shape_list, .value_list = &value_list, .donation_list = &donation_list }; + var context: LocalContext = .{ + .shape_list = &shape_list, + .value_list = &value_list, + .donation_list = &donation_list, + .output_memory_kind_list = &output_memory_kind_list, + }; try meta.visit(struct { fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { try ctx_.shape_list.append(tensor.shape()); try ctx_.value_list.append(tensor.value()); try ctx_.donation_list.append(tensor.donation()); + try ctx_.output_memory_kind_list.append(tensor.outputMemoryKind()); } }.cb, &context, v); @@ -250,6 +264,7 @@ fn collectOutputInfo(allocator: std.mem.Allocator, v: anytype) !OutputInfo { .shapes = try shape_list.toOwnedSlice(), .values = try value_list.toOwnedSlice(), .donations = try donation_list.toOwnedSlice(), + .output_memory_kinds = try output_memory_kind_list.toOwnedSlice(), }; } @@ -353,15 +368,18 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar for (output_info.donations, 0..) |donation, index| if (donation) |argument_index| { input_attributes[argument_index].appendAssumeCapacity(.named(compilation_context.mlir_ctx, "tf.aliasing_output", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, index))); }; + for (output_info.output_memory_kinds, 0..) |output_memory_kind, index| { + if (output_memory_kind == .device) continue; + output_attributes[index].appendAssumeCapacity(.named(compilation_context.mlir_ctx, "mhlo.memory_kind", mlir.stringAttribute(compilation_context.mlir_ctx, output_memory_kind.pjrtName()))); + } _ = dialects.func.returns(compilation_context.mlir_ctx, output_info.values, .unknown(compilation_context.mlir_ctx)).appendTo(compilation_context.currentScope().block); - //self.add - const mlir_func = dialects.func.func(compilation_context.mlir_ctx, .{ .name = "main", .block = compilation_context.currentScope().block, .location = .unknown(compilation_context.mlir_ctx), .args_attributes = try finalizeAttributeList(arena.allocator(), compilation_context.mlir_ctx, input_attributes), + .results_attributes = try finalizeAttributeList(arena.allocator(), compilation_context.mlir_ctx, output_attributes), }); compilation_context.mlir_pass_manager.runOnOp(mlir_func) catch |err| switch (err) { diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 6ffcdc46..6ae8cea8 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -16,6 +16,7 @@ const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; +const Memory = Buffer.Memory; pub const Tensor = struct { var current_id: std.atomic.Value(usize) = .{ .raw = 1 }; @@ -149,10 +150,33 @@ pub const Tensor = struct { @panic("TODO"); } - // TODO(Corentin) - //pub fn toMemory(self: Tensor, kind: Memory) Tensor { - // @panic("TODO"); - //} + pub fn toMemory(self: Tensor, kind: Memory) Tensor { + const ctx = CompilationContext.current(); + if (ctx.platform.target == .cpu) return self; + + const frontend_attributes = mlir.dictionaryAttribute(ctx.mlir_ctx, &.{ + .named(ctx.mlir_ctx, "_xla_buffer_placement", mlir.stringAttribute(ctx.mlir_ctx, kind.pjrtName())), + }); + + const op = dialects.stablehlo.custom_call( + ctx.mlir_ctx, + &.{self.value()}, + &.{self.value().type_()}, + .{ + .call_target_name = "annotate_device_placement", + .has_side_effect = true, + .backend_config = .{ .original = "" }, + .additional_attributes = &.{ + .named(ctx.mlir_ctx, "mhlo.frontend_attributes", frontend_attributes), + }, + }, + .unknown(ctx.mlir_ctx), + ).appendTo(currentBlock()); + + var res = _result(self._shape, op.result(0)); + ctx.currentScope().id_to_output_memory_kind.put(ctx.currentScope().arena.allocator(), res.id, kind) catch unreachable; + return res; + } /// Returns a Tensor with new tag names. pub fn rename(self: Tensor, renames: anytype) Tensor { @@ -3967,6 +3991,11 @@ pub const Tensor = struct { pub fn donation(self: Tensor) ?usize { return CompilationContext.current().currentScope().id_to_donation.get(self.id); } + + /// Returns the output memory kind of the tensor. + pub fn outputMemoryKind(self: Tensor) Buffer.Memory { + return CompilationContext.current().currentScope().id_to_output_memory_kind.get(self.id) orelse .device; + } }; fn initPoolArg(rank: usize, data: []const i64) [constants.MAX_RANK]i64 { From be6d93990fea6421d2180120f06748c97eb2e4f3 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 16 Dec 2025 17:19:14 +0100 Subject: [PATCH 143/881] io: perf on linux (#365) --- .bazelrc | 5 ++ zml2/io/vfs.zig | 36 +++++----- zml2/io/vfs/hf.zig | 32 ++++----- zml2/io/vfs/http.zig | 95 ++++++++++---------------- zml2/main_io.zig | 159 +++++++++++++++++++++++++++++++++++-------- zml2/safetensors.zig | 6 +- 6 files changed, 211 insertions(+), 122 deletions(-) diff --git a/.bazelrc b/.bazelrc index 2b7fbd0e..e3c17bb7 100644 --- a/.bazelrc +++ b/.bazelrc @@ -115,3 +115,8 @@ common:alldebug --strategy=local common:release --compilation_mode=opt common:release --copt=-Ofast common:release --@rules_zig//zig/settings:mode=release_safe + +# --config=native builds everything optimized for the local CPU +common:native --copt=-march=native +common:native --@rules_zig//zig/settings:zigopt=-mcpu=native +common:native --@rules_rust//rust/settings:extra_rustc_flag=-Ctarget-cpu=native diff --git a/zml2/io/vfs.zig b/zml2/io/vfs.zig index f982e7f0..551198de 100644 --- a/zml2/io/vfs.zig +++ b/zml2/io/vfs.zig @@ -36,7 +36,7 @@ pub const VFS = struct { const ROOT_DIR_HANDLE: std.Io.Dir.Handle = 0; allocator: std.mem.Allocator, - mutex: std.Thread.Mutex, + mutex: std.Io.Mutex, backends: std.StringHashMapUnmanaged(std.Io), // Maps virtual file and dir handles to backend handles @@ -105,7 +105,7 @@ pub const VFS = struct { return .{ .allocator = allocator, - .mutex = .{}, + .mutex = .init, .backends = .{}, .files = .{}, .dirs = .{}, @@ -133,15 +133,15 @@ pub const VFS = struct { } pub fn register(self: *VFS, scheme: []const u8, backend: std.Io) std.mem.Allocator.Error!void { - self.mutex.lock(); - defer self.mutex.unlock(); + self.mutex.lockUncancelable(self.base_io); + defer self.mutex.unlock(self.base_io); try self.backends.put(self.allocator, scheme, backend); } pub fn unregister(self: *VFS, scheme: []const u8) bool { - self.mutex.lock(); - defer self.mutex.unlock(); + self.mutex.lockUncancelable(self.base_io); + defer self.mutex.unlock(self.base_io); return self.backends.remove(scheme); } @@ -167,8 +167,8 @@ pub const VFS = struct { .handle = handle, }; - self.mutex.lock(); - defer self.mutex.unlock(); + self.mutex.lockUncancelable(self.base_io); + defer self.mutex.unlock(self.base_io); const virtual_handle = self.next_file_handle; self.next_file_handle += 1; @@ -179,9 +179,9 @@ pub const VFS = struct { } fn closeFile(self: *VFS, virtual_handle: std.Io.File.Handle) void { - self.mutex.lock(); + self.mutex.lockUncancelable(self.base_io); const entry = self.files.fetchRemove(virtual_handle); - self.mutex.unlock(); + self.mutex.unlock(self.base_io); if (entry) |kv| { const fs_file = kv.value; @@ -191,8 +191,8 @@ pub const VFS = struct { } fn fsFile(self: *VFS, virtual_handle: std.Io.File.Handle) FileEntryError!FsFile { - self.mutex.lock(); - defer self.mutex.unlock(); + self.mutex.lockUncancelable(self.base_io); + defer self.mutex.unlock(self.base_io); const entry = self.files.get(virtual_handle) orelse return FileEntryError.MissingVirtualFileHandle; return entry.*; @@ -207,8 +207,8 @@ pub const VFS = struct { .handle = handle, }; - self.mutex.lock(); - defer self.mutex.unlock(); + self.mutex.lockUncancelable(self.base_io); + defer self.mutex.unlock(self.base_io); const virtual_handle = self.next_dir_handle; self.next_dir_handle += 1; @@ -220,9 +220,9 @@ pub const VFS = struct { fn closeDir(self: *VFS, virtual_handle: std.Io.Dir.Handle) void { if (virtual_handle == ROOT_DIR_HANDLE) return; - self.mutex.lock(); + self.mutex.lockUncancelable(self.base_io); const entry = self.dirs.fetchRemove(virtual_handle); - self.mutex.unlock(); + self.mutex.unlock(self.base_io); if (entry) |kv| { const fs_dir = kv.value; @@ -238,8 +238,8 @@ pub const VFS = struct { return .{ .backend = backend, .handle = ROOT_DIR_HANDLE }; } - self.mutex.lock(); - defer self.mutex.unlock(); + self.mutex.lockUncancelable(self.base_io); + defer self.mutex.unlock(self.base_io); const entry = self.dirs.get(virtual_handle) orelse return DirEntryError.MissingVirtualDirHandle; return entry.*; diff --git a/zml2/io/vfs/hf.zig b/zml2/io/vfs/hf.zig index 835fac8e..28fcfb1c 100644 --- a/zml2/io/vfs/hf.zig +++ b/zml2/io/vfs/hf.zig @@ -81,7 +81,7 @@ pub const HF = struct { file_handles: std.AutoHashMapUnmanaged(HandleId, *FileHandle), dir_handles: std.AutoHashMapUnmanaged(HandleId, *DirHandle), next_handle_id: HandleId, - mutex: std.Thread.Mutex, + mutex: std.Io.Mutex, vtable: std.Io.VTable, pub const HandleId = i32; @@ -195,7 +195,7 @@ pub const HF = struct { .file_handles = .{}, .dir_handles = .{}, .next_handle_id = 1, - .mutex = .{}, + .mutex = .init, .vtable = makeVTable(args.io), }; @@ -232,8 +232,8 @@ pub const HF = struct { } fn allocateHandleId(self: *HF) HandleId { - self.mutex.lock(); - defer self.mutex.unlock(); + self.mutex.lockUncancelable(self.io()); + defer self.mutex.unlock(self.io()); const id = self.next_handle_id; self.next_handle_id += 1; @@ -241,45 +241,45 @@ pub const HF = struct { } fn registerFileHandle(self: *HF, handle: *FileHandle) std.mem.Allocator.Error!void { - self.mutex.lock(); - defer self.mutex.unlock(); + self.mutex.lockUncancelable(self.io()); + defer self.mutex.unlock(self.io()); try self.file_handles.put(self.allocator, handle.id, handle); } fn unregisterFileHandle(self: *HF, id: HandleId) ?*FileHandle { - self.mutex.lock(); - defer self.mutex.unlock(); + self.mutex.lockUncancelable(self.io()); + defer self.mutex.unlock(self.io()); const kv = self.file_handles.fetchRemove(id) orelse return null; return kv.value; } fn getFileHandle(self: *HF, id: HandleId) ?*FileHandle { - self.mutex.lock(); - defer self.mutex.unlock(); + self.mutex.lockUncancelable(self.io()); + defer self.mutex.unlock(self.io()); return self.file_handles.get(id); } fn registerDirHandle(self: *HF, handle: *DirHandle) std.mem.Allocator.Error!void { - self.mutex.lock(); - defer self.mutex.unlock(); + self.mutex.lockUncancelable(self.io()); + defer self.mutex.unlock(self.io()); try self.dir_handles.put(self.allocator, handle.id, handle); } fn unregisterDirHandle(self: *HF, id: HandleId) ?*DirHandle { - self.mutex.lock(); - defer self.mutex.unlock(); + self.mutex.lockUncancelable(self.io()); + defer self.mutex.unlock(self.io()); const kv = self.dir_handles.fetchRemove(id) orelse return null; return kv.value; } fn getDirHandle(self: *HF, id: HandleId) ?*DirHandle { - self.mutex.lock(); - defer self.mutex.unlock(); + self.mutex.lockUncancelable(self.io()); + defer self.mutex.unlock(self.io()); return self.dir_handles.get(id); } diff --git a/zml2/io/vfs/http.zig b/zml2/io/vfs/http.zig index f459b484..8fbbca76 100644 --- a/zml2/io/vfs/http.zig +++ b/zml2/io/vfs/http.zig @@ -111,7 +111,7 @@ pub const HTTP = struct { }; allocator: std.mem.Allocator, - mutex: std.Thread.Mutex, + mutex: std.Io.Mutex, file_handles: std.AutoHashMapUnmanaged(HandleId, *FileHandle), dir_handles: std.AutoHashMapUnmanaged(HandleId, *DirHandle), @@ -126,7 +126,7 @@ pub const HTTP = struct { pub fn init(allocator: std.mem.Allocator, base_io: std.Io, http_client: *std.http.Client, config: Config) std.mem.Allocator.Error!HTTP { var self: HTTP = .{ .allocator = allocator, - .mutex = .{}, + .mutex = .init, .file_handles = .{}, .dir_handles = .{}, .next_file_handle_id = 0, @@ -170,8 +170,8 @@ pub const HTTP = struct { } fn registerFileHandle(self: *HTTP, handle: *FileHandle) std.mem.Allocator.Error!HandleId { - self.mutex.lock(); - defer self.mutex.unlock(); + self.mutex.lockUncancelable(self.io()); + defer self.mutex.unlock(self.io()); const handle_id = self.next_file_handle_id; self.next_file_handle_id += 1; @@ -182,15 +182,15 @@ pub const HTTP = struct { } fn fileHandle(self: *HTTP, file: std.Io.File) ?*FileHandle { - self.mutex.lock(); - defer self.mutex.unlock(); + self.mutex.lockUncancelable(self.io()); + defer self.mutex.unlock(self.io()); return self.file_handles.get(file.handle); } fn registerDirHandle(self: *HTTP, handle: *DirHandle) std.mem.Allocator.Error!HandleId { - self.mutex.lock(); - defer self.mutex.unlock(); + self.mutex.lockUncancelable(self.io()); + defer self.mutex.unlock(self.io()); const handle_id = self.next_dir_handle_id; self.next_dir_handle_id += 1; @@ -201,8 +201,8 @@ pub const HTTP = struct { } fn dirHandle(self: *HTTP, dir: std.Io.Dir) ?*DirHandle { - self.mutex.lock(); - defer self.mutex.unlock(); + self.mutex.lockUncancelable(self.io()); + defer self.mutex.unlock(self.io()); return self.dir_handles.get(dir.handle); } @@ -253,8 +253,8 @@ pub const HTTP = struct { } fn closeDir(self: *HTTP, dir: std.Io.Dir) void { - self.mutex.lock(); - defer self.mutex.unlock(); + self.mutex.lockUncancelable(self.io()); + defer self.mutex.unlock(self.io()); const handle_kv = self.dir_handles.fetchRemove(dir.handle) orelse { log.warn("Attempted to close non-existent dir handle: {d}", .{dir.handle}); @@ -265,8 +265,8 @@ pub const HTTP = struct { } fn closeFile(self: *HTTP, file: std.Io.File) void { - self.mutex.lock(); - defer self.mutex.unlock(); + self.mutex.lockUncancelable(self.io()); + defer self.mutex.unlock(self.io()); const handle_kv = self.file_handles.fetchRemove(file.handle) orelse { log.warn("Attempted to close non-existent file handle: {d}", .{file.handle}); @@ -318,31 +318,35 @@ pub const HTTP = struct { var total_read: usize = 0; - log.debug("Starting HTTP read handle={d} pos={d} buffers={d} url={s}", .{ file.handle, handle.pos, data.len, handle.url }); - for (data) |buf| { - log.debug(" buffer len={d}", .{buf.len}); - } - - for (data, 0..) |buf, buf_index| { var offset: usize = 0; while (offset < buf.len) { + const file_size = blk: { + if (handle.size) |size| break :blk size; + + const stat = self.fetchFileStat(file) catch { + log.err("Failed to fetch file stat during read for handle={d} url={s}", .{ file.handle, handle.url }); + return std.Io.File.Reader.Error.Unexpected; + }; + + handle.size = stat.size; + + break :blk stat.size; + }; + // Check EOF against known file size - if (handle.size) |size| { - if (handle.pos >= size) { - return total_read; - } + if (handle.pos >= file_size) { + return total_read; } - var desired: u64 = 0; - for (data[buf_index..]) |b| desired += b.len; - desired -= offset; + const start = handle.pos; + var desired_read_len: u64 = @max(@as(u64, @intCast(buf.len)), self.config.request_range_min); + if (desired_read_len > self.config.request_range_max) desired_read_len = self.config.request_range_max; - if (desired < self.config.request_range_min) desired = self.config.request_range_min; - if (desired > self.config.request_range_max) desired = self.config.request_range_max; + const end = if (handle.size.? > 0 and start < file_size) @min(start + desired_read_len - 1, file_size - 1) else start + desired_read_len - 1; - try self.requestRead(file, desired); + try self.requestRead(file, start, end); const reader = handle.body_reader.?; const dest = buf[offset..]; @@ -359,43 +363,20 @@ pub const HTTP = struct { }, }; - if (n == 0) { - // log.debug("readVec returned 0 (no progress) handle={d} pos={d} buf_index={d} offset={d}", .{ file.handle, handle.pos, buf_index, offset }); - continue; - } - - // log.debug("readVec wrote {d} bytes for handle={d} pos(before)={d} buf_index={d} offset(before)={d}", .{ n, file.handle, handle.pos, buf_index, offset }); - offset += n; handle.pos += n; total_read += n; } } - return total_read; } - fn requestRead(self: *HTTP, file: std.Io.File, desired: u64) std.Io.File.Reader.Error!void { + fn requestRead(self: *HTTP, file: std.Io.File, start: u64, end: u64) std.Io.File.Reader.Error!void { const handle = self.fileHandle(file) orelse return std.Io.File.Reader.Error.Unexpected; - if (handle.body_reader != null) return; + if (handle.request) |_| return; - log.debug("Starting HTTP request lifecycle handle={d} pos={d} desired={d} url={s}", .{ file.handle, handle.pos, desired, handle.url }); - - if (handle.size == null) { - const stat = self.fetchFileStat(file) catch |err| { - log.err("Failed to fetch file stat during read: {}", .{err}); - return std.Io.File.Reader.Error.SystemResources; - }; - handle.size = stat.size; - } - - const size = handle.size.?; - - if (handle.pos >= size) return; - - const start = handle.pos; - const end = @min(start + desired, size) - 1; + std.debug.assert(start <= end); var range_buf: [64]u8 = undefined; const range_header = std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ start, end }) catch unreachable; @@ -461,7 +442,6 @@ pub const HTTP = struct { handle.request = request; handle.response = response; handle.body_reader = handle.response.?.reader(&.{}); - log.debug("Initialized body_reader len={d} seek={d} for handle={s}", .{ handle.body_reader.?.buffer.len, handle.body_reader.?.seek, handle.url }); } pub const FileStat = struct { @@ -542,7 +522,6 @@ pub const HTTP = struct { log.err("HTTP stat response missing Content-Length", .{}); return std.Io.File.StatError.SystemResources; }; - handle.size = size; return .{ .inode = @intCast(file.handle), diff --git a/zml2/main_io.zig b/zml2/main_io.zig index a805c257..06310233 100644 --- a/zml2/main_io.zig +++ b/zml2/main_io.zig @@ -14,10 +14,12 @@ pub const std_options: std.Options = .{ }; pub fn main() !void { - var debug_allocator: std.heap.DebugAllocator(.{}) = .init; - defer std.debug.assert(debug_allocator.deinit() == .ok); + // var debug_allocator: std.heap.DebugAllocator(.{}) = .init; + // defer std.debug.assert(debug_allocator.deinit() == .ok); - const allocator = debug_allocator.allocator(); + // const allocator = debug_allocator.allocator(); + + const allocator = std.heap.c_allocator; var threaded: std.Io.Threaded = .init(allocator); defer threaded.deinit(); @@ -104,30 +106,46 @@ pub fn main() !void { log.info("Parsed {d} tensors", .{registry.tensors.count()}); - { - const tensor_name = "model.layers.31.mlp.down_proj.weight"; - // const tensor_name = "model.layers.31.input_layernorm.weight"; - const tensor = registry.tensors.get(tensor_name) orelse return error.TensorNotFound; - log.info("Reading {f}...", .{tensor}); + // { + // const tensor_name = "model.layers.31.mlp.down_proj.weight"; + // // const tensor_name = "model.layers.31.input_layernorm.weight"; + // // const tensor_name = "lm_head.weight"; + // const tensor = registry.tensors.get(tensor_name) orelse return error.TensorNotFound; + // log.info("Reading {f}...", .{tensor}); - const tensor_reader_buf = try allocator.alloc(u8, tensor.byteSize()); - defer allocator.free(tensor_reader_buf); + // const tensor_reader_buf = try allocator.alloc(u8, tensor.byteSize()); + // defer allocator.free(tensor_reader_buf); - const writer_buffer = try allocator.alloc(u8, tensor.byteSize()); - defer allocator.free(writer_buffer); + // const writer_buffer = try allocator.alloc(u8, tensor.byteSize()); + // defer allocator.free(writer_buffer); - var tensor_reader = try registry.reader(io, &vfs, tensor_name, tensor_reader_buf); - defer tensor_reader.deinit(); + // var tensor_reader = try registry.reader(io, &vfs, tensor_name, tensor_reader_buf); + // defer tensor_reader.deinit(); - var writer: std.Io.Writer = .fixed(writer_buffer); - const read = try tensor_reader.interface.streamRemaining(&writer); + // var writer: std.Io.Writer = .fixed(writer_buffer); - log.info("Read tensor data: {d} bytes", .{read}); - } + // var timer_read: std.time.Timer = try .start(); + + // const read = try tensor_reader.interface.streamRemaining(&writer); + + // const elapsed = timer_read.read(); + // const read_mb = @as(f64, @floatFromInt(read)) / (1024.0 * 1024.0); + // const read_time_s = @as(f64, @floatFromInt(elapsed)) / @as(f64, std.time.ns_per_s); + // const throughput_gbps = (read_mb * 8.0) / 1024.0 / read_time_s; + + // log.info("Read completed in {d} ms | {d:.2} MB/s | {d:.2} Gbps", .{ + // elapsed / std.time.ns_per_ms, + // read_mb / read_time_s, + // throughput_gbps, + // }); + // } { var err: ?anyerror = null; + var pool: MemoryPool = try .init(allocator, threaded.async_limit.toInt() orelse 16, 256 * 1024 * 1024); + defer pool.deinit(io); + var group: std.Io.Group = .init; defer group.cancel(io); @@ -135,7 +153,7 @@ pub fn main() !void { var it = registry.iterator(); while (it.next()) |entry| { const tensor_name = entry.key_ptr.*; - group.async(threaded.io(), readTensor, .{ allocator, io, &vfs, ®istry, tensor_name, &err }); + group.async(io, readTensor, .{ allocator, io, &vfs, &pool, ®istry, tensor_name, &err }); } group.wait(io); @@ -148,7 +166,12 @@ pub fn main() !void { const elapsed = timer_async_read.read(); const read_mb = @as(f64, @floatFromInt(registry.totalBytes())) / (1024.0 * 1024.0); const read_time_s = @as(f64, @floatFromInt(elapsed)) / @as(f64, std.time.ns_per_s); - log.info("Throughput: {d:.2} MB in {d:.2} s = {d:.2} MB/s", .{ read_mb, read_time_s, read_mb / read_time_s }); + log.info("Throughput: {d:.2} MB in {d:.2} s = {d:.2} MB/s | {d:.2} Gbps", .{ + read_mb, + read_time_s, + read_mb / read_time_s, + (read_mb * 8.0) / 1024.0 / read_time_s, + }); } } @@ -274,20 +297,22 @@ fn readTensor( allocator: std.mem.Allocator, io: std.Io, vfs: *zml.io.VFS, + pool: *MemoryPool, registry: *zml.safetensors.TensorRegistry, tensor_name: []const u8, err_ptr: *?anyerror, ) void { - readTensorImpl(allocator, io, vfs, registry, tensor_name) catch |err| { + readTensorImpl(allocator, io, vfs, pool, registry, tensor_name) catch |err| { err_ptr.* = err; log.err("Failed to read tensor {s}: {any}", .{ tensor_name, err }); }; } pub fn readTensorImpl( - allocator: std.mem.Allocator, + _: std.mem.Allocator, io: std.Io, vfs: *zml.io.VFS, + pool: *MemoryPool, registry: *zml.safetensors.TensorRegistry, tensor_name: []const u8, ) !void { @@ -295,15 +320,16 @@ pub fn readTensorImpl( const tensor = registry.tensors.get(tensor_name) orelse return error.TensorNotFound; - const buff = try allocator.alloc(u8, tensor.byteSize() * 2); - defer allocator.free(buff); + const mem_slot = try pool.alloc(io, tensor.byteSize() * 2); + defer pool.free(io, mem_slot.index); - var tensor_reader = try registry.reader(io, vfs, tensor_name, buff[0..tensor.byteSize()]); + var tensor_reader = try registry.reader(io, vfs, tensor_name, mem_slot.data[0..tensor.byteSize()]); defer tensor_reader.deinit(); - var writer: std.Io.Writer = .fixed(buff[tensor.byteSize()..]); - var read_timer = try std.time.Timer.start(); + + var writer: std.Io.Writer = .fixed(mem_slot.data[tensor.byteSize()..]); + const read = try tensor_reader.interface.streamRemaining(&writer); const read_time = read_timer.read(); @@ -323,6 +349,85 @@ pub fn readTensorImpl( }); } +const MemoryPool = struct { + const Slots = std.ArrayList(Slot); + const Slot = struct { + buf: []u8, + capacity: usize, + in_use: bool, + }; + + allocator: std.mem.Allocator, + mutex: std.Io.Mutex, + + slots: Slots, + default_capacity: usize, + + pub fn init(allocator: std.mem.Allocator, initial_count: usize, slot_size: usize) !MemoryPool { + var slots: Slots = .{}; + + for (0..initial_count) |_| { + const buf = try allocator.alloc(u8, slot_size); + try slots.append(allocator, .{ .buf = buf, .capacity = slot_size, .in_use = false }); + } + + return .{ + .allocator = allocator, + .slots = slots, + .default_capacity = slot_size, + .mutex = .init, + }; + } + + pub fn deinit(self: *MemoryPool, io: std.Io) void { + self.mutex.lockUncancelable(io); + + for (self.slots.items) |slot| { + self.allocator.free(slot.buf); + } + + self.mutex.unlock(io); + self.slots.deinit(self.allocator); + } + + pub fn alloc(self: *MemoryPool, io: std.Io, n: usize) !struct { data: []u8, index: usize } { + self.mutex.lockUncancelable(io); + + var idx: usize = 0; + while (idx < self.slots.items.len) : (idx += 1) { + if (!self.slots.items[idx].in_use and self.slots.items[idx].capacity >= n) { + self.slots.items[idx].in_use = true; + const data = self.slots.items[idx].buf[0..n]; + self.mutex.unlock(io); + return .{ .data = data, .index = idx }; + } + } + + const capacity = if (n > self.default_capacity) n else self.default_capacity; + + log.warn("MemoryPool: growing pool, allocating new slot of size {d} bytes", .{capacity}); + + const buf = try self.allocator.alloc(u8, capacity); + try self.slots.append(self.allocator, .{ .buf = buf, .capacity = capacity, .in_use = true }); + + const new_index = self.slots.items.len - 1; + const data = buf[0..n]; + + self.mutex.unlock(io); + + return .{ .data = data, .index = new_index }; + } + + pub fn free(self: *MemoryPool, io: std.Io, index: usize) void { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + + if (index < self.slots.items.len) { + self.slots.items[index].in_use = false; + } + } +}; + fn readData( allocator: std.mem.Allocator, io: std.Io, diff --git a/zml2/safetensors.zig b/zml2/safetensors.zig index dc09c631..c257b6d0 100644 --- a/zml2/safetensors.zig +++ b/zml2/safetensors.zig @@ -26,13 +26,13 @@ pub fn parseFromPath( const repo_path = try resolveModelRepoPath(allocator, io, vfs, path); defer allocator.free(repo_path); - log.info("Resolved model repo path for '{s}': {s}", .{ path, repo_path }); + log.debug("Resolved model repo path for '{s}': {s}", .{ path, repo_path }); const repo = try vfs.openAbsoluteDir(io, repo_path, .{}); defer repo.close(io); const file_type = resolveFiletype(file_path); - log.info("Resolved file type for '{s}': {any}", .{ path, file_type }); + log.debug("Resolved file type for '{s}': {any}", .{ path, file_type }); return switch (file_type) { .index => blk: { @@ -225,7 +225,7 @@ pub const TensorReader = struct { .stream = stream, .discard = discard, }, - .buffer = buffer, + .buffer = &.{}, .seek = 0, .end = 0, }, From 850c8f31529437e7d5ee50064ae6bc5acf693077 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 16 Dec 2025 17:58:02 +0100 Subject: [PATCH 144/881] Re-introduced zml/tokenizer --- zml2/BUILD.bazel | 1 + zml2/tokenizer/BUILD.bazel | 41 + zml2/tokenizer/hftokenizers/BUILD.bazel | 32 + zml2/tokenizer/hftokenizers/Cargo.lock | 713 ++++++++++ zml2/tokenizer/hftokenizers/Cargo.toml | 15 + zml2/tokenizer/hftokenizers/hftokenizers.h | 17 + zml2/tokenizer/hftokenizers/hftokenizers.rs | 109 ++ zml2/tokenizer/hftokenizers/hftokenizers.zig | 120 ++ zml2/tokenizer/homemade.zig | 1248 +++++++++++++++++ zml2/tokenizer/main.zig | 66 + zml2/tokenizer/sentencepiece/BUILD.bazel | 35 + zml2/tokenizer/sentencepiece/sentencepiece.i | 92 ++ .../tokenizer/sentencepiece/sentencepiece.zig | 191 +++ zml2/tokenizer/tokenizer.zig | 138 ++ zml2/zml.zig | 1 + 15 files changed, 2819 insertions(+) create mode 100644 zml2/tokenizer/BUILD.bazel create mode 100644 zml2/tokenizer/hftokenizers/BUILD.bazel create mode 100644 zml2/tokenizer/hftokenizers/Cargo.lock create mode 100644 zml2/tokenizer/hftokenizers/Cargo.toml create mode 100644 zml2/tokenizer/hftokenizers/hftokenizers.h create mode 100644 zml2/tokenizer/hftokenizers/hftokenizers.rs create mode 100644 zml2/tokenizer/hftokenizers/hftokenizers.zig create mode 100644 zml2/tokenizer/homemade.zig create mode 100644 zml2/tokenizer/main.zig create mode 100644 zml2/tokenizer/sentencepiece/BUILD.bazel create mode 100644 zml2/tokenizer/sentencepiece/sentencepiece.i create mode 100644 zml2/tokenizer/sentencepiece/sentencepiece.zig create mode 100644 zml2/tokenizer/tokenizer.zig diff --git a/zml2/BUILD.bazel b/zml2/BUILD.bazel index 4abf9ad8..ac8f913a 100644 --- a/zml2/BUILD.bazel +++ b/zml2/BUILD.bazel @@ -56,6 +56,7 @@ zig_library( "//runtimes", "//stdx", "//upb", + "//zml2/tokenizer", "//zml2/io", "@rules_zig//zig/runfiles", ], diff --git a/zml2/tokenizer/BUILD.bazel b/zml2/tokenizer/BUILD.bazel new file mode 100644 index 00000000..f34e30b3 --- /dev/null +++ b/zml2/tokenizer/BUILD.bazel @@ -0,0 +1,41 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_test") +load("@zml//bazel:zig_srcs.bzl", "zig_srcs") + +zig_library( + name = "tokenizer", + srcs = ["homemade.zig"], + import_name = "zml/tokenizer", + main = "tokenizer.zig", + visibility = ["//visibility:public"], + deps = [ + "//ffi:zig", + "//stdx", + "//zml2/tokenizer/hftokenizers", + "//zml2/tokenizer/sentencepiece", + ], +) + +zig_binary( + name = "main", + main = "main.zig", + visibility = ["//visibility:public"], + deps = [ + ":tokenizer", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "homemade.zig", + visibility = ["//visibility:public"], + deps = [ + "//ffi:zig", + "//stdx", + ], +) + +zig_srcs( + name = "sources", + zig_lib = ":tokenizer", +) diff --git a/zml2/tokenizer/hftokenizers/BUILD.bazel b/zml2/tokenizer/hftokenizers/BUILD.bazel new file mode 100644 index 00000000..a07d855b --- /dev/null +++ b/zml2/tokenizer/hftokenizers/BUILD.bazel @@ -0,0 +1,32 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_rust//rust:defs.bzl", "rust_static_library") +load("@rules_zig//zig:defs.bzl", "zig_library") + +rust_static_library( + name = "hftokenizers_rs", + srcs = ["hftokenizers.rs"], + crate_name = "zml_tokenizer_hftokenizers", + edition = "2021", + deps = ["@crates//:tokenizers"], +) + +cc_library( + name = "hftokenizers_cc", + hdrs = ["hftokenizers.h"], + visibility = ["//visibility:public"], + deps = [ + ":hftokenizers_rs", + "//ffi:cc", + ], +) + +zig_library( + name = "hftokenizers", + main = "hftokenizers.zig", + visibility = ["//visibility:public"], + deps = [ + ":hftokenizers_cc", + "//ffi:zig", + "//stdx", + ], +) diff --git a/zml2/tokenizer/hftokenizers/Cargo.lock b/zml2/tokenizer/hftokenizers/Cargo.lock new file mode 100644 index 00000000..68d1a36c --- /dev/null +++ b/zml2/tokenizer/hftokenizers/Cargo.lock @@ -0,0 +1,713 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "serde", + "version_check", + "zerocopy 0.8.26", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "bitflags" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.2.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "compact_str" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "darling" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dary_heap" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728" +dependencies = [ + "serde", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn", +] + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" +dependencies = [ + "cc", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" + +[[package]] +name = "libc" +version = "0.2.169" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "macro_rules_attribute" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "monostate" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e" +dependencies = [ + "monostate-impl", + "serde", +] + +[[package]] +name = "monostate-impl" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "onig" +version = "6.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0" +dependencies = [ + "bitflags", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy 0.7.35", +] + +[[package]] +name = "proc-macro2" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools", + "rayon", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rustversion" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "serde" +version = "1.0.216" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.216" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.134" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64", + "nom", + "serde", + "unicode-segmentation", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokenizers" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476" +dependencies = [ + "ahash", + "aho-corasick", + "compact_str", + "dary_heap", + "derive_builder", + "esaxx-rs", + "getrandom", + "itertools", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + +[[package]] +name = "unicode-ident" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" + +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive 0.7.35", +] + +[[package]] +name = "zerocopy" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" +dependencies = [ + "zerocopy-derive 0.8.26", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zml_tokenizer_hftokenizers" +version = "0.1.0" +dependencies = [ + "tokenizers", +] diff --git a/zml2/tokenizer/hftokenizers/Cargo.toml b/zml2/tokenizer/hftokenizers/Cargo.toml new file mode 100644 index 00000000..8054cd49 --- /dev/null +++ b/zml2/tokenizer/hftokenizers/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "zml_tokenizer_hftokenizers" +version = "0.1.0" +edition = "2021" + +[dependencies] +tokenizers = { version = "0.21.4", default-features = false, features = [ + # We don't need/want the progressbar feature + "onig", + "esaxx_fast", +] } + +[lib] +name = "zml_tokenizer_hftokenizers" +path = "hftokenizers.rs" diff --git a/zml2/tokenizer/hftokenizers/hftokenizers.h b/zml2/tokenizer/hftokenizers/hftokenizers.h new file mode 100644 index 00000000..f779b575 --- /dev/null +++ b/zml2/tokenizer/hftokenizers/hftokenizers.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include + +#include "ffi/zig_slice.h" + +typedef struct hftokenizers hftokenizers; + +hftokenizers *hftokenizers_new(zig_slice); +hftokenizers *hftokenizers_new_from_bytes(zig_slice); +void hftokenizers_drop(hftokenizers *tokenizer); +zig_slice hftokenizers_encode(hftokenizers *tokenizer, zig_slice text); +void hftokenizers_tokens_drop(zig_slice tokens); +zig_slice hftokenizers_decode(hftokenizers *tokenizer, zig_slice tokens); +void hftokenizers_str_drop(zig_slice text); +uint32_t hftokenizers_token_to_id(hftokenizers *tokenizer, zig_slice token); diff --git a/zml2/tokenizer/hftokenizers/hftokenizers.rs b/zml2/tokenizer/hftokenizers/hftokenizers.rs new file mode 100644 index 00000000..048283cc --- /dev/null +++ b/zml2/tokenizer/hftokenizers/hftokenizers.rs @@ -0,0 +1,109 @@ +#[repr(C)] +struct ZigSlice { + ptr: *mut T, + len: usize, +} + +impl ZigSlice { + fn as_slice(&self) -> &[T] { + unsafe { std::slice::from_raw_parts(self.ptr, self.len) } + } + + fn as_slice_mut(&self) -> &mut [T] { + unsafe { std::slice::from_raw_parts_mut(self.ptr, self.len) } + } +} + +#[no_mangle] +extern "C" fn hftokenizers_new(path: ZigSlice) -> *mut tokenizers::Tokenizer { + return Box::into_raw(Box::new( + tokenizers::Tokenizer::from_file(std::path::Path::new( + std::str::from_utf8(path.as_slice()).unwrap(), + )) + .unwrap(), + )); +} + +#[no_mangle] +extern "C" fn hftokenizers_new_from_bytes(bytes: ZigSlice) -> *mut tokenizers::Tokenizer { + return Box::into_raw(Box::new( + tokenizers::Tokenizer::from_bytes(bytes.as_slice()) + .unwrap(), + )); +} + +#[no_mangle] +extern "C" fn hftokenizers_drop(t: *mut tokenizers::Tokenizer) { + drop(unsafe { Box::from_raw(t) }); +} + +#[no_mangle] +extern "C" fn hftokenizers_encode( + t: *mut tokenizers::Tokenizer, + string: ZigSlice, +) -> ZigSlice { + let input_str = std::str::from_utf8(string.as_slice()).unwrap(); + let encoded = unsafe { t.as_ref() } + .unwrap() + .encode_fast(input_str, false) + .unwrap(); + + // Convert the result to a boxed slice + let mut ids: Box<[u32]> = encoded.get_ids().to_owned().into_boxed_slice(); + + // Retrieve the zig slice associated to the boxed slice. + let slice = ZigSlice { + ptr: ids.as_mut_ptr(), + len: ids.len(), + }; + + // Leak the box so that it's not deallocated. + Box::leak(ids); + + return slice; +} + +#[no_mangle] +extern "C" fn hftokenizers_tokens_drop(tokens: ZigSlice) { + // Reconstruct the Box from the zig slice so that it's dropped. + drop(unsafe { Box::from_raw(tokens.as_slice_mut()) }); +} + +#[no_mangle] +extern "C" fn hftokenizers_decode( + t: *mut tokenizers::Tokenizer, + ids: ZigSlice, +) -> ZigSlice { + let decoded = unsafe { t.as_ref() } + .unwrap() + .decode(ids.as_slice(), false) + .unwrap(); + + // Convert the result to a boxed slice + let mut string: Box<[u8]> = decoded.into_bytes().into_boxed_slice(); + + // Retrieve the zig slice associated to the boxed slice. + let slice = ZigSlice { + ptr: string.as_mut_ptr(), + len: string.len(), + }; + + // Leak the box so that it's not deallocated. + Box::leak(string); + + return slice; +} + +#[no_mangle] +extern "C" fn hftokenizers_str_drop(tokens: ZigSlice) { + drop(unsafe { Box::from_raw(tokens.as_slice_mut()) }); +} + +#[no_mangle] +extern "C" fn hftokenizers_token_to_id(t: *mut tokenizers::Tokenizer, token: ZigSlice) -> u32 { + let id = unsafe { t.as_ref() } + .unwrap() + .token_to_id(std::str::from_utf8(token.as_slice()).unwrap()) + .unwrap_or(u32::MAX); + return id; +} diff --git a/zml2/tokenizer/hftokenizers/hftokenizers.zig b/zml2/tokenizer/hftokenizers/hftokenizers.zig new file mode 100644 index 00000000..0871d6f9 --- /dev/null +++ b/zml2/tokenizer/hftokenizers/hftokenizers.zig @@ -0,0 +1,120 @@ +const std = @import("std"); + +const c = @import("c"); +const ffi = @import("ffi"); +const stdx = @import("stdx"); + +pub const Encoder = struct { + inner: *HFTokenizer, + current_ids: ?[]const u32 = null, + + fn init(inner: *HFTokenizer) Encoder { + return .{ .inner = inner }; + } + + pub fn reset(self: *Encoder) void { + if (self.current_ids) |current_ids_| { + c.hftokenizers_tokens_drop(ffi.ZigSlice.from(current_ids_)); + self.current_ids = null; + } + } + + pub fn deinit(self: *Encoder) void { + self.reset(); + } + + pub fn encode(self: *Encoder, input: []const u8) ![]const u32 { + self.reset(); + self.current_ids = ffi.ZigSlice.to(u32, c.hftokenizers_encode(@ptrCast(self.inner), ffi.ZigSlice.from(input))); + return self.ids(); + } + + pub fn ids(self: *const Encoder) []const u32 { + return self.current_ids orelse &.{}; + } +}; + +pub const Decoder = struct { + const StringBuffer = stdx.BoundedArray(u8, 128); + const TokensIdsBuffer = stdx.BoundedArray(u32, 4); + + inner: *HFTokenizer, + current_string: ?[]const u8 = null, + last_string: StringBuffer = .{ .len = 0 }, + last_token_ids: TokensIdsBuffer = .{ .len = 0 }, + + fn init(inner: *HFTokenizer) Decoder { + return .{ .inner = inner }; + } + + pub fn deinit(self: *Decoder) void { + self.reset(); + } + + pub fn reset(self: *Decoder) void { + if (self.current_string) |current_string_| { + c.hftokenizers_str_drop(ffi.ZigSlice.from(current_string_)); + self.current_string = null; + } + } + + pub fn decode(self: *Decoder, ids: []const u32) ![]const u8 { + self.reset(); + self.current_string = ffi.ZigSlice.to(u8, c.hftokenizers_decode(@ptrCast(self.inner), ffi.ZigSlice.from(ids))); + return self.string(); + } + + pub fn string(self: *const Decoder) []const u8 { + return self.current_string orelse &.{}; + } + + pub fn next(self: *Decoder, token_id: u32) !?[]const u8 { + if (self.last_token_ids.len >= self.last_token_ids.capacity()) { + _ = self.last_token_ids.orderedRemove(0); + } + self.last_token_ids.appendAssumeCapacity(token_id); + const new_string = try self.decode(self.last_token_ids.constSlice()); + if (self.last_string.len == 0) { + self.last_string = try StringBuffer.fromSlice(new_string); + return new_string; + } + var view = try std.unicode.Utf8View.init(self.last_string.constSlice()); + var it = view.iterator(); + while (it.nextCodepointSlice()) |cp| { + const start = it.i - cp.len; + if (std.mem.startsWith(u8, new_string, self.last_string.constSlice()[start..])) { + const chunk = new_string[self.last_string.len - start ..]; + self.last_string = try StringBuffer.fromSlice(new_string); + return chunk; + } + } + return null; + } +}; + +pub const HFTokenizer = opaque { + pub fn fromFile(model: []const u8) !*HFTokenizer { + return @ptrCast(c.hftokenizers_new(ffi.ZigSlice.from(model))); + } + + pub fn fromBytes(bytes: []const u8) !*HFTokenizer { + return @ptrCast(c.hftokenizers_new_from_bytes(ffi.ZigSlice.from(bytes))); + } + + pub fn deinit(self: *HFTokenizer) void { + return c.hftokenizers_drop(@ptrCast(self)); + } + + pub fn encoder(self: *HFTokenizer) !Encoder { + return Encoder.init(self); + } + + pub fn decoder(self: *HFTokenizer) !Decoder { + return Decoder.init(self); + } + + pub fn tokenToId(self: *HFTokenizer, token: []const u8) ?u32 { + const id = c.hftokenizers_token_to_id(@ptrCast(self), ffi.ZigSlice.from(token)); + return if (id == std.math.maxInt(u32)) null else id; + } +}; diff --git a/zml2/tokenizer/homemade.zig b/zml2/tokenizer/homemade.zig new file mode 100644 index 00000000..95274bc8 --- /dev/null +++ b/zml2/tokenizer/homemade.zig @@ -0,0 +1,1248 @@ +//! Text tokenizer implementations +//! Disclaimer this is not a very robust implementation: +//! In particular the normalization is pretty minimalist, only works with ascii, and don't do unicode normalization. +//! Mostly used for testing models that don't have an official HF/sentencepiece tokenizer. +const std = @import("std"); +const testing = std.testing; +const builtin = @import("builtin"); + +const stdx = @import("stdx"); + +const log = std.log.scoped(.@"zml/tokenizer"); + +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(Normalizer); + std.testing.refAllDecls(Tokenizer); +} + +/// Byte Pair Encoding tokenizer generally used for LLM. +pub const Tokenizer = struct { + tokens: [][]const u8, + token_lookup: std.StringHashMapUnmanaged(u32), + special_tokens: SpecialTokens, + + scores: []f32, + max_token_len: u32, + normalizer: ?Normalizer, + // Allows to split unknown unicode characters into bytes. + byte_fallback: bool = false, + + arena_state: std.heap.ArenaAllocator, + vocab_size: u32, + next_token_id: u32 = 0, + + pub const SpecialTokens = struct { + eos: u32, + bos: u32, + unk: u32, + pad: u32 = std.math.maxInt(u32), + hard_space: u32 = std.math.maxInt(u32), + }; + + pub fn init( + allocator: std.mem.Allocator, + vocab_size: u32, + max_token_len: u32, + normalizer: ?Normalizer, + special_tokens: SpecialTokens, + alloc_tokens: bool, + ) !Tokenizer { + var arena_state = std.heap.ArenaAllocator.init(allocator); + errdefer arena_state.deinit(); + const arena = arena_state.allocator(); + + var token_lookup: std.StringHashMapUnmanaged(u32) = .{}; + errdefer token_lookup.deinit(arena); + + try token_lookup.ensureTotalCapacity(arena, @intCast(vocab_size)); + + const tokens: [][]const u8 = if (alloc_tokens) try arena.alloc([]const u8, vocab_size) else &.{}; + errdefer if (alloc_tokens) arena.free(tokens); + + const scores: []f32 = if (alloc_tokens) try arena.alloc(f32, vocab_size) else &.{}; + errdefer if (alloc_tokens) arena.free(scores); + + return .{ + .tokens = tokens, + .scores = scores, + .max_token_len = max_token_len, + .token_lookup = token_lookup, + .arena_state = arena_state, + .normalizer = normalizer, + .vocab_size = vocab_size, + .special_tokens = special_tokens, + }; + } + + pub fn deinit(self: Tokenizer) void { + self.arena_state.deinit(); + } + + pub fn encoder(self: *Tokenizer) !Encoder { + return Encoder.init(self); + } + + pub fn decoder(self: *Tokenizer) !Decoder { + return Decoder.init(self); + } + + /// Reads a new word directly into the tokenizer arena. + pub fn readTokenInto(self: *Tokenizer, score: f32, len: usize, tok_reader: *std.Io.Reader) !void { + const arena = self.arena_state.allocator(); + + const token = try arena.alloc(u8, len); + try tok_reader.readSliceAll(token); + return self.addOwnedToken(score, token); + } + + /// Adds a new token (and copy it) + pub fn addToken(self: *Tokenizer, score: f32, token: []const u8) !void { + const arena = self.arena_state.allocator(); + return self.addOwnedToken(score, try arena.dupe(u8, token)); + } + + /// Adds a new token (without copying it) + pub fn addOwnedToken(self: *Tokenizer, score: f32, token: []const u8) void { + const i = self.next_token_id; + std.debug.assert(i < self.vocab_size); + self.next_token_id += 1; + + self.scores[i] = score; + self.tokens[i] = token; + const v = self.token_lookup.getOrPutAssumeCapacity(token); + if (!v.found_existing) { + v.value_ptr.* = i; + } + } + + pub fn addOwnedTokenByIndex(self: *Tokenizer, i: u32, score: f32, token: []const u8) void { + std.debug.assert(i < self.vocab_size); + self.next_token_id += 1; + self.scores[i] = score; + self.tokens[i] = token; + const v = self.token_lookup.getOrPutAssumeCapacity(token); + if (!v.found_existing) { + v.value_ptr.* = @intCast(i); + } + } + + pub fn lookup(self: *const Tokenizer, str: []const u8) ?u32 { + return self.token_lookup.get(str); + } + + pub fn tokenToId(self: *const Tokenizer, token: []const u8) ?u32 { + return self.token_lookup.get(token); + } + + pub const EncodeOptions = struct { + /// Should the beginning of sentence '' token be added. + add_bos: bool = true, + add_eos: bool = false, + pad_to: u32 = 0, + // Print tokenization intermediary steps. + debug: bool = false, + }; + + pub fn encode(self: *const Tokenizer, allocator: std.mem.Allocator, raw: []const u8, options: EncodeOptions) ![]u32 { + if (options.debug) log.debug("Tokenizer.encode('{s}')", .{raw}); + const input = if (self.normalizer) |n| try n.normalize(allocator, raw) else raw; + defer if (self.normalizer) |_| allocator.free(input); + if (options.debug) log.debug("Tokenizer.encode.normalize -> '{s}'", .{input}); + + // Allocate a buffer that can fit all indices as well as extra character if requested. + // We then slice it so that the token merging code doesn't see the bos token. + const tok_buff_alloc = try allocator.alloc(u32, @max(options.pad_to, input.len + 2)); + const tok_buff = if (options.add_bos) tok_buff_alloc[1..] else tok_buff_alloc; + + const MergeState = union(enum) { ready: u32, nope, hard_space, idk }; + const mergeable = try allocator.alloc(MergeState, tok_buff.len); + + var num_tokens: usize = 0; + var it: CharTokenIterator = .{ .input = input }; + while (try it.nextCodepointToken(self)) |token| : (num_tokens += 1) { + tok_buff[num_tokens] = token; + mergeable[num_tokens] = if (token == self.special_tokens.hard_space) + .hard_space + else + .idk; + } + + var stable_prefix: usize = 0; + var stable_off: usize = 0; + while (true) { + // This code is a bit overcomplicated cause I'm abstracting over two algorithms: + // BPE and sentencepiece unigram model. + // Normally BPE is pre-split on spaces then the regular merge algorithm is applied. + // With unigram model you work at sentence level and you handle spaces as you would any other bytes, + // hoping the final tokens mostly align with spaces. + // This seemed like a good idea, but is kinda bad because I had to add special code to speed up BPE + // by detecting when the first "word" is treated and can be safely removed from sequence. + // Also it doesn't work well with BPE vocab which have multi-space tokens (for indentation) + // and have custom splitting rules. + // This is fine for now cause we now have bindings to HF tokenizers for complexe use cases + // and are only using this for tinyllama/gguf models. + // If we come back to use this in production, the implementation would gain in speed/clarity + // by splitting in two. + // The merging token logic isn't that complicated anyway. + + // Step by step visualization of the progress. + if (options.debug) { + var _debug_buf: [256]u8 = undefined; + var debug_progress: std.Io.Writer = .fixed(&_debug_buf); + self.decodeWithOpts(tok_buff[0..num_tokens], &debug_progress, .{ .sep = "|" }) catch {}; + log.debug("tokens: {any} -> {s}", .{ tok_buff[0..num_tokens], debug_progress.buffered() }); + } + var best_score: f32 = -1e10; + var best_token: u32 = 0; + var best_idx: ?usize = null; + var input_off: usize = stable_off; + + // Find best tokens to merge in all available tokens + for (stable_prefix..num_tokens - 1) |i| { + if (tok_buff[i] == self.special_tokens.unk) { + input_off += 1; + continue; + } + const cur_tok = self.tokens[tok_buff[i]]; + defer input_off += cur_tok.len; + + // Lookup merge for current token, if not already done. + switch (mergeable[i]) { + .nope => continue, + .ready => {}, + .hard_space => { + // Since tokens are not allowed to merge through hard sep, + // we don't need to merge the sentence-wide best token. + // We can just merge the best token since beginning. + if (best_idx != null) break; + // OTOH if there was no merge possible since beginning, + // we can skip the beginning in future iterations. + stable_prefix = i + 1; + stable_off = input_off + cur_tok.len; + continue; + }, + .idk => { + + // Special tokens can't be concatenated. + if (builtin.mode == .Debug and tok_buff[i] != self.special_tokens.unk) { + // Detects memory corruption of tokens. + if (cur_tok.len == 0 or cur_tok.len > self.max_token_len) @panic("Token looks corrupted !"); + + if (!std.mem.eql(u8, cur_tok, input[input_off..][0..cur_tok.len])) { + log.err("current token '{s}' not found in input string '{s}' !", .{ cur_tok, input[input_off..] }); + @panic("invalid tokenization"); + } + } + const next_tok = self.tokens[tok_buff[i + 1]]; + // if `next_tok` is `.unk`, length is 1; otherwise, it's the length of the token. + const next_tok_len = if (tok_buff[i + 1] == self.special_tokens.unk) 1 else next_tok.len; + const concat_tokens = input[input_off..][0 .. cur_tok.len + next_tok_len]; + // Save the result + mergeable[i] = if (self.lookup(concat_tokens)) |tok| + .{ .ready = tok } + else + .nope; + }, + } + + switch (mergeable[i]) { + .idk, .hard_space => unreachable, + .nope => continue, + .ready => |tok| { + if (self.scores[tok] > best_score) { + best_score = self.scores[tok]; + best_token = tok; + best_idx = i; + } + }, + } + } + + if (best_idx) |bidx| { + // Apply the merge. + tok_buff[bidx] = best_token; + std.mem.copyForwards(u32, tok_buff[bidx + 1 ..], tok_buff[bidx + 2 .. num_tokens]); + std.mem.copyForwards(MergeState, mergeable[bidx + 1 ..], mergeable[bidx + 2 .. num_tokens]); + num_tokens -= 1; + // We got two new merge lookups to do. + mergeable[bidx] = .idk; + if (bidx > 0 and mergeable[bidx - 1] != .hard_space) mergeable[bidx - 1] = .idk; + } else { + // No merge candidate => we are done ! + break; + } + } + + if (options.add_eos) { + tok_buff[num_tokens] = self.special_tokens.eos; + num_tokens += 1; + } + if (options.add_bos) { + tok_buff_alloc[0] = self.special_tokens.bos; + num_tokens += 1; + } + if (num_tokens < options.pad_to) { + for (num_tokens..options.pad_to) |i| { + tok_buff_alloc[i] = self.special_tokens.pad; + } + num_tokens = options.pad_to; + } + + // Release extra memory we don't need anymore. + allocator.free(mergeable); + _ = allocator.resize(tok_buff_alloc, num_tokens); + return tok_buff_alloc[0..num_tokens]; + } + + /// Returns a slice corresponding to the given id. Handles unknown ids and special ids. + pub fn lookupPiece(self: *const Tokenizer, id: usize) []const u8 { + return if (id == self.special_tokens.bos or id == self.special_tokens.eos or id == self.special_tokens.pad) + "" + else if (id == self.special_tokens.unk) + "" + else if (id > self.tokens.len) + "" // this means we received an invalid id, but we didn't want to panic. + else + self.tokens[id]; + } + + /// Converts the given slice of tokens back into bytes. + /// Note that if the tokenizer allows sub-unicode bytes, it's possible + /// the output is not valid utf8. + pub fn decode(self: *const Tokenizer, input: []const u32, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try self.decodeWithOpts(input, writer, .{}); + } + + pub fn decodeWithOpts( + self: *const Tokenizer, + input: []const u32, + output: *std.Io.Writer, + opts: struct { sep: []const u8 = "" }, + ) std.Io.Writer.Error!void { + const escaped = if (self.normalizer) |n| n.escapedSpace() else null; + // Flag used to indicate if the first dummy whitespace has been consumed. + var first_token: bool = true; + for (input) |id| { + // Retrieve the slice corresponding to the id. + var piece = self.lookupPiece(id); + + // Convert `▁` to a regular space. + if (escaped) |escspc| { + // we modify piece inside the loop, so we can use it in the condition + while (std.mem.startsWith(u8, piece, escspc)) { + piece = piece[escspc.len..]; + // don't output a space at beginning of text. + if (!first_token) try output.writeByte(' '); + } + } + + try output.writeAll(piece); + first_token = false; + try output.writeAll(opts.sep); + } + } + + /// Some tokenizers have bytes encoded in hex like this: "<0x40>". + /// This break the tokenization algorithm because the input text + /// will contain "@" not "<0x40>", + /// and if the input contains "<0x40>" it needs to not be treated as a single byte. + /// So we replace byte fallbacks strings, by their corresponding character. + /// This enables the normal tokenization algorithm to work. + pub fn rewriteByteFallbackTokens(tokenizer: *Tokenizer) !void { + tokenizer.byte_fallback = true; + var single_bytes = try tokenizer.arena_state.allocator().alloc(u8, 256); + var byte_fallback_buf = "<0x00>".*; + + for (0..256) |i| { + const c: u8 = @truncate(i); + single_bytes[i] = c; + + // First lookup the byte fallback entry. + // Note: we assume upper case, but we could try both upper and lower case if needed. + var writer: std.Io.Writer = .fixed(byte_fallback_buf[3..5]); + try writer.printInt(c, 16, .upper, .{ .fill = '0', .width = 2 }); + const entry = tokenizer.token_lookup.getEntry(&byte_fallback_buf) orelse { + log.err("Tokenizer has \"byte_fallback\" = true, but doesn't contains the byte fallback token {s}", .{byte_fallback_buf}); + return error.InvalidInput; + }; + + // Check if the character is already present in the vocab. + // In that case, nothing to do, + // but note that the fallback token will be "unreachable", + // ie there is no way the tokenizer can produce it. + if (tokenizer.token_lookup.get(&.{c})) |_| continue; + + const idx: u32 = entry.value_ptr.*; + tokenizer.token_lookup.removeByPtr(entry.key_ptr); + tokenizer.addOwnedTokenByIndex(idx, tokenizer.scores[idx], single_bytes[i .. i + 1]); + } + } +}; + +test Tokenizer { + const allocator = std.testing.allocator; + const special_tokens: Tokenizer.SpecialTokens = .{ + .unk = 0, + .bos = 1, + .eos = 2, + }; + + var tokenizer = try Tokenizer.init(allocator, 10, 5, null, special_tokens, true); + defer tokenizer.deinit(); + + try tokenizer.addToken(10, "hello"); + try tokenizer.addToken(3.5, "world"); + + try testing.expect(tokenizer.lookup("hello") == 0); + try testing.expect(tokenizer.lookup("world") == 1); + + // TODO: test Tokenizer.decode, Tokenizer.encode, Tokenizer.readTokenInto +} + +pub const Encoder = struct { + inner: *Tokenizer, + arena: std.heap.ArenaAllocator, + current_ids: []const u32 = &.{}, + + fn init(inner: *Tokenizer) !Encoder { + var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); + // Warmup the arena. Page allocator is expensive, avoid calling it for small reallocs. + _ = try arena.allocator().alloc(u32, 4096); + std.debug.assert(arena.reset(.retain_capacity)); + return .{ .inner = inner, .arena = arena }; + } + + pub fn reset(self: *Encoder) void { + self.current_ids = &.{}; + std.debug.assert(self.arena.reset(.retain_capacity)); + } + + pub fn deinit(self: *Encoder) void { + self.arena.deinit(); + } + + pub fn encode(self: *Encoder, input: []const u8) ![]const u32 { + self.reset(); + const res = try self.inner.encode(self.arena.allocator(), input, .{ + .add_bos = true, + .add_eos = false, + .pad_to = 0, + // Print tokenization intermediary steps. + .debug = false, + }); + self.current_ids = res; + return res; + } + + pub fn ids(self: *const Encoder) []const u32 { + return self.current_ids; + } +}; + +pub const Decoder = struct { + const StringBuffer = stdx.BoundedArray(u8, 128); + const TokensIdsBuffer = stdx.BoundedArray(u32, 4); + + inner: *Tokenizer, + arena: std.heap.ArenaAllocator, + + current_string: ?[]const u8 = null, + last_string: StringBuffer = .{ .len = 0 }, + last_token_ids: TokensIdsBuffer = .{ .len = 0 }, + + fn init(inner: *Tokenizer) !Decoder { + var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); + // Warmup the arena. Page allocator is expensive, avoid calling it for small reallocs. + _ = try arena.allocator().alloc(u32, 4096); + std.debug.assert(arena.reset(.retain_capacity)); + return .{ .inner = inner, .arena = arena }; + } + + pub fn deinit(self: *Decoder) void { + self.arena.deinit(); + } + + pub fn reset(self: *Decoder) void { + std.debug.assert(self.arena.reset(.retain_capacity)); + self.current_string = null; + } + + pub fn decode(self: *Decoder, ids: []const u32) ![]const u8 { + // TODO: revisite tokenizer api to use std.Io.Writer. + self.reset(); + // Reuse the same warmup than in init, to maximize contiguous writes. + var arena_writer = std.Io.Writer.Allocating.initCapacity(self.arena.allocator(), 4096) catch unreachable; + try self.inner.decode(ids, &arena_writer.writer); + self.current_string = arena_writer.written(); + return self.current_string.?; + } + + pub fn string(self: *const Decoder) []const u8 { + return self.current_string; + } + + pub fn next(self: *Decoder, token_id: u32) !?[]const u8 { + if (self.last_token_ids.len >= self.last_token_ids.capacity()) { + _ = self.last_token_ids.orderedRemove(0); + } + self.last_token_ids.appendAssumeCapacity(token_id); + const new_string = try self.decode(self.last_token_ids.constSlice()); + if (self.last_string.len == 0) { + self.last_string = try StringBuffer.fromSlice(new_string); + return new_string; + } + var view = try std.unicode.Utf8View.init(self.last_string.constSlice()); + var it = view.iterator(); + while (it.nextCodepointSlice()) |cp| { + const start = it.i - cp.len; + if (std.mem.startsWith(u8, new_string, self.last_string.constSlice()[start..])) { + const chunk = new_string[self.last_string.len - start ..]; + self.last_string = try StringBuffer.fromSlice(new_string); + return chunk; + } + } + return null; + } +}; + +/// Given a slice, split it in the most simple tokens using the given tokenizer tokens. +/// The output of this can be used to initialize the tokenization algorithm. +/// Normally we split the input text into utf8 codepoint, +/// but if we find an unknown codepoint we either split it in bytes, or use the special "unknown" token, +/// depending on the tokenizer configuration. +const CharTokenIterator = struct { + state: union(enum) { by_codepoint, by_byte: u8 } = .by_codepoint, + input: []const u8, + + fn nextCodepointToken(self: *CharTokenIterator, tokenizer: *const Tokenizer) error{ TruncatedInput, Utf8InvalidStartByte }!?u32 { + if (self.input.len == 0) return null; + return switch (self.state) { + .by_byte => |*byte_left| { + const idx = tokenizer.lookup(self.input[0..1]) orelse { + // Normally this has been caught when calling `rewriteByteFallbackTokens`. + std.debug.panic("Tokenizer has \"byte_fallback\" = true, but doesn't contains the byte fallback for token '<0x{X:02}>'", .{self.input[0]}); + }; + + self.input = self.input[1..]; + byte_left.* -|= 1; + if (byte_left.* == 0) self.state = .by_codepoint; + return idx; + }, + .by_codepoint => { + // Try to lookup valid utf8 codepoint first. + const utf8_len = try std.unicode.utf8ByteSequenceLength(self.input[0]); + if (self.input.len < utf8_len) return error.TruncatedInput; + if (tokenizer.lookup(self.input[0..utf8_len])) |idx| { + self.input = self.input[utf8_len..]; + return idx; + } + + // Otherwise split in bytes if it's allowed. + if (tokenizer.byte_fallback) { + // TODO: replace this by a continue statement next time we bump Zig. + self.state = .{ .by_byte = utf8_len }; + return self.nextCodepointToken(tokenizer); + } + + // Or mark the full utf8 codepoint as unknown. + log.debug("Token not found for char '{s}'", .{self.input[0..utf8_len]}); + self.input = self.input[utf8_len..]; + return tokenizer.special_tokens.unk; + }, + }; + } +}; + +test CharTokenIterator { + const special_tokens: Tokenizer.SpecialTokens = .{ .unk = 0, .bos = 1, .eos = 2 }; + var tokenizer = try Tokenizer.init(std.testing.allocator, 16, 4, null, special_tokens, true); + defer tokenizer.deinit(); + + tokenizer.addOwnedToken(1.0, ""); // 0 + tokenizer.addOwnedToken(1.0, ""); // 1 + tokenizer.addOwnedToken(1.0, ""); // 2 + tokenizer.addOwnedToken(1.0, "ζ"); // 3 + tokenizer.addOwnedToken(1.0, &.{0xE2}); // 4: ℳ, first byte + tokenizer.addOwnedToken(1.0, &.{0x84}); // 5: ℳ, second byte + tokenizer.addOwnedToken(1.0, &.{0xB3}); // 6: ℳ, third byte + tokenizer.addOwnedToken(1.0, "L"); // 7 + + // No byte fallback + { + tokenizer.byte_fallback = false; + var it: CharTokenIterator = .{ .input = "ζℳL" }; + var res: stdx.BoundedArray(u32, 8) = .{}; + while (try it.nextCodepointToken(&tokenizer)) |token| { + res.appendAssumeCapacity(token); + } + try std.testing.expectEqualSlices(u32, &[_]u32{ 3, 0, 7 }, res.constSlice()); + } + + // with byte fallback + { + tokenizer.byte_fallback = true; + var it: CharTokenIterator = .{ .input = "ζℳL" }; + var res: stdx.BoundedArray(u32, 8) = .{}; + while (try it.nextCodepointToken(&tokenizer)) |token| { + res.appendAssumeCapacity(token); + } + try std.testing.expectEqualSlices(u32, &[_]u32{ 3, 4, 5, 6, 7 }, res.constSlice()); + } +} + +/// Text normalizer. +/// Most tokenizer assumes the input text have been prepocessed with on of those. +pub const Normalizer = struct { + /// Space token used by sentencepiece derived tokenizer. + pub const sentencepiece_space = "▁"; // \xe2\x96\x81 + + _whitespace: stdx.BoundedArray(u8, 8) = .{}, + + flags: packed struct { + remove_extra_whitespaces: bool, + add_dummy_prefix: bool, + add_dummy_suffix: bool, + /// Cheap lower casing. + /// TODO: try to match Python "lower" + lower_case_ascii: bool, + /// cheap ascii punct splitting. + // doing this processing ahead of time simplifies the logic + split_on_punct_ascii: bool, + }, + + pub fn init(flags: @FieldType(Normalizer, "flags"), escaped_whitespace: ?[]const u8) Normalizer { + var res: Normalizer = .{ .flags = flags }; + if (escaped_whitespace) |escaped| { + res._whitespace.appendSliceAssumeCapacity(escaped); + } + return res; + } + + pub inline fn escapedSpace(self: Normalizer) ?[]const u8 { + return if (self._whitespace.len > 1) self._whitespace.constSlice() else null; + } + + fn addSlice(allocator: std.mem.Allocator, data: []const u8, consumed: usize, normalized: *std.ArrayList(u8), normalized_to_origin: *std.ArrayList(usize)) !void { + try normalized.appendSlice(allocator, data); + for (data) |_| try normalized_to_origin.append(allocator, consumed); + } + + pub const Result = struct { + /// Normalized string + normalized: []const u8, + /// Mapping between chars in the original string and chars in the new string + normalized_to_origin: []const usize, + + pub fn deinit(self: Result, allocator: std.mem.Allocator) void { + allocator.free(self.normalized); + allocator.free(self.normalized_to_origin); + } + }; + + /// Simplifed version of Sentencepiece normalizer. + /// + /// Llama2 uses a normalizer called "identity" so this basically only handles trailing + /// whitespaces and replaces whitespace with the "▁" (U+2581) character. + pub fn normalize(self: Normalizer, allocator: std.mem.Allocator, input: []const u8) ![]const u8 { + const res = try self.normalizeWithMapping(allocator, input); + allocator.free(res.normalized_to_origin); + return res.normalized; + } + + /// Returns both the normalized string and a mapping between the normalized string and the original. + pub fn normalizeWithMapping(self: Normalizer, allocator: std.mem.Allocator, input: []const u8) !Result { + // Number of bytes consumed from the input. + var consumed: usize = 0; + var trimmed_input = input; + + // Skip leading whitespaces. + if (self.flags.remove_extra_whitespaces) { + while (trimmed_input.len != 0) { + if (trimmed_input[0] != ' ') break; + trimmed_input = trimmed_input[1..]; + consumed += 1; + } + } + + // If the trimmed input is empty, we are done. + if (trimmed_input.len == 0) { + return .{ .normalized = &.{}, .normalized_to_origin = &.{} }; + } + + // Pre-allocate outputs + const space = self.escapedSpace() orelse " "; + const overhead = if (self.flags.split_on_punct_ascii) space.len + 1 else space.len; + var normalized: std.ArrayList(u8) = try .initCapacity(allocator, trimmed_input.len * overhead + 2 * space.len); + errdefer normalized.deinit(allocator); + var normalized_to_origin: std.ArrayList(usize) = try .initCapacity(allocator, normalized.capacity); + errdefer normalized_to_origin.deinit(allocator); + + // If the spec asks for it, add a whitespace at the beginning. + if (self.flags.add_dummy_prefix) try addSlice(allocator, space, consumed, &normalized, &normalized_to_origin); + + var is_prev_space: bool = true; + var is_prev_word: bool = false; + + while (trimmed_input.len != 0) { + // NOTE(Corendos): This might feel weird but normally the slice we get comes from a normalizing process and can contain multiple codepoints. + // Since we have an "identity" normalizer, each slice is actually a unicode character. + const multibyte_length = try std.unicode.utf8ByteSequenceLength(trimmed_input[0]); + var slice = trimmed_input[0..multibyte_length]; + const origin = consumed; + consumed += multibyte_length; + trimmed_input = trimmed_input[multibyte_length..]; + + if (self.flags.remove_extra_whitespaces and is_prev_space) { + while (slice.len > 0 and slice[0] == ' ') { + slice = slice[1..]; + } + if (slice.len == 0) continue; + } + is_prev_space = slice[slice.len - 1] == ' '; + + if (slice.len == 1) ascii: { + // The more advanced logic only works with ascii atm + var byte = slice[0]; + if (self.escapedSpace() != null and byte == ' ') { + // replace the space token by the special token + try addSlice(allocator, space, origin, &normalized, &normalized_to_origin); + is_prev_word = false; + break :ascii; + } else if (self.flags.split_on_punct_ascii) { + if (is_prev_word and isPunct(slice)) { + // Insert a space, but continue handling the rest + try addSlice(allocator, space, origin, &normalized, &normalized_to_origin); + } + } + if (self.flags.lower_case_ascii) { + byte = std.ascii.toLower(byte); + } + try normalized.append(allocator, byte); + try normalized_to_origin.append(allocator, origin); + } else { + // we can safely copy to the output. + try addSlice(allocator, slice, origin, &normalized, &normalized_to_origin); + } + is_prev_word = !is_prev_space and !isPunct(slice); + } + + // Skip trailing whitespaces + if (self.flags.remove_extra_whitespaces) { + while (std.mem.endsWith(u8, normalized.items, space)) { + const length = normalized.items.len - space.len; + consumed = normalized_to_origin.items[length]; + try normalized.resize(allocator, length); + try normalized_to_origin.resize(allocator, length); + } + } + + try normalized_to_origin.append(allocator, consumed); + + std.debug.assert(normalized_to_origin.items.len == normalized.items.len + 1); + + if (self.flags.add_dummy_suffix) try addSlice(allocator, space, consumed, &normalized, &normalized_to_origin); + + return .{ + .normalized_to_origin = try normalized_to_origin.toOwnedSlice(allocator), + .normalized = try normalized.toOwnedSlice(allocator), + }; + } + + pub fn wellKnown(impl: KnownImplementation) Normalizer { + return switch (impl) { + .sentencepiece => init(.{ + .remove_extra_whitespaces = true, + .add_dummy_prefix = true, + .add_dummy_suffix = false, + .lower_case_ascii = false, + .split_on_punct_ascii = false, + }, sentencepiece_space), + .llama3 => init(.{ + .remove_extra_whitespaces = true, + .add_dummy_prefix = false, + .add_dummy_suffix = false, + .lower_case_ascii = false, + .split_on_punct_ascii = false, + }, null), + .gpt2 => init(.{ + .remove_extra_whitespaces = true, + .add_dummy_prefix = true, + .add_dummy_suffix = false, + .lower_case_ascii = false, + .split_on_punct_ascii = false, + }, null), + }; + } + + pub fn fromHfJson(config: std.json.ObjectMap) error{InvalidNormalizerJson}!Normalizer { + var normalizer: Normalizer = .{ .flags = .{ + .remove_extra_whitespaces = false, + .add_dummy_suffix = false, + .add_dummy_prefix = false, + .lower_case_ascii = false, + .split_on_punct_ascii = false, + } }; + + // Normalizer config can be a single normalizer, or a sequence of normalizers. + const maybe_steps = objectGet(config, .array, "normalizers"); + const steps = if (maybe_steps) |st| st.items else &.{std.json.Value{ .object = config }}; + + for (steps) |step_val| { + if (step_val != .object) { + return error.InvalidNormalizerJson; + } + const step = step_val.object; + + const step_type = objectGet(step, .string, "type") orelse { + return error.InvalidNormalizerJson; + }; + if (std.mem.eql(u8, "Prepend", step_type)) { + normalizer.flags.add_dummy_prefix = true; + } else if (std.mem.eql(u8, "Append", step_type)) { + normalizer.flags.add_dummy_suffix = true; + } else if (std.mem.eql(u8, "Replace", step_type)) { + const pattern = objectGet(step, .object, "pattern") orelse return error.InvalidNormalizerJson; + const str_pattern = objectGet(pattern, .string, "String") orelse return error.InvalidNormalizerJson; + + if (std.mem.eql(u8, str_pattern, " ")) { + normalizer._whitespace.appendSliceAssumeCapacity( + objectGet(step, .string, "content") orelse return error.InvalidNormalizerJson, + ); + } else { + log.warn("Normalizer Replace pattern not supported: '{s}' -> '{s}'", .{ str_pattern, objectGet(pattern, .string, "content") orelse "" }); + } + } else { + log.warn("Unknown normalizer type: {s}", .{step_type}); + } + } + + return normalizer; + } + + test "Normalizer.fromHfJson" { + const config_json = + \\{ + \\ "type": "Sequence", + \\ "normalizers": [ + \\ { + \\ "type": "Prepend", + \\ "prepend": "▁" + \\ }, + \\ { + \\ "type": "Replace", + \\ "pattern": { + \\ "String": " " + \\ }, + \\ "content": "▁" + \\ } + \\ ] + \\} + ; + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const config = try std.json.parseFromSliceLeaky(std.json.Value, arena.allocator(), config_json, .{}); + const normalizer = try Normalizer.fromHfJson(config.object); + + const expected = Normalizer{ + ._whitespace = .{ .buffer = [_]u8{ 0xe2, 0x96, 0x81 } ++ [_]u8{0} ** 5, .len = 3 }, + .flags = .{ + .remove_extra_whitespaces = false, + .add_dummy_prefix = true, + .add_dummy_suffix = false, + .lower_case_ascii = false, + .split_on_punct_ascii = false, + }, + }; + try std.testing.expectEqual(expected.flags, normalizer.flags); + try std.testing.expectEqualStrings(expected.escapedSpace().?, normalizer.escapedSpace().?); + } +}; +pub const KnownImplementation = enum(u8) { + sentencepiece, + gpt2, + llama3, +}; + +fn isPunct(unicode_char: []const u8) bool { + // TODO use unicode categories + if (unicode_char.len > 1) return false; + + return switch (unicode_char[0]) { + ' ', '\t' => false, + 0...8 => true, + 10...31 => true, + '!'...'/' => true, + ':'...'@' => true, + '['...'`' => true, + '{'...'~' => true, + else => false, + }; +} + +test Normalizer { + { + const n: Normalizer = .{ .flags = .{ + .remove_extra_whitespaces = true, + .add_dummy_prefix = true, + .add_dummy_suffix = false, + .lower_case_ascii = false, + .split_on_punct_ascii = false, + } }; + const res = try n.normalizeWithMapping(testing.allocator, "Hellŏ world!"); + defer res.deinit(testing.allocator); + + try testing.expectEqualSlices(u8, " Hellŏ world!", res.normalized); + try testing.expectEqualSlices( + usize, + // H e l l ŏ ␣ w o r l d ! + &.{ 0, 0, 1, 2, 3, 4, 4, 6, 8, 9, 10, 11, 12, 13, 14 }, + res.normalized_to_origin, + ); + } + + { + const n: Normalizer = .{ .flags = .{ + .remove_extra_whitespaces = true, + .add_dummy_prefix = true, + .add_dummy_suffix = true, + .lower_case_ascii = false, + .split_on_punct_ascii = false, + } }; + const res = try n.normalize(testing.allocator, "Hello world!"); + defer testing.allocator.free(res); + + try testing.expectEqualSlices(u8, " Hello world! ", res); + } + + { + const n = Normalizer.init( + .{ + .remove_extra_whitespaces = false, + .add_dummy_prefix = true, + .add_dummy_suffix = false, + .lower_case_ascii = false, + .split_on_punct_ascii = false, + }, + Normalizer.sentencepiece_space, + ); + const res = try n.normalize(testing.allocator, "Hello world!"); + defer testing.allocator.free(res); + + try testing.expectEqualSlices(u8, "▁Hello▁▁world!", res); + } + + { + const n: Normalizer = .{ .flags = .{ + .remove_extra_whitespaces = true, + .add_dummy_prefix = false, + .add_dummy_suffix = true, + .lower_case_ascii = true, + .split_on_punct_ascii = false, + } }; + const res = try n.normalize(testing.allocator, "Hello world!"); + defer testing.allocator.free(res); + + try testing.expectEqualSlices(u8, "hello world! ", res); + } + + { + const n: Normalizer = .{ .flags = .{ + .remove_extra_whitespaces = true, + .add_dummy_prefix = false, + .add_dummy_suffix = true, + .lower_case_ascii = false, + .split_on_punct_ascii = true, + } }; + const res = try n.normalize(testing.allocator, "Hello world!"); + defer testing.allocator.free(res); + + try testing.expectEqualSlices(u8, "Hello world ! ", res); + } +} + +/// gpt2 had their own way of storing text. +/// Unfortunately this has contaminated other models. +/// This implementation precompupte a mapping between bytes encoded with GPT2 algorithm, +/// into utf8 bytes, and do lookups at runtime. +pub const Gpt2TextDecoder = struct { + const Code = stdx.BoundedArray(u8, 2); + + // TODO: benchmark this is more efficient than doing the conversion at runtime. + code_to_byte: std.AutoArrayHashMap(Code, u8), + + pub fn init(allocator: std.mem.Allocator) !Gpt2TextDecoder { + var self = Gpt2TextDecoder{ + .code_to_byte = std.AutoArrayHashMap(Code, u8).init(allocator), + }; + try self.code_to_byte.ensureTotalCapacity(256); + errdefer unreachable; + + var n: usize = 0; + for (0..256) |index| { + var code: Code = .{ .buffer = .{ 0, 0 }, .len = 0 }; // 0-init + const i: u8 = @intCast(index); + if (isPrintableByte(i)) { + if (std.ascii.isAscii(i)) { + code.appendAssumeCapacity(i); + } else { + const codepoint: u21 = @as(u21, @intCast(i)); + code.len = @intCast(std.unicode.utf8Encode(codepoint, &code.buffer) catch unreachable); + } + } else { + const codepoint: u21 = 256 + @as(u21, @intCast(n)); + code.len = @intCast(std.unicode.utf8Encode(codepoint, &code.buffer) catch unreachable); + n += 1; + } + + self.code_to_byte.putAssumeCapacityNoClobber(code, i); + } + return self; + } + + pub fn deinit(self: *Gpt2TextDecoder) void { + self.code_to_byte.deinit(); + } + + /// Transform bytes representing text under the gpt2 encoding, + /// and write to the `unicode` buffer utf-8 bytes. + pub fn decode(self: Gpt2TextDecoder, bytes: []const u8, writer: *std.Io.Writer) (error{InvalidInput} || std.Io.Writer.Error)!void { + var it = std.unicode.Utf8Iterator{ .i = 0, .bytes = bytes }; + while (it.nextCodepointSlice()) |codepoint| { + const code: Code = switch (codepoint.len) { + 1 => .{ .buffer = .{ codepoint[0], 0 }, .len = 1 }, // 0-init + 2 => .{ .buffer = .{ codepoint[0], codepoint[1] }, .len = 2 }, + else => return error.InvalidInput, + }; + const byte = self.code_to_byte.get(code) orelse return error.InvalidInput; + try writer.writeByte(byte); + } + } + + inline fn isPrintableByte(c: u8) bool { + return ('!' <= c and c <= '~') or (0xa1 <= c and c <= 0xac) or (0xae <= c and c <= 0xff); + } +}; + +test Gpt2TextDecoder { + var decoder = try Gpt2TextDecoder.init(testing.allocator); + defer decoder.deinit(); + + var buf: [128]u8 = undefined; + + // Ascii is not changed. + { + var out: std.Io.Writer = .fixed(&buf); + try decoder.decode("getTitle", &out); + try testing.expectEqualStrings("getTitle", out.buffered()); + } + + { + var out: std.Io.Writer = .fixed(&buf); + try decoder.decode("Ċ", &out); + try testing.expectEqualStrings("\n", out.buffered()); + } + + // Leading space are represented with 'Ġ' + { + var out: std.Io.Writer = .fixed(&buf); + try decoder.decode("ĠUINavigationController", &out); + try testing.expectEqualStrings(" UINavigationController", out.buffered()); + } + // Russian is wild + { + var out: std.Io.Writer = .fixed(&buf); + try decoder.decode("ĠÑĢабоÑĤ", &out); + try testing.expectEqualStrings(" работ", out.buffered()); + } +} + +/// Open a json file in HF format and load the vocab from it. +pub fn fromHfJson(allocator: std.mem.Allocator, io: std.Io, tokenizer_path: []const u8) !Tokenizer { + const file = try std.fs.cwd().openFile(tokenizer_path, .{}); + defer file.close(); + + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + const file_content = b: { + var reader = file.reader(io, &.{}); + var writer: std.Io.Writer.Allocating = .init(allocator); + defer writer.deinit(); + _ = try reader.interface.streamRemaining(&writer.writer); + break :b try writer.toOwnedSlice(); + }; + defer allocator.free(file_content); + + const info = try std.json.parseFromSliceLeaky(std.json.Value, arena, file_content, .{ + .duplicate_field_behavior = .use_last, + }); + const main_object = switch (info) { + .object => |obj| if (obj.get("added_tokens") == null or obj.get("model") == null) { + return error.InvalidFormat; + } else obj, + else => return error.InvalidFormat, + }; + + const model = objectGet(main_object, .object, "model") orelse return error.InvalidFormat; + const vocab = objectGet(model, .object, "vocab") orelse return error.InvalidFormat; + const added_tokens = if (objectGet(main_object, .array, "added_tokens")) |added| added.items else &.{}; + const vocab_size: u32 = @intCast(vocab.count() + added_tokens.len); + + const normalizer = if (objectGet(main_object, .object, "normalizer")) |normalizer_config| + try Normalizer.fromHfJson(normalizer_config) + else + Normalizer.wellKnown(.llama3); + + // delay init of special tokens. + var tokenizer = try Tokenizer.init(allocator, vocab_size, 256, normalizer, undefined, true); + errdefer tokenizer.deinit(); + + // Buffer containing all concatenated tokens. + // Reserve a big chunk, to avoid grow event, but release over-allocated memory. + var all_tokens: std.Io.Writer.Allocating = try .initCapacity(tokenizer.arena_state.allocator(), file_content.len); + const original_alloc = all_tokens.writer.buffer.ptr; + // A re-alloc event here means we have invalidated all slices inside the tokenizer. + // If this is too annoying we could switch to a custom type instead of slices. + defer { + std.debug.assert(all_tokens.writer.buffer.ptr == original_alloc); + } + + // gpt2 based tokenizer got a special way of encoding unicode. + // we don't know in advance if this will be used by this tokenizer or not. + // so we assume it is the case, but if we find some unicode character, + // outside of the range used by gpt2 we know it was wrong, and start over. + var is_gpt2_vocab: bool = true; + var gpt2_decoder = try Gpt2TextDecoder.init(allocator); + defer gpt2_decoder.deinit(); + var it = vocab.iterator(); + while (it.next()) |kv| { + const n = all_tokens.writer.buffered().len; + gpt2_decoder.decode(kv.key_ptr.*, &all_tokens.writer) catch |err| { + switch (err) { + error.InvalidInput => { + is_gpt2_vocab = false; + break; + }, + error.WriteFailed => return error.OutOfMemory, + } + }; + const idx: u32 = @intCast(kv.value_ptr.*.integer); + const token = all_tokens.written()[n..]; + tokenizer.addOwnedTokenByIndex(idx, @floatFromInt(vocab_size - idx), token); + } + + if (!is_gpt2_vocab) { + // We where wrong, this is not a gpt2 vocab, start over, + // and reset the tokenizer state. + tokenizer.next_token_id = 0; + tokenizer.token_lookup.clearRetainingCapacity(); + all_tokens.clearRetainingCapacity(); + it = vocab.iterator(); + while (it.next()) |kv| { + const idx: u32 = @intCast(kv.value_ptr.*.integer); + const token = try dup(&all_tokens, kv.key_ptr.*); + tokenizer.addOwnedTokenByIndex(idx, @floatFromInt(vocab_size - idx), token); + } + } + + // More tokens, typically added during fine tuning of the model. + for (added_tokens) |token_obj| { + if (token_obj != .object) return error.InvalidFormat; + const v = objectGet(token_obj.object, .string, "content") orelse return error.InvalidFormat; + const id: u32 = @intCast(objectGet(token_obj.object, .integer, "id") orelse return error.InvalidFormat); + const n = all_tokens.written().len; + try if (is_gpt2_vocab) + gpt2_decoder.decode(v, &all_tokens.writer) + else + all_tokens.writer.writeAll(v); + const token = all_tokens.written()[n..]; + tokenizer.addOwnedTokenByIndex(id, 0, token); + } + // We won't add more tokens here, let release. + _ = try all_tokens.toOwnedSlice(); + + var unk = tokenizer.lookup(""); + if (objectGet(model, .integer, "unk_token")) |unk_tok| { + unk = @intCast(unk_tok); + } + + tokenizer.special_tokens = .{ + // TODO allow users to specify special tokens or read them from a tokenizer_config.json file + .bos = tokenizer.lookup("") orelse tokenizer.lookup("<|begin_of_text|>") orelse @panic("bos token not found !"), + .eos = tokenizer.lookup("") orelse tokenizer.lookup("<|end_of_text|>") orelse @panic("eos token not found !"), + .unk = unk orelse std.math.maxInt(u32), + }; + + const byte_fallback = objectGet(model, .bool, "byte_fallback") orelse false; + if (!byte_fallback and unk == null) { + // GPT2 tokenizer have byte fallback already encoded in the model, + // but the json generally don't have the field set. + // We can detect it though because they don't specify an unknown token. + if (is_gpt2_vocab) { + tokenizer.byte_fallback = true; + } else { + log.warn("The given tokenizer can't handle unknown token: no unknown token was set, and byte_fallback is disabled too ! The tokenizer will panic when facing unknown tokens.", .{}); + } + } else if (byte_fallback) { + try tokenizer.rewriteByteFallbackTokens(); + } + return tokenizer; +} + +/// Returns a copy of the given string, stored inside the given ArrayList. +fn dup(allocating: *std.Io.Writer.Allocating, str: []const u8) std.Io.Writer.Error![]const u8 { + const n = allocating.written().len; + try allocating.writer.writeAll(str); + return allocating.written()[n..]; +} + +/// Returns the given entry in a json object only if it has the right type. +fn objectGet( + object: std.json.ObjectMap, + comptime kind: std.meta.FieldEnum(std.json.Value), + key: []const u8, +) ?@FieldType(std.json.Value, @tagName(kind)) { + const val = object.get(key) orelse return null; + if (val != kind) return null; + return @field(val, @tagName(kind)); +} + +pub fn fromTinyLlamaFile(allocator: std.mem.Allocator, io: std.Io, tokenizer_path: []const u8, vocab_size: u32) !Tokenizer { + const tokenizer_file = try std.fs.cwd().openFile(tokenizer_path, .{}); + defer tokenizer_file.close(); + var read_buff: [4096]u8 = undefined; + var tok_reader = tokenizer_file.reader(io, &read_buff); + const r: *std.Io.Reader = &tok_reader.interface; + + const max_token_len = try readValueLE(u32, r); + const special_tokens: Tokenizer.SpecialTokens = .{ + .unk = 0, + .bos = 1, + .eos = 2, + }; + var tokenizer = try Tokenizer.init(allocator, vocab_size, max_token_len, null, special_tokens, true); + var i: u32 = 0; + while (readToken(&tokenizer, r)) : (i += 1) { + // Pass + } else |_| { + if (i < vocab_size) { + log.info("Read {d} words out of {d}", .{ i, vocab_size }); + } + tokenizer.vocab_size = i; + } + try tokenizer.rewriteByteFallbackTokens(); + return tokenizer; +} + +fn readToken(tokenizer: *Tokenizer, tok_reader: *std.Io.Reader) !void { + const score: f32 = try readValueLE(f32, tok_reader); + const len: usize = try readValueLE(u32, tok_reader); + try tokenizer.readTokenInto(score, len, tok_reader); +} + +fn readValueLE(T: type, reader: *std.Io.Reader) !T { + var res: [1]T = undefined; + try reader.readSliceEndian(T, &res, .little); + return res[0]; +} diff --git a/zml2/tokenizer/main.zig b/zml2/tokenizer/main.zig new file mode 100644 index 00000000..a35de7d5 --- /dev/null +++ b/zml2/tokenizer/main.zig @@ -0,0 +1,66 @@ +const std = @import("std"); + +const stdx = @import("stdx"); +const zml_tokenizer = @import("zml/tokenizer"); + +const log = std.log.scoped(.@"//zml/tokenizer"); + +const Flags = struct { + tokenizer: []const u8, + prompt: []const u8, + expected: []const u8 = "", + verbose: bool = false, +}; + +pub fn main() !void { + var gpa: std.heap.GeneralPurposeAllocator(.{}) = .{}; + const allocator = gpa.allocator(); + + var threaded: std.Io.Threaded = .init(allocator); + defer threaded.deinit(); + + const io = threaded.io(); + + const args = stdx.flags.parseProcessArgs(Flags); + + log.info("\tLoading tokenizer from {s}", .{args.tokenizer}); + var tokenizer = try zml_tokenizer.Tokenizer.fromFile(allocator, io, args.tokenizer); + log.info("✅\tLoaded tokenizer from {s}", .{args.tokenizer}); + defer tokenizer.deinit(); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + var decoder = try tokenizer.decoder(); + defer decoder.deinit(); + + const prompt_tok = try encoder.encode(args.prompt); + + log.info("Input: {s}\nOutput: {any}", .{ args.prompt, prompt_tok }); + + var errors: u8 = 0; + { + const reconstructed = try decoder.decode(prompt_tok); + if (!std.mem.eql(u8, args.prompt, reconstructed)) { + log.err("Reconstructed string from tokens doesn't match source: {s}", .{reconstructed}); + errors += 1; + } + } + + if (args.expected.len > 0) { + var expected: std.ArrayList(u32) = try .initCapacity(allocator, args.prompt.len); + var it = std.mem.splitSequence(u8, args.expected, ","); + while (it.next()) |int_token| { + const tok = try std.fmt.parseInt(u32, int_token, 10); + try expected.append(allocator, tok); + } + if (!std.mem.eql(u32, expected.items, prompt_tok)) { + log.err("Doesn't match expected: {any}", .{expected.items}); + errors += 1; + } + } + + if (errors == 0) log.info("All good !", .{}); + + std.process.exit(errors); +} diff --git a/zml2/tokenizer/sentencepiece/BUILD.bazel b/zml2/tokenizer/sentencepiece/BUILD.bazel new file mode 100644 index 00000000..10ac080b --- /dev/null +++ b/zml2/tokenizer/sentencepiece/BUILD.bazel @@ -0,0 +1,35 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") +load("//bazel:swig.bzl", "swig_cc_library") +load("//bazel:zig_srcs.bzl", "zig_srcs") + +swig_cc_library( + name = "sentencepiece_swig", + interface = "sentencepiece.i", + module = "sentencepiece", + deps = [ + "//ffi:cc", + "@com_google_sentencepiece//:sentencepiece_processor", + ], +) + +zig_library( + name = "sentencepiece", + import_name = "sentencepiece", + main = "sentencepiece.zig", + visibility = ["//visibility:public"], + deps = [ + ":sentencepiece_swig", + "//ffi:zig", + "//stdx", + ], +) + +zig_srcs( + name = "sources", + zig_lib = ":sentencepiece", +) + +cc_static_library( + name="sentencepiece_static", + deps = [":sentencepiece_swig"] +) diff --git a/zml2/tokenizer/sentencepiece/sentencepiece.i b/zml2/tokenizer/sentencepiece/sentencepiece.i new file mode 100644 index 00000000..f2413dc3 --- /dev/null +++ b/zml2/tokenizer/sentencepiece/sentencepiece.i @@ -0,0 +1,92 @@ +/* File : example.i */ +%module sentencepiece +%include +%include + +%{ +#include +#include +#include +#include +#include "ffi/zig_slice.h" +%} + +%insert("cheader") %{ +#include "ffi/zig_slice.h" +%} + +%typemap(in) absl::string_view { + $1 = absl::string_view((char *)$input.ptr, $input.len); +} + +%typemap(out, optimal="1") const std::string& %{ + $result.ptr = (void *)($1->data()); + $result.len = (size_t)($1->length()); +%} +%typemap(ctype) absl::string_view, const std::string& "zig_slice" + +%rename(std_string) std::string; +namespace std { + class string { + public: + void reserve(size_t n); + void clear(); + size_t capacity() const; + }; +} + +%extend std::string { + const std::string& data() const { + return *$self; + }; +} + +%extend std::vector { + T* data() { + return $self->data(); + }; +} + +%template(std_vector_int) std::vector; + +%typemap(out) sentencepiece::util::Status %{ + $result = $1.code(); +%} +%typemap(ctype) sentencepiece::util::Status "unsigned int" +%rename(sentencepiece_util_StatusCode, fullname=1) sentencepiece::util::StatusCode; + +namespace sentencepiece { + namespace util { + enum class StatusCode : int { + kOk = 0, + kCancelled = 1, + kUnknown = 2, + kInvalidArgument = 3, + kDeadlineExceeded = 4, + kNotFound = 5, + kAlreadyExists = 6, + kPermissionDenied = 7, + kResourceExhausted = 8, + kFailedPrecondition = 9, + kAborted = 10, + kOutOfRange = 11, + kUnimplemented = 12, + kInternal = 13, + kUnavailable = 14, + kDataLoss = 15, + kUnauthenticated = 16, + }; + } + + class SentencePieceProcessor { + public: + virtual sentencepiece::util::Status Load(absl::string_view filename); + virtual sentencepiece::util::Status Encode(absl::string_view input, std::vector *ids) const; + virtual sentencepiece::util::Status Decode(const std::vector &ids, std::string *detokenized) const; + virtual int PieceToId(absl::string_view piece) const; + virtual int unk_id() const; + virtual int bos_id() const; + virtual int eos_id() const; + virtual int pad_id() const; + }; +} diff --git a/zml2/tokenizer/sentencepiece/sentencepiece.zig b/zml2/tokenizer/sentencepiece/sentencepiece.zig new file mode 100644 index 00000000..c3623cb5 --- /dev/null +++ b/zml2/tokenizer/sentencepiece/sentencepiece.zig @@ -0,0 +1,191 @@ +const std = @import("std"); + +const c = @import("c"); +const ffi = @import("ffi"); +const stdx = @import("stdx"); + +const StringToTokenRatio = 3; + +pub const Error = error{ + Cancelled, + Unknown, + InvalidArgument, + DeadlineExceeded, + NotFound, + AlreadyExists, + PermissionDenied, + ResourceExhausted, + FailedPrecondition, + Aborted, + OutOfRange, + Unimplemented, + Internal, + Unavailable, + DataLoss, + Unauthenticated, +}; + +fn assertOk(code: c.sentencepiece_util_StatusCode) Error!void { + return switch (code) { + c.sentencepiece_util_StatusCode_kOk => {}, + c.sentencepiece_util_StatusCode_kCancelled => Error.Cancelled, + c.sentencepiece_util_StatusCode_kUnknown => Error.Unknown, + c.sentencepiece_util_StatusCode_kInvalidArgument => Error.InvalidArgument, + c.sentencepiece_util_StatusCode_kDeadlineExceeded => Error.DeadlineExceeded, + c.sentencepiece_util_StatusCode_kNotFound => Error.NotFound, + c.sentencepiece_util_StatusCode_kAlreadyExists => Error.AlreadyExists, + c.sentencepiece_util_StatusCode_kPermissionDenied => Error.PermissionDenied, + c.sentencepiece_util_StatusCode_kResourceExhausted => Error.ResourceExhausted, + c.sentencepiece_util_StatusCode_kFailedPrecondition => Error.FailedPrecondition, + c.sentencepiece_util_StatusCode_kAborted => Error.Aborted, + c.sentencepiece_util_StatusCode_kOutOfRange => Error.OutOfRange, + c.sentencepiece_util_StatusCode_kUnimplemented => Error.Unimplemented, + c.sentencepiece_util_StatusCode_kInternal => Error.Internal, + c.sentencepiece_util_StatusCode_kUnavailable => Error.Unavailable, + c.sentencepiece_util_StatusCode_kDataLoss => Error.DataLoss, + c.sentencepiece_util_StatusCode_kUnauthenticated => Error.Unauthenticated, + else => unreachable, + }; +} + +pub const Encoder = struct { + inner: *SentencePieceProcessor, + vec: *c.std_vector_int, + + fn init(inner: *SentencePieceProcessor) Encoder { + return .{ + .inner = inner, + .vec = c.std_vector_int_new() orelse unreachable, + }; + } + + pub fn deinit(self: *Encoder) void { + c.std_vector_int_delete(self.vec); + } + + pub fn reset(self: *Encoder) void { + c.std_vector_int_clear(self.vec); + } + + pub fn encode(self: *Encoder, input: []const u8) ![]const u32 { + c.std_vector_int_reserve(self.vec, input.len / StringToTokenRatio); + try assertOk(c.SentencePieceProcessor_Encode(@ptrCast(self.inner), ffi.ZigSlice.from(input), self.vec)); + return self.ids(); + } + + pub fn ids(self: *const Encoder) []const u32 { + return ffi.ZigSlice.to(u32, .{ + .ptr = c.std_vector_int_data(self.vec), + .len = c.std_vector_int_size(self.vec), + }); + } +}; + +pub const Decoder = struct { + const StringBufferSize = 64; + const StringBuffer = stdx.BoundedArray(u8, StringBufferSize); + const TokenIdsBufferSize = 4; + + inner: *SentencePieceProcessor, + vec: *c.std_vector_int, + str: *c.std_string, + last_string: StringBuffer = .{ .len = 0 }, + + fn init(inner: *SentencePieceProcessor) !Decoder { + const vec = try (c.std_vector_int_new() orelse std.mem.Allocator.Error.OutOfMemory); + c.std_vector_int_reserve(vec, TokenIdsBufferSize); + errdefer c.std_vector_int_delete(vec); + + const str = try (c.std_string_new() orelse std.mem.Allocator.Error.OutOfMemory); + c.std_string_reserve(str, StringBufferSize); + errdefer c.std_string_delete(str); + + return .{ + .inner = inner, + .vec = vec, + .str = str, + }; + } + + pub fn deinit(self: *Decoder) void { + c.std_vector_int_delete(self.vec); + c.std_string_delete(self.str); + } + + pub fn reset(self: *Decoder) void { + c.std_vector_int_clear(self.vec); + c.std_string_clear(self.str); + } + + pub fn decode(self: *Decoder, ids_: []const u32) ![]const u8 { + c.std_vector_int_reserve(self.vec, ids_.len); + c.std_string_reserve(self.str, ids_.len * StringToTokenRatio); + for (ids_) |id| { + c.std_vector_int_push_back(self.vec, @intCast(id)); + } + try assertOk(c.SentencePieceProcessor_Decode(@ptrCast(self.inner), self.vec, self.str)); + return self.string(); + } + + pub fn string(self: *const Decoder) []const u8 { + const res = c.std_string_data(self.str); + return ffi.ZigSlice.to(u8, res); + } + + fn ids(self: *const Decoder) []u32 { + const ptr: [*c]u32 = @ptrCast(c.std_vector_int_data(self.vec)); + return ptr[0..c.std_vector_int_size(self.vec)]; + } + + pub fn next(self: *Decoder, token_id: u32) !?[]const u8 { + const current_ids = self.ids(); + if (current_ids.len >= c.std_vector_int_capacity(self.vec)) { + std.mem.copyForwards(u32, current_ids[0 .. current_ids.len - 1], current_ids[1..]); + current_ids[current_ids.len - 1] = token_id; + } else { + c.std_vector_int_push_back(self.vec, @intCast(token_id)); + } + try assertOk(c.SentencePieceProcessor_Decode(@ptrCast(self.inner), self.vec, self.str)); + const new_string = self.string(); + if (self.last_string.len == 0) { + self.last_string = try StringBuffer.fromSlice(new_string); + return new_string; + } + var view = try std.unicode.Utf8View.init(self.last_string.constSlice()); + var it = view.iterator(); + while (it.nextCodepointSlice()) |cp| { + const start = it.i - cp.len; + if (std.mem.startsWith(u8, new_string, self.last_string.constSlice()[start..])) { + const chunk = new_string[self.last_string.len - start ..]; + self.last_string = try StringBuffer.fromSlice(new_string); + return chunk; + } + } + return &.{}; + } +}; + +pub const SentencePieceProcessor = opaque { + pub fn fromFile(model: []const u8) !*SentencePieceProcessor { + const sp: *SentencePieceProcessor = @ptrCast(c.SentencePieceProcessor_new()); + errdefer sp.deinit(); + try assertOk(c.SentencePieceProcessor_Load(@ptrCast(sp), ffi.ZigSlice.from(model))); + return sp; + } + + pub fn deinit(self: *SentencePieceProcessor) void { + c.SentencePieceProcessor_delete(@ptrCast(self)); + } + + pub fn encoder(self: *SentencePieceProcessor) !Encoder { + return Encoder.init(self); + } + + pub fn decoder(self: *SentencePieceProcessor) !Decoder { + return try Decoder.init(self); + } + + pub fn tokenToId(self: *SentencePieceProcessor, token: []const u8) u32 { + return @intCast(c.SentencePieceProcessor_PieceToId(@ptrCast(self), ffi.ZigSlice.from(token))); + } +}; diff --git a/zml2/tokenizer/tokenizer.zig b/zml2/tokenizer/tokenizer.zig new file mode 100644 index 00000000..bb80ae4a --- /dev/null +++ b/zml2/tokenizer/tokenizer.zig @@ -0,0 +1,138 @@ +const std = @import("std"); + +pub const hftokenizers = @import("hftokenizers"); +pub const sentencepiece = @import("sentencepiece"); + +pub const homemade = @import("homemade.zig"); + +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(hftokenizers); + std.testing.refAllDecls(sentencepiece); + std.testing.refAllDecls(homemade); +} + +const Tokenizers = enum { + hftokenizers, + sentencepiece, + homemade, +}; + +pub const Tokenizer = union(Tokenizers) { + pub const Encoder = union(Tokenizers) { + hftokenizers: hftokenizers.Encoder, + sentencepiece: sentencepiece.Encoder, + homemade: homemade.Encoder, + + pub fn deinit(self: *Encoder) void { + switch (self.*) { + inline else => |*v| v.deinit(), + } + } + + pub fn reset(self: *Encoder) void { + switch (self.*) { + inline else => |*v| v.reset(), + } + } + + pub fn encode(self: *Encoder, input: []const u8) ![]const u32 { + return switch (self.*) { + inline else => |*v| v.encode(input), + }; + } + + pub fn ids(self: Encoder) []const u32 { + return switch (self) { + inline else => |v| v.ids(), + }; + } + }; + + pub const Decoder = union(Tokenizers) { + hftokenizers: hftokenizers.Decoder, + sentencepiece: sentencepiece.Decoder, + homemade: homemade.Decoder, + + pub fn deinit(self: *Decoder) void { + switch (self.*) { + inline else => |*v| v.deinit(), + } + } + + pub fn reset(self: *Decoder) void { + switch (self.*) { + inline else => |*v| v.reset(), + } + } + + pub fn decode(self: *Decoder, ids_: []const u32) ![]const u8 { + return switch (self.*) { + inline else => |*v| v.decode(ids_), + }; + } + + pub fn string(self: Decoder) []const u8 { + return switch (self.*) { + inline else => |v| v.string(), + }; + } + + pub fn ids(self: Decoder) []u32 { + return switch (self.*) { + inline else => |v| v.ids(), + }; + } + + pub fn next(self: *Decoder, token_id: u32) !?[]const u8 { + return switch (self.*) { + inline else => |*v| v.next(token_id), + }; + } + }; + + hftokenizers: *hftokenizers.HFTokenizer, + sentencepiece: *sentencepiece.SentencePieceProcessor, + homemade: *homemade.Tokenizer, + + pub fn fromFile(allocator: std.mem.Allocator, io: std.Io, model: []const u8) !Tokenizer { + if (std.mem.endsWith(u8, model, ".pb")) { + return .{ .sentencepiece = try sentencepiece.SentencePieceProcessor.fromFile(model) }; + } + if (std.mem.endsWith(u8, model, ".json")) { + return .{ .hftokenizers = try hftokenizers.HFTokenizer.fromFile(model) }; + } + + if (std.mem.endsWith(u8, model, ".tinyllama")) { + const tokenizer = try allocator.create(homemade.Tokenizer); + tokenizer.* = try homemade.fromTinyLlamaFile(allocator, io, model, 32000); + return .{ .homemade = tokenizer }; + } + + return error.InvalidArgument; + } + + pub fn deinit(self: *Tokenizer) void { + switch (self.*) { + inline else => |t| t.deinit(), + } + } + + pub fn encoder(self: Tokenizer) !Encoder { + return switch (self) { + inline else => |v, tag| @unionInit(Encoder, @tagName(tag), try v.encoder()), + }; + } + + pub fn decoder(self: Tokenizer) !Decoder { + return switch (self) { + inline else => |v, tag| @unionInit(Decoder, @tagName(tag), try v.decoder()), + }; + } + + pub fn tokenToId(self: Tokenizer, token: []const u8) ?u32 { + return switch (self) { + inline else => |v| v.tokenToId(token), + }; + } +}; diff --git a/zml2/zml.zig b/zml2/zml.zig index f6b6e6a7..5f56360b 100644 --- a/zml2/zml.zig +++ b/zml2/zml.zig @@ -35,6 +35,7 @@ pub const ConstSlice = slice.ConstSlice; pub const tensor = @import("tensor.zig"); pub const Tensor = tensor.Tensor; pub const testing = @import("testing.zig"); +pub const tokenizer = @import("zml/tokenizer"); var runfiles_once = std.once(struct { fn call_() !void { From 95401d52aa07aa828aa18cbddc5690c1c7a84f23 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 16 Dec 2025 17:58:16 +0100 Subject: [PATCH 145/881] wip llama --- examples/llama/BUILD.bazel | 3 +- examples/llama/llama.zig | 192 +++++----- examples/llama/main.zig | 234 ++++++------ examples/mnist/mnist.zig | 381 +------------------ third_party/com_github_hejsil_clap/repo.bzl | 2 +- zml2/io.zig | 393 ++++++++++++++++++++ zml2/nn.zig | 21 +- zml2/tensor.zig | 2 +- 8 files changed, 638 insertions(+), 590 deletions(-) diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel index 635eb087..ad8d914e 100644 --- a/examples/llama/BUILD.bazel +++ b/examples/llama/BUILD.bazel @@ -12,9 +12,8 @@ zig_binary( main = "main.zig", visibility = ["//visibility:public"], deps = [ - "//async", "//stdx", - "//zml", + "//zml2:zml", "@com_github_hejsil_clap//:clap", ], ) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 606598fb..1857e588 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -43,81 +43,24 @@ pub const LlamaLM = struct { gen_opts: zml.nn.SamplingStrategy = .{}, config: Config, - pub fn init(allocator: std.mem.Allocator, config: Config, options: Options, store: zml.aio.BufferStore) !LlamaLM { - const rope_opts: zml.nn.RopeOpts = .{ - .layout = if (config.hf_rope_impl) .sequential else .interleaved, - .freq_base = config.rope_theta, - .scaling = config.rope_scaling, - }; - - const layers = try allocator.alloc(TransformerLayer, config.num_hidden_layers); - var prefix = try zml.aio.PrefixBuilder.initCapacity(allocator, 1024); - try prefix.push(stdx.noalloc, "model.layers"); - for (0.., layers) |i, *layer| { - try prefix.pushDigit(stdx.noalloc, i); - defer prefix.pop(); - var self_attn = try zml.aio.populateModelWithPrefix(SelfAttn, allocator, store, prefix.concat("self_attn")); - self_attn.num_heads = config.num_attention_heads; - self_attn.num_kv_heads = config.num_key_value_heads; - self_attn.rope_opts = rope_opts; - self_attn.q_proj.weight = self_attn.q_proj.weight.withSharding(.{0}); - self_attn.k_proj.weight = self_attn.k_proj.weight.withSharding(.{0}); - self_attn.v_proj.weight = self_attn.v_proj.weight.withSharding(.{0}); - self_attn.o_proj.weight = self_attn.o_proj.weight.withSharding(.{1}); - - var input_layernorm = try zml.aio.populateModelWithPrefix(RmsNorm, allocator, store, prefix.concat("input_layernorm")); - input_layernorm.eps = config.rms_norm_eps; - - var post_attention_layernorm = try zml.aio.populateModelWithPrefix(RmsNorm, allocator, store, prefix.concat("post_attention_layernorm")); - post_attention_layernorm.eps = config.rms_norm_eps; - - var mlp = try zml.aio.populateModelWithPrefix(Mlp, allocator, store, prefix.concat("mlp")); - mlp.up_proj.weight = mlp.up_proj.weight.withSharding(.{0}); - mlp.gate_proj.weight = mlp.gate_proj.weight.withSharding(.{0}); - mlp.down_proj.weight = mlp.down_proj.weight.withSharding(.{1}); - - layer.* = .{ - .self_attn = self_attn, - .input_layernorm = input_layernorm, - .post_attention_layernorm = post_attention_layernorm, - .mlp = mlp, - }; - } - - var lm_head: ?zml.nn.Linear = null; - if (!config.tie_word_embeddings) { - lm_head = .{ .weight = store.getTensor("lm_head.weight") }; - if (options.sampling_strategy) |gen_opts| { - if (gen_opts.topk == 1) - lm_head.?.weight = lm_head.?.weight.withSharding(.{0}); - } - } + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config, options: Options) !LlamaLM { + const lm_head: ?zml.nn.Linear = if (store.withPrefix("lm_head").maybeCreateTensorWithTags("weight", .{ .dout, .d })) |weight| + .init(weight, null, .d) + else + null; return .{ - .config = config, - .gen_opts = options.sampling_strategy orelse .{}, - .model = .{ - // Weights - .layers = layers, - .embed_tokens = .{ .weight = store.getTensor("model.embed_tokens.weight") }, - .norm = .{ - .weight = store.getTensor("model.norm.weight"), - .eps = config.rms_norm_eps, - }, - // Push down some configs - .max_seq_len = options.max_seq_len, - .num_heads = config.num_attention_heads, - .num_kv_heads = config.num_key_value_heads, - .rope_opts = .{ - .layout = if (config.hf_rope_impl) .sequential else .interleaved, - .freq_base = config.rope_theta, - .scaling = config.rope_scaling, - }, - }, .lm_head = lm_head, + .model = try .init(allocator, store.withPrefix("model"), config, options), + .gen_opts = options.sampling_strategy orelse .{}, + .config = config, }; } + pub fn deinit(self: LlamaLM, allocator: std.mem.Allocator) void { + self.model.deinit(allocator); + } + /// Predicts the token at `token_index` position. /// Returns: /// - updated `tokens`, @@ -132,7 +75,7 @@ pub const LlamaLM = struct { ) struct { Tensor, KvCache, Tensor.Rng } { stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Llama ! Expected >=1d tokens and 0d token_index, got: {f} and {f}", .{ tokens_, token_index }); const tokens = tokens_.withPartialTags(.{.s}); - const out, const updated_kv_cache = zml.call(self.model, .forward, .{ tokens, token_index, kv_cache }); + const out, const updated_kv_cache = self.model.forward(tokens, token_index, kv_cache); const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; } @@ -148,9 +91,9 @@ pub const LlamaLM = struct { var logits = blk: { if (lm_head_) |lm_head| { - break :blk zml.call(lm_head, .forward, .{out}); + break :blk lm_head.forward(out).rename(.{ .dout = .d }); } else { - break :blk self.model.embed_tokens.weight.withTags(.{ .voc, .d }).dot(out, .{.d}); + break :blk self.model.embed_tokens.weight.withTags(.{ .voc, .d }).dot(out, .d); } }; @@ -179,6 +122,25 @@ pub const Llama = struct { .freq_base = 10_000, }, + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: LlamaLM.Config, options: LlamaLM.Options) !Llama { + const layers = try allocator.alloc(TransformerLayer, config.num_hidden_layers); + errdefer allocator.free(layers); + + for (layers, 0..) |*layer, i| { + layer.* = try .init(store.withPrefix("layers").withLayer(i), config, options); + } + + return .{ + .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight") }, + .norm = .{ .weight = store.withPrefix("norm").createTensorWithTags("weight", .{.d}), .eps = config.rms_norm_eps }, + .layers = layers, + }; + } + + pub fn deinit(self: Llama, allocator: std.mem.Allocator) void { + allocator.free(self.layers); + } + /// Forward one token, using KV cache for previous tokens. /// Returns result and updated KV cache. pub fn forward(self: Llama, tokens: Tensor, token_index: Tensor, kv_cache: KvCache) struct { Tensor, KvCache } { @@ -187,15 +149,15 @@ pub const Llama = struct { var updated_kv_cache = kv_cache; for (self.layers, 0..) |layer, i| { - hidden, updated_kv_cache = zml.call(layer, .forward, .{ hidden, token_index, updated_kv_cache.atLayer(i) }); + hidden, updated_kv_cache = layer.forward(hidden, token_index, updated_kv_cache.atLayer(i)); } - const output = zml.call(self.norm, .forward, .{hidden}); + const output = self.norm.forward(hidden); return .{ output, updated_kv_cache.reuseBuffer(kv_cache) }; } pub fn embed(embed_tokens_: zml.nn.TokenEmbedding, tokens_: Tensor) Tensor { - return zml.call(embed_tokens_, .forward, .{tokens_}).withPartialTags(.{.d}); + return embed_tokens_.forward(tokens_).withPartialTags(.{.d}); } }; @@ -205,6 +167,15 @@ pub const TransformerLayer = struct { post_attention_layernorm: RmsNorm, mlp: Mlp, + pub fn init(store: zml.io.TensorStore.View, config: LlamaLM.Config, options: LlamaLM.Options) !TransformerLayer { + return .{ + .input_layernorm = .init(store.withPrefix("input_layernorm"), config.rms_norm_eps), + .self_attn = try .init(store.withPrefix("self_attn"), config, options), + .post_attention_layernorm = .init(store.withPrefix("post_attention_layernorm"), config.rms_norm_eps), + .mlp = .init(store.withPrefix("mlp")), + }; + } + pub fn forward( self: TransformerLayer, x0: Tensor, @@ -215,13 +186,13 @@ pub const TransformerLayer = struct { //log.debug("TransformerLayer({f}) -> {f}", .{ x0, self.input_layernorm.forward(x0) }); stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {f}", .{x0}); - const x0_normalized = zml.call(self.input_layernorm, .forward, .{x0}); - const delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); + const x0_normalized = self.input_layernorm.forward(x0); + const delta0, const updated_kv_cache = self.self_attn.forward(x0_normalized, token_index, kv_cache); const x1 = x0.add(delta0); // Fully Connected - const x1_normalized = zml.call(self.post_attention_layernorm, .forward, .{x1}); - const x2 = zml.call(self.mlp, .forward, .{x1_normalized}).add(x1); + const x1_normalized = self.post_attention_layernorm.forward(x1); + const x2 = self.mlp.forward(x1_normalized).add(x1).rename(.{ .dout = .d }); return .{ x2.reuseBuffer(x0), updated_kv_cache }; } @@ -231,6 +202,10 @@ const RmsNorm = struct { weight: Tensor, eps: f32 = 1e-5, + pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { + return .{ .weight = store.createTensorWithTags("weight", .{.d}), .eps = eps }; + } + /// L2 normalization of input tensor along `.d` axis. pub fn forward(self: RmsNorm, input: Tensor) Tensor { const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); @@ -244,11 +219,23 @@ const Mlp = struct { gate_proj: zml.nn.Linear, // (dim -> hidden_dim) down_proj: zml.nn.Linear, // (hidden_dim -> dim) + fn initLinear(store: zml.io.TensorStore.View) zml.nn.Linear { + return .init(store.createTensorWithTags("weight", .{ .dout, .d }), store.maybeCreateTensorWithTags("bias", .{.dout}), .d); + } + + pub fn init(store: zml.io.TensorStore.View) Mlp { + return .{ + .up_proj = initLinear(store.withPrefix("up_proj")), + .gate_proj = initLinear(store.withPrefix("gate_proj")), + .down_proj = initLinear(store.withPrefix("down_proj")), + }; + } + pub fn forward(self: Mlp, x: Tensor) Tensor { - const proj = zml.call(self.up_proj, .forward, .{x}); - var output = zml.call(self.gate_proj, .forward, .{x}); - output = output.silu().mul(proj); - return zml.call(self.down_proj, .forward, .{output}); + const proj = self.up_proj.forward(x); + var output = self.gate_proj.forward(x); + output = output.silu().mul(proj).rename(.{ .dout = .d }); + return self.down_proj.forward(output); } }; @@ -265,6 +252,30 @@ pub const SelfAttn = struct { num_kv_heads: i64 = 0, rope_opts: zml.nn.RopeOpts = undefined, + fn initProj(store: zml.io.TensorStore.View) zml.nn.Linear { + return .init(store.createTensorWithTags("weight", .{ .dout, .d }), store.maybeCreateTensorWithTags("bias", .{.dout}), .d); + } + + pub fn init(store: zml.io.TensorStore.View, config: LlamaLM.Config, options: LlamaLM.Options) !SelfAttn { + _ = options; // autofix + return .{ + .q_proj = initProj(store.withPrefix("q_proj")), + .k_proj = initProj(store.withPrefix("k_proj")), + .v_proj = initProj(store.withPrefix("v_proj")), + .o_proj = initProj(store.withPrefix("o_proj")), + // TODO(Corentin): fix that + .q_norm = null, + .k_norm = null, + .num_heads = @intCast(config.num_attention_heads), + .num_kv_heads = @intCast(config.num_key_value_heads), + .rope_opts = .{ + .layout = if (config.hf_rope_impl) .sequential else .interleaved, + .freq_base = config.rope_theta, + .scaling = config.rope_scaling, + }, + }; + } + /// Self Attention. /// - If token_index is set, x is assumed to be the representation of one new token, /// and kv_cache will be read for the previous tokens. @@ -279,9 +290,9 @@ pub const SelfAttn = struct { kv_cache: KvCache, ) struct { Tensor, KvCache } { const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; - var q = zml.call(self.q_proj, .forward, .{x}).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }).withSharding(.{.h}); - var k = zml.call(self.k_proj, .forward, .{x}).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{.h}); - var v = zml.call(self.v_proj, .forward, .{x}).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{.h}); + var q = self.q_proj.forward(x).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); + var k = self.k_proj.forward(x).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); + var v = self.v_proj.forward(x).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); // Generate the attention mask. const seq_len = kv_cache.k.dim(.k); @@ -313,7 +324,7 @@ pub const SelfAttn = struct { const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask, .allow_cudnn = true }); // const attn_output = zml.nn.sdpaMemEfficient(q, k, v, .{ .attn_mask = attn_mask }, .{ .q_chunk_size = 4096, .k_chunk_size = 1024 }); const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); - return .{ zml.call(self.o_proj, .forward, .{attn}), new_kv_cache }; + return .{ self.o_proj.forward(attn), new_kv_cache }; } }; @@ -323,11 +334,10 @@ pub const KvCache = struct { layer_index: Tensor, pub fn init(kv_shape: zml.Shape) KvCache { - // The KV-cache is initialized with ones to detect reads of uninitialized memory. return .{ - .k = Tensor.constant(kv_shape, kv_shape.dtype().one()).withSharding(.{.h}), - .v = Tensor.constant(kv_shape, kv_shape.dtype().one()).withSharding(.{.h}), - .layer_index = Tensor.scalar(-1, .u32), + .k = .fromShape(kv_shape), + .v = .fromShape(kv_shape), + .layer_index = .init(.{}, .u32), }; } diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 6d577695..5df1e639 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -1,6 +1,5 @@ const std = @import("std"); -const async = @import("async"); const clap = @import("clap"); const stdx = @import("stdx"); const zml = @import("zml"); @@ -19,7 +18,6 @@ const log = std.log.scoped(.llama); pub const std_options: std.Options = .{ .log_level = .info, - .logFn = async.logFn(std.log.defaultLog), }; const params = clap.parseParamsComptime( @@ -152,13 +150,28 @@ pub fn generateText( } pub fn main() !void { - try async.AsyncThread.main(std.heap.c_allocator, asyncMain); -} - -pub fn asyncMain() !void { log.info(" LLama was compiled with {}", .{@import("builtin").mode}); - const allocator = std.heap.c_allocator; + //const allocator = std.heap.c_allocator; + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + + const allocator = gpa.allocator(); + + var threaded: std.Io.Threaded = .init(allocator); + defer threaded.deinit(); + + var vfs_file: zml.io.VFS.File = .init(threaded.io()); + + var vfs: zml.io.VFS = .init(allocator, threaded.io()); + defer vfs.deinit(); + + try vfs.register("file", vfs_file.io()); + + const io = vfs.io(); + + zml.init(); + defer zml.deinit(); const parsers = comptime .{ .BOOL = bool_parser, @@ -192,54 +205,40 @@ pub fn asyncMain() !void { log.err("Missing --hf-model-path", .{}); return; }; - - const model_config_path = try std.fs.path.join(allocator, &.{ hf_model_path, "config.json" }); - defer allocator.free(model_config_path); - - const model_weights_path = b: { - const simple_path = try std.fs.path.join(allocator, &.{ hf_model_path, "model.safetensors" }); - if (async.File.access(simple_path, .{})) { - break :b simple_path; - } else |_| { - allocator.free(simple_path); - } - - const sharded_path = try std.fs.path.join(allocator, &.{ hf_model_path, "model.safetensors.index.json" }); - break :b sharded_path; - }; - defer allocator.free(model_weights_path); + const repo = try vfs.openAbsoluteDir(io, hf_model_path, .{}); + defer repo.close(io); const model_tokenizer_path = try std.fs.path.join(allocator, &.{ hf_model_path, "tokenizer.json" }); defer allocator.free(model_tokenizer_path); const config = blk: { - var config_json_file = try async.File.open(model_config_path, .{ .mode = .read_only }); - defer config_json_file.close() catch unreachable; + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + const config_json_file = try repo.openFile(io, "config.json", .{}); + defer config_json_file.close(io); var config_json_buffer: [256]u8 = undefined; - var config_reader = config_json_file.reader(&config_json_buffer); + var config_reader = config_json_file.reader(io, &config_json_buffer); var reader = std.json.Reader.init(allocator, &config_reader.interface); defer reader.deinit(); - const config_obj = try std.json.parseFromTokenSourceLeaky(llama.LlamaLM.Config, allocator, &reader, .{ .ignore_unknown_fields = true }); + const config_obj = try std.json.parseFromTokenSourceLeaky(llama.LlamaLM.Config, arena.allocator(), &reader, .{ .ignore_unknown_fields = true }); break :blk config_obj; }; - var context = try zml.Context.init(); - defer context.deinit(); - - const compilation_options = zml.CompilationOptions{ - .xla_dump_to = "/tmp/zml/llama", - .sharding_enabled = cli.args.sharding orelse true, - }; - // initialize ZML platform with optional create options // eg: --create-options='{"cuda":{"allocator":{"bfc":{"memory_fraction": 0.99}}}}' - const create_opts_json = cli.args.@"create-options" orelse "{}"; - const create_opts = try std.json.parseFromSlice(zml.Platform.CreateOptions, allocator, create_opts_json, .{}); - const platform = context.autoPlatform(create_opts.value).withCompilationOptions(compilation_options); - create_opts.deinit(); - context.printAvailablePlatforms(platform); + var platform = b: { + const create_opts_json = cli.args.@"create-options" orelse "{}"; + const create_opts = try std.json.parseFromSlice(zml.platform.CreateOptions, allocator, create_opts_json, .{}); + defer create_opts.deinit(); + const platform: zml.Platform = try .auto(threaded.io(), create_opts.value); + break :b platform; + }; + defer platform.deinit(); - var store = try zml.aio.detectFormatAndOpen(allocator, model_weights_path); + var registry = try zml.safetensors.parseFromPath(allocator, io, &vfs, hf_model_path); + defer registry.deinit(); + + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); defer store.deinit(); // Write metadata from the config file into the LlamaLm struct. @@ -252,97 +251,110 @@ pub fn asyncMain() !void { }, }; - // Contains memory for llama_tensors and llama_buffers. - var compiler_arena = std.heap.ArenaAllocator.init(allocator); - defer compiler_arena.deinit(); - // Initialize the Llama struct and map the content of the .safetensors to the model tensors. - const llama_tensors: llama.LlamaLM = try .init(compiler_arena.allocator(), config, llama_options, store); + const llama_model: llama.LlamaLM = try .init(allocator, store.view(), config, llama_options); + defer llama_model.deinit(allocator); // Specify shapes of input arguments - const prefill_tokens_shape = zml.Shape.init(.{ .s = llama_options.max_seq_len }, .u32); - const gen_tokens_shape = zml.Shape.init(.{ .s = 1 }, .u32); - const token_idx_shape = zml.Shape.init(.{}, .u32); + const prefill_tokens: zml.Tensor = .init(.{ .s = llama_options.max_seq_len }, .u32); + const decode_tokens: zml.Tensor = .init(.{ .s = 1 }, .u32); + const token_index: zml.Tensor = .init(.{}, .u32); - const dtype = llama_tensors.model.embed_tokens.weight.dtype(); - const kv_shape = zml.Shape.init(.{ - .layer = llama_tensors.model.layers.len, + const dtype = llama_model.model.embed_tokens.weight.dtype(); + const kv_cache: llama.KvCache = .init(zml.Shape.init(.{ + .layer = llama_model.model.layers.len, .k = seq_len, .h = config.num_key_value_heads, .hd = config.head_dim orelse @divExact(config.hidden_size, config.num_attention_heads), - }, dtype).withSharding(.{.h}); - const kv_cache_shape: zml.ShapeOf(llama.KvCache) = llama.KvCache.initShape(kv_shape); - const rng_shape = zml.Tensor.Rng.shape(); + }, dtype)); + const rng: zml.Tensor.Rng = .init(); - // Compile the model twice, one for prefill, one for generation. - var start = try std.time.Timer.start(); - var fut_mod_prefill = try async.async(zml.compileModel, .{ - allocator, llama.LlamaLM.forward, llama_tensors, - .{ - prefill_tokens_shape, - token_idx_shape, - kv_cache_shape, - rng_shape, - }, - platform, - }); + var prefill_exe = try platform.compileModel(allocator, io, llama.LlamaLM.forward, llama_model, .{ prefill_tokens, token_index, kv_cache, rng }); + defer prefill_exe.deinit(); + var decode_exe = try platform.compileModel(allocator, io, llama.LlamaLM.forward, llama_model, .{ decode_tokens, token_index, kv_cache, rng }); + defer decode_exe.deinit(); - var fut_mod = try async.async(zml.compileModel, .{ - allocator, llama.LlamaLM.forward, llama_tensors, - .{ - gen_tokens_shape, - token_idx_shape, - kv_cache_shape, - rng_shape, - }, - platform, - }); + //// Compile the model twice, one for prefill, one for generation. + //var start = try std.time.Timer.start(); + //var fut_mod_prefill = try async.async(zml.compileModel, .{ + // allocator, llama.LlamaLM.forward, llama_tensors, + // .{ + // prefill_tokens_shape, + // token_idx_shape, + // kv_cache_shape, + // rng_shape, + // }, + // platform, + //}); - // While we are still compiling load the weights to the device. - log.info("\tLoading Llama weights from {s}...", .{model_weights_path}); - var llama_buffers = try store.loadModelById(llama.LlamaLM, compiler_arena.allocator(), llama_tensors, platform); - defer zml.aio.unloadBuffers(&llama_buffers); - log.info("✅\tLoaded weights in {D}", .{start.read()}); + //var fut_mod = try async.async(zml.compileModel, .{ + // allocator, llama.LlamaLM.forward, llama_tensors, + // .{ + // gen_tokens_shape, + // token_idx_shape, + // kv_cache_shape, + // rng_shape, + // }, + // platform, + //}); - var llama_module_prefill = (try fut_mod_prefill.await()).prepare(llama_buffers); - defer llama_module_prefill.deinit(); - var llama_module = (try fut_mod.await()).prepare(llama_buffers); - defer llama_module.deinit(); - log.info("✅\tCompiled model in {D}", .{start.read()}); - log.info("Creating KvCache", .{}); - const kv_cache = try llama.KvCache.initBuffer(kv_shape, platform); + //// While we are still compiling load the weights to the device. + //log.info("\tLoading Llama weights from {s}...", .{model_weights_path}); + //var llama_buffers = try store.loadModelById(llama.LlamaLM, compiler_arena.allocator(), llama_tensors, platform); + //defer zml.aio.unloadBuffers(&llama_buffers); + //log.info("✅\tLoaded weights in {D}", .{start.read()}); + + //var llama_module_prefill = (try fut_mod_prefill.await()).prepare(llama_buffers); + //defer llama_module_prefill.deinit(); + //var llama_module = (try fut_mod.await()).prepare(llama_buffers); + //defer llama_module.deinit(); + //log.info("✅\tCompiled model in {D}", .{start.read()}); + //log.info("Creating KvCache", .{}); + //const kv_cache = try llama.KvCache.initBuffer(kv_shape, platform); var tokenizer = blk: { - log.info("Loading tokenizer from {s}", .{model_tokenizer_path}); + log.info("Loading tokenizer", .{}); var timer = try stdx.time.Timer.start(); - defer log.info("Loaded tokenizer from {s} [{D}]", .{ model_tokenizer_path, timer.read() }); + defer log.info("Loaded tokenizer [{D}]", .{timer.read()}); + const bytes = b: { + const file = try repo.openFile(io, "tokenizer.json", .{}); + defer file.close(io); - break :blk try zml.tokenizer.Tokenizer.fromFile(allocator, model_tokenizer_path); + var reader = file.reader(io, &.{}); + var writer: std.Io.Writer.Allocating = .init(allocator); + defer writer.deinit(); + _ = try reader.interface.streamRemaining(&writer.writer); + break :b try writer.toOwnedSlice(); + }; + defer allocator.free(bytes); + + const tokenizer: zml.tokenizer.Tokenizer = .{ .hftokenizers = try zml.tokenizer.hftokenizers.HFTokenizer.fromBytes(bytes) }; + break :blk tokenizer; }; errdefer tokenizer.deinit(); - const prompt = cli.args.prompt orelse "What is the capital of France?"; - log.info("✅\tPrompt: {s}", .{prompt}); + //const prompt = cli.args.prompt orelse "What is the capital of France?"; + //log.info("✅\tPrompt: {s}", .{prompt}); - // Unbuffered writing of the tokens to stdout. - var stdout = std.fs.File.stdout().writer(&.{}); + //// Unbuffered writing of the tokens to stdout. + //var stdout = std.fs.File.stdout().writer(&.{}); - const seed: u128 = cli.args.seed orelse @bitCast(std.time.nanoTimestamp()); - const skip_llama3_encoding = cli.args.@"no-llama3" orelse false; + //const seed: u128 = cli.args.seed orelse @bitCast(std.time.nanoTimestamp()); + //const skip_llama3_encoding = cli.args.@"no-llama3" orelse false; - try generateText( - config, - llama_tensors, - llama_module_prefill, - llama_module, - kv_cache, - tokenizer, - allocator, - seed, - prompt[0..], - skip_llama3_encoding, - &stdout.interface, - ); + //try generateText( + // config, + // llama_tensors, + // llama_module_prefill, + // llama_module, + // kv_cache, + // tokenizer, + // allocator, + // seed, + // prompt[0..], + // skip_llama3_encoding, + // &stdout.interface, + //); } fn bool_parser(in: []const u8) error{}!bool { diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index f7a22ff0..f8cb288e 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -18,7 +18,7 @@ const Mnist = struct { weight: zml.Tensor, bias: zml.Tensor, - pub fn init(store: TensorStore.View) Layer { + pub fn init(store: zml.io.TensorStore.View) Layer { return .{ .weight = store.createTensorWithTags("weight", .{ .d_out, .d }), .bias = store.createTensorWithTags("bias", .{.d_out}), @@ -30,15 +30,15 @@ const Mnist = struct { } }; - pub fn init(store: TensorStore.View) Mnist { + pub fn init(store: zml.io.TensorStore.View) Mnist { return .{ .fc1 = .init(store.withPrefix("fc1")), .fc2 = .init(store.withPrefix("fc2")), }; } - pub fn loadBuffers(self: Mnist, allocator: std.mem.Allocator, io: std.Io, vfs: *zml.io.VFS, store: TensorStore.View, platform: zml.Platform) !zml.Bufferized(Mnist) { - return loadBuffersFromId(allocator, io, vfs, self, store, platform); + pub fn loadBuffers(self: Mnist, allocator: std.mem.Allocator, io: std.Io, vfs: *zml.io.VFS, store: zml.io.TensorStore.View, platform: zml.Platform) !zml.Bufferized(Mnist) { + return zml.io.loadBuffersFromId(allocator, io, vfs, self, store, platform); } pub fn unloadBuffers(self: *zml.Bufferized(Mnist)) void { @@ -96,7 +96,7 @@ pub fn main() !void { defer registry.deinit(); // Init model - var store: TensorStore = .fromRegistry(allocator, ®istry); + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); defer store.deinit(); const mnist_model = Mnist.init(store.view()); @@ -266,374 +266,3 @@ const digits = [_][]const u8{ \\ | ##### | , }; - -pub const TensorStore = struct { - registry: *zml.safetensors.TensorRegistry, - id_map: std.AutoHashMapUnmanaged(usize, *zml.safetensors.Tensor), - allocator: std.mem.Allocator, - - pub fn fromRegistry(allocator: std.mem.Allocator, registry: *zml.safetensors.TensorRegistry) TensorStore { - return .{ - .registry = registry, - .id_map = .empty, - .allocator = allocator, - }; - } - - pub fn deinit(self: *TensorStore) void { - self.id_map.deinit(self.allocator); - } - - fn bindIdToKey(self: *TensorStore, key: []const u8, id: usize) !void { - const tensor_desc_ptr = self.registry.tensors.getPtr(key).?; - - const gop = try self.id_map.getOrPut(self.allocator, id); - if (gop.found_existing) { - stdx.debug.panic("Key {s} already has an associated tensor (id: {})", .{ key, gop.key_ptr.* }); - } - errdefer self.id_map.removeByPtr(gop.key_ptr); - - gop.value_ptr.* = tensor_desc_ptr; - } - - fn getPtrFromKey(self: *const TensorStore, key: []const u8) ?*zml.safetensors.Tensor { - const tensor_desc_ptr = self.registry.tensors.getPtr(key) orelse return null; - return tensor_desc_ptr; - } - - fn getPtrFromId(self: *const TensorStore, id: usize) ?*zml.safetensors.Tensor { - const tensor_desc_ptr = self.id_map.get(id) orelse return null; - return tensor_desc_ptr; - } - - pub fn getReader(self: *const TensorStore, key: []const u8, io: std.Io, vfs: *zml.io.VFS, buffer: []u8) !zml.safetensors.TensorReader { - return self.registry.reader(io, vfs, key, buffer); - } - - pub fn getReaderById(self: *const TensorStore, id: usize, io: std.Io, vfs: *zml.io.VFS, buffer: []u8) !zml.safetensors.TensorReader { - const tensor_desc = self.id_map.get(id) orelse return error.NotFound; - - return zml.safetensors.TensorReader.init(io, vfs, tensor_desc.*, buffer); - } - - pub fn view(self: *TensorStore) View { - return .{ .store = self }; - } - - pub const View = struct { - store: *TensorStore, - - prefix_buffer: [256]u8 = undefined, - prefix_length: usize = 0, - - pub fn root(self: *const View) View { - return .{ - .store = self.store, - }; - } - - pub fn parent(self: *const View) View { - const slice = self.prefix() orelse unreachable; - const index = std.mem.lastIndexOfScalar(u8, slice[0 .. slice.len - 1], '.') orelse return self.root(); - var buffer: [256]u8 = undefined; - @memcpy(buffer[0 .. index + 1], slice[0 .. index + 1]); - return .{ - .store = self.store, - .prefix_buffer = buffer, - .prefix_length = index + 1, - }; - } - - pub fn withPrefix(self: *const View, prefix_: []const u8) View { - var buffer: [256]u8 = undefined; - const new_prefix = std.fmt.bufPrint(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }) catch unreachable; - - return .{ - .store = self.store, - .prefix_buffer = buffer, - .prefix_length = new_prefix.len, - }; - } - - fn prefix(self: *const View) ?[]const u8 { - return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; - } - - pub fn maybeCreateTensor(self: View, subkey: []const u8) ?zml.Tensor { - var buffer: [256]u8 = undefined; - const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; - - const ptr = self.store.getPtrFromKey(key) orelse return null; - - const tensor: zml.Tensor = .init(ptr.shape); - self.store.bindIdToKey(key, tensor.id) catch unreachable; - - return tensor; - } - - pub fn createTensor(self: View, subkey: []const u8) zml.Tensor { - return self.maybeCreateTensor(subkey).?; - } - - pub fn maybeCreateTensorWithTags(self: View, subkey: []const u8, tagz: anytype) ?zml.Tensor { - var buffer: [256]u8 = undefined; - const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; - - const ptr = self.store.getPtrFromKey(key) orelse return null; - ptr.shape = ptr.shape.withTags(tagz); - - const tensor: zml.Tensor = .fromShape(ptr.shape); - self.store.bindIdToKey(key, tensor.id) catch unreachable; - - return tensor; - } - - pub fn createTensorWithTags(self: View, subkey: []const u8, tagz: anytype) zml.Tensor { - return self.maybeCreateTensorWithTags(subkey, tagz).?; - } - - pub fn getShape(self: View, subkey: []const u8) ?zml.Shape { - var buffer: [256]u8 = undefined; - const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; - const entry_ptr = self.store.getPtrFromKey(key) orelse return null; - return entry_ptr.shape; - } - - pub fn getShapeOpts(self: View, subkey: []const u8, opts: struct { no_prefix: bool = false }) ?zml.Shape { - var buffer: [256]u8 = undefined; - const key = if (opts.no_prefix) - subkey - else b: { - break :b std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; - }; - const entry_ptr = self.store.getPtrFromKey(key) orelse return null; - return entry_ptr.shape; - } - - pub fn getReader(self: View, subkey: []const u8, io: std.Io, vfs: *zml.io.VFS, buffer: []u8) !zml.safetensors.TensorReader { - var key_buffer: [256]u8 = undefined; - const key = std.fmt.bufPrint(&key_buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; - return self.store.getReader(key, io, vfs, buffer); - } - }; -}; - -fn collectShapes(allocator: std.mem.Allocator, v: anytype) ![]zml.Shape { - const LocalContext = struct { - list: *std.array_list.Managed(zml.Shape), - }; - var list = std.array_list.Managed(zml.Shape).init(allocator); - errdefer list.deinit(); - - var context: LocalContext = .{ .list = &list }; - try zml.meta.visit(struct { - fn cb(ctx_: *LocalContext, tensor: *const zml.Tensor) !void { - try ctx_.list.append(tensor.shape()); - } - }.cb, &context, v); - - return try list.toOwnedSlice(); -} - -fn collectTensorDesc(allocator: std.mem.Allocator, store: TensorStore.View, v: anytype) ![]zml.safetensors.Tensor { - const LocalContext = struct { - list: *std.array_list.Managed(zml.safetensors.Tensor), - store: TensorStore.View, - }; - var list = std.array_list.Managed(zml.safetensors.Tensor).init(allocator); - var context: LocalContext = .{ .list = &list, .store = store }; - zml.meta.visit(struct { - fn cb(ctx_: *LocalContext, tensor: *const zml.Tensor) void { - const tensor_desc = ctx_.store.store.getPtrFromId(tensor.id).?.*; - ctx_.list.append(tensor_desc) catch unreachable; - } - }.cb, &context, v); - - return try list.toOwnedSlice(); -} - -pub fn loadBuffersFromId(allocator: std.mem.Allocator, io: std.Io, vfs: *zml.io.VFS, model: anytype, store: TensorStore.View, platform: zml.Platform) !zml.Bufferized(@TypeOf(model)) { - const Model = @TypeOf(model); - var result: zml.Bufferized(Model) = undefined; - initBufferizedFrom(model, &result); - - var arena: std.heap.ArenaAllocator = .init(allocator); - defer arena.deinit(); - - const shapes = try collectShapes(arena.allocator(), &model); - - var transfer: Transfer = try .init(arena.allocator(), shapes, platform); - defer transfer.deinit(platform); - - const tensor_descs = try collectTensorDesc(arena.allocator(), store, &model); - - // TODO(Corentin): Find a way to inject that - const buffer_reader = try allocator.alloc(u8, 16 * 1024 * 1024); - defer allocator.free(buffer_reader); - const buffer_writer = try allocator.alloc(u8, 16 * 1024 * 1024); - defer allocator.free(buffer_writer); - - const LocalContext = struct { - tensor_descs: []zml.safetensors.Tensor, - shapes: []const zml.Shape, - platform: zml.Platform, - transfer: *Transfer, - index: usize = 0, - buffer_reader: []u8, - buffer_writer: []u8, - store: TensorStore.View, - allocator: std.mem.Allocator, - io: std.Io, - vfs: *zml.io.VFS, - }; - var context: LocalContext = .{ - .tensor_descs = tensor_descs, - .shapes = shapes, - .platform = platform, - .transfer = &transfer, - .buffer_reader = buffer_reader, - .buffer_writer = buffer_writer, - .store = store, - .allocator = allocator, - .io = io, - .vfs = vfs, - }; - try zml.meta.visit(struct { - fn cb(context_: *LocalContext, buffer: *zml.Buffer) !void { - const tensor_desc = context_.tensor_descs[context_.index]; - - var reader = try zml.safetensors.TensorReader.init(context_.io, context_.vfs, tensor_desc, context_.buffer_reader); - defer reader.deinit(); - - var writer = try context_.transfer.getWriter(context_.index, context_.buffer_writer); - - _ = try reader.interface.streamRemaining(&writer.interface); - try writer.interface.flush(); - - buffer.* = try context_.transfer.getBuffer(context_.index); - context_.index += 1; - } - }.cb, &context, &result); - - return result; -} - -pub fn initBufferizedFrom(model: anytype, bufferized_: *zml.Bufferized(@TypeOf(model))) void { - const Model = @TypeOf(model); - const type_info = @typeInfo(zml.Bufferized(Model)); - switch (type_info) { - .@"struct" => |struct_type_info| { - if (zml.Bufferized(Model) == zml.Buffer) return; - inline for (struct_type_info.fields) |field| { - initBufferizedFrom(@field(model, field.name), &@field(bufferized_, field.name)); - } - }, - .@"union" => { - switch (model) { - inline else => |v, tag| { - bufferized_.* = @unionInit(zml.Bufferized(Model), @tagName(tag), undefined); - initBufferizedFrom(v, @field(bufferized_, @tagName(tag))); - }, - } - }, - .optional => { - if (model == null) { - bufferized_.* = null; - } else { - bufferized_.* = undefined; - initBufferizedFrom(model.?, &bufferized_.*.?); - } - }, - .void, .int, .@"enum", .bool, .enum_literal, .float, .pointer, .vector => {}, - else => unreachable, - } -} - -const SimpleWriter = struct { - offset: usize = 0, - transfer_manager: *zml.pjrt.AsyncHostToDeviceTransferManager, - shape: zml.Shape, - buffer_index: usize, - platform: zml.Platform, - interface: std.Io.Writer, - - pub fn init(buffer: []u8, transfer_manager: *zml.pjrt.AsyncHostToDeviceTransferManager, shape: zml.Shape, buffer_index: usize, platform: zml.Platform) SimpleWriter { - return .{ - .transfer_manager = transfer_manager, - .shape = shape, - .buffer_index = buffer_index, - .platform = platform, - .interface = .{ - .buffer = buffer, - .end = 0, - .vtable = &.{ - .drain = drain, - }, - }, - }; - } - - pub fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { - _ = data; // autofix - _ = splat; // autofix - const writer: *SimpleWriter = @alignCast(@fieldParentPtr("interface", w)); - stdx.debug.assert(writer.offset + w.end <= writer.shape.byteSize(), "Can't write more data than required", .{}); - const is_last_transfer = writer.offset + w.end >= writer.shape.byteSize(); - log.debug("Writing {} bytes", .{w.end}); - _ = writer.transfer_manager.transferData(writer.platform.pjrt_api, writer.buffer_index, w.buffer[0..w.end], @intCast(writer.offset), is_last_transfer) catch return error.WriteFailed; - const written = w.end; - writer.offset += written; - w.end = 0; - return 0; - } -}; - -const Transfer = struct { - shapes: []zml.Shape, - transfer_manager: *zml.pjrt.AsyncHostToDeviceTransferManager, - arena: std.heap.ArenaAllocator, - platform: zml.Platform, - - pub fn init(allocator: std.mem.Allocator, shapes: []const zml.Shape, platform: zml.Platform) !Transfer { - const shape_specs = try allocator.alloc(zml.pjrt.ShapeSpec, shapes.len); - defer allocator.free(shape_specs); - - var temp_arena = std.heap.ArenaAllocator.init(allocator); - defer temp_arena.deinit(); - - var arena = std.heap.ArenaAllocator.init(allocator); - errdefer arena.deinit(); - - for (shape_specs, shapes) |*spec, shape| { - const dims = try temp_arena.allocator().dupe(i64, shape.dims()); - spec.* = zml.pjrt.ShapeSpec.init(dims, zml.pjrt.bufferTypeFromDtype(shape.dtype())); - } - - const memory = platform.pjrt_client.memoryByKind(platform.pjrt_api, .device).?; - - const transfer_manager = try platform.pjrt_client.createBuffersForAsyncHostToDevice(platform.pjrt_api, .{ .shape_specs = shape_specs, .memory = memory }); - errdefer transfer_manager.deinit(platform.pjrt_api); - - return .{ - .shapes = try arena.allocator().dupe(zml.Shape, shapes), - .transfer_manager = transfer_manager, - .arena = arena, - .platform = platform, - }; - } - - pub fn deinit(self: Transfer, platform: zml.Platform) void { - self.arena.deinit(); - self.transfer_manager.deinit(platform.pjrt_api); - } - - pub fn getBuffer(self: *const Transfer, index: usize) !zml.Buffer { - const pjrt_buffer = self.transfer_manager.retrieveBuffer(self.platform.pjrt_api, index) catch return error.NotFound; - return .fromPjrtBuffers(self.platform, self.shapes[index], &.{pjrt_buffer}); - } - - pub fn getWriter(self: *const Transfer, index: usize, buffer: []u8) !SimpleWriter { - const writer: SimpleWriter = .init(buffer, self.transfer_manager, self.shapes[index], index, self.platform); - return writer; - } -}; diff --git a/third_party/com_github_hejsil_clap/repo.bzl b/third_party/com_github_hejsil_clap/repo.bzl index 4359f511..b88dac44 100644 --- a/third_party/com_github_hejsil_clap/repo.bzl +++ b/third_party/com_github_hejsil_clap/repo.bzl @@ -4,6 +4,6 @@ def repo(): new_git_repository( name = "com_github_hejsil_clap", remote = "https://github.com/Hejsil/zig-clap.git", - commit = "5289e0753cd274d65344bef1c114284c633536ea", + commit = "3ae92228409d82a2c906fafc228b7920d9ff046b", build_file = "//:third_party/com_github_hejsil_clap/clap.bazel", ) diff --git a/zml2/io.zig b/zml2/io.zig index 67f54a6c..19ef5d64 100644 --- a/zml2/io.zig +++ b/zml2/io.zig @@ -1,6 +1,399 @@ const builtin = @import("builtin"); const std = @import("std"); +const stdx = @import("stdx"); + pub const VFS = @import("io").VFS; const log = std.log.scoped(.@"zml/io"); + +const safetensors = @import("safetensors.zig"); +const Tensor = @import("tensor.zig").Tensor; +const Shape = @import("shape.zig").Shape; +const Platform = @import("platform.zig").Platform; +const Buffer = @import("buffer.zig").Buffer; +const meta = @import("meta.zig"); +const Bufferized = @import("zml.zig").Bufferized; +const pjrt = @import("pjrtx.zig"); + +pub const TensorStore = struct { + registry: *safetensors.TensorRegistry, + id_map: std.AutoHashMapUnmanaged(usize, *safetensors.Tensor), + allocator: std.mem.Allocator, + + pub fn fromRegistry(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry) TensorStore { + return .{ + .registry = registry, + .id_map = .empty, + .allocator = allocator, + }; + } + + pub fn deinit(self: *TensorStore) void { + self.id_map.deinit(self.allocator); + } + + fn bindIdToKey(self: *TensorStore, key: []const u8, id: usize) !void { + const tensor_desc_ptr = self.registry.tensors.getPtr(key).?; + + const gop = try self.id_map.getOrPut(self.allocator, id); + if (gop.found_existing) { + stdx.debug.panic("Key {s} already has an associated tensor (id: {})", .{ key, gop.key_ptr.* }); + } + errdefer self.id_map.removeByPtr(gop.key_ptr); + + gop.value_ptr.* = tensor_desc_ptr; + } + + fn getPtrFromKey(self: *const TensorStore, key: []const u8) ?*safetensors.Tensor { + const tensor_desc_ptr = self.registry.tensors.getPtr(key) orelse return null; + return tensor_desc_ptr; + } + + fn getPtrFromId(self: *const TensorStore, id: usize) ?*safetensors.Tensor { + const tensor_desc_ptr = self.id_map.get(id) orelse return null; + return tensor_desc_ptr; + } + + pub fn getReader(self: *const TensorStore, key: []const u8, io: std.Io, vfs: *VFS, buffer: []u8) !safetensors.TensorReader { + return self.registry.reader(io, vfs, key, buffer); + } + + pub fn getReaderById(self: *const TensorStore, id: usize, io: std.Io, vfs: *VFS, buffer: []u8) !safetensors.TensorReader { + const tensor_desc = self.id_map.get(id) orelse return error.NotFound; + + return safetensors.TensorReader.init(io, vfs, tensor_desc.*, buffer); + } + + pub fn view(self: *TensorStore) View { + return .{ .store = self }; + } + + pub const View = struct { + store: *TensorStore, + + prefix_buffer: [256]u8 = undefined, + prefix_length: usize = 0, + + pub fn root(self: *const View) View { + return .{ + .store = self.store, + }; + } + + pub fn parent(self: *const View) View { + const slice = self.prefix() orelse unreachable; + const index = std.mem.lastIndexOfScalar(u8, slice[0 .. slice.len - 1], '.') orelse return self.root(); + var buffer: [256]u8 = undefined; + @memcpy(buffer[0 .. index + 1], slice[0 .. index + 1]); + return .{ + .store = self.store, + .prefix_buffer = buffer, + .prefix_length = index + 1, + }; + } + + pub fn withPrefix(self: *const View, prefix_: []const u8) View { + var buffer: [256]u8 = undefined; + const new_prefix = std.fmt.bufPrint(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }) catch unreachable; + + return .{ + .store = self.store, + .prefix_buffer = buffer, + .prefix_length = new_prefix.len, + }; + } + + pub fn withLayer(self: *const View, index: usize) View { + var buffer: [256]u8 = undefined; + const new_prefix = std.fmt.bufPrint(&buffer, "{s}{d}.", .{ self.prefix() orelse "", index }) catch unreachable; + + return .{ + .store = self.store, + .prefix_buffer = buffer, + .prefix_length = new_prefix.len, + }; + } + + fn prefix(self: *const View) ?[]const u8 { + return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; + } + + pub fn maybeCreateTensor(self: View, subkey: []const u8) ?Tensor { + var buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + + const ptr = self.store.getPtrFromKey(key) orelse return null; + + const tensor: Tensor = .fromShape(ptr.shape); + self.store.bindIdToKey(key, tensor.id) catch unreachable; + + return tensor; + } + + pub fn createTensor(self: View, subkey: []const u8) Tensor { + return self.maybeCreateTensor(subkey).?; + } + + pub fn maybeCreateTensorWithTags(self: View, subkey: []const u8, tagz: anytype) ?Tensor { + var buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + + const ptr = self.store.getPtrFromKey(key) orelse return null; + ptr.shape = ptr.shape.withTags(tagz); + + const tensor: Tensor = .fromShape(ptr.shape); + self.store.bindIdToKey(key, tensor.id) catch unreachable; + + return tensor; + } + + pub fn createTensorWithTags(self: View, subkey: []const u8, tagz: anytype) Tensor { + return self.maybeCreateTensorWithTags(subkey, tagz).?; + } + + pub fn getShape(self: View, subkey: []const u8) ?Shape { + var buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + const entry_ptr = self.store.getPtrFromKey(key) orelse return null; + return entry_ptr.shape; + } + + pub fn getShapeOpts(self: View, subkey: []const u8, opts: struct { no_prefix: bool = false }) ?Shape { + var buffer: [256]u8 = undefined; + const key = if (opts.no_prefix) + subkey + else b: { + break :b std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + }; + const entry_ptr = self.store.getPtrFromKey(key) orelse return null; + return entry_ptr.shape; + } + + pub fn getReader(self: View, subkey: []const u8, io: std.Io, vfs: *VFS, buffer: []u8) !safetensors.TensorReader { + var key_buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&key_buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + return self.store.getReader(key, io, vfs, buffer); + } + }; +}; + +fn collectShapes(allocator: std.mem.Allocator, v: anytype) ![]Shape { + const LocalContext = struct { + list: *std.array_list.Managed(Shape), + }; + var list = std.array_list.Managed(Shape).init(allocator); + errdefer list.deinit(); + + var context: LocalContext = .{ .list = &list }; + try meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { + try ctx_.list.append(tensor.shape()); + } + }.cb, &context, v); + + return try list.toOwnedSlice(); +} + +fn collectTensorDesc(allocator: std.mem.Allocator, store: TensorStore.View, v: anytype) ![]safetensors.Tensor { + const LocalContext = struct { + list: *std.array_list.Managed(safetensors.Tensor), + store: TensorStore.View, + }; + var list = std.array_list.Managed(safetensors.Tensor).init(allocator); + var context: LocalContext = .{ .list = &list, .store = store }; + meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { + const tensor_desc = ctx_.store.store.getPtrFromId(tensor.id).?.*; + ctx_.list.append(tensor_desc) catch unreachable; + } + }.cb, &context, v); + + return try list.toOwnedSlice(); +} + +pub fn loadBuffersFromId(allocator: std.mem.Allocator, io: std.Io, vfs: *VFS, model: anytype, store: TensorStore.View, platform: Platform) !Bufferized(@TypeOf(model)) { + const Model = @TypeOf(model); + var result: Bufferized(Model) = undefined; + initBufferizedFrom(model, &result); + + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + + const shapes = try collectShapes(arena.allocator(), &model); + + var transfer: Transfer = try .init(arena.allocator(), shapes, platform); + defer transfer.deinit(platform); + + const tensor_descs = try collectTensorDesc(arena.allocator(), store, &model); + + // TODO(Corentin): Find a way to inject that + const buffer_reader = try allocator.alloc(u8, 16 * 1024 * 1024); + defer allocator.free(buffer_reader); + const buffer_writer = try allocator.alloc(u8, 16 * 1024 * 1024); + defer allocator.free(buffer_writer); + + const LocalContext = struct { + tensor_descs: []safetensors.Tensor, + shapes: []const Shape, + platform: Platform, + transfer: *Transfer, + index: usize = 0, + buffer_reader: []u8, + buffer_writer: []u8, + store: TensorStore.View, + allocator: std.mem.Allocator, + io: std.Io, + vfs: *VFS, + }; + var context: LocalContext = .{ + .tensor_descs = tensor_descs, + .shapes = shapes, + .platform = platform, + .transfer = &transfer, + .buffer_reader = buffer_reader, + .buffer_writer = buffer_writer, + .store = store, + .allocator = allocator, + .io = io, + .vfs = vfs, + }; + try meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *Buffer) !void { + const tensor_desc = context_.tensor_descs[context_.index]; + + var reader = try safetensors.TensorReader.init(context_.io, context_.vfs, tensor_desc, context_.buffer_reader); + defer reader.deinit(); + + var writer = try context_.transfer.getWriter(context_.index, context_.buffer_writer); + + _ = try reader.interface.streamRemaining(&writer.interface); + try writer.interface.flush(); + + buffer.* = try context_.transfer.getBuffer(context_.index); + context_.index += 1; + } + }.cb, &context, &result); + + return result; +} + +pub fn initBufferizedFrom(model: anytype, bufferized_: *Bufferized(@TypeOf(model))) void { + const Model = @TypeOf(model); + const type_info = @typeInfo(Bufferized(Model)); + switch (type_info) { + .@"struct" => |struct_type_info| { + if (Bufferized(Model) == Buffer) return; + inline for (struct_type_info.fields) |field| { + initBufferizedFrom(@field(model, field.name), &@field(bufferized_, field.name)); + } + }, + .@"union" => { + switch (model) { + inline else => |v, tag| { + bufferized_.* = @unionInit(Bufferized(Model), @tagName(tag), undefined); + initBufferizedFrom(v, @field(bufferized_, @tagName(tag))); + }, + } + }, + .optional => { + if (model == null) { + bufferized_.* = null; + } else { + bufferized_.* = undefined; + initBufferizedFrom(model.?, &bufferized_.*.?); + } + }, + .void, .int, .@"enum", .bool, .enum_literal, .float, .pointer, .vector => {}, + else => unreachable, + } +} + +pub const SimpleWriter = struct { + offset: usize = 0, + transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, + shape: Shape, + buffer_index: usize, + platform: Platform, + interface: std.Io.Writer, + + pub fn init(buffer: []u8, transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, shape: Shape, buffer_index: usize, platform: Platform) SimpleWriter { + return .{ + .transfer_manager = transfer_manager, + .shape = shape, + .buffer_index = buffer_index, + .platform = platform, + .interface = .{ + .buffer = buffer, + .end = 0, + .vtable = &.{ + .drain = drain, + }, + }, + }; + } + + pub fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + _ = data; // autofix + _ = splat; // autofix + const writer: *SimpleWriter = @alignCast(@fieldParentPtr("interface", w)); + stdx.debug.assert(writer.offset + w.end <= writer.shape.byteSize(), "Can't write more data than required", .{}); + const is_last_transfer = writer.offset + w.end >= writer.shape.byteSize(); + log.debug("Writing {} bytes", .{w.end}); + _ = writer.transfer_manager.transferData(writer.platform.pjrt_api, writer.buffer_index, w.buffer[0..w.end], @intCast(writer.offset), is_last_transfer) catch return error.WriteFailed; + const written = w.end; + writer.offset += written; + w.end = 0; + return 0; + } +}; + +pub const Transfer = struct { + shapes: []Shape, + transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, + arena: std.heap.ArenaAllocator, + platform: Platform, + + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, platform: Platform) !Transfer { + const shape_specs = try allocator.alloc(pjrt.ShapeSpec, shapes.len); + defer allocator.free(shape_specs); + + var temp_arena = std.heap.ArenaAllocator.init(allocator); + defer temp_arena.deinit(); + + var arena = std.heap.ArenaAllocator.init(allocator); + errdefer arena.deinit(); + + for (shape_specs, shapes) |*spec, shape| { + const dims = try temp_arena.allocator().dupe(i64, shape.dims()); + spec.* = pjrt.ShapeSpec.init(dims, pjrt.bufferTypeFromDtype(shape.dtype())); + } + + const memory = platform.pjrt_client.memoryByKind(platform.pjrt_api, .device).?; + + const transfer_manager = try platform.pjrt_client.createBuffersForAsyncHostToDevice(platform.pjrt_api, .{ .shape_specs = shape_specs, .memory = memory }); + errdefer transfer_manager.deinit(platform.pjrt_api); + + return .{ + .shapes = try arena.allocator().dupe(Shape, shapes), + .transfer_manager = transfer_manager, + .arena = arena, + .platform = platform, + }; + } + + pub fn deinit(self: Transfer, platform: Platform) void { + self.arena.deinit(); + self.transfer_manager.deinit(platform.pjrt_api); + } + + pub fn getBuffer(self: *const Transfer, index: usize) !Buffer { + const pjrt_buffer = self.transfer_manager.retrieveBuffer(self.platform.pjrt_api, index) catch return error.NotFound; + return .fromPjrtBuffers(self.platform, self.shapes[index], &.{pjrt_buffer}); + } + + pub fn getWriter(self: *const Transfer, index: usize, buffer: []u8) !SimpleWriter { + const writer: SimpleWriter = .init(buffer, self.transfer_manager, self.shapes[index], index, self.platform); + return writer; + } +}; diff --git a/zml2/nn.zig b/zml2/nn.zig index 32dd36a7..d244f235 100644 --- a/zml2/nn.zig +++ b/zml2/nn.zig @@ -16,16 +16,21 @@ const log = std.log.scoped(.@"zml/nn"); pub const Linear = struct { weight: Tensor, bias: ?Tensor = null, + tag: Shape.Tag, + + pub fn init(weight: Tensor, bias: ?Tensor, tag: anytype) Linear { + return .{ + .weight = weight, + .bias = bias, + .tag = zml.Shape.toTag(tag), + }; + } pub fn forward(self: Linear, x: Tensor) Tensor { - var y = x.dotGeneral(self.weight.convert(x.dtype()), &.{.{ -1, -1 }}, &.{}); - // If self.weight doesn't have tags, preserve tags from x. - if (y.shape().tag(-1) == Shape.TagUnknown) { - y._shape._tags.set(y.rank() - 1, x.shape().tag(-1)); - } + var y = x.dot(self.weight.convert(x.dtype()), self.tag); // log.debug("Linear({*}): {d} -> {d} -> {d}", .{ self, x.dims(), y.dims(), if (self.bias) |bias| y.add(bias).dims() else y.dims() }); - return if (self.bias) |bias| y.add(bias.broadcast(y.shape(), &.{y.axis(-1)})) else y; + return if (self.bias) |bias| y.add(bias.broad(y.shape())) else y; } }; @@ -939,8 +944,8 @@ pub fn causalAttnMask( } if (dtype.isFloat()) { - const zeros = Tensor.constant(mask.shape(), dtype.zero()); - const minus_inf = Tensor.constant(mask.shape(), dtype.minValue()); + const zeros = Tensor.constant(dtype.zero()).broad(mask.shape()); + const minus_inf = Tensor.constant(dtype.minValue()).broad(mask.shape()); mask = Tensor.select(mask, zeros, minus_inf); } else { mask = mask.convert(dtype); diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 6ae8cea8..b2f73c2f 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -218,7 +218,7 @@ pub const Tensor = struct { } else if (scope.id_to_donation.get(origin.id)) |origin_donation| { const gop = scope.id_to_donation.getOrPut(scope.arena.allocator(), self.id) catch unreachable; gop.value_ptr.* = origin_donation; - } else @panic("Something went really wrong, tensor is not an argument nor has an mlir.Value"); + } return self; } From 0463e503c4c8f7145a0cd61112bcac9f19f70e43 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Wed, 17 Dec 2025 12:01:23 +0000 Subject: [PATCH 146/881] Await transfer event to avoid segfault --- zml2/io.zig | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/zml2/io.zig b/zml2/io.zig index 19ef5d64..764958d9 100644 --- a/zml2/io.zig +++ b/zml2/io.zig @@ -265,7 +265,7 @@ pub fn loadBuffersFromId(allocator: std.mem.Allocator, io: std.Io, vfs: *VFS, mo var reader = try safetensors.TensorReader.init(context_.io, context_.vfs, tensor_desc, context_.buffer_reader); defer reader.deinit(); - var writer = try context_.transfer.getWriter(context_.index, context_.buffer_writer); + var writer = try context_.transfer.getWriter(context_.io, context_.index, context_.buffer_writer); _ = try reader.interface.streamRemaining(&writer.interface); try writer.interface.flush(); @@ -311,14 +311,16 @@ pub fn initBufferizedFrom(model: anytype, bufferized_: *Bufferized(@TypeOf(model pub const SimpleWriter = struct { offset: usize = 0, + io: std.Io, transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, shape: Shape, buffer_index: usize, platform: Platform, interface: std.Io.Writer, - pub fn init(buffer: []u8, transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, shape: Shape, buffer_index: usize, platform: Platform) SimpleWriter { + pub fn init(buffer: []u8, io: std.Io, transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, shape: Shape, buffer_index: usize, platform: Platform) SimpleWriter { return .{ + .io = io, .transfer_manager = transfer_manager, .shape = shape, .buffer_index = buffer_index, @@ -340,7 +342,8 @@ pub const SimpleWriter = struct { stdx.debug.assert(writer.offset + w.end <= writer.shape.byteSize(), "Can't write more data than required", .{}); const is_last_transfer = writer.offset + w.end >= writer.shape.byteSize(); log.debug("Writing {} bytes", .{w.end}); - _ = writer.transfer_manager.transferData(writer.platform.pjrt_api, writer.buffer_index, w.buffer[0..w.end], @intCast(writer.offset), is_last_transfer) catch return error.WriteFailed; + const event = writer.transfer_manager.transferData(writer.platform.pjrt_api, writer.buffer_index, w.buffer[0..w.end], @intCast(writer.offset), is_last_transfer) catch return error.WriteFailed; + event.await(writer.platform.pjrt_api, writer.io) catch return error.WriteFailed; const written = w.end; writer.offset += written; w.end = 0; @@ -392,8 +395,8 @@ pub const Transfer = struct { return .fromPjrtBuffers(self.platform, self.shapes[index], &.{pjrt_buffer}); } - pub fn getWriter(self: *const Transfer, index: usize, buffer: []u8) !SimpleWriter { - const writer: SimpleWriter = .init(buffer, self.transfer_manager, self.shapes[index], index, self.platform); + pub fn getWriter(self: *const Transfer, io: std.Io, index: usize, buffer: []u8) !SimpleWriter { + const writer: SimpleWriter = .init(buffer, io, self.transfer_manager, self.shapes[index], index, self.platform); return writer; } }; From cc0349fd7034a46112678199deca45de1913c183 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Wed, 17 Dec 2025 13:32:55 +0100 Subject: [PATCH 147/881] Update ROCm runtime and sandbox to 7.1.1 (#364) Co-authored-by: Corentin Godeau Co-authored-by: Hugo Mano --- MODULE.bazel | 2 +- MODULE.bazel.lock | 13 +- runtimes/cpu/cpu.bzl | 12 +- runtimes/cuda/cuda.bzl | 58 +- runtimes/cuda/cuda.zig | 4 +- runtimes/cuda/libpjrt_cuda.BUILD.bazel | 1 + runtimes/rocm/libpjrt_rocm.BUILD.bazel | 5 + runtimes/rocm/packages.lock.json | 6902 ++++++++++++++++-------- runtimes/rocm/packages.yaml | 5 +- runtimes/rocm/rocm.bzl | 35 +- runtimes/rocm/zmlxrocm.zig | 12 +- 11 files changed, 4631 insertions(+), 2418 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 176b7ffc..7fab36f7 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -21,7 +21,7 @@ bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep(name = "rules_oci", version = "2.2.6") bazel_dep(name = "rules_proto", version = "7.1.0") bazel_dep(name = "rules_python", version = "1.5.3") -bazel_dep(name = "rules_rust", version = "0.63.0") +bazel_dep(name = "rules_rust", version = "0.68.1") bazel_dep(name = "rules_zig", version = "0.12.2") bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.3.1") bazel_dep(name = "with_cfg.bzl", version = "0.11.0") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 06aa29e2..104ba2c3 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -36,6 +36,7 @@ "https://bazel-registry.zml.ai/modules/bazel_features/1.28.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.3.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.30.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/bazel_features/1.32.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.33.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.34.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/bazel_features/1.4.1/MODULE.bazel": "not found", @@ -114,6 +115,7 @@ "https://bazel-registry.zml.ai/modules/rules_cc/0.2.11/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.12/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.14/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_cc/0.2.4/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_distroless/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_fuzzing/0.5.2/MODULE.bazel": "not found", @@ -166,7 +168,7 @@ "https://bazel-registry.zml.ai/modules/rules_python/0.40.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_python/1.5.3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_python/1.6.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_rust/0.63.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_rust/0.68.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.2.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.3.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.4.1/MODULE.bazel": "not found", @@ -232,6 +234,7 @@ "https://bcr.bazel.build/modules/bazel_features/1.28.0/MODULE.bazel": "4b4200e6cbf8fa335b2c3f43e1d6ef3e240319c33d43d60cc0fbd4b87ece299d", "https://bcr.bazel.build/modules/bazel_features/1.3.0/MODULE.bazel": "cdcafe83ec318cda34e02948e81d790aab8df7a929cec6f6969f13a489ccecd9", "https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87", + "https://bcr.bazel.build/modules/bazel_features/1.32.0/MODULE.bazel": "095d67022a58cb20f7e20e1aefecfa65257a222c18a938e2914fd257b5f1ccdc", "https://bcr.bazel.build/modules/bazel_features/1.33.0/MODULE.bazel": "8b8dc9d2a4c88609409c3191165bccec0e4cb044cd7a72ccbe826583303459f6", "https://bcr.bazel.build/modules/bazel_features/1.34.0/MODULE.bazel": "e8475ad7c8965542e0c7aac8af68eb48c4af904be3d614b6aa6274c092c2ea1e", "https://bcr.bazel.build/modules/bazel_features/1.34.0/source.json": "dfa5c4b01110313153b484a735764d247fee5624bbab63d25289e43b151a657a", @@ -331,6 +334,7 @@ "https://bcr.bazel.build/modules/rules_cc/0.2.12/MODULE.bazel": "4216c383ce3223c7dfbd3afcb09c7056265156150b2a3bd3fd9d02949a7ee1bc", "https://bcr.bazel.build/modules/rules_cc/0.2.14/MODULE.bazel": "353c99ed148887ee89c54a17d4100ae7e7e436593d104b668476019023b58df8", "https://bcr.bazel.build/modules/rules_cc/0.2.14/source.json": "55d0a4587c5592fad350f6e698530f4faf0e7dd15e69d43f8d87e220c78bea54", + "https://bcr.bazel.build/modules/rules_cc/0.2.4/MODULE.bazel": "1ff1223dfd24f3ecf8f028446d4a27608aa43c3f41e346d22838a4223980b8cc", "https://bcr.bazel.build/modules/rules_distroless/0.5.1/MODULE.bazel": "2a63f4744d30749128105da5f96adf7caf5628e37548293f89e7fa39c3b3f2c2", "https://bcr.bazel.build/modules/rules_distroless/0.5.1/source.json": "c6b9ff7f325bfed89c3671757f14c1d1bc6077d0fcef809b8aa2d007cac7dd1d", "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", @@ -392,8 +396,8 @@ "https://bcr.bazel.build/modules/rules_python/1.5.3/MODULE.bazel": "d0b7fb08458ca7fd80a26bc00c9e0f1d011609cc3da0381faa2eccd88c6ebd98", "https://bcr.bazel.build/modules/rules_python/1.6.0/MODULE.bazel": "7e04ad8f8d5bea40451cf80b1bd8262552aa73f841415d20db96b7241bd027d8", "https://bcr.bazel.build/modules/rules_python/1.6.0/source.json": "e980f654cf66ec4928672f41fc66c4102b5ea54286acf4aecd23256c84211be6", - "https://bcr.bazel.build/modules/rules_rust/0.63.0/MODULE.bazel": "4144e1606661c7168d23e8b4e7c5f6fb28ef519d9d5d63e0bd789d1b2a4611f8", - "https://bcr.bazel.build/modules/rules_rust/0.63.0/source.json": "638d4731ad05d31835ba45cffc06e8dc1cca01692a681daf831378cf952ee7e6", + "https://bcr.bazel.build/modules/rules_rust/0.68.1/MODULE.bazel": "8d3332ef4079673385eb81f8bd68b012decc04ac00c9d5a01a40eff90301732c", + "https://bcr.bazel.build/modules/rules_rust/0.68.1/source.json": "3378e746f81b62457fdfd37391244fa8ff075ba85c05931ee4f3a20ac1efe963", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", @@ -1190,7 +1194,7 @@ }, "@@rules_rust+//crate_universe:extensions.bzl%crate": { "general": { - "bzlTransitiveDigest": "dgvl+ECngVEy/RaoBW5zRnnG7OibigZhl1yu8+X1gzg=", + "bzlTransitiveDigest": "xnOdcJlDn8jAdyIY4BnC1iO+osURpMjNcjvqwmM3hfE=", "usagesDigest": "mup6mgSFatk34Q0znMZPpIEOFZyk9v3xf6jRIXEFTxc=", "recordedFileInputs": { "@@//zml/tokenizer/hftokenizers/Cargo.lock": "d78ffab37ef69bf8ff45a4d686b25175f30e8fed4d8375612e84dcf6db0cfc9c", @@ -1204,6 +1208,7 @@ "CARGO_BAZEL_ISOLATED": null, "CARGO_BAZEL_REPIN": null, "CARGO_BAZEL_REPIN_ONLY": null, + "CARGO_BAZEL_TIMEOUT": null, "REPIN": null }, "generatedRepoSpecs": { diff --git a/runtimes/cpu/cpu.bzl b/runtimes/cpu/cpu.bzl index 675ff3b4..40dc257d 100644 --- a/runtimes/cpu/cpu.bzl +++ b/runtimes/cpu/cpu.bzl @@ -23,22 +23,22 @@ def _cpu_pjrt_plugin_impl(mctx): http_archive( name = "libpjrt_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "ecc26dc792d2577474348eb48f3989aba8c3bb8d3cbd6df77ccf43357092a700", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v14.0.1/pjrt-cpu_linux-amd64.tar.gz", + sha256 = "880a9575be4c950cd49b6ca20c2782869cea7ec445bac843ac55b21053b648ef", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v15.0.2/pjrt-cpu_linux-amd64.tar.gz", ) http_archive( name = "libpjrt_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "4a21db4ecd015fb772614ce4b491551d483ce11321c8784e3d0e07a9a425d5eb", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v14.0.1/pjrt-cpu_darwin-amd64.tar.gz", + sha256 = "e1bd183a862fa1c504ade5aca1e000b273628ab2bd2a9eb2971067584934d882", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v15.0.2/pjrt-cpu_darwin-amd64.tar.gz", ) http_archive( name = "libpjrt_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "e0ab4492468999ae7861a27837427846a708f4346fdae9ad1e84b80e1313566a", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v14.0.1/pjrt-cpu_darwin-arm64.tar.gz", + sha256 = "fb733aeec1284bfe4573acbe78ca38292ab521e8f207a3547d56d1a6d7173d47", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v15.0.2/pjrt-cpu_darwin-arm64.tar.gz", ) return mctx.extension_metadata( diff --git a/runtimes/cuda/cuda.bzl b/runtimes/cuda/cuda.bzl index 0b1a35bc..ea2e0a95 100644 --- a/runtimes/cuda/cuda.bzl +++ b/runtimes/cuda/cuda.bzl @@ -10,16 +10,16 @@ package(default_visibility = ["//visibility:public"]) ARCH = "linux-x86_64" CUDA_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cuda/redist/" -CUDA_VERSION = "12.8.1" -CUDA_REDIST_JSON_SHA256 = "249e28a83008d711d5f72880541c8be6253f6d61608461de4fcb715554a6cf17" +CUDA_VERSION = "13.0.2" +CUDA_REDIST_JSON_SHA256 = "" CUDNN_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cudnn/redist/" -CUDNN_VERSION = "9.8.0" -CUDNN_REDIST_JSON_SHA256 = "a1599fa1f8dcb81235157be5de5ab7d3936e75dfc4e1e442d07970afad3c4843" +CUDNN_VERSION = "9.14.0" +CUDNN_REDIST_JSON_SHA256 = "fe58e8e9559ef5c61ab7a9954472d16acdcbad3b099004296ae410d25982830d" NVSHMEM_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/nvshmem/redist/" -NVSHMEM_VERSION = "3.2.5" -NVSHMEM_REDIST_JSON_SHA256 = "6945425d3bfd24de23c045996f93ec720c010379bfd6f0860ac5f2716659442d" +NVSHMEM_VERSION = "3.4.5" +NVSHMEM_REDIST_JSON_SHA256 = "a656614a6ec638d85922bc816e5e26063308c3905273a72a863cf0f24e188f38" _UBUNTU_PACKAGES = { "zlib1g": packages.filegroup(name = "zlib1g", srcs = ["lib/x86_64-linux-gnu/libz.so.1"]), @@ -36,31 +36,26 @@ CUDA_PACKAGES = { #TODO: Remove me as soon we use the Driver API in tracer.zig packages.filegroup( name = "cuda_cudart", - srcs = ["lib/libcudart.so.12"], + srcs = ["lib/libcudart.so.13"], ), ]), "cuda_cupti": packages.filegroup( name = "cuda_cupti", - srcs = ["lib/libcupti.so.12"], + srcs = ["lib/libcupti.so.13"], ), "cuda_nvtx": "\n".join([ - # packages.cc_library( - # name = "nvtx", - # hdrs = glob(["include/nvtx3/**/*.h"]), - # visibility = ["//visibility:public"], - # ), packages.filegroup( name = "cuda_nvtx", - srcs = ["lib/libnvToolsExt.so.1"], + srcs = ["lib/libnvtx3interop.so"], ), ]), "libcufft": packages.filegroup( name = "libcufft", - srcs = ["lib/libcufft.so.11"], + srcs = ["lib/libcufft.so.12"], ), "libcusolver": packages.filegroup( name = "libcusolver", - srcs = ["lib/libcusolver.so.11"], + srcs = ["lib/libcusolver.so.12"], ), "libcusparse": packages.filegroup( name = "libcusparse", @@ -68,7 +63,7 @@ CUDA_PACKAGES = { ), "libnvjitlink": packages.filegroup( name = "libnvjitlink", - srcs = ["lib/libnvJitLink.so.12"], + srcs = ["lib/libnvJitLink.so.13"], ), "cuda_nvcc": "\n".join([ packages.filegroup( @@ -76,8 +71,19 @@ CUDA_PACKAGES = { srcs = [ "bin/ptxas", "bin/nvlink", + ], + ), + packages.cc_import( + name = "nvptxcompiler", + static_library = "lib/libnvptxcompiler_static.a", + ), + ]), + "libnvvm": "\n".join([ + packages.filegroup( + name = "libnvvm", + srcs = [ + "nvvm/bin/cicc", "nvvm/libdevice/libdevice.10.bc", - "nvvm/lib64/libnvvm.so.4", ], ), packages.cc_import( @@ -89,8 +95,8 @@ CUDA_PACKAGES = { packages.filegroup( name = "cuda_nvrtc", srcs = [ - "lib/libnvrtc.so.12", - "lib/libnvrtc-builtins.so.12.8", + "lib/libnvrtc.so.13", + "lib/libnvrtc-builtins.so.13.0", ], ), ]), @@ -98,8 +104,8 @@ CUDA_PACKAGES = { packages.filegroup( name = "libcublas", srcs = [ - "lib/libcublasLt.so.12", - "lib/libcublas.so.12", + "lib/libcublasLt.so.13", + "lib/libcublas.so.13", ], ), ]), @@ -217,9 +223,9 @@ def _cuda_impl(mctx): http_archive( name = "nccl", - urls = ["https://files.pythonhosted.org/packages/11/0c/8c78b7603f4e685624a3ea944940f1e75f36d71bd6504330511f4a0e1557/nvidia_nccl_cu12-2.25.1-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl"], + urls = ["https://pypi.nvidia.com/nvidia-nccl-cu13/nvidia_nccl_cu13-2.27.7-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl"], type = "zip", - sha256 = "362aed5963fb9ea2ed2f264409baae30143498fd0e5c503aeaa1badd88cdc54a", + sha256 = "b28a524abd8389b76a4a3f133c76a7aaa7005e47fcaa9d9603b90103927a3f93", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + packages.filegroup( name = "nccl", srcs = ["nvidia/nccl/lib/libnccl.so.2"], @@ -229,8 +235,8 @@ def _cuda_impl(mctx): http_archive( name = "libpjrt_cuda", build_file = "libpjrt_cuda.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v14.0.1/pjrt-cuda_linux-amd64.tar.gz", - sha256 = "4b618f05f9cd4cd14966717f7a521b1aa80b425999755870ce2d1caf45685578", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v15.0.2/pjrt-cuda_linux-amd64.tar.gz", + sha256 = "ebc5f0fa54d38ec85346f0f02b29a6497ee394b9e5fa4832da5db16d3296ae84", ) return mctx.extension_metadata( diff --git a/runtimes/cuda/cuda.zig b/runtimes/cuda/cuda.zig index 30148e9b..01f236cb 100644 --- a/runtimes/cuda/cuda.zig +++ b/runtimes/cuda/cuda.zig @@ -69,9 +69,9 @@ pub fn load(io: std.Io) !*const pjrt.Api { { var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; - const path = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libnvToolsExt.so.1" }); + const path = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libnvtx3interop.so" }); _ = std.c.dlopen(path, .{ .NOW = true, .GLOBAL = true }) orelse { - log.err("Unable to dlopen libnvToolsExt.so.1: {s}", .{std.c.dlerror().?}); + log.err("Unable to dlopen libnvtx3interop.so: {s}", .{std.c.dlerror().?}); return error.DlError; }; } diff --git a/runtimes/cuda/libpjrt_cuda.BUILD.bazel b/runtimes/cuda/libpjrt_cuda.BUILD.bazel index b63a30fb..92c3ca31 100644 --- a/runtimes/cuda/libpjrt_cuda.BUILD.bazel +++ b/runtimes/cuda/libpjrt_cuda.BUILD.bazel @@ -20,6 +20,7 @@ copy_to_directory( "@cuda_cudart", "@cuda_cupti", "@cuda_nvcc", + "@libnvvm", "@cuda_nvrtc", "@cuda_nvtx", "@cudnn", diff --git a/runtimes/rocm/libpjrt_rocm.BUILD.bazel b/runtimes/rocm/libpjrt_rocm.BUILD.bazel index 7a59252b..72fd2a12 100644 --- a/runtimes/rocm/libpjrt_rocm.BUILD.bazel +++ b/runtimes/rocm/libpjrt_rocm.BUILD.bazel @@ -40,6 +40,7 @@ copy_to_directory( "@hip-runtime-amd//:amdhip", "@hip-runtime-amd//:hiprtc", "@hipblaslt//:hipblaslt", + "@hipblaslt//:rocroller", "@hipfft", "@hipsolver", "@hsa-amd-aqlprofile//:hsa-amd-aqlprofile", @@ -60,6 +61,9 @@ copy_to_directory( "@rocm-device-libs//:runfiles", "@rocm-smi-lib//:rocm_smi", "@rocprofiler-register", + "@rocprofiler-sdk//:rocprofiler-sdk", + "@rocprofiler-sdk-rocpd//:rocprofiler-sdk-rocpd", + "@rocprofiler-sdk-roctx//:rocprofiler-sdk-roctx", "@rocsolver", "@roctracer", "@roctracer//:roctx", @@ -79,6 +83,7 @@ copy_to_directory( "opt/amdgpu/lib/x86_64-linux-gnu": "lib", "opt/amdgpu/share": "share", "rocblas": "lib", + "rocroller": "lib", "runtimes/rocm": "lib", "usr/lib/x86_64-linux-gnu": "lib", }, diff --git a/runtimes/rocm/packages.lock.json b/runtimes/rocm/packages.lock.json index 213b57c9..4737457d 100755 --- a/runtimes/rocm/packages.lock.json +++ b/runtimes/rocm/packages.lock.json @@ -4,37 +4,37 @@ "arch": "amd64", "dependencies": [ { - "key": "amdgpu-core_1-6.4.60401-2164967.22.04_amd64", + "key": "amdgpu-core_1-7.1.70101-2255337.22.04_amd64", "name": "amdgpu-core", - "version": "1:6.4.60401-2164967.22.04" + "version": "1:7.1.70101-2255337.22.04" } ], - "key": "libdrm-amdgpu-common_1.0.0.60401-2164967.22.04_amd64", + "key": "libdrm-amdgpu-common_1.0.0.70101-2255337.22.04_amd64", "name": "libdrm-amdgpu-common", - "sha256": "e37673ce63a3f01d33d40b38f3aa8b5d08f5fc10c985909df5cd7acf5bb44267", + "sha256": "19330c218a1643f8b683b6c68c7bb0754932d839948e8a1d3f78e64ac5e111ff", "urls": [ - "https://repo.radeon.com/amdgpu/6.4.1/ubuntu/pool/main/libd/libdrm-amdgpu-common/libdrm-amdgpu-common_1.0.0.60401-2164967.22.04_all.deb" + "https://repo.radeon.com/graphics/7.1.1/ubuntu/pool/main/libd/libdrm-amdgpu-common/libdrm-amdgpu-common_1.0.0.70101-2255337.22.04_all.deb" ], - "version": "1.0.0.60401-2164967.22.04" + "version": "1.0.0.70101-2255337.22.04" }, { "arch": "amd64", "dependencies": [], - "key": "amdgpu-core_1-6.4.60401-2164967.22.04_amd64", + "key": "amdgpu-core_1-7.1.70101-2255337.22.04_amd64", "name": "amdgpu-core", - "sha256": "9d4244a588009ae3951fa4e1c9d20c243a663bda14043849ecff40c493a0e73f", + "sha256": "2b1b81ab71327771fa7ec0dad3673441a97d5c20348128fe71d8ef6b6ce7c9ca", "urls": [ - "https://repo.radeon.com/amdgpu/6.4.1/ubuntu/pool/main/a/amdgpu-core/amdgpu-core_6.4.60401-2164967.22.04_all.deb" + "https://repo.radeon.com/graphics/7.1.1/ubuntu/pool/main/a/amdgpu-core/amdgpu-core_7.1.70101-2255337.22.04_all.deb" ], - "version": "1:6.4.60401-2164967.22.04" + "version": "1:7.1.70101-2255337.22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocprofiler-register_0.4.0.60401-83_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70101-38_22.04_amd64", "name": "rocprofiler-register", - "version": "0.4.0.60401-83~22.04" + "version": "0.6.0.70101-38~22.04" }, { "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", @@ -62,311 +62,29 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", + "key": "rocm-core_7.1.1.70101-38_22.04_amd64", "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" + "version": "7.1.1.70101-38~22.04" }, { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" }, { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" }, { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" }, { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.124.60401-2164967.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.124.60401-2164967.22.04" - }, - { - "key": "libdrm-amdgpu-common_1.0.0.60401-2164967.22.04_amd64", - "name": "libdrm-amdgpu-common", - "version": "1.0.0.60401-2164967.22.04" - }, - { - "key": "amdgpu-core_1-6.4.60401-2164967.22.04_amd64", - "name": "amdgpu-core", - "version": "1:6.4.60401-2164967.22.04" - }, - { - "key": "libdrm2-amdgpu_1-2.4.124.60401-2164967.22.04_amd64", - "name": "libdrm2-amdgpu", - "version": "1:2.4.124.60401-2164967.22.04" - } - ], - "key": "hsa-rocr_1.15.0.60401-83_22.04_amd64", - "name": "hsa-rocr", - "sha256": "07a37ac162acbc5f054b0ea9cfd4e37e1640adee367d31aad15d504c49999372", - "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/h/hsa-rocr/hsa-rocr_1.15.0.60401-83~22.04_amd64.deb" - ], - "version": "1.15.0.60401-83~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", - "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" - } - ], - "key": "rocprofiler-register_0.4.0.60401-83_22.04_amd64", - "name": "rocprofiler-register", - "sha256": "da49a66ca3e6ee8b9491777c2b5170b6020e8308371e26b869d7af81bc50f571", - "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/r/rocprofiler-register/rocprofiler-register_0.4.0.60401-83~22.04_amd64.deb" - ], - "version": "0.4.0.60401-83~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "sha256": "83873058e692936a09649ede2f8e70b87dde1f3f5488db53da8081b81c79a5d9", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libstdc++6_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "sha256": "8ca7b573446d22ef46a97b4ec537c38acc8653f76391127253040acc7b74af5f", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libgcc-s1_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "sha256": "d8e36ba6cc55719d728c38166bf753650710878e02e30af2aa79aefa244eb275", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc6_2.35-0ubuntu3.10_amd64.deb" - ], - "version": "2.35-0ubuntu3.10" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "sha256": "3fa566e9f861a08736cbc5a97562d9d6e4f0c00450fbeafcb6d7583423b04a98", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libx/libxcrypt/libcrypt1_4.4.27-1_amd64.deb" - ], - "version": "1:4.4.27-1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "sha256": "80a1f5df097af3978f7df2808006ca77197865e5eb7a923712aee6876e7a14cb", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/gcc-12-base_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", - "name": "rocm-core", - "sha256": "221236cf665597beea8c6faeffa1569fc35a6ad4cab89d5c8931974120b42621", - "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/r/rocm-core/rocm-core_6.4.1.60401-83~22.04_amd64.deb" - ], - "version": "6.4.1.60401-83~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "sha256": "2cc781614b045b450a1e073ee3b0902f393ee58ede1d2ea4526ad805fcccf538", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/e/elfutils/libelf1_0.186-1ubuntu0.1_amd64.deb" - ], - "version": "0.186-1ubuntu0.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "sha256": "9dc17e51a1be2d9ed63b7b84ef0e4e29c5abe6f1bc62cb03e7181483cce8a2f2", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/z/zlib/zlib1g_1.2.11.dfsg-2ubuntu9.2_amd64.deb" - ], - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "sha256": "0721c89001fbbd1ada23e89da5d60e762763c1a7b3dc814a2e9a518480a8043d", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/numactl/libnuma1_2.0.14-3ubuntu2_amd64.deb" - ], - "version": "2.0.14-3ubuntu2" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libdrm-amdgpu-amdgpu1_1-2.4.124.60401-2164967.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "sha256": "5c01c079bf95f08e920ad41ab8133faef5ce6e5e6908a5b5fa55d2365557a497", - "urls": [ - "https://repo.radeon.com/amdgpu/6.4.1/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-amdgpu1_2.4.124.60401-2164967.22.04_amd64.deb" - ], - "version": "1:2.4.124.60401-2164967.22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libdrm2-amdgpu_1-2.4.124.60401-2164967.22.04_amd64", - "name": "libdrm2-amdgpu", - "sha256": "a0eb10e4765d721d83d93312e47247a618a491dcc73c39e5e4414adc87e6ea93", - "urls": [ - "https://repo.radeon.com/amdgpu/6.4.1/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm2-amdgpu_2.4.124.60401-2164967.22.04_amd64.deb" - ], - "version": "1:2.4.124.60401-2164967.22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", - "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" - } - ], - "key": "hsa-amd-aqlprofile_1.0.0.60401-83_22.04_amd64", - "name": "hsa-amd-aqlprofile", - "sha256": "9e0917b47d40318f73d4323bdc0fdaa27202931544bc4e89f706c4ddd9bd9428", - "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/h/hsa-amd-aqlprofile/hsa-amd-aqlprofile_1.0.0.60401-83~22.04_amd64.deb" - ], - "version": "1.0.0.60401-83~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", - "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" }, { "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", @@ -433,6 +151,16 @@ "name": "libacl1", "version": "2.3.1-1" }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, { "key": "liblzma5_5.2.5-2ubuntu1_amd64", "name": "liblzma5", @@ -458,137 +186,466 @@ "name": "libcom-err2", "version": "1.46.5-2ubuntu1.2" }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, { "key": "libnsl2_1.3.0-2build2_amd64", "name": "libnsl2", "version": "1.3.0-2build2" }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, { "key": "libncursesw6_6.3-2ubuntu0.1_amd64", "name": "libncursesw6", "version": "6.3-2ubuntu0.1" }, { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "version": "6.3-2ubuntu0.1" + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libelf1_0.186-1ubuntu0.1_amd64", + "name": "libelf1", + "version": "0.186-1ubuntu0.1" + }, + { + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" + }, + { + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70101-2255337.22.04_amd64", + "name": "libdrm-amdgpu-amdgpu1", + "version": "1:2.4.125.70101-2255337.22.04" + }, + { + "key": "libdrm-amdgpu-common_1.0.0.70101-2255337.22.04_amd64", + "name": "libdrm-amdgpu-common", + "version": "1.0.0.70101-2255337.22.04" + }, + { + "key": "amdgpu-core_1-7.1.70101-2255337.22.04_amd64", + "name": "amdgpu-core", + "version": "1:7.1.70101-2255337.22.04" + }, + { + "key": "libdrm2-amdgpu_1-2.4.125.70101-2255337.22.04_amd64", + "name": "libdrm2-amdgpu", + "version": "1:2.4.125.70101-2255337.22.04" } ], - "key": "comgr_3.0.0.60401-83_22.04_amd64", - "name": "comgr", - "sha256": "ba02bdf830458668a39300ad082419f4bac3644bf5727c496846697e9fddf429", + "key": "hsa-rocr_1.18.0.70101-38_22.04_amd64", + "name": "hsa-rocr", + "sha256": "34cb8d18abbad9c9bfada312e5e72b80cdd8d6e4b3410b4090cf5697ce06e1f0", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/c/comgr/comgr_3.0.0.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/h/hsa-rocr/hsa-rocr_1.18.0.70101-38~22.04_amd64.deb" ], - "version": "3.0.0.60401-83~22.04" + "version": "1.18.0.70101-38~22.04" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "name": "rocm-core", + "version": "7.1.1.70101-38~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + } + ], + "key": "rocprofiler-register_0.6.0.70101-38_22.04_amd64", + "name": "rocprofiler-register", + "sha256": "379c4129de4f0087c142e75b3891822569d1bde6abfee3cad9d7e83bd8054b63", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocprofiler-register/rocprofiler-register_0.6.0.70101-38~22.04_amd64.deb" + ], + "version": "0.6.0.70101-38~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "sha256": "ae7db00ce8b093e50c994518b90203544e063b4bc574836a048bb142b950b2c9", + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "sha256": "83873058e692936a09649ede2f8e70b87dde1f3f5488db53da8081b81c79a5d9", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libz/libzstd/libzstd1_1.4.8+dfsg-3build1_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libstdc++6_12.3.0-1ubuntu1~22.04_amd64.deb" ], - "version": "1.4.8+dfsg-3build1" + "version": "12.3.0-1ubuntu1~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "sha256": "145736b5bec52cd4fc1b2c09517759383b95255f9fd304ace1de7d830295bb33", + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "sha256": "8ca7b573446d22ef46a97b4ec537c38acc8653f76391127253040acc7b74af5f", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libtinfo-dev_6.3-2ubuntu0.1_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libgcc-s1_12.3.0-1ubuntu1~22.04_amd64.deb" ], - "version": "6.3-2ubuntu0.1" + "version": "12.3.0-1ubuntu1~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "sha256": "1de196ffdd885b5953a44cde13e68b77dc7ceb9c9921482dc84e358bd1ad8b02", + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "sha256": "d8e36ba6cc55719d728c38166bf753650710878e02e30af2aa79aefa244eb275", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libncurses-dev_6.3-2ubuntu0.1_amd64.deb" - ], - "version": "6.3-2ubuntu0.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "sha256": "6382f5c1d7d0bf39cbb94e208ece2bdf556cb3a8dfe278748542c7b8b57a2e6d", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/ncurses-bin_6.3-2ubuntu0.1_amd64.deb" - ], - "version": "6.3-2ubuntu0.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "sha256": "1ec03fe34a7dfd8640a0564a3cad385a10b4f68ebf2190a4c08ccadb395133d3", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libtinfo6_6.3-2ubuntu0.1_amd64.deb" - ], - "version": "6.3-2ubuntu0.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "sha256": "c2e3b50af761886617462ca47ebc9303a83968129acca2add42b8eba0a7abca3", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc6-dev_2.35-0ubuntu3.10_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc6_2.35-0ubuntu3.10_amd64.deb" ], "version": "2.35-0ubuntu3.10" }, { "arch": "amd64", "dependencies": [], - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "sha256": "e93aa448fe708a4bbfcc40b2a8cab6f5f88896174a301caab2719f734e08a621", + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "sha256": "3fa566e9f861a08736cbc5a97562d9d6e4f0c00450fbeafcb6d7583423b04a98", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libn/libnsl/libnsl-dev_1.3.0-2build2_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libx/libxcrypt/libcrypt1_4.4.27-1_amd64.deb" ], - "version": "1.3.0-2build2" + "version": "1:4.4.27-1" }, { "arch": "amd64", "dependencies": [], - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "sha256": "f4254aff5a32ef3c9fca06d3f4f2eff71c93b9393cc1ecc718f7478a5d6a83da", + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "sha256": "80a1f5df097af3978f7df2808006ca77197865e5eb7a923712aee6876e7a14cb", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libt/libtirpc/libtirpc-dev_1.3.2-2ubuntu0.1_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/gcc-12-base_12.3.0-1ubuntu1~22.04_amd64.deb" ], - "version": "1.3.2-2ubuntu0.1" + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "name": "rocm-core", + "sha256": "7332c2d6101392fae6b83889e25208e6a261f1152407a5d332aa6ad4da711931", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocm-core/rocm-core_7.1.1.70101-38~22.04_amd64.deb" + ], + "version": "7.1.1.70101-38~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "sha256": "43dbcc12d790ed9360be4a1dc690ac0a7464a5e7c3170785886df84811a3f5a5", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3-defaults/python3_3.10.6-1~22.04.1_amd64.deb" + ], + "version": "3.10.6-1~22.04.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "sha256": "41b8f285f0a1661ece778a63592dcb556710cce3c559d83114fad245bfdf56ea", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3-defaults/libpython3-stdlib_3.10.6-1~22.04.1_amd64.deb" + ], + "version": "3.10.6-1~22.04.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "sha256": "123b338b5857a35e145c2bd7ce890434dbed2e7f6cd03505e86377f9e0aac40b", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3.10/libpython3.10-stdlib_3.10.12-1~22.04.10_amd64.deb" + ], + "version": "3.10.12-1~22.04.10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "sha256": "3a88f3c8c58a32c623b79f1caace557f63cb997c03fa18eaa31ca0285f635919", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/u/util-linux/libuuid1_2.37.2-4ubuntu3.4_amd64.deb" + ], + "version": "2.37.2-4ubuntu3.4" }, { "arch": "amd64", @@ -733,6 +790,28 @@ ], "version": "2.3.1-1" }, + { + "arch": "amd64", + "dependencies": [], + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "sha256": "9dc17e51a1be2d9ed63b7b84ef0e4e29c5abe6f1bc62cb03e7181483cce8a2f2", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/z/zlib/zlib1g_1.2.11.dfsg-2ubuntu9.2_amd64.deb" + ], + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "sha256": "ae7db00ce8b093e50c994518b90203544e063b4bc574836a048bb142b950b2c9", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libz/libzstd/libzstd1_1.4.8+dfsg-3build1_amd64.deb" + ], + "version": "1.4.8+dfsg-3build1" + }, { "arch": "amd64", "dependencies": [], @@ -791,1281 +870,14 @@ { "arch": "amd64", "dependencies": [], - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "sha256": "cfeef478f96ace59617f4f93c2497776b98a33c99bf3602af46844ccf9cba9d3", + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "sha256": "1ec03fe34a7dfd8640a0564a3cad385a10b4f68ebf2190a4c08ccadb395133d3", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libn/libnsl/libnsl2_1.3.0-2build2_amd64.deb" - ], - "version": "1.3.0-2build2" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "sha256": "447c6d8f7624a3d728a12232440d3f10e6e53bbf60cc202b7beb8a3bc9f7acc9", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/r/rpcsvc-proto/rpcsvc-proto_1.4.2-0ubuntu6_amd64.deb" - ], - "version": "1.4.2-0ubuntu6" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "sha256": "a79be2f6e45823dcc09e04d5e98c88ec88d07d5b8895d05b875a8ade8b345efa", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libx/libxcrypt/libcrypt-dev_4.4.27-1_amd64.deb" - ], - "version": "1:4.4.27-1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "sha256": "f534bb052e96efb9f9e08279b18950d1b1f2a7958f3a06c426fed7d2c1500840", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/l/linux/linux-libc-dev_5.15.0-143.153_amd64.deb" - ], - "version": "5.15.0-143.153" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "sha256": "d93f29bc5ad18f69a4fe292fd392dc17e0a9d9bf0a22ee11fcf43e6395b6ecf8", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc-dev-bin_2.35-0ubuntu3.10_amd64.deb" - ], - "version": "2.35-0ubuntu3.10" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "sha256": "88e5178ccdd4317e263dbe03db1a54a6e35ae1f1fd057585655c2b06ab487f6e", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libncursesw6_6.3-2ubuntu0.1_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libtinfo6_6.3-2ubuntu0.1_amd64.deb" ], "version": "6.3-2ubuntu0.1" }, - { - "arch": "amd64", - "dependencies": [], - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "sha256": "4a828e6cb2a721be6f4ed7b34256d3fc9aab1495b2b4923fd579d4dc589496fd", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libncurses6_6.3-2ubuntu0.1_amd64.deb" - ], - "version": "6.3-2ubuntu0.1" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", - "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" - }, - { - "key": "rocrand_3.3.0.60401-83_22.04_amd64", - "name": "rocrand", - "version": "3.3.0.60401-83~22.04" - }, - { - "key": "hipblaslt_0.12.1.60401-83_22.04_amd64", - "name": "hipblaslt", - "version": "0.12.1.60401-83~22.04" - }, - { - "key": "roctracer_4.1.60401.60401-83_22.04_amd64", - "name": "roctracer", - "version": "4.1.60401.60401-83~22.04" - }, - { - "key": "rocblas_4.4.0.60401-83_22.04_amd64", - "name": "rocblas", - "version": "4.4.0.60401-83~22.04" - }, - { - "key": "hip-runtime-amd_6.4.43483.60401-83_22.04_amd64", - "name": "hip-runtime-amd", - "version": "6.4.43483.60401-83~22.04" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "rocprofiler-register_0.4.0.60401-83_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.4.0.60401-83~22.04" - }, - { - "key": "comgr_3.0.0.60401-83_22.04_amd64", - "name": "comgr", - "version": "3.0.0.60401-83~22.04" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "rocminfo_1.0.0.60401-83_22.04_amd64", - "name": "rocminfo", - "version": "1.0.0.60401-83~22.04" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "pciutils_1-3.7.0-6_amd64", - "name": "pciutils", - "version": "1:3.7.0-6" - }, - { - "key": "libpci3_1-3.7.0-6_amd64", - "name": "libpci3", - "version": "1:3.7.0-6" - }, - { - "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", - "name": "pci.ids", - "version": "0.0~2022.01.22-1ubuntu0.1" - }, - { - "key": "libudev1_249.11-0ubuntu3.16_amd64", - "name": "libudev1", - "version": "249.11-0ubuntu3.16" - }, - { - "key": "libkmod2_29-1ubuntu1_amd64", - "name": "libkmod2", - "version": "29-1ubuntu1" - }, - { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.10-2166611.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.10-2166611.22.04" - }, - { - "key": "amdgpu-core_1-6.4.60401-2164967.22.04_amd64", - "name": "amdgpu-core", - "version": "1:6.4.60401-2164967.22.04" - }, - { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" - } - ], - "key": "miopen-hip_3.4.0.60401-83_22.04_amd64", - "name": "miopen-hip", - "sha256": "584431b94f2809f553633a887a7f88d308391114cc9547da305d5ddcd84bd3e5", - "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/m/miopen-hip/miopen-hip_3.4.0.60401-83~22.04_amd64.deb" - ], - "version": "3.4.0.60401-83~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "rocrand_3.3.0.60401-83_22.04_amd64", - "name": "rocrand", - "sha256": "8d77ce48dc4256bcdba122c9243febd083ffc9147c340ef793f362d8611e35d3", - "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/r/rocrand/rocrand_3.3.0.60401-83~22.04_amd64.deb" - ], - "version": "3.3.0.60401-83~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", - "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" - }, - { - "key": "roctracer_4.1.60401.60401-83_22.04_amd64", - "name": "roctracer", - "version": "4.1.60401.60401-83~22.04" - } - ], - "key": "hipblaslt_0.12.1.60401-83_22.04_amd64", - "name": "hipblaslt", - "sha256": "23e6facad7bd1a17223761e06e9c6d1561c5a9e80f052e5f1589d21ab7167e9e", - "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/h/hipblaslt/hipblaslt_0.12.1.60401-83~22.04_amd64.deb" - ], - "version": "0.12.1.60401-83~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", - "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" - } - ], - "key": "roctracer_4.1.60401.60401-83_22.04_amd64", - "name": "roctracer", - "sha256": "58cead537cf07c8a8770bfe28346c3b3c92cc4297b51e307c9032b04434b187c", - "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/r/roctracer/roctracer_4.1.60401.60401-83~22.04_amd64.deb" - ], - "version": "4.1.60401.60401-83~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", - "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" - }, - { - "key": "hip-runtime-amd_6.4.43483.60401-83_22.04_amd64", - "name": "hip-runtime-amd", - "version": "6.4.43483.60401-83~22.04" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "rocprofiler-register_0.4.0.60401-83_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.4.0.60401-83~22.04" - }, - { - "key": "comgr_3.0.0.60401-83_22.04_amd64", - "name": "comgr", - "version": "3.0.0.60401-83~22.04" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "rocminfo_1.0.0.60401-83_22.04_amd64", - "name": "rocminfo", - "version": "1.0.0.60401-83~22.04" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "pciutils_1-3.7.0-6_amd64", - "name": "pciutils", - "version": "1:3.7.0-6" - }, - { - "key": "libpci3_1-3.7.0-6_amd64", - "name": "libpci3", - "version": "1:3.7.0-6" - }, - { - "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", - "name": "pci.ids", - "version": "0.0~2022.01.22-1ubuntu0.1" - }, - { - "key": "libudev1_249.11-0ubuntu3.16_amd64", - "name": "libudev1", - "version": "249.11-0ubuntu3.16" - }, - { - "key": "libkmod2_29-1ubuntu1_amd64", - "name": "libkmod2", - "version": "29-1ubuntu1" - }, - { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.10-2166611.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.10-2166611.22.04" - }, - { - "key": "amdgpu-core_1-6.4.60401-2164967.22.04_amd64", - "name": "amdgpu-core", - "version": "1:6.4.60401-2164967.22.04" - }, - { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" - }, - { - "key": "hipblaslt_0.12.1.60401-83_22.04_amd64", - "name": "hipblaslt", - "version": "0.12.1.60401-83~22.04" - }, - { - "key": "roctracer_4.1.60401.60401-83_22.04_amd64", - "name": "roctracer", - "version": "4.1.60401.60401-83~22.04" - } - ], - "key": "rocblas_4.4.0.60401-83_22.04_amd64", - "name": "rocblas", - "sha256": "0f850a080c312d80c0b928b48efca4fb507c64e95536aaefe7f23c6fc0c1fe09", - "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/r/rocblas/rocblas_4.4.0.60401-83~22.04_amd64.deb" - ], - "version": "4.4.0.60401-83~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "rocprofiler-register_0.4.0.60401-83_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.4.0.60401-83~22.04" - }, - { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", - "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" - }, - { - "key": "comgr_3.0.0.60401-83_22.04_amd64", - "name": "comgr", - "version": "3.0.0.60401-83~22.04" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "rocminfo_1.0.0.60401-83_22.04_amd64", - "name": "rocminfo", - "version": "1.0.0.60401-83~22.04" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "pciutils_1-3.7.0-6_amd64", - "name": "pciutils", - "version": "1:3.7.0-6" - }, - { - "key": "libpci3_1-3.7.0-6_amd64", - "name": "libpci3", - "version": "1:3.7.0-6" - }, - { - "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", - "name": "pci.ids", - "version": "0.0~2022.01.22-1ubuntu0.1" - }, - { - "key": "libudev1_249.11-0ubuntu3.16_amd64", - "name": "libudev1", - "version": "249.11-0ubuntu3.16" - }, - { - "key": "libkmod2_29-1ubuntu1_amd64", - "name": "libkmod2", - "version": "29-1ubuntu1" - }, - { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.10-2166611.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.10-2166611.22.04" - }, - { - "key": "amdgpu-core_1-6.4.60401-2164967.22.04_amd64", - "name": "amdgpu-core", - "version": "1:6.4.60401-2164967.22.04" - }, - { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" - } - ], - "key": "hip-runtime-amd_6.4.43483.60401-83_22.04_amd64", - "name": "hip-runtime-amd", - "sha256": "dee21498aa4fd4dd267efad563b4cc7fc3363d60f46b626c23681631f5f0cf79", - "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/h/hip-runtime-amd/hip-runtime-amd_6.4.43483.60401-83~22.04_amd64.deb" - ], - "version": "6.4.43483.60401-83~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "rocminfo_1.0.0.60401-83_22.04_amd64", - "name": "rocminfo", - "sha256": "55d19b59464dd444002d12bf1fe8c762c15f2ae79f9dccbbf38360b42e3156b6", - "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/r/rocminfo/rocminfo_1.0.0.60401-83~22.04_amd64.deb" - ], - "version": "1.0.0.60401-83~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "sha256": "43dbcc12d790ed9360be4a1dc690ac0a7464a5e7c3170785886df84811a3f5a5", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3-defaults/python3_3.10.6-1~22.04.1_amd64.deb" - ], - "version": "3.10.6-1~22.04.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "sha256": "41b8f285f0a1661ece778a63592dcb556710cce3c559d83114fad245bfdf56ea", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3-defaults/libpython3-stdlib_3.10.6-1~22.04.1_amd64.deb" - ], - "version": "3.10.6-1~22.04.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "sha256": "123b338b5857a35e145c2bd7ce890434dbed2e7f6cd03505e86377f9e0aac40b", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3.10/libpython3.10-stdlib_3.10.12-1~22.04.10_amd64.deb" - ], - "version": "3.10.12-1~22.04.10" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "sha256": "3a88f3c8c58a32c623b79f1caace557f63cb997c03fa18eaa31ca0285f635919", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/u/util-linux/libuuid1_2.37.2-4ubuntu3.4_amd64.deb" - ], - "version": "2.37.2-4ubuntu3.4" - }, { "arch": "amd64", "dependencies": [], @@ -2099,6 +911,28 @@ ], "version": "8.1.2-1" }, + { + "arch": "amd64", + "dependencies": [], + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "sha256": "cfeef478f96ace59617f4f93c2497776b98a33c99bf3602af46844ccf9cba9d3", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libn/libnsl/libnsl2_1.3.0-2build2_amd64.deb" + ], + "version": "1.3.0-2build2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "sha256": "88e5178ccdd4317e263dbe03db1a54a6e35ae1f1fd057585655c2b06ab487f6e", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libncursesw6_6.3-2ubuntu0.1_amd64.deb" + ], + "version": "6.3-2ubuntu0.1" + }, { "arch": "amd64", "dependencies": [], @@ -2201,77 +1035,327 @@ { "arch": "amd64", "dependencies": [], - "key": "pciutils_1-3.7.0-6_amd64", - "name": "pciutils", - "sha256": "830f2957662723860d3ebac949b22867df00d9a4c5ab1c2bf98e10674dc1c756", + "key": "libelf1_0.186-1ubuntu0.1_amd64", + "name": "libelf1", + "sha256": "2cc781614b045b450a1e073ee3b0902f393ee58ede1d2ea4526ad805fcccf538", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pciutils/pciutils_3.7.0-6_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/e/elfutils/libelf1_0.186-1ubuntu0.1_amd64.deb" ], - "version": "1:3.7.0-6" + "version": "0.186-1ubuntu0.1" }, { "arch": "amd64", "dependencies": [], - "key": "libpci3_1-3.7.0-6_amd64", - "name": "libpci3", - "sha256": "624bea7b5a9073a5bc2dfa5d956571d6127c20400fcfcfca1e47b7448ba5d351", + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "sha256": "0721c89001fbbd1ada23e89da5d60e762763c1a7b3dc814a2e9a518480a8043d", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pciutils/libpci3_3.7.0-6_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/numactl/libnuma1_2.0.14-3ubuntu2_amd64.deb" ], - "version": "1:3.7.0-6" + "version": "2.0.14-3ubuntu2" }, { "arch": "amd64", "dependencies": [], - "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", - "name": "pci.ids", - "sha256": "026609ed71363cd6ed13ace2dac0c9813d94be3874c778539a8a64d6d2cc5566", + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70101-2255337.22.04_amd64", + "name": "libdrm-amdgpu-amdgpu1", + "sha256": "c97887b0d1198d3e572ead52ff3061d088d3821430c0066ae489133389ef7117", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pci.ids/pci.ids_0.0~2022.01.22-1ubuntu0.1_all.deb" + "https://repo.radeon.com/graphics/7.1.1/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-amdgpu1_2.4.125.70101-2255337.22.04_amd64.deb" ], - "version": "0.0~2022.01.22-1ubuntu0.1" + "version": "1:2.4.125.70101-2255337.22.04" }, { "arch": "amd64", "dependencies": [], - "key": "libudev1_249.11-0ubuntu3.16_amd64", - "name": "libudev1", - "sha256": "451d4eaa84dbeabc045611937a78489710ad2c2baf277a63aacda7ee5226e3ea", + "key": "libdrm2-amdgpu_1-2.4.125.70101-2255337.22.04_amd64", + "name": "libdrm2-amdgpu", + "sha256": "5314dc876a0a84f60575af8bdc6773cd9fe9cc8072089adac72ca04304d0411c", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/s/systemd/libudev1_249.11-0ubuntu3.16_amd64.deb" + "https://repo.radeon.com/graphics/7.1.1/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm2-amdgpu_2.4.125.70101-2255337.22.04_amd64.deb" ], - "version": "249.11-0ubuntu3.16" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libkmod2_29-1ubuntu1_amd64", - "name": "libkmod2", - "sha256": "f355cebb66405a84cc88bb313440abd39f97cf45d3f468ea9ec7ba34e11b6c8a", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/k/kmod/libkmod2_29-1ubuntu1_amd64.deb" - ], - "version": "29-1ubuntu1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "hsa-runtime-rocr4wsl-amdgpu_25.10-2166611.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "sha256": "5d726440fdedc0a14a818747adc1ce26e5447e88a4260a580afacfafbf68cfba", - "urls": [ - "https://repo.radeon.com/amdgpu/6.4.1/ubuntu/pool/main/h/hsa-runtime-rocr4wsl-amdgpu/hsa-runtime-rocr4wsl-amdgpu_25.10-2166611.22.04_amd64.deb" - ], - "version": "25.10-2166611.22.04" + "version": "1:2.4.125.70101-2255337.22.04" }, { "arch": "amd64", "dependencies": [ + { + "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "name": "rocm-core", + "version": "7.1.1.70101-38~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, { "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", "name": "libstdc++6", "version": "12.3.0-1ubuntu1~22.04" }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "hsa-rocr_1.18.0.70101-38_22.04_amd64", + "name": "hsa-rocr", + "version": "1.18.0.70101-38~22.04" + }, + { + "key": "rocprofiler-register_0.6.0.70101-38_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70101-38~22.04" + }, + { + "key": "libelf1_0.186-1ubuntu0.1_amd64", + "name": "libelf1", + "version": "0.186-1ubuntu0.1" + }, + { + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" + }, + { + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70101-2255337.22.04_amd64", + "name": "libdrm-amdgpu-amdgpu1", + "version": "1:2.4.125.70101-2255337.22.04" + }, + { + "key": "libdrm-amdgpu-common_1.0.0.70101-2255337.22.04_amd64", + "name": "libdrm-amdgpu-common", + "version": "1.0.0.70101-2255337.22.04" + }, + { + "key": "amdgpu-core_1-7.1.70101-2255337.22.04_amd64", + "name": "amdgpu-core", + "version": "1:7.1.70101-2255337.22.04" + }, + { + "key": "libdrm2-amdgpu_1-2.4.125.70101-2255337.22.04_amd64", + "name": "libdrm2-amdgpu", + "version": "1:2.4.125.70101-2255337.22.04" + } + ], + "key": "hsa-amd-aqlprofile_1.0.0.70101-38_22.04_amd64", + "name": "hsa-amd-aqlprofile", + "sha256": "d55331fdf6e4e27dce1b02a2f8ae8b5df811357bb0632df921df176044e6a36f", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/h/hsa-amd-aqlprofile/hsa-amd-aqlprofile_1.0.0.70101-38~22.04_amd64.deb" + ], + "version": "1.0.0.70101-38~22.04" + }, + { + "arch": "amd64", + "dependencies": [ { "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", "name": "libgcc-s1", @@ -2293,14 +1377,1975 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", - "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "rocm-smi-lib_7.5.0.60401-83_22.04_amd64", - "name": "rocm-smi-lib", - "version": "7.5.0.60401-83~22.04" + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "name": "rocm-core", + "version": "7.1.1.70101-38~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", + "name": "libtinfo-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", + "name": "libncurses-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", + "name": "ncurses-bin", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libncurses6_6.3-2ubuntu0.1_amd64", + "name": "libncurses6", + "version": "6.3-2ubuntu0.1" + } + ], + "key": "comgr_3.0.0.70101-38_22.04_amd64", + "name": "comgr", + "sha256": "0b3e5d433a512a7ee07b076fbcf1d3867d0061160d23a741ab37d2be4af141b4", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/c/comgr/comgr_3.0.0.70101-38~22.04_amd64.deb" + ], + "version": "3.0.0.70101-38~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", + "name": "libtinfo-dev", + "sha256": "145736b5bec52cd4fc1b2c09517759383b95255f9fd304ace1de7d830295bb33", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libtinfo-dev_6.3-2ubuntu0.1_amd64.deb" + ], + "version": "6.3-2ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", + "name": "libncurses-dev", + "sha256": "1de196ffdd885b5953a44cde13e68b77dc7ceb9c9921482dc84e358bd1ad8b02", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libncurses-dev_6.3-2ubuntu0.1_amd64.deb" + ], + "version": "6.3-2ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", + "name": "ncurses-bin", + "sha256": "6382f5c1d7d0bf39cbb94e208ece2bdf556cb3a8dfe278748542c7b8b57a2e6d", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/ncurses-bin_6.3-2ubuntu0.1_amd64.deb" + ], + "version": "6.3-2ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "sha256": "c2e3b50af761886617462ca47ebc9303a83968129acca2add42b8eba0a7abca3", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc6-dev_2.35-0ubuntu3.10_amd64.deb" + ], + "version": "2.35-0ubuntu3.10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "sha256": "e93aa448fe708a4bbfcc40b2a8cab6f5f88896174a301caab2719f734e08a621", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libn/libnsl/libnsl-dev_1.3.0-2build2_amd64.deb" + ], + "version": "1.3.0-2build2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "sha256": "f4254aff5a32ef3c9fca06d3f4f2eff71c93b9393cc1ecc718f7478a5d6a83da", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libt/libtirpc/libtirpc-dev_1.3.2-2ubuntu0.1_amd64.deb" + ], + "version": "1.3.2-2ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "sha256": "447c6d8f7624a3d728a12232440d3f10e6e53bbf60cc202b7beb8a3bc9f7acc9", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/r/rpcsvc-proto/rpcsvc-proto_1.4.2-0ubuntu6_amd64.deb" + ], + "version": "1.4.2-0ubuntu6" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "sha256": "a79be2f6e45823dcc09e04d5e98c88ec88d07d5b8895d05b875a8ade8b345efa", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libx/libxcrypt/libcrypt-dev_4.4.27-1_amd64.deb" + ], + "version": "1:4.4.27-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "sha256": "f534bb052e96efb9f9e08279b18950d1b1f2a7958f3a06c426fed7d2c1500840", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/l/linux/linux-libc-dev_5.15.0-143.153_amd64.deb" + ], + "version": "5.15.0-143.153" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "sha256": "d93f29bc5ad18f69a4fe292fd392dc17e0a9d9bf0a22ee11fcf43e6395b6ecf8", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc-dev-bin_2.35-0ubuntu3.10_amd64.deb" + ], + "version": "2.35-0ubuntu3.10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libncurses6_6.3-2ubuntu0.1_amd64", + "name": "libncurses6", + "sha256": "4a828e6cb2a721be6f4ed7b34256d3fc9aab1495b2b4923fd579d4dc589496fd", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libncurses6_6.3-2ubuntu0.1_amd64.deb" + ], + "version": "6.3-2ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "rocprofiler-sdk-rocpd_1.0.0-38_22.04_amd64", + "name": "rocprofiler-sdk-rocpd", + "version": "1.0.0-38~22.04" + }, + { + "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "name": "rocm-core", + "version": "7.1.1.70101-38~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "rocprofiler-sdk-roctx_1.0.0-38_22.04_amd64", + "name": "rocprofiler-sdk-roctx", + "version": "1.0.0-38~22.04" + }, + { + "key": "rocprofiler-register_0.6.0.70101-38_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70101-38~22.04" + } + ], + "key": "rocprofiler-sdk_1.0.0-38_22.04_amd64", + "name": "rocprofiler-sdk", + "sha256": "49268825761685f71efd3a07b538b4b9f4038e4a00b90968877b0c7d451f944e", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocprofiler-sdk/rocprofiler-sdk_1.0.0-38~22.04_amd64.deb" + ], + "version": "1.0.0-38~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "rocprofiler-sdk-rocpd_1.0.0-38_22.04_amd64", + "name": "rocprofiler-sdk-rocpd", + "sha256": "6be1beb6fcbdf4e01a65806ed7b759b2499a708a294a29889f3788d47e2ed0e8", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocprofiler-sdk-rocpd/rocprofiler-sdk-rocpd_1.0.0-38~22.04_amd64.deb" + ], + "version": "1.0.0-38~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "rocprofiler-sdk-roctx_1.0.0-38_22.04_amd64", + "name": "rocprofiler-sdk-roctx", + "sha256": "35112e33ca1cce1b86eb7e31a703320e022c6825c5c8c0f45f9eec01ac58e9ba", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocprofiler-sdk-roctx/rocprofiler-sdk-roctx_1.0.0-38~22.04_amd64.deb" + ], + "version": "1.0.0-38~22.04" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "name": "rocm-core", + "version": "7.1.1.70101-38~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "rocrand_4.1.0.70101-38_22.04_amd64", + "name": "rocrand", + "version": "4.1.0.70101-38~22.04" + }, + { + "key": "hipblaslt_1.1.0.70101-38_22.04_amd64", + "name": "hipblaslt", + "version": "1.1.0.70101-38~22.04" + }, + { + "key": "roctracer_4.1.70101.70101-38_22.04_amd64", + "name": "roctracer", + "version": "4.1.70101.70101-38~22.04" + }, + { + "key": "rocblas_5.1.1.70101-38_22.04_amd64", + "name": "rocblas", + "version": "5.1.1.70101-38~22.04" + }, + { + "key": "hip-runtime-amd_7.1.52802.70101-38_22.04_amd64", + "name": "hip-runtime-amd", + "version": "7.1.52802.70101-38~22.04" + }, + { + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" + }, + { + "key": "rocprofiler-register_0.6.0.70101-38_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70101-38~22.04" + }, + { + "key": "comgr_3.0.0.70101-38_22.04_amd64", + "name": "comgr", + "version": "3.0.0.70101-38~22.04" + }, + { + "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", + "name": "libtinfo-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", + "name": "libncurses-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", + "name": "ncurses-bin", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libncurses6_6.3-2ubuntu0.1_amd64", + "name": "libncurses6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "rocminfo_1.0.0.70101-38_22.04_amd64", + "name": "rocminfo", + "version": "1.0.0.70101-38~22.04" + }, + { + "key": "pciutils_1-3.7.0-6_amd64", + "name": "pciutils", + "version": "1:3.7.0-6" + }, + { + "key": "libpci3_1-3.7.0-6_amd64", + "name": "libpci3", + "version": "1:3.7.0-6" + }, + { + "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", + "name": "pci.ids", + "version": "0.0~2022.01.22-1ubuntu0.1" + }, + { + "key": "libudev1_249.11-0ubuntu3.16_amd64", + "name": "libudev1", + "version": "249.11-0ubuntu3.16" + }, + { + "key": "libkmod2_29-1ubuntu1_amd64", + "name": "libkmod2", + "version": "29-1ubuntu1" + }, + { + "key": "hsa-rocr_1.18.0.70101-38_22.04_amd64", + "name": "hsa-rocr", + "version": "1.18.0.70101-38~22.04" + }, + { + "key": "libelf1_0.186-1ubuntu0.1_amd64", + "name": "libelf1", + "version": "0.186-1ubuntu0.1" + }, + { + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70101-2255337.22.04_amd64", + "name": "libdrm-amdgpu-amdgpu1", + "version": "1:2.4.125.70101-2255337.22.04" + }, + { + "key": "libdrm-amdgpu-common_1.0.0.70101-2255337.22.04_amd64", + "name": "libdrm-amdgpu-common", + "version": "1.0.0.70101-2255337.22.04" + }, + { + "key": "amdgpu-core_1-7.1.70101-2255337.22.04_amd64", + "name": "amdgpu-core", + "version": "1:7.1.70101-2255337.22.04" + }, + { + "key": "libdrm2-amdgpu_1-2.4.125.70101-2255337.22.04_amd64", + "name": "libdrm2-amdgpu", + "version": "1:2.4.125.70101-2255337.22.04" + } + ], + "key": "miopen-hip_3.5.1.70101-38_22.04_amd64", + "name": "miopen-hip", + "sha256": "0ec588284420c27a7b2ef1e0c2578d669a9455a5ee14aa6917675cb765c47f35", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/m/miopen-hip/miopen-hip_3.5.1.70101-38~22.04_amd64.deb" + ], + "version": "3.5.1.70101-38~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "rocrand_4.1.0.70101-38_22.04_amd64", + "name": "rocrand", + "sha256": "0c8ed29a197a1444cf11bd52cc20dcf981ea0310e0db83f404eec14edd9d0520", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocrand/rocrand_4.1.0.70101-38~22.04_amd64.deb" + ], + "version": "4.1.0.70101-38~22.04" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "name": "rocm-core", + "version": "7.1.1.70101-38~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "roctracer_4.1.70101.70101-38_22.04_amd64", + "name": "roctracer", + "version": "4.1.70101.70101-38~22.04" + } + ], + "key": "hipblaslt_1.1.0.70101-38_22.04_amd64", + "name": "hipblaslt", + "sha256": "c2689d9a1ec1a236ac6545cc0b60ca99704e5ebfc6d536a0f2a28f061817fd7f", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/h/hipblaslt/hipblaslt_1.1.0.70101-38~22.04_amd64.deb" + ], + "version": "1.1.0.70101-38~22.04" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "name": "rocm-core", + "version": "7.1.1.70101-38~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + } + ], + "key": "roctracer_4.1.70101.70101-38_22.04_amd64", + "name": "roctracer", + "sha256": "621441c073c1a85ce22a9d9df9b6b2f40cbd0f4ecb1c6647eef1db57df02b6b2", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/roctracer/roctracer_4.1.70101.70101-38~22.04_amd64.deb" + ], + "version": "4.1.70101.70101-38~22.04" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "name": "rocm-core", + "version": "7.1.1.70101-38~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "hip-runtime-amd_7.1.52802.70101-38_22.04_amd64", + "name": "hip-runtime-amd", + "version": "7.1.52802.70101-38~22.04" + }, + { + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" + }, + { + "key": "rocprofiler-register_0.6.0.70101-38_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70101-38~22.04" + }, + { + "key": "comgr_3.0.0.70101-38_22.04_amd64", + "name": "comgr", + "version": "3.0.0.70101-38~22.04" + }, + { + "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", + "name": "libtinfo-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", + "name": "libncurses-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", + "name": "ncurses-bin", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libncurses6_6.3-2ubuntu0.1_amd64", + "name": "libncurses6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "rocminfo_1.0.0.70101-38_22.04_amd64", + "name": "rocminfo", + "version": "1.0.0.70101-38~22.04" + }, + { + "key": "pciutils_1-3.7.0-6_amd64", + "name": "pciutils", + "version": "1:3.7.0-6" + }, + { + "key": "libpci3_1-3.7.0-6_amd64", + "name": "libpci3", + "version": "1:3.7.0-6" + }, + { + "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", + "name": "pci.ids", + "version": "0.0~2022.01.22-1ubuntu0.1" + }, + { + "key": "libudev1_249.11-0ubuntu3.16_amd64", + "name": "libudev1", + "version": "249.11-0ubuntu3.16" + }, + { + "key": "libkmod2_29-1ubuntu1_amd64", + "name": "libkmod2", + "version": "29-1ubuntu1" + }, + { + "key": "hsa-rocr_1.18.0.70101-38_22.04_amd64", + "name": "hsa-rocr", + "version": "1.18.0.70101-38~22.04" + }, + { + "key": "libelf1_0.186-1ubuntu0.1_amd64", + "name": "libelf1", + "version": "0.186-1ubuntu0.1" + }, + { + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70101-2255337.22.04_amd64", + "name": "libdrm-amdgpu-amdgpu1", + "version": "1:2.4.125.70101-2255337.22.04" + }, + { + "key": "libdrm-amdgpu-common_1.0.0.70101-2255337.22.04_amd64", + "name": "libdrm-amdgpu-common", + "version": "1.0.0.70101-2255337.22.04" + }, + { + "key": "amdgpu-core_1-7.1.70101-2255337.22.04_amd64", + "name": "amdgpu-core", + "version": "1:7.1.70101-2255337.22.04" + }, + { + "key": "libdrm2-amdgpu_1-2.4.125.70101-2255337.22.04_amd64", + "name": "libdrm2-amdgpu", + "version": "1:2.4.125.70101-2255337.22.04" + }, + { + "key": "hipblaslt_1.1.0.70101-38_22.04_amd64", + "name": "hipblaslt", + "version": "1.1.0.70101-38~22.04" + }, + { + "key": "roctracer_4.1.70101.70101-38_22.04_amd64", + "name": "roctracer", + "version": "4.1.70101.70101-38~22.04" + } + ], + "key": "rocblas_5.1.1.70101-38_22.04_amd64", + "name": "rocblas", + "sha256": "46c414775f070518afba3326611fd577059a5bb277182356a0bae720e13086a7", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocblas/rocblas_5.1.1.70101-38~22.04_amd64.deb" + ], + "version": "5.1.1.70101-38~22.04" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" + }, + { + "key": "rocprofiler-register_0.6.0.70101-38_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70101-38~22.04" + }, + { + "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "name": "rocm-core", + "version": "7.1.1.70101-38~22.04" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -2498,24 +3543,9 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "hip-runtime-amd_6.4.43483.60401-83_22.04_amd64", - "name": "hip-runtime-amd", - "version": "6.4.43483.60401-83~22.04" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "rocprofiler-register_0.4.0.60401-83_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.4.0.60401-83~22.04" - }, - { - "key": "comgr_3.0.0.60401-83_22.04_amd64", + "key": "comgr_3.0.0.70101-38_22.04_amd64", "name": "comgr", - "version": "3.0.0.60401-83~22.04" + "version": "3.0.0.70101-38~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -2573,9 +3603,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.60401-83_22.04_amd64", + "key": "rocminfo_1.0.0.70101-38_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.60401-83~22.04" + "version": "1.0.0.70101-38~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -2603,84 +3633,505 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.10-2166611.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.10-2166611.22.04" - }, - { - "key": "amdgpu-core_1-6.4.60401-2164967.22.04_amd64", - "name": "amdgpu-core", - "version": "1:6.4.60401-2164967.22.04" + "key": "hsa-rocr_1.18.0.70101-38_22.04_amd64", + "name": "hsa-rocr", + "version": "1.18.0.70101-38~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", "name": "libelf1", "version": "0.186-1ubuntu0.1" + }, + { + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70101-2255337.22.04_amd64", + "name": "libdrm-amdgpu-amdgpu1", + "version": "1:2.4.125.70101-2255337.22.04" + }, + { + "key": "libdrm-amdgpu-common_1.0.0.70101-2255337.22.04_amd64", + "name": "libdrm-amdgpu-common", + "version": "1.0.0.70101-2255337.22.04" + }, + { + "key": "amdgpu-core_1-7.1.70101-2255337.22.04_amd64", + "name": "amdgpu-core", + "version": "1:7.1.70101-2255337.22.04" + }, + { + "key": "libdrm2-amdgpu_1-2.4.125.70101-2255337.22.04_amd64", + "name": "libdrm2-amdgpu", + "version": "1:2.4.125.70101-2255337.22.04" } ], - "key": "rccl_2.22.3.60401-83_22.04_amd64", - "name": "rccl", - "sha256": "6b5675f7bb81b03387a64478a896a671d10e83a5629e54a0c3a8bf503a7ac106", + "key": "hip-runtime-amd_7.1.52802.70101-38_22.04_amd64", + "name": "hip-runtime-amd", + "sha256": "5f4e042d2cc08c1acc40a710e619dc6fc4e98ff1285329d0a2981b4bbcd6436c", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/r/rccl/rccl_2.22.3.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/h/hip-runtime-amd/hip-runtime-amd_7.1.52802.70101-38~22.04_amd64.deb" ], - "version": "2.22.3.60401-83~22.04" + "version": "7.1.52802.70101-38~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocm-smi-lib_7.5.0.60401-83_22.04_amd64", - "name": "rocm-smi-lib", - "sha256": "3db3358e0ae302b4589e586d6b9dae2ae183d521e61979723d9eb918c7bfbde1", + "key": "rocminfo_1.0.0.70101-38_22.04_amd64", + "name": "rocminfo", + "sha256": "996f635e8bb55e6ea37e4910c44dffd461ffa6ee5189c4bdb57906bc57dcbd0e", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/r/rocm-smi-lib/rocm-smi-lib_7.5.0.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocminfo/rocminfo_1.0.0.70101-38~22.04_amd64.deb" ], - "version": "7.5.0.60401-83~22.04" + "version": "1.0.0.70101-38~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "pciutils_1-3.7.0-6_amd64", + "name": "pciutils", + "sha256": "830f2957662723860d3ebac949b22867df00d9a4c5ab1c2bf98e10674dc1c756", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pciutils/pciutils_3.7.0-6_amd64.deb" + ], + "version": "1:3.7.0-6" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libpci3_1-3.7.0-6_amd64", + "name": "libpci3", + "sha256": "624bea7b5a9073a5bc2dfa5d956571d6127c20400fcfcfca1e47b7448ba5d351", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pciutils/libpci3_3.7.0-6_amd64.deb" + ], + "version": "1:3.7.0-6" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", + "name": "pci.ids", + "sha256": "026609ed71363cd6ed13ace2dac0c9813d94be3874c778539a8a64d6d2cc5566", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pci.ids/pci.ids_0.0~2022.01.22-1ubuntu0.1_all.deb" + ], + "version": "0.0~2022.01.22-1ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libudev1_249.11-0ubuntu3.16_amd64", + "name": "libudev1", + "sha256": "451d4eaa84dbeabc045611937a78489710ad2c2baf277a63aacda7ee5226e3ea", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/s/systemd/libudev1_249.11-0ubuntu3.16_amd64.deb" + ], + "version": "249.11-0ubuntu3.16" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libkmod2_29-1ubuntu1_amd64", + "name": "libkmod2", + "sha256": "f355cebb66405a84cc88bb313440abd39f97cf45d3f468ea9ec7ba34e11b6c8a", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/k/kmod/libkmod2_29-1ubuntu1_amd64.deb" + ], + "version": "29-1ubuntu1" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "rocm-core_7.1.1.70101-38_22.04_amd64", "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" + "version": "7.1.1.70101-38~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "rocm-smi-lib_7.8.0.70101-38_22.04_amd64", + "name": "rocm-smi-lib", + "version": "7.8.0.70101-38~22.04" + }, + { + "key": "hip-runtime-amd_7.1.52802.70101-38_22.04_amd64", + "name": "hip-runtime-amd", + "version": "7.1.52802.70101-38~22.04" + }, + { + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" + }, + { + "key": "rocprofiler-register_0.6.0.70101-38_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70101-38~22.04" + }, + { + "key": "comgr_3.0.0.70101-38_22.04_amd64", + "name": "comgr", + "version": "3.0.0.70101-38~22.04" + }, + { + "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", + "name": "libtinfo-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", + "name": "libncurses-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", + "name": "ncurses-bin", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libncurses6_6.3-2ubuntu0.1_amd64", + "name": "libncurses6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "rocminfo_1.0.0.70101-38_22.04_amd64", + "name": "rocminfo", + "version": "1.0.0.70101-38~22.04" + }, + { + "key": "pciutils_1-3.7.0-6_amd64", + "name": "pciutils", + "version": "1:3.7.0-6" + }, + { + "key": "libpci3_1-3.7.0-6_amd64", + "name": "libpci3", + "version": "1:3.7.0-6" + }, + { + "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", + "name": "pci.ids", + "version": "0.0~2022.01.22-1ubuntu0.1" + }, + { + "key": "libudev1_249.11-0ubuntu3.16_amd64", + "name": "libudev1", + "version": "249.11-0ubuntu3.16" + }, + { + "key": "libkmod2_29-1ubuntu1_amd64", + "name": "libkmod2", + "version": "29-1ubuntu1" + }, + { + "key": "hsa-rocr_1.18.0.70101-38_22.04_amd64", + "name": "hsa-rocr", + "version": "1.18.0.70101-38~22.04" + }, + { + "key": "libelf1_0.186-1ubuntu0.1_amd64", + "name": "libelf1", + "version": "0.186-1ubuntu0.1" + }, + { + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70101-2255337.22.04_amd64", + "name": "libdrm-amdgpu-amdgpu1", + "version": "1:2.4.125.70101-2255337.22.04" + }, + { + "key": "libdrm-amdgpu-common_1.0.0.70101-2255337.22.04_amd64", + "name": "libdrm-amdgpu-common", + "version": "1.0.0.70101-2255337.22.04" + }, + { + "key": "amdgpu-core_1-7.1.70101-2255337.22.04_amd64", + "name": "amdgpu-core", + "version": "1:7.1.70101-2255337.22.04" + }, + { + "key": "libdrm2-amdgpu_1-2.4.125.70101-2255337.22.04_amd64", + "name": "libdrm2-amdgpu", + "version": "1:2.4.125.70101-2255337.22.04" } ], - "key": "rocm-device-libs_1.0.0.60401-83_22.04_amd64", - "name": "rocm-device-libs", - "sha256": "080c1cbcade5a22cea800256eccc993116ae05b60f11785fda76beffe68c38bf", + "key": "rccl_2.27.7.70101-38_22.04_amd64", + "name": "rccl", + "sha256": "4fa485a696d6450e528c81c1f9dc0b8afba6f464ab43f6560b8594906c2de5f6", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/r/rocm-device-libs/rocm-device-libs_1.0.0.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rccl/rccl_2.27.7.70101-38~22.04_amd64.deb" ], - "version": "1.0.0.60401-83~22.04" + "version": "2.27.7.70101-38~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "rocm-smi-lib_7.8.0.70101-38_22.04_amd64", + "name": "rocm-smi-lib", + "sha256": "0daa96b94592fc1c6886115b7ec0d530ad53bc2fe5b5bb4ece05915ef8db479c", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocm-smi-lib/rocm-smi-lib_7.8.0.70101-38~22.04_amd64.deb" + ], + "version": "7.8.0.70101-38~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "hipcc_1.1.1.60401-83_22.04_amd64", - "name": "hipcc", - "version": "1.1.1.60401-83~22.04" - }, - { - "key": "rocm-llvm_19.0.0.25184.60401-83_22.04_amd64", - "name": "rocm-llvm", - "version": "19.0.0.25184.60401-83~22.04" - }, - { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", + "key": "rocm-core_7.1.1.70101-38_22.04_amd64", "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" - }, - { - "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libgcc-11-dev", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", - "name": "libquadmath0", - "version": "12.3.0-1ubuntu1~22.04" + "version": "7.1.1.70101-38~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -2703,15 +4154,467 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libubsan1", - "version": "12.3.0-1ubuntu1~22.04" + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" }, { "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", "name": "libstdc++6", "version": "12.3.0-1ubuntu1~22.04" }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + } + ], + "key": "rocm-device-libs_1.0.0.70101-38_22.04_amd64", + "name": "rocm-device-libs", + "sha256": "e042075ac40c261efe7bcff48ad784d8ea5f3b701dfb1692320cd93fa1805df0", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocm-device-libs/rocm-device-libs_1.0.0.70101-38~22.04_amd64.deb" + ], + "version": "1.0.0.70101-38~22.04" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "hipcc_1.1.1.70101-38_22.04_amd64", + "name": "hipcc", + "version": "1.1.1.70101-38~22.04" + }, + { + "key": "rocm-llvm_20.0.0.25444.70101-38_22.04_amd64", + "name": "rocm-llvm", + "version": "20.0.0.25444.70101-38~22.04" + }, + { + "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "name": "rocm-core", + "version": "7.1.1.70101-38~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, { "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", "name": "libtsan0", @@ -2767,111 +4670,6 @@ "name": "libtirpc-dev", "version": "1.3.2-2ubuntu0.1" }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, { "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", "name": "rpcsvc-proto", @@ -2893,109 +4691,19 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "hsa-rocr-dev_1.15.0.60401-83_22.04_amd64", + "key": "hsa-rocr-dev_1.18.0.70101-38_22.04_amd64", "name": "hsa-rocr-dev", - "version": "1.15.0.60401-83~22.04" + "version": "1.18.0.70101-38~22.04" }, { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.10-2166611.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.10-2166611.22.04" + "key": "hsa-rocr_1.18.0.70101-38_22.04_amd64", + "name": "hsa-rocr", + "version": "1.18.0.70101-38~22.04" }, { - "key": "amdgpu-core_1-6.4.60401-2164967.22.04_amd64", - "name": "amdgpu-core", - "version": "1:6.4.60401-2164967.22.04" + "key": "rocprofiler-register_0.6.0.70101-38_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70101-38~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", @@ -3003,9 +4711,34 @@ "version": "0.186-1ubuntu0.1" }, { - "key": "libdrm-amdgpu-dev_1-2.4.124.60401-2164967.22.04_amd64", + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" + }, + { + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70101-2255337.22.04_amd64", + "name": "libdrm-amdgpu-amdgpu1", + "version": "1:2.4.125.70101-2255337.22.04" + }, + { + "key": "libdrm-amdgpu-common_1.0.0.70101-2255337.22.04_amd64", + "name": "libdrm-amdgpu-common", + "version": "1.0.0.70101-2255337.22.04" + }, + { + "key": "amdgpu-core_1-7.1.70101-2255337.22.04_amd64", + "name": "amdgpu-core", + "version": "1:7.1.70101-2255337.22.04" + }, + { + "key": "libdrm2-amdgpu_1-2.4.125.70101-2255337.22.04_amd64", + "name": "libdrm2-amdgpu", + "version": "1:2.4.125.70101-2255337.22.04" + }, + { + "key": "libdrm-amdgpu-dev_1-2.4.125.70101-2255337.22.04_amd64", "name": "libdrm-amdgpu-dev", - "version": "1:2.4.124.60401-2164967.22.04" + "version": "1:2.4.125.70101-2255337.22.04" }, { "key": "valgrind_1-3.18.1-1ubuntu2_amd64", @@ -3023,44 +4756,19 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "libdrm-amdgpu-radeon1_1-2.4.124.60401-2164967.22.04_amd64", + "key": "libdrm-amdgpu-radeon1_1-2.4.125.70101-2255337.22.04_amd64", "name": "libdrm-amdgpu-radeon1", - "version": "1:2.4.124.60401-2164967.22.04" + "version": "1:2.4.125.70101-2255337.22.04" }, { - "key": "libdrm2-amdgpu_1-2.4.124.60401-2164967.22.04_amd64", - "name": "libdrm2-amdgpu", - "version": "1:2.4.124.60401-2164967.22.04" - }, - { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.124.60401-2164967.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.124.60401-2164967.22.04" - }, - { - "key": "libdrm-amdgpu-common_1.0.0.60401-2164967.22.04_amd64", - "name": "libdrm-amdgpu-common", - "version": "1.0.0.60401-2164967.22.04" - }, - { - "key": "hip-runtime-amd_6.4.43483.60401-83_22.04_amd64", + "key": "hip-runtime-amd_7.1.52802.70101-38_22.04_amd64", "name": "hip-runtime-amd", - "version": "6.4.43483.60401-83~22.04" + "version": "7.1.52802.70101-38~22.04" }, { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "rocprofiler-register_0.4.0.60401-83_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.4.0.60401-83~22.04" - }, - { - "key": "comgr_3.0.0.60401-83_22.04_amd64", + "key": "comgr_3.0.0.70101-38_22.04_amd64", "name": "comgr", - "version": "3.0.0.60401-83~22.04" + "version": "3.0.0.70101-38~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -3083,9 +4791,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.60401-83_22.04_amd64", + "key": "rocminfo_1.0.0.70101-38_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.60401-83~22.04" + "version": "1.0.0.70101-38~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -3128,35 +4836,35 @@ "version": "1:5.41-3ubuntu0.1" } ], - "key": "hip-dev_6.4.43483.60401-83_22.04_amd64", + "key": "hip-dev_7.1.52802.70101-38_22.04_amd64", "name": "hip-dev", - "sha256": "e10323a872b833c7846f7a6d204efa8126b2b6b1727f6da317b687ea4da8fb0e", + "sha256": "9f33ea2f9c62bad88cd2942f15a82eb59082ce6604feede4b662fc12f13042cf", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/h/hip-dev/hip-dev_6.4.43483.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/h/hip-dev/hip-dev_7.1.52802.70101-38~22.04_amd64.deb" ], - "version": "6.4.43483.60401-83~22.04" + "version": "7.1.52802.70101-38~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "hipcc_1.1.1.60401-83_22.04_amd64", + "key": "hipcc_1.1.1.70101-38_22.04_amd64", "name": "hipcc", - "sha256": "71aabfdefdd126fe6145b0e21be6964371d550b4be1b955095db6cbb53f9e146", + "sha256": "49355c960eef059dcbcc12134ac6b621505f429b6a9d20ac0902dd9546963593", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/h/hipcc/hipcc_1.1.1.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/h/hipcc/hipcc_1.1.1.70101-38~22.04_amd64.deb" ], - "version": "1.1.1.60401-83~22.04" + "version": "1.1.1.70101-38~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocm-llvm_19.0.0.25184.60401-83_22.04_amd64", + "key": "rocm-llvm_20.0.0.25444.70101-38_22.04_amd64", "name": "rocm-llvm", - "sha256": "1dec6406f10c8f2b8fa2baefd1b7a8af844908d7da1493fca407ea696f5b0877", + "sha256": "23a2682b3a1e718c04ca04b39ed10f52fc8a3b524b12e1de2245ad7120988e6a", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/r/rocm-llvm/rocm-llvm_19.0.0.25184.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocm-llvm/rocm-llvm_20.0.0.25444.70101-38~22.04_amd64.deb" ], - "version": "19.0.0.25184.60401-83~22.04" + "version": "20.0.0.25444.70101-38~22.04" }, { "arch": "amd64", @@ -3282,24 +4990,24 @@ { "arch": "amd64", "dependencies": [], - "key": "hsa-rocr-dev_1.15.0.60401-83_22.04_amd64", + "key": "hsa-rocr-dev_1.18.0.70101-38_22.04_amd64", "name": "hsa-rocr-dev", - "sha256": "e1221293f8a5501ef01324c381e61d76a5d2e0f11032e4d33766dc53efc75813", + "sha256": "92135d3d9775fc07a446641272e39f9022830afb9cb35042b7ed3cfd4bd504e4", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/h/hsa-rocr-dev/hsa-rocr-dev_1.15.0.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/h/hsa-rocr-dev/hsa-rocr-dev_1.18.0.70101-38~22.04_amd64.deb" ], - "version": "1.15.0.60401-83~22.04" + "version": "1.18.0.70101-38~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "libdrm-amdgpu-dev_1-2.4.124.60401-2164967.22.04_amd64", + "key": "libdrm-amdgpu-dev_1-2.4.125.70101-2255337.22.04_amd64", "name": "libdrm-amdgpu-dev", - "sha256": "71c05b464d721e24c8dc2c9b6ca43f8cc9f92714f98c7c8d32022aef6ffb9df7", + "sha256": "af20b1033129e0797efc2fcc071cf213f2739483674b1a1a312d31aad3166aef", "urls": [ - "https://repo.radeon.com/amdgpu/6.4.1/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-dev_2.4.124.60401-2164967.22.04_amd64.deb" + "https://repo.radeon.com/graphics/7.1.1/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-dev_2.4.125.70101-2255337.22.04_amd64.deb" ], - "version": "1:2.4.124.60401-2164967.22.04" + "version": "1:2.4.125.70101-2255337.22.04" }, { "arch": "amd64", @@ -3337,13 +5045,13 @@ { "arch": "amd64", "dependencies": [], - "key": "libdrm-amdgpu-radeon1_1-2.4.124.60401-2164967.22.04_amd64", + "key": "libdrm-amdgpu-radeon1_1-2.4.125.70101-2255337.22.04_amd64", "name": "libdrm-amdgpu-radeon1", - "sha256": "1b5a5d927f604020efc5d3ccd857036cd43857cab05e8d357e63d2fd3fc59644", + "sha256": "d864ae2ec21b164d20b48807f9343ee6e57a4b937e7c218b96099479af53e643", "urls": [ - "https://repo.radeon.com/amdgpu/6.4.1/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-radeon1_2.4.124.60401-2164967.22.04_amd64.deb" + "https://repo.radeon.com/graphics/7.1.1/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-radeon1_2.4.125.70101-2255337.22.04_amd64.deb" ], - "version": "1:2.4.124.60401-2164967.22.04" + "version": "1:2.4.125.70101-2255337.22.04" }, { "arch": "amd64", @@ -3382,19 +5090,9 @@ "arch": "amd64", "dependencies": [ { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", + "key": "rocm-core_7.1.1.70101-38_22.04_amd64", "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" - }, - { - "key": "rocblas_4.4.0.60401-83_22.04_amd64", - "name": "rocblas", - "version": "4.4.0.60401-83~22.04" - }, - { - "key": "hip-runtime-amd_6.4.43483.60401-83_22.04_amd64", - "name": "hip-runtime-amd", - "version": "6.4.43483.60401-83~22.04" + "version": "7.1.1.70101-38~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -3417,69 +5115,24 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" }, { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" }, { - "key": "rocprofiler-register_0.4.0.60401-83_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.4.0.60401-83~22.04" + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" }, { - "key": "comgr_3.0.0.60401-83_22.04_amd64", - "name": "comgr", - "version": "3.0.0.60401-83~22.04" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" }, { "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", @@ -3546,6 +5199,16 @@ "name": "libacl1", "version": "2.3.1-1" }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, { "key": "liblzma5_5.2.5-2ubuntu1_amd64", "name": "liblzma5", @@ -3572,65 +5235,10 @@ "version": "1.46.5-2ubuntu1.2" }, { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", "version": "6.3-2ubuntu0.1" }, - { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "rocminfo_1.0.0.60401-83_22.04_amd64", - "name": "rocminfo", - "version": "1.0.0.60401-83~22.04" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, { "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", "name": "libsqlite3-0", @@ -3646,11 +5254,26 @@ "name": "readline-common", "version": "8.1.2-1" }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, { "key": "libmpdec3_2.5.1-2build2_amd64", "name": "libmpdec3", "version": "2.5.1-2build2" }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, { "key": "libffi8_3.4.2-4_amd64", "name": "libffi8", @@ -3691,6 +5314,91 @@ "name": "python3-minimal", "version": "3.10.6-1~22.04.1" }, + { + "key": "rocblas_5.1.1.70101-38_22.04_amd64", + "name": "rocblas", + "version": "5.1.1.70101-38~22.04" + }, + { + "key": "hip-runtime-amd_7.1.52802.70101-38_22.04_amd64", + "name": "hip-runtime-amd", + "version": "7.1.52802.70101-38~22.04" + }, + { + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" + }, + { + "key": "rocprofiler-register_0.6.0.70101-38_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70101-38~22.04" + }, + { + "key": "comgr_3.0.0.70101-38_22.04_amd64", + "name": "comgr", + "version": "3.0.0.70101-38~22.04" + }, + { + "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", + "name": "libtinfo-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", + "name": "libncurses-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", + "name": "ncurses-bin", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libncurses6_6.3-2ubuntu0.1_amd64", + "name": "libncurses6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "rocminfo_1.0.0.70101-38_22.04_amd64", + "name": "rocminfo", + "version": "1.0.0.70101-38~22.04" + }, { "key": "pciutils_1-3.7.0-6_amd64", "name": "pciutils", @@ -3717,14 +5425,9 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.10-2166611.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.10-2166611.22.04" - }, - { - "key": "amdgpu-core_1-6.4.60401-2164967.22.04_amd64", - "name": "amdgpu-core", - "version": "1:6.4.60401-2164967.22.04" + "key": "hsa-rocr_1.18.0.70101-38_22.04_amd64", + "name": "hsa-rocr", + "version": "1.18.0.70101-38~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", @@ -3732,46 +5435,51 @@ "version": "0.186-1ubuntu0.1" }, { - "key": "hipblaslt_0.12.1.60401-83_22.04_amd64", - "name": "hipblaslt", - "version": "0.12.1.60401-83~22.04" + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70101-2255337.22.04_amd64", + "name": "libdrm-amdgpu-amdgpu1", + "version": "1:2.4.125.70101-2255337.22.04" }, { - "key": "roctracer_4.1.60401.60401-83_22.04_amd64", + "key": "libdrm-amdgpu-common_1.0.0.70101-2255337.22.04_amd64", + "name": "libdrm-amdgpu-common", + "version": "1.0.0.70101-2255337.22.04" + }, + { + "key": "amdgpu-core_1-7.1.70101-2255337.22.04_amd64", + "name": "amdgpu-core", + "version": "1:7.1.70101-2255337.22.04" + }, + { + "key": "libdrm2-amdgpu_1-2.4.125.70101-2255337.22.04_amd64", + "name": "libdrm2-amdgpu", + "version": "1:2.4.125.70101-2255337.22.04" + }, + { + "key": "hipblaslt_1.1.0.70101-38_22.04_amd64", + "name": "hipblaslt", + "version": "1.1.0.70101-38~22.04" + }, + { + "key": "roctracer_4.1.70101.70101-38_22.04_amd64", "name": "roctracer", - "version": "4.1.60401.60401-83~22.04" + "version": "4.1.70101.70101-38~22.04" } ], - "key": "rocsolver_3.28.0.60401-83_22.04_amd64", + "key": "rocsolver_3.31.0.70101-38_22.04_amd64", "name": "rocsolver", - "sha256": "82451d985c3f9ea9472df45e044651a16cc7a8d510a338e57e0d7c19b04a7243", + "sha256": "0fdf9d402336e592f80d41358eb324b21b08ae9b8b6e1a312bcd2e0da0fdf8da", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/r/rocsolver/rocsolver_3.28.0.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocsolver/rocsolver_3.31.0.70101-38~22.04_amd64.deb" ], - "version": "3.28.0.60401-83~22.04" + "version": "3.31.0.70101-38~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", + "key": "rocm-core_7.1.1.70101-38_22.04_amd64", "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" - }, - { - "key": "rocsolver_3.28.0.60401-83_22.04_amd64", - "name": "rocsolver", - "version": "3.28.0.60401-83~22.04" - }, - { - "key": "rocblas_4.4.0.60401-83_22.04_amd64", - "name": "rocblas", - "version": "4.4.0.60401-83~22.04" - }, - { - "key": "hip-runtime-amd_6.4.43483.60401-83_22.04_amd64", - "name": "hip-runtime-amd", - "version": "6.4.43483.60401-83~22.04" + "version": "7.1.1.70101-38~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -3794,69 +5502,24 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" }, { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" }, { - "key": "rocprofiler-register_0.4.0.60401-83_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.4.0.60401-83~22.04" + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" }, { - "key": "comgr_3.0.0.60401-83_22.04_amd64", - "name": "comgr", - "version": "3.0.0.60401-83~22.04" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" }, { "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", @@ -3923,6 +5586,16 @@ "name": "libacl1", "version": "2.3.1-1" }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, { "key": "liblzma5_5.2.5-2ubuntu1_amd64", "name": "liblzma5", @@ -3949,65 +5622,10 @@ "version": "1.46.5-2ubuntu1.2" }, { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", "version": "6.3-2ubuntu0.1" }, - { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "rocminfo_1.0.0.60401-83_22.04_amd64", - "name": "rocminfo", - "version": "1.0.0.60401-83~22.04" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, { "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", "name": "libsqlite3-0", @@ -4023,11 +5641,26 @@ "name": "readline-common", "version": "8.1.2-1" }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, { "key": "libmpdec3_2.5.1-2build2_amd64", "name": "libmpdec3", "version": "2.5.1-2build2" }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, { "key": "libffi8_3.4.2-4_amd64", "name": "libffi8", @@ -4068,6 +5701,96 @@ "name": "python3-minimal", "version": "3.10.6-1~22.04.1" }, + { + "key": "rocsolver_3.31.0.70101-38_22.04_amd64", + "name": "rocsolver", + "version": "3.31.0.70101-38~22.04" + }, + { + "key": "rocblas_5.1.1.70101-38_22.04_amd64", + "name": "rocblas", + "version": "5.1.1.70101-38~22.04" + }, + { + "key": "hip-runtime-amd_7.1.52802.70101-38_22.04_amd64", + "name": "hip-runtime-amd", + "version": "7.1.52802.70101-38~22.04" + }, + { + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" + }, + { + "key": "rocprofiler-register_0.6.0.70101-38_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70101-38~22.04" + }, + { + "key": "comgr_3.0.0.70101-38_22.04_amd64", + "name": "comgr", + "version": "3.0.0.70101-38~22.04" + }, + { + "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", + "name": "libtinfo-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", + "name": "libncurses-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", + "name": "ncurses-bin", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libncurses6_6.3-2ubuntu0.1_amd64", + "name": "libncurses6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "rocminfo_1.0.0.70101-38_22.04_amd64", + "name": "rocminfo", + "version": "1.0.0.70101-38~22.04" + }, { "key": "pciutils_1-3.7.0-6_amd64", "name": "pciutils", @@ -4094,14 +5817,9 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.10-2166611.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.10-2166611.22.04" - }, - { - "key": "amdgpu-core_1-6.4.60401-2164967.22.04_amd64", - "name": "amdgpu-core", - "version": "1:6.4.60401-2164967.22.04" + "key": "hsa-rocr_1.18.0.70101-38_22.04_amd64", + "name": "hsa-rocr", + "version": "1.18.0.70101-38~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", @@ -4109,99 +5827,559 @@ "version": "0.186-1ubuntu0.1" }, { - "key": "hipblaslt_0.12.1.60401-83_22.04_amd64", - "name": "hipblaslt", - "version": "0.12.1.60401-83~22.04" + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70101-2255337.22.04_amd64", + "name": "libdrm-amdgpu-amdgpu1", + "version": "1:2.4.125.70101-2255337.22.04" }, { - "key": "roctracer_4.1.60401.60401-83_22.04_amd64", + "key": "libdrm-amdgpu-common_1.0.0.70101-2255337.22.04_amd64", + "name": "libdrm-amdgpu-common", + "version": "1.0.0.70101-2255337.22.04" + }, + { + "key": "amdgpu-core_1-7.1.70101-2255337.22.04_amd64", + "name": "amdgpu-core", + "version": "1:7.1.70101-2255337.22.04" + }, + { + "key": "libdrm2-amdgpu_1-2.4.125.70101-2255337.22.04_amd64", + "name": "libdrm2-amdgpu", + "version": "1:2.4.125.70101-2255337.22.04" + }, + { + "key": "hipblaslt_1.1.0.70101-38_22.04_amd64", + "name": "hipblaslt", + "version": "1.1.0.70101-38~22.04" + }, + { + "key": "roctracer_4.1.70101.70101-38_22.04_amd64", "name": "roctracer", - "version": "4.1.60401.60401-83~22.04" + "version": "4.1.70101.70101-38~22.04" } ], - "key": "hipsolver_2.4.0.60401-83_22.04_amd64", + "key": "hipsolver_3.1.0.70101-38_22.04_amd64", "name": "hipsolver", - "sha256": "d461a5a321aad9743dd91906b393615cc2066e7ca792092d4d7619aa14903612", + "sha256": "1ebd0afa7bb9b52f341755603fc432ab1ede8218c014b1df1bff03d7a6d11c0a", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/h/hipsolver/hipsolver_2.4.0.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/h/hipsolver/hipsolver_3.1.0.70101-38~22.04_amd64.deb" ], - "version": "2.4.0.60401-83~22.04" + "version": "3.1.0.70101-38~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", + "key": "rocm-core_7.1.1.70101-38_22.04_amd64", "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" + "version": "7.1.1.70101-38~22.04" }, { - "key": "rocfft_1.0.32.60401-83_22.04_amd64", + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "rocfft_1.0.35.70101-38_22.04_amd64", "name": "rocfft", - "version": "1.0.32.60401-83~22.04" + "version": "1.0.35.70101-38~22.04" } ], - "key": "hipfft_1.0.18.60401-83_22.04_amd64", + "key": "hipfft_1.0.21.70101-38_22.04_amd64", "name": "hipfft", - "sha256": "64f2bc139adf53f00affdda5b189d60d491b93a7260c4cbcffc7b26af6569348", + "sha256": "14adaf6995d9b26295d5ae20d6eb3d76357521294349796f88bb49becbb7717a", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/h/hipfft/hipfft_1.0.18.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/h/hipfft/hipfft_1.0.21.70101-38~22.04_amd64.deb" ], - "version": "1.0.18.60401-83~22.04" + "version": "1.0.21.70101-38~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocfft_1.0.32.60401-83_22.04_amd64", + "key": "rocfft_1.0.35.70101-38_22.04_amd64", "name": "rocfft", - "sha256": "0c2ec9e4b492adb6d782db21d6438751d93be4c0ac64a9b8d0dd294a32ca7586", + "sha256": "c9cb19ce041ab9694355cf6fc3e58514c6ccfdbd1af409829e03aaa7093555cb", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/r/rocfft/rocfft_1.0.32.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocfft/rocfft_1.0.35.70101-38~22.04_amd64.deb" ], - "version": "1.0.32.60401-83~22.04" + "version": "1.0.35.70101-38~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "hipblaslt_0.12.1.60401-83_22.04_amd64", + "key": "hipblaslt_1.1.0.70101-38_22.04_amd64", "name": "hipblaslt", - "version": "0.12.1.60401-83~22.04" + "version": "1.1.0.70101-38~22.04" }, { - "key": "rocm-core_6.4.1.60401-83_22.04_amd64", + "key": "rocm-core_7.1.1.70101-38_22.04_amd64", "name": "rocm-core", - "version": "6.4.1.60401-83~22.04" + "version": "7.1.1.70101-38~22.04" }, { - "key": "roctracer_4.1.60401.60401-83_22.04_amd64", + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "roctracer_4.1.70101.70101-38_22.04_amd64", "name": "roctracer", - "version": "4.1.60401.60401-83~22.04" + "version": "4.1.70101.70101-38~22.04" }, { - "key": "hipblas-common-dev_1.0.0.60401-83_22.04_amd64", + "key": "hipblas-common-dev_1.3.0.70101-38_22.04_amd64", "name": "hipblas-common-dev", - "version": "1.0.0.60401-83~22.04" + "version": "1.3.0.70101-38~22.04" } ], - "key": "hipblaslt-dev_0.12.1.60401-83_22.04_amd64", + "key": "hipblaslt-dev_1.1.0.70101-38_22.04_amd64", "name": "hipblaslt-dev", - "sha256": "46eb2285c76d246b162eb54cc7f9e5cb7bcdd0aa83d57ecaea440e57260f2f4a", + "sha256": "a54d9203412a4b2ecfe446c23358d87d2446bab4c49b874706ac8ec420321a47", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/h/hipblaslt-dev/hipblaslt-dev_0.12.1.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/h/hipblaslt-dev/hipblaslt-dev_1.1.0.70101-38~22.04_amd64.deb" ], - "version": "0.12.1.60401-83~22.04" + "version": "1.1.0.70101-38~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "hipblas-common-dev_1.0.0.60401-83_22.04_amd64", + "key": "hipblas-common-dev_1.3.0.70101-38_22.04_amd64", "name": "hipblas-common-dev", - "sha256": "5df3e4a8a1959cbf94106f7bf87d7fb71bf06e726cde00c6092ef29bbd8156f0", + "sha256": "e6a3e7dd89519d3a0ce0875a722d74e4ec125264c99195231f27b9a59a3c2bed", "urls": [ - "https://repo.radeon.com/rocm/apt/6.4.1/pool/main/h/hipblas-common-dev/hipblas-common-dev_1.0.0.60401-83~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/h/hipblas-common-dev/hipblas-common-dev_1.3.0.70101-38~22.04_amd64.deb" ], - "version": "1.0.0.60401-83~22.04" + "version": "1.3.0.70101-38~22.04" } ], "version": 1 diff --git a/runtimes/rocm/packages.yaml b/runtimes/rocm/packages.yaml index fcb6583d..19b45fca 100644 --- a/runtimes/rocm/packages.yaml +++ b/runtimes/rocm/packages.yaml @@ -5,9 +5,9 @@ version: 1 sources: - channel: jammy main - url: https://repo.radeon.com/amdgpu/6.4.1/ubuntu + url: https://repo.radeon.com/rocm/apt/7.1.1 - channel: jammy main - url: https://repo.radeon.com/rocm/apt/6.4.1 + url: https://repo.radeon.com/graphics/7.1.1/ubuntu - channel: jammy main url: https://snapshot.ubuntu.com/ubuntu/20250711T030400Z - channel: jammy-security main @@ -27,6 +27,7 @@ packages: - "hsa-amd-aqlprofile" - "comgr" - "rocprofiler-register" + - "rocprofiler-sdk" - "miopen-hip" - "rccl" - "rocm-device-libs" diff --git a/runtimes/rocm/rocm.bzl b/runtimes/rocm/rocm.bzl index f06daacb..96ff4690 100644 --- a/runtimes/rocm/rocm.bzl +++ b/runtimes/rocm/rocm.bzl @@ -6,7 +6,7 @@ _BUILD_FILE_DEFAULT_VISIBILITY = """\ package(default_visibility = ["//visibility:public"]) """ -_ROCM_STRIP_PREFIX = "opt/rocm-6.4.1" +_ROCM_STRIP_PREFIX = "opt/rocm-7.1.1" _UBUNTU_PACKAGES = { "libdrm2-amdgpu": packages.filegroup(name = "libdrm2-amdgpu", srcs = ["opt/amdgpu/lib/x86_64-linux-gnu/libdrm.so.2"]), @@ -39,7 +39,15 @@ _UBUNTU_PACKAGES = { _ROCM_PACKAGES = { "rocm-core": packages.filegroup(name = "rocm-core", srcs = ["lib/librocm-core.so.1"]), - "rocm-smi-lib": packages.filegroup(name = "rocm_smi", srcs = ["lib/librocm_smi64.so.7"]), + "rocm-smi-lib": packages.filegroup(name = "rocm_smi", srcs = ["lib/librocm_smi64.so.1"]), + "rocprofiler-sdk": "\n".join([ + packages.filegroup(name = "rocprofiler-sdk", srcs = [ + "lib/librocprofiler-sdk.so.1", + "lib/librocprofiler-sdk-attach.so.1", + ]), + ]), + "rocprofiler-sdk-rocpd": packages.filegroup(name = "rocprofiler-sdk-rocpd", srcs = ["lib/librocprofiler-sdk-rocpd.so.1"]), + "rocprofiler-sdk-roctx": packages.filegroup(name = "rocprofiler-sdk-roctx", srcs = ["lib/librocprofiler-sdk-roctx.so.1"]), "hsa-rocr": packages.filegroup(name = "hsa-runtime", srcs = ["lib/libhsa-runtime64.so.1"]), "hsa-amd-aqlprofile": packages.filegroup(name = "hsa-amd-aqlprofile", srcs = ["lib/libhsa-amd-aqlprofile64.so.1"]), "comgr": packages.filegroup(name = "amd_comgr", srcs = ["lib/libamd_comgr.so.3"]), @@ -56,7 +64,7 @@ _ROCM_PACKAGES = { packages.load_("@zml//runtimes/rocm:gfx.bzl", "bytecode_select"), packages.patchelf( name = "rocblas", - src = "lib/librocblas.so.4", + src = "lib/librocblas.so.5", add_needed = ["libzmlxrocm.so.0"], rename_dynamic_symbols = { "dlopen": "zmlxrocm_dlopen", @@ -87,7 +95,16 @@ _ROCM_PACKAGES = { packages.load_("@zml//runtimes/rocm:gfx.bzl", "bytecode_select"), packages.patchelf( name = "hipblaslt", - src = "lib/libhipblaslt.so.0", + src = "lib/libhipblaslt.so.1", + add_needed = ["libzmlxrocm.so.0"], + rename_dynamic_symbols = { + "dlopen": "zmlxrocm_dlopen", + }, + ), + packages.patchelf( + name = "rocroller", + src = "lib/librocroller.so.1", + set_rpath = "$ORIGIN", add_needed = ["libzmlxrocm.so.0"], rename_dynamic_symbols = { "dlopen": "zmlxrocm_dlopen", @@ -115,15 +132,15 @@ _ROCM_PACKAGES = { packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), packages.patchelf( name = "amdhip", - src = "lib/libamdhip64.so.6", + src = "lib/libamdhip64.so.7", add_needed = ["libzmlxrocm.so.0"], rename_dynamic_symbols = { "dlopen": "zmlxrocm_dlopen", }, ), - packages.filegroup(name = "hiprtc", srcs = ["lib/libhiprtc.so.6", "lib/libhiprtc-builtins.so.6"]), + packages.filegroup(name = "hiprtc", srcs = ["lib/libhiprtc.so.7", "lib/libhiprtc-builtins.so.7"]), ]), - "hipsolver": packages.filegroup(name = "hipsolver", srcs = ["lib/libhipsolver.so.0"]), + "hipsolver": packages.filegroup(name = "hipsolver", srcs = ["lib/libhipsolver.so.1"]), } def _rocm_impl(mctx): @@ -153,8 +170,8 @@ def _rocm_impl(mctx): http_archive( name = "libpjrt_rocm", build_file = "libpjrt_rocm.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v14.0.1/pjrt-rocm_linux-amd64.tar.gz", - sha256 = "087858044f17bc06b70d7cbffc33e7f2bf590d732f3ce2c24425e41453ea1cf4", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v15.0.2/pjrt-rocm_linux-amd64.tar.gz", + sha256 = "b3f0dc66eec0f72835ba7fac66a88eef967f5caa2ba5cb3bb5ef7ee040d736cd", ) return mctx.extension_metadata( diff --git a/runtimes/rocm/zmlxrocm.zig b/runtimes/rocm/zmlxrocm.zig index fccfbe9c..30791f3a 100644 --- a/runtimes/rocm/zmlxrocm.zig +++ b/runtimes/rocm/zmlxrocm.zig @@ -5,7 +5,7 @@ const stdx = @import("stdx"); pub export fn zmlxrocm_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque { const replacements: std.StaticStringMap([:0]const u8) = .initComptime(.{ .{ "librocm-core.so", "librocm-core.so.1" }, - .{ "librocm_smi64.so", "librocm_smi64.so.7" }, + .{ "librocm_smi64.so", "librocm_smi64.so.1" }, .{ "libhsa-runtime64.so", "libhsa-runtime64.so.1" }, .{ "libhsa-amd-aqlprofile64.so", "libhsa-amd-aqlprofile64.so.1" }, .{ "libamd_comgr.so", "libamd_comgr.so.3" }, @@ -13,13 +13,13 @@ pub export fn zmlxrocm_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque .{ "libMIOpen.so", "libMIOpen.so.1" }, .{ "libMIOpen.so.1", "libMIOpen.so.1" }, .{ "librccl.so", "librccl.so.1" }, - .{ "librocblas.so.4", "librocblas.so.4" }, - .{ "librocblas.so", "librocblas.so.4" }, + .{ "librocblas.so.5", "librocblas.so.5" }, + .{ "librocblas.so", "librocblas.so.5" }, .{ "libroctracer64.so", "libroctracer64.so.4" }, .{ "libroctx64.so", "libroctx64.so.4" }, - .{ "libhipblaslt.so", "libhipblaslt.so.0" }, - .{ "libamdhip64.so", "libamdhip64.so.6" }, - .{ "libhiprtc.so", "libhiprtc.so.6" }, + .{ "libhipblaslt.so", "libhipblaslt.so.1" }, + .{ "libamdhip64.so", "libamdhip64.so.7" }, + .{ "libhiprtc.so", "libhiprtc.so.7" }, }); var buf: [std.fs.max_path_bytes]u8 = undefined; From 860a52e8f08fb3a2691917679ff57a2550bcf8c5 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 17 Dec 2025 16:34:59 +0100 Subject: [PATCH 148/881] io: fixes (#366) --- examples/llama/main.zig | 30 +++- examples/mnist/BUILD.bazel | 4 +- examples/mnist/mnist.zig | 10 +- zml2/io.zig | 18 +-- zml2/io/vfs.zig | 58 +++---- zml2/io/vfs/http.zig | 111 ++++++++----- zml2/main.zig | 6 +- zml2/main_io.zig | 37 +++-- zml2/safetensors.zig | 324 +++++++++++++++++++------------------ 9 files changed, 324 insertions(+), 274 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 5df1e639..d75a825a 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -163,10 +163,26 @@ pub fn main() !void { var vfs_file: zml.io.VFS.File = .init(threaded.io()); + var http_client: std.http.Client = .{ + .allocator = allocator, + .io = threaded.io(), + .connection_pool = .{ + .free_size = threaded.async_limit.toInt() orelse 16, + }, + }; + + try http_client.initDefaultProxies(allocator); + defer http_client.deinit(); + + var vfs_http: zml.io.VFS.HTTP = try .init(allocator, threaded.io(), &http_client, .{}); + defer vfs_http.deinit(); + var vfs: zml.io.VFS = .init(allocator, threaded.io()); defer vfs.deinit(); try vfs.register("file", vfs_file.io()); + try vfs.register("http", vfs_http.io()); + try vfs.register("https", vfs_http.io()); const io = vfs.io(); @@ -205,16 +221,14 @@ pub fn main() !void { log.err("Missing --hf-model-path", .{}); return; }; - const repo = try vfs.openAbsoluteDir(io, hf_model_path, .{}); - defer repo.close(io); - const model_tokenizer_path = try std.fs.path.join(allocator, &.{ hf_model_path, "tokenizer.json" }); - defer allocator.free(model_tokenizer_path); + var repo = try zml.safetensors.resolveModelRepo(allocator, io, hf_model_path); + defer repo.deinit(allocator, io); const config = blk: { var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); - const config_json_file = try repo.openFile(io, "config.json", .{}); + const config_json_file = try repo.dir.openFile(io, "config.json", .{}); defer config_json_file.close(io); var config_json_buffer: [256]u8 = undefined; var config_reader = config_json_file.reader(io, &config_json_buffer); @@ -230,12 +244,12 @@ pub fn main() !void { const create_opts_json = cli.args.@"create-options" orelse "{}"; const create_opts = try std.json.parseFromSlice(zml.platform.CreateOptions, allocator, create_opts_json, .{}); defer create_opts.deinit(); - const platform: zml.Platform = try .auto(threaded.io(), create_opts.value); + const platform: zml.Platform = try .auto(io, create_opts.value); break :b platform; }; defer platform.deinit(); - var registry = try zml.safetensors.parseFromPath(allocator, io, &vfs, hf_model_path); + var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); defer registry.deinit(); var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); @@ -317,7 +331,7 @@ pub fn main() !void { var timer = try stdx.time.Timer.start(); defer log.info("Loaded tokenizer [{D}]", .{timer.read()}); const bytes = b: { - const file = try repo.openFile(io, "tokenizer.json", .{}); + const file = try repo.dir.openFile(io, "tokenizer.json", .{}); defer file.close(io); var reader = file.reader(io, &.{}); diff --git a/examples/mnist/BUILD.bazel b/examples/mnist/BUILD.bazel index f76bfc48..69ea3c2c 100644 --- a/examples/mnist/BUILD.bazel +++ b/examples/mnist/BUILD.bazel @@ -4,8 +4,8 @@ load("@rules_zig//zig:defs.bzl", "zig_binary") zig_binary( name = "mnist", args = [ - #"$(location @mnist//:mnist.safetensors)", - #"$(location @mnist//:t10k-images.idx3-ubyte)", + "$(location @mnist//:mnist.safetensors)", + "$(location @mnist//:t10k-images.idx3-ubyte)", ], data = [ "@mnist//:mnist.safetensors", diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index f8cb288e..eedc9c37 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -37,8 +37,8 @@ const Mnist = struct { }; } - pub fn loadBuffers(self: Mnist, allocator: std.mem.Allocator, io: std.Io, vfs: *zml.io.VFS, store: zml.io.TensorStore.View, platform: zml.Platform) !zml.Bufferized(Mnist) { - return zml.io.loadBuffersFromId(allocator, io, vfs, self, store, platform); + pub fn loadBuffers(self: Mnist, allocator: std.mem.Allocator, io: std.Io, store: zml.io.TensorStore.View, platform: zml.Platform) !zml.Bufferized(Mnist) { + return zml.io.loadBuffersFromId(allocator, io, self, store, platform); } pub fn unloadBuffers(self: *zml.Bufferized(Mnist)) void { @@ -92,7 +92,7 @@ pub fn main() !void { const t10kfilename = process_args[2]; // Read model shapes. - var registry = try zml.safetensors.parseFromPath(allocator, io, &vfs, model_path); + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); defer registry.deinit(); // Init model @@ -116,14 +116,14 @@ pub fn main() !void { defer results.deinit(allocator); // Load buffers - var mnist_buffers = try mnist_model.loadBuffers(allocator, io, &vfs, store.view(), platform); + var mnist_buffers = try mnist_model.loadBuffers(allocator, io, store.view(), platform); defer Mnist.unloadBuffers(&mnist_buffers); log.info("✅ Weights transferred in {f}", .{timer.read()}); log.info("Starting inference...", .{}); // Load a random digit image from the dataset. - const dataset = try vfs.openAbsoluteFile(io, t10kfilename, .{ .mode = .read_only }); + const dataset = try std.Io.Dir.openFile(.cwd(), io, t10kfilename, .{ .mode = .read_only }); defer dataset.close(io); const now = std.Io.Clock.now(.awake, io) catch unreachable; diff --git a/zml2/io.zig b/zml2/io.zig index 764958d9..b42b157b 100644 --- a/zml2/io.zig +++ b/zml2/io.zig @@ -55,14 +55,14 @@ pub const TensorStore = struct { return tensor_desc_ptr; } - pub fn getReader(self: *const TensorStore, key: []const u8, io: std.Io, vfs: *VFS, buffer: []u8) !safetensors.TensorReader { - return self.registry.reader(io, vfs, key, buffer); + pub fn getReader(self: *const TensorStore, key: []const u8, io: std.Io, buffer: []u8) !safetensors.TensorReader { + return self.registry.reader(io, key, buffer); } - pub fn getReaderById(self: *const TensorStore, id: usize, io: std.Io, vfs: *VFS, buffer: []u8) !safetensors.TensorReader { + pub fn getReaderById(self: *const TensorStore, id: usize, io: std.Io, buffer: []u8) !safetensors.TensorReader { const tensor_desc = self.id_map.get(id) orelse return error.NotFound; - return safetensors.TensorReader.init(io, vfs, tensor_desc.*, buffer); + return safetensors.TensorReader.init(io, tensor_desc.*, buffer); } pub fn view(self: *TensorStore) View { @@ -170,10 +170,10 @@ pub const TensorStore = struct { return entry_ptr.shape; } - pub fn getReader(self: View, subkey: []const u8, io: std.Io, vfs: *VFS, buffer: []u8) !safetensors.TensorReader { + pub fn getReader(self: View, subkey: []const u8, io: std.Io, buffer: []u8) !safetensors.TensorReader { var key_buffer: [256]u8 = undefined; const key = std.fmt.bufPrint(&key_buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; - return self.store.getReader(key, io, vfs, buffer); + return self.store.getReader(key, io, buffer); } }; }; @@ -212,7 +212,7 @@ fn collectTensorDesc(allocator: std.mem.Allocator, store: TensorStore.View, v: a return try list.toOwnedSlice(); } -pub fn loadBuffersFromId(allocator: std.mem.Allocator, io: std.Io, vfs: *VFS, model: anytype, store: TensorStore.View, platform: Platform) !Bufferized(@TypeOf(model)) { +pub fn loadBuffersFromId(allocator: std.mem.Allocator, io: std.Io, model: anytype, store: TensorStore.View, platform: Platform) !Bufferized(@TypeOf(model)) { const Model = @TypeOf(model); var result: Bufferized(Model) = undefined; initBufferizedFrom(model, &result); @@ -244,7 +244,6 @@ pub fn loadBuffersFromId(allocator: std.mem.Allocator, io: std.Io, vfs: *VFS, mo store: TensorStore.View, allocator: std.mem.Allocator, io: std.Io, - vfs: *VFS, }; var context: LocalContext = .{ .tensor_descs = tensor_descs, @@ -256,13 +255,12 @@ pub fn loadBuffersFromId(allocator: std.mem.Allocator, io: std.Io, vfs: *VFS, mo .store = store, .allocator = allocator, .io = io, - .vfs = vfs, }; try meta.visit(struct { fn cb(context_: *LocalContext, buffer: *Buffer) !void { const tensor_desc = context_.tensor_descs[context_.index]; - var reader = try safetensors.TensorReader.init(context_.io, context_.vfs, tensor_desc, context_.buffer_reader); + var reader = try safetensors.TensorReader.init(context_.io, tensor_desc, context_.buffer_reader); defer reader.deinit(); var writer = try context_.transfer.getWriter(context_.io, context_.index, context_.buffer_writer); diff --git a/zml2/io/vfs.zig b/zml2/io/vfs.zig index 551198de..413761a7 100644 --- a/zml2/io/vfs.zig +++ b/zml2/io/vfs.zig @@ -33,8 +33,6 @@ pub const VFS = struct { handle: std.Io.Dir.Handle, }; - const ROOT_DIR_HANDLE: std.Io.Dir.Handle = 0; - allocator: std.mem.Allocator, mutex: std.Io.Mutex, @@ -110,7 +108,7 @@ pub const VFS = struct { .files = .{}, .dirs = .{}, .next_file_handle = 0, - .next_dir_handle = 1, // 0 reserved for ROOT_DIR_HANDLE + .next_dir_handle = 0, .vtable = vtable, .base_io = base_io, }; @@ -154,6 +152,7 @@ pub const VFS = struct { } fn backendFromAbsolutePath(self: *VFS, path: []const u8) ResolveAbsolutePathError!std.Io { + if (std.fs.path.isAbsolutePosix(path)) return self.base_io; const uri = std.Uri.parse(path) catch return ResolveAbsolutePathError.InvalidURI; return self.backends.get(uri.scheme) orelse return ResolveAbsolutePathError.SchemeNotRegistered; } @@ -218,8 +217,6 @@ pub const VFS = struct { } fn closeDir(self: *VFS, virtual_handle: std.Io.Dir.Handle) void { - if (virtual_handle == ROOT_DIR_HANDLE) return; - self.mutex.lockUncancelable(self.base_io); const entry = self.dirs.fetchRemove(virtual_handle); self.mutex.unlock(self.base_io); @@ -231,38 +228,23 @@ pub const VFS = struct { } } - fn fsDir(self: *VFS, virtual_handle: std.Io.Dir.Handle, sub_path: ?[]const u8) DirEntryError!FsDir { - if (virtual_handle == ROOT_DIR_HANDLE) { + fn fsDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) DirEntryError!FsDir { + if (std.meta.eql(dir, std.Io.Dir.cwd())) { std.debug.assert(sub_path != null); - const backend = try self.backendFromAbsolutePath(sub_path.?); - return .{ .backend = backend, .handle = ROOT_DIR_HANDLE }; + const backend = self.backendFromAbsolutePath(sub_path.?) catch blk: { + log.warn("Unable to resolve backend for relative path {s} in cwd, falling back to base_io", .{sub_path.?}); + break :blk self.base_io; + }; + return .{ .backend = backend, .handle = dir.handle }; } self.mutex.lockUncancelable(self.base_io); defer self.mutex.unlock(self.base_io); - const entry = self.dirs.get(virtual_handle) orelse return DirEntryError.MissingVirtualDirHandle; + const entry = self.dirs.get(dir.handle) orelse return DirEntryError.MissingVirtualDirHandle; return entry.*; } - pub fn rootDir(self: *VFS) std.Io.Dir { - _ = self; - return .{ .handle = ROOT_DIR_HANDLE }; - } - - pub fn openAbsoluteDir(self: *VFS, vfs_io: std.Io, path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { - const backend_dir = try vfs_io.vtable.dirOpenDir(vfs_io.userdata, .{ .handle = ROOT_DIR_HANDLE }, path, options); - const virtual_handle = self.registerDir(vfs_io, backend_dir.handle) catch return std.Io.Dir.OpenError.Unexpected; - - return .{ .handle = virtual_handle }; - } - - pub fn openAbsoluteFile(self: *VFS, vfs_io: std.Io, path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { - const file = try vfs_io.vtable.dirOpenFile(vfs_io.userdata, .{ .handle = ROOT_DIR_HANDLE }, path, flags); - const virtual_handle = self.registerFile(vfs_io, file.handle) catch return std.Io.File.OpenError.SystemResources; - return .{ .handle = virtual_handle }; - } - // VTable implementations fn dirMake( @@ -273,7 +255,7 @@ pub const VFS = struct { ) std.Io.Dir.MakeError!void { const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const fs_dir = self.fsDir(dir.handle, sub_path) catch |err| { + const fs_dir = self.fsDir(dir, sub_path) catch |err| { return switch (err) { DirEntryError.SchemeNotRegistered => std.Io.Dir.MakeError.BadPathName, DirEntryError.InvalidURI => std.Io.Dir.MakeError.BadPathName, @@ -292,7 +274,7 @@ pub const VFS = struct { ) std.Io.Dir.MakeError!void { const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const fs_dir = self.fsDir(dir.handle, sub_path) catch |err| { + const fs_dir = self.fsDir(dir, sub_path) catch |err| { return switch (err) { DirEntryError.SchemeNotRegistered => std.Io.Dir.MakeError.BadPathName, DirEntryError.InvalidURI => std.Io.Dir.MakeError.BadPathName, @@ -311,7 +293,7 @@ pub const VFS = struct { ) std.Io.Dir.MakeOpenPathError!std.Io.Dir { const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const fs_dir = self.fsDir(dir.handle, sub_path) catch |err| { + const fs_dir = self.fsDir(dir, sub_path) catch |err| { return switch (err) { DirEntryError.SchemeNotRegistered => std.Io.Dir.MakeOpenPathError.BadPathName, DirEntryError.InvalidURI => std.Io.Dir.MakeOpenPathError.BadPathName, @@ -336,9 +318,7 @@ pub const VFS = struct { ) std.Io.Dir.StatError!std.Io.Dir.Stat { const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - if (dir.handle == ROOT_DIR_HANDLE) return std.Io.Dir.StatError.Unexpected; - - const fs_dir = self.fsDir(dir.handle, null) catch |err| { + const fs_dir = self.fsDir(dir, null) catch |err| { return switch (err) { DirEntryError.SchemeNotRegistered => std.Io.Dir.StatError.Unexpected, DirEntryError.InvalidURI => std.Io.Dir.StatError.Unexpected, @@ -357,7 +337,7 @@ pub const VFS = struct { ) std.Io.Dir.StatPathError!std.Io.Dir.Stat { const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const fs_dir = self.fsDir(dir.handle, sub_path) catch |err| { + const fs_dir = self.fsDir(dir, sub_path) catch |err| { return switch (err) { DirEntryError.SchemeNotRegistered => std.Io.Dir.StatPathError.BadPathName, DirEntryError.InvalidURI => std.Io.Dir.StatPathError.BadPathName, @@ -376,7 +356,7 @@ pub const VFS = struct { ) std.Io.Dir.AccessError!void { const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const fs_dir = self.fsDir(dir.handle, sub_path) catch |err| { + const fs_dir = self.fsDir(dir, sub_path) catch |err| { return switch (err) { DirEntryError.SchemeNotRegistered => std.Io.Dir.AccessError.BadPathName, DirEntryError.InvalidURI => std.Io.Dir.AccessError.BadPathName, @@ -395,7 +375,7 @@ pub const VFS = struct { ) std.Io.File.OpenError!std.Io.File { const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const fs_dir = self.fsDir(dir.handle, sub_path) catch |err| { + const fs_dir = self.fsDir(dir, sub_path) catch |err| { return switch (err) { DirEntryError.SchemeNotRegistered => std.Io.File.OpenError.BadPathName, DirEntryError.InvalidURI => std.Io.File.OpenError.BadPathName, @@ -422,7 +402,7 @@ pub const VFS = struct { ) std.Io.File.OpenError!std.Io.File { const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const fs_dir = self.fsDir(dir.handle, sub_path) catch |err| { + const fs_dir = self.fsDir(dir, sub_path) catch |err| { return switch (err) { DirEntryError.SchemeNotRegistered => std.Io.File.OpenError.BadPathName, DirEntryError.InvalidURI => std.Io.File.OpenError.BadPathName, @@ -449,7 +429,7 @@ pub const VFS = struct { ) std.Io.Dir.OpenError!std.Io.Dir { const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const fs_dir = self.fsDir(dir.handle, sub_path) catch |err| { + const fs_dir = self.fsDir(dir, sub_path) catch |err| { return switch (err) { DirEntryError.SchemeNotRegistered => std.Io.Dir.OpenError.BadPathName, DirEntryError.InvalidURI => std.Io.Dir.OpenError.BadPathName, diff --git a/zml2/io/vfs/http.zig b/zml2/io/vfs/http.zig index 8fbbca76..2d1f6d4c 100644 --- a/zml2/io/vfs/http.zig +++ b/zml2/io/vfs/http.zig @@ -5,12 +5,18 @@ const log = std.log.scoped(.@"zml/io/vfs/http"); pub const HTTP = struct { const HandleId = i32; + pub const InitHandleError = error{ + PathTooLong, + AllocationFailed, + PathUnresolvable, + } || std.mem.Allocator.Error; + pub const FileHandle = struct { allocator: std.mem.Allocator, url: []const u8, pos: u64, - size: ?u64, + size: u64, redirect_buffer: [std.fs.max_path_bytes]u8, request: ?*std.http.Client.Request, response: ?std.http.Client.Response, @@ -21,7 +27,7 @@ pub const HTTP = struct { min_latency_ns: u64 = std.math.maxInt(u64), max_latency_ns: u64 = 0, - pub fn init(allocator: std.mem.Allocator, parent_path: []const u8, sub_path: []const u8) std.mem.Allocator.Error!*FileHandle { + pub fn init(allocator: std.mem.Allocator, parent_path: []const u8, sub_path: []const u8) InitHandleError!*FileHandle { const handle = try allocator.create(FileHandle); errdefer allocator.destroy(handle); @@ -29,7 +35,7 @@ pub const HTTP = struct { .allocator = allocator, .url = try resolveUri(allocator, parent_path, sub_path), .pos = 0, - .size = null, + .size = undefined, .redirect_buffer = undefined, .request = null, .response = null, @@ -67,15 +73,28 @@ pub const HTTP = struct { return (self.total_latency_ns / self.total_requests) / std.time.ns_per_ms; } - fn resolveUri(allocator: std.mem.Allocator, parent_path: []const u8, sub_path: []const u8) std.mem.Allocator.Error![]const u8 { - // todo: check uri + fn resolveUri(allocator: std.mem.Allocator, parent_path: []const u8, sub_path: []const u8) InitHandleError![]const u8 { if (std.mem.startsWith(u8, sub_path, "http://") or std.mem.startsWith(u8, sub_path, "https://")) { return try allocator.dupe(u8, sub_path); } - return std.fs.path.join(allocator, &.{ parent_path, sub_path }); + const base_uri = std.Uri.parse(parent_path) catch return InitHandleError.PathTooLong; + var aux_buf: [std.fs.max_path_bytes]u8 = undefined; + + if (sub_path.len > aux_buf.len) return InitHandleError.PathTooLong; + + @memcpy(aux_buf[0..sub_path.len], sub_path); + var aux_slice: []u8 = aux_buf[0..]; + const resolved = std.Uri.resolveInPlace(base_uri, sub_path.len, &aux_slice) catch return InitHandleError.PathUnresolvable; + return std.fmt.allocPrint(allocator, "{f}", .{resolved.fmt(.{ + .scheme = true, + .authority = true, + .path = true, + .query = true, + .fragment = true, + })}); } }; @@ -83,13 +102,21 @@ pub const HTTP = struct { allocator: std.mem.Allocator, url: []const u8, - pub fn init(allocator: std.mem.Allocator, parent_path: []const u8, sub_path: []const u8) std.mem.Allocator.Error!*DirHandle { + pub fn init(allocator: std.mem.Allocator, parent_path: []const u8, sub_path: []const u8) InitHandleError!*DirHandle { const self = try allocator.create(DirHandle); errdefer allocator.destroy(self); - const url = try std.fs.path.join(allocator, &.{ parent_path, sub_path }); + var url = try FileHandle.resolveUri(allocator, parent_path, sub_path); errdefer allocator.free(url); + // Ensure directory URLs have a trailing slash. + // According to RFC 3986, if the parent path lacks a trailing slash, it is treated as a file and the last segment is replaced. + if (url.len == 0 or url[url.len - 1] != '/') { + const with_slash = try std.fmt.allocPrint(allocator, "{s}/", .{url}); + allocator.free(url); + url = with_slash; + } + self.* = .{ .allocator = allocator, .url = url, @@ -124,7 +151,7 @@ pub const HTTP = struct { vtable: std.Io.VTable, pub fn init(allocator: std.mem.Allocator, base_io: std.Io, http_client: *std.http.Client, config: Config) std.mem.Allocator.Error!HTTP { - var self: HTTP = .{ + return .{ .allocator = allocator, .mutex = .init, .file_handles = .{}, @@ -135,17 +162,6 @@ pub const HTTP = struct { .config = config, .vtable = makeVTable(base_io), }; - - // Used by VFS - const root_dir_handle = DirHandle.init(allocator, "", "") catch { - log.err("Failed to create root DirHandle", .{}); - return self; - }; - errdefer root_dir_handle.deinit(); - - _ = try self.registerDirHandle(root_dir_handle); - - return self; } pub fn deinit(self: *HTTP) void { @@ -200,11 +216,22 @@ pub const HTTP = struct { return handle_id; } - fn dirHandle(self: *HTTP, dir: std.Io.Dir) ?*DirHandle { + fn dirHandle(self: *HTTP, dir: std.Io.Dir) ?DirHandle { self.mutex.lockUncancelable(self.io()); defer self.mutex.unlock(self.io()); - return self.dir_handles.get(dir.handle); + if (std.meta.eql(dir, std.Io.Dir.cwd())) { + return .{ + .allocator = self.allocator, + .url = "", + }; + } + + if (self.dir_handles.get(dir.handle)) |handle_ptr| { + return handle_ptr.*; + } else { + return null; + } } fn openFile( @@ -228,7 +255,22 @@ pub const HTTP = struct { return std.Io.File.OpenError.SystemResources; }; - return .{ .handle = @intCast(handle_id) }; + const file: std.Io.File = .{ .handle = @intCast(handle_id) }; + + const stat = self.fetchFileStat(file) catch |err| switch (err) { + error.AccessDenied, error.SystemResources => { + log.err("File not found during openFile for handle={d} url={s}", .{ file.handle, handle.url }); + return std.Io.File.OpenError.FileNotFound; + }, + else => { + log.err("Failed to fetch file stat during openFile for handle={d} url={s}", .{ file.handle, handle.url }); + return std.Io.File.OpenError.Unexpected; + }, + }; + + handle.size = stat.size; + + return file; } fn openDir( @@ -322,21 +364,8 @@ pub const HTTP = struct { var offset: usize = 0; while (offset < buf.len) { - const file_size = blk: { - if (handle.size) |size| break :blk size; - - const stat = self.fetchFileStat(file) catch { - log.err("Failed to fetch file stat during read for handle={d} url={s}", .{ file.handle, handle.url }); - return std.Io.File.Reader.Error.Unexpected; - }; - - handle.size = stat.size; - - break :blk stat.size; - }; - // Check EOF against known file size - if (handle.pos >= file_size) { + if (handle.pos >= handle.size) { return total_read; } @@ -344,7 +373,7 @@ pub const HTTP = struct { var desired_read_len: u64 = @max(@as(u64, @intCast(buf.len)), self.config.request_range_min); if (desired_read_len > self.config.request_range_max) desired_read_len = self.config.request_range_max; - const end = if (handle.size.? > 0 and start < file_size) @min(start + desired_read_len - 1, file_size - 1) else start + desired_read_len - 1; + const end = if (handle.size > 0 and start < handle.size) @min(start + desired_read_len - 1, handle.size - 1) else start + desired_read_len - 1; try self.requestRead(file, start, end); const reader = handle.body_reader.?; @@ -454,8 +483,10 @@ pub const HTTP = struct { var attempt: u32 = 0; while (attempt < self.config.max_fetch_attempts) : (attempt += 1) { - const req = self.allocator.create(std.http.Client.Request) catch continue; - errdefer self.allocator.destroy(req); + const req = self.allocator.create(std.http.Client.Request) catch { + log.err("Failed to allocate HTTP HEAD request", .{}); + return std.Io.File.StatError.SystemResources; + }; req.* = self.client.request(.HEAD, std.Uri.parse(handle.url) catch return std.Io.File.StatError.Unexpected, .{ .headers = .{ .accept_encoding = .{ .override = "identity" } }, diff --git a/zml2/main.zig b/zml2/main.zig index c3f2f897..3d6f9036 100644 --- a/zml2/main.zig +++ b/zml2/main.zig @@ -61,16 +61,16 @@ pub fn main() !void { const io = vfs.io(); - const file = try vfs.openAbsoluteFile(io, "//private/var/tmp/_bazel_corendos/c1731bd4fcb7bdb2687c1972ca03d70f/external/+non_module_deps+mnist/mnist.safetensors", .{ .mode = .read_only }); + const file = try std.Io.Dir.openFile(.cwd(), io, "/private/var/tmp/_bazel_corendos/c1731bd4fcb7bdb2687c1972ca03d70f/external/+non_module_deps+mnist/mnist.safetensors", .{ .mode = .read_only }); defer file.close(io); zml.init(); defer zml.deinit(); - const available = zml.Platform.availablePlatforms(threaded.io()); + const available = zml.Platform.availablePlatforms(io); std.log.info("Available platforms: {any}", .{available}); - var platform = try zml.Platform.auto(threaded.io(), .{}); + var platform = try zml.Platform.auto(io, .{}); defer platform.deinit(); std.log.info("Selected platform: {f}", .{platform}); diff --git a/zml2/main_io.zig b/zml2/main_io.zig index 06310233..50b4deb4 100644 --- a/zml2/main_io.zig +++ b/zml2/main_io.zig @@ -14,12 +14,12 @@ pub const std_options: std.Options = .{ }; pub fn main() !void { - // var debug_allocator: std.heap.DebugAllocator(.{}) = .init; - // defer std.debug.assert(debug_allocator.deinit() == .ok); + var debug_allocator: std.heap.DebugAllocator(.{}) = .init; + defer std.debug.assert(debug_allocator.deinit() == .ok); - // const allocator = debug_allocator.allocator(); + const allocator = debug_allocator.allocator(); - const allocator = std.heap.c_allocator; + // const allocator = std.heap.c_allocator; var threaded: std.Io.Threaded = .init(allocator); defer threaded.deinit(); @@ -88,6 +88,23 @@ pub fn main() !void { break :blk default_uri; }; + { + // const initial_path = "/Users/hugo/Developer/Llama-3.1-8B-Instruct/Llama-3.1-8B-Instruct"; + // const initial_path = "file:///Users/hugo/Developer/Llama-3.1-8B-Instruct/Llama-3.1-8B-Instruct"; + const initial_path = "https://storage.googleapis.com/zig-vfs/Llama-3.1-8B-Instruct/Llama-3.1-8B-Instruct"; + + const llama_dir = try std.Io.Dir.openDir(.cwd(), io, initial_path, .{}); + defer llama_dir.close(io); + + const filepath = "../model.safetensors.index.json"; + + const index_file = try llama_dir.openFile(io, filepath, .{}); + defer index_file.close(io); + + const stat = try index_file.stat(io); + log.info("Opened local index file with size: {d} bytes", .{stat.size}); + } + { var timer: std.time.Timer = try .start(); defer { @@ -95,7 +112,7 @@ pub fn main() !void { log.info("Completed in {d} ms", .{elapsed / std.time.ns_per_ms}); } - var registry = try zml.safetensors.parseFromPath(allocator, io, &vfs, uri); + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, uri); defer registry.deinit(); // var it = registry.iterator(); @@ -153,7 +170,7 @@ pub fn main() !void { var it = registry.iterator(); while (it.next()) |entry| { const tensor_name = entry.key_ptr.*; - group.async(io, readTensor, .{ allocator, io, &vfs, &pool, ®istry, tensor_name, &err }); + group.async(io, readTensor, .{ io, &pool, ®istry, tensor_name, &err }); } group.wait(io); @@ -294,24 +311,20 @@ pub fn main() !void { } fn readTensor( - allocator: std.mem.Allocator, io: std.Io, - vfs: *zml.io.VFS, pool: *MemoryPool, registry: *zml.safetensors.TensorRegistry, tensor_name: []const u8, err_ptr: *?anyerror, ) void { - readTensorImpl(allocator, io, vfs, pool, registry, tensor_name) catch |err| { + readTensorImpl(io, pool, registry, tensor_name) catch |err| { err_ptr.* = err; log.err("Failed to read tensor {s}: {any}", .{ tensor_name, err }); }; } pub fn readTensorImpl( - _: std.mem.Allocator, io: std.Io, - vfs: *zml.io.VFS, pool: *MemoryPool, registry: *zml.safetensors.TensorRegistry, tensor_name: []const u8, @@ -323,7 +336,7 @@ pub fn readTensorImpl( const mem_slot = try pool.alloc(io, tensor.byteSize() * 2); defer pool.free(io, mem_slot.index); - var tensor_reader = try registry.reader(io, vfs, tensor_name, mem_slot.data[0..tensor.byteSize()]); + var tensor_reader = try registry.reader(io, tensor_name, mem_slot.data[0..tensor.byteSize()]); defer tensor_reader.deinit(); var read_timer = try std.time.Timer.start(); diff --git a/zml2/safetensors.zig b/zml2/safetensors.zig index c257b6d0..fd0d2418 100644 --- a/zml2/safetensors.zig +++ b/zml2/safetensors.zig @@ -2,7 +2,6 @@ const std = @import("std"); const Shape = @import("shape.zig").Shape; const DataType = @import("dtype.zig").DataType; -const VFS = @import("io").VFS; const Dims = Shape.DimsArray; const StringBuilder = std.ArrayListUnmanaged(u8); @@ -14,77 +13,6 @@ const log = std.log.scoped(.@"zml/safetensors"); const BYTES_HEADER = 8; -pub fn parseFromPath( - allocator: std.mem.Allocator, - io: std.Io, - vfs: *VFS, // todo: remove pointer when std.Io.Dir/File ready - path: []const u8, -) !TensorRegistry { - const file_path = try resolveModelPathEntrypoint(allocator, io, vfs, path); - defer allocator.free(file_path); - log.debug("Resolved model path entrypoint for '{s}': {s}", .{ path, file_path }); - - const repo_path = try resolveModelRepoPath(allocator, io, vfs, path); - defer allocator.free(repo_path); - log.debug("Resolved model repo path for '{s}': {s}", .{ path, repo_path }); - - const repo = try vfs.openAbsoluteDir(io, repo_path, .{}); - defer repo.close(io); - - const file_type = resolveFiletype(file_path); - log.debug("Resolved file type for '{s}': {any}", .{ path, file_type }); - - return switch (file_type) { - .index => blk: { - const index_file = try vfs.openAbsoluteFile(io, file_path, .{}); - defer index_file.close(io); - - const file_reader_buf = try allocator.alloc(u8, 64 * 1024); - defer allocator.free(file_reader_buf); - - var index_reader = index_file.reader(io, file_reader_buf); - - var safetensors_index = try parseSafetensorsIndex( - allocator, - &index_reader.interface, - ); - defer safetensors_index.deinit(); - - var registry: TensorRegistry = try .initWithMetadata(allocator, safetensors_index.metadata); - - try parseSafetensorsIndexFiles( - allocator, - io, - safetensors_index, - ®istry, - repo, - repo_path, - ); - - break :blk registry; - }, - .safetensors => blk: { - var registry: TensorRegistry = .init(allocator); - - log.info("Parsing single safetensors file '{s}'", .{file_path}); - const file = try repo.openFile(io, file_path, .{}); - defer file.close(io); - - log.info("Opened safetensors file '{s}'", .{file_path}); - - const header_buf = try allocator.alloc(u8, 16 * 1024); - defer allocator.free(header_buf); - - var reader = file.reader(io, header_buf); - - try parseSafetensors(allocator, file_path, &reader.interface, ®istry); - - break :blk registry; - }, - else => return error.InvalidPath, - }; -} - pub const FileType = enum { index, safetensors, @@ -106,92 +34,147 @@ pub const ModelPathResolutionError = error{ InvalidPath, } || std.mem.Allocator.Error; -pub fn resolveModelRepoPath(allocator: std.mem.Allocator, io: std.Io, vfs: *VFS, path: []const u8) ModelPathResolutionError![]const u8 { - const resolved_path = if (std.mem.startsWith(u8, path, "/")) - try std.fmt.allocPrint(allocator, "file://{s}", .{path}) - else - try allocator.dupe(u8, path); - defer allocator.free(resolved_path); +pub const ModelRepo = struct { + dir: std.Io.Dir, + path: []const u8, - var supported_schemes_it = vfs.backends.keyIterator(); - - while (supported_schemes_it.next()) |scheme| { - if (std.mem.startsWith(u8, resolved_path, scheme.*)) { - break; - } - } else { - log.err("Unsupported URI scheme in path: {s}", .{resolved_path}); - return ModelPathResolutionError.InvalidPath; + pub fn deinit(self: *ModelRepo, allocator: std.mem.Allocator, io: std.Io) void { + self.dir.close(io); + allocator.free(self.path); } +}; + +pub fn resolveModelRepo( + allocator: std.mem.Allocator, + io: std.Io, + path: []const u8, +) ModelPathResolutionError!ModelRepo { + const resolved_path = try allocator.dupe(u8, path); if (std.mem.endsWith(u8, resolved_path, ".safetensors.index.json") or std.mem.endsWith(u8, resolved_path, ".safetensors")) { - const dir_path = std.fs.path.dirname(resolved_path) orelse return ModelPathResolutionError.InvalidPath; - return try allocator.dupe(u8, dir_path); + const dir_path = std.fs.path.dirname(resolved_path) orelse { + allocator.free(resolved_path); + return ModelPathResolutionError.InvalidPath; + }; + const dir_path_dup = try allocator.dupe(u8, dir_path); + allocator.free(resolved_path); + + const dir = std.Io.Dir.openDir(.cwd(), io, dir_path_dup, .{}) catch |e| { + allocator.free(dir_path_dup); + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; + }; + + return .{ + .dir = dir, + .path = dir_path_dup, + }; } - if (vfs.openAbsoluteDir(io, resolved_path, .{})) |dir| { - defer dir.close(io); - return try allocator.dupe(u8, resolved_path); - } else |_| {} + const dir = std.Io.Dir.openDir(.cwd(), io, resolved_path, .{}) catch |e| { + allocator.free(resolved_path); + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; + }; + + return .{ + .dir = dir, + .path = resolved_path, + }; +} + +pub const ModelEntrypoint = struct { + file: std.Io.File, + path: []const u8, + + pub fn deinit(self: *ModelEntrypoint, allocator: std.mem.Allocator, io: std.Io) void { + self.file.close(io); + allocator.free(self.path); + } +}; + +pub fn resolveModelEntrypoint(allocator: std.mem.Allocator, io: std.Io, repo: ModelRepo) ModelPathResolutionError!ModelEntrypoint { + { + const index_name = "model.safetensors.index.json"; + log.warn("Trying to resolve model entrypoint: {s}/{s}", .{ repo.path, index_name }); + if (repo.dir.openFile(io, index_name, .{ .mode = .read_only })) |index_file| { + const index_path = try std.fmt.allocPrint(allocator, "{s}/{s}", .{ repo.path, index_name }); + return .{ .file = index_file, .path = index_path }; + } else |_| {} + } + + { + const model_name = "model.safetensors"; + log.warn("Trying to resolve model entrypoint: {s}/{s}", .{ repo.path, model_name }); + if (repo.dir.openFile(io, model_name, .{ .mode = .read_only })) |model_file| { + const model_path = try std.fmt.allocPrint(allocator, "{s}/{s}", .{ repo.path, model_name }); + return .{ .file = model_file, .path = model_path }; + } else |_| {} + } return ModelPathResolutionError.FileNotFound; } -pub fn resolveModelPathEntrypoint(allocator: std.mem.Allocator, io: std.Io, vfs: *VFS, path: []const u8) ModelPathResolutionError![]const u8 { - const resolved_path = if (std.mem.startsWith(u8, path, "/")) - try std.fmt.allocPrint(allocator, "file://{s}", .{path}) - else - try allocator.dupe(u8, path); - defer allocator.free(resolved_path); +pub fn fetchRegistry( + allocator: std.mem.Allocator, + io: std.Io, + repo: ModelRepo, + entrypoint: ModelEntrypoint, +) !TensorRegistry { + const file_type = resolveFiletype(entrypoint.path); + log.debug("Resolved file type: {any}", .{file_type}); - var supported_schemes_it = vfs.backends.keyIterator(); + return switch (file_type) { + .index => blk: { + const index_file = try repo.dir.openFile(io, entrypoint.path, .{ .mode = .read_only }); + defer index_file.close(io); - while (supported_schemes_it.next()) |scheme| { - if (std.mem.startsWith(u8, resolved_path, scheme.*)) { - break; - } - } else { - log.err("Unsupported URI scheme in path: {s}", .{resolved_path}); - return ModelPathResolutionError.InvalidPath; - } + const file_reader_buf = try allocator.alloc(u8, 64 * 1024); + defer allocator.free(file_reader_buf); - if (std.mem.endsWith(u8, resolved_path, ".safetensors.index.json") or - std.mem.endsWith(u8, resolved_path, ".safetensors")) - { - if (vfs.openAbsoluteFile(io, resolved_path, .{})) |file| { + var index_reader = index_file.reader(io, file_reader_buf); + + var safetensors_index = try parseSafetensorsIndex( + allocator, + &index_reader.interface, + ); + defer safetensors_index.deinit(); + + var registry: TensorRegistry = try .initWithMetadata(allocator, safetensors_index.metadata); + + try parseSafetensorsIndexFiles( + allocator, + io, + safetensors_index, + ®istry, + repo, + ); + + break :blk registry; + }, + .safetensors => blk: { + var registry: TensorRegistry = .init(allocator); + + const file = try repo.dir.openFile(io, entrypoint.path, .{}); defer file.close(io); - if (file.stat(io)) |_| { - return try allocator.dupe(u8, resolved_path); - } else |_| {} - } else |_| {} - return ModelPathResolutionError.FileNotFound; - } - const index_path = try std.fs.path.join(allocator, &[_][]const u8{ resolved_path, "model.safetensors.index.json" }); + const header_buf = try allocator.alloc(u8, 16 * 1024); + defer allocator.free(header_buf); - if (vfs.openAbsoluteFile(io, index_path, .{})) |index_file| { - defer index_file.close(io); - if (index_file.stat(io)) |_| { - return index_path; - } else |_| {} - } else |_| {} + var reader = file.reader(io, header_buf); - allocator.free(index_path); + try parseSafetensors(allocator, entrypoint.path, &reader.interface, ®istry); - const model_path = try std.fs.path.join(allocator, &[_][]const u8{ resolved_path, "model.safetensors" }); - - if (vfs.openAbsoluteFile(io, model_path, .{})) |model_file| { - defer model_file.close(io); - if (model_file.stat(io)) |_| { - return model_path; - } else |_| {} - } else |_| {} - - allocator.free(model_path); - - return ModelPathResolutionError.FileNotFound; + break :blk registry; + }, + else => return error.InvalidPath, + }; } pub const TensorReader = struct { @@ -205,11 +188,10 @@ pub const TensorReader = struct { pub fn init( io: std.Io, - vfs: *VFS, tensor: Tensor, buffer: []u8, ) Error!TensorReader { - const file = try vfs.openAbsoluteFile(io, tensor.file_uri, .{}); + const file = try std.Io.Dir.openFile(.cwd(), io, tensor.file_uri, .{ .mode = .read_only }); errdefer file.close(io); var file_reader = file.reader(io, buffer); @@ -311,6 +293,46 @@ pub const TensorRegistry = struct { return self; } + pub fn fromRepo( + allocator: std.mem.Allocator, + io: std.Io, + repo: ModelRepo, + ) !TensorRegistry { + var entrypoint = try resolveModelEntrypoint(allocator, io, repo); + defer entrypoint.deinit(allocator, io); + log.debug("Resolved model path entrypoint: {s}", .{entrypoint.path}); + + return try fetchRegistry(allocator, io, repo, entrypoint); + } + + pub fn fromPath( + allocator: std.mem.Allocator, + io: std.Io, + path: []const u8, + ) !TensorRegistry { + var repo = try resolveModelRepo(allocator, io, path); + defer repo.deinit(allocator, io); + + if (std.mem.endsWith(u8, path, ".safetensors.index.json") or + std.mem.endsWith(u8, path, ".safetensors")) + { + var entrypoint: ModelEntrypoint = .{ + .file = try repo.dir.openFile(io, path, .{ .mode = .read_only }), + .path = try allocator.dupe(u8, path), + }; + defer entrypoint.deinit(allocator, io); + log.debug("Resolved model path entrypoint: {s}", .{entrypoint.path}); + + return try fetchRegistry(allocator, io, repo, entrypoint); + } else { + var entrypoint = try resolveModelEntrypoint(allocator, io, repo); + defer entrypoint.deinit(allocator, io); + log.debug("Resolved model path entrypoint: {s}", .{entrypoint.path}); + + return try fetchRegistry(allocator, io, repo, entrypoint); + } + } + pub fn deinit(self: *TensorRegistry) void { const allocator = self.arena.allocator(); self.tensors.deinit(allocator); @@ -364,7 +386,6 @@ pub const TensorRegistry = struct { pub fn reader( self: *TensorRegistry, io: std.Io, - vfs: *VFS, tensor_name: []const u8, buffer: []u8, ) TensorReader.Error!TensorReader { @@ -373,7 +394,7 @@ pub const TensorRegistry = struct { return TensorReader.Error.TensorNotFound; }; - return try .init(io, vfs, tensor, buffer); + return try .init(io, tensor, buffer); } pub fn iterator(self: *TensorRegistry) Tensors.Iterator { @@ -508,9 +529,6 @@ pub fn parseSafetensorsIndex( const weight_map = index.object.get("weight_map"); if (weight_map) |wm| { - const num_tensors = wm.object.count(); - log.debug("Found {} tensors in weight_map", .{num_tensors}); - var it = wm.object.iterator(); while (it.next()) |entry| { @@ -541,8 +559,7 @@ pub fn parseSafetensorsIndexFiles( io: std.Io, safetensors_index: SafetensorsIndex, registry: *TensorRegistry, - repo: std.Io.Dir, - repo_path: []const u8, + repo: ModelRepo, ) !void { var group: std.Io.Group = .init; defer group.cancel(io); @@ -557,7 +574,6 @@ pub fn parseSafetensorsIndexFiles( io, registry, repo, - repo_path, filename, &err, }); @@ -576,12 +592,11 @@ const AsyncParseSafetensorsIndexFile = struct { allocator: std.mem.Allocator, io: std.Io, registry: *TensorRegistry, - repo: std.Io.Dir, - repo_path: []const u8, // todo: Dir.realPath when ready + repo: ModelRepo, filename: []const u8, err_ptr: *?anyerror, ) void { - parse(allocator, io, registry, repo, repo_path, filename) catch |err| { + parse(allocator, io, registry, repo, filename) catch |err| { err_ptr.* = err; }; } @@ -590,14 +605,13 @@ const AsyncParseSafetensorsIndexFile = struct { allocator: std.mem.Allocator, io: std.Io, registry: *TensorRegistry, - repo: std.Io.Dir, - repo_path: []const u8, // todo: Dir.realPath when ready + repo: ModelRepo, filename: []const u8, ) !void { - const file_uri = try std.fs.path.join(allocator, &.{ repo_path, filename }); + const file_uri = try std.fs.path.join(allocator, &.{ repo.path, filename }); defer allocator.free(file_uri); - const file = try repo.openFile(io, filename, .{}); + const file = try repo.dir.openFile(io, filename, .{}); defer file.close(io); const header_buf = try allocator.alloc(u8, 64 * 1024); From ad93a5803056a77fa947000649d3bb5135a92b56 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 17 Dec 2025 16:36:37 +0100 Subject: [PATCH 149/881] wip loading --- examples/llama/llama.zig | 75 ++++++++++++++++++++++++++++++++++++++++ examples/llama/main.zig | 43 +++++++++-------------- 2 files changed, 91 insertions(+), 27 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 1857e588..f22b6fd9 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -61,6 +61,23 @@ pub const LlamaLM = struct { self.model.deinit(allocator); } + pub fn loadBuffers(self: *const LlamaLM, allocator: std.mem.Allocator, io: std.Io, store: zml.io.TensorStore.View, platform: zml.Platform) !zml.Bufferized(LlamaLM) { + const lm_head = if (self.lm_head) |lm_head| try zml.io.loadBuffersFromId(allocator, io, lm_head, store.withPrefix("lm_head"), platform) else null; + // TODO(Corentin): errdefer + + const model = try self.model.loadBuffers(allocator, io, store.withPrefix("model"), platform); + + return .{ + .lm_head = lm_head, + .model = model, + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(LlamaLM), allocator: std.mem.Allocator) void { + if (self.lm_head) |*lm_head| lm_head.weight.deinit(); + Llama.unloadBuffers(&self.model, allocator); + } + /// Predicts the token at `token_index` position. /// Returns: /// - updated `tokens`, @@ -141,6 +158,30 @@ pub const Llama = struct { allocator.free(self.layers); } + pub fn loadBuffers(self: *const Llama, allocator: std.mem.Allocator, io: std.Io, store: zml.io.TensorStore.View, platform: zml.Platform) !zml.Bufferized(Llama) { + const layers = try allocator.alloc(zml.Bufferized(TransformerLayer), self.layers.len); + errdefer allocator.free(layers); + + for (layers, 0..) |*layer, i| { + layer.* = try zml.io.loadBuffersFromId(allocator, io, self.layers[i], store.withLayer(i), platform); + } + + return .{ + .embed_tokens = try zml.io.loadBuffersFromId(allocator, io, self.embed_tokens, store.withPrefix("embed_tokens"), platform), + .layers = layers, + .norm = try zml.io.loadBuffersFromId(allocator, io, self.norm, store.withPrefix("norm"), platform), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Llama), allocator: std.mem.Allocator) void { + self.embed_tokens.weight.deinit(); + RmsNorm.unloadBuffers(&self.norm); + for (self.layers) |*layer| { + TransformerLayer.unloadBuffers(layer); + } + allocator.free(self.layers); + } + /// Forward one token, using KV cache for previous tokens. /// Returns result and updated KV cache. pub fn forward(self: Llama, tokens: Tensor, token_index: Tensor, kv_cache: KvCache) struct { Tensor, KvCache } { @@ -176,6 +217,13 @@ pub const TransformerLayer = struct { }; } + pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer)) void { + RmsNorm.unloadBuffers(&self.input_layernorm); + SelfAttn.unloadBuffers(&self.self_attn); + RmsNorm.unloadBuffers(&self.post_attention_layernorm); + Mlp.unloadBuffers(&self.mlp); + } + pub fn forward( self: TransformerLayer, x0: Tensor, @@ -206,6 +254,10 @@ const RmsNorm = struct { return .{ .weight = store.createTensorWithTags("weight", .{.d}), .eps = eps }; } + pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { + self.weight.deinit(); + } + /// L2 normalization of input tensor along `.d` axis. pub fn forward(self: RmsNorm, input: Tensor) Tensor { const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); @@ -231,6 +283,15 @@ const Mlp = struct { }; } + pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { + self.up_proj.weight.deinit(); + if (self.up_proj.bias) |*bias| bias.deinit(); + self.gate_proj.weight.deinit(); + if (self.gate_proj.bias) |*bias| bias.deinit(); + self.down_proj.weight.deinit(); + if (self.down_proj.bias) |*bias| bias.deinit(); + } + pub fn forward(self: Mlp, x: Tensor) Tensor { const proj = self.up_proj.forward(x); var output = self.gate_proj.forward(x); @@ -276,6 +337,20 @@ pub const SelfAttn = struct { }; } + pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { + self.q_proj.weight.deinit(); + if (self.q_proj.bias) |*bias| bias.deinit(); + self.k_proj.weight.deinit(); + if (self.k_proj.bias) |*bias| bias.deinit(); + self.v_proj.weight.deinit(); + if (self.v_proj.bias) |*bias| bias.deinit(); + self.o_proj.weight.deinit(); + if (self.o_proj.bias) |*bias| bias.deinit(); + + if (self.q_norm) |*q_norm| RmsNorm.unloadBuffers(q_norm); + if (self.k_norm) |*k_norm| RmsNorm.unloadBuffers(k_norm); + } + /// Self Attention. /// - If token_index is set, x is assumed to be the representation of one new token, /// and kv_cache will be read for the previous tokens. diff --git a/examples/llama/main.zig b/examples/llama/main.zig index d75a825a..6fd5d9ca 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -271,8 +271,11 @@ pub fn main() !void { // Specify shapes of input arguments const prefill_tokens: zml.Tensor = .init(.{ .s = llama_options.max_seq_len }, .u32); + _ = prefill_tokens; // autofix const decode_tokens: zml.Tensor = .init(.{ .s = 1 }, .u32); + _ = decode_tokens; // autofix const token_index: zml.Tensor = .init(.{}, .u32); + _ = token_index; // autofix const dtype = llama_model.model.embed_tokens.weight.dtype(); const kv_cache: llama.KvCache = .init(zml.Shape.init(.{ @@ -281,36 +284,22 @@ pub fn main() !void { .h = config.num_key_value_heads, .hd = config.head_dim orelse @divExact(config.hidden_size, config.num_attention_heads), }, dtype)); + _ = kv_cache; // autofix const rng: zml.Tensor.Rng = .init(); + _ = rng; // autofix - var prefill_exe = try platform.compileModel(allocator, io, llama.LlamaLM.forward, llama_model, .{ prefill_tokens, token_index, kv_cache, rng }); - defer prefill_exe.deinit(); - var decode_exe = try platform.compileModel(allocator, io, llama.LlamaLM.forward, llama_model, .{ decode_tokens, token_index, kv_cache, rng }); - defer decode_exe.deinit(); + // Compile the model twice, one for prefill, one for generation. + //var prefill_exe = try platform.compileModel(allocator, io, llama.LlamaLM.forward, llama_model, .{ prefill_tokens, token_index, kv_cache, rng }); + //defer prefill_exe.deinit(); + //var decode_exe = try platform.compileModel(allocator, io, llama.LlamaLM.forward, llama_model, .{ decode_tokens, token_index, kv_cache, rng }); + //defer decode_exe.deinit(); - //// Compile the model twice, one for prefill, one for generation. - //var start = try std.time.Timer.start(); - //var fut_mod_prefill = try async.async(zml.compileModel, .{ - // allocator, llama.LlamaLM.forward, llama_tensors, - // .{ - // prefill_tokens_shape, - // token_idx_shape, - // kv_cache_shape, - // rng_shape, - // }, - // platform, - //}); - - //var fut_mod = try async.async(zml.compileModel, .{ - // allocator, llama.LlamaLM.forward, llama_tensors, - // .{ - // gen_tokens_shape, - // token_idx_shape, - // kv_cache_shape, - // rng_shape, - // }, - // platform, - //}); + { + var timer = try stdx.time.Timer.start(); + defer log.info("Model loaded [{D}]", .{timer.read()}); + var llama_buffers = try llama_model.loadBuffers(allocator, io, store.view(), platform); + defer LlamaLM.unloadBuffers(&llama_buffers, allocator); + } //// While we are still compiling load the weights to the device. //log.info("\tLoading Llama weights from {s}...", .{model_weights_path}); From 0776802b4151d1be2607cb7e8615ea3b7df6c766 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 17 Dec 2025 17:19:29 +0100 Subject: [PATCH 150/881] wip llama working --- examples/llama/llama.zig | 14 ++- examples/llama/main.zig | 182 ++++++++++++++++++++------------------- zml2/buffer.zig | 67 +++++++++++++- zml2/tensor.zig | 4 + 4 files changed, 175 insertions(+), 92 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index f22b6fd9..442544e3 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -424,14 +424,20 @@ pub const KvCache = struct { }; } - pub fn initBuffer(kv_shape: zml.Shape, platform: zml.Platform) !zml.Bufferized(KvCache) { + pub fn initBuffer(self: KvCache, io: std.Io, platform: zml.Platform) !zml.Bufferized(KvCache) { return .{ - .k = try zml.Buffer.uninitialized(platform, kv_shape, .{}), - .v = try zml.Buffer.uninitialized(platform, kv_shape, .{}), - .layer_index = try zml.Buffer.scalar(platform, 0, .u32), + .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), + .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}), + .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), }; } + pub fn deinitBuffer(self: *zml.Bufferized(KvCache)) void { + self.k.deinit(); + self.v.deinit(); + self.layer_index.deinit(); + } + pub fn keys(self: KvCache) Tensor { return self.k.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); } diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 6fd5d9ca..120f8b97 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -59,17 +59,20 @@ pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tok } pub fn generateText( - config: LlamaLM.Config, - llama_: LlamaLM, - mod_prefill: zml.ModuleExe(LlamaLM.forward), - mod_generate: zml.ModuleExe(LlamaLM.forward), - kv_cache_: zml.Bufferized(llama.KvCache), - tokenizer: zml.tokenizer.Tokenizer, allocator: std.mem.Allocator, + io: std.Io, + llama_buffers: zml.Bufferized(LlamaLM), + prefill_exe: zml.exe.Exe, + decode_exe: zml.exe.Exe, + kv_cache_buffers: *zml.Bufferized(llama.KvCache), + tokenizer: zml.tokenizer.Tokenizer, + config: LlamaLM.Config, + options: LlamaLM.Options, seed: u128, prompt: []const u8, skip_llama3_encoding: bool, writer: *std.Io.Writer, + platform: zml.Platform, ) !void { const prompt_tok: []const u32 = try tokenizePrompt(allocator, tokenizer, config, prompt, skip_llama3_encoding); defer allocator.free(prompt_tok); @@ -77,37 +80,54 @@ pub fn generateText( var tokenizer_decoder = try tokenizer.decoder(); defer tokenizer_decoder.deinit(); - const platform = mod_generate.platform(); - const max_seq_len = llama_.model.max_seq_len; + const max_seq_len = options.max_seq_len; // init RNG and buffers - var rng = try zml.Tensor.Rng.init(platform, seed); - var generated_token_buffer = [_]u32{undefined}; + var rng_buffers = try zml.Tensor.Rng.initBuffer(platform, seed, io); + defer zml.Tensor.Rng.deinitBuffer(&rng_buffers); + var generated_token_slice: zml.Slice = try .alloc(allocator, zml.Shape.init(.{ .s = 1 }, .u32)); + defer generated_token_slice.free(allocator); + + { + var prefill_args = try prefill_exe.args(allocator); + defer prefill_args.deinit(allocator); + + var prefill_results = try prefill_exe.results(allocator); + defer prefill_results.deinit(allocator); - var kv_cache = prefill: { // prepare device buffers for the prefill tokens and their positions - const prefill_buffer = try allocator.alloc(u32, max_seq_len); - @memcpy(prefill_buffer[0..prompt_tok.len], prompt_tok); + const prefill_tokens_slice: zml.Slice = try .alloc(allocator, .init(.{max_seq_len}, .u32)); + defer prefill_tokens_slice.free(allocator); + @memcpy(prefill_tokens_slice.items(u32)[0..prompt_tok.len], prompt_tok); - var prefill_tokens = try zml.Buffer.fromSlice(platform, .{max_seq_len}, prefill_buffer); - defer prefill_tokens.deinit(); - var prefill_token_pos = try zml.Buffer.scalar(platform, 0, .u32); - defer prefill_token_pos.deinit(); + var prefill_tokens_buffer: zml.Buffer = try .fromSlice(io, platform, prefill_tokens_slice); + defer prefill_tokens_buffer.deinit(); + var prefill_token_pos_buffer = try zml.Buffer.scalar(io, platform, 0, .u32); + defer prefill_token_pos_buffer.deinit(); - const prefilled_tokens, const kv_cache, rng = mod_prefill.call(.{ prefill_tokens, prefill_token_pos, kv_cache_, rng }); - _ = try prefilled_tokens.toHost(std.mem.sliceAsBytes(prefill_buffer)); - generated_token_buffer[0] = prefill_buffer[prompt_tok.len - 1]; - break :prefill kv_cache; - }; - defer zml.aio.unloadBuffers(&kv_cache); + prefill_args.set(.{ llama_buffers, prefill_tokens_buffer, prefill_token_pos_buffer, kv_cache_buffers, rng_buffers }); + + prefill_exe.call(prefill_args, &prefill_results, io); + + prefill_results.fill(&.{ &prefill_tokens_buffer, kv_cache_buffers, &rng_buffers }); + try prefill_tokens_buffer.toSlice(prefill_tokens_slice, io); + generated_token_slice.items(u32)[0] = prefill_tokens_slice.items(u32)[prompt_tok.len - 1]; + } // Prepare for token-by-token generation, + + var decode_args = try decode_exe.args(allocator); + defer decode_args.deinit(allocator); + + var decode_results = try decode_exe.results(allocator); + defer decode_results.deinit(allocator); + // start with the token generated based on the full prompt. - var current_token = try zml.Buffer.fromSlice(platform, .{1}, &generated_token_buffer); - defer current_token.deinit(); + var current_token_buffer: zml.Buffer = try .fromSlice(io, platform, generated_token_slice); + defer current_token_buffer.deinit(); const output_tokens_len = max_seq_len - prompt_tok.len - 1; - const start = std.time.microTimestamp(); + var timer = try stdx.time.Timer.start(); // One token has alreadyh been generated by the prefill. var num_tokens_generated: usize = 1; @@ -115,7 +135,7 @@ pub fn generateText( generation: for (0..output_tokens_len + 1) |i| { // collect and print generated sequence num_tokens_generated += 1; - const generated_token = generated_token_buffer[0]; + const generated_token = generated_token_slice.items(u32)[0]; if (try tokenizer_decoder.next(generated_token)) |chunk| { try writer.writeAll(chunk); } @@ -132,21 +152,25 @@ pub fn generateText( } // current token pos needs to go into a zml.Buffer - const token_pos_buffer = &[_]u32{@intCast(prompt_tok.len + i)}; - const token_pos = try zml.Buffer.fromSlice(platform, .{}, token_pos_buffer); - defer token_pos.deinit(); + const token_pos_slice: zml.ConstSlice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(prompt_tok.len + i)})); + const token_pos_buffer: zml.Buffer = try .fromSlice(io, platform, token_pos_slice); + defer token_pos_buffer.deinit(); // call to generate the next token - current_token, kv_cache, rng = mod_generate.call(.{ current_token, token_pos, kv_cache, rng }); + decode_args.set(.{ llama_buffers, current_token_buffer, token_pos_buffer, kv_cache_buffers, rng_buffers }); + + decode_exe.call(decode_args, &decode_results, io); + + decode_results.fill(&.{ ¤t_token_buffer, kv_cache_buffers, &rng_buffers }); // extract the generated token from the buffer - _ = try current_token.toHost(std.mem.sliceAsBytes(&generated_token_buffer)); + try current_token_buffer.toSlice(generated_token_slice, io); } - const end = std.time.microTimestamp(); - const duration = stdx.math.divFloat(f64, end - start, std.time.us_per_s); - const speed = @as(f64, @floatFromInt(num_tokens_generated)) / duration; + const duration = timer.read(); + const duration_s = stdx.math.divFloat(f64, duration.ns, std.time.ns_per_s); + const speed = @as(f64, @floatFromInt(num_tokens_generated)) / duration_s; std.debug.print("\n", .{}); - log.info("✅ Generated {d} tokens in {:.3}s: {d:.3}tok/s", .{ num_tokens_generated, duration, speed }); + log.info("✅ Generated {} tokens in {D}: {:.3}tok/s", .{ num_tokens_generated, duration.ns, speed }); } pub fn main() !void { @@ -225,18 +249,17 @@ pub fn main() !void { var repo = try zml.safetensors.resolveModelRepo(allocator, io, hf_model_path); defer repo.deinit(allocator, io); - const config = blk: { - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); + const parsed_config = blk: { const config_json_file = try repo.dir.openFile(io, "config.json", .{}); defer config_json_file.close(io); var config_json_buffer: [256]u8 = undefined; var config_reader = config_json_file.reader(io, &config_json_buffer); var reader = std.json.Reader.init(allocator, &config_reader.interface); defer reader.deinit(); - const config_obj = try std.json.parseFromTokenSourceLeaky(llama.LlamaLM.Config, arena.allocator(), &reader, .{ .ignore_unknown_fields = true }); - break :blk config_obj; + break :blk try std.json.parseFromTokenSource(llama.LlamaLM.Config, allocator, &reader, .{ .ignore_unknown_fields = true }); }; + defer parsed_config.deinit(); + const config = parsed_config.value; // initialize ZML platform with optional create options // eg: --create-options='{"cuda":{"allocator":{"bfc":{"memory_fraction": 0.99}}}}' @@ -271,11 +294,8 @@ pub fn main() !void { // Specify shapes of input arguments const prefill_tokens: zml.Tensor = .init(.{ .s = llama_options.max_seq_len }, .u32); - _ = prefill_tokens; // autofix const decode_tokens: zml.Tensor = .init(.{ .s = 1 }, .u32); - _ = decode_tokens; // autofix const token_index: zml.Tensor = .init(.{}, .u32); - _ = token_index; // autofix const dtype = llama_model.model.embed_tokens.weight.dtype(); const kv_cache: llama.KvCache = .init(zml.Shape.init(.{ @@ -284,36 +304,20 @@ pub fn main() !void { .h = config.num_key_value_heads, .hd = config.head_dim orelse @divExact(config.hidden_size, config.num_attention_heads), }, dtype)); - _ = kv_cache; // autofix const rng: zml.Tensor.Rng = .init(); - _ = rng; // autofix // Compile the model twice, one for prefill, one for generation. - //var prefill_exe = try platform.compileModel(allocator, io, llama.LlamaLM.forward, llama_model, .{ prefill_tokens, token_index, kv_cache, rng }); - //defer prefill_exe.deinit(); - //var decode_exe = try platform.compileModel(allocator, io, llama.LlamaLM.forward, llama_model, .{ decode_tokens, token_index, kv_cache, rng }); - //defer decode_exe.deinit(); + var prefill_exe = try platform.compileModel(allocator, io, llama.LlamaLM.forward, llama_model, .{ prefill_tokens, token_index, kv_cache, rng }); + defer prefill_exe.deinit(); + var decode_exe = try platform.compileModel(allocator, io, llama.LlamaLM.forward, llama_model, .{ decode_tokens, token_index, kv_cache, rng }); + defer decode_exe.deinit(); - { - var timer = try stdx.time.Timer.start(); - defer log.info("Model loaded [{D}]", .{timer.read()}); - var llama_buffers = try llama_model.loadBuffers(allocator, io, store.view(), platform); - defer LlamaLM.unloadBuffers(&llama_buffers, allocator); - } + var llama_buffers = try llama_model.loadBuffers(allocator, io, store.view(), platform); + defer LlamaLM.unloadBuffers(&llama_buffers, allocator); - //// While we are still compiling load the weights to the device. - //log.info("\tLoading Llama weights from {s}...", .{model_weights_path}); - //var llama_buffers = try store.loadModelById(llama.LlamaLM, compiler_arena.allocator(), llama_tensors, platform); - //defer zml.aio.unloadBuffers(&llama_buffers); - //log.info("✅\tLoaded weights in {D}", .{start.read()}); - - //var llama_module_prefill = (try fut_mod_prefill.await()).prepare(llama_buffers); - //defer llama_module_prefill.deinit(); - //var llama_module = (try fut_mod.await()).prepare(llama_buffers); - //defer llama_module.deinit(); - //log.info("✅\tCompiled model in {D}", .{start.read()}); - //log.info("Creating KvCache", .{}); - //const kv_cache = try llama.KvCache.initBuffer(kv_shape, platform); + log.info("Creating KvCache", .{}); + var kv_cache_buffers = try kv_cache.initBuffer(io, platform); + defer llama.KvCache.deinitBuffer(&kv_cache_buffers); var tokenizer = blk: { log.info("Loading tokenizer", .{}); @@ -336,28 +340,32 @@ pub fn main() !void { }; errdefer tokenizer.deinit(); - //const prompt = cli.args.prompt orelse "What is the capital of France?"; - //log.info("✅\tPrompt: {s}", .{prompt}); + const prompt = cli.args.prompt orelse "What is the capital of France?"; + log.info("✅\tPrompt: {s}", .{prompt}); - //// Unbuffered writing of the tokens to stdout. - //var stdout = std.fs.File.stdout().writer(&.{}); + // Unbuffered writing of the tokens to stdout. + var stdout = std.fs.File.stdout().writer(&.{}); //const seed: u128 = cli.args.seed orelse @bitCast(std.time.nanoTimestamp()); - //const skip_llama3_encoding = cli.args.@"no-llama3" orelse false; + const seed: u128 = cli.args.seed orelse 0; + const skip_llama3_encoding = cli.args.@"no-llama3" orelse false; - //try generateText( - // config, - // llama_tensors, - // llama_module_prefill, - // llama_module, - // kv_cache, - // tokenizer, - // allocator, - // seed, - // prompt[0..], - // skip_llama3_encoding, - // &stdout.interface, - //); + try generateText( + allocator, + io, + llama_buffers, + prefill_exe, + decode_exe, + &kv_cache_buffers, + tokenizer, + config, + llama_options, + seed, + prompt[0..], + skip_llama3_encoding, + &stdout.interface, + platform, + ); } fn bool_parser(in: []const u8) error{}!bool { diff --git a/zml2/buffer.zig b/zml2/buffer.zig index b47d7942..f98466f9 100644 --- a/zml2/buffer.zig +++ b/zml2/buffer.zig @@ -10,6 +10,7 @@ const Shape = @import("shape.zig").Shape; const Slice = @import("slice.zig").Slice; const Target = @import("platform.zig").Target; const testing = @import("testing.zig"); +const constants = @import("constants.zig"); const log = std.log.scoped(.zml); @@ -117,9 +118,13 @@ pub const Buffer = struct { /// Copies the given zml.Slice or zml.ConstSlice to the accelerator memory and /// return a Buffer. pub fn fromSlice(io: std.Io, platform: Platform, slice: anytype) !Buffer { + return fromSliceOpts(io, platform, slice, .{}); + } + + pub fn fromSliceOpts(io: std.Io, platform: Platform, slice: anytype, opts: FromOptions) !Buffer { if (@TypeOf(slice) != Slice and @TypeOf(slice) != ConstSlice) @compileError("Buffer.fromSlice expects a Slice or ConstSlice, got " ++ @typeName(@TypeOf(slice))); const real_slice: ConstSlice = if (@TypeOf(slice) == Slice) slice.constSlice() else slice; - return from(io, platform, real_slice.shape, std.mem.sliceAsBytes(real_slice.data), .{}); + return from(io, platform, real_slice.shape, std.mem.sliceAsBytes(real_slice.data), opts); } /// Creates a Buffer with a single element. @@ -138,6 +143,66 @@ pub const Buffer = struct { return self; } + pub const UnitializedOptions = struct { memory: Memory = .device }; + + pub fn uninitialized(io: std.Io, platform: Platform, shape_: Shape, opts: UnitializedOptions) !Buffer { + if (opts.memory != .device) { + // XLA uninitialized doesn't respect memory see https://github.com/openxla/xla/pull/31292 + // TODO: use uninitialized when it works again. + const slice: Slice = try .alloc(std.heap.smp_allocator, shape_); + defer slice.free(std.heap.smp_allocator); + return .fromSliceOpts(io, platform, slice, .{ .memory = opts.memory }); + } + + var res: Buffer = .{ + ._api = platform.pjrt_api, + ._shape = shape_, + ._shards = .{}, + ._target = platform.target, + }; + errdefer for (res._shards.slice()) |shard| { + shard.deinit(platform.pjrt_api); + }; + + stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); + //const sharding_ax: ?u3 = std.simd.firstTrue(shape_._sharding_info); + const n_partitions = platform.sharding().num_partitions; + + //const shard_shape = if (sharding_ax) |ax| s: { + // // This kind of sharding error should be detected earlier on. + // stdx.debug.assert(@rem(shape_.dim(ax), n_partitions) == 0, "Buffer.uninitialized() expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ ax, n_partitions }); + // const shard_shape = shape_.set(ax, @divExact(shape_.dim(ax), n_partitions)); + // break :s shard_shape; + //} else shape_; + + var args = pjrt.Client.CreateUninitializedBufferArgs{ + .dims = shape_.dims(), + .element_type = pjrt.bufferTypeFromDtype(shape_.dtype()), + .layout = .{ + .tiled = .{ + .minor_to_major = constants.minorToMajor(shape_.rank()), + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }, + // set per device, see below. + .dst = undefined, + }; + + const devices = platform.getDevices(); + for (0..n_partitions) |i| { + args.dst = if (platform.target == .cpu or opts.memory == .device) + .{ .device = devices[i] } + else + .{ .memory = platform.memoryForDevice(opts.memory, devices[i]) }; + + const shard = try platform.pjrt_client.createUnitializedBuffer(platform.pjrt_api, args); + res._shards.appendAssumeCapacity(shard); + } + + return res; + } + /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. pub fn fromPjrtBuffers(platform: Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); diff --git a/zml2/tensor.zig b/zml2/tensor.zig index b2f73c2f..9be42366 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -481,6 +481,10 @@ pub const Tensor = struct { }; } + pub fn deinitBuffer(self: *Bufferized(Rng)) void { + self._state.deinit(); + } + /// Returns a Tensor of the given shape, filled with uniform random bits, and a new Rng state. /// /// The given Rng state should not be used anymore (or you'll get the same numbers again). From 190141926ff0e46ec918600a3dfe2eca6e9388b4 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 18 Dec 2025 13:35:06 +0100 Subject: [PATCH 151/881] io(vfs/file): implement direct io (#367) --- zml2/io/vfs/file.zig | 185 ++++++++++++++++++++++++++++++++++++++++++- zml2/main_io.zig | 115 ++++++++++++++++----------- 2 files changed, 252 insertions(+), 48 deletions(-) diff --git a/zml2/io/vfs/file.zig b/zml2/io/vfs/file.zig index 0dfbbeaa..d7fb94a3 100644 --- a/zml2/io/vfs/file.zig +++ b/zml2/io/vfs/file.zig @@ -1,12 +1,72 @@ +const builtin = @import("builtin"); const std = @import("std"); const log = std.log.scoped(.@"zml/io/vfs/file"); +pub const DirectIoError = error{ + UnsupportedPlatform, + MisalignedBuffer, + BufferTooSmall, + UnexpectedFcntlResult, +} || std.posix.FcntlError; + +fn canUseDirectIO() bool { + if (builtin.target.os.tag == .linux) { + return @hasField(std.posix.O, "DIRECT"); + } + return false; +} + +fn useDirectIO(file: std.Io.File) DirectIoError!bool { + if (canUseDirectIO()) { + const flags = try std.posix.fcntl(file.handle, std.posix.F.GETFL, 0); + const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); + return (flags & direct_flag) != 0; + } else { + return DirectIoError.UnsupportedPlatform; + } +} + +fn switchToBufferedIO(file: std.fs.File) DirectIoError!void { + if (canUseDirectIO()) { + const flags = try std.posix.fcntl(file.handle, std.posix.F.GETFL, 0); + const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); + if ((flags & direct_flag) == 0) return; + + const result = try std.posix.fcntl(file.handle, std.posix.F.SETFL, flags & ~@as(c_uint, @bitCast(@as(u32, @intCast(direct_flag))))); + if (result != 0) return DirectIoError.UnexpectedFcntlResult; + } else { + return DirectIoError.UnsupportedPlatform; + } +} + +fn switchToDirectIO(file: std.Io.File) DirectIoError!void { + if (canUseDirectIO()) { + const flags = try std.posix.fcntl(file.handle, std.posix.F.GETFL, 0); + const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); + + const result = try std.posix.fcntl(file.handle, std.posix.F.SETFL, flags | direct_flag); + if (result != 0) return DirectIoError.UnexpectedFcntlResult; + } else { + return DirectIoError.UnsupportedPlatform; + } +} + pub const File = struct { + pub const Config = struct { + direct_io: bool = false, + direct_io_alignment: std.mem.Alignment = .fromByteUnits(4 * 1024), + }; + + allocator: std.mem.Allocator, + direct_io_map: std.AutoHashMapUnmanaged(std.Io.File.Handle, std.Io.File.Handle), + mutex: std.Io.Mutex, + config: Config, + inner: std.Io, vtable: std.Io.VTable, - pub fn init(inner: std.Io) File { + pub fn init(allocator: std.mem.Allocator, inner: std.Io, config: Config) File { var vtable = inner.vtable.*; vtable.dirMake = dirMake; vtable.dirMakePath = dirMakePath; @@ -15,14 +75,25 @@ pub const File = struct { vtable.dirAccess = dirAccess; vtable.dirCreateFile = dirCreateFile; vtable.dirOpenFile = dirOpenFile; + vtable.fileClose = fileClose; vtable.dirOpenDir = dirOpenDir; + vtable.fileReadPositional = fileReadPositional; + vtable.fileReadStreaming = fileReadStreaming; return .{ + .allocator = allocator, + .direct_io_map = .{}, + .mutex = .init, + .config = config, .inner = inner, .vtable = vtable, }; } + pub fn deinit(self: *File) void { + self.direct_io_map.deinit(self.allocator); + } + pub fn io(self: *File) std.Io { return .{ .userdata = self, @@ -37,6 +108,51 @@ pub const File = struct { return path; } + fn canOpenWithDirectIO(self: *File, file: std.Io.File, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!bool { + const file_stat = self.inner.vtable.fileStat(self.inner.userdata, file) catch |err| switch (err) { + else => return std.Io.File.OpenError.Unexpected, + }; + + return self.config.direct_io and flags.mode == .read_only and file_stat.size >= self.config.direct_io_alignment.toByteUnits(); + } + + fn innerFile( + self: *File, + file: std.Io.File, + buffers: [][]u8, + position: u64, + ) std.Io.File { + const alignment_bytes: usize = self.config.direct_io_alignment.toByteUnits(); + var buffers_aligned = true; + + for (buffers) |buf| { + const ptr_addr = @intFromPtr(buf.ptr); + if (!std.mem.isAligned(ptr_addr, alignment_bytes) or + !std.mem.isAligned(buf.len, alignment_bytes) or + (buf.len < alignment_bytes)) + { + buffers_aligned = false; + break; + } + } + + const pos_aligned = std.mem.isAligned(@as(usize, position), alignment_bytes); + + self.mutex.lockUncancelable(self.inner); + const direct_io_handle = self.direct_io_map.get(file.handle); + self.mutex.unlock(self.inner); + + return if (direct_io_handle) |handle| blk: { + if (buffers_aligned and pos_aligned) { + break :blk std.Io.File{ .handle = handle }; + } else { + break :blk file; + } + } else blk: { + break :blk file; + }; + } + fn dirMake( userdata: ?*anyopaque, dir: std.Io.Dir, @@ -104,7 +220,48 @@ pub const File = struct { flags: std.Io.File.OpenFlags, ) std.Io.File.OpenError!std.Io.File { const self: *File = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirOpenFile(self.inner.userdata, dir, stripScheme(sub_path), flags); + const file = try self.inner.vtable.dirOpenFile(self.inner.userdata, dir, stripScheme(sub_path), flags); + + const use_direct_io = try self.canOpenWithDirectIO(file, flags); + errdefer file.close(self.inner); + + if (use_direct_io) { + const direct_io_file = try self.inner.vtable.dirOpenFile(self.inner.userdata, dir, stripScheme(sub_path), flags); + errdefer direct_io_file.close(self.inner); + + switchToDirectIO(direct_io_file) catch |err| { + log.err("Failed to switch to Direct I/O mode: {any}", .{err}); + return std.Io.File.OpenError.Unexpected; + }; + + self.mutex.lockUncancelable(self.inner); + defer self.mutex.unlock(self.inner); + + self.direct_io_map.put(self.allocator, file.handle, direct_io_file.handle) catch |err| { + log.err("Failed to insert Direct I/O file into map: {any}", .{err}); + return std.Io.File.OpenError.Unexpected; + }; + } + + return file; + } + + fn fileClose(userdata: ?*anyopaque, file: std.Io.File) void { + const self: *File = @ptrCast(@alignCast(userdata orelse unreachable)); + + self.mutex.lockUncancelable(self.inner); + const direct_io_handle = self.direct_io_map.fetchRemove(file.handle); + self.mutex.unlock(self.inner); + + if (direct_io_handle) |handle| { + const direct_io_file: std.Io.File = .{ + .handle = handle.value, + }; + + self.inner.vtable.fileClose(self.inner.userdata, direct_io_file); + } + + return self.inner.vtable.fileClose(self.inner.userdata, file); } fn dirOpenDir( @@ -116,4 +273,28 @@ pub const File = struct { const self: *File = @ptrCast(@alignCast(userdata orelse unreachable)); return self.inner.vtable.dirOpenDir(self.inner.userdata, dir, stripScheme(sub_path), options); } + + fn fileReadPositional( + userdata: ?*anyopaque, + file: std.Io.File, + buffer: [][]u8, + position: u64, + ) std.Io.File.ReadPositionalError!usize { + const self: *File = @ptrCast(@alignCast(userdata orelse unreachable)); + const inner_file = self.innerFile(file, buffer, position); + + return self.inner.vtable.fileReadPositional(self.inner.userdata, inner_file, buffer, position); + } + + fn fileReadStreaming( + userdata: ?*anyopaque, + file: std.Io.File, + buffer: [][]u8, + ) std.Io.File.Reader.Error!usize { + const self: *File = @ptrCast(@alignCast(userdata orelse unreachable)); + const position = std.posix.lseek_CUR_get(file.handle) catch return std.Io.File.Reader.Error.Unexpected; + const inner_file = self.innerFile(file, buffer, position); + + return self.inner.vtable.fileReadStreaming(self.inner.userdata, inner_file, buffer); + } }; diff --git a/zml2/main_io.zig b/zml2/main_io.zig index 50b4deb4..c8e4d113 100644 --- a/zml2/main_io.zig +++ b/zml2/main_io.zig @@ -19,7 +19,7 @@ pub fn main() !void { const allocator = debug_allocator.allocator(); - // const allocator = std.heap.c_allocator; + // const allocator = std.heap.smp_allocator; var threaded: std.Io.Threaded = .init(allocator); defer threaded.deinit(); @@ -44,7 +44,15 @@ pub fn main() !void { try http_client.initDefaultProxies(allocator); defer http_client.deinit(); - var vfs_file: zml.io.VFS.File = .init(threaded.io()); + var vfs_file: zml.io.VFS.File = .init( + allocator, + threaded.io(), + .{ + .direct_io = std.process.hasEnvVarConstant("DIRECT"), + .direct_io_alignment = .fromByteUnits(4 * 1024), + }, + ); + defer vfs_file.deinit(); var vfs_http: zml.io.VFS.HTTP = try .init(allocator, threaded.io(), &http_client, .{}); defer vfs_http.deinit(); @@ -88,21 +96,70 @@ pub fn main() !void { break :blk default_uri; }; + // { + // // const initial_path = "/Users/hugo/Developer/Llama-3.1-8B-Instruct/Llama-3.1-8B-Instruct"; + // // const initial_path = "file:///Users/hugo/Developer/Llama-3.1-8B-Instruct/Llama-3.1-8B-Instruct"; + // const initial_path = "https://storage.googleapis.com/zig-vfs/Llama-3.1-8B-Instruct/Llama-3.1-8B-Instruct"; + + // const llama_dir = try std.Io.Dir.openDir(.cwd(), io, initial_path, .{}); + // defer llama_dir.close(io); + + // const filepath = "../model.safetensors.index.json"; + + // const index_file = try llama_dir.openFile(io, filepath, .{}); + // defer index_file.close(io); + + // const stat = try index_file.stat(io); + // log.info("Opened local index file with size: {d} bytes", .{stat.size}); + // } + { - // const initial_path = "/Users/hugo/Developer/Llama-3.1-8B-Instruct/Llama-3.1-8B-Instruct"; - // const initial_path = "file:///Users/hugo/Developer/Llama-3.1-8B-Instruct/Llama-3.1-8B-Instruct"; - const initial_path = "https://storage.googleapis.com/zig-vfs/Llama-3.1-8B-Instruct/Llama-3.1-8B-Instruct"; + const repo = try std.Io.Dir.openDir(.cwd(), io, "file:///home/hugo/Llama-3.1-8B-Instruct/", .{}); + defer repo.close(io); - const llama_dir = try std.Io.Dir.openDir(.cwd(), io, initial_path, .{}); - defer llama_dir.close(io); + const file = try repo.openFile(io, "model-00001-of-00004.safetensors", .{}); + defer file.close(io); - const filepath = "../model.safetensors.index.json"; + const buf_size = 256 * 1024 * 1024; + const buf = try allocator.alignedAlloc(u8, .fromByteUnits(4 * 1024), buf_size); + defer allocator.free(buf); - const index_file = try llama_dir.openFile(io, filepath, .{}); - defer index_file.close(io); + var sha256: std.crypto.hash.sha2.Sha256 = .init(.{}); + const compute_sha = false; - const stat = try index_file.stat(io); - log.info("Opened local index file with size: {d} bytes", .{stat.size}); + var total_read: usize = 1 * 1024; + var bufs = [_][]u8{buf}; + + var timer: std.time.Timer = try .start(); + defer { + const elapsed = timer.read(); + log.info("File read in {d} ms", .{elapsed / std.time.ns_per_ms}); + } + + while (true) { + const n = try io.vtable.fileReadStreaming(io.userdata, file, &bufs); + if (n == 0) break; + if (compute_sha) sha256.update(buf[0..n]); + total_read += n; + } + + const elapsed = timer.read(); + const read_mb = @as(f64, @floatFromInt(total_read)) / (1024.0 * 1024.0); + const read_time_s = @as(f64, @floatFromInt(elapsed)) / @as(f64, std.time.ns_per_s); + const throughput_mb_s = if (read_time_s > 0) read_mb / read_time_s else 0; + const throughput_gbps = if (read_time_s > 0) (read_mb * 8.0) / 1024.0 / read_time_s else 0; + log.info("Read {d:.2} MB in {d:.2} s = {d:.2} MB/s | {d:.2} Gbps", .{ + read_mb, + read_time_s, + throughput_mb_s, + throughput_gbps, + }); + + if (compute_sha) { + var hash: [32]u8 = undefined; + sha256.final(&hash); + log.info("SHA256: {x}", .{hash}); + } } { @@ -123,40 +180,6 @@ pub fn main() !void { log.info("Parsed {d} tensors", .{registry.tensors.count()}); - // { - // const tensor_name = "model.layers.31.mlp.down_proj.weight"; - // // const tensor_name = "model.layers.31.input_layernorm.weight"; - // // const tensor_name = "lm_head.weight"; - // const tensor = registry.tensors.get(tensor_name) orelse return error.TensorNotFound; - // log.info("Reading {f}...", .{tensor}); - - // const tensor_reader_buf = try allocator.alloc(u8, tensor.byteSize()); - // defer allocator.free(tensor_reader_buf); - - // const writer_buffer = try allocator.alloc(u8, tensor.byteSize()); - // defer allocator.free(writer_buffer); - - // var tensor_reader = try registry.reader(io, &vfs, tensor_name, tensor_reader_buf); - // defer tensor_reader.deinit(); - - // var writer: std.Io.Writer = .fixed(writer_buffer); - - // var timer_read: std.time.Timer = try .start(); - - // const read = try tensor_reader.interface.streamRemaining(&writer); - - // const elapsed = timer_read.read(); - // const read_mb = @as(f64, @floatFromInt(read)) / (1024.0 * 1024.0); - // const read_time_s = @as(f64, @floatFromInt(elapsed)) / @as(f64, std.time.ns_per_s); - // const throughput_gbps = (read_mb * 8.0) / 1024.0 / read_time_s; - - // log.info("Read completed in {d} ms | {d:.2} MB/s | {d:.2} Gbps", .{ - // elapsed / std.time.ns_per_ms, - // read_mb / read_time_s, - // throughput_gbps, - // }); - // } - { var err: ?anyerror = null; From 3581770d26189cbd51eb171dced5636edc000234 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 18 Dec 2025 16:07:17 +0100 Subject: [PATCH 152/881] Fix tests --- mlir/mlir.zig | 2 +- pjrt/pjrt.zig | 4 +-- zml2/exe.zig | 2 +- zml2/testing.zig | 60 +----------------------------------- zml2/tokenizer/homemade.zig | 8 +---- zml2/tokenizer/tokenizer.zig | 18 +++++------ zml2/zml.zig | 2 +- 7 files changed, 16 insertions(+), 80 deletions(-) diff --git a/mlir/mlir.zig b/mlir/mlir.zig index b90e3f3e..79177cf7 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -6,7 +6,7 @@ const stdx = @import("stdx"); const log = std.log.scoped(.mlir); test { - std.testing.refAllDeclsRecursive(@This()); + std.testing.refAllDecls(@This()); } pub const Error = error{ diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 0093681c..87a0fa19 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -280,12 +280,12 @@ pub const ShapeSpec = extern struct { }; } - pub fn dims(self: ShapeSpec) []usize { + pub fn dims(self: ShapeSpec) []const i64 { return self.inner.dims[0..self.inner.num_dims]; } pub fn bufferType(self: ShapeSpec) BufferType { - return @enumFromInt(self.inner.buffer_type); + return @enumFromInt(self.inner.element_type); } }; diff --git a/zml2/exe.zig b/zml2/exe.zig index 735b3213..47a4f39c 100644 --- a/zml2/exe.zig +++ b/zml2/exe.zig @@ -193,7 +193,7 @@ pub const Exe = struct { buffer.* = Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], shards.constSlice()); context_.current_index += 1; } - }.cb, &context, v); + }.cb, &context, &v); } }; diff --git a/zml2/testing.zig b/zml2/testing.zig index 551b6ed3..0a5f904c 100644 --- a/zml2/testing.zig +++ b/zml2/testing.zig @@ -118,64 +118,6 @@ fn BufferizedWithArgs(comptime T: type) type { return zml.meta.MapType(zml.Tensor, zml.Buffer).map(T); } -/// Compile a function and immediatly call it with the given buffers. -/// The compiled module is discarded after the call. -/// Useful during testing when a module is typically called only once. -/// -/// Note: `func` needs explicit types on all parameters. -/// To test a function with `anytype` (typically for tagged API), you need to create a specialized version of it with specific types. -pub fn compileAndCall( - io: std.Io, - platform: zml.Platform, - func: anytype, - buffer_and_args: BufferizedWithArgs(stdx.meta.FnArgs(func)), -) !zml.Bufferized(stdx.meta.FnResult(func)) { - // This simplify test API and also ensure this fn isn't used outside of tests. - const allocator = std.testing.allocator; - - var shape_list: std.array_list.Managed(zml.Shape) = .init(allocator); - defer shape_list.deinit(); - const LocalContext1 = struct { - shape_list: *std.array_list.Managed(zml.Shape), - }; - var context1: LocalContext1 = .{ .shape_list = &shape_list }; - try zml.meta.visit(struct { - fn cb(context_: *LocalContext1, buffer: *const zml.Buffer) !void { - try context_.shape_list.append(buffer.shape()); - } - }.cb, &context1, &buffer_and_args); - - var func_args: stdx.meta.FnArgs(func) = undefined; - const LocalContext2 = struct { - shape_list: *std.array_list.Managed(zml.Shape), - index: usize = 0, - }; - var context2: LocalContext2 = .{ .shape_list = &shape_list }; - try zml.meta.visit(struct { - fn cb(context_: *LocalContext2, tensor: *zml.Tensor) !void { - tensor.* = .fromShape(context_.shape_list.items[context_.index]); - context_.index += 1; - } - }.cb, &context2, &func_args); - - var exe = try zml.module.compile(allocator, io, func, func_args, platform); - defer exe.deinit(); - - var args = try exe.args(allocator); - defer args.deinit(allocator); - - args.set(buffer_and_args); - - var results = try exe.results(allocator); - defer results.deinit(allocator); - - exe.call(args, &results, io); - - var result: zml.Bufferized(stdx.meta.FnResult(func)) = undefined; - results.fill(&result); - return result; -} - /// Automatically calls the executable with the given arguments, taking care of arguments and results allocation. /// This helper can be used in tests to make the code a bit less verbose. /// It doesn't handle pointers in inputs/outputs structs. @@ -190,7 +132,7 @@ pub fn autoCall(allocator: std.mem.Allocator, io: std.Io, exe: *const zml.exe.Ex exe.call(args, &results, io); var output: zml.Bufferized(stdx.meta.FnResult(func)) = undefined; - results.fill(&output); + results.fill(.{&output}); return output; } diff --git a/zml2/tokenizer/homemade.zig b/zml2/tokenizer/homemade.zig index 95274bc8..bd16d183 100644 --- a/zml2/tokenizer/homemade.zig +++ b/zml2/tokenizer/homemade.zig @@ -10,12 +10,6 @@ const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/tokenizer"); -test { - std.testing.refAllDecls(@This()); - std.testing.refAllDecls(Normalizer); - std.testing.refAllDecls(Tokenizer); -} - /// Byte Pair Encoding tokenizer generally used for LLM. pub const Tokenizer = struct { tokens: [][]const u8, @@ -479,7 +473,7 @@ pub const Decoder = struct { } pub fn string(self: *const Decoder) []const u8 { - return self.current_string; + return self.current_string orelse ""; } pub fn next(self: *Decoder, token_id: u32) !?[]const u8 { diff --git a/zml2/tokenizer/tokenizer.zig b/zml2/tokenizer/tokenizer.zig index bb80ae4a..1ba1d1d6 100644 --- a/zml2/tokenizer/tokenizer.zig +++ b/zml2/tokenizer/tokenizer.zig @@ -7,9 +7,6 @@ pub const homemade = @import("homemade.zig"); test { std.testing.refAllDecls(@This()); - std.testing.refAllDecls(hftokenizers); - std.testing.refAllDecls(sentencepiece); - std.testing.refAllDecls(homemade); } const Tokenizers = enum { @@ -72,17 +69,17 @@ pub const Tokenizer = union(Tokenizers) { }; } - pub fn string(self: Decoder) []const u8 { + pub fn string(self: *Decoder) []const u8 { return switch (self.*) { inline else => |v| v.string(), }; } - pub fn ids(self: Decoder) []u32 { - return switch (self.*) { - inline else => |v| v.ids(), - }; - } + //pub fn ids(self: *Decoder) []u32 { + // return switch (self.*) { + // inline else => |v| v.ids(), + // }; + //} pub fn next(self: *Decoder, token_id: u32) !?[]const u8 { return switch (self.*) { @@ -136,3 +133,6 @@ pub const Tokenizer = union(Tokenizers) { }; } }; + +test {} +test {} diff --git a/zml2/zml.zig b/zml2/zml.zig index 5f56360b..1df477a5 100644 --- a/zml2/zml.zig +++ b/zml2/zml.zig @@ -82,5 +82,5 @@ pub fn Bufferized(comptime T: type) type { } test "zml" { - std.testing.refAllDeclsRecursive(@This()); + std.testing.refAllDecls(@This()); } From f46e649a290d2b014e76d8561509a0bf01c5fbb9 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 19 Dec 2025 11:11:30 +0100 Subject: [PATCH 153/881] Small fixe on LLama --- examples/llama/llama.zig | 17 ++++++----------- examples/llama/main.zig | 11 ++++++----- zml2/module.zig | 4 ++-- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 442544e3..5d89e100 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -51,7 +51,7 @@ pub const LlamaLM = struct { return .{ .lm_head = lm_head, - .model = try .init(allocator, store.withPrefix("model"), config, options), + .model = try .init(allocator, store.withPrefix("model"), config), .gen_opts = options.sampling_strategy orelse .{}, .config = config, }; @@ -120,10 +120,6 @@ pub const LlamaLM = struct { const next_tokens, const new_rng = zml.nn.sampleTokens(logits, opts, rng); return .{ next_tokens, new_rng }; } - - pub fn increment(_: u8, token_index: Tensor) Tensor { - return token_index.addConstant(1).reuseBuffer(token_index); - } }; pub const Llama = struct { @@ -139,12 +135,12 @@ pub const Llama = struct { .freq_base = 10_000, }, - pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: LlamaLM.Config, options: LlamaLM.Options) !Llama { + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: LlamaLM.Config) !Llama { const layers = try allocator.alloc(TransformerLayer, config.num_hidden_layers); errdefer allocator.free(layers); for (layers, 0..) |*layer, i| { - layer.* = try .init(store.withPrefix("layers").withLayer(i), config, options); + layer.* = try .init(store.withPrefix("layers").withLayer(i), config); } return .{ @@ -208,10 +204,10 @@ pub const TransformerLayer = struct { post_attention_layernorm: RmsNorm, mlp: Mlp, - pub fn init(store: zml.io.TensorStore.View, config: LlamaLM.Config, options: LlamaLM.Options) !TransformerLayer { + pub fn init(store: zml.io.TensorStore.View, config: LlamaLM.Config) !TransformerLayer { return .{ .input_layernorm = .init(store.withPrefix("input_layernorm"), config.rms_norm_eps), - .self_attn = try .init(store.withPrefix("self_attn"), config, options), + .self_attn = try .init(store.withPrefix("self_attn"), config), .post_attention_layernorm = .init(store.withPrefix("post_attention_layernorm"), config.rms_norm_eps), .mlp = .init(store.withPrefix("mlp")), }; @@ -317,8 +313,7 @@ pub const SelfAttn = struct { return .init(store.createTensorWithTags("weight", .{ .dout, .d }), store.maybeCreateTensorWithTags("bias", .{.dout}), .d); } - pub fn init(store: zml.io.TensorStore.View, config: LlamaLM.Config, options: LlamaLM.Options) !SelfAttn { - _ = options; // autofix + pub fn init(store: zml.io.TensorStore.View, config: LlamaLM.Config) !SelfAttn { return .{ .q_proj = initProj(store.withPrefix("q_proj")), .k_proj = initProj(store.withPrefix("k_proj")), diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 120f8b97..1326618a 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -109,7 +109,7 @@ pub fn generateText( prefill_exe.call(prefill_args, &prefill_results, io); - prefill_results.fill(&.{ &prefill_tokens_buffer, kv_cache_buffers, &rng_buffers }); + prefill_results.fill(.{ &prefill_tokens_buffer, kv_cache_buffers, &rng_buffers }); try prefill_tokens_buffer.toSlice(prefill_tokens_slice, io); generated_token_slice.items(u32)[0] = prefill_tokens_slice.items(u32)[prompt_tok.len - 1]; } @@ -161,7 +161,7 @@ pub fn generateText( decode_exe.call(decode_args, &decode_results, io); - decode_results.fill(&.{ ¤t_token_buffer, kv_cache_buffers, &rng_buffers }); + decode_results.fill(.{ ¤t_token_buffer, kv_cache_buffers, &rng_buffers }); // extract the generated token from the buffer try current_token_buffer.toSlice(generated_token_slice, io); @@ -283,7 +283,7 @@ pub fn main() !void { const llama_options: llama.LlamaLM.Options = .{ .max_seq_len = seq_len, .sampling_strategy = .{ - .topk = 1, + .topk = 2, .temperature = 1.0, }, }; @@ -346,8 +346,9 @@ pub fn main() !void { // Unbuffered writing of the tokens to stdout. var stdout = std.fs.File.stdout().writer(&.{}); - //const seed: u128 = cli.args.seed orelse @bitCast(std.time.nanoTimestamp()); - const seed: u128 = cli.args.seed orelse 0; + const now = try std.Io.Clock.now(.boot, io); + std.log.info("now: {}", .{now.nanoseconds}); + const seed: u128 = cli.args.seed orelse @intCast(now.nanoseconds); const skip_llama3_encoding = cli.args.@"no-llama3" orelse false; try generateText( diff --git a/zml2/module.zig b/zml2/module.zig index 337d5d03..0659d923 100644 --- a/zml2/module.zig +++ b/zml2/module.zig @@ -186,8 +186,8 @@ pub fn compile(allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), io, platform, compilation_context.module, null) catch unreachable; - log.warn("******** ZML generated MLIR ********", .{}); - log.warn("{f}", .{compilation_context.module.operation()}); + log.debug("******** ZML generated MLIR ********", .{}); + log.debug("{f}", .{compilation_context.module.operation()}); const num_devices = sharding.num_partitions * sharding.num_replicas; const exe = try Exe.init(allocator, platform, loaded_executable, result.input_info.shapes, result.output_info.shapes, num_devices); From ec52dc22f1610d6b6d51e65b585dabbb5314e254 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 19 Dec 2025 11:16:10 +0100 Subject: [PATCH 154/881] Small fix in File VFS DIRECT flag is not supported on macOS --- zml2/io/vfs/file.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml2/io/vfs/file.zig b/zml2/io/vfs/file.zig index d7fb94a3..31a7b873 100644 --- a/zml2/io/vfs/file.zig +++ b/zml2/io/vfs/file.zig @@ -41,7 +41,7 @@ fn switchToBufferedIO(file: std.fs.File) DirectIoError!void { } fn switchToDirectIO(file: std.Io.File) DirectIoError!void { - if (canUseDirectIO()) { + if (builtin.os.tag == .linux and canUseDirectIO()) { const flags = try std.posix.fcntl(file.handle, std.posix.F.GETFL, 0); const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); From 3988755c73d085df4d20faaaf1c9f0f8b3e6c41a Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 19 Dec 2025 11:16:47 +0100 Subject: [PATCH 155/881] Fix VFS.File init in llama --- examples/llama/main.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 1326618a..6ee1253f 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -185,7 +185,7 @@ pub fn main() !void { var threaded: std.Io.Threaded = .init(allocator); defer threaded.deinit(); - var vfs_file: zml.io.VFS.File = .init(threaded.io()); + var vfs_file: zml.io.VFS.File = .init(allocator, threaded.io(), .{}); var http_client: std.http.Client = .{ .allocator = allocator, From e17f07e1fc52d79555b1bcc579e6ff561ba81a9f Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 19 Dec 2025 12:48:00 +0100 Subject: [PATCH 156/881] Use correct default parameter for TopK --- zml2/tensor.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 9be42366..8d850b8f 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -2965,7 +2965,7 @@ pub const Tensor = struct { } } - pub const TopKOpts = SortOpts; + pub const TopKOpts = struct { descending: bool = true }; /// Returns a Tensor representing the result of Top-K over the given axis. pub fn topK(self: Tensor, named_axis_: anytype, k: u32, opts: TopKOpts) SortRes { @@ -2979,7 +2979,7 @@ pub const Tensor = struct { }, else => stdx.debug.compileError(err_msg, .{}), }; - var result = self.sort(a, opts); + var result = self.sort(a, .{ .descending = opts.descending }); result.values = result.values.slice1d(a, .{ .end = k }); result.indices = result.indices.slice1d(a, .{ .end = k }); if (has_name) |new_name| { From 14d8f2359b9047b1fece7240f3b5df6cfee6605e Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Fri, 19 Dec 2025 14:00:26 +0100 Subject: [PATCH 157/881] io: HF VFS v2 + alignment with HTTP VFS + fix for File VFS (#368) --- zml2/io/vfs/file.zig | 15 +- zml2/io/vfs/hf.zig | 1590 +++++++++++++++++++++++++----------------- zml2/io/vfs/http.zig | 274 ++++---- zml2/main_io.zig | 229 +++--- zml2/safetensors.zig | 220 +++--- 5 files changed, 1368 insertions(+), 960 deletions(-) diff --git a/zml2/io/vfs/file.zig b/zml2/io/vfs/file.zig index 31a7b873..b0f3d757 100644 --- a/zml2/io/vfs/file.zig +++ b/zml2/io/vfs/file.zig @@ -18,7 +18,7 @@ fn canUseDirectIO() bool { } fn useDirectIO(file: std.Io.File) DirectIoError!bool { - if (canUseDirectIO()) { + if (comptime canUseDirectIO()) { const flags = try std.posix.fcntl(file.handle, std.posix.F.GETFL, 0); const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); return (flags & direct_flag) != 0; @@ -28,7 +28,7 @@ fn useDirectIO(file: std.Io.File) DirectIoError!bool { } fn switchToBufferedIO(file: std.fs.File) DirectIoError!void { - if (canUseDirectIO()) { + if (comptime canUseDirectIO()) { const flags = try std.posix.fcntl(file.handle, std.posix.F.GETFL, 0); const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); if ((flags & direct_flag) == 0) return; @@ -229,9 +229,14 @@ pub const File = struct { const direct_io_file = try self.inner.vtable.dirOpenFile(self.inner.userdata, dir, stripScheme(sub_path), flags); errdefer direct_io_file.close(self.inner); - switchToDirectIO(direct_io_file) catch |err| { - log.err("Failed to switch to Direct I/O mode: {any}", .{err}); - return std.Io.File.OpenError.Unexpected; + switchToDirectIO(direct_io_file) catch |err| switch (err) { + DirectIoError.UnsupportedPlatform => { + log.debug("Direct I/O is not supported on this platform", .{}); + }, + else => { + log.err("Failed to switch to Direct I/O mode: {any}", .{err}); + return std.Io.File.OpenError.Unexpected; + }, }; self.mutex.lockUncancelable(self.inner); diff --git a/zml2/io/vfs/hf.zig b/zml2/io/vfs/hf.zig index 28fcfb1c..47683ba7 100644 --- a/zml2/io/vfs/hf.zig +++ b/zml2/io/vfs/hf.zig @@ -1,8 +1,422 @@ const std = @import("std"); +const json = @import("std").json; const log = std.log.scoped(.@"zml/io/vfs/hf"); -// todo : refactor this code to align with http vfs +const HFApi = struct { + pub const Error = error{ + MalformedURI, + RequestFailed, + ResponseParsingFailed, + } || std.mem.Allocator.Error; + + const PickleImport = struct { + module: []const u8, + name: []const u8, + safety: []const u8, + }; + + const Scan = struct { + status: ?[]const u8 = null, + message: ?[]const u8 = null, + reportLink: ?[]const u8 = null, + reportLabel: ?[]const u8 = null, + pickleImports: ?[]PickleImport = null, + version: ?[]const u8 = null, + }; + + const SecurityFileStatus = struct { + status: ?[]const u8 = null, + jFrogScan: ?Scan = null, + protectAiScan: ?Scan = null, + avScan: ?Scan = null, + pickleImportScan: ?Scan = null, + virusTotalScan: ?Scan = null, + }; + + const LastCommit = struct { + id: []const u8, + title: []const u8, + date: []const u8, + }; + + const Lfs = struct { + oid: []const u8, + size: u64, + pointerSize: u64, + }; + + pub const TreeEntry = struct { + type: []const u8, + oid: []const u8, + path: []const u8, + size: u64, + lfs: ?Lfs = null, + lastCommit: ?LastCommit = null, + xetHash: ?[]const u8 = null, + securityFileStatus: ?SecurityFileStatus = null, + }; + + pub const TreeSize = struct { + path: []const u8, + size: u64, + }; + + pub const Model = struct { + namespace: []const u8, + repo: []const u8, + revision: ?[]const u8, + }; + + pub fn modelFromUri(uri: []const u8) Error!Model { + const parsed = std.Uri.parse(uri) catch { + log.err("Failed to parse HF model URI: {s}", .{uri}); + return Error.MalformedURI; + }; + + const namespace = parsed.host.?.percent_encoded; + + var path = parsed.path.percent_encoded; + // strip leading slash + if (path.len > 0 and path[0] == '/') path = path[1..]; + // strip trailing slash + if (path.len > 0 and path[path.len - 1] == '/') path = path[0 .. path.len - 1]; + + var it = std.mem.splitSequence(u8, path, "/"); + const repo_and_rev = it.next(); + + if (repo_and_rev == null) { + log.err("Malformed HF model URI, missing repo: {s}", .{uri}); + return Error.MalformedURI; + } + + // repo may contain an optional @revision suffix + var rev: ?[]const u8 = null; + var repo = repo_and_rev.?; + const at_index = std.mem.indexOf(u8, repo_and_rev.?, "@"); + if (at_index) |idx| { + repo = repo_and_rev.?[0..idx]; + rev = repo_and_rev.?[idx + 1 ..]; + if (rev.?.len == 0) rev = null; + } + + return .{ + .namespace = namespace, + .repo = repo, + .revision = rev, + }; + } + + pub fn filePathFromUri(uri: []const u8) Error![]const u8 { + const parsed = std.Uri.parse(uri) catch { + log.err("Failed to parse HF file URI: {s}", .{uri}); + return Error.MalformedURI; + }; + + var path = parsed.path.percent_encoded; + // strip leading slash + if (path.len > 0 and path[0] == '/') path = path[1..]; + // strip trailing slash + if (path.len > 0 and path[path.len - 1] == '/') path = path[0 .. path.len - 1]; + + // Remove the first segment (repo[@rev]) + var it = std.mem.splitSequence(u8, path, "/"); + _ = it.next(); // skip repo[@rev] + const rest = it.rest(); + path = rest; + + return path; + } + + pub fn isModelPath(uri: []const u8) bool { + // Determine whether the directory URI represents an HF repo root. + // Valid root examples: + // hf://user/repo + // hf://user/repo/ + // hf://user/repo@rev + // hf://user/repo@rev/ + // Invalid examples: + // hf:// + // hf://user + // hf://user/ + // hf://user/repo/some/path + + const parsed_uri = std.Uri.parse(uri) catch return false; + + // Must be hf scheme with a host + if (!std.mem.eql(u8, parsed_uri.scheme, "hf")) return false; + if (parsed_uri.host == null) return false; + + var path = parsed_uri.path.percent_encoded; + + // No path or root-only path ("/") are not repo roots + if (path.len == 0) return false; + if (path.len == 1 and path[0] == '/') return false; + + // Strip leading slash if present + if (path[0] == '/') path = path[1..]; + + // Strip trailing slash if present + if (path.len > 0 and path[path.len - 1] == '/') path = path[0 .. path.len - 1]; + + // After trimming, there must be exactly one segment (no additional '/') + if (path.len == 0) return false; + + // Check for additional '/' in the path + var i: usize = 0; + while (i < path.len) : (i += 1) { + if (path[i] == '/') return false; + } + + return true; + } + + pub fn fetchModelSize( + allocator: std.mem.Allocator, + client: *std.http.Client, + extra_headers: []std.http.Header, + repo_uri: []const u8, + ) Error!json.Parsed(TreeSize) { + const model = try HFApi.modelFromUri(repo_uri); + const rev = model.revision orelse blk: { + log.debug("No revision specified in repo URI; defaulting to 'main'", .{}); + break :blk "main"; + }; + + var redirect_buffer: [std.fs.max_path_bytes]u8 = undefined; + var url_buf: [std.fs.max_path_bytes]u8 = undefined; + const api_url = std.fmt.bufPrint( + &url_buf, + "https://huggingface.co/api/models/{s}/{s}/treesize/{s}/", + .{ model.namespace, model.repo, rev }, + ) catch { + log.err("Failed to format HF model size API URL for repo URI {s}", .{repo_uri}); + return Error.OutOfMemory; + }; + + var req = try allocator.create(std.http.Client.Request); + defer allocator.destroy(req); + + const api_uri = std.Uri.parse(api_url) catch { + log.err("Failed to parse HF model size API URL {s}", .{api_url}); + return Error.MalformedURI; + }; + + req.* = client.request(.GET, api_uri, .{ + .headers = .{ .accept_encoding = .{ .override = "identity" } }, + .extra_headers = extra_headers, + }) catch { + log.err("Failed to create HTTP request for HF model size API URL {s}", .{api_url}); + return Error.RequestFailed; + }; + defer req.deinit(); + + req.sendBodiless() catch { + log.err("Failed to send HTTP request for HF model size API URL {s}", .{api_url}); + return Error.RequestFailed; + }; + + var resp = req.receiveHead(&redirect_buffer) catch { + log.err("Failed to receive HTTP response for HF model size API URL {s}", .{api_url}); + return Error.RequestFailed; + }; + + if (resp.head.status != .ok) { + log.err("Failed to fetch HF model size, got {s} for {s}", .{ @tagName(resp.head.status), api_url }); + return Error.RequestFailed; + } + + const content_length = resp.head.content_length orelse { + log.err("Missing Content-Length in HF model size response for {s}", .{api_url}); + return Error.ResponseParsingFailed; + }; + + const body = resp.reader(&.{}).readAlloc(allocator, content_length) catch { + log.err("Failed to read HF model size response body for {s} of content length {d}", .{ api_url, content_length }); + return Error.ResponseParsingFailed; + }; + defer allocator.free(body); + + const parsed = json.parseFromSlice( + HFApi.TreeSize, + allocator, + body, + .{ .ignore_unknown_fields = true }, + ) catch { + log.err("Failed to parse HF model size response JSON for {s}", .{api_url}); + return Error.ResponseParsingFailed; + }; + + return parsed; + } +}; + +// todo: ref counters +const HFRecords = struct { + pub const Error = HFApi.Error || std.mem.Allocator.Error; + + const Mapping = std.StringArrayHashMapUnmanaged(HFApi.TreeEntry); + + arena: std.heap.ArenaAllocator, + mapping: Mapping, + + pub fn init(allocator: std.mem.Allocator) HFRecords { + return .{ + .arena = std.heap.ArenaAllocator.init(allocator), + .mapping = .{}, + }; + } + + pub fn get(self: *HFRecords, key: []const u8) ?HFApi.TreeEntry { + // Normalize key by stripping directory trailing slash if present + const normalized_key = if (key.len > 0 and key[key.len - 1] == '/') key[0 .. key.len - 1] else key; + + return self.mapping.get(normalized_key); + } + + pub fn containsPathPrefix(self: *HFRecords, prefix: []const u8) bool { + var it = self.mapping.iterator(); + while (it.next()) |kv| { + const key = kv.key_ptr.*; + if (key.len >= prefix.len and std.mem.startsWith(u8, key, prefix)) { + return true; + } + } + + return false; + } + + pub fn fetch( + self: *HFRecords, + client: *std.http.Client, + limit: usize, + extra_headers: []std.http.Header, + repo_uri: []const u8, + url: ?[]const u8, // used for pagination + ) Error!void { + const allocator = self.arena.allocator(); + + var url_buf: [std.fs.max_path_bytes]u8 = undefined; + var redirect_buffer: [std.fs.max_path_bytes]u8 = undefined; + + const api_url = url orelse blk: { + const model = try HFApi.modelFromUri(repo_uri); + const rev = model.revision orelse blk_: { + log.debug("No revision specified in repo URI; defaulting to 'main'", .{}); + break :blk_ "main"; + }; + + break :blk std.fmt.bufPrint( + &url_buf, + "https://huggingface.co/api/models/{s}/{s}/tree/{s}/?expand=true&recursive=true&limit={d}", + .{ model.namespace, model.repo, rev, limit }, + ) catch { + log.err("Failed to format HF repo tree API URL for repo URI {s}", .{repo_uri}); + return Error.OutOfMemory; + }; + }; + + const api_uri = std.Uri.parse(api_url) catch { + log.err("Failed to parse HF repo tree API URL {s}", .{api_url}); + return Error.MalformedURI; + }; + + var req = try allocator.create(std.http.Client.Request); + defer allocator.destroy(req); + + req.* = client.request(.GET, api_uri, .{ + .headers = .{ .accept_encoding = .{ .override = "identity" } }, + .extra_headers = extra_headers, + }) catch { + log.err("Failed to create HTTP request for HF repo tree API URL {s}", .{api_url}); + return Error.RequestFailed; + }; + defer req.deinit(); + + req.sendBodiless() catch { + log.err("Failed to send HTTP request for HF repo tree API URL {s}", .{api_url}); + return Error.RequestFailed; + }; + + var resp = req.receiveHead(&redirect_buffer) catch { + log.err("Failed to receive HTTP response for HF repo tree API URL {s}", .{api_url}); + return Error.RequestFailed; + }; + if (resp.head.status != .ok) { + log.err("Failed to fetch HF repo tree, got {s} for {s}", .{ @tagName(resp.head.status), api_url }); + return Error.RequestFailed; + } + + var next_url: ?[]const u8 = null; + var it = resp.head.iterateHeaders(); + while (it.next()) |hdr| { + if (std.mem.eql(u8, hdr.name, "Link") or std.mem.eql(u8, hdr.name, "link")) { + const hv = hdr.value; + var i: usize = 0; + while (i < hv.len) : (i += 1) { + const lt = std.mem.indexOf(u8, hv[i..], "<"); + if (lt == null) break; + const start = i + lt.? + 1; + const gt = std.mem.indexOf(u8, hv[start..], ">"); + if (gt == null) break; + const end = start + gt.?; + const url_slice = hv[start..end]; + const after = hv[end..]; + if (std.mem.indexOf(u8, after, "rel=\"next\"") != null) { + next_url = url_slice; + break; + } + i = end + 1; + } + } + } + + const content_length = resp.head.content_length orelse { + log.err("Missing Content-Length in HF repo tree response for {s}", .{api_url}); + return Error.ResponseParsingFailed; + }; + const body = resp.reader(&.{}).readAlloc(allocator, content_length) catch { + log.err("Failed to read HF repo tree response body for {s} of content length {d}", .{ api_url, content_length }); + return Error.ResponseParsingFailed; + }; + + const entries = json.parseFromSliceLeaky( + []HFApi.TreeEntry, + allocator, + body, + .{ .ignore_unknown_fields = true }, + ) catch { + log.err("Failed to parse HF repo tree response JSON for {s}", .{api_url}); + return Error.ResponseParsingFailed; + }; + + for (entries) |entry| { + try self.add(repo_uri, entry); + } + + if (next_url) |url_| { + try self.fetch(client, limit, extra_headers, repo_uri, url_); + } + } + + fn add(self: *HFRecords, repo_uri: []const u8, entry: HFApi.TreeEntry) Error!void { + const allocator = self.arena.allocator(); + + // Ensure keys are built from the canonical repo root (hf://namespace/repo[@rev]/) + const model = try HFApi.modelFromUri(repo_uri); + const base = if (model.revision) |rev| blk: { + break :blk try std.fmt.allocPrint(allocator, "hf://{s}/{s}@{s}/", .{ model.namespace, model.repo, rev }); + } else blk: { + break :blk try std.fmt.allocPrint(allocator, "hf://{s}/{s}/", .{ model.namespace, model.repo }); + }; + const key = try std.fmt.allocPrint(allocator, "{s}{s}", .{ base, entry.path }); + try self.mapping.put(allocator, key, entry); + } + + pub fn deinit(self: *HFRecords) void { + self.arena.deinit(); + } +}; pub const HF = struct { pub const Auth = union(enum) { @@ -27,8 +441,13 @@ pub const HF = struct { pub fn auto(allocator: std.mem.Allocator, io_: std.Io) AuthError!Auth { return fromHomeConfig(allocator, io_) catch |err| switch (err) { - AuthError.MissingHomePath => return fromEnv(allocator), - AuthError.MissingConfigFile => return fromEnv(allocator), + AuthError.MissingHomePath, AuthError.MissingConfigFile => return fromEnv(allocator) catch |e| switch (e) { + AuthError.MissingHFToken => { + log.warn("No Hugging Face authentication token found in environment or home config; proceeding without authentication.", .{}); + return .none; + }, + else => return e, + }, else => |e| return e, }; } @@ -68,60 +487,34 @@ pub const HF = struct { } }; - const Config = struct { - auth: Auth = .none, - range_size: u64 = 128 * 1024 * 1024, // todo: rename field - }; + const HandleId = i32; - allocator: std.mem.Allocator, - client: *std.http.Client, // todo: rename http_client - http_headers: []std.http.Header, - config: Config, - - file_handles: std.AutoHashMapUnmanaged(HandleId, *FileHandle), - dir_handles: std.AutoHashMapUnmanaged(HandleId, *DirHandle), - next_handle_id: HandleId, - mutex: std.Io.Mutex, - vtable: std.Io.VTable, - - pub const HandleId = i32; - const max_retries = 10; + pub const InitHandleError = error{ + PathTooLong, + PathUnresolvable, + FailedToFetchRemoteResource, + MissingRessource, + } || std.mem.Allocator.Error; pub const FileHandle = struct { - id: HandleId, allocator: std.mem.Allocator, - hf_path: []const u8, - pos: u64, - built_url: []const u8, - size: ?u64, - redirect_buffer: [1024 * 1024]u8, - request: ?*std.http.Client.Request, - response: ?std.http.Client.Response, - body_reader: ?*std.Io.Reader, - response_bytes_remaining: u64, - pub fn init(allocator: std.mem.Allocator, id: HandleId, hf_path: []const u8) !*FileHandle { - const handle = try allocator.create(FileHandle); - handle.* = .{ - .id = id, - .allocator = allocator, - .hf_path = try allocator.dupe(u8, hf_path), - .pos = 0, - .built_url = "", - .size = null, - .redirect_buffer = undefined, - .request = null, - .response = null, - .body_reader = null, - .response_bytes_remaining = 0, - }; - return handle; - } + uri: []const u8, + pos: u64, + size: u64, + redirect_buffer: [std.fs.max_path_bytes]u8 = undefined, + request: ?*std.http.Client.Request = null, + response: ?std.http.Client.Response = null, + body_reader: ?*std.Io.Reader = null, + + total_requests: u64 = 0, + total_latency_ns: u64 = 0, + min_latency_ns: u64 = std.math.maxInt(u64), + max_latency_ns: u64 = 0, pub fn deinit(self: *FileHandle) void { self.releaseRequest(); - self.allocator.free(self.hf_path); - self.allocator.free(self.built_url); + self.allocator.free(self.uri); self.allocator.destroy(self); } @@ -132,71 +525,69 @@ pub const HF = struct { self.request = null; self.response = null; self.body_reader = null; - self.response_bytes_remaining = 0; } } + + pub fn recordLatency(self: *FileHandle, latency_ns: u64) void { + self.total_requests += 1; + self.total_latency_ns += latency_ns; + self.min_latency_ns = @min(self.min_latency_ns, latency_ns); + self.max_latency_ns = @max(self.max_latency_ns, latency_ns); + } + + pub fn avgLatencyMs(self: *FileHandle) u64 { + if (self.total_requests == 0) return 0; + return (self.total_latency_ns / self.total_requests) / std.time.ns_per_ms; + } }; pub const DirHandle = struct { - id: HandleId, allocator: std.mem.Allocator, - hf_path: []const u8, - built_url: []const u8, - redirect_buffer: []u8, - request: ?*std.http.Client.Request, - response: ?std.http.Client.Response, - body_reader: ?*std.Io.Reader, - - pub fn init(allocator: std.mem.Allocator, id: HandleId, url: []const u8) std.mem.Allocator.Error!*DirHandle { - const self = try allocator.create(DirHandle); - errdefer allocator.destroy(self); - - const redirect_buffer = try allocator.alloc(u8, 1024 * 1024); - errdefer allocator.free(redirect_buffer); - - const owned_path = try allocator.dupe(u8, url); - errdefer allocator.free(owned_path); - - self.* = .{ - .id = id, - .allocator = allocator, - .hf_path = owned_path, - .built_url = "", - .redirect_buffer = redirect_buffer, - .request = null, - .response = null, - .body_reader = null, - }; - - return self; - } + uri: []const u8, + size: u64, pub fn deinit(self: *DirHandle) void { - self.allocator.free(self.hf_path); - self.allocator.free(self.built_url); - self.allocator.free(self.redirect_buffer); + self.allocator.free(self.uri); self.allocator.destroy(self); } }; - pub const InitArgs = struct { - allocator: std.mem.Allocator, - io: std.Io, - http_client: *std.http.Client, - config: Config, + pub const Config = struct { + auth: Auth = .none, + request_range_min: u64 = 8 * 1024, + request_range_max: u64 = 128 * 1024 * 1024, + hf_pagination_limit: usize = 100, }; - pub fn init(args: InitArgs) !HF { + allocator: std.mem.Allocator, + mutex: std.Io.Mutex, + + file_handles: std.AutoHashMapUnmanaged(HandleId, *FileHandle), + dir_handles: std.AutoHashMapUnmanaged(HandleId, *DirHandle), + next_file_handle_id: HandleId, + next_dir_handle_id: HandleId, + + hf_records: HFRecords, + + client: *std.http.Client, + http_headers: []std.http.Header, + config: Config, + + vtable: std.Io.VTable, + + pub fn init(allocator: std.mem.Allocator, base_io: std.Io, http_client: *std.http.Client, config: Config) std.mem.Allocator.Error!HF { var self: HF = .{ - .allocator = args.allocator, - .client = args.http_client, - .http_headers = try args.allocator.alloc(std.http.Header, 2), - .config = args.config, + .allocator = allocator, + .mutex = .init, .file_handles = .{}, .dir_handles = .{}, - .next_handle_id = 1, - .mutex = .init, - .vtable = makeVTable(args.io), + .next_file_handle_id = 0, + .next_dir_handle_id = 0, + .hf_records = .init(allocator), + .client = http_client, + .http_headers = try allocator.alloc(std.http.Header, 2), + .config = config, + .vtable = makeVTable(base_io), }; switch (self.config.auth) { @@ -221,6 +612,7 @@ pub const HF = struct { handle_ptr.*.deinit(); } self.dir_handles.deinit(self.allocator); + self.hf_records.deinit(); self.allocator.free(self.http_headers); } @@ -231,90 +623,460 @@ pub const HF = struct { }; } - fn allocateHandleId(self: *HF) HandleId { - self.mutex.lockUncancelable(self.io()); - defer self.mutex.unlock(self.io()); + const URIKind = union(enum) { + dir, + file, + }; - const id = self.next_handle_id; - self.next_handle_id += 1; - return id; - } - - fn registerFileHandle(self: *HF, handle: *FileHandle) std.mem.Allocator.Error!void { - self.mutex.lockUncancelable(self.io()); - defer self.mutex.unlock(self.io()); - - try self.file_handles.put(self.allocator, handle.id, handle); - } - - fn unregisterFileHandle(self: *HF, id: HandleId) ?*FileHandle { - self.mutex.lockUncancelable(self.io()); - defer self.mutex.unlock(self.io()); - - const kv = self.file_handles.fetchRemove(id) orelse return null; - return kv.value; - } - - fn getFileHandle(self: *HF, id: HandleId) ?*FileHandle { - self.mutex.lockUncancelable(self.io()); - defer self.mutex.unlock(self.io()); - - return self.file_handles.get(id); - } - - fn registerDirHandle(self: *HF, handle: *DirHandle) std.mem.Allocator.Error!void { - self.mutex.lockUncancelable(self.io()); - defer self.mutex.unlock(self.io()); - - try self.dir_handles.put(self.allocator, handle.id, handle); - } - - fn unregisterDirHandle(self: *HF, id: HandleId) ?*DirHandle { - self.mutex.lockUncancelable(self.io()); - defer self.mutex.unlock(self.io()); - - const kv = self.dir_handles.fetchRemove(id) orelse return null; - return kv.value; - } - - fn getDirHandle(self: *HF, id: HandleId) ?*DirHandle { - self.mutex.lockUncancelable(self.io()); - defer self.mutex.unlock(self.io()); - return self.dir_handles.get(id); - } - - fn resolvePath(self: *HF, dir_handle: std.Io.Dir.Handle, sub_path: []const u8) ![]const u8 { + fn resolveUri(self: *HF, parent_path: []const u8, sub_path: []const u8, kind: URIKind) InitHandleError![]const u8 { if (std.mem.startsWith(u8, sub_path, "hf://")) { - return try self.allocator.dupe(u8, sub_path); + if (kind == .dir and !std.mem.endsWith(u8, sub_path, "/")) { + return std.fmt.allocPrint(self.allocator, "{s}/", .{sub_path}); + } else { + return try self.allocator.dupe(u8, sub_path); + } } - const dir = self.getDirHandle(dir_handle) orelse { - return error.DirectoryNotfound; + const base_uri = std.Uri.parse(parent_path) catch return InitHandleError.PathUnresolvable; + var aux_buf: [std.fs.max_path_bytes]u8 = undefined; + + if (sub_path.len > aux_buf.len) return InitHandleError.PathTooLong; + + @memcpy(aux_buf[0..sub_path.len], sub_path); + var aux_slice: []u8 = aux_buf[0..]; + const resolved = std.Uri.resolveInPlace(base_uri, sub_path.len, &aux_slice) catch return InitHandleError.PathUnresolvable; + if (kind == .dir and !std.mem.endsWith(u8, resolved.path.percent_encoded, "/")) { + return std.fmt.allocPrint(self.allocator, "{f}/", .{resolved.fmt(.{ + .scheme = true, + .authority = true, + .path = true, + .query = false, + .fragment = false, + })}); + } else { + return std.fmt.allocPrint(self.allocator, "{f}", .{resolved.fmt(.{ + .scheme = true, + .authority = true, + .path = true, + .query = false, + .fragment = false, + })}); + } + } + + fn fetchHFRecordsIfNeeded( + self: *HF, + uri: []const u8, + ) HFRecords.Error!void { + var extra_headers: []std.http.Header = &.{}; + switch (self.config.auth) { + .hf_token => |_| { + extra_headers = self.http_headers[0..1]; + }, + else => {}, + } + + if (!self.hf_records.containsPathPrefix(uri)) { + log.debug("Fetching HF records for URI {s}", .{uri}); + try self.hf_records.fetch(self.client, self.config.hf_pagination_limit, extra_headers, uri, null); + } + } + + fn dirSize(self: *HF, uri: []const u8) HFApi.Error!u64 { + var extra_headers: []std.http.Header = &.{}; + switch (self.config.auth) { + .hf_token => |_| { + extra_headers = self.http_headers[0..1]; + }, + else => {}, + } + + if (HFApi.isModelPath(uri)) { + const model_size = HFApi.fetchModelSize( + self.allocator, + self.client, + extra_headers, + uri, + ) catch |err| { + log.err("Failed to fetch model size for dirSize with URI {s}", .{uri}); + return err; + }; + defer model_size.deinit(); + + return model_size.value.size; + } else { + const entry = self.hf_records.get(uri) orelse { + log.err("Directory path not found in repo tree entries with URI {s}", .{uri}); + return HFApi.Error.ResponseParsingFailed; + }; + + return entry.size; + } + } + + fn initFileHandle( + self: *HF, + dir: DirHandle, + sub_path: []const u8, + ) InitHandleError!*FileHandle { + const handle = try self.allocator.create(FileHandle); + errdefer self.allocator.destroy(handle); + + const uri = try self.resolveUri(dir.uri, sub_path, .file); + errdefer self.allocator.free(uri); + + self.fetchHFRecordsIfNeeded(uri) catch { + log.err("Failed to fetch records in dirOpenFile for URI {s}", .{uri}); + return InitHandleError.FailedToFetchRemoteResource; }; - return joinPath(self.allocator, dir.hf_path, sub_path); + const entry = self.hf_records.get(uri) orelse { + log.err("Directory path not found in repo tree entries with URI {s}", .{uri}); + return InitHandleError.MissingRessource; + }; + + handle.* = .{ + .allocator = self.allocator, + .uri = uri, + .pos = 0, + .size = entry.size, + }; + + return handle; } - pub fn joinPath(allocator: std.mem.Allocator, base: []const u8, path: []const u8) ![]const u8 { - if (base.len == 0) { - return try allocator.dupe(u8, path); - } - if (path.len == 0) { - return try allocator.dupe(u8, base); + fn registerFileHandle(self: *HF, handle: *FileHandle) std.mem.Allocator.Error!HandleId { + self.mutex.lockUncancelable(self.io()); + defer self.mutex.unlock(self.io()); + + const handle_id = self.next_file_handle_id; + self.next_file_handle_id += 1; + + try self.file_handles.put(self.allocator, handle_id, handle); + + return handle_id; + } + + fn fileHandle(self: *HF, file: std.Io.File) ?*FileHandle { + self.mutex.lockUncancelable(self.io()); + defer self.mutex.unlock(self.io()); + + return self.file_handles.get(file.handle); + } + + fn initDirHandle( + self: *HF, + parent_path: []const u8, + sub_path: []const u8, + ) InitHandleError!*DirHandle { + const dir_handle = try self.allocator.create(DirHandle); + errdefer self.allocator.destroy(dir_handle); + + const uri = try self.resolveUri(parent_path, sub_path, .dir); + errdefer self.allocator.free(uri); + + self.fetchHFRecordsIfNeeded(uri) catch { + log.err("Failed to fetch HF records in dirOpenFile for URI {s}", .{uri}); + return InitHandleError.FailedToFetchRemoteResource; + }; + + const dir_size = self.dirSize(uri) catch { + log.err("Failed to get directory size in dirOpenFile for URI {s}", .{uri}); + return InitHandleError.FailedToFetchRemoteResource; + }; + + dir_handle.* = .{ + .allocator = self.allocator, + .uri = uri, + .size = dir_size, + }; + + return dir_handle; + } + + fn registerDirHandle(self: *HF, handle: *DirHandle) std.mem.Allocator.Error!HandleId { + self.mutex.lockUncancelable(self.io()); + defer self.mutex.unlock(self.io()); + + const handle_id = self.next_dir_handle_id; + self.next_dir_handle_id += 1; + + try self.dir_handles.put(self.allocator, handle_id, handle); + + return handle_id; + } + + fn dirHandle(self: *HF, dir: std.Io.Dir) ?DirHandle { + self.mutex.lockUncancelable(self.io()); + defer self.mutex.unlock(self.io()); + + if (std.meta.eql(dir, std.Io.Dir.cwd())) { + return .{ + .allocator = self.allocator, + .uri = "", + .size = 0, + }; } - const base_has_slash = base[base.len - 1] == '/'; - const path_has_slash = path[0] == '/'; - - if (base_has_slash and path_has_slash) { - return try std.fmt.allocPrint(allocator, "{s}{s}", .{ base, path[1..] }); - } else if (base_has_slash or path_has_slash) { - return try std.fmt.allocPrint(allocator, "{s}{s}", .{ base, path }); + if (self.dir_handles.get(dir.handle)) |handle_ptr| { + return handle_ptr.*; } else { - return try std.fmt.allocPrint(allocator, "{s}/{s}", .{ base, path }); + return null; } } + fn openFile( + self: *HF, + dir: std.Io.Dir, + sub_path: []const u8, + ) std.Io.File.OpenError!std.Io.File { + const dir_handle = self.dirHandle(dir) orelse { + log.err("Directory handle not found for dirOpenFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); + return std.Io.File.OpenError.FileNotFound; + }; + + const handle = self.initFileHandle(dir_handle, sub_path) catch { + log.err("Failed to init FileHandle in openFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); + return std.Io.File.OpenError.SystemResources; + }; + + const handle_id = self.registerFileHandle(handle) catch { + log.err("Failed to register FileHandle in openFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); + return std.Io.File.OpenError.SystemResources; + }; + + return .{ .handle = @intCast(handle_id) }; + } + + fn openDir( + self: *HF, + dir: std.Io.Dir, + sub_path: []const u8, + ) std.Io.Dir.OpenError!std.Io.Dir { + const dir_handle = self.dirHandle(dir) orelse { + log.err("Directory handle not found for dirOpenFile", .{}); + return std.Io.Dir.OpenError.FileNotFound; + }; + + const handle = self.initDirHandle(dir_handle.uri, sub_path) catch { + log.err("Failed to init DirHandle in openDir with dir={any} sub_path={s}", .{ dir.handle, sub_path }); + return std.Io.Dir.OpenError.SystemResources; + }; + + const handle_id = self.registerDirHandle(handle) catch { + log.err("Failed to register DirHandle in openDir with dir={any} sub_path={s}", .{ dir.handle, sub_path }); + return std.Io.Dir.OpenError.SystemResources; + }; + + return .{ .handle = @intCast(handle_id) }; + } + + fn closeDir(self: *HF, dir: std.Io.Dir) void { + self.mutex.lockUncancelable(self.io()); + defer self.mutex.unlock(self.io()); + + const handle_kv = self.dir_handles.fetchRemove(dir.handle) orelse { + log.warn("Attempted to close non-existent dir handle: {d}", .{dir.handle}); + return; + }; + + handle_kv.value.deinit(); + } + + fn closeFile(self: *HF, file: std.Io.File) void { + self.mutex.lockUncancelable(self.io()); + defer self.mutex.unlock(self.io()); + + const handle_kv = self.file_handles.fetchRemove(file.handle) orelse { + log.warn("Attempted to close non-existent file handle: {d}", .{file.handle}); + return; + }; + + handle_kv.value.deinit(); + } + + fn setAbsolutePos(self: *HF, file: std.Io.File, pos: u64) std.Io.File.SeekError!void { + const handle = self.fileHandle(file) orelse return std.Io.File.SeekError.Unexpected; + handle.releaseRequest(); + handle.pos = pos; + } + + fn setRelativeOffset(self: *HF, file: std.Io.File, offset: i64) std.Io.File.SeekError!void { + const handle = self.fileHandle(file) orelse return std.Io.File.SeekError.Unexpected; + handle.releaseRequest(); + + const new_pos = @as(i64, @intCast(handle.pos)) + offset; + handle.pos = @intCast(new_pos); + } + + fn performPositionalRead(self: *HF, file: std.Io.File, data: [][]u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const handle = self.fileHandle(file) orelse return std.Io.File.ReadPositionalError.Unexpected; + + const saved_pos = handle.pos; + + if (handle.pos != offset) { + handle.releaseRequest(); + handle.pos = offset; + } + + const bytes_read = try self.performRead(file, data); + + handle.pos = saved_pos; + handle.releaseRequest(); + + return bytes_read; + } + + fn performRead(self: *HF, file: std.Io.File, data: [][]u8) std.Io.File.Reader.Error!usize { + const handle = self.fileHandle(file) orelse { + log.err("Invalid file handle in VFS read", .{}); + return std.Io.File.Reader.Error.Unexpected; + }; + + if (data.len == 0) return 0; + + var total_read: usize = 0; + + for (data) |buf| { + var offset: usize = 0; + + while (offset < buf.len) { + // Check EOF against known file size + if (handle.pos >= handle.size) { + return total_read; + } + + const start = handle.pos; + var desired_read_len: u64 = @max(@as(u64, @intCast(buf.len)), self.config.request_range_min); + if (desired_read_len > self.config.request_range_max) desired_read_len = self.config.request_range_max; + + const end = if (handle.size > 0 and start < handle.size) @min(start + desired_read_len - 1, handle.size - 1) else start + desired_read_len - 1; + + try self.requestRead(file, start, end); + const reader = handle.body_reader.?; + + const dest = buf[offset..]; + var iov = [_][]u8{dest}; + + const n = reader.readVec(&iov) catch |err| switch (err) { + error.EndOfStream => { + log.debug("Reached EndOfStream for handle={d} pos={d}, will re-request remaining", .{ file.handle, handle.pos }); + handle.releaseRequest(); + continue; + }, + else => { + return std.Io.File.Reader.Error.Unexpected; + }, + }; + + offset += n; + handle.pos += n; + total_read += n; + } + } + return total_read; + } + + fn requestRead(self: *HF, file: std.Io.File, start: u64, end: u64) std.Io.File.Reader.Error!void { + const handle = self.fileHandle(file) orelse return std.Io.File.Reader.Error.Unexpected; + + if (handle.request) |_| return; + + std.debug.assert(start <= end); + + var range_buf: [64]u8 = undefined; + const range_header = std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ start, end }) catch unreachable; + + var timer = std.time.Timer.start() catch { + log.err("Failed to start timer for request", .{}); + return std.Io.File.Reader.Error.SystemResources; + }; + + const request = self.allocator.create(std.http.Client.Request) catch { + log.err("Failed to allocate request", .{}); + return std.Io.File.Reader.Error.SystemResources; + }; + errdefer self.allocator.destroy(request); + + var url_buf: [std.fs.max_path_bytes]u8 = undefined; + + const path = HFApi.filePathFromUri(handle.uri) catch |err| { + log.err("Failed to extract file path from URI {s}: {}", .{ handle.uri, err }); + return std.Io.File.Reader.Error.Unexpected; + }; + + const api_url = blk: { + const model = HFApi.modelFromUri(handle.uri) catch |err| { + log.err("Failed to parse model from URI {s}: {}", .{ handle.uri, err }); + return std.Io.File.Reader.Error.Unexpected; + }; + const rev = model.revision orelse blk_: { + log.debug("No revision specified in repo URI; defaulting to 'main'", .{}); + break :blk_ "main"; + }; + + break :blk std.fmt.bufPrint( + &url_buf, + "https://huggingface.co/{s}/{s}/resolve/{s}/{s}", + .{ model.namespace, model.repo, rev, path }, + ) catch { + log.err("Failed to format HF repo tree API URL for repo URI {s}", .{handle.uri}); + return std.Io.File.Reader.Error.SystemResources; + }; + }; + + const api_uri = std.Uri.parse(api_url) catch { + log.err("Failed to parse URL during read request", .{}); + return std.Io.File.Reader.Error.Unexpected; + }; + + request.* = self.client.request(.GET, api_uri, .{ + .headers = .{ .accept_encoding = .{ .override = "identity" } }, + .extra_headers = &.{.{ .name = "Range", .value = range_header }}, + }) catch |err| { + log.err("Failed to create GET request: {}", .{err}); + return std.Io.File.Reader.Error.SystemResources; + }; + errdefer request.deinit(); + + request.sendBodiless() catch |err| { + log.err("Failed to send GET request: {}", .{err}); + return std.Io.File.Reader.Error.SystemResources; + }; + + const response = request.receiveHead(&handle.redirect_buffer) catch { + log.err("Failed to receive header response for URL {s} ", .{api_url}); + return std.Io.File.Reader.Error.Unexpected; + }; + + const latency = timer.read(); + handle.recordLatency(latency); + + log.debug("GET call: handle={d} status={s} content_length={?} range={d}-{d} ({d}) latency={d}ms avg={d}ms min={d}ms max={d}ms for {s}", .{ + file.handle, + @tagName(response.head.status), + response.head.content_length, + start, + end, + end - start + 1, + latency / std.time.ns_per_ms, + handle.avgLatencyMs(), + handle.min_latency_ns / std.time.ns_per_ms, + handle.max_latency_ns / std.time.ns_per_ms, + api_url, + }); + + if (response.head.status != .partial_content and response.head.status != .ok) { + request.deinit(); + self.allocator.destroy(request); + return std.Io.File.Reader.Error.Unexpected; + } + + handle.request = request; + handle.response = response; + handle.body_reader = handle.response.?.reader(&.{}); + } + fn makeVTable(base_io: std.Io) std.Io.VTable { var vtable = base_io.vtable.*; vtable.dirMake = dirMake; @@ -350,7 +1112,8 @@ pub const HF = struct { _ = dir; _ = sub_path; _ = mode; - return error.Unexpected; + log.err("VFS is read-only, dirMake not supported", .{}); + return std.Io.Dir.MakeError.ReadOnlyFileSystem; } fn dirMakePath( @@ -363,7 +1126,8 @@ pub const HF = struct { _ = dir; _ = sub_path; _ = mode; - return error.Unexpected; + log.err("VFS is read-only, dirMakePath not supported", .{}); + return std.Io.Dir.MakeError.ReadOnlyFileSystem; } fn dirMakeOpenPath( @@ -376,137 +1140,18 @@ pub const HF = struct { _ = dir; _ = sub_path; _ = options; - return error.Unexpected; - } - - pub fn updateBuiltUrl(self: *HF, url_ptr: *[]const u8, new_url: []const u8) !void { - const owned_url = try self.allocator.dupe(u8, new_url); - errdefer self.allocator.free(owned_url); - - self.allocator.free(url_ptr.*); - url_ptr.* = owned_url; + log.err("VFS is read-only, dirMakeOpenPath not supported", .{}); + return std.Io.Dir.MakeOpenPathError.Unexpected; } fn dirStat( userdata: ?*anyopaque, dir: std.Io.Dir, ) std.Io.Dir.StatError!std.Io.Dir.Stat { - const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); - const handle = self.getDirHandle(@intCast(dir.handle)) orelse return error.Unexpected; - const stat_result = self.statDir(handle) catch return error.Unexpected; - - return .{ - .inode = @intCast(handle.id), - .kind = .directory, - .size = stat_result.size, - .mode = 0o555, - .atime = .{ .nanoseconds = 0 }, - .mtime = .{ .nanoseconds = 0 }, - .ctime = .{ .nanoseconds = 0 }, - }; - } - - fn statDir(self: *HF, handle: *DirHandle) !StatResult { - const path = handle.hf_path[5..]; - var path_it = std.mem.splitSequence(u8, path, "/"); - const namespace = path_it.next() orelse return error.Unexpected; - const repo = path_it.next() orelse return error.Unexpected; - const rev = "main"; - // Build API URL - var url_buf: [512]u8 = undefined; - const url = std.fmt.bufPrint( - &url_buf, - "https://huggingface.co/api/models/{s}/{s}/treesize/{s}/", - .{ namespace, repo, rev }, - ) catch return error.SystemResources; - - self.updateBuiltUrl(&handle.built_url, url) catch return error.SystemResources; - - return try self.fetchStatDir(handle); - } - - pub const StatResult = struct { - size: u64, - }; - - fn fetchStatDir(self: *HF, handle: *DirHandle) !StatResult { - var attempt: u32 = 0; - while (attempt < max_retries) : (attempt += 1) { - const req = self.allocator.create(std.http.Client.Request) catch continue; - - req.* = self.client.request(.GET, std.Uri.parse(handle.built_url) catch return error.Unexpected, .{ - .headers = .{ .accept_encoding = .{ .override = "identity" } }, - .extra_headers = self.http_headers[0..1], - }) catch { - self.allocator.destroy(req); - continue; - }; - - req.sendBodiless() catch { - req.deinit(); - self.allocator.destroy(req); - continue; - }; - - var response = req.receiveHead(handle.redirect_buffer) catch { - req.deinit(); - self.allocator.destroy(req); - continue; - }; - - defer { - req.deinit(); - self.allocator.destroy(req); - } - - switch (response.head.status) { - .moved_permanently, .found, .see_other, .temporary_redirect, .permanent_redirect => { - if (response.head.location) |location| { - var buf: [4096]u8 = undefined; - if (location.len > buf.len) return error.InvalidRedirect; - @memcpy(buf[0..location.len], location); - - var aux_buf: []u8 = buf[0..]; - const resolved = try std.Uri.resolveInPlace(try std.Uri.parse(handle.built_url), location.len, &aux_buf); - const new_url = try std.fmt.allocPrint(self.allocator, "{f}", .{resolved.fmt(.{ - .scheme = true, - .authority = true, - .path = true, - .query = true, - .fragment = true, - })}); - self.allocator.free(handle.built_url); - handle.built_url = new_url; - - continue; - } - }, - .ok => { - var body_buf: [256]u8 = undefined; - var body_reader = response.reader(&.{}); - var iov = [_][]u8{&body_buf}; - const body_len = try body_reader.readVec(&iov); - const body = body_buf[0..body_len]; - - var parsed = try std.json.parseFromSlice(std.json.Value, self.allocator, body, .{}); - defer parsed.deinit(); - - const size_value = parsed.value.object.get("size") orelse return error.InvalidFormat; - const size = size_value.integer; - - return .{ .size = @intCast(size) }; - }, - .internal_server_error, .bad_gateway, .service_unavailable, .gateway_timeout => { - continue; - }, - else => { - log.err("HTTP stat dir failed with status: {s} for {s}", .{ @tagName(response.head.status), handle.built_url }); - return error.HttpRequestFailed; - }, - } - } - - return error.Unexpected; + _ = userdata; + _ = dir; + log.err("VFS does not support dirStat", .{}); + return std.Io.Dir.StatError.AccessDenied; } fn dirStatPath( @@ -515,25 +1160,11 @@ pub const HF = struct { sub_path: []const u8, options: std.Io.Dir.StatPathOptions, ) std.Io.Dir.StatPathError!std.Io.File.Stat { - const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); + _ = userdata; + _ = dir; + _ = sub_path; _ = options; - - const file = dirOpenFile(userdata, dir, sub_path, .{}) catch return std.Io.Dir.StatPathError.BadPathName; - - const handle = self.getFileHandle(@intCast(file.handle)) orelse return error.Unexpected; - defer fileClose(userdata, file); - - const stat_result = self.fetchStat(handle) catch return error.Unexpected; - - return .{ - .inode = @intCast(file.handle), - .kind = .file, - .size = stat_result.size, - .mode = 0o444, - .atime = .{ .nanoseconds = 0 }, - .mtime = .{ .nanoseconds = 0 }, - .ctime = .{ .nanoseconds = 0 }, - }; + @panic("TODO: implement VFS dirStatPath"); } fn dirAccess( @@ -545,11 +1176,13 @@ pub const HF = struct { _ = userdata; _ = dir; _ = sub_path; + if (options.write) { return std.Io.Dir.AccessError.ReadOnlyFileSystem; } - // todo: check + log.err("VFS does not support dirAccess", .{}); + return std.Io.Dir.AccessError.AccessDenied; } fn dirCreateFile( @@ -562,6 +1195,7 @@ pub const HF = struct { _ = dir; _ = sub_path; _ = flags; + log.err("VFS is read-only, dirCreateFile not supported", .{}); return std.Io.File.OpenError.Unexpected; } @@ -571,32 +1205,9 @@ pub const HF = struct { sub_path: []const u8, flags: std.Io.File.OpenFlags, ) std.Io.File.OpenError!std.Io.File { - const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); _ = flags; - - const path = self.resolvePath(@intCast(dir.handle), sub_path) catch return error.SystemResources; - defer self.allocator.free(path); - - return try self.openFile(path); - } - - fn openFile(self: *HF, path: []const u8) std.Io.File.OpenError!std.Io.File { - const url = hfPathToHttps(self.allocator, path) catch return error.SystemResources; - defer self.allocator.free(url); - const id = self.allocateHandleId(); - - // Check if it's a valid file - var handle = FileHandle.init(self.allocator, id, path) catch return error.SystemResources; - errdefer handle.deinit(); - self.updateBuiltUrl(&handle.built_url, url) catch return error.SystemResources; - - _ = self.fetchStat(handle) catch return error.FileNotFound; - - // URL ownership transferred to handle - - self.registerFileHandle(handle) catch return error.SystemResources; - - return .{ .handle = @intCast(id) }; + const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); + return try self.openFile(dir, sub_path); } fn dirOpenDir( @@ -605,43 +1216,14 @@ pub const HF = struct { sub_path: []const u8, options: std.Io.Dir.OpenOptions, ) std.Io.Dir.OpenError!std.Io.Dir { - const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); _ = options; - const path = self.resolvePath(@intCast(dir.handle), sub_path) catch return error.Unexpected; - defer self.allocator.free(path); - - return try self.openDir(path); - } - - fn openDir(self: *HF, path: []const u8) std.Io.Dir.OpenError!std.Io.Dir { - const url = hfPathToHttps(self.allocator, path) catch return error.SystemResources; - defer self.allocator.free(url); - // Ensure URL ends with / - const dir_url = if (url.len > 0 and url[url.len - 1] != '/') blk: { - const new_url = std.fmt.allocPrint(self.allocator, "{s}/", .{url}) catch return error.Unexpected; - self.allocator.free(url); - break :blk new_url; - } else url; - errdefer if (dir_url.ptr != url.ptr) self.allocator.free(dir_url); - - const id = self.allocateHandleId(); - const handle = DirHandle.init(self.allocator, id, path) catch return error.Unexpected; - - // Ensure it's a correct directory - const stat = self.statDir(handle) catch return error.NotDir; - _ = stat; - - errdefer handle.deinit(); - - self.registerDirHandle(handle) catch return error.Unexpected; - - return .{ .handle = @intCast(id) }; + const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); + return try self.openDir(dir, sub_path); } fn dirClose(userdata: ?*anyopaque, dir: std.Io.Dir) void { const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); - const handle = self.unregisterDirHandle(@intCast(dir.handle)) orelse return; - handle.deinit(); + self.closeDir(dir); } fn fileStat( @@ -649,16 +1231,15 @@ pub const HF = struct { file: std.Io.File, ) std.Io.File.StatError!std.Io.File.Stat { const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); - const handle = self.getFileHandle(@intCast(file.handle)) orelse return error.Unexpected; - - // call self.httpengine.fetchStat(handle) - // log.debug("Fetching stat for file handle {d}", handle.id); - const stat_result = self.fetchStat(handle) catch return error.Unexpected; + const handle = self.fileHandle(file) orelse { + log.err("Invalid file handle in VFS stat", .{}); + return std.Io.File.StatError.Unexpected; + }; return .{ .inode = @intCast(file.handle), .kind = .file, - .size = stat_result.size, + .size = handle.size, .mode = 0o444, .atime = .{ .nanoseconds = 0 }, .mtime = .{ .nanoseconds = 0 }, @@ -666,81 +1247,9 @@ pub const HF = struct { }; } - pub fn fetchStat(self: *HF, handle: *FileHandle) !StatResult { - var attempt: u32 = 0; - while (attempt < max_retries) : (attempt += 1) { - const req = self.allocator.create(std.http.Client.Request) catch continue; - - req.* = self.client.request(.HEAD, std.Uri.parse(handle.built_url) catch return error.Unexpected, .{ - .headers = .{ .accept_encoding = .{ .override = "identity" } }, - .extra_headers = self.http_headers[0..1], - }) catch { - self.allocator.destroy(req); - continue; - }; - - req.sendBodiless() catch { - req.deinit(); - self.allocator.destroy(req); - continue; - }; - - var response = req.receiveHead(&handle.redirect_buffer) catch { - req.deinit(); - self.allocator.destroy(req); - continue; - }; - - defer { - req.deinit(); - self.allocator.destroy(req); - } - - switch (response.head.status) { - .moved_permanently, .found, .see_other, .temporary_redirect, .permanent_redirect => { - if (response.head.location) |location| { - var buf: [4096]u8 = undefined; - if (location.len > buf.len) return error.InvalidRedirect; - @memcpy(buf[0..location.len], location); - - var aux_buf: []u8 = buf[0..]; - const resolved = try std.Uri.resolveInPlace(try std.Uri.parse(handle.built_url), location.len, &aux_buf); - const new_url = try std.fmt.allocPrint(self.allocator, "{f}", .{resolved.fmt(.{ - .scheme = true, - .authority = true, - .path = true, - .query = true, - .fragment = true, - })}); - self.allocator.free(handle.built_url); - handle.built_url = new_url; - - continue; - } - }, - .ok, .partial_content => { - const size = response.head.content_length orelse return error.MissingContentLength; - handle.size = size; - - return .{ .size = size }; - }, - .internal_server_error, .bad_gateway, .service_unavailable, .gateway_timeout => { - continue; - }, - else => { - log.err("HTTP stat failed with status: {s} - {s}", .{ @tagName(response.head.status), handle.built_url }); - return error.HttpRequestFailed; - }, - } - } - - return error.Unexpected; - } - fn fileClose(userdata: ?*anyopaque, file: std.Io.File) void { const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); - const handle = self.unregisterFileHandle(@intCast(file.handle)) orelse return; - handle.deinit(); + self.closeFile(file); } fn fileWriteStreaming( @@ -751,7 +1260,8 @@ pub const HF = struct { _ = userdata; _ = file; _ = buffer; - return error.Unexpected; + log.err("VFS is read-only, fileWriteStreaming not supported", .{}); + return std.Io.File.WriteStreamingError.Unexpected; } fn fileWritePositional( @@ -764,7 +1274,8 @@ pub const HF = struct { _ = file; _ = buffer; _ = offset; - return error.Unexpected; + log.err("VFS is read-only, fileWritePositional not supported", .{}); + return std.Io.File.WritePositionalError.Unexpected; } fn fileReadStreaming( @@ -773,8 +1284,7 @@ pub const HF = struct { data: [][]u8, ) std.Io.File.Reader.Error!usize { const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); - const handle = self.getFileHandle(@intCast(file.handle)) orelse return error.Unexpected; - return self.performRead(handle, data, null) catch return error.Unexpected; + return try self.performRead(file, data); } fn fileReadPositional( @@ -784,8 +1294,7 @@ pub const HF = struct { offset: u64, ) std.Io.File.ReadPositionalError!usize { const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); - const handle = self.getFileHandle(@intCast(file.handle)) orelse return error.Unexpected; - return self.performRead(handle, data, offset) catch return error.Unexpected; + return try self.performPositionalRead(file, data, offset); } fn fileSeekBy( @@ -794,9 +1303,7 @@ pub const HF = struct { relative_offset: i64, ) std.Io.File.SeekError!void { const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); - const handle = self.getFileHandle(@intCast(file.handle)) orelse return error.Unexpected; - - seekBy(handle, relative_offset) catch return error.Unexpected; + try self.setRelativeOffset(file, relative_offset); } fn fileSeekTo( @@ -805,29 +1312,7 @@ pub const HF = struct { absolute_offset: u64, ) std.Io.File.SeekError!void { const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); - const handle = self.getFileHandle(@intCast(file.handle)) orelse return error.Unexpected; - - seekTo(handle, absolute_offset) catch return error.Unexpected; - } - - pub fn seekBy(handle: *FileHandle, relative_offset: i64) !void { - handle.releaseRequest(); - if (relative_offset >= 0) { - handle.pos +|= @intCast(relative_offset); - } else { - const abs_offset: u64 = @intCast(-relative_offset); - if (abs_offset > handle.pos) { - handle.pos = 0; - } else { - handle.pos -= abs_offset; - } - } - } - - // Go back to VFS - pub fn seekTo(handle: *FileHandle, absolute_offset: u64) !void { - handle.releaseRequest(); - handle.pos = absolute_offset; + try self.setAbsolutePos(file, absolute_offset); } fn openSelfExe( @@ -836,146 +1321,7 @@ pub const HF = struct { ) std.Io.File.OpenSelfExeError!std.Io.File { _ = userdata; _ = flags; - return error.NotSupported; - } - - fn hfPathToHttps(allocator: std.mem.Allocator, hf_path: []const u8) ![]const u8 { - // Remove the "hf://" prefix - const path = hf_path[5..hf_path.len]; - var path_it = std.mem.splitSequence(u8, path, "/"); - const namespace = path_it.next() orelse return error.InvalidPath; - const repo = path_it.next() orelse return error.InvalidPath; - const rev = "main"; - var file: []const u8 = ""; - if (path_it.next()) |file_path| { - file = file_path; - } - - var final_url_buf: [std.fs.max_path_bytes]u8 = undefined; - - const final_url = try std.fmt.bufPrint( - &final_url_buf, - "https://huggingface.co/{s}/{s}/resolve/{s}/{s}", - .{ namespace, repo, rev, file }, - ); - - return try allocator.dupe(u8, final_url); - } - - fn ensureConnection(self: *HF, handle: *FileHandle) !void { - if (handle.body_reader != null) return; - - if (handle.size == null) { - const stat = try self.fetchStat(handle); - handle.size = stat.size; - } - - const size = handle.size.?; - - if (handle.pos >= size) return; - - const start = handle.pos; - const end = @min(start + self.config.range_size, size) - 1; - - var range_buf: [64]u8 = undefined; - const range_header = std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ start, end }) catch unreachable; - - const request = try self.allocator.create(std.http.Client.Request); - errdefer self.allocator.destroy(request); - - const uri = try std.Uri.parse(handle.built_url); - - self.http_headers[1] = .{ .name = "Range", .value = range_header }; - - request.* = try self.client.request(.GET, uri, .{ - .headers = .{ .accept_encoding = .{ .override = "identity" } }, - .extra_headers = self.http_headers[0..2], - }); - // errdefer request.deinit(); - - try request.sendBodiless(); - - const response = try request.receiveHead(&handle.redirect_buffer); - - if (response.head.status != .partial_content and response.head.status != .ok) { - request.deinit(); - self.allocator.destroy(request); - return error.Unexpected; - } - - handle.request = request; - handle.response = response; - handle.body_reader = handle.response.?.reader(&.{}); - } - - fn doRead(self: *HF, handle: *FileHandle, data: [][]u8) !usize { - if (data.len == 0) return 0; - - var total_read: usize = 0; - - for (data) |buf| { - var offset: usize = 0; - - while (offset < buf.len) { - // Check EOF against known file size - if (handle.size) |size| { - if (handle.pos >= size) { - return total_read; - } - } - - try self.ensureConnection(handle); - const reader = handle.body_reader orelse { - return total_read; - }; - - // Read into remaining buffer - const dest = buf[offset..]; - var iov = [_][]u8{dest}; - - const n = reader.readVec(&iov) catch |err| switch (err) { - error.EndOfStream => { - // Current HTTP range exhausted, need new request - handle.releaseRequest(); - continue; - }, - else => { - return err; - }, - }; - - // Data not ready yet, just retry same read - if (n == 0) { - continue; - } - - offset += n; - handle.pos += n; - total_read += n; - } - } - - return total_read; - } - - fn performRead(self: *HF, handle: *FileHandle, data: [][]u8, offset: ?u64) !usize { - const saved_pos = handle.pos; - const is_positional = offset != null; - - if (offset) |o| { - if (handle.pos != o) { - handle.releaseRequest(); - handle.pos = o; - } - } - - const bytes_read = try self.doRead(handle, data); - - if (is_positional) { - handle.pos = saved_pos; - handle.releaseRequest(); - } - - return bytes_read; + log.err("VFS does not support openSelfExe", .{}); + return std.Io.File.OpenSelfExeError.NotSupported; } }; diff --git a/zml2/io/vfs/http.zig b/zml2/io/vfs/http.zig index 2d1f6d4c..f26ffa29 100644 --- a/zml2/io/vfs/http.zig +++ b/zml2/io/vfs/http.zig @@ -7,47 +7,30 @@ pub const HTTP = struct { pub const InitHandleError = error{ PathTooLong, - AllocationFailed, PathUnresolvable, + FailedToFetchRemoteResource, + MissingRessource, } || std.mem.Allocator.Error; pub const FileHandle = struct { allocator: std.mem.Allocator, - url: []const u8, + uri: []const u8, pos: u64, size: u64, - redirect_buffer: [std.fs.max_path_bytes]u8, - request: ?*std.http.Client.Request, - response: ?std.http.Client.Response, - body_reader: ?*std.Io.Reader, + redirect_buffer: [std.fs.max_path_bytes]u8 = undefined, + request: ?*std.http.Client.Request = null, + response: ?std.http.Client.Response = null, + body_reader: ?*std.Io.Reader = null, total_requests: u64 = 0, total_latency_ns: u64 = 0, min_latency_ns: u64 = std.math.maxInt(u64), max_latency_ns: u64 = 0, - pub fn init(allocator: std.mem.Allocator, parent_path: []const u8, sub_path: []const u8) InitHandleError!*FileHandle { - const handle = try allocator.create(FileHandle); - errdefer allocator.destroy(handle); - - handle.* = .{ - .allocator = allocator, - .url = try resolveUri(allocator, parent_path, sub_path), - .pos = 0, - .size = undefined, - .redirect_buffer = undefined, - .request = null, - .response = null, - .body_reader = null, - }; - - return handle; - } - pub fn deinit(self: *FileHandle) void { self.releaseRequest(); - self.allocator.free(self.url); + self.allocator.free(self.uri); self.allocator.destroy(self); } @@ -72,61 +55,14 @@ pub const HTTP = struct { if (self.total_requests == 0) return 0; return (self.total_latency_ns / self.total_requests) / std.time.ns_per_ms; } - - fn resolveUri(allocator: std.mem.Allocator, parent_path: []const u8, sub_path: []const u8) InitHandleError![]const u8 { - if (std.mem.startsWith(u8, sub_path, "http://") or - std.mem.startsWith(u8, sub_path, "https://")) - { - return try allocator.dupe(u8, sub_path); - } - - const base_uri = std.Uri.parse(parent_path) catch return InitHandleError.PathTooLong; - var aux_buf: [std.fs.max_path_bytes]u8 = undefined; - - if (sub_path.len > aux_buf.len) return InitHandleError.PathTooLong; - - @memcpy(aux_buf[0..sub_path.len], sub_path); - var aux_slice: []u8 = aux_buf[0..]; - const resolved = std.Uri.resolveInPlace(base_uri, sub_path.len, &aux_slice) catch return InitHandleError.PathUnresolvable; - return std.fmt.allocPrint(allocator, "{f}", .{resolved.fmt(.{ - .scheme = true, - .authority = true, - .path = true, - .query = true, - .fragment = true, - })}); - } }; pub const DirHandle = struct { allocator: std.mem.Allocator, - url: []const u8, - - pub fn init(allocator: std.mem.Allocator, parent_path: []const u8, sub_path: []const u8) InitHandleError!*DirHandle { - const self = try allocator.create(DirHandle); - errdefer allocator.destroy(self); - - var url = try FileHandle.resolveUri(allocator, parent_path, sub_path); - errdefer allocator.free(url); - - // Ensure directory URLs have a trailing slash. - // According to RFC 3986, if the parent path lacks a trailing slash, it is treated as a file and the last segment is replaced. - if (url.len == 0 or url[url.len - 1] != '/') { - const with_slash = try std.fmt.allocPrint(allocator, "{s}/", .{url}); - allocator.free(url); - url = with_slash; - } - - self.* = .{ - .allocator = allocator, - .url = url, - }; - - return self; - } + uri: []const u8, pub fn deinit(self: *DirHandle) void { - self.allocator.free(self.url); + self.allocator.free(self.uri); self.allocator.destroy(self); } }; @@ -185,6 +121,75 @@ pub const HTTP = struct { }; } + const URIKind = union(enum) { + dir, + file, + }; + + fn resolveUri(self: *HTTP, parent_path: []const u8, sub_path: []const u8, kind: URIKind) InitHandleError![]const u8 { + if (std.mem.startsWith(u8, sub_path, "http://") or std.mem.startsWith(u8, sub_path, "https://")) { + if (kind == .dir and !std.mem.endsWith(u8, sub_path, "/")) { + return std.fmt.allocPrint(self.allocator, "{s}/", .{sub_path}); + } else { + return try self.allocator.dupe(u8, sub_path); + } + } + + const base_uri = std.Uri.parse(parent_path) catch return InitHandleError.PathUnresolvable; + var aux_buf: [std.fs.max_path_bytes]u8 = undefined; + + if (sub_path.len > aux_buf.len) return InitHandleError.PathTooLong; + + @memcpy(aux_buf[0..sub_path.len], sub_path); + var aux_slice: []u8 = aux_buf[0..]; + const resolved = std.Uri.resolveInPlace(base_uri, sub_path.len, &aux_slice) catch return InitHandleError.PathUnresolvable; + if (kind == .dir and !std.mem.endsWith(u8, resolved.path.percent_encoded, "/")) { + return std.fmt.allocPrint(self.allocator, "{f}/", .{resolved.fmt(.{ + .scheme = true, + .authority = true, + .path = true, + .query = false, + .fragment = false, + })}); + } else { + return std.fmt.allocPrint(self.allocator, "{f}", .{resolved.fmt(.{ + .scheme = true, + .authority = true, + .path = true, + .query = false, + .fragment = false, + })}); + } + } + + fn initFileHandle( + self: *HTTP, + dir: DirHandle, + sub_path: []const u8, + ) InitHandleError!*FileHandle { + const handle = try self.allocator.create(FileHandle); + errdefer self.allocator.destroy(handle); + + const uri = try self.resolveUri(dir.uri, sub_path, .file); + errdefer self.allocator.free(uri); + + handle.* = .{ + .allocator = self.allocator, + .uri = uri, + .pos = 0, + .size = undefined, + }; + + const stat = self.fetchFileStat(handle) catch { + log.err("Failed to fetch file stat for URI {s}", .{uri}); + return InitHandleError.FailedToFetchRemoteResource; + }; + + handle.size = stat.size; + + return handle; + } + fn registerFileHandle(self: *HTTP, handle: *FileHandle) std.mem.Allocator.Error!HandleId { self.mutex.lockUncancelable(self.io()); defer self.mutex.unlock(self.io()); @@ -204,6 +209,25 @@ pub const HTTP = struct { return self.file_handles.get(file.handle); } + fn initDirHandle( + self: *HTTP, + parent_path: []const u8, + sub_path: []const u8, + ) InitHandleError!*DirHandle { + const dir_handle = try self.allocator.create(DirHandle); + errdefer self.allocator.destroy(dir_handle); + + const uri = try self.resolveUri(parent_path, sub_path, .dir); + errdefer self.allocator.free(uri); + + dir_handle.* = .{ + .allocator = self.allocator, + .uri = uri, + }; + + return dir_handle; + } + fn registerDirHandle(self: *HTTP, handle: *DirHandle) std.mem.Allocator.Error!HandleId { self.mutex.lockUncancelable(self.io()); defer self.mutex.unlock(self.io()); @@ -223,7 +247,7 @@ pub const HTTP = struct { if (std.meta.eql(dir, std.Io.Dir.cwd())) { return .{ .allocator = self.allocator, - .url = "", + .uri = "", }; } @@ -244,33 +268,17 @@ pub const HTTP = struct { return std.Io.File.OpenError.FileNotFound; }; - const handle = FileHandle.init(self.allocator, dir_handle.url, sub_path) catch { - log.err("Failed to create FileHandle in openFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); + const handle = self.initFileHandle(dir_handle, sub_path) catch { + log.err("Failed to init FileHandle in openFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); return std.Io.File.OpenError.SystemResources; }; - errdefer handle.deinit(); const handle_id = self.registerFileHandle(handle) catch { log.err("Failed to register FileHandle in openFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); return std.Io.File.OpenError.SystemResources; }; - const file: std.Io.File = .{ .handle = @intCast(handle_id) }; - - const stat = self.fetchFileStat(file) catch |err| switch (err) { - error.AccessDenied, error.SystemResources => { - log.err("File not found during openFile for handle={d} url={s}", .{ file.handle, handle.url }); - return std.Io.File.OpenError.FileNotFound; - }, - else => { - log.err("Failed to fetch file stat during openFile for handle={d} url={s}", .{ file.handle, handle.url }); - return std.Io.File.OpenError.Unexpected; - }, - }; - - handle.size = stat.size; - - return file; + return .{ .handle = @intCast(handle_id) }; } fn openDir( @@ -280,16 +288,18 @@ pub const HTTP = struct { ) std.Io.Dir.OpenError!std.Io.Dir { const dir_handle = self.dirHandle(dir) orelse { log.err("Directory handle not found for dirOpenFile", .{}); - return std.Io.File.OpenError.FileNotFound; + return std.Io.Dir.OpenError.FileNotFound; }; - const handle = DirHandle.init(self.allocator, dir_handle.url, sub_path) catch { - log.err("Failed to create DirHandle in dirOpenFile", .{}); + const handle = self.initDirHandle(dir_handle.uri, sub_path) catch { + log.err("Failed to init DirHandle in openDir with dir={any} sub_path={s}", .{ dir.handle, sub_path }); return std.Io.Dir.OpenError.SystemResources; }; - errdefer handle.deinit(); - const handle_id = self.registerDirHandle(handle) catch return std.Io.Dir.OpenError.SystemResources; + const handle_id = self.registerDirHandle(handle) catch { + log.err("Failed to register DirHandle in openDir with dir={any} sub_path={s}", .{ dir.handle, sub_path }); + return std.Io.Dir.OpenError.SystemResources; + }; return .{ .handle = @intCast(handle_id) }; } @@ -352,7 +362,7 @@ pub const HTTP = struct { fn performRead(self: *HTTP, file: std.Io.File, data: [][]u8) std.Io.File.Reader.Error!usize { const handle = self.fileHandle(file) orelse { - log.err("Invalid file handle in HTTP VFS read", .{}); + log.err("Invalid file handle in VFS read", .{}); return std.Io.File.Reader.Error.Unexpected; }; @@ -411,18 +421,18 @@ pub const HTTP = struct { const range_header = std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ start, end }) catch unreachable; var timer = std.time.Timer.start() catch { - log.err("Failed to start timer for HTTP request", .{}); + log.err("Failed to start timer for request", .{}); return std.Io.File.Reader.Error.SystemResources; }; const request = self.allocator.create(std.http.Client.Request) catch { - log.err("Failed to allocate HTTP request", .{}); + log.err("Failed to allocate request", .{}); return std.Io.File.Reader.Error.SystemResources; }; errdefer self.allocator.destroy(request); - const uri = std.Uri.parse(handle.url) catch |err| { - log.err("Failed to parse URL during read request: {}", .{err}); + const uri = std.Uri.parse(handle.uri) catch |err| { + log.err("Failed to parse URI during read request: {}", .{err}); return std.Io.File.Reader.Error.Unexpected; }; @@ -430,25 +440,25 @@ pub const HTTP = struct { .headers = .{ .accept_encoding = .{ .override = "identity" } }, .extra_headers = &.{.{ .name = "Range", .value = range_header }}, }) catch |err| { - log.err("Failed to create HTTP GET request: {}", .{err}); + log.err("Failed to create GET request: {}", .{err}); return std.Io.File.Reader.Error.SystemResources; }; errdefer request.deinit(); request.sendBodiless() catch |err| { - log.err("Failed to send HTTP GET request: {}", .{err}); + log.err("Failed to send GET request: {}", .{err}); return std.Io.File.Reader.Error.SystemResources; }; const response = request.receiveHead(&handle.redirect_buffer) catch |err| { - log.err("Failed to receive HTTP response head for handle={d} url={s}: {}", .{ file.handle, handle.url, err }); + log.err("Failed to receive HTTP response head for handle={d} uri={s}: {}", .{ file.handle, handle.uri, err }); return std.Io.File.Reader.Error.Unexpected; }; const latency = timer.read(); handle.recordLatency(latency); - log.debug("HTTP call: handle={d} status={s} content_length={?} range={d}-{d} ({d}) latency={d}ms avg={d}ms min={d}ms max={d}ms for {s}", .{ + log.debug("GET call: handle={d} status={s} content_length={?} range={d}-{d} ({d}) latency={d}ms avg={d}ms min={d}ms max={d}ms for {s}", .{ file.handle, @tagName(response.head.status), response.head.content_length, @@ -459,7 +469,7 @@ pub const HTTP = struct { handle.avgLatencyMs(), handle.min_latency_ns / std.time.ns_per_ms, handle.max_latency_ns / std.time.ns_per_ms, - handle.url, + handle.uri, }); if (response.head.status != .partial_content and response.head.status != .ok) { @@ -474,13 +484,10 @@ pub const HTTP = struct { } pub const FileStat = struct { - inode: u64, size: u64, }; - fn fetchFileStat(self: *HTTP, file: std.Io.File) std.Io.File.StatError!FileStat { - const handle = self.fileHandle(file) orelse return std.Io.File.StatError.Unexpected; - + fn fetchFileStat(self: *HTTP, handle: *FileHandle) std.Io.File.StatError!FileStat { var attempt: u32 = 0; while (attempt < self.config.max_fetch_attempts) : (attempt += 1) { const req = self.allocator.create(std.http.Client.Request) catch { @@ -488,7 +495,7 @@ pub const HTTP = struct { return std.Io.File.StatError.SystemResources; }; - req.* = self.client.request(.HEAD, std.Uri.parse(handle.url) catch return std.Io.File.StatError.Unexpected, .{ + req.* = self.client.request(.HEAD, std.Uri.parse(handle.uri) catch return std.Io.File.StatError.Unexpected, .{ .headers = .{ .accept_encoding = .{ .override = "identity" } }, }) catch { self.allocator.destroy(req); @@ -522,7 +529,7 @@ pub const HTTP = struct { var aux_buf: []u8 = buf[0..]; - const uri = std.Uri.parse(handle.url) catch |err| { + const uri = std.Uri.parse(handle.uri) catch |err| { log.err("Failed to parse URL during redirect: {}", .{err}); return std.Io.File.StatError.Unexpected; }; @@ -531,7 +538,7 @@ pub const HTTP = struct { log.err("Failed to resolve redirect URL: {}", .{err}); return std.Io.File.StatError.Unexpected; }; - const new_url = std.fmt.allocPrint(self.allocator, "{f}", .{resolved.fmt(.{ + const new_uri = std.fmt.allocPrint(self.allocator, "{f}", .{resolved.fmt(.{ .scheme = true, .authority = true, .path = true, @@ -542,8 +549,8 @@ pub const HTTP = struct { return std.Io.File.StatError.SystemResources; }; - self.allocator.free(handle.url); - handle.url = new_url; + self.allocator.free(handle.uri); + handle.uri = new_uri; continue; } @@ -555,7 +562,6 @@ pub const HTTP = struct { }; return .{ - .inode = @intCast(file.handle), .size = size, }; }, @@ -611,7 +617,7 @@ pub const HTTP = struct { _ = dir; _ = sub_path; _ = mode; - log.err("HTTP VFS is read-only, dirMake not supported", .{}); + log.err("VFS is read-only, dirMake not supported", .{}); return std.Io.Dir.MakeError.ReadOnlyFileSystem; } @@ -625,7 +631,7 @@ pub const HTTP = struct { _ = dir; _ = sub_path; _ = mode; - log.err("HTTP VFS is read-only, dirMakePath not supported", .{}); + log.err("VFS is read-only, dirMakePath not supported", .{}); return std.Io.Dir.MakeError.ReadOnlyFileSystem; } @@ -639,7 +645,7 @@ pub const HTTP = struct { _ = dir; _ = sub_path; _ = options; - log.err("HTTP VFS is read-only, dirMakeOpenPath not supported", .{}); + log.err("VFS is read-only, dirMakeOpenPath not supported", .{}); return std.Io.Dir.MakeOpenPathError.Unexpected; } @@ -649,7 +655,7 @@ pub const HTTP = struct { ) std.Io.Dir.StatError!std.Io.Dir.Stat { _ = userdata; _ = dir; - log.err("HTTP VFS does not support dirStat", .{}); + log.err("VFS does not support dirStat", .{}); return std.Io.Dir.StatError.AccessDenied; } @@ -663,7 +669,7 @@ pub const HTTP = struct { _ = dir; _ = sub_path; _ = options; - @panic("TODO: implement HTTP VFS dirStatPath"); + @panic("TODO: implement VFS dirStatPath"); } fn dirAccess( @@ -680,7 +686,7 @@ pub const HTTP = struct { return std.Io.Dir.AccessError.ReadOnlyFileSystem; } - log.err("HTTP VFS does not support dirAccess", .{}); + log.err("VFS does not support dirAccess", .{}); return std.Io.Dir.AccessError.AccessDenied; } @@ -694,7 +700,7 @@ pub const HTTP = struct { _ = dir; _ = sub_path; _ = flags; - log.err("HTTP VFS is read-only, dirCreateFile not supported", .{}); + log.err("VFS is read-only, dirCreateFile not supported", .{}); return std.Io.File.OpenError.Unexpected; } @@ -730,12 +736,16 @@ pub const HTTP = struct { file: std.Io.File, ) std.Io.File.StatError!std.Io.File.Stat { const self: *HTTP = @ptrCast(@alignCast(userdata orelse unreachable)); - const stat = try self.fetchFileStat(file); + const handle = self.fileHandle(file) orelse { + log.err("Invalid file handle in VFS stat", .{}); + + return std.Io.File.StatError.Unexpected; + }; return .{ - .inode = stat.inode, + .inode = @intCast(file.handle), .kind = .file, - .size = stat.size, + .size = handle.size, .mode = 0o444, .atime = .{ .nanoseconds = 0 }, .mtime = .{ .nanoseconds = 0 }, @@ -756,7 +766,7 @@ pub const HTTP = struct { _ = userdata; _ = file; _ = buffer; - log.err("HTTP VFS is read-only, fileWriteStreaming not supported", .{}); + log.err("VFS is read-only, fileWriteStreaming not supported", .{}); return std.Io.File.WriteStreamingError.Unexpected; } @@ -770,7 +780,7 @@ pub const HTTP = struct { _ = file; _ = buffer; _ = offset; - log.err("HTTP VFS is read-only, fileWritePositional not supported", .{}); + log.err("VFS is read-only, fileWritePositional not supported", .{}); return std.Io.File.WritePositionalError.Unexpected; } @@ -817,7 +827,7 @@ pub const HTTP = struct { ) std.Io.File.OpenSelfExeError!std.Io.File { _ = userdata; _ = flags; - log.err("HTTP VFS does not support openSelfExe", .{}); + log.err("VFS does not support openSelfExe", .{}); return std.Io.File.OpenSelfExeError.NotSupported; } }; diff --git a/zml2/main_io.zig b/zml2/main_io.zig index c8e4d113..f9a612df 100644 --- a/zml2/main_io.zig +++ b/zml2/main_io.zig @@ -9,7 +9,9 @@ pub const std_options: std.Options = .{ .log_level = .debug, .log_scope_levels = &[_]std.log.ScopeLevel{ .{ .scope = .@"zml/safetensors", .level = .debug }, + .{ .scope = .@"zml/io/vfs/file", .level = .info }, .{ .scope = .@"zml/io/vfs/http", .level = .info }, + .{ .scope = .@"zml/io/vfs/hf", .level = .info }, }, }; @@ -48,7 +50,7 @@ pub fn main() !void { allocator, threaded.io(), .{ - .direct_io = std.process.hasEnvVarConstant("DIRECT"), + .direct_io = true, .direct_io_alignment = .fromByteUnits(4 * 1024), }, ); @@ -60,14 +62,17 @@ pub fn main() !void { var hf_auth: zml.io.VFS.HF.Auth = try .auto(allocator, threaded.io()); defer hf_auth.deinit(allocator); - var hf_vfs: zml.io.VFS.HF = try .init(.{ - .allocator = allocator, - .io = threaded.io(), - .http_client = &http_client, - .config = .{ + var hf_vfs: zml.io.VFS.HF = try .init( + allocator, + threaded.io(), + &http_client, + .{ + .request_range_min = 8 * 1024, + .request_range_max = 128 * 1024 * 1024, + .hf_pagination_limit = 100, .auth = hf_auth, }, - }); + ); defer hf_vfs.deinit(); var vfs: zml.io.VFS = .init(allocator, threaded.io()); @@ -80,7 +85,7 @@ pub fn main() !void { const io = vfs.io(); - const default_uri = "hf://Qwen/Qwen3-8B"; + const default_uri = "hf://Qwen/Qwen3-8B@b968826"; // const default_uri = "file:///Users/hugo/Developer/Llama-3.1-8B-Instruct"; // const default_uri = "https://storage.googleapis.com/zig-vfs/Llama-3.1-8B-Instruct/"; // const default_uri = "http://9960x-5090x2:8003/"; @@ -96,73 +101,109 @@ pub fn main() !void { break :blk default_uri; }; - // { - // // const initial_path = "/Users/hugo/Developer/Llama-3.1-8B-Instruct/Llama-3.1-8B-Instruct"; - // // const initial_path = "file:///Users/hugo/Developer/Llama-3.1-8B-Instruct/Llama-3.1-8B-Instruct"; - // const initial_path = "https://storage.googleapis.com/zig-vfs/Llama-3.1-8B-Instruct/Llama-3.1-8B-Instruct"; - - // const llama_dir = try std.Io.Dir.openDir(.cwd(), io, initial_path, .{}); - // defer llama_dir.close(io); - - // const filepath = "../model.safetensors.index.json"; - - // const index_file = try llama_dir.openFile(io, filepath, .{}); - // defer index_file.close(io); - - // const stat = try index_file.stat(io); - // log.info("Opened local index file with size: {d} bytes", .{stat.size}); - // } - { - const repo = try std.Io.Dir.openDir(.cwd(), io, "file:///home/hugo/Llama-3.1-8B-Instruct/", .{}); + const local_uri = "hf://openai/gpt-oss-20b@6cee5e8"; + const repo = try std.Io.Dir.openDir(.cwd(), io, local_uri, .{}); defer repo.close(io); - const file = try repo.openFile(io, "model-00001-of-00004.safetensors", .{}); - defer file.close(io); + const index_file = try repo.openFile(io, "model.safetensors.index.json", .{}); + defer index_file.close(io); - const buf_size = 256 * 1024 * 1024; - const buf = try allocator.alignedAlloc(u8, .fromByteUnits(4 * 1024), buf_size); - defer allocator.free(buf); + const stat = try index_file.stat(io); + log.info("Opened local index file with size: {d} bytes", .{stat.size}); - var sha256: std.crypto.hash.sha2.Sha256 = .init(.{}); - const compute_sha = false; + const original = try repo.openDir(io, "original", .{}); + defer original.close(io); - var total_read: usize = 1 * 1024; - var bufs = [_][]u8{buf}; + const original_config = try original.openFile(io, "config.json", .{}); + defer original_config.close(io); - var timer: std.time.Timer = try .start(); - defer { - const elapsed = timer.read(); - log.info("File read in {d} ms", .{elapsed / std.time.ns_per_ms}); - } + // dir access / stat - while (true) { - const n = try io.vtable.fileReadStreaming(io.userdata, file, &bufs); - if (n == 0) break; - if (compute_sha) sha256.update(buf[0..n]); - total_read += n; - } - - const elapsed = timer.read(); - const read_mb = @as(f64, @floatFromInt(total_read)) / (1024.0 * 1024.0); - const read_time_s = @as(f64, @floatFromInt(elapsed)) / @as(f64, std.time.ns_per_s); - const throughput_mb_s = if (read_time_s > 0) read_mb / read_time_s else 0; - const throughput_gbps = if (read_time_s > 0) (read_mb * 8.0) / 1024.0 / read_time_s else 0; - log.info("Read {d:.2} MB in {d:.2} s = {d:.2} MB/s | {d:.2} Gbps", .{ - read_mb, - read_time_s, - throughput_mb_s, - throughput_gbps, - }); - - if (compute_sha) { - var hash: [32]u8 = undefined; - sha256.final(&hash); - log.info("SHA256: {x}", .{hash}); - } + const stat_config = try original_config.stat(io); + log.info("Opened local original/config.json file with size: {d} bytes", .{stat_config.size}); } { + const local_uri = "hf://openai/gpt-oss-20b@6cee5e8"; + const repo = try std.Io.Dir.openDir(.cwd(), io, local_uri, .{}); + defer repo.close(io); + + const original_config = try repo.openFile(io, "original/config.json", .{}); + defer original_config.close(io); + + const stat_config = try original_config.stat(io); + log.info("Opened local original/config.json file with size: {d} bytes", .{stat_config.size}); + } + + { + const local_uri = "hf://openai/gpt-oss-20b@6cee5e8"; + const original_uri = try std.fmt.allocPrint(allocator, "{s}/original", .{local_uri}); + defer allocator.free(original_uri); + + const repo = try std.Io.Dir.openDir(.cwd(), io, original_uri, .{}); + defer repo.close(io); + + const original_config = try repo.openFile(io, "config.json", .{}); + defer original_config.close(io); + + const stat_config = try original_config.stat(io); + log.info("Opened local original/config.json file with size: {d} bytes", .{stat_config.size}); + } + + // { + // const repo = try std.Io.Dir.openDir(.cwd(), io, "file:///home/hugo/Llama-3.1-8B-Instruct/", .{}); + // defer repo.close(io); + + // const file = try repo.openFile(io, "model-00001-of-00004.safetensors", .{}); + // defer file.close(io); + + // const buf_size = 256 * 1024 * 1024; + // const buf = try allocator.alignedAlloc(u8, .fromByteUnits(4 * 1024), buf_size); + // defer allocator.free(buf); + + // var sha256: std.crypto.hash.sha2.Sha256 = .init(.{}); + // const compute_sha = false; + + // var total_read: usize = 1 * 1024; + // var bufs = [_][]u8{buf}; + + // var timer: std.time.Timer = try .start(); + // defer { + // const elapsed = timer.read(); + // log.info("File read in {d} ms", .{elapsed / std.time.ns_per_ms}); + // } + + // while (true) { + // const n = try io.vtable.fileReadStreaming(io.userdata, file, &bufs); + // if (n == 0) break; + // if (compute_sha) sha256.update(buf[0..n]); + // total_read += n; + // } + + // const elapsed = timer.read(); + // const read_mb = @as(f64, @floatFromInt(total_read)) / (1024.0 * 1024.0); + // const read_time_s = @as(f64, @floatFromInt(elapsed)) / @as(f64, std.time.ns_per_s); + // const throughput_mb_s = if (read_time_s > 0) read_mb / read_time_s else 0; + // const throughput_gbps = if (read_time_s > 0) (read_mb * 8.0) / 1024.0 / read_time_s else 0; + // log.info("Read {d:.2} MB in {d:.2} s = {d:.2} MB/s | {d:.2} Gbps", .{ + // read_mb, + // read_time_s, + // throughput_mb_s, + // throughput_gbps, + // }); + + // if (compute_sha) { + // var hash: [32]u8 = undefined; + // sha256.final(&hash); + // log.info("SHA256: {x}", .{hash}); + // } + // } + + { + var pool: MemoryPool = try .init(allocator, threaded.async_limit.toInt() orelse 16, 256 * 1024 * 1024); + defer pool.deinit(io); + var timer: std.time.Timer = try .start(); defer { const elapsed = timer.read(); @@ -172,47 +213,37 @@ pub fn main() !void { var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, uri); defer registry.deinit(); - // var it = registry.iterator(); - // while (it.next()) |entry| { - // const tensor = entry.value_ptr.*; - // log.info("{f}", .{tensor}); - // } - log.info("Parsed {d} tensors", .{registry.tensors.count()}); - { - var err: ?anyerror = null; + var err: ?anyerror = null; - var pool: MemoryPool = try .init(allocator, threaded.async_limit.toInt() orelse 16, 256 * 1024 * 1024); - defer pool.deinit(io); + var group: std.Io.Group = .init; + defer group.cancel(io); - var group: std.Io.Group = .init; - defer group.cancel(io); + var timer_async_read: std.time.Timer = try .start(); + var it = registry.iterator(); - var timer_async_read: std.time.Timer = try .start(); - var it = registry.iterator(); - while (it.next()) |entry| { - const tensor_name = entry.key_ptr.*; - group.async(io, readTensor, .{ io, &pool, ®istry, tensor_name, &err }); - } - - group.wait(io); - - if (err) |e| { - log.err("Error reading tensor: {any}", .{e}); - return e; - } - - const elapsed = timer_async_read.read(); - const read_mb = @as(f64, @floatFromInt(registry.totalBytes())) / (1024.0 * 1024.0); - const read_time_s = @as(f64, @floatFromInt(elapsed)) / @as(f64, std.time.ns_per_s); - log.info("Throughput: {d:.2} MB in {d:.2} s = {d:.2} MB/s | {d:.2} Gbps", .{ - read_mb, - read_time_s, - read_mb / read_time_s, - (read_mb * 8.0) / 1024.0 / read_time_s, - }); + while (it.next()) |entry| { + const tensor_name = entry.key_ptr.*; + group.async(io, readTensor, .{ io, &pool, ®istry, tensor_name, &err }); } + + group.wait(io); + + if (err) |e| { + log.err("Error reading tensor: {any}", .{e}); + return e; + } + + const elapsed = timer_async_read.read(); + const read_mb = @as(f64, @floatFromInt(registry.totalBytes())) / (1024.0 * 1024.0); + const read_time_s = @as(f64, @floatFromInt(elapsed)) / @as(f64, std.time.ns_per_s); + log.info("Throughput: {d:.2} MB in {d:.2} s = {d:.2} MB/s | {d:.2} Gbps", .{ + read_mb, + read_time_s, + read_mb / read_time_s, + (read_mb * 8.0) / 1024.0 / read_time_s, + }); } // { diff --git a/zml2/safetensors.zig b/zml2/safetensors.zig index fd0d2418..be899ac1 100644 --- a/zml2/safetensors.zig +++ b/zml2/safetensors.zig @@ -19,108 +19,6 @@ pub const FileType = enum { unknown, }; -pub fn resolveFiletype(path: []const u8) FileType { - if (std.mem.endsWith(u8, path, ".safetensors.index.json")) { - return .index; - } else if (std.mem.endsWith(u8, path, ".safetensors")) { - return .safetensors; - } else { - return .unknown; - } -} - -pub const ModelPathResolutionError = error{ - FileNotFound, - InvalidPath, -} || std.mem.Allocator.Error; - -pub const ModelRepo = struct { - dir: std.Io.Dir, - path: []const u8, - - pub fn deinit(self: *ModelRepo, allocator: std.mem.Allocator, io: std.Io) void { - self.dir.close(io); - allocator.free(self.path); - } -}; - -pub fn resolveModelRepo( - allocator: std.mem.Allocator, - io: std.Io, - path: []const u8, -) ModelPathResolutionError!ModelRepo { - const resolved_path = try allocator.dupe(u8, path); - - if (std.mem.endsWith(u8, resolved_path, ".safetensors.index.json") or - std.mem.endsWith(u8, resolved_path, ".safetensors")) - { - const dir_path = std.fs.path.dirname(resolved_path) orelse { - allocator.free(resolved_path); - return ModelPathResolutionError.InvalidPath; - }; - const dir_path_dup = try allocator.dupe(u8, dir_path); - allocator.free(resolved_path); - - const dir = std.Io.Dir.openDir(.cwd(), io, dir_path_dup, .{}) catch |e| { - allocator.free(dir_path_dup); - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, - }; - }; - - return .{ - .dir = dir, - .path = dir_path_dup, - }; - } - - const dir = std.Io.Dir.openDir(.cwd(), io, resolved_path, .{}) catch |e| { - allocator.free(resolved_path); - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, - }; - }; - - return .{ - .dir = dir, - .path = resolved_path, - }; -} - -pub const ModelEntrypoint = struct { - file: std.Io.File, - path: []const u8, - - pub fn deinit(self: *ModelEntrypoint, allocator: std.mem.Allocator, io: std.Io) void { - self.file.close(io); - allocator.free(self.path); - } -}; - -pub fn resolveModelEntrypoint(allocator: std.mem.Allocator, io: std.Io, repo: ModelRepo) ModelPathResolutionError!ModelEntrypoint { - { - const index_name = "model.safetensors.index.json"; - log.warn("Trying to resolve model entrypoint: {s}/{s}", .{ repo.path, index_name }); - if (repo.dir.openFile(io, index_name, .{ .mode = .read_only })) |index_file| { - const index_path = try std.fmt.allocPrint(allocator, "{s}/{s}", .{ repo.path, index_name }); - return .{ .file = index_file, .path = index_path }; - } else |_| {} - } - - { - const model_name = "model.safetensors"; - log.warn("Trying to resolve model entrypoint: {s}/{s}", .{ repo.path, model_name }); - if (repo.dir.openFile(io, model_name, .{ .mode = .read_only })) |model_file| { - const model_path = try std.fmt.allocPrint(allocator, "{s}/{s}", .{ repo.path, model_name }); - return .{ .file = model_file, .path = model_path }; - } else |_| {} - } - - return ModelPathResolutionError.FileNotFound; -} - pub fn fetchRegistry( allocator: std.mem.Allocator, io: std.Io, @@ -623,6 +521,124 @@ const AsyncParseSafetensorsIndexFile = struct { } }; +pub fn resolveFiletype(path: []const u8) FileType { + if (std.mem.endsWith(u8, path, ".safetensors.index.json")) { + return .index; + } else if (std.mem.endsWith(u8, path, ".safetensors")) { + return .safetensors; + } else { + return .unknown; + } +} + +pub const ModelPathResolutionError = error{ + FileNotFound, + InvalidPath, +} || std.mem.Allocator.Error; + +pub const ModelRepo = struct { + dir: std.Io.Dir, + path: []const u8, + + pub fn deinit(self: *ModelRepo, allocator: std.mem.Allocator, io: std.Io) void { + self.dir.close(io); + allocator.free(self.path); + } +}; + +// Helper to ensure trailing slash from the given user input path. +fn ensureTrailingSlash(allocator: std.mem.Allocator, input: []const u8) ![]u8 { + if (input.len == 0 or input[input.len - 1] == std.fs.path.sep) { + return try allocator.dupe(u8, input); + } else { + var out = try allocator.alloc(u8, input.len + 1); + @memcpy(out[0..input.len], input); + out[input.len] = std.fs.path.sep; + return out; + } +} + +pub fn resolveModelRepo( + allocator: std.mem.Allocator, + io: std.Io, + path: []const u8, +) ModelPathResolutionError!ModelRepo { + const resolved_path = try allocator.dupe(u8, path); + + if (std.mem.endsWith(u8, resolved_path, ".safetensors.index.json") or + std.mem.endsWith(u8, resolved_path, ".safetensors")) + { + const dir_path = std.fs.path.dirname(resolved_path) orelse { + allocator.free(resolved_path); + return ModelPathResolutionError.InvalidPath; + }; + + const dir_path_with_slash = try ensureTrailingSlash(allocator, dir_path); + allocator.free(resolved_path); + + const dir = std.Io.Dir.openDir(.cwd(), io, dir_path_with_slash, .{}) catch |e| { + allocator.free(dir_path_with_slash); + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; + }; + + return .{ + .dir = dir, + .path = dir_path_with_slash, + }; + } + + const path_with_slash = try ensureTrailingSlash(allocator, resolved_path); + allocator.free(resolved_path); + + const dir = std.Io.Dir.openDir(.cwd(), io, path_with_slash, .{}) catch |e| { + allocator.free(path_with_slash); + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; + }; + + return .{ + .dir = dir, + .path = path_with_slash, + }; +} + +pub const ModelEntrypoint = struct { + file: std.Io.File, + path: []const u8, + + pub fn deinit(self: *ModelEntrypoint, allocator: std.mem.Allocator, io: std.Io) void { + self.file.close(io); + allocator.free(self.path); + } +}; + +pub fn resolveModelEntrypoint(allocator: std.mem.Allocator, io: std.Io, repo: ModelRepo) ModelPathResolutionError!ModelEntrypoint { + { + const index_name = "model.safetensors.index.json"; + log.warn("Trying to resolve model entrypoint: {s}{s}", .{ repo.path, index_name }); + if (repo.dir.openFile(io, index_name, .{ .mode = .read_only })) |index_file| { + const index_path = try std.fmt.allocPrint(allocator, "{s}{s}", .{ repo.path, index_name }); + return .{ .file = index_file, .path = index_path }; + } else |_| {} + } + + { + const model_name = "model.safetensors"; + log.warn("Trying to resolve model entrypoint: {s}{s}", .{ repo.path, model_name }); + if (repo.dir.openFile(io, model_name, .{ .mode = .read_only })) |model_file| { + const model_path = try std.fmt.allocPrint(allocator, "{s}{s}", .{ repo.path, model_name }); + return .{ .file = model_file, .path = model_path }; + } else |_| {} + } + + return ModelPathResolutionError.FileNotFound; +} + pub const Metadata = union(enum) { null: void, int: i64, From 09df6d757d535bcc842d9a956c9a3fc7fb0a6df9 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 19 Dec 2025 15:18:36 +0100 Subject: [PATCH 158/881] Make Llama more async --- examples/llama/main.zig | 485 ++++++++++++++++++++++++---------------- zml2/zml.zig | 1 + 2 files changed, 290 insertions(+), 196 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 6ee1253f..65a281fa 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -31,6 +31,295 @@ const params = clap.parseParamsComptime( \\--sharding default: true: sharding on or off ); +pub fn main() !void { + log.info("LLama was compiled with {}", .{@import("builtin").mode}); + + const allocator = std.heap.c_allocator; + //var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + //defer _ = gpa.deinit(); + + //const allocator = gpa.allocator(); + + var threaded: std.Io.Threaded = .init(allocator); + defer threaded.deinit(); + + var vfs_file: zml.io.VFS.File = .init(allocator, threaded.io(), .{}); + + var http_client: std.http.Client = .{ + .allocator = allocator, + .io = threaded.io(), + .connection_pool = .{ + .free_size = threaded.async_limit.toInt() orelse 16, + }, + }; + + try http_client.initDefaultProxies(allocator); + defer http_client.deinit(); + + var vfs_http: zml.io.VFS.HTTP = try .init(allocator, threaded.io(), &http_client, .{}); + defer vfs_http.deinit(); + + var hf_auth: zml.io.VFS.HF.Auth = try .auto(allocator, threaded.io()); + defer hf_auth.deinit(allocator); + + var hf_vfs: zml.io.VFS.HF = try .init( + allocator, + threaded.io(), + &http_client, + .{ + .request_range_min = 8 * 1024, + .request_range_max = 128 * 1024 * 1024, + .hf_pagination_limit = 100, + .auth = hf_auth, + }, + ); + defer hf_vfs.deinit(); + + var vfs: zml.io.VFS = .init(allocator, threaded.io()); + defer vfs.deinit(); + + try vfs.register("file", vfs_file.io()); + try vfs.register("http", vfs_http.io()); + try vfs.register("https", vfs_http.io()); + try vfs.register("hf", hf_vfs.io()); + + const io = vfs.io(); + + zml.init(); + defer zml.deinit(); + + const parsers = comptime .{ + .BOOL = bool_parser, + .UINT = clap.parsers.int(u32, 0), + .STRING = clap.parsers.string, + .PATH = clap.parsers.string, + }; + var diag: clap.Diagnostic = .{}; + var stderr_buffer: [1024]u8 = undefined; + var stderr = std.fs.File.stderr().writer(&stderr_buffer); + defer stderr.interface.flush() catch {}; + + var cli = clap.parse(clap.Help, ¶ms, parsers, .{ + .diagnostic = &diag, + .allocator = allocator, + }) catch |err| { + diag.report(&stderr.interface, err) catch {}; + stderr.interface.writeAll("usage: ") catch {}; + clap.usage(&stderr.interface, clap.Help, ¶ms) catch {}; + stderr.interface.writeAll("\n") catch {}; + return; + }; + defer cli.deinit(); + + if (cli.args.help != 0) { + clap.help(&stderr.interface, clap.Help, ¶ms, .{}) catch {}; + return; + } + + const hf_model_path = cli.args.@"hf-model-path" orelse { + log.err("Missing --hf-model-path", .{}); + return; + }; + + log.info("Resolving Model repo", .{}); + var repo = try zml.safetensors.resolveModelRepo(allocator, io, hf_model_path); + defer repo.deinit(allocator, io); + + const parsed_config = try parseConfig(allocator, io, repo.dir); + defer parsed_config.deinit(); + const config = parsed_config.value; + + // initialize ZML platform with optional create options + // eg: --create-options='{"cuda":{"allocator":{"bfc":{"memory_fraction": 0.99}}}}' + var platform = b: { + const create_opts_json = cli.args.@"create-options" orelse "{}"; + const create_opts = try std.json.parseFromSlice(zml.platform.CreateOptions, allocator, create_opts_json, .{}); + defer create_opts.deinit(); + const platform: zml.Platform = try .auto(io, create_opts.value); + break :b platform; + }; + defer platform.deinit(); + + var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); + defer registry.deinit(); + + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + defer store.deinit(); + + // Write metadata from the config file into the LlamaLm struct. + const seq_len: u32 = cli.args.@"seq-len" orelse 256; + const llama_options: llama.LlamaLM.Options = .{ + .max_seq_len = seq_len, + .sampling_strategy = .{ + .topk = 2, + .temperature = 1.0, + }, + }; + + // Initialize the Llama struct and map the content of the .safetensors to the model tensors. + const llama_model: llama.LlamaLM = try .init(allocator, store.view(), config, llama_options); + defer llama_model.deinit(allocator); + + // Specify shapes of input arguments + const dtype = llama_model.model.embed_tokens.weight.dtype(); + const llama_parameters: LlamaParameters = .{ + .prefill_tokens = .init(.{ .s = llama_options.max_seq_len }, .u32), + .decode_tokens = .init(.{ .s = 1 }, .u32), + .token_index = .init(.{}, .u32), + .kv_cache = .init(zml.Shape.init(.{ + .layer = llama_model.model.layers.len, + .k = seq_len, + .h = config.num_key_value_heads, + .hd = config.head_dim orelse @divExact(config.hidden_size, config.num_attention_heads), + }, dtype)), + .rng = .init(), + }; + + var compiled_model_result_future = io.async(struct { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: zml.Platform, llama_model_: llama.LlamaLM, llama_parameters_: LlamaParameters) !CompileModelResult { + return compileModel(allocator_, io_, platform_, llama_model_, llama_parameters_); + } + }.call, .{ allocator, io, platform, llama_model, llama_parameters }); + errdefer if (compiled_model_result_future.cancel(io)) |v| { + defer v.prefill_exe.deinit(); + defer v.decode_exe.deinit(); + } else |_| {}; + + var llama_buffers_future = io.async(struct { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: zml.Platform, store_: *zml.io.TensorStore, llama_model_: llama.LlamaLM) !zml.Bufferized(llama.LlamaLM) { + return loadModelBuffers(allocator_, io_, platform_, store_, llama_model_); + } + }.call, .{ allocator, io, platform, &store, llama_model }); + errdefer b: { + var v = llama_buffers_future.cancel(io) catch break :b; + LlamaLM.unloadBuffers(&v, allocator); + } + + const compiled_model_result = try compiled_model_result_future.await(io); + defer compiled_model_result.prefill_exe.deinit(); + defer compiled_model_result.decode_exe.deinit(); + + var llama_buffers = try llama_buffers_future.await(io); + defer LlamaLM.unloadBuffers(&llama_buffers, allocator); + + log.info("Creating KvCache", .{}); + var kv_cache_buffers = try llama_parameters.kv_cache.initBuffer(io, platform); + defer llama.KvCache.deinitBuffer(&kv_cache_buffers); + + var tokenizer = try loadTokenizer(allocator, io, repo.dir); + defer tokenizer.deinit(); + + const prompt = cli.args.prompt orelse "What is the capital of France?"; + log.info("✅\tPrompt: {s}", .{prompt}); + + // Unbuffered writing of the tokens to stdout. + var stdout = std.fs.File.stdout().writer(&.{}); + + const now = try std.Io.Clock.now(.boot, io); + const seed: u128 = cli.args.seed orelse @intCast(now.nanoseconds); + const skip_llama3_encoding = cli.args.@"no-llama3" orelse false; + + try generateText( + allocator, + io, + llama_buffers, + compiled_model_result.prefill_exe, + compiled_model_result.decode_exe, + &kv_cache_buffers, + tokenizer, + config, + llama_options, + seed, + prompt[0..], + skip_llama3_encoding, + &stdout.interface, + platform, + ); +} + +fn parseConfig(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !std.json.Parsed(LlamaLM.Config) { + var timer = try stdx.time.Timer.start(); + log.info("Loading model config", .{}); + defer log.info("Loaded model config [{D}]", .{timer.read()}); + const parsed_config = blk: { + const config_json_file = try dir.openFile(io, "config.json", .{}); + defer config_json_file.close(io); + var config_json_buffer: [256]u8 = undefined; + var config_reader = config_json_file.reader(io, &config_json_buffer); + var reader = std.json.Reader.init(allocator, &config_reader.interface); + defer reader.deinit(); + break :blk try std.json.parseFromTokenSource(llama.LlamaLM.Config, allocator, &reader, .{ .ignore_unknown_fields = true }); + }; + errdefer parsed_config.deinit(); + + return parsed_config; +} + +fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !zml.tokenizer.Tokenizer { + var timer = try stdx.time.Timer.start(); + log.info("Loading tokenizer", .{}); + defer log.info("Loaded tokenizer [{D}]", .{timer.read()}); + const bytes = b: { + const file = try dir.openFile(io, "tokenizer.json", .{}); + defer file.close(io); + + var reader = file.reader(io, &.{}); + var writer: std.Io.Writer.Allocating = .init(allocator); + defer writer.deinit(); + _ = try reader.interface.streamRemaining(&writer.writer); + break :b try writer.toOwnedSlice(); + }; + defer allocator.free(bytes); + + return .{ .hftokenizers = try zml.tokenizer.hftokenizers.HFTokenizer.fromBytes(bytes) }; +} + +const LlamaParameters = struct { + prefill_tokens: zml.Tensor, + decode_tokens: zml.Tensor, + token_index: zml.Tensor, + kv_cache: llama.KvCache, + rng: zml.Tensor.Rng, +}; + +const CompileModelResult = struct { + prefill_exe: zml.Exe, + decode_exe: zml.Exe, +}; + +fn compileModel(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform, llama_model: llama.LlamaLM, parameters: LlamaParameters) !CompileModelResult { + var timer = try stdx.time.Timer.start(); + log.info("Compiling model", .{}); + defer log.info("Compiled model [{D}]", .{timer.read()}); + + // Compile the model twice, one for prefill, one for generation. + var prefill_future = io.async(struct { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: zml.Platform, llama_model_: llama.LlamaLM, parameters_: LlamaParameters) !zml.Exe { + return platform_.compileModel(allocator_, io_, llama.LlamaLM.forward, llama_model_, .{ parameters_.prefill_tokens, parameters_.token_index, parameters_.kv_cache, parameters_.rng }); + } + }.call, .{ allocator, io, platform, llama_model, parameters }); + errdefer if (prefill_future.cancel(io)) |v| v.deinit() else |_| {}; + + var decode_future = io.async(struct { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: zml.Platform, llama_model_: llama.LlamaLM, parameters_: LlamaParameters) !zml.Exe { + return platform_.compileModel(allocator_, io_, llama.LlamaLM.forward, llama_model_, .{ parameters_.decode_tokens, parameters_.token_index, parameters_.kv_cache, parameters_.rng }); + } + }.call, .{ allocator, io, platform, llama_model, parameters }); + errdefer if (decode_future.cancel(io)) |v| v.deinit() else |_| {}; + + const prefill_exe = try prefill_future.await(io); + const decode_exe = try decode_future.await(io); + + return .{ .prefill_exe = prefill_exe, .decode_exe = decode_exe }; +} + +fn loadModelBuffers(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform, store: *zml.io.TensorStore, llama_model: llama.LlamaLM) !zml.Bufferized(llama.LlamaLM) { + var timer = try stdx.time.Timer.start(); + log.info("Loading model", .{}); + defer log.info("Loaded model [{D}]", .{timer.read()}); + return llama_model.loadBuffers(allocator, io, store.view(), platform); +} + pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: LlamaLM.Config, prompt: []const u8, skip_llama3_encoding: bool) ![]u32 { var encoder = try tokenizer.encoder(); defer encoder.deinit(); @@ -173,202 +462,6 @@ pub fn generateText( log.info("✅ Generated {} tokens in {D}: {:.3}tok/s", .{ num_tokens_generated, duration.ns, speed }); } -pub fn main() !void { - log.info(" LLama was compiled with {}", .{@import("builtin").mode}); - - //const allocator = std.heap.c_allocator; - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - - const allocator = gpa.allocator(); - - var threaded: std.Io.Threaded = .init(allocator); - defer threaded.deinit(); - - var vfs_file: zml.io.VFS.File = .init(allocator, threaded.io(), .{}); - - var http_client: std.http.Client = .{ - .allocator = allocator, - .io = threaded.io(), - .connection_pool = .{ - .free_size = threaded.async_limit.toInt() orelse 16, - }, - }; - - try http_client.initDefaultProxies(allocator); - defer http_client.deinit(); - - var vfs_http: zml.io.VFS.HTTP = try .init(allocator, threaded.io(), &http_client, .{}); - defer vfs_http.deinit(); - - var vfs: zml.io.VFS = .init(allocator, threaded.io()); - defer vfs.deinit(); - - try vfs.register("file", vfs_file.io()); - try vfs.register("http", vfs_http.io()); - try vfs.register("https", vfs_http.io()); - - const io = vfs.io(); - - zml.init(); - defer zml.deinit(); - - const parsers = comptime .{ - .BOOL = bool_parser, - .UINT = clap.parsers.int(u32, 0), - .STRING = clap.parsers.string, - .PATH = clap.parsers.string, - }; - var diag: clap.Diagnostic = .{}; - var stderr_buffer: [1024]u8 = undefined; - var stderr = std.fs.File.stderr().writer(&stderr_buffer); - defer stderr.interface.flush() catch {}; - - var cli = clap.parse(clap.Help, ¶ms, parsers, .{ - .diagnostic = &diag, - .allocator = allocator, - }) catch |err| { - diag.report(&stderr.interface, err) catch {}; - stderr.interface.writeAll("usage: ") catch {}; - clap.usage(&stderr.interface, clap.Help, ¶ms) catch {}; - stderr.interface.writeAll("\n") catch {}; - return; - }; - defer cli.deinit(); - - if (cli.args.help != 0) { - clap.help(&stderr.interface, clap.Help, ¶ms, .{}) catch {}; - return; - } - - const hf_model_path = cli.args.@"hf-model-path" orelse { - log.err("Missing --hf-model-path", .{}); - return; - }; - - var repo = try zml.safetensors.resolveModelRepo(allocator, io, hf_model_path); - defer repo.deinit(allocator, io); - - const parsed_config = blk: { - const config_json_file = try repo.dir.openFile(io, "config.json", .{}); - defer config_json_file.close(io); - var config_json_buffer: [256]u8 = undefined; - var config_reader = config_json_file.reader(io, &config_json_buffer); - var reader = std.json.Reader.init(allocator, &config_reader.interface); - defer reader.deinit(); - break :blk try std.json.parseFromTokenSource(llama.LlamaLM.Config, allocator, &reader, .{ .ignore_unknown_fields = true }); - }; - defer parsed_config.deinit(); - const config = parsed_config.value; - - // initialize ZML platform with optional create options - // eg: --create-options='{"cuda":{"allocator":{"bfc":{"memory_fraction": 0.99}}}}' - var platform = b: { - const create_opts_json = cli.args.@"create-options" orelse "{}"; - const create_opts = try std.json.parseFromSlice(zml.platform.CreateOptions, allocator, create_opts_json, .{}); - defer create_opts.deinit(); - const platform: zml.Platform = try .auto(io, create_opts.value); - break :b platform; - }; - defer platform.deinit(); - - var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); - defer registry.deinit(); - - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); - defer store.deinit(); - - // Write metadata from the config file into the LlamaLm struct. - const seq_len: u32 = cli.args.@"seq-len" orelse 256; - const llama_options: llama.LlamaLM.Options = .{ - .max_seq_len = seq_len, - .sampling_strategy = .{ - .topk = 2, - .temperature = 1.0, - }, - }; - - // Initialize the Llama struct and map the content of the .safetensors to the model tensors. - const llama_model: llama.LlamaLM = try .init(allocator, store.view(), config, llama_options); - defer llama_model.deinit(allocator); - - // Specify shapes of input arguments - const prefill_tokens: zml.Tensor = .init(.{ .s = llama_options.max_seq_len }, .u32); - const decode_tokens: zml.Tensor = .init(.{ .s = 1 }, .u32); - const token_index: zml.Tensor = .init(.{}, .u32); - - const dtype = llama_model.model.embed_tokens.weight.dtype(); - const kv_cache: llama.KvCache = .init(zml.Shape.init(.{ - .layer = llama_model.model.layers.len, - .k = seq_len, - .h = config.num_key_value_heads, - .hd = config.head_dim orelse @divExact(config.hidden_size, config.num_attention_heads), - }, dtype)); - const rng: zml.Tensor.Rng = .init(); - - // Compile the model twice, one for prefill, one for generation. - var prefill_exe = try platform.compileModel(allocator, io, llama.LlamaLM.forward, llama_model, .{ prefill_tokens, token_index, kv_cache, rng }); - defer prefill_exe.deinit(); - var decode_exe = try platform.compileModel(allocator, io, llama.LlamaLM.forward, llama_model, .{ decode_tokens, token_index, kv_cache, rng }); - defer decode_exe.deinit(); - - var llama_buffers = try llama_model.loadBuffers(allocator, io, store.view(), platform); - defer LlamaLM.unloadBuffers(&llama_buffers, allocator); - - log.info("Creating KvCache", .{}); - var kv_cache_buffers = try kv_cache.initBuffer(io, platform); - defer llama.KvCache.deinitBuffer(&kv_cache_buffers); - - var tokenizer = blk: { - log.info("Loading tokenizer", .{}); - var timer = try stdx.time.Timer.start(); - defer log.info("Loaded tokenizer [{D}]", .{timer.read()}); - const bytes = b: { - const file = try repo.dir.openFile(io, "tokenizer.json", .{}); - defer file.close(io); - - var reader = file.reader(io, &.{}); - var writer: std.Io.Writer.Allocating = .init(allocator); - defer writer.deinit(); - _ = try reader.interface.streamRemaining(&writer.writer); - break :b try writer.toOwnedSlice(); - }; - defer allocator.free(bytes); - - const tokenizer: zml.tokenizer.Tokenizer = .{ .hftokenizers = try zml.tokenizer.hftokenizers.HFTokenizer.fromBytes(bytes) }; - break :blk tokenizer; - }; - errdefer tokenizer.deinit(); - - const prompt = cli.args.prompt orelse "What is the capital of France?"; - log.info("✅\tPrompt: {s}", .{prompt}); - - // Unbuffered writing of the tokens to stdout. - var stdout = std.fs.File.stdout().writer(&.{}); - - const now = try std.Io.Clock.now(.boot, io); - std.log.info("now: {}", .{now.nanoseconds}); - const seed: u128 = cli.args.seed orelse @intCast(now.nanoseconds); - const skip_llama3_encoding = cli.args.@"no-llama3" orelse false; - - try generateText( - allocator, - io, - llama_buffers, - prefill_exe, - decode_exe, - &kv_cache_buffers, - tokenizer, - config, - llama_options, - seed, - prompt[0..], - skip_llama3_encoding, - &stdout.interface, - platform, - ); -} - fn bool_parser(in: []const u8) error{}!bool { return std.mem.indexOfScalar(u8, "tTyY1", in[0]) != null; } diff --git a/zml2/zml.zig b/zml2/zml.zig index 1df477a5..f56946a3 100644 --- a/zml2/zml.zig +++ b/zml2/zml.zig @@ -15,6 +15,7 @@ pub const dtype = @import("dtype.zig"); pub const Data = dtype.Data; pub const DataType = dtype.DataType; pub const exe = @import("exe.zig"); +pub const Exe = exe.Exe; pub const floats = @import("floats.zig"); pub const meta = @import("meta.zig"); pub const mlir = @import("mlirx.zig"); From 5d2dc7c8c1fa1eab804964a9ea66be600b7244a7 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Mon, 22 Dec 2025 11:29:55 +0100 Subject: [PATCH 159/881] io: fixes (#369) Co-authored-by: Corentin Godeau --- examples/llama/main.zig | 1 + zml2/io/vfs/hf.zig | 40 ++++++++++++++++++++++++++++++---------- zml2/io/vfs/http.zig | 26 +++++++++++++++++--------- zml2/safetensors.zig | 4 +--- 4 files changed, 49 insertions(+), 22 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 65a281fa..f7b27734 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -241,6 +241,7 @@ fn parseConfig(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !std.j var timer = try stdx.time.Timer.start(); log.info("Loading model config", .{}); defer log.info("Loaded model config [{D}]", .{timer.read()}); + const parsed_config = blk: { const config_json_file = try dir.openFile(io, "config.json", .{}); defer config_json_file.close(io); diff --git a/zml2/io/vfs/hf.zig b/zml2/io/vfs/hf.zig index 47683ba7..c7746658 100644 --- a/zml2/io/vfs/hf.zig +++ b/zml2/io/vfs/hf.zig @@ -831,17 +831,17 @@ pub const HF = struct { sub_path: []const u8, ) std.Io.File.OpenError!std.Io.File { const dir_handle = self.dirHandle(dir) orelse { - log.err("Directory handle not found for dirOpenFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); + log.warn("Directory handle not found for dirOpenFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); return std.Io.File.OpenError.FileNotFound; }; const handle = self.initFileHandle(dir_handle, sub_path) catch { - log.err("Failed to init FileHandle in openFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); + log.warn("Failed to init FileHandle in openFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); return std.Io.File.OpenError.SystemResources; }; const handle_id = self.registerFileHandle(handle) catch { - log.err("Failed to register FileHandle in openFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); + log.warn("Failed to register FileHandle in openFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); return std.Io.File.OpenError.SystemResources; }; @@ -854,17 +854,17 @@ pub const HF = struct { sub_path: []const u8, ) std.Io.Dir.OpenError!std.Io.Dir { const dir_handle = self.dirHandle(dir) orelse { - log.err("Directory handle not found for dirOpenFile", .{}); + log.warn("Directory handle not found for dirOpenFile", .{}); return std.Io.Dir.OpenError.FileNotFound; }; const handle = self.initDirHandle(dir_handle.uri, sub_path) catch { - log.err("Failed to init DirHandle in openDir with dir={any} sub_path={s}", .{ dir.handle, sub_path }); + log.warn("Failed to init DirHandle in openDir with dir={any} sub_path={s}", .{ dir.handle, sub_path }); return std.Io.Dir.OpenError.SystemResources; }; const handle_id = self.registerDirHandle(handle) catch { - log.err("Failed to register DirHandle in openDir with dir={any} sub_path={s}", .{ dir.handle, sub_path }); + log.warn("Failed to register DirHandle in openDir with dir={any} sub_path={s}", .{ dir.handle, sub_path }); return std.Io.Dir.OpenError.SystemResources; }; @@ -1030,9 +1030,21 @@ pub const HF = struct { return std.Io.File.Reader.Error.Unexpected; }; + self.http_headers[1] = .{ .name = "Range", .value = range_header }; + + var extra_headers: []std.http.Header = &.{}; + switch (self.config.auth) { + .hf_token => |_| { + extra_headers = self.http_headers[0..2]; + }, + else => { + extra_headers = self.http_headers[1..2]; + }, + } + request.* = self.client.request(.GET, api_uri, .{ .headers = .{ .accept_encoding = .{ .override = "identity" } }, - .extra_headers = &.{.{ .name = "Range", .value = range_header }}, + .extra_headers = extra_headers, }) catch |err| { log.err("Failed to create GET request: {}", .{err}); return std.Io.File.Reader.Error.SystemResources; @@ -1067,9 +1079,17 @@ pub const HF = struct { }); if (response.head.status != .partial_content and response.head.status != .ok) { - request.deinit(); - self.allocator.destroy(request); - return std.Io.File.Reader.Error.Unexpected; + log.err("Unexpected HTTP status ({s}) for GET {s}", .{ @tagName(response.head.status), api_url }); + var it = response.head.iterateHeaders(); + while (it.next()) |hdr| { + log.err("Header: {s}: {s}", .{ hdr.name, hdr.value }); + } + + if (response.head.status == .unauthorized) { + return std.Io.File.Reader.Error.AccessDenied; + } else { + return std.Io.File.Reader.Error.Unexpected; + } } handle.request = request; diff --git a/zml2/io/vfs/http.zig b/zml2/io/vfs/http.zig index f26ffa29..27df2698 100644 --- a/zml2/io/vfs/http.zig +++ b/zml2/io/vfs/http.zig @@ -264,17 +264,17 @@ pub const HTTP = struct { sub_path: []const u8, ) std.Io.File.OpenError!std.Io.File { const dir_handle = self.dirHandle(dir) orelse { - log.err("Directory handle not found for dirOpenFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); + log.warn("Directory handle not found for dirOpenFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); return std.Io.File.OpenError.FileNotFound; }; const handle = self.initFileHandle(dir_handle, sub_path) catch { - log.err("Failed to init FileHandle in openFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); + log.warn("Failed to init FileHandle in openFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); return std.Io.File.OpenError.SystemResources; }; const handle_id = self.registerFileHandle(handle) catch { - log.err("Failed to register FileHandle in openFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); + log.warn("Failed to register FileHandle in openFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); return std.Io.File.OpenError.SystemResources; }; @@ -287,17 +287,17 @@ pub const HTTP = struct { sub_path: []const u8, ) std.Io.Dir.OpenError!std.Io.Dir { const dir_handle = self.dirHandle(dir) orelse { - log.err("Directory handle not found for dirOpenFile", .{}); + log.warn("Directory handle not found for dirOpenFile", .{}); return std.Io.Dir.OpenError.FileNotFound; }; const handle = self.initDirHandle(dir_handle.uri, sub_path) catch { - log.err("Failed to init DirHandle in openDir with dir={any} sub_path={s}", .{ dir.handle, sub_path }); + log.warn("Failed to init DirHandle in openDir with dir={any} sub_path={s}", .{ dir.handle, sub_path }); return std.Io.Dir.OpenError.SystemResources; }; const handle_id = self.registerDirHandle(handle) catch { - log.err("Failed to register DirHandle in openDir with dir={any} sub_path={s}", .{ dir.handle, sub_path }); + log.warn("Failed to register DirHandle in openDir with dir={any} sub_path={s}", .{ dir.handle, sub_path }); return std.Io.Dir.OpenError.SystemResources; }; @@ -473,9 +473,17 @@ pub const HTTP = struct { }); if (response.head.status != .partial_content and response.head.status != .ok) { - request.deinit(); - self.allocator.destroy(request); - return std.Io.File.Reader.Error.Unexpected; + log.err("Unexpected HTTP status ({s}) for GET {s}", .{ @tagName(response.head.status), handle.uri }); + var it = response.head.iterateHeaders(); + while (it.next()) |hdr| { + log.err("Header: {s}: {s}", .{ hdr.name, hdr.value }); + } + + if (response.head.status == .unauthorized) { + return std.Io.File.Reader.Error.AccessDenied; + } else { + return std.Io.File.Reader.Error.Unexpected; + } } handle.request = request; diff --git a/zml2/safetensors.zig b/zml2/safetensors.zig index be899ac1..7c47992b 100644 --- a/zml2/safetensors.zig +++ b/zml2/safetensors.zig @@ -256,7 +256,7 @@ pub const TensorRegistry = struct { if (gop.found_existing) { gop.value_ptr.*.deinit(allocator); gop.value_ptr.* = value; - log.warn("Overwrote existing metadata key={s} with value={f}", .{ key, value }); + log.info("Overwrote existing metadata key={s} with value={f}", .{ key, value }); } else { gop.value_ptr.* = value; log.debug("Added new metadata key={s} with value={f}", .{ key, value }); @@ -620,7 +620,6 @@ pub const ModelEntrypoint = struct { pub fn resolveModelEntrypoint(allocator: std.mem.Allocator, io: std.Io, repo: ModelRepo) ModelPathResolutionError!ModelEntrypoint { { const index_name = "model.safetensors.index.json"; - log.warn("Trying to resolve model entrypoint: {s}{s}", .{ repo.path, index_name }); if (repo.dir.openFile(io, index_name, .{ .mode = .read_only })) |index_file| { const index_path = try std.fmt.allocPrint(allocator, "{s}{s}", .{ repo.path, index_name }); return .{ .file = index_file, .path = index_path }; @@ -629,7 +628,6 @@ pub fn resolveModelEntrypoint(allocator: std.mem.Allocator, io: std.Io, repo: Mo { const model_name = "model.safetensors"; - log.warn("Trying to resolve model entrypoint: {s}{s}", .{ repo.path, model_name }); if (repo.dir.openFile(io, model_name, .{ .mode = .read_only })) |model_file| { const model_path = try std.fmt.allocPrint(allocator, "{s}{s}", .{ repo.path, model_name }); return .{ .file = model_file, .path = model_path }; From 058bd48abeb26d6b72775785b6475a7e66a48166 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 22 Dec 2025 11:51:39 +0100 Subject: [PATCH 160/881] Implemented formatNumber on Slice and ConstSlice --- zml2/slice.zig | 178 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 172 insertions(+), 6 deletions(-) diff --git a/zml2/slice.zig b/zml2/slice.zig index 4acafb7d..ff13f6bf 100644 --- a/zml2/slice.zig +++ b/zml2/slice.zig @@ -1,6 +1,7 @@ const std = @import("std"); const stdx = @import("stdx"); +const floats = @import("floats.zig"); const DataType = @import("dtype.zig").DataType; const Shape = @import("shape.zig").Shape; @@ -53,9 +54,7 @@ pub const Slice = struct { } pub fn formatNumber(self: @This(), writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { - _ = self; // autofix - _ = n; // autofix - return writer.print("TODO", .{}); + return self.constSlice().formatNumber(writer, n); } }; @@ -96,8 +95,175 @@ pub const ConstSlice = struct { } pub fn formatNumber(self: @This(), writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { - _ = self; // autofix - _ = n; // autofix - return writer.print("TODO", .{}); + return self.prettyPrintIndented(writer, 4, 0, n); + } + + pub fn prettyPrint(self: ConstSlice, writer: *std.Io.Writer, options: std.fmt.Number) !void { + return self.prettyPrintIndented(writer, 4, 0, options); + } + + fn prettyPrintIndented(self: ConstSlice, writer: *std.Io.Writer, num_rows: u8, indent_level: u8, options: std.fmt.Number) !void { + if (self.shape.rank() == 0) { + // Special case input tensor is a scalar + return switch (self.dtype()) { + inline else => |dt| { + const val: dt.toZigType() = self.items(dt.toZigType())[0]; + return switch (comptime dt.class()) { + // Since we have custom floats, we need to explicitly convert to float32 ourselves. + .float => stdx.fmt.formatFloat(floats.floatCast(f32, val), options, writer), + .integer => stdx.fmt.formatInt(val, options, writer), + .bool => stdx.fmt.formatBool(val, options, writer), + .complex => stdx.fmt.formatComplex(val, options, writer), + }; + }, + }; + } + + if (self.shape.rank() == 1) { + // Print a contiguous slice of items from the buffer in one line. + // The number of items printed is controlled by the user through format syntax. + try writer.splatByteAll(' ', indent_level); + switch (self.dtype()) { + inline else => |dt| { + const T = dt.toZigType(); + // TODO: handle negative strides + const elem_strides: i64 = self.shape.computeElementStrides().get(0); + const values = self.items(T); + switch (comptime dt.class()) { + .float => try stdx.fmt.formatFloatSlice(values, options, elem_strides, writer), + .integer => try stdx.fmt.formatIntSlice(values, options, elem_strides, writer), + .complex => try stdx.fmt.formatComplexSlice(values, options, elem_strides, writer), + .bool => try stdx.fmt.formatBoolSlice(values, options, elem_strides, writer), + } + }, + } + return; + } + // TODO: consider removing the \n if dim is 1 for this axis. + try writer.splatByteAll(' ', indent_level); + _ = try writer.write("{\n"); + + // Write first rows + const n: u64 = @intCast(self.shape.dim(0)); + for (0..@min(num_rows, n)) |d| { + const byte_stride: usize = @intCast(self.shape.computeByteStrides().get(0)); + const sub_slice: ConstSlice = .init(self.shape.drop(0), self.data[d * byte_stride ..][0..byte_stride]); + try sub_slice.prettyPrintIndented(writer, num_rows, indent_level + 2, options); + if (self.shape.rank() > 1) { + try writer.writeAll(",\n"); + } + } + + if (n < num_rows) return; + // Skip middle rows + if (n > 2 * num_rows) { + try writer.splatByteAll(' ', indent_level + 2); + _ = try writer.write("...\n"); + } + // Write last rows + for (@max(n - num_rows, num_rows)..n) |d| { + const byte_stride: usize = @intCast(self.shape.computeByteStrides().get(0)); + const sub_slice: ConstSlice = .init(self.shape.drop(0), self.data[d * byte_stride ..][0..byte_stride]); + try sub_slice.prettyPrintIndented(writer, num_rows, indent_level + 2, options); + if (self.shape.rank() > 1) { + try writer.writeAll(",\n"); + } + } + + try writer.splatByteAll(' ', indent_level); + _ = try writer.write("}"); } }; + +test "slice pretty print rank 3" { + const data: [4][4][4]i32 = .{ + .{ + .{ 0, 1, 2, 3 }, + .{ 4, 5, 6, 7 }, + .{ 8, 9, 10, 11 }, + .{ 12, 13, 14, 15 }, + }, + .{ + .{ 16, 17, 18, 19 }, + .{ 20, 21, 22, 23 }, + .{ 24, 25, 26, 27 }, + .{ 28, 29, 30, 31 }, + }, + .{ + .{ 32, 33, 34, 35 }, + .{ 36, 37, 38, 39 }, + .{ 40, 41, 42, 43 }, + .{ 44, 45, 46, 47 }, + }, + .{ + .{ 48, 49, 50, 51 }, + .{ 52, 53, 54, 55 }, + .{ 56, 57, 58, 59 }, + .{ 60, 61, 62, 63 }, + }, + }; + const slice = ConstSlice.init(.init(.{ 4, 4, 4 }, .i32), std.mem.asBytes(&data)); + const expected = + \\{ + \\ { + \\ {0,1,2,3}, + \\ {4,5,6,7}, + \\ {8,9,10,11}, + \\ {12,13,14,15}, + \\ }, + \\ { + \\ {16,17,18,19}, + \\ {20,21,22,23}, + \\ {24,25,26,27}, + \\ {28,29,30,31}, + \\ }, + \\ { + \\ {32,33,34,35}, + \\ {36,37,38,39}, + \\ {40,41,42,43}, + \\ {44,45,46,47}, + \\ }, + \\ { + \\ {48,49,50,51}, + \\ {52,53,54,55}, + \\ {56,57,58,59}, + \\ {60,61,62,63}, + \\ }, + \\} + ; + + try std.testing.expectFmt(expected, "{d}", .{slice}); +} + +test "slice pretty print rank 1" { + const data: [4]i32 = .{ 0, 1, 2, 3 }; + const slice = ConstSlice.init(.init(.{4}, .i32), std.mem.asBytes(&data)); + const expected = "{0,1,2,3}"; + try std.testing.expectFmt(expected, "{d}", .{slice}); +} + +test "slice pretty print rank 0" { + const data: [1]i32 = .{ 0 }; + const slice = ConstSlice.init(.init(.{}, .i32), std.mem.asBytes(&data)); + const expected = "0"; + try std.testing.expectFmt(expected, "{d}", .{slice}); +} + +test "slice pretty print ellipsis" { + const data: [9][1]i32 = .{ .{0}, .{1}, .{2}, .{3}, .{4}, .{5}, .{6}, .{7}, .{8} }; + const slice = ConstSlice.init(.init(.{ 9, 1 }, .i32), std.mem.asBytes(&data)); + const expected = + \\{ + \\ {0}, + \\ {1}, + \\ {2}, + \\ {3}, + \\ ... + \\ {5}, + \\ {6}, + \\ {7}, + \\ {8}, + \\} + ; + try std.testing.expectFmt(expected, "{d}", .{slice}); +} From 2739c925a64a61622f9ef7957d6fa231e1dad7d3 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 22 Dec 2025 11:52:00 +0100 Subject: [PATCH 161/881] Fixed mnist example with new VFS.File changes --- examples/mnist/mnist.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index eedc9c37..d9f464ad 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -70,7 +70,7 @@ pub fn main() !void { var threaded: std.Io.Threaded = .init(allocator); defer threaded.deinit(); - var vfs_file: zml.io.VFS.File = .init(threaded.io()); + var vfs_file: zml.io.VFS.File = .init(allocator, threaded.io(), .{}); var vfs: zml.io.VFS = .init(allocator, threaded.io()); defer vfs.deinit(); @@ -83,7 +83,7 @@ pub fn main() !void { defer zml.deinit(); // Auto-select platform - const platform: zml.Platform = try .auto(threaded.io(), .{}); + const platform: zml.Platform = try .auto(io, .{}); // Parse program args const process_args = try std.process.argsAlloc(allocator); From 2d0aa3ceaed93b22203a67fa12dcb59731967162 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 22 Dec 2025 11:52:36 +0100 Subject: [PATCH 162/881] Removed autofix comments --- runtimes/cuda/cuda.zig | 3 ++- runtimes/neuron/neuron.zig | 6 ++++-- runtimes/rocm/rocm.zig | 3 ++- runtimes/tpu/tpu.zig | 3 ++- zml2/buffer.zig | 2 +- zml2/io.zig | 4 ++-- zml2/tensor.zig | 18 ++++++++---------- 7 files changed, 21 insertions(+), 18 deletions(-) diff --git a/runtimes/cuda/cuda.zig b/runtimes/cuda/cuda.zig index 01f236cb..b1289869 100644 --- a/runtimes/cuda/cuda.zig +++ b/runtimes/cuda/cuda.zig @@ -16,7 +16,8 @@ pub fn isEnabled() bool { } fn hasNvidiaDevice(io: std.Io) bool { - _ = io; // autofix + // TODO(Corentin): Plug that when fs stuff is merged in std.Io + _ = io; std.fs.accessAbsolute("/dev/nvidiactl", .{ .read = true }) catch return false; return true; } diff --git a/runtimes/neuron/neuron.zig b/runtimes/neuron/neuron.zig index 3638e27e..bc460288 100644 --- a/runtimes/neuron/neuron.zig +++ b/runtimes/neuron/neuron.zig @@ -14,13 +14,15 @@ pub fn isEnabled() bool { } fn hasNeuronDevice(io: std.Io) bool { - _ = io; // autofix + // TODO(Corentin): Plug that when fs stuff is merged in std.Io + _ = io; std.fs.accessAbsolute("/dev/neuron0", .{ .read = true }) catch return false; return true; } fn isRunningOnEC2(io: std.Io) !bool { - _ = io; // autofix + // TODO(Corentin): Plug that when fs stuff is merged in std.Io + _ = io; const AmazonEC2 = "Amazon EC2"; var f = try std.fs.openFileAbsolute("/sys/devices/virtual/dmi/id/sys_vendor", .{ .read = true }); diff --git a/runtimes/rocm/rocm.zig b/runtimes/rocm/rocm.zig index 95e85fb4..aa37350d 100644 --- a/runtimes/rocm/rocm.zig +++ b/runtimes/rocm/rocm.zig @@ -14,7 +14,8 @@ pub fn isEnabled() bool { } fn hasRocmDevices(io: std.Io) bool { - _ = io; // autofix + // TODO(Corentin): Plug that when fs stuff is merged in std.Io + _ = io; inline for (&.{ "/dev/kfd", "/dev/dri" }) |path| { std.fs.accessAbsolute(path, .{ .read = true }) catch return false; } diff --git a/runtimes/tpu/tpu.zig b/runtimes/tpu/tpu.zig index adf17923..1648c320 100644 --- a/runtimes/tpu/tpu.zig +++ b/runtimes/tpu/tpu.zig @@ -18,7 +18,8 @@ pub fn isEnabled() bool { /// Do it using the official method at: /// https://cloud.google.com/compute/docs/instances/detect-compute-engine?hl=en#use_operating_system_tools_to_detect_if_a_vm_is_running_in fn isOnGCP(io: std.Io) !bool { - _ = io; // autofix + // TODO(Corentin): Plug that when fs stuff is merged in std.Io + _ = io; // TODO: abstract that in the client and fail init const GoogleComputeEngine = "Google Compute Engine"; diff --git a/zml2/buffer.zig b/zml2/buffer.zig index f98466f9..ebd7591b 100644 --- a/zml2/buffer.zig +++ b/zml2/buffer.zig @@ -69,7 +69,7 @@ pub const Buffer = struct { // stdx.debug.assert(@rem(shape_.dim(ax), n_partitions) == 0, "Buffer.from({f}) expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ shape_, ax, n_partitions }); // break :cs @divExact(shape_.dim(ax), n_partitions); //} else 0; - //_ = chunk_size; // autofix + //_ = chunk_size; const buffer_type = pjrt.bufferTypeFromDtype(shape_.dtype()); const byte_strides = shape_.computeByteStrides(); diff --git a/zml2/io.zig b/zml2/io.zig index b42b157b..277df35d 100644 --- a/zml2/io.zig +++ b/zml2/io.zig @@ -334,8 +334,8 @@ pub const SimpleWriter = struct { } pub fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { - _ = data; // autofix - _ = splat; // autofix + _ = data; + _ = splat; const writer: *SimpleWriter = @alignCast(@fieldParentPtr("interface", w)); stdx.debug.assert(writer.offset + w.end <= writer.shape.byteSize(), "Can't write more data than required", .{}); const is_last_transfer = writer.offset + w.end >= writer.shape.byteSize(); diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 8d850b8f..a88def6c 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -145,9 +145,9 @@ pub const Tensor = struct { // TODO(Corentin) pub fn withSharding(self: Tensor, axes_: anytype) Tensor { - _ = self; // autofix - _ = axes_; // autofix - @panic("TODO"); + _ = self; + _ = axes_; + @panic("Unimplemented"); } pub fn toMemory(self: Tensor, kind: Memory) Tensor { @@ -730,8 +730,7 @@ pub const Tensor = struct { return _result(self._shape, op.result(0)); } - fn convolution(self: Tensor, other: Tensor, opts: dialects.stablehlo.ConvolutionOpts, loc: *const mlir.Location) Tensor { - _ = loc; // autofix + fn convolution(self: Tensor, other: Tensor, opts: dialects.stablehlo.ConvolutionOpts) Tensor { stdx.debug.assert(self.rank() == other.rank(), "convolution expects tensor ranks to match, got {} and {}", .{ self.rank(), other.rank() }); const N = self.rank(); stdx.debug.guard(opts.window_strides.len == N - 2, @src()); @@ -866,7 +865,7 @@ pub const Tensor = struct { .output_spatial_dimensions = &.{opts.output_spatial_dimensions}, .feature_group_count = opts.feature_group_count, .batch_group_count = opts.batch_group_count, - }, .unknown(mlirCtx())); + }); } /// Returns a Tensor containing the result of the 2D convolution of 'input' by 'kernel'. @@ -910,7 +909,7 @@ pub const Tensor = struct { .output_spatial_dimensions = opts.output_spatial_dimensions, .feature_group_count = opts.feature_group_count, .batch_group_count = opts.batch_group_count, - }, .unknown(mlirCtx())); + }); } /// Returns a Tensor containing the element-wise addition of the input Tensors. @@ -3977,10 +3976,9 @@ pub const Tensor = struct { /// Only for debug purpose, it inserts device to host synchronization /// so it will slow down the program execution. pub fn print(input: Tensor) Tensor { - _ = input; // autofix - unreachable; // TODO(Corentin) - //return callback.call(callback.Print, .{input}, &.{input.shape()})[0]; + _ = input; + @panic("Unimplemented"); } fn mlirCtx() *mlir.Context { From e3ad3115368f7ea14743c34bc80726c20e580b7a Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 22 Dec 2025 12:13:30 +0100 Subject: [PATCH 163/881] Ported simple_layer example to new API --- examples/simple_layer/BUILD.bazel | 3 +- examples/simple_layer/main.zig | 112 ++++++++++++++++-------------- 2 files changed, 59 insertions(+), 56 deletions(-) diff --git a/examples/simple_layer/BUILD.bazel b/examples/simple_layer/BUILD.bazel index b10b6dad..7feb8aad 100644 --- a/examples/simple_layer/BUILD.bazel +++ b/examples/simple_layer/BUILD.bazel @@ -8,8 +8,7 @@ zig_binary( name = "simple_layer", main = "main.zig", deps = [ - "//async", - "//zml", + "//zml2:zml", ], ) diff --git a/examples/simple_layer/main.zig b/examples/simple_layer/main.zig index 57d5d16b..f7f38906 100644 --- a/examples/simple_layer/main.zig +++ b/examples/simple_layer/main.zig @@ -1,6 +1,5 @@ const std = @import("std"); -const async = @import("async"); const zml = @import("zml"); /// Model definition @@ -18,77 +17,82 @@ const Layer = struct { }; pub fn main() !void { - try async.AsyncThread.main(std.heap.c_allocator, asyncMain); -} - -pub fn asyncMain() !void { // Short lived allocations var gpa = std.heap.GeneralPurposeAllocator(.{}){}; defer _ = gpa.deinit(); const allocator = gpa.allocator(); - // Arena allocator for BufferStore etc. - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); + // Io runtime + var threaded: std.Io.Threaded = .init(allocator); + defer threaded.deinit(); - var context = try zml.Context.init(); - defer context.deinit(); + const io = threaded.io(); - const platform = context.autoPlatform(.{}); - context.printAvailablePlatforms(platform); + // Initialize ZML globally + zml.init(); + defer zml.deinit(); + + // Auto-select platform + const platform: zml.Platform = try .auto(io, .{}); + + // Initialize model instance and input + const model: Layer = .{ + .weight = zml.Tensor.init(.{4}, .f32), + .bias = zml.Tensor.init(.{4}, .f32), + }; + const input: zml.Tensor = .init(.{4}, .f32); + + // Compile our executable + var exe = try platform.compileModel(allocator, io, Layer.forward, model, .{input}); + defer exe.deinit(); // Our weights and bias to use - var weights = [4]f32{ 2.0, 2.0, 2.0, 2.0 }; - var bias = [4]f32{ 1.0, 2.0, 3.0, 4.0 }; - const input_shape = zml.Shape.init(.{4}, .f32); + const weights_data = [4]f32{ 2.0, 2.0, 2.0, 2.0 }; + const weights_slice: zml.ConstSlice = .init(model.weight.shape(), std.mem.sliceAsBytes(&weights_data)); + var weights_buffer: zml.Buffer = try .fromSlice(io, platform, weights_slice); - // We manually produce a BufferStore. You would not normally do that. - // A BufferStore is usually created by loading model data from a file. - var store: zml.aio.BufferStore = .init(allocator); - defer store.deinit(); - try store.buffers.put(store.arena.allocator(), "weight", zml.HostBuffer.fromArray(&weights)); - try store.buffers.put(store.arena.allocator(), "bias", zml.HostBuffer.fromArray(&bias)); + const bias_data = [4]f32{ 1.0, 2.0, 3.0, 4.0 }; + const bias_slice: zml.ConstSlice = .init(model.bias.?.shape(), std.mem.sliceAsBytes(&bias_data)); + var bias_buffer: zml.Buffer = try .fromSlice(io, platform, bias_slice); - // A clone of our model, consisting of shapes. We only need shapes for compiling. - // We use the BufferStore to infer the shapes. - var model_shapes = try zml.aio.populateModel(Layer, allocator, store); - model_shapes.weight = model_shapes.weight.withSharding(.{-1}); - model_shapes.bias = model_shapes.bias.?.withSharding(.{-1}); + // Don't forget to deinit buffers + defer weights_buffer.deinit(); + defer bias_buffer.deinit(); - // Start compiling. This uses the inferred shapes from the BufferStore. - // The shape of the input tensor, we have to pass in manually. - var compilation = try async.async(zml.compileModel, .{ allocator, Layer.forward, model_shapes, .{input_shape}, platform }); + // Init the bufferized version of the model + const model_buffers: zml.Bufferized(Layer) = .{ + .weight = weights_buffer, + .bias = bias_buffer, + }; - // Produce a bufferized weights struct from the fake BufferStore. - // This is like the inferred shapes, but with actual values. - // We will need to send those to the computation device later. - var model_weights = try zml.aio.loadModelBuffers(Layer, model_shapes, store, arena, platform); - defer zml.aio.unloadBuffers(&model_weights); // for good practice - - // Wait for compilation to finish - const compiled = try compilation.await(); - - // pass the model weights to the compiled module to create an executable module - var executable = compiled.prepare(model_weights); - defer executable.deinit(); - - // prepare an input buffer - // Here, we use zml.HostBuffer.fromSlice to show how you would create a HostBuffer - // with a specific shape from an array. - // For situations where e.g. you have an [4]f32 array but need a .{2, 2} input shape. - var input = [4]f32{ 5.0, 5.0, 5.0, 5.0 }; - var input_buffer = try zml.Buffer.from(platform, zml.HostBuffer.fromSlice(input_shape, &input), .{}); + // Prepare an input buffer + var input_data = [4]f32{ 5.0, 5.0, 5.0, 5.0 }; + const input_slice: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&input_data)); + var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice); defer input_buffer.deinit(); - // call our executable module - var result: zml.Buffer = executable.call(.{input_buffer}); + // Create the Arguments and Results struct to interact with the executable + var args = try exe.args(allocator); + defer args.deinit(allocator); + + var results = try exe.results(allocator); + defer results.deinit(allocator); + + // Set the arguments from the bufferized version of the model and the input buffer + args.set(.{ model_buffers, input_buffer }); + + // Call our executable module + exe.call(args, &results, io); + + // Retrieve the output + var result = results.get(zml.Buffer); defer result.deinit(); // fetch the result to CPU memory - const cpu_result = try result.toHostAlloc(arena); + const cpu_result = try result.toSliceAlloc(allocator, io); + defer cpu_result.free(allocator); std.debug.print( - "\nThe result of {any} * {any} + {any} = {any}\n", - .{ &weights, &input, &bias, cpu_result.items(f32) }, + "\nThe result of {d} * {d} + {d} = {d}\n", + .{ weights_slice, input_slice, bias_slice, cpu_result }, ); } From 573fabad95b4fc0dcadd7296cb80abba3a2804fd Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Mon, 22 Dec 2025 12:27:35 +0100 Subject: [PATCH 164/881] Upgrade TPU to latest nightly --- runtimes/tpu/tpu.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtimes/tpu/tpu.bzl b/runtimes/tpu/tpu.bzl index e637d8dd..fe111015 100644 --- a/runtimes/tpu/tpu.bzl +++ b/runtimes/tpu/tpu.bzl @@ -4,9 +4,9 @@ def _tpu_impl(mctx): # https://storage.googleapis.com/jax-releases/libtpu_releases.html http_archive( name = "libpjrt_tpu", - url = "https://storage.googleapis.com/libtpu-nightly-releases/wheels/libtpu-nightly/libtpu_nightly-0.1.dev20250807+nightly-py3-none-manylinux_2_31_x86_64.whl", + url = "https://storage.googleapis.com/libtpu-nightly-releases/wheels/libtpu/libtpu-0.0.33.dev20251218+nightly-cp314-cp314t-manylinux_2_31_x86_64.whl", type = "zip", - sha256 = "41c19fa5ae4a32fbd05f0260527ba2d93afb6cf128e6c4de7773e9011c7b3df5", + sha256 = "b7d9855c74a7fc56d11c4c7a2c55d5395c80594d43caec1549c69d5f908583cf", build_file = "libpjrt_tpu.BUILD.bazel", ) return mctx.extension_metadata( From da596e30f3df465041e7cb8296cc476de8920864 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 22 Dec 2025 14:31:39 +0100 Subject: [PATCH 165/881] Remove Tensor.matmul as it's not used anymore --- zml2/tensor.zig | 6 ------ 1 file changed, 6 deletions(-) diff --git a/zml2/tensor.zig b/zml2/tensor.zig index a88def6c..2c23938e 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -1064,12 +1064,6 @@ pub const Tensor = struct { return _result(self._shape, op.result(0)); } - // TODO(Corentin) - /// See torch.matmul - //pub fn matmul(lhs: Tensor, rhs: Tensor) Tensor { - // return @import("torch.zig").matmul(lhs, rhs); - //} - /// Matrix multiplication, where contracting axes are specified using their tags. /// eg dot(.{ .a, .b, .c }, .{ .a, .c, .d }, .c) -> .{ .a, .b, .d } /// Axes with the same tag on both sides, and which aren't contracting, From 1c7c520d8efd327b21c2cf9e230f4a1848466187 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 22 Dec 2025 15:08:47 +0100 Subject: [PATCH 166/881] Use best platform for tests by default --- zml2/testing.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml2/testing.zig b/zml2/testing.zig index 0a5f904c..88783f52 100644 --- a/zml2/testing.zig +++ b/zml2/testing.zig @@ -16,7 +16,7 @@ pub fn env() Platform { if (_platform == null) { zml.init(); - _platform = Platform.init(.cpu, std.testing.io, .{}) catch unreachable; + _platform = Platform.auto(std.testing.io, .{}) catch unreachable; } return _platform.?; From c0522ca6aa465da9abdc5a71271ceda281ddf576 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 22 Dec 2025 15:10:41 +0100 Subject: [PATCH 167/881] Events can be null when executing a LoadedExecutable --- pjrt/pjrt.zig | 4 ++-- zml2/pjrtx.zig | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 87a0fa19..01bfd1e6 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -767,7 +767,7 @@ pub const LoadedExecutable = opaque { num_args: usize, arguments: []const [*]const *const Buffer, results: []const [*]*Buffer, - events: []?*Event, + events: ?[]?*Event, non_donatable_input_indices: []const i64 = &.{}, context: ?*ExecuteContext, }; @@ -790,7 +790,7 @@ pub const LoadedExecutable = opaque { .num_devices = @as(usize, @intCast(args.arguments.len)), .num_args = args.num_args, .output_lists = @as([*c]const [*c]?*c.PJRT_Buffer, @ptrCast(args.results.ptr)), - .device_complete_events = @as([*c]?*c.PJRT_Event, @ptrCast(args.events.ptr)), + .device_complete_events = @as([*c]?*c.PJRT_Event, if (args.events) |ev| @ptrCast(ev.ptr) else null), .execute_device = null, }); } diff --git a/zml2/pjrtx.zig b/zml2/pjrtx.zig index db672b8f..3a96c4b4 100644 --- a/zml2/pjrtx.zig +++ b/zml2/pjrtx.zig @@ -301,7 +301,7 @@ pub const LoadedExecutable = opaque { arguments: []const [*]const *const Buffer, num_args: usize, results: []const [*]*Buffer, - events: []?*Event, + events: ?[]?*Event, non_donatable_input_indices: []const i64 = &.{}, context: ?*ExecuteContext, }; From 11ca63293187b660e60b48ca40f8c618047d96e9 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 22 Dec 2025 15:11:17 +0100 Subject: [PATCH 168/881] Introduced variant of Exe.call By default, we don't wait for execution completion when calling `Exe.call`. If you want to await the result, you can use `Exe.callOpts`. --- examples/benchmark/main.zig | 4 ++-- examples/llama/main.zig | 4 ++-- examples/mnist/mnist.zig | 2 +- examples/simple_layer/main.zig | 2 +- zml2/exe.zig | 26 +++++++++++++++++++++----- zml2/main.zig | 2 +- zml2/testing.zig | 2 +- 7 files changed, 29 insertions(+), 13 deletions(-) diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index f0321e88..fe095a52 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -77,14 +77,14 @@ pub fn main() !void { // Ignore first run { - exe.call(exe_args, &exe_results, io); + exe.call(exe_args, &exe_results); var result = exe_results.get(zml.Buffer); defer result.deinit(); } // call our executable module timer.reset(); - exe.call(exe_args, &exe_results, io); + exe.call(exe_args, &exe_results); var result = exe_results.get(zml.Buffer); defer result.deinit(); const elapsed_ns = timer.lap(); diff --git a/examples/llama/main.zig b/examples/llama/main.zig index f7b27734..976e7ca5 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -397,7 +397,7 @@ pub fn generateText( prefill_args.set(.{ llama_buffers, prefill_tokens_buffer, prefill_token_pos_buffer, kv_cache_buffers, rng_buffers }); - prefill_exe.call(prefill_args, &prefill_results, io); + prefill_exe.call(prefill_args, &prefill_results); prefill_results.fill(.{ &prefill_tokens_buffer, kv_cache_buffers, &rng_buffers }); try prefill_tokens_buffer.toSlice(prefill_tokens_slice, io); @@ -449,7 +449,7 @@ pub fn generateText( // call to generate the next token decode_args.set(.{ llama_buffers, current_token_buffer, token_pos_buffer, kv_cache_buffers, rng_buffers }); - decode_exe.call(decode_args, &decode_results, io); + decode_exe.call(decode_args, &decode_results); decode_results.fill(.{ ¤t_token_buffer, kv_cache_buffers, &rng_buffers }); diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index d9f464ad..9ab07497 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -143,7 +143,7 @@ pub fn main() !void { args.set(.{ mnist_buffers, input_buffer }); printDigit(sample); - exe.call(args, &results, io); + exe.call(args, &results); var result: zml.Buffer = results.get(zml.Buffer); defer result.deinit(); diff --git a/examples/simple_layer/main.zig b/examples/simple_layer/main.zig index f7f38906..ae46ccbb 100644 --- a/examples/simple_layer/main.zig +++ b/examples/simple_layer/main.zig @@ -82,7 +82,7 @@ pub fn main() !void { args.set(.{ model_buffers, input_buffer }); // Call our executable module - exe.call(args, &results, io); + exe.call(args, &results); // Retrieve the output var result = results.get(zml.Buffer); diff --git a/zml2/exe.zig b/zml2/exe.zig index 47a4f39c..122f99d8 100644 --- a/zml2/exe.zig +++ b/zml2/exe.zig @@ -197,15 +197,17 @@ pub const Exe = struct { } }; - pub fn call(self: *const Exe, arguments: Arguments, results_: *Results, io: std.Io) void { + pub fn internalCall(self: *const Exe, io: ?std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { + stdx.debug.assert(opts.wait == false or io != null, "io should not be null when waiting for execution completion", .{}); var events = [_]?*pjrt.Event{null} ** Platform.MAX_NUM_DEVICES; const sharding = self.platform.sharding(); + const events_slice: ?[]?*pjrt.Event = if (opts.wait) events[0..sharding.num_partitions] else null; self.exe.execute(self.platform.pjrt_api, .{ .arguments = arguments.flat_buffers.buffers, .num_args = arguments.expected_shapes.len, .results = results_.flat_buffers.buffers, - .events = events[0..sharding.num_partitions], + .events = events_slice, // this allows to tell a specific buffer shouldn't be donated, // even if it has been marked as "can be donated" during compilation. // TODO: expose it ? @@ -215,10 +217,24 @@ pub const Exe = struct { std.debug.panic("PJRT_LoadedExecutable_Execute failed with: {}", .{err}); }; - for (events[0..sharding.num_partitions]) |e| { - if (e) |ev| { - ev.await(self.platform.pjrt_api, io) catch unreachable; + if (opts.wait) { + for (events_slice.?) |e| { + if (e) |ev| { + ev.await(self.platform.pjrt_api, io.?) catch unreachable; + } } } } + + pub const CallOpts = struct { + wait: bool = false, + }; + + pub fn callOpts(self: *const Exe, io: std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { + return self.internalCall(io, arguments, results_, opts); + } + + pub fn call(self: *const Exe, arguments: Arguments, results_: *Results) void { + return self.internalCall(null, arguments, results_, .{}); + } }; diff --git a/zml2/main.zig b/zml2/main.zig index 3d6f9036..f62c98da 100644 --- a/zml2/main.zig +++ b/zml2/main.zig @@ -99,7 +99,7 @@ pub fn main() !void { defer results.deinit(allocator); args.set(.{ model_buffers, input_buffer }); - exe.call(args, &results, io); + exe.call(args, &results); const output = results.get(zml.Buffer); defer output.deinit(); diff --git a/zml2/testing.zig b/zml2/testing.zig index 88783f52..bf968551 100644 --- a/zml2/testing.zig +++ b/zml2/testing.zig @@ -129,7 +129,7 @@ pub fn autoCall(allocator: std.mem.Allocator, io: std.Io, exe: *const zml.exe.Ex defer results.deinit(allocator); args.set(inputs); - exe.call(args, &results, io); + exe.callOpts(io, args, &results, .{ .wait = true }); var output: zml.Bufferized(stdx.meta.FnResult(func)) = undefined; results.fill(.{&output}); From 5ca3dad667b9abcce19891b1a358d58bb2c597e1 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 22 Dec 2025 15:57:56 +0100 Subject: [PATCH 169/881] Change parameter order for Buffer.toSlice std.Io should be as close to the beginning as possible for consistency --- examples/llama/main.zig | 4 ++-- zml2/buffer.zig | 6 +++--- zml2/tensor.zig | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 976e7ca5..6272dcaa 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -400,7 +400,7 @@ pub fn generateText( prefill_exe.call(prefill_args, &prefill_results); prefill_results.fill(.{ &prefill_tokens_buffer, kv_cache_buffers, &rng_buffers }); - try prefill_tokens_buffer.toSlice(prefill_tokens_slice, io); + try prefill_tokens_buffer.toSlice(io, prefill_tokens_slice); generated_token_slice.items(u32)[0] = prefill_tokens_slice.items(u32)[prompt_tok.len - 1]; } @@ -454,7 +454,7 @@ pub fn generateText( decode_results.fill(.{ ¤t_token_buffer, kv_cache_buffers, &rng_buffers }); // extract the generated token from the buffer - try current_token_buffer.toSlice(generated_token_slice, io); + try current_token_buffer.toSlice(io, generated_token_slice); } const duration = timer.read(); const duration_s = stdx.math.divFloat(f64, duration.ns, std.time.ns_per_s); diff --git a/zml2/buffer.zig b/zml2/buffer.zig index ebd7591b..ef398f09 100644 --- a/zml2/buffer.zig +++ b/zml2/buffer.zig @@ -227,13 +227,13 @@ pub const Buffer = struct { stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {f} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); var res: T = undefined; - try self.toSlice(.init(self.shape(), std.mem.asBytes(&res)), io); + try self.toSlice(io, .init(self.shape(), std.mem.asBytes(&res))); return res; } /// Copies the content of the Buffer to the provided slice. - pub fn toSlice(self: Buffer, slice: Slice, io: std.Io) !void { + pub fn toSlice(self: Buffer, io: std.Io, slice: Slice) !void { //stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); const maybe_event = try self._shards.get(0).toHostBuffer(self._api, slice.data); if (maybe_event) |event| { @@ -247,7 +247,7 @@ pub const Buffer = struct { const slice = try Slice.alloc(allocator, self.shape()); errdefer slice.free(allocator); - try self.toSlice(slice, io); + try self.toSlice(io, slice); return slice; } diff --git a/zml2/tensor.zig b/zml2/tensor.zig index 2c23938e..776af877 100644 --- a/zml2/tensor.zig +++ b/zml2/tensor.zig @@ -4105,7 +4105,7 @@ test "Tensor.maxPool2d" { try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2, 4 }, .f32), result.values.shape()); try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2, 4 }, .i32), result.indices.shape()); var buffer: [2][2][2][4]f32 = undefined; - _ = try result.values.toSlice(.init(result.values.shape(), std.mem.asBytes(&buffer)), std.testing.io); + _ = try result.values.toSlice(std.testing.io, .init(result.values.shape(), std.mem.asBytes(&buffer))); try std.testing.expectEqualDeep( [2][2][2][4]f32{ .{ From db3ded4f8014bcd12586706769aa7f3381a1dc94 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 1 Jan 2026 08:47:52 +0000 Subject: [PATCH 170/881] WIP --- BUILD.bazel | 8 + MODULE.bazel | 4 +- MODULE.bazel.lock | 98 +++---- bazel/zig_index.json | 154 +++++------ examples/llama/main.zig | 2 +- third_party/arocc/repo.bzl | 4 +- third_party/translate-c/repo.bzl | 4 +- third_party/zls/workspace_printer.zig | 2 +- third_party/zls/zls_runner.zig | 34 +-- zml2/io.zig | 17 +- zml2/io/vfs/BUILD.bazel | 12 +- zml2/io/vfs/base.zig | 374 +++++++++++++++++++++++++ zml2/io/vfs/hf2.zig | 384 ++++++++++++++++++++++++++ zml2/io/vfs/proxy.zig | 331 ++++++++++++++++++++++ 14 files changed, 1262 insertions(+), 166 deletions(-) create mode 100644 zml2/io/vfs/base.zig create mode 100644 zml2/io/vfs/hf2.zig create mode 100644 zml2/io/vfs/proxy.zig diff --git a/BUILD.bazel b/BUILD.bazel index 5d775990..477668b3 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -2,6 +2,14 @@ load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_c_proto_library") load("@rules_zig//zig:defs.bzl", "zig_binary") load("//third_party/zls:zls_completion.bzl", "zls_completion") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_binary") + +zig_binary( + name = "http", + main = "http.zig", + visibility = ["//visibility:public"], +) + zls_completion( name = "completion", visibility = ["//visibility:public"], diff --git a/MODULE.bazel b/MODULE.bazel index 7fab36f7..7a110629 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -31,7 +31,7 @@ bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2", dev_dependency = Tr ### Zig toolchain zig = use_extension("@rules_zig//zig:extensions.bzl", "zig") zig.index(file = "//bazel:zig_index.json") -zig.toolchain(zig_version = "0.16.0-dev.1484+d0ba6642b") +zig.toolchain(zig_version = "0.16.0-dev.1878+2a02c7a0d") zig.mirrors(urls = [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/", @@ -227,7 +227,7 @@ use_repo(huggingface, "ModernBERT-base", "ModernBERT-large") git_override( module_name = "rules_zig", - commit = "15cf8dfd3007025a607fe228d16cdc106a307aed", + commit = "b4ae496abfd61f880927dd01ea59823df7b42c29", remote = "https://github.com/zml/rules_zig.git", ) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 104ba2c3..d31e0ba0 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -112,9 +112,9 @@ "https://bazel-registry.zml.ai/modules/rules_cc/0.0.9/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.1.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.2.11/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.12/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.14/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_cc/0.2.16/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.4/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_distroless/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "not found", @@ -330,10 +330,10 @@ "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", "https://bcr.bazel.build/modules/rules_cc/0.1.1/MODULE.bazel": "2f0222a6f229f0bf44cd711dc13c858dad98c62d52bd51d8fc3a764a83125513", "https://bcr.bazel.build/modules/rules_cc/0.2.0/MODULE.bazel": "b5c17f90458caae90d2ccd114c81970062946f49f355610ed89bebf954f5783c", - "https://bcr.bazel.build/modules/rules_cc/0.2.11/MODULE.bazel": "e94f24f065bf2191dba2dace951814378b66a94bb3bcc48077492fe0508059b5", "https://bcr.bazel.build/modules/rules_cc/0.2.12/MODULE.bazel": "4216c383ce3223c7dfbd3afcb09c7056265156150b2a3bd3fd9d02949a7ee1bc", "https://bcr.bazel.build/modules/rules_cc/0.2.14/MODULE.bazel": "353c99ed148887ee89c54a17d4100ae7e7e436593d104b668476019023b58df8", - "https://bcr.bazel.build/modules/rules_cc/0.2.14/source.json": "55d0a4587c5592fad350f6e698530f4faf0e7dd15e69d43f8d87e220c78bea54", + "https://bcr.bazel.build/modules/rules_cc/0.2.16/MODULE.bazel": "9242fa89f950c6ef7702801ab53922e99c69b02310c39fb6e62b2bd30df2a1d4", + "https://bcr.bazel.build/modules/rules_cc/0.2.16/source.json": "d03d5cde49376d87e14ec14b666c56075e5e3926930327fd5d0484a1ff2ac1cc", "https://bcr.bazel.build/modules/rules_cc/0.2.4/MODULE.bazel": "1ff1223dfd24f3ecf8f028446d4a27608aa43c3f41e346d22838a4223980b8cc", "https://bcr.bazel.build/modules/rules_distroless/0.5.1/MODULE.bazel": "2a63f4744d30749128105da5f96adf7caf5628e37548293f89e7fa39c3b3f2c2", "https://bcr.bazel.build/modules/rules_distroless/0.5.1/source.json": "c6b9ff7f325bfed89c3671757f14c1d1bc6077d0fcef809b8aa2d007cac7dd1d", @@ -436,7 +436,7 @@ "moduleExtensions": { "@@apple_support+//crosstool:setup.bzl%apple_cc_configure_extension": { "general": { - "bzlTransitiveDigest": "6HFMfNkxiYUs5AKI+9d2/9J5tWKUXMh2sUT0MsHY6jI=", + "bzlTransitiveDigest": "cw6mKbsar0LsMbAePOLZdW+szZNKRE8/CrhomQFNbNE=", "usagesDigest": "GXVjAqzvGUnh20kBPs+f0GSdAtaqLPet4sogeCfejZY=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -1194,7 +1194,7 @@ }, "@@rules_rust+//crate_universe:extensions.bzl%crate": { "general": { - "bzlTransitiveDigest": "xnOdcJlDn8jAdyIY4BnC1iO+osURpMjNcjvqwmM3hfE=", + "bzlTransitiveDigest": "NvjljZ+5m2nc1Qpf7ekjzs+FJrjYYY9DwXvwuXnippI=", "usagesDigest": "mup6mgSFatk34Q0znMZPpIEOFZyk9v3xf6jRIXEFTxc=", "recordedFileInputs": { "@@//zml/tokenizer/hftokenizers/Cargo.lock": "d78ffab37ef69bf8ff45a4d686b25175f30e8fed4d8375612e84dcf6db0cfc9c", @@ -2849,89 +2849,89 @@ "@@rules_zig+//zig:extensions.bzl%zig": { "general": { "bzlTransitiveDigest": "wV3euhBZSI0D2ZlqbY1YW592mghFY88d9H6zBMQRaJY=", - "usagesDigest": "eS/vW6tmN9XXr9OKW9PDWeVMh8wVIBSfvoOHCD420XU=", + "usagesDigest": "gADTTSTOnyxlP1vQhqp5HillVM6887O94Xj7VDt4tdk=", "recordedFileInputs": { - "@@//bazel/zig_index.json": "83fdd8898d3a527681082cd98c02b91570aa439ae2209807169ad4e41326d75e", + "@@//bazel/zig_index.json": "b9733b7c49787cd95b384a0b5ee9e4171367d48576aeaa8b3a6fd05e27184e4d", "@@rules_zig+//zig/private/versions.json": "6eb85ebaee72c4e6fbc82d8ad73f69e0899d839b70a7e19caa792ab6a5fa0c36" }, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { - "zig_0.16.0-dev.1484_Pd0ba6642b_aarch64-linux": { + "zig_0.16.0-dev.1878_P2a02c7a0d_aarch64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1484+d0ba6642b.tar.xz", + "url": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1878+2a02c7a0d.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "f8240e9708376f3e808c59dbaf69a2839eb1a14507432ee9f7d6817482346bf9", - "zig_version": "0.16.0-dev.1484+d0ba6642b", + "sha256": "76622697d65885d3d2b0aa21de9a680662db14d04174b022b83e9292d106e4f6", + "zig_version": "0.16.0-dev.1878+2a02c7a0d", "platform": "aarch64-linux" } }, - "zig_0.16.0-dev.1484_Pd0ba6642b_aarch64-macos": { + "zig_0.16.0-dev.1878_P2a02c7a0d_aarch64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1484+d0ba6642b.tar.xz", + "url": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1878+2a02c7a0d.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "b521ed99eecad23575de1c138d9efc2fa4e3c3b9afd454af66c91707e3c89d32", - "zig_version": "0.16.0-dev.1484+d0ba6642b", + "sha256": "bf264609d8c3b07fc315324ceb1e4b3fb0cbe17c9842a4c663fca29715805a58", + "zig_version": "0.16.0-dev.1878+2a02c7a0d", "platform": "aarch64-macos" } }, - "zig_0.16.0-dev.1484_Pd0ba6642b_aarch64-windows": { + "zig_0.16.0-dev.1878_P2a02c7a0d_aarch64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1484+d0ba6642b.zip", + "url": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1878+2a02c7a0d.zip", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "62beade8036e74f10009c133ae81454df690506c447c215baf891d828f38761e", - "zig_version": "0.16.0-dev.1484+d0ba6642b", + "sha256": "552e4e7ddd868a8f9aa59d7c3f252f16b4317642d59ee64545ceecedeba789bf", + "zig_version": "0.16.0-dev.1878+2a02c7a0d", "platform": "aarch64-windows" } }, - "zig_0.16.0-dev.1484_Pd0ba6642b_x86_64-linux": { + "zig_0.16.0-dev.1878_P2a02c7a0d_x86_64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1484+d0ba6642b.tar.xz", + "url": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1878+2a02c7a0d.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "3f90582cde6e4ea4a9b9b1dd283a6720131dfefbece1c85be2f4c82ab74a2e7b", - "zig_version": "0.16.0-dev.1484+d0ba6642b", + "sha256": "a9095696174976aae6da0b925f5d994e324be3fc4f0a55497ecfea9331ba1f0b", + "zig_version": "0.16.0-dev.1878+2a02c7a0d", "platform": "x86_64-linux" } }, - "zig_0.16.0-dev.1484_Pd0ba6642b_x86_64-macos": { + "zig_0.16.0-dev.1878_P2a02c7a0d_x86_64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1484+d0ba6642b.tar.xz", + "url": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1878+2a02c7a0d.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "7055d0af0dd99af4e845a016e25fdd3be26c0e2c331b98902d2510c737e8b50e", - "zig_version": "0.16.0-dev.1484+d0ba6642b", + "sha256": "93ca19d2d2d4f05689d934dd691594533972b2adfa6c8f6b219a215fc9d37e1b", + "zig_version": "0.16.0-dev.1878+2a02c7a0d", "platform": "x86_64-macos" } }, - "zig_0.16.0-dev.1484_Pd0ba6642b_x86_64-windows": { + "zig_0.16.0-dev.1878_P2a02c7a0d_x86_64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1484+d0ba6642b.zip", + "url": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1878+2a02c7a0d.zip", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "14cd84fa0977c6893257c10a89a87b730815f3a71426e9494feea2accb768144", - "zig_version": "0.16.0-dev.1484+d0ba6642b", + "sha256": "7912bb842839ec03887f9a8b86da24afb1ac511e29471be44caec5440f7d7f12", + "zig_version": "0.16.0-dev.1878+2a02c7a0d", "platform": "x86_64-windows" } }, @@ -2939,28 +2939,28 @@ "repoRuleId": "@@rules_zig+//zig/private/repo:toolchains_repo.bzl%toolchains_repo", "attributes": { "names": [ - "zig_0.16.0-dev.1484_Pd0ba6642b_aarch64-linux", - "zig_0.16.0-dev.1484_Pd0ba6642b_aarch64-macos", - "zig_0.16.0-dev.1484_Pd0ba6642b_aarch64-windows", - "zig_0.16.0-dev.1484_Pd0ba6642b_x86_64-linux", - "zig_0.16.0-dev.1484_Pd0ba6642b_x86_64-macos", - "zig_0.16.0-dev.1484_Pd0ba6642b_x86_64-windows" + "zig_0.16.0-dev.1878_P2a02c7a0d_aarch64-linux", + "zig_0.16.0-dev.1878_P2a02c7a0d_aarch64-macos", + "zig_0.16.0-dev.1878_P2a02c7a0d_aarch64-windows", + "zig_0.16.0-dev.1878_P2a02c7a0d_x86_64-linux", + "zig_0.16.0-dev.1878_P2a02c7a0d_x86_64-macos", + "zig_0.16.0-dev.1878_P2a02c7a0d_x86_64-windows" ], "labels": [ - "@zig_0.16.0-dev.1484_Pd0ba6642b_aarch64-linux//:zig_toolchain", - "@zig_0.16.0-dev.1484_Pd0ba6642b_aarch64-macos//:zig_toolchain", - "@zig_0.16.0-dev.1484_Pd0ba6642b_aarch64-windows//:zig_toolchain", - "@zig_0.16.0-dev.1484_Pd0ba6642b_x86_64-linux//:zig_toolchain", - "@zig_0.16.0-dev.1484_Pd0ba6642b_x86_64-macos//:zig_toolchain", - "@zig_0.16.0-dev.1484_Pd0ba6642b_x86_64-windows//:zig_toolchain" + "@zig_0.16.0-dev.1878_P2a02c7a0d_aarch64-linux//:zig_toolchain", + "@zig_0.16.0-dev.1878_P2a02c7a0d_aarch64-macos//:zig_toolchain", + "@zig_0.16.0-dev.1878_P2a02c7a0d_aarch64-windows//:zig_toolchain", + "@zig_0.16.0-dev.1878_P2a02c7a0d_x86_64-linux//:zig_toolchain", + "@zig_0.16.0-dev.1878_P2a02c7a0d_x86_64-macos//:zig_toolchain", + "@zig_0.16.0-dev.1878_P2a02c7a0d_x86_64-windows//:zig_toolchain" ], "zig_versions": [ - "0.16.0-dev.1484+d0ba6642b", - "0.16.0-dev.1484+d0ba6642b", - "0.16.0-dev.1484+d0ba6642b", - "0.16.0-dev.1484+d0ba6642b", - "0.16.0-dev.1484+d0ba6642b", - "0.16.0-dev.1484+d0ba6642b" + "0.16.0-dev.1878+2a02c7a0d", + "0.16.0-dev.1878+2a02c7a0d", + "0.16.0-dev.1878+2a02c7a0d", + "0.16.0-dev.1878+2a02c7a0d", + "0.16.0-dev.1878+2a02c7a0d", + "0.16.0-dev.1878+2a02c7a0d" ], "exec_lengths": [ 2, diff --git a/bazel/zig_index.json b/bazel/zig_index.json index 43dbe8a6..c8561921 100644 --- a/bazel/zig_index.json +++ b/bazel/zig_index.json @@ -1,133 +1,133 @@ { "master": { - "version": "0.16.0-dev.1484+d0ba6642b", - "date": "2025-11-26", + "version": "0.16.0-dev.1878+2a02c7a0d", + "date": "2025-12-30", "docs": "https://ziglang.org/documentation/master/", "stdDocs": "https://ziglang.org/documentation/master/std/", "src": { - "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.1484+d0ba6642b.tar.xz", - "shasum": "a7ee7eae112fda7d3ddaab0fb110eee34692e0a1e12fd016e121a1f954c0a50c", - "size": "21703792" + "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "shasum": "23a81e334522ea78dbef5a63c4570446adfbefe5d44f4fae216b615b5786bf02", + "size": "21859732" }, "bootstrap": { - "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.1484+d0ba6642b.tar.xz", - "shasum": "db7d3943bfb389fca4e77f8f55b1e17710b3453fe05fd3bfdc97fa5cf955b7d5", - "size": "54466044" + "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "shasum": "9b03560d3531e6cf24827afc966f51a75673fe08948da943d9740441d7ef4540", + "size": "54621664" }, "x86_64-macos": { - "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1484+d0ba6642b.tar.xz", - "shasum": "7055d0af0dd99af4e845a016e25fdd3be26c0e2c331b98902d2510c737e8b50e", - "size": "56848400" + "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "shasum": "93ca19d2d2d4f05689d934dd691594533972b2adfa6c8f6b219a215fc9d37e1b", + "size": "56995132" }, "aarch64-macos": { - "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1484+d0ba6642b.tar.xz", - "shasum": "b521ed99eecad23575de1c138d9efc2fa4e3c3b9afd454af66c91707e3c89d32", - "size": "51712376" + "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "shasum": "bf264609d8c3b07fc315324ceb1e4b3fb0cbe17c9842a4c663fca29715805a58", + "size": "51853136" }, "x86_64-linux": { - "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1484+d0ba6642b.tar.xz", - "shasum": "3f90582cde6e4ea4a9b9b1dd283a6720131dfefbece1c85be2f4c82ab74a2e7b", - "size": "54737932" + "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "shasum": "a9095696174976aae6da0b925f5d994e324be3fc4f0a55497ecfea9331ba1f0b", + "size": "54913924" }, "aarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1484+d0ba6642b.tar.xz", - "shasum": "f8240e9708376f3e808c59dbaf69a2839eb1a14507432ee9f7d6817482346bf9", - "size": "50502404" + "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "shasum": "76622697d65885d3d2b0aa21de9a680662db14d04174b022b83e9292d106e4f6", + "size": "50654532" }, "arm-linux": { - "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.1484+d0ba6642b.tar.xz", - "shasum": "5bf3b779b33ad4479e4380700393551df22804c6ca44e9126cbde0719b84a788", - "size": "51392924" + "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "shasum": "2cb8f15af2a49509929f751675080055cbf03377ccf72c8fb96dd8a79b561040", + "size": "51524380" }, "riscv64-linux": { - "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.1484+d0ba6642b.tar.xz", - "shasum": "3dfc535a064d60443a61c0fc672860287e006db5667da3b345e614a4ca75a577", - "size": "54552624" + "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "shasum": "576edcdc9404c25d6f0c106d6ef860c2098803866702f675e9431d61b87d3343", + "size": "54717592" }, "powerpc64le-linux": { - "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.1484+d0ba6642b.tar.xz", - "shasum": "e8fa27e9bfaffe74ec85dc07133658587d3ab6f84947640acb92b4b24c06b348", - "size": "54592728" + "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "shasum": "5fdd195feec0fc7b4a9f8d4276e73793de4723115d193767d34a48617f81eca1", + "size": "54727752" }, "x86-linux": { - "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.1484+d0ba6642b.tar.xz", - "shasum": "ca98b5a992fbf4e4f66b8ce6ad37e1280e4cdc0e4d5aa4556b4dd84c2b9a4161", - "size": "57448984" + "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "shasum": "ff2e70a50d5fd548b40a199511abd69b3b6ceb1faf6d695e6c096f74693c4833", + "size": "57578920" }, "loongarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.1484+d0ba6642b.tar.xz", - "shasum": "d6a3829bd4c18603261e9b4a9e7ece7eb44ac3c9e784e0edacd8527a70e2f245", - "size": "51807684" + "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "shasum": "e6bde81c6bb63e1135006ee156d9563d28aa1b69202f1d7ee79642e0a8eb819f", + "size": "51954144" }, "s390x-linux": { - "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.1484+d0ba6642b.tar.xz", - "shasum": "bb1e5bba6d3b1e4d95bb9bf51042e3887b0aa1e55f39800eafbc49d1eece67bf", - "size": "54272976" + "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "shasum": "7552d69fc7b78fbc662af4df837849c40f00a49aa7d24f141797b3cc65a35f0d", + "size": "54411980" }, "x86_64-windows": { - "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1484+d0ba6642b.zip", - "shasum": "14cd84fa0977c6893257c10a89a87b730815f3a71426e9494feea2accb768144", - "size": "94726172" + "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1878+2a02c7a0d.zip", + "shasum": "7912bb842839ec03887f9a8b86da24afb1ac511e29471be44caec5440f7d7f12", + "size": "94976682" }, "aarch64-windows": { - "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1484+d0ba6642b.zip", - "shasum": "62beade8036e74f10009c133ae81454df690506c447c215baf891d828f38761e", - "size": "90710289" + "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1878+2a02c7a0d.zip", + "shasum": "552e4e7ddd868a8f9aa59d7c3f252f16b4317642d59ee64545ceecedeba789bf", + "size": "90930367" }, "x86-windows": { - "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.1484+d0ba6642b.zip", - "shasum": "f5331ec9511d5b85f51635aa9487b9ebb93acdde8b973c92019b1226eda9b853", - "size": "96574941" + "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.1878+2a02c7a0d.zip", + "shasum": "a44128f8aedc38152e57552687a988b7163d852a835df1cc2c8e67871e7d017d", + "size": "96809617" }, "aarch64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.1484+d0ba6642b.tar.xz", - "shasum": "5291d755fe5b5a43449c3d763298e33b8c370eb99572f298a1fc1296e72b3ddb", - "size": "50454224" + "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "shasum": "13f878e9f68d135ed6130f2fb514fa96485bbbd9b4d356c76d1fdcd17e7c8a3f", + "size": "50590876" }, "arm-freebsd": { - "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.1484+d0ba6642b.tar.xz", - "shasum": "3949292db30948364bf7e5c2a012b24ae8449096218c8f3ed558b61391211fdc", - "size": "52058888" + "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "shasum": "e02f83f42f985b30496a9ceeae05e8ad99cb09d5462c7ee4c920557293d48e66", + "size": "52205192" }, "powerpc64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-powerpc64-freebsd-0.16.0-dev.1484+d0ba6642b.tar.xz", - "shasum": "1916ec21059513d70764017753f22df316b1375c6e1258aa3464aad443e9cda1", - "size": "53141628" + "tarball": "https://ziglang.org/builds/zig-powerpc64-freebsd-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "shasum": "1dc506c30e954385e2499662738bf60546f4ed667b46cac3dc0402af8dd15f05", + "size": "53265316" }, "powerpc64le-freebsd": { - "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.1484+d0ba6642b.tar.xz", - "shasum": "d3381907956625bc0585f1fdc66a14b9b418cca6fd7b79ff01e8d5af11f25694", - "size": "54590608" + "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "shasum": "e2c2365a8d4da7828de6e7f6363f146a68dca9dbebc6296e6777e9d600c30464", + "size": "54738884" }, "riscv64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.1484+d0ba6642b.tar.xz", - "shasum": "96d66952a130af6a958c25aa8f131333c2e8fa0fed6582004ce4b45d9d97708b", - "size": "54719008" + "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "shasum": "c205e8a5c20a879ca13be5c9322b49ad7247cb581ac5922ed38d30b773ba48e4", + "size": "54844712" }, "x86_64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.1484+d0ba6642b.tar.xz", - "shasum": "fa5bddd439bacb8401553299d8e66aa3008804312aed0702f5a317e96cc713c0", - "size": "54855936" + "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "shasum": "2f9309473d1acf253a8e5eba3db43c814739794dc7316fa0364058b402535815", + "size": "55002612" }, "aarch64-netbsd": { - "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.1484+d0ba6642b.tar.xz", - "shasum": "64fec627697895f4ae979b9f05586d27b8114f18ec6c067139798f9928077e1f", - "size": "50418132" + "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "shasum": "d4db68ff2d81fa41d40db82da9c41d6d46baf247a3f2eb05ba9c1aaf903a0d07", + "size": "50568412" }, "arm-netbsd": { - "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.1484+d0ba6642b.tar.xz", - "shasum": "7e9d09a4971fb0a4b257527f0414b0adb04f83e0ffd6ebb067b3555095ab193b", - "size": "53085432" + "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "shasum": "3a39f6ceb25f0162e8a3cb0f4586a9a5d4ea24050bc30e418c60bc57f6155adc", + "size": "53258248" }, "x86-netbsd": { - "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.1484+d0ba6642b.tar.xz", - "shasum": "6fed94706492135b5a9db2421df435bdd3ca502d2a2eb1a4892d443340a05cac", - "size": "58054536" + "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "shasum": "2910bfbf92119bdbb46ee9e0cf37c3cd828160563e48a1b007683bb9ccda5b76", + "size": "58192980" }, "x86_64-netbsd": { - "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.1484+d0ba6642b.tar.xz", - "shasum": "3e375f4aa4c7e31391ec5bbef5e8a5eaf14750faf5510c09074b76d61525ef4a", - "size": "54805628" + "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "shasum": "8454376bb6031dbd2eb809de0b6309a8109db811f284110bbae4c84a7fc77ed5", + "size": "54951836" } }, "0.15.2": { diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 6272dcaa..a6781a8b 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -40,7 +40,7 @@ pub fn main() !void { //const allocator = gpa.allocator(); - var threaded: std.Io.Threaded = .init(allocator); + var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); var vfs_file: zml.io.VFS.File = .init(allocator, threaded.io(), .{}); diff --git a/third_party/arocc/repo.bzl b/third_party/arocc/repo.bzl index 818ad68e..ac5f8573 100644 --- a/third_party/arocc/repo.bzl +++ b/third_party/arocc/repo.bzl @@ -3,7 +3,7 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") def repo(): new_git_repository( name = "arocc", - remote = "https://github.com/Vexu/arocc.git", - commit = "3c7b4545b2ac30b9da41dbc8eacd2f14af44bc6c", + remote = "https://github.com/zml/arocc.git", + commit = "7e60c78a9660016e46d8be8907591b143ba2e700", build_file = "//third_party/arocc:arocc.bazel", ) diff --git a/third_party/translate-c/repo.bzl b/third_party/translate-c/repo.bzl index 0390cb87..2b3facf1 100644 --- a/third_party/translate-c/repo.bzl +++ b/third_party/translate-c/repo.bzl @@ -3,7 +3,7 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") def repo(): new_git_repository( name = "translate-c", - remote = "https://codeberg.org/ziglang/translate-c.git", - commit = "55f225f0a37b9627d459ed23b7df21f458f492d9", + remote = "https://github.com/zml/translate-c.git", + commit = "5308ec6eba13a96dcaec452bffd0cc946384909a", build_file = "//third_party/translate-c:translate-c.bazel", ) diff --git a/third_party/zls/workspace_printer.zig b/third_party/zls/workspace_printer.zig index 52a3a652..7fe077d3 100644 --- a/third_party/zls/workspace_printer.zig +++ b/third_party/zls/workspace_printer.zig @@ -79,7 +79,7 @@ pub fn main_016() !void { var arena = std.heap.ArenaAllocator.init(std.heap.smp_allocator); defer arena.deinit(); - var threaded = std.Io.Threaded.init(arena.allocator()); + var threaded: std.Io.Threaded = .init(arena.allocator(), .{}); defer threaded.deinit(); const io = threaded.io(); diff --git a/third_party/zls/zls_runner.zig b/third_party/zls/zls_runner.zig index 0bfd0aca..5ca4f695 100644 --- a/third_party/zls/zls_runner.zig +++ b/third_party/zls/zls_runner.zig @@ -7,21 +7,9 @@ const std = @import("std"); const runfiles = @import("runfiles"); const bazel_builtin = @import("bazel_builtin"); -fn getRandomFilename(buf: *[std.fs.max_name_bytes]u8, extension: []const u8) ![]const u8 { - const random_bytes_count = 12; - const sub_path_len = comptime std.fs.base64_encoder.calcSize(random_bytes_count); - - var random_bytes: [random_bytes_count]u8 = undefined; - std.crypto.random.bytes(&random_bytes); - var random_name: [sub_path_len]u8 = undefined; - _ = std.fs.base64_encoder.encode(&random_name, &random_bytes); - - const fmt_template = "/tmp/{s}{s}"; - const fmt_args = .{ - @as([]const u8, &random_name), - extension, - }; - return std.fmt.bufPrint(buf, fmt_template, fmt_args) catch @panic("OOM"); +fn getRandomFilename(io: std.Io, buf: []u8, extension: []const u8) ![]const u8 { + const now = try std.Io.Clock.real.now(io); + return std.fmt.bufPrint(buf, "/tmp/{d}{s}", .{ now.nanoseconds, extension }) catch @panic("OOM"); } const Config = struct { @@ -43,10 +31,12 @@ pub fn main() !void { var arena = std.heap.ArenaAllocator.init(gpa); const allocator = arena.allocator(); - var threaded: std.Io.Threaded = .init(gpa); + var threaded: std.Io.Threaded = .init(gpa, .{}); defer threaded.deinit(); - var r_ = try runfiles.Runfiles.create(.{ .allocator = allocator, .io = threaded.io() }) orelse + const io = threaded.io(); + + var r_ = try runfiles.Runfiles.create(.{ .allocator = allocator, .io = io }) orelse return error.RunfilesNotFound; defer r_.deinit(allocator); @@ -91,15 +81,15 @@ pub fn main() !void { .global_cache_path = global_cache_path, }; - var buf: [std.fs.max_name_bytes]u8 = undefined; - const tmp_file_path = try getRandomFilename(&buf, ".json"); + var buf: [std.Io.Dir.max_name_bytes]u8 = undefined; + const tmp_file_path = try getRandomFilename(io, &buf, ".json"); { - var tmp_file = try std.fs.createFileAbsolute(tmp_file_path, .{}); - defer tmp_file.close(); + var tmp_file = try std.Io.Dir.createFileAbsolute(io, tmp_file_path, .{}); + defer tmp_file.close(io); var out_buf: [4096]u8 = undefined; - var tmp_file_writer = tmp_file.writer(&out_buf); + var tmp_file_writer = tmp_file.writer(io, &out_buf); const formatter = std.json.fmt(config, .{ .whitespace = .indent_2 }); diff --git a/zml2/io.zig b/zml2/io.zig index 277df35d..d49c7edc 100644 --- a/zml2/io.zig +++ b/zml2/io.zig @@ -1,20 +1,19 @@ -const builtin = @import("builtin"); const std = @import("std"); +const builtin = @import("builtin"); const stdx = @import("stdx"); - pub const VFS = @import("io").VFS; -const log = std.log.scoped(.@"zml/io"); - -const safetensors = @import("safetensors.zig"); -const Tensor = @import("tensor.zig").Tensor; -const Shape = @import("shape.zig").Shape; -const Platform = @import("platform.zig").Platform; const Buffer = @import("buffer.zig").Buffer; -const meta = @import("meta.zig"); const Bufferized = @import("zml.zig").Bufferized; +const meta = @import("meta.zig"); const pjrt = @import("pjrtx.zig"); +const Platform = @import("platform.zig").Platform; +const safetensors = @import("safetensors.zig"); +const Shape = @import("shape.zig").Shape; +const Tensor = @import("tensor.zig").Tensor; + +const log = std.log.scoped(.@"zml/io"); pub const TensorStore = struct { registry: *safetensors.TensorRegistry, diff --git a/zml2/io/vfs/BUILD.bazel b/zml2/io/vfs/BUILD.bazel index c60e0dce..0a0bc3c4 100644 --- a/zml2/io/vfs/BUILD.bazel +++ b/zml2/io/vfs/BUILD.bazel @@ -1,12 +1,22 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_binary") zig_library( name = "vfs", srcs = [ + "base.zig", "file.zig", "hf.zig", + "proxy.zig", "http.zig", ], main = "index.zig", visibility = ["//visibility:public"], ) + +zig_binary( + name = "hf2", + main = "hf2.zig", + srcs = ["base.zig"], + deps = ["//stdx"], + visibility = ["//visibility:public"], +) diff --git a/zml2/io/vfs/base.zig b/zml2/io/vfs/base.zig new file mode 100644 index 00000000..d98c338e --- /dev/null +++ b/zml2/io/vfs/base.zig @@ -0,0 +1,374 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +pub const VFSBase = struct { + pub fn HandleRegistry(comptime InnerType: type) type { + return struct { + const Self = @This(); + + const Node = struct { + value: InnerType, + next_closed: ?usize = null, + }; + + mutex: std.Io.Mutex = .init, + handles: stdx.SegmentedList(InnerType, 0) = .{}, + next_closed: ?usize = null, + + pub fn open(self: *Self, allocator: std.mem.Allocator, io: std.Io) !struct { usize, *InnerType } { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + + if (self.next_closed) |handle| { + const node = self.handles.at(handle); + self.next_closed = node.next_closed; + return .{ handle, node.value }; + } + return .{ self.handles.count(), try self.handles.addOne(allocator) }; + } + + pub fn close(self: *Self, io: std.Io, handle: usize) void { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + + const node = self.handles.at(handle); + node.next_closed = self.next_closed; + self.next_closed = handle; + } + + pub fn get(self: *Self, io: std.Io, handle: usize) *InnerType { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + + return &self.handles.at(handle).value; + } + }; + } + + inner: std.Io, + + pub fn init(io: std.Io) VFSBase { + return .{ .inner = io }; + } + + pub fn vtable(overrides: anytype) std.Io.VTable { + var new_vtable: std.Io.VTable = .{ + .dirMake = dirMake, + .dirMakePath = dirMakePath, + .dirMakeOpenPath = dirMakeOpenPath, + .dirStat = dirStat, + .dirStatPath = dirStatPath, + .dirAccess = dirAccess, + .dirCreateFile = dirCreateFile, + .dirOpenFile = dirOpenFile, + .dirOpenDir = dirOpenDir, + .dirClose = dirClose, + .fileStat = fileStat, + .fileClose = fileClose, + .fileWriteStreaming = fileWriteStreaming, + .fileWritePositional = fileWritePositional, + .fileReadStreaming = fileReadStreaming, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .openSelfExe = openSelfExe, + .async = async, + .concurrent = concurrent, + .await = await, + .cancel = cancel, + .cancelRequested = cancelRequested, + .groupAsync = groupAsync, + .groupConcurrent = groupConcurrent, + .groupWait = groupWait, + .groupCancel = groupCancel, + .select = select, + .mutexLock = mutexLock, + .mutexLockUncancelable = mutexLockUncancelable, + .mutexUnlock = mutexUnlock, + .conditionWait = conditionWait, + .conditionWaitUncancelable = conditionWaitUncancelable, + .conditionWake = conditionWake, + .now = now, + .sleep = sleep, + .netListenIp = netListenIp, + .netAccept = netAccept, + .netBindIp = netBindIp, + .netConnectIp = netConnectIp, + .netListenUnix = netListenUnix, + .netConnectUnix = netConnectUnix, + .netSend = netSend, + .netReceive = netReceive, + .netRead = netRead, + .netWrite = netWrite, + .netClose = netClose, + .netInterfaceNameResolve = netInterfaceNameResolve, + .netInterfaceName = netInterfaceName, + .netLookup = netLookup, + }; + for (std.meta.fieldNames(@TypeOf(overrides))) |field_name| { + @field(new_vtable, field_name) = @field(overrides, field_name); + } + return new_vtable; + } + + pub fn as(userdata: ?*anyopaque) *VFSBase { + return @ptrCast(@alignCast(userdata orelse unreachable)); + } + + pub fn dirMake(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, mode: std.Io.Dir.Mode) std.Io.Dir.MakeError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirMake(self.inner.userdata, dir, sub_path, mode); + } + + pub fn dirMakePath(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, mode: std.Io.Dir.Mode) std.Io.Dir.MakeError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirMakePath(self.inner.userdata, dir, sub_path, mode); + } + + pub fn dirMakeOpenPath(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.MakeOpenPathError!std.Io.Dir { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirMakeOpenPath(self.inner.userdata, dir, sub_path, options); + } + + pub fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirStat(self.inner.userdata, dir); + } + + pub fn dirStatPath(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatPathOptions) std.Io.Dir.StatPathError!std.Io.Dir.Stat { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirStatPath(self.inner.userdata, dir, sub_path, options); + } + + pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirAccess(self.inner.userdata, dir, sub_path, options); + } + + pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.CreateFlags) std.Io.File.OpenError!std.Io.File { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirCreateFile(self.inner.userdata, dir, sub_path, flags); + } + + pub fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirOpenFile(self.inner.userdata, dir, sub_path, flags); + } + + pub fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirOpenDir(self.inner.userdata, dir, sub_path, options); + } + + pub fn dirClose(userdata: ?*anyopaque, dir: std.Io.Dir) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + self.inner.vtable.dirClose(self.inner.userdata, dir); + } + + pub fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileStat(self.inner.userdata, file); + } + + pub fn fileClose(userdata: ?*anyopaque, file: std.Io.File) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + self.inner.vtable.fileClose(self.inner.userdata, file); + } + + pub fn fileWriteStreaming(userdata: ?*anyopaque, file: std.Io.File, buffer: [][]const u8) std.Io.File.WriteStreamingError!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileWriteStreaming(self.inner.userdata, file, buffer); + } + + pub fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, buffer: [][]const u8, offset: u64) std.Io.File.WritePositionalError!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileWritePositional(self.inner.userdata, file, buffer, offset); + } + + pub fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: [][]u8) std.Io.File.Reader.Error!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileReadStreaming(self.inner.userdata, file, data); + } + + pub fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: [][]u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileReadPositional(self.inner.userdata, file, data, offset); + } + + pub fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileSeekBy(self.inner.userdata, file, relative_offset); + } + + pub fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileSeekTo(self.inner.userdata, file, absolute_offset); + } + + pub fn openSelfExe(userdata: ?*anyopaque, flags: std.Io.File.OpenFlags) std.Io.File.OpenSelfExeError!std.Io.File { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.openSelfExe(self.inner.userdata, flags); + } + + pub fn async(userdata: ?*anyopaque, result: []u8, result_alignment: std.mem.Alignment, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque, result: *anyopaque) void) ?*std.Io.AnyFuture { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.async(self.inner.userdata, result, result_alignment, context, context_alignment, start); + } + + pub fn concurrent(userdata: ?*anyopaque, result_len: usize, result_alignment: std.mem.Alignment, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque, result: *anyopaque) void) std.Io.ConcurrentError!*std.Io.AnyFuture { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.concurrent(self.inner.userdata, result_len, result_alignment, context, context_alignment, start); + } + + pub fn await(userdata: ?*anyopaque, any_future: *std.Io.AnyFuture, result: []u8, result_alignment: std.mem.Alignment) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.await(self.inner.userdata, any_future, result, result_alignment); + } + + pub fn cancel(userdata: ?*anyopaque, any_future: *std.Io.AnyFuture, result: []u8, result_alignment: std.mem.Alignment) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.cancel(self.inner.userdata, any_future, result, result_alignment); + } + + pub fn cancelRequested(userdata: ?*anyopaque) bool { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.cancelRequested(self.inner.userdata); + } + + pub fn groupAsync(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (*std.Io.Group, context: *const anyopaque) void) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.groupAsync(self.inner.userdata, group, context, context_alignment, start); + } + + pub fn groupConcurrent(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (*std.Io.Group, context: *const anyopaque) void) std.Io.ConcurrentError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.groupConcurrent(self.inner.userdata, group, context, context_alignment, start); + } + + pub fn groupWait(userdata: ?*anyopaque, group: *std.Io.Group, token: *anyopaque) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.groupWait(self.inner.userdata, group, token); + } + + pub fn groupCancel(userdata: ?*anyopaque, group: *std.Io.Group, token: *anyopaque) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.groupCancel(self.inner.userdata, group, token); + } + + pub fn select(userdata: ?*anyopaque, futures: []const *std.Io.AnyFuture) std.Io.Cancelable!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.select(self.inner.userdata, futures); + } + + pub fn mutexLock(userdata: ?*anyopaque, prev_state: std.Io.Mutex.State, mutex: *std.Io.Mutex) std.Io.Cancelable!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.mutexLock(self.inner.userdata, prev_state, mutex); + } + + pub fn mutexLockUncancelable(userdata: ?*anyopaque, prev_state: std.Io.Mutex.State, mutex: *std.Io.Mutex) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + std.debug.print(">>>>>>>> {*}\n", .{self}); + self.inner.vtable.mutexLockUncancelable(self.inner.userdata, prev_state, mutex); + } + + pub fn mutexUnlock(userdata: ?*anyopaque, prev_state: std.Io.Mutex.State, mutex: *std.Io.Mutex) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + self.inner.vtable.mutexUnlock(self.inner.userdata, prev_state, mutex); + } + + pub fn conditionWait(userdata: ?*anyopaque, cond: *std.Io.Condition, mutex: *std.Io.Mutex) std.Io.Cancelable!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.conditionWait(self.inner.userdata, cond, mutex); + } + + pub fn conditionWaitUncancelable(userdata: ?*anyopaque, cond: *std.Io.Condition, mutex: *std.Io.Mutex) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + self.inner.vtable.conditionWaitUncancelable(self.inner.userdata, cond, mutex); + } + + pub fn conditionWake(userdata: ?*anyopaque, cond: *std.Io.Condition, wake: std.Io.Condition.Wake) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + self.inner.vtable.conditionWake(self.inner.userdata, cond, wake); + } + + pub fn now(userdata: ?*anyopaque, clock: std.Io.Clock) std.Io.Clock.Error!std.Io.Timestamp { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.now(self.inner.userdata, clock); + } + + pub fn sleep(userdata: ?*anyopaque, timeout: std.Io.Timeout) std.Io.SleepError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.sleep(self.inner.userdata, timeout); + } + + pub fn netListenIp(userdata: ?*anyopaque, address: std.Io.net.IpAddress, options: std.Io.net.IpAddress.ListenOptions) std.Io.net.IpAddress.ListenError!std.Io.net.Server { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netListenIp(self.inner.userdata, address, options); + } + + pub fn netAccept(userdata: ?*anyopaque, server: std.Io.net.Socket.Handle) std.Io.net.Server.AcceptError!std.Io.net.Stream { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netAccept(self.inner.userdata, server); + } + + pub fn netBindIp(userdata: ?*anyopaque, address: *const std.Io.net.IpAddress, options: std.Io.net.IpAddress.BindOptions) std.Io.net.IpAddress.BindError!std.Io.net.Socket { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netBindIp(self.inner.userdata, address, options); + } + + pub fn netConnectIp(userdata: ?*anyopaque, address: *const std.Io.net.IpAddress, options: std.Io.net.IpAddress.ConnectOptions) std.Io.net.IpAddress.ConnectError!std.Io.net.Stream { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netConnectIp(self.inner.userdata, address, options); + } + + pub fn netListenUnix(userdata: ?*anyopaque, address: *const std.Io.net.UnixAddress, options: std.Io.net.UnixAddress.ListenOptions) std.Io.net.UnixAddress.ListenError!std.Io.net.Socket.Handle { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netListenUnix(self.inner.userdata, address, options); + } + + pub fn netConnectUnix(userdata: ?*anyopaque, address: *const std.Io.net.UnixAddress) std.Io.net.UnixAddress.ConnectError!std.Io.net.Socket.Handle { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netConnectUnix(self.inner.userdata, address); + } + + pub fn netSend(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, msgs: []std.Io.net.OutgoingMessage, flags: std.Io.net.SendFlags) struct { ?std.Io.net.Socket.SendError, usize } { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netSend(self.inner.userdata, handle, msgs, flags); + } + + pub fn netReceive(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, message_buffer: []std.Io.net.IncomingMessage, data_buffer: []u8, flags: std.Io.net.ReceiveFlags, timeout: std.Io.Timeout) struct { ?std.Io.net.Socket.ReceiveTimeoutError, usize } { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netReceive(self.inner.userdata, handle, message_buffer, data_buffer, flags, timeout); + } + + pub fn netRead(userdata: ?*anyopaque, src: std.Io.net.Socket.Handle, data: [][]u8) std.Io.net.Stream.Reader.Error!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netRead(self.inner.userdata, src, data); + } + + pub fn netWrite(userdata: ?*anyopaque, dest: std.Io.net.Socket.Handle, header: []const u8, data: []const []const u8, splat: usize) std.Io.net.Stream.Writer.Error!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netWrite(self.inner.userdata, dest, header, data, splat); + } + + pub fn netClose(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + self.inner.vtable.netClose(self.inner.userdata, handle); + } + + pub fn netInterfaceNameResolve(userdata: ?*anyopaque, name: *const std.Io.net.Interface.Name) std.Io.net.Interface.Name.ResolveError!std.Io.net.Interface { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netInterfaceNameResolve(self.inner.userdata, name); + } + + pub fn netInterfaceName(userdata: ?*anyopaque, iface: std.Io.net.Interface) std.Io.net.Interface.NameError!std.Io.net.Interface.Name { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netInterfaceName(self.inner.userdata, iface); + } + + pub fn netLookup(userdata: ?*anyopaque, host: std.Io.net.HostName, q: *std.Io.Queue(std.Io.net.HostName.LookupResult), options: std.Io.net.HostName.LookupOptions) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netLookup(self.inner.userdata, host, q, options); + } +}; diff --git a/zml2/io/vfs/hf2.zig b/zml2/io/vfs/hf2.zig new file mode 100644 index 00000000..7864514c --- /dev/null +++ b/zml2/io/vfs/hf2.zig @@ -0,0 +1,384 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const VFSBase = @import("base.zig").VFSBase; + +const log = std.log.scoped(.@"zml/io/vfs/hf"); + +pub const API = struct { + const TREE_URL_TEMPLATE = "https://huggingface.co/api/models/{[repo]s}/{[model]s}/tree/{[rev]s}/{[path]s}"; + const TREESIZE_URL_TEMPLATE = "https://huggingface.co/api/models/{[repo]s}/{[model]s}/treesize/{[rev]s}/{[path]s}"; + const RAW_FILE_URL_REMPLATE = "https://huggingface.co/{model}/raw/{rev}/{path}"; + const LFS_FILE_URL_TEMPLATE = "https://huggingface.co/{[repo]s}/{[model]s}/resolve/{[rev]s}/{[path]s}"; + + pub const Entry = struct { + pub const Type = enum { + file, + directory, + }; + + pub const LFS = struct { + oid: []const u8, + size: u64, + pointerSize: u64, + }; + + type: Type, + oid: []const u8, + size: u64, + path: []const u8, + lfs: ?LFS = null, + }; +}; + +pub const HF = struct { + pub const Repo = struct { + repo: []const u8, + model: []const u8, + rev: []const u8, + path: []const u8, + + pub fn parse(uri: []const u8) !Repo { + var parts = std.mem.splitScalar(u8, uri, '/'); + var repo: Repo = .{ + .repo = parts.first(), + .model = parts.next() orelse return error.InvalidURI, + .rev = "main", + .path = parts.rest(), + }; + if (std.mem.findScalar(u8, repo.model, '@')) |at_index| { + repo.rev = repo.model[at_index + 1 ..]; + repo.model = repo.model[0..at_index]; + } + return repo; + } + }; + + pub const Handle = struct { + pub const Type = enum { + file, + directory, + }; + + type: Type, + url: []const u8, + uri: std.Uri, + pos: u64, + size: u64, + + pub fn init(allocator: std.mem.Allocator, type_: Type, sub_path: []const u8, size: u64) Handle { + const repo = Repo.parse(sub_path) catch unreachable; + const lfs_url = std.fmt.allocPrint(allocator, API.LFS_FILE_URL_TEMPLATE, repo) catch unreachable; + errdefer allocator.free(lfs_url); + + return .{ + .type = type_, + .url = lfs_url, + .uri = std.Uri.parse(lfs_url) catch unreachable, + .pos = 0, + .size = size, + }; + } + + pub fn deinit(self: *Handle, allocator: std.mem.Allocator) void { + allocator.free(self.url); + } + }; + + base: VFSBase, + authorization: std.http.Client.Request.Headers.Value = .default, + token_len: usize = 0, + allocator: std.mem.Allocator, + client: *std.http.Client, + handles: std.ArrayList(Handle) = .{}, + closed_handles: std.ArrayList(u32) = .{}, + + pub fn init(allocator: std.mem.Allocator, io_: std.Io, http_client_: *std.http.Client) !HF { + return .{ + .allocator = allocator, + .base = .init(io_), + .client = http_client_, + .authorization = .{ + .override = "Bearer XXX", + }, + }; + } + + pub fn deinit(self: *HF) void { + _ = self; // autofix + } + + pub fn io(self: *HF) std.Io { + return .{ + .userdata = &self.base, + .vtable = &comptime VFSBase.vtable(.{ + .dirOpenFile = dirOpenFile, + .fileReadPositional = fileReadPositional, + .fileReadStreaming = fileReadStreaming, + .fileClose = fileClose, + }), + }; + } + + pub fn openHandle(self: *HF) !struct { u32, *Handle } { + if (self.closed_handles.pop()) |idx| { + return .{ idx, &self.handles.items[idx] }; + } + return .{ @intCast(self.handles.items.len), try self.handles.addOne(self.allocator) }; + } + + pub fn closeHandle(self: *HF, idx: u32) !void { + try self.closed_handles.append(self.allocator, idx); + } + + fn getFileHandle(self: *HF, file: std.Io.File) *Handle { + return &self.handles.items[@intCast(file.handle)]; + } + + fn parseContentRange(value: []const u8) struct { u64, u64, u64 } { + const space = std.mem.indexOfScalar(u8, value, ' ').?; + const range = std.mem.indexOfScalar(u8, value, '-').?; + const slash = std.mem.indexOfScalar(u8, value, '/').?; + + return .{ + std.fmt.parseInt(u64, value[space + 1 .. range], 10) catch 0, + std.fmt.parseInt(u64, value[range + 1 .. slash], 10) catch 0, + std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch 0, + }; + } + + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + _ = dir; // autofix + _ = flags; // autofix + // _ = flags; // autofix + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + + var url_buffer: [1024]u8 = undefined; + var redirect_buffer: [8 * 1024]u8 = undefined; + var aux_buffer: []u8 = &redirect_buffer; + + const repo = Repo.parse(sub_path) catch return std.Io.File.OpenError.BadPathName; + const url: []const u8 = std.fmt.bufPrint(&url_buffer, API.LFS_FILE_URL_TEMPLATE, repo) catch return std.Io.File.OpenError.BadPathName; + + log.info("Opening url: {s}", .{url}); + + var uri = std.Uri.parse(url) catch return std.Io.File.OpenError.BadPathName; + while (true) { + var req = self.client.request(.HEAD, uri, .{ + .headers = .{ .authorization = self.authorization }, + .redirect_behavior = .not_allowed, + }) catch return std.Io.File.Reader.Error.Unexpected; + defer req.deinit(); + + req.sendBodiless() catch { + log.err("Failed to send HTTP request for HF model size API URL", .{}); + return std.Io.File.Reader.Error.Unexpected; + }; + + var resp = req.receiveHead(&.{}) catch { + log.err("Failed to receive HTTP response for HF model size API URL", .{}); + return std.Io.File.Reader.Error.Unexpected; + }; + + if (resp.head.status.class() == .redirect) { + const location = resp.head.location.?; + @memcpy(aux_buffer[0..location.len], location); + uri = uri.resolveInPlace(location.len, &aux_buffer) catch unreachable; + log.info("Redirect {f}", .{uri}); + continue; + } + + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; + handle.* = .init(self.allocator, .file, sub_path, resp.head.content_length.?); + + return .{ .handle = @intCast(idx) }; + } + } + + pub fn performRead(self: *HF, handle: *Handle, data: [][]u8, offset: u64) std.Io.File.Reader.Error!usize { + if (offset >= handle.size) { + log.info("EOF", .{}); + return 0; + } + + var range_buf: [64]u8 = undefined; + const range_header = blk: { + var total_bytes: u64 = 0; + for (data) |buf| { + total_bytes += buf.len; + } + break :blk std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ offset, offset + total_bytes }) catch unreachable; + }; + + var req = self.client.request(.GET, handle.uri, .{ + .headers = .{ .authorization = self.authorization }, + .extra_headers = &.{ + .{ .name = "Range", .value = range_header }, + }, + }) catch return std.Io.File.Reader.Error.Unexpected; + defer req.deinit(); + + req.sendBodiless() catch { + log.err("Failed to send HTTP request for HF model size API URL", .{}); + return std.Io.File.Reader.Error.Unexpected; + }; + + var redirect_buffer: [8 * 1024]u8 = undefined; + var resp = req.receiveHead(&redirect_buffer) catch { + log.err("Failed to receive HTTP response for HF model size API URL", .{}); + return std.Io.File.Reader.Error.Unexpected; + }; + + const reader = resp.reader(&.{}); + var total_n: usize = 0; + while (true) { + total_n += reader.readSliceShort(buffer: []u8) .readVec(data) catch |err| return switch (err) { + std.Io.Reader.Error.EndOfStream => return total_n, + else => std.Io.File.Reader.Error.Unexpected, + }; + } + } + + pub fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: [][]u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return try self.performRead(handle, data, offset); + } + + fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: [][]u8) std.Io.File.Reader.Error!usize { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + const total = try self.performRead(handle, data, handle.pos); + handle.pos += @intCast(total); + return total; + } + + fn fileClose(userdata: ?*anyopaque, file: std.Io.File) void { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + self.closeHandle(@intCast(file.handle)) catch unreachable; + } +}; + +pub fn main() !void { + const allocator = std.heap.smp_allocator; + + var threaded: std.Io.Threaded = .init(allocator); + defer threaded.deinit(); + + var client: std.http.Client = .{ + .allocator = allocator, + .io = threaded.io(), + }; + + try client.initDefaultProxies(allocator); + defer client.deinit(); + + var hf: HF = try .init(allocator, threaded.io(), &client); + defer hf.deinit(); + + const io = hf.io(); + + const file = std.Io.Dir.cwd().openFile(io, "Qwen/Qwen3-8B/model-00001-of-00005.safetensors", .{}) catch |err| { + log.err("Failed to open HF file: {any}", .{err}); + unreachable; + }; + + log.info(">>>>>>> {any}", .{file}); + + const buffer = try allocator.alloc(u8, 128 * 1024 * 1024); + var reader = file.reader(io, &.{}); + defer log.info("DONE", .{}); + while (true) { + const read = try reader.interface.readSliceShort(buffer); + log.info("Read {d} bytes", .{read}); + if (read == 0) break; + } + + // var uri_buffer: [512]u8 = undefined; + // const repo: HF.Repo = try .parse("Qwen/Qwen3-8B@main/config.json"); + + // var buffer: [256]u8 = undefined; + // const gna = try std.fmt.bufPrint(&buffer, API.TREE_URL_TEMPLATE, repo); + // log.info(">>>> repo |{s}|", .{gna}); + + // var req = try self.client.request(.GET, try .parse(std.fmt.bufPrint(&uri_buffer, API.TREE_URL_TEMPLATE, .{repo})), .{ + // .headers = .{ + // .authorization = .{ .override = self.token() }, + // }, + // }); + // defer req.deinit(); + + // req.sendBodiless() catch { + // log.err("Failed to send HTTP request for HF model size API URL", .{}); + // return; + // }; + + // var redirect_buffer: [8 * 1024]u8 = undefined; + // var resp = req.receiveHead(&redirect_buffer) catch { + // log.err("Failed to receive HTTP response for HF model size API URL", .{}); + // return std.Io.Dir.OpenError.FileNotFound; + // }; + + // var read_buffer: [8 * 1024]u8 = undefined; + // const reader = resp.reader(&read_buffer); + // _ = reader; // autofix + + // https://huggingface.co/Qwen/Qwen3-8B/resolve/main/config.json + // https://huggingface.co/api/models/meta-llama/Llama-3.1-8B-Instruct/tree/main + // + // var req = try client.request(.GET, try .parse("https://huggingface.co/Qwen/Qwen3-8B/resolve/main/config.json"), .{ + // .headers = .{ + // .accept_encoding = .{ .override = "identity" }, + // .authorization = .{ .override = "Bearer XXX" }, + // }, + // }); + // defer req.deinit(); + + // req.sendBodiless() catch { + // log.err("Failed to send HTTP request for HF model size API URL", .{}); + // return; + // }; + + // var redirect_buffer: [16 * 1024]u8 = undefined; + // var resp = req.receiveHead(&redirect_buffer) catch { + // log.err("Failed to receive HTTP response for HF model size API URL", .{}); + // return; + // }; + + // log.info(">>>> {any}", .{resp.head}); + + // var read_buffer: [8 * 1024]u8 = undefined; + // const reader = resp.reader(&read_buffer); + + // var json_reader: std.json.Reader = .init(allocator, reader); + // const entries = std.json.parseFromTokenSourceLeaky([]API.Entry, allocator, &json_reader, .{ .ignore_unknown_fields = true }) catch |err| { + // log.err("Failed to parse HF model size response: {any}", .{err}); + // return; + // }; + + // log.info(">>>> {any}", .{entries}); + + // req. + // defer req.deinit(); + + // req.sendBodiless() catch { + // log.err("Failed to send HTTP request for HF model size API URL {s}", .{api_url}); + // return Error.RequestFailed; + // }; + + // var resp = req.receiveHead(&redirect_buffer) catch { + // log.err("Failed to receive HTTP response for HF model size API URL {s}", .{api_url}); + // return Error.RequestFailed; + // }; + + // if (resp.head.status != .ok) { + // log.err("Failed to fetch HF model size, got {s} for {s}", .{ @tagName(resp.head.status), api_url }); + // return Error.RequestFailed; + // } + + // const content_length = resp.head.content_length orelse { + // log.err("Missing Content-Length in HF model size response for {s}", .{api_url}); + // return Error.ResponseParsingFailed; + // }; +} diff --git a/zml2/io/vfs/proxy.zig b/zml2/io/vfs/proxy.zig new file mode 100644 index 00000000..75cae361 --- /dev/null +++ b/zml2/io/vfs/proxy.zig @@ -0,0 +1,331 @@ +const std = @import("std"); + +pub const IoProxy = struct { + inner: std.Io, + + pub fn init(io: std.Io) IoProxy { + return .{ .inner = io }; + } + + pub fn partial(overrides: anytype) std.Io.VTable { + var new_vtable = vtable; + for (std.meta.fieldNames(@TypeOf(overrides))) |field_name| { + @field(new_vtable, field_name) = @field(overrides, field_name); + } + return new_vtable; + } + + pub fn as(userdata: ?*anyopaque) *IoProxy { + return @ptrCast(@alignCast(userdata orelse unreachable)); + } + + pub const vtable: std.Io.VTable = .{ + .dirMake = dirMake, + .dirMakePath = dirMakePath, + .dirMakeOpenPath = dirMakeOpenPath, + .dirStat = dirStat, + .dirStatPath = dirStatPath, + .dirAccess = dirAccess, + .dirCreateFile = dirCreateFile, + .dirOpenFile = dirOpenFile, + .dirOpenDir = dirOpenDir, + .dirClose = dirClose, + .fileStat = fileStat, + .fileClose = fileClose, + .fileWriteStreaming = fileWriteStreaming, + .fileWritePositional = fileWritePositional, + .fileReadStreaming = fileReadStreaming, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .openSelfExe = openSelfExe, + .async = async, + .concurrent = concurrent, + .await = await, + .cancel = cancel, + .cancelRequested = cancelRequested, + .groupAsync = groupAsync, + .groupConcurrent = groupConcurrent, + .groupWait = groupWait, + .groupCancel = groupCancel, + .select = select, + .mutexLock = mutexLock, + .mutexLockUncancelable = mutexLockUncancelable, + .mutexUnlock = mutexUnlock, + .conditionWait = conditionWait, + .conditionWaitUncancelable = conditionWaitUncancelable, + .conditionWake = conditionWake, + .now = now, + .sleep = sleep, + .netListenIp = netListenIp, + .netAccept = netAccept, + .netBindIp = netBindIp, + .netConnectIp = netConnectIp, + .netListenUnix = netListenUnix, + .netConnectUnix = netConnectUnix, + .netSend = netSend, + .netReceive = netReceive, + .netRead = netRead, + .netWrite = netWrite, + .netClose = netClose, + .netInterfaceNameResolve = netInterfaceNameResolve, + .netInterfaceName = netInterfaceName, + .netLookup = netLookup, + }; + + pub fn dirMake(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, mode: std.Io.Dir.Mode) std.Io.Dir.MakeError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirMake(self.inner.userdata, dir, sub_path, mode); + } + + pub fn dirMakePath(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, mode: std.Io.Dir.Mode) std.Io.Dir.MakeError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirMakePath(self.inner.userdata, dir, sub_path, mode); + } + + pub fn dirMakeOpenPath(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.MakeOpenPathError!std.Io.Dir { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirMakeOpenPath(self.inner.userdata, dir, sub_path, options); + } + + pub fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirStat(self.inner.userdata, dir); + } + + pub fn dirStatPath(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatPathOptions) std.Io.Dir.StatPathError!std.Io.Dir.Stat { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirStatPath(self.inner.userdata, dir, sub_path, options); + } + + pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirAccess(self.inner.userdata, dir, sub_path, options); + } + + pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.CreateFlags) std.Io.File.OpenError!std.Io.File { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirCreateFile(self.inner.userdata, dir, sub_path, flags); + } + + pub fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirOpenFile(self.inner.userdata, dir, sub_path, flags); + } + + pub fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirOpenDir(self.inner.userdata, dir, sub_path, options); + } + + pub fn dirClose(userdata: ?*anyopaque, dir: std.Io.Dir) void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + self.inner.vtable.dirClose(self.inner.userdata, dir); + } + + pub fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileStat(self.inner.userdata, file); + } + + pub fn fileClose(userdata: ?*anyopaque, file: std.Io.File) void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + self.inner.vtable.fileClose(self.inner.userdata, file); + } + + pub fn fileWriteStreaming(userdata: ?*anyopaque, file: std.Io.File, buffer: [][]const u8) std.Io.File.WriteStreamingError!usize { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileWriteStreaming(self.inner.userdata, file, buffer); + } + + pub fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, buffer: [][]const u8, offset: u64) std.Io.File.WritePositionalError!usize { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileWritePositional(self.inner.userdata, file, buffer, offset); + } + + pub fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: [][]u8) std.Io.File.Reader.Error!usize { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileReadStreaming(self.inner.userdata, file, data); + } + + pub fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: [][]u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileReadPositional(self.inner.userdata, file, data, offset); + } + + pub fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileSeekBy(self.inner.userdata, file, relative_offset); + } + + pub fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileSeekTo(self.inner.userdata, file, absolute_offset); + } + + pub fn openSelfExe(userdata: ?*anyopaque, flags: std.Io.File.OpenFlags) std.Io.File.OpenSelfExeError!std.Io.File { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.openSelfExe(self.inner.userdata, flags); + } + + pub fn async(userdata: ?*anyopaque, result: []u8, result_alignment: std.mem.Alignment, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque, result: *anyopaque) void) ?*std.Io.AnyFuture { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.async(self.inner.userdata, result, result_alignment, context, context_alignment, start); + } + + pub fn concurrent(userdata: ?*anyopaque, result_len: usize, result_alignment: std.mem.Alignment, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque, result: *anyopaque) void) std.Io.ConcurrentError!*std.Io.AnyFuture { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.concurrent(self.inner.userdata, result_len, result_alignment, context, context_alignment, start); + } + + pub fn await(userdata: ?*anyopaque, any_future: *std.Io.AnyFuture, result: []u8, result_alignment: std.mem.Alignment) void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.await(self.inner.userdata, any_future, result, result_alignment); + } + + pub fn cancel(userdata: ?*anyopaque, any_future: *std.Io.AnyFuture, result: []u8, result_alignment: std.mem.Alignment) void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.cancel(self.inner.userdata, any_future, result, result_alignment); + } + + pub fn cancelRequested(userdata: ?*anyopaque) bool { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.cancelRequested(self.inner.userdata); + } + + pub fn groupAsync(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (*std.Io.Group, context: *const anyopaque) void) void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.groupAsync(self.inner.userdata, group, context, context_alignment, start); + } + + pub fn groupConcurrent(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (*std.Io.Group, context: *const anyopaque) void) std.Io.ConcurrentError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.groupConcurrent(self.inner.userdata, group, context, context_alignment, start); + } + + pub fn groupWait(userdata: ?*anyopaque, group: *std.Io.Group, token: *anyopaque) void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.groupWait(self.inner.userdata, group, token); + } + + pub fn groupCancel(userdata: ?*anyopaque, group: *std.Io.Group, token: *anyopaque) void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.groupCancel(self.inner.userdata, group, token); + } + + pub fn select(userdata: ?*anyopaque, futures: []const *std.Io.AnyFuture) std.Io.Cancelable!usize { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.select(self.inner.userdata, futures); + } + + pub fn mutexLock(userdata: ?*anyopaque, prev_state: std.Io.Mutex.State, mutex: *std.Io.Mutex) std.Io.Cancelable!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.mutexLock(self.inner.userdata, prev_state, mutex); + } + + pub fn mutexLockUncancelable(userdata: ?*anyopaque, prev_state: std.Io.Mutex.State, mutex: *std.Io.Mutex) void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + std.debug.print(">>>>>>>> {*}\n", .{self}); + self.inner.vtable.mutexLockUncancelable(self.inner.userdata, prev_state, mutex); + } + + pub fn mutexUnlock(userdata: ?*anyopaque, prev_state: std.Io.Mutex.State, mutex: *std.Io.Mutex) void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + self.inner.vtable.mutexUnlock(self.inner.userdata, prev_state, mutex); + } + + pub fn conditionWait(userdata: ?*anyopaque, cond: *std.Io.Condition, mutex: *std.Io.Mutex) std.Io.Cancelable!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.conditionWait(self.inner.userdata, cond, mutex); + } + + pub fn conditionWaitUncancelable(userdata: ?*anyopaque, cond: *std.Io.Condition, mutex: *std.Io.Mutex) void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + self.inner.vtable.conditionWaitUncancelable(self.inner.userdata, cond, mutex); + } + + pub fn conditionWake(userdata: ?*anyopaque, cond: *std.Io.Condition, wake: std.Io.Condition.Wake) void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + self.inner.vtable.conditionWake(self.inner.userdata, cond, wake); + } + + pub fn now(userdata: ?*anyopaque, clock: std.Io.Clock) std.Io.Clock.Error!std.Io.Timestamp { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.now(self.inner.userdata, clock); + } + + pub fn sleep(userdata: ?*anyopaque, timeout: std.Io.Timeout) std.Io.SleepError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.sleep(self.inner.userdata, timeout); + } + + pub fn netListenIp(userdata: ?*anyopaque, address: std.Io.net.IpAddress, options: std.Io.net.IpAddress.ListenOptions) std.Io.net.IpAddress.ListenError!std.Io.net.Server { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netListenIp(self.inner.userdata, address, options); + } + + pub fn netAccept(userdata: ?*anyopaque, server: std.Io.net.Socket.Handle) std.Io.net.Server.AcceptError!std.Io.net.Stream { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netAccept(self.inner.userdata, server); + } + + pub fn netBindIp(userdata: ?*anyopaque, address: *const std.Io.net.IpAddress, options: std.Io.net.IpAddress.BindOptions) std.Io.net.IpAddress.BindError!std.Io.net.Socket { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netBindIp(self.inner.userdata, address, options); + } + + pub fn netConnectIp(userdata: ?*anyopaque, address: *const std.Io.net.IpAddress, options: std.Io.net.IpAddress.ConnectOptions) std.Io.net.IpAddress.ConnectError!std.Io.net.Stream { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netConnectIp(self.inner.userdata, address, options); + } + + pub fn netListenUnix(userdata: ?*anyopaque, address: *const std.Io.net.UnixAddress, options: std.Io.net.UnixAddress.ListenOptions) std.Io.net.UnixAddress.ListenError!std.Io.net.Socket.Handle { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netListenUnix(self.inner.userdata, address, options); + } + + pub fn netConnectUnix(userdata: ?*anyopaque, address: *const std.Io.net.UnixAddress) std.Io.net.UnixAddress.ConnectError!std.Io.net.Socket.Handle { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netConnectUnix(self.inner.userdata, address); + } + + pub fn netSend(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, msgs: []std.Io.net.OutgoingMessage, flags: std.Io.net.SendFlags) struct { ?std.Io.net.Socket.SendError, usize } { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netSend(self.inner.userdata, handle, msgs, flags); + } + + pub fn netReceive(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, message_buffer: []std.Io.net.IncomingMessage, data_buffer: []u8, flags: std.Io.net.ReceiveFlags, timeout: std.Io.Timeout) struct { ?std.Io.net.Socket.ReceiveTimeoutError, usize } { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netReceive(self.inner.userdata, handle, message_buffer, data_buffer, flags, timeout); + } + + pub fn netRead(userdata: ?*anyopaque, src: std.Io.net.Socket.Handle, data: [][]u8) std.Io.net.Stream.Reader.Error!usize { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netRead(self.inner.userdata, src, data); + } + + pub fn netWrite(userdata: ?*anyopaque, dest: std.Io.net.Socket.Handle, header: []const u8, data: []const []const u8, splat: usize) std.Io.net.Stream.Writer.Error!usize { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netWrite(self.inner.userdata, dest, header, data, splat); + } + + pub fn netClose(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle) void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + self.inner.vtable.netClose(self.inner.userdata, handle); + } + + pub fn netInterfaceNameResolve(userdata: ?*anyopaque, name: *const std.Io.net.Interface.Name) std.Io.net.Interface.Name.ResolveError!std.Io.net.Interface { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netInterfaceNameResolve(self.inner.userdata, name); + } + + pub fn netInterfaceName(userdata: ?*anyopaque, iface: std.Io.net.Interface) std.Io.net.Interface.NameError!std.Io.net.Interface.Name { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netInterfaceName(self.inner.userdata, iface); + } + + pub fn netLookup(userdata: ?*anyopaque, host: std.Io.net.HostName, q: *std.Io.Queue(std.Io.net.HostName.LookupResult), options: std.Io.net.HostName.LookupOptions) void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netLookup(self.inner.userdata, host, q, options); + } +}; From f3245c596883b8216a7dbbb5f14373a0fdb84db4 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 1 Jan 2026 17:56:29 +0000 Subject: [PATCH 171/881] MNIST running again --- MODULE.bazel | 2 +- MODULE.bazel.lock | 88 +++++++++++----------- bazel/zig_index.json | 154 +++++++++++++++++++-------------------- examples/mnist/mnist.zig | 16 ++-- zml2/pjrtx.zig | 27 ++----- zml2/safetensors.zig | 6 +- 6 files changed, 140 insertions(+), 153 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 7a110629..3f5bb2c2 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -31,7 +31,7 @@ bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2", dev_dependency = Tr ### Zig toolchain zig = use_extension("@rules_zig//zig:extensions.bzl", "zig") zig.index(file = "//bazel:zig_index.json") -zig.toolchain(zig_version = "0.16.0-dev.1878+2a02c7a0d") +zig.toolchain(zig_version = "0.16.0-dev.1888+9d497d0d7") zig.mirrors(urls = [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index d31e0ba0..87594272 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -2849,89 +2849,89 @@ "@@rules_zig+//zig:extensions.bzl%zig": { "general": { "bzlTransitiveDigest": "wV3euhBZSI0D2ZlqbY1YW592mghFY88d9H6zBMQRaJY=", - "usagesDigest": "gADTTSTOnyxlP1vQhqp5HillVM6887O94Xj7VDt4tdk=", + "usagesDigest": "1F/XDUJlOsUsE2Gup2fXDs6ZiU30P6wuDDvns9uNyxM=", "recordedFileInputs": { - "@@//bazel/zig_index.json": "b9733b7c49787cd95b384a0b5ee9e4171367d48576aeaa8b3a6fd05e27184e4d", + "@@//bazel/zig_index.json": "f7ac8e30d820efe4ab7e6c0c770badc8aa18ff082b8d1a56f1ea0bfa1282bec5", "@@rules_zig+//zig/private/versions.json": "6eb85ebaee72c4e6fbc82d8ad73f69e0899d839b70a7e19caa792ab6a5fa0c36" }, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { - "zig_0.16.0-dev.1878_P2a02c7a0d_aarch64-linux": { + "zig_0.16.0-dev.1888_P9d497d0d7_aarch64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "url": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1888+9d497d0d7.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "76622697d65885d3d2b0aa21de9a680662db14d04174b022b83e9292d106e4f6", - "zig_version": "0.16.0-dev.1878+2a02c7a0d", + "sha256": "26147ba4330bd01a06a9755f42445122be227dec4445117e450754968f4d613b", + "zig_version": "0.16.0-dev.1888+9d497d0d7", "platform": "aarch64-linux" } }, - "zig_0.16.0-dev.1878_P2a02c7a0d_aarch64-macos": { + "zig_0.16.0-dev.1888_P9d497d0d7_aarch64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "url": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1888+9d497d0d7.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "bf264609d8c3b07fc315324ceb1e4b3fb0cbe17c9842a4c663fca29715805a58", - "zig_version": "0.16.0-dev.1878+2a02c7a0d", + "sha256": "f1e092e51cd026a600f68c53ad610e12ecad0cfdc9aac71ffc86f21afe3db5e5", + "zig_version": "0.16.0-dev.1888+9d497d0d7", "platform": "aarch64-macos" } }, - "zig_0.16.0-dev.1878_P2a02c7a0d_aarch64-windows": { + "zig_0.16.0-dev.1888_P9d497d0d7_aarch64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1878+2a02c7a0d.zip", + "url": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1888+9d497d0d7.zip", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "552e4e7ddd868a8f9aa59d7c3f252f16b4317642d59ee64545ceecedeba789bf", - "zig_version": "0.16.0-dev.1878+2a02c7a0d", + "sha256": "a7c248888bdf545ef6ce96e0585dc1313ce19ca5dd7fee59d6a93c392a1fa8fb", + "zig_version": "0.16.0-dev.1888+9d497d0d7", "platform": "aarch64-windows" } }, - "zig_0.16.0-dev.1878_P2a02c7a0d_x86_64-linux": { + "zig_0.16.0-dev.1888_P9d497d0d7_x86_64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "url": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1888+9d497d0d7.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "a9095696174976aae6da0b925f5d994e324be3fc4f0a55497ecfea9331ba1f0b", - "zig_version": "0.16.0-dev.1878+2a02c7a0d", + "sha256": "51e17ce99f3e10a21a56a7bfe163a98736495bd10cbc974c6d4648748ac710c4", + "zig_version": "0.16.0-dev.1888+9d497d0d7", "platform": "x86_64-linux" } }, - "zig_0.16.0-dev.1878_P2a02c7a0d_x86_64-macos": { + "zig_0.16.0-dev.1888_P9d497d0d7_x86_64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1878+2a02c7a0d.tar.xz", + "url": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1888+9d497d0d7.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "93ca19d2d2d4f05689d934dd691594533972b2adfa6c8f6b219a215fc9d37e1b", - "zig_version": "0.16.0-dev.1878+2a02c7a0d", + "sha256": "247d6e0fe0bdbbc2433737caba10688594ca62469d3e782c62ef7257dad0b167", + "zig_version": "0.16.0-dev.1888+9d497d0d7", "platform": "x86_64-macos" } }, - "zig_0.16.0-dev.1878_P2a02c7a0d_x86_64-windows": { + "zig_0.16.0-dev.1888_P9d497d0d7_x86_64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1878+2a02c7a0d.zip", + "url": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1888+9d497d0d7.zip", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "7912bb842839ec03887f9a8b86da24afb1ac511e29471be44caec5440f7d7f12", - "zig_version": "0.16.0-dev.1878+2a02c7a0d", + "sha256": "3dc6a275d9ef7ae622b128bf971201bada1f81b46f16f9d2771b8dd058d7db7e", + "zig_version": "0.16.0-dev.1888+9d497d0d7", "platform": "x86_64-windows" } }, @@ -2939,28 +2939,28 @@ "repoRuleId": "@@rules_zig+//zig/private/repo:toolchains_repo.bzl%toolchains_repo", "attributes": { "names": [ - "zig_0.16.0-dev.1878_P2a02c7a0d_aarch64-linux", - "zig_0.16.0-dev.1878_P2a02c7a0d_aarch64-macos", - "zig_0.16.0-dev.1878_P2a02c7a0d_aarch64-windows", - "zig_0.16.0-dev.1878_P2a02c7a0d_x86_64-linux", - "zig_0.16.0-dev.1878_P2a02c7a0d_x86_64-macos", - "zig_0.16.0-dev.1878_P2a02c7a0d_x86_64-windows" + "zig_0.16.0-dev.1888_P9d497d0d7_aarch64-linux", + "zig_0.16.0-dev.1888_P9d497d0d7_aarch64-macos", + "zig_0.16.0-dev.1888_P9d497d0d7_aarch64-windows", + "zig_0.16.0-dev.1888_P9d497d0d7_x86_64-linux", + "zig_0.16.0-dev.1888_P9d497d0d7_x86_64-macos", + "zig_0.16.0-dev.1888_P9d497d0d7_x86_64-windows" ], "labels": [ - "@zig_0.16.0-dev.1878_P2a02c7a0d_aarch64-linux//:zig_toolchain", - "@zig_0.16.0-dev.1878_P2a02c7a0d_aarch64-macos//:zig_toolchain", - "@zig_0.16.0-dev.1878_P2a02c7a0d_aarch64-windows//:zig_toolchain", - "@zig_0.16.0-dev.1878_P2a02c7a0d_x86_64-linux//:zig_toolchain", - "@zig_0.16.0-dev.1878_P2a02c7a0d_x86_64-macos//:zig_toolchain", - "@zig_0.16.0-dev.1878_P2a02c7a0d_x86_64-windows//:zig_toolchain" + "@zig_0.16.0-dev.1888_P9d497d0d7_aarch64-linux//:zig_toolchain", + "@zig_0.16.0-dev.1888_P9d497d0d7_aarch64-macos//:zig_toolchain", + "@zig_0.16.0-dev.1888_P9d497d0d7_aarch64-windows//:zig_toolchain", + "@zig_0.16.0-dev.1888_P9d497d0d7_x86_64-linux//:zig_toolchain", + "@zig_0.16.0-dev.1888_P9d497d0d7_x86_64-macos//:zig_toolchain", + "@zig_0.16.0-dev.1888_P9d497d0d7_x86_64-windows//:zig_toolchain" ], "zig_versions": [ - "0.16.0-dev.1878+2a02c7a0d", - "0.16.0-dev.1878+2a02c7a0d", - "0.16.0-dev.1878+2a02c7a0d", - "0.16.0-dev.1878+2a02c7a0d", - "0.16.0-dev.1878+2a02c7a0d", - "0.16.0-dev.1878+2a02c7a0d" + "0.16.0-dev.1888+9d497d0d7", + "0.16.0-dev.1888+9d497d0d7", + "0.16.0-dev.1888+9d497d0d7", + "0.16.0-dev.1888+9d497d0d7", + "0.16.0-dev.1888+9d497d0d7", + "0.16.0-dev.1888+9d497d0d7" ], "exec_lengths": [ 2, diff --git a/bazel/zig_index.json b/bazel/zig_index.json index c8561921..d6f49100 100644 --- a/bazel/zig_index.json +++ b/bazel/zig_index.json @@ -1,133 +1,133 @@ { "master": { - "version": "0.16.0-dev.1878+2a02c7a0d", - "date": "2025-12-30", + "version": "0.16.0-dev.1888+9d497d0d7", + "date": "2025-12-31", "docs": "https://ziglang.org/documentation/master/", "stdDocs": "https://ziglang.org/documentation/master/std/", "src": { - "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.1878+2a02c7a0d.tar.xz", - "shasum": "23a81e334522ea78dbef5a63c4570446adfbefe5d44f4fae216b615b5786bf02", - "size": "21859732" + "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.1888+9d497d0d7.tar.xz", + "shasum": "5a496e6563059129da0faf07299bf1df2a58d9024b15080a3f7a090fd59ab5f3", + "size": "21858752" }, "bootstrap": { - "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.1878+2a02c7a0d.tar.xz", - "shasum": "9b03560d3531e6cf24827afc966f51a75673fe08948da943d9740441d7ef4540", - "size": "54621664" + "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.1888+9d497d0d7.tar.xz", + "shasum": "0d2be93accccb4b28991d46a0a10a73b2b2627a897e49b4dff0220aadb1ba577", + "size": "54621884" }, "x86_64-macos": { - "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1878+2a02c7a0d.tar.xz", - "shasum": "93ca19d2d2d4f05689d934dd691594533972b2adfa6c8f6b219a215fc9d37e1b", - "size": "56995132" + "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1888+9d497d0d7.tar.xz", + "shasum": "247d6e0fe0bdbbc2433737caba10688594ca62469d3e782c62ef7257dad0b167", + "size": "57003700" }, "aarch64-macos": { - "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1878+2a02c7a0d.tar.xz", - "shasum": "bf264609d8c3b07fc315324ceb1e4b3fb0cbe17c9842a4c663fca29715805a58", - "size": "51853136" + "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1888+9d497d0d7.tar.xz", + "shasum": "f1e092e51cd026a600f68c53ad610e12ecad0cfdc9aac71ffc86f21afe3db5e5", + "size": "51844052" }, "x86_64-linux": { - "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1878+2a02c7a0d.tar.xz", - "shasum": "a9095696174976aae6da0b925f5d994e324be3fc4f0a55497ecfea9331ba1f0b", - "size": "54913924" + "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1888+9d497d0d7.tar.xz", + "shasum": "51e17ce99f3e10a21a56a7bfe163a98736495bd10cbc974c6d4648748ac710c4", + "size": "54905484" }, "aarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1878+2a02c7a0d.tar.xz", - "shasum": "76622697d65885d3d2b0aa21de9a680662db14d04174b022b83e9292d106e4f6", - "size": "50654532" + "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1888+9d497d0d7.tar.xz", + "shasum": "26147ba4330bd01a06a9755f42445122be227dec4445117e450754968f4d613b", + "size": "50654628" }, "arm-linux": { - "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.1878+2a02c7a0d.tar.xz", - "shasum": "2cb8f15af2a49509929f751675080055cbf03377ccf72c8fb96dd8a79b561040", - "size": "51524380" + "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.1888+9d497d0d7.tar.xz", + "shasum": "269d22e85c097ec7627815366f4148344b5bba7d893006578c8608a09de17ffc", + "size": "51544652" }, "riscv64-linux": { - "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.1878+2a02c7a0d.tar.xz", - "shasum": "576edcdc9404c25d6f0c106d6ef860c2098803866702f675e9431d61b87d3343", - "size": "54717592" + "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.1888+9d497d0d7.tar.xz", + "shasum": "63db2a0c2af03a6028857ae403ccb21fd24ac9ab3d21f81bf7f44e4c3e855c14", + "size": "54712804" }, "powerpc64le-linux": { - "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.1878+2a02c7a0d.tar.xz", - "shasum": "5fdd195feec0fc7b4a9f8d4276e73793de4723115d193767d34a48617f81eca1", - "size": "54727752" + "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.1888+9d497d0d7.tar.xz", + "shasum": "a05cd2af0703978e12216e7af36bacfd94d6a56a0a1f89fb77420de35f423515", + "size": "54723848" }, "x86-linux": { - "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.1878+2a02c7a0d.tar.xz", - "shasum": "ff2e70a50d5fd548b40a199511abd69b3b6ceb1faf6d695e6c096f74693c4833", - "size": "57578920" + "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.1888+9d497d0d7.tar.xz", + "shasum": "12b45ec048bc07b9d83c7536a4c2a828f90f4494a78251c58e0c1982637adbb0", + "size": "57602100" }, "loongarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.1878+2a02c7a0d.tar.xz", - "shasum": "e6bde81c6bb63e1135006ee156d9563d28aa1b69202f1d7ee79642e0a8eb819f", - "size": "51954144" + "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.1888+9d497d0d7.tar.xz", + "shasum": "a4eb9adc0e8a5ec0cce5e2082d8b3b0e6a87eda49f7272702377838ecd8c859e", + "size": "51952008" }, "s390x-linux": { - "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.1878+2a02c7a0d.tar.xz", - "shasum": "7552d69fc7b78fbc662af4df837849c40f00a49aa7d24f141797b3cc65a35f0d", - "size": "54411980" + "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.1888+9d497d0d7.tar.xz", + "shasum": "d7d3d1cc379cfc070a0b6f0f6161b2c584870cc659928037e3f315bec2626951", + "size": "54397912" }, "x86_64-windows": { - "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1878+2a02c7a0d.zip", - "shasum": "7912bb842839ec03887f9a8b86da24afb1ac511e29471be44caec5440f7d7f12", - "size": "94976682" + "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1888+9d497d0d7.zip", + "shasum": "3dc6a275d9ef7ae622b128bf971201bada1f81b46f16f9d2771b8dd058d7db7e", + "size": "94973867" }, "aarch64-windows": { - "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1878+2a02c7a0d.zip", - "shasum": "552e4e7ddd868a8f9aa59d7c3f252f16b4317642d59ee64545ceecedeba789bf", - "size": "90930367" + "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1888+9d497d0d7.zip", + "shasum": "a7c248888bdf545ef6ce96e0585dc1313ce19ca5dd7fee59d6a93c392a1fa8fb", + "size": "90931027" }, "x86-windows": { - "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.1878+2a02c7a0d.zip", - "shasum": "a44128f8aedc38152e57552687a988b7163d852a835df1cc2c8e67871e7d017d", - "size": "96809617" + "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.1888+9d497d0d7.zip", + "shasum": "33bb0222efff1df27f24cb8afa0cf24c6e1dfaa43c537cd73a4c25a351cb8dd7", + "size": "96811069" }, "aarch64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.1878+2a02c7a0d.tar.xz", - "shasum": "13f878e9f68d135ed6130f2fb514fa96485bbbd9b4d356c76d1fdcd17e7c8a3f", - "size": "50590876" + "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.1888+9d497d0d7.tar.xz", + "shasum": "9de0a0c4f14c435225827ef4b70569841b97cb934487b176ea929ea3eae497bf", + "size": "50602712" }, "arm-freebsd": { - "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.1878+2a02c7a0d.tar.xz", - "shasum": "e02f83f42f985b30496a9ceeae05e8ad99cb09d5462c7ee4c920557293d48e66", - "size": "52205192" + "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.1888+9d497d0d7.tar.xz", + "shasum": "0f81ff46f9b94bab09d2eb38a12f0ef1818577f3b4ce5da0e4d2eb593da7b50d", + "size": "52207800" }, "powerpc64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-powerpc64-freebsd-0.16.0-dev.1878+2a02c7a0d.tar.xz", - "shasum": "1dc506c30e954385e2499662738bf60546f4ed667b46cac3dc0402af8dd15f05", - "size": "53265316" + "tarball": "https://ziglang.org/builds/zig-powerpc64-freebsd-0.16.0-dev.1888+9d497d0d7.tar.xz", + "shasum": "347bc5af3d520427fc9976c3bd8a213d5a2dc6d175d243823714c48d80d604ba", + "size": "53293836" }, "powerpc64le-freebsd": { - "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.1878+2a02c7a0d.tar.xz", - "shasum": "e2c2365a8d4da7828de6e7f6363f146a68dca9dbebc6296e6777e9d600c30464", - "size": "54738884" + "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.1888+9d497d0d7.tar.xz", + "shasum": "73d7a142e0acb93dd90fd67bd6b1602b970bd40b12007470f45dd42ea82cf797", + "size": "54741496" }, "riscv64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.1878+2a02c7a0d.tar.xz", - "shasum": "c205e8a5c20a879ca13be5c9322b49ad7247cb581ac5922ed38d30b773ba48e4", - "size": "54844712" + "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.1888+9d497d0d7.tar.xz", + "shasum": "8ba301c25212133614b440eef1d9f21337adf0f93c676c87ee6a996e011966e3", + "size": "54847220" }, "x86_64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.1878+2a02c7a0d.tar.xz", - "shasum": "2f9309473d1acf253a8e5eba3db43c814739794dc7316fa0364058b402535815", - "size": "55002612" + "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.1888+9d497d0d7.tar.xz", + "shasum": "afaf144768b88ee25c9a0b88897a81a35daf603990b5abcaecc36677da382209", + "size": "55013484" }, "aarch64-netbsd": { - "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.1878+2a02c7a0d.tar.xz", - "shasum": "d4db68ff2d81fa41d40db82da9c41d6d46baf247a3f2eb05ba9c1aaf903a0d07", - "size": "50568412" + "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.1888+9d497d0d7.tar.xz", + "shasum": "b72aff3c96ab11283b3d9a37d1741adeb7a6a910b7949912c0b68da5475a85ab", + "size": "50543896" }, "arm-netbsd": { - "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.1878+2a02c7a0d.tar.xz", - "shasum": "3a39f6ceb25f0162e8a3cb0f4586a9a5d4ea24050bc30e418c60bc57f6155adc", - "size": "53258248" + "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.1888+9d497d0d7.tar.xz", + "shasum": "2cae2f1daf158eac60887ac2c0c7e1c6f92746ca17a85a951ef91103a02f9b8b", + "size": "53245616" }, "x86-netbsd": { - "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.1878+2a02c7a0d.tar.xz", - "shasum": "2910bfbf92119bdbb46ee9e0cf37c3cd828160563e48a1b007683bb9ccda5b76", - "size": "58192980" + "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.1888+9d497d0d7.tar.xz", + "shasum": "457a413d2aa1e1f789b8caa3f375e7c04cf09adc5eee1a2577452d85fbced86a", + "size": "58200316" }, "x86_64-netbsd": { - "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.1878+2a02c7a0d.tar.xz", - "shasum": "8454376bb6031dbd2eb809de0b6309a8109db811f284110bbae4c84a7fc77ed5", - "size": "54951836" + "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.1888+9d497d0d7.tar.xz", + "shasum": "73ac63473a6fb06b4b3dc23fdc0fbd7367eabac5225b78238ce3458519ecdf8b", + "size": "54969392" } }, "0.15.2": { diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 9ab07497..0ad82c2d 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -1,6 +1,6 @@ const std = @import("std"); -const stdx = @import("stdx"); +const stdx = @import("stdx"); const zml = @import("zml"); const log = std.log.scoped(.mnist); @@ -67,17 +67,19 @@ pub fn main() !void { //const allocator = gpa.allocator(); - var threaded: std.Io.Threaded = .init(allocator); + var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); - var vfs_file: zml.io.VFS.File = .init(allocator, threaded.io(), .{}); + // var vfs_file: zml.io.VFS.File = .init(allocator, threaded.io(), .{}); - var vfs: zml.io.VFS = .init(allocator, threaded.io()); - defer vfs.deinit(); + // var vfs: zml.io.VFS = .init(allocator, threaded.io()); + // defer vfs.deinit(); - try vfs.register("file", vfs_file.io()); + // try vfs.register("file", vfs_file.io()); - const io = vfs.io(); + // const io = vfs.io(); + // + const io = threaded.io(); zml.init(); defer zml.deinit(); diff --git a/zml2/pjrtx.zig b/zml2/pjrtx.zig index 3a96c4b4..70424edd 100644 --- a/zml2/pjrtx.zig +++ b/zml2/pjrtx.zig @@ -239,35 +239,20 @@ pub const Event = opaque { return; } - var mutex: std.Io.Mutex = .init; - var condvar: std.Io.Condition = .{}; - var done: bool = false; - var ctx = struct { err: ?*pjrt.Error = null, - mutex: *std.Io.Mutex, - condvar: *std.Io.Condition, - done: *bool, + event: std.Io.Event = .unset, io: std.Io, - }{ .mutex = &mutex, .condvar = &condvar, .done = &done, .io = io }; + }{ .io = io }; - try self.inner().onReady(api, &(struct { + try self.inner().onReady(api, struct { fn call(err: ?*pjrt.Error, user_arg: ?*anyopaque) callconv(.c) void { const ctx_: *@TypeOf(ctx) = @ptrCast(@alignCast(user_arg.?)); ctx_.err = err; - { - ctx_.mutex.lock(ctx_.io) catch unreachable; - defer ctx_.mutex.unlock(ctx_.io); - ctx_.done.* = true; - } - - ctx_.condvar.signal(ctx_.io); + ctx_.event.set(ctx_.io); } - }.call), &ctx); - mutex.lock(io) catch unreachable; - while (!done) { - ctx.condvar.wait(io, &mutex) catch unreachable; - } + }.call, &ctx); + ctx.event.waitUncancelable(io); if (ctx.err) |e| { defer e.deinit(api); diff --git a/zml2/safetensors.zig b/zml2/safetensors.zig index 7c47992b..d7653216 100644 --- a/zml2/safetensors.zig +++ b/zml2/safetensors.zig @@ -1,9 +1,9 @@ const std = @import("std"); -const Shape = @import("shape.zig").Shape; const DataType = @import("dtype.zig").DataType; - +const Shape = @import("shape.zig").Shape; const Dims = Shape.DimsArray; + const StringBuilder = std.ArrayListUnmanaged(u8); pub const Tensors = std.StringArrayHashMapUnmanaged(Tensor); @@ -477,7 +477,7 @@ pub fn parseSafetensorsIndexFiles( }); } - group.wait(io); + try group.await(io); if (err) |e| { log.err("Error parsing safetensors index files: {any}", .{e}); From 39fb5e590bca9ab3901afc0f315ed396c4825a3e Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 2 Jan 2026 16:02:30 +0100 Subject: [PATCH 172/881] WIP --- zml2/io/vfs/base.zig | 509 ++++++++++++++++++++++++++++++------------ zml2/io/vfs/hf2.zig | 115 +--------- zml2/io/vfs/proxy.zig | 509 ++++++++++++++++++++++++++++++------------ 3 files changed, 736 insertions(+), 397 deletions(-) diff --git a/zml2/io/vfs/base.zig b/zml2/io/vfs/base.zig index d98c338e..ae8204b8 100644 --- a/zml2/io/vfs/base.zig +++ b/zml2/io/vfs/base.zig @@ -54,41 +54,75 @@ pub const VFSBase = struct { pub fn vtable(overrides: anytype) std.Io.VTable { var new_vtable: std.Io.VTable = .{ - .dirMake = dirMake, - .dirMakePath = dirMakePath, - .dirMakeOpenPath = dirMakeOpenPath, - .dirStat = dirStat, - .dirStatPath = dirStatPath, - .dirAccess = dirAccess, - .dirCreateFile = dirCreateFile, - .dirOpenFile = dirOpenFile, - .dirOpenDir = dirOpenDir, - .dirClose = dirClose, - .fileStat = fileStat, - .fileClose = fileClose, - .fileWriteStreaming = fileWriteStreaming, - .fileWritePositional = fileWritePositional, - .fileReadStreaming = fileReadStreaming, - .fileReadPositional = fileReadPositional, - .fileSeekBy = fileSeekBy, - .fileSeekTo = fileSeekTo, - .openSelfExe = openSelfExe, .async = async, .concurrent = concurrent, .await = await, .cancel = cancel, - .cancelRequested = cancelRequested, .groupAsync = groupAsync, .groupConcurrent = groupConcurrent, - .groupWait = groupWait, + .groupAwait = groupAwait, .groupCancel = groupCancel, + .recancel = recancel, + .swapCancelProtection = swapCancelProtection, + .checkCancel = checkCancel, .select = select, - .mutexLock = mutexLock, - .mutexLockUncancelable = mutexLockUncancelable, - .mutexUnlock = mutexUnlock, - .conditionWait = conditionWait, - .conditionWaitUncancelable = conditionWaitUncancelable, - .conditionWake = conditionWake, + .futexWait = futexWait, + .futexWaitUncancelable = futexWaitUncancelable, + .futexWake = futexWake, + .dirCreateDir = dirCreateDir, + .dirCreateDirPath = dirCreateDirPath, + .dirCreateDirPathOpen = dirCreateDirPathOpen, + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirCreateFile = dirCreateFile, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRead = dirRead, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .dirDeleteFile = dirDeleteFile, + .dirDeleteDir = dirDeleteDir, + .dirRename = dirRename, + .dirSymLink = dirSymLink, + .dirReadLink = dirReadLink, + .dirSetOwner = dirSetOwner, + .dirSetFileOwner = dirSetFileOwner, + .dirSetPermissions = dirSetPermissions, + .dirSetFilePermissions = dirSetFilePermissions, + .dirSetTimestamps = dirSetTimestamps, + .dirHardLink = dirHardLink, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileWriteStreaming = fileWriteStreaming, + .fileWritePositional = fileWritePositional, + .fileWriteFileStreaming = fileWriteFileStreaming, + .fileWriteFilePositional = fileWriteFilePositional, + .fileReadStreaming = fileReadStreaming, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileSync = fileSync, + .fileIsTty = fileIsTty, + .fileEnableAnsiEscapeCodes = fileEnableAnsiEscapeCodes, + .fileSupportsAnsiEscapeCodes = fileSupportsAnsiEscapeCodes, + .fileSetLength = fileSetLength, + .fileSetOwner = fileSetOwner, + .fileSetPermissions = fileSetPermissions, + .fileSetTimestamps = fileSetTimestamps, + .fileLock = fileLock, + .fileTryLock = fileTryLock, + .fileUnlock = fileUnlock, + .fileDowngradeLock = fileDowngradeLock, + .fileRealPath = fileRealPath, + .processExecutableOpen = processExecutableOpen, + .processExecutablePath = processExecutablePath, + .lockStderr = lockStderr, + .tryLockStderr = tryLockStderr, + .unlockStderr = unlockStderr, + .processSetCurrentDir = processSetCurrentDir, .now = now, .sleep = sleep, .netListenIp = netListenIp, @@ -101,7 +135,9 @@ pub const VFSBase = struct { .netReceive = netReceive, .netRead = netRead, .netWrite = netWrite, + .netWriteFile = netWriteFile, .netClose = netClose, + .netShutdown = netShutdown, .netInterfaceNameResolve = netInterfaceNameResolve, .netInterfaceName = netInterfaceName, .netLookup = netLookup, @@ -116,101 +152,6 @@ pub const VFSBase = struct { return @ptrCast(@alignCast(userdata orelse unreachable)); } - pub fn dirMake(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, mode: std.Io.Dir.Mode) std.Io.Dir.MakeError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirMake(self.inner.userdata, dir, sub_path, mode); - } - - pub fn dirMakePath(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, mode: std.Io.Dir.Mode) std.Io.Dir.MakeError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirMakePath(self.inner.userdata, dir, sub_path, mode); - } - - pub fn dirMakeOpenPath(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.MakeOpenPathError!std.Io.Dir { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirMakeOpenPath(self.inner.userdata, dir, sub_path, options); - } - - pub fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirStat(self.inner.userdata, dir); - } - - pub fn dirStatPath(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatPathOptions) std.Io.Dir.StatPathError!std.Io.Dir.Stat { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirStatPath(self.inner.userdata, dir, sub_path, options); - } - - pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirAccess(self.inner.userdata, dir, sub_path, options); - } - - pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.CreateFlags) std.Io.File.OpenError!std.Io.File { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirCreateFile(self.inner.userdata, dir, sub_path, flags); - } - - pub fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirOpenFile(self.inner.userdata, dir, sub_path, flags); - } - - pub fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirOpenDir(self.inner.userdata, dir, sub_path, options); - } - - pub fn dirClose(userdata: ?*anyopaque, dir: std.Io.Dir) void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - self.inner.vtable.dirClose(self.inner.userdata, dir); - } - - pub fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileStat(self.inner.userdata, file); - } - - pub fn fileClose(userdata: ?*anyopaque, file: std.Io.File) void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - self.inner.vtable.fileClose(self.inner.userdata, file); - } - - pub fn fileWriteStreaming(userdata: ?*anyopaque, file: std.Io.File, buffer: [][]const u8) std.Io.File.WriteStreamingError!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileWriteStreaming(self.inner.userdata, file, buffer); - } - - pub fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, buffer: [][]const u8, offset: u64) std.Io.File.WritePositionalError!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileWritePositional(self.inner.userdata, file, buffer, offset); - } - - pub fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: [][]u8) std.Io.File.Reader.Error!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileReadStreaming(self.inner.userdata, file, data); - } - - pub fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: [][]u8, offset: u64) std.Io.File.ReadPositionalError!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileReadPositional(self.inner.userdata, file, data, offset); - } - - pub fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileSeekBy(self.inner.userdata, file, relative_offset); - } - - pub fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileSeekTo(self.inner.userdata, file, absolute_offset); - } - - pub fn openSelfExe(userdata: ?*anyopaque, flags: std.Io.File.OpenFlags) std.Io.File.OpenSelfExeError!std.Io.File { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.openSelfExe(self.inner.userdata, flags); - } - pub fn async(userdata: ?*anyopaque, result: []u8, result_alignment: std.mem.Alignment, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque, result: *anyopaque) void) ?*std.Io.AnyFuture { const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); return self.inner.vtable.async(self.inner.userdata, result, result_alignment, context, context_alignment, start); @@ -231,24 +172,19 @@ pub const VFSBase = struct { return self.inner.vtable.cancel(self.inner.userdata, any_future, result, result_alignment); } - pub fn cancelRequested(userdata: ?*anyopaque) bool { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.cancelRequested(self.inner.userdata); - } - - pub fn groupAsync(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (*std.Io.Group, context: *const anyopaque) void) void { + pub fn groupAsync(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (*std.Io.Group, context: *const anyopaque) std.Io.Cancelable!void) void { const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); return self.inner.vtable.groupAsync(self.inner.userdata, group, context, context_alignment, start); } - pub fn groupConcurrent(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (*std.Io.Group, context: *const anyopaque) void) std.Io.ConcurrentError!void { + pub fn groupConcurrent(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (*std.Io.Group, context: *const anyopaque) std.Io.Cancelable!void) std.Io.ConcurrentError!void { const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); return self.inner.vtable.groupConcurrent(self.inner.userdata, group, context, context_alignment, start); } - pub fn groupWait(userdata: ?*anyopaque, group: *std.Io.Group, token: *anyopaque) void { + pub fn groupAwait(userdata: ?*anyopaque, group: *std.Io.Group, token: *anyopaque) std.Io.Cancelable!void { const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.groupWait(self.inner.userdata, group, token); + return self.inner.vtable.groupAwait(self.inner.userdata, group, token); } pub fn groupCancel(userdata: ?*anyopaque, group: *std.Io.Group, token: *anyopaque) void { @@ -256,40 +192,309 @@ pub const VFSBase = struct { return self.inner.vtable.groupCancel(self.inner.userdata, group, token); } + pub fn recancel(userdata: ?*anyopaque) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.recancel(self.inner.userdata); + } + + pub fn swapCancelProtection(userdata: ?*anyopaque, new: std.Io.CancelProtection) std.Io.CancelProtection { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.swapCancelProtection(self.inner.userdata, new); + } + + pub fn checkCancel(userdata: ?*anyopaque) std.Io.Cancelable!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.checkCancel(self.inner.userdata); + } + pub fn select(userdata: ?*anyopaque, futures: []const *std.Io.AnyFuture) std.Io.Cancelable!usize { const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); return self.inner.vtable.select(self.inner.userdata, futures); } - pub fn mutexLock(userdata: ?*anyopaque, prev_state: std.Io.Mutex.State, mutex: *std.Io.Mutex) std.Io.Cancelable!void { + pub fn futexWait(userdata: ?*anyopaque, ptr: *const u32, expected: u32, timeout: std.Io.Timeout) std.Io.Cancelable!void { const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.mutexLock(self.inner.userdata, prev_state, mutex); + return self.inner.vtable.futexWait(self.inner.userdata, ptr, expected, timeout); } - pub fn mutexLockUncancelable(userdata: ?*anyopaque, prev_state: std.Io.Mutex.State, mutex: *std.Io.Mutex) void { + pub fn futexWaitUncancelable(userdata: ?*anyopaque, ptr: *const u32, expected: u32) void { const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - std.debug.print(">>>>>>>> {*}\n", .{self}); - self.inner.vtable.mutexLockUncancelable(self.inner.userdata, prev_state, mutex); + return self.inner.vtable.futexWaitUncancelable(self.inner.userdata, ptr, expected); } - pub fn mutexUnlock(userdata: ?*anyopaque, prev_state: std.Io.Mutex.State, mutex: *std.Io.Mutex) void { + pub fn futexWake(userdata: ?*anyopaque, ptr: *const u32, max_waiters: u32) void { const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - self.inner.vtable.mutexUnlock(self.inner.userdata, prev_state, mutex); + return self.inner.vtable.futexWake(self.inner.userdata, ptr, max_waiters); } - pub fn conditionWait(userdata: ?*anyopaque, cond: *std.Io.Condition, mutex: *std.Io.Mutex) std.Io.Cancelable!void { + pub fn dirCreateDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions) std.Io.Dir.CreateDirError!void { const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.conditionWait(self.inner.userdata, cond, mutex); + return self.inner.vtable.dirCreateDir(self.inner.userdata, dir, sub_path, permissions); } - pub fn conditionWaitUncancelable(userdata: ?*anyopaque, cond: *std.Io.Condition, mutex: *std.Io.Mutex) void { + pub fn dirCreateDirPath(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions) std.Io.Dir.CreateDirPathError!std.Io.Dir.CreatePathStatus { const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - self.inner.vtable.conditionWaitUncancelable(self.inner.userdata, cond, mutex); + return self.inner.vtable.dirCreateDirPath(self.inner.userdata, dir, sub_path, permissions); } - pub fn conditionWake(userdata: ?*anyopaque, cond: *std.Io.Condition, wake: std.Io.Condition.Wake) void { + pub fn dirCreateDirPathOpen(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions, options: std.Io.Dir.OpenOptions) std.Io.Dir.CreateDirPathOpenError!std.Io.Dir { const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - self.inner.vtable.conditionWake(self.inner.userdata, cond, wake); + return self.inner.vtable.dirCreateDirPathOpen(self.inner.userdata, dir, sub_path, permissions, options); + } + + pub fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirOpenDir(self.inner.userdata, dir, sub_path, options); + } + + pub fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirStat(self.inner.userdata, dir); + } + + pub fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirStatFile(self.inner.userdata, dir, sub_path, options); + } + + pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirAccess(self.inner.userdata, dir, sub_path, options); + } + + pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.CreateFlags) std.Io.File.OpenError!std.Io.File { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirCreateFile(self.inner.userdata, dir, sub_path, flags); + } + + pub fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirOpenFile(self.inner.userdata, dir, sub_path, flags); + } + + pub fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + self.inner.vtable.dirClose(self.inner.userdata, dirs); + } + + pub fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirRead(self.inner.userdata, reader, entries); + } + + pub fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirRealPath(self.inner.userdata, dir, out_buffer); + } + + pub fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirRealPathFile(self.inner.userdata, dir, path_name, out_buffer); + } + + pub fn dirDeleteFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8) std.Io.Dir.DeleteFileError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirDeleteFile(self.inner.userdata, dir, sub_path); + } + + pub fn dirDeleteDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8) std.Io.Dir.DeleteDirError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirDeleteDir(self.inner.userdata, dir, sub_path); + } + + pub fn dirRename(userdata: ?*anyopaque, old_dir: std.Io.Dir, old_sub_path: []const u8, new_dir: std.Io.Dir, new_sub_path: []const u8) std.Io.Dir.RenameError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirRename(self.inner.userdata, old_dir, old_sub_path, new_dir, new_sub_path); + } + + pub fn dirSymLink(userdata: ?*anyopaque, dir: std.Io.Dir, target_path: []const u8, sym_link_path: []const u8, flags: std.Io.Dir.SymLinkFlags) std.Io.Dir.SymLinkError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirSymLink(self.inner.userdata, dir, target_path, sym_link_path, flags); + } + + pub fn dirReadLink(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, buffer: []u8) std.Io.Dir.ReadLinkError!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirReadLink(self.inner.userdata, dir, sub_path, buffer); + } + + pub fn dirSetOwner(userdata: ?*anyopaque, dir: std.Io.Dir, uid: ?std.Io.File.Uid, gid: ?std.Io.File.Gid) std.Io.Dir.SetOwnerError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirSetOwner(self.inner.userdata, dir, uid, gid); + } + + pub fn dirSetFileOwner(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, uid: ?std.Io.File.Uid, gid: ?std.Io.File.Gid, options: std.Io.Dir.SetFileOwnerOptions) std.Io.Dir.SetFileOwnerError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirSetFileOwner(self.inner.userdata, dir, sub_path, uid, gid, options); + } + + pub fn dirSetPermissions(userdata: ?*anyopaque, dir: std.Io.Dir, permissions: std.Io.Dir.Permissions) std.Io.Dir.SetPermissionsError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirSetPermissions(self.inner.userdata, dir, permissions); + } + + pub fn dirSetFilePermissions(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.File.Permissions, options: std.Io.Dir.SetFilePermissionsOptions) std.Io.Dir.SetFilePermissionsError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirSetFilePermissions(self.inner.userdata, dir, sub_path, permissions, options); + } + + pub fn dirSetTimestamps(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.SetTimestampsOptions) std.Io.Dir.SetTimestampsError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirSetTimestamps(self.inner.userdata, dir, sub_path, options); + } + + pub fn dirHardLink(userdata: ?*anyopaque, old_dir: std.Io.Dir, old_sub_path: []const u8, new_dir: std.Io.Dir, new_sub_path: []const u8, options: std.Io.Dir.HardLinkOptions) std.Io.Dir.HardLinkError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirHardLink(self.inner.userdata, old_dir, old_sub_path, new_dir, new_sub_path, options); + } + + pub fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileStat(self.inner.userdata, file); + } + + pub fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileLength(self.inner.userdata, file); + } + + pub fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + self.inner.vtable.fileClose(self.inner.userdata, files); + } + + pub fn fileWriteStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize) std.Io.File.Writer.Error!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileWriteStreaming(self.inner.userdata, file, header, data, splat); + } + + pub fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) std.Io.File.WritePositionalError!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileWritePositional(self.inner.userdata, file, header, data, splat, offset); + } + + pub fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileWriteFileStreaming(self.inner.userdata, file, header, reader, limit); + } + + pub fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileWriteFilePositional(self.inner.userdata, file, header, reader, limit, offset); + } + + pub fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileReadStreaming(self.inner.userdata, file, data); + } + + pub fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileReadPositional(self.inner.userdata, file, data, offset); + } + + pub fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileSeekBy(self.inner.userdata, file, relative_offset); + } + + pub fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileSeekTo(self.inner.userdata, file, absolute_offset); + } + + pub fn fileSync(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.SyncError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileSync(self.inner.userdata, file); + } + + pub fn fileIsTty(userdata: ?*anyopaque, file: std.Io.File) std.Io.Cancelable!bool { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileIsTty(self.inner.userdata, file); + } + + pub fn fileEnableAnsiEscapeCodes(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.EnableAnsiEscapeCodesError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileEnableAnsiEscapeCodes(self.inner.userdata, file); + } + + pub fn fileSupportsAnsiEscapeCodes(userdata: ?*anyopaque, file: std.Io.File) std.Io.Cancelable!bool { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileSupportsAnsiEscapeCodes(self.inner.userdata, file); + } + + pub fn fileSetLength(userdata: ?*anyopaque, file: std.Io.File, length: u64) std.Io.File.SetLengthError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileSetLength(self.inner.userdata, file, length); + } + + pub fn fileSetOwner(userdata: ?*anyopaque, file: std.Io.File, uid: ?std.Io.File.Uid, gid: ?std.Io.File.Gid) std.Io.File.SetOwnerError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileSetOwner(self.inner.userdata, file, uid, gid); + } + + pub fn fileSetPermissions(userdata: ?*anyopaque, file: std.Io.File, permissions: std.Io.File.Permissions) std.Io.File.SetPermissionsError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileSetPermissions(self.inner.userdata, file, permissions); + } + + pub fn fileSetTimestamps(userdata: ?*anyopaque, file: std.Io.File, options: std.Io.File.SetTimestampsOptions) std.Io.File.SetTimestampsError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileSetTimestamps(self.inner.userdata, file, options); + } + + pub fn fileLock(userdata: ?*anyopaque, file: std.Io.File, lock: std.Io.File.Lock) std.Io.File.LockError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileLock(self.inner.userdata, file, lock); + } + + pub fn fileTryLock(userdata: ?*anyopaque, file: std.Io.File, lock: std.Io.File.Lock) std.Io.File.LockError!bool { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileTryLock(self.inner.userdata, file, lock); + } + + pub fn fileUnlock(userdata: ?*anyopaque, file: std.Io.File) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileUnlock(self.inner.userdata, file); + } + + pub fn fileDowngradeLock(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.DowngradeLockError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileDowngradeLock(self.inner.userdata, file); + } + + pub fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileRealPath(self.inner.userdata, file, out_buffer); + } + + pub fn processExecutableOpen(userdata: ?*anyopaque, flags: std.Io.File.OpenFlags) std.process.OpenExecutableError!std.Io.File { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.processExecutableOpen(self.inner.userdata, flags); + } + + pub fn processExecutablePath(userdata: ?*anyopaque, buffer: []u8) std.process.ExecutablePathError!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.processExecutablePath(self.inner.userdata, buffer); + } + + pub fn lockStderr(userdata: ?*anyopaque, mode: ?std.Io.Terminal.Mode) std.Io.Cancelable!std.Io.LockedStderr { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.lockStderr(self.inner.userdata, mode); + } + + pub fn tryLockStderr(userdata: ?*anyopaque, mode: ?std.Io.Terminal.Mode) std.Io.Cancelable!?std.Io.LockedStderr { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.tryLockStderr(self.inner.userdata, mode); + } + + pub fn unlockStderr(userdata: ?*anyopaque) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.unlockStderr(self.inner.userdata); + } + + pub fn processSetCurrentDir(userdata: ?*anyopaque, dir: std.Io.Dir) std.process.SetCurrentDirError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.processSetCurrentDir(self.inner.userdata, dir); } pub fn now(userdata: ?*anyopaque, clock: std.Io.Clock) std.Io.Clock.Error!std.Io.Timestamp { @@ -352,9 +557,19 @@ pub const VFSBase = struct { return self.inner.vtable.netWrite(self.inner.userdata, dest, header, data, splat); } - pub fn netClose(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle) void { + pub fn netWriteFile(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.net.Stream.Writer.WriteFileError!usize { const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); - self.inner.vtable.netClose(self.inner.userdata, handle); + return self.inner.vtable.netWriteFile(self.inner.userdata, handle, header, reader, limit); + } + + pub fn netClose(userdata: ?*anyopaque, handles: []const std.Io.net.Socket.Handle) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + self.inner.vtable.netClose(self.inner.userdata, handles); + } + + pub fn netShutdown(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, how: std.Io.net.ShutdownHow) std.Io.net.ShutdownError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netShutdown(self.inner.userdata, handle, how); } pub fn netInterfaceNameResolve(userdata: ?*anyopaque, name: *const std.Io.net.Interface.Name) std.Io.net.Interface.Name.ResolveError!std.Io.net.Interface { @@ -367,7 +582,7 @@ pub const VFSBase = struct { return self.inner.vtable.netInterfaceName(self.inner.userdata, iface); } - pub fn netLookup(userdata: ?*anyopaque, host: std.Io.net.HostName, q: *std.Io.Queue(std.Io.net.HostName.LookupResult), options: std.Io.net.HostName.LookupOptions) void { + pub fn netLookup(userdata: ?*anyopaque, host: std.Io.net.HostName, q: *std.Io.Queue(std.Io.net.HostName.LookupResult), options: std.Io.net.HostName.LookupOptions) std.Io.net.HostName.LookupError!void { const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); return self.inner.vtable.netLookup(self.inner.userdata, host, q, options); } diff --git a/zml2/io/vfs/hf2.zig b/zml2/io/vfs/hf2.zig index 7864514c..c231eca7 100644 --- a/zml2/io/vfs/hf2.zig +++ b/zml2/io/vfs/hf2.zig @@ -196,7 +196,7 @@ pub const HF = struct { } } - pub fn performRead(self: *HF, handle: *Handle, data: [][]u8, offset: u64) std.Io.File.Reader.Error!usize { + pub fn performRead(self: *HF, handle: *Handle, data: []const []u8, offset: u64) std.Io.File.Reader.Error!usize { if (offset >= handle.size) { log.info("EOF", .{}); return 0; @@ -231,22 +231,16 @@ pub const HF = struct { }; const reader = resp.reader(&.{}); - var total_n: usize = 0; - while (true) { - total_n += reader.readSliceShort(buffer: []u8) .readVec(data) catch |err| return switch (err) { - std.Io.Reader.Error.EndOfStream => return total_n, - else => std.Io.File.Reader.Error.Unexpected, - }; - } + return reader.readSliceShort(data[0]) catch return std.Io.File.Reader.Error.Unexpected; } - pub fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: [][]u8, offset: u64) std.Io.File.ReadPositionalError!usize { + pub fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); return try self.performRead(handle, data, offset); } - fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: [][]u8) std.Io.File.Reader.Error!usize { + fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); const total = try self.performRead(handle, data, handle.pos); @@ -254,16 +248,18 @@ pub const HF = struct { return total; } - fn fileClose(userdata: ?*anyopaque, file: std.Io.File) void { + fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); - self.closeHandle(@intCast(file.handle)) catch unreachable; + for (files) |file| { + self.closeHandle(@intCast(file.handle)) catch unreachable; + } } }; pub fn main() !void { const allocator = std.heap.smp_allocator; - var threaded: std.Io.Threaded = .init(allocator); + var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); var client: std.http.Client = .{ @@ -289,96 +285,9 @@ pub fn main() !void { const buffer = try allocator.alloc(u8, 128 * 1024 * 1024); var reader = file.reader(io, &.{}); defer log.info("DONE", .{}); - while (true) { - const read = try reader.interface.readSliceShort(buffer); - log.info("Read {d} bytes", .{read}); - if (read == 0) break; - } - // var uri_buffer: [512]u8 = undefined; - // const repo: HF.Repo = try .parse("Qwen/Qwen3-8B@main/config.json"); + const stdout: std.Io.File = .stdout(); + var stdout_writer = stdout.writer(io, buffer); - // var buffer: [256]u8 = undefined; - // const gna = try std.fmt.bufPrint(&buffer, API.TREE_URL_TEMPLATE, repo); - // log.info(">>>> repo |{s}|", .{gna}); - - // var req = try self.client.request(.GET, try .parse(std.fmt.bufPrint(&uri_buffer, API.TREE_URL_TEMPLATE, .{repo})), .{ - // .headers = .{ - // .authorization = .{ .override = self.token() }, - // }, - // }); - // defer req.deinit(); - - // req.sendBodiless() catch { - // log.err("Failed to send HTTP request for HF model size API URL", .{}); - // return; - // }; - - // var redirect_buffer: [8 * 1024]u8 = undefined; - // var resp = req.receiveHead(&redirect_buffer) catch { - // log.err("Failed to receive HTTP response for HF model size API URL", .{}); - // return std.Io.Dir.OpenError.FileNotFound; - // }; - - // var read_buffer: [8 * 1024]u8 = undefined; - // const reader = resp.reader(&read_buffer); - // _ = reader; // autofix - - // https://huggingface.co/Qwen/Qwen3-8B/resolve/main/config.json - // https://huggingface.co/api/models/meta-llama/Llama-3.1-8B-Instruct/tree/main - // - // var req = try client.request(.GET, try .parse("https://huggingface.co/Qwen/Qwen3-8B/resolve/main/config.json"), .{ - // .headers = .{ - // .accept_encoding = .{ .override = "identity" }, - // .authorization = .{ .override = "Bearer XXX" }, - // }, - // }); - // defer req.deinit(); - - // req.sendBodiless() catch { - // log.err("Failed to send HTTP request for HF model size API URL", .{}); - // return; - // }; - - // var redirect_buffer: [16 * 1024]u8 = undefined; - // var resp = req.receiveHead(&redirect_buffer) catch { - // log.err("Failed to receive HTTP response for HF model size API URL", .{}); - // return; - // }; - - // log.info(">>>> {any}", .{resp.head}); - - // var read_buffer: [8 * 1024]u8 = undefined; - // const reader = resp.reader(&read_buffer); - - // var json_reader: std.json.Reader = .init(allocator, reader); - // const entries = std.json.parseFromTokenSourceLeaky([]API.Entry, allocator, &json_reader, .{ .ignore_unknown_fields = true }) catch |err| { - // log.err("Failed to parse HF model size response: {any}", .{err}); - // return; - // }; - - // log.info(">>>> {any}", .{entries}); - - // req. - // defer req.deinit(); - - // req.sendBodiless() catch { - // log.err("Failed to send HTTP request for HF model size API URL {s}", .{api_url}); - // return Error.RequestFailed; - // }; - - // var resp = req.receiveHead(&redirect_buffer) catch { - // log.err("Failed to receive HTTP response for HF model size API URL {s}", .{api_url}); - // return Error.RequestFailed; - // }; - - // if (resp.head.status != .ok) { - // log.err("Failed to fetch HF model size, got {s} for {s}", .{ @tagName(resp.head.status), api_url }); - // return Error.RequestFailed; - // } - - // const content_length = resp.head.content_length orelse { - // log.err("Missing Content-Length in HF model size response for {s}", .{api_url}); - // return Error.ResponseParsingFailed; - // }; + _ = try reader.interface.stream(&stdout_writer.interface, .unlimited); } diff --git a/zml2/io/vfs/proxy.zig b/zml2/io/vfs/proxy.zig index 75cae361..0382d5ac 100644 --- a/zml2/io/vfs/proxy.zig +++ b/zml2/io/vfs/proxy.zig @@ -20,41 +20,75 @@ pub const IoProxy = struct { } pub const vtable: std.Io.VTable = .{ - .dirMake = dirMake, - .dirMakePath = dirMakePath, - .dirMakeOpenPath = dirMakeOpenPath, - .dirStat = dirStat, - .dirStatPath = dirStatPath, - .dirAccess = dirAccess, - .dirCreateFile = dirCreateFile, - .dirOpenFile = dirOpenFile, - .dirOpenDir = dirOpenDir, - .dirClose = dirClose, - .fileStat = fileStat, - .fileClose = fileClose, - .fileWriteStreaming = fileWriteStreaming, - .fileWritePositional = fileWritePositional, - .fileReadStreaming = fileReadStreaming, - .fileReadPositional = fileReadPositional, - .fileSeekBy = fileSeekBy, - .fileSeekTo = fileSeekTo, - .openSelfExe = openSelfExe, .async = async, .concurrent = concurrent, .await = await, .cancel = cancel, - .cancelRequested = cancelRequested, .groupAsync = groupAsync, .groupConcurrent = groupConcurrent, - .groupWait = groupWait, + .groupAwait = groupAwait, .groupCancel = groupCancel, + .recancel = recancel, + .swapCancelProtection = swapCancelProtection, + .checkCancel = checkCancel, .select = select, - .mutexLock = mutexLock, - .mutexLockUncancelable = mutexLockUncancelable, - .mutexUnlock = mutexUnlock, - .conditionWait = conditionWait, - .conditionWaitUncancelable = conditionWaitUncancelable, - .conditionWake = conditionWake, + .futexWait = futexWait, + .futexWaitUncancelable = futexWaitUncancelable, + .futexWake = futexWake, + .dirCreateDir = dirCreateDir, + .dirCreateDirPath = dirCreateDirPath, + .dirCreateDirPathOpen = dirCreateDirPathOpen, + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirCreateFile = dirCreateFile, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRead = dirRead, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .dirDeleteFile = dirDeleteFile, + .dirDeleteDir = dirDeleteDir, + .dirRename = dirRename, + .dirSymLink = dirSymLink, + .dirReadLink = dirReadLink, + .dirSetOwner = dirSetOwner, + .dirSetFileOwner = dirSetFileOwner, + .dirSetPermissions = dirSetPermissions, + .dirSetFilePermissions = dirSetFilePermissions, + .dirSetTimestamps = dirSetTimestamps, + .dirHardLink = dirHardLink, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileWriteStreaming = fileWriteStreaming, + .fileWritePositional = fileWritePositional, + .fileWriteFileStreaming = fileWriteFileStreaming, + .fileWriteFilePositional = fileWriteFilePositional, + .fileReadStreaming = fileReadStreaming, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileSync = fileSync, + .fileIsTty = fileIsTty, + .fileEnableAnsiEscapeCodes = fileEnableAnsiEscapeCodes, + .fileSupportsAnsiEscapeCodes = fileSupportsAnsiEscapeCodes, + .fileSetLength = fileSetLength, + .fileSetOwner = fileSetOwner, + .fileSetPermissions = fileSetPermissions, + .fileSetTimestamps = fileSetTimestamps, + .fileLock = fileLock, + .fileTryLock = fileTryLock, + .fileUnlock = fileUnlock, + .fileDowngradeLock = fileDowngradeLock, + .fileRealPath = fileRealPath, + .processExecutableOpen = processExecutableOpen, + .processExecutablePath = processExecutablePath, + .lockStderr = lockStderr, + .tryLockStderr = tryLockStderr, + .unlockStderr = unlockStderr, + .processSetCurrentDir = processSetCurrentDir, .now = now, .sleep = sleep, .netListenIp = netListenIp, @@ -67,107 +101,14 @@ pub const IoProxy = struct { .netReceive = netReceive, .netRead = netRead, .netWrite = netWrite, + .netWriteFile = netWriteFile, .netClose = netClose, + .netShutdown = netShutdown, .netInterfaceNameResolve = netInterfaceNameResolve, .netInterfaceName = netInterfaceName, .netLookup = netLookup, }; - pub fn dirMake(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, mode: std.Io.Dir.Mode) std.Io.Dir.MakeError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirMake(self.inner.userdata, dir, sub_path, mode); - } - - pub fn dirMakePath(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, mode: std.Io.Dir.Mode) std.Io.Dir.MakeError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirMakePath(self.inner.userdata, dir, sub_path, mode); - } - - pub fn dirMakeOpenPath(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.MakeOpenPathError!std.Io.Dir { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirMakeOpenPath(self.inner.userdata, dir, sub_path, options); - } - - pub fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirStat(self.inner.userdata, dir); - } - - pub fn dirStatPath(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatPathOptions) std.Io.Dir.StatPathError!std.Io.Dir.Stat { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirStatPath(self.inner.userdata, dir, sub_path, options); - } - - pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirAccess(self.inner.userdata, dir, sub_path, options); - } - - pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.CreateFlags) std.Io.File.OpenError!std.Io.File { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirCreateFile(self.inner.userdata, dir, sub_path, flags); - } - - pub fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirOpenFile(self.inner.userdata, dir, sub_path, flags); - } - - pub fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirOpenDir(self.inner.userdata, dir, sub_path, options); - } - - pub fn dirClose(userdata: ?*anyopaque, dir: std.Io.Dir) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - self.inner.vtable.dirClose(self.inner.userdata, dir); - } - - pub fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileStat(self.inner.userdata, file); - } - - pub fn fileClose(userdata: ?*anyopaque, file: std.Io.File) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - self.inner.vtable.fileClose(self.inner.userdata, file); - } - - pub fn fileWriteStreaming(userdata: ?*anyopaque, file: std.Io.File, buffer: [][]const u8) std.Io.File.WriteStreamingError!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileWriteStreaming(self.inner.userdata, file, buffer); - } - - pub fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, buffer: [][]const u8, offset: u64) std.Io.File.WritePositionalError!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileWritePositional(self.inner.userdata, file, buffer, offset); - } - - pub fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: [][]u8) std.Io.File.Reader.Error!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileReadStreaming(self.inner.userdata, file, data); - } - - pub fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: [][]u8, offset: u64) std.Io.File.ReadPositionalError!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileReadPositional(self.inner.userdata, file, data, offset); - } - - pub fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileSeekBy(self.inner.userdata, file, relative_offset); - } - - pub fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileSeekTo(self.inner.userdata, file, absolute_offset); - } - - pub fn openSelfExe(userdata: ?*anyopaque, flags: std.Io.File.OpenFlags) std.Io.File.OpenSelfExeError!std.Io.File { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.openSelfExe(self.inner.userdata, flags); - } - pub fn async(userdata: ?*anyopaque, result: []u8, result_alignment: std.mem.Alignment, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque, result: *anyopaque) void) ?*std.Io.AnyFuture { const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); return self.inner.vtable.async(self.inner.userdata, result, result_alignment, context, context_alignment, start); @@ -188,24 +129,19 @@ pub const IoProxy = struct { return self.inner.vtable.cancel(self.inner.userdata, any_future, result, result_alignment); } - pub fn cancelRequested(userdata: ?*anyopaque) bool { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.cancelRequested(self.inner.userdata); - } - - pub fn groupAsync(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (*std.Io.Group, context: *const anyopaque) void) void { + pub fn groupAsync(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (*std.Io.Group, context: *const anyopaque) std.Io.Cancelable!void) void { const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); return self.inner.vtable.groupAsync(self.inner.userdata, group, context, context_alignment, start); } - pub fn groupConcurrent(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (*std.Io.Group, context: *const anyopaque) void) std.Io.ConcurrentError!void { + pub fn groupConcurrent(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (*std.Io.Group, context: *const anyopaque) std.Io.Cancelable!void) std.Io.ConcurrentError!void { const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); return self.inner.vtable.groupConcurrent(self.inner.userdata, group, context, context_alignment, start); } - pub fn groupWait(userdata: ?*anyopaque, group: *std.Io.Group, token: *anyopaque) void { + pub fn groupAwait(userdata: ?*anyopaque, group: *std.Io.Group, token: *anyopaque) std.Io.Cancelable!void { const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.groupWait(self.inner.userdata, group, token); + return self.inner.vtable.groupAwait(self.inner.userdata, group, token); } pub fn groupCancel(userdata: ?*anyopaque, group: *std.Io.Group, token: *anyopaque) void { @@ -213,40 +149,309 @@ pub const IoProxy = struct { return self.inner.vtable.groupCancel(self.inner.userdata, group, token); } + pub fn recancel(userdata: ?*anyopaque) void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.recancel(self.inner.userdata); + } + + pub fn swapCancelProtection(userdata: ?*anyopaque, new: std.Io.CancelProtection) std.Io.CancelProtection { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.swapCancelProtection(self.inner.userdata, new); + } + + pub fn checkCancel(userdata: ?*anyopaque) std.Io.Cancelable!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.checkCancel(self.inner.userdata); + } + pub fn select(userdata: ?*anyopaque, futures: []const *std.Io.AnyFuture) std.Io.Cancelable!usize { const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); return self.inner.vtable.select(self.inner.userdata, futures); } - pub fn mutexLock(userdata: ?*anyopaque, prev_state: std.Io.Mutex.State, mutex: *std.Io.Mutex) std.Io.Cancelable!void { + pub fn futexWait(userdata: ?*anyopaque, ptr: *const u32, expected: u32, timeout: std.Io.Timeout) std.Io.Cancelable!void { const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.mutexLock(self.inner.userdata, prev_state, mutex); + return self.inner.vtable.futexWait(self.inner.userdata, ptr, expected, timeout); } - pub fn mutexLockUncancelable(userdata: ?*anyopaque, prev_state: std.Io.Mutex.State, mutex: *std.Io.Mutex) void { + pub fn futexWaitUncancelable(userdata: ?*anyopaque, ptr: *const u32, expected: u32) void { const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - std.debug.print(">>>>>>>> {*}\n", .{self}); - self.inner.vtable.mutexLockUncancelable(self.inner.userdata, prev_state, mutex); + return self.inner.vtable.futexWaitUncancelable(self.inner.userdata, ptr, expected); } - pub fn mutexUnlock(userdata: ?*anyopaque, prev_state: std.Io.Mutex.State, mutex: *std.Io.Mutex) void { + pub fn futexWake(userdata: ?*anyopaque, ptr: *const u32, max_waiters: u32) void { const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - self.inner.vtable.mutexUnlock(self.inner.userdata, prev_state, mutex); + return self.inner.vtable.futexWake(self.inner.userdata, ptr, max_waiters); } - pub fn conditionWait(userdata: ?*anyopaque, cond: *std.Io.Condition, mutex: *std.Io.Mutex) std.Io.Cancelable!void { + pub fn dirCreateDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions) std.Io.Dir.CreateDirError!void { const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.conditionWait(self.inner.userdata, cond, mutex); + return self.inner.vtable.dirCreateDir(self.inner.userdata, dir, sub_path, permissions); } - pub fn conditionWaitUncancelable(userdata: ?*anyopaque, cond: *std.Io.Condition, mutex: *std.Io.Mutex) void { + pub fn dirCreateDirPath(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions) std.Io.Dir.CreateDirPathError!std.Io.Dir.CreatePathStatus { const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - self.inner.vtable.conditionWaitUncancelable(self.inner.userdata, cond, mutex); + return self.inner.vtable.dirCreateDirPath(self.inner.userdata, dir, sub_path, permissions); } - pub fn conditionWake(userdata: ?*anyopaque, cond: *std.Io.Condition, wake: std.Io.Condition.Wake) void { + pub fn dirCreateDirPathOpen(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions, options: std.Io.Dir.OpenOptions) std.Io.Dir.CreateDirPathOpenError!std.Io.Dir { const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - self.inner.vtable.conditionWake(self.inner.userdata, cond, wake); + return self.inner.vtable.dirCreateDirPathOpen(self.inner.userdata, dir, sub_path, permissions, options); + } + + pub fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirOpenDir(self.inner.userdata, dir, sub_path, options); + } + + pub fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirStat(self.inner.userdata, dir); + } + + pub fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirStatFile(self.inner.userdata, dir, sub_path, options); + } + + pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirAccess(self.inner.userdata, dir, sub_path, options); + } + + pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.CreateFlags) std.Io.File.OpenError!std.Io.File { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirCreateFile(self.inner.userdata, dir, sub_path, flags); + } + + pub fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirOpenFile(self.inner.userdata, dir, sub_path, flags); + } + + pub fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + self.inner.vtable.dirClose(self.inner.userdata, dirs); + } + + pub fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirRead(self.inner.userdata, reader, entries); + } + + pub fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirRealPath(self.inner.userdata, dir, out_buffer); + } + + pub fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirRealPathFile(self.inner.userdata, dir, path_name, out_buffer); + } + + pub fn dirDeleteFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8) std.Io.Dir.DeleteFileError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirDeleteFile(self.inner.userdata, dir, sub_path); + } + + pub fn dirDeleteDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8) std.Io.Dir.DeleteDirError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirDeleteDir(self.inner.userdata, dir, sub_path); + } + + pub fn dirRename(userdata: ?*anyopaque, old_dir: std.Io.Dir, old_sub_path: []const u8, new_dir: std.Io.Dir, new_sub_path: []const u8) std.Io.Dir.RenameError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirRename(self.inner.userdata, old_dir, old_sub_path, new_dir, new_sub_path); + } + + pub fn dirSymLink(userdata: ?*anyopaque, dir: std.Io.Dir, target_path: []const u8, sym_link_path: []const u8, flags: std.Io.Dir.SymLinkFlags) std.Io.Dir.SymLinkError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirSymLink(self.inner.userdata, dir, target_path, sym_link_path, flags); + } + + pub fn dirReadLink(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, buffer: []u8) std.Io.Dir.ReadLinkError!usize { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirReadLink(self.inner.userdata, dir, sub_path, buffer); + } + + pub fn dirSetOwner(userdata: ?*anyopaque, dir: std.Io.Dir, uid: ?std.Io.File.Uid, gid: ?std.Io.File.Gid) std.Io.Dir.SetOwnerError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirSetOwner(self.inner.userdata, dir, uid, gid); + } + + pub fn dirSetFileOwner(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, uid: ?std.Io.File.Uid, gid: ?std.Io.File.Gid, options: std.Io.Dir.SetFileOwnerOptions) std.Io.Dir.SetFileOwnerError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirSetFileOwner(self.inner.userdata, dir, sub_path, uid, gid, options); + } + + pub fn dirSetPermissions(userdata: ?*anyopaque, dir: std.Io.Dir, permissions: std.Io.Dir.Permissions) std.Io.Dir.SetPermissionsError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirSetPermissions(self.inner.userdata, dir, permissions); + } + + pub fn dirSetFilePermissions(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.File.Permissions, options: std.Io.Dir.SetFilePermissionsOptions) std.Io.Dir.SetFilePermissionsError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirSetFilePermissions(self.inner.userdata, dir, sub_path, permissions, options); + } + + pub fn dirSetTimestamps(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.SetTimestampsOptions) std.Io.Dir.SetTimestampsError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirSetTimestamps(self.inner.userdata, dir, sub_path, options); + } + + pub fn dirHardLink(userdata: ?*anyopaque, old_dir: std.Io.Dir, old_sub_path: []const u8, new_dir: std.Io.Dir, new_sub_path: []const u8, options: std.Io.Dir.HardLinkOptions) std.Io.Dir.HardLinkError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.dirHardLink(self.inner.userdata, old_dir, old_sub_path, new_dir, new_sub_path, options); + } + + pub fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileStat(self.inner.userdata, file); + } + + pub fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileLength(self.inner.userdata, file); + } + + pub fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + self.inner.vtable.fileClose(self.inner.userdata, files); + } + + pub fn fileWriteStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize) std.Io.File.Writer.Error!usize { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileWriteStreaming(self.inner.userdata, file, header, data, splat); + } + + pub fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) std.Io.File.WritePositionalError!usize { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileWritePositional(self.inner.userdata, file, header, data, splat, offset); + } + + pub fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileWriteFileStreaming(self.inner.userdata, file, header, reader, limit); + } + + pub fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileWriteFilePositional(self.inner.userdata, file, header, reader, limit, offset); + } + + pub fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileReadStreaming(self.inner.userdata, file, data); + } + + pub fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileReadPositional(self.inner.userdata, file, data, offset); + } + + pub fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileSeekBy(self.inner.userdata, file, relative_offset); + } + + pub fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileSeekTo(self.inner.userdata, file, absolute_offset); + } + + pub fn fileSync(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.SyncError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileSync(self.inner.userdata, file); + } + + pub fn fileIsTty(userdata: ?*anyopaque, file: std.Io.File) std.Io.Cancelable!bool { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileIsTty(self.inner.userdata, file); + } + + pub fn fileEnableAnsiEscapeCodes(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.EnableAnsiEscapeCodesError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileEnableAnsiEscapeCodes(self.inner.userdata, file); + } + + pub fn fileSupportsAnsiEscapeCodes(userdata: ?*anyopaque, file: std.Io.File) std.Io.Cancelable!bool { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileSupportsAnsiEscapeCodes(self.inner.userdata, file); + } + + pub fn fileSetLength(userdata: ?*anyopaque, file: std.Io.File, length: u64) std.Io.File.SetLengthError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileSetLength(self.inner.userdata, file, length); + } + + pub fn fileSetOwner(userdata: ?*anyopaque, file: std.Io.File, uid: ?std.Io.File.Uid, gid: ?std.Io.File.Gid) std.Io.File.SetOwnerError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileSetOwner(self.inner.userdata, file, uid, gid); + } + + pub fn fileSetPermissions(userdata: ?*anyopaque, file: std.Io.File, permissions: std.Io.File.Permissions) std.Io.File.SetPermissionsError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileSetPermissions(self.inner.userdata, file, permissions); + } + + pub fn fileSetTimestamps(userdata: ?*anyopaque, file: std.Io.File, options: std.Io.File.SetTimestampsOptions) std.Io.File.SetTimestampsError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileSetTimestamps(self.inner.userdata, file, options); + } + + pub fn fileLock(userdata: ?*anyopaque, file: std.Io.File, lock: std.Io.File.Lock) std.Io.File.LockError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileLock(self.inner.userdata, file, lock); + } + + pub fn fileTryLock(userdata: ?*anyopaque, file: std.Io.File, lock: std.Io.File.Lock) std.Io.File.LockError!bool { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileTryLock(self.inner.userdata, file, lock); + } + + pub fn fileUnlock(userdata: ?*anyopaque, file: std.Io.File) void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileUnlock(self.inner.userdata, file); + } + + pub fn fileDowngradeLock(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.DowngradeLockError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileDowngradeLock(self.inner.userdata, file); + } + + pub fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.fileRealPath(self.inner.userdata, file, out_buffer); + } + + pub fn processExecutableOpen(userdata: ?*anyopaque, flags: std.Io.File.OpenFlags) std.process.OpenExecutableError!std.Io.File { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.processExecutableOpen(self.inner.userdata, flags); + } + + pub fn processExecutablePath(userdata: ?*anyopaque, buffer: []u8) std.process.ExecutablePathError!usize { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.processExecutablePath(self.inner.userdata, buffer); + } + + pub fn lockStderr(userdata: ?*anyopaque, mode: ?std.Io.Terminal.Mode) std.Io.Cancelable!std.Io.LockedStderr { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.lockStderr(self.inner.userdata, mode); + } + + pub fn tryLockStderr(userdata: ?*anyopaque, mode: ?std.Io.Terminal.Mode) std.Io.Cancelable!?std.Io.LockedStderr { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.tryLockStderr(self.inner.userdata, mode); + } + + pub fn unlockStderr(userdata: ?*anyopaque) void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.unlockStderr(self.inner.userdata); + } + + pub fn processSetCurrentDir(userdata: ?*anyopaque, dir: std.Io.Dir) std.process.SetCurrentDirError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.processSetCurrentDir(self.inner.userdata, dir); } pub fn now(userdata: ?*anyopaque, clock: std.Io.Clock) std.Io.Clock.Error!std.Io.Timestamp { @@ -309,9 +514,19 @@ pub const IoProxy = struct { return self.inner.vtable.netWrite(self.inner.userdata, dest, header, data, splat); } - pub fn netClose(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle) void { + pub fn netWriteFile(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.net.Stream.Writer.WriteFileError!usize { const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - self.inner.vtable.netClose(self.inner.userdata, handle); + return self.inner.vtable.netWriteFile(self.inner.userdata, handle, header, reader, limit); + } + + pub fn netClose(userdata: ?*anyopaque, handles: []const std.Io.net.Socket.Handle) void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + self.inner.vtable.netClose(self.inner.userdata, handles); + } + + pub fn netShutdown(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, how: std.Io.net.ShutdownHow) std.Io.net.ShutdownError!void { + const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); + return self.inner.vtable.netShutdown(self.inner.userdata, handle, how); } pub fn netInterfaceNameResolve(userdata: ?*anyopaque, name: *const std.Io.net.Interface.Name) std.Io.net.Interface.Name.ResolveError!std.Io.net.Interface { @@ -324,7 +539,7 @@ pub const IoProxy = struct { return self.inner.vtable.netInterfaceName(self.inner.userdata, iface); } - pub fn netLookup(userdata: ?*anyopaque, host: std.Io.net.HostName, q: *std.Io.Queue(std.Io.net.HostName.LookupResult), options: std.Io.net.HostName.LookupOptions) void { + pub fn netLookup(userdata: ?*anyopaque, host: std.Io.net.HostName, q: *std.Io.Queue(std.Io.net.HostName.LookupResult), options: std.Io.net.HostName.LookupOptions) std.Io.net.HostName.LookupError!void { const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); return self.inner.vtable.netLookup(self.inner.userdata, host, q, options); } From b7a9bd68ccb3ad05367aaecc67f9c50688fbd623 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 2 Jan 2026 20:37:14 +0000 Subject: [PATCH 173/881] third_party/clap: switch to fork for Zig 0.16 compat --- third_party/com_github_hejsil_clap/repo.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/third_party/com_github_hejsil_clap/repo.bzl b/third_party/com_github_hejsil_clap/repo.bzl index b88dac44..11a00b3c 100644 --- a/third_party/com_github_hejsil_clap/repo.bzl +++ b/third_party/com_github_hejsil_clap/repo.bzl @@ -3,7 +3,7 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") def repo(): new_git_repository( name = "com_github_hejsil_clap", - remote = "https://github.com/Hejsil/zig-clap.git", - commit = "3ae92228409d82a2c906fafc228b7920d9ff046b", + remote = "https://github.com/devoc09/zig-clap.git", + commit = "3c35ff18bb32fec2f53e12e6bdbb00422d6c33fe", build_file = "//:third_party/com_github_hejsil_clap/clap.bazel", ) From e7b6651c49241cf06e4943cdc3bf8edd1d968df3 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 2 Jan 2026 20:37:42 +0000 Subject: [PATCH 174/881] runtimes/cuda: update for Zig 0.16 --- runtimes/cuda/cuda.zig | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/runtimes/cuda/cuda.zig b/runtimes/cuda/cuda.zig index b1289869..57fff810 100644 --- a/runtimes/cuda/cuda.zig +++ b/runtimes/cuda/cuda.zig @@ -16,9 +16,7 @@ pub fn isEnabled() bool { } fn hasNvidiaDevice(io: std.Io) bool { - // TODO(Corentin): Plug that when fs stuff is merged in std.Io - _ = io; - std.fs.accessAbsolute("/dev/nvidiactl", .{ .read = true }) catch return false; + std.Io.Dir.accessAbsolute(io, "/dev/nvidiactl", .{ .read = true }) catch return false; return true; } From f9828e445fda259caf2fae6531d52f6190cde177 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 2 Jan 2026 20:37:56 +0000 Subject: [PATCH 175/881] examples/llama: disable VFS --- examples/llama/main.zig | 50 +++-------------------------------------- 1 file changed, 3 insertions(+), 47 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index a6781a8b..4bbd3627 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -35,55 +35,11 @@ pub fn main() !void { log.info("LLama was compiled with {}", .{@import("builtin").mode}); const allocator = std.heap.c_allocator; - //var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - //defer _ = gpa.deinit(); - - //const allocator = gpa.allocator(); var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); - var vfs_file: zml.io.VFS.File = .init(allocator, threaded.io(), .{}); - - var http_client: std.http.Client = .{ - .allocator = allocator, - .io = threaded.io(), - .connection_pool = .{ - .free_size = threaded.async_limit.toInt() orelse 16, - }, - }; - - try http_client.initDefaultProxies(allocator); - defer http_client.deinit(); - - var vfs_http: zml.io.VFS.HTTP = try .init(allocator, threaded.io(), &http_client, .{}); - defer vfs_http.deinit(); - - var hf_auth: zml.io.VFS.HF.Auth = try .auto(allocator, threaded.io()); - defer hf_auth.deinit(allocator); - - var hf_vfs: zml.io.VFS.HF = try .init( - allocator, - threaded.io(), - &http_client, - .{ - .request_range_min = 8 * 1024, - .request_range_max = 128 * 1024 * 1024, - .hf_pagination_limit = 100, - .auth = hf_auth, - }, - ); - defer hf_vfs.deinit(); - - var vfs: zml.io.VFS = .init(allocator, threaded.io()); - defer vfs.deinit(); - - try vfs.register("file", vfs_file.io()); - try vfs.register("http", vfs_http.io()); - try vfs.register("https", vfs_http.io()); - try vfs.register("hf", hf_vfs.io()); - - const io = vfs.io(); + const io = threaded.io(); zml.init(); defer zml.deinit(); @@ -96,7 +52,7 @@ pub fn main() !void { }; var diag: clap.Diagnostic = .{}; var stderr_buffer: [1024]u8 = undefined; - var stderr = std.fs.File.stderr().writer(&stderr_buffer); + var stderr = std.Io.File.stderr().writer(io, &stderr_buffer); defer stderr.interface.flush() catch {}; var cli = clap.parse(clap.Help, ¶ms, parsers, .{ @@ -213,7 +169,7 @@ pub fn main() !void { log.info("✅\tPrompt: {s}", .{prompt}); // Unbuffered writing of the tokens to stdout. - var stdout = std.fs.File.stdout().writer(&.{}); + var stdout = std.Io.File.stdout().writer(io, &.{}); const now = try std.Io.Clock.now(.boot, io); const seed: u128 = cli.args.seed orelse @intCast(now.nanoseconds); From 89afb43e645d8951203000f395db8b912a272b50 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 2 Jan 2026 22:14:00 +0000 Subject: [PATCH 176/881] pjrt,zml: trim pjrtx with the aim to remove it --- pjrt/pjrt.zig | 39 ++++++++- zml2/buffer.zig | 6 +- zml2/pjrtx.zig | 215 ++---------------------------------------------- 3 files changed, 47 insertions(+), 213 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 01bfd1e6..f2aa4830 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -345,7 +345,7 @@ pub const Client = opaque { compile_options_pb: []const u8, }; - pub fn compile(self: *const Client, api: *const Api, args: CompileArgs) ApiError!*LoadedExecutable { + pub fn compileRaw(self: *const Client, api: *const Api, args: CompileArgs) ApiError!*LoadedExecutable { const bytecode_format_ = @tagName(args.bytecode_format); const ret = try api.call(.PJRT_Client_Compile, .{ .program = &pjrtStruct2(c.PJRT_Program, .{ @@ -361,6 +361,11 @@ pub const Client = opaque { return @ptrCast(ret.executable.?); } + pub fn compile(self: *const Client, api: *const Api, io: std.Io, args: CompileArgs) ApiError!*LoadedExecutable { + var future = io.async(compileRaw, .{ self, api, args }); + return future.await(io); + } + pub const BufferFromHostBufferArgs = struct { data: [*]const u8, buffer_type: BufferType, @@ -1058,7 +1063,37 @@ pub const Event = opaque { return @ptrCast(result); } - pub fn await(self: *const Event, api: *const Api) ApiError!void { + pub fn await(self: *Event, api: *const Api, io: std.Io) ApiError!void { + defer self.deinit(api); + + if (self.isReady(api)) { + return; + } + + var ctx = struct { + err: ?*Error = null, + event: std.Io.Event = .unset, + io: std.Io, + }{ .io = io }; + + try self.onReady(api, struct { + fn call(err: ?*Error, user_arg: ?*anyopaque) callconv(.c) void { + const ctx_: *@TypeOf(ctx) = @ptrCast(@alignCast(user_arg.?)); + ctx_.err = err; + ctx_.event.set(ctx_.io); + } + }.call, &ctx); + ctx.event.waitUncancelable(io); + + if (ctx.err) |e| { + defer e.deinit(api); + const err_code = e.getCode(api).toApiError(); + log.err("{t} {s}", .{ err_code, e.getMessage(api) }); + return err_code; + } + } + + pub fn awaitRaw(self: *const Event, api: *const Api) ApiError!void { _ = try api.call(.PJRT_Event_Await, .{ .event = self.inner(), }); diff --git a/zml2/buffer.zig b/zml2/buffer.zig index ef398f09..bafca61c 100644 --- a/zml2/buffer.zig +++ b/zml2/buffer.zig @@ -2,6 +2,7 @@ const std = @import("std"); const stdx = @import("stdx"); +const constants = @import("constants.zig"); const ConstSlice = @import("slice.zig").ConstSlice; const DataType = @import("dtype.zig").DataType; const pjrt = @import("pjrtx.zig"); @@ -10,7 +11,6 @@ const Shape = @import("shape.zig").Shape; const Slice = @import("slice.zig").Slice; const Target = @import("platform.zig").Target; const testing = @import("testing.zig"); -const constants = @import("constants.zig"); const log = std.log.scoped(.zml); @@ -135,9 +135,7 @@ pub const Buffer = struct { pub fn await(self: Buffer, io: std.Io) !Buffer { for (self._shards.constSlice()) |buffer| { - if (buffer.getReadyEvent(self._api)) |ev| { - try ev.await(self._api, io); - } + try buffer.getReadyEvent(self._api).await(self._api, io); } return self; diff --git a/zml2/pjrtx.zig b/zml2/pjrtx.zig index 70424edd..459ee1e7 100644 --- a/zml2/pjrtx.zig +++ b/zml2/pjrtx.zig @@ -23,6 +23,10 @@ pub const NamedValue = pjrt.NamedValue; pub const SerializeResult = pjrt.SerializeResult; pub const Stream = pjrt.Stream; pub const ShapeSpec = pjrt.ShapeSpec; +pub const Buffer = pjrt.Buffer; +pub const Event = pjrt.Event; +pub const LoadedExecutable = pjrt.LoadedExecutable; +pub const AsyncHostToDeviceTransferManager = pjrt.AsyncHostToDeviceTransferManager; const zml = struct { pub const DataType = @import("dtype.zig").DataType; @@ -79,7 +83,7 @@ pub const Client = opaque { return @ptrCast(try self.inner().createViewOfDeviceBuffer(api, args)); } - fn compileSync(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { + pub fn compile(self: *const Client, api: *const Api, allocator: std.mem.Allocator, io: std.Io, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { var bytecode: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); defer bytecode.deinit(); module.operation().writeBytecode(.{ .desired_emit_version = 1 }, &bytecode.writer) catch |err| { @@ -90,9 +94,6 @@ pub const Client = opaque { }; }; - var serialized_buffer: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); - defer serialized_buffer.deinit(); - const stablehlo_version = blk: { if (api.stablehloCurrentVersion()) |requested_version| { break :blk dialects.stablehlo.smallerVersion(requested_version, dialects.stablehlo.currentVersion()); @@ -100,6 +101,8 @@ pub const Client = opaque { break :blk dialects.stablehlo.minimumVersion(); }; + var serialized_buffer: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); + defer serialized_buffer.deinit(); dialects.stablehlo.serializePortableArtifact(bytecode.written(), stablehlo_version, &serialized_buffer.writer) catch |err| { log.err("failed to serialize to portable artifact: {}", .{err}); return switch (err) { @@ -108,18 +111,13 @@ pub const Client = opaque { }; }; - return @ptrCast(try self.inner().compile(api, .{ + return @ptrCast(try self.inner().compile(api, io, .{ .bytecode = serialized_buffer.written(), .bytecode_format = .mlir, .compile_options_pb = compile_options_pb, })); } - pub fn compile(self: *const Client, api: *const Api, allocator: std.mem.Allocator, io: std.Io, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { - var future = io.async(compileSync, .{ self, api, allocator, module, compile_options_pb }); - return future.await(io); - } - pub fn addressableMemories(self: *const Client, api: *const Api) []*const Memory { return self.inner().addressableMemories(api); } @@ -146,203 +144,6 @@ pub const Client = opaque { } }; -pub const Buffer = opaque { - pub const inner = InnerMixin(pjrt.Buffer).inner; - - pub fn deinit(self: *Buffer, api: *const Api) void { - self.inner().deinit(api); - } - - pub fn getDevice(self: *const Buffer, api: *const Api) ApiError!*Device { - return try self.inner().getDevice(api); - } - - pub fn delete(self: *Buffer, api: *const Api) void { - self.inner().delete(api); - } - - pub fn isDeleted(self: *const Buffer, api: *const Api) bool { - return self.inner().isDeleted(api); - } - - pub fn isOnCpu(self: *const Buffer, api: *const Api) bool { - return self.inner().isOnCpu(api); - } - - pub fn memory(self: *const Buffer, api: *const Api) *const Memory { - return self.inner().memory(api); - } - - pub fn toHostBuffer(self: *const Buffer, api: *const Api, dst: []u8) ApiError!?*Event { - return @ptrCast(try self.inner().toHostBuffer(api, dst)); - } - - pub fn getElementType(self: *const Buffer, api: *const Api) zml.DataType { - return dtypeFromBufferType(self.inner().getElementType(api)); - } - - pub fn getDimensions(self: *const Buffer, api: *const Api) []const i64 { - return self.inner().getDimensions(api); - } - - pub fn getUnpaddedDimensions(self: *const Buffer, api: *const Api) ApiError![]const i64 { - return try self.inner().getUnpaddedDimensions(api); - } - - pub fn getOnDeviceSizeInBytes(self: *const Buffer, api: *const Api) ApiError!usize { - return try self.inner().getOnDeviceSizeInBytes(api); - } - - pub fn copyToDevice(self: *const Buffer, api: *const Api, device: *Device) ApiError!*Buffer { - return @ptrCast(try self.inner().copyToDevice(api, device)); - } - - pub fn copyToMemory(self: *const Buffer, api: *const Api, memory_: *const Memory) ApiError!*Buffer { - return @ptrCast(try self.inner().copyToMemory(api, memory_)); - } - - pub fn getReadyEvent(self: *const Buffer, api: *const Api) ?*Event { - return @ptrCast(self.inner().getReadyEvent(api)); - } - - pub fn getOpaqueDeviceMemoryDataPointer(self: *const Buffer, api: *const Api) ApiError!*anyopaque { - return try self.inner().getOpaqueDeviceMemoryDataPointer(api); - } -}; - -pub const Event = opaque { - pub const inner = InnerMixin(pjrt.Event).inner; - - pub fn deinit(self: *Event, api: *const Api) void { - self.inner().deinit(api); - } - - pub fn isReady(self: *const Event, api: *const Api) bool { - return self.inner().isReady(api); - } - - pub fn getEventError(self: *const Event, api: *const Api) ?*Error { - return self.inner().getEventError(api); - } - - pub fn awaitBlocking(self: *Event, api: *const Api) ApiError!void { - if (self.isReady(api)) { - return; - } - try self.inner().await(api); - } - - pub fn await(self: *Event, api: *const Api, io: std.Io) ApiError!void { - defer self.deinit(api); - - if (self.isReady(api)) { - return; - } - - var ctx = struct { - err: ?*pjrt.Error = null, - event: std.Io.Event = .unset, - io: std.Io, - }{ .io = io }; - - try self.inner().onReady(api, struct { - fn call(err: ?*pjrt.Error, user_arg: ?*anyopaque) callconv(.c) void { - const ctx_: *@TypeOf(ctx) = @ptrCast(@alignCast(user_arg.?)); - ctx_.err = err; - ctx_.event.set(ctx_.io); - } - }.call, &ctx); - ctx.event.waitUncancelable(io); - - if (ctx.err) |e| { - defer e.deinit(api); - const err_code = e.getCode(api).toApiError(); - log.err("{t} {s}", .{ err_code, e.getMessage(api) }); - return err_code; - } - } -}; - -pub const LoadedExecutable = opaque { - const inner = InnerMixin(pjrt.LoadedExecutable).inner; - - pub fn deinit(self: *LoadedExecutable, api: *const Api) void { - self.inner().deinit(api); - } - - pub fn delete(self: *LoadedExecutable, api: *const Api) void { - self.inner().delete(api); - } - - pub fn isDeleted(self: *const LoadedExecutable, api: *const Api) bool { - return self.inner().isDeleted(api); - } - - pub fn getAddressableDevices(self: *const LoadedExecutable, api: *const Api) []const *Device { - return self.inner().getAddressableDevices(api); - } - - pub const ExecuteArgs = struct { - arguments: []const [*]const *const Buffer, - num_args: usize, - results: []const [*]*Buffer, - events: ?[]?*Event, - non_donatable_input_indices: []const i64 = &.{}, - context: ?*ExecuteContext, - }; - - pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ - .num_args = args.num_args, - .arguments = @ptrCast(args.arguments), - .results = @ptrCast(args.results), - .events = @ptrCast(args.events), - .non_donatable_input_indices = args.non_donatable_input_indices, - .context = args.context, - }); - } - - pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { - return try self.inner().getExecutable(api); - } -}; - -pub const AsyncHostToDeviceTransferManager = opaque { - const inner = InnerMixin(pjrt.AsyncHostToDeviceTransferManager).inner; - - pub fn deinit(self: *AsyncHostToDeviceTransferManager, api: *const Api) void { - self.inner().deinit(api); - } - - pub fn transferData(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize, data: []const u8, offset: i64, is_last_transfer: bool) ApiError!*Event { - return @ptrCast(try self.inner().transferData(api, buffer_index, data, offset, is_last_transfer)); - } - - pub fn retrieveBuffer(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize) ApiError!*Buffer { - return @ptrCast(try self.inner().retrieveBuffer(api, buffer_index)); - } - - pub fn device(self: *AsyncHostToDeviceTransferManager, api: *const Api) ApiError!*Device { - return @ptrCast(try self.inner().device(api)); - } - - pub fn bufferCount(self: *AsyncHostToDeviceTransferManager, api: *const Api) ApiError!usize { - return self.inner().bufferCount(api); - } - - pub fn bufferSize(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize) ApiError!usize { - return self.inner().bufferSize(api, buffer_index); - } - - pub fn setBufferError(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize, error_code: ErrorCode, error_message: []const u8) ApiError!void { - return self.inner().setBufferError(api, buffer_index, @intFromEnum(error_code), error_message); - } - - pub fn addMetadata(self: *AsyncHostToDeviceTransferManager, api: *const Api, transfer_metadata: []const NamedValue) ApiError!void { - return self.inner().addMetadata(api, transfer_metadata); - } -}; - pub fn bufferTypeFromDtype(dt: zml.DataType) pjrt.BufferType { return switch (dt) { inline else => |tag| @field(pjrt.BufferType, @tagName(tag)), From 39876f6936730453ca2f15af59723680042e79f8 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sat, 3 Jan 2026 09:16:16 +0000 Subject: [PATCH 177/881] zml/tokenizers: add fromBytes --- examples/llama/main.zig | 21 +++++++++++-------- zml2/tokenizer/sentencepiece/sentencepiece.i | 1 + .../tokenizer/sentencepiece/sentencepiece.zig | 7 +++++++ zml2/tokenizer/tokenizer.zig | 21 ++++++++++++------- 4 files changed, 33 insertions(+), 17 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 4bbd3627..8d06fc79 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -131,6 +131,9 @@ pub fn main() !void { .rng = .init(), }; + var tokenizer_future = io.async(loadTokenizer, .{ allocator, io, repo.dir }); + errdefer _ = tokenizer_future.cancel(io) catch {}; + var compiled_model_result_future = io.async(struct { fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: zml.Platform, llama_model_: llama.LlamaLM, llama_parameters_: LlamaParameters) !CompileModelResult { return compileModel(allocator_, io_, platform_, llama_model_, llama_parameters_); @@ -162,7 +165,7 @@ pub fn main() !void { var kv_cache_buffers = try llama_parameters.kv_cache.initBuffer(io, platform); defer llama.KvCache.deinitBuffer(&kv_cache_buffers); - var tokenizer = try loadTokenizer(allocator, io, repo.dir); + var tokenizer = try tokenizer_future.await(io); defer tokenizer.deinit(); const prompt = cli.args.prompt orelse "What is the capital of France?"; @@ -171,7 +174,7 @@ pub fn main() !void { // Unbuffered writing of the tokens to stdout. var stdout = std.Io.File.stdout().writer(io, &.{}); - const now = try std.Io.Clock.now(.boot, io); + const now = try std.Io.Clock.boot.now(io); const seed: u128 = cli.args.seed orelse @intCast(now.nanoseconds); const skip_llama3_encoding = cli.args.@"no-llama3" orelse false; @@ -213,7 +216,7 @@ fn parseConfig(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !std.j } fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !zml.tokenizer.Tokenizer { - var timer = try stdx.time.Timer.start(); + var timer = try std.time.Timer.start(); log.info("Loading tokenizer", .{}); defer log.info("Loaded tokenizer [{D}]", .{timer.read()}); const bytes = b: { @@ -228,7 +231,7 @@ fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !zml }; defer allocator.free(bytes); - return .{ .hftokenizers = try zml.tokenizer.hftokenizers.HFTokenizer.fromBytes(bytes) }; + return try .fromBytes(allocator, io, bytes); } const LlamaParameters = struct { @@ -245,7 +248,7 @@ const CompileModelResult = struct { }; fn compileModel(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform, llama_model: llama.LlamaLM, parameters: LlamaParameters) !CompileModelResult { - var timer = try stdx.time.Timer.start(); + var timer = try std.time.Timer.start(); log.info("Compiling model", .{}); defer log.info("Compiled model [{D}]", .{timer.read()}); @@ -271,7 +274,7 @@ fn compileModel(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform } fn loadModelBuffers(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform, store: *zml.io.TensorStore, llama_model: llama.LlamaLM) !zml.Bufferized(llama.LlamaLM) { - var timer = try stdx.time.Timer.start(); + var timer = try std.time.Timer.start(); log.info("Loading model", .{}); defer log.info("Loaded model [{D}]", .{timer.read()}); return llama_model.loadBuffers(allocator, io, store.view(), platform); @@ -373,7 +376,7 @@ pub fn generateText( defer current_token_buffer.deinit(); const output_tokens_len = max_seq_len - prompt_tok.len - 1; - var timer = try stdx.time.Timer.start(); + var timer = try std.time.Timer.start(); // One token has alreadyh been generated by the prefill. var num_tokens_generated: usize = 1; @@ -413,10 +416,10 @@ pub fn generateText( try current_token_buffer.toSlice(io, generated_token_slice); } const duration = timer.read(); - const duration_s = stdx.math.divFloat(f64, duration.ns, std.time.ns_per_s); + const duration_s = stdx.math.divFloat(f64, duration, std.time.ns_per_s); const speed = @as(f64, @floatFromInt(num_tokens_generated)) / duration_s; std.debug.print("\n", .{}); - log.info("✅ Generated {} tokens in {D}: {:.3}tok/s", .{ num_tokens_generated, duration.ns, speed }); + log.info("✅ Generated {} tokens in {D}: {:.3}tok/s", .{ num_tokens_generated, duration, speed }); } fn bool_parser(in: []const u8) error{}!bool { diff --git a/zml2/tokenizer/sentencepiece/sentencepiece.i b/zml2/tokenizer/sentencepiece/sentencepiece.i index f2413dc3..35ccc057 100644 --- a/zml2/tokenizer/sentencepiece/sentencepiece.i +++ b/zml2/tokenizer/sentencepiece/sentencepiece.i @@ -81,6 +81,7 @@ namespace sentencepiece { class SentencePieceProcessor { public: virtual sentencepiece::util::Status Load(absl::string_view filename); + virtual sentencepiece::util::Status LoadFromSerializedProto(absl::string_view serialized); virtual sentencepiece::util::Status Encode(absl::string_view input, std::vector *ids) const; virtual sentencepiece::util::Status Decode(const std::vector &ids, std::string *detokenized) const; virtual int PieceToId(absl::string_view piece) const; diff --git a/zml2/tokenizer/sentencepiece/sentencepiece.zig b/zml2/tokenizer/sentencepiece/sentencepiece.zig index c3623cb5..c98b0740 100644 --- a/zml2/tokenizer/sentencepiece/sentencepiece.zig +++ b/zml2/tokenizer/sentencepiece/sentencepiece.zig @@ -173,6 +173,13 @@ pub const SentencePieceProcessor = opaque { return sp; } + pub fn fromBytes(bytes: []const u8) !*SentencePieceProcessor { + const sp: *SentencePieceProcessor = @ptrCast(c.SentencePieceProcessor_new()); + errdefer sp.deinit(); + try assertOk(c.SentencePieceProcessor_LoadFromSerializedProto(@ptrCast(sp), ffi.ZigSlice.from(bytes))); + return sp; + } + pub fn deinit(self: *SentencePieceProcessor) void { c.SentencePieceProcessor_delete(@ptrCast(self)); } diff --git a/zml2/tokenizer/tokenizer.zig b/zml2/tokenizer/tokenizer.zig index 1ba1d1d6..c72ec803 100644 --- a/zml2/tokenizer/tokenizer.zig +++ b/zml2/tokenizer/tokenizer.zig @@ -93,22 +93,27 @@ pub const Tokenizer = union(Tokenizers) { homemade: *homemade.Tokenizer, pub fn fromFile(allocator: std.mem.Allocator, io: std.Io, model: []const u8) !Tokenizer { + _ = allocator; // autofix + _ = io; // autofix if (std.mem.endsWith(u8, model, ".pb")) { - return .{ .sentencepiece = try sentencepiece.SentencePieceProcessor.fromFile(model) }; + return .{ .sentencepiece = try .fromFile(model) }; } if (std.mem.endsWith(u8, model, ".json")) { - return .{ .hftokenizers = try hftokenizers.HFTokenizer.fromFile(model) }; - } - - if (std.mem.endsWith(u8, model, ".tinyllama")) { - const tokenizer = try allocator.create(homemade.Tokenizer); - tokenizer.* = try homemade.fromTinyLlamaFile(allocator, io, model, 32000); - return .{ .homemade = tokenizer }; + return .{ .hftokenizers = try .fromFile(model) }; } return error.InvalidArgument; } + pub fn fromBytes(allocator: std.mem.Allocator, io: std.Io, bytes: []const u8) !Tokenizer { + _ = allocator; // autofix + _ = io; // autofix + if (bytes[0] == '{') { + return .{ .hftokenizers = try .fromBytes(bytes) }; + } + return .{ .sentencepiece = try .fromBytes(bytes) }; + } + pub fn deinit(self: *Tokenizer) void { switch (self.*) { inline else => |t| t.deinit(), From 7cdce4ffae3ef37e4acd70938112900a2cc8f6cf Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sat, 3 Jan 2026 10:36:44 +0000 Subject: [PATCH 178/881] zml/platform: new compile prototype --- examples/llama/main.zig | 4 ++-- examples/mnist/mnist.zig | 2 +- zml2/platform.zig | 18 +++++++++++++++++- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 8d06fc79..a369a3ba 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -255,14 +255,14 @@ fn compileModel(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform // Compile the model twice, one for prefill, one for generation. var prefill_future = io.async(struct { fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: zml.Platform, llama_model_: llama.LlamaLM, parameters_: LlamaParameters) !zml.Exe { - return platform_.compileModel(allocator_, io_, llama.LlamaLM.forward, llama_model_, .{ parameters_.prefill_tokens, parameters_.token_index, parameters_.kv_cache, parameters_.rng }); + return platform_.compile(allocator_, io_, llama_model_, .forward, .{ parameters_.prefill_tokens, parameters_.token_index, parameters_.kv_cache, parameters_.rng }); } }.call, .{ allocator, io, platform, llama_model, parameters }); errdefer if (prefill_future.cancel(io)) |v| v.deinit() else |_| {}; var decode_future = io.async(struct { fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: zml.Platform, llama_model_: llama.LlamaLM, parameters_: LlamaParameters) !zml.Exe { - return platform_.compileModel(allocator_, io_, llama.LlamaLM.forward, llama_model_, .{ parameters_.decode_tokens, parameters_.token_index, parameters_.kv_cache, parameters_.rng }); + return platform_.compile(allocator_, io_, llama_model_, .forward, .{ parameters_.decode_tokens, parameters_.token_index, parameters_.kv_cache, parameters_.rng }); } }.call, .{ allocator, io, platform, llama_model, parameters }); errdefer if (decode_future.cancel(io)) |v| v.deinit() else |_| {}; diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 0ad82c2d..af9d7d88 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -106,7 +106,7 @@ pub fn main() !void { log.info("Compiling model to MLIR....", .{}); var timer = try stdx.time.Timer.start(); const input: zml.Tensor = .init(.{ 28, 28 }, .u8); - var exe = try platform.compileModel(allocator, io, Mnist.forward, mnist_model, .{input}); + var exe = try platform.compile(allocator, io, mnist_model, .forward, .{input}); defer exe.deinit(); log.info("✅ Compiled model in {f}", .{timer.read()}); diff --git a/zml2/platform.zig b/zml2/platform.zig index df212333..6a5c7093 100644 --- a/zml2/platform.zig +++ b/zml2/platform.zig @@ -176,11 +176,27 @@ pub const Platform = struct { return device.memoryStats(platform.pjrt_api) catch .zeroes; } + pub fn compile( + self: Platform, + allocator: std.mem.Allocator, + io: std.Io, + model_: anytype, + comptime func: std.meta.DeclEnum(@TypeOf(model_)), + args: stdx.meta.Tail(stdx.meta.FnArgs(@field(@TypeOf(model_), @tagName(func)))), + ) !Exe { + return self.compileFn( + allocator, + io, + @field(@TypeOf(model_), @tagName(func)), + .{model_} ++ args, + ); + } + pub fn compileModel(self: Platform, allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, model: stdx.meta.Head(stdx.meta.FnArgs(func)), args: stdx.meta.Tail(stdx.meta.FnArgs(func))) !Exe { return self.compile(allocator, io, func, .{model} ++ args); } - pub fn compile(self: Platform, allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, args: stdx.meta.FnArgs(func)) !Exe { + pub fn compileFn(self: Platform, allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, args: stdx.meta.FnArgs(func)) !Exe { return zml.module.compile(allocator, io, func, args, self); } From 1ec373368b5d3c3b361b80766527e6c9d3685f00 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sat, 3 Jan 2026 10:43:26 +0000 Subject: [PATCH 179/881] examples/llama: remove stdx.time.Timer usage --- examples/llama/main.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index a369a3ba..4f210327 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -197,7 +197,7 @@ pub fn main() !void { } fn parseConfig(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !std.json.Parsed(LlamaLM.Config) { - var timer = try stdx.time.Timer.start(); + var timer = try std.time.Timer.start(); log.info("Loading model config", .{}); defer log.info("Loaded model config [{D}]", .{timer.read()}); From a854a29cb33c606c469843dfa2ab2104f1df450c Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sat, 3 Jan 2026 10:44:43 +0000 Subject: [PATCH 180/881] examples/llama: cosmetic changes --- examples/llama/main.zig | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 4f210327..cd515189 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -206,7 +206,7 @@ fn parseConfig(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !std.j defer config_json_file.close(io); var config_json_buffer: [256]u8 = undefined; var config_reader = config_json_file.reader(io, &config_json_buffer); - var reader = std.json.Reader.init(allocator, &config_reader.interface); + var reader: std.json.Reader = .init(allocator, &config_reader.interface); defer reader.deinit(); break :blk try std.json.parseFromTokenSource(llama.LlamaLM.Config, allocator, &reader, .{ .ignore_unknown_fields = true }); }; @@ -222,12 +222,8 @@ fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !zml const bytes = b: { const file = try dir.openFile(io, "tokenizer.json", .{}); defer file.close(io); - var reader = file.reader(io, &.{}); - var writer: std.Io.Writer.Allocating = .init(allocator); - defer writer.deinit(); - _ = try reader.interface.streamRemaining(&writer.writer); - break :b try writer.toOwnedSlice(); + break :b try reader.interface.readAlloc(allocator, try file.length(io)); }; defer allocator.free(bytes); From 7eaf650d0ef52e173a4964c8595d13f011b29405 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sat, 3 Jan 2026 11:04:45 +0000 Subject: [PATCH 181/881] zml: Platform.Device and iterator --- examples/llama/main.zig | 6 +++++ zml2/platform.zig | 58 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index cd515189..a2627fe4 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -96,6 +96,12 @@ pub fn main() !void { }; defer platform.deinit(); + log.info("{f}", .{platform.fmtVerbose()}); + var it = platform.devicesIterator(); + while (it.next()) |device| { + log.info(" - {f}", .{device}); + } + var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); defer registry.deinit(); diff --git a/zml2/platform.zig b/zml2/platform.zig index 6a5c7093..a5c664c1 100644 --- a/zml2/platform.zig +++ b/zml2/platform.zig @@ -55,6 +55,33 @@ pub const Platform = struct { pub const MAX_NUM_DEVICES: u8 = if (runtimes.isEnabled(.tpu)) 32 else 8; + pub const Device = struct { + pub const Iterator = struct { + api: *const pjrt.Api, + devices: []const *const pjrt.Device, + current: usize = 0, + + pub fn next(self: *Iterator) ?Device { + if (self.current >= self.devices.len) { + return null; + } + defer self.current += 1; + return .{ + .api = self.api, + .device = self.devices[self.current], + }; + } + }; + + api: *const pjrt.Api, + device: *const pjrt.Device, + + pub fn format(self: Device, writer: *std.Io.Writer) std.Io.Writer.Error!void { + const description = self.device.getDescription(self.api); + try writer.print("{s} ({s})", .{ description.getKind(self.api), description.debugString(self.api) }); + } + }; + pub fn init(target: Target, io: std.Io, options: CreateOptions) !Platform { const api = try loadOrGetApi(target, io); @@ -73,7 +100,13 @@ pub const Platform = struct { } pub fn auto(io: std.Io, options: CreateOptions) !Platform { - const ordered_targets = [_]Target{ .tpu, .neuron, .cuda, .rocm, .cpu }; + const ordered_targets = [_]Target{ + .tpu, + .neuron, + .cuda, + .rocm, + .cpu, + }; return for (ordered_targets) |target| { break init(target, io, options) catch continue; } else error.Unavailable; @@ -104,6 +137,27 @@ pub const Platform = struct { } } + pub fn devicesIterator(self: Platform) Device.Iterator { + return .{ + .api = self.pjrt_api, + .devices = self.getDevices(), + }; + } + + pub fn formatWithDevices(self: Platform, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("Platform: {s}\n", .{@tagName(self.target)}); + const devices = self.getDevices(); + try writer.print("Devices total={d}\n", .{devices.len}); + for (devices, 0..) |device, i| { + const description = device.getDescription(self.pjrt_api); + try writer.print("\t#{d}: {s}\n", .{ i, description.getKind(self.pjrt_api) }); + } + } + + pub fn fmtVerbose(self: Platform) std.fmt.Alt(Platform, formatWithDevices) { + return .{ .data = self }; + } + pub fn getDevices(self: Platform) []const *const pjrt.Device { const all_devices = self.pjrt_client.getAddressableDevices(self.pjrt_api); if (all_devices.len > MAX_NUM_DEVICES) { @@ -193,7 +247,7 @@ pub const Platform = struct { } pub fn compileModel(self: Platform, allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, model: stdx.meta.Head(stdx.meta.FnArgs(func)), args: stdx.meta.Tail(stdx.meta.FnArgs(func))) !Exe { - return self.compile(allocator, io, func, .{model} ++ args); + return self.compileFn(allocator, io, func, .{model} ++ args); } pub fn compileFn(self: Platform, allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, args: stdx.meta.FnArgs(func)) !Exe { From 799b69e2b7963e2dcb16e5fe7f52611a86a27c85 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sat, 3 Jan 2026 11:05:16 +0000 Subject: [PATCH 182/881] examples/llama: properly cancel tokenizer loading --- examples/llama/main.zig | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index a2627fe4..c8976b37 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -138,7 +138,10 @@ pub fn main() !void { }; var tokenizer_future = io.async(loadTokenizer, .{ allocator, io, repo.dir }); - errdefer _ = tokenizer_future.cancel(io) catch {}; + errdefer blk: { + var v = tokenizer_future.cancel(io) catch break :blk; + v.deinit(); + } var compiled_model_result_future = io.async(struct { fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: zml.Platform, llama_model_: llama.LlamaLM, llama_parameters_: LlamaParameters) !CompileModelResult { From 8e9a28fc2c0b496b4bf6a022528849189a9efbc7 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sun, 4 Jan 2026 19:37:42 +0000 Subject: [PATCH 183/881] stdx: mirror new std layout --- runtimes/cpu/cpu.zig | 12 ++++++------ runtimes/cuda/cuda.zig | 16 ++++++---------- runtimes/cuda/zmlxcuda.zig | 6 +++--- runtimes/neuron/libneuronxla.zig | 4 ++-- runtimes/neuron/libpjrt_neuron.zig | 10 +++++----- runtimes/neuron/neuron.zig | 4 ++-- runtimes/neuron/neuronx-cc.zig | 6 +++--- runtimes/rocm/rocm.zig | 6 +++--- runtimes/rocm/zmlxrocm.zig | 4 ++-- runtimes/tpu/tpu.zig | 4 ++-- stdx/BUILD.bazel | 3 ++- stdx/Io.zig | 15 +++++++++++++++ stdx/{fs.zig => process.zig} | 12 ------------ stdx/stdx.zig | 3 ++- zml2/io/vfs/hf.zig | 16 ++++++++-------- zml2/io/vfs/http.zig | 6 +++--- 16 files changed, 64 insertions(+), 63 deletions(-) create mode 100644 stdx/Io.zig rename stdx/{fs.zig => process.zig} (60%) diff --git a/runtimes/cpu/cpu.zig b/runtimes/cpu/cpu.zig index e5c05e68..35abd5bf 100644 --- a/runtimes/cpu/cpu.zig +++ b/runtimes/cpu/cpu.zig @@ -1,11 +1,11 @@ +const std = @import("std"); const builtin = @import("builtin"); +const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); -const bazel_builtin = @import("bazel_builtin"); -const std = @import("std"); -const stdx = @import("stdx"); const runfiles = @import("runfiles"); +const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/runtime/cpu"); @@ -28,7 +28,7 @@ pub fn load(io: std.Io) !*const pjrt.Api { const source_repo = bazel_builtin.current_repository; const r = r_.withSourceRepo(source_repo); - var path_buf: [std.fs.max_path_bytes]u8 = undefined; + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const sandbox_path = try r.rlocation("zml/runtimes/cpu/sandbox/lib", &path_buf) orelse { log.err("Failed to find sandbox path for CPU runtime", .{}); return error.FileNotFound; @@ -41,8 +41,8 @@ pub fn load(io: std.Io) !*const pjrt.Api { else => ".so", }; - var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; - const path = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "libpjrt_cpu" ++ ext }); + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "libpjrt_cpu" ++ ext }); var future = io.async(struct { fn call(path_: [:0]const u8) !*const pjrt.Api { return pjrt.Api.loadFrom(path_); diff --git a/runtimes/cuda/cuda.zig b/runtimes/cuda/cuda.zig index 57fff810..26796e37 100644 --- a/runtimes/cuda/cuda.zig +++ b/runtimes/cuda/cuda.zig @@ -56,7 +56,7 @@ pub fn load(io: std.Io) !*const pjrt.Api { const source_repo = bazel_builtin.current_repository; const r = r_.withSourceRepo(source_repo); - var path_buf: [std.fs.max_path_bytes]u8 = undefined; + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const sandbox_path = try r.rlocation("libpjrt_cuda/sandbox", &path_buf) orelse { log.err("Failed to find sandbox path for CUDA runtime", .{}); return error.FileNotFound; @@ -67,8 +67,8 @@ pub fn load(io: std.Io) !*const pjrt.Api { try setupXlaGpuCudaDirFlag(arena.allocator(), sandbox_path); { - var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; - const path = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libnvtx3interop.so" }); + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libnvtx3interop.so" }); _ = std.c.dlopen(path, .{ .NOW = true, .GLOBAL = true }) orelse { log.err("Unable to dlopen libnvtx3interop.so: {s}", .{std.c.dlerror().?}); return error.DlError; @@ -76,13 +76,9 @@ pub fn load(io: std.Io) !*const pjrt.Api { } return blk: { - var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; - const path = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_cuda.so" }); - var future = io.async(struct { - fn call(path_: [:0]const u8) !*const pjrt.Api { - return pjrt.Api.loadFrom(path_); - } - }.call, .{path}); + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_cuda.so" }); + var future = io.async(pjrt.Api.loadFrom, .{path}); break :blk future.await(io); }; } diff --git a/runtimes/cuda/zmlxcuda.zig b/runtimes/cuda/zmlxcuda.zig index 1ccd4baa..425cbbf9 100644 --- a/runtimes/cuda/zmlxcuda.zig +++ b/runtimes/cuda/zmlxcuda.zig @@ -15,11 +15,11 @@ pub export fn zmlxcuda_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque .{ "libnccl.so", "libnccl.so.2" }, }); - var buf: [std.fs.max_path_bytes]u8 = undefined; + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const new_filename: [*c]const u8 = if (filename) |f| blk: { const replacement = replacements.get(std.fs.path.basename(std.mem.span(f))) orelse break :blk f; - break :blk stdx.fs.path.bufJoinZ(&buf, &.{ - stdx.fs.selfSharedObjectDirPath(), + break :blk stdx.Io.Dir.path.bufJoinZ(&buf, &.{ + stdx.process.selfSharedObjectDirPath(), replacement, }) catch unreachable; } else null; diff --git a/runtimes/neuron/libneuronxla.zig b/runtimes/neuron/libneuronxla.zig index 59d8ae6b..e2e25878 100644 --- a/runtimes/neuron/libneuronxla.zig +++ b/runtimes/neuron/libneuronxla.zig @@ -169,7 +169,7 @@ fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) return error.UnknownPlatformVersion; }; - var tmp_dir_buf: [std.fs.max_path_bytes]u8 = undefined; + var tmp_dir_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const tmp_dir = try makeTempDir(&tmp_dir_buf, "zml-neuronxcc-"); defer std.fs.deleteTreeAbsolute(tmp_dir) catch |err| { log.err("Error deleting temporary directory {s}: {}\n", .{ tmp_dir, err }); @@ -184,7 +184,7 @@ fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) const neff_file = try std.fs.path.join(arena.allocator(), &.{ tmp_dir, "file.neff" }); - var neuronx_cc_buf: [std.fs.max_path_bytes]u8 = undefined; + var neuronx_cc_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; var child = std.process.Child.init(&.{ try stdx.fs.path.bufJoin(&neuronx_cc_buf, &.{ stdx.fs.selfSharedObjectDirPath(), diff --git a/runtimes/neuron/libpjrt_neuron.zig b/runtimes/neuron/libpjrt_neuron.zig index 9a13d5ed..33900bb8 100644 --- a/runtimes/neuron/libpjrt_neuron.zig +++ b/runtimes/neuron/libpjrt_neuron.zig @@ -28,7 +28,7 @@ pub export fn zmlxneuron_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaqu .{ "libncfw.so", "libncfw.so.2" }, }); - var buf: [std.fs.max_path_bytes]u8 = undefined; + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const new_filename: [*c]const u8 = if (filename) |f| blk: { const replacement = replacements.get(std.fs.path.basename(std.mem.span(f))) orelse break :blk f; break :blk stdx.fs.path.bufJoinZ(&buf, &.{ @@ -100,7 +100,7 @@ fn setupPythonEnv(sandbox_path: []const u8) !void { Static.py_config.write_bytecode = 0; { - var buf: [std.fs.max_path_bytes]u8 = undefined; + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const home = try std.fmt.bufPrintZ(&buf, "{f}{d}.{d}", .{ std.fs.path.fmtJoin(&.{ sandbox_path, @@ -115,7 +115,7 @@ fn setupPythonEnv(sandbox_path: []const u8) !void { } { - var buf: [std.fs.max_path_bytes]u8 = undefined; + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const site_packages = try stdx.fs.path.bufJoin(&buf, &.{ sandbox_path, "site-packages", @@ -146,7 +146,7 @@ fn getPjrtApi() !*c.PJRT_Api { var proxy: c.PJRT_Api = undefined; }; - var sandbox_path_buf: [std.fs.max_path_bytes]u8 = undefined; + var sandbox_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const sandbox_path = try stdx.fs.path.bufJoin(&sandbox_path_buf, &.{ stdx.fs.selfSharedObjectDirPath(), "..", @@ -160,7 +160,7 @@ fn getPjrtApi() !*c.PJRT_Api { var lib: std.DynLib = .{ .inner = .{ .handle = handle_blk: { - var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const library = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libneuronpjrt.so" }); break :handle_blk std.c.dlopen(library, .{ .LAZY = true, .NODELETE = true }) orelse { log.err("Unable to dlopen plugin: {?s}", .{std.mem.span(std.c.dlerror())}); diff --git a/runtimes/neuron/neuron.zig b/runtimes/neuron/neuron.zig index bc460288..3b7f4947 100644 --- a/runtimes/neuron/neuron.zig +++ b/runtimes/neuron/neuron.zig @@ -58,14 +58,14 @@ pub fn load(io: std.Io) !*const pjrt.Api { const source_repo = bazel_builtin.current_repository; const r = r_.withSourceRepo(source_repo); - var sandbox_path_buf: [std.fs.max_path_bytes]u8 = undefined; + var sandbox_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const sandbox_path = try r.rlocation("libpjrt_neuron/sandbox", &sandbox_path_buf) orelse { log.err("Failed to find sandbox path for NEURON runtime", .{}); return error.FileNotFound; }; return blk: { - var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const path = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_neuron.so" }); var future = io.async(struct { fn call(path_: [:0]const u8) !*const pjrt.Api { diff --git a/runtimes/neuron/neuronx-cc.zig b/runtimes/neuron/neuronx-cc.zig index 828a1d42..e39c1748 100644 --- a/runtimes/neuron/neuronx-cc.zig +++ b/runtimes/neuron/neuronx-cc.zig @@ -42,11 +42,11 @@ pub fn main() !void { _ = c.PyConfig_SetBytesArgv(&config, @intCast(std.os.argv.len), @ptrCast(std.os.argv)); - var self_exe_dir_buf: [std.fs.max_path_bytes]u8 = undefined; + var self_exe_dir_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const self_exe_dir = try std.fs.selfExeDirPath(&self_exe_dir_buf); { - var buf: [std.fs.max_path_bytes]u8 = undefined; + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const home = try std.fmt.bufPrintZ(&buf, "{f}{d}.{d}", .{ std.fs.path.fmtJoin(&.{ self_exe_dir, @@ -62,7 +62,7 @@ pub fn main() !void { } { - var buf: [std.fs.max_path_bytes]u8 = undefined; + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const site_packages = try std.fmt.bufPrint(&buf, "{f}", .{ std.fs.path.fmtJoin(&.{ self_exe_dir, diff --git a/runtimes/rocm/rocm.zig b/runtimes/rocm/rocm.zig index aa37350d..28af4fb8 100644 --- a/runtimes/rocm/rocm.zig +++ b/runtimes/rocm/rocm.zig @@ -23,7 +23,7 @@ fn hasRocmDevices(io: std.Io) bool { } fn setupRocmEnv(rocm_data_dir: []const u8) !void { - var buf: [std.fs.max_path_bytes]u8 = undefined; + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; _ = c.setenv("ROCM_PATH", try stdx.fs.path.bufJoinZ(&buf, &.{rocm_data_dir}), 1); // must be zero terminated } @@ -48,7 +48,7 @@ pub fn load(io: std.Io) !*const pjrt.Api { const source_repo = bazel_builtin.current_repository; const r = r_.withSourceRepo(source_repo); - var path_buf: [std.fs.max_path_bytes]u8 = undefined; + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const sandbox_path = try r.rlocation("libpjrt_rocm/sandbox", &path_buf) orelse { log.err("Failed to find sandbox path for ROCm runtime", .{}); return error.FileNotFound; @@ -56,7 +56,7 @@ pub fn load(io: std.Io) !*const pjrt.Api { try setupRocmEnv(sandbox_path); - var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const lib_path = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_rocm.so" }); // We must load the PJRT plugin from the main thread. diff --git a/runtimes/rocm/zmlxrocm.zig b/runtimes/rocm/zmlxrocm.zig index 30791f3a..eeac9826 100644 --- a/runtimes/rocm/zmlxrocm.zig +++ b/runtimes/rocm/zmlxrocm.zig @@ -22,7 +22,7 @@ pub export fn zmlxrocm_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque .{ "libhiprtc.so", "libhiprtc.so.7" }, }); - var buf: [std.fs.max_path_bytes]u8 = undefined; + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const new_filename: [*c]const u8 = if (filename) |f| blk: { const replacement = replacements.get(std.fs.path.basename(std.mem.span(f))) orelse break :blk f; break :blk stdx.fs.path.bufJoinZ(&buf, &.{ @@ -39,7 +39,7 @@ pub export fn zmlxrocm_fopen64(pathname: [*c]const u8, mode: [*c]const u8) ?*std .{ "/opt/amdgpu/share/libdrm/amdgpu.ids", "../share/libdrm/amdgpu.ids" }, }); - var buf: [std.fs.max_path_bytes]u8 = undefined; + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const new_pathname: [*c]const u8 = blk: { const replacement = replacements.get(std.mem.span(pathname)) orelse break :blk pathname; break :blk stdx.fs.path.bufJoinZ(&buf, &.{ diff --git a/runtimes/tpu/tpu.zig b/runtimes/tpu/tpu.zig index 1648c320..32b6b84e 100644 --- a/runtimes/tpu/tpu.zig +++ b/runtimes/tpu/tpu.zig @@ -53,14 +53,14 @@ pub fn load(io: std.Io) !*const pjrt.Api { const source_repo = bazel_builtin.current_repository; const r = r_.withSourceRepo(source_repo); - var path_buf: [std.fs.max_path_bytes]u8 = undefined; + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const sandbox_path = try r.rlocation("libpjrt_tpu/sandbox", &path_buf) orelse { log.err("Failed to find sandbox path for TPU runtime", .{}); return error.FileNotFound; }; return blk: { - var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const path = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_tpu.so" }); var future = io.async(struct { fn call(path_: [:0]const u8) !*const pjrt.Api { diff --git a/stdx/BUILD.bazel b/stdx/BUILD.bazel index 23c270e0..6d915882 100644 --- a/stdx/BUILD.bazel +++ b/stdx/BUILD.bazel @@ -3,14 +3,15 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "stdx", srcs = [ + "Io.zig", "bounded_array.zig", "debug.zig", "flags.zig", "fmt.zig", - "fs.zig", "json.zig", "math.zig", "meta.zig", + "process.zig", "queue.zig", "signature.zig", "time.zig", diff --git a/stdx/Io.zig b/stdx/Io.zig new file mode 100644 index 00000000..324b6c68 --- /dev/null +++ b/stdx/Io.zig @@ -0,0 +1,15 @@ +const std = @import("std"); + +pub const Dir = struct { + pub const path = struct { + pub fn bufJoin(buf: []u8, paths: []const []const u8) ![]u8 { + var fa: std.heap.FixedBufferAllocator = .init(buf); + return try std.Io.Dir.path.join(fa.allocator(), paths); + } + + pub fn bufJoinZ(buf: []u8, paths: []const []const u8) ![:0]u8 { + var fa: std.heap.FixedBufferAllocator = .init(buf); + return try std.Io.Dir.path.joinZ(fa.allocator(), paths); + } + }; +}; diff --git a/stdx/fs.zig b/stdx/process.zig similarity index 60% rename from stdx/fs.zig rename to stdx/process.zig index ea8a5fce..434a5bf7 100644 --- a/stdx/fs.zig +++ b/stdx/process.zig @@ -22,15 +22,3 @@ pub fn selfSharedObjectPath() []const u8 { pub fn selfSharedObjectDirPath() []const u8 { return std.fs.path.dirname(selfSharedObjectPathImpl(@returnAddress())).?; } - -pub const path = struct { - pub fn bufJoin(buf: []u8, paths: []const []const u8) ![]u8 { - var fa: std.heap.FixedBufferAllocator = .init(buf); - return try std.fs.path.join(fa.allocator(), paths); - } - - pub fn bufJoinZ(buf: []u8, paths: []const []const u8) ![:0]u8 { - var fa: std.heap.FixedBufferAllocator = .init(buf); - return try std.fs.path.joinZ(fa.allocator(), paths); - } -}; diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 3ec8b8bb..74959e62 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -6,10 +6,11 @@ pub const BoundedArrayAligned = @import("bounded_array.zig").BoundedArrayAligned pub const debug = @import("debug.zig"); pub const flags = @import("flags.zig"); pub const fmt = @import("fmt.zig"); -pub const fs = @import("fs.zig"); +pub const Io = @import("Io.zig"); pub const json = @import("json.zig"); pub const math = @import("math.zig"); pub const meta = @import("meta.zig"); +pub const process = @import("process.zig"); pub const queue = @import("queue.zig"); pub const time = @import("time.zig"); diff --git a/zml2/io/vfs/hf.zig b/zml2/io/vfs/hf.zig index c7746658..7cb488e8 100644 --- a/zml2/io/vfs/hf.zig +++ b/zml2/io/vfs/hf.zig @@ -1,5 +1,5 @@ -const std = @import("std"); const json = @import("std").json; +const std = @import("std"); const log = std.log.scoped(.@"zml/io/vfs/hf"); @@ -183,8 +183,8 @@ const HFApi = struct { break :blk "main"; }; - var redirect_buffer: [std.fs.max_path_bytes]u8 = undefined; - var url_buf: [std.fs.max_path_bytes]u8 = undefined; + var redirect_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; + var url_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const api_url = std.fmt.bufPrint( &url_buf, "https://huggingface.co/api/models/{s}/{s}/treesize/{s}/", @@ -296,8 +296,8 @@ const HFRecords = struct { ) Error!void { const allocator = self.arena.allocator(); - var url_buf: [std.fs.max_path_bytes]u8 = undefined; - var redirect_buffer: [std.fs.max_path_bytes]u8 = undefined; + var url_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + var redirect_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; const api_url = url orelse blk: { const model = try HFApi.modelFromUri(repo_uri); @@ -502,7 +502,7 @@ pub const HF = struct { uri: []const u8, pos: u64, size: u64, - redirect_buffer: [std.fs.max_path_bytes]u8 = undefined, + redirect_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined, request: ?*std.http.Client.Request = null, response: ?std.http.Client.Response = null, body_reader: ?*std.Io.Reader = null, @@ -638,7 +638,7 @@ pub const HF = struct { } const base_uri = std.Uri.parse(parent_path) catch return InitHandleError.PathUnresolvable; - var aux_buf: [std.fs.max_path_bytes]u8 = undefined; + var aux_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; if (sub_path.len > aux_buf.len) return InitHandleError.PathTooLong; @@ -998,7 +998,7 @@ pub const HF = struct { }; errdefer self.allocator.destroy(request); - var url_buf: [std.fs.max_path_bytes]u8 = undefined; + var url_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const path = HFApi.filePathFromUri(handle.uri) catch |err| { log.err("Failed to extract file path from URI {s}: {}", .{ handle.uri, err }); diff --git a/zml2/io/vfs/http.zig b/zml2/io/vfs/http.zig index 27df2698..e8168c9c 100644 --- a/zml2/io/vfs/http.zig +++ b/zml2/io/vfs/http.zig @@ -18,7 +18,7 @@ pub const HTTP = struct { uri: []const u8, pos: u64, size: u64, - redirect_buffer: [std.fs.max_path_bytes]u8 = undefined, + redirect_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined, request: ?*std.http.Client.Request = null, response: ?std.http.Client.Response = null, body_reader: ?*std.Io.Reader = null, @@ -136,7 +136,7 @@ pub const HTTP = struct { } const base_uri = std.Uri.parse(parent_path) catch return InitHandleError.PathUnresolvable; - var aux_buf: [std.fs.max_path_bytes]u8 = undefined; + var aux_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; if (sub_path.len > aux_buf.len) return InitHandleError.PathTooLong; @@ -530,7 +530,7 @@ pub const HTTP = struct { switch (response.head.status) { .moved_permanently, .found, .see_other, .temporary_redirect, .permanent_redirect => { if (response.head.location) |location| { - var buf: [std.fs.max_path_bytes]u8 = undefined; + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; if (location.len > buf.len) return std.Io.File.StatError.Unexpected; @memcpy(buf[0..location.len], location); From bfcdf27527b7f1578c57ff595955be337148178b Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 5 Jan 2026 09:07:18 +0000 Subject: [PATCH 184/881] stdx: add time.hzFloat --- stdx/time.zig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/stdx/time.zig b/stdx/time.zig index 4d2fde10..d724c0c2 100644 --- a/stdx/time.zig +++ b/stdx/time.zig @@ -11,6 +11,10 @@ pub const Duration = struct { return (1 * std.time.ns_per_s) / self.ns; } + pub fn hzFloat(self: Duration) f64 { + return (1 * std.time.ns_per_s) / @as(f64, @floatFromInt(self.ns)); + } + pub fn formatDuration(duration: Duration, writer: *std.Io.Writer) std.Io.Writer.Error!void { try writer.printDuration(duration.ns, .{}); } From 1b0242b2f1c6c6c325d940c96f647eb4471a16ef Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 5 Jan 2026 09:07:50 +0000 Subject: [PATCH 185/881] zml: bump Zig --- MODULE.bazel | 2 +- MODULE.bazel.lock | 88 ++++++++++----------- zml2/io/vfs/base.zig | 180 +++++++++++++++++++++---------------------- 3 files changed, 135 insertions(+), 135 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 3f5bb2c2..db3aa426 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -31,7 +31,7 @@ bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2", dev_dependency = Tr ### Zig toolchain zig = use_extension("@rules_zig//zig:extensions.bzl", "zig") zig.index(file = "//bazel:zig_index.json") -zig.toolchain(zig_version = "0.16.0-dev.1888+9d497d0d7") +zig.toolchain(zig_version = "0.16.0-dev.1912+0cbaaa5eb") zig.mirrors(urls = [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 87594272..378c9caa 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -2849,89 +2849,89 @@ "@@rules_zig+//zig:extensions.bzl%zig": { "general": { "bzlTransitiveDigest": "wV3euhBZSI0D2ZlqbY1YW592mghFY88d9H6zBMQRaJY=", - "usagesDigest": "1F/XDUJlOsUsE2Gup2fXDs6ZiU30P6wuDDvns9uNyxM=", + "usagesDigest": "MUggfXR9Y92m2LJD1coQy9DezZvA1s0RkFDA7YzrxBw=", "recordedFileInputs": { - "@@//bazel/zig_index.json": "f7ac8e30d820efe4ab7e6c0c770badc8aa18ff082b8d1a56f1ea0bfa1282bec5", + "@@//bazel/zig_index.json": "d38c409f2a4af04ad52d52f817ff71b77bb919194f5be6c34f827c68296d5795", "@@rules_zig+//zig/private/versions.json": "6eb85ebaee72c4e6fbc82d8ad73f69e0899d839b70a7e19caa792ab6a5fa0c36" }, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { - "zig_0.16.0-dev.1888_P9d497d0d7_aarch64-linux": { + "zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1888+9d497d0d7.tar.xz", + "url": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1912+0cbaaa5eb.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "26147ba4330bd01a06a9755f42445122be227dec4445117e450754968f4d613b", - "zig_version": "0.16.0-dev.1888+9d497d0d7", + "sha256": "de4eddf26f6f5e95d1bd055577cba7e9ebe9fbe8b1425bd4d40f073303ccc65b", + "zig_version": "0.16.0-dev.1912+0cbaaa5eb", "platform": "aarch64-linux" } }, - "zig_0.16.0-dev.1888_P9d497d0d7_aarch64-macos": { + "zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1888+9d497d0d7.tar.xz", + "url": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1912+0cbaaa5eb.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "f1e092e51cd026a600f68c53ad610e12ecad0cfdc9aac71ffc86f21afe3db5e5", - "zig_version": "0.16.0-dev.1888+9d497d0d7", + "sha256": "241fa43a37803a9e50e7f4a2d2c436882ee2215e6c420c454cb4de57adae84d3", + "zig_version": "0.16.0-dev.1912+0cbaaa5eb", "platform": "aarch64-macos" } }, - "zig_0.16.0-dev.1888_P9d497d0d7_aarch64-windows": { + "zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1888+9d497d0d7.zip", + "url": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1912+0cbaaa5eb.zip", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "a7c248888bdf545ef6ce96e0585dc1313ce19ca5dd7fee59d6a93c392a1fa8fb", - "zig_version": "0.16.0-dev.1888+9d497d0d7", + "sha256": "824044c2dd76d4fdf9dd3d3fc7e8ed9bd39a1a03a3eb3acf1fecde25cee640cc", + "zig_version": "0.16.0-dev.1912+0cbaaa5eb", "platform": "aarch64-windows" } }, - "zig_0.16.0-dev.1888_P9d497d0d7_x86_64-linux": { + "zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1888+9d497d0d7.tar.xz", + "url": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1912+0cbaaa5eb.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "51e17ce99f3e10a21a56a7bfe163a98736495bd10cbc974c6d4648748ac710c4", - "zig_version": "0.16.0-dev.1888+9d497d0d7", + "sha256": "d0bfbe4c6e6871a36ad7c9be577df2abda37c69af437fedb6865d8729d60cbdc", + "zig_version": "0.16.0-dev.1912+0cbaaa5eb", "platform": "x86_64-linux" } }, - "zig_0.16.0-dev.1888_P9d497d0d7_x86_64-macos": { + "zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1888+9d497d0d7.tar.xz", + "url": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1912+0cbaaa5eb.tar.xz", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "247d6e0fe0bdbbc2433737caba10688594ca62469d3e782c62ef7257dad0b167", - "zig_version": "0.16.0-dev.1888+9d497d0d7", + "sha256": "9bfa7e5237c5e5b132229e3686eb849ca2afb23f7b6f6b72cbae0e5e8766aade", + "zig_version": "0.16.0-dev.1912+0cbaaa5eb", "platform": "x86_64-macos" } }, - "zig_0.16.0-dev.1888_P9d497d0d7_x86_64-windows": { + "zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1888+9d497d0d7.zip", + "url": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1912+0cbaaa5eb.zip", "mirrors": [ "https://mirror.zml.ai/zig", "https://ziglang.org/builds/" ], - "sha256": "3dc6a275d9ef7ae622b128bf971201bada1f81b46f16f9d2771b8dd058d7db7e", - "zig_version": "0.16.0-dev.1888+9d497d0d7", + "sha256": "719859787d0be77a8dc19f5ecc37d8cc7802ddca06f616d81e9ebde34e610dd4", + "zig_version": "0.16.0-dev.1912+0cbaaa5eb", "platform": "x86_64-windows" } }, @@ -2939,28 +2939,28 @@ "repoRuleId": "@@rules_zig+//zig/private/repo:toolchains_repo.bzl%toolchains_repo", "attributes": { "names": [ - "zig_0.16.0-dev.1888_P9d497d0d7_aarch64-linux", - "zig_0.16.0-dev.1888_P9d497d0d7_aarch64-macos", - "zig_0.16.0-dev.1888_P9d497d0d7_aarch64-windows", - "zig_0.16.0-dev.1888_P9d497d0d7_x86_64-linux", - "zig_0.16.0-dev.1888_P9d497d0d7_x86_64-macos", - "zig_0.16.0-dev.1888_P9d497d0d7_x86_64-windows" + "zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-linux", + "zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-macos", + "zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-windows", + "zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-linux", + "zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-macos", + "zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-windows" ], "labels": [ - "@zig_0.16.0-dev.1888_P9d497d0d7_aarch64-linux//:zig_toolchain", - "@zig_0.16.0-dev.1888_P9d497d0d7_aarch64-macos//:zig_toolchain", - "@zig_0.16.0-dev.1888_P9d497d0d7_aarch64-windows//:zig_toolchain", - "@zig_0.16.0-dev.1888_P9d497d0d7_x86_64-linux//:zig_toolchain", - "@zig_0.16.0-dev.1888_P9d497d0d7_x86_64-macos//:zig_toolchain", - "@zig_0.16.0-dev.1888_P9d497d0d7_x86_64-windows//:zig_toolchain" + "@zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-linux//:zig_toolchain", + "@zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-macos//:zig_toolchain", + "@zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-windows//:zig_toolchain", + "@zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-linux//:zig_toolchain", + "@zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-macos//:zig_toolchain", + "@zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-windows//:zig_toolchain" ], "zig_versions": [ - "0.16.0-dev.1888+9d497d0d7", - "0.16.0-dev.1888+9d497d0d7", - "0.16.0-dev.1888+9d497d0d7", - "0.16.0-dev.1888+9d497d0d7", - "0.16.0-dev.1888+9d497d0d7", - "0.16.0-dev.1888+9d497d0d7" + "0.16.0-dev.1912+0cbaaa5eb", + "0.16.0-dev.1912+0cbaaa5eb", + "0.16.0-dev.1912+0cbaaa5eb", + "0.16.0-dev.1912+0cbaaa5eb", + "0.16.0-dev.1912+0cbaaa5eb", + "0.16.0-dev.1912+0cbaaa5eb" ], "exec_lengths": [ 2, diff --git a/zml2/io/vfs/base.zig b/zml2/io/vfs/base.zig index ae8204b8..d2d8f30e 100644 --- a/zml2/io/vfs/base.zig +++ b/zml2/io/vfs/base.zig @@ -149,441 +149,441 @@ pub const VFSBase = struct { } pub fn as(userdata: ?*anyopaque) *VFSBase { - return @ptrCast(@alignCast(userdata orelse unreachable)); + return @ptrCast(@alignCast(userdata.?)); } pub fn async(userdata: ?*anyopaque, result: []u8, result_alignment: std.mem.Alignment, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque, result: *anyopaque) void) ?*std.Io.AnyFuture { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.async(self.inner.userdata, result, result_alignment, context, context_alignment, start); } pub fn concurrent(userdata: ?*anyopaque, result_len: usize, result_alignment: std.mem.Alignment, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque, result: *anyopaque) void) std.Io.ConcurrentError!*std.Io.AnyFuture { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.concurrent(self.inner.userdata, result_len, result_alignment, context, context_alignment, start); } pub fn await(userdata: ?*anyopaque, any_future: *std.Io.AnyFuture, result: []u8, result_alignment: std.mem.Alignment) void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.await(self.inner.userdata, any_future, result, result_alignment); } pub fn cancel(userdata: ?*anyopaque, any_future: *std.Io.AnyFuture, result: []u8, result_alignment: std.mem.Alignment) void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.cancel(self.inner.userdata, any_future, result, result_alignment); } - pub fn groupAsync(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (*std.Io.Group, context: *const anyopaque) std.Io.Cancelable!void) void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + pub fn groupAsync(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque) std.Io.Cancelable!void) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.groupAsync(self.inner.userdata, group, context, context_alignment, start); } - pub fn groupConcurrent(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (*std.Io.Group, context: *const anyopaque) std.Io.Cancelable!void) std.Io.ConcurrentError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + pub fn groupConcurrent(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque) std.Io.Cancelable!void) std.Io.ConcurrentError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.groupConcurrent(self.inner.userdata, group, context, context_alignment, start); } pub fn groupAwait(userdata: ?*anyopaque, group: *std.Io.Group, token: *anyopaque) std.Io.Cancelable!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.groupAwait(self.inner.userdata, group, token); } pub fn groupCancel(userdata: ?*anyopaque, group: *std.Io.Group, token: *anyopaque) void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.groupCancel(self.inner.userdata, group, token); } pub fn recancel(userdata: ?*anyopaque) void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.recancel(self.inner.userdata); } pub fn swapCancelProtection(userdata: ?*anyopaque, new: std.Io.CancelProtection) std.Io.CancelProtection { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.swapCancelProtection(self.inner.userdata, new); } pub fn checkCancel(userdata: ?*anyopaque) std.Io.Cancelable!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.checkCancel(self.inner.userdata); } pub fn select(userdata: ?*anyopaque, futures: []const *std.Io.AnyFuture) std.Io.Cancelable!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.select(self.inner.userdata, futures); } pub fn futexWait(userdata: ?*anyopaque, ptr: *const u32, expected: u32, timeout: std.Io.Timeout) std.Io.Cancelable!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.futexWait(self.inner.userdata, ptr, expected, timeout); } pub fn futexWaitUncancelable(userdata: ?*anyopaque, ptr: *const u32, expected: u32) void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.futexWaitUncancelable(self.inner.userdata, ptr, expected); } pub fn futexWake(userdata: ?*anyopaque, ptr: *const u32, max_waiters: u32) void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.futexWake(self.inner.userdata, ptr, max_waiters); } pub fn dirCreateDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions) std.Io.Dir.CreateDirError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirCreateDir(self.inner.userdata, dir, sub_path, permissions); } pub fn dirCreateDirPath(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions) std.Io.Dir.CreateDirPathError!std.Io.Dir.CreatePathStatus { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirCreateDirPath(self.inner.userdata, dir, sub_path, permissions); } pub fn dirCreateDirPathOpen(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions, options: std.Io.Dir.OpenOptions) std.Io.Dir.CreateDirPathOpenError!std.Io.Dir { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirCreateDirPathOpen(self.inner.userdata, dir, sub_path, permissions, options); } pub fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirOpenDir(self.inner.userdata, dir, sub_path, options); } pub fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirStat(self.inner.userdata, dir); } pub fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirStatFile(self.inner.userdata, dir, sub_path, options); } pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirAccess(self.inner.userdata, dir, sub_path, options); } pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.CreateFlags) std.Io.File.OpenError!std.Io.File { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirCreateFile(self.inner.userdata, dir, sub_path, flags); } pub fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirOpenFile(self.inner.userdata, dir, sub_path, flags); } pub fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); self.inner.vtable.dirClose(self.inner.userdata, dirs); } pub fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirRead(self.inner.userdata, reader, entries); } pub fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirRealPath(self.inner.userdata, dir, out_buffer); } pub fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirRealPathFile(self.inner.userdata, dir, path_name, out_buffer); } pub fn dirDeleteFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8) std.Io.Dir.DeleteFileError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirDeleteFile(self.inner.userdata, dir, sub_path); } pub fn dirDeleteDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8) std.Io.Dir.DeleteDirError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirDeleteDir(self.inner.userdata, dir, sub_path); } pub fn dirRename(userdata: ?*anyopaque, old_dir: std.Io.Dir, old_sub_path: []const u8, new_dir: std.Io.Dir, new_sub_path: []const u8) std.Io.Dir.RenameError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirRename(self.inner.userdata, old_dir, old_sub_path, new_dir, new_sub_path); } pub fn dirSymLink(userdata: ?*anyopaque, dir: std.Io.Dir, target_path: []const u8, sym_link_path: []const u8, flags: std.Io.Dir.SymLinkFlags) std.Io.Dir.SymLinkError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirSymLink(self.inner.userdata, dir, target_path, sym_link_path, flags); } pub fn dirReadLink(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, buffer: []u8) std.Io.Dir.ReadLinkError!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirReadLink(self.inner.userdata, dir, sub_path, buffer); } pub fn dirSetOwner(userdata: ?*anyopaque, dir: std.Io.Dir, uid: ?std.Io.File.Uid, gid: ?std.Io.File.Gid) std.Io.Dir.SetOwnerError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirSetOwner(self.inner.userdata, dir, uid, gid); } pub fn dirSetFileOwner(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, uid: ?std.Io.File.Uid, gid: ?std.Io.File.Gid, options: std.Io.Dir.SetFileOwnerOptions) std.Io.Dir.SetFileOwnerError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirSetFileOwner(self.inner.userdata, dir, sub_path, uid, gid, options); } pub fn dirSetPermissions(userdata: ?*anyopaque, dir: std.Io.Dir, permissions: std.Io.Dir.Permissions) std.Io.Dir.SetPermissionsError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirSetPermissions(self.inner.userdata, dir, permissions); } pub fn dirSetFilePermissions(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.File.Permissions, options: std.Io.Dir.SetFilePermissionsOptions) std.Io.Dir.SetFilePermissionsError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirSetFilePermissions(self.inner.userdata, dir, sub_path, permissions, options); } pub fn dirSetTimestamps(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.SetTimestampsOptions) std.Io.Dir.SetTimestampsError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirSetTimestamps(self.inner.userdata, dir, sub_path, options); } pub fn dirHardLink(userdata: ?*anyopaque, old_dir: std.Io.Dir, old_sub_path: []const u8, new_dir: std.Io.Dir, new_sub_path: []const u8, options: std.Io.Dir.HardLinkOptions) std.Io.Dir.HardLinkError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirHardLink(self.inner.userdata, old_dir, old_sub_path, new_dir, new_sub_path, options); } pub fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileStat(self.inner.userdata, file); } pub fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileLength(self.inner.userdata, file); } pub fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); self.inner.vtable.fileClose(self.inner.userdata, files); } pub fn fileWriteStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize) std.Io.File.Writer.Error!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileWriteStreaming(self.inner.userdata, file, header, data, splat); } pub fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) std.Io.File.WritePositionalError!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileWritePositional(self.inner.userdata, file, header, data, splat, offset); } pub fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileWriteFileStreaming(self.inner.userdata, file, header, reader, limit); } pub fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileWriteFilePositional(self.inner.userdata, file, header, reader, limit, offset); } pub fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileReadStreaming(self.inner.userdata, file, data); } pub fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileReadPositional(self.inner.userdata, file, data, offset); } pub fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileSeekBy(self.inner.userdata, file, relative_offset); } pub fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileSeekTo(self.inner.userdata, file, absolute_offset); } pub fn fileSync(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.SyncError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileSync(self.inner.userdata, file); } pub fn fileIsTty(userdata: ?*anyopaque, file: std.Io.File) std.Io.Cancelable!bool { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileIsTty(self.inner.userdata, file); } pub fn fileEnableAnsiEscapeCodes(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.EnableAnsiEscapeCodesError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileEnableAnsiEscapeCodes(self.inner.userdata, file); } pub fn fileSupportsAnsiEscapeCodes(userdata: ?*anyopaque, file: std.Io.File) std.Io.Cancelable!bool { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileSupportsAnsiEscapeCodes(self.inner.userdata, file); } pub fn fileSetLength(userdata: ?*anyopaque, file: std.Io.File, length: u64) std.Io.File.SetLengthError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileSetLength(self.inner.userdata, file, length); } pub fn fileSetOwner(userdata: ?*anyopaque, file: std.Io.File, uid: ?std.Io.File.Uid, gid: ?std.Io.File.Gid) std.Io.File.SetOwnerError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileSetOwner(self.inner.userdata, file, uid, gid); } pub fn fileSetPermissions(userdata: ?*anyopaque, file: std.Io.File, permissions: std.Io.File.Permissions) std.Io.File.SetPermissionsError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileSetPermissions(self.inner.userdata, file, permissions); } pub fn fileSetTimestamps(userdata: ?*anyopaque, file: std.Io.File, options: std.Io.File.SetTimestampsOptions) std.Io.File.SetTimestampsError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileSetTimestamps(self.inner.userdata, file, options); } pub fn fileLock(userdata: ?*anyopaque, file: std.Io.File, lock: std.Io.File.Lock) std.Io.File.LockError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileLock(self.inner.userdata, file, lock); } pub fn fileTryLock(userdata: ?*anyopaque, file: std.Io.File, lock: std.Io.File.Lock) std.Io.File.LockError!bool { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileTryLock(self.inner.userdata, file, lock); } pub fn fileUnlock(userdata: ?*anyopaque, file: std.Io.File) void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileUnlock(self.inner.userdata, file); } pub fn fileDowngradeLock(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.DowngradeLockError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileDowngradeLock(self.inner.userdata, file); } pub fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileRealPath(self.inner.userdata, file, out_buffer); } pub fn processExecutableOpen(userdata: ?*anyopaque, flags: std.Io.File.OpenFlags) std.process.OpenExecutableError!std.Io.File { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.processExecutableOpen(self.inner.userdata, flags); } pub fn processExecutablePath(userdata: ?*anyopaque, buffer: []u8) std.process.ExecutablePathError!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.processExecutablePath(self.inner.userdata, buffer); } pub fn lockStderr(userdata: ?*anyopaque, mode: ?std.Io.Terminal.Mode) std.Io.Cancelable!std.Io.LockedStderr { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.lockStderr(self.inner.userdata, mode); } pub fn tryLockStderr(userdata: ?*anyopaque, mode: ?std.Io.Terminal.Mode) std.Io.Cancelable!?std.Io.LockedStderr { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.tryLockStderr(self.inner.userdata, mode); } pub fn unlockStderr(userdata: ?*anyopaque) void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.unlockStderr(self.inner.userdata); } pub fn processSetCurrentDir(userdata: ?*anyopaque, dir: std.Io.Dir) std.process.SetCurrentDirError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.processSetCurrentDir(self.inner.userdata, dir); } pub fn now(userdata: ?*anyopaque, clock: std.Io.Clock) std.Io.Clock.Error!std.Io.Timestamp { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.now(self.inner.userdata, clock); } pub fn sleep(userdata: ?*anyopaque, timeout: std.Io.Timeout) std.Io.SleepError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.sleep(self.inner.userdata, timeout); } pub fn netListenIp(userdata: ?*anyopaque, address: std.Io.net.IpAddress, options: std.Io.net.IpAddress.ListenOptions) std.Io.net.IpAddress.ListenError!std.Io.net.Server { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netListenIp(self.inner.userdata, address, options); } pub fn netAccept(userdata: ?*anyopaque, server: std.Io.net.Socket.Handle) std.Io.net.Server.AcceptError!std.Io.net.Stream { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netAccept(self.inner.userdata, server); } pub fn netBindIp(userdata: ?*anyopaque, address: *const std.Io.net.IpAddress, options: std.Io.net.IpAddress.BindOptions) std.Io.net.IpAddress.BindError!std.Io.net.Socket { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netBindIp(self.inner.userdata, address, options); } pub fn netConnectIp(userdata: ?*anyopaque, address: *const std.Io.net.IpAddress, options: std.Io.net.IpAddress.ConnectOptions) std.Io.net.IpAddress.ConnectError!std.Io.net.Stream { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netConnectIp(self.inner.userdata, address, options); } pub fn netListenUnix(userdata: ?*anyopaque, address: *const std.Io.net.UnixAddress, options: std.Io.net.UnixAddress.ListenOptions) std.Io.net.UnixAddress.ListenError!std.Io.net.Socket.Handle { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netListenUnix(self.inner.userdata, address, options); } pub fn netConnectUnix(userdata: ?*anyopaque, address: *const std.Io.net.UnixAddress) std.Io.net.UnixAddress.ConnectError!std.Io.net.Socket.Handle { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netConnectUnix(self.inner.userdata, address); } pub fn netSend(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, msgs: []std.Io.net.OutgoingMessage, flags: std.Io.net.SendFlags) struct { ?std.Io.net.Socket.SendError, usize } { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netSend(self.inner.userdata, handle, msgs, flags); } pub fn netReceive(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, message_buffer: []std.Io.net.IncomingMessage, data_buffer: []u8, flags: std.Io.net.ReceiveFlags, timeout: std.Io.Timeout) struct { ?std.Io.net.Socket.ReceiveTimeoutError, usize } { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netReceive(self.inner.userdata, handle, message_buffer, data_buffer, flags, timeout); } pub fn netRead(userdata: ?*anyopaque, src: std.Io.net.Socket.Handle, data: [][]u8) std.Io.net.Stream.Reader.Error!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netRead(self.inner.userdata, src, data); } pub fn netWrite(userdata: ?*anyopaque, dest: std.Io.net.Socket.Handle, header: []const u8, data: []const []const u8, splat: usize) std.Io.net.Stream.Writer.Error!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netWrite(self.inner.userdata, dest, header, data, splat); } pub fn netWriteFile(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.net.Stream.Writer.WriteFileError!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netWriteFile(self.inner.userdata, handle, header, reader, limit); } pub fn netClose(userdata: ?*anyopaque, handles: []const std.Io.net.Socket.Handle) void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); self.inner.vtable.netClose(self.inner.userdata, handles); } pub fn netShutdown(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, how: std.Io.net.ShutdownHow) std.Io.net.ShutdownError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netShutdown(self.inner.userdata, handle, how); } pub fn netInterfaceNameResolve(userdata: ?*anyopaque, name: *const std.Io.net.Interface.Name) std.Io.net.Interface.Name.ResolveError!std.Io.net.Interface { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netInterfaceNameResolve(self.inner.userdata, name); } pub fn netInterfaceName(userdata: ?*anyopaque, iface: std.Io.net.Interface) std.Io.net.Interface.NameError!std.Io.net.Interface.Name { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netInterfaceName(self.inner.userdata, iface); } pub fn netLookup(userdata: ?*anyopaque, host: std.Io.net.HostName, q: *std.Io.Queue(std.Io.net.HostName.LookupResult), options: std.Io.net.HostName.LookupOptions) std.Io.net.HostName.LookupError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata orelse unreachable)); + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netLookup(self.inner.userdata, host, q, options); } }; From 03e71446923aaa03b0944157a30568fd36631783 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 5 Jan 2026 09:08:27 +0000 Subject: [PATCH 186/881] zml: cosmetic logging changes --- zml2/module.zig | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/zml2/module.zig b/zml2/module.zig index 0659d923..525cfbd6 100644 --- a/zml2/module.zig +++ b/zml2/module.zig @@ -6,6 +6,7 @@ const mlir = @import("mlir"); const stdx = @import("stdx"); const upb = @import("upb"); +const Buffer = @import("buffer.zig").Buffer; const Exe = @import("exe.zig").Exe; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); @@ -13,7 +14,6 @@ const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const Tensor = @import("tensor.zig").Tensor; -const Buffer = @import("buffer.zig").Buffer; const log = std.log.scoped(.@"zml/module"); @@ -186,8 +186,7 @@ pub fn compile(allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), io, platform, compilation_context.module, null) catch unreachable; - log.debug("******** ZML generated MLIR ********", .{}); - log.debug("{f}", .{compilation_context.module.operation()}); + log.debug("\n******** ZML generated MLIR ********\n{f}", .{compilation_context.module.operation()}); const num_devices = sharding.num_partitions * sharding.num_replicas; const exe = try Exe.init(allocator, platform, loaded_executable, result.input_info.shapes, result.output_info.shapes, num_devices); From 3934392360e59cd23904014c06483205d8cef31c Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 5 Jan 2026 09:08:43 +0000 Subject: [PATCH 187/881] runtimes/cuda: disable XLA logging --- runtimes/cuda/cuda.zig | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/runtimes/cuda/cuda.zig b/runtimes/cuda/cuda.zig index 26796e37..4a255557 100644 --- a/runtimes/cuda/cuda.zig +++ b/runtimes/cuda/cuda.zig @@ -25,6 +25,10 @@ fn hasCudaPathInLDPath() bool { return std.ascii.indexOfIgnoreCase(std.mem.span(ldLibraryPath), nvidiaLibsPath) != null; } +pub fn disableXlaLogs() void { + _ = c.setenv("TF_CPP_MIN_LOG_LEVEL", "3", 1); +} + fn setupXlaGpuCudaDirFlag(allocator: std.mem.Allocator, sandbox: []const u8) !void { const xla_flags = std.process.getEnvVarOwned(allocator, "XLA_FLAGS") catch ""; const new_xla_flagsZ = try std.fmt.allocPrintSentinel(allocator, "{s} --xla_gpu_cuda_data_dir={s}", .{ xla_flags, sandbox }, 0); @@ -66,6 +70,8 @@ pub fn load(io: std.Io) !*const pjrt.Api { // See https://github.com/openxla/xla/issues/21428 try setupXlaGpuCudaDirFlag(arena.allocator(), sandbox_path); + disableXlaLogs(); + { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libnvtx3interop.so" }); From df0961525654397c8d482dd2dbe0a341a614f93c Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 5 Jan 2026 09:09:04 +0000 Subject: [PATCH 188/881] zig bump --- bazel/zig_index.json | 154 +++++++++++++++++++++---------------------- 1 file changed, 77 insertions(+), 77 deletions(-) diff --git a/bazel/zig_index.json b/bazel/zig_index.json index d6f49100..4720c0e1 100644 --- a/bazel/zig_index.json +++ b/bazel/zig_index.json @@ -1,133 +1,133 @@ { "master": { - "version": "0.16.0-dev.1888+9d497d0d7", - "date": "2025-12-31", + "version": "0.16.0-dev.1912+0cbaaa5eb", + "date": "2026-01-04", "docs": "https://ziglang.org/documentation/master/", "stdDocs": "https://ziglang.org/documentation/master/std/", "src": { - "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.1888+9d497d0d7.tar.xz", - "shasum": "5a496e6563059129da0faf07299bf1df2a58d9024b15080a3f7a090fd59ab5f3", - "size": "21858752" + "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "shasum": "4730e30cf16fb088b706bf2834cd973a54a1ff06d56cba2dc845b28408d7e8bd", + "size": "21866640" }, "bootstrap": { - "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.1888+9d497d0d7.tar.xz", - "shasum": "0d2be93accccb4b28991d46a0a10a73b2b2627a897e49b4dff0220aadb1ba577", - "size": "54621884" + "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "shasum": "e928729be6613ba78dc0dc01773380148a8b1e94a4d13b0e0b7a36bfbaa3823f", + "size": "54630236" }, "x86_64-macos": { - "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1888+9d497d0d7.tar.xz", - "shasum": "247d6e0fe0bdbbc2433737caba10688594ca62469d3e782c62ef7257dad0b167", - "size": "57003700" + "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "shasum": "9bfa7e5237c5e5b132229e3686eb849ca2afb23f7b6f6b72cbae0e5e8766aade", + "size": "57011520" }, "aarch64-macos": { - "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1888+9d497d0d7.tar.xz", - "shasum": "f1e092e51cd026a600f68c53ad610e12ecad0cfdc9aac71ffc86f21afe3db5e5", - "size": "51844052" + "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "shasum": "241fa43a37803a9e50e7f4a2d2c436882ee2215e6c420c454cb4de57adae84d3", + "size": "51849528" }, "x86_64-linux": { - "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1888+9d497d0d7.tar.xz", - "shasum": "51e17ce99f3e10a21a56a7bfe163a98736495bd10cbc974c6d4648748ac710c4", - "size": "54905484" + "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "shasum": "d0bfbe4c6e6871a36ad7c9be577df2abda37c69af437fedb6865d8729d60cbdc", + "size": "54905880" }, "aarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1888+9d497d0d7.tar.xz", - "shasum": "26147ba4330bd01a06a9755f42445122be227dec4445117e450754968f4d613b", - "size": "50654628" + "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "shasum": "de4eddf26f6f5e95d1bd055577cba7e9ebe9fbe8b1425bd4d40f073303ccc65b", + "size": "50663184" }, "arm-linux": { - "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.1888+9d497d0d7.tar.xz", - "shasum": "269d22e85c097ec7627815366f4148344b5bba7d893006578c8608a09de17ffc", - "size": "51544652" + "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "shasum": "d3bc2a7973a89e58018244be6020ab5594ac50b31075f62b191bcb59a4394161", + "size": "51546152" }, "riscv64-linux": { - "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.1888+9d497d0d7.tar.xz", - "shasum": "63db2a0c2af03a6028857ae403ccb21fd24ac9ab3d21f81bf7f44e4c3e855c14", - "size": "54712804" + "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "shasum": "4b492fc3b85fc71c9f89ecdce9b1a40cc74d3c5460fd1aeacca15cfcf75e39f2", + "size": "54724024" }, "powerpc64le-linux": { - "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.1888+9d497d0d7.tar.xz", - "shasum": "a05cd2af0703978e12216e7af36bacfd94d6a56a0a1f89fb77420de35f423515", - "size": "54723848" + "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "shasum": "f9d8a4b1747042cdd8845ddf6c89f44efdb3c8d09306321b86efa1241d340bad", + "size": "54736740" }, "x86-linux": { - "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.1888+9d497d0d7.tar.xz", - "shasum": "12b45ec048bc07b9d83c7536a4c2a828f90f4494a78251c58e0c1982637adbb0", - "size": "57602100" + "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "shasum": "4b47620480328f5f82925022ace2c98e0b13d0730defcd3af74c82e9da1c2b2d", + "size": "57621640" }, "loongarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.1888+9d497d0d7.tar.xz", - "shasum": "a4eb9adc0e8a5ec0cce5e2082d8b3b0e6a87eda49f7272702377838ecd8c859e", - "size": "51952008" + "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "shasum": "897d10a9f0cd6b8ab8454adc5c0fda9dfcecc65df97316400bf71396f78d7fe9", + "size": "51956176" }, "s390x-linux": { - "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.1888+9d497d0d7.tar.xz", - "shasum": "d7d3d1cc379cfc070a0b6f0f6161b2c584870cc659928037e3f315bec2626951", - "size": "54397912" + "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "shasum": "07501ac4c472d652d8f6128cc4026a68fd461c08a8a2006bc6fe66a9a8f70415", + "size": "54434044" }, "x86_64-windows": { - "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1888+9d497d0d7.zip", - "shasum": "3dc6a275d9ef7ae622b128bf971201bada1f81b46f16f9d2771b8dd058d7db7e", - "size": "94973867" + "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1912+0cbaaa5eb.zip", + "shasum": "719859787d0be77a8dc19f5ecc37d8cc7802ddca06f616d81e9ebde34e610dd4", + "size": "94983652" }, "aarch64-windows": { - "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1888+9d497d0d7.zip", - "shasum": "a7c248888bdf545ef6ce96e0585dc1313ce19ca5dd7fee59d6a93c392a1fa8fb", - "size": "90931027" + "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1912+0cbaaa5eb.zip", + "shasum": "824044c2dd76d4fdf9dd3d3fc7e8ed9bd39a1a03a3eb3acf1fecde25cee640cc", + "size": "90939398" }, "x86-windows": { - "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.1888+9d497d0d7.zip", - "shasum": "33bb0222efff1df27f24cb8afa0cf24c6e1dfaa43c537cd73a4c25a351cb8dd7", - "size": "96811069" + "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.1912+0cbaaa5eb.zip", + "shasum": "aa8048f1ddc6c5dbd14d2d04665d2d9b82b6c8fc07d61813607a16e371e98149", + "size": "96824007" }, "aarch64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.1888+9d497d0d7.tar.xz", - "shasum": "9de0a0c4f14c435225827ef4b70569841b97cb934487b176ea929ea3eae497bf", - "size": "50602712" + "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "shasum": "e5d6ad13df1b22d0a9706c9df248890dcbf864a42daf1a307634816523f0134d", + "size": "50596788" }, "arm-freebsd": { - "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.1888+9d497d0d7.tar.xz", - "shasum": "0f81ff46f9b94bab09d2eb38a12f0ef1818577f3b4ce5da0e4d2eb593da7b50d", - "size": "52207800" + "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "shasum": "903745269c221596c4d9c59d727b89810f95498c86f015fd1627f30d00d830f7", + "size": "52216472" }, "powerpc64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-powerpc64-freebsd-0.16.0-dev.1888+9d497d0d7.tar.xz", - "shasum": "347bc5af3d520427fc9976c3bd8a213d5a2dc6d175d243823714c48d80d604ba", - "size": "53293836" + "tarball": "https://ziglang.org/builds/zig-powerpc64-freebsd-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "shasum": "de9f318029687e616118a44e4d8b3da5d3d3ffb05a22fc6a14f0d90f32b13b3b", + "size": "53314532" }, "powerpc64le-freebsd": { - "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.1888+9d497d0d7.tar.xz", - "shasum": "73d7a142e0acb93dd90fd67bd6b1602b970bd40b12007470f45dd42ea82cf797", - "size": "54741496" + "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "shasum": "8e2ec39e098f83de0408d40f58b201f8734e07f3763a3c6048b3ba76b8ca68ab", + "size": "54750456" }, "riscv64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.1888+9d497d0d7.tar.xz", - "shasum": "8ba301c25212133614b440eef1d9f21337adf0f93c676c87ee6a996e011966e3", - "size": "54847220" + "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "shasum": "b4aa67fc5b08f68dca234ba671934512765421e46a84500c57b239a9c3e1b308", + "size": "54857948" }, "x86_64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.1888+9d497d0d7.tar.xz", - "shasum": "afaf144768b88ee25c9a0b88897a81a35daf603990b5abcaecc36677da382209", - "size": "55013484" + "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "shasum": "8dc6475b7b9b68c0322513b727da1f168d4e3d8b29f215cdf0e236e7650fa383", + "size": "55013180" }, "aarch64-netbsd": { - "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.1888+9d497d0d7.tar.xz", - "shasum": "b72aff3c96ab11283b3d9a37d1741adeb7a6a910b7949912c0b68da5475a85ab", - "size": "50543896" + "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "shasum": "9ecfc5b13dc620ccdb37452b59f8123d723db848d9e038413514a43715403f5f", + "size": "50554668" }, "arm-netbsd": { - "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.1888+9d497d0d7.tar.xz", - "shasum": "2cae2f1daf158eac60887ac2c0c7e1c6f92746ca17a85a951ef91103a02f9b8b", - "size": "53245616" + "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "shasum": "66e0ef652963906fccd9b2e4ff0af4d9d6968093c25dd1fff4b3de1f56e29f0e", + "size": "53254492" }, "x86-netbsd": { - "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.1888+9d497d0d7.tar.xz", - "shasum": "457a413d2aa1e1f789b8caa3f375e7c04cf09adc5eee1a2577452d85fbced86a", - "size": "58200316" + "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "shasum": "a2f092bfe2c34fa0193263fa3df18318886fda437767f1cc2a2d589baa65f6da", + "size": "58215316" }, "x86_64-netbsd": { - "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.1888+9d497d0d7.tar.xz", - "shasum": "73ac63473a6fb06b4b3dc23fdc0fbd7367eabac5225b78238ce3458519ecdf8b", - "size": "54969392" + "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "shasum": "612f5e287a482b32eeb62e16f44ef41ce64b6ba5a9cc296f88a06e41014c3783", + "size": "54954084" } }, "0.15.2": { From 749a22c881d28ecaa7f2c00137295a51200378aa Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 5 Jan 2026 09:09:17 +0000 Subject: [PATCH 189/881] examples/benchmark: update --- examples/benchmark/main.zig | 72 +++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 34 deletions(-) diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index fe095a52..d678551e 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -1,12 +1,8 @@ const std = @import("std"); -const zml = @import("zml"); -const stdx = @import("stdx"); -const flags = stdx.flags; +const log = std.log; -// set log level to debug to print the generated IR -pub const std_options: std.Options = .{ - .log_level = .warn, -}; +const stdx = @import("stdx"); +const zml = @import("zml"); pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { return a.dot(b, .k); @@ -21,13 +17,9 @@ pub fn main() !void { dtype: zml.DataType = .f16, }; - // Short lived allocations - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); + const allocator = std.heap.smp_allocator; - const allocator = gpa.allocator(); - - var threaded: std.Io.Threaded = .init(allocator); + var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); const io = threaded.io(); @@ -38,25 +30,31 @@ pub fn main() !void { // Auto-select platform const platform: zml.Platform = try .auto(io, .{}); - var args = std.process.args(); - const cli_args = flags.parse(&args, CliArgs); + const cli_args: CliArgs = blk: { + var ret: CliArgs = .{}; + var it = std.process.args(); + defer it.deinit(); + while (it.next()) |arg| { + if (std.mem.startsWith(u8, arg, "--dtype=")) { + ret.dtype = std.meta.stringToEnum(zml.DataType, arg["--dtype=".len..]) orelse unreachable; + } else if (std.mem.startsWith(u8, arg, "--size=")) { + ret.size = try std.fmt.parseUnsigned(usize, arg["--size=".len..], 10); + } + } + break :blk ret; + }; - const a = zml.Tensor.init(.{ cli_args.size, cli_args.size }, cli_args.dtype).withTags(.{ .m, .k }); - const b = zml.Tensor.init(.{ cli_args.size, cli_args.size }, cli_args.dtype).withTags(.{ .k, .n }); - var timer = try std.time.Timer.start(); + const a: zml.Tensor = .init(.{ .m = cli_args.size, .k = cli_args.size }, cli_args.dtype); + const b: zml.Tensor = .init(.{ .k = cli_args.size, .n = cli_args.size }, cli_args.dtype); - std.debug.print("\nCompiling model to MLIR....\n", .{}); - std.debug.print("-" ** 160 ++ "\n", .{}); - - timer.reset(); - - var exe = try platform.compile(allocator, io, benchmark, .{ a, b }); + var exe = blk: { + log.info("⏱️ Compiling benchmark...", .{}); + var timer = try std.time.Timer.start(); + defer log.info("✅ Compiled benchmark [{D}]", .{timer.read()}); + break :blk try platform.compileFn(allocator, io, benchmark, .{ a, b }); + }; defer exe.deinit(); - const compilation_elapsed = timer.lap() / std.time.ns_per_ms; - std.debug.print("-" ** 160 ++ "\n\n", .{}); - std.debug.print("✅ Compiled Benchmark model in {d} milliseconds! \n", .{compilation_elapsed}); - var rng = std.Random.DefaultPrng.init(0); const random = rng.random(); @@ -73,7 +71,7 @@ pub fn main() !void { exe_args.set(.{ a_buffer, b_buffer }); - std.debug.print("\nRunning benchmark....\n", .{}); + log.info("⏱️ Running benchmark...", .{}); // Ignore first run { @@ -83,19 +81,25 @@ pub fn main() !void { } // call our executable module - timer.reset(); + var timer = try std.time.Timer.start(); exe.call(exe_args, &exe_results); var result = exe_results.get(zml.Buffer); + _ = try result.await(io); defer result.deinit(); - const elapsed_ns = timer.lap(); - const elapsed_ms = @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_ms; + const elapsed_ns = timer.read(); const elapsed_s = @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_s; - std.debug.print("\n✅ Benchmark done!\n\n", .{}); + log.info("✅ Benchmark done!", .{}); const floating_op_count = 2 * cli_args.size * cli_args.size * cli_args.size; const flops = @as(f64, @floatFromInt(floating_op_count)) / elapsed_s; - std.debug.print("Dot product size: {d}x{d} - Datatype: {s} - Elapsed: {d:.3}ms - {d:.3} GFLOP/s\n\n", .{ cli_args.size, cli_args.size, @tagName(cli_args.dtype), elapsed_ms, flops / 1_000_000_000 }); + log.info("Dot product size: {d}x{d} - Datatype: {s} - Elapsed: {D} - {d:.3} GFLOP/s", .{ + cli_args.size, + cli_args.size, + @tagName(cli_args.dtype), + elapsed_ns, + flops / 1_000_000_000, + }); } fn createRandomBuffer(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform, shape: zml.Shape, random: std.Random) !zml.Buffer { From 541a0b2f41f33338242348bbc1c2ee0e0f856d4c Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 5 Jan 2026 09:09:34 +0000 Subject: [PATCH 190/881] examples/llama: remove clap, take prompt from stdin --- examples/llama/BUILD.bazel | 43 ----------- examples/llama/main.zig | 141 ++++++++++++++----------------------- 2 files changed, 53 insertions(+), 131 deletions(-) diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel index ad8d914e..bb087644 100644 --- a/examples/llama/BUILD.bazel +++ b/examples/llama/BUILD.bazel @@ -14,52 +14,9 @@ zig_binary( deps = [ "//stdx", "//zml2:zml", - "@com_github_hejsil_clap//:clap", ], ) -# TODO(steeve): Re-enable with a small, tokenless huggingface model. -# -# zig_test( -# name = "test-implementation", -# srcs = ["llama.zig"], -# args = [ -# "--weights=$(location @Meta-Llama-3.1-8B-Instruct//:model.safetensors.index.json)", -# "--config=$(location @Meta-Llama-3.1-8B-Instruct//:config.json)", -# ], -# data = [ -# "@Meta-Llama-3.1-8B-Instruct//:config.json", -# "@Meta-Llama-3.1-8B-Instruct//:model.safetensors.index.json", -# ], -# main = "test.zig", -# deps = [ -# "//async", -# "//stdx", -# "//zml", -# ], -# ) - -# native_test( -# name = "test_tokenizer", -# src = "//zml/tokenizer:main", -# # Note: all Llama-3.x tokenizers are the same, -# # but using the 3.2-1B version because downloading the tokenizer triggers downloading the model. -# args = [ -# "--tokenizer=$(location @Meta-Llama-3.2-1B-Instruct//:tokenizer.json)", -# """--prompt='Examples of titles: -# 📉 Stock Market Trends -# 🍪 Perfect Chocolate Chip Recipe -# Evolution of Music Streaming -# Remote Work Productivity Tips -# Artificial Intelligence in Healthcare -# 🎮 Video Game Development Insights -# '""", -# # this correspond to encoding with HF tokenizers, with bos=False -# "--expected=41481,315,15671,512,9468,241,231,12937,8152,50730,198,9468,235,103,24118,39520,32013,26371,198,35212,3294,315,10948,45910,198,25732,5664,5761,1968,26788,198,9470,16895,22107,304,39435,198,9468,236,106,8519,4140,11050,73137,198", -# ], -# data = ["@Meta-Llama-3.2-1B-Instruct//:tokenizer.json"], -# ) - mtree_spec( name = "mtree", srcs = [":llama"], diff --git a/examples/llama/main.zig b/examples/llama/main.zig index c8976b37..4d457c72 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -1,6 +1,5 @@ const std = @import("std"); -const clap = @import("clap"); const stdx = @import("stdx"); const zml = @import("zml"); const Buffer = zml.Buffer; @@ -20,16 +19,10 @@ pub const std_options: std.Options = .{ .log_level = .info, }; -const params = clap.parseParamsComptime( - \\--help print this help - \\--prompt the prompt - \\--hf-model-path path to the directory containing model weights, config and tokenizer - \\--seed random seed (optional) - \\--seq-len sequence length - \\--create-options platform creation options JSON, defaults to {} - \\--no-llama3 skip prompt template - \\--sharding default: true: sharding on or off -); +pub const Args = struct { + model: ?[]const u8 = null, + seqlen: u32 = 512, +}; pub fn main() !void { log.info("LLama was compiled with {}", .{@import("builtin").mode}); @@ -44,64 +37,34 @@ pub fn main() !void { zml.init(); defer zml.deinit(); - const parsers = comptime .{ - .BOOL = bool_parser, - .UINT = clap.parsers.int(u32, 0), - .STRING = clap.parsers.string, - .PATH = clap.parsers.string, - }; - var diag: clap.Diagnostic = .{}; - var stderr_buffer: [1024]u8 = undefined; - var stderr = std.Io.File.stderr().writer(io, &stderr_buffer); - defer stderr.interface.flush() catch {}; + const args: Args = blk: { + var ret: Args = .{}; + var it = std.process.args(); + defer it.deinit(); + while (it.next()) |arg| { + if (std.mem.startsWith(u8, arg, "--model=")) { + ret.model = arg["--model=".len..]; + } else if (std.mem.startsWith(u8, arg, "--seqlen=")) { + ret.seqlen = try std.fmt.parseUnsigned(u32, arg["--seqlen=".len..], 10); + } + } - var cli = clap.parse(clap.Help, ¶ms, parsers, .{ - .diagnostic = &diag, - .allocator = allocator, - }) catch |err| { - diag.report(&stderr.interface, err) catch {}; - stderr.interface.writeAll("usage: ") catch {}; - clap.usage(&stderr.interface, clap.Help, ¶ms) catch {}; - stderr.interface.writeAll("\n") catch {}; - return; - }; - defer cli.deinit(); + if (ret.model == null) { + log.err("Missing --model", .{}); + return; + } - if (cli.args.help != 0) { - clap.help(&stderr.interface, clap.Help, ¶ms, .{}) catch {}; - return; - } - - const hf_model_path = cli.args.@"hf-model-path" orelse { - log.err("Missing --hf-model-path", .{}); - return; + break :blk ret; }; log.info("Resolving Model repo", .{}); - var repo = try zml.safetensors.resolveModelRepo(allocator, io, hf_model_path); + var repo = try zml.safetensors.resolveModelRepo(allocator, io, args.model.?); defer repo.deinit(allocator, io); const parsed_config = try parseConfig(allocator, io, repo.dir); defer parsed_config.deinit(); const config = parsed_config.value; - // initialize ZML platform with optional create options - // eg: --create-options='{"cuda":{"allocator":{"bfc":{"memory_fraction": 0.99}}}}' - var platform = b: { - const create_opts_json = cli.args.@"create-options" orelse "{}"; - const create_opts = try std.json.parseFromSlice(zml.platform.CreateOptions, allocator, create_opts_json, .{}); - defer create_opts.deinit(); - const platform: zml.Platform = try .auto(io, create_opts.value); - break :b platform; - }; - defer platform.deinit(); - - log.info("{f}", .{platform.fmtVerbose()}); - var it = platform.devicesIterator(); - while (it.next()) |device| { - log.info(" - {f}", .{device}); - } - var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); defer registry.deinit(); @@ -109,9 +72,8 @@ pub fn main() !void { defer store.deinit(); // Write metadata from the config file into the LlamaLm struct. - const seq_len: u32 = cli.args.@"seq-len" orelse 256; const llama_options: llama.LlamaLM.Options = .{ - .max_seq_len = seq_len, + .max_seq_len = args.seqlen, .sampling_strategy = .{ .topk = 2, .temperature = 1.0, @@ -128,9 +90,9 @@ pub fn main() !void { .prefill_tokens = .init(.{ .s = llama_options.max_seq_len }, .u32), .decode_tokens = .init(.{ .s = 1 }, .u32), .token_index = .init(.{}, .u32), - .kv_cache = .init(zml.Shape.init(.{ + .kv_cache = .init(.init(.{ .layer = llama_model.model.layers.len, - .k = seq_len, + .k = args.seqlen, .h = config.num_key_value_heads, .hd = config.head_dim orelse @divExact(config.hidden_size, config.num_attention_heads), }, dtype)), @@ -143,21 +105,21 @@ pub fn main() !void { v.deinit(); } - var compiled_model_result_future = io.async(struct { - fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: zml.Platform, llama_model_: llama.LlamaLM, llama_parameters_: LlamaParameters) !CompileModelResult { - return compileModel(allocator_, io_, platform_, llama_model_, llama_parameters_); - } - }.call, .{ allocator, io, platform, llama_model, llama_parameters }); + var platform: zml.Platform = try .auto(io, .{}); + defer platform.deinit(); + var it = platform.devicesIterator(); + log.info("Devices:", .{}); + while (it.next()) |device| { + log.info("\t- {f}", .{device}); + } + + var compiled_model_result_future = io.async(compileModel, .{ allocator, io, platform, llama_model, llama_parameters }); errdefer if (compiled_model_result_future.cancel(io)) |v| { defer v.prefill_exe.deinit(); defer v.decode_exe.deinit(); } else |_| {}; - var llama_buffers_future = io.async(struct { - fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: zml.Platform, store_: *zml.io.TensorStore, llama_model_: llama.LlamaLM) !zml.Bufferized(llama.LlamaLM) { - return loadModelBuffers(allocator_, io_, platform_, store_, llama_model_); - } - }.call, .{ allocator, io, platform, &store, llama_model }); + var llama_buffers_future = io.async(loadModelBuffers, .{ allocator, io, platform, &store, llama_model }); errdefer b: { var v = llama_buffers_future.cancel(io) catch break :b; LlamaLM.unloadBuffers(&v, allocator); @@ -177,16 +139,18 @@ pub fn main() !void { var tokenizer = try tokenizer_future.await(io); defer tokenizer.deinit(); - const prompt = cli.args.prompt orelse "What is the capital of France?"; + const prompt = blk: { + var reader = std.Io.File.stdin().reader(io, &.{}); + break :blk try reader.interface.allocRemaining(allocator, .unlimited); + }; + defer allocator.free(prompt); + + // const prompt = cli.args.prompt orelse "What is the capital of France?"; log.info("✅\tPrompt: {s}", .{prompt}); // Unbuffered writing of the tokens to stdout. var stdout = std.Io.File.stdout().writer(io, &.{}); - const now = try std.Io.Clock.boot.now(io); - const seed: u128 = cli.args.seed orelse @intCast(now.nanoseconds); - const skip_llama3_encoding = cli.args.@"no-llama3" orelse false; - try generateText( allocator, io, @@ -197,9 +161,9 @@ pub fn main() !void { tokenizer, config, llama_options, - seed, + @intCast((try std.Io.Clock.now(.real, io)).toNanoseconds()), prompt[0..], - skip_llama3_encoding, + false, &stdout.interface, platform, ); @@ -225,8 +189,8 @@ fn parseConfig(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !std.j } fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !zml.tokenizer.Tokenizer { - var timer = try std.time.Timer.start(); log.info("Loading tokenizer", .{}); + var timer = try std.time.Timer.start(); defer log.info("Loaded tokenizer [{D}]", .{timer.read()}); const bytes = b: { const file = try dir.openFile(io, "tokenizer.json", .{}); @@ -260,6 +224,9 @@ fn compileModel(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform // Compile the model twice, one for prefill, one for generation. var prefill_future = io.async(struct { fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: zml.Platform, llama_model_: llama.LlamaLM, parameters_: LlamaParameters) !zml.Exe { + var timer_ = try std.time.Timer.start(); + log.info("Compiling prefill", .{}); + defer log.info("Compiled prefill [{D}]", .{timer_.read()}); return platform_.compile(allocator_, io_, llama_model_, .forward, .{ parameters_.prefill_tokens, parameters_.token_index, parameters_.kv_cache, parameters_.rng }); } }.call, .{ allocator, io, platform, llama_model, parameters }); @@ -267,6 +234,9 @@ fn compileModel(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform var decode_future = io.async(struct { fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: zml.Platform, llama_model_: llama.LlamaLM, parameters_: LlamaParameters) !zml.Exe { + var timer_ = try std.time.Timer.start(); + log.info("Compiling decode", .{}); + defer log.info("Compiled decode [{D}]", .{timer_.read()}); return platform_.compile(allocator_, io_, llama_model_, .forward, .{ parameters_.decode_tokens, parameters_.token_index, parameters_.kv_cache, parameters_.rng }); } }.call, .{ allocator, io, platform, llama_model, parameters }); @@ -381,7 +351,7 @@ pub fn generateText( defer current_token_buffer.deinit(); const output_tokens_len = max_seq_len - prompt_tok.len - 1; - var timer = try std.time.Timer.start(); + var timer = try stdx.time.Timer.start(); // One token has alreadyh been generated by the prefill. var num_tokens_generated: usize = 1; @@ -392,6 +362,7 @@ pub fn generateText( const generated_token = generated_token_slice.items(u32)[0]; if (try tokenizer_decoder.next(generated_token)) |chunk| { try writer.writeAll(chunk); + try writer.flush(); } // check for eos @@ -421,12 +392,6 @@ pub fn generateText( try current_token_buffer.toSlice(io, generated_token_slice); } const duration = timer.read(); - const duration_s = stdx.math.divFloat(f64, duration, std.time.ns_per_s); - const speed = @as(f64, @floatFromInt(num_tokens_generated)) / duration_s; std.debug.print("\n", .{}); - log.info("✅ Generated {} tokens in {D}: {:.3}tok/s", .{ num_tokens_generated, duration, speed }); -} - -fn bool_parser(in: []const u8) error{}!bool { - return std.mem.indexOfScalar(u8, "tTyY1", in[0]) != null; + log.info("✅ Generated {} tokens in {D}: {:.3}tok/s", .{ num_tokens_generated, duration, duration.div(num_tokens_generated).hzFloat() }); } From 48eddd98b6a5909e314bdd2b41e402e70b5be38f Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 5 Jan 2026 09:09:43 +0000 Subject: [PATCH 191/881] examples/mnist: cosmetic changes --- examples/mnist/mnist.zig | 52 +++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index af9d7d88..0e81a952 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -84,9 +84,6 @@ pub fn main() !void { zml.init(); defer zml.deinit(); - // Auto-select platform - const platform: zml.Platform = try .auto(io, .{}); - // Parse program args const process_args = try std.process.argsAlloc(allocator); defer std.process.argsFree(allocator, process_args); @@ -100,16 +97,29 @@ pub fn main() !void { // Init model var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); defer store.deinit(); - const mnist_model = Mnist.init(store.view()); + const mnist_model: Mnist = .init(store.view()); + + // Auto-select platform + const platform: zml.Platform = try .auto(io, .{}); // Compile model - log.info("Compiling model to MLIR....", .{}); - var timer = try stdx.time.Timer.start(); const input: zml.Tensor = .init(.{ 28, 28 }, .u8); - var exe = try platform.compile(allocator, io, mnist_model, .forward, .{input}); + var exe = blk: { + log.info("Compiling model....", .{}); + var timer = try std.time.Timer.start(); + defer log.info("✅ Compiled model in {D}", .{timer.read()}); + break :blk try platform.compile(allocator, io, mnist_model, .forward, .{input}); + }; defer exe.deinit(); - log.info("✅ Compiled model in {f}", .{timer.read()}); + // Load buffers + var mnist_buffers = blk: { + log.info("Transfering weights....", .{}); + var timer = try std.time.Timer.start(); + defer log.info("✅ Transferred weights in {D}", .{timer.read()}); + break :blk try mnist_model.loadBuffers(allocator, io, store.view(), platform); + }; + defer Mnist.unloadBuffers(&mnist_buffers); var args = try exe.args(allocator); defer args.deinit(allocator); @@ -117,36 +127,28 @@ pub fn main() !void { var results = try exe.results(allocator); defer results.deinit(allocator); - // Load buffers - var mnist_buffers = try mnist_model.loadBuffers(allocator, io, store.view(), platform); - defer Mnist.unloadBuffers(&mnist_buffers); - log.info("✅ Weights transferred in {f}", .{timer.read()}); - - log.info("Starting inference...", .{}); - // Load a random digit image from the dataset. const dataset = try std.Io.Dir.openFile(.cwd(), io, t10kfilename, .{ .mode = .read_only }); defer dataset.close(io); - const now = std.Io.Clock.now(.awake, io) catch unreachable; - var rng = std.Random.Xoshiro256.init(@intCast(now.toMilliseconds())); + var rng: std.Random.DefaultPrng = blk: { + const now = try std.Io.Clock.now(.awake, io); + break :blk .init(@intCast(now.toMilliseconds())); + }; // inference - can be looped - { - const idx = rng.random().intRangeAtMost(u64, 0, 10000 - 1); + while (true) { + const idx = rng.random().uintLessThan(u64, 10000); var sample: [28 * 28]u8 align(16) = undefined; - var reader = dataset.reader(io, &.{}); - try reader.seekTo(16 + (idx * 28 * 28)); - _ = try reader.interface.readSliceShort(&sample); + _ = try dataset.readPositionalAll(io, &sample, 16 + (idx * 28 * 28)); var input_buffer: zml.Buffer = try .fromSlice(io, platform, zml.Slice.init(input.shape(), &sample)); defer input_buffer.deinit(); - args.set(.{ mnist_buffers, input_buffer }); - printDigit(sample); - exe.call(args, &results); + args.set(.{ mnist_buffers, input_buffer }); + exe.call(args, &results); var result: zml.Buffer = results.get(zml.Buffer); defer result.deinit(); From bdce67fb496576e166d569b8dd8de8f05e29925b Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 5 Jan 2026 11:46:36 +0000 Subject: [PATCH 192/881] zml/io/vfs: hf2 working again --- zml2/io/vfs/hf2.zig | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/zml2/io/vfs/hf2.zig b/zml2/io/vfs/hf2.zig index c231eca7..1092ab98 100644 --- a/zml2/io/vfs/hf2.zig +++ b/zml2/io/vfs/hf2.zig @@ -45,7 +45,7 @@ pub const HF = struct { .repo = parts.first(), .model = parts.next() orelse return error.InvalidURI, .rev = "main", - .path = parts.rest(), + .path = std.mem.trimEnd(u8, parts.rest(), "/"), }; if (std.mem.findScalar(u8, repo.model, '@')) |at_index| { repo.rev = repo.model[at_index + 1 ..]; @@ -88,7 +88,6 @@ pub const HF = struct { base: VFSBase, authorization: std.http.Client.Request.Headers.Value = .default, - token_len: usize = 0, allocator: std.mem.Allocator, client: *std.http.Client, handles: std.ArrayList(Handle) = .{}, @@ -100,7 +99,7 @@ pub const HF = struct { .base = .init(io_), .client = http_client_, .authorization = .{ - .override = "Bearer XXX", + .override = "Bearer xxxx", }, }; } @@ -208,7 +207,7 @@ pub const HF = struct { for (data) |buf| { total_bytes += buf.len; } - break :blk std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ offset, offset + total_bytes }) catch unreachable; + break :blk std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ offset, @min(handle.size - offset, offset + total_bytes) }) catch unreachable; }; var req = self.client.request(.GET, handle.uri, .{ @@ -230,7 +229,23 @@ pub const HF = struct { return std.Io.File.Reader.Error.Unexpected; }; + const range = blk: { + var it = resp.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { + break :blk parseContentRange(header.value); + } + } else break :blk .{ 0, 0, 0 }; + }; + + log.info(">>>>> {any}", .{range}); + const reader = resp.reader(&.{}); + + if (range.@"0" < offset) { + reader.discardAll(offset - range.@"0") catch return std.Io.File.Reader.Error.Unexpected; + } + return reader.readSliceShort(data[0]) catch return std.Io.File.Reader.Error.Unexpected; } @@ -282,12 +297,20 @@ pub fn main() !void { log.info(">>>>>>> {any}", .{file}); - const buffer = try allocator.alloc(u8, 128 * 1024 * 1024); var reader = file.reader(io, &.{}); + reader.mode = reader.mode.toSimple(); defer log.info("DONE", .{}); - const stdout: std.Io.File = .stdout(); - var stdout_writer = stdout.writer(io, buffer); + // const data = try reader.interface.readAlloc(allocator, 1024); + // defer allocator.free(data); + // log.info("READ {s}", .{data}); - _ = try reader.interface.stream(&stdout_writer.interface, .unlimited); + const buffer = try allocator.alloc(u8, 256 * 1024 * 1024); + var stdout_writer = std.Io.File.stdout().writer(io, buffer); + defer stdout_writer.interface.flush() catch {}; + + _ = reader.interface.streamRemaining(&stdout_writer.interface) catch |err| { + log.err("Failed to stream HF file: {any}", .{err}); + unreachable; + }; } From 0f1d49daf49fb58fd323087c0b80a18cf1ee5eff Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Mon, 5 Jan 2026 16:01:43 +0000 Subject: [PATCH 193/881] Fix ZLS --- third_party/zls/workspace_printer.zig | 2 +- third_party/zls/zls_build_runner.zig | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/third_party/zls/workspace_printer.zig b/third_party/zls/workspace_printer.zig index 7fe077d3..d55785ac 100644 --- a/third_party/zls/workspace_printer.zig +++ b/third_party/zls/workspace_printer.zig @@ -101,7 +101,7 @@ pub fn main_016() !void { }; var write_buffer: [8192]u8 = undefined; - var writer = std.fs.File.stdout().writer(&write_buffer); + var writer = std.Io.File.stdout().writer(io, &write_buffer); _ = try replacer.interface.streamRemaining(&writer.interface); try writer.interface.flush(); } diff --git a/third_party/zls/zls_build_runner.zig b/third_party/zls/zls_build_runner.zig index 675f61be..a2ce8dad 100644 --- a/third_party/zls/zls_build_runner.zig +++ b/third_party/zls/zls_build_runner.zig @@ -6,6 +6,11 @@ pub fn main() !void { defer arena_.deinit(); const arena = arena_.allocator(); + var threaded: std.Io.Threaded = .init(std.heap.smp_allocator, .{}); + defer threaded.deinit(); + + const io = threaded.io(); + const build_workspace_directory = try std.process.getEnvVarOwned(arena, "BUILD_WORKSPACE_DIRECTORY"); var child = std.process.Child.init(&.{ "bazel", @@ -17,7 +22,7 @@ pub fn main() !void { child.stderr_behavior = .Inherit; child.cwd = build_workspace_directory; - try child.spawn(); + try child.spawn(io); // var out = child.stdout.?; // _ = out; // autofix From a1ce0a387b425be454a8c7cb4b12fcfbe7c2c68e Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 5 Jan 2026 23:38:40 +0000 Subject: [PATCH 194/881] runtimes: remove usage of std.fs, add std.Io --- runtimes/cuda/zmlxcuda.zig | 2 +- runtimes/neuron/libneuronxla.zig | 61 ++++++++++++++++++------------ runtimes/neuron/libpjrt_neuron.zig | 14 +++---- runtimes/neuron/neuron.zig | 27 +++++++------ runtimes/neuron/neuronx-cc.zig | 6 +-- runtimes/rocm/rocm.zig | 8 ++-- runtimes/rocm/zmlxrocm.zig | 10 ++--- runtimes/tpu/tpu.zig | 24 ++++++------ stdx/Io.zig | 8 ++++ 9 files changed, 89 insertions(+), 71 deletions(-) diff --git a/runtimes/cuda/zmlxcuda.zig b/runtimes/cuda/zmlxcuda.zig index 425cbbf9..dc9be25f 100644 --- a/runtimes/cuda/zmlxcuda.zig +++ b/runtimes/cuda/zmlxcuda.zig @@ -17,7 +17,7 @@ pub export fn zmlxcuda_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const new_filename: [*c]const u8 = if (filename) |f| blk: { - const replacement = replacements.get(std.fs.path.basename(std.mem.span(f))) orelse break :blk f; + const replacement = replacements.get(std.Io.Dir.path.basename(std.mem.span(f))) orelse break :blk f; break :blk stdx.Io.Dir.path.bufJoinZ(&buf, &.{ stdx.process.selfSharedObjectDirPath(), replacement, diff --git a/runtimes/neuron/libneuronxla.zig b/runtimes/neuron/libneuronxla.zig index e2e25878..f14d389d 100644 --- a/runtimes/neuron/libneuronxla.zig +++ b/runtimes/neuron/libneuronxla.zig @@ -6,22 +6,26 @@ const upb = @import("upb"); const log = std.log.scoped(.@"zml/runtimes/neuron/libneuronxla"); -pub fn makeTempDir(buf: []u8, prefix: []const u8) ![]const u8 { +pub fn makeTempDir(io: std.Io, buf: []u8, prefix: []const u8) ![]const u8 { const tmp_dir = std.posix.getenv("TMPDIR") orelse "/tmp"; const ret = try std.fmt.bufPrint(buf, "{s}{s}{s}{d}", .{ tmp_dir, - std.fs.path.sep_str_posix, + std.Io.Dir.path.sep_str_posix, prefix, std.time.microTimestamp(), }); - try std.fs.makeDirAbsolute(ret); + try std.Io.Dir.createDir(.cwd(), io, ret, .fromMode(0o700)); return ret; } +const ModuleState = struct { + threaded: std.Io.Threaded, +}; + var module_def: c.PyModuleDef = .{ .m_base = .{}, .m_name = "libneuronxla", - .m_size = 0, + .m_size = @sizeOf(ModuleState), .m_methods = @constCast(&[_]c.PyMethodDef{ .{ .ml_name = "hook", @@ -66,7 +70,7 @@ pub fn PyBytes_AsStringAndSize(object: *c.PyObject) []u8 { return buf[0..@intCast(len)]; } -fn wrapNeffAsCustomCall(allocator: std.mem.Allocator, hlo_code: []const u8, neff_file_path: []const u8) ![]const u8 { +fn wrapNeffAsCustomCall(allocator: std.mem.Allocator, io: std.Io, hlo_code: []const u8, neff_file_path: []const u8) ![]const u8 { var upb_alloc: upb.Allocator = .init(allocator); const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); @@ -98,14 +102,12 @@ fn wrapNeffAsCustomCall(allocator: std.mem.Allocator, hlo_code: []const u8, neff c.xla_HloInstructionProto_set_opcode(fused_root, upb.stringView("custom-call")); c.xla_HloInstructionProto_set_custom_call_target(fused_root, upb.stringView("AwsNeuronNeff")); - c.xla_HloInstructionProto_set_backend_config(fused_root, blk: { - const neff_file = try std.fs.openFileAbsolute(neff_file_path, .{}); - defer neff_file.close(); - const stat = try neff_file.stat(); - const neff_buf = try allocator.alloc(u8, stat.size); - const size = try neff_file.readAll(neff_buf); - break :blk upb.stringView(neff_buf[0..size]); - }); + c.xla_HloInstructionProto_set_backend_config( + fused_root, + try upb.stringView( + try stdx.Io.Dir.readFileAlloc(.cwd(), io, neff_file_path, allocator, .unlimited), + ), + ); const parameters_len = blk: { var size: usize = undefined; @@ -150,7 +152,8 @@ fn wrapNeffAsCustomCall(allocator: std.mem.Allocator, hlo_code: []const u8, neff } fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) !?*c.PyObject { - _ = self; + const state: *ModuleState = @ptrCast(@alignCast(c.PyModule_GetState(self))); + const io = state.threaded.io(); var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); defer arena.deinit(); @@ -170,24 +173,24 @@ fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) }; var tmp_dir_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const tmp_dir = try makeTempDir(&tmp_dir_buf, "zml-neuronxcc-"); - defer std.fs.deleteTreeAbsolute(tmp_dir) catch |err| { + const tmp_dir = try makeTempDir(io, &tmp_dir_buf, "zml-neuronxcc-"); + defer std.Io.Dir.deleteTree(.cwd(), io, tmp_dir) catch |err| { log.err("Error deleting temporary directory {s}: {}\n", .{ tmp_dir, err }); }; - const code_file = try std.fs.path.join(arena.allocator(), &.{ tmp_dir, "file.code" }); + const code_file = try std.Io.Dir.path.join(arena.allocator(), &.{ tmp_dir, "file.code" }); { - const file = try std.fs.cwd().createFile(code_file, .{ .truncate = true }); + const file = try std.Io.Dir.createFile(.cwd(), io, code_file, .{ .truncate = true }); defer file.close(); try file.writeAll(code); } - const neff_file = try std.fs.path.join(arena.allocator(), &.{ tmp_dir, "file.neff" }); + const neff_file = try std.Io.Dir.path.join(arena.allocator(), &.{ tmp_dir, "file.neff" }); var neuronx_cc_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - var child = std.process.Child.init(&.{ - try stdx.fs.path.bufJoin(&neuronx_cc_buf, &.{ - stdx.fs.selfSharedObjectDirPath(), + var child: std.process.Child = .init(&.{ + try stdx.Io.Dir.path.bufJoin(&neuronx_cc_buf, &.{ + stdx.process.selfSharedObjectDirPath(), "..", "bin", "neuronx-cc", @@ -212,7 +215,7 @@ fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) child.stdout_behavior = .Inherit; child.stderr_behavior = .Inherit; child.cwd = tmp_dir; - _ = try child.spawnAndWait(); + _ = try child.spawnAndWait(io); std.debug.print(">>>> {s}\n", .{tmp_dir}); @@ -245,5 +248,15 @@ fn neuronx_cc(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) } pub export fn PyInit_libneuronxla() callconv(.c) ?*c.PyObject { - return c.PyModuleDef_Init(&module_def); + const module = c.PyModuleDef_Init(&module_def); + if (module == null) { + return null; + } + + const state: *ModuleState = @ptrCast(@alignCast(c.PyModule_GetState(module))); + state.* = .{ + .threaded = .init(std.heap.c_allocator, .{}), + }; + + return module; } diff --git a/runtimes/neuron/libpjrt_neuron.zig b/runtimes/neuron/libpjrt_neuron.zig index 33900bb8..a4406e41 100644 --- a/runtimes/neuron/libpjrt_neuron.zig +++ b/runtimes/neuron/libpjrt_neuron.zig @@ -30,9 +30,9 @@ pub export fn zmlxneuron_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaqu var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const new_filename: [*c]const u8 = if (filename) |f| blk: { - const replacement = replacements.get(std.fs.path.basename(std.mem.span(f))) orelse break :blk f; + const replacement = replacements.get(std.Io.Dir.path.basename(std.mem.span(f))) orelse break :blk f; break :blk stdx.fs.path.bufJoinZ(&buf, &.{ - stdx.fs.selfSharedObjectDirPath(), + stdx.process.selfSharedObjectDirPath(), replacement, }) catch unreachable; } else null; @@ -102,7 +102,7 @@ fn setupPythonEnv(sandbox_path: []const u8) !void { { var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const home = try std.fmt.bufPrintZ(&buf, "{f}{d}.{d}", .{ - std.fs.path.fmtJoin(&.{ + std.Io.Dir.path.fmtJoin(&.{ sandbox_path, "lib", "python", @@ -116,7 +116,7 @@ fn setupPythonEnv(sandbox_path: []const u8) !void { { var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const site_packages = try stdx.fs.path.bufJoin(&buf, &.{ + const site_packages = try stdx.Io.Dir.path.bufJoin(&buf, &.{ sandbox_path, "site-packages", }); @@ -147,8 +147,8 @@ fn getPjrtApi() !*c.PJRT_Api { }; var sandbox_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = try stdx.fs.path.bufJoin(&sandbox_path_buf, &.{ - stdx.fs.selfSharedObjectDirPath(), + const sandbox_path = try stdx.Io.Dir.path.bufJoin(&sandbox_path_buf, &.{ + stdx.process.selfSharedObjectDirPath(), "..", }); @@ -161,7 +161,7 @@ fn getPjrtApi() !*c.PJRT_Api { .inner = .{ .handle = handle_blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const library = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libneuronpjrt.so" }); + const library = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libneuronpjrt.so" }); break :handle_blk std.c.dlopen(library, .{ .LAZY = true, .NODELETE = true }) orelse { log.err("Unable to dlopen plugin: {?s}", .{std.mem.span(std.c.dlerror())}); return error.DlOpenFailed; diff --git a/runtimes/neuron/neuron.zig b/runtimes/neuron/neuron.zig index 3b7f4947..9d57e629 100644 --- a/runtimes/neuron/neuron.zig +++ b/runtimes/neuron/neuron.zig @@ -14,24 +14,23 @@ pub fn isEnabled() bool { } fn hasNeuronDevice(io: std.Io) bool { - // TODO(Corentin): Plug that when fs stuff is merged in std.Io - _ = io; - std.fs.accessAbsolute("/dev/neuron0", .{ .read = true }) catch return false; + std.Io.Dir.accessAbsolute(io, "/dev/neuron0", .{ .read = true }) catch return false; return true; } fn isRunningOnEC2(io: std.Io) !bool { - // TODO(Corentin): Plug that when fs stuff is merged in std.Io - _ = io; const AmazonEC2 = "Amazon EC2"; - - var f = try std.fs.openFileAbsolute("/sys/devices/virtual/dmi/id/sys_vendor", .{ .read = true }); - defer f.close() catch {}; - - var content: [AmazonEC2.len]u8 = undefined; - const n_read = try f.pread(&content, 0); - - return std.mem.eql(u8, content[0..n_read], AmazonEC2); + var buffer: [AmazonEC2.len]u8 = undefined; + return std.mem.eql( + u8, + AmazonEC2, + try std.Io.Dir.readFile( + .cwd(), + io, + "/sys/devices/virtual/dmi/id/product_name", + &buffer, + ), + ); } pub fn load(io: std.Io) !*const pjrt.Api { @@ -66,7 +65,7 @@ pub fn load(io: std.Io) !*const pjrt.Api { return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_neuron.so" }); + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_neuron.so" }); var future = io.async(struct { fn call(path_: [:0]const u8) !*const pjrt.Api { return pjrt.Api.loadFrom(path_); diff --git a/runtimes/neuron/neuronx-cc.zig b/runtimes/neuron/neuronx-cc.zig index e39c1748..3f2a8fe8 100644 --- a/runtimes/neuron/neuronx-cc.zig +++ b/runtimes/neuron/neuronx-cc.zig @@ -43,12 +43,12 @@ pub fn main() !void { _ = c.PyConfig_SetBytesArgv(&config, @intCast(std.os.argv.len), @ptrCast(std.os.argv)); var self_exe_dir_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const self_exe_dir = try std.fs.selfExeDirPath(&self_exe_dir_buf); + const self_exe_dir = try std.process.executableDirPath(io, &self_exe_dir_buf); { var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const home = try std.fmt.bufPrintZ(&buf, "{f}{d}.{d}", .{ - std.fs.path.fmtJoin(&.{ + std.Io.Dir.path.fmtJoin(&.{ self_exe_dir, "..", "lib", @@ -64,7 +64,7 @@ pub fn main() !void { { var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const site_packages = try std.fmt.bufPrint(&buf, "{f}", .{ - std.fs.path.fmtJoin(&.{ + std.Io.Dir.path.fmtJoin(&.{ self_exe_dir, "..", "site-packages", diff --git a/runtimes/rocm/rocm.zig b/runtimes/rocm/rocm.zig index 28af4fb8..95ac8d91 100644 --- a/runtimes/rocm/rocm.zig +++ b/runtimes/rocm/rocm.zig @@ -14,17 +14,15 @@ pub fn isEnabled() bool { } fn hasRocmDevices(io: std.Io) bool { - // TODO(Corentin): Plug that when fs stuff is merged in std.Io - _ = io; inline for (&.{ "/dev/kfd", "/dev/dri" }) |path| { - std.fs.accessAbsolute(path, .{ .read = true }) catch return false; + std.Io.Dir.accessAbsolute(io, path, .{ .read = true }) catch return false; } return true; } fn setupRocmEnv(rocm_data_dir: []const u8) !void { var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - _ = c.setenv("ROCM_PATH", try stdx.fs.path.bufJoinZ(&buf, &.{rocm_data_dir}), 1); // must be zero terminated + _ = c.setenv("ROCM_PATH", try stdx.Io.Dir.path.bufJoinZ(&buf, &.{rocm_data_dir}), 1); // must be zero terminated } pub fn load(io: std.Io) !*const pjrt.Api { @@ -57,7 +55,7 @@ pub fn load(io: std.Io) !*const pjrt.Api { try setupRocmEnv(sandbox_path); var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const lib_path = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_rocm.so" }); + const lib_path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_rocm.so" }); // We must load the PJRT plugin from the main thread. // diff --git a/runtimes/rocm/zmlxrocm.zig b/runtimes/rocm/zmlxrocm.zig index eeac9826..e3024de6 100644 --- a/runtimes/rocm/zmlxrocm.zig +++ b/runtimes/rocm/zmlxrocm.zig @@ -24,9 +24,9 @@ pub export fn zmlxrocm_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const new_filename: [*c]const u8 = if (filename) |f| blk: { - const replacement = replacements.get(std.fs.path.basename(std.mem.span(f))) orelse break :blk f; - break :blk stdx.fs.path.bufJoinZ(&buf, &.{ - stdx.fs.selfSharedObjectDirPath(), + const replacement = replacements.get(std.Io.Dir.path.basename(std.mem.span(f))) orelse break :blk f; + break :blk stdx.Io.Dir.path.bufJoinZ(&buf, &.{ + stdx.process.selfSharedObjectDirPath(), replacement, }) catch unreachable; } else null; @@ -42,8 +42,8 @@ pub export fn zmlxrocm_fopen64(pathname: [*c]const u8, mode: [*c]const u8) ?*std var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const new_pathname: [*c]const u8 = blk: { const replacement = replacements.get(std.mem.span(pathname)) orelse break :blk pathname; - break :blk stdx.fs.path.bufJoinZ(&buf, &.{ - stdx.fs.selfSharedObjectDirPath(), + break :blk stdx.Io.Dir.path.bufJoinZ(&buf, &.{ + stdx.process.selfSharedObjectDirPath(), replacement, }) catch unreachable; }; diff --git a/runtimes/tpu/tpu.zig b/runtimes/tpu/tpu.zig index 32b6b84e..83d6c4a9 100644 --- a/runtimes/tpu/tpu.zig +++ b/runtimes/tpu/tpu.zig @@ -18,18 +18,18 @@ pub fn isEnabled() bool { /// Do it using the official method at: /// https://cloud.google.com/compute/docs/instances/detect-compute-engine?hl=en#use_operating_system_tools_to_detect_if_a_vm_is_running_in fn isOnGCP(io: std.Io) !bool { - // TODO(Corentin): Plug that when fs stuff is merged in std.Io - _ = io; - // TODO: abstract that in the client and fail init const GoogleComputeEngine = "Google Compute Engine"; - - var f = try std.fs.openFileAbsolute("/sys/devices/virtual/dmi/id/product_name", .{ .read = true }); - defer f.close() catch {}; - - var content: [GoogleComputeEngine.len]u8 = undefined; - const n_read = try f.pread(&content, 0); - - return std.mem.eql(u8, content[0..n_read], GoogleComputeEngine); + var buffer: [GoogleComputeEngine.len]u8 = undefined; + return std.mem.eql( + u8, + GoogleComputeEngine, + try std.Io.Dir.readFile( + .cwd(), + io, + "/sys/devices/virtual/dmi/id/product_name", + &buffer, + ), + ); } pub fn load(io: std.Io) !*const pjrt.Api { @@ -61,7 +61,7 @@ pub fn load(io: std.Io) !*const pjrt.Api { return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_tpu.so" }); + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_tpu.so" }); var future = io.async(struct { fn call(path_: [:0]const u8) !*const pjrt.Api { return pjrt.Api.loadFrom(path_); diff --git a/stdx/Io.zig b/stdx/Io.zig index 324b6c68..83db497d 100644 --- a/stdx/Io.zig +++ b/stdx/Io.zig @@ -12,4 +12,12 @@ pub const Dir = struct { return try std.Io.Dir.path.joinZ(fa.allocator(), paths); } }; + + pub fn readFileAlloc(dir: std.Io.Dir, io: std.Io, sub_path: []const u8, gpa: std.mem.Allocator, limit: std.Io.Limit) ![]u8 { + const stat = try std.Io.Dir.statFile(dir, io, sub_path, .{}); + const buffer = try gpa.alloc(u8, limit.min(.limited64(stat.size))); + errdefer gpa.free(buffer); + _ = try std.Io.Dir.readFile(.cwd(), io, sub_path, buffer); + return buffer; + } }; From e7ee7175dd8d665ef133b846c3e0fc51a74ac09c Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 6 Jan 2026 08:54:16 +0000 Subject: [PATCH 195/881] third_party/zls: update to 0.16.0-dev.82+cbb62284 --- third_party/zls/zls_toolchain.bzl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/third_party/zls/zls_toolchain.bzl b/third_party/zls/zls_toolchain.bzl index b5f7e6c1..e487f8bc 100644 --- a/third_party/zls/zls_toolchain.bzl +++ b/third_party/zls/zls_toolchain.bzl @@ -2,7 +2,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -_VERSION = "0.15.0" +_VERSION = "0.16.0-dev.82+cbb62284" _ZLS_PLATFORMS = { "x86_64-linux": struct( @@ -26,9 +26,9 @@ _ZLS_PLATFORMS = { } _ZLS_VERSIONS = { - "x86_64-linux": "508bfe3fd637d2a02f07f3fc7da8900351f407116b03685c5dae26b4f01a30de", - "x86_64-macos": "46c31838bfef5adcc7fee82428c3ec2b9abbfae38242639afea5f242ee133d93", - "aarch64-macos": "76c7a23190f67e67970024065f689c2c49b3c7b0fc16876fb24ef199fb05fc2a", + "x86_64-linux": "27277ee215687fda4a80fa518fb38061f4abb56a469349853ac3e5671e005421", + "x86_64-macos": "98e3618d41532fcb1b6148bed75f081fc2f42e74fa6a23827870eb328dfbf87c", + "aarch64-macos": "083bc9b5fd9f70c7cd380766bffa7ce72a626237c62d7cf26ae3ac0395530ee0", } ZlsToolchainInfo = provider( @@ -74,7 +74,7 @@ def _zls_toolchains_repos_impl(mctx): for platform, _ in _ZLS_PLATFORMS.items(): http_archive( name = "zls_{}".format(platform), - url = "https://github.com/zigtools/zls/releases/download/{version}/zls-{platform}.tar.xz".format( + url = "https://builds.zigtools.org/zls-{platform}-{version}.tar.xz".format( version = _VERSION, platform = platform, ), From 3a94593467bf4448d3edc21ff1001318fc0131b6 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 6 Jan 2026 09:05:56 +0000 Subject: [PATCH 196/881] workspace: fix and bump ZLS --- .zed/settings.json | 2 +- bazel/BUILD.bazel | 15 +++++++++++++++ third_party/zls/zls_completion.bzl | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.zed/settings.json b/.zed/settings.json index 26af5ce3..55eff7c8 100644 --- a/.zed/settings.json +++ b/.zed/settings.json @@ -34,7 +34,7 @@ "lsp": { "zls": { "binary": { - "path": "./tools/zls.sh" + "path": "tools/zls.sh" } } } diff --git a/bazel/BUILD.bazel b/bazel/BUILD.bazel index e69de29b..27870cab 100644 --- a/bazel/BUILD.bazel +++ b/bazel/BUILD.bazel @@ -0,0 +1,15 @@ +load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") +load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_binary( + name = "bazel", + main = "bazel.zig", + visibility = ["//visibility:public"], + deps = [ + "//stdx", + "@rules_zig//zig/runfiles", + ], +) diff --git a/third_party/zls/zls_completion.bzl b/third_party/zls/zls_completion.bzl index f2b59387..ffb7cbf8 100644 --- a/third_party/zls/zls_completion.bzl +++ b/third_party/zls/zls_completion.bzl @@ -23,7 +23,7 @@ def zls_completion(name, deps, **kwargs): ) # Create a target that will be invoked by ZLS using our customer build_runner. - build_config_printer = "print_{}_build_config".format(name) + build_config_printer = "{}.print_build_config".format(name) zig_binary( name = build_config_printer, main = Label("//third_party/zls:workspace_printer.zig"), From c2780ca7c58cd09e4fb88a1cc168164109640d45 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 6 Jan 2026 10:08:17 +0000 Subject: [PATCH 197/881] workspace: zml2 -> zml --- BUILD.bazel | 13 +- examples/benchmark/BUILD.bazel | 2 +- examples/llama/BUILD.bazel | 2 +- examples/mnist/BUILD.bazel | 2 +- examples/simple_layer/BUILD.bazel | 2 +- zml/BUILD.bazel | 37 +- zml/aio.zig | 805 ---- zml/aio/json.zig | 107 - zml/aio/safetensors.zig | 149 - zml/aio/tinyllama.zig | 129 - zml/aio/torch.zig | 41 - zml/aio/torch/eval.zig | 436 -- zml/aio/torch/file.zig | 523 -- zml/aio/torch/pickle.zig | 1072 ----- zml/aio/torch/py.zig | 431 -- zml/aio/torch/simple.pt | Bin 2708 -> 0 bytes zml/aio/torch/simple_test_4.pickle | Bin 130 -> 0 bytes zml/buffer.zig | 525 +-- zml/callback.zig | 322 -- {zml2 => zml}/constants.zig | 0 zml/context.zig | 217 - zml/dtype.zig | 296 +- zml/exe.zig | 608 +-- zml/floats.zig | 207 +- zml/helpers.zig | 141 - zml/hostbuffer.zig | 409 -- {zml2 => zml}/io.zig | 0 {zml2 => zml}/io/BUILD.bazel | 2 +- {zml2 => zml}/io/io.zig | 0 {zml2 => zml}/io/vfs.zig | 0 {zml2 => zml}/io/vfs/BUILD.bazel | 0 {zml2 => zml}/io/vfs/base.zig | 1 + {zml2 => zml}/io/vfs/file.zig | 0 {zml2 => zml}/io/vfs/hf.zig | 0 {zml2 => zml}/io/vfs/hf2.zig | 0 {zml2 => zml}/io/vfs/http.zig | 0 {zml2 => zml}/io/vfs/index.zig | 0 {zml2 => zml}/io/vfs/proxy.zig | 0 zml/meta.zig | 161 +- zml/mlirx.zig | 149 +- zml/module.zig | 1386 ++---- zml/nn.zig | 522 +- zml/nn/cuda.zig | 141 - zml/ops.zig | 1314 ++---- zml/pjrtx.zig | 229 +- zml/platform.zig | 141 +- zml/posix.zig | 16 - zml/quantization.zig | 101 - {zml2 => zml}/safetensors.zig | 0 zml/shape.zig | 811 +++- {zml2 => zml}/slice.zig | 0 zml/tensor.zig | 2260 +++++---- zml/test_runner.zig | 12 +- zml/testing.zig | 260 +- zml/tokenizer/BUILD.bazel | 2 - zml/tokenizer/hftokenizers/hftokenizers.h | 1 + zml/tokenizer/hftokenizers/hftokenizers.rs | 8 + zml/tokenizer/hftokenizers/hftokenizers.zig | 4 + zml/tokenizer/homemade.zig | 23 +- zml/tokenizer/main.zig | 12 +- zml/tokenizer/sentencepiece/sentencepiece.i | 1 + zml/tokenizer/sentencepiece/sentencepiece.zig | 7 + zml/tokenizer/tokenizer.zig | 47 +- zml/tools/BUILD.bazel | 29 - zml/tools/macos.c | 26 - zml/tools/macos.h | 17 - zml/tools/tools.zig | 1 - zml/tools/tracer.zig | 106 - zml/torch.zig | 285 -- zml/zml.zig | 97 +- zml2/BUILD.bazel | 101 - zml2/buffer.zig | 252 - zml2/dtype.zig | 373 -- zml2/exe.zig | 240 - zml2/floats.zig | 556 --- zml2/main.zig | 111 - zml2/main_io.zig | 537 --- zml2/meta.zig | 793 ---- zml2/mlirx.zig | 93 - zml2/module.zig | 503 -- zml2/nn.zig | 1383 ------ zml2/ops.zig | 1035 ---- zml2/pjrtx.zig | 171 - zml2/platform.zig | 350 -- zml2/posix.h | 2 - zml2/shape.zig | 1738 ------- zml2/tensor.zig | 4197 ----------------- zml2/test_runner.zig | 111 - zml2/testing.zig | 149 - zml2/tokenizer/BUILD.bazel | 41 - zml2/tokenizer/hftokenizers/BUILD.bazel | 32 - zml2/tokenizer/hftokenizers/Cargo.lock | 713 --- zml2/tokenizer/hftokenizers/Cargo.toml | 15 - zml2/tokenizer/hftokenizers/hftokenizers.h | 17 - zml2/tokenizer/hftokenizers/hftokenizers.rs | 109 - zml2/tokenizer/hftokenizers/hftokenizers.zig | 120 - zml2/tokenizer/homemade.zig | 1242 ----- zml2/tokenizer/main.zig | 66 - zml2/tokenizer/sentencepiece/BUILD.bazel | 35 - zml2/tokenizer/sentencepiece/sentencepiece.i | 93 - .../tokenizer/sentencepiece/sentencepiece.zig | 198 - zml2/tokenizer/tokenizer.zig | 143 - zml2/zml.zig | 87 - 103 files changed, 4076 insertions(+), 26178 deletions(-) delete mode 100644 zml/aio.zig delete mode 100644 zml/aio/json.zig delete mode 100644 zml/aio/safetensors.zig delete mode 100644 zml/aio/tinyllama.zig delete mode 100644 zml/aio/torch.zig delete mode 100644 zml/aio/torch/eval.zig delete mode 100644 zml/aio/torch/file.zig delete mode 100644 zml/aio/torch/pickle.zig delete mode 100644 zml/aio/torch/py.zig delete mode 100644 zml/aio/torch/simple.pt delete mode 100644 zml/aio/torch/simple_test_4.pickle delete mode 100644 zml/callback.zig rename {zml2 => zml}/constants.zig (100%) delete mode 100644 zml/context.zig delete mode 100644 zml/helpers.zig delete mode 100644 zml/hostbuffer.zig rename {zml2 => zml}/io.zig (100%) rename {zml2 => zml}/io/BUILD.bazel (89%) rename {zml2 => zml}/io/io.zig (100%) rename {zml2 => zml}/io/vfs.zig (100%) rename {zml2 => zml}/io/vfs/BUILD.bazel (100%) rename {zml2 => zml}/io/vfs/base.zig (99%) rename {zml2 => zml}/io/vfs/file.zig (100%) rename {zml2 => zml}/io/vfs/hf.zig (100%) rename {zml2 => zml}/io/vfs/hf2.zig (100%) rename {zml2 => zml}/io/vfs/http.zig (100%) rename {zml2 => zml}/io/vfs/index.zig (100%) rename {zml2 => zml}/io/vfs/proxy.zig (100%) delete mode 100644 zml/nn/cuda.zig delete mode 100644 zml/posix.zig delete mode 100644 zml/quantization.zig rename {zml2 => zml}/safetensors.zig (100%) rename {zml2 => zml}/slice.zig (100%) delete mode 100644 zml/tools/BUILD.bazel delete mode 100644 zml/tools/macos.c delete mode 100644 zml/tools/macos.h delete mode 100644 zml/tools/tools.zig delete mode 100644 zml/tools/tracer.zig delete mode 100644 zml/torch.zig delete mode 100644 zml2/BUILD.bazel delete mode 100644 zml2/buffer.zig delete mode 100644 zml2/dtype.zig delete mode 100644 zml2/exe.zig delete mode 100644 zml2/floats.zig delete mode 100644 zml2/main.zig delete mode 100644 zml2/main_io.zig delete mode 100644 zml2/meta.zig delete mode 100644 zml2/mlirx.zig delete mode 100644 zml2/module.zig delete mode 100644 zml2/nn.zig delete mode 100644 zml2/ops.zig delete mode 100644 zml2/pjrtx.zig delete mode 100644 zml2/platform.zig delete mode 100644 zml2/posix.h delete mode 100644 zml2/shape.zig delete mode 100644 zml2/tensor.zig delete mode 100644 zml2/test_runner.zig delete mode 100644 zml2/testing.zig delete mode 100644 zml2/tokenizer/BUILD.bazel delete mode 100644 zml2/tokenizer/hftokenizers/BUILD.bazel delete mode 100644 zml2/tokenizer/hftokenizers/Cargo.lock delete mode 100644 zml2/tokenizer/hftokenizers/Cargo.toml delete mode 100644 zml2/tokenizer/hftokenizers/hftokenizers.h delete mode 100644 zml2/tokenizer/hftokenizers/hftokenizers.rs delete mode 100644 zml2/tokenizer/hftokenizers/hftokenizers.zig delete mode 100644 zml2/tokenizer/homemade.zig delete mode 100644 zml2/tokenizer/main.zig delete mode 100644 zml2/tokenizer/sentencepiece/BUILD.bazel delete mode 100644 zml2/tokenizer/sentencepiece/sentencepiece.i delete mode 100644 zml2/tokenizer/sentencepiece/sentencepiece.zig delete mode 100644 zml2/tokenizer/tokenizer.zig delete mode 100644 zml2/zml.zig diff --git a/BUILD.bazel b/BUILD.bazel index 477668b3..ec4bcf23 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,21 +1,10 @@ -load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_c_proto_library") -load("@rules_zig//zig:defs.bzl", "zig_binary") load("//third_party/zls:zls_completion.bzl", "zls_completion") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_binary") - -zig_binary( - name = "http", - main = "http.zig", - visibility = ["//visibility:public"], -) - zls_completion( name = "completion", visibility = ["//visibility:public"], deps = [ - #"//examples/llama", "//stdx", - "//zml2:zml", + "//zml:zml", ], ) diff --git a/examples/benchmark/BUILD.bazel b/examples/benchmark/BUILD.bazel index ab10bd77..ce2f0194 100644 --- a/examples/benchmark/BUILD.bazel +++ b/examples/benchmark/BUILD.bazel @@ -6,7 +6,7 @@ zig_binary( main = "main.zig", deps = [ "//stdx", - "//zml2:zml", + "//zml", ], ) diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel index bb087644..9a43f707 100644 --- a/examples/llama/BUILD.bazel +++ b/examples/llama/BUILD.bazel @@ -13,7 +13,7 @@ zig_binary( visibility = ["//visibility:public"], deps = [ "//stdx", - "//zml2:zml", + "//zml", ], ) diff --git a/examples/mnist/BUILD.bazel b/examples/mnist/BUILD.bazel index 69ea3c2c..719b4f73 100644 --- a/examples/mnist/BUILD.bazel +++ b/examples/mnist/BUILD.bazel @@ -14,7 +14,7 @@ zig_binary( main = "mnist.zig", deps = [ "//stdx", - "//zml2:zml", + "//zml", ], ) diff --git a/examples/simple_layer/BUILD.bazel b/examples/simple_layer/BUILD.bazel index 7feb8aad..754816cf 100644 --- a/examples/simple_layer/BUILD.bazel +++ b/examples/simple_layer/BUILD.bazel @@ -8,7 +8,7 @@ zig_binary( name = "simple_layer", main = "main.zig", deps = [ - "//zml2:zml", + "//zml", ], ) diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index efe4abb0..aa5774c0 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -1,6 +1,6 @@ load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_c_proto_library") load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_test") upb_c_proto_library( name = "xla_data_upb", @@ -20,51 +20,36 @@ cc_library( zig_library( name = "zml", srcs = [ - "aio.zig", - "aio/json.zig", - "aio/safetensors.zig", - "aio/tinyllama.zig", - "aio/torch.zig", - "aio/torch/eval.zig", - "aio/torch/file.zig", - "aio/torch/pickle.zig", - "aio/torch/py.zig", "buffer.zig", - "context.zig", - "callback.zig", + "constants.zig", "dtype.zig", "exe.zig", "floats.zig", - "helpers.zig", - "hostbuffer.zig", + "io.zig", "meta.zig", "mlirx.zig", "module.zig", "nn.zig", - "nn/cuda.zig", "ops.zig", "pjrtx.zig", "platform.zig", - "posix.zig", - "quantization.zig", + "safetensors.zig", "shape.zig", + "slice.zig", "tensor.zig", - "test_runner.zig", "testing.zig", - "torch.zig", "zml.zig", ], + main = "zml.zig", + visibility = ["//visibility:public"], zigopts = [ "-lc", "-freference-trace=20", ], - main = "zml.zig", - visibility = ["//visibility:public"], deps = [ ":posix", ":xla_compile_options_upb", ":xla_data_upb", - "//async", "//mlir", "//mlir/dialects", "//pjrt", @@ -72,7 +57,7 @@ zig_library( "//stdx", "//upb", "//zml/tokenizer", - "//zml/tools", + "//zml/io", "@rules_zig//zig/runfiles", ], ) @@ -82,11 +67,11 @@ zig_library( zig_test( name = "test", data = [ - "aio/torch/simple.pt", - "aio/torch/simple_test_4.pickle", + #"aio/torch/simple.pt", + #"aio/torch/simple_test_4.pickle", ], test_runner = ":test_runner", - deps = [":zml"], + deps = ["//zml"], ) filegroup( diff --git a/zml/aio.zig b/zml/aio.zig deleted file mode 100644 index 8114e0a1..00000000 --- a/zml/aio.zig +++ /dev/null @@ -1,805 +0,0 @@ -const std = @import("std"); - -const async = @import("async"); -const c = @import("c"); -const stdx = @import("stdx"); - -pub const safetensors = @import("aio/safetensors.zig"); -pub const torch = @import("aio/torch.zig"); -const HostBuffer = @import("hostbuffer.zig").HostBuffer; -const posix = @import("posix.zig"); -const zml = @import("zml.zig"); - -pub const log = std.log.scoped(.@"zml/aio"); -test { - std.testing.refAllDecls(@This()); - std.testing.refAllDecls(safetensors); - std.testing.refAllDecls(torch); -} - -// TODO error set for weight loading - -/// Detects the format of the model file (base on filename) and open it. -pub fn detectFormatAndOpen(allocator: std.mem.Allocator, model_path: []const u8) !BufferStore { - return if (std.mem.endsWith(u8, model_path, ".safetensors")) - try safetensors.open(allocator, model_path) - else if (std.mem.endsWith(u8, model_path, ".safetensors.index.json")) - try safetensors.open(allocator, model_path) - else if (std.mem.endsWith(u8, model_path, ".pt")) - try torch.open(allocator, model_path) - // else if (std.mem.endsWith(u8, model_path, ".gguf")) - // try gguf.open(allocator, model_path) - // else if (std.mem.endsWith(u8, model_path, ".tinyllama")) - // try tinyllama.open(allocator, model_path) - else { - std.debug.panic("File extension not recognized: {s}", .{model_path}); - }; -} - -/// Creates a Model struct with tensor shapes read from the given BufferStore. -/// The result can be used to pass to `compileModel`. -/// -/// * The `Tensor` field `Model.a.b` will be populated with a `Tensor` -/// whose shape is read from the "a.b" tensor. -/// * If `Model` contains a list of layers, then the field: -/// `Model.layers[2].a.b` will be populated from the "layers.2.a.b" tensor. -pub fn populateModel(comptime Model: type, allocator: std.mem.Allocator, store: BufferStore) !Model { - return populateModelWithPrefix(Model, allocator, store, ""); -} - -/// Creates a Model struct with tensor shapes read from the given TensorStore, -/// using a given prefix. -/// The result can be used to pass to `compileWithModel`. -/// -/// * The `Tensor` field `Model.a.b` will be populated with a `Tensor` -/// whose shape is read from the "prefix.a.b" tensor. -/// * If `Model` contains a list of layers, then the field: -/// `Model.layers[2].a.b` will be populated from the "prefix.layers.2.a.b" tensor. -pub fn populateModelWithPrefix(comptime Model: type, allocator: std.mem.Allocator, store: BufferStore, prefix: []const u8) !Model { - var model: Model = undefined; - - var prefix_builder: PrefixBuilder = .{}; - try prefix_builder.push(allocator, prefix); - defer prefix_builder.deinit(allocator); - - const ok = _populateStruct(allocator, &prefix_builder, store, &model, true) catch |err| { - std.debug.panic("Can't populate model of type {s}: {s}", .{ @typeName(type), @errorName(err) }); - }; - if (!ok) return error.TensorNotFound; - return model; -} - -/// A struct containing all the buffers and metadata found in a model file. -pub const BufferStore = struct { - pub const Buffers = std.StringArrayHashMapUnmanaged(HostBuffer); - pub const Metadatas = std.StringArrayHashMapUnmanaged(Metadata); - var _unique_store_id: std.atomic.Value(u64) = .init(0); - const _store_id_range: u64 = 1024 * 1024 * 1024; - - arena: std.heap.ArenaAllocator, - files: []MemoryMappedFile = &.{}, - buffers: Buffers = .{}, - _metadata: Metadatas = .{}, - _unique_id: u64, - - /// Create an empty BufferStore. - /// Takes owneship of the given files. - pub fn init(allocator: std.mem.Allocator) BufferStore { - return .{ - .arena = std.heap.ArenaAllocator.init(allocator), - ._unique_id = _unique_store_id.fetchAdd(_store_id_range, .monotonic), - }; - } - - /// Create an empty BufferStore. - /// Takes owneship of the given files. - pub fn initWithFiles(allocator: std.mem.Allocator, files: []const MemoryMappedFile) error{OutOfMemory}!BufferStore { - var self: BufferStore = .init(allocator); - self.files = try self.arena.allocator().dupe(MemoryMappedFile, files); - return self; - } - - pub fn deinit(self: BufferStore) void { - for (self.files) |*file| { - file.deinit(); - } - self.arena.deinit(); - } - - pub fn get(self: BufferStore, key: []const u8) ?HostBuffer { - return self.buffers.get(key); - } - - pub fn loadBufferById(self: BufferStore, x: zml.Tensor, platform: zml.Platform) !zml.Buffer { - var host_buffer: zml.HostBuffer = switch (x._id) { - .buffer_id => |id| hb: { - if (id < self._unique_id or self._unique_id + _store_id_range <= id) { - @panic("`store.loadBufferById()` only works on Tensor created by `store.getTensor()`, using the same store object."); - } - if (platform.target != .cpu) mem_debug: { - const stats = platform.getDevices()[0].memoryStats(platform.pjrt_api) catch break :mem_debug; - log.debug("Loading {s} -> {f} {d:>10} bytes ({d:>10} allocated / {d:>10} reserved)", .{ self.buffers.keys()[id - self._unique_id], x._shape, x.shape().byteSize(), stats.bytes_in_use, stats.bytes_reserved }); - } - break :hb self.buffers.values()[id - self._unique_id]; - }, - else => @panic("`store.loadBufferById()` only works on Tensor created by `store.getTensor()`"), - }; - - // Use the sharding information stored in the tensor. - std.debug.assert(host_buffer.shape().eql(x.shape())); - host_buffer._shape = x.shape(); - return try host_buffer.toDevice(platform); - } - - /// Creates a bufferized version of a model from the given BufferStore. - /// - /// This will represent the weights of the model, loaded on a specific platform. - /// It can be used with a `module.Exe` (a compiled version of the same Model), to make a - /// `module.ExeWithWeights` ready to be called. - pub fn loadModelById(self: BufferStore, Model: type, allocator: std.mem.Allocator, model: Model, platform: zml.Platform) !zml.Bufferized(Model) { - const Ctx = struct { - platform: *const zml.Platform, - store: *const BufferStore, - - pub fn cb(ctx: @This(), x: zml.Tensor) zml.Buffer { - return ctx.store.loadBufferById(x, ctx.platform.*) catch @panic("Failed to load buffer to device"); - } - }; - - var res: zml.Bufferized(Model) = undefined; - try zml.meta.mapAlloc(Ctx.cb, allocator, .{ .platform = &platform, .store = &self }, model, &res); - return res; - } - - pub fn getTensor(self: BufferStore, key: []const u8) zml.Tensor { - return self.getTensorOrNull(key) orelse { - log.err("Tensor not found: {s}", .{key}); - self.findSimilarBufferKeys(std.heap.smp_allocator, key); - @panic("Tensor not found"); - }; - } - - pub fn getTensorOrNull(self: BufferStore, key: []const u8) ?zml.Tensor { - return if (self.buffers.getIndex(key)) |entry_idx| - .{ - ._shape = self.buffers.values()[entry_idx].shape(), - ._id = .{ .buffer_id = self._unique_id + entry_idx }, - ._donation = .input_buffer, - } - else - return null; - } - - /// Count layers starting with the given prefix. - pub fn countLayers(self: BufferStore, prefix: []const u8) usize { - // Note: This is kinda inefficient - const digit_start_index = prefix.len + 1; - var it = self.buffers.iterator(); - var maybe_max_index: ?usize = null; - while (it.next()) |entry| { - if (!std.mem.startsWith(u8, entry.key_ptr.*, prefix)) continue; - - const next_dot_index = std.mem.indexOfScalarPos(u8, entry.key_ptr.*, digit_start_index, '.') orelse entry.key_ptr.len; - const index = std.fmt.parseInt(usize, entry.key_ptr.*[digit_start_index..next_dot_index], 10) catch continue; - if (maybe_max_index) |*max_index| { - max_index.* = @max(max_index.*, index); - } else { - maybe_max_index = index; - } - } - - return if (maybe_max_index) |index| index + 1 else 0; - } - - pub fn metadata(self: BufferStore, key: []const u8, comptime tag: std.meta.FieldEnum(Metadata)) ?std.meta.FieldType(Metadata, tag) { - const wrapped_value = self._metadata.get(key) orelse return null; - - if (wrapped_value != tag) { - zml.log.err("Tried to interpret metadata '{s}' as {}, but was of type {}", .{ key, tag, wrapped_value }); - @panic("invalid metadata type"); - } - return @field(wrapped_value, @tagName(tag)); - } - - pub fn metadataSlice(self: BufferStore, key: []const u8, comptime tag: Metadata.ItemType) ?[]const tag.toZigType() { - const wrapped_value = self._metadata.get(key) orelse return null; - const true_tag = std.meta.stringToEnum(std.meta.FieldEnum(Metadata), @tagName(tag)).?; - if (wrapped_value == true_tag) { - return @field(wrapped_value, "array_" ++ @tagName(tag)); - } - - return null; - } - - /// Assists in debuggigng `BufferNotFound` error - /// This is useful when a buffer key is not found and you want to identify possible alternatives (or typos) - pub fn findSimilarBufferKeys(store: BufferStore, tmp_alloc: std.mem.Allocator, original_key: []const u8) void { - const suffixes = [_][]const u8{ "", ".weight", ".bias" }; - var shown_keys = std.StringHashMap(void).init(tmp_alloc); - defer shown_keys.deinit(); - - // remove suffix .weight and .bias - var base_key = original_key; - for (suffixes) |suffix| { - if (std.mem.endsWith(u8, original_key, suffix)) { - base_key = original_key[0 .. original_key.len - suffix.len]; - break; - } - } - - // first test: look for exact matches - var matches: usize = 0; - var it = store.buffers.iterator(); - while (it.next()) |entry| { - const key = entry.key_ptr.*; - if (std.mem.startsWith(u8, key, base_key)) { - if (matches == 0) log.warn("Similar buffers found:", .{}); - if (!shown_keys.contains(key)) { - log.warn(" - {s}: {f}", .{ key, entry.value_ptr.*.shape() }); - shown_keys.put(key, {}) catch continue; - matches += 1; - } - } - } - - // second test: progressive partial matches - if (matches == 0) { - var components = std.mem.splitScalar(u8, base_key, '.'); - while (components.next()) |component| { - matches = 0; - it = store.buffers.iterator(); - while (it.next()) |entry| { - const key = entry.key_ptr.*; - if (std.mem.indexOf(u8, key, component) != null and !shown_keys.contains(key)) { - if (matches == 0) log.warn("Partial matches for '{s}':", .{component}); - log.warn(" - {s}: {f}", .{ key, entry.value_ptr.*.shape() }); - shown_keys.put(key, {}) catch continue; - matches += 1; - if (matches >= 5) break; - } - } - if (matches > 0) break; - } - } - } -}; - -pub const Metadata = union(enum) { - null: void, - int: i64, - float: f64, - bool: bool, - string: []const u8, - - array_bool: []const bool, - array_int: []const i64, - array_float: []const f64, - array_string: []const []const u8, - - pub const ItemType = enum { - int, - float, - bool, - string, - - pub fn toZigType(comptime kind: ItemType) type { - return switch (kind) { - .int => i64, - .float => f64, - .bool => bool, - .string => []const u8, - }; - } - }; - - pub fn wrap(x: anytype) Metadata { - return switch (@TypeOf(x)) { - inline u8, i8, u16, i16, u32, i32, u64, i64 => .{ .int = @intCast(x) }, - inline f16, f32, f64 => .{ .float = @floatCast(x) }, - bool => .{ .bool = x }, - []const u8 => .{ .string = x }, - else => @panic("Unsupported type for zml.aio.Value: " ++ @typeName(@TypeOf(x))), - }; - } - - pub fn copySlice(allocator: std.mem.Allocator, any_slice: anytype) !Metadata { - return switch (@TypeOf(any_slice[0])) { - inline u8, i8, u16, i16, u32, i32, u64, i64 => { - const res = try allocator.alloc(i64, any_slice.len); - for (res, any_slice) |*r, val| r.* = @intCast(val); - return .{ .array_int = res }; - }, - inline f16, f32, f64 => { - const res = try allocator.alloc(f64, any_slice.len); - for (res, any_slice) |*r, val| r.* = @floatCast(val); - return .{ .array_float = res }; - }, - bool => .{ .array_bool = try allocator.dupe(bool, any_slice) }, - []const u8 => .{ .array_string = try allocator.dupe([]const u8, @alignCast(any_slice)) }, - else => @panic("Unsupported type for zml.aio.Value: " ++ @typeName(@TypeOf(any_slice))), - }; - } - - pub fn format(self: Metadata, writer: *std.Io.Writer) !void { - switch (self) { - .null => _ = try writer.write("null"), - inline .bool, .array_bool => |b| try writer.print("{any}", .{b}), - inline else => |v| try writer.print("{d}", .{v}), - } - } -}; - -/// A file containing contiguous/non-contiguous buffers, that can be read with mmap -/// (assumes contiguous if `strides` is `null`). -/// This struct is meant to be wrapped into a format specific struct, like io.gguf.File. -pub const MemoryMappedFile = struct { - /// underlying file handle - file: async.File, - data: []align(std.heap.page_size_min) const u8, - data_offset: u64 = 0, - - pub fn init(file: async.File) !MemoryMappedFile { - const data_len: usize = (try file.stat()).size; - const data_ = try async.callBlocking(std.posix.mmap, .{ - null, - data_len, - std.posix.PROT.READ, - std.posix.system.MAP{ .TYPE = .PRIVATE }, - file.handle(), - 0, - }); - - try async.callBlocking(posix.madvise, .{ - data_.ptr, - @as(usize, @intCast(data_.len)), - @as(u32, @intCast(c.MADV_SEQUENTIAL)), - }); - - return .{ - .file = file, - .data = data_, - }; - } - - pub fn mappedSlice(self: MemoryMappedFile, start: usize, len: usize) []const u8 { - return self.data[self.data_offset + start ..][0..len]; - } - - pub fn deinit(self: *MemoryMappedFile) void { - std.posix.munmap(self.data); - self.file.close() catch @panic("failed to close file"); - self.* = undefined; - } -}; - -/// Helper handling prefix building. -/// -/// This allows to easily push/pop prefixes and handles the generation of the string with the correct format. -pub const PrefixBuilder = struct { - /// Stores the computed prefix. - data: std.ArrayList(u8) = .{}, - /// Stack storing the size of the intermediary prefix. - subprefixes: std.ArrayList(u32) = .{}, - - pub fn initCapacity(allocator: std.mem.Allocator, capacity: usize) !PrefixBuilder { - return .{ - .data = try .initCapacity(allocator, capacity), - .subprefixes = try .initCapacity(allocator, @divFloor(capacity, 4)), - }; - } - - pub fn deinit(self: *PrefixBuilder, allocator: std.mem.Allocator) void { - self.data.deinit(allocator); - self.subprefixes.deinit(allocator); - } - - pub fn items(self: PrefixBuilder) []const u8 { - return self.data.items; - } - - pub fn concat(self: *PrefixBuilder, prefix: []const u8) []const u8 { - self.push(stdx.noalloc, prefix) catch unreachable; - const res = self.items(); - self.pop(); - return res; - } - - pub fn push(self: *PrefixBuilder, allocator: std.mem.Allocator, prefix: []const u8) !void { - const old_len: u32 = @intCast(self.data.items.len); - try self.subprefixes.append(allocator, old_len); - errdefer _ = self.subprefixes.pop(); - - if (old_len == 0) { - try self.data.appendSlice(allocator, prefix); - } else { - try self.data.ensureUnusedCapacity(allocator, prefix.len + 1); - self.data.appendAssumeCapacity('.'); - self.data.appendSliceAssumeCapacity(prefix); - } - } - - pub fn pushDigit(self: *PrefixBuilder, allocator: std.mem.Allocator, idx: usize) !void { - const old_len: u32 = @intCast(self.data.items.len); - try self.subprefixes.append(allocator, old_len); - errdefer _ = self.subprefixes.pop(); - - try self.data.ensureUnusedCapacity(allocator, 16); - if (old_len > 0) { - self.data.appendAssumeCapacity('.'); - } - try self.data.writer(allocator).print("{d}", .{idx}); - } - - pub fn pop(self: *PrefixBuilder) void { - const last_prefix_len = self.subprefixes.pop() orelse unreachable; - self.data.shrinkRetainingCapacity(last_prefix_len); - } - - pub fn checkpoint(self: PrefixBuilder) [2]usize { - return .{ self.data.items.len, self.subprefixes.items.len }; - } - - pub fn restore(self: *PrefixBuilder, ckpt: [2]usize) void { - self.data.items.len, self.subprefixes.items.len = ckpt; - } -}; - -fn _populateStruct( - allocator: std.mem.Allocator, - prefix_builder: *PrefixBuilder, - store: BufferStore, - obj: anytype, - required: bool, -) !bool { - const err_msg = "_populateStruct must be called with a pointer to type. Received "; - const type_info, const T = switch (@typeInfo(@TypeOf(obj))) { - .pointer => |ptr_info| switch (ptr_info.size) { - .one => .{ @typeInfo(ptr_info.child), ptr_info.child }, - else => @compileError(err_msg ++ @typeName(@TypeOf(obj))), - }, - else => @compileError(err_msg ++ @typeName(@TypeOf(obj))), - }; - - const prefix = prefix_builder.data.items; - if (T == zml.Tensor) { - return if (store.getTensorOrNull(prefix)) |tensor| { - obj.* = tensor; - return true; - } else { - if (required) { - log.err("Tensor not found: {s} ({d})", .{ prefix, store.buffers.count() }); - } - return false; - }; - } - - return switch (type_info) { - .pointer => |ptr_info| { - if (ptr_info.size == .slice) { - obj.* = &.{}; - - const len = store.countLayers(prefix); - if (len > 0) { - obj.* = try allocator.alloc(ptr_info.child, len); - - for (obj.*, 0..) |*value, i| { - try prefix_builder.pushDigit(allocator, i); - defer prefix_builder.pop(); - const found = try _populateStruct(allocator, prefix_builder, store, value, required); - if (!found) { - log.err("Not able to load {s} as {s}", .{ prefix_builder.data.items, @typeName(ptr_info.child) }); - return false; - } - } - } else if (required) { - log.warn("No layer found at {s}", .{prefix}); - } - return true; - } else { - log.err("{s} - {s}: {s} type not supported", .{ @src().fn_name, prefix, @typeName(T) }); - return false; - } - }, - .array => |arr_info| { - for (obj, 0..) |*value, i| { - try prefix_builder.pushDigit(allocator, i); - defer prefix_builder.pop(); - const found = try _populateStruct(allocator, prefix_builder, store, value, required); - if (!found) { - log.err("Not able to load {s} as {s}", .{ prefix_builder.data.items, @typeName(arr_info.child) }); - return false; - } - } - return true; - }, - .@"struct" => |struct_info| { - var partial_struct = false; - inline for (struct_info.fields) |field| { - if (field.is_comptime or @sizeOf(field.type) == 0) continue; - try prefix_builder.push(allocator, field.name); - defer prefix_builder.pop(); - - var has_default = false; - if (field.default_value_ptr) |_| has_default = true; - - if (zml.meta.Contains(field.type, zml.Tensor)) { - const field_found = try _populateStruct(allocator, prefix_builder, store, &@field(obj, field.name), required and !has_default); - partial_struct = partial_struct or field_found; - if (!field_found) { - if (field.default_value_ptr) |v| { - @field(obj, field.name) = @as(*const field.type, @ptrCast(@alignCast(v))).*; - } else { - if (partial_struct) { - log.warn("Incomplete struct '{0s}': {1s}. Missing field: '{2s}'. '{0s}' will be ignored.", .{ prefix, @typeName(T), field.name }); - obj.* = undefined; - return false; - } - - return false; - } - - return false; - } - } - } - return true; - }, - .optional => |opt_info| { - obj.* = @as(opt_info.child, undefined); - const found = try _populateStruct(allocator, prefix_builder, store, &(obj.*.?), false); - if (!found) obj.* = null; - return true; - }, - .int => { - obj.* = undefined; - return true; - }, - .float => { - obj.* = std.math.nan(@TypeOf(obj.*)); - return true; - }, - .void => true, - .@"union" => true, - .bool => { - obj.* = undefined; - return true; - }, - else => if (required) { - log.err("{s}: {s} type not supported", .{ prefix, @typeName(T) }); - return error.UnsupportedMetadataType; - } else return false, - }; -} - -test populateModel { - const Model = struct { - a: zml.Tensor, - b: struct { a: zml.Tensor, b: u32 }, - c: []zml.Tensor, - d: []struct { a: zml.Tensor, b: u32 }, - e: struct { zml.Tensor, u32, struct { a: u32, b: zml.Tensor, c: void } }, - f: ?zml.Tensor, - g: ?zml.Tensor, - - // Create a fake HostBuffer, we use the given integer to identify the created buffer. - fn _newHostBuffer(n: u32) zml.HostBuffer { - return .{ ._shape = zml.Shape.init(.{n}, .f16), ._strides = undefined, ._data = undefined }; - } - }; - - var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena_state.deinit(); - var store: BufferStore = .init(arena_state.allocator()); - try store.buffers.ensureUnusedCapacity(arena_state.allocator(), 16); - store.buffers.putAssumeCapacity("a", Model._newHostBuffer(10)); - store.buffers.putAssumeCapacity("b.a", Model._newHostBuffer(20)); - store.buffers.putAssumeCapacity("c.0", Model._newHostBuffer(30)); - store.buffers.putAssumeCapacity("c.1", Model._newHostBuffer(31)); - store.buffers.putAssumeCapacity("c.2", Model._newHostBuffer(32)); - store.buffers.putAssumeCapacity("d.0.a", Model._newHostBuffer(40)); - store.buffers.putAssumeCapacity("d.1.a", Model._newHostBuffer(41)); - store.buffers.putAssumeCapacity("d.2.a", Model._newHostBuffer(42)); - store.buffers.putAssumeCapacity("e.0", Model._newHostBuffer(50)); - store.buffers.putAssumeCapacity("e.2.b", Model._newHostBuffer(51)); - store.buffers.putAssumeCapacity("f", Model._newHostBuffer(60)); - // no entry for g. - store.buffers.putAssumeCapacity("unused_entry", Model._newHostBuffer(1000)); - - const model = try populateModel(Model, arena_state.allocator(), store); - - try std.testing.expectEqual(10, model.a.dim(0)); - try std.testing.expectEqual(20, model.b.a.dim(0)); - try std.testing.expectEqual(3, model.c.len); - try std.testing.expectEqual(30, model.c[0].dim(0)); - try std.testing.expectEqual(31, model.c[1].dim(0)); - try std.testing.expectEqual(32, model.c[2].dim(0)); - try std.testing.expectEqual(3, model.d.len); - try std.testing.expectEqual(40, model.d[0].a.dim(0)); - try std.testing.expectEqual(41, model.d[1].a.dim(0)); - try std.testing.expectEqual(42, model.d[2].a.dim(0)); - try std.testing.expectEqual(50, model.e[0].dim(0)); - try std.testing.expectEqual(51, model.e[2].b.dim(0)); - try std.testing.expectEqual(60, model.f.?.dim(0)); - try std.testing.expectEqual(null, model.g); -} - -/// Creates a bufferized version of a Model from the given BufferStore. For details about -/// bufferization, see the documentation of Bufferized(T). -/// -/// This will represent the weights of the model, loaded on a specific platform. -/// It can be used with a `module.Exe` (a compiled version of the same Model), to make a -/// `module.ExeWithWeights` ready to be called. -/// -/// The `init_args` are used to initialize the non Buffer fields, using `Model.init` function. -pub fn loadBuffers( - comptime Model: type, - init_args: if (@hasDecl(Model, "init")) stdx.meta.Tail(stdx.meta.FnArgs(Model.init)) else void, - store: BufferStore, - allocator: std.mem.Allocator, - platform: zml.Platform, -) !zml.Bufferized(Model) { - return loadBuffersWithPrefix(Model, init_args, store, allocator, platform, ""); -} - -/// Creates a bufferized version of a Model from the given BufferStore with a specified prefix. -/// For details about bufferization, see the documentation of Bufferized(T). -/// -/// This will represent the weights of the model, loaded on a specific platform. -/// It can be used with a `module.Exe` (a compiled version of the same Model), to make a -/// `module.ExeWithWeights` ready to be called. -/// -/// The `init_args` are used to initialize the non Buffer fields, using `Model.init` function. -pub fn loadBuffersWithPrefix( - comptime Model: type, - init_args: if (@hasDecl(Model, "init")) stdx.meta.Tail(stdx.meta.FnArgs(Model.init)) else void, - store: BufferStore, - allocator: std.mem.Allocator, - platform: zml.Platform, - prefix: []const u8, -) !zml.Bufferized(Model) { - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - // Get model structure with tensor shapes from the buffer store with prefix - var model: Model = try zml.aio.populateModelWithPrefix(Model, arena, store, prefix); - - // If the Model has a "init" function, call it with the given parameters. - if (@hasDecl(Model, "init")) { - @call(.auto, Model.init, .{&model} ++ init_args); - } - - return loadModelBuffersWithPrefix(Model, model, store, allocator, platform, prefix); -} - -/// Creates a bufferized version of a Model from the given BufferStore. For details about -/// bufferization, see the documentation of Bufferized(T). -/// -/// This will represent the weights of the model, loaded on a specific platform. -/// It can be used with a `module.Exe` (a compiled version of the same Model), to make a -/// `module.ExeWithWeights` ready to be called. -pub fn loadModelBuffers( - comptime Model: type, - model: Model, - store: BufferStore, - allocator: std.mem.Allocator, - platform: zml.Platform, -) !zml.Bufferized(Model) { - return try loadModelBuffersWithPrefix(Model, model, store, allocator, platform, ""); -} - -/// Creates a bufferized version of a Model from the given BufferStore and the given prefix. -/// For details about bufferization, see the documentation of Bufferized(T). -/// -/// This will represent the weights of the model, loaded on a specific platform. -/// It can be used with a `module.Exe` (a compiled version of the same Model), to make a -/// `module.ExeWithWeights` ready to be called. -pub fn loadModelBuffersWithPrefix( - comptime Model: type, - model: Model, - store: BufferStore, - allocator: std.mem.Allocator, - platform: zml.Platform, - prefix: []const u8, -) !zml.Bufferized(Model) { - // Allocate the bufferized version. - // We copy the shape, and let visitStructAndLoadBuffer write the other fields. - // to write them just afterward. - var res: zml.Bufferized(Model) = undefined; - try zml.meta.mapAlloc(struct { - pub fn initBuffer(_: void, tensor: zml.Tensor) zml.Buffer { - return .{ ._shape = tensor.shape(), ._api = undefined, ._shards = undefined, ._target = undefined }; - } - }.initBuffer, allocator, {}, model, &res); - - var prefix_builder: PrefixBuilder = .{}; - try prefix_builder.push(allocator, prefix); - defer prefix_builder.deinit(allocator); - - try visitStructAndLoadBuffer(allocator, &prefix_builder, store, &res, platform); - return res; -} - -/// Takes a bufferized version of a `model`, ie a mirror struct of the `model`, and deinit all the -/// Buffer found. -pub fn unloadBuffers(model: anytype) void { - zml.meta.visit((struct { - fn cb(_: void, buffer: *zml.Buffer) void { - buffer.deinit(); - } - }).cb, {}, model); -} - -/// deinit all buffers in the given struct -pub fn awaitAll(buffers: anytype) !void { - zml.meta.visit((struct { - fn cb(_: void, buffer: *zml.Buffer) void { - buffer.* = buffer.await() catch unreachable; - } - }).cb, {}, buffers); -} - -fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *PrefixBuilder, store: BufferStore, obj: anytype, platform: zml.Platform) !void { - const err_msg = "visitStructAndLoadBuffer must be called with a pointer to type. Received "; - const type_info, const T = switch (@typeInfo(@TypeOf(obj))) { - .pointer => |ptr_info| switch (ptr_info.size) { - .one => .{ @typeInfo(ptr_info.child), ptr_info.child }, - else => @compileError(err_msg ++ @typeName(@TypeOf(obj))), - }, - else => @compileError(err_msg ++ @typeName(@TypeOf(obj))), - }; - - const prefix = prefix_builder.data.items; - if (T == zml.Buffer) { - return if (store.get(prefix)) |host_buffer| { - // obj._shape has been set inside `loadModelBuffersWithPrefix`, before calling us. - var buf_with_metadata = host_buffer; - log.debug("Loading buffer {s} ({f})", .{ prefix, obj._shape }); - stdx.debug.assert(host_buffer.shape().eql(obj._shape), "loadModelBuffers expects to find the same shapes in the model and in the buffer store, got {f} and {f} for tensor {s}", .{ obj._shape, host_buffer, prefix }); - buf_with_metadata._shape = obj._shape; - obj.* = try zml.Buffer.from(platform, buf_with_metadata, .{}); - } else { - log.err("Buffer not found: {s}", .{prefix}); - - store.findSimilarBufferKeys(allocator, prefix); - - return error.BufferNotFound; - }; - } else if (T == zml.Shape) return; - - switch (type_info) { - .pointer => |ptr_info| { - if (ptr_info.size == .slice) { - for (obj.*, 0..) |*value, i| { - try prefix_builder.pushDigit(allocator, i); - defer prefix_builder.pop(); - - try visitStructAndLoadBuffer(allocator, prefix_builder, store, value, platform); - } - } else stdx.debug.compileError("type not supported by visitStructAndLoadBuffer: {}", .{T}); - }, - .array => { - for (obj, 0..) |*value, i| { - try prefix_builder.pushDigit(allocator, i); - defer prefix_builder.pop(); - try visitStructAndLoadBuffer(allocator, prefix_builder, store, value, platform); - } - }, - - .@"struct" => |struct_info| { - inline for (struct_info.fields) |field| { - if (field.is_comptime or @sizeOf(field.type) == 0) continue; - try prefix_builder.push(allocator, field.name); - defer prefix_builder.pop(); - - try visitStructAndLoadBuffer(allocator, prefix_builder, store, &@field(obj, field.name), platform); - } - }, - .optional => { - if (obj.*) |*obj_val| { - try visitStructAndLoadBuffer(allocator, prefix_builder, store, obj_val, platform); - } - }, - else => {}, - } -} diff --git a/zml/aio/json.zig b/zml/aio/json.zig deleted file mode 100644 index 6f465134..00000000 --- a/zml/aio/json.zig +++ /dev/null @@ -1,107 +0,0 @@ -const std = @import("std"); - -const async = @import("async"); - -const zml = @import("../zml.zig"); - -const StringBuilder = std.ArrayList(u8); -pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore { - const file = try std.fs.cwd().openFile(path, .{}); - defer file.close(); - var res: zml.aio.BufferStore = .{ - .arena = std.heap.ArenaAllocator.init(allocator), - }; - errdefer res.arena.deinit(); - const arena = res.arena.allocator(); - - const json_data = try file.reader().readAllAlloc(arena, (try file.metadata()).size()); - const metadata = try std.json.parseFromSliceLeaky(std.json.Value, allocator, json_data, .{ .allocate = .alloc_if_needed }); - - var it = metadata.object.iterator(); - while (it.next()) |entry| { - var prefix_buf: [1024]u8 = undefined; - try parseMetadata(allocator, &res, StringBuilder.initBuffer(&prefix_buf), entry.value_ptr.*); - } - - return res; -} - -pub fn parseMetadata(allocator: std.mem.Allocator, store: *zml.aio.BufferStore, prefix: StringBuilder, val: std.json.Value) !void { - const metadata = &store._metadata; - const key = prefix.items; - return switch (val) { - .null => try metadata.put(allocator, try allocator.dupe(u8, key), .null), - .bool => |v| try metadata.put(allocator, try allocator.dupe(u8, key), .{ .bool = v }), - .integer => |v| try metadata.put(allocator, try allocator.dupe(u8, key), .{ .int = v }), - .float => |v| try metadata.put(allocator, try allocator.dupe(u8, key), .{ .float = v }), - .number_string, .string => |v| try metadata.put(allocator, try allocator.dupe(u8, key), .{ .string = try allocator.dupe(u8, v) }), - .array => |v| { - if (v.items.len == 0) return; - return if (validSlice(v)) |item_type| { - const data: zml.aio.Metadata = switch (item_type) { - .bool => blk: { - const values = try allocator.alloc(bool, v.items.len); - for (v.items, 0..) |item, i| values[i] = item.bool; - break :blk .{ .array_bool = values }; - }, - .integer => blk: { - const values = try allocator.alloc(i64, v.items.len); - for (v.items, 0..) |item, i| values[i] = item.integer; - break :blk .{ .array_int = values }; - }, - .float => blk: { - const values = try allocator.alloc(f64, v.items.len); - for (v.items, 0..) |item, i| values[i] = item.float; - break :blk .{ .array_float = values }; - }, - inline .string, .number_string => |tag| blk: { - const values = try allocator.alloc([]const u8, v.items.len); - for (v.items, 0..) |item, i| { - values[i] = @field(item, @tagName(tag)); - } - break :blk .{ .array_string = values }; - }, - .null, .array, .object => unreachable, - }; - try metadata.put(allocator, try allocator.dupe(u8, key), data); - } else { - for (v.items, 0..) |item, i| { - var new_prefix = prefix; - if (prefix.items.len > 0) - new_prefix.appendAssumeCapacity('.'); - new_prefix.items.len += std.fmt.printInt(new_prefix.unusedCapacitySlice(), i, 10, .lower, .{}); - try parseMetadata(allocator, store, new_prefix, item); - } - }; - }, - .object => |v| { - var obj_iter = v.iterator(); - while (obj_iter.next()) |entry| { - var new_prefix = prefix; - if (prefix.items.len > 0) - new_prefix.appendAssumeCapacity('.'); - new_prefix.appendSliceAssumeCapacity(entry.key_ptr.*); - try parseMetadata(allocator, store, new_prefix, entry.value_ptr.*); - } - }, - }; -} - -/// We can only create a Zig slice out of json array, if all values -/// in the array have the same type. -fn validSlice(v: std.json.Array) ?std.meta.Tag(std.json.Value) { - if (v.items.len == 0) return null; - - const item_type: std.meta.Tag(std.json.Value) = v.items[0]; - switch (item_type) { - .null, .array, .object => return null, - else => {}, - } - - for (v.items[1..]) |item| { - if (item != item_type) - return null; - } - - return item_type; -} diff --git a/zml/aio/safetensors.zig b/zml/aio/safetensors.zig deleted file mode 100644 index de18f446..00000000 --- a/zml/aio/safetensors.zig +++ /dev/null @@ -1,149 +0,0 @@ -const std = @import("std"); -const Allocator = std.mem.Allocator; - -const async = @import("async"); -const stdx = @import("stdx"); - -const MemoryMappedFile = @import("../aio.zig").MemoryMappedFile; -const zml = @import("../zml.zig"); -const HostBuffer = zml.HostBuffer; -const json = @import("json.zig"); - -const log = std.log.scoped(.@"zml/io"); - -pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore { - var res: zml.aio.BufferStore = .init(allocator); - errdefer res.arena.deinit(); - const arena = res.arena.allocator(); - - var files: std.ArrayList(MemoryMappedFile) = .empty; - - if (std.mem.endsWith(u8, path, ".safetensors.index.json")) { - try loadFromIndex(arena, &res, &files, path); - } else { - try loadFile(arena, &res, &files, path); - } - res.files = try files.toOwnedSlice(arena); - return res; -} - -fn loadFromIndex(allocator: Allocator, store: *zml.aio.BufferStore, files: *std.ArrayList(MemoryMappedFile), path: []const u8) !void { - const file = async.File.open(path, .{}) catch |err| { - log.err("Failed to open {s}: {}", .{ path, err }); - return err; - }; - errdefer file.close() catch unreachable; - var buffer: [4096]u8 = undefined; - var r = file.reader(&buffer); - - // const json_data = try allocator.alloc(u8, (try file.stat()).size); - var json_reader = std.json.Reader.init(allocator, &r.interface); - // _ = try r.readAtLeast(json_data, json_data.len); - const index = try std.json.parseFromTokenSourceLeaky(std.json.Value, allocator, &json_reader, .{ .allocate = .alloc_if_needed }); - var loaded_files = std.StringHashMap(void).init(allocator); - - const weight_map = index.object.get("weight_map").?.object; - var it = weight_map.iterator(); - while (it.next()) |entry| { - const filename = entry.value_ptr.string; - if (loaded_files.contains(filename)) { - continue; - } - - log.debug("Loading shard: {s}", .{filename}); - try loaded_files.put(filename, {}); - - const full_filename = try std.fs.path.join(allocator, &.{ std.fs.path.dirname(path).?, filename }); - try loadFile(allocator, store, files, full_filename); - } - - if (index.object.get("__metadata__")) |metadata| { - var prefix_buf: [1024]u8 = undefined; - try json.parseMetadata(allocator, store, .initBuffer(&prefix_buf), metadata); - } -} - -fn loadFile(allocator: Allocator, store: *zml.aio.BufferStore, files: *std.ArrayList(MemoryMappedFile), path: []const u8) !void { - const file = async.File.open(path, .{}) catch |err| { - log.err("Failed to open {s}: {}", .{ path, err }); - return err; - }; - errdefer file.close() catch unreachable; - var buffer: [16 * 1024]u8 = undefined; - var r = file.reader(&buffer); - - const json_header_length: usize = @intCast(try r.interface.takeInt(u64, .little)); - const json_data = try allocator.alloc(u8, json_header_length); - try r.interface.readSliceAll(json_data); - // if (n != json_header_length) { - // log.err("Failed to read the full {} bytes of json header from file {s}", .{ n, path }); - // return error.CorruptedFile; - // } - - const metadata = try std.json.parseFromSliceLeaky(std.json.Value, allocator, json_data, .{}); - var buffer_file = try MemoryMappedFile.init(file); - errdefer buffer_file.deinit(); - buffer_file.data_offset = 8 + json_header_length; - - try files.append(allocator, buffer_file); - errdefer _ = files.pop(); - - var it = metadata.object.iterator(); - while (it.next()) |entry| { - const key = entry.key_ptr.*; - if (std.mem.eql(u8, key, "__metadata__")) { - var prefix_buf: [1024]u8 = undefined; - try json.parseMetadata(allocator, store, .initBuffer(&prefix_buf), entry.value_ptr.*); - continue; - } - const val = entry.value_ptr.*; - const shape_field = val.object.get("shape").?.array; - if (shape_field.items.len > zml.Shape.MAX_RANK) { - // Not an error until someone tries to read the tensor itself. - log.warn("Can't load tensor {s}, too many dims: {}", .{ key, shape_field.items.len }); - continue; - } - const offset_field = val.object.get("data_offsets").?; - const start: usize = @intCast(offset_field.array.items[0].integer); - const end: usize = @intCast(offset_field.array.items[1].integer); - const dtype = try stringToDtype(val.object.get("dtype").?.string); - var dims: stdx.BoundedArray(i64, zml.Shape.MAX_RANK) = .{}; - for (shape_field.items) |d| { - dims.appendAssumeCapacity(d.integer); - } - - const out_shape = zml.Shape.init(dims.constSlice(), dtype); - // We aren't storing 'end', so check we can infer it from the tensor shape. - // This is fine cause safetensor only allow storing contiguous tensors. - // https://github.com/huggingface/safetensors/blob/main/README.md#format - // > The byte buffer needs to be entirely indexed, and cannot contain holes. This prevents the creation of polyglot files. - std.debug.assert(end - start == out_shape.byteSize()); - - const buf = HostBuffer.fromBytes(out_shape, buffer_file.mappedSlice(start, out_shape.byteSize())); - try store.buffers.put(allocator, try allocator.dupe(u8, key), buf); - } -} - -fn stringToDtype(safetensor_type: []const u8) !zml.DataType { - const map = std.StaticStringMap(zml.DataType).initComptime(.{ - .{ "F64", .f64 }, - .{ "F32", .f32 }, - .{ "F16", .f16 }, - .{ "BF16", .bf16 }, - .{ "F8_E4M3", .f8e4m3fn }, - .{ "I64", .i64 }, - .{ "I32", .i32 }, - .{ "I16", .i16 }, - .{ "I8", .i8 }, - .{ "U64", .u64 }, - .{ "U32", .u32 }, - .{ "U16", .u16 }, - .{ "U8", .u8 }, - .{ "BOOL", .bool }, - }); - - return map.get(safetensor_type) orelse { - log.err("Unsupported safetensor data type: {s}", .{safetensor_type}); - return error.UnsupportedDataType; - }; -} diff --git a/zml/aio/tinyllama.zig b/zml/aio/tinyllama.zig deleted file mode 100644 index 5adca927..00000000 --- a/zml/aio/tinyllama.zig +++ /dev/null @@ -1,129 +0,0 @@ -/// Tools to load models from https://huggingface.co/karpathy/tinyllamas/ -/// Originally made to be run with https://github.com/karpathy/llama2.c -const async = @import("async"); -const std = @import("std"); -const stdx = @import("stdx"); - -const zml = @import("../zml.zig"); - -const TinyLlamaConfig = extern struct { - dim: u32, - hidden_dim: u32, - n_layers: u32, - n_heads: u32, - n_kv_heads: u32, - // llama2.c uses the sign bit of the vocab size to store if we need - // a second embedding layer. - vocab: packed struct(u32) { - size: u31, - has_lm_head: bool, - }, - seq_len: u32, -}; - -/// Open a tinyllama file. -/// For convenience we use the same layer names -/// than the one used by the Llama-3.1 models. -pub fn open(allocator: std.mem.Allocator, model_path: []const u8) !zml.aio.BufferStore { - var res: zml.aio.BufferStore = .init(allocator); - errdefer res.arena.deinit(); - const arena = res.arena.allocator(); - - const file = try async.File.open(model_path, .{}); - res.files = try arena.alloc(zml.aio.MemoryMappedFile, 1); - res.files[0] = try zml.aio.MemoryMappedFile.init(file); - - const c = try file.reader().readStructEndian(TinyLlamaConfig, .little); - zml.log.info("Read TinyLlamaConfig: {}", .{c}); - - const dim = c.dim; - const hidden_dim = c.hidden_dim; - const n_layers = c.n_layers; - const n_heads = c.n_heads; - const n_kv_heads = c.n_kv_heads; - const seq_len = c.seq_len; - const head_size = c.dim / c.n_heads; - - var off: usize = @sizeOf(TinyLlamaConfig); - try res.buffers.ensureUnusedCapacity(arena, 8 + 9 * n_layers); - - // Map tinyllama naming to Meta llama3 naming. - // token_embedding_table - off = newBuff(&res, "model.embed_tokens.weight", .{ .voc = c.vocab.size, .d = dim }, off); - // rms_att_weight - off = try splitBuff(&res, "model.layers.{d}.input_layernorm.weight", .{dim}, n_layers, off); - // wq - off = try splitBuff(&res, "model.layers.{d}.self_attn.q_proj.weight", .{ dim, n_heads * head_size }, n_layers, off); - // wk - off = try splitBuff(&res, "model.layers.{d}.self_attn.k_proj.weight", .{ dim, n_kv_heads * head_size }, n_layers, off); - // wv - off = try splitBuff(&res, "model.layers.{d}.self_attn.v_proj.weight", .{ dim, n_kv_heads * head_size }, n_layers, off); - // wo - off = try splitBuff(&res, "model.layers.{d}.self_attn.o_proj.weight", .{ n_heads * head_size, dim }, n_layers, off); - // rms_ffn_weight - off = try splitBuff(&res, "model.layers.{d}.post_attention_layernorm.weight", .{dim}, n_layers, off); - // w1 - off = try splitBuff(&res, "model.layers.{d}.mlp.gate_proj.weight", .{ hidden_dim, dim }, n_layers, off); - // w2 - off = try splitBuff(&res, "model.layers.{d}.mlp.down_proj.weight", .{ dim, hidden_dim }, n_layers, off); - // w3 - off = try splitBuff(&res, "model.layers.{d}.mlp.up_proj.weight", .{ hidden_dim, dim }, n_layers, off); - // rms_final_weight - off = newBuff(&res, "model.norm.weight", .{dim}, off); - // freq_cis_real (not used) - off = newBuff(&res, "freq_cis_real", .{ seq_len, head_size / 2 }, off); - off = newBuff(&res, "freq_cis_imag", .{ seq_len, head_size / 2 }, off); - - // wcls - if (c.vocab.has_lm_head) { - off = newBuff(&res, "lm_head.weight", .{ c.vocab.size, c.dim }, off); - } else { - res.buffers.putAssumeCapacityNoClobber("lm_head.weight", res.buffers.get("model.embed_tokens.weight").?); - } - - const weights_size = off; - std.log.info("Loaded a tinyllama file of {} bytes.\nThis is the parsed configuration of this llama model: {}", .{ weights_size, c }); - if (file.stat() catch null) |stat| { - stdx.debug.assert(weights_size == stat.size, "Expected to have a tinyllama file of {} bytes but file only got {} !\nThis is the parsed configuration of this llama model: {}", .{ weights_size, stat.size, c }); - } - - { - try res._metadata.ensureUnusedCapacity(arena, 11); - res._metadata.putAssumeCapacityNoClobber("dim", .{ .int = c.dim }); - res._metadata.putAssumeCapacityNoClobber("hidden_dim", .{ .int = c.hidden_dim }); - res._metadata.putAssumeCapacityNoClobber("n_layers", .{ .int = c.n_layers }); - res._metadata.putAssumeCapacityNoClobber("num_heads", .{ .int = c.n_heads }); - res._metadata.putAssumeCapacityNoClobber("num_kv_heads", .{ .int = c.n_kv_heads }); - res._metadata.putAssumeCapacityNoClobber("vocab_size", .{ .int = c.vocab.size }); - res._metadata.putAssumeCapacityNoClobber("has_lm_head", .{ .bool = c.vocab.has_lm_head }); - res._metadata.putAssumeCapacityNoClobber("max_seq_len", .{ .int = c.seq_len }); - res._metadata.putAssumeCapacityNoClobber("rope_impl", .{ .string = "interleaved" }); - res._metadata.putAssumeCapacityNoClobber("rope_freq_base", .{ .float = 10_000 }); - res._metadata.putAssumeCapacityNoClobber("rms_norm_eps", .{ .float = 1e-6 }); - } - - return res; -} - -fn newBuff(store: *zml.aio.BufferStore, name: []const u8, sh: anytype, offset: usize) usize { - var shape = zml.Shape.init(sh, .f32); - const n = shape.byteSize(); - const buff = zml.HostBuffer.fromBytes(shape, store.files[0].data[offset..][0..n]); - store.buffers.putAssumeCapacityNoClobber(name, buff); - zml.log.debug("Found {s}: {}", .{ name, shape }); - return offset + n; -} - -fn splitBuff(store: *zml.aio.BufferStore, comptime fmt: []const u8, sh: anytype, layers: usize, offset: usize) !usize { - var shape = zml.Shape.init(sh, .f32); - const n = shape.byteSize(); - var off = offset; - for (0..layers) |i| { - const name = try std.fmt.allocPrint(store.arena.allocator(), fmt, .{i}); - const buff = zml.HostBuffer.fromBytes(shape, store.files[0].data[off..][0..n]); - store.buffers.putAssumeCapacityNoClobber(name, buff); - off += n; - if (i == 0) zml.log.debug("Found {s}: {}", .{ name, shape }); - } - return off; -} diff --git a/zml/aio/torch.zig b/zml/aio/torch.zig deleted file mode 100644 index 7c244f61..00000000 --- a/zml/aio/torch.zig +++ /dev/null @@ -1,41 +0,0 @@ -const std = @import("std"); - -const async = @import("async"); - -const zml = @import("../zml.zig"); -const eval = @import("torch/eval.zig"); -const File = @import("torch/file.zig").File; - -const log = std.log.scoped(.@"zml/aio"); - -test { - std.testing.refAllDecls(@This()); - std.testing.refAllDecls(eval); - std.testing.refAllDecls(@import("torch/py.zig")); - std.testing.refAllDecls(File); -} - -/// Opens and loads a BufferStore from the torch file at the given path. -pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore { - const file = async.File.open(path, .{}) catch |err| { - log.err("Failed to open {s}: {}", .{ path, err }); - return err; - }; - errdefer file.close() catch unreachable; - - // Temporary memory needed to parse the pytorch file. - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - const tmp_alloc = arena.allocator(); - - const mmap_file = try zml.aio.MemoryMappedFile.init(file); - var torch_file = try async.callBlocking(File.init, .{ tmp_alloc, mmap_file }); - - const ops = try torch_file.parsePickle(tmp_alloc); - const py_values = try eval.evaluate(tmp_alloc, ops, true); - - // file ownership is transferred to the BufferStore - var res = try zml.aio.BufferStore.initWithFiles(allocator, &.{mmap_file}); - try torch_file.parseModel(py_values, &res); - return res; -} diff --git a/zml/aio/torch/eval.zig b/zml/aio/torch/eval.zig deleted file mode 100644 index 01604e73..00000000 --- a/zml/aio/torch/eval.zig +++ /dev/null @@ -1,436 +0,0 @@ -const std = @import("std"); -const stdx = @import("stdx"); - -const py = @import("py.zig"); -const pickle = @import("pickle.zig"); - -const MAX_DEPTH: usize = 250; -const MAX_PROTOCOL: u8 = 5; - -pub const PickleMemo = struct { - map: std.AutoHashMap(u32, py.Any), - - pub fn init(allocator: std.mem.Allocator) PickleMemo { - return .{ - .map = std.AutoHashMap(u32, py.Any).init(allocator), - }; - } - - pub fn deinit(self: *PickleMemo) void { - const allocator = self.map.allocator; - var iterator = self.map.iterator(); - defer iterator.deinit(); - while (iterator.next()) |entry| { - entry.value_ptr.deinit(allocator); - } - self.map.deinit() catch unreachable; - self.* = undefined; - } - - pub fn resolve(self: *PickleMemo, allocator: std.mem.Allocator, op: py.Any, recursive: bool) !py.Any { - var used_op = op; - while (used_op == .ref) { - var count: usize = 0; - const val = self.map.get(op.ref) orelse { - return error.BadMemoRef; - }; - if (!recursive) { - return val.clone(allocator); - } - - count += 1; - if (count >= MAX_DEPTH or val != .ref) { - used_op = try val.clone(allocator); - break; - } - used_op = val; - } - if (used_op.containsRef()) { - switch (used_op) { - .app, .object, .global => |v| { - if (v.member.containsRef()) { - v.member = try self.resolve(allocator, v.member, recursive); - } - for (v.args) |*item| { - if (item.containsRef()) { - item.* = try self.resolve(allocator, item.*, recursive); - } - } - }, - .set_state => |v| { - if (v.obj.containsRef()) { - v.obj = try self.resolve(allocator, v.obj, recursive); - } - if (v.state.containsRef()) { - v.state = try self.resolve(allocator, v.state, recursive); - } - }, - .pers_id => |v| { - if (v.ref.containsRef()) { - v.ref = try self.resolve(allocator, v.ref, recursive); - } - }, - .seq => |*v| { - for (v.values) |*item| { - if (item.containsRef()) { - item.* = try self.resolve(allocator, item.*, recursive); - } - } - }, - else => {}, - } - } - return used_op; - } - - pub fn insert(self: *PickleMemo, mid: u32, val: py.Any) !void { - _ = try self.map.fetchPut(mid, val); - } - - pub fn resolveMut(self: *PickleMemo, op: *py.Any, recursive: bool) !*py.Any { - if (op.* != .ref) return op; - var lastmid = op.ref; - var count: usize = 0; - var val = self.map.get(lastmid) orelse { - return error.BadMemoRef; - }; - while (val == .ref) { - lastmid = val.ref; - if (!recursive) { - break; - } - count += 1; - if (count >= MAX_DEPTH) { - break; - } - val = self.map.get(lastmid) orelse { - return error.BadMemoRef; - }; - } - return (self.map.getPtr(lastmid) orelse { - return error.BadMemoRef; - }); - } - - const MemoError = py.Any.UnpickleError || error{BadMemoRef}; - - pub fn resolveAllRefsIter(self: *PickleMemo, allocator: std.mem.Allocator, depth: usize, vals: []py.Any, fix_values: bool) MemoError![]py.Any { - if (depth >= MAX_DEPTH) { - return vals; - } - const res = try allocator.alloc(py.Any, vals.len); - for (vals, 0..) |v, i| { - res[i] = try self.resolveAllRefs(allocator, depth + 1, v, fix_values); - } - return res; - } - - pub fn resolveAllRefs(self: *PickleMemo, allocator: std.mem.Allocator, depth: usize, val: py.Any, fix_values: bool) !py.Any { - var output: py.Any = switch (val) { - .ref => try self.resolve(allocator, val, true), - inline .app, .object, .global => |v, tag| @unionInit(py.Any, @tagName(tag), try py.Object.init( - allocator, - try self.resolveAllRefs(allocator, depth + 1, v.member, fix_values), - try self.resolveAllRefsIter(allocator, depth + 1, v.args, fix_values), - try self.resolveAllRefsIter(allocator, depth + 1, v.kwargs, fix_values), - )), - .set_state => |v| .{ .set_state = try py.SetState.init( - allocator, - try self.resolveAllRefs(allocator, depth + 1, v.obj, fix_values), - try self.resolveAllRefs(allocator, depth + 1, v.state, fix_values), - ) }, - .seq => |v| .{ .seq = .{ .type = v.type, .values = try self.resolveAllRefsIter(allocator, depth + 1, v.values, fix_values) } }, - .pers_id => |v| .{ .pers_id = try py.PersId.init(allocator, try self.resolveAllRefs(allocator, depth + 1, v.ref, fix_values)) }, - else => try val.clone(allocator), - }; - if (fix_values) { - output = try output.coerceFromRaw(allocator); - } - return output; - } -}; - -pub fn evaluate(arena: std.mem.Allocator, x: []const pickle.Op, resolve_refs: bool) ![]const py.Any { - var stack: std.ArrayList(py.Any) = .{}; - var memo = PickleMemo.init(arena); - - for (x) |op| { - switch (op) { - .mark => try stack.append(arena, .{ .raw = op }), - .frame => {}, - .stop => break, - .pop => _ = try pop(&stack), - .pop_mark => _ = try popMark(&stack), - .dup => if (stack.getLastOrNull()) |item| - try stack.append(arena, try item.clone(arena)) - else - return error.CannotDupEmptyStack, - .persid => |v| try stack.append(arena, .{ .pers_id = try py.PersId.init(arena, .{ .string = try arena.dupe(u8, v) }) }), - .binpersid => try stack.append(arena, .{ .pers_id = try py.PersId.init(arena, try pop(&stack)) }), - .reduce => try stack.append(arena, .{ .global = blk: { - var args = try pop(&stack); - args = try memo.resolve(arena, args, true); - if (args != .seq) return error.InvalidInput; - var func = try pop(&stack); - func = try memo.resolve(arena, func, true); - break :blk try py.Object.init(arena, func, args.seq.values, &.{}); - } }), - .build => try stack.append(arena, blk: { - const args = try memo.resolve(arena, try pop(&stack), true); - const member = try memo.resolve(arena, try pop(&stack), true); - break :blk .{ .set_state = try py.SetState.init(arena, member, args) }; - }), - .empty_dict => try stack.append(arena, .{ .seq = .{ .type = .dict, .values = &[_]py.Any{} } }), - .get => |v| try stack.append(arena, .{ .ref = v }), - .empty_list => try stack.append(arena, .{ .seq = .{ .type = .list, .values = &[_]py.Any{} } }), - .put => |v| { - try memo.insert(v, try pop(&stack)); - try stack.append(arena, .{ .ref = v }); - }, - .tuple => try stack.append(arena, blk: { - const popped = try popMark(&stack); - break :blk .{ .seq = .{ .type = .tuple, .values = try arena.dupe(py.Any, popped) } }; - }), - .empty_tuple => try stack.append(arena, .{ .seq = .{ .type = .tuple, .values = &[_]py.Any{} } }), - .setitem => { - const v = try memo.resolve(arena, try pop(&stack), true); - const k = try memo.resolve(arena, try pop(&stack), true); - const top = try lastMut(&stack); - const rtop = try memo.resolveMut(top, true); - switch (rtop.*) { - .global => |obj| { - try append(arena, &obj.kwargs, &.{ k, v }); - }, - .seq => |*dict| { - if (dict.type != .dict) return error.BadStackTopForSetItem; - try append(arena, &dict.values, &.{ k, v }); - }, - else => { - return error.BadStackTopForSetItem; - }, - } - }, - .setitems => { - const popped = try memo.resolveAllRefsIter(arena, 0, try popMark(&stack), true); - const top = try lastMut(&stack); - const rtop = try memo.resolveMut(top, true); - switch (rtop.*) { - .global => |obj| { - try append(arena, &obj.kwargs, popped); - }, - .seq => |*dict| { - if (dict.type != .dict) return error.BadStackTopForSetItems; - try append(arena, &dict.values, popped); - }, - else => { - return error.BadStackTopForSetItems; - }, - } - }, - .proto => |proto| stdx.debug.assert(proto <= MAX_PROTOCOL, "Unsupported protocol {d}", .{proto}), - .tuple1 => try stack.append(arena, blk: { - const tup_values = try arena.alloc(py.Any, 1); - tup_values[0] = try pop(&stack); - break :blk .{ .seq = .{ .type = .tuple, .values = tup_values } }; - }), - .tuple2 => try stack.append(arena, blk: { - const tup_values = try arena.alloc(py.Any, 2); - inline for (0..2) |i| tup_values[(tup_values.len - 1) - i] = try pop(&stack); - break :blk .{ .seq = .{ .type = .tuple, .values = tup_values } }; - }), - .tuple3 => try stack.append(arena, blk: { - const tup_values = try arena.alloc(py.Any, 3); - inline for (0..3) |i| tup_values[(tup_values.len - 1) - i] = try pop(&stack); - break :blk .{ .seq = .{ .type = .tuple, .values = tup_values } }; - }), - .append => { - const v = try pop(&stack); - const top = try lastMut(&stack); - const rtop = try memo.resolveMut(top, true); - switch (rtop.*) { - .global => |obj| { - // can this happen ? - try append(arena, &obj.args, &.{v}); - }, - .seq => |*seq| { - if (seq.type != .list) return error.BadStackTopForAppend; - try append(arena, &seq.values, &.{v}); - }, - else => { - return error.BadStackTopForAppend; - }, - } - }, - .appends => { - const postmark = try popMark(&stack); - const top = try lastMut(&stack); - const rtop = try memo.resolveMut(top, true); - switch (rtop.*) { - .global => try append(arena, &rtop.global.args, postmark), - .seq => |*seq| { - if (seq.type != .list) return error.BadStackTopForAppend; - try append(arena, &seq.values, postmark); - }, - else => { - return error.BadStackTopForAppends; - }, - } - }, - .dict => try stack.append(arena, .{ .seq = .{ - .type = .dict, - .values = try arena.dupe(py.Any, try popMark(&stack)), - } }), - .list => try stack.append(arena, .{ .seq = .{ - .type = .list, - .values = try arena.dupe(py.Any, try popMark(&stack)), - } }), - .inst => |v| try stack.append(arena, .{ .object = try py.Object.init( - arena, - try py.tuple(&.{ .{ .string = v.module }, .{ .string = v.class } }).clone(arena), - try arena.dupe(py.Any, try popMark(&stack)), - &.{}, - ) }), - .obj => try stack.append(arena, blk: { - const mark = try findMark(&stack); - const args = try arena.dupe(py.Any, stack.items[mark + 2 ..]); - const member = stack.items[mark + 1]; - break :blk .{ .object = try py.Object.init(arena, member, args, &.{}) }; - }), - .newobj => try stack.append(arena, blk: { - const args = try arena.alloc(py.Any, 1); - args[0] = try pop(&stack); - break :blk .{ .object = try py.Object.init(arena, try pop(&stack), args, &.{}) }; - }), - .empty_set => try stack.append(arena, .{ .seq = .{ .type = .set, .values = &[_]py.Any{} } }), - .additems => { - const postmark = try popMark(&stack); - const top = try lastMut(&stack); - const rtop = try memo.resolveMut(top, true); - switch (rtop.*) { - .seq => |*seq| { - if (seq.type != .set) return error.BadStackTopForAppend; - try append(arena, &seq.values, postmark); - }, - else => { - return error.BadStackTopForAppends; - }, - } - }, - .frozenset => try stack.append(arena, .{ .seq = .{ - .type = .frozen_set, - .values = try arena.dupe(py.Any, try popMark(&stack)), - } }), - .newobj_ex => try stack.append(arena, blk: { - const kwargs, const args, const cls = .{ try pop(&stack), try pop(&stack), try pop(&stack) }; - break :blk .{ .object = try py.Object.init(arena, cls, args.seq.values, kwargs.seq.values) }; - }), - .stack_global => try stack.append(arena, blk: { - const gn, const mn = .{ - try memo.resolve(arena, try pop(&stack), true), - try memo.resolve(arena, try pop(&stack), true), - }; - const new_seq: py.Sequence = .{ .type = .tuple, .values = try arena.dupe(py.Any, &.{ gn, mn }) }; - break :blk .{ .object = try py.Object.init(arena, .{ .seq = new_seq }, &.{}, &.{}) }; - }), - .memoize => { - const item = stack.getLastOrNull() orelse { - return error.StackUnderrun; - }; - try memo.insert(@intCast(memo.map.count()), try item.clone(arena)); - }, - else => try stack.append(arena, .{ .raw = try op.clone(arena) }), - } - } - if (resolve_refs) { - return try memo.resolveAllRefsIter(arena, 0, stack.items, true); - } - return stack.toOwnedSlice(arena); -} - -fn append(allocator: std.mem.Allocator, current: *[]py.Any, values: []const py.Any) !void { - var array_list = std.ArrayList(py.Any).fromOwnedSlice(current.*); - try array_list.appendSlice(allocator, values); - current.* = array_list.items; -} - -test evaluate { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const allocator = arena.allocator(); - const file = try std.fs.cwd().openFile("zml/aio/torch/simple_test_4.pickle", .{ .mode = .read_only }); - var reader_buffer: [1024]u8 = undefined; - var reader = file.reader(&reader_buffer); - const ops = try pickle.parse(allocator, &reader.interface); - - const vals = try evaluate(allocator, ops, true); - defer allocator.free(vals); - - try std.testing.expect(vals.len == 1); - try std.testing.expect(vals[0] == .seq); - try std.testing.expect(vals[0].seq.type == .dict); - const entries = vals[0].seq.values; - const expected: []const py.Any = &.{ - // Key, followed by its value - .{ .string = "hello" }, .{ .string = "world" }, - .{ .string = "int" }, .{ .int64 = 1 }, - .{ .string = "float" }, .{ .float64 = 3.141592 }, - .{ .string = "list" }, - .{ - .seq = .{ - .type = .list, - .values = @constCast(&[_]py.Any{ - .{ .int64 = 255 }, - .{ .int64 = 1234 }, - .{ .int64 = -123 }, - .{ .int64 = 1_000_000_000 }, - .{ .int64 = 999_000_000_000 }, - .{ .bigint = (try std.math.big.int.Managed.initSet(allocator, 999_000_000_000_000_000_000_000_000_000)).toConst() }, - }), - }, - }, - .{ .string = "bool" }, .{ .boolval = false }, - .{ .string = "tuple" }, - .{ .seq = .{ - .type = .tuple, - .values = @constCast(&[_]py.Any{ - .{ .string = "a" }, - .{ .int64 = 10 }, - }), - } }, - }; - - try std.testing.expectEqualDeep(expected, entries); -} - -pub fn pop(values: *std.ArrayList(py.Any)) error{StackUnderrun}!py.Any { - if (values.items.len == 0) { - return error.StackUnderrun; - } - return values.pop().?; -} - -fn popMark(values: *std.ArrayList(py.Any)) ![]py.Any { - const mark = try findMark(values); - const popping = values.items[mark + 1 ..]; - values.shrinkRetainingCapacity(mark); - return popping; -} - -fn lastMut(values: *std.ArrayList(py.Any)) !*py.Any { - if (values.items.len == 0) { - return error.UnexpectedEmptyStack; - } - return &values.items[values.items.len - 1]; -} - -fn findMark(values: *std.ArrayList(py.Any)) !usize { - const len = values.items.len; - for (0..len) |i| { - const idx = (len - 1) - i; - const val = values.items[idx]; - if (val == .raw and val.raw == .mark) { - return idx; - } - } - return error.MarkNotFound; -} diff --git a/zml/aio/torch/file.zig b/zml/aio/torch/file.zig deleted file mode 100644 index 42da1192..00000000 --- a/zml/aio/torch/file.zig +++ /dev/null @@ -1,523 +0,0 @@ -const std = @import("std"); -const testing = std.testing; - -const async = @import("async"); -const stdx = @import("stdx"); - -const zml = @import("../../zml.zig"); -const HostBuffer = zml.HostBuffer; -const eval = @import("eval.zig"); -const pickle = @import("pickle.zig"); -const py = @import("py.zig"); - -const log = std.log.scoped(.@"zml/aio"); - -test { - std.testing.refAllDecls(@This()); - std.testing.refAllDecls(File); -} - -pub const File = struct { - mmap_file: zml.aio.MemoryMappedFile, - /// Map names to sub file - file_map: std.StringArrayHashMapUnmanaged([]const u8) = .{}, - zip_prefix: []const u8, - pickle_subfile: []const u8, - - const magic = "PK\x03\x04"; - - pub fn init(allocator: std.mem.Allocator, mmap_file: zml.aio.MemoryMappedFile) !File { - var pkl: []const u8 = mmap_file.data; - var zip_prefix: []const u8 = &.{}; - var file_map: std.StringArrayHashMapUnmanaged([]const u8) = .{}; - if (std.mem.eql(u8, mmap_file.data[0..magic.len], magic)) { - // We are dealing with a zip file. - // Let's look for the `data.pkl` file and keep a map of all other files. - // The other files will be the tensor storage and will be reference from `data.pkl`. - var header_parsing_buffer: [4096]u8 = undefined; - - // std.zip requires on a std.fs.File and don't leverage std.Io.Reader directly. - // So we use the synchronous API to parse the headers, - // then we rely only on the memory map data to parse the pickle and load the buffers. - // To mitigate this we use `async.launchBlocking` in `torch.open`. - const raw_file: std.fs.File = .{ .handle = mmap_file.file._handle }; - var reader = raw_file.reader(&header_parsing_buffer); - var it: std.zip.Iterator = try .init(&reader); - - while (try it.next()) |header| { - if (header.filename_len == 0) { - continue; - } - if (header.compression_method != .store) { - return error.Unsupported; - } - - const filename = mmap_file.data[header.header_zip_offset + @sizeOf(std.zip.CentralDirectoryFileHeader) ..][0..header.filename_len]; - - var local_reader: std.Io.Reader = .fixed(mmap_file.data); - local_reader.discardAll(header.file_offset) catch return error.InvalidZipFile; - const local_header = local_reader.takeStruct(std.zip.LocalFileHeader, .little) catch return error.InvalidZipFile; - local_reader.discardAll(local_header.filename_len) catch return error.InvalidZipFile; - local_reader.discardAll(local_header.extra_len) catch return error.InvalidZipFile; - - // normalize path separators - const file_content = mmap_file.data[local_reader.seek..][0..header.compressed_size]; - const my_filename: []u8 = try allocator.dupe(u8, filename); - std.mem.replaceScalar(u8, my_filename, '\\', '/'); - - try file_map.put(allocator, my_filename, file_content); - - if (std.mem.endsWith(u8, filename, "data.pkl")) { - pkl = file_content; - zip_prefix = filename[0 .. filename.len - "data.pkl".len]; - } - } - - if (pkl.len == 0) { - log.err("Could not find file ending in `data.pkl` in archive", .{}); - return error.PickleNotFound; - } - } - - return .{ - .mmap_file = mmap_file, - .file_map = file_map, - .pickle_subfile = pkl, - .zip_prefix = zip_prefix, - }; - } - - pub fn close(self: *File) void { - self.mmap_file.deinit(); - } - - pub fn parsePickle(self: *File, allocator: std.mem.Allocator) ![]const pickle.Op { - var reader: std.Io.Reader = .fixed(self.pickle_subfile); - return try pickle.parse(allocator, &reader); - } - - fn basicTypeCheck(object: *const py.Object, module: []const u8, class: []const u8) bool { - return switch (object.member) { - .raw => |raw| return (std.mem.eql(u8, module, raw.global.module) and - std.mem.eql(u8, class, raw.global.class)), - else => false, - }; - } - - pub fn parseModel(self: File, values: []const py.Any, store: *zml.aio.BufferStore) !void { - var prefix_buf: [1024]u8 = undefined; - const allocator = store.arena.allocator(); - for (values) |item| { - try self.parseValue(allocator, store, .initBuffer(&prefix_buf), item); - } - } - - pub fn parseValue(self: File, allocator: std.mem.Allocator, store: *zml.aio.BufferStore, prefix: std.ArrayList(u8), v: py.Any) !void { - // log.warn("Parsing {}", .{v}); - switch (v) { - .app, .object, .global => |object| { - if (!(try self.parseTorchGlobal(allocator, store, prefix, v))) { - try self.parseValue(allocator, store, prefix, object.member); - for (object.args) |item| { - try self.parseValue(allocator, store, prefix, item); - } - if (object.kwargs.len % 2 != 0) return error.InvalidInput; - const n_kwargs = @divExact(object.kwargs.len, 2); - - for (0..n_kwargs) |i| { - const key, const val = object.kwargs[2 * i ..][0..2].*; - // kwargs can only be keyed by string. - if (key != .string) return error.InvalidInput; - // Handle Pytorch specific fields - const s = key.string; - if (std.mem.eql(u8, s, "_modules") or std.mem.eql(u8, s, "_parameters") or std.mem.eql(u8, s, "_buffers")) { - try self.parseValue(allocator, store, prefix, val); - } else { - var new_prefix = prefix; - if (prefix.items.len > 0) { - new_prefix.appendAssumeCapacity('.'); - } - new_prefix.appendSliceAssumeCapacity(s); - try self.parseValue(allocator, store, new_prefix, val); - } - } - } - }, - .set_state => |set_state| { - // `set_state` contains info about python struct being constructed - switch (set_state.obj) { - .object => |obj| switch (obj.member) { - .raw => |raw| switch (raw) { - .global => |global| { - // in this case, we can capture the name of the python type - // which can be used for codegen (e.g. `torch.nn.modules.conv.Conv2d`) - var new_prefix = prefix; - if (prefix.items.len > 0) { - new_prefix.appendAssumeCapacity('.'); - } - new_prefix.appendSliceAssumeCapacity("_gen_type_helper"); - const key = try allocator.dupe(u8, new_prefix.items); - const d = try store._metadata.getOrPut(allocator, key); - if (d.found_existing) { - log.err("Duplicate key: {s}", .{new_prefix.items}); - allocator.free(key); - } else { - const val = try std.mem.join(allocator, ".", &.{ global.module, global.class }); - d.value_ptr.* = .{ .string = val }; - } - }, - else => try self.parseValue(allocator, store, prefix, set_state.obj), // parse normally - }, - else => try self.parseValue(allocator, store, prefix, set_state.obj), // parse normally - }, - else => try self.parseValue(allocator, store, prefix, set_state.obj), // parse normally - } - try self.parseValue(allocator, store, prefix, set_state.state); - }, - .pers_id => |pers_id| try self.parseValue(allocator, store, prefix, pers_id.ref), - .seq => |seq| { - switch (seq.type) { - .list, .tuple, .set, .frozen_set => { - if (seq.values.len == 0) return; - var valid_slice = true; - switch (seq.values[0]) { - inline .int64, .float64, .boolval => |val0, tag| { - const ItemType = switch (tag) { - .int64 => i64, - .float64 => f64, - .boolval => bool, - else => unreachable, - }; - var values: std.ArrayList(ItemType) = .{}; - try values.append(allocator, val0); - for (seq.values[1..], 1..) |val, i| { - if (std.meta.activeTag(val) != tag) valid_slice = false; - if (valid_slice) { - try values.append(allocator, @field(val, @tagName(tag))); - } else { - var new_prefix = prefix; - if (prefix.items.len > 0) { - new_prefix.appendAssumeCapacity('.'); - } - new_prefix.items.len += std.fmt.printInt(new_prefix.unusedCapacitySlice(), i, 10, .lower, .{}); - try self.parseValue(allocator, store, new_prefix, val); - } - } - - if (valid_slice) { - try store._metadata.put( - allocator, - try allocator.dupe(u8, prefix.items), - try zml.aio.Metadata.copySlice(allocator, values.items), - ); - } else { - for (values.items, 0..) |val, i| { - var new_prefix = prefix; - if (prefix.items.len > 0) { - new_prefix.appendAssumeCapacity('.'); - } - new_prefix.items.len += std.fmt.printInt(new_prefix.unusedCapacitySlice(), i, 10, .lower, .{}); - const new_tag = switch (tag) { - .int64 => "int", - .float64 => "float", - .boolval => "bool", - else => unreachable, // we are already inside a switch - }; - try store._metadata.put(allocator, try allocator.dupe(u8, new_prefix.items), @unionInit(zml.aio.Metadata, new_tag, val)); - } - } - }, - else => { - for (seq.values, 0..) |item, i| { - var new_prefix = prefix; - if (v.isPrimitive()) { - if (prefix.items.len > 0) { - new_prefix.appendAssumeCapacity('.'); - } - new_prefix.items.len += std.fmt.printInt(new_prefix.unusedCapacitySlice(), i, 10, .lower, .{}); - } - try self.parseValue(allocator, store, new_prefix, item); - } - }, - } - }, - .dict => { - const n = @divExact(seq.values.len, 2); - log.debug("found dict with {} entries", .{n}); - for (0..n) |i| { - const key, const val = seq.values[2 * i ..][0..2].*; - switch (key) { - .string => |s| { - // Handle Pytorch specific fields. - if (std.mem.eql(u8, s, "_modules") or std.mem.eql(u8, s, "_parameters") or std.mem.eql(u8, s, "_buffers")) { - try self.parseValue(allocator, store, prefix, val); - } else { - var new_prefix = prefix; - if (prefix.items.len > 0) { - new_prefix.appendAssumeCapacity('.'); - } - new_prefix.appendSliceAssumeCapacity(s); - - try self.parseValue(allocator, store, new_prefix, val); - } - }, - .int64 => |int| { - var new_prefix = prefix; - if (prefix.items.len > 0) { - new_prefix.appendAssumeCapacity('.'); - } - new_prefix.items.len += std.fmt.printInt(new_prefix.unusedCapacitySlice(), int, 10, .lower, .{}); - try self.parseValue(allocator, store, new_prefix, val); - }, - inline else => |_, tag| { - log.debug("Ignoring unsupported key type found in torch file: {s}", .{@tagName(tag)}); - continue; - }, - } - } - }, - } - }, - .bytes => |val| { - const key = try allocator.dupe(u8, prefix.items); - const d = try store._metadata.getOrPut(allocator, key); - if (d.found_existing) { - log.warn("Duplicate key: {s}", .{prefix.items}); - allocator.free(key); - } else d.value_ptr.* = .{ .string = val }; - }, - inline .float64, .int64, .boolval, .bigint, .string => |val| { - const key = try allocator.dupe(u8, prefix.items); - const d = try store._metadata.getOrPut(allocator, key); - if (d.found_existing) { - log.warn("Duplicate key: {s}", .{prefix.items}); - allocator.free(key); - } else { - d.value_ptr.* = zml.aio.Metadata.wrap(val); - } - }, - else => {}, - } - } - - fn parseTorchGlobal(self: File, allocator: std.mem.Allocator, store: *zml.aio.BufferStore, prefix: std.ArrayList(u8), v: py.Any) !bool { - return switch (v) { - .global => |object| { - if (try self.parseTensor(allocator, object)) |host_buffer| { - const key = try allocator.dupe(u8, prefix.items); - const entry = try store.buffers.getOrPut(allocator, key); - if (entry.found_existing) { - log.warn("Duplicate key: {s}", .{prefix.items}); - allocator.free(key); - } - entry.value_ptr.* = host_buffer; - return true; - } else if (basicTypeCheck(object, "torch", "Size")) { - const size = object.args; - const key = try allocator.dupe(u8, prefix.items); - const entry = try store._metadata.getOrPut(allocator, key); - if (entry.found_existing) { - log.warn("Duplicate key: {s}", .{prefix.items}); - allocator.free(key); - } - const d = try allocator.alloc(i64, size.len); - for (d, 0..) |*di, i| di.* = size[i].int64; - entry.value_ptr.* = .{ .array_int = d }; - return true; - } else if (basicTypeCheck(object, "fractions", "Fraction")) { - const fraction_str = object.args[0].string; - if (std.mem.indexOfScalar(u8, fraction_str, '/')) |split_idx| { - { - var new_prefix = prefix; - new_prefix.appendSliceAssumeCapacity(".numerator"); - try store._metadata.put(allocator, try allocator.dupe(u8, new_prefix.items), .{ .int = try std.fmt.parseInt(i64, fraction_str[0..split_idx], 10) }); - } - { - var new_prefix = prefix; - new_prefix.appendSliceAssumeCapacity(".denominator"); - try store._metadata.put(allocator, try allocator.dupe(u8, new_prefix.items), .{ .int = try std.fmt.parseInt(i64, fraction_str[split_idx + 1 ..], 10) }); - } - return true; - } - } - return false; - }, - else => false, - }; - } - - fn parseTensor(self: File, tmp_allocator: std.mem.Allocator, object: *py.Object) !?zml.HostBuffer { - if (!basicTypeCheck(object, "torch._utils", "_rebuild_tensor_v2")) { - return null; - } - - const args = object.args; - if (args.len < 4 or - args[0] != .pers_id or - args[1] != .int64 or - args[2] != .seq or args[2].seq.type != .tuple or - args[3] != .seq or args[3].seq.type != .tuple) - { - log.err("Unexpected py.Any in call to torch._utils._rebuild_tensor_v2: {}", .{object.*}); - return error.InvalidInput; - } - - const pid: *py.PersId = args[0].pers_id; - var offset: u64 = @intCast(args[1].int64); - const raw_dims: py.Sequence = args[2].seq; - const raw_strides: py.Sequence = args[3].seq; - const dims = try parseDims(raw_dims.values); - var strides = try parseDims(raw_strides.values); - - const dtype, const storage_file = try parseStorage(pid.ref); - // Pytorch store "item" strides, while ZML uses byte strides. - for (strides.slice()) |*s| s.* *= dtype.sizeOf(); - // Same thing for the offset. - offset = offset * dtype.sizeOf(); - - const filename = try std.mem.join(tmp_allocator, "", &.{ self.zip_prefix, "data/", storage_file }); - defer tmp_allocator.free(filename); - - // The offset in the pickle is the offset inside the storage_file. - // But .pt are made of several files, so we need to append the file offset. - const storage = try self.getStorage(filename); - return HostBuffer.fromStridedSlice( - zml.Shape.init(dims.constSlice(), dtype), - storage[offset..], - strides.constSlice(), - ); - } - - fn parseStorage(val: py.Any) !struct { zml.DataType, []const u8 } { - if (val != .seq) return error.InvalidInput; - const sargs = val.seq.values; - if (val.seq.type == .tuple and - sargs.len >= 5 and - sargs[0] == .string and std.mem.eql(u8, sargs[0].string, "storage") and - sargs[1] == .raw and sargs[1].raw == .global and - sargs[2] == .string and - sargs[3] == .string) - { - const op = sargs[1].raw.global; - const storage_file = sargs[2].string; - // const sdev = sargs[3].string; - if (!std.mem.eql(u8, "torch", op.module) or - !std.mem.endsWith(u8, op.class, "Storage")) - return error.InvalidInput; - - return .{ - try storageToDtype(op.class), - storage_file, - }; - } else { - return error.InvalidInput; - } - } - - /// Given the name of one of the files in the .pt tarball, - /// return the slice of the memory-mapped .pt corresponding to it. - fn getStorage(self: File, filename: []const u8) ![]const u8 { - return self.file_map.get(filename) orelse { - std.log.err("Could not find file ending in `{s}` in archive", .{filename}); - return error.TensorNotFound; - }; - } - - fn parseDims(values: []py.Any) error{InvalidInput}!zml.Shape.DimsArray { - stdx.debug.assert(values.len <= zml.Tensor.MAX_RANK, "Found Pytorch tensor with unsupported rank {}", .{values.len}); - var result: zml.Shape.DimsArray = .{}; - for (values) |val| { - switch (val) { - .int64 => |d| result.appendAssumeCapacity(d), - else => return error.InvalidInput, - } - } - return result; - } -}; - -/// Convert from a torch.Storage to a `zml.DataType`. -/// TODO: make this future proof, storage type are going to get replaced with torch.UntypedStorage -/// See https://pytorch.org/docs/stable/storage.html -fn storageToDtype(storage_type: []const u8) !zml.DataType { - const torch_type = storage_type[0 .. storage_type.len - "Storage".len]; - const map = std.StaticStringMap(zml.DataType).initComptime(.{ - .{ "Double", .f64 }, - .{ "Float", .f32 }, - .{ "Half", .f16 }, - .{ "Long", .i64 }, - .{ "Int", .i32 }, - .{ "Short", .i16 }, - .{ "Char", .i8 }, - .{ "Byte", .u8 }, - .{ "Bool", .bool }, - .{ "BFloat16", .bf16 }, - .{ "ComplexDouble", .c128 }, - .{ "ComplexFloat", .c64 }, - // QUInt8Storage - // QInt8Storage - // QInt32Storage - // QUInt4x2Storage - // QUInt2x4Storage - }); - - return map.get(torch_type) orelse { - log.err("Unsupported torch storage type: {s}", .{storage_type}); - return error.UnsupportedDataType; - }; -} - -test "Read pickle (zipped)" { - // test file created with following python snippet: - // - // import torch - // torch.manual_seed(0) - // model = torch.nn.Conv2d(2, 2, 3, stride=2, padding=[2, 4], dtype=torch.float16) - // tensor = torch.tensor([[2, 4, 3, 2]], dtype=torch.uint8) - // torch.save({ "model": model, "tensor": tensor}, "simple.pt") - const file = try async.File.open("zml/aio/torch/simple.pt", .{ .mode = .read_only }); - const mmap_file = try zml.aio.MemoryMappedFile.init(file); - var store = try zml.aio.BufferStore.initWithFiles(testing.allocator, &.{mmap_file}); - defer store.deinit(); - - { - var arena = std.heap.ArenaAllocator.init(testing.allocator); - defer arena.deinit(); - var torch_file = try File.init(arena.allocator(), mmap_file); - // We don't close the file directly, it will be closed by the store. - - const ops = try torch_file.parsePickle(arena.allocator()); - try std.testing.expectEqual(302, ops.len); - - const py_values = try eval.evaluate(arena.allocator(), ops, true); - try torch_file.parseModel(py_values, &store); - } - - // now we have freed the arena. - // all data needed should have been copied into the store arena. - try zml.testing.expectEqualShapes( - zml.Shape.init(.{ 1, 4 }, .u8), - store.get("tensor").?.shape(), - ); - try zml.testing.expectEqualShapes( - zml.Shape.init(.{ 2, 2, 3, 3 }, .f16), - store.get("model.weight").?.shape(), - ); - try zml.testing.expectEqualShapes( - zml.Shape.init(.{2}, .f16), - store.get("model.bias").?.shape(), - ); -} - -fn isBadFilename(filename: []const u8) bool { - if (filename.len == 0 or filename[0] == '/') - return true; - - var it = std.mem.splitScalar(u8, filename, '/'); - while (it.next()) |part| { - if (std.mem.eql(u8, part, "..")) - return true; - } - - return false; -} diff --git a/zml/aio/torch/pickle.zig b/zml/aio/torch/pickle.zig deleted file mode 100644 index 8172ddc7..00000000 --- a/zml/aio/torch/pickle.zig +++ /dev/null @@ -1,1072 +0,0 @@ -const std = @import("std"); - -const log = std.log.scoped(.@"zml/aio"); - -/// All possible pickle operators. -/// Reference: https://github.com/python/cpython/blob/3.13/Lib/pickletools.py -pub const OpCode = enum(u8) { - /// Push an integer or bool. - /// - /// The argument is a newline-terminated decimal literal string. - /// - /// The intent may have been that this always fit in a short Python int, - /// but INT can be generated in pickles written on a 64-bit box that - /// require a Python long on a 32-bit box. The difference between this - /// and LONG then is that INT skips a trailing 'L', and produces a short - /// int whenever possible. - /// - /// Another difference is due to that, when bool was introduced as a - /// distinct type in 2.3, builtin names True and False were also added to - /// 2.2.2, mapping to ints 1 and 0. For compatibility in both directions, - /// True gets pickled as INT + "I01\n", and False as INT + "I00\n". - /// Leading zeroes are never produced for a genuine integer. The 2.3 - /// (and later) unpicklers special-case these and return bool instead; - /// earlier unpicklers ignore the leading "0" and return the int. - int = 'I', - - /// Push a four-byte signed integer. - /// Introduced in protocol 1. - /// - /// This handles the full range of Python (short) integers on a 32-bit - /// box, directly as binary bytes (1 for the opcode and 4 for the integer). - /// If the integer is non-negative and fits in 1 or 2 bytes, pickling via - /// BININT1 or BININT2 saves space. - binint = 'J', - - /// Push a one-byte unsigned integer. - /// Introduced in protocol 1. - /// - /// This is a space optimization for pickling very small non-negative ints, - /// in range(256). - binint1 = 'K', - - /// Push a two-byte unsigned integer. - /// Introduced in protocol 1. - /// - /// This is a space optimization for pickling small positive ints, in - /// range(256, 2**16). Integers in range(256) can also be pickled via - /// BININT2, but BININT1 instead saves a byte. - binint2 = 'M', - - /// Push a long integer. - /// - /// The same as INT, except that the literal ends with 'L', and always - /// unpickles to a Python long. There doesn't seem a real purpose to the - /// trailing 'L'. - /// - /// Note that LONG takes time quadratic in the number of digits when - /// unpickling (this is simply due to the nature of decimal->binary - /// conversion). Proto 2 added linear-time (in C; still quadratic-time - /// in Python) LONG1 and LONG4 opcodes. - long = 'L', - - /// Long integer using one-byte length. - /// Introduced in protocol 2. - /// - /// A more efficient encoding of a Python long; the long1 encoding - long1 = 0x8a, - - /// Long integer using four-byte length. - /// Introduced in protocol 2. - /// - /// A more efficient encoding of a Python long; the long4 encoding - long4 = 0x8b, - - /// Push a Python string object. - /// - /// The argument is a repr-style string, with bracketing quote characters, - /// and perhaps embedded escapes. The argument extends until the next - /// newline character. These are usually decoded into a str instance - /// using the encoding given to the Unpickler constructor. or the default, - /// 'ASCII'. If the encoding given was 'bytes' however, they will be - /// decoded as bytes object instead. - string = 'S', - - /// Push a Python string object. - /// Introduced in protocol 1. - /// - /// There are two arguments: the first is a 4-byte little-endian - /// signed int giving the number of bytes in the string, and the - /// second is that many bytes, which are taken literally as the string - /// content. These are usually decoded into a str instance using the - /// encoding given to the Unpickler constructor. or the default, - /// 'ASCII'. If the encoding given was 'bytes' however, they will be - /// decoded as bytes object instead. - binstring = 'T', - - /// Push a Python string object. - /// Introduced in protocol 1. - /// - /// There are two arguments: the first is a 1-byte unsigned int giving - /// the number of bytes in the string, and the second is that many - /// bytes, which are taken literally as the string content. These are - /// usually decoded into a str instance using the encoding given to - /// the Unpickler constructor. or the default, 'ASCII'. If the - /// encoding given was 'bytes' however, they will be decoded as bytes - /// object instead. - short_binstring = 'U', - - /// Push a Python bytes object. - /// Introduced in protocol 3. - /// - /// There are two arguments: the first is a 4-byte little-endian unsigned int - /// giving the number of bytes, and the second is that many bytes, which are - /// taken literally as the bytes content. - binbytes = 'B', - - /// Push a Python bytes object. - /// Introduced in protocol 3. - /// - /// There are two arguments: the first is a 1-byte unsigned int giving - /// the number of bytes, and the second is that many bytes, which are taken - /// literally as the string content. - short_binbytes = 'C', - - /// Push a Python bytes object. - /// Introduced in protocol 4. - /// - /// There are two arguments: the first is an 8-byte unsigned int giving - /// the number of bytes in the string, and the second is that many bytes, - /// which are taken literally as the string content. - binbytes8 = 0x8e, - - /// Push a Python bytearray object. - /// Introduced in protocol 5. - /// - /// There are two arguments: the first is an 8-byte unsigned int giving - /// the number of bytes in the bytearray, and the second is that many bytes, - /// which are taken literally as the bytearray content. - bytearray8 = 0x96, - - /// Introduced in protocol 5. - next_buffer = 0x97, - - /// Introduced in protocol 5. - readonly_buffer = 0x98, - - none = 'N', - - /// Introduced in protocol 2. - newtrue = 0x88, - - /// Introduced in protocol 2. - newfalse = 0x89, - - /// Push a Python Unicode string object. - /// - /// The argument is a raw-unicode-escape encoding of a Unicode string, - /// and so may contain embedded escape sequences. The argument extends - /// until the next newline character. - unicode = 'V', - - /// Push a Python Unicode string object. - /// Introduced in protocol 4. - /// - /// There are two arguments: the first is a 1-byte little-endian signed int - /// giving the number of bytes in the string. The second is that many - /// bytes, and is the UTF-8 encoding of the Unicode string. - short_binunicode = 0x8c, - - /// Push a Python Unicode string object. - /// Introduced in protocol 1. - /// - /// There are two arguments: the first is a 4-byte little-endian unsigned int - /// giving the number of bytes in the string. The second is that many - /// bytes, and is the UTF-8 encoding of the Unicode string. - binunicode = 'X', - - /// Push a Python Unicode string object. - /// Introduced in protocol 4. - /// - /// There are two arguments: the first is an 8-byte little-endian signed int - /// giving the number of bytes in the string. The second is that many - /// bytes, and is the UTF-8 encoding of the Unicode string. - binunicode8 = 0x8d, - - /// Newline-terminated decimal float literal. - /// - /// The argument is repr(a_float), and in general requires 17 significant - /// digits for roundtrip conversion to be an identity (this is so for - /// IEEE-754 double precision values, which is what Python float maps to - /// on most boxes). - /// - /// In general, FLOAT cannot be used to transport infinities, NaNs, or - /// minus zero across boxes (or even on a single box, if the platform C - /// library can't read the strings it produces for such things -- Windows - /// is like that), but may do less damage than BINFLOAT on boxes with - /// greater precision or dynamic range than IEEE-754 double. - float = 'F', - - /// Float stored in binary form, with 8 bytes of data. - /// Introduced in protocol 1. - /// - /// This generally requires less than half the space of FLOAT encoding. - /// In general, BINFLOAT cannot be used to transport infinities, NaNs, or - /// minus zero, raises an exception if the exponent exceeds the range of - /// an IEEE-754 double, and retains no more than 53 bits of precision (if - /// there are more than that, "add a half and chop" rounding is used to - /// cut it back to 53 significant bits). - binfloat = 'G', - - /// Introduced in protocol 1. - empty_list = ']', - - /// Append an object to a list. - /// - /// Stack before: ... pylist anyobject - /// Stack after: ... pylist+[anyobject] - /// - /// although pylist is really extended in-place. - append = 'a', - - /// Extend a list by a slice of stack objects. - /// Introduced in protocol 1. - /// - /// Stack before: ... pylist markobject stackslice - /// Stack after: ... pylist+stackslice - /// - /// although pylist is really extended in-place. - appends = 'e', - - /// Build a list out of the topmost stack slice, after markobject. - /// - /// All the stack entries following the topmost markobject are placed into - /// a single Python list, which single list object replaces all of the - /// stack from the topmost markobject onward. For example, - /// - /// Stack before: ... markobject 1 2 3 'abc' - /// Stack after: ... [1, 2, 3, 'abc'] - list = 'l', - - /// Introduced in protocol 1. - empty_tuple = ')', - - /// Build a tuple out of the topmost stack slice, after markobject. - /// - /// All the stack entries following the topmost markobject are placed into - /// a single Python tuple, which single tuple object replaces all of the - /// stack from the topmost markobject onward. For example, - /// - /// Stack before: ... markobject 1 2 3 'abc' - /// Stack after: ... (1, 2, 3, 'abc') - tuple = 't', - - /// Build a one-tuple out of the topmost item on the stack. - /// Introduced in protocol 2. - /// - /// This code pops one value off the stack and pushes a tuple of - /// length 1 whose one item is that value back onto it. In other - /// words: - /// - /// stack[-1] = tuple(stack[-1:]) - tuple1 = 0x85, - - /// Build a two-tuple out of the top two items on the stack. - /// Introduced in protocol 2. - /// - /// This code pops two values off the stack and pushes a tuple of - /// length 2 whose items are those values back onto it. In other - /// words: - /// - /// stack[-2:] = [tuple(stack[-2:])] - tuple2 = 0x86, - - /// Build a three-tuple out of the top three items on the stack. - /// Introduced in protocol 2. - /// - /// This code pops three values off the stack and pushes a tuple of - /// length 3 whose items are those values back onto it. In other - /// words: - /// - /// stack[-3:] = [tuple(stack[-3:])] - tuple3 = 0x87, - - /// Introduced in protocol 1. - empty_dict = '}', - - /// Build a dict out of the topmost stack slice, after markobject. - /// - /// All the stack entries following the topmost markobject are placed into - /// a single Python dict, which single dict object replaces all of the - /// stack from the topmost markobject onward. The stack slice alternates - /// key, value, key, value, .... For example, - /// - /// Stack before: ... markobject 1 2 3 'abc' - /// Stack after: ... {1: 2, 3: 'abc'} - dict = 'd', - - /// Add a key+value pair to an existing dict. - /// - /// Stack before: ... pydict key value - /// Stack after: ... pydict - /// - /// where pydict has been modified via pydict[key] = value. - setitem = 's', - - /// Add an arbitrary number of key+value pairs to an existing dict. - /// Introduced in protocol 1. - /// - /// The slice of the stack following the topmost markobject is taken as - /// an alternating sequence of keys and values, added to the dict - /// immediately under the topmost markobject. Everything at and after the - /// topmost markobject is popped, leaving the mutated dict at the top - /// of the stack. - /// - /// Stack before: ... pydict markobject key_1 value_1 ... key_n value_n - /// Stack after: ... pydict - /// - /// where pydict has been modified via pydict[key_i] = value_i for i in - /// 1, 2, ..., n, and in that order. - setitems = 'u', - - /// Introduced in protocol 4. - empty_set = 0x8f, - - /// Add an arbitrary number of items to an existing set. - /// Introduced in protocol 4. - /// - /// The slice of the stack following the topmost markobject is taken as - /// a sequence of items, added to the set immediately under the topmost - /// markobject. Everything at and after the topmost markobject is popped, - /// leaving the mutated set at the top of the stack. - /// - /// Stack before: ... pyset markobject item_1 ... item_n - /// Stack after: ... pyset - /// - /// where pyset has been modified via pyset.add(item_i) = item_i for i in - /// 1, 2, ..., n, and in that order. - additems = 0x90, - - /// Build a frozenset out of the topmost slice, after markobject. - /// Introduced in protocol 4. - /// - /// All the stack entries following the topmost markobject are placed into - /// a single Python frozenset, which single frozenset object replaces all - /// of the stack from the topmost markobject onward. For example, - /// - /// Stack before: ... markobject 1 2 3 - /// Stack after: ... frozenset({1, 2, 3}) - frozenset = 0x91, - - pop = '0', - - dup = '2', - - /// Push markobject onto the stack. - /// - /// markobject is a unique object, used by other opcodes to identify a - /// region of the stack containing a variable number of objects for them - /// to work on. See markobject.doc for more detail. - mark = '(', - - /// Pop all the stack objects at and above the topmost markobject. - /// Introduced in protocol 1. - /// - /// When an opcode using a variable number of stack objects is done, - /// POP_MARK is used to remove those objects, and to remove the markobject - /// that delimited their starting position on the stack. - pop_mark = '1', - - /// Read an object from the memo and push it on the stack. - /// - /// The index of the memo object to push is given by the newline-terminated - /// decimal string following. BINGET and LONG_BINGET are space-optimized - /// versions. - get = 'g', - - /// Read an object from the memo and push it on the stack. - /// Introduced in protocol 1. - /// - /// The index of the memo object to push is given by the 1-byte unsigned - /// integer following. - binget = 'h', - - /// Read an object from the memo and push it on the stack. - /// Introduced in protocol 1. - /// - /// The index of the memo object to push is given by the 4-byte unsigned - /// little-endian integer following. - long_binget = 'j', - - /// Store the stack top into the memo. The stack is not popped. - /// - /// The index of the memo location to write into is given by the newline- - /// terminated decimal string following. BINPUT and LONG_BINPUT are - /// space-optimized versions. - put = 'p', - - /// Store the stack top into the memo. The stack is not popped. - /// Introduced in protocol 1. - /// - /// The index of the memo location to write into is given by the 1-byte - /// unsigned integer following. - binput = 'q', - - /// Store the stack top into the memo. The stack is not popped. - /// Introduced in protocol 1. - /// - /// The index of the memo location to write into is given by the 4-byte - /// unsigned little-endian integer following. - long_binput = 'r', - - /// Store the stack top into the memo. The stack is not popped. - /// Introduced in protocol 4. - /// - /// The index of the memo location to write is the number of - /// elements currently present in the memo. - memoize = 0x94, - - /// Extension code. - /// Introduced in protocol 2. - /// - /// This code and the similar EXT2 and EXT4 allow using a registry - /// of popular objects that are pickled by name, typically classes. - /// It is envisioned that through a global negotiation and - /// registration process, third parties can set up a mapping between - /// ints and object names. - /// - /// In order to guarantee pickle interchangeability, the extension - /// code registry ought to be global, although a range of codes may - /// be reserved for private use. - /// - /// EXT1 has a 1-byte integer argument. This is used to index into the - /// extension registry, and the object at that index is pushed on the stack. - ext1 = 0x82, - - /// Extension code. - /// Introduced in protocol 2. - /// - /// See EXT1. EXT2 has a two-byte integer argument. - ext2 = 0x83, - - /// Extension code. - /// Introduced in protocol 2. - /// - /// See EXT1. EXT4 has a four-byte integer argument. - ext4 = 0x84, - - /// Push a global object (module.attr) on the stack. - /// - /// Two newline-terminated strings follow the GLOBAL opcode. The first is - /// taken as a module name, and the second as a class name. The class - /// object module.class is pushed on the stack. More accurately, the - /// object returned by self.find_class(module, class) is pushed on the - /// stack, so unpickling subclasses can override this form of lookup. - global = 'c', - - /// Push a global object (module.attr) on the stack. - /// Introduced in protocol 4. - stack_global = 0x93, - - /// Push an object built from a callable and an argument tuple. - /// - /// The opcode is named to remind of the __reduce__() method. - /// - /// Stack before: ... callable pytuple - /// Stack after: ... callable(*pytuple) - /// - /// The callable and the argument tuple are the first two items returned - /// by a __reduce__ method. Applying the callable to the argtuple is - /// supposed to reproduce the original object, or at least get it started. - /// If the __reduce__ method returns a 3-tuple, the last component is an - /// argument to be passed to the object's __setstate__, and then the REDUCE - /// opcode is followed by code to create setstate's argument, and then a - /// BUILD opcode to apply __setstate__ to that argument. - /// - /// If not isinstance(callable, type), REDUCE complains unless the - /// callable has been registered with the copyreg module's - /// safe_constructors dict, or the callable has a magic - /// '__safe_for_unpickling__' attribute with a true value. I'm not sure - /// why it does this, but I've sure seen this complaint often enough when - /// I didn't want to . - reduce = 'R', - - /// Finish building an object, via __setstate__ or dict update. - /// - /// Stack before: ... anyobject argument - /// Stack after: ... anyobject - /// - /// where anyobject may have been mutated, as follows: - /// - /// If the object has a __setstate__ method, - /// - /// anyobject.__setstate__(argument) - /// - /// is called. - /// - /// Else the argument must be a dict, the object must have a __dict__, and - /// the object is updated via - /// - /// anyobject.__dict__.update(argument) - build = 'b', - - /// Build a class instance. - /// - /// This is the protocol 0 version of protocol 1's OBJ opcode. - /// INST is followed by two newline-terminated strings, giving a - /// module and class name, just as for the GLOBAL opcode (and see - /// GLOBAL for more details about that). self.find_class(module, name) - /// is used to get a class object. - /// - /// In addition, all the objects on the stack following the topmost - /// markobject are gathered into a tuple and popped (along with the - /// topmost markobject), just as for the TUPLE opcode. - /// - /// Now it gets complicated. If all of these are true: - /// - /// + The argtuple is empty (markobject was at the top of the stack - /// at the start). - /// - /// + The class object does not have a __getinitargs__ attribute. - /// - /// then we want to create an old-style class instance without invoking - /// its __init__() method (pickle has waffled on this over the years; not - /// calling __init__() is current wisdom). In this case, an instance of - /// an old-style dummy class is created, and then we try to rebind its - /// __class__ attribute to the desired class object. If this succeeds, - /// the new instance object is pushed on the stack, and we're done. - /// - /// Else (the argtuple is not empty, it's not an old-style class object, - /// or the class object does have a __getinitargs__ attribute), the code - /// first insists that the class object have a __safe_for_unpickling__ - /// attribute. Unlike as for the __safe_for_unpickling__ check in REDUCE, - /// it doesn't matter whether this attribute has a true or false value, it - /// only matters whether it exists (XXX this is a bug). If - /// __safe_for_unpickling__ doesn't exist, UnpicklingError is raised. - /// - /// Else (the class object does have a __safe_for_unpickling__ attr), - /// the class object obtained from INST's arguments is applied to the - /// argtuple obtained from the stack, and the resulting instance object - /// is pushed on the stack. - /// - /// NOTE: checks for __safe_for_unpickling__ went away in Python 2.3. - /// NOTE: the distinction between old-style and new-style classes does - /// not make sense in Python 3. - inst = 'i', - - /// Build a class instance. - /// Introduced in protocol 1. - /// - /// This is the protocol 1 version of protocol 0's INST opcode, and is - /// very much like it. The major difference is that the class object - /// is taken off the stack, allowing it to be retrieved from the memo - /// repeatedly if several instances of the same class are created. This - /// can be much more efficient (in both time and space) than repeatedly - /// embedding the module and class names in INST opcodes. - /// - /// Unlike INST, OBJ takes no arguments from the opcode stream. Instead - /// the class object is taken off the stack, immediately above the - /// topmost markobject: - /// - /// Stack before: ... markobject classobject stackslice - /// Stack after: ... new_instance_object - /// - /// As for INST, the remainder of the stack above the markobject is - /// gathered into an argument tuple, and then the logic seems identical, - /// except that no __safe_for_unpickling__ check is done (XXX this is - /// a bug). See INST for the gory details. - /// - /// NOTE: In Python 2.3, INST and OBJ are identical except for how they - /// get the class object. That was always the intent; the implementations - /// had diverged for accidental reasons. - obj = 'o', - - /// Build an object instance. - /// Introduced in protocol 2. - /// - /// The stack before should be thought of as containing a class - /// object followed by an argument tuple (the tuple being the stack - /// top). Call these cls and args. They are popped off the stack, - /// and the value returned by cls.__new__(cls, *args) is pushed back - /// onto the stack. - newobj = 0x81, - - /// Build an object instance. - /// Introduced in protocol 4. - /// - /// The stack before should be thought of as containing a class - /// object followed by an argument tuple and by a keyword argument dict - /// (the dict being the stack top). Call these cls and args. They are - /// popped off the stack, and the value returned by - /// cls.__new__(cls, *args, *kwargs) is pushed back onto the stack. - newobj_ex = 0x92, - - /// Protocol version indicator. - /// Introduced in protocol 2. - /// - /// For protocol 2 and above, a pickle must start with this opcode. - /// The argument is the protocol version, an int in range(2, 256). - proto = 0x80, - - /// Stop the unpickling machine. - /// - /// Every pickle ends with this opcode. The object at the top of the stack - /// is popped, and that's the result of unpickling. The stack should be - /// empty then. - stop = '.', - - /// Indicate the beginning of a new frame. - /// Introduced in protocol 4. - /// - /// The unpickler may use this opcode to safely prefetch data from its - /// underlying stream. - frame = 0x95, - - /// Push an object identified by a persistent ID. - /// - /// The pickle module doesn't define what a persistent ID means. PERSID's - /// argument is a newline-terminated str-style (no embedded escapes, no - /// bracketing quote characters) string, which *is* "the persistent ID". - /// The unpickler passes this string to self.persistent_load(). Whatever - /// object that returns is pushed on the stack. There is no implementation - /// of persistent_load() in Python's unpickler: it must be supplied by an - /// unpickler subclass. - persid = 'P', - - /// Push an object identified by a persistent ID. - /// Introduced in protocol 1. - /// - /// Like PERSID, except the persistent ID is popped off the stack (instead - /// of being a string embedded in the opcode bytestream). The persistent - /// ID is passed to self.persistent_load(), and whatever object that - /// returns is pushed on the stack. See PERSID for more detail. - binpersid = 'Q', - - _, -}; - -// The above enum was generated with the following Python code, -// run inside pickletools.py -// -// def generate_zig(): -// print("""/// All possible pickle operators. -// /// Reference: https://github.com/python/cpython/blob/3.13/Lib/pickletools.py -// pub const OpCode = enum(u8) { -// """) -// for op in opcodes: -// lines = [_cleanup(l) for l in op.doc.split("\n")[:-1]] -// if op.proto > 0: -// lines.insert(1, _cleanup(f"Introduced in protocol {op.proto}.")) -// doc = "\n".join(lines) -// op_code = op.code.__repr__() -// if op_code.startswith("'\\x"): -// op_code = "0x" + op_code[3:-1] -// print(f"""{doc} -// {op.name.lower()} = {op_code}, -// """) -// -// print(" _,") -// print("};") -// -// def _cleanup(line: str) -> str: -// indent = " " -// if (line.startswith(indent)): -// line = line[len(indent):] -// line = line.replace(". ", ". ") -// if line: -// line = " " + line -// line = " ///" + line -// return line - -/// A decoded Pickle operation in its natural state. -/// This is a bit different from Op enum, -/// because operators having same semantics, but different encoding have been merged. -/// ex: string, binstring, short_binstring -> string. -pub const Op = union(enum) { - int: i32, - // Python can represent arbitrary long integers - long: []const u8, - binlong: []const u8, - string: []const u8, - bytes: []const u8, - bytearray: []u8, - next_buffer, - readonly_buffer, - none, - bool: bool, - unicode: []const u8, - float: []const u8, - binfloat: f64, - empty_list, - append, - appends, - list, - empty_tuple, - tuple, - tuple1, - tuple2, - tuple3, - empty_dict, - dict, - setitem, - setitems, - empty_set, - additems, - frozenset, - pop, - dup, - mark, - pop_mark, - get: u32, - put: u32, - memoize, - ext1: u8, - ext2: i16, - ext4: i32, - global: PyType, - stack_global, - reduce, - build, - inst: PyType, - obj, - newobj, - newobj_ex, - proto: u8, - stop, - frame: u64, // new frame and its size - persid: []const u8, - binpersid, - - pub const PyType = struct { module: []const u8, class: []const u8 }; - - pub fn deinit(self: Op, allocator: std.mem.Allocator) void { - switch (self) { - // Use a switch on the type of the stored data, - // this is easier than listing every opcode. - inline else => |v| switch (@TypeOf(v)) { - void, bool, u8, u16, u32, u64, i16, i32, f64 => {}, - []const u8, []u8 => allocator.free(v), - PyType => { - allocator.free(v.module); - allocator.free(v.class); - }, - else => @compileError("please explicit how to free this new opcode: " ++ @typeName(@TypeOf(v))), - }, - } - } - - pub fn clone(self: Op, allocator: std.mem.Allocator) !Op { - return switch (self) { - // Use a switch on the type of the stored data, - // this is easier than listing every opcode. - inline else => |v, tag| switch (@TypeOf(v)) { - void, bool, u8, u16, u32, u64, i16, i32, f64 => self, - []const u8, []u8 => @unionInit(Op, @tagName(tag), try allocator.dupe(u8, v)), - PyType => @unionInit(Op, @tagName(tag), .{ - .module = try allocator.dupe(u8, v.module), - .class = try allocator.dupe(u8, v.class), - }), - else => @compileError("please explicit how to close this new opcode: " ++ @typeName(@TypeOf(v))), - }, - }; - } -}; - -pub const ParseError = error{ - UnknownPickleOp, -} || std.fmt.ParseIntError || std.Io.Reader.DelimiterError || std.mem.Allocator.Error; - -/// Read a stream of bytes, and interpret it as a stream of Pickle operators. -/// The given allocator needs to be an arena cause we are not aligning allocations to avoid copies. -pub fn parse(arena: std.mem.Allocator, reader: *std.Io.Reader) ParseError![]const Op { - // It's not very efficient to interleave the results with the data copied from the stream, - // because growth event in the results ArrayList will lead to fragmentation. - // Trying to mitigate that by using a generous default size. - var results: std.ArrayList(Op) = try .initCapacity(arena, 512); - errdefer results.deinit(arena); - var alloc_writer = try std.Io.Writer.Allocating.initCapacity(arena, 512); - - while (true) { - const code: OpCode = @enumFromInt(try reader.takeByte()); - const op: Op = switch (code) { - .int => int: { - const bytes_with_ln = try reader.takeDelimiterInclusive('\n'); - const bytes = bytes_with_ln[0 .. bytes_with_ln.len - 1]; - // Legacy hack, see OpCode.int documentation - // We do this parsing right away to simplify downstream code. - break :int if (bytes.len == 2 and bytes[0] == '0' and bytes[1] == '0') - .{ .bool = false } - else if (bytes.len == 2 and bytes[0] == '0' and bytes[1] == '1') - .{ .bool = true } - else - .{ .int = try std.fmt.parseInt(i32, bytes, 10) }; - }, - .binint => .{ .int = try reader.takeInt(i32, .little) }, - .binint1 => .{ .int = try reader.takeByte() }, - .binint2 => .{ .int = try reader.takeInt(u16, .little) }, - .long => .{ .long = try readLine(reader, &alloc_writer) }, - .long1 => .{ .binlong = try _readSlice(reader, arena, 1) }, - .long4 => .{ .binlong = try _readSlice(reader, arena, 4) }, - .string => .{ .string = try readLine(reader, &alloc_writer) }, - .binstring => .{ .string = try _readSlice(reader, arena, 4) }, - .short_binstring => .{ .string = try _readSlice(reader, arena, 1) }, - .binbytes => .{ .bytes = try _readSlice(reader, arena, 4) }, - .binbytes8 => .{ .bytes = try _readSlice(reader, arena, 8) }, - .short_binbytes => .{ .bytes = try _readSlice(reader, arena, 1) }, - .bytearray8 => .{ .bytearray = try _readSlice(reader, arena, 8) }, - .next_buffer => .next_buffer, - .readonly_buffer => .readonly_buffer, - .none => .none, - .newtrue => .{ .bool = true }, - .newfalse => .{ .bool = false }, - .unicode => .{ .unicode = try readLine(reader, &alloc_writer) }, - .short_binunicode => .{ .unicode = try _readSlice(reader, arena, 1) }, - .binunicode => .{ .unicode = try _readSlice(reader, arena, 4) }, - .binunicode8 => .{ .unicode = try _readSlice(reader, arena, 8) }, - .float => .{ .float = try readLine(reader, &alloc_writer) }, - .binfloat => .{ .binfloat = @bitCast(try reader.takeInt(u64, .big)) }, - .empty_list => .empty_list, - .append => .append, - .appends => .appends, - .list => .list, - .empty_tuple => .empty_tuple, - .tuple => .tuple, - .tuple1 => .tuple1, - .tuple2 => .tuple2, - .tuple3 => .tuple3, - .empty_dict => .empty_dict, - .dict => .dict, - .setitem => .setitem, - .setitems => .setitems, - .empty_set => .empty_set, - .additems => .additems, - .frozenset => .frozenset, - .pop => .pop, - .dup => .dup, - .mark => .mark, - .pop_mark => .pop_mark, - .get => .{ .get = try readTextU32(reader) }, - .binget => .{ .get = try reader.takeByte() }, - .long_binget => .{ .get = try reader.takeInt(u32, .little) }, - .put => .{ .put = try readTextU32(reader) }, - .binput => .{ .put = try reader.takeByte() }, - .long_binput => .{ .put = try reader.takeInt(u32, .little) }, - .memoize => .memoize, - .ext1 => .{ .ext1 = try reader.takeByte() }, - .ext2 => .{ .ext2 = try reader.takeInt(i16, .little) }, - .ext4 => .{ .ext4 = try reader.takeInt(i32, .little) }, - .global => .{ .global = .{ - .module = try readLine(reader, &alloc_writer), - .class = try readLine(reader, &alloc_writer), - } }, - .stack_global => .stack_global, - .reduce => .reduce, - .build => .build, - .inst => .{ .inst = .{ - .module = try readLine(reader, &alloc_writer), - .class = try readLine(reader, &alloc_writer), - } }, - .obj => .obj, - .newobj => .newobj, - .newobj_ex => .newobj_ex, - .proto => blk: { - const version = try reader.takeByte(); - if (version > 5) log.warn("zml.aio.torch.pickle.parse expects a Python pickle object of version <=5, got version {}. Will try to interpret anyway, but this may lead to more errors.", .{version}); - break :blk .{ .proto = version }; - }, - .stop => .stop, - .frame => frame: { - // This is not documented in pickletools but in https://peps.python.org/pep-3154/ - // The loader is allowed to prefetch framesize from the underlying reader, - // and ops are not allowed to cross a frame boundary. - const frame_size = try reader.takeInt(u64, .little); - reader.fill(@min(frame_size, reader.buffer.len)) catch |err| switch (err) { - error.EndOfStream => {}, - else => return err, - }; - break :frame .{ .frame = frame_size }; - }, - .persid => .{ .persid = try readLine(reader, &alloc_writer) }, - .binpersid => .binpersid, - _ => |unk_tag| { - log.err("Unknow pickle operator {}, note we are only supporting pickle protocol up to version 5.", .{unk_tag}); - return error.UnknownPickleOp; - }, - }; - try results.append(arena, op); - if (op == .stop) break; - } - return results.items; -} - -test "parse protocol 4" { - var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); - defer arena.deinit(); - - const file = try std.fs.cwd().openFile("zml/aio/torch/simple_test_4.pickle", .{ .mode = .read_only }); - var read_buffer: [1024]u8 = undefined; - var reader = file.reader(&read_buffer); - const ops = try parse(arena.allocator(), &reader.interface); - - // this can be obtained by running: `python -m pickletools simple_test_4.pickle` - const expected: []const Op = &.{ - .{ .proto = 4 }, - .{ .frame = 119 }, - .empty_dict, - .memoize, - .mark, - .{ .unicode = "hello" }, - .memoize, - .{ .unicode = "world" }, - .memoize, - .{ .unicode = "int" }, - .memoize, - .{ .int = 1 }, - .{ .unicode = "float" }, - .memoize, - .{ .binfloat = 3.141592 }, - .{ .unicode = "list" }, - .memoize, - .empty_list, - .memoize, - .mark, - .{ .int = 255 }, - .{ .int = 1234 }, - .{ .int = -123 }, - .{ .int = 1_000_000_000 }, - .{ .binlong = &writeIntBuff(u48, 999_000_000_000) }, - .{ .binlong = &writeIntBuff(u104, 999_000_000_000_000_000_000_000_000_000) }, - .appends, - .{ .unicode = "bool" }, - .memoize, - .{ .bool = false }, - .{ .unicode = "tuple" }, - .memoize, - .{ .unicode = "a" }, - .memoize, - .{ .int = 10 }, - .tuple2, - .memoize, - .setitems, - .stop, - }; - try std.testing.expectEqualDeep(expected, ops); -} - -test "parse protocol 0" { - // We also test protocol 0, cause it's more text oriented. - var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); - defer arena.deinit(); - - const pickle_0 = - \\(dp0 - \\Vhello - \\p1 - \\Vworld - \\p2 - \\sVint - \\p3 - \\I1 - \\sVfloat - \\p4 - \\F3.141592 - \\sVlist - \\p5 - \\(lp6 - \\I255 - \\aI1234 - \\aI-123 - \\aI1000000000 - \\aL999000000000L - \\aL999000000000000000000000000000L - \\asVbool - \\p7 - \\I00 - \\sVtuple - \\p8 - \\(Va - \\p9 - \\I10 - \\tp10 - \\s. - ; - - var reader: std.Io.Reader = .fixed(pickle_0); - const ops = try parse(arena.allocator(), &reader); - - var expected = [_]Op{ - .mark, - .dict, - .{ .put = 0 }, - .{ .unicode = "hello" }, - .{ .put = 1 }, - .{ .unicode = "world" }, - .{ .put = 2 }, - .setitem, - .{ .unicode = "int" }, - .{ .put = 3 }, - .{ .int = 1 }, - .setitem, - .{ .unicode = "float" }, - .{ .put = 4 }, - .{ .float = "3.141592" }, - .setitem, - .{ .unicode = "list" }, - .{ .put = 5 }, - .mark, - .list, - .{ .put = 6 }, - .{ .int = 255 }, - .append, - .{ .int = 1234 }, - .append, - .{ .int = -123 }, - .append, - .{ .int = 1_000_000_000 }, - .append, - .{ .long = "999000000000L" }, - .append, - .{ .long = "999000000000000000000000000000L" }, - .append, - .setitem, - .{ .unicode = "bool" }, - .{ .put = 7 }, - .{ .bool = false }, - .setitem, - .{ .unicode = "tuple" }, - .{ .put = 8 }, - .mark, - .{ .unicode = "a" }, - .{ .put = 9 }, - .{ .int = 10 }, - .tuple, - .{ .put = 10 }, - .setitem, - .stop, - }; - try std.testing.expectEqualDeep(&expected, ops); -} - -fn _readSlice(reader: anytype, allocator: std.mem.Allocator, comptime len_bytes: u8) ![]u8 { - const T = std.meta.Int(.unsigned, 8 * len_bytes); - const str_len: u64 = try reader.takeInt(T, .little); - const buf = try allocator.alloc(u8, str_len); - _ = try reader.readSliceAll(buf); - return buf; -} - -fn writeIntBuff(comptime T: type, value: T) [@divExact(@typeInfo(T).int.bits, 8)]u8 { - var res: [@divExact(@typeInfo(T).int.bits, 8)]u8 = undefined; - std.mem.writeInt(T, &res, value, .little); - return res; -} - -fn readLine(reader: *std.Io.Reader, alloc_writer: *std.Io.Writer.Allocating) ParseError![]const u8 { - const n = reader.streamDelimiter(&alloc_writer.writer, '\n') catch |err| switch (err) { - error.WriteFailed => return error.OutOfMemory, - else => |e| return e, - }; - std.debug.assert(try reader.takeByte() == '\n'); - const w = &alloc_writer.writer; - std.debug.assert(w.end == n); - const items = w.buffer[0..n]; - w.buffer = w.buffer[n + 1 ..]; - w.end = 0; - return items; -} - -fn readTextU32(reader: *std.Io.Reader) ParseError!u32 { - // Note we use takeDelimiterInclusive because the newline must always be there. - const digits = try reader.takeDelimiterInclusive('\n'); - return try std.fmt.parseInt(u32, digits[0 .. digits.len - 1], 10); -} diff --git a/zml/aio/torch/py.zig b/zml/aio/torch/py.zig deleted file mode 100644 index 539c28a0..00000000 --- a/zml/aio/torch/py.zig +++ /dev/null @@ -1,431 +0,0 @@ -const std = @import("std"); -const math = std.math; - -const pickle = @import("pickle.zig"); - -const log = std.log.scoped(.@"zml/aio"); - -/// Correspond to a function/constructor call -pub const Object = struct { - member: Any, - args: []Any, - kwargs: []Any, - - pub fn init(allocator: std.mem.Allocator, member: Any, args: []Any, kwargs: []Any) !*Object { - const self = try allocator.create(Object); - self.* = .{ .member = member, .args = args, .kwargs = kwargs }; - return self; - } - - pub fn clone(self: *Object, allocator: std.mem.Allocator) std.mem.Allocator.Error!*Object { - const res = try allocator.create(Object); - res.* = .{ - .member = try self.member.clone(allocator), - .args = try allocator.alloc(Any, self.args.len), - .kwargs = try allocator.alloc(Any, self.kwargs.len), - }; - for (self.args, 0..) |v, i| res.args[i] = try v.clone(allocator); - for (self.kwargs, 0..) |v, i| res.kwargs[i] = try v.clone(allocator); - return res; - } - - pub fn deinit(self: *Object, allocator: std.mem.Allocator) void { - self.member.deinit(allocator); - for (self.args) |*v| v.deinit(allocator); - allocator.free(self.args); - allocator.destroy(self); - } -}; - -/// Correspond to the __set_state__ call when pickle finishes building an object. -pub const SetState = struct { - obj: Any, - state: Any, - - pub fn init(allocator: std.mem.Allocator, obj: Any, state: Any) !*SetState { - const res = try allocator.create(SetState); - res.* = .{ .obj = obj, .state = state }; - return res; - } - - pub fn clone(self: *SetState, allocator: std.mem.Allocator) std.mem.Allocator.Error!*SetState { - const res = try allocator.create(SetState); - res.* = .{ .obj = try self.obj.clone(allocator), .state = try self.state.clone(allocator) }; - return res; - } - - pub fn deinit(self: *SetState, allocator: std.mem.Allocator) void { - self.obj.deinit(allocator); - self.state.deinit(allocator); - self.allocator.destroy(self); - } -}; - -/// The types of sequences that exist. -pub const SequenceType = enum { - list, - dict, - tuple, - set, - frozen_set, -}; - -pub const Sequence = struct { - type: SequenceType, - values: []Any, -}; - -pub fn tuple(values: []const Any) Any { - // tuple are readonly, but sequence in general aren't - return .{ .seq = .{ .type = .tuple, .values = @constCast(values) } }; -} - -pub const PersId = struct { - ref: Any, - - pub fn init(allocator: std.mem.Allocator, ref: Any) !*PersId { - const self = try allocator.create(PersId); - self.* = .{ .ref = ref }; - return self; - } - - pub fn clone(self: *PersId, allocator: std.mem.Allocator) std.mem.Allocator.Error!*PersId { - const res = try allocator.create(PersId); - res.* = .{ .ref = try self.ref.clone(allocator) }; - return res; - } - - pub fn deinit(self: *PersId, allocator: std.mem.Allocator) void { - self.ref.deinit(allocator); - allocator.destroy(self); - } -}; - -pub const Kind = enum { - raw, - ref, - app, - object, - set_state, - pers_id, - global, - seq, - string, - bytes, - int64, - bigint, - float64, - raw_num, - boolval, - none, -}; - -/// A pickle operator that has been interpreted. -pub const Any = union(Kind) { - /// Types that we can't handle or just had to give up on processing. - raw: pickle.Op, - - /// A reference. You might be able to look it up in the memo map - /// unless there's something weird going on like recursive references. - /// You generally shouldn't see this in the result unless bad things - /// are going on... - ref: u32, - - /// The result of applying a thing to another thing. We're not - /// Python so we don't really know what a "thing" is. - app: *Object, - - /// An object or something. The first tuple member is the - /// thing, the second one is the arguments it got applied to. - object: *Object, - - /// Correspond to the __set_state__ call when pickle finishes building an object. - /// The first tuple member is the target object, - /// the second one is the "state" argument - set_state: *SetState, - - /// References to persistant storage. They basically could be anything. - /// You kind of have to know what the thing you're trying to - /// interface wants to use as keys for persistant storage. - /// Good luck. - pers_id: *PersId, - - /// A global value of some kind. The first tuple member is - /// the thing, the second one is the arguments it got applied to. - global: *Object, - - /// A sequence. We don't really distinguish between them - /// much. The one exception is when the SequenceType is - /// Dict we try to split the flat list of `[k, v, k, v, k, v]` - /// into a list of tuples with the key and value. - seq: Sequence, - - /// A string, but not the crazy strings that have to be - /// unescaped as if they were Python strings. If you - /// need one of those, look for it inside a `Value.raw`. - string: []const u8, - - /// Some bytes. It might be a byte array or a binary - /// string that couldn't get UTF8 decoded. We do the best - /// we can. - bytes: []const u8, - - /// An integer, but not the crazy kind that comes as a string - /// that has to be parsed. You can look in `Value.raw_num` for - /// those. - int64: i64, - - /// An integer that can't fit in i64. - bigint: math.big.int.Const, - - /// An float, but not the crazy kind that comes as a string - /// that has to be parsed. You can look in `Value.raw_num` for - /// those. - float64: f64, - - /// Some kind of weird number we can't handle. - raw_num: pickle.Op, - - /// A boolean value. - boolval: bool, - - /// Python `None`. - none: void, - - pub fn deinit(self: *Any, allocator: std.mem.Allocator) void { - switch (self.*) { - .raw, .raw_num => |v| v.deinit(allocator), - inline .app, .object, .global, .build, .pers_id => |v| v.deinit(), - .seq => |v| { - for (v.values) |*val| val.deinit(allocator); - allocator.free(v.values); - }, - .string, .bytes => |v| allocator.free(v), - .bigint => |big| allocator.free(big.limbs), - else => {}, - } - self.* = undefined; - } - - inline fn writeIndents(indents: usize, writer: *std.Io.Writer) !void { - try writer.writeBytesNTimes(" ", indents); // resolve tab = 2 spaces - // try writer.writeByteNTimes('\t'); - } - - pub fn format(self: Any, writer: *std.Io.Writer) !void { - return internalFormat(self, 0, writer); - } - - fn internalFormat(value: Any, indents: usize, writer: *std.Io.Writer) !void { - try writeIndents(indents, writer); - try writer.writeAll(".{\n"); - try writeIndents(indents + 1, writer); - try writer.print(".{s} = ", .{@tagName(std.meta.activeTag(value))}); - switch (value) { - inline .ref, .int64, .float64 => |v| try writer.print("{d} ", .{v}), - .app, .object, .global => |v| { - try writer.writeAll(".{\n"); - try writeIndents(indents + 2, writer); - try writer.writeAll(".fn ="); - try internalFormat(v.member, indents + 2, writer); - try writer.writeAll(",\n"); - try writeIndents(indents + 2, writer); - try writer.writeAll(".args = "); - if (v.args.len > 0) { - try writer.writeAll(".{\n"); - for (v.args, 0..) |arg, i| { - try internalFormat(arg, indents + 3, writer); - if (i < v.args.len - 1) try writer.writeAll(","); - try writer.writeByte('\n'); - } - try writeIndents(indents + 2, writer); - try writer.writeAll("},\n"); - } else { - try writer.writeAll(".{},\n"); - } - try writeIndents(indents + 2, writer); - try writer.writeAll(".kwargs ="); - if (v.kwargs.len > 0) { - try writer.writeAll(".{\n"); - for (v.kwargs, 0..) |arg, i| { - try internalFormat(arg, indents + 3, writer); - if (i < v.kwargs.len - 1) try writer.writeAll(","); - try writer.writeByte('\n'); - } - try writeIndents(indents + 2, writer); - try writer.writeAll("}\n"); - } else { - try writer.writeAll(".{}\n"); - } - try writeIndents(indents + 1, writer); - try writer.writeAll("}"); - }, - .set_state => |v| { - try writer.writeAll(".{\n"); - try internalFormat(v.obj, indents + 2, writer); - try writer.writeAll(",\n"); - try internalFormat(v.state, indents + 2, writer); - try writer.writeAll(",\n"); - try writeIndents(indents + 1, writer); - try writer.writeAll("}"); - }, - .pers_id => |v| { - try writer.writeByte('\n'); - try internalFormat(v.ref, indents + 2, writer); - }, - .seq => |v| { - try writer.writeAll(".{\n"); - try writeIndents(indents + 2, writer); - try writer.print(".{s},\n", .{@tagName(v.type)}); - try writeIndents(indents + 2, writer); - if (v.values.len > 0) { - try writer.writeAll(".{\n"); - for (v.values, 0..) |arg, i| { - try internalFormat(arg, indents + 3, writer); - if (i < v.values.len - 1) try writer.writeAll(","); - try writer.writeByte('\n'); - } - try writeIndents(indents + 2, writer); - try writer.writeAll("}\n"); - } else { - try writer.writeAll(".{}\n"); - } - - try writeIndents(indents + 1, writer); - try writer.writeAll("}"); - }, - .string => |v| try writer.print("\"{s}\"", .{v}), - .raw => |v| switch (v) { - .global => |py_type| try writer.print("\"{s}\", \"{s}\"", .{ py_type.module, py_type.class }), - else => try writer.print("{any}", .{v}), - }, - inline else => |v| { - try writer.print("{any}", .{v}); - }, - } - try writer.writeByte('\n'); - try writeIndents(indents, writer); - try writer.writeByte('}'); - } - - pub fn clone(self: Any, allocator: std.mem.Allocator) !Any { - return switch (self) { - inline .raw, .raw_num => |v, tag| @unionInit(Any, @tagName(tag), try v.clone(allocator)), - inline .app, .object, .global, .set_state, .pers_id => |v, tag| @unionInit(Any, @tagName(tag), try v.clone(allocator)), - .seq => |seq| { - const values = try allocator.alloc(Any, seq.values.len); - for (seq.values, 0..) |v, i| values[i] = try v.clone(allocator); - return .{ .seq = .{ .type = seq.type, .values = values } }; - }, - inline .string, .bytes => |v, tag| @unionInit(Any, @tagName(tag), try allocator.dupe(u8, v)), - .bigint => |v| .{ .bigint = (try v.toManaged(allocator)).toConst() }, - else => self, - }; - } - - pub fn isPrimitive(self: Any) bool { - return switch (self) { - .int64, .bigint, .float64, .string, .bytes, .boolval, .none => true, - .seq => |seq| { - for (seq.values) |v| { - if (!v.isPrimitive()) return false; - } - return true; - }, - else => false, - }; - } - - pub fn containsRef(self: Any) bool { - switch (self) { - .ref => return true, - .app, .object, .global => |v| { - if (v.member.containsRef()) return true; - for (v.args) |arg| if (arg.containsRef()) return true; - return false; - }, - .set_state => |v| { - if (v.obj.containsRef()) return true; - if (v.state.containsRef()) return true; - return false; - }, - .pers_id => |v| return v.ref.containsRef(), - .seq => |v| { - for (v.values) |val| if (val.containsRef()) return true; - return false; - }, - else => return false, - } - } - - pub const UnpickleError = error{ InvalidCharacter, OutOfMemory }; - - pub fn coerceFromRaw(self: Any, allocator: std.mem.Allocator) UnpickleError!Any { - return switch (self) { - .raw => |raw_val| switch (raw_val) { - .none => .none, - .bool => |b| .{ .boolval = b }, - .float => |b| .{ .float64 = std.fmt.parseFloat(f64, b) catch std.math.nan(f64) }, - .int => |val| .{ .int64 = val }, - .long => |digits| { - const n = std.fmt.parseInt(i64, digits[0 .. digits.len - 1], 10) catch |err| { - switch (err) { - error.Overflow => { - log.warn("Not parsing long integer: {s}", .{digits}); - return self; - }, - error.InvalidCharacter => return error.InvalidCharacter, - } - }; - return .{ .int64 = n }; - }, - .binlong => |bytes| if (bytes.len <= 8) - .{ .int64 = std.mem.readVarInt(i64, bytes, .little) } - else { - // Note: we need to copy here, because Zig big int limbs are usize aligned, - // whereas pickle big int are byte aligned. - const n_limbs = std.math.divCeil(usize, bytes.len, @sizeOf(math.big.Limb)) catch unreachable; - var big = (try math.big.int.Managed.initCapacity(allocator, n_limbs)).toMutable(); - big.readTwosComplement(bytes, bytes.len * 8, .little, .signed); - - return .{ .bigint = big.toConst() }; - }, - .binfloat => |val| .{ .float64 = val }, - .unicode => |s| .{ .string = s }, - inline .bytes, .bytearray => |b| .{ .bytes = b }, - // This isn't how Pickle actually works but we just try to UTF8 decode the - // string and if it fails, we make it a bytes value instead. If anyone - // actually cares they can just fix values themselves or recover the raw bytes - // from the UTF8 string (it's guaranteed to be reversible, as far as I know). - .string => |b| if (std.unicode.utf8ValidateSlice(b)) - .{ .string = b } - else - .{ .bytes = b }, - else => self, - }, - .app, .object, .global => |v| blk: { - v.member = try v.member.coerceFromRaw(allocator); - for (v.args) |*arg| { - arg.* = try arg.coerceFromRaw(allocator); - } - break :blk self; - }, - .set_state => |v| blk: { - v.obj = try v.obj.coerceFromRaw(allocator); - v.state = try v.state.coerceFromRaw(allocator); - break :blk self; - }, - .pers_id => |v| blk: { - v.ref = try v.ref.coerceFromRaw(allocator); - break :blk self; - }, - .seq => |v| blk: { - for (v.values) |*val| { - val.* = try val.coerceFromRaw(allocator); - } - break :blk self; - }, - else => self, - }; - } -}; diff --git a/zml/aio/torch/simple.pt b/zml/aio/torch/simple.pt deleted file mode 100644 index f6978e65eaa6a8429f4ced64c8370b7294eefa6f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2708 zcmbtWOKjX!6!pxc`M4!%6I%Ks6bLYZW-`BIlK#?8%JgGsr_&Zpp;(UnGGoQH=h|-) zTB#CUByAV4V8Idz#G+LpsObWU4Qtqe0HJJIuz>{<3nYZZz0dPc(~9y&FOJ{+oO|E9 z=lbQ9NQWeKcS~!=W~pD&J)>mJvr$v$daOKW=d!)hg_iMd_$o+Ibpx7Ta!qDK$lxwo z#T+Mw3l*FBF~fD{$vxe2ET;$?Zm1jaT`TLJeu;7B`Os+? zu5B}eTdw2Fr#+K-%sgrtT!yaU0(3{!PPABHR25T}rriM@@4!vpwpGLU?eaaAr&1V!T zs)X?uQ6e0+M3sJ}2mcRo=)YN_hzB@4UVyDP#LKqp@I(Q&tGN5pjHUaqqa>FQzqF|o z03={%sZR-W*oBUj@FMJ1LW}TZ;A)V=Qw7))xO)0JJX3&YD`H2rnab=eZQqc9c~0Gh z0Gi`ES{YX@AG75(9nEJPo;Ngnw`^`XnkM6l3@==S=ptamFRBA%J);|Q3%ZB9=`pS3 zx^q6fB)soc`IY(&Un>Nb*DzBXI2?S>E1fzeZHFG*)zPdE+?iZ%7vhXMcwcb~jG*S@G#0hA z1bG2g^xR6>hcik@L|WC+sG~qZG?y#PbA33g?m^#3?s=?bM(h>=;X;|=nV1@1unY$0 zqDlw;Fmev(D>DKjU^%=NaIDJf>}AgCynbEqIteP$u7krHM7t2>c>#T4Vs8f2dyB)x z0=!*`<(01P!>j(6e)0uQSGm)4~p)4cL=$oEzk3=y-nt3He!Pajc z%|zNvf|k*K4{5h<<_@*!#!&EPQnRj8uid=4aOtzu7#}v72Xt$GObt#Cc@{wL7%wl1M+${TH=J!j}L5 diff --git a/zml/aio/torch/simple_test_4.pickle b/zml/aio/torch/simple_test_4.pickle deleted file mode 100644 index 4a5176590cd0d4a7aa79b39b80fb411b6513808e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 130 zcmZo*nOe>O0ku;!dRQ}3b8_;h^stua7v-c(>0!>yE1BZW*u$EZlb={J#od8Z@zQxoXp~qDX~*Dy#M=NV)1JI|Np-i!>L);U2F_)Tr*!VbnyZWjaZwzWZC4rkF$AF fdsve4^K+(j_OO0wNq;?31IrBn|9AKNkd diff --git a/zml/buffer.zig b/zml/buffer.zig index e76039d2..bafca61c 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -1,19 +1,16 @@ const std = @import("std"); -const async = @import("async"); const stdx = @import("stdx"); +const constants = @import("constants.zig"); +const ConstSlice = @import("slice.zig").ConstSlice; const DataType = @import("dtype.zig").DataType; -const HostBuffer = @import("hostbuffer.zig").HostBuffer; const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; +const Slice = @import("slice.zig").Slice; const Target = @import("platform.zig").Target; - -test { - std.testing.refAllDecls(@This()); - std.testing.refAllDecls(Buffer); -} +const testing = @import("testing.zig"); const log = std.log.scoped(.zml); @@ -34,293 +31,6 @@ pub const Buffer = struct { pub const Memory = pjrt.Memory.Kind; pub const FromOptions = struct { wait: bool = true, memory: Memory = .device }; - /// Copies the content of the given buffer from host memory to the accelerator memory. - pub fn from(platform: Platform, host_buffer: HostBuffer, opts: FromOptions) !Buffer { - var res: Buffer = .{ - ._api = platform.pjrt_api, - ._target = platform.target, - ._shape = host_buffer.shape(), - ._shards = .{}, - }; - - // We shard only on the first axis so that the chunks are still contiguous. - // TODO: support more advanced sharding specs - stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); - const sharding_ax: ?u3 = std.simd.firstTrue(host_buffer.shape()._sharding_info); - const n_partitions = platform.sharding().num_partitions; - const chunk_size = if (sharding_ax) |ax| cs: { - // This kind of sharding error should be detected earlier on. - stdx.debug.assert(@rem(host_buffer.dim(ax), n_partitions) == 0, "Buffer.from({f}) expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ host_buffer, ax, n_partitions }); - break :cs @divExact(host_buffer.dim(ax), n_partitions); - } else 0; - - const buffer_type = pjrt.bufferTypeFromDtype(host_buffer.shape().dtype()); - const byte_strides = host_buffer.strides(); - - const devices = platform.getDevices(); - for (0..n_partitions) |i| { - // If no sharding if found, the given buffer is replicated on all devices. - const buf = if (sharding_ax) |ax| buf: { - const start: i64 = @as(i64, @intCast(i)) * chunk_size; - break :buf host_buffer.slice1d(ax, .{ .start = start, .end = start + chunk_size }); - } else host_buffer; - - const args = pjrt.Client.BufferFromHostBufferArgs{ - .data = buf._data, - .buffer_type = buffer_type, - .dims = buf.shape().dims(), - .byte_strides = byte_strides, - .host_buffer_semantics = .ImmutableUntilTransferCompletes, - // CPU has no distinctions between memories. - .dst = if (platform.target == .cpu or opts.memory == .device) - .{ .device = devices[i] } - else - .{ .memory = platform.memoryForDevice(opts.memory, devices[i]) }, - }; - - const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); - - if (event) |ev| ev.deinit(platform.pjrt_api); - res._shards.appendAssumeCapacity(pjrt_buffer); - } - - if (opts.wait) { - res = try res.await(); - } - - return res; - } - - pub fn await(self: Buffer) !Buffer { - for (self._shards.constSlice()) |buffer| { - if (buffer.getReadyEvent(self._api)) |ev| { - try ev.await(self._api); - } - } - - return self; - } - - pub fn awaitBlocking(self: Buffer) !Buffer { - for (self._shards.constSlice()) |buffer| { - if (buffer.getReadyEvent(self._api)) |ev| { - try ev.awaitBlocking(self._api); - } - } - return self; - } - - /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. - pub fn fromPjrtBuffers(platform: Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { - stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); - stdx.debug.assert(pjrt_buffers.len > 0, "fromPjrtBuffers expects at least one buffer, got 0.", .{}); - var shards: Shards = .{}; - shards.appendSliceAssumeCapacity(pjrt_buffers); - return .{ - ._api = platform.pjrt_api, - ._target = platform.target, - ._shape = shape_, - ._shards = shards, - }; - } - - /// Copies the given Zig slice to the accelerator memory and - /// return a Buffer with the given dimensions. - pub fn fromSlice(platform: Platform, dimz: anytype, s: anytype) !Buffer { - const sh = Shape.init(dimz, DataType.fromSliceElementType(s)); - return from(platform, HostBuffer.fromBytes(sh, std.mem.sliceAsBytes(s)), .{}); - } - - /// Copies the given Zig slice to the accelerator memory and - /// return a Buffer with the given dimensions. - pub fn fromBytes(platform: Platform, sh: Shape, data: []const u8) !Buffer { - return from(platform, HostBuffer.fromBytes(sh, data), .{}); - } - - /// Copies the given Zig array to the accelerator memory and - /// return a Buffer using the array shape. - pub fn fromArray(platform: Platform, arr: anytype) !Buffer { - const host_buffer = HostBuffer.fromArray(&arr); - return try from(platform, host_buffer, .{ .wait = true }); - } - - /// Copies the given Zig slice to the accelerator memory and - /// return a Buffer with the given dimensions. - pub fn fromSliceOpts(platform: Platform, dimz: anytype, s: anytype, opts: FromOptions) !Buffer { - const sh = Shape.init(dimz, DataType.fromSliceElementType(s)); - return from(platform, HostBuffer.fromBytes(sh, std.mem.sliceAsBytes(s)), opts); - } - - /// Copies the given Zig slice to the accelerator memory and - /// return a Buffer with the given dimensions. - pub fn fromBytesOpts(platform: Platform, sh: Shape, data: []const u8, opts: FromOptions) !Buffer { - return from(platform, HostBuffer.fromBytes(sh, data), opts); - } - - /// Copies the given Zig array to the accelerator memory and - /// return a Buffer using the array shape. - pub fn fromArrayOpts(platform: Platform, arr: anytype, opts: FromOptions) !Buffer { - const host_buffer = HostBuffer.fromArray(&arr); - return try from(platform, host_buffer, opts); - } - - pub fn asHostBuffer(self: Buffer) HostBuffer { - if (self._target != .cpu) { - const memory = self.getMemory().kind(self._api); - stdx.debug.assert((memory == .host_pinned) or (memory == .host_unpinned), "asHostBuffer({f}) expects a buffer allocated on host memory, got {t}. see `copyToMemory`", .{ self, memory }); - } - const ptr: [*]u8 = @ptrCast(self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable); - return HostBuffer.fromBytes(self._shape, ptr[0..self._shape.byteSize()]); - } - - /// Creates a Buffer with a single element. - pub fn scalar(platform: Platform, val: anytype, dtype_: DataType) !Buffer { - const x = dtype_.constant(val); - const host_buffer = HostBuffer.fromBytes(Shape.init(.{}, dtype_), x.constSlice()); - return try from(platform, host_buffer, .{ .wait = true }); - } - - /// Creates a Buffer with a single element repeated manytime. - pub fn constant(platform: Platform, shape_: Shape, val: anytype, opts: FromOptions) !Buffer { - var start = try std.time.Timer.start(); - defer { - const duration_ms = stdx.math.divFloat(f32, start.read(), std.time.ns_per_ms); - if (duration_ms > 100) { - const size_gb = stdx.math.divFloat(f32, shape_.byteSize(), 1024 * 1024 * 1024); - log.debug("Wrote constant({f}) to device ({d:.2}Gb) in {d:.0}ms: {d:.2}Gb/s", .{ shape_, size_gb, duration_ms, size_gb / duration_ms * 1000 }); - } - } - - // Constant is always blocking because it uses pointer to stack memory. - const cst_opts: FromOptions = .{ .memory = opts.memory, .wait = true }; - // Convert val to the requested dtype. - const x = shape_.dtype().constant(val); - const byte_size = shape_.dtype().sizeOf(); - const max_bytes = 1024; - - // Naive version for scalars and buffers with long last axis. - if (shape_.rank() < 1 or byte_size * shape_.dim(-1) > max_bytes) { - const host_buffer: HostBuffer = .{ - ._shape = shape_, - ._strides = @splat(0), - ._data = x.constSlice().ptr, - }; - return try from(platform, host_buffer, cst_opts); - } - - // To speed up copies, duplicate the scalar value into a vector, - // so that PJRT can copy row by row. - // Because this is respecting the shape, it won't work if the last axis is too big. - // If this becomes an issue, we should create a new intermediary Buffer by splitting last axis into { n, max_bytes } - // so that the trick works, and then reshape it - // We could also handle sharded constant directly in this function to avoid having to create too big arrays. - var bytes: [max_bytes]u8 align(64) = undefined; - var strides = [1]i64{0} ** Shape.MAX_RANK; - strides[shape_.rank() - 1] = byte_size; - - switch (byte_size) { - inline 1, 2, 4, 8, 16 => |b| { - const Int = std.meta.Int(.unsigned, b * 8); - const x_as_int: Int = @bitCast(x.constSlice()[0..b].*); - const bytes_as_int: [*]Int = @ptrCast(&bytes); - @memset(bytes_as_int[0..@intCast(shape_.dim(-1))], x_as_int); - }, - else => unreachable, - } - const host_buffer: HostBuffer = .{ ._shape = shape_, ._strides = strides, ._data = &bytes }; - return try from(platform, host_buffer, cst_opts); - } - - test constant { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const x = try constant(platform, Shape.init(.{ 4, 3, 2 }, .u16), 42, .{ .wait = true }); - const y = try x.getValue([4 * 3 * 2]u16); - try std.testing.expectEqual([_]u16{42} ** (4 * 3 * 2), y); - } - - /// Creates a Buffer as a view of host memory visible from the device, - /// thus avoiding a copy. - /// - /// Be careful though, as it requires a specific alignment - /// and it might not work on all platforms, - /// could lead to crashes and operations on the buffer will be slower. - /// Tested on Cuda 12.4. - pub fn asViewOfHostBuffer(platform: Platform, buf: HostBuffer) Buffer { - return asViewOfDeviceBuffer(platform, buf.shape(), null, @constCast(buf._data)); - } - - /// Creates a Buffer from a pointer into device memory. - /// This allows to interface with other libraries producing buffers. - pub fn asViewOfDeviceBuffer(platform: Platform, shape_: Shape, stream: ?*const pjrt.Stream, device_data: *anyopaque) Buffer { - const pjrt_buffer = platform.pjrt_client.createViewOfDeviceBuffer(platform.pjrt_api, .{ - .data = device_data, - .element_type = pjrt.bufferTypeFromDtype(shape_.dtype()), - .dims = shape_.dims(), - // TODO: exposes sharding in the API. - .device = platform.getDevices()[0], - .layout = .{ - .tiled = .{ - .minor_to_major = minorToMajor(shape_.rank()), - .tile_dims = &.{}, - .tile_dims_sizes = &.{}, - }, - }, - .stream = stream, - }) catch @panic("failed to createViewOfDeviceBuffer"); - - var shards: Shards = .{}; - shards.appendAssumeCapacity(pjrt_buffer); - return .{ - ._api = platform.pjrt_api, - ._target = platform.target, - ._shape = shape_, - ._shards = shards, - }; - } - - pub fn devicePtr(self: Buffer) *anyopaque { - stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer", .{}); - return self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable; - } - - /// Fetches the content of the given buffer into a stack variable of the given type. - pub fn getValue(self: Buffer, T: type) !T { - stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {f} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); - var res: T = undefined; - stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, std.mem.asBytes(&res)); - if (maybe_event) |event| { - try event.await(self._api); - } - return res; - } - - /// Copies the content of the Buffer back to host, in the given buffer, - /// and return a new `HostBuffer` object with the same shape. - /// The returned `HostBuffer` doesn't own the memory. - pub fn toHost(self: Buffer, output: []u8) !HostBuffer { - stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, output); - if (maybe_event) |event| { - try event.await(self._api); - } - return HostBuffer.fromBytes(self.shape(), output); - } - - /// Copies the content of the Buffer to the host. - /// The returned `HostBuffer` does own the memory. - pub fn toHostAlloc(self: Buffer, allocator: std.mem.Allocator) !HostBuffer { - const output = try HostBuffer.empty(allocator, self.shape()); - stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, @constCast(output.bytes())); - if (maybe_event) |event| { - try event.await(self._api); - } - return output; - } - /// Frees the accelerator memory. /// Depending on the platform, the memory is typically not released to the OS /// but just marked as available in the memory pool. @@ -336,81 +46,110 @@ pub const Buffer = struct { return self._shape; } - /// This Buffer shape as a slice of dims. - pub fn dims(self: *const Buffer) []const i64 { - return self._shape.dims(); - } - - /// This Buffer element type. - pub fn dtype(self: Buffer) DataType { - return self._shape.dtype(); - } - - /// This Buffer rank. - pub fn rank(self: Buffer) u4 { - return self._shape.rank(); - } - - /// Test helper: returns a new Buffer with the given tags. - /// Allows to call `zml.testing.compileAndCall` when the tested - /// functions requires tagged tensors. - pub fn withTags(self: Buffer, tags_: anytype) Buffer { - var res = self; - res._shape = self._shape.withTags(tags_); - return res; - } - pub fn format(self: Buffer, writer: *std.Io.Writer) !void { try writer.print("Buffer({f})@{x}", .{ self._shape, @intFromPtr(self.devicePtr()) }); } - pub fn getMemory(self: Buffer) *const pjrt.Memory { - const shard = self._shards.get(0); - return shard.memory(self._api); - } + /// Copies the content of the given buffer from host memory to the accelerator memory. + pub fn from(io: std.Io, platform: Platform, shape_: Shape, data_: []const u8, opts: FromOptions) !Buffer { + var res: Buffer = .{ + ._api = platform.pjrt_api, + ._target = platform.target, + ._shape = shape_, + ._shards = .{}, + }; - fn hasShardedAxis(self: Buffer) bool { - if (self._shards.len == 1) return false; - return @reduce(.Or, self._shape._sharding_info); - } + // We shard only on the first axis so that the chunks are still contiguous. + // TODO: support more advanced sharding specs + stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); + //const sharding_ax: ?u3 = std.simd.firstTrue(shape_._sharding_info); + const n_partitions = platform.sharding().num_partitions; + //const chunk_size = if (sharding_ax) |ax| cs: { + // // This kind of sharding error should be detected earlier on. + // stdx.debug.assert(@rem(shape_.dim(ax), n_partitions) == 0, "Buffer.from({f}) expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ shape_, ax, n_partitions }); + // break :cs @divExact(shape_.dim(ax), n_partitions); + //} else 0; + //_ = chunk_size; - pub const CopyToMemoryOpts = struct { - wait: bool = true, - }; + const buffer_type = pjrt.bufferTypeFromDtype(shape_.dtype()); + const byte_strides = shape_.computeByteStrides(); - pub fn copyToMemory(self: Buffer, platform: Platform, memory: pjrt.Memory.Kind, opts: CopyToMemoryOpts) !Buffer { - const pjrt_memory = platform.pjrt_client.memoryByKind(self._api, memory); - if (pjrt_memory == null) { - stdx.debug.panic("Memory destination `{t}` for {f}", .{ memory, self }); - } + const devices = platform.getDevices(); + for (0..n_partitions) |i| { + // If no sharding if found, the given buffer is replicated on all devices. + //const buf = if (sharding_ax) |ax| buf: { + // @panic("sharding not implemented"); + // //const start: i64 = @as(i64, @intCast(i)) * chunk_size; + // //break :buf host_buffer.slice1d(ax, .{ .start = start, .end = start + chunk_size }); + //} else host_buffer; - var new_shards: Buffer.Shards = .{}; - for (self._shards.slice()) |shard| { - const new_shard = try shard.copyToMemory(self._api, pjrt_memory.?); - new_shards.appendAssumeCapacity(new_shard); + const args = pjrt.Client.BufferFromHostBufferArgs{ + .data = data_.ptr, + .buffer_type = buffer_type, + .dims = shape_.dims(), + .byte_strides = byte_strides.slice(), + .host_buffer_semantics = .ImmutableUntilTransferCompletes, + // CPU has no distinctions between memories. + .dst = if (platform.target == .cpu or opts.memory == .device) + .{ .device = devices[i] } + else + .{ .memory = platform.memoryForDevice(opts.memory, devices[i]) }, + }; + + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); + + if (event) |ev| ev.deinit(platform.pjrt_api); + res._shards.appendAssumeCapacity(pjrt_buffer); } if (opts.wait) { - for (new_shards.constSlice()) |shard| { - const event = shard.getReadyEvent(self._api); - if (event) |e| { - try e.awaitBlocking(self._api); - } - } + res = try res.await(io); } - return Buffer{ ._api = self._api, ._target = platform.target, ._shape = self._shape, ._shards = new_shards }; + return res; + } + + /// Copies the given Zig bytes to the accelerator memory and + /// return a Buffer with the given dimensions. + pub fn fromBytes(io: std.Io, platform: Platform, sh: Shape, data: []const u8) !Buffer { + return from(io, platform, sh, data, .{}); + } + + /// Copies the given zml.Slice or zml.ConstSlice to the accelerator memory and + /// return a Buffer. + pub fn fromSlice(io: std.Io, platform: Platform, slice: anytype) !Buffer { + return fromSliceOpts(io, platform, slice, .{}); + } + + pub fn fromSliceOpts(io: std.Io, platform: Platform, slice: anytype, opts: FromOptions) !Buffer { + if (@TypeOf(slice) != Slice and @TypeOf(slice) != ConstSlice) @compileError("Buffer.fromSlice expects a Slice or ConstSlice, got " ++ @typeName(@TypeOf(slice))); + const real_slice: ConstSlice = if (@TypeOf(slice) == Slice) slice.constSlice() else slice; + return from(io, platform, real_slice.shape, std.mem.sliceAsBytes(real_slice.data), opts); + } + + /// Creates a Buffer with a single element. + pub fn scalar(io: std.Io, platform: Platform, val: anytype, dtype_: DataType) !Buffer { + const x = dtype_.constant(val); + return fromBytes(io, platform, Shape.init(.{}, dtype_), x.asBytes()); + } + + pub fn await(self: Buffer, io: std.Io) !Buffer { + for (self._shards.constSlice()) |buffer| { + try buffer.getReadyEvent(self._api).await(self._api, io); + } + + return self; } pub const UnitializedOptions = struct { memory: Memory = .device }; - pub fn uninitialized(platform: Platform, shape_: Shape, opts: UnitializedOptions) !Buffer { + pub fn uninitialized(io: std.Io, platform: Platform, shape_: Shape, opts: UnitializedOptions) !Buffer { if (opts.memory != .device) { // XLA uninitialized doesn't respect memory see https://github.com/openxla/xla/pull/31292 // TODO: use uninitialized when it works again. - const host_buffer: HostBuffer = try .empty(std.heap.smp_allocator, shape_); - defer host_buffer.deinit(std.heap.smp_allocator); - return try .from(platform, host_buffer, .{ .wait = true, .memory = opts.memory }); + const slice: Slice = try .alloc(std.heap.smp_allocator, shape_); + defer slice.free(std.heap.smp_allocator); + return .fromSliceOpts(io, platform, slice, .{ .memory = opts.memory }); } var res: Buffer = .{ @@ -424,22 +163,22 @@ pub const Buffer = struct { }; stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); - const sharding_ax: ?u3 = std.simd.firstTrue(shape_._sharding_info); + //const sharding_ax: ?u3 = std.simd.firstTrue(shape_._sharding_info); const n_partitions = platform.sharding().num_partitions; - const shard_shape = if (sharding_ax) |ax| s: { - // This kind of sharding error should be detected earlier on. - stdx.debug.assert(@rem(shape_.dim(ax), n_partitions) == 0, "Buffer.uninitialized() expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ ax, n_partitions }); - const shard_shape = shape_.set(ax, @divExact(shape_.dim(ax), n_partitions)); - break :s shard_shape; - } else shape_; + //const shard_shape = if (sharding_ax) |ax| s: { + // // This kind of sharding error should be detected earlier on. + // stdx.debug.assert(@rem(shape_.dim(ax), n_partitions) == 0, "Buffer.uninitialized() expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ ax, n_partitions }); + // const shard_shape = shape_.set(ax, @divExact(shape_.dim(ax), n_partitions)); + // break :s shard_shape; + //} else shape_; var args = pjrt.Client.CreateUninitializedBufferArgs{ - .dims = shard_shape.dims(), + .dims = shape_.dims(), .element_type = pjrt.bufferTypeFromDtype(shape_.dtype()), .layout = .{ .tiled = .{ - .minor_to_major = minorToMajor(shape_.rank()), + .minor_to_major = constants.minorToMajor(shape_.rank()), .tile_dims = &.{}, .tile_dims_sizes = &.{}, }, @@ -462,38 +201,52 @@ pub const Buffer = struct { return res; } - test uninitialized { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); + /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. + pub fn fromPjrtBuffers(platform: Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { + stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); + stdx.debug.assert(pjrt_buffers.len > 0, "fromPjrtBuffers expects at least one buffer, got 0.", .{}); + var shards: Shards = .{}; + shards.appendSliceAssumeCapacity(pjrt_buffers); + return .{ + ._api = platform.pjrt_api, + ._target = platform.target, + ._shape = shape_, + ._shards = shards, + }; + } - const host_visible_memories: []const Memory = &.{ .host_pinned, .host_unpinned }; - for (host_visible_memories) |memory| { - const x = try uninitialized(platform, .init(.{6}, .u8), .{ .memory = memory }); - const x_ptr: [*]u8 = @ptrCast(x.devicePtr()); - @memcpy(x_ptr, &[_]u8{ 104, 101, 108, 108, 111, 33 }); + pub fn devicePtr(self: Buffer) *anyopaque { + //stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer", .{}); + return self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable; + } - const y = try x.getValue([6]u8); - try std.testing.expectEqualStrings("hello!", &y); + /// Fetches the content of the given buffer into a stack variable of the given type. + pub fn getValue(self: Buffer, T: type, io: std.Io) !T { + stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {f} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); + var res: T = undefined; + + try self.toSlice(io, .init(self.shape(), std.mem.asBytes(&res))); + + return res; + } + + /// Copies the content of the Buffer to the provided slice. + pub fn toSlice(self: Buffer, io: std.Io, slice: Slice) !void { + //stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); + const maybe_event = try self._shards.get(0).toHostBuffer(self._api, slice.data); + if (maybe_event) |event| { + try event.await(self._api, io); } } - pub fn isDeleted(self: Buffer) bool { - const deleted: bool = self._shards.get(0).isDeleted(self._api); - for (self._shards.slice()[1..]) |shard| { - stdx.debug.assert(shard.isDeleted(self._api) == deleted, "Buffer has some shards deleted but not others.", .{}); - } - return deleted; + /// Copies the content of the Buffer to the provided slice. + /// The returned slice owns the memory. + pub fn toSliceAlloc(self: Buffer, allocator: std.mem.Allocator, io: std.Io) !Slice { + const slice = try Slice.alloc(allocator, self.shape()); + errdefer slice.free(allocator); + + try self.toSlice(io, slice); + + return slice; } }; - -const _MINOR_TO_MAJOR = blk: { - var ret: [Shape.MAX_RANK]i64 = undefined; - for (0..Shape.MAX_RANK) |i| { - ret[i] = @intCast(Shape.MAX_RANK - i - 1); - } - break :blk ret; -}; - -fn minorToMajor(rank: u8) []const i64 { - return _MINOR_TO_MAJOR[_MINOR_TO_MAJOR.len - rank ..]; -} diff --git a/zml/callback.zig b/zml/callback.zig deleted file mode 100644 index 04d09de7..00000000 --- a/zml/callback.zig +++ /dev/null @@ -1,322 +0,0 @@ -const std = @import("std"); - -const async = @import("async"); -const mlir = @import("mlir"); -const pjrt = @import("pjrt"); -const stablehlo = @import("mlir/dialects").stablehlo; -const stdx = @import("stdx"); - -const Buffer = @import("buffer.zig").Buffer; -const CompilationContext = @import("module.zig").CompilationContext; -const DataType = @import("dtype.zig").DataType; -const HostBuffer = @import("hostbuffer.zig").HostBuffer; -const mlirx = @import("mlirx.zig"); -const pjrtx = @import("pjrtx.zig"); -const Platform = @import("platform.zig").Platform; -const Shape = @import("shape.zig").Shape; -const Tensor = @import("tensor.zig").Tensor; - -const log = std.log.scoped(.@"zml/callback"); - -/// Inserts a user-defined callback into the computation graph. -/// The callback is defined with a struct, that store runtime information needed by the callback. -/// -/// ❗Experimental API❗ -/// -/// ```zig -/// pub const MyCallback = struct { -/// // a unique type_id will be set by the PJRT plugin during registration. -/// pub var type_id: pjrt.ffi.TypeId = undefined; -/// -/// pub const callback_config: zml.callback.Config = .{ -/// // assumption this custom call makes about the input / output buffers -/// }; -/// -/// // Required, this will tell the callback in which env it runs. -/// platform: zml.Platform, -/// // data needed by the callback -/// my_data: []const u8, -/// -/// // storage modified by the runtime to tell the callback where it should write its results. -/// // Normally the callback doesn't need to allocate as the input and output buffers are given. -/// results: [1]Buffer = undefined, -/// -/// pub fn init(my_data: []const u8) !MyCallback { -/// return .{ .my_data = my_data }; -/// } -/// -/// pub fn call(callback: *MyCallback, input: Buffer) !void { -/// // Do something with `input` and `callback.my_data`, write the results inside `callback.results[0]` -/// } -/// }; -/// ``` -/// -/// See eg the implementation of the `zml.callback.Print` callback, for a practical example. -/// -/// Note calling this during the compilation of a module, isn't enough: -/// -/// * backend need to be made aware of the callback, see `zml.Platform.registerCallback` -/// * executable need to know the specific data needed by `MyCallback`, see `zml.Exe.bind` -pub fn call( - comptime Callback: type, - inputs: TensorArgs(Callback), - output_shapes: []const Shape, -) []Tensor { - checkIsValidCallback(Callback); - - const ctx = CompilationContext.current(); - const allocator = ctx.allocator(); - const mlir_ctx = ctx.mlirCtx(); - const platform = ctx._platform; - const pjrt_api = platform.pjrt_api; - - if (pjrt_api.ffi() == null) { - stdx.debug.panic("Custom calls are not supported for target {s}", .{@tagName(platform.target)}); - } - - const output_tensors = allocator.alloc(Tensor, output_shapes.len) catch @panic("OOM"); - // Note: we don't always free output_tensor, because it's returned to the caller. - // It's also why we allocate it first so that it doesn't fragment the arena. - errdefer allocator.free(output_tensors); - - const output_types = allocator.alloc(mlir.Type, output_shapes.len) catch @panic("OOM"); - defer allocator.free(output_types); - for (output_types, output_shapes) |*output_type, output_shape| { - output_type.* = mlirx.tensorType(mlir_ctx, output_shape); - } - const input_values = allocator.alloc(mlir.Value, inputs.len) catch @panic("OOM"); - defer allocator.free(input_values); - for (input_values, inputs) |*input_value, input_tensor| { - input_value.* = input_tensor.value(); - } - - const target_name = "zml$" ++ @typeName(Callback); - const op = stablehlo.custom_call( - mlir_ctx, - input_values, - .{ - .call_target_name = target_name, - .api_version = .typed_ffi, - .backend_config = .dict(mlir_ctx, &.{}), - .additional_attributes = &.{.{ "mhlo.frontend_attributes", .dict(mlir_ctx, &.{}) }}, - .has_side_effect = Callback.callback_config.has_side_effect, - .output_operand_aliases = Callback.callback_config.output_operand_aliases, - }, - output_types, - mlir_ctx.location(@src()), - ); - - for (output_tensors, output_shapes, 0..) |*output_tensor, output_shape, i| { - output_tensor.* = Tensor._result(output_shape, op.result(i)); - } - return output_tensors; -} - -/// Describe properties of a callback -/// -/// * output_operand_aliases: the callback reuse input buffer to write the output -/// * copy_inputs_to_host_pinned: the callback need to work on host visible buffers -/// * traits: PJRT specified properties of the callback -pub const Config = struct { - output_operand_aliases: []const i64 = &.{}, - copy_inputs_to_host_pinned: bool = false, - - /// Indicates that the handler is compatible with command buffers (ie Cuda graphs). - /// The Cuda backend traces the execution of the callback and records it as a Cuda graph. - /// To be compatible with CUDA graphs, the callback need: - /// 1. to not use any synchronous operations - /// 2. to not allocate any new host/device memory - /// See: https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#prohibited-and-unhandled-operations. - traits: pjrt.ffi.HandlerTraits = .{ .command_buffer_compatible = false }, - // TODO: handle sharded inputs - has_side_effect: bool = true, -}; - -/// Compile-time check that a callback has all informations we require. -pub fn checkIsValidCallback(Callback: type) void { - stdx.debug.assertComptime(@hasDecl(Callback, "call"), "Expected callback {} to have a call method", .{Callback}); - const ArgsT = stdx.meta.FnArgs(Callback.call); - inline for (@typeInfo(ArgsT).@"struct".fields[1..]) |field| { - stdx.debug.assertComptime(field.type == Buffer, "Expected callback {}.call arguments to be of type zml.Buffer, got {}", .{ Callback, field.type }); - } - - stdx.debug.assertComptime(@hasDecl(Callback, "type_id") and @TypeOf(Callback.type_id) == pjrt.ffi.TypeId, "Expected callback {} to have a field `pub var type_id: pjrt.ffi.TypeId`", .{Callback}); - stdx.debug.assertComptime(@hasDecl(Callback, "callback_config") and @TypeOf(Callback.callback_config) == Config, "Expected callback {} to have a field `pub const callback_config: zml.CustomCallOptions`", .{Callback}); -} - -pub fn register(Callback: type, platform: Platform) pjrt.ApiError!void { - checkIsValidCallback(Callback); - - const ffi = platform.pjrt_api.ffi() orelse return error.Unavailable; - const target_name = "zml$" ++ @typeName(Callback); - - const proxy_cb = proxy(Callback); - const type_info: pjrt.Ffi.TypeInfo = .{}; - Callback.type_id = try ffi.registerTypeId(platform.pjrt_api, @typeName(Callback), &type_info.toCStruct()); - try ffi.register(platform.pjrt_api, target_name, @tagName(platform.target), &proxy_cb, Callback.callback_config.traits); - log.debug("Registered custom call {} with target name \"{s}\"", .{ Callback, target_name }); -} - -fn proxy(Callback: type) pjrt.ffi.Handler { - return struct { - pub fn cb(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { - return CallbackImpl(Callback, call_frame); - } - }.cb; -} - -fn CallbackImpl(comptime Callback: type, call_frame: *pjrt.ffi.CallFrame) ?*pjrt.ffi.Error { - if (call_frame.registeringHook()) return null; - - const opts = Callback.callback_config; - - const execution_context = call_frame.ctx; - log.debug("Custom call {s} called !", .{@typeName(Callback)}); - const user_ctx_opaque = execution_context.getContext(Callback.type_id, call_frame.api) catch { - log.err("{} user data was never given for current executable", .{Callback}); - return .create(call_frame.api, .failed_precondition, "failed to fetch user context" ++ @typeName(Callback)); - }; - const user_ctx: *Callback = @ptrCast(@alignCast(user_ctx_opaque)); - // We actually have one more constraint here, we force the Callback to have a platform field, - // and to correctly set it. - // Is this good ? We could also simplify this by registering ourselves the `Platform` type id. - const platform: Platform = user_ctx.platform; - - // Hook to get a cuda stream in the callback. - if (@hasField(Callback, "stream") and platform.target != .cpu) { - const stream = call_frame.api.stream(execution_context); - user_ctx.stream = stream; - } - - var callback_args: std.meta.ArgsTuple(@TypeOf(Callback.call)) = undefined; - callback_args[0] = user_ctx; - - inline for (1..callback_args.len, call_frame.args.buffers()) |i, ffi_buffer| { - const shape = shapeFromFfi(ffi_buffer); - var zml_buffer: Buffer = if (platform.target == .cpu) - .asViewOfHostBuffer(platform, .fromBytes(shape, ffi_buffer.data[0..shape.byteSize()])) - else - .asViewOfDeviceBuffer(platform, shape, null, ffi_buffer.data); - if (opts.copy_inputs_to_host_pinned and platform.target != .cpu) { - // log.debug("Copying argument {d} {f} {x} to host_pinned memory !", .{ i, zml_buffer, @intFromPtr(zml_buffer.devicePtr()) }); - zml_buffer = zml_buffer.copyToMemory(platform, .host_pinned, .{ .wait = true }) catch |err| { - log.err("Failed to copy input buffer {d} {f} {x} to host_pinned: {}", .{ i, zml_buffer, @intFromPtr(zml_buffer.devicePtr()), err }); - return .create(call_frame.api, .resource_exhausted, "host pinned OOM"); - }; - // log.debug("--> {f} {x}", .{ zml_buffer, @intFromPtr(zml_buffer.devicePtr()) }); - } - callback_args[i] = zml_buffer; - } - - defer { - if (opts.copy_inputs_to_host_pinned and platform.target != .cpu) { - inline for (1..callback_args.len) |i| callback_args[i].deinit(); - } - } - - for (0..call_frame.results.len) |i| { - const ffi_buffer = call_frame.results.buffers()[i]; - const ffi_buffer_shape = shapeFromFfi(ffi_buffer); - - if (platform.target == .cpu) { - user_ctx.results[i] = Buffer.asViewOfHostBuffer(platform, HostBuffer.fromBytes(ffi_buffer_shape, ffi_buffer.data[0..ffi_buffer_shape.byteSize()])); - } else { - user_ctx.results[i] = Buffer.asViewOfDeviceBuffer(platform, shapeFromFfi(ffi_buffer), null, ffi_buffer.data); - } - } - - @call(.auto, Callback.call, callback_args) catch |err| { - log.err("Callback {} failed with {}", .{ Callback, err }); - return .create(call_frame.api, .internal, "internal callback error"); - }; - - return .ok; -} - -/// Internal custom calls. -/// These are not meant to be used by users, but rather by the library itself. -pub const internal_callbacks = [_]type{ - Print, -}; - -pub fn registerInternalCallbacks(platform: Platform) !void { - inline for (internal_callbacks) |Callback| { - try register(Callback, platform); - // log.debug("Registered internal custom call {s} with type_id {d}", .{ @typeName(Callback), Callback.type_id.type_id }); - } -} - -/// Allocate user data data needed by the ZML provided custom calls. -pub fn bindInternalCallbacks( - arena: std.mem.Allocator, - platform: Platform, - ffi: pjrt.Ffi, - execute_context: *pjrt.ExecuteContext, -) (std.mem.Allocator.Error || pjrt.ApiError)!void { - // Atm we don't have a mechanism to detect which ZML callbacks the executable needs, - // so we always allocate. - { - // Print - const print_ptr = try arena.create(Print); - print_ptr.* = try .init(platform); - try addUserData(Print, platform.pjrt_api, ffi, execute_context, print_ptr); - } -} - -pub fn addUserData( - Callback: type, - api: *const pjrt.Api, - ffi: pjrt.Ffi, - execute_context: *pjrt.ExecuteContext, - user_data: *Callback, -) pjrt.ApiError!void { - try ffi.addUserData( - api, - execute_context, - .{ .type_id = Callback.type_id.type_id, .user_data = @ptrCast(user_data) }, - ); - log.debug("Bound {s}@{x} with type id {d} on {any}", .{ @typeName(Callback), @intFromPtr(user_data), Callback.type_id.type_id, execute_context }); -} - -/// The print callback -pub const Print = struct { - // a unique type_id will be set by the PJRT plugin during registration. - pub var type_id: pjrt.ffi.TypeId = undefined; - - pub const callback_config: Config = .{ - // Print callback pretends to modify the given input buffer, but just returns it unmodified. - .output_operand_aliases = &.{0}, - // It also needs PJRT to copy the data on the host first so it can print it. - .copy_inputs_to_host_pinned = true, - // Print is fairly predictable and can be captured in an execution graph. - .traits = .{ .command_buffer_compatible = false }, - .has_side_effect = false, - }; - - platform: Platform, - results: [1]Buffer = undefined, - - pub fn init(platform: Platform) !Print { - return .{ .platform = platform }; - } - - pub fn call(_: *Print, input: Buffer) !void { - std.log.defaultLog(.info, .zml, "Device buffer: {f}: {d:20.3}", .{ input, input.asHostBuffer() }); - } -}; - -fn shapeFromFfi(ffi_buffer: *const pjrt.ffi.Buffer) Shape { - const dt: DataType = switch (ffi_buffer.dtype) { - .invalid => stdx.debug.panic("Invalid FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), - .token, .f8e4m3, .f8e3m4 => stdx.debug.panic("Unsupported FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), - inline else => |t| @field(DataType, @tagName(t)), - }; - return Shape.init(ffi_buffer.dims(), dt); -} - -fn TensorArgs(Callback: type) type { - const ArgsT = stdx.meta.FnArgs(Callback.call); - - const args = @typeInfo(ArgsT).@"struct".fields; - return [args.len - 1]Tensor; -} diff --git a/zml2/constants.zig b/zml/constants.zig similarity index 100% rename from zml2/constants.zig rename to zml/constants.zig diff --git a/zml/context.zig b/zml/context.zig deleted file mode 100644 index 27d635a0..00000000 --- a/zml/context.zig +++ /dev/null @@ -1,217 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); - -const c = @import("c"); -const mlir = @import("mlir"); -const runfiles = @import("runfiles"); -const runtimes = @import("runtimes"); -const stdx = @import("stdx"); - -const pjrt = @import("pjrtx.zig"); - -const zml = struct { - pub const callback = @import("callback.zig"); - pub const HostBuffer = @import("hostbuffer.zig").HostBuffer; - pub const Platform = @import("platform.zig").Platform; - pub const platform = @import("platform.zig"); - pub const Shape = @import("shape.zig").Shape; - pub const Target = @import("platform.zig").Target; -}; - -const PjrtApiMap = std.EnumArray(zml.Target, ?*const pjrt.Api); -const PlatformsMap = std.EnumArray(zml.Target, ?zml.Platform); -const log = std.log.scoped(.@"zml/context"); - -test { - std.testing.refAllDecls(Context); -} - -/// Every program using ZML must start with a `zml.Context.init(.{});` -/// The ZML context contains global state to interact with the different -/// devices available on your system. -/// Note that the runtimes available depends on how the program was compiled. -/// For example you need to compile your program with `--//runtimes:cuda=true` -/// to have the CUDA runtime available. -pub const Context = struct { - var apis = PjrtApiMap.initFill(null); - var apis_once = std.once(struct { - fn call() void { - inline for (comptime std.enums.values(runtimes.Platform)) |t| { - if (runtimes.load(t)) |api| { - Context.apis.set(t, api); - } else |_| {} - } - } - }.call); - - var mlir_once = std.once(struct { - fn call() void { - mlir.registerPasses("Transforms"); - } - }.call); - - var runfiles_once = std.once(struct { - fn call_() !void { - if (std.process.hasEnvVarConstant("RUNFILES_MANIFEST_FILE") or std.process.hasEnvVarConstant("RUNFILES_DIR")) { - return; - } - - var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); - const allocator = arena.allocator(); - defer arena.deinit(); - - var envMap = std.process.EnvMap.init(allocator); - var r = (try runfiles.Runfiles.create(.{ .allocator = allocator })) orelse return; - try r.environment(&envMap); - - var it = envMap.iterator(); - while (it.next()) |entry| { - const keyZ = try allocator.dupeZ(u8, entry.key_ptr.*); - const valueZ = try allocator.dupeZ(u8, entry.value_ptr.*); - _ = c.setenv(keyZ.ptr, valueZ.ptr, 1); - } - } - - fn call() void { - call_() catch @panic("Unable to init runfiles env"); - } - }.call); - - platforms: PlatformsMap, - - /// Creates a ZML Context and returns it. - pub fn init() !Context { - Context.runfiles_once.call(); - Context.apis_once.call(); - Context.mlir_once.call(); - - var num_platforms: u8 = 0; - for (Context.apis.values) |api| { - if (api != null) num_platforms += 1; - } - if (num_platforms == 0) { - log.err("No platform available", .{}); - return error.NoPlatformAvailable; - } - - return .{ .platforms = PlatformsMap.initFill(null) }; - } - - fn platformToLibrary(comptime target: zml.Target) []const u8 { - const ext = switch (builtin.os.tag) { - .windows => ".dll", - .macos, .ios, .watchos => ".dylib", - else => ".so", - }; - return switch (target) { - inline else => "libpjrt_" ++ @tagName(target) ++ ext, - }; - } - - pub fn pjrtApi(target: zml.Target) *const pjrt.Api { - return Context.apis.get(target).?; - } - - pub fn deinit(self: *Context) void { - var iterator = self.platforms.iterator(); - while (iterator.next()) |entry| { - if (entry.value.*) |*p| { - p.deinit(); - } - } - self.* = undefined; - } - - const prefered_targets = [_]zml.Target{ .tpu, .neuron, .cuda, .rocm, .cpu }; - - /// Automatically selects the best Platform loaded in the current Context. - /// - /// For example, if supported, this will select a platform corresponding to an accelerator (GPU, TPU, ...). - pub fn autoPlatform(self: *Context, opts: zml.Platform.CreateOptions) zml.Platform { - stdx.debug.assert(prefered_targets.len == apis.values.len, "New target need to be inserted inside `zml.Context.preferred_targets`", .{}); - - return self.platformByPreferences(opts, &prefered_targets); - } - - /// Given a list of preferred targets to select the best Platform - /// - /// For example, if supported, this will select a platform corresponding to an accelerator (GPU, TPU, ...). - pub fn platformByPreferences(self: *Context, opts: zml.Platform.CreateOptions, prefered: []const zml.Target) zml.Platform { - // Try prefered targets. - for (prefered) |target| { - if (apis.get(target) == null) continue; - return self.platform(target, opts) catch |err| { - log.err("Failed to load platform .{s}: {}", .{ @tagName(target), err }); - continue; - }; - } - - // Try unlisted targets - var it = Context.apis.iterator(); - while (it.next()) |entry| { - const target = entry.key; - // CPU should only be use as fallback. - if (target == .cpu) continue; - if (entry.value.* == null) continue; - if (std.mem.indexOfScalar(zml.Target, prefered, target) != null) continue; - return self.platform(target, opts) catch |err| { - log.err("Failed to load platform .{s}: {}", .{ @tagName(target), err }); - continue; - }; - } - - // Finally fallback to cpu. - return self.platform(.cpu, opts) catch { - log.err("No platform available", .{}); - @panic("No platform available !"); - }; - } - - pub fn platform(self: *Context, target: zml.Target, opts: zml.Platform.CreateOptions) !zml.Platform { - if (self.platforms.get(target)) |p| { - return p; - } - const api = Context.apis.get(target); - if (api == null) return error.PlatformNotCompiled; - const p = try zml.Platform.init(target, api.?, opts); - if (p.getDevices().len == 0) { - log.err("No device found for platform {} !", .{target}); - return error.NoDevicesFound; - } - - self.platforms.set(target, p); - try zml.callback.registerInternalCallbacks(p); - - return p; - } - - pub fn printAvailablePlatforms(self: Context, selected: zml.Platform) void { - // List available targets - log.info("Available Platforms:", .{}); - const selected_prefix = "✅"; - const not_selected_prefix = "• "; - const selected_postfix = "(AUTO-SELECTED)"; - const not_selected_postfix = ""; - - for (zml.platform.available_targets) |target| { - log.info(" {s} {s} {s}", .{ - if (target == selected.target) selected_prefix else not_selected_prefix, - @tagName(target), - if (target == selected.target) selected_postfix else not_selected_postfix, - }); - - // now the platform's devices - if (self.platforms.get(target)) |pfm| { - for (pfm.getDevices(), 0..) |device, index| { - const deviceKind = device.getDescription(pfm.pjrt_api).getKind(pfm.pjrt_api); - log.info(" ◦ #{d}: {s}", .{ - index, - deviceKind, - }); - // we only list 1 CPU device - if (target == .cpu) break; - } - } - } - } -}; diff --git a/zml/dtype.zig b/zml/dtype.zig index b14f483e..8ec73c3d 100644 --- a/zml/dtype.zig +++ b/zml/dtype.zig @@ -5,10 +5,6 @@ const floats = @import("floats.zig"); const C64 = std.math.Complex(f32); const C128 = std.math.Complex(f64); -test { - std.testing.refAllDecls(@This()); -} - pub const DataType = enum(u8) { bool, // Note: the support of the float8 is a bit spotty, f8e4m3b11fnuz seems to be the most supported one on Cuda. @@ -137,7 +133,7 @@ pub const DataType = enum(u8) { } pub fn toZigType(comptime dtype: DataType) type { - return @FieldType(Data, @tagName(dtype)); + return @FieldType(Value, @tagName(dtype)); } pub fn isSignedInt(dtype: DataType) bool { @@ -195,171 +191,171 @@ pub const DataType = enum(u8) { try std.testing.expectEqual(DataType.c64.resolvePeerType(.f32), null); } - pub fn zero(dtype: DataType) Data { - return Data.init(dtype, 0); + pub fn zero(dtype: DataType) Value { + return .init(dtype, 0); } - pub fn one(dtype: DataType) Data { - return Data.init(dtype, 1); + pub fn one(dtype: DataType) Value { + return .init(dtype, 1); } - pub fn minValue(dtype: DataType) Data { + pub fn minValue(dtype: DataType) Value { return switch (dtype) { .bool => .{ .bool = false }, - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).zero), - inline .f8e5m2, .bf16 => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).minus_inf), - inline .f16, .f32, .f64 => |tag| @unionInit(Data, @tagName(tag), -std.math.inf(@FieldType(Data, @tagName(tag)))), - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Data, @tagName(tag), std.math.minInt(@FieldType(Data, @tagName(tag)))), - inline else => |tag| @panic("Unsupported type: " ++ @tagName(tag)), + inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz, .f8e8m0, .f4e2m1 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).min), + inline .f8e5m2, .f8e3m4, .f8e4m3, .bf16 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).minus_inf), + inline .f16, .f32, .f64 => |tag| @unionInit(Value, @tagName(tag), -std.math.inf(@FieldType(Value, @tagName(tag)))), + inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Value, @tagName(tag), std.math.minInt(@FieldType(Value, @tagName(tag)))), + inline .c64, .c128 => |tag| @panic("DataType doesn't have a min value: " ++ @tagName(tag)), }; } - pub fn maxValue(dtype: DataType) Data { + pub fn maxValue(dtype: DataType) Value { return switch (dtype) { .bool => .{ .bool = true }, - inline .f4e2m1, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz, .f8e8m0 => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), - inline .f8e3m4, .f8e4m3, .f8e5m2, .bf16 => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).inf), - inline .f16, .f32, .f64 => |tag| @unionInit(Data, @tagName(tag), std.math.inf(@FieldType(Data, @tagName(tag)))), - inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Data, @tagName(tag), std.math.maxInt(@FieldType(Data, @tagName(tag)))), + inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz, .f8e8m0, .f4e2m1 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).max), + inline .f8e5m2, .f8e3m4, .f8e4m3, .bf16 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).inf), + inline .f16, .f32, .f64 => |tag| @unionInit(Value, @tagName(tag), std.math.inf(@FieldType(Value, @tagName(tag)))), + inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Value, @tagName(tag), std.math.maxInt(@FieldType(Value, @tagName(tag)))), inline .c64, .c128 => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), }; } - pub fn constant(dtype: DataType, value: anytype) Data { - return Data.init(dtype, value); + pub fn constant(dtype: DataType, value: anytype) Value { + return .init(dtype, value); } -}; -pub const Data = union(DataType) { - bool: bool, - f4e2m1: floats.Float4E2M1, - f8e3m4: floats.Float8E3M4, - f8e4m3: floats.Float8E4M3, - f8e4m3b11fnuz: floats.Float8E4M3B11FNUZ, - f8e4m3fn: floats.Float8E4M3FN, - f8e4m3fnuz: floats.Float8E4M3FNUZ, - f8e5m2: floats.Float8E5M2, - f8e5m2fnuz: floats.Float8E5M2FNUZ, - f8e8m0: floats.Float8E8M0, - bf16: floats.BFloat16, - f16: f16, - f32: f32, - f64: f64, - i2: i2, - i4: i4, - i8: i8, - i16: i16, - i32: i32, - i64: i64, - u2: u2, - u4: u4, - u8: u8, - u16: u16, - u32: u32, - u64: u64, - c64: C64, - c128: C128, + pub const Value = union(DataType) { + bool: bool, + f4e2m1: floats.Float4E2M1, + f8e3m4: floats.Float8E3M4, + f8e4m3: floats.Float8E4M3, + f8e4m3b11fnuz: floats.Float8E4M3B11FNUZ, + f8e4m3fn: floats.Float8E4M3FN, + f8e4m3fnuz: floats.Float8E4M3FNUZ, + f8e5m2: floats.Float8E5M2, + f8e5m2fnuz: floats.Float8E5M2FNUZ, + f8e8m0: floats.Float8E8M0, + bf16: floats.BFloat16, + f16: f16, + f32: f32, + f64: f64, + i2: i2, + i4: i4, + i8: i8, + i16: i16, + i32: i32, + i64: i64, + u2: u2, + u4: u4, + u8: u8, + u16: u16, + u32: u32, + u64: u64, + c64: C64, + c128: C128, - /// Creates `Data` from a `value`. - /// - /// If the `dtype` and `@TypeOf(value)` are incompatible - /// or a cast from `value` to `FieldType(dtype)` would - /// be lossy, a panic occurs. - pub fn init(dtype_: DataType, value: anytype) Data { - const T = @TypeOf(value); - const Ti = @typeInfo(T); + /// Creates `Data` from a `value`. + /// + /// If the `dtype` and `@TypeOf(value)` are incompatible + /// or a cast from `value` to `FieldType(dtype)` would + /// be lossy, a panic occurs. + pub fn init(dtype_: DataType, value: anytype) Value { + const T = @TypeOf(value); + const Ti = @typeInfo(T); - return switch (dtype_) { - .bool => switch (Ti) { - .bool => .{ .bool = value }, - .comptime_int, .int, .comptime_float, .float => .{ .bool = value != 0 }, - else => @panic("Could not create Data of type bool from value of type " ++ @typeName(T)), - }, - inline .f4e2m1, .f8e3m4, .f8e4m3, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .f8e8m0, .bf16 => |tag| switch (Ti) { - .comptime_int, .int => @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).fromF32(@floatFromInt(value))), - .comptime_float, .float => @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).fromF32(@floatCast(value))), - else => @panic("Could not create Data of type bf16 from value of type " ++ @typeName(T)), - }, - inline .f16, .f32, .f64 => |tag| switch (Ti) { - .comptime_int, .int => @unionInit(Data, @tagName(tag), @floatFromInt(value)), - .comptime_float, .float => @unionInit(Data, @tagName(tag), @floatCast(value)), - else => @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), - }, - inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| switch (Ti) { - .comptime_int => blk: { - const OutT = @FieldType(Data, @tagName(tag)); - if (value >= std.math.minInt(OutT) and value <= std.math.maxInt(OutT)) { - break :blk @unionInit(Data, @tagName(tag), @intCast(value)); - } else { - @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)); - } + return switch (dtype_) { + .bool => switch (Ti) { + .bool => .{ .bool = value }, + .comptime_int, .int, .comptime_float, .float => .{ .bool = value != 0 }, + else => @panic("Could not create Value of type bool from value of type " ++ @typeName(T)), }, - .int => @unionInit(Data, @tagName(tag), @intCast(value)), - else => @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), - }, - .c64 => switch (T) { - C64 => .{ .c64 = value }, - C128 => .{ .c64 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, - else => @panic("Could not create Data of type c64 from value of type " ++ @typeName(T)), - }, - .c128 => switch (T) { - C64 => .{ .c128 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, - C128 => .{ .c128 = value }, - else => @panic("Could not create Data of type c128 from value of type " ++ @typeName(T)), - }, - }; - } - - test init { - try std.testing.expectEqual(20.0, Data.init(.f16, 20).f16); - try std.testing.expectEqual(20.5, Data.init(.f16, 20.5).f16); - try std.testing.expectEqual(20, Data.init(.f16, @as(u8, 20)).f16); - try std.testing.expectEqual(-20, Data.init(.f16, @as(i8, -20)).f16); - try std.testing.expectEqual(2000.5, Data.init(.f16, @as(f32, 2000.5)).f16); - - try std.testing.expectEqual(true, Data.init(.bool, true).bool); - - try std.testing.expectEqual(10, Data.init(.u8, 10).u8); - try std.testing.expectEqual(10, Data.init(.u8, @as(u16, 10)).u8); - - try std.testing.expectEqual(10, Data.init(.i8, 10).i8); - try std.testing.expectEqual(10, Data.init(.i8, @as(u16, 10)).i8); - try std.testing.expectEqual(-10, Data.init(.i8, -10).i8); - try std.testing.expectEqual(-10, Data.init(.i8, @as(i16, -10)).i8); - - try std.testing.expectEqual(C64.init(1, 2), Data.init(.c64, C64.init(1, 2)).c64); - try std.testing.expectEqual(C64.init(1, 2), Data.init(.c64, C128.init(1, 2)).c64); - try std.testing.expectEqual(C128.init(1, 2), Data.init(.c128, C128.init(1, 2)).c128); - try std.testing.expectEqual(C128.init(1, 2), Data.init(.c128, C64.init(1, 2)).c128); - } - - pub fn dtype(self: Data) DataType { - return std.meta.activeTag(self); - } - - pub fn constSlice(data: *const Data) []const u8 { - return switch (data.*) { - inline else => |*value| std.mem.asBytes(value), - }; - } - - pub fn as(self: Data, comptime T: type) T { - // TODO allow more lossless conversions - switch (@typeInfo(T)) { - .bool => return self.bool, - .float => switch (self) { - inline .f16, .f32, .f64 => |v| return @floatCast(v), - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16 => |v| return @floatCast(v.toF32()), - else => {}, - }, - .int => switch (self) { - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |v| return @intCast(v), - else => {}, - }, - else => {}, + inline .f4e2m1, .f8e3m4, .f8e4m3, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .f8e8m0, .bf16 => |tag| switch (Ti) { + .comptime_int, .int => @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).fromF32(@floatFromInt(value))), + .comptime_float, .float => @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).fromF32(@floatCast(value))), + else => @panic("Could not create Value of type bf16 from value of type " ++ @typeName(T)), + }, + inline .f16, .f32, .f64 => |tag| switch (Ti) { + .comptime_int, .int => @unionInit(Value, @tagName(tag), @floatFromInt(value)), + .comptime_float, .float => @unionInit(Value, @tagName(tag), @floatCast(value)), + else => @panic("Could not create Value of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), + }, + inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| switch (Ti) { + .comptime_int => blk: { + const OutT = @FieldType(Value, @tagName(tag)); + if (value >= std.math.minInt(OutT) and value <= std.math.maxInt(OutT)) { + break :blk @unionInit(Value, @tagName(tag), @intCast(value)); + } else { + @panic("Could not create Value of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)); + } + }, + .int => @unionInit(Value, @tagName(tag), @intCast(value)), + else => @panic("Could not create Value of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), + }, + .c64 => switch (T) { + C64 => .{ .c64 = value }, + C128 => .{ .c64 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, + else => @panic("Could not create Value of type c64 from value of type " ++ @typeName(T)), + }, + .c128 => switch (T) { + C64 => .{ .c128 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, + C128 => .{ .c128 = value }, + else => @panic("Could not create Value of type c128 from value of type " ++ @typeName(T)), + }, + }; } - std.debug.panic("Unsupported conversion {} -> {s}", .{ self.dtype(), @typeName(T) }); - } + + test init { + try std.testing.expectEqual(20.0, Value.init(.f16, 20).f16); + try std.testing.expectEqual(20.5, Value.init(.f16, 20.5).f16); + try std.testing.expectEqual(20, Value.init(.f16, @as(u8, 20)).f16); + try std.testing.expectEqual(-20, Value.init(.f16, @as(i8, -20)).f16); + try std.testing.expectEqual(2000.5, Value.init(.f16, @as(f32, 2000.5)).f16); + + try std.testing.expectEqual(true, Value.init(.bool, true).bool); + + try std.testing.expectEqual(10, Value.init(.u8, 10).u8); + try std.testing.expectEqual(10, Value.init(.u8, @as(u16, 10)).u8); + + try std.testing.expectEqual(10, Value.init(.i8, 10).i8); + try std.testing.expectEqual(10, Value.init(.i8, @as(u16, 10)).i8); + try std.testing.expectEqual(-10, Value.init(.i8, -10).i8); + try std.testing.expectEqual(-10, Value.init(.i8, @as(i16, -10)).i8); + + try std.testing.expectEqual(C64.init(1, 2), Value.init(.c64, C64.init(1, 2)).c64); + try std.testing.expectEqual(C64.init(1, 2), Value.init(.c64, C128.init(1, 2)).c64); + try std.testing.expectEqual(C128.init(1, 2), Value.init(.c128, C128.init(1, 2)).c128); + try std.testing.expectEqual(C128.init(1, 2), Value.init(.c128, C64.init(1, 2)).c128); + } + + pub fn dtype(self: Value) DataType { + return std.meta.activeTag(self); + } + + pub fn asBytes(data: *const Value) []const u8 { + return switch (data.*) { + inline else => |*value| std.mem.asBytes(value), + }; + } + + pub fn as(self: Value, comptime T: type) T { + // TODO allow more lossless conversions + switch (@typeInfo(T)) { + .bool => return self.bool, + .float => switch (self) { + inline .f16, .f32, .f64 => |v| return @floatCast(v), + inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16 => |v| return @floatCast(v.toF32()), + else => {}, + }, + .int => switch (self) { + inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |v| return @intCast(v), + else => {}, + }, + else => {}, + } + std.debug.panic("Unsupported conversion {} -> {s}", .{ self.dtype(), @typeName(T) }); + } + }; }; pub fn mantissaSize(dtype: DataType) usize { @@ -373,3 +369,5 @@ pub fn mantissaSize(dtype: DataType) usize { else => @panic("Can't get mantissa size for a non-float dtype"), }; } + +pub const Data = DataType.Value; diff --git a/zml/exe.zig b/zml/exe.zig index 15d33b0c..122f99d8 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -2,475 +2,239 @@ const std = @import("std"); const stdx = @import("stdx"); -const aio = @import("aio.zig"); const Buffer = @import("buffer.zig").Buffer; -const Bufferized = @import("tensor.zig").Bufferized; -const callback = @import("callback.zig"); -const CompilationContext = @import("module.zig").CompilationContext; const meta = @import("meta.zig"); const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; -const ShapeOf = @import("tensor.zig").ShapeOf; -const log = std.log.scoped(.@"zml/exe"); - -test { - std.testing.refAllDecls(@This()); -} - -/// Compiles a Model struct with the given configuration and shapes, for the given platform. -/// The steps are: -/// * lookup at tensors available in the store and create a `model: Model` struct with them -/// * call `model.init(init_args)` to fields of the model that aren't Tensor, ie hyperparemeters/config -/// * generate MLIR by calling `model.forward` with tensor of the given shapes and other arguments -pub fn compile( - allocator: std.mem.Allocator, - comptime func: anytype, - init_args: anytype, - args_shapes: ShapeOf(ModuleSignature(func).ArgsT), - buffer_store: aio.BufferStore, +pub const Exe = struct { platform: Platform, -) !FnExe(func) { - return compileWithPrefix(allocator, func, init_args, args_shapes, buffer_store, platform, ""); -} - -/// Compiles a Model struct with the given configuration and shapes, for the given platform. -/// Uses a prefix for looking up model weights in the buffer store. -/// The steps are: -/// * lookup at tensors available in the store and create a `model: Model` struct with them -/// * call `model.init(init_args)` to fields of the model that aren't Tensor, ie hyperparemeters/config -/// * generate MLIR by calling `model.forward` with tensor of the given shapes and other arguments -pub fn compileWithPrefix( - allocator: std.mem.Allocator, - comptime func: anytype, - init_args: anytype, - args_shapes: ShapeOf(ModuleSignature(func).ArgsT), - buffer_store: aio.BufferStore, - platform: Platform, - prefix: []const u8, -) !FnExe(func) { - const ModelT = ModuleSignature(func).ModelT; - - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - var model = try aio.populateModelWithPrefix(ModelT, arena, buffer_store, prefix); - - // If the Model has a "init" function, call it with the given parameters. - if (@hasDecl(ModelT, "init")) { - // TODO(Corentin,@Improvement): Add a warning/error if there is no init function but init_args is non-void. - @call(.auto, ModelT.init, .{@as(*ModelT, &model)} ++ init_args); - } - - return compileModel(allocator, func, model, args_shapes, platform); -} - -/// Compiles a Model struct with the given configuration and shapes, for the given platform. -/// Generate MLIR by calling `model.forward` with tensor of the given shapes and other arguments -pub fn compileModel( - allocator: std.mem.Allocator, - comptime func: anytype, - model: ModuleSignature(func).ModelT, - args_shapes: ShapeOf(ModuleSignature(func).ArgsT), - platform: Platform, -) !FnExe(func) { - const ModelT = ModuleSignature(func).ModelT; - const name = @typeName(ModelT) ++ ".forward"; - log.info("Compiling {s} with {f}", .{ name, stdx.fmt.any(args_shapes) }); - - var context = try CompilationContext.init(allocator, name, platform); - defer context.deinit(); - - return .{ .inner = try context.compileInternal(allocator, func, .{model} ++ args_shapes) }; -} - -/// Compiles a function with the given configuration and shapes, for the given platform. -/// Generate MLIR by calling the given function with tensor of the given shapes. -pub fn compileFn( - allocator: std.mem.Allocator, - comptime func: anytype, - args: ShapeOf(stdx.meta.FnArgs(func)), - platform: Platform, -) !FnExe(func) { - var pretty_name = try prettyFnName(func, allocator); - defer pretty_name.deinit(allocator); - log.info("Compiling {s} with {f}", .{ pretty_name.items, stdx.fmt.any(args) }); - - var context = try CompilationContext.init(allocator, pretty_name.items, platform); - defer context.deinit(); - - return .{ .inner = try context.compileInternal(allocator, func, args) }; -} - -pub fn FnExe(comptime func: anytype) type { - return Exe(stdx.meta.FnArgs(func), stdx.meta.FnResult(func)); -} - -/// Represents a ZML model, compiled into a PJRT executable, and ready to call. -/// The buffers for the model weights are saved inside the struct and will be used in `call`. -/// You only need to pass the remaining arguments. -/// Creating a `ModuleExe` is a two steps proccess: -/// -/// ``` -/// const exe: zml.FnExe(MyModel.forward) = try zml.compile(allocator, MyModel.forward, init_args, model_shapes, buffer_store, platform);` -/// const module: zml.ModuleExe(MyModel.forward) = exe.prepare(model_buffers); -/// ``` -pub fn ModuleExe(comptime func: anytype) type { - const AllArgs = stdx.meta.FnArgs(func); - const len = @typeInfo(AllArgs).@"struct".fields.len; - stdx.debug.assertComptime(len > 0, "ModuleExe expects a function with at least one argument where the first one is treated as the module, got {}", .{func}); - return Exe(stdx.meta.Tail(AllArgs), stdx.meta.FnResult(func)); -} - -// making this a struct force all fields to be evaluted on creation, -// which gives a better error stacktrace -// than delaying the error to when the object fields are read. -const Sign = struct { - ModelT: type, - ArgsT: type, - ReturnT: type, -}; - -pub fn ModuleSignature(comptime func: anytype) Sign { - const AllArgsT = stdx.meta.FnArgs(func); - const len = @typeInfo(AllArgsT).@"struct".fields.len; - stdx.debug.assertComptime(len > 0, "ModuleExe expects a function with at least one argument where the first one is treated as the module, got {}", .{func}); - - return .{ - .ModelT = stdx.meta.Head(AllArgsT), - .ArgsT = stdx.meta.Tail(AllArgsT), - .ReturnT = stdx.meta.FnResult(func), - }; -} - -/// Represents an MLIR module compiled into a PJRT executable. -/// The BaseExe is a plain old struct and doesn't have information about Zig types. -/// -/// It also contains pre-allocated buffers so that we can pass them to PJRT_LoadedExecutable_Execute -/// without allocations. -pub const BaseExe = struct { - /// The platform for which this module was compiled. - platform: Platform, - - /// The PJRT executable representing the compiled module. exe: *pjrt.LoadedExecutable, - /// The execution context for this executable. - execute_context: ?*pjrt.ExecuteContext, + context: ?*pjrt.ExecuteContext = null, - /// Pre-allocated slice of buffers to use as inputs when the module is called. - input_per_device: []const [*]*pjrt.Buffer, + input_shapes: []const Shape, + output_shapes: []const Shape, - /// Pre-allocated slice of buffers to use as outputs when the module is called. - output_per_device: []const [*]*pjrt.Buffer, - - /// Number of buffers already fed to the executable. - ready_buffer_count: u32, - - /// Total number of buffers needed by this executable. - input_buffer_count: u32, - - input_shapes: []Shape, - result_shapes: []Shape, - - /// Num devices used (>1 for sharded executable) num_devices: u8, - /// Allocator backing memory - _arena: std.heap.ArenaAllocator, + arena: std.heap.ArenaAllocator, pub fn init( - parent_allocator: std.mem.Allocator, + allocator: std.mem.Allocator, platform: Platform, exe: *pjrt.LoadedExecutable, - args: struct { input_shapes: []const Shape, result_shapes: []const Shape, n_devices: u8 }, - ) !BaseExe { - var arena = std.heap.ArenaAllocator.init(parent_allocator); + input_shapes: []const Shape, + output_shapes: []const Shape, + num_devices: u8, + ) !Exe { + var arena = std.heap.ArenaAllocator.init(allocator); errdefer arena.deinit(); - const allocator = arena.allocator(); - const n_in = args.input_shapes.len; - const n_out = args.result_shapes.len; - const n_devices = args.n_devices; - // Allocate once for all the *pjrt.Buffer we need to store ... - const all_buffers = try allocator.alloc(*pjrt.Buffer, (n_in + n_out) * n_devices); - const all_input_buffers, const all_output_buffers = splitBuffer(*pjrt.Buffer, all_buffers, .{ n_in * n_devices, n_out * n_devices }); - // ... and once for all the [*]*pjrt.Buffer. - const all_per_device = try allocator.alloc([*]*pjrt.Buffer, 2 * n_devices); - const input_per_device, const output_per_device = splitBuffer([*]*pjrt.Buffer, all_per_device, .{ n_devices, n_devices }); - - for (0..n_devices) |i| { - input_per_device[i] = all_input_buffers[i * n_in ..].ptr; - output_per_device[i] = all_output_buffers[i * n_out ..].ptr; - } - - const all_shapes = try allocator.alloc(Shape, n_in + n_out); - @memcpy(all_shapes[0..n_in], args.input_shapes); - @memcpy(all_shapes[n_in..], args.result_shapes); - - var execute_context: ?*pjrt.ExecuteContext = null; - if (platform.pjrt_api.ffi()) |ffi| { - execute_context = try platform.pjrt_api.createExecuteContext(); - try callback.bindInternalCallbacks(allocator, platform, ffi, execute_context.?); - // log.info("Created context execution {*} for {*}", .{ execute_context, exe }); - } + const input_shapes_copy = try arena.allocator().dupe(Shape, input_shapes); + const output_shapes_copy = try arena.allocator().dupe(Shape, output_shapes); return .{ .platform = platform, .exe = exe, - .execute_context = execute_context, - .ready_buffer_count = 0, - .input_buffer_count = @intCast(n_in), - .num_devices = args.n_devices, - .input_per_device = input_per_device, - .output_per_device = output_per_device, - .input_shapes = all_shapes[0..n_in], - .result_shapes = all_shapes[n_in..], - ._arena = arena, + .input_shapes = input_shapes_copy, + .output_shapes = output_shapes_copy, + .num_devices = num_devices, + .arena = arena, }; } - pub fn deinit(self: BaseExe) void { - if (self.execute_context) |ctx| { - ctx.deinit(self.platform.pjrt_api); + pub fn deinit(self: *const Exe) void { + self.arena.deinit(); + } + + pub fn args(self: *const Exe, allocator: std.mem.Allocator) !Arguments { + return Arguments.init(allocator, self.input_shapes, self.num_devices); + } + + pub fn results(self: *const Exe, allocator: std.mem.Allocator) !Results { + return Results.init(allocator, self.output_shapes, self.num_devices, self.platform); + } + + pub const FlatBuffers = struct { + buffers: []const [*]*pjrt.Buffer, + raw_buffers: []const *pjrt.Buffer, + + num_devices: usize, + + pub fn init(allocator: std.mem.Allocator, count: usize, num_devices: usize) !FlatBuffers { + const raw_buffers = try allocator.alloc(*pjrt.Buffer, num_devices * count); + errdefer allocator.free(raw_buffers); + + const buffers = try allocator.alloc([*]*pjrt.Buffer, num_devices); + errdefer allocator.free(buffers); + + for (0..num_devices) |i| { + buffers[i] = raw_buffers[i * count ..].ptr; + } + + return .{ + .buffers = buffers, + .raw_buffers = raw_buffers, + .num_devices = num_devices, + }; } - self.exe.deinit(self.platform.pjrt_api); - self._arena.deinit(); - } - pub fn call(self: BaseExe) void { - stdx.debug.assert(self.input_buffer_count == self.ready_buffer_count, "BaseExe isn't ready to be called, expected {} buffer inputs got {}", .{ self.input_buffer_count, self.ready_buffer_count }); - return self._unsafeCall(); - } + pub fn deinit(self: *const FlatBuffers, allocator: std.mem.Allocator) void { + allocator.free(self.buffers); + allocator.free(self.raw_buffers); + } + }; - pub fn _unsafeCall(self: BaseExe) void { + pub const Arguments = struct { + flat_buffers: FlatBuffers, + expected_shapes: []const Shape, + + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize) !Arguments { + const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); + errdefer flat_buffers.deinit(allocator); + + const expected_shapes = try allocator.dupe(Shape, shapes); + errdefer allocator.free(expected_shapes); + + return .{ + .flat_buffers = flat_buffers, + .expected_shapes = expected_shapes, + }; + } + + pub fn deinit(self: *const Arguments, allocator: std.mem.Allocator) void { + allocator.free(self.expected_shapes); + self.flat_buffers.deinit(allocator); + } + + pub fn set(self: *Arguments, v: anytype) void { + return self.setPartial(v, 0); + } + + pub fn setPartial(self: *Arguments, v: anytype, offset: usize) void { + const LocalContext = struct { + self: *Arguments, + current_index: usize = 0, + }; + var context: LocalContext = .{ .self = self, .current_index = offset }; + meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *const Buffer) void { + stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected argument {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); + for (0..context_.self.flat_buffers.num_devices) |device_index| { + context_.self.flat_buffers.buffers[device_index][context_.current_index] = buffer._shards.get(device_index); + } + + context_.current_index += 1; + } + }.cb, &context, &v); + } + }; + + pub const Results = struct { + platform: Platform, + flat_buffers: FlatBuffers, + + expected_shapes: []const Shape, + + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize, platform: Platform) !Results { + const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); + errdefer flat_buffers.deinit(allocator); + + const expected_shapes = try allocator.dupe(Shape, shapes); + errdefer allocator.free(expected_shapes); + + return .{ + .platform = platform, + .flat_buffers = flat_buffers, + .expected_shapes = expected_shapes, + }; + } + + pub fn deinit(self: *const Results, allocator: std.mem.Allocator) void { + allocator.free(self.expected_shapes); + self.flat_buffers.deinit(allocator); + } + + pub fn get(self: *Results, comptime T: type) T { + var result: T = undefined; + const LocalContext = struct { + self: *Results, + current_index: usize = 0, + }; + var context: LocalContext = .{ .self = self, .current_index = 0 }; + meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *Buffer) void { + var shards: Buffer.Shards = .{}; + for (0..context_.self.flat_buffers.num_devices) |device_index| { + shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); + } + buffer.* = Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], shards.constSlice()); + context_.current_index += 1; + } + }.cb, &context, &result); + return result; + } + + pub fn fill(self: *Results, v: anytype) void { + const LocalContext = struct { + self: *Results, + current_index: usize = 0, + }; + var context: LocalContext = .{ .self = self, .current_index = 0 }; + meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *Buffer) void { + //stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected result {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); + var shards: Buffer.Shards = .{}; + for (0..context_.self.flat_buffers.num_devices) |device_index| { + shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); + } + buffer.* = Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], shards.constSlice()); + context_.current_index += 1; + } + }.cb, &context, &v); + } + }; + + pub fn internalCall(self: *const Exe, io: ?std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { + stdx.debug.assert(opts.wait == false or io != null, "io should not be null when waiting for execution completion", .{}); var events = [_]?*pjrt.Event{null} ** Platform.MAX_NUM_DEVICES; const sharding = self.platform.sharding(); + const events_slice: ?[]?*pjrt.Event = if (opts.wait) events[0..sharding.num_partitions] else null; self.exe.execute(self.platform.pjrt_api, .{ - .arguments = self.input_per_device, - .num_args = self.input_buffer_count, - .results = self.output_per_device, - .events = events[0..sharding.num_partitions], + .arguments = arguments.flat_buffers.buffers, + .num_args = arguments.expected_shapes.len, + .results = results_.flat_buffers.buffers, + .events = events_slice, // this allows to tell a specific buffer shouldn't be donated, // even if it has been marked as "can be donated" during compilation. // TODO: expose it ? .non_donatable_input_indices = &.{}, - .context = self.execute_context, + .context = self.context, }) catch |err| { std.debug.panic("PJRT_LoadedExecutable_Execute failed with: {}", .{err}); }; - // for (events[0..sharding.num_partitions]) |e| { - // if (e) |ev| { - // ev.await(self.platform.pjrt_api) catch unreachable; - // } - // } - } - - pub fn _unsafeAssignResults(self: BaseExe, T: type, result: *T) void { - const LocalContext = struct { - index: u32, - platform: Platform, - outputs: []const [*]*pjrt.Buffer, - output_shapes: []Shape, - }; - var local_ctx: LocalContext = .{ - .index = 0, - .platform = self.platform, - .outputs = self.output_per_device, - .output_shapes = self.result_shapes, - }; - meta.visit((struct { - fn cb(ctx: *LocalContext, buffer: *Buffer) void { - const i = ctx.index; - ctx.index += 1; - if (i >= ctx.output_shapes.len) return; - - var shards: Buffer.Shards = .{}; - for (ctx.outputs) |buff| { - shards.appendAssumeCapacity(buff[i]); + if (opts.wait) { + for (events_slice.?) |e| { + if (e) |ev| { + ev.await(self.platform.pjrt_api, io.?) catch unreachable; } - buffer.* = Buffer.fromPjrtBuffers(ctx.platform, ctx.output_shapes[i], shards.constSlice()); } - }).cb, &local_ctx, result); - stdx.debug.internalAssert(local_ctx.index == self.result_shapes.len, "Pjrt call returned {} tensors, but the return type {s}, contains {} Buffers. Note that modules need to have a comptime know number of returned tensors.", .{ self.output_per_device.len, @typeName(T), local_ctx.index }); - } - - pub fn bind(exe: BaseExe, Callback: type, op: *Callback) !void { - stdx.debug.assert(exe.execute_context != null, "Exe doesn't have an execution context", .{}); - const pjrt_api = exe.platform.pjrt_api; - - if (pjrt_api.ffi()) |ffi| { - try callback.addUserData(Callback, pjrt_api, ffi, exe.execute_context.?, op); - } else { - stdx.debug.panic("Callbacks are not supported for target {s}", .{@tagName(exe.platform.target)}); } } - pub fn serialize(self: BaseExe, writer: *std.Io.Writer) !void { - var executable = try self.exe.getExecutable(self.platform.pjrt_api); - var serialize_result = try executable.serialize(self.platform.pjrt_api); - defer serialize_result.deinit(); - try writer.writeAll(serialize_result.bytes); + pub const CallOpts = struct { + wait: bool = false, + }; + + pub fn callOpts(self: *const Exe, io: std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { + return self.internalCall(io, arguments, results_, opts); } - // pub fn deserialize(allocator: std.mem.Allocator, platform: Platform, reader: anytype) !Self { - // const bytes = try reader.readToEndAlloc(allocator, max_pjrt_executable_size); - // defer allocator.free(bytes); - // return platform.pjrt_client.deserializeAndLoad(platform.pjrt_api, bytes); - // } - - pub fn prepare(self: *BaseExe, x: anytype) void { - const n = fillBuffers(&x, self.input_shapes, self.input_per_device, self.ready_buffer_count); - self.ready_buffer_count += n; - } - - pub fn getOutputBuffer(self: BaseExe, i: usize) Buffer { - var shards: Buffer.Shards = .{}; - for (self.output_per_device) |dev_out| { - shards.appendAssumeCapacity(dev_out[i]); - } - - return Buffer.fromPjrtBuffers(self.platform, self.result_shapes[i], shards.constSlice()); - } - - pub fn clone(self: BaseExe, parent_allocator: std.mem.Allocator) !BaseExe { - var exe: BaseExe = try .init(parent_allocator, self.platform, self.exe, .{ - .input_shapes = self.input_shapes, - .result_shapes = self.result_shapes, - .n_devices = self.num_devices, - }); - exe.execute_context = self.execute_context; - return exe; + pub fn call(self: *const Exe, arguments: Arguments, results_: *Results) void { + return self.internalCall(null, arguments, results_, .{}); } }; - -/// Represents a ZML function, compiled into a PJRT executable. -/// The signature of the Exe reflects the arguments that are needed for `call`. -pub fn Exe(ArgsT: type, ReturnT: type) type { - return struct { - const Self = @This(); - - /// The raw untyped compiled module. - inner: BaseExe, - - pub fn deinit(self: Self) void { - self.inner.deinit(); - } - - /// Hardcode the first argument of the function to the given buffers. - /// Returns an Exe with one less argument in `call`. - /// In functional languages this is known as partial application. - /// - /// **Warning:** the new Exe reuses the underlying memory of the previous one. - /// The caller is responsible to come up with a strategy to call `deinit` exactly once. - pub fn prepare(self: Self, first_arg: Bufferized(stdx.meta.Head(ArgsT))) Exe(stdx.meta.Tail(ArgsT), ReturnT) { - var new: Exe(stdx.meta.Tail(ArgsT), ReturnT) = .{ .inner = self.inner }; - new.inner.prepare(first_arg); - return new; - } - - /// For a given customCall inside this executable, - /// provide a pointer to runtime data. - /// The caller keeps memory ownership and need to ensure that the value - /// stays alive as long as the executable. - pub fn bind(self: Self, comptime T: type, value: *T) !void { - try self.inner.bind(T, value); - } - - pub fn serialize(self: Self, writer: *std.Io.Writer) !void { - return try self.inner.serialize(writer); - } - - pub fn platform(self: Self) Platform { - return self.inner.platform; - } - - pub fn call(self: Self, args: Bufferized(ArgsT)) Bufferized(ReturnT) { - const total_ready = fillBuffers(&args, self.inner.input_shapes, self.inner.input_per_device, self.inner.ready_buffer_count); - std.debug.assert(total_ready == self.inner.input_buffer_count); - self.inner._unsafeCall(); - var result: Bufferized(ReturnT) = undefined; - self.inner._unsafeAssignResults(Bufferized(ReturnT), &result); - return result; - } - - pub fn clone(self: Self, allocator: std.mem.Allocator) !Self { - return .{ .inner = try self.inner.clone(allocator) }; - } - }; -} - -fn splitBuffer(T: type, buffer: []T, lengths: anytype) [lengths.len][]T { - var res: [lengths.len][]T = undefined; - var i: usize = 0; - inline for (&res, lengths) |*r, len| { - r.* = buffer[i .. i + len]; - i += len; - } - std.debug.assert(i == buffer.len); - return res; -} - -/// Visit the given struct and fill the `buffers` slice with the buffer associated with encountered Tensor. -fn fillBuffers(v: anytype, shapes: []const Shape, buffers: []const [*]*pjrt.Buffer, start: u32) u32 { - const LocalContext = struct { - index: u32, - buffers: []const [*]*pjrt.Buffer, - shapes: []const Shape, - }; - var context: LocalContext = .{ - .index = start, - .buffers = buffers, - .shapes = shapes, - }; - meta.visit((struct { - fn cb(ctx: *LocalContext, buffer: *const Buffer) void { - // stdx.debug.assert(!buffer._data.isDeleted(), "Can't use {} (argument buffer {}) because its pjrt buffer has been donated", .{ buffer, ctx.index }); - const model_sharding = ctx.buffers.len; - stdx.debug.assert(buffer._shards.len == model_sharding, "Can't feed a {d}-sharded tensor into a {d}-sharded model", .{ buffer._shards.len, ctx.buffers.len }); - stdx.debug.assert(ctx.shapes[ctx.index].eql(buffer.shape()), "Executable expected argument {} to have shape {f}, got {f}", .{ ctx.index, ctx.shapes[ctx.index], buffer.shape() }); - for (buffer._shards.constSlice(), 0..) |shard, d| { - ctx.buffers[d][ctx.index] = shard; - } - ctx.index += 1; - } - }).cb, &context, v); - return context.index; -} - -fn prettyFnName( - comptime func: anytype, - allocator: std.mem.Allocator, -) !std.ArrayList(u8) { - const full_noisy_name = @typeName(@TypeOf(func)); - const og_len = full_noisy_name.len; - const buffer = try allocator.alloc(u8, og_len); - errdefer comptime unreachable; // No errors below this point. - var out: []u8 = buffer; - - { - const verbose = "tensor.Tensor"; - const compact = "Tensor"; - const num_replacements = std.mem.replace(u8, full_noisy_name, verbose, compact, buffer); - out.len = out.len + num_replacements * compact.len - num_replacements * verbose.len; - } - - { - const verbose = "tensor.Tensor."; - const compact = ""; - const num_replacements = std.mem.replace(u8, out, verbose, compact, buffer); - out.len = out.len + num_replacements * compact.len - num_replacements * verbose.len; - } - - { - const verbose = "shape.Shape"; - const compact = "Shape"; - const num_replacements = std.mem.replace(u8, out, verbose, compact, buffer); - out.len = out.len + num_replacements * compact.len - num_replacements * verbose.len; - } - - return .{ .items = out, .capacity = og_len }; -} diff --git a/zml/floats.zig b/zml/floats.zig index 54240485..538722cd 100644 --- a/zml/floats.zig +++ b/zml/floats.zig @@ -1,11 +1,7 @@ ///! Conversion utilities between different Floating point formats. const std = @import("std"); -test { - std.testing.refAllDecls(@This()); -} - -fn FloatHelpers(Float: type) type { +fn FloatConversionHelpers(Float: type) type { const info = @typeInfo(Float); const err_msg = "FloatHelpers expect a packed struct { mantissa: uXX, exponent: uXX, sign: u1}"; if (info != .@"struct" or info.@"struct".backing_integer == null) { @@ -18,22 +14,14 @@ fn FloatHelpers(Float: type) type { } } + const zero = FloatHelpers(Float).zero; + return struct { const mantissa_bits: u8 = @typeInfo(@FieldType(Float, "mantissa")).int.bits; const exponent_bits: u8 = @typeInfo(@FieldType(Float, "exponent")).int.bits; const f32_mantissa_bits: u8 = @typeInfo(@FieldType(Float32, "mantissa")).int.bits; const exp_bias: i16 = std.math.maxInt(std.meta.Int(.unsigned, exponent_bits - 1)); - const exp_off: u8 = FloatHelpers(Float32).exp_bias - exp_bias; - - pub const zero: Float = .{ .sign = 0, .exponent = 0, .mantissa = 0 }; - - pub fn neg(x: Float) Float { - return .{ - .sign = x.sign ^ 1, - .exponent = x.exponent, - .mantissa = x.mantissa, - }; - } + const exp_off: u8 = FloatConversionHelpers(Float32).exp_bias - exp_bias; /// Lossy conversion from f32, similar to @floatCast pub fn fromF32(f: f32) Float { @@ -46,8 +34,11 @@ fn FloatHelpers(Float: type) type { @branchHint(.unlikely); return if (@hasDecl(Float, "inf")) if (vf32.sign == 0) Float.inf else Float.minus_inf - else - Float.nan; + else if (@hasDecl(Float, "nan")) + Float.nan + else b: { + break :b if (vf32.sign == 0) Float.max else Float.min; + }; } return if (exponent <= 0) @@ -106,8 +97,34 @@ fn FloatHelpers(Float: type) type { const T = @FieldType(Float32, "mantissa"); return @as(T, x.mantissa) << f32_mantissa_bits - mantissa_bits; } + }; +} - pub fn formatNumber(x: Float, writer: *std.io.Writer, n: std.fmt.Number) std.io.Writer.Error!void { +fn FloatHelpers(Float: type) type { + const info = @typeInfo(Float); + const err_msg = "FloatHelpers expect a packed struct { mantissa: uXX, exponent: uXX, sign: u1}"; + if (info != .@"struct" or info.@"struct".backing_integer == null) { + @compileError(err_msg); + } + comptime { + for (info.@"struct".fields, &.{ "mantissa", "exponent", "sign" }) |field, expected_name| { + if (!std.mem.eql(u8, field.name, expected_name)) + @compileError(err_msg); + } + } + + return struct { + pub const zero: Float = .{ .sign = 0, .exponent = 0, .mantissa = 0 }; + + pub fn neg(x: Float) Float { + return .{ + .sign = x.sign ^ 1, + .exponent = x.exponent, + .mantissa = x.mantissa, + }; + } + + pub fn formatNumber(x: Float, writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { switch (n.mode) { .binary, .octal, .hex => try writer.print("{{ .sign={}, .exp={}, .mantissa={} }}", .{ x.sign, x.exponent, x.mantissa }), else => try writer.printFloat(x.toF32(), n), @@ -121,14 +138,28 @@ pub const Float32 = packed struct(u32) { exponent: u8, sign: u1, + pub const nan: Float32 = .{ .sign = 0, .exponent = std.math.maxInt(u8), .mantissa = 1 }; + pub const max: Float32 = .{ .sign = 0, .exponent = std.math.maxInt(u8) - 1, .mantissa = std.math.maxInt(u23) }; + pub const min: Float32 = neg(max); pub const inf: Float32 = .{ .sign = 0, .exponent = std.math.maxInt(u8), .mantissa = 0 }; pub const minus_inf = neg(inf); + pub fn isNan(self: Float32) bool { + return self.exponent == nan.exponent and self.mantissa != 0; + } + const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; pub const neg = Helpers.neg; - pub const fromF32 = Helpers.fromF32; - pub const toF32 = Helpers.toF32; + + pub fn fromF32(f: f32) Float32 { + return @bitCast(f); + } + + pub fn toF32(x: Float32) f32 { + return @bitCast(x); + } + pub const formatNumber = Helpers.formatNumber; }; @@ -139,11 +170,27 @@ pub const Float64 = packed struct(u64) { exponent: u11, sign: u1, + pub const nan: Float64 = .{ .sign = 0, .exponent = std.math.maxInt(u11), .mantissa = 1 }; + pub const max: Float64 = .{ .sign = 0, .exponent = std.math.maxInt(u11) - 1, .mantissa = std.math.maxInt(u32) }; + pub const min: Float64 = neg(max); + pub const inf: Float64 = .{ .sign = 0, .exponent = std.math.maxInt(u11), .mantissa = 0 }; + pub const minus_inf = neg(inf); + + pub fn isNan(self: Float64) bool { + return self.exponent == nan.exponent and self.mantissa != 0; + } + const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; pub const neg = Helpers.neg; - pub const fromF32 = Helpers.fromF32; - pub const toF32 = Helpers.toF32; + pub fn fromF32(f: f32) Float64 { + return @bitCast(@as(f64, @floatCast(f))); + } + + pub fn toF32(x: Float64) f32 { + return @floatCast(@as(f64, @bitCast(x))); + } + pub const formatNumber = Helpers.formatNumber; }; @@ -153,16 +200,18 @@ pub const Float8E4M3B11FNUZ = packed struct(u8) { sign: u1, pub const nan: Float8E4M3B11FNUZ = .{ .sign = 1, .exponent = 0, .mantissa = 0 }; + pub const max: Float8E4M3B11FNUZ = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = std.math.maxInt(u3) }; + pub const min: Float8E4M3B11FNUZ = neg(max); pub fn isNan(self: Float8E4M3B11FNUZ) bool { - return self.sign == 1 and self.exponent == 0 and self.mantissa == 0; + return self == nan; } const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; pub const neg = Helpers.neg; - pub const fromF32 = Helpers.fromF32; - pub const toF32 = Helpers.toF32; + pub const fromF32 = FloatConversionHelpers(@This()).fromF32; + pub const toF32 = FloatConversionHelpers(@This()).toF32; pub const formatNumber = Helpers.formatNumber; }; @@ -172,15 +221,18 @@ pub const Float8E4M3FN = packed struct(u8) { sign: u1, pub const nan: Float8E4M3FN = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = std.math.maxInt(u3) }; + pub const max: Float8E4M3FN = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = 0b110 }; + pub const min: Float8E4M3FN = neg(max); pub fn isNan(self: Float8E4M3FN) bool { return self.exponent == nan.exponent and self.mantissa == nan.mantissa; } + const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; pub const neg = Helpers.neg; - pub const fromF32 = Helpers.fromF32; - pub const toF32 = Helpers.toF32; + pub const fromF32 = FloatConversionHelpers(@This()).fromF32; + pub const toF32 = FloatConversionHelpers(@This()).toF32; pub const formatNumber = Helpers.formatNumber; }; @@ -190,16 +242,18 @@ pub const Float8E4M3FNUZ = packed struct(u8) { sign: u1, pub const nan: Float8E4M3FNUZ = .{ .sign = 1, .exponent = 0, .mantissa = 0 }; + pub const max: Float8E4M3FNUZ = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = std.math.maxInt(u3) }; + pub const min: Float8E4M3FNUZ = neg(max); pub fn isNan(self: Float8E4M3FNUZ) bool { - return self.sign == 1 and self.exponent == 0 and self.mantissa == 0; + return self == nan; } const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; pub const neg = Helpers.neg; - pub const fromF32 = Helpers.fromF32; - pub const toF32 = Helpers.toF32; + pub const fromF32 = FloatConversionHelpers(@This()).fromF32; + pub const toF32 = FloatConversionHelpers(@This()).toF32; pub const formatNumber = Helpers.formatNumber; }; @@ -229,25 +283,21 @@ pub const Float8E5M2 = packed struct(u8) { exponent: u5, sign: u1, - pub const nan: Float8E5M2 = .{ .sign = 0, .exponent = std.math.maxInt(u5), .mantissa = 1 }; + pub const nan: Float8E5M2 = .{ .sign = 0, .exponent = std.math.maxInt(u5), .mantissa = 0b01 }; + pub const max: Float8E5M2 = .{ .sign = 0, .exponent = 0b11110, .mantissa = std.math.maxInt(u2) }; + pub const min: Float8E5M2 = neg(max); + pub const inf: Float8E5M2 = .{ .sign = 0, .exponent = std.math.maxInt(u5), .mantissa = 0 }; + pub const minus_inf: Float8E5M2 = .neg(inf); pub fn isNan(self: Float8E5M2) bool { return self.exponent == nan.exponent and self.mantissa != 0; } - pub const inf: Float8E5M2 = .{ - .sign = 0, - .exponent = std.math.maxInt(u5), - .mantissa = 0, - }; - - pub const minus_inf: Float8E5M2 = .neg(inf); - const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; pub const neg = Helpers.neg; - pub const fromF32 = Helpers.fromF32; - pub const toF32 = Helpers.toF32; + pub const fromF32 = FloatConversionHelpers(@This()).fromF32; + pub const toF32 = FloatConversionHelpers(@This()).toF32; pub const formatNumber = Helpers.formatNumber; }; @@ -257,16 +307,18 @@ pub const Float8E5M2FNUZ = packed struct(u8) { sign: u1, pub const nan: Float8E5M2FNUZ = .{ .sign = 1, .exponent = 0, .mantissa = 0 }; + pub const max: Float8E5M2FNUZ = .{ .sign = 0, .exponent = std.math.maxInt(u5), .mantissa = std.math.maxInt(u2) }; + pub const min: Float8E5M2FNUZ = neg(max); pub fn isNan(self: Float8E5M2FNUZ) bool { - return self.sign == 1 and self.exponent == 0 and self.mantissa == 0; + return self == nan; } const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; pub const neg = Helpers.neg; - pub const fromF32 = Helpers.fromF32; - pub const toF32 = Helpers.toF32; + pub const fromF32 = FloatConversionHelpers(@This()).fromF32; + pub const toF32 = FloatConversionHelpers(@This()).toF32; pub const formatNumber = Helpers.formatNumber; }; @@ -286,19 +338,15 @@ pub const BFloat16 = packed struct(u16) { sign: u1, pub const nan: BFloat16 = .{ .sign = 0, .exponent = std.math.maxInt(u8), .mantissa = 1 }; + pub const max: BFloat16 = .{ .sign = 0, .exponent = 0b11111110, .mantissa = std.math.maxInt(u7) }; + pub const min: BFloat16 = neg(max); + pub const inf: BFloat16 = .{ .sign = 0, .exponent = std.math.maxInt(u8), .mantissa = 0 }; + pub const minus_inf: BFloat16 = .neg(inf); pub fn isNan(self: BFloat16) bool { - return allBitsOne(self.exponent) and self.mantissa != 0; + return self.exponent == nan.exponent and self.mantissa != 0; } - pub const inf: BFloat16 = .{ - .sign = 0, - .exponent = std.math.maxInt(u8), - .mantissa = 0, - }; - - pub const minus_inf: BFloat16 = .neg(inf); - // Specialized versions of to/from F32. Since BFloat16 has the same exponent range than F32, // no overflow/underflow can happen, simplifiying conversion logic. pub fn toF32(self: BFloat16) f32 { @@ -340,25 +388,21 @@ pub const Float8E4M3 = packed struct(u8) { exponent: u4, sign: u1, - pub const nan: Float8E4M3 = @bitCast(0xFF); + pub const nan: Float8E4M3 = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = 1 }; + pub const max: Float8E4M3 = .{ .sign = 0, .exponent = 0b1110, .mantissa = std.math.maxInt(u3) }; + pub const min: Float8E4M3 = neg(max); + pub const inf: Float8E4M3 = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = 0 }; + pub const minus_inf = neg(inf); pub fn isNan(self: Float8E4M3) bool { - return self == nan or self == comptime nan.neg(); + return self.exponent == nan.exponent and self.mantissa != 0; } - pub const inf: Float8E4M3 = .{ - .sign = 0, - .exponent = std.math.maxInt(u4), - .mantissa = 0, - }; - - pub const minus_inf = neg(inf); - const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; pub const neg = Helpers.neg; - pub const fromF32 = Helpers.fromF32; - pub const toF32 = Helpers.toF32; + pub const fromF32 = FloatConversionHelpers(@This()).fromF32; + pub const toF32 = FloatConversionHelpers(@This()).toF32; pub const formatNumber = Helpers.formatNumber; }; @@ -367,25 +411,21 @@ pub const Float8E3M4 = packed struct(u8) { exponent: u3, sign: u1, - pub const nan: Float8E3M4 = @bitCast(0xFF); + pub const nan: Float8E3M4 = .{ .sign = 0, .exponent = std.math.maxInt(u3), .mantissa = 1 }; + pub const max: Float8E3M4 = .{ .sign = 0, .exponent = 0b110, .mantissa = std.math.maxInt(u4) }; + pub const min: Float8E3M4 = neg(max); + pub const inf: Float8E3M4 = .{ .sign = 0, .exponent = std.math.maxInt(u3), .mantissa = 0 }; + pub const minus_inf = neg(inf); pub fn isNan(self: Float8E3M4) bool { - return self == nan or self == comptime nan.neg(); + return self.exponent == nan.exponent and self.mantissa != 0; } - pub const inf: Float8E3M4 = .{ - .sign = 0, - .exponent = std.math.maxInt(u3), - .mantissa = 0, - }; - - pub const minus_inf = neg(inf); - const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; pub const neg = Helpers.neg; - pub const fromF32 = Helpers.fromF32; - pub const toF32 = Helpers.toF32; + pub const fromF32 = FloatConversionHelpers(@This()).fromF32; + pub const toF32 = FloatConversionHelpers(@This()).toF32; pub const formatNumber = Helpers.formatNumber; }; @@ -394,6 +434,8 @@ pub const Float8E8M0 = packed struct(u8) { exponent: u8, sign: u0 = 0, + pub const max: Float8E8M0 = .{ .exponent = std.math.maxInt(u8) }; + pub const min: Float8E8M0 = .{ .exponent = 0 }; pub const min_scale: f32 = @bitCast(Float32{ .sign = 0, .exponent = 0, .mantissa = 0b1 << 22 }); /// Lossy conversion from f32, similar to @floatCast @@ -432,11 +474,12 @@ pub const Float4E2M1 = packed struct(u4) { exponent: u2, sign: u1, - pub const nan: Float4E2M1 = @bitCast(@as(u4, 0xF)); + pub const max: Float4E2M1 = .{ .sign = 0, .exponent = std.math.maxInt(u2), .mantissa = std.math.maxInt(u1) }; + pub const min: Float4E2M1 = neg(max); const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; pub const neg = Helpers.neg; - pub const fromF32 = Helpers.fromF32; + pub const fromF32 = FloatConversionHelpers(@This()).fromF32; pub const formatNumber = Helpers.formatNumber; pub const values = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; @@ -490,10 +533,6 @@ pub fn isInf(x: anytype) bool { return @as(FBits, @bitCast(x)) & remove_sign == @as(FBits, @bitCast(Float.inf)); } -fn allBitsOne(v: anytype) bool { - return v == std.math.maxInt(@TypeOf(v)); -} - const TestCase = struct { lossless: []const f32, lossy: []const f32, diff --git a/zml/helpers.zig b/zml/helpers.zig deleted file mode 100644 index 85c93517..00000000 --- a/zml/helpers.zig +++ /dev/null @@ -1,141 +0,0 @@ -//! Public helpers to manipulate tensors or shapes. -const std = @import("std"); - -const meta = @import("meta.zig"); -const Shape = @import("shape.zig").Shape; - -const log = std.log.scoped(.@"zml/tensor"); - -test { - std.testing.refAllDecls(@This()); -} - -const ShapeError = error{ DimMismatch, NotFound }; -const NOT_SET: i64 = -2; -const DIM_MISMATCH: i64 = -1; - -/// Collect the given dimensions inside a struct containing tagged tensors. -pub fn collectDims( - comptime dims: anytype, - v: anytype, - comptime mode: enum { strict, allow_extra_dims, ignore_errors }, -) ShapeError!ShapeStruct(dims) { - const LocalContext = struct { - res: ShapeStruct(dims), - mode: @TypeOf(mode), - }; - - var context = LocalContext{ - .res = undefined, - .mode = mode, - }; - @memset(std.mem.bytesAsSlice(i64, std.mem.asBytes(&context.res)), NOT_SET); - - meta.visit((struct { - fn cb(ctx: *LocalContext, shape: *const Shape) void { - inline for (dims) |a| { - if (shape.hasTag(a)) |axis| { - const dim = shape.dim(axis); - - const expected_dim = &@field(ctx.res, @tagName(a)); - if (expected_dim.* == NOT_SET) { - expected_dim.* = dim; - } else if (expected_dim.* == DIM_MISMATCH) { - // this axis has already been reported as invalid. - } else if (dim != expected_dim.*) { - if (mode != .ignore_errors) { - log.warn("Dim mismatch ! Axis {0s}={1d} but received a new tensor where {0s}={2d}", .{ @tagName(a), expected_dim.*, dim }); - } - expected_dim.* = DIM_MISMATCH; - } - } - } - } - }).cb, &context, v); - - if (context.mode != .ignore_errors) { - inline for (shapeToDims(context.res), dims) |dim, dim_tag| { - if (dim == NOT_SET) { - log.warn("Axis not found: {s}", .{@tagName(dim_tag)}); - return error.NotFound; - } - if (dim == DIM_MISMATCH) return error.DimMismatch; - } - } - return context.res; -} - -fn shapeToDims(shape: anytype) [@divExact(@sizeOf(@TypeOf(shape)), @sizeOf(i64))]i64 { - return @bitCast(shape); -} - -test collectDims { - const zml = @import("zml.zig"); - - const Model = struct { - x: Shape, - y: Shape, - bias: Shape, - }; - - { - var model: Model = .{ - .x = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), - .y = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), - .bias = Shape.init(.{5}, .f32).withTags(.{.d}), - }; - try zml.testing.expectEqual(collectDims(.{ .b, .d }, &model, .strict), .{ .b = 2, .d = 5 }); - } - { - var model: Model = .{ - .x = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), - .y = Shape.init(.{ 3, 5 }, .f32).withTags(.{ .b, .d }), - .bias = Shape.init(.{5}, .f32).withTags(.{.d}), - }; - try std.testing.expectEqual( - collectDims(.{ .b, .d }, &model, .strict), - error.DimMismatch, - ); - try zml.testing.expectEqual(collectDims(.{ .b, .d }, &model, .ignore_errors), .{ .b = DIM_MISMATCH, .d = 5 }); - } - { - var model: Model = .{ - .x = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), - .y = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), - .bias = Shape.init(.{5}, .f32).withTags(.{.d}), - }; - try std.testing.expectEqual(collectDims(.{ .b, .d, .c }, &model, .strict), error.NotFound); - try zml.testing.expectEqual(collectDims(.{ .b, .d, .c }, &model, .ignore_errors), .{ .b = 2, .d = 5, .c = NOT_SET }); - } - { - var model: Model = .{ - .x = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), - .y = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), - .bias = Shape.init(.{7}, .f32).withTags(.{.d}), - }; - try std.testing.expectEqual(collectDims(.{ .b, .d }, &model, .strict), error.DimMismatch); - try zml.testing.expectEqual(collectDims(.{ .b, .d }, &model, .ignore_errors), .{ .b = 2, .d = DIM_MISMATCH }); - } -} - -fn ShapeStruct(comptime dims: anytype) type { - const rank = dims.len; - @setEvalBranchQuota(rank + 5); - var struct_fields: [rank]std.builtin.Type.StructField = undefined; - const default: i64 = NOT_SET; - for (&struct_fields, dims) |*struct_field, axis| { - struct_field.* = .{ - .name = @tagName(axis), - .type = i64, - .default_value_ptr = &default, - .is_comptime = false, - .alignment = @alignOf(i64), - }; - } - return @Type(.{ .@"struct" = .{ - .layout = .@"extern", - .fields = &struct_fields, - .decls = &.{}, - .is_tuple = false, - } }); -} diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig deleted file mode 100644 index 0469f1ec..00000000 --- a/zml/hostbuffer.zig +++ /dev/null @@ -1,409 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -const Buffer = @import("buffer.zig").Buffer; -const DataType = @import("dtype.zig").DataType; -const floats = @import("floats.zig"); -const Platform = @import("platform.zig").Platform; -const Shape = @import("shape.zig").Shape; - -test { - std.testing.refAllDecls(HostBuffer); -} - -/// Represents a tensor with associated data allocated by user code. -/// If the memory is `.managed` it needs to be freed with `x.deinit(allocator)` -/// If the memory is `.unmanaged` it doesn't need to be freed (eg memory mapped, or tracked elsewhere). -pub const HostBuffer = struct { - _shape: Shape, - _strides: [Shape.MAX_RANK]i64, - _data: [*]const u8, - _memory: union(enum) { - managed: std.mem.Alignment, - unmanaged, - } = .unmanaged, - - /// Allocates a HostBuffer with the given shape. - /// The memory is left undefined. - /// The caller owns the memory, and need to call `deinit()`. - pub fn empty(allocator: std.mem.Allocator, sh: Shape) error{OutOfMemory}!HostBuffer { - return .{ - ._shape = sh, - ._strides = sh.computeStrides().buffer, - ._data = (try allocator.alignedAlloc(u8, .@"64", sh.byteSize())).ptr, - ._memory = .{ .managed = .@"64" }, - }; - } - - /// Wraps an exisiting slice of bytes into a HostBuffer. - /// The returned HostBuffer doesn't take ownership of the slice - /// that will still need to be deallocated. - pub fn fromBytes(shape_: Shape, data_: []const u8) HostBuffer { - stdx.debug.assert(shape_.byteSize() == data_.len, "shape {} and data {} don't match", .{ shape_.byteSize(), data_.len }); - return .{ - ._shape = shape_, - ._strides = shape_.computeStrides().buffer, - ._data = data_.ptr, - ._memory = .unmanaged, - }; - } - - /// Frees the underlying memory if we owned it, ie if we've been created with `HostBuffer.empty`. - pub fn deinit(self: *const HostBuffer, allocator: std.mem.Allocator) void { - // This means we don't own the data. - if (self._memory == .unmanaged) return; - const log2_align = self._memory.managed; - allocator.rawFree(self.mutBytes(), log2_align, @returnAddress()); - } - - /// Wraps an exisiting slice into a HostBuffer. - /// The element type is inferred from the slice type. - /// The returned HostBuffer doesn't take ownership of the slice - /// that will still need to be deallocated. - pub fn fromSlice(sh: anytype, s: anytype) HostBuffer { - const shape_ = Shape.init(sh, DataType.fromSliceElementType(s)); - const raw_bytes = std.mem.sliceAsBytes(s); - std.debug.assert(shape_.byteSize() == raw_bytes.len); - return .{ - ._shape = shape_, - ._strides = shape_.computeStrides().buffer, - ._data = raw_bytes.ptr, - ._memory = .unmanaged, - }; - } - - /// Wraps an exisiting slice into a HostBuffer. - /// The element type is inferred from the slice type. - /// The values in the slice doesn't need to be contiguous, - /// strides can be specified. - /// The returned HostBuffer doesn't take ownership of the slice. - pub fn fromStridedSlice(sh: Shape, s: anytype, strides_: []const i64) HostBuffer { - // std.debug.assert(sh.count() == s.len); - var tmp: [Shape.MAX_RANK]i64 = undefined; - @memcpy(tmp[0..strides_.len], strides_); - return .{ - ._shape = sh, - ._data = @alignCast(std.mem.sliceAsBytes(s).ptr), - ._strides = tmp, - ._memory = .unmanaged, - }; - } - - /// Creates a tensor from a **pointer** to a "multi dimension" array. - /// Note this doesn't copy, the pointee array need to survive the `HostBuffer` object. - /// Typically this is use with constant arrays. - pub fn fromArray(arr_ptr: anytype) HostBuffer { - const T = @TypeOf(arr_ptr.*); - const sh = parseArrayInfo(T); - std.debug.assert(sh.byteSize() == @sizeOf(T)); - return .{ - ._shape = sh, - ._strides = sh.computeStrides().buffer, - ._data = @ptrCast(arr_ptr), - ._memory = .unmanaged, - }; - } - - /// Returns a HostBuffer tagged with the tags in 'tagz'. - pub fn withTags(self: HostBuffer, tagz: anytype) HostBuffer { - var res = self; - res._shape = self._shape.withTags(tagz); - return res; - } - - pub const ArangeArgs = struct { - start: i64 = 0, - end: i64, - step: i64 = 1, - }; - - /// Allocates a HostBuffer with the given shape. - /// The memory is initialized with increasing numbers. - /// The caller owns the memory, and need to call `deinit()`. - pub fn arange(allocator: std.mem.Allocator, args: ArangeArgs, dt: DataType) !HostBuffer { - stdx.debug.assert(args.start < args.end, "arange expects 'args.start' to be less than 'args.end', got {} and {}", .{ args.start, args.end }); - stdx.debug.assert(args.step > 0, "arange expects 'args.step' to be positive, got {}", .{args.step}); - - const n_steps = std.math.divCeil(i64, args.end - args.start, args.step) catch unreachable; - const res = try empty(allocator, Shape.init(.{n_steps}, dt)); - switch (dt) { - inline else => |d| if (comptime d.class() != .integer) { - stdx.debug.assert(dt.class() == .integer, "arange expects type to be integer, got {} instead.", .{dt}); - } else { - const Zt = d.toZigType(); - var j: i64 = args.start; - for (res.mutItems(Zt)) |*val| { - val.* = @intCast(j); - j +%= args.step; - } - }, - } - return res; - } - - test arange { - { - var x = try arange(std.testing.allocator, .{ .end = 8 }, .i32); - defer x.deinit(std.testing.allocator); - try std.testing.expectEqualSlices(i32, &.{ 0, 1, 2, 3, 4, 5, 6, 7 }, x.items(i32)); - } - { - var x = try arange(std.testing.allocator, .{ .start = -3, .end = 12, .step = 2 }, .i32); - defer x.deinit(std.testing.allocator); - try std.testing.expectEqualSlices(i32, &.{ -3, -1, 1, 3, 5, 7, 9, 11 }, x.items(i32)); - } - } - - /// Copies this HostBuffer to the given accelerator. - pub fn toDevice(self: HostBuffer, platform_: Platform) !Buffer { - return try self.toDeviceOpts(platform_, .{}); - } - - /// Copies this HostBuffer to the given accelerator (with options). - pub fn toDeviceOpts(self: HostBuffer, platform_: Platform, opts: Buffer.FromOptions) !Buffer { - return try Buffer.from(platform_, self, opts); - } - - /// Interpret the underlying data as a contiguous slice. - /// WARNING: It's only valid if the buffer is contiguous. - /// Strided buffers can't use this method. - pub fn items(self: HostBuffer, comptime T: type) []const T { - // TODO we should allow interpreting the output as @Vector(8, f32) when the tensor is f32. - stdx.debug.assert(DataType.fromZigType(T) == self.dtype(), "Can't reinterpret {f} as {s}", .{ self, @typeName(T) }); - stdx.debug.assert(self.isContiguous(), "{f} isn't contiguous, can't interpret as []const u8", .{self}); - const ptr: [*]const T = @ptrCast(@alignCast(self._data)); - return ptr[0..self._shape.count()]; - } - - pub fn mutItems(self: HostBuffer, comptime T: type) []T { - return @constCast(self.items(T)); - } - - pub fn bytes(self: HostBuffer) []const u8 { - stdx.debug.assert(self.isContiguous(), "{f} isn't contiguous, can't interpret as []const u8", .{self}); - return self._data[0..self._shape.byteSize()]; - } - - pub fn mutBytes(self: HostBuffer) []u8 { - return @constCast(self.bytes()); - } - - pub fn shape(self: HostBuffer) Shape { - return self._shape; - } - - pub fn dtype(self: HostBuffer) DataType { - return self._shape.dtype(); - } - - pub fn strides(self: *const HostBuffer) []const i64 { - // Pass strides per pointer otherwise we return a pointer to this stack frame. - return self._strides[0..self._shape.rank()]; - } - - pub inline fn rank(self: HostBuffer) u4 { - return self._shape.rank(); - } - - pub inline fn count(self: HostBuffer) usize { - return self._shape.count(); - } - - pub fn dim(self: HostBuffer, axis_: anytype) i64 { - return self._shape.dim(axis_); - } - - pub fn axis(self: HostBuffer, axis_: anytype) u3 { - return self._shape.axis(axis_); - } - - pub fn isContiguous(self: HostBuffer) bool { - const _strides = self._strides; - const cont_strides = self._shape.computeStrides(); - for (self._shape.dims(), _strides[0..self.rank()], cont_strides.constSlice()) |d, stride, cont_stride| { - if (d != 1 and stride != cont_stride) return false; - } - return true; - } - - pub fn reshape(self: HostBuffer, shape_: anytype) HostBuffer { - stdx.debug.assert(self.isContiguous(), "reshape expects a contiguous tensor, got: {f}", .{self}); - var res = self; - res._shape = self._shape.reshape(shape_); - res._strides = res._shape.computeStrides().buffer; - return res; - } - - pub const Slice = struct { - start: i64 = 0, - end: ?i64 = null, - }; - - /// Slices the input Tensor over the given axis_ using the given parameters. - pub fn slice1d(self: HostBuffer, axis_: anytype, s: Slice) HostBuffer { - const ax = self._shape.axis(axis_); - const d = self.dim(ax); - const start: i64 = if (s.start < 0) s.start + d else s.start; - var end = s.end orelse d; - if (end < 0) end += d; - stdx.debug.assert(start >= 0 and start < d, "slice1d({f}, {}) expects the slice start to be between 0 and {} got: {}", .{ self, ax, d, s }); - stdx.debug.assert(end >= 1 and end <= d, "slice1d({f}, {}) expects the slice end to be between 1 and {} got: {}", .{ self, ax, d, s }); - stdx.debug.assert(start < end, "slice1d({f}, {}) expects the slice start ({}) to be smaller than the end ({}), got: {}", .{ self, ax, start, end, s }); - - const offset: usize = @intCast(start * self._strides[ax]); - const new_shape = self.shape().set(ax, end - start); - return .{ - ._shape = new_shape, - ._data = self._data[offset..], - ._strides = self._strides, - ._memory = .unmanaged, - }; - } - - pub fn choose1d(self: HostBuffer, axis_: anytype, start: i64) HostBuffer { - const ax = self.axis(axis_); - return self.slice1d(ax, .{ .start = start, .end = start + 1 }).squeeze(ax); - } - - pub fn choose(self: HostBuffer, offsets: anytype) HostBuffer { - const off, const tags = Shape.parseDimensions(offsets); - var sh = self._shape; - var offset: i64 = 0; - for (off.constSlice(), tags.constSlice()) |o, t| { - const ax = sh.axis(t); - offset += o * self._strides[ax]; - sh._dims.buffer[ax] = 0; - } - - var new_strides: [Shape.MAX_RANK]i64 = @splat(self.dtype().sizeOf()); - - // TODO rewrite with simd. This is a pshuf, but it's not supported by @shuffle. - var res_ax: u32 = 0; - for (0..self._shape.rank()) |ax| { - if (sh._dims.buffer[ax] == 0) { - continue; - } - - sh._dims.buffer[res_ax] = self._shape._dims.buffer[ax]; - sh._tags.buffer[res_ax] = self._shape._tags.buffer[ax]; - new_strides[res_ax] = self._strides[ax]; - res_ax += 1; - } - sh._dims.len -= off.len; - sh._tags.len -= off.len; - - return HostBuffer{ - ._shape = sh, - ._strides = new_strides, - ._data = self._data[@intCast(offset)..], - ._memory = .unmanaged, - }; - } - - pub fn squeeze(self: HostBuffer, axis_: anytype) HostBuffer { - const ax = self._shape.axis(axis_); - stdx.debug.assert(self.dim(ax) == 1, "squeeze expects a 1-d axis got {} in {f}", .{ ax, self }); - - var strd: stdx.BoundedArray(i64, Shape.MAX_RANK) = .{ .buffer = self._strides, .len = self.rank() }; - _ = strd.orderedRemove(ax); - - return .{ - ._shape = self.shape().drop(ax), - ._data = self._data, - ._strides = strd.buffer, - ._memory = self._memory, - }; - } - - pub fn format(self: HostBuffer, writer: *std.Io.Writer) !void { - try writer.print("HostBuffer(.{f})@{x}", .{ self._shape, @intFromPtr(self._data) }); - } - - pub fn formatNumber(self: HostBuffer, writer: *std.io.Writer, n: std.fmt.Number) std.io.Writer.Error!void { - return self.prettyPrintIndented(writer, 4, 0, n); - } - - pub fn prettyPrint(self: HostBuffer, writer: *std.Io.Writer, options: std.fmt.Number) !void { - return self.prettyPrintIndented(writer, 4, 0, options); - } - - fn prettyPrintIndented(self: HostBuffer, writer: *std.Io.Writer, num_rows: u8, indent_level: u8, options: std.fmt.Number) !void { - if (self.rank() == 0) { - // Special case input tensor is a scalar - return switch (self.dtype()) { - inline else => |dt| { - const val: dt.toZigType() = self.items(dt.toZigType())[0]; - return switch (comptime dt.class()) { - // Since we have custom floats, we need to explicitly convert to float32 ourselves. - .float => stdx.fmt.formatFloat(floats.floatCast(f32, val), options, writer), - .integer => stdx.fmt.formatInt(val, options, writer), - .bool => stdx.fmt.formatBool(val, options, writer), - .complex => stdx.fmt.formatComplex(val, options, writer), - }; - }, - }; - } - if (self.rank() == 1) { - // Print a contiguous slice of items from the buffer in one line. - // The number of items printed is controlled by the user through format syntax. - try writer.splatByteAll(' ', indent_level); - switch (self.dtype()) { - inline else => |dt| { - const T = dt.toZigType(); - const n: i64 = self._shape.dim(0); - // TODO: handle negative strides - const byte_stride: i64 = self._strides[0]; - const elem_strides: i64 = @divExact(byte_stride, @sizeOf(T)); - const values: []const T = @ptrCast(@alignCast(self._data[0..@intCast(n * byte_stride)])); - switch (comptime dt.class()) { - .float => try stdx.fmt.formatFloatSlice(values, options, elem_strides, writer), - .integer => try stdx.fmt.formatIntSlice(values, options, elem_strides, writer), - .complex => try stdx.fmt.formatComplexSlice(values, options, elem_strides, writer), - .bool => try stdx.fmt.formatBoolSlice(values, options, elem_strides, writer), - } - }, - } - try writer.writeByte('\n'); - return; - } - // TODO: consider removing the \n if dim is 1 for this axis. - try writer.splatByteAll(' ', indent_level); - _ = try writer.write("{\n"); - defer { - writer.splatByteAll(' ', indent_level) catch {}; - _ = writer.write("},\n") catch {}; - } - - // Write first rows - const n: u64 = @intCast(self.dim(0)); - for (0..@min(num_rows, n)) |d| { - const di: i64 = @intCast(d); - const sliced_self = self.slice1d(0, .{ .start = di, .end = di + 1 }).squeeze(0); - try sliced_self.prettyPrintIndented(writer, num_rows, indent_level + 2, options); - } - - if (n < num_rows) return; - // Skip middle rows - if (n > 2 * num_rows) { - try writer.splatByteAll(' ', indent_level + 2); - _ = try writer.write("...\n"); - } - // Write last rows - for (@max(n - num_rows, num_rows)..n) |d| { - const di: i64 = @intCast(d); - const sliced_self = self.slice1d(0, .{ .start = di, .end = di + 1 }).squeeze(0); - try sliced_self.prettyPrintIndented(writer, num_rows, indent_level + 2, options); - } - } -}; - -fn parseArrayInfo(T: type) Shape { - return switch (@typeInfo(T)) { - .array => |arr| { - const s = parseArrayInfo(arr.child); - return s.insert(0, .{arr.len}); - }, - else => .{ ._dtype = DataType.fromZigType(T) }, - }; -} diff --git a/zml2/io.zig b/zml/io.zig similarity index 100% rename from zml2/io.zig rename to zml/io.zig diff --git a/zml2/io/BUILD.bazel b/zml/io/BUILD.bazel similarity index 89% rename from zml2/io/BUILD.bazel rename to zml/io/BUILD.bazel index 6d5241ca..8407a5c6 100644 --- a/zml2/io/BUILD.bazel +++ b/zml/io/BUILD.bazel @@ -9,6 +9,6 @@ zig_library( visibility = ["//visibility:public"], deps = [ "//stdx", - "//zml2/io/vfs", + "//zml/io/vfs", ], ) diff --git a/zml2/io/io.zig b/zml/io/io.zig similarity index 100% rename from zml2/io/io.zig rename to zml/io/io.zig diff --git a/zml2/io/vfs.zig b/zml/io/vfs.zig similarity index 100% rename from zml2/io/vfs.zig rename to zml/io/vfs.zig diff --git a/zml2/io/vfs/BUILD.bazel b/zml/io/vfs/BUILD.bazel similarity index 100% rename from zml2/io/vfs/BUILD.bazel rename to zml/io/vfs/BUILD.bazel diff --git a/zml2/io/vfs/base.zig b/zml/io/vfs/base.zig similarity index 99% rename from zml2/io/vfs/base.zig rename to zml/io/vfs/base.zig index d2d8f30e..6b032562 100644 --- a/zml2/io/vfs/base.zig +++ b/zml/io/vfs/base.zig @@ -378,6 +378,7 @@ pub const VFSBase = struct { } pub fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { + std.log.info("cocou bg", .{}); const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileWriteFilePositional(self.inner.userdata, file, header, reader, limit, offset); } diff --git a/zml2/io/vfs/file.zig b/zml/io/vfs/file.zig similarity index 100% rename from zml2/io/vfs/file.zig rename to zml/io/vfs/file.zig diff --git a/zml2/io/vfs/hf.zig b/zml/io/vfs/hf.zig similarity index 100% rename from zml2/io/vfs/hf.zig rename to zml/io/vfs/hf.zig diff --git a/zml2/io/vfs/hf2.zig b/zml/io/vfs/hf2.zig similarity index 100% rename from zml2/io/vfs/hf2.zig rename to zml/io/vfs/hf2.zig diff --git a/zml2/io/vfs/http.zig b/zml/io/vfs/http.zig similarity index 100% rename from zml2/io/vfs/http.zig rename to zml/io/vfs/http.zig diff --git a/zml2/io/vfs/index.zig b/zml/io/vfs/index.zig similarity index 100% rename from zml2/io/vfs/index.zig rename to zml/io/vfs/index.zig diff --git a/zml2/io/vfs/proxy.zig b/zml/io/vfs/proxy.zig similarity index 100% rename from zml2/io/vfs/proxy.zig rename to zml/io/vfs/proxy.zig diff --git a/zml/meta.zig b/zml/meta.zig index 891a27ec..e26fa014 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -5,10 +5,6 @@ const stdx = @import("stdx"); const FnParam = stdx.meta.FnParam; const asSlice = stdx.meta.asSlice; -test { - std.testing.refAllDecls(@This()); -} - /// Visit a given type `T` and replace all fields containing `From` by fields containing `To`. pub fn MapType(From: type, To: type) type { return struct { @@ -26,64 +22,73 @@ pub fn MapType(From: type, To: type) type { .@"struct" => |struct_infos| { const fields = struct_infos.fields; var same: bool = true; - var struct_fields: [fields.len]std.builtin.Type.StructField = undefined; - for (struct_fields[0..], fields) |*struct_field, field| { + var struct_field_names: [fields.len][]const u8 = undefined; + var struct_field_types: [fields.len]type = undefined; + var struct_field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; + for (struct_field_names[0..], struct_field_types[0..], struct_field_attrs[0..], fields) |*field_name, *field_type, *field_attr, field| { if (!field.is_comptime) { const R = map(field.type); if (R == field.type) { - struct_field.* = field; + field_name.* = field.name; + field_type.* = field.type; + field_attr.* = .{ + .@"comptime" = field.is_comptime, + .@"align" = field.alignment, + .default_value_ptr = field.default_value_ptr, + }; } else { - struct_field.* = .{ - .name = field.name, - .type = R, - .default_value_ptr = null, - .is_comptime = field.is_comptime, - .alignment = @alignOf(R), + field_name.* = field.name; + field_type.* = R; + field_attr.* = .{ + .@"comptime" = field.is_comptime, + .@"align" = @alignOf(R), }; same = false; // Handle the case `field: ?Tensor = null` // Generic handling of default value is complicated, // it would require to call the callback at comptime. if (R == ?To) { - struct_field.default_value_ptr = &@as(R, null); + field_attr.default_value_ptr = &@as(R, null); } } } else { - struct_field.* = field; + field_name.* = field.name; + field_type.* = field.type; + field_attr.* = .{ + .@"comptime" = field.is_comptime, + .@"align" = field.alignment, + .default_value_ptr = field.default_value_ptr, + }; } } if (same) return T; - return @Type(.{ .@"struct" = .{ - .layout = .auto, - .fields = struct_fields[0..], - .decls = &.{}, - .is_tuple = struct_infos.is_tuple, - } }); + return if (struct_infos.is_tuple) @Tuple(&struct_field_types) else @Struct(.auto, null, &struct_field_names, &struct_field_types, &struct_field_attrs); }, .@"union" => |union_info| { const fields = union_info.fields; var same: bool = true; - var union_fields: [fields.len]std.builtin.Type.UnionField = undefined; - for (union_fields[0..], fields) |*union_field, field| { + var union_field_names: [fields.len][]const u8 = undefined; + var union_field_types: [fields.len]type = undefined; + var union_field_attrs: [fields.len]std.builtin.Type.UnionField.Attributes = undefined; + for (union_field_names[0..], union_field_types[0..], union_field_attrs[0..], fields) |*field_name, *field_type, *field_attr, field| { const R = map(field.type); if (R == field.type) { - union_field.* = field; + field_name.* = field.name; + field_type.* = field.type; + field_attr.* = .{ + .@"align" = field.alignment, + }; } else { - union_field.* = .{ - .name = field.name, - .type = R, - .alignment = @alignOf(R), + field_name.* = field.name; + field_type.* = R; + field_attr.* = .{ + .@"align" = @alignOf(R), }; same = false; } } if (same) return T; - return @Type(.{ .@"union" = .{ - .layout = .auto, - .tag_type = union_info.tag_type, - .fields = union_fields[0..], - .decls = &.{}, - } }); + return @Union(.auto, union_info.tag_type, &union_field_names, &union_field_types, &union_field_attrs); }, .array => |arr_info| [arr_info.len]map(arr_info.child), .pointer => |ptr_info| switch (ptr_info.size) { @@ -321,57 +326,61 @@ pub fn MapRestrict(From: type, To: type) type { const fields = struct_infos.fields; var num_fields: usize = 0; - var struct_fields: [fields.len]std.builtin.Type.StructField = undefined; + var struct_field_names: [fields.len][]const u8 = undefined; + var struct_field_types: [fields.len]type = undefined; + var struct_field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; for (fields) |field| { if (!field.is_comptime and Contains(field.type, From)) { const R = map(field.type); if (R == field.type) { - struct_fields[num_fields] = field; + struct_field_names[num_fields] = field.name; + struct_field_types[num_fields] = field.type; + struct_field_attrs[num_fields] = .{ + .@"comptime" = field.is_comptime, + .@"align" = field.alignment, + .default_value_ptr = field.default_value_ptr, + }; } else { const name = if (struct_infos.is_tuple) struct_infos.fields[num_fields].name else field.name; - struct_fields[num_fields] = .{ - .name = name, - .type = R, - .default_value_ptr = null, - .is_comptime = false, - .alignment = @alignOf(R), + struct_field_names[num_fields] = name; + struct_field_types[num_fields] = R; + struct_field_attrs[num_fields] = .{ + .@"align" = @alignOf(R), }; // Handle the case `field: ?Tensor = null` // Generic handling of default value is not possible. if (R == ?To) { - struct_fields[num_fields].default_value_ptr = &@as(R, null); + struct_field_attrs[num_fields].default_value_ptr = &@as(R, null); } } num_fields += 1; } } if (num_fields == 0) return void; - return @Type(.{ .@"struct" = .{ - .layout = .auto, - .fields = struct_fields[0..num_fields], - .decls = &.{}, - .is_tuple = struct_infos.is_tuple, - } }); + return if (struct_infos.is_tuple) @Tuple(struct_field_types[0..num_fields]) else @Struct( + .auto, + null, + struct_field_names[0..num_fields], + struct_field_types[0..num_fields], + struct_field_attrs[0..num_fields], + ); }, .@"union" => |union_info| { // We know that at least one of the union field contains a From. // We map each field individually. Fields without From, are replaced by "void". const fields = union_info.fields; - var union_fields: [fields.len]std.builtin.Type.UnionField = undefined; + var union_field_names: [fields.len][]const u8 = undefined; + var union_field_types: [fields.len]type = undefined; + var union_field_attrs: [fields.len]std.builtin.Type.UnionField.Attributes = undefined; for (0.., fields) |i, field| { const FT = map(field.type); - union_fields[i] = .{ - .name = field.name, - .type = FT, - .alignment = @alignOf(FT), + union_field_names[i] = field.name; + union_field_types[i] = FT; + union_field_attrs[i] = .{ + .@"align" = @alignOf(FT), }; } - return @Type(.{ .@"union" = .{ - .layout = .auto, - .tag_type = union_info.tag_type, - .fields = union_fields[0..], - .decls = &.{}, - } }); + return @Union(.auto, union_info.tag_type, &union_field_names, &union_field_types, &union_field_attrs); }, .array => |arr_info| [arr_info.len]map(arr_info.child), .pointer => |ptr_info| switch (ptr_info.size) { @@ -427,7 +436,14 @@ test MapRestrict { /// Recursively visit the given struct and calls the callback for each K found. /// The `v` parameter must me a pointer, and tensor data need to be mutable if callbacks needs it. -pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) void { +pub fn VisitReturn(comptime cb: anytype) type { + return if (stdx.meta.FnSignature(cb, null).ReturnErrorSet) |error_set| + error_set!void + else + void; +} + +pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) VisitReturn(cb) { const Callback = @TypeOf(cb); const Ptr = @TypeOf(v); const type_info_v = @typeInfo(Ptr); @@ -436,6 +452,7 @@ pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) void { } const ptr_info = type_info_v.pointer; const Child = ptr_info.child; + const can_error = stdx.meta.FnSignature(cb, null).ReturnErrorSet != null; const K, const mutating_cb = switch (@typeInfo(FnParam(cb, 1))) { .pointer => |info| .{ info.child, !info.is_const }, @@ -449,7 +466,7 @@ pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) void { *const K, *K => return cb(ctx, v), *const ?K, *?K => return if (v.*) |*val| cb(ctx, val) else {}, []const K, []K => { - for (v) |*v_elem| cb(ctx, v_elem); + for (v) |*v_elem| if (can_error) try cb(ctx, v_elem) else cb(ctx, v_elem); return; }, else => {}, @@ -468,15 +485,15 @@ pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) void { const field_type_info = @typeInfo(field.type); // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. switch (field_type_info) { - .pointer => visit(cb, ctx, @field(v, field.name)), - .array, .optional, .@"union", .@"struct" => visit(cb, ctx, &@field(v, field.name)), + .pointer => if (can_error) try visit(cb, ctx, @field(v, field.name)) else visit(cb, ctx, @field(v, field.name)), + .array, .optional, .@"union", .@"struct" => if (can_error) try visit(cb, ctx, &@field(v, field.name)) else visit(cb, ctx, &@field(v, field.name)), else => {}, } }, - .array => |_| for (v) |*elem| visit(cb, ctx, elem), - .optional => if (v.* != null) visit(cb, ctx, &v.*.?), + .array => |_| for (v) |*elem| if (can_error) try visit(cb, ctx, elem) else visit(cb, ctx, elem), + .optional => if (v.* != null) if (can_error) try visit(cb, ctx, &v.*.?) else visit(cb, ctx, &v.*.?), .@"union" => switch (v.*) { - inline else => |*v_field| visit(cb, ctx, v_field), + inline else => |*v_field| if (can_error) try visit(cb, ctx, v_field) else visit(cb, ctx, v_field), }, else => stdx.debug.compileError("zml.meta.visit({}) doesn't support fields of type: {}", .{ Callback, Child }), }, @@ -488,15 +505,15 @@ pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) void { const field_type_info = @typeInfo(field.type); // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. if (field_type_info == .pointer) { - visit(cb, ctx, @field(v_elem, field.name)); + if (can_error) try visit(cb, ctx, @field(v_elem, field.name)) else visit(cb, ctx, @field(v_elem, field.name)); } else { - visit(cb, ctx, &@field(v_elem, field.name)); + if (can_error) try visit(cb, ctx, &@field(v_elem, field.name)) else visit(cb, ctx, &@field(v_elem, field.name)); } }, - .array => |_| for (v) |*elem| visit(cb, ctx, elem), - .optional => if (v.* != null) visit(cb, ctx, &v.*.?), + .array => |_| for (v) |*elem| if (can_error) try visit(cb, ctx, elem) else visit(cb, ctx, elem), + .optional => if (v.* != null) if (can_error) try visit(cb, ctx, &v.*.?) else visit(cb, ctx, &v.*.?), .@"union" => switch (v_elem.*) { - inline else => |*v_field| visit(cb, ctx, v_field), + inline else => |*v_field| if (can_error) try visit(cb, ctx, v_field) else visit(cb, ctx, v_field), }, else => stdx.debug.compileError("zml.meta.visit({}) doesn't support fields of type: {}", .{ Callback, Child }), } diff --git a/zml/mlirx.zig b/zml/mlirx.zig index 286166aa..2c3ddb17 100644 --- a/zml/mlirx.zig +++ b/zml/mlirx.zig @@ -7,104 +7,77 @@ const Shape = @import("shape.zig").Shape; const mlirx = @This(); -/// Returns the mlir.Type corresponding to a given zml.Shape. -pub fn tensorType(ctx: mlir.Context, sh: Shape) mlir.Type { - return .tensor(sh.dims(), mlirx.Type.fromDType(ctx, sh.dtype())); -} - -pub fn denseElementAttrType(dt: dtype.DataType) ?mlir.DenseElementsAttributeTypes { - return switch (dt) { - .bool => .bool, - .i8 => .i8, - .i16 => .i16, - .i32 => .i32, - .i64 => .i64, - .u8 => .u8, - .u16 => .u16, - .u32 => .u32, - .u64 => .u64, - .bf16 => .bf16, - .f16 => .f16, - .f32 => .f32, - .f64 => .f64, - else => null, - }; -} - pub const Type = struct { - pub fn fromDType(ctx: mlir.Context, dt: dtype.DataType) mlir.Type { + pub fn fromDType(ctx: *mlir.Context, dt: dtype.DataType) *const mlir.Type { return switch (dt) { - .bool => .int(ctx, .i1), - .f4e2m1 => .float(ctx, .f4e2m1fn), - .f8e3m4 => .float(ctx, .f8e3m4), - .f8e4m3 => .float(ctx, .f8e4m3), - .f8e4m3b11fnuz => .float(ctx, .f8e4m3b11fnuz), - .f8e4m3fn => .float(ctx, .f8e4m3fn), - .f8e4m3fnuz => .float(ctx, .f8e4m3fnuz), - .f8e5m2 => .float(ctx, .f8e5m2), - .f8e5m2fnuz => .float(ctx, .f8e5m2fnuz), - .f8e8m0 => .float(ctx, .f8e8m0fnu), - .bf16 => .float(ctx, .bf16), - .f16 => .float(ctx, .f16), - .f32 => .float(ctx, .f32), - .f64 => .float(ctx, .f64), - .i2 => .int(ctx, .i2), - .i4 => .int(ctx, .i4), - .i8 => .int(ctx, .i8), - .i16 => .int(ctx, .i16), - .i32 => .int(ctx, .i32), - .i64 => .int(ctx, .i64), - .u2 => .int(ctx, .u2), - .u4 => .int(ctx, .u4), - .u8 => .int(ctx, .u8), - .u16 => .int(ctx, .u16), - .u32 => .int(ctx, .u32), - .u64 => .int(ctx, .u64), - .c64 => .complex(ctx, .c64), - .c128 => .complex(ctx, .c128), + .bool => mlir.integerType(ctx, .i1), + .f4e2m1 => mlir.floatType(ctx, .f4e2m1fn), + .f8e3m4 => mlir.floatType(ctx, .f8e3m4), + .f8e4m3 => mlir.floatType(ctx, .f8e4m3), + .f8e4m3fn => mlir.floatType(ctx, .f8e4m3fn), + .f8e4m3fnuz => mlir.floatType(ctx, .f8e4m3fnuz), + .f8e4m3b11fnuz => mlir.floatType(ctx, .f8e4m3b11fnuz), + .f8e5m2 => mlir.floatType(ctx, .f8e5m2), + .f8e5m2fnuz => mlir.floatType(ctx, .f8e5m2fnuz), + .f8e8m0 => @panic("Unsupported type"), + .bf16 => mlir.floatType(ctx, .bf16), + .f16 => mlir.floatType(ctx, .f16), + .f32 => mlir.floatType(ctx, .f32), + .f64 => mlir.floatType(ctx, .f64), + .i2 => mlir.integerType(ctx, .i2), + .i4 => mlir.integerType(ctx, .i4), + .i8 => mlir.integerType(ctx, .i8), + .i16 => mlir.integerType(ctx, .i16), + .i32 => mlir.integerType(ctx, .i32), + .i64 => mlir.integerType(ctx, .i64), + .u2 => mlir.integerType(ctx, .u2), + .u4 => mlir.integerType(ctx, .u4), + .u8 => mlir.integerType(ctx, .u8), + .u16 => mlir.integerType(ctx, .u16), + .u32 => mlir.integerType(ctx, .u32), + .u64 => mlir.integerType(ctx, .u64), + .c64 => mlir.complexType(ctx, .c64), + .c128 => mlir.complexType(ctx, .c128), }; } - pub fn toDType(mlir_type: mlir.Type) dtype.DataType { + // TODO(Corentin): Maybe remove that as it now requires a mlir context. + // It's weird to have to provide a mlir context to get a zml.DataType from a mlir.Type. + pub fn toDType(ctx: *mlir.Context, mlir_type: *const mlir.Type) dtype.DataType { const mapping = .{ - .{ .bool, mlir.IntegerType(.i1) }, - .{ .f4e2m1, mlir.FloatType(.f4e2m1fn) }, - .{ .f8e3m4, mlir.FloatType(.f8e3m4) }, - .{ .f8e4m3, mlir.FloatType(.f8e4m3) }, - .{ .f8e4m3b11fnuz, mlir.FloatType(.f8e4m3b11fnuz) }, - .{ .f8e4m3fn, mlir.FloatType(.f8e4m3fn) }, - .{ .f8e4m3fnuz, mlir.FloatType(.f8e4m3fnuz) }, - .{ .f8e5m2, mlir.FloatType(.f8e5m2) }, - .{ .f8e5m2fnuz, mlir.FloatType(.f8e5m2fnuz) }, - .{ .f8e8m0, mlir.FloatType(.f8e8m0fnu) }, - .{ .bf16, mlir.FloatType(.bf16) }, - .{ .f16, mlir.FloatType(.f16) }, - .{ .f32, mlir.FloatType(.f32) }, - .{ .f64, mlir.FloatType(.f64) }, + .{ .bool, mlir.integerType(ctx, .i1) }, - .{ .i2, mlir.IntegerType(.i2) }, - .{ .i4, mlir.IntegerType(.i4) }, - .{ .i8, mlir.IntegerType(.i8) }, - .{ .i16, mlir.IntegerType(.i16) }, - .{ .i32, mlir.IntegerType(.i32) }, - .{ .i64, mlir.IntegerType(.i64) }, + .{ .f8e4m3b11fnuz, mlir.floatType(ctx, .f8e4m3b11fnuz) }, + .{ .f8e4m3fn, mlir.floatType(ctx, .f8e4m3fn) }, + .{ .f8e4m3fnuz, mlir.floatType(ctx, .f8e4m3fnuz) }, + .{ .f8e3m4, mlir.floatType(ctx, .f8e3m4) }, + .{ .f8e5m2, mlir.floatType(ctx, .f8e5m2) }, + .{ .f8e5m2fnuz, mlir.floatType(ctx, .f8e5m2fnuz) }, + .{ .f4e2m1, mlir.floatType(ctx, .f4e2m1fn) }, + .{ .bf16, mlir.floatType(ctx, .bf16) }, + .{ .f16, mlir.floatType(ctx, .f16) }, + .{ .f32, mlir.floatType(ctx, .f32) }, + .{ .f64, mlir.floatType(ctx, .f64) }, - .{ .u2, mlir.IntegerType(.u2) }, - .{ .u4, mlir.IntegerType(.u4) }, - .{ .u8, mlir.IntegerType(.u8) }, - .{ .u16, mlir.IntegerType(.u16) }, - .{ .u32, mlir.IntegerType(.u32) }, - .{ .u64, mlir.IntegerType(.u64) }, + .{ .i4, mlir.integerType(ctx, .i4) }, + .{ .i8, mlir.integerType(ctx, .i8) }, + .{ .i16, mlir.integerType(ctx, .i16) }, + .{ .i32, mlir.integerType(ctx, .i32) }, + .{ .i64, mlir.integerType(ctx, .i64) }, - .{ .c64, mlir.ComplexType(.c64) }, - .{ .c128, mlir.ComplexType(.c128) }, + .{ .u4, mlir.integerType(ctx, .u4) }, + .{ .u8, mlir.integerType(ctx, .u8) }, + .{ .u16, mlir.integerType(ctx, .u16) }, + .{ .u32, mlir.integerType(ctx, .u32) }, + .{ .u64, mlir.integerType(ctx, .u64) }, + + .{ .c64, mlir.complexType(ctx, .c64) }, + .{ .c128, mlir.complexType(ctx, .c128) }, }; - // TODO: this seems quite slow to have all of those functions calls. - // Maybe we should memoize the ptr of a set of mlir types when creating the context. inline for (mapping) |entry| { const dt, const mlirT = entry; - if (mlirT.is_a_fn(mlir_type._inner)) { + if (mlirT.eql(mlir_type)) { return dt; } } @@ -112,3 +85,9 @@ pub const Type = struct { std.debug.panic("Could not convert mlir.Type to DataType: {f}", .{mlir_type}); } }; + +pub var once = std.once(struct { + fn call() void { + mlir.registerPasses("Transforms"); + } +}.call); diff --git a/zml/module.zig b/zml/module.zig index 7ba63b23..525cfbd6 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -1,848 +1,399 @@ const std = @import("std"); -const async = @import("async"); const c = @import("c"); -const dialect = @import("mlir/dialects"); +const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); const stdx = @import("stdx"); const upb = @import("upb"); -const BaseExe = @import("exe.zig").BaseExe; const Buffer = @import("buffer.zig").Buffer; +const Exe = @import("exe.zig").Exe; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); -const ops = @import("ops.zig"); const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; -const Target = @import("platform.zig").Target; const Tensor = @import("tensor.zig").Tensor; -const Tracer = @import("tools/tracer.zig").Tracer; const log = std.log.scoped(.@"zml/module"); -test { - std.testing.refAllDecls(@This()); -} - -pub const MlirFn = struct { - name: []const u8, - args_shapes: []Shape, - res_tensors: *const anyopaque, - res_types: []mlir.Type, - res_shapes: []Shape, - res_donations: []Tensor._Donation, - mlir_fn: mlir.Operation, - - pub const Kind = enum { - main, - private, - }; -}; +const AttributeList = stdx.BoundedArray(mlir.NamedAttribute, 3); pub const CompilationContext = struct { - _platform: Platform, - _name: []const u8, + pub const Scope = struct { + block: *mlir.Block, + id_to_argument: std.AutoArrayHashMapUnmanaged(usize, usize), + id_to_donation: std.AutoArrayHashMapUnmanaged(usize, usize), + id_to_output_memory_kind: std.AutoArrayHashMapUnmanaged(usize, Buffer.Memory), + arena: std.heap.ArenaAllocator, - _arena: std.heap.ArenaAllocator, - _mlir_ctx: mlir.Context, - _mlir_registry: mlir.Registry, - _mlir_canonicalizer: mlir.PassManager, + pub fn initFromBlock(allocator: std.mem.Allocator, block: *mlir.Block) Scope { + const arena: std.heap.ArenaAllocator = .init(allocator); + return .{ + .block = block, + .id_to_argument = .empty, + .id_to_donation = .empty, + .id_to_output_memory_kind = .empty, + .arena = arena, + }; + } - _module: mlir.Module, + pub fn deinit(self: *Scope) void { + self.arena.deinit(); + } + }; - _blocks: stdx.BoundedArray(TaggedBlock, 64) = .{}, - _fn_cache: FnCache = .{}, + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, - _block_args: TensorToBlockArg = .{}, - _unique_id: u64 = 10000, - _tracer: Tracer, + mlir_registry: *mlir.DialectRegistry, + mlir_ctx: *mlir.Context, + mlir_pass_manager: *mlir.PassManager, + //mlir_op_pass_manager: *mlir.OpPassManager, + module: *mlir.Module, + platform: Platform, + + scopes: stdx.BoundedArray(Scope, 16) = .{}, - _previous: ?*CompilationContext = null, threadlocal var _current: ?*CompilationContext = null; - const TaggedBlock = struct { mlir.Block, mlir.Block.RecursiveOpts }; - const TensorToBlockArg = std.AutoHashMapUnmanaged(Tensor._Id, struct { mlir.Value, Tensor._Donation }); - const AttributeList = stdx.BoundedArray(mlir.NamedAttribute, 3); - - pub fn init(allocator_: std.mem.Allocator, full_name: []const u8, platform: Platform) !CompilationContext { - const mlir_registry = mlir.Registry.init() catch unreachable; + pub fn init(allocator: std.mem.Allocator, platform: Platform) CompilationContext { + mlir.registerPasses("Transforms"); + const mlir_registry = mlir.DialectRegistry.init() catch unreachable; inline for (.{ "func", "stablehlo" }) |d| { mlir.DialectHandle.fromString(d).insertDialect(mlir_registry); } - var mlir_ctx = mlir.Context.initWithRegistry(mlir_registry, false) catch unreachable; + var mlir_ctx = mlir.Context.init(.{ .registry = mlir_registry, .threading = false }) catch unreachable; mlir_ctx.loadAllAvailableDialects(); - // Too long module names create too long file paths and files failed to create. - // * leave half of the space for parent folder and XLA generated filename, - // * leave 17 bytes for the module hash (16 + 1 for underscore). - const max_name_len = @divFloor(std.fs.max_path_bytes, 2) - 17; - const name = full_name[0..@min(max_name_len, full_name.len)]; + //const loc = mlir.Location.fromSrc(mlir_ctx, @src()).named(mlir_ctx, "main"); + const module = mlir.Module.init(.unknown(mlir_ctx)); + module.operation().setAttributeByName("sym_name", mlir.stringAttribute(mlir_ctx, "zml")); - const loc = mlir_ctx.location(@src()).named(mlir_ctx, "main"); - const module = mlir.Module.init(loc); - module.op().setAttributeByName("sym_name", .string(mlir_ctx, "zml")); - - var canonicalizer = try mlir.PassManager.init(mlir_ctx); + const pass_manager = mlir.PassManager.init(mlir_ctx); { - var opm = canonicalizer.asOpPassManager(); - try opm.addPipeline("canonicalize"); - try opm.addPipeline("cse"); - try opm.addPipeline("canonicalize"); + var opm = pass_manager.asOpPassManager(); + const passes: []const []const u8 = &.{ + "canonicalize", + "cse", + "canonicalize", + }; + for (passes) |pass| { + opm.addPipeline(pass) catch unreachable; + } } - var arena = std.heap.ArenaAllocator.init(allocator_); - _ = try arena.allocator().alloc(u8, 4096); - _ = arena.reset(.retain_capacity); - return .{ - ._platform = platform, - ._name = try arena.allocator().dupe(u8, name), - ._mlir_ctx = mlir_ctx, - ._mlir_registry = mlir_registry, - ._mlir_canonicalizer = canonicalizer, - ._module = module, - ._blocks = .{}, - ._fn_cache = .{}, - ._arena = arena, - ._tracer = Tracer.init("ai.zml.compilation"), + .allocator = allocator, + .arena = std.heap.ArenaAllocator.init(allocator), + .mlir_registry = mlir_registry, + .mlir_ctx = mlir_ctx, + .mlir_pass_manager = pass_manager, + .module = module, + .platform = platform, }; } pub fn deinit(self: *CompilationContext) void { - // No need to deinit self._fn_cache cause it uses our arena - self._mlir_ctx.deinit(); - self._mlir_registry.deinit(); - self._arena.deinit(); - } - - pub fn allocator(self: *CompilationContext) std.mem.Allocator { - return self._arena.allocator(); + self.mlir_pass_manager.deinit(); + self.module.deinit(); + self.mlir_ctx.deinit(); + self.mlir_registry.deinit(); + self.arena.deinit(); } pub fn activate(self: *CompilationContext) void { - self._previous = _current; + std.debug.assert(_current == null); _current = self; } pub fn deactivate(self: *CompilationContext) void { - std.debug.assert(_current != null and _current.? == self); - _current = self._previous; + _ = self; + _current = null; } pub fn current() *CompilationContext { return _current.?; } - pub fn target(self: *const CompilationContext) Target { - return self._platform.target; + pub fn currentScope(self: *CompilationContext) *Scope { + return &self.scopes.slice()[self.scopes.len - 1]; } - pub fn mlirCtx(self: *const CompilationContext) mlir.Context { - return self._mlir_ctx; + pub fn pushBlock(self: *CompilationContext, block: *mlir.Block) void { + const scope = Scope.initFromBlock(self.allocator, block); + self.scopes.appendAssumeCapacity(scope); } - pub fn location(self: *const CompilationContext, src: std.builtin.SourceLocation, comptime name: [:0]const u8, args: anytype) mlir.Location { - return self._mlir_ctx.location(src).namedFmt(self._mlir_ctx, name, args); - } - - /// Compiles the given function with the given arguments. - /// This is the untyped API and is not meant to be use directly. - /// - /// * allocator is used to allocate the result Exe - /// * args can contain a mix of tensors and shapes, allowing to pass a "model struct" containig tensors. - pub fn compileInternal( - self: *CompilationContext, - allocator_: std.mem.Allocator, - comptime func: anytype, - args: anytype, - ) !BaseExe { - const arena = self.allocator(); - - var timer = std.time.Timer.start() catch null; - const tensor_args = try self.tensorFromShapes(stdx.meta.FnArgs(func), arena, args); - // Run in a dedicated thread because compilation relies on `threadlocal`. - const f = try async.callBlocking(CompilationContext.emitMlir, .{ self, func, &tensor_args, CompilationContext.EmitMlirOpts{ .name = "main", .kind = .main } }); - const module = self._module; - module.getBody().appendOperation(f.mlir_fn); - - const sharding = self._platform.sharding(); - const mlir_ctx = self._mlir_ctx; - module.op().setAttributeByName("mhlo.num_replicas", .int(mlir_ctx, .i32, sharding.num_replicas)); - module.op().setAttributeByName("mhlo.num_partitions", .int(mlir_ctx, .i32, sharding.num_partitions)); - - const module_hash = computeModuleHash(self._platform, module); - var module_dir: ?[]const u8 = null; - var pjrt_location: ?[:0]const u8 = null; - - if (self._platform.compilation_options.xla_dump_to) |xla_dump_to| { - const sep = std.fs.path.sep_str; - const module_dir_name = try std.fmt.allocPrint(arena, "{s}{s}{s}{s}{s}_{x}", .{ xla_dump_to, sep, @tagName(self._platform.target), sep, self._name, module_hash }); - try std.fs.cwd().makePath(module_dir_name); - module_dir = try std.fs.cwd().realpathAlloc(arena, module_dir_name); - const cache_dir = try std.fs.cwd().openDir(module_dir.?, .{}); - - // Write the mlir to a file. All errors are discarded, since this is for debugging only. - const mlir_name = "module.mlir"; - if (cache_dir.createFile(mlir_name, .{ .truncate = true })) |file| { - var write_buf: [4096]u8 = undefined; - var writer = file.writer(&write_buf); - try module.op().print(&writer.interface, .{ .debug_info = true, .debug_info_pretty_form = false }); - try writer.interface.flush(); - log.info("Wrote MLIR to {s}/{s}", .{ module_dir.?, mlir_name }); - } else |_| { - log.warn("Failed to open {s}", .{mlir_name}); - } - - pjrt_location = try std.fs.path.joinZ(arena, &.{ module_dir.?, "module.pjrt" }); + pub fn popBlock(self: *CompilationContext) void { + var maybe_popped_scope = self.scopes.pop(); + if (maybe_popped_scope) |*popped| { + popped.deinit(); } - - const loaded_executable: *pjrt.LoadedExecutable = blk: { - if (pjrt_location) |pjrt_loc| { - if (loadPjrtExecutable(arena, self._platform, pjrt_loc)) |exe| { - log.info("Loaded pre-compiled module from {s} (generated from {s}/module.mlir)", .{ pjrt_loc, module_dir.? }); - break :blk exe; - } else |err| { - if (err != error.FileNotFound) log.warn("Failed to load pre-compiled module: {} at {s}", .{ err, pjrt_loc }); - } - } - - const loaded_executable = compileModuleToPjrtExecutable(arena, self._platform, module, module_dir) catch |err| { - log.err("pjrt-{s} failed to compile: {}", .{ @tagName(self._platform.target), err }); - if (module_dir) |dir| log.err("mlir can be found at {s}/module.mlir", .{dir}); - return err; - }; - - if (pjrt_location) |pjrt_loc| { - storePjrtExecutable(self._platform, loaded_executable, pjrt_loc) catch |err| { - log.warn("Failed to store compiled module: {} at {s}", .{ err, pjrt_loc }); - }; - } - break :blk loaded_executable; - }; - - { - const exe = try loaded_executable.getExecutable(self._platform.pjrt_api); - const stats = try exe.getCompiledMemoryStats(self._platform.pjrt_api); - log.debug("Compiled {s}: {any}", .{ self._name, stats }); - } - - log.debug("******** ZML generated MLIR ********", .{}); - log.debug("{f}", .{module.op().mlirFormatter(.{})}); - - if (timer) |*t| { - const time_ms = @divFloor(t.lap(), std.time.ns_per_ms); - if (time_ms > 1000) log.info("Compilation took {d:.3}s", .{stdx.math.divFloat(f32, time_ms, 1000)}); - } - - return BaseExe.init( - allocator_, - self._platform, - loaded_executable, - .{ - .input_shapes = f.args_shapes, - .result_shapes = f.res_shapes, - .n_devices = sharding.num_replicas * sharding.num_partitions, - }, - ); } - fn currentBlock(self: *const CompilationContext) ?TaggedBlock { - return if (self._blocks.len > 0) self._blocks.get(self._blocks.len - 1) else null; - } + //pub fn addShardingAttributes( + // self: *const CompilationContext, + // mesh: Mesh, + // arg_attrs: []AttributeList, + // res_attrs: []AttributeList, + // input_shapes: []const Shape, + // input_meshes: []const ?Mesh, + // output_info: OutputInfo, + //) void { + // if (mesh.isSinglePartition()) return; // No sharding attributes for single partition. - pub fn openBlock(self: *CompilationContext, kind: mlir.Block.RecursiveOpts, args: []const mlir.Type, locs: []const mlir.Location) !TaggedBlock { - const block: TaggedBlock = .{ try mlir.Block.init(args, locs), kind }; - self.pushBlock(block); - return block; - } + // const default_layout = mlir.NamedAttribute.named(self.mlir_ctx, "mhlo.layout_mode", mlir.stringAttribute(self.mlir_ctx, "default")); + // for (arg_attrs, input_shapes, input_meshes) |*attr, shape, input_mesh| { + // attr.appendAssumeCapacity(default_layout); + // if (input_mesh) |m| { + // // log.warn("addShardingAttributes for {} with input mesh: {}", .{ shape, m }); + // attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(m, shape)))); + // } else { + // // log.warn("addShardingAttributes for {} with progran mesh: {}", .{ shape, mesh }); + // attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(mesh, shape)))); + // } + // } - pub fn closeBlock(self: *CompilationContext, block: TaggedBlock) void { - const popped = self._blocks.pop(); - std.debug.assert(block[0].eql(popped.?[0])); - } + // for (res_attrs, output_info.shapes) |*attr, shape| { + // attr.appendAssumeCapacity(default_layout); + // attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(mesh, shape)))); + // } + //} - fn pushBlock(self: *CompilationContext, block: TaggedBlock) void { - self._blocks.appendAssumeCapacity(block); - } + //pub fn getShardingAttr(self: *const CompilationContext, sharding: Sharding) *const mlir.Attribute { + // var sharding_str: stdx.BoundedArray(u8, 128) = .{}; + // sharding.writeShardingRepresentation(sharding_str.writer()) catch unreachable; + // return mlir.stringAttribute(self.mlir_ctx, sharding_str.constSlice()); + //} +}; - /// Transform a Tensor -> Tensor function into an Mlir block. - /// `blkctx` represents values from outside the block that can be accessed inside the block. - /// Returns both the mlir.Block created and also the Tensors returned by `func`. - /// The returned tensors should not be returned to the user, - /// because their `mlir.Value` must not escape the block that created them. - /// But their shapes/tags can be safely propagated further. - pub fn makeBlock( - self: *CompilationContext, - kind: mlir.Block.RecursiveOpts, - comptime S: ops.BlockSignature, - func: *const S.Fn, - blkctx: S.BlkCtx, - args: S.Args, - ) struct { mlir.Block, S.Return } { - const N = S.nIn; - const loc = self.mlirCtx().location(@src()); - const locations = .{loc} ** N; - var input_types: [N]mlir.Type = undefined; - fillMlirTypes(&args, self.mlirCtx(), &input_types); +pub fn compile(allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, args: stdx.meta.FnArgs(func), platform: Platform) !Exe { + var compilation_context: CompilationContext = .init(allocator, platform); + defer compilation_context.deinit(); - // Before creating a new block, assign all received values to previous block, - // otherwise they will be assign to this block - if (self.currentBlock()) |prev_block| { - meta.visit(_appendTensorRecursive, prev_block, &blkctx); - } + const result = emitMlir(&compilation_context, func, args) catch unreachable; + defer result.output_info.deinit(compilation_context.allocator); + defer result.input_info.deinit(compilation_context.allocator); - const block = self.openBlock(kind, &input_types, &locations) catch unreachable; - defer self.closeBlock(block); + _ = result.func.appendTo(compilation_context.module.body()); + const sharding = platform.sharding(); + compilation_context.module.operation().setAttributeByName("mhlo.num_replicas", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, sharding.num_replicas)); + compilation_context.module.operation().setAttributeByName("mhlo.num_partitions", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, sharding.num_partitions)); - // Here we want to create the block with the correct mlir types. - // but we don't want to use the values themselves. - // So we create a copy of the arguments, and replace values - // by the block arguments. - var blk_args = args; - std.debug.assert(assignBlockArguments(&blk_args, block[0], 0) == N); + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); - const block_res = @call(.auto, func, S.blkArgs(blkctx, blk_args)); - var block_res_values: [S.nOut]mlir.Value = undefined; - self.extractValues(&block_res, &block_res_values); - const block_ret = dialect.stablehlo.returns_(self.mlirCtx(), &block_res_values, loc); - block[0].appendOperationRecursive(block_ret, block[1]); + const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), io, platform, compilation_context.module, null) catch unreachable; - return .{ block[0], block_res }; - } + log.debug("\n******** ZML generated MLIR ********\n{f}", .{compilation_context.module.operation()}); - fn _appendTensorRecursive(tagged_block: TaggedBlock, x: *const Tensor) void { - const block, const tag = tagged_block; - block.appendValueRecursive(x.value(), tag); - } + const num_devices = sharding.num_partitions * sharding.num_replicas; + const exe = try Exe.init(allocator, platform, loaded_executable, result.input_info.shapes, result.output_info.shapes, num_devices); + errdefer exe.deinit(); - pub const EmitMlirOpts = struct { - name: []const u8, - kind: MlirFn.Kind = .private, + return exe; +} + +fn collectShapes(allocator: std.mem.Allocator, v: anytype) ![]Shape { + const LocalContext = struct { + list: *std.array_list.Managed(Shape), }; + var list = std.array_list.Managed(Shape).init(allocator); + errdefer list.deinit(); - /// Generate an MLIR function from a ZML function. - /// The caller is responsible to have properly created the input - /// tensors with unique tensor ids. - pub fn emitMlir( - self: *CompilationContext, - comptime func: anytype, - args: *const stdx.meta.FnArgs(func), - opts: EmitMlirOpts, - ) error{OutOfMemory}!MlirFn { - const frame = self._tracer.frameStart("emitMlir.emit"); - errdefer self._tracer.frameEnd(frame, "emitMlir.emit"); - - const res_allocator = self.allocator(); - // Note: only temp allocations are done in the arena, - // the other allocations are in the context allocator. - var arena_state = std.heap.ArenaAllocator.init(self._arena.child_allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - const tensor_count = meta.count(Tensor, args); - - const mlir_ctx = self.mlirCtx(); - const loc = mlir_ctx.location(@src()); - - const locations = try arena.alloc(mlir.Location, tensor_count); - @memset(locations, mlir.Location.unknown(mlir_ctx)); - - const input_shapes = try res_allocator.alloc(Shape, tensor_count); - meta.collectBuf(Tensor.shape, {}, args, input_shapes); - - const input_types = try arena.alloc(mlir.Type, tensor_count); - for (input_types, input_shapes) |*t, sh| t.* = mlirx.tensorType(mlir_ctx, sh); - - const og_block_args = self._block_args; - defer { - self._block_args.deinit(self.allocator()); - self._block_args = og_block_args; + var context: LocalContext = .{ .list = &list }; + try meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { + try ctx_.list.append(tensor.shape()); } + }.cb, &context, v); - // Reset the buffer -> assignement - self._block_args = .{}; + return try list.toOwnedSlice(); +} - // Note: this isn't stricly necessary. We call `countTensor` on `fn_res`. - // But it forces user to have simpler function. - const ReturnT = stdx.meta.FnResult(func); - const out_tensor_count = comptime ops.staticCountTensors(ReturnT) orelse @compileError("Can't use " ++ @typeName(ReturnT) ++ " in an MLIR function, because it has a variable number of tensors"); +pub const OutputInfo = struct { + shapes: []Shape, + values: []*const mlir.Value, + donations: []?usize, + output_memory_kinds: []Buffer.Memory, - // Those are returned to caller so we don't put them in the arena, but in the module allocator. - const fn_res = try res_allocator.create(ReturnT); - const fn_res_types = try res_allocator.alloc(mlir.Type, out_tensor_count); - const fn_res_shapes = try res_allocator.alloc(Shape, out_tensor_count); - const fn_res_donations = try res_allocator.alloc(Tensor._Donation, out_tensor_count); - const fn_res_output_memory_kind = try res_allocator.alloc(Buffer.Memory, out_tensor_count); - var fn_body = self.openBlock(.hermetic, input_types, locations) catch unreachable; - { - defer self.closeBlock(fn_body); - - try self._block_args.ensureUnusedCapacity(self.allocator(), @intCast(tensor_count)); - const assigned_args_count = self.mapBlockArguments(args, fn_body[0], 0); - std.debug.assert(assigned_args_count == tensor_count); - - fn_res.* = forward: { - self.activate(); - defer self.deactivate(); - break :forward @call(.auto, func, args.*); - }; - - var fn_res_values: [out_tensor_count]mlir.Value = undefined; - self.extractValuesAndTypes(fn_res, &fn_res_values, fn_res_types, fn_res_shapes, fn_res_donations, fn_res_output_memory_kind); - - const fn_ret = dialect.func.return_(mlir_ctx, &fn_res_values, loc); - fn_body[0].appendOperationRecursive(fn_ret, fn_body[1]); - } - - const arg_attrs = try arena.alloc(AttributeList, tensor_count); - @memset(arg_attrs, .{}); - - const res_attrs = try arena.alloc(AttributeList, out_tensor_count); - @memset(res_attrs, .{}); - - if (opts.kind == .main) { - self.addDonationsAttributes(arg_attrs, fn_res_donations); - self.addOutputMemoryKindAttributes(res_attrs, fn_res_output_memory_kind); - if (self._platform.sharding().num_partitions > 1) { - self.addShardingAttributes(arg_attrs, res_attrs, input_shapes, fn_res_shapes); - } - } - - const mlir_fn = dialect.func.func(self.mlirCtx(), .{ - .sym_name = opts.name, - .args = input_types, - .arg_attrs = try finalizeAttributeList(arena, mlir_ctx, arg_attrs), - .results = fn_res_types, - .res_attrs = try finalizeAttributeList(arena, mlir_ctx, res_attrs), - .block = fn_body[0], - .location = loc, - }); - - self._tracer.frameEnd(frame, "emitMlir.emit"); - const canonicalize_frame = self._tracer.frameStart("emitMlir.canonicalize"); - defer self._tracer.frameEnd(canonicalize_frame, "emitMlir.canonicalize"); - self._mlir_canonicalizer.runOnOp(mlir_fn) catch |err| switch (err) { - error.InvalidMlir => { - log.err("Failed to canonicalize invalid mlir: {f}", .{mlir_fn.mlirFormatter(.{})}); - // user errors should have triggered a panic before we reach this. - @panic("ZML generated invalid mlir. Please open a bug report"); - }, - }; - - return .{ - .mlir_fn = mlir_fn, - .name = opts.name, - .args_shapes = input_shapes, - .res_tensors = fn_res, - .res_types = fn_res_types, - .res_shapes = fn_res_shapes, - .res_donations = fn_res_donations, - }; - } - - fn addOutputMemoryKindAttributes(self: CompilationContext, attributes: []AttributeList, output_memory_kind: []const Buffer.Memory) void { - const mlir_ctx = self.mlirCtx(); - for (attributes, output_memory_kind) |*attr, memory_kind| { - // .device is the default output, don't explicitly emit the attribute - if (memory_kind == .device) continue; - - attr.appendAssumeCapacity(.named( - mlir_ctx, - "mhlo.memory_kind", - .string(mlir_ctx, memory_kind.pjrtName()), - )); - } - } - - /// Given a list of donations mapping output buffers to input buffers, - /// generate donation attribute for each `n_args` input argument. - fn addDonationsAttributes(self: CompilationContext, attributes: []AttributeList, donations: []const Tensor._Donation) void { - const ctx = self.mlirCtx(); - var n_donations: usize = 0; - for (donations, 0..) |donation, index| { - switch (donation) { - .no_buffer => {}, - // This is an input buffer that has been returned, - // but without explicitly calling `reuseBuffer`. - // So we assume the intent was to return a new buffer. - .input_buffer => {}, - .arg => |a| { - n_donations += 1; - // This will break the day we writer another attribute before donation. - // When the time come, do a more fancy lookup here to check if an argument - // is donated twice. - stdx.debug.assert(attributes[a].len == 0, "Donation error ! Argument {d} has been donated twice ! To {d} and to {any}", .{ a, index, attributes[a].buffer[0] }); - attributes[a].appendAssumeCapacity(.named(ctx, "tf.aliasing_output", .int(ctx, .i32, @intCast(index)))); - // log.debug("attribute: {}", .{attributes[a].constSlice()}); - }, - } - } - } - - test addDonationsAttributes { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - - const s = Shape.init(.{8}, .f16); - - const Local = struct { - bias: Tensor, - - pub fn _fwd(self: @This(), x: Tensor, y: Tensor) [2]Tensor { - const x1 = zml.ops.call(self, ._inner, .{x}); - const x2 = zml.ops.call(self, ._inner, .{x1}); - return .{ x1.reuseBuffer(y), x2 }; - } - - pub fn _inner(self: @This(), x: Tensor) Tensor { - const y = x.add(self.bias); - return y.reuseBuffer(x); - } - }; - - const model: Local = .{ - .bias = zml.Tensor{ ._shape = s, ._id = .{ .buffer_id = 0 } }, - }; - - var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); - defer comp.deinit(); - var tensor_args = .{ model, Tensor{ ._shape = s, ._id = .{ .buffer_id = 1234 } }, Tensor{ ._shape = s, ._id = .{ .buffer_id = 1235 } } }; - const f = try comp.emitMlir(Local._fwd, &tensor_args, .{ .name = "test.emitMlir.Local.forward", .kind = .main }); - - var mlir_code: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer mlir_code.deinit(); - try f.mlir_fn.print(&mlir_code.writer, .{}); - - // Check that the `x` input argument gives its buffer to the result tensor. - // `%arg0` is the bias of the model, `%arg1` is `x`, `%arg2` is `y`. - try std.testing.expectEqual(3, f.args_shapes.len); - // We should have two buffers being donated. - const template = "tf.aliasing_output = {d} : i32"; - var buf = template.*; - for (0..2) |i| { - const alias_attr = std.fmt.bufPrint(&buf, template, .{i}) catch unreachable; - std.testing.expect(std.mem.indexOf(u8, mlir_code.written(), alias_attr) != null) catch |err| { - log.warn("Didn't produced the expected IR:\n{s}", .{mlir_code.written()}); - return err; - }; - } - } - - fn addShardingAttributes(self: CompilationContext, arg_attrs: []AttributeList, res_attrs: []AttributeList, input_shapes: []const Shape, output_shapes: []const Shape) void { - const ctx = self.mlirCtx(); - if (!self._platform.compilation_options.sharding_enabled) return; - const default_layout = mlir.NamedAttribute.named(ctx, "mhlo.layout_mode", .string(ctx, "default")); - for (arg_attrs, input_shapes) |*attr, shape| { - attr.appendAssumeCapacity(default_layout); - attr.appendAssumeCapacity(.named(ctx, "mhlo.sharding", self.getShardingAttr(shape))); - } - - for (res_attrs, output_shapes) |*attr, shape| { - attr.appendAssumeCapacity(default_layout); - attr.appendAssumeCapacity(.named(ctx, "mhlo.sharding", self.getShardingAttr(shape))); - } - } - - pub fn numPartitions(self: CompilationContext) u8 { - return self._platform.sharding().num_partitions; - } - - pub fn getShardingAttr(self: CompilationContext, shape: Shape) mlir.Attribute { - const ctx = self.mlirCtx(); - const num_partitions = self.numPartitions(); - // This is big enough, see test below for examples values - var sharding_str_buf: [64]u8 = undefined; - var writer: std.Io.Writer = .fixed(&sharding_str_buf); - writeShardingRepresentation(shape, num_partitions, &writer) catch unreachable; - return mlir.Attribute.string(ctx, writer.buffered()); - } - - fn writeShardingRepresentation(shape: Shape, num_partitions: u8, writer: *std.Io.Writer) std.Io.Writer.Error!void { - const n_sharded: u8 = @popCount(@as(u8, @bitCast(shape._sharding_info))); - if (n_sharded == 0 or num_partitions == 1) { - try writer.writeAll("{replicated}"); - return; - } - try writer.writeAll("{devices=["); - for (0..shape.rank()) |i| { - try writer.print("{d}", .{if (shape._sharding_info[i]) num_partitions else 1}); - if (i < shape.rank() - 1) try writer.writeByte(','); - } - try writer.print("]<=[{d}]}}", .{num_partitions}); - } - - test writeShardingRepresentation { - var attr_buf: [64]u8 = undefined; - const x = Shape.init(.{ 16, 8 }, .f32); - - // By default tensors are replicated. - { - var writer: std.Io.Writer = .fixed(&attr_buf); - try writeShardingRepresentation(x, 4, &writer); - try std.testing.expectEqualStrings("{replicated}", writer.buffered()); - } - // Shard along first axis. - { - var writer: std.Io.Writer = .fixed(&attr_buf); - try writeShardingRepresentation(x.withSharding(.{0}), 4, &writer); - try std.testing.expectEqualStrings("{devices=[4,1]<=[4]}", writer.buffered()); - } - // Also shard along second axis. - { - var writer: std.Io.Writer = .fixed(&attr_buf); - try writeShardingRepresentation(x.withSharding(.{ 0, 1 }), 2, &writer); - try std.testing.expectEqualStrings("{devices=[2,2]<=[2]}", writer.buffered()); - } - } - - fn finalizeAttributeList(allocator_: std.mem.Allocator, mlir_ctx: mlir.Context, attributes: []AttributeList) ![]mlir.Attribute { - const res = try allocator_.alloc(mlir.Attribute, attributes.len); - for (res, attributes) |*r, attr| { - r.* = mlir.DictionaryAttribute.init(mlir_ctx, attr.constSlice()).asAttr(); - } - return res; - } - - /// Generates an MLIR `func.call` of the given function. - /// If the function has not been seen yet, we generate MLIR for it, - /// in a independent function. - /// The main benefit of this is to generate MLIR that maps more closely - /// to the Zig code, but compilation speed stays similar. - pub fn callFunc( - self: *CompilationContext, - func_name: [:0]const u8, - comptime func: anytype, - args: stdx.meta.FnArgs(func), - ) error{OutOfMemory}!stdx.meta.FnResult(func) { - var arena_state = std.heap.ArenaAllocator.init(self._arena.child_allocator); - defer arena_state.deinit(); - // This arena is used for allocations which won't outlive the function call, - // but the function creation uses `self.allocator()` which we'll live for the duration of the compilation. - const arena = arena_state.allocator(); - - // first, do the "compile" and check the bytecode - // the result of this will also have the correct tags of the result shapes - const args_hash = hashArgs(args); - const key: FnKey = .{ .fn_ptr = &func, .input_hash = args_hash }; - - const function = self._fn_cache.get(key) orelse b: { - const full_name: [:0]const u8 = if (std.mem.eql(u8, "main", func_name)) - try self.allocator().dupeZ(u8, func_name) - else - try std.fmt.allocPrintSentinel(self.allocator(), "{s}_{x}", .{ func_name, key.input_hash }, 0); - - var arg_id: u16 = 0; - var tensor_args: @TypeOf(args) = args; - try meta.mapAlloc(struct { - fn cb(arg_id_: *u16, x: Tensor) Tensor { - const a = arg_id_.*; - arg_id_.* += 1; - return Tensor{ ._shape = x._shape, ._id = .{ .arg_id = a }, ._donation = .{ .arg = a } }; - } - }.cb, arena, &arg_id, args, &tensor_args); - - const f = try self.emitMlir( - func, - &tensor_args, - .{ .name = full_name }, - ); - self._module.getBody().appendOperation(f.mlir_fn); - - try self._fn_cache.putNoClobber(self.allocator(), key, f); - break :b f; - }; - - const loc = self.mlirCtx().location(@src()); - - const num_args = function.args_shapes.len; - const values = try arena.alloc(mlir.Value, num_args); - self.extractValues(&args, values); - - const donations = try arena.alloc(Tensor._Donation, num_args); - meta.collectBuf(struct { - pub fn cb(ctx: *const CompilationContext, x: Tensor) Tensor._Donation { - return ctx.getValueAndDonation(x)[1]; - } - }.cb, self, &args, donations); - - const op = dialect.func.call(self.mlirCtx(), @ptrCast(function.name), values, function.res_types, loc); - // Create the result tensor object by combining the operand results, - // as well as the registered shapes and donations. - // Note: this assume res can be stack-allocated. - var res = @as(*const stdx.meta.FnResult(func), @ptrCast(@alignCast(function.res_tensors))).*; - const LocalContext = struct { index: usize = 0, op: mlir.Operation, function: MlirFn, donations: []Tensor._Donation }; - var context: LocalContext = .{ .op = op, .function = function, .donations = donations }; - meta.visit((struct { - fn cb(ctx: *LocalContext, tensor: *Tensor) void { - const i = ctx.index; - ctx.index += 1; - var new = Tensor.fromMlirValue(ctx.op.result(i)); - new._shape = ctx.function.res_shapes[i]; - new._donation = switch (ctx.function.res_donations[i]) { - .no_buffer => .no_buffer, - .arg => |input_arg| ctx.donations[input_arg], - .input_buffer => .no_buffer, // user escaped the sandbox - }; - tensor.* = new; - } - }).cb, &context, &res); - std.debug.assert(context.index == op.numResults()); - return res; - } - - /// Visit the given struct and recursively associate the `block` arguments with the `value` field of each encountered Tensor. - /// - /// This is done so that we have a mapping between the arguments of the kernel associated with a module and the actual Tensors - /// stored in the Module. - /// Caller need to allocate required memory in self._block_args. - pub fn mapBlockArguments(self: *CompilationContext, v: anytype, block: mlir.Block, start: usize) usize { - const LocalContext = struct { - index: usize, - block: mlir.Block, - self: *CompilationContext, - }; - var context = LocalContext{ .self = self, .block = block, .index = start }; - meta.visit((struct { - fn cb(ctx: *LocalContext, tensor: *const Tensor) void { - const arg_value = ctx.block.argument(ctx.index); - // log.debug("mapping {} to arg {}", .{ tensor._id, ctx.index }); - - const res = ctx.self._block_args.getOrPutAssumeCapacity(tensor._id); - if (res.found_existing) { - stdx.debug.panic("Failed compilation because received two tensors arguments with the same ID: {f} and {f} at index {} ({}).", .{ res.value_ptr.*[0], tensor, ctx.index, tensor._id }); - } else { - res.value_ptr.* = .{ arg_value, .{ .arg = @intCast(ctx.index) } }; - } - ctx.index += 1; - } - }).cb, &context, v); - return context.index; - } - - /// Create tensor from the given shapes. - /// Each created tensor will receive a unique id, local to this CompilationContext. - pub fn tensorFromShapes(self: *CompilationContext, ArgsT: type, allocator_: std.mem.Allocator, args_shapes: anytype) !ArgsT { - const Local = struct { - fn tensorFromShape(arg_id: *u64, shape: Shape) Tensor { - defer arg_id.* += 1; - return Tensor{ - ._shape = shape, - ._id = .{ .arg_id = arg_id.* }, - ._donation = .input_buffer, - }; - } - }; - var tensor_args: ArgsT = undefined; - try meta.mapAlloc(Local.tensorFromShape, allocator_, &self._unique_id, args_shapes, &tensor_args); - return tensor_args; - } - - /// Visit the given struct and extract the mlir.Value and mlir.Type associated with each tensor found. - pub fn extractValuesAndTypes( - self: *const CompilationContext, - v: anytype, - values: []mlir.Value, - types: []mlir.Type, - shapes: []Shape, - donations: []Tensor._Donation, - output_memory_kind: []Buffer.Memory, - ) void { - std.debug.assert(values.len == types.len); - const LocalContext = struct { - self: *const CompilationContext, - index: usize = 0, - values: []mlir.Value, - types: []mlir.Type, - shapes: []Shape, - donations: []Tensor._Donation, - output_memory_kind: []Buffer.Memory, - }; - var context = LocalContext{ - .self = self, - .values = values, - .types = types, - .shapes = shapes, - .donations = donations, - .output_memory_kind = output_memory_kind, - }; - meta.visit((struct { - fn cb(ctx: *LocalContext, tensor: *const Tensor) void { - const value, const donation = ctx.self.getValueAndDonation(tensor.*); - ctx.values[ctx.index] = value; - ctx.types[ctx.index] = value.getType(); - ctx.shapes[ctx.index] = tensor._shape; - ctx.donations[ctx.index] = donation; - ctx.output_memory_kind[ctx.index] = tensor._output_memory_kind; - ctx.index += 1; - } - }).cb, &context, v); - std.debug.assert(context.index == values.len); - } - - pub fn getValueAndDonation(self: *const CompilationContext, tensor: Tensor) struct { mlir.Value, Tensor._Donation } { - return switch (tensor._id) { - .buffer_id, .arg_id => if (self._block_args.get(tensor._id)) |res| - .{ res[0], res[1] } - else { - log.err("Found unknown tensor id {f}({})", .{ tensor, tensor._id }); - @panic("Found unknown tensor id"); - }, - .mlir => |v| .{ v, tensor._donation }, - }; - } - - pub fn getValue(self: *const CompilationContext, tensor: Tensor) mlir.Value { - return self.getValueAndDonation(tensor)[0]; - } - - pub fn extractValues(self: *const CompilationContext, v: anytype, values: []mlir.Value) void { - meta.collectBuf(getValue, self, v, values); + pub fn deinit(self: OutputInfo, allocator: std.mem.Allocator) void { + allocator.free(self.shapes); + allocator.free(self.values); + allocator.free(self.donations); + allocator.free(self.output_memory_kinds); } }; -fn computeModuleHash(platform: Platform, module: mlir.Module) u64 { - var hasher = std.hash.XxHash64.init(0); - module.hash(&hasher); +fn collectOutputInfo(allocator: std.mem.Allocator, v: anytype) !OutputInfo { + const LocalContext = struct { + shape_list: *std.array_list.Managed(Shape), + value_list: *std.array_list.Managed(*const mlir.Value), + donation_list: *std.array_list.Managed(?usize), + output_memory_kind_list: *std.array_list.Managed(Buffer.Memory), + }; - hasher.update(platform.pjrt_client.getPlatformName(platform.pjrt_api)); - const api_version = platform.pjrt_api.version(); - hasher.update(std.mem.sliceAsBytes(&[_]i64{ api_version.major, api_version.minor })); + var shape_list = std.array_list.Managed(Shape).init(allocator); + errdefer shape_list.deinit(); + var value_list = std.array_list.Managed(*const mlir.Value).init(allocator); + errdefer value_list.deinit(); + var donation_list = std.array_list.Managed(?usize).init(allocator); + errdefer donation_list.deinit(); + var output_memory_kind_list = std.array_list.Managed(Buffer.Memory).init(allocator); + errdefer output_memory_kind_list.deinit(); - return hasher.final(); + var context: LocalContext = .{ + .shape_list = &shape_list, + .value_list = &value_list, + .donation_list = &donation_list, + .output_memory_kind_list = &output_memory_kind_list, + }; + + try meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { + try ctx_.shape_list.append(tensor.shape()); + try ctx_.value_list.append(tensor.value()); + try ctx_.donation_list.append(tensor.donation()); + try ctx_.output_memory_kind_list.append(tensor.outputMemoryKind()); + } + }.cb, &context, v); + + return .{ + .shapes = try shape_list.toOwnedSlice(), + .values = try value_list.toOwnedSlice(), + .donations = try donation_list.toOwnedSlice(), + .output_memory_kinds = try output_memory_kind_list.toOwnedSlice(), + }; } -const max_pjrt_executable_size = 400 * 1024 * 1024; +pub const InputInfo = struct { + shapes: []Shape, -fn loadPjrtExecutable(arena: std.mem.Allocator, platform: Platform, absolute_file: [:0]const u8) !*pjrt.LoadedExecutable { - const tracer = Tracer.init("ai.zml.load_exe"); - const compile_frame = tracer.frameStart("pjrt load executable"); - defer tracer.frameEnd(compile_frame, "pjrt load executable"); + pub fn deinit(self: InputInfo, allocator: std.mem.Allocator) void { + allocator.free(self.shapes); + } +}; - const loaded_executable_file = try std.fs.openFileAbsoluteZ(absolute_file, .{}); - defer loaded_executable_file.close(); +fn collectInputInfo(allocator: std.mem.Allocator, v: anytype) !InputInfo { + const LocalContext = struct { + shape_list: *std.array_list.Managed(Shape), + }; - const exe_size = if (loaded_executable_file.stat()) |stat| stat.size else |_| max_pjrt_executable_size; - const bytes = try arena.alloc(u8, exe_size); - defer arena.free(bytes); + var shape_list = std.array_list.Managed(Shape).init(allocator); + errdefer shape_list.deinit(); - const size = try loaded_executable_file.readAll(bytes); - return try platform.pjrt_client.deserializeAndLoad(platform.pjrt_api, bytes[0..size]); + var context: LocalContext = .{ .shape_list = &shape_list }; + + try meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { + try ctx_.shape_list.append(tensor.shape()); + } + }.cb, &context, v); + + return .{ + .shapes = try shape_list.toOwnedSlice(), + }; } -fn storePjrtExecutable(platform: Platform, loaded_executable: *pjrt.LoadedExecutable, absolute_file: [:0]const u8) !void { - const loaded_executable_file = try std.fs.createFileAbsoluteZ(absolute_file, .{}); - defer loaded_executable_file.close(); +const EmitMlirResult = struct { + func: *mlir.Operation, + input_info: InputInfo, + output_info: OutputInfo, +}; - var executable = try loaded_executable.getExecutable(platform.pjrt_api); - defer executable.deinit(platform.pjrt_api); +fn finalizeAttributeList(allocator_: std.mem.Allocator, mlir_ctx: *mlir.Context, attributes: []AttributeList) ![]*const mlir.Attribute { + const res = try allocator_.alloc(*const mlir.Attribute, attributes.len); + for (res, attributes) |*r, attr| { + r.* = mlir.dictionaryAttribute(mlir_ctx, attr.constSlice()); + } + return res; +} - var serialize_result = try executable.serialize(platform.pjrt_api); - defer serialize_result.deinit(); +fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, args: stdx.meta.FnArgs(func)) !EmitMlirResult { + var arena = std.heap.ArenaAllocator.init(compilation_context.allocator); + defer arena.deinit(); - try loaded_executable_file.writeAll(serialize_result.bytes); + const module = mlir.Module.init(.unknown(compilation_context.mlir_ctx)); + errdefer module.deinit(); + + const block = mlir.Block.init(&.{}, &.{}); + errdefer block.deinit(); + + compilation_context.pushBlock(block); + defer compilation_context.popBlock(); + + const LocalContext = struct { + compilation_context: *CompilationContext, + current_argument_id: usize = 0, + }; + var context: LocalContext = .{ + .compilation_context = compilation_context, + }; + meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { + const mlir_type = mlir.rankedTensorType( + tensor.dims(), + mlirx.Type.fromDType(ctx_.compilation_context.mlir_ctx, tensor.dtype()), + ); + _ = ctx_.compilation_context.currentScope().block.addArgument(mlir_type, .unknown(ctx_.compilation_context.mlir_ctx)); + ctx_.compilation_context.currentScope().id_to_argument.put(ctx_.compilation_context.currentScope().arena.allocator(), tensor.id, ctx_.current_argument_id) catch unreachable; + ctx_.current_argument_id += 1; + } + }.cb, &context, &args); + + const input_info = try collectInputInfo(compilation_context.allocator, &args); + errdefer input_info.deinit(compilation_context.allocator); + + const input_attributes = try arena.allocator().alloc(AttributeList, input_info.shapes.len); + @memset(input_attributes, .{}); + + const output_info = b: { + compilation_context.activate(); + defer compilation_context.deactivate(); + + const result = @call(.auto, func, args); + + const output_info = try collectOutputInfo(compilation_context.allocator, &result); + errdefer output_info.deinit(compilation_context.allocator); + + break :b output_info; + }; + errdefer output_info.deinit(compilation_context.allocator); + + const output_attributes = try arena.allocator().alloc(AttributeList, output_info.shapes.len); + @memset(output_attributes, .{}); + + for (output_info.donations, 0..) |donation, index| if (donation) |argument_index| { + input_attributes[argument_index].appendAssumeCapacity(.named(compilation_context.mlir_ctx, "tf.aliasing_output", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, index))); + }; + for (output_info.output_memory_kinds, 0..) |output_memory_kind, index| { + if (output_memory_kind == .device) continue; + output_attributes[index].appendAssumeCapacity(.named(compilation_context.mlir_ctx, "mhlo.memory_kind", mlir.stringAttribute(compilation_context.mlir_ctx, output_memory_kind.pjrtName()))); + } + _ = dialects.func.returns(compilation_context.mlir_ctx, output_info.values, .unknown(compilation_context.mlir_ctx)).appendTo(compilation_context.currentScope().block); + + const mlir_func = dialects.func.func(compilation_context.mlir_ctx, .{ + .name = "main", + .block = compilation_context.currentScope().block, + .location = .unknown(compilation_context.mlir_ctx), + .args_attributes = try finalizeAttributeList(arena.allocator(), compilation_context.mlir_ctx, input_attributes), + .results_attributes = try finalizeAttributeList(arena.allocator(), compilation_context.mlir_ctx, output_attributes), + }); + + compilation_context.mlir_pass_manager.runOnOp(mlir_func) catch |err| switch (err) { + error.MlirUnexpected => { + std.log.err("Failed to canonicalize invalid mlir: {f}", .{mlir_func}); + // user errors should have triggered a panic before we reach this. + @panic("ZML generated invalid mlir. Please open a bug report"); + }, + }; + + return .{ + .func = mlir_func, + .input_info = input_info, + .output_info = output_info, + }; } fn setXlaOverrideFlag(map: *c.upb_Map, flag: []const u8, value: anytype, upb_arena: *c.upb_Arena) !void { @@ -867,10 +418,10 @@ fn setXlaOverrideFlag(map: *c.upb_Map, flag: []const u8, value: anytype, upb_are } } -fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, module: mlir.Module, xla_dump_to_: ?[]const u8) !*pjrt.LoadedExecutable { - const tracer = Tracer.init("ai.zml.compilation"); - const compile_frame = tracer.frameStart("pjrt compilation"); - defer tracer.frameEnd(compile_frame, "pjrt compilation"); +fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: Platform, module: *const mlir.Module, xla_dump_to_: ?[]const u8) !*pjrt.LoadedExecutable { + //const tracer = Tracer.init("ai.zml.compilation"); + //const compile_frame = tracer.frameStart("pjrt compilation"); + //defer tracer.frameEnd(compile_frame, "pjrt compilation"); const sharding = platform.sharding(); @@ -887,9 +438,6 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m c.xla_ExecutableBuildOptionsProto_set_num_replicas(exec_build_options, sharding.num_replicas); c.xla_ExecutableBuildOptionsProto_set_num_partitions(exec_build_options, sharding.num_partitions); c.xla_ExecutableBuildOptionsProto_set_use_spmd_partitioning(exec_build_options, sharding.num_partitions > 1 or sharding.num_replicas > 1); - if (platform.compilation_options.device_memory_size > 0) { - c.xla_ExecutableBuildOptionsProto_set_device_memory_size(exec_build_options, @intCast(platform.compilation_options.device_memory_size)); - } c.xla_ExecutableBuildOptionsProto_set_device_assignment(exec_build_options, device_assignment_blk: { const device_assignment = try upb.new(c.xla_DeviceAssignmentProto, upb_arena); @@ -904,6 +452,7 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m } break :device_assignment_blk device_assignment; }); + break :executable_build_options_blk exec_build_options; }); @@ -944,6 +493,7 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m const loaded_executable = try platform.pjrt_client.compile( platform.pjrt_api, arena, + io, module, try upb.serialize(options, upb_arena), ); @@ -951,321 +501,3 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m return loaded_executable; } - -/// Visit the given struct and recursively counts the number of tensors found. -pub fn countTensors(v: anytype) usize { - const LocalContext = struct { - count: usize = 0, - }; - var context = LocalContext{}; - meta.visit((struct { - fn cb(inner_context: *LocalContext, _: *const Tensor) void { - inner_context.count += 1; - } - }).cb, &context, v); - return context.count; -} - -/// Visit the given struct and recursively fill the `types` slice with the mlir.Type associated with encountered Tensor. -pub fn fillMlirTypes(v: anytype, mlir_ctx: mlir.Context, types: []mlir.Type) void { - const LocalContext = struct { - index: usize = 0, - mlir_ctx: mlir.Context, - types: []mlir.Type, - }; - var context = LocalContext{ .mlir_ctx = mlir_ctx, .types = types }; - meta.visit((struct { - fn cb(inner_context: *LocalContext, tensor: *const Tensor) void { - inner_context.types[inner_context.index] = mlirx.tensorType(inner_context.mlir_ctx, tensor.shape()); - inner_context.index += 1; - } - }).cb, &context, v); - std.debug.assert(context.index == types.len); -} - -/// Visit the given struct and recursively associate the `block` arguments with the `value` field of each encountered Tensor. -/// -/// This is done so that we have a mapping between the arguments of the kernel associated with a module and the actual Tensors -/// stored in the Module. -fn assignBlockArguments(v: anytype, block: mlir.Block, start: usize) usize { - const LocalContext = struct { index: usize, block: mlir.Block }; - var context = LocalContext{ .block = block, .index = start }; - meta.visit((struct { - fn cb(ctx: *LocalContext, tensor: *Tensor) void { - tensor._id = .{ .mlir = ctx.block.argument(ctx.index) }; - tensor._donation = .{ .arg = @intCast(ctx.index) }; - ctx.index += 1; - } - }).cb, &context, v); - return context.index; -} - -pub const FnCache = std.AutoHashMapUnmanaged(FnKey, MlirFn); -pub const FnKey = struct { fn_ptr: *const anyopaque, input_hash: u64 }; - -test FnCache { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const Layer = struct { - const Layer_ = @This(); - - w: Tensor, - b: Tensor, - - pub fn _fwd(self: Layer_, x: Tensor) Tensor { - const wx = self.w.dotGeneral(x, &.{.{ -1, 0 }}, &.{}); - return wx.add(self.b.broad(wx.shape())).relu(); - } - }; - - const NN = struct { - const NN_ = @This(); - layers: [3]Layer, - - pub fn _fwd(self: NN_, x0: Tensor) Tensor { - var x = x0; - for (self.layers) |layer| { - x = ops.call(layer, ._fwd, .{x}); - } - return x; - } - - pub fn _forwardRefImpl(self: NN_, x0: Tensor) Tensor { - var x = x0; - for (self.layers) |layer| { - x = layer._fwd(x); - } - return x; - } - }; - - const x = try zml.Buffer.fromArray(platform, [2]f16{ -1, 1 }); - const nn: zml.testing.BufferizedWithArgs(NN) = .{ - .layers = .{ - .{ - .w = try .fromArray(platform, [2][2]f16{ .{ 1, -1 }, .{ 0, 1 } }), - .b = try .fromArray(platform, [2]f16{ 0, 0 }), - }, - .{ - .w = try .fromArray(platform, [2][2]f16{ .{ 1, 2 }, .{ 1, -1 } }), - .b = try .fromArray(platform, [2]f16{ 10, 10 }), - }, - // third layer is different - .{ - .w = try .fromArray(platform, [3][2]f16{ .{ 1, 2 }, .{ 0, 1 }, .{ -1, 0 } }), - .b = try .fromArray(platform, [3]f16{ -10, -10, -10 }), - }, - }, - }; - const res = try zml.testing.compileAndCall(platform, NN._fwd, .{ nn, x }); - const expected = try zml.testing.compileAndCall(platform, NN._forwardRefImpl, .{ nn, x }); - try zml.testing.expectClose(expected, res, 1e-4); -} - -test "FnCache with mixed integer/tensor" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const Layer = struct { - const Layer_ = @This(); - var num_call: u32 = 0; - - w: Tensor, - - pub fn _fwd(self: Layer_, x: Tensor) struct { Tensor, usize } { - const wx = self.w.dotGeneral(x, &.{.{ -1, 0 }}, &.{}); - // Note: this is for testing only, it's a bad idea to mutate global state - // from a forward function because it can mess with caching. - num_call += 1; - return .{ wx.addConstant(num_call), num_call }; - } - }; - - const NN = struct { - const NN_ = @This(); - layers: [3]Layer, - - pub fn _fwd(self: NN_, x0: Tensor) Tensor { - var x = x0; - var y: usize = 0; - x, y = ops.call(self.layers[0], ._fwd, .{x}); - std.debug.assert(Layer.num_call == 1); - std.debug.assert(y == 1); - // Here we call a second time but since first two layers have the same shape, - // We hit the function cache, and "num_call" is not incremented. - x, y = ops.call(self.layers[1], ._fwd, .{x}); - std.debug.assert(Layer.num_call == 1); - std.debug.assert(y == 1); - x, y = ops.call(self.layers[2], ._fwd, .{x}); - std.debug.assert(Layer.num_call == 2); - std.debug.assert(y == 2); - return x; - } - - pub fn _forwardRefImpl(self: NN_, x0: Tensor) Tensor { - var x = x0; - for (self.layers, &[_]u32{ 1, 1, 2 }) |layer, bias| { - const wx = layer.w.dotGeneral(x, &.{.{ -1, 0 }}, &.{}); - x = wx.addConstant(bias); - } - return x; - } - }; - - const x = try zml.Buffer.fromArray(platform, [2]f16{ -1, 1 }); - const nn: zml.testing.BufferizedWithArgs(NN) = .{ - .layers = .{ - .{ .w = try .fromArray(platform, [2][2]f16{ .{ 1, -1 }, .{ 0, 1 } }) }, - .{ .w = try .fromArray(platform, [2][2]f16{ .{ 1, 2 }, .{ 1, -1 } }) }, - // third layer has different shape - .{ .w = try .fromArray(platform, [3][2]f16{ .{ 1, 2 }, .{ 0, 1 }, .{ -1, 0 } }) }, - }, - }; - const res = try zml.testing.compileAndCall(platform, NN._fwd, .{ nn, x }); - const expected = try zml.testing.compileAndCall(platform, NN._forwardRefImpl, .{ nn, x }); - try zml.testing.expectClose(expected, res, 1e-4); -} - -pub fn hashArgs(mod: anytype) u64 { - var hasher = std.hash.Wyhash.init(0); - hash(&hasher, mod, .DeepRecursive); - return hasher.final(); -} - -pub fn hashShape(hasher: *std.hash.Wyhash, shape: Shape) void { - // Note: if we enforced 0-init dims then we could hash dims instead. - hashArray(hasher, shape.dims(), .Shallow); - hash(hasher, shape._dtype, .Shallow); - hash(hasher, shape._sharding_info, .Shallow); - for (shape.tags()) |tag| { - hashArray(hasher, std.mem.span(tag), .Shallow); - } -} - -const tensorAwareHash = hash; // alias for when "hash" is ambiguous - -/// Provides generic hashing for any eligible type. -/// Strategy is provided to determine if pointers should be followed or not. -pub fn hash(hasher: *std.hash.Wyhash, key: anytype, comptime strat: std.hash.Strategy) void { - const Key = @TypeOf(key); - if (Key == Tensor) return hashShape(hasher, key.shape()); - if (Key == Shape) return hashShape(hasher, key); - - if (strat == .Shallow and std.meta.hasUniqueRepresentation(Key)) { - hasher.update(std.mem.asBytes(&key)); - return; - } - - switch (@typeInfo(Key)) { - .noreturn, .@"opaque", .undefined, .null, .comptime_float, .comptime_int, .type, .enum_literal, .frame, .void => return, - - // Help the optimizer see that hashing an int is easy by inlining! - // TODO Check if the situation is better after #561 is resolved. - .int => |int| switch (int.signedness) { - .signed => hash(hasher, @as(@Type(.{ .int = .{ - .bits = int.bits, - .signedness = .unsigned, - } }), @bitCast(key)), strat), - .unsigned => { - if (std.meta.hasUniqueRepresentation(Key)) { - hasher.update(std.mem.asBytes(&key)); - } else { - // Take only the part containing the key value, the remaining - // bytes are undefined and must not be hashed! - const byte_size = comptime std.math.divCeil(comptime_int, @bitSizeOf(Key), 8) catch unreachable; - hasher.update(std.mem.asBytes(&key)[0..byte_size]); - } - }, - }, - // Note: contrary to Zig we accept hashing floats. - // Typically the float we are going to hash here are hyperparameters, - // and not the result of an operation, so bytes should be the same everytime. - .float => hasher.update(std.mem.asBytes(&key)), - .bool => hash(hasher, @intFromBool(key), strat), - .@"enum" => hash(hasher, @intFromEnum(key), strat), - .error_set => hash(hasher, @intFromError(key), strat), - .@"anyframe", .@"fn" => hash(hasher, @intFromPtr(key), strat), - .pointer => |info| switch (info.size) { - .one => switch (strat) { - .Shallow => hash(hasher, @intFromPtr(key), .Shallow), - .Deep => hash(hasher, key.*, .Shallow), - .DeepRecursive => switch (@typeInfo(info.child)) { - .@"opaque", .@"fn" => hash(hasher, @intFromPtr(key), .Shallow), - else => hash(hasher, key.*, .DeepRecursive), - }, - }, - .slice => { - switch (strat) { - .Shallow => hash(hasher, @intFromPtr(key.ptr), .Shallow), - .Deep => hashArray(hasher, key, .Shallow), - .DeepRecursive => hashArray(hasher, key, .DeepRecursive), - } - hash(hasher, key.len, .Shallow); - }, - .many, - .c, - => switch (strat) { - .Shallow => hash(hasher, @intFromPtr(key), .Shallow), - else => @compileError( - \\ unknown-length pointers and C pointers cannot be hashed deeply. - \\ Consider providing your own hash function. - ), - }, - }, - .optional => if (key) |k| hash(hasher, k, strat), - - .array => hashArray(hasher, key, strat), - - .vector => |info| { - if (std.meta.hasUniqueRepresentation(Key)) { - hasher.update(std.mem.asBytes(&key)); - } else { - comptime var i = 0; - inline while (i < info.len) : (i += 1) { - hash(hasher, key[i], strat); - } - } - }, - - .@"struct" => |info| { - inline for (info.fields) |field| { - // We reuse the hash of the previous field as the seed for the - // next one so that they're dependant. - hash(hasher, @field(key, field.name), strat); - } - }, - - .@"union" => |info| { - if (info.tag_type) |tag_type| { - const tag = std.meta.activeTag(key); - hash(hasher, tag, strat); - inline for (info.fields) |field| { - if (@field(tag_type, field.name) == tag) { - if (field.type != void) { - hash(hasher, @field(key, field.name), strat); - } - // TODO use a labelled break when it does not crash the compiler. cf #2908 - // break :blk; - return; - } - } - unreachable; - } else @compileError("cannot hash untagged union type: " ++ @typeName(Key) ++ ", provide your own hash function"); - }, - - .error_union => blk: { - const payload = key catch |err| { - hash(hasher, err, strat); - break :blk; - }; - hash(hasher, payload, strat); - }, - } -} - -fn hashArray(hasher: anytype, key: anytype, comptime strat: std.hash.Strategy) void { - for (key) |element| { - hash(hasher, element, strat); - } -} diff --git a/zml/nn.zig b/zml/nn.zig index 8bfd2f2d..d244f235 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -1,38 +1,36 @@ //! Common layer definition and functions for Neural Networks (NN) const std = @import("std"); -const assert = std.debug.assert; -const testing = std.testing; const stdx = @import("stdx"); +const ConstSlice = @import("slice.zig").ConstSlice; const DataType = @import("dtype.zig").DataType; -const helpers = @import("helpers.zig"); const meta = @import("meta.zig"); -const cuda = @import("nn/cuda.zig"); const ops = @import("ops.zig"); const Shape = @import("shape.zig").Shape; const Tensor = @import("tensor.zig").Tensor; const zml = @import("zml.zig"); -const log = std.log.scoped(.@"zml/tensor"); -test { - _ = cuda; - std.testing.refAllDecls(@This()); -} +const log = std.log.scoped(.@"zml/nn"); pub const Linear = struct { weight: Tensor, bias: ?Tensor = null, + tag: Shape.Tag, + + pub fn init(weight: Tensor, bias: ?Tensor, tag: anytype) Linear { + return .{ + .weight = weight, + .bias = bias, + .tag = zml.Shape.toTag(tag), + }; + } pub fn forward(self: Linear, x: Tensor) Tensor { - var y = x.dotGeneral(self.weight.convert(x.dtype()), &.{.{ -1, -1 }}, &.{}); - // If self.weight doesn't have tags, preserve tags from x. - if (y.shape().tag(-1) == Shape.TagUnknown) { - y._shape._tags.set(y.rank() - 1, x.shape().tag(-1)); - } + var y = x.dot(self.weight.convert(x.dtype()), self.tag); // log.debug("Linear({*}): {d} -> {d} -> {d}", .{ self, x.dims(), y.dims(), if (self.bias) |bias| y.add(bias).dims() else y.dims() }); - return if (self.bias) |bias| y.add(bias.broadcast(y.shape(), &.{y.axis(-1)})) else y; + return if (self.bias) |bias| y.add(bias.broad(y.shape())) else y; } }; @@ -77,20 +75,6 @@ pub fn elu(x: Tensor, alpha: f32) Tensor { ); } -pub fn chainModules(module_list: anytype, input: Tensor) Tensor { - const T = @TypeOf(module_list); - switch (@typeInfo(T)) { - .Struct => |struct_info| { - var x = input; - inline for (struct_info.fields) |field| { - x = @field(module_list, field.name).forward(x); - } - return x; - }, - else => @compileError("chainModules only works on a struct with only containing 'module' struct."), - } -} - /// Layer Normalization pub const LayerNorm = struct { weight: Tensor, @@ -141,11 +125,19 @@ pub fn normalizeL2(input: Tensor, eps: f32) Tensor { test normalizeL2 { const platform = zml.testing.env(); - const input = try zml.Buffer.fromSlice(platform, .{ 2, 2 }, &[_]f32{ -0.9686, -1.0058, -1.7808, 0.6698 }); + const input: zml.Tensor = .init(.{ 2, 2 }, .f32); - const res = try zml.testing.compileAndCall(platform, zml.nn.normalizeL2, .{ input, 1e-12 }); - const expectation = zml.HostBuffer.fromSlice(.{ 2, 2 }, &[_]f32{ -0.6937, -0.7203, -0.9360, 0.3520 }); - try zml.testing.expectClose(expectation, res, 1e-4); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, normalizeL2, .{ input, 1e-12 }, platform); + defer exe.deinit(); + + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.9686, -1.0058, -1.7808, 0.6698 })); + defer input_buffer.deinit(); + + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, normalizeL2, .{input_buffer}); + defer res.deinit(); + + const expectation: ConstSlice = .init(input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6937, -0.7203, -0.9360, 0.3520 })); + try zml.testing.expectClose(std.testing.io, expectation, res, 1e-4); } pub const RopeOpts = struct { @@ -274,14 +266,14 @@ pub fn mergeRealImg(x_real: Tensor, x_imag: Tensor, layout: RopeOpts.Layout) Ten } /// {exp( - n * ln(10_000) / N ) | n in [0..N] } -pub fn invFreq(N: i64, opts: RopeOpts) Tensor { - const allocator = zml.module.CompilationContext.current().allocator(); +fn invFreq(N: i64, opts: RopeOpts) Tensor { + const allocator = zml.module.CompilationContext.current().allocator; const N_half: usize = @intCast(@divExact(N, 2)); const inv_freq = allocator.alloc(f32, N_half) catch @panic("OOM"); defer allocator.free(inv_freq); _invFreq(opts, inv_freq); - return zml.Tensor.constantTensor(.fromSlice(.{@divExact(N, 2)}, inv_freq)); + return zml.Tensor.constantTensor(zml.Shape.init(.{@divExact(N, 2)}, .f32), std.mem.sliceAsBytes(inv_freq)); } fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { @@ -453,30 +445,51 @@ test "real/img" { ); } }; + { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.interleaved}, platform); + defer exe.deinit(); - const d_interleaved = try zml.testing.compileAndCall(platform, Fns.testSplitMergeIsId, .{.interleaved}); - try testing.expectEqual(20, d_interleaved.getValue(i32)); + var d_interleaved = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitMergeIsId, {}); + defer d_interleaved.deinit(); + try std.testing.expectEqual(20, try d_interleaved.getValue(i32, std.testing.io)); + } + { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.sequential}, platform); + defer exe.deinit(); - const d_sequential = try zml.testing.compileAndCall(platform, Fns.testSplitMergeIsId, .{.sequential}); - try testing.expectEqual(20, d_sequential.getValue(i32)); + var d_sequential = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitMergeIsId, {}); + defer d_sequential.deinit(); + try std.testing.expectEqual(20, try d_sequential.getValue(i32, std.testing.io)); + } // test the function that accepts 1 void argument - const d_split_seq_void = try zml.testing.compileAndCall(platform, Fns.testSplitSeqVoid, .{{}}); - try testing.expectEqual(20, d_split_seq_void.getValue(i32)); + { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitSeqVoid, .{{}}, platform); + defer exe.deinit(); + + var d_split_seq_void = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitSeqVoid, {}); + defer d_split_seq_void.deinit(); + try std.testing.expectEqual(20, try d_split_seq_void.getValue(i32, std.testing.io)); + } // test the function that takes NO arguments - const d_split_seq = try zml.testing.compileAndCall(platform, Fns.testSplitSeq, .{}); - try testing.expectEqual(20, d_split_seq.getValue(i32)); - - // now try compiling and calling ourselves { - const mod = try zml.compileFn(std.testing.allocator, Fns.testSplitSeq, .{}, platform); - defer mod.deinit(); - const ret = mod.call({}); - try testing.expectEqual(20, ret.getValue(i32)); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitSeq, .{}, platform); + defer exe.deinit(); + + var d_split_seq = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitSeq, {}); + defer d_split_seq.deinit(); + try std.testing.expectEqual(20, try d_split_seq.getValue(i32, std.testing.io)); + } + + { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitInterleaved, .{}, platform); + defer exe.deinit(); + + var d_split_seq = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitInterleaved, {}); + defer d_split_seq.deinit(); + try std.testing.expectEqual(20, try d_split_seq.getValue(i32, std.testing.io)); } - const d_split_interleaved = try zml.testing.compileAndCall(platform, Fns.testSplitInterleaved, .{}); - try testing.expectEqual(20, d_split_interleaved.getValue(i32)); } test rope { @@ -503,19 +516,22 @@ test rope { // x is made such as the interleaved and sequential reps are the same. // So the two implementations should give the same results. - const x = try zml.Buffer.fromSlice(platform, .{ .b = 1, .s = 5, .hd = 4 }, &[_]f32{ 1.0, 0.1, -1.0, -0.5 } ** 5); - const res1 = try zml.testing.compileAndCall(platform, Local._fwd, .{ x, RopeOpts{ .layout = .interleaved } }); - const res2 = try zml.testing.compileAndCall(platform, Local._fwd, .{ x, RopeOpts{ .layout = .sequential } }); - try zml.testing.expectClose(res1, res2, 1e-4); -} + const x: zml.Tensor = .init(.{ .b = 1, .s = 5, .hd = 4 }, .f32); + var exe_interleaved = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .interleaved } }, platform); + defer exe_interleaved.deinit(); -/// In neural network we generally care about the relative precision, -/// but on a given dimension, if the output is close to 0, then the precision -/// don't matter as much. -fn approxEq(comptime Float: type, l: Float, r: Float, tolerance: Float) bool { - const closeRel = std.math.approxEqRel(Float, l, r, @floatCast(tolerance)); - const closeAbs = std.math.approxEqAbs(Float, l, r, @floatCast(tolerance / 2)); - return closeRel or closeAbs; + var exe_sequential = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .sequential } }, platform); + defer exe_sequential.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ 1.0, 0.1, -1.0, -0.5 } ** 5)); + defer x_buffer.deinit(); + + var res1 = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_interleaved, Local._fwd, .{x_buffer}); + defer res1.deinit(); + var res2 = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_sequential, Local._fwd, .{x_buffer}); + defer res2.deinit(); + + try zml.testing.expectClose(std.testing.io, res1, res2, 1e-4); } pub const UpsampleMode = enum { @@ -569,7 +585,7 @@ pub fn nearest(input: Tensor, scale_factor: []const f64) Tensor { const n = out_shape.dim(ax); const ratio = stdx.math.divFloat(f32, input.dim(ax), n); const offsets = Tensor.arange(.{ .end = n }, .f32).addConstant(0.5).scale(ratio).floor().convert(.i32); - res = res.gather_(&.{ax}, &.{offsets}, .{ .indices_are_sorted = true }); + res = ops.gather(res, &.{ax}, &.{offsets}, .{ .indices_are_sorted = true }); } return res; } @@ -579,21 +595,38 @@ test nearest { // 3D Tensor (basic) { - const input_3d_basic = try zml.Buffer.fromArray(platform, [1][1][2]i32{.{.{ 1, 2 }}}); - const result = try zml.testing.compileAndCall(platform, upsample, .{ input_3d_basic, .{ .scale_factor = &.{3}, .mode = .nearest } }); - try std.testing.expectEqualSlices(i64, &.{ 1, 1, 6 }, result.dims()); - const expected: [1][1][6]i32 = .{.{.{ 1, 1, 1, 2, 2, 2 }}}; - try zml.testing.expectClose(zml.HostBuffer.fromArray(&expected), result, 0); + const input_3d_basic: zml.Tensor = .init(.{ 1, 1, 2 }, .i32); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_3d_basic, .{ .scale_factor = &.{3}, .mode = .nearest } }, platform); + defer exe.deinit(); + + var input_3d_basic_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_3d_basic.shape(), std.mem.sliceAsBytes(&[1][1][2]i32{.{.{ 1, 2 }}})); + defer input_3d_basic_buffer.deinit(); + + const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_3d_basic_buffer}); + defer result.deinit(); + + try std.testing.expectEqualSlices(i64, &.{ 1, 1, 6 }, result.shape().dims()); + const expected: ConstSlice = .init(Shape.init(.{ 1, 1, 6 }, .i32), std.mem.sliceAsBytes(&[1][1][6]i32{.{.{ 1, 1, 1, 2, 2, 2 }}})); + try zml.testing.expectClose(std.testing.io, expected, result, 0); } + // 3D Tensor (advanced) { - const input_3d_advanced = try zml.Buffer.fromArray(platform, [2][3][4]i32{ + const input_3d_advanced: zml.Tensor = .init(.{ 2, 3, 4 }, .i32); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_3d_advanced, .{ .scale_factor = &.{2}, .mode = .nearest } }, platform); + defer exe.deinit(); + + var input_3d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_3d_advanced.shape(), std.mem.sliceAsBytes(&[2][3][4]i32{ .{ .{ 1, 2, 3, 4 }, .{ 5, 6, 7, 8 }, .{ 9, 10, 11, 12 } }, .{ .{ 13, 14, 15, 16 }, .{ 17, 18, 19, 20 }, .{ 21, 22, 23, 24 } }, - }); - const result = try zml.testing.compileAndCall(platform, upsample, .{ input_3d_advanced, .{ .scale_factor = &.{2}, .mode = .nearest } }); - try std.testing.expectEqualSlices(i64, &.{ 2, 3, 8 }, result.dims()); - const expected: [2][3][8]i32 = .{ + })); + defer input_3d_advanced_buffer.deinit(); + + const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_3d_advanced_buffer}); + defer result.deinit(); + + try std.testing.expectEqualSlices(i64, &.{ 2, 3, 8 }, result.shape().dims()); + const expected: ConstSlice = .init(Shape.init(.{ 2, 3, 8 }, .i32), std.mem.sliceAsBytes(&[2][3][8]i32{ .{ .{ 1, 1, 2, 2, 3, 3, 4, 4 }, .{ 5, 5, 6, 6, 7, 7, 8, 8 }, @@ -604,36 +637,53 @@ test nearest { .{ 17, 17, 18, 18, 19, 19, 20, 20 }, .{ 21, 21, 22, 22, 23, 23, 24, 24 }, }, - }; - try zml.testing.expectClose(zml.HostBuffer.fromArray(&expected), result, 0); + })); + try zml.testing.expectClose(std.testing.io, expected, result, 0); } + // 4D Tensor (basic) { - const input_4d_basic = try zml.Buffer.fromSlice(platform, .{ 1, 1, 2, 2 }, &[_]i32{ 1, 2, 3, 4 }); - const result = try zml.testing.compileAndCall(platform, upsample, .{ input_4d_basic, .{ .scale_factor = &.{ 3, 3 }, .mode = .nearest } }); - try std.testing.expectEqualSlices(i64, &.{ 1, 1, 6, 6 }, result.dims()); - const expected: [1][1][6][6]i32 = .{.{.{ + const input_4d_basic: zml.Tensor = .init(.{ 1, 1, 2, 2 }, .i32); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_4d_basic, .{ .scale_factor = &.{ 3, 3 }, .mode = .nearest } }, platform); + defer exe.deinit(); + + var input_4d_basic_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_basic.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); + defer input_4d_basic_buffer.deinit(); + + const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_4d_basic_buffer}); + defer result.deinit(); + + try std.testing.expectEqualSlices(i64, &.{ 1, 1, 6, 6 }, result.shape().dims()); + const expected: ConstSlice = .init(Shape.init(.{ 1, 1, 6, 6 }, .i32), std.mem.sliceAsBytes(&[1][1][6][6]i32{.{.{ .{ 1, 1, 1, 2, 2, 2 }, .{ 1, 1, 1, 2, 2, 2 }, .{ 1, 1, 1, 2, 2, 2 }, .{ 3, 3, 3, 4, 4, 4 }, .{ 3, 3, 3, 4, 4, 4 }, .{ 3, 3, 3, 4, 4, 4 }, - }}}; - try std.testing.expectEqual(expected, result.getValue([1][1][6][6]i32)); + }}})); + try zml.testing.expectClose(std.testing.io, expected, result, 0); } // 4D Tensor (advanced) { - const input_4d_advanced = try zml.Buffer.fromArray(platform, [2][2][2][2]i32{ .{ + const input_4d_advanced: zml.Tensor = .init(.{ 2, 2, 2, 2 }, .i32); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_4d_advanced, .{ .scale_factor = &.{ 2, 2 }, .mode = .nearest } }, platform); + defer exe.deinit(); + + var input_4d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_advanced.shape(), std.mem.sliceAsBytes(&[2][2][2][2]i32{ .{ .{ .{ 1, 2 }, .{ 3, 4 } }, .{ .{ 5, 6 }, .{ 7, 8 } }, }, .{ .{ .{ 9, 10 }, .{ 11, 12 } }, .{ .{ 13, 14 }, .{ 15, 16 } }, - } }); - const result = try zml.testing.compileAndCall(platform, upsample, .{ input_4d_advanced, .{ .scale_factor = &.{ 2, 2 }, .mode = .nearest } }); - try std.testing.expectEqualSlices(i64, &.{ 2, 2, 4, 4 }, result.dims()); - const expected: [2][2][4][4]i32 = .{ + } })); + defer input_4d_advanced_buffer.deinit(); + + const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_4d_advanced_buffer}); + defer result.deinit(); + + try std.testing.expectEqualSlices(i64, &.{ 2, 2, 4, 4 }, result.shape().dims()); + const expected: ConstSlice = .init(Shape.init(.{ 2, 2, 4, 4 }, .i32), std.mem.sliceAsBytes(&[2][2][4][4]i32{ .{ .{ .{ 1, 1, 2, 2 }, @@ -662,15 +712,23 @@ test nearest { .{ 15, 15, 16, 16 }, }, }, - }; - try zml.testing.expectClose(zml.HostBuffer.fromArray(&expected), result, 0); + })); + try zml.testing.expectClose(std.testing.io, expected, result, 0); } // 5D Tensor (basic) { - const input_5d = try zml.Buffer.fromSlice(platform, .{ 1, 1, 1, 2, 2 }, &[_]i32{ 1, 2, 3, 4 }); - const result = try zml.testing.compileAndCall(platform, upsample, .{ input_5d, .{ .scale_factor = &.{2}, .mode = .nearest } }); - try std.testing.expectEqualSlices(i64, &.{ 1, 1, 2, 4, 4 }, result.dims()); - const expected: [1][1][2][4][4]i32 = .{ + const input_5d: zml.Tensor = .init(.{ 1, 1, 1, 2, 2 }, .i32); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_5d, .{ .scale_factor = &.{2}, .mode = .nearest } }, platform); + defer exe.deinit(); + + var input_5d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_5d.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); + defer input_5d_buffer.deinit(); + + const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_5d_buffer}); + defer result.deinit(); + + try std.testing.expectEqualSlices(i64, &.{ 1, 1, 2, 4, 4 }, result.shape().dims()); + const expected: ConstSlice = .init(Shape.init(.{ 1, 1, 2, 4, 4 }, .i32), std.mem.sliceAsBytes(&[1][1][2][4][4]i32{ .{ .{ .{ @@ -687,8 +745,8 @@ test nearest { }, }, }, - }; - try zml.testing.expectClose(zml.HostBuffer.fromArray(&expected), result, 0); + })); + try zml.testing.expectClose(std.testing.io, expected, result, 0); } } @@ -720,20 +778,23 @@ pub fn resizeBilinear(image: Tensor, resized_axes: anytype, opt: ResizeOpts) Ten test resizeBilinear { const platform = zml.testing.env(); - // Only test shapes - var comp = try zml.module.CompilationContext.init(std.heap.page_allocator, "test", platform); + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); + const block = @import("mlir").Block.init(&.{}, &.{}); + comp.pushBlock(block); + defer comp.popBlock(); + inline for (.{ .{ .{ .a = 10, .b = 10 }, .{ .a = 20 }, .{ .a = 20, .b = 10 } }, .{ .{ .a = 10, .b = 10 }, .{ .b = 5 }, .{ .a = 10, .b = 5 } }, .{ .{ .a = 10, .b = 10 }, .{ .a = 20, .b = 5 }, .{ .a = 20, .b = 5 } }, }) |testcase| { const x_shape, const resizing, const res_shape = testcase; - const x = Tensor.constant(x_shape, .{ .f16 = 0 }); + const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); const y = resizeBilinear(x, resizing, .{}); try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); try std.testing.expect(y.value().owner().verify()); @@ -754,8 +815,8 @@ pub fn resizeLinear1d(image: Tensor, axis: i8, new_len: u63, opt: ResizeOpts) Te // TODO: check that two gather isn't too bad perf wise. // Normally we should use gatherSlices to collect the values 2 by 2, // but gatherSlices messes up with the order of axes. - const left_val = image.gather_(&.{ax}, &.{left.convert(.i32)}, .{ .indices_are_sorted = true }).convert(dtype); - const right_val = image.gather_(&.{ax}, &.{right.convert(.i32)}, .{ .indices_are_sorted = true }).convert(dtype); + const left_val = ops.gather(image, &.{ax}, &.{left.convert(.i32)}, .{ .indices_are_sorted = true }).convert(dtype); + const right_val = ops.gather(image, &.{ax}, &.{right.convert(.i32)}, .{ .indices_are_sorted = true }).convert(dtype); const left_weight = right.sub(scaled).broadcast(res_shape, &.{ax}); const right_weight = scaled.sub(left).broadcast(res_shape, &.{ax}); @@ -783,20 +844,23 @@ pub fn resizeBicubic(image: Tensor, resized_axes: anytype, opt: ResizeOpts) Tens test resizeBicubic { const platform = zml.testing.env(); - // Only test shapes - var comp = try zml.module.CompilationContext.init(std.heap.page_allocator, "test", platform); + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); + const block = @import("mlir").Block.init(&.{}, &.{}); + comp.pushBlock(block); + defer comp.popBlock(); + inline for (.{ .{ .{ .a = 10, .b = 10 }, .{ .a = 20 }, .{ .a = 20, .b = 10 } }, .{ .{ .a = 10, .b = 10 }, .{ .b = 5 }, .{ .a = 10, .b = 5 } }, .{ .{ .a = 10, .b = 10 }, .{ .a = 20, .b = 5 }, .{ .a = 20, .b = 5 } }, }) |testcase| { const x_shape, const resizing, const res_shape = testcase; - const x = Tensor.constant(x_shape, .{ .f16 = 0 }); + const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); const y = resizeBicubic(x, resizing, .{}); try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); try std.testing.expect(y.value().owner().verify()); @@ -812,7 +876,7 @@ pub fn resizeCubic1d(image: Tensor, axis: i8, new_len: u63, opt: ResizeOpts) Ten const scaled = Tensor.arange(.{ .end = new_len }, dtype).mul(ratio); const t = scaled.sub(scaled.floor()); const pos = Tensor.stack(&.{ - Tensor.constant(t.shape(), dtype.one()), + Tensor.constant(dtype.one()).broad(t.shape()), t, t.mul(t), t.powByConst(3), @@ -835,7 +899,7 @@ pub fn resizeCubic1d(image: Tensor, axis: i8, new_len: u63, opt: ResizeOpts) Ten .{ 1, -2.5, 2, -0.5 }, .{ -0.5, 1.5, -1.5, 0.5 }, }; - const weights = zml.Tensor.constantTensor(zml.HostBuffer.fromArray(&weights_)).convert(dtype).withTags(.{ ._interpolated, ._neighbors }); + const weights = zml.Tensor.constantTensor(Shape.init(.{ 4, 4 }, .f32), std.mem.sliceAsBytes(&weights_)).convert(dtype).withTags(.{ ._interpolated, ._neighbors }); // actually do the interpolation. // Note: ideally this matmul should be inlined with the gather, but that's currently not the case. @@ -880,8 +944,8 @@ pub fn causalAttnMask( } if (dtype.isFloat()) { - const zeros = Tensor.constant(mask.shape(), dtype.zero()); - const minus_inf = Tensor.constant(mask.shape(), dtype.minValue()); + const zeros = Tensor.constant(dtype.zero()).broad(mask.shape()); + const minus_inf = Tensor.constant(dtype.minValue()).broad(mask.shape()); mask = Tensor.select(mask, zeros, minus_inf); } else { mask = mask.convert(dtype); @@ -922,9 +986,10 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { stdx.debug.assert(k.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "k is missing tags {{.h, .k, .hd}}", err_args); stdx.debug.assert(v.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "v is missing tags {{.h, .k, .hd}}", err_args); - if (opts.allow_cudnn and cuda.canUseCudnnSdpa(q.shape()) and opts.softmax_bias == null) { - return cuda.sdpa(q, k, v, opts); - } + // TODO(Corentin): Re-enable that + //if (opts.allow_cudnn and cuda.canUseCudnnSdpa(q.shape()) and opts.softmax_bias == null) { + // return cuda.sdpa(q, k, v, opts); + //} // Handle different numbers of head by splitting q heads. // This is a bit error prone in the sense that it depends of the layout of q heads. @@ -932,14 +997,14 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { q = q.splitAxis(.h, .{ .h = k.dim(.h), .hq = .auto }); const attn_mask = if (opts.attn_mask) |m| m else null; - const dims = helpers.collectDims(.{ .h, .q, .k, .hd }, &.{ q, k, v, attn_mask }, .strict) catch { + const dims = collectDims(.{ .h, .q, .k, .hd }, &.{ q, k, v, attn_mask }, .strict) catch { stdx.debug.panic(err_template ++ "Inputs have incompatible shapes.", err_args); }; const sqrtHeadDim: f32 = 1.0 / std.math.sqrt(@as(f32, @floatFromInt(dims.hd))); const head_scaling = if (opts.scale) |s| s else Tensor.scalar(sqrtHeadDim, k.dtype()); k = k.mul(head_scaling.convert(k.dtype())); - var attn_weights = q.dot(k, .{.hd}); + var attn_weights = q.dot(k, .hd); // log.debug("attn_weights : {f}, attn_mask : {?f}", .{ attn_weights, attn_mask }); if (attn_mask) |mask| attn_weights = attn_weights.add(mask.broad(attn_weights.shape())); attn_weights = attn_weights.convert(.f32); @@ -950,7 +1015,7 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { break :attn attn_weights.convert(.f32).softmaxBiased(.k, bias).convert(q.dtype()); } else attn_weights.convert(.f32).softmax(.k).convert(q.dtype()); - var attn = attn_weights.dot(v, .{.k}); + var attn = attn_weights.dot(v, .k); return attn.transpose(q.shape()).merge(.{ .h = .{ .h, .hq } }); } @@ -993,14 +1058,17 @@ pub fn sampleTokens(activations: Tensor, opts: SamplingStrategy, rng: Tensor.Rng test sampleTokens { const platform = zml.testing.env(); - const allocator = std.testing.allocator; + + const rng: zml.Tensor.Rng = .init(); + const activations: zml.Tensor = .init(.{ .voc = 4 }, .f32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, sampleTokens, .{ activations, .{ .topk = 4, .temperature = 2.0 }, rng }, platform); + defer exe.deinit(); + + var rng_buffer = try zml.Tensor.Rng.initBuffer(platform, 0xdeadbeef, std.testing.io); + defer rng_buffer._state.deinit(); const inf = std.math.inf(f32); - var rng_buff = try zml.Tensor.Rng.init(platform, 0xdeadbeef); - defer rng_buff._state.deinit(); - - const mod = try zml.compileFn(allocator, sampleTokens, .{ Shape.init(.{ .voc = 4 }, .f32), .{ .topk = 4, .temperature = 2.0 }, zml.Tensor.Rng.shape() }, platform); - defer mod.deinit(); inline for (.{ .{ [_]f32{ inf, 3.0, 2.0, 1.0 }, 0 }, @@ -1008,11 +1076,14 @@ test sampleTokens { .{ [_]f32{ 3.0, 2, inf, inf }, 2 }, }) |logits_expected| { const logits, const expected: i32 = logits_expected; - var logits_buff = try zml.Buffer.fromArray(platform, logits); - defer logits_buff.deinit(); - var sampled, rng_buff = mod.call(.{ logits_buff, rng_buff }); + + const activations_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, activations.shape(), std.mem.sliceAsBytes(&logits)); + defer activations_buffer.deinit(); + + var sampled, rng_buffer = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, sampleTokens, .{ activations_buffer, rng_buffer }); defer sampled.deinit(); - try zml.testing.expectEqual(expected, try sampled.getValue(i32)); + + try zml.testing.expectEqual(expected, try sampled.getValue(i32, std.testing.io)); } } @@ -1030,30 +1101,38 @@ pub const DynamicSamplingStrategy = struct { min_p: f32 = 0.0, }; - pub fn shapes(dtype: DataType, max_top_k: u32) zml.ShapeOf(DynamicSamplingStrategy) { - const scalar_float = Shape.init(.{}, dtype); - const scalar_i32 = Shape.init(.{}, .i32); + pub fn init(dtype: DataType, max_top_k: u32) DynamicSamplingStrategy { + const scalar_float_shape = Shape.init(.{}, dtype); + const scalar_i32_shape = Shape.init(.{}, .i32); return .{ .max_top_k = max_top_k, - .top_k = scalar_i32, - .temperature = scalar_float, - .top_p = scalar_float, - .min_p = scalar_float, + .top_k = .fromShape(scalar_i32_shape), + .temperature = .fromShape(scalar_float_shape), + .top_p = .fromShape(scalar_float_shape), + .min_p = .fromShape(scalar_float_shape), }; } pub fn makeBuffers( + io: std.Io, platform: zml.Platform, dtype: zml.DataType, opts: Opts, ) !zml.Bufferized(DynamicSamplingStrategy) { return .{ - .top_k = try zml.Buffer.scalar(platform, opts.top_k, .i32), - .temperature = try zml.Buffer.scalar(platform, opts.temperature, dtype), - .top_p = try zml.Buffer.scalar(platform, opts.top_p, dtype), - .min_p = try zml.Buffer.scalar(platform, opts.min_p, dtype), + .top_k = try zml.Buffer.scalar(io, platform, opts.top_k, .i32), + .temperature = try zml.Buffer.scalar(io, platform, opts.temperature, dtype), + .top_p = try zml.Buffer.scalar(io, platform, opts.top_p, dtype), + .min_p = try zml.Buffer.scalar(io, platform, opts.min_p, dtype), }; } + + pub fn deinitBuffers(self: *zml.Bufferized(DynamicSamplingStrategy)) void { + self.top_k.deinit(); + self.temperature.deinit(); + self.top_p.deinit(); + self.min_p.deinit(); + } }; /// Given the output of the last layer of a LM with a `.voc` axis, @@ -1079,7 +1158,7 @@ pub fn sampleTokensDynamic(logits: Tensor, opts: DynamicSamplingStrategy, rng: T } fn fixupLogits(logits: Tensor, opts: DynamicSamplingStrategy) [2]Tensor { - const min_inf = Tensor.constant(.{}, logits.dtype().minValue()); + const min_inf = Tensor.constant(logits.dtype().minValue()); // First reduce the vocab size to a reasonable sub set of candidate. const full_topk = if (opts.max_top_k > 0) @@ -1115,14 +1194,19 @@ fn fixupLogits(logits: Tensor, opts: DynamicSamplingStrategy) [2]Tensor { test sampleTokensDynamic { const platform = zml.testing.env(); - const allocator = std.testing.allocator; const ___ = -std.math.inf(f32); - const logits = [_]f32{ @log(2.0), @log(1.0), @log(4.0), @log(3.0) }; + const logits_data = [_]f32{ @log(2.0), @log(1.0), @log(4.0), @log(3.0) }; const top_k_indices = [_]i32{ 2, 3, 0, 1 }; - const logits_buff = try zml.Buffer.fromArray(platform, logits); - const mod = try zml.compileFn(allocator, fixupLogits, .{ Shape.init(.{ .voc = logits.len }, .f32), DynamicSamplingStrategy.shapes(.f32, 0) }, platform); - defer mod.deinit(); + + const logits: zml.Tensor = .init(.{ .voc = logits_data.len }, .f32); + const dynamic_sampling_strategy = DynamicSamplingStrategy.init(.f32, 0); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, fixupLogits, .{ logits, dynamic_sampling_strategy }, platform); + defer exe.deinit(); + + var logits_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits.shape(), std.mem.sliceAsBytes(&logits_data)); + defer logits_buffer.deinit(); const Args = struct { DynamicSamplingStrategy.Opts, [4]f32 }; inline for ([_]Args{ @@ -1140,22 +1224,160 @@ test sampleTokensDynamic { .{ .{ .top_k = 4, .top_p = 0.901, .min_p = 0.6 }, [_]f32{ @log(4.0), @log(3.0), ___, ___ } }, }) |args_expected| { const args, const expected = args_expected; - const new_logits, const indices = mod.call(.{ logits_buff, try DynamicSamplingStrategy.makeBuffers(platform, .f32, args) }); - try std.testing.expectEqual(top_k_indices, try indices.getValue(@TypeOf(top_k_indices))); - try zml.testing.expectEqual(expected, try new_logits.getValue(@TypeOf(expected))); + var dynamic_sampling_strategy_buffers = try DynamicSamplingStrategy.makeBuffers(std.testing.io, platform, .f32, args); + defer DynamicSamplingStrategy.deinitBuffers(&dynamic_sampling_strategy_buffers); + var new_logits, var indices = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, fixupLogits, .{ logits_buffer, dynamic_sampling_strategy_buffers }); + defer new_logits.deinit(); + defer indices.deinit(); + try std.testing.expectEqual(top_k_indices, try indices.getValue(@TypeOf(top_k_indices), std.testing.io)); + try zml.testing.expectEqual(expected, try new_logits.getValue(@TypeOf(expected), std.testing.io)); } { // Similar but use bf16, and uses infinity to trigger nans after the softmax. const bf16 = zml.floats.BFloat16; - const mod_bf16 = try zml.compileFn(allocator, fixupLogits, .{ Shape.init(.{ .voc = logits.len }, .bf16), DynamicSamplingStrategy.shapes(.bf16, 0) }, platform); - defer mod_bf16.deinit(); + const logits_bf16: zml.Tensor = .init(.{ .voc = logits_data.len }, .bf16); + const dynamic_sampling_strategy_bf16 = DynamicSamplingStrategy.init(.bf16, 0); + var exe_bf16 = try zml.module.compile(std.testing.allocator, std.testing.io, fixupLogits, .{ logits_bf16, dynamic_sampling_strategy_bf16 }, platform); + defer exe_bf16.deinit(); + const boost = bf16.inf; const nerf = bf16.minus_inf; - const logits_buff_2 = try zml.Buffer.fromArray(platform, [4]bf16{ boost, boost, bf16.fromF32(2), nerf }); - const new_logits, const indices = mod_bf16.call(.{ logits_buff_2, try DynamicSamplingStrategy.makeBuffers(platform, .bf16, .{ .top_k = 4, .top_p = 0.9, .min_p = 0.1 }) }); - try std.testing.expectEqual([_]i32{ 0, 1, 2, 3 }, try indices.getValue([4]i32)); - try zml.testing.expectEqual([_]bf16{ boost, nerf, nerf, nerf }, try new_logits.getValue([4]bf16)); + + var logits_bf16_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits_bf16.shape(), std.mem.sliceAsBytes(&[4]bf16{ boost, boost, bf16.fromF32(2), nerf })); + defer logits_bf16_buffer.deinit(); + var dynamic_sampling_strategy_bf16_buffers = try DynamicSamplingStrategy.makeBuffers(std.testing.io, platform, .bf16, .{ .top_k = 4, .top_p = 0.9, .min_p = 0.1 }); + defer DynamicSamplingStrategy.deinitBuffers(&dynamic_sampling_strategy_bf16_buffers); + + var new_logits, var indices = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_bf16, fixupLogits, .{ logits_bf16_buffer, dynamic_sampling_strategy_bf16_buffers }); + + try std.testing.expectEqual([_]i32{ 0, 1, 2, 3 }, try indices.getValue([4]i32, std.testing.io)); + try zml.testing.expectEqual([_]bf16{ boost, nerf, nerf, nerf }, try new_logits.getValue([4]bf16, std.testing.io)); } } + +const ShapeError = error{ DimMismatch, NotFound }; +const NOT_SET: i64 = -2; +const DIM_MISMATCH: i64 = -1; + +/// Collect the given dimensions inside a struct containing tagged tensors. +pub fn collectDims( + comptime dims: anytype, + v: anytype, + comptime mode: enum { strict, allow_extra_dims, ignore_errors }, +) ShapeError!ShapeStruct(dims) { + const LocalContext = struct { + res: ShapeStruct(dims), + mode: @TypeOf(mode), + }; + + var context = LocalContext{ + .res = undefined, + .mode = mode, + }; + @memset(std.mem.bytesAsSlice(i64, std.mem.asBytes(&context.res)), NOT_SET); + + meta.visit((struct { + fn cb(ctx: *LocalContext, shape: *const Shape) void { + inline for (dims) |a| { + if (shape.hasTag(a)) |axis| { + const dim = shape.dim(axis); + + const expected_dim = &@field(ctx.res, @tagName(a)); + if (expected_dim.* == NOT_SET) { + expected_dim.* = dim; + } else if (expected_dim.* == DIM_MISMATCH) { + // this axis has already been reported as invalid. + } else if (dim != expected_dim.*) { + if (mode != .ignore_errors) { + log.warn("Dim mismatch ! Axis {0s}={1d} but received a new tensor where {0s}={2d}", .{ @tagName(a), expected_dim.*, dim }); + } + expected_dim.* = DIM_MISMATCH; + } + } + } + } + }).cb, &context, v); + + if (context.mode != .ignore_errors) { + inline for (shapeToDims(context.res), dims) |dim, dim_tag| { + if (dim == NOT_SET) { + log.warn("Axis not found: {s}", .{@tagName(dim_tag)}); + return error.NotFound; + } + if (dim == DIM_MISMATCH) return error.DimMismatch; + } + } + return context.res; +} + +fn shapeToDims(shape: anytype) [@divExact(@sizeOf(@TypeOf(shape)), @sizeOf(i64))]i64 { + return @bitCast(shape); +} + +test collectDims { + const Model = struct { + x: Shape, + y: Shape, + bias: Shape, + }; + + { + var model: Model = .{ + .x = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), + .y = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), + .bias = Shape.init(.{5}, .f32).withTags(.{.d}), + }; + try zml.testing.expectEqual(collectDims(.{ .b, .d }, &model, .strict), .{ .b = 2, .d = 5 }); + } + { + var model: Model = .{ + .x = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), + .y = Shape.init(.{ 3, 5 }, .f32).withTags(.{ .b, .d }), + .bias = Shape.init(.{5}, .f32).withTags(.{.d}), + }; + try std.testing.expectEqual( + collectDims(.{ .b, .d }, &model, .strict), + error.DimMismatch, + ); + try zml.testing.expectEqual(collectDims(.{ .b, .d }, &model, .ignore_errors), .{ .b = DIM_MISMATCH, .d = 5 }); + } + { + var model: Model = .{ + .x = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), + .y = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), + .bias = Shape.init(.{5}, .f32).withTags(.{.d}), + }; + try std.testing.expectEqual(collectDims(.{ .b, .d, .c }, &model, .strict), error.NotFound); + try zml.testing.expectEqual(collectDims(.{ .b, .d, .c }, &model, .ignore_errors), .{ .b = 2, .d = 5, .c = NOT_SET }); + } + { + var model: Model = .{ + .x = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), + .y = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), + .bias = Shape.init(.{7}, .f32).withTags(.{.d}), + }; + try std.testing.expectEqual(collectDims(.{ .b, .d }, &model, .strict), error.DimMismatch); + try zml.testing.expectEqual(collectDims(.{ .b, .d }, &model, .ignore_errors), .{ .b = 2, .d = DIM_MISMATCH }); + } +} + +fn ShapeStruct(comptime dims: anytype) type { + const rank = dims.len; + @setEvalBranchQuota(rank + 5); + var struct_field_names: [rank][]const u8 = undefined; + var struct_field_types: [rank]type = undefined; + var struct_field_attrs: [rank]std.builtin.Type.StructField.Attributes = undefined; + const default: i64 = NOT_SET; + for (&struct_field_names, &struct_field_types, &struct_field_attrs, dims) |*field_name, *field_type, *field_attr, axis| { + field_name.* = @tagName(axis); + field_type.* = i64; + field_attr.* = .{ + .@"align" = @alignOf(i64), + .default_value_ptr = &default, + }; + } + + return @Struct(.@"extern", null, &struct_field_names, &struct_field_types, &struct_field_attrs); +} diff --git a/zml/nn/cuda.zig b/zml/nn/cuda.zig deleted file mode 100644 index 4a682f31..00000000 --- a/zml/nn/cuda.zig +++ /dev/null @@ -1,141 +0,0 @@ -const std = @import("std"); - -const dialect = @import("mlir/dialects"); - -const Context = @import("../context.zig").Context; -const DataType = @import("../dtype.zig").DataType; -const Data = @import("../dtype.zig").Data; -const mlirx = @import("../mlirx.zig"); -const module = @import("../module.zig"); -const CompilationContext = module.CompilationContext; -const SdpaOpts = @import("../nn.zig").SdpaOpts; -const Shape = @import("../shape.zig").Shape; -const Tensor = @import("../tensor.zig").Tensor; - -pub fn canUseCudnnSdpa(q_shape: Shape) bool { - const ctx = CompilationContext.current(); - // TODO(Corendos): Check cuda version, cudnn version, device compatibility. - if (ctx.target() != .cuda) return false; - - if (q_shape.rank() != 4) return false; - - // NOTE(Corentin): In Cudnn fused MHA head_dim is limited to 128. - if (q_shape.dim(.hd) > 128) return false; - - // NOTE(Corentin): In Cudnn fused MHA data type is limited to F16 and BF16. - if (q_shape.dtype() != .f16 and q_shape.dtype() != .bf16) return false; - - return true; -} - -fn elementTypeFromDataType(dtype: DataType) [:0]const u8 { - return switch (dtype) { - .f16 => "F16", - .bf16 => "BF16", - else => @panic("Unsupported DataType"), - }; -} - -pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { - const ctx = CompilationContext.current(); - const q = q_.transpose(.{ .b, .h, .q, .hd }); - var k = k_.transpose(.{ .b, .h, .k, .hd }); - const v = v_.transpose(.{ .b, .h, .k, .hd }); - - const sqrtHeadDim: f32 = 1.0 / std.math.sqrt(@as(f32, @floatFromInt(q.dim(.hd)))); - const scale: f32 = if (opts.scale) |_| 1.0 else sqrtHeadDim; - - if (opts.scale) |s| k = k.mul(s); - - var buffer: [4096]u8 = undefined; - var fba = std.heap.FixedBufferAllocator.init(&buffer); - const allocator = fba.allocator(); - - const backend_config = std.fmt.allocPrintSentinel( - allocator, - \\{{ - \\ "operation_queue_id":"0", - \\ "wait_on_operation_queues":[], - \\ "cudnn_fmha_backend_config": {{ - \\ "algorithm": {{ - \\ "algo_id": "0", - \\ "math_type": "TENSOR_OP_MATH", - \\ "tuning_knobs": {{"17": "1", "24": "0"}}, - \\ "is_cudnn_frontend": true, - \\ "workspace_size": "0", - \\ }}, - \\ "fmha_scale": {d}, - \\ "dropout_rate": 0, - \\ "intermediate_tensor_shape": {{ - \\ "element_type": "{s}", - \\ "dimensions": ["{}", "{}", "{}", "{}"], - \\ "tuple_shapes": [], - \\ "layout": {{ - \\ "dim_level_types": [], - \\ "dim_unique": [], - \\ "dim_ordered": [], - \\ "minor_to_major": ["3", "2", "1", "0"], - \\ "tiles": [], - \\ "element_size_in_bits": "0", - \\ "memory_space": "0", - \\ "index_primitive_type": "PRIMITIVE_TYPE_INVALID", - \\ "pointer_primitive_type": "PRIMITIVE_TYPE_INVALID", - \\ "dynamic_shape_metadata_prefix_bytes": "0", - \\ }}, - \\ "is_dynamic_dimension": [false, false, false, false], - \\ }}, - \\ "seed": 0, - \\ "is_flash_attention": true, - \\ "mask_type": "NO_MASK", - \\ "bmm1_dot_dimension_numbers": {{ - \\ "lhs_contracting_dimensions": ["3"], - \\ "rhs_contracting_dimensions": ["3"], - \\ "lhs_batch_dimensions": ["0", "1"], - \\ "rhs_batch_dimensions": ["0", "1"], - \\ }}, - \\ "bmm2_dot_dimension_numbers": {{ - \\ "lhs_contracting_dimensions": ["3"], - \\ "rhs_contracting_dimensions": ["2"], - \\ "lhs_batch_dimensions": ["0", "1"], - \\ "rhs_batch_dimensions": ["0", "1"], - \\ }} - \\ }} - \\}} - , - .{ - scale, - elementTypeFromDataType(q.dtype()), - q.dim(.b), - q.dim(.h), - q.dim(.q), - k.dim(.k), - }, - 0, - ) catch unreachable; - - var bias = Tensor.constant(Shape.init(.{ .b = q.dim(.b), .h = q.dim(.h), .q = q.dim(.q), .k = k.dim(.k) }, q.dtype()), Data.init(q.dtype(), 0)); - - if (opts.attn_mask) |attn_mask| { - bias = bias.add(attn_mask.broad(bias.shape())); - } - - const mlir_ctx = ctx.mlirCtx(); - const loc = mlir_ctx.location(@src()); - const op = dialect.stablehlo.custom_call( - mlir_ctx, - &.{ q.value(), k.value(), v.value(), bias.value() }, - .{ - .call_target_name = "__cudnn$fmhaScaleBiasSoftmax", - .backend_config = .string(mlir_ctx, backend_config), - .has_side_effect = false, - .api_version = .original, - }, - &.{ - mlirx.tensorType(mlir_ctx, q.shape()), - .tensor(&.{0}, .int(mlir_ctx, .u8)), - }, - loc, - ); - const result = Tensor._result(q.shape(), op.result(0)).transpose(q_.shape()); - return result; -} diff --git a/zml/ops.zig b/zml/ops.zig index edbcd4a2..637e0947 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1,206 +1,113 @@ const std = @import("std"); +const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); const stdx = @import("stdx"); -const _collectAxes = @import("tensor.zig")._collectAxes; -const Buffer = @import("buffer.zig").Buffer; const CompilationContext = @import("module.zig").CompilationContext; -const Context = @import("context.zig").Context; +const constants = @import("constants.zig"); const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); -const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const Tensor = @import("tensor.zig").Tensor; -const EnumLiteral = @TypeOf(.enum_literal); -const dialect = struct { - const stablehlo = @import("mlir/dialects").stablehlo; +pub const ReduceArgs = struct { + left: Tensor, + right: Tensor, }; -const log = std.log.scoped(.@"zml/tensor"); +pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); -test { - std.testing.refAllDecls(@This()); -} + const mlir_ctx = CompilationContext.current().mlir_ctx; -/// Generate an MLIR call to the given member function with the given tensors. -pub fn call(self: anytype, comptime func: stdx.meta.DeclEnum(@TypeOf(self)), args: anytype) @TypeOf(@call(.auto, @field(stdx.meta.UnwrapPtr(@TypeOf(self)), @tagName(func)), .{self} ++ args)) { - const ctx = CompilationContext.current(); - const name = @typeName(@TypeOf(self)) ++ "." ++ @tagName(func); - const actual_fn = @field(@TypeOf(self), @tagName(func)); - return ctx.callFunc(name, actual_fn, .{self} ++ args) catch @panic("OOM"); -} + const reduce_block, var result = b: { + const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inits.len); + var args: ArgsType = undefined; + var block_types: [2 * inits.len]*const mlir.Type = undefined; -pub fn while_( - comptime cond_fn: anytype, - comptime body_fn: anytype, - blkctx: BlockSign(body_fn).BlkCtx, - inputs: BlockSign(body_fn).Args, -) BlockSign(body_fn).Return { - const CondS = comptime BlockSign(cond_fn); - const BodyS = comptime BlockSign(body_fn); - if (CondS.Args != BodyS.Args) { - @compileError("cond_fn and body_fn signatures don't match ! " ++ @typeName(@TypeOf(cond_fn)) ++ " and " ++ @typeName(@TypeOf(body_fn))); - } - const ctx = CompilationContext.current(); - const cond_block, _ = ctx.makeBlock(.open, CondS, &cond_fn, blkctx, inputs); - const body_block, const body_res = ctx.makeBlock(.open, BodyS, &body_fn, blkctx, inputs); - var input_values: [BodyS.nIn]mlir.Value = undefined; - ctx.extractValues(&inputs, &input_values); + inline for (0..inits.len) |i| { + args[i].left = .fromShape(inits[i].shape()); + args[i].right = .fromShape(inits[i].shape()); - const loc = ctx.mlirCtx().location(@src()); - - const op = mlir.Operation.make(ctx.mlirCtx(), "stablehlo.while", .{ - .variadic_operands = &.{&input_values}, - .result_type_inference = true, - .blocks = &.{ cond_block, body_block }, - // We can't verify right away, cause the weights captured by the while haven't been added yet. - .verify = false, - .location = loc, - }); - - return fromMlirOperationWithTags(op, body_res); -} - -test "simple while" { - const CountInts = struct { - step: Tensor, - end: Tensor, - const CountInts = @This(); - - pub fn _hasNext(self: CountInts, i: Tensor, sum: Tensor) Tensor { - _ = sum; - return i.cmp(.LT, self.end); + block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); + block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); } - pub fn _next(self: CountInts, i: Tensor, sum: Tensor) [2]Tensor { - const r1 = i.add(self.step); - const r2 = sum.add(i); - return .{ r1, r2 }; + const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); + const reduce_block = mlir.Block.init(&block_types, &block_locs); + errdefer reduce_block.deinit(); + + CompilationContext.current().pushBlock(reduce_block); + defer CompilationContext.current().popBlock(); + + const scope = CompilationContext.current().currentScope(); + inline for (0..inits.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inits.len) catch unreachable; } - pub fn _fwd(self: CountInts, init_i: Tensor, init_sum: Tensor) [2]Tensor { - const x = init_i.scale(2); - return while_(CountInts._hasNext, CountInts._next, self, .{ x, init_sum }); + var result = @call(.auto, func, args ++ context); + + var result_values: [inits.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| { + result_values[i] = result[i].value(); } - pub fn _zigForward(step: i64, end: i64, init_i: i64, init_sum: i64) [2]i64 { - const x = init_i * 2; - var i = x; - var sum = init_sum; - while (i < end) { - const r1 = i + step; - const r2 = sum + i; - i, sum = .{ r1, r2 }; - } - return .{ i, sum }; - } + _ = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)).appendTo(reduce_block); + break :b .{ reduce_block, result }; }; - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const res0, const res1 = try zml.testing.compileAndCall( - platform, - CountInts._fwd, - .{ - .{ .step = try .scalar(platform, 1, .i64), .end = try .scalar(platform, 10, .i64) }, - try .scalar(platform, 0, .i64), - try .scalar(platform, 0, .i64), - }, - ); - const last_i = try res0.getValue(i64); - const sum = try res1.getValue(i64); - - try std.testing.expectEqual(10, last_i); - try std.testing.expectEqual(45, sum); - - try std.testing.expectEqual(.{ 10, 45 }, CountInts._zigForward(1, 10, 0, 0)); -} - -pub fn reduce( - comptime body_fn: anytype, - inputs: stdx.meta.FnParam(body_fn, 0), - inits: stdx.meta.FnParam(body_fn, 0), - axes: []const i64, -) BlockSignNoCtx(body_fn).Return { - // TODO: actualAxes - const BodyS = comptime BlockSignNoCtx(body_fn); - comptime { - if (BodyS.Return != @TypeOf(inputs)) @compileError("reduce body function need to have the following signature `fn (left: T, right: T) T`, got: " ++ @typeName(body_fn)); + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); } - const ctx = CompilationContext.current(); - const N = comptime @divExact(BodyS.nIn, 2); - var input_values: [N]mlir.Value = undefined; - ctx.extractValues(&inputs, &input_values); - var init_values: [N]mlir.Value = undefined; - ctx.extractValues(&inits, &init_values); - const body_block, _ = ctx.makeBlock(.hermetic, BodyS, &body_fn, {}, .{ inits, inits }); + var init_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| init_values[i] = inits[i].value(); - const loc = ctx.mlirCtx().location(@src()); - - const op = mlir.Operation.make(ctx.mlirCtx(), "stablehlo.reduce", .{ - .variadic_operands = &.{ &input_values, &init_values }, + const reduce_op = mlir.Operation.make(mlir_ctx, "stablehlo.reduce", .{ + .operands = .{ .variadic = &.{ &input_values, &init_values } }, .result_type_inference = true, - .blocks = &.{body_block}, - .attributes = &.{.{ "dimensions", .dense(ctx.mlirCtx(), .i64, axes) }}, - // We can't verify right away, cause the weights captured by the reduce haven't been added yet. - .verify = false, - .location = loc, - }); + .blocks = &.{reduce_block}, + .attributes = &.{ + .named(mlir_ctx, "dimensions", mlir.denseArrayAttribute(mlir_ctx, .i64, axes_)), + }, + .verify = true, + .location = .unknown(mlir_ctx), + }).appendTo(CompilationContext.current().currentScope().block); // `stablehlo.reduce` drops axes. We want to avoid that to propagate tags. // So we need to broadcast the output of `stablehlo.reduce` to the input shapes. // To that order, we initialize `result` to `inputs`, then we use stdx.meta.visit, // to find the correct mlir.Value, but we first broadcast before creating the final // Tensor struct. - var broadcasting_axes: stdx.BoundedArray(i64, Tensor.MAX_RANK) = .{}; - for (0..Tensor.MAX_RANK) |i| { - if (std.mem.indexOfScalar(i64, axes, @intCast(i)) == null) { + var broadcasting_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; + for (0..constants.MAX_RANK) |i| { + if (std.mem.indexOfScalar(i64, axes_, @intCast(i)) == null) { broadcasting_axes.append(@intCast(i)) catch unreachable; } } - var res: BodyS.Return = inputs; - const LocalContext = struct { - axes: []const i64, - broadcasting_axes: []const i64, - n_reduced: u8, - op: mlir.Operation, - loc: mlir.Location, - index: usize = 0, - }; - var local_context = LocalContext{ - .axes = axes, - .broadcasting_axes = broadcasting_axes.constSlice(), - .n_reduced = @intCast(axes.len), - .op = op, - .loc = loc, - }; - meta.visit((struct { - fn cb(inner_ctx: *LocalContext, tensor: *Tensor) void { - const val = inner_ctx.op.result(inner_ctx.index); - // compute the target reduced shape - var reduced_shape = tensor.shape(); - for (inner_ctx.axes) |a| { - reduced_shape = reduced_shape.setDim(a, 1); - } - const mlir_ctx = CompilationContext.current().mlirCtx(); - - const broad_val = dialect.stablehlo.broadcast_in_dim( - mlir_ctx, - val, - inner_ctx.broadcasting_axes[0 .. tensor.rank() - inner_ctx.n_reduced], - mlirx.tensorType(mlir_ctx, reduced_shape), - inner_ctx.loc, - ); - tensor.* = Tensor._result(reduced_shape, broad_val.result(0)); - inner_ctx.index += 1; + inline for (0..result.len) |i| { + var reduced_shape: Shape = inputs[i].shape(); + for (axes_) |a| { + reduced_shape = reduced_shape.setDim(a, 1); } - }).cb, &local_context, &res); - std.debug.assert(local_context.index == op.numResults()); - return res; + + const tensor_type = mlir.rankedTensorType(reduced_shape.dims(), mlirx.Type.fromDType(mlir_ctx, reduced_shape.dtype())); + const broad_op = dialects.stablehlo.broadcast_in_dim( + mlir_ctx, + reduce_op.result(i), + broadcasting_axes.slice()[0 .. reduced_shape.rank() - axes_.len], + tensor_type, + .unknown(mlir_ctx), + ).appendTo(CompilationContext.current().currentScope().block); + + result[i] = Tensor._result(reduced_shape, broad_op.result(0)); + } + + return result; } pub const ReduceWindowOpts = struct { @@ -212,550 +119,145 @@ pub const ReduceWindowOpts = struct { padding: []const [2]i64, }; -pub fn reduceWindow( - comptime body_fn: anytype, - inputs: stdx.meta.FnParam(body_fn, 0), - inits: stdx.meta.FnParam(body_fn, 0), - opts: ReduceWindowOpts, -) stdx.meta.FnResult(body_fn) { - const BodyS = comptime BlockSignNoCtx(body_fn); - comptime { - if (BodyS.Return != @TypeOf(inputs)) @compileError("reduce body function need to have the following signature `fn (left: T, right: T) T`, got: " ++ @typeName(body_fn)); - } - const ctx = CompilationContext.current(); - const body_block, _ = ctx.makeBlock(.hermetic, BodyS, &body_fn, {}, .{ inits, inits }); - const N = comptime @divExact(BodyS.nIn, 2); - var input_values: [N]mlir.Value = undefined; - ctx.extractValues(&inputs, &input_values); - var init_values: [N]mlir.Value = undefined; - ctx.extractValues(&inits, &init_values); +pub fn reduceWindow(inputs: anytype, inits: anytype, opts: ReduceWindowOpts, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); - const loc = ctx.mlirCtx().location(@src()); - const op = mlir.Operation.make(ctx.mlirCtx(), "stablehlo.reduce_window", .{ - .variadic_operands = &.{ input_values[0..], init_values[0..] }, + const mlir_ctx = CompilationContext.current().mlir_ctx; + + const reduce_block, var result = b: { + const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inits.len); + var args: ArgsType = undefined; + var block_types: [2 * inits.len]*const mlir.Type = undefined; + + inline for (0..inits.len) |i| { + args[i].left = .fromShape(inits[i].shape()); + args[i].right = .fromShape(inits[i].shape()); + + block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); + block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); + } + + const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); + const reduce_block = mlir.Block.init(&block_types, &block_locs); + errdefer reduce_block.deinit(); + + CompilationContext.current().pushBlock(reduce_block); + defer CompilationContext.current().popBlock(); + + const scope = CompilationContext.current().currentScope(); + inline for (0..inits.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inits.len) catch unreachable; + } + + var result = @call(.auto, func, args ++ context); + + var result_values: [inits.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| { + result_values[i] = result[i].value(); + } + + _ = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)).appendTo(reduce_block); + break :b .{ reduce_block, result }; + }; + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); + } + + var init_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| init_values[i] = inits[i].value(); + + const reduce_op = mlir.Operation.make(mlir_ctx, "stablehlo.reduce_window", .{ + .operands = .{ .variadic = &.{ &input_values, &init_values } }, .result_type_inference = true, - .blocks = &.{body_block}, + .blocks = &.{reduce_block}, .attributes = &.{ - .{ "window_dimensions", .dense(ctx.mlirCtx(), .i64, opts.window_dimensions) }, - .{ "window_strides", .dense(ctx.mlirCtx(), .i64, opts.window_strides) }, - .{ "base_dilations", .dense(ctx.mlirCtx(), .i64, opts.base_dilations) }, - .{ "window_dilations", .dense(ctx.mlirCtx(), .i64, opts.window_dilations) }, + .named(mlir_ctx, "window_dimensions", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.window_dimensions)), + .named(mlir_ctx, "window_strides", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.window_strides)), + .named(mlir_ctx, "base_dilations", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.base_dilations)), + .named(mlir_ctx, "window_dilations", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.window_dilations)), // Cast the [][2]i64 to []i64 (safe) - .{ "padding", .denseElements(ctx.mlirCtx(), &.{ @intCast(opts.padding.len), 2 }, .i64, @ptrCast(opts.padding)) }, + .named(mlir_ctx, "padding", mlir.denseElementsAttribute(mlir.RankedTensorType.get(&.{ @intCast(opts.padding.len), 2 }, mlir.integerType(mlir_ctx, .i64), null).shaped(), @as([]const i64, @ptrCast(opts.padding)))), }, - .location = loc, - }); + .verify = true, + .location = .unknown(mlir_ctx), + }).appendTo(CompilationContext.current().currentScope().block); - return fromMlirOperationWithTags(op, inputs); + inline for (0..result.len) |i| { + result[i] = Tensor.fromMlirValue(reduce_op.result(i)).withTags(inputs[i].shape()); + } + + return result; } -/// Runs a given function for several steps, and returns a stack of each step output. -/// The step outputs will be stacked along the first axis. -pub fn for_(comptime func: anytype, blk_ctx: BlockSign(func).BlkCtx, num_steps_: anytype) BlockSign(func).Return { - const num_steps: u32, const step_tag = blk: { - const dims, const tags = Shape.parseDimensions(num_steps_); - stdx.debug.assert(dims.len == 1, "zml.for_ only supports one num_step, Received: {any}", .{num_steps_}); - break :blk .{ @intCast(dims.get(0)), tags.get(0) }; - }; - const S = comptime BlockSign(func); - - const ForBlk = struct { - blk_ctx: S.BlkCtx, - step_tag: Shape.Tag, - num_steps: u32, - const Self = @This(); - - fn next(self: Self, res: S.Return, idx: Tensor) struct { S.Return, Tensor } { - const step_res = @call(.auto, func, .{ self.blk_ctx, idx }); - var buf: [@sizeOf(S.Return) * 2]u8 = undefined; - var fba = std.heap.FixedBufferAllocator.init(&buf); - return .{ - meta.zip(updateResBuffer, fba.allocator(), &[_]S.Return{ res, step_res }, .{idx}) catch unreachable, - idx.addConstant(1), - }; - } - - fn done(self: Self, res: S.Return, idx: Tensor) Tensor { - _ = res; - return idx.cmp(.LT, Tensor.scalar(self.num_steps, idx.dtype())); - } - - fn updateResBuffer(inputs: []const Tensor, idx: Tensor) Tensor { - stdx.debug.internalAssert(inputs.len == 2, "too many tensors", .{}); - const res, const step_res = inputs[0..2].*; - return res.dynamicUpdateSlice1d(step_res.insertAxes(0, .{._}), 0, idx); - } - - /// Prepare buffer to store all results steps. - fn prep(self: Self, first_step: Tensor) Tensor { - const shape = first_step.shape().insertTag(0, 1, self.step_tag); - // Reuse the first step Tensor. - // TODO: this is needed because of https://github.com/zml/zml/issues/97 - // Normally I'd rather NOT reuse first_step to streamline the stablehlo IR. - return first_step.reshape(shape).pad(0, .{ ._0 = Tensor.Pad{ .high = self.num_steps - 1 } }); - } - - fn wrapFirstStep(tag_: @TypeOf(step_tag), x: Tensor) Tensor { - var shape = x.shape(); - shape._dims.insert(0, 1) catch unreachable; - shape._tags.insert(0, tag_) catch unreachable; - return x.reshape(shape); - } - }; - - // Compute first step to infer the output shapes. - // Normally this shouldn't be reused apart from the unrolled cases, - // but because of https://github.com/zml/zml/issues/97 we also reuse it to start the while_ loop. - const first_step = @call(.auto, func, .{ blk_ctx, Tensor.scalar(0, .i32) }); - log.debug("for_ first_step: {}", .{first_step}); - const allocator = CompilationContext.current().allocator(); - // Optimize for small num reps - if (num_steps == 1) { - var res = first_step; - meta.mapAlloc(ForBlk.wrapFirstStep, allocator, step_tag, first_step, &res) catch unreachable; - return res; - } - - if (num_steps <= 4) { - var steps: [4]S.Return = undefined; - steps[0] = first_step; - for (1..num_steps) |i| { - steps[i] = @call(.auto, func, .{ blk_ctx, Tensor.scalar(i, .i32) }); - } - - const res = meta.zip(Tensor.stack, allocator, steps[0..num_steps], .{ 0, step_tag }) catch unreachable; - return res; - } - - const for_blk: ForBlk = .{ .blk_ctx = blk_ctx, .step_tag = step_tag, .num_steps = num_steps }; - var result_buffers: @TypeOf(first_step) = undefined; - try meta.mapAlloc(ForBlk.prep, allocator, for_blk, first_step, &result_buffers); - - return while_( - ForBlk.done, - ForBlk.next, - for_blk, - .{ - result_buffers, - // First step is already done - Tensor.scalar(1, .i32), - }, - )[0]; -} - -test for_ { - const Squares = struct { - const Squares = @This(); - - pub fn sq(self: Squares, i: Tensor) Tensor { - _ = self; - const f = i.convert(.f32); - return f.mul(f); - } - - pub fn _fwd(num_steps: u63) Tensor { - return for_(Squares.sq, .{}, .{num_steps}); - } - }; - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - // Just one baby step - { - const squares = try zml.testing.compileAndCall(platform, Squares._fwd, .{1}); - try zml.testing.expectEqualShapes(Shape.init(.{1}, .f32), squares.shape()); - try std.testing.expectEqual(0, squares.getValue(f32)); - } - // Wow 4 in rows ! - { - const squares = try zml.testing.compileAndCall(platform, Squares._fwd, .{4}); - try zml.testing.expectEqualShapes(Shape.init(.{4}, .f32), squares.shape()); - try std.testing.expectEqual([_]f32{ 0, 1, 4, 9 }, try squares.getValue([4]f32)); - } - // AGI is coming, computing 10 squares as it's nothing. - { - const squares = try zml.testing.compileAndCall(platform, Squares._fwd, .{10}); - try zml.testing.expectEqualShapes(Shape.init(.{10}, .f32), squares.shape()); - try std.testing.expectEqual( - [_]f32{ 0, 1, 4, 9, 16, 25, 36, 49, 64, 81 }, - try squares.getValue([10]f32), - ); - } -} - -test "nested for" { - const OuterProd = struct { - const OuterProd = @This(); - - x: Tensor, - x_row: Tensor, - - pub fn _fwd(x: Tensor) Tensor { - return for_(OuterProd.scanRow, x, .{x.dim(0)}); - } - - pub fn scanRow(x: Tensor, i: Tensor) Tensor { - const row = x.dynamicSlice(.{Tensor.DynSlice{ .start = i, .len = 1 }}); - return for_(OuterProd.scanCol, .{ .x = x, .x_row = row }, .{x.dim(0)}); - } - - pub fn scanCol(self: OuterProd, j: Tensor) Tensor { - const col = self.x.dynamicSlice(.{Tensor.DynSlice{ .start = j, .len = 1 }}); - return self.x_row.mul(col); - } - }; - - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - // 5 to prevent inlining - const x = try zml.Buffer.fromArray(platform, [5]f32{ 0, 1.0, -1.0, 2.0, -2.0 }); - const outer_prod = try zml.testing.compileAndCall(platform, OuterProd._fwd, .{x}); - const expected: [5][5]f32 = .{ - .{ 0, 0, 0, 0, 0 }, - .{ 0, 1.0, -1.0, 2.0, -2.0 }, - .{ 0, -1.0, 1.0, -2.0, 2.0 }, - .{ 0, 2.0, -2.0, 4.0, -4.0 }, - .{ 0, -2.0, 2.0, -4.0, 4.0 }, - }; - try std.testing.expectEqual(expected, outer_prod.getValue(@TypeOf(expected))); -} - -pub fn if_2(pred: Tensor, comptime Closure: type, blkctx: BlockSignNoArgs(@field(Closure, "then")).BlkCtx) BlockSignNoArgs(@field(Closure, "then")).Return { - return if_(pred, @field(Closure, "then"), @field(Closure, "else_"), blkctx); -} - -pub fn if_( - pred: Tensor, - comptime true_branch_fn: anytype, - comptime false_branch_fn: anytype, - blkctx: BlockSignNoArgs(true_branch_fn).BlkCtx, -) BlockSignNoArgs(true_branch_fn).Return { - const TrueBlockSignature = comptime BlockSignNoArgs(true_branch_fn); - const FalseBlockSignature = comptime BlockSignNoArgs(false_branch_fn); - if (TrueBlockSignature.Return != FalseBlockSignature.Return) { - @compileError("true_branch_fn and false_branch_fn return types don't match ! " ++ @typeName(TrueBlockSignature.Return) ++ " and " ++ @typeName(FalseBlockSignature.Return)); - } - - stdx.debug.assert(pred.dtype() == .bool and pred.count() == 1, "zml.ops.if_ expects the condition to have exactly one element of dtype .bool, got {f}", .{pred}); - - const ctx = CompilationContext.current(); - const true_branch_block, const true_branch_res = ctx.makeBlock(.open, TrueBlockSignature, &true_branch_fn, blkctx, {}); - const false_branch_block, const false_branch_res = ctx.makeBlock(.open, TrueBlockSignature, &false_branch_fn, blkctx, {}); - - check: { - const arena = ctx.allocator(); - const true_shapes = meta.collectAlloc(Tensor.shape, {}, arena, &true_branch_res) catch break :check; - defer arena.free(true_shapes); - - const false_shapes = meta.collectAlloc(Tensor.shape, {}, arena, &false_branch_res) catch break :check; - defer arena.free(false_shapes); - - stdx.debug.assert(true_shapes.len == false_shapes.len, "zml.ops.if_ expects the true and false branch to produce the same number of tensors. Got: \n - true branch: {f}\n -false branch: {f}", .{ stdx.fmt.slice(true_shapes), stdx.fmt.slice(false_shapes) }); - for (true_shapes, false_shapes) |true_shape, false_shape| { - stdx.debug.assert(true_shape.eqlWithTags(false_shape), "zml.ops.if_ expects the true and false branch to produce tensors of the same shape. Got: \n - true branch: {f}\n -false branch: {f}", .{ stdx.fmt.slice(true_shapes), stdx.fmt.slice(false_shapes) }); - } - } - - const loc = ctx.mlirCtx().location(@src()); - const op = mlir.Operation.make(ctx.mlirCtx(), "stablehlo.if", .{ - .operands = &.{pred.asScalar().value()}, - .result_type_inference = true, - .blocks = &.{ true_branch_block, false_branch_block }, - // We can't verify right away, cause the weights captured by the if haven't been added yet. - .verify = false, - .location = loc, - }); - - return fromMlirOperationWithTags(op, true_branch_res); -} - -test "if" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - const allocator = std.testing.allocator; - - const IfMod = struct { - pub fn _fwd(pred: Tensor, a: Tensor, b: Tensor) Tensor { - const result = if_(pred.convert(.bool), condTrue, condFalse, .{ a, b }); - return result; - } - - pub fn condTrue(a: Tensor, b: Tensor) Tensor { - return a.matmul(b); - } - - pub fn condFalse(a: Tensor, b: Tensor) Tensor { - return b.matmul(a); - } - }; - - { - const pred = Shape.init(.{}, .i32); - const a = Shape.init(.{ 4, 4 }, .f32); - const b = Shape.init(.{ 4, 4 }, .f32); - const mod = try zml.compileFn(allocator, IfMod._fwd, .{ pred, a, b }, platform); - defer mod.deinit(); - } -} - -/// Execute exactly one of the `branches` given by `index`. -/// -/// If `index` is out of bound, it is clamped withing bounds. -/// -/// The branches take no parameters but can access the local context given by `blkctx`. -pub fn case( - index: Tensor, - comptime branches: anytype, - blkctx: BlockSignNoArgs(branches[0]).BlkCtx, -) BlockSign(branches[0]).Return { - const Signature = BlockSignNoArgs(branches[0]); - const ctx = CompilationContext.current(); - - const branch_count = branches.len; - var blocks: [branch_count]mlir.Block = undefined; - var res: [branch_count]Signature.Return = undefined; - inline for (branches, 0..) |branch, i| { - blocks[i], res[i] = ctx.makeBlock(.open, Signature, &branch, blkctx, {}); - } - - const loc = ctx.mlirCtx().location(@src()); - const op = mlir.Operation.make(ctx.mlirCtx(), "stablehlo.case", .{ - .operands = &.{index.value()}, - .result_type_inference = true, - .blocks = &blocks, - // We can't verify right away, cause the weights captured by the if haven't been added yet. - .verify = false, - .location = loc, - }); - - return fromMlirOperationWithTags(op, res[0]); -} - -test "case" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const CaseMod = struct { - pub fn _fwd(index: Tensor, a: Tensor, b: Tensor) Tensor { - const result = case(index, .{ case1, case2, case3 }, .{ a, b }); - return result; - } - - pub fn case1(a: Tensor, b: Tensor) Tensor { - return a.matmul(b); - } - - pub fn case2(a: Tensor, b: Tensor) Tensor { - return a.add(b); - } - - pub fn case3(a: Tensor, b: Tensor) Tensor { - return a.sub(b); - } - }; - - { - const index = try zml.Buffer.fromSlice(platform, .{}, &[1]i32{1}); - const a = try zml.Buffer.fromSlice(platform, .{ 2, 2 }, &[4]f32{ 1, 1, 2, 2 }); - const b = try zml.Buffer.fromSlice(platform, .{ 2, 2 }, &[4]f32{ 1, 1, 1, 1 }); - const result = try zml.testing.compileAndCall(platform, CaseMod._fwd, .{ index, a, b }); - - const expected: [2][2]f32 = .{ - .{ 2, 2 }, - .{ 3, 3 }, - }; - try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected))); - } -} - -pub fn sort( - comptime comp_fn: anytype, - blkctx: BlockSign(comp_fn).BlkCtx, - inputs: [@divExact(BlockSign(comp_fn).nIn, 2)]Tensor, - dimension: i64, - is_stable: bool, -) [@divExact(BlockSign(comp_fn).nIn, 2)]Tensor { - const BodyS = comptime BlockSign(comp_fn); - var inits: BlockSign(comp_fn).Args = undefined; - inline for (0..@divExact(BlockSign(comp_fn).nIn, 2)) |i| { - const arg_shape = Shape.init(.{}, inputs[i].dtype()); - // Note: the id doesn't matter cause makeBlock will correctly fill it. - inits[i * 2] = Tensor{ ._shape = arg_shape, ._id = undefined, ._donation = .no_buffer }; - inits[i * 2 + 1] = Tensor{ ._shape = arg_shape, ._id = undefined, ._donation = .no_buffer }; - } - const ctx = CompilationContext.current(); - const block, _ = ctx.makeBlock(.hermetic, BodyS, &comp_fn, blkctx, inits); - var input_values: [@divExact(BodyS.nIn, 2)]mlir.Value = undefined; - ctx.extractValues(&inputs, &input_values); - - const loc = ctx.mlirCtx().location(@src()).namedFmt(ctx.mlirCtx(), "sort(dimension={d}, is_stable={})", .{ dimension, is_stable }); - - const op = mlir.Operation.make(ctx.mlirCtx(), "stablehlo.sort", .{ - .variadic_operands = &.{&input_values}, - .result_type_inference = true, - .blocks = &.{block}, - .attributes = &.{ - .{ "dimension", .int(ctx.mlirCtx(), .i64, dimension) }, - .{ "is_stable", .boolean(ctx.mlirCtx(), is_stable) }, - }, - .location = loc, - }); - - var res: [@divExact(BlockSign(comp_fn).nIn, 2)]Tensor = undefined; - inline for (0..@divExact(BlockSign(comp_fn).nIn, 2)) |i| { - res[i] = Tensor._result(inputs[i].shape(), op.result(i)); - } - return res; -} - -pub const BlockSignature = struct { - Fn: type, - BlkCtx: type, - Args: type, - FullArgs: type, - Return: type, - nIn: usize, - nOut: usize, - - pub inline fn blkArgs(self: BlockSignature, blk_ctx: self.BlkCtx, args: self.Args) self.FullArgs { - if (self.BlkCtx == void) return args; - if (self.Args == void) return blk_ctx; - return .{blk_ctx} ++ args; - } +pub const SortArgs = struct { + left: Tensor, + right: Tensor, }; -const BlockType = enum { default, no_ctx, no_args }; +pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytype, is_stable: bool) [inputs.len]Tensor { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); -pub inline fn BlockSign(comptime func: anytype) BlockSignature { - return _BlockSign(func, .default); -} + const mlir_ctx = CompilationContext.current().mlir_ctx; -pub inline fn BlockSignNoCtx(comptime func: anytype) BlockSignature { - return _BlockSign(func, .no_ctx); -} + const sort_block = b: { + const ArgsType = std.meta.Tuple(&[1]type{SortArgs} ** inputs.len); + var args: ArgsType = undefined; + var block_types: [2 * inputs.len]*const mlir.Type = undefined; -pub inline fn BlockSignNoArgs(comptime func: anytype) BlockSignature { - return _BlockSign(func, .no_args); -} + inline for (0..inputs.len) |i| { + args[i].left = Tensor.init(.{}, inputs[i].shape().dtype()); + args[i].right = Tensor.init(.{}, inputs[i].shape().dtype()); -pub fn fnInfo(comptime func: anytype) std.builtin.Type.Fn { - if (@TypeOf(func) == type) { - if (@typeInfo(func) == .Struct and @hasDecl(func, "forward")) { - return fnInfo(func.forward); + block_types[2 * i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); + block_types[2 * i + 1] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); } - @compileError("Given type doesn't have a forward function: " ++ @typeName(func)); - } - const type_info = @typeInfo(@TypeOf(func)); - const err_msg = "`func` must be a function and return one or more `Tensor`. Got: "; - if (type_info != .@"fn" or type_info.@"fn".return_type == null) { - @compileError(err_msg ++ @typeName(@TypeOf(func))); - } - return type_info.@"fn"; -} -fn _BlockSign(comptime func: anytype, blk_type: BlockType) BlockSignature { - const fn_info = fnInfo(func); - const err_msg = "`func` must be a function and return one or more `Tensor`. Got: "; + const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); + const sort_block = mlir.Block.init(&block_types, &block_locs); + errdefer sort_block.deinit(); - var full_args: [fn_info.params.len]type = undefined; - const arg_start = switch (blk_type) { - .default => 1, - .no_ctx => 0, - .no_args => fn_info.params.len, - }; - var n_tensors: usize = 0; - // var n_inner_tensors: usize = 0; - inline for (fn_info.params, 0..) |arg, i| { - const ArgType = if (arg.type) |T| T else @compileError(err_msg ++ @typeName(@TypeOf(func))); - full_args[i] = ArgType; - if (i >= arg_start) { - n_tensors += staticCountTensors(ArgType) orelse @compileError("Can't use " ++ @typeName(ArgType) ++ " in an MLIR function, because it has a variable number of tensors"); + CompilationContext.current().pushBlock(sort_block); + defer CompilationContext.current().popBlock(); + + const scope = CompilationContext.current().currentScope(); + inline for (0..inputs.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, 2 * i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, 2 * i + 1) catch unreachable; } + + var result = @call(.auto, func, args ++ context); + + _ = dialects.stablehlo.return_(mlir_ctx, result.value(), .unknown(mlir_ctx)).appendTo(sort_block); + break :b sort_block; + }; + + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); } - const FullArgs = std.meta.Tuple(&full_args); - const BlkCtx = switch (blk_type) { - .default => full_args[0], - .no_ctx => void, - .no_args => FullArgs, - }; - const Args = switch (blk_type) { - .default => std.meta.Tuple(full_args[1..]), - .no_ctx => FullArgs, - .no_args => void, - }; - return .{ - .Fn = @TypeOf(func), - .BlkCtx = BlkCtx, - .Args = Args, - .FullArgs = FullArgs, - .Return = fn_info.return_type.?, - .nIn = n_tensors, - .nOut = staticCountTensors(fn_info.return_type.?) orelse @compileError("Can't use " ++ @typeName(fn_info.return_type.?) ++ " in an MLIR function, because it has a variable number of tensors"), - }; -} - -pub fn staticIsOnlyTensors(comptime T: type) bool { - if (T == Tensor) return true; - - return switch (@typeInfo(T)) { - .Array => |array_info| staticIsOnlyTensors(array_info.child), - .Pointer => |ptr_info| ptr_info.size == .One and staticIsOnlyTensors(ptr_info.child), - .Struct => |struct_info| { - inline for (struct_info.fields) |field| { - if (!staticIsOnlyTensors(field.type)) return false; - } - return true; + const sort_op = mlir.Operation.make(mlir_ctx, "stablehlo.sort", .{ + .operands = .{ .variadic = &.{&input_values} }, + .result_type_inference = true, + .blocks = &.{sort_block}, + .attributes = &.{ + .named(mlir_ctx, "dimension", mlir.integerAttribute(mlir_ctx, .i64, axis_)), + .named(mlir_ctx, "is_stable", mlir.boolAttribute(mlir_ctx, is_stable)), }, - else => false, - }; -} + .verify = true, + .location = .unknown(mlir_ctx), + }).appendTo(CompilationContext.current().currentScope().block); -pub fn staticCountTensors(comptime T: type) ?usize { - if (T == Tensor) return 1; + var result: [inputs.len]Tensor = undefined; + inline for (0..inputs.len) |i| { + result[i] = Tensor._result(inputs[i].shape(), sort_op.result(i)); + } - return switch (@typeInfo(T)) { - .array => |array_info| array_info.len * (staticCountTensors(array_info.child) orelse return null), - .pointer => |ptr_info| { - const n = staticCountTensors(ptr_info.child) orelse return null; - if (ptr_info.size != .One and n > 0) return null; - return n; - }, - .@"struct" => |struct_info| { - var count: usize = 0; - inline for (struct_info.fields) |field| { - count += staticCountTensors(field.type) orelse return null; - } - return count; - }, - else => 0, - }; -} - -/// Create a Tensor struct similar to base, keeping base tags, -/// but using mlir value and dims from the mlir operation. -pub fn fromMlirOperationWithTags(op: mlir.Operation, base: anytype) @TypeOf(base) { - const LocalContext = struct { - index: usize, - op: mlir.Operation, - }; - var context = LocalContext{ .index = 0, .op = op }; - var res = base; - meta.visit((struct { - fn cb(inner_ctx: *LocalContext, tensor: *Tensor) void { - var new = Tensor.fromMlirValue(inner_ctx.op.result(inner_ctx.index)); - stdx.debug.internalAssert(new.rank() == tensor.rank(), "expected operand result to have rank {} but got {f}", .{ tensor.rank(), new }); - // copy tags and sharding info over - // some ops can change dims eg reduceWindow, so we trust mlir here. - new._shape._tags = tensor._shape._tags; - new._shape._sharding_info = tensor._shape._sharding_info; - tensor.* = new; - inner_ctx.index += 1; - } - }).cb, &context, &res); - std.debug.assert(context.index == op.numResults()); - return res; + return result; } pub const TritonOps = struct { @@ -770,51 +272,54 @@ pub const TritonOps = struct { /// Generate an MLIR call to the given member function with the given tensors. pub fn triton(inputs: anytype, outputs: anytype, opts: TritonOps) [outputs.len]Tensor { - const ctx = CompilationContext.current(); + const mlir_ctx = CompilationContext.current().mlir_ctx; + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); - var values: [inputs.len]mlir.Value = undefined; - ctx.extractValues(&inputs, &values); - - var res_types: [outputs.len]mlir.Type = undefined; - inline for (outputs, 0..) |output, i| { - res_types[i] = mlirx.tensorType(ctx.mlirCtx(), output); + var values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + values[i] = inputs[i].value(); } - const backend_config = mlir.Attribute.dict(ctx.mlirCtx(), &.{ - .{ "name", .string(ctx.mlirCtx(), opts.name) }, - .{ "ir", .string(ctx.mlirCtx(), opts.ir) }, - .{ "grid_x", .int(ctx.mlirCtx(), .i32, opts.grid[0]) }, - .{ "grid_y", .int(ctx.mlirCtx(), .i32, opts.grid[1]) }, - .{ "grid_z", .int(ctx.mlirCtx(), .i32, opts.grid[2]) }, - .{ "num_stages", .int(ctx.mlirCtx(), .i32, opts.num_stages) }, - .{ "num_warps", .int(ctx.mlirCtx(), .i32, opts.num_warps) }, + var res_types: [outputs.len]*const mlir.Type = undefined; + inline for (outputs, 0..) |output, i| { + res_types[i] = mlir.rankedTensorType(output.dims(), mlirx.Type.fromDType(mlir_ctx, output.dtype())); + } + + const backend_config = mlir.dictionaryAttribute(mlir_ctx, &.{ + mlir.NamedAttribute.named(mlir_ctx, "name", mlir.stringAttribute(mlir_ctx, opts.name)), + mlir.NamedAttribute.named(mlir_ctx, "ir", mlir.stringAttribute(mlir_ctx, opts.ir)), + mlir.NamedAttribute.named(mlir_ctx, "grid_x", mlir.integerAttribute(mlir_ctx, .i32, opts.grid[0])), + mlir.NamedAttribute.named(mlir_ctx, "grid_y", mlir.integerAttribute(mlir_ctx, .i32, opts.grid[1])), + mlir.NamedAttribute.named(mlir_ctx, "grid_z", mlir.integerAttribute(mlir_ctx, .i32, opts.grid[2])), + mlir.NamedAttribute.named(mlir_ctx, "num_stages", mlir.integerAttribute(mlir_ctx, .i32, opts.num_stages)), + mlir.NamedAttribute.named(mlir_ctx, "num_warps", mlir.integerAttribute(mlir_ctx, .i32, opts.num_warps)), }); var operands_layouts: [inputs.len][]const usize = undefined; inline for (inputs, 0..) |input, i| { - operands_layouts[i] = minorToMajor(input.rank()); + operands_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(input.rank())).constSlice()) catch unreachable; } var results_layouts: [outputs.len][]const usize = undefined; inline for (outputs, 0..) |output, i| { - results_layouts[i] = minorToMajor(output.rank()); + results_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(output.rank())).constSlice()) catch unreachable; } - const op = dialect.stablehlo.custom_call( - ctx.mlirCtx(), + const op = dialects.stablehlo.custom_call( + mlir_ctx, &values, + &res_types, .{ .call_target_name = "__gpu$xla.gpu.triton", - .backend_config = backend_config, + .backend_config = .{ .typed_ffi = backend_config }, .has_side_effect = false, - .api_version = .typed_ffi, .operand_layouts = &operands_layouts, .result_layouts = &results_layouts, .output_operand_aliases = opts.output_operand_aliases, }, - &res_types, - ctx.mlirCtx().location(@src()), - ); + .unknown(mlir_ctx), + ).appendTo(CompilationContext.current().currentScope().block); var outputs_: [outputs.len]Tensor = undefined; inline for (outputs, 0..) |output, i| { @@ -858,15 +363,25 @@ test "triton" { } }; - const a = try zml.Buffer.fromSlice(platform, .{}, &[1]f32{1}); - const b = try zml.Buffer.fromSlice(platform, .{}, &[1]f32{3}); + const a: zml.Tensor = .init(.{}, .f32); + const b: zml.Tensor = .init(.{}, .f32); - const results = try zml.testing.compileAndCall(platform, TritonMod.forward, .{ a, b }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, TritonMod.forward, .{ a, b }, platform); + defer exe.deinit(); - var cpu_result_0 = try results[0].toHostAlloc(std.testing.allocator); - defer cpu_result_0.deinit(std.testing.allocator); - var cpu_result_1 = try results[1].toHostAlloc(std.testing.allocator); - defer cpu_result_1.deinit(std.testing.allocator); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[1]f32{1})); + defer a_buffer.deinit(); + var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), std.mem.sliceAsBytes(&[1]f32{3})); + defer b_buffer.deinit(); + + const results = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, TritonMod.forward, .{ a_buffer, b_buffer }); + defer results[0].deinit(); + defer results[1].deinit(); + + var cpu_result_0 = try results[0].toSliceAlloc(std.testing.allocator, std.testing.io); + defer cpu_result_0.free(std.testing.allocator); + var cpu_result_1 = try results[1].toSliceAlloc(std.testing.allocator, std.testing.io); + defer cpu_result_1.free(std.testing.allocator); const expected_result_a: f32 = 2.0; const expected_result_b: f32 = 3.0; @@ -875,6 +390,11 @@ test "triton" { try std.testing.expectEqual(expected_result_b, cpu_result_1.items(f32)[0]); } +pub const ScatterArgs = struct { + input: Tensor, + update: Tensor, +}; + /// Generalized version of scatter to many inputs. /// See `zml.Tensor.scatterSlices` for documentation on scatter. /// @@ -885,21 +405,54 @@ test "triton" { /// This sounds nice but in practice XLA doesn't support this well on GPU, /// and will generate slow code. In practice stick with `zml.Tensor.scatterSlices`. pub fn scatter( - comptime T: type, - comptime BlkCtx: type, - comptime update_fn: fn (BlkCtx, T, T) T, - inputs: T, - blkctx: BlkCtx, + inputs: anytype, index_tensors: anytype, - updates: T, + updates: anytype, + comptime func: anytype, + context: anytype, opts: Tensor.ScatterOpts, -) T { - const loc = @src(); - const ctx = CompilationContext.current(); +) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); - const n_inputs = meta.count(Tensor, &inputs); - const n_updates = meta.count(Tensor, &updates); - stdx.debug.assert(n_inputs == n_updates, "zml.ops.scatter expects the same number of tensors in inputs and updates, got {d} and {d}", .{ n_inputs, n_updates }); + const mlir_ctx = CompilationContext.current().mlir_ctx; + + const update_block, var result = b: { + const ArgsType = std.meta.Tuple(&[1]type{ScatterArgs} ** inputs.len); + var args: ArgsType = undefined; + var block_types: [2 * inputs.len]*const mlir.Type = undefined; + + inline for (0..inputs.len) |i| { + args[i].input = Tensor.init(.{}, inputs[i].dtype()); + args[i].update = Tensor.init(.{}, inputs[i].dtype()); + + block_types[i] = mlir.rankedTensorType(args[i].input.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].input.dtype())); + block_types[i + inputs.len] = mlir.rankedTensorType(args[i].update.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].update.dtype())); + } + + const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); + const update_block = mlir.Block.init(&block_types, &block_locs); + errdefer update_block.deinit(); + + CompilationContext.current().pushBlock(update_block); + defer CompilationContext.current().popBlock(); + + const scope = CompilationContext.current().currentScope(); + inline for (0..inputs.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), args[i].input.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].update.id, i + inputs.len) catch unreachable; + } + + var result = @call(.auto, func, args ++ context); + + var result_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + result_values[i] = result[i].value(); + } + + _ = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)).appendTo(update_block); + break :b .{ update_block, result }; + }; // Note: I was a bit lazy here, and I only look at tags on the first tensor. // we probably should check all of them. @@ -931,8 +484,8 @@ pub fn scatter( } // rewrite simple scatters to dynamicUpdateSlice. - if (T == Tensor and indices_shape.rank() == 0) { - return self.dynamicUpdateSlice(index_tensors, updates); + if (@TypeOf(inputs) == struct { Tensor } and indices_shape.rank() == 0) { + return .{self.dynamicUpdateSlice(index_tensors, update)}; } // TODO: ideally we should catch all possible scatter errors and provide nice error messages. @@ -941,80 +494,58 @@ pub fn scatter( // const n_indices_axes = update.rank() - _collectAxes(AxisKind, up_kind, .update_window).len; // stdx.debug.assert(n_indices_axe == indices_axes.len, "scatter({f}, {any}) expects 'updates' to contain all axes from 'indices', got indices={s}, updates={f}", .{ self, index_tensors, indices_axes.constSlice(), update }); - const mlir_ctx = ctx.mlirCtx(); - var _scalar: T = inputs; - meta.visit(struct { - pub fn cb(_: void, x: *Tensor) void { - x.* = .{ ._shape = Shape.init(.{}, x.dtype()), ._id = undefined }; - } - }.cb, {}, &_scalar); + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); + } - const UpdateS = BlockSign(update_fn); - const update_block, _ = ctx.makeBlock(.hermetic, UpdateS, update_fn, blkctx, .{ _scalar, _scalar }); + var updates_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..updates.len) |i| updates_values[i] = updates[i].value(); - const arena = ctx.allocator(); - const input_values = arena.alloc(mlir.Value, n_inputs) catch @panic("OOM"); - defer arena.free(input_values); - meta.collectBuf(CompilationContext.getValue, ctx, &inputs, input_values); - - const updates_values = arena.alloc(mlir.Value, n_updates) catch @panic("OOM"); - defer arena.free(updates_values); - meta.collectBuf(CompilationContext.getValue, ctx, &updates, updates_values); - - const op = dialect.stablehlo.scatter( + const op = dialects.stablehlo.scatter( mlir_ctx, - input_values, + &input_values, &.{indices.value()}, - updates_values, + &updates_values, update_block, .{ - .update_window_dims = _collectAxes(AxisKind, config.up_kind, .update_window).constSlice(), - .inserted_window_dims = _collectAxes(AxisKind, config.op_kind, .inserted_window).constSlice(), - .input_batching_dims = _collectAxes(AxisKind, config.op_kind, .batching).constSlice(), + .update_window_dims = _collectAxes(ScatterAxisKind, config.up_kind, .update_window).constSlice(), + .inserted_window_dims = _collectAxes(ScatterAxisKind, config.op_kind, .inserted_window).constSlice(), + .input_batching_dims = _collectAxes(ScatterAxisKind, config.op_kind, .batching).constSlice(), .scatter_indices_batching_dims = config.indices_batch_axes.constSlice(), .scatter_dims_to_operand_dims = config.scatter_to_operand_axes.constSlice(), .index_vector_dim = indices.rank() - 1, .indices_are_sorted = opts.indices_are_sorted, .unique_indices = opts.indices_are_unique, }, - mlir_ctx.location(loc), - ); + .unknown(mlir_ctx), + ).appendTo(CompilationContext.current().currentScope().block); - var res: T = inputs; - const LocalContext = struct { - op: mlir.Operation, - index: usize = 0, - }; - var local_context = LocalContext{ .op = op }; - meta.visit((struct { - fn cb(inner_ctx: *LocalContext, tensor: *Tensor) void { - const val = inner_ctx.op.result(inner_ctx.index); - tensor.* = Tensor._result(tensor.shape(), val); - inner_ctx.index += 1; - } - }).cb, &local_context, &res); - std.debug.assert(local_context.index == op.numResults()); - return res; + inline for (0..result.len) |i| { + result[i] = Tensor._result(inputs[i].shape(), op.result(i)); + } + + return result; } const ScatterConfig = struct { - op_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}, - up_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}, + op_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .{}, + up_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .{}, indices_batch_axes: Shape.DimsArray = .{}, scatter_to_operand_axes: Shape.DimsArray = .{}, updates_transpose: Shape.AxesArray = .{}, }; -const AxisKind = enum { batching, update_window, inserted_window, window_id }; +const ScatterAxisKind = enum { batching, update_window, inserted_window, window_id }; fn scatterConfig( op: Shape, update: Shape, - indices_per_axis: stdx.BoundedArray(Tensor, Tensor.MAX_RANK), + indices_per_axis: stdx.BoundedArray(Tensor, constants.MAX_RANK), indices_axes: Shape.TagsArray, ) ScatterConfig { - var op_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}; - var up_kind: stdx.BoundedArray(AxisKind, Tensor.MAX_RANK) = .{}; + var op_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .{}; + var up_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .{}; var indices_batch_axes: Shape.DimsArray = .{}; var scatter_to_operand_axes: Shape.DimsArray = .{}; var updates_transpose: Shape.AxesArray = .{}; @@ -1100,14 +631,18 @@ test scatterConfig { const zml = @import("zml.zig"); const platform = zml.testing.env(); - var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); + const block = mlir.Block.init(&.{}, &.{}); + comp.pushBlock(block); + defer comp.popBlock(); + const Local = struct { pub fn _idx(idx_shape: anytype) Tensor { - return Tensor.constant(idx_shape, .{ .i32 = 0 }); + return Tensor.constant(.{ .i32 = 0 }).broad(Shape.init(idx_shape, .i32)); } }; @@ -1120,8 +655,8 @@ test scatterConfig { const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); const cfg = scatterConfig(op, update, indices, coords_tags); - try std.testing.expectEqualSlices(AxisKind, &.{ .batching, .update_window }, cfg.op_kind.constSlice()); - try std.testing.expectEqualSlices(AxisKind, &.{ .batching, .window_id, .update_window }, cfg.up_kind.constSlice()); + try std.testing.expectEqualSlices(ScatterAxisKind, &.{ .batching, .update_window }, cfg.op_kind.constSlice()); + try std.testing.expectEqualSlices(ScatterAxisKind, &.{ .batching, .window_id, .update_window }, cfg.up_kind.constSlice()); } // similar, but use the normalized form where .a is no longer an explicit batching axis. @@ -1130,8 +665,8 @@ test scatterConfig { const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); const cfg = scatterConfig(op, update, indices, coords_tags); - try std.testing.expectEqualSlices(AxisKind, &.{ .inserted_window, .update_window }, cfg.op_kind.constSlice()); - try std.testing.expectEqualSlices(AxisKind, &.{ .window_id, .window_id, .update_window }, cfg.up_kind.constSlice()); + try std.testing.expectEqualSlices(ScatterAxisKind, &.{ .inserted_window, .update_window }, cfg.op_kind.constSlice()); + try std.testing.expectEqualSlices(ScatterAxisKind, &.{ .window_id, .window_id, .update_window }, cfg.up_kind.constSlice()); } } @@ -1143,11 +678,11 @@ fn scatterPrepareIndices( cfg: *ScatterConfig, op: Shape, update: Shape, - indices_per_axis: *stdx.BoundedArray(Tensor, Tensor.MAX_RANK), + indices_per_axis: *stdx.BoundedArray(Tensor, constants.MAX_RANK), indices_axes: *Shape.TagsArray, ) Tensor { var old_scatter_to_op_axes = cfg.scatter_to_operand_axes; - const batching = _collectAxes(AxisKind, cfg.op_kind, .batching); + const batching = _collectAxes(ScatterAxisKind, cfg.op_kind, .batching); for (batching.constSlice()) |batch_ax| { const id_shape = indices_per_axis.get(0).shape(); // batching requires tagging, so we're sure to have a tag here. @@ -1163,7 +698,7 @@ fn scatterPrepareIndices( // Reorder the axes so that in indices_per_axis is ordered like in op if possible. // TODO: transpose updates if needed - var indices: stdx.BoundedArray(Tensor, Tensor.MAX_RANK) = .{}; + var indices: stdx.BoundedArray(Tensor, constants.MAX_RANK) = .{}; var scatter_to_op_axes: Shape.DimsArray = .{}; while (old_scatter_to_op_axes.len > 0) { @@ -1178,13 +713,133 @@ fn scatterPrepareIndices( for (scatter_to_op_axes.constSlice(), 0..) |sc_ax, i| { if (i != sc_ax) { - log.warn("Found a slow scatter pattern, which is going to generate a while loop: scatter({f}, {any}, {f}). Because the index axes aren't the major ones in the input tensor.", .{ op, scatter_to_op_axes.constSlice(), update }); + //log.warn("Found a slow scatter pattern, which is going to generate a while loop: scatter({f}, {any}, {f}). Because the index axes aren't the major ones in the input tensor.", .{ op, scatter_to_op_axes.constSlice(), update }); break; } } return Tensor.stack(indices.constSlice(), .last, .coord); } +pub const GatherAxisKind = enum { batching, offset, collapsed, indices }; + +pub const GatherOpts = struct { indices_are_sorted: bool = false }; + +pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, opts: GatherOpts) Tensor { + const mlir_ctx = CompilationContext.current().mlir_ctx; + + stdx.debug.assert(idx_axes.len > 0, "gather expects 1 or more axes to operate one, received none. Example: `x.gather(.a, indices, .{{}})`", .{}); + for (idx_axes, 0..) |a, i| { + if (i > 0) { + stdx.debug.assert(a == idx_axes[i - 1] + 1, "gather expects 'idx_axes' to be sequential. But {any} aren't sequential in {f}", .{ idx_axes, self }); + } + } + var indices_shape = idx_per_axis[0].shape(); + for (idx_per_axis[1..]) |idx| { + if (idx.rank() > indices_shape.rank()) { + indices_shape = idx.shape(); + } + } + for (idx_per_axis) |idx| { + stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "gather indices can't be broadcasted together {any}", .{idx_per_axis}); + } + + var idx_batch_axes: Shape.DimsArray = .{}; + + var self_kind: stdx.BoundedArray(GatherAxisKind, constants.MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; + + for (self._shape.tags(), 0..self.rank()) |t, self_ax| { + const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, 1, @intCast(self_ax)); + if (indices_shape.hasTag(t)) |id_ax| { + // tag is both in self and indices -> it's a batching dim + // Note: tags are required for batching. + self_kind.buffer[self_ax] = .batching; + idx_batch_axes.appendAssumeCapacity(id_ax); + stdx.debug.assert(!is_gather_axis, "gather expects axes to appear at most twice. Axis {s} has been found both in 'self={f}', in 'idx_axes={any}' and in 'indices={f}'", .{ t, self, idx_axes, indices_shape }); + } else if (is_gather_axis) { + // we collapsed all gathered axes + self_kind.buffer[self_ax] = .collapsed; + // idx_kind.buffer[id_ax] = .indices; + } else { + self_kind.buffer[self_ax] = .offset; + } + } + + // compute res shape + var res_shape = Shape.init(.{}, self.dtype()); + var res_kind: stdx.BoundedArray(GatherAxisKind, constants.MAX_RANK) = .{}; + for (self_kind.slice(), 0..) |kind, ax_usize| { + const ax: u3 = @intCast(ax_usize); + if (ax == idx_axes[0]) { + // The first val_ax is special cause this is the place where we insert indices axes. + for (0.., indices_shape.tags(), indices_shape.dims()) |id_axis_order, id_axis, id_inserted_dim| { + const is_batching_axis = std.mem.containsAtLeastScalar(i64, idx_batch_axes.constSlice(), 1, @intCast(id_axis_order)); + // Batching axis is already in self. + if (is_batching_axis) continue; + + res_shape = res_shape.appendDim(id_inserted_dim, id_axis); + res_kind.appendAssumeCapacity(.indices); + } + } + switch (kind) { + .collapsed => continue, + else => { + res_shape = res_shape.appendDim(self.dim(ax), self._shape.tag(ax)); + res_kind.appendAssumeCapacity(kind); + }, + } + } + + // This is not a gather, but a dynamicSlice. + // Sometimes the backend recognize this pattern, but not always. + // So let us handle that. + if (indices_shape.count() == 1 and idx_axes.len == 1) { + return self.dynamicSlice1d(idx_axes[0], .{ .start = idx_per_axis[0].asScalar(), .len = 1 }).reshape(res_shape); + } + + var slice_dims: Shape.DimsArray = .{}; + for (self_kind.slice(), self.dims()) |k, d| { + slice_dims.appendAssumeCapacity(switch (k) { + .batching, .collapsed => 1, + .offset => d, + .indices => unreachable, + }); + } + + // TODO: try changing .last by other axis and see the perf impact. + const indices = Tensor.stack(idx_per_axis, .last, .coord); + // scoped_log.debug("gather --> {} {any}", .{ res_shape, res_kind.constSlice() }); + const gather_op = dialects.stablehlo.gather( + mlir_ctx, + self.value(), + indices.value(), + slice_dims.constSlice(), + .{ + .offset_dims = _collectAxes(GatherAxisKind, res_kind, .offset).constSlice(), + .collapsed_slice_dims = _collectAxes(GatherAxisKind, self_kind, .collapsed).constSlice(), + .operand_batching_dims = _collectAxes(GatherAxisKind, self_kind, .batching).constSlice(), + .start_indices_batching_dims = idx_batch_axes.constSlice(), + .start_index_map = _collectAxes(GatherAxisKind, self_kind, .collapsed).constSlice(), + .index_vector_dim = indices.axis(.coord), + .indices_are_sorted = opts.indices_are_sorted, + }, + .unknown(mlir_ctx), + ).appendTo(CompilationContext.current().currentScope().block); + + const mlir_shape = Tensor.fromMlirValue(gather_op.result(0)).shape(); + stdx.debug.assert(mlir_shape.eql(res_shape), "gather expects that batching indices appear in the same order in 'self' and 'indices', got: self={f}, indices={f}. You should transpose one or the other.", .{ self, indices }); + return Tensor._result(res_shape, gather_op.result(0)); +} + +pub fn _collectAxes(T: type, bounded_array: stdx.BoundedArray(T, constants.MAX_RANK), value: T) stdx.BoundedArray(i64, constants.MAX_RANK) { + var res: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; + for (bounded_array.constSlice(), 0..) |v, ax| { + if (v == value) { + res.appendAssumeCapacity(@intCast(ax)); + } + } + return res; +} + fn TensorOrTensorArray(comptime T: type) type { const type_info = @typeInfo(T); return switch (type_info) { @@ -1290,57 +945,60 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs: []const Shape, metadata: anytype, opts: CustomCallOptions) []Tensor { const ctx = CompilationContext.current(); + var arena = std.heap.ArenaAllocator.init(ctx.allocator); + defer arena.deinit(); - const values = ctx.allocator().alloc(mlir.Value, inputs.len) catch unreachable; - ctx.extractValues(inputs, values); + const values = arena.allocator().alloc(*const mlir.Value, inputs.len) catch unreachable; + for (0..inputs.len) |i| { + values[i] = inputs[i].value(); + } - const res_types = ctx.allocator().alloc(mlir.Type, outputs.len) catch unreachable; + const res_types = arena.allocator().alloc(*const mlir.Type, outputs.len) catch unreachable; for (outputs, 0..) |output, i| { - res_types[i] = mlirx.tensorType(ctx.mlirCtx(), output); + res_types[i] = mlir.rankedTensorType(output.dims(), mlirx.Type.fromDType(ctx.mlir_ctx, output.dtype())); } const metadata_type_info = @typeInfo(@TypeOf(metadata)); - var metadata_attributes_tuple: [metadata_type_info.@"struct".fields.len]mlir.AttrTuple = undefined; + var metadata_attributes: [metadata_type_info.@"struct".fields.len]mlir.NamedAttribute = undefined; inline for (metadata_type_info.@"struct".fields, 0..) |field, i| { - const attribute: mlir.Attribute = switch (@typeInfo(field.type)) { - .int, .comptime_int => .int(ctx.mlirCtx(), .u64, @bitCast(@field(metadata, field.name))), + const attribute: *const mlir.Attribute = switch (@typeInfo(field.type)) { + .int, .comptime_int => mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@field(metadata, field.name)))), else => @compileError("Unsupported metadata type: " ++ @typeName(field.type)), }; - metadata_attributes_tuple[i] = .{ field.name, attribute }; + metadata_attributes[i] = mlir.NamedAttribute.named(ctx.mlir_ctx, field.name, attribute); } - const backend_config = mlir.Attribute.dict(ctx.mlirCtx(), &(metadata_attributes_tuple ++ [_]mlir.AttrTuple{ - .{ "pjrt_api", .int(ctx.mlirCtx(), .u64, @bitCast(@intFromPtr(ctx._platform.pjrt_api))) }, - .{ "pjrt_client", .int(ctx.mlirCtx(), .u64, @bitCast(@intFromPtr(ctx._platform.pjrt_client))) }, + const backend_config = mlir.dictionaryAttribute(ctx.mlir_ctx, &(metadata_attributes ++ [_]mlir.NamedAttribute{ + .named(ctx.mlir_ctx, "pjrt_api", mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_api))))), + .named(ctx.mlir_ctx, "pjrt_client", mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_client))))), })); - const operands_layouts = ctx.allocator().alloc([]const usize, inputs.len) catch unreachable; + const operands_layouts = arena.allocator().alloc([]const usize, inputs.len) catch unreachable; for (inputs, 0..) |input, i| { - operands_layouts[i] = minorToMajor(input.rank()); + operands_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(input.rank())).constSlice()) catch unreachable; } - const results_layouts = ctx.allocator().alloc([]const usize, outputs.len) catch unreachable; + const results_layouts = arena.allocator().alloc([]const usize, outputs.len) catch unreachable; for (outputs, 0..) |output, i| { - results_layouts[i] = minorToMajor(output.rank()); + results_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(output.rank())).constSlice()) catch unreachable; } - const op = dialect.stablehlo.custom_call( - ctx.mlirCtx(), + const op = dialects.stablehlo.custom_call( + ctx.mlir_ctx, values, + res_types, .{ .call_target_name = target_name, - .backend_config = backend_config, + .backend_config = .{ .typed_ffi = backend_config }, .has_side_effect = opts.has_side_effect, - .api_version = .typed_ffi, .operand_layouts = operands_layouts, .result_layouts = results_layouts, .output_operand_aliases = opts.output_operand_aliases orelse &.{}, }, - res_types, - ctx.mlirCtx().location(@src()), + .unknown(ctx.mlir_ctx), ); - const outputs_ = ctx.allocator().alloc(Tensor, outputs.len) catch unreachable; + const outputs_ = ctx.arena.allocator().alloc(Tensor, outputs.len) catch unreachable; for (outputs, 0..) |output, i| { outputs_[i] = Tensor._result(output, op.result(i)); } @@ -1348,20 +1006,30 @@ fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs return outputs_; } -inline fn toI64(values: anytype) []i64 { - var res: [Tensor.MAX_RANK]i64 = undefined; - for (values, 0..) |val, i| res[i] = @intCast(val); - return res[0..values.len]; +test customCall { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); + defer comp.deinit(); + comp.activate(); + defer comp.deactivate(); + + const block = mlir.Block.init(&.{}, &.{}); + comp.pushBlock(block); + defer comp.popBlock(); + + const input = Tensor.constant(zml.DataType.bf16.constant(0)).broad(Shape.init(.{128}, .bf16)); + const output = customCall("my_custom_call", .{input}, .{zml.Shape.init(.{128}, .bf16)}, .{}, .{ + .has_side_effect = false, + .output_operand_aliases = &.{0}, + }); + + try zml.testing.expectEqualShapes(input.shape(), output.shape()); } -const _MINOR_TO_MAJOR = blk: { - var ret: [Shape.MAX_RANK]usize = undefined; - for (0..Shape.MAX_RANK) |i| { - ret[i] = @intCast(Shape.MAX_RANK - i - 1); - } - break :blk ret; -}; - -fn minorToMajor(rank: u8) []const usize { - return _MINOR_TO_MAJOR[_MINOR_TO_MAJOR.len - rank ..]; +fn toUsize(values: anytype) stdx.BoundedArray(usize, constants.MAX_RANK) { + var res: stdx.BoundedArray(usize, constants.MAX_RANK) = .{}; + for (values) |val| res.appendAssumeCapacity(@intCast(val)); + return res; } diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index f09ffa1e..459ee1e7 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -1,6 +1,5 @@ const std = @import("std"); -const async = @import("async"); const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); @@ -21,9 +20,13 @@ pub const GetCostAnalysisError = pjrt.GetCostAnalysisError; pub const Memory = pjrt.Memory; pub const MemoryStats = pjrt.MemoryStats; pub const NamedValue = pjrt.NamedValue; -pub const Profiler = pjrt.Profiler; pub const SerializeResult = pjrt.SerializeResult; pub const Stream = pjrt.Stream; +pub const ShapeSpec = pjrt.ShapeSpec; +pub const Buffer = pjrt.Buffer; +pub const Event = pjrt.Event; +pub const LoadedExecutable = pjrt.LoadedExecutable; +pub const AsyncHostToDeviceTransferManager = pjrt.AsyncHostToDeviceTransferManager; const zml = struct { pub const DataType = @import("dtype.zig").DataType; @@ -70,8 +73,9 @@ pub const Client = opaque { return .{ @ptrCast(buffer), @ptrCast(event_) }; } - pub fn deserializeAndLoad(self: *const Client, api: *const Api, bytes: []const u8) ApiError!*LoadedExecutable { - return @ptrCast(try async.callBlocking(pjrt.Client.deserializeAndLoad, .{ self.inner(), api, bytes })); + pub fn deserializeAndLoad(self: *const Client, api: *const Api, io: std.Io, bytes: []const u8) ApiError!*LoadedExecutable { + var future = io.async(pjrt.Client.deserializeAndLoad, .{ self.inner(), api, bytes }); + return @ptrCast(try future.await(io)); } pub const CreateViewOfDeviceBufferArgs = pjrt.Client.CreateViewOfDeviceBufferArgs; @@ -79,27 +83,26 @@ pub const Client = opaque { return @ptrCast(try self.inner().createViewOfDeviceBuffer(api, args)); } - fn compileSync(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { + pub fn compile(self: *const Client, api: *const Api, allocator: std.mem.Allocator, io: std.Io, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { var bytecode: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); defer bytecode.deinit(); - module.op().writeBytecodeWithConfig(&bytecode.writer, .{ .desiredEmitedVersion = 1 }) catch |err| { + module.operation().writeBytecode(.{ .desired_emit_version = 1 }, &bytecode.writer) catch |err| { log.err("failed to write module bytecode: {}", .{err}); return switch (err) { - std.Io.Writer.Error.WriteFailed => error.OutOfMemory, + error.WriteFailed => error.OutOfMemory, else => |e| e, }; }; - var serialized_buffer: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); - defer serialized_buffer.deinit(); - const stablehlo_version = blk: { if (api.stablehloCurrentVersion()) |requested_version| { - break :blk dialects.stablehlo.stablehloGetSmallerVersion(requested_version, dialects.stablehlo.getCurrentVersion()); + break :blk dialects.stablehlo.smallerVersion(requested_version, dialects.stablehlo.currentVersion()); } - break :blk dialects.stablehlo.getMinimumVersion(); + break :blk dialects.stablehlo.minimumVersion(); }; + var serialized_buffer: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); + defer serialized_buffer.deinit(); dialects.stablehlo.serializePortableArtifact(bytecode.written(), stablehlo_version, &serialized_buffer.writer) catch |err| { log.err("failed to serialize to portable artifact: {}", .{err}); return switch (err) { @@ -108,17 +111,13 @@ pub const Client = opaque { }; }; - return @ptrCast(try self.inner().compile(api, .{ + return @ptrCast(try self.inner().compile(api, io, .{ .bytecode = serialized_buffer.written(), .bytecode_format = .mlir, .compile_options_pb = compile_options_pb, })); } - pub fn compile(self: *const Client, api: *const Api, allocator: std.mem.Allocator, module: mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { - return try async.callBlocking(compileSync, .{ self, api, allocator, module, compile_options_pb }); - } - pub fn addressableMemories(self: *const Client, api: *const Api) []*const Memory { return self.inner().addressableMemories(api); } @@ -137,201 +136,11 @@ pub const Client = opaque { pub fn createUnitializedBuffer(self: *const Client, api: *const Api, args: CreateUninitializedBufferArgs) ApiError!*Buffer { return @ptrCast(try self.inner().createUninitializedBuffer(api, args)); } -}; -pub const Buffer = opaque { - pub const inner = InnerMixin(pjrt.Buffer).inner; + pub const CreateBuffersForAsyncHostToDeviceArgs = pjrt.Client.CreateBuffersForAsyncHostToDeviceArgs; - pub fn deinit(self: *Buffer, api: *const Api) void { - self.inner().deinit(api); - } - - pub fn getDevice(self: *const Buffer, api: *const Api) ApiError!*Device { - return try self.inner().getDevice(api); - } - - pub fn delete(self: *Buffer, api: *const Api) void { - self.inner().delete(api); - } - - pub fn isDeleted(self: *const Buffer, api: *const Api) bool { - return self.inner().isDeleted(api); - } - - pub fn isOnCpu(self: *const Buffer, api: *const Api) bool { - return self.inner().isOnCpu(api); - } - - pub fn memory(self: *const Buffer, api: *const Api) *const Memory { - return self.inner().memory(api); - } - - pub fn toHostBuffer(self: *const Buffer, api: *const Api, dst: []u8) ApiError!?*Event { - return @ptrCast(try self.inner().toHostBuffer(api, dst)); - } - - pub fn getElementType(self: *const Buffer, api: *const Api) zml.DataType { - return dtypeFromBufferType(self.inner().getElementType(api)); - } - - pub fn getDimensions(self: *const Buffer, api: *const Api) []const i64 { - return self.inner().getDimensions(api); - } - - pub fn getUnpaddedDimensions(self: *const Buffer, api: *const Api) ApiError![]const i64 { - return try self.inner().getUnpaddedDimensions(api); - } - - pub fn getOnDeviceSizeInBytes(self: *const Buffer, api: *const Api) ApiError!usize { - return try self.inner().getOnDeviceSizeInBytes(api); - } - - pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!*Buffer { - return @ptrCast(self.inner().copyToDevice(api, device)); - } - - pub fn copyToMemory(self: *const Buffer, api: *const Api, memory_: *const Memory) ApiError!*Buffer { - return @ptrCast(try self.inner().copyToMemory(api, memory_)); - } - - pub fn getReadyEvent(self: *const Buffer, api: *const Api) ?*Event { - return @ptrCast(self.inner().getReadyEvent(api)); - } - - pub fn getOpaqueDeviceMemoryDataPointer(self: *const Buffer, api: *const Api) ApiError!*anyopaque { - return try self.inner().getOpaqueDeviceMemoryDataPointer(api); - } -}; - -pub const Event = opaque { - pub const inner = InnerMixin(pjrt.Event).inner; - - pub fn deinit(self: *Event, api: *const Api) void { - self.inner().deinit(api); - } - - pub fn isReady(self: *const Event, api: *const Api) bool { - return self.inner().isReady(api); - } - - pub fn getEventError(self: *const Event, api: *const Api) ?*Error { - return self.inner().getEventError(api); - } - - pub fn awaitBlocking(self: *Event, api: *const Api) ApiError!void { - if (self.isReady(api)) { - return; - } - try self.inner().await(api); - } - - pub fn await(self: *Event, api: *const Api) ApiError!void { - defer self.deinit(api); - - if (self.isReady(api)) { - return; - } - - var ctx = struct { - err: ?*pjrt.Error = null, - event: async.threading.ResetEventSingle = .{}, - }{}; - - try self.inner().onReady(api, &(struct { - fn call(err: ?*pjrt.Error, user_arg: ?*anyopaque) callconv(.c) void { - const ctx_: *@TypeOf(ctx) = @ptrCast(@alignCast(user_arg.?)); - ctx_.err = err; - ctx_.event.set(); - } - }.call), &ctx); - ctx.event.wait(); - - if (ctx.err) |e| { - defer e.deinit(api); - const err_code = e.getCode(api).toApiError(); - log.err("{t} {s}", .{ err_code, e.getMessage(api) }); - return err_code; - } - } -}; - -pub const LoadedExecutable = opaque { - const inner = InnerMixin(pjrt.LoadedExecutable).inner; - - pub fn deinit(self: *LoadedExecutable, api: *const Api) void { - self.inner().deinit(api); - } - - pub fn delete(self: *LoadedExecutable, api: *const Api) void { - self.inner().delete(api); - } - - pub fn isDeleted(self: *const LoadedExecutable, api: *const Api) bool { - return self.inner().isDeleted(api); - } - - pub fn getAddressableDevices(self: *const LoadedExecutable, api: *const Api) []*const Device { - return self.inner().getAddressableDevices(api); - } - - pub const ExecuteArgs = struct { - arguments: []const [*]const *const Buffer, - num_args: usize, - results: []const [*]*Buffer, - events: []?*Event, - non_donatable_input_indices: []const i64 = &.{}, - context: ?*ExecuteContext, - }; - - pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ - .num_args = args.num_args, - .arguments = @ptrCast(args.arguments), - .results = @ptrCast(args.results), - .events = @ptrCast(args.events), - .non_donatable_input_indices = args.non_donatable_input_indices, - .context = args.context, - }); - } - - pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { - return try self.inner().getExecutable(api); - } -}; - -pub const AsyncHostToDeviceTransferManager = opaque { - const inner = InnerMixin(pjrt.AsyncHostToDeviceTransferManager).inner; - - pub fn deinit(self: *AsyncHostToDeviceTransferManager, api: *const Api) void { - self.inner().deinit(api); - } - - pub fn transferData(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize, data: []const u8, offset: i64, is_last_transfer: bool) ApiError!*Event { - return @ptrCast(try self.inner().transferData(api, buffer_index, data, offset, is_last_transfer)); - } - - pub fn retrieveBuffer(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize) ApiError!*Buffer { - return @ptrCast(try self.inner().retrieveBuffer(api, buffer_index)); - } - - pub fn device(self: *AsyncHostToDeviceTransferManager, api: *const Api) *Device { - return @ptrCast(self.inner().device(api)); - } - - pub fn bufferCount(self: *AsyncHostToDeviceTransferManager, api: *const Api) usize { - return self.inner().bufferCount(api); - } - - pub fn bufferSize(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize) usize { - return self.inner().bufferSize(api, buffer_index); - } - - pub fn setBufferError(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize, error_code: ErrorCode, error_message: []const u8) void { - self.inner().setBufferError(api, buffer_index, error_code, error_message); - } - - pub fn addMetadata(self: *AsyncHostToDeviceTransferManager, api: *const Api, transfer_metadata: []const NamedValue) void { - return self.inner().addMetadata(api, transfer_metadata); + pub fn createBuffersForAsyncHostToDevice(self: *const Client, api: *const Api, args: CreateBuffersForAsyncHostToDeviceArgs) ApiError!*AsyncHostToDeviceTransferManager { + return @ptrCast(try self.inner().createBuffersForAsyncHostToDevice(api, args)); } }; diff --git a/zml/platform.zig b/zml/platform.zig index c5872a57..3a26d30f 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -4,16 +4,12 @@ const runtimes = @import("runtimes"); pub const Target = runtimes.Platform; const stdx = @import("stdx"); +const Exe = @import("exe.zig").Exe; const pjrt = @import("pjrtx.zig"); +const zml = @import("zml.zig"); const log = std.log.scoped(.zml); -pub const available_targets = std.enums.values(Target); - -test { - std.testing.refAllDecls(@This()); -} - pub const CompilationOptions = struct { xla_dump_to: ?[]const u8 = null, xla_dump_fusion_visualization: bool = false, @@ -23,6 +19,29 @@ pub const CompilationOptions = struct { device_memory_size: u64 = 0, }; +fn StaticPlatformMap(comptime E: type, comptime T: type) type { + const tag_count = @typeInfo(E).@"enum".fields.len; + var struct_field_names: [tag_count][]const u8 = undefined; + var struct_field_types: [tag_count]type = @splat(T); + const is_optional = @typeInfo(T) == .optional; + const default_value_ptr: ?*const anyopaque = if (is_optional) @ptrCast(&@as(T, null)) else null; + var struct_field_attrs: [tag_count]std.builtin.Type.StructField.Attributes = @splat(.{ .default_value_ptr = default_value_ptr }); + inline for (@typeInfo(E).@"enum".fields, 0..) |f, i| struct_field_names[i] = f.name; + return @Struct(.auto, null, &struct_field_names, &struct_field_types, &struct_field_attrs); +} + +var api_map: StaticPlatformMap(Target, ?*const pjrt.Api) = .{}; + +fn loadOrGetApi(target: Target, io: std.Io) !*const pjrt.Api { + return switch (target) { + inline else => |tag| @field(api_map, @tagName(tag)) orelse b: { + const api = try runtimes.load(tag, io); + @field(api_map, @tagName(tag)) = api; + break :b api; + }, + }; +} + pub const Platform = struct { target: Target, pjrt_api: *const pjrt.Api, @@ -35,9 +54,37 @@ pub const Platform = struct { compilation_options: CompilationOptions = .{}, pub const MAX_NUM_DEVICES: u8 = if (runtimes.isEnabled(.tpu)) 32 else 8; - pub const CreateOptions = _CreateOptions; - pub fn init(target: Target, api: *const pjrt.Api, options: CreateOptions) !Platform { + pub const Device = struct { + pub const Iterator = struct { + api: *const pjrt.Api, + devices: []const *const pjrt.Device, + current: usize = 0, + + pub fn next(self: *Iterator) ?Device { + if (self.current >= self.devices.len) { + return null; + } + defer self.current += 1; + return .{ + .api = self.api, + .device = self.devices[self.current], + }; + } + }; + + api: *const pjrt.Api, + device: *const pjrt.Device, + + pub fn format(self: Device, writer: *std.Io.Writer) std.Io.Writer.Error!void { + const description = self.device.getDescription(self.api); + try writer.print("{s} ({s})", .{ description.getKind(self.api), description.debugString(self.api) }); + } + }; + + pub fn init(target: Target, io: std.Io, options: CreateOptions) !Platform { + const api = try loadOrGetApi(target, io); + var named_values_buf: [16]pjrt.NamedValue = undefined; const pjrt_client = try pjrt.Client.init(api, options.toNamedValues(target, &named_values_buf)); const true_num_devices = pjrt_client.getAddressableDevices(api).len; @@ -52,6 +99,40 @@ pub const Platform = struct { }; } + pub fn auto(io: std.Io, options: CreateOptions) !Platform { + const ordered_targets: []const Target = &.{ + .tpu, + .neuron, + .rocm, + .cuda, + .cpu, + }; + return for (ordered_targets) |target| { + break init(target, io, options) catch continue; + } else error.Unavailable; + } + + pub fn devicesIterator(self: Platform) Device.Iterator { + return .{ + .api = self.pjrt_api, + .devices = self.getDevices(), + }; + } + + pub fn formatWithDevices(self: Platform, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("Platform: {s}\n", .{@tagName(self.target)}); + const devices = self.getDevices(); + try writer.print("Devices total={d}\n", .{devices.len}); + for (devices, 0..) |device, i| { + const description = device.getDescription(self.pjrt_api); + try writer.print("\t#{d}: {s}\n", .{ i, description.getKind(self.pjrt_api) }); + } + } + + pub fn fmtVerbose(self: Platform) std.fmt.Alt(Platform, formatWithDevices) { + return .{ .data = self }; + } + pub fn getDevices(self: Platform) []const *const pjrt.Device { const all_devices = self.pjrt_client.getAddressableDevices(self.pjrt_api); if (all_devices.len > MAX_NUM_DEVICES) { @@ -108,7 +189,6 @@ pub const Platform = struct { } test memoryForDevice { - const zml = @import("zml.zig"); const platform = zml.testing.env(); const memory_fields = @typeInfo(pjrt.Memory.Kind).@"enum".fields; inline for (memory_fields) |field| { @@ -124,15 +204,50 @@ pub const Platform = struct { const device = platform.getDevices()[device_id]; return device.memoryStats(platform.pjrt_api) catch .zeroes; } + + pub fn compile( + self: Platform, + allocator: std.mem.Allocator, + io: std.Io, + model_: anytype, + comptime func: std.meta.DeclEnum(@TypeOf(model_)), + args: stdx.meta.Tail(stdx.meta.FnArgs(@field(@TypeOf(model_), @tagName(func)))), + ) !Exe { + return self.compileFn( + allocator, + io, + @field(@TypeOf(model_), @tagName(func)), + .{model_} ++ args, + ); + } + + pub fn compileModel(self: Platform, allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, model: stdx.meta.Head(stdx.meta.FnArgs(func)), args: stdx.meta.Tail(stdx.meta.FnArgs(func))) !Exe { + return self.compileFn(allocator, io, func, .{model} ++ args); + } + + pub fn compileFn(self: Platform, allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, args: stdx.meta.FnArgs(func)) !Exe { + return zml.module.compile(allocator, io, func, args, self); + } + + pub fn format(self: @This(), writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("{s} {{ ", .{@tagName(self.target)}); + const devices = self.getDevices(); + for (0..devices.len) |i| { + const description = devices[i].getDescription(self.pjrt_api); + try writer.print("{s}(\"{s}\")", .{ description.toString(self.pjrt_api), description.getKind(self.pjrt_api) }); + if (i < devices.len - 1) try writer.writeAll(", "); + } + try writer.writeAll(" }"); + } }; -const _CreateOptions = struct { +pub const CreateOptions = struct { cpu: Cpu = .{ .device_count = 4 }, // bump memory fraction from XLA defaults of 75% to 90%. // Even on a 8GB GPU it should leave enough space for the Cuda driver // https://github.com/openxla/xla/blob/3e87afa11a865cf91137522492918ad18bfe5b7c/xla/pjrt/plugin/xla_gpu/xla_gpu_allocator_config.h#L25-L60 - cuda: Cuda = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.90 } } }, + cuda: Cuda = .{ .allocator = .{ .async = .{ .preallocate = true, .memory_fraction = 0.90 } } }, rocm: struct {} = .{}, tpu: struct {} = .{}, neuron: struct {} = .{}, @@ -193,14 +308,14 @@ const _CreateOptions = struct { } }; - pub fn toNamedValues(self: _CreateOptions, target: Target, out: []pjrt.NamedValue) []pjrt.NamedValue { + pub fn toNamedValues(self: CreateOptions, target: Target, out: []pjrt.NamedValue) []pjrt.NamedValue { var values = std.ArrayList(pjrt.NamedValue).fromOwnedSlice(out); values.shrinkRetainingCapacity(0); switch (target) { .cpu => self.cpu.writeNamedValues(&values), .cuda => self.cuda.writeNamedValues(&values), inline else => |t| { - stdx.debug.assertComptime(@hasField(_CreateOptions, @tagName(t)), "zml.platform.CreateOptions doesn't list target {s}", .{@tagName(t)}); + stdx.debug.assertComptime(@hasField(CreateOptions, @tagName(t)), "zml.platform.CreateOptions doesn't list target {s}", .{@tagName(t)}); const options = @field(self, @tagName(t)); stdx.debug.assertComptime(@sizeOf(@TypeOf(options)) == 0, "zml.platform.CreateOptions.{s} is discarded", .{@tagName(t)}); }, diff --git a/zml/posix.zig b/zml/posix.zig deleted file mode 100644 index c826845f..00000000 --- a/zml/posix.zig +++ /dev/null @@ -1,16 +0,0 @@ -const std = @import("std"); -const c = @import("c"); - -pub fn madvise(ptr: [*]align(std.heap.page_size_min) u8, length: usize, advice: u32) std.posix.MadviseError!void { - switch (std.posix.errno(c.madvise(ptr, @intCast(length), @intCast(advice)))) { - .SUCCESS => return, - .ACCES => return error.AccessDenied, - .AGAIN => return error.SystemResources, - .BADF => unreachable, // The map exists, but the area maps something that isn't a file. - .INVAL => return error.InvalidSyscall, - .IO => return error.WouldExceedMaximumResidentSetSize, - .NOMEM => return error.OutOfMemory, - .NOSYS => return error.MadviseUnavailable, - else => |err| return std.posix.unexpectedErrno(err), - } -} diff --git a/zml/quantization.zig b/zml/quantization.zig deleted file mode 100644 index 90a35bda..00000000 --- a/zml/quantization.zig +++ /dev/null @@ -1,101 +0,0 @@ -const std = @import("std"); -const Allocator = std.mem.Allocator; - -const zml = @import("zml.zig"); -const module = zml.module; - -// TODO add tests, use modern zml - -pub fn Q4_0(comptime dtype: zml.DataType) type { - return struct { - const Self = @This(); - - const QuantType = zml.io.gguf.quants.QuantType.q4_0; - - quant_buffer: zml.Tensor, - - pub fn compile( - allocator: Allocator, - ctx: *zml.Context, - input: zml.Tensor, - shape: zml.Shape, - ) !module.CompiledModule(Self.forward) { - std.debug.assert(input.dtype() == .u8); - std.debug.assert(input.rank() == 1); - return module.compile( - allocator, - ctx, - Self.forward, - Self{ .quant_buffer = input }, - .{shape}, - ) catch unreachable; - } - - /// Each block is composed of a f16 scale and 32 4-bit values. - const block_stride = 18; - - pub fn forward(self: Self, shape: zml.Shape) zml.Tensor { - const input = self.quant_buffer; - const block_count: u63 = @intCast(@divExact(input.dim(0), block_stride)); - const scales = extractScales(block_count, input); - const weights = extractWeights(block_count, input); - - return scales.broadcast(weights.shape(), &.{0}) - .mul(weights) - .convert(dtype) - .reshape(shape); - } - - pub fn extractScales(block_count: u63, input: zml.Tensor) zml.Tensor { - // The goal here is to get the first two bytes of every 18-bytes block in the input. For that, - // we generate a list of indices that we will use to gather from the input. - - // indices1 is the offsets of the scale bytes, repeated block_count times. - const indices1 = zml.Tensor.arange(.{ .start = 0, .end = 2 }, .i32).repeat1d(0, block_count); - - // indices2 is the offsets of the blocks, repeated for each scale byte, repeated block_count times. - const indices2 = zml.Tensor.arange(.{ .start = 0, .end = block_stride * block_count, .step = block_stride }, .i32) - .reshape(.{ block_count, 1 }).broadcastLeft(zml.Shape.init(.{ block_count, 2 }, .i32)).reshape(.{2 * block_count}); - - // indices is the sum of the two, which is the offsets to all the bytes we are interested in. - const indices = indices1.add(indices2); - - // We select the values we are interested in with the indices, group them by pair and bitcast them to f16, then convert them to f32. - const scales = input.gather_(&.{0}, &.{indices}, .{ .indices_are_sorted = true }).reshape(.{ block_count, 2 }).bitCast(.f16).convert(.f32); - - return scales; - } - - pub fn extractWeights(block_count: u63, input: zml.Tensor) zml.Tensor { - // The goal here is to get everything but the first two bytes of every 18-bytes block in the input. For that, - // we generate a list of indices that we will use to gather from the input. - - // indices1 is the offsets of the data bytes, repeated block_count times. - const indices1 = zml.Tensor.arange(.{ .start = 2, .end = 18 }, .i32).repeat1d(0, block_count); - - // indices2 is the offsets of the blocks, repeated for each data byte, repeated block_count times. - const indices2 = zml.Tensor.arange(.{ .start = 0, .end = block_stride * block_count, .step = block_stride }, .i32) - .reshape(.{ block_count, 1 }).broadcastLeft(zml.Shape.init(.{ block_count, 16 }, .i32)).reshape(.{16 * block_count}); - - // indices is the sum of the two, which is the offsets to all the bytes we are interested in. - const indices = indices1.add(indices2); - - // NOTE(Corendos): i4 is not supported by bitcast convert, so we need the following workaround. - - // We select the values we are interested in with the indices, these are our quantized_weights. - const quantized_weights = input.gather_(&.{0}, &.{indices}, .{ .indices_are_sorted = true }); - const lb_weights = quantized_weights - .logical(.And, zml.Tensor.constant(.{16 * block_count}, zml.Data.init(.u8, 0xf))) - .bitCast(.i8); - const hb_weights = quantized_weights - .shiftRightLogical(zml.Tensor.constant(.{16 * block_count}, zml.Data.init(.u8, 4))).bitCast(.i8); - const weights = zml.Tensor.concatenate( - &.{ lb_weights.reshape(.{ block_count, 16 }), hb_weights.reshape(.{ block_count, 16 }) }, - 1, - ) - .sub(zml.Tensor.constant(.{ block_count, 32 }, zml.Data.init(.i8, 8))) - .convert(.f32); - return weights; - } - }; -} diff --git a/zml2/safetensors.zig b/zml/safetensors.zig similarity index 100% rename from zml2/safetensors.zig rename to zml/safetensors.zig diff --git a/zml/shape.zig b/zml/shape.zig index 564f33b5..56c724e0 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -4,35 +4,161 @@ const builtin = @import("builtin"); const stdx = @import("stdx"); +const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; -const EnumLiteral = @TypeOf(.enum_literal); - const log = std.log.scoped(.shape); -test { - std.testing.refAllDecls(Shape); -} - /// Represent the shape of a tensor. pub const Shape = struct { pub const MAX_RANK: u8 = 8; + pub const PartitionSpec = union(enum) { + axis: Tag, + open, + replicated, + unknown, + + pub fn init(comptime value: anytype) PartitionSpec { + const T = @TypeOf(value); + + if (T == PartitionSpec) { + return value; + } + + if (comptime T == @EnumLiteral()) { + const name = @tagName(value); + inline for (std.meta.fields(PartitionSpec)) |field| { + if (field.type == void and std.mem.eql(u8, field.name, name)) { + return @field(PartitionSpec, field.name); + } + } + } + + if (comptime isTagConvertible(T)) { + const tag_ = Shape.toTag(value); + + return .{ .axis = tag_ }; + } + + stdx.debug.compileError("PartitionSpec.init expected a tag, got: {any}", .{T}); + } + + test "PartitionSpec.init" { + const spec_axis: PartitionSpec = .{ .axis = Shape.toTag(.a) }; + try testing.expectEqual(spec_axis, PartitionSpec.init(.a)); + + const spec_axis_open: PartitionSpec = .open; + try testing.expect(spec_axis_open.eql(.init(.open))); + } + + pub fn eql(self: PartitionSpec, other: PartitionSpec) bool { + if (@as(u4, @intFromEnum(self)) != @as(u4, @intFromEnum(other))) { + return false; + } + + switch (self) { + .axis => |t1| { + const t2 = other.axis; + return std.mem.eql(u8, std.mem.span(t1), std.mem.span(t2)); + }, + else => return true, + } + } + + test "PartitionSpec.eql" { + const spec_axis: PartitionSpec = .init(.a); + const spec_axis_b: PartitionSpec = .init(.b); + const spec_open: PartitionSpec = .open; + const spec_replicated: PartitionSpec = .replicated; + const spec_unknown: PartitionSpec = .unknown; + + try testing.expect(spec_axis.eql(.{ .axis = Shape.toTag(.a) })); + try testing.expect(!spec_axis.eql(spec_axis_b)); + try testing.expect(!spec_axis.eql(.open)); + try testing.expect(!spec_axis.eql(.replicated)); + try testing.expect(!spec_axis.eql(.unknown)); + + try testing.expect(spec_open.eql(.open)); + try testing.expect(!spec_open.eql(spec_axis)); + try testing.expect(!spec_open.eql(.replicated)); + try testing.expect(!spec_open.eql(.unknown)); + + try testing.expect(spec_replicated.eql(.replicated)); + try testing.expect(!spec_replicated.eql(spec_axis)); + try testing.expect(!spec_replicated.eql(.open)); + try testing.expect(!spec_replicated.eql(.unknown)); + + try testing.expect(spec_unknown.eql(.unknown)); + try testing.expect(!spec_unknown.eql(spec_axis)); + try testing.expect(!spec_unknown.eql(.open)); + try testing.expect(!spec_unknown.eql(.replicated)); + } + + pub fn isClosed(self: PartitionSpec) bool { + return switch (self) { + .axis => true, + .open => false, + .replicated => true, + .unknown => false, + }; + } + + test isClosed { + const spec_axis: PartitionSpec = .init(.a); + try testing.expect(spec_axis.isClosed()); + + const spec_open: PartitionSpec = .open; + try testing.expect(!spec_open.isClosed()); + + const spec_replicated: PartitionSpec = .replicated; + try testing.expect(spec_replicated.isClosed()); + + const spec_unknown: PartitionSpec = .unknown; + try testing.expect(!spec_unknown.isClosed()); + } + + pub fn toTag(self: PartitionSpec) Tag { + return switch (self) { + .axis => |t| return t, + .open => TagOpen, + .replicated => TagReplicated, + .unknown => TagUnknown, + }; + } + + test "PartitionSpec.toTag" { + const spec_axis: PartitionSpec = .init(.a); + try testing.expect(spec_axis.toTag() == Shape.toTag(.a)); + + const spec_open: PartitionSpec = .open; + try testing.expect(spec_open.toTag() == TagOpen); + + const spec_replicated: PartitionSpec = .replicated; + try testing.expect(spec_replicated.toTag() == TagReplicated); + + const spec_unknown: PartitionSpec = .unknown; + try testing.expect(spec_unknown.toTag() == TagUnknown); + } + }; + pub const Tag = [*:0]const u8; pub const TagUnknown = "_".ptr; const TagLast = "last".ptr; + const TagReplicated = "replicated".ptr; + const TagOpen = "open".ptr; - pub const DimsArray = stdx.BoundedArray(i64, MAX_RANK); - pub const TagsArray = stdx.BoundedArray(Tag, MAX_RANK); - pub const AxesArray = stdx.BoundedArray(u3, MAX_RANK); - pub const ShardingInfo = @Vector(MAX_RANK, bool); + pub const DimsArray = stdx.BoundedArray(i64, constants.MAX_RANK); + pub const TagsArray = stdx.BoundedArray(Tag, constants.MAX_RANK); + pub const AxesArray = stdx.BoundedArray(u3, constants.MAX_RANK); + pub const PartitionArray = stdx.BoundedArray(PartitionSpec, constants.MAX_RANK); - const UnknownTags: TagsArray = .{ .len = 0, .buffer = [_]Tag{TagUnknown} ** MAX_RANK }; + const UnknownTags: TagsArray = .{ .len = 0, .buffer = [_]Tag{TagUnknown} ** constants.MAX_RANK }; _dtype: DataType, _dims: DimsArray = .{}, _tags: TagsArray = UnknownTags, - _sharding_info: ShardingInfo = @splat(false), + _partitioning: PartitionArray = .{}, pub fn parseDimensions(v: anytype) struct { DimsArray, TagsArray } { const T = @TypeOf(v); @@ -58,7 +184,7 @@ pub const Shape = struct { const fv = @field(v, field.name); if (comptime stdx.meta.isInteger(field.type)) { dims_.appendAssumeCapacity(@intCast(fv)); - } else if (@TypeOf(fv) == EnumLiteral and comptime isAutoDim(fv)) { + } else if (@TypeOf(fv) == @EnumLiteral() and comptime isAutoDim(fv)) { dims_.appendAssumeCapacity(-1); } else { stdx.debug.compileError("Field {s} should be an integer or an auto dimension, got {}", .{ field.name, field.type }); @@ -115,7 +241,7 @@ pub const Shape = struct { pub fn parseTags(v: anytype) TagsArray { const T = @TypeOf(v); - stdx.debug.assertComptime(stdx.meta.isTupleOf(T, EnumLiteral), "Wrong type, got {}. Expected .{{ .a, .b }}", .{T}); + stdx.debug.assertComptime(stdx.meta.isTupleOf(T, @EnumLiteral()), "Wrong type, got {}. Expected .{{ .a, .b }}", .{T}); var tags_ = TagsArray.init(0) catch unreachable; inline for (v) |field| { tags_.appendAssumeCapacity(toTag(field)); @@ -129,6 +255,8 @@ pub const Shape = struct { pub fn init(dimz: anytype, dt: DataType) Shape { var res: Shape = .{ ._dtype = dt }; res._dims, res._tags = parseDimensions(dimz); + res._partitioning.appendNTimes(.unknown, res._dims.len) catch @panic("Rank too large"); + return res; } @@ -145,6 +273,7 @@ pub const Shape = struct { }; res._tags.append(TagUnknown) catch unreachable; } + res._partitioning.appendNTimes(.unknown, rank_) catch @panic("Rank too large"); return res; } @@ -153,17 +282,17 @@ pub const Shape = struct { } pub fn rank(self: Shape) u4 { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return @intCast(self._dims.len); } pub fn dim(self: Shape, ax: anytype) i64 { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return self._dims.get(self.axis(ax)); } pub fn dims(self: *const Shape) []const i64 { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return self._dims.constSlice(); } @@ -173,29 +302,29 @@ pub const Shape = struct { fn isTagConvertible(comptime T: type) bool { return switch (T) { - EnumLiteral => true, + @EnumLiteral() => true, std.builtin.Type.StructField => true, Tag => true, else => false, }; } - fn toTag(v: anytype) Tag { + pub fn toTag(v: anytype) Tag { const T = @TypeOf(v); return switch (T) { - EnumLiteral => @tagName(v).ptr, + @EnumLiteral() => @tagName(v).ptr, std.builtin.Type.StructField => v.name.ptr, Tag => v, - else => stdx.debug.compileError("Shape tag should be an EnumLiteral, a Shape.Tag or a StructField, got {}", .{T}), + else => stdx.debug.compileError("Shape tag should be an @EnumLiteral(), a Shape.Tag or a StructField, got {}", .{T}), }; } - inline fn ensureDimsAndTagsAreSync(self: Shape) void { - stdx.debug.assert(self._dims.len == self._tags.len, "Tags and dims have diverged! dims={d} tags={d}", .{ self._dims.len, self._tags.len }); + inline fn ensureAttributesAreSync(self: Shape) void { + stdx.debug.assert(self._dims.len == self._tags.len and self._dims.len == self._partitioning.len, "Tags, dims and partitioning have diverged! dims={d} tags={d} partitioning={d}", .{ self._dims.len, self._tags.len, self._partitioning.len }); } pub fn tag(self: Shape, ax: anytype) Tag { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return self._tags.get(self.axis(ax)); } @@ -215,7 +344,7 @@ pub const Shape = struct { } pub fn tags(self: *const Shape) []const Tag { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return self._tags.constSlice(); } @@ -226,7 +355,7 @@ pub const Shape = struct { pub fn hasTags(self: Shape, tagz: anytype) bool { const T = @TypeOf(tagz); - if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, EnumLiteral)) { + if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, @EnumLiteral())) { for (tagz) |t| { if (self.hasTag(t) == null) { return false; @@ -235,7 +364,7 @@ pub const Shape = struct { return true; } - if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, EnumLiteral)) { + if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, @EnumLiteral())) { inline for (tagz) |t| { if (self.hasTag(t) == null) { return false; @@ -255,7 +384,7 @@ pub const Shape = struct { } pub fn axis(self: Shape, axis_: anytype) u3 { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); const T = @TypeOf(axis_); if (comptime stdx.meta.isInteger(T)) { @@ -270,7 +399,7 @@ pub const Shape = struct { } pub fn axes(self: Shape, axes_: anytype) AxesArray { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); const T = @TypeOf(axes_); @@ -398,9 +527,12 @@ pub const Shape = struct { } else { try writer.print("{d}", .{d}); } - if (self._sharding_info[i]) { - try writer.writeByte('!'); + + const part = self._partitioning.get(i); + if (part.toTag() != TagUnknown) { + try writer.print("/{s}", .{part.toTag()}); } + need_comma = true; } if (need_comma) try writer.writeByte(','); @@ -413,13 +545,6 @@ pub const Shape = struct { // Already the right shape if (std.mem.eql(i64, self.dims(), other.dims())) return true; - if (std.mem.eql(Tag, self.tags(), other.tags())) { - for (0..self.rank()) |i| { - if (self.dim(i) != 1 and self.dim(i) != other.dim(i)) return false; - } - return true; - } - // Non ambiguous broadcasting // TODO: broad is error prone because of this: // it will happily broadcast .{ .a = 10, .b = 1 } to .{ .b = 10, .a = 5 } @@ -441,6 +566,7 @@ pub const Shape = struct { pub fn reshape(self: Shape, new_shape_: anytype) Shape { var new_shape: Shape = .{ ._dtype = self.dtype() }; new_shape._dims, new_shape._tags = parseDimensions(new_shape_); + new_shape._partitioning.appendNTimes(.unknown, new_shape._dims.len) catch @panic("Rank too large"); new_shape.inferMissingAxis(self.count()); stdx.debug.assert(self.count() == new_shape.count(), "Can't reshape {any} to {any}", .{ self.dims(), new_shape.dims() }); return new_shape; @@ -507,7 +633,7 @@ pub const Shape = struct { /// ``` pub fn insert(self: Shape, axis_: anytype, dimz: anytype) Shape { const dims_, const tags_ = parseDimensions(dimz); - const ax = if (@TypeOf(axis_) == EnumLiteral and axis_ == .last) + const ax = if (@TypeOf(axis_) == @EnumLiteral() and axis_ == .last) self.rank() else self.axis(axis_); @@ -515,6 +641,11 @@ pub const Shape = struct { var res = self; res._dims.insertSlice(ax, dims_.constSlice()) catch unreachable; res._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; + + for (0..dims_.len) |_| { + res._partitioning.insert(ax, .unknown) catch unreachable; + } + return res; } @@ -525,9 +656,9 @@ pub const Shape = struct { } pub fn insertTag(self: Shape, axis_: anytype, d: i64, tag_: anytype) Shape { - stdx.debug.assert(self.rank() < MAX_RANK - 1, "Can't insert new axis in {f}, it's already at max rank.", .{self}); + stdx.debug.assert(self.rank() < constants.MAX_RANK - 1, "Can't insert new axis in {f}, it's already at max rank.", .{self}); - const ax = if (@TypeOf(axis_) == EnumLiteral and axis_ == .last) + const ax = if (@TypeOf(axis_) == @EnumLiteral() and axis_ == .last) self.rank() else self.axis(axis_); @@ -535,6 +666,8 @@ pub const Shape = struct { var res = self; res._dims.insert(ax, d) catch unreachable; res._tags.insert(ax, toTag(tag_)) catch unreachable; + res._partitioning.insert(ax, .unknown) catch unreachable; + return res; } @@ -543,6 +676,7 @@ pub const Shape = struct { const dims_, const tags_ = parseDimensions(v); res._dims.appendSliceAssumeCapacity(dims_.constSlice()); res._tags.appendSliceAssumeCapacity(tags_.constSlice()); + res._partitioning.appendNTimesAssumeCapacity(.unknown, dims_.len); return res; } @@ -564,6 +698,7 @@ pub const Shape = struct { var res = self; res._dims.appendAssumeCapacity(d); res._tags.appendAssumeCapacity(if (tag_) |t| t else TagUnknown); + res._partitioning.appendAssumeCapacity(.unknown); return res; } @@ -572,14 +707,15 @@ pub const Shape = struct { const a = self.axis(axis_); _ = res._dims.orderedRemove(a); _ = res._tags.orderedRemove(a); + _ = res._partitioning.orderedRemove(a); return res; } pub const drop = remove; test remove { - try std.testing.expectEqualSlices(i64, &.{ 10, 12 }, Shape.init(.{ 10, 11, 12 }, .f32).remove(1).dims()); - try std.testing.expectEqualSlices(i64, &.{ 10, 11, 12 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).remove(-1).dims()); + try testing.expectEqualSlices(i64, &.{ 10, 12 }, Shape.init(.{ 10, 11, 12 }, .f32).remove(1).dims()); + try testing.expectEqualSlices(i64, &.{ 10, 11, 12 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).remove(-1).dims()); } pub fn removeMany(self: Shape, axes_: anytype) Shape { @@ -597,20 +733,22 @@ pub const Shape = struct { sh._dims.buffer[res_ax] = self._dims.buffer[ax]; sh._tags.buffer[res_ax] = self._tags.buffer[ax]; + sh._partitioning.buffer[res_ax] = self._partitioning.buffer[ax]; res_ax += 1; } sh._dims.len = rk - to_remove.len; sh._tags.len = rk - to_remove.len; + sh._partitioning.len = rk - to_remove.len; return sh; } test removeMany { - try std.testing.expectEqualSlices( + try testing.expectEqualSlices( i64, &.{12}, Shape.init(.{ 10, 11, 12 }, .f32).removeMany(.{ 0, 1 }).dims(), ); - try std.testing.expectEqualSlices( + try testing.expectEqualSlices( i64, &.{ 10, 11 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).removeMany(.{ -1, -2 }).dims(), @@ -621,10 +759,16 @@ pub const Shape = struct { std.debug.assert(self.rank() == permutations.len); const permutations_ = self.axes(permutations); var res = self; + const original_dims = self._dims; + const original_tags = self._tags; + const original_parts = self._partitioning; + for (permutations_.constSlice(), 0..) |permutation, i| { - res._dims.set(i, self.dim(permutation)); - res._tags.set(i, self.tag(permutation)); + res._dims.set(i, original_dims.get(permutation)); + res._tags.set(i, original_tags.get(permutation)); + res._partitioning.set(i, original_parts.get(permutation)); } + return res; } @@ -640,6 +784,10 @@ pub const Shape = struct { Shape.init(.{ .a = 10, .b = 11, .c = 12, .d = 13 }, .f32).transpose(.{ 0, 2, 1, 3 }), ), ); + + const shape = Shape.init(.{ .a = 10, .c = 12, .b = 11 }, .f32).withPartitioning(.{ .a = .batch, .c = .colors, .b = .open }).transpose(.{ 0, 2, 1 }); + try testing.expect(shape.eqlWithTags(Shape.init(.{ .a = 10, .b = 11, .c = 12 }, .f32))); + try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .open, .init(.colors) }, shape._partitioning.constSlice()); } /// Tag each ax of this shape with tags from a tuple. @@ -652,7 +800,7 @@ pub const Shape = struct { var res = self; - if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, EnumLiteral)) { + if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, @EnumLiteral())) { stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {any}", .{ self, tagz }); for (tagz, 0..) |tag_, i| { res._tags.set(i, toTag(tag_)); @@ -660,7 +808,7 @@ pub const Shape = struct { return res; } - if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, EnumLiteral)) { + if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, @EnumLiteral())) { stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {}", .{ self, tagz }); inline for (tagz, 0..) |tag_, i| { res._tags.set(i, toTag(tag_)); @@ -699,7 +847,7 @@ pub const Shape = struct { var res = self; - if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, EnumLiteral)) { + if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, @EnumLiteral())) { stdx.debug.assert(tagz.len <= self.rank(), "Too many tags for shape {f}, got {any}", .{ self, tagz }); for (tagz, self.rank() - tagz.len..) |tag_, i| { res._tags.set(i, toTag(tag_)); @@ -707,7 +855,7 @@ pub const Shape = struct { return res; } - if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, EnumLiteral)) { + if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, @EnumLiteral())) { stdx.debug.assert(tagz.len <= self.rank(), "Too many tags for shape {f}, got {}", .{ self, tagz }); inline for (tagz, self.rank() - tagz.len..) |tag_, i| { res._tags.set(i, toTag(tag_)); @@ -748,16 +896,233 @@ pub const Shape = struct { return res; } - pub fn withSharding(self: Shape, axes_: anytype) Shape { + pub fn withPartitioning(self: Shape, specs: anytype) Shape { + const T = @TypeOf(specs); + + var res = self.withDefaultPartitioning(); // todo add test for this new change + + if (stdx.meta.isStruct(T)) { + inline for (std.meta.fields(T)) |field| { + const partition_axis = @field(specs, field.name); + const axis_ = res.axisFromTagMaybe(toTag(field)); + + if (axis_) |ax| { + res._partitioning.set(ax, PartitionSpec.init(partition_axis)); + } else { + stdx.debug.panic("Partitioning axis {s} not found", .{field.name}); + } + } + } else { + stdx.debug.panic("Expected a struct of enum literals, got: {any}", .{T}); + } + + // Check that no mesh axis is used to partition multiple tensor dimensions. + var used_mesh_axes: stdx.BoundedArray(Tag, constants.MAX_RANK) = .{ .len = 0 }; + + for (res._partitioning.constSlice()) |spec| { + const axis_tag = spec.toTag(); + + if (std.mem.indexOfScalar(Tag, used_mesh_axes.constSlice(), axis_tag)) |_| { + stdx.debug.panic("Mesh axis '{s}' used to partition multiple tensor dimensions. This is not allowed.", .{axis_tag}); + } + + switch (spec) { + .unknown => {}, + .open => {}, + .replicated => {}, + .axis => used_mesh_axes.appendAssumeCapacity(axis_tag), + } + } + + return res; + } + + test withPartitioning { + var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(.{ .c = .feature, .a = .batch }); + try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .unknown, .init(.feature) }, shape._partitioning.constSlice()); + + shape = shape.withDefaultPartitioning(); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + + shape = shape.withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); + try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .open, .init(.feature) }, shape._partitioning.constSlice()); + } + + pub fn mapPartitioningAxes(self: Shape, mapping: anytype) Shape { + const T = @TypeOf(mapping); + stdx.debug.assertComptime(stdx.meta.isStruct(T), "Mapping must be a struct, e.g., .{ .old_axis = .new_axis }", .{}); + var res = self; - // Reset sharding. - res._sharding_info = @splat(false); - for (self.axes(axes_).constSlice()) |ax| { - res._sharding_info[ax] = true; + for (res._partitioning.slice()) |*spec| { + switch (spec.*) { + .unknown => {}, + .open => {}, + .replicated => {}, + .axis => |*old_axis_tag| { + inline for (std.meta.fields(T)) |field| { + const logical_axis_in_map = toTag(field); + + if (std.mem.eql(u8, std.mem.span(old_axis_tag.*), std.mem.span(logical_axis_in_map))) { + const physical_axis_from_map = @field(mapping, field.name); + spec.* = PartitionSpec.init(physical_axis_from_map); + break; + } + } + }, + } } return res; } + test mapPartitioningAxes { + const logical_shape = Shape.init(.{ .batch = 64, .seq = 128, .features = 512 }, .f32).withPartitioning(.{ + .batch = .data, + .seq = .replicated, + .features = .model, + }); + + const physical_shape = logical_shape.mapPartitioningAxes(.{ .data = .x, .model = .y }); + + try testing.expect(physical_shape.partition(.batch).eql(.init(.x))); + try testing.expect(physical_shape.partition(.features).eql(.init(.y))); + try testing.expect(physical_shape.partition(.seq).eql(.replicated)); + + const partial_mapped_shape = logical_shape.mapPartitioningAxes(.{ .data = .z }); + try testing.expect(partial_mapped_shape.partition(.batch).eql(.init(.z))); + try testing.expect(partial_mapped_shape.partition(.features).eql(.init(.model))); + } + + pub fn withDefaultPartitioning(self: Shape) Shape { + var res = self; + res._partitioning.clear(); + res._partitioning.appendNTimes(.unknown, self._dims.len) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); + return res; + } + + test withDefaultPartitioning { + var shape = Shape.init(.{ 10, 20, 30 }, .f32); + try testing.expectEqual(3, shape._partitioning.len); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + + shape = shape.withPartitioning(.{ ._1 = .batch }); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .init(.batch), .unknown }, shape._partitioning.constSlice()); + + shape = shape.withDefaultPartitioning(); + try testing.expectEqual(3, shape._partitioning.len); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + } + + pub fn withReplicatedPartitioning(self: Shape) Shape { + var res = self; + res._partitioning.clear(); + res._partitioning.appendNTimes(.replicated, self._dims.len) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); + return res; + } + + test withReplicatedPartitioning { + var shape = Shape.init(.{ 10, 20, 30 }, .f32); + try testing.expectEqual(3, shape._partitioning.len); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + + shape = shape.withPartitioning(.{ ._1 = .batch }); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .init(.batch), .unknown }, shape._partitioning.constSlice()); + + shape = shape.withReplicatedPartitioning(); + try testing.expectEqual(3, shape._partitioning.len); + try testing.expectEqualSlices(PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._partitioning.constSlice()); + } + + pub fn hasAtLeastOnePartitionedAxis(self: Shape) bool { + for (self._partitioning.constSlice()) |spec| { + if (spec == .axis) return true; + } + return false; + } + + test hasAtLeastOnePartitionedAxis { + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .x, .b = .replicated }); + try testing.expect(shape.hasAtLeastOnePartitionedAxis()); + + shape = shape.withPartitioning(.{ .a = .replicated, .b = .replicated }); + try testing.expect(!shape.hasAtLeastOnePartitionedAxis()); + + shape = shape.withDefaultPartitioning(); + try testing.expect(!shape.hasAtLeastOnePartitionedAxis()); + } + + pub fn isFullyPartitioned(self: Shape) bool { + for (self._partitioning.constSlice()) |spec| { + switch (spec) { + .replicated, .open, .unknown => return false, + .axis => {}, + } + } + return true; + } + + test isFullyPartitioned { + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .x, .b = .y }); + try testing.expect(shape.isFullyPartitioned()); + + shape = shape.withPartitioning(.{ .a = .x, .b = .replicated }); + try testing.expect(!shape.isFullyPartitioned()); + + shape = shape.withDefaultPartitioning(); + try testing.expect(!shape.isFullyPartitioned()); + } + + pub fn isFullyReplicated(self: Shape) bool { + for (self._partitioning.constSlice()) |spec| { + if (spec != .replicated and spec != .unknown) { + return false; + } + } + return true; + } + + test isFullyReplicated { + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .replicated, .b = .replicated }); + try testing.expect(shape.isFullyReplicated()); + + shape = shape.withPartitioning(.{ .a = .replicated, .b = .unknown }); + try testing.expect(shape.isFullyReplicated()); + + shape = shape.withPartitioning(.{ .a = .x, .b = .replicated }); + try testing.expect(!shape.isFullyReplicated()); + } + + pub fn partition(self: Shape, ax: anytype) PartitionSpec { + return self._partitioning.get(self.axis(ax)); + } + + test partition { + const shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); + try testing.expectEqual(PartitionSpec.init(.batch), shape.partition(.a)); + try testing.expectEqual(.open, shape.partition(.b)); + try testing.expectEqual(PartitionSpec.init(.feature), shape.partition(.c)); + } + + pub fn containsPartitionSpec(self: Shape, part: PartitionSpec) bool { + for (self._partitioning.constSlice()[0..self.rank()]) |dim_part| { + if (dim_part.eql(part)) return true; + } + return false; + } + + test containsPartitionSpec { + var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30, .d = 40 }, .f32).withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); + try testing.expect(shape.containsPartitionSpec(.init(.batch))); + try testing.expect(shape.containsPartitionSpec(.open)); + try testing.expect(!shape.containsPartitionSpec(.replicated)); + try testing.expect(shape.containsPartitionSpec(.unknown)); + + shape = shape.withDefaultPartitioning(); + try testing.expect(!shape.containsPartitionSpec(.init(.batch))); + try testing.expect(shape.containsPartitionSpec(.unknown)); + try testing.expect(!shape.containsPartitionSpec(.replicated)); + try testing.expect(!shape.containsPartitionSpec(.open)); + } + /// Renames some of the tags in this shape. /// Shape.init(.{ .a = 10, .b = 20 }).rename(.{ .b = .batch }); // .{ .a = 10, .batch = 20 }; pub fn rename(self: Shape, renames: anytype) Shape { @@ -766,7 +1131,7 @@ pub const Shape = struct { var res = self; inline for (std.meta.fields(T)) |field| { const new_field = @field(renames, field.name); - stdx.debug.assert(self.hasTag(new_field) == null, "{f}.rename({any}) failed because of duplicated axis {}", .{ self, renames, new_field }); + stdx.debug.assert(self.hasTag(new_field) == null, "{f}.rename({any}) failed because of duplicated axis {any}", .{ self, renames, new_field }); res._tags.set(self.axis(field), toTag(new_field)); } return res; @@ -786,18 +1151,114 @@ pub const Shape = struct { } } - pub fn computeStrides(self: Shape) stdx.BoundedArray(i64, MAX_RANK) { + /// Computes the strides of the shape in units of elements. + /// For a shape {d0, d1, d2}, the strides would be {d1*d2, d2, 1}. + pub fn computeElementStrides(self: Shape) stdx.BoundedArray(i64, constants.MAX_RANK) { + return self.computeStrides(1); + } + + test computeElementStrides { + { + const s = Shape.init(.{ 4, 1024 }, .bf16); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 1024, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 2, 3, 4 }, .f32); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 12, 4, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 5, 6 }, .i16); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 6, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{10}, .u8); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{1}, strides.constSlice()); + } + + { + const s = Shape.scalar(.f64); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{}, strides.constSlice()); + } + + { + const s = Shape.init(.{ 4, 1, 5 }, .i32); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 5, 5, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ .n = 8, .c = 3, .h = 224, .w = 224 }, .f32); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 3 * 224 * 224, 224 * 224, 224, 1 }, strides.constSlice()); + } + } + + /// Computes the strides of the shape in units of bytes. + /// For a shape {d0, d1, d2} of type T, the strides would be + /// {(d1*d2)*sizeof(T), d2*sizeof(T), 1*sizeof(T)}. + pub fn computeByteStrides(self: Shape) stdx.BoundedArray(i64, constants.MAX_RANK) { + return self.computeStrides(self.dtype().sizeOf()); + } + + test computeByteStrides { + { + const s = Shape.init(.{ 2, 3, 4 }, .f16); // 2 bytes + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 12 * 2, 4 * 2, 1 * 2 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 4, 1024 }, .bf16); // 2 bytes + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 1024 * 2, 1 * 2 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 5, 6 }, .c64); // 8 bytes (f32 + f32) + const strides = s.computeByteStrides(); + try std.testing.expectEqual(@sizeOf(std.math.Complex(f32)), 8); + try std.testing.expectEqualSlices(i64, &.{ 6 * 8, 1 * 8 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 128, 512 }, .i8); // 1 byte + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 512, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 1, 4096 }, .u32); // 4 bytes + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 4096 * 4, 1 * 4 }, strides.constSlice()); + } + + { + const s = Shape.scalar(.f64); // 8 bytes + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{}, strides.constSlice()); + } + } + + fn computeStrides(self: Shape, element_size: i64) stdx.BoundedArray(i64, constants.MAX_RANK) { const rk = self.rank(); - var strides: stdx.BoundedArray(i64, MAX_RANK) = .{ .len = rk }; + var strides: stdx.BoundedArray(i64, constants.MAX_RANK) = .{ .len = rk }; if (rk == 0) return strides; - const V = @Vector(MAX_RANK, i64); - const rank_mask = std.simd.iota(u8, MAX_RANK) < @as(@Vector(MAX_RANK, u8), @splat(rk)); + const V = @Vector(constants.MAX_RANK, i64); + const rank_mask = std.simd.iota(u8, constants.MAX_RANK) < @as(@Vector(constants.MAX_RANK, u8), @splat(rk)); // For each axis compute the product of all following dimensions // and the element size in bytes. var d: V = @bitCast(self._dims.buffer); d = @select(i64, rank_mask, d, @as(V, @splat(1))); - d = std.simd.shiftElementsLeft(d, 1, self.dtype().sizeOf()); + d = std.simd.shiftElementsLeft(d, 1, element_size); d = std.simd.prefixScan(.Mul, -1, d); strides.buffer = @bitCast(d); @@ -856,6 +1317,12 @@ pub const Shape = struct { var new_shape = self; new_shape._dims.replaceRange(ax, 1, dims_.constSlice()) catch unreachable; new_shape._tags.replaceRange(ax, 1, tags_.constSlice()) catch unreachable; + _ = new_shape._partitioning.orderedRemove(ax); + + for (0..dims_.len) |_| { + new_shape._partitioning.insert(ax, .unknown) catch unreachable; + } + new_shape.inferMissingAxis(self.count()); return new_shape; } @@ -900,22 +1367,43 @@ pub const Shape = struct { /// Merge the given axes into one axis. /// eg: `Shape.init(.{.a1 = 5, .a2 = 2, .b = 3}).merge(.{ .a = .{ .a1, .a2 }); -> .{ .a = 10, .b = 3 }` pub fn mergeAxis(self: Shape, axis_: anytype, axes_: anytype) Shape { - const axes__ = self.axes(axes_); - - const first_axis = axes__.get(0); - const last_axis = axes__.get(axes__.len - 1); + const axes_to_merge = self.axes(axes_); + stdx.debug.assert(axes_to_merge.len > 1, "Must merge at least two axes, got {any}", .{axes_}); var new_dim: i64 = 1; - for (axes__.constSlice(), first_axis..) |ax, counter| { - new_dim *= self.dim(ax); - stdx.debug.assert(ax == counter, "Can't merge shape {f} along non-contiguous axes {any}", .{ self, axes_ }); + // Merging partitioned axes is complex. If ANY axis being merged is partitioned, + // the resulting sharding is ambiguous. A propagation pass must resolve this. + // Marking it 'open' is the correct signal. If all are replicated/unknown, the + // result is also replicated/unknown. + // rule: .replicated wins over .unknown. .open wins over everything. + var resulting_partition_spec: PartitionSpec = .unknown; + + for (0..axes_to_merge.len) |i| { + const current_axis = axes_to_merge.get(i); + new_dim *= self.dim(current_axis); + + if (i > 0) { + stdx.debug.assert(current_axis == axes_to_merge.get(i - 1) + 1, "Can't merge shape {f} along non-contiguous axes {any}", .{ self, axes_ }); + } + + switch (self.partition(current_axis)) { + .axis, .open => resulting_partition_spec = .open, + .replicated => if (resulting_partition_spec != .open) { + resulting_partition_spec = .replicated; + }, + .unknown => {}, + } } + const first_axis_to_merge = axes_to_merge.get(0); + const num_axes_to_merge = axes_to_merge.len; + var new_shape = self; - new_shape._dims.set(first_axis, new_dim); - new_shape._dims.replaceRange(first_axis + 1, self.dims()[first_axis + 1 ..].len, self.dims()[last_axis + 1 ..]) catch unreachable; - new_shape._tags.set(first_axis, if (comptime isTagConvertible(@TypeOf(axis_))) toTag(axis_) else TagUnknown); - new_shape._tags.replaceRange(first_axis + 1, self.dims()[first_axis + 1 ..].len, self.tags()[last_axis + 1 ..]) catch unreachable; + + new_shape._dims.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{new_dim}) catch @panic("mergeAxis failed on dims"); + new_shape._tags.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{toTag(axis_)}) catch @panic("mergeAxis failed on tags"); + new_shape._partitioning.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{resulting_partition_spec}) catch @panic("mergeAxis failed on partitioning"); + return new_shape; } @@ -947,6 +1435,28 @@ pub const Shape = struct { ); } + test "mergeAxis partitioning" { + // Merging a partitioned axis with a replicated one -> open + const shape1 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32) + .withPartitioning(.{ .a = .replicated, .b = .x, .c = .replicated }); + const merged1 = shape1.mergeAxis(.ab, .{ .a, .b }); + try testing.expect(merged1.partition(.ab).eql(.open)); + try testing.expect(merged1.dim(.ab) == 10); + try testing.expect(merged1.rank() == 2); + + // Merging two replicated axes -> replicated + const shape2 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32) + .withPartitioning(.{ .a = .replicated, .b = .replicated, .c = .y }); + const merged2 = shape2.mergeAxis(.ab, .{ .a, .b }); + try testing.expect(merged2.partition(.ab).eql(.replicated)); + try testing.expect(merged2.partition(.c).eql(.init(.y))); + + // Merging two unknown axes -> unknown + const shape3 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32); // default is .unknown + const merged3 = shape3.mergeAxis(.ab, .{ .a, .b }); + try testing.expect(merged3.partition(.ab).eql(.unknown)); + } + pub fn mergeAxes(self: Shape, axes_: anytype) Shape { const T = @TypeOf(axes_); stdx.debug.assertComptime(stdx.meta.isStruct(T), "Must pass struct of enum literals like .{ .a = .{ .a1, .a2 } }. Passed: {any}", .{T}); @@ -992,10 +1502,10 @@ pub const Shape = struct { return res; } - pub fn parseStruct(T: type, v: anytype) struct { stdx.BoundedArray(T, MAX_RANK), TagsArray } { + pub fn parseStruct(T: type, v: anytype) struct { stdx.BoundedArray(T, constants.MAX_RANK), TagsArray } { const V = @TypeOf(v); - var vals_: stdx.BoundedArray(T, MAX_RANK) = .{}; + var vals_: stdx.BoundedArray(T, constants.MAX_RANK) = .{}; var tags_: TagsArray = .{}; if (comptime stdx.meta.isSliceOf(V, T)) { @@ -1007,7 +1517,7 @@ pub const Shape = struct { if (comptime stdx.meta.isStruct(V)) { const fields = std.meta.fields(V); - stdx.debug.assertComptime(fields.len <= MAX_RANK, "Too many fields in struct {} ({d}). Max supported is {d}.", .{ V, fields.len, MAX_RANK }); + stdx.debug.assertComptime(fields.len <= constants.MAX_RANK, "Too many fields in struct {} ({d}). Max supported is {d}.", .{ V, fields.len, constants.MAX_RANK }); inline for (fields) |field| { const fv = @field(v, field.name); vals_.appendAssumeCapacity(fv); @@ -1030,10 +1540,10 @@ pub const Shape = struct { } /// Parses a struct literal into a list of options for each axes. - pub fn parseAxesOptions(self: Shape, T: type, options: anytype, default: T) stdx.BoundedArray(T, MAX_RANK) { + pub fn parseAxesOptions(self: Shape, T: type, options: anytype, default: T) stdx.BoundedArray(T, constants.MAX_RANK) { const V = @TypeOf(options); - var res: stdx.BoundedArray(T, MAX_RANK) = .{}; + var res: stdx.BoundedArray(T, constants.MAX_RANK) = .{}; if (comptime stdx.meta.isSliceOf(V, T)) { stdx.debug.assert(options.len == self.rank(), "expects exactly {} options in slice, for {} got {}", .{ self.rank(), self, options.len }); for (options) |d| { @@ -1044,7 +1554,7 @@ pub const Shape = struct { if (comptime stdx.meta.isStruct(V)) { for (0..self.rank()) |_| res.appendAssumeCapacity(default); const fields = std.meta.fields(V); - stdx.debug.assertComptime(fields.len <= MAX_RANK, "expects up to {} options struct literal, got {}", .{ V, MAX_RANK, fields.len }); + stdx.debug.assertComptime(fields.len <= constants.MAX_RANK, "expects up to {} options struct literal, got {}", .{ V, constants.MAX_RANK, fields.len }); inline for (fields) |field| { const a = self.axis(field); res.buffer[a] = @field(options, field.name); @@ -1065,17 +1575,17 @@ pub const Shape = struct { test "comptimeShape" { comptime { const s = Shape.init(.{ .a = 5, .b = 6, .c = 7 }, .f32); - try std.testing.expectEqual(3, s.rank()); - try std.testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); - try std.testing.expectEqual(1, s.axis(.b)); + try testing.expectEqual(3, s.rank()); + try testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); + try testing.expectEqual(1, s.axis(.b)); } // comptime only the shape { const s = comptime Shape.init(.{ .a = 5, .b = 6, .c = 7 }, .f32); - try std.testing.expectEqual(3, s.rank()); - try std.testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); - try std.testing.expectEqual(1, s.axis(.b)); + try testing.expectEqual(3, s.rank()); + try testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); + try testing.expectEqual(1, s.axis(.b)); } } @@ -1094,4 +1604,135 @@ pub const Shape = struct { } return res_shape; } + + pub fn iterator(self: Shape) MultiDimIterator { + return .{ + .shape = self, + .current_coords = .{0} ** Shape.MAX_RANK, + .flat_index = 0, + }; + } }; + +/// Iterates over all coordinates of a shape, yielding the multi-dimensional +/// coordinates and the corresponding flat index for each element. +pub const MultiDimIterator = struct { + shape: Shape, + current_coords: [Shape.MAX_RANK]i64, + flat_index: usize, + + pub const Item = struct { + coords: [Shape.MAX_RANK]i64, + rank: u4, + flat_index: usize, + }; + + pub fn next(self: *MultiDimIterator) ?Item { + if (self.flat_index >= self.shape.count()) { + return null; + } + + const rank = self.shape.rank(); + const item = Item{ + .coords = self.current_coords, + .rank = rank, + .flat_index = self.flat_index, + }; + + self.flat_index += 1; + + var i: usize = rank; + while (i > 0) { + i -= 1; + self.current_coords[i] += 1; + if (self.current_coords[i] < self.shape.dim(i)) { + return item; + } + self.current_coords[i] = 0; + } + + return item; + } +}; + +test "MultiDimIterator / Scalar" { + const shape = Shape.scalar(.f32); + var iter = shape.iterator(); + const item = iter.next().?; + + try std.testing.expectEqual(0, item.rank); + try std.testing.expectEqual(0, item.flat_index); + try std.testing.expect(iter.next() == null); +} + +test "MultiDimIterator / Empty" { + const shape = Shape.init(.{ .x = 5, .y = 0, .z = 2 }, .f32); + var iter = shape.iterator(); + + try std.testing.expect(iter.next() == null); +} + +test "MultiDimIterator / 1D shape" { + const shape = Shape.init(.{5}, .f32); + var iter = shape.iterator(); + var count: i64 = 0; + + while (iter.next()) |item| : (count += 1) { + try std.testing.expectEqual(count, item.coords[0]); + } + try std.testing.expectEqual(5, count); +} + +test "MultiDimIterator / 2D shape" { + const shape = Shape.init(.{ .rows = 2, .cols = 3 }, .f32); + var iter = shape.iterator(); + var items: std.ArrayList(MultiDimIterator.Item) = .empty; + defer items.deinit(std.testing.allocator); + + while (iter.next()) |item| { + try items.append(std.testing.allocator, item); + } + + try std.testing.expectEqual(6, items.items.len); + try std.testing.expectEqualSlices(i64, &.{ 0, 0 }, items.items[0].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 0, 1 }, items.items[1].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 0, 2 }, items.items[2].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 1, 0 }, items.items[3].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 1, 1 }, items.items[4].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 1, 2 }, items.items[5].coords[0..2]); + try std.testing.expectEqual(5, items.items[5].flat_index); +} + +test "MultiDimIterator / 3D shape" { + const shape = Shape.init(.{ .d = 2, .h = 2, .w = 2 }, .f32); + var iter = shape.iterator(); + const expected_coords = [_][3]i64{ + .{ 0, 0, 0 }, + .{ 0, 0, 1 }, + .{ 0, 1, 0 }, + .{ 0, 1, 1 }, + .{ 1, 0, 0 }, + .{ 1, 0, 1 }, + .{ 1, 1, 0 }, + .{ 1, 1, 1 }, + }; + var count: usize = 0; + + while (iter.next()) |item| : (count += 1) { + try std.testing.expectEqual(count, item.flat_index); + try std.testing.expectEqualSlices(i64, &expected_coords[count], item.coords[0..3]); + } + + try std.testing.expectEqual(8, count); +} + +test "MultiDimIterator / Shape with dimension of size 1" { + const shape = Shape.init(.{ .a = 2, .b = 1, .c = 3 }, .f32); + var iter = shape.iterator(); + var count: usize = 0; + + while (iter.next()) |item| : (count += 1) { + try std.testing.expectEqual(0, item.coords[1]); + } + try std.testing.expectEqual(6, count); +} diff --git a/zml2/slice.zig b/zml/slice.zig similarity index 100% rename from zml2/slice.zig rename to zml/slice.zig diff --git a/zml/tensor.zig b/zml/tensor.zig index d5acc541..776af877 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1,55 +1,37 @@ const std = @import("std"); const builtin = @import("builtin"); +const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; -const callback = @import("callback.zig"); +const Bufferized = @import("zml.zig").Bufferized; const CompilationContext = @import("module.zig").CompilationContext; -const Data = @import("dtype.zig").Data; +const constants = @import("constants.zig"); +const ConstSlice = @import("slice.zig").ConstSlice; const DataType = @import("dtype.zig").DataType; -const HostBuffer = @import("hostbuffer.zig").HostBuffer; -const Memory = @import("buffer.zig").Buffer.Memory; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; +const Memory = Buffer.Memory; -const EnumLiteral = @TypeOf(.enum_literal); - -const dialect = struct { - const stablehlo = @import("mlir/dialects").stablehlo; -}; - -const scoped_log = std.log.scoped(.@"zml/tensor"); - -test { - std.testing.refAllDecls(Tensor); -} - -/// Represents an abstract Tensor object, which can be the input, -/// output, weight or activations of a neural network. -/// Tensor are abstract in the sense they only represent a computation, -/// but not a specific memory buffer. -/// Tensor namespace contains most of linear algebra needed to -/// represent mathematical operations. -/// More operations are available in `zml.nn` and `zml.torch` namespaces. pub const Tensor = struct { + var current_id: std.atomic.Value(usize) = .{ .raw = 1 }; + + id: usize, + auto_broadcast: bool = false, _shape: Shape, - _id: _Id, - _donation: _Donation = .no_buffer, - _output_memory_kind: Memory = .device, + _value: ?*const mlir.Value = null, - pub const _Donation = union(enum) { no_buffer, input_buffer, arg: u16 }; - pub const _Id = union(enum) { mlir: mlir.Value, buffer_id: u64, arg_id: u64 }; - pub const MAX_RANK = Shape.MAX_RANK; + pub fn init(shape_like: anytype, dt: DataType) Tensor { + return .fromShape(.init(shape_like, dt)); + } - /// Returns the current compilation context. - pub fn getContext(self: Tensor) *CompilationContext { - _ = self; - return CompilationContext.current(); + pub fn fromShape(shape_: Shape) Tensor { + return .{ .id = Tensor.current_id.fetchAdd(1, .seq_cst), ._shape = shape_ }; } pub fn format(self: Tensor, writer: *std.Io.Writer) !void { @@ -67,7 +49,7 @@ pub const Tensor = struct { } /// Returns the rank of a Tensor. - pub inline fn rank(self: Tensor) u4 { + pub fn rank(self: Tensor) u4 { return self._shape.rank(); } @@ -84,10 +66,11 @@ pub const Tensor = struct { /// Internal use /// /// Creates a tensor from a Shape and an mlir.Value. - pub fn _result(sh: Shape, val: mlir.Value) Tensor { + pub fn _result(sh: Shape, val: *const mlir.Value) Tensor { const res: Tensor = .{ ._shape = sh, - ._id = .{ .mlir = val }, + ._value = val, + .id = Tensor.current_id.fetchAdd(1, .seq_cst), }; if (builtin.mode == .Debug) { @@ -102,19 +85,20 @@ pub const Tensor = struct { /// Creates a Tensor from a mlir.Value /// /// The shape is derived from the type of the mlir.Value. - pub fn fromMlirValue(val: mlir.Value) Tensor { - const ranked_tensor = val.getType().as(mlir.RankedTensorType).?; - const n = ranked_tensor.getRank(); + pub fn fromMlirValue(val: *const mlir.Value) Tensor { + const ranked_tensor = val.type_().isA(mlir.RankedTensorType).?; + const n = ranked_tensor.rank(); - stdx.debug.assert(n <= MAX_RANK, "Can't represent MLIR tensor of rank {}, max supported rank is {}.", .{ n, MAX_RANK }); + stdx.debug.assert(n <= constants.MAX_RANK, "Can't represent MLIR tensor of rank {}, max supported rank is {}.", .{ n, constants.MAX_RANK }); - var sh: Shape = .{ ._dtype = mlirx.Type.toDType(ranked_tensor.getElementType()) }; + var sh: Shape = .{ ._dtype = mlirx.Type.toDType(mlirCtx(), ranked_tensor.elementType()) }; for (0..n) |i| { - sh._dims.appendAssumeCapacity(ranked_tensor.getDimension(i)); + sh._dims.appendAssumeCapacity(ranked_tensor.dimension(i)); } sh._tags.resize(n) catch unreachable; + sh._partitioning.resize(n) catch unreachable; - return .{ ._shape = sh, ._id = .{ .mlir = val } }; + return .{ ._shape = sh, ._value = val, .id = Tensor.current_id.fetchAdd(1, .seq_cst) }; } /// Returns the dimension of axis 'axis_'. @@ -139,7 +123,7 @@ pub const Tensor = struct { /// Returns the indices of each of the given axes. /// /// 'axis_' can be an integer or a tag. - pub fn axes(self: Tensor, axes_: anytype) stdx.BoundedArray(u3, Tensor.MAX_RANK) { + pub fn axes(self: Tensor, axes_: anytype) stdx.BoundedArray(u3, constants.MAX_RANK) { return self._shape.axes(axes_); } @@ -159,68 +143,39 @@ pub const Tensor = struct { return res; } + // TODO(Corentin) pub fn withSharding(self: Tensor, axes_: anytype) Tensor { - return switch (self._id) { - .arg_id, .mlir => { - const ctx = self.getContext(); - const mlir_ctx = ctx.mlirCtx(); - var res = self; - res._shape = self._shape.withSharding(axes_); - - if (ctx.numPartitions() <= 1) return self; - const op = dialect.stablehlo.custom_call( - mlir_ctx, - &.{self.value()}, - .{ - .call_target_name = "Sharding", - .has_side_effect = false, - .backend_config = null, - .additional_attributes = &.{.{ "mhlo.sharding", ctx.getShardingAttr(res._shape) }}, - .api_version = .original, - }, - &.{self.value().getType()}, - mlir_ctx.location(@src()), - ); - - return _result(res._shape, op.result(0)); - }, - .buffer_id => { - var res = self; - res._shape = self._shape.withSharding(axes_); - return res; - }, - }; + _ = self; + _ = axes_; + @panic("Unimplemented"); } pub fn toMemory(self: Tensor, kind: Memory) Tensor { - return switch (self._id) { - .arg_id, .mlir => { - const ctx = self.getContext(); - const mlir_ctx = ctx.mlirCtx(); - if (ctx.target() == .cpu) return self; + const ctx = CompilationContext.current(); + if (ctx.platform.target == .cpu) return self; - const frontend_attributes = mlir.Attribute.dict(mlir_ctx, &.{ - .{ "_xla_buffer_placement", .string(mlir_ctx, kind.pjrtName()) }, - }); + const frontend_attributes = mlir.dictionaryAttribute(ctx.mlir_ctx, &.{ + .named(ctx.mlir_ctx, "_xla_buffer_placement", mlir.stringAttribute(ctx.mlir_ctx, kind.pjrtName())), + }); - const op = dialect.stablehlo.custom_call(mlir_ctx, &.{self.value()}, .{ - .call_target_name = "annotate_device_placement", - .has_side_effect = true, - .backend_config = null, - .additional_attributes = &.{.{ "mhlo.frontend_attributes", frontend_attributes }}, - .api_version = .original, - }, &.{self.value().getType()}, mlir_ctx.location(@src())); - - var res = _result(self._shape, op.result(0)); - res._output_memory_kind = kind; - return res; + const op = dialects.stablehlo.custom_call( + ctx.mlir_ctx, + &.{self.value()}, + &.{self.value().type_()}, + .{ + .call_target_name = "annotate_device_placement", + .has_side_effect = true, + .backend_config = .{ .original = "" }, + .additional_attributes = &.{ + .named(ctx.mlir_ctx, "mhlo.frontend_attributes", frontend_attributes), + }, }, - .buffer_id => { - var res = self; - res._output_memory_kind = kind; - return res; - }, - }; + .unknown(ctx.mlir_ctx), + ).appendTo(currentBlock()); + + var res = _result(self._shape, op.result(0)); + ctx.currentScope().id_to_output_memory_kind.put(ctx.currentScope().arena.allocator(), res.id, kind) catch unreachable; + return res; } /// Returns a Tensor with new tag names. @@ -230,7 +185,7 @@ pub const Tensor = struct { return res; } - pub fn renameAxis(self: Tensor, ax: i8, name: EnumLiteral) Tensor { + pub fn renameAxis(self: Tensor, ax: i8, name: @EnumLiteral()) Tensor { var res = self; res._shape._tags.set(self.axis(ax), @tagName(name).ptr); return res; @@ -239,8 +194,12 @@ pub const Tensor = struct { /// Returns the mlir.Value associated with the Tensor. /// /// This will fail if used outside of a compilation context. - pub fn value(self: Tensor) mlir.Value { - return self.getContext().getValueAndDonation(self)[0]; + pub fn value(self: Tensor) *const mlir.Value { + if (CompilationContext.current().currentScope().id_to_argument.get(self.id)) |argument_index| { + return CompilationContext.current().currentScope().block.argument(argument_index); + } else if (self._value) |v| { + return v; + } else @panic("Something went really wrong, tensor is not an argument nor has an mlir.Value"); } /// Tell PJRT compiler that memory should be reuse between the two tensors. @@ -251,20 +210,21 @@ pub const Tensor = struct { /// is not allowed to reuse the donated input buffer after the call. /// For `reuseBuffer` to be effective, it needs to propagate all the way through the output. pub fn reuseBuffer(self: Tensor, origin: Tensor) Tensor { - // Note: check donation docs, this may be too permissive. - stdx.debug.assert(self.byteSize() == origin.byteSize(), "Can't reuse buffers between tensors of different size: {f} and {f}", .{ self, origin }); - - // TODO: should we store all donations inside the context ? - var res = self; - res._donation = self.getContext().getValueAndDonation(origin)[1]; - - return res; + const compilation_context = CompilationContext.current(); + const scope = compilation_context.currentScope(); + if (scope.id_to_argument.get(origin.id)) |argument_index| { + const gop = scope.id_to_donation.getOrPut(scope.arena.allocator(), self.id) catch unreachable; + gop.value_ptr.* = argument_index; + } else if (scope.id_to_donation.get(origin.id)) |origin_donation| { + const gop = scope.id_to_donation.getOrPut(scope.arena.allocator(), self.id) catch unreachable; + gop.value_ptr.* = origin_donation; + } + return self; } /// Returns a Tensor containing the absolute value of each element of the input Tensor. pub fn abs(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.abs(self.getContext().mlirCtx(), self.value(), loc); + const op = dialects.stablehlo.abs(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); const dt = switch (self.dtype()) { .c64 => .f32, .c128 => .f64, @@ -295,13 +255,12 @@ pub const Tensor = struct { res_shape = res_shape.withDtype(dt); - const loc = self.getContext().location(@src(), "bitCast({s})", .{@tagName(dt)}); - const op = dialect.stablehlo.bitcast_convert( - self.getContext().mlirCtx(), + const op = dialects.stablehlo.bitcast_convert( + mlirCtx(), self.value(), - mlirx.tensorType(self.getContext().mlirCtx(), res_shape), - loc, - ); + mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); return _result(res_shape, op.result(0)); } @@ -313,30 +272,26 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise number of leading 0 bits in the input Tensor. pub fn countLeadingZeros(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.count_leading_zeros(self.getContext().mlirCtx(), self.value(), loc); + const op = dialects.stablehlo.count_leading_zeros(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing booleans indicating if each element of the input Tensor is finite. pub fn isFinite(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.is_finite(self.getContext().mlirCtx(), self.value(), loc); + const op = dialects.stablehlo.is_finite(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape.withDtype(.bool), op.result(0)); } /// Returns a Tensor containing the element-wise number of bits set in the input Tensor. pub fn popcnt(self: Tensor) Tensor { stdx.debug.assert(self.dtype().isInteger(), "popcnt expects tensor type to be an integer, got {}", .{self.dtype()}); - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.popcnt(self.getContext().mlirCtx(), self.value(), loc); + const op = dialects.stablehlo.popcnt(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing the sign of the input Tensor element-wise. pub fn sign(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.sign(self.getContext().mlirCtx(), self.value(), loc); + const op = dialects.stablehlo.sign(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -344,7 +299,7 @@ pub const Tensor = struct { /// /// See https://pytorch.org/docs/stable/generated/torch.fmod.html for more details. pub fn fmod(self: Tensor, divisor: f32) Tensor { - return self.remainder(Tensor.scalar(divisor, .f32).broadcast(self._shape, &.{})); + return self.remainder(scalar(divisor, .f32).broadcast(self._shape, &.{})); } test fmod { @@ -356,26 +311,33 @@ pub const Tensor = struct { const divisors: [2]f32 = .{ 2, -1.5 }; inline for (inputs, expectations, divisors) |i, e, d| { - const input = try zml.Buffer.fromSlice(platform, .{6}, &i); - const output = try zml.testing.compileAndCall(platform, Tensor.fmod, .{ input, d }); + const input: zml.Tensor = .init(.{6}, .f32); - try zml.testing.expectClose(zml.HostBuffer.fromSlice(.{6}, &e), output, 1e-4); + const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.fmod, .{ input, d }, platform); + defer exe.deinit(); + + const input_buffer = try zml.Buffer.fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&i)); + defer input_buffer.deinit(); + + const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.fmod, .{input_buffer}); + + try zml.testing.expectClose(std.testing.io, ConstSlice.init(zml.Shape.init(.{6}, .f32), std.mem.sliceAsBytes(&e)), output, 1e-4); } } /// Returns a Tensor containing the element-wise left-shift operation of 'self' by 'other'. pub fn shiftLeft(self: Tensor, other: Tensor) Tensor { - return binaryOp(@src(), "shiftLeft", dialect.stablehlo.shift_left)(self, other); + return binaryOp("shiftLeft", dialects.stablehlo.shift_left)(self, other); } /// Returns a Tensor containing the element-wise arithmetic right-shift operation of 'self' by 'other'. pub fn shiftRightArithmetic(self: Tensor, other: Tensor) Tensor { - return binaryOp(@src(), "shiftRightArithmetic", dialect.stablehlo.shift_right_arithmetic)(self, other); + return binaryOp("shiftRightArithmetic", dialects.stablehlo.shift_right_arithmetic)(self, other); } /// Returns a Tensor containing the element-wise logical right-shift operation of 'self' by 'other'. pub fn shiftRightLogical(self: Tensor, other: Tensor) Tensor { - return binaryOp(@src(), "shiftRightLogical", dialect.stablehlo.shift_right_logical)(self, other); + return binaryOp("shiftRightLogical", dialects.stablehlo.shift_right_logical)(self, other); } /// Returns the Cholesky decomposition of the input Tensor. @@ -385,18 +347,16 @@ pub const Tensor = struct { pub fn cholesky(self: Tensor, lower: bool) Tensor { stdx.debug.assert(self.rank() <= 2, "cholesky expects tensor rank to be <= 2, got {}", .{self.rank()}); - const loc = self.getContext().location(@src(), "lower={}", .{lower}); - const op = dialect.stablehlo.cholesky(self.getContext().mlirCtx(), self.value(), lower, loc); + const op = dialects.stablehlo.cholesky(mlirCtx(), self.value(), lower, .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Solves the system of linear equations formed by the input tensors. - pub fn triangularSolve(self: Tensor, other: Tensor, opts: dialect.stablehlo.TriangularSolveOpts) Tensor { + pub fn triangularSolve(self: Tensor, other: Tensor, opts: dialects.stablehlo.TriangularSolveOpts) Tensor { stdx.debug.assert(self.dtype() == other.dtype(), "triangularSolve expects tensors to be of the same type, got {} and {}", .{ self.dtype(), other.dtype() }); stdx.debug.assert(self.rank() <= 2 and self.rank() == other.rank(), "triangularSolve expects tensors to have the same rank and be <= 2, got {} and {}", .{ self.rank(), other.rank() }); - const loc = self.getContext().location(@src(), "triangularSolve({f}, {})", .{ self, opts }); - const op = dialect.stablehlo.triangular_solve(self.getContext().mlirCtx(), self.value(), other.value(), loc, opts); + const op = dialects.stablehlo.triangular_solve(mlirCtx(), self.value(), other.value(), opts, .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -404,8 +364,7 @@ pub const Tensor = struct { pub fn roundNearestAfz(self: Tensor) Tensor { stdx.debug.assert(self.dtype().isFloat(), "roundNearestAfz expects tensor type to be a float, got {}", .{self.dtype()}); - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.round_nearest_afz(self.getContext().mlirCtx(), self.value(), loc); + const op = dialects.stablehlo.round_nearest_afz(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -413,8 +372,7 @@ pub const Tensor = struct { pub fn roundNearestEven(self: Tensor) Tensor { stdx.debug.assert(self.dtype().isFloat(), "roundNearestEven expects tensor type to be a float, got {}", .{self.dtype()}); - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.round_nearest_even(self.getContext().mlirCtx(), self.value(), loc); + const op = dialects.stablehlo.round_nearest_even(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -423,8 +381,7 @@ pub const Tensor = struct { stdx.debug.assert(re._shape.eql(im._shape), "complex expects tensor shapes to match, got {f} and {f}", .{ re._shape, im._shape }); stdx.debug.assert(re.dtype() == .f32 or re.dtype() == .f64, "complex expects tensors type to be f32 or f64, got {}", .{re.dtype()}); - const loc = re.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.complex(re.getContext().mlirCtx(), re.value(), im.value(), loc); + const op = dialects.stablehlo.complex(mlirCtx(), re.value(), im.value(), .unknown(mlirCtx())).appendTo(currentBlock()); const dt: DataType = if (re.dtype() == .f32) .c64 else .c128; return _result(re._shape.withDtype(dt), op.result(0)); } @@ -444,8 +401,7 @@ pub const Tensor = struct { .c128 => .f64, else => unreachable, }; - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.real(self.getContext().mlirCtx(), self.value(), loc); + const op = dialects.stablehlo.real(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape.withDtype(dt), op.result(0)); } @@ -457,7 +413,7 @@ pub const Tensor = struct { // Real tensors don't have imaginary part. if (self.dtype().isFloat()) { - return Tensor.constant(self._shape, self.dtype().zero()); + return Tensor.constant(self.dtype().zero()).broad(self._shape); } const dt: DataType = switch (self.dtype()) { @@ -466,13 +422,12 @@ pub const Tensor = struct { .c128 => .f64, else => unreachable, }; - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.imag(self.getContext().mlirCtx(), self.value(), loc); + const op = dialects.stablehlo.imag(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape.withDtype(dt), op.result(0)); } /// Returns the Fast Fourier Transform (FFT) of the input Tensor. - pub fn fft(self: Tensor, opts: dialect.stablehlo.FftOpts) Tensor { + pub fn fft(self: Tensor, opts: dialects.stablehlo.FftOpts) Tensor { // TODO: support tagged API. stdx.debug.assert(1 <= opts.length.len and opts.length.len <= 3, "fft expects 'opts.length' length to be between 1 and 3 (inclusive), got {}", .{opts.length.len}); @@ -508,25 +463,26 @@ pub const Tensor = struct { }, }; - const loc = self.getContext().location(@src(), "fft({f},{})", .{ self, opts }); - const op = dialect.stablehlo.fft(self.getContext().mlirCtx(), self.value(), loc, opts); + const op = dialects.stablehlo.fft(mlirCtx(), self.value(), opts, .unknown(mlirCtx())).appendTo(currentBlock()); return _result(sh, op.result(0)); } pub const Rng = struct { - _state: Tensor = .{ ._shape = .init(.{2}, .u64), ._id = .{ .buffer_id = 0 } }, - algorithm: dialect.stablehlo.RngAlgorithm.Type = .DEFAULT, + _state: Tensor, + algorithm: dialects.stablehlo.RngAlgorithm.Type = .DEFAULT, - pub fn shape() ShapeOf(Rng) { + pub fn init() Rng { + return .{ ._state = .init(.{2}, .u64) }; + } + + pub fn initBuffer(platform: Platform, seed: u128, io: std.Io) !Bufferized(Rng) { return .{ - ._state = Shape.init(.{2}, .u64), + ._state = try .fromBytes(io, platform, Shape.init(.{2}, .u64), std.mem.asBytes(&seed)), }; } - pub fn init(platform: Platform, seed: u128) !Bufferized(Rng) { - return .{ - ._state = try Buffer.fromBytes(platform, Rng.shape()._state, std.mem.asBytes(&seed)), - }; + pub fn deinitBuffer(self: *Bufferized(Rng)) void { + self._state.deinit(); } /// Returns a Tensor of the given shape, filled with uniform random bits, and a new Rng state. @@ -535,20 +491,18 @@ pub const Tensor = struct { /// The output is guaranteed to be deterministic function of `self` Rng state, /// but it is not guaranteed to be deterministic between implementations. pub fn bitGenerator(self: Rng, sh: Shape) struct { Rng, Tensor } { - const ctx = CompilationContext.current(); - const loc = ctx.location(@src(), "rand.bitGen({f})", .{sh}); - const op = dialect.stablehlo.rng_bit_generator( - ctx.mlirCtx(), + const op = dialects.stablehlo.rng_bit_generator( + mlirCtx(), self.algorithm, self._state.value(), - mlirx.tensorType(ctx.mlirCtx(), self._state._shape), - mlirx.tensorType(ctx.mlirCtx(), sh), - loc, - ); + mlir.rankedTensorType(self._state.dims(), mlirx.Type.fromDType(mlirCtx(), self._state.dtype())), + mlir.rankedTensorType(sh.dims(), mlirx.Type.fromDType(mlirCtx(), sh.dtype())), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); return .{ self.update(op.result(0)), _result(sh, op.result(1)) }; } - fn update(self: Rng, new_state: mlir.Value) Rng { + fn update(self: Rng, new_state: *const mlir.Value) Rng { return .{ ._state = _result(self._state._shape, new_state).reuseBuffer(self._state), .algorithm = self.algorithm, @@ -623,30 +577,36 @@ pub const Tensor = struct { const platform = zml.testing.env(); // Compute stats over a uniform distribution on [-2, 10]. - const rand, const stats = try zml.testing.compileAndCallWithTensors( - platform, - Stats.uniformStats, - .{ Rng.shape(), zml.Shape.init(.{1024}, .f32), .{ .min = -2, .max = 10 } }, - .{try Rng.init(platform, 1234)}, - ); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.uniformStats, .{ Rng.init(), Shape.init(.{1024}, .f32), .{ .min = -2, .max = 10 } }, platform); + defer exe.deinit(); + + const rng_buffer = try Rng.initBuffer(platform, 1234, std.testing.io); + defer rng_buffer._state.deinit(); + + const rand, const stats = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Stats.uniformStats, .{rng_buffer}); + defer rand._state.deinit(); + defer stats.mean.deinit(); + defer stats.variance.deinit(); + defer stats.min.deinit(); + defer stats.max.deinit(); // Check the Rng state has been modified. - try std.testing.expect(try rand._state.getValue(u128) != 1234); + try std.testing.expect(try rand._state.getValue(u128, std.testing.io) != 1234); // Check the mean and variance are close to theoritical values. - const mean_ = try stats.mean.getValue(f32); + const mean_ = try stats.mean.getValue(f32, std.testing.io); try std.testing.expectApproxEqAbs(4, mean_, 0.03); - const variance = try stats.variance.getValue(f32); + const variance = try stats.variance.getValue(f32, std.testing.io); try std.testing.expectApproxEqAbs(12.0 * 12.0 / 12.0, variance, 0.01); // Check that no value is outside of the interval // and we have samples close to the edges. - const min_ = try stats.min.getValue(f32); + const min_ = try stats.min.getValue(f32, std.testing.io); try std.testing.expect(min_ >= -2); try std.testing.expectApproxEqAbs(-2, min_, 0.05); - const max_ = try stats.max.getValue(f32); + const max_ = try stats.max.getValue(f32, std.testing.io); try std.testing.expect(max_ < 10); try std.testing.expectApproxEqAbs(10, max_, 0.05); } @@ -658,12 +618,10 @@ pub const Tensor = struct { pub fn normal(sh: Shape, opts: struct { mean: f64 = 0, stddev: f64 = 1 }) Tensor { stdx.debug.assert(sh.dtype().isFloat(), "normal expects tensor type to be a float, got {}", .{sh.dtype()}); - const ctx = CompilationContext.current(); - const loc = ctx.location(@src(), "rand.normal({f}, mean={},stddev={})", .{ sh, opts.mean, opts.stddev }); - const a = Tensor.constant(.{}, Data.init(sh.dtype(), opts.mean)); - const b = Tensor.constant(.{}, Data.init(sh.dtype(), opts.stddev)); - const res_shape = Tensor.constantTensor(HostBuffer.fromSlice(.{sh.rank()}, sh.dims())); - const op = dialect.stablehlo.rng(ctx.mlirCtx(), a.value(), b.value(), res_shape.value(), .NORMAL, loc); + const a = Tensor.constant(DataType.Value.init(sh.dtype(), opts.mean)); + const b = Tensor.constant(DataType.Value.init(sh.dtype(), opts.stddev)); + const res_tensor_shape = Tensor.constantTensor(Shape.init(.{sh.rank()}, .i64), std.mem.sliceAsBytes(sh.dims())); + const op = dialects.stablehlo.rng(mlirCtx(), a.value(), b.value(), res_tensor_shape.value(), .NORMAL, .unknown(mlirCtx())).appendTo(currentBlock()); return _result(sh, op.result(0)); } @@ -715,7 +673,7 @@ pub const Tensor = struct { for (&powers, 0..) |*p, i| p.* = std.math.pow(u64, 2, i * 16); break :blk powers; }; - const values = Tensor.constantTensor(HostBuffer.fromArray(&powers)).withTags(.{.d}); + const values = Tensor.constantTensor(Shape.init(.{4}, .u64), std.mem.sliceAsBytes(&powers)).withTags(.{.d}); const counts = values.gather(.{ .d = samples }, .{}).sum(.n).bitCast(.u16); const actual_dist = counts.reshape(target_dist.shape()).convert(target_dist.dtype()).divByConst(s.dim(.n)); return .{ rng, .{ .mean = mean_, .variance = variance, .actual_dist = actual_dist } }; @@ -723,28 +681,37 @@ pub const Tensor = struct { }; const platform = zml.testing.env(); - const tgt_dist = [_]f32{ 2.0, 1.0, 4.0, 3.0 }; - const rand, const stats = try zml.testing.compileAndCallWithTensors( - platform, - Stats.gumbelStats, - .{ Rng.shape(), zml.Shape.init(.{tgt_dist.len}, .f32) }, - .{ try Rng.init(platform, 1234), try .fromArray(platform, tgt_dist) }, - ); + const tgt_dist_data = [_]f32{ 2.0, 1.0, 4.0, 3.0 }; + const tgt_dist: Tensor = .init(.{tgt_dist_data.len}, .f32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.gumbelStats, .{ Rng.init(), tgt_dist }, platform); + defer exe.deinit(); + + const rng_buffer = try Rng.initBuffer(platform, 1234, std.testing.io); + defer rng_buffer._state.deinit(); + const tgt_dist_buffer: Buffer = try .fromBytes(std.testing.io, platform, tgt_dist.shape(), std.mem.sliceAsBytes(&tgt_dist_data)); + defer tgt_dist_buffer.deinit(); + + const rand, const stats = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Stats.gumbelStats, .{ rng_buffer, tgt_dist_buffer }); + defer rand._state.deinit(); + defer stats.mean.deinit(); + defer stats.variance.deinit(); + defer stats.actual_dist.deinit(); + // Check the Rng state has been modified. - try std.testing.expect(try rand._state.getValue(i128) != 1234); + try std.testing.expect(try rand._state.getValue(i128, std.testing.io) != 1234); // Check the mean and variance are close to theoritical values. - const mean_ = try stats.mean.getValue(f32); + const mean_ = try stats.mean.getValue(f32, std.testing.io); try std.testing.expectApproxEqAbs(0.5772, mean_, 0.02); - const variance = try stats.variance.getValue(f32); + const variance = try stats.variance.getValue(f32, std.testing.io); const pi = std.math.pi; try std.testing.expectApproxEqAbs(pi * pi / 6.0, variance, 0.03); // Check the distribution obtained with the gumbel trick matches the target distribution. - const actual_dist = try stats.actual_dist.getValue([4]f32); - scoped_log.debug("tgt_dist: {d}, actual_dist: {d}", .{ tgt_dist, actual_dist }); - for (tgt_dist, actual_dist) |tgt, actual| { + const actual_dist = try stats.actual_dist.getValue([4]f32, std.testing.io); + for (tgt_dist_data, actual_dist) |tgt, actual| { // We normalize tgt_dist to make it a well formed distribution. // We didn't do it before calling gumbel, because the gumbel trick // doesn't require normalized distributions as input. @@ -759,12 +726,11 @@ pub const Tensor = struct { stdx.debug.assert(1 <= exponent_bits, "reducePrecision expects 'exponent_bits' to be >= 1, got {}", .{exponent_bits}); stdx.debug.assert(0 <= mantissa_bits, "reducePrecision expects 'mantissa_bits' to be positive, got {}", .{mantissa_bits}); - const loc = self.getContext().location(@src(), "reducePrecision(exponent_bits={}, mantissa_bits={})", .{ exponent_bits, mantissa_bits }); - const op = dialect.stablehlo.reduce_precision(self.getContext().mlirCtx(), self.value(), exponent_bits, mantissa_bits, loc); + const op = dialects.stablehlo.reduce_precision(mlirCtx(), self.value(), exponent_bits, mantissa_bits, .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } - inline fn convolution(self: Tensor, other: Tensor, opts: dialect.stablehlo.ConvolutionOpts, loc: mlir.Location) Tensor { + fn convolution(self: Tensor, other: Tensor, opts: dialects.stablehlo.ConvolutionOpts) Tensor { stdx.debug.assert(self.rank() == other.rank(), "convolution expects tensor ranks to match, got {} and {}", .{ self.rank(), other.rank() }); const N = self.rank(); stdx.debug.guard(opts.window_strides.len == N - 2, @src()); @@ -847,14 +813,14 @@ pub const Tensor = struct { } // inferred shape '[1, 256, 1, 12008]' is incompatible with return type of operation '[1, 256, 1, 11978]' - const op = dialect.stablehlo.convolution( - self.getContext().mlirCtx(), + const op = dialects.stablehlo.convolution( + mlirCtx(), self.value(), other.value(), + mlir.rankedTensorType(new_shape.dims(), mlirx.Type.fromDType(mlirCtx(), new_shape.dtype())), used_opts, - mlirx.tensorType(self.getContext().mlirCtx(), new_shape), - loc, - ); + .unknown(mlirCtx()), + ).appendTo(currentBlock()); return _result(new_shape, op.result(0)); } @@ -882,7 +848,6 @@ pub const Tensor = struct { batch_group_count: i64 = 1, }, ) Tensor { - const loc = input.getContext().location(@src(), "opts={}", .{opts}); return input.convolution(kernel, .{ .window_strides = &.{opts.window_strides}, .pad_value = opts.padding, @@ -900,7 +865,7 @@ pub const Tensor = struct { .output_spatial_dimensions = &.{opts.output_spatial_dimensions}, .feature_group_count = opts.feature_group_count, .batch_group_count = opts.batch_group_count, - }, loc); + }); } /// Returns a Tensor containing the result of the 2D convolution of 'input' by 'kernel'. @@ -927,7 +892,6 @@ pub const Tensor = struct { batch_group_count: i64 = 1, }, ) Tensor { - const loc = input.getContext().location(@src(), "opts={}", .{opts}); return input.convolution(kernel, .{ .window_strides = opts.window_strides, .pad_value = opts.padding, @@ -945,47 +909,47 @@ pub const Tensor = struct { .output_spatial_dimensions = opts.output_spatial_dimensions, .feature_group_count = opts.feature_group_count, .batch_group_count = opts.batch_group_count, - }, loc); + }); } /// Returns a Tensor containing the element-wise addition of the input Tensors. pub fn add(self: Tensor, other: Tensor) Tensor { - return binaryOp(@src(), "add", dialect.stablehlo.add)(self, other); + return binaryOp("add", dialects.stablehlo.add)(self, other); } /// Returns a Tensor containing the element-wise subtraction of the input Tensors. pub fn sub(self: Tensor, other: Tensor) Tensor { - return binaryOp(@src(), "subtract", dialect.stablehlo.subtract)(self, other); + return binaryOp("subtract", dialects.stablehlo.subtract)(self, other); } /// Returns a Tensor containing the element-wise multiplication of the input Tensors. pub fn mul(self: Tensor, other: Tensor) Tensor { - return binaryOp(@src(), "mul", dialect.stablehlo.multiply)(self, other); + return binaryOp("mul", dialects.stablehlo.multiply)(self, other); } /// Returns a Tensor containing the element-wise division of the input Tensors. pub fn div(self: Tensor, other: Tensor) Tensor { - return binaryOp(@src(), "div", dialect.stablehlo.divide)(self, other); + return binaryOp("div", dialects.stablehlo.divide)(self, other); } /// Returns a Tensor containing the element-wise exponentiation of the input Tensors. pub fn pow(self: Tensor, other: Tensor) Tensor { - return binaryOp(@src(), "pow", dialect.stablehlo.power)(self, other); + return binaryOp("pow", dialects.stablehlo.power)(self, other); } /// Returns a Tensor containing the element-wise maximum operation of the input Tensors. pub fn maximum(self: Tensor, other: Tensor) Tensor { - return binaryOp(@src(), "maximum", dialect.stablehlo.maximum)(self, other); + return binaryOp("maximum", dialects.stablehlo.maximum)(self, other); } /// Returns a Tensor containing the element-wise minimum operation of the input Tensors. pub fn minimum(self: Tensor, other: Tensor) Tensor { - return binaryOp(@src(), "minimum", dialect.stablehlo.minimum)(self, other); + return binaryOp("minimum", dialects.stablehlo.minimum)(self, other); } /// Returns a Tensor containing the element-wise remainder of dividend 'self' and divisor 'other'. pub fn remainder(self: Tensor, other: Tensor) Tensor { - return binaryOp(@src(), "remainder", dialect.stablehlo.remainder)(self, other); + return binaryOp("remainder", dialects.stablehlo.remainder)(self, other); } /// Returns a Tensor containing the element-wise addition of the input Tensor with a constant. @@ -1004,7 +968,7 @@ pub const Tensor = struct { } /// Returns a Tensor containing the element-wise multiplication of the input Tensor by a constant. - pub inline fn scale(self: Tensor, val: anytype) Tensor { + pub fn scale(self: Tensor, val: anytype) Tensor { return self.mul(.scalar(val, self.dtype())); } @@ -1013,22 +977,20 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise logical operation of the input Tensors. pub fn logical(self: Tensor, comptime logical_op: LogicalOp, other: Tensor) Tensor { return switch (logical_op) { - .OR => binaryOp(@src(), "or", dialect.stablehlo.or_)(self, other), - .XOR => binaryOp(@src(), "xor", dialect.stablehlo.xor)(self, other), - .AND => binaryOp(@src(), "and", dialect.stablehlo.and_)(self, other), + .OR => binaryOp("or", dialects.stablehlo.or_)(self, other), + .XOR => binaryOp("xor", dialects.stablehlo.xor)(self, other), + .AND => binaryOp("and", dialects.stablehlo.and_)(self, other), }; } /// Returns a Tensor containing the element-wise floor operation of the input Tensor. pub fn floor(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - return _result(self._shape, dialect.stablehlo.floor(self.getContext().mlirCtx(), self.value(), loc).result(0)); + return _result(self._shape, dialects.stablehlo.floor(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()).result(0)); } /// Returns a Tensor containing the element-wise ceil operation of the input Tensor. pub fn ceil(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - return _result(self._shape, dialect.stablehlo.ceil(self.getContext().mlirCtx(), self.value(), loc).result(0)); + return _result(self._shape, dialects.stablehlo.ceil(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()).result(0)); } /// Returns a Tensor containing the element-wise conversion to another type. @@ -1036,11 +998,9 @@ pub const Tensor = struct { if (to == self.dtype()) { return self; } - const loc = self.getContext().location(@src(), "convert({f},to={s})", .{ self, @tagName(to) }); - const mlir_ctx = self.getContext().mlirCtx(); - const res_type = mlirx.tensorType(mlir_ctx, self.shape().withDtype(to)); - const op = dialect.stablehlo.convert(mlir_ctx, self.value(), res_type, loc); + const res_type = mlir.rankedTensorType(self.shape().dims(), mlirx.Type.fromDType(mlirCtx(), to)); + const op = dialects.stablehlo.convert(mlirCtx(), self.value(), res_type, .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape.withDtype(to), op.result(0)); } @@ -1055,12 +1015,19 @@ pub const Tensor = struct { var x_f4: [x.len]floats.Float4E2M1 = undefined; for (&x_f4, &x) |*xi_f4, xi| xi_f4.* = .fromF32(xi); - const x_d = try zml.Buffer.fromArray(platform, x); - const x_f4_xla_d = try zml.testing.compileAndCall(platform, Tensor.convert, .{ x_d, .f4e2m1 }); + const x_d: Tensor = .init(.{x.len}, .f32); + const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.convert, .{ x_d, .f4e2m1 }, platform); + defer exe.deinit(); + + const x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), std.mem.sliceAsBytes(&x)); + defer x_d_buffer.deinit(); + + const x_f4_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); + const x_f4_xla = try x_f4_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); + defer x_f4_xla.free(std.testing.allocator); - const x_f4_xla = x_f4_xla_d.getValue(@TypeOf(x_f4)); errdefer std.log.warn("convert(.f4e2m1) failed !\ninput f32:\n{e}\nzml.floats computed:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x), x_f4, x_f4_xla }); - try std.testing.expectEqualDeep(x_f4, x_f4_xla); + try std.testing.expectEqualDeep(&x_f4, x_f4_xla.items(floats.Float4E2M1)); } // f8e3m4 @@ -1069,108 +1036,93 @@ pub const Tensor = struct { var x_f8e3: [x.len]floats.Float8E3M4 = undefined; for (&x_f8e3, &x) |*xi_f8e3, xi| xi_f8e3.* = .fromF32(xi); - const x_d = try zml.Buffer.fromArray(platform, x); - const x_f8e3_xla_d = try zml.testing.compileAndCall(platform, Tensor.convert, .{ x_d, .f8e3m4 }); + const x_d: Tensor = .init(.{x.len}, .f32); + const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.convert, .{ x_d, .f8e3m4 }, platform); + defer exe.deinit(); + + const x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), std.mem.sliceAsBytes(&x)); + defer x_d_buffer.deinit(); + + const x_f8e3_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); + const x_f8e3_xla = try x_f8e3_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); + defer x_f8e3_xla.free(std.testing.allocator); - const x_f8e3_xla = x_f8e3_xla_d.getValue(@TypeOf(x_f8e3)); errdefer std.log.warn("convert(.f8e3m4) failed !\ninput f32:\n{e}\nzml.floats computed:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x), x_f8e3, x_f8e3_xla }); - try std.testing.expectEqualDeep(x_f8e3, x_f8e3_xla); + try std.testing.expectEqualDeep(&x_f8e3, x_f8e3_xla.items(floats.Float8E3M4)); } } /// Returns a Tensor containing the element-wise rounding operation of the input Tensor. pub fn round(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const sine_op = dialect.stablehlo.round_nearest_even(self.getContext().mlirCtx(), self.value(), loc); - return _result(self._shape, sine_op.result(0)); + const round_op = dialects.stablehlo.round_nearest_even(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, round_op.result(0)); } /// Returns a Tensor containing the element-wise clamping operation of the input Tensor. pub fn clamp(self: Tensor, min_: Tensor, max_: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.clamp(self.getContext().mlirCtx(), min_.value(), self.value(), max_.value(), loc); + const op = dialects.stablehlo.clamp(mlirCtx(), min_.value(), self.value(), max_.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } - /// See torch.matmul - pub fn matmul(lhs: Tensor, rhs: Tensor) Tensor { - return @import("torch.zig").matmul(lhs, rhs); - } - /// Matrix multiplication, where contracting axes are specified using their tags. - /// eg dot(.{ .a, .b, .c }, .{ .a, .c, .d }, .{ .c }) -> .{ .a, .c, .d } + /// eg dot(.{ .a, .b, .c }, .{ .a, .c, .d }, .c) -> .{ .a, .b, .d } /// Axes with the same tag on both sides, and which aren't contracting, /// are considered "batching axes". - pub fn dot(lhs: Tensor, rhs: Tensor, comptime contracting: anytype) Tensor { - var contracting_axes: stdx.BoundedArray([2]i8, MAX_RANK) = .{}; - if (@TypeOf(contracting) == EnumLiteral) { - contracting_axes.appendAssumeCapacity(.{ lhs.axis(contracting), rhs.axis(contracting) }); - } else { - inline for (contracting) |c| { - contracting_axes.appendAssumeCapacity(.{ lhs.axis(c), rhs.axis(c) }); + pub fn dot(lhs: Tensor, rhs: Tensor, args: anytype) Tensor { + const lhs_contracting_dim: i8 = @intCast(lhs.shape().hasTag(args).?); + const rhs_contracting_dim: i8 = @intCast(rhs.shape().hasTag(args).?); + + var batching_axes: stdx.BoundedArray([2]i8, constants.MAX_RANK) = .{}; + for (0..lhs.rank()) |lhs_tag_index| { + const lhs_tag = lhs.shape().tag(lhs_tag_index); + if (lhs_tag == Shape.toTag(args)) continue; + if (rhs.shape().hasTag(lhs_tag)) |rhs_tag_index| { + batching_axes.appendAssumeCapacity(.{ @intCast(lhs_tag_index), @intCast(rhs_tag_index) }); } } - - var batching_axes: stdx.BoundedArray([2]i8, MAX_RANK) = .{}; - for (lhs._shape.tags(), 0..) |l, li| { - stdx.debug.assert(l != Shape.TagUnknown, "Can't use `dot(..., {any})` on {any}, it need to be explictily tagged.", .{ contracting, lhs }); - - for (rhs._shape.tags(), 0..) |r, ri| { - stdx.debug.assert(r != Shape.TagUnknown, "Can't use `dot(..., {any})` on {any}, it need to be explictily tagged.", .{ contracting, rhs }); - - if (l == r) { - for (contracting_axes.slice()) |ct| { - if (l == lhs._shape.tag(ct[0])) { - break; - } - } else { - // tag is both in lhs and rhs but not in contracting -> it's a batching dim. - batching_axes.appendAssumeCapacity(.{ @intCast(li), @intCast(ri) }); - } - } - } - } - - return dotGeneral(lhs, rhs, contracting_axes.slice(), batching_axes.slice()); + return lhs.dotGeneral(rhs, &.{.{ lhs_contracting_dim, rhs_contracting_dim }}, batching_axes.slice()); } test dot { const zml = @import("zml.zig"); const platform = zml.testing.env(); - var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); - defer comp.deinit(); - - comp.activate(); - defer comp.deactivate(); - inline for (.{ - .{ .{ .c = 20 }, .{ .c = 20 }, .{.c}, .{} }, + .{ .{ .c = 20 }, .{ .c = 20 }, .c, .{} }, .{ .{ .a = 20, .b = 21, .c = 22 }, .{ .a = 20, .d = 23, .c = 22 }, - .{.c}, + .c, .{ .a = 20, .b = 21, .d = 23 }, }, .{ .{ .a = 20, .b = 21, .c = 22 }, .{ .c = 22, .d = 23, .e = 24 }, - .{.c}, + .c, .{ .a = 20, .b = 21, .d = 23, .e = 24 }, }, - .{ - .{ .a = 20, .b = 21, .c = 22 }, - .{ .c = 22, .d = 23, .a = 20 }, - .{ .c, .a }, - .{ .b = 21, .d = 23 }, - }, + // TODO(Corentin): Re-enable that + //.{ + // .{ .a = 20, .b = 21, .c = 22 }, + // .{ .c = 22, .d = 23, .a = 20 }, + // .{ .c, .a }, + // .{ .b = 21, .d = 23 }, + //}, }) |testcase| { - const x_shape, const y_shape, const ctr, const z_shape = testcase; - const x = Tensor.constant(x_shape, .{ .f32 = 0.0 }); - const y = Tensor.constant(y_shape, .{ .f32 = 0.0 }); - const z = x.dot(y, ctr); + const x: Tensor = .init(testcase[0], .f32); + const y: Tensor = .init(testcase[1], .f32); + const ctr = Shape.toTag(testcase[2]); + const z_shape: Shape = .init(testcase[3], .f32); + const forward = struct { + fn forward(x_: Tensor, y_: Tensor, tag: Shape.Tag) Tensor { + return x_.dot(y_, tag); + } + }.forward; - try zml.testing.expectEqualShapes(Shape.init(z_shape, .f32), z.shape()); + const exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, y, ctr }, platform); + defer exe.deinit(); + + try zml.testing.expectEqualShapes(Shape.init(z_shape, .f32), exe.output_shapes[0]); } } @@ -1184,9 +1136,9 @@ pub const Tensor = struct { contracting_axes: []const [2]i8, batching_axes: []const [2]i8, ) Tensor { - stdx.debug.assert(lhs.dtype() == rhs.dtype(), "dotGeneral expects tensors to be of the same type, got {f} and {f}", .{ lhs, rhs }); + stdx.debug.assert(lhs.dtype() == rhs.dtype(), "dotGeneral expects tensors to be of the same type, got {} and {}", .{ lhs.dtype(), rhs.dtype() }); - const Axes = stdx.BoundedArray(i64, MAX_RANK); + const Axes = stdx.BoundedArray(i64, constants.MAX_RANK); var res_shape: Shape = .{ ._dtype = lhs.dtype() }; // Validate batching axes @@ -1194,7 +1146,7 @@ pub const Tensor = struct { var rhs_batching_axes: Axes = .{}; for (batching_axes) |b_axes| { const l, const r = b_axes; - stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects batching dimensions to be equal, got {d} and {d} in {f} and {f}", .{ l, r, lhs, rhs }); + stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects batching dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); var t = lhs._shape.tag(l); if (t == Shape.TagUnknown) t = rhs._shape.tag(r); res_shape = res_shape.appendDim(lhs._shape.dim(l), t); @@ -1234,29 +1186,26 @@ pub const Tensor = struct { res_shape = res_shape.appendDim(rhs._shape.dim(r), rhs._shape.tag(r)); } - const mlir_ctx = lhs.getContext().mlirCtx(); - const loc = lhs.getContext().location(@src(), "dot({f},{f},contracting={any},batching={any}", .{ lhs, rhs, contracting_axes, batching_axes }); - const op = dialect.stablehlo.dot_general( - mlir_ctx, + const op = dialects.stablehlo.dot_general( + mlirCtx(), lhs.value(), rhs.value(), - mlirx.tensorType(mlir_ctx, res_shape), - loc, + mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), .{ .lhs_batching_dimensions = lhs_batching_axes.constSlice(), .rhs_batching_dimensions = rhs_batching_axes.constSlice(), .lhs_contracting_dimensions = lhs_contracting_axes.constSlice(), .rhs_contracting_dimensions = rhs_contracting_axes.constSlice(), - .precision = .fast, + .dot_precision = .fast, }, - ); + .unknown(mlirCtx()), + ).appendTo(currentBlock()); return _result(res_shape, op.result(0)); } /// Returns a Tensor containing the sigmoid function applied to each element of the input Tensor. pub fn sigmoid(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.logistic(self.getContext().mlirCtx(), self.value(), loc); + const op = dialects.stablehlo.logistic(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -1264,7 +1213,7 @@ pub const Tensor = struct { /// Returns a Tensor containing the ReLU activation function applied to each element of the input Tensor. pub fn relu(self: Tensor) Tensor { - return self.maximum(Tensor.constant(self.dims(), self.dtype().zero())); + return self.maximum(Tensor.constant(self.dtype().zero()).broad(.init(self.dims(), self.dtype()))); } /// Returns a Tensor containing the leaky-ReLU activation function applied to each element of the input Tensor. @@ -1272,7 +1221,7 @@ pub const Tensor = struct { /// LeakyReLU(x) = max(0,x) + negative_slope * min(0,x) /// ref: https://paperswithcode.com/method/leaky-relu pub fn leakyReLU(self: Tensor, negative_slope: f32) Tensor { - const below_zero = self.scale(negative_slope).minimum(Tensor.scalar(0, self.dtype())); + const below_zero = self.scale(negative_slope).minimum(.scalar(0, self.dtype())); return self.relu().add(below_zero); } @@ -1280,19 +1229,26 @@ pub const Tensor = struct { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const input = try zml.Buffer.fromSlice(platform, .{2}, &[_]f32{ -0.6884, 1.6795 }); - const res = try zml.testing.compileAndCall(platform, leakyReLU, .{ input, 0.1 }); + const input: Tensor = .init(.{2}, .f32); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.leakyReLU, .{ input, 0.1 }, platform); + defer exe.deinit(); - const expectation = zml.HostBuffer.fromArray(&[2]f32{ -0.0688, 1.6795 }); - try zml.testing.expectClose(expectation, res, 1e-4); + const input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6884, 1.6795 })); + defer input_buffer.deinit(); + + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.leakyReLU, .{input_buffer}); + defer res.deinit(); + + const expectation: ConstSlice = .init(input.shape(), std.mem.sliceAsBytes(&[2]f32{ -0.0688, 1.6795 })); + try zml.testing.expectClose(std.testing.io, expectation, res, 1e-4); } /// Returns a Tensor containing the SwiGLU activation function applied to the input Tensor. - pub fn swiglu(self: Tensor, beta: f32, w: Tensor, b: Tensor) Tensor { - const sigmoid_tensor = self.mul(Tensor.constant(self._shape, Data.init(self.dtype(), beta))).sigmoid(); - const one_minus_sigmoid_tensor = Tensor.constant(self._shape, Data.init(self.dtype(), 1)).sub(sigmoid_tensor); + pub fn swiglu(self: Tensor, beta: f32, w: Tensor, b: Tensor, tag: Shape.Tag) Tensor { + const sigmoid_tensor = self.mul(Tensor.constant(DataType.Value.init(self.dtype(), beta)).broad(.init(self._shape, self.dtype()))).sigmoid(); + const one_minus_sigmoid_tensor = Tensor.constant(.init(self.dtype(), 1)).broad(.init(self._shape, self.dtype())).sub(sigmoid_tensor); - return self.mul(sigmoid_tensor).add(one_minus_sigmoid_tensor.mul(w.matmul(self).add(b))); + return self.mul(sigmoid_tensor).add(one_minus_sigmoid_tensor.mul(w.dot(self, tag).add(b))); } /// Returns a Tensor containing the Gaussian Error Linear Units (GeLU) activation function applied to each element of the input Tensor. @@ -1356,26 +1312,21 @@ pub const Tensor = struct { pub fn logSumExp(self: Tensor, axis_: anytype) Tensor { const a = self.axis(axis_); // stabilization: shift `self` by it's max value before passing to exponent. - const M = self.max(a); - const log_sum_exp = log(sum(exp(self.sub(M.broad(self._shape))), a)); + const max_ = self.max(a); + const log_sum_exp = log(sum(exp(self.sub(max_.broad(self._shape))), a)); // restore the shift again - return M.add(log_sum_exp); + return max_.add(log_sum_exp); } /// Returns a Tensor containing the sum of elements over the given axis. /// Output shape is the input shape with the axis_ dim set to 1. pub fn sum(self: Tensor, axis_: anytype) Tensor { const a = self.axis(axis_); - return ops.reduce( - struct { - pub fn acc(x: Tensor, res: Tensor) Tensor { - return res.add(x.convert(res.dtype())); - } - }.acc, - self, - Tensor.scalar(0, self.dtype()), - &.{a}, - ); + return ops.reduce(.{self}, .{Tensor.constant(self.dtype().zero())}, &.{a}, struct { + pub fn acc(args: ops.ReduceArgs) struct { Tensor } { + return .{args.right.add(args.left.convert(args.right.dtype()))}; + } + }.acc, .{})[0]; } /// Returns a Tensor containing the mean of elements over the given axis. @@ -1392,16 +1343,15 @@ pub const Tensor = struct { const rk = self.rank(); const a = self.axis(axis_); - const ones = [_]i64{1} ** MAX_RANK; + const ones = [_]i64{1} ** constants.MAX_RANK; var window_dimensions = ones; window_dimensions[a] = self.dim(a); - var padding = [_][2]i64{.{ 0, 0 }} ** MAX_RANK; + var padding = [_][2]i64{.{ 0, 0 }} ** constants.MAX_RANK; padding[a] = .{ self.dim(a) - 1, 0 }; - return ops.reduceWindow( - Tensor.add, - self, - Tensor.scalar(0, self.dtype()), + const result = ops.reduceWindow( + .{self}, + .{Tensor.scalar(0, self.dtype())}, .{ .base_dilations = ones[0..rk], .window_dilations = ones[0..rk], @@ -1409,7 +1359,15 @@ pub const Tensor = struct { .window_dimensions = window_dimensions[0..rk], .padding = padding[0..rk], }, + struct { + fn add(values: ops.ReduceArgs) struct { Tensor } { + return .{values.left.add(values.right)}; + } + }.add, + .{}, ); + + return result[0]; } test cumulativeSum { @@ -1426,29 +1384,36 @@ pub const Tensor = struct { } }; - const x = try zml.Buffer.fromArray( - platform, - [2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } }, - ); - const res = try zml.testing.compileAndCall(platform, Local._cumsum, .{x}); - try std.testing.expectEqual( - [2][5]f32{ .{ 0, 1, 2, 2, 3 }, .{ 3, 4, 4, 6, 7 } }, - try res.getValue([2][5]f32), - ); + const x: Tensor = .init(.{ 2, 5 }, .f32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cumsum, .{x}, platform); + defer exe.deinit(); + + const x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } })); + defer x_buffer.deinit(); + + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._cumsum, .{x_buffer}); + defer res.deinit(); + + try std.testing.expectEqual([2][5]f32{ .{ 0, 1, 2, 2, 3 }, .{ 3, 4, 4, 6, 7 } }, try res.getValue([2][5]f32, std.testing.io)); } /// Returns a transposed Tensor computed using the given axes. pub fn transpose(self: Tensor, axes_: anytype) Tensor { const axes__ = self.axes(axes_).constSlice(); - const default_perm = [MAX_RANK]i64{ 7, 6, 5, 4, 3, 2, 1, 0 }; - const no_op = [MAX_RANK]i64{ 0, 1, 2, 3, 4, 5, 6, 7 }; + const no_op = constants.AXES_IOTA; + const default_perm = b: { + var buf = constants.AXES_IOTA; + std.mem.reverse(i64, &buf); + break :b buf; + }; const permutation: []const i64 = if (axes__.len == 0) - default_perm[MAX_RANK - self.rank() ..] + default_perm[constants.MAX_RANK - self.rank() ..] else - toI64(axes__); + toI64(axes__).constSlice(); - stdx.debug.assert(permutation.len == self.rank(), "transpose expects input tensor rank and 'axes_' length to be equal, got {f} and {any}", .{ self, permutation[0..@min(permutation.len, MAX_RANK + 2)] }); + stdx.debug.assert(permutation.len == self.rank(), "transpose expects input tensor rank and 'axes_' length to be equal, got {f} and {any}", .{ self, permutation[0..@min(permutation.len, constants.MAX_RANK + 2)] }); if (std.mem.eql(i64, permutation, no_op[0..self.rank()])) { return self; @@ -1459,14 +1424,13 @@ pub const Tensor = struct { return self.reshape(res_shape); } - const loc = self.getContext().location(@src(), "transpose({f}, {any})", .{ self, permutation }); - const op = dialect.stablehlo.transpose( - self.getContext().mlirCtx(), + const op = dialects.stablehlo.transpose( + mlirCtx(), self.value(), - mlirx.tensorType(self.getContext().mlirCtx(), res_shape), - loc, - .{ .permutation = toI64(permutation) }, - ); + mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), + .{ .permutation = toI64(permutation).constSlice() }, + .unknown(mlirCtx()), + ).appendTo(currentBlock()); return _result(res_shape, op.result(0)); } @@ -1486,19 +1450,18 @@ pub const Tensor = struct { /// unflatten((d0, d1, axis_m, d3), 2, n) -> (d0, d1, n, d2_m, d3) pub fn unflatten(self: Tensor, axis_: i8, n: i64) Tensor { // TODO: move to torch.zig, this equivalent to `spitAxis` - stdx.debug.assert(self.rank() < Tensor.MAX_RANK, "unflatten expects input tensor rank to be less than {}, got {}", .{ Tensor.MAX_RANK, self.rank() }); + stdx.debug.assert(self.rank() < constants.MAX_RANK, "unflatten expects input tensor rank to be less than {}, got {}", .{ constants.MAX_RANK, self.rank() }); const a = if (axis_ >= 0) self.axis(axis_) else self.axis(axis_) + 1; const new_dim = std.math.divExact(i64, self.dim(a), n) catch std.debug.panic("unflatten expects chosen dimension to be divisible by 'n' but {} is not divisible by {}", .{ self.dim(a), n }); const new_shape = self._shape.set(a, n).insert(a + 1, .{ ._ = new_dim }); - const loc = self.getContext().location(@src(), "axis={}, n={}", .{ axis_, n }); - const reshaped_val = dialect.stablehlo.reshape( - self.getContext().mlirCtx(), + const reshaped_val = dialects.stablehlo.reshape( + mlirCtx(), self.value(), - mlirx.tensorType(self.getContext().mlirCtx(), new_shape), - loc, - ); + mlir.rankedTensorType(new_shape.dims(), mlirx.Type.fromDType(mlirCtx(), new_shape.dtype())), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); return _result(new_shape, reshaped_val.result(0)); } @@ -1509,13 +1472,12 @@ pub const Tensor = struct { pub fn splitAxis(self: Tensor, ax: anytype, split_shape: anytype) Tensor { const new_shape = self._shape.splitAxis(ax, split_shape); - const loc = self.getContext().location(@src(), "splitAxis({}, {any})", .{ ax, split_shape }); - const reshaped_val = dialect.stablehlo.reshape( - self.getContext().mlirCtx(), + const reshaped_val = dialects.stablehlo.reshape( + mlirCtx(), self.value(), - mlirx.tensorType(self.getContext().mlirCtx(), new_shape), - loc, - ); + mlir.rankedTensorType(new_shape.dims(), mlirx.Type.fromDType(mlirCtx(), new_shape.dtype())), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); return _result(new_shape, reshaped_val.result(0)); } @@ -1527,7 +1489,7 @@ pub const Tensor = struct { /// Merges two or more non-contiguous axes into one axis. /// Will make a transpose if needed. /// .{ .a, .b, .c }.mergeTranspose(.{ .a, .c }, .ac) -> .{ .b, .ac } - pub fn mergeTranspose(self: Tensor, axes_: anytype, merged: EnumLiteral) Tensor { + pub fn mergeTranspose(self: Tensor, axes_: anytype, merged: @EnumLiteral()) Tensor { const cont = self.contiguous(axes_); return cont.reshape(cont._shape.mergeAxis(merged, axes_)); } @@ -1539,7 +1501,7 @@ pub const Tensor = struct { return self.transpose(perm.constSlice()); } - pub inline fn flatten(self: Tensor) Tensor { + pub fn flatten(self: Tensor) Tensor { return self.reshape(.{self.count()}); } @@ -1580,16 +1542,16 @@ pub const Tensor = struct { /// Slices the input Tensor over the given axis using the given parameters. pub fn slice1d(self: Tensor, axis_: anytype, s: Slice) Tensor { - var slices = [_]Slice{.{}} ** MAX_RANK; + var slices = [_]Slice{.{}} ** constants.MAX_RANK; slices[self.axis(axis_)] = s; return self.slice(slices[0..self.rank()]); } /// Slices the input Tensor using the given parameters. pub fn slice(self: Tensor, slices: []const Slice) Tensor { - var start_indices: [MAX_RANK]i64 = undefined; - var strides: [MAX_RANK]i64 = undefined; - var limit_indices: [MAX_RANK]i64 = undefined; + var start_indices: [constants.MAX_RANK]i64 = undefined; + var strides: [constants.MAX_RANK]i64 = undefined; + var limit_indices: [constants.MAX_RANK]i64 = undefined; var res_shape: Shape = self._shape; for (slices, 0..) |s, a| { @@ -1602,18 +1564,16 @@ pub const Tensor = struct { res_shape = res_shape.setDim(a, std.math.divCeil(i64, args.end - args.start, args.step) catch unreachable); } - const mlir_ctx = self.getContext().mlirCtx(); - const loc = mlir_ctx.location(@src()).namedFmt(mlir_ctx, "slices={any}", .{slices}); - const result_type = mlirx.tensorType(mlir_ctx, res_shape); - const slice_op = dialect.stablehlo.slice( - mlir_ctx, + const result_type = mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())); + const slice_op = dialects.stablehlo.slice( + mlirCtx(), self.value(), start_indices[0..self.rank()], limit_indices[0..self.rank()], strides[0..self.rank()], result_type, - loc, - ); + .unknown(mlirCtx()), + ).appendTo(currentBlock()); var res = _result(res_shape, slice_op.result(0)); var to_remove: Shape.AxesArray = .{}; @@ -1627,7 +1587,10 @@ pub const Tensor = struct { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const x = try zml.Buffer.fromSlice(platform, .{ 2, 5 }, &[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); + const x: Tensor = .init(.{ 2, 5 }, .f32); + + const x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); + defer x_buffer.deinit(); // Wrap slice1d to hide the anytype in the signature. const Local = struct { @@ -1636,17 +1599,19 @@ pub const Tensor = struct { } }; - { - const res = try zml.testing.compileAndCall(platform, Local._slice1dAxis, .{ x, 0, .{ .end = 1 } }); - try std.testing.expectEqual([5]f32{ 0, 1, 2, 3, 4 }, try res.getValue([5]f32)); - } - { - const res = try zml.testing.compileAndCall(platform, Local._slice1dAxis, .{ x, 1, .{ .start = 1, .step = 2 } }); - try std.testing.expectEqual([4]f32{ 1, 3, 6, 8 }, try res.getValue([4]f32)); - } - { - const res = try zml.testing.compileAndCall(platform, Local._slice1dAxis, .{ x, -1, .{ .start = -2 } }); - try std.testing.expectEqual([4]f32{ 3, 4, 8, 9 }, try res.getValue([4]f32)); + inline for (.{ + .{ 0, Tensor.Slice{ .end = 1 }, [5]f32{ 0, 1, 2, 3, 4 } }, + .{ 1, Tensor.Slice{ .start = 1, .step = 2 }, [4]f32{ 1, 3, 6, 8 } }, + .{ -1, Tensor.Slice{ .start = -2 }, [4]f32{ 3, 4, 8, 9 } }, + }) |testcase| { + const ax, const slice_, const expectation = testcase; + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._slice1dAxis, .{ x, ax, slice_ }, platform); + defer exe.deinit(); + + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._slice1dAxis, .{x_buffer}); + defer res.deinit(); + + try std.testing.expectEqual(expectation, try res.getValue(@TypeOf(expectation), std.testing.io)); } } @@ -1660,7 +1625,7 @@ pub const Tensor = struct { pub fn choose(self: Tensor, offsets: anytype) Tensor { const off, const tags = Shape.parseDimensions(offsets); - var slices = [_]Slice{.{}} ** MAX_RANK; + var slices = [_]Slice{.{}} ** constants.MAX_RANK; for (off.constSlice(), tags.constSlice()) |o, t| { const ax = self.axis(t); slices[ax] = .single(o); @@ -1672,7 +1637,7 @@ pub const Tensor = struct { pub fn concatenate(tensors: []const Tensor, axis_: anytype) Tensor { if (tensors.len == 1) return tensors[0]; stdx.debug.assert(tensors.len <= 32, "concatenate only supports up to 32 tensors, got {}", .{tensors.len}); - var buffer: [32]mlir.Value = undefined; + var buffer: [32]*const mlir.Value = undefined; std.debug.assert(tensors.len <= buffer.len); std.debug.assert(tensors.len > 0); const a = tensors[0].axis(axis_); @@ -1685,9 +1650,7 @@ pub const Tensor = struct { } const res_shape = tensors[0]._shape.set(a, concatenated_dim); - const ctx = tensors[0].getContext(); - const loc = ctx.location(@src(), "axis={}", .{axis_}); - const op = dialect.stablehlo.concatenate(ctx.mlirCtx(), buffer[0..tensors.len], a, loc); + const op = dialects.stablehlo.concatenate(mlirCtx(), buffer[0..tensors.len], a, .unknown(mlirCtx())).appendTo(currentBlock()); // log.debug("concatenate({}, {}, {d}) -> {d}", .{ tensors[0], tensors[1], a, res_shape }); return _result(res_shape, op.result(0)); } @@ -1715,7 +1678,7 @@ pub const Tensor = struct { // Be careful here: we need to resolve ax before calling concatenate, // because we added an axis, so all - const ax = if (@TypeOf(axis_) == EnumLiteral and axis_ == .last) + const ax = if (@TypeOf(axis_) == @EnumLiteral() and axis_ == .last) shape0.rank() else shape0.axis(axis_); @@ -1765,23 +1728,21 @@ pub const Tensor = struct { } }; - { - const inputs: [3]u8 = .{ 1, 2, 3 }; - const expectations: [6]u8 = .{ 1, 2, 3, 1, 2, 3 }; + inline for (.{ + .{ [3]u8{ 1, 2, 3 }, Shape.init(.{3}, .u8), [6]u8{ 1, 2, 3, 1, 2, 3 }, 0, 2 }, + .{ [2][3]u8{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }, Shape.init(.{ 2, 3 }, .u8), [2][6]u8{ .{ 1, 2, 3, 1, 2, 3 }, .{ 4, 5, 6, 4, 5, 6 } }, 1, 2 }, + }) |testcase| { + const input_data, const shape_, const expectation, const ax, const reps = testcase; + const input: Tensor = .fromShape(shape_); - const input = try zml.Buffer.fromArray(platform, inputs); - const output = try zml.testing.compileAndCall(platform, Local.repeat1d, .{ input, 0, 2 }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.repeat1d, .{ input, ax, reps }, platform); + defer exe.deinit(); - try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); - } - { - const inputs: [2][3]u8 = .{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }; - const expectations: [2][6]u8 = .{ .{ 1, 2, 3, 1, 2, 3 }, .{ 4, 5, 6, 4, 5, 6 } }; + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&input_data)); + defer input_buffer.deinit(); - const input = try zml.Buffer.fromArray(platform, inputs); - const output = try zml.testing.compileAndCall(platform, Local.repeat1d, .{ input, 1, 2 }); - - try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local.repeat1d, .{input_buffer}); + try std.testing.expectEqual(expectation, try output.getValue(@TypeOf(expectation), std.testing.io)); } } @@ -1821,111 +1782,98 @@ pub const Tensor = struct { } }; - { - const inputs: [3]u8 = .{ 1, 2, 3 }; - const expectations: [6]u8 = .{ 1, 1, 2, 2, 3, 3 }; + inline for (.{ + .{ [3]u8{ 1, 2, 3 }, Shape.init(.{3}, .u8), [6]u8{ 1, 1, 2, 2, 3, 3 }, 0, 2 }, + .{ [2][3]u8{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }, Shape.init(.{ 2, 3 }, .u8), [2][6]u8{ .{ 1, 1, 2, 2, 3, 3 }, .{ 4, 4, 5, 5, 6, 6 } }, 1, 2 }, + .{ [2][3]u8{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }, Shape.init(.{ 2, 3 }, .u8), [2][6]u8{ .{ 1, 2, 3, 1, 2, 3 }, .{ 4, 5, 6, 4, 5, 6 } }, 0, 2 }, + }) |testcase| { + const input_data, const shape_, const expectation, const ax, const reps = testcase; + const input: Tensor = .fromShape(shape_); - const input = try zml.Buffer.fromArray(platform, inputs); - const output = try zml.testing.compileAndCall(platform, Local.stutter1d, .{ input, 0, 2 }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.stutter1d, .{ input, ax, reps }, platform); + defer exe.deinit(); - try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); - } - { - const inputs: [2][3]u8 = .{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }; - const expectations: [2][6]u8 = .{ .{ 1, 1, 2, 2, 3, 3 }, .{ 4, 4, 5, 5, 6, 6 } }; + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&input_data)); + defer input_buffer.deinit(); - const input = try zml.Buffer.fromArray(platform, inputs); - const output = try zml.testing.compileAndCall(platform, Local.stutter1d, .{ input, 1, 2 }); - - try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); - } - { - const inputs: [2][3]u8 = .{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }; - const expectations: [2][6]u8 = .{ .{ 1, 2, 3, 1, 2, 3 }, .{ 4, 5, 6, 4, 5, 6 } }; - - const input = try zml.Buffer.fromArray(platform, inputs); - const output = try zml.testing.compileAndCall(platform, Local.stutter1d, .{ input, 0, 2 }); - - try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local.stutter1d, .{input_buffer}); + try std.testing.expectEqual(expectation, try output.getValue(@TypeOf(expectation), std.testing.io)); } } /// Returns a Tensor containing the element-wise negation of the input Tensor. pub fn negate(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const negate_op = dialect.stablehlo.negate(self.getContext().mlirCtx(), self.value(), loc); + const negate_op = dialects.stablehlo.negate(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, negate_op.result(0)); } /// Returns a Tensor containing the element-wise cosine of the input Tensor. pub fn cos(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const cosine_op = dialect.stablehlo.cosine(self.getContext().mlirCtx(), self.value(), loc); + const cosine_op = dialects.stablehlo.cosine(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, cosine_op.result(0)); } /// Returns a Tensor containing the element-wise sine of the input Tensor. pub fn sin(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const sine_op = dialect.stablehlo.sine(self.getContext().mlirCtx(), self.value(), loc); + const sine_op = dialects.stablehlo.sine(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, sine_op.result(0)); } /// Returns a Tensor containing the element-wise exponential operation of the input Tensor. pub fn exp(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.exponential(self.getContext().mlirCtx(), self.value(), loc); + const op = dialects.stablehlo.exponential(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing the element-wise logarithm operation of the input Tensor. pub fn log(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.log(self.getContext().mlirCtx(), self.value(), loc); + const op = dialects.stablehlo.log(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing the element-wise square-root of the input Tensor. pub fn sqrt(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const sqrt_op = dialect.stablehlo.sqrt(self.getContext().mlirCtx(), self.value(), loc); + const sqrt_op = dialects.stablehlo.sqrt(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, sqrt_op.result(0)); } /// Returns a Tensor containing the element-wise reverse square-root of the input Tensor. pub fn rsqrt(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const rsqrt_op = dialect.stablehlo.rsqrt(self.getContext().mlirCtx(), self.value(), loc); + const rsqrt_op = dialects.stablehlo.rsqrt(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, rsqrt_op.result(0)); } /// Returns a Tensor containing the element-wise hyperbolic tangent of the input Tensor. pub fn tanh(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const tanh_op = dialect.stablehlo.tanh(self.getContext().mlirCtx(), self.value(), loc); + const tanh_op = dialects.stablehlo.tanh(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, tanh_op.result(0)); } /// Returns a Tensor containing the element-wise exponential minus one operation of the input Tensor. pub fn exponentialMinusOne(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const expm1_op = dialect.stablehlo.exponential_minus_one(self.getContext().mlirCtx(), self.value(), loc); + const expm1_op = dialects.stablehlo.exponential_minus_one(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, expm1_op.result(0)); } - pub const ArangeArgs = HostBuffer.ArangeArgs; - + pub const ArangeArgs = struct { + start: i64 = 0, + end: i64, + step: i64 = 1, + }; + /// /// Returns a Tensor containing evenly spaced values within a given interval. pub fn arange(args: ArangeArgs, dt: DataType) Tensor { stdx.debug.assert(args.start <= args.end, "arange expects 'args.start' to be less than 'args.end', got {} and {}", .{ args.start, args.end }); stdx.debug.assert(args.step > 0, "arange expects 'args.step' to be positive, got {}", .{args.step}); - const ctx = CompilationContext.current(); - const loc = ctx.location(@src(), "arange({}, dtype={})", .{ args, dt }); - const n_steps = std.math.divCeil(i64, args.end - args.start, args.step) catch unreachable; const sh = Shape.init(.{n_steps}, dt); - var op = dialect.stablehlo.iota(ctx.mlirCtx(), 0, mlirx.tensorType(ctx.mlirCtx(), sh), loc); + var op = dialects.stablehlo.iota( + mlirCtx(), + 0, + mlir.rankedTensorType(sh.dims(), mlirx.Type.fromDType(mlirCtx(), sh.dtype())), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); var res = _result(sh, op.result(0)); if (args.step != 1) { @@ -1948,16 +1896,13 @@ pub const Tensor = struct { const a = sh.axis(axis_); const dt: DataType = if (sh.dim(a) <= std.math.maxInt(i32)) .i32 else .i64; const res_shape = sh.withDtype(dt); - const ctx = CompilationContext.current(); - const loc = ctx.location(@src(), "iota({f}, {})", .{ res_shape, a }); - const mlir_ctx = ctx.mlirCtx(); - var op = dialect.stablehlo.iota( - mlir_ctx, + var op = dialects.stablehlo.iota( + mlirCtx(), a, - mlirx.tensorType(mlir_ctx, res_shape), - loc, - ); + mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); return _result(res_shape, op.result(0)); } @@ -1973,11 +1918,13 @@ pub const Tensor = struct { stdx.debug.assert(args.steps > 0, "linspace expects 'args.steps' to be positive, got {}", .{args.steps}); stdx.debug.assert(dt.isFloat(), "linspace expects type to be a float, got {} (hint: use arange instead)", .{dt}); - const ctx = CompilationContext.current(); - const loc = ctx.location(@src(), "linspace({}, dtype={})", .{ args, dt }); - const sh = Shape.init(.{args.steps}, dt); - var iota_op = dialect.stablehlo.iota(ctx.mlirCtx(), 0, mlirx.tensorType(ctx.mlirCtx(), sh), loc); + var iota_op = dialects.stablehlo.iota( + mlirCtx(), + 0, + mlir.rankedTensorType(sh.dims(), mlirx.Type.fromDType(mlirCtx(), sh.dtype())), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); var res = _result(sh, iota_op.result(0)); if (args.steps != 1) { @@ -1993,12 +1940,12 @@ pub const Tensor = struct { /// Returns a 0-rank Tensor with the given value. pub fn scalar(val: anytype, dt: DataType) Tensor { - const data = Data.init(dt, val); + const data = DataType.Value.init(dt, val); switch (dt.class()) { .float => stdx.debug.assert(!std.math.isNan(val), "scalar(NaN) is probably due to compiling a model with an uninitialized field", .{}), else => {}, } - return Tensor.constant(.{}, data); + return Tensor.constant(data); } test scalar { @@ -2014,40 +1961,32 @@ pub const Tensor = struct { } }; - _ = try zml.testing.compileAndCall(platform, Local._fwd, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{}, platform); + defer exe.deinit(); } /// Returns a constant Tensor with the given value. - pub fn constant(dimz: anytype, val: Data) Tensor { - const sh = Shape.init(dimz, val.dtype()); - const ctx = CompilationContext.current().mlirCtx(); - const loc = CompilationContext.current().location(@src(), "dims={f}, value={}", .{ sh, val }); - - var constant_op = if (mlirx.denseElementAttrType(val.dtype())) |elem_type| - dialect.stablehlo.constant(ctx, &.{}, elem_type, val.constSlice(), loc) - else blk: { - // Not all dtype can be serialized in the IR. If that's not possible, use f32. - const val_f32 = val.as(f32); - break :blk dialect.stablehlo.constant(ctx, &.{}, .f32, std.mem.asBytes(&val_f32), loc); - }; - - if (sh.rank() > 0) { - constant_op = dialect.stablehlo.broadcast_in_dim(ctx, constant_op.result(0), &.{}, mlirx.tensorType(ctx, sh), loc); - } - return _result(sh, constant_op.result(0)).convert(val.dtype()); + pub fn constant(val: DataType.Value) Tensor { + const op = dialects.stablehlo.constant( + mlirCtx(), + &.{}, + mlirx.Type.fromDType(mlirCtx(), val.dtype()), + val.asBytes(), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + return _result(.init(&.{}, val.dtype()), op.result(0)); } pub fn zeroes(sh: Shape) Tensor { - return .constant(sh, sh.dtype().zero()); + return Tensor.constant(sh.dtype().zero()).broad(sh); } /// Embeds a buffer with concrete values into an Mlir program. - pub fn constantTensor(val: HostBuffer) Tensor { - const ctx = CompilationContext.current().mlirCtx(); - const loc = ctx.location(@src()); - const elem_type = mlirx.denseElementAttrType(val.dtype()) orelse std.debug.panic("constantTensor expects a dtype that can be serialized to MLIR, like f32 or i32, got {f}", .{val.shape()}); - const constant_op = dialect.stablehlo.constant(ctx, val.shape().dims(), elem_type, val.bytes(), loc); - return _result(val.shape(), constant_op.result(0)); + pub fn constantTensor(sh: Shape, bytes_: []const u8) Tensor { + const elem_type = mlirx.Type.fromDType(mlirCtx(), sh.dtype()); + //const elem_type = mlirx.denseElementAttrType(val.dtype()) orelse std.debug.panic("constantTensor expects a dtype that can be serialized to MLIR, like f32 or i32, got {f}", .{val.shape()}); + const constant_op = dialects.stablehlo.constant(mlirCtx(), sh.dims(), elem_type, bytes_, .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(sh, constant_op.result(0)); } /// Returns a Tensor containing the result of the outer product between the input Tensors. @@ -2081,11 +2020,11 @@ pub const Tensor = struct { // but we propagate output_shape tags. return _result(res_shape, self.value()); } - const ctx = self.getContext(); - const result_type = mlirx.tensorType(ctx.mlirCtx(), res_shape); - const loc = ctx.location(@src(), "broadcast({f}, {f}, axes={any})", .{ self, res_shape, axes_ }); - const broadcast_op = dialect.stablehlo.broadcast_in_dim(ctx.mlirCtx(), self.value(), axes_, result_type, loc); - + const result_type = mlir.rankedTensorType( + res_shape.dims(), + mlirx.Type.fromDType(mlirCtx(), res_shape.dtype()), + ); + const broadcast_op = dialects.stablehlo.broadcast_in_dim(mlirCtx(), self.value(), axes_, result_type, .unknown(mlirCtx())).appendTo(currentBlock()); return _result(res_shape, broadcast_op.result(0)); } @@ -2125,12 +2064,11 @@ pub const Tensor = struct { // TODO: broad is error prone because of this: // it will happily broadcast .{ .a = 10, .b = 1 } to .{ .b = 10, .a = 5 } if (self._shape.rank() == 0 or self._shape.rank() == other.rank()) { - const all_axes = [MAX_RANK]i64{ 0, 1, 2, 3, 4, 5, 6, 7 }; - return self.broadcast(other, all_axes[0..self.rank()]); + return self.broadcast(other, constants.AXES_IOTA[0..self.rank()]); } // check that each axis of self maps to an axis of other - var axes_: stdx.BoundedArray(i64, MAX_RANK) = .{}; + var axes_: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; for (self._shape.tags()) |t| { axes_.appendAssumeCapacity(@intCast(other.axis(t))); } @@ -2140,9 +2078,8 @@ pub const Tensor = struct { /// Reshapes the input Tensor with the given shape. pub fn reshape(self: Tensor, output_shape_: anytype) Tensor { const output_shape = self._shape.reshape(output_shape_); - const tensor_type = mlirx.tensorType(self.getContext().mlirCtx(), output_shape); - const loc = self.getContext().location(@src(), "reshape({f})", .{output_shape}); - const reshape_value = dialect.stablehlo.reshape(self.getContext().mlirCtx(), self.value(), tensor_type, loc); + const tensor_type = mlir.rankedTensorType(output_shape.dims(), mlirx.Type.fromDType(mlirCtx(), output_shape.dtype())); + const reshape_value = dialects.stablehlo.reshape(mlirCtx(), self.value(), tensor_type, .unknown(mlirCtx())).appendTo(currentBlock()); return _result(output_shape, reshape_value.result(0)); } @@ -2163,7 +2100,7 @@ pub const Tensor = struct { pub fn pad(self: Tensor, padding_value: anytype, paddings: anytype) Tensor { const _paddings = self.shape().parseAxesOptions(Pad, paddings, .{}); - const ZEROS = [_]i64{0} ** MAX_RANK; + const ZEROS = [_]i64{0} ** constants.MAX_RANK; var low = ZEROS; var high = ZEROS; var interior = ZEROS; @@ -2180,15 +2117,13 @@ pub const Tensor = struct { } const rk = self.rank(); - const mlir_ctx = self.getContext().mlirCtx(); - const loc = mlir_ctx.location(@src()).namedFmt(mlir_ctx, "pad({},{})", .{ padding_value, _paddings }); - const pad_op = dialect.stablehlo.pad( - mlir_ctx, + const pad_op = dialects.stablehlo.pad( + mlirCtx(), self.value(), Tensor.scalar(padding_value, self.dtype()).value(), .{ .low = low[0..rk], .high = high[0..rk], .interior = interior[0..rk] }, - loc, - ); + .unknown(mlirCtx()), + ).appendTo(currentBlock()); return _result(res_shape, pad_op.result(0)); } @@ -2197,13 +2132,13 @@ pub const Tensor = struct { /// `.{.a = 5, .b = 4}.insert(.b, .{ .c, .d }) -> .{ .a = 5, .c = 1, .d = 1, .b = 4 }` pub fn insertAxes(self: Tensor, axis_: anytype, tags: anytype) Tensor { const tags_ = Shape.parseTags(tags); - const ax = if (@TypeOf(axis_) == EnumLiteral and axis_ == .last) + const ax = if (@TypeOf(axis_) == @EnumLiteral() and axis_ == .last) self.rank() else self.axis(axis_); var res_shape = self._shape; - const ones = [_]i64{1} ** MAX_RANK; + const ones = [_]i64{1} ** constants.MAX_RANK; res_shape._dims.insertSlice(ax, ones[0..tags_.len]) catch unreachable; res_shape._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; @@ -2232,12 +2167,11 @@ pub const Tensor = struct { pub fn reverse(self: Tensor, axes_: anytype) Tensor { const actual_axes = self._shape.axes(axes_); - const loc = self.getContext().location(@src(), "reverse({any})", .{axes_}); - const reverse_op = dialect.stablehlo.reverse(self.getContext().mlirCtx(), self.value(), toI64(actual_axes.constSlice()), loc); + const reverse_op = dialects.stablehlo.reverse(mlirCtx(), self.value(), toI64(actual_axes.constSlice()).constSlice(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, reverse_op.result(0)); } - pub const GatherOpts = struct { indices_are_sorted: bool = false }; + pub const GatherOpts = ops.GatherOpts; /// `gather` extracts slices from the given tensor at the specified offsets. /// example: `values.gather(.{ .a = idx }, .{})` @@ -2276,113 +2210,7 @@ pub const Tensor = struct { } // TODO: sort indices following self.shape instead of asking the user to do it. - return self.gather_(idx_axes.slice(), idx_per_axis.slice(), opts); - } - - pub fn gather_(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, opts: GatherOpts) Tensor { - stdx.debug.assert(idx_axes.len > 0, "gather expects 1 or more axes to operate one, received none. Example: `x.gather(.a, indices, .{{}})`", .{}); - for (idx_axes, 0..) |a, i| { - if (i > 0) { - stdx.debug.assert(a == idx_axes[i - 1] + 1, "gather expects 'idx_axes' to be sequential. But {any} aren't sequential in {f}", .{ idx_axes, self }); - } - } - var indices_shape = idx_per_axis[0].shape(); - for (idx_per_axis[1..]) |idx| { - if (idx.rank() > indices_shape.rank()) { - indices_shape = idx.shape(); - } - } - for (idx_per_axis) |idx| { - stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "gather indices can't be broadcasted together {any}", .{idx_per_axis}); - } - - var idx_batch_axes: Shape.DimsArray = .{}; - - const AxisKind = enum { batching, offset, collapsed, indices }; - var self_kind: stdx.BoundedArray(AxisKind, MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; - - for (self._shape.tags(), 0..self.rank()) |t, self_ax| { - const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, 1, @intCast(self_ax)); - if (indices_shape.hasTag(t)) |id_ax| { - // tag is both in self and indices -> it's a batching dim - // Note: tags are required for batching. - self_kind.buffer[self_ax] = .batching; - idx_batch_axes.appendAssumeCapacity(id_ax); - stdx.debug.assert(!is_gather_axis, "gather expects axes to appear at most twice. Axis {s} has been found both in 'self={f}', in 'idx_axes={any}' and in 'indices={f}'", .{ t, self, idx_axes, indices_shape }); - } else if (is_gather_axis) { - // we collapsed all gathered axes - self_kind.buffer[self_ax] = .collapsed; - // idx_kind.buffer[id_ax] = .indices; - } else { - self_kind.buffer[self_ax] = .offset; - } - } - - // compute res shape - var res_shape = Shape.init(.{}, self.dtype()); - var res_kind: stdx.BoundedArray(AxisKind, MAX_RANK) = .{}; - for (self_kind.slice(), 0..) |kind, ax_usize| { - const ax: u3 = @intCast(ax_usize); - if (ax == idx_axes[0]) { - // The first val_ax is special cause this is the place where we insert indices axes. - for (0.., indices_shape.tags(), indices_shape.dims()) |id_axis_order, id_axis, id_inserted_dim| { - const is_batching_axis = std.mem.containsAtLeastScalar(i64, idx_batch_axes.constSlice(), 1, @intCast(id_axis_order)); - // Batching axis is already in self. - if (is_batching_axis) continue; - - res_shape = res_shape.appendDim(id_inserted_dim, id_axis); - res_kind.appendAssumeCapacity(.indices); - } - } - switch (kind) { - .collapsed => continue, - else => { - res_shape = res_shape.appendDim(self.dim(ax), self._shape.tag(ax)); - res_kind.appendAssumeCapacity(kind); - }, - } - } - - // This is not a gather, but a dynamicSlice. - // Sometimes the backend recognize this pattern, but not always. - // So let us handle that. - if (indices_shape.count() == 1 and idx_axes.len == 1) { - return self.dynamicSlice1d(idx_axes[0], .{ .start = idx_per_axis[0].asScalar(), .len = 1 }).reshape(res_shape); - } - - var slice_dims: Shape.DimsArray = .{}; - for (self_kind.slice(), self.dims()) |k, d| { - slice_dims.appendAssumeCapacity(switch (k) { - .batching, .collapsed => 1, - .offset => d, - .indices => unreachable, - }); - } - - // TODO: try changing .last by other axis and see the perf impact. - const indices = Tensor.stack(idx_per_axis, .last, .coord); - // scoped_log.debug("gather --> {} {any}", .{ res_shape, res_kind.constSlice() }); - const loc = self.getContext().mlirCtx().location(@src()); - const gather_op = dialect.stablehlo.gather( - self.getContext().mlirCtx(), - self.value(), - indices.value(), - slice_dims.constSlice(), - loc, - .{ - .offset_dims = _collectAxes(AxisKind, res_kind, .offset).constSlice(), - .collapsed_slice_dims = _collectAxes(AxisKind, self_kind, .collapsed).constSlice(), - .operand_batching_dims = _collectAxes(AxisKind, self_kind, .batching).constSlice(), - .start_indices_batching_dims = idx_batch_axes.constSlice(), - .start_index_map = _collectAxes(AxisKind, self_kind, .collapsed).constSlice(), - .index_vector_dim = indices.axis(.coord), - .indices_are_sorted = opts.indices_are_sorted, - }, - ); - - const mlir_shape = fromMlirValue(gather_op.result(0)).shape(); - stdx.debug.assert(mlir_shape.eql(res_shape), "gather expects that batching indices appear in the same order in 'self' and 'indices', got: self={f}, indices={f}. You should transpose one or the other.", .{ self, indices }); - return _result(res_shape, gather_op.result(0)); + return ops.gather(self, idx_axes.slice(), idx_per_axis.slice(), opts); } test gather { @@ -2391,7 +2219,7 @@ pub const Tensor = struct { const Local = struct { pub fn _idx(idx_shape: anytype) Tensor { - return Tensor.constant(idx_shape, .{ .i32 = 0 }); + return Tensor.constant(.{ .i32 = 0 }).broad(Shape.init(idx_shape, .i64)); } }; @@ -2399,11 +2227,15 @@ pub const Tensor = struct { { // Only test shapes - var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); + const block = mlir.Block.init(&.{}, &.{}); + comp.pushBlock(block); + defer comp.popBlock(); + inline for (.{ .{ .{ .a = 10 }, .{ .a = idx(.{}) }, .{} }, .{ .{ .a = 10 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8 } }, @@ -2425,7 +2257,7 @@ pub const Tensor = struct { .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }), .b = idx(.{ .n = 8 }) }, .{ .n = 8 } }, }) |testcase| { const x_shape, const indices, const res_shape = testcase; - const x = Tensor.constant(x_shape, .{ .f16 = 0 }); + const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .i64)); const y = gather(x, indices, .{}); try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); try std.testing.expect(y.value().owner().verify()); @@ -2435,8 +2267,8 @@ pub const Tensor = struct { .{ .{ 10, 20 }, &[_]u3{ 0, 1 }, &[_]Tensor{ idx(.{8}), idx(.{8}) }, .{8} }, }) |testcase| { const x_shape, const idx_axes, const idx_per_axis, const res_shape = testcase; - const x = Tensor.constant(x_shape, .{ .f16 = 0 }); - const y = gather_(x, idx_axes, idx_per_axis, .{}); + const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .i64)); + const y = ops.gather(x, idx_axes, idx_per_axis, .{}); try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); try std.testing.expect(y.value().owner().verify()); } @@ -2486,10 +2318,10 @@ pub const Tensor = struct { // Compute result shape var res_shape = indices._shape.remove(index_coord_axis).withDtype(self.dtype()); var slice_dims = self._shape._dims; - var self_batch_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; - var indices_batch_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; - var start_index_map: stdx.BoundedArray(i64, MAX_RANK) = .{}; - var self_offset_axes: stdx.BoundedArray(i64, MAX_RANK) = .{}; + var self_batch_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; + var indices_batch_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; + var start_index_map: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; + var self_offset_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; for (self._shape.tags(), 0..self.rank()) |t, self_ax| { const maybe_slice_ax: ?u3 = if (tagged_api) slice_shape.hasTag(t) else @intCast(self_ax); @@ -2516,13 +2348,11 @@ pub const Tensor = struct { } } - const loc = self.getContext().location(@src(), "gatherSlices({f}, slice_shape={f}, idx={f})", .{ self, slice_shape, indices }); - const gather_op = dialect.stablehlo.gather( - self.getContext().mlirCtx(), + const gather_op = dialects.stablehlo.gather( + mlirCtx(), self.value(), indices.value(), slice_dims.constSlice(), - loc, .{ .offset_dims = self_offset_axes.constSlice(), .collapsed_slice_dims = &.{}, @@ -2532,7 +2362,8 @@ pub const Tensor = struct { .index_vector_dim = index_coord_axis, .indices_are_sorted = opts.indices_are_sorted, }, - ); + .unknown(mlirCtx()), + ).appendTo(currentBlock()); return _result(res_shape, gather_op.result(0)); } @@ -2548,11 +2379,15 @@ pub const Tensor = struct { { // Only test shapes - var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); + const block = mlir.Block.init(&.{}, &.{}); + comp.pushBlock(block); + defer comp.popBlock(); + inline for (.{ .{ .{ .a = 10 }, .{}, .{ ._ = 0 }, .{ .a = 10 } }, .{ .{ .a = 10 }, .{ .a = 7 }, .{ ._ = 1 }, .{ .a = 7 } }, @@ -2573,34 +2408,42 @@ pub const Tensor = struct { .{ .{ .b = 200, .a = 100, .c = 300 }, .{ .c = 300 }, .{ .a = 100, .b = 200, ._ = 1 }, .{ .a = 100, .b = 200, .c = 300 } }, }) |testcase| { const x_shape, const slice_dims, const idx_shape, const res_shape = testcase; - const x = Tensor.constant(x_shape, .{ .f16 = 0 }); + const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); const slice_shape = Shape.init(slice_dims, .u16); - const idx = Tensor.constant(idx_shape, .{ .i32 = 0 }); + const idx = Tensor.constant(.{ .i32 = 0 }).broad(Shape.init(idx_shape, .i32)); const y = gatherSlices(x, slice_shape, idx, .{}); try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); try std.testing.expect(y.value().owner().verify()); - const mod = try zml.compileFn( - std.testing.allocator, - Local._gatherSlices, - .{ x.shape(), slice_shape, idx.shape(), .{ .indices_are_sorted = true } }, - platform, - ); - defer mod.deinit(); + _ = Local._gatherSlices(x, slice_shape, idx, .{ .indices_are_sorted = true }); } } // Test with actual values. - const range = try zml.HostBuffer.arange(std.testing.allocator, .{ .end = 2 * 4 * 6 }, .u16); - defer range.deinit(std.testing.allocator); - const operand = try range.reshape(.{ .a = 2, .b = 4, .c = 6 }).toDevice(platform); - defer operand.deinit(); - const start_indices = (try zml.Buffer.fromArray(platform, [2][2]i32{ .{ 2, 1 }, .{ 0, 3 } })).withTags(.{ .n, ._ }); - defer start_indices.deinit(); + const operand: Tensor = .init(.{ .a = 2, .b = 4, .c = 6 }, .u16); + const indices: Tensor = .init(.{ .n = 2, ._ = 2 }, .i32); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._gatherSlices, .{ operand, Shape.init(.{ .b = 2, .c = 3 }, .u16), indices, .{} }, platform); + defer exe.deinit(); - const result = try zml.testing.compileAndCall(platform, Local._gatherSlices, .{ operand, Shape.init(.{ .b = 2, .c = 3 }, .u16), start_indices, .{} }); + const indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, indices.shape(), std.mem.sliceAsBytes(&[2][2]i32{ .{ 2, 1 }, .{ 0, 3 } })); + defer indices_buffer.deinit(); + const operand_buffer: zml.Buffer = b: { + const temp_func = struct { + fn forward() Tensor { + return Tensor.arange(.{ .end = 2 * 4 * 6 }, .u16).reshape(.{ .a = 2, .b = 4, .c = 6 }); + } + }.forward; + var temp_exe = try zml.module.compile(std.testing.allocator, std.testing.io, temp_func, .{}, platform); + defer temp_exe.deinit(); - const expected = zml.HostBuffer.fromArray(&[2][2][2][3]u16{ + const buffer = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &temp_exe, temp_func, {}); + break :b buffer; + }; + defer operand_buffer.deinit(); + const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._gatherSlices, .{ operand_buffer, indices_buffer }); + defer result.deinit(); + + const expected: ConstSlice = .init(Shape.init(.{ 2, 2, 2, 3 }, .u16), std.mem.sliceAsBytes(&[2][2][2][3]u16{ .{ .{ .{ 13, 14, 15 }, .{ 19, 20, 21 } }, .{ .{ 37, 38, 39 }, .{ 43, 44, 45 } }, @@ -2609,8 +2452,8 @@ pub const Tensor = struct { .{ .{ 3, 4, 5 }, .{ 9, 10, 11 } }, .{ .{ 27, 28, 29 }, .{ 33, 34, 35 } }, }, - }); - try zml.testing.expectClose(expected, result, 0); + })); + try zml.testing.expectClose(std.testing.io, expected, result, 0); } pub const ScatterOpts = struct { @@ -2628,15 +2471,14 @@ pub const Tensor = struct { /// then you should make sure the slices don't overlap, /// otherwise the result will depend on the runtime scheduling /// of the operator which is backend specific. - update_fn: *const fn (Tensor, Tensor) Tensor = increment, + update_fn: *const fn (ops.ScatterArgs) struct { Tensor } = increment, - pub fn increment(old_value: Tensor, new_value: Tensor) Tensor { - return old_value.add(new_value); + pub fn increment(values: ops.ScatterArgs) struct { Tensor } { + return .{values.input.add(values.update)}; } - pub fn override(old_value: Tensor, new_value: Tensor) Tensor { - _ = old_value; - return new_value; + pub fn override(values: ops.ScatterArgs) struct { Tensor } { + return .{values.update}; } }; @@ -2714,17 +2556,17 @@ pub const Tensor = struct { /// that requires host<->device synchronization. /// ZML tries to generate the easiest to optimize IR, and will warn you if it generates known problematic IR. pub fn scatterSlices(self: Tensor, indices: anytype, updates: Tensor, opts: ScatterOpts) Tensor { - scoped_log.debug("scatterSlices({f}, {any}, {f})", .{ self, indices, updates }); + //scoped_log.debug("scatterSlices({}, {any}, {})", .{ self, indices, updates }); const UpdateType = @TypeOf(ScatterOpts.increment); const Custom = struct { - pub fn inc(custom: *const UpdateType, old_value: Tensor, new_value: Tensor) Tensor { - return @call(.auto, custom, .{ old_value, new_value }); + pub fn inc(values: ops.ScatterArgs, custom: *const UpdateType) struct { Tensor } { + return @call(.auto, custom, .{values}); } }; - return ops.scatter(Tensor, *const UpdateType, Custom.inc, self, opts.update_fn, indices, updates, opts); + return ops.scatter(.{self}, indices, .{updates}, Custom.inc, .{opts.update_fn}, opts)[0]; } test scatterSlices { @@ -2749,16 +2591,22 @@ pub const Tensor = struct { } pub fn _idx(idx_shape: anytype) Tensor { - return Tensor.constant(idx_shape, .{ .i32 = 0 }); + return Tensor.constant(.{ .i32 = 0 }).broad(Shape.init(idx_shape, .i32)); } }; { // Only test shapes - var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); + + const block = mlir.Block.init(&.{}, &.{}); + defer block.deinit(); + comp.pushBlock(block); + defer comp.popBlock(); + const idx = Local._idx; inline for (.{ @@ -2776,8 +2624,8 @@ pub const Tensor = struct { .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }), .b = idx(.{ .n = 8 }) }, .{ .a = 3, .n = 8, .b = 2 } }, }) |testcase| { const x_shape, const indices, const updates_shapes = testcase; - const x = Tensor.constant(x_shape, .{ .f16 = 0 }); - const updates = Tensor.constant(updates_shapes, .{ .f16 = 0 }); + const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); + const updates = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(updates_shapes, .f16)); const y = scatterSlices(x, indices, updates, .{}); // Shape doesn't change with scatterSlices @@ -2787,70 +2635,79 @@ pub const Tensor = struct { } // Test with actual values, no batching. { - const a_host = try zml.HostBuffer.arange(std.testing.allocator, .{ .end = 9 }, .i32); - const a = (try zml.Buffer.from(platform, a_host.reshape(.{ 3, 3 }), .{})).withTags(.{ .a, .b }); - defer a.deinit(); - a_host.deinit(std.testing.allocator); + const a: Tensor = .init(.{ 3, 3 }, .i32); - const scatter_indices = try zml.Buffer.fromArray(platform, [2]i32{ 0, 2 }); - const updates = try zml.Buffer.fromArray(platform, [2][3]i32{ .{ 10, 20, 30 }, .{ 70, 80, 90 } }); + const scatter_indices = Tensor.init(.{2}, .i32).withTags(.{.n}); + const updates = Tensor.init(.{ 2, 3 }, .i32).withTags(.{ .n, .b }); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform); + defer exe.deinit(); + + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); + defer a_buffer.deinit(); + var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), std.mem.sliceAsBytes(&[2]i32{ 0, 2 })); + defer scatter_indices_buffer.deinit(); + var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), std.mem.sliceAsBytes(&[2][3]i32{ .{ 10, 20, 30 }, .{ 70, 80, 90 } })); + defer updates_buffer.deinit(); + + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatter, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); + defer result.deinit(); const expected = [3][3]i32{ .{ 10, 21, 32 }, .{ 3, 4, 5 }, .{ 76, 87, 98 } }; - const result = try zml.testing.compileAndCall(platform, Local._scatter, .{ - a, - &.{scatter_indices.withTags(.{.n})}, - updates.withTags(.{ .n, .b }), - }); try std.testing.expect(a.shape().eql(result.shape())); - try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected))); + try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected), std.testing.io)); } // Test with setting individual values (no batching) { - const a_host = try zml.HostBuffer.arange(std.testing.allocator, .{ .end = 9 }, .i32); - const a = try zml.Buffer.from(platform, a_host, .{}); - defer a.deinit(); - a_host.deinit(std.testing.allocator); + const a: Tensor = .init(.{9}, .i32); - const scatter_indices = try zml.Buffer.fromArray(platform, [2]i32{ 2, 7 }); - const updates = try zml.Buffer.fromArray(platform, [2]i32{ 20, 70 }); + const scatter_indices = Tensor.init(.{2}, .i32).withTags(.{.n}); + const updates = Tensor.init(.{2}, .i32).withTags(.{.n}); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform); + defer exe.deinit(); + + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); + defer a_buffer.deinit(); + var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), std.mem.sliceAsBytes(&[2]i32{ 2, 7 })); + defer scatter_indices_buffer.deinit(); + var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), std.mem.sliceAsBytes(&[2]i32{ 20, 70 })); + defer updates_buffer.deinit(); + + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatter, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); + defer result.deinit(); const expected = [9]i32{ 0, 1, 22, 3, 4, 5, 6, 77, 8 }; - const result = try zml.testing.compileAndCall(platform, Local._scatter, .{ - a, - &.{scatter_indices.withTags(.{.n})}, - updates.withTags(.{.n}), - }); try std.testing.expect(a.shape().eql(result.shape())); - try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected))); + try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected), std.testing.io)); } { // Test with actual values and batching along axis .a - const operand = try zml.Buffer.constant(platform, Shape.init(.{ .a = 2, .b = 3, .c = 4, .d = 2 }, .u16), 0, .{}); - defer operand.deinit(); - const start_indices = (try zml.Buffer.fromArray( - platform, - [2][2][3][2]i32{ - .{ - .{ .{ 0, 0 }, .{ 1, 0 }, .{ 2, 1 } }, - .{ .{ 0, 1 }, .{ 1, 1 }, .{ 0, 9 } }, - }, - .{ - .{ .{ 0, 0 }, .{ 2, 1 }, .{ 2, 2 } }, - .{ .{ 1, 2 }, .{ 0, 1 }, .{ 1, 0 } }, - }, + const operand: Tensor = .init(.{ .a = 2, .b = 3, .c = 4, .d = 2 }, .u16); + const start_indices: Tensor = .init(.{ .n = 2, .a = 2, .m = 3, .coord = 2 }, .i32); + const values: Tensor = .init(.{ .n = 2, .a = 2, .m = 3, .c = 2, .d = 2 }, .u16); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatterCB, .{ operand, start_indices, values }, platform); + defer exe.deinit(); + + var operand_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, operand.shape(), std.mem.sliceAsBytes(&@as([2 * 3 * 4 * 2]u16, @splat(0)))); + defer operand_buffer.deinit(); + var start_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, start_indices.shape(), std.mem.sliceAsBytes(&[2][2][3][2]i32{ + .{ + .{ .{ 0, 0 }, .{ 1, 0 }, .{ 2, 1 } }, + .{ .{ 0, 1 }, .{ 1, 1 }, .{ 0, 9 } }, }, - )).withTags(.{ .n, .a, .m, .coord }); - defer start_indices.deinit(); + .{ + .{ .{ 0, 0 }, .{ 2, 1 }, .{ 2, 2 } }, + .{ .{ 1, 2 }, .{ 0, 1 }, .{ 1, 0 } }, + }, + })); + defer start_indices_buffer.deinit(); + var values_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, values.shape(), std.mem.sliceAsBytes(&@as([2 * 2 * 3 * 2 * 2]u16, @splat(1)))); + defer values_buffer.deinit(); - const values = try zml.Buffer.constant( - platform, - Shape.init(.{ .n = 2, .a = 2, .m = 3, .c = 2, .d = 2 }, .u16), - 1, - .{}, - ); - defer values.deinit(); - - const result = try zml.testing.compileAndCall(platform, Local._scatterCB, .{ operand, start_indices, values }); + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatterCB, .{ operand_buffer, start_indices_buffer, values_buffer }); + defer result.deinit(); const expected = [2][3][4][2]u16{ .{ @@ -2865,7 +2722,7 @@ pub const Tensor = struct { }, }; try std.testing.expect(operand.shape().eql(result.shape())); - try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected))); + try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected), std.testing.io)); } } @@ -2873,74 +2730,68 @@ pub const Tensor = struct { pub fn max(self: Tensor, axis_: anytype) Tensor { const a = self.axis(axis_); return ops.reduce( + .{self}, + .{Tensor.constant(self.dtype().minValue())}, + &.{a}, struct { - pub fn cmp(x: Tensor, res: Tensor) Tensor { - return res.maximum(x.convert(res.dtype())); + pub fn cmp(args: ops.ReduceArgs) struct { Tensor } { + return .{args.left.maximum(args.right)}; } }.cmp, - self, - Tensor.constant(&.{}, self.dtype().minValue()), - &.{a}, - ); + .{}, + )[0]; } /// Returns a Tensor containing the minimum over a given axis. pub fn min(self: Tensor, axis_: anytype) Tensor { const a = self.axis(axis_); return ops.reduce( + .{self}, + .{Tensor.constant(self.dtype().maxValue())}, + &.{a}, struct { - pub fn cmp(x: Tensor, res: Tensor) Tensor { - return res.minimum(x.convert(res.dtype())); + pub fn cmp(args: ops.ReduceArgs) struct { Tensor } { + return .{args.left.minimum(args.right)}; } }.cmp, - self, - Tensor.constant(&.{}, self.dtype().maxValue()), - &.{a}, - ); + .{}, + )[0]; } pub const ArgMaxRes = struct { values: Tensor, indices: Tensor, - fn cmp(left: ArgMaxRes, right: ArgMaxRes) ArgMaxRes { - const left_val = left.values; - const right_val = right.values; - const left_idx = left.indices; - const right_idx = right.indices; - - const left_gt_right = left_val.cmp(.GT, right_val); - const is_nan = left_val.cmp(.NE, left_val); + fn cmp(values: ops.ReduceArgs, indices: ops.ReduceArgs) struct { Tensor, Tensor } { + const left_gt_right = values.left.cmp(.GT, values.right); + const is_nan = values.left.cmp(.NE, values.left); const left_gt_or_nan = left_gt_right.logical(.OR, is_nan); // we are bubbling up Nan. - const max_val = left_gt_or_nan.select(left_val, right_val); + const max_val = left_gt_or_nan.select(values.left, values.right); - // If left_val == right_val: keep the smallest idx. - const is_same = left_val.cmp(.EQ, right_val); - const is_first = left_idx.cmp(.LT, right_idx); + // If values.left == values.right: keep the smallest idx. + const is_same = values.left.cmp(.EQ, values.right); + const is_first = indices.left.cmp(.LT, indices.right); const is_same_but_first = is_same.logical(.AND, is_first); const keep_left_idx = left_gt_or_nan.logical(.OR, is_same_but_first); - const max_idx = keep_left_idx.select(left_idx, right_idx); + const max_idx = keep_left_idx.select(indices.left, indices.right); - return .{ .values = max_val, .indices = max_idx }; + return .{ max_val, max_idx }; } }; - /// Returns two Tensors containing the maximum and the index of this maximum over a given axis. - /// - /// Stable argmax: - /// * bubbles up Nan - /// * in case of equality the smallest index matching the maximum pub fn argMax(x: Tensor, axis_: anytype) ArgMaxRes { const a = x.axis(axis_); const dt: DataType = if (x.dim(a) <= std.math.maxInt(i32)) .i32 else .i64; - return ops.reduce( - ArgMaxRes.cmp, - .{ .values = x, .indices = Tensor.arange(.{ .end = x.dim(a) }, dt).broadcast(x.shape(), &.{a}) }, - .{ .values = Tensor.constant(&.{}, x.dtype().minValue()), .indices = Tensor.scalar(0, dt) }, + const values, const indices = ops.reduce( + .{ x, Tensor.arange(.{ .end = x.dim(axis_) }, dt).broadcast(x.shape(), &.{a}) }, + .{ Tensor.constant(x.dtype().minValue()), Tensor.constant(dt.zero()) }, &.{a}, + ArgMaxRes.cmp, + .{}, ); + return .{ .values = values, .indices = indices }; } test argMax { @@ -2953,25 +2804,33 @@ pub const Tensor = struct { } }; - const argmax = try zml.compileFn(allocator, ArgMaxTest._fwd, .{Shape.init(.{ 1, 5 }, .f32)}, platform); - defer argmax.deinit(); - // Test with tie + const x: Tensor = .init(.{ 1, 5 }, .f32); + var exe = try zml.module.compile(allocator, std.testing.io, ArgMaxTest._fwd, .{x}, platform); + defer exe.deinit(); + { - const x = try zml.Buffer.fromArray(platform, [1][5]f32{.{ 5.0, 4.1, 7.9, 0, 7.9 }}); - const res = argmax.call(.{x}); - const max_ = res.values.getValue(f32); - const max_idx = res.indices.getValue(i32); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, 4.1, 7.9, 0, 7.9 }})); + defer x_buffer.deinit(); + + const res = try zml.testing.autoCall(allocator, std.testing.io, &exe, ArgMaxTest._fwd, .{x_buffer}); + defer res.values.deinit(); + defer res.indices.deinit(); + const max_ = res.values.getValue(f32, std.testing.io); + const max_idx = res.indices.getValue(i32, std.testing.io); try std.testing.expectEqual(max_, 7.9); // We should always return the first max found. try std.testing.expectEqual(max_idx, 2); } - // Test with Nan { - const x = try zml.Buffer.fromArray(platform, [1][5]f32{.{ 5.0, std.math.nan(f32), 7.9, 0, 7.9 }}); - const res = argmax.call(.{x}); - const max_ = try res.values.getValue(f32); - const max_idx = try res.indices.getValue(i32); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, std.math.nan(f32), 7.9, 0, 7.9 }})); + defer x_buffer.deinit(); + + const res = try zml.testing.autoCall(allocator, std.testing.io, &exe, ArgMaxTest._fwd, .{x_buffer}); + defer res.values.deinit(); + defer res.indices.deinit(); + const max_ = try res.values.getValue(f32, std.testing.io); + const max_idx = try res.indices.getValue(i32, std.testing.io); try std.testing.expect(std.math.isNan(max_)); try std.testing.expectEqual(max_idx, 1); } @@ -2979,25 +2838,22 @@ pub const Tensor = struct { pub const SortRes = ArgMaxRes; + pub const SortOpts = struct { descending: bool = false }; + /// Returns two Tensors. The first contains the sorted values and the second one contains the sorted indices. - pub fn sort(self: Tensor, axis_: anytype, opts: struct { descending: bool = false }) SortRes { + pub fn sort(self: Tensor, axis_: anytype, opts: SortOpts) SortRes { const a = self.axis(axis_); const indices = Tensor.arange(.{ .end = self.dim(a) }, .i32).broadcast(self._shape, &.{a}); - const res = ops.sort( - struct { - fn call(direction: dialect.stablehlo.ComparisonDirection.Direction, lhs: Tensor, rhs: Tensor, _: Tensor, _: Tensor) Tensor { - return lhs.cmp(direction, rhs); - } - }.call, - if (opts.descending) .GT else .LT, - .{ self, indices }, - self.axis(axis_), - true, - ); + const direction: dialects.stablehlo.ComparisonDirection.Direction = if (opts.descending) .GT else .LT; + const res = ops.sort(.{ self, indices }, a, struct { + fn call(values: ops.SortArgs, _: ops.SortArgs, direction_: dialects.stablehlo.ComparisonDirection.Direction) Tensor { + return values.left.cmp(direction_, values.right); + } + }.call, .{direction}, true); return .{ .values = res[0], .indices = res[1] }; } - pub const ArgSortOpts = struct { descending: bool = false }; + pub const ArgSortOpts = SortOpts; /// Returns a Tensor containing the indices corresponding to the sorted values over the given axis. pub fn argsort(self: Tensor, axis_: anytype, opts: ArgSortOpts) Tensor { @@ -3014,38 +2870,64 @@ pub const Tensor = struct { } }; - var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena_state.deinit(); - const allocator = arena_state.allocator(); // 2D Tensor - dim = 1, ascending { - const x = try zml.Buffer.fromSlice(platform, .{ 2, 5 }, &[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 }); - const res = try zml.testing.compileAndCall(platform, Local._argsort, .{ x, 1, .{} }); - const res_cpu = try res.toHostAlloc(allocator); + const x: Tensor = .init(.{ 2, 5 }, .f32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{} }, platform); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 })); + defer x_buffer.deinit(); + + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); + defer res.deinit(); + + const res_cpu = try res.toSliceAlloc(std.testing.allocator, std.testing.io); + defer res_cpu.free(std.testing.allocator); + try std.testing.expectEqualSlices(i32, &.{ 0, 3, 1, 2, 4, 1, 4, 3, 0, 2 }, res_cpu.items(i32)); } + // 3D Tensor, dim = 1, descending { - const x = try zml.Buffer.fromSlice(platform, .{ 1, 5, 10 }, &[_]f16{ + const x: Tensor = .init(.{ 1, 5, 10 }, .f16); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{ .descending = true } }, platform); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f16{ -0.2505, 1.2520, -0.7041, 0.1066, 1.2773, -1.7246, 0.8389, 1.1094, 0.0601, 1.0684, 0.9619, 1.3916, 1.2246, -0.1406, 0.3674, -1.2480, -1.7051, -0.0934, 0.3435, 0.4373, 1.3809, 0.5444, -0.6079, 1.2031, -0.6880, 1.2979, -0.1869, 0.2991, 0.0156, 0.1847, 0.6626, -0.3040, -0.8726, -1.4805, -1.6943, 1.1055, -2.0078, -0.5288, 0.8813, 0.8008, 2.0527, 1.1230, 0.5430, 0.2494, -0.9434, 0.7876, 0.1818, 0.9258, -2.4902, 1.5918, - }); - const res_dev = try zml.testing.compileAndCall(platform, Local._argsort, .{ x, 1, .{ .descending = true } }); - const res = try res_dev.toHostAlloc(allocator); + })); + defer x_buffer.deinit(); + + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); + defer res.deinit(); + + const res_cpu = try res.toSliceAlloc(std.testing.allocator, std.testing.io); + defer res_cpu.free(std.testing.allocator); + try std.testing.expectEqualSlices(i32, &.{ 4, 1, 1, 2, 0, 2, 0, 0, 3, 4, 2, 0, 4, 4, 1, 3, 4, 4, 1, 0, 1, 4, 2, 0, 2, 4, 2, 2, 0, 3, 3, 2, 0, 1, 4, 1, 1, 1, 2, 1, 0, 3, 3, 3, 3, 0, 3, 3, 4, 2, - }, res.items(i32)); + }, res_cpu.items(i32)); } + // 4D Tensor, dim = 3, ascending { - const x = try zml.Buffer.fromSlice(platform, .{ 4, 2, 1, 4 }, &[_]i32{ + const x: Tensor = .init(.{ 4, 2, 1, 4 }, .i32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 3, .{} }, platform); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 89, 31, 22, 42, 64, 39, 0, 30, 64, 71, 46, 31, @@ -3054,9 +2936,15 @@ pub const Tensor = struct { 93, 24, 45, 72, 64, 86, 62, 88, 57, 21, 19, 12, - }); - const res_dev = try zml.testing.compileAndCall(platform, Local._argsort, .{ x, 3, .{} }); - const res = try res_dev.toHostAlloc(allocator); + })); + defer x_buffer.deinit(); + + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); + defer res.deinit(); + + const res_cpu = try res.toSliceAlloc(std.testing.allocator, std.testing.io); + defer res_cpu.free(std.testing.allocator); + try std.testing.expectEqualSlices(i32, &.{ 2, 1, 3, 0, 2, 3, 1, 0, @@ -3066,12 +2954,14 @@ pub const Tensor = struct { 1, 2, 3, 0, 2, 0, 1, 3, 3, 2, 1, 0, - }, res.items(i32)); + }, res_cpu.items(i32)); } } + pub const TopKOpts = struct { descending: bool = true }; + /// Returns a Tensor representing the result of Top-K over the given axis. - pub fn topK(self: Tensor, named_axis_: anytype, k: u32, opts: struct { descending: bool = true }) SortRes { + pub fn topK(self: Tensor, named_axis_: anytype, k: u32, opts: TopKOpts) SortRes { stdx.debug.assert(k > 0, "topK expects a k > 0, got 0", .{}); const err_msg = "topK named axis should be an integer or a named axis, eg `x.topK(.{{ .best_token = .token }}, 16)` or `x.topK(-1, 16)`"; const has_name: ?[:0]const u8, const a = switch (@typeInfo(@TypeOf(named_axis_))) { @@ -3113,7 +3003,7 @@ pub const Tensor = struct { // TODO: support maxPool on non last axis const a = self.axis(-1); - const ones = [_]i64{1} ** Tensor.MAX_RANK; + const ones = [_]i64{1} ** constants.MAX_RANK; var window_dimensions = ones; window_dimensions[a] = opts.window_dimensions; var window_strides = window_dimensions; @@ -3124,13 +3014,12 @@ pub const Tensor = struct { var window_dilations = ones; window_dilations[a] = opts.window_dilations; - var padding = [_][2]i64{.{ 0, 0 }} ** Tensor.MAX_RANK; + var padding = [_][2]i64{.{ 0, 0 }} ** constants.MAX_RANK; padding[a] = opts.padding; - return ops.reduceWindow( - MaxPoolRes.cmp, - .{ .values = self, .indices = iota(self._shape, a) }, - .{ .values = Tensor.constant(.{}, self.dtype().minValue()), .indices = Tensor.scalar(0, .i32) }, + const result = ops.reduceWindow( + .{ self, iota(self._shape, a) }, + .{ Tensor.constant(self.dtype().minValue()), Tensor.scalar(0, .i32) }, .{ .window_dimensions = window_dimensions[0..self.rank()], .window_strides = window_strides[0..self.rank()], @@ -3138,7 +3027,10 @@ pub const Tensor = struct { .window_dilations = window_dilations[0..self.rank()], .padding = padding[0..self.rank()], }, + MaxPoolRes.cmp, + .{}, ); + return .{ .values = result[0], .indices = result[1] }; } /// Computes the 2d maxPool operation on the input Tensor. @@ -3161,14 +3053,13 @@ pub const Tensor = struct { const base_dilation = initPoolArg(self.rank(), &opts.base_dilations); const window_dilations = initPoolArg(self.rank(), &opts.window_dilations); - var padding = [_][2]i64{.{ 0, 0 }} ** Tensor.MAX_RANK; + var padding = [_][2]i64{.{ 0, 0 }} ** constants.MAX_RANK; padding[a - 1] = opts.padding[0]; padding[a] = opts.padding[1]; - return ops.reduceWindow( - MaxPoolRes.cmp, - .{ .values = self, .indices = iota(self._shape, a) }, - .{ .values = Tensor.constant(.{}, self.dtype().minValue()), .indices = Tensor.scalar(0, .i32) }, + const result = ops.reduceWindow( + .{ self, iota(self._shape, a) }, + .{ Tensor.constant(self.dtype().minValue()), Tensor.scalar(0, .i32) }, .{ .window_dimensions = window_dimensions[0..self.rank()], .window_strides = window_strides[0..self.rank()], @@ -3176,7 +3067,10 @@ pub const Tensor = struct { .window_dilations = window_dilations[0..self.rank()], .padding = padding[0..self.rank()], }, + MaxPoolRes.cmp, + .{}, ); + return .{ .values = result[0], .indices = result[1] }; } /// Chunk a given tensor into exactly n parts of equal shape. @@ -3198,18 +3092,23 @@ pub const Tensor = struct { const platform = zml.testing.env(); // Only test shapes - var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); + const block = mlir.Block.init(&.{}, &.{}); + defer block.deinit(); + comp.pushBlock(block); + defer comp.popBlock(); + inline for (.{ .{ .{ .a = 12 }, .a, 3, .{ .a = 4 } }, .{ .{ .a = 12, .b = 2 }, .a, 3, .{ .a = 4, .b = 2 } }, .{ .{ 12, 2 }, 0, 3, .{ 4, 2 } }, }) |testcase| { const x_shape, const ax, const n_chunks, const res = testcase; - const x = Tensor.constant(x_shape, .{ .f16 = 0 }); + const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); const chunks = x.chunkExact(ax, n_chunks); const res_shape = Shape.init(res, .f16); @@ -3225,14 +3124,17 @@ pub const Tensor = struct { self: Tensor, axis_: i64, n_chunks: comptime_int, - ) []Tensor { + ) ![]Tensor { const a = self.axis(axis_); const d = self.dim(a); const chunk_size: i64 = @divFloor(d, n_chunks); const tail_chunk_size: i64 = @rem(d, chunk_size); - const allocator = self.getContext().allocator(); + const allocator = CompilationContext.current().arena.allocator(); + var chunks = std.ArrayList(Tensor).initCapacity(allocator, n_chunks + 1) catch @panic("OOM"); + defer chunks.deinit(allocator); + for (0..n_chunks) |i| { const start: i64 = @as(i64, @intCast(i)) * chunk_size; chunks.appendAssumeCapacity( @@ -3243,7 +3145,7 @@ pub const Tensor = struct { const start: i64 = n_chunks * chunk_size; chunks.appendAssumeCapacity(self.slice1d(a, .{ .start = start })); } - return chunks.items; + return chunks.toOwnedSlice(allocator); } test chunkAllowTrailing { @@ -3251,11 +3153,16 @@ pub const Tensor = struct { const platform = zml.testing.env(); // Only test shapes - var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); + const block = mlir.Block.init(&.{}, &.{}); + defer block.deinit(); + comp.pushBlock(block); + defer comp.popBlock(); + inline for (.{ .{ .{ .a = 10 }, .a, 3, .{ .a = 3 }, .{ .a = 1 } }, .{ .{ .a = 10, .b = 2 }, .a, 3, .{ .a = 3, .b = 2 }, .{ .a = 1, .b = 2 } }, @@ -3263,8 +3170,8 @@ pub const Tensor = struct { .{ .{ 12, 2 }, 0, 3, .{ 4, 2 }, .{} }, }) |testcase| { const x_shape, const ax, const n_chunks, const res, const trailing = testcase; - const x = Tensor.constant(x_shape, .{ .f16 = 0 }); - const chunks = x.chunkAllowTrailing(x.axis(ax), n_chunks); + const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); + const chunks = try x.chunkAllowTrailing(x.axis(ax), n_chunks); const res_shape = Shape.init(res, .f16); for (chunks[0..n_chunks]) |chk| { @@ -3289,9 +3196,10 @@ pub const Tensor = struct { for (split_sizes) |n| split_sum += n; stdx.debug.assert(split_sum == d, "split expects sum of 'split_sizes' values and axis dimension to be equal, got {} and {}", .{ split_sum, d }); - const allocator = self.getContext().allocator(); - const res = allocator.alloc(Tensor, split_sizes.len) catch @panic("OOM"); - errdefer allocator.dealloc(res); + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const res = arena.allocator().alloc(Tensor, split_sizes.len) catch @panic("OOM"); var start: i64 = 0; for (split_sizes, 0..) |n, i| { @@ -3309,18 +3217,17 @@ pub const Tensor = struct { const a = self.axis(axis_); const new_shape = self._shape.set(a, slice_.len); - const loc = self.getContext().location(@src(), "dynSlice({}, len={})", .{ axis_, slice_.len }); - var start_indices = [_]mlir.Value{constant(.{}, slice_.start.dtype().zero()).value()} ** MAX_RANK; + var start_indices = [_]*const mlir.Value{constant(slice_.start.dtype().zero()).value()} ** constants.MAX_RANK; start_indices[a] = slice_.start.value(); - const op = dialect.stablehlo.dynamic_slice( - self.getContext().mlirCtx(), + const op = dialects.stablehlo.dynamic_slice( + mlirCtx(), self.value(), new_shape.dims(), start_indices[0..self.rank()], - loc, - ); + .unknown(mlirCtx()), + ).appendTo(currentBlock()); return _result(new_shape, op.result(0)); } @@ -3349,11 +3256,10 @@ pub const Tensor = struct { // TODO use slices and slices_tags for the format. // Currently this prints: "dynSlice(struct{q: struct{start: tensor.Tensor, comptime len: comptime_int = 1}}{ .q = struct{start: tensor.Tensor, comptime len: comptime_int = 1}{ .start = Tensor({1,10}, dtype=.i64), .len = 1 } })" // which is kinda ugly. - const loc = self.getContext().location(@src(), "dynSlice({any})", .{slices_}); const idx_dtype = if (slices.len > 0) slices.get(0).start.dtype() else .i32; const zero = Tensor.scalar(0, idx_dtype).value(); - var offset_values = [_]mlir.Value{zero} ** MAX_RANK; + var offset_values = [_]*const mlir.Value{zero} ** constants.MAX_RANK; var res_shape = self._shape; for (slices.constSlice(), 0..) |slice_, i| { const offset = slice_.start; @@ -3375,30 +3281,34 @@ pub const Tensor = struct { res_shape._dims.set(a, len); } } - const op = dialect.stablehlo.dynamic_slice(self.getContext().mlirCtx(), self.value(), res_shape.dims(), offset_values[0..self.rank()], loc); + const op = dialects.stablehlo.dynamic_slice(mlirCtx(), self.value(), res_shape.dims(), offset_values[0..self.rank()], .unknown(mlirCtx())).appendTo(currentBlock()); return _result(res_shape, op.result(0)); } test dynamicSlice { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const T = f32; - { - const x = try zml.Buffer.fromArray(platform, [10]T{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); - const z = try zml.Buffer.scalar(platform, 4, .i32); - const res = try zml.testing.compileAndCall(platform, Tensor.dynamicSlice1d, .{ x, 0, .{ .len = 2, .start = z } }); + inline for (.{ + .{ [10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, Shape.init(.{10}, .f32), [2]f32{ 4, 5 }, 4, 0 }, + .{ [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }, Shape.init(.{ 2, 5 }, .f32), [4]f32{ 3, 4, 8, 9 }, 3, 1 }, + }) |testcase| { + const x_data, const x_shape, const expectation, const z_value: i32, const ax = testcase; + const x: Tensor = .fromShape(x_shape); + const z: Tensor = .init(.{}, .i32); - try std.testing.expectEqual([2]T{ 4, 5 }, try res.getValue([2]T)); - } + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.dynamicSlice1d, .{ x, ax, .{ .len = 2, .start = z } }, platform); + defer exe.deinit(); - { - // Strided - const x = try zml.Buffer.fromArray(platform, [2][5]T{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }); - const z = try zml.Buffer.scalar(platform, 3, .i32); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&x_data)); + defer x_buffer.deinit(); + var z_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, z.shape(), std.mem.asBytes(&z_value)); + defer z_buffer.deinit(); - const res = try zml.testing.compileAndCall(platform, Tensor.dynamicSlice1d, .{ x, 1, .{ .len = 2, .start = z } }); - try std.testing.expectEqual([4]T{ 3, 4, 8, 9 }, res.getValue([4]T)); + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.dynamicSlice1d, .{ x_buffer, .{ .start = z_buffer } }); + defer res.deinit(); + + try std.testing.expectEqual(expectation, try res.getValue(@TypeOf(expectation), std.testing.io)); } } @@ -3406,7 +3316,7 @@ pub const Tensor = struct { /// Note this is the untagged api, if you have tags, you should use dynamicUpdateSlice directly. pub fn dynamicUpdateSlice1d(self: Tensor, update: Tensor, axis_: i64, offset: Tensor) Tensor { const placeholder = Tensor.scalar(0, .i32); - var start_indices = [_]Tensor{placeholder} ** MAX_RANK; + var start_indices = [_]Tensor{placeholder} ** constants.MAX_RANK; start_indices[self.axis(axis_)] = offset; return self.dynamicUpdateSlice(start_indices[0..self.rank()], update); } @@ -3470,7 +3380,7 @@ pub const Tensor = struct { const idx_dtype = if (offset.len > 0) offset.get(0).dtype() else .i32; const zero = Tensor.scalar(0, idx_dtype).value(); - var offset_values: [MAX_RANK]mlir.Value = undefined; + var offset_values: [constants.MAX_RANK]*const mlir.Value = undefined; if (offset_tags.len == 0) { // Without offset tags we need the same number of offset than rank. stdx.debug.assert(self.rank() == offset.len, "dynamicUpdateSlice expects input tensor rank and 'offset_' length to be equal, got {} and {}", .{ self.rank(), offset.len }); @@ -3481,21 +3391,20 @@ pub const Tensor = struct { } else { // If an axis isn't specified, update the full slice. // This is only allowed when using tagged sliced. - offset_values = .{zero} ** MAX_RANK; + offset_values = .{zero} ** constants.MAX_RANK; for (offset.constSlice(), offset_tags.constSlice()) |start, t| { const a = self._shape.hasTag(t) orelse stdx.debug.panic("dynamicUpdateSlice expects input tensor to have tags used in 'offset_' but {s} is missing (input shape is {f})", .{ t, self._shape }); offset_values[a] = start.value(); } } - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.dynamic_update_slice( - self.getContext().mlirCtx(), + const op = dialects.stablehlo.dynamic_update_slice( + mlirCtx(), self.value(), update.value(), offset_values[0..self.rank()], - loc, - ); + .unknown(mlirCtx()), + ).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -3504,104 +3413,155 @@ pub const Tensor = struct { const platform = zml.testing.env(); { - const x = try zml.Buffer.fromArray(platform, [10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); - const y = try zml.Buffer.fromArray(platform, [2]f32{ -1, -1 }); - const idx = try zml.Buffer.scalar(platform, 4, .i32); - const res = try zml.testing.compileAndCall( - platform, - struct { - pub fn _fwd(x_: Tensor, idx_: struct { a: Tensor }, y_: Tensor) Tensor { - return x_.dynamicUpdateSlice(idx_, y_); - } - }._fwd, - .{ x.withTags(.{.a}), .{ .a = idx }, y.withTags(.{.a}) }, - ); - try std.testing.expectEqual([10]f32{ 0, 1, 2, 3, -1, -1, 6, 7, 8, 9 }, try res.getValue([10]f32)); + const x = Tensor.init(.{10}, .f32).withTags(.{.a}); + const y = Tensor.init(.{2}, .f32).withTags(.{.a}); + const ids: Tensor = .init(.{}, .i32); + const forward = struct { + pub fn _fwd(x_: Tensor, idx_: struct { a: Tensor }, y_: Tensor) Tensor { + return x_.dynamicUpdateSlice(idx_, y_); + } + }._fwd; + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = ids }, y }, platform); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); + defer x_buffer.deinit(); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); + defer y_buffer.deinit(); + var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{4})); + defer ids_buffer.deinit(); + + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ .a = ids_buffer }, y_buffer }); + defer res.deinit(); + + try std.testing.expectEqual([10]f32{ 0, 1, 2, 3, -1, -1, 6, 7, 8, 9 }, try res.getValue([10]f32, std.testing.io)); } { // Updates 2D, tagged api - const x = try zml.Buffer.fromArray(platform, [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }); - const y = try zml.Buffer.fromArray(platform, [2]f32{ -1, -1 }); - const idx = try zml.Buffer.scalar(platform, 3, .i32); + const x = Tensor.init(.{ 2, 5 }, .f32).withTags(.{ .a, .b }); + const y = Tensor.init(.{2}, .f32).withTags(.{.a}); + const ids: Tensor = .init(.{}, .i32); + const forward = struct { + pub fn _fwd(x_: Tensor, idx_: Tensor, y_: Tensor) Tensor { + return x_.dynamicUpdateSlice(.{ .b = idx_ }, y_); + } + }._fwd; + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + defer x_buffer.deinit(); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); + defer y_buffer.deinit(); + var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3})); + defer ids_buffer.deinit(); + + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, ids_buffer, y_buffer }); + defer res.deinit(); - const res = try zml.testing.compileAndCall( - platform, - struct { - pub fn _fwd(x_: Tensor, idx_: Tensor, y_: Tensor) Tensor { - return x_.dynamicUpdateSlice(.{ .b = idx_ }, y_); - } - }._fwd, - .{ x.withTags(.{ .a, .b }), idx, y.withTags(.{.a}) }, - ); try std.testing.expectEqualDeep( [2][5]f32{ .{ 0, 1, 2, -1, 4 }, .{ 5, 6, 7, -1, 9 } }, - try res.getValue([2][5]f32), + try res.getValue([2][5]f32, std.testing.io), ); } { // Updates 2D slice, un-tagged api. Note that `y` needs to have a 1 dimension axis. - const x = try zml.Buffer.fromArray(platform, [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }); - const y = try zml.Buffer.fromArray(platform, [2][1]f32{ .{-1}, .{-1} }); - const idx = try zml.Buffer.scalar(platform, 3, .i32); - const res = try zml.testing.compileAndCall( - platform, - struct { - pub fn _fwd(x_: Tensor, idx_: Tensor, y_: Tensor) Tensor { - return x_.dynamicUpdateSlice(.{ zml.Tensor.scalar(0, .i32), idx_ }, y_); - } - }._fwd, - .{ x, idx, y }, - ); + const x: Tensor = .init(.{ 2, 5 }, .f32); + const y: Tensor = .init(.{ 2, 1 }, .f32); + const ids: Tensor = .init(.{}, .i32); + const forward = struct { + pub fn _fwd(x_: Tensor, idx_: Tensor, y_: Tensor) Tensor { + return x_.dynamicUpdateSlice(.{ zml.Tensor.scalar(0, .i32), idx_ }, y_); + } + }._fwd; + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + defer x_buffer.deinit(); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[2][1]f32{ .{-1}, .{-1} })); + defer y_buffer.deinit(); + var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3})); + defer ids_buffer.deinit(); + + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, ids_buffer, y_buffer }); + defer res.deinit(); + try std.testing.expectEqualDeep( [2][5]f32{ .{ 0, 1, 2, -1, 4 }, .{ 5, 6, 7, -1, 9 } }, - res.getValue([2][5]f32), + res.getValue([2][5]f32, std.testing.io), ); } { // Updates 2D, partial update - const x = try zml.Buffer.fromArray(platform, [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }); - const y = try zml.Buffer.fromArray(platform, [1]f32{-1}); - const idx_a = try zml.Buffer.scalar(platform, 1, .i32); - const idx_b = try zml.Buffer.scalar(platform, 3, .i32); - const res = try zml.testing.compileAndCall( - platform, - struct { - pub fn _fwd(x_: Tensor, idx_: struct { a: Tensor, b: Tensor }, y_: Tensor) Tensor { - return x_.dynamicUpdateSlice(idx_, y_); - } - }._fwd, - .{ x.withTags(.{ .a, .b }), .{ .a = idx_a, .b = idx_b }, y.withTags(.{.a}) }, - ); + const x = Tensor.init(.{ 2, 5 }, .f32).withTags(.{ .a, .b }); + const y = Tensor.init(.{1}, .f32).withTags(.{.a}); + const idx_a: Tensor = .init(.{}, .i32); + const idx_b: Tensor = .init(.{}, .i32); + const forward = struct { + pub fn _fwd(x_: Tensor, idx_: struct { a: Tensor, b: Tensor }, y_: Tensor) Tensor { + return x_.dynamicUpdateSlice(idx_, y_); + } + }._fwd; + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = idx_a, .b = idx_b }, y }, platform); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + defer x_buffer.deinit(); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[1]f32{-1})); + defer y_buffer.deinit(); + var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), std.mem.sliceAsBytes(&[1]i32{1})); + defer idx_a_buffer.deinit(); + var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3})); + defer idx_b_buffer.deinit(); + + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ .a = idx_a_buffer, .b = idx_b_buffer }, y_buffer }); + defer res.deinit(); + try std.testing.expectEqualDeep( [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, -1, 9 } }, - res.getValue([2][5]f32), + res.getValue([2][5]f32, std.testing.io), ); } { // Updates 2D, partial update, un-tagged api. - const x = try zml.Buffer.fromArray(platform, [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }); - const y = try zml.Buffer.fromArray(platform, [1][1]f32{.{-1}}); - const idx_a = try zml.Buffer.scalar(platform, 1, .i32); - const idx_b = try zml.Buffer.scalar(platform, 3, .i32); - const A = struct { + const x: Tensor = .init(.{ 2, 5 }, .f32); + const y: Tensor = .init(.{ 1, 1 }, .f32); + const idx_a: Tensor = .init(.{}, .i32); + const idx_b: Tensor = .init(.{}, .i32); + const forward = struct { pub fn _fwd(x_: Tensor, idx_: [2]Tensor, y_: Tensor) Tensor { return x_.dynamicUpdateSlice(&idx_, y_); } - }; - const res = try zml.testing.compileAndCall(platform, A._fwd, .{ x, .{ idx_a, idx_b }, y }); + }._fwd; + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ idx_a, idx_b }, y }, platform); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + defer x_buffer.deinit(); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[1][1]f32{.{-1}})); + defer y_buffer.deinit(); + var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), std.mem.sliceAsBytes(&[1]i32{1})); + defer idx_a_buffer.deinit(); + var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3})); + defer idx_b_buffer.deinit(); + + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ idx_a_buffer, idx_b_buffer }, y_buffer }); + defer res.deinit(); + try std.testing.expectEqualDeep( [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, -1, 9 } }, - res.getValue([2][5]f32), + res.getValue([2][5]f32, std.testing.io), ); } } /// Returns a Tensor containing the element-wise result of the given 'cmp' comparison between the two input Tensors. - pub fn cmp(self: Tensor, direction: dialect.stablehlo.ComparisonDirection.Direction, other: Tensor) Tensor { + pub fn cmp(self: Tensor, direction: dialects.stablehlo.ComparisonDirection.Direction, other: Tensor) Tensor { stdx.debug.assert(self.dtype() == other.dtype(), "cmp expects input tensors to be of the same type, got {t} and {t}", .{ self.dtype(), other.dtype() }); if (self.rank() == 0 and other.rank() != 0) return self.broadcast(other._shape, &.{}).cmp(direction, other); @@ -3609,31 +3569,33 @@ pub const Tensor = struct { stdx.debug.assert(self._shape.eql(other._shape), "cmp expects input tensor shapes to match, got {f} and {f}", .{ self._shape, other._shape }); - const loc = self.getContext().location(@src(), "cmp(.{s})", .{@tagName(direction)}); - const op = dialect.stablehlo.compare( - self.getContext().mlirCtx(), + const op = dialects.stablehlo.compare( + mlirCtx(), self.value(), other.value(), - dialect.stablehlo.ComparisonDirection.init(self.getContext().mlirCtx(), direction), - getComparisonType(self.getContext().mlirCtx(), self.dtype()), - loc, - ); + dialects.stablehlo.ComparisonDirection.init(mlirCtx(), direction).getValue(), + getComparisonType(mlirCtx(), self.dtype()).getValue(), + .unknown(mlirCtx()), + ).appendTo(currentBlock()); return _result(self._shape.withDtype(.bool), op.result(0)); } /// For each vector in the input tensor, /// creates a diagonal-matrix where diagonal values are set to the vector values. - pub fn toDiagonal(self: Tensor, axis_: anytype, new_tags: [2]EnumLiteral) Tensor { - stdx.debug.assert(self.rank() < MAX_RANK - 1, "toDiagonal expects input up to {d} rank, got {f}", .{ MAX_RANK - 1, self }); + pub fn toDiagonal(self: Tensor, axis_: anytype, new_tags: [2]@EnumLiteral()) Tensor { + stdx.debug.assert(self.rank() < constants.MAX_RANK - 1, "toDiagonal expects input up to {d} rank, got {f}", .{ constants.MAX_RANK - 1, self }); const a = self.axis(axis_); const d = self.dim(a); + const p = self.shape()._partitioning.get(a); var res_shape = self._shape; res_shape._dims.replaceRange(a, 1, &.{ d, d }) catch unreachable; res_shape._tags.replaceRange(a, 1, &.{ @tagName(new_tags[0]), @tagName(new_tags[1]) }) catch unreachable; + // TODO(Corentin): Not sure about that + res_shape._partitioning.replaceRange(a, 1, &.{ p, p }) catch unreachable; const values = self.insertAxes(a + 1, .{new_tags[1]}).broad(res_shape); - const zeros = Tensor.constant(res_shape, self.dtype().zero()); + const zeros = Tensor.constant(self.dtype().zero()).broad(res_shape); const x = Tensor.iota(res_shape, a); const y = Tensor.iota(res_shape, a + 1); @@ -3655,20 +3617,26 @@ pub const Tensor = struct { } }; - const x = try zml.Buffer.fromArray(platform, [2][2]u8{ .{ 1, 2 }, .{ 3, 4 } }); - { - const res = try zml.testing.compileAndCall(platform, Local._toDiag, .{x}); - try std.testing.expectEqual( - [2][2][2]u8{ .{ - .{ 1, 0 }, - .{ 0, 2 }, - }, .{ - .{ 3, 0 }, - .{ 0, 4 }, - } }, - try res.getValue([2][2][2]u8), - ); - } + const x: Tensor = .init(.{ 2, 2 }, .u8); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._toDiag, .{x}, platform); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][2]u8{ .{ 1, 2 }, .{ 3, 4 } })); + defer x_buffer.deinit(); + + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._toDiag, .{x_buffer}); + defer res.deinit(); + try std.testing.expectEqual( + [2][2][2]u8{ .{ + .{ 1, 0 }, + .{ 0, 2 }, + }, .{ + .{ 3, 0 }, + .{ 0, 4 }, + } }, + try res.getValue([2][2][2]u8, std.testing.io), + ); } /// For each matrix specified by the two axes, returns the lower triangular part of it. @@ -3688,7 +3656,7 @@ pub const Tensor = struct { const x = Tensor.iota(self.shape(), _axes.get(0)); const y = Tensor.iota(self.shape(), _axes.get(1)); - const zeros = Tensor.constant(self.shape(), self.dtype().zero()); + const zeros = Tensor.constant(self.dtype().zero()).broad(self.shape()); return x.addConstant(num_diagonals).cmp(.GE, y).select(self, zeros); } @@ -3702,42 +3670,61 @@ pub const Tensor = struct { } }; - const x = try zml.Buffer.fromArray(platform, [3][3]u8{ + const x: Tensor = .init(.{ 3, 3 }, .u8); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[3][3]u8{ .{ 1, 1, 1 }, .{ 1, 1, 1 }, .{ 1, 1, 1 }, - }); + })); + defer x_buffer.deinit(); + { - const res = try zml.testing.compileAndCall(platform, Local._tri, .{ x, 0 }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 0 }, platform); + defer exe.deinit(); + + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); + defer res.deinit(); + try std.testing.expectEqual( [3][3]u8{ .{ 1, 0, 0 }, .{ 1, 1, 0 }, .{ 1, 1, 1 }, }, - try res.getValue([3][3]u8), + try res.getValue([3][3]u8, std.testing.io), ); } { - const res = try zml.testing.compileAndCall(platform, Local._tri, .{ x, 1 }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 1 }, platform); + defer exe.deinit(); + + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); + defer res.deinit(); + try std.testing.expectEqual( [3][3]u8{ .{ 1, 1, 0 }, .{ 1, 1, 1 }, .{ 1, 1, 1 }, }, - try res.getValue([3][3]u8), + try res.getValue([3][3]u8, std.testing.io), ); } { - const res = try zml.testing.compileAndCall(platform, Local._tri, .{ x, -1 }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, -1 }, platform); + defer exe.deinit(); + + const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); + defer res.deinit(); + try std.testing.expectEqual( [3][3]u8{ .{ 0, 0, 0 }, .{ 1, 0, 0 }, .{ 1, 1, 0 }, }, - try res.getValue([3][3]u8), + try res.getValue([3][3]u8, std.testing.io), ); } } @@ -3758,22 +3745,20 @@ pub const Tensor = struct { stdx.debug.assert(bool_tensor._shape.eqlDims(on_true._shape), "select expects input tensor and 'on_true' tensor dimensions to match, got {f} and {f}", .{ bool_tensor._shape, on_true._shape }); stdx.debug.assert(bool_tensor._shape.eqlDims(on_false._shape), "select expects input tensor and 'on_false' tensor dimensions to match, got {f} and {f}", .{ bool_tensor._shape, on_false._shape }); - const loc = bool_tensor.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.select( - bool_tensor.getContext().mlirCtx(), + const op = dialects.stablehlo.select( + mlirCtx(), bool_tensor.value(), on_true.value(), on_false.value(), - loc, - ); + .unknown(mlirCtx()), + ).appendTo(currentBlock()); return _result(on_true._shape, op.result(0)); } /// Returns a Tensor containing the element-wise not logical operation of the input Tensor. pub fn not(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.not(self.getContext().mlirCtx(), self.value(), loc); + const op = dialects.stablehlo.not(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -3781,30 +3766,32 @@ pub const Tensor = struct { pub fn any(self: Tensor, axis_: anytype) Tensor { const pred = self.cmp(.NE, Tensor.constant(self.dims(), self.dtype().zero())); return ops.reduce( + .{pred}, + .{Tensor.scalar(false, .bool)}, + &.{self.axis(axis_)}, struct { - pub fn acc(x: Tensor, res: Tensor) Tensor { - return res.logical(.OR, x); + pub fn acc(args: ops.ReduceArgs) Tensor { + return args.left.logical(.OR, args.right); } }.acc, - pred, - Tensor.scalar(false, .bool), - &.{self.axis(axis_)}, - ); + .{}, + )[0]; } /// Returns a Tensor containing boolean indicating if there is a non-zero value over the given axis. pub fn all(self: Tensor, axis_: anytype) Tensor { const pred = if (self.dtype() == .bool) self else self.cmp(.NE, Tensor.scalar(0, self.dtype())); return ops.reduce( + .{pred}, + .{Tensor.scalar(true, .bool)}, + &.{self.axis(axis_)}, struct { - pub fn acc(x: Tensor, res: Tensor) Tensor { - return res.logical(.AND, x); + pub fn acc(args: ops.ReduceArgs) Tensor { + return args.left.logical(.AND, args.right); } }.acc, - pred, - Tensor.scalar(true, .bool), - &.{self.axis(axis_)}, - ); + .{}, + )[0]; } /// Given a set of N vectors of lengths A, B, C, D, @@ -3827,7 +3814,7 @@ pub const Tensor = struct { fn _cartesianProduct(vectors: []const Tensor, out: []Tensor) void { stdx.debug.assert(vectors.len >= 1, "cartesianProduct expects at least one input.", .{}); - stdx.debug.assert(vectors.len < Tensor.MAX_RANK, "cartesianProduct expects at most {} input vectors, received {} !", .{ Tensor.MAX_RANK - 1, vectors.len }); + stdx.debug.assert(vectors.len < constants.MAX_RANK, "cartesianProduct expects at most {} input vectors, received {} !", .{ constants.MAX_RANK - 1, vectors.len }); for (vectors) |x| { stdx.debug.assert(x.rank() <= 1, "cartesianProduct expects 0 or 1 rank input vectors. Got: {any}", .{vectors}); stdx.debug.assert(vectors[0].dtype() == x.dtype(), "cartesianProduct expects input vectors to have all the same dtype. Got: {any}", .{vectors}); @@ -3849,10 +3836,10 @@ pub const Tensor = struct { test cartesianProduct { const zml = @import("zml.zig"); - const client = zml.testing.env(); + const platform = zml.testing.env(); - const x = try zml.Buffer.fromSlice(client, .{6}, &[_]i32{ 0, 1, 2, 3, 4, 5 }); - const y = try zml.Buffer.fromSlice(client, .{4}, &[_]i32{ 0, 1, 2, 3 }); + const x: Tensor = .init(.{6}, .i32); + const y: Tensor = .init(.{4}, .i32); const Local = struct { pub fn _cartesianProduct2(a: Tensor, b: Tensor) [2]Tensor { @@ -3860,33 +3847,42 @@ pub const Tensor = struct { } }; - { - const xs, const ys = try zml.testing.compileAndCall(client, Local._cartesianProduct2, .{ x, y }); - try std.testing.expectEqualSlices(i64, &.{ 6, 4 }, xs.shape().dims()); - try std.testing.expectEqualSlices(i64, &.{ 6, 4 }, ys.shape().dims()); - try std.testing.expectEqualDeep( - [6][4]i32{ - .{ 0, 0, 0, 0 }, - .{ 1, 1, 1, 1 }, - .{ 2, 2, 2, 2 }, - .{ 3, 3, 3, 3 }, - .{ 4, 4, 4, 4 }, - .{ 5, 5, 5, 5 }, - }, - try xs.getValue([6][4]i32), - ); - try std.testing.expectEqualDeep( - [6][4]i32{ - .{ 0, 1, 2, 3 }, - .{ 0, 1, 2, 3 }, - .{ 0, 1, 2, 3 }, - .{ 0, 1, 2, 3 }, - .{ 0, 1, 2, 3 }, - .{ 0, 1, 2, 3 }, - }, - try ys.getValue([6][4]i32), - ); - } + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cartesianProduct2, .{ x, y }, platform); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); + defer x_buffer.deinit(); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); + defer y_buffer.deinit(); + + var xs, var ys = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._cartesianProduct2, .{ x_buffer, y_buffer }); + defer xs.deinit(); + defer ys.deinit(); + + try std.testing.expectEqualSlices(i64, &.{ 6, 4 }, xs.shape().dims()); + try std.testing.expectEqualSlices(i64, &.{ 6, 4 }, ys.shape().dims()); + try std.testing.expectEqualDeep( + [6][4]i32{ + .{ 0, 0, 0, 0 }, + .{ 1, 1, 1, 1 }, + .{ 2, 2, 2, 2 }, + .{ 3, 3, 3, 3 }, + .{ 4, 4, 4, 4 }, + .{ 5, 5, 5, 5 }, + }, + try xs.getValue([6][4]i32, std.testing.io), + ); + try std.testing.expectEqualDeep( + [6][4]i32{ + .{ 0, 1, 2, 3 }, + .{ 0, 1, 2, 3 }, + .{ 0, 1, 2, 3 }, + .{ 0, 1, 2, 3 }, + .{ 0, 1, 2, 3 }, + .{ 0, 1, 2, 3 }, + }, + try ys.getValue([6][4]i32, std.testing.io), + ); } /// Given a set of N vectors of lengths A, B, C, D, @@ -3895,7 +3891,7 @@ pub const Tensor = struct { /// /// - res[a, b, c, d] == (A[a], B[b], C[c], D[d]) pub fn cartesianProductStacked(vectors: []const Tensor) Tensor { - var out = stdx.BoundedArray(Tensor, Tensor.MAX_RANK).init(vectors.len) catch unreachable; + var out = stdx.BoundedArray(Tensor, constants.MAX_RANK).init(vectors.len) catch unreachable; _cartesianProduct(vectors, out.slice()); return Tensor.stack(out.constSlice(), .last, .coord); @@ -3904,8 +3900,8 @@ pub const Tensor = struct { test cartesianProductStacked { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const x = try zml.Buffer.fromSlice(platform, .{6}, &[_]i32{ 0, 1, 2, 3, 4, 5 }); - const y = try zml.Buffer.fromSlice(platform, .{4}, &[_]i32{ 0, 1, 2, 3 }); + const x: Tensor = .init(.{6}, .i32); + const y: Tensor = .init(.{4}, .i32); const Local = struct { pub fn _fwd(a: Tensor, b: Tensor) Tensor { @@ -3913,7 +3909,17 @@ pub const Tensor = struct { } }; - const z = try zml.testing.compileAndCall(platform, Local._fwd, .{ x, y }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, y }, platform); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); + defer x_buffer.deinit(); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); + defer y_buffer.deinit(); + + var z = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._fwd, .{ x_buffer, y_buffer }); + defer z.deinit(); + try std.testing.expectEqualDeep( [6][4][2]i32{ .{ .{ 0, 0 }, .{ 0, 1 }, .{ 0, 2 }, .{ 0, 3 } }, @@ -3923,14 +3929,13 @@ pub const Tensor = struct { .{ .{ 4, 0 }, .{ 4, 1 }, .{ 4, 2 }, .{ 4, 3 } }, .{ .{ 5, 0 }, .{ 5, 1 }, .{ 5, 2 }, .{ 5, 3 } }, }, - try z.getValue([6][4][2]i32), + try z.getValue([6][4][2]i32, std.testing.io), ); } fn binaryOp( - src: std.builtin.SourceLocation, op_name: []const u8, - op_fn: fn (mlir.Context, mlir.Value, mlir.Value, mlir.Location) mlir.Operation, + op_fn: fn (*mlir.Context, *const mlir.Value, *const mlir.Value, *const mlir.Location) *mlir.Operation, ) fn (Tensor, Tensor) Tensor { return struct { pub fn binaryOpHelper(self: Tensor, other: Tensor) Tensor { @@ -3955,9 +3960,7 @@ pub const Tensor = struct { stdx.debug.assert(same_shape, "{s} expects tensor shapes to match, got {f} and {f}", .{ op_name, self._shape, other._shape }); - const ctx = self.getContext(); - const location = ctx.location(src, "{s}({f}, {f})", .{ op_name, self, other_ }); - const ret = @call(.auto, op_fn, .{ ctx.mlirCtx(), self.value(), other_.value(), location }); + const ret = @call(.auto, op_fn, .{ mlirCtx(), self.value(), other_.value(), mlir.Location.unknown(mlirCtx()) }).appendTo(currentBlock()); return _result(self._shape, ret.result(0)); } }.binaryOpHelper; @@ -3967,28 +3970,48 @@ pub const Tensor = struct { /// Only for debug purpose, it inserts device to host synchronization /// so it will slow down the program execution. pub fn print(input: Tensor) Tensor { - return callback.call(callback.Print, .{input}, &.{input.shape()})[0]; + // TODO(Corentin) + _ = input; + @panic("Unimplemented"); + } + + fn mlirCtx() *mlir.Context { + return CompilationContext.current().mlir_ctx; + } + + fn currentBlock() *mlir.Block { + return CompilationContext.current().currentScope().block; + } + + /// Returns the donation data of the tensor. + pub fn donation(self: Tensor) ?usize { + return CompilationContext.current().currentScope().id_to_donation.get(self.id); + } + + /// Returns the output memory kind of the tensor. + pub fn outputMemoryKind(self: Tensor) Buffer.Memory { + return CompilationContext.current().currentScope().id_to_output_memory_kind.get(self.id) orelse .device; } }; -fn initPoolArg(rank: usize, data: []const i64) [Tensor.MAX_RANK]i64 { +fn initPoolArg(rank: usize, data: []const i64) [constants.MAX_RANK]i64 { // TODO use shape - var result = [_]i64{1} ** Tensor.MAX_RANK; + var result = [_]i64{1} ** constants.MAX_RANK; const start = rank - data.len; @memcpy(result[start .. start + data.len], data); return result; } -fn getPoolResDims(dt: DataType, in_dims: []const i64, base_dilations: @Vector(Tensor.MAX_RANK, i64), padding: []const i64, window_dimensions: @Vector(Tensor.MAX_RANK, i64), window_dilations: @Vector(Tensor.MAX_RANK, i64), window_strides: @Vector(Tensor.MAX_RANK, i64)) Shape { +fn getPoolResDims(dt: DataType, in_dims: []const i64, base_dilations: @Vector(constants.MAX_RANK, i64), padding: []const i64, window_dimensions: @Vector(constants.MAX_RANK, i64), window_dilations: @Vector(constants.MAX_RANK, i64), window_strides: @Vector(constants.MAX_RANK, i64)) Shape { // TODO use shape - var input_dims = [_]i64{1} ** Tensor.MAX_RANK; + var input_dims = [_]i64{1} ** constants.MAX_RANK; @memcpy(input_dims[0..in_dims.len], in_dims); - const input_dims_: @Vector(Tensor.MAX_RANK, i64) = input_dims; - const splat_one: @Vector(Tensor.MAX_RANK, i64) = @splat(1); - const dilated_input_shape: @Vector(Tensor.MAX_RANK, i64) = (input_dims_ - splat_one) * base_dilations + splat_one; - var pad_slice0: @Vector(Tensor.MAX_RANK, i64) = @splat(padding[0]); - var pad_slice1: @Vector(Tensor.MAX_RANK, i64) = @splat(padding[0]); + const input_dims_: @Vector(constants.MAX_RANK, i64) = input_dims; + const splat_one: @Vector(constants.MAX_RANK, i64) = @splat(1); + const dilated_input_shape: @Vector(constants.MAX_RANK, i64) = (input_dims_ - splat_one) * base_dilations + splat_one; + var pad_slice0: @Vector(constants.MAX_RANK, i64) = @splat(padding[0]); + var pad_slice1: @Vector(constants.MAX_RANK, i64) = @splat(padding[0]); if (padding.len > 1) { var idx: usize = 0; while (idx < in_dims.len * 2) : (idx += 2) { @@ -3996,15 +4019,15 @@ fn getPoolResDims(dt: DataType, in_dims: []const i64, base_dilations: @Vector(Te pad_slice1[idx / 2] = padding[idx + 1]; } } - const padded_input_shape: @Vector(Tensor.MAX_RANK, i64) = pad_slice0 + dilated_input_shape + pad_slice1; + const padded_input_shape: @Vector(constants.MAX_RANK, i64) = pad_slice0 + dilated_input_shape + pad_slice1; const dilated_window_shape = (window_dimensions - splat_one) * window_dilations + splat_one; const dims = @divFloor(padded_input_shape - dilated_window_shape, window_strides) + splat_one; - const dims_arr: [Tensor.MAX_RANK]i64 = @bitCast(dims); + const dims_arr: [constants.MAX_RANK]i64 = @bitCast(dims); return Shape.init(dims_arr[0..in_dims.len], dt); } -fn getComparisonType(ctx: mlir.Context, dtype: DataType) dialect.stablehlo.CompareType { - return dialect.stablehlo.CompareType.init(ctx, switch (dtype.class()) { +fn getComparisonType(ctx: *mlir.Context, dtype: DataType) *const dialects.stablehlo.CompareType { + return dialects.stablehlo.CompareType.init(ctx, switch (dtype.class()) { .bool => .UNSIGNED, .integer => if (dtype.isSignedInt()) .SIGNED else .UNSIGNED, .float => .FLOAT, @@ -4028,8 +4051,18 @@ test "Tensor.maxPool1d" { var data: [20]f32 = undefined; for (&data, 0..) |*v, i| v.* = @floatFromInt(i); - const x = try zml.Buffer.fromSlice(platform, .{ 2, 2, 5 }, &data); - const result = try zml.testing.compileAndCall(platform, MaxPool._fwd, .{x}); + const x: Tensor = .init(.{ 2, 2, 5 }, .f32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&data)); + defer x_buffer.deinit(); + + const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, MaxPool._fwd, .{x_buffer}); + defer result.values.deinit(); + defer result.indices.deinit(); + try zml.testing.expectEqualShapes(.init(.{ 2, 2, 2 }, .f32), result.values.shape()); try zml.testing.expectEqualShapes(.init(.{ 2, 2, 2 }, .i32), result.indices.shape()); try std.testing.expectEqualDeep( @@ -4037,7 +4070,7 @@ test "Tensor.maxPool1d" { .{ .{ 2, 4 }, .{ 7, 9 } }, .{ .{ 12, 14 }, .{ 17, 19 } }, }, - result.values.getValue([2][2][2]f32), + result.values.getValue([2][2][2]f32, std.testing.io), ); } @@ -4056,13 +4089,23 @@ test "Tensor.maxPool2d" { var data: [100]f32 = undefined; for (&data, 0..) |*v, i| v.* = @floatFromInt(i); - const x = try zml.Buffer.fromSlice(platform, .{ 2, 2, 5, 5 }, &data); - const result = try zml.testing.compileAndCall(platform, MaxPool._fwd, .{x}); + const x: Tensor = .init(.{ 2, 2, 5, 5 }, .f32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&data)); + defer x_buffer.deinit(); + + const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, MaxPool._fwd, .{x_buffer}); + defer result.values.deinit(); + defer result.indices.deinit(); + try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2, 4 }, .f32), result.values.shape()); try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2, 4 }, .i32), result.indices.shape()); var buffer: [2][2][2][4]f32 = undefined; - _ = try result.values.toHost(std.mem.asBytes(&buffer)); + _ = try result.values.toSlice(std.testing.io, .init(result.values.shape(), std.mem.asBytes(&buffer))); try std.testing.expectEqualDeep( [2][2][2][4]f32{ .{ @@ -4078,147 +4121,26 @@ test "Tensor.maxPool2d" { ); } -/// Return a clone of a type with Tensors replaced by Buffer. -/// Non-Tensor metadata is stripped out of the resulting struct. -/// Recursively descends into the type. -pub fn Bufferized(comptime T: type) type { - return meta.MapRestrict(Tensor, Buffer).map(T); -} - -/// Return a clone of a type with Tensors replaced by Shapes. -/// Recursively descends into the type. -/// See also: shapesOf() and its tests, and meta.MapType(). -pub fn ShapeOf(comptime T: type) type { - const M = meta.MapType(Tensor, Shape); - return M.map(T); -} - -/// Return a clone of the argument where each instance of a Tensor is replaced -/// by its Shape. This is similar to ShapeOf(), but with runtime values. -/// See also: meta.mapAlloc(). -pub fn shapesOf(model: anytype, allocator: std.mem.Allocator) !ShapeOf(@TypeOf(model)) { - var shapes: ShapeOf(@TypeOf(model)) = undefined; - try meta.mapAlloc(struct { - fn shapeFromTensorCallback(_: void, tensor: Tensor) Shape { - return tensor.shape(); - } - }.shapeFromTensorCallback, allocator, {}, model, &shapes); - return shapes; -} - -test shapesOf { - const alloc = std.testing.allocator; - - // Tensor in struct - { - const S = struct { - a: Tensor, - }; - const shape = Shape.init(.{ 28, 28 }, .f32); - const s: S = .{ - .a = Tensor{ ._shape = shape, ._id = undefined }, - }; - const shapes = try shapesOf(s, alloc); - try std.testing.expectEqual(shape, shapes.a); - } - - // single Tensor - { - const shape = Shape.init(.{ 28, 28 }, .f32); - const tensor = Tensor{ ._shape = shape, ._id = undefined }; - const shapes = try shapesOf(tensor, alloc); - try std.testing.expectEqual(shape, shapes); - } - - // nn linear layer, no bias - { - const nn = @import("nn.zig"); - const shape = Shape.init(.{ 28, 28 }, .f32); - const layer: nn.Linear = .{ - .weight = Tensor{ ._shape = shape, ._id = undefined }, - .bias = null, - }; - - const shapes = try shapesOf(layer, alloc); - try std.testing.expectEqual(shape, shapes.weight); - try std.testing.expectEqual(null, shapes.bias); - } - - // model - { - const Mnist = struct { - fc1: Layer, - fc2: Layer, - - const Layer = struct { - weight: Tensor, - bias: Tensor, - }; - }; - - const fc1_weight_shape = Shape.init(.{ 500, 784 }, .f32); - const fc1_bias_shape = Shape.init(.{500}, .f32); - const fc2_weight_shape = Shape.init(.{ 10, 500 }, .f32); - const fc2_bias_shape = Shape.init(.{10}, .f32); - const mnist: Mnist = .{ - .fc1 = .{ - .weight = Tensor{ ._shape = fc1_weight_shape, ._id = undefined }, - .bias = Tensor{ ._shape = fc1_bias_shape, ._id = undefined }, - }, - .fc2 = .{ - .weight = Tensor{ ._shape = fc2_weight_shape, ._id = undefined }, - .bias = Tensor{ ._shape = fc2_bias_shape, ._id = undefined }, - }, - }; - - const shapes = try shapesOf(mnist, alloc); - try std.testing.expectEqual(fc1_weight_shape, shapes.fc1.weight); - try std.testing.expectEqual(fc1_bias_shape, shapes.fc1.bias); - try std.testing.expectEqual(fc2_weight_shape, shapes.fc2.weight); - try std.testing.expectEqual(fc2_bias_shape, shapes.fc2.bias); - } -} - -pub fn _collectAxes(T: type, bounded_array: stdx.BoundedArray(T, Tensor.MAX_RANK), value: T) stdx.BoundedArray(i64, Tensor.MAX_RANK) { - var res: stdx.BoundedArray(i64, Tensor.MAX_RANK) = .{}; - for (bounded_array.constSlice(), 0..) |v, ax| { - if (v == value) { - res.appendAssumeCapacity(@intCast(ax)); - } - } - return res; -} - -fn _parseGatherCoord(self: Tensor, axes_: anytype) struct { bool, stdx.BoundedArray(u3, Tensor.MAX_RANK) } { +fn _parseGatherCoord(self: Tensor, axes_: anytype) struct { bool, stdx.BoundedArray(u3, constants.MAX_RANK) } { const AxesT = @TypeOf(axes_); - const axes_is_scalar = AxesT == EnumLiteral or AxesT == comptime_int or @typeInfo(AxesT) == .int; + const axes_is_scalar = AxesT == @EnumLiteral() or AxesT == comptime_int or @typeInfo(AxesT) == .int; const coord_axes = if (axes_is_scalar) - stdx.BoundedArray(u3, Tensor.MAX_RANK).fromSlice(&.{self.axis(axes_)}) catch unreachable + stdx.BoundedArray(u3, constants.MAX_RANK).fromSlice(&.{self.axis(axes_)}) catch unreachable else self.axes(axes_); return .{ axes_is_scalar, coord_axes }; } -fn parseArrayInfo(T: type) Shape { - return switch (@typeInfo(T)) { - .Array => |arr| { - const s = parseArrayInfo(arr.child); - return s.insert(0, .{arr.len}); - }, - else => .{ ._dtype = DataType.fromZigType(T) }, - }; -} - -inline fn toI64(values: anytype) []i64 { - var res: [Tensor.MAX_RANK]i64 = undefined; - for (values, 0..) |val, i| res[i] = @intCast(val); - return res[0..values.len]; +fn toI64(values: anytype) stdx.BoundedArray(i64, constants.MAX_RANK) { + var res: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; + for (values) |val| res.appendAssumeCapacity(@intCast(val)); + return res; } fn transposeIsJustAReshape(x: Shape, permutation: []const i64) bool { - var perm: stdx.BoundedArray(struct { u8, bool }, Tensor.MAX_RANK) = .{}; + var perm: stdx.BoundedArray(struct { u8, bool }, constants.MAX_RANK) = .{}; // Don't rewrite on invalid inputs. if (permutation.len > x.rank()) return false; for (permutation) |ax| { @@ -4270,6 +4192,6 @@ test "unused tensor" { } }; - const mod = try zml.compileFn(std.testing.allocator, Local._fwd, .{Shape.init(.{10}, .f32)}, platform); - defer mod.deinit(); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{Tensor.init(.{10}, .f32)}, platform); + defer exe.deinit(); } diff --git a/zml/test_runner.zig b/zml/test_runner.zig index 85acadef..e3c801f3 100644 --- a/zml/test_runner.zig +++ b/zml/test_runner.zig @@ -3,8 +3,6 @@ const std = @import("std"); const testing = std.testing; const builtin = @import("builtin"); -const async = @import("async"); - // note: std_options.log_level does not respect testing.log_level // ref: https://github.com/ziglang/zig/issues/5738 const log_level: std.log.Level = .warn; @@ -17,12 +15,9 @@ var log_err_count: usize = 0; var fba_buffer: [8192]u8 = undefined; var fba = std.heap.FixedBufferAllocator.init(&fba_buffer); -pub fn main() anyerror!void { +pub fn main() !void { testing.log_level = log_level; - try async.AsyncThread.main(testing.allocator, asyncMain); -} -pub fn asyncMain() !void { const test_fn_list: []const std.builtin.TestFn = builtin.test_functions; var ok_count: usize = 0; var skip_count: usize = 0; @@ -58,6 +53,9 @@ pub fn asyncMain() !void { } } + testing.io_instance = .init(testing.allocator_instance.allocator()); + defer testing.io_instance.deinit(); + const test_node = root_node.start(test_fn.name, 0); if (!have_tty) { std.debug.print("{d}/{d} {s}...", .{ i + 1, test_fn_list.len, test_fn.name }); @@ -86,7 +84,7 @@ pub fn asyncMain() !void { std.debug.print("FAIL ({s})\n", .{@errorName(err)}); } if (@errorReturnTrace()) |trace| { - std.debug.dumpStackTrace(trace.*); + std.debug.dumpStackTrace(trace); } test_node.end(); }, diff --git a/zml/testing.zig b/zml/testing.zig index 38952dd6..bf968551 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -3,22 +3,20 @@ const builtin = @import("builtin"); const stdx = @import("stdx"); -const meta = @import("meta.zig"); -const shapesOf = @import("tensor.zig").shapesOf; +const ConstSlice = @import("slice.zig").ConstSlice; +const Platform = @import("platform.zig").Platform; const zml = @import("zml.zig"); const log = std.log.scoped(.@"zml/testing"); -var _platform: ?zml.Platform = null; +var _platform: ?Platform = null; -pub fn env() zml.Platform { +pub fn env() Platform { if (!builtin.is_test) @compileError("Cannot use zml.testing.env outside of a test block"); if (_platform == null) { - var ctx = zml.Context.init() catch unreachable; - _platform = ctx.autoPlatform(.{}).withCompilationOptions(.{ - .xla_dump_to = "/tmp/zml/tests/", - .sharding_enabled = true, - }); + zml.init(); + + _platform = Platform.auto(std.testing.io, .{}) catch unreachable; } return _platform.?; @@ -35,31 +33,32 @@ pub fn approxEq(comptime Float: type, l: Float, r: Float, tolerance: Float) bool /// Testing utility. Accepts both zml.Buffer and zml.HostBuffer but zml.Buffer will be copied to the /// host for comparison ! -pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { +pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, tolerance: f32) !void { const allocator = if (builtin.is_test) std.testing.allocator else std.heap.smp_allocator; - var left: zml.HostBuffer, const should_free_left = if (@TypeOf(left_) == zml.Buffer) - .{ try left_.toHostAlloc(allocator), true } - else - .{ left_, false }; + var left: ConstSlice, const should_free_left = if (@TypeOf(left_) == zml.Buffer) b: { + const slice = try left_.toSliceAlloc(allocator, io); + break :b .{ slice.constSlice(), true }; + } else .{ left_, false }; - var right: zml.HostBuffer, const should_free_right = if (@TypeOf(right_) == zml.Buffer) - .{ try right_.toHostAlloc(allocator), true } - else - .{ right_, false }; + var right: ConstSlice, const should_free_right = if (@TypeOf(right_) == zml.Buffer) b: { + const slice = try right_.toSliceAlloc(allocator, io); + break :b .{ slice.constSlice(), true }; + } else .{ right_, false }; defer { - if (should_free_left) left.deinit(allocator); - if (should_free_right) right.deinit(allocator); + if (should_free_left) left.free(allocator); + if (should_free_right) right.free(allocator); } errdefer log.err("\n--> Left: {0f}{0d:24.3}\n--> Right: {1f}{1d:24.3}", .{ left, right }); - if (!std.mem.eql(i64, left.shape().dims(), right.shape().dims())) { - log.err("left.shape() {f} != right.shape() {f}", .{ left.shape(), right.shape() }); + if (!std.mem.eql(i64, left.shape.dims(), right.shape.dims())) { + log.err("left.shape() {f} != right.shape() {f}", .{ left.shape, right.shape }); return error.TestUnexpectedResult; } if (left.dtype() != right.dtype() and !(left.dtype() == .f16 and right.dtype() == .bf16)) { - log.err("left.dtype ({f}) != right.dtype ({f})", .{ left.shape(), right.shape() }); + log.err("left.dtype ({}) != right.dtype ({})", .{ left.dtype(), right.dtype() }); return error.TestUnexpectedResult; } + switch (left.dtype()) { inline .bf16, .f16, @@ -115,208 +114,27 @@ pub fn expectEqualShapes(expected: zml.Shape, actual: zml.Shape) error{TestExpec return error.TestExpectedEqual; } -/// Returns a mirrored version of T where each Tensor has been replaced by a Buffer. -/// This is similar to zml.Bufferized, -/// but also keep every other fields that could be used during compilation. -/// -/// see `compileAndCall`. -pub fn BufferizedWithArgs(comptime T: type) type { - // TODO: we should strip out the non-buffer fields. - // Currently it's confusing cause the Bufferized struct contains field that are never read. - // Also it will simplify the layout of the Bufferized struct. - // accelerating the calls to execute. - return meta.MapType(zml.Tensor, zml.Buffer).map(T); +fn BufferizedWithArgs(comptime T: type) type { + return zml.meta.MapType(zml.Tensor, zml.Buffer).map(T); } -/// Compile a function and immediatly call it with the given buffers. -/// The compiled module is discarded after the call. -/// Useful during testing when a module is typically called only once. -/// -/// Note: `func` needs explicit types on all parameters. -/// To test a function with `anytype` (typically for tagged API), you need to create a specialized version of it with specific types. -pub fn compileAndCall( - platform: zml.Platform, - func: anytype, - buffer_and_args: BufferizedWithArgs(stdx.meta.FnArgs(func)), -) !zml.Bufferized(stdx.meta.FnResult(func)) { - // This simplify test API and also ensure this fn isn't used outside of tests. - const allocator = std.testing.allocator; - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); +/// Automatically calls the executable with the given arguments, taking care of arguments and results allocation. +/// This helper can be used in tests to make the code a bit less verbose. +/// It doesn't handle pointers in inputs/outputs structs. +pub fn autoCall(allocator: std.mem.Allocator, io: std.Io, exe: *const zml.exe.Exe, func: anytype, inputs: zml.Bufferized(stdx.meta.FnArgs(func))) !zml.Bufferized(stdx.meta.FnResult(func)) { + var args = try exe.args(allocator); + defer args.deinit(allocator); - const Local = struct { - pub fn bufferToShape(_: void, x: zml.Buffer) zml.Shape { - return x.shape(); - } - }; - var shape_args: zml.ShapeOf(stdx.meta.FnArgs(func)) = undefined; - try meta.mapAlloc(Local.bufferToShape, arena.allocator(), {}, buffer_and_args, &shape_args); + var results = try exe.results(allocator); + defer results.deinit(allocator); - var mod = try zml.compileFn(allocator, func, shape_args, platform); - defer mod.deinit(); + args.set(inputs); + exe.callOpts(io, args, &results, .{ .wait = true }); - // Note: we don't use the type safe API of mod, - // cause mod.call expects a `zml.Bufferized` while we have `BufferizedWithArgs`. - mod.inner.prepare(buffer_and_args); - mod.inner._unsafeCall(); + var output: zml.Bufferized(stdx.meta.FnResult(func)) = undefined; + results.fill(.{&output}); - var result: zml.Bufferized(stdx.meta.FnResult(func)) = undefined; - mod.inner._unsafeAssignResults(@TypeOf(result), &result); - return result; -} - -/// Compile a function and immediatly call it with the given buffers. -/// The compiled module is discarded after the call. -/// Useful during testing when a module is typically called only once. -pub fn compileAndCallWithTensors( - platform: zml.Platform, - func: anytype, - shape_args: zml.ShapeOf(stdx.meta.FnArgs(func)), - buffer_args: zml.Bufferized(stdx.meta.FnArgs(func)), -) !zml.Bufferized(stdx.meta.FnResult(func)) { - // This simplify test API and also ensure this fn isn't used outside of tests. - const allocator = std.testing.allocator; - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - - const mod = try zml.compileFn(allocator, func, shape_args, platform); - defer mod.deinit(); - - return mod.call(buffer_args); -} - -pub fn testLayer( - platform: zml.Platform, - activations: zml.aio.BufferStore, - comptime name: []const u8, - layer: anytype, - layer_weights: zml.Bufferized(@TypeOf(layer)), - tolerance: f32, -) !void { - try testLayerOut(platform, activations, name, name ++ ".out", layer, layer_weights, tolerance); -} - -pub fn testLayerOut( - platform: zml.Platform, - activations: zml.aio.BufferStore, - comptime name: []const u8, - comptime out_name: []const u8, - layer: anytype, - layer_weights: zml.Bufferized(@TypeOf(layer)), - tolerance: f32, -) !void { - var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); - defer arena.deinit(); - const alloc = arena.allocator(); - log.info("Testing {s}", .{name}); - - const fwd = @TypeOf(layer).forward; - const FwdSign = zml.ModuleSignature(fwd); - - const input_tensors = try zml.aio.populateModelWithPrefix(FwdSign.ArgsT, alloc, activations, name ++ ".in"); - const input_shapes = try shapesOf(input_tensors, alloc); - - const n_in = zml.module.countTensors(&input_tensors); - const n_in_exp = activations.countLayers(name ++ ".in"); - if (n_in != n_in_exp) { - log.warn("Reference models uses {d} inputs, but implementation uses {d}", .{ n_in_exp, n_in }); - } - - const exe = try zml.compileModel(alloc, fwd, layer, input_shapes, platform); - - const n_out_exp = activations.countLayers(out_name); - if (exe.inner.result_shapes.len != n_out_exp) { - log.warn("Reference models produces {d} outputs, but implementation produces {d}", .{ n_out_exp, exe.inner.result_shapes.len }); - } - const mod = exe.prepare(layer_weights); - - const FetchCtx = struct { - store: zml.aio.BufferStore, - index: u32, - prefix: std.ArrayList(u8), - platform: zml.Platform, - - fn fetch(ctx: *@This(), x: zml.Tensor) zml.Buffer { - _ = x; - defer ctx.index += 1; - var full_prefix = ctx.*.prefix; - _ = full_prefix.writer(undefined).print("{d}", .{ctx.index}) catch unreachable; - log.info("prefix: {s}", .{full_prefix.items}); - const host = ctx.store.get(full_prefix.items) orelse { - log.err("Didn't find test input: {s}", .{full_prefix.items}); - @panic("Missing test input"); - }; - return host.toDevice(ctx.platform) catch unreachable; - } - }; - - // Note: zml.populateModelWithPrefix isn't enough, - // because it assumes we have the same structure in the activation file - // than in the function signature. - // But for sake of decoupling the reference implementation - // and ZML code that's not always the case. - { - var input_buffers: zml.Bufferized(FwdSign.ArgsT) = undefined; - var fetch_ctx: FetchCtx = .{ .store = activations, .index = 0, .prefix = .{}, .platform = platform }; - try fetch_ctx.prefix.ensureTotalCapacity(alloc, name.len + 32); - fetch_ctx.prefix.appendSliceAssumeCapacity(name ++ ".in."); - try zml.meta.mapAlloc(FetchCtx.fetch, alloc, &fetch_ctx, input_tensors, &input_buffers); - defer zml.aio.unloadBuffers(&input_buffers); - _ = mod.call(input_buffers); - } - - var buf: [1024]u8 = undefined; - var failed: bool = false; - for (0..mod.inner.result_shapes.len) |i| { - const full_name = std.fmt.bufPrint(&buf, "{s}.{d}", .{ out_name, i }) catch unreachable; - const expected_out = activations.get(full_name) orelse { - log.warn("Output buffer not found: {s}", .{full_name}); - continue; - }; - zml.testing.expectClose(expected_out, mod.inner.getOutputBuffer(i), tolerance) catch |err| switch (err) { - error.TestUnexpectedResult => { - log.err("{s}.{d} doesn't match !", .{ out_name, i }); - failed = true; - continue; - }, - else => return err, - }; - } - - if (failed) return error.TestUnexpectedResult; - log.info("all good for {s} !", .{name}); -} - -test testLayer { - const platform = env(); - - // create a model - const layer: zml.nn.Linear = .{ - .weight = zml.Tensor{ ._shape = zml.Shape.init(.{ 5, 2 }, .f32), ._id = .{ .buffer_id = 42 } }, - }; - const layer_weights: zml.Bufferized(zml.nn.Linear) = .{ - .weight = try zml.Buffer.fromArray( - platform, - [5][2]f32{ .{ 0, 0 }, .{ 0, 1 }, .{ 1, 2 }, .{ -1, -1 }, .{ -1, 0 } }, - ), - }; - - // create a buffer store containing the activations: - var activations = zml.aio.BufferStore.init(std.testing.allocator); - defer activations.deinit(); - { - const input = zml.HostBuffer.fromArray(&[2]f32{ 1, -1 }); - try activations.buffers.put(activations.arena.allocator(), "model.layer.in.0", input); - const output = zml.HostBuffer.fromArray(&[5]f32{ 0, -1, -1, 0, -1 }); - try activations.buffers.put(activations.arena.allocator(), "model.layer.out.0", output); - } - - // test the ZML layer reproduces the "captured" activations: - try zml.testing.testLayer(platform, activations, "model.layer", layer, layer_weights, 1e-5); -} - -pub inline fn expectEqual(expected: anytype, actual: @TypeOf(expected)) !void { - return std.testing.expectEqual(expected, actual); + return output; } fn center(slice: anytype, i: usize) @TypeOf(slice) { @@ -325,3 +143,7 @@ fn center(slice: anytype, i: usize) @TypeOf(slice) { const end = @min(start + 2 * c, slice.len); return slice[start..end]; } + +pub inline fn expectEqual(expected: anytype, actual: @TypeOf(expected)) !void { + return std.testing.expectEqual(expected, actual); +} diff --git a/zml/tokenizer/BUILD.bazel b/zml/tokenizer/BUILD.bazel index 012e218c..691bd0ee 100644 --- a/zml/tokenizer/BUILD.bazel +++ b/zml/tokenizer/BUILD.bazel @@ -21,7 +21,6 @@ zig_binary( visibility = ["//visibility:public"], deps = [ ":tokenizer", - "//async", "//stdx", ], ) @@ -31,7 +30,6 @@ zig_test( main = "homemade.zig", visibility = ["//visibility:public"], deps = [ - "//async", "//ffi:zig", "//stdx", ], diff --git a/zml/tokenizer/hftokenizers/hftokenizers.h b/zml/tokenizer/hftokenizers/hftokenizers.h index 7bc8790c..f779b575 100644 --- a/zml/tokenizer/hftokenizers/hftokenizers.h +++ b/zml/tokenizer/hftokenizers/hftokenizers.h @@ -8,6 +8,7 @@ typedef struct hftokenizers hftokenizers; hftokenizers *hftokenizers_new(zig_slice); +hftokenizers *hftokenizers_new_from_bytes(zig_slice); void hftokenizers_drop(hftokenizers *tokenizer); zig_slice hftokenizers_encode(hftokenizers *tokenizer, zig_slice text); void hftokenizers_tokens_drop(zig_slice tokens); diff --git a/zml/tokenizer/hftokenizers/hftokenizers.rs b/zml/tokenizer/hftokenizers/hftokenizers.rs index 4035afe4..048283cc 100644 --- a/zml/tokenizer/hftokenizers/hftokenizers.rs +++ b/zml/tokenizer/hftokenizers/hftokenizers.rs @@ -24,6 +24,14 @@ extern "C" fn hftokenizers_new(path: ZigSlice) -> *mut tokenizers::Tokenizer )); } +#[no_mangle] +extern "C" fn hftokenizers_new_from_bytes(bytes: ZigSlice) -> *mut tokenizers::Tokenizer { + return Box::into_raw(Box::new( + tokenizers::Tokenizer::from_bytes(bytes.as_slice()) + .unwrap(), + )); +} + #[no_mangle] extern "C" fn hftokenizers_drop(t: *mut tokenizers::Tokenizer) { drop(unsafe { Box::from_raw(t) }); diff --git a/zml/tokenizer/hftokenizers/hftokenizers.zig b/zml/tokenizer/hftokenizers/hftokenizers.zig index 01b9c997..0871d6f9 100644 --- a/zml/tokenizer/hftokenizers/hftokenizers.zig +++ b/zml/tokenizer/hftokenizers/hftokenizers.zig @@ -97,6 +97,10 @@ pub const HFTokenizer = opaque { return @ptrCast(c.hftokenizers_new(ffi.ZigSlice.from(model))); } + pub fn fromBytes(bytes: []const u8) !*HFTokenizer { + return @ptrCast(c.hftokenizers_new_from_bytes(ffi.ZigSlice.from(bytes))); + } + pub fn deinit(self: *HFTokenizer) void { return c.hftokenizers_drop(@ptrCast(self)); } diff --git a/zml/tokenizer/homemade.zig b/zml/tokenizer/homemade.zig index 93383b0f..bd16d183 100644 --- a/zml/tokenizer/homemade.zig +++ b/zml/tokenizer/homemade.zig @@ -10,12 +10,6 @@ const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/tokenizer"); -test { - std.testing.refAllDecls(@This()); - std.testing.refAllDecls(Normalizer); - std.testing.refAllDecls(Tokenizer); -} - /// Byte Pair Encoding tokenizer generally used for LLM. pub const Tokenizer = struct { tokens: [][]const u8, @@ -479,7 +473,7 @@ pub const Decoder = struct { } pub fn string(self: *const Decoder) []const u8 { - return self.current_string; + return self.current_string orelse ""; } pub fn next(self: *Decoder, token_id: u32) !?[]const u8 { @@ -1058,14 +1052,21 @@ test Gpt2TextDecoder { } /// Open a json file in HF format and load the vocab from it. -pub fn fromHfJson(allocator: std.mem.Allocator, tokenizer_path: []const u8) !Tokenizer { +pub fn fromHfJson(allocator: std.mem.Allocator, io: std.Io, tokenizer_path: []const u8) !Tokenizer { const file = try std.fs.cwd().openFile(tokenizer_path, .{}); defer file.close(); var arena_state = std.heap.ArenaAllocator.init(allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); - const file_content = try file.readToEndAlloc(arena, 32 * 1024 * 1024); + const file_content = b: { + var reader = file.reader(io, &.{}); + var writer: std.Io.Writer.Allocating = .init(allocator); + defer writer.deinit(); + _ = try reader.interface.streamRemaining(&writer.writer); + break :b try writer.toOwnedSlice(); + }; + defer allocator.free(file_content); const info = try std.json.parseFromSliceLeaky(std.json.Value, arena, file_content, .{ .duplicate_field_behavior = .use_last, @@ -1201,11 +1202,11 @@ fn objectGet( return @field(val, @tagName(kind)); } -pub fn fromTinyLlamaFile(allocator: std.mem.Allocator, tokenizer_path: []const u8, vocab_size: u32) !Tokenizer { +pub fn fromTinyLlamaFile(allocator: std.mem.Allocator, io: std.Io, tokenizer_path: []const u8, vocab_size: u32) !Tokenizer { const tokenizer_file = try std.fs.cwd().openFile(tokenizer_path, .{}); defer tokenizer_file.close(); var read_buff: [4096]u8 = undefined; - var tok_reader = tokenizer_file.reader(&read_buff); + var tok_reader = tokenizer_file.reader(io, &read_buff); const r: *std.Io.Reader = &tok_reader.interface; const max_token_len = try readValueLE(u32, r); diff --git a/zml/tokenizer/main.zig b/zml/tokenizer/main.zig index 5d78cb80..a35de7d5 100644 --- a/zml/tokenizer/main.zig +++ b/zml/tokenizer/main.zig @@ -1,6 +1,5 @@ const std = @import("std"); -const async = @import("async"); const stdx = @import("stdx"); const zml_tokenizer = @import("zml/tokenizer"); @@ -14,17 +13,18 @@ const Flags = struct { }; pub fn main() !void { - try async.AsyncThread.main(std.heap.c_allocator, asyncMain); -} - -pub fn asyncMain() !void { var gpa: std.heap.GeneralPurposeAllocator(.{}) = .{}; const allocator = gpa.allocator(); + var threaded: std.Io.Threaded = .init(allocator); + defer threaded.deinit(); + + const io = threaded.io(); + const args = stdx.flags.parseProcessArgs(Flags); log.info("\tLoading tokenizer from {s}", .{args.tokenizer}); - var tokenizer = try zml_tokenizer.Tokenizer.fromFile(allocator, args.tokenizer); + var tokenizer = try zml_tokenizer.Tokenizer.fromFile(allocator, io, args.tokenizer); log.info("✅\tLoaded tokenizer from {s}", .{args.tokenizer}); defer tokenizer.deinit(); diff --git a/zml/tokenizer/sentencepiece/sentencepiece.i b/zml/tokenizer/sentencepiece/sentencepiece.i index f2413dc3..35ccc057 100644 --- a/zml/tokenizer/sentencepiece/sentencepiece.i +++ b/zml/tokenizer/sentencepiece/sentencepiece.i @@ -81,6 +81,7 @@ namespace sentencepiece { class SentencePieceProcessor { public: virtual sentencepiece::util::Status Load(absl::string_view filename); + virtual sentencepiece::util::Status LoadFromSerializedProto(absl::string_view serialized); virtual sentencepiece::util::Status Encode(absl::string_view input, std::vector *ids) const; virtual sentencepiece::util::Status Decode(const std::vector &ids, std::string *detokenized) const; virtual int PieceToId(absl::string_view piece) const; diff --git a/zml/tokenizer/sentencepiece/sentencepiece.zig b/zml/tokenizer/sentencepiece/sentencepiece.zig index c3623cb5..c98b0740 100644 --- a/zml/tokenizer/sentencepiece/sentencepiece.zig +++ b/zml/tokenizer/sentencepiece/sentencepiece.zig @@ -173,6 +173,13 @@ pub const SentencePieceProcessor = opaque { return sp; } + pub fn fromBytes(bytes: []const u8) !*SentencePieceProcessor { + const sp: *SentencePieceProcessor = @ptrCast(c.SentencePieceProcessor_new()); + errdefer sp.deinit(); + try assertOk(c.SentencePieceProcessor_LoadFromSerializedProto(@ptrCast(sp), ffi.ZigSlice.from(bytes))); + return sp; + } + pub fn deinit(self: *SentencePieceProcessor) void { c.SentencePieceProcessor_delete(@ptrCast(self)); } diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig index 5109ce31..c72ec803 100644 --- a/zml/tokenizer/tokenizer.zig +++ b/zml/tokenizer/tokenizer.zig @@ -1,15 +1,12 @@ const std = @import("std"); -const hftokenizers = @import("hftokenizers"); -const sentencepiece = @import("sentencepiece"); +pub const hftokenizers = @import("hftokenizers"); +pub const sentencepiece = @import("sentencepiece"); -const homemade = @import("homemade.zig"); +pub const homemade = @import("homemade.zig"); test { std.testing.refAllDecls(@This()); - std.testing.refAllDecls(hftokenizers); - std.testing.refAllDecls(sentencepiece); - std.testing.refAllDecls(homemade); } const Tokenizers = enum { @@ -72,17 +69,17 @@ pub const Tokenizer = union(Tokenizers) { }; } - pub fn string(self: Decoder) []const u8 { + pub fn string(self: *Decoder) []const u8 { return switch (self.*) { inline else => |v| v.string(), }; } - pub fn ids(self: Decoder) []u32 { - return switch (self.*) { - inline else => |v| v.ids(), - }; - } + //pub fn ids(self: *Decoder) []u32 { + // return switch (self.*) { + // inline else => |v| v.ids(), + // }; + //} pub fn next(self: *Decoder, token_id: u32) !?[]const u8 { return switch (self.*) { @@ -95,23 +92,28 @@ pub const Tokenizer = union(Tokenizers) { sentencepiece: *sentencepiece.SentencePieceProcessor, homemade: *homemade.Tokenizer, - pub fn fromFile(allocator: std.mem.Allocator, model: []const u8) !Tokenizer { + pub fn fromFile(allocator: std.mem.Allocator, io: std.Io, model: []const u8) !Tokenizer { + _ = allocator; // autofix + _ = io; // autofix if (std.mem.endsWith(u8, model, ".pb")) { - return .{ .sentencepiece = try sentencepiece.SentencePieceProcessor.fromFile(model) }; + return .{ .sentencepiece = try .fromFile(model) }; } if (std.mem.endsWith(u8, model, ".json")) { - return .{ .hftokenizers = try hftokenizers.HFTokenizer.fromFile(model) }; - } - - if (std.mem.endsWith(u8, model, ".tinyllama")) { - const tokenizer = try allocator.create(homemade.Tokenizer); - tokenizer.* = try homemade.fromTinyLlamaFile(allocator, model, 32000); - return .{ .homemade = tokenizer }; + return .{ .hftokenizers = try .fromFile(model) }; } return error.InvalidArgument; } + pub fn fromBytes(allocator: std.mem.Allocator, io: std.Io, bytes: []const u8) !Tokenizer { + _ = allocator; // autofix + _ = io; // autofix + if (bytes[0] == '{') { + return .{ .hftokenizers = try .fromBytes(bytes) }; + } + return .{ .sentencepiece = try .fromBytes(bytes) }; + } + pub fn deinit(self: *Tokenizer) void { switch (self.*) { inline else => |t| t.deinit(), @@ -136,3 +138,6 @@ pub const Tokenizer = union(Tokenizers) { }; } }; + +test {} +test {} diff --git a/zml/tools/BUILD.bazel b/zml/tools/BUILD.bazel deleted file mode 100644 index 21ad5654..00000000 --- a/zml/tools/BUILD.bazel +++ /dev/null @@ -1,29 +0,0 @@ -load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library") - -cc_library( - name = "macos_c", - srcs = ["macos.c"], - hdrs = ["macos.h"], - target_compatible_with = [ - "@platforms//os:macos", - ], - visibility = ["//zml/tools:__subpackages__"], -) - -zig_library( - name = "tools", - srcs = ["tracer.zig"], - main = "tools.zig", - visibility = ["//visibility:public"], - deps = select({ - # TODO(cerisier): fix MacOsTracer - # "@platforms//os:macos": [ ":macos_c" ], - "//conditions:default": [], - }), -) - -cc_static_library( - name = "macos_static_tools", - deps = ["macos_c"], -) diff --git a/zml/tools/macos.c b/zml/tools/macos.c deleted file mode 100644 index e59dc638..00000000 --- a/zml/tools/macos.c +++ /dev/null @@ -1,26 +0,0 @@ -#include -#include - -void zml_os_signpost_event( - os_log_t log, - os_signpost_id_t signpost_id, - const char *message) -{ - os_signpost_event_emit(log, signpost_id, "zml", "%s", message); -} - -void zml_os_signpost_interval_begin( - os_log_t log, - os_signpost_id_t signpost_id, - const char *message) -{ - os_signpost_interval_begin(log, signpost_id, "zml", "%s", message); -} - -void zml_os_signpost_interval_end( - os_log_t log, - os_signpost_id_t signpost_id, - const char *message) -{ - os_signpost_interval_end(log, signpost_id, "zml", "%s", message); -} diff --git a/zml/tools/macos.h b/zml/tools/macos.h deleted file mode 100644 index 0d2f9b4f..00000000 --- a/zml/tools/macos.h +++ /dev/null @@ -1,17 +0,0 @@ -#include -#include - -void zml_os_signpost_event( - os_log_t log, - os_signpost_id_t signpost_id, - const char *message); - -void zml_os_signpost_interval_begin( - os_log_t log, - os_signpost_id_t signpost_id, - const char *message); - -void zml_os_signpost_interval_end( - os_log_t log, - os_signpost_id_t signpost_id, - const char *message); diff --git a/zml/tools/tools.zig b/zml/tools/tools.zig deleted file mode 100644 index 2bc03172..00000000 --- a/zml/tools/tools.zig +++ /dev/null @@ -1 +0,0 @@ -pub const Tracer = @import("tracer.zig").Tracer; diff --git a/zml/tools/tracer.zig b/zml/tools/tracer.zig deleted file mode 100644 index 3946cb6c..00000000 --- a/zml/tools/tracer.zig +++ /dev/null @@ -1,106 +0,0 @@ -const builtin = @import("builtin"); - -const c = @import("c"); - -pub const Tracer = switch (builtin.os.tag) { - // TODO(cerisier): fix MacOsTracer - // .macos => MacOsTracer, - .linux => if (@hasDecl(c, "ZML_RUNTIME_CUDA")) CudaTracer else FakeTracer, - else => FakeTracer, -}; - -const CudaTracer = struct { - - // Those symbols are defined in cudaProfiler.h but their implementation is in libcuda.so - // They will be bound at call time after libcuda.so is loaded (as a needed dependency of libpjrt_cuda.so). - const cuProfilerStart = @extern(*const fn () callconv(.c) c_int, .{ .name = "cuProfilerStart", .linkage = .weak }) orelse unreachable; - const cuProfilerStop = @extern(*const fn () callconv(.c) c_int, .{ .name = "cuProfilerStop", .linkage = .weak }) orelse unreachable; - - // Those symbols are defined in nvToolsExt.h which we don't want to provide. - // However, we link with libnvToolsExt.so which provides them. - // They will be bound at call time after libnvToolsExt.so is loaded (manually dlopen'ed by us). - const nvtxMarkA = @extern(*const fn ([*:0]const u8) callconv(.c) void, .{ .name = "nvtxMarkA", .linkage = .weak }) orelse unreachable; - const nvtxRangeStartA = @extern(*const fn ([*:0]const u8) callconv(.c) c_int, .{ .name = "nvtxRangeStartA", .linkage = .weak }) orelse unreachable; - const nvtxRangeEnd = @extern(*const fn (c_int) callconv(.c) void, .{ .name = "nvtxRangeEnd", .linkage = .weak }) orelse unreachable; - - pub fn init(name: [:0]const u8) CudaTracer { - _ = name; - _ = cuProfilerStart(); - return .{}; - } - - pub fn deinit(self: *const CudaTracer) void { - _ = self; - _ = cuProfilerStop(); - } - - pub fn event(self: *const CudaTracer, message: [:0]const u8) void { - _ = self; - nvtxMarkA(message.ptr); - } - - pub fn frameStart(self: *const CudaTracer, message: [:0]const u8) u64 { - _ = self; - return @intCast(nvtxRangeStartA(message.ptr)); - } - - pub fn frameEnd(self: *const CudaTracer, interval_id: u64, message: [:0]const u8) void { - _ = self; - _ = message; - nvtxRangeEnd(@intCast(interval_id)); - return; - } -}; - -const MacOsTracer = struct { - logger: c.os_log_t, - - pub fn init(name: [:0]const u8) MacOsTracer { - const logger = c.os_log_create(name.ptr, c.OS_LOG_CATEGORY_POINTS_OF_INTEREST); - return .{ - .logger = logger, - }; - } - - pub fn event(self: *const MacOsTracer, message: [:0]const u8) void { - const interval_id = c.os_signpost_id_generate(self.logger); - c.zml_os_signpost_event(self.logger, interval_id, message); - } - - pub fn frameStart(self: *const MacOsTracer, message: [:0]const u8) c.os_signpost_id_t { - const interval_id = c.os_signpost_id_generate(self.logger); - c.zml_os_signpost_interval_begin(self.logger, interval_id, message); - return interval_id; - } - - pub fn frameEnd(self: *const MacOsTracer, interval_id: c.os_signpost_id_t, message: [:0]const u8) void { - c.zml_os_signpost_interval_end(self.logger, interval_id, message); - } -}; - -/// Mock tracer for OS which don't have an impl. -const FakeTracer = struct { - pub fn init(name: [:0]const u8) FakeTracer { - _ = name; - return .{}; - } - - pub fn event(self: *const FakeTracer, message: [:0]const u8) void { - _ = self; - _ = message; - return; - } - - pub fn frameStart(self: *const FakeTracer, message: [:0]const u8) u64 { - _ = self; - _ = message; - return 0; - } - - pub fn frameEnd(self: *const FakeTracer, interval_id: u64, message: [:0]const u8) void { - _ = self; - _ = interval_id; - _ = message; - return; - } -}; diff --git a/zml/torch.zig b/zml/torch.zig deleted file mode 100644 index 09888aee..00000000 --- a/zml/torch.zig +++ /dev/null @@ -1,285 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -const zml = @import("zml.zig"); -const Tensor = zml.Tensor; - -/// Multiplies a matrix or a vector with a tensor, -/// following the semantic of pytorch `@` operator. -/// When both sides are matrices, it's the textbook matrix multiplication : -/// `matmul(.{ 8, 9 }, .{ 9, 10 }) -> .{ 8, 10 }` -/// When one of the input is a tensor, it assumes the first dimensions are batches, -/// and the last two ones are used for the regular matmul. -/// * `matmul(.{10}, .{10}) -> .{}` -/// * `matmul(.{10}, .{10}) -> .{}` -pub fn matmul(lhs: Tensor, rhs: Tensor) Tensor { - stdx.debug.assert(lhs.rank() >= 1 and rhs.rank() >= 1, "Can't matmul({f}, {f}) ! The two tensors need to have at least rank 1.", .{ lhs, rhs }); - - const contracting = [_][2]i8{.{ -1, if (rhs.rank() >= 2) rhs.rank() - 2 else 0 }}; - if (lhs.rank() == 1 or rhs.rank() <= 2) { - // When lhs is a vector or rhs is small the torch semantics match the dot_general semantics and life is easy. - return lhs.dotGeneral(rhs, &contracting, &.{}); - } - - stdx.debug.assert(lhs.rank() == 2, "Can't matmul({f}, {f}) ! One of the two tensors need to have a rank less than 2.", .{ lhs, rhs }); - - // Pytorch treats the extra dimensions of rhs has batching dimensions, - // and implicitly broadcast lhs along those. - // We make this broadcasting explicit. - var left_shape = rhs.shape(); - left_shape._dims.set(left_shape.axis(-2), lhs.dim(-2)); - left_shape._tags.set(left_shape.axis(-2), lhs.shape().tag(-2)); - left_shape._dims.set(left_shape.axis(-1), lhs.dim(-1)); - left_shape._tags.set(left_shape.axis(-1), lhs.shape().tag(-1)); - const lhs_broad = lhs.broadcastLeft(left_shape); - - const n_batching_axes = rhs.rank() - lhs.rank(); - var batching: [Tensor.MAX_RANK][2]i8 = undefined; - for (0..n_batching_axes) |i| { - batching[i] = .{ @intCast(i), @intCast(i) }; - } - return lhs_broad.dotGeneral(rhs, &contracting, batching[0..n_batching_axes]); -} - -test matmul { - const platform = zml.testing.env(); - - var comp = try zml.module.CompilationContext.init(std.heap.page_allocator, "test", platform); - defer comp.deinit(); - - comp.activate(); - defer comp.deactivate(); - - // Generated with pytorch - inline for (.{ - .{ .{20}, .{20}, .{} }, - .{ .{20}, .{ 20, 15 }, .{15} }, - .{ .{20}, .{ 11, 20, 15 }, .{ 11, 15 } }, - .{ .{20}, .{ 9, 11, 20, 15 }, .{ 9, 11, 15 } }, - .{ .{20}, .{ 7, 9, 11, 20, 15 }, .{ 7, 9, 11, 15 } }, - .{ .{20}, .{ 5, 7, 9, 11, 20, 15 }, .{ 5, 7, 9, 11, 15 } }, - .{ .{ 12, 20 }, .{20}, .{12} }, - .{ .{ 12, 20 }, .{ 20, 15 }, .{ 12, 15 } }, - .{ .{ 12, 20 }, .{ 11, 20, 15 }, .{ 11, 12, 15 } }, - .{ .{ 12, 20 }, .{ 9, 11, 20, 15 }, .{ 9, 11, 12, 15 } }, - .{ .{ 12, 20 }, .{ 7, 9, 11, 20, 15 }, .{ 7, 9, 11, 12, 15 } }, - .{ .{ 12, 20 }, .{ 5, 7, 9, 11, 20, 15 }, .{ 5, 7, 9, 11, 12, 15 } }, - .{ .{ 10, 12, 20 }, .{20}, .{ 10, 12 } }, - .{ .{ 10, 12, 20 }, .{ 20, 15 }, .{ 10, 12, 15 } }, - .{ .{ 8, 10, 12, 20 }, .{20}, .{ 8, 10, 12 } }, - .{ .{ 8, 10, 12, 20 }, .{ 20, 15 }, .{ 8, 10, 12, 15 } }, - .{ .{ 6, 8, 10, 12, 20 }, .{20}, .{ 6, 8, 10, 12 } }, - .{ .{ 6, 8, 10, 12, 20 }, .{ 20, 15 }, .{ 6, 8, 10, 12, 15 } }, - .{ .{ 4, 6, 8, 10, 12, 20 }, .{20}, .{ 4, 6, 8, 10, 12 } }, - .{ .{ 4, 6, 8, 10, 12, 20 }, .{ 20, 15 }, .{ 4, 6, 8, 10, 12, 15 } }, - }) |testcase| { - const x_shape, const y_shape, const z_shape = testcase; - const x = Tensor.constant(x_shape, .{ .f32 = 0.0 }); - const y = Tensor.constant(y_shape, .{ .f32 = 0.0 }); - const z = matmul(x, y); - - try std.testing.expectEqualSlices(i64, &z_shape, z.dims()); - } -} - -/// Inserts a 1-dim axis at the given position. -/// Negative indexes are handled like pytorch, ie they are relative to the returned shaped: -/// - `.{5, 4}.unsqueeze(1)` returns `.{5, 1, 4}` -/// - `.{5, 4}.unsqueeze(-1)` returns `.{5, 4, 1}` -pub fn unsqueeze( - self: Tensor, - axis_: anytype, -) Tensor { - stdx.debug.assert(self.rank() < Tensor.MAX_RANK - 1, "Can't unsqueeze {f}, it's already at max rank.", .{self}); - const a = switch (@typeInfo(@TypeOf(axis_))) { - .int, .comptime_int => if (axis_ < 0) - @as(i8, self.rank()) + 1 + axis_ - else - self.axis(axis_), - else => self.axis(axis_), - }; - return self.insertAxes(a, .{._}); -} - -test unsqueeze { - const Local = struct { - pub fn _fwd(x: Tensor) Tensor { - var y = x; - y = unsqueeze(y, 0); - y = unsqueeze(y, -1); - y = unsqueeze(y, -1); - return y; - } - }; - const platform = zml.testing.env(); - - const x = try zml.Buffer.fromArray(platform, @as([8]f16, undefined)); - const res = try zml.testing.compileAndCall(platform, Local._fwd, .{x}); - try zml.testing.expectEqualShapes(zml.Shape.init(.{ 1, 8, 1, 1 }, .f16), res.shape()); -} - -/// Given an input images with .{ .c, .w, .h } tags, -/// shuffle values between the channel (.c), width (.w) and height (.h) axes. -/// pixelShuffle(.{ .c, .w, .h }, u) -> .{ .c / u / u, .w * u, .h * u} -/// ref: https://pytorch.org/docs/stable/generated/torch.nn.PixelShuffle.html#pixelshuffle -pub fn pixelShuffle(tensor: Tensor, upscale_factor: u32) Tensor { - const shape = tensor.shape(); - stdx.debug.assert(shape.hasTags(.{ .c, .w, .h }), "pixelShuffle({f}) is invalide. Missing tags {{.c, .w, .h}}", .{tensor}); - - stdx.debug.assert(@mod(shape.dim(.c), upscale_factor * upscale_factor) == 0, "pixelShuffle({f}) is invalide. Number of channels {}, isn't divisible by upscale factor {}**2", .{ tensor, shape.dim(.c), upscale_factor }); - - const s = tensor.splitAxis(.c, .{ .c = -1, .upscale_h = upscale_factor, .upscale_w = upscale_factor }); - const perm = s.shape().contiguousPerm(.{ .h, .upscale_h, .w, .upscale_w }); - const cont = s.transpose(perm.constSlice()); - return cont.merge(.{ .h = .{ .h, .upscale_h }, .w = .{ .w, .upscale_w } }).transpose(tensor.shape()); -} - -test pixelShuffle { - const platform = zml.testing.env(); - - const upscale_factor = 3; - const shape = zml.Shape.init(.{ .b = 1, .c = 9, .h = 4, .w = 4 }, .i32); - const input = input: { - var digits: [9 * 4 * 4]i32 = undefined; - for (&digits, 0..) |*d, i| d.* = @intCast(i); - break :input try zml.Buffer.fromSlice(platform, shape, &digits); - }; - - const output = try zml.testing.compileAndCall(platform, pixelShuffle, .{ input, upscale_factor }); - - const exp = zml.HostBuffer.fromArray(&[1][1][12][12]i32{.{.{ - .{ 0, 16, 32, 1, 17, 33, 2, 18, 34, 3, 19, 35 }, - .{ 48, 64, 80, 49, 65, 81, 50, 66, 82, 51, 67, 83 }, - .{ 96, 112, 128, 97, 113, 129, 98, 114, 130, 99, 115, 131 }, - .{ 4, 20, 36, 5, 21, 37, 6, 22, 38, 7, 23, 39 }, - .{ 52, 68, 84, 53, 69, 85, 54, 70, 86, 55, 71, 87 }, - .{ 100, 116, 132, 101, 117, 133, 102, 118, 134, 103, 119, 135 }, - .{ 8, 24, 40, 9, 25, 41, 10, 26, 42, 11, 27, 43 }, - .{ 56, 72, 88, 57, 73, 89, 58, 74, 90, 59, 75, 91 }, - .{ 104, 120, 136, 105, 121, 137, 106, 122, 138, 107, 123, 139 }, - .{ 12, 28, 44, 13, 29, 45, 14, 30, 46, 15, 31, 47 }, - .{ 60, 76, 92, 61, 77, 93, 62, 78, 94, 63, 79, 95 }, - .{ 108, 124, 140, 109, 125, 141, 110, 126, 142, 111, 127, 143 }, - }}}); - try zml.testing.expectClose(exp, output, 0); -} - -/// Implementation of `torch.roll`. -/// -/// Note: at the difference of Pytorch, shifts need to be explicitly repeated, even if they are the same for all axes. -/// ref: https://pytorch.org/docs/stable/generated/torch.roll.html -pub fn roll(self: Tensor, shifts: []const i64, axes_: []const i8) Tensor { - // TODO(hugo) accept following syntax: x.roll(.{ .a = 5, .b = 8 }) - stdx.debug.assert(self.rank() > 0 and shifts.len == axes_.len, "Shifts length ({d}) and dims length ({d}) are not equal, we expect the same length.", .{ shifts.len, axes_.len }); - - if (shifts.len != 1 or axes_.len != 1) { - const tail_shifts = shifts[1..shifts.len]; - const tail_dims = axes_[1..axes_.len]; - const first_dim_rolled = roll(self, &.{shifts[0]}, &.{axes_[0]}); - return roll(first_dim_rolled, tail_shifts, tail_dims); - } - - const a = self.axis(axes_[0]); - const start = @mod(self.dim(a) - shifts[0], self.dim(a)); - const idx = Tensor.arange(.{ .start = start, .end = start + self.dim(a) }, .f32); - const divisor: f32 = @floatFromInt(self.dim(a)); - return self.gather_(&.{a}, &.{idx.fmod(divisor).convert(.i32)}, .{}); -} - -test roll { - const platform = zml.testing.env(); - - const input = try zml.Buffer.fromSlice(platform, .{ 4, 2 }, &[_]f32{ 2, 2, 3, 4, 5, 6, 7, 8 }); - const res = try zml.testing.compileAndCall( - platform, - roll, - .{ input, &[_]i64{ 2, 1 }, &[_]i8{ 0, 1 } }, - ); - - const expectation = zml.HostBuffer.fromSlice(.{ 4, 2 }, &[_]f32{ 6, 5, 8, 7, 2, 1, 4, 3 }); - try zml.testing.expectClose(expectation, res, 1e0); -} - -pub const MeshgridIndexing = enum { xy, ij }; - -/// Mimic Pytorch and Numpy api. -/// The .ij mode is just calling to `zml.nn.cartesianProduct` -/// and has simple semantics. -/// The .xy mode swap the role of the first two vectors, it's generally best -/// to rewrite the calling code to use .ij mode if possible. -/// See Numpy docs: -/// https://numpy.org/doc/stable/reference/generated/numpy.meshgrid.html#numpy.meshgrid -/// - In the 2-D case with vectors of length M and N: -/// * for ‘ij’ indexing, outputs are of shape (M, N) -/// * for ‘xy’ indexing, outputs are of shape (N, M) -/// - In the 3-D case with vectors of length M, N and P: -/// * for ‘ij’ indexing, outputs are of shape (M, N, P) -/// * for ‘xy’ indexing, outputs are of shape (N, M, P) -pub fn meshgrid(comptime N: u3, vectors: [N]Tensor, indexing: MeshgridIndexing) [N]Tensor { - stdx.debug.assertComptime(vectors.len >= 1, "Invalid meshgrid. No input.", .{}); - stdx.debug.assertComptime(vectors.len <= Tensor.MAX_RANK, "Invalid meshgrid(...). Too many inputs: {}", .{vectors.len}); - - if (vectors.len == 1) return vectors; - - return switch (indexing) { - .ij => zml.Tensor.cartesianProduct(N, vectors), - .xy => { - const x, const y = vectors[0..2].*; - var new_vectors = vectors; - new_vectors[0..2].* = .{ y, x }; - var res = zml.Tensor.cartesianProduct(N, new_vectors); - const y_res, const x_res = res[0..2].*; - res[0..2].* = .{ x_res, y_res }; - return res; - }, - }; -} - -test meshgrid { - const platform = zml.testing.env(); - - const x = try zml.Buffer.fromSlice(platform, .{6}, &[_]i32{ 0, 1, 2, 3, 4, 5 }); - const y = try zml.Buffer.fromSlice(platform, .{4}, &[_]i32{ 0, 1, 2, 3 }); - - const Local = struct { - pub fn _meshgrid2(a: Tensor, b: Tensor, indexing: MeshgridIndexing) [2]Tensor { - return meshgrid(2, .{ a, b }, indexing); - } - }; - - // Only test .xy mode, sinc .ij is just calling cartesianProduct which - // got its own tests. - { - const xs, const ys = try zml.testing.compileAndCall(platform, Local._meshgrid2, .{ x, y, .xy }); - try std.testing.expectEqualSlices(i64, &.{ 4, 6 }, xs.dims()); - try std.testing.expectEqualSlices(i64, &.{ 4, 6 }, ys.dims()); - try std.testing.expectEqualDeep( - [4][6]i32{ - .{ 0, 1, 2, 3, 4, 5 }, - .{ 0, 1, 2, 3, 4, 5 }, - .{ 0, 1, 2, 3, 4, 5 }, - .{ 0, 1, 2, 3, 4, 5 }, - }, - try xs.getValue([4][6]i32), - ); - try std.testing.expectEqualDeep( - [4][6]i32{ - .{ 0, 0, 0, 0, 0, 0 }, - .{ 1, 1, 1, 1, 1, 1 }, - .{ 2, 2, 2, 2, 2, 2 }, - .{ 3, 3, 3, 3, 3, 3 }, - }, - try ys.getValue([4][6]i32), - ); - } -} - -/// Flattens the given axis and the next one, into one new axis. -pub fn flatten(self: Tensor, axis_: anytype) Tensor { - const old_shape = self._shape; - const a = self.axis(axis_); - stdx.debug.assert(a + 1 < self.rank(), "Can't flatten {f} on the last axis {}.", .{ self, axis_ }); - const new_shape = old_shape.mergeAxis(a, .{ a, a + 1 }); - return self.reshape(new_shape); -} diff --git a/zml/zml.zig b/zml/zml.zig index 55559f5c..f56946a3 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -6,53 +6,82 @@ // Namespaces const std = @import("std"); -pub const platform_specific = @import("c"); -pub const tokenizer = @import("zml/tokenizer"); +const c = @import("c"); +const runfiles = @import("runfiles"); -pub const aio = @import("aio.zig"); pub const Buffer = @import("buffer.zig").Buffer; -pub const Bufferized = @import("tensor.zig").Bufferized; -pub const callback = @import("callback.zig"); -pub const CompilationOptions = @import("platform.zig").CompilationOptions; -pub const context = @import("context.zig"); -pub const Context = @import("context.zig").Context; -pub const Data = @import("dtype.zig").Data; -pub const DataType = @import("dtype.zig").DataType; +pub const constants = @import("constants.zig"); +pub const dtype = @import("dtype.zig"); +pub const Data = dtype.Data; +pub const DataType = dtype.DataType; pub const exe = @import("exe.zig"); -pub const compile = exe.compile; -pub const compileWithPrefix = exe.compileWithPrefix; -pub const compileFn = exe.compileFn; -pub const compileModel = exe.compileModel; -pub const FnExe = exe.FnExe; -pub const ModuleExe = exe.ModuleExe; -pub const ModuleSignature = exe.ModuleSignature; +pub const Exe = exe.Exe; pub const floats = @import("floats.zig"); -pub const helpers = @import("helpers.zig"); -pub const HostBuffer = @import("hostbuffer.zig").HostBuffer; pub const meta = @import("meta.zig"); pub const mlir = @import("mlirx.zig"); pub const module = @import("module.zig"); pub const nn = @import("nn.zig"); -pub const ops = @import("ops.zig"); -pub const call = ops.call; pub const pjrt = @import("pjrtx.zig"); +pub const io = @import("io.zig"); pub const platform = @import("platform.zig"); -pub const Platform = @import("platform.zig").Platform; -pub const Shape = @import("shape.zig").Shape; -pub const ShapeOf = @import("tensor.zig").ShapeOf; -pub const Target = @import("platform.zig").Target; -pub const Tensor = @import("tensor.zig").Tensor; +pub const Platform = platform.Platform; +pub const Target = platform.Target; +pub const CompilationOptions = platform.CompilationOptions; +pub const shape = @import("shape.zig"); +pub const Shape = shape.Shape; +pub const safetensors = @import("safetensors.zig"); +pub const slice = @import("slice.zig"); +pub const Slice = slice.Slice; +pub const ConstSlice = slice.ConstSlice; +pub const tensor = @import("tensor.zig"); +pub const Tensor = tensor.Tensor; pub const testing = @import("testing.zig"); -pub const torch = @import("torch.zig"); +pub const tokenizer = @import("zml/tokenizer"); -pub const tools = struct { - pub const Tracer = @import("tools/tracer.zig").Tracer; -}; +var runfiles_once = std.once(struct { + fn call_() !void { + if (std.process.hasEnvVarConstant("RUNFILES_MANIFEST_FILE") or std.process.hasEnvVarConstant("RUNFILES_DIR")) { + return; + } -pub const log = std.log.scoped(.zml); + var io_: std.Io.Threaded = .init_single_threaded; + defer io_.deinit(); -test { - // NOTE : testing entrypoint. - // Don't forget to import your module if you want to declare tests declarations that will be run by //zml:test + var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); + const allocator = arena.allocator(); + defer arena.deinit(); + + var envMap = std.process.EnvMap.init(allocator); + var r = (try runfiles.Runfiles.create(.{ .allocator = allocator, .io = io_.io() })) orelse return; + try r.environment(&envMap); + + var it = envMap.iterator(); + while (it.next()) |entry| { + const keyZ = try allocator.dupeZ(u8, entry.key_ptr.*); + const valueZ = try allocator.dupeZ(u8, entry.value_ptr.*); + _ = c.setenv(keyZ.ptr, valueZ.ptr, 1); + } + } + + fn call() void { + call_() catch @panic("Unable to init runfiles env"); + } +}.call); + +pub fn init() void { + runfiles_once.call(); + mlir.once.call(); +} + +pub fn deinit() void {} + +/// Return a clone of a type with Tensors replaced by Buffer. +/// Non-Tensor metadata is stripped out of the resulting struct. +/// Recursively descends into the type. +pub fn Bufferized(comptime T: type) type { + return meta.MapRestrict(Tensor, Buffer).map(T); +} + +test "zml" { std.testing.refAllDecls(@This()); } diff --git a/zml2/BUILD.bazel b/zml2/BUILD.bazel deleted file mode 100644 index ac8f913a..00000000 --- a/zml2/BUILD.bazel +++ /dev/null @@ -1,101 +0,0 @@ -load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_c_proto_library") -load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_test") - -upb_c_proto_library( - name = "xla_data_upb", - deps = ["@xla//xla:xla_data_proto"], -) - -upb_c_proto_library( - name = "xla_compile_options_upb", - deps = ["@xla//xla/pjrt/proto:compile_options_proto"], -) - -cc_library( - name = "posix", - hdrs = ["posix.h"], -) - -zig_library( - name = "zml", - srcs = [ - "buffer.zig", - "constants.zig", - "dtype.zig", - "exe.zig", - "floats.zig", - "io.zig", - "meta.zig", - "mlirx.zig", - "module.zig", - "nn.zig", - "ops.zig", - "pjrtx.zig", - "platform.zig", - "safetensors.zig", - "shape.zig", - "slice.zig", - "tensor.zig", - "testing.zig", - "zml.zig", - ], - main = "zml.zig", - visibility = ["//visibility:public"], - zigopts = [ - "-lc", - "-freference-trace=20", - ], - deps = [ - ":posix", - ":xla_compile_options_upb", - ":xla_data_upb", - "//mlir", - "//mlir/dialects", - "//pjrt", - "//runtimes", - "//stdx", - "//upb", - "//zml2/tokenizer", - "//zml2/io", - "@rules_zig//zig/runfiles", - ], -) - -zig_binary( - name = "main", - args = [], - data = [], - main = "main.zig", - deps = [ - "//zml2:zml", - ], -) - -zig_binary( - name = "main_io", - args = [], - data = [], - main = "main_io.zig", - deps = [ - "//zml2:zml", - ], -) - -# All ZML Tests - -zig_test( - name = "test", - data = [ - #"aio/torch/simple.pt", - #"aio/torch/simple_test_4.pickle", - ], - test_runner = ":test_runner", - deps = ["//zml2:zml"], -) - -filegroup( - name = "test_runner", - srcs = ["test_runner.zig"], - visibility = ["//visibility:public"], -) diff --git a/zml2/buffer.zig b/zml2/buffer.zig deleted file mode 100644 index bafca61c..00000000 --- a/zml2/buffer.zig +++ /dev/null @@ -1,252 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -const constants = @import("constants.zig"); -const ConstSlice = @import("slice.zig").ConstSlice; -const DataType = @import("dtype.zig").DataType; -const pjrt = @import("pjrtx.zig"); -const Platform = @import("platform.zig").Platform; -const Shape = @import("shape.zig").Shape; -const Slice = @import("slice.zig").Slice; -const Target = @import("platform.zig").Target; -const testing = @import("testing.zig"); - -const log = std.log.scoped(.zml); - -/// Buffer is a multi-dimension array, whose memory is allocated on an accelerator. -/// -/// * contains a handle that the ZML runtime can use to convert into a physical address, but there is no guarantee this address is visible from the CPU. -/// * loading weights from disk directly to the `device zml.aio.loadBuffers` -/// * can be created by calling `HostBuffer.toDevice(platform)`. -pub const Buffer = struct { - _shape: Shape, - _api: *const pjrt.Api, - _target: Target, - _shards: Shards, - - pub const MAX_NUM_SHARDS: u8 = Platform.MAX_NUM_DEVICES; - pub const Shards = stdx.BoundedArray(*pjrt.Buffer, MAX_NUM_SHARDS); - - pub const Memory = pjrt.Memory.Kind; - pub const FromOptions = struct { wait: bool = true, memory: Memory = .device }; - - /// Frees the accelerator memory. - /// Depending on the platform, the memory is typically not released to the OS - /// but just marked as available in the memory pool. - pub fn deinit(self: *const Buffer) void { - // log.warn("Unloading {f} {d} bytes", .{ self._shape, self._shape.byteSize() }); - for (self._shards.constSlice()) |buffer| { - buffer.deinit(self._api); - } - } - - /// This Buffer shape. - pub fn shape(self: Buffer) Shape { - return self._shape; - } - - pub fn format(self: Buffer, writer: *std.Io.Writer) !void { - try writer.print("Buffer({f})@{x}", .{ self._shape, @intFromPtr(self.devicePtr()) }); - } - - /// Copies the content of the given buffer from host memory to the accelerator memory. - pub fn from(io: std.Io, platform: Platform, shape_: Shape, data_: []const u8, opts: FromOptions) !Buffer { - var res: Buffer = .{ - ._api = platform.pjrt_api, - ._target = platform.target, - ._shape = shape_, - ._shards = .{}, - }; - - // We shard only on the first axis so that the chunks are still contiguous. - // TODO: support more advanced sharding specs - stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); - //const sharding_ax: ?u3 = std.simd.firstTrue(shape_._sharding_info); - const n_partitions = platform.sharding().num_partitions; - //const chunk_size = if (sharding_ax) |ax| cs: { - // // This kind of sharding error should be detected earlier on. - // stdx.debug.assert(@rem(shape_.dim(ax), n_partitions) == 0, "Buffer.from({f}) expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ shape_, ax, n_partitions }); - // break :cs @divExact(shape_.dim(ax), n_partitions); - //} else 0; - //_ = chunk_size; - - const buffer_type = pjrt.bufferTypeFromDtype(shape_.dtype()); - const byte_strides = shape_.computeByteStrides(); - - const devices = platform.getDevices(); - for (0..n_partitions) |i| { - // If no sharding if found, the given buffer is replicated on all devices. - //const buf = if (sharding_ax) |ax| buf: { - // @panic("sharding not implemented"); - // //const start: i64 = @as(i64, @intCast(i)) * chunk_size; - // //break :buf host_buffer.slice1d(ax, .{ .start = start, .end = start + chunk_size }); - //} else host_buffer; - - const args = pjrt.Client.BufferFromHostBufferArgs{ - .data = data_.ptr, - .buffer_type = buffer_type, - .dims = shape_.dims(), - .byte_strides = byte_strides.slice(), - .host_buffer_semantics = .ImmutableUntilTransferCompletes, - // CPU has no distinctions between memories. - .dst = if (platform.target == .cpu or opts.memory == .device) - .{ .device = devices[i] } - else - .{ .memory = platform.memoryForDevice(opts.memory, devices[i]) }, - }; - - const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); - - if (event) |ev| ev.deinit(platform.pjrt_api); - res._shards.appendAssumeCapacity(pjrt_buffer); - } - - if (opts.wait) { - res = try res.await(io); - } - - return res; - } - - /// Copies the given Zig bytes to the accelerator memory and - /// return a Buffer with the given dimensions. - pub fn fromBytes(io: std.Io, platform: Platform, sh: Shape, data: []const u8) !Buffer { - return from(io, platform, sh, data, .{}); - } - - /// Copies the given zml.Slice or zml.ConstSlice to the accelerator memory and - /// return a Buffer. - pub fn fromSlice(io: std.Io, platform: Platform, slice: anytype) !Buffer { - return fromSliceOpts(io, platform, slice, .{}); - } - - pub fn fromSliceOpts(io: std.Io, platform: Platform, slice: anytype, opts: FromOptions) !Buffer { - if (@TypeOf(slice) != Slice and @TypeOf(slice) != ConstSlice) @compileError("Buffer.fromSlice expects a Slice or ConstSlice, got " ++ @typeName(@TypeOf(slice))); - const real_slice: ConstSlice = if (@TypeOf(slice) == Slice) slice.constSlice() else slice; - return from(io, platform, real_slice.shape, std.mem.sliceAsBytes(real_slice.data), opts); - } - - /// Creates a Buffer with a single element. - pub fn scalar(io: std.Io, platform: Platform, val: anytype, dtype_: DataType) !Buffer { - const x = dtype_.constant(val); - return fromBytes(io, platform, Shape.init(.{}, dtype_), x.asBytes()); - } - - pub fn await(self: Buffer, io: std.Io) !Buffer { - for (self._shards.constSlice()) |buffer| { - try buffer.getReadyEvent(self._api).await(self._api, io); - } - - return self; - } - - pub const UnitializedOptions = struct { memory: Memory = .device }; - - pub fn uninitialized(io: std.Io, platform: Platform, shape_: Shape, opts: UnitializedOptions) !Buffer { - if (opts.memory != .device) { - // XLA uninitialized doesn't respect memory see https://github.com/openxla/xla/pull/31292 - // TODO: use uninitialized when it works again. - const slice: Slice = try .alloc(std.heap.smp_allocator, shape_); - defer slice.free(std.heap.smp_allocator); - return .fromSliceOpts(io, platform, slice, .{ .memory = opts.memory }); - } - - var res: Buffer = .{ - ._api = platform.pjrt_api, - ._shape = shape_, - ._shards = .{}, - ._target = platform.target, - }; - errdefer for (res._shards.slice()) |shard| { - shard.deinit(platform.pjrt_api); - }; - - stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); - //const sharding_ax: ?u3 = std.simd.firstTrue(shape_._sharding_info); - const n_partitions = platform.sharding().num_partitions; - - //const shard_shape = if (sharding_ax) |ax| s: { - // // This kind of sharding error should be detected earlier on. - // stdx.debug.assert(@rem(shape_.dim(ax), n_partitions) == 0, "Buffer.uninitialized() expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ ax, n_partitions }); - // const shard_shape = shape_.set(ax, @divExact(shape_.dim(ax), n_partitions)); - // break :s shard_shape; - //} else shape_; - - var args = pjrt.Client.CreateUninitializedBufferArgs{ - .dims = shape_.dims(), - .element_type = pjrt.bufferTypeFromDtype(shape_.dtype()), - .layout = .{ - .tiled = .{ - .minor_to_major = constants.minorToMajor(shape_.rank()), - .tile_dims = &.{}, - .tile_dims_sizes = &.{}, - }, - }, - // set per device, see below. - .dst = undefined, - }; - - const devices = platform.getDevices(); - for (0..n_partitions) |i| { - args.dst = if (platform.target == .cpu or opts.memory == .device) - .{ .device = devices[i] } - else - .{ .memory = platform.memoryForDevice(opts.memory, devices[i]) }; - - const shard = try platform.pjrt_client.createUnitializedBuffer(platform.pjrt_api, args); - res._shards.appendAssumeCapacity(shard); - } - - return res; - } - - /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. - pub fn fromPjrtBuffers(platform: Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { - stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); - stdx.debug.assert(pjrt_buffers.len > 0, "fromPjrtBuffers expects at least one buffer, got 0.", .{}); - var shards: Shards = .{}; - shards.appendSliceAssumeCapacity(pjrt_buffers); - return .{ - ._api = platform.pjrt_api, - ._target = platform.target, - ._shape = shape_, - ._shards = shards, - }; - } - - pub fn devicePtr(self: Buffer) *anyopaque { - //stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer", .{}); - return self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable; - } - - /// Fetches the content of the given buffer into a stack variable of the given type. - pub fn getValue(self: Buffer, T: type, io: std.Io) !T { - stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {f} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); - var res: T = undefined; - - try self.toSlice(io, .init(self.shape(), std.mem.asBytes(&res))); - - return res; - } - - /// Copies the content of the Buffer to the provided slice. - pub fn toSlice(self: Buffer, io: std.Io, slice: Slice) !void { - //stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, slice.data); - if (maybe_event) |event| { - try event.await(self._api, io); - } - } - - /// Copies the content of the Buffer to the provided slice. - /// The returned slice owns the memory. - pub fn toSliceAlloc(self: Buffer, allocator: std.mem.Allocator, io: std.Io) !Slice { - const slice = try Slice.alloc(allocator, self.shape()); - errdefer slice.free(allocator); - - try self.toSlice(io, slice); - - return slice; - } -}; diff --git a/zml2/dtype.zig b/zml2/dtype.zig deleted file mode 100644 index 8ec73c3d..00000000 --- a/zml2/dtype.zig +++ /dev/null @@ -1,373 +0,0 @@ -const std = @import("std"); - -const floats = @import("floats.zig"); - -const C64 = std.math.Complex(f32); -const C128 = std.math.Complex(f64); - -pub const DataType = enum(u8) { - bool, - // Note: the support of the float8 is a bit spotty, f8e4m3b11fnuz seems to be the most supported one on Cuda. - f4e2m1, - f8e3m4, - f8e4m3, - f8e4m3b11fnuz, - f8e4m3fn, - f8e4m3fnuz, - f8e5m2, - f8e5m2fnuz, - f8e8m0, - bf16, - f16, - f32, - f64, - i2, - i4, - i8, - i16, - i32, - i64, - u2, - u4, - u8, - u16, - u32, - u64, - c64, - c128, - - pub fn str(self: DataType) [:0]const u8 { - return switch (self) { - inline else => |tag| @tagName(tag), - }; - } - - pub const Class = enum(u8) { - bool, - float, - integer, - complex, - }; - - pub fn class(self: DataType) Class { - return switch (self) { - .bool => .bool, - .f4e2m1, - .f8e3m4, - .f8e4m3, - .f8e4m3b11fnuz, - .f8e4m3fn, - .f8e4m3fnuz, - .f8e5m2, - .f8e5m2fnuz, - .f8e8m0, - .bf16, - .f16, - .f32, - .f64, - => .float, - .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => .integer, - .c64, .c128 => .complex, - }; - } - - pub fn isInteger(self: DataType) bool { - return self.class() == .integer; - } - - pub fn isFloat(self: DataType) bool { - return self.class() == .float; - } - - pub fn isComplex(self: DataType) bool { - return self.class() == .complex; - } - - pub fn fromZigType(comptime T: type) DataType { - return switch (T) { - floats.Float4E2M1 => .f4e2m1, - floats.Float8E3M4 => .f8e3m4, - floats.Float8E4M3 => .f8e4m3, - floats.Float8E4M3B11FNUZ => .f8e4m3b11fnuz, - floats.Float8E4M3FN => .f8e4m3fn, - floats.Float8E4M3FNUZ => .f8e4m3fnuz, - floats.Float8E5M2 => .f8e5m2, - floats.Float8E5M2FNUZ => .f8e5m2fnuz, - floats.Float8E8M0 => .f8e8m0, - floats.BFloat16 => .bf16, - f16 => .f16, - f32 => .f32, - f64 => .f64, - bool => .bool, - i2 => .i2, - i4 => .i4, - i8 => .i8, - i16 => .i16, - i32 => .i32, - i64 => .i64, - u2 => .u2, - u4 => .u4, - u8 => .u8, - u16 => .u16, - u32 => .u32, - u64 => .u64, - C64 => .c64, - C128 => .c128, - else => @compileError("Unsupported Zig type: " ++ @typeName(T)), - }; - } - - pub fn fromSliceElementType(slice: anytype) DataType { - const type_info = @typeInfo(@TypeOf(slice)); - if (type_info != .pointer) { - @compileError("`initFromSlice` expects a slice, got " ++ @tagName(type_info)); - } - - return switch (type_info.pointer.size) { - .slice, .c, .many => DataType.fromZigType(type_info.pointer.child), - .one => b: { - const child_type_info = @typeInfo(type_info.pointer.child); - break :b DataType.fromZigType(child_type_info.array.child); - }, - }; - } - - pub fn toZigType(comptime dtype: DataType) type { - return @FieldType(Value, @tagName(dtype)); - } - - pub fn isSignedInt(dtype: DataType) bool { - return switch (dtype) { - .i4, .i8, .i16, .i32, .i64 => true, - else => false, - }; - } - - pub fn sizeOf(self: DataType) u16 { - return switch (self) { - inline else => |tag| @sizeOf(tag.toZigType()), - }; - } - - pub fn bitSizeOf(self: DataType) u16 { - return switch (self) { - inline else => |tag| @bitSizeOf(tag.toZigType()), - }; - } - - pub fn alignOf(self: DataType) u29 { - return switch (self) { - inline else => |tag| @alignOf(tag.toZigType()), - }; - } - - /// Try to find a type compatible with both dtype. - pub fn resolvePeerType(a: DataType, b: DataType) ?DataType { - if (a == b) { - return a; - } - - // only resolve types in the same class - if (a.class() != b.class()) { - return null; - } - - return if (a.sizeOf() >= b.sizeOf()) a else b; - } - - test resolvePeerType { - try std.testing.expectEqual(DataType.f16.resolvePeerType(.f16), .f16); - try std.testing.expectEqual(DataType.f32.resolvePeerType(.f32), .f32); - - try std.testing.expectEqual(DataType.f16.resolvePeerType(.f32), .f32); - try std.testing.expectEqual(DataType.f32.resolvePeerType(.f16), .f32); - - try std.testing.expectEqual(DataType.f32.resolvePeerType(.f64), .f64); - try std.testing.expectEqual(DataType.f64.resolvePeerType(.f32), .f64); - try std.testing.expectEqual(DataType.f32.resolvePeerType(.i32), null); - - try std.testing.expectEqual(DataType.c64.resolvePeerType(.c128), .c128); - try std.testing.expectEqual(DataType.c128.resolvePeerType(.i32), null); - try std.testing.expectEqual(DataType.c64.resolvePeerType(.f32), null); - } - - pub fn zero(dtype: DataType) Value { - return .init(dtype, 0); - } - - pub fn one(dtype: DataType) Value { - return .init(dtype, 1); - } - - pub fn minValue(dtype: DataType) Value { - return switch (dtype) { - .bool => .{ .bool = false }, - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz, .f8e8m0, .f4e2m1 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).min), - inline .f8e5m2, .f8e3m4, .f8e4m3, .bf16 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).minus_inf), - inline .f16, .f32, .f64 => |tag| @unionInit(Value, @tagName(tag), -std.math.inf(@FieldType(Value, @tagName(tag)))), - inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Value, @tagName(tag), std.math.minInt(@FieldType(Value, @tagName(tag)))), - inline .c64, .c128 => |tag| @panic("DataType doesn't have a min value: " ++ @tagName(tag)), - }; - } - - pub fn maxValue(dtype: DataType) Value { - return switch (dtype) { - .bool => .{ .bool = true }, - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz, .f8e8m0, .f4e2m1 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).max), - inline .f8e5m2, .f8e3m4, .f8e4m3, .bf16 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).inf), - inline .f16, .f32, .f64 => |tag| @unionInit(Value, @tagName(tag), std.math.inf(@FieldType(Value, @tagName(tag)))), - inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Value, @tagName(tag), std.math.maxInt(@FieldType(Value, @tagName(tag)))), - inline .c64, .c128 => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), - }; - } - - pub fn constant(dtype: DataType, value: anytype) Value { - return .init(dtype, value); - } - - pub const Value = union(DataType) { - bool: bool, - f4e2m1: floats.Float4E2M1, - f8e3m4: floats.Float8E3M4, - f8e4m3: floats.Float8E4M3, - f8e4m3b11fnuz: floats.Float8E4M3B11FNUZ, - f8e4m3fn: floats.Float8E4M3FN, - f8e4m3fnuz: floats.Float8E4M3FNUZ, - f8e5m2: floats.Float8E5M2, - f8e5m2fnuz: floats.Float8E5M2FNUZ, - f8e8m0: floats.Float8E8M0, - bf16: floats.BFloat16, - f16: f16, - f32: f32, - f64: f64, - i2: i2, - i4: i4, - i8: i8, - i16: i16, - i32: i32, - i64: i64, - u2: u2, - u4: u4, - u8: u8, - u16: u16, - u32: u32, - u64: u64, - c64: C64, - c128: C128, - - /// Creates `Data` from a `value`. - /// - /// If the `dtype` and `@TypeOf(value)` are incompatible - /// or a cast from `value` to `FieldType(dtype)` would - /// be lossy, a panic occurs. - pub fn init(dtype_: DataType, value: anytype) Value { - const T = @TypeOf(value); - const Ti = @typeInfo(T); - - return switch (dtype_) { - .bool => switch (Ti) { - .bool => .{ .bool = value }, - .comptime_int, .int, .comptime_float, .float => .{ .bool = value != 0 }, - else => @panic("Could not create Value of type bool from value of type " ++ @typeName(T)), - }, - inline .f4e2m1, .f8e3m4, .f8e4m3, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .f8e8m0, .bf16 => |tag| switch (Ti) { - .comptime_int, .int => @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).fromF32(@floatFromInt(value))), - .comptime_float, .float => @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).fromF32(@floatCast(value))), - else => @panic("Could not create Value of type bf16 from value of type " ++ @typeName(T)), - }, - inline .f16, .f32, .f64 => |tag| switch (Ti) { - .comptime_int, .int => @unionInit(Value, @tagName(tag), @floatFromInt(value)), - .comptime_float, .float => @unionInit(Value, @tagName(tag), @floatCast(value)), - else => @panic("Could not create Value of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), - }, - inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| switch (Ti) { - .comptime_int => blk: { - const OutT = @FieldType(Value, @tagName(tag)); - if (value >= std.math.minInt(OutT) and value <= std.math.maxInt(OutT)) { - break :blk @unionInit(Value, @tagName(tag), @intCast(value)); - } else { - @panic("Could not create Value of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)); - } - }, - .int => @unionInit(Value, @tagName(tag), @intCast(value)), - else => @panic("Could not create Value of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), - }, - .c64 => switch (T) { - C64 => .{ .c64 = value }, - C128 => .{ .c64 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, - else => @panic("Could not create Value of type c64 from value of type " ++ @typeName(T)), - }, - .c128 => switch (T) { - C64 => .{ .c128 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, - C128 => .{ .c128 = value }, - else => @panic("Could not create Value of type c128 from value of type " ++ @typeName(T)), - }, - }; - } - - test init { - try std.testing.expectEqual(20.0, Value.init(.f16, 20).f16); - try std.testing.expectEqual(20.5, Value.init(.f16, 20.5).f16); - try std.testing.expectEqual(20, Value.init(.f16, @as(u8, 20)).f16); - try std.testing.expectEqual(-20, Value.init(.f16, @as(i8, -20)).f16); - try std.testing.expectEqual(2000.5, Value.init(.f16, @as(f32, 2000.5)).f16); - - try std.testing.expectEqual(true, Value.init(.bool, true).bool); - - try std.testing.expectEqual(10, Value.init(.u8, 10).u8); - try std.testing.expectEqual(10, Value.init(.u8, @as(u16, 10)).u8); - - try std.testing.expectEqual(10, Value.init(.i8, 10).i8); - try std.testing.expectEqual(10, Value.init(.i8, @as(u16, 10)).i8); - try std.testing.expectEqual(-10, Value.init(.i8, -10).i8); - try std.testing.expectEqual(-10, Value.init(.i8, @as(i16, -10)).i8); - - try std.testing.expectEqual(C64.init(1, 2), Value.init(.c64, C64.init(1, 2)).c64); - try std.testing.expectEqual(C64.init(1, 2), Value.init(.c64, C128.init(1, 2)).c64); - try std.testing.expectEqual(C128.init(1, 2), Value.init(.c128, C128.init(1, 2)).c128); - try std.testing.expectEqual(C128.init(1, 2), Value.init(.c128, C64.init(1, 2)).c128); - } - - pub fn dtype(self: Value) DataType { - return std.meta.activeTag(self); - } - - pub fn asBytes(data: *const Value) []const u8 { - return switch (data.*) { - inline else => |*value| std.mem.asBytes(value), - }; - } - - pub fn as(self: Value, comptime T: type) T { - // TODO allow more lossless conversions - switch (@typeInfo(T)) { - .bool => return self.bool, - .float => switch (self) { - inline .f16, .f32, .f64 => |v| return @floatCast(v), - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16 => |v| return @floatCast(v.toF32()), - else => {}, - }, - .int => switch (self) { - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |v| return @intCast(v), - else => {}, - }, - else => {}, - } - std.debug.panic("Unsupported conversion {} -> {s}", .{ self.dtype(), @typeName(T) }); - } - }; -}; - -pub fn mantissaSize(dtype: DataType) usize { - return switch (dtype) { - .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz => 3, - .f8e5m2, .f8e5m2fnuz => 2, - .f16 => 10, - .bf16 => 7, - .f32 => 23, - .f64 => 52, - else => @panic("Can't get mantissa size for a non-float dtype"), - }; -} - -pub const Data = DataType.Value; diff --git a/zml2/exe.zig b/zml2/exe.zig deleted file mode 100644 index 122f99d8..00000000 --- a/zml2/exe.zig +++ /dev/null @@ -1,240 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -const Buffer = @import("buffer.zig").Buffer; -const meta = @import("meta.zig"); -const pjrt = @import("pjrtx.zig"); -const Platform = @import("platform.zig").Platform; -const Shape = @import("shape.zig").Shape; - -pub const Exe = struct { - platform: Platform, - exe: *pjrt.LoadedExecutable, - - context: ?*pjrt.ExecuteContext = null, - - input_shapes: []const Shape, - output_shapes: []const Shape, - - num_devices: u8, - - arena: std.heap.ArenaAllocator, - - pub fn init( - allocator: std.mem.Allocator, - platform: Platform, - exe: *pjrt.LoadedExecutable, - input_shapes: []const Shape, - output_shapes: []const Shape, - num_devices: u8, - ) !Exe { - var arena = std.heap.ArenaAllocator.init(allocator); - errdefer arena.deinit(); - - const input_shapes_copy = try arena.allocator().dupe(Shape, input_shapes); - const output_shapes_copy = try arena.allocator().dupe(Shape, output_shapes); - - return .{ - .platform = platform, - .exe = exe, - .input_shapes = input_shapes_copy, - .output_shapes = output_shapes_copy, - .num_devices = num_devices, - .arena = arena, - }; - } - - pub fn deinit(self: *const Exe) void { - self.arena.deinit(); - } - - pub fn args(self: *const Exe, allocator: std.mem.Allocator) !Arguments { - return Arguments.init(allocator, self.input_shapes, self.num_devices); - } - - pub fn results(self: *const Exe, allocator: std.mem.Allocator) !Results { - return Results.init(allocator, self.output_shapes, self.num_devices, self.platform); - } - - pub const FlatBuffers = struct { - buffers: []const [*]*pjrt.Buffer, - raw_buffers: []const *pjrt.Buffer, - - num_devices: usize, - - pub fn init(allocator: std.mem.Allocator, count: usize, num_devices: usize) !FlatBuffers { - const raw_buffers = try allocator.alloc(*pjrt.Buffer, num_devices * count); - errdefer allocator.free(raw_buffers); - - const buffers = try allocator.alloc([*]*pjrt.Buffer, num_devices); - errdefer allocator.free(buffers); - - for (0..num_devices) |i| { - buffers[i] = raw_buffers[i * count ..].ptr; - } - - return .{ - .buffers = buffers, - .raw_buffers = raw_buffers, - .num_devices = num_devices, - }; - } - - pub fn deinit(self: *const FlatBuffers, allocator: std.mem.Allocator) void { - allocator.free(self.buffers); - allocator.free(self.raw_buffers); - } - }; - - pub const Arguments = struct { - flat_buffers: FlatBuffers, - expected_shapes: []const Shape, - - pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize) !Arguments { - const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); - errdefer flat_buffers.deinit(allocator); - - const expected_shapes = try allocator.dupe(Shape, shapes); - errdefer allocator.free(expected_shapes); - - return .{ - .flat_buffers = flat_buffers, - .expected_shapes = expected_shapes, - }; - } - - pub fn deinit(self: *const Arguments, allocator: std.mem.Allocator) void { - allocator.free(self.expected_shapes); - self.flat_buffers.deinit(allocator); - } - - pub fn set(self: *Arguments, v: anytype) void { - return self.setPartial(v, 0); - } - - pub fn setPartial(self: *Arguments, v: anytype, offset: usize) void { - const LocalContext = struct { - self: *Arguments, - current_index: usize = 0, - }; - var context: LocalContext = .{ .self = self, .current_index = offset }; - meta.visit(struct { - fn cb(context_: *LocalContext, buffer: *const Buffer) void { - stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected argument {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); - for (0..context_.self.flat_buffers.num_devices) |device_index| { - context_.self.flat_buffers.buffers[device_index][context_.current_index] = buffer._shards.get(device_index); - } - - context_.current_index += 1; - } - }.cb, &context, &v); - } - }; - - pub const Results = struct { - platform: Platform, - flat_buffers: FlatBuffers, - - expected_shapes: []const Shape, - - pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize, platform: Platform) !Results { - const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); - errdefer flat_buffers.deinit(allocator); - - const expected_shapes = try allocator.dupe(Shape, shapes); - errdefer allocator.free(expected_shapes); - - return .{ - .platform = platform, - .flat_buffers = flat_buffers, - .expected_shapes = expected_shapes, - }; - } - - pub fn deinit(self: *const Results, allocator: std.mem.Allocator) void { - allocator.free(self.expected_shapes); - self.flat_buffers.deinit(allocator); - } - - pub fn get(self: *Results, comptime T: type) T { - var result: T = undefined; - const LocalContext = struct { - self: *Results, - current_index: usize = 0, - }; - var context: LocalContext = .{ .self = self, .current_index = 0 }; - meta.visit(struct { - fn cb(context_: *LocalContext, buffer: *Buffer) void { - var shards: Buffer.Shards = .{}; - for (0..context_.self.flat_buffers.num_devices) |device_index| { - shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); - } - buffer.* = Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], shards.constSlice()); - context_.current_index += 1; - } - }.cb, &context, &result); - return result; - } - - pub fn fill(self: *Results, v: anytype) void { - const LocalContext = struct { - self: *Results, - current_index: usize = 0, - }; - var context: LocalContext = .{ .self = self, .current_index = 0 }; - meta.visit(struct { - fn cb(context_: *LocalContext, buffer: *Buffer) void { - //stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected result {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); - var shards: Buffer.Shards = .{}; - for (0..context_.self.flat_buffers.num_devices) |device_index| { - shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); - } - buffer.* = Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], shards.constSlice()); - context_.current_index += 1; - } - }.cb, &context, &v); - } - }; - - pub fn internalCall(self: *const Exe, io: ?std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { - stdx.debug.assert(opts.wait == false or io != null, "io should not be null when waiting for execution completion", .{}); - var events = [_]?*pjrt.Event{null} ** Platform.MAX_NUM_DEVICES; - const sharding = self.platform.sharding(); - const events_slice: ?[]?*pjrt.Event = if (opts.wait) events[0..sharding.num_partitions] else null; - - self.exe.execute(self.platform.pjrt_api, .{ - .arguments = arguments.flat_buffers.buffers, - .num_args = arguments.expected_shapes.len, - .results = results_.flat_buffers.buffers, - .events = events_slice, - // this allows to tell a specific buffer shouldn't be donated, - // even if it has been marked as "can be donated" during compilation. - // TODO: expose it ? - .non_donatable_input_indices = &.{}, - .context = self.context, - }) catch |err| { - std.debug.panic("PJRT_LoadedExecutable_Execute failed with: {}", .{err}); - }; - - if (opts.wait) { - for (events_slice.?) |e| { - if (e) |ev| { - ev.await(self.platform.pjrt_api, io.?) catch unreachable; - } - } - } - } - - pub const CallOpts = struct { - wait: bool = false, - }; - - pub fn callOpts(self: *const Exe, io: std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { - return self.internalCall(io, arguments, results_, opts); - } - - pub fn call(self: *const Exe, arguments: Arguments, results_: *Results) void { - return self.internalCall(null, arguments, results_, .{}); - } -}; diff --git a/zml2/floats.zig b/zml2/floats.zig deleted file mode 100644 index 538722cd..00000000 --- a/zml2/floats.zig +++ /dev/null @@ -1,556 +0,0 @@ -///! Conversion utilities between different Floating point formats. -const std = @import("std"); - -fn FloatConversionHelpers(Float: type) type { - const info = @typeInfo(Float); - const err_msg = "FloatHelpers expect a packed struct { mantissa: uXX, exponent: uXX, sign: u1}"; - if (info != .@"struct" or info.@"struct".backing_integer == null) { - @compileError(err_msg); - } - comptime { - for (info.@"struct".fields, &.{ "mantissa", "exponent", "sign" }) |field, expected_name| { - if (!std.mem.eql(u8, field.name, expected_name)) - @compileError(err_msg); - } - } - - const zero = FloatHelpers(Float).zero; - - return struct { - const mantissa_bits: u8 = @typeInfo(@FieldType(Float, "mantissa")).int.bits; - const exponent_bits: u8 = @typeInfo(@FieldType(Float, "exponent")).int.bits; - const f32_mantissa_bits: u8 = @typeInfo(@FieldType(Float32, "mantissa")).int.bits; - const exp_bias: i16 = std.math.maxInt(std.meta.Int(.unsigned, exponent_bits - 1)); - const exp_off: u8 = FloatConversionHelpers(Float32).exp_bias - exp_bias; - - /// Lossy conversion from f32, similar to @floatCast - pub fn fromF32(f: f32) Float { - @setRuntimeSafety(false); - - const vf32: Float32 = @bitCast(f); - const exponent: i16 = @as(i16, vf32.exponent) - exp_off; - const overflow = exponent > std.math.maxInt(@FieldType(Float, "exponent")); - if (overflow) { - @branchHint(.unlikely); - return if (@hasDecl(Float, "inf")) - if (vf32.sign == 0) Float.inf else Float.minus_inf - else if (@hasDecl(Float, "nan")) - Float.nan - else b: { - break :b if (vf32.sign == 0) Float.max else Float.min; - }; - } - - return if (exponent <= 0) - .{ - .sign = vf32.sign, - .exponent = 0, - .mantissa = shiftMantissa(vf32.mantissa, @intCast(-exponent)), - } - else - .{ - .sign = vf32.sign, - .exponent = @intCast(exponent), - .mantissa = truncMantissa(vf32.mantissa), - }; - } - - /// Lossless conversion to f32. - pub fn toF32(x: Float) f32 { - @setRuntimeSafety(false); - - if (x == zero) return 0.0; - if (isInf(x)) { - @branchHint(.unlikely); - return if (x.sign == 0) std.math.inf(f32) else -std.math.inf(f32); - } - - const vf32: Float32 = if (x.exponent > 0) - .{ - .sign = x.sign, - .exponent = @as(u8, x.exponent) + exp_off, - .mantissa = f32Mantissa(x), - } - else - .{ - .sign = x.sign, - .exponent = exp_off - @clz(x.mantissa), - .mantissa = @as(u23, x.mantissa) << @clz(x.mantissa), - }; - return @bitCast(vf32); - } - - fn truncMantissa(f32_mantissa: u32) @FieldType(Float, "mantissa") { - const rounding_val: u32 = @as(u32, 1) << (f32_mantissa_bits - mantissa_bits - 1); - return @truncate((f32_mantissa + rounding_val) >> (f32_mantissa_bits - mantissa_bits)); - } - - fn shiftMantissa(f32_mantissa: u32, underflow: u8) @FieldType(Float, "mantissa") { - const upper_bit: u32 = @as(u32, 1) << f32_mantissa_bits; - const full_mant32: u32 = f32_mantissa | upper_bit; - // divide the mantissa proportionally to the exponent underflow - const shifted_mant: u32 = full_mant32 >> @truncate(underflow + 1); - return truncMantissa(shifted_mant); - } - - fn f32Mantissa(x: Float) @FieldType(Float32, "mantissa") { - const T = @FieldType(Float32, "mantissa"); - return @as(T, x.mantissa) << f32_mantissa_bits - mantissa_bits; - } - }; -} - -fn FloatHelpers(Float: type) type { - const info = @typeInfo(Float); - const err_msg = "FloatHelpers expect a packed struct { mantissa: uXX, exponent: uXX, sign: u1}"; - if (info != .@"struct" or info.@"struct".backing_integer == null) { - @compileError(err_msg); - } - comptime { - for (info.@"struct".fields, &.{ "mantissa", "exponent", "sign" }) |field, expected_name| { - if (!std.mem.eql(u8, field.name, expected_name)) - @compileError(err_msg); - } - } - - return struct { - pub const zero: Float = .{ .sign = 0, .exponent = 0, .mantissa = 0 }; - - pub fn neg(x: Float) Float { - return .{ - .sign = x.sign ^ 1, - .exponent = x.exponent, - .mantissa = x.mantissa, - }; - } - - pub fn formatNumber(x: Float, writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { - switch (n.mode) { - .binary, .octal, .hex => try writer.print("{{ .sign={}, .exp={}, .mantissa={} }}", .{ x.sign, x.exponent, x.mantissa }), - else => try writer.printFloat(x.toF32(), n), - } - } - }; -} - -pub const Float32 = packed struct(u32) { - mantissa: u23, - exponent: u8, - sign: u1, - - pub const nan: Float32 = .{ .sign = 0, .exponent = std.math.maxInt(u8), .mantissa = 1 }; - pub const max: Float32 = .{ .sign = 0, .exponent = std.math.maxInt(u8) - 1, .mantissa = std.math.maxInt(u23) }; - pub const min: Float32 = neg(max); - pub const inf: Float32 = .{ .sign = 0, .exponent = std.math.maxInt(u8), .mantissa = 0 }; - pub const minus_inf = neg(inf); - - pub fn isNan(self: Float32) bool { - return self.exponent == nan.exponent and self.mantissa != 0; - } - - const Helpers = FloatHelpers(@This()); - pub const zero = Helpers.zero; - pub const neg = Helpers.neg; - - pub fn fromF32(f: f32) Float32 { - return @bitCast(f); - } - - pub fn toF32(x: Float32) f32 { - return @bitCast(x); - } - - pub const formatNumber = Helpers.formatNumber; -}; - -const f32_exp_bias = FloatHelpers(Float32).expBias(); - -pub const Float64 = packed struct(u64) { - mantissa: u52, - exponent: u11, - sign: u1, - - pub const nan: Float64 = .{ .sign = 0, .exponent = std.math.maxInt(u11), .mantissa = 1 }; - pub const max: Float64 = .{ .sign = 0, .exponent = std.math.maxInt(u11) - 1, .mantissa = std.math.maxInt(u32) }; - pub const min: Float64 = neg(max); - pub const inf: Float64 = .{ .sign = 0, .exponent = std.math.maxInt(u11), .mantissa = 0 }; - pub const minus_inf = neg(inf); - - pub fn isNan(self: Float64) bool { - return self.exponent == nan.exponent and self.mantissa != 0; - } - - const Helpers = FloatHelpers(@This()); - pub const zero = Helpers.zero; - pub const neg = Helpers.neg; - pub fn fromF32(f: f32) Float64 { - return @bitCast(@as(f64, @floatCast(f))); - } - - pub fn toF32(x: Float64) f32 { - return @floatCast(@as(f64, @bitCast(x))); - } - - pub const formatNumber = Helpers.formatNumber; -}; - -pub const Float8E4M3B11FNUZ = packed struct(u8) { - mantissa: u3, - exponent: u4, - sign: u1, - - pub const nan: Float8E4M3B11FNUZ = .{ .sign = 1, .exponent = 0, .mantissa = 0 }; - pub const max: Float8E4M3B11FNUZ = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = std.math.maxInt(u3) }; - pub const min: Float8E4M3B11FNUZ = neg(max); - - pub fn isNan(self: Float8E4M3B11FNUZ) bool { - return self == nan; - } - - const Helpers = FloatHelpers(@This()); - pub const zero = Helpers.zero; - pub const neg = Helpers.neg; - pub const fromF32 = FloatConversionHelpers(@This()).fromF32; - pub const toF32 = FloatConversionHelpers(@This()).toF32; - pub const formatNumber = Helpers.formatNumber; -}; - -pub const Float8E4M3FN = packed struct(u8) { - mantissa: u3, - exponent: u4, - sign: u1, - - pub const nan: Float8E4M3FN = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = std.math.maxInt(u3) }; - pub const max: Float8E4M3FN = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = 0b110 }; - pub const min: Float8E4M3FN = neg(max); - - pub fn isNan(self: Float8E4M3FN) bool { - return self.exponent == nan.exponent and self.mantissa == nan.mantissa; - } - - const Helpers = FloatHelpers(@This()); - pub const zero = Helpers.zero; - pub const neg = Helpers.neg; - pub const fromF32 = FloatConversionHelpers(@This()).fromF32; - pub const toF32 = FloatConversionHelpers(@This()).toF32; - pub const formatNumber = Helpers.formatNumber; -}; - -pub const Float8E4M3FNUZ = packed struct(u8) { - mantissa: u3, - exponent: u4, - sign: u1, - - pub const nan: Float8E4M3FNUZ = .{ .sign = 1, .exponent = 0, .mantissa = 0 }; - pub const max: Float8E4M3FNUZ = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = std.math.maxInt(u3) }; - pub const min: Float8E4M3FNUZ = neg(max); - - pub fn isNan(self: Float8E4M3FNUZ) bool { - return self == nan; - } - - const Helpers = FloatHelpers(@This()); - pub const zero = Helpers.zero; - pub const neg = Helpers.neg; - pub const fromF32 = FloatConversionHelpers(@This()).fromF32; - pub const toF32 = FloatConversionHelpers(@This()).toF32; - pub const formatNumber = Helpers.formatNumber; -}; - -test "Float8E4" { - // With 4 bits of exponents power of two can be represented exactly up to 64. - const test_case_e4: TestCase = .{ - .lossless = &[_]f32{ 0, 1.0, -2, 1.0 / 64.0, -128, -1.125 / 64.0 }, - .lossy = &[_]f32{ 3.02344107628, 1.0 / 128.0, 1.0 / 512.0 }, - }; - - inline for (.{ - Float8E4M3B11FNUZ, - Float8E4M3FN, - Float8E4M3FNUZ, - }) |Float8T| { - try testCustomFloat(Float8T, test_case_e4); - try std.testing.expectEqual(0.0, Float8T.fromF32(1.0 / 2048.0).toF32()); - if (@hasDecl(Float8T, "inf")) { - try std.testing.expectEqual(Float8T.inf, Float8T.fromF32(128.0)); - try std.testing.expectEqual(Float8T.inf.neg(), Float8T.fromF32(-128.0)); - } - } -} - -pub const Float8E5M2 = packed struct(u8) { - mantissa: u2, - exponent: u5, - sign: u1, - - pub const nan: Float8E5M2 = .{ .sign = 0, .exponent = std.math.maxInt(u5), .mantissa = 0b01 }; - pub const max: Float8E5M2 = .{ .sign = 0, .exponent = 0b11110, .mantissa = std.math.maxInt(u2) }; - pub const min: Float8E5M2 = neg(max); - pub const inf: Float8E5M2 = .{ .sign = 0, .exponent = std.math.maxInt(u5), .mantissa = 0 }; - pub const minus_inf: Float8E5M2 = .neg(inf); - - pub fn isNan(self: Float8E5M2) bool { - return self.exponent == nan.exponent and self.mantissa != 0; - } - - const Helpers = FloatHelpers(@This()); - pub const zero = Helpers.zero; - pub const neg = Helpers.neg; - pub const fromF32 = FloatConversionHelpers(@This()).fromF32; - pub const toF32 = FloatConversionHelpers(@This()).toF32; - pub const formatNumber = Helpers.formatNumber; -}; - -pub const Float8E5M2FNUZ = packed struct(u8) { - mantissa: u2, - exponent: u5, - sign: u1, - - pub const nan: Float8E5M2FNUZ = .{ .sign = 1, .exponent = 0, .mantissa = 0 }; - pub const max: Float8E5M2FNUZ = .{ .sign = 0, .exponent = std.math.maxInt(u5), .mantissa = std.math.maxInt(u2) }; - pub const min: Float8E5M2FNUZ = neg(max); - - pub fn isNan(self: Float8E5M2FNUZ) bool { - return self == nan; - } - - const Helpers = FloatHelpers(@This()); - pub const zero = Helpers.zero; - pub const neg = Helpers.neg; - pub const fromF32 = FloatConversionHelpers(@This()).fromF32; - pub const toF32 = FloatConversionHelpers(@This()).toF32; - pub const formatNumber = Helpers.formatNumber; -}; - -test "Float8E5" { - const test_case_e5: TestCase = .{ - .lossless = &[_]f32{ 0, 1.0, -2, 1.0 / 128.0, -128 }, - .lossy = &[_]f32{3.02344107628}, - }; - inline for (.{ Float8E5M2, Float8E5M2FNUZ }) |Float8T| { - try testCustomFloat(Float8T, test_case_e5); - } -} - -pub const BFloat16 = packed struct(u16) { - mantissa: u7, - exponent: u8, - sign: u1, - - pub const nan: BFloat16 = .{ .sign = 0, .exponent = std.math.maxInt(u8), .mantissa = 1 }; - pub const max: BFloat16 = .{ .sign = 0, .exponent = 0b11111110, .mantissa = std.math.maxInt(u7) }; - pub const min: BFloat16 = neg(max); - pub const inf: BFloat16 = .{ .sign = 0, .exponent = std.math.maxInt(u8), .mantissa = 0 }; - pub const minus_inf: BFloat16 = .neg(inf); - - pub fn isNan(self: BFloat16) bool { - return self.exponent == nan.exponent and self.mantissa != 0; - } - - // Specialized versions of to/from F32. Since BFloat16 has the same exponent range than F32, - // no overflow/underflow can happen, simplifiying conversion logic. - pub fn toF32(self: BFloat16) f32 { - // Pad the BF16 with zeros 0 - return @bitCast([2]u16{ 0, @bitCast(self) }); - } - - pub fn fromF32(float32: f32) BFloat16 { - var int: u32 = @bitCast(float32); - // Round up if needed. - int += 0x8000; - const parts: [2]u16 = @bitCast(int); - return @bitCast(parts[1]); - } - - const Helpers = FloatHelpers(@This()); - pub const zero = Helpers.zero; - pub const neg = Helpers.neg; - pub const formatNumber = Helpers.formatNumber; -}; - -test BFloat16 { - // From https://en.wikipedia.org/wiki/Bfloat16_floating-point_format#Examples - try std.testing.expectEqual(BFloat16.fromF32(0), BFloat16{ .sign = 0, .exponent = 0, .mantissa = 0 }); - try std.testing.expectEqual(BFloat16.fromF32(-2), BFloat16{ .sign = 1, .exponent = 127 + 1, .mantissa = 0 }); - try std.testing.expectEqual(BFloat16.fromF32(3.02344107628), BFloat16{ .sign = 0, .exponent = 127 + 1, .mantissa = 66 }); - try std.testing.expectEqual(BFloat16.fromF32(1.0 / 128.0), BFloat16{ .sign = 0, .exponent = 127 - 7, .mantissa = 0 }); - try std.testing.expectEqual(std.mem.toBytes(BFloat16.inf.neg()), [_]u8{ 0x80, 0xff }); - try std.testing.expectEqual(BFloat16.inf, BFloat16.fromF32(std.math.inf(f32))); - - try testCustomFloat(BFloat16, .{ - .lossless = &[_]f32{ 0, -2, 1.0 / 128.0, -1e64, std.math.inf(f32) }, - .lossy = &[_]f32{3.02344107628}, - }); -} - -pub const Float8E4M3 = packed struct(u8) { - mantissa: u3, - exponent: u4, - sign: u1, - - pub const nan: Float8E4M3 = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = 1 }; - pub const max: Float8E4M3 = .{ .sign = 0, .exponent = 0b1110, .mantissa = std.math.maxInt(u3) }; - pub const min: Float8E4M3 = neg(max); - pub const inf: Float8E4M3 = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = 0 }; - pub const minus_inf = neg(inf); - - pub fn isNan(self: Float8E4M3) bool { - return self.exponent == nan.exponent and self.mantissa != 0; - } - - const Helpers = FloatHelpers(@This()); - pub const zero = Helpers.zero; - pub const neg = Helpers.neg; - pub const fromF32 = FloatConversionHelpers(@This()).fromF32; - pub const toF32 = FloatConversionHelpers(@This()).toF32; - pub const formatNumber = Helpers.formatNumber; -}; - -pub const Float8E3M4 = packed struct(u8) { - mantissa: u4, - exponent: u3, - sign: u1, - - pub const nan: Float8E3M4 = .{ .sign = 0, .exponent = std.math.maxInt(u3), .mantissa = 1 }; - pub const max: Float8E3M4 = .{ .sign = 0, .exponent = 0b110, .mantissa = std.math.maxInt(u4) }; - pub const min: Float8E3M4 = neg(max); - pub const inf: Float8E3M4 = .{ .sign = 0, .exponent = std.math.maxInt(u3), .mantissa = 0 }; - pub const minus_inf = neg(inf); - - pub fn isNan(self: Float8E3M4) bool { - return self.exponent == nan.exponent and self.mantissa != 0; - } - - const Helpers = FloatHelpers(@This()); - pub const zero = Helpers.zero; - pub const neg = Helpers.neg; - pub const fromF32 = FloatConversionHelpers(@This()).fromF32; - pub const toF32 = FloatConversionHelpers(@This()).toF32; - pub const formatNumber = Helpers.formatNumber; -}; - -pub const Float8E8M0 = packed struct(u8) { - mantissa: u0 = 0, - exponent: u8, - sign: u0 = 0, - - pub const max: Float8E8M0 = .{ .exponent = std.math.maxInt(u8) }; - pub const min: Float8E8M0 = .{ .exponent = 0 }; - pub const min_scale: f32 = @bitCast(Float32{ .sign = 0, .exponent = 0, .mantissa = 0b1 << 22 }); - - /// Lossy conversion from f32, similar to @floatCast - pub fn fromF32(f: f32) Float8E8M0 { - const vf32: Float32 = @bitCast(f); - return .{ .exponent = @intCast(vf32.exponent) }; - } - - /// Lossless conversion to f32. - pub fn toF32(x: Float8E8M0) f32 { - if (x.exponent == 0) return min_scale; - const vf32: Float32 = .{ - .sign = 0, - .exponent = x.exponent, - .mantissa = 0, - }; - return @bitCast(vf32); - } - - const Helpers = FloatHelpers(@This()); - pub const formatNumber = Helpers.formatNumber; -}; - -test Float8E8M0 { - try std.testing.expectEqual(Float8E8M0{ .exponent = 127 }, Float8E8M0.fromF32(1.0)); - // try std.testing.expectEqual(5.877472e-39, Float8E8M0.toF32(.{ .exponent = 0})); - - try testCustomFloat(Float8E8M0, .{ - .lossless = &[_]f32{ Float8E8M0.min_scale, 1.0, 64.0, 1.0 / 128.0, std.math.pow(f32, 2.0, 127) }, - .lossy = &[_]f32{1.00001}, - }); -} - -pub const Float4E2M1 = packed struct(u4) { - mantissa: u1, - exponent: u2, - sign: u1, - - pub const max: Float4E2M1 = .{ .sign = 0, .exponent = std.math.maxInt(u2), .mantissa = std.math.maxInt(u1) }; - pub const min: Float4E2M1 = neg(max); - const Helpers = FloatHelpers(@This()); - pub const zero = Helpers.zero; - pub const neg = Helpers.neg; - pub const fromF32 = FloatConversionHelpers(@This()).fromF32; - pub const formatNumber = Helpers.formatNumber; - - pub const values = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; - - pub fn toF32(x: Float4E2M1) f32 { - // faster implementation - return values[@as(u4, @bitCast(x))]; - } - - test toF32 { - var to_f32_res: [16]f32 = undefined; - for (&to_f32_res, 0..) |*r, i| { - const x_f4: Float4E2M1 = @bitCast(@as(u4, @intCast(i))); - r.* = x_f4.toF32(); - } - try std.testing.expectEqualSlices(f32, &Float4E2M1.values, &to_f32_res); - } - - test fromF32 { - var from_f32_res: [16]Float4E2M1 = undefined; - for (&from_f32_res, 0..) |*r, i| { - r.* = .fromF32(Float4E2M1.values[i]); - } - try std.testing.expectEqualSlices(u4, &.{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, @ptrCast(&from_f32_res)); - } -}; - -pub fn floatCast(T: type, x: anytype) T { - return switch (T) { - f64, f32, f16 => switch (@TypeOf(x)) { - f64, f32, f16 => @floatCast(x), - else => @floatCast(x.toF32()), - }, - else => switch (@TypeOf(x)) { - f64, f32, f16 => .fromF32(x), - else => .fromF32(x.toF32()), - }, - }; -} - -pub fn isInf(x: anytype) bool { - const Float = @TypeOf(x); - switch (Float) { - f64, f32, f16 => return std.math.isInf(x), - else => {}, - } - if (!@hasDecl(Float, "inf")) return false; - - const FBits = std.meta.Int(.unsigned, @bitSizeOf(Float)); - const remove_sign = ~@as(FBits, 0) >> 1; - return @as(FBits, @bitCast(x)) & remove_sign == @as(FBits, @bitCast(Float.inf)); -} - -const TestCase = struct { - lossless: []const f32, - lossy: []const f32, - tolerance: f32 = 1e-2, -}; - -fn testCustomFloat(FloatT: type, test_case: TestCase) !void { - for (test_case.lossless) |x| { - try std.testing.expectEqual(x, FloatT.fromF32(x).toF32()); - } - for (test_case.lossy) |x| { - try expectApproxEqRel(f32, x, FloatT.fromF32(x).toF32(), test_case.tolerance); - } -} - -fn expectApproxEqRel(FloatT: type, x: FloatT, y: FloatT, tolerance: FloatT) !void { - if (!std.math.approxEqRel(f32, x, y, tolerance)) { - std.log.err("expected ~{d}, got {d}", .{ x, y }); - return error.TestUnexpectedResult; - } -} diff --git a/zml2/main.zig b/zml2/main.zig deleted file mode 100644 index f62c98da..00000000 --- a/zml2/main.zig +++ /dev/null @@ -1,111 +0,0 @@ -const std = @import("std"); -const zml = @import("zml"); - -pub const Model = struct { - weight: zml.Tensor, - bias: zml.Tensor, - - pub fn init() !Model { - return .{ - .weight = .init(.{ .m = 4096, .n = 4096 }, .f32), - .bias = .init(.{ .m = 4096 }, .f32), - }; - } - - pub fn deinit(self: *Model) void { - _ = self; - } - - pub fn loadBuffers(model: Model, allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform) !zml.Bufferized(Model) { - const weight_slice: zml.Slice = try .alloc(allocator, model.weight.shape()); - defer weight_slice.free(allocator); - @memset(weight_slice.items(f32), 1); - - const bias_slice: zml.Slice = try .alloc(allocator, model.bias.shape()); - defer bias_slice.free(allocator); - @memset(bias_slice.items(f32), 2); - - const weight_buffer: zml.Buffer = try .fromSlice(io, platform, weight_slice); - errdefer weight_buffer.deinit(); - const bias_buffer: zml.Buffer = try .fromSlice(io, platform, bias_slice); - errdefer bias_buffer.deinit(); - - return .{ .weight = weight_buffer, .bias = bias_buffer }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Model)) void { - self.weight.deinit(); - self.bias.deinit(); - } - - pub fn forward(self: Model, input: zml.Tensor) zml.Tensor { - return self.weight.dot(input, .n).add(self.bias); - } -}; - -pub fn main() !void { - var gpa: std.heap.DebugAllocator(.{}) = .init; - defer _ = gpa.deinit(); - - const allocator = gpa.allocator(); - - var threaded: std.Io.Threaded = .init(allocator); - defer threaded.deinit(); - - var vfs_file: zml.io.VFS.File = .init(threaded.io()); - - var vfs: zml.io.VFS = .init(allocator, threaded.io()); - defer vfs.deinit(); - - try vfs.register("file", vfs_file.io()); - - const io = vfs.io(); - - const file = try std.Io.Dir.openFile(.cwd(), io, "/private/var/tmp/_bazel_corendos/c1731bd4fcb7bdb2687c1972ca03d70f/external/+non_module_deps+mnist/mnist.safetensors", .{ .mode = .read_only }); - defer file.close(io); - - zml.init(); - defer zml.deinit(); - - const available = zml.Platform.availablePlatforms(io); - std.log.info("Available platforms: {any}", .{available}); - - var platform = try zml.Platform.auto(io, .{}); - defer platform.deinit(); - - std.log.info("Selected platform: {f}", .{platform}); - - var model: Model = try .init(); - defer model.deinit(); - - const input: zml.Tensor = .init(.{ .n = 4096 }, .f32); - var exe = try platform.compileModel(allocator, io, Model.forward, model, .{input}); - defer exe.deinit(); - - const slice: zml.Slice = try .alloc(allocator, input.shape()); - defer slice.free(allocator); - @memset(slice.items(f32), 1); - - var model_buffers = try model.loadBuffers(allocator, io, platform); - defer Model.unloadBuffers(&model_buffers); - - const input_buffer: zml.Buffer = try .fromSlice(io, platform, slice); - defer input_buffer.deinit(); - - var args = try exe.args(allocator); - defer args.deinit(allocator); - - var results = try exe.results(allocator); - defer results.deinit(allocator); - - args.set(.{ model_buffers, input_buffer }); - exe.call(args, &results); - - const output = results.get(zml.Buffer); - defer output.deinit(); - - const output_slice = try output.toSliceAlloc(allocator, io); - defer output_slice.free(allocator); - - std.log.info("Output: {any}", .{output_slice.items(f32)[0..10]}); -} diff --git a/zml2/main_io.zig b/zml2/main_io.zig deleted file mode 100644 index f9a612df..00000000 --- a/zml2/main_io.zig +++ /dev/null @@ -1,537 +0,0 @@ -const builtin = @import("builtin"); -const std = @import("std"); - -const zml = @import("zml"); - -const log = std.log.scoped(.@"zml/main_io"); - -pub const std_options: std.Options = .{ - .log_level = .debug, - .log_scope_levels = &[_]std.log.ScopeLevel{ - .{ .scope = .@"zml/safetensors", .level = .debug }, - .{ .scope = .@"zml/io/vfs/file", .level = .info }, - .{ .scope = .@"zml/io/vfs/http", .level = .info }, - .{ .scope = .@"zml/io/vfs/hf", .level = .info }, - }, -}; - -pub fn main() !void { - var debug_allocator: std.heap.DebugAllocator(.{}) = .init; - defer std.debug.assert(debug_allocator.deinit() == .ok); - - const allocator = debug_allocator.allocator(); - - // const allocator = std.heap.smp_allocator; - - var threaded: std.Io.Threaded = .init(allocator); - defer threaded.deinit(); - - if (try std.process.hasEnvVar(allocator, "ASYNC_LIMIT")) { - const limit = try std.process.getEnvVarOwned(allocator, "ASYNC_LIMIT"); - defer allocator.free(limit); - - threaded.async_limit = .limited(try std.fmt.parseInt(usize, limit, 10)); - } - - log.debug("Running with threaded async limit: {?d}", .{threaded.async_limit.toInt()}); - - var http_client: std.http.Client = .{ - .allocator = allocator, - .io = threaded.io(), - .connection_pool = .{ - .free_size = threaded.async_limit.toInt() orelse 16, - }, - }; - - try http_client.initDefaultProxies(allocator); - defer http_client.deinit(); - - var vfs_file: zml.io.VFS.File = .init( - allocator, - threaded.io(), - .{ - .direct_io = true, - .direct_io_alignment = .fromByteUnits(4 * 1024), - }, - ); - defer vfs_file.deinit(); - - var vfs_http: zml.io.VFS.HTTP = try .init(allocator, threaded.io(), &http_client, .{}); - defer vfs_http.deinit(); - - var hf_auth: zml.io.VFS.HF.Auth = try .auto(allocator, threaded.io()); - defer hf_auth.deinit(allocator); - - var hf_vfs: zml.io.VFS.HF = try .init( - allocator, - threaded.io(), - &http_client, - .{ - .request_range_min = 8 * 1024, - .request_range_max = 128 * 1024 * 1024, - .hf_pagination_limit = 100, - .auth = hf_auth, - }, - ); - defer hf_vfs.deinit(); - - var vfs: zml.io.VFS = .init(allocator, threaded.io()); - defer vfs.deinit(); - - try vfs.register("file", vfs_file.io()); - try vfs.register("http", vfs_http.io()); - try vfs.register("https", vfs_http.io()); - try vfs.register("hf", hf_vfs.io()); - - const io = vfs.io(); - - const default_uri = "hf://Qwen/Qwen3-8B@b968826"; - // const default_uri = "file:///Users/hugo/Developer/Llama-3.1-8B-Instruct"; - // const default_uri = "https://storage.googleapis.com/zig-vfs/Llama-3.1-8B-Instruct/"; - // const default_uri = "http://9960x-5090x2:8003/"; - - var args = std.process.args(); - _ = args.next(); // skip program name - - const uri = if (args.next()) |uri| blk: { - log.info("Using repo URI from args: {s}", .{uri}); - break :blk uri; - } else blk: { - log.info("Using default repo URI: {s}", .{default_uri}); - break :blk default_uri; - }; - - { - const local_uri = "hf://openai/gpt-oss-20b@6cee5e8"; - const repo = try std.Io.Dir.openDir(.cwd(), io, local_uri, .{}); - defer repo.close(io); - - const index_file = try repo.openFile(io, "model.safetensors.index.json", .{}); - defer index_file.close(io); - - const stat = try index_file.stat(io); - log.info("Opened local index file with size: {d} bytes", .{stat.size}); - - const original = try repo.openDir(io, "original", .{}); - defer original.close(io); - - const original_config = try original.openFile(io, "config.json", .{}); - defer original_config.close(io); - - // dir access / stat - - const stat_config = try original_config.stat(io); - log.info("Opened local original/config.json file with size: {d} bytes", .{stat_config.size}); - } - - { - const local_uri = "hf://openai/gpt-oss-20b@6cee5e8"; - const repo = try std.Io.Dir.openDir(.cwd(), io, local_uri, .{}); - defer repo.close(io); - - const original_config = try repo.openFile(io, "original/config.json", .{}); - defer original_config.close(io); - - const stat_config = try original_config.stat(io); - log.info("Opened local original/config.json file with size: {d} bytes", .{stat_config.size}); - } - - { - const local_uri = "hf://openai/gpt-oss-20b@6cee5e8"; - const original_uri = try std.fmt.allocPrint(allocator, "{s}/original", .{local_uri}); - defer allocator.free(original_uri); - - const repo = try std.Io.Dir.openDir(.cwd(), io, original_uri, .{}); - defer repo.close(io); - - const original_config = try repo.openFile(io, "config.json", .{}); - defer original_config.close(io); - - const stat_config = try original_config.stat(io); - log.info("Opened local original/config.json file with size: {d} bytes", .{stat_config.size}); - } - - // { - // const repo = try std.Io.Dir.openDir(.cwd(), io, "file:///home/hugo/Llama-3.1-8B-Instruct/", .{}); - // defer repo.close(io); - - // const file = try repo.openFile(io, "model-00001-of-00004.safetensors", .{}); - // defer file.close(io); - - // const buf_size = 256 * 1024 * 1024; - // const buf = try allocator.alignedAlloc(u8, .fromByteUnits(4 * 1024), buf_size); - // defer allocator.free(buf); - - // var sha256: std.crypto.hash.sha2.Sha256 = .init(.{}); - // const compute_sha = false; - - // var total_read: usize = 1 * 1024; - // var bufs = [_][]u8{buf}; - - // var timer: std.time.Timer = try .start(); - // defer { - // const elapsed = timer.read(); - // log.info("File read in {d} ms", .{elapsed / std.time.ns_per_ms}); - // } - - // while (true) { - // const n = try io.vtable.fileReadStreaming(io.userdata, file, &bufs); - // if (n == 0) break; - // if (compute_sha) sha256.update(buf[0..n]); - // total_read += n; - // } - - // const elapsed = timer.read(); - // const read_mb = @as(f64, @floatFromInt(total_read)) / (1024.0 * 1024.0); - // const read_time_s = @as(f64, @floatFromInt(elapsed)) / @as(f64, std.time.ns_per_s); - // const throughput_mb_s = if (read_time_s > 0) read_mb / read_time_s else 0; - // const throughput_gbps = if (read_time_s > 0) (read_mb * 8.0) / 1024.0 / read_time_s else 0; - // log.info("Read {d:.2} MB in {d:.2} s = {d:.2} MB/s | {d:.2} Gbps", .{ - // read_mb, - // read_time_s, - // throughput_mb_s, - // throughput_gbps, - // }); - - // if (compute_sha) { - // var hash: [32]u8 = undefined; - // sha256.final(&hash); - // log.info("SHA256: {x}", .{hash}); - // } - // } - - { - var pool: MemoryPool = try .init(allocator, threaded.async_limit.toInt() orelse 16, 256 * 1024 * 1024); - defer pool.deinit(io); - - var timer: std.time.Timer = try .start(); - defer { - const elapsed = timer.read(); - log.info("Completed in {d} ms", .{elapsed / std.time.ns_per_ms}); - } - - var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, uri); - defer registry.deinit(); - - log.info("Parsed {d} tensors", .{registry.tensors.count()}); - - var err: ?anyerror = null; - - var group: std.Io.Group = .init; - defer group.cancel(io); - - var timer_async_read: std.time.Timer = try .start(); - var it = registry.iterator(); - - while (it.next()) |entry| { - const tensor_name = entry.key_ptr.*; - group.async(io, readTensor, .{ io, &pool, ®istry, tensor_name, &err }); - } - - group.wait(io); - - if (err) |e| { - log.err("Error reading tensor: {any}", .{e}); - return e; - } - - const elapsed = timer_async_read.read(); - const read_mb = @as(f64, @floatFromInt(registry.totalBytes())) / (1024.0 * 1024.0); - const read_time_s = @as(f64, @floatFromInt(elapsed)) / @as(f64, std.time.ns_per_s); - log.info("Throughput: {d:.2} MB in {d:.2} s = {d:.2} MB/s | {d:.2} Gbps", .{ - read_mb, - read_time_s, - read_mb / read_time_s, - (read_mb * 8.0) / 1024.0 / read_time_s, - }); - } - - // { - // const root_dir = try vfs.openAbsoluteDir(io, "https://storage.googleapis.com", .{}); - // defer root_dir.close(io); - - // const buckets_dir = try root_dir.openDir(io, "zig-vfs", .{}); - // defer buckets_dir.close(io); - - // const model_dir = try buckets_dir.openDir(io, "Llama-3.1-8B-Instruct", .{}); - // defer model_dir.close(io); - - // const file = try model_dir.openFile(io, "model.safetensors.index.json", .{}); - // defer file.close(io); - - // const stat = try file.stat(io); - // log.info("Opened remote file with size: {d} bytes by navigating dirs", .{stat.size}); - // } - - // { - // const path = try std.fs.path.join(allocator, &.{ default_uri, "model-00004-of-00005.safetensors" }); - // defer allocator.free(path); - - // const file = try vfs.openAbsoluteFile(io, path, .{}); - // defer file.close(io); - - // const stat = try file.stat(io); - - // var group: std.Io.Group = .init; - // defer group.cancel(io); - - // const chunk_size: usize = 32 * 1024 * 1024; - // const num_chunks = (stat.size + chunk_size - 1) / chunk_size; - - // for (0..num_chunks) |i| { - // const offset = i * chunk_size; - // const size = if (offset + chunk_size > stat.size) stat.size - offset else chunk_size; - - // group.async(threaded.io(), readData, .{ allocator, io, &vfs, path, offset, size, i }); - // } - - // group.wait(io); - // } - - // { - // const path = try std.fs.path.join(allocator, &.{ repo_uri, "model.safetensors.index.json" }); - // defer allocator.free(path); - - // const file = try vfs.openAbsoluteFile(io, path, .{}); - // defer file.close(io); - - // const stat = try file.stat(io); - // log.info("Opened remote file with size: {d} bytes with openAbsoluteFile", .{stat.size}); - // } - - // { - // const model_dir = try vfs.openAbsoluteDir(io, repo_uri, .{}); - - // const safetensors_index = try model_dir.openFile(io, "model.safetensors.index.json", .{}); - // defer safetensors_index.close(io); - - // const stat = try safetensors_index.stat(io); - // log.info("Opened remote index file with size: {d} bytes with openAbsoluteDir + openFile", .{stat.size}); - // } - - // { - // const path = try std.fs.path.join(allocator, &.{ repo_uri, "model-00004-of-00005.safetensors" }); - // defer allocator.free(path); - - // const file = try vfs.openAbsoluteFile(io, path, .{}); - // defer file.close(io); - - // var reader = file.reader(io, &.{}); - - // const read_size = 16 * 1024 * 1024; - - // var allocating_writer: std.Io.Writer.Allocating = try .initCapacity(allocator, read_size); - // defer allocating_writer.deinit(); - - // const read = try reader.interface.stream(&allocating_writer.writer, .limited(read_size)); - // std.debug.assert(read == read_size); - // } - - // { - // const path = try std.fs.path.join(allocator, &.{ default_uri, "model-00004-of-00005.safetensors" }); // SHA256: 92ecfe1a2414458b4821ac8c13cf8cb70aed66b5eea8dc5ad9eeb4ff309d6d7b - // defer allocator.free(path); - - // const file = try vfs.openAbsoluteFile(io, path, .{}); - // defer file.close(io); - - // const stat = try file.stat(io); - - // const buf_size = 64 * 1024 * 1024; - // const buf = try allocator.alloc(u8, buf_size); - // defer allocator.free(buf); - - // var sha256: std.crypto.hash.sha2.Sha256 = .init(.{}); - // const compute_sha = true; - - // // Read directly without extra buffering layers - // var total_read: usize = 0; - // var bufs = [_][]u8{buf}; - - // while (true) { - // const n = try io.vtable.fileReadStreaming(io.userdata, file, &bufs); - // if (n == 0) break; - // if (compute_sha) sha256.update(buf[0..n]); - // total_read += n; - // } - - // if (compute_sha) { - // var hash: [32]u8 = undefined; - // sha256.final(&hash); - // log.info("SHA256: {x}", .{hash}); - // } - - // std.debug.assert(total_read == stat.size); - // } -} - -fn readTensor( - io: std.Io, - pool: *MemoryPool, - registry: *zml.safetensors.TensorRegistry, - tensor_name: []const u8, - err_ptr: *?anyerror, -) void { - readTensorImpl(io, pool, registry, tensor_name) catch |err| { - err_ptr.* = err; - log.err("Failed to read tensor {s}: {any}", .{ tensor_name, err }); - }; -} - -pub fn readTensorImpl( - io: std.Io, - pool: *MemoryPool, - registry: *zml.safetensors.TensorRegistry, - tensor_name: []const u8, -) !void { - var total_timer = try std.time.Timer.start(); - - const tensor = registry.tensors.get(tensor_name) orelse return error.TensorNotFound; - - const mem_slot = try pool.alloc(io, tensor.byteSize() * 2); - defer pool.free(io, mem_slot.index); - - var tensor_reader = try registry.reader(io, tensor_name, mem_slot.data[0..tensor.byteSize()]); - defer tensor_reader.deinit(); - - var read_timer = try std.time.Timer.start(); - - var writer: std.Io.Writer = .fixed(mem_slot.data[tensor.byteSize()..]); - - const read = try tensor_reader.interface.streamRemaining(&writer); - - const read_time = read_timer.read(); - const total_time = total_timer.read(); - const read_mb = @as(f64, @floatFromInt(read)) / (1024.0 * 1024.0); - const read_time_s = @as(f64, @floatFromInt(read_time)) / @as(f64, std.time.ns_per_s); - const throughput = if (read_time_s > 0) read_mb / read_time_s else 0; - - log.info("Tensor {s} read {d}/{d} bytes at offset {d} | read={d}ms total={d}ms | {d:.2} MB/s", .{ - tensor.name, - read, - tensor.byteSize(), - tensor.offset, - read_time / std.time.ns_per_ms, - total_time / std.time.ns_per_ms, - throughput, - }); -} - -const MemoryPool = struct { - const Slots = std.ArrayList(Slot); - const Slot = struct { - buf: []u8, - capacity: usize, - in_use: bool, - }; - - allocator: std.mem.Allocator, - mutex: std.Io.Mutex, - - slots: Slots, - default_capacity: usize, - - pub fn init(allocator: std.mem.Allocator, initial_count: usize, slot_size: usize) !MemoryPool { - var slots: Slots = .{}; - - for (0..initial_count) |_| { - const buf = try allocator.alloc(u8, slot_size); - try slots.append(allocator, .{ .buf = buf, .capacity = slot_size, .in_use = false }); - } - - return .{ - .allocator = allocator, - .slots = slots, - .default_capacity = slot_size, - .mutex = .init, - }; - } - - pub fn deinit(self: *MemoryPool, io: std.Io) void { - self.mutex.lockUncancelable(io); - - for (self.slots.items) |slot| { - self.allocator.free(slot.buf); - } - - self.mutex.unlock(io); - self.slots.deinit(self.allocator); - } - - pub fn alloc(self: *MemoryPool, io: std.Io, n: usize) !struct { data: []u8, index: usize } { - self.mutex.lockUncancelable(io); - - var idx: usize = 0; - while (idx < self.slots.items.len) : (idx += 1) { - if (!self.slots.items[idx].in_use and self.slots.items[idx].capacity >= n) { - self.slots.items[idx].in_use = true; - const data = self.slots.items[idx].buf[0..n]; - self.mutex.unlock(io); - return .{ .data = data, .index = idx }; - } - } - - const capacity = if (n > self.default_capacity) n else self.default_capacity; - - log.warn("MemoryPool: growing pool, allocating new slot of size {d} bytes", .{capacity}); - - const buf = try self.allocator.alloc(u8, capacity); - try self.slots.append(self.allocator, .{ .buf = buf, .capacity = capacity, .in_use = true }); - - const new_index = self.slots.items.len - 1; - const data = buf[0..n]; - - self.mutex.unlock(io); - - return .{ .data = data, .index = new_index }; - } - - pub fn free(self: *MemoryPool, io: std.Io, index: usize) void { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - - if (index < self.slots.items.len) { - self.slots.items[index].in_use = false; - } - } -}; - -fn readData( - allocator: std.mem.Allocator, - io: std.Io, - vfs: *zml.io.VFS, - uri: []const u8, - offset: usize, - size: usize, - chunk_index: usize, -) void { - const buffer = allocator.alloc(u8, size) catch { - log.err("Failed to allocate buffer of size {d} bytes for chunk {d}", .{ size, chunk_index }); - return; - }; - defer allocator.free(buffer); - - var file = vfs.openAbsoluteFile(io, uri, .{}) catch { - log.err("Failed to open file at {s}", .{uri}); - return; - }; - - var reader = file.reader(io, &.{}); - - reader.seekTo(@intCast(offset)) catch { - log.err("Failed to seek to offset {d} for chunk {d}", .{ offset, chunk_index }); - return; - }; - - var total_read: usize = 0; - while (total_read < size) { - const n = reader.interface.readSliceShort(buffer[total_read..]) catch { - log.err("Failed to read data for chunk {d} at offset {d}", .{ chunk_index, offset + total_read }); - return; - }; - if (n == 0) break; - total_read += n; - } - - std.debug.assert(total_read == size); - log.info("Read chunk {d} of size {d} bytes at offset {d}", .{ chunk_index, size, offset }); -} diff --git a/zml2/meta.zig b/zml2/meta.zig deleted file mode 100644 index e26fa014..00000000 --- a/zml2/meta.zig +++ /dev/null @@ -1,793 +0,0 @@ -const std = @import("std"); -const testing = std.testing; - -const stdx = @import("stdx"); -const FnParam = stdx.meta.FnParam; -const asSlice = stdx.meta.asSlice; - -/// Visit a given type `T` and replace all fields containing `From` by fields containing `To`. -pub fn MapType(From: type, To: type) type { - return struct { - pub fn map(T: type) type { - switch (T) { - To => return To, - ?To => return ?To, - From => return To, - *From => return *To, - ?From => return ?To, - else => {}, - } - - return switch (@typeInfo(T)) { - .@"struct" => |struct_infos| { - const fields = struct_infos.fields; - var same: bool = true; - var struct_field_names: [fields.len][]const u8 = undefined; - var struct_field_types: [fields.len]type = undefined; - var struct_field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; - for (struct_field_names[0..], struct_field_types[0..], struct_field_attrs[0..], fields) |*field_name, *field_type, *field_attr, field| { - if (!field.is_comptime) { - const R = map(field.type); - if (R == field.type) { - field_name.* = field.name; - field_type.* = field.type; - field_attr.* = .{ - .@"comptime" = field.is_comptime, - .@"align" = field.alignment, - .default_value_ptr = field.default_value_ptr, - }; - } else { - field_name.* = field.name; - field_type.* = R; - field_attr.* = .{ - .@"comptime" = field.is_comptime, - .@"align" = @alignOf(R), - }; - same = false; - // Handle the case `field: ?Tensor = null` - // Generic handling of default value is complicated, - // it would require to call the callback at comptime. - if (R == ?To) { - field_attr.default_value_ptr = &@as(R, null); - } - } - } else { - field_name.* = field.name; - field_type.* = field.type; - field_attr.* = .{ - .@"comptime" = field.is_comptime, - .@"align" = field.alignment, - .default_value_ptr = field.default_value_ptr, - }; - } - } - if (same) return T; - return if (struct_infos.is_tuple) @Tuple(&struct_field_types) else @Struct(.auto, null, &struct_field_names, &struct_field_types, &struct_field_attrs); - }, - .@"union" => |union_info| { - const fields = union_info.fields; - var same: bool = true; - var union_field_names: [fields.len][]const u8 = undefined; - var union_field_types: [fields.len]type = undefined; - var union_field_attrs: [fields.len]std.builtin.Type.UnionField.Attributes = undefined; - for (union_field_names[0..], union_field_types[0..], union_field_attrs[0..], fields) |*field_name, *field_type, *field_attr, field| { - const R = map(field.type); - if (R == field.type) { - field_name.* = field.name; - field_type.* = field.type; - field_attr.* = .{ - .@"align" = field.alignment, - }; - } else { - field_name.* = field.name; - field_type.* = R; - field_attr.* = .{ - .@"align" = @alignOf(R), - }; - same = false; - } - } - if (same) return T; - return @Union(.auto, union_info.tag_type, &union_field_names, &union_field_types, &union_field_attrs); - }, - .array => |arr_info| [arr_info.len]map(arr_info.child), - .pointer => |ptr_info| switch (ptr_info.size) { - .slice => if (ptr_info.is_const) - []const map(ptr_info.child) - else - []map(ptr_info.child), - .one => if (ptr_info.is_const) - *const map(ptr_info.child) - else - *map(ptr_info.child), - else => T, - }, - .optional => |opt_info| ?map(opt_info.child), - else => T, - }; - } - }; -} - -test MapType { - const A = struct { a: u32 }; - const B = struct { b: u32 }; - - const A2B = MapType(A, B); - - const StructA = struct { some: []const A, one: A, maybe: ?A, other: u32 }; - const struct_b = A2B.map(StructA){ - .some = &[2]B{ .{ .b = 0 }, .{ .b = 1 } }, - .maybe = null, - .one = .{ .b = 2 }, - .other = 43, - }; - _ = struct_b; - - const UnionA = union { some: []const A, one: A, maybe: ?A, other: u32 }; - const union_b = [_]A2B.map(UnionA){ - .{ .some = &[2]B{ .{ .b = 0 }, .{ .b = 1 } } }, - .{ .one = .{ .b = 2 } }, - .{ .maybe = null }, - .{ .other = 43 }, - }; - _ = union_b; -} - -/// Given a callback: `fn(Ctx, From) To`, recursively visits the given `from` struct -/// and calls the callback when it finds a `From` element, and writes it to the `to` struct. -/// The `to` parameter must be passed with mutable pointer, and tensor data need to be mutable if callback needs it. -/// `mapAlloc` tries as much as possible to respect the conversions made by Zig itself. -/// For example it can convert from a comptime array to a runtime slice. -/// `mapAlloc` can allocate new slices to write the result if the result struct requires it. -/// The caller is owning said allocations, using an `ArenaAllocator` might help tracking them. -/// -/// Note: to avoid infinite loop, mapAlloc doesn't look for `From` fields inside `To` struct. -/// Any `To` struct inside `from` will be copied over to the target. -pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam(cb, 0), from: anytype, to: anytype) !void { - // TODO: handle tuple to slice conversion - // TODO: handle error bubbling up - const From = FnParam(cb, 1); - const To = stdx.meta.FnResult(cb); - const FromStruct = @TypeOf(from); - const type_info_to_ptr = @typeInfo(@TypeOf(to)); - if (type_info_to_ptr != .pointer) { - stdx.debug.compileError("convertType is expecting a mutable `to` argument but received: {}", .{@TypeOf(to)}); - } - const ToStruct = type_info_to_ptr.pointer.child; - const type_info_to = @typeInfo(ToStruct); - - if (FromStruct == From) { - // We have an issues with `Tensor` -> `Shape` -> `Tensor` conversion when compiling ZML functions where one argument is a Shape itself. - // Normally we should call `cb` on all `Shape`. - // But the "ShapeOf" struct will have more Shape than need on the output. - // So here we take a hint from the receiving object. - // If the target is indeed a Tensor, use the callback, but if the target is `Shape` just copy it over. - if (ToStruct != To and FromStruct == ToStruct) { - to.* = from; - } else { - to.* = @call(.auto, cb, .{ ctx, from }); - } - return; - } - - if (FromStruct == To) { - to.* = from; - return; - } - - if (@sizeOf(ToStruct) == 0) return; - - switch (type_info_to) { - .@"struct" => |info| inline for (info.fields) |field| { - if (field.is_comptime or @sizeOf(field.type) == 0) continue; - const field_type_info = @typeInfo(field.type); - // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. - switch (field_type_info) { - // .pointer => try convertType(From, To, allocator, @field(from, field.name), @field(to, field.name), Ctx, ctx, cb), - .array, .optional, .@"union", .@"struct", .pointer => if (@hasField(FromStruct, field.name)) { - try mapAlloc( - cb, - allocator, - ctx, - @field(from, field.name), - &@field(to, field.name), - ); - } else if (field.default_value_ptr) |_| { - @field(to, field.name) = null; - } else { - stdx.debug.compileError("Mapping {} -> {} inside {} failed. Missing field {s} in {}", .{ From, To, FromStruct, field.name, ToStruct }); - }, - else => @field(to, field.name) = @field(from, field.name), - } - }, - .@"union" => { - switch (from) { - inline else => |_, tag| { - to.* = @unionInit(ToStruct, @tagName(tag), undefined); - try mapAlloc(cb, allocator, ctx, @field(from, @tagName(tag)), &@field(to, @tagName(tag))); - }, - } - }, - .array => for (from, to) |f, *t| { - try mapAlloc(cb, allocator, ctx, f, t); - }, - .pointer => |ptr_info| switch (ptr_info.size) { - .one => switch (type_info_to_ptr.pointer.size) { - // pointer to array -> slice promotion - .slice => { - const items = try allocator.alloc(type_info_to_ptr.pointer.child, from.len); - for (from, items) |f, *t| { - try mapAlloc(cb, allocator, ctx, f, t); - } - to.* = items; - }, - else => try mapAlloc(cb, allocator, ctx, from.*, to.*), - }, - .slice => { - const items = try allocator.alloc(@typeInfo(ToStruct).pointer.child, from.len); - for (from, items) |f, *t| { - try mapAlloc(cb, allocator, ctx, f, t); - } - to.* = items; - }, - else => stdx.debug.compileError("zml.meta.mapAlloc doesn't support: {}", .{FromStruct}), - }, - .optional => if (from) |f| { - to.* = @as(@typeInfo(type_info_to_ptr.pointer.child).optional.child, undefined); - try mapAlloc(cb, allocator, ctx, f, &(to.*.?)); - } else { - to.* = null; - }, - .int, .float, .@"enum" => to.* = from, - else => stdx.debug.compileError("zml.meta.mapAlloc doesn't support: {}", .{FromStruct}), - } -} - -test mapAlloc { - const B = struct { b: u8 }; - const A = struct { - a: u8, - pub fn convert(_: void, a: @This()) B { - return .{ .b = a.a }; - } - }; - - const Empty = struct {}; - - const AA = struct { - field: A, - array: [2]A, - slice: []const A, - other: u8, - // We want to allow conversion from comptime to runtime, because Zig type inference works like this. - comptime static_val: u8 = 8, - comptime static_slice: [2]A = .{ .{ .a = 11 }, .{ .a = 12 } }, - field_with_empty: struct { A, Empty }, - }; - const BB = struct { - field: B, - array: [2]B, - slice: []const B, - other: u8, - static_val: u8, - static_slice: []B, - field_with_empty: struct { B, Empty }, - }; - - const aa: AA = .{ - .field = .{ .a = 4 }, - .array = .{ .{ .a = 5 }, .{ .a = 6 } }, - .other = 7, - .slice = &.{ .{ .a = 9 }, .{ .a = 10 } }, - .field_with_empty = .{ .{ .a = 9 }, .{} }, - }; - var bb: BB = undefined; - - try mapAlloc(A.convert, testing.allocator, {}, aa, &bb); - defer testing.allocator.free(bb.slice); - defer testing.allocator.free(bb.static_slice); - - try testing.expectEqual(4, bb.field.b); - try testing.expectEqual(5, bb.array[0].b); - try testing.expectEqual(6, bb.array[1].b); - try testing.expectEqual(7, bb.other); - try testing.expectEqual(8, bb.static_val); - try testing.expectEqual(9, bb.slice[0].b); - try testing.expectEqual(10, bb.slice[1].b); - try testing.expectEqual(11, bb.static_slice[0].b); - try testing.expectEqual(12, bb.static_slice[1].b); -} - -/// Visit a given type `T` and: -/// * replace all fields containing `From` by fields containing `To` -/// * drop all fields not containing any `From`. -/// The returned type will contains only `To` making it easy for the compiler to produce compact layout. -/// Used by `zml.Bufferized` to strip compile time arguments from a model struct. -pub fn MapRestrict(From: type, To: type) type { - return struct { - pub fn map(T: type) type { - switch (T) { - From => return To, - ?From => return ?To, - *From => return *To, - *const From => return *const To, - []From => return []To, - []const From => return []const To, - else => {}, - } - - if (!Contains(T, From)) return void; - - return switch (@typeInfo(T)) { - .@"struct" => |struct_infos| { - // We know that at least one of the struct field contains a From. - // We map each field individually. Fields without From and comptime fields are removed. - const fields = struct_infos.fields; - var num_fields: usize = 0; - - var struct_field_names: [fields.len][]const u8 = undefined; - var struct_field_types: [fields.len]type = undefined; - var struct_field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; - for (fields) |field| { - if (!field.is_comptime and Contains(field.type, From)) { - const R = map(field.type); - if (R == field.type) { - struct_field_names[num_fields] = field.name; - struct_field_types[num_fields] = field.type; - struct_field_attrs[num_fields] = .{ - .@"comptime" = field.is_comptime, - .@"align" = field.alignment, - .default_value_ptr = field.default_value_ptr, - }; - } else { - const name = if (struct_infos.is_tuple) struct_infos.fields[num_fields].name else field.name; - struct_field_names[num_fields] = name; - struct_field_types[num_fields] = R; - struct_field_attrs[num_fields] = .{ - .@"align" = @alignOf(R), - }; - // Handle the case `field: ?Tensor = null` - // Generic handling of default value is not possible. - if (R == ?To) { - struct_field_attrs[num_fields].default_value_ptr = &@as(R, null); - } - } - num_fields += 1; - } - } - if (num_fields == 0) return void; - return if (struct_infos.is_tuple) @Tuple(struct_field_types[0..num_fields]) else @Struct( - .auto, - null, - struct_field_names[0..num_fields], - struct_field_types[0..num_fields], - struct_field_attrs[0..num_fields], - ); - }, - .@"union" => |union_info| { - // We know that at least one of the union field contains a From. - // We map each field individually. Fields without From, are replaced by "void". - const fields = union_info.fields; - var union_field_names: [fields.len][]const u8 = undefined; - var union_field_types: [fields.len]type = undefined; - var union_field_attrs: [fields.len]std.builtin.Type.UnionField.Attributes = undefined; - for (0.., fields) |i, field| { - const FT = map(field.type); - union_field_names[i] = field.name; - union_field_types[i] = FT; - union_field_attrs[i] = .{ - .@"align" = @alignOf(FT), - }; - } - return @Union(.auto, union_info.tag_type, &union_field_names, &union_field_types, &union_field_attrs); - }, - .array => |arr_info| [arr_info.len]map(arr_info.child), - .pointer => |ptr_info| switch (ptr_info.size) { - .slice => if (ptr_info.is_const) - []const map(ptr_info.child) - else - []map(ptr_info.child), - .one => if (ptr_info.is_const) - *const map(ptr_info.child) - else - *map(ptr_info.child), - .many => if (ptr_info.is_const) - [*]const map(ptr_info.child) - else - [*]map(ptr_info.child), - .c => if (ptr_info.is_const) - [*c]map(ptr_info.child) - else - [*c]map(ptr_info.child), - }, - .optional => |opt_info| ?map(opt_info.child), - else => T, - }; - } - }; -} - -test MapRestrict { - const A = struct { a: u32 }; - const B = struct { b: u32 }; - - const A2B = MapRestrict(A, B); - - const StructA = struct { some: []const A, one: A, maybe: ?A, other: u32 }; - const struct_b = A2B.map(StructA){ - .some = &[2]B{ .{ .b = 0 }, .{ .b = 1 } }, - .maybe = null, - .one = .{ .b = 2 }, - // Note how struct_b doesn't even have a .other field now. - }; - _ = struct_b; - - const UnionA = union { some: []const A, one: A, maybe: ?A, other: u32 }; - const union_b = [_]A2B.map(UnionA){ - .{ .some = &[2]B{ .{ .b = 0 }, .{ .b = 1 } } }, - .{ .one = .{ .b = 2 } }, - .{ .maybe = null }, - // Note how union_b.other is void now. - .{ .other = {} }, - }; - _ = union_b; -} - -/// Recursively visit the given struct and calls the callback for each K found. -/// The `v` parameter must me a pointer, and tensor data need to be mutable if callbacks needs it. -pub fn VisitReturn(comptime cb: anytype) type { - return if (stdx.meta.FnSignature(cb, null).ReturnErrorSet) |error_set| - error_set!void - else - void; -} - -pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) VisitReturn(cb) { - const Callback = @TypeOf(cb); - const Ptr = @TypeOf(v); - const type_info_v = @typeInfo(Ptr); - if (type_info_v != .pointer) { - stdx.debug.compileError("zml.meta.visit({}) is expecting a pointer/slice input, but received: {}", .{ Callback, Ptr }); - } - const ptr_info = type_info_v.pointer; - const Child = ptr_info.child; - const can_error = stdx.meta.FnSignature(cb, null).ReturnErrorSet != null; - - const K, const mutating_cb = switch (@typeInfo(FnParam(cb, 1))) { - .pointer => |info| .{ info.child, !info.is_const }, - else => stdx.debug.compileError("zml.meta.visit is expecting a callback with a pointer as second argument but found {}", .{FnParam(cb, 1)}), - }; - // Abort if v doesnt' contain any K. - if (comptime !Contains(Ptr, K)) return; - - // Handle simple cases. - switch (Ptr) { - *const K, *K => return cb(ctx, v), - *const ?K, *?K => return if (v.*) |*val| cb(ctx, val) else {}, - []const K, []K => { - for (v) |*v_elem| if (can_error) try cb(ctx, v_elem) else cb(ctx, v_elem); - return; - }, - else => {}, - } - - // Handle stdx.BoundedArray that contains uninitalized data. - if (@typeInfo(Child) == .@"struct" and @hasDecl(Child, "constSlice") and @hasDecl(Child, "slice")) { - return visit(cb, ctx, if (mutating_cb) v.slice() else v.constSlice()); - } - - // Recursively visit fields of v. - switch (ptr_info.size) { - .one => switch (@typeInfo(Child)) { - .@"struct" => |s| inline for (s.fields) |field| { - if (field.is_comptime or comptime !Contains(field.type, K)) continue; - const field_type_info = @typeInfo(field.type); - // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. - switch (field_type_info) { - .pointer => if (can_error) try visit(cb, ctx, @field(v, field.name)) else visit(cb, ctx, @field(v, field.name)), - .array, .optional, .@"union", .@"struct" => if (can_error) try visit(cb, ctx, &@field(v, field.name)) else visit(cb, ctx, &@field(v, field.name)), - else => {}, - } - }, - .array => |_| for (v) |*elem| if (can_error) try visit(cb, ctx, elem) else visit(cb, ctx, elem), - .optional => if (v.* != null) if (can_error) try visit(cb, ctx, &v.*.?) else visit(cb, ctx, &v.*.?), - .@"union" => switch (v.*) { - inline else => |*v_field| if (can_error) try visit(cb, ctx, v_field) else visit(cb, ctx, v_field), - }, - else => stdx.debug.compileError("zml.meta.visit({}) doesn't support fields of type: {}", .{ Callback, Child }), - }, - .slice => { - for (v) |*v_elem| { - switch (@typeInfo(Child)) { - .@"struct" => |s| inline for (s.fields) |field| { - if (field.is_comptime or comptime !Contains(field.type, K)) continue; - const field_type_info = @typeInfo(field.type); - // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. - if (field_type_info == .pointer) { - if (can_error) try visit(cb, ctx, @field(v_elem, field.name)) else visit(cb, ctx, @field(v_elem, field.name)); - } else { - if (can_error) try visit(cb, ctx, &@field(v_elem, field.name)) else visit(cb, ctx, &@field(v_elem, field.name)); - } - }, - .array => |_| for (v) |*elem| if (can_error) try visit(cb, ctx, elem) else visit(cb, ctx, elem), - .optional => if (v.* != null) if (can_error) try visit(cb, ctx, &v.*.?) else visit(cb, ctx, &v.*.?), - .@"union" => switch (v_elem.*) { - inline else => |*v_field| if (can_error) try visit(cb, ctx, v_field) else visit(cb, ctx, v_field), - }, - else => stdx.debug.compileError("zml.meta.visit({}) doesn't support fields of type: {}", .{ Callback, Child }), - } - } - }, - .many, .c => stdx.debug.compileError("zml.meta.visit({}) doesn't support [*] style pointers, got: {}", .{ Callback, Ptr }), - } -} - -test visit { - const Attr = struct { data: usize }; - const OtherAttr = struct { other: []const u8 }; - const NestedAttr = struct { nested: Attr }; - const NestedAttrOptional = struct { nested: ?Attr }; - const SimpleStruct = struct { prop: Attr }; - const MultipleTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: ?Attr }; - const NestedTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: NestedAttr, prop4: NestedAttrOptional, prop5: stdx.BoundedArray(Attr, 8) }; - - const LocalContext = struct { result: usize }; - - { - var context: LocalContext = .{ .result = 0 }; - const container: SimpleStruct = .{ .prop = .{ .data = 1 } }; - - visit((struct { - fn cb(ctx: *LocalContext, attr: *const Attr) void { - ctx.result += attr.data; - } - }).cb, &context, &container); - - try std.testing.expectEqual(1, context.result); - } - { - var context: LocalContext = .{ .result = 0 }; - var container: SimpleStruct = .{ .prop = .{ .data = 1 } }; - - visit((struct { - fn cb(ctx: *LocalContext, attr: *Attr) void { - ctx.result += attr.data; - } - }).cb, &context, &container); - - try std.testing.expectEqual(1, context.result); - } - { - var context: LocalContext = .{ .result = 0 }; - var container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = null }; - - visit((struct { - fn cb(ctx: *LocalContext, attr: *Attr) void { - ctx.result += attr.data; - } - }).cb, &context, &container); - - try std.testing.expectEqual(1, context.result); - } - { - var context: LocalContext = .{ .result = 0 }; - const container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = .{ .data = 2 } }; - - visit((struct { - fn cb(ctx: *LocalContext, attr: *const Attr) void { - ctx.result += attr.data; - } - }).cb, &context, &container); - - try std.testing.expectEqual(3, context.result); - } - { - var context: LocalContext = .{ .result = 0 }; - const prop5: stdx.BoundedArray(Attr, 8) = .{ - .buffer = @splat(.{ .data = 4 }), - .len = 2, - }; - const container: NestedTypesStruct = .{ - .prop1 = .{ .data = 1 }, - .prop2 = .{ .other = "hello" }, - .prop3 = .{ .nested = .{ .data = 2 } }, - .prop4 = .{ .nested = .{ .data = 3 } }, - .prop5 = prop5, // 4 will be counted twice. - }; - - visit((struct { - fn cb(ctx: *LocalContext, attr: *const Attr) void { - ctx.result += attr.data; - } - }).cb, &context, &container); - - try std.testing.expectEqual(14, context.result); - } -} - -pub fn count(T: type, value: anytype) u32 { - var counter: u32 = 0; - visit(struct { - pub fn cb(res: *u32, _: *const T) void { - res.* += 1; - } - }.cb, &counter, value); - return counter; -} - -pub fn first(T: type, value: anytype) T { - var res: ?T = null; - visit(struct { - pub fn cb(res_ptr: *?T, x: *const T) void { - if (res_ptr.* == null) res_ptr.* = x.*; - } - }.cb, &res, &value); - return res.?; -} - -/// Given a `fn([]const T, Args) T` and a slice of values, -/// will combine all values in one value. -/// Only T elements of values will be looked at. -/// This only works for simple types, in particular `zip` doesn't follow pointers. -/// Which means that zip only allocate temp memory, and nothing need to be freed after the call. -pub fn zip(comptime func: anytype, allocator: std.mem.Allocator, values: anytype, args: anytype) error{OutOfMemory}!asSlice(@TypeOf(values)) { - const sliceT = @typeInfo(FnParam(func, 0)); - const T = sliceT.pointer.child; - const V = asSlice(@TypeOf(values)); - if (V == T) { - return @call(.auto, func, .{values} ++ args); - } - // const fn_args - - return switch (@typeInfo(V)) { - .pointer => stdx.debug.compileError("zip only accept by value arguments. Received: {}", .{V}), - .@"struct" => |struct_info| { - var out: V = values[0]; - inline for (struct_info.fields) |f| { - if (f.is_comptime) continue; - if (@typeInfo(f.type) == .pointer) { - stdx.debug.compileError("zip doesn't follow pointers and don't accept struct containing them. Received: {}", .{V}); - } - var fields = try allocator.alloc(f.type, values.len); - defer allocator.free(fields); - for (values, 0..) |val, i| { - fields[i] = @field(val, f.name); - } - @field(out, f.name) = try zip(func, allocator, fields, args); - } - return out; - }, - .array => |arr_info| { - if (@typeInfo(arr_info.child) == .pointer) { - stdx.debug.compileError("zip doesn't follow pointers and don't accept struct containing them. Received: {}", .{V}); - } - var out: V = undefined; - var slice = try allocator.alloc(arr_info.child, values.len); - defer allocator.free(slice); - for (&out, 0..) |*o, j| { - for (values, 0..) |val, i| { - slice[i] = val[j]; - } - o.* = try zip(func, allocator, slice, args); - } - return out; - }, - .@"union", .optional => stdx.debug.compileError("zip doesn't yet support {}", .{V}), - else => values[0], - }; -} - -test zip { - const A = struct { a: u8, b: [2]u8 }; - const a0: A = .{ .a = 1, .b = .{ 2, 3 } }; - const a1: A = .{ .a = 4, .b = .{ 5, 6 } }; - - const Sum = struct { - pub fn call(x: []const u8) u8 { - var res: u8 = 0; - for (x) |xx| res += xx; - return res; - } - }; - const a_sum: A = try zip(Sum.call, testing.allocator, &[_]A{ a0, a1 }, .{}); - try testing.expectEqual(A{ .a = 5, .b = .{ 7, 9 } }, a_sum); -} - -/// Given a func(X) -> Y or a func(Ctx, X) -> Y, -/// finds all X in the given object, and write the result of func(X) into an arraylist. -pub fn collectAlloc( - func: anytype, - func_ctx: _CollectCtx(func), - allocator: std.mem.Allocator, - obj: anytype, -) std.mem.Allocator.Error![]stdx.meta.FnSignature(func, null).ReturnT { - stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".params.len <= 2, "zml.meta.collect expects a func with two arguments, got: {}", .{@TypeOf(func)}); - - const CollectAllocCtx = struct { - func_ctx: _CollectCtx(func), - allocator: std.mem.Allocator, - out: std.ArrayList(stdx.meta.FnSignature(func, null).ReturnT) = .empty, - oom: bool = false, - - fn cb(ctx: *@This(), val: *const _CollectArg(func)) void { - if (ctx.oom) return; - const res = if (_CollectCtx(func) == void) func(val.*) else func(ctx.func_ctx, val.*); - ctx.out.append(ctx.allocator, res) catch { - ctx.oom = true; - }; - } - }; - var context = CollectAllocCtx{ .func_ctx = func_ctx, .allocator = allocator }; - visit(CollectAllocCtx.cb, &context, obj); - if (context.oom) return error.OutOfMemory; - - return context.out.toOwnedSlice(allocator); -} - -/// Given a func(X) -> Y or a func(Ctx, X) -> Y, -/// finds all X in the given object, and write the result of func(X) into a given slice. -/// Asserts that the number of X found is equal to the slice len. -pub fn collectBuf(func: anytype, func_ctx: _CollectCtx(func), obj: anytype, out: []stdx.meta.FnResult(func)) void { - stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".params.len <= 2, "zml.meta.collectBuf expects a func with one or two arguments, got: {}", .{@TypeOf(func)}); - const LocalContext = struct { - func_ctx: _CollectCtx(func), - out: @TypeOf(out), - idx: usize = 0, - }; - var context = LocalContext{ .func_ctx = func_ctx, .out = out }; - visit((struct { - fn cb(ctx: *LocalContext, val: *const _CollectArg(func)) void { - if (ctx.idx >= ctx.out.len) return; - - const res = if (_CollectCtx(func) == void) func(val.*) else func(ctx.func_ctx, val.*); - ctx.out[ctx.idx] = res; - ctx.idx += 1; - } - }).cb, &context, obj); - std.debug.assert(context.idx == context.out.len); -} - -fn _CollectCtx(func: anytype) type { - const params = @typeInfo(@TypeOf(func)).@"fn".params; - if (params.len == 1) return void; - return params[0].type orelse @compileError("anytype not supported in collect"); -} - -fn _CollectArg(func: anytype) type { - const params = @typeInfo(@TypeOf(func)).@"fn".params; - return params[params.len - 1].type orelse @compileError("anytype not supported in collect"); -} - -pub fn Contains(Haystack: type, T: type) bool { - switch (Haystack) { - T, ?T => return true, - *T, ?*T => return true, - *const T, ?*const T => return true, - []const T, ?[]const T => return true, - anyopaque => return false, - else => {}, - } - - return switch (@typeInfo(Haystack)) { - .@"struct" => |info| { - inline for (info.fields) |field| { - if (!field.is_comptime and Contains(field.type, T)) - return true; - } - return false; - }, - .@"union" => |info| { - inline for (info.fields) |field| { - if (Contains(field.type, T)) - return true; - } - return false; - }, - .array => |info| Contains(info.child, T), - .pointer => |info| Contains(info.child, T), - .optional => |info| Contains(info.child, T), - .vector => |info| Contains(info.child, T), - else => false, - }; -} diff --git a/zml2/mlirx.zig b/zml2/mlirx.zig deleted file mode 100644 index 2c3ddb17..00000000 --- a/zml2/mlirx.zig +++ /dev/null @@ -1,93 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); - -const dtype = @import("dtype.zig"); -const Shape = @import("shape.zig").Shape; - -const mlirx = @This(); - -pub const Type = struct { - pub fn fromDType(ctx: *mlir.Context, dt: dtype.DataType) *const mlir.Type { - return switch (dt) { - .bool => mlir.integerType(ctx, .i1), - .f4e2m1 => mlir.floatType(ctx, .f4e2m1fn), - .f8e3m4 => mlir.floatType(ctx, .f8e3m4), - .f8e4m3 => mlir.floatType(ctx, .f8e4m3), - .f8e4m3fn => mlir.floatType(ctx, .f8e4m3fn), - .f8e4m3fnuz => mlir.floatType(ctx, .f8e4m3fnuz), - .f8e4m3b11fnuz => mlir.floatType(ctx, .f8e4m3b11fnuz), - .f8e5m2 => mlir.floatType(ctx, .f8e5m2), - .f8e5m2fnuz => mlir.floatType(ctx, .f8e5m2fnuz), - .f8e8m0 => @panic("Unsupported type"), - .bf16 => mlir.floatType(ctx, .bf16), - .f16 => mlir.floatType(ctx, .f16), - .f32 => mlir.floatType(ctx, .f32), - .f64 => mlir.floatType(ctx, .f64), - .i2 => mlir.integerType(ctx, .i2), - .i4 => mlir.integerType(ctx, .i4), - .i8 => mlir.integerType(ctx, .i8), - .i16 => mlir.integerType(ctx, .i16), - .i32 => mlir.integerType(ctx, .i32), - .i64 => mlir.integerType(ctx, .i64), - .u2 => mlir.integerType(ctx, .u2), - .u4 => mlir.integerType(ctx, .u4), - .u8 => mlir.integerType(ctx, .u8), - .u16 => mlir.integerType(ctx, .u16), - .u32 => mlir.integerType(ctx, .u32), - .u64 => mlir.integerType(ctx, .u64), - .c64 => mlir.complexType(ctx, .c64), - .c128 => mlir.complexType(ctx, .c128), - }; - } - - // TODO(Corentin): Maybe remove that as it now requires a mlir context. - // It's weird to have to provide a mlir context to get a zml.DataType from a mlir.Type. - pub fn toDType(ctx: *mlir.Context, mlir_type: *const mlir.Type) dtype.DataType { - const mapping = .{ - .{ .bool, mlir.integerType(ctx, .i1) }, - - .{ .f8e4m3b11fnuz, mlir.floatType(ctx, .f8e4m3b11fnuz) }, - .{ .f8e4m3fn, mlir.floatType(ctx, .f8e4m3fn) }, - .{ .f8e4m3fnuz, mlir.floatType(ctx, .f8e4m3fnuz) }, - .{ .f8e3m4, mlir.floatType(ctx, .f8e3m4) }, - .{ .f8e5m2, mlir.floatType(ctx, .f8e5m2) }, - .{ .f8e5m2fnuz, mlir.floatType(ctx, .f8e5m2fnuz) }, - .{ .f4e2m1, mlir.floatType(ctx, .f4e2m1fn) }, - .{ .bf16, mlir.floatType(ctx, .bf16) }, - .{ .f16, mlir.floatType(ctx, .f16) }, - .{ .f32, mlir.floatType(ctx, .f32) }, - .{ .f64, mlir.floatType(ctx, .f64) }, - - .{ .i4, mlir.integerType(ctx, .i4) }, - .{ .i8, mlir.integerType(ctx, .i8) }, - .{ .i16, mlir.integerType(ctx, .i16) }, - .{ .i32, mlir.integerType(ctx, .i32) }, - .{ .i64, mlir.integerType(ctx, .i64) }, - - .{ .u4, mlir.integerType(ctx, .u4) }, - .{ .u8, mlir.integerType(ctx, .u8) }, - .{ .u16, mlir.integerType(ctx, .u16) }, - .{ .u32, mlir.integerType(ctx, .u32) }, - .{ .u64, mlir.integerType(ctx, .u64) }, - - .{ .c64, mlir.complexType(ctx, .c64) }, - .{ .c128, mlir.complexType(ctx, .c128) }, - }; - - inline for (mapping) |entry| { - const dt, const mlirT = entry; - if (mlirT.eql(mlir_type)) { - return dt; - } - } - - std.debug.panic("Could not convert mlir.Type to DataType: {f}", .{mlir_type}); - } -}; - -pub var once = std.once(struct { - fn call() void { - mlir.registerPasses("Transforms"); - } -}.call); diff --git a/zml2/module.zig b/zml2/module.zig deleted file mode 100644 index 525cfbd6..00000000 --- a/zml2/module.zig +++ /dev/null @@ -1,503 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const dialects = @import("mlir/dialects"); -const mlir = @import("mlir"); -const stdx = @import("stdx"); -const upb = @import("upb"); - -const Buffer = @import("buffer.zig").Buffer; -const Exe = @import("exe.zig").Exe; -const meta = @import("meta.zig"); -const mlirx = @import("mlirx.zig"); -const pjrt = @import("pjrtx.zig"); -const Platform = @import("platform.zig").Platform; -const Shape = @import("shape.zig").Shape; -const Tensor = @import("tensor.zig").Tensor; - -const log = std.log.scoped(.@"zml/module"); - -const AttributeList = stdx.BoundedArray(mlir.NamedAttribute, 3); - -pub const CompilationContext = struct { - pub const Scope = struct { - block: *mlir.Block, - id_to_argument: std.AutoArrayHashMapUnmanaged(usize, usize), - id_to_donation: std.AutoArrayHashMapUnmanaged(usize, usize), - id_to_output_memory_kind: std.AutoArrayHashMapUnmanaged(usize, Buffer.Memory), - arena: std.heap.ArenaAllocator, - - pub fn initFromBlock(allocator: std.mem.Allocator, block: *mlir.Block) Scope { - const arena: std.heap.ArenaAllocator = .init(allocator); - return .{ - .block = block, - .id_to_argument = .empty, - .id_to_donation = .empty, - .id_to_output_memory_kind = .empty, - .arena = arena, - }; - } - - pub fn deinit(self: *Scope) void { - self.arena.deinit(); - } - }; - - allocator: std.mem.Allocator, - arena: std.heap.ArenaAllocator, - - mlir_registry: *mlir.DialectRegistry, - mlir_ctx: *mlir.Context, - mlir_pass_manager: *mlir.PassManager, - //mlir_op_pass_manager: *mlir.OpPassManager, - module: *mlir.Module, - platform: Platform, - - scopes: stdx.BoundedArray(Scope, 16) = .{}, - - threadlocal var _current: ?*CompilationContext = null; - - pub fn init(allocator: std.mem.Allocator, platform: Platform) CompilationContext { - mlir.registerPasses("Transforms"); - const mlir_registry = mlir.DialectRegistry.init() catch unreachable; - inline for (.{ "func", "stablehlo" }) |d| { - mlir.DialectHandle.fromString(d).insertDialect(mlir_registry); - } - var mlir_ctx = mlir.Context.init(.{ .registry = mlir_registry, .threading = false }) catch unreachable; - mlir_ctx.loadAllAvailableDialects(); - - //const loc = mlir.Location.fromSrc(mlir_ctx, @src()).named(mlir_ctx, "main"); - const module = mlir.Module.init(.unknown(mlir_ctx)); - module.operation().setAttributeByName("sym_name", mlir.stringAttribute(mlir_ctx, "zml")); - - const pass_manager = mlir.PassManager.init(mlir_ctx); - { - var opm = pass_manager.asOpPassManager(); - const passes: []const []const u8 = &.{ - "canonicalize", - "cse", - "canonicalize", - }; - for (passes) |pass| { - opm.addPipeline(pass) catch unreachable; - } - } - - return .{ - .allocator = allocator, - .arena = std.heap.ArenaAllocator.init(allocator), - .mlir_registry = mlir_registry, - .mlir_ctx = mlir_ctx, - .mlir_pass_manager = pass_manager, - .module = module, - .platform = platform, - }; - } - - pub fn deinit(self: *CompilationContext) void { - self.mlir_pass_manager.deinit(); - self.module.deinit(); - self.mlir_ctx.deinit(); - self.mlir_registry.deinit(); - self.arena.deinit(); - } - - pub fn activate(self: *CompilationContext) void { - std.debug.assert(_current == null); - _current = self; - } - - pub fn deactivate(self: *CompilationContext) void { - _ = self; - _current = null; - } - - pub fn current() *CompilationContext { - return _current.?; - } - - pub fn currentScope(self: *CompilationContext) *Scope { - return &self.scopes.slice()[self.scopes.len - 1]; - } - - pub fn pushBlock(self: *CompilationContext, block: *mlir.Block) void { - const scope = Scope.initFromBlock(self.allocator, block); - self.scopes.appendAssumeCapacity(scope); - } - - pub fn popBlock(self: *CompilationContext) void { - var maybe_popped_scope = self.scopes.pop(); - if (maybe_popped_scope) |*popped| { - popped.deinit(); - } - } - - //pub fn addShardingAttributes( - // self: *const CompilationContext, - // mesh: Mesh, - // arg_attrs: []AttributeList, - // res_attrs: []AttributeList, - // input_shapes: []const Shape, - // input_meshes: []const ?Mesh, - // output_info: OutputInfo, - //) void { - // if (mesh.isSinglePartition()) return; // No sharding attributes for single partition. - - // const default_layout = mlir.NamedAttribute.named(self.mlir_ctx, "mhlo.layout_mode", mlir.stringAttribute(self.mlir_ctx, "default")); - // for (arg_attrs, input_shapes, input_meshes) |*attr, shape, input_mesh| { - // attr.appendAssumeCapacity(default_layout); - // if (input_mesh) |m| { - // // log.warn("addShardingAttributes for {} with input mesh: {}", .{ shape, m }); - // attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(m, shape)))); - // } else { - // // log.warn("addShardingAttributes for {} with progran mesh: {}", .{ shape, mesh }); - // attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(mesh, shape)))); - // } - // } - - // for (res_attrs, output_info.shapes) |*attr, shape| { - // attr.appendAssumeCapacity(default_layout); - // attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(mesh, shape)))); - // } - //} - - //pub fn getShardingAttr(self: *const CompilationContext, sharding: Sharding) *const mlir.Attribute { - // var sharding_str: stdx.BoundedArray(u8, 128) = .{}; - // sharding.writeShardingRepresentation(sharding_str.writer()) catch unreachable; - // return mlir.stringAttribute(self.mlir_ctx, sharding_str.constSlice()); - //} -}; - -pub fn compile(allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, args: stdx.meta.FnArgs(func), platform: Platform) !Exe { - var compilation_context: CompilationContext = .init(allocator, platform); - defer compilation_context.deinit(); - - const result = emitMlir(&compilation_context, func, args) catch unreachable; - defer result.output_info.deinit(compilation_context.allocator); - defer result.input_info.deinit(compilation_context.allocator); - - _ = result.func.appendTo(compilation_context.module.body()); - const sharding = platform.sharding(); - compilation_context.module.operation().setAttributeByName("mhlo.num_replicas", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, sharding.num_replicas)); - compilation_context.module.operation().setAttributeByName("mhlo.num_partitions", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, sharding.num_partitions)); - - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - - const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), io, platform, compilation_context.module, null) catch unreachable; - - log.debug("\n******** ZML generated MLIR ********\n{f}", .{compilation_context.module.operation()}); - - const num_devices = sharding.num_partitions * sharding.num_replicas; - const exe = try Exe.init(allocator, platform, loaded_executable, result.input_info.shapes, result.output_info.shapes, num_devices); - errdefer exe.deinit(); - - return exe; -} - -fn collectShapes(allocator: std.mem.Allocator, v: anytype) ![]Shape { - const LocalContext = struct { - list: *std.array_list.Managed(Shape), - }; - var list = std.array_list.Managed(Shape).init(allocator); - errdefer list.deinit(); - - var context: LocalContext = .{ .list = &list }; - try meta.visit(struct { - fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { - try ctx_.list.append(tensor.shape()); - } - }.cb, &context, v); - - return try list.toOwnedSlice(); -} - -pub const OutputInfo = struct { - shapes: []Shape, - values: []*const mlir.Value, - donations: []?usize, - output_memory_kinds: []Buffer.Memory, - - pub fn deinit(self: OutputInfo, allocator: std.mem.Allocator) void { - allocator.free(self.shapes); - allocator.free(self.values); - allocator.free(self.donations); - allocator.free(self.output_memory_kinds); - } -}; - -fn collectOutputInfo(allocator: std.mem.Allocator, v: anytype) !OutputInfo { - const LocalContext = struct { - shape_list: *std.array_list.Managed(Shape), - value_list: *std.array_list.Managed(*const mlir.Value), - donation_list: *std.array_list.Managed(?usize), - output_memory_kind_list: *std.array_list.Managed(Buffer.Memory), - }; - - var shape_list = std.array_list.Managed(Shape).init(allocator); - errdefer shape_list.deinit(); - var value_list = std.array_list.Managed(*const mlir.Value).init(allocator); - errdefer value_list.deinit(); - var donation_list = std.array_list.Managed(?usize).init(allocator); - errdefer donation_list.deinit(); - var output_memory_kind_list = std.array_list.Managed(Buffer.Memory).init(allocator); - errdefer output_memory_kind_list.deinit(); - - var context: LocalContext = .{ - .shape_list = &shape_list, - .value_list = &value_list, - .donation_list = &donation_list, - .output_memory_kind_list = &output_memory_kind_list, - }; - - try meta.visit(struct { - fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { - try ctx_.shape_list.append(tensor.shape()); - try ctx_.value_list.append(tensor.value()); - try ctx_.donation_list.append(tensor.donation()); - try ctx_.output_memory_kind_list.append(tensor.outputMemoryKind()); - } - }.cb, &context, v); - - return .{ - .shapes = try shape_list.toOwnedSlice(), - .values = try value_list.toOwnedSlice(), - .donations = try donation_list.toOwnedSlice(), - .output_memory_kinds = try output_memory_kind_list.toOwnedSlice(), - }; -} - -pub const InputInfo = struct { - shapes: []Shape, - - pub fn deinit(self: InputInfo, allocator: std.mem.Allocator) void { - allocator.free(self.shapes); - } -}; - -fn collectInputInfo(allocator: std.mem.Allocator, v: anytype) !InputInfo { - const LocalContext = struct { - shape_list: *std.array_list.Managed(Shape), - }; - - var shape_list = std.array_list.Managed(Shape).init(allocator); - errdefer shape_list.deinit(); - - var context: LocalContext = .{ .shape_list = &shape_list }; - - try meta.visit(struct { - fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { - try ctx_.shape_list.append(tensor.shape()); - } - }.cb, &context, v); - - return .{ - .shapes = try shape_list.toOwnedSlice(), - }; -} - -const EmitMlirResult = struct { - func: *mlir.Operation, - input_info: InputInfo, - output_info: OutputInfo, -}; - -fn finalizeAttributeList(allocator_: std.mem.Allocator, mlir_ctx: *mlir.Context, attributes: []AttributeList) ![]*const mlir.Attribute { - const res = try allocator_.alloc(*const mlir.Attribute, attributes.len); - for (res, attributes) |*r, attr| { - r.* = mlir.dictionaryAttribute(mlir_ctx, attr.constSlice()); - } - return res; -} - -fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, args: stdx.meta.FnArgs(func)) !EmitMlirResult { - var arena = std.heap.ArenaAllocator.init(compilation_context.allocator); - defer arena.deinit(); - - const module = mlir.Module.init(.unknown(compilation_context.mlir_ctx)); - errdefer module.deinit(); - - const block = mlir.Block.init(&.{}, &.{}); - errdefer block.deinit(); - - compilation_context.pushBlock(block); - defer compilation_context.popBlock(); - - const LocalContext = struct { - compilation_context: *CompilationContext, - current_argument_id: usize = 0, - }; - var context: LocalContext = .{ - .compilation_context = compilation_context, - }; - meta.visit(struct { - fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { - const mlir_type = mlir.rankedTensorType( - tensor.dims(), - mlirx.Type.fromDType(ctx_.compilation_context.mlir_ctx, tensor.dtype()), - ); - _ = ctx_.compilation_context.currentScope().block.addArgument(mlir_type, .unknown(ctx_.compilation_context.mlir_ctx)); - ctx_.compilation_context.currentScope().id_to_argument.put(ctx_.compilation_context.currentScope().arena.allocator(), tensor.id, ctx_.current_argument_id) catch unreachable; - ctx_.current_argument_id += 1; - } - }.cb, &context, &args); - - const input_info = try collectInputInfo(compilation_context.allocator, &args); - errdefer input_info.deinit(compilation_context.allocator); - - const input_attributes = try arena.allocator().alloc(AttributeList, input_info.shapes.len); - @memset(input_attributes, .{}); - - const output_info = b: { - compilation_context.activate(); - defer compilation_context.deactivate(); - - const result = @call(.auto, func, args); - - const output_info = try collectOutputInfo(compilation_context.allocator, &result); - errdefer output_info.deinit(compilation_context.allocator); - - break :b output_info; - }; - errdefer output_info.deinit(compilation_context.allocator); - - const output_attributes = try arena.allocator().alloc(AttributeList, output_info.shapes.len); - @memset(output_attributes, .{}); - - for (output_info.donations, 0..) |donation, index| if (donation) |argument_index| { - input_attributes[argument_index].appendAssumeCapacity(.named(compilation_context.mlir_ctx, "tf.aliasing_output", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, index))); - }; - for (output_info.output_memory_kinds, 0..) |output_memory_kind, index| { - if (output_memory_kind == .device) continue; - output_attributes[index].appendAssumeCapacity(.named(compilation_context.mlir_ctx, "mhlo.memory_kind", mlir.stringAttribute(compilation_context.mlir_ctx, output_memory_kind.pjrtName()))); - } - _ = dialects.func.returns(compilation_context.mlir_ctx, output_info.values, .unknown(compilation_context.mlir_ctx)).appendTo(compilation_context.currentScope().block); - - const mlir_func = dialects.func.func(compilation_context.mlir_ctx, .{ - .name = "main", - .block = compilation_context.currentScope().block, - .location = .unknown(compilation_context.mlir_ctx), - .args_attributes = try finalizeAttributeList(arena.allocator(), compilation_context.mlir_ctx, input_attributes), - .results_attributes = try finalizeAttributeList(arena.allocator(), compilation_context.mlir_ctx, output_attributes), - }); - - compilation_context.mlir_pass_manager.runOnOp(mlir_func) catch |err| switch (err) { - error.MlirUnexpected => { - std.log.err("Failed to canonicalize invalid mlir: {f}", .{mlir_func}); - // user errors should have triggered a panic before we reach this. - @panic("ZML generated invalid mlir. Please open a bug report"); - }, - }; - - return .{ - .func = mlir_func, - .input_info = input_info, - .output_info = output_info, - }; -} - -fn setXlaOverrideFlag(map: *c.upb_Map, flag: []const u8, value: anytype, upb_arena: *c.upb_Arena) !void { - const result = c.upb_Map_Set( - map, - .{ .str_val = upb.stringView(flag) }, - .{ .msg_val = blk: { - const field = try upb.new(c.xla_OptionOverrideProto, upb_arena); - switch (@typeInfo(@TypeOf(value))) { - .bool => c.xla_OptionOverrideProto_set_bool_field(field, value), - .comptime_int, .int => c.xla_OptionOverrideProto_set_int_field(field, @intCast(value)), - .comptime_float, .float => c.xla_OptionOverrideProto_set_double_field(field, @floatCast(value)), - else => c.xla_OptionOverrideProto_set_string_field(field, upb.stringView(value)), - } - break :blk @ptrCast(field); - } }, - upb_arena, - ); - - if (result == false) { - return std.mem.Allocator.Error.OutOfMemory; - } -} - -fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: Platform, module: *const mlir.Module, xla_dump_to_: ?[]const u8) !*pjrt.LoadedExecutable { - //const tracer = Tracer.init("ai.zml.compilation"); - //const compile_frame = tracer.frameStart("pjrt compilation"); - //defer tracer.frameEnd(compile_frame, "pjrt compilation"); - - const sharding = platform.sharding(); - - var upb_alloc: upb.Allocator = .init(arena); - const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); - defer c.upb_Arena_Free(upb_arena); - - const options = blk: { - const options = try upb.new(c.xla_CompileOptionsProto, upb_arena); - c.xla_CompileOptionsProto_set_executable_build_options(options, executable_build_options_blk: { - const exec_build_options = try upb.new(c.xla_ExecutableBuildOptionsProto, upb_arena); - - c.xla_ExecutableBuildOptionsProto_set_device_ordinal(exec_build_options, -1); - c.xla_ExecutableBuildOptionsProto_set_num_replicas(exec_build_options, sharding.num_replicas); - c.xla_ExecutableBuildOptionsProto_set_num_partitions(exec_build_options, sharding.num_partitions); - c.xla_ExecutableBuildOptionsProto_set_use_spmd_partitioning(exec_build_options, sharding.num_partitions > 1 or sharding.num_replicas > 1); - - c.xla_ExecutableBuildOptionsProto_set_device_assignment(exec_build_options, device_assignment_blk: { - const device_assignment = try upb.new(c.xla_DeviceAssignmentProto, upb_arena); - - c.xla_DeviceAssignmentProto_set_replica_count(device_assignment, sharding.num_replicas); - c.xla_DeviceAssignmentProto_set_computation_count(device_assignment, sharding.num_partitions); - - const computation_devices = c.xla_DeviceAssignmentProto_resize_computation_devices(device_assignment, sharding.num_partitions, upb_arena); - for (computation_devices[0..sharding.num_partitions], 0..) |*computation_device, i| { - computation_device.* = try upb.new(c.xla_DeviceAssignmentProto_ComputationDevice, upb_arena); - _ = c.xla_DeviceAssignmentProto_ComputationDevice_add_replica_device_ids(computation_device.*, @intCast(i), upb_arena); - } - break :device_assignment_blk device_assignment; - }); - - break :executable_build_options_blk exec_build_options; - }); - - const overrides_map = c._xla_CompileOptionsProto_env_option_overrides_mutable_upb_map(options, upb_arena); - switch (platform.target) { - .cuda => { - // NVIDIA recommends these settings - // https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_triton_gemm", false, upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_latency_hiding_scheduler", true, upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_llvm_module_compilation_parallelism", true, upb_arena); - }, - .rocm => { - // Disable Triton GEMM on ROCM. For some reason it's much, much slower when - // enabled on CDNA and it's used on RDNA. Disable it altogether. - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_triton_gemm", false, upb_arena); - // Use lld from libllvm instead of invoking the ld.lld binary. - // This saves us from having to sandbox it. - try setXlaOverrideFlag(overrides_map, "xla_gpu_use_inprocess_lld", true, upb_arena); - }, - else => {}, - } - - if (xla_dump_to_ orelse platform.compilation_options.xla_dump_to) |xla_dump_to| { - try setXlaOverrideFlag(overrides_map, "xla_dump_to", xla_dump_to, upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_dump_hlo_as_proto", true, upb_arena); - if (platform.compilation_options.xla_dump_fusion_visualization) { - try setXlaOverrideFlag(overrides_map, "xla_dump_fusion_visualization", true, upb_arena); - } - if (platform.compilation_options.xla_dump_hlo_pass_re) |re| { - try setXlaOverrideFlag(overrides_map, "xla_dump_hlo_pass_re", re, upb_arena); - } - } - - break :blk options; - }; - - const loaded_executable = try platform.pjrt_client.compile( - platform.pjrt_api, - arena, - io, - module, - try upb.serialize(options, upb_arena), - ); - errdefer loaded_executable.deinit(); - - return loaded_executable; -} diff --git a/zml2/nn.zig b/zml2/nn.zig deleted file mode 100644 index d244f235..00000000 --- a/zml2/nn.zig +++ /dev/null @@ -1,1383 +0,0 @@ -//! Common layer definition and functions for Neural Networks (NN) -const std = @import("std"); - -const stdx = @import("stdx"); - -const ConstSlice = @import("slice.zig").ConstSlice; -const DataType = @import("dtype.zig").DataType; -const meta = @import("meta.zig"); -const ops = @import("ops.zig"); -const Shape = @import("shape.zig").Shape; -const Tensor = @import("tensor.zig").Tensor; -const zml = @import("zml.zig"); - -const log = std.log.scoped(.@"zml/nn"); - -pub const Linear = struct { - weight: Tensor, - bias: ?Tensor = null, - tag: Shape.Tag, - - pub fn init(weight: Tensor, bias: ?Tensor, tag: anytype) Linear { - return .{ - .weight = weight, - .bias = bias, - .tag = zml.Shape.toTag(tag), - }; - } - - pub fn forward(self: Linear, x: Tensor) Tensor { - var y = x.dot(self.weight.convert(x.dtype()), self.tag); - - // log.debug("Linear({*}): {d} -> {d} -> {d}", .{ self, x.dims(), y.dims(), if (self.bias) |bias| y.add(bias).dims() else y.dims() }); - return if (self.bias) |bias| y.add(bias.broad(y.shape())) else y; - } -}; - -pub const TokenEmbedding = struct { - weight: Tensor, - - pub fn forward(self: TokenEmbedding, idx: Tensor) Tensor { - stdx.debug.assert(idx.dtype().isInteger(), "TokenEmbedding expects an integer input, received: {f}", .{idx}); - stdx.debug.assert(self.weight.rank() == 2, "TokenEmbedding expects it's weight Tensor to be a 2D matrix, got {f}", .{self.weight}); - return self.weight.withTags(.{ .voc, .d }).gather(.{ .voc = idx }, .{}); - } -}; - -pub const Activation = union(enum) { - sigmoid, - tanh, - relu, - leakyReLU: f32, - elu: f32, - silu, - gelu, - quick_gelu, - - pub fn forward(self: Activation, x: Tensor) Tensor { - return switch (self) { - .sigmoid => x.sigmoid(), - .tanh => x.tanh(), - .relu => x.relu(), - .silu => x.silu(), - .gelu => x.gelu(), - .elu => |alpha| elu(x, alpha), - .quick_gelu => x.quickGelu(), - .leakyReLU => |slope| x.leakyReLU(slope), - }; - } -}; - -pub fn elu(x: Tensor, alpha: f32) Tensor { - return x.cmp(.GE, Tensor.scalar(0, x.dtype())).select( - x, - x.exp().addConstant(-1).scale(alpha), - ); -} - -/// Layer Normalization -pub const LayerNorm = struct { - weight: Tensor, - bias: ?Tensor = null, - eps: f32 = 1e-5, - - pub fn forward(self: LayerNorm, x: Tensor) Tensor { - const normed = normalizeVariance(x, self.eps); - const ax = x.axis(-1); - var out = normed.mul(self.weight.broadcast(x.shape(), &.{ax})); - if (self.bias) |bias| out = out.add(bias.broadcast(x.shape(), &.{ax})); - - return out; - } -}; - -pub fn rmsNorm(x: Tensor, axis: anytype, eps: f32) Tensor { - const ax = x.axis(axis); - // upcast to improve precision - const variance = x.convert(.f32).powByConst(2).mean(ax); - const rsqrt = Tensor.rsqrt(variance.addConstant(eps)).convert(x.dtype()); - return x.mul(rsqrt.broad(x.shape())); -} - -/// Center and scale by the variance. -/// normalize(x, eps) = (x - mean(x)) / sqrt(var(x) + eps) -/// Work on the last axis. -pub fn normalizeVariance(x: Tensor, eps: f32) Tensor { - const N: f32 = @floatFromInt(x.dim(-1)); - - // Upcast to improve precision - const xf32 = x.convert(.f32); - const mean = xf32.sum(-1).scale(1.0 / N); - const mean_dev = xf32.sub(mean); - const variance = mean_dev.mul(mean_dev).sum(-1).divByConst(N); - const rsqrt = Tensor.rsqrt(variance.addConstant(eps)); - - return mean_dev.mul(rsqrt).convert(x.dtype()); -} - -// ref: https://pytorch.org/docs/stable/generated/torch.nn.functional.normalize.html -// Implementation equivalent to `nn.functional.normalize(tensor, dim=-1)` call -pub fn normalizeL2(input: Tensor, eps: f32) Tensor { - const inv_norm = input.powByConst(2).sum(-1).addConstant(eps).rsqrt(); - return input.mul(inv_norm); -} - -test normalizeL2 { - const platform = zml.testing.env(); - - const input: zml.Tensor = .init(.{ 2, 2 }, .f32); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, normalizeL2, .{ input, 1e-12 }, platform); - defer exe.deinit(); - - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.9686, -1.0058, -1.7808, 0.6698 })); - defer input_buffer.deinit(); - - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, normalizeL2, .{input_buffer}); - defer res.deinit(); - - const expectation: ConstSlice = .init(input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6937, -0.7203, -0.9360, 0.3520 })); - try zml.testing.expectClose(std.testing.io, expectation, res, 1e-4); -} - -pub const RopeOpts = struct { - layout: Layout = .sequential, - freq_base: f32 = 10_000, - scaling: Scaling = .default, - - /// There are two layouts corresponding to how to split `x` in real/imag parts. - /// * Interleaved means that the real/imag of each scalar is contiguous. - /// * Sequential means that you first read all real values then all imag values. - /// Typically HF models use sequential, while GGUF use interleaved. - pub const Layout = enum { interleaved, sequential }; - - /// There are several ways to init the scaling aka "inv_freq" - pub const Scaling = union(enum) { - default: void, - custom: []const f32, - llama3: Llama3, - yarn: Yarn, - - pub const Llama3 = struct { - factor: f32, - high_freq_factor: f32, - low_freq_factor: f32, - original_max_position_embeddings: u32, - truncate: bool = true, - }; - - pub const Yarn = struct { - beta_fast: f32 = 32.0, - beta_slow: f32 = 1.0, - factor: f32, - truncate: bool = true, - original_max_position_embeddings: u32, - }; - - /// Read a Rope scaling config from HF config.json format. - pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !Scaling { - const content = try std.json.Value.jsonParse(allocator, source, options); - if (content == .null) return .default; - - if (content != .object) return error.InvalidEnumTag; - - const obj = content.object; - const impl = obj.get("rope_type") orelse return error.MissingField; - if (impl != .string) return error.InvalidEnumTag; - if (std.mem.eql(u8, impl.string, "llama3")) { - // Note: leaky is fine here cause Llama3 struct don't need to allocate memory. - return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, stdx.noalloc, content, .{ .ignore_unknown_fields = true }) }; - } else if (std.mem.eql(u8, impl.string, "yarn")) { - return .{ .yarn = try std.json.parseFromValueLeaky(Yarn, stdx.noalloc, content, .{ .ignore_unknown_fields = true }) }; - } else { - log.warn("Unsupported Rope implementation: {s}, will use the default one which will produce altered results", .{impl.string}); - return .{ .default = {} }; - } - } - - pub fn attentionScaling(scaling: Scaling) f32 { - return switch (scaling) { - .yarn => |yarn| 0.1 * @log(yarn.factor) + 1.0, - else => 1.0, - }; - } - }; -}; - -/// Rotary position embedding modify queries and keys tensor before compute Q * K in self attention. -/// This biases a token to look at token near him. -/// The nice thing with rope is that you can cache the modified queries and keys directly. -/// See: https://paperswithcode.com/method/rope -/// -/// Expected shapes of tensor: -/// - x: .{ .s, .hd } where .s is the sequence length and .hd the head dimension -/// - pos_idx: optional tensor which indicates which positions are needed. -/// When not set `rope` return all positions from 0 to x.dim(.s) which is the max seq len. -pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { - stdx.debug.assert(@mod(x.dim(.hd), 2) == 0, "rope expects a even head dim (.hd), got {f}", .{x}); - - const idx = if (pos_idx) |idx| blk: { - stdx.debug.assert(x.shape().hasTags(.{.hd}), "rope expects x argument to have .hd axes got: rope(x={f}, idx={f})", .{ x, idx }); - break :blk idx; - } else blk: { - stdx.debug.assert(x.shape().hasTags(.{ .s, .hd }), "rope expects x argument to have both .s and .hd axes got: rope(x={f})", .{x}); - break :blk Tensor.arange(.{ .end = x.dim(.s) }, .f32).withTags(.{.s}); - }; - const x_real, const x_imag = splitRealImg(x, opts.layout); - - // compute sin and cos in f32 before downcasting to x type. - const inv_freq = invFreq(x.dim(.hd), opts).withTags(.{.hd}); - const inv_freq_pos = Tensor.outer(idx.convert(.f32), inv_freq); - const scaling = opts.scaling.attentionScaling(); - const cos = inv_freq_pos.cos().scale(scaling).convert(x.dtype()).broad(x_real.shape()); - const sin = inv_freq_pos.sin().scale(scaling).convert(x.dtype()).broad(x_real.shape()); - - // apply rotation - const y_real = x_real.mul(cos).sub(x_imag.mul(sin)); - const y_imag = x_real.mul(sin).add(x_imag.mul(cos)); - - // flatten last dimensions - return mergeRealImg(y_real, y_imag, opts.layout); -} - -pub fn splitRealImg(x: Tensor, layout: RopeOpts.Layout) [2]Tensor { - const n = x.dim(-1); - - return switch (layout) { - .sequential => .{ - x.slice1d(-1, .{ .end = @divExact(n, 2) }), - x.slice1d(-1, .{ .start = @divExact(n, 2), .end = n }), - }, - .interleaved => .{ - x.slice1d(-1, .{ .start = 0, .step = 2 }), - x.slice1d(-1, .{ .start = 1, .step = 2 }), - }, - }; -} - -pub fn mergeRealImg(x_real: Tensor, x_imag: Tensor, layout: RopeOpts.Layout) Tensor { - return switch (layout) { - .sequential => Tensor.concatenate(&.{ x_real, x_imag }, -1), - .interleaved => Tensor.concatenate(&.{ - x_real.appendAxes(.{.interleaved_real_img}), - x_imag.appendAxes(.{.interleaved_real_img}), - }, -1).reshape(x_imag.shape().setDim(-1, -1)), - }; -} - -/// {exp( - n * ln(10_000) / N ) | n in [0..N] } -fn invFreq(N: i64, opts: RopeOpts) Tensor { - const allocator = zml.module.CompilationContext.current().allocator; - const N_half: usize = @intCast(@divExact(N, 2)); - const inv_freq = allocator.alloc(f32, N_half) catch @panic("OOM"); - defer allocator.free(inv_freq); - - _invFreq(opts, inv_freq); - return zml.Tensor.constantTensor(zml.Shape.init(.{@divExact(N, 2)}, .f32), std.mem.sliceAsBytes(inv_freq)); -} - -fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { - const N = inv_freq.len; - // Default frequencies - for (0.., inv_freq) |n, *f| { - f.* = @exp(-@log(opts.freq_base) * stdx.math.divFloat(f32, n, N)); - } - - switch (opts.scaling) { - .default => {}, - .custom => { - stdx.debug.assert(opts.scaling.custom.len == N, "rope expected custom inv_freq to match half head dimension {d}, got {d}", .{ N, opts.scaling.custom.len }); - @memcpy(inv_freq, opts.scaling.custom); - }, - .llama3 => |s| { - // https://arxiv.org/pdf/2309.16039 - // After Llama2 they observed that the rope frequencies where too sharp and hurting long distance attention. - // In Llama3 they used a higher base freq and also downscaled low frequencies. - std.debug.assert(s.low_freq_factor < s.high_freq_factor); - const M: f64 = @floatFromInt(s.original_max_position_embeddings); - const f_high = s.high_freq_factor * (2 * std.math.pi) / M; - const f_low = s.low_freq_factor * (2 * std.math.pi) / M; - const downscaling = 1.0 / s.factor; - - for (0..N, inv_freq) |n, f| { - if (f > f_high) { - // High freq match default implem - } else if (f < f_low) { - // Downscaling for low freq - inv_freq[n] *= downscaling; - } else { - // Linear interpolation for middle freq - const lerp: f64 = (inv_freq[n] - f_low) / (f_high - f_low); - inv_freq[n] *= @floatCast(lerp + (1 - lerp) * downscaling); - } - } - }, - .yarn => |s| { - const N_f: f64 = @floatFromInt(inv_freq.len); - const M: f64 = @floatFromInt(s.original_max_position_embeddings); - const f_high = s.beta_fast * (2 * std.math.pi) / M; - const f_low = s.beta_slow * (2 * std.math.pi) / M; - const downscaling = 1.0 / s.factor; - - // This isn't a typo: low n have a high frequency, high n have a low frequency. - var n_low: f64 = -@log(f_high) / @log(opts.freq_base) * N_f; - var n_high: f64 = -@log(f_low) / @log(opts.freq_base) * N_f; - if (s.truncate) { - n_high = std.math.ceil(n_high); - n_low = std.math.floor(n_low); - } - std.debug.assert(n_high > n_low); - for (0..N, inv_freq) |n, f| { - if (f > f_high) { - // High freq match default implem - } else if (f < f_low) { - // Downscaling for low freq - inv_freq[n] *= downscaling; - } else { - // Yarn use lerp too but not in the frequency space, in the time space. - const lerp: f64 = (n_high - @as(f64, @floatFromInt(n))) / (n_high - n_low); - inv_freq[n] *= @floatCast(lerp + (1 - lerp) * downscaling); - } - } - }, - } -} - -test "invFreq Llama3" { - // Llama 3.2-1B config - const llama_conf: RopeOpts = .{ - .freq_base = 500_000, - .scaling = .{ .llama3 = .{ - .factor = 32, - .high_freq_factor = 4, - .low_freq_factor = 1, - .original_max_position_embeddings = 8192, - } }, - }; - const llama_freq = [_]f32{ 1.000000000000e+00, 6.636012792587e-01, 4.403666257858e-01, 2.922278344631e-01, 1.939227581024e-01, 1.286873817444e-01, 8.539710193872e-02, 5.666961893439e-02, 3.760603070259e-02, 2.495540864766e-02, 1.656044088304e-02, 1.098952908069e-02, 7.292665075511e-03, 4.839421249926e-03, 3.211446106434e-03, 1.290548010729e-03, 4.295567050576e-04, 9.708286233945e-05, 1.946163865796e-05, 1.291476746701e-05, 8.570255886298e-06, 5.687232260243e-06, 3.774054448513e-06, 2.504467147446e-06, 1.661967417022e-06, 1.102883629756e-06, 7.318749339902e-07, 4.856731266045e-07, 3.222932889457e-07, 2.138742303259e-07, 1.419272024350e-07, 9.418306490261e-08 }; - - var inv_freq: @TypeOf(llama_freq) = undefined; - _invFreq(llama_conf, &inv_freq); - for (llama_freq, inv_freq, 0..) |expected, actual, i| { - errdefer log.err("Mismatch at position {d}.\nExpected: {any}\nActual: {any}", .{ i, llama_freq, inv_freq }); - try std.testing.expectApproxEqRel(expected, actual, 1e-5); - } -} - -test "invFreq Yarn" { - const yarn_conf: RopeOpts = .{ - .freq_base = 150_000, - .scaling = .{ .yarn = .{ - .factor = 32.0, - .beta_fast = 32.0, - .beta_slow = 1.0, - .original_max_position_embeddings = 4096, - .truncate = true, - } }, - }; - const yarn_freq = [_]f32{ 1.000000000000e+00, 6.890442967415e-01, 4.747820496559e-01, 3.271458745003e-01, 2.254180014133e-01, 1.553229838610e-01, 1.070244237781e-01, 7.374456524849e-02, 5.081327259541e-02, 3.162075206637e-02, 1.945096626878e-02, 1.179219130427e-02, 7.015713956207e-03, 4.069554619491e-03, 2.277272054926e-03, 1.206130953506e-03, 5.809474969283e-04, 2.279478358105e-04, 3.830881178146e-05, 2.639646845637e-05, 1.818833698053e-05, 1.253256959899e-05, 8.635495760245e-06, 5.950239483354e-06, 4.099978468730e-06, 2.825066758305e-06, 1.946596285052e-06, 1.341290953860e-06, 9.242089618056e-07, 6.368209142238e-07, 4.387978549403e-07, 3.023511396805e-07 }; - - var inv_freq: @TypeOf(yarn_freq) = undefined; - _invFreq(yarn_conf, &inv_freq); - for (yarn_freq, inv_freq, 0..) |expected, actual, i| { - errdefer log.err("Mismatch at position {d}.\nExpected: {d}\nActual: {d}", .{ i, stdx.fmt.slice(&yarn_freq), stdx.fmt.slice(&inv_freq) }); - try std.testing.expectApproxEqRel(expected, actual, 1e-5); - } - try std.testing.expectApproxEqRel(1.3465735902799727, yarn_conf.scaling.attentionScaling(), 1e-5); -} - -test "real/img" { - const platform = zml.testing.env(); - - const Fns = struct { - fn testSplitMergeIsId(layout: RopeOpts.Layout) Tensor { - const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ 5, 4 }); - const real, const imag = splitRealImg(x, layout); - const y = mergeRealImg(real, imag, layout); - const real2, const imag2 = splitRealImg(y, layout); - return real.cmp(.EQ, real2).flatten().convert(.i32).sum(-1).add( - imag.cmp(.EQ, imag2).flatten().convert(.i32).sum(-1), - ); - } - - fn testSplitSeqVoid(_: void) Tensor { - const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ 5, 4 }); - const real, const imag = splitRealImg(x, .sequential); - const x_real = Tensor.concatenate(&.{ - Tensor.arange(.{ .start = 0, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), - Tensor.arange(.{ .start = 1, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), - }, 1); - const x_imag = Tensor.concatenate(&.{ - Tensor.arange(.{ .start = 2, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), - Tensor.arange(.{ .start = 3, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), - }, 1); - - return real.cmp(.EQ, x_real).flatten().convert(.i32).sum(-1).add( - imag.cmp(.EQ, x_imag).flatten().convert(.i32).sum(-1), - ); - } - - fn testSplitSeq() Tensor { - const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ 5, 4 }); - const real, const imag = splitRealImg(x, .sequential); - const x_real = Tensor.concatenate(&.{ - Tensor.arange(.{ .start = 0, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), - Tensor.arange(.{ .start = 1, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), - }, 1); - const x_imag = Tensor.concatenate(&.{ - Tensor.arange(.{ .start = 2, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), - Tensor.arange(.{ .start = 3, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), - }, 1); - - return real.cmp(.EQ, x_real).flatten().convert(.i32).sum(-1).add( - imag.cmp(.EQ, x_imag).flatten().convert(.i32).sum(-1), - ); - } - - fn testSplitInterleaved() Tensor { - const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ 5, 4 }); - const real, const imag = splitRealImg(x, .interleaved); - const x_real = Tensor.arange(.{ .start = 0, .end = 20, .step = 2 }, .f32).reshape(.{ 5, 2 }); - const x_imag = Tensor.arange(.{ .start = 1, .end = 20, .step = 2 }, .f32).reshape(.{ 5, 2 }); - - return real.cmp(.EQ, x_real).flatten().convert(.i32).sum(-1).add( - imag.cmp(.EQ, x_imag).flatten().convert(.i32).sum(-1), - ); - } - }; - { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.interleaved}, platform); - defer exe.deinit(); - - var d_interleaved = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitMergeIsId, {}); - defer d_interleaved.deinit(); - try std.testing.expectEqual(20, try d_interleaved.getValue(i32, std.testing.io)); - } - { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.sequential}, platform); - defer exe.deinit(); - - var d_sequential = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitMergeIsId, {}); - defer d_sequential.deinit(); - try std.testing.expectEqual(20, try d_sequential.getValue(i32, std.testing.io)); - } - - // test the function that accepts 1 void argument - { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitSeqVoid, .{{}}, platform); - defer exe.deinit(); - - var d_split_seq_void = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitSeqVoid, {}); - defer d_split_seq_void.deinit(); - try std.testing.expectEqual(20, try d_split_seq_void.getValue(i32, std.testing.io)); - } - - // test the function that takes NO arguments - { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitSeq, .{}, platform); - defer exe.deinit(); - - var d_split_seq = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitSeq, {}); - defer d_split_seq.deinit(); - try std.testing.expectEqual(20, try d_split_seq.getValue(i32, std.testing.io)); - } - - { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitInterleaved, .{}, platform); - defer exe.deinit(); - - var d_split_seq = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitInterleaved, {}); - defer d_split_seq.deinit(); - try std.testing.expectEqual(20, try d_split_seq.getValue(i32, std.testing.io)); - } -} - -test rope { - const platform = zml.testing.env(); - - const Local = struct { - fn _fwd(x: Tensor, opts: RopeOpts) Tensor { - var input = x; - { - // Convert input to the requested format - const real, const imag = splitRealImg(input, .sequential); - input = mergeRealImg(real, imag, opts.layout); - } - var res = rope(input, null, opts).squeeze(0); - - { - // Convert back to sequential - const real, const imag = splitRealImg(res, opts.layout); - res = mergeRealImg(real, imag, .sequential); - } - return res; - } - }; - - // x is made such as the interleaved and sequential reps are the same. - // So the two implementations should give the same results. - const x: zml.Tensor = .init(.{ .b = 1, .s = 5, .hd = 4 }, .f32); - var exe_interleaved = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .interleaved } }, platform); - defer exe_interleaved.deinit(); - - var exe_sequential = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .sequential } }, platform); - defer exe_sequential.deinit(); - - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ 1.0, 0.1, -1.0, -0.5 } ** 5)); - defer x_buffer.deinit(); - - var res1 = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_interleaved, Local._fwd, .{x_buffer}); - defer res1.deinit(); - var res2 = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_sequential, Local._fwd, .{x_buffer}); - defer res2.deinit(); - - try zml.testing.expectClose(std.testing.io, res1, res2, 1e-4); -} - -pub const UpsampleMode = enum { - nearest, - // TODO: Linear, - // TODO: Bilinear, - // TODO: Bicubic, - // TODO: Trilinear, -}; - -/// Upsample -pub fn upsample( - input: Tensor, - opts: struct { mode: UpsampleMode, scale_factor: []const f64 }, -) Tensor { - // TODO(james): make `nearest` compatible with resizeBilinear and resizeBicubic, and wrap them here. - // resize* have API which are more explicit, this assume you want to scale the N-2 last axes. - stdx.debug.assert(3 <= input.rank() and input.rank() <= 5, "upsample is only implemented for (3,4,5)-D tensors, received {f}", .{input}); - stdx.debug.assert(opts.scale_factor.len == 1 or opts.scale_factor.len == input.rank() - 2, "scale factors", .{}); - return switch (opts.mode) { - .nearest => { - var scale_factors: [3]f64 = undefined; - switch (opts.scale_factor.len) { - 1 => { - for (0..input.rank() - 2) |i| scale_factors[i] = opts.scale_factor[0]; - }, - else => @memcpy(scale_factors[0..opts.scale_factor.len], opts.scale_factor), - } - return nearest(input, scale_factors[0 .. input.rank() - 2]); - }, - }; -} - -pub fn nearest(input: Tensor, scale_factor: []const f64) Tensor { - var out_shape = input.shape(); - for (scale_factor, 0..) |sf, i| { - out_shape._dims.set(i + 2, @intFromFloat(@floor(@as(f64, @floatFromInt(out_shape.dim(i + 2))) * sf))); - } - // TODO(james): remove this implicit two batching dims - var sd: [3]u3 = undefined; - var len_sd: usize = 0; - for (2..input.rank()) |i| { - if (input.dim(i) != out_shape.dim(i)) { - sd[len_sd] = @intCast(i); - len_sd += 1; - } - } - const spatial_axes = sd[0..len_sd]; - var res = input; - for (spatial_axes) |ax| { - const n = out_shape.dim(ax); - const ratio = stdx.math.divFloat(f32, input.dim(ax), n); - const offsets = Tensor.arange(.{ .end = n }, .f32).addConstant(0.5).scale(ratio).floor().convert(.i32); - res = ops.gather(res, &.{ax}, &.{offsets}, .{ .indices_are_sorted = true }); - } - return res; -} - -test nearest { - const platform = zml.testing.env(); - - // 3D Tensor (basic) - { - const input_3d_basic: zml.Tensor = .init(.{ 1, 1, 2 }, .i32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_3d_basic, .{ .scale_factor = &.{3}, .mode = .nearest } }, platform); - defer exe.deinit(); - - var input_3d_basic_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_3d_basic.shape(), std.mem.sliceAsBytes(&[1][1][2]i32{.{.{ 1, 2 }}})); - defer input_3d_basic_buffer.deinit(); - - const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_3d_basic_buffer}); - defer result.deinit(); - - try std.testing.expectEqualSlices(i64, &.{ 1, 1, 6 }, result.shape().dims()); - const expected: ConstSlice = .init(Shape.init(.{ 1, 1, 6 }, .i32), std.mem.sliceAsBytes(&[1][1][6]i32{.{.{ 1, 1, 1, 2, 2, 2 }}})); - try zml.testing.expectClose(std.testing.io, expected, result, 0); - } - - // 3D Tensor (advanced) - { - const input_3d_advanced: zml.Tensor = .init(.{ 2, 3, 4 }, .i32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_3d_advanced, .{ .scale_factor = &.{2}, .mode = .nearest } }, platform); - defer exe.deinit(); - - var input_3d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_3d_advanced.shape(), std.mem.sliceAsBytes(&[2][3][4]i32{ - .{ .{ 1, 2, 3, 4 }, .{ 5, 6, 7, 8 }, .{ 9, 10, 11, 12 } }, - .{ .{ 13, 14, 15, 16 }, .{ 17, 18, 19, 20 }, .{ 21, 22, 23, 24 } }, - })); - defer input_3d_advanced_buffer.deinit(); - - const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_3d_advanced_buffer}); - defer result.deinit(); - - try std.testing.expectEqualSlices(i64, &.{ 2, 3, 8 }, result.shape().dims()); - const expected: ConstSlice = .init(Shape.init(.{ 2, 3, 8 }, .i32), std.mem.sliceAsBytes(&[2][3][8]i32{ - .{ - .{ 1, 1, 2, 2, 3, 3, 4, 4 }, - .{ 5, 5, 6, 6, 7, 7, 8, 8 }, - .{ 9, 9, 10, 10, 11, 11, 12, 12 }, - }, - .{ - .{ 13, 13, 14, 14, 15, 15, 16, 16 }, - .{ 17, 17, 18, 18, 19, 19, 20, 20 }, - .{ 21, 21, 22, 22, 23, 23, 24, 24 }, - }, - })); - try zml.testing.expectClose(std.testing.io, expected, result, 0); - } - - // 4D Tensor (basic) - { - const input_4d_basic: zml.Tensor = .init(.{ 1, 1, 2, 2 }, .i32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_4d_basic, .{ .scale_factor = &.{ 3, 3 }, .mode = .nearest } }, platform); - defer exe.deinit(); - - var input_4d_basic_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_basic.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); - defer input_4d_basic_buffer.deinit(); - - const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_4d_basic_buffer}); - defer result.deinit(); - - try std.testing.expectEqualSlices(i64, &.{ 1, 1, 6, 6 }, result.shape().dims()); - const expected: ConstSlice = .init(Shape.init(.{ 1, 1, 6, 6 }, .i32), std.mem.sliceAsBytes(&[1][1][6][6]i32{.{.{ - .{ 1, 1, 1, 2, 2, 2 }, - .{ 1, 1, 1, 2, 2, 2 }, - .{ 1, 1, 1, 2, 2, 2 }, - .{ 3, 3, 3, 4, 4, 4 }, - .{ 3, 3, 3, 4, 4, 4 }, - .{ 3, 3, 3, 4, 4, 4 }, - }}})); - try zml.testing.expectClose(std.testing.io, expected, result, 0); - } - // 4D Tensor (advanced) - { - const input_4d_advanced: zml.Tensor = .init(.{ 2, 2, 2, 2 }, .i32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_4d_advanced, .{ .scale_factor = &.{ 2, 2 }, .mode = .nearest } }, platform); - defer exe.deinit(); - - var input_4d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_advanced.shape(), std.mem.sliceAsBytes(&[2][2][2][2]i32{ .{ - .{ .{ 1, 2 }, .{ 3, 4 } }, - .{ .{ 5, 6 }, .{ 7, 8 } }, - }, .{ - .{ .{ 9, 10 }, .{ 11, 12 } }, - .{ .{ 13, 14 }, .{ 15, 16 } }, - } })); - defer input_4d_advanced_buffer.deinit(); - - const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_4d_advanced_buffer}); - defer result.deinit(); - - try std.testing.expectEqualSlices(i64, &.{ 2, 2, 4, 4 }, result.shape().dims()); - const expected: ConstSlice = .init(Shape.init(.{ 2, 2, 4, 4 }, .i32), std.mem.sliceAsBytes(&[2][2][4][4]i32{ - .{ - .{ - .{ 1, 1, 2, 2 }, - .{ 1, 1, 2, 2 }, - .{ 3, 3, 4, 4 }, - .{ 3, 3, 4, 4 }, - }, - .{ - .{ 5, 5, 6, 6 }, - .{ 5, 5, 6, 6 }, - .{ 7, 7, 8, 8 }, - .{ 7, 7, 8, 8 }, - }, - }, - .{ - .{ - .{ 9, 9, 10, 10 }, - .{ 9, 9, 10, 10 }, - .{ 11, 11, 12, 12 }, - .{ 11, 11, 12, 12 }, - }, - .{ - .{ 13, 13, 14, 14 }, - .{ 13, 13, 14, 14 }, - .{ 15, 15, 16, 16 }, - .{ 15, 15, 16, 16 }, - }, - }, - })); - try zml.testing.expectClose(std.testing.io, expected, result, 0); - } - // 5D Tensor (basic) - { - const input_5d: zml.Tensor = .init(.{ 1, 1, 1, 2, 2 }, .i32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_5d, .{ .scale_factor = &.{2}, .mode = .nearest } }, platform); - defer exe.deinit(); - - var input_5d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_5d.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); - defer input_5d_buffer.deinit(); - - const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_5d_buffer}); - defer result.deinit(); - - try std.testing.expectEqualSlices(i64, &.{ 1, 1, 2, 4, 4 }, result.shape().dims()); - const expected: ConstSlice = .init(Shape.init(.{ 1, 1, 2, 4, 4 }, .i32), std.mem.sliceAsBytes(&[1][1][2][4][4]i32{ - .{ - .{ - .{ - .{ 1, 1, 2, 2 }, - .{ 1, 1, 2, 2 }, - .{ 3, 3, 4, 4 }, - .{ 3, 3, 4, 4 }, - }, - .{ - .{ 1, 1, 2, 2 }, - .{ 1, 1, 2, 2 }, - .{ 3, 3, 4, 4 }, - .{ 3, 3, 4, 4 }, - }, - }, - }, - })); - try zml.testing.expectClose(std.testing.io, expected, result, 0); - } -} - -pub const ResizeOpts = struct { - /// scalar tensor containing the original dimension of the image. - /// It can be different from the image shape, - /// if the image has been padded. - /// This allows to compile one module that handle different input image sizes. - original_len: ?Tensor = null, - - /// Internal precision to do the interpolation. - /// Result will always use the same dtype than the original. - /// If not set, will use the image dtype, unless it's an integer type, in which case f32 will be used. - precision: ?zml.DataType = null, -}; - -pub fn resizeBilinear(image: Tensor, resized_axes: anytype, opt: ResizeOpts) Tensor { - const new_size, const tags_ = Shape.parseStruct(u63, resized_axes); - var out = image; - for (new_size.constSlice(), tags_.constSlice()) |d, t| { - const ax = image.shape().axis(t); - const child_opt: ResizeOpts = .{ - .original_len = if (opt.original_len) |o| o.choose1d(0, ax) else null, - }; - out = resizeLinear1d(out, ax, d, child_opt); - } - return out; -} - -test resizeBilinear { - const platform = zml.testing.env(); - // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); - defer comp.deinit(); - comp.activate(); - defer comp.deactivate(); - - const block = @import("mlir").Block.init(&.{}, &.{}); - comp.pushBlock(block); - defer comp.popBlock(); - - inline for (.{ - .{ .{ .a = 10, .b = 10 }, .{ .a = 20 }, .{ .a = 20, .b = 10 } }, - .{ .{ .a = 10, .b = 10 }, .{ .b = 5 }, .{ .a = 10, .b = 5 } }, - .{ .{ .a = 10, .b = 10 }, .{ .a = 20, .b = 5 }, .{ .a = 20, .b = 5 } }, - }) |testcase| { - const x_shape, const resizing, const res_shape = testcase; - const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); - const y = resizeBilinear(x, resizing, .{}); - try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); - try std.testing.expect(y.value().owner().verify()); - } -} - -pub fn resizeLinear1d(image: Tensor, axis: i8, new_len: u63, opt: ResizeOpts) Tensor { - const ax = image.axis(axis); - const res_shape = image.shape().set(ax, new_len); - - const dtype = opt.precision orelse if (image.dtype().class() == .integer) .f32 else image.dtype(); - const og_len = opt.original_len orelse Tensor.scalar(image.dim(ax), dtype); - const ratio = og_len.convert(dtype).scale(stdx.math.divFloat(f32, 1, new_len)); - const scaled = Tensor.arange(.{ .end = new_len }, dtype).mul(ratio); - const left = scaled.floor(); - const right = left.addConstant(1); - - // TODO: check that two gather isn't too bad perf wise. - // Normally we should use gatherSlices to collect the values 2 by 2, - // but gatherSlices messes up with the order of axes. - const left_val = ops.gather(image, &.{ax}, &.{left.convert(.i32)}, .{ .indices_are_sorted = true }).convert(dtype); - const right_val = ops.gather(image, &.{ax}, &.{right.convert(.i32)}, .{ .indices_are_sorted = true }).convert(dtype); - - const left_weight = right.sub(scaled).broadcast(res_shape, &.{ax}); - const right_weight = scaled.sub(left).broadcast(res_shape, &.{ax}); - - const res = left_val.mul(left_weight).add(right_val.mul(right_weight)); - return res.convert(image.dtype()).withTags(image.shape().tags()); -} - -/// Bicubic interpolation of the given image. -/// Warning as of May 2024 the cpu backend don't optimize this very well -/// and is not able to merge the weighting with the gather, -/// leading to 20x slow down compared to STB implementation. -pub fn resizeBicubic(image: Tensor, resized_axes: anytype, opt: ResizeOpts) Tensor { - const new_size, const tags_ = Shape.parseStruct(u63, resized_axes); - var out = image; - for (new_size.constSlice(), tags_.constSlice()) |d, t| { - const ax = image.shape().axis(t); - const child_opt: ResizeOpts = .{ - .original_len = if (opt.original_len) |o| o.choose1d(0, ax) else null, - }; - out = resizeCubic1d(out, ax, d, child_opt); - } - return out; -} - -test resizeBicubic { - const platform = zml.testing.env(); - // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); - defer comp.deinit(); - comp.activate(); - defer comp.deactivate(); - - const block = @import("mlir").Block.init(&.{}, &.{}); - comp.pushBlock(block); - defer comp.popBlock(); - - inline for (.{ - .{ .{ .a = 10, .b = 10 }, .{ .a = 20 }, .{ .a = 20, .b = 10 } }, - .{ .{ .a = 10, .b = 10 }, .{ .b = 5 }, .{ .a = 10, .b = 5 } }, - .{ .{ .a = 10, .b = 10 }, .{ .a = 20, .b = 5 }, .{ .a = 20, .b = 5 } }, - }) |testcase| { - const x_shape, const resizing, const res_shape = testcase; - const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); - const y = resizeBicubic(x, resizing, .{}); - try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); - try std.testing.expect(y.value().owner().verify()); - } -} - -pub fn resizeCubic1d(image: Tensor, axis: i8, new_len: u63, opt: ResizeOpts) Tensor { - // Extract neighboring pixels from the image. - const dtype = opt.precision orelse if (image.dtype().class() == .integer) .f32 else image.dtype(); - const og_len = opt.original_len orelse Tensor.scalar(image.dim(axis), dtype); - - const ratio = og_len.convert(dtype).scale(stdx.math.divFloat(f32, 1, new_len)); - const scaled = Tensor.arange(.{ .end = new_len }, dtype).mul(ratio); - const t = scaled.sub(scaled.floor()); - const pos = Tensor.stack(&.{ - Tensor.constant(dtype.one()).broad(t.shape()), - t, - t.mul(t), - t.powByConst(3), - }, .last, ._interpolated); - - std.debug.assert(pos.dim(0) == new_len); - std.debug.assert(pos.dim(1) == 4); - - const neighbors = scaled.floor().addConstant(-1).convert(.i32).maximum(Tensor.scalar(0, .i32)); - - const values = image.renameAxis(axis, ._neighbors).gatherSlices( - Shape.init(.{ ._neighbors = 4 }, image.dtype()), - neighbors.appendAxes(.{.coord}), - .{ .indices_are_sorted = true }, - ).convert(dtype); - - const weights_: [4][4]f32 = .{ - .{ 0, 1, 0, 0 }, - .{ -0.5, 0, 0.5, 0 }, - .{ 1, -2.5, 2, -0.5 }, - .{ -0.5, 1.5, -1.5, 0.5 }, - }; - const weights = zml.Tensor.constantTensor(Shape.init(.{ 4, 4 }, .f32), std.mem.sliceAsBytes(&weights_)).convert(dtype).withTags(.{ ._interpolated, ._neighbors }); - - // actually do the interpolation. - // Note: ideally this matmul should be inlined with the gather, but that's currently not the case. - // TODO: not being able to use dot here is a bit annoying. - var res = values.dotGeneral(weights, &.{.{ values.axis(._neighbors), weights.axis(._neighbors) }}, &.{}); - res = pos.dotGeneral(res, &.{.{ pos.axis(._interpolated), res.axis(._interpolated) }}, &.{.{ 0, 0 }}); - - // the current axis is outputted in first position because it's a batching dim, put it back in place. - if (axis != 0) { - res = res.swapAxes(0, axis); - } - - // verify the shape - const res_shape = image.shape().set(axis, new_len); - // log.debug("resizeCubic1d: ({}, {}, {}, {}) -> {}", .{ image, axis, new_len, opt, res }); - std.debug.assert(std.mem.eql(i64, res_shape.dims(), res.dims())); - return res.convert(image.dtype()).withTags(image.shape()); -} - -/// Return causal attention masks for the given shape. -/// The last dimensions are -pub fn causalAttnMask( - attn_shape_: anytype, - dtype: DataType, - attn_window_len: ?u32, -) Tensor { - const attn_shape = Shape.init(attn_shape_, dtype); - stdx.debug.assert(attn_shape.rank() == 2, "causalAttnMask({f}) shape need to be exactly 2 axes", .{attn_shape}); - const qlen = attn_shape.dim(-2); - const q_idx = Tensor.iota(attn_shape, -2); - const klen = attn_shape.dim(-1); - const k_idx = Tensor.iota(attn_shape, -1); - - // all elements > main diagonal must be 0 - // (q_idx - window_len < k_idx <= q_idx) - var mask = k_idx.cmp(.LE, q_idx); - if (attn_window_len) |window_len| { - if (qlen >= window_len or klen >= window_len) { - const window_mask = q_idx.cmp(.LT, k_idx.addConstant(window_len)); - mask = mask.logical(.AND, window_mask); - } - } - - if (dtype.isFloat()) { - const zeros = Tensor.constant(dtype.zero()).broad(mask.shape()); - const minus_inf = Tensor.constant(dtype.minValue()).broad(mask.shape()); - mask = Tensor.select(mask, zeros, minus_inf); - } else { - mask = mask.convert(dtype); - } - - return mask; -} - -pub const SdpaOpts = struct { - attn_mask: ?Tensor = null, - scale: ?Tensor = null, - softmax_bias: ?Tensor = null, - allow_cudnn: bool = true, - // TODO: put a callback instead of all this field, - // so that -}; - -/// Scaled dot product attention. -/// -/// **Shapes**: -/// - q, result: .{ .h, .q, .hd } -/// - k, v: .{ .h, .k, .hd } -/// -/// Where: -/// - .h is the number of head -/// - .q is the number of queries -/// - .k is the number of keys -/// - .hd is the head dimension -/// -/// .h is allowed to differ from queries and keys as long as the key heads -/// can be repeated to match query heads. -pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { - var q, var k, var v = .{ q_, k_, v_ }; - - const err_template = "sdpa(q: {f}, k: {f}, v: {f}, attn: {?f}) is invalid ! "; - const err_args = .{ q, k, v, opts.attn_mask }; - stdx.debug.assert(q.shape().hasTags(.{ .h, .q, .hd }), err_template ++ "q is missing tags {{.h, .q, .hd}}", err_args); - stdx.debug.assert(k.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "k is missing tags {{.h, .k, .hd}}", err_args); - stdx.debug.assert(v.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "v is missing tags {{.h, .k, .hd}}", err_args); - - // TODO(Corentin): Re-enable that - //if (opts.allow_cudnn and cuda.canUseCudnnSdpa(q.shape()) and opts.softmax_bias == null) { - // return cuda.sdpa(q, k, v, opts); - //} - - // Handle different numbers of head by splitting q heads. - // This is a bit error prone in the sense that it depends of the layout of q heads. - // This is the Llama convention though. - q = q.splitAxis(.h, .{ .h = k.dim(.h), .hq = .auto }); - const attn_mask = if (opts.attn_mask) |m| m else null; - - const dims = collectDims(.{ .h, .q, .k, .hd }, &.{ q, k, v, attn_mask }, .strict) catch { - stdx.debug.panic(err_template ++ "Inputs have incompatible shapes.", err_args); - }; - const sqrtHeadDim: f32 = 1.0 / std.math.sqrt(@as(f32, @floatFromInt(dims.hd))); - const head_scaling = if (opts.scale) |s| s else Tensor.scalar(sqrtHeadDim, k.dtype()); - k = k.mul(head_scaling.convert(k.dtype())); - - var attn_weights = q.dot(k, .hd); - // log.debug("attn_weights : {f}, attn_mask : {?f}", .{ attn_weights, attn_mask }); - if (attn_mask) |mask| attn_weights = attn_weights.add(mask.broad(attn_weights.shape())); - attn_weights = attn_weights.convert(.f32); - attn_weights = if (opts.softmax_bias) |softmax_bias| attn: { - // The split is needed because we also split q ourselves. - // TODO: consider letting the user do that. - const bias = softmax_bias.splitAxis(.h, .{ .h = k.dim(.h), .hq = .auto }); - break :attn attn_weights.convert(.f32).softmaxBiased(.k, bias).convert(q.dtype()); - } else attn_weights.convert(.f32).softmax(.k).convert(q.dtype()); - - var attn = attn_weights.dot(v, .k); - return attn.transpose(q.shape()).merge(.{ .h = .{ .h, .hq } }); -} - -/// Options controlling generation. The default values correspond to greedy decoding. -pub const SamplingStrategy = struct { - topk: u32 = 1, - temperature: f32 = 1.0, -}; - -/// Given the output of the last layer of a LM with a `.voc` axis, -/// Compute indices for the next tokens, following the given sampling strategy. -/// Returns an integer tensor with a shape similar to the input, but without the .voc axis. -pub fn sampleTokens(activations: Tensor, opts: SamplingStrategy, rng: Tensor.Rng) struct { Tensor, Tensor.Rng } { - if (opts.topk <= 1) { - const next_tokens = activations.argMax(.voc).indices.squeeze(.voc); - return .{ next_tokens, rng }; - } - - const topk = activations.topK(.{ .topk = .voc }, opts.topk, .{}); - // After the topk, we don't have .voc values, anymore, only .topk. - var x = topk.values; - if (opts.temperature != 1.0) { - x = x.scale(1 / opts.temperature); - } - - // Gumbel reparametrization trick: - // Adding gumbel noise and taking the argmax is equivalent - // to sampling from the categorical distribution produced by the softmax. - // https://en.wikipedia.org/wiki/Gumbel_distribution#Gumbel_reparametrization_tricks - const next_rng, const gumbel_noise = rng.gumbel(x.shape()); - x = x.add(gumbel_noise); - const topk_idx = x.argMax(.topk).indices; - - // topk_idx is indices into topk.values ! so in the range [0, topk] - // Convert for the original indices from the full [0, voc] range. - const next_tokens = topk.indices.gather(.{ .topk = topk_idx.squeeze(.topk) }, .{}); - // log.debug("sampleTokens({}) -> {} -> {} -> {}", .{ activations, topk.indices, topk_idx, next_tokens }); - return .{ next_tokens, next_rng }; -} - -test sampleTokens { - const platform = zml.testing.env(); - - const rng: zml.Tensor.Rng = .init(); - const activations: zml.Tensor = .init(.{ .voc = 4 }, .f32); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, sampleTokens, .{ activations, .{ .topk = 4, .temperature = 2.0 }, rng }, platform); - defer exe.deinit(); - - var rng_buffer = try zml.Tensor.Rng.initBuffer(platform, 0xdeadbeef, std.testing.io); - defer rng_buffer._state.deinit(); - - const inf = std.math.inf(f32); - - inline for (.{ - .{ [_]f32{ inf, 3.0, 2.0, 1.0 }, 0 }, - .{ [_]f32{ -inf, 3.0, -inf, -inf }, 1 }, - .{ [_]f32{ 3.0, 2, inf, inf }, 2 }, - }) |logits_expected| { - const logits, const expected: i32 = logits_expected; - - const activations_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, activations.shape(), std.mem.sliceAsBytes(&logits)); - defer activations_buffer.deinit(); - - var sampled, rng_buffer = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, sampleTokens, .{ activations_buffer, rng_buffer }); - defer sampled.deinit(); - - try zml.testing.expectEqual(expected, try sampled.getValue(i32, std.testing.io)); - } -} - -pub const DynamicSamplingStrategy = struct { - max_top_k: u32, - top_k: Tensor, - temperature: Tensor, - top_p: Tensor, - min_p: Tensor, - - pub const Opts = struct { - top_k: u32, - temperature: f32 = 1.0, - top_p: f32 = 1.0, - min_p: f32 = 0.0, - }; - - pub fn init(dtype: DataType, max_top_k: u32) DynamicSamplingStrategy { - const scalar_float_shape = Shape.init(.{}, dtype); - const scalar_i32_shape = Shape.init(.{}, .i32); - return .{ - .max_top_k = max_top_k, - .top_k = .fromShape(scalar_i32_shape), - .temperature = .fromShape(scalar_float_shape), - .top_p = .fromShape(scalar_float_shape), - .min_p = .fromShape(scalar_float_shape), - }; - } - - pub fn makeBuffers( - io: std.Io, - platform: zml.Platform, - dtype: zml.DataType, - opts: Opts, - ) !zml.Bufferized(DynamicSamplingStrategy) { - return .{ - .top_k = try zml.Buffer.scalar(io, platform, opts.top_k, .i32), - .temperature = try zml.Buffer.scalar(io, platform, opts.temperature, dtype), - .top_p = try zml.Buffer.scalar(io, platform, opts.top_p, dtype), - .min_p = try zml.Buffer.scalar(io, platform, opts.min_p, dtype), - }; - } - - pub fn deinitBuffers(self: *zml.Bufferized(DynamicSamplingStrategy)) void { - self.top_k.deinit(); - self.temperature.deinit(); - self.top_p.deinit(); - self.min_p.deinit(); - } -}; - -/// Given the output of the last layer of a LM with a `.voc` axis, -/// Compute indices for the next tokens, following the given sampling strategy. -/// The dynamic sampling strategy is more expressive but top_p requires computing the softmax. -/// -/// Options are: -/// -/// * top_k: only sample among the k top scoring tokens, -/// * max_top_k: limit a compilation time what is the max possible runtime value for top_k, saving memory and compute by not having to fully sort the tokens. -/// * top_p: only sample among top scoring tokens whose probabilities sum up to top_p -/// * min_p: drop tokens whose probabilities are lower than a ratio of the most likely token -pub fn sampleTokensDynamic(logits: Tensor, opts: DynamicSamplingStrategy, rng: Tensor.Rng) struct { Tensor, Tensor.Rng } { - var x, const topk_indices = fixupLogits(logits, opts); - - // the rest is similar to sampleTokens - const next_rng, const gumbel_noise = rng.gumbel(x.shape()); - x = x.add(gumbel_noise); - - const topk_idx = x.argMax(.topk).indices; - const next_tokens = topk_indices.gather(.{ .voc = topk_idx.squeeze(.topk) }, .{}); - return .{ next_tokens, next_rng }; -} - -fn fixupLogits(logits: Tensor, opts: DynamicSamplingStrategy) [2]Tensor { - const min_inf = Tensor.constant(logits.dtype().minValue()); - - // First reduce the vocab size to a reasonable sub set of candidate. - const full_topk = if (opts.max_top_k > 0) - logits.topK(.{ .voc = .voc }, opts.max_top_k, .{ .descending = true }) - else - logits.sort(.voc, .{ .descending = true }); - - // After the topk, we don't have .voc indices, anymore, only topk. - var x = full_topk.values.rename(.{ .voc = .topk }); - // mask values above the dynamic top_k - x = Tensor.iota(x.shape(), .topk).cmp(.GE, opts.top_k).select(min_inf, x); - x = x.mul(opts.temperature); - - // if there are high values in x, softmax can overflow and will create nans in full probs - // this propagate to probs_sum and probs_max. - const probs = x.softmax(.topk); - const probs_sum = probs.cumulativeSum(.topk); - const probs_max = probs.slice1d(.topk, .{ .start = 0, .end = 1 }); - - const top_p = opts.top_p.broad(x.shape()); - const min_p = probs_max.mul(opts.min_p).broad(x.shape()); - - // * if first candidate has very high prob, then probs_sum is always greater than top_p and candidate is full false - // * if first candidate score is even bigger, the probs become Nan because of the softmax, - // then cmp is is full false, and candidate is full false too. - const candidate = probs_sum.cmp(.LE, top_p).logical(.AND, probs.cmp(.GE, min_p)); - // * so we explicitly always accept first candidate. - const first_token = Tensor.iota(x.shape(), .topk).cmp(.EQ, Tensor.scalar(0, .i32)); - x = candidate.logical(.OR, first_token).select(x, min_inf); - - return .{ x, full_topk.indices }; -} - -test sampleTokensDynamic { - const platform = zml.testing.env(); - - const ___ = -std.math.inf(f32); - const logits_data = [_]f32{ @log(2.0), @log(1.0), @log(4.0), @log(3.0) }; - const top_k_indices = [_]i32{ 2, 3, 0, 1 }; - - const logits: zml.Tensor = .init(.{ .voc = logits_data.len }, .f32); - const dynamic_sampling_strategy = DynamicSamplingStrategy.init(.f32, 0); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, fixupLogits, .{ logits, dynamic_sampling_strategy }, platform); - defer exe.deinit(); - - var logits_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits.shape(), std.mem.sliceAsBytes(&logits_data)); - defer logits_buffer.deinit(); - - const Args = struct { DynamicSamplingStrategy.Opts, [4]f32 }; - inline for ([_]Args{ - // top_k == logits.len -> just sort the input - .{ .{ .top_k = 4 }, [_]f32{ @log(4.0), @log(3.0), @log(2.0), @log(1.0) } }, - .{ .{ .top_k = 2 }, [_]f32{ @log(4.0), @log(3.0), ___, ___ } }, - .{ .{ .top_k = 2, .temperature = 0.1 }, [_]f32{ @log(4.0) * 0.1, @log(3.0) * 0.1, ___, ___ } }, - // top_k == logits.len and small top_p -> make sure at least one is returned - .{ .{ .top_k = 4, .top_p = 0.1 }, [_]f32{ @log(4.0), ___, ___, ___ } }, - .{ .{ .top_k = 4, .top_p = 0.701 }, [_]f32{ @log(4.0), @log(3.0), ___, ___ } }, - .{ .{ .top_k = 4, .top_p = 0.901 }, [_]f32{ @log(4.0), @log(3.0), @log(2.0), ___ } }, - // Here top_p is computed on the top 3 items, so 0.701 isn't enougth anymore to allow @log(3.0) - .{ .{ .top_k = 3, .top_p = 0.701 }, [_]f32{ @log(4.0), ___, ___, ___ } }, - // Here top_p allows the first 3 results, but min_p only accepts the first two. - .{ .{ .top_k = 4, .top_p = 0.901, .min_p = 0.6 }, [_]f32{ @log(4.0), @log(3.0), ___, ___ } }, - }) |args_expected| { - const args, const expected = args_expected; - var dynamic_sampling_strategy_buffers = try DynamicSamplingStrategy.makeBuffers(std.testing.io, platform, .f32, args); - defer DynamicSamplingStrategy.deinitBuffers(&dynamic_sampling_strategy_buffers); - var new_logits, var indices = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, fixupLogits, .{ logits_buffer, dynamic_sampling_strategy_buffers }); - defer new_logits.deinit(); - defer indices.deinit(); - try std.testing.expectEqual(top_k_indices, try indices.getValue(@TypeOf(top_k_indices), std.testing.io)); - try zml.testing.expectEqual(expected, try new_logits.getValue(@TypeOf(expected), std.testing.io)); - } - - { - // Similar but use bf16, and uses infinity to trigger nans after the softmax. - const bf16 = zml.floats.BFloat16; - - const logits_bf16: zml.Tensor = .init(.{ .voc = logits_data.len }, .bf16); - const dynamic_sampling_strategy_bf16 = DynamicSamplingStrategy.init(.bf16, 0); - var exe_bf16 = try zml.module.compile(std.testing.allocator, std.testing.io, fixupLogits, .{ logits_bf16, dynamic_sampling_strategy_bf16 }, platform); - defer exe_bf16.deinit(); - - const boost = bf16.inf; - const nerf = bf16.minus_inf; - - var logits_bf16_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits_bf16.shape(), std.mem.sliceAsBytes(&[4]bf16{ boost, boost, bf16.fromF32(2), nerf })); - defer logits_bf16_buffer.deinit(); - var dynamic_sampling_strategy_bf16_buffers = try DynamicSamplingStrategy.makeBuffers(std.testing.io, platform, .bf16, .{ .top_k = 4, .top_p = 0.9, .min_p = 0.1 }); - defer DynamicSamplingStrategy.deinitBuffers(&dynamic_sampling_strategy_bf16_buffers); - - var new_logits, var indices = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_bf16, fixupLogits, .{ logits_bf16_buffer, dynamic_sampling_strategy_bf16_buffers }); - - try std.testing.expectEqual([_]i32{ 0, 1, 2, 3 }, try indices.getValue([4]i32, std.testing.io)); - try zml.testing.expectEqual([_]bf16{ boost, nerf, nerf, nerf }, try new_logits.getValue([4]bf16, std.testing.io)); - } -} - -const ShapeError = error{ DimMismatch, NotFound }; -const NOT_SET: i64 = -2; -const DIM_MISMATCH: i64 = -1; - -/// Collect the given dimensions inside a struct containing tagged tensors. -pub fn collectDims( - comptime dims: anytype, - v: anytype, - comptime mode: enum { strict, allow_extra_dims, ignore_errors }, -) ShapeError!ShapeStruct(dims) { - const LocalContext = struct { - res: ShapeStruct(dims), - mode: @TypeOf(mode), - }; - - var context = LocalContext{ - .res = undefined, - .mode = mode, - }; - @memset(std.mem.bytesAsSlice(i64, std.mem.asBytes(&context.res)), NOT_SET); - - meta.visit((struct { - fn cb(ctx: *LocalContext, shape: *const Shape) void { - inline for (dims) |a| { - if (shape.hasTag(a)) |axis| { - const dim = shape.dim(axis); - - const expected_dim = &@field(ctx.res, @tagName(a)); - if (expected_dim.* == NOT_SET) { - expected_dim.* = dim; - } else if (expected_dim.* == DIM_MISMATCH) { - // this axis has already been reported as invalid. - } else if (dim != expected_dim.*) { - if (mode != .ignore_errors) { - log.warn("Dim mismatch ! Axis {0s}={1d} but received a new tensor where {0s}={2d}", .{ @tagName(a), expected_dim.*, dim }); - } - expected_dim.* = DIM_MISMATCH; - } - } - } - } - }).cb, &context, v); - - if (context.mode != .ignore_errors) { - inline for (shapeToDims(context.res), dims) |dim, dim_tag| { - if (dim == NOT_SET) { - log.warn("Axis not found: {s}", .{@tagName(dim_tag)}); - return error.NotFound; - } - if (dim == DIM_MISMATCH) return error.DimMismatch; - } - } - return context.res; -} - -fn shapeToDims(shape: anytype) [@divExact(@sizeOf(@TypeOf(shape)), @sizeOf(i64))]i64 { - return @bitCast(shape); -} - -test collectDims { - const Model = struct { - x: Shape, - y: Shape, - bias: Shape, - }; - - { - var model: Model = .{ - .x = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), - .y = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), - .bias = Shape.init(.{5}, .f32).withTags(.{.d}), - }; - try zml.testing.expectEqual(collectDims(.{ .b, .d }, &model, .strict), .{ .b = 2, .d = 5 }); - } - { - var model: Model = .{ - .x = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), - .y = Shape.init(.{ 3, 5 }, .f32).withTags(.{ .b, .d }), - .bias = Shape.init(.{5}, .f32).withTags(.{.d}), - }; - try std.testing.expectEqual( - collectDims(.{ .b, .d }, &model, .strict), - error.DimMismatch, - ); - try zml.testing.expectEqual(collectDims(.{ .b, .d }, &model, .ignore_errors), .{ .b = DIM_MISMATCH, .d = 5 }); - } - { - var model: Model = .{ - .x = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), - .y = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), - .bias = Shape.init(.{5}, .f32).withTags(.{.d}), - }; - try std.testing.expectEqual(collectDims(.{ .b, .d, .c }, &model, .strict), error.NotFound); - try zml.testing.expectEqual(collectDims(.{ .b, .d, .c }, &model, .ignore_errors), .{ .b = 2, .d = 5, .c = NOT_SET }); - } - { - var model: Model = .{ - .x = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), - .y = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), - .bias = Shape.init(.{7}, .f32).withTags(.{.d}), - }; - try std.testing.expectEqual(collectDims(.{ .b, .d }, &model, .strict), error.DimMismatch); - try zml.testing.expectEqual(collectDims(.{ .b, .d }, &model, .ignore_errors), .{ .b = 2, .d = DIM_MISMATCH }); - } -} - -fn ShapeStruct(comptime dims: anytype) type { - const rank = dims.len; - @setEvalBranchQuota(rank + 5); - var struct_field_names: [rank][]const u8 = undefined; - var struct_field_types: [rank]type = undefined; - var struct_field_attrs: [rank]std.builtin.Type.StructField.Attributes = undefined; - const default: i64 = NOT_SET; - for (&struct_field_names, &struct_field_types, &struct_field_attrs, dims) |*field_name, *field_type, *field_attr, axis| { - field_name.* = @tagName(axis); - field_type.* = i64; - field_attr.* = .{ - .@"align" = @alignOf(i64), - .default_value_ptr = &default, - }; - } - - return @Struct(.@"extern", null, &struct_field_names, &struct_field_types, &struct_field_attrs); -} diff --git a/zml2/ops.zig b/zml2/ops.zig deleted file mode 100644 index 637e0947..00000000 --- a/zml2/ops.zig +++ /dev/null @@ -1,1035 +0,0 @@ -const std = @import("std"); - -const dialects = @import("mlir/dialects"); -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -const CompilationContext = @import("module.zig").CompilationContext; -const constants = @import("constants.zig"); -const meta = @import("meta.zig"); -const mlirx = @import("mlirx.zig"); -const Shape = @import("shape.zig").Shape; -const Tensor = @import("tensor.zig").Tensor; - -pub const ReduceArgs = struct { - left: Tensor, - right: Tensor, -}; - -pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); - defer arena.deinit(); - - const mlir_ctx = CompilationContext.current().mlir_ctx; - - const reduce_block, var result = b: { - const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inits.len); - var args: ArgsType = undefined; - var block_types: [2 * inits.len]*const mlir.Type = undefined; - - inline for (0..inits.len) |i| { - args[i].left = .fromShape(inits[i].shape()); - args[i].right = .fromShape(inits[i].shape()); - - block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); - block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); - } - - const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); - const reduce_block = mlir.Block.init(&block_types, &block_locs); - errdefer reduce_block.deinit(); - - CompilationContext.current().pushBlock(reduce_block); - defer CompilationContext.current().popBlock(); - - const scope = CompilationContext.current().currentScope(); - inline for (0..inits.len) |i| { - scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; - scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inits.len) catch unreachable; - } - - var result = @call(.auto, func, args ++ context); - - var result_values: [inits.len]*const mlir.Value = undefined; - inline for (0..inits.len) |i| { - result_values[i] = result[i].value(); - } - - _ = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)).appendTo(reduce_block); - break :b .{ reduce_block, result }; - }; - var input_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| { - input_values[i] = inputs[i].value(); - } - - var init_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inits.len) |i| init_values[i] = inits[i].value(); - - const reduce_op = mlir.Operation.make(mlir_ctx, "stablehlo.reduce", .{ - .operands = .{ .variadic = &.{ &input_values, &init_values } }, - .result_type_inference = true, - .blocks = &.{reduce_block}, - .attributes = &.{ - .named(mlir_ctx, "dimensions", mlir.denseArrayAttribute(mlir_ctx, .i64, axes_)), - }, - .verify = true, - .location = .unknown(mlir_ctx), - }).appendTo(CompilationContext.current().currentScope().block); - - // `stablehlo.reduce` drops axes. We want to avoid that to propagate tags. - // So we need to broadcast the output of `stablehlo.reduce` to the input shapes. - // To that order, we initialize `result` to `inputs`, then we use stdx.meta.visit, - // to find the correct mlir.Value, but we first broadcast before creating the final - // Tensor struct. - var broadcasting_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; - for (0..constants.MAX_RANK) |i| { - if (std.mem.indexOfScalar(i64, axes_, @intCast(i)) == null) { - broadcasting_axes.append(@intCast(i)) catch unreachable; - } - } - - inline for (0..result.len) |i| { - var reduced_shape: Shape = inputs[i].shape(); - for (axes_) |a| { - reduced_shape = reduced_shape.setDim(a, 1); - } - - const tensor_type = mlir.rankedTensorType(reduced_shape.dims(), mlirx.Type.fromDType(mlir_ctx, reduced_shape.dtype())); - const broad_op = dialects.stablehlo.broadcast_in_dim( - mlir_ctx, - reduce_op.result(i), - broadcasting_axes.slice()[0 .. reduced_shape.rank() - axes_.len], - tensor_type, - .unknown(mlir_ctx), - ).appendTo(CompilationContext.current().currentScope().block); - - result[i] = Tensor._result(reduced_shape, broad_op.result(0)); - } - - return result; -} - -pub const ReduceWindowOpts = struct { - // TODO replace with Shape - window_dimensions: []const i64, - window_strides: []const i64, - base_dilations: []const i64, - window_dilations: []const i64, - padding: []const [2]i64, -}; - -pub fn reduceWindow(inputs: anytype, inits: anytype, opts: ReduceWindowOpts, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); - defer arena.deinit(); - - const mlir_ctx = CompilationContext.current().mlir_ctx; - - const reduce_block, var result = b: { - const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inits.len); - var args: ArgsType = undefined; - var block_types: [2 * inits.len]*const mlir.Type = undefined; - - inline for (0..inits.len) |i| { - args[i].left = .fromShape(inits[i].shape()); - args[i].right = .fromShape(inits[i].shape()); - - block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); - block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); - } - - const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); - const reduce_block = mlir.Block.init(&block_types, &block_locs); - errdefer reduce_block.deinit(); - - CompilationContext.current().pushBlock(reduce_block); - defer CompilationContext.current().popBlock(); - - const scope = CompilationContext.current().currentScope(); - inline for (0..inits.len) |i| { - scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; - scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inits.len) catch unreachable; - } - - var result = @call(.auto, func, args ++ context); - - var result_values: [inits.len]*const mlir.Value = undefined; - inline for (0..inits.len) |i| { - result_values[i] = result[i].value(); - } - - _ = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)).appendTo(reduce_block); - break :b .{ reduce_block, result }; - }; - var input_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| { - input_values[i] = inputs[i].value(); - } - - var init_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inits.len) |i| init_values[i] = inits[i].value(); - - const reduce_op = mlir.Operation.make(mlir_ctx, "stablehlo.reduce_window", .{ - .operands = .{ .variadic = &.{ &input_values, &init_values } }, - .result_type_inference = true, - .blocks = &.{reduce_block}, - .attributes = &.{ - .named(mlir_ctx, "window_dimensions", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.window_dimensions)), - .named(mlir_ctx, "window_strides", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.window_strides)), - .named(mlir_ctx, "base_dilations", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.base_dilations)), - .named(mlir_ctx, "window_dilations", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.window_dilations)), - // Cast the [][2]i64 to []i64 (safe) - .named(mlir_ctx, "padding", mlir.denseElementsAttribute(mlir.RankedTensorType.get(&.{ @intCast(opts.padding.len), 2 }, mlir.integerType(mlir_ctx, .i64), null).shaped(), @as([]const i64, @ptrCast(opts.padding)))), - }, - .verify = true, - .location = .unknown(mlir_ctx), - }).appendTo(CompilationContext.current().currentScope().block); - - inline for (0..result.len) |i| { - result[i] = Tensor.fromMlirValue(reduce_op.result(i)).withTags(inputs[i].shape()); - } - - return result; -} - -pub const SortArgs = struct { - left: Tensor, - right: Tensor, -}; - -pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytype, is_stable: bool) [inputs.len]Tensor { - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); - defer arena.deinit(); - - const mlir_ctx = CompilationContext.current().mlir_ctx; - - const sort_block = b: { - const ArgsType = std.meta.Tuple(&[1]type{SortArgs} ** inputs.len); - var args: ArgsType = undefined; - var block_types: [2 * inputs.len]*const mlir.Type = undefined; - - inline for (0..inputs.len) |i| { - args[i].left = Tensor.init(.{}, inputs[i].shape().dtype()); - args[i].right = Tensor.init(.{}, inputs[i].shape().dtype()); - - block_types[2 * i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); - block_types[2 * i + 1] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); - } - - const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); - const sort_block = mlir.Block.init(&block_types, &block_locs); - errdefer sort_block.deinit(); - - CompilationContext.current().pushBlock(sort_block); - defer CompilationContext.current().popBlock(); - - const scope = CompilationContext.current().currentScope(); - inline for (0..inputs.len) |i| { - scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, 2 * i) catch unreachable; - scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, 2 * i + 1) catch unreachable; - } - - var result = @call(.auto, func, args ++ context); - - _ = dialects.stablehlo.return_(mlir_ctx, result.value(), .unknown(mlir_ctx)).appendTo(sort_block); - break :b sort_block; - }; - - var input_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| { - input_values[i] = inputs[i].value(); - } - - const sort_op = mlir.Operation.make(mlir_ctx, "stablehlo.sort", .{ - .operands = .{ .variadic = &.{&input_values} }, - .result_type_inference = true, - .blocks = &.{sort_block}, - .attributes = &.{ - .named(mlir_ctx, "dimension", mlir.integerAttribute(mlir_ctx, .i64, axis_)), - .named(mlir_ctx, "is_stable", mlir.boolAttribute(mlir_ctx, is_stable)), - }, - .verify = true, - .location = .unknown(mlir_ctx), - }).appendTo(CompilationContext.current().currentScope().block); - - var result: [inputs.len]Tensor = undefined; - inline for (0..inputs.len) |i| { - result[i] = Tensor._result(inputs[i].shape(), sort_op.result(i)); - } - - return result; -} - -pub const TritonOps = struct { - debug: bool = false, - name: [:0]const u8, - ir: [:0]const u8, - grid: [3]i32, - num_stages: i32, - num_warps: i32, - output_operand_aliases: []const i64 = &.{}, -}; - -/// Generate an MLIR call to the given member function with the given tensors. -pub fn triton(inputs: anytype, outputs: anytype, opts: TritonOps) [outputs.len]Tensor { - const mlir_ctx = CompilationContext.current().mlir_ctx; - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); - defer arena.deinit(); - - var values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| { - values[i] = inputs[i].value(); - } - - var res_types: [outputs.len]*const mlir.Type = undefined; - inline for (outputs, 0..) |output, i| { - res_types[i] = mlir.rankedTensorType(output.dims(), mlirx.Type.fromDType(mlir_ctx, output.dtype())); - } - - const backend_config = mlir.dictionaryAttribute(mlir_ctx, &.{ - mlir.NamedAttribute.named(mlir_ctx, "name", mlir.stringAttribute(mlir_ctx, opts.name)), - mlir.NamedAttribute.named(mlir_ctx, "ir", mlir.stringAttribute(mlir_ctx, opts.ir)), - mlir.NamedAttribute.named(mlir_ctx, "grid_x", mlir.integerAttribute(mlir_ctx, .i32, opts.grid[0])), - mlir.NamedAttribute.named(mlir_ctx, "grid_y", mlir.integerAttribute(mlir_ctx, .i32, opts.grid[1])), - mlir.NamedAttribute.named(mlir_ctx, "grid_z", mlir.integerAttribute(mlir_ctx, .i32, opts.grid[2])), - mlir.NamedAttribute.named(mlir_ctx, "num_stages", mlir.integerAttribute(mlir_ctx, .i32, opts.num_stages)), - mlir.NamedAttribute.named(mlir_ctx, "num_warps", mlir.integerAttribute(mlir_ctx, .i32, opts.num_warps)), - }); - - var operands_layouts: [inputs.len][]const usize = undefined; - inline for (inputs, 0..) |input, i| { - operands_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(input.rank())).constSlice()) catch unreachable; - } - - var results_layouts: [outputs.len][]const usize = undefined; - inline for (outputs, 0..) |output, i| { - results_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(output.rank())).constSlice()) catch unreachable; - } - - const op = dialects.stablehlo.custom_call( - mlir_ctx, - &values, - &res_types, - .{ - .call_target_name = "__gpu$xla.gpu.triton", - .backend_config = .{ .typed_ffi = backend_config }, - .has_side_effect = false, - .operand_layouts = &operands_layouts, - .result_layouts = &results_layouts, - .output_operand_aliases = opts.output_operand_aliases, - }, - .unknown(mlir_ctx), - ).appendTo(CompilationContext.current().currentScope().block); - - var outputs_: [outputs.len]Tensor = undefined; - inline for (outputs, 0..) |output, i| { - outputs_[i] = Tensor._result(output, op.result(i)); - } - - return outputs_; -} - -test "triton" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - if (platform.target != .cuda and platform.target != .rocm) return error.SkipZigTest; - - const ir = - \\ module { - \\ tt.func public @add_one(%arg0: !tt.ptr {tt.divisibility = 32 : i32}, %arg1: !tt.ptr {tt.divisibility = 32 : i32}, %arg2: !tt.ptr {tt.divisibility = 32 : i32}, %arg3: !tt.ptr {tt.divisibility = 32 : i32}) { - \\ %0 = tt.get_program_id x : i32 - \\ %1 = tt.load %arg0 {cache = 1 : i32, evict = 1 : i32, isVolatile = false} : !tt.ptr - \\ %2 = tt.load %arg1 {cache = 1 : i32, evict = 1 : i32, isVolatile = false} : !tt.ptr - \\ %cst = arith.constant 1.000000e+00 : f32 - \\ %3 = arith.addf %1, %cst : f32 - \\ tt.store %arg2, %3 {cache = 1 : i32, evict = 1 : i32} : !tt.ptr - \\ tt.store %arg3, %2 {cache = 1 : i32, evict = 1 : i32} : !tt.ptr - \\ tt.return - \\ } - \\ } - ; - - const TritonMod = struct { - pub fn forward(a: Tensor, b: Tensor) [2]Tensor { - return triton(.{ a, b }, .{ a.shape(), b.shape() }, .{ - .debug = false, - .name = "add_one", - .ir = ir, - .grid = .{ 1, 1, 1 }, - .num_stages = 1, - .num_warps = 1, - }); - } - }; - - const a: zml.Tensor = .init(.{}, .f32); - const b: zml.Tensor = .init(.{}, .f32); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, TritonMod.forward, .{ a, b }, platform); - defer exe.deinit(); - - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[1]f32{1})); - defer a_buffer.deinit(); - var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), std.mem.sliceAsBytes(&[1]f32{3})); - defer b_buffer.deinit(); - - const results = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, TritonMod.forward, .{ a_buffer, b_buffer }); - defer results[0].deinit(); - defer results[1].deinit(); - - var cpu_result_0 = try results[0].toSliceAlloc(std.testing.allocator, std.testing.io); - defer cpu_result_0.free(std.testing.allocator); - var cpu_result_1 = try results[1].toSliceAlloc(std.testing.allocator, std.testing.io); - defer cpu_result_1.free(std.testing.allocator); - - const expected_result_a: f32 = 2.0; - const expected_result_b: f32 = 3.0; - - try std.testing.expectEqual(expected_result_a, cpu_result_0.items(f32)[0]); - try std.testing.expectEqual(expected_result_b, cpu_result_1.items(f32)[0]); -} - -pub const ScatterArgs = struct { - input: Tensor, - update: Tensor, -}; - -/// Generalized version of scatter to many inputs. -/// See `zml.Tensor.scatterSlices` for documentation on scatter. -/// -/// This allows to use the same indices to update several tensors at once, -/// and where the update function is allow to look at elements from the different tensors -/// to compute the final value. -/// -/// This sounds nice but in practice XLA doesn't support this well on GPU, -/// and will generate slow code. In practice stick with `zml.Tensor.scatterSlices`. -pub fn scatter( - inputs: anytype, - index_tensors: anytype, - updates: anytype, - comptime func: anytype, - context: anytype, - opts: Tensor.ScatterOpts, -) stdx.meta.FnResult(func) { - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); - defer arena.deinit(); - - const mlir_ctx = CompilationContext.current().mlir_ctx; - - const update_block, var result = b: { - const ArgsType = std.meta.Tuple(&[1]type{ScatterArgs} ** inputs.len); - var args: ArgsType = undefined; - var block_types: [2 * inputs.len]*const mlir.Type = undefined; - - inline for (0..inputs.len) |i| { - args[i].input = Tensor.init(.{}, inputs[i].dtype()); - args[i].update = Tensor.init(.{}, inputs[i].dtype()); - - block_types[i] = mlir.rankedTensorType(args[i].input.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].input.dtype())); - block_types[i + inputs.len] = mlir.rankedTensorType(args[i].update.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].update.dtype())); - } - - const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); - const update_block = mlir.Block.init(&block_types, &block_locs); - errdefer update_block.deinit(); - - CompilationContext.current().pushBlock(update_block); - defer CompilationContext.current().popBlock(); - - const scope = CompilationContext.current().currentScope(); - inline for (0..inputs.len) |i| { - scope.id_to_argument.put(scope.arena.allocator(), args[i].input.id, i) catch unreachable; - scope.id_to_argument.put(scope.arena.allocator(), args[i].update.id, i + inputs.len) catch unreachable; - } - - var result = @call(.auto, func, args ++ context); - - var result_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| { - result_values[i] = result[i].value(); - } - - _ = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)).appendTo(update_block); - break :b .{ update_block, result }; - }; - - // Note: I was a bit lazy here, and I only look at tags on the first tensor. - // we probably should check all of them. - const self = meta.first(Tensor, inputs); - const update = meta.first(Tensor, updates); - var indices_per_axis, var indices_axes = Shape.parseStruct(Tensor, index_tensors); - - if (indices_per_axis.len == 0) return inputs; - - // validate coord axes: all coord_axes should exist inside self - for (indices_axes.constSlice()) |t| { - stdx.debug.assert(self._shape.hasTag(t) != null, "zml.ops.scatter expects axes of indices to be axes of inputs, got input={f} and indices={any}", .{ self, indices_axes.constSlice() }); - } - - // Handle scalar indices by broadcasting them to the indices with the highest rank. - const indices_shape = blk: { - var higher_rank = indices_per_axis.get(0).shape(); - for (indices_per_axis.constSlice()[1..]) |indices| { - if (indices.rank() > higher_rank.rank()) { - higher_rank = indices.shape(); - } - } - break :blk higher_rank; - }; - for (indices_per_axis.slice()) |*idx| { - stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "zml.ops.scatter expects all indices tensor to have the same shape, got {any}", .{indices_per_axis.slice()}); - stdx.debug.assert(idx.dtype() == indices_shape.dtype(), "zml.ops.scatter expects all indices tensor to have the same dtype, got {any}", .{indices_per_axis.slice()}); - idx.* = idx.broad(indices_shape); - } - - // rewrite simple scatters to dynamicUpdateSlice. - if (@TypeOf(inputs) == struct { Tensor } and indices_shape.rank() == 0) { - return .{self.dynamicUpdateSlice(index_tensors, update)}; - } - - // TODO: ideally we should catch all possible scatter errors and provide nice error messages. - var config = scatterConfig(self.shape(), update.shape(), indices_per_axis, indices_axes); - const indices = scatterPrepareIndices(&config, self.shape(), update.shape(), &indices_per_axis, &indices_axes); - // const n_indices_axes = update.rank() - _collectAxes(AxisKind, up_kind, .update_window).len; - // stdx.debug.assert(n_indices_axe == indices_axes.len, "scatter({f}, {any}) expects 'updates' to contain all axes from 'indices', got indices={s}, updates={f}", .{ self, index_tensors, indices_axes.constSlice(), update }); - - var input_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| { - input_values[i] = inputs[i].value(); - } - - var updates_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..updates.len) |i| updates_values[i] = updates[i].value(); - - const op = dialects.stablehlo.scatter( - mlir_ctx, - &input_values, - &.{indices.value()}, - &updates_values, - update_block, - .{ - .update_window_dims = _collectAxes(ScatterAxisKind, config.up_kind, .update_window).constSlice(), - .inserted_window_dims = _collectAxes(ScatterAxisKind, config.op_kind, .inserted_window).constSlice(), - .input_batching_dims = _collectAxes(ScatterAxisKind, config.op_kind, .batching).constSlice(), - .scatter_indices_batching_dims = config.indices_batch_axes.constSlice(), - .scatter_dims_to_operand_dims = config.scatter_to_operand_axes.constSlice(), - .index_vector_dim = indices.rank() - 1, - .indices_are_sorted = opts.indices_are_sorted, - .unique_indices = opts.indices_are_unique, - }, - .unknown(mlir_ctx), - ).appendTo(CompilationContext.current().currentScope().block); - - inline for (0..result.len) |i| { - result[i] = Tensor._result(inputs[i].shape(), op.result(i)); - } - - return result; -} - -const ScatterConfig = struct { - op_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .{}, - up_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .{}, - indices_batch_axes: Shape.DimsArray = .{}, - scatter_to_operand_axes: Shape.DimsArray = .{}, - updates_transpose: Shape.AxesArray = .{}, -}; - -const ScatterAxisKind = enum { batching, update_window, inserted_window, window_id }; - -fn scatterConfig( - op: Shape, - update: Shape, - indices_per_axis: stdx.BoundedArray(Tensor, constants.MAX_RANK), - indices_axes: Shape.TagsArray, -) ScatterConfig { - var op_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .{}; - var up_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .{}; - var indices_batch_axes: Shape.DimsArray = .{}; - var scatter_to_operand_axes: Shape.DimsArray = .{}; - var updates_transpose: Shape.AxesArray = .{}; - - const tagged_api = indices_axes.len > 0; - const indices = indices_per_axis.get(0).shape(); - - if (tagged_api) { - for (indices_axes.constSlice()) |t| { - scatter_to_operand_axes.appendAssumeCapacity(op.axis(t)); - } - for (indices.tags()) |t| { - stdx.debug.assert(update.hasTag(t) != null, "scatter expects 'updates' to have all axes of 'indices', got self={f}, updates={f} and indices={f}", .{ op, update, indices }); - updates_transpose.appendAssumeCapacity(update.axis(t)); - } - - for (op.tags()) |t| { - if (update.hasTag(t)) |up_ax| { - updates_transpose.appendAssumeCapacity(up_ax); - - if (indices.hasTag(t)) |id_ax| { - if (std.mem.indexOfScalar(Shape.Tag, indices_axes.constSlice(), t) != null) { - // tag is in indices AND in coords -> it's a batching dim that has been rewritten to a regular insertion dim - op_kind.appendAssumeCapacity(.inserted_window); - } else { - // tag is in op, indices and updates -> it's a batching dim - op_kind.appendAssumeCapacity(.batching); - indices_batch_axes.appendAssumeCapacity(@intCast(id_ax)); - } - } else { - op_kind.appendAssumeCapacity(.update_window); - } - } else { - op_kind.appendAssumeCapacity(.inserted_window); - } - } - - for (update.tags(), 0..) |t, up_ax| { - // Handle batch axes right away. - if (op.hasTag(t)) |self_ax| { - if (op_kind.get(self_ax) == .batching) { - up_kind.appendAssumeCapacity(.batching); - continue; - } - } - if (indices.hasTag(t) != null) { - up_kind.appendAssumeCapacity(.window_id); - } else if (op.hasTag(t)) |self_ax| { - stdx.debug.assert(update.dim(up_ax) <= op.dim(self_ax), "scatter expects the slices described in 'updates' to fit inside 'op', but along axis .{s} it doesn't. Got op={f}, updates={f}.", .{ t, op, update }); - up_kind.appendAssumeCapacity(.update_window); - } else { - // TODO: consider accepting untagged update here. - std.debug.panic("scatter expects 'updates' to be made of axes from op={f} and from indices={any}, got unknown tag {s} in {f}", .{ op, indices_axes.constSlice(), std.mem.sliceTo(t, 0), update }); - } - } - } else { - for (0..indices_per_axis.len) |i| { - op_kind.appendAssumeCapacity(.inserted_window); - scatter_to_operand_axes.appendAssumeCapacity(@intCast(i)); - up_kind.appendAssumeCapacity(.window_id); - } - for (indices_per_axis.len..op.rank()) |_| { - op_kind.appendAssumeCapacity(.update_window); - } - for (indices_per_axis.len..update.rank()) |_| { - up_kind.appendAssumeCapacity(.update_window); - } - for (0..update.rank()) |i| { - updates_transpose.appendAssumeCapacity(@intCast(i)); - } - } - - return .{ - .op_kind = op_kind, - .up_kind = up_kind, - .indices_batch_axes = indices_batch_axes, - .scatter_to_operand_axes = scatter_to_operand_axes, - .updates_transpose = updates_transpose, - }; -} - -test scatterConfig { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); - defer comp.deinit(); - comp.activate(); - defer comp.deactivate(); - - const block = mlir.Block.init(&.{}, &.{}); - comp.pushBlock(block); - defer comp.popBlock(); - - const Local = struct { - pub fn _idx(idx_shape: anytype) Tensor { - return Tensor.constant(.{ .i32 = 0 }).broad(Shape.init(idx_shape, .i32)); - } - }; - - const idx = Local._idx; - const op = Shape.init(.{ .a = 10, .b = 20 }, .f32); - - // Use .a as a batching axis with .a=10 x .n=8 updates of 2 elements of .b - { - const indices, const coords_tags = Shape.parseStruct(Tensor, .{ .b = idx(.{ .a = 10, .n = 8 }) }); - const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); - - const cfg = scatterConfig(op, update, indices, coords_tags); - try std.testing.expectEqualSlices(ScatterAxisKind, &.{ .batching, .update_window }, cfg.op_kind.constSlice()); - try std.testing.expectEqualSlices(ScatterAxisKind, &.{ .batching, .window_id, .update_window }, cfg.up_kind.constSlice()); - } - - // similar, but use the normalized form where .a is no longer an explicit batching axis. - { - const indices, const coords_tags = Shape.parseStruct(Tensor, .{ .a = idx(.{ .a = 10, .n = 8 }), .b = idx(.{ .a = 10, .n = 8 }) }); - const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); - - const cfg = scatterConfig(op, update, indices, coords_tags); - try std.testing.expectEqualSlices(ScatterAxisKind, &.{ .inserted_window, .update_window }, cfg.op_kind.constSlice()); - try std.testing.expectEqualSlices(ScatterAxisKind, &.{ .window_id, .window_id, .update_window }, cfg.up_kind.constSlice()); - } -} - -/// Concatenate all indices tensor in one tensor. -/// -/// Is allowed to reorder stuff to simplify the job of the backend, -/// and to expand the batching dims. -fn scatterPrepareIndices( - cfg: *ScatterConfig, - op: Shape, - update: Shape, - indices_per_axis: *stdx.BoundedArray(Tensor, constants.MAX_RANK), - indices_axes: *Shape.TagsArray, -) Tensor { - var old_scatter_to_op_axes = cfg.scatter_to_operand_axes; - const batching = _collectAxes(ScatterAxisKind, cfg.op_kind, .batching); - for (batching.constSlice()) |batch_ax| { - const id_shape = indices_per_axis.get(0).shape(); - // batching requires tagging, so we're sure to have a tag here. - const batch_tag = op.tag(batch_ax); - indices_axes.appendAssumeCapacity(batch_tag); - const batch_id = Tensor.iota(id_shape, batch_tag).convert(id_shape.dtype()); - indices_per_axis.appendAssumeCapacity(batch_id); - cfg.op_kind.buffer[@intCast(batch_ax)] = .inserted_window; - cfg.up_kind.buffer[update.axis(batch_tag)] = .window_id; - old_scatter_to_op_axes.appendAssumeCapacity(batch_ax); - } - cfg.indices_batch_axes = .{}; - - // Reorder the axes so that in indices_per_axis is ordered like in op if possible. - // TODO: transpose updates if needed - var indices: stdx.BoundedArray(Tensor, constants.MAX_RANK) = .{}; - var scatter_to_op_axes: Shape.DimsArray = .{}; - - while (old_scatter_to_op_axes.len > 0) { - const scatter_ax = std.sort.argMin(i64, old_scatter_to_op_axes.constSlice(), {}, std.sort.asc(i64)).?; - const op_ax = old_scatter_to_op_axes.orderedRemove(scatter_ax); - const scatter_idx = indices_per_axis.orderedRemove(scatter_ax); - - scatter_to_op_axes.appendAssumeCapacity(op_ax); - indices.appendAssumeCapacity(scatter_idx); - } - cfg.scatter_to_operand_axes = scatter_to_op_axes; - - for (scatter_to_op_axes.constSlice(), 0..) |sc_ax, i| { - if (i != sc_ax) { - //log.warn("Found a slow scatter pattern, which is going to generate a while loop: scatter({f}, {any}, {f}). Because the index axes aren't the major ones in the input tensor.", .{ op, scatter_to_op_axes.constSlice(), update }); - break; - } - } - return Tensor.stack(indices.constSlice(), .last, .coord); -} - -pub const GatherAxisKind = enum { batching, offset, collapsed, indices }; - -pub const GatherOpts = struct { indices_are_sorted: bool = false }; - -pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, opts: GatherOpts) Tensor { - const mlir_ctx = CompilationContext.current().mlir_ctx; - - stdx.debug.assert(idx_axes.len > 0, "gather expects 1 or more axes to operate one, received none. Example: `x.gather(.a, indices, .{{}})`", .{}); - for (idx_axes, 0..) |a, i| { - if (i > 0) { - stdx.debug.assert(a == idx_axes[i - 1] + 1, "gather expects 'idx_axes' to be sequential. But {any} aren't sequential in {f}", .{ idx_axes, self }); - } - } - var indices_shape = idx_per_axis[0].shape(); - for (idx_per_axis[1..]) |idx| { - if (idx.rank() > indices_shape.rank()) { - indices_shape = idx.shape(); - } - } - for (idx_per_axis) |idx| { - stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "gather indices can't be broadcasted together {any}", .{idx_per_axis}); - } - - var idx_batch_axes: Shape.DimsArray = .{}; - - var self_kind: stdx.BoundedArray(GatherAxisKind, constants.MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; - - for (self._shape.tags(), 0..self.rank()) |t, self_ax| { - const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, 1, @intCast(self_ax)); - if (indices_shape.hasTag(t)) |id_ax| { - // tag is both in self and indices -> it's a batching dim - // Note: tags are required for batching. - self_kind.buffer[self_ax] = .batching; - idx_batch_axes.appendAssumeCapacity(id_ax); - stdx.debug.assert(!is_gather_axis, "gather expects axes to appear at most twice. Axis {s} has been found both in 'self={f}', in 'idx_axes={any}' and in 'indices={f}'", .{ t, self, idx_axes, indices_shape }); - } else if (is_gather_axis) { - // we collapsed all gathered axes - self_kind.buffer[self_ax] = .collapsed; - // idx_kind.buffer[id_ax] = .indices; - } else { - self_kind.buffer[self_ax] = .offset; - } - } - - // compute res shape - var res_shape = Shape.init(.{}, self.dtype()); - var res_kind: stdx.BoundedArray(GatherAxisKind, constants.MAX_RANK) = .{}; - for (self_kind.slice(), 0..) |kind, ax_usize| { - const ax: u3 = @intCast(ax_usize); - if (ax == idx_axes[0]) { - // The first val_ax is special cause this is the place where we insert indices axes. - for (0.., indices_shape.tags(), indices_shape.dims()) |id_axis_order, id_axis, id_inserted_dim| { - const is_batching_axis = std.mem.containsAtLeastScalar(i64, idx_batch_axes.constSlice(), 1, @intCast(id_axis_order)); - // Batching axis is already in self. - if (is_batching_axis) continue; - - res_shape = res_shape.appendDim(id_inserted_dim, id_axis); - res_kind.appendAssumeCapacity(.indices); - } - } - switch (kind) { - .collapsed => continue, - else => { - res_shape = res_shape.appendDim(self.dim(ax), self._shape.tag(ax)); - res_kind.appendAssumeCapacity(kind); - }, - } - } - - // This is not a gather, but a dynamicSlice. - // Sometimes the backend recognize this pattern, but not always. - // So let us handle that. - if (indices_shape.count() == 1 and idx_axes.len == 1) { - return self.dynamicSlice1d(idx_axes[0], .{ .start = idx_per_axis[0].asScalar(), .len = 1 }).reshape(res_shape); - } - - var slice_dims: Shape.DimsArray = .{}; - for (self_kind.slice(), self.dims()) |k, d| { - slice_dims.appendAssumeCapacity(switch (k) { - .batching, .collapsed => 1, - .offset => d, - .indices => unreachable, - }); - } - - // TODO: try changing .last by other axis and see the perf impact. - const indices = Tensor.stack(idx_per_axis, .last, .coord); - // scoped_log.debug("gather --> {} {any}", .{ res_shape, res_kind.constSlice() }); - const gather_op = dialects.stablehlo.gather( - mlir_ctx, - self.value(), - indices.value(), - slice_dims.constSlice(), - .{ - .offset_dims = _collectAxes(GatherAxisKind, res_kind, .offset).constSlice(), - .collapsed_slice_dims = _collectAxes(GatherAxisKind, self_kind, .collapsed).constSlice(), - .operand_batching_dims = _collectAxes(GatherAxisKind, self_kind, .batching).constSlice(), - .start_indices_batching_dims = idx_batch_axes.constSlice(), - .start_index_map = _collectAxes(GatherAxisKind, self_kind, .collapsed).constSlice(), - .index_vector_dim = indices.axis(.coord), - .indices_are_sorted = opts.indices_are_sorted, - }, - .unknown(mlir_ctx), - ).appendTo(CompilationContext.current().currentScope().block); - - const mlir_shape = Tensor.fromMlirValue(gather_op.result(0)).shape(); - stdx.debug.assert(mlir_shape.eql(res_shape), "gather expects that batching indices appear in the same order in 'self' and 'indices', got: self={f}, indices={f}. You should transpose one or the other.", .{ self, indices }); - return Tensor._result(res_shape, gather_op.result(0)); -} - -pub fn _collectAxes(T: type, bounded_array: stdx.BoundedArray(T, constants.MAX_RANK), value: T) stdx.BoundedArray(i64, constants.MAX_RANK) { - var res: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; - for (bounded_array.constSlice(), 0..) |v, ax| { - if (v == value) { - res.appendAssumeCapacity(@intCast(ax)); - } - } - return res; -} - -fn TensorOrTensorArray(comptime T: type) type { - const type_info = @typeInfo(T); - return switch (type_info) { - .@"struct" => |struct_info| b: { - if (T == Tensor) break :b Tensor; - if (!struct_info.is_tuple) @compileError("Expected tuple"); - break :b if (struct_info.fields.len == 1) - Tensor - else - [struct_info.fields.len]Tensor; - }, - .array => |array_info| b: { - break :b if (array_info.len == 1) - Tensor - else - [array_info.len]Tensor; - }, - .pointer => |pointer_info| b: { - if (pointer_info.size != .slice) @compileError("Expected slice"); - break :b []Tensor; - }, - else => @compileError("Unsupported type: " ++ @typeName(T)), - }; -} - -pub const CustomCallOptions = struct { - has_side_effect: bool, - output_operand_aliases: ?[]const i64 = null, -}; - -pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, metadata: anytype, opts: CustomCallOptions) TensorOrTensorArray(@TypeOf(outputs)) { - // Transform generic inputs to flat slice. - const inputs_: []const Tensor = switch (@typeInfo(@TypeOf(inputs))) { - .@"struct" => |struct_info| b: { - if (@TypeOf(inputs) == Tensor) { - break :b &[1]Tensor{inputs}; - } - if (!struct_info.is_tuple) @compileError("Expected tuple"); - var inputs_: [struct_info.fields.len]Tensor = undefined; - meta.collectBuf((struct { - pub fn func(t: Tensor) Tensor { - return t; - } - }).func, {}, &inputs, &inputs_); - break :b &inputs_; - }, - .array => &inputs, - .pointer => |pointer_info| b: { - if (pointer_info.size != .slice) @compileError("Expected slice"); - break :b inputs; - }, - else => @compileError("Unsupported type: " ++ @typeName(@TypeOf(inputs))), - }; - - // Transform generic outputs to flat slice. - const output_shapes: []const Shape = switch (@typeInfo(@TypeOf(outputs))) { - .@"struct" => |struct_info| b: { - if (@TypeOf(outputs) == Shape) { - break :b &[1]Shape{outputs}; - } - if (!struct_info.is_tuple) @compileError("Expected tuple"); - var output_shapes: [struct_info.fields.len]Shape = undefined; - meta.collectBuf((struct { - pub fn func(t: Shape) Shape { - return t; - } - }).func, {}, &outputs, &output_shapes); - break :b &output_shapes; - }, - .array => &outputs, - .pointer => |pointer_info| b: { - if (pointer_info.size != .slice) @compileError("Expected slice"); - break :b outputs; - }, - else => @compileError("Unsupported type: " ++ @typeName(@TypeOf(outputs))), - }; - - const outputs_flat = customCallInternal(target_name, inputs_, output_shapes, metadata, opts); - - // Transform flat slice to generic outputs. - return switch (@typeInfo(@TypeOf(outputs))) { - .@"struct" => |struct_info| b: { - if (@TypeOf(outputs) == Shape) break :b outputs_flat[0]; - if (!struct_info.is_tuple) @compileError("Expected tuple"); - if (struct_info.fields.len == 1) break :b outputs_flat[0]; - var outputs_: [struct_info.fields.len]Tensor = undefined; - @memcpy(&outputs_, outputs_flat); - break :b outputs_; - }, - .array => |array_info| b: { - if (array_info.len == 1) break :b outputs_flat[0]; - var outputs_: [array_info.fields.len]Tensor = undefined; - @memcpy(&outputs_, outputs_flat); - break :b outputs_; - }, - .pointer => |pointer_info| b: { - if (pointer_info.size != .slice) @compileError("Expected slice"); - break :b outputs_flat; - }, - else => @compileError("Unsupported type: " ++ @typeName(@TypeOf(outputs))), - }; -} - -fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs: []const Shape, metadata: anytype, opts: CustomCallOptions) []Tensor { - const ctx = CompilationContext.current(); - var arena = std.heap.ArenaAllocator.init(ctx.allocator); - defer arena.deinit(); - - const values = arena.allocator().alloc(*const mlir.Value, inputs.len) catch unreachable; - for (0..inputs.len) |i| { - values[i] = inputs[i].value(); - } - - const res_types = arena.allocator().alloc(*const mlir.Type, outputs.len) catch unreachable; - for (outputs, 0..) |output, i| { - res_types[i] = mlir.rankedTensorType(output.dims(), mlirx.Type.fromDType(ctx.mlir_ctx, output.dtype())); - } - - const metadata_type_info = @typeInfo(@TypeOf(metadata)); - var metadata_attributes: [metadata_type_info.@"struct".fields.len]mlir.NamedAttribute = undefined; - inline for (metadata_type_info.@"struct".fields, 0..) |field, i| { - const attribute: *const mlir.Attribute = switch (@typeInfo(field.type)) { - .int, .comptime_int => mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@field(metadata, field.name)))), - else => @compileError("Unsupported metadata type: " ++ @typeName(field.type)), - }; - metadata_attributes[i] = mlir.NamedAttribute.named(ctx.mlir_ctx, field.name, attribute); - } - - const backend_config = mlir.dictionaryAttribute(ctx.mlir_ctx, &(metadata_attributes ++ [_]mlir.NamedAttribute{ - .named(ctx.mlir_ctx, "pjrt_api", mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_api))))), - .named(ctx.mlir_ctx, "pjrt_client", mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_client))))), - })); - - const operands_layouts = arena.allocator().alloc([]const usize, inputs.len) catch unreachable; - for (inputs, 0..) |input, i| { - operands_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(input.rank())).constSlice()) catch unreachable; - } - - const results_layouts = arena.allocator().alloc([]const usize, outputs.len) catch unreachable; - for (outputs, 0..) |output, i| { - results_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(output.rank())).constSlice()) catch unreachable; - } - - const op = dialects.stablehlo.custom_call( - ctx.mlir_ctx, - values, - res_types, - .{ - .call_target_name = target_name, - .backend_config = .{ .typed_ffi = backend_config }, - .has_side_effect = opts.has_side_effect, - .operand_layouts = operands_layouts, - .result_layouts = results_layouts, - .output_operand_aliases = opts.output_operand_aliases orelse &.{}, - }, - .unknown(ctx.mlir_ctx), - ); - - const outputs_ = ctx.arena.allocator().alloc(Tensor, outputs.len) catch unreachable; - for (outputs, 0..) |output, i| { - outputs_[i] = Tensor._result(output, op.result(i)); - } - - return outputs_; -} - -test customCall { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); - defer comp.deinit(); - comp.activate(); - defer comp.deactivate(); - - const block = mlir.Block.init(&.{}, &.{}); - comp.pushBlock(block); - defer comp.popBlock(); - - const input = Tensor.constant(zml.DataType.bf16.constant(0)).broad(Shape.init(.{128}, .bf16)); - const output = customCall("my_custom_call", .{input}, .{zml.Shape.init(.{128}, .bf16)}, .{}, .{ - .has_side_effect = false, - .output_operand_aliases = &.{0}, - }); - - try zml.testing.expectEqualShapes(input.shape(), output.shape()); -} - -fn toUsize(values: anytype) stdx.BoundedArray(usize, constants.MAX_RANK) { - var res: stdx.BoundedArray(usize, constants.MAX_RANK) = .{}; - for (values) |val| res.appendAssumeCapacity(@intCast(val)); - return res; -} diff --git a/zml2/pjrtx.zig b/zml2/pjrtx.zig deleted file mode 100644 index 459ee1e7..00000000 --- a/zml2/pjrtx.zig +++ /dev/null @@ -1,171 +0,0 @@ -const std = @import("std"); - -const dialects = @import("mlir/dialects"); -const mlir = @import("mlir"); -const pjrt = @import("pjrt"); -pub const ffi = pjrt.ffi; -pub const Api = pjrt.Api; -pub const ApiError = pjrt.ApiError; -pub const BufferType = pjrt.BufferType; -pub const ClientInitError = pjrt.ClientInitError; -pub const CompiledMemoryStats = pjrt.CompiledMemoryStats; -pub const Device = pjrt.Device; -pub const DeviceDescription = pjrt.DeviceDescription; -pub const Error = pjrt.Error; -pub const ErrorCode = pjrt.ErrorCode; -pub const Executable = pjrt.Executable; -pub const ExecuteContext = pjrt.ExecuteContext; -pub const ExecuteError = ApiError; -pub const GetCostAnalysisError = pjrt.GetCostAnalysisError; -pub const Memory = pjrt.Memory; -pub const MemoryStats = pjrt.MemoryStats; -pub const NamedValue = pjrt.NamedValue; -pub const SerializeResult = pjrt.SerializeResult; -pub const Stream = pjrt.Stream; -pub const ShapeSpec = pjrt.ShapeSpec; -pub const Buffer = pjrt.Buffer; -pub const Event = pjrt.Event; -pub const LoadedExecutable = pjrt.LoadedExecutable; -pub const AsyncHostToDeviceTransferManager = pjrt.AsyncHostToDeviceTransferManager; - -const zml = struct { - pub const DataType = @import("dtype.zig").DataType; -}; - -const log = std.log.scoped(.zml); - -pub const CompileError = std.mem.Allocator.Error || error{InvalidMlirBytecodeVersion} || ApiError; - -fn InnerMixin(comptime innerT: type) type { - return struct { - inline fn inner(self: anytype) if (@typeInfo(@TypeOf(self)).pointer.is_const) *const innerT else *innerT { - return @ptrCast(self); - } - }; -} - -pub const Client = opaque { - const inner = InnerMixin(pjrt.Client).inner; - - pub fn init(api: *const Api, options: []const NamedValue) ClientInitError!*Client { - return @ptrCast(try pjrt.Client.init(api, options)); - } - - pub fn deinit(self: *Client, api: *const Api) void { - self.inner().deinit(api); - } - - pub fn getPlatformName(self: *const Client, api: *const Api) []const u8 { - return self.inner().getPlatformName(api); - } - - pub fn getDevices(self: *const Client, api: *const Api) []const *const Device { - return self.inner().getDevices(api); - } - - pub fn getAddressableDevices(self: *const Client, api: *const Api) []const *const Device { - return self.inner().getAddressableDevices(api); - } - - pub const BufferFromHostBufferArgs = pjrt.Client.BufferFromHostBufferArgs; - pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) ApiError!struct { *Buffer, ?*Event } { - const buffer, const event_ = try self.inner().bufferFromHostBuffer(api, args); - return .{ @ptrCast(buffer), @ptrCast(event_) }; - } - - pub fn deserializeAndLoad(self: *const Client, api: *const Api, io: std.Io, bytes: []const u8) ApiError!*LoadedExecutable { - var future = io.async(pjrt.Client.deserializeAndLoad, .{ self.inner(), api, bytes }); - return @ptrCast(try future.await(io)); - } - - pub const CreateViewOfDeviceBufferArgs = pjrt.Client.CreateViewOfDeviceBufferArgs; - pub fn createViewOfDeviceBuffer(self: *const Client, api: *const Api, args: CreateViewOfDeviceBufferArgs) ApiError!*Buffer { - return @ptrCast(try self.inner().createViewOfDeviceBuffer(api, args)); - } - - pub fn compile(self: *const Client, api: *const Api, allocator: std.mem.Allocator, io: std.Io, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { - var bytecode: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); - defer bytecode.deinit(); - module.operation().writeBytecode(.{ .desired_emit_version = 1 }, &bytecode.writer) catch |err| { - log.err("failed to write module bytecode: {}", .{err}); - return switch (err) { - error.WriteFailed => error.OutOfMemory, - else => |e| e, - }; - }; - - const stablehlo_version = blk: { - if (api.stablehloCurrentVersion()) |requested_version| { - break :blk dialects.stablehlo.smallerVersion(requested_version, dialects.stablehlo.currentVersion()); - } - break :blk dialects.stablehlo.minimumVersion(); - }; - - var serialized_buffer: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); - defer serialized_buffer.deinit(); - dialects.stablehlo.serializePortableArtifact(bytecode.written(), stablehlo_version, &serialized_buffer.writer) catch |err| { - log.err("failed to serialize to portable artifact: {}", .{err}); - return switch (err) { - std.Io.Writer.Error.WriteFailed => error.OutOfMemory, - else => |e| e, - }; - }; - - return @ptrCast(try self.inner().compile(api, io, .{ - .bytecode = serialized_buffer.written(), - .bytecode_format = .mlir, - .compile_options_pb = compile_options_pb, - })); - } - - pub fn addressableMemories(self: *const Client, api: *const Api) []*const Memory { - return self.inner().addressableMemories(api); - } - - pub fn memoryByKind(self: *const Client, api: *const Api, kind: Memory.Kind) ?*const Memory { - for (self.addressableMemories(api)) |mem| { - if (mem.kind(api) == kind) { - return mem; - } - } - return null; - } - - pub const CreateUninitializedBufferArgs = pjrt.Client.CreateUninitializedBufferArgs; - - pub fn createUnitializedBuffer(self: *const Client, api: *const Api, args: CreateUninitializedBufferArgs) ApiError!*Buffer { - return @ptrCast(try self.inner().createUninitializedBuffer(api, args)); - } - - pub const CreateBuffersForAsyncHostToDeviceArgs = pjrt.Client.CreateBuffersForAsyncHostToDeviceArgs; - - pub fn createBuffersForAsyncHostToDevice(self: *const Client, api: *const Api, args: CreateBuffersForAsyncHostToDeviceArgs) ApiError!*AsyncHostToDeviceTransferManager { - return @ptrCast(try self.inner().createBuffersForAsyncHostToDevice(api, args)); - } -}; - -pub fn bufferTypeFromDtype(dt: zml.DataType) pjrt.BufferType { - return switch (dt) { - inline else => |tag| @field(pjrt.BufferType, @tagName(tag)), - }; -} - -pub fn dtypeFromBufferType(pjrt_type: pjrt.BufferType) zml.DataType { - return switch (pjrt_type) { - .invalid => @panic("Found an invalid pjrt buffer"), - inline else => |tag| @field(zml.DataType, @tagName(tag)), - }; -} - -test bufferTypeFromDtype { - inline for (@typeInfo(zml.DataType).@"enum".fields) |field| { - const dt: zml.DataType = @enumFromInt(field.value); - try std.testing.expectEqual(dt, dtypeFromBufferType(bufferTypeFromDtype(dt))); - } - - inline for (@typeInfo(pjrt.BufferType).@"enum".fields) |field| { - const dt: pjrt.BufferType = @enumFromInt(field.value); - if (dt == .invalid) continue; - try std.testing.expectEqual(dt, bufferTypeFromDtype(dtypeFromBufferType(dt))); - } -} diff --git a/zml2/platform.zig b/zml2/platform.zig deleted file mode 100644 index a5c664c1..00000000 --- a/zml2/platform.zig +++ /dev/null @@ -1,350 +0,0 @@ -const std = @import("std"); - -const runtimes = @import("runtimes"); -pub const Target = runtimes.Platform; -const stdx = @import("stdx"); - -const Exe = @import("exe.zig").Exe; -const pjrt = @import("pjrtx.zig"); -const zml = @import("zml.zig"); - -const log = std.log.scoped(.zml); - -pub const CompilationOptions = struct { - xla_dump_to: ?[]const u8 = null, - xla_dump_fusion_visualization: bool = false, - xla_dump_hlo_pass_re: ?[]const u8 = null, - sharding_enabled: bool = false, - sharding_axes: stdx.BoundedArray([*:0]const u8, 8) = .{}, - device_memory_size: u64 = 0, -}; - -fn StaticPlatformMap(comptime E: type, comptime T: type) type { - const tag_count = @typeInfo(E).@"enum".fields.len; - var struct_field_names: [tag_count][]const u8 = undefined; - var struct_field_types: [tag_count]type = @splat(T); - const is_optional = @typeInfo(T) == .optional; - const default_value_ptr: ?*const anyopaque = if (is_optional) @ptrCast(&@as(T, null)) else null; - var struct_field_attrs: [tag_count]std.builtin.Type.StructField.Attributes = @splat(.{ .default_value_ptr = default_value_ptr }); - inline for (@typeInfo(E).@"enum".fields, 0..) |f, i| struct_field_names[i] = f.name; - return @Struct(.auto, null, &struct_field_names, &struct_field_types, &struct_field_attrs); -} - -var api_map: StaticPlatformMap(Target, ?*const pjrt.Api) = .{}; - -fn loadOrGetApi(target: Target, io: std.Io) !*const pjrt.Api { - return switch (target) { - inline else => |tag| @field(api_map, @tagName(tag)) orelse b: { - const api = try runtimes.load(tag, io); - @field(api_map, @tagName(tag)) = api; - break :b api; - }, - }; -} - -pub const Platform = struct { - target: Target, - pjrt_api: *const pjrt.Api, - pjrt_client: *pjrt.Client, - - // This make the pjrt struct quite fat, but is only used during compilation. - // TODO: Reconsider having it here, and maybe pass explicitly to compile, - // or create an intermediary struct: - // `const comp = platform.compiler(compile_opts); const exe = comp.compile(...);` - compilation_options: CompilationOptions = .{}, - - pub const MAX_NUM_DEVICES: u8 = if (runtimes.isEnabled(.tpu)) 32 else 8; - - pub const Device = struct { - pub const Iterator = struct { - api: *const pjrt.Api, - devices: []const *const pjrt.Device, - current: usize = 0, - - pub fn next(self: *Iterator) ?Device { - if (self.current >= self.devices.len) { - return null; - } - defer self.current += 1; - return .{ - .api = self.api, - .device = self.devices[self.current], - }; - } - }; - - api: *const pjrt.Api, - device: *const pjrt.Device, - - pub fn format(self: Device, writer: *std.Io.Writer) std.Io.Writer.Error!void { - const description = self.device.getDescription(self.api); - try writer.print("{s} ({s})", .{ description.getKind(self.api), description.debugString(self.api) }); - } - }; - - pub fn init(target: Target, io: std.Io, options: CreateOptions) !Platform { - const api = try loadOrGetApi(target, io); - - var named_values_buf: [16]pjrt.NamedValue = undefined; - const pjrt_client = try pjrt.Client.init(api, options.toNamedValues(target, &named_values_buf)); - const true_num_devices = pjrt_client.getAddressableDevices(api).len; - if (true_num_devices > MAX_NUM_DEVICES) { - log.warn("platform {} got {} devices, but ZML only support up to {} devices. Some devices won't be used.", .{ target, true_num_devices, MAX_NUM_DEVICES }); - } - return .{ - .target = target, - .pjrt_api = api, - .pjrt_client = pjrt_client, - .compilation_options = .{}, - }; - } - - pub fn auto(io: std.Io, options: CreateOptions) !Platform { - const ordered_targets = [_]Target{ - .tpu, - .neuron, - .cuda, - .rocm, - .cpu, - }; - return for (ordered_targets) |target| { - break init(target, io, options) catch continue; - } else error.Unavailable; - } - - pub fn availablePlatforms(io: std.Io) StaticPlatformMap(Target, bool) { - var result: StaticPlatformMap(Target, bool) = undefined; - inline for (@typeInfo(Target).@"enum".fields) |field| { - if (@field(api_map, field.name) != null) { - @field(result, field.name) = true; - } else { - if (loadOrGetApi(@enumFromInt(field.value), io)) |_| { - @field(result, field.name) = true; - } else |_| { - @field(result, field.name) = false; - } - } - } - return result; - } - - pub fn printAvailablePlatforms(io: std.Io) void { - const available_platforms = availablePlatforms(io); - inline for (@typeInfo(Target).@"enum".fields) |field| { - if (@field(available_platforms, field.name)) { - std.log.info("{f}", .{@field(api_map, field.name).?}); - } - } - } - - pub fn devicesIterator(self: Platform) Device.Iterator { - return .{ - .api = self.pjrt_api, - .devices = self.getDevices(), - }; - } - - pub fn formatWithDevices(self: Platform, writer: *std.Io.Writer) std.Io.Writer.Error!void { - try writer.print("Platform: {s}\n", .{@tagName(self.target)}); - const devices = self.getDevices(); - try writer.print("Devices total={d}\n", .{devices.len}); - for (devices, 0..) |device, i| { - const description = device.getDescription(self.pjrt_api); - try writer.print("\t#{d}: {s}\n", .{ i, description.getKind(self.pjrt_api) }); - } - } - - pub fn fmtVerbose(self: Platform) std.fmt.Alt(Platform, formatWithDevices) { - return .{ .data = self }; - } - - pub fn getDevices(self: Platform) []const *const pjrt.Device { - const all_devices = self.pjrt_client.getAddressableDevices(self.pjrt_api); - if (all_devices.len > MAX_NUM_DEVICES) { - return all_devices[0..MAX_NUM_DEVICES]; - } - return all_devices; - } - - pub const Sharding = struct { num_replicas: u8, num_partitions: u8 }; - - pub fn sharding(self: Platform) Sharding { - // replicas run the same function but with different inputs, - // while partitions contribute to one evaluation over a shared input. - // Inside an inference process, we generally don't want replicas, - // as it's best to fully isolate replicas on different processes. - // For now we hardcode num_replicas = 1. - const num_devices: u8 = @intCast(self.getDevices().len); - return if (self.compilation_options.sharding_enabled) - .{ .num_replicas = 1, .num_partitions = num_devices } - else - .{ .num_replicas = 1, .num_partitions = 1 }; - } - - pub fn withCompilationOptions(self: Platform, opts: CompilationOptions) Platform { - var res = self; - res.compilation_options = opts; - return res; - } - - pub fn deinit(self: *Platform) void { - self.pjrt_client.deinit(self.pjrt_api); - } - - pub fn memoryForDevice(platform: Platform, memory: pjrt.Memory.Kind, device: *const pjrt.Device) *const pjrt.Memory { - const memory_target: pjrt.Memory.Kind = switch (memory) { - .host_unpinned => switch (platform.target) { - // Cuda doesn't have host_unpinned. - // ROCm doesn't seem to have it either. - // TODO(gwenzek): investigate why it was not forced before. - .cuda, .rocm => .host_pinned, - else => .host_unpinned, - }, - inline else => |t| t, - }; - // TODO measure the cost of this and consider caching. - const device_memories = device.addressableMemories(platform.pjrt_api); - for (device_memories) |m| { - if (memory_target == m.kind(platform.pjrt_api)) { - return m; - } - } - log.err("Platform {t} doesn't have memory {t}", .{ platform.target, memory }); - @panic("Memory kind not found"); - } - - test memoryForDevice { - const platform = zml.testing.env(); - const memory_fields = @typeInfo(pjrt.Memory.Kind).@"enum".fields; - inline for (memory_fields) |field| { - for (platform.getDevices()) |dev| { - _ = platform.memoryForDevice(@field(pjrt.Memory.Kind, field.name), dev); - } - } - } - - pub fn memoryStats(platform: Platform, device_id: usize) pjrt.MemoryStats { - if (platform.target == .cpu) return .zeroes; - - const device = platform.getDevices()[device_id]; - return device.memoryStats(platform.pjrt_api) catch .zeroes; - } - - pub fn compile( - self: Platform, - allocator: std.mem.Allocator, - io: std.Io, - model_: anytype, - comptime func: std.meta.DeclEnum(@TypeOf(model_)), - args: stdx.meta.Tail(stdx.meta.FnArgs(@field(@TypeOf(model_), @tagName(func)))), - ) !Exe { - return self.compileFn( - allocator, - io, - @field(@TypeOf(model_), @tagName(func)), - .{model_} ++ args, - ); - } - - pub fn compileModel(self: Platform, allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, model: stdx.meta.Head(stdx.meta.FnArgs(func)), args: stdx.meta.Tail(stdx.meta.FnArgs(func))) !Exe { - return self.compileFn(allocator, io, func, .{model} ++ args); - } - - pub fn compileFn(self: Platform, allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, args: stdx.meta.FnArgs(func)) !Exe { - return zml.module.compile(allocator, io, func, args, self); - } - - pub fn format(self: @This(), writer: *std.Io.Writer) std.Io.Writer.Error!void { - try writer.print("{s} {{ ", .{@tagName(self.target)}); - const devices = self.getDevices(); - for (0..devices.len) |i| { - const description = devices[i].getDescription(self.pjrt_api); - try writer.print("{s}(\"{s}\")", .{ description.toString(self.pjrt_api), description.getKind(self.pjrt_api) }); - if (i < devices.len - 1) try writer.writeAll(", "); - } - try writer.writeAll(" }"); - } -}; - -pub const CreateOptions = struct { - cpu: Cpu = .{ .device_count = 4 }, - - // bump memory fraction from XLA defaults of 75% to 90%. - // Even on a 8GB GPU it should leave enough space for the Cuda driver - // https://github.com/openxla/xla/blob/3e87afa11a865cf91137522492918ad18bfe5b7c/xla/pjrt/plugin/xla_gpu/xla_gpu_allocator_config.h#L25-L60 - cuda: Cuda = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.90 } } }, - rocm: struct {} = .{}, - tpu: struct {} = .{}, - neuron: struct {} = .{}, - - pub const Cpu = struct { - device_count: u32, - - fn writeNamedValues(self: Cpu, values: *std.ArrayList(pjrt.NamedValue)) void { - values.appendAssumeCapacity(pjrt.NamedValue.from("cpu_device_count", @as(i64, self.device_count))); - } - }; - - pub const Cuda = struct { - allocator: Allocator = .{ .bfc = .{} }, - // TODO support all of https://github.com/openxla/xla/blob/3d31c48c719d331d432132b3e0c2c5ce52650675/xla/pjrt/c/pjrt_c_api_gpu_internal.cc#L76-L86 - // visible_devices: []const i64 = &.{}, - // node_id - // num_nodes - // enable_mock_nccl - // mock_gpu_topology - - pub const Allocator = union(enum) { - /// "Best-Fit with Coalescing" algorithm - bfc: Options, - /// use cudaMallocAsync - async: Options, - /// use raw cuMalloc - platform, - - pub const Options = struct { - preallocate: bool = true, - memory_fraction: f32 = 0.90, - collective_memory_size_mb: u32 = 0, - }; - }; - - fn writeNamedValues(self: Cuda, values: *std.ArrayList(pjrt.NamedValue)) void { - switch (self.allocator) { - .platform => { - values.appendAssumeCapacity(.fromString("allocator", "platform")); - }, - .bfc, .async => |opt| { - values.appendAssumeCapacity(.fromString("allocator", switch (self.allocator) { - .bfc => "bfc", - .async => "cuda_async", - .platform => unreachable, - })); - values.appendAssumeCapacity(.from("preallocate", opt.preallocate)); - if (opt.memory_fraction > 0) { - values.appendAssumeCapacity(.from("memory_fraction", opt.memory_fraction)); - } - if (opt.collective_memory_size_mb > 0) { - const collective = @as(i64, opt.collective_memory_size_mb) * 1024 * 1024; - values.appendAssumeCapacity(.from("collective_memory_size", collective)); - } - }, - } - } - }; - - pub fn toNamedValues(self: CreateOptions, target: Target, out: []pjrt.NamedValue) []pjrt.NamedValue { - var values = std.ArrayList(pjrt.NamedValue).fromOwnedSlice(out); - values.shrinkRetainingCapacity(0); - switch (target) { - .cpu => self.cpu.writeNamedValues(&values), - .cuda => self.cuda.writeNamedValues(&values), - inline else => |t| { - stdx.debug.assertComptime(@hasField(CreateOptions, @tagName(t)), "zml.platform.CreateOptions doesn't list target {s}", .{@tagName(t)}); - const options = @field(self, @tagName(t)); - stdx.debug.assertComptime(@sizeOf(@TypeOf(options)) == 0, "zml.platform.CreateOptions.{s} is discarded", .{@tagName(t)}); - }, - } - return values.items; - } -}; diff --git a/zml2/posix.h b/zml2/posix.h deleted file mode 100644 index 0d66200c..00000000 --- a/zml2/posix.h +++ /dev/null @@ -1,2 +0,0 @@ -#include -#include diff --git a/zml2/shape.zig b/zml2/shape.zig deleted file mode 100644 index 56c724e0..00000000 --- a/zml2/shape.zig +++ /dev/null @@ -1,1738 +0,0 @@ -const std = @import("std"); -const testing = std.testing; -const builtin = @import("builtin"); - -const stdx = @import("stdx"); - -const constants = @import("constants.zig"); -const DataType = @import("dtype.zig").DataType; - -const log = std.log.scoped(.shape); - -/// Represent the shape of a tensor. -pub const Shape = struct { - pub const MAX_RANK: u8 = 8; - - pub const PartitionSpec = union(enum) { - axis: Tag, - open, - replicated, - unknown, - - pub fn init(comptime value: anytype) PartitionSpec { - const T = @TypeOf(value); - - if (T == PartitionSpec) { - return value; - } - - if (comptime T == @EnumLiteral()) { - const name = @tagName(value); - inline for (std.meta.fields(PartitionSpec)) |field| { - if (field.type == void and std.mem.eql(u8, field.name, name)) { - return @field(PartitionSpec, field.name); - } - } - } - - if (comptime isTagConvertible(T)) { - const tag_ = Shape.toTag(value); - - return .{ .axis = tag_ }; - } - - stdx.debug.compileError("PartitionSpec.init expected a tag, got: {any}", .{T}); - } - - test "PartitionSpec.init" { - const spec_axis: PartitionSpec = .{ .axis = Shape.toTag(.a) }; - try testing.expectEqual(spec_axis, PartitionSpec.init(.a)); - - const spec_axis_open: PartitionSpec = .open; - try testing.expect(spec_axis_open.eql(.init(.open))); - } - - pub fn eql(self: PartitionSpec, other: PartitionSpec) bool { - if (@as(u4, @intFromEnum(self)) != @as(u4, @intFromEnum(other))) { - return false; - } - - switch (self) { - .axis => |t1| { - const t2 = other.axis; - return std.mem.eql(u8, std.mem.span(t1), std.mem.span(t2)); - }, - else => return true, - } - } - - test "PartitionSpec.eql" { - const spec_axis: PartitionSpec = .init(.a); - const spec_axis_b: PartitionSpec = .init(.b); - const spec_open: PartitionSpec = .open; - const spec_replicated: PartitionSpec = .replicated; - const spec_unknown: PartitionSpec = .unknown; - - try testing.expect(spec_axis.eql(.{ .axis = Shape.toTag(.a) })); - try testing.expect(!spec_axis.eql(spec_axis_b)); - try testing.expect(!spec_axis.eql(.open)); - try testing.expect(!spec_axis.eql(.replicated)); - try testing.expect(!spec_axis.eql(.unknown)); - - try testing.expect(spec_open.eql(.open)); - try testing.expect(!spec_open.eql(spec_axis)); - try testing.expect(!spec_open.eql(.replicated)); - try testing.expect(!spec_open.eql(.unknown)); - - try testing.expect(spec_replicated.eql(.replicated)); - try testing.expect(!spec_replicated.eql(spec_axis)); - try testing.expect(!spec_replicated.eql(.open)); - try testing.expect(!spec_replicated.eql(.unknown)); - - try testing.expect(spec_unknown.eql(.unknown)); - try testing.expect(!spec_unknown.eql(spec_axis)); - try testing.expect(!spec_unknown.eql(.open)); - try testing.expect(!spec_unknown.eql(.replicated)); - } - - pub fn isClosed(self: PartitionSpec) bool { - return switch (self) { - .axis => true, - .open => false, - .replicated => true, - .unknown => false, - }; - } - - test isClosed { - const spec_axis: PartitionSpec = .init(.a); - try testing.expect(spec_axis.isClosed()); - - const spec_open: PartitionSpec = .open; - try testing.expect(!spec_open.isClosed()); - - const spec_replicated: PartitionSpec = .replicated; - try testing.expect(spec_replicated.isClosed()); - - const spec_unknown: PartitionSpec = .unknown; - try testing.expect(!spec_unknown.isClosed()); - } - - pub fn toTag(self: PartitionSpec) Tag { - return switch (self) { - .axis => |t| return t, - .open => TagOpen, - .replicated => TagReplicated, - .unknown => TagUnknown, - }; - } - - test "PartitionSpec.toTag" { - const spec_axis: PartitionSpec = .init(.a); - try testing.expect(spec_axis.toTag() == Shape.toTag(.a)); - - const spec_open: PartitionSpec = .open; - try testing.expect(spec_open.toTag() == TagOpen); - - const spec_replicated: PartitionSpec = .replicated; - try testing.expect(spec_replicated.toTag() == TagReplicated); - - const spec_unknown: PartitionSpec = .unknown; - try testing.expect(spec_unknown.toTag() == TagUnknown); - } - }; - - pub const Tag = [*:0]const u8; - pub const TagUnknown = "_".ptr; - const TagLast = "last".ptr; - const TagReplicated = "replicated".ptr; - const TagOpen = "open".ptr; - - pub const DimsArray = stdx.BoundedArray(i64, constants.MAX_RANK); - pub const TagsArray = stdx.BoundedArray(Tag, constants.MAX_RANK); - pub const AxesArray = stdx.BoundedArray(u3, constants.MAX_RANK); - pub const PartitionArray = stdx.BoundedArray(PartitionSpec, constants.MAX_RANK); - - const UnknownTags: TagsArray = .{ .len = 0, .buffer = [_]Tag{TagUnknown} ** constants.MAX_RANK }; - - _dtype: DataType, - _dims: DimsArray = .{}, - _tags: TagsArray = UnknownTags, - _partitioning: PartitionArray = .{}, - - pub fn parseDimensions(v: anytype) struct { DimsArray, TagsArray } { - const T = @TypeOf(v); - - if (T == Shape) { - return .{ v._dims, v._tags }; - } - - if (comptime stdx.meta.isSliceOfAny(T, stdx.meta.isInteger)) { - var dims_ = DimsArray.init(0) catch unreachable; - var tags_ = TagsArray.init(0) catch unreachable; - for (v) |d| { - dims_.appendAssumeCapacity(@intCast(d)); - tags_.appendAssumeCapacity(TagUnknown); - } - return .{ dims_, tags_ }; - } - - if (comptime stdx.meta.isStruct(T)) { - var dims_: DimsArray = .{}; - var tags_: TagsArray = .{}; - inline for (std.meta.fields(T)) |field| { - const fv = @field(v, field.name); - if (comptime stdx.meta.isInteger(field.type)) { - dims_.appendAssumeCapacity(@intCast(fv)); - } else if (@TypeOf(fv) == @EnumLiteral() and comptime isAutoDim(fv)) { - dims_.appendAssumeCapacity(-1); - } else { - stdx.debug.compileError("Field {s} should be an integer or an auto dimension, got {}", .{ field.name, field.type }); - } - if (comptime stdx.meta.isTuple(T)) { - tags_.appendAssumeCapacity(TagUnknown); - } else { - tags_.appendAssumeCapacity(toTag(field)); - } - } - - return .{ dims_, tags_ }; - } - - stdx.debug.compileError("expected a dimension tuple eg '.{{ .a = 10, .b = 20}}' or '.{{ 10, 20 }}', got {}", .{T}); - } - - test parseDimensions { - const ref = Shape.init(.{ .a = 0, .b = 1 }, .f32); - const dims_, const tags_ = parseDimensions(.{ .a = 0, .b = 1 }); - - try testing.expectEqualSlices(i64, ref.dims(), dims_.constSlice()); - try testing.expectEqualSlices(Tag, ref.tags(), tags_.constSlice()); - } - - pub fn parseAxes(self: Shape, v: anytype) struct { AxesArray, TagsArray } { - const T = @TypeOf(v); - - if (T == Shape) { - return self.parseAxes(self.tags()); - } - - var axes_ = AxesArray.init(0) catch unreachable; - var tags_ = TagsArray.init(0) catch unreachable; - - if (comptime stdx.meta.isSliceOfAny(T, isAxisConvertible)) { - for (v) |d| { - axes_.appendAssumeCapacity(self.axis(d)); - tags_.appendAssumeCapacity(self.tag(d)); - } - return .{ axes_, tags_ }; - } - - if (comptime stdx.meta.isTupleOfAny(T, isAxisConvertible)) { - inline for (std.meta.fields(T)) |field| { - axes_.appendAssumeCapacity(self.axis(@field(v, field.name))); - tags_.appendAssumeCapacity(self.tag(@field(v, field.name))); - } - return .{ axes_, tags_ }; - } - - stdx.debug.compileError("Wrong type, got {}. Expected .{{.a, .b}}", .{T}); - } - - pub fn parseTags(v: anytype) TagsArray { - const T = @TypeOf(v); - stdx.debug.assertComptime(stdx.meta.isTupleOf(T, @EnumLiteral()), "Wrong type, got {}. Expected .{{ .a, .b }}", .{T}); - var tags_ = TagsArray.init(0) catch unreachable; - inline for (v) |field| { - tags_.appendAssumeCapacity(toTag(field)); - } - return tags_; - } - - /// Create a shape from a struct literal, eg: - /// Shape.init(.{ .h = 1024, .w = 512, .c = 3 }); - /// Shape.init(.{ 1024, 512, 3 }); - pub fn init(dimz: anytype, dt: DataType) Shape { - var res: Shape = .{ ._dtype = dt }; - res._dims, res._tags = parseDimensions(dimz); - res._partitioning.appendNTimes(.unknown, res._dims.len) catch @panic("Rank too large"); - - return res; - } - - pub fn scalar(dt: DataType) Shape { - return .{ ._dtype = dt }; - } - - /// Creates a Shape with dims set to `.{0, 1, 2, ..., rank-1}`. - pub fn range(rank_: usize, dt: DataType) Shape { - var res: Shape = .{ ._dtype = dt }; - for (0..rank_) |i| { - res._dims.append(@intCast(i)) catch { - stdx.debug.panic("Too many dimensions! Max: {d}, passed: {d}", .{ res._dims.capacity(), rank_ }); - }; - res._tags.append(TagUnknown) catch unreachable; - } - res._partitioning.appendNTimes(.unknown, rank_) catch @panic("Rank too large"); - return res; - } - - pub fn dtype(self: Shape) DataType { - return self._dtype; - } - - pub fn rank(self: Shape) u4 { - self.ensureAttributesAreSync(); - return @intCast(self._dims.len); - } - - pub fn dim(self: Shape, ax: anytype) i64 { - self.ensureAttributesAreSync(); - return self._dims.get(self.axis(ax)); - } - - pub fn dims(self: *const Shape) []const i64 { - self.ensureAttributesAreSync(); - return self._dims.constSlice(); - } - - fn isAxisConvertible(comptime T: type) bool { - return stdx.meta.isInteger(T) or isTagConvertible(T); - } - - fn isTagConvertible(comptime T: type) bool { - return switch (T) { - @EnumLiteral() => true, - std.builtin.Type.StructField => true, - Tag => true, - else => false, - }; - } - - pub fn toTag(v: anytype) Tag { - const T = @TypeOf(v); - return switch (T) { - @EnumLiteral() => @tagName(v).ptr, - std.builtin.Type.StructField => v.name.ptr, - Tag => v, - else => stdx.debug.compileError("Shape tag should be an @EnumLiteral(), a Shape.Tag or a StructField, got {}", .{T}), - }; - } - - inline fn ensureAttributesAreSync(self: Shape) void { - stdx.debug.assert(self._dims.len == self._tags.len and self._dims.len == self._partitioning.len, "Tags, dims and partitioning have diverged! dims={d} tags={d} partitioning={d}", .{ self._dims.len, self._tags.len, self._partitioning.len }); - } - - pub fn tag(self: Shape, ax: anytype) Tag { - self.ensureAttributesAreSync(); - return self._tags.get(self.axis(ax)); - } - - /// Returns a printable name for a given axis. - /// Either the tag itself, or a digit if it's not tagged. - pub fn debugTag(self: Shape, ax: usize) []const u8 { - const t = self.tag(ax); - if (t != TagUnknown) return std.mem.span(t); - - return "01234567"[ax .. ax + 1]; - } - - pub fn setTag(self: Shape, ax: anytype, tag_: anytype) Shape { - var res = self; - res._tags.set(self.axis(ax), toTag(tag_)); - return res; - } - - pub fn tags(self: *const Shape) []const Tag { - self.ensureAttributesAreSync(); - return self._tags.constSlice(); - } - - pub fn hasTag(self: Shape, tag_: anytype) ?u3 { - return self.axisFromTagMaybe(toTag(tag_)); - } - - pub fn hasTags(self: Shape, tagz: anytype) bool { - const T = @TypeOf(tagz); - - if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, @EnumLiteral())) { - for (tagz) |t| { - if (self.hasTag(t) == null) { - return false; - } - } - return true; - } - - if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, @EnumLiteral())) { - inline for (tagz) |t| { - if (self.hasTag(t) == null) { - return false; - } - } - return true; - } - - stdx.debug.compileError("Expected tuple of tags, got {any}", .{T}); - } - - pub fn isFullyTagged(self: Shape) bool { - for (self._tags.constSlice()) |t| { - if (t == TagUnknown) return false; - } - return true; - } - - pub fn axis(self: Shape, axis_: anytype) u3 { - self.ensureAttributesAreSync(); - - const T = @TypeOf(axis_); - if (comptime stdx.meta.isInteger(T)) { - return self.axisFromInt(@intCast(axis_)); - } - - if (comptime isTagConvertible(T)) { - return self.axisFromTag(toTag(axis_)); - } - - stdx.debug.compileError("Wrong axis type, expected .literal, or an integer, got: {any}", .{T}); - } - - pub fn axes(self: Shape, axes_: anytype) AxesArray { - self.ensureAttributesAreSync(); - - const T = @TypeOf(axes_); - - if (T == Shape) { - return self.axes(axes_.tags()); - } - - var res = AxesArray.init(0) catch unreachable; - - if (comptime stdx.meta.isSliceOfAny(T, stdx.meta.isInteger) or stdx.meta.isSliceOf(T, Tag)) { - for (axes_) |ax| { - res.appendAssumeCapacity(self.axis(ax)); - } - return res; - } - - if (comptime stdx.meta.isStruct(T)) { - inline for (std.meta.fields(T)) |field| { - res.appendAssumeCapacity(self.axis(@field(axes_, field.name))); - } - return res; - } - - stdx.debug.compileError("axes expects an int-tuple or a tuple of enum literal, got {}", .{T}); - } - - fn axisFromInt(self: Shape, a: isize) u3 { - const rk: i8 = self.rank(); - if (a < -rk or a > rk) { - stdx.debug.panic("Tensor {f} doesn't have dimension: {d}", .{ self, a }); - } - return if (a < 0) - @intCast(a + rk) - else - @intCast(a); - } - - fn axisFromTagMaybe(self: Shape, t: Tag) ?u3 { - if (t == TagUnknown) return null; - - if (axisFromLiteralInt(t)) |ax| return ax; - - if (@inComptime()) { - // At comptime two duplicated strings may have two different representations - const t_bytes: []const u8 = std.mem.span(t); - for (self.tags(), 0..) |self_tag, ax| { - if (std.mem.eql(u8, t_bytes, std.mem.span(self_tag))) { - return @truncate(ax); - } - } - return null; - } - - // But at runtime the comptime strings have been deduplicated and ptr match is enough. - if (std.mem.indexOfScalar(Tag, self.tags(), t)) |ax| { - return @truncate(ax); - } - return null; - } - - /// Handle .{ ._0 = x } syntax. - fn axisFromLiteralInt(t: Tag) ?u3 { - // match .{ '_', '0-9', null } - if (t[0] == '_' and t[1] >= '0' and t[1] < '8' and t[2] == 0) { - return @intCast(t[1] - '0'); - } - return null; - } - - fn axisFromTag(self: Shape, d: Tag) u3 { - stdx.debug.assert(d != TagUnknown, "The unknown tag .{s} can't be used to fetch axis in {f}", .{ d, self }); - return self.axisFromTagMaybe(d) orelse { - stdx.debug.panic("Tensor {f} doesn't have dimension with tag: {s}", .{ self, d }); - }; - } - - test axis { - try testing.expectEqual(1, Shape.init(.{ 5, 2 }, .f32).axis(1)); - try testing.expectEqual(1, Shape.init(.{ 5, 2 }, .f32).axis(-1)); - try testing.expectEqual(1, Shape.init(.{ .a = 5, .b = 2 }, .f32).axis(.b)); - } - - /// The number of element inside a Tensor described by this shape. - pub fn count(self: Shape) usize { - var res: i64 = 1; - for (self.dims()) |d| { - stdx.debug.assert(d >= 0, "Can't count elements in shape with negative dimension: {f}", .{self}); - res *= d; - } - return @intCast(res); - } - - /// Total size in bytes needed to represent this shape. - pub fn byteSize(self: Shape) usize { - return self.dtype().sizeOf() * self.count(); - } - - /// Compares the two shapes described, ignoring tagging. - pub fn eql(self: Shape, other: Shape) bool { - return std.mem.eql(i64, self.dims(), other.dims()) and self.dtype() == other.dtype(); - } - - /// Compares the two shapes described, ignoring tagging and dtype. - pub fn eqlDims(self: Shape, other: Shape) bool { - return std.mem.eql(i64, self.dims(), other.dims()); - } - - /// Compares the two shapes described including tags. - pub fn eqlWithTags(self: Shape, other: Shape) bool { - return self.eql(other) and std.mem.eql(Tag, self.tags(), other.tags()) and self.dtype() == other.dtype(); - } - - /// Format the shape. - /// Default format: "Shape({.a=10, .b=20}, dtype=.f32)" - /// Bare format {_}: "{.a=10, .b=20}, dtype=.f32" - pub fn format(self: Shape, writer: *std.Io.Writer) !void { - _ = try writer.writeByte('{'); - - var need_comma = false; - for (self.dims(), 0..) |d, i| { - if (need_comma) try writer.writeByte(','); - const t = self.tag(i); - if (t != TagUnknown) { - try writer.print("{s}={d}", .{ t, d }); - } else { - try writer.print("{d}", .{d}); - } - - const part = self._partitioning.get(i); - if (part.toTag() != TagUnknown) { - try writer.print("/{s}", .{part.toTag()}); - } - - need_comma = true; - } - if (need_comma) try writer.writeByte(','); - _ = try writer.write(@tagName(self.dtype())); - _ = try writer.writeByte('}'); - } - - /// Broadcasts a Tensor to the given shape, extending dimensions if needed. - pub fn canBroadcastTo(self: Shape, other: Shape) bool { - // Already the right shape - if (std.mem.eql(i64, self.dims(), other.dims())) return true; - - // Non ambiguous broadcasting - // TODO: broad is error prone because of this: - // it will happily broadcast .{ .a = 10, .b = 1 } to .{ .b = 10, .a = 5 } - if (self.rank() == 0 or self.rank() == other.rank()) { - for (0..self.rank()) |i| { - if (self.dim(i) != 1 and self.dim(i) != other.dim(i)) return false; - } - return true; - } - - for (self.dims(), self.tags()) |d, t| { - // TODO this is also wrong when the axes are in different order in the two shapes. - const other_ax = other.hasTag(t) orelse return false; - if (d != 1 and d != other.dim(other_ax)) return false; - } - return true; - } - - pub fn reshape(self: Shape, new_shape_: anytype) Shape { - var new_shape: Shape = .{ ._dtype = self.dtype() }; - new_shape._dims, new_shape._tags = parseDimensions(new_shape_); - new_shape._partitioning.appendNTimes(.unknown, new_shape._dims.len) catch @panic("Rank too large"); - new_shape.inferMissingAxis(self.count()); - stdx.debug.assert(self.count() == new_shape.count(), "Can't reshape {any} to {any}", .{ self.dims(), new_shape.dims() }); - return new_shape; - } - - fn inferMissingAxis(self: *Shape, n_: usize) void { - stdx.debug.assert(std.mem.count(i64, self.dims(), &.{-1}) < 2, "Cannot infer multiple dimensions when reshaping to: {f}", .{self.*}); - - const inferred_ax = std.mem.indexOfScalar(i64, self.dims(), -1) orelse return; - // We can't use `self.count()` yet cause we have negative dims. - var tmp_count: i64 = 1; - for (self.dims()) |d| { - if (d > 0) { - tmp_count *= d; - } - } - const n: i64 = @intCast(n_); - // Abort, `reshape` will panic with more context. - if (@mod(n, tmp_count) != 0) { - return; - } - self._dims.set(inferred_ax, @divExact(n, tmp_count)); - } - - test reshape { - const x = Shape.init(.{ 2, 5, 3 }, .f32); - { - const res = x.reshape(.{ .auto, 3 }); - try testing.expectEqualSlices(i64, &.{ 10, 3 }, res.dims()); - } - { - const res = x.reshape(.{ 10, -1 }); - try testing.expectEqualSlices(i64, &.{ 10, 3 }, res.dims()); - } - { - const res = x.reshape(.{-1}); - try testing.expectEqualSlices(i64, &.{30}, res.dims()); - } - } - - pub fn setDim(self: Shape, ax: anytype, d: i64) Shape { - var res = self; - res._dims.set(self.axis(ax), d); - return res; - } - - pub const set = setDim; - - fn isAutoDim(v: anytype) bool { - return toTag(v) == toTag(.auto); - } - - fn isDynDim(v: anytype) bool { - return toTag(v) == toTag(.dyn); - } - - /// Inserts one ore more axes with the given dimensions, before the given axis. - /// Negative axis is interpreted wrt the current shape. - /// `.last` axis can be used to insert at the end (ie to append). - /// ``` - /// .{10, 11, 12}.insert(1, 2) -> .{10, 2, 11, 12} - /// .{10, 11, 12}.insert(-1, 2) -> .{10, 11, 2, 12} - /// .{10, 11, 12}.insert(.last, 2) -> .{10, 11, 12, 2} - /// ``` - pub fn insert(self: Shape, axis_: anytype, dimz: anytype) Shape { - const dims_, const tags_ = parseDimensions(dimz); - const ax = if (@TypeOf(axis_) == @EnumLiteral() and axis_ == .last) - self.rank() - else - self.axis(axis_); - - var res = self; - res._dims.insertSlice(ax, dims_.constSlice()) catch unreachable; - res._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; - - for (0..dims_.len) |_| { - res._partitioning.insert(ax, .unknown) catch unreachable; - } - - return res; - } - - test insert { - try testing.expectEqualSlices(i64, &.{ 10, 1, 11, 12 }, Shape.init(.{ 10, 11, 12 }, .f32).insert(1, .{1}).dims()); - try testing.expectEqualSlices(i64, &.{ 10, 11, 12, 1, 13 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).insert(-1, .{1}).dims()); - try testing.expectEqualSlices(i64, &.{ 10, 11, 12, 13, 1 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).insert(.last, .{1}).dims()); - } - - pub fn insertTag(self: Shape, axis_: anytype, d: i64, tag_: anytype) Shape { - stdx.debug.assert(self.rank() < constants.MAX_RANK - 1, "Can't insert new axis in {f}, it's already at max rank.", .{self}); - - const ax = if (@TypeOf(axis_) == @EnumLiteral() and axis_ == .last) - self.rank() - else - self.axis(axis_); - - var res = self; - res._dims.insert(ax, d) catch unreachable; - res._tags.insert(ax, toTag(tag_)) catch unreachable; - res._partitioning.insert(ax, .unknown) catch unreachable; - - return res; - } - - pub fn append(self: Shape, v: anytype) Shape { - var res = self; - const dims_, const tags_ = parseDimensions(v); - res._dims.appendSliceAssumeCapacity(dims_.constSlice()); - res._tags.appendSliceAssumeCapacity(tags_.constSlice()); - res._partitioning.appendNTimesAssumeCapacity(.unknown, dims_.len); - return res; - } - - test append { - try testing.expectEqualSlices( - i64, - &.{ 10, 11, 12, 1 }, - Shape.init(.{ 10, 11, 12 }, .f32).append(.{1}).dims(), - ); - - try testing.expect( - Shape.init(.{ .a = 10, .b = 11, .c = 12 }, .f32).eqlWithTags( - Shape.init(.{ .a = 10, .b = 11 }, .f32).append(.{ .c = 12 }), - ), - ); - } - - pub fn appendDim(self: Shape, d: i64, tag_: ?Tag) Shape { - var res = self; - res._dims.appendAssumeCapacity(d); - res._tags.appendAssumeCapacity(if (tag_) |t| t else TagUnknown); - res._partitioning.appendAssumeCapacity(.unknown); - return res; - } - - pub fn remove(self: Shape, axis_: anytype) Shape { - var res = self; - const a = self.axis(axis_); - _ = res._dims.orderedRemove(a); - _ = res._tags.orderedRemove(a); - _ = res._partitioning.orderedRemove(a); - return res; - } - - pub const drop = remove; - - test remove { - try testing.expectEqualSlices(i64, &.{ 10, 12 }, Shape.init(.{ 10, 11, 12 }, .f32).remove(1).dims()); - try testing.expectEqualSlices(i64, &.{ 10, 11, 12 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).remove(-1).dims()); - } - - pub fn removeMany(self: Shape, axes_: anytype) Shape { - var to_remove = self.axes(axes_); - if (to_remove.len == 0) return self; - std.mem.sort(u3, to_remove.slice(), {}, std.sort.asc(u3)); - - var sh: Shape = self; - const rk = self.rank(); - var res_ax: u32 = 0; - for (0..rk) |ax| { - if (std.mem.indexOfScalar(u3, to_remove.constSlice(), @intCast(ax))) |_| { - continue; - } - - sh._dims.buffer[res_ax] = self._dims.buffer[ax]; - sh._tags.buffer[res_ax] = self._tags.buffer[ax]; - sh._partitioning.buffer[res_ax] = self._partitioning.buffer[ax]; - res_ax += 1; - } - sh._dims.len = rk - to_remove.len; - sh._tags.len = rk - to_remove.len; - sh._partitioning.len = rk - to_remove.len; - return sh; - } - - test removeMany { - try testing.expectEqualSlices( - i64, - &.{12}, - Shape.init(.{ 10, 11, 12 }, .f32).removeMany(.{ 0, 1 }).dims(), - ); - try testing.expectEqualSlices( - i64, - &.{ 10, 11 }, - Shape.init(.{ 10, 11, 12, 13 }, .f32).removeMany(.{ -1, -2 }).dims(), - ); - } - - pub fn transpose(self: Shape, permutations: anytype) Shape { - std.debug.assert(self.rank() == permutations.len); - const permutations_ = self.axes(permutations); - var res = self; - const original_dims = self._dims; - const original_tags = self._tags; - const original_parts = self._partitioning; - - for (permutations_.constSlice(), 0..) |permutation, i| { - res._dims.set(i, original_dims.get(permutation)); - res._tags.set(i, original_tags.get(permutation)); - res._partitioning.set(i, original_parts.get(permutation)); - } - - return res; - } - - test transpose { - try testing.expect( - Shape.init(.{ 12, 11, 10 }, .f32).eql( - Shape.init(.{ 10, 11, 12 }, .f32).transpose(.{ 2, 1, 0 }), - ), - ); - - try testing.expect( - Shape.init(.{ .a = 10, .c = 12, .b = 11, .d = 13 }, .f32).eqlWithTags( - Shape.init(.{ .a = 10, .b = 11, .c = 12, .d = 13 }, .f32).transpose(.{ 0, 2, 1, 3 }), - ), - ); - - const shape = Shape.init(.{ .a = 10, .c = 12, .b = 11 }, .f32).withPartitioning(.{ .a = .batch, .c = .colors, .b = .open }).transpose(.{ 0, 2, 1 }); - try testing.expect(shape.eqlWithTags(Shape.init(.{ .a = 10, .b = 11, .c = 12 }, .f32))); - try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .open, .init(.colors) }, shape._partitioning.constSlice()); - } - - /// Tag each ax of this shape with tags from a tuple. - pub fn withTags(self: Shape, tagz: anytype) Shape { - const T = @TypeOf(tagz); - - if (T == Shape) { - return self.withTags(tagz.tags()); - } - - var res = self; - - if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, @EnumLiteral())) { - stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {any}", .{ self, tagz }); - for (tagz, 0..) |tag_, i| { - res._tags.set(i, toTag(tag_)); - } - return res; - } - - if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, @EnumLiteral())) { - stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {}", .{ self, tagz }); - inline for (tagz, 0..) |tag_, i| { - res._tags.set(i, toTag(tag_)); - } - return res; - } - - stdx.debug.compileError("Expected a tuple of enum literals eg: .{ .a, .b, .c } got: {any}", .{@TypeOf(tagz)}); - } - - test withTags { - { - const tagged = Shape.init(.{ 0, 1 }, .f32).withTags(.{ .a, .b }); - try testing.expectEqual(0, tagged.axis(.a)); - try testing.expectEqual(1, tagged.axis(.b)); - } - { - const tagged = Shape.init(.{ 0, 1, 2 }, .f32).withTags(.{ ._, .a, .b }); - try testing.expectEqual(1, tagged.axis(.a)); - try testing.expectEqual(2, tagged.axis(.b)); - } - { - const tagged = Shape.init(.{ 0, 1, 2, 3 }, .f32).withTags(.{ ._, ._, .a, .b }); - try testing.expectEqual(2, tagged.axis(.a)); - try testing.expectEqual(3, tagged.axis(.b)); - } - } - - /// Tag the last axes of this shape with tags from a tuple. - pub fn withPartialTags(self: Shape, tagz: anytype) Shape { - const T = @TypeOf(tagz); - - if (T == Shape) { - return self.withPartialTags(tagz.tags()); - } - - var res = self; - - if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, @EnumLiteral())) { - stdx.debug.assert(tagz.len <= self.rank(), "Too many tags for shape {f}, got {any}", .{ self, tagz }); - for (tagz, self.rank() - tagz.len..) |tag_, i| { - res._tags.set(i, toTag(tag_)); - } - return res; - } - - if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, @EnumLiteral())) { - stdx.debug.assert(tagz.len <= self.rank(), "Too many tags for shape {f}, got {}", .{ self, tagz }); - inline for (tagz, self.rank() - tagz.len..) |tag_, i| { - res._tags.set(i, toTag(tag_)); - } - return res; - } - - stdx.debug.compileError("Expected a tuple of enum literals eg: .{ .a, .b, .c } got: {any}", .{@TypeOf(tagz)}); - } - - test withPartialTags { - { - const tagged = Shape.init(.{ 0, 1 }, .f32).withPartialTags(.{ .a, .b }); - try testing.expectEqual(0, tagged.axis(.a)); - try testing.expectEqual(1, tagged.axis(.b)); - } - { - const tagged = Shape.init(.{ 0, 1, 2 }, .f32).withPartialTags(.{ .a, .b }); - try testing.expectEqual(1, tagged.axis(.a)); - try testing.expectEqual(2, tagged.axis(.b)); - } - { - const tagged = Shape.init(.{ 0, 1, 2, 3, 4 }, .f32).withPartialTags(.{ .a, .b }); - try testing.expectEqual(3, tagged.axis(.a)); - try testing.expectEqual(4, tagged.axis(.b)); - } - { - const tagged = Shape.init(.{ 0, 1, 2, 3, 4, 5, 6 }, .f32).withPartialTags(.{ .a, .b, .c }); - try testing.expectEqual(4, tagged.axis(.a)); - try testing.expectEqual(5, tagged.axis(.b)); - try testing.expectEqual(6, tagged.axis(.c)); - } - } - - pub fn withDtype(self: Shape, dt: DataType) Shape { - var res = self; - res._dtype = dt; - return res; - } - - pub fn withPartitioning(self: Shape, specs: anytype) Shape { - const T = @TypeOf(specs); - - var res = self.withDefaultPartitioning(); // todo add test for this new change - - if (stdx.meta.isStruct(T)) { - inline for (std.meta.fields(T)) |field| { - const partition_axis = @field(specs, field.name); - const axis_ = res.axisFromTagMaybe(toTag(field)); - - if (axis_) |ax| { - res._partitioning.set(ax, PartitionSpec.init(partition_axis)); - } else { - stdx.debug.panic("Partitioning axis {s} not found", .{field.name}); - } - } - } else { - stdx.debug.panic("Expected a struct of enum literals, got: {any}", .{T}); - } - - // Check that no mesh axis is used to partition multiple tensor dimensions. - var used_mesh_axes: stdx.BoundedArray(Tag, constants.MAX_RANK) = .{ .len = 0 }; - - for (res._partitioning.constSlice()) |spec| { - const axis_tag = spec.toTag(); - - if (std.mem.indexOfScalar(Tag, used_mesh_axes.constSlice(), axis_tag)) |_| { - stdx.debug.panic("Mesh axis '{s}' used to partition multiple tensor dimensions. This is not allowed.", .{axis_tag}); - } - - switch (spec) { - .unknown => {}, - .open => {}, - .replicated => {}, - .axis => used_mesh_axes.appendAssumeCapacity(axis_tag), - } - } - - return res; - } - - test withPartitioning { - var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(.{ .c = .feature, .a = .batch }); - try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .unknown, .init(.feature) }, shape._partitioning.constSlice()); - - shape = shape.withDefaultPartitioning(); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); - - shape = shape.withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); - try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .open, .init(.feature) }, shape._partitioning.constSlice()); - } - - pub fn mapPartitioningAxes(self: Shape, mapping: anytype) Shape { - const T = @TypeOf(mapping); - stdx.debug.assertComptime(stdx.meta.isStruct(T), "Mapping must be a struct, e.g., .{ .old_axis = .new_axis }", .{}); - - var res = self; - for (res._partitioning.slice()) |*spec| { - switch (spec.*) { - .unknown => {}, - .open => {}, - .replicated => {}, - .axis => |*old_axis_tag| { - inline for (std.meta.fields(T)) |field| { - const logical_axis_in_map = toTag(field); - - if (std.mem.eql(u8, std.mem.span(old_axis_tag.*), std.mem.span(logical_axis_in_map))) { - const physical_axis_from_map = @field(mapping, field.name); - spec.* = PartitionSpec.init(physical_axis_from_map); - break; - } - } - }, - } - } - return res; - } - - test mapPartitioningAxes { - const logical_shape = Shape.init(.{ .batch = 64, .seq = 128, .features = 512 }, .f32).withPartitioning(.{ - .batch = .data, - .seq = .replicated, - .features = .model, - }); - - const physical_shape = logical_shape.mapPartitioningAxes(.{ .data = .x, .model = .y }); - - try testing.expect(physical_shape.partition(.batch).eql(.init(.x))); - try testing.expect(physical_shape.partition(.features).eql(.init(.y))); - try testing.expect(physical_shape.partition(.seq).eql(.replicated)); - - const partial_mapped_shape = logical_shape.mapPartitioningAxes(.{ .data = .z }); - try testing.expect(partial_mapped_shape.partition(.batch).eql(.init(.z))); - try testing.expect(partial_mapped_shape.partition(.features).eql(.init(.model))); - } - - pub fn withDefaultPartitioning(self: Shape) Shape { - var res = self; - res._partitioning.clear(); - res._partitioning.appendNTimes(.unknown, self._dims.len) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); - return res; - } - - test withDefaultPartitioning { - var shape = Shape.init(.{ 10, 20, 30 }, .f32); - try testing.expectEqual(3, shape._partitioning.len); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); - - shape = shape.withPartitioning(.{ ._1 = .batch }); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .init(.batch), .unknown }, shape._partitioning.constSlice()); - - shape = shape.withDefaultPartitioning(); - try testing.expectEqual(3, shape._partitioning.len); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); - } - - pub fn withReplicatedPartitioning(self: Shape) Shape { - var res = self; - res._partitioning.clear(); - res._partitioning.appendNTimes(.replicated, self._dims.len) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); - return res; - } - - test withReplicatedPartitioning { - var shape = Shape.init(.{ 10, 20, 30 }, .f32); - try testing.expectEqual(3, shape._partitioning.len); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); - - shape = shape.withPartitioning(.{ ._1 = .batch }); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .init(.batch), .unknown }, shape._partitioning.constSlice()); - - shape = shape.withReplicatedPartitioning(); - try testing.expectEqual(3, shape._partitioning.len); - try testing.expectEqualSlices(PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._partitioning.constSlice()); - } - - pub fn hasAtLeastOnePartitionedAxis(self: Shape) bool { - for (self._partitioning.constSlice()) |spec| { - if (spec == .axis) return true; - } - return false; - } - - test hasAtLeastOnePartitionedAxis { - var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .x, .b = .replicated }); - try testing.expect(shape.hasAtLeastOnePartitionedAxis()); - - shape = shape.withPartitioning(.{ .a = .replicated, .b = .replicated }); - try testing.expect(!shape.hasAtLeastOnePartitionedAxis()); - - shape = shape.withDefaultPartitioning(); - try testing.expect(!shape.hasAtLeastOnePartitionedAxis()); - } - - pub fn isFullyPartitioned(self: Shape) bool { - for (self._partitioning.constSlice()) |spec| { - switch (spec) { - .replicated, .open, .unknown => return false, - .axis => {}, - } - } - return true; - } - - test isFullyPartitioned { - var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .x, .b = .y }); - try testing.expect(shape.isFullyPartitioned()); - - shape = shape.withPartitioning(.{ .a = .x, .b = .replicated }); - try testing.expect(!shape.isFullyPartitioned()); - - shape = shape.withDefaultPartitioning(); - try testing.expect(!shape.isFullyPartitioned()); - } - - pub fn isFullyReplicated(self: Shape) bool { - for (self._partitioning.constSlice()) |spec| { - if (spec != .replicated and spec != .unknown) { - return false; - } - } - return true; - } - - test isFullyReplicated { - var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .replicated, .b = .replicated }); - try testing.expect(shape.isFullyReplicated()); - - shape = shape.withPartitioning(.{ .a = .replicated, .b = .unknown }); - try testing.expect(shape.isFullyReplicated()); - - shape = shape.withPartitioning(.{ .a = .x, .b = .replicated }); - try testing.expect(!shape.isFullyReplicated()); - } - - pub fn partition(self: Shape, ax: anytype) PartitionSpec { - return self._partitioning.get(self.axis(ax)); - } - - test partition { - const shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); - try testing.expectEqual(PartitionSpec.init(.batch), shape.partition(.a)); - try testing.expectEqual(.open, shape.partition(.b)); - try testing.expectEqual(PartitionSpec.init(.feature), shape.partition(.c)); - } - - pub fn containsPartitionSpec(self: Shape, part: PartitionSpec) bool { - for (self._partitioning.constSlice()[0..self.rank()]) |dim_part| { - if (dim_part.eql(part)) return true; - } - return false; - } - - test containsPartitionSpec { - var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30, .d = 40 }, .f32).withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); - try testing.expect(shape.containsPartitionSpec(.init(.batch))); - try testing.expect(shape.containsPartitionSpec(.open)); - try testing.expect(!shape.containsPartitionSpec(.replicated)); - try testing.expect(shape.containsPartitionSpec(.unknown)); - - shape = shape.withDefaultPartitioning(); - try testing.expect(!shape.containsPartitionSpec(.init(.batch))); - try testing.expect(shape.containsPartitionSpec(.unknown)); - try testing.expect(!shape.containsPartitionSpec(.replicated)); - try testing.expect(!shape.containsPartitionSpec(.open)); - } - - /// Renames some of the tags in this shape. - /// Shape.init(.{ .a = 10, .b = 20 }).rename(.{ .b = .batch }); // .{ .a = 10, .batch = 20 }; - pub fn rename(self: Shape, renames: anytype) Shape { - const T = @TypeOf(renames); - stdx.debug.assertComptime(stdx.meta.isStructOfAny(T, isAxisConvertible), "Must pass a struct of enum literals. Passed: {any}", .{T}); - var res = self; - inline for (std.meta.fields(T)) |field| { - const new_field = @field(renames, field.name); - stdx.debug.assert(self.hasTag(new_field) == null, "{f}.rename({any}) failed because of duplicated axis {any}", .{ self, renames, new_field }); - res._tags.set(self.axis(field), toTag(new_field)); - } - return res; - } - - test rename { - { - const tagged = Shape.init(.{ .a = 0, .b = 1 }, .f32).rename(.{ .a = .x, .b = .y }); - try testing.expectEqual(0, tagged.dim(.x)); - try testing.expectEqual(1, tagged.dim(.y)); - } - { - const tagged = Shape.init(.{ .a = 0, .b = 1, .c = 2 }, .f32).rename(.{ .a = .x, .c = .z }); - try testing.expectEqual(0, tagged.dim(.x)); - try testing.expectEqual(1, tagged.dim(.b)); - try testing.expectEqual(2, tagged.dim(.z)); - } - } - - /// Computes the strides of the shape in units of elements. - /// For a shape {d0, d1, d2}, the strides would be {d1*d2, d2, 1}. - pub fn computeElementStrides(self: Shape) stdx.BoundedArray(i64, constants.MAX_RANK) { - return self.computeStrides(1); - } - - test computeElementStrides { - { - const s = Shape.init(.{ 4, 1024 }, .bf16); - const strides = s.computeElementStrides(); - try std.testing.expectEqualSlices(i64, &.{ 1024, 1 }, strides.constSlice()); - } - - { - const s = Shape.init(.{ 2, 3, 4 }, .f32); - const strides = s.computeElementStrides(); - try std.testing.expectEqualSlices(i64, &.{ 12, 4, 1 }, strides.constSlice()); - } - - { - const s = Shape.init(.{ 5, 6 }, .i16); - const strides = s.computeElementStrides(); - try std.testing.expectEqualSlices(i64, &.{ 6, 1 }, strides.constSlice()); - } - - { - const s = Shape.init(.{10}, .u8); - const strides = s.computeElementStrides(); - try std.testing.expectEqualSlices(i64, &.{1}, strides.constSlice()); - } - - { - const s = Shape.scalar(.f64); - const strides = s.computeElementStrides(); - try std.testing.expectEqualSlices(i64, &.{}, strides.constSlice()); - } - - { - const s = Shape.init(.{ 4, 1, 5 }, .i32); - const strides = s.computeElementStrides(); - try std.testing.expectEqualSlices(i64, &.{ 5, 5, 1 }, strides.constSlice()); - } - - { - const s = Shape.init(.{ .n = 8, .c = 3, .h = 224, .w = 224 }, .f32); - const strides = s.computeElementStrides(); - try std.testing.expectEqualSlices(i64, &.{ 3 * 224 * 224, 224 * 224, 224, 1 }, strides.constSlice()); - } - } - - /// Computes the strides of the shape in units of bytes. - /// For a shape {d0, d1, d2} of type T, the strides would be - /// {(d1*d2)*sizeof(T), d2*sizeof(T), 1*sizeof(T)}. - pub fn computeByteStrides(self: Shape) stdx.BoundedArray(i64, constants.MAX_RANK) { - return self.computeStrides(self.dtype().sizeOf()); - } - - test computeByteStrides { - { - const s = Shape.init(.{ 2, 3, 4 }, .f16); // 2 bytes - const strides = s.computeByteStrides(); - try std.testing.expectEqualSlices(i64, &.{ 12 * 2, 4 * 2, 1 * 2 }, strides.constSlice()); - } - - { - const s = Shape.init(.{ 4, 1024 }, .bf16); // 2 bytes - const strides = s.computeByteStrides(); - try std.testing.expectEqualSlices(i64, &.{ 1024 * 2, 1 * 2 }, strides.constSlice()); - } - - { - const s = Shape.init(.{ 5, 6 }, .c64); // 8 bytes (f32 + f32) - const strides = s.computeByteStrides(); - try std.testing.expectEqual(@sizeOf(std.math.Complex(f32)), 8); - try std.testing.expectEqualSlices(i64, &.{ 6 * 8, 1 * 8 }, strides.constSlice()); - } - - { - const s = Shape.init(.{ 128, 512 }, .i8); // 1 byte - const strides = s.computeByteStrides(); - try std.testing.expectEqualSlices(i64, &.{ 512, 1 }, strides.constSlice()); - } - - { - const s = Shape.init(.{ 1, 4096 }, .u32); // 4 bytes - const strides = s.computeByteStrides(); - try std.testing.expectEqualSlices(i64, &.{ 4096 * 4, 1 * 4 }, strides.constSlice()); - } - - { - const s = Shape.scalar(.f64); // 8 bytes - const strides = s.computeByteStrides(); - try std.testing.expectEqualSlices(i64, &.{}, strides.constSlice()); - } - } - - fn computeStrides(self: Shape, element_size: i64) stdx.BoundedArray(i64, constants.MAX_RANK) { - const rk = self.rank(); - var strides: stdx.BoundedArray(i64, constants.MAX_RANK) = .{ .len = rk }; - if (rk == 0) return strides; - - const V = @Vector(constants.MAX_RANK, i64); - const rank_mask = std.simd.iota(u8, constants.MAX_RANK) < @as(@Vector(constants.MAX_RANK, u8), @splat(rk)); - // For each axis compute the product of all following dimensions - // and the element size in bytes. - var d: V = @bitCast(self._dims.buffer); - d = @select(i64, rank_mask, d, @as(V, @splat(1))); - d = std.simd.shiftElementsLeft(d, 1, element_size); - d = std.simd.prefixScan(.Mul, -1, d); - - strides.buffer = @bitCast(d); - return strides; - } - - /// Returns the permutation needed to transpose this shape - /// so that the given axes are contiguous. - pub fn contiguousPerm(self: Shape, axes_: anytype) AxesArray { - const axes__, _ = self.parseAxes(axes_); - var perms = AxesArray.init(0) catch unreachable; - for (0..self.rank()) |i| { - if (std.mem.indexOfScalar(u3, axes__.constSlice(), @intCast(i))) |_| { - continue; - } - perms.appendAssumeCapacity(@intCast(i)); - } - perms.appendSliceAssumeCapacity(axes__.constSlice()); - return perms; - } - - test contiguousPerm { - const abc = Shape.init(.{ .a = 10, .b = 11, .c = 12 }, .f32); - try testing.expect( - Shape.init(.{ .b = 11, .c = 12, .a = 10 }, .f32).eqlWithTags( - abc.transpose(abc.contiguousPerm(.{.a}).constSlice()), - ), - ); - try testing.expect( - Shape.init(.{ .c = 12, .b = 11, .a = 10 }, .f32).eqlWithTags( - abc.transpose(abc.contiguousPerm(.{ .b, .a }).constSlice()), - ), - ); - const abcd = Shape.init(.{ .a = 10, .b = 11, .c = 12, .d = 13 }, .f32); - try testing.expect( - Shape.init(.{ .a = 10, .c = 12, .b = 11, .d = 13 }, .f32).eqlWithTags( - abcd.transpose(abcd.contiguousPerm(.{ .b, .d }).constSlice()), - ), - ); - - const abcde = Shape.init(.{ .a = 10, .b = 11, .c = 12, .d = 13, .e = 14 }, .f32); - try testing.expect( - Shape.init(.{ .a = 10, .b = 11, .d = 13, .c = 12, .e = 14 }, .f32).eqlWithTags( - abcde.transpose(abcde.contiguousPerm(.{ .b, .d, .c, .e }).constSlice()), - ), - ); - } - - /// Splits the given axis in several axes. - /// eg: `Shape.init(.{ .a = 10, .b = 3 }).split(.a, .{.a1 = 5, .a2 = 2}); -> .{.a1 = 5, .a2 = 2, .b = 3}` - /// The number of elements in the split shape must match the number of element - /// in the target axis. - pub fn splitAxis(self: Shape, axis_: anytype, split_shape_: anytype) Shape { - const ax = self.axis(axis_); - const dims_, const tags_ = parseDimensions(split_shape_); - var new_shape = self; - new_shape._dims.replaceRange(ax, 1, dims_.constSlice()) catch unreachable; - new_shape._tags.replaceRange(ax, 1, tags_.constSlice()) catch unreachable; - _ = new_shape._partitioning.orderedRemove(ax); - - for (0..dims_.len) |_| { - new_shape._partitioning.insert(ax, .unknown) catch unreachable; - } - - new_shape.inferMissingAxis(self.count()); - return new_shape; - } - - test splitAxis { - try testing.expect( - Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).eql( - Shape.init(.{ .a = 10, .b = 3 }, .f32).splitAxis(.a, .{ .a1 = 5, .a2 = 2 }), - ), - ); - try testing.expect( - Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).eql( - Shape.init(.{ .a = 10, .b = 3 }, .f32).splitAxis(.a, .{ .a1 = .auto, .a2 = 2 }), - ), - ); - } - - pub fn splitAxes(self: Shape, axes_: anytype) Shape { - const T = @TypeOf(axes_); - stdx.debug.assertComptime(stdx.meta.isStruct(T), "Must pass struct of enum literals like .{ .a = .{ .a1, .a2 } }. Passed: {any}", .{T}); - - var res = self; - inline for (std.meta.fields(T)) |field| { - res = res.splitAxis(field, @field(axes_, field.name)); - } - return res; - } - - test splitAxes { - try testing.expect( - Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).eql( - Shape.init(.{ .a = 10, .b = 3 }, .f32).splitAxes(.{ .a = .{ .a1 = 5, .a2 = .auto } }), - ), - ); - try testing.expect( - Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).eql( - Shape.init(.{ .a = 10, .b = 3 }, .f32).splitAxes(.{ .a = .{ .a1 = 5, .a2 = .auto } }), - ), - ); - } - - /// Merge the given axes into one axis. - /// eg: `Shape.init(.{.a1 = 5, .a2 = 2, .b = 3}).merge(.{ .a = .{ .a1, .a2 }); -> .{ .a = 10, .b = 3 }` - pub fn mergeAxis(self: Shape, axis_: anytype, axes_: anytype) Shape { - const axes_to_merge = self.axes(axes_); - stdx.debug.assert(axes_to_merge.len > 1, "Must merge at least two axes, got {any}", .{axes_}); - - var new_dim: i64 = 1; - // Merging partitioned axes is complex. If ANY axis being merged is partitioned, - // the resulting sharding is ambiguous. A propagation pass must resolve this. - // Marking it 'open' is the correct signal. If all are replicated/unknown, the - // result is also replicated/unknown. - // rule: .replicated wins over .unknown. .open wins over everything. - var resulting_partition_spec: PartitionSpec = .unknown; - - for (0..axes_to_merge.len) |i| { - const current_axis = axes_to_merge.get(i); - new_dim *= self.dim(current_axis); - - if (i > 0) { - stdx.debug.assert(current_axis == axes_to_merge.get(i - 1) + 1, "Can't merge shape {f} along non-contiguous axes {any}", .{ self, axes_ }); - } - - switch (self.partition(current_axis)) { - .axis, .open => resulting_partition_spec = .open, - .replicated => if (resulting_partition_spec != .open) { - resulting_partition_spec = .replicated; - }, - .unknown => {}, - } - } - - const first_axis_to_merge = axes_to_merge.get(0); - const num_axes_to_merge = axes_to_merge.len; - - var new_shape = self; - - new_shape._dims.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{new_dim}) catch @panic("mergeAxis failed on dims"); - new_shape._tags.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{toTag(axis_)}) catch @panic("mergeAxis failed on tags"); - new_shape._partitioning.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{resulting_partition_spec}) catch @panic("mergeAxis failed on partitioning"); - - return new_shape; - } - - test mergeAxis { - try testing.expect( - Shape.init(.{ .a = 10, .b = 3, .c = 4 }, .f32).eqlWithTags( - Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3, .c = 4 }, .f32).mergeAxis(.a, .{ .a1, .a2 }), - ), - ); - try testing.expect( - Shape.init(.{ .a = 5, .c = 6 }, .f32).eqlWithTags( - Shape.init(.{ .a = 5, .b1 = 2, .b2 = 3 }, .f32).mergeAxis(.c, .{ .b1, .b2 }), - ), - ); - try testing.expect( - Shape.init(.{ .a = 10, .b = 3 }, .f32).eqlWithTags( - Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).mergeAxis(.a, .{ toTag(.a1), toTag(.a2) }), - ), - ); - try testing.expect( - Shape.init(.{ .a = 10, .b = 3 }, .f32).eqlWithTags( - Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).mergeAxis(toTag(.a), @as([]const Tag, &.{ toTag(.a1), toTag(.a2) })), - ), - ); - try testing.expect( - Shape.init(.{ .a = 10, .b = 3 }, .f32).eqlWithTags( - Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).mergeAxis(.a, @as([]const usize, &.{ 0, 1 })), - ), - ); - } - - test "mergeAxis partitioning" { - // Merging a partitioned axis with a replicated one -> open - const shape1 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32) - .withPartitioning(.{ .a = .replicated, .b = .x, .c = .replicated }); - const merged1 = shape1.mergeAxis(.ab, .{ .a, .b }); - try testing.expect(merged1.partition(.ab).eql(.open)); - try testing.expect(merged1.dim(.ab) == 10); - try testing.expect(merged1.rank() == 2); - - // Merging two replicated axes -> replicated - const shape2 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32) - .withPartitioning(.{ .a = .replicated, .b = .replicated, .c = .y }); - const merged2 = shape2.mergeAxis(.ab, .{ .a, .b }); - try testing.expect(merged2.partition(.ab).eql(.replicated)); - try testing.expect(merged2.partition(.c).eql(.init(.y))); - - // Merging two unknown axes -> unknown - const shape3 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32); // default is .unknown - const merged3 = shape3.mergeAxis(.ab, .{ .a, .b }); - try testing.expect(merged3.partition(.ab).eql(.unknown)); - } - - pub fn mergeAxes(self: Shape, axes_: anytype) Shape { - const T = @TypeOf(axes_); - stdx.debug.assertComptime(stdx.meta.isStruct(T), "Must pass struct of enum literals like .{ .a = .{ .a1, .a2 } }. Passed: {any}", .{T}); - - var res = self; - inline for (std.meta.fields(T)) |field| { - stdx.debug.assertComptime(stdx.meta.isTupleOfAny(field.type, isAxisConvertible) or stdx.meta.isSliceOfAny(field.type, isAxisConvertible), "Must pass struct of axes. Passed: {any}", .{field.type}); - res = res.mergeAxis(field, @field(axes_, field.name)); - } - return res; - } - - test mergeAxes { - try testing.expect( - Shape.init(.{ .a = 10, .b = 3 }, .f32).eqlWithTags( - Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).mergeAxes(.{ .a = .{ .a1, .a2 } }), - ), - ); - try testing.expect( - Shape.init(.{ .a = 10, .b = 3 }, .f32).eqlWithTags( - Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).mergeAxes(.{ .a = .{ toTag(.a1), toTag(.a2) } }), - ), - ); - try testing.expect( - Shape.init(.{ .a = 10, .b = 3 }, .f32).eqlWithTags( - Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).mergeAxes(.{ .a = .{ 0, 1 } }), - ), - ); - try testing.expect( - Shape.init(.{ .a = 10, .b = 3 }, .f32).eqlWithTags( - Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).mergeAxes(.{ .a = @as([]const usize, &.{ 0, 1 }) }), - ), - ); - } - - fn intersectTags(a: []const Tag, b: []const Tag) TagsArray { - var res = TagsArray.init(0) catch unreachable; - for (a) |tag_| { - if (std.mem.indexOfScalar(Tag, b, tag_)) { - res.appendAssumeCapacity(tag_); - } - } - return res; - } - - pub fn parseStruct(T: type, v: anytype) struct { stdx.BoundedArray(T, constants.MAX_RANK), TagsArray } { - const V = @TypeOf(v); - - var vals_: stdx.BoundedArray(T, constants.MAX_RANK) = .{}; - var tags_: TagsArray = .{}; - - if (comptime stdx.meta.isSliceOf(V, T)) { - for (v) |d| { - vals_.appendAssumeCapacity(d); - } - return .{ vals_, tags_ }; - } - - if (comptime stdx.meta.isStruct(V)) { - const fields = std.meta.fields(V); - stdx.debug.assertComptime(fields.len <= constants.MAX_RANK, "Too many fields in struct {} ({d}). Max supported is {d}.", .{ V, fields.len, constants.MAX_RANK }); - inline for (fields) |field| { - const fv = @field(v, field.name); - vals_.appendAssumeCapacity(fv); - - if (!comptime stdx.meta.isTuple(V)) { - tags_.appendAssumeCapacity(toTag(field)); - } - } - return .{ vals_, tags_ }; - } - - stdx.debug.compileError("parseStruct expects struct or tuple, got {}", .{V}); - } - - test parseStruct { - const vals_, const tags_ = parseStruct(f32, .{ .a = 0.1, .b = 1.2 }); - - try testing.expectEqualSlices(f32, &.{ 0.1, 1.2 }, vals_.constSlice()); - try testing.expectEqualSlices(Tag, &.{ "a".ptr, "b".ptr }, tags_.constSlice()); - } - - /// Parses a struct literal into a list of options for each axes. - pub fn parseAxesOptions(self: Shape, T: type, options: anytype, default: T) stdx.BoundedArray(T, constants.MAX_RANK) { - const V = @TypeOf(options); - - var res: stdx.BoundedArray(T, constants.MAX_RANK) = .{}; - if (comptime stdx.meta.isSliceOf(V, T)) { - stdx.debug.assert(options.len == self.rank(), "expects exactly {} options in slice, for {} got {}", .{ self.rank(), self, options.len }); - for (options) |d| { - res.appendAssumeCapacity(d); - } - } - - if (comptime stdx.meta.isStruct(V)) { - for (0..self.rank()) |_| res.appendAssumeCapacity(default); - const fields = std.meta.fields(V); - stdx.debug.assertComptime(fields.len <= constants.MAX_RANK, "expects up to {} options struct literal, got {}", .{ V, constants.MAX_RANK, fields.len }); - inline for (fields) |field| { - const a = self.axis(field); - res.buffer[a] = @field(options, field.name); - } - return res; - } - - stdx.debug.compileError("parseStruct expects struct or tuple, got {}", .{V}); - } - - test parseAxesOptions { - const shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .u8); - const scaling = shape.parseAxesOptions(f32, .{ .b = 1.2, .a = 0.1 }, 1.0); - - try testing.expectEqualSlices(f32, &.{ 0.1, 1.2, 1.0 }, scaling.constSlice()); - } - - test "comptimeShape" { - comptime { - const s = Shape.init(.{ .a = 5, .b = 6, .c = 7 }, .f32); - try testing.expectEqual(3, s.rank()); - try testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); - try testing.expectEqual(1, s.axis(.b)); - } - - // comptime only the shape - { - const s = comptime Shape.init(.{ .a = 5, .b = 6, .c = 7 }, .f32); - try testing.expectEqual(3, s.rank()); - try testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); - try testing.expectEqual(1, s.axis(.b)); - } - } - - pub fn outer(self: Shape, other: Shape) Shape { - var res_shape = self; - var batching_axes: u8 = 0; - for (0..other.rank()) |ax| { - if (other.tag(ax) != Shape.TagUnknown) { - if (self.hasTag(other.tag(ax))) |batching_ax| { - stdx.debug.assert(batching_ax == batching_axes and batching_ax == ax, "outer expects batching dims to be the first dims in both tensors, got outer({f}, {f})", .{ self, other }); - batching_axes += 1; - } - } - - res_shape = res_shape.appendDim(other.dim(ax), other.tag(ax)); - } - return res_shape; - } - - pub fn iterator(self: Shape) MultiDimIterator { - return .{ - .shape = self, - .current_coords = .{0} ** Shape.MAX_RANK, - .flat_index = 0, - }; - } -}; - -/// Iterates over all coordinates of a shape, yielding the multi-dimensional -/// coordinates and the corresponding flat index for each element. -pub const MultiDimIterator = struct { - shape: Shape, - current_coords: [Shape.MAX_RANK]i64, - flat_index: usize, - - pub const Item = struct { - coords: [Shape.MAX_RANK]i64, - rank: u4, - flat_index: usize, - }; - - pub fn next(self: *MultiDimIterator) ?Item { - if (self.flat_index >= self.shape.count()) { - return null; - } - - const rank = self.shape.rank(); - const item = Item{ - .coords = self.current_coords, - .rank = rank, - .flat_index = self.flat_index, - }; - - self.flat_index += 1; - - var i: usize = rank; - while (i > 0) { - i -= 1; - self.current_coords[i] += 1; - if (self.current_coords[i] < self.shape.dim(i)) { - return item; - } - self.current_coords[i] = 0; - } - - return item; - } -}; - -test "MultiDimIterator / Scalar" { - const shape = Shape.scalar(.f32); - var iter = shape.iterator(); - const item = iter.next().?; - - try std.testing.expectEqual(0, item.rank); - try std.testing.expectEqual(0, item.flat_index); - try std.testing.expect(iter.next() == null); -} - -test "MultiDimIterator / Empty" { - const shape = Shape.init(.{ .x = 5, .y = 0, .z = 2 }, .f32); - var iter = shape.iterator(); - - try std.testing.expect(iter.next() == null); -} - -test "MultiDimIterator / 1D shape" { - const shape = Shape.init(.{5}, .f32); - var iter = shape.iterator(); - var count: i64 = 0; - - while (iter.next()) |item| : (count += 1) { - try std.testing.expectEqual(count, item.coords[0]); - } - try std.testing.expectEqual(5, count); -} - -test "MultiDimIterator / 2D shape" { - const shape = Shape.init(.{ .rows = 2, .cols = 3 }, .f32); - var iter = shape.iterator(); - var items: std.ArrayList(MultiDimIterator.Item) = .empty; - defer items.deinit(std.testing.allocator); - - while (iter.next()) |item| { - try items.append(std.testing.allocator, item); - } - - try std.testing.expectEqual(6, items.items.len); - try std.testing.expectEqualSlices(i64, &.{ 0, 0 }, items.items[0].coords[0..2]); - try std.testing.expectEqualSlices(i64, &.{ 0, 1 }, items.items[1].coords[0..2]); - try std.testing.expectEqualSlices(i64, &.{ 0, 2 }, items.items[2].coords[0..2]); - try std.testing.expectEqualSlices(i64, &.{ 1, 0 }, items.items[3].coords[0..2]); - try std.testing.expectEqualSlices(i64, &.{ 1, 1 }, items.items[4].coords[0..2]); - try std.testing.expectEqualSlices(i64, &.{ 1, 2 }, items.items[5].coords[0..2]); - try std.testing.expectEqual(5, items.items[5].flat_index); -} - -test "MultiDimIterator / 3D shape" { - const shape = Shape.init(.{ .d = 2, .h = 2, .w = 2 }, .f32); - var iter = shape.iterator(); - const expected_coords = [_][3]i64{ - .{ 0, 0, 0 }, - .{ 0, 0, 1 }, - .{ 0, 1, 0 }, - .{ 0, 1, 1 }, - .{ 1, 0, 0 }, - .{ 1, 0, 1 }, - .{ 1, 1, 0 }, - .{ 1, 1, 1 }, - }; - var count: usize = 0; - - while (iter.next()) |item| : (count += 1) { - try std.testing.expectEqual(count, item.flat_index); - try std.testing.expectEqualSlices(i64, &expected_coords[count], item.coords[0..3]); - } - - try std.testing.expectEqual(8, count); -} - -test "MultiDimIterator / Shape with dimension of size 1" { - const shape = Shape.init(.{ .a = 2, .b = 1, .c = 3 }, .f32); - var iter = shape.iterator(); - var count: usize = 0; - - while (iter.next()) |item| : (count += 1) { - try std.testing.expectEqual(0, item.coords[1]); - } - try std.testing.expectEqual(6, count); -} diff --git a/zml2/tensor.zig b/zml2/tensor.zig deleted file mode 100644 index 776af877..00000000 --- a/zml2/tensor.zig +++ /dev/null @@ -1,4197 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); - -const dialects = @import("mlir/dialects"); -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -const Buffer = @import("buffer.zig").Buffer; -const Bufferized = @import("zml.zig").Bufferized; -const CompilationContext = @import("module.zig").CompilationContext; -const constants = @import("constants.zig"); -const ConstSlice = @import("slice.zig").ConstSlice; -const DataType = @import("dtype.zig").DataType; -const meta = @import("meta.zig"); -const mlirx = @import("mlirx.zig"); -const ops = @import("ops.zig"); -const Platform = @import("platform.zig").Platform; -const Shape = @import("shape.zig").Shape; -const Memory = Buffer.Memory; - -pub const Tensor = struct { - var current_id: std.atomic.Value(usize) = .{ .raw = 1 }; - - id: usize, - auto_broadcast: bool = false, - _shape: Shape, - _value: ?*const mlir.Value = null, - - pub fn init(shape_like: anytype, dt: DataType) Tensor { - return .fromShape(.init(shape_like, dt)); - } - - pub fn fromShape(shape_: Shape) Tensor { - return .{ .id = Tensor.current_id.fetchAdd(1, .seq_cst), ._shape = shape_ }; - } - - pub fn format(self: Tensor, writer: *std.Io.Writer) !void { - try writer.print("Tensor({f})", .{self._shape}); - } - - /// Returns the shape of a Tensor. - pub fn shape(self: Tensor) Shape { - return self._shape; - } - - /// Returns the datatype of a Tensor. - pub fn dtype(self: Tensor) DataType { - return self._shape.dtype(); - } - - /// Returns the rank of a Tensor. - pub fn rank(self: Tensor) u4 { - return self._shape.rank(); - } - - /// Returns the number of element of a Tensor. - pub fn count(self: Tensor) usize { - return self._shape.count(); - } - - /// Returns the size in bytes of a Tensor. - pub fn byteSize(self: Tensor) usize { - return self._shape.byteSize(); - } - - /// Internal use - /// - /// Creates a tensor from a Shape and an mlir.Value. - pub fn _result(sh: Shape, val: *const mlir.Value) Tensor { - const res: Tensor = .{ - ._shape = sh, - ._value = val, - .id = Tensor.current_id.fetchAdd(1, .seq_cst), - }; - - if (builtin.mode == .Debug) { - // Check that the MLIR value actually have the same shape. - const other = fromMlirValue(val); - stdx.debug.internalAssert(sh.eql(other._shape), "Created a {f} from Mlir value but expected {f}", .{ other._shape, res._shape }); - } - - return res; - } - - /// Creates a Tensor from a mlir.Value - /// - /// The shape is derived from the type of the mlir.Value. - pub fn fromMlirValue(val: *const mlir.Value) Tensor { - const ranked_tensor = val.type_().isA(mlir.RankedTensorType).?; - const n = ranked_tensor.rank(); - - stdx.debug.assert(n <= constants.MAX_RANK, "Can't represent MLIR tensor of rank {}, max supported rank is {}.", .{ n, constants.MAX_RANK }); - - var sh: Shape = .{ ._dtype = mlirx.Type.toDType(mlirCtx(), ranked_tensor.elementType()) }; - for (0..n) |i| { - sh._dims.appendAssumeCapacity(ranked_tensor.dimension(i)); - } - sh._tags.resize(n) catch unreachable; - sh._partitioning.resize(n) catch unreachable; - - return .{ ._shape = sh, ._value = val, .id = Tensor.current_id.fetchAdd(1, .seq_cst) }; - } - - /// Returns the dimension of axis 'axis_'. - /// - /// 'axis_' can be an integer or a tag. - pub fn dim(self: Tensor, axis_: anytype) i64 { - return self._shape.dim(axis_); - } - - /// Returns the dimensions of a Tensor as a slice. - pub fn dims(self: *const Tensor) []const i64 { - return self._shape.dims(); - } - - /// Returns the index of axis 'axis_'. - /// - /// 'axis_' can be an integer or a tag. - pub fn axis(self: Tensor, axis_: anytype) u3 { - return self._shape.axis(axis_); - } - - /// Returns the indices of each of the given axes. - /// - /// 'axis_' can be an integer or a tag. - pub fn axes(self: Tensor, axes_: anytype) stdx.BoundedArray(u3, constants.MAX_RANK) { - return self._shape.axes(axes_); - } - - /// Returns a Tensor tagged with the tags in 'tagz'. - pub fn withTags(self: Tensor, tagz: anytype) Tensor { - var res = self; - res._shape = self._shape.withTags(tagz); - return res; - } - - /// Returns a Tensor tagged partially with the tags in 'tagz'. - /// - /// If 'tagz' is of length n, the n last dimensions of the Tensor will be tagged. - pub fn withPartialTags(self: Tensor, tagz: anytype) Tensor { - var res = self; - res._shape = self._shape.withPartialTags(tagz); - return res; - } - - // TODO(Corentin) - pub fn withSharding(self: Tensor, axes_: anytype) Tensor { - _ = self; - _ = axes_; - @panic("Unimplemented"); - } - - pub fn toMemory(self: Tensor, kind: Memory) Tensor { - const ctx = CompilationContext.current(); - if (ctx.platform.target == .cpu) return self; - - const frontend_attributes = mlir.dictionaryAttribute(ctx.mlir_ctx, &.{ - .named(ctx.mlir_ctx, "_xla_buffer_placement", mlir.stringAttribute(ctx.mlir_ctx, kind.pjrtName())), - }); - - const op = dialects.stablehlo.custom_call( - ctx.mlir_ctx, - &.{self.value()}, - &.{self.value().type_()}, - .{ - .call_target_name = "annotate_device_placement", - .has_side_effect = true, - .backend_config = .{ .original = "" }, - .additional_attributes = &.{ - .named(ctx.mlir_ctx, "mhlo.frontend_attributes", frontend_attributes), - }, - }, - .unknown(ctx.mlir_ctx), - ).appendTo(currentBlock()); - - var res = _result(self._shape, op.result(0)); - ctx.currentScope().id_to_output_memory_kind.put(ctx.currentScope().arena.allocator(), res.id, kind) catch unreachable; - return res; - } - - /// Returns a Tensor with new tag names. - pub fn rename(self: Tensor, renames: anytype) Tensor { - var res = self; - res._shape = self._shape.rename(renames); - return res; - } - - pub fn renameAxis(self: Tensor, ax: i8, name: @EnumLiteral()) Tensor { - var res = self; - res._shape._tags.set(self.axis(ax), @tagName(name).ptr); - return res; - } - - /// Returns the mlir.Value associated with the Tensor. - /// - /// This will fail if used outside of a compilation context. - pub fn value(self: Tensor) *const mlir.Value { - if (CompilationContext.current().currentScope().id_to_argument.get(self.id)) |argument_index| { - return CompilationContext.current().currentScope().block.argument(argument_index); - } else if (self._value) |v| { - return v; - } else @panic("Something went really wrong, tensor is not an argument nor has an mlir.Value"); - } - - /// Tell PJRT compiler that memory should be reuse between the two tensors. - /// The compiler is already aggressively reusing tensors for intermediate results, - /// but this API allows to reuse buffer between input and output arguments - /// of a given function. - /// Note this is visible from the outside. The caller of a function with donations - /// is not allowed to reuse the donated input buffer after the call. - /// For `reuseBuffer` to be effective, it needs to propagate all the way through the output. - pub fn reuseBuffer(self: Tensor, origin: Tensor) Tensor { - const compilation_context = CompilationContext.current(); - const scope = compilation_context.currentScope(); - if (scope.id_to_argument.get(origin.id)) |argument_index| { - const gop = scope.id_to_donation.getOrPut(scope.arena.allocator(), self.id) catch unreachable; - gop.value_ptr.* = argument_index; - } else if (scope.id_to_donation.get(origin.id)) |origin_donation| { - const gop = scope.id_to_donation.getOrPut(scope.arena.allocator(), self.id) catch unreachable; - gop.value_ptr.* = origin_donation; - } - return self; - } - - /// Returns a Tensor containing the absolute value of each element of the input Tensor. - pub fn abs(self: Tensor) Tensor { - const op = dialects.stablehlo.abs(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); - const dt = switch (self.dtype()) { - .c64 => .f32, - .c128 => .f64, - else => self.dtype(), - }; - - return _result(self._shape.withDtype(dt), op.result(0)); - } - - /// Returns a Tensor whose elements have been bitcast to a target datatype. - /// - /// The Tensor shape needs to be compatible with the target datatype. - pub fn bitCast(self: Tensor, dt: DataType) Tensor { - const src_bit_size = self.dtype().bitSizeOf(); - const tgt_bit_size = dt.bitSizeOf(); - var res_shape = if (src_bit_size == tgt_bit_size) - self._shape - else if (src_bit_size > tgt_bit_size) gt: { - const new_dim = std.math.divExact(u16, src_bit_size, tgt_bit_size) catch std.debug.panic("bitcast expects target datatype to be a multiple of source datatype when upcasting, got {} (bitsize of {}) and {} (bitsize of {})", .{ self.dtype(), src_bit_size, dt, tgt_bit_size }); - var res = self._shape; - res = res.append(.{ .bitcast = new_dim }); - break :gt res; - } else lt: { - // several contiguous elements of self maps to one element of the result - stdx.debug.assert(self.dim(-1) * src_bit_size == tgt_bit_size, "bitcast expects elements of the input tensor last dimension to map to one element of the target datatype, got {0} elements (bitsize of {0}x{1}={2}) and {3} (bitsize of {4})", .{ self.dim(-1), src_bit_size, self.dim(-1) * src_bit_size, dt, tgt_bit_size }); - break :lt self._shape.remove(-1); - }; - - res_shape = res_shape.withDtype(dt); - - const op = dialects.stablehlo.bitcast_convert( - mlirCtx(), - self.value(), - mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - - return _result(res_shape, op.result(0)); - } - - /// Returns the given tensor as one contiguous buffer of bytes. - pub fn bytes(self: Tensor) Tensor { - return self.bitCast(.u8).flatten().withTags(.{.bytes}); - } - - /// Returns a Tensor containing the element-wise number of leading 0 bits in the input Tensor. - pub fn countLeadingZeros(self: Tensor) Tensor { - const op = dialects.stablehlo.count_leading_zeros(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape, op.result(0)); - } - - /// Returns a Tensor containing booleans indicating if each element of the input Tensor is finite. - pub fn isFinite(self: Tensor) Tensor { - const op = dialects.stablehlo.is_finite(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape.withDtype(.bool), op.result(0)); - } - - /// Returns a Tensor containing the element-wise number of bits set in the input Tensor. - pub fn popcnt(self: Tensor) Tensor { - stdx.debug.assert(self.dtype().isInteger(), "popcnt expects tensor type to be an integer, got {}", .{self.dtype()}); - const op = dialects.stablehlo.popcnt(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape, op.result(0)); - } - - /// Returns a Tensor containing the sign of the input Tensor element-wise. - pub fn sign(self: Tensor) Tensor { - const op = dialects.stablehlo.sign(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape, op.result(0)); - } - - /// Returns a Tensor containing the element-wise remainder of dividend 'self' and divisor 'other'. - /// - /// See https://pytorch.org/docs/stable/generated/torch.fmod.html for more details. - pub fn fmod(self: Tensor, divisor: f32) Tensor { - return self.remainder(scalar(divisor, .f32).broadcast(self._shape, &.{})); - } - - test fmod { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const inputs: [2][6]f32 = .{ .{ -3.0, -2, -1, 1, 2, 3 }, .{ 1, 2, 3, 4, 5, -5 } }; - const expectations: [2][6]f32 = .{ .{ -1.0, -0.0, -1.0, 1.0, 0.0, 1.0 }, .{ 1.0000, 0.5000, 0.0000, 1.0000, 0.5000, -0.5000 } }; - const divisors: [2]f32 = .{ 2, -1.5 }; - - inline for (inputs, expectations, divisors) |i, e, d| { - const input: zml.Tensor = .init(.{6}, .f32); - - const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.fmod, .{ input, d }, platform); - defer exe.deinit(); - - const input_buffer = try zml.Buffer.fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&i)); - defer input_buffer.deinit(); - - const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.fmod, .{input_buffer}); - - try zml.testing.expectClose(std.testing.io, ConstSlice.init(zml.Shape.init(.{6}, .f32), std.mem.sliceAsBytes(&e)), output, 1e-4); - } - } - - /// Returns a Tensor containing the element-wise left-shift operation of 'self' by 'other'. - pub fn shiftLeft(self: Tensor, other: Tensor) Tensor { - return binaryOp("shiftLeft", dialects.stablehlo.shift_left)(self, other); - } - - /// Returns a Tensor containing the element-wise arithmetic right-shift operation of 'self' by 'other'. - pub fn shiftRightArithmetic(self: Tensor, other: Tensor) Tensor { - return binaryOp("shiftRightArithmetic", dialects.stablehlo.shift_right_arithmetic)(self, other); - } - - /// Returns a Tensor containing the element-wise logical right-shift operation of 'self' by 'other'. - pub fn shiftRightLogical(self: Tensor, other: Tensor) Tensor { - return binaryOp("shiftRightLogical", dialects.stablehlo.shift_right_logical)(self, other); - } - - /// Returns the Cholesky decomposition of the input Tensor. - /// - /// 'lower' controls the form of the output Tensor. The output will be lower-triangular if 'lower' is true - /// and upper-triangular otherwise. - pub fn cholesky(self: Tensor, lower: bool) Tensor { - stdx.debug.assert(self.rank() <= 2, "cholesky expects tensor rank to be <= 2, got {}", .{self.rank()}); - - const op = dialects.stablehlo.cholesky(mlirCtx(), self.value(), lower, .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape, op.result(0)); - } - - /// Solves the system of linear equations formed by the input tensors. - pub fn triangularSolve(self: Tensor, other: Tensor, opts: dialects.stablehlo.TriangularSolveOpts) Tensor { - stdx.debug.assert(self.dtype() == other.dtype(), "triangularSolve expects tensors to be of the same type, got {} and {}", .{ self.dtype(), other.dtype() }); - stdx.debug.assert(self.rank() <= 2 and self.rank() == other.rank(), "triangularSolve expects tensors to have the same rank and be <= 2, got {} and {}", .{ self.rank(), other.rank() }); - - const op = dialects.stablehlo.triangular_solve(mlirCtx(), self.value(), other.value(), opts, .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape, op.result(0)); - } - - /// Returns a Tensor containing the element-wise rounding towards the nearest integer, breaking ties away from zero, of the input Tensor. - pub fn roundNearestAfz(self: Tensor) Tensor { - stdx.debug.assert(self.dtype().isFloat(), "roundNearestAfz expects tensor type to be a float, got {}", .{self.dtype()}); - - const op = dialects.stablehlo.round_nearest_afz(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape, op.result(0)); - } - - /// Returns a Tensor containing the element-wise rounding towards the nearest integer, breaking ties towards the even integer, of the input Tensor. - pub fn roundNearestEven(self: Tensor) Tensor { - stdx.debug.assert(self.dtype().isFloat(), "roundNearestEven expects tensor type to be a float, got {}", .{self.dtype()}); - - const op = dialects.stablehlo.round_nearest_even(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape, op.result(0)); - } - - /// Returns a Tensor of complex number converted from a pair of real and imaginary Tensors. - pub fn complex(re: Tensor, im: Tensor) Tensor { - stdx.debug.assert(re._shape.eql(im._shape), "complex expects tensor shapes to match, got {f} and {f}", .{ re._shape, im._shape }); - stdx.debug.assert(re.dtype() == .f32 or re.dtype() == .f64, "complex expects tensors type to be f32 or f64, got {}", .{re.dtype()}); - - const op = dialects.stablehlo.complex(mlirCtx(), re.value(), im.value(), .unknown(mlirCtx())).appendTo(currentBlock()); - const dt: DataType = if (re.dtype() == .f32) .c64 else .c128; - return _result(re._shape.withDtype(dt), op.result(0)); - } - - /// Returns a Tensor containing the element-wise real part of the input Tensor. - /// - /// Tensor type can float or complex. - pub fn real(self: Tensor) Tensor { - stdx.debug.assert(self.dtype().isComplex() or self.dtype().isFloat(), "real expects tensor type to be a float or a complex, got {}", .{self.dtype()}); - - if (self.dtype().isFloat()) { - return self; - } - - const dt: DataType = switch (self.dtype()) { - .c64 => .f32, - .c128 => .f64, - else => unreachable, - }; - const op = dialects.stablehlo.real(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape.withDtype(dt), op.result(0)); - } - - /// Returns a Tensor containing the element-wise imaginary part of the input Tensor. - /// - /// Tensor type can float or complex. - pub fn imag(self: Tensor) Tensor { - stdx.debug.assert(self.dtype().isFloat() or self.dtype().isComplex(), "imag expects tensor type to be a float or a complex, got {}", .{self.dtype()}); - - // Real tensors don't have imaginary part. - if (self.dtype().isFloat()) { - return Tensor.constant(self.dtype().zero()).broad(self._shape); - } - - const dt: DataType = switch (self.dtype()) { - .bf16, .f16, .f32, .f64 => self.dtype(), - .c64 => .f32, - .c128 => .f64, - else => unreachable, - }; - const op = dialects.stablehlo.imag(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape.withDtype(dt), op.result(0)); - } - - /// Returns the Fast Fourier Transform (FFT) of the input Tensor. - pub fn fft(self: Tensor, opts: dialects.stablehlo.FftOpts) Tensor { - // TODO: support tagged API. - - stdx.debug.assert(1 <= opts.length.len and opts.length.len <= 3, "fft expects 'opts.length' length to be between 1 and 3 (inclusive), got {}", .{opts.length.len}); - stdx.debug.assert(opts.length.len <= self.rank(), "fft expects 'opts.length' length to be less than tensor rank, got {} and {}", .{ opts.length.len, self.rank() }); - - const sh = switch (opts.kind) { - .FFT, .IFFT => blk: { - stdx.debug.assert(self.dtype().isComplex(), "fft({any}) expects tensor type to be complex, got {}", .{ opts, self.dtype() }); - - break :blk self._shape; - }, - .RFFT => blk: { - stdx.debug.assert(self.dtype() == .f32 or self.dtype() == .f64, "fft({}) expects tensor type to be f32 or f64, got {}", .{ opts, self.dtype() }); - stdx.debug.assert(std.mem.eql(i64, self.dims()[self.rank() - opts.length.len ..], opts.length), "fft({}) expects tensor last dimensions to match given lengths, got {} and {}", .{ opts, self.dims()[self.rank() - opts.length.len ..].len, opts.length.len }); - - const dt: DataType = switch (self.dtype()) { - .f32 => .c64, - else => .c128, - }; - const shape_ = self._shape.setDim(-1, @divExact(self.dim(-1), 2) + 1); - break :blk shape_.withDtype(dt); - }, - .IRFFT => blk: { - stdx.debug.assert(self.dtype().isComplex(), "fft({any}) expects tensor type to be complex, got {}", .{ opts, self.dtype() }); - stdx.debug.assert(std.mem.eql(i64, self.dims()[self.rank() - opts.length.len ..], opts.length), "fft({any}) expects tensor last dimensions to match given lengths, got {} and {}", .{ opts, self.dims()[self.rank() - opts.length.len ..].len, opts.length.len }); - - const dt: DataType = switch (self.dtype()) { - .c64 => .f32, - else => .f64, - }; - const shape_ = self._shape.setDim(-1, @divExact(self.dim(-1) - 1, 2)); - break :blk shape_.withDtype(dt); - }, - }; - - const op = dialects.stablehlo.fft(mlirCtx(), self.value(), opts, .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(sh, op.result(0)); - } - - pub const Rng = struct { - _state: Tensor, - algorithm: dialects.stablehlo.RngAlgorithm.Type = .DEFAULT, - - pub fn init() Rng { - return .{ ._state = .init(.{2}, .u64) }; - } - - pub fn initBuffer(platform: Platform, seed: u128, io: std.Io) !Bufferized(Rng) { - return .{ - ._state = try .fromBytes(io, platform, Shape.init(.{2}, .u64), std.mem.asBytes(&seed)), - }; - } - - pub fn deinitBuffer(self: *Bufferized(Rng)) void { - self._state.deinit(); - } - - /// Returns a Tensor of the given shape, filled with uniform random bits, and a new Rng state. - /// - /// The given Rng state should not be used anymore (or you'll get the same numbers again). - /// The output is guaranteed to be deterministic function of `self` Rng state, - /// but it is not guaranteed to be deterministic between implementations. - pub fn bitGenerator(self: Rng, sh: Shape) struct { Rng, Tensor } { - const op = dialects.stablehlo.rng_bit_generator( - mlirCtx(), - self.algorithm, - self._state.value(), - mlir.rankedTensorType(self._state.dims(), mlirx.Type.fromDType(mlirCtx(), self._state.dtype())), - mlir.rankedTensorType(sh.dims(), mlirx.Type.fromDType(mlirCtx(), sh.dtype())), - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - return .{ self.update(op.result(0)), _result(sh, op.result(1)) }; - } - - fn update(self: Rng, new_state: *const mlir.Value) Rng { - return .{ - ._state = _result(self._state._shape, new_state).reuseBuffer(self._state), - .algorithm = self.algorithm, - }; - } - - /// Returns a Tensor of the given shape, filled with uniformly sampled floating point numbers from an interval, - /// and a new Rng state. - /// - /// https://en.wikipedia.org/wiki/Continuous_uniform_distribution - pub fn uniform( - self: Rng, - shape_: Shape, - opts: struct { min: f64 = 0, max: f64 = 1 }, - ) struct { Rng, Tensor } { - const dt = if (shape_.dtype().isFloat()) shape_.dtype() else .f32; - - const mantissa_bit_count = @import("dtype.zig").mantissaSize(dt); - const bit_count: usize = dt.bitSizeOf(); - const rng_bit_count = if (mantissa_bit_count < 8) 8 else bit_count; - const uint_dtype: DataType = switch (bit_count) { - 8 => .u8, - 16 => .u16, - 32 => .u32, - 64 => .u64, - else => stdx.debug.panic("uniform don't support non-byte aligned dtype. Got: {f}", .{shape_}), - }; - - const rng, const bits = self.bitGenerator(shape_.withDtype(uint_dtype)); - - // Erase bits outside of mantissa. - var float_bits = bits.shiftRightLogical(scalar(rng_bit_count - mantissa_bit_count, uint_dtype)); - - // Set exponent bits to represent e^0 (eg 127 for f32). - float_bits = float_bits.logical(.OR, scalar(1, dt).bitCast(uint_dtype)); - - // float_bits now uniformly represents number in [1, 2[ range. - // Let's convert to floats, and subtract one to go to [0, 1[ range. - var floats = float_bits.bitCast(dt).sub(scalar(1, dt)); - floats = floats.mul(scalar(opts.max - opts.min, dt)).addConstant(opts.min); - - // Convert back to integer if needed. - return .{ rng, floats.convert(shape_.dtype()) }; - } - - test uniform { - const zml = @import("zml.zig"); - const Stats = struct { - const Stats = @This(); - - mean: Tensor, - variance: Tensor, - min: Tensor, - max: Tensor, - - pub fn uniformStats( - rand: Rng, - shape_: Shape, - opts: struct { min: f64, max: f64 }, - ) struct { Rng, Stats } { - const rng, const data = rand.uniform(shape_, .{ .min = opts.min, .max = opts.max }); - const mean_ = data.mean(0); - const variance = data.sub(mean_.broad(data.shape())).pow(Tensor.scalar(2, .f32)).mean(0); - return .{ rng, .{ - .mean = mean_, - .variance = variance, - .min = data.min(0), - .max = data.max(0), - } }; - } - }; - - const platform = zml.testing.env(); - // Compute stats over a uniform distribution on [-2, 10]. - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.uniformStats, .{ Rng.init(), Shape.init(.{1024}, .f32), .{ .min = -2, .max = 10 } }, platform); - defer exe.deinit(); - - const rng_buffer = try Rng.initBuffer(platform, 1234, std.testing.io); - defer rng_buffer._state.deinit(); - - const rand, const stats = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Stats.uniformStats, .{rng_buffer}); - defer rand._state.deinit(); - defer stats.mean.deinit(); - defer stats.variance.deinit(); - defer stats.min.deinit(); - defer stats.max.deinit(); - - // Check the Rng state has been modified. - try std.testing.expect(try rand._state.getValue(u128, std.testing.io) != 1234); - - // Check the mean and variance are close to theoritical values. - const mean_ = try stats.mean.getValue(f32, std.testing.io); - try std.testing.expectApproxEqAbs(4, mean_, 0.03); - - const variance = try stats.variance.getValue(f32, std.testing.io); - try std.testing.expectApproxEqAbs(12.0 * 12.0 / 12.0, variance, 0.01); - - // Check that no value is outside of the interval - // and we have samples close to the edges. - const min_ = try stats.min.getValue(f32, std.testing.io); - try std.testing.expect(min_ >= -2); - try std.testing.expectApproxEqAbs(-2, min_, 0.05); - - const max_ = try stats.max.getValue(f32, std.testing.io); - try std.testing.expect(max_ < 10); - try std.testing.expectApproxEqAbs(10, max_, 0.05); - } - - /// Returns a Tensor of the given shape, filled with floating point numbers sampled from a normal distribution. - /// - /// Note: this uses stablehlo.rng which is deprecated. - /// https://github.com/openxla/stablehlo/blob/main/rfcs/20240503-opset-deprecations.md - pub fn normal(sh: Shape, opts: struct { mean: f64 = 0, stddev: f64 = 1 }) Tensor { - stdx.debug.assert(sh.dtype().isFloat(), "normal expects tensor type to be a float, got {}", .{sh.dtype()}); - - const a = Tensor.constant(DataType.Value.init(sh.dtype(), opts.mean)); - const b = Tensor.constant(DataType.Value.init(sh.dtype(), opts.stddev)); - const res_tensor_shape = Tensor.constantTensor(Shape.init(.{sh.rank()}, .i64), std.mem.sliceAsBytes(sh.dims())); - const op = dialects.stablehlo.rng(mlirCtx(), a.value(), b.value(), res_tensor_shape.value(), .NORMAL, .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(sh, op.result(0)); - } - - /// Returns a Tensor of the given shape, filled with floating point numbers sampled from a Gumbel distribution, and a new Rng state. - /// - /// Often used in ML because of the reparametrization tricks. - /// Sampling from a gumbel distribution is equivalent to sample - /// from a softmax distribution, but doesn't require to compute the sum of exponentials. - /// https://en.wikipedia.org/wiki/Gumbel_distribution#Gumbel_reparametrization_tricks - /// See `sampleTokens` for a practical use case. - /// Note: we only implement the μ=0, β=1 version. - pub fn gumbel(self: Rng, shape_: Shape) struct { Rng, Tensor } { - const rand, const u = self.uniform( - // Always use .f32 to have a big enough mantissa. - shape_.withDtype(.f32), - // We don't want 0 to be sampled otherwise `log` will return -inf. - .{ .min = std.math.floatEps(f32), .max = 1 }, - ); - return .{ rand, u.log().scale(-1).log().scale(-1).convert(shape_.dtype()) }; - } - - test gumbel { - const zml = @import("zml.zig"); - const Stats = struct { - const Stats = @This(); - - mean: Tensor, - variance: Tensor, - actual_dist: Tensor, - - pub fn gumbelStats(rand: Rng, target_dist: Tensor) struct { Rng, Stats } { - const s = Shape.init(.{ .n = 1024, .d = 4 }, .f32); - const rng, const data = rand.gumbel(s); - const flat = data.flatten(); - const mean_ = flat.mean(0); - const variance = flat.sub(mean_.broad(flat.shape())).pow(Tensor.scalar(2, .f32)).mean(0); - - // Test out the gumbel reparametrization trick - var x = target_dist.log().withTags(.{.d}).broad(s); - x = x.add(data); - const samples = x.argMax(.d).indices.squeeze(.d); - - // count 0, 1, 2 and 3 in samples: - // - map 0 to 1, 1 to 2**16, 2 to 2**32, 3 to N**58 - // - sum in u64 - // - split to [4]u16 - const powers = blk: { - var powers: [4]u64 = undefined; - for (&powers, 0..) |*p, i| p.* = std.math.pow(u64, 2, i * 16); - break :blk powers; - }; - const values = Tensor.constantTensor(Shape.init(.{4}, .u64), std.mem.sliceAsBytes(&powers)).withTags(.{.d}); - const counts = values.gather(.{ .d = samples }, .{}).sum(.n).bitCast(.u16); - const actual_dist = counts.reshape(target_dist.shape()).convert(target_dist.dtype()).divByConst(s.dim(.n)); - return .{ rng, .{ .mean = mean_, .variance = variance, .actual_dist = actual_dist } }; - } - }; - - const platform = zml.testing.env(); - const tgt_dist_data = [_]f32{ 2.0, 1.0, 4.0, 3.0 }; - const tgt_dist: Tensor = .init(.{tgt_dist_data.len}, .f32); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.gumbelStats, .{ Rng.init(), tgt_dist }, platform); - defer exe.deinit(); - - const rng_buffer = try Rng.initBuffer(platform, 1234, std.testing.io); - defer rng_buffer._state.deinit(); - const tgt_dist_buffer: Buffer = try .fromBytes(std.testing.io, platform, tgt_dist.shape(), std.mem.sliceAsBytes(&tgt_dist_data)); - defer tgt_dist_buffer.deinit(); - - const rand, const stats = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Stats.gumbelStats, .{ rng_buffer, tgt_dist_buffer }); - defer rand._state.deinit(); - defer stats.mean.deinit(); - defer stats.variance.deinit(); - defer stats.actual_dist.deinit(); - - // Check the Rng state has been modified. - try std.testing.expect(try rand._state.getValue(i128, std.testing.io) != 1234); - - // Check the mean and variance are close to theoritical values. - const mean_ = try stats.mean.getValue(f32, std.testing.io); - try std.testing.expectApproxEqAbs(0.5772, mean_, 0.02); - - const variance = try stats.variance.getValue(f32, std.testing.io); - const pi = std.math.pi; - try std.testing.expectApproxEqAbs(pi * pi / 6.0, variance, 0.03); - - // Check the distribution obtained with the gumbel trick matches the target distribution. - const actual_dist = try stats.actual_dist.getValue([4]f32, std.testing.io); - for (tgt_dist_data, actual_dist) |tgt, actual| { - // We normalize tgt_dist to make it a well formed distribution. - // We didn't do it before calling gumbel, because the gumbel trick - // doesn't require normalized distributions as input. - try std.testing.expectApproxEqAbs(tgt / 10.0, actual, 0.05); - } - } - }; - - /// Returns a Tensor containing the element-wise conversion to another floating point type. - pub fn reducePrecision(self: Tensor, exponent_bits: i32, mantissa_bits: i32) Tensor { - stdx.debug.assert(self.dtype().isFloat(), "reducePrecision expects tensor type to be a float, got {}", .{self.dtype()}); - stdx.debug.assert(1 <= exponent_bits, "reducePrecision expects 'exponent_bits' to be >= 1, got {}", .{exponent_bits}); - stdx.debug.assert(0 <= mantissa_bits, "reducePrecision expects 'mantissa_bits' to be positive, got {}", .{mantissa_bits}); - - const op = dialects.stablehlo.reduce_precision(mlirCtx(), self.value(), exponent_bits, mantissa_bits, .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape, op.result(0)); - } - - fn convolution(self: Tensor, other: Tensor, opts: dialects.stablehlo.ConvolutionOpts) Tensor { - stdx.debug.assert(self.rank() == other.rank(), "convolution expects tensor ranks to match, got {} and {}", .{ self.rank(), other.rank() }); - const N = self.rank(); - stdx.debug.guard(opts.window_strides.len == N - 2, @src()); - for (opts.window_strides) |s| stdx.debug.guard(0 < s, @src()); - stdx.debug.guard(opts.lhs_dilation.len == N - 2, @src()); - for (opts.lhs_dilation) |d| stdx.debug.guard(0 < d, @src()); - stdx.debug.guard(opts.rhs_dilation.len == N - 2, @src()); - for (opts.rhs_dilation) |d| stdx.debug.guard(0 < d, @src()); - stdx.debug.guard(opts.window_reversal.len == N - 2, @src()); - stdx.debug.guard(@rem(self.dim(opts.input_batch_dimension), opts.batch_group_count) == 0, @src()); - stdx.debug.guard(@rem(self.dim(opts.input_feature_dimension), opts.feature_group_count) == 0, @src()); - stdx.debug.guard(opts.input_spatial_dimensions.len == N - 2, @src()); - stdx.debug.guard(opts.input_batch_dimension != opts.input_feature_dimension, @src()); - stdx.debug.guard(0 <= opts.input_batch_dimension and opts.input_batch_dimension < N, @src()); - stdx.debug.guard(0 <= opts.input_feature_dimension and opts.input_feature_dimension < N, @src()); - for (opts.input_spatial_dimensions, 0..) |d, i| { - stdx.debug.guard(d != opts.input_batch_dimension, @src()); - stdx.debug.guard(d != opts.input_feature_dimension, @src()); - stdx.debug.guard(0 <= d and d < N, @src()); - if (i < opts.input_spatial_dimensions.len - 1) continue; - stdx.debug.guard(std.mem.indexOfScalar(i64, opts.input_spatial_dimensions[i + 1 ..], d) == null, @src()); - } - stdx.debug.guard(other.dim(opts.kernel_input_feature_dimension) == @divTrunc(self.dim(opts.input_feature_dimension), opts.feature_group_count), @src()); - stdx.debug.guard(@rem(other.dim(opts.kernel_output_feature_dimension), opts.batch_group_count) == 0, @src()); - stdx.debug.guard(@rem(other.dim(opts.kernel_output_feature_dimension), opts.feature_group_count) == 0, @src()); - stdx.debug.guard(opts.kernel_spatial_dimensions.len == N - 2, @src()); - stdx.debug.guard(opts.kernel_input_feature_dimension != opts.kernel_output_feature_dimension, @src()); - stdx.debug.guard(0 <= opts.kernel_input_feature_dimension and opts.kernel_input_feature_dimension < N, @src()); - stdx.debug.guard(0 <= opts.kernel_output_feature_dimension and opts.kernel_output_feature_dimension < N, @src()); - for (opts.kernel_spatial_dimensions, 0..) |d, i| { - stdx.debug.guard(d != opts.kernel_input_feature_dimension, @src()); - stdx.debug.guard(d != opts.kernel_output_feature_dimension, @src()); - stdx.debug.guard(0 <= d and d < N, @src()); - if (i < opts.kernel_spatial_dimensions.len - 1) continue; - stdx.debug.guard(std.mem.indexOfScalar(i64, opts.kernel_spatial_dimensions[i + 1 ..], d) == null, @src()); - } - stdx.debug.guard(opts.output_spatial_dimensions.len == N - 2, @src()); - stdx.debug.guard(opts.output_batch_dimension != opts.output_feature_dimension, @src()); - stdx.debug.guard(0 <= opts.output_batch_dimension and opts.output_batch_dimension < N, @src()); - stdx.debug.guard(0 <= opts.output_feature_dimension and opts.output_feature_dimension < N, @src()); - for (opts.output_spatial_dimensions, 0..) |d, i| { - stdx.debug.guard(d != opts.output_batch_dimension, @src()); - stdx.debug.guard(d != opts.output_feature_dimension, @src()); - stdx.debug.guard(0 <= d and d < N, @src()); - if (i < opts.output_spatial_dimensions.len - 1) continue; - stdx.debug.guard(std.mem.indexOfScalar(i64, opts.output_spatial_dimensions[i + 1 ..], d) == null, @src()); - } - stdx.debug.guard(0 < opts.feature_group_count, @src()); - stdx.debug.guard(0 < opts.batch_group_count, @src()); - stdx.debug.guard(opts.feature_group_count == 1 or opts.batch_group_count == 1, @src()); - var used_opts = opts; - used_opts.pad_shape = &.{ @intCast(N - 2), 2 }; - used_opts.precision_config = &.{ .DEFAULT, .DEFAULT }; - - var new_shape = self._shape; - var res_dim: i64 = undefined; - - for (0..N) |i| { - if (i == @as(usize, @intCast(opts.output_batch_dimension))) { - res_dim = @divTrunc(self.dim(opts.input_batch_dimension), opts.batch_group_count); - } else if (i == @as(usize, @intCast(opts.output_feature_dimension))) { - res_dim = other.dim(opts.kernel_output_feature_dimension); - } else { - // calculate spatial dimension value - const spatial_dim: usize = std.mem.indexOfScalar(i64, opts.output_spatial_dimensions, @as(i64, @intCast(i))).?; - const lhs_dim = opts.input_spatial_dimensions[spatial_dim]; - const rhs_dim = opts.kernel_spatial_dimensions[spatial_dim]; - const dilated_input_shape_lhs_dim: i64 = if (self.dim(lhs_dim) == 0) 0 else (self.dim(lhs_dim) - 1) * opts.lhs_dilation[spatial_dim] + 1; - const left_pad_value, const right_pad_value = if (opts.pad_value.len == 1) - .{ opts.pad_value[0], opts.pad_value[0] } - else - .{ opts.pad_value[2 * spatial_dim], opts.pad_value[2 * spatial_dim + 1] }; - const padded_input_shape_lhs_dim = left_pad_value + dilated_input_shape_lhs_dim + right_pad_value; - const dilated_window_shape_lhs_dim: i64 = if (other.dim(rhs_dim) == 0) 0 else (other.dim(rhs_dim) - 1) * opts.rhs_dilation[spatial_dim] + 1; - const is_empty_window_lhs_dim = padded_input_shape_lhs_dim == 0 or dilated_window_shape_lhs_dim > padded_input_shape_lhs_dim; - res_dim = if (is_empty_window_lhs_dim) 0 else @divTrunc(padded_input_shape_lhs_dim - dilated_window_shape_lhs_dim, opts.window_strides[spatial_dim]) + 1; - } - - new_shape = new_shape.set(i, res_dim); - } - - // inferred shape '[1, 256, 1, 12008]' is incompatible with return type of operation '[1, 256, 1, 11978]' - const op = dialects.stablehlo.convolution( - mlirCtx(), - self.value(), - other.value(), - mlir.rankedTensorType(new_shape.dims(), mlirx.Type.fromDType(mlirCtx(), new_shape.dtype())), - used_opts, - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - - return _result(new_shape, op.result(0)); - } - - /// Returns a Tensor containing the result of the 1D convolution of 'input' by 'kernel'. - pub fn conv1d( - input: Tensor, - kernel: Tensor, - opts: struct { - window_strides: i64 = 1, - padding: []const i64 = &.{ 0, 0 }, - lhs_dilation: i64 = 1, - rhs_dilation: i64 = 1, - window_reversal: bool = false, - input_batch_dimension: i64 = 0, - input_feature_dimension: i64 = 1, - input_spatial_dimensions: i64 = 2, - kernel_output_feature_dimension: i64 = 0, - kernel_input_feature_dimension: i64 = 1, - kernel_spatial_dimensions: i64 = 2, - output_batch_dimension: i64 = 0, - output_feature_dimension: i64 = 1, - output_spatial_dimensions: i64 = 2, - feature_group_count: i64 = 1, - batch_group_count: i64 = 1, - }, - ) Tensor { - return input.convolution(kernel, .{ - .window_strides = &.{opts.window_strides}, - .pad_value = opts.padding, - .lhs_dilation = &.{opts.lhs_dilation}, - .rhs_dilation = &.{opts.rhs_dilation}, - .window_reversal = &.{opts.window_reversal}, - .input_batch_dimension = opts.input_batch_dimension, - .input_feature_dimension = opts.input_feature_dimension, - .input_spatial_dimensions = &.{opts.input_spatial_dimensions}, - .kernel_input_feature_dimension = opts.kernel_input_feature_dimension, - .kernel_output_feature_dimension = opts.kernel_output_feature_dimension, - .kernel_spatial_dimensions = &.{opts.kernel_spatial_dimensions}, - .output_batch_dimension = opts.output_batch_dimension, - .output_feature_dimension = opts.output_feature_dimension, - .output_spatial_dimensions = &.{opts.output_spatial_dimensions}, - .feature_group_count = opts.feature_group_count, - .batch_group_count = opts.batch_group_count, - }); - } - - /// Returns a Tensor containing the result of the 2D convolution of 'input' by 'kernel'. - /// Defaults values correspond to a (B, C_in, W, H) image, (C_out, C_in, W, H) kernel weights and (B, C_out, W, H) output. - pub fn conv2d( - input: Tensor, - kernel: Tensor, - opts: struct { - window_strides: []const i64 = &.{ 1, 1 }, - padding: []const i64 = &.{ 0, 0, 0, 0 }, - lhs_dilation: []const i64 = &.{ 1, 1 }, - rhs_dilation: []const i64 = &.{ 1, 1 }, - window_reversal: []const bool = &.{ false, false }, - input_batch_dimension: i64 = 0, - input_feature_dimension: i64 = 1, - input_spatial_dimensions: []const i64 = &.{ 2, 3 }, - kernel_input_feature_dimension: i64 = 1, - kernel_output_feature_dimension: i64 = 0, - kernel_spatial_dimensions: []const i64 = &.{ 2, 3 }, - output_batch_dimension: i64 = 0, - output_feature_dimension: i64 = 1, - output_spatial_dimensions: []const i64 = &.{ 2, 3 }, - feature_group_count: i64 = 1, - batch_group_count: i64 = 1, - }, - ) Tensor { - return input.convolution(kernel, .{ - .window_strides = opts.window_strides, - .pad_value = opts.padding, - .lhs_dilation = opts.lhs_dilation, - .rhs_dilation = opts.rhs_dilation, - .window_reversal = opts.window_reversal, - .input_batch_dimension = opts.input_batch_dimension, - .input_feature_dimension = opts.input_feature_dimension, - .input_spatial_dimensions = opts.input_spatial_dimensions, - .kernel_input_feature_dimension = opts.kernel_input_feature_dimension, - .kernel_output_feature_dimension = opts.kernel_output_feature_dimension, - .kernel_spatial_dimensions = opts.kernel_spatial_dimensions, - .output_batch_dimension = opts.output_batch_dimension, - .output_feature_dimension = opts.output_feature_dimension, - .output_spatial_dimensions = opts.output_spatial_dimensions, - .feature_group_count = opts.feature_group_count, - .batch_group_count = opts.batch_group_count, - }); - } - - /// Returns a Tensor containing the element-wise addition of the input Tensors. - pub fn add(self: Tensor, other: Tensor) Tensor { - return binaryOp("add", dialects.stablehlo.add)(self, other); - } - - /// Returns a Tensor containing the element-wise subtraction of the input Tensors. - pub fn sub(self: Tensor, other: Tensor) Tensor { - return binaryOp("subtract", dialects.stablehlo.subtract)(self, other); - } - - /// Returns a Tensor containing the element-wise multiplication of the input Tensors. - pub fn mul(self: Tensor, other: Tensor) Tensor { - return binaryOp("mul", dialects.stablehlo.multiply)(self, other); - } - - /// Returns a Tensor containing the element-wise division of the input Tensors. - pub fn div(self: Tensor, other: Tensor) Tensor { - return binaryOp("div", dialects.stablehlo.divide)(self, other); - } - - /// Returns a Tensor containing the element-wise exponentiation of the input Tensors. - pub fn pow(self: Tensor, other: Tensor) Tensor { - return binaryOp("pow", dialects.stablehlo.power)(self, other); - } - - /// Returns a Tensor containing the element-wise maximum operation of the input Tensors. - pub fn maximum(self: Tensor, other: Tensor) Tensor { - return binaryOp("maximum", dialects.stablehlo.maximum)(self, other); - } - - /// Returns a Tensor containing the element-wise minimum operation of the input Tensors. - pub fn minimum(self: Tensor, other: Tensor) Tensor { - return binaryOp("minimum", dialects.stablehlo.minimum)(self, other); - } - - /// Returns a Tensor containing the element-wise remainder of dividend 'self' and divisor 'other'. - pub fn remainder(self: Tensor, other: Tensor) Tensor { - return binaryOp("remainder", dialects.stablehlo.remainder)(self, other); - } - - /// Returns a Tensor containing the element-wise addition of the input Tensor with a constant. - pub fn addConstant(self: Tensor, b: anytype) Tensor { - return self.add(.scalar(b, self.dtype())); - } - - /// Returns a Tensor containing the element-wise division of the input Tensor by a constant. - pub fn divByConst(self: Tensor, b: anytype) Tensor { - return self.div(.scalar(b, self.dtype())); - } - - /// Returns a Tensor containing the element-wise power of the input Tensor by a constant. - pub fn powByConst(self: Tensor, b: anytype) Tensor { - return self.pow(.scalar(b, self.dtype())); - } - - /// Returns a Tensor containing the element-wise multiplication of the input Tensor by a constant. - pub fn scale(self: Tensor, val: anytype) Tensor { - return self.mul(.scalar(val, self.dtype())); - } - - pub const LogicalOp = enum { OR, XOR, AND }; - - /// Returns a Tensor containing the element-wise logical operation of the input Tensors. - pub fn logical(self: Tensor, comptime logical_op: LogicalOp, other: Tensor) Tensor { - return switch (logical_op) { - .OR => binaryOp("or", dialects.stablehlo.or_)(self, other), - .XOR => binaryOp("xor", dialects.stablehlo.xor)(self, other), - .AND => binaryOp("and", dialects.stablehlo.and_)(self, other), - }; - } - - /// Returns a Tensor containing the element-wise floor operation of the input Tensor. - pub fn floor(self: Tensor) Tensor { - return _result(self._shape, dialects.stablehlo.floor(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()).result(0)); - } - - /// Returns a Tensor containing the element-wise ceil operation of the input Tensor. - pub fn ceil(self: Tensor) Tensor { - return _result(self._shape, dialects.stablehlo.ceil(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()).result(0)); - } - - /// Returns a Tensor containing the element-wise conversion to another type. - pub fn convert(self: Tensor, to: DataType) Tensor { - if (to == self.dtype()) { - return self; - } - - const res_type = mlir.rankedTensorType(self.shape().dims(), mlirx.Type.fromDType(mlirCtx(), to)); - const op = dialects.stablehlo.convert(mlirCtx(), self.value(), res_type, .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape.withDtype(to), op.result(0)); - } - - test convert { - const floats = @import("floats.zig"); - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - // f4e2m1 - { - const x = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; - var x_f4: [x.len]floats.Float4E2M1 = undefined; - for (&x_f4, &x) |*xi_f4, xi| xi_f4.* = .fromF32(xi); - - const x_d: Tensor = .init(.{x.len}, .f32); - const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.convert, .{ x_d, .f4e2m1 }, platform); - defer exe.deinit(); - - const x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), std.mem.sliceAsBytes(&x)); - defer x_d_buffer.deinit(); - - const x_f4_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); - const x_f4_xla = try x_f4_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); - defer x_f4_xla.free(std.testing.allocator); - - errdefer std.log.warn("convert(.f4e2m1) failed !\ninput f32:\n{e}\nzml.floats computed:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x), x_f4, x_f4_xla }); - try std.testing.expectEqualDeep(&x_f4, x_f4_xla.items(floats.Float4E2M1)); - } - - // f8e3m4 - { - const x = [_]f32{ 1.1 / 4.0, 1.1 / 8.0, 1.1 / 16.0, 1.1 / 32.0, 1.1 / 64.0, 1.1 / 128.0 }; - var x_f8e3: [x.len]floats.Float8E3M4 = undefined; - for (&x_f8e3, &x) |*xi_f8e3, xi| xi_f8e3.* = .fromF32(xi); - - const x_d: Tensor = .init(.{x.len}, .f32); - const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.convert, .{ x_d, .f8e3m4 }, platform); - defer exe.deinit(); - - const x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), std.mem.sliceAsBytes(&x)); - defer x_d_buffer.deinit(); - - const x_f8e3_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); - const x_f8e3_xla = try x_f8e3_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); - defer x_f8e3_xla.free(std.testing.allocator); - - errdefer std.log.warn("convert(.f8e3m4) failed !\ninput f32:\n{e}\nzml.floats computed:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x), x_f8e3, x_f8e3_xla }); - try std.testing.expectEqualDeep(&x_f8e3, x_f8e3_xla.items(floats.Float8E3M4)); - } - } - - /// Returns a Tensor containing the element-wise rounding operation of the input Tensor. - pub fn round(self: Tensor) Tensor { - const round_op = dialects.stablehlo.round_nearest_even(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape, round_op.result(0)); - } - - /// Returns a Tensor containing the element-wise clamping operation of the input Tensor. - pub fn clamp(self: Tensor, min_: Tensor, max_: Tensor) Tensor { - const op = dialects.stablehlo.clamp(mlirCtx(), min_.value(), self.value(), max_.value(), .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape, op.result(0)); - } - - /// Matrix multiplication, where contracting axes are specified using their tags. - /// eg dot(.{ .a, .b, .c }, .{ .a, .c, .d }, .c) -> .{ .a, .b, .d } - /// Axes with the same tag on both sides, and which aren't contracting, - /// are considered "batching axes". - pub fn dot(lhs: Tensor, rhs: Tensor, args: anytype) Tensor { - const lhs_contracting_dim: i8 = @intCast(lhs.shape().hasTag(args).?); - const rhs_contracting_dim: i8 = @intCast(rhs.shape().hasTag(args).?); - - var batching_axes: stdx.BoundedArray([2]i8, constants.MAX_RANK) = .{}; - for (0..lhs.rank()) |lhs_tag_index| { - const lhs_tag = lhs.shape().tag(lhs_tag_index); - if (lhs_tag == Shape.toTag(args)) continue; - if (rhs.shape().hasTag(lhs_tag)) |rhs_tag_index| { - batching_axes.appendAssumeCapacity(.{ @intCast(lhs_tag_index), @intCast(rhs_tag_index) }); - } - } - return lhs.dotGeneral(rhs, &.{.{ lhs_contracting_dim, rhs_contracting_dim }}, batching_axes.slice()); - } - - test dot { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - inline for (.{ - .{ .{ .c = 20 }, .{ .c = 20 }, .c, .{} }, - .{ - .{ .a = 20, .b = 21, .c = 22 }, - .{ .a = 20, .d = 23, .c = 22 }, - .c, - .{ .a = 20, .b = 21, .d = 23 }, - }, - .{ - .{ .a = 20, .b = 21, .c = 22 }, - .{ .c = 22, .d = 23, .e = 24 }, - .c, - .{ .a = 20, .b = 21, .d = 23, .e = 24 }, - }, - // TODO(Corentin): Re-enable that - //.{ - // .{ .a = 20, .b = 21, .c = 22 }, - // .{ .c = 22, .d = 23, .a = 20 }, - // .{ .c, .a }, - // .{ .b = 21, .d = 23 }, - //}, - }) |testcase| { - const x: Tensor = .init(testcase[0], .f32); - const y: Tensor = .init(testcase[1], .f32); - const ctr = Shape.toTag(testcase[2]); - const z_shape: Shape = .init(testcase[3], .f32); - const forward = struct { - fn forward(x_: Tensor, y_: Tensor, tag: Shape.Tag) Tensor { - return x_.dot(y_, tag); - } - }.forward; - - const exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, y, ctr }, platform); - defer exe.deinit(); - - try zml.testing.expectEqualShapes(Shape.init(z_shape, .f32), exe.output_shapes[0]); - } - } - - /// Generalized matrix multiplication of two tensors along the specified axes. - /// In this version batching dimensions need to be explicitly specified. - /// The result shape is made of (batching_axes ++ lhs_result_axes ++ rhs_result_axes. - /// Where "result axes" are non-contracting, non-batching axes of each input tensor. - pub fn dotGeneral( - lhs: Tensor, - rhs: Tensor, - contracting_axes: []const [2]i8, - batching_axes: []const [2]i8, - ) Tensor { - stdx.debug.assert(lhs.dtype() == rhs.dtype(), "dotGeneral expects tensors to be of the same type, got {} and {}", .{ lhs.dtype(), rhs.dtype() }); - - const Axes = stdx.BoundedArray(i64, constants.MAX_RANK); - - var res_shape: Shape = .{ ._dtype = lhs.dtype() }; - // Validate batching axes - var lhs_batching_axes: Axes = .{}; - var rhs_batching_axes: Axes = .{}; - for (batching_axes) |b_axes| { - const l, const r = b_axes; - stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects batching dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); - var t = lhs._shape.tag(l); - if (t == Shape.TagUnknown) t = rhs._shape.tag(r); - res_shape = res_shape.appendDim(lhs._shape.dim(l), t); - lhs_batching_axes.appendAssumeCapacity(lhs._shape.axis(l)); - rhs_batching_axes.appendAssumeCapacity(rhs._shape.axis(r)); - } - - // Validate contracting axes - var lhs_contracting_axes: Axes = .{}; - var rhs_contracting_axes: Axes = .{}; - for (contracting_axes) |c_axes| { - const l, const r = c_axes; - stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects contracting dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); - lhs_contracting_axes.appendAssumeCapacity(lhs._shape.axis(l)); - rhs_contracting_axes.appendAssumeCapacity(rhs._shape.axis(r)); - } - - // Result shape is obtained by concatenating batching dimensions, (already done) - // then dimensions from lhs axes that aren't contracting nor batching, - // then dimensions from rhs axes that aren't contracting nor batching. - for (0..lhs.rank()) |l| { - if (std.mem.indexOfScalar(i64, lhs_contracting_axes.constSlice(), @intCast(l))) |_| { - continue; - } - if (std.mem.indexOfScalar(i64, lhs_batching_axes.constSlice(), @intCast(l))) |_| { - continue; - } - res_shape = res_shape.appendDim(lhs._shape.dim(l), lhs._shape.tag(l)); - } - for (0..rhs.rank()) |r| { - if (std.mem.indexOfScalar(i64, rhs_contracting_axes.constSlice(), @intCast(r))) |_| { - continue; - } - if (std.mem.indexOfScalar(i64, rhs_batching_axes.constSlice(), @intCast(r))) |_| { - continue; - } - res_shape = res_shape.appendDim(rhs._shape.dim(r), rhs._shape.tag(r)); - } - - const op = dialects.stablehlo.dot_general( - mlirCtx(), - lhs.value(), - rhs.value(), - mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), - .{ - .lhs_batching_dimensions = lhs_batching_axes.constSlice(), - .rhs_batching_dimensions = rhs_batching_axes.constSlice(), - .lhs_contracting_dimensions = lhs_contracting_axes.constSlice(), - .rhs_contracting_dimensions = rhs_contracting_axes.constSlice(), - .dot_precision = .fast, - }, - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - return _result(res_shape, op.result(0)); - } - - /// Returns a Tensor containing the sigmoid function applied to each element of the input Tensor. - pub fn sigmoid(self: Tensor) Tensor { - const op = dialects.stablehlo.logistic(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape, op.result(0)); - } - - pub const logistic = sigmoid; - - /// Returns a Tensor containing the ReLU activation function applied to each element of the input Tensor. - pub fn relu(self: Tensor) Tensor { - return self.maximum(Tensor.constant(self.dtype().zero()).broad(.init(self.dims(), self.dtype()))); - } - - /// Returns a Tensor containing the leaky-ReLU activation function applied to each element of the input Tensor. - /// - /// LeakyReLU(x) = max(0,x) + negative_slope * min(0,x) - /// ref: https://paperswithcode.com/method/leaky-relu - pub fn leakyReLU(self: Tensor, negative_slope: f32) Tensor { - const below_zero = self.scale(negative_slope).minimum(.scalar(0, self.dtype())); - return self.relu().add(below_zero); - } - - test leakyReLU { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const input: Tensor = .init(.{2}, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.leakyReLU, .{ input, 0.1 }, platform); - defer exe.deinit(); - - const input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6884, 1.6795 })); - defer input_buffer.deinit(); - - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.leakyReLU, .{input_buffer}); - defer res.deinit(); - - const expectation: ConstSlice = .init(input.shape(), std.mem.sliceAsBytes(&[2]f32{ -0.0688, 1.6795 })); - try zml.testing.expectClose(std.testing.io, expectation, res, 1e-4); - } - - /// Returns a Tensor containing the SwiGLU activation function applied to the input Tensor. - pub fn swiglu(self: Tensor, beta: f32, w: Tensor, b: Tensor, tag: Shape.Tag) Tensor { - const sigmoid_tensor = self.mul(Tensor.constant(DataType.Value.init(self.dtype(), beta)).broad(.init(self._shape, self.dtype()))).sigmoid(); - const one_minus_sigmoid_tensor = Tensor.constant(.init(self.dtype(), 1)).broad(.init(self._shape, self.dtype())).sub(sigmoid_tensor); - - return self.mul(sigmoid_tensor).add(one_minus_sigmoid_tensor.mul(w.dot(self, tag).add(b))); - } - - /// Returns a Tensor containing the Gaussian Error Linear Units (GeLU) activation function applied to each element of the input Tensor. - /// - /// We use an approximation of the erf function using tanh: - /// gelu(x) ≃ 0.5 * x * (1 + tanh(sqrt(2 / pi) * (x + 0.044715 * x^3))) - /// see: https://paperswithcode.com/method/gelu - pub fn gelu(x: Tensor) Tensor { - const scaled_x_cube = x.mul(x).mul(x).scale(0.044715); - const beta = std.math.sqrt(2.0 / std.math.pi); - const tanh_ = x.add(scaled_x_cube).scale(beta).tanh(); - return tanh_.addConstant(1).mul(x).scale(0.5); - } - - /// Returns a Tensor containing an approximation of the Gaussian Error Linear Units (GeLU) activation function applied to each element of the input Tensor. - /// - /// It's an even more crude approximation than gelu. - pub fn quickGelu(x: Tensor) Tensor { - return x.scale(1.702).sigmoid().mul(x); - } - - /// Returns a Tensor containing the Sigmoid Linear Unit (SiLU) activation function applied to each element of the input Tensor. - /// - /// silu(x) = x σ(x) - pub fn silu(x: Tensor) Tensor { - return x.mul(.sigmoid(x)); - } - - /// Computes softmax along the given axis. - /// y[i] = exp(x[i]) / ( Σ_k exp(x[k]) + bias ) - pub fn softmax(self: Tensor, axis_: anytype) Tensor { - const a = self.axis(axis_); - const max_val = self.max(a); - const row_mask = max_val.cmp(.GT, .scalar(-std.math.inf(f64), self.dtype())); - - const exp_diff_max = self.sub(max_val).convert(.f32).exp(); - const res = exp_diff_max.div(exp_diff_max.sum(a)).convert(self.dtype()); - - // If a row is full -inf return full 0 instead of full nan, - // this fix attention when mask hides a full row. - return row_mask.broad(self.shape()).select(res, .scalar(0, self.dtype())); - } - - /// Computes softmax, but adds a bias to the sum of exponentiel. - /// y[i] = exp(x[i]) / ( Σ_k exp(x[k]) + bias ) - pub fn softmaxBiased(self: Tensor, axis_: anytype, bias: ?Tensor) Tensor { - const a = self.axis(axis_); - - if (bias == null) return self.softmax(axis_); - const b = bias.?.convert(self.dtype()).broad(self.shape().setDim(a, 1)); - const max_val: Tensor = maximum(self.max(a), b); - const exp_diff_max = self.sub(max_val).exp(); - const bias_diff_max = b.sub(max_val).exp(); - const res = exp_diff_max.div(exp_diff_max.sum(a).add(bias_diff_max)); - - // The bias means that denominator won't be 0, we don't need to handle that case. - return res; - } - - /// Returns a Tensor containing the log of the sum of exponential over the given axis. - pub fn logSumExp(self: Tensor, axis_: anytype) Tensor { - const a = self.axis(axis_); - // stabilization: shift `self` by it's max value before passing to exponent. - const max_ = self.max(a); - const log_sum_exp = log(sum(exp(self.sub(max_.broad(self._shape))), a)); - // restore the shift again - return max_.add(log_sum_exp); - } - - /// Returns a Tensor containing the sum of elements over the given axis. - /// Output shape is the input shape with the axis_ dim set to 1. - pub fn sum(self: Tensor, axis_: anytype) Tensor { - const a = self.axis(axis_); - return ops.reduce(.{self}, .{Tensor.constant(self.dtype().zero())}, &.{a}, struct { - pub fn acc(args: ops.ReduceArgs) struct { Tensor } { - return .{args.right.add(args.left.convert(args.right.dtype()))}; - } - }.acc, .{})[0]; - } - - /// Returns a Tensor containing the mean of elements over the given axis. - /// Output shape is the input shape with the axis_ dim set to 1. - pub fn mean(self: Tensor, axis_: anytype) Tensor { - return self.sum(axis_).divByConst(self.dim(axis_)); - } - - /// Returns a Tensor containing the cumulative sum of elements over the given axis. - /// Output shape is the same as input shape. - /// [0, 1, 0, 1, 0, 0, 1, 1].cumulativeSum(0) -> [0, 1, 1, 2, 2, 2, 3, 4] - /// The last value contains the sum of all element in the array. - pub fn cumulativeSum(self: Tensor, axis_: anytype) Tensor { - const rk = self.rank(); - const a = self.axis(axis_); - - const ones = [_]i64{1} ** constants.MAX_RANK; - var window_dimensions = ones; - window_dimensions[a] = self.dim(a); - var padding = [_][2]i64{.{ 0, 0 }} ** constants.MAX_RANK; - padding[a] = .{ self.dim(a) - 1, 0 }; - - const result = ops.reduceWindow( - .{self}, - .{Tensor.scalar(0, self.dtype())}, - .{ - .base_dilations = ones[0..rk], - .window_dilations = ones[0..rk], - .window_strides = ones[0..rk], - .window_dimensions = window_dimensions[0..rk], - .padding = padding[0..rk], - }, - struct { - fn add(values: ops.ReduceArgs) struct { Tensor } { - return .{values.left.add(values.right)}; - } - }.add, - .{}, - ); - - return result[0]; - } - - test cumulativeSum { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const Local = struct { - pub fn _cumsum(input: Tensor) Tensor { - const x = input.withPartialTags(.{.n}); - const y = x.cumulativeSum(.n); - // Check that tags are propagated - std.debug.assert(y.shape().eqlWithTags(x.shape())); - return y; - } - }; - - const x: Tensor = .init(.{ 2, 5 }, .f32); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cumsum, .{x}, platform); - defer exe.deinit(); - - const x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } })); - defer x_buffer.deinit(); - - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._cumsum, .{x_buffer}); - defer res.deinit(); - - try std.testing.expectEqual([2][5]f32{ .{ 0, 1, 2, 2, 3 }, .{ 3, 4, 4, 6, 7 } }, try res.getValue([2][5]f32, std.testing.io)); - } - - /// Returns a transposed Tensor computed using the given axes. - pub fn transpose(self: Tensor, axes_: anytype) Tensor { - const axes__ = self.axes(axes_).constSlice(); - const no_op = constants.AXES_IOTA; - const default_perm = b: { - var buf = constants.AXES_IOTA; - std.mem.reverse(i64, &buf); - break :b buf; - }; - - const permutation: []const i64 = if (axes__.len == 0) - default_perm[constants.MAX_RANK - self.rank() ..] - else - toI64(axes__).constSlice(); - - stdx.debug.assert(permutation.len == self.rank(), "transpose expects input tensor rank and 'axes_' length to be equal, got {f} and {any}", .{ self, permutation[0..@min(permutation.len, constants.MAX_RANK + 2)] }); - - if (std.mem.eql(i64, permutation, no_op[0..self.rank()])) { - return self; - } - - const res_shape = self._shape.transpose(permutation); - if (transposeIsJustAReshape(self.shape(), permutation)) { - return self.reshape(res_shape); - } - - const op = dialects.stablehlo.transpose( - mlirCtx(), - self.value(), - mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), - .{ .permutation = toI64(permutation).constSlice() }, - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - return _result(res_shape, op.result(0)); - } - - pub fn swapAxes(self: Tensor, a: anytype, b: anytype) Tensor { - if (self.axis(a) == self.axis(b)) return self; - var perm: Shape.AxesArray = .{}; - for (0..self.rank()) |i| { - perm.appendAssumeCapacity(@intCast(i)); - } - perm.set(self.axis(a), self.axis(b)); - perm.set(self.axis(b), self.axis(a)); - return self.transpose(perm.constSlice()); - } - - /// Returns a Tensor with the given axis unflattened. - /// - /// unflatten((d0, d1, axis_m, d3), 2, n) -> (d0, d1, n, d2_m, d3) - pub fn unflatten(self: Tensor, axis_: i8, n: i64) Tensor { - // TODO: move to torch.zig, this equivalent to `spitAxis` - stdx.debug.assert(self.rank() < constants.MAX_RANK, "unflatten expects input tensor rank to be less than {}, got {}", .{ constants.MAX_RANK, self.rank() }); - - const a = if (axis_ >= 0) self.axis(axis_) else self.axis(axis_) + 1; - const new_dim = std.math.divExact(i64, self.dim(a), n) catch std.debug.panic("unflatten expects chosen dimension to be divisible by 'n' but {} is not divisible by {}", .{ self.dim(a), n }); - const new_shape = self._shape.set(a, n).insert(a + 1, .{ ._ = new_dim }); - - const reshaped_val = dialects.stablehlo.reshape( - mlirCtx(), - self.value(), - mlir.rankedTensorType(new_shape.dims(), mlirx.Type.fromDType(mlirCtx(), new_shape.dtype())), - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - return _result(new_shape, reshaped_val.result(0)); - } - - /// Splits the given axis in several axes. - /// eg: `Tensor.init(.{ .a = 10, .b = 3 }).split(.a, .{.a1 = 5, .a2 = 2});` - /// The number of elements in the split shape must match the number of element - /// in the target axis. - pub fn splitAxis(self: Tensor, ax: anytype, split_shape: anytype) Tensor { - const new_shape = self._shape.splitAxis(ax, split_shape); - - const reshaped_val = dialects.stablehlo.reshape( - mlirCtx(), - self.value(), - mlir.rankedTensorType(new_shape.dims(), mlirx.Type.fromDType(mlirCtx(), new_shape.dtype())), - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - return _result(new_shape, reshaped_val.result(0)); - } - - /// Merges two or more contiguous axes into one axis. - pub fn merge(self: Tensor, merges_: anytype) Tensor { - return self.reshape(self._shape.mergeAxes(merges_)); - } - - /// Merges two or more non-contiguous axes into one axis. - /// Will make a transpose if needed. - /// .{ .a, .b, .c }.mergeTranspose(.{ .a, .c }, .ac) -> .{ .b, .ac } - pub fn mergeTranspose(self: Tensor, axes_: anytype, merged: @EnumLiteral()) Tensor { - const cont = self.contiguous(axes_); - return cont.reshape(cont._shape.mergeAxis(merged, axes_)); - } - - /// Transposes the input Tensor, such has the given axes end up in contiguous position. - /// .{ .a, .b, .c, .d }.contiguous(.{ .c, .a }) -> .{ .b, .d, .c, .a } - pub fn contiguous(self: Tensor, axes_: anytype) Tensor { - const perm = self._shape.contiguousPerm(axes_); - return self.transpose(perm.constSlice()); - } - - pub fn flatten(self: Tensor) Tensor { - return self.reshape(.{self.count()}); - } - - pub const Slice = struct { - start: i64 = 0, - end: i64 = to_the_end, - step: i32 = 1, - singleton: bool = false, - - const full = .{ .start = 0, .end = to_the_end, .step = 1 }; - - pub fn single(offset: i64) Slice { - return .{ .start = offset, .end = offset + 1, .singleton = true }; - } - - pub fn absolute(s: Slice, d: i64) Slice { - const start = if (s.start < 0) d + s.start else s.start; - const end = if (s.end == to_the_end) d else if (s.end < 0) d + s.end else s.end; - const res: Slice = .{ .start = start, .end = end, .step = s.step, .singleton = s.singleton }; - stdx.debug.assert(start < end, "Slice {f} is invalid for axis of dimension {d} (resolved to {f}", .{ s, d, res }); - return res; - } - - const to_the_end = std.math.maxInt(i64); - - pub fn format(self: Slice, writer: *std.Io.Writer) !void { - if (self.singleton) { - try writer.print("[{d}]", .{self.start}); - } else if (self.end == to_the_end and self.step == 1) { - try writer.print("[{d}..]", .{self.start}); - } else if (self.step == 1) { - try writer.print("[{d}..{d}]", .{ self.start, self.end }); - } else { - try writer.print("[{d}..{d}:{d}]", .{ self.start, self.end, self.step }); - } - } - }; - - /// Slices the input Tensor over the given axis using the given parameters. - pub fn slice1d(self: Tensor, axis_: anytype, s: Slice) Tensor { - var slices = [_]Slice{.{}} ** constants.MAX_RANK; - slices[self.axis(axis_)] = s; - return self.slice(slices[0..self.rank()]); - } - - /// Slices the input Tensor using the given parameters. - pub fn slice(self: Tensor, slices: []const Slice) Tensor { - var start_indices: [constants.MAX_RANK]i64 = undefined; - var strides: [constants.MAX_RANK]i64 = undefined; - var limit_indices: [constants.MAX_RANK]i64 = undefined; - var res_shape: Shape = self._shape; - - for (slices, 0..) |s, a| { - stdx.debug.assert(s.step > 0, "slice expects 'step' to be positive, got {} at index {}", .{ s.step, a }); - - const args: Slice = s.absolute(self.dim(a)); - start_indices[a] = args.start; - limit_indices[a] = args.end; - strides[a] = args.step; - res_shape = res_shape.setDim(a, std.math.divCeil(i64, args.end - args.start, args.step) catch unreachable); - } - - const result_type = mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())); - const slice_op = dialects.stablehlo.slice( - mlirCtx(), - self.value(), - start_indices[0..self.rank()], - limit_indices[0..self.rank()], - strides[0..self.rank()], - result_type, - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - - var res = _result(res_shape, slice_op.result(0)); - var to_remove: Shape.AxesArray = .{}; - for (slices, 0..) |s, a| { - if (s.singleton) to_remove.appendAssumeCapacity(@intCast(a)); - } - return res.reshape(res_shape.removeMany(to_remove.constSlice())); - } - - test slice { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const x: Tensor = .init(.{ 2, 5 }, .f32); - - const x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); - defer x_buffer.deinit(); - - // Wrap slice1d to hide the anytype in the signature. - const Local = struct { - pub fn _slice1dAxis(input: Tensor, ax: i8, slice_: Tensor.Slice) Tensor { - return input.slice1d(ax, slice_); - } - }; - - inline for (.{ - .{ 0, Tensor.Slice{ .end = 1 }, [5]f32{ 0, 1, 2, 3, 4 } }, - .{ 1, Tensor.Slice{ .start = 1, .step = 2 }, [4]f32{ 1, 3, 6, 8 } }, - .{ -1, Tensor.Slice{ .start = -2 }, [4]f32{ 3, 4, 8, 9 } }, - }) |testcase| { - const ax, const slice_, const expectation = testcase; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._slice1dAxis, .{ x, ax, slice_ }, platform); - defer exe.deinit(); - - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._slice1dAxis, .{x_buffer}); - defer res.deinit(); - - try std.testing.expectEqual(expectation, try res.getValue(@TypeOf(expectation), std.testing.io)); - } - } - - inline fn wrapIndex(self: Tensor, axis_: usize, idx: i64) i64 { - return if (idx < 0) self.dim(axis_) + idx else idx; - } - - pub fn choose1d(self: Tensor, axis_: anytype, i: i64) Tensor { - return self.slice1d(axis_, .single(i)); - } - - pub fn choose(self: Tensor, offsets: anytype) Tensor { - const off, const tags = Shape.parseDimensions(offsets); - var slices = [_]Slice{.{}} ** constants.MAX_RANK; - for (off.constSlice(), tags.constSlice()) |o, t| { - const ax = self.axis(t); - slices[ax] = .single(o); - } - return self.slice(slices[0..self.rank()]); - } - - /// Concatenates the input Tensors along the given axis. - pub fn concatenate(tensors: []const Tensor, axis_: anytype) Tensor { - if (tensors.len == 1) return tensors[0]; - stdx.debug.assert(tensors.len <= 32, "concatenate only supports up to 32 tensors, got {}", .{tensors.len}); - var buffer: [32]*const mlir.Value = undefined; - std.debug.assert(tensors.len <= buffer.len); - std.debug.assert(tensors.len > 0); - const a = tensors[0].axis(axis_); - // TODO(Corendos): Check that tensor axes match. - - var concatenated_dim: i64 = 0; - for (tensors, 0..) |t, i| { - buffer[i] = t.value(); - concatenated_dim += t.dim(a); - } - - const res_shape = tensors[0]._shape.set(a, concatenated_dim); - const op = dialects.stablehlo.concatenate(mlirCtx(), buffer[0..tensors.len], a, .unknown(mlirCtx())).appendTo(currentBlock()); - // log.debug("concatenate({}, {}, {d}) -> {d}", .{ tensors[0], tensors[1], a, res_shape }); - return _result(res_shape, op.result(0)); - } - - /// Concatenates the input Tensors along a new axis. The Tensors must have the same shape. - /// For x, y, z of shape .{ .a = 10, .b = 11, .c = 12 }: - /// - Tensor.stack(&.{x, y, z}, .b, .layers) -> .{ .a, .layers, .b, .c } - /// - Tensor.stack(&.{x, y, z}, 1, .layers) -> .{ .a, .layers, .b, .c } - /// - Tensor.stack(&.{x, y, z}, .last, .layers) -> .{ .a, .b, .c, .layers } - pub fn stack(tensors: []const Tensor, axis_: anytype, tag: anytype) Tensor { - // Note: we could ask the compilation context for some memory instead of stack allocating - stdx.debug.assert(tensors.len <= 32, "stack only supports up to 32 tensors, got {}", .{tensors.len}); - - const shape0 = tensors[0]._shape; - const res_shape = shape0.insertTag(axis_, 1, tag); - - for (tensors[1..]) |tensor| { - stdx.debug.assert(shape0.eqlWithTags(tensor._shape), "stack expects tensor shapes to match, got {f} and {f}", .{ shape0, tensor._shape }); - } - - var reshaped: [32]Tensor = undefined; - for (tensors, 0..) |tensor, i| { - reshaped[i] = tensor.reshape(res_shape); - } - - // Be careful here: we need to resolve ax before calling concatenate, - // because we added an axis, so all - const ax = if (@TypeOf(axis_) == @EnumLiteral() and axis_ == .last) - shape0.rank() - else - shape0.axis(axis_); - - return Tensor.concatenate(reshaped[0..tensors.len], ax); - } - - /// Repeats a Tensor several times along the given axis. - /// - /// * repeat1d(x, axis, 4) = concat(&.{x, x, x, x}, axis); - /// * repeat1d([0, 1, 2, 3], 0, 2) = [0, 1, 2, 3, 0, 1, 2, 3] - pub fn repeat1d(self: Tensor, axis_: anytype, n_rep: u63) Tensor { - if (n_rep == 1) { - return self; - } - - const a = self.axis(axis_); - const res_shape = self._shape.setDim(a, self.dim(a) * n_rep); - - const broadshape = self._shape.insert(a, .{n_rep}); - const repeat_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a); - - return self.broadcast(broadshape, repeat_dims.dims()).reshape(res_shape); - } - - /// Repeats a Tensor several times along the given axes. - pub fn repeat(self: Tensor, n_reps: []const u63) Tensor { - // TODO: this should support the tagged syntax: x.repeat(.{ .a = 3, .b = 2}); - stdx.debug.assert(n_reps.len == self.rank(), "repeat expects tensor rank and 'n_reps' length to be equal, got {} and {}", .{ self.rank(), n_reps.len }); - - var res = self; - for (n_reps, 0..) |n_rep, a| { - if (n_rep == 1) continue; - - res = res.repeat1d(a, n_rep); - } - return res; - } - - test repeat1d { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const Local = struct { - fn repeat1d(x: Tensor, axis_: u3, n_reps: u32) Tensor { - return x.repeat1d(axis_, n_reps); - } - }; - - inline for (.{ - .{ [3]u8{ 1, 2, 3 }, Shape.init(.{3}, .u8), [6]u8{ 1, 2, 3, 1, 2, 3 }, 0, 2 }, - .{ [2][3]u8{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }, Shape.init(.{ 2, 3 }, .u8), [2][6]u8{ .{ 1, 2, 3, 1, 2, 3 }, .{ 4, 5, 6, 4, 5, 6 } }, 1, 2 }, - }) |testcase| { - const input_data, const shape_, const expectation, const ax, const reps = testcase; - const input: Tensor = .fromShape(shape_); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.repeat1d, .{ input, ax, reps }, platform); - defer exe.deinit(); - - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&input_data)); - defer input_buffer.deinit(); - - const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local.repeat1d, .{input_buffer}); - try std.testing.expectEqual(expectation, try output.getValue(@TypeOf(expectation), std.testing.io)); - } - } - - /// Repeats in line each value along the given axis. - /// - /// * stutter1d([0, 1, 2, 3], -1, 2) = [0, 0, 1, 1, 2, 2, 3, 3] - /// This is equivalent to repeat(ax+1) unless ax is the last axis. - pub fn stutter1d(self: Tensor, axis_: i8, n_rep: u63) Tensor { - const a = self.axis(axis_); - const broadshape = self._shape.insert(a + 1, .{n_rep}); - const res_shape = self._shape.setDim(a, self.dim(a) * n_rep); - - const stutter_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a + 1); - return self.broadcast(broadshape, stutter_dims.dims()).reshape(res_shape); - } - - /// Repeats in line each value along the given axes. - pub fn stutter(self: Tensor, n_reps: []const u63) Tensor { - // TODO: this should support the tagged syntax: x.repeat(.{ .a = 3, .b = 2}); - stdx.debug.assert(n_reps.len == self.rank(), "stutter expects tensor rank and 'n_reps' length to be equal, got {} and {}", .{ self.rank(), n_reps.len }); - - var res = self; - for (n_reps, 0..) |n_rep, a| { - if (n_rep == 1) continue; - res = res.stutter1d(@intCast(a), n_rep); - } - return res; - } - - test stutter1d { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const Local = struct { - fn stutter1d(x: Tensor, axis_: u3, n_reps: u32) Tensor { - return x.stutter1d(axis_, n_reps); - } - }; - - inline for (.{ - .{ [3]u8{ 1, 2, 3 }, Shape.init(.{3}, .u8), [6]u8{ 1, 1, 2, 2, 3, 3 }, 0, 2 }, - .{ [2][3]u8{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }, Shape.init(.{ 2, 3 }, .u8), [2][6]u8{ .{ 1, 1, 2, 2, 3, 3 }, .{ 4, 4, 5, 5, 6, 6 } }, 1, 2 }, - .{ [2][3]u8{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }, Shape.init(.{ 2, 3 }, .u8), [2][6]u8{ .{ 1, 2, 3, 1, 2, 3 }, .{ 4, 5, 6, 4, 5, 6 } }, 0, 2 }, - }) |testcase| { - const input_data, const shape_, const expectation, const ax, const reps = testcase; - const input: Tensor = .fromShape(shape_); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.stutter1d, .{ input, ax, reps }, platform); - defer exe.deinit(); - - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&input_data)); - defer input_buffer.deinit(); - - const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local.stutter1d, .{input_buffer}); - try std.testing.expectEqual(expectation, try output.getValue(@TypeOf(expectation), std.testing.io)); - } - } - - /// Returns a Tensor containing the element-wise negation of the input Tensor. - pub fn negate(self: Tensor) Tensor { - const negate_op = dialects.stablehlo.negate(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape, negate_op.result(0)); - } - - /// Returns a Tensor containing the element-wise cosine of the input Tensor. - pub fn cos(self: Tensor) Tensor { - const cosine_op = dialects.stablehlo.cosine(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape, cosine_op.result(0)); - } - - /// Returns a Tensor containing the element-wise sine of the input Tensor. - pub fn sin(self: Tensor) Tensor { - const sine_op = dialects.stablehlo.sine(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape, sine_op.result(0)); - } - - /// Returns a Tensor containing the element-wise exponential operation of the input Tensor. - pub fn exp(self: Tensor) Tensor { - const op = dialects.stablehlo.exponential(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape, op.result(0)); - } - - /// Returns a Tensor containing the element-wise logarithm operation of the input Tensor. - pub fn log(self: Tensor) Tensor { - const op = dialects.stablehlo.log(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape, op.result(0)); - } - - /// Returns a Tensor containing the element-wise square-root of the input Tensor. - pub fn sqrt(self: Tensor) Tensor { - const sqrt_op = dialects.stablehlo.sqrt(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape, sqrt_op.result(0)); - } - - /// Returns a Tensor containing the element-wise reverse square-root of the input Tensor. - pub fn rsqrt(self: Tensor) Tensor { - const rsqrt_op = dialects.stablehlo.rsqrt(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape, rsqrt_op.result(0)); - } - - /// Returns a Tensor containing the element-wise hyperbolic tangent of the input Tensor. - pub fn tanh(self: Tensor) Tensor { - const tanh_op = dialects.stablehlo.tanh(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape, tanh_op.result(0)); - } - - /// Returns a Tensor containing the element-wise exponential minus one operation of the input Tensor. - pub fn exponentialMinusOne(self: Tensor) Tensor { - const expm1_op = dialects.stablehlo.exponential_minus_one(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape, expm1_op.result(0)); - } - - pub const ArangeArgs = struct { - start: i64 = 0, - end: i64, - step: i64 = 1, - }; - /// - /// Returns a Tensor containing evenly spaced values within a given interval. - pub fn arange(args: ArangeArgs, dt: DataType) Tensor { - stdx.debug.assert(args.start <= args.end, "arange expects 'args.start' to be less than 'args.end', got {} and {}", .{ args.start, args.end }); - stdx.debug.assert(args.step > 0, "arange expects 'args.step' to be positive, got {}", .{args.step}); - - const n_steps = std.math.divCeil(i64, args.end - args.start, args.step) catch unreachable; - const sh = Shape.init(.{n_steps}, dt); - var op = dialects.stablehlo.iota( - mlirCtx(), - 0, - mlir.rankedTensorType(sh.dims(), mlirx.Type.fromDType(mlirCtx(), sh.dtype())), - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - var res = _result(sh, op.result(0)); - - if (args.step != 1) { - res = res.scale(args.step); - } - - if (args.start != 0) { - res = res.addConstant(args.start); - } - - return res; - } - - /// Returns a Tensor containing values in increasing order starting from 0 along the given axis. - /// - /// The output dtype will be `.i32`, unless the given axis has a too big dimension, in that case we use `.i64`. - /// In most program this shouldn't matter, because typically this will be used in a comparison, - /// or explicitly converted by the user to do floating point arithmetic. - pub fn iota(sh: Shape, axis_: anytype) Tensor { - const a = sh.axis(axis_); - const dt: DataType = if (sh.dim(a) <= std.math.maxInt(i32)) .i32 else .i64; - const res_shape = sh.withDtype(dt); - - var op = dialects.stablehlo.iota( - mlirCtx(), - a, - mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - return _result(res_shape, op.result(0)); - } - - pub const LinspaceArgs = struct { - start: f64, - end: f64, - steps: i64, - }; - - /// Returns a Tensor containing 'args.steps' values evenly spaced from 'args.start' to 'args.end', inclusive. - pub fn linspace(args: LinspaceArgs, dt: DataType) Tensor { - stdx.debug.assert(args.start < args.end, "linspace expects 'args.start' to be less than 'args.end', got {} and {}", .{ args.start, args.end }); - stdx.debug.assert(args.steps > 0, "linspace expects 'args.steps' to be positive, got {}", .{args.steps}); - stdx.debug.assert(dt.isFloat(), "linspace expects type to be a float, got {} (hint: use arange instead)", .{dt}); - - const sh = Shape.init(.{args.steps}, dt); - var iota_op = dialects.stablehlo.iota( - mlirCtx(), - 0, - mlir.rankedTensorType(sh.dims(), mlirx.Type.fromDType(mlirCtx(), sh.dtype())), - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - var res = _result(sh, iota_op.result(0)); - - if (args.steps != 1) { - res = res.scale(args.steps); - } - - if (args.start != 0) { - res = res.addConstant(args.start); - } - - return res; - } - - /// Returns a 0-rank Tensor with the given value. - pub fn scalar(val: anytype, dt: DataType) Tensor { - const data = DataType.Value.init(dt, val); - switch (dt.class()) { - .float => stdx.debug.assert(!std.math.isNan(val), "scalar(NaN) is probably due to compiling a model with an uninitialized field", .{}), - else => {}, - } - return Tensor.constant(data); - } - - test scalar { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const Local = struct { - pub fn _fwd() [6]Tensor { - var res: [6]Tensor = undefined; - const dtypes = .{ .bool, .u8, .i32, .f32, .bf16, .u64 }; - inline for (0..6) |i| res[i] = scalar(0, dtypes[i]); - return res; - } - }; - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{}, platform); - defer exe.deinit(); - } - - /// Returns a constant Tensor with the given value. - pub fn constant(val: DataType.Value) Tensor { - const op = dialects.stablehlo.constant( - mlirCtx(), - &.{}, - mlirx.Type.fromDType(mlirCtx(), val.dtype()), - val.asBytes(), - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - return _result(.init(&.{}, val.dtype()), op.result(0)); - } - - pub fn zeroes(sh: Shape) Tensor { - return Tensor.constant(sh.dtype().zero()).broad(sh); - } - - /// Embeds a buffer with concrete values into an Mlir program. - pub fn constantTensor(sh: Shape, bytes_: []const u8) Tensor { - const elem_type = mlirx.Type.fromDType(mlirCtx(), sh.dtype()); - //const elem_type = mlirx.denseElementAttrType(val.dtype()) orelse std.debug.panic("constantTensor expects a dtype that can be serialized to MLIR, like f32 or i32, got {f}", .{val.shape()}); - const constant_op = dialects.stablehlo.constant(mlirCtx(), sh.dims(), elem_type, bytes_, .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(sh, constant_op.result(0)); - } - - /// Returns a Tensor containing the result of the outer product between the input Tensors. - pub fn outer(self: Tensor, other: Tensor) Tensor { - if (self.rank() + other.rank() == 1) { - return self.mul(other); - } - - const res_shape = self.shape().outer(other.shape()); - return self.broad(res_shape).mul(other.broad(res_shape)); - } - - /// Given a tensor and a shape of the same rank, - /// will "broadcast" the given axes, so that `self` has the given shape. - /// This happens by virtually repeating the data several time along each give axes. - /// Note: most of the time the optimizer will make it so that the broadcast doesn't trigger a copy. - /// Note: the tags of the return tensor will be from the `output_shape`. - /// This means if you use and un-tagged broadcast on a tagged tensor, - /// you will lose the tags. - /// To avoid use favorise `.broad(shape)` when working with tagged tensors. - pub fn broadcast(self: Tensor, output_shape: Shape, axes_: []const i64) Tensor { - stdx.debug.assert(axes_.len == self.rank(), "broadcast expects axes_ to map all axes from self to axes of the output shape, got broadcast({f}, {f}, {any})", .{ self, output_shape, axes_ }); - for (0.., axes_) |self_ax, other_ax| { - const d = self.dim(self_ax); - stdx.debug.assert(d == 1 or d == output_shape.dim(other_ax), "broadcast expects shape axes to either be 1-sized or to match the target size. got broadcast({f}, {f}, {any}), error on self axis {d} mapping to other axis {d}", .{ self, output_shape, axes_, self_ax, other_ax }); - } - - const res_shape = output_shape.withDtype(self.dtype()); - if (std.mem.eql(i64, self.dims(), output_shape.dims())) { - // No broadcast needed. We don't emit a new stablehlo value - // but we propagate output_shape tags. - return _result(res_shape, self.value()); - } - const result_type = mlir.rankedTensorType( - res_shape.dims(), - mlirx.Type.fromDType(mlirCtx(), res_shape.dtype()), - ); - const broadcast_op = dialects.stablehlo.broadcast_in_dim(mlirCtx(), self.value(), axes_, result_type, .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(res_shape, broadcast_op.result(0)); - } - - /// Broadcasts a Tensor to the given shape, adding axes at the beginning. - pub fn broadcastLeft(self: Tensor, output_shape: Shape) Tensor { - stdx.debug.assert(self.rank() <= output_shape.rank(), "broadcastLeft expects tensor rank to be less than output tensor rank, got {d} and {d}", .{ self.rank(), output_shape.rank() }); - - const a = output_shape.rank() - self.rank(); - if (self.rank() == output_shape.rank() and std.mem.eql(i64, self.dims(), output_shape.dims())) { - return self; - } - - return self.broadcast(output_shape, Shape.range(output_shape.rank(), output_shape.dtype()).dims()[a..]); - } - - /// Broadcasts a Tensor to the given shape, adding axes at the end. - pub fn broadcastRight(self: Tensor, output_shape: Shape) Tensor { - stdx.debug.assert(self.rank() <= output_shape.rank(), "broadcastRight expects tensor rank to be less than output tensor rank, got {d} and {d}", .{ self.rank(), output_shape.rank() }); - - if (self.rank() == output_shape.rank() and self._shape.eql(output_shape)) { - return self; - } - - return self.broadcast(output_shape, Shape.range(self.rank(), output_shape.dtype()).dims()); - } - - /// Broadcasts a Tensor to the given shape, extending dimensions if needed. - pub fn broad(self: Tensor, other: Shape) Tensor { - // TODO: broad is too restrictive because sometime you only want to specify one specific axis - // Note: if you code below, make sure to update Shape.canBroadcastTo. - stdx.debug.assert(self._shape.canBroadcastTo(other), "Can't broadcast {f} to {f}", .{ self, other }); - - // Already the right shape - if (std.mem.eql(i64, self.dims(), other.dims())) return self; - - // Non ambiguous broadcasting - // TODO: broad is error prone because of this: - // it will happily broadcast .{ .a = 10, .b = 1 } to .{ .b = 10, .a = 5 } - if (self._shape.rank() == 0 or self._shape.rank() == other.rank()) { - return self.broadcast(other, constants.AXES_IOTA[0..self.rank()]); - } - - // check that each axis of self maps to an axis of other - var axes_: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; - for (self._shape.tags()) |t| { - axes_.appendAssumeCapacity(@intCast(other.axis(t))); - } - return self.broadcast(other, axes_.constSlice()); - } - - /// Reshapes the input Tensor with the given shape. - pub fn reshape(self: Tensor, output_shape_: anytype) Tensor { - const output_shape = self._shape.reshape(output_shape_); - const tensor_type = mlir.rankedTensorType(output_shape.dims(), mlirx.Type.fromDType(mlirCtx(), output_shape.dtype())); - const reshape_value = dialects.stablehlo.reshape(mlirCtx(), self.value(), tensor_type, .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(output_shape, reshape_value.result(0)); - } - - /// Converts the given 1 element Tensor into a 0-rank Tensor. - pub fn asScalar(self: Tensor) Tensor { - stdx.debug.assert(self.count() == 1, "Tensor.asScalar expects an input with exactly 1-element got {f}", .{self}); - return if (self.rank() == 0) self else self.reshape(.{}); - } - - pub const Pad = struct { - low: i64 = 0, - high: i64 = 0, - interior: i64 = 0, - }; - - /// Pads the input Tensor with the given values. - /// Usage: x.pad(0, .{ .a = .{ .low = 1, .high = 1 }}); - pub fn pad(self: Tensor, padding_value: anytype, paddings: anytype) Tensor { - const _paddings = self.shape().parseAxesOptions(Pad, paddings, .{}); - - const ZEROS = [_]i64{0} ** constants.MAX_RANK; - var low = ZEROS; - var high = ZEROS; - var interior = ZEROS; - - var res_shape = self._shape; - for (_paddings.constSlice(), 0..) |padding, i| { - low[i] = padding.low; - high[i] = padding.high; - interior[i] = padding.interior; - - var d: i64 = self.dim(i); - d += low[i] + (@max(d - 1, 0) * interior[i]) + high[i]; - res_shape._dims.set(i, d); - } - - const rk = self.rank(); - const pad_op = dialects.stablehlo.pad( - mlirCtx(), - self.value(), - Tensor.scalar(padding_value, self.dtype()).value(), - .{ .low = low[0..rk], .high = high[0..rk], .interior = interior[0..rk] }, - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - - return _result(res_shape, pad_op.result(0)); - } - - /// Inserts 1-dim axes at the given position, with the given tags. - /// `.{.a = 5, .b = 4}.insert(.b, .{ .c, .d }) -> .{ .a = 5, .c = 1, .d = 1, .b = 4 }` - pub fn insertAxes(self: Tensor, axis_: anytype, tags: anytype) Tensor { - const tags_ = Shape.parseTags(tags); - const ax = if (@TypeOf(axis_) == @EnumLiteral() and axis_ == .last) - self.rank() - else - self.axis(axis_); - - var res_shape = self._shape; - const ones = [_]i64{1} ** constants.MAX_RANK; - res_shape._dims.insertSlice(ax, ones[0..tags_.len]) catch unreachable; - res_shape._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; - - return self.reshape(res_shape); - } - - /// Appends a 1-dim axis, with the given tag. - pub fn appendAxes(self: Tensor, t: anytype) Tensor { - // stdx.debug.assert(self.rank() < Tensor.MAX_RANK - t.len, "appendAxis expects tensor rank to be small enough in order to extend it, got {} and {} (max is {})", .{ self.rank(), t.len, Tensor.MAX_RANK }); - - return self.insertAxes(.last, t); - } - - /// Drops a 1-dim axis at the given index - pub fn squeeze(self: Tensor, axis_: anytype) Tensor { - const a = self.axis(axis_); - stdx.debug.assert(self.dim(a) == 1, "squeeze expects axis to be squeezed to have a dimension of 1, got {}", .{self.dim(a)}); - - const new_shape = self._shape.remove(a); - // log.debug("squeeze({}, {d}={d}) -> ({})", .{ self, axis, a, new_shape }); - - return _result(new_shape, self.reshape(new_shape).value()); - } - - /// Returns a Tensor with the given axes reversed. - pub fn reverse(self: Tensor, axes_: anytype) Tensor { - const actual_axes = self._shape.axes(axes_); - - const reverse_op = dialects.stablehlo.reverse(mlirCtx(), self.value(), toI64(actual_axes.constSlice()).constSlice(), .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape, reverse_op.result(0)); - } - - pub const GatherOpts = ops.GatherOpts; - - /// `gather` extracts slices from the given tensor at the specified offsets. - /// example: `values.gather(.{ .a = idx }, .{})` - /// - /// * indices is a named list of integer tensors: eg `.{ .a = idx }`. - /// Each names specify a gathering axis, it must refer to an axis of self, - /// and the corresponding idx Tensor must contains valid indices into axis .a. - /// All indices must have the same shape or broadcast to the same shape. - /// - /// * result is a tensor whose shape is similar to the input shape - /// where the gathered axes have been replaced by axes from 'indices'. - /// - /// Some example input for the base case where we work on one axis: - /// - gather(f:[a], .{ .a = idx:[n]})[n] == f[idx[n]] - /// - gather(f:[a, b], .a, idx:[n])[n, b] == f[idx[n], b] - /// - gather(f:[a,b,c], .{.b = idx:[n,m]})[a, n, m, c] == f[a, idx[n, m], c] - /// - /// If an axis in common between `self` and `indices`, - /// it is treated as a "batching" axis, meaning that semantically - /// the operator is doing a gather one time per dimension of this axis: - /// - gather(f: [a,b,c], .{.b=idx: [a,n]})[a, n] == f[a, idx[a, n]] - /// - /// It's possible to pass several indices: - /// - gather(f: [a,b,c], .{.b=idx_b[n], .c=idx_c[n]})[a, n] == f[a, idx_b[n], idx_c[n]] - /// - gather(f: [a,b,c,d], .{.b=idx_b[a,n], .c=idx_c[a, n]})[a, n, d] == f[a, idx_b[a, n], idx_c[a, n], d] - /// - /// If `self` isn't tagged, you can use `gather_` to specify gathered axis by their position but batching won't be available. - /// - /// For performance it's better to have batching and gathering axes of `self` be the first one, - /// so that gather can - pub fn gather(self: Tensor, _indices: anytype, opts: GatherOpts) Tensor { - const idx_per_axis, const idx_tags = Shape.parseStruct(Tensor, _indices); - var idx_axes: Shape.AxesArray = .{}; - for (idx_tags.slice()) |t| { - idx_axes.appendAssumeCapacity(self.axis(t)); - } - - // TODO: sort indices following self.shape instead of asking the user to do it. - return ops.gather(self, idx_axes.slice(), idx_per_axis.slice(), opts); - } - - test gather { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const Local = struct { - pub fn _idx(idx_shape: anytype) Tensor { - return Tensor.constant(.{ .i32 = 0 }).broad(Shape.init(idx_shape, .i64)); - } - }; - - const idx = Local._idx; - - { - // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); - defer comp.deinit(); - comp.activate(); - defer comp.deactivate(); - - const block = mlir.Block.init(&.{}, &.{}); - comp.pushBlock(block); - defer comp.popBlock(); - - inline for (.{ - .{ .{ .a = 10 }, .{ .a = idx(.{}) }, .{} }, - .{ .{ .a = 10 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8 } }, - .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{}) }, .{ .b = 20 } }, - .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8, .b = 20 } }, - // .{ .{ .a = 10, .b = 20 }, 0, idx(.{ .n = 8 }), .{ .n = 8, .b = 20 } }, - // Favor val shape, instead of indices shape. - .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .n = 8 }) }, .{ .a = 10, .n = 8 } }, - .{ .{ .a = 10, .b = 20, .c = 30 }, .{ .b = idx(.{ .n = 8 }) }, .{ .a = 10, .n = 8, .c = 30 } }, - // batching axes are implicits. - .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10 }) }, .{ .a = 10 } }, - .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .b = 20 }) }, .{ .b = 20 } }, - .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .n = 8 } }, - // stablehlo.gather is biased toward indices shape (like gatherSlice). - // This make it awkward to use when you have both batching dimension and new indices dimensions. - // For now we reject those, and let user explicitly transpose self or indices if needed. - // .{ .{ .a = 10, .b = 20 }, .{.b = idx(.{ .n = 8, .a = 10 })}, .{ .a = 10, .n = 8 } }, - // Also handle tuples - .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }), .b = idx(.{ .n = 8 }) }, .{ .n = 8 } }, - }) |testcase| { - const x_shape, const indices, const res_shape = testcase; - const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .i64)); - const y = gather(x, indices, .{}); - try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); - try std.testing.expect(y.value().owner().verify()); - } - - inline for (.{ - .{ .{ 10, 20 }, &[_]u3{ 0, 1 }, &[_]Tensor{ idx(.{8}), idx(.{8}) }, .{8} }, - }) |testcase| { - const x_shape, const idx_axes, const idx_per_axis, const res_shape = testcase; - const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .i64)); - const y = ops.gather(x, idx_axes, idx_per_axis, .{}); - try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); - try std.testing.expect(y.value().owner().verify()); - } - } - } - - /// Gathers slices along the given axes with runtime indices. - /// * slice_shape represents the shape of the slices to extract, - /// it must be smaller than original shape. - /// It must use a subset of self axes. - /// If slice_shape is **not** tagged, then it must have the same rank than self. - /// * `indices` represents a set of coordinates. - /// The coordinates are read from the `.coord` axis, or last axis if `.coord` is not found. - /// The coordinate axis must have `slice_shape.rank()` dims. - /// The coordinates represent the "top-left" corner of the slice to extract. - /// * the output tensor starts with axes from `indices`. - /// * if the input tensor has tagged axes, matching `indices` axes, - /// they will be considered "batching" axes. - /// - /// Sample input/output shapes: - /// * gatherSlices([A, B, C, D], .{.b=B', .c=C'}, [N, 2]) -> [N, A, B', C', D] - /// * gatherSlices(x(a,b,c,d), .{.b=B', .c=C'}, g(n,m)) = z(n, a, b', c', d) = x(a, g(n, 0) + b', g(n, 1) + c', d) - /// - /// Note: the axis order of the result is different from gatherValues. - /// This is because gatherSlices, favorizes contiguous copy of the extracted slices, - /// while gatherValues, always copy values one by one, and as such don't have the same issues. - /// In our example the contiguous dimension .d is not sliced - /// and gatherSlices can copy data by group of C'*D elements. - pub fn gatherSlices(self: Tensor, slice_shape_: anytype, indices: Tensor, opts: GatherOpts) Tensor { - const slice_shape = if (@TypeOf(slice_shape_) == Shape) slice_shape_ else Shape.init(slice_shape_, .i32); - // scoped_log.debug("gatherSlice({}, {f}, {})", .{ self, slice_shape, indices }); - - const tagged_api = slice_shape.isFullyTagged(); - if (tagged_api) { - for (slice_shape.tags()) |t| { - stdx.debug.assert(self._shape.hasTag(t) != null, "gatherSlices expects `slices_shape` to only use tags from `self`. But {s} wasn't found in {f}", .{ t, self }); - } - } else { - // For untagged api, we require all slices to be specified. - // Note: we could relax this and right align the slice. - stdx.debug.assert(slice_shape.rank() == self.rank(), "gatherSlices expects `slice_shape.rank()` to match `self.rank()`. Got: gatherSlices({f}, slice={f}). To avoid specifying all axes in `slice_shape`, you can use tags.", .{ self, slice_shape }); - } - - const index_coord_axis = indices._shape.hasTag(.coord) orelse indices._shape.axis(-1); - stdx.debug.assert(indices.dim(index_coord_axis) == slice_shape.rank(), "gatherSlices({f}, slice={f}, indices) expects 'indices' to be a tensor [..., {}], got {f}", .{ self, slice_shape, slice_shape.rank(), indices }); - - // Compute result shape - var res_shape = indices._shape.remove(index_coord_axis).withDtype(self.dtype()); - var slice_dims = self._shape._dims; - var self_batch_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; - var indices_batch_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; - var start_index_map: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; - var self_offset_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; - for (self._shape.tags(), 0..self.rank()) |t, self_ax| { - const maybe_slice_ax: ?u3 = if (tagged_api) slice_shape.hasTag(t) else @intCast(self_ax); - - if (tagged_api and indices._shape.hasTag(t) != null) { - // tag is both in self and indices -> it's a batching dim - // Note: tags are required for batching. - self_batch_axes.appendAssumeCapacity(@intCast(self_ax)); - indices_batch_axes.appendAssumeCapacity(indices._shape.axis(t)); - slice_dims.set(self_ax, 1); - stdx.debug.assert(slice_shape.hasTag(t) == null, "gatherSlices expect axes to be either batches or slices axes. Axis {s} has been found both in `slices={f}` and `indices={f}`", .{ t, slice_shape, indices }); - } else if (maybe_slice_ax) |slice_ax| { - // Specified axes contains the start offset of the slices, - // and are collected in `start_index_map`. - const slice_dim = slice_shape.dim(slice_ax); - stdx.debug.assert(slice_dim <= self._shape.dim(self_ax), "gatherSlices expects `slice_shape` to be smaller than `self.shape()`. On axis {s}, got {f} > {f}.", .{ t, slice_shape, self._shape }); - slice_dims.set(self_ax, slice_dim); - res_shape = res_shape.appendDim(slice_dim, t); - start_index_map.appendAssumeCapacity(@intCast(self_ax)); - self_offset_axes.appendAssumeCapacity(res_shape.rank() - 1); - } else { - // non-batching, non-indexed axes - res_shape = res_shape.appendDim(self.dim(self_ax), t); - self_offset_axes.appendAssumeCapacity(res_shape.rank() - 1); - } - } - - const gather_op = dialects.stablehlo.gather( - mlirCtx(), - self.value(), - indices.value(), - slice_dims.constSlice(), - .{ - .offset_dims = self_offset_axes.constSlice(), - .collapsed_slice_dims = &.{}, - .operand_batching_dims = self_batch_axes.constSlice(), - .start_indices_batching_dims = indices_batch_axes.constSlice(), - .start_index_map = start_index_map.constSlice(), - .index_vector_dim = index_coord_axis, - .indices_are_sorted = opts.indices_are_sorted, - }, - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - return _result(res_shape, gather_op.result(0)); - } - - test gatherSlices { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const Local = struct { - pub fn _gatherSlices(self: Tensor, slice_shape: Shape, indices: Tensor, opts: GatherOpts) Tensor { - return self.gatherSlices(slice_shape, indices, opts); - } - }; - - { - // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); - defer comp.deinit(); - comp.activate(); - defer comp.deactivate(); - - const block = mlir.Block.init(&.{}, &.{}); - comp.pushBlock(block); - defer comp.popBlock(); - - inline for (.{ - .{ .{ .a = 10 }, .{}, .{ ._ = 0 }, .{ .a = 10 } }, - .{ .{ .a = 10 }, .{ .a = 7 }, .{ ._ = 1 }, .{ .a = 7 } }, - .{ .{ .a = 10 }, .{ .a = 7 }, .{ .n = 8, ._ = 1 }, .{ .n = 8, .a = 7 } }, - .{ .{ .a = 10 }, .{ .a = 7 }, .{ .coord = 1, .n = 8 }, .{ .n = 8, .a = 7 } }, - // tags aren't required. - .{ .{10}, .{7}, .{ .n = 8, ._ = 1 }, .{ .n = 8, ._ = 7 } }, - .{ .{ .a = 10, .b = 20 }, .{ .a = 7 }, .{ ._ = 1 }, .{ .a = 7, .b = 20 } }, - .{ .{ .a = 10, .b = 20 }, .{ .a = 7 }, .{ .n = 8, ._ = 1 }, .{ .n = 8, .a = 7, .b = 20 } }, - .{ .{ .a = 10, .b = 20 }, .{ .a = 7 }, .{ .n = 8, .coord = 1, .m = 9 }, .{ .n = 8, .m = 9, .a = 7, .b = 20 } }, - .{ .{ .a = 10, .b = 20 }, .{ .b = 17 }, .{ .n = 8, ._ = 1 }, .{ .n = 8, .a = 10, .b = 17 } }, - .{ .{ .a = 10, .b = 20 }, .{ .a = 7, .b = 17 }, .{ .n = 8, ._ = 2 }, .{ .n = 8, .a = 7, .b = 17 } }, - // Note: currently the order of the axes in the slice is not used. - .{ .{ .a = 10, .b = 20 }, .{ .b = 17, .a = 7 }, .{ .n = 8, ._ = 2 }, .{ .n = 8, .a = 7, .b = 17 } }, - .{ .{ .a = 10, .b = 20, .c = 20 }, .{ .b = 17 }, .{ .n = 8, ._ = 1 }, .{ .n = 8, .a = 10, .b = 17, .c = 20 } }, - // batching dims - .{ .{ .a = 10, .b = 20 }, .{ .b = 17 }, .{ .a = 10, ._ = 1 }, .{ .a = 10, .b = 17 } }, - .{ .{ .b = 200, .a = 100, .c = 300 }, .{ .c = 300 }, .{ .a = 100, .b = 200, ._ = 1 }, .{ .a = 100, .b = 200, .c = 300 } }, - }) |testcase| { - const x_shape, const slice_dims, const idx_shape, const res_shape = testcase; - const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); - const slice_shape = Shape.init(slice_dims, .u16); - const idx = Tensor.constant(.{ .i32 = 0 }).broad(Shape.init(idx_shape, .i32)); - const y = gatherSlices(x, slice_shape, idx, .{}); - try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); - try std.testing.expect(y.value().owner().verify()); - - _ = Local._gatherSlices(x, slice_shape, idx, .{ .indices_are_sorted = true }); - } - } - - // Test with actual values. - const operand: Tensor = .init(.{ .a = 2, .b = 4, .c = 6 }, .u16); - const indices: Tensor = .init(.{ .n = 2, ._ = 2 }, .i32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._gatherSlices, .{ operand, Shape.init(.{ .b = 2, .c = 3 }, .u16), indices, .{} }, platform); - defer exe.deinit(); - - const indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, indices.shape(), std.mem.sliceAsBytes(&[2][2]i32{ .{ 2, 1 }, .{ 0, 3 } })); - defer indices_buffer.deinit(); - const operand_buffer: zml.Buffer = b: { - const temp_func = struct { - fn forward() Tensor { - return Tensor.arange(.{ .end = 2 * 4 * 6 }, .u16).reshape(.{ .a = 2, .b = 4, .c = 6 }); - } - }.forward; - var temp_exe = try zml.module.compile(std.testing.allocator, std.testing.io, temp_func, .{}, platform); - defer temp_exe.deinit(); - - const buffer = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &temp_exe, temp_func, {}); - break :b buffer; - }; - defer operand_buffer.deinit(); - const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._gatherSlices, .{ operand_buffer, indices_buffer }); - defer result.deinit(); - - const expected: ConstSlice = .init(Shape.init(.{ 2, 2, 2, 3 }, .u16), std.mem.sliceAsBytes(&[2][2][2][3]u16{ - .{ - .{ .{ 13, 14, 15 }, .{ 19, 20, 21 } }, - .{ .{ 37, 38, 39 }, .{ 43, 44, 45 } }, - }, - .{ - .{ .{ 3, 4, 5 }, .{ 9, 10, 11 } }, - .{ .{ 27, 28, 29 }, .{ 33, 34, 35 } }, - }, - })); - try zml.testing.expectClose(std.testing.io, expected, result, 0); - } - - pub const ScatterOpts = struct { - /// Promise scatter that all coordinates in `indices` are sorted, wrt to the final offset in `self` - /// Result is undefined if the promise is violated. - indices_are_sorted: bool = false, - - /// Promise scatter that slices don't overlap. - /// Result is undefined if the promise is violated. - /// This allows for better code generation, because it means that updates can be applied in parallel. - indices_are_unique: bool = false, - - /// Function used to update previous value in `self` with values from `updates`. - /// If `update_fn` is not associative (ie the order of execution matters), - /// then you should make sure the slices don't overlap, - /// otherwise the result will depend on the runtime scheduling - /// of the operator which is backend specific. - update_fn: *const fn (ops.ScatterArgs) struct { Tensor } = increment, - - pub fn increment(values: ops.ScatterArgs) struct { Tensor } { - return .{values.input.add(values.update)}; - } - - pub fn override(values: ops.ScatterArgs) struct { Tensor } { - return .{values.update}; - } - }; - - /// Update the given tensor, by copying `values` into slice by slice into `self`. - /// The slices are chosen at runtime by interpreting indices as coordinates into `self`. - /// This is a generalized version of `dynamicUpdateSlice` where more than one offset can be specified at a time. - /// - /// ### Arguments - /// - /// - Return a tensor with same shape than `self`, with updated content. - /// - `indices` is a set of Tensor (typically rank 1), representing coordinates into `self`. - /// all indices must have the same shape, but scalars are accepted. - /// - each `indices` entry contains offset along an axes into `self`. - /// Typically axes are identified by their tags, but in the absence of tags on `indices`, - /// The entry in indices will be assigned to axes of `self` from major to minor axis. - /// It is recommended to have indices referencing only major axes of `self` for better performance. - /// - `values` shape is obtained by concatenating the shape of `indices` with the shape of the slices to be extracted. - /// - `opts`: `zml.Tensor.ScatterOpts` des - /// - /// ### Sample input/output shapes with corresponding pseudo-code. - /// - /// Basic `scatterSlices` with the first two axes (.a, .b) being indexed, and full (.c, .d) slice copies: - /// - /// ``` - /// fn scatterSlices(x[A, B, C, D], .{.a=off_a[N], .b=off_b[N]}, y[N, C, D]) [A, B, C, D] { - /// var z = x; - /// for (0..N) |n| { - /// for (0..C) |c| for (0..D) |d| {{ - /// z[off_a[n],off_b[n],c,d] += y[n, c, d]; - /// }} - /// } - /// return z; - /// } - /// ``` - /// - /// `scatterSlices` with the first three axes (.a, .b, .c) being indexed, and a partial copy of (.c, .d). - /// Note that .c axis is present both in the indices and updates, and `updates.dim(.c) < self.dim(.c)`. - /// - /// ``` - /// fn scatterSlices(x[A, B, C, D], .{.a=off_a[N], .b=off_b[N], .c=off_c[N]}, y[N, C', D]) [A, B, C, D] { - /// var z = x; - /// for (0..N) |n| { - /// for (0..C') |c| for (0..D) |d| {{ - /// z[off_a[n],off_b[n],off_c[n]+c,d] += y[n, c, d]; - /// }} - /// } - /// return z; - /// } - /// ``` - /// - /// `scatterSlices` with the first axis .a being indexed, and where .b is used as a batching axis. - /// Note that here .b axis is present in `self`, `off_a`, and `updates`, - /// and is not mentionned in the axes of indices. - /// - /// ``` - /// fn scatterSlices(x[A, B, C, D], .{.a=off_a[B,N]}, y[N, B, C, D]) [A, B, C, D] { - /// var z = x; - /// for (0..B) |b| { - /// for (0..N) |n| { - /// for (0..C) |c| for (0..D) |d| {{ - /// z[off_a[b,n],b,c,d] += y[n, b, c, d]; - /// }} - /// } - /// } - /// return z; - /// } - /// ``` - /// - /// ### Warnings - /// - /// - if `opts.update_fn` is not associative not all calls to `scatterSlices` are sound. - /// In particular if you scatter overlapping slices, with `zml.Tensor.ScatterOpts.override`, - /// then the result will depend on the execution order that you don't control. - /// - `scatterSlices` is a very expressive operator, and can lead to complicated code generation - /// that requires host<->device synchronization. - /// ZML tries to generate the easiest to optimize IR, and will warn you if it generates known problematic IR. - pub fn scatterSlices(self: Tensor, indices: anytype, updates: Tensor, opts: ScatterOpts) Tensor { - //scoped_log.debug("scatterSlices({}, {any}, {})", .{ self, indices, updates }); - - const UpdateType = @TypeOf(ScatterOpts.increment); - - const Custom = struct { - pub fn inc(values: ops.ScatterArgs, custom: *const UpdateType) struct { Tensor } { - return @call(.auto, custom, .{values}); - } - }; - - return ops.scatter(.{self}, indices, .{updates}, Custom.inc, .{opts.update_fn}, opts)[0]; - } - - test scatterSlices { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const Local = struct { - pub fn _scatter(self: Tensor, indices: []const Tensor, updates: Tensor) Tensor { - return self.scatterSlices( - indices, - updates, - .{ .update_fn = ScatterOpts.increment }, - ); - } - - pub fn _scatterCB(self: Tensor, coords: Tensor, updates: Tensor) Tensor { - return self.scatterSlices( - .{ .c = coords.choose1d(.coord, 0), .b = coords.choose1d(.coord, 1) }, - updates, - .{ .update_fn = ScatterOpts.increment }, - ); - } - - pub fn _idx(idx_shape: anytype) Tensor { - return Tensor.constant(.{ .i32 = 0 }).broad(Shape.init(idx_shape, .i32)); - } - }; - - { - // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); - defer comp.deinit(); - comp.activate(); - defer comp.deactivate(); - - const block = mlir.Block.init(&.{}, &.{}); - defer block.deinit(); - comp.pushBlock(block); - defer comp.popBlock(); - - const idx = Local._idx; - - inline for (.{ - // This is equivalent to a dynamic update slice, update 3 values at given offset of axis .a: - .{ .{ .a = 10 }, .{ .a = idx(.{}) }, .{ .a = 3 } }, - // Use .a as a batching axis with .a=10 x .n=8 updates of 2 elements of .b - .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .n = 8, .b = 2 } }, - // Same but with update transposed - .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .b = 2, .n = 8 } }, - // similar, but use the normalized form where a is no longer an explicit batching axis. - .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .a2 = 10, .n = 8 }), .b = idx(.{ .a2 = 10, .n = 8 }) }, .{ .a2 = 10, .n = 8, .b = 2 } }, - .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .a = 10, .n = 8 }), .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .n = 8, .b = 2 } }, - .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8, .a = 2 } }, - .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .n = 8 }), .a = idx(.{ .n = 8 }) }, .{ .n = 8, .a = 3, .b = 2 } }, - .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }), .b = idx(.{ .n = 8 }) }, .{ .a = 3, .n = 8, .b = 2 } }, - }) |testcase| { - const x_shape, const indices, const updates_shapes = testcase; - const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); - const updates = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(updates_shapes, .f16)); - - const y = scatterSlices(x, indices, updates, .{}); - // Shape doesn't change with scatterSlices - try zml.testing.expectEqualShapes(x.shape(), y.shape()); - try std.testing.expect(y.value().owner().verify()); - } - } - // Test with actual values, no batching. - { - const a: Tensor = .init(.{ 3, 3 }, .i32); - - const scatter_indices = Tensor.init(.{2}, .i32).withTags(.{.n}); - const updates = Tensor.init(.{ 2, 3 }, .i32).withTags(.{ .n, .b }); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform); - defer exe.deinit(); - - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); - defer a_buffer.deinit(); - var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), std.mem.sliceAsBytes(&[2]i32{ 0, 2 })); - defer scatter_indices_buffer.deinit(); - var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), std.mem.sliceAsBytes(&[2][3]i32{ .{ 10, 20, 30 }, .{ 70, 80, 90 } })); - defer updates_buffer.deinit(); - - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatter, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); - defer result.deinit(); - - const expected = [3][3]i32{ .{ 10, 21, 32 }, .{ 3, 4, 5 }, .{ 76, 87, 98 } }; - try std.testing.expect(a.shape().eql(result.shape())); - try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected), std.testing.io)); - } - // Test with setting individual values (no batching) - { - const a: Tensor = .init(.{9}, .i32); - - const scatter_indices = Tensor.init(.{2}, .i32).withTags(.{.n}); - const updates = Tensor.init(.{2}, .i32).withTags(.{.n}); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform); - defer exe.deinit(); - - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); - defer a_buffer.deinit(); - var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), std.mem.sliceAsBytes(&[2]i32{ 2, 7 })); - defer scatter_indices_buffer.deinit(); - var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), std.mem.sliceAsBytes(&[2]i32{ 20, 70 })); - defer updates_buffer.deinit(); - - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatter, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); - defer result.deinit(); - - const expected = [9]i32{ 0, 1, 22, 3, 4, 5, 6, 77, 8 }; - try std.testing.expect(a.shape().eql(result.shape())); - try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected), std.testing.io)); - } - { - // Test with actual values and batching along axis .a - const operand: Tensor = .init(.{ .a = 2, .b = 3, .c = 4, .d = 2 }, .u16); - const start_indices: Tensor = .init(.{ .n = 2, .a = 2, .m = 3, .coord = 2 }, .i32); - const values: Tensor = .init(.{ .n = 2, .a = 2, .m = 3, .c = 2, .d = 2 }, .u16); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatterCB, .{ operand, start_indices, values }, platform); - defer exe.deinit(); - - var operand_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, operand.shape(), std.mem.sliceAsBytes(&@as([2 * 3 * 4 * 2]u16, @splat(0)))); - defer operand_buffer.deinit(); - var start_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, start_indices.shape(), std.mem.sliceAsBytes(&[2][2][3][2]i32{ - .{ - .{ .{ 0, 0 }, .{ 1, 0 }, .{ 2, 1 } }, - .{ .{ 0, 1 }, .{ 1, 1 }, .{ 0, 9 } }, - }, - .{ - .{ .{ 0, 0 }, .{ 2, 1 }, .{ 2, 2 } }, - .{ .{ 1, 2 }, .{ 0, 1 }, .{ 1, 0 } }, - }, - })); - defer start_indices_buffer.deinit(); - var values_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, values.shape(), std.mem.sliceAsBytes(&@as([2 * 2 * 3 * 2 * 2]u16, @splat(1)))); - defer values_buffer.deinit(); - - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatterCB, .{ operand_buffer, start_indices_buffer, values_buffer }); - defer result.deinit(); - - const expected = [2][3][4][2]u16{ - .{ - .{ .{ 2, 2 }, .{ 3, 3 }, .{ 1, 1 }, .{ 0, 0 } }, - .{ .{ 0, 0 }, .{ 0, 0 }, .{ 2, 2 }, .{ 2, 2 } }, - .{ .{ 0, 0 }, .{ 0, 0 }, .{ 1, 1 }, .{ 1, 1 } }, - }, - .{ - .{ .{ 0, 0 }, .{ 1, 1 }, .{ 1, 1 }, .{ 0, 0 } }, - .{ .{ 2, 2 }, .{ 3, 3 }, .{ 1, 1 }, .{ 0, 0 } }, - .{ .{ 0, 0 }, .{ 1, 1 }, .{ 1, 1 }, .{ 0, 0 } }, - }, - }; - try std.testing.expect(operand.shape().eql(result.shape())); - try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected), std.testing.io)); - } - } - - /// Returns a Tensor containing the maximum over a given axis. - pub fn max(self: Tensor, axis_: anytype) Tensor { - const a = self.axis(axis_); - return ops.reduce( - .{self}, - .{Tensor.constant(self.dtype().minValue())}, - &.{a}, - struct { - pub fn cmp(args: ops.ReduceArgs) struct { Tensor } { - return .{args.left.maximum(args.right)}; - } - }.cmp, - .{}, - )[0]; - } - - /// Returns a Tensor containing the minimum over a given axis. - pub fn min(self: Tensor, axis_: anytype) Tensor { - const a = self.axis(axis_); - return ops.reduce( - .{self}, - .{Tensor.constant(self.dtype().maxValue())}, - &.{a}, - struct { - pub fn cmp(args: ops.ReduceArgs) struct { Tensor } { - return .{args.left.minimum(args.right)}; - } - }.cmp, - .{}, - )[0]; - } - - pub const ArgMaxRes = struct { - values: Tensor, - indices: Tensor, - - fn cmp(values: ops.ReduceArgs, indices: ops.ReduceArgs) struct { Tensor, Tensor } { - const left_gt_right = values.left.cmp(.GT, values.right); - const is_nan = values.left.cmp(.NE, values.left); - const left_gt_or_nan = left_gt_right.logical(.OR, is_nan); - // we are bubbling up Nan. - const max_val = left_gt_or_nan.select(values.left, values.right); - - // If values.left == values.right: keep the smallest idx. - const is_same = values.left.cmp(.EQ, values.right); - const is_first = indices.left.cmp(.LT, indices.right); - const is_same_but_first = is_same.logical(.AND, is_first); - const keep_left_idx = left_gt_or_nan.logical(.OR, is_same_but_first); - const max_idx = keep_left_idx.select(indices.left, indices.right); - - return .{ max_val, max_idx }; - } - }; - - pub fn argMax(x: Tensor, axis_: anytype) ArgMaxRes { - const a = x.axis(axis_); - const dt: DataType = if (x.dim(a) <= std.math.maxInt(i32)) .i32 else .i64; - - const values, const indices = ops.reduce( - .{ x, Tensor.arange(.{ .end = x.dim(axis_) }, dt).broadcast(x.shape(), &.{a}) }, - .{ Tensor.constant(x.dtype().minValue()), Tensor.constant(dt.zero()) }, - &.{a}, - ArgMaxRes.cmp, - .{}, - ); - return .{ .values = values, .indices = indices }; - } - - test argMax { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - const allocator = std.testing.allocator; - const ArgMaxTest = struct { - pub fn _fwd(x: Tensor) Tensor.ArgMaxRes { - return x.argMax(1); - } - }; - - const x: Tensor = .init(.{ 1, 5 }, .f32); - var exe = try zml.module.compile(allocator, std.testing.io, ArgMaxTest._fwd, .{x}, platform); - defer exe.deinit(); - - { - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, 4.1, 7.9, 0, 7.9 }})); - defer x_buffer.deinit(); - - const res = try zml.testing.autoCall(allocator, std.testing.io, &exe, ArgMaxTest._fwd, .{x_buffer}); - defer res.values.deinit(); - defer res.indices.deinit(); - const max_ = res.values.getValue(f32, std.testing.io); - const max_idx = res.indices.getValue(i32, std.testing.io); - try std.testing.expectEqual(max_, 7.9); - // We should always return the first max found. - try std.testing.expectEqual(max_idx, 2); - } - - { - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, std.math.nan(f32), 7.9, 0, 7.9 }})); - defer x_buffer.deinit(); - - const res = try zml.testing.autoCall(allocator, std.testing.io, &exe, ArgMaxTest._fwd, .{x_buffer}); - defer res.values.deinit(); - defer res.indices.deinit(); - const max_ = try res.values.getValue(f32, std.testing.io); - const max_idx = try res.indices.getValue(i32, std.testing.io); - try std.testing.expect(std.math.isNan(max_)); - try std.testing.expectEqual(max_idx, 1); - } - } - - pub const SortRes = ArgMaxRes; - - pub const SortOpts = struct { descending: bool = false }; - - /// Returns two Tensors. The first contains the sorted values and the second one contains the sorted indices. - pub fn sort(self: Tensor, axis_: anytype, opts: SortOpts) SortRes { - const a = self.axis(axis_); - const indices = Tensor.arange(.{ .end = self.dim(a) }, .i32).broadcast(self._shape, &.{a}); - const direction: dialects.stablehlo.ComparisonDirection.Direction = if (opts.descending) .GT else .LT; - const res = ops.sort(.{ self, indices }, a, struct { - fn call(values: ops.SortArgs, _: ops.SortArgs, direction_: dialects.stablehlo.ComparisonDirection.Direction) Tensor { - return values.left.cmp(direction_, values.right); - } - }.call, .{direction}, true); - return .{ .values = res[0], .indices = res[1] }; - } - - pub const ArgSortOpts = SortOpts; - - /// Returns a Tensor containing the indices corresponding to the sorted values over the given axis. - pub fn argsort(self: Tensor, axis_: anytype, opts: ArgSortOpts) Tensor { - return self.sort(axis_, .{ .descending = opts.descending }).indices; - } - - test argsort { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const Local = struct { - pub fn _argsort(x: Tensor, axis_: u3, opts: ArgSortOpts) Tensor { - return x.argsort(axis_, opts); - } - }; - - // 2D Tensor - dim = 1, ascending - { - const x: Tensor = .init(.{ 2, 5 }, .f32); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{} }, platform); - defer exe.deinit(); - - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 })); - defer x_buffer.deinit(); - - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); - defer res.deinit(); - - const res_cpu = try res.toSliceAlloc(std.testing.allocator, std.testing.io); - defer res_cpu.free(std.testing.allocator); - - try std.testing.expectEqualSlices(i32, &.{ 0, 3, 1, 2, 4, 1, 4, 3, 0, 2 }, res_cpu.items(i32)); - } - - // 3D Tensor, dim = 1, descending - { - const x: Tensor = .init(.{ 1, 5, 10 }, .f16); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{ .descending = true } }, platform); - defer exe.deinit(); - - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f16{ - -0.2505, 1.2520, -0.7041, 0.1066, 1.2773, -1.7246, 0.8389, 1.1094, 0.0601, 1.0684, - 0.9619, 1.3916, 1.2246, -0.1406, 0.3674, -1.2480, -1.7051, -0.0934, 0.3435, 0.4373, - 1.3809, 0.5444, -0.6079, 1.2031, -0.6880, 1.2979, -0.1869, 0.2991, 0.0156, 0.1847, - 0.6626, -0.3040, -0.8726, -1.4805, -1.6943, 1.1055, -2.0078, -0.5288, 0.8813, 0.8008, - 2.0527, 1.1230, 0.5430, 0.2494, -0.9434, 0.7876, 0.1818, 0.9258, -2.4902, 1.5918, - })); - defer x_buffer.deinit(); - - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); - defer res.deinit(); - - const res_cpu = try res.toSliceAlloc(std.testing.allocator, std.testing.io); - defer res_cpu.free(std.testing.allocator); - - try std.testing.expectEqualSlices(i32, &.{ - 4, 1, 1, 2, 0, 2, 0, 0, 3, 4, - 2, 0, 4, 4, 1, 3, 4, 4, 1, 0, - 1, 4, 2, 0, 2, 4, 2, 2, 0, 3, - 3, 2, 0, 1, 4, 1, 1, 1, 2, 1, - 0, 3, 3, 3, 3, 0, 3, 3, 4, 2, - }, res_cpu.items(i32)); - } - - // 4D Tensor, dim = 3, ascending - { - const x: Tensor = .init(.{ 4, 2, 1, 4 }, .i32); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 3, .{} }, platform); - defer exe.deinit(); - - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ - 89, 31, 22, 42, - 64, 39, 0, 30, - 64, 71, 46, 31, - 89, 82, 78, 86, - 55, 32, 43, 19, - 93, 24, 45, 72, - 64, 86, 62, 88, - 57, 21, 19, 12, - })); - defer x_buffer.deinit(); - - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); - defer res.deinit(); - - const res_cpu = try res.toSliceAlloc(std.testing.allocator, std.testing.io); - defer res_cpu.free(std.testing.allocator); - - try std.testing.expectEqualSlices(i32, &.{ - 2, 1, 3, 0, - 2, 3, 1, 0, - 3, 2, 0, 1, - 2, 1, 3, 0, - 3, 1, 2, 0, - 1, 2, 3, 0, - 2, 0, 1, 3, - 3, 2, 1, 0, - }, res_cpu.items(i32)); - } - } - - pub const TopKOpts = struct { descending: bool = true }; - - /// Returns a Tensor representing the result of Top-K over the given axis. - pub fn topK(self: Tensor, named_axis_: anytype, k: u32, opts: TopKOpts) SortRes { - stdx.debug.assert(k > 0, "topK expects a k > 0, got 0", .{}); - const err_msg = "topK named axis should be an integer or a named axis, eg `x.topK(.{{ .best_token = .token }}, 16)` or `x.topK(-1, 16)`"; - const has_name: ?[:0]const u8, const a = switch (@typeInfo(@TypeOf(named_axis_))) { - .int, .comptime_int => .{ null, self.axis(@as(i64, @intCast(named_axis_))) }, - .@"struct" => |info| blk: { - stdx.debug.assertComptime(info.fields.len == 1, err_msg, .{}); - break :blk .{ info.fields[0].name, self.axis(@field(named_axis_, info.fields[0].name)) }; - }, - else => stdx.debug.compileError(err_msg, .{}), - }; - var result = self.sort(a, .{ .descending = opts.descending }); - result.values = result.values.slice1d(a, .{ .end = k }); - result.indices = result.indices.slice1d(a, .{ .end = k }); - if (has_name) |new_name| { - result.values._shape._tags.set(a, new_name.ptr); - result.indices._shape._tags.set(a, new_name.ptr); - } - return result; - } - - pub const MaxPoolRes = ArgMaxRes; - - /// Computes the 1d maxPool operation on the input Tensor. - pub fn maxPool1d(self: Tensor, opts: struct { - window_dimensions: i64, - window_strides: ?i64, - base_dilations: i64 = 1, - window_dilations: i64 = 1, - padding: [2]i64 = .{ 0, 0 }, - }) MaxPoolRes { - // TODO migrate to the following syntax. - // maxPool(.{.a = .{ .stride = 5, .dilation = 2, .padding = .{0, 1} }, - // .b = .{ .stride = 8, .dilation = 2, .padding = .{0, 1} }), - // maxPool(.{ - // .stride = .{ .a = 5, .b = 8 }, - // .dilation = .{ .a = 2, .b = 2 }, - // .padding = .{ .a = .{ 0, 2 }, .b = .{0, 2} - // }) - - // TODO: support maxPool on non last axis - const a = self.axis(-1); - const ones = [_]i64{1} ** constants.MAX_RANK; - var window_dimensions = ones; - window_dimensions[a] = opts.window_dimensions; - var window_strides = window_dimensions; - if (opts.window_strides) |stride| window_strides[a] = stride; - - var base_dilations = ones; - base_dilations[a] = opts.base_dilations; - var window_dilations = ones; - window_dilations[a] = opts.window_dilations; - - var padding = [_][2]i64{.{ 0, 0 }} ** constants.MAX_RANK; - padding[a] = opts.padding; - - const result = ops.reduceWindow( - .{ self, iota(self._shape, a) }, - .{ Tensor.constant(self.dtype().minValue()), Tensor.scalar(0, .i32) }, - .{ - .window_dimensions = window_dimensions[0..self.rank()], - .window_strides = window_strides[0..self.rank()], - .base_dilations = base_dilations[0..self.rank()], - .window_dilations = window_dilations[0..self.rank()], - .padding = padding[0..self.rank()], - }, - MaxPoolRes.cmp, - .{}, - ); - return .{ .values = result[0], .indices = result[1] }; - } - - /// Computes the 2d maxPool operation on the input Tensor. - pub fn maxPool2d(self: Tensor, opts: struct { - window_dimensions: [2]i64, - window_strides: ?[2]i64 = null, - base_dilations: [2]i64 = .{ 1, 1 }, - window_dilations: [2]i64 = .{ 1, 1 }, - padding: [2][2]i64 = .{ .{ 0, 0 }, .{ 0, 0 } }, - }) MaxPoolRes { - // TODO: rewrite using modern ZML - stdx.debug.guard(self.rank() == 3 or self.rank() == 4, @src()); - - // TODO: support maxPool on non last axis - // Note: the problem is initPoolArg assuming last axis - const a = self.axis(-1); - - const window_dimensions = initPoolArg(self.rank(), &opts.window_dimensions); - const window_strides = if (opts.window_strides) |ws| initPoolArg(self.rank(), &ws) else window_dimensions; - const base_dilation = initPoolArg(self.rank(), &opts.base_dilations); - const window_dilations = initPoolArg(self.rank(), &opts.window_dilations); - - var padding = [_][2]i64{.{ 0, 0 }} ** constants.MAX_RANK; - padding[a - 1] = opts.padding[0]; - padding[a] = opts.padding[1]; - - const result = ops.reduceWindow( - .{ self, iota(self._shape, a) }, - .{ Tensor.constant(self.dtype().minValue()), Tensor.scalar(0, .i32) }, - .{ - .window_dimensions = window_dimensions[0..self.rank()], - .window_strides = window_strides[0..self.rank()], - .base_dilations = base_dilation[0..self.rank()], - .window_dilations = window_dilations[0..self.rank()], - .padding = padding[0..self.rank()], - }, - MaxPoolRes.cmp, - .{}, - ); - return .{ .values = result[0], .indices = result[1] }; - } - - /// Chunk a given tensor into exactly n parts of equal shape. - /// `self.dim(axis_)` must be divisible by n_chunks. - pub fn chunkExact(self: Tensor, axis_: anytype, n_chunks: comptime_int) [n_chunks]Tensor { - const a = self.axis(axis_); - const d = self.dim(a); - const chunk_size = @divExact(d, n_chunks); - var chunks: [n_chunks]Tensor = undefined; - for (0..n_chunks) |i| { - const start: i64 = @as(i64, @intCast(i)) * chunk_size; - chunks[i] = self.slice1d(a, .{ .start = start, .end = start + chunk_size }); - } - return chunks; - } - - test chunkExact { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); - defer comp.deinit(); - comp.activate(); - defer comp.deactivate(); - - const block = mlir.Block.init(&.{}, &.{}); - defer block.deinit(); - comp.pushBlock(block); - defer comp.popBlock(); - - inline for (.{ - .{ .{ .a = 12 }, .a, 3, .{ .a = 4 } }, - .{ .{ .a = 12, .b = 2 }, .a, 3, .{ .a = 4, .b = 2 } }, - .{ .{ 12, 2 }, 0, 3, .{ 4, 2 } }, - }) |testcase| { - const x_shape, const ax, const n_chunks, const res = testcase; - const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); - const chunks = x.chunkExact(ax, n_chunks); - - const res_shape = Shape.init(res, .f16); - for (chunks) |chk| { - try zml.testing.expectEqualShapes(res_shape, chk.shape()); - } - } - } - - /// Chunk a given tensor into n parts of equal shape, and one part with the remaining items. - /// When `self.dim(axis_)` is divisible by `n_chunks` it will return exactly `n_chunks`. - pub fn chunkAllowTrailing( - self: Tensor, - axis_: i64, - n_chunks: comptime_int, - ) ![]Tensor { - const a = self.axis(axis_); - const d = self.dim(a); - const chunk_size: i64 = @divFloor(d, n_chunks); - const tail_chunk_size: i64 = @rem(d, chunk_size); - - const allocator = CompilationContext.current().arena.allocator(); - - var chunks = std.ArrayList(Tensor).initCapacity(allocator, n_chunks + 1) catch @panic("OOM"); - defer chunks.deinit(allocator); - - for (0..n_chunks) |i| { - const start: i64 = @as(i64, @intCast(i)) * chunk_size; - chunks.appendAssumeCapacity( - self.slice1d(a, .{ .start = start, .end = start + chunk_size }), - ); - } - if (tail_chunk_size != 0) { - const start: i64 = n_chunks * chunk_size; - chunks.appendAssumeCapacity(self.slice1d(a, .{ .start = start })); - } - return chunks.toOwnedSlice(allocator); - } - - test chunkAllowTrailing { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); - defer comp.deinit(); - comp.activate(); - defer comp.deactivate(); - - const block = mlir.Block.init(&.{}, &.{}); - defer block.deinit(); - comp.pushBlock(block); - defer comp.popBlock(); - - inline for (.{ - .{ .{ .a = 10 }, .a, 3, .{ .a = 3 }, .{ .a = 1 } }, - .{ .{ .a = 10, .b = 2 }, .a, 3, .{ .a = 3, .b = 2 }, .{ .a = 1, .b = 2 } }, - .{ .{ 10, 2 }, 0, 3, .{ 3, 2 }, .{ 1, 2 } }, - .{ .{ 12, 2 }, 0, 3, .{ 4, 2 }, .{} }, - }) |testcase| { - const x_shape, const ax, const n_chunks, const res, const trailing = testcase; - const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); - const chunks = try x.chunkAllowTrailing(x.axis(ax), n_chunks); - - const res_shape = Shape.init(res, .f16); - for (chunks[0..n_chunks]) |chk| { - try zml.testing.expectEqualShapes(res_shape, chk.shape()); - } - const trailing_shape = Shape.init(trailing, .f16); - if (trailing_shape.rank() > 0) { - try std.testing.expectEqual(n_chunks + 1, chunks.len); - try zml.testing.expectEqualShapes(trailing_shape, chunks[n_chunks].shape()); - } else { - try std.testing.expectEqual(n_chunks, chunks.len); - } - } - } - - pub fn split(self: Tensor, axis_: anytype, split_sizes: []const i64) []Tensor { - stdx.debug.assert(split_sizes.len > 0, "split expects at least one 'split_sizes', got 0", .{}); - - const a = self.axis(axis_); - const d = self.dim(a); - var split_sum: i64 = 0; - for (split_sizes) |n| split_sum += n; - stdx.debug.assert(split_sum == d, "split expects sum of 'split_sizes' values and axis dimension to be equal, got {} and {}", .{ split_sum, d }); - - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); - defer arena.deinit(); - - const res = arena.allocator().alloc(Tensor, split_sizes.len) catch @panic("OOM"); - - var start: i64 = 0; - for (split_sizes, 0..) |n, i| { - res[i] = self.slice1d(a, .{ .start = start, .end = start + n }); - start += n; - } - return res; - } - - /// Slices the input Tensor along a specific axis, with a start offset known at runtime. - /// Note: this doesn't support tagging, if you have tags, - /// you should use `dynamicSlice` directly. - pub fn dynamicSlice1d(self: Tensor, axis_: i8, slice_: DynSlice) Tensor { - stdx.debug.assert(slice_.start.rank() == 0, "dynamicSlice1d expects 'slice_.start' tensor rank to be a scalar, got {f}", .{slice_.start}); - - const a = self.axis(axis_); - const new_shape = self._shape.set(a, slice_.len); - - var start_indices = [_]*const mlir.Value{constant(slice_.start.dtype().zero()).value()} ** constants.MAX_RANK; - start_indices[a] = slice_.start.value(); - - const op = dialects.stablehlo.dynamic_slice( - mlirCtx(), - self.value(), - new_shape.dims(), - start_indices[0..self.rank()], - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - - return _result(new_shape, op.result(0)); - } - - pub const DynSlice = struct { start: Tensor, len: i64 }; - - /// Slices a Tensor across many axes, with runtime known offsets. - /// - /// Due to the nature of stablehlo, the length of the slices need to be known when compiling the IR. - /// When using the tagged API it is allowed to not specify some axes. - /// But with the non-tagged API all slices need to be specified. - /// Examples: - /// ``` - /// Tensor(.{.a=20,.b=30,.c=40 }).dynamicSlice(.{ .a = .{ .start = a_off, .len = 11}); - /// Tensor(.{.a=20,.b=30,.c=40 }).dynamicSlice(.{ - /// .a = .{ .start = a_off, .len = 11 }, - /// .b = .{ .start = b_off, .len = 12 }, - /// }); - /// Tensor(.{ 20,30,40}).dynamicSlice(.{.{ .start = scalar(0, .i32), .len = 20 }, .{ .start = b_off, .len = 12 }, .{ .start = scalar(0, .i32), .len = 40 }}); - /// ``` - pub fn dynamicSlice(self: Tensor, slices_: anytype) Tensor { - // TODO: the untagged api is a bit verbose. Should I allow: `Tensor(.{ 20,30,40}).dynamicSlice(.{.{}, .{ .start = b_off, .len = 12 }, .{}});` ?? - // - const slices, const slices_tags = Shape.parseStruct(DynSlice, slices_); - - // TODO use slices and slices_tags for the format. - // Currently this prints: "dynSlice(struct{q: struct{start: tensor.Tensor, comptime len: comptime_int = 1}}{ .q = struct{start: tensor.Tensor, comptime len: comptime_int = 1}{ .start = Tensor({1,10}, dtype=.i64), .len = 1 } })" - // which is kinda ugly. - - const idx_dtype = if (slices.len > 0) slices.get(0).start.dtype() else .i32; - const zero = Tensor.scalar(0, idx_dtype).value(); - var offset_values = [_]*const mlir.Value{zero} ** constants.MAX_RANK; - var res_shape = self._shape; - for (slices.constSlice(), 0..) |slice_, i| { - const offset = slice_.start; - const len = slice_.len; - if (slices_tags.len == 0) { - stdx.debug.assert(self.rank() == slices.len, "dynamicSlice expects tensor rank and 'slices_' length to be equal, got {d} and {d}", .{ self.rank(), slices.len }); - - offset_values[i] = offset.value(); - res_shape._dims.set(i, len); - - stdx.debug.assert(len <= self.dim(i), "dynamicSlice expects slices 'len' to be less than or equal to their corresponding dimension in input tensor, got {d} and {d} for index {d}", .{ len, self.dim(i), i }); - } else { - const t = slices_tags.get(i); - const a = res_shape.hasTag(t) orelse stdx.debug.panic("dynamicSlice expects input tensor to have tags used in 'slices_' but {s} is missing (input shape is {f})", .{ t, self._shape }); - - stdx.debug.assert(len <= self.dim(a), "dynamicSlice expects slices 'len' to be less than their corresponding dimension in input tensor, got {d} and {d} for axis {s}", .{ len, self.dim(a), t }); - - offset_values[a] = offset.value(); - res_shape._dims.set(a, len); - } - } - const op = dialects.stablehlo.dynamic_slice(mlirCtx(), self.value(), res_shape.dims(), offset_values[0..self.rank()], .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(res_shape, op.result(0)); - } - - test dynamicSlice { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - inline for (.{ - .{ [10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, Shape.init(.{10}, .f32), [2]f32{ 4, 5 }, 4, 0 }, - .{ [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }, Shape.init(.{ 2, 5 }, .f32), [4]f32{ 3, 4, 8, 9 }, 3, 1 }, - }) |testcase| { - const x_data, const x_shape, const expectation, const z_value: i32, const ax = testcase; - const x: Tensor = .fromShape(x_shape); - const z: Tensor = .init(.{}, .i32); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.dynamicSlice1d, .{ x, ax, .{ .len = 2, .start = z } }, platform); - defer exe.deinit(); - - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&x_data)); - defer x_buffer.deinit(); - var z_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, z.shape(), std.mem.asBytes(&z_value)); - defer z_buffer.deinit(); - - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.dynamicSlice1d, .{ x_buffer, .{ .start = z_buffer } }); - defer res.deinit(); - - try std.testing.expectEqual(expectation, try res.getValue(@TypeOf(expectation), std.testing.io)); - } - } - - /// Updates a slice of the input Tensor along a specific axis using the given 'update' Tensor, with a start offset known at runtime. - /// Note this is the untagged api, if you have tags, you should use dynamicUpdateSlice directly. - pub fn dynamicUpdateSlice1d(self: Tensor, update: Tensor, axis_: i64, offset: Tensor) Tensor { - const placeholder = Tensor.scalar(0, .i32); - var start_indices = [_]Tensor{placeholder} ** constants.MAX_RANK; - start_indices[self.axis(axis_)] = offset; - return self.dynamicUpdateSlice(start_indices[0..self.rank()], update); - } - - /// Updates a part of the input Tensor using the given 'update' Tensor, with runtime known offsets. - /// - /// The offsets are specified similarly to the dynamicSlice api. - /// It's semantically equivalent to: - /// self.dynamicSlice(offsets_) := update - /// Examples: - /// ``` - /// Tensor(.{ .a = 2, .b = 5 }).dynamicUpdateSlice(.{ .a = scalar(1, .i32) }, Tensor(.{ .b = 5 })); - /// ``` - pub fn dynamicUpdateSlice(self: Tensor, offset_: anytype, update_: Tensor) Tensor { - // TODO: add updateSlice for when the offset isn't dynamic - stdx.debug.assert(self.dtype() == update_.dtype(), "dynamicUpdateSlice expects input and 'update_' tensors to be of the same type, got {} and {}", .{ self.dtype(), update_.dtype() }); - - const offset, const offset_tags = Shape.parseStruct(Tensor, offset_); - // log.debug("offset: {any}, offset_tags: {any}", .{ offset, offset_tags }); - for (offset.constSlice(), 0..) |start_idx, i| { - stdx.debug.assert(start_idx.rank() == 0, "dynamicUpdateSlice expects 'offset_' tensor ranks to be equal to 0, got {} at index {}", .{ start_idx.rank(), i }); - } - - const tagged_api = update_._shape.isFullyTagged() and self._shape.isFullyTagged() and offset_tags.len > 0; - // When using tags, we can safely insert axis with a 1-dim. - // the offset into the inserted axis will need to be specified through indices. - var update = update_; - if (tagged_api) { - // Check that all update tags are known. - for (update._shape._tags.constSlice()) |t| { - stdx.debug.assert(self._shape.hasTag(t) != null, "dynamicUpdateSlice expects 'update_' tensor tags to be a subset of input tensor tags but {s} is missing (input shape is {f})", .{ t, self._shape }); - } - - var update_shape = self._shape; - var prev_ax: i8 = -1; - for (self._shape.tags(), 0..) |t, self_ax| { - if (update._shape.hasTag(t)) |up_ax| { - stdx.debug.assert(up_ax == prev_ax + 1, "dynamicUpdateSlice expects 'update_' and input tensor axis to have the same order, got {f} and {f}. (hint: you need to explicitly transpose 'update_')", .{ update_, self }); - - update_shape._dims.set(self_ax, update.dim(up_ax)); - prev_ax = up_ax; - } else { - update_shape._dims.set(self_ax, 1); - } - } - update = update.reshape(update_shape); - } - - stdx.debug.assert(self.rank() == update.rank(), "dynamicUpdateSlice expects input and computed update tensors to have the same rank, got {f} and {f}", .{ self, update }); - - for (self.dims(), update.dims(), 0..) |self_d, up_d, ax| { - const t = self._shape.debugTag(ax); - stdx.debug.assert(up_d <= self_d, "dynamicUpdateSlice expects 'update_' dimensions to be less than or equal to their corresponding dimension in input tensor, got {} and {} for axis .{s}", .{ up_d, self_d, t }); - - if (tagged_api and up_d < self_d) { - const axis_has_offset = std.mem.indexOfScalar(Shape.Tag, offset_tags.constSlice(), self._shape._tags.get(ax)) != null; - - stdx.debug.assert(axis_has_offset, "dynamicUpdateSlice expects 'update_' dimensions to be equal to their corresponding dimension in input tensor, got {} and {} for axis .{s} (hint: you need to provide an offset)", .{ up_d, self_d, t }); - } - } - - const idx_dtype = if (offset.len > 0) offset.get(0).dtype() else .i32; - const zero = Tensor.scalar(0, idx_dtype).value(); - var offset_values: [constants.MAX_RANK]*const mlir.Value = undefined; - if (offset_tags.len == 0) { - // Without offset tags we need the same number of offset than rank. - stdx.debug.assert(self.rank() == offset.len, "dynamicUpdateSlice expects input tensor rank and 'offset_' length to be equal, got {} and {}", .{ self.rank(), offset.len }); - - for (offset.constSlice(), 0..) |idx, i| { - offset_values[i] = idx.value(); - } - } else { - // If an axis isn't specified, update the full slice. - // This is only allowed when using tagged sliced. - offset_values = .{zero} ** constants.MAX_RANK; - for (offset.constSlice(), offset_tags.constSlice()) |start, t| { - const a = self._shape.hasTag(t) orelse stdx.debug.panic("dynamicUpdateSlice expects input tensor to have tags used in 'offset_' but {s} is missing (input shape is {f})", .{ t, self._shape }); - offset_values[a] = start.value(); - } - } - - const op = dialects.stablehlo.dynamic_update_slice( - mlirCtx(), - self.value(), - update.value(), - offset_values[0..self.rank()], - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - return _result(self._shape, op.result(0)); - } - - test dynamicUpdateSlice { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - { - const x = Tensor.init(.{10}, .f32).withTags(.{.a}); - const y = Tensor.init(.{2}, .f32).withTags(.{.a}); - const ids: Tensor = .init(.{}, .i32); - const forward = struct { - pub fn _fwd(x_: Tensor, idx_: struct { a: Tensor }, y_: Tensor) Tensor { - return x_.dynamicUpdateSlice(idx_, y_); - } - }._fwd; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = ids }, y }, platform); - defer exe.deinit(); - - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); - defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); - defer y_buffer.deinit(); - var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{4})); - defer ids_buffer.deinit(); - - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ .a = ids_buffer }, y_buffer }); - defer res.deinit(); - - try std.testing.expectEqual([10]f32{ 0, 1, 2, 3, -1, -1, 6, 7, 8, 9 }, try res.getValue([10]f32, std.testing.io)); - } - - { - // Updates 2D, tagged api - const x = Tensor.init(.{ 2, 5 }, .f32).withTags(.{ .a, .b }); - const y = Tensor.init(.{2}, .f32).withTags(.{.a}); - const ids: Tensor = .init(.{}, .i32); - const forward = struct { - pub fn _fwd(x_: Tensor, idx_: Tensor, y_: Tensor) Tensor { - return x_.dynamicUpdateSlice(.{ .b = idx_ }, y_); - } - }._fwd; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform); - defer exe.deinit(); - - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); - defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); - defer y_buffer.deinit(); - var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3})); - defer ids_buffer.deinit(); - - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, ids_buffer, y_buffer }); - defer res.deinit(); - - try std.testing.expectEqualDeep( - [2][5]f32{ .{ 0, 1, 2, -1, 4 }, .{ 5, 6, 7, -1, 9 } }, - try res.getValue([2][5]f32, std.testing.io), - ); - } - - { - // Updates 2D slice, un-tagged api. Note that `y` needs to have a 1 dimension axis. - const x: Tensor = .init(.{ 2, 5 }, .f32); - const y: Tensor = .init(.{ 2, 1 }, .f32); - const ids: Tensor = .init(.{}, .i32); - const forward = struct { - pub fn _fwd(x_: Tensor, idx_: Tensor, y_: Tensor) Tensor { - return x_.dynamicUpdateSlice(.{ zml.Tensor.scalar(0, .i32), idx_ }, y_); - } - }._fwd; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform); - defer exe.deinit(); - - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); - defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[2][1]f32{ .{-1}, .{-1} })); - defer y_buffer.deinit(); - var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3})); - defer ids_buffer.deinit(); - - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, ids_buffer, y_buffer }); - defer res.deinit(); - - try std.testing.expectEqualDeep( - [2][5]f32{ .{ 0, 1, 2, -1, 4 }, .{ 5, 6, 7, -1, 9 } }, - res.getValue([2][5]f32, std.testing.io), - ); - } - - { - // Updates 2D, partial update - const x = Tensor.init(.{ 2, 5 }, .f32).withTags(.{ .a, .b }); - const y = Tensor.init(.{1}, .f32).withTags(.{.a}); - const idx_a: Tensor = .init(.{}, .i32); - const idx_b: Tensor = .init(.{}, .i32); - const forward = struct { - pub fn _fwd(x_: Tensor, idx_: struct { a: Tensor, b: Tensor }, y_: Tensor) Tensor { - return x_.dynamicUpdateSlice(idx_, y_); - } - }._fwd; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = idx_a, .b = idx_b }, y }, platform); - defer exe.deinit(); - - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); - defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[1]f32{-1})); - defer y_buffer.deinit(); - var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), std.mem.sliceAsBytes(&[1]i32{1})); - defer idx_a_buffer.deinit(); - var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3})); - defer idx_b_buffer.deinit(); - - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ .a = idx_a_buffer, .b = idx_b_buffer }, y_buffer }); - defer res.deinit(); - - try std.testing.expectEqualDeep( - [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, -1, 9 } }, - res.getValue([2][5]f32, std.testing.io), - ); - } - - { - // Updates 2D, partial update, un-tagged api. - const x: Tensor = .init(.{ 2, 5 }, .f32); - const y: Tensor = .init(.{ 1, 1 }, .f32); - const idx_a: Tensor = .init(.{}, .i32); - const idx_b: Tensor = .init(.{}, .i32); - const forward = struct { - pub fn _fwd(x_: Tensor, idx_: [2]Tensor, y_: Tensor) Tensor { - return x_.dynamicUpdateSlice(&idx_, y_); - } - }._fwd; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ idx_a, idx_b }, y }, platform); - defer exe.deinit(); - - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); - defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[1][1]f32{.{-1}})); - defer y_buffer.deinit(); - var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), std.mem.sliceAsBytes(&[1]i32{1})); - defer idx_a_buffer.deinit(); - var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3})); - defer idx_b_buffer.deinit(); - - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ idx_a_buffer, idx_b_buffer }, y_buffer }); - defer res.deinit(); - - try std.testing.expectEqualDeep( - [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, -1, 9 } }, - res.getValue([2][5]f32, std.testing.io), - ); - } - } - - /// Returns a Tensor containing the element-wise result of the given 'cmp' comparison between the two input Tensors. - pub fn cmp(self: Tensor, direction: dialects.stablehlo.ComparisonDirection.Direction, other: Tensor) Tensor { - stdx.debug.assert(self.dtype() == other.dtype(), "cmp expects input tensors to be of the same type, got {t} and {t}", .{ self.dtype(), other.dtype() }); - - if (self.rank() == 0 and other.rank() != 0) return self.broadcast(other._shape, &.{}).cmp(direction, other); - if (self.rank() != 0 and other.rank() == 0) return self.cmp(direction, other.broadcast(self._shape, &.{})); - - stdx.debug.assert(self._shape.eql(other._shape), "cmp expects input tensor shapes to match, got {f} and {f}", .{ self._shape, other._shape }); - - const op = dialects.stablehlo.compare( - mlirCtx(), - self.value(), - other.value(), - dialects.stablehlo.ComparisonDirection.init(mlirCtx(), direction).getValue(), - getComparisonType(mlirCtx(), self.dtype()).getValue(), - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - - return _result(self._shape.withDtype(.bool), op.result(0)); - } - - /// For each vector in the input tensor, - /// creates a diagonal-matrix where diagonal values are set to the vector values. - pub fn toDiagonal(self: Tensor, axis_: anytype, new_tags: [2]@EnumLiteral()) Tensor { - stdx.debug.assert(self.rank() < constants.MAX_RANK - 1, "toDiagonal expects input up to {d} rank, got {f}", .{ constants.MAX_RANK - 1, self }); - const a = self.axis(axis_); - const d = self.dim(a); - const p = self.shape()._partitioning.get(a); - var res_shape = self._shape; - res_shape._dims.replaceRange(a, 1, &.{ d, d }) catch unreachable; - res_shape._tags.replaceRange(a, 1, &.{ @tagName(new_tags[0]), @tagName(new_tags[1]) }) catch unreachable; - // TODO(Corentin): Not sure about that - res_shape._partitioning.replaceRange(a, 1, &.{ p, p }) catch unreachable; - - const values = self.insertAxes(a + 1, .{new_tags[1]}).broad(res_shape); - const zeros = Tensor.constant(self.dtype().zero()).broad(res_shape); - - const x = Tensor.iota(res_shape, a); - const y = Tensor.iota(res_shape, a + 1); - var res = x.cmp(.EQ, y).select(values, zeros); - res._shape = res_shape; - return res; - } - - test toDiagonal { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const Local = struct { - pub fn _toDiag(input: Tensor) Tensor { - const res = input.toDiagonal(-1, .{ .x, .y }); - std.debug.assert(res.dim(.x) == input.dim(-1)); - std.debug.assert(res.dim(.y) == input.dim(-1)); - return res; - } - }; - - const x: Tensor = .init(.{ 2, 2 }, .u8); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._toDiag, .{x}, platform); - defer exe.deinit(); - - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][2]u8{ .{ 1, 2 }, .{ 3, 4 } })); - defer x_buffer.deinit(); - - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._toDiag, .{x_buffer}); - defer res.deinit(); - try std.testing.expectEqual( - [2][2][2]u8{ .{ - .{ 1, 0 }, - .{ 0, 2 }, - }, .{ - .{ 3, 0 }, - .{ 0, 4 }, - } }, - try res.getValue([2][2][2]u8, std.testing.io), - ); - } - - /// For each matrix specified by the two axes, returns the lower triangular part of it. - /// The other elements are set to 0. - /// Usage: `.{ .b = 32, .w = 20, .h = 20 }.triangular(.{ .w, .h}, 0);` - /// - /// * if `num_diagonals` is set to 0, the diagonal is not modified. - /// * if set to -1, the diagonal is set to 0 - /// * if set to n, the n "quasi diagonal" above the diagonal are conserved. - /// - /// To get the upper triangular part, swap the order of axes: - /// `.{ .b = 32, .w = 20, .h = 20 }.triangular(.{ .h, .w }, 0);` - pub fn triangular(self: Tensor, axes_: anytype, num_diagonals: i32) Tensor { - stdx.debug.assertComptime(stdx.meta.isTuple(@TypeOf(axes_)) and axes_.len == 2, "triangular expects exactly two axes to work on.", .{}); - const _axes = self.axes(axes_); - - const x = Tensor.iota(self.shape(), _axes.get(0)); - const y = Tensor.iota(self.shape(), _axes.get(1)); - - const zeros = Tensor.constant(self.dtype().zero()).broad(self.shape()); - return x.addConstant(num_diagonals).cmp(.GE, y).select(self, zeros); - } - - test triangular { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const Local = struct { - pub fn _tri(input: Tensor, num_diagonals: i32) Tensor { - return input.triangular(.{ -2, -1 }, num_diagonals); - } - }; - - const x: Tensor = .init(.{ 3, 3 }, .u8); - - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[3][3]u8{ - .{ 1, 1, 1 }, - .{ 1, 1, 1 }, - .{ 1, 1, 1 }, - })); - defer x_buffer.deinit(); - - { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 0 }, platform); - defer exe.deinit(); - - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); - defer res.deinit(); - - try std.testing.expectEqual( - [3][3]u8{ - .{ 1, 0, 0 }, - .{ 1, 1, 0 }, - .{ 1, 1, 1 }, - }, - try res.getValue([3][3]u8, std.testing.io), - ); - } - { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 1 }, platform); - defer exe.deinit(); - - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); - defer res.deinit(); - - try std.testing.expectEqual( - [3][3]u8{ - .{ 1, 1, 0 }, - .{ 1, 1, 1 }, - .{ 1, 1, 1 }, - }, - try res.getValue([3][3]u8, std.testing.io), - ); - } - { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, -1 }, platform); - defer exe.deinit(); - - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); - defer res.deinit(); - - try std.testing.expectEqual( - [3][3]u8{ - .{ 0, 0, 0 }, - .{ 1, 0, 0 }, - .{ 1, 1, 0 }, - }, - try res.getValue([3][3]u8, std.testing.io), - ); - } - } - - /// For each element at index `i`, if `bool_tensor[i] == true`, `output[i] = on_true[i]` - /// otherwise, if `bool_tensor[i] == false`, `output[i] = on_false[i]` - pub fn select(bool_tensor: Tensor, on_true: Tensor, on_false: Tensor) Tensor { - stdx.debug.assert(bool_tensor.dtype() == .bool, "select expects input tensor type to be a boolean, got {}", .{bool_tensor.dtype()}); - stdx.debug.assert(on_true.dtype() == on_false.dtype(), "select expects 'on_true' and 'on_false' tensor types to be equal, got {} and {}", .{ on_true.dtype(), on_false.dtype() }); - - if (bool_tensor.rank() != 0 and on_true.rank() == 0) { - return bool_tensor.select(on_true.broad(bool_tensor.shape()), on_false); - } - if (bool_tensor.rank() != 0 and on_false.rank() == 0) { - return bool_tensor.select(on_true, on_false.broad(bool_tensor.shape())); - } - - stdx.debug.assert(bool_tensor._shape.eqlDims(on_true._shape), "select expects input tensor and 'on_true' tensor dimensions to match, got {f} and {f}", .{ bool_tensor._shape, on_true._shape }); - stdx.debug.assert(bool_tensor._shape.eqlDims(on_false._shape), "select expects input tensor and 'on_false' tensor dimensions to match, got {f} and {f}", .{ bool_tensor._shape, on_false._shape }); - - const op = dialects.stablehlo.select( - mlirCtx(), - bool_tensor.value(), - on_true.value(), - on_false.value(), - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - - return _result(on_true._shape, op.result(0)); - } - - /// Returns a Tensor containing the element-wise not logical operation of the input Tensor. - pub fn not(self: Tensor) Tensor { - const op = dialects.stablehlo.not(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape, op.result(0)); - } - - /// Returns a Tensor containing boolean indicating if there is a non-zero value over the given axis. - pub fn any(self: Tensor, axis_: anytype) Tensor { - const pred = self.cmp(.NE, Tensor.constant(self.dims(), self.dtype().zero())); - return ops.reduce( - .{pred}, - .{Tensor.scalar(false, .bool)}, - &.{self.axis(axis_)}, - struct { - pub fn acc(args: ops.ReduceArgs) Tensor { - return args.left.logical(.OR, args.right); - } - }.acc, - .{}, - )[0]; - } - - /// Returns a Tensor containing boolean indicating if there is a non-zero value over the given axis. - pub fn all(self: Tensor, axis_: anytype) Tensor { - const pred = if (self.dtype() == .bool) self else self.cmp(.NE, Tensor.scalar(0, self.dtype())); - return ops.reduce( - .{pred}, - .{Tensor.scalar(true, .bool)}, - &.{self.axis(axis_)}, - struct { - pub fn acc(args: ops.ReduceArgs) Tensor { - return args.left.logical(.AND, args.right); - } - }.acc, - .{}, - )[0]; - } - - /// Given a set of N vectors of lengths A, B, C, D, - /// returns N tensors of rank N, and shape (A, B, C, D). - /// For any coordinate (a, b, c, d), we have: - /// - /// - res[0][a, b, c, d] == A[a] - /// - res[1][a, b, c, d] == B[b] - /// - res[2][a, b, c, d] == C[c] - /// - res[3][a, b, c, d] == D[d] - /// - /// This is implemented with broadcasting, so typically it won't copy. - /// In Pytorch/Numpy this is know as `meshgrid` with "ij" mode. - /// See `zml.torch.meshgrid` for the "xy" mode. - pub fn cartesianProduct(comptime N: u3, vectors: [N]Tensor) [N]Tensor { - var out: @TypeOf(vectors) = undefined; - _cartesianProduct(&vectors, &out); - return out; - } - - fn _cartesianProduct(vectors: []const Tensor, out: []Tensor) void { - stdx.debug.assert(vectors.len >= 1, "cartesianProduct expects at least one input.", .{}); - stdx.debug.assert(vectors.len < constants.MAX_RANK, "cartesianProduct expects at most {} input vectors, received {} !", .{ constants.MAX_RANK - 1, vectors.len }); - for (vectors) |x| { - stdx.debug.assert(x.rank() <= 1, "cartesianProduct expects 0 or 1 rank input vectors. Got: {any}", .{vectors}); - stdx.debug.assert(vectors[0].dtype() == x.dtype(), "cartesianProduct expects input vectors to have all the same dtype. Got: {any}", .{vectors}); - } - - var res_shape = Shape.init(.{}, vectors[0].dtype()); - for (vectors) |x| { - if (x.rank() == 0) { - res_shape = res_shape.appendDim(1, null); - } else { - res_shape = res_shape.appendDim(x.dim(0), x.shape().tag(0)); - } - } - - for (out, vectors, 0..) |*o, x, i| { - o.* = x.broadcast(res_shape, &[1]i64{@intCast(i)}); - } - } - - test cartesianProduct { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const x: Tensor = .init(.{6}, .i32); - const y: Tensor = .init(.{4}, .i32); - - const Local = struct { - pub fn _cartesianProduct2(a: Tensor, b: Tensor) [2]Tensor { - return cartesianProduct(2, .{ a, b }); - } - }; - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cartesianProduct2, .{ x, y }, platform); - defer exe.deinit(); - - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); - defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); - defer y_buffer.deinit(); - - var xs, var ys = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._cartesianProduct2, .{ x_buffer, y_buffer }); - defer xs.deinit(); - defer ys.deinit(); - - try std.testing.expectEqualSlices(i64, &.{ 6, 4 }, xs.shape().dims()); - try std.testing.expectEqualSlices(i64, &.{ 6, 4 }, ys.shape().dims()); - try std.testing.expectEqualDeep( - [6][4]i32{ - .{ 0, 0, 0, 0 }, - .{ 1, 1, 1, 1 }, - .{ 2, 2, 2, 2 }, - .{ 3, 3, 3, 3 }, - .{ 4, 4, 4, 4 }, - .{ 5, 5, 5, 5 }, - }, - try xs.getValue([6][4]i32, std.testing.io), - ); - try std.testing.expectEqualDeep( - [6][4]i32{ - .{ 0, 1, 2, 3 }, - .{ 0, 1, 2, 3 }, - .{ 0, 1, 2, 3 }, - .{ 0, 1, 2, 3 }, - .{ 0, 1, 2, 3 }, - .{ 0, 1, 2, 3 }, - }, - try ys.getValue([6][4]i32, std.testing.io), - ); - } - - /// Given a set of N vectors of lengths A, B, C, D, - /// returns 1 tensors of rank N+1, and shape (A, B, C, D, N). - /// For any coordinate (a, b, c, d), we have: - /// - /// - res[a, b, c, d] == (A[a], B[b], C[c], D[d]) - pub fn cartesianProductStacked(vectors: []const Tensor) Tensor { - var out = stdx.BoundedArray(Tensor, constants.MAX_RANK).init(vectors.len) catch unreachable; - _cartesianProduct(vectors, out.slice()); - - return Tensor.stack(out.constSlice(), .last, .coord); - } - - test cartesianProductStacked { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - const x: Tensor = .init(.{6}, .i32); - const y: Tensor = .init(.{4}, .i32); - - const Local = struct { - pub fn _fwd(a: Tensor, b: Tensor) Tensor { - return cartesianProductStacked(&.{ a, b }); - } - }; - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, y }, platform); - defer exe.deinit(); - - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); - defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); - defer y_buffer.deinit(); - - var z = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._fwd, .{ x_buffer, y_buffer }); - defer z.deinit(); - - try std.testing.expectEqualDeep( - [6][4][2]i32{ - .{ .{ 0, 0 }, .{ 0, 1 }, .{ 0, 2 }, .{ 0, 3 } }, - .{ .{ 1, 0 }, .{ 1, 1 }, .{ 1, 2 }, .{ 1, 3 } }, - .{ .{ 2, 0 }, .{ 2, 1 }, .{ 2, 2 }, .{ 2, 3 } }, - .{ .{ 3, 0 }, .{ 3, 1 }, .{ 3, 2 }, .{ 3, 3 } }, - .{ .{ 4, 0 }, .{ 4, 1 }, .{ 4, 2 }, .{ 4, 3 } }, - .{ .{ 5, 0 }, .{ 5, 1 }, .{ 5, 2 }, .{ 5, 3 } }, - }, - try z.getValue([6][4][2]i32, std.testing.io), - ); - } - - fn binaryOp( - op_name: []const u8, - op_fn: fn (*mlir.Context, *const mlir.Value, *const mlir.Value, *const mlir.Location) *mlir.Operation, - ) fn (Tensor, Tensor) Tensor { - return struct { - pub fn binaryOpHelper(self: Tensor, other: Tensor) Tensor { - stdx.debug.assert(self.dtype() == other.dtype(), "{s} expects tensor to be of same type, got {f} and {f}", .{ op_name, self, other }); - - if (self.rank() == 0 and other.rank() != 0) { - return binaryOpHelper(self.broad(other._shape), other); - } - - if (self.rank() != 0 and other.rank() == 0) { - return binaryOpHelper(self, other.broad(self._shape)); - } - - var other_ = other; - var same_shape = self._shape.eql(other._shape); - if (!same_shape and std.mem.eql(Shape.Tag, self._shape.tags(), other._shape.tags()) and other._shape.canBroadcastTo(self._shape)) { - // Only a restrictive version of broadcasting is allowed here, where all the tags matches already. - // Typical use case: `x.div(x.sum(.a))` - same_shape = true; - other_ = other.broad(self._shape); - } - - stdx.debug.assert(same_shape, "{s} expects tensor shapes to match, got {f} and {f}", .{ op_name, self._shape, other._shape }); - - const ret = @call(.auto, op_fn, .{ mlirCtx(), self.value(), other_.value(), mlir.Location.unknown(mlirCtx()) }).appendTo(currentBlock()); - return _result(self._shape, ret.result(0)); - } - }.binaryOpHelper; - } - - /// Insert code that will print the content of the given buffer at runtime. - /// Only for debug purpose, it inserts device to host synchronization - /// so it will slow down the program execution. - pub fn print(input: Tensor) Tensor { - // TODO(Corentin) - _ = input; - @panic("Unimplemented"); - } - - fn mlirCtx() *mlir.Context { - return CompilationContext.current().mlir_ctx; - } - - fn currentBlock() *mlir.Block { - return CompilationContext.current().currentScope().block; - } - - /// Returns the donation data of the tensor. - pub fn donation(self: Tensor) ?usize { - return CompilationContext.current().currentScope().id_to_donation.get(self.id); - } - - /// Returns the output memory kind of the tensor. - pub fn outputMemoryKind(self: Tensor) Buffer.Memory { - return CompilationContext.current().currentScope().id_to_output_memory_kind.get(self.id) orelse .device; - } -}; - -fn initPoolArg(rank: usize, data: []const i64) [constants.MAX_RANK]i64 { - // TODO use shape - var result = [_]i64{1} ** constants.MAX_RANK; - const start = rank - data.len; - @memcpy(result[start .. start + data.len], data); - return result; -} - -fn getPoolResDims(dt: DataType, in_dims: []const i64, base_dilations: @Vector(constants.MAX_RANK, i64), padding: []const i64, window_dimensions: @Vector(constants.MAX_RANK, i64), window_dilations: @Vector(constants.MAX_RANK, i64), window_strides: @Vector(constants.MAX_RANK, i64)) Shape { - // TODO use shape - var input_dims = [_]i64{1} ** constants.MAX_RANK; - @memcpy(input_dims[0..in_dims.len], in_dims); - - const input_dims_: @Vector(constants.MAX_RANK, i64) = input_dims; - const splat_one: @Vector(constants.MAX_RANK, i64) = @splat(1); - const dilated_input_shape: @Vector(constants.MAX_RANK, i64) = (input_dims_ - splat_one) * base_dilations + splat_one; - var pad_slice0: @Vector(constants.MAX_RANK, i64) = @splat(padding[0]); - var pad_slice1: @Vector(constants.MAX_RANK, i64) = @splat(padding[0]); - if (padding.len > 1) { - var idx: usize = 0; - while (idx < in_dims.len * 2) : (idx += 2) { - pad_slice0[idx / 2] = padding[idx]; - pad_slice1[idx / 2] = padding[idx + 1]; - } - } - const padded_input_shape: @Vector(constants.MAX_RANK, i64) = pad_slice0 + dilated_input_shape + pad_slice1; - const dilated_window_shape = (window_dimensions - splat_one) * window_dilations + splat_one; - const dims = @divFloor(padded_input_shape - dilated_window_shape, window_strides) + splat_one; - const dims_arr: [constants.MAX_RANK]i64 = @bitCast(dims); - return Shape.init(dims_arr[0..in_dims.len], dt); -} - -fn getComparisonType(ctx: *mlir.Context, dtype: DataType) *const dialects.stablehlo.CompareType { - return dialects.stablehlo.CompareType.init(ctx, switch (dtype.class()) { - .bool => .UNSIGNED, - .integer => if (dtype.isSignedInt()) .SIGNED else .UNSIGNED, - .float => .FLOAT, - .complex => @panic("Can't compare complex numbers"), - }); -} - -test "Tensor.maxPool1d" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const MaxPool = struct { - pub fn _fwd(x: zml.Tensor) Tensor.ArgMaxRes { - return x.maxPool1d(.{ - .window_dimensions = 3, - .window_strides = 2, - }); - } - }; - - var data: [20]f32 = undefined; - for (&data, 0..) |*v, i| v.* = @floatFromInt(i); - - const x: Tensor = .init(.{ 2, 2, 5 }, .f32); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform); - defer exe.deinit(); - - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&data)); - defer x_buffer.deinit(); - - const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, MaxPool._fwd, .{x_buffer}); - defer result.values.deinit(); - defer result.indices.deinit(); - - try zml.testing.expectEqualShapes(.init(.{ 2, 2, 2 }, .f32), result.values.shape()); - try zml.testing.expectEqualShapes(.init(.{ 2, 2, 2 }, .i32), result.indices.shape()); - try std.testing.expectEqualDeep( - [2][2][2]f32{ - .{ .{ 2, 4 }, .{ 7, 9 } }, - .{ .{ 12, 14 }, .{ 17, 19 } }, - }, - result.values.getValue([2][2][2]f32, std.testing.io), - ); -} - -test "Tensor.maxPool2d" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const MaxPool = struct { - pub fn _fwd(x: Tensor) Tensor.ArgMaxRes { - return x.maxPool2d(.{ - .window_dimensions = .{ 3, 2 }, - .window_strides = .{ 2, 1 }, - }); - } - }; - - var data: [100]f32 = undefined; - for (&data, 0..) |*v, i| v.* = @floatFromInt(i); - - const x: Tensor = .init(.{ 2, 2, 5, 5 }, .f32); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform); - defer exe.deinit(); - - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&data)); - defer x_buffer.deinit(); - - const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, MaxPool._fwd, .{x_buffer}); - defer result.values.deinit(); - defer result.indices.deinit(); - - try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2, 4 }, .f32), result.values.shape()); - try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2, 4 }, .i32), result.indices.shape()); - var buffer: [2][2][2][4]f32 = undefined; - _ = try result.values.toSlice(std.testing.io, .init(result.values.shape(), std.mem.asBytes(&buffer))); - try std.testing.expectEqualDeep( - [2][2][2][4]f32{ - .{ - .{ .{ 11, 12, 13, 14 }, .{ 21, 22, 23, 24 } }, - .{ .{ 36, 37, 38, 39 }, .{ 46, 47, 48, 49 } }, - }, - .{ - .{ .{ 61, 62, 63, 64 }, .{ 71, 72, 73, 74 } }, - .{ .{ 86, 87, 88, 89 }, .{ 96, 97, 98, 99 } }, - }, - }, - buffer, - ); -} - -fn _parseGatherCoord(self: Tensor, axes_: anytype) struct { bool, stdx.BoundedArray(u3, constants.MAX_RANK) } { - const AxesT = @TypeOf(axes_); - const axes_is_scalar = AxesT == @EnumLiteral() or AxesT == comptime_int or @typeInfo(AxesT) == .int; - - const coord_axes = if (axes_is_scalar) - stdx.BoundedArray(u3, constants.MAX_RANK).fromSlice(&.{self.axis(axes_)}) catch unreachable - else - self.axes(axes_); - - return .{ axes_is_scalar, coord_axes }; -} - -fn toI64(values: anytype) stdx.BoundedArray(i64, constants.MAX_RANK) { - var res: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; - for (values) |val| res.appendAssumeCapacity(@intCast(val)); - return res; -} - -fn transposeIsJustAReshape(x: Shape, permutation: []const i64) bool { - var perm: stdx.BoundedArray(struct { u8, bool }, constants.MAX_RANK) = .{}; - // Don't rewrite on invalid inputs. - if (permutation.len > x.rank()) return false; - for (permutation) |ax| { - const squeezable = x.dim(ax) == 1; - perm.appendAssumeCapacity(.{ @intCast(ax), squeezable }); - } - - var effective_ax: u8 = 0; - for (0..perm.len) |i| { - const ax, const squeezable = perm.get(i); - if (squeezable) { - // Effectively squeeze this axis by decrementing axes coming after by 1. - for (i..perm.len) |j| { - if (perm.buffer[j][0] > ax) { - perm.buffer[j][0] -= 1; - } - } - continue; - } - - if (ax != effective_ax) return false; - effective_ax += 1; - } - - return true; -} - -test transposeIsJustAReshape { - try std.testing.expect(transposeIsJustAReshape(Shape.init(.{ 5, 1, 3 }, .i32), &.{ 0, 1, 2 })); - try std.testing.expect(transposeIsJustAReshape(Shape.init(.{ 5, 1, 3 }, .i32), &.{ 1, 0, 2 })); - try std.testing.expect(!transposeIsJustAReshape(Shape.init(.{ 5, 1, 3 }, .i32), &.{ 2, 1, 0 })); - try std.testing.expect(transposeIsJustAReshape(Shape.init(.{ 64, 8, 1, 128 }, .bf16), &.{ 0, 2, 1, 3 })); - try std.testing.expect(!transposeIsJustAReshape(Shape.init(.{ 64, 8, 155, 128 }, .bf16), &.{ 0, 2, 1, 3 })); - try std.testing.expect(transposeIsJustAReshape(Shape.init(.{ 64, 1, 1, 128 }, .bf16), &.{ 1, 2, 0, 3 })); - try std.testing.expect(!transposeIsJustAReshape(Shape.init(.{ .b = 1, .h = 10, .q = 155, .hd = 1 }, .f32), &.{ 0, 2, 1, 3 })); - try std.testing.expect(!transposeIsJustAReshape(Shape.init(.{ 1, 10, 155, 1 }, .f32), &.{ 0, 2, 3, 1 })); - try std.testing.expect(transposeIsJustAReshape(Shape.init(.{ 1, 10, 155, 1 }, .f32), &.{ 0, 1, 3, 2 })); -} - -test "unused tensor" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const Local = struct { - pub fn _fwd(x: Tensor) Tensor { - const y = x.addConstant(1); - _ = y; - return x; - } - }; - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{Tensor.init(.{10}, .f32)}, platform); - defer exe.deinit(); -} diff --git a/zml2/test_runner.zig b/zml2/test_runner.zig deleted file mode 100644 index e3c801f3..00000000 --- a/zml2/test_runner.zig +++ /dev/null @@ -1,111 +0,0 @@ -//! Test runner for unit test based on https://github.com/ziglang/zig/blob/master/lib/compiler/test_runner.zig with async -const std = @import("std"); -const testing = std.testing; -const builtin = @import("builtin"); - -// note: std_options.log_level does not respect testing.log_level -// ref: https://github.com/ziglang/zig/issues/5738 -const log_level: std.log.Level = .warn; - -pub const std_options: std.Options = .{ - .log_level = log_level, -}; - -var log_err_count: usize = 0; -var fba_buffer: [8192]u8 = undefined; -var fba = std.heap.FixedBufferAllocator.init(&fba_buffer); - -pub fn main() !void { - testing.log_level = log_level; - - const test_fn_list: []const std.builtin.TestFn = builtin.test_functions; - var ok_count: usize = 0; - var skip_count: usize = 0; - var fail_count: usize = 0; - const root_node = std.Progress.start(.{ - .root_name = "Test", - .estimated_total_items = test_fn_list.len, - }); - const have_tty = std.fs.File.stderr().isTty(); - - var args = std.process.args(); - // Skip executable path - _ = args.next().?; - - const identifier_query = if (args.next()) |arg| blk: { - std.debug.print("Only tests with identifiers that includes `{s}` will be run\n", .{arg}); - break :blk arg; - } else blk: { - break :blk ""; - }; - - var leaks: usize = 0; - - for (test_fn_list, 0..) |test_fn, i| { - if (std.mem.indexOf(u8, test_fn.name, identifier_query) == null) { - continue; - } - - testing.allocator_instance = .{}; - defer { - if (testing.allocator_instance.deinit() == .leak) { - leaks += 1; - } - } - - testing.io_instance = .init(testing.allocator_instance.allocator()); - defer testing.io_instance.deinit(); - - const test_node = root_node.start(test_fn.name, 0); - if (!have_tty) { - std.debug.print("{d}/{d} {s}...", .{ i + 1, test_fn_list.len, test_fn.name }); - } - if (test_fn.func()) |_| { - ok_count += 1; - test_node.end(); - if (!have_tty) std.debug.print("OK\n", .{}); - } else |err| switch (err) { - error.SkipZigTest => { - skip_count += 1; - if (have_tty) { - std.debug.print("{d}/{d} {s}...SKIP\n", .{ i + 1, test_fn_list.len, test_fn.name }); - } else { - std.debug.print("SKIP\n", .{}); - } - test_node.end(); - }, - else => { - fail_count += 1; - if (have_tty) { - std.debug.print("{d}/{d} {s}...FAIL ({s})\n", .{ - i + 1, test_fn_list.len, test_fn.name, @errorName(err), - }); - } else { - std.debug.print("FAIL ({s})\n", .{@errorName(err)}); - } - if (@errorReturnTrace()) |trace| { - std.debug.dumpStackTrace(trace); - } - test_node.end(); - }, - } - } - root_node.end(); - if (ok_count == test_fn_list.len) { - std.debug.print("All {d} tests passed.\n", .{ok_count}); - } else { - std.debug.print("{d} passed; {d} skipped; {d} failed.\n", .{ ok_count, skip_count, fail_count }); - } - if (log_err_count != 0) { - std.debug.print("{d} errors were logged.\n", .{log_err_count}); - } - if (leaks != 0) { - std.debug.print("{d} tests leaked memory.\n", .{leaks}); - } - if (leaks != 0 or log_err_count != 0 or fail_count != 0) { - std.process.exit(1); - } - - // Explicit exit 0 to exit from async main thread and main properly - std.process.exit(0); -} diff --git a/zml2/testing.zig b/zml2/testing.zig deleted file mode 100644 index bf968551..00000000 --- a/zml2/testing.zig +++ /dev/null @@ -1,149 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); - -const stdx = @import("stdx"); - -const ConstSlice = @import("slice.zig").ConstSlice; -const Platform = @import("platform.zig").Platform; -const zml = @import("zml.zig"); - -const log = std.log.scoped(.@"zml/testing"); - -var _platform: ?Platform = null; - -pub fn env() Platform { - if (!builtin.is_test) @compileError("Cannot use zml.testing.env outside of a test block"); - if (_platform == null) { - zml.init(); - - _platform = Platform.auto(std.testing.io, .{}) catch unreachable; - } - - return _platform.?; -} - -/// In neural network we generally care about the relative precision, -/// but on a given dimension, if the output is close to 0, then the precision -/// don't matter as much. -pub fn approxEq(comptime Float: type, l: Float, r: Float, tolerance: Float) bool { - const closeRel = std.math.approxEqRel(Float, l, r, @floatCast(tolerance)); - const closeAbs = std.math.approxEqAbs(Float, l, r, @floatCast(tolerance / 2)); - return closeRel or closeAbs; -} - -/// Testing utility. Accepts both zml.Buffer and zml.HostBuffer but zml.Buffer will be copied to the -/// host for comparison ! -pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, tolerance: f32) !void { - const allocator = if (builtin.is_test) std.testing.allocator else std.heap.smp_allocator; - var left: ConstSlice, const should_free_left = if (@TypeOf(left_) == zml.Buffer) b: { - const slice = try left_.toSliceAlloc(allocator, io); - break :b .{ slice.constSlice(), true }; - } else .{ left_, false }; - - var right: ConstSlice, const should_free_right = if (@TypeOf(right_) == zml.Buffer) b: { - const slice = try right_.toSliceAlloc(allocator, io); - break :b .{ slice.constSlice(), true }; - } else .{ right_, false }; - - defer { - if (should_free_left) left.free(allocator); - if (should_free_right) right.free(allocator); - } - errdefer log.err("\n--> Left: {0f}{0d:24.3}\n--> Right: {1f}{1d:24.3}", .{ left, right }); - if (!std.mem.eql(i64, left.shape.dims(), right.shape.dims())) { - log.err("left.shape() {f} != right.shape() {f}", .{ left.shape, right.shape }); - return error.TestUnexpectedResult; - } - if (left.dtype() != right.dtype() and !(left.dtype() == .f16 and right.dtype() == .bf16)) { - log.err("left.dtype ({}) != right.dtype ({})", .{ left.dtype(), right.dtype() }); - return error.TestUnexpectedResult; - } - - switch (left.dtype()) { - inline .bf16, - .f16, - .f32, - .f64, - .f4e2m1, - .f8e3m4, - .f8e4m3, - .f8e4m3b11fnuz, - .f8e4m3fn, - .f8e4m3fnuz, - .f8e5m2, - .f8e5m2fnuz, - .f8e8m0, - => |t| { - const L = t.toZigType(); - const left_data = left.items(L); - switch (right.dtype()) { - inline .bf16, - .f16, - .f32, - .f64, - .f8e4m3b11fnuz, - .f8e4m3fn, - .f8e4m3fnuz, - .f8e5m2, - .f8e5m2fnuz, - => |rt| { - const R = rt.toZigType(); - const right_data = right.items(R); - for (left_data, right_data, 0..) |l, r, i| { - if (!approxEq(f32, zml.floats.floatCast(f32, l), zml.floats.floatCast(f32, r), tolerance)) { - log.err("left.data != right_data.\n < {d:40.3} \n > {d:40.3}\n error at idx {d}: {d:.3} != {d:.3}", .{ stdx.fmt.slice(center(left_data, i)), stdx.fmt.slice(center(right_data, i)), i, left_data[i], right_data[i] }); - return error.TestUnexpectedResult; - } - } - }, - else => unreachable, - } - }, - inline .bool, .u2, .u4, .u8, .u16, .u32, .u64, .i2, .i4, .i8, .i16, .i32, .i64 => |t| { - const T = t.toZigType(); - return std.testing.expectEqualSlices(T, left.items(T), right.items(T)); - }, - .c64, .c128 => @panic("TODO: support comparison of complex"), - } -} - -pub fn expectEqualShapes(expected: zml.Shape, actual: zml.Shape) error{TestExpectedEqual}!void { - if (expected.eqlWithTags(actual)) return; - - std.debug.print("Expected {f}, got {f}", .{ expected, actual }); - return error.TestExpectedEqual; -} - -fn BufferizedWithArgs(comptime T: type) type { - return zml.meta.MapType(zml.Tensor, zml.Buffer).map(T); -} - -/// Automatically calls the executable with the given arguments, taking care of arguments and results allocation. -/// This helper can be used in tests to make the code a bit less verbose. -/// It doesn't handle pointers in inputs/outputs structs. -pub fn autoCall(allocator: std.mem.Allocator, io: std.Io, exe: *const zml.exe.Exe, func: anytype, inputs: zml.Bufferized(stdx.meta.FnArgs(func))) !zml.Bufferized(stdx.meta.FnResult(func)) { - var args = try exe.args(allocator); - defer args.deinit(allocator); - - var results = try exe.results(allocator); - defer results.deinit(allocator); - - args.set(inputs); - exe.callOpts(io, args, &results, .{ .wait = true }); - - var output: zml.Bufferized(stdx.meta.FnResult(func)) = undefined; - results.fill(.{&output}); - - return output; -} - -fn center(slice: anytype, i: usize) @TypeOf(slice) { - const c = 20; - const start = if (i < c) 0 else i - c; - const end = @min(start + 2 * c, slice.len); - return slice[start..end]; -} - -pub inline fn expectEqual(expected: anytype, actual: @TypeOf(expected)) !void { - return std.testing.expectEqual(expected, actual); -} diff --git a/zml2/tokenizer/BUILD.bazel b/zml2/tokenizer/BUILD.bazel deleted file mode 100644 index f34e30b3..00000000 --- a/zml2/tokenizer/BUILD.bazel +++ /dev/null @@ -1,41 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_test") -load("@zml//bazel:zig_srcs.bzl", "zig_srcs") - -zig_library( - name = "tokenizer", - srcs = ["homemade.zig"], - import_name = "zml/tokenizer", - main = "tokenizer.zig", - visibility = ["//visibility:public"], - deps = [ - "//ffi:zig", - "//stdx", - "//zml2/tokenizer/hftokenizers", - "//zml2/tokenizer/sentencepiece", - ], -) - -zig_binary( - name = "main", - main = "main.zig", - visibility = ["//visibility:public"], - deps = [ - ":tokenizer", - "//stdx", - ], -) - -zig_test( - name = "test", - main = "homemade.zig", - visibility = ["//visibility:public"], - deps = [ - "//ffi:zig", - "//stdx", - ], -) - -zig_srcs( - name = "sources", - zig_lib = ":tokenizer", -) diff --git a/zml2/tokenizer/hftokenizers/BUILD.bazel b/zml2/tokenizer/hftokenizers/BUILD.bazel deleted file mode 100644 index a07d855b..00000000 --- a/zml2/tokenizer/hftokenizers/BUILD.bazel +++ /dev/null @@ -1,32 +0,0 @@ -load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_rust//rust:defs.bzl", "rust_static_library") -load("@rules_zig//zig:defs.bzl", "zig_library") - -rust_static_library( - name = "hftokenizers_rs", - srcs = ["hftokenizers.rs"], - crate_name = "zml_tokenizer_hftokenizers", - edition = "2021", - deps = ["@crates//:tokenizers"], -) - -cc_library( - name = "hftokenizers_cc", - hdrs = ["hftokenizers.h"], - visibility = ["//visibility:public"], - deps = [ - ":hftokenizers_rs", - "//ffi:cc", - ], -) - -zig_library( - name = "hftokenizers", - main = "hftokenizers.zig", - visibility = ["//visibility:public"], - deps = [ - ":hftokenizers_cc", - "//ffi:zig", - "//stdx", - ], -) diff --git a/zml2/tokenizer/hftokenizers/Cargo.lock b/zml2/tokenizer/hftokenizers/Cargo.lock deleted file mode 100644 index 68d1a36c..00000000 --- a/zml2/tokenizer/hftokenizers/Cargo.lock +++ /dev/null @@ -1,713 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "ahash" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" -dependencies = [ - "cfg-if", - "getrandom", - "once_cell", - "serde", - "version_check", - "zerocopy 0.8.26", -] - -[[package]] -name = "aho-corasick" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" -dependencies = [ - "memchr", -] - -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - -[[package]] -name = "bitflags" -version = "2.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "castaway" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" -dependencies = [ - "rustversion", -] - -[[package]] -name = "cc" -version = "1.2.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f" -dependencies = [ - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "compact_str" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" -dependencies = [ - "castaway", - "cfg-if", - "itoa", - "rustversion", - "ryu", - "serde", - "static_assertions", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" - -[[package]] -name = "darling" -version = "0.20.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" -dependencies = [ - "darling_core", - "darling_macro", -] - -[[package]] -name = "darling_core" -version = "0.20.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn", -] - -[[package]] -name = "darling_macro" -version = "0.20.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" -dependencies = [ - "darling_core", - "quote", - "syn", -] - -[[package]] -name = "dary_heap" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728" -dependencies = [ - "serde", -] - -[[package]] -name = "derive_builder" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "derive_builder_macro" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" -dependencies = [ - "derive_builder_core", - "syn", -] - -[[package]] -name = "either" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" - -[[package]] -name = "esaxx-rs" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" -dependencies = [ - "cc", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "getrandom" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" -dependencies = [ - "cfg-if", - "libc", - "r-efi", - "wasi", -] - -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - -[[package]] -name = "itertools" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" - -[[package]] -name = "libc" -version = "0.2.169" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" - -[[package]] -name = "log" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" - -[[package]] -name = "macro_rules_attribute" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13" -dependencies = [ - "macro_rules_attribute-proc_macro", - "paste", -] - -[[package]] -name = "macro_rules_attribute-proc_macro" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568" - -[[package]] -name = "memchr" -version = "2.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" - -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - -[[package]] -name = "monostate" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e" -dependencies = [ - "monostate-impl", - "serde", -] - -[[package]] -name = "monostate-impl" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - -[[package]] -name = "once_cell" -version = "1.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" - -[[package]] -name = "onig" -version = "6.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0" -dependencies = [ - "bitflags", - "libc", - "once_cell", - "onig_sys", -] - -[[package]] -name = "onig_sys" -version = "69.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc" -dependencies = [ - "cc", - "pkg-config", -] - -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - -[[package]] -name = "pkg-config" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" - -[[package]] -name = "ppv-lite86" -version = "0.2.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" -dependencies = [ - "zerocopy 0.7.35", -] - -[[package]] -name = "proc-macro2" -version = "1.0.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "rand" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" -dependencies = [ - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" -dependencies = [ - "getrandom", -] - -[[package]] -name = "rayon" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-cond" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" -dependencies = [ - "either", - "itertools", - "rayon", -] - -[[package]] -name = "rayon-core" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - -[[package]] -name = "regex" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" - -[[package]] -name = "rustversion" -version = "1.0.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" - -[[package]] -name = "ryu" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" - -[[package]] -name = "serde" -version = "1.0.216" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.216" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.134" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" -dependencies = [ - "itoa", - "memchr", - "ryu", - "serde", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "smallvec" -version = "1.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" - -[[package]] -name = "spm_precompiled" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" -dependencies = [ - "base64", - "nom", - "serde", - "unicode-segmentation", -] - -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "syn" -version = "2.0.91" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "thiserror" -version = "2.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tokenizers" -version = "0.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476" -dependencies = [ - "ahash", - "aho-corasick", - "compact_str", - "dary_heap", - "derive_builder", - "esaxx-rs", - "getrandom", - "itertools", - "log", - "macro_rules_attribute", - "monostate", - "onig", - "paste", - "rand", - "rayon", - "rayon-cond", - "regex", - "regex-syntax", - "serde", - "serde_json", - "spm_precompiled", - "thiserror", - "unicode-normalization-alignments", - "unicode-segmentation", - "unicode_categories", -] - -[[package]] -name = "unicode-ident" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" - -[[package]] -name = "unicode-normalization-alignments" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" -dependencies = [ - "smallvec", -] - -[[package]] -name = "unicode-segmentation" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" - -[[package]] -name = "unicode_categories" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "wasi" -version = "0.14.2+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" -dependencies = [ - "wit-bindgen-rt", -] - -[[package]] -name = "wit-bindgen-rt" -version = "0.39.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" -dependencies = [ - "bitflags", -] - -[[package]] -name = "zerocopy" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" -dependencies = [ - "byteorder", - "zerocopy-derive 0.7.35", -] - -[[package]] -name = "zerocopy" -version = "0.8.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" -dependencies = [ - "zerocopy-derive 0.8.26", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zml_tokenizer_hftokenizers" -version = "0.1.0" -dependencies = [ - "tokenizers", -] diff --git a/zml2/tokenizer/hftokenizers/Cargo.toml b/zml2/tokenizer/hftokenizers/Cargo.toml deleted file mode 100644 index 8054cd49..00000000 --- a/zml2/tokenizer/hftokenizers/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "zml_tokenizer_hftokenizers" -version = "0.1.0" -edition = "2021" - -[dependencies] -tokenizers = { version = "0.21.4", default-features = false, features = [ - # We don't need/want the progressbar feature - "onig", - "esaxx_fast", -] } - -[lib] -name = "zml_tokenizer_hftokenizers" -path = "hftokenizers.rs" diff --git a/zml2/tokenizer/hftokenizers/hftokenizers.h b/zml2/tokenizer/hftokenizers/hftokenizers.h deleted file mode 100644 index f779b575..00000000 --- a/zml2/tokenizer/hftokenizers/hftokenizers.h +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -#include -#include - -#include "ffi/zig_slice.h" - -typedef struct hftokenizers hftokenizers; - -hftokenizers *hftokenizers_new(zig_slice); -hftokenizers *hftokenizers_new_from_bytes(zig_slice); -void hftokenizers_drop(hftokenizers *tokenizer); -zig_slice hftokenizers_encode(hftokenizers *tokenizer, zig_slice text); -void hftokenizers_tokens_drop(zig_slice tokens); -zig_slice hftokenizers_decode(hftokenizers *tokenizer, zig_slice tokens); -void hftokenizers_str_drop(zig_slice text); -uint32_t hftokenizers_token_to_id(hftokenizers *tokenizer, zig_slice token); diff --git a/zml2/tokenizer/hftokenizers/hftokenizers.rs b/zml2/tokenizer/hftokenizers/hftokenizers.rs deleted file mode 100644 index 048283cc..00000000 --- a/zml2/tokenizer/hftokenizers/hftokenizers.rs +++ /dev/null @@ -1,109 +0,0 @@ -#[repr(C)] -struct ZigSlice { - ptr: *mut T, - len: usize, -} - -impl ZigSlice { - fn as_slice(&self) -> &[T] { - unsafe { std::slice::from_raw_parts(self.ptr, self.len) } - } - - fn as_slice_mut(&self) -> &mut [T] { - unsafe { std::slice::from_raw_parts_mut(self.ptr, self.len) } - } -} - -#[no_mangle] -extern "C" fn hftokenizers_new(path: ZigSlice) -> *mut tokenizers::Tokenizer { - return Box::into_raw(Box::new( - tokenizers::Tokenizer::from_file(std::path::Path::new( - std::str::from_utf8(path.as_slice()).unwrap(), - )) - .unwrap(), - )); -} - -#[no_mangle] -extern "C" fn hftokenizers_new_from_bytes(bytes: ZigSlice) -> *mut tokenizers::Tokenizer { - return Box::into_raw(Box::new( - tokenizers::Tokenizer::from_bytes(bytes.as_slice()) - .unwrap(), - )); -} - -#[no_mangle] -extern "C" fn hftokenizers_drop(t: *mut tokenizers::Tokenizer) { - drop(unsafe { Box::from_raw(t) }); -} - -#[no_mangle] -extern "C" fn hftokenizers_encode( - t: *mut tokenizers::Tokenizer, - string: ZigSlice, -) -> ZigSlice { - let input_str = std::str::from_utf8(string.as_slice()).unwrap(); - let encoded = unsafe { t.as_ref() } - .unwrap() - .encode_fast(input_str, false) - .unwrap(); - - // Convert the result to a boxed slice - let mut ids: Box<[u32]> = encoded.get_ids().to_owned().into_boxed_slice(); - - // Retrieve the zig slice associated to the boxed slice. - let slice = ZigSlice { - ptr: ids.as_mut_ptr(), - len: ids.len(), - }; - - // Leak the box so that it's not deallocated. - Box::leak(ids); - - return slice; -} - -#[no_mangle] -extern "C" fn hftokenizers_tokens_drop(tokens: ZigSlice) { - // Reconstruct the Box from the zig slice so that it's dropped. - drop(unsafe { Box::from_raw(tokens.as_slice_mut()) }); -} - -#[no_mangle] -extern "C" fn hftokenizers_decode( - t: *mut tokenizers::Tokenizer, - ids: ZigSlice, -) -> ZigSlice { - let decoded = unsafe { t.as_ref() } - .unwrap() - .decode(ids.as_slice(), false) - .unwrap(); - - // Convert the result to a boxed slice - let mut string: Box<[u8]> = decoded.into_bytes().into_boxed_slice(); - - // Retrieve the zig slice associated to the boxed slice. - let slice = ZigSlice { - ptr: string.as_mut_ptr(), - len: string.len(), - }; - - // Leak the box so that it's not deallocated. - Box::leak(string); - - return slice; -} - -#[no_mangle] -extern "C" fn hftokenizers_str_drop(tokens: ZigSlice) { - drop(unsafe { Box::from_raw(tokens.as_slice_mut()) }); -} - -#[no_mangle] -extern "C" fn hftokenizers_token_to_id(t: *mut tokenizers::Tokenizer, token: ZigSlice) -> u32 { - let id = unsafe { t.as_ref() } - .unwrap() - .token_to_id(std::str::from_utf8(token.as_slice()).unwrap()) - .unwrap_or(u32::MAX); - return id; -} diff --git a/zml2/tokenizer/hftokenizers/hftokenizers.zig b/zml2/tokenizer/hftokenizers/hftokenizers.zig deleted file mode 100644 index 0871d6f9..00000000 --- a/zml2/tokenizer/hftokenizers/hftokenizers.zig +++ /dev/null @@ -1,120 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const ffi = @import("ffi"); -const stdx = @import("stdx"); - -pub const Encoder = struct { - inner: *HFTokenizer, - current_ids: ?[]const u32 = null, - - fn init(inner: *HFTokenizer) Encoder { - return .{ .inner = inner }; - } - - pub fn reset(self: *Encoder) void { - if (self.current_ids) |current_ids_| { - c.hftokenizers_tokens_drop(ffi.ZigSlice.from(current_ids_)); - self.current_ids = null; - } - } - - pub fn deinit(self: *Encoder) void { - self.reset(); - } - - pub fn encode(self: *Encoder, input: []const u8) ![]const u32 { - self.reset(); - self.current_ids = ffi.ZigSlice.to(u32, c.hftokenizers_encode(@ptrCast(self.inner), ffi.ZigSlice.from(input))); - return self.ids(); - } - - pub fn ids(self: *const Encoder) []const u32 { - return self.current_ids orelse &.{}; - } -}; - -pub const Decoder = struct { - const StringBuffer = stdx.BoundedArray(u8, 128); - const TokensIdsBuffer = stdx.BoundedArray(u32, 4); - - inner: *HFTokenizer, - current_string: ?[]const u8 = null, - last_string: StringBuffer = .{ .len = 0 }, - last_token_ids: TokensIdsBuffer = .{ .len = 0 }, - - fn init(inner: *HFTokenizer) Decoder { - return .{ .inner = inner }; - } - - pub fn deinit(self: *Decoder) void { - self.reset(); - } - - pub fn reset(self: *Decoder) void { - if (self.current_string) |current_string_| { - c.hftokenizers_str_drop(ffi.ZigSlice.from(current_string_)); - self.current_string = null; - } - } - - pub fn decode(self: *Decoder, ids: []const u32) ![]const u8 { - self.reset(); - self.current_string = ffi.ZigSlice.to(u8, c.hftokenizers_decode(@ptrCast(self.inner), ffi.ZigSlice.from(ids))); - return self.string(); - } - - pub fn string(self: *const Decoder) []const u8 { - return self.current_string orelse &.{}; - } - - pub fn next(self: *Decoder, token_id: u32) !?[]const u8 { - if (self.last_token_ids.len >= self.last_token_ids.capacity()) { - _ = self.last_token_ids.orderedRemove(0); - } - self.last_token_ids.appendAssumeCapacity(token_id); - const new_string = try self.decode(self.last_token_ids.constSlice()); - if (self.last_string.len == 0) { - self.last_string = try StringBuffer.fromSlice(new_string); - return new_string; - } - var view = try std.unicode.Utf8View.init(self.last_string.constSlice()); - var it = view.iterator(); - while (it.nextCodepointSlice()) |cp| { - const start = it.i - cp.len; - if (std.mem.startsWith(u8, new_string, self.last_string.constSlice()[start..])) { - const chunk = new_string[self.last_string.len - start ..]; - self.last_string = try StringBuffer.fromSlice(new_string); - return chunk; - } - } - return null; - } -}; - -pub const HFTokenizer = opaque { - pub fn fromFile(model: []const u8) !*HFTokenizer { - return @ptrCast(c.hftokenizers_new(ffi.ZigSlice.from(model))); - } - - pub fn fromBytes(bytes: []const u8) !*HFTokenizer { - return @ptrCast(c.hftokenizers_new_from_bytes(ffi.ZigSlice.from(bytes))); - } - - pub fn deinit(self: *HFTokenizer) void { - return c.hftokenizers_drop(@ptrCast(self)); - } - - pub fn encoder(self: *HFTokenizer) !Encoder { - return Encoder.init(self); - } - - pub fn decoder(self: *HFTokenizer) !Decoder { - return Decoder.init(self); - } - - pub fn tokenToId(self: *HFTokenizer, token: []const u8) ?u32 { - const id = c.hftokenizers_token_to_id(@ptrCast(self), ffi.ZigSlice.from(token)); - return if (id == std.math.maxInt(u32)) null else id; - } -}; diff --git a/zml2/tokenizer/homemade.zig b/zml2/tokenizer/homemade.zig deleted file mode 100644 index bd16d183..00000000 --- a/zml2/tokenizer/homemade.zig +++ /dev/null @@ -1,1242 +0,0 @@ -//! Text tokenizer implementations -//! Disclaimer this is not a very robust implementation: -//! In particular the normalization is pretty minimalist, only works with ascii, and don't do unicode normalization. -//! Mostly used for testing models that don't have an official HF/sentencepiece tokenizer. -const std = @import("std"); -const testing = std.testing; -const builtin = @import("builtin"); - -const stdx = @import("stdx"); - -const log = std.log.scoped(.@"zml/tokenizer"); - -/// Byte Pair Encoding tokenizer generally used for LLM. -pub const Tokenizer = struct { - tokens: [][]const u8, - token_lookup: std.StringHashMapUnmanaged(u32), - special_tokens: SpecialTokens, - - scores: []f32, - max_token_len: u32, - normalizer: ?Normalizer, - // Allows to split unknown unicode characters into bytes. - byte_fallback: bool = false, - - arena_state: std.heap.ArenaAllocator, - vocab_size: u32, - next_token_id: u32 = 0, - - pub const SpecialTokens = struct { - eos: u32, - bos: u32, - unk: u32, - pad: u32 = std.math.maxInt(u32), - hard_space: u32 = std.math.maxInt(u32), - }; - - pub fn init( - allocator: std.mem.Allocator, - vocab_size: u32, - max_token_len: u32, - normalizer: ?Normalizer, - special_tokens: SpecialTokens, - alloc_tokens: bool, - ) !Tokenizer { - var arena_state = std.heap.ArenaAllocator.init(allocator); - errdefer arena_state.deinit(); - const arena = arena_state.allocator(); - - var token_lookup: std.StringHashMapUnmanaged(u32) = .{}; - errdefer token_lookup.deinit(arena); - - try token_lookup.ensureTotalCapacity(arena, @intCast(vocab_size)); - - const tokens: [][]const u8 = if (alloc_tokens) try arena.alloc([]const u8, vocab_size) else &.{}; - errdefer if (alloc_tokens) arena.free(tokens); - - const scores: []f32 = if (alloc_tokens) try arena.alloc(f32, vocab_size) else &.{}; - errdefer if (alloc_tokens) arena.free(scores); - - return .{ - .tokens = tokens, - .scores = scores, - .max_token_len = max_token_len, - .token_lookup = token_lookup, - .arena_state = arena_state, - .normalizer = normalizer, - .vocab_size = vocab_size, - .special_tokens = special_tokens, - }; - } - - pub fn deinit(self: Tokenizer) void { - self.arena_state.deinit(); - } - - pub fn encoder(self: *Tokenizer) !Encoder { - return Encoder.init(self); - } - - pub fn decoder(self: *Tokenizer) !Decoder { - return Decoder.init(self); - } - - /// Reads a new word directly into the tokenizer arena. - pub fn readTokenInto(self: *Tokenizer, score: f32, len: usize, tok_reader: *std.Io.Reader) !void { - const arena = self.arena_state.allocator(); - - const token = try arena.alloc(u8, len); - try tok_reader.readSliceAll(token); - return self.addOwnedToken(score, token); - } - - /// Adds a new token (and copy it) - pub fn addToken(self: *Tokenizer, score: f32, token: []const u8) !void { - const arena = self.arena_state.allocator(); - return self.addOwnedToken(score, try arena.dupe(u8, token)); - } - - /// Adds a new token (without copying it) - pub fn addOwnedToken(self: *Tokenizer, score: f32, token: []const u8) void { - const i = self.next_token_id; - std.debug.assert(i < self.vocab_size); - self.next_token_id += 1; - - self.scores[i] = score; - self.tokens[i] = token; - const v = self.token_lookup.getOrPutAssumeCapacity(token); - if (!v.found_existing) { - v.value_ptr.* = i; - } - } - - pub fn addOwnedTokenByIndex(self: *Tokenizer, i: u32, score: f32, token: []const u8) void { - std.debug.assert(i < self.vocab_size); - self.next_token_id += 1; - self.scores[i] = score; - self.tokens[i] = token; - const v = self.token_lookup.getOrPutAssumeCapacity(token); - if (!v.found_existing) { - v.value_ptr.* = @intCast(i); - } - } - - pub fn lookup(self: *const Tokenizer, str: []const u8) ?u32 { - return self.token_lookup.get(str); - } - - pub fn tokenToId(self: *const Tokenizer, token: []const u8) ?u32 { - return self.token_lookup.get(token); - } - - pub const EncodeOptions = struct { - /// Should the beginning of sentence '' token be added. - add_bos: bool = true, - add_eos: bool = false, - pad_to: u32 = 0, - // Print tokenization intermediary steps. - debug: bool = false, - }; - - pub fn encode(self: *const Tokenizer, allocator: std.mem.Allocator, raw: []const u8, options: EncodeOptions) ![]u32 { - if (options.debug) log.debug("Tokenizer.encode('{s}')", .{raw}); - const input = if (self.normalizer) |n| try n.normalize(allocator, raw) else raw; - defer if (self.normalizer) |_| allocator.free(input); - if (options.debug) log.debug("Tokenizer.encode.normalize -> '{s}'", .{input}); - - // Allocate a buffer that can fit all indices as well as extra character if requested. - // We then slice it so that the token merging code doesn't see the bos token. - const tok_buff_alloc = try allocator.alloc(u32, @max(options.pad_to, input.len + 2)); - const tok_buff = if (options.add_bos) tok_buff_alloc[1..] else tok_buff_alloc; - - const MergeState = union(enum) { ready: u32, nope, hard_space, idk }; - const mergeable = try allocator.alloc(MergeState, tok_buff.len); - - var num_tokens: usize = 0; - var it: CharTokenIterator = .{ .input = input }; - while (try it.nextCodepointToken(self)) |token| : (num_tokens += 1) { - tok_buff[num_tokens] = token; - mergeable[num_tokens] = if (token == self.special_tokens.hard_space) - .hard_space - else - .idk; - } - - var stable_prefix: usize = 0; - var stable_off: usize = 0; - while (true) { - // This code is a bit overcomplicated cause I'm abstracting over two algorithms: - // BPE and sentencepiece unigram model. - // Normally BPE is pre-split on spaces then the regular merge algorithm is applied. - // With unigram model you work at sentence level and you handle spaces as you would any other bytes, - // hoping the final tokens mostly align with spaces. - // This seemed like a good idea, but is kinda bad because I had to add special code to speed up BPE - // by detecting when the first "word" is treated and can be safely removed from sequence. - // Also it doesn't work well with BPE vocab which have multi-space tokens (for indentation) - // and have custom splitting rules. - // This is fine for now cause we now have bindings to HF tokenizers for complexe use cases - // and are only using this for tinyllama/gguf models. - // If we come back to use this in production, the implementation would gain in speed/clarity - // by splitting in two. - // The merging token logic isn't that complicated anyway. - - // Step by step visualization of the progress. - if (options.debug) { - var _debug_buf: [256]u8 = undefined; - var debug_progress: std.Io.Writer = .fixed(&_debug_buf); - self.decodeWithOpts(tok_buff[0..num_tokens], &debug_progress, .{ .sep = "|" }) catch {}; - log.debug("tokens: {any} -> {s}", .{ tok_buff[0..num_tokens], debug_progress.buffered() }); - } - var best_score: f32 = -1e10; - var best_token: u32 = 0; - var best_idx: ?usize = null; - var input_off: usize = stable_off; - - // Find best tokens to merge in all available tokens - for (stable_prefix..num_tokens - 1) |i| { - if (tok_buff[i] == self.special_tokens.unk) { - input_off += 1; - continue; - } - const cur_tok = self.tokens[tok_buff[i]]; - defer input_off += cur_tok.len; - - // Lookup merge for current token, if not already done. - switch (mergeable[i]) { - .nope => continue, - .ready => {}, - .hard_space => { - // Since tokens are not allowed to merge through hard sep, - // we don't need to merge the sentence-wide best token. - // We can just merge the best token since beginning. - if (best_idx != null) break; - // OTOH if there was no merge possible since beginning, - // we can skip the beginning in future iterations. - stable_prefix = i + 1; - stable_off = input_off + cur_tok.len; - continue; - }, - .idk => { - - // Special tokens can't be concatenated. - if (builtin.mode == .Debug and tok_buff[i] != self.special_tokens.unk) { - // Detects memory corruption of tokens. - if (cur_tok.len == 0 or cur_tok.len > self.max_token_len) @panic("Token looks corrupted !"); - - if (!std.mem.eql(u8, cur_tok, input[input_off..][0..cur_tok.len])) { - log.err("current token '{s}' not found in input string '{s}' !", .{ cur_tok, input[input_off..] }); - @panic("invalid tokenization"); - } - } - const next_tok = self.tokens[tok_buff[i + 1]]; - // if `next_tok` is `.unk`, length is 1; otherwise, it's the length of the token. - const next_tok_len = if (tok_buff[i + 1] == self.special_tokens.unk) 1 else next_tok.len; - const concat_tokens = input[input_off..][0 .. cur_tok.len + next_tok_len]; - // Save the result - mergeable[i] = if (self.lookup(concat_tokens)) |tok| - .{ .ready = tok } - else - .nope; - }, - } - - switch (mergeable[i]) { - .idk, .hard_space => unreachable, - .nope => continue, - .ready => |tok| { - if (self.scores[tok] > best_score) { - best_score = self.scores[tok]; - best_token = tok; - best_idx = i; - } - }, - } - } - - if (best_idx) |bidx| { - // Apply the merge. - tok_buff[bidx] = best_token; - std.mem.copyForwards(u32, tok_buff[bidx + 1 ..], tok_buff[bidx + 2 .. num_tokens]); - std.mem.copyForwards(MergeState, mergeable[bidx + 1 ..], mergeable[bidx + 2 .. num_tokens]); - num_tokens -= 1; - // We got two new merge lookups to do. - mergeable[bidx] = .idk; - if (bidx > 0 and mergeable[bidx - 1] != .hard_space) mergeable[bidx - 1] = .idk; - } else { - // No merge candidate => we are done ! - break; - } - } - - if (options.add_eos) { - tok_buff[num_tokens] = self.special_tokens.eos; - num_tokens += 1; - } - if (options.add_bos) { - tok_buff_alloc[0] = self.special_tokens.bos; - num_tokens += 1; - } - if (num_tokens < options.pad_to) { - for (num_tokens..options.pad_to) |i| { - tok_buff_alloc[i] = self.special_tokens.pad; - } - num_tokens = options.pad_to; - } - - // Release extra memory we don't need anymore. - allocator.free(mergeable); - _ = allocator.resize(tok_buff_alloc, num_tokens); - return tok_buff_alloc[0..num_tokens]; - } - - /// Returns a slice corresponding to the given id. Handles unknown ids and special ids. - pub fn lookupPiece(self: *const Tokenizer, id: usize) []const u8 { - return if (id == self.special_tokens.bos or id == self.special_tokens.eos or id == self.special_tokens.pad) - "" - else if (id == self.special_tokens.unk) - "" - else if (id > self.tokens.len) - "" // this means we received an invalid id, but we didn't want to panic. - else - self.tokens[id]; - } - - /// Converts the given slice of tokens back into bytes. - /// Note that if the tokenizer allows sub-unicode bytes, it's possible - /// the output is not valid utf8. - pub fn decode(self: *const Tokenizer, input: []const u32, writer: *std.Io.Writer) std.Io.Writer.Error!void { - try self.decodeWithOpts(input, writer, .{}); - } - - pub fn decodeWithOpts( - self: *const Tokenizer, - input: []const u32, - output: *std.Io.Writer, - opts: struct { sep: []const u8 = "" }, - ) std.Io.Writer.Error!void { - const escaped = if (self.normalizer) |n| n.escapedSpace() else null; - // Flag used to indicate if the first dummy whitespace has been consumed. - var first_token: bool = true; - for (input) |id| { - // Retrieve the slice corresponding to the id. - var piece = self.lookupPiece(id); - - // Convert `▁` to a regular space. - if (escaped) |escspc| { - // we modify piece inside the loop, so we can use it in the condition - while (std.mem.startsWith(u8, piece, escspc)) { - piece = piece[escspc.len..]; - // don't output a space at beginning of text. - if (!first_token) try output.writeByte(' '); - } - } - - try output.writeAll(piece); - first_token = false; - try output.writeAll(opts.sep); - } - } - - /// Some tokenizers have bytes encoded in hex like this: "<0x40>". - /// This break the tokenization algorithm because the input text - /// will contain "@" not "<0x40>", - /// and if the input contains "<0x40>" it needs to not be treated as a single byte. - /// So we replace byte fallbacks strings, by their corresponding character. - /// This enables the normal tokenization algorithm to work. - pub fn rewriteByteFallbackTokens(tokenizer: *Tokenizer) !void { - tokenizer.byte_fallback = true; - var single_bytes = try tokenizer.arena_state.allocator().alloc(u8, 256); - var byte_fallback_buf = "<0x00>".*; - - for (0..256) |i| { - const c: u8 = @truncate(i); - single_bytes[i] = c; - - // First lookup the byte fallback entry. - // Note: we assume upper case, but we could try both upper and lower case if needed. - var writer: std.Io.Writer = .fixed(byte_fallback_buf[3..5]); - try writer.printInt(c, 16, .upper, .{ .fill = '0', .width = 2 }); - const entry = tokenizer.token_lookup.getEntry(&byte_fallback_buf) orelse { - log.err("Tokenizer has \"byte_fallback\" = true, but doesn't contains the byte fallback token {s}", .{byte_fallback_buf}); - return error.InvalidInput; - }; - - // Check if the character is already present in the vocab. - // In that case, nothing to do, - // but note that the fallback token will be "unreachable", - // ie there is no way the tokenizer can produce it. - if (tokenizer.token_lookup.get(&.{c})) |_| continue; - - const idx: u32 = entry.value_ptr.*; - tokenizer.token_lookup.removeByPtr(entry.key_ptr); - tokenizer.addOwnedTokenByIndex(idx, tokenizer.scores[idx], single_bytes[i .. i + 1]); - } - } -}; - -test Tokenizer { - const allocator = std.testing.allocator; - const special_tokens: Tokenizer.SpecialTokens = .{ - .unk = 0, - .bos = 1, - .eos = 2, - }; - - var tokenizer = try Tokenizer.init(allocator, 10, 5, null, special_tokens, true); - defer tokenizer.deinit(); - - try tokenizer.addToken(10, "hello"); - try tokenizer.addToken(3.5, "world"); - - try testing.expect(tokenizer.lookup("hello") == 0); - try testing.expect(tokenizer.lookup("world") == 1); - - // TODO: test Tokenizer.decode, Tokenizer.encode, Tokenizer.readTokenInto -} - -pub const Encoder = struct { - inner: *Tokenizer, - arena: std.heap.ArenaAllocator, - current_ids: []const u32 = &.{}, - - fn init(inner: *Tokenizer) !Encoder { - var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); - // Warmup the arena. Page allocator is expensive, avoid calling it for small reallocs. - _ = try arena.allocator().alloc(u32, 4096); - std.debug.assert(arena.reset(.retain_capacity)); - return .{ .inner = inner, .arena = arena }; - } - - pub fn reset(self: *Encoder) void { - self.current_ids = &.{}; - std.debug.assert(self.arena.reset(.retain_capacity)); - } - - pub fn deinit(self: *Encoder) void { - self.arena.deinit(); - } - - pub fn encode(self: *Encoder, input: []const u8) ![]const u32 { - self.reset(); - const res = try self.inner.encode(self.arena.allocator(), input, .{ - .add_bos = true, - .add_eos = false, - .pad_to = 0, - // Print tokenization intermediary steps. - .debug = false, - }); - self.current_ids = res; - return res; - } - - pub fn ids(self: *const Encoder) []const u32 { - return self.current_ids; - } -}; - -pub const Decoder = struct { - const StringBuffer = stdx.BoundedArray(u8, 128); - const TokensIdsBuffer = stdx.BoundedArray(u32, 4); - - inner: *Tokenizer, - arena: std.heap.ArenaAllocator, - - current_string: ?[]const u8 = null, - last_string: StringBuffer = .{ .len = 0 }, - last_token_ids: TokensIdsBuffer = .{ .len = 0 }, - - fn init(inner: *Tokenizer) !Decoder { - var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); - // Warmup the arena. Page allocator is expensive, avoid calling it for small reallocs. - _ = try arena.allocator().alloc(u32, 4096); - std.debug.assert(arena.reset(.retain_capacity)); - return .{ .inner = inner, .arena = arena }; - } - - pub fn deinit(self: *Decoder) void { - self.arena.deinit(); - } - - pub fn reset(self: *Decoder) void { - std.debug.assert(self.arena.reset(.retain_capacity)); - self.current_string = null; - } - - pub fn decode(self: *Decoder, ids: []const u32) ![]const u8 { - // TODO: revisite tokenizer api to use std.Io.Writer. - self.reset(); - // Reuse the same warmup than in init, to maximize contiguous writes. - var arena_writer = std.Io.Writer.Allocating.initCapacity(self.arena.allocator(), 4096) catch unreachable; - try self.inner.decode(ids, &arena_writer.writer); - self.current_string = arena_writer.written(); - return self.current_string.?; - } - - pub fn string(self: *const Decoder) []const u8 { - return self.current_string orelse ""; - } - - pub fn next(self: *Decoder, token_id: u32) !?[]const u8 { - if (self.last_token_ids.len >= self.last_token_ids.capacity()) { - _ = self.last_token_ids.orderedRemove(0); - } - self.last_token_ids.appendAssumeCapacity(token_id); - const new_string = try self.decode(self.last_token_ids.constSlice()); - if (self.last_string.len == 0) { - self.last_string = try StringBuffer.fromSlice(new_string); - return new_string; - } - var view = try std.unicode.Utf8View.init(self.last_string.constSlice()); - var it = view.iterator(); - while (it.nextCodepointSlice()) |cp| { - const start = it.i - cp.len; - if (std.mem.startsWith(u8, new_string, self.last_string.constSlice()[start..])) { - const chunk = new_string[self.last_string.len - start ..]; - self.last_string = try StringBuffer.fromSlice(new_string); - return chunk; - } - } - return null; - } -}; - -/// Given a slice, split it in the most simple tokens using the given tokenizer tokens. -/// The output of this can be used to initialize the tokenization algorithm. -/// Normally we split the input text into utf8 codepoint, -/// but if we find an unknown codepoint we either split it in bytes, or use the special "unknown" token, -/// depending on the tokenizer configuration. -const CharTokenIterator = struct { - state: union(enum) { by_codepoint, by_byte: u8 } = .by_codepoint, - input: []const u8, - - fn nextCodepointToken(self: *CharTokenIterator, tokenizer: *const Tokenizer) error{ TruncatedInput, Utf8InvalidStartByte }!?u32 { - if (self.input.len == 0) return null; - return switch (self.state) { - .by_byte => |*byte_left| { - const idx = tokenizer.lookup(self.input[0..1]) orelse { - // Normally this has been caught when calling `rewriteByteFallbackTokens`. - std.debug.panic("Tokenizer has \"byte_fallback\" = true, but doesn't contains the byte fallback for token '<0x{X:02}>'", .{self.input[0]}); - }; - - self.input = self.input[1..]; - byte_left.* -|= 1; - if (byte_left.* == 0) self.state = .by_codepoint; - return idx; - }, - .by_codepoint => { - // Try to lookup valid utf8 codepoint first. - const utf8_len = try std.unicode.utf8ByteSequenceLength(self.input[0]); - if (self.input.len < utf8_len) return error.TruncatedInput; - if (tokenizer.lookup(self.input[0..utf8_len])) |idx| { - self.input = self.input[utf8_len..]; - return idx; - } - - // Otherwise split in bytes if it's allowed. - if (tokenizer.byte_fallback) { - // TODO: replace this by a continue statement next time we bump Zig. - self.state = .{ .by_byte = utf8_len }; - return self.nextCodepointToken(tokenizer); - } - - // Or mark the full utf8 codepoint as unknown. - log.debug("Token not found for char '{s}'", .{self.input[0..utf8_len]}); - self.input = self.input[utf8_len..]; - return tokenizer.special_tokens.unk; - }, - }; - } -}; - -test CharTokenIterator { - const special_tokens: Tokenizer.SpecialTokens = .{ .unk = 0, .bos = 1, .eos = 2 }; - var tokenizer = try Tokenizer.init(std.testing.allocator, 16, 4, null, special_tokens, true); - defer tokenizer.deinit(); - - tokenizer.addOwnedToken(1.0, ""); // 0 - tokenizer.addOwnedToken(1.0, ""); // 1 - tokenizer.addOwnedToken(1.0, ""); // 2 - tokenizer.addOwnedToken(1.0, "ζ"); // 3 - tokenizer.addOwnedToken(1.0, &.{0xE2}); // 4: ℳ, first byte - tokenizer.addOwnedToken(1.0, &.{0x84}); // 5: ℳ, second byte - tokenizer.addOwnedToken(1.0, &.{0xB3}); // 6: ℳ, third byte - tokenizer.addOwnedToken(1.0, "L"); // 7 - - // No byte fallback - { - tokenizer.byte_fallback = false; - var it: CharTokenIterator = .{ .input = "ζℳL" }; - var res: stdx.BoundedArray(u32, 8) = .{}; - while (try it.nextCodepointToken(&tokenizer)) |token| { - res.appendAssumeCapacity(token); - } - try std.testing.expectEqualSlices(u32, &[_]u32{ 3, 0, 7 }, res.constSlice()); - } - - // with byte fallback - { - tokenizer.byte_fallback = true; - var it: CharTokenIterator = .{ .input = "ζℳL" }; - var res: stdx.BoundedArray(u32, 8) = .{}; - while (try it.nextCodepointToken(&tokenizer)) |token| { - res.appendAssumeCapacity(token); - } - try std.testing.expectEqualSlices(u32, &[_]u32{ 3, 4, 5, 6, 7 }, res.constSlice()); - } -} - -/// Text normalizer. -/// Most tokenizer assumes the input text have been prepocessed with on of those. -pub const Normalizer = struct { - /// Space token used by sentencepiece derived tokenizer. - pub const sentencepiece_space = "▁"; // \xe2\x96\x81 - - _whitespace: stdx.BoundedArray(u8, 8) = .{}, - - flags: packed struct { - remove_extra_whitespaces: bool, - add_dummy_prefix: bool, - add_dummy_suffix: bool, - /// Cheap lower casing. - /// TODO: try to match Python "lower" - lower_case_ascii: bool, - /// cheap ascii punct splitting. - // doing this processing ahead of time simplifies the logic - split_on_punct_ascii: bool, - }, - - pub fn init(flags: @FieldType(Normalizer, "flags"), escaped_whitespace: ?[]const u8) Normalizer { - var res: Normalizer = .{ .flags = flags }; - if (escaped_whitespace) |escaped| { - res._whitespace.appendSliceAssumeCapacity(escaped); - } - return res; - } - - pub inline fn escapedSpace(self: Normalizer) ?[]const u8 { - return if (self._whitespace.len > 1) self._whitespace.constSlice() else null; - } - - fn addSlice(allocator: std.mem.Allocator, data: []const u8, consumed: usize, normalized: *std.ArrayList(u8), normalized_to_origin: *std.ArrayList(usize)) !void { - try normalized.appendSlice(allocator, data); - for (data) |_| try normalized_to_origin.append(allocator, consumed); - } - - pub const Result = struct { - /// Normalized string - normalized: []const u8, - /// Mapping between chars in the original string and chars in the new string - normalized_to_origin: []const usize, - - pub fn deinit(self: Result, allocator: std.mem.Allocator) void { - allocator.free(self.normalized); - allocator.free(self.normalized_to_origin); - } - }; - - /// Simplifed version of Sentencepiece normalizer. - /// - /// Llama2 uses a normalizer called "identity" so this basically only handles trailing - /// whitespaces and replaces whitespace with the "▁" (U+2581) character. - pub fn normalize(self: Normalizer, allocator: std.mem.Allocator, input: []const u8) ![]const u8 { - const res = try self.normalizeWithMapping(allocator, input); - allocator.free(res.normalized_to_origin); - return res.normalized; - } - - /// Returns both the normalized string and a mapping between the normalized string and the original. - pub fn normalizeWithMapping(self: Normalizer, allocator: std.mem.Allocator, input: []const u8) !Result { - // Number of bytes consumed from the input. - var consumed: usize = 0; - var trimmed_input = input; - - // Skip leading whitespaces. - if (self.flags.remove_extra_whitespaces) { - while (trimmed_input.len != 0) { - if (trimmed_input[0] != ' ') break; - trimmed_input = trimmed_input[1..]; - consumed += 1; - } - } - - // If the trimmed input is empty, we are done. - if (trimmed_input.len == 0) { - return .{ .normalized = &.{}, .normalized_to_origin = &.{} }; - } - - // Pre-allocate outputs - const space = self.escapedSpace() orelse " "; - const overhead = if (self.flags.split_on_punct_ascii) space.len + 1 else space.len; - var normalized: std.ArrayList(u8) = try .initCapacity(allocator, trimmed_input.len * overhead + 2 * space.len); - errdefer normalized.deinit(allocator); - var normalized_to_origin: std.ArrayList(usize) = try .initCapacity(allocator, normalized.capacity); - errdefer normalized_to_origin.deinit(allocator); - - // If the spec asks for it, add a whitespace at the beginning. - if (self.flags.add_dummy_prefix) try addSlice(allocator, space, consumed, &normalized, &normalized_to_origin); - - var is_prev_space: bool = true; - var is_prev_word: bool = false; - - while (trimmed_input.len != 0) { - // NOTE(Corendos): This might feel weird but normally the slice we get comes from a normalizing process and can contain multiple codepoints. - // Since we have an "identity" normalizer, each slice is actually a unicode character. - const multibyte_length = try std.unicode.utf8ByteSequenceLength(trimmed_input[0]); - var slice = trimmed_input[0..multibyte_length]; - const origin = consumed; - consumed += multibyte_length; - trimmed_input = trimmed_input[multibyte_length..]; - - if (self.flags.remove_extra_whitespaces and is_prev_space) { - while (slice.len > 0 and slice[0] == ' ') { - slice = slice[1..]; - } - if (slice.len == 0) continue; - } - is_prev_space = slice[slice.len - 1] == ' '; - - if (slice.len == 1) ascii: { - // The more advanced logic only works with ascii atm - var byte = slice[0]; - if (self.escapedSpace() != null and byte == ' ') { - // replace the space token by the special token - try addSlice(allocator, space, origin, &normalized, &normalized_to_origin); - is_prev_word = false; - break :ascii; - } else if (self.flags.split_on_punct_ascii) { - if (is_prev_word and isPunct(slice)) { - // Insert a space, but continue handling the rest - try addSlice(allocator, space, origin, &normalized, &normalized_to_origin); - } - } - if (self.flags.lower_case_ascii) { - byte = std.ascii.toLower(byte); - } - try normalized.append(allocator, byte); - try normalized_to_origin.append(allocator, origin); - } else { - // we can safely copy to the output. - try addSlice(allocator, slice, origin, &normalized, &normalized_to_origin); - } - is_prev_word = !is_prev_space and !isPunct(slice); - } - - // Skip trailing whitespaces - if (self.flags.remove_extra_whitespaces) { - while (std.mem.endsWith(u8, normalized.items, space)) { - const length = normalized.items.len - space.len; - consumed = normalized_to_origin.items[length]; - try normalized.resize(allocator, length); - try normalized_to_origin.resize(allocator, length); - } - } - - try normalized_to_origin.append(allocator, consumed); - - std.debug.assert(normalized_to_origin.items.len == normalized.items.len + 1); - - if (self.flags.add_dummy_suffix) try addSlice(allocator, space, consumed, &normalized, &normalized_to_origin); - - return .{ - .normalized_to_origin = try normalized_to_origin.toOwnedSlice(allocator), - .normalized = try normalized.toOwnedSlice(allocator), - }; - } - - pub fn wellKnown(impl: KnownImplementation) Normalizer { - return switch (impl) { - .sentencepiece => init(.{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = true, - .add_dummy_suffix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - }, sentencepiece_space), - .llama3 => init(.{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = false, - .add_dummy_suffix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - }, null), - .gpt2 => init(.{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = true, - .add_dummy_suffix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - }, null), - }; - } - - pub fn fromHfJson(config: std.json.ObjectMap) error{InvalidNormalizerJson}!Normalizer { - var normalizer: Normalizer = .{ .flags = .{ - .remove_extra_whitespaces = false, - .add_dummy_suffix = false, - .add_dummy_prefix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - } }; - - // Normalizer config can be a single normalizer, or a sequence of normalizers. - const maybe_steps = objectGet(config, .array, "normalizers"); - const steps = if (maybe_steps) |st| st.items else &.{std.json.Value{ .object = config }}; - - for (steps) |step_val| { - if (step_val != .object) { - return error.InvalidNormalizerJson; - } - const step = step_val.object; - - const step_type = objectGet(step, .string, "type") orelse { - return error.InvalidNormalizerJson; - }; - if (std.mem.eql(u8, "Prepend", step_type)) { - normalizer.flags.add_dummy_prefix = true; - } else if (std.mem.eql(u8, "Append", step_type)) { - normalizer.flags.add_dummy_suffix = true; - } else if (std.mem.eql(u8, "Replace", step_type)) { - const pattern = objectGet(step, .object, "pattern") orelse return error.InvalidNormalizerJson; - const str_pattern = objectGet(pattern, .string, "String") orelse return error.InvalidNormalizerJson; - - if (std.mem.eql(u8, str_pattern, " ")) { - normalizer._whitespace.appendSliceAssumeCapacity( - objectGet(step, .string, "content") orelse return error.InvalidNormalizerJson, - ); - } else { - log.warn("Normalizer Replace pattern not supported: '{s}' -> '{s}'", .{ str_pattern, objectGet(pattern, .string, "content") orelse "" }); - } - } else { - log.warn("Unknown normalizer type: {s}", .{step_type}); - } - } - - return normalizer; - } - - test "Normalizer.fromHfJson" { - const config_json = - \\{ - \\ "type": "Sequence", - \\ "normalizers": [ - \\ { - \\ "type": "Prepend", - \\ "prepend": "▁" - \\ }, - \\ { - \\ "type": "Replace", - \\ "pattern": { - \\ "String": " " - \\ }, - \\ "content": "▁" - \\ } - \\ ] - \\} - ; - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const config = try std.json.parseFromSliceLeaky(std.json.Value, arena.allocator(), config_json, .{}); - const normalizer = try Normalizer.fromHfJson(config.object); - - const expected = Normalizer{ - ._whitespace = .{ .buffer = [_]u8{ 0xe2, 0x96, 0x81 } ++ [_]u8{0} ** 5, .len = 3 }, - .flags = .{ - .remove_extra_whitespaces = false, - .add_dummy_prefix = true, - .add_dummy_suffix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - }, - }; - try std.testing.expectEqual(expected.flags, normalizer.flags); - try std.testing.expectEqualStrings(expected.escapedSpace().?, normalizer.escapedSpace().?); - } -}; -pub const KnownImplementation = enum(u8) { - sentencepiece, - gpt2, - llama3, -}; - -fn isPunct(unicode_char: []const u8) bool { - // TODO use unicode categories - if (unicode_char.len > 1) return false; - - return switch (unicode_char[0]) { - ' ', '\t' => false, - 0...8 => true, - 10...31 => true, - '!'...'/' => true, - ':'...'@' => true, - '['...'`' => true, - '{'...'~' => true, - else => false, - }; -} - -test Normalizer { - { - const n: Normalizer = .{ .flags = .{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = true, - .add_dummy_suffix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - } }; - const res = try n.normalizeWithMapping(testing.allocator, "Hellŏ world!"); - defer res.deinit(testing.allocator); - - try testing.expectEqualSlices(u8, " Hellŏ world!", res.normalized); - try testing.expectEqualSlices( - usize, - // H e l l ŏ ␣ w o r l d ! - &.{ 0, 0, 1, 2, 3, 4, 4, 6, 8, 9, 10, 11, 12, 13, 14 }, - res.normalized_to_origin, - ); - } - - { - const n: Normalizer = .{ .flags = .{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = true, - .add_dummy_suffix = true, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - } }; - const res = try n.normalize(testing.allocator, "Hello world!"); - defer testing.allocator.free(res); - - try testing.expectEqualSlices(u8, " Hello world! ", res); - } - - { - const n = Normalizer.init( - .{ - .remove_extra_whitespaces = false, - .add_dummy_prefix = true, - .add_dummy_suffix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - }, - Normalizer.sentencepiece_space, - ); - const res = try n.normalize(testing.allocator, "Hello world!"); - defer testing.allocator.free(res); - - try testing.expectEqualSlices(u8, "▁Hello▁▁world!", res); - } - - { - const n: Normalizer = .{ .flags = .{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = false, - .add_dummy_suffix = true, - .lower_case_ascii = true, - .split_on_punct_ascii = false, - } }; - const res = try n.normalize(testing.allocator, "Hello world!"); - defer testing.allocator.free(res); - - try testing.expectEqualSlices(u8, "hello world! ", res); - } - - { - const n: Normalizer = .{ .flags = .{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = false, - .add_dummy_suffix = true, - .lower_case_ascii = false, - .split_on_punct_ascii = true, - } }; - const res = try n.normalize(testing.allocator, "Hello world!"); - defer testing.allocator.free(res); - - try testing.expectEqualSlices(u8, "Hello world ! ", res); - } -} - -/// gpt2 had their own way of storing text. -/// Unfortunately this has contaminated other models. -/// This implementation precompupte a mapping between bytes encoded with GPT2 algorithm, -/// into utf8 bytes, and do lookups at runtime. -pub const Gpt2TextDecoder = struct { - const Code = stdx.BoundedArray(u8, 2); - - // TODO: benchmark this is more efficient than doing the conversion at runtime. - code_to_byte: std.AutoArrayHashMap(Code, u8), - - pub fn init(allocator: std.mem.Allocator) !Gpt2TextDecoder { - var self = Gpt2TextDecoder{ - .code_to_byte = std.AutoArrayHashMap(Code, u8).init(allocator), - }; - try self.code_to_byte.ensureTotalCapacity(256); - errdefer unreachable; - - var n: usize = 0; - for (0..256) |index| { - var code: Code = .{ .buffer = .{ 0, 0 }, .len = 0 }; // 0-init - const i: u8 = @intCast(index); - if (isPrintableByte(i)) { - if (std.ascii.isAscii(i)) { - code.appendAssumeCapacity(i); - } else { - const codepoint: u21 = @as(u21, @intCast(i)); - code.len = @intCast(std.unicode.utf8Encode(codepoint, &code.buffer) catch unreachable); - } - } else { - const codepoint: u21 = 256 + @as(u21, @intCast(n)); - code.len = @intCast(std.unicode.utf8Encode(codepoint, &code.buffer) catch unreachable); - n += 1; - } - - self.code_to_byte.putAssumeCapacityNoClobber(code, i); - } - return self; - } - - pub fn deinit(self: *Gpt2TextDecoder) void { - self.code_to_byte.deinit(); - } - - /// Transform bytes representing text under the gpt2 encoding, - /// and write to the `unicode` buffer utf-8 bytes. - pub fn decode(self: Gpt2TextDecoder, bytes: []const u8, writer: *std.Io.Writer) (error{InvalidInput} || std.Io.Writer.Error)!void { - var it = std.unicode.Utf8Iterator{ .i = 0, .bytes = bytes }; - while (it.nextCodepointSlice()) |codepoint| { - const code: Code = switch (codepoint.len) { - 1 => .{ .buffer = .{ codepoint[0], 0 }, .len = 1 }, // 0-init - 2 => .{ .buffer = .{ codepoint[0], codepoint[1] }, .len = 2 }, - else => return error.InvalidInput, - }; - const byte = self.code_to_byte.get(code) orelse return error.InvalidInput; - try writer.writeByte(byte); - } - } - - inline fn isPrintableByte(c: u8) bool { - return ('!' <= c and c <= '~') or (0xa1 <= c and c <= 0xac) or (0xae <= c and c <= 0xff); - } -}; - -test Gpt2TextDecoder { - var decoder = try Gpt2TextDecoder.init(testing.allocator); - defer decoder.deinit(); - - var buf: [128]u8 = undefined; - - // Ascii is not changed. - { - var out: std.Io.Writer = .fixed(&buf); - try decoder.decode("getTitle", &out); - try testing.expectEqualStrings("getTitle", out.buffered()); - } - - { - var out: std.Io.Writer = .fixed(&buf); - try decoder.decode("Ċ", &out); - try testing.expectEqualStrings("\n", out.buffered()); - } - - // Leading space are represented with 'Ġ' - { - var out: std.Io.Writer = .fixed(&buf); - try decoder.decode("ĠUINavigationController", &out); - try testing.expectEqualStrings(" UINavigationController", out.buffered()); - } - // Russian is wild - { - var out: std.Io.Writer = .fixed(&buf); - try decoder.decode("ĠÑĢабоÑĤ", &out); - try testing.expectEqualStrings(" работ", out.buffered()); - } -} - -/// Open a json file in HF format and load the vocab from it. -pub fn fromHfJson(allocator: std.mem.Allocator, io: std.Io, tokenizer_path: []const u8) !Tokenizer { - const file = try std.fs.cwd().openFile(tokenizer_path, .{}); - defer file.close(); - - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - const file_content = b: { - var reader = file.reader(io, &.{}); - var writer: std.Io.Writer.Allocating = .init(allocator); - defer writer.deinit(); - _ = try reader.interface.streamRemaining(&writer.writer); - break :b try writer.toOwnedSlice(); - }; - defer allocator.free(file_content); - - const info = try std.json.parseFromSliceLeaky(std.json.Value, arena, file_content, .{ - .duplicate_field_behavior = .use_last, - }); - const main_object = switch (info) { - .object => |obj| if (obj.get("added_tokens") == null or obj.get("model") == null) { - return error.InvalidFormat; - } else obj, - else => return error.InvalidFormat, - }; - - const model = objectGet(main_object, .object, "model") orelse return error.InvalidFormat; - const vocab = objectGet(model, .object, "vocab") orelse return error.InvalidFormat; - const added_tokens = if (objectGet(main_object, .array, "added_tokens")) |added| added.items else &.{}; - const vocab_size: u32 = @intCast(vocab.count() + added_tokens.len); - - const normalizer = if (objectGet(main_object, .object, "normalizer")) |normalizer_config| - try Normalizer.fromHfJson(normalizer_config) - else - Normalizer.wellKnown(.llama3); - - // delay init of special tokens. - var tokenizer = try Tokenizer.init(allocator, vocab_size, 256, normalizer, undefined, true); - errdefer tokenizer.deinit(); - - // Buffer containing all concatenated tokens. - // Reserve a big chunk, to avoid grow event, but release over-allocated memory. - var all_tokens: std.Io.Writer.Allocating = try .initCapacity(tokenizer.arena_state.allocator(), file_content.len); - const original_alloc = all_tokens.writer.buffer.ptr; - // A re-alloc event here means we have invalidated all slices inside the tokenizer. - // If this is too annoying we could switch to a custom type instead of slices. - defer { - std.debug.assert(all_tokens.writer.buffer.ptr == original_alloc); - } - - // gpt2 based tokenizer got a special way of encoding unicode. - // we don't know in advance if this will be used by this tokenizer or not. - // so we assume it is the case, but if we find some unicode character, - // outside of the range used by gpt2 we know it was wrong, and start over. - var is_gpt2_vocab: bool = true; - var gpt2_decoder = try Gpt2TextDecoder.init(allocator); - defer gpt2_decoder.deinit(); - var it = vocab.iterator(); - while (it.next()) |kv| { - const n = all_tokens.writer.buffered().len; - gpt2_decoder.decode(kv.key_ptr.*, &all_tokens.writer) catch |err| { - switch (err) { - error.InvalidInput => { - is_gpt2_vocab = false; - break; - }, - error.WriteFailed => return error.OutOfMemory, - } - }; - const idx: u32 = @intCast(kv.value_ptr.*.integer); - const token = all_tokens.written()[n..]; - tokenizer.addOwnedTokenByIndex(idx, @floatFromInt(vocab_size - idx), token); - } - - if (!is_gpt2_vocab) { - // We where wrong, this is not a gpt2 vocab, start over, - // and reset the tokenizer state. - tokenizer.next_token_id = 0; - tokenizer.token_lookup.clearRetainingCapacity(); - all_tokens.clearRetainingCapacity(); - it = vocab.iterator(); - while (it.next()) |kv| { - const idx: u32 = @intCast(kv.value_ptr.*.integer); - const token = try dup(&all_tokens, kv.key_ptr.*); - tokenizer.addOwnedTokenByIndex(idx, @floatFromInt(vocab_size - idx), token); - } - } - - // More tokens, typically added during fine tuning of the model. - for (added_tokens) |token_obj| { - if (token_obj != .object) return error.InvalidFormat; - const v = objectGet(token_obj.object, .string, "content") orelse return error.InvalidFormat; - const id: u32 = @intCast(objectGet(token_obj.object, .integer, "id") orelse return error.InvalidFormat); - const n = all_tokens.written().len; - try if (is_gpt2_vocab) - gpt2_decoder.decode(v, &all_tokens.writer) - else - all_tokens.writer.writeAll(v); - const token = all_tokens.written()[n..]; - tokenizer.addOwnedTokenByIndex(id, 0, token); - } - // We won't add more tokens here, let release. - _ = try all_tokens.toOwnedSlice(); - - var unk = tokenizer.lookup(""); - if (objectGet(model, .integer, "unk_token")) |unk_tok| { - unk = @intCast(unk_tok); - } - - tokenizer.special_tokens = .{ - // TODO allow users to specify special tokens or read them from a tokenizer_config.json file - .bos = tokenizer.lookup("") orelse tokenizer.lookup("<|begin_of_text|>") orelse @panic("bos token not found !"), - .eos = tokenizer.lookup("") orelse tokenizer.lookup("<|end_of_text|>") orelse @panic("eos token not found !"), - .unk = unk orelse std.math.maxInt(u32), - }; - - const byte_fallback = objectGet(model, .bool, "byte_fallback") orelse false; - if (!byte_fallback and unk == null) { - // GPT2 tokenizer have byte fallback already encoded in the model, - // but the json generally don't have the field set. - // We can detect it though because they don't specify an unknown token. - if (is_gpt2_vocab) { - tokenizer.byte_fallback = true; - } else { - log.warn("The given tokenizer can't handle unknown token: no unknown token was set, and byte_fallback is disabled too ! The tokenizer will panic when facing unknown tokens.", .{}); - } - } else if (byte_fallback) { - try tokenizer.rewriteByteFallbackTokens(); - } - return tokenizer; -} - -/// Returns a copy of the given string, stored inside the given ArrayList. -fn dup(allocating: *std.Io.Writer.Allocating, str: []const u8) std.Io.Writer.Error![]const u8 { - const n = allocating.written().len; - try allocating.writer.writeAll(str); - return allocating.written()[n..]; -} - -/// Returns the given entry in a json object only if it has the right type. -fn objectGet( - object: std.json.ObjectMap, - comptime kind: std.meta.FieldEnum(std.json.Value), - key: []const u8, -) ?@FieldType(std.json.Value, @tagName(kind)) { - const val = object.get(key) orelse return null; - if (val != kind) return null; - return @field(val, @tagName(kind)); -} - -pub fn fromTinyLlamaFile(allocator: std.mem.Allocator, io: std.Io, tokenizer_path: []const u8, vocab_size: u32) !Tokenizer { - const tokenizer_file = try std.fs.cwd().openFile(tokenizer_path, .{}); - defer tokenizer_file.close(); - var read_buff: [4096]u8 = undefined; - var tok_reader = tokenizer_file.reader(io, &read_buff); - const r: *std.Io.Reader = &tok_reader.interface; - - const max_token_len = try readValueLE(u32, r); - const special_tokens: Tokenizer.SpecialTokens = .{ - .unk = 0, - .bos = 1, - .eos = 2, - }; - var tokenizer = try Tokenizer.init(allocator, vocab_size, max_token_len, null, special_tokens, true); - var i: u32 = 0; - while (readToken(&tokenizer, r)) : (i += 1) { - // Pass - } else |_| { - if (i < vocab_size) { - log.info("Read {d} words out of {d}", .{ i, vocab_size }); - } - tokenizer.vocab_size = i; - } - try tokenizer.rewriteByteFallbackTokens(); - return tokenizer; -} - -fn readToken(tokenizer: *Tokenizer, tok_reader: *std.Io.Reader) !void { - const score: f32 = try readValueLE(f32, tok_reader); - const len: usize = try readValueLE(u32, tok_reader); - try tokenizer.readTokenInto(score, len, tok_reader); -} - -fn readValueLE(T: type, reader: *std.Io.Reader) !T { - var res: [1]T = undefined; - try reader.readSliceEndian(T, &res, .little); - return res[0]; -} diff --git a/zml2/tokenizer/main.zig b/zml2/tokenizer/main.zig deleted file mode 100644 index a35de7d5..00000000 --- a/zml2/tokenizer/main.zig +++ /dev/null @@ -1,66 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); -const zml_tokenizer = @import("zml/tokenizer"); - -const log = std.log.scoped(.@"//zml/tokenizer"); - -const Flags = struct { - tokenizer: []const u8, - prompt: []const u8, - expected: []const u8 = "", - verbose: bool = false, -}; - -pub fn main() !void { - var gpa: std.heap.GeneralPurposeAllocator(.{}) = .{}; - const allocator = gpa.allocator(); - - var threaded: std.Io.Threaded = .init(allocator); - defer threaded.deinit(); - - const io = threaded.io(); - - const args = stdx.flags.parseProcessArgs(Flags); - - log.info("\tLoading tokenizer from {s}", .{args.tokenizer}); - var tokenizer = try zml_tokenizer.Tokenizer.fromFile(allocator, io, args.tokenizer); - log.info("✅\tLoaded tokenizer from {s}", .{args.tokenizer}); - defer tokenizer.deinit(); - - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - var decoder = try tokenizer.decoder(); - defer decoder.deinit(); - - const prompt_tok = try encoder.encode(args.prompt); - - log.info("Input: {s}\nOutput: {any}", .{ args.prompt, prompt_tok }); - - var errors: u8 = 0; - { - const reconstructed = try decoder.decode(prompt_tok); - if (!std.mem.eql(u8, args.prompt, reconstructed)) { - log.err("Reconstructed string from tokens doesn't match source: {s}", .{reconstructed}); - errors += 1; - } - } - - if (args.expected.len > 0) { - var expected: std.ArrayList(u32) = try .initCapacity(allocator, args.prompt.len); - var it = std.mem.splitSequence(u8, args.expected, ","); - while (it.next()) |int_token| { - const tok = try std.fmt.parseInt(u32, int_token, 10); - try expected.append(allocator, tok); - } - if (!std.mem.eql(u32, expected.items, prompt_tok)) { - log.err("Doesn't match expected: {any}", .{expected.items}); - errors += 1; - } - } - - if (errors == 0) log.info("All good !", .{}); - - std.process.exit(errors); -} diff --git a/zml2/tokenizer/sentencepiece/BUILD.bazel b/zml2/tokenizer/sentencepiece/BUILD.bazel deleted file mode 100644 index 10ac080b..00000000 --- a/zml2/tokenizer/sentencepiece/BUILD.bazel +++ /dev/null @@ -1,35 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") -load("//bazel:swig.bzl", "swig_cc_library") -load("//bazel:zig_srcs.bzl", "zig_srcs") - -swig_cc_library( - name = "sentencepiece_swig", - interface = "sentencepiece.i", - module = "sentencepiece", - deps = [ - "//ffi:cc", - "@com_google_sentencepiece//:sentencepiece_processor", - ], -) - -zig_library( - name = "sentencepiece", - import_name = "sentencepiece", - main = "sentencepiece.zig", - visibility = ["//visibility:public"], - deps = [ - ":sentencepiece_swig", - "//ffi:zig", - "//stdx", - ], -) - -zig_srcs( - name = "sources", - zig_lib = ":sentencepiece", -) - -cc_static_library( - name="sentencepiece_static", - deps = [":sentencepiece_swig"] -) diff --git a/zml2/tokenizer/sentencepiece/sentencepiece.i b/zml2/tokenizer/sentencepiece/sentencepiece.i deleted file mode 100644 index 35ccc057..00000000 --- a/zml2/tokenizer/sentencepiece/sentencepiece.i +++ /dev/null @@ -1,93 +0,0 @@ -/* File : example.i */ -%module sentencepiece -%include -%include - -%{ -#include -#include -#include -#include -#include "ffi/zig_slice.h" -%} - -%insert("cheader") %{ -#include "ffi/zig_slice.h" -%} - -%typemap(in) absl::string_view { - $1 = absl::string_view((char *)$input.ptr, $input.len); -} - -%typemap(out, optimal="1") const std::string& %{ - $result.ptr = (void *)($1->data()); - $result.len = (size_t)($1->length()); -%} -%typemap(ctype) absl::string_view, const std::string& "zig_slice" - -%rename(std_string) std::string; -namespace std { - class string { - public: - void reserve(size_t n); - void clear(); - size_t capacity() const; - }; -} - -%extend std::string { - const std::string& data() const { - return *$self; - }; -} - -%extend std::vector { - T* data() { - return $self->data(); - }; -} - -%template(std_vector_int) std::vector; - -%typemap(out) sentencepiece::util::Status %{ - $result = $1.code(); -%} -%typemap(ctype) sentencepiece::util::Status "unsigned int" -%rename(sentencepiece_util_StatusCode, fullname=1) sentencepiece::util::StatusCode; - -namespace sentencepiece { - namespace util { - enum class StatusCode : int { - kOk = 0, - kCancelled = 1, - kUnknown = 2, - kInvalidArgument = 3, - kDeadlineExceeded = 4, - kNotFound = 5, - kAlreadyExists = 6, - kPermissionDenied = 7, - kResourceExhausted = 8, - kFailedPrecondition = 9, - kAborted = 10, - kOutOfRange = 11, - kUnimplemented = 12, - kInternal = 13, - kUnavailable = 14, - kDataLoss = 15, - kUnauthenticated = 16, - }; - } - - class SentencePieceProcessor { - public: - virtual sentencepiece::util::Status Load(absl::string_view filename); - virtual sentencepiece::util::Status LoadFromSerializedProto(absl::string_view serialized); - virtual sentencepiece::util::Status Encode(absl::string_view input, std::vector *ids) const; - virtual sentencepiece::util::Status Decode(const std::vector &ids, std::string *detokenized) const; - virtual int PieceToId(absl::string_view piece) const; - virtual int unk_id() const; - virtual int bos_id() const; - virtual int eos_id() const; - virtual int pad_id() const; - }; -} diff --git a/zml2/tokenizer/sentencepiece/sentencepiece.zig b/zml2/tokenizer/sentencepiece/sentencepiece.zig deleted file mode 100644 index c98b0740..00000000 --- a/zml2/tokenizer/sentencepiece/sentencepiece.zig +++ /dev/null @@ -1,198 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const ffi = @import("ffi"); -const stdx = @import("stdx"); - -const StringToTokenRatio = 3; - -pub const Error = error{ - Cancelled, - Unknown, - InvalidArgument, - DeadlineExceeded, - NotFound, - AlreadyExists, - PermissionDenied, - ResourceExhausted, - FailedPrecondition, - Aborted, - OutOfRange, - Unimplemented, - Internal, - Unavailable, - DataLoss, - Unauthenticated, -}; - -fn assertOk(code: c.sentencepiece_util_StatusCode) Error!void { - return switch (code) { - c.sentencepiece_util_StatusCode_kOk => {}, - c.sentencepiece_util_StatusCode_kCancelled => Error.Cancelled, - c.sentencepiece_util_StatusCode_kUnknown => Error.Unknown, - c.sentencepiece_util_StatusCode_kInvalidArgument => Error.InvalidArgument, - c.sentencepiece_util_StatusCode_kDeadlineExceeded => Error.DeadlineExceeded, - c.sentencepiece_util_StatusCode_kNotFound => Error.NotFound, - c.sentencepiece_util_StatusCode_kAlreadyExists => Error.AlreadyExists, - c.sentencepiece_util_StatusCode_kPermissionDenied => Error.PermissionDenied, - c.sentencepiece_util_StatusCode_kResourceExhausted => Error.ResourceExhausted, - c.sentencepiece_util_StatusCode_kFailedPrecondition => Error.FailedPrecondition, - c.sentencepiece_util_StatusCode_kAborted => Error.Aborted, - c.sentencepiece_util_StatusCode_kOutOfRange => Error.OutOfRange, - c.sentencepiece_util_StatusCode_kUnimplemented => Error.Unimplemented, - c.sentencepiece_util_StatusCode_kInternal => Error.Internal, - c.sentencepiece_util_StatusCode_kUnavailable => Error.Unavailable, - c.sentencepiece_util_StatusCode_kDataLoss => Error.DataLoss, - c.sentencepiece_util_StatusCode_kUnauthenticated => Error.Unauthenticated, - else => unreachable, - }; -} - -pub const Encoder = struct { - inner: *SentencePieceProcessor, - vec: *c.std_vector_int, - - fn init(inner: *SentencePieceProcessor) Encoder { - return .{ - .inner = inner, - .vec = c.std_vector_int_new() orelse unreachable, - }; - } - - pub fn deinit(self: *Encoder) void { - c.std_vector_int_delete(self.vec); - } - - pub fn reset(self: *Encoder) void { - c.std_vector_int_clear(self.vec); - } - - pub fn encode(self: *Encoder, input: []const u8) ![]const u32 { - c.std_vector_int_reserve(self.vec, input.len / StringToTokenRatio); - try assertOk(c.SentencePieceProcessor_Encode(@ptrCast(self.inner), ffi.ZigSlice.from(input), self.vec)); - return self.ids(); - } - - pub fn ids(self: *const Encoder) []const u32 { - return ffi.ZigSlice.to(u32, .{ - .ptr = c.std_vector_int_data(self.vec), - .len = c.std_vector_int_size(self.vec), - }); - } -}; - -pub const Decoder = struct { - const StringBufferSize = 64; - const StringBuffer = stdx.BoundedArray(u8, StringBufferSize); - const TokenIdsBufferSize = 4; - - inner: *SentencePieceProcessor, - vec: *c.std_vector_int, - str: *c.std_string, - last_string: StringBuffer = .{ .len = 0 }, - - fn init(inner: *SentencePieceProcessor) !Decoder { - const vec = try (c.std_vector_int_new() orelse std.mem.Allocator.Error.OutOfMemory); - c.std_vector_int_reserve(vec, TokenIdsBufferSize); - errdefer c.std_vector_int_delete(vec); - - const str = try (c.std_string_new() orelse std.mem.Allocator.Error.OutOfMemory); - c.std_string_reserve(str, StringBufferSize); - errdefer c.std_string_delete(str); - - return .{ - .inner = inner, - .vec = vec, - .str = str, - }; - } - - pub fn deinit(self: *Decoder) void { - c.std_vector_int_delete(self.vec); - c.std_string_delete(self.str); - } - - pub fn reset(self: *Decoder) void { - c.std_vector_int_clear(self.vec); - c.std_string_clear(self.str); - } - - pub fn decode(self: *Decoder, ids_: []const u32) ![]const u8 { - c.std_vector_int_reserve(self.vec, ids_.len); - c.std_string_reserve(self.str, ids_.len * StringToTokenRatio); - for (ids_) |id| { - c.std_vector_int_push_back(self.vec, @intCast(id)); - } - try assertOk(c.SentencePieceProcessor_Decode(@ptrCast(self.inner), self.vec, self.str)); - return self.string(); - } - - pub fn string(self: *const Decoder) []const u8 { - const res = c.std_string_data(self.str); - return ffi.ZigSlice.to(u8, res); - } - - fn ids(self: *const Decoder) []u32 { - const ptr: [*c]u32 = @ptrCast(c.std_vector_int_data(self.vec)); - return ptr[0..c.std_vector_int_size(self.vec)]; - } - - pub fn next(self: *Decoder, token_id: u32) !?[]const u8 { - const current_ids = self.ids(); - if (current_ids.len >= c.std_vector_int_capacity(self.vec)) { - std.mem.copyForwards(u32, current_ids[0 .. current_ids.len - 1], current_ids[1..]); - current_ids[current_ids.len - 1] = token_id; - } else { - c.std_vector_int_push_back(self.vec, @intCast(token_id)); - } - try assertOk(c.SentencePieceProcessor_Decode(@ptrCast(self.inner), self.vec, self.str)); - const new_string = self.string(); - if (self.last_string.len == 0) { - self.last_string = try StringBuffer.fromSlice(new_string); - return new_string; - } - var view = try std.unicode.Utf8View.init(self.last_string.constSlice()); - var it = view.iterator(); - while (it.nextCodepointSlice()) |cp| { - const start = it.i - cp.len; - if (std.mem.startsWith(u8, new_string, self.last_string.constSlice()[start..])) { - const chunk = new_string[self.last_string.len - start ..]; - self.last_string = try StringBuffer.fromSlice(new_string); - return chunk; - } - } - return &.{}; - } -}; - -pub const SentencePieceProcessor = opaque { - pub fn fromFile(model: []const u8) !*SentencePieceProcessor { - const sp: *SentencePieceProcessor = @ptrCast(c.SentencePieceProcessor_new()); - errdefer sp.deinit(); - try assertOk(c.SentencePieceProcessor_Load(@ptrCast(sp), ffi.ZigSlice.from(model))); - return sp; - } - - pub fn fromBytes(bytes: []const u8) !*SentencePieceProcessor { - const sp: *SentencePieceProcessor = @ptrCast(c.SentencePieceProcessor_new()); - errdefer sp.deinit(); - try assertOk(c.SentencePieceProcessor_LoadFromSerializedProto(@ptrCast(sp), ffi.ZigSlice.from(bytes))); - return sp; - } - - pub fn deinit(self: *SentencePieceProcessor) void { - c.SentencePieceProcessor_delete(@ptrCast(self)); - } - - pub fn encoder(self: *SentencePieceProcessor) !Encoder { - return Encoder.init(self); - } - - pub fn decoder(self: *SentencePieceProcessor) !Decoder { - return try Decoder.init(self); - } - - pub fn tokenToId(self: *SentencePieceProcessor, token: []const u8) u32 { - return @intCast(c.SentencePieceProcessor_PieceToId(@ptrCast(self), ffi.ZigSlice.from(token))); - } -}; diff --git a/zml2/tokenizer/tokenizer.zig b/zml2/tokenizer/tokenizer.zig deleted file mode 100644 index c72ec803..00000000 --- a/zml2/tokenizer/tokenizer.zig +++ /dev/null @@ -1,143 +0,0 @@ -const std = @import("std"); - -pub const hftokenizers = @import("hftokenizers"); -pub const sentencepiece = @import("sentencepiece"); - -pub const homemade = @import("homemade.zig"); - -test { - std.testing.refAllDecls(@This()); -} - -const Tokenizers = enum { - hftokenizers, - sentencepiece, - homemade, -}; - -pub const Tokenizer = union(Tokenizers) { - pub const Encoder = union(Tokenizers) { - hftokenizers: hftokenizers.Encoder, - sentencepiece: sentencepiece.Encoder, - homemade: homemade.Encoder, - - pub fn deinit(self: *Encoder) void { - switch (self.*) { - inline else => |*v| v.deinit(), - } - } - - pub fn reset(self: *Encoder) void { - switch (self.*) { - inline else => |*v| v.reset(), - } - } - - pub fn encode(self: *Encoder, input: []const u8) ![]const u32 { - return switch (self.*) { - inline else => |*v| v.encode(input), - }; - } - - pub fn ids(self: Encoder) []const u32 { - return switch (self) { - inline else => |v| v.ids(), - }; - } - }; - - pub const Decoder = union(Tokenizers) { - hftokenizers: hftokenizers.Decoder, - sentencepiece: sentencepiece.Decoder, - homemade: homemade.Decoder, - - pub fn deinit(self: *Decoder) void { - switch (self.*) { - inline else => |*v| v.deinit(), - } - } - - pub fn reset(self: *Decoder) void { - switch (self.*) { - inline else => |*v| v.reset(), - } - } - - pub fn decode(self: *Decoder, ids_: []const u32) ![]const u8 { - return switch (self.*) { - inline else => |*v| v.decode(ids_), - }; - } - - pub fn string(self: *Decoder) []const u8 { - return switch (self.*) { - inline else => |v| v.string(), - }; - } - - //pub fn ids(self: *Decoder) []u32 { - // return switch (self.*) { - // inline else => |v| v.ids(), - // }; - //} - - pub fn next(self: *Decoder, token_id: u32) !?[]const u8 { - return switch (self.*) { - inline else => |*v| v.next(token_id), - }; - } - }; - - hftokenizers: *hftokenizers.HFTokenizer, - sentencepiece: *sentencepiece.SentencePieceProcessor, - homemade: *homemade.Tokenizer, - - pub fn fromFile(allocator: std.mem.Allocator, io: std.Io, model: []const u8) !Tokenizer { - _ = allocator; // autofix - _ = io; // autofix - if (std.mem.endsWith(u8, model, ".pb")) { - return .{ .sentencepiece = try .fromFile(model) }; - } - if (std.mem.endsWith(u8, model, ".json")) { - return .{ .hftokenizers = try .fromFile(model) }; - } - - return error.InvalidArgument; - } - - pub fn fromBytes(allocator: std.mem.Allocator, io: std.Io, bytes: []const u8) !Tokenizer { - _ = allocator; // autofix - _ = io; // autofix - if (bytes[0] == '{') { - return .{ .hftokenizers = try .fromBytes(bytes) }; - } - return .{ .sentencepiece = try .fromBytes(bytes) }; - } - - pub fn deinit(self: *Tokenizer) void { - switch (self.*) { - inline else => |t| t.deinit(), - } - } - - pub fn encoder(self: Tokenizer) !Encoder { - return switch (self) { - inline else => |v, tag| @unionInit(Encoder, @tagName(tag), try v.encoder()), - }; - } - - pub fn decoder(self: Tokenizer) !Decoder { - return switch (self) { - inline else => |v, tag| @unionInit(Decoder, @tagName(tag), try v.decoder()), - }; - } - - pub fn tokenToId(self: Tokenizer, token: []const u8) ?u32 { - return switch (self) { - inline else => |v| v.tokenToId(token), - }; - } -}; - -test {} -test {} diff --git a/zml2/zml.zig b/zml2/zml.zig deleted file mode 100644 index f56946a3..00000000 --- a/zml2/zml.zig +++ /dev/null @@ -1,87 +0,0 @@ -//! Welcome to the ZML API documentation! -//! ZML provides tools to write high level code describing a neural network, -//! compiling it for various accelerators and targets, and executing it. -//! - -// Namespaces -const std = @import("std"); - -const c = @import("c"); -const runfiles = @import("runfiles"); - -pub const Buffer = @import("buffer.zig").Buffer; -pub const constants = @import("constants.zig"); -pub const dtype = @import("dtype.zig"); -pub const Data = dtype.Data; -pub const DataType = dtype.DataType; -pub const exe = @import("exe.zig"); -pub const Exe = exe.Exe; -pub const floats = @import("floats.zig"); -pub const meta = @import("meta.zig"); -pub const mlir = @import("mlirx.zig"); -pub const module = @import("module.zig"); -pub const nn = @import("nn.zig"); -pub const pjrt = @import("pjrtx.zig"); -pub const io = @import("io.zig"); -pub const platform = @import("platform.zig"); -pub const Platform = platform.Platform; -pub const Target = platform.Target; -pub const CompilationOptions = platform.CompilationOptions; -pub const shape = @import("shape.zig"); -pub const Shape = shape.Shape; -pub const safetensors = @import("safetensors.zig"); -pub const slice = @import("slice.zig"); -pub const Slice = slice.Slice; -pub const ConstSlice = slice.ConstSlice; -pub const tensor = @import("tensor.zig"); -pub const Tensor = tensor.Tensor; -pub const testing = @import("testing.zig"); -pub const tokenizer = @import("zml/tokenizer"); - -var runfiles_once = std.once(struct { - fn call_() !void { - if (std.process.hasEnvVarConstant("RUNFILES_MANIFEST_FILE") or std.process.hasEnvVarConstant("RUNFILES_DIR")) { - return; - } - - var io_: std.Io.Threaded = .init_single_threaded; - defer io_.deinit(); - - var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); - const allocator = arena.allocator(); - defer arena.deinit(); - - var envMap = std.process.EnvMap.init(allocator); - var r = (try runfiles.Runfiles.create(.{ .allocator = allocator, .io = io_.io() })) orelse return; - try r.environment(&envMap); - - var it = envMap.iterator(); - while (it.next()) |entry| { - const keyZ = try allocator.dupeZ(u8, entry.key_ptr.*); - const valueZ = try allocator.dupeZ(u8, entry.value_ptr.*); - _ = c.setenv(keyZ.ptr, valueZ.ptr, 1); - } - } - - fn call() void { - call_() catch @panic("Unable to init runfiles env"); - } -}.call); - -pub fn init() void { - runfiles_once.call(); - mlir.once.call(); -} - -pub fn deinit() void {} - -/// Return a clone of a type with Tensors replaced by Buffer. -/// Non-Tensor metadata is stripped out of the resulting struct. -/// Recursively descends into the type. -pub fn Bufferized(comptime T: type) type { - return meta.MapRestrict(Tensor, Buffer).map(T); -} - -test "zml" { - std.testing.refAllDecls(@This()); -} From 73950032f51fe6fc408db3d071569f432252d5ae Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 6 Jan 2026 10:34:29 +0000 Subject: [PATCH 198/881] zml: remove pjrtx proxy, only keep helpers --- zml/buffer.zig | 9 ++-- zml/exe.zig | 2 +- zml/io.zig | 7 +-- zml/module.zig | 6 ++- zml/pjrtx.zig | 112 +++++------------------------------------------ zml/platform.zig | 3 +- zml/zml.zig | 8 ++-- 7 files changed, 32 insertions(+), 115 deletions(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index bafca61c..d5409a04 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -1,11 +1,12 @@ const std = @import("std"); +const pjrt = @import("pjrt"); const stdx = @import("stdx"); const constants = @import("constants.zig"); const ConstSlice = @import("slice.zig").ConstSlice; const DataType = @import("dtype.zig").DataType; -const pjrt = @import("pjrtx.zig"); +const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const Slice = @import("slice.zig").Slice; @@ -71,7 +72,7 @@ pub const Buffer = struct { //} else 0; //_ = chunk_size; - const buffer_type = pjrt.bufferTypeFromDtype(shape_.dtype()); + const buffer_type = pjrtx.bufferTypeFromDtype(shape_.dtype()); const byte_strides = shape_.computeByteStrides(); const devices = platform.getDevices(); @@ -175,7 +176,7 @@ pub const Buffer = struct { var args = pjrt.Client.CreateUninitializedBufferArgs{ .dims = shape_.dims(), - .element_type = pjrt.bufferTypeFromDtype(shape_.dtype()), + .element_type = pjrtx.bufferTypeFromDtype(shape_.dtype()), .layout = .{ .tiled = .{ .minor_to_major = constants.minorToMajor(shape_.rank()), @@ -194,7 +195,7 @@ pub const Buffer = struct { else .{ .memory = platform.memoryForDevice(opts.memory, devices[i]) }; - const shard = try platform.pjrt_client.createUnitializedBuffer(platform.pjrt_api, args); + const shard = try platform.pjrt_client.createUninitializedBuffer(platform.pjrt_api, args); res._shards.appendAssumeCapacity(shard); } diff --git a/zml/exe.zig b/zml/exe.zig index 122f99d8..49361e0b 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -1,10 +1,10 @@ const std = @import("std"); +const pjrt = @import("pjrt"); const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; const meta = @import("meta.zig"); -const pjrt = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; diff --git a/zml/io.zig b/zml/io.zig index d49c7edc..24454e33 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -1,13 +1,14 @@ const std = @import("std"); const builtin = @import("builtin"); +const pjrt = @import("pjrt"); const stdx = @import("stdx"); pub const VFS = @import("io").VFS; const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("zml.zig").Bufferized; const meta = @import("meta.zig"); -const pjrt = @import("pjrtx.zig"); +const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const safetensors = @import("safetensors.zig"); const Shape = @import("shape.zig").Shape; @@ -366,10 +367,10 @@ pub const Transfer = struct { for (shape_specs, shapes) |*spec, shape| { const dims = try temp_arena.allocator().dupe(i64, shape.dims()); - spec.* = pjrt.ShapeSpec.init(dims, pjrt.bufferTypeFromDtype(shape.dtype())); + spec.* = pjrt.ShapeSpec.init(dims, pjrtx.bufferTypeFromDtype(shape.dtype())); } - const memory = platform.pjrt_client.memoryByKind(platform.pjrt_api, .device).?; + const memory = pjrtx.Client.memoryByKind(platform.pjrt_client, platform.pjrt_api, .device).?; const transfer_manager = try platform.pjrt_client.createBuffersForAsyncHostToDevice(platform.pjrt_api, .{ .shape_specs = shape_specs, .memory = memory }); errdefer transfer_manager.deinit(platform.pjrt_api); diff --git a/zml/module.zig b/zml/module.zig index 525cfbd6..7d7e0cbe 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -3,6 +3,7 @@ const std = @import("std"); const c = @import("c"); const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); +const pjrt = @import("pjrt"); const stdx = @import("stdx"); const upb = @import("upb"); @@ -10,7 +11,7 @@ const Buffer = @import("buffer.zig").Buffer; const Exe = @import("exe.zig").Exe; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); -const pjrt = @import("pjrtx.zig"); +const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const Tensor = @import("tensor.zig").Tensor; @@ -490,7 +491,8 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: break :blk options; }; - const loaded_executable = try platform.pjrt_client.compile( + const loaded_executable = try pjrtx.Client.compile( + platform.pjrt_client, platform.pjrt_api, arena, io, diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 459ee1e7..37c1e239 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -3,87 +3,15 @@ const std = @import("std"); const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); -pub const ffi = pjrt.ffi; -pub const Api = pjrt.Api; -pub const ApiError = pjrt.ApiError; -pub const BufferType = pjrt.BufferType; -pub const ClientInitError = pjrt.ClientInitError; -pub const CompiledMemoryStats = pjrt.CompiledMemoryStats; -pub const Device = pjrt.Device; -pub const DeviceDescription = pjrt.DeviceDescription; -pub const Error = pjrt.Error; -pub const ErrorCode = pjrt.ErrorCode; -pub const Executable = pjrt.Executable; -pub const ExecuteContext = pjrt.ExecuteContext; -pub const ExecuteError = ApiError; -pub const GetCostAnalysisError = pjrt.GetCostAnalysisError; -pub const Memory = pjrt.Memory; -pub const MemoryStats = pjrt.MemoryStats; -pub const NamedValue = pjrt.NamedValue; -pub const SerializeResult = pjrt.SerializeResult; -pub const Stream = pjrt.Stream; -pub const ShapeSpec = pjrt.ShapeSpec; -pub const Buffer = pjrt.Buffer; -pub const Event = pjrt.Event; -pub const LoadedExecutable = pjrt.LoadedExecutable; -pub const AsyncHostToDeviceTransferManager = pjrt.AsyncHostToDeviceTransferManager; -const zml = struct { - pub const DataType = @import("dtype.zig").DataType; -}; +pub const DataType = @import("dtype.zig").DataType; -const log = std.log.scoped(.zml); - -pub const CompileError = std.mem.Allocator.Error || error{InvalidMlirBytecodeVersion} || ApiError; - -fn InnerMixin(comptime innerT: type) type { - return struct { - inline fn inner(self: anytype) if (@typeInfo(@TypeOf(self)).pointer.is_const) *const innerT else *innerT { - return @ptrCast(self); - } - }; -} +const log = std.log.scoped(.@"zml/pjrtx"); pub const Client = opaque { - const inner = InnerMixin(pjrt.Client).inner; + pub const CompileError = std.mem.Allocator.Error || error{InvalidMlirBytecodeVersion} || pjrt.ApiError; - pub fn init(api: *const Api, options: []const NamedValue) ClientInitError!*Client { - return @ptrCast(try pjrt.Client.init(api, options)); - } - - pub fn deinit(self: *Client, api: *const Api) void { - self.inner().deinit(api); - } - - pub fn getPlatformName(self: *const Client, api: *const Api) []const u8 { - return self.inner().getPlatformName(api); - } - - pub fn getDevices(self: *const Client, api: *const Api) []const *const Device { - return self.inner().getDevices(api); - } - - pub fn getAddressableDevices(self: *const Client, api: *const Api) []const *const Device { - return self.inner().getAddressableDevices(api); - } - - pub const BufferFromHostBufferArgs = pjrt.Client.BufferFromHostBufferArgs; - pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) ApiError!struct { *Buffer, ?*Event } { - const buffer, const event_ = try self.inner().bufferFromHostBuffer(api, args); - return .{ @ptrCast(buffer), @ptrCast(event_) }; - } - - pub fn deserializeAndLoad(self: *const Client, api: *const Api, io: std.Io, bytes: []const u8) ApiError!*LoadedExecutable { - var future = io.async(pjrt.Client.deserializeAndLoad, .{ self.inner(), api, bytes }); - return @ptrCast(try future.await(io)); - } - - pub const CreateViewOfDeviceBufferArgs = pjrt.Client.CreateViewOfDeviceBufferArgs; - pub fn createViewOfDeviceBuffer(self: *const Client, api: *const Api, args: CreateViewOfDeviceBufferArgs) ApiError!*Buffer { - return @ptrCast(try self.inner().createViewOfDeviceBuffer(api, args)); - } - - pub fn compile(self: *const Client, api: *const Api, allocator: std.mem.Allocator, io: std.Io, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*LoadedExecutable { + pub fn compile(client: *const pjrt.Client, api: *const pjrt.Api, allocator: std.mem.Allocator, io: std.Io, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*pjrt.LoadedExecutable { var bytecode: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); defer bytecode.deinit(); module.operation().writeBytecode(.{ .desired_emit_version = 1 }, &bytecode.writer) catch |err| { @@ -111,55 +39,39 @@ pub const Client = opaque { }; }; - return @ptrCast(try self.inner().compile(api, io, .{ + return @ptrCast(try client.compile(api, io, .{ .bytecode = serialized_buffer.written(), .bytecode_format = .mlir, .compile_options_pb = compile_options_pb, })); } - pub fn addressableMemories(self: *const Client, api: *const Api) []*const Memory { - return self.inner().addressableMemories(api); - } - - pub fn memoryByKind(self: *const Client, api: *const Api, kind: Memory.Kind) ?*const Memory { - for (self.addressableMemories(api)) |mem| { + pub fn memoryByKind(client: *const pjrt.Client, api: *const pjrt.Api, kind: pjrt.Memory.Kind) ?*const pjrt.Memory { + for (client.addressableMemories(api)) |mem| { if (mem.kind(api) == kind) { return mem; } } return null; } - - pub const CreateUninitializedBufferArgs = pjrt.Client.CreateUninitializedBufferArgs; - - pub fn createUnitializedBuffer(self: *const Client, api: *const Api, args: CreateUninitializedBufferArgs) ApiError!*Buffer { - return @ptrCast(try self.inner().createUninitializedBuffer(api, args)); - } - - pub const CreateBuffersForAsyncHostToDeviceArgs = pjrt.Client.CreateBuffersForAsyncHostToDeviceArgs; - - pub fn createBuffersForAsyncHostToDevice(self: *const Client, api: *const Api, args: CreateBuffersForAsyncHostToDeviceArgs) ApiError!*AsyncHostToDeviceTransferManager { - return @ptrCast(try self.inner().createBuffersForAsyncHostToDevice(api, args)); - } }; -pub fn bufferTypeFromDtype(dt: zml.DataType) pjrt.BufferType { +pub fn bufferTypeFromDtype(dt: DataType) pjrt.BufferType { return switch (dt) { inline else => |tag| @field(pjrt.BufferType, @tagName(tag)), }; } -pub fn dtypeFromBufferType(pjrt_type: pjrt.BufferType) zml.DataType { +pub fn dtypeFromBufferType(pjrt_type: pjrt.BufferType) DataType { return switch (pjrt_type) { .invalid => @panic("Found an invalid pjrt buffer"), - inline else => |tag| @field(zml.DataType, @tagName(tag)), + inline else => |tag| @field(DataType, @tagName(tag)), }; } test bufferTypeFromDtype { - inline for (@typeInfo(zml.DataType).@"enum".fields) |field| { - const dt: zml.DataType = @enumFromInt(field.value); + inline for (@typeInfo(DataType).@"enum".fields) |field| { + const dt: DataType = @enumFromInt(field.value); try std.testing.expectEqual(dt, dtypeFromBufferType(bufferTypeFromDtype(dt))); } diff --git a/zml/platform.zig b/zml/platform.zig index 3a26d30f..f135a1a9 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -1,11 +1,12 @@ const std = @import("std"); +const pjrt = @import("pjrt"); +const pjrtx = @import("pjrt"); const runtimes = @import("runtimes"); pub const Target = runtimes.Platform; const stdx = @import("stdx"); const Exe = @import("exe.zig").Exe; -const pjrt = @import("pjrtx.zig"); const zml = @import("zml.zig"); const log = std.log.scoped(.zml); diff --git a/zml/zml.zig b/zml/zml.zig index f56946a3..13b60c72 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -7,7 +7,9 @@ const std = @import("std"); const c = @import("c"); +pub const pjrt = @import("pjrt"); const runfiles = @import("runfiles"); +pub const tokenizer = @import("zml/tokenizer"); pub const Buffer = @import("buffer.zig").Buffer; pub const constants = @import("constants.zig"); @@ -17,26 +19,24 @@ pub const DataType = dtype.DataType; pub const exe = @import("exe.zig"); pub const Exe = exe.Exe; pub const floats = @import("floats.zig"); +pub const io = @import("io.zig"); pub const meta = @import("meta.zig"); pub const mlir = @import("mlirx.zig"); pub const module = @import("module.zig"); pub const nn = @import("nn.zig"); -pub const pjrt = @import("pjrtx.zig"); -pub const io = @import("io.zig"); pub const platform = @import("platform.zig"); pub const Platform = platform.Platform; pub const Target = platform.Target; pub const CompilationOptions = platform.CompilationOptions; +pub const safetensors = @import("safetensors.zig"); pub const shape = @import("shape.zig"); pub const Shape = shape.Shape; -pub const safetensors = @import("safetensors.zig"); pub const slice = @import("slice.zig"); pub const Slice = slice.Slice; pub const ConstSlice = slice.ConstSlice; pub const tensor = @import("tensor.zig"); pub const Tensor = tensor.Tensor; pub const testing = @import("testing.zig"); -pub const tokenizer = @import("zml/tokenizer"); var runfiles_once = std.once(struct { fn call_() !void { From 93d92f4757f476910235be3601541786fa2a4565 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 6 Jan 2026 10:44:02 +0000 Subject: [PATCH 199/881] third_party: remove clap --- third_party/com_github_hejsil_clap/BUILD.bazel | 0 third_party/com_github_hejsil_clap/clap.bazel | 9 --------- third_party/com_github_hejsil_clap/repo.bzl | 9 --------- third_party/non_module_deps.bzl | 2 -- 4 files changed, 20 deletions(-) delete mode 100644 third_party/com_github_hejsil_clap/BUILD.bazel delete mode 100644 third_party/com_github_hejsil_clap/clap.bazel delete mode 100644 third_party/com_github_hejsil_clap/repo.bzl diff --git a/third_party/com_github_hejsil_clap/BUILD.bazel b/third_party/com_github_hejsil_clap/BUILD.bazel deleted file mode 100644 index e69de29b..00000000 diff --git a/third_party/com_github_hejsil_clap/clap.bazel b/third_party/com_github_hejsil_clap/clap.bazel deleted file mode 100644 index 07caec46..00000000 --- a/third_party/com_github_hejsil_clap/clap.bazel +++ /dev/null @@ -1,9 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "clap", - import_name = "clap", - srcs = glob(["clap/*.zig"]), - main = "clap.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/com_github_hejsil_clap/repo.bzl b/third_party/com_github_hejsil_clap/repo.bzl deleted file mode 100644 index 11a00b3c..00000000 --- a/third_party/com_github_hejsil_clap/repo.bzl +++ /dev/null @@ -1,9 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") - -def repo(): - new_git_repository( - name = "com_github_hejsil_clap", - remote = "https://github.com/devoc09/zig-clap.git", - commit = "3c35ff18bb32fec2f53e12e6bdbb00422d6c33fe", - build_file = "//:third_party/com_github_hejsil_clap/clap.bazel", - ) diff --git a/third_party/non_module_deps.bzl b/third_party/non_module_deps.bzl index 02eee8fc..36791eb3 100644 --- a/third_party/non_module_deps.bzl +++ b/third_party/non_module_deps.bzl @@ -1,5 +1,4 @@ load("//third_party/arocc:repo.bzl", arocc = "repo") -load("//third_party/com_github_hejsil_clap:repo.bzl", com_github_hejsil_clap = "repo") load("//third_party/com_google_sentencepiece:repo.bzl", com_google_sentencepiece = "repo") load("//third_party/mnist:repo.bzl", mnist = "repo") load("//third_party/org_swig_swig:repo.bzl", org_swig_swig = "repo") @@ -9,7 +8,6 @@ load("//third_party/xla:repo.bzl", xla = "repo") def _non_module_deps_impl(mctx): com_google_sentencepiece() org_swig_swig() - com_github_hejsil_clap() mnist() xla() arocc() From 53a4b6905d812a735c8821bbbd9f949f7d843bf4 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 6 Jan 2026 10:45:28 +0000 Subject: [PATCH 200/881] examples: remove outdated examples (will reintroduce later) --- examples/gpt_oss/BUILD.bazel | 16 - examples/gpt_oss/GptOss.zig | 741 ------------------ examples/gpt_oss/README.md | 19 - examples/gpt_oss/main.zig | 376 --------- examples/modernbert/BUILD.bazel | 46 -- .../modernbert/activations/activations.py | 55 -- .../modernbert/activations/requirements.in | 4 - examples/modernbert/main.zig | 275 ------- examples/modernbert/modernbert.zig | 269 ------- examples/modernbert/test.zig | 238 ------ 10 files changed, 2039 deletions(-) delete mode 100644 examples/gpt_oss/BUILD.bazel delete mode 100644 examples/gpt_oss/GptOss.zig delete mode 100644 examples/gpt_oss/README.md delete mode 100644 examples/gpt_oss/main.zig delete mode 100644 examples/modernbert/BUILD.bazel delete mode 100644 examples/modernbert/activations/activations.py delete mode 100644 examples/modernbert/activations/requirements.in delete mode 100644 examples/modernbert/main.zig delete mode 100644 examples/modernbert/modernbert.zig delete mode 100644 examples/modernbert/test.zig diff --git a/examples/gpt_oss/BUILD.bazel b/examples/gpt_oss/BUILD.bazel deleted file mode 100644 index 7efba06c..00000000 --- a/examples/gpt_oss/BUILD.bazel +++ /dev/null @@ -1,16 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_test") - -zig_binary( - name = "gpt_oss", - srcs = [ - "GptOss.zig", - ], - main = "main.zig", - deps = [ - "@com_github_hejsil_clap//:clap", - "@zml//async", - "@zml//stdx", - "@zml//zml", - ], - visibility = ["//visibility:public"], -) diff --git a/examples/gpt_oss/GptOss.zig b/examples/gpt_oss/GptOss.zig deleted file mode 100644 index bce16daf..00000000 --- a/examples/gpt_oss/GptOss.zig +++ /dev/null @@ -1,741 +0,0 @@ -///! GptOss architecture, using huggingface transformers naming. -///! Dimensions of activations: {.b, .s, .d} -const std = @import("std"); - -const stdx = @import("stdx"); -const zml = @import("zml"); - -const GptOss = @This(); -const log = std.log.scoped(.GptOss); - -pub const Config = struct { - bos_token_id: u32 = 199998, - eos_token_id: stdx.json.Union(union(enum) { - int: u32, - ints: []const u32, - }), - head_dim: u32, - num_hidden_layers: u32, - num_attention_heads: u32, - num_key_value_heads: u32, - experts_per_token: u32, - rope_theta: f32, - max_position_embeddings: u32, - rms_norm_eps: f32, - sliding_window: u32, - hf_rope_impl: bool = true, - rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = {} }, -}; - -pub const Options = struct { - sampling_strategy: zml.nn.SamplingStrategy, - max_seq_len: u32, - max_prompt_len: u32, - tokens_per_expert_ratio: f32, -}; - -pub const Mode = union(enum) { - /// In prefill mode we pass the actual len of the prompt - prefill: zml.Tensor, - /// In gen mode we pass the position of the next token - gen: zml.Tensor, -}; - -lm_head: ?zml.nn.Linear, -model: Model, - -config: Config, -options: Options, - -pub fn init(allocator: std.mem.Allocator, store: zml.aio.BufferStore, config: Config, options: Options) !GptOss { - var self: GptOss = .{ - .config = config, - .options = options, - .model = .{ - .max_seq_len = @intCast(options.max_seq_len), - .num_heads = @intCast(config.num_attention_heads), - .num_kv_heads = @intCast(config.num_key_value_heads), - .rope_opts = .{ - .layout = if (config.hf_rope_impl) .sequential else .interleaved, - .freq_base = config.rope_theta, - .scaling = config.rope_scaling, - }, - - .embed_tokens = .{ - .weight = store.getTensor("model.embed_tokens.weight").withSharding(.{1}), - }, - .layers = try allocator.alloc(TransformerLayer, config.num_hidden_layers), - .norm = .{ - .weight = store.getTensor("model.norm.weight"), - .eps = config.rms_norm_eps, - }, - }, - .lm_head = .{ .weight = store.getTensor("lm_head.weight").withSharding(.{0}) }, - }; - - var prefix: zml.aio.PrefixBuilder = try .initCapacity(allocator, 1024); - try prefix.push(stdx.noalloc, "model.layers"); - for (self.model.layers, 0..) |*layer, i| { - try prefix.pushDigit(stdx.noalloc, i); - defer prefix.pop(); - - var self_attn: SelfAttn = .{ - .sinks = store.getTensor(prefix.concat("self_attn.sinks")), - .q_proj = try zml.aio.populateModelWithPrefix(zml.nn.Linear, allocator, store, prefix.concat("self_attn.q_proj")), - .k_proj = try zml.aio.populateModelWithPrefix(zml.nn.Linear, allocator, store, prefix.concat("self_attn.k_proj")), - .v_proj = try zml.aio.populateModelWithPrefix(zml.nn.Linear, allocator, store, prefix.concat("self_attn.v_proj")), - .o_proj = try zml.aio.populateModelWithPrefix(zml.nn.Linear, allocator, store, prefix.concat("self_attn.o_proj")), - - .sliding_window = if (i % 2 == 0) config.sliding_window else null, - .num_heads = self.model.num_heads, - .num_kv_heads = self.model.num_kv_heads, - .rope_opts = self.model.rope_opts, - }; - - self_attn.q_proj.weight = self_attn.q_proj.weight.withSharding(.{0}); - self_attn.k_proj.weight = self_attn.k_proj.weight.withSharding(.{0}); - self_attn.v_proj.weight = self_attn.v_proj.weight.withSharding(.{0}); - self_attn.o_proj.weight = self_attn.o_proj.weight.withSharding(.{1}); - - const on_disk_moe = try zml.aio.populateModelWithPrefix(MoE.OnDisk, allocator, store, prefix.concat("mlp")); - var moe = on_disk_moe.rewrite(config.experts_per_token, options); - { - moe.experts.gate_up_proj.blocks = moe.experts.gate_up_proj.blocks.withSharding(.{.expert}); - moe.experts.down_proj.blocks = moe.experts.down_proj.blocks.withSharding(.{.expert}); - } - - layer.* = .{ - .input_layernorm = .{ - .weight = store.getTensor(prefix.concat("input_layernorm.weight")), - .eps = config.rms_norm_eps, - }, - .post_attention_layernorm = .{ - .weight = store.getTensor(prefix.concat("post_attention_layernorm.weight")), - .eps = config.rms_norm_eps, - }, - .self_attn = self_attn, - .mlp = moe, - }; - } - - // TODO(Corentin): Fix lm_head sharding when top-k sampling is enabled. - // It currently crashes/compilation fails - if (self.options.sampling_strategy.topk == 1 and self.lm_head != null) { - self.lm_head.?.weight = self.lm_head.?.weight.withSharding(.{0}); - } - - return self; -} - -/// Predicts the token at `token_index` position. -/// Returns: -/// - updated `tokens`, -/// - updated KV cache -/// - a Rng state to allow for probabilistic generation -pub fn forward( - self: GptOss, - tokens_: zml.Tensor, - mode: Mode, - kv_cache: KvCache, - rng: zml.Tensor.Rng, -) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { - const tokens = tokens_.withPartialTags(.{.s}); - - // token index is the position in the kv cache where to write results. - const token_index: zml.Tensor = switch (mode) { - .gen => |token_index| token_index, - .prefill => .scalar(0, .u32), - }; - var out, const updated_kv_cache = zml.call(self.model, .forward, .{ tokens, token_index, kv_cache }); - - switch (mode) { - // In prefill we only pass the last token to the lm head. - .prefill => |prompt_len| out = out.gather(.{ .s = prompt_len.convert(.i32).addConstant(-1) }, .{ .indices_are_sorted = true }), - .gen => {}, - } - - var new_token, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.options.sampling_strategy); - new_token = new_token.convert(.u32); - new_token = switch (mode) { - .gen => new_token.reuseBuffer(tokens), - .prefill => new_token.appendAxes(.{.s}), - }; - return .{ new_token, updated_kv_cache, new_rng }; -} - -fn sampleTokens( - self: GptOss, - lm_head_: ?zml.nn.Linear, - out_: zml.Tensor, - rng: zml.Tensor.Rng, - opts: zml.nn.SamplingStrategy, -) struct { zml.Tensor, zml.Tensor.Rng } { - const out = out_.withPartialTags(.{.d}); - - var logits = blk: { - if (lm_head_) |lm_head| { - break :blk zml.call(lm_head, .forward, .{out}); - } else { - break :blk self.model.embed_tokens.weight.withTags(.{ .voc, .d }).dot(out, .{.d}); - } - }; - - if (logits.shape().hasTag(.voc) == null) - logits = logits.rename(.{ .d = .voc }); - - const next_tokens, const new_rng = zml.nn.sampleTokens(logits, opts, rng); - return .{ next_tokens, new_rng }; -} - -pub fn loadBuffers(self: GptOss, allocator: std.mem.Allocator, store: zml.aio.BufferStore, platform: zml.Platform) !zml.Bufferized(GptOss) { - var prefix: zml.aio.PrefixBuilder = try .initCapacity(allocator, 256); - defer prefix.deinit(allocator); - - const noalloc = stdx.noalloc; - const loaded: zml.Bufferized(GptOss) = .{ - .model = .{ - .embed_tokens = try store.loadModelById(zml.nn.TokenEmbedding, noalloc, self.model.embed_tokens, platform), - .layers = try allocator.alloc(zml.Bufferized(TransformerLayer), self.model.layers.len), - .norm = try store.loadModelById(RmsNorm, noalloc, self.model.norm, platform), - }, - .lm_head = try store.loadModelById(?zml.nn.Linear, noalloc, self.lm_head, platform), - }; - - prefix.push(noalloc, "model.layers") catch unreachable; - for (loaded.model.layers, self.model.layers, 0..) |*d_layer, layer, layer_id| { - const ckpt = prefix.checkpoint(); - defer prefix.restore(ckpt); - - prefix.pushDigit(noalloc, layer_id) catch unreachable; - d_layer.* = .{ - .input_layernorm = try store.loadModelById(RmsNorm, noalloc, layer.input_layernorm, platform), - .self_attn = try store.loadModelById(SelfAttn, noalloc, layer.self_attn, platform), - .post_attention_layernorm = try store.loadModelById(RmsNorm, noalloc, layer.post_attention_layernorm, platform), - .mlp = try store.loadModelById(MoE, noalloc, layer.mlp, platform), - }; - } - - return loaded; -} - -pub const Model = struct { - embed_tokens: zml.nn.TokenEmbedding, - norm: RmsNorm, - layers: []TransformerLayer, - - max_seq_len: u32 = 0, - num_heads: i64 = 32, - num_kv_heads: i64 = 32, - rope_opts: zml.nn.RopeOpts = .{ - .layout = .interleaved, - .freq_base = 10_000, - }, - - /// Forward one token, using KV cache for previous tokens. - /// Returns result and updated KV cache. - pub fn forward(self: Model, tokens: zml.Tensor, token_index: zml.Tensor, kv_cache: KvCache) struct { zml.Tensor, KvCache } { - const embeds = embed(self.embed_tokens, tokens); - var hidden = embeds; - - var updated_kv_cache = kv_cache; - for (self.layers, 0..) |layer, i| { - hidden, updated_kv_cache = zml.call(layer, .forward, .{ hidden, token_index, updated_kv_cache.atLayer(i) }); - } - const output = zml.call(self.norm, .forward, .{hidden}); - - return .{ output, updated_kv_cache.reuseBuffer(kv_cache) }; - } - - pub fn embed(embed_tokens_: zml.nn.TokenEmbedding, tokens_: zml.Tensor) zml.Tensor { - return zml.call(embed_tokens_, .forward, .{tokens_}).withPartialTags(.{.d}); - } -}; - -pub const TransformerLayer = struct { - input_layernorm: RmsNorm, - self_attn: SelfAttn, - post_attention_layernorm: RmsNorm, - mlp: MoE, - - pub fn forward( - self: TransformerLayer, - x0: zml.Tensor, - token_index: zml.Tensor, - kv_cache: KvCache, - ) struct { zml.Tensor, KvCache } { - // Self Attention - //log.debug("TransformerLayer({}) -> {}", .{ x0, self.input_layernorm.forward(x0) }); - stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {f}", .{x0}); - - const x0_normalized = zml.call(self.input_layernorm, .forward, .{x0}); - const delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); - const x1 = x0.add(delta0); - - // Fully Connected - const x1_normalized = zml.call(self.post_attention_layernorm, .forward, .{x1}); - const x2 = zml.call(self.mlp, .forward, .{x1_normalized}).add(x1); - - return .{ x2.reuseBuffer(x0), updated_kv_cache }; - } -}; - -const RmsNorm = struct { - weight: zml.Tensor, - eps: f32 = 1e-6, - - /// L2 normalization of input tensor along `.d` axis. - pub fn forward(self: RmsNorm, input: zml.Tensor) zml.Tensor { - const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); - // Note: contrary to Llama here the full layer is done in .f32, not just the variance computation. - const normalized = zml.nn.rmsNorm(x.convert(.f32), .d, self.eps); - return normalized.mul(self.weight.convert(.f32).withTags(.{.d}).broad(x.shape())).convert(input.dtype()); - } -}; - -const MoE = struct { - experts: Mlp, - router: zml.nn.Linear, - moe_opts: MoeOpts, - - pub fn forward(self: MoE, input: zml.Tensor) zml.Tensor { - log.warn("compiling moe with {f}", .{input}); - // Note: GptOss applies softmax on the routing score. - // We delay the softmax to mixtureOfExperts where the actual routing is done. - // This allow to do re-routing without introducing nans. - const gating = self.router.forward(input); - return mixtureOfExperts(Mlp, self.experts, input, gating, self.moe_opts); - } - - pub const OnDisk = struct { - router: zml.nn.Linear, - experts: struct { - down_proj_bias: zml.Tensor, - down_proj_blocks: zml.Tensor, - down_proj_scales: zml.Tensor, - gate_up_proj_bias: zml.Tensor, - gate_up_proj_blocks: zml.Tensor, - gate_up_proj_scales: zml.Tensor, - }, - - pub fn rewrite(on_disk: OnDisk, experts_per_token: u32, options: Options) MoE { - const e = on_disk.experts; - return .{ - .experts = .{ - .gate_up_proj = .{ - // We need to bitcast the scale cause safetensors doesn't encode f8 types correctly - .scale = e.gate_up_proj_scales.withTags(.{ .expert, .out, .d }), - // We don't bitcast here because PJRT doesn't handle packed host buffers - .blocks = e.gate_up_proj_blocks.withTags(.{ .expert, .out, .d, .d_block }), - .blocks_dtype = .f4e2m1, - .bias = e.gate_up_proj_bias.withTags(.{ .expert, .d }), - }, - .down_proj = .{ - .blocks = e.down_proj_blocks.withTags(.{ .expert, .out, .d, .d_block }), - .blocks_dtype = .f4e2m1, - .scale = e.down_proj_scales.withTags(.{ .expert, .out, .d }), - .bias = e.down_proj_bias.withTags(.{ .expert, .d }), - }, - }, - - .router = .{ - .weight = on_disk.router.weight.withTags(.{ .expert, .d }), - .bias = on_disk.router.bias.?.withTags(.{.expert}), - }, - - .moe_opts = .{ - .experts_per_token = experts_per_token, - .tokens_per_expert_ratio = options.tokens_per_expert_ratio, - .normalization = .softmax, - }, - }; - } - }; -}; - -pub const Mlp = struct { - gate_up_proj: BlockScaledLinear, // {.out = intermediate_size * 2, .d = hidden_size / block_size, .d_block = block_size } - down_proj: BlockScaledLinear, // {.out = hidden_size * 2, .d = intermediate_size / block_size, .d_block = block_size } - - pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { - const dt = x.dtype(); - var gate, var up = zml.nn.splitRealImg(self.gate_up_proj.forward(x), .interleaved); - gate = .minimum(gate, .scalar(7, dt)); - up = .clamp(up, .scalar(-7, dt), .scalar(7, dt)); - - const out = gate.quickGelu().mul(up.addConstant(1)); - return zml.call(self.down_proj, .forward, .{out}); - } - - pub fn format(self: Mlp, writer: *std.Io.Writer) std.Io.Writer.Error!void { - try writer.print("Mlp(gate_up_proj=.{f}, down_proj=.{f})", .{ self.gate_up_proj, self.down_proj }); - } -}; - -pub const SelfAttn = struct { - q_proj: zml.nn.Linear, - k_proj: zml.nn.Linear, - v_proj: zml.nn.Linear, - sinks: zml.Tensor, - - o_proj: zml.nn.Linear, - - sliding_window: ?u32, - num_heads: i64, - num_kv_heads: i64, - rope_opts: zml.nn.RopeOpts, - - /// Self Attention. - /// - If token_index is set, x is assumed to be the representation of one new token, - /// and kv_cache will be read for the previous tokens. - /// - If token_index is not set, x is assumed to be the representation of all tokens - /// since the beginning of the sequence, and kv_cache won't be read. - /// In both case, kv_cache will be updated with the computed key and value. - /// x: {.b, .s, .d } -> .{.b, .s, .d} - pub fn forward( - self: SelfAttn, - x: zml.Tensor, - token_index: zml.Tensor, - kv_cache: KvCache, - ) struct { zml.Tensor, KvCache } { - const num_kv_heads = self.num_kv_heads; - var q = zml.call(self.q_proj, .forward, .{x}).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }).withSharding(.{.h}); - var k = zml.call(self.k_proj, .forward, .{x}).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{.h}); - var v = zml.call(self.v_proj, .forward, .{x}).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{.h}); - - // Generate the attention mask. - const seq_len = kv_cache.k.dim(.k); - var attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, x.dtype(), self.sliding_window); - - // Note: in Pytorch it would be very inefficient to generate the full attn_mask, - // then slice into it, but XLA is able to optimize this correctly. - attn_mask = attn_mask.gatherSlices(zml.Shape.init(.{ .q = x.dim(.s) }, attn_mask.dtype()), token_index.reshape(.{ .coord = 1 }), .{}); - - // In self-attention, .s axis is used both for keys and queries. - const pos_index = b: { - const temp = zml.Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()).withTags(.{.s}).broad(zml.Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); - break :b temp.add(token_index.broad(temp.shape())); - }; - - q = zml.nn.rope(q, pos_index, self.rope_opts); - k = zml.nn.rope(k, pos_index, self.rope_opts); - q = q.rename(.{ .s = .q }); - k = k.rename(.{ .s = .k }); - v = v.rename(.{ .s = .k }); - - const dtype = q.dtype(); - const new_kv_cache = kv_cache.update(k, v, token_index); - k = new_kv_cache.keys().convert(dtype); - v = new_kv_cache.values().convert(dtype); - - // TODO ringbuffer kv cache. - - const softmax_bias = self.sinks.withTags(.{.h}); - const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask, .softmax_bias = softmax_bias }); - const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); - return .{ zml.call(self.o_proj, .forward, .{attn}), new_kv_cache }; - } -}; - -pub const KvCache = struct { - k: zml.Tensor, - v: zml.Tensor, - layer_index: zml.Tensor, - - pub fn init(kv_shape: zml.Shape) KvCache { - // The KV-cache is initialized with ones to detect reads of uninitialized memory. - return .{ - .k = .constant(kv_shape, kv_shape.dtype().one()).withSharding(.{.h}), - .v = .constant(kv_shape, kv_shape.dtype().one()).withSharding(.{.h}), - .layer_index = .scalar(-1, .u32), - }; - } - - pub fn initShape(kv_shape: zml.Shape) zml.ShapeOf(KvCache) { - return .{ - .k = kv_shape, - .v = kv_shape, - .layer_index = zml.Shape.init(.{}, .u32), - }; - } - - pub fn initBuffer(kv_shape: zml.Shape, platform: zml.Platform) !zml.Bufferized(KvCache) { - return .{ - .k = try zml.Buffer.uninitialized(platform, kv_shape, .{}), - .v = try zml.Buffer.uninitialized(platform, kv_shape, .{}), - .layer_index = try zml.Buffer.uninitialized(platform, .scalar(.u32), .{}), - }; - } - - pub fn keys(self: KvCache) zml.Tensor { - return self.k.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); - } - - pub fn values(self: KvCache) zml.Tensor { - return self.v.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); - } - - pub fn update(self: KvCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: ?zml.Tensor) KvCache { - const idx = if (token_index) |idx| idx else zml.Tensor.arange(.{ .end = new_k.dim(.k) }, .u32); - - return .{ - .k = self.k.scatterSlices( - .{ .k = idx, .layer = self.layer_index }, - new_k.convert(self.k.dtype()), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.k), - .v = self.v.scatterSlices( - .{ .k = idx, .layer = self.layer_index }, - new_v.convert(self.v.dtype()), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.v), - .layer_index = self.layer_index, - }; - } - - pub fn atLayer(self: KvCache, layer_index: usize) KvCache { - return .{ - .k = self.k, - .v = self.v, - .layer_index = .scalar(layer_index, .u32), - }; - } - - pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { - return .{ - .k = self.k.reuseBuffer(other.k), - .v = self.v.reuseBuffer(other.v), - .layer_index = self.layer_index.reuseBuffer(other.layer_index), - }; - } -}; - -pub const BlockScaledLinear = struct { - blocks: zml.Tensor, - scale: zml.Tensor, - bias: ?zml.Tensor = null, - blocks_dtype: zml.DataType, - - pub fn forward(self: BlockScaledLinear, x: zml.Tensor) zml.Tensor { - const ctx = x.getContext(); - const res_shape = x.shape().setDim(-1, self.blocks.dim(-3)); - - // Bitcast to our actual type. This allows to load weights in a packed layout. - const blocks_0 = self.blocks.bitCast(self.blocks_dtype); - const blocks = blocks_0.merge(.{ .d_block = .{ .d_block, .bitcast } }); - - const scale = self.scale.bitCast(.f8e8m0); - - // log.warn("BlockScaledLinear({}): {f} -> {f}", .{ self, x, res_shape }); - const y = switch (ctx._platform.target) { - else => y: { - var dequantized_weight: zml.Tensor = .mul( - blocks.convert(x.dtype()), - scale.convert(x.dtype()).appendAxes(.{.d_block}), - ); - var y = x.dot(dequantized_weight.merge(.{ .d = .{ .d, .d_block } }), .{.d}); - // std.log.warn("output shape: {f}", .{y}); - std.debug.assert(y.shape().eql(res_shape)); - y._shape = res_shape; - break :y y; - }, - }; - return if (self.bias) |bias| y.add(bias.broad(y.shape())) else y; - } - - pub fn format(self: BlockScaledLinear, writer: *std.Io.Writer) !void { - try writer.print("BlockScaledLinear(blocks={f}, scale={f}, bias={?f}, dt={t})", .{ self.blocks, self.scale, self.bias, self.blocks_dtype }); - } -}; - -const MoeOpts = struct { - experts_per_token: u32, - tokens_per_expert_ratio: ?f32 = 0.0, - normalization: Normalization, - - pub const Normalization = enum { linear, softmax }; -}; - -/// We have three algorithms, -/// * one for single-stream inference (naive), -/// * one for small batch sized with exact precision that sends all tokens to all experts. -/// this isn't too costly as long as the batch size is small and the experts are IO bound. -/// * one for big batch size that assign a fixed compute budget per expert and -/// experts chose the tokens they want to handle. This introduces noise since it's possible -/// a token doesn't get their requested expert. -/// The parameter `tokens_per_expert_ratio` control how much compute budget is granted: -/// expert_budget = ratio * (num_tokens * experts_per_token / num_experts). -/// Bigger values of ratio will ensure it's rare a token doesn't get it's top 2 tokens. -/// -/// The preferred algorithm is the batched one, -/// it is selected as soon there is enough tokens to guarantee that experts will be active most of the time. -/// -/// - input: .{ .s, .d } per-entry vector -/// - gating: .{ .s, .expert } per-entry expert-affinity -/// - experts: .{ .expert, .d_out, .d } expert layer (need to have a .forward method). -/// -> output: .{ .s, .d_out } -pub fn mixtureOfExperts(Expert: type, experts: Expert, input: zml.Tensor, gating: zml.Tensor, opts: MoeOpts) zml.Tensor { - log.warn("mixtureOfExperts({s}, {f}, {f}, {})", .{ @typeName(Expert), input, gating, opts }); - const num_tokens: u32 = @intCast(input.dim(.s)); - const num_experts = gating.dim(.expert); - stdx.debug.assert(opts.experts_per_token > 0, "mixtureOfExperts expects opts.experts_per_token > 0, got {}", .{opts}); - - if (num_tokens == 1) { - return moePerTokenRouting(Expert, experts, input, gating, opts); - } - - const tokens_per_expert: u32 = if (opts.tokens_per_expert_ratio) |ratio| tpe: { - const compute_budget = ratio * @as(f32, @floatFromInt(num_tokens * opts.experts_per_token)); - var tpe: u32 = @intFromFloat(stdx.math.divFloat(f32, compute_budget, num_experts)); - // Round to next multiple of 8 to avoid weird shapes. - if (tpe % 8 != 0) tpe += 8 - (tpe % 8); - break :tpe tpe; - } else num_tokens; - - if (3 * tokens_per_expert <= 2 * num_tokens) { - const routing, const tokens_ids_per_expert = dispatchTokens(gating, .{ - .tokens_per_expert = tokens_per_expert, - .experts_per_token = opts.experts_per_token, - .normalization = opts.normalization, - }); - const scores_per_expert = routing.transpose(.{ .expert, .s }).gather(.{ .s = tokens_ids_per_expert }, .{}); - const input_per_expert = input.gather(.{ .s = tokens_ids_per_expert }, .{}); - var output_per_expert = experts.forward(input_per_expert); - output_per_expert = output_per_expert.mul(scores_per_expert.convert(output_per_expert.dtype()).broad(output_per_expert.shape())); - - // Reverse engineer the normal output shape that one expert would have produced for all tokens. - // If this fall short, we could use the "sliced_expert" strategy and call forward ourselves. - const output_shape = output_per_expert.shape().drop(.expert).rename(.{ .top_token = .s }).setDim(.s, num_tokens); - const output = zml.Tensor.scatterSlices( - .constant(output_shape, output_shape.dtype().zero()), - .{ .s = tokens_ids_per_expert }, - output_per_expert, - .{ .update_fn = zml.Tensor.ScatterOpts.increment }, - ); - - log.warn("mixtureOfExperts({s}, {f}, {f}) -> fixed budget impl tpe: {d}, tokens: {d}", .{ @typeName(Expert), input, gating, tokens_per_expert, num_tokens }); - return output; - } else { - return mixtureOfExpertsAllToAll(Expert, experts, input, gating, opts); - } -} - -/// Few tokens: most experts are unused, experts have at most one token. -/// Select active experts and compute with that. -pub fn moePerTokenRouting(Expert: type, experts: Expert, input: zml.Tensor, gating: zml.Tensor, opts: MoeOpts) zml.Tensor { - const num_tokens: u32 = @intCast(input.dim(.s)); - stdx.debug.assert(num_tokens < 32, "Trying to unroll a lot of tokens !", .{}); - const per_token_outputs = input.getContext().allocator().alloc(zml.Tensor, num_tokens) catch @panic("OOM"); - - const routing = gating.topK(.{ .top_expert = .expert }, opts.experts_per_token, .{}); - const per_token_score = switch (opts.normalization) { - .linear => routing.values.div(routing.values.sum(.top_expert)), - .softmax => routing.values.softmax(.top_expert), - }; - - for (per_token_outputs, 0..num_tokens) |*output, tok_id| { - for (0..opts.experts_per_token) |expert_rank| { - const expert_id = routing.indices.choose(.{ .s = tok_id, .top_expert = expert_rank }).asScalar(); - const expert_score = per_token_score.choose(.{ .s = tok_id, .top_expert = expert_rank }).asScalar(); - - var sliced_expert: Expert = undefined; - zml.meta.mapAlloc(struct { - pub fn cb(expert_id_: zml.Tensor, expert_weight: zml.Tensor) zml.Tensor { - return expert_weight.gather(.{ .expert = expert_id_ }, .{}); - } - }.cb, stdx.noalloc, expert_id, experts, &sliced_expert) catch unreachable; - - // TODO how does this work when the two experts are on different gpus? - // does the compute overlap ? - var expert_output = sliced_expert.forward(input.choose(.{ .s = tok_id })); - expert_output = .mul( - expert_output, - expert_score.convert(input.dtype()).broad(expert_output.shape()), - ); - output.* = if (expert_rank > 0) output.add(expert_output) else expert_output; - } - } - - log.warn("mixtureOfExperts({s}, {f}, {f}) -> single-stream impl", .{ @typeName(Expert), input, gating }); - return .stack(per_token_outputs, 0, .s); -} - -/// Send all tokens to all experts, and apply gating. -pub fn mixtureOfExpertsAllToAll(Expert: type, experts: Expert, input: zml.Tensor, gating: zml.Tensor, opts: MoeOpts) zml.Tensor { - log.warn("mixtureOfExperts({s}, {f}, {f}) -> all to all impl", .{ @typeName(Expert), input, gating }); - const num_experts = gating.dim(.expert); - const hard_gating = hardGating(gating, opts).print(); - // TODO: `input.insertAxes(0, .{.expert}).repeat1d(.expert, num_experts)` is too verbose for just broadcasting along a new axis` - const output_per_expert = experts.forward(input.insertAxes(0, .{.expert}).repeat1d(.expert, @intCast(num_experts))); - return output_per_expert.dot(hard_gating.convert(input.dtype()), .expert); -} - -/// Given `(token, expert) -> scores`, -/// keeps only the top-k expert per token, and normalize the scores accordingly. -/// Non selected experts will have a 0 score. -pub fn hardGating(gating: zml.Tensor, opts: MoeOpts) zml.Tensor { - const routing = gating.topK(.{ .top_expert = .expert }, opts.experts_per_token, .{}); - - const per_token_score = switch (opts.normalization) { - .linear => routing.values.div(routing.values.sum(.top_expert)), - .softmax => routing.values.softmax(.top_expert), - }; - - return zml.Tensor.scatterSlices( - .zeroes(gating.shape()), - .{ .expert = routing.indices }, - per_token_score, - .{ .indices_are_unique = true }, - ); -} - -/// Lot of tokens, each experts chose their tokens. -/// It means that some tokens may have only one expert assigned. -/// Each token will get assigned to at least one expert IIF the input gating is sums up to 1 (typically softmax output). -/// Returns the actual `(token, expert) -> scores` used. -pub fn dispatchTokens( - gating: zml.Tensor, - opts: struct { - tokens_per_expert: u32, - experts_per_token: u32, - normalization: MoeOpts.Normalization, - }, -) [2]zml.Tensor { - const num_experts = gating.dim(.expert); - - const token_pref = gating.argsort(.expert, .{ .descending = true }); - var expert_rank: zml.Tensor = .scatterSlices( - .zeroes(gating.shape().withDtype(.i32)), - .{ .expert = token_pref }, - .addConstant(.iota(gating.shape(), .expert), 1), - .{ .indices_are_unique = true }, - ); - // The pow(expert_rank) here means that we strongly favor top 1 over top 2 and top 2 over top 3. - // expert_routing: (expert, top_token) -> token - const expert_routing = gating.pow(expert_rank.convert(gating.dtype())).topK(.{ .top_token = .s }, opts.tokens_per_expert, .{}); - const scores_per_expert = gating.gather(.{ .s = expert_routing.indices }, .{}); - - // Update the gating coefficient to account for the expert routing. - // Each (token, expert) which can't be computed within the given budget is left to 0. - const gating_v2: zml.Tensor = .scatterSlices( - .zeroes(gating.shape()), - .{ .s = expert_routing.indices }, - scores_per_expert, - .{ .indices_are_unique = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ); - // Now set to zero the scores (token, expert) for tokens that have been assigned more than experts_per_token. - const lowest_experts = gating_v2.topK(.{ .top_expert = .expert }, @intCast(num_experts - opts.experts_per_token), .{ .descending = false }); - var gating_v3: zml.Tensor = .scatterSlices( - gating_v2, - .{ .expert = lowest_experts.indices }, - .zeroes(lowest_experts.values.shape()), - .{ .indices_are_unique = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ); - // Then normalize so the sum of experts scores for one token sums up to 1. - gating_v3 = switch (opts.normalization) { - .linear => gating_v3.div(gating_v3.sum(.expert)), - .softmax => gating_v3.softmax(.expert), - }; - const tokens_ids_per_expert = expert_routing.indices.transpose(.{ .expert, .top_token }); - - return .{ gating_v3, tokens_ids_per_expert }; -} diff --git a/examples/gpt_oss/README.md b/examples/gpt_oss/README.md deleted file mode 100644 index d569d1c9..00000000 --- a/examples/gpt_oss/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# GptOss - -Download the model (you can change 20b to 120b for the bigger version if you have the required hardware). - -``` -bazel run //tools/hf -- download openai/gpt-oss-20b --local-dir /tmp/models/openai/gpt-oss-20b --exclude metal/ original/ -``` - -Run the model, use `--@zml//runtimes:xxx` flags to target the relevant platform for your hardware. - -``` -bazel run --config=release //examples/gpt_oss \ - --@zml//runtimes:cuda=true \ - --@zml//runtimes:cpu=false \ - --@zml//runtimes:tpu=false \ - --@zml//runtimes:rocm=false \ - -- --hf-model-path=/tmp/models/openai/gpt-oss-20b/ \ - --prompt='What are some spooky funfacts about animal for a Halloween party ?' -``` diff --git a/examples/gpt_oss/main.zig b/examples/gpt_oss/main.zig deleted file mode 100644 index 72aead20..00000000 --- a/examples/gpt_oss/main.zig +++ /dev/null @@ -1,376 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); - -const async = @import("async"); -const clap = @import("clap"); -const stdx = @import("stdx"); -const zml = @import("zml"); -const Buffer = zml.Buffer; -const Tensor = zml.Tensor; -const ShapeOf = zml.ShapeOf; - -const GptOss = @import("GptOss.zig"); - -const log = std.log.scoped(.GptOss); - -pub const std_options: std.Options = .{ - .log_level = .info, - .logFn = async.logFn(std.log.defaultLog), -}; - -const cli_params = clap.parseParamsComptime( - \\--help print this help - \\--prompt the prompt - \\--hf-model-path path to the directory containing model weights, config and tokenizer - \\--seed random seed (optional) - \\--seq-len max sequence length - \\--prompt-len max prompt length - \\--temperature temperature (default 1.0) - \\--topk topk (default 10) - \\--expert-budget token budget per expert - \\--platform-options platform options, using Zon syntax, eg '.{.cuda=.{.allocator=.{.async=.{.memory_fraction=0.95}}}}' - \\--nochat skip prompt template - \\--sharding default: true: sharding on or off -); - -pub fn tokenizePrompt(tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8, no_chat: bool, out: []u32) ![]u32 { - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - if (no_chat) { - const tokens = try encoder.encode(prompt); - if (tokens.len > out.len) return error.PromptTooLong; - @memcpy(out[0..tokens.len], tokens); - return out[0..tokens.len]; - } - - const start_header = tokenizer.tokenToId("<|start|>") orelse return error.NoSuchToken; - const end_header_start_message = tokenizer.tokenToId("<|message|>") orelse return error.NoSuchToken; - const end_message = tokenizer.tokenToId("<|end|>") orelse return error.NoSuchToken; - - var tokens: std.ArrayList(u32) = .initBuffer(out); - - const system_prompt = try encoder.encode("You are ChatGPT, a large language model trained by OpenAI.\n"); - if (system_prompt.len + 4 > tokens.unusedCapacitySlice().len) return error.PromptTooLong; - tokens.appendSliceAssumeCapacity(&.{ start_header, tokenizer.tokenToId("system").?, end_header_start_message }); - tokens.appendSliceAssumeCapacity(system_prompt); - tokens.appendAssumeCapacity(end_message); - - const user_prompt = try encoder.encode(prompt); - if (user_prompt.len + 9 > tokens.unusedCapacitySlice().len) return error.PromptTooLong; - tokens.appendSliceAssumeCapacity(&.{ start_header, tokenizer.tokenToId("user").?, end_header_start_message }); - tokens.appendSliceAssumeCapacity(user_prompt); - tokens.appendSliceAssumeCapacity(&.{ - end_message, - start_header, - tokenizer.tokenToId("assistant").?, - tokenizer.tokenToId("<|channel|>") orelse return error.NoSuchToken, - tokenizer.tokenToId("analysis") orelse return error.NoSuchToken, - end_header_start_message, - }); - - return tokens.items; -} - -pub fn generateText( - config: GptOss.Config, - options: GptOss.Options, - mod_prefill: zml.ModuleExe(GptOss.forward), - mod_generate: zml.ModuleExe(GptOss.forward), - kv_cache_: zml.Bufferized(GptOss.KvCache), - tokenizer: zml.tokenizer.Tokenizer, - allocator: std.mem.Allocator, - seed: u128, - prompt_tok: []const u32, - output: *std.Io.Writer, -) !void { - var tokenizer_decoder = try tokenizer.decoder(); - defer tokenizer_decoder.deinit(); - - const platform = mod_generate.platform(); - - // init RNG and buffers - var rng = try zml.Tensor.Rng.init(platform, seed); - var generated_token_buffer = [_]u32{undefined}; - - var current_token, var kv_cache = prefill: { - // prepare device buffers for the prefill tokens and their positions - const prefill_buffer = try allocator.alloc(u32, options.max_prompt_len); - @memcpy(prefill_buffer[0..prompt_tok.len], prompt_tok); - - var prefill_tokens = try zml.Buffer.fromSlice(platform, .{options.max_prompt_len}, prefill_buffer); - defer prefill_tokens.deinit(); - var prefill_token_pos = try zml.Buffer.scalar(platform, prompt_tok.len, .u32); - defer prefill_token_pos.deinit(); - - const first_token, const kv_cache, rng = mod_prefill.call(.{ prefill_tokens, .{ .prefill = prefill_token_pos }, kv_cache_, rng }); - - // extract the first generated token - _ = try first_token.toHost(std.mem.sliceAsBytes(&generated_token_buffer)); - log.warn("first_token: {d}", .{generated_token_buffer[0]}); - break :prefill .{ first_token, kv_cache }; - }; - defer zml.aio.unloadBuffers(&kv_cache); - defer current_token.deinit(); - - const output_tokens_len = options.max_seq_len - prompt_tok.len - 1; - const start = std.time.microTimestamp(); - - // One token has already been generated by the prefill. - var num_tokens_generated: usize = 1; - - generation: for (0..output_tokens_len + 1) |i| { - // collect and print generated sequence - num_tokens_generated += 1; - const generated_token = generated_token_buffer[0]; - if (try tokenizer_decoder.next(generated_token)) |chunk| { - try output.writeAll(chunk); - } - - // check for eos - if (i == output_tokens_len) break :generation; - switch (config.eos_token_id.value) { - .int => |eos| if (generated_token == @as(u32, @intCast(eos))) break :generation, - .ints => |eos_list| { - for (eos_list) |eos| { - if (generated_token == @as(u32, @intCast(eos))) break :generation; - } - }, - } - - // current token pos needs to go into a zml.Buffer - const token_pos_buffer = &[_]u32{@intCast(prompt_tok.len + i)}; - const token_pos = try zml.Buffer.fromSlice(platform, .{}, token_pos_buffer); - defer token_pos.deinit(); - - // call to generate the next token - current_token, kv_cache, rng = mod_generate.call(.{ current_token, .{ .gen = token_pos }, kv_cache, rng }); - - // extract the generated token from the buffer - _ = try current_token.toHost(std.mem.sliceAsBytes(&generated_token_buffer)); - } - const end = std.time.microTimestamp(); - const duration = stdx.math.divFloat(f64, end - start, std.time.us_per_s); - const speed = @as(f64, @floatFromInt(num_tokens_generated)) / duration; - - log.info("✅ Generated {d} tokens in {:.3}s: {d:.3}tok/s", .{ num_tokens_generated, duration, speed }); -} - -pub fn main() !void { - try async.AsyncThread.main(std.heap.smp_allocator, asyncMain); -} - -pub fn asyncMain() !void { - log.info(" GptOss was compiled with {}", .{@import("builtin").mode}); - - var allocator: std.mem.Allocator = alloc: { - if (builtin.mode == .Debug) { - var dbg_alloc: std.heap.DebugAllocator(.{ - .never_unmap = true, - .retain_metadata = true, - }) = .init; - break :alloc dbg_alloc.allocator(); - } - break :alloc std.heap.smp_allocator; - }; - - const cli = ClapBoilerplate.parseCli(allocator); - defer cli.deinit(); - - const hf_model_path = cli.args.@"hf-model-path" orelse { - log.err("Missing --hf-model-path", .{}); - return; - }; - - const config = config: { - var arena: std.heap.ArenaAllocator = .init(allocator); - defer arena.deinit(); - - const model_config_path = try std.fs.path.join(arena.allocator(), &.{ hf_model_path, "config.json" }); - - var config_json_file = try async.File.open(model_config_path, .{ .mode = .read_only }); - defer config_json_file.close() catch unreachable; - - var config_reader = config_json_file.reader(try arena.allocator().alloc(u8, 256)); - var reader = std.json.Reader.init(allocator, &config_reader.interface); - defer reader.deinit(); - var config = try std.json.parseFromTokenSourceLeaky(GptOss.Config, arena.allocator(), &reader, .{ .ignore_unknown_fields = true }); - - // From generation_config.json - config.eos_token_id = .{ .value = .{ .ints = &.{ 200002, 199999, 200012 } } }; - break :config config; - }; - - var context = try zml.Context.init(); - defer context.deinit(); - - // initialize ZML platform - const platform: zml.Platform = platform: { - const arena: std.heap.ArenaAllocator = .init(allocator); - defer arena.deinit(); - - // eg: --platform-options='.{.cuda=.{.allocator=.{.bfc=.{.memory_fraction=0.99}}}}' - // eg: --platform-options='.{.cpu=.{.device_count=8}}' - const platform_opts = std.zon.parse.fromSlice(zml.Platform.CreateOptions, allocator, @ptrCast(cli.args.@"platform-options" orelse ".{}"), null, .{ .free_on_error = false }) catch |err| { - log.err("Failed to parse --platform-options as json ({}): {s}", .{ err, cli.args.@"platform-options".? }); - return err; - }; - - const compilation_options = zml.CompilationOptions{ - .xla_dump_to = "/tmp/zml/gpt_oss", - .sharding_enabled = cli.args.sharding orelse true, - }; - - const platform = context - .autoPlatform(platform_opts) - .withCompilationOptions(compilation_options); - context.printAvailablePlatforms(platform); - - break :platform platform; - }; - - const options: GptOss.Options = .{ - .max_seq_len = cli.args.@"seq-len" orelse 8192, - .max_prompt_len = cli.args.@"prompt-len" orelse 256, - .tokens_per_expert_ratio = cli.args.@"expert-budget" orelse 4.0, - .sampling_strategy = .{ - .topk = cli.args.topk orelse 10, - .temperature = 1.0, - }, - }; - - var compiler_arena = std.heap.ArenaAllocator.init(allocator); - defer compiler_arena.deinit(); - - const model_weights_path = try std.fs.path.join(allocator, &.{ hf_model_path, "model.safetensors.index.json" }); - defer allocator.free(model_weights_path); - - var store = try zml.aio.detectFormatAndOpen(allocator, model_weights_path); - defer store.deinit(); - - const model: GptOss = try GptOss.init(compiler_arena.allocator(), store, config, options); - - const tokens_shape_prefill = zml.Shape.init(.{ .s = options.max_prompt_len }, .u32); - const tokens_shape = zml.Shape.init(.{ .s = 1 }, .u32); - - const dtype = model.model.embed_tokens.weight.dtype(); - - const kv_shape = zml.Shape.init(.{ - .layer = model.model.layers.len, - .k = options.max_seq_len, - .h = config.num_key_value_heads, - .hd = config.head_dim, - }, dtype).withSharding(.{.h}); - - const kv_cache_shape: zml.ShapeOf(GptOss.KvCache) = GptOss.KvCache.initShape(kv_shape); - const rng_shape = zml.Tensor.Rng.shape(); - - var start = try std.time.Timer.start(); - var fut_mod_prefill = try async.async(zml.compileModel, .{ - allocator, GptOss.forward, model, - .{ - tokens_shape_prefill, - zml.ShapeOf(GptOss.Mode){ .prefill = .scalar(.u32) }, - kv_cache_shape, - rng_shape, - }, - platform, - }); - - var fut_mod = try async.async(zml.compileModel, .{ - allocator, GptOss.forward, model, - .{ - tokens_shape, - zml.ShapeOf(GptOss.Mode){ .gen = .scalar(.u32) }, - kv_cache_shape, - rng_shape, - }, - platform, - }); - - log.info("\tLoading GptOss weights from {s}...", .{model_weights_path}); - var gpt_oss_weights = try model.loadBuffers(compiler_arena.allocator(), store, platform); - defer zml.aio.unloadBuffers(&gpt_oss_weights); - log.info("✅\tLoaded weights in {D}", .{start.read()}); - - var module_prefill = (try fut_mod_prefill.await()).prepare(gpt_oss_weights); - defer module_prefill.deinit(); - var module_gen = (try fut_mod.await()).prepare(gpt_oss_weights); - defer module_gen.deinit(); - log.info("✅\tCompiled model in {D}", .{start.read()}); - - log.info("Creating KvCache", .{}); - const kv_cache = try GptOss.KvCache.initBuffer(kv_shape, platform); - - var tokenizer = blk: { - const model_tokenizer_path = try std.fs.path.join(allocator, &.{ hf_model_path, "tokenizer.json" }); - defer allocator.free(model_tokenizer_path); - - log.info("Loading tokenizer from {s}", .{model_tokenizer_path}); - var timer = try stdx.time.Timer.start(); - defer log.info("Loaded tokenizer from {s} [{f}]", .{ model_tokenizer_path, timer.read() }); - - break :blk try zml.tokenizer.Tokenizer.fromFile(allocator, model_tokenizer_path); - }; - errdefer tokenizer.deinit(); - - const prompt = cli.args.prompt orelse "What are some fun facts about animals?"; - log.info("✅\tPrompt: {s}", .{prompt}); - - const no_chat = cli.args.nochat orelse false; - const prompt_tok_buf = try allocator.alloc(u32, options.max_prompt_len); - defer allocator.free(prompt_tok_buf); - - const prompt_tok = tokenizePrompt(tokenizer, prompt, no_chat, prompt_tok_buf) catch |err| switch (err) { - error.PromptTooLong => std.debug.panic("Prompt too long, expected at most {d} tokens. Consider increasing --max-prompt-len", .{prompt_tok_buf.len}), - else => |e| return e, - }; - log.info("\t Tokenized prompt: {any} ({d} tokens)", .{ prompt_tok, prompt_tok.len }); - - const seed = cli.args.seed orelse @as(u128, @bitCast(std.time.nanoTimestamp())); - - // Unbuffered writing of the tokens to stdout. - // generated text will be printed token by token. - var output = std.fs.File.stdout().writer(&.{}); - - try generateText(config, options, module_prefill, module_gen, kv_cache, tokenizer, allocator, seed, prompt_tok, &output.interface); -} - -const ClapBoilerplate = struct { - pub const Cli = clap.Result(clap.Help, &cli_params, parsers); - - fn bool_parser(in: []const u8) error{}!bool { - return std.mem.indexOfScalar(u8, "tTyY1", in[0]) != null; - } - - const parsers = .{ - .BOOL = bool_parser, - .UINT = clap.parsers.int(u32, 0), - .FLOAT = clap.parsers.float(f32), - .STRING = clap.parsers.string, - .PATH = clap.parsers.string, - }; - - pub fn parseCli(allocator: std.mem.Allocator) Cli { - var diag: clap.Diagnostic = .{}; - var stderr_buffer: [1024]u8 = undefined; - var stderr = std.fs.File.stderr().writer(&stderr_buffer); - const cli = clap.parse(clap.Help, &cli_params, parsers, .{ - .diagnostic = &diag, - .allocator = allocator, - }) catch |err| { - diag.report(&stderr.interface, err) catch {}; - stderr.interface.print("usage: ", .{}) catch {}; - clap.usage(&stderr.interface, clap.Help, &cli_params) catch {}; - stderr.interface.print("\n", .{}) catch {}; - stderr.interface.flush() catch {}; - std.process.exit(1); - }; - if (cli.args.help != 0) { - clap.help(&stderr.interface, clap.Help, &cli_params, .{}) catch {}; - stderr.interface.flush() catch {}; - std.process.exit(0); - } - return cli; - } -}; diff --git a/examples/modernbert/BUILD.bazel b/examples/modernbert/BUILD.bazel deleted file mode 100644 index 081127ef..00000000 --- a/examples/modernbert/BUILD.bazel +++ /dev/null @@ -1,46 +0,0 @@ -load("@bazel_skylib//rules:build_test.bzl", "build_test") -load("@rules_zig//zig:defs.bzl", "zig_binary") - -zig_binary( - name = "modernbert", - srcs = ["modernbert.zig"], - args = [ - "--model=$(location @ModernBERT-base//:model.safetensors)", - "--tokenizer=$(location @ModernBERT-base//:tokenizer.json)", - "--num-attention-heads=12", - "--tie-word-embeddings=true", - ], - data = [ - "@ModernBERT-base//:model.safetensors", - "@ModernBERT-base//:tokenizer.json", - ], - main = "main.zig", - deps = [ - "//async", - "//stdx", - "//zml", - "@com_github_hejsil_clap//:clap", - ], -) - -# zig_cc_binary( -# name = "test-implementation", -# srcs = ["modernbert.zig"], -# args = [ -# "--model=$(location @ModernBERT-base//:model.safetensors)", -# ], -# data = [ -# "@ModernBERT-base//:model.safetensors", -# ], -# main = "test.zig", -# deps = [ -# "//async", -# "//zml", -# "@com_github_hejsil_clap//:clap", -# ], -# ) - -build_test( - name = "test", - targets = [":modernbert"], -) diff --git a/examples/modernbert/activations/activations.py b/examples/modernbert/activations/activations.py deleted file mode 100644 index b82416d0..00000000 --- a/examples/modernbert/activations/activations.py +++ /dev/null @@ -1,55 +0,0 @@ -import logging -import torch -from transformers import pipeline -from tools.zml_utils import ActivationCollector - -logging.basicConfig( - level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s" -) -log = logging.getLogger(__name__) - -MODEL_NAME: str = "answerdotai/ModernBERT-base" - - -def main() -> None: - try: - log.info("Start running main()") - - log.info(f"CPU capability : `{torch.backends.cpu.get_cpu_capability()}`") - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - - log.info(f"Loading model : `{MODEL_NAME}`") - - fill_mask_pipeline = pipeline( - "fill-mask", - model=MODEL_NAME, - device_map=device, - ) - model, tokenizer = fill_mask_pipeline.model, fill_mask_pipeline.tokenizer - log.info( - f"Model loaded successfully {model.config.architectures} - `{model.config.torch_dtype}` - {tokenizer.model_max_length} max tokens" # noqa: E501 - ) - - # Wrap the pipeline, and extract activations. - # Activations files can be huge for big models, - # so let's stop collecting after 1000 layers. - zml_pipeline = ActivationCollector( - fill_mask_pipeline, max_layers=1000, stop_after_first_step=True - ) - - input_text = "Paris is the [MASK] of France." - outputs, activations = zml_pipeline(input_text) - log.info(f"ouputs : {outputs}") - - filename = MODEL_NAME.split("/")[-1] + ".activations.pt" - torch.save(activations, filename) - log.info(f"Saved {len(activations)} activations to {filename}") - - log.info("End running main()") - except Exception as exception: - log.error(exception) - raise - - -if __name__ == "__main__": - main() diff --git a/examples/modernbert/activations/requirements.in b/examples/modernbert/activations/requirements.in deleted file mode 100644 index 415d610a..00000000 --- a/examples/modernbert/activations/requirements.in +++ /dev/null @@ -1,4 +0,0 @@ -torch -transformers==4.48.1 -accelerate -numpy==1.26.4 \ No newline at end of file diff --git a/examples/modernbert/main.zig b/examples/modernbert/main.zig deleted file mode 100644 index 0df16b39..00000000 --- a/examples/modernbert/main.zig +++ /dev/null @@ -1,275 +0,0 @@ -const std = @import("std"); - -const async = @import("async"); -const clap = @import("clap"); -const stdx = @import("stdx"); -const zml = @import("zml"); -const Tensor = zml.Tensor; - -const modernbert = @import("modernbert.zig"); - -const log = std.log.scoped(.modernbert); - -pub const std_options: std.Options = .{ - .log_level = .info, - .log_scope_levels = &[_]std.log.ScopeLevel{ - .{ .scope = .modernbert, .level = .info }, - }, - .logFn = async.logFn(std.log.defaultLog), -}; - -const params = clap.parseParamsComptime( - \\--help print this help - \\--text the prompt - \\--model model path - \\--tokenizer tokenizer path - \\--seq-len sequence length - \\--num-attention-heads number of attention heads - \\--tie-word-embeddings default: false: tied weights - \\--create-options platform creation options JSON, defaults to {} - \\--sharding default: true: sharding on or off -); - -const clap_parsers = .{ - .BOOL = bool_parser, - .UINT = clap.parsers.int(usize, 0), - .STRING = clap.parsers.string, - .PATH = clap.parsers.string, -}; - -pub fn main() !void { - try async.AsyncThread.main(std.heap.c_allocator, asyncMain); -} - -pub fn asyncMain() !void { - const allocator = std.heap.c_allocator; - const stderr = std.fs.File.stderr(); - - var diag: clap.Diagnostic = .{}; - var cli = clap.parse(clap.Help, ¶ms, clap_parsers, .{ - .diagnostic = &diag, - .allocator = allocator, - }) catch |err| { - try diag.reportToFile(stderr, err); - try printUsageAndExit(stderr); - }; - defer cli.deinit(); - - if (cli.args.help != 0) { - try clap.helpToFile(stderr, clap.Help, ¶ms, .{}); - return; - } - - const tmp = try std.fs.openDirAbsolute("/tmp", .{}); - try tmp.makePath("zml/modernbert/cache"); - - // Create ZML context - var context = try zml.Context.init(); - defer context.deinit(); - - // Platform and compilation options - const create_opts_json = cli.args.@"create-options" orelse "{}"; - const create_opts = try std.json.parseFromSliceLeaky(zml.Platform.CreateOptions, allocator, create_opts_json, .{}); - const compilation_options = zml.CompilationOptions{ - .xla_dump_to = "/tmp/zml/modernbert", - .sharding_enabled = cli.args.sharding orelse true, - }; - - // Auto-select platform - const platform = context.autoPlatform(create_opts).withCompilationOptions(compilation_options); - context.printAvailablePlatforms(platform); - - // Detects the format of the model file (base on filename) and open it. - const model_file = cli.args.model orelse { - var buf: [256]u8 = undefined; - var writer = stderr.writer(&buf); - writer.interface.print("Error: missing --model=...\n\n", .{}) catch {}; - printUsageAndExit(stderr); - unreachable; - }; - var tensor_store = try zml.aio.detectFormatAndOpen(allocator, model_file); - defer tensor_store.deinit(); - - // Memory arena dedicated to model shapes and weights - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const model_arena = arena_state.allocator(); - - var tokenizer = blk: { - if (cli.args.tokenizer) |tok| { - log.info("\tLoading tokenizer from {s}", .{tok}); - var timer = try stdx.time.Timer.start(); - defer log.info("✅\tLoaded tokenizer from {s} [{D}]", .{ tok, timer.read() }); - - break :blk try zml.tokenizer.Tokenizer.fromFile(model_arena, tok); - } else { - log.err("Error: missing --tokenizer", .{}); - return; - } - }; - defer tokenizer.deinit(); - - // Create the model struct, with tensor shapes extracted from the tensor_store - // TODO: read from config.json - const modernbert_options = modernbert.ModernBertOptions{ - .pad_token = tokenizer.tokenToId("[PAD]") orelse return error.NoSuchToken, - .num_attention_heads = @intCast(cli.args.@"num-attention-heads" orelse 12), - .tie_word_embeddings = cli.args.@"tie-word-embeddings" orelse false, - .local_attention = 128, - }; - var modern_bert_for_masked_lm = try zml.aio.populateModel(modernbert.ModernBertForMaskedLM, model_arena, tensor_store); - modern_bert_for_masked_lm.init(modernbert_options); - - log.info("\tModernBERT options: {}", .{modernbert_options}); - - // Prepare shapes for compilation - const seq_len = @as(i64, @intCast(cli.args.@"seq-len" orelse 256)); - const input_shape = zml.Shape.init(.{ .b = 1, .s = seq_len }, .u32); - - var start = try stdx.time.Timer.start(); - - // Load weights - log.info("\tLoading ModernBERT weights from {s}...", .{model_file}); - var bert_weights = try zml.aio.loadBuffers(modernbert.ModernBertForMaskedLM, .{modernbert_options}, tensor_store, model_arena, platform); - defer zml.aio.unloadBuffers(&bert_weights); - log.info("✅\tLoaded weights in {D}", .{start.read()}); - - // Compile the model - log.info("\tCompiling ModernBERT model...", .{}); - var fut_mod = try async.async(zml.compile, .{ - allocator, - modernbert.ModernBertForMaskedLM.forward, - .{modernbert_options}, - .{input_shape}, - tensor_store, - platform, - }); - var bert_module = (try fut_mod.await()).prepare(bert_weights); - defer bert_module.deinit(); - log.info("✅\tLoaded weights and compiled model in {D}", .{start.read()}); - - const text = cli.args.text orelse "Paris is the [MASK] of France."; - log.info("\tInput text: {s}", .{text}); - - try unmask(allocator, bert_module, tokenizer, seq_len, text); -} - -/// fill-mask pipeline -/// ref: https://github.com/huggingface/transformers/blob/main/src/transformers/pipelines/fill_mask.py -pub fn unmask( - allocator: std.mem.Allocator, - mod: zml.ModuleExe(modernbert.ModernBertForMaskedLM.forward), - tokenizer: zml.tokenizer.Tokenizer, - seq_len: i64, - text: []const u8, -) !void { - var tokenizer_decoder = try tokenizer.decoder(); - defer tokenizer_decoder.deinit(); - - const pad_token = tokenizer.tokenToId("[PAD]") orelse return error.NoSuchToken; - const mask_token = tokenizer.tokenToId("[MASK]") orelse return error.NoSuchToken; - - // Tokenize input text - const tokens: []const u32 = try tokenize(allocator, tokenizer, text); - defer allocator.free(tokens); - - // Find "[MASK]" positions - const mask_positions = try findMaskPositions(allocator, tokens, mask_token); - defer allocator.free(mask_positions); - - // Prepare input tensors - const inputs = try prepareTensorInputs(allocator, tokens, seq_len, pad_token); - defer allocator.free(inputs); - - // Create input tensors (on the accelerator) - const input_shape = zml.Shape.init(.{ .b = 1, .s = seq_len }, .i64); - const input_ids_tensor = try zml.Buffer.fromSlice(mod.platform(), input_shape.dims(), inputs); - defer input_ids_tensor.deinit(); - - // Model inference (retrieve indices) - var inference_timer = try std.time.Timer.start(); - var topk = mod.call(.{input_ids_tensor}); - defer zml.aio.unloadBuffers(&topk); - const inference_time = inference_timer.read(); - - // Transfer the result to host memory (CPU) - var indices_host_buffer = try topk.indices.toHostAlloc(allocator); - defer indices_host_buffer.deinit(allocator); - var values_host_buffer = try topk.values.toHostAlloc(allocator); - defer values_host_buffer.deinit(allocator); - - // We consider only the first occurrence of [MASK], which has five predictions - const pred_offset = mask_positions[0] * 5; - const predictions = indices_host_buffer.items(i32)[pred_offset..][0..5]; - const scores = values_host_buffer.items(f32)[pred_offset..][0..5]; - - // Log timing information - log.info("⏱️\tModel inference in {d}ms", .{inference_time / std.time.ns_per_ms}); - - log.info("✅\tTop 5 predictions:", .{}); - for (predictions, scores) |token_id, score| { - const token_text = try tokenizer_decoder.next(@intCast(token_id)); - if (token_text) |word| { - log.info("\t • score: {d:.4} word: '{s}' token: {}", .{ score, word, token_id }); - } - } -} - -pub fn tokenize(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]const u32 { - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - const bos = tokenizer.tokenToId("[CLS]") orelse return error.NoSuchToken; - const eos = tokenizer.tokenToId("[SEP]") orelse return error.NoSuchToken; - - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); - try tokens.append(allocator, bos); - try tokens.appendSlice(allocator, try encoder.encode(prompt)); - try tokens.append(allocator, eos); - - return tokens.toOwnedSlice(allocator); -} - -fn findMaskPositions(allocator: std.mem.Allocator, tokens: []const u32, mask_token: u32) ![]usize { - var mask_positions: std.ArrayList(usize) = .empty; - - for (tokens, 0..) |token, i| { - if (token == mask_token) { - try mask_positions.append(allocator, i); - } - } - - if (mask_positions.items.len == 0) { - log.err("Input text must contains `[MASK]`", .{}); - return error.InvalidInput; - } - - if (mask_positions.items.len > 1) log.warn("Currently only supporting one [MASK] per input", .{}); - - return mask_positions.toOwnedSlice(allocator); -} - -fn prepareTensorInputs( - allocator: std.mem.Allocator, - tokens: []const u32, - seq_len: i64, - pad_token: u32, -) ![]u32 { - const input_ids = try allocator.alloc(u32, @intCast(seq_len)); - - @memset(input_ids, pad_token); - for (tokens, 0..) |token, i| { - input_ids[i] = @intCast(token); - } - - return input_ids; -} - -fn bool_parser(in: []const u8) error{}!bool { - return std.mem.indexOfScalar(u8, "tTyY1", in[0]) != null; -} - -fn printUsageAndExit(stderr: std.fs.File) noreturn { - clap.usageToFile(stderr, clap.Help, ¶ms) catch {}; - std.process.exit(0); -} diff --git a/examples/modernbert/modernbert.zig b/examples/modernbert/modernbert.zig deleted file mode 100644 index 1ba35578..00000000 --- a/examples/modernbert/modernbert.zig +++ /dev/null @@ -1,269 +0,0 @@ -const std = @import("std"); - -const async = @import("async"); -const stdx = @import("stdx"); -const zml = @import("zml"); -const Tensor = zml.Tensor; - -const log = std.log.scoped(.modernbert); - -pub const ModernBertOptions = struct { - num_attention_heads: i64, - pad_token: u32, - local_attention: u32, - tie_word_embeddings: bool = false, -}; - -pub const ModernBertForMaskedLM = struct { - model: ModernBertModel, - head: ModernBertPredictionHead, - decoder: struct { weight: ?zml.Tensor, bias: zml.Tensor }, - - pub fn init(self: *ModernBertForMaskedLM, options: ModernBertOptions) void { - self.model.init(options); - self.head.norm.eps = 1e-5; - - self.head.dense.weight = self.head.dense.weight.withSharding(.{0}); - - if (options.tie_word_embeddings == true) { - self.decoder.weight = null; - } else if (self.decoder.weight) |decoder_weight| { - self.decoder.weight = decoder_weight.withSharding(.{1}); - } - } - - pub fn forward(self: ModernBertForMaskedLM, input_ids: Tensor) zml.Tensor.ArgMaxRes { - const outputs: Tensor = zml.call(self.model, .forward, .{input_ids}); - const head_outputs: Tensor = zml.call(self.head, .forward, .{outputs}); - - // either use decoder or tied weights - const decoder_weights = self.decoder.weight orelse self.model.embeddings.tok_embeddings.weight; - - const logits = head_outputs.withTags(.{ .b, .s, .d }).dot(decoder_weights.withTags(.{ .voc, .d }), .{.d}); - const biased_logits = logits.add(self.decoder.bias.withTags(.{.voc}).broad(logits.shape())); - - const probabilities = biased_logits.softmax(.voc); - return probabilities.topK(.{ .best_words = .voc }, 5, .{ .descending = true }); - } -}; - -pub const ModernBertModel = struct { - options: ModernBertOptions, - embeddings: ModernBertEmbeddings, - layers: []ModernBertEncoderLayer, - final_norm: zml.nn.LayerNorm, - - pub fn init(self: *ModernBertModel, options: ModernBertOptions) void { - self.options = options; - self.final_norm.eps = 1e-5; - self.embeddings.norm.eps = 1e-5; - for (self.layers, 0..) |*encoder_layer, layer_idx| { - encoder_layer.attn.Wqkv.weight = encoder_layer.attn.Wqkv.weight.withSharding(.{0}); - encoder_layer.attn.Wo.weight = encoder_layer.attn.Wo.weight.withSharding(.{1}); - - encoder_layer.mlp.Wi.weight = encoder_layer.mlp.Wi.weight.withSharding(.{0}); - encoder_layer.mlp.Wo.weight = encoder_layer.mlp.Wo.weight.withSharding(.{1}); - - if (encoder_layer.attn_norm) |*norm| norm.eps = 1e-5; - encoder_layer.mlp_norm.eps = 1e-5; - encoder_layer.attn.is_global_attention = (layer_idx % 3 == 0); - encoder_layer.attn.num_heads = options.num_attention_heads; - } - } - - pub fn forward(self: ModernBertModel, input_ids: Tensor) Tensor { - var hidden_states: Tensor = zml.call(self.embeddings, .forward, .{input_ids}).withTags(.{ .b, .src, .d }); - - const global_mask = globalAttnMask(input_ids, hidden_states.dtype(), self.options.pad_token); - const local_mask = localAttnMask(global_mask, self.options.local_attention); - - // Process through all encoder layers - for (self.layers) |encoder_layer| { - hidden_states = zml.call(encoder_layer, .forward, .{ - hidden_states, - global_mask, - local_mask, - }); - } - - // Final layer normalization - hidden_states = zml.call(self.final_norm, .forward, .{hidden_states}); - - return hidden_states; - } - - /// Find [PAD] tokens in inputs, and assign them a -inf attention mask. - /// Output shapes follows zml.nn.sdpa convention: .{ .b, .q, .k } - pub fn globalAttnMask(input_ids: Tensor, dt: zml.DataType, pad_token: u32) Tensor { - const ids = input_ids.withTags(.{ .b, .k }); - - // Mask keys where corresponding token is [PAD] - const padding = ids.cmp(.EQ, Tensor.scalar(pad_token, ids.dtype())); - const pad_mask = padding.select(Tensor.constant(.{}, dt.minValue()), Tensor.constant(.{}, dt.zero())); - - // Broadcast to the desired output shape. - const seq_len = ids.dim(.k); - const pad_mask_shape = zml.Shape.init(.{ .b = pad_mask.dim(.b), .q = seq_len, .k = seq_len }, dt); - return pad_mask.broad(pad_mask_shape); - } - - /// Restrict global attn mask to a sliding window. - /// Output shapes follows zml.nn.sdpa convention: .{ .b, .q, .k } - pub fn localAttnMask(global_mask: Tensor, window_size: u32) Tensor { - const mask_shape = global_mask.shape(); - - // Calculate distance between positions - const rows = Tensor.iota(mask_shape, .q); - const cols = Tensor.iota(mask_shape, .k); - const distance = rows.sub(cols).abs(); - - // Note: we divide by two because the BERT local attention is symetric around the query token. - // Create sliding window mask (1 for positions within window, 0 outside) - const window_mask = distance.cmp(.LE, Tensor.scalar(@divExact(window_size, 2), .i32)); - const minus_inf = Tensor.constant(mask_shape, mask_shape.dtype().minValue()); - return window_mask.select(global_mask, minus_inf); - } -}; - -pub const ModernBertPredictionHead = struct { - dense: zml.nn.Linear, - norm: zml.nn.LayerNorm, - - pub fn forward(self: ModernBertPredictionHead, hidden_states: Tensor) Tensor { - const dense_output: Tensor = zml.call(self.dense, .forward, .{hidden_states}); - - const activated_output = dense_output.gelu(); - - return zml.call(self.norm, .forward, .{activated_output}); - } -}; - -pub const ModernBertEmbeddings = struct { - tok_embeddings: zml.nn.TokenEmbedding, - norm: zml.nn.LayerNorm, - - pub fn forward(self: ModernBertEmbeddings, input_ids: Tensor) Tensor { - // Perform tok_embeddings - const hidden_states = zml.call(self.tok_embeddings, .forward, .{input_ids}); - - // Perform norm - return zml.call(self.norm, .forward, .{hidden_states}); - } -}; - -pub const ModernBertEncoderLayer = struct { - attn_norm: ?zml.nn.LayerNorm = null, - attn: ModernBertAttention, - mlp_norm: zml.nn.LayerNorm, - mlp: ModernBertMLP, - - pub fn forward( - self: ModernBertEncoderLayer, - hidden_states: Tensor, - global_mask: Tensor, - local_mask: Tensor, - ) Tensor { - const attn_norm_output = if (self.attn_norm) |attn_norm| - zml.call(attn_norm, .forward, .{hidden_states}) - else - hidden_states; - - const attn_output: Tensor = zml.call(self.attn, .forward, .{ - attn_norm_output, - global_mask, - local_mask, - }); - - var output = hidden_states.add(attn_output); - - const mlp_norm_output: Tensor = zml.call(self.mlp_norm, .forward, .{output}); - const mlp_output = zml.call(self.mlp, .forward, .{mlp_norm_output}); - output = output.add(mlp_output); - - return output; - } -}; - -/// Performs multi-headed self attention on a batch of unpadded sequences. -/// -/// If Flash Attention 2 is installed, this module uses Flash Attention to improve throughput. -/// If Flash Attention 2 is not installed, the implementation will use SDPA, -pub const ModernBertAttention = struct { - Wqkv: zml.nn.Linear, - Wo: zml.nn.Linear, - is_global_attention: bool = false, - num_heads: i64 = undefined, - - /// sdpa_attention_forward - pub fn forward( - self: ModernBertAttention, - hidden_states: Tensor, - global_mask: Tensor, - local_mask: Tensor, - ) Tensor { - const batch_size = hidden_states.shape().dim(0); - const seq_length = hidden_states.shape().dim(1); - const hidden_size = hidden_states.shape().dim(2); - const num_heads = self.num_heads; - const head_dim = @divExact(hidden_size, num_heads); - - // Project to query, key, value - { batch_size, seq_len, 3 * num_heads * head_dim } - var qkv: Tensor = zml.call(self.Wqkv, .forward, .{hidden_states}); - - // Reshape to { batch_size, seq_len, 3, num_heads, head_dim } - qkv = qkv.reshape(.{ batch_size, seq_length, 3, num_heads, head_dim }).withTags(.{ .b, .s, .chunk, .h, .hd }); - - // Split into query, key, value tensors - each { batch_size, seq_length, num_heads, head_dim } - var q, var k, var v = qkv.chunkExact(.chunk, 3); - q = q.squeeze(.chunk); - k = k.squeeze(.chunk); - v = v.squeeze(.chunk); - - // Apply rotary position embeddings (RoPE) - // Layer 0, 3, 6, 9, 12 ... use global RoPE - // Layer 1, 2, 4, 5, 7, 8, 10, 11 ... use local RoPE - const rope_opts = zml.nn.RopeOpts{ - .layout = .sequential, - .freq_base = if (self.is_global_attention) 160_000 else 10_000, - }; - - q = zml.nn.rope(q, null, rope_opts); - k = zml.nn.rope(k, null, rope_opts); - - // rename dimensions for sdpa - q = q.rename(.{ .s = .q }); - k = k.rename(.{ .s = .k }); - v = v.rename(.{ .s = .k }); - - // Scaled dot product attention - const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = if (self.is_global_attention) global_mask else local_mask }); - const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); - - // Final projection - return zml.call(self.Wo, .forward, .{attn}); - } -}; - -/// Switch out the old MLP layers for GeGLU layers, improving on the original BERT’s GeLU activation function. -/// -/// The GeGLU activation function is a combination of the Gated Linear Unit (GLU) and the Gaussian Error Linear Unit (GeLU). -/// -/// see: https://paperswithcode.com/method/geglu -pub const ModernBertMLP = struct { - Wi: zml.nn.Linear, - Wo: zml.nn.Linear, - - pub fn forward(self: ModernBertMLP, hidden_states: Tensor) Tensor { - // Perform Wi - const wi_output: Tensor = zml.call(self.Wi, .forward, .{hidden_states}); - - // Split into input and gate tensors along the last dimension - const input, const gate = wi_output.chunkExact(-1, 2); - - // Apply activation - const activated_input = input.gelu().mul(gate); - - // Perform Wo - return zml.call(self.Wo, .forward, .{activated_input}); - } -}; diff --git a/examples/modernbert/test.zig b/examples/modernbert/test.zig deleted file mode 100644 index 5a6d4496..00000000 --- a/examples/modernbert/test.zig +++ /dev/null @@ -1,238 +0,0 @@ -const clap = @import("clap"); -const std = @import("std"); -const zml = @import("zml"); -const async = @import("async"); -const log = std.log; -const Tensor = zml.Tensor; -const modernbert_module = @import("modernbert.zig"); -const ModernBertOptions = modernbert_module.ModernBertOptions; - -const params = clap.parseParamsComptime( - \\--help print this help - \\--model model weights path - \\--activations model activations path -); - -fn printUsageAndExit(stderr: anytype) noreturn { - stderr.print("usage: ", .{}) catch {}; - clap.usage(stderr, clap.Help, ¶ms) catch {}; - stderr.print("\n", .{}) catch {}; - std.process.exit(0); -} -pub fn main() !void { - try async.AsyncThread.main(std.heap.c_allocator, asyncMain); -} - -pub fn asyncMain() !void { - // Short lived allocations - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - - const allocator = gpa.allocator(); - const stderr = std.io.getStdErr().writer(); - - // Read CLI arguments - const parsers = comptime .{ - .PATH = clap.parsers.string, - }; - var diag: clap.Diagnostic = .{}; - var res = clap.parse(clap.Help, ¶ms, parsers, .{ - .diagnostic = &diag, - .allocator = allocator, - }) catch |err| { - try diag.report(stderr, err); - try printUsageAndExit(stderr); - }; - defer res.deinit(); - - if (res.args.help != 0) { - try clap.help(stderr, clap.Help, ¶ms, .{}); - return; - } - - const model_file = res.args.model orelse { - stderr.print("Error: missing --model=...\n\n", .{}) catch {}; - printUsageAndExit(stderr); - unreachable; - }; - const activations_file = res.args.activations orelse { - stderr.print("Error: missing --activations=...\n\n", .{}) catch {}; - printUsageAndExit(stderr); - unreachable; - }; - - // Initialize the ZML context - var context = try zml.Context.init(); - defer context.deinit(); - - // Auto-select platform - const compute_platform = context.autoPlatform(.{}); - log.info("Selected platform: {s}", .{@tagName(compute_platform.target)}); - - // Create a dedicated memory arena for model-related allocations (dedicated to model shapes and weights) - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const model_arena = arena_state.allocator(); - - // Load the model weights file and parse its structure (shape) - var weights_file = try zml.aio.detectFormatAndOpen(allocator, model_file); - defer weights_file.deinit(); - log.info("Model contains {d} layers. Loaded from: {s}", .{ weights_file.buffers.count(), model_file }); - - // Load the activation data file - const activations = try zml.aio.torch.open(model_arena, activations_file); - defer activations.deinit(); - log.info("Found {} activations in {s}", .{ activations.buffers.count(), activations_file }); - - // Initialize model - var model = try zml.aio.populateModel( - modernbert_module.ModernBertForMaskedLM, - model_arena, - weights_file, - ); - - const modernbert_base_options: modernbert_module.ModernBertOptions = .{ - .num_attention_heads = 12, - .tie_word_embeddings = true, - .pad_token = 50283, - .local_attention = 128, - }; - model.init(modernbert_base_options); - - // Load model weights - const model_weights = try zml.aio.loadModelBuffers( - modernbert_module.ModernBertForMaskedLM, - model, - weights_file, - model_arena, - compute_platform, - ); - - // Test implementation - try testImplementation(compute_platform, model, model_weights, activations); -} - -fn testImplementation( - compute_platform: zml.Platform, - model: modernbert_module.ModernBertForMaskedLM, - model_weights: zml.Bufferized(modernbert_module.ModernBertForMaskedLM), - activations: zml.aio.BufferStore, -) !void { - try zml.testing.testLayer( - compute_platform, - activations, - "model.model.embeddings.tok_embeddings", - model.model.embeddings.tok_embeddings, - model_weights.model.embeddings.tok_embeddings, - 1e-6, - ); - - try zml.testing.testLayer( - compute_platform, - activations, - "model.model.embeddings.norm", - model.model.embeddings.norm, - model_weights.model.embeddings.norm, - 1e-3, - ); - - try zml.testing.testLayer( - compute_platform, - activations, - "model.model.embeddings", - model.model.embeddings, - model_weights.model.embeddings, - 1e-3, - ); - - try zml.testing.testLayer( - compute_platform, - activations, - "model.model.final_norm", - model.model.final_norm, - model_weights.model.final_norm, - 1e-5, - ); - - try zml.testing.testLayer( - compute_platform, - activations, - "model.model.layers.2.mlp", - model.model.layers[2].mlp, - model_weights.model.layers[2].mlp, - 2e-3, - ); - - try zml.testing.testLayer( - compute_platform, - activations, - "model.model.layers.2.mlp_norm", - model.model.layers[2].mlp_norm, - model_weights.model.layers[2].mlp_norm, - 1e-4, - ); - - try zml.testing.testLayer( - compute_platform, - activations, - "model.model.layers.2.attn", - model.model.layers[2].attn, - model_weights.model.layers[2].attn, - 1e-6, - ); - - try zml.testing.testLayer( - compute_platform, - activations, - "model.model.layers.2", - model.model.layers[2], - model_weights.model.layers[2], - 2e-3, - ); - - try zml.testing.testLayer( - compute_platform, - activations, - "model.model.layers.3.attn", - model.model.layers[3].attn, - model_weights.model.layers[3].attn, - 1e-5, - ); - - try zml.testing.testLayer( - compute_platform, - activations, - "model.model", - model.model, - model_weights.model, - 1e-2, - ); - - const TiedDecoder = struct { - weight: Tensor, - bias: Tensor, - - pub fn forward(self: @This(), head_outputs: Tensor) Tensor { - const results = head_outputs.withTags(.{ .b, .s, .d }).dot(self.weight.withTags(.{ .voc, .d }), .{.d}); - return results.add(self.bias.withTags(.{.voc}).broad(results.shape())); - } - }; - - try zml.testing.testLayer( - compute_platform, - activations, - "model.decoder", - TiedDecoder{ .weight = model.decoder.weight orelse model.model.embeddings.tok_embeddings.weight, .bias = model.decoder.bias }, - .{ .weight = model_weights.model.embeddings.tok_embeddings.weight, .bias = model_weights.decoder.bias }, - 1e-3, - ); - - try zml.testing.testLayer( - compute_platform, - activations, - "model.head", - model.head, - model_weights.head, - 0.1, // TODO: too high tolerance - ); -} From 31c9357a371eb8edc8c1f7429ba6aef14cf63d41 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 6 Jan 2026 10:49:45 +0000 Subject: [PATCH 201/881] workspace: remove clap from MODULE --- MODULE.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index db3aa426..146ca13b 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -112,7 +112,7 @@ use_repo(zls, "zls_toolchains") register_toolchains("@zls_toolchains//:all") non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") -use_repo(non_module_deps, "arocc", "com_github_hejsil_clap", "com_google_sentencepiece", "mnist", "org_swig_swig", "translate-c", "xla") +use_repo(non_module_deps, "arocc", "com_google_sentencepiece", "mnist", "org_swig_swig", "translate-c", "xla") xla = use_extension("//third_party/xla:xla.bzl", "xla") use_repo( From 84e8b281310669ae38a26d96b51bcf696c3efdcf Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 6 Jan 2026 10:51:18 +0000 Subject: [PATCH 202/881] examples: remove outdated examples --- examples/callback/BUILD.bazel | 11 -- examples/callback/main.zig | 214 ------------------------------ examples/simple_layer/BUILD.bazel | 78 ----------- examples/simple_layer/main.zig | 98 -------------- 4 files changed, 401 deletions(-) delete mode 100644 examples/callback/BUILD.bazel delete mode 100644 examples/callback/main.zig delete mode 100644 examples/simple_layer/BUILD.bazel delete mode 100644 examples/simple_layer/main.zig diff --git a/examples/callback/BUILD.bazel b/examples/callback/BUILD.bazel deleted file mode 100644 index 3f55040f..00000000 --- a/examples/callback/BUILD.bazel +++ /dev/null @@ -1,11 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_binary") - -zig_binary( - name = "callback", - main = "main.zig", - deps = [ - "@zml//async", - "@zml//zml", - "@zml//runtimes", - ], -) diff --git a/examples/callback/main.zig b/examples/callback/main.zig deleted file mode 100644 index 5f207001..00000000 --- a/examples/callback/main.zig +++ /dev/null @@ -1,214 +0,0 @@ -const std = @import("std"); - -const async = @import("async"); -const runtimes = @import("runtimes"); -const zml = @import("zml"); -const cu = zml.platform_specific; - -pub const std_options: std.Options = .{ - .log_level = .info, - .logFn = async.logFn(std.log.defaultLog), -}; - -const log = std.log.scoped(.@"examples/custom_call"); - -/// Demonstration of the custom_call mechanism. -/// -/// * defines a function to compute the grayscale version of an image. -/// * using simple for loop on CPU -/// * using a custom PTX kernel on Cuda, with manual cuLaunchKernel -/// * Use `zml.customCall` to create an executable calling our callback and our kernel. -pub const GrayScale = struct { - // Mandatory fields to work with ZML custom call api - pub var type_id: zml.pjrt.ffi.TypeId = undefined; - pub const callback_config: zml.callback.Config = .{}; - platform: zml.Platform, - - // Custom field to store our cuda module and function. - cu_data: [2]*anyopaque, - - // Set by ZML before `call` is entered - results: [1]zml.Buffer = undefined, - stream: *zml.pjrt.Stream = undefined, - - pub fn init(platform: zml.Platform) !GrayScale { - var cu_data: [2]*anyopaque = undefined; - if (comptime runtimes.isEnabled(.cuda)) { - var module: cu.CUmodule = undefined; - try cuda.check(cuda.moduleLoadData.?(&module, grayscale_ptx)); - log.info("Loaded Grayscale cuda module", .{}); - var function: cu.CUfunction = undefined; - try cuda.check(cuda.moduleGetFunction.?(&function, module, "rgba_to_grayscale")); - log.info("Found Grayscale cuda function", .{}); - cu_data = .{ module.?, function.? }; - } - return .{ - .platform = platform, - .cu_data = cu_data, - }; - } - - pub fn deinit(self: *GrayScale) void { - if (comptime runtimes.isEnabled(.cuda)) { - const module: cu.CUmodule = @ptrCast(self.cu_data[0]); - _ = cuda.moduleUnload.?(module); - } - } - - pub fn call(self: *GrayScale, rgb_d: zml.Buffer) !void { - switch (self.platform.target) { - .cpu => grayScaleCpu(rgb_d, self.results[0]), - // Only try to compile `grayScaleCuda` if we have cuda symbols. - .cuda => if (comptime runtimes.isEnabled(.cuda)) - try self.grayScaleCuda(rgb_d, self.results[0]) - else - unreachable, - else => @panic("Platform not supported"), - } - } - - pub fn grayScaleCpu(rgb_d: zml.Buffer, gray_d: zml.Buffer) void { - const rgb_h = rgb_d.asHostBuffer().items(u8); - const gray_h = gray_d.asHostBuffer().mutItems(u8); - - for (gray_h, 0..) |*gray, i| { - const px = rgb_h[i * 3 .. i * 3 + 3]; - const R: u32 = @intCast(px[0]); - const G: u32 = @intCast(px[1]); - const B: u32 = @intCast(px[2]); - const gray_u32: u32 = @divFloor(299 * R + 587 * G + 114 * B, 1000); - gray.* = @intCast(gray_u32); - } - } - - pub fn grayScaleCuda(self: GrayScale, rgb_d: zml.Buffer, gray_d: zml.Buffer) !void { - var args: [2:null]?*anyopaque = .{ rgb_d.devicePtr(), gray_d.devicePtr() }; - // This is a naive kernel with one block per pixel. - try cuda.check(cuda.launchKernel.?( - @ptrCast(self.cu_data[1]), // function - @intCast(rgb_d.shape().count() / 3), // num blocks x - 1, // num blocks y - 1, // num blocks z - 1, // num grids x - 1, // num grids y - 1, // num grids z - 0, // shared mem - @ptrCast(self.stream), - &args, - null, - )); - // Note: no explicit synchronization, we just enqueue work in the stream. - } - - const cuda = struct { - // Here we leverage ZML sandboxing to access cuda symbols and their definitions. - const moduleLoadData = @extern(*const @TypeOf(cu.cuModuleLoadData), .{ .name = "cuModuleLoadData", .linkage = .weak }); - const moduleUnload = @extern(*const @TypeOf(cu.cuModuleUnload), .{ .name = "cuModuleUnload", .linkage = .weak }); - const moduleGetFunction = @extern(*const @TypeOf(cu.cuModuleGetFunction), .{ .name = "cuModuleGetFunction", .linkage = .weak }); - const launchKernel = @extern(*const @TypeOf(cu.cuLaunchKernel), .{ .name = "cuLaunchKernel", .linkage = .weak }); - - pub fn check(result: cu.CUresult) error{CudaError}!void { - if (result == cu.CUDA_SUCCESS) return; - std.log.err("cuda error: {}", .{result}); - return error.CudaError; - } - }; -}; - -pub fn grayscale(rgb: zml.Tensor) zml.Tensor { - const gray_shape = rgb.shape().setDim(0, @divExact(rgb.dim(0), 3)); - const result = zml.callback.call(GrayScale, .{rgb.print()}, &.{gray_shape}); - return result[0]; -} - -pub fn main() !void { - try async.AsyncThread.main(std.heap.smp_allocator, asyncMain); -} - -pub fn asyncMain() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - var context = try zml.Context.init(); - defer context.deinit(); - - const platform = context.autoPlatform(.{}); - context.printAvailablePlatforms(platform); - - // Register our custom call - try zml.callback.register(GrayScale, platform); - - // Compile MLIR code containing our custom call. - const rgb_shape = zml.Shape.init(.{12 * 3}, .u8); - const exe = try zml.compileFn(allocator, grayscale, .{rgb_shape}, platform); - defer exe.deinit(); - - // Provide runtime information needed by our custom call. - var gray_op: GrayScale = try .init(platform); - defer gray_op.deinit(); - try exe.bind(GrayScale, &gray_op); - - // Load data and run the executable. - const rgb_h: [12][3]u8 = @splat(.{ 0xFF, 0xAA, 0x00 }); - const rgb_d = try zml.Buffer.fromBytes(platform, rgb_shape, @ptrCast(&rgb_h)); - defer rgb_d.deinit(); - - var gray_d: zml.Buffer = exe.call(.{rgb_d}); - defer gray_d.deinit(); - - // Inspect results - const gray_h = try gray_d.toHostAlloc(allocator); - defer gray_h.deinit(allocator); - - std.debug.print("Grayscale conversion of {any} -> {d}\n", .{ rgb_h, gray_h }); - try std.testing.expectEqualSlices(u8, &@as([12]u8, @splat(0xB0)), gray_h.items(u8)); -} - -// Compiled with Zig and a little help from `https://github.com/gwenzek/cudaz` -const grayscale_ptx = - \\.version 4.0 - \\.target sm_32 - \\.address_size 64 - \\ - \\.visible .entry rgba_to_grayscale( - \\ .param .align 8 .b8 rgba_to_grayscale_param_0[16], - \\ .param .align 8 .b8 rgba_to_grayscale_param_1[16] - \\) - \\{ - \\ .reg .pred %p<2>; - \\ .reg .b16 %rs<4>; - \\ .reg .b32 %r<13>; - \\ .reg .b64 %rd<9>; - \\ - \\ ld.param.u64 %rd5, [rgba_to_grayscale_param_1+8]; - \\ mov.u32 %r1, %tid.x; - \\ mov.u32 %r3, %ntid.x; - \\ mov.u32 %r2, %ctaid.x; - \\ mad.lo.s32 %r4, %r2, %r3, %r1; - \\ cvt.u64.u32 %rd1, %r4; - \\ setp.gt.u64 %p1, %rd5, %rd1; - \\ @%p1 bra $L__BB0_2; - \\ bra.uni $L__BB0_1; - \\$L__BB0_2: - \\ ld.param.u64 %rd4, [rgba_to_grayscale_param_1]; - \\ ld.param.u64 %rd2, [rgba_to_grayscale_param_0]; - \\ cvt.u32.u64 %r5, %rd1; - \\ mul.lo.s32 %r6, %r5, 3; - \\ cvt.u64.u32 %rd6, %r6; - \\ add.s64 %rd7, %rd2, %rd6; - \\ ld.u8 %rs1, [%rd7]; - \\ ld.u8 %rs2, [%rd7+1]; - \\ ld.u8 %rs3, [%rd7+2]; - \\ mul.wide.u16 %r7, %rs1, 299; - \\ mul.wide.u16 %r8, %rs2, 587; - \\ add.s32 %r9, %r8, %r7; - \\ mul.wide.u16 %r10, %rs3, 114; - \\ add.s32 %r11, %r9, %r10; - \\ mul.hi.u32 %r12, %r11, 4294968; - \\ add.s64 %rd8, %rd4, %rd1; - \\ st.u8 [%rd8], %r12; - \\$L__BB0_1: - \\ ret; - \\} -; diff --git a/examples/simple_layer/BUILD.bazel b/examples/simple_layer/BUILD.bazel deleted file mode 100644 index 754816cf..00000000 --- a/examples/simple_layer/BUILD.bazel +++ /dev/null @@ -1,78 +0,0 @@ -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") -load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") -load("@bazel_skylib//rules:build_test.bzl", "build_test") -load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") -load("@rules_zig//zig:defs.bzl", "zig_binary") - -zig_binary( - name = "simple_layer", - main = "main.zig", - deps = [ - "//zml", - ], -) - -# Manifest created from the simple_layer binary and friends -mtree_spec( - name = "mtree", - srcs = [":simple_layer"], -) - -# Create a tar archive from the above manifest -tar( - name = "archive", - srcs = [":simple_layer"], - args = [ - "--options", - ",".join([ - "zstd:compression-level=9", - "zstd:threads=16", - ]), - ], - compress = "zstd", - mtree = ":mtree", -) - -# The actual docker image, with entrypoint, created from tar archive -oci_image( - name = "image_", - base = "@distroless_cc_debian12", - entrypoint = ["/{}/simple_layer".format(package_name())], - target_compatible_with = [ - "@platforms//os:linux", - ], - tars = [":archive"], -) - -# We always want to create the image for Linux -platform_transition_filegroup( - name = "image", - srcs = [":image_"], - tags = ["manual"], - target_platform = "//platforms:linux_amd64", -) - -# Load will immediatly load the image (eg: docker load) -oci_load( - name = "load", - image = ":image", - repo_tags = [ - "distroless/simple_layer:latest", - ], - tags = ["manual"], -) - -# Bazel target for pushing the Linux image to the docker registry -oci_push( - name = "push", - image = ":image", - remote_tags = ["latest"], - # override with -- --repository foo.bar/org/image - repository = "index.docker.io/zmlai/simple_layer", - tags = ["manual"], -) - -build_test( - name = "test", - targets = [":simple_layer"], -) diff --git a/examples/simple_layer/main.zig b/examples/simple_layer/main.zig deleted file mode 100644 index ae46ccbb..00000000 --- a/examples/simple_layer/main.zig +++ /dev/null @@ -1,98 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); - -/// Model definition -const Layer = struct { - bias: ?zml.Tensor = null, - weight: zml.Tensor, - - pub fn forward(self: Layer, x: zml.Tensor) zml.Tensor { - var y = self.weight.mul(x); - if (self.bias) |bias| { - y = y.add(bias); - } - return y; - } -}; - -pub fn main() !void { - // Short lived allocations - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - // Io runtime - var threaded: std.Io.Threaded = .init(allocator); - defer threaded.deinit(); - - const io = threaded.io(); - - // Initialize ZML globally - zml.init(); - defer zml.deinit(); - - // Auto-select platform - const platform: zml.Platform = try .auto(io, .{}); - - // Initialize model instance and input - const model: Layer = .{ - .weight = zml.Tensor.init(.{4}, .f32), - .bias = zml.Tensor.init(.{4}, .f32), - }; - const input: zml.Tensor = .init(.{4}, .f32); - - // Compile our executable - var exe = try platform.compileModel(allocator, io, Layer.forward, model, .{input}); - defer exe.deinit(); - - // Our weights and bias to use - const weights_data = [4]f32{ 2.0, 2.0, 2.0, 2.0 }; - const weights_slice: zml.ConstSlice = .init(model.weight.shape(), std.mem.sliceAsBytes(&weights_data)); - var weights_buffer: zml.Buffer = try .fromSlice(io, platform, weights_slice); - - const bias_data = [4]f32{ 1.0, 2.0, 3.0, 4.0 }; - const bias_slice: zml.ConstSlice = .init(model.bias.?.shape(), std.mem.sliceAsBytes(&bias_data)); - var bias_buffer: zml.Buffer = try .fromSlice(io, platform, bias_slice); - - // Don't forget to deinit buffers - defer weights_buffer.deinit(); - defer bias_buffer.deinit(); - - // Init the bufferized version of the model - const model_buffers: zml.Bufferized(Layer) = .{ - .weight = weights_buffer, - .bias = bias_buffer, - }; - - // Prepare an input buffer - var input_data = [4]f32{ 5.0, 5.0, 5.0, 5.0 }; - const input_slice: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&input_data)); - var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice); - defer input_buffer.deinit(); - - // Create the Arguments and Results struct to interact with the executable - var args = try exe.args(allocator); - defer args.deinit(allocator); - - var results = try exe.results(allocator); - defer results.deinit(allocator); - - // Set the arguments from the bufferized version of the model and the input buffer - args.set(.{ model_buffers, input_buffer }); - - // Call our executable module - exe.call(args, &results); - - // Retrieve the output - var result = results.get(zml.Buffer); - defer result.deinit(); - - // fetch the result to CPU memory - const cpu_result = try result.toSliceAlloc(allocator, io); - defer cpu_result.free(allocator); - std.debug.print( - "\nThe result of {d} * {d} + {d} = {d}\n", - .{ weights_slice, input_slice, bias_slice, cpu_result }, - ); -} From 2ba6d73af287a675eec02cd885abeeb8e4fc7d3c Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 6 Jan 2026 10:52:28 +0000 Subject: [PATCH 203/881] zml: expose stdx from inside zml --- examples/benchmark/BUILD.bazel | 5 +---- examples/benchmark/main.zig | 2 +- examples/llama/BUILD.bazel | 9 ++------- examples/llama/llama.zig | 2 +- examples/llama/main.zig | 2 +- examples/llama/test.zig | 8 ++++---- examples/mnist/BUILD.bazel | 5 +---- examples/mnist/mnist.zig | 2 +- zml/zml.zig | 1 + 9 files changed, 13 insertions(+), 23 deletions(-) diff --git a/examples/benchmark/BUILD.bazel b/examples/benchmark/BUILD.bazel index ce2f0194..72a2dc6c 100644 --- a/examples/benchmark/BUILD.bazel +++ b/examples/benchmark/BUILD.bazel @@ -4,10 +4,7 @@ load("@rules_zig//zig:defs.bzl", "zig_binary") zig_binary( name = "benchmark", main = "main.zig", - deps = [ - "//stdx", - "//zml", - ], + deps = ["//zml"], ) build_test( diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index d678551e..98162058 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -1,8 +1,8 @@ const std = @import("std"); const log = std.log; -const stdx = @import("stdx"); const zml = @import("zml"); +const stdx = zml.stdx; pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { return a.dot(b, .k); diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel index 9a43f707..39d3cd21 100644 --- a/examples/llama/BUILD.bazel +++ b/examples/llama/BUILD.bazel @@ -6,15 +6,10 @@ load("@rules_zig//zig:defs.bzl", "zig_binary") zig_binary( name = "llama", - srcs = [ - "llama.zig", - ], + srcs = ["llama.zig"], main = "main.zig", visibility = ["//visibility:public"], - deps = [ - "//stdx", - "//zml", - ], + deps = ["//zml"], ) mtree_spec( diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 5d89e100..9a2a9937 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -1,11 +1,11 @@ const std = @import("std"); const testing = std.testing; -const stdx = @import("stdx"); const zml = @import("zml"); const Buffer = zml.Buffer; const Tensor = zml.Tensor; const ShapeOf = zml.ShapeOf; +const stdx = zml.stdx; const log = std.log.scoped(.llama); diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 4d457c72..3be7dbcb 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -1,10 +1,10 @@ const std = @import("std"); -const stdx = @import("stdx"); const zml = @import("zml"); const Buffer = zml.Buffer; const Tensor = zml.Tensor; const ShapeOf = zml.ShapeOf; +const stdx = zml.stdx; const llama = @import("llama.zig"); const LlamaLM = llama.LlamaLM; diff --git a/examples/llama/test.zig b/examples/llama/test.zig index 0e1edba0..6c4ebc84 100644 --- a/examples/llama/test.zig +++ b/examples/llama/test.zig @@ -1,14 +1,14 @@ -const async = @import("async"); const std = @import("std"); -const stdx = @import("stdx"); + +const async = @import("async"); const zml = @import("zml"); +const Tensor = zml.Tensor; +const stdx = zml.stdx; const flags = stdx.flags; const llama_mod = @import("./llama.zig"); const LlamaLM = llama_mod.LlamaLM; -const Tensor = zml.Tensor; - pub fn main() !void { try async.AsyncThread.main(std.heap.c_allocator, asyncMain); } diff --git a/examples/mnist/BUILD.bazel b/examples/mnist/BUILD.bazel index 719b4f73..787d30f4 100644 --- a/examples/mnist/BUILD.bazel +++ b/examples/mnist/BUILD.bazel @@ -12,10 +12,7 @@ zig_binary( "@mnist//:t10k-images.idx3-ubyte", ], main = "mnist.zig", - deps = [ - "//stdx", - "//zml", - ], + deps = ["//zml"], ) build_test( diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 0e81a952..1ffd65e0 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -1,7 +1,7 @@ const std = @import("std"); -const stdx = @import("stdx"); const zml = @import("zml"); +const stdx = zml.stdx; const log = std.log.scoped(.mnist); diff --git a/zml/zml.zig b/zml/zml.zig index 13b60c72..7fd9e07e 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -9,6 +9,7 @@ const std = @import("std"); const c = @import("c"); pub const pjrt = @import("pjrt"); const runfiles = @import("runfiles"); +pub const stdx = @import("stdx"); pub const tokenizer = @import("zml/tokenizer"); pub const Buffer = @import("buffer.zig").Buffer; From f3a2992365ce5f19321e793b54f122896050a95f Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 6 Jan 2026 12:30:12 +0000 Subject: [PATCH 204/881] zml: disable XLA logs by default, unless overriden --- runtimes/cuda/cuda.zig | 6 ------ zml/platform.zig | 10 ++++++++++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/runtimes/cuda/cuda.zig b/runtimes/cuda/cuda.zig index 4a255557..26796e37 100644 --- a/runtimes/cuda/cuda.zig +++ b/runtimes/cuda/cuda.zig @@ -25,10 +25,6 @@ fn hasCudaPathInLDPath() bool { return std.ascii.indexOfIgnoreCase(std.mem.span(ldLibraryPath), nvidiaLibsPath) != null; } -pub fn disableXlaLogs() void { - _ = c.setenv("TF_CPP_MIN_LOG_LEVEL", "3", 1); -} - fn setupXlaGpuCudaDirFlag(allocator: std.mem.Allocator, sandbox: []const u8) !void { const xla_flags = std.process.getEnvVarOwned(allocator, "XLA_FLAGS") catch ""; const new_xla_flagsZ = try std.fmt.allocPrintSentinel(allocator, "{s} --xla_gpu_cuda_data_dir={s}", .{ xla_flags, sandbox }, 0); @@ -70,8 +66,6 @@ pub fn load(io: std.Io) !*const pjrt.Api { // See https://github.com/openxla/xla/issues/21428 try setupXlaGpuCudaDirFlag(arena.allocator(), sandbox_path); - disableXlaLogs(); - { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libnvtx3interop.so" }); diff --git a/zml/platform.zig b/zml/platform.zig index f135a1a9..e03783c1 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -1,5 +1,6 @@ const std = @import("std"); +const c = @import("c"); const pjrt = @import("pjrt"); const pjrtx = @import("pjrt"); const runtimes = @import("runtimes"); @@ -33,9 +34,18 @@ fn StaticPlatformMap(comptime E: type, comptime T: type) type { var api_map: StaticPlatformMap(Target, ?*const pjrt.Api) = .{}; +fn disableXlaLogs() void { + _ = c.setenv( + "TF_CPP_MIN_LOG_LEVEL", + std.posix.getenv("TF_CPP_MIN_LOG_LEVEL") orelse "3", + 1, + ); +} + fn loadOrGetApi(target: Target, io: std.Io) !*const pjrt.Api { return switch (target) { inline else => |tag| @field(api_map, @tagName(tag)) orelse b: { + disableXlaLogs(); const api = try runtimes.load(tag, io); @field(api_map, @tagName(tag)) = api; break :b api; From bb01106ace407dd3d958a2b20fbaa7c47de6b13f Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 6 Jan 2026 15:35:28 +0000 Subject: [PATCH 205/881] runtimes/rocm: bump PJRT plugin --- runtimes/rocm/rocm.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtimes/rocm/rocm.bzl b/runtimes/rocm/rocm.bzl index 96ff4690..0d392389 100644 --- a/runtimes/rocm/rocm.bzl +++ b/runtimes/rocm/rocm.bzl @@ -170,8 +170,8 @@ def _rocm_impl(mctx): http_archive( name = "libpjrt_rocm", build_file = "libpjrt_rocm.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v15.0.2/pjrt-rocm_linux-amd64.tar.gz", - sha256 = "b3f0dc66eec0f72835ba7fac66a88eef967f5caa2ba5cb3bb5ef7ee040d736cd", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v16.0.0/pjrt-rocm_linux-amd64.tar.gz", + sha256 = "6a70cf6a5a8330809f173072dd5fae5332c3ec8fba80acb9b77ddd97bda6572b", ) return mctx.extension_metadata( From fd44f8e63ea689b774020a45ef0bd980abd2abb2 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 6 Jan 2026 16:36:35 +0000 Subject: [PATCH 206/881] workspace: bump hf tokenizers and rules_rust --- MODULE.bazel | 4 ++-- MODULE.bazel.lock | 18 +++++++++--------- zml/tokenizer/hftokenizers/Cargo.lock | 4 ++-- zml/tokenizer/hftokenizers/Cargo.toml | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 146ca13b..6df1a6bb 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -140,14 +140,14 @@ use_repo(llvm, "llvm-project") rust = use_extension("@rules_rust//rust:extensions.bzl", "rust") rust.toolchain( - edition = "2021", + edition = "2024", extra_target_triples = [ "aarch64-apple-darwin", "x86_64-apple-darwin", "aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu", ], - versions = ["1.89.0"], + versions = ["1.92.0"], ) use_repo(rust, "rust_toolchains") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 378c9caa..1f3593cd 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1197,8 +1197,8 @@ "bzlTransitiveDigest": "NvjljZ+5m2nc1Qpf7ekjzs+FJrjYYY9DwXvwuXnippI=", "usagesDigest": "mup6mgSFatk34Q0znMZPpIEOFZyk9v3xf6jRIXEFTxc=", "recordedFileInputs": { - "@@//zml/tokenizer/hftokenizers/Cargo.lock": "d78ffab37ef69bf8ff45a4d686b25175f30e8fed4d8375612e84dcf6db0cfc9c", - "@@//zml/tokenizer/hftokenizers/Cargo.toml": "eeccbf5ff71c5f26b94cb289b3fc64c5fc6eb8708b815f39ce5f16bf225b6774" + "@@//zml/tokenizer/hftokenizers/Cargo.lock": "7fb683b6a36181a92d7564c8552b4fc5bad00229a41d520a4536c4e577c2b888", + "@@//zml/tokenizer/hftokenizers/Cargo.toml": "e46e67c2ae6792917dc1259d6ba5db389becc8bafa9fa1911bb3b6cec6017620" }, "recordedDirentsInputs": {}, "envVariables": { @@ -1216,9 +1216,9 @@ "repoRuleId": "@@rules_rust+//crate_universe:extensions.bzl%_generate_repo", "attributes": { "contents": { - "BUILD.bazel": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files(\n [\n \"cargo-bazel.json\",\n \"crates.bzl\",\n \"defs.bzl\",\n ] + glob(\n allow_empty = True,\n include = [\"*.bazel\"],\n ),\n)\n\nfilegroup(\n name = \"srcs\",\n srcs = glob(\n allow_empty = True,\n include = [\n \"*.bazel\",\n \"*.bzl\",\n ],\n ),\n)\n\n# Workspace Member Dependencies\nalias(\n name = \"tokenizers-0.21.4\",\n actual = \"@crates__tokenizers-0.21.4//:tokenizers\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"tokenizers\",\n actual = \"@crates__tokenizers-0.21.4//:tokenizers\",\n tags = [\"manual\"],\n)\n", + "BUILD.bazel": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files(\n [\n \"cargo-bazel.json\",\n \"crates.bzl\",\n \"defs.bzl\",\n ] + glob(\n allow_empty = True,\n include = [\"*.bazel\"],\n ),\n)\n\nfilegroup(\n name = \"srcs\",\n srcs = glob(\n allow_empty = True,\n include = [\n \"*.bazel\",\n \"*.bzl\",\n ],\n ),\n)\n\n# Workspace Member Dependencies\nalias(\n name = \"tokenizers-0.22.2\",\n actual = \"@crates__tokenizers-0.22.2//:tokenizers\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"tokenizers\",\n actual = \"@crates__tokenizers-0.22.2//:tokenizers\",\n tags = [\"manual\"],\n)\n", "alias_rules.bzl": "\"\"\"Alias that transitions its target to `compilation_mode=opt`. Use `transition_alias=\"opt\"` to enable.\"\"\"\n\nload(\"@rules_cc//cc:defs.bzl\", \"CcInfo\")\nload(\"@rules_rust//rust:rust_common.bzl\", \"COMMON_PROVIDERS\")\n\ndef _transition_alias_impl(ctx):\n # `ctx.attr.actual` is a list of 1 item due to the transition\n providers = [ctx.attr.actual[0][provider] for provider in COMMON_PROVIDERS]\n if CcInfo in ctx.attr.actual[0]:\n providers.append(ctx.attr.actual[0][CcInfo])\n return providers\n\ndef _change_compilation_mode(compilation_mode):\n def _change_compilation_mode_impl(_settings, _attr):\n return {\n \"//command_line_option:compilation_mode\": compilation_mode,\n }\n\n return transition(\n implementation = _change_compilation_mode_impl,\n inputs = [],\n outputs = [\n \"//command_line_option:compilation_mode\",\n ],\n )\n\ndef _transition_alias_rule(compilation_mode):\n return rule(\n implementation = _transition_alias_impl,\n provides = COMMON_PROVIDERS,\n attrs = {\n \"actual\": attr.label(\n mandatory = True,\n doc = \"`rust_library()` target to transition to `compilation_mode=opt`.\",\n providers = COMMON_PROVIDERS,\n cfg = _change_compilation_mode(compilation_mode),\n ),\n \"_allowlist_function_transition\": attr.label(\n default = \"@bazel_tools//tools/allowlists/function_transition_allowlist\",\n ),\n },\n doc = \"Transitions a Rust library crate to the `compilation_mode=opt`.\",\n )\n\ntransition_alias_dbg = _transition_alias_rule(\"dbg\")\ntransition_alias_fastbuild = _transition_alias_rule(\"fastbuild\")\ntransition_alias_opt = _transition_alias_rule(\"opt\")\n", - "defs.bzl": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\"\"\"\n# `crates_repository` API\n\n- [aliases](#aliases)\n- [crate_deps](#crate_deps)\n- [all_crate_deps](#all_crate_deps)\n- [crate_repositories](#crate_repositories)\n\n\"\"\"\n\nload(\"@bazel_tools//tools/build_defs/repo:git.bzl\", \"new_git_repository\")\nload(\"@bazel_tools//tools/build_defs/repo:http.bzl\", \"http_archive\")\nload(\"@bazel_tools//tools/build_defs/repo:utils.bzl\", \"maybe\")\nload(\"@bazel_skylib//lib:selects.bzl\", \"selects\")\nload(\"@rules_rust//crate_universe/private:local_crate_mirror.bzl\", \"local_crate_mirror\")\n\n###############################################################################\n# MACROS API\n###############################################################################\n\n# An identifier that represent common dependencies (unconditional).\n_COMMON_CONDITION = \"\"\n\ndef _flatten_dependency_maps(all_dependency_maps):\n \"\"\"Flatten a list of dependency maps into one dictionary.\n\n Dependency maps have the following structure:\n\n ```python\n DEPENDENCIES_MAP = {\n # The first key in the map is a Bazel package\n # name of the workspace this file is defined in.\n \"workspace_member_package\": {\n\n # Not all dependencies are supported for all platforms.\n # the condition key is the condition required to be true\n # on the host platform.\n \"condition\": {\n\n # An alias to a crate target. # The label of the crate target the\n # Aliases are only crate names. # package name refers to.\n \"package_name\": \"@full//:label\",\n }\n }\n }\n ```\n\n Args:\n all_dependency_maps (list): A list of dicts as described above\n\n Returns:\n dict: A dictionary as described above\n \"\"\"\n dependencies = {}\n\n for workspace_deps_map in all_dependency_maps:\n for pkg_name, conditional_deps_map in workspace_deps_map.items():\n if pkg_name not in dependencies:\n non_frozen_map = dict()\n for key, values in conditional_deps_map.items():\n non_frozen_map.update({key: dict(values.items())})\n dependencies.setdefault(pkg_name, non_frozen_map)\n continue\n\n for condition, deps_map in conditional_deps_map.items():\n # If the condition has not been recorded, do so and continue\n if condition not in dependencies[pkg_name]:\n dependencies[pkg_name].setdefault(condition, dict(deps_map.items()))\n continue\n\n # Alert on any miss-matched dependencies\n inconsistent_entries = []\n for crate_name, crate_label in deps_map.items():\n existing = dependencies[pkg_name][condition].get(crate_name)\n if existing and existing != crate_label:\n inconsistent_entries.append((crate_name, existing, crate_label))\n dependencies[pkg_name][condition].update({crate_name: crate_label})\n\n return dependencies\n\ndef crate_deps(deps, package_name = None):\n \"\"\"Finds the fully qualified label of the requested crates for the package where this macro is called.\n\n Args:\n deps (list): The desired list of crate targets.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()`.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if not deps:\n return []\n\n if package_name == None:\n package_name = native.package_name()\n\n # Join both sets of dependencies\n dependencies = _flatten_dependency_maps([\n _NORMAL_DEPENDENCIES,\n _NORMAL_DEV_DEPENDENCIES,\n _PROC_MACRO_DEPENDENCIES,\n _PROC_MACRO_DEV_DEPENDENCIES,\n _BUILD_DEPENDENCIES,\n _BUILD_PROC_MACRO_DEPENDENCIES,\n ]).pop(package_name, {})\n\n # Combine all conditional packages so we can easily index over a flat list\n # TODO: Perhaps this should actually return select statements and maintain\n # the conditionals of the dependencies\n flat_deps = {}\n for deps_set in dependencies.values():\n for crate_name, crate_label in deps_set.items():\n flat_deps.update({crate_name: crate_label})\n\n missing_crates = []\n crate_targets = []\n for crate_target in deps:\n if crate_target not in flat_deps:\n missing_crates.append(crate_target)\n else:\n crate_targets.append(flat_deps[crate_target])\n\n if missing_crates:\n fail(\"Could not find crates `{}` among dependencies of `{}`. Available dependencies were `{}`\".format(\n missing_crates,\n package_name,\n dependencies,\n ))\n\n return crate_targets\n\ndef all_crate_deps(\n normal = False, \n normal_dev = False, \n proc_macro = False, \n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Finds the fully qualified label of all requested direct crate dependencies \\\n for the package where this macro is called.\n\n If no parameters are set, all normal dependencies are returned. Setting any one flag will\n otherwise impact the contents of the returned list.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_dependency_maps = []\n if normal:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n if normal_dev:\n all_dependency_maps.append(_NORMAL_DEV_DEPENDENCIES)\n if proc_macro:\n all_dependency_maps.append(_PROC_MACRO_DEPENDENCIES)\n if proc_macro_dev:\n all_dependency_maps.append(_PROC_MACRO_DEV_DEPENDENCIES)\n if build:\n all_dependency_maps.append(_BUILD_DEPENDENCIES)\n if build_proc_macro:\n all_dependency_maps.append(_BUILD_PROC_MACRO_DEPENDENCIES)\n\n # Default to always using normal dependencies\n if not all_dependency_maps:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n\n dependencies = _flatten_dependency_maps(all_dependency_maps).pop(package_name, None)\n\n if not dependencies:\n if dependencies == None:\n fail(\"Tried to get all_crate_deps for package \" + package_name + \" but that package had no Cargo.toml file\")\n else:\n return []\n\n crate_deps = list(dependencies.pop(_COMMON_CONDITION, {}).values())\n for condition, deps in dependencies.items():\n crate_deps += selects.with_or({\n tuple(_CONDITIONS[condition]): deps.values(),\n \"//conditions:default\": [],\n })\n\n return crate_deps\n\ndef aliases(\n normal = False,\n normal_dev = False,\n proc_macro = False,\n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Produces a map of Crate alias names to their original label\n\n If no dependency kinds are specified, `normal` and `proc_macro` are used by default.\n Setting any one flag will otherwise determine the contents of the returned dict.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n dict: The aliases of all associated packages\n \"\"\"\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_aliases_maps = []\n if normal:\n all_aliases_maps.append(_NORMAL_ALIASES)\n if normal_dev:\n all_aliases_maps.append(_NORMAL_DEV_ALIASES)\n if proc_macro:\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n if proc_macro_dev:\n all_aliases_maps.append(_PROC_MACRO_DEV_ALIASES)\n if build:\n all_aliases_maps.append(_BUILD_ALIASES)\n if build_proc_macro:\n all_aliases_maps.append(_BUILD_PROC_MACRO_ALIASES)\n\n # Default to always using normal aliases\n if not all_aliases_maps:\n all_aliases_maps.append(_NORMAL_ALIASES)\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n\n aliases = _flatten_dependency_maps(all_aliases_maps).pop(package_name, None)\n\n if not aliases:\n return dict()\n\n common_items = aliases.pop(_COMMON_CONDITION, {}).items()\n\n # If there are only common items in the dictionary, immediately return them\n if not len(aliases.keys()) == 1:\n return dict(common_items)\n\n # Build a single select statement where each conditional has accounted for the\n # common set of aliases.\n crate_aliases = {\"//conditions:default\": dict(common_items)}\n for condition, deps in aliases.items():\n condition_triples = _CONDITIONS[condition]\n for triple in condition_triples:\n if triple in crate_aliases:\n crate_aliases[triple].update(deps)\n else:\n crate_aliases.update({triple: dict(deps.items() + common_items)})\n\n return select(crate_aliases)\n\n###############################################################################\n# WORKSPACE MEMBER DEPS AND ALIASES\n###############################################################################\n\n_NORMAL_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n _COMMON_CONDITION: {\n \"tokenizers\": Label(\"@crates//:tokenizers-0.21.4\"),\n },\n },\n}\n\n\n_NORMAL_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n _COMMON_CONDITION: {\n },\n },\n}\n\n\n_NORMAL_DEV_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_NORMAL_DEV_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEV_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEV_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_CONDITIONS = {\n \"aarch64-apple-darwin\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\"],\n \"aarch64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\"],\n \"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\",\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\": [],\n \"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\": [],\n \"cfg(any())\": [],\n \"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\": [],\n \"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\": [],\n \"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\": [],\n \"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\",\"@rules_rust//rust/platform:x86_64-apple-darwin\"],\n \"cfg(not(all(target_arch = \\\"arm\\\", target_os = \\\"none\\\")))\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\",\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\",\"@rules_rust//rust/platform:x86_64-apple-darwin\",\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"cfg(target_os = \\\"netbsd\\\")\": [],\n \"cfg(target_os = \\\"solaris\\\")\": [],\n \"cfg(target_os = \\\"vxworks\\\")\": [],\n \"cfg(windows)\": [],\n \"x86_64-apple-darwin\": [\"@rules_rust//rust/platform:x86_64-apple-darwin\"],\n \"x86_64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n}\n\n###############################################################################\n\ndef crate_repositories():\n \"\"\"A macro for defining repositories for all generated crates.\n\n Returns:\n A list of repos visible to the module through the module extension.\n \"\"\"\n maybe(\n http_archive,\n name = \"crates__ahash-0.8.12\",\n sha256 = \"5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ahash/0.8.12/download\"],\n strip_prefix = \"ahash-0.8.12\",\n build_file = Label(\"@crates//crates:BUILD.ahash-0.8.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__aho-corasick-1.1.3\",\n sha256 = \"8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/aho-corasick/1.1.3/download\"],\n strip_prefix = \"aho-corasick-1.1.3\",\n build_file = Label(\"@crates//crates:BUILD.aho-corasick-1.1.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__base64-0.13.1\",\n sha256 = \"9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/base64/0.13.1/download\"],\n strip_prefix = \"base64-0.13.1\",\n build_file = Label(\"@crates//crates:BUILD.base64-0.13.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__bitflags-2.9.1\",\n sha256 = \"1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/bitflags/2.9.1/download\"],\n strip_prefix = \"bitflags-2.9.1\",\n build_file = Label(\"@crates//crates:BUILD.bitflags-2.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__byteorder-1.5.0\",\n sha256 = \"1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/byteorder/1.5.0/download\"],\n strip_prefix = \"byteorder-1.5.0\",\n build_file = Label(\"@crates//crates:BUILD.byteorder-1.5.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__castaway-0.2.4\",\n sha256 = \"dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/castaway/0.2.4/download\"],\n strip_prefix = \"castaway-0.2.4\",\n build_file = Label(\"@crates//crates:BUILD.castaway-0.2.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__cc-1.2.33\",\n sha256 = \"3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/cc/1.2.33/download\"],\n strip_prefix = \"cc-1.2.33\",\n build_file = Label(\"@crates//crates:BUILD.cc-1.2.33.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__cfg-if-1.0.0\",\n sha256 = \"baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/cfg-if/1.0.0/download\"],\n strip_prefix = \"cfg-if-1.0.0\",\n build_file = Label(\"@crates//crates:BUILD.cfg-if-1.0.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__compact_str-0.9.0\",\n sha256 = \"3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/compact_str/0.9.0/download\"],\n strip_prefix = \"compact_str-0.9.0\",\n build_file = Label(\"@crates//crates:BUILD.compact_str-0.9.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-deque-0.8.6\",\n sha256 = \"9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-deque/0.8.6/download\"],\n strip_prefix = \"crossbeam-deque-0.8.6\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-deque-0.8.6.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-epoch-0.9.18\",\n sha256 = \"5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-epoch/0.9.18/download\"],\n strip_prefix = \"crossbeam-epoch-0.9.18\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-epoch-0.9.18.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-utils-0.8.21\",\n sha256 = \"d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-utils/0.8.21/download\"],\n strip_prefix = \"crossbeam-utils-0.8.21\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-utils-0.8.21.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling-0.20.10\",\n sha256 = \"6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling/0.20.10/download\"],\n strip_prefix = \"darling-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling_core-0.20.10\",\n sha256 = \"95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling_core/0.20.10/download\"],\n strip_prefix = \"darling_core-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling_core-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling_macro-0.20.10\",\n sha256 = \"d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling_macro/0.20.10/download\"],\n strip_prefix = \"darling_macro-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling_macro-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__dary_heap-0.3.7\",\n sha256 = \"04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/dary_heap/0.3.7/download\"],\n strip_prefix = \"dary_heap-0.3.7\",\n build_file = Label(\"@crates//crates:BUILD.dary_heap-0.3.7.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder-0.20.2\",\n sha256 = \"507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder/0.20.2/download\"],\n strip_prefix = \"derive_builder-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder_core-0.20.2\",\n sha256 = \"2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder_core/0.20.2/download\"],\n strip_prefix = \"derive_builder_core-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder_core-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder_macro-0.20.2\",\n sha256 = \"ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder_macro/0.20.2/download\"],\n strip_prefix = \"derive_builder_macro-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder_macro-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__either-1.13.0\",\n sha256 = \"60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/either/1.13.0/download\"],\n strip_prefix = \"either-1.13.0\",\n build_file = Label(\"@crates//crates:BUILD.either-1.13.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__esaxx-rs-0.1.10\",\n sha256 = \"d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/esaxx-rs/0.1.10/download\"],\n strip_prefix = \"esaxx-rs-0.1.10\",\n build_file = Label(\"@crates//crates:BUILD.esaxx-rs-0.1.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__fnv-1.0.7\",\n sha256 = \"3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/fnv/1.0.7/download\"],\n strip_prefix = \"fnv-1.0.7\",\n build_file = Label(\"@crates//crates:BUILD.fnv-1.0.7.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__getrandom-0.3.3\",\n sha256 = \"26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/getrandom/0.3.3/download\"],\n strip_prefix = \"getrandom-0.3.3\",\n build_file = Label(\"@crates//crates:BUILD.getrandom-0.3.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ident_case-1.0.1\",\n sha256 = \"b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ident_case/1.0.1/download\"],\n strip_prefix = \"ident_case-1.0.1\",\n build_file = Label(\"@crates//crates:BUILD.ident_case-1.0.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__itertools-0.14.0\",\n sha256 = \"2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/itertools/0.14.0/download\"],\n strip_prefix = \"itertools-0.14.0\",\n build_file = Label(\"@crates//crates:BUILD.itertools-0.14.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__itoa-1.0.14\",\n sha256 = \"d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/itoa/1.0.14/download\"],\n strip_prefix = \"itoa-1.0.14\",\n build_file = Label(\"@crates//crates:BUILD.itoa-1.0.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__libc-0.2.169\",\n sha256 = \"b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/libc/0.2.169/download\"],\n strip_prefix = \"libc-0.2.169\",\n build_file = Label(\"@crates//crates:BUILD.libc-0.2.169.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__log-0.4.22\",\n sha256 = \"a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/log/0.4.22/download\"],\n strip_prefix = \"log-0.4.22\",\n build_file = Label(\"@crates//crates:BUILD.log-0.4.22.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__macro_rules_attribute-0.2.0\",\n sha256 = \"8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/macro_rules_attribute/0.2.0/download\"],\n strip_prefix = \"macro_rules_attribute-0.2.0\",\n build_file = Label(\"@crates//crates:BUILD.macro_rules_attribute-0.2.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__macro_rules_attribute-proc_macro-0.2.0\",\n sha256 = \"b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/macro_rules_attribute-proc_macro/0.2.0/download\"],\n strip_prefix = \"macro_rules_attribute-proc_macro-0.2.0\",\n build_file = Label(\"@crates//crates:BUILD.macro_rules_attribute-proc_macro-0.2.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__memchr-2.7.4\",\n sha256 = \"78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/memchr/2.7.4/download\"],\n strip_prefix = \"memchr-2.7.4\",\n build_file = Label(\"@crates//crates:BUILD.memchr-2.7.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__minimal-lexical-0.2.1\",\n sha256 = \"68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/minimal-lexical/0.2.1/download\"],\n strip_prefix = \"minimal-lexical-0.2.1\",\n build_file = Label(\"@crates//crates:BUILD.minimal-lexical-0.2.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__monostate-0.1.13\",\n sha256 = \"0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/monostate/0.1.13/download\"],\n strip_prefix = \"monostate-0.1.13\",\n build_file = Label(\"@crates//crates:BUILD.monostate-0.1.13.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__monostate-impl-0.1.13\",\n sha256 = \"a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/monostate-impl/0.1.13/download\"],\n strip_prefix = \"monostate-impl-0.1.13\",\n build_file = Label(\"@crates//crates:BUILD.monostate-impl-0.1.13.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__nom-7.1.3\",\n sha256 = \"d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/nom/7.1.3/download\"],\n strip_prefix = \"nom-7.1.3\",\n build_file = Label(\"@crates//crates:BUILD.nom-7.1.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__once_cell-1.20.2\",\n sha256 = \"1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/once_cell/1.20.2/download\"],\n strip_prefix = \"once_cell-1.20.2\",\n build_file = Label(\"@crates//crates:BUILD.once_cell-1.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__onig-6.5.1\",\n sha256 = \"336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/onig/6.5.1/download\"],\n strip_prefix = \"onig-6.5.1\",\n build_file = Label(\"@crates//crates:BUILD.onig-6.5.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__onig_sys-69.9.1\",\n sha256 = \"c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/onig_sys/69.9.1/download\"],\n strip_prefix = \"onig_sys-69.9.1\",\n build_file = Label(\"@crates//crates:BUILD.onig_sys-69.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__paste-1.0.15\",\n sha256 = \"57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/paste/1.0.15/download\"],\n strip_prefix = \"paste-1.0.15\",\n build_file = Label(\"@crates//crates:BUILD.paste-1.0.15.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__pkg-config-0.3.32\",\n sha256 = \"7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/pkg-config/0.3.32/download\"],\n strip_prefix = \"pkg-config-0.3.32\",\n build_file = Label(\"@crates//crates:BUILD.pkg-config-0.3.32.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ppv-lite86-0.2.20\",\n sha256 = \"77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ppv-lite86/0.2.20/download\"],\n strip_prefix = \"ppv-lite86-0.2.20\",\n build_file = Label(\"@crates//crates:BUILD.ppv-lite86-0.2.20.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__proc-macro2-1.0.92\",\n sha256 = \"37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/proc-macro2/1.0.92/download\"],\n strip_prefix = \"proc-macro2-1.0.92\",\n build_file = Label(\"@crates//crates:BUILD.proc-macro2-1.0.92.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__quote-1.0.37\",\n sha256 = \"b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/quote/1.0.37/download\"],\n strip_prefix = \"quote-1.0.37\",\n build_file = Label(\"@crates//crates:BUILD.quote-1.0.37.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__r-efi-5.3.0\",\n sha256 = \"69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/r-efi/5.3.0/download\"],\n strip_prefix = \"r-efi-5.3.0\",\n build_file = Label(\"@crates//crates:BUILD.r-efi-5.3.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand-0.9.1\",\n sha256 = \"9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand/0.9.1/download\"],\n strip_prefix = \"rand-0.9.1\",\n build_file = Label(\"@crates//crates:BUILD.rand-0.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand_chacha-0.9.0\",\n sha256 = \"d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand_chacha/0.9.0/download\"],\n strip_prefix = \"rand_chacha-0.9.0\",\n build_file = Label(\"@crates//crates:BUILD.rand_chacha-0.9.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand_core-0.9.3\",\n sha256 = \"99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand_core/0.9.3/download\"],\n strip_prefix = \"rand_core-0.9.3\",\n build_file = Label(\"@crates//crates:BUILD.rand_core-0.9.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-1.10.0\",\n sha256 = \"b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon/1.10.0/download\"],\n strip_prefix = \"rayon-1.10.0\",\n build_file = Label(\"@crates//crates:BUILD.rayon-1.10.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-cond-0.4.0\",\n sha256 = \"2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon-cond/0.4.0/download\"],\n strip_prefix = \"rayon-cond-0.4.0\",\n build_file = Label(\"@crates//crates:BUILD.rayon-cond-0.4.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-core-1.12.1\",\n sha256 = \"1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon-core/1.12.1/download\"],\n strip_prefix = \"rayon-core-1.12.1\",\n build_file = Label(\"@crates//crates:BUILD.rayon-core-1.12.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-1.11.1\",\n sha256 = \"b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex/1.11.1/download\"],\n strip_prefix = \"regex-1.11.1\",\n build_file = Label(\"@crates//crates:BUILD.regex-1.11.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-automata-0.4.9\",\n sha256 = \"809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-automata/0.4.9/download\"],\n strip_prefix = \"regex-automata-0.4.9\",\n build_file = Label(\"@crates//crates:BUILD.regex-automata-0.4.9.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-syntax-0.8.5\",\n sha256 = \"2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-syntax/0.8.5/download\"],\n strip_prefix = \"regex-syntax-0.8.5\",\n build_file = Label(\"@crates//crates:BUILD.regex-syntax-0.8.5.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rustversion-1.0.21\",\n sha256 = \"8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rustversion/1.0.21/download\"],\n strip_prefix = \"rustversion-1.0.21\",\n build_file = Label(\"@crates//crates:BUILD.rustversion-1.0.21.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ryu-1.0.18\",\n sha256 = \"f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ryu/1.0.18/download\"],\n strip_prefix = \"ryu-1.0.18\",\n build_file = Label(\"@crates//crates:BUILD.ryu-1.0.18.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde-1.0.216\",\n sha256 = \"0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde/1.0.216/download\"],\n strip_prefix = \"serde-1.0.216\",\n build_file = Label(\"@crates//crates:BUILD.serde-1.0.216.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde_derive-1.0.216\",\n sha256 = \"46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde_derive/1.0.216/download\"],\n strip_prefix = \"serde_derive-1.0.216\",\n build_file = Label(\"@crates//crates:BUILD.serde_derive-1.0.216.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde_json-1.0.134\",\n sha256 = \"d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde_json/1.0.134/download\"],\n strip_prefix = \"serde_json-1.0.134\",\n build_file = Label(\"@crates//crates:BUILD.serde_json-1.0.134.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__shlex-1.3.0\",\n sha256 = \"0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/shlex/1.3.0/download\"],\n strip_prefix = \"shlex-1.3.0\",\n build_file = Label(\"@crates//crates:BUILD.shlex-1.3.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__smallvec-1.13.2\",\n sha256 = \"3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/smallvec/1.13.2/download\"],\n strip_prefix = \"smallvec-1.13.2\",\n build_file = Label(\"@crates//crates:BUILD.smallvec-1.13.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__spm_precompiled-0.1.4\",\n sha256 = \"5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/spm_precompiled/0.1.4/download\"],\n strip_prefix = \"spm_precompiled-0.1.4\",\n build_file = Label(\"@crates//crates:BUILD.spm_precompiled-0.1.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__static_assertions-1.1.0\",\n sha256 = \"a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/static_assertions/1.1.0/download\"],\n strip_prefix = \"static_assertions-1.1.0\",\n build_file = Label(\"@crates//crates:BUILD.static_assertions-1.1.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__strsim-0.11.1\",\n sha256 = \"7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/strsim/0.11.1/download\"],\n strip_prefix = \"strsim-0.11.1\",\n build_file = Label(\"@crates//crates:BUILD.strsim-0.11.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__syn-2.0.91\",\n sha256 = \"d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/syn/2.0.91/download\"],\n strip_prefix = \"syn-2.0.91\",\n build_file = Label(\"@crates//crates:BUILD.syn-2.0.91.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__thiserror-2.0.12\",\n sha256 = \"567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/thiserror/2.0.12/download\"],\n strip_prefix = \"thiserror-2.0.12\",\n build_file = Label(\"@crates//crates:BUILD.thiserror-2.0.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__thiserror-impl-2.0.12\",\n sha256 = \"7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/thiserror-impl/2.0.12/download\"],\n strip_prefix = \"thiserror-impl-2.0.12\",\n build_file = Label(\"@crates//crates:BUILD.thiserror-impl-2.0.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__tokenizers-0.21.4\",\n sha256 = \"a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/tokenizers/0.21.4/download\"],\n strip_prefix = \"tokenizers-0.21.4\",\n build_file = Label(\"@crates//crates:BUILD.tokenizers-0.21.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-ident-1.0.14\",\n sha256 = \"adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-ident/1.0.14/download\"],\n strip_prefix = \"unicode-ident-1.0.14\",\n build_file = Label(\"@crates//crates:BUILD.unicode-ident-1.0.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-normalization-alignments-0.1.12\",\n sha256 = \"43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-normalization-alignments/0.1.12/download\"],\n strip_prefix = \"unicode-normalization-alignments-0.1.12\",\n build_file = Label(\"@crates//crates:BUILD.unicode-normalization-alignments-0.1.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-segmentation-1.12.0\",\n sha256 = \"f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-segmentation/1.12.0/download\"],\n strip_prefix = \"unicode-segmentation-1.12.0\",\n build_file = Label(\"@crates//crates:BUILD.unicode-segmentation-1.12.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode_categories-0.1.1\",\n sha256 = \"39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode_categories/0.1.1/download\"],\n strip_prefix = \"unicode_categories-0.1.1\",\n build_file = Label(\"@crates//crates:BUILD.unicode_categories-0.1.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__version_check-0.9.5\",\n sha256 = \"0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/version_check/0.9.5/download\"],\n strip_prefix = \"version_check-0.9.5\",\n build_file = Label(\"@crates//crates:BUILD.version_check-0.9.5.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasi-0.14.2-wasi-0.2.4\",\n sha256 = \"9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasi/0.14.2+wasi-0.2.4/download\"],\n strip_prefix = \"wasi-0.14.2+wasi-0.2.4\",\n build_file = Label(\"@crates//crates:BUILD.wasi-0.14.2+wasi-0.2.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wit-bindgen-rt-0.39.0\",\n sha256 = \"6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wit-bindgen-rt/0.39.0/download\"],\n strip_prefix = \"wit-bindgen-rt-0.39.0\",\n build_file = Label(\"@crates//crates:BUILD.wit-bindgen-rt-0.39.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-0.7.35\",\n sha256 = \"1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy/0.7.35/download\"],\n strip_prefix = \"zerocopy-0.7.35\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-0.7.35.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-0.8.26\",\n sha256 = \"1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy/0.8.26/download\"],\n strip_prefix = \"zerocopy-0.8.26\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-0.8.26.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-derive-0.7.35\",\n sha256 = \"fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy-derive/0.7.35/download\"],\n strip_prefix = \"zerocopy-derive-0.7.35\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-derive-0.7.35.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-derive-0.8.26\",\n sha256 = \"9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy-derive/0.8.26/download\"],\n strip_prefix = \"zerocopy-derive-0.8.26\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-derive-0.8.26.bazel\"),\n )\n\n return [\n struct(repo=\"crates__tokenizers-0.21.4\", is_dev_dep = False),\n ]\n" + "defs.bzl": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\"\"\"\n# `crates_repository` API\n\n- [aliases](#aliases)\n- [crate_deps](#crate_deps)\n- [all_crate_deps](#all_crate_deps)\n- [crate_repositories](#crate_repositories)\n\n\"\"\"\n\nload(\"@bazel_tools//tools/build_defs/repo:git.bzl\", \"new_git_repository\")\nload(\"@bazel_tools//tools/build_defs/repo:http.bzl\", \"http_archive\")\nload(\"@bazel_tools//tools/build_defs/repo:utils.bzl\", \"maybe\")\nload(\"@bazel_skylib//lib:selects.bzl\", \"selects\")\nload(\"@rules_rust//crate_universe/private:local_crate_mirror.bzl\", \"local_crate_mirror\")\n\n###############################################################################\n# MACROS API\n###############################################################################\n\n# An identifier that represent common dependencies (unconditional).\n_COMMON_CONDITION = \"\"\n\ndef _flatten_dependency_maps(all_dependency_maps):\n \"\"\"Flatten a list of dependency maps into one dictionary.\n\n Dependency maps have the following structure:\n\n ```python\n DEPENDENCIES_MAP = {\n # The first key in the map is a Bazel package\n # name of the workspace this file is defined in.\n \"workspace_member_package\": {\n\n # Not all dependencies are supported for all platforms.\n # the condition key is the condition required to be true\n # on the host platform.\n \"condition\": {\n\n # An alias to a crate target. # The label of the crate target the\n # Aliases are only crate names. # package name refers to.\n \"package_name\": \"@full//:label\",\n }\n }\n }\n ```\n\n Args:\n all_dependency_maps (list): A list of dicts as described above\n\n Returns:\n dict: A dictionary as described above\n \"\"\"\n dependencies = {}\n\n for workspace_deps_map in all_dependency_maps:\n for pkg_name, conditional_deps_map in workspace_deps_map.items():\n if pkg_name not in dependencies:\n non_frozen_map = dict()\n for key, values in conditional_deps_map.items():\n non_frozen_map.update({key: dict(values.items())})\n dependencies.setdefault(pkg_name, non_frozen_map)\n continue\n\n for condition, deps_map in conditional_deps_map.items():\n # If the condition has not been recorded, do so and continue\n if condition not in dependencies[pkg_name]:\n dependencies[pkg_name].setdefault(condition, dict(deps_map.items()))\n continue\n\n # Alert on any miss-matched dependencies\n inconsistent_entries = []\n for crate_name, crate_label in deps_map.items():\n existing = dependencies[pkg_name][condition].get(crate_name)\n if existing and existing != crate_label:\n inconsistent_entries.append((crate_name, existing, crate_label))\n dependencies[pkg_name][condition].update({crate_name: crate_label})\n\n return dependencies\n\ndef crate_deps(deps, package_name = None):\n \"\"\"Finds the fully qualified label of the requested crates for the package where this macro is called.\n\n Args:\n deps (list): The desired list of crate targets.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()`.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if not deps:\n return []\n\n if package_name == None:\n package_name = native.package_name()\n\n # Join both sets of dependencies\n dependencies = _flatten_dependency_maps([\n _NORMAL_DEPENDENCIES,\n _NORMAL_DEV_DEPENDENCIES,\n _PROC_MACRO_DEPENDENCIES,\n _PROC_MACRO_DEV_DEPENDENCIES,\n _BUILD_DEPENDENCIES,\n _BUILD_PROC_MACRO_DEPENDENCIES,\n ]).pop(package_name, {})\n\n # Combine all conditional packages so we can easily index over a flat list\n # TODO: Perhaps this should actually return select statements and maintain\n # the conditionals of the dependencies\n flat_deps = {}\n for deps_set in dependencies.values():\n for crate_name, crate_label in deps_set.items():\n flat_deps.update({crate_name: crate_label})\n\n missing_crates = []\n crate_targets = []\n for crate_target in deps:\n if crate_target not in flat_deps:\n missing_crates.append(crate_target)\n else:\n crate_targets.append(flat_deps[crate_target])\n\n if missing_crates:\n fail(\"Could not find crates `{}` among dependencies of `{}`. Available dependencies were `{}`\".format(\n missing_crates,\n package_name,\n dependencies,\n ))\n\n return crate_targets\n\ndef all_crate_deps(\n normal = False, \n normal_dev = False, \n proc_macro = False, \n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Finds the fully qualified label of all requested direct crate dependencies \\\n for the package where this macro is called.\n\n If no parameters are set, all normal dependencies are returned. Setting any one flag will\n otherwise impact the contents of the returned list.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_dependency_maps = []\n if normal:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n if normal_dev:\n all_dependency_maps.append(_NORMAL_DEV_DEPENDENCIES)\n if proc_macro:\n all_dependency_maps.append(_PROC_MACRO_DEPENDENCIES)\n if proc_macro_dev:\n all_dependency_maps.append(_PROC_MACRO_DEV_DEPENDENCIES)\n if build:\n all_dependency_maps.append(_BUILD_DEPENDENCIES)\n if build_proc_macro:\n all_dependency_maps.append(_BUILD_PROC_MACRO_DEPENDENCIES)\n\n # Default to always using normal dependencies\n if not all_dependency_maps:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n\n dependencies = _flatten_dependency_maps(all_dependency_maps).pop(package_name, None)\n\n if not dependencies:\n if dependencies == None:\n fail(\"Tried to get all_crate_deps for package \" + package_name + \" but that package had no Cargo.toml file\")\n else:\n return []\n\n crate_deps = list(dependencies.pop(_COMMON_CONDITION, {}).values())\n for condition, deps in dependencies.items():\n crate_deps += selects.with_or({\n tuple(_CONDITIONS[condition]): deps.values(),\n \"//conditions:default\": [],\n })\n\n return crate_deps\n\ndef aliases(\n normal = False,\n normal_dev = False,\n proc_macro = False,\n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Produces a map of Crate alias names to their original label\n\n If no dependency kinds are specified, `normal` and `proc_macro` are used by default.\n Setting any one flag will otherwise determine the contents of the returned dict.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n dict: The aliases of all associated packages\n \"\"\"\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_aliases_maps = []\n if normal:\n all_aliases_maps.append(_NORMAL_ALIASES)\n if normal_dev:\n all_aliases_maps.append(_NORMAL_DEV_ALIASES)\n if proc_macro:\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n if proc_macro_dev:\n all_aliases_maps.append(_PROC_MACRO_DEV_ALIASES)\n if build:\n all_aliases_maps.append(_BUILD_ALIASES)\n if build_proc_macro:\n all_aliases_maps.append(_BUILD_PROC_MACRO_ALIASES)\n\n # Default to always using normal aliases\n if not all_aliases_maps:\n all_aliases_maps.append(_NORMAL_ALIASES)\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n\n aliases = _flatten_dependency_maps(all_aliases_maps).pop(package_name, None)\n\n if not aliases:\n return dict()\n\n common_items = aliases.pop(_COMMON_CONDITION, {}).items()\n\n # If there are only common items in the dictionary, immediately return them\n if not len(aliases.keys()) == 1:\n return dict(common_items)\n\n # Build a single select statement where each conditional has accounted for the\n # common set of aliases.\n crate_aliases = {\"//conditions:default\": dict(common_items)}\n for condition, deps in aliases.items():\n condition_triples = _CONDITIONS[condition]\n for triple in condition_triples:\n if triple in crate_aliases:\n crate_aliases[triple].update(deps)\n else:\n crate_aliases.update({triple: dict(deps.items() + common_items)})\n\n return select(crate_aliases)\n\n###############################################################################\n# WORKSPACE MEMBER DEPS AND ALIASES\n###############################################################################\n\n_NORMAL_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n _COMMON_CONDITION: {\n \"tokenizers\": Label(\"@crates//:tokenizers-0.22.2\"),\n },\n },\n}\n\n\n_NORMAL_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n _COMMON_CONDITION: {\n },\n },\n}\n\n\n_NORMAL_DEV_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_NORMAL_DEV_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEV_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEV_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_CONDITIONS = {\n \"aarch64-apple-darwin\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\"],\n \"aarch64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\"],\n \"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\",\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\": [],\n \"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\": [],\n \"cfg(any())\": [],\n \"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\": [],\n \"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\": [],\n \"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\": [],\n \"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\",\"@rules_rust//rust/platform:x86_64-apple-darwin\"],\n \"cfg(not(all(target_arch = \\\"arm\\\", target_os = \\\"none\\\")))\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\",\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\",\"@rules_rust//rust/platform:x86_64-apple-darwin\",\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"cfg(target_os = \\\"netbsd\\\")\": [],\n \"cfg(target_os = \\\"solaris\\\")\": [],\n \"cfg(target_os = \\\"vxworks\\\")\": [],\n \"cfg(windows)\": [],\n \"x86_64-apple-darwin\": [\"@rules_rust//rust/platform:x86_64-apple-darwin\"],\n \"x86_64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n}\n\n###############################################################################\n\ndef crate_repositories():\n \"\"\"A macro for defining repositories for all generated crates.\n\n Returns:\n A list of repos visible to the module through the module extension.\n \"\"\"\n maybe(\n http_archive,\n name = \"crates__ahash-0.8.12\",\n sha256 = \"5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ahash/0.8.12/download\"],\n strip_prefix = \"ahash-0.8.12\",\n build_file = Label(\"@crates//crates:BUILD.ahash-0.8.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__aho-corasick-1.1.3\",\n sha256 = \"8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/aho-corasick/1.1.3/download\"],\n strip_prefix = \"aho-corasick-1.1.3\",\n build_file = Label(\"@crates//crates:BUILD.aho-corasick-1.1.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__base64-0.13.1\",\n sha256 = \"9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/base64/0.13.1/download\"],\n strip_prefix = \"base64-0.13.1\",\n build_file = Label(\"@crates//crates:BUILD.base64-0.13.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__bitflags-2.9.1\",\n sha256 = \"1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/bitflags/2.9.1/download\"],\n strip_prefix = \"bitflags-2.9.1\",\n build_file = Label(\"@crates//crates:BUILD.bitflags-2.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__byteorder-1.5.0\",\n sha256 = \"1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/byteorder/1.5.0/download\"],\n strip_prefix = \"byteorder-1.5.0\",\n build_file = Label(\"@crates//crates:BUILD.byteorder-1.5.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__castaway-0.2.4\",\n sha256 = \"dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/castaway/0.2.4/download\"],\n strip_prefix = \"castaway-0.2.4\",\n build_file = Label(\"@crates//crates:BUILD.castaway-0.2.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__cc-1.2.33\",\n sha256 = \"3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/cc/1.2.33/download\"],\n strip_prefix = \"cc-1.2.33\",\n build_file = Label(\"@crates//crates:BUILD.cc-1.2.33.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__cfg-if-1.0.0\",\n sha256 = \"baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/cfg-if/1.0.0/download\"],\n strip_prefix = \"cfg-if-1.0.0\",\n build_file = Label(\"@crates//crates:BUILD.cfg-if-1.0.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__compact_str-0.9.0\",\n sha256 = \"3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/compact_str/0.9.0/download\"],\n strip_prefix = \"compact_str-0.9.0\",\n build_file = Label(\"@crates//crates:BUILD.compact_str-0.9.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-deque-0.8.6\",\n sha256 = \"9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-deque/0.8.6/download\"],\n strip_prefix = \"crossbeam-deque-0.8.6\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-deque-0.8.6.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-epoch-0.9.18\",\n sha256 = \"5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-epoch/0.9.18/download\"],\n strip_prefix = \"crossbeam-epoch-0.9.18\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-epoch-0.9.18.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-utils-0.8.21\",\n sha256 = \"d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-utils/0.8.21/download\"],\n strip_prefix = \"crossbeam-utils-0.8.21\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-utils-0.8.21.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling-0.20.10\",\n sha256 = \"6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling/0.20.10/download\"],\n strip_prefix = \"darling-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling_core-0.20.10\",\n sha256 = \"95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling_core/0.20.10/download\"],\n strip_prefix = \"darling_core-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling_core-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling_macro-0.20.10\",\n sha256 = \"d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling_macro/0.20.10/download\"],\n strip_prefix = \"darling_macro-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling_macro-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__dary_heap-0.3.7\",\n sha256 = \"04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/dary_heap/0.3.7/download\"],\n strip_prefix = \"dary_heap-0.3.7\",\n build_file = Label(\"@crates//crates:BUILD.dary_heap-0.3.7.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder-0.20.2\",\n sha256 = \"507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder/0.20.2/download\"],\n strip_prefix = \"derive_builder-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder_core-0.20.2\",\n sha256 = \"2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder_core/0.20.2/download\"],\n strip_prefix = \"derive_builder_core-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder_core-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder_macro-0.20.2\",\n sha256 = \"ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder_macro/0.20.2/download\"],\n strip_prefix = \"derive_builder_macro-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder_macro-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__either-1.13.0\",\n sha256 = \"60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/either/1.13.0/download\"],\n strip_prefix = \"either-1.13.0\",\n build_file = Label(\"@crates//crates:BUILD.either-1.13.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__esaxx-rs-0.1.10\",\n sha256 = \"d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/esaxx-rs/0.1.10/download\"],\n strip_prefix = \"esaxx-rs-0.1.10\",\n build_file = Label(\"@crates//crates:BUILD.esaxx-rs-0.1.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__fnv-1.0.7\",\n sha256 = \"3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/fnv/1.0.7/download\"],\n strip_prefix = \"fnv-1.0.7\",\n build_file = Label(\"@crates//crates:BUILD.fnv-1.0.7.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__getrandom-0.3.3\",\n sha256 = \"26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/getrandom/0.3.3/download\"],\n strip_prefix = \"getrandom-0.3.3\",\n build_file = Label(\"@crates//crates:BUILD.getrandom-0.3.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ident_case-1.0.1\",\n sha256 = \"b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ident_case/1.0.1/download\"],\n strip_prefix = \"ident_case-1.0.1\",\n build_file = Label(\"@crates//crates:BUILD.ident_case-1.0.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__itertools-0.14.0\",\n sha256 = \"2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/itertools/0.14.0/download\"],\n strip_prefix = \"itertools-0.14.0\",\n build_file = Label(\"@crates//crates:BUILD.itertools-0.14.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__itoa-1.0.14\",\n sha256 = \"d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/itoa/1.0.14/download\"],\n strip_prefix = \"itoa-1.0.14\",\n build_file = Label(\"@crates//crates:BUILD.itoa-1.0.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__libc-0.2.169\",\n sha256 = \"b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/libc/0.2.169/download\"],\n strip_prefix = \"libc-0.2.169\",\n build_file = Label(\"@crates//crates:BUILD.libc-0.2.169.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__log-0.4.22\",\n sha256 = \"a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/log/0.4.22/download\"],\n strip_prefix = \"log-0.4.22\",\n build_file = Label(\"@crates//crates:BUILD.log-0.4.22.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__macro_rules_attribute-0.2.0\",\n sha256 = \"8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/macro_rules_attribute/0.2.0/download\"],\n strip_prefix = \"macro_rules_attribute-0.2.0\",\n build_file = Label(\"@crates//crates:BUILD.macro_rules_attribute-0.2.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__macro_rules_attribute-proc_macro-0.2.0\",\n sha256 = \"b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/macro_rules_attribute-proc_macro/0.2.0/download\"],\n strip_prefix = \"macro_rules_attribute-proc_macro-0.2.0\",\n build_file = Label(\"@crates//crates:BUILD.macro_rules_attribute-proc_macro-0.2.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__memchr-2.7.4\",\n sha256 = \"78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/memchr/2.7.4/download\"],\n strip_prefix = \"memchr-2.7.4\",\n build_file = Label(\"@crates//crates:BUILD.memchr-2.7.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__minimal-lexical-0.2.1\",\n sha256 = \"68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/minimal-lexical/0.2.1/download\"],\n strip_prefix = \"minimal-lexical-0.2.1\",\n build_file = Label(\"@crates//crates:BUILD.minimal-lexical-0.2.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__monostate-0.1.13\",\n sha256 = \"0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/monostate/0.1.13/download\"],\n strip_prefix = \"monostate-0.1.13\",\n build_file = Label(\"@crates//crates:BUILD.monostate-0.1.13.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__monostate-impl-0.1.13\",\n sha256 = \"a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/monostate-impl/0.1.13/download\"],\n strip_prefix = \"monostate-impl-0.1.13\",\n build_file = Label(\"@crates//crates:BUILD.monostate-impl-0.1.13.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__nom-7.1.3\",\n sha256 = \"d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/nom/7.1.3/download\"],\n strip_prefix = \"nom-7.1.3\",\n build_file = Label(\"@crates//crates:BUILD.nom-7.1.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__once_cell-1.20.2\",\n sha256 = \"1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/once_cell/1.20.2/download\"],\n strip_prefix = \"once_cell-1.20.2\",\n build_file = Label(\"@crates//crates:BUILD.once_cell-1.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__onig-6.5.1\",\n sha256 = \"336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/onig/6.5.1/download\"],\n strip_prefix = \"onig-6.5.1\",\n build_file = Label(\"@crates//crates:BUILD.onig-6.5.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__onig_sys-69.9.1\",\n sha256 = \"c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/onig_sys/69.9.1/download\"],\n strip_prefix = \"onig_sys-69.9.1\",\n build_file = Label(\"@crates//crates:BUILD.onig_sys-69.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__paste-1.0.15\",\n sha256 = \"57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/paste/1.0.15/download\"],\n strip_prefix = \"paste-1.0.15\",\n build_file = Label(\"@crates//crates:BUILD.paste-1.0.15.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__pkg-config-0.3.32\",\n sha256 = \"7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/pkg-config/0.3.32/download\"],\n strip_prefix = \"pkg-config-0.3.32\",\n build_file = Label(\"@crates//crates:BUILD.pkg-config-0.3.32.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ppv-lite86-0.2.20\",\n sha256 = \"77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ppv-lite86/0.2.20/download\"],\n strip_prefix = \"ppv-lite86-0.2.20\",\n build_file = Label(\"@crates//crates:BUILD.ppv-lite86-0.2.20.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__proc-macro2-1.0.92\",\n sha256 = \"37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/proc-macro2/1.0.92/download\"],\n strip_prefix = \"proc-macro2-1.0.92\",\n build_file = Label(\"@crates//crates:BUILD.proc-macro2-1.0.92.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__quote-1.0.37\",\n sha256 = \"b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/quote/1.0.37/download\"],\n strip_prefix = \"quote-1.0.37\",\n build_file = Label(\"@crates//crates:BUILD.quote-1.0.37.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__r-efi-5.3.0\",\n sha256 = \"69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/r-efi/5.3.0/download\"],\n strip_prefix = \"r-efi-5.3.0\",\n build_file = Label(\"@crates//crates:BUILD.r-efi-5.3.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand-0.9.1\",\n sha256 = \"9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand/0.9.1/download\"],\n strip_prefix = \"rand-0.9.1\",\n build_file = Label(\"@crates//crates:BUILD.rand-0.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand_chacha-0.9.0\",\n sha256 = \"d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand_chacha/0.9.0/download\"],\n strip_prefix = \"rand_chacha-0.9.0\",\n build_file = Label(\"@crates//crates:BUILD.rand_chacha-0.9.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand_core-0.9.3\",\n sha256 = \"99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand_core/0.9.3/download\"],\n strip_prefix = \"rand_core-0.9.3\",\n build_file = Label(\"@crates//crates:BUILD.rand_core-0.9.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-1.10.0\",\n sha256 = \"b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon/1.10.0/download\"],\n strip_prefix = \"rayon-1.10.0\",\n build_file = Label(\"@crates//crates:BUILD.rayon-1.10.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-cond-0.4.0\",\n sha256 = \"2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon-cond/0.4.0/download\"],\n strip_prefix = \"rayon-cond-0.4.0\",\n build_file = Label(\"@crates//crates:BUILD.rayon-cond-0.4.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-core-1.12.1\",\n sha256 = \"1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon-core/1.12.1/download\"],\n strip_prefix = \"rayon-core-1.12.1\",\n build_file = Label(\"@crates//crates:BUILD.rayon-core-1.12.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-1.11.1\",\n sha256 = \"b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex/1.11.1/download\"],\n strip_prefix = \"regex-1.11.1\",\n build_file = Label(\"@crates//crates:BUILD.regex-1.11.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-automata-0.4.9\",\n sha256 = \"809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-automata/0.4.9/download\"],\n strip_prefix = \"regex-automata-0.4.9\",\n build_file = Label(\"@crates//crates:BUILD.regex-automata-0.4.9.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-syntax-0.8.5\",\n sha256 = \"2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-syntax/0.8.5/download\"],\n strip_prefix = \"regex-syntax-0.8.5\",\n build_file = Label(\"@crates//crates:BUILD.regex-syntax-0.8.5.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rustversion-1.0.21\",\n sha256 = \"8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rustversion/1.0.21/download\"],\n strip_prefix = \"rustversion-1.0.21\",\n build_file = Label(\"@crates//crates:BUILD.rustversion-1.0.21.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ryu-1.0.18\",\n sha256 = \"f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ryu/1.0.18/download\"],\n strip_prefix = \"ryu-1.0.18\",\n build_file = Label(\"@crates//crates:BUILD.ryu-1.0.18.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde-1.0.216\",\n sha256 = \"0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde/1.0.216/download\"],\n strip_prefix = \"serde-1.0.216\",\n build_file = Label(\"@crates//crates:BUILD.serde-1.0.216.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde_derive-1.0.216\",\n sha256 = \"46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde_derive/1.0.216/download\"],\n strip_prefix = \"serde_derive-1.0.216\",\n build_file = Label(\"@crates//crates:BUILD.serde_derive-1.0.216.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde_json-1.0.134\",\n sha256 = \"d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde_json/1.0.134/download\"],\n strip_prefix = \"serde_json-1.0.134\",\n build_file = Label(\"@crates//crates:BUILD.serde_json-1.0.134.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__shlex-1.3.0\",\n sha256 = \"0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/shlex/1.3.0/download\"],\n strip_prefix = \"shlex-1.3.0\",\n build_file = Label(\"@crates//crates:BUILD.shlex-1.3.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__smallvec-1.13.2\",\n sha256 = \"3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/smallvec/1.13.2/download\"],\n strip_prefix = \"smallvec-1.13.2\",\n build_file = Label(\"@crates//crates:BUILD.smallvec-1.13.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__spm_precompiled-0.1.4\",\n sha256 = \"5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/spm_precompiled/0.1.4/download\"],\n strip_prefix = \"spm_precompiled-0.1.4\",\n build_file = Label(\"@crates//crates:BUILD.spm_precompiled-0.1.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__static_assertions-1.1.0\",\n sha256 = \"a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/static_assertions/1.1.0/download\"],\n strip_prefix = \"static_assertions-1.1.0\",\n build_file = Label(\"@crates//crates:BUILD.static_assertions-1.1.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__strsim-0.11.1\",\n sha256 = \"7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/strsim/0.11.1/download\"],\n strip_prefix = \"strsim-0.11.1\",\n build_file = Label(\"@crates//crates:BUILD.strsim-0.11.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__syn-2.0.91\",\n sha256 = \"d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/syn/2.0.91/download\"],\n strip_prefix = \"syn-2.0.91\",\n build_file = Label(\"@crates//crates:BUILD.syn-2.0.91.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__thiserror-2.0.12\",\n sha256 = \"567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/thiserror/2.0.12/download\"],\n strip_prefix = \"thiserror-2.0.12\",\n build_file = Label(\"@crates//crates:BUILD.thiserror-2.0.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__thiserror-impl-2.0.12\",\n sha256 = \"7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/thiserror-impl/2.0.12/download\"],\n strip_prefix = \"thiserror-impl-2.0.12\",\n build_file = Label(\"@crates//crates:BUILD.thiserror-impl-2.0.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__tokenizers-0.22.2\",\n sha256 = \"b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/tokenizers/0.22.2/download\"],\n strip_prefix = \"tokenizers-0.22.2\",\n build_file = Label(\"@crates//crates:BUILD.tokenizers-0.22.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-ident-1.0.14\",\n sha256 = \"adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-ident/1.0.14/download\"],\n strip_prefix = \"unicode-ident-1.0.14\",\n build_file = Label(\"@crates//crates:BUILD.unicode-ident-1.0.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-normalization-alignments-0.1.12\",\n sha256 = \"43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-normalization-alignments/0.1.12/download\"],\n strip_prefix = \"unicode-normalization-alignments-0.1.12\",\n build_file = Label(\"@crates//crates:BUILD.unicode-normalization-alignments-0.1.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-segmentation-1.12.0\",\n sha256 = \"f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-segmentation/1.12.0/download\"],\n strip_prefix = \"unicode-segmentation-1.12.0\",\n build_file = Label(\"@crates//crates:BUILD.unicode-segmentation-1.12.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode_categories-0.1.1\",\n sha256 = \"39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode_categories/0.1.1/download\"],\n strip_prefix = \"unicode_categories-0.1.1\",\n build_file = Label(\"@crates//crates:BUILD.unicode_categories-0.1.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__version_check-0.9.5\",\n sha256 = \"0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/version_check/0.9.5/download\"],\n strip_prefix = \"version_check-0.9.5\",\n build_file = Label(\"@crates//crates:BUILD.version_check-0.9.5.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasi-0.14.2-wasi-0.2.4\",\n sha256 = \"9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasi/0.14.2+wasi-0.2.4/download\"],\n strip_prefix = \"wasi-0.14.2+wasi-0.2.4\",\n build_file = Label(\"@crates//crates:BUILD.wasi-0.14.2+wasi-0.2.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wit-bindgen-rt-0.39.0\",\n sha256 = \"6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wit-bindgen-rt/0.39.0/download\"],\n strip_prefix = \"wit-bindgen-rt-0.39.0\",\n build_file = Label(\"@crates//crates:BUILD.wit-bindgen-rt-0.39.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-0.7.35\",\n sha256 = \"1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy/0.7.35/download\"],\n strip_prefix = \"zerocopy-0.7.35\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-0.7.35.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-0.8.26\",\n sha256 = \"1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy/0.8.26/download\"],\n strip_prefix = \"zerocopy-0.8.26\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-0.8.26.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-derive-0.7.35\",\n sha256 = \"fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy-derive/0.7.35/download\"],\n strip_prefix = \"zerocopy-derive-0.7.35\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-derive-0.7.35.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-derive-0.8.26\",\n sha256 = \"9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy-derive/0.8.26/download\"],\n strip_prefix = \"zerocopy-derive-0.8.26\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-derive-0.8.26.bazel\"),\n )\n\n return [\n struct(repo=\"crates__tokenizers-0.22.2\", is_dev_dep = False),\n ]\n" } } }, @@ -2278,20 +2278,20 @@ "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"thiserror_impl\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=thiserror-impl\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.0.12\",\n)\n" } }, - "crates__tokenizers-0.21.4": { + "crates__tokenizers-0.22.2": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { "patch_args": [], "patch_tool": "", "patches": [], "remote_patch_strip": 1, - "sha256": "a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476", + "sha256": "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223", "type": "tar.gz", "urls": [ - "https://static.crates.io/crates/tokenizers/0.21.4/download" + "https://static.crates.io/crates/tokenizers/0.22.2/download" ], - "strip_prefix": "tokenizers-0.21.4", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"tokenizers\",\n deps = [\n \"@crates__ahash-0.8.12//:ahash\",\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__compact_str-0.9.0//:compact_str\",\n \"@crates__dary_heap-0.3.7//:dary_heap\",\n \"@crates__derive_builder-0.20.2//:derive_builder\",\n \"@crates__esaxx-rs-0.1.10//:esaxx_rs\",\n \"@crates__getrandom-0.3.3//:getrandom\",\n \"@crates__itertools-0.14.0//:itertools\",\n \"@crates__log-0.4.22//:log\",\n \"@crates__macro_rules_attribute-0.2.0//:macro_rules_attribute\",\n \"@crates__monostate-0.1.13//:monostate\",\n \"@crates__onig-6.5.1//:onig\",\n \"@crates__rand-0.9.1//:rand\",\n \"@crates__rayon-1.10.0//:rayon\",\n \"@crates__rayon-cond-0.4.0//:rayon_cond\",\n \"@crates__regex-1.11.1//:regex\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__serde_json-1.0.134//:serde_json\",\n \"@crates__spm_precompiled-0.1.4//:spm_precompiled\",\n \"@crates__thiserror-2.0.12//:thiserror\",\n \"@crates__unicode-normalization-alignments-0.1.12//:unicode_normalization_alignments\",\n \"@crates__unicode-segmentation-1.12.0//:unicode_segmentation\",\n \"@crates__unicode_categories-0.1.1//:unicode_categories\",\n ],\n proc_macro_deps = [\n \"@crates__paste-1.0.15//:paste\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"esaxx_fast\",\n \"onig\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=tokenizers\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.21.4\",\n)\n" + "strip_prefix": "tokenizers-0.22.2", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"tokenizers\",\n deps = [\n \"@crates__ahash-0.8.12//:ahash\",\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__compact_str-0.9.0//:compact_str\",\n \"@crates__dary_heap-0.3.7//:dary_heap\",\n \"@crates__derive_builder-0.20.2//:derive_builder\",\n \"@crates__esaxx-rs-0.1.10//:esaxx_rs\",\n \"@crates__getrandom-0.3.3//:getrandom\",\n \"@crates__itertools-0.14.0//:itertools\",\n \"@crates__log-0.4.22//:log\",\n \"@crates__macro_rules_attribute-0.2.0//:macro_rules_attribute\",\n \"@crates__monostate-0.1.13//:monostate\",\n \"@crates__onig-6.5.1//:onig\",\n \"@crates__rand-0.9.1//:rand\",\n \"@crates__rayon-1.10.0//:rayon\",\n \"@crates__rayon-cond-0.4.0//:rayon_cond\",\n \"@crates__regex-1.11.1//:regex\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__serde_json-1.0.134//:serde_json\",\n \"@crates__spm_precompiled-0.1.4//:spm_precompiled\",\n \"@crates__thiserror-2.0.12//:thiserror\",\n \"@crates__unicode-normalization-alignments-0.1.12//:unicode_normalization_alignments\",\n \"@crates__unicode-segmentation-1.12.0//:unicode_segmentation\",\n \"@crates__unicode_categories-0.1.1//:unicode_categories\",\n ],\n proc_macro_deps = [\n \"@crates__paste-1.0.15//:paste\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"esaxx_fast\",\n \"onig\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=tokenizers\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.22.2\",\n)\n" } }, "crates__unicode-ident-1.0.14": { diff --git a/zml/tokenizer/hftokenizers/Cargo.lock b/zml/tokenizer/hftokenizers/Cargo.lock index 68d1a36c..7164e77b 100644 --- a/zml/tokenizer/hftokenizers/Cargo.lock +++ b/zml/tokenizer/hftokenizers/Cargo.lock @@ -582,9 +582,9 @@ dependencies = [ [[package]] name = "tokenizers" -version = "0.21.4" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476" +checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223" dependencies = [ "ahash", "aho-corasick", diff --git a/zml/tokenizer/hftokenizers/Cargo.toml b/zml/tokenizer/hftokenizers/Cargo.toml index 8054cd49..f905c004 100644 --- a/zml/tokenizer/hftokenizers/Cargo.toml +++ b/zml/tokenizer/hftokenizers/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] -tokenizers = { version = "0.21.4", default-features = false, features = [ +tokenizers = { version = "0.22.2", default-features = false, features = [ # We don't need/want the progressbar feature "onig", "esaxx_fast", From b3614f6f0ca458d3f3a2c62d04e0e19e63faeaf1 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 6 Jan 2026 17:32:19 +0000 Subject: [PATCH 207/881] third_party: bump swig=4.4.1 --- third_party/org_swig_swig/repo.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/third_party/org_swig_swig/repo.bzl b/third_party/org_swig_swig/repo.bzl index 247c9d46..c35659bf 100644 --- a/third_party/org_swig_swig/repo.bzl +++ b/third_party/org_swig_swig/repo.bzl @@ -3,8 +3,8 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") def repo(): http_archive( name = "org_swig_swig", - url = "http://prdownloads.sourceforge.net/swig/swig-4.3.0.tar.gz", - sha256 = "f7203ef796f61af986c70c05816236cbd0d31b7aa9631e5ab53020ab7804aa9e", - strip_prefix = "swig-4.3.0", + url = "http://prdownloads.sourceforge.net/swig/swig-4.4.1.tar.gz", + sha256 = "40162a706c56f7592d08fd52ef5511cb7ac191f3593cf07306a0a554c6281fcf", + strip_prefix = "swig-4.4.1", build_file = "//:third_party/org_swig_swig/swig.bazel", ) From 1f73e85005d96ce298e05ec2fff0f4aba1b99c47 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 8 Jan 2026 09:08:52 +0000 Subject: [PATCH 208/881] pjrt: rework NamedValue it's simpler --- pjrt/pjrt.zig | 117 +++++++++++++++++------------------------------ zml/platform.zig | 15 +++--- 2 files changed, 50 insertions(+), 82 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index f2aa4830..74b416f3 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -1263,6 +1263,14 @@ pub const NamedValue = extern struct { bool = c.PJRT_NamedValue_kBool, }; + pub const Value = union(Kind) { + string: []const u8, + int64: i64, + int64list: []const i64, + float: f32, + bool: bool, + }; + pub fn kind(self: NamedValue) Kind { return @enumFromInt(self.inner.type); } @@ -1271,86 +1279,47 @@ pub const NamedValue = extern struct { return self.inner.name[0..self.inner.name_size]; } - pub fn from(name_: []const u8, value: anytype) NamedValue { - return switch (@TypeOf(value)) { - []u8, []const u8 => fromString(name_, value), - i64 => fromInt64(name_, value), - []i64, []const i64 => fromInt64List(name_, value), - f32 => fromFloat(name_, value), - bool => fromBool(name_, value), - else => fromString(name_, @tagName(value)), + pub fn value(self: NamedValue) Value { + return switch (self.kind()) { + .string => .{ .string = self.inner.unnamed_0.string_value[0..self.inner.value_size] }, + .int64 => .{ .int64 = self.inner.unnamed_0.int64_value }, + .int64list => .{ .int64list = self.inner.unnamed_0.int64_array_value[0..self.inner.value_size] }, + .float => .{ .float = self.inner.unnamed_0.float_value }, + .bool => .{ .bool = self.inner.unnamed_0.bool_value }, }; } - pub fn fromString(name_: []const u8, value: []const u8) NamedValue { - return .{ .inner = c.PJRT_NamedValue{ - .struct_size = c.PJRT_NamedValue_STRUCT_SIZE, - .extension_start = null, - .name = @as([*c]const u8, @ptrCast(@constCast(name_.ptr))), - .name_size = name_.len, - .type = c.PJRT_NamedValue_kString, - .unnamed_0 = .{ .string_value = @ptrCast(@constCast(value.ptr)) }, - .value_size = value.len, - } }; - } - - pub fn fromInt64(name_: []const u8, value: i64) NamedValue { - return .{ .inner = c.PJRT_NamedValue{ - .struct_size = c.PJRT_NamedValue_STRUCT_SIZE, - .extension_start = null, - .name = @as([*c]const u8, @ptrCast(@constCast(name_.ptr))), - .name_size = name_.len, - .type = c.PJRT_NamedValue_kInt64, - .unnamed_0 = .{ .int64_value = value }, - .value_size = 1, - } }; - } - - pub fn fromInt64List(name_: []const u8, value: []const i64) NamedValue { - return .{ .inner = c.PJRT_NamedValue{ - .struct_size = c.PJRT_NamedValue_STRUCT_SIZE, - .extension_start = null, - .name = @as([*c]const u8, @ptrCast(@constCast(name_.ptr))), - .name_size = name_.len, - .type = c.PJRT_NamedValue_kInt64List, - .unnamed_0 = .{ .int64_array_value = @ptrCast(@constCast(value.ptr)) }, - .value_size = value.len, - } }; - } - - pub fn fromFloat(name_: []const u8, value: f32) NamedValue { - return .{ .inner = c.PJRT_NamedValue{ - .struct_size = c.PJRT_NamedValue_STRUCT_SIZE, - .extension_start = null, - .name = @as([*c]const u8, @ptrCast(@constCast(name_.ptr))), - .name_size = name_.len, - .type = c.PJRT_NamedValue_kFloat, - .unnamed_0 = .{ .float_value = value }, - .value_size = 1, - } }; - } - - pub fn fromBool(name_: []const u8, value: bool) NamedValue { - return .{ .inner = c.PJRT_NamedValue{ - .struct_size = c.PJRT_NamedValue_STRUCT_SIZE, - .extension_start = null, - .name = @as([*c]const u8, @ptrCast(@constCast(name_.ptr))), - .name_size = name_.len, - .type = c.PJRT_NamedValue_kBool, - .unnamed_0 = .{ .bool_value = value }, - .value_size = 1, - } }; + pub fn init(comptime kind_: Kind, name_: []const u8, value_: std.meta.fieldInfo(Value, kind_).type) NamedValue { + return .{ + .inner = .{ + .struct_size = c.PJRT_NamedValue_STRUCT_SIZE, + .extension_start = null, + .name = @ptrCast(name_), + .name_size = name_.len, + .type = @intFromEnum(kind_), + .unnamed_0 = switch (kind_) { + .string => .{ .string_value = @ptrCast(@constCast(value_.ptr)) }, + .int64 => .{ .int64_value = value_ }, + .int64list => .{ .int64_array_value = @ptrCast(@constCast(value_.ptr)) }, + .float => .{ .float_value = value_ }, + .bool => .{ .bool_value = value_ }, + }, + .value_size = switch (kind_) { + .string, .int64list => value_.len, + inline else => 1, + }, + }, + }; } pub fn format(self: NamedValue, writer: *std.Io.Writer) !void { - try writer.print("{s}{{ .name = {s},", .{ @typeName(NamedValue), self.inner.name[0..self.inner.name_size] }); - const u = self.inner.unnamed_0; - switch (self.kind()) { - .string => try writer.print(" .string = {s} ", .{u.string_value[0..self.inner.value_size]}), - .int64 => try writer.print(" .int64 = {d} ", .{u.int64_value}), - .int64list => try writer.print(" .int64list = {any} ", .{u.int64_array_value[0..self.inner.value_size]}), - .float => try writer.print(" .float = {d} ", .{u.float_value}), - .bool => try writer.print(" .bool = {} ", .{u.bool_value}), + try writer.print("{s}{{ .name = {s},", .{ @typeName(NamedValue), self.name() }); + switch (self.value()) { + .string => |v| try writer.print(" .string = \"{s}\" ", .{v}), + .int64 => |v| try writer.print(" .int64 = {d} ", .{v}), + .int64list => |v| try writer.print(" .int64list = {any} ", .{v}), + .float => |v| try writer.print(" .float = {d} ", .{v}), + .bool => |v| try writer.print(" .bool = {} ", .{v}), } try writer.writeAll("}"); } diff --git a/zml/platform.zig b/zml/platform.zig index e03783c1..6687001d 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -267,7 +267,7 @@ pub const CreateOptions = struct { device_count: u32, fn writeNamedValues(self: Cpu, values: *std.ArrayList(pjrt.NamedValue)) void { - values.appendAssumeCapacity(pjrt.NamedValue.from("cpu_device_count", @as(i64, self.device_count))); + values.appendAssumeCapacity(.init(.int64, "cpu_device_count", self.device_count)); } }; @@ -291,28 +291,27 @@ pub const CreateOptions = struct { pub const Options = struct { preallocate: bool = true, memory_fraction: f32 = 0.90, - collective_memory_size_mb: u32 = 0, + collective_memory_size_mb: i64 = 0, }; }; fn writeNamedValues(self: Cuda, values: *std.ArrayList(pjrt.NamedValue)) void { switch (self.allocator) { .platform => { - values.appendAssumeCapacity(.fromString("allocator", "platform")); + values.appendAssumeCapacity(.init(.string, "allocator", "platform")); }, .bfc, .async => |opt| { - values.appendAssumeCapacity(.fromString("allocator", switch (self.allocator) { + values.appendAssumeCapacity(.init(.string, "allocator", switch (self.allocator) { .bfc => "bfc", .async => "cuda_async", .platform => unreachable, })); - values.appendAssumeCapacity(.from("preallocate", opt.preallocate)); + values.appendAssumeCapacity(.init(.bool, "preallocate", opt.preallocate)); if (opt.memory_fraction > 0) { - values.appendAssumeCapacity(.from("memory_fraction", opt.memory_fraction)); + values.appendAssumeCapacity(.init(.float, "memory_fraction", opt.memory_fraction)); } if (opt.collective_memory_size_mb > 0) { - const collective = @as(i64, opt.collective_memory_size_mb) * 1024 * 1024; - values.appendAssumeCapacity(.from("collective_memory_size", collective)); + values.appendAssumeCapacity(.init(.int64, "collective_memory_size", opt.collective_memory_size_mb * 1024 * 1024)); } }, } From d42e4f99adaabe49faa8eda4c27f50a1789fc703 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 8 Jan 2026 09:09:31 +0000 Subject: [PATCH 209/881] pjrt: add DeviceDescription.attributes and attribute helper for getting only one --- pjrt/pjrt.zig | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 74b416f3..5d35689f 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -648,6 +648,22 @@ pub const DeviceDescription = opaque { }) catch unreachable; return ret.to_string[0..ret.to_string_size]; } + + pub fn attributes(self: *const DeviceDescription, api: *const Api) []const NamedValue { + const ret = api.call(.PJRT_DeviceDescription_Attributes, .{ + .device_description = self.inner(), + }) catch unreachable; + return @ptrCast(ret.attributes[0..ret.num_attributes]); + } + + pub fn attribute(self: *const DeviceDescription, api: *const Api, name: []const u8) ?NamedValue.Value { + for (self.attributes(api)) |attr| { + if (std.mem.eql(u8, attr.name(), name)) { + return attr.value(); + } + } + return null; + } }; pub const GetCostAnalysisError = std.mem.Allocator.Error || ApiError; From e20e22483365dc5c847a049f5724ccea23693093 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 8 Jan 2026 09:11:18 +0000 Subject: [PATCH 210/881] zml: on rocm, only enable triton gemm when using CDNA --- zml/module.zig | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/zml/module.zig b/zml/module.zig index 7d7e0cbe..5f82062b 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -467,9 +467,28 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_llvm_module_compilation_parallelism", true, upb_arena); }, .rocm => { - // Disable Triton GEMM on ROCM. For some reason it's much, much slower when - // enabled on CDNA and it's used on RDNA. Disable it altogether. - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_triton_gemm", false, upb_arena); + // enable only on CDNA + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_triton_gemm", gemm_blk: { + var supported: std.StaticStringMap(void) = .initComptime(.{ + .{ "gfx950", {} }, + .{ "gfx950", {} }, + .{ "gfx942", {} }, + .{ "gfx942", {} }, + .{ "gfx942", {} }, + .{ "gfx90a", {} }, + .{ "gfx90a", {} }, + .{ "gfx90a", {} }, + .{ "gfx908", {} }, + .{ "gfx906", {} }, + .{ "gfx900", {} }, + }); + const first_device = platform.pjrt_client.getDevices(platform.pjrt_api)[0]; + const description = first_device.getDescription(platform.pjrt_api); + break :gemm_blk if (description.attribute(platform.pjrt_api, "compute_capability")) |cc| + supported.has(cc.string) + else + false; + }, upb_arena); // Use lld from libllvm instead of invoking the ld.lld binary. // This saves us from having to sandbox it. try setXlaOverrideFlag(overrides_map, "xla_gpu_use_inprocess_lld", true, upb_arena); From 33775378b99d54a12062dee3586214e73e5fcf51 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 8 Jan 2026 09:12:54 +0000 Subject: [PATCH 211/881] zml: disable command buffer on rocm This avoids weird crashes, AMD recommended disabling it for now. --- zml/module.zig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zml/module.zig b/zml/module.zig index 5f82062b..5a9607b4 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -492,6 +492,9 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: // Use lld from libllvm instead of invoking the ld.lld binary. // This saves us from having to sandbox it. try setXlaOverrideFlag(overrides_map, "xla_gpu_use_inprocess_lld", true, upb_arena); + // Disable command buffer to avoid some weird crashes. + // This is what AMD recommended in the meantime. + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "", upb_arena); }, else => {}, } From 2b0a8e653c840e792054dc9b184248c70554db9c Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 8 Jan 2026 09:13:07 +0000 Subject: [PATCH 212/881] zml: on cuda, re-enable triton gemm by default NVIDIA doesn't recommend disabling it anymore --- zml/module.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/zml/module.zig b/zml/module.zig index 5a9607b4..fc4aa495 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -462,7 +462,6 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: .cuda => { // NVIDIA recommends these settings // https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_triton_gemm", false, upb_arena); try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_latency_hiding_scheduler", true, upb_arena); try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_llvm_module_compilation_parallelism", true, upb_arena); }, From fe2d33eca144e1700341684906ee6f5a3068e5fc Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 8 Jan 2026 09:31:44 +0000 Subject: [PATCH 213/881] zml: force unwrap on compute_capability attribute on rocm this should absolutely fail if this doesn't exist --- zml/module.zig | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/zml/module.zig b/zml/module.zig index fc4aa495..ab419bf9 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -468,7 +468,7 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: .rocm => { // enable only on CDNA try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_triton_gemm", gemm_blk: { - var supported: std.StaticStringMap(void) = .initComptime(.{ + const supported: std.StaticStringMap(void) = .initComptime(.{ .{ "gfx950", {} }, .{ "gfx950", {} }, .{ "gfx942", {} }, @@ -483,10 +483,7 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: }); const first_device = platform.pjrt_client.getDevices(platform.pjrt_api)[0]; const description = first_device.getDescription(platform.pjrt_api); - break :gemm_blk if (description.attribute(platform.pjrt_api, "compute_capability")) |cc| - supported.has(cc.string) - else - false; + break :gemm_blk supported.has(description.attribute(platform.pjrt_api, "compute_capability").?.string); }, upb_arena); // Use lld from libllvm instead of invoking the ld.lld binary. // This saves us from having to sandbox it. From a39fece38ce8a7af5cea7d496975335b922f59f2 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 9 Jan 2026 10:45:00 +0100 Subject: [PATCH 214/881] WIP BUFFEREDWRITER --- zml/io.zig | 58 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/zml/io.zig b/zml/io.zig index 24454e33..8a27f842 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -222,8 +222,8 @@ pub fn loadBuffersFromId(allocator: std.mem.Allocator, io: std.Io, model: anytyp const shapes = try collectShapes(arena.allocator(), &model); - var transfer: Transfer = try .init(arena.allocator(), shapes, platform); - defer transfer.deinit(platform); + // var transfer: Transfer = try .init(arena.allocator(), shapes, platform); + // defer transfer.deinit(platform); const tensor_descs = try collectTensorDesc(arena.allocator(), store, &model); @@ -237,7 +237,7 @@ pub fn loadBuffersFromId(allocator: std.mem.Allocator, io: std.Io, model: anytyp tensor_descs: []safetensors.Tensor, shapes: []const Shape, platform: Platform, - transfer: *Transfer, + // transfer: *Transfer, index: usize = 0, buffer_reader: []u8, buffer_writer: []u8, @@ -249,7 +249,7 @@ pub fn loadBuffersFromId(allocator: std.mem.Allocator, io: std.Io, model: anytyp .tensor_descs = tensor_descs, .shapes = shapes, .platform = platform, - .transfer = &transfer, + // .transfer = &transfer, .buffer_reader = buffer_reader, .buffer_writer = buffer_writer, .store = store, @@ -263,12 +263,19 @@ pub fn loadBuffersFromId(allocator: std.mem.Allocator, io: std.Io, model: anytyp var reader = try safetensors.TensorReader.init(context_.io, tensor_desc, context_.buffer_reader); defer reader.deinit(); - var writer = try context_.transfer.getWriter(context_.io, context_.index, context_.buffer_writer); + // var writer = try context_.transfer.getWriter(context_.io, context_.index, context_.buffer_writer); + var writer = try BufferedDeviceWriter.init( + context_.allocator, + context_.io, + context_.platform, + context_.shapes[context_.index], + ); _ = try reader.interface.streamRemaining(&writer.interface); try writer.interface.flush(); - buffer.* = try context_.transfer.getBuffer(context_.index); + // buffer.* = try context_.transfer.getBuffer(context_.index); + buffer.* = writer.buffer.?; context_.index += 1; } }.cb, &context, &result); @@ -398,3 +405,42 @@ pub const Transfer = struct { return writer; } }; + +pub const BufferedDeviceWriter = struct { + allocator: std.mem.Allocator, + io: std.Io, + transfer_buffer: []u8, + platform: Platform, + shape: Shape, + buffer: ?Buffer = null, + interface: std.Io.Writer, + + pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: Platform, shape: Shape) !BufferedDeviceWriter { + const transfer_buffer = try allocator.alloc(u8, shape.byteSize()); + return .{ + .allocator = allocator, + .io = io, + .transfer_buffer = transfer_buffer, + .shape = shape, + .platform = platform, + .interface = .{ + .buffer = transfer_buffer, + .vtable = &.{ + .drain = std.Io.Writer.fixedDrain, + .flush = flush, + .rebase = std.Io.Writer.failingRebase, + }, + }, + }; + } + + pub fn deinit(self: *BufferedDeviceWriter) void { + self.allocator.free(self.transfer_buffer); + } + + pub fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { + const self: *BufferedDeviceWriter = @alignCast(@fieldParentPtr("interface", w)); + self.buffer = Buffer.fromBytes(self.io, self.platform, self.shape, self.transfer_buffer) catch return std.Io.Writer.Error.WriteFailed; + _ = self.buffer.?.await(self.io) catch return std.Io.Writer.Error.WriteFailed; + } +}; From 008390f3058322e8d80d57e3a37439d6a13c8034 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 9 Jan 2026 16:37:52 +0100 Subject: [PATCH 215/881] runtimes/neuron: bump neuron dependencies --- MODULE.bazel | 12 +- MODULE.bazel.lock | 247 ++++++++++- runtimes/neuron/BUILD.bazel | 3 +- runtimes/neuron/packages.lock.json | 16 +- runtimes/neuron/requirements.in | 4 +- runtimes/neuron/requirements.lock.txt | 584 ++++++++++++++++---------- 6 files changed, 613 insertions(+), 253 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 6df1a6bb..6c7ccdc0 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -20,7 +20,7 @@ bazel_dep(name = "rules_distroless", version = "0.5.1") bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep(name = "rules_oci", version = "2.2.6") bazel_dep(name = "rules_proto", version = "7.1.0") -bazel_dep(name = "rules_python", version = "1.5.3") +bazel_dep(name = "rules_python", version = "1.8.0-rc1") bazel_dep(name = "rules_rust", version = "0.68.1") bazel_dep(name = "rules_zig", version = "0.12.2") bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.3.1") @@ -60,16 +60,16 @@ uv = use_extension("@rules_python//python/uv:uv.bzl", "uv", dev_dependency = Tru uv.configure(version = "0.6.5") python = use_extension("@rules_python//python/extensions:python.bzl", "python") -python.toolchain(python_version = "3.11") +python.toolchain(python_version = "3.12") use_repo(python, "python_versions") pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") pip.parse( download_only = True, extra_pip_args = [ - "--abi=cp311", + "--abi=cp312", "--implementation=cp", - "--python-version=311", + "--python-version=312", "--platform=linux_x86_64", "--platform=manylinux2014_x86_64", "--platform=manylinux_2_27_x86_64", @@ -77,7 +77,7 @@ pip.parse( "--platform=manylinux_2_29_x86_64", ], hub_name = "neuron_py_deps", - python_version = "3.11", + python_version = "3.12", requirements_lock = "//runtimes/neuron:requirements.lock.txt", ) use_repo(pip, "neuron_py_deps") @@ -86,7 +86,7 @@ use_repo(pip, "neuron_py_deps") pip.parse( download_only = True, hub_name = "huggingface_hub", - python_version = "3.11", + python_version = "3.12", requirements_lock = "//tools/hf:requirements.lock.txt", ) use_repo(pip, "huggingface_hub") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 1f3593cd..454f2511 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -111,6 +111,7 @@ "https://bazel-registry.zml.ai/modules/rules_cc/0.0.8/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.9/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.1.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_cc/0.1.5/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.12/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.14/MODULE.bazel": "not found", @@ -166,8 +167,8 @@ "https://bazel-registry.zml.ai/modules/rules_python/0.33.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_python/0.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_python/0.40.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/1.5.3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_python/1.6.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_python/1.8.0-rc1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_rust/0.68.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.2.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.3.0/MODULE.bazel": "not found", @@ -329,6 +330,7 @@ "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", "https://bcr.bazel.build/modules/rules_cc/0.1.1/MODULE.bazel": "2f0222a6f229f0bf44cd711dc13c858dad98c62d52bd51d8fc3a764a83125513", + "https://bcr.bazel.build/modules/rules_cc/0.1.5/MODULE.bazel": "88dfc9361e8b5ae1008ac38f7cdfd45ad738e4fa676a3ad67d19204f045a1fd8", "https://bcr.bazel.build/modules/rules_cc/0.2.0/MODULE.bazel": "b5c17f90458caae90d2ccd114c81970062946f49f355610ed89bebf954f5783c", "https://bcr.bazel.build/modules/rules_cc/0.2.12/MODULE.bazel": "4216c383ce3223c7dfbd3afcb09c7056265156150b2a3bd3fd9d02949a7ee1bc", "https://bcr.bazel.build/modules/rules_cc/0.2.14/MODULE.bazel": "353c99ed148887ee89c54a17d4100ae7e7e436593d104b668476019023b58df8", @@ -393,9 +395,9 @@ "https://bcr.bazel.build/modules/rules_python/0.33.2/MODULE.bazel": "3e036c4ad8d804a4dad897d333d8dce200d943df4827cb849840055be8d2e937", "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", "https://bcr.bazel.build/modules/rules_python/0.40.0/MODULE.bazel": "9d1a3cd88ed7d8e39583d9ffe56ae8a244f67783ae89b60caafc9f5cf318ada7", - "https://bcr.bazel.build/modules/rules_python/1.5.3/MODULE.bazel": "d0b7fb08458ca7fd80a26bc00c9e0f1d011609cc3da0381faa2eccd88c6ebd98", "https://bcr.bazel.build/modules/rules_python/1.6.0/MODULE.bazel": "7e04ad8f8d5bea40451cf80b1bd8262552aa73f841415d20db96b7241bd027d8", - "https://bcr.bazel.build/modules/rules_python/1.6.0/source.json": "e980f654cf66ec4928672f41fc66c4102b5ea54286acf4aecd23256c84211be6", + "https://bcr.bazel.build/modules/rules_python/1.8.0-rc1/MODULE.bazel": "d5348333fd8be9589c3ea8d9110fd00b1c7d84bc7b505668307cbe9f105e9c8f", + "https://bcr.bazel.build/modules/rules_python/1.8.0-rc1/source.json": "d26719d5b92a569eaa387bad9667a042d4810193f279ea64590c9d76ae38f3ba", "https://bcr.bazel.build/modules/rules_rust/0.68.1/MODULE.bazel": "8d3332ef4079673385eb81f8bd68b012decc04ac00c9d5a01a40eff90301732c", "https://bcr.bazel.build/modules/rules_rust/0.68.1/source.json": "3378e746f81b62457fdfd37391244fa8ff075ba85c05931ee4f3a20ac1efe963", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", @@ -1020,10 +1022,245 @@ ] } }, + "@@rules_python+//python/extensions:config.bzl%config": { + "general": { + "bzlTransitiveDigest": "0rWbrtB3JZ8y7lycaRGpCqLuh1Pt+DXhVnHBD0vU2JI=", + "usagesDigest": "tIEieEA/gbsjNF3L/Oouyg6UdqGOVxFsPqBjFxkTAKM=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "rules_python_internal": { + "repoRuleId": "@@rules_python+//python/private:internal_config_repo.bzl%internal_config_repo", + "attributes": { + "transition_setting_generators": {}, + "transition_settings": [] + } + }, + "pypi__build": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/e2/03/f3c8ba0a6b6e30d7d18c40faab90807c9bb5e9a1e3b2fe2008af624a9c97/build-1.2.1-py3-none-any.whl", + "sha256": "75e10f767a433d9a86e50d83f418e83efc18ede923ee5ff7df93b6cb0306c5d4", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__click": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl", + "sha256": "ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__colorama": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", + "sha256": "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__importlib_metadata": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/2d/0a/679461c511447ffaf176567d5c496d1de27cbe34a87df6677d7171b2fbd4/importlib_metadata-7.1.0-py3-none-any.whl", + "sha256": "30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__installer": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl", + "sha256": "05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__more_itertools": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/50/e2/8e10e465ee3987bb7c9ab69efb91d867d93959095f4807db102d07995d94/more_itertools-10.2.0-py3-none-any.whl", + "sha256": "686b06abe565edfab151cb8fd385a05651e1fdf8f0a14191e4439283421f8684", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__packaging": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/49/df/1fceb2f8900f8639e278b056416d49134fb8d84c5942ffaa01ad34782422/packaging-24.0-py3-none-any.whl", + "sha256": "2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__pep517": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/25/6e/ca4a5434eb0e502210f591b97537d322546e4833dcb4d470a48c375c5540/pep517-0.13.1-py3-none-any.whl", + "sha256": "31b206f67165b3536dd577c5c3f1518e8fbaf38cbc57efff8369a392feff1721", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__pip": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/8a/6a/19e9fe04fca059ccf770861c7d5721ab4c2aebc539889e97c7977528a53b/pip-24.0-py3-none-any.whl", + "sha256": "ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__pip_tools": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/0d/dc/38f4ce065e92c66f058ea7a368a9c5de4e702272b479c0992059f7693941/pip_tools-7.4.1-py3-none-any.whl", + "sha256": "4c690e5fbae2f21e87843e89c26191f0d9454f362d8acdbd695716493ec8b3a9", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__pyproject_hooks": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/ae/f3/431b9d5fe7d14af7a32340792ef43b8a714e7726f1d7b69cc4e8e7a3f1d7/pyproject_hooks-1.1.0-py3-none-any.whl", + "sha256": "7ceeefe9aec63a1064c18d939bdc3adf2d8aa1988a510afec15151578b232aa2", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__setuptools": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/90/99/158ad0609729111163fc1f674a5a42f2605371a4cf036d0441070e2f7455/setuptools-78.1.1-py3-none-any.whl", + "sha256": "c3a9c4211ff4c309edb8b8c4f1cbfa7ae324c4ba9f91ff254e3d305b9fd54561", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__tomli": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", + "sha256": "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__wheel": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/7d/cd/d7460c9a869b16c3dd4e1e403cce337df165368c71d6af229a74699622ce/wheel-0.43.0-py3-none-any.whl", + "sha256": "55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__zipp": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/da/55/a03fd7240714916507e1fcf7ae355bd9d9ed2e6db492595f1a67f61681be/zipp-3.18.2-py3-none-any.whl", + "sha256": "dce197b859eb796242b0622af1b8beb0a722d52aa2f57133ead08edd5bf5374e", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_python+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_python+", + "pypi__build", + "rules_python++config+pypi__build" + ], + [ + "rules_python+", + "pypi__click", + "rules_python++config+pypi__click" + ], + [ + "rules_python+", + "pypi__colorama", + "rules_python++config+pypi__colorama" + ], + [ + "rules_python+", + "pypi__importlib_metadata", + "rules_python++config+pypi__importlib_metadata" + ], + [ + "rules_python+", + "pypi__installer", + "rules_python++config+pypi__installer" + ], + [ + "rules_python+", + "pypi__more_itertools", + "rules_python++config+pypi__more_itertools" + ], + [ + "rules_python+", + "pypi__packaging", + "rules_python++config+pypi__packaging" + ], + [ + "rules_python+", + "pypi__pep517", + "rules_python++config+pypi__pep517" + ], + [ + "rules_python+", + "pypi__pip", + "rules_python++config+pypi__pip" + ], + [ + "rules_python+", + "pypi__pip_tools", + "rules_python++config+pypi__pip_tools" + ], + [ + "rules_python+", + "pypi__pyproject_hooks", + "rules_python++config+pypi__pyproject_hooks" + ], + [ + "rules_python+", + "pypi__setuptools", + "rules_python++config+pypi__setuptools" + ], + [ + "rules_python+", + "pypi__tomli", + "rules_python++config+pypi__tomli" + ], + [ + "rules_python+", + "pypi__wheel", + "rules_python++config+pypi__wheel" + ], + [ + "rules_python+", + "pypi__zipp", + "rules_python++config+pypi__zipp" + ] + ] + } + }, "@@rules_python+//python/uv:uv.bzl%uv": { "general": { - "bzlTransitiveDigest": "Vd/0cN91U7O//KVXEBJsnr64LUyHdC6DRiDTTnQTrUw=", - "usagesDigest": "IlyRxs7yhBn46gx9AgZj1f1kRAknvGnAoqPCzcMEPDA=", + "bzlTransitiveDigest": "bb4QpQxvjqUS7bhlSKPbiLJgYNlcA0T5b9vYziRFobM=", + "usagesDigest": "gx0mA10eprg2o8cg9R+m/jpqCuVjnTnNzD17TPQdFu0=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, diff --git a/runtimes/neuron/BUILD.bazel b/runtimes/neuron/BUILD.bazel index 993ed5e5..c9eb2f84 100644 --- a/runtimes/neuron/BUILD.bazel +++ b/runtimes/neuron/BUILD.bazel @@ -105,8 +105,9 @@ uv_lock( "--emit-find-links", "--index-strategy=unsafe-best-match", "--upgrade", + "--python-platform=x86_64-unknown-linux-gnu", ], - python_version = "3.11", + python_version = "3.12", tags = ["manual"], ) diff --git a/runtimes/neuron/packages.lock.json b/runtimes/neuron/packages.lock.json index ff484ae5..134d55e4 100755 --- a/runtimes/neuron/packages.lock.json +++ b/runtimes/neuron/packages.lock.json @@ -181,13 +181,13 @@ { "arch": "amd64", "dependencies": [], - "key": "aws-neuronx-runtime-lib_2.26.42.0-2ff3b5c7d_amd64", + "key": "aws-neuronx-runtime-lib_2.29.40.0-f954cd7a5_amd64", "name": "aws-neuronx-runtime-lib", - "sha256": "57f6fd8e850f4054c2d229cd41a83fe1489ec9d9ee55bedf33a14cd6e8beed16", + "sha256": "60f5dc0c8731fdd48a9726493375dbb0f0d178d05d84f70716bcc36789fd9be9", "urls": [ - "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-runtime-lib/aws-neuronx-runtime-lib_2.26.42.0-2ff3b5c7d_amd64.deb" + "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-runtime-lib/aws-neuronx-runtime-lib_2.29.40.0-f954cd7a5_amd64.deb" ], - "version": "2.26.42.0-2ff3b5c7d" + "version": "2.29.40.0-f954cd7a5" }, { "arch": "amd64", @@ -333,13 +333,13 @@ "version": "252.36-1~deb12u1" } ], - "key": "aws-neuronx-collectives_2.26.43.0-47cc904ea_amd64", + "key": "aws-neuronx-collectives_2.29.41.0-681fef5f5_amd64", "name": "aws-neuronx-collectives", - "sha256": "4886f5aeeea21e4d502d445654a838190e2174a52ffb757da2cc3a8bf09be3f8", + "sha256": "c4bd37dfd6a18883ea87b9e7e665ee266f3e344536d0f4e9c9d6da4aa38e7417", "urls": [ - "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-collectives/aws-neuronx-collectives_2.26.43.0-47cc904ea_amd64.deb" + "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-collectives/aws-neuronx-collectives_2.29.41.0-681fef5f5_amd64.deb" ], - "version": "2.26.43.0-47cc904ea" + "version": "2.29.41.0-681fef5f5" }, { "arch": "amd64", diff --git a/runtimes/neuron/requirements.in b/runtimes/neuron/requirements.in index 902c14c6..f574686b 100644 --- a/runtimes/neuron/requirements.in +++ b/runtimes/neuron/requirements.in @@ -1,4 +1,4 @@ --find-links https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html --extra-index-url=https://pip.repos.neuron.amazonaws.com -libneuronxla -neuronx-cc +libneuronxla>=2.2 +neuronx-cc>=2.22 diff --git a/runtimes/neuron/requirements.lock.txt b/runtimes/neuron/requirements.lock.txt index 9332b2f4..cf1301fa 100644 --- a/runtimes/neuron/requirements.lock.txt +++ b/runtimes/neuron/requirements.lock.txt @@ -4,109 +4,143 @@ --extra-index-url https://pip.repos.neuron.amazonaws.com --find-links https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html -boto3==1.40.14 \ - --hash=sha256:ab5798a03582d09c0de132d080c9aee53d5647b6461261a5b7621170ec80d92b \ - --hash=sha256:d1d9998fc2b9619fc796c859d263ac81793d783e79331be62931b353dd1b68b9 +boto3==1.42.24 \ + --hash=sha256:8ed6ad670a5a2d7f66c1b0d3362791b48392c7a08f78479f5d8ab319a4d9118f \ + --hash=sha256:c47a2f40df933e3861fc66fd8d6b87ee36d4361663a7e7ba39a87f5a78b2eae1 # via libneuronxla -botocore==1.40.14 \ - --hash=sha256:06c852be83543c8d45e18a530abcad31659db8b16bcb66fd371c930d38017e7c \ - --hash=sha256:1810494c8c4190f20f9e17f2da4f7ee91eda863f429c6d690ea17ec41a8f83c4 +botocore==1.42.24 \ + --hash=sha256:8fca9781d7c84f7ad070fceffaff7179c4aa7a5ffb27b43df9d1d957801e0a8d \ + --hash=sha256:be8d1bea64fb91eea08254a1e5fea057e4428d08e61f4e11083a02cafc1f8cc6 # via # boto3 # libneuronxla # s3transfer -certifi==2025.8.3 \ - --hash=sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407 \ - --hash=sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5 +certifi==2026.1.4 \ + --hash=sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c \ + --hash=sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120 # via requests -charset-normalizer==3.4.3 \ - --hash=sha256:00237675befef519d9af72169d8604a067d92755e84fe76492fef5441db05b91 \ - --hash=sha256:02425242e96bcf29a49711b0ca9f37e451da7c70562bc10e8ed992a5a7a25cc0 \ - --hash=sha256:027b776c26d38b7f15b26a5da1044f376455fb3766df8fc38563b4efbc515154 \ - --hash=sha256:07a0eae9e2787b586e129fdcbe1af6997f8d0e5abaa0bc98c0e20e124d67e601 \ - --hash=sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884 \ - --hash=sha256:0e78314bdc32fa80696f72fa16dc61168fda4d6a0c014e0380f9d02f0e5d8a07 \ - --hash=sha256:0f2be7e0cf7754b9a30eb01f4295cc3d4358a479843b31f328afd210e2c7598c \ - --hash=sha256:13faeacfe61784e2559e690fc53fa4c5ae97c6fcedb8eb6fb8d0a15b475d2c64 \ - --hash=sha256:14c2a87c65b351109f6abfc424cab3927b3bdece6f706e4d12faaf3d52ee5efe \ - --hash=sha256:1606f4a55c0fd363d754049cdf400175ee96c992b1f8018b993941f221221c5f \ - --hash=sha256:16a8770207946ac75703458e2c743631c79c59c5890c80011d536248f8eaa432 \ - --hash=sha256:18343b2d246dc6761a249ba1fb13f9ee9a2bcd95decc767319506056ea4ad4dc \ - --hash=sha256:18b97b8404387b96cdbd30ad660f6407799126d26a39ca65729162fd810a99aa \ - --hash=sha256:1bb60174149316da1c35fa5233681f7c0f9f514509b8e399ab70fea5f17e45c9 \ - --hash=sha256:1e8ac75d72fa3775e0b7cb7e4629cec13b7514d928d15ef8ea06bca03ef01cae \ - --hash=sha256:1ef99f0456d3d46a50945c98de1774da86f8e992ab5c77865ea8b8195341fc19 \ - --hash=sha256:2001a39612b241dae17b4687898843f254f8748b796a2e16f1051a17078d991d \ - --hash=sha256:23b6b24d74478dc833444cbd927c338349d6ae852ba53a0d02a2de1fce45b96e \ - --hash=sha256:252098c8c7a873e17dd696ed98bbe91dbacd571da4b87df3736768efa7a792e4 \ - --hash=sha256:257f26fed7d7ff59921b78244f3cd93ed2af1800ff048c33f624c87475819dd7 \ - --hash=sha256:2c322db9c8c89009a990ef07c3bcc9f011a3269bc06782f916cd3d9eed7c9312 \ - --hash=sha256:30a96e1e1f865f78b030d65241c1ee850cdf422d869e9028e2fc1d5e4db73b92 \ - --hash=sha256:30d006f98569de3459c2fc1f2acde170b7b2bd265dc1943e87e1a4efe1b67c31 \ - --hash=sha256:31a9a6f775f9bcd865d88ee350f0ffb0e25936a7f930ca98995c05abf1faf21c \ - --hash=sha256:320e8e66157cc4e247d9ddca8e21f427efc7a04bbd0ac8a9faf56583fa543f9f \ - --hash=sha256:34a7f768e3f985abdb42841e20e17b330ad3aaf4bb7e7aeeb73db2e70f077b99 \ - --hash=sha256:3653fad4fe3ed447a596ae8638b437f827234f01a8cd801842e43f3d0a6b281b \ - --hash=sha256:3cd35b7e8aedeb9e34c41385fda4f73ba609e561faedfae0a9e75e44ac558a15 \ - --hash=sha256:3cfb2aad70f2c6debfbcb717f23b7eb55febc0bb23dcffc0f076009da10c6392 \ - --hash=sha256:416175faf02e4b0810f1f38bcb54682878a4af94059a1cd63b8747244420801f \ - --hash=sha256:41d1fc408ff5fdfb910200ec0e74abc40387bccb3252f3f27c0676731df2b2c8 \ - --hash=sha256:42e5088973e56e31e4fa58eb6bd709e42fc03799c11c42929592889a2e54c491 \ - --hash=sha256:4ca4c094de7771a98d7fbd67d9e5dbf1eb73efa4f744a730437d8a3a5cf994f0 \ - --hash=sha256:511729f456829ef86ac41ca78c63a5cb55240ed23b4b737faca0eb1abb1c41bc \ - --hash=sha256:53cd68b185d98dde4ad8990e56a58dea83a4162161b1ea9272e5c9182ce415e0 \ - --hash=sha256:585f3b2a80fbd26b048a0be90c5aae8f06605d3c92615911c3a2b03a8a3b796f \ - --hash=sha256:5b413b0b1bfd94dbf4023ad6945889f374cd24e3f62de58d6bb102c4d9ae534a \ - --hash=sha256:5d8d01eac18c423815ed4f4a2ec3b439d654e55ee4ad610e153cf02faf67ea40 \ - --hash=sha256:6aab0f181c486f973bc7262a97f5aca3ee7e1437011ef0c2ec04b5a11d16c927 \ - --hash=sha256:6cf8fd4c04756b6b60146d98cd8a77d0cdae0e1ca20329da2ac85eed779b6849 \ - --hash=sha256:6fb70de56f1859a3f71261cbe41005f56a7842cc348d3aeb26237560bfa5e0ce \ - --hash=sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14 \ - --hash=sha256:70bfc5f2c318afece2f5838ea5e4c3febada0be750fcf4775641052bbba14d05 \ - --hash=sha256:73dc19b562516fc9bcf6e5d6e596df0b4eb98d87e4f79f3ae71840e6ed21361c \ - --hash=sha256:74d77e25adda8581ffc1c720f1c81ca082921329452eba58b16233ab1842141c \ - --hash=sha256:78deba4d8f9590fe4dae384aeff04082510a709957e968753ff3c48399f6f92a \ - --hash=sha256:86df271bf921c2ee3818f0522e9a5b8092ca2ad8b065ece5d7d9d0e9f4849bcc \ - --hash=sha256:88ab34806dea0671532d3f82d82b85e8fc23d7b2dd12fa837978dad9bb392a34 \ - --hash=sha256:8999f965f922ae054125286faf9f11bc6932184b93011d138925a1773830bbe9 \ - --hash=sha256:8dcfc373f888e4fb39a7bc57e93e3b845e7f462dacc008d9749568b1c4ece096 \ - --hash=sha256:939578d9d8fd4299220161fdd76e86c6a251987476f5243e8864a7844476ba14 \ - --hash=sha256:96b2b3d1a83ad55310de8c7b4a2d04d9277d5591f40761274856635acc5fcb30 \ - --hash=sha256:a2d08ac246bb48479170408d6c19f6385fa743e7157d716e144cad849b2dd94b \ - --hash=sha256:b256ee2e749283ef3ddcff51a675ff43798d92d746d1a6e4631bf8c707d22d0b \ - --hash=sha256:b5e3b2d152e74e100a9e9573837aba24aab611d39428ded46f4e4022ea7d1942 \ - --hash=sha256:b89bc04de1d83006373429975f8ef9e7932534b8cc9ca582e4db7d20d91816db \ - --hash=sha256:bd28b817ea8c70215401f657edef3a8aa83c29d447fb0b622c35403780ba11d5 \ - --hash=sha256:c60e092517a73c632ec38e290eba714e9627abe9d301c8c8a12ec32c314a2a4b \ - --hash=sha256:c6dbd0ccdda3a2ba7c2ecd9d77b37f3b5831687d8dc1b6ca5f56a4880cc7b7ce \ - --hash=sha256:c6e490913a46fa054e03699c70019ab869e990270597018cef1d8562132c2669 \ - --hash=sha256:c6f162aabe9a91a309510d74eeb6507fab5fff92337a15acbe77753d88d9dcf0 \ - --hash=sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018 \ - --hash=sha256:cc34f233c9e71701040d772aa7490318673aa7164a0efe3172b2981218c26d93 \ - --hash=sha256:cc9370a2da1ac13f0153780040f465839e6cccb4a1e44810124b4e22483c93fe \ - --hash=sha256:ccf600859c183d70eb47e05a44cd80a4ce77394d1ac0f79dbd2dd90a69a3a049 \ - --hash=sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a \ - --hash=sha256:cf1ebb7d78e1ad8ec2a8c4732c7be2e736f6e5123a4146c5b89c9d1f585f8cef \ - --hash=sha256:d0e909868420b7049dafd3a31d45125b31143eec59235311fc4c57ea26a4acd2 \ - --hash=sha256:d22dbedd33326a4a5190dd4fe9e9e693ef12160c77382d9e87919bce54f3d4ca \ - --hash=sha256:d716a916938e03231e86e43782ca7878fb602a125a91e7acb8b5112e2e96ac16 \ - --hash=sha256:d79c198e27580c8e958906f803e63cddb77653731be08851c7df0b1a14a8fc0f \ - --hash=sha256:d95bfb53c211b57198bb91c46dd5a2d8018b3af446583aab40074bf7988401cb \ - --hash=sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1 \ - --hash=sha256:ec557499516fc90fd374bf2e32349a2887a876fbf162c160e3c01b6849eaf557 \ - --hash=sha256:fb6fecfd65564f208cbf0fba07f107fb661bcd1a7c389edbced3f7a493f70e37 \ - --hash=sha256:fb731e5deb0c7ef82d698b0f4c5bb724633ee2a489401594c5c88b02e6cb15f7 \ - --hash=sha256:fb7f67a1bfa6e40b438170ebdc8158b78dc465a5a67b6dde178a46987b244a72 \ - --hash=sha256:fd10de089bcdcd1be95a2f73dbe6254798ec1bda9f450d5828c96f93e2536b9c \ - --hash=sha256:fdabf8315679312cfa71302f9bd509ded4f2f263fb5b765cf1433b39106c3cc9 +charset-normalizer==3.4.4 \ + --hash=sha256:027f6de494925c0ab2a55eab46ae5129951638a49a34d87f4c3eda90f696b4ad \ + --hash=sha256:077fbb858e903c73f6c9db43374fd213b0b6a778106bc7032446a8e8b5b38b93 \ + --hash=sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394 \ + --hash=sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89 \ + --hash=sha256:0f04b14ffe5fdc8c4933862d8306109a2c51e0704acfa35d51598eb45a1e89fc \ + --hash=sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86 \ + --hash=sha256:194f08cbb32dc406d6e1aea671a68be0823673db2832b38405deba2fb0d88f63 \ + --hash=sha256:1bee1e43c28aa63cb16e5c14e582580546b08e535299b8b6158a7c9c768a1f3d \ + --hash=sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f \ + --hash=sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8 \ + --hash=sha256:244bfb999c71b35de57821b8ea746b24e863398194a4014e4c76adc2bbdfeff0 \ + --hash=sha256:2677acec1a2f8ef614c6888b5b4ae4060cc184174a938ed4e8ef690e15d3e505 \ + --hash=sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161 \ + --hash=sha256:2aaba3b0819274cc41757a1da876f810a3e4d7b6eb25699253a4effef9e8e4af \ + --hash=sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152 \ + --hash=sha256:2c9d3c380143a1fedbff95a312aa798578371eb29da42106a29019368a475318 \ + --hash=sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72 \ + --hash=sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4 \ + --hash=sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e \ + --hash=sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3 \ + --hash=sha256:44c2a8734b333e0578090c4cd6b16f275e07aa6614ca8715e6c038e865e70576 \ + --hash=sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c \ + --hash=sha256:4902828217069c3c5c71094537a8e623f5d097858ac6ca8252f7b4d10b7560f1 \ + --hash=sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8 \ + --hash=sha256:4fe7859a4e3e8457458e2ff592f15ccb02f3da787fcd31e0183879c3ad4692a1 \ + --hash=sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2 \ + --hash=sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44 \ + --hash=sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26 \ + --hash=sha256:5947809c8a2417be3267efc979c47d76a079758166f7d43ef5ae8e9f92751f88 \ + --hash=sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016 \ + --hash=sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede \ + --hash=sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf \ + --hash=sha256:5cb4d72eea50c8868f5288b7f7f33ed276118325c1dfd3957089f6b519e1382a \ + --hash=sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc \ + --hash=sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0 \ + --hash=sha256:64b55f9dce520635f018f907ff1b0df1fdc31f2795a922fb49dd14fbcdf48c84 \ + --hash=sha256:6515f3182dbe4ea06ced2d9e8666d97b46ef4c75e326b79bb624110f122551db \ + --hash=sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1 \ + --hash=sha256:6aee717dcfead04c6eb1ce3bd29ac1e22663cdea57f943c87d1eab9a025438d7 \ + --hash=sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed \ + --hash=sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8 \ + --hash=sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133 \ + --hash=sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e \ + --hash=sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef \ + --hash=sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14 \ + --hash=sha256:778d2e08eda00f4256d7f672ca9fef386071c9202f5e4607920b86d7803387f2 \ + --hash=sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0 \ + --hash=sha256:798d75d81754988d2565bff1b97ba5a44411867c0cf32b77a7e8f8d84796b10d \ + --hash=sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828 \ + --hash=sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f \ + --hash=sha256:7c308f7e26e4363d79df40ca5b2be1c6ba9f02bdbccfed5abddb7859a6ce72cf \ + --hash=sha256:7fa17817dc5625de8a027cb8b26d9fefa3ea28c8253929b8d6649e705d2835b6 \ + --hash=sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328 \ + --hash=sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090 \ + --hash=sha256:837c2ce8c5a65a2035be9b3569c684358dfbf109fd3b6969630a87535495ceaa \ + --hash=sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381 \ + --hash=sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c \ + --hash=sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb \ + --hash=sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc \ + --hash=sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a \ + --hash=sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec \ + --hash=sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc \ + --hash=sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac \ + --hash=sha256:9cd98cdc06614a2f768d2b7286d66805f94c48cde050acdbbb7db2600ab3197e \ + --hash=sha256:9d1bb833febdff5c8927f922386db610b49db6e0d4f4ee29601d71e7c2694313 \ + --hash=sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569 \ + --hash=sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3 \ + --hash=sha256:a61900df84c667873b292c3de315a786dd8dac506704dea57bc957bd31e22c7d \ + --hash=sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525 \ + --hash=sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894 \ + --hash=sha256:a8bf8d0f749c5757af2142fe7903a9df1d2e8aa3841559b2bad34b08d0e2bcf3 \ + --hash=sha256:a9768c477b9d7bd54bc0c86dbaebdec6f03306675526c9927c0e8a04e8f94af9 \ + --hash=sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a \ + --hash=sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9 \ + --hash=sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14 \ + --hash=sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25 \ + --hash=sha256:b5d84d37db046c5ca74ee7bb47dd6cbc13f80665fdde3e8040bdd3fb015ecb50 \ + --hash=sha256:b7cf1017d601aa35e6bb650b6ad28652c9cd78ee6caff19f3c28d03e1c80acbf \ + --hash=sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1 \ + --hash=sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3 \ + --hash=sha256:c4ef880e27901b6cc782f1b95f82da9313c0eb95c3af699103088fa0ac3ce9ac \ + --hash=sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e \ + --hash=sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815 \ + --hash=sha256:cb01158d8b88ee68f15949894ccc6712278243d95f344770fa7593fa2d94410c \ + --hash=sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6 \ + --hash=sha256:cc00f04ed596e9dc0da42ed17ac5e596c6ccba999ba6bd92b0e0aef2f170f2d6 \ + --hash=sha256:cd09d08005f958f370f539f186d10aec3377d55b9eeb0d796025d4886119d76e \ + --hash=sha256:cd4b7ca9984e5e7985c12bc60a6f173f3c958eae74f3ef6624bb6b26e2abbae4 \ + --hash=sha256:ce8a0633f41a967713a59c4139d29110c07e826d131a316b50ce11b1d79b4f84 \ + --hash=sha256:cead0978fc57397645f12578bfd2d5ea9138ea0fac82b2f63f7f7c6877986a69 \ + --hash=sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15 \ + --hash=sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191 \ + --hash=sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0 \ + --hash=sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897 \ + --hash=sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd \ + --hash=sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2 \ + --hash=sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794 \ + --hash=sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d \ + --hash=sha256:e912091979546adf63357d7e2ccff9b44f026c075aeaf25a52d0e95ad2281074 \ + --hash=sha256:eaabd426fe94daf8fd157c32e571c85cb12e66692f15516a83a03264b08d06c3 \ + --hash=sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224 \ + --hash=sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838 \ + --hash=sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a \ + --hash=sha256:f155a433c2ec037d4e8df17d18922c3a0d9b3232a396690f17175d2946f0218d \ + --hash=sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d \ + --hash=sha256:f34be2938726fc13801220747472850852fe6b1ea75869a048d6f896838c896f \ + --hash=sha256:f820802628d2694cb7e56db99213f930856014862f3fd943d290ea8438d07ca8 \ + --hash=sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490 \ + --hash=sha256:f8e160feb2aed042cd657a72acc0b481212ed28b1b9a95c0cee1621b524e1966 \ + --hash=sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9 \ + --hash=sha256:fa09f53c465e532f4d3db095e0c55b615f010ad81803d383195b6b5ca6cbf5f3 \ + --hash=sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e \ + --hash=sha256:fd44c878ea55ba351104cb93cc85e74916eb8fa440ca7903e57575e97394f608 # via requests -ec2-metadata==2.15.0 \ - --hash=sha256:b066b268091ca40369fdd644b90a34ecbfd78225bce6f0e269f0bf9ce3ab1b75 \ - --hash=sha256:daeec6b93229623942ac61f5ccc3026d0079ea1e7c84a5b18a3fb631db85600f +ec2-metadata==2.17.0 \ + --hash=sha256:a696a899cb8ee1f0101eb9ec7bc2050e1ffc8574ee4cd748a52d9c748fb19d9e \ + --hash=sha256:ad98a5afd8f4f49f688e4899dc505257da3237348761eceeb0fc7fc7d00cc90d # via neuronx-cc -idna==3.10 \ - --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \ - --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 +idna==3.11 \ + --hash=sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea \ + --hash=sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902 # via requests islpy==2023.2.5 \ --hash=sha256:06446395028b88701fd84b685b1c82b9de493728906f96f576a4dca69d12dd1f \ @@ -156,126 +190,178 @@ jmespath==1.0.1 \ # via # boto3 # botocore -libneuronxla==2.2.8201.0+f46ac1ef \ - --hash=sha256:c5bb9d2c4ab45a786561e09919ef6a3e3b7839bb7b18f539130e8c8659a29b1e +libneuronxla==2.2.14584.0+06ac23d1 \ + --hash=sha256:d1e594b27716bc59b937ccd8f40e7f2b74f6c309643e83dcf511b7ea392924f2 # via -r runtimes/neuron/requirements.in lockfile==0.12.2 \ --hash=sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799 \ --hash=sha256:6c3cb24f344923d30b2785d5ad75182c8ea7ac1b6171b08657258ec7429d50fa # via python-daemon -ml-dtypes==0.5.3 \ - --hash=sha256:01de48de4537dc3c46e684b969a40ec36594e7eeb7c69e9a093e7239f030a28a \ - --hash=sha256:0a1d68a7cb53e3f640b2b6a34d12c0542da3dd935e560fdf463c0c77f339fc20 \ - --hash=sha256:0cd5a6c711b5350f3cbc2ac28def81cd1c580075ccb7955e61e9d8f4bfd40d24 \ - --hash=sha256:0e44a3761f64bc009d71ddb6d6c71008ba21b53ab6ee588dadab65e2fa79eafc \ - --hash=sha256:156418abeeda48ea4797db6776db3c5bdab9ac7be197c1233771e0880c304057 \ - --hash=sha256:19f6c3a4f635c2fc9e2aa7d91416bd7a3d649b48350c51f7f715a09370a90d93 \ - --hash=sha256:1b255acada256d1fa8c35ed07b5f6d18bc21d1556f842fbc2d5718aea2cd9e55 \ - --hash=sha256:1db60c154989af253f6c4a34e8a540c2c9dce4d770784d426945e09908fbb177 \ - --hash=sha256:2db74788fc01914a3c7f7da0763427280adfc9cd377e9604b6b64eb8097284bd \ - --hash=sha256:4a177b882667c69422402df6ed5c3428ce07ac2c1f844d8a1314944651439458 \ - --hash=sha256:4cae435a68861660af81fa3c5af16b70ca11a17275c5b662d9c6f58294e0f113 \ - --hash=sha256:5103856a225465371fe119f2fef737402b705b810bd95ad5f348e6e1a6ae21af \ - --hash=sha256:58e39349d820b5702bb6f94ea0cb2dc8ec62ee81c0267d9622067d8333596a46 \ - --hash=sha256:5ab039ffb40f3dc0aeeeba84fd6c3452781b5e15bef72e2d10bcb33e4bbffc39 \ - --hash=sha256:5ee72568d46b9533ad54f78b1e1f3067c0534c5065120ea8ecc6f210d22748b3 \ - --hash=sha256:66c2756ae6cfd7f5224e355c893cfd617fa2f747b8bbd8996152cbdebad9a184 \ - --hash=sha256:6936283b56d74fbec431ca57ce58a90a908fdbd14d4e2d22eea6d72bb208a7b7 \ - --hash=sha256:8b1a6e231b0770f2894910f1dce6d2f31d65884dbf7668f9b08d73623cdca909 \ - --hash=sha256:8bb9cd1ce63096567f5f42851f5843b5a0ea11511e50039a7649619abfb4ba6d \ - --hash=sha256:93c36a08a6d158db44f2eb9ce3258e53f24a9a4a695325a689494f0fdbc71770 \ - --hash=sha256:95ce33057ba4d05df50b1f3cfefab22e351868a843b3b15a46c65836283670c9 \ - --hash=sha256:9849ce7267444c0a717c80c6900997de4f36e2815ce34ac560a3edb2d9a64cd2 \ - --hash=sha256:9d55ea7f7baf2aed61bf1872116cefc9d0c3693b45cae3916897ee27ef4b835e \ - --hash=sha256:a4f39b9bf6555fab9bfb536cf5fdd1c1c727e8d22312078702e9ff005354b37f \ - --hash=sha256:aec640bd94c4c85c0d11e2733bd13cbb10438fb004852996ec0efbc6cacdaf70 \ - --hash=sha256:aecbd7c5272c82e54d5b99d8435fd10915d1bc704b7df15e4d9ca8dc3902be61 \ - --hash=sha256:bda32ce212baa724e03c68771e5c69f39e584ea426bfe1a701cb01508ffc7035 \ - --hash=sha256:bdcf26c2dbc926b8a35ec8cbfad7eff1a8bd8239e12478caca83a1fc2c400dc2 \ - --hash=sha256:bdf40d2aaabd3913dec11840f0d0ebb1b93134f99af6a0a4fd88ffe924928ab4 \ - --hash=sha256:c205cac07d24a29840c163d6469f61069ce4b065518519216297fc2f261f8db9 \ - --hash=sha256:c3f5ae0309d9f888fd825c2e9d0241102fadaca81d888f26f845bc8c13c1e4ee \ - --hash=sha256:cd7c0bb22d4ff86d65ad61b5dd246812e8993fbc95b558553624c33e8b6903ea \ - --hash=sha256:d0f730a17cf4f343b2c7ad50cee3bd19e969e793d2be6ed911f43086460096e4 \ - --hash=sha256:da65e5fd3eea434ccb8984c3624bc234ddcc0d9f4c81864af611aaebcc08a50e \ - --hash=sha256:e12e29764a0e66a7a31e9b8bf1de5cc0423ea72979f45909acd4292de834ccd3 +ml-dtypes==0.5.4 \ + --hash=sha256:0d2ffd05a2575b1519dc928c0b93c06339eb67173ff53acb00724502cda231cf \ + --hash=sha256:11942cbf2cf92157db91e5022633c0d9474d4dfd813a909383bd23ce828a4b7d \ + --hash=sha256:14a4fd3228af936461db66faccef6e4f41c1d82fcc30e9f8d58a08916b1d811f \ + --hash=sha256:19b9a53598f21e453ea2fbda8aa783c20faff8e1eeb0d7ab899309a0053f1483 \ + --hash=sha256:2314892cdc3fcf05e373d76d72aaa15fda9fb98625effa73c1d646f331fcecb7 \ + --hash=sha256:2b857d3af6ac0d39db1de7c706e69c7f9791627209c3d6dedbfca8c7e5faec22 \ + --hash=sha256:304ad47faa395415b9ccbcc06a0350800bc50eda70f0e45326796e27c62f18b6 \ + --hash=sha256:35f29491a3e478407f7047b8a4834e4640a77d2737e0b294d049746507af5175 \ + --hash=sha256:388d399a2152dd79a3f0456a952284a99ee5c93d3e2f8dfe25977511e0515270 \ + --hash=sha256:3bbbe120b915090d9dd1375e4684dd17a20a2491ef25d640a908281da85e73f1 \ + --hash=sha256:3d277bf3637f2a62176f4575512e9ff9ef51d00e39626d9fe4a161992f355af2 \ + --hash=sha256:4381fe2f2452a2d7589689693d3162e876b3ddb0a832cde7a414f8e1adf7eab1 \ + --hash=sha256:4ff7f3e7ca2972e7de850e7b8fcbb355304271e2933dd90814c1cb847414d6e2 \ + --hash=sha256:531eff30e4d368cb6255bc2328d070e35836aa4f282a0fb5f3a0cd7260257298 \ + --hash=sha256:533ce891ba774eabf607172254f2e7260ba5f57bdd64030c9a4fcfbd99815d0d \ + --hash=sha256:557a31a390b7e9439056644cb80ed0735a6e3e3bb09d67fd5687e4b04238d1de \ + --hash=sha256:5a0f68ca8fd8d16583dfa7793973feb86f2fbb56ce3966daf9c9f748f52a2049 \ + --hash=sha256:6a0df4223b514d799b8a1629c65ddc351b3efa833ccf7f8ea0cf654a61d1e35d \ + --hash=sha256:6c7ecb74c4bd71db68a6bea1edf8da8c34f3d9fe218f038814fd1d310ac76c90 \ + --hash=sha256:7c23c54a00ae43edf48d44066a7ec31e05fdc2eee0be2b8b50dd1903a1db94bb \ + --hash=sha256:805cef3a38f4eafae3a5bf9ebdcdb741d0bcfd9e1bd90eb54abd24f928cd2465 \ + --hash=sha256:88c982aac7cb1cbe8cbb4e7f253072b1df872701fcaf48d84ffbb433b6568f24 \ + --hash=sha256:8ab06a50fb9bf9666dd0fe5dfb4676fa2b0ac0f31ecff72a6c3af8e22c063453 \ + --hash=sha256:8c6a2dcebd6f3903e05d51960a8058d6e131fe69f952a5397e5dbabc841b6d56 \ + --hash=sha256:8c760d85a2f82e2bed75867079188c9d18dae2ee77c25a54d60e9cc79be1bc48 \ + --hash=sha256:9ad459e99793fa6e13bd5b7e6792c8f9190b4e5a1b45c63aba14a4d0a7f1d5ff \ + --hash=sha256:9bad06436568442575beb2d03389aa7456c690a5b05892c471215bfd8cf39460 \ + --hash=sha256:a174837a64f5b16cab6f368171a1a03a27936b31699d167684073ff1c4237dac \ + --hash=sha256:a7f7c643e8b1320fd958bf098aa7ecf70623a42ec5154e3be3be673f4c34d900 \ + --hash=sha256:a9b61c19040397970d18d7737375cffd83b1f36a11dd4ad19f83a016f736c3ef \ + --hash=sha256:b4b801ebe0b477be666696bda493a9be8356f1f0057a57f1e35cd26928823e5a \ + --hash=sha256:b95e97e470fe60ed493fd9ae3911d8da4ebac16bd21f87ffa2b7c588bf22ea2c \ + --hash=sha256:bc11d7e8c44a65115d05e2ab9989d1e045125d7be8e05a071a48bc76eb6d6040 \ + --hash=sha256:bfc534409c5d4b0bf945af29e5d0ab075eae9eecbb549ff8a29280db822f34f9 \ + --hash=sha256:c1a953995cccb9e25a4ae19e34316671e4e2edaebe4cf538229b1fc7109087b7 \ + --hash=sha256:cb73dccfc991691c444acc8c0012bee8f2470da826a92e3a20bb333b1a7894e6 \ + --hash=sha256:ce756d3a10d0c4067172804c9cc276ba9cc0ff47af9078ad439b075d1abdc29b \ + --hash=sha256:d81fdb088defa30eb37bf390bb7dde35d3a83ec112ac8e33d75ab28cc29dd8b0 \ + --hash=sha256:f21c9219ef48ca5ee78402d5cc831bd58ea27ce89beda894428bc67a52da5328 # via neuronx-cc networkx==2.8.8 \ --hash=sha256:230d388117af870fce5647a3c52401fcf753e94720e6ea6b4197a5355648885e \ --hash=sha256:e435dfa75b1d7195c7b8378c3859f0445cd88c6b0375c181ed66823a9ceb7524 # via neuronx-cc -neuronx-cc==2.20.9961.0+0acef03a \ - --hash=sha256:22db1e09975f400b74edc2fab24a1b86d967386f060f5a29a3ec21756ecf5b0c +neuronx-cc==2.22.12471.0+b4a00d10 \ + --hash=sha256:19fcf633f6f453b73e791b6f23d0e3f741fd967a61b7b82208bd74d80908c81a # via # -r runtimes/neuron/requirements.in # libneuronxla -numpy==1.26.4 \ - --hash=sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b \ - --hash=sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818 \ - --hash=sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20 \ - --hash=sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0 \ - --hash=sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010 \ - --hash=sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a \ - --hash=sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea \ - --hash=sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c \ - --hash=sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71 \ - --hash=sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110 \ - --hash=sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be \ - --hash=sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a \ - --hash=sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a \ - --hash=sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5 \ - --hash=sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed \ - --hash=sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd \ - --hash=sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c \ - --hash=sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e \ - --hash=sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0 \ - --hash=sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c \ - --hash=sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a \ - --hash=sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b \ - --hash=sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0 \ - --hash=sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6 \ - --hash=sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2 \ - --hash=sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a \ - --hash=sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30 \ - --hash=sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218 \ - --hash=sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5 \ - --hash=sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07 \ - --hash=sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2 \ - --hash=sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4 \ - --hash=sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764 \ - --hash=sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef \ - --hash=sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3 \ - --hash=sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f +numpy==2.4.0 \ + --hash=sha256:009bd0ea12d3c784b6639a8457537016ce5172109e585338e11334f6a7bb88ee \ + --hash=sha256:02a2038eb27f9443a8b266a66911e926566b5a6ffd1a689b588f7f35b81e7dc3 \ + --hash=sha256:0804a8e4ab070d1d35496e65ffd3cf8114c136a2b81f61dfab0de4b218aacfd5 \ + --hash=sha256:0feafc9e03128074689183031181fac0897ff169692d8492066e949041096548 \ + --hash=sha256:1889b3a3f47a7b5bee16bc25a2145bd7cb91897f815ce3499db64c7458b6d91d \ + --hash=sha256:1d8f9fde5f6dc1b6fc34df8162f3b3079365468703fee7f31d4e0cc8c63baed9 \ + --hash=sha256:1ee32359fb7543b7b7bd0b2f46294db27e29e7bbdf70541e81b190836cd83ded \ + --hash=sha256:20c115517513831860c573996e395707aa9fb691eb179200125c250e895fcd93 \ + --hash=sha256:21555da4ec4a0c942520ead42c3b0dc9477441e085c42b0fbdd6a084869a6f6b \ + --hash=sha256:23a3e9d1a6f360267e8fbb38ba5db355a6a7e9be71d7fce7ab3125e88bb646c8 \ + --hash=sha256:2a8b6bb8369abefb8bd1801b054ad50e02b3275c8614dc6e5b0373c305291037 \ + --hash=sha256:2e284ca13d5a8367e43734148622caf0b261b275673823593e3e3634a6490f83 \ + --hash=sha256:2f585f52b2baf07ff3356158d9268ea095e221371f1074fadea2f42544d58b4d \ + --hash=sha256:316b2f2584682318539f0bcaca5a496ce9ca78c88066579ebd11fd06f8e4741e \ + --hash=sha256:32ed06d0fe9cae27d8fb5f400c63ccee72370599c75e683a6358dd3a4fb50aaf \ + --hash=sha256:355354388cba60f2132df297e2d53053d4063f79077b67b481d21276d61fc4df \ + --hash=sha256:35ddc8f4914466e6fc954c76527aa91aa763682a4f6d73249ef20b418fe6effb \ + --hash=sha256:36667db4d6c1cea79c8930ab72fadfb4060feb4bfe724141cd4bd064d2e5f8ce \ + --hash=sha256:36cbfb13c152b1c7c184ddac43765db8ad672567e7bafff2cc755a09917ed2e6 \ + --hash=sha256:39699233bc72dd482da1415dcb06076e32f60eddc796a796c5fb6c5efce94667 \ + --hash=sha256:39b19251dec4de8ff8496cd0806cbe27bf0684f765abb1f4809554de93785f2d \ + --hash=sha256:3d857f55e7fdf7c38ab96c4558c95b97d1c685be6b05c249f5fdafcbd6f9899e \ + --hash=sha256:3de19cfecd1465d0dcf8a5b5ea8b3155b42ed0b639dba4b71e323d74f2a3be5e \ + --hash=sha256:3f3096405acc48887458bbf9f6814d43785ac7ba2a57ea6442b581dedbc60ce6 \ + --hash=sha256:40483b2f2d3ba7aad426443767ff5632ec3156ef09742b96913787d13c336471 \ + --hash=sha256:413aa561266a4be2d06cd2b9665e89d9f54c543f418773076a76adcf2af08bc7 \ + --hash=sha256:44aba2f0cafd287871a495fb3163408b0bd25bbce135c6f621534a07f4f7875c \ + --hash=sha256:49ff32b09f5aa0cd30a20c2b39db3e669c845589f2b7fc910365210887e39344 \ + --hash=sha256:4d1cfce39e511069b11e67cd0bd78ceff31443b7c9e5c04db73c7a19f572967c \ + --hash=sha256:57c540ed8fb1f05cb997c6761cd56db72395b0d6985e90571ff660452ade4f98 \ + --hash=sha256:5f48cb3e88fbc294dc90e215d86fbaf1c852c63dbdb6c3a3e63f45c4b57f7344 \ + --hash=sha256:5fe44e277225fd3dff6882d86d3d447205d43532c3627313d17e754fb3905a0e \ + --hash=sha256:60e8c196cd82cbbd4f130b5290007e13e6de3eca79f0d4d38014769d96a7c475 \ + --hash=sha256:65167da969cd1ec3a1df31cb221ca3a19a8aaa25370ecb17d428415e93c1935e \ + --hash=sha256:680060061adb2d74ce352628cb798cfdec399068aa7f07ba9fb818b2b3305f98 \ + --hash=sha256:6bbaebf0d11567fa8926215ae731e1d58e6ec28a8a25235b8a47405d301332db \ + --hash=sha256:6c05483c3136ac4c91b4e81903cb53a8707d316f488124d0398499a4f8e8ef51 \ + --hash=sha256:6dc8b7e2f4eb184b37655195f421836cfae6f58197b67e3ffc501f1333d993fa \ + --hash=sha256:6e504f7b16118198f138ef31ba24d985b124c2c469fe8467007cf30fd992f934 \ + --hash=sha256:74623f2ab5cc3f7c886add4f735d1031a1d2be4a4ae63c0546cfd74e7a31ddf6 \ + --hash=sha256:77f0d13fa87036d7553bf81f0e1fe3ce68d14c9976c9851744e4d3e91127e95f \ + --hash=sha256:85eef4cb5625c47ee6425c58a3502555e10f45ee973da878ac8248ad58c136f3 \ + --hash=sha256:882567b7ae57c1b1a0250208cc21a7976d8cbcc49d5a322e607e6f09c9e0bd53 \ + --hash=sha256:8b986403023c8f3bf8f487c2e6186afda156174d31c175f747d8934dfddf3479 \ + --hash=sha256:8cfa5f29a695cb7438965e6c3e8d06e0416060cf0d709c1b1c1653a939bf5c2a \ + --hash=sha256:8eafe7c36c8430b7794edeab3087dec7bf31d634d92f2af9949434b9d1964cba \ + --hash=sha256:9198f447e1dc5647d07c9a6bbe2063cc0132728cc7175b39dbc796da5b54920d \ + --hash=sha256:92190db305a6f48734d3982f2c60fa30d6b5ee9bff10f2887b930d7b40119f4c \ + --hash=sha256:927ccf5cd17c48f801f4ed43a7e5673a2724bd2171460be3e3894e6e332ef83a \ + --hash=sha256:98e81648e0b36e325ab67e46b5400a7a6d4a22b8a7c8e8bbfe20e7db7906bf95 \ + --hash=sha256:9a818668b674047fd88c4cddada7ab8f1c298812783e8328e956b78dc4807f9f \ + --hash=sha256:a152d86a3ae00ba5f47b3acf3b827509fd0b6cb7d3259665e63dafbad22a75ea \ + --hash=sha256:a2718c1de8504121714234b6f8241d0019450353276c88b9453c9c3d92e101db \ + --hash=sha256:a39fb973a726e63223287adc6dafe444ce75af952d711e400f3bf2b36ef55a7b \ + --hash=sha256:a899699294f28f7be8992853c0c60741f16ff199205e2e6cdca155762cbaa59d \ + --hash=sha256:a8fdfed3deaf1928fb7667d96e0567cdf58c2b370ea2ee7e586aa383ec2cb346 \ + --hash=sha256:aabb081ca0ec5d39591fc33018cd4b3f96e1a2dd6756282029986d00a785fba4 \ + --hash=sha256:b1f5b45829ac1848893f0ddf5cb326110604d6df96cdc255b0bf9edd154104d4 \ + --hash=sha256:b48e35f4ab6f6a7597c46e301126ceba4c44cd3280e3750f85db48b082624fa4 \ + --hash=sha256:b54c83f1c0c0f1d748dca0af516062b8829d53d1f0c402be24b4257a9c48ada6 \ + --hash=sha256:ba0cb30acd3ef11c94dc27fbfba68940652492bc107075e7ffe23057f9425681 \ + --hash=sha256:bb50ce5fb202a26fd5404620e7ef820ad1ab3558b444cb0b55beb7ef66cd2d63 \ + --hash=sha256:c95eb6db2884917d86cde0b4d4cf31adf485c8ec36bf8696dd66fa70de96f36b \ + --hash=sha256:d0ffd9e2e4441c96a9c91ec1783285d80bf835b677853fc2770a89d50c1e48ac \ + --hash=sha256:d57b5046c120561ba8fa8e4030fbb8b822f3063910fa901ffadf16e2b7128ad6 \ + --hash=sha256:d9e6a7664ddd9746e20b7325351fe1a8408d0a2bf9c63b5e898290ddc8f09544 \ + --hash=sha256:dc578891de1db95b2a35001b695451767b580bb45753717498213c5ff3c41d63 \ + --hash=sha256:e0434aa22c821f44eeb4c650b81c7fbdd8c0122c6c4b5a576a76d5a35625ecd9 \ + --hash=sha256:e06a922a469cae9a57100864caf4f8a97a1026513793969f8ba5b63137a35d25 \ + --hash=sha256:e493962256a38f58283de033d8af176c5c91c084ea30f15834f7545451c42059 \ + --hash=sha256:ecb0019d44f4cdb50b676c5d0cb4b1eae8e15d1ed3d3e6639f986fc92b2ec52c \ + --hash=sha256:f935c4493eda9069851058fa0d9e39dbf6286be690066509305e52912714dbb2 # via # ml-dtypes # neuronx-cc # scipy -pgzip==0.3.5 \ - --hash=sha256:4e13ab66ecface5c51c5af51d8cd676aa51675cf85df000f501a86cf38c208c1 \ - --hash=sha256:dd35510f59f6bd6b64e31c4baf90c10cdbb2775235fcc079b14b404fbd7f46bf +pgzip==0.4.0 \ + --hash=sha256:41e44c03a9f1bbbd390dd6b6e37a11484113f8c159a049fa81cd8a9ddde82a21 \ + --hash=sha256:91f6526f0c6e1f6c2d3522707777e5843fad7c791401745593fd01bd869b7a05 # via neuronx-cc -protobuf==6.32.0 \ - --hash=sha256:15eba1b86f193a407607112ceb9ea0ba9569aed24f93333fe9a497cf2fda37d3 \ - --hash=sha256:501fe6372fd1c8ea2a30b4d9be8f87955a64d6be9c88a973996cef5ef6f0abf1 \ - --hash=sha256:75a2aab2bd1aeb1f5dc7c5f33bcb11d82ea8c055c9becbb41c26a8c43fd7092c \ - --hash=sha256:7db8ed09024f115ac877a1427557b838705359f047b2ff2f2b2364892d19dacb \ - --hash=sha256:84f9e3c1ff6fb0308dbacb0950d8aa90694b0d0ee68e75719cb044b7078fe741 \ - --hash=sha256:a81439049127067fc49ec1d36e25c6ee1d1a2b7be930675f919258d03c04e7d2 \ - --hash=sha256:a8bdbb2f009cfc22a36d031f22a625a38b615b5e19e558a7b756b3279723e68e \ - --hash=sha256:ba377e5b67b908c8f3072a57b63e2c6a4cbd18aea4ed98d2584350dbf46f2783 \ - --hash=sha256:d52691e5bee6c860fff9a1c86ad26a13afbeb4b168cd4445c922b7e2cf85aaf0 +protobuf==6.33.2 \ + --hash=sha256:1f8017c48c07ec5859106533b682260ba3d7c5567b1ca1f24297ce03384d1b4f \ + --hash=sha256:2981c58f582f44b6b13173e12bb8656711189c2a70250845f264b877f00b1913 \ + --hash=sha256:56dc370c91fbb8ac85bc13582c9e373569668a290aa2e66a590c2a0d35ddb9e4 \ + --hash=sha256:7109dcc38a680d033ffb8bf896727423528db9163be1b6a02d6a49606dcadbfe \ + --hash=sha256:7636aad9bb01768870266de5dc009de2d1b936771b38a793f73cbbf279c91c5c \ + --hash=sha256:87eb388bd2d0f78febd8f4c8779c79247b26a5befad525008e49a6955787ff3d \ + --hash=sha256:8cd7640aee0b7828b6d03ae518b5b4806fdfc1afe8de82f79c3454f8aef29872 \ + --hash=sha256:b5d3b5625192214066d99b2b605f5783483575656784de223f00a8d00754fc0e \ + --hash=sha256:d9b19771ca75935b3a4422957bc518b0cecb978b31d1dd12037b088f6bcc0e43 \ + --hash=sha256:fc2a0e8b05b180e5fc0dd1559fe8ebdae21a27e81ac77728fb6c42b12c7419b4 # via neuronx-cc -psutil==7.0.0 \ - --hash=sha256:101d71dc322e3cffd7cea0650b09b3d08b8e7c4109dd6809fe452dfd00e58b25 \ - --hash=sha256:1e744154a6580bc968a0195fd25e80432d3afec619daf145b9e5ba16cc1d688e \ - --hash=sha256:1fcee592b4c6f146991ca55919ea3d1f8926497a713ed7faaf8225e174581e91 \ - --hash=sha256:39db632f6bb862eeccf56660871433e111b6ea58f2caea825571951d4b6aa3da \ - --hash=sha256:4b1388a4f6875d7e2aff5c4ca1cc16c545ed41dd8bb596cefea80111db353a34 \ - --hash=sha256:4cf3d4eb1aa9b348dec30105c55cd9b7d4629285735a102beb4441e38db90553 \ - --hash=sha256:7be9c3eba38beccb6495ea33afd982a44074b78f28c434a1f51cc07fd315c456 \ - --hash=sha256:84df4eb63e16849689f76b1ffcb36db7b8de703d1bc1fe41773db487621b6c17 \ - --hash=sha256:a5f098451abc2828f7dc6b58d44b532b22f2088f4999a937557b603ce72b1993 \ - --hash=sha256:ba3fcef7523064a6c9da440fc4d6bd07da93ac726b5733c29027d7dc95b39d99 +psutil==7.2.1 \ + --hash=sha256:05cc68dbb8c174828624062e73078e7e35406f4ca2d0866c272c2410d8ef06d1 \ + --hash=sha256:08a2f175e48a898c8eb8eace45ce01777f4785bc744c90aa2cc7f2fa5462a266 \ + --hash=sha256:0d67c1822c355aa6f7314d92018fb4268a76668a536f133599b91edd48759442 \ + --hash=sha256:2ceae842a78d1603753561132d5ad1b2f8a7979cb0c283f5b52fb4e6e14b1a79 \ + --hash=sha256:35630d5af80d5d0d49cfc4d64c1c13838baf6717a13effb35869a5919b854cdf \ + --hash=sha256:3fce5f92c22b00cdefd1645aa58ab4877a01679e901555067b1bd77039aa589f \ + --hash=sha256:494c513ccc53225ae23eec7fe6e1482f1b8a44674241b54561f755a898650679 \ + --hash=sha256:5e38404ca2bb30ed7267a46c02f06ff842e92da3bb8c5bfdadbd35a5722314d8 \ + --hash=sha256:81442dac7abfc2f4f4385ea9e12ddf5a796721c0f6133260687fec5c3780fa49 \ + --hash=sha256:923f8653416604e356073e6e0bccbe7c09990acef442def2f5640dd0faa9689f \ + --hash=sha256:93f3f7b0bb07711b49626e7940d6fe52aa9940ad86e8f7e74842e73189712129 \ + --hash=sha256:99a4cd17a5fdd1f3d014396502daa70b5ec21bf4ffe38393e152f8e449757d67 \ + --hash=sha256:ab2b98c9fc19f13f59628d94df5cc4cc4844bc572467d113a8b517d634e362c6 \ + --hash=sha256:b1b0671619343aa71c20ff9767eced0483e4fc9e1f489d50923738caf6a03c17 \ + --hash=sha256:b2e953fcfaedcfbc952b44744f22d16575d3aa78eb4f51ae74165b4e96e55f42 \ + --hash=sha256:ba9f33bb525b14c3ea563b2fd521a84d2fa214ec59e3e6a2858f78d0844dd60d \ + --hash=sha256:cfbe6b40ca48019a51827f20d830887b3107a74a79b01ceb8cc8de4ccb17b672 \ + --hash=sha256:d34d2ca888208eea2b5c68186841336a7f5e0b990edec929be909353a202768a \ + --hash=sha256:ea46c0d060491051d39f0d2cff4f98d5c72b288289f57a21556cc7d504db37fc \ + --hash=sha256:f7583aec590485b43ca601dd9cea0dcd65bd7bb21d30ef4ddbf4ea6b5ed1bdd3 \ + --hash=sha256:f78baafb38436d5a128f837fab2d92c276dfb48af01a240b861ae02b2413ada8 # via neuronx-cc python-daemon==3.1.2 \ --hash=sha256:b906833cef63502994ad48e2eab213259ed9bb18d54fa8774dcba2ff7864cec6 \ @@ -295,36 +381,72 @@ requests-unixsocket==0.4.1 \ --hash=sha256:60c4942e9dbecc2f64d611039fb1dfc25da382083c6434ac0316dca3ff908f4d \ --hash=sha256:b2596158c356ecee68d27ba469a52211230ac6fb0cde8b66afb19f0ed47a1995 # via neuronx-cc -s3transfer==0.13.1 \ - --hash=sha256:a981aa7429be23fe6dfc13e80e4020057cbab622b08c0315288758d67cabc724 \ - --hash=sha256:c3fdba22ba1bd367922f27ec8032d6a1cf5f10c934fb5d68cf60fd5a23d936cf +s3transfer==0.16.0 \ + --hash=sha256:18e25d66fed509e3868dc1572b3f427ff947dd2c56f844a5bf09481ad3f3b2fe \ + --hash=sha256:8e990f13268025792229cd52fa10cb7163744bf56e719e0b9cb925ab79abf920 # via boto3 -scipy==1.12.0 \ - --hash=sha256:196ebad3a4882081f62a5bf4aeb7326aa34b110e533aab23e4374fcccb0890dc \ - --hash=sha256:408c68423f9de16cb9e602528be4ce0d6312b05001f3de61fe9ec8b1263cad08 \ - --hash=sha256:4bf5abab8a36d20193c698b0f1fc282c1d083c94723902c447e5d2f1780936a3 \ - --hash=sha256:4c1020cad92772bf44b8e4cdabc1df5d87376cb219742549ef69fc9fd86282dd \ - --hash=sha256:5adfad5dbf0163397beb4aca679187d24aec085343755fcdbdeb32b3679f254c \ - --hash=sha256:5e32847e08da8d895ce09d108a494d9eb78974cf6de23063f93306a3e419960c \ - --hash=sha256:6546dc2c11a9df6926afcbdd8a3edec28566e4e785b915e849348c6dd9f3f490 \ - --hash=sha256:730badef9b827b368f351eacae2e82da414e13cf8bd5051b4bdfd720271a5371 \ - --hash=sha256:75ea2a144096b5e39402e2ff53a36fecfd3b960d786b7efd3c180e29c39e53f2 \ - --hash=sha256:78e4402e140879387187f7f25d91cc592b3501a2e51dfb320f48dfb73565f10b \ - --hash=sha256:8b8066bce124ee5531d12a74b617d9ac0ea59245246410e19bca549656d9a40a \ - --hash=sha256:8bee4993817e204d761dba10dbab0774ba5a8612e57e81319ea04d84945375ba \ - --hash=sha256:913d6e7956c3a671de3b05ccb66b11bc293f56bfdef040583a7221d9e22a2e35 \ - --hash=sha256:95e5c750d55cf518c398a8240571b0e0782c2d5a703250872f36eaf737751338 \ - --hash=sha256:9c39f92041f490422924dfdb782527a4abddf4707616e07b021de33467f917bc \ - --hash=sha256:a24024d45ce9a675c1fb8494e8e5244efea1c7a09c60beb1eeb80373d0fecc70 \ - --hash=sha256:a7ebda398f86e56178c2fa94cad15bf457a218a54a35c2a7b4490b9f9cb2676c \ - --hash=sha256:b360f1b6b2f742781299514e99ff560d1fe9bd1bff2712894b52abe528d1fd1e \ - --hash=sha256:bba1b0c7256ad75401c73e4b3cf09d1f176e9bd4248f0d3112170fb2ec4db067 \ - --hash=sha256:c3003652496f6e7c387b1cf63f4bb720951cfa18907e998ea551e6de51a04467 \ - --hash=sha256:e53958531a7c695ff66c2e7bb7b79560ffdc562e2051644c5576c39ff8efb563 \ - --hash=sha256:e646d8571804a304e1da01040d21577685ce8e2db08ac58e543eaca063453e1c \ - --hash=sha256:e7e76cc48638228212c747ada851ef355c2bb5e7f939e10952bc504c11f4e372 \ - --hash=sha256:f5f00ebaf8de24d14b8449981a2842d404152774c1a1d880c901bf454cb8e2a1 \ - --hash=sha256:f7ce148dffcd64ade37b2df9315541f9adad6efcaa86866ee7dd5db0c8f041c3 +scipy==1.16.3 \ + --hash=sha256:0151a0749efeaaab78711c78422d413c583b8cdd2011a3c1d6c794938ee9fdb2 \ + --hash=sha256:01e87659402762f43bd2fee13370553a17ada367d42e7487800bf2916535aecb \ + --hash=sha256:03192a35e661470197556de24e7cb1330d84b35b94ead65c46ad6f16f6b28f2a \ + --hash=sha256:0553371015692a898e1aa858fed67a3576c34edefa6b7ebdb4e9dde49ce5c203 \ + --hash=sha256:062246acacbe9f8210de8e751b16fc37458213f124bef161a5a02c7a39284304 \ + --hash=sha256:0c3b4dd3d9b08dbce0f3440032c52e9e2ab9f96ade2d3943313dfe51a7056959 \ + --hash=sha256:0c623a54f7b79dd88ef56da19bc2873afec9673a48f3b85b18e4d402bdd29a5a \ + --hash=sha256:16b8bc35a4cc24db80a0ec836a9286d0e31b2503cb2fd7ff7fb0e0374a97081d \ + --hash=sha256:1fb2472e72e24d1530debe6ae078db70fb1605350c88a3d14bc401d6306dbffe \ + --hash=sha256:21d9d6b197227a12dcbf9633320a4e34c6b0e51c57268df255a0942983bac562 \ + --hash=sha256:2a207a6ce9c24f1951241f4693ede2d393f59c07abc159b2cb2be980820e01fb \ + --hash=sha256:2b71d93c8a9936046866acebc915e2af2e292b883ed6e2cbe5c34beb094b82d9 \ + --hash=sha256:2d1ae2cf0c350e7705168ff2429962a89ad90c2d49d1dd300686d8b2a5af22fc \ + --hash=sha256:3a4c460301fb2cffb7f88528f30b3127742cff583603aa7dc964a52c463b385d \ + --hash=sha256:3d4a07a8e785d80289dfe66b7c27d8634a773020742ec7187b85ccc4b0e7b686 \ + --hash=sha256:40be6cf99e68b6c4321e9f8782e7d5ff8265af28ef2cd56e9c9b2638fa08ad97 \ + --hash=sha256:4aff59800a3b7f786b70bfd6ab551001cb553244988d7d6b8299cb1ea653b353 \ + --hash=sha256:50a3dbf286dbc7d84f176f9a1574c705f277cb6565069f88f60db9eafdbe3ee2 \ + --hash=sha256:532fb5ad6a87e9e9cd9c959b106b73145a03f04c7d57ea3e6f6bb60b86ab0876 \ + --hash=sha256:53c3844d527213631e886621df5695d35e4f6a75f620dca412bcd292f6b87d78 \ + --hash=sha256:56edc65510d1331dae01ef9b658d428e33ed48b4f77b1d51caf479a0253f96dc \ + --hash=sha256:57d01cb6f85e34f0946b33caa66e892aae072b64b034183f3d87c4025802a119 \ + --hash=sha256:5803c5fadd29de0cf27fa08ccbfe7a9e5d741bf63e4ab1085437266f12460ff9 \ + --hash=sha256:6020470b9d00245926f2d5bb93b119ca0340f0d564eb6fbaad843eaebf9d690f \ + --hash=sha256:63d3cdacb8a824a295191a723ee5e4ea7768ca5ca5f2838532d9f2e2b3ce2135 \ + --hash=sha256:663b8d66a8748051c3ee9c96465fb417509315b99c71550fda2591d7dd634234 \ + --hash=sha256:72d1717fd3b5e6ec747327ce9bda32d5463f472c9dce9f54499e81fbd50245a1 \ + --hash=sha256:7dc1360c06535ea6116a2220f760ae572db9f661aba2d88074fe30ec2aa1ff88 \ + --hash=sha256:7f68154688c515cdb541a31ef8eb66d8cd1050605be9dcd74199cbd22ac739bc \ + --hash=sha256:81fc5827606858cf71446a5e98715ba0e11f0dbc83d71c7409d05486592a45d6 \ + --hash=sha256:875555ce62743e1d54f06cdf22c1e0bc47b91130ac40fe5d783b6dfa114beeb6 \ + --hash=sha256:8b3c820ddb80029fe9f43d61b81d8b488d3ef8ca010d15122b152db77dc94c22 \ + --hash=sha256:8be1ca9170fcb6223cc7c27f4305d680ded114a1567c0bd2bfcbf947d1b17511 \ + --hash=sha256:8d09d72dc92742988b0e7750bddb8060b0c7079606c0d24a8cc8e9c9c11f9079 \ + --hash=sha256:9452781bd879b14b6f055b26643703551320aa8d79ae064a71df55c00286a184 \ + --hash=sha256:96491a6a54e995f00a28a3c3badfff58fd093bf26cd5fb34a2188c8c756a3a2c \ + --hash=sha256:9b9c9c07b6d56a35777a1b4cc8966118fb16cfd8daf6743867d17d36cfad2d40 \ + --hash=sha256:a8a26c78ef223d3e30920ef759e25625a0ecdd0d60e5a8818b7513c3e5384cf2 \ + --hash=sha256:aadd23f98f9cb069b3bd64ddc900c4d277778242e961751f77a8cb5c4b946fb0 \ + --hash=sha256:b7180967113560cca57418a7bc719e30366b47959dd845a93206fbed693c867e \ + --hash=sha256:b7c5f1bda1354d6a19bc6af73a649f8285ca63ac6b52e64e658a5a11d4d69800 \ + --hash=sha256:b81c27fc41954319a943d43b20e07c40bdcd3ff7cf013f4fb86286faefe546c4 \ + --hash=sha256:bb61878c18a470021fb515a843dc7a76961a8daceaaaa8bad1332f1bf4b54657 \ + --hash=sha256:bea0a62734d20d67608660f69dcda23e7f90fb4ca20974ab80b6ed40df87a005 \ + --hash=sha256:c5192722cffe15f9329a3948c4b1db789fbb1f05c97899187dcf009b283aea70 \ + --hash=sha256:c97176013d404c7346bf57874eaac5187d969293bf40497140b0a2b2b7482e07 \ + --hash=sha256:cd13e354df9938598af2be05822c323e97132d5e6306b83a3b4ee6724c6e522e \ + --hash=sha256:d2ec56337675e61b312179a1ad124f5f570c00f920cc75e1000025451b88241c \ + --hash=sha256:d3837938ae715fc0fe3c39c0202de3a8853aff22ca66781ddc2ade7554b7e2cc \ + --hash=sha256:d9f48cafc7ce94cf9b15c6bffdc443a81a27bf7075cf2dcd5c8b40f85d10c4e7 \ + --hash=sha256:da7763f55885045036fabcebd80144b757d3db06ab0861415d1c3b7c69042146 \ + --hash=sha256:deb3841c925eeddb6afc1e4e4a45e418d19ec7b87c5df177695224078e8ec733 \ + --hash=sha256:e1d27cbcb4602680a49d787d90664fa4974063ac9d4134813332a8c53dbe667c \ + --hash=sha256:e5d42a9472e7579e473879a1990327830493a7047506d58d73fc429b84c1d49d \ + --hash=sha256:e7efa2681ea410b10dde31a52b18b0154d66f2485328830e45fdf183af5aefc6 \ + --hash=sha256:eab43fae33a0c39006a88096cd7b4f4ef545ea0447d250d5ac18202d40b6611d \ + --hash=sha256:f2622206f5559784fa5c4b53a950c3c7c1cf3e84ca1b9c4b6c03f062f289ca26 \ + --hash=sha256:f379b54b77a597aa7ee5e697df0d66903e41b9c85a6dd7946159e356319158e8 \ + --hash=sha256:f667a4542cc8917af1db06366d3f78a5c8e83badd56409f94d1eac8d8d9133fa \ + --hash=sha256:fb4b29f4cf8cc5a8d628bc8d8e26d12d7278cd1f219f22698a378c3d67db5e4b \ + --hash=sha256:ffa6eea95283b2b8079b821dc11f50a17d0571c92b43e2b5b12764dc5f9b285d # via neuronx-cc six==1.17.0 \ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ @@ -334,13 +456,13 @@ tqdm==4.67.1 \ --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 \ --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2 # via neuronx-cc -typing-extensions==4.14.1 \ - --hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 \ - --hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76 +typing-extensions==4.15.0 \ + --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ + --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 # via neuronx-cc -urllib3==2.5.0 \ - --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 \ - --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc +urllib3==2.6.3 \ + --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ + --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 # via # botocore # requests From ab62c536e274bde52bfdd0cefda3080fa767d6c8 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 9 Jan 2026 17:54:00 +0000 Subject: [PATCH 216/881] runtimes/neuron: wip --- pjrt/pjrt.zig | 1 + runtimes/neuron/libneuronxla.zig | 42 ++++++++---------- runtimes/neuron/libpjrt_neuron.BUILD.bazel | 4 +- runtimes/neuron/libpjrt_neuron.zig | 51 ++++++++++++---------- runtimes/neuron/neuron.bzl | 4 ++ runtimes/neuron/neuron.zig | 2 +- runtimes/neuron/neuronx-cc.zig | 14 +++++- stdx/Io.zig | 2 +- 8 files changed, 67 insertions(+), 53 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 5d35689f..d222ece4 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -80,6 +80,7 @@ pub const Api = struct { inner: c.PJRT_Api, pub fn loadFrom(library: [:0]const u8) !*const Api { + log.info("Loading library {s}...", .{library}); var lib: std.DynLib = switch (builtin.os.tag) { .linux => blk: { // We use RTLD_GLOBAL so that symbols from NEEDED libraries are available in the global namespace. diff --git a/runtimes/neuron/libneuronxla.zig b/runtimes/neuron/libneuronxla.zig index f14d389d..57d0dea1 100644 --- a/runtimes/neuron/libneuronxla.zig +++ b/runtimes/neuron/libneuronxla.zig @@ -12,7 +12,7 @@ pub fn makeTempDir(io: std.Io, buf: []u8, prefix: []const u8) ![]const u8 { tmp_dir, std.Io.Dir.path.sep_str_posix, prefix, - std.time.microTimestamp(), + (try std.Io.Clock.now(.real, io)).toNanoseconds(), }); try std.Io.Dir.createDir(.cwd(), io, ret, .fromMode(0o700)); return ret; @@ -23,7 +23,8 @@ const ModuleState = struct { }; var module_def: c.PyModuleDef = .{ - .m_base = .{}, + .m_base = std.mem.zeroes(c.PyModuleDef_Base), + .m_doc = "Zig bindings for libneuronxla", .m_name = "libneuronxla", .m_size = @sizeOf(ModuleState), .m_methods = @constCast(&[_]c.PyMethodDef{ @@ -39,11 +40,11 @@ var module_def: c.PyModuleDef = .{ .ml_flags = c.METH_FASTCALL, .ml_doc = "Return a greeting from Zig.", }, - .{}, + std.mem.zeroes(c.PyMethodDef), }), .m_slots = @constCast(&[_]c.PyModuleDef_Slot{ .{ .slot = c.Py_mod_exec, .value = @ptrCast(@constCast(&module_exec)) }, - .{}, + std.mem.zeroes(c.PyModuleDef_Slot), }), .m_traverse = null, .m_clear = null, @@ -51,7 +52,10 @@ var module_def: c.PyModuleDef = .{ }; fn module_exec(module: ?*c.PyObject) callconv(.c) c_int { - _ = module; + const state: *ModuleState = @ptrCast(@alignCast(c.PyModule_GetState(module))); + state.* = .{ + .threaded = .init(std.heap.c_allocator, .{}), + }; return 0; } @@ -104,7 +108,7 @@ fn wrapNeffAsCustomCall(allocator: std.mem.Allocator, io: std.Io, hlo_code: []co c.xla_HloInstructionProto_set_custom_call_target(fused_root, upb.stringView("AwsNeuronNeff")); c.xla_HloInstructionProto_set_backend_config( fused_root, - try upb.stringView( + upb.stringView( try stdx.Io.Dir.readFileAlloc(.cwd(), io, neff_file_path, allocator, .unlimited), ), ); @@ -179,11 +183,11 @@ fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) }; const code_file = try std.Io.Dir.path.join(arena.allocator(), &.{ tmp_dir, "file.code" }); - { - const file = try std.Io.Dir.createFile(.cwd(), io, code_file, .{ .truncate = true }); - defer file.close(); - try file.writeAll(code); - } + try std.Io.Dir.writeFile(.cwd(), io, .{ + .data = code, + .sub_path = code_file, + .flags = .{ .truncate = true }, + }); const neff_file = try std.Io.Dir.path.join(arena.allocator(), &.{ tmp_dir, "file.neff" }); @@ -199,7 +203,7 @@ fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) "--framework=XLA", "--target", target, - "--verbose=info", + "--verbose=user", "--enable-internal-neff-wrapper", "--output", neff_file, @@ -219,7 +223,7 @@ fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) std.debug.print(">>>> {s}\n", .{tmp_dir}); - const neff_hlo_bytes = wrapNeffAsCustomCall(arena.allocator(), code, neff_file) catch |err| { + const neff_hlo_bytes = wrapNeffAsCustomCall(arena.allocator(), io, code, neff_file) catch |err| { log.err("Error wrapping NEFF as custom call: {}\n", .{err}); return err; }; @@ -248,15 +252,5 @@ fn neuronx_cc(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) } pub export fn PyInit_libneuronxla() callconv(.c) ?*c.PyObject { - const module = c.PyModuleDef_Init(&module_def); - if (module == null) { - return null; - } - - const state: *ModuleState = @ptrCast(@alignCast(c.PyModule_GetState(module))); - state.* = .{ - .threaded = .init(std.heap.c_allocator, .{}), - }; - - return module; + return c.PyModuleDef_Init(&module_def); } diff --git a/runtimes/neuron/libpjrt_neuron.BUILD.bazel b/runtimes/neuron/libpjrt_neuron.BUILD.bazel index 01c0bede..876d9d9b 100644 --- a/runtimes/neuron/libpjrt_neuron.BUILD.bazel +++ b/runtimes/neuron/libpjrt_neuron.BUILD.bazel @@ -41,6 +41,7 @@ copy_to_directory( "@rules_python//python:current_py_toolchain", "@aws-neuronx-runtime-lib//:libnrt.patchelf", "@aws-neuronx-runtime-lib//:libncfw.patchelf", + "@aws-neuronx-runtime-lib//:libnrtucode_extisa", "@aws-neuronx-collectives//:libnccom", "@zlib1g", "@libgomp1", @@ -52,13 +53,14 @@ copy_to_directory( "libpjrt_neuron_so": "lib", "libnrt.patchelf": "lib", "libncfw.patchelf": "lib", + "libnrtucode_extisa": "lib", "lib/x86_64-linux-gnu": "lib", "usr/lib/x86_64-linux-gnu": "lib", "opt/neuron": "lib", }, exclude_srcs_patterns = [ "**/tests/**", - "**/include/**", + # "**/include/**", "**/_solib_*/**", ], include_srcs_patterns = [ diff --git a/runtimes/neuron/libpjrt_neuron.zig b/runtimes/neuron/libpjrt_neuron.zig index a4406e41..e87d1f23 100644 --- a/runtimes/neuron/libpjrt_neuron.zig +++ b/runtimes/neuron/libpjrt_neuron.zig @@ -7,18 +7,14 @@ const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/runtimes/neuron"); -fn findFreeTcpPort() !u16 { - var address = std.net.Address.initIp4(.{ 127, 0, 0, 1 }, 0); - const sockfd = try std.posix.socket( - std.posix.AF.INET, - std.posix.SOCK.STREAM, - std.posix.IPPROTO.TCP, +fn findFreeTcpPort(io: std.Io) !u16 { + var server = try std.Io.net.IpAddress.listen( + .{ .ip4 = .{ .bytes = .{ 127, 0, 0, 1 }, .port = 0 } }, + io, + .{}, ); - defer std.posix.close(sockfd); - var socklen = address.getOsSockLen(); - try std.posix.bind(sockfd, &address.any, socklen); - try std.posix.getsockname(sockfd, &address.any, &socklen); - return address.getPort(); + defer server.deinit(io); + return server.socket.address.getPort(); } pub export fn zmlxneuron_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque { @@ -31,7 +27,7 @@ pub export fn zmlxneuron_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaqu var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const new_filename: [*c]const u8 = if (filename) |f| blk: { const replacement = replacements.get(std.Io.Dir.path.basename(std.mem.span(f))) orelse break :blk f; - break :blk stdx.fs.path.bufJoinZ(&buf, &.{ + break :blk stdx.Io.Dir.path.bufJoinZ(&buf, &.{ stdx.process.selfSharedObjectDirPath(), replacement, }) catch unreachable; @@ -41,11 +37,11 @@ pub export fn zmlxneuron_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaqu } extern fn setenv(name: [*:0]const u8, value: [*:0]const u8, overwrite: c_int) c_int; -fn setupNeuronEnv() !void { +fn setupNeuronEnv(io: std.Io) !void { var buf: [256]u8 = undefined; _ = setenv( "NEURON_RT_ROOT_COMM_ID", - try std.fmt.bufPrintZ(&buf, "127.0.0.1:{d}", .{try findFreeTcpPort()}), + try std.fmt.bufPrintZ(&buf, "127.0.0.1:{d}", .{try findFreeTcpPort(io)}), 1, ); _ = setenv( @@ -81,6 +77,9 @@ fn toPosixPathW(file_path: []const u8) error{NameTooLong}![std.posix.PATH_MAX - return path_with_null; } +// redefine it because for some reason PyThreadState can't be sized +extern fn PyEval_SaveThread() *anyopaque; + fn setupPythonEnv(sandbox_path: []const u8) !void { const Static = struct { var py_config: c.PyConfig = undefined; @@ -126,7 +125,7 @@ fn setupPythonEnv(sandbox_path: []const u8) !void { pyStatusCheck(c.Py_InitializeFromConfig(&Static.py_config)); // release the GIL - _ = c.PyEval_SaveThread(); + _ = PyEval_SaveThread(); } // Duplicates a PJRT Api object while being careful about struct size differences @@ -141,6 +140,10 @@ fn dupePjrtApi(api: *c.PJRT_Api) c.PJRT_Api { } fn getPjrtApi() !*c.PJRT_Api { + var threaded: std.Io.Threaded = .init(std.heap.c_allocator, .{}); + defer threaded.deinit(); + const io = threaded.io(); + const Static = struct { var inner: *c.PJRT_Api = undefined; var proxy: c.PJRT_Api = undefined; @@ -152,7 +155,7 @@ fn getPjrtApi() !*c.PJRT_Api { "..", }); - try setupNeuronEnv(); + try setupNeuronEnv(io); try setupPythonEnv(sandbox_path); Static.inner = blk: { @@ -181,15 +184,15 @@ fn getPjrtApi() !*c.PJRT_Api { Static.proxy = dupePjrtApi(Static.inner); // Setup the API proxy functions - Static.proxy.PJRT_Plugin_Attributes = &struct { - const STRUCT_SIZE = 24; // according to the failing assertion + // Static.proxy.PJRT_Plugin_Attributes = &struct { + // const STRUCT_SIZE = 24; // according to the failing assertion - fn call(args: [*c]c.PJRT_Plugin_Attributes_Args) callconv(.c) ?*c.PJRT_Error { - var new_args = args.*; - new_args.struct_size = @min(new_args.struct_size, STRUCT_SIZE); - return Static.inner.PJRT_Plugin_Attributes.?(&new_args); - } - }.call; + // fn call(args: [*c]c.PJRT_Plugin_Attributes_Args) callconv(.c) ?*c.PJRT_Error { + // var new_args = args.*; + // new_args.struct_size = @min(new_args.struct_size, STRUCT_SIZE); + // return Static.inner.PJRT_Plugin_Attributes.?(&new_args); + // } + // }.call; return &Static.proxy; } diff --git a/runtimes/neuron/neuron.bzl b/runtimes/neuron/neuron.bzl index e03e8ca6..d329c319 100644 --- a/runtimes/neuron/neuron.bzl +++ b/runtimes/neuron/neuron.bzl @@ -46,6 +46,10 @@ _NEURON_PACKAGES = { src = "lib/libncfw.so", soname = "libncfw.so.2", ), + packages.filegroup( + name = "libnrtucode_extisa", + srcs = ["lib/libnrtucode_extisa.so"], + ), ]), "aws-neuronx-collectives": "\n".join([ packages.filegroup( diff --git a/runtimes/neuron/neuron.zig b/runtimes/neuron/neuron.zig index 9d57e629..5ea729c3 100644 --- a/runtimes/neuron/neuron.zig +++ b/runtimes/neuron/neuron.zig @@ -27,7 +27,7 @@ fn isRunningOnEC2(io: std.Io) !bool { try std.Io.Dir.readFile( .cwd(), io, - "/sys/devices/virtual/dmi/id/product_name", + "/sys/devices/virtual/dmi/id/sys_vendor", &buffer, ), ); diff --git a/runtimes/neuron/neuronx-cc.zig b/runtimes/neuron/neuronx-cc.zig index 3f2a8fe8..e77c4858 100644 --- a/runtimes/neuron/neuronx-cc.zig +++ b/runtimes/neuron/neuronx-cc.zig @@ -22,7 +22,14 @@ pub fn toPosixPathW(file_path: []const u8) error{NameTooLong}![std.posix.PATH_MA } pub fn main() !void { - var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); + const allocator = std.heap.c_allocator; + + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + + const io = threaded.io(); + + var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); { @@ -43,7 +50,10 @@ pub fn main() !void { _ = c.PyConfig_SetBytesArgv(&config, @intCast(std.os.argv.len), @ptrCast(std.os.argv)); var self_exe_dir_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const self_exe_dir = try std.process.executableDirPath(io, &self_exe_dir_buf); + const self_exe_dir = blk: { + const n = try std.process.executableDirPath(io, &self_exe_dir_buf); + break :blk self_exe_dir_buf[0..n]; + }; { var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; diff --git a/stdx/Io.zig b/stdx/Io.zig index 83db497d..52319906 100644 --- a/stdx/Io.zig +++ b/stdx/Io.zig @@ -15,7 +15,7 @@ pub const Dir = struct { pub fn readFileAlloc(dir: std.Io.Dir, io: std.Io, sub_path: []const u8, gpa: std.mem.Allocator, limit: std.Io.Limit) ![]u8 { const stat = try std.Io.Dir.statFile(dir, io, sub_path, .{}); - const buffer = try gpa.alloc(u8, limit.min(.limited64(stat.size))); + const buffer = try gpa.alloc(u8, limit.minInt64(stat.size)); errdefer gpa.free(buffer); _ = try std.Io.Dir.readFile(.cwd(), io, sub_path, buffer); return buffer; From 75903732c12c22101ef819cb19c3c4eb5d08284c Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sat, 10 Jan 2026 18:50:18 +0000 Subject: [PATCH 217/881] pjrt: do not deinit Event after await PJRT_Buffer_ReadyEvent semantics say it allocates a new object everytime, of which the caller is the owner. --- pjrt/pjrt.zig | 2 -- zml/buffer.zig | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index d222ece4..2bf88b61 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -1081,8 +1081,6 @@ pub const Event = opaque { } pub fn await(self: *Event, api: *const Api, io: std.Io) ApiError!void { - defer self.deinit(api); - if (self.isReady(api)) { return; } diff --git a/zml/buffer.zig b/zml/buffer.zig index d5409a04..4f2a2920 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -136,7 +136,9 @@ pub const Buffer = struct { pub fn await(self: Buffer, io: std.Io) !Buffer { for (self._shards.constSlice()) |buffer| { - try buffer.getReadyEvent(self._api).await(self._api, io); + const ev = buffer.getReadyEvent(self._api); + defer ev.deinit(self._api); + try ev.await(self._api, io); } return self; From 132c7cf390e4cd352c57e6b6a5fd267600b89032 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sun, 11 Jan 2026 09:48:15 +0100 Subject: [PATCH 218/881] pjrt:WIP --- pjrt/pjrt.zig | 139 +++++++++++++++++++++++++++++--------------------- 1 file changed, 80 insertions(+), 59 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 2bf88b61..c7c36d18 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -12,35 +12,56 @@ test { std.testing.refAllDecls(@This()); } -// We could calculate it like PJRT does, but it turns out that some of those -// were wrong in PJRT itself [1], which gets propagated to binary plugins. In -// order to mirror that, we just the value as computed by PJRT itself, through -// comptime reflection. We could make the argument to remove that one day since -// [1] has been fixed. The problem is that this problem could happen again in -// as the way PJRT does it is not very robust. -// -// 1. https://github.com/openxla/xla/issues/10032 -pub fn pjrtStructSize(comptime T: type) usize { - // unsafe on purpose, we want this to fail if that ever changes - const typedef_name = comptime blk: { - const needle = ".struct_"; - const idx = std.mem.indexOf(u8, @typeName(T), needle).?; - break :blk @typeName(T)[idx + needle.len ..]; - }; - return @field(c, typedef_name ++ "_STRUCT_SIZE"); -} +const meta = struct { + pub fn isPjrtStruct(comptime T: type) bool { + if (std.mem.indexOf(u8, @typeName(T), ".struct_PJRT_") == null) { + return false; + } + return true; + } -pub inline fn pjrtStruct(v: anytype) @TypeOf(v) { - var ret = v; - ret.struct_size = pjrtStructSize(@TypeOf(v)); - return ret; -} + // We could calculate it like PJRT does, but it turns out that some of those + // were wrong in PJRT itself [1], which gets propagated to binary plugins. In + // order to mirror that, we just the value as computed by PJRT itself, through + // comptime reflection. We could make the argument to remove that one day since + // [1] has been fixed. The problem is that this problem could happen again in + // as the way PJRT does it is not very robust. + // + // 1. https://github.com/openxla/xla/issues/10032 + pub fn structSize(comptime T: type) usize { + // unsafe on purpose, we want this to fail if that ever changes + const typedef_name = comptime blk: { + const needle = ".struct_"; + const idx = std.mem.indexOf(u8, @typeName(T), needle).?; + break :blk @typeName(T)[idx + needle.len ..]; + }; + return @field(c, typedef_name ++ "_STRUCT_SIZE"); + } -pub fn pjrtStruct2(T: type, v: anytype) T { - var ret = std.mem.zeroInit(T, v); - ret.struct_size = pjrtStructSize(T); - return ret; -} + pub fn Struct(comptime T: type) type { + const fields = std.meta.fields(T); + var names: [fields.len][]const u8 = undefined; + var types: [fields.len]type = undefined; + var attributes: [fields.len]std.builtin.Type.StructField.Attributes = undefined; + for (fields, &names, &types, &attributes) |field, *name, *type_, *attr| { + name.* = field.name; + type_.* = field.type; + attr.* = .{ + .default_value_ptr = @ptrCast(if (std.mem.eql(u8, field.name, "struct_size")) + &structSize(T) + else + &std.mem.zeroes(field.type)), + }; + } + return @Struct( + .@"extern", + null, + &names, + &types, + &attributes, + ); + } +}; pub const ApiError = error{ Cancelled, @@ -101,12 +122,12 @@ pub const Api = struct { const api = DynGetPjrtApi(); log.info("Loaded library: {s}", .{library}); - _ = api.call(.PJRT_Plugin_Initialize, pjrtStruct2(c.PJRT_Plugin_Initialize_Args, .{})) catch unreachable; + _ = api.call(.PJRT_Plugin_Initialize, .{}) catch unreachable; return api; } - fn CallFnArgType(comptime func: Funcs) type { + fn PJRTFnArg(comptime func: Funcs) type { const fti = @typeInfo(@FieldType(c.PJRT_Api, @tagName(func))); const fn_ptr = @typeInfo(fti.optional.child); const fn_type_info = @typeInfo(fn_ptr.pointer.child); @@ -114,20 +135,30 @@ pub const Api = struct { return arg_array_type_info.pointer.child; } - inline fn call(self: *const Api, comptime method: Funcs, arg: anytype) ApiError!CallFnArgType(method) { - var ret = pjrtStruct2(CallFnArgType(method), arg); - const fn_ptr = @field(&self.inner, @tagName(method)).?; - const result = fn_ptr(&ret); - if (@TypeOf(result) == void) { - return ret; - } + fn PJRTFnArgWithDefault(comptime func: Funcs) type { + const argT = PJRTFnArg(func); + return switch (@typeInfo(argT)) { + .@"struct" => meta.Struct(argT), + else => argT, + }; + } + inline fn callInner(self: *const Api, comptime method: Funcs, arg: *PJRTFnArg(method)) ApiError!void { + const fn_ptr = @field(&self.inner, @tagName(method)).?; + const result = fn_ptr(arg); + if (@TypeOf(result) == void) { + return; + } if (result) |pjrt_c_error| { const pjrt_error: *Error = @ptrCast(pjrt_c_error); log.err("[{s}] {s}", .{ @tagName(method), pjrt_error.getMessage(self) }); return pjrt_error.getCode(self).toApiError(); } + } + inline fn call(self: *const Api, comptime method: Funcs, arg: PJRTFnArgWithDefault(method)) ApiError!PJRTFnArgWithDefault(method) { + var ret = arg; + try callInner(self, method, @ptrCast(&ret)); return ret; } @@ -269,15 +300,15 @@ pub const ShapeSpec = extern struct { std.debug.assert(@sizeOf(ShapeSpec) == @sizeOf(c.PJRT_ShapeSpec)); } - inner: c.PJRT_ShapeSpec, + inner: meta.Struct(c.PJRT_ShapeSpec), pub fn init(dims_: []const i64, bt: BufferType) ShapeSpec { return .{ - .inner = pjrtStruct2(c.PJRT_ShapeSpec, .{ + .inner = .{ .dims = @as([*c]const i64, @ptrCast(@constCast(dims_.ptr))), .num_dims = dims_.len, .element_type = @intFromEnum(bt), - }), + }, }; } @@ -303,12 +334,8 @@ pub const Client = opaque { pub fn init(api: *const Api, create_options: []const NamedValue) ClientInitError!*Client { // log.info("Loaded PJRT runtime plugin: {s}", .{api.Platform}); const ret = try api.call(.PJRT_Client_Create, .{ - .create_options = @as([*c]const c.PJRT_NamedValue, @ptrCast(create_options.ptr)), + .create_options = @ptrCast(create_options.ptr), .num_options = create_options.len, - .kv_get_callback = null, - .kv_put_callback = null, - .kv_put_user_arg = null, - .kv_get_user_arg = null, }); return @ptrCast(ret.client.?); } @@ -349,13 +376,13 @@ pub const Client = opaque { pub fn compileRaw(self: *const Client, api: *const Api, args: CompileArgs) ApiError!*LoadedExecutable { const bytecode_format_ = @tagName(args.bytecode_format); const ret = try api.call(.PJRT_Client_Compile, .{ - .program = &pjrtStruct2(c.PJRT_Program, .{ - .code = @as([*c]u8, @ptrCast(@constCast(args.bytecode.ptr))), + .program = @ptrCast(&meta.Struct(c.PJRT_Program){ + .code = @ptrCast(@constCast(args.bytecode)), .code_size = args.bytecode.len, - .format = @as([*c]const u8, @ptrCast(@constCast(bytecode_format_.ptr))), + .format = @ptrCast(@constCast(bytecode_format_)), .format_size = bytecode_format_.len, }), - .compile_options = @as([*c]const u8, @ptrCast(@constCast(args.compile_options_pb.ptr))), + .compile_options = @ptrCast(@constCast(args.compile_options_pb)), .compile_options_size = args.compile_options_pb.len, .client = self.inner(), }); @@ -795,19 +822,13 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ApiError!void { - var options = pjrtStruct2(c.PJRT_ExecuteOptions, .{ - .send_callbacks = null, - .recv_callbacks = null, - .num_send_ops = 0, - .num_recv_ops = 0, - .launch_id = 0, - .non_donatable_input_indices = @as([*c]const i64, @ptrCast(args.non_donatable_input_indices.ptr)), - .num_non_donatable_input_indices = args.non_donatable_input_indices.len, - .context = @as(?*c.PJRT_ExecuteContext, @ptrCast(args.context)), - }); _ = try api.call(.PJRT_LoadedExecutable_Execute, .{ .executable = self.inner(), - .options = @as(*c.PJRT_ExecuteOptions, @ptrCast(&options)), + .options = &.{ + .non_donatable_input_indices = @ptrCast(args.non_donatable_input_indices), + .num_non_donatable_input_indices = args.non_donatable_input_indices.len, + .context = @ptrCast(args.context), + }, .argument_lists = @as([*c]const [*c]const ?*c.PJRT_Buffer, @ptrCast(args.arguments.ptr)), .num_devices = @as(usize, @intCast(args.arguments.len)), .num_args = args.num_args, From 55488f676dda4f9cdba22a23d4030f39d51aa4d8 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sun, 11 Jan 2026 09:56:47 +0100 Subject: [PATCH 219/881] PJRT: meta --- pjrt/pjrt.zig | 94 +++++++++++++++++++++++++-------------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index c7c36d18..83ddf3aa 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -274,20 +274,20 @@ pub const ErrorCode = enum(c.PJRT_Error_Code) { pub const Error = opaque { pub fn deinit(self: *Error, api: *const Api) void { _ = api.call(.PJRT_Error_Destroy, .{ - .@"error" = @as(*c.PJRT_Error, @ptrCast(self)), + .@"error" = @ptrCast(self), }) catch unreachable; } pub fn getCode(self: *Error, api: *const Api) ErrorCode { const ret = api.call(.PJRT_Error_GetCode, .{ - .@"error" = @as(*c.PJRT_Error, @ptrCast(self)), + .@"error" = @ptrCast(self), }) catch unreachable; return @enumFromInt(ret.code); } pub fn getMessage(self: *Error, api: *const Api) []const u8 { const ret = api.call(.PJRT_Error_Message, .{ - .@"error" = @as(*c.PJRT_Error, @ptrCast(self)), + .@"error" = @ptrCast(self), }) catch unreachable; return ret.message[0..ret.message_size]; } @@ -305,7 +305,7 @@ pub const ShapeSpec = extern struct { pub fn init(dims_: []const i64, bt: BufferType) ShapeSpec { return .{ .inner = .{ - .dims = @as([*c]const i64, @ptrCast(@constCast(dims_.ptr))), + .dims = @ptrCast(@constCast(dims_.ptr)), .num_dims = dims_.len, .element_type = @intFromEnum(bt), }, @@ -411,13 +411,13 @@ pub const Client = opaque { .client = self.inner(), .data = @constCast(args.data), .type = @intFromEnum(args.buffer_type), - .dims = @as([*c]const i64, @ptrCast(@constCast(args.dims.ptr))), + .dims = @ptrCast(@constCast(args.dims.ptr)), .num_dims = args.dims.len, - .byte_strides = @as([*c]const i64, if (args.byte_strides) |bs| @ptrCast(@constCast(bs.ptr)) else null), + .byte_strides = if (args.byte_strides) |bs| @ptrCast(@constCast(bs.ptr)) else null, .num_byte_strides = if (args.byte_strides) |bs| bs.len else 0, .host_buffer_semantics = @intFromEnum(args.host_buffer_semantics), - .device = @as(?*c.PJRT_Device, if (args.dst == .device) @ptrCast(@constCast(args.dst.device)) else null), - .memory = @as(?*c.PJRT_Memory, if (args.dst == .memory) @ptrCast(@constCast(args.dst.memory)) else null), + .device = if (args.dst == .device) @ptrCast(@constCast(args.dst.device)) else null, + .memory = if (args.dst == .memory) @ptrCast(@constCast(args.dst.memory)) else null, .device_layout = null, // TODO .done_with_host_buffer = null, // out .buffer = null, // out @@ -459,11 +459,11 @@ pub const Client = opaque { .dims = args.dims.ptr, .num_dims = args.dims.len, .element_type = @intFromEnum(args.element_type), - .layout = @as([*c]c.PJRT_Buffer_MemoryLayout, @ptrCast(@constCast(&layout))), - .device = @as(?*c.PJRT_Device, @ptrCast(@constCast(args.device))), + .layout = @ptrCast(@constCast(&layout)), + .device = @ptrCast(@constCast(args.device)), .on_delete_callback = args.on_delete_callback, .on_delete_callback_arg = args.on_delete_callback_arg, - .stream = @as(isize, @bitCast(@intFromPtr(args.stream))), + .stream = @bitCast(@intFromPtr(args.stream)), }); return @ptrCast(ret.buffer.?); } @@ -502,11 +502,11 @@ pub const Client = opaque { pub fn createBuffersForAsyncHostToDevice(self: *const Client, api: *const Api, args: CreateBuffersForAsyncHostToDeviceArgs) ApiError!*AsyncHostToDeviceTransferManager { const ret = try api.call(.PJRT_Client_CreateBuffersForAsyncHostToDevice, .{ .client = self.inner(), - .shape_specs = @as([*c]c.PJRT_ShapeSpec, @ptrCast(@constCast(args.shape_specs.ptr))), + .shape_specs = @ptrCast(@constCast(args.shape_specs.ptr)), .num_shape_specs = args.shape_specs.len, - .device_layouts = @as([*c][*c]c.PJRT_Buffer_MemoryLayout, if (args.device_layouts) |layouts| @ptrCast(@constCast(layouts.ptr)) else null), - .num_device_layouts = @as(usize, if (args.device_layouts) |layouts| @intCast(layouts.len) else 0), - .memory = @as(?*c.PJRT_Memory, @ptrCast(@constCast(args.memory))), + .device_layouts = if (args.device_layouts) |layouts| @ptrCast(@constCast(layouts.ptr)) else null, + .num_device_layouts = if (args.device_layouts) |layouts| layouts.len else 0, + .memory = @ptrCast(@constCast(args.memory)), }); return @ptrCast(ret.transfer_manager.?); } @@ -526,11 +526,11 @@ pub const Client = opaque { const ret = try api.call(.PJRT_Client_CreateUninitializedBuffer, .{ .client = self.inner(), .shape_dims = args.dims.ptr, - .shape_num_dims = @as(usize, @intCast(args.dims.len)), + .shape_num_dims = args.dims.len, .shape_element_type = @intFromEnum(args.element_type), - .shape_layout = @as([*c]c.PJRT_Buffer_MemoryLayout, @ptrCast(&layout)), - .device = if (args.dst == .device) @as(?*c.PJRT_Device, @ptrCast(@constCast(args.dst.device))) else null, - .memory = if (args.dst == .memory) @as(?*c.PJRT_Memory, @ptrCast(@constCast(args.dst.memory))) else null, + .shape_layout = @ptrCast(&layout), + .device = if (args.dst == .device) @ptrCast(@constCast(args.dst.device)) else null, + .memory = if (args.dst == .memory) @ptrCast(@constCast(args.dst.memory)) else null, }); return @ptrCast(ret.buffer.?); } @@ -711,7 +711,7 @@ pub const SerializeResult = struct { pub const ExecuteContext = opaque { pub fn deinit(self: *ExecuteContext, api: *const Api) void { _ = api.call(.PJRT_ExecuteContext_Destroy, .{ - .context = @as(?*c.PJRT_ExecuteContext, @ptrCast(self)), + .context = @ptrCast(self), }) catch {}; } }; @@ -822,19 +822,19 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ApiError!void { + var options: meta.Struct(c.PJRT_ExecuteOptions) = .{ + .non_donatable_input_indices = @ptrCast(args.non_donatable_input_indices), + .num_non_donatable_input_indices = args.non_donatable_input_indices.len, + .context = @ptrCast(args.context), + }; _ = try api.call(.PJRT_LoadedExecutable_Execute, .{ .executable = self.inner(), - .options = &.{ - .non_donatable_input_indices = @ptrCast(args.non_donatable_input_indices), - .num_non_donatable_input_indices = args.non_donatable_input_indices.len, - .context = @ptrCast(args.context), - }, - .argument_lists = @as([*c]const [*c]const ?*c.PJRT_Buffer, @ptrCast(args.arguments.ptr)), - .num_devices = @as(usize, @intCast(args.arguments.len)), + .options = @ptrCast(&options), + .argument_lists = @ptrCast(args.arguments.ptr), + .num_devices = args.arguments.len, .num_args = args.num_args, - .output_lists = @as([*c]const [*c]?*c.PJRT_Buffer, @ptrCast(args.results.ptr)), - .device_complete_events = @as([*c]?*c.PJRT_Event, if (args.events) |ev| @ptrCast(ev.ptr) else null), - .execute_device = null, + .output_lists = @ptrCast(args.results.ptr), + .device_complete_events = if (args.events) |ev| @ptrCast(ev.ptr) else null, }); } @@ -1052,7 +1052,7 @@ pub const Buffer = opaque { pub fn copyToMemory(self: *const Buffer, api: *const Api, dst_memory: *const Memory) ApiError!*Buffer { const ret = try api.call(.PJRT_Buffer_CopyToMemory, .{ .buffer = self.inner(), - .dst_memory = @as(?*c.PJRT_Memory, @ptrCast(@constCast(dst_memory))), + .dst_memory = @ptrCast(@constCast(dst_memory)), }); return @ptrCast(ret.dst_buffer); } @@ -1094,10 +1094,10 @@ pub const Event = opaque { } pub fn getEventError(self: *const Event, api: *const Api) ?*Error { - var args = pjrtStruct2(c.PJRT_Event_Error_Args, .{ + var args: meta.Struct(c.PJRT_Event_Error_Args) = .{ .event = self.inner(), - }); - const result: ?*c.PJRT_Error = api.inner.PJRT_Event_Error.?(&args); + }; + const result: ?*c.PJRT_Error = api.inner.PJRT_Event_Error.?(@ptrCast(&args)); return @ptrCast(result); } @@ -1138,7 +1138,7 @@ pub const Event = opaque { pub fn onReady(self: *Event, api: *const Api, func: *const fn (err: ?*Error, user_arg: ?*anyopaque) callconv(.c) void, user_arg: ?*anyopaque) ApiError!void { _ = try api.call(.PJRT_Event_OnReady, .{ .event = self.inner(), - .callback = @as(c.PJRT_Event_OnReadyCallback, @ptrCast(func)), + .callback = @ptrCast(func), .user_arg = user_arg, }); } @@ -1226,10 +1226,10 @@ pub const AsyncHostToDeviceTransferManager = opaque { pub fn transferData(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize, data: []const u8, offset: i64, is_last_transfer: bool) ApiError!*Event { const ret = try api.call(.PJRT_AsyncHostToDeviceTransferManager_TransferData, .{ .transfer_manager = self.inner(), - .buffer_index = @as(c_int, @intCast(buffer_index)), + .buffer_index = @intCast(buffer_index), .data = data.ptr, .offset = offset, - .transfer_size = @as(i64, @intCast(data.len)), + .transfer_size = @intCast(data.len), .is_last_transfer = is_last_transfer, }); return @ptrCast(ret.done_with_h2d_transfer.?); @@ -1238,7 +1238,7 @@ pub const AsyncHostToDeviceTransferManager = opaque { pub fn retrieveBuffer(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize) ApiError!*Buffer { const ret = try api.call(.PJRT_AsyncHostToDeviceTransferManager_RetrieveBuffer, .{ .transfer_manager = self.inner(), - .buffer_index = @as(c_int, @intCast(buffer_index)), + .buffer_index = @intCast(buffer_index), }); return @ptrCast(ret.buffer_out.?); } @@ -1260,7 +1260,7 @@ pub const AsyncHostToDeviceTransferManager = opaque { pub fn bufferSize(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize) ApiError!usize { const ret = try api.call(.PJRT_AsyncHostToDeviceTransferManager_BufferSize, .{ .transfer_manager = self.inner(), - .buffer_index = @as(c_int, @intCast(buffer_index)), + .buffer_index = @intCast(buffer_index), }); return ret.buffer_size; } @@ -1268,7 +1268,7 @@ pub const AsyncHostToDeviceTransferManager = opaque { pub fn setBufferError(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize, error_code: c.PJRT_Error_Code, error_message: []const u8) ApiError!void { _ = try api.call(.PJRT_AsyncHostToDeviceTransferManager_SetBufferError, .{ .transfer_manager = self.inner(), - .buffer_index = @as(c_int, @intCast(buffer_index)), + .buffer_index = @intCast(buffer_index), .error_code = error_code, .error_message = error_message.ptr, .error_message_size = error_message.len, @@ -1278,7 +1278,7 @@ pub const AsyncHostToDeviceTransferManager = opaque { pub fn addMetadata(self: *AsyncHostToDeviceTransferManager, api: *const Api, transfer_metadata: []const NamedValue) ApiError!void { _ = try api.call(.PJRT_AsyncHostToDeviceTransferManager_AddMetadata, .{ .transfer_manager = self.inner(), - .transfer_metadata = @as([*c]const c.PJRT_NamedValue, @ptrCast(transfer_metadata.ptr)), + .transfer_metadata = @ptrCast(transfer_metadata.ptr), .num_metadata = transfer_metadata.len, }); } @@ -1400,14 +1400,14 @@ pub const Ffi = extern struct { func: *const ffi.Handler, traits: ffi.HandlerTraits, ) ApiError!void { - var ret = pjrtStruct2(c.PJRT_FFI_Register_Handler_Args, .{ + var ret: meta.Struct(c.PJRT_FFI_Register_Handler_Args) = .{ .target_name = target_name.ptr, .target_name_size = target_name.len, .handler = @ptrCast(@constCast(func)), .platform_name = platform_name.ptr, .platform_name_size = platform_name.len, .traits = @bitCast(traits), - }); + }; const result = self.inner.register_handler.?(&ret); if (result) |pjrt_c_error| { const pjrt_error: *Error = @ptrCast(pjrt_c_error); @@ -1417,12 +1417,12 @@ pub const Ffi = extern struct { } pub fn registerTypeId(self: *const Ffi, api: *const Api, type_name: []const u8, type_info: ?*const c.PJRT_FFI_Type_Info) ApiError!ffi.TypeId { - var ret = pjrtStruct2(c.PJRT_FFI_Type_Register_Args, .{ + var ret: meta.Struct(c.PJRT_FFI_Type_Register_Args) = .{ .type_name = type_name.ptr, .type_name_size = type_name.len, .type_id = 0, // let the plugin assign a unique type ID .type_info = @ptrCast(@constCast(type_info)), - }); + }; const result = self.inner.type_register.?(&ret); if (result) |pjrt_c_error| { const pjrt_error: *Error = @ptrCast(pjrt_c_error); @@ -1433,10 +1433,10 @@ pub const Ffi = extern struct { } pub fn addUserData(self: *const Ffi, api: *const Api, context: *ExecuteContext, user_data: UserData) ApiError!void { - var ret = pjrtStruct2(c.PJRT_FFI_UserData_Add_Args, .{ + var ret: meta.Struct(c.PJRT_FFI_UserData_Add_Args) = .{ .context = @ptrCast(context), .user_data = user_data.toCStruct(), - }); + }; const result = self.inner.user_data_add.?(&ret); if (result) |pjrt_c_error| { const pjrt_error: *Error = @ptrCast(pjrt_c_error); From 9eae5baaf4f141d9e376ac70530025b6e672e906 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sun, 11 Jan 2026 10:17:39 +0100 Subject: [PATCH 220/881] pjrt: meta --- pjrt/pjrt.zig | 177 +++++++++++++++++++++++--------------------------- 1 file changed, 80 insertions(+), 97 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 83ddf3aa..fc3e22fc 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -13,13 +13,6 @@ test { } const meta = struct { - pub fn isPjrtStruct(comptime T: type) bool { - if (std.mem.indexOf(u8, @typeName(T), ".struct_PJRT_") == null) { - return false; - } - return true; - } - // We could calculate it like PJRT does, but it turns out that some of those // were wrong in PJRT itself [1], which gets propagated to binary plugins. In // order to mirror that, we just the value as computed by PJRT itself, through @@ -162,14 +155,26 @@ pub const Api = struct { return ret; } - pub fn lookupExtension(self: *const Api, comptime ExtensionT: type, ext_id: c_int) ?*const ExtensionT { - var cur: [*c]const c.PJRT_Extension_Base = @ptrCast(@alignCast(self.inner.extension_start)); - while (cur != null) : (cur = cur.*.next) { - if (cur.*.type == ext_id) { - return @ptrCast(@alignCast(cur)); + pub const ExtensionsIterator = struct { + current: ?*const c.PJRT_Extension_Base, + + pub fn next(self: *ExtensionsIterator) ?*const c.PJRT_Extension_Base { + self.current = self.current.next; + return self.current; + } + }; + + pub fn extensions(self: *const Api) ExtensionsIterator { + return .{ .current = @ptrCast(@alignCast(self.inner.extension_start)) }; + } + + pub fn extension(self: *const Api, comptime ExtensionT: type, ext_id: c_int) ?*const ExtensionT { + var it = extensions(self); + while (it.next()) |ext| { + if (ext.type == ext_id) { + return @ptrCast(@alignCast(ext)); } } - return null; } @@ -188,11 +193,13 @@ pub const Api = struct { if (state.str) |str| { return str; } - if (self.getPluginAttribute("stablehlo_current_version")) |v| { - stdx.debug.assert(v.kind() == .int64list, "fetched attribute \"stablehlo_current_version\" from the plugin with type `{}`, expected `.int64list`", .{v.kind()}); - stdx.debug.assert(v.inner.value_size == 3, "expect version format to have 3 elements representing `major.minor.patch` format, got {} elements", .{v.inner.value_size}); - const value = v.inner.unnamed_0.int64_array_value[0..v.inner.value_size]; - state.str = std.fmt.bufPrintZ(&state.buf, "{d}.{d}.{d}", .{ value[0], value[1], value[2] }) catch unreachable; + if (self.getPluginAttribute("stablehlo_current_version")) |nv| { + switch (nv.value()) { + .int64list => |v| { + state.str = std.fmt.bufPrintZ(&state.buf, "{d}.{d}.{d}", .{ v[0], v[1], v[2] }) catch unreachable; + }, + else => unreachable, + } } return state.str; } @@ -203,7 +210,7 @@ pub const Api = struct { } pub fn ffi(api: *const Api) ?Ffi { - if (api.lookupExtension(c.PJRT_FFI_Extension, c.PJRT_Extension_Type_FFI)) |ext| { + if (api.extension(c.PJRT_FFI_Extension, c.PJRT_Extension_Type_FFI)) |ext| { return .{ .inner = ext }; } return null; @@ -216,7 +223,6 @@ pub const Api = struct { return attr; } } - return null; } @@ -305,7 +311,7 @@ pub const ShapeSpec = extern struct { pub fn init(dims_: []const i64, bt: BufferType) ShapeSpec { return .{ .inner = .{ - .dims = @ptrCast(@constCast(dims_.ptr)), + .dims = @ptrCast(@constCast(dims_)), .num_dims = dims_.len, .element_type = @intFromEnum(bt), }, @@ -334,7 +340,7 @@ pub const Client = opaque { pub fn init(api: *const Api, create_options: []const NamedValue) ClientInitError!*Client { // log.info("Loaded PJRT runtime plugin: {s}", .{api.Platform}); const ret = try api.call(.PJRT_Client_Create, .{ - .create_options = @ptrCast(create_options.ptr), + .create_options = @ptrCast(create_options), .num_options = create_options.len, }); return @ptrCast(ret.client.?); @@ -411,16 +417,13 @@ pub const Client = opaque { .client = self.inner(), .data = @constCast(args.data), .type = @intFromEnum(args.buffer_type), - .dims = @ptrCast(@constCast(args.dims.ptr)), + .dims = @ptrCast(@constCast(args.dims)), .num_dims = args.dims.len, - .byte_strides = if (args.byte_strides) |bs| @ptrCast(@constCast(bs.ptr)) else null, + .byte_strides = if (args.byte_strides) |bs| @ptrCast(@constCast(bs)) else null, .num_byte_strides = if (args.byte_strides) |bs| bs.len else 0, .host_buffer_semantics = @intFromEnum(args.host_buffer_semantics), .device = if (args.dst == .device) @ptrCast(@constCast(args.dst.device)) else null, .memory = if (args.dst == .memory) @ptrCast(@constCast(args.dst.memory)) else null, - .device_layout = null, // TODO - .done_with_host_buffer = null, // out - .buffer = null, // out }); return .{ @@ -481,7 +484,7 @@ pub const Client = opaque { pub fn dmaMap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { try api.call(.PJRT_Client_DmaMap, .{ .client = self.inner(), - .data = @ptrCast(@constCast(data.ptr)), + .data = @ptrCast(@constCast(data)), .size = @intCast(data.len), }); } @@ -489,7 +492,7 @@ pub const Client = opaque { pub fn dmaUnmap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { try api.call(.PJRT_Client_DmaUnmap, .{ .client = self.inner(), - .data = @ptrCast(@constCast(data.ptr)), + .data = @ptrCast(@constCast(data)), }); } @@ -502,9 +505,9 @@ pub const Client = opaque { pub fn createBuffersForAsyncHostToDevice(self: *const Client, api: *const Api, args: CreateBuffersForAsyncHostToDeviceArgs) ApiError!*AsyncHostToDeviceTransferManager { const ret = try api.call(.PJRT_Client_CreateBuffersForAsyncHostToDevice, .{ .client = self.inner(), - .shape_specs = @ptrCast(@constCast(args.shape_specs.ptr)), + .shape_specs = @ptrCast(@constCast(args.shape_specs)), .num_shape_specs = args.shape_specs.len, - .device_layouts = if (args.device_layouts) |layouts| @ptrCast(@constCast(layouts.ptr)) else null, + .device_layouts = if (args.device_layouts) |layouts| @ptrCast(@constCast(layouts)) else null, .num_device_layouts = if (args.device_layouts) |layouts| layouts.len else 0, .memory = @ptrCast(@constCast(args.memory)), }); @@ -537,44 +540,46 @@ pub const Client = opaque { }; pub const MemoryStats = struct { - // Number of bytes in use. - bytes_in_use: u64, // out - - // The peak bytes in use. - peak_bytes_in_use: u64, // out - peak_bytes_in_use_is_set: bool, // out - // Number of allocations. - num_allocs: u64, // out - num_allocs_is_set: bool, // out - // The largest single allocation seen. - largest_alloc_size: u64, // out - largest_alloc_size_is_set: bool, // out - // The upper limit of user-allocatable device memory in bytes. - bytes_limit: u64, // out - bytes_limit_is_set: bool, // out - - // Number of bytes reserved. - bytes_reserved: u64, // out - bytes_reserved_is_set: bool, // out - // The peak number of bytes reserved. - peak_bytes_reserved: u64, // out - peak_bytes_reserved_is_set: bool, // out - // The upper limit on the number bytes of reservable memory. - bytes_reservable_limit: u64, // out - bytes_reservable_limit_is_set: bool, // out - - // Largest free block size in bytes. - largest_free_block_bytes: u64, // out - largest_free_block_bytes_is_set: bool, // out - - // Number of bytes of memory held by the allocator. This may be higher than - // bytes_in_use if the allocator holds a pool of memory (e.g. BFCAllocator). - pool_bytes: u64, // out - pool_bytes_is_set: bool, // out - peak_pool_bytes: u64, // out - peak_pool_bytes_is_set: bool, // out + /// Number of bytes in use. + bytes_in_use: u64, + /// The peak bytes in use. + peak_bytes_in_use: ?u64, + /// Number of allocations. + num_allocs: ?u64, + /// The largest single allocation seen. + largest_alloc_size: ?u64, + /// The upper limit of user-allocatable device memory in bytes. + bytes_limit: ?u64, + /// Number of bytes reserved. + bytes_reserved: ?u64, + /// The peak number of bytes reserved. + peak_bytes_reserved: ?u64, + /// The upper limit on the number bytes of reservable memory. + bytes_reservable_limit: ?u64, + /// Largest free block size in bytes. + largest_free_block_bytes: ?u64, + /// Number of bytes of memory held by the allocator. This may be higher than + /// bytes_in_use if the allocator holds a pool of memory (e.g. BFCAllocator). + pool_bytes: ?u64, + peak_pool_bytes: ?u64, pub const zeroes = std.mem.zeroes(MemoryStats); + + fn fromCStruct(v: c.PJRT_Device_MemoryStats_Args) MemoryStats { + return .{ + .bytes_in_use = @intCast(v.bytes_in_use), + .peak_bytes_in_use = if (v.peak_bytes_in_use_is_set) @intCast(v.peak_bytes_in_use) else null, + .num_allocs = if (v.num_allocs_is_set) @intCast(v.num_allocs) else null, + .largest_alloc_size = if (v.largest_alloc_size_is_set) @intCast(v.largest_alloc_size) else null, + .bytes_limit = if (v.bytes_limit_is_set) @intCast(v.bytes_limit) else null, + .bytes_reserved = if (v.bytes_reserved_is_set) @intCast(v.bytes_reserved) else null, + .peak_bytes_reserved = if (v.peak_bytes_reserved_is_set) @intCast(v.peak_bytes_reserved) else null, + .bytes_reservable_limit = if (v.bytes_reservable_limit_is_set) @intCast(v.bytes_reservable_limit) else null, + .largest_free_block_bytes = if (v.largest_free_block_bytes_is_set) @intCast(v.largest_free_block_bytes) else null, + .pool_bytes = if (v.pool_bytes_is_set) @intCast(v.pool_bytes) else null, + .peak_pool_bytes = if (v.peak_pool_bytes_is_set) @intCast(v.peak_pool_bytes) else null, + }; + } }; pub const Device = opaque { @@ -613,29 +618,7 @@ pub const Device = opaque { const ret = try api.call(.PJRT_Device_MemoryStats, .{ .device = self.inner(), }); - return .{ - .bytes_in_use = @intCast(ret.bytes_in_use), - .peak_bytes_in_use = @intCast(ret.peak_bytes_in_use), - .peak_bytes_in_use_is_set = ret.peak_bytes_in_use_is_set, - .num_allocs = @intCast(ret.num_allocs), - .num_allocs_is_set = ret.num_allocs_is_set, - .largest_alloc_size = @intCast(ret.largest_alloc_size), - .largest_alloc_size_is_set = ret.largest_alloc_size_is_set, - .bytes_limit = @intCast(ret.bytes_limit), - .bytes_limit_is_set = ret.bytes_limit_is_set, - .bytes_reserved = @intCast(ret.bytes_reserved), - .bytes_reserved_is_set = ret.bytes_reserved_is_set, - .peak_bytes_reserved = @intCast(ret.peak_bytes_reserved), - .peak_bytes_reserved_is_set = ret.peak_bytes_reserved_is_set, - .bytes_reservable_limit = @intCast(ret.bytes_reservable_limit), - .bytes_reservable_limit_is_set = ret.bytes_reservable_limit_is_set, - .largest_free_block_bytes = @intCast(ret.largest_free_block_bytes), - .largest_free_block_bytes_is_set = ret.largest_free_block_bytes_is_set, - .pool_bytes = @intCast(ret.pool_bytes), - .pool_bytes_is_set = ret.pool_bytes_is_set, - .peak_pool_bytes = @intCast(ret.peak_pool_bytes), - .peak_pool_bytes_is_set = ret.peak_pool_bytes_is_set, - }; + return .fromCStruct(ret); } }; @@ -830,11 +813,11 @@ pub const LoadedExecutable = opaque { _ = try api.call(.PJRT_LoadedExecutable_Execute, .{ .executable = self.inner(), .options = @ptrCast(&options), - .argument_lists = @ptrCast(args.arguments.ptr), + .argument_lists = @ptrCast(args.arguments), .num_devices = args.arguments.len, .num_args = args.num_args, - .output_lists = @ptrCast(args.results.ptr), - .device_complete_events = if (args.events) |ev| @ptrCast(ev.ptr) else null, + .output_lists = @ptrCast(args.results), + .device_complete_events = if (args.events) |ev| @ptrCast(ev) else null, }); } @@ -1042,7 +1025,7 @@ pub const Buffer = opaque { pub fn copyRawToHost(self: *const Buffer, api: *const Api, dst: []u8, offset: i64) ApiError!?*Event { const ret = try api.call(.PJRT_Buffer_CopyRawToHost, .{ .buffer = self.inner(), - .dst = @ptrCast(dst.ptr), + .dst = @ptrCast(dst), .offset = offset, .transfer_size = @intCast(dst.len), }); @@ -1278,7 +1261,7 @@ pub const AsyncHostToDeviceTransferManager = opaque { pub fn addMetadata(self: *AsyncHostToDeviceTransferManager, api: *const Api, transfer_metadata: []const NamedValue) ApiError!void { _ = try api.call(.PJRT_AsyncHostToDeviceTransferManager_AddMetadata, .{ .transfer_manager = self.inner(), - .transfer_metadata = @ptrCast(transfer_metadata.ptr), + .transfer_metadata = @ptrCast(transfer_metadata), .num_metadata = transfer_metadata.len, }); } @@ -1334,9 +1317,9 @@ pub const NamedValue = extern struct { .name_size = name_.len, .type = @intFromEnum(kind_), .unnamed_0 = switch (kind_) { - .string => .{ .string_value = @ptrCast(@constCast(value_.ptr)) }, + .string => .{ .string_value = @ptrCast(@constCast(value_)) }, .int64 => .{ .int64_value = value_ }, - .int64list => .{ .int64_array_value = @ptrCast(@constCast(value_.ptr)) }, + .int64list => .{ .int64_array_value = @ptrCast(@constCast(value_)) }, .float => .{ .float_value = value_ }, .bool => .{ .bool_value = value_ }, }, From 470359f39af8e1a848bde6b083e63a3717ab39b0 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sun, 11 Jan 2026 10:22:12 +0100 Subject: [PATCH 221/881] pjrt: memorystats structs in structs --- pjrt/pjrt.zig | 123 +++++++++++++++++++++++++------------------------- 1 file changed, 61 insertions(+), 62 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index fc3e22fc..4f996a0c 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -539,49 +539,6 @@ pub const Client = opaque { } }; -pub const MemoryStats = struct { - /// Number of bytes in use. - bytes_in_use: u64, - /// The peak bytes in use. - peak_bytes_in_use: ?u64, - /// Number of allocations. - num_allocs: ?u64, - /// The largest single allocation seen. - largest_alloc_size: ?u64, - /// The upper limit of user-allocatable device memory in bytes. - bytes_limit: ?u64, - /// Number of bytes reserved. - bytes_reserved: ?u64, - /// The peak number of bytes reserved. - peak_bytes_reserved: ?u64, - /// The upper limit on the number bytes of reservable memory. - bytes_reservable_limit: ?u64, - /// Largest free block size in bytes. - largest_free_block_bytes: ?u64, - /// Number of bytes of memory held by the allocator. This may be higher than - /// bytes_in_use if the allocator holds a pool of memory (e.g. BFCAllocator). - pool_bytes: ?u64, - peak_pool_bytes: ?u64, - - pub const zeroes = std.mem.zeroes(MemoryStats); - - fn fromCStruct(v: c.PJRT_Device_MemoryStats_Args) MemoryStats { - return .{ - .bytes_in_use = @intCast(v.bytes_in_use), - .peak_bytes_in_use = if (v.peak_bytes_in_use_is_set) @intCast(v.peak_bytes_in_use) else null, - .num_allocs = if (v.num_allocs_is_set) @intCast(v.num_allocs) else null, - .largest_alloc_size = if (v.largest_alloc_size_is_set) @intCast(v.largest_alloc_size) else null, - .bytes_limit = if (v.bytes_limit_is_set) @intCast(v.bytes_limit) else null, - .bytes_reserved = if (v.bytes_reserved_is_set) @intCast(v.bytes_reserved) else null, - .peak_bytes_reserved = if (v.peak_bytes_reserved_is_set) @intCast(v.peak_bytes_reserved) else null, - .bytes_reservable_limit = if (v.bytes_reservable_limit_is_set) @intCast(v.bytes_reservable_limit) else null, - .largest_free_block_bytes = if (v.largest_free_block_bytes_is_set) @intCast(v.largest_free_block_bytes) else null, - .pool_bytes = if (v.pool_bytes_is_set) @intCast(v.pool_bytes) else null, - .peak_pool_bytes = if (v.peak_pool_bytes_is_set) @intCast(v.peak_pool_bytes) else null, - }; - } -}; - pub const Device = opaque { const inner = InnerMixin(c.PJRT_Device).inner; @@ -614,6 +571,49 @@ pub const Device = opaque { return @ptrCast(ret.memories[0..ret.num_memories]); } + pub const MemoryStats = struct { + /// Number of bytes in use. + bytes_in_use: u64, + /// The peak bytes in use. + peak_bytes_in_use: ?u64, + /// Number of allocations. + num_allocs: ?u64, + /// The largest single allocation seen. + largest_alloc_size: ?u64, + /// The upper limit of user-allocatable device memory in bytes. + bytes_limit: ?u64, + /// Number of bytes reserved. + bytes_reserved: ?u64, + /// The peak number of bytes reserved. + peak_bytes_reserved: ?u64, + /// The upper limit on the number bytes of reservable memory. + bytes_reservable_limit: ?u64, + /// Largest free block size in bytes. + largest_free_block_bytes: ?u64, + /// Number of bytes of memory held by the allocator. This may be higher than + /// bytes_in_use if the allocator holds a pool of memory (e.g. BFCAllocator). + pool_bytes: ?u64, + peak_pool_bytes: ?u64, + + pub const zeroes = std.mem.zeroes(MemoryStats); + + fn fromCStruct(v: c.PJRT_Device_MemoryStats_Args) MemoryStats { + return .{ + .bytes_in_use = @intCast(v.bytes_in_use), + .peak_bytes_in_use = if (v.peak_bytes_in_use_is_set) @intCast(v.peak_bytes_in_use) else null, + .num_allocs = if (v.num_allocs_is_set) @intCast(v.num_allocs) else null, + .largest_alloc_size = if (v.largest_alloc_size_is_set) @intCast(v.largest_alloc_size) else null, + .bytes_limit = if (v.bytes_limit_is_set) @intCast(v.bytes_limit) else null, + .bytes_reserved = if (v.bytes_reserved_is_set) @intCast(v.bytes_reserved) else null, + .peak_bytes_reserved = if (v.peak_bytes_reserved_is_set) @intCast(v.peak_bytes_reserved) else null, + .bytes_reservable_limit = if (v.bytes_reservable_limit_is_set) @intCast(v.bytes_reservable_limit) else null, + .largest_free_block_bytes = if (v.largest_free_block_bytes_is_set) @intCast(v.largest_free_block_bytes) else null, + .pool_bytes = if (v.pool_bytes_is_set) @intCast(v.pool_bytes) else null, + .peak_pool_bytes = if (v.peak_pool_bytes_is_set) @intCast(v.peak_pool_bytes) else null, + }; + } + }; + pub fn memoryStats(self: *const Device, api: *const Api) ApiError!MemoryStats { const ret = try api.call(.PJRT_Device_MemoryStats, .{ .device = self.inner(), @@ -728,11 +728,28 @@ pub const Executable = opaque { }; } + pub const CompiledMemoryStats = struct { + /// Mirrors xla::CompiledMemoryStats. + /// Device default memory (e.g., HBM for GPU/TPU) usage stats. + generated_code_size_in_bytes: u64, + argument_size_in_bytes: u64, + output_size_in_bytes: u64, + /// much: How argument is reused for output. + alias_size_in_bytes: u64, + temp_size_in_bytes: u64, + + /// memory: Host usage stats. + host_generated_code_size_in_bytes: u64, + host_argument_size_in_bytes: u64, + host_output_size_in_bytes: u64, + host_alias_size_in_bytes: u64, + host_temp_size_in_bytes: u64, + }; + pub fn getCompiledMemoryStats(self: *const Executable, api: *const Api) ApiError!CompiledMemoryStats { const ret = try api.call(.PJRT_Executable_GetCompiledMemoryStats, .{ .executable = self.inner(), }); - return .{ .generated_code_size_in_bytes = @intCast(ret.generated_code_size_in_bytes), .argument_size_in_bytes = @intCast(ret.argument_size_in_bytes), @@ -748,24 +765,6 @@ pub const Executable = opaque { } }; -pub const CompiledMemoryStats = struct { - // Mirrors xla::CompiledMemoryStats. - // Device default memory (e.g., HBM for GPU/TPU) usage stats. - generated_code_size_in_bytes: u64, - argument_size_in_bytes: u64, - output_size_in_bytes: u64, - // much: How argument is reused for output. - alias_size_in_bytes: u64, - temp_size_in_bytes: u64, - - // memory: Host usage stats. - host_generated_code_size_in_bytes: u64, - host_argument_size_in_bytes: u64, - host_output_size_in_bytes: u64, - host_alias_size_in_bytes: u64, - host_temp_size_in_bytes: u64, -}; - pub const LoadedExecutable = opaque { const inner = InnerMixin(c.PJRT_LoadedExecutable).inner; From 89aa189d8d2fa8945897919818d7f3473e9e2527 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sun, 11 Jan 2026 10:26:29 +0100 Subject: [PATCH 222/881] pjrt: add addressableMemory from pjrtx it should be part of the binding --- pjrt/pjrt.zig | 9 +++++++++ zml/io.zig | 2 +- zml/pjrtx.zig | 9 --------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 4f996a0c..825c343d 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -571,6 +571,15 @@ pub const Device = opaque { return @ptrCast(ret.memories[0..ret.num_memories]); } + pub fn addressableMemory(client: *const Client, api: *const Api, kind: Memory.Kind) ?*const Memory { + for (client.addressableMemories(api)) |mem| { + if (mem.kind(api) == kind) { + return mem; + } + } + return null; + } + pub const MemoryStats = struct { /// Number of bytes in use. bytes_in_use: u64, diff --git a/zml/io.zig b/zml/io.zig index 8a27f842..a962edaf 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -377,7 +377,7 @@ pub const Transfer = struct { spec.* = pjrt.ShapeSpec.init(dims, pjrtx.bufferTypeFromDtype(shape.dtype())); } - const memory = pjrtx.Client.memoryByKind(platform.pjrt_client, platform.pjrt_api, .device).?; + const memory = platform.pjrt_client.addressableMemory(platform.pjrt_api, .device).?; const transfer_manager = try platform.pjrt_client.createBuffersForAsyncHostToDevice(platform.pjrt_api, .{ .shape_specs = shape_specs, .memory = memory }); errdefer transfer_manager.deinit(platform.pjrt_api); diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 37c1e239..6cb4ca3b 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -45,15 +45,6 @@ pub const Client = opaque { .compile_options_pb = compile_options_pb, })); } - - pub fn memoryByKind(client: *const pjrt.Client, api: *const pjrt.Api, kind: pjrt.Memory.Kind) ?*const pjrt.Memory { - for (client.addressableMemories(api)) |mem| { - if (mem.kind(api) == kind) { - return mem; - } - } - return null; - } }; pub fn bufferTypeFromDtype(dt: DataType) pjrt.BufferType { From c1c97d8dec3493534310aad1f533bb7cbb640a49 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sun, 11 Jan 2026 10:31:05 +0100 Subject: [PATCH 223/881] pjrt: more cosmetics --- pjrt/pjrt.zig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 825c343d..a542fcc0 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -136,7 +136,7 @@ pub const Api = struct { }; } - inline fn callInner(self: *const Api, comptime method: Funcs, arg: *PJRTFnArg(method)) ApiError!void { + inline fn innerCall(self: *const Api, comptime method: Funcs, arg: *PJRTFnArg(method)) ApiError!void { const fn_ptr = @field(&self.inner, @tagName(method)).?; const result = fn_ptr(arg); if (@TypeOf(result) == void) { @@ -151,7 +151,7 @@ pub const Api = struct { inline fn call(self: *const Api, comptime method: Funcs, arg: PJRTFnArgWithDefault(method)) ApiError!PJRTFnArgWithDefault(method) { var ret = arg; - try callInner(self, method, @ptrCast(&ret)); + try innerCall(self, method, @ptrCast(&ret)); return ret; } @@ -169,7 +169,7 @@ pub const Api = struct { } pub fn extension(self: *const Api, comptime ExtensionT: type, ext_id: c_int) ?*const ExtensionT { - var it = extensions(self); + var it = self.extensions(); while (it.next()) |ext| { if (ext.type == ext_id) { return @ptrCast(@alignCast(ext)); From 8c8f233f6aa8bcd72d97ed9cecf0589121ccebd9 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sun, 11 Jan 2026 15:57:45 +0100 Subject: [PATCH 224/881] third_party/xla: bump to head --- MODULE.bazel | 2 +- ...header-C-compliant-for-PJRT-FFI-exte.patch | 45 ------------------- third_party/xla/repo.bzl | 6 +-- third_party/xla/xla.bzl | 9 +--- 4 files changed, 3 insertions(+), 59 deletions(-) delete mode 100644 third_party/xla/patches/0001-PjRT-C-API-male-header-C-compliant-for-PJRT-FFI-exte.patch diff --git a/MODULE.bazel b/MODULE.bazel index 6c7ccdc0..266c86b2 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -22,6 +22,7 @@ bazel_dep(name = "rules_oci", version = "2.2.6") bazel_dep(name = "rules_proto", version = "7.1.0") bazel_dep(name = "rules_python", version = "1.8.0-rc1") bazel_dep(name = "rules_rust", version = "0.68.1") +bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "rules_zig", version = "0.12.2") bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.3.1") bazel_dep(name = "with_cfg.bzl", version = "0.11.0") @@ -126,7 +127,6 @@ use_repo( "local_config_tensorrt", "python_version_repo", "rules_ml_toolchain", - "rules_shell", "stablehlo", "sycl_configure", "sycl_configure_ext", diff --git a/third_party/xla/patches/0001-PjRT-C-API-male-header-C-compliant-for-PJRT-FFI-exte.patch b/third_party/xla/patches/0001-PjRT-C-API-male-header-C-compliant-for-PJRT-FFI-exte.patch deleted file mode 100644 index bfb65808..00000000 --- a/third_party/xla/patches/0001-PjRT-C-API-male-header-C-compliant-for-PJRT-FFI-exte.patch +++ /dev/null @@ -1,45 +0,0 @@ -From 0d88ac9b06c8bc78db817d85e90cd60d38e6561a Mon Sep 17 00:00:00 2001 -From: Hugo Mano -Date: Mon, 3 Nov 2025 16:54:54 +0100 -Subject: [PATCH] PjRT C API: make PJRT FFI C extension header compliant - - -XLA PR: https://github.com/openxla/xla/pull/33470 - ---- - xla/pjrt/c/pjrt_c_api_ffi_extension.h | 8 ++++---- - 1 file changed, 4 insertions(+), 4 deletions(-) - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -index e756650911..33b78238b9 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -@@ -32,13 +32,13 @@ extern "C" { - // See: https://en.wikipedia.org/wiki/Foreign_function_interface - #define PJRT_API_FFI_EXTENSION_VERSION 3 - --struct PJRT_FFI_Type_Info { -+typedef struct PJRT_FFI_Type_Info { - void (*deleter)(void* object); - void (*serialize)(); // placeholder for future use - void (*deserialize)(); // placeholder for future use --}; -+} PJRT_FFI_Type_Info; - --struct PJRT_FFI_Type_Register_Args { -+typedef struct PJRT_FFI_Type_Register_Args { - size_t struct_size; - PJRT_Extension_Base* extension_start; - -@@ -46,7 +46,7 @@ struct PJRT_FFI_Type_Register_Args { - size_t type_name_size; - int64_t type_id; // in-out - PJRT_FFI_Type_Info* type_info; --}; -+} PJRT_FFI_Type_Register_Args; - PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Type_Register_Args, type_info); - - // Registers external type in a static type registry. If `type_id` is set to `0` --- -2.50.1 (Apple Git-155) - diff --git a/third_party/xla/repo.bzl b/third_party/xla/repo.bzl index 4e45d2d3..67740474 100644 --- a/third_party/xla/repo.bzl +++ b/third_party/xla/repo.bzl @@ -4,9 +4,5 @@ def repo(): git_repository( name = "xla", remote = "https://github.com/openxla/xla.git", - commit = "9a77a882bb2bc75cb8c29620ff8cd0fd089bdc86", - patch_args = ["-p1"], - patches = [ - "third_party/xla/patches/0001-PjRT-C-API-male-header-C-compliant-for-PJRT-FFI-exte.patch", - ], + commit = "01da52b9afe3a2e694bd926323a649a6e63a3785", ) diff --git a/third_party/xla/xla.bzl b/third_party/xla/xla.bzl index 8444ffdb..43a6629b 100644 --- a/third_party/xla/xla.bzl +++ b/third_party/xla/xla.bzl @@ -42,6 +42,7 @@ if_rocm_newer_than = always_newer_than is_rocm_configured = always_false if_gpu_is_configured = always_if_false if_cuda_or_rocm = always_if_false +get_rbe_amdgpu_pool = always_false """, }) simple_files(name = "local_config_sycl", files = { @@ -69,14 +70,6 @@ if_sycl_is_configured = always_if_false }) simple_files(name = "sycl_configure_ext", files = {}) simple_files(name = "sycl_configure", files = {}) - simple_files(name = "rules_shell", files = { - "BUILD.bazel": "", - "shell/BUILD.bazel": "", - "shell/sh_binary.bzl": """ -def sh_binary(**kwargs): - native.sh_binary(**kwargs) -""", - }) def _xla_impl(mctx): llvm("llvm-raw") From a05fa3299e7cbded36b472fb000c47ffc66fc466 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sun, 11 Jan 2026 15:59:43 +0100 Subject: [PATCH 225/881] pjrt: rework extensions support through tagged union --- pjrt/BUILD.bazel | 9 ++++++++ pjrt/pjrt.zig | 60 ++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 59 insertions(+), 10 deletions(-) diff --git a/pjrt/BUILD.bazel b/pjrt/BUILD.bazel index 173bea7e..282cd1dc 100644 --- a/pjrt/BUILD.bazel +++ b/pjrt/BUILD.bazel @@ -8,9 +8,18 @@ zig_library( deps = [ "//stdx", "@xla//xla/ffi/api:c_api", + # TODO: fix non conformance to C standard + # "@xla//xla/pjrt/c:pjrt_c_api_callback_extension_hdrs", + "@xla//xla/pjrt/c:pjrt_c_api_custom_partitioner_extension_hdrs", "@xla//xla/pjrt/c:pjrt_c_api_ffi_extension_hdrs", "@xla//xla/pjrt/c:pjrt_c_api_gpu_extension_hdrs", "@xla//xla/pjrt/c:pjrt_c_api_hdrs", + "@xla//xla/pjrt/c:pjrt_c_api_layouts_extension_hdrs", + "@xla//xla/pjrt/c:pjrt_c_api_memory_descriptions_extension_hdrs", + "@xla//xla/pjrt/c:pjrt_c_api_phase_compile_extension_hdrs", + "@xla//xla/pjrt/c:pjrt_c_api_profiler_extension_hdrs", + "@xla//xla/pjrt/c:pjrt_c_api_raw_buffer_extension_hdrs", + "@xla//xla/pjrt/c:pjrt_c_api_stream_extension_hdrs", "@xla//xla/pjrt/c:pjrt_c_api_triton_extension_hdrs", ], ) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index a542fcc0..1c08ab2a 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -155,24 +155,64 @@ pub const Api = struct { return ret; } - pub const ExtensionsIterator = struct { - current: ?*const c.PJRT_Extension_Base, + pub const Extensions = struct { + pub const Type = enum(c.PJRT_Extension_Type) { + custom_call = c.PJRT_Extension_Type_Gpu_Custom_Call, + profiler = c.PJRT_Extension_Type_Profiler, + custom_partitioner = c.PJRT_Extension_Type_Custom_Partitioner, + stream = c.PJRT_Extension_Type_Stream, + layouts = c.PJRT_Extension_Type_Layouts, + ffi = c.PJRT_Extension_Type_FFI, + memory_descriptions = c.PJRT_Extension_Type_MemoryDescriptions, + triton = c.PJRT_Extension_Type_Triton, + raw_buffer = c.PJRT_Extension_Type_RawBuffer, + phase_compile = c.PJRT_Extension_Type_PhaseCompile, + unknown = c.PJRT_Extension_Type_Unknown, + }; - pub fn next(self: *ExtensionsIterator) ?*const c.PJRT_Extension_Base { - self.current = self.current.next; - return self.current; - } + pub const Extension = union(Type) { + custom_call: *const c.PJRT_Gpu_Custom_Call, + profiler: *const c.PJRT_Profiler_Extension, + custom_partitioner: *const c.PJRT_Custom_Partitioner_Extension, + stream: *const c.PJRT_Stream_Extension, + layouts: *const c.PJRT_Layouts_Extension, + ffi: *const c.PJRT_FFI_Extension, + memory_descriptions: *const c.PJRT_MemoryDescriptions_Extension, + triton: *const c.PJRT_Triton_Extension, + raw_buffer: *const c.PJRT_RawBuffer_Extension, + phase_compile: *const c.PJRT_PhaseCompile_Extension, + unknown: *const c.PJRT_Extension_Base, + }; + + pub const Iterator = struct { + current: ?*const c.PJRT_Extension_Base, + + pub fn next(self: *Iterator) ?Extension { + defer if (self.current) |cur| { + self.current = cur.next; + }; + if (self.current) |cur| { + if (std.enums.fromInt(Extensions.Type, cur.type)) |e| { + return switch (e) { + inline else => |t| @unionInit(Extension, @tagName(t), @ptrCast(self.current)), + }; + } + return .{ .unknown = cur }; + } + return null; + } + }; }; - pub fn extensions(self: *const Api) ExtensionsIterator { + pub fn extensions(self: *const Api) Extensions.Iterator { return .{ .current = @ptrCast(@alignCast(self.inner.extension_start)) }; } - pub fn extension(self: *const Api, comptime ExtensionT: type, ext_id: c_int) ?*const ExtensionT { + pub fn extension(self: *const Api, ext_type: Extensions.Type) ?Extensions.Extension { var it = self.extensions(); while (it.next()) |ext| { - if (ext.type == ext_id) { - return @ptrCast(@alignCast(ext)); + if (std.meta.activeTag(ext) == ext_type) { + return ext; } } return null; From 431dcd7a1c26e6f67061ee1b06b297f9443ed542 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sun, 11 Jan 2026 16:00:38 +0100 Subject: [PATCH 226/881] pjrt: guard against out of bounds access when calling into an API, make sure that the method is inside the struct_size, else we would read into undefined memory --- pjrt/pjrt.zig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 1c08ab2a..ad39f95c 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -137,6 +137,9 @@ pub const Api = struct { } inline fn innerCall(self: *const Api, comptime method: Funcs, arg: *PJRTFnArg(method)) ApiError!void { + if (@offsetOf(c.PJRT_Api, @tagName(method)) > self.inner.struct_size) { + std.debug.panic("PJRT Api method {s} not available in this plugin", .{@tagName(method)}); + } const fn_ptr = @field(&self.inner, @tagName(method)).?; const result = fn_ptr(arg); if (@TypeOf(result) == void) { From 989f5aef25ec56971b64fc1a2237fc78d53c7c40 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sun, 11 Jan 2026 17:14:37 +0100 Subject: [PATCH 227/881] workspace: bump bazel=8.5.0 --- .bazelversion | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bazelversion b/.bazelversion index 56b6be4e..6d289079 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -8.3.1 +8.5.0 From faee73c5ee95d63577cc28d30ea1c1d861c43aa3 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sun, 11 Jan 2026 17:19:30 +0100 Subject: [PATCH 228/881] zml: explicit TF_CPP_LOG_LEVEL --- zml/platform.zig | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/zml/platform.zig b/zml/platform.zig index 6687001d..23675eab 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -35,9 +35,16 @@ fn StaticPlatformMap(comptime E: type, comptime T: type) type { var api_map: StaticPlatformMap(Target, ?*const pjrt.Api) = .{}; fn disableXlaLogs() void { + // https://deepreg.readthedocs.io/en/latest/docs/logging.html#tensorflow-logging + const TF_CPP_LOG_LEVEL = struct { + const DEBUG = "0"; + const INFO = "1"; + const WARNING = "2"; + const ERROR = "3"; + }; _ = c.setenv( "TF_CPP_MIN_LOG_LEVEL", - std.posix.getenv("TF_CPP_MIN_LOG_LEVEL") orelse "3", + std.posix.getenv("TF_CPP_MIN_LOG_LEVEL") orelse TF_CPP_LOG_LEVEL.ERROR, 1, ); } From dc4f6e2f002b6f71b6925214df7ab07dbe3abcf6 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sun, 11 Jan 2026 21:24:57 +0100 Subject: [PATCH 229/881] zml: remove zml.init and deinit --- examples/benchmark/main.zig | 3 --- examples/llama/main.zig | 3 --- examples/mnist/mnist.zig | 3 --- zml/mlirx.zig | 6 ------ zml/module.zig | 8 +++++++- zml/zml.zig | 37 ------------------------------------- 6 files changed, 7 insertions(+), 53 deletions(-) diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index 98162058..06d4f047 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -24,9 +24,6 @@ pub fn main() !void { const io = threaded.io(); - zml.init(); - defer zml.deinit(); - // Auto-select platform const platform: zml.Platform = try .auto(io, .{}); diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 3be7dbcb..91b218dc 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -34,9 +34,6 @@ pub fn main() !void { const io = threaded.io(); - zml.init(); - defer zml.deinit(); - const args: Args = blk: { var ret: Args = .{}; var it = std.process.args(); diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 1ffd65e0..49633891 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -81,9 +81,6 @@ pub fn main() !void { // const io = threaded.io(); - zml.init(); - defer zml.deinit(); - // Parse program args const process_args = try std.process.argsAlloc(allocator); defer std.process.argsFree(allocator, process_args); diff --git a/zml/mlirx.zig b/zml/mlirx.zig index 2c3ddb17..748d988c 100644 --- a/zml/mlirx.zig +++ b/zml/mlirx.zig @@ -85,9 +85,3 @@ pub const Type = struct { std.debug.panic("Could not convert mlir.Type to DataType: {f}", .{mlir_type}); } }; - -pub var once = std.once(struct { - fn call() void { - mlir.registerPasses("Transforms"); - } -}.call); diff --git a/zml/module.zig b/zml/module.zig index ab419bf9..3af04fde 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -58,8 +58,14 @@ pub const CompilationContext = struct { threadlocal var _current: ?*CompilationContext = null; + var mlir_once = std.once(struct { + fn call() void { + mlir.registerPasses("Transforms"); + } + }.call); + pub fn init(allocator: std.mem.Allocator, platform: Platform) CompilationContext { - mlir.registerPasses("Transforms"); + mlir_once.call(); const mlir_registry = mlir.DialectRegistry.init() catch unreachable; inline for (.{ "func", "stablehlo" }) |d| { mlir.DialectHandle.fromString(d).insertDialect(mlir_registry); diff --git a/zml/zml.zig b/zml/zml.zig index 7fd9e07e..a68b9dc3 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -39,43 +39,6 @@ pub const tensor = @import("tensor.zig"); pub const Tensor = tensor.Tensor; pub const testing = @import("testing.zig"); -var runfiles_once = std.once(struct { - fn call_() !void { - if (std.process.hasEnvVarConstant("RUNFILES_MANIFEST_FILE") or std.process.hasEnvVarConstant("RUNFILES_DIR")) { - return; - } - - var io_: std.Io.Threaded = .init_single_threaded; - defer io_.deinit(); - - var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); - const allocator = arena.allocator(); - defer arena.deinit(); - - var envMap = std.process.EnvMap.init(allocator); - var r = (try runfiles.Runfiles.create(.{ .allocator = allocator, .io = io_.io() })) orelse return; - try r.environment(&envMap); - - var it = envMap.iterator(); - while (it.next()) |entry| { - const keyZ = try allocator.dupeZ(u8, entry.key_ptr.*); - const valueZ = try allocator.dupeZ(u8, entry.value_ptr.*); - _ = c.setenv(keyZ.ptr, valueZ.ptr, 1); - } - } - - fn call() void { - call_() catch @panic("Unable to init runfiles env"); - } -}.call); - -pub fn init() void { - runfiles_once.call(); - mlir.once.call(); -} - -pub fn deinit() void {} - /// Return a clone of a type with Tensors replaced by Buffer. /// Non-Tensor metadata is stripped out of the resulting struct. /// Recursively descends into the type. From b0f35d481509cb5d41b9c33cf611f11c402ff459 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sun, 11 Jan 2026 22:13:57 +0100 Subject: [PATCH 230/881] pjrt: remove dep to stdx --- pjrt/BUILD.bazel | 1 - pjrt/pjrt.zig | 1 - 2 files changed, 2 deletions(-) diff --git a/pjrt/BUILD.bazel b/pjrt/BUILD.bazel index 282cd1dc..e9f365d1 100644 --- a/pjrt/BUILD.bazel +++ b/pjrt/BUILD.bazel @@ -6,7 +6,6 @@ zig_library( main = "pjrt.zig", visibility = ["//visibility:public"], deps = [ - "//stdx", "@xla//xla/ffi/api:c_api", # TODO: fix non conformance to C standard # "@xla//xla/pjrt/c:pjrt_c_api_callback_extension_hdrs", diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index ad39f95c..135487a1 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -2,7 +2,6 @@ const std = @import("std"); const builtin = @import("builtin"); const c = @import("c"); -const stdx = @import("stdx"); pub const ffi = @import("ffi.zig"); From b03b6299fc13d7723bfdcb4b408e9971f6e27001 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sun, 11 Jan 2026 22:23:25 +0100 Subject: [PATCH 231/881] pjrt: remove get prefixes when not matching with PJRT --- pjrt/pjrt.zig | 40 ++++++++++++++++++++-------------------- zml/buffer.zig | 4 ++-- zml/module.zig | 2 +- zml/platform.zig | 10 +++++----- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 135487a1..7b07b3a9 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -394,21 +394,21 @@ pub const Client = opaque { }) catch {}; } - pub fn getPlatformName(self: *const Client, api: *const Api) []const u8 { + pub fn platformName(self: *const Client, api: *const Api) []const u8 { const ret = api.call(.PJRT_Client_PlatformName, .{ .client = self.inner(), }) catch unreachable; return ret.platform_name[0..ret.platform_name_size]; } - pub fn getDevices(self: *const Client, api: *const Api) []const *Device { + pub fn devices(self: *const Client, api: *const Api) []const *Device { const ret = api.call(.PJRT_Client_Devices, .{ .client = self.inner(), }) catch unreachable; return @ptrCast(ret.devices[0..ret.num_devices]); } - pub fn getAddressableDevices(self: *const Client, api: *const Api) []const *Device { + pub fn addressableDevices(self: *const Client, api: *const Api) []const *Device { const ret = api.call(.PJRT_Client_AddressableDevices, .{ .client = self.inner(), }) catch unreachable; @@ -598,7 +598,7 @@ pub const Device = opaque { return ret.is_addressable; } - pub fn getLocalHardwareId(self: *const Device, api: *const Api) usize { + pub fn localHardwareId(self: *const Device, api: *const Api) usize { const ret = api.call(.PJRT_Device_LocalHardwareId, .{ .device = self.inner(), }) catch unreachable; @@ -676,21 +676,21 @@ pub const Device = opaque { pub const DeviceDescription = opaque { const inner = InnerMixin(c.PJRT_DeviceDescription).inner; - pub fn getId(self: *const DeviceDescription, api: *const Api) usize { + pub fn id(self: *const DeviceDescription, api: *const Api) usize { const ret = api.call(.PJRT_DeviceDescription_Id, .{ .device_description = self.inner(), }) catch unreachable; return @intCast(ret.id); } - pub fn getProcessIndex(self: *const DeviceDescription, api: *const Api) usize { + pub fn processIndex(self: *const DeviceDescription, api: *const Api) usize { const ret = api.call(.PJRT_DeviceDescription_ProcessIndex, .{ .device_description = self.inner(), }) catch unreachable; return @intCast(ret.process_index); } - pub fn getKind(self: *const DeviceDescription, api: *const Api) []const u8 { + pub fn kind(self: *const DeviceDescription, api: *const Api) []const u8 { const ret = api.call(.PJRT_DeviceDescription_Kind, .{ .device_description = self.inner(), }) catch unreachable; @@ -759,7 +759,7 @@ pub const Executable = opaque { }) catch unreachable; } - pub fn getCostAnalysis(self: *const Executable, api: *const Api) GetCostAnalysisError![]const NamedValue { + pub fn costAnalysis(self: *const Executable, api: *const Api) GetCostAnalysisError![]const NamedValue { const ret = try api.call(.PJRT_Executable_GetCostAnalysis, .{ .executable = self.inner(), }); @@ -838,7 +838,7 @@ pub const LoadedExecutable = opaque { return ret.is_deleted; } - pub fn getAddressableDevices(self: *const LoadedExecutable, api: *const Api) []const *Device { + pub fn addressableDevices(self: *const LoadedExecutable, api: *const Api) []const *Device { const ret = api.call(.PJRT_LoadedExecutable_AddressableDevices, .{ .executable = self.inner(), }) catch unreachable; @@ -871,7 +871,7 @@ pub const LoadedExecutable = opaque { }); } - pub fn getExecutable(self: *const LoadedExecutable, api: *const Api) ApiError!*Executable { + pub fn executable(self: *const LoadedExecutable, api: *const Api) ApiError!*Executable { const ret = try api.call(.PJRT_LoadedExecutable_GetExecutable, .{ .loaded_executable = self.inner(), }); @@ -983,7 +983,7 @@ pub const Buffer = opaque { }) catch unreachable; } - pub fn getDevice(self: *const Buffer, api: *const Api) ApiError!*Device { + pub fn device(self: *const Buffer, api: *const Api) ApiError!*Device { const ret = try api.call(.PJRT_Buffer_Device, .{ .buffer = self.inner(), }); @@ -1019,14 +1019,14 @@ pub const Buffer = opaque { return @ptrCast(ret.event); } - pub fn getElementType(self: *const Buffer, api: *const Api) BufferType { + pub fn elementType(self: *const Buffer, api: *const Api) BufferType { const ret = api.call(.PJRT_Buffer_ElementType, .{ .buffer = self.inner(), }) catch unreachable; return @enumFromInt(ret.type); } - pub fn getDimensions(self: *const Buffer, api: *const Api) []const i64 { + pub fn dimensions(self: *const Buffer, api: *const Api) []const i64 { const ret = api.call(.PJRT_Buffer_Dimensions, .{ .buffer = self.inner(), }) catch unreachable; @@ -1036,36 +1036,36 @@ pub const Buffer = opaque { return ret.dims[0..ret.num_dims]; } - pub fn getUnpaddedDimensions(self: *const Buffer, api: *const Api) ApiError![]const i64 { + pub fn unpaddedDimensions(self: *const Buffer, api: *const Api) ApiError![]const i64 { const ret = try api.call(.PJRT_Buffer_UnpaddedDimensions, .{ .buffer = self.inner(), }); return ret.unpadded_dims[0..ret.num_dims]; } - pub fn getOnDeviceSizeInBytes(self: *const Buffer, api: *const Api) ApiError!usize { + pub fn onDeviceSizeInBytes(self: *const Buffer, api: *const Api) ApiError!usize { const ret = try api.call(.PJRT_Buffer_OnDeviceSizeInBytes, .{ .buffer = self.inner(), }); - return ret.on_device_size_in_bytes; + return @intCast(ret.on_device_size_in_bytes); } - pub fn copyToDevice(self: *const Buffer, api: *const Api, device: *Device) ApiError!*Buffer { + pub fn copyToDevice(self: *const Buffer, api: *const Api, dst_device: *Device) ApiError!*Buffer { const ret = try api.call(.PJRT_Buffer_CopyToDevice, .{ .buffer = self.inner(), - .dst_device = device.inner(), + .dst_device = dst_device.inner(), }); return @ptrCast(ret.dst_buffer.?); } - pub fn getReadyEvent(self: *const Buffer, api: *const Api) *Event { + pub fn readyEvent(self: *const Buffer, api: *const Api) *Event { const ret = api.call(.PJRT_Buffer_ReadyEvent, .{ .buffer = self.inner(), }) catch unreachable; return @ptrCast(ret.event.?); } - pub fn getOpaqueDeviceMemoryDataPointer(self: *const Buffer, api: *const Api) ApiError!*anyopaque { + pub fn opaqueDeviceMemoryDataPointer(self: *const Buffer, api: *const Api) ApiError!*anyopaque { const ret = try api.call(.PJRT_Buffer_OpaqueDeviceMemoryDataPointer, .{ .buffer = self.inner(), }); diff --git a/zml/buffer.zig b/zml/buffer.zig index 4f2a2920..43067706 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -136,7 +136,7 @@ pub const Buffer = struct { pub fn await(self: Buffer, io: std.Io) !Buffer { for (self._shards.constSlice()) |buffer| { - const ev = buffer.getReadyEvent(self._api); + const ev = buffer.readyEvent(self._api); defer ev.deinit(self._api); try ev.await(self._api, io); } @@ -220,7 +220,7 @@ pub const Buffer = struct { pub fn devicePtr(self: Buffer) *anyopaque { //stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer", .{}); - return self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable; + return self._shards.get(0).opaqueDeviceMemoryDataPointer(self._api) catch unreachable; } /// Fetches the content of the given buffer into a stack variable of the given type. diff --git a/zml/module.zig b/zml/module.zig index 3af04fde..25082857 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -487,7 +487,7 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: .{ "gfx906", {} }, .{ "gfx900", {} }, }); - const first_device = platform.pjrt_client.getDevices(platform.pjrt_api)[0]; + const first_device = platform.pjrt_client.devices(platform.pjrt_api)[0]; const description = first_device.getDescription(platform.pjrt_api); break :gemm_blk supported.has(description.attribute(platform.pjrt_api, "compute_capability").?.string); }, upb_arena); diff --git a/zml/platform.zig b/zml/platform.zig index 23675eab..00578890 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -96,7 +96,7 @@ pub const Platform = struct { pub fn format(self: Device, writer: *std.Io.Writer) std.Io.Writer.Error!void { const description = self.device.getDescription(self.api); - try writer.print("{s} ({s})", .{ description.getKind(self.api), description.debugString(self.api) }); + try writer.print("{s} ({s})", .{ description.kind(self.api), description.debugString(self.api) }); } }; @@ -105,7 +105,7 @@ pub const Platform = struct { var named_values_buf: [16]pjrt.NamedValue = undefined; const pjrt_client = try pjrt.Client.init(api, options.toNamedValues(target, &named_values_buf)); - const true_num_devices = pjrt_client.getAddressableDevices(api).len; + const true_num_devices = pjrt_client.addressableDevices(api).len; if (true_num_devices > MAX_NUM_DEVICES) { log.warn("platform {} got {} devices, but ZML only support up to {} devices. Some devices won't be used.", .{ target, true_num_devices, MAX_NUM_DEVICES }); } @@ -143,7 +143,7 @@ pub const Platform = struct { try writer.print("Devices total={d}\n", .{devices.len}); for (devices, 0..) |device, i| { const description = device.getDescription(self.pjrt_api); - try writer.print("\t#{d}: {s}\n", .{ i, description.getKind(self.pjrt_api) }); + try writer.print("\t#{d}: {s}\n", .{ i, description.kind(self.pjrt_api) }); } } @@ -152,7 +152,7 @@ pub const Platform = struct { } pub fn getDevices(self: Platform) []const *const pjrt.Device { - const all_devices = self.pjrt_client.getAddressableDevices(self.pjrt_api); + const all_devices = self.pjrt_client.addressableDevices(self.pjrt_api); if (all_devices.len > MAX_NUM_DEVICES) { return all_devices[0..MAX_NUM_DEVICES]; } @@ -252,7 +252,7 @@ pub const Platform = struct { const devices = self.getDevices(); for (0..devices.len) |i| { const description = devices[i].getDescription(self.pjrt_api); - try writer.print("{s}(\"{s}\")", .{ description.toString(self.pjrt_api), description.getKind(self.pjrt_api) }); + try writer.print("{s}(\"{s}\")", .{ description.toString(self.pjrt_api), description.kind(self.pjrt_api) }); if (i < devices.len - 1) try writer.writeAll(", "); } try writer.writeAll(" }"); From 3112f1138a7efee4148bf68a697860d406b2d747 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Wed, 14 Jan 2026 18:00:22 +0100 Subject: [PATCH 232/881] stdx: add utf32 helpers --- stdx/BUILD.bazel | 2 ++ stdx/stdx.zig | 1 + stdx/unicode.zig | 24 ++++++++++++++++++++++++ 3 files changed, 27 insertions(+) create mode 100644 stdx/unicode.zig diff --git a/stdx/BUILD.bazel b/stdx/BUILD.bazel index 6d915882..5f44c3ed 100644 --- a/stdx/BUILD.bazel +++ b/stdx/BUILD.bazel @@ -15,10 +15,12 @@ zig_library( "queue.zig", "signature.zig", "time.zig", + "unicode.zig", ], zigopts = ["-lc"], main = "stdx.zig", visibility = ["//visibility:public"], + zigopts = ["-lc"], ) zig_test( diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 74959e62..a149f013 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -13,6 +13,7 @@ pub const meta = @import("meta.zig"); pub const process = @import("process.zig"); pub const queue = @import("queue.zig"); pub const time = @import("time.zig"); +pub const unicode = @import("unicode.zig"); test { std.testing.refAllDecls(@This()); diff --git a/stdx/unicode.zig b/stdx/unicode.zig new file mode 100644 index 00000000..89c65a28 --- /dev/null +++ b/stdx/unicode.zig @@ -0,0 +1,24 @@ +const std = @import("std"); + +pub fn calcUtf32LeLen(utf8: []const u8) !usize { + return (try std.unicode.utf8CountCodepoints(utf8)) * @sizeOf(u32); +} + +pub fn utf8ToUtf32Le(utf32le: []u32, utf8: []const u8) !usize { + var view: std.unicode.Utf8View = .initUnchecked(utf8); + var it = view.iterator(); + var idx: usize = 0; + while (it.nextCodepoint()) |cp| : (idx += 1) { + if (idx >= utf32le.len) { + return error.BufferTooSmall; + } + utf32le[idx] = @intCast(cp); + } + return idx; +} + +pub fn utf8ToUtf32LeZ(utf32le: []u32, utf8: []const u8) !usize { + const n = try utf8ToUtf32Le(utf32le, utf8); + utf32le[n] = 0; + return n + 1; +} From 25a5ee10f65d614e2f61d291cce78f93561c1325 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 7 Jan 2026 12:08:31 +0100 Subject: [PATCH 233/881] rework zml.io.* --- MODULE.bazel.lock | 2814 +---------------------------------- examples/llama/llama.zig | 59 +- examples/llama/main.zig | 193 ++- examples/mnist/mnist.zig | 35 +- examples/vfs/BUILD.bazel | 15 + examples/vfs/playground.zig | 209 +++ pjrt/pjrt.zig | 4 +- stdx/BUILD.bazel | 3 +- stdx/Io.zig | 101 ++ stdx/segmented_list.zig | 539 +++++++ stdx/stdx.zig | 1 + zml/buffer.zig | 9 + zml/io.zig | 562 ++++--- zml/io/vfs.zig | 1022 ++++--------- zml/io/vfs/BUILD.bazel | 12 +- zml/io/vfs/base.zig | 44 - zml/io/vfs/file.zig | 357 ++--- zml/io/vfs/hf.zig | 1782 ++++++++-------------- zml/io/vfs/hf2.zig | 316 ---- zml/io/vfs/http.zig | 1100 +++++--------- zml/io/vfs/index.zig | 1 + zml/io/vfs/proxy.zig | 546 ------- zml/safetensors.zig | 184 +-- 23 files changed, 2958 insertions(+), 6950 deletions(-) create mode 100644 examples/vfs/BUILD.bazel create mode 100644 examples/vfs/playground.zig create mode 100644 stdx/segmented_list.zig delete mode 100644 zml/io/vfs/hf2.zig delete mode 100644 zml/io/vfs/proxy.zig diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 454f2511..fc687128 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1,5 +1,5 @@ { - "lockFileVersion": 18, + "lockFileVersion": 24, "registryFileHashes": { "https://bazel-registry.zml.ai/modules/abseil-cpp/20210324.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20211102.0/MODULE.bazel": "not found", @@ -131,7 +131,7 @@ "https://bazel-registry.zml.ai/modules/rules_java/7.2.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_java/7.3.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_java/7.6.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/8.12.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_java/8.14.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_java/8.3.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_java/8.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_java/8.6.0/MODULE.bazel": "not found", @@ -352,8 +352,8 @@ "https://bcr.bazel.build/modules/rules_java/7.2.0/MODULE.bazel": "06c0334c9be61e6cef2c8c84a7800cef502063269a5af25ceb100b192453d4ab", "https://bcr.bazel.build/modules/rules_java/7.3.2/MODULE.bazel": "50dece891cfdf1741ea230d001aa9c14398062f2b7c066470accace78e412bc2", "https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe", - "https://bcr.bazel.build/modules/rules_java/8.12.0/MODULE.bazel": "8e6590b961f2defdfc2811c089c75716cb2f06c8a4edeb9a8d85eaa64ee2a761", - "https://bcr.bazel.build/modules/rules_java/8.12.0/source.json": "cbd5d55d9d38d4008a7d00bee5b5a5a4b6031fcd4a56515c9accbcd42c7be2ba", + "https://bcr.bazel.build/modules/rules_java/8.14.0/MODULE.bazel": "717717ed40cc69994596a45aec6ea78135ea434b8402fb91b009b9151dd65615", + "https://bcr.bazel.build/modules/rules_java/8.14.0/source.json": "8a88c4ca9e8759da53cddc88123880565c520503321e2566b4e33d0287a3d4bc", "https://bcr.bazel.build/modules/rules_java/8.3.2/MODULE.bazel": "7336d5511ad5af0b8615fdc7477535a2e4e723a357b6713af439fe8cf0195017", "https://bcr.bazel.build/modules/rules_java/8.5.1/MODULE.bazel": "d8a9e38cc5228881f7055a6079f6f7821a073df3744d441978e7a43e20226939", "https://bcr.bazel.build/modules/rules_java/8.6.0/MODULE.bazel": "9c064c434606d75a086f15ade5edb514308cccd1544c2b2a89bbac4310e41c71", @@ -435,2808 +435,6 @@ "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198" }, "selectedYankedVersions": {}, - "moduleExtensions": { - "@@apple_support+//crosstool:setup.bzl%apple_cc_configure_extension": { - "general": { - "bzlTransitiveDigest": "cw6mKbsar0LsMbAePOLZdW+szZNKRE8/CrhomQFNbNE=", - "usagesDigest": "GXVjAqzvGUnh20kBPs+f0GSdAtaqLPet4sogeCfejZY=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "local_config_apple_cc_toolchains": { - "repoRuleId": "@@apple_support+//crosstool:setup.bzl%_apple_cc_autoconf_toolchains", - "attributes": {} - }, - "local_config_apple_cc": { - "repoRuleId": "@@apple_support+//crosstool:setup.bzl%_apple_cc_autoconf", - "attributes": {} - } - }, - "recordedRepoMappingEntries": [ - [ - "apple_support+", - "bazel_tools", - "bazel_tools" - ], - [ - "bazel_tools", - "rules_cc", - "rules_cc+" - ] - ] - } - }, - "@@aspect_rules_py+//py:extensions.bzl%py_tools": { - "general": { - "bzlTransitiveDigest": "Udhh0icjIe0dHLjbgfmj2tzBz0geB4SXJl0FoPcpF84=", - "usagesDigest": "tVlELaqQFX7HPl1uFzRY9m7nVV7tY9roq1qfK75ApwE=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "bsd_tar_darwin_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "bsd_tar_darwin_arm64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "bsd_tar_linux_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "bsd_tar_linux_arm64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "bsd_tar_windows_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "windows_amd64" - } - }, - "bsd_tar_windows_arm64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "windows_arm64" - } - }, - "bsd_tar_toolchains": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%tar_toolchains_repo", - "attributes": { - "user_repository_name": "bsd_tar" - } - }, - "rules_py_tools.darwin_amd64": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "rules_py_tools.darwin_arm64": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "rules_py_tools.linux_amd64": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "rules_py_tools.linux_arm64": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "rules_py_tools": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%toolchains_repo", - "attributes": { - "user_repository_name": "rules_py_tools" - } - }, - "rules_py_pex_2_3_1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://files.pythonhosted.org/packages/e7/d0/fbda2a4d41d62d86ce53f5ae4fbaaee8c34070f75bb7ca009090510ae874/pex-2.3.1-py2.py3-none-any.whl" - ], - "sha256": "64692a5bf6f298403aab930d22f0d836ae4736c5bc820e262e9092fe8c56f830", - "downloaded_file_path": "pex-2.3.1-py2.py3-none-any.whl" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "aspect_bazel_lib+", - "bazel_tools", - "bazel_tools" - ], - [ - "aspect_rules_py+", - "aspect_bazel_lib", - "aspect_bazel_lib+" - ], - [ - "aspect_rules_py+", - "aspect_tools_telemetry_report", - "aspect_tools_telemetry++telemetry+aspect_tools_telemetry_report" - ], - [ - "aspect_rules_py+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "aspect_rules_py+", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@aspect_tools_telemetry+//:extension.bzl%telemetry": { - "general": { - "bzlTransitiveDigest": "aVNc9R1NtH1OtynCXvZ9j7WtILv+X93Qxzmkyn4x++8=", - "usagesDigest": "bLd5DJLF7d7awlSuBaWvY0qEqBLja5XVKriiQmSNEi8=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "aspect_tools_telemetry_report": { - "repoRuleId": "@@aspect_tools_telemetry+//:extension.bzl%tel_repository", - "attributes": { - "deps": { - "aspect_rules_py": "1.6.3", - "aspect_tools_telemetry": "0.2.6" - } - } - } - }, - "recordedRepoMappingEntries": [ - [ - "aspect_tools_telemetry+", - "aspect_bazel_lib", - "aspect_bazel_lib+" - ], - [ - "aspect_tools_telemetry+", - "bazel_skylib", - "bazel_skylib+" - ] - ] - } - }, - "@@pybind11_bazel+//:internal_configure.bzl%internal_configure_extension": { - "general": { - "bzlTransitiveDigest": "YX3gHhSTjjghUwEDKUr1WO9E6h/uiehMZuE9vZup5+g=", - "usagesDigest": "D1r3lfzMuUBFxgG8V6o0bQTLMk3GkaGOaPzw53wrwyw=", - "recordedFileInputs": { - "@@pybind11_bazel+//MODULE.bazel": "45d98c531db5f8430f847b195357dca36665d9c926cbbee77a85105fc6865d27" - }, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "pybind11": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "build_file": "@@pybind11_bazel+//:pybind11-BUILD.bazel", - "strip_prefix": "pybind11-2.12.0", - "urls": [ - "https://github.com/pybind/pybind11/archive/v2.12.0.zip" - ] - } - } - }, - "recordedRepoMappingEntries": [ - [ - "pybind11_bazel+", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@rules_apple+//apple:apple.bzl%provisioning_profile_repository_extension": { - "general": { - "bzlTransitiveDigest": "asHlWWwDeYTUAyZk9zlK+dm0Ezyb97q8UBEfsY8p83k=", - "usagesDigest": "vsJl8Rw5NL+5Ag2wdUDoTeRF/5klkXO8545Iy7U1Q08=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "local_provisioning_profiles": { - "repoRuleId": "@@rules_apple+//apple/internal:local_provisioning_profiles.bzl%provisioning_profile_repository", - "attributes": {} - } - }, - "recordedRepoMappingEntries": [ - [ - "apple_support+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "bazel_tools", - "rules_cc", - "rules_cc+" - ], - [ - "rules_apple+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "rules_apple+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_apple+", - "build_bazel_apple_support", - "apple_support+" - ], - [ - "rules_apple+", - "build_bazel_rules_swift", - "rules_swift+" - ], - [ - "rules_cc+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_cc+", - "cc_compatibility_proxy", - "rules_cc++compatibility_proxy+cc_compatibility_proxy" - ], - [ - "rules_cc+", - "rules_cc", - "rules_cc+" - ], - [ - "rules_cc++compatibility_proxy+cc_compatibility_proxy", - "rules_cc", - "rules_cc+" - ], - [ - "rules_swift+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "rules_swift+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_swift+", - "build_bazel_apple_support", - "apple_support+" - ], - [ - "rules_swift+", - "build_bazel_rules_swift", - "rules_swift+" - ], - [ - "rules_swift+", - "build_bazel_rules_swift_local_config", - "rules_swift++non_module_deps+build_bazel_rules_swift_local_config" - ] - ] - } - }, - "@@rules_apple+//apple:extensions.bzl%non_module_deps": { - "general": { - "bzlTransitiveDigest": "MABz1Esd3zxY4bS7aJoMhMKwBUdwC27e+3XkbPb3QcQ=", - "usagesDigest": "M3VqFpeTCo4qmrNKGZw0dxBHvTYDrfV3cscGzlSAhQ4=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "xctestrunner": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/google/xctestrunner/archive/b7698df3d435b6491b4b4c0f9fc7a63fbed5e3a6.tar.gz" - ], - "strip_prefix": "xctestrunner-b7698df3d435b6491b4b4c0f9fc7a63fbed5e3a6", - "sha256": "ae3a063c985a8633cb7eb566db21656f8db8eb9a0edb8c182312c7f0db53730d" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_apple+", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { - "general": { - "bzlTransitiveDigest": "URQTcVYyatfFO8w2HD1Z1tb8lTBnhHn1wkiu5DYTSow=", - "usagesDigest": "QI2z8ZUR+mqtbwsf2fLqYdJAkPOHdOV+tF2yVAUgRzw=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "com_github_jetbrains_kotlin_git": { - "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:compiler.bzl%kotlin_compiler_git_repository", - "attributes": { - "urls": [ - "https://github.com/JetBrains/kotlin/releases/download/v1.9.23/kotlin-compiler-1.9.23.zip" - ], - "sha256": "93137d3aab9afa9b27cb06a824c2324195c6b6f6179d8a8653f440f5bd58be88" - } - }, - "com_github_jetbrains_kotlin": { - "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:compiler.bzl%kotlin_capabilities_repository", - "attributes": { - "git_repository_name": "com_github_jetbrains_kotlin_git", - "compiler_version": "1.9.23" - } - }, - "com_github_google_ksp": { - "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:ksp.bzl%ksp_compiler_plugin_repository", - "attributes": { - "urls": [ - "https://github.com/google/ksp/releases/download/1.9.23-1.0.20/artifacts.zip" - ], - "sha256": "ee0618755913ef7fd6511288a232e8fad24838b9af6ea73972a76e81053c8c2d", - "strip_version": "1.9.23-1.0.20" - } - }, - "com_github_pinterest_ktlint": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "sha256": "01b2e0ef893383a50dbeb13970fe7fa3be36ca3e83259e01649945b09d736985", - "urls": [ - "https://github.com/pinterest/ktlint/releases/download/1.3.0/ktlint" - ], - "executable": true - } - }, - "rules_android": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "sha256": "cd06d15dd8bb59926e4d65f9003bfc20f9da4b2519985c27e190cddc8b7a7806", - "strip_prefix": "rules_android-0.1.1", - "urls": [ - "https://github.com/bazelbuild/rules_android/archive/v0.1.1.zip" - ] - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_kotlin+", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@rules_oci+//oci:extensions.bzl%oci": { - "general": { - "bzlTransitiveDigest": "PTQgYMhZAtC3X0ttcTBbDaaOBjIGyz0G4vX/ThEUHDY=", - "usagesDigest": "vK9qUa3pyGJik2hb1NI2fKVJrjIszor+wN/44XMkKi0=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "distroless_cc_debian12_linux_amd64": { - "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_pull", - "attributes": { - "www_authenticate_challenges": {}, - "scheme": "https", - "registry": "gcr.io", - "repository": "distroless/cc-debian12", - "identifier": "sha256:eccec5274132c1be0ce5d2c8e6fe41033e64af5e987ccee9007826e4c012069d", - "platform": "linux/amd64", - "target_name": "distroless_cc_debian12_linux_amd64", - "bazel_tags": [] - } - }, - "distroless_cc_debian12": { - "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_alias", - "attributes": { - "target_name": "distroless_cc_debian12", - "www_authenticate_challenges": {}, - "scheme": "https", - "registry": "gcr.io", - "repository": "distroless/cc-debian12", - "identifier": "sha256:eccec5274132c1be0ce5d2c8e6fe41033e64af5e987ccee9007826e4c012069d", - "platforms": { - "@@platforms//cpu:x86_64": "@distroless_cc_debian12_linux_amd64" - }, - "bzlmod_repository": "distroless_cc_debian12", - "reproducible": true - } - }, - "oci_crane_darwin_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "darwin_amd64", - "crane_version": "v0.18.0" - } - }, - "oci_crane_darwin_arm64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "darwin_arm64", - "crane_version": "v0.18.0" - } - }, - "oci_crane_linux_arm64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "linux_arm64", - "crane_version": "v0.18.0" - } - }, - "oci_crane_linux_armv6": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "linux_armv6", - "crane_version": "v0.18.0" - } - }, - "oci_crane_linux_i386": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "linux_i386", - "crane_version": "v0.18.0" - } - }, - "oci_crane_linux_s390x": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "linux_s390x", - "crane_version": "v0.18.0" - } - }, - "oci_crane_linux_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "linux_amd64", - "crane_version": "v0.18.0" - } - }, - "oci_crane_windows_armv6": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "windows_armv6", - "crane_version": "v0.18.0" - } - }, - "oci_crane_windows_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "windows_amd64", - "crane_version": "v0.18.0" - } - }, - "oci_crane_toolchains": { - "repoRuleId": "@@rules_oci+//oci/private:toolchains_repo.bzl%toolchains_repo", - "attributes": { - "toolchain_type": "@rules_oci//oci:crane_toolchain_type", - "toolchain": "@oci_crane_{platform}//:crane_toolchain" - } - }, - "oci_regctl_darwin_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "darwin_amd64" - } - }, - "oci_regctl_darwin_arm64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "darwin_arm64" - } - }, - "oci_regctl_linux_arm64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "linux_arm64" - } - }, - "oci_regctl_linux_s390x": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "linux_s390x" - } - }, - "oci_regctl_linux_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "linux_amd64" - } - }, - "oci_regctl_windows_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "windows_amd64" - } - }, - "oci_regctl_toolchains": { - "repoRuleId": "@@rules_oci+//oci/private:toolchains_repo.bzl%toolchains_repo", - "attributes": { - "toolchain_type": "@rules_oci//oci:regctl_toolchain_type", - "toolchain": "@oci_regctl_{platform}//:regctl_toolchain" - } - } - }, - "moduleExtensionMetadata": { - "explicitRootModuleDirectDeps": [ - "distroless_cc_debian12", - "distroless_cc_debian12_linux_amd64" - ], - "explicitRootModuleDirectDevDeps": [], - "useAllRepos": "NO", - "reproducible": false - }, - "recordedRepoMappingEntries": [ - [ - "aspect_bazel_lib+", - "bazel_tools", - "bazel_tools" - ], - [ - "bazel_features+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_oci+", - "aspect_bazel_lib", - "aspect_bazel_lib+" - ], - [ - "rules_oci+", - "bazel_features", - "bazel_features+" - ], - [ - "rules_oci+", - "bazel_skylib", - "bazel_skylib+" - ] - ] - } - }, - "@@rules_python+//python/extensions:config.bzl%config": { - "general": { - "bzlTransitiveDigest": "0rWbrtB3JZ8y7lycaRGpCqLuh1Pt+DXhVnHBD0vU2JI=", - "usagesDigest": "tIEieEA/gbsjNF3L/Oouyg6UdqGOVxFsPqBjFxkTAKM=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "rules_python_internal": { - "repoRuleId": "@@rules_python+//python/private:internal_config_repo.bzl%internal_config_repo", - "attributes": { - "transition_setting_generators": {}, - "transition_settings": [] - } - }, - "pypi__build": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/e2/03/f3c8ba0a6b6e30d7d18c40faab90807c9bb5e9a1e3b2fe2008af624a9c97/build-1.2.1-py3-none-any.whl", - "sha256": "75e10f767a433d9a86e50d83f418e83efc18ede923ee5ff7df93b6cb0306c5d4", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__click": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl", - "sha256": "ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__colorama": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", - "sha256": "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__importlib_metadata": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/2d/0a/679461c511447ffaf176567d5c496d1de27cbe34a87df6677d7171b2fbd4/importlib_metadata-7.1.0-py3-none-any.whl", - "sha256": "30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__installer": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl", - "sha256": "05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__more_itertools": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/50/e2/8e10e465ee3987bb7c9ab69efb91d867d93959095f4807db102d07995d94/more_itertools-10.2.0-py3-none-any.whl", - "sha256": "686b06abe565edfab151cb8fd385a05651e1fdf8f0a14191e4439283421f8684", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__packaging": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/49/df/1fceb2f8900f8639e278b056416d49134fb8d84c5942ffaa01ad34782422/packaging-24.0-py3-none-any.whl", - "sha256": "2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__pep517": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/25/6e/ca4a5434eb0e502210f591b97537d322546e4833dcb4d470a48c375c5540/pep517-0.13.1-py3-none-any.whl", - "sha256": "31b206f67165b3536dd577c5c3f1518e8fbaf38cbc57efff8369a392feff1721", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__pip": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/8a/6a/19e9fe04fca059ccf770861c7d5721ab4c2aebc539889e97c7977528a53b/pip-24.0-py3-none-any.whl", - "sha256": "ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__pip_tools": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/0d/dc/38f4ce065e92c66f058ea7a368a9c5de4e702272b479c0992059f7693941/pip_tools-7.4.1-py3-none-any.whl", - "sha256": "4c690e5fbae2f21e87843e89c26191f0d9454f362d8acdbd695716493ec8b3a9", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__pyproject_hooks": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/ae/f3/431b9d5fe7d14af7a32340792ef43b8a714e7726f1d7b69cc4e8e7a3f1d7/pyproject_hooks-1.1.0-py3-none-any.whl", - "sha256": "7ceeefe9aec63a1064c18d939bdc3adf2d8aa1988a510afec15151578b232aa2", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__setuptools": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/90/99/158ad0609729111163fc1f674a5a42f2605371a4cf036d0441070e2f7455/setuptools-78.1.1-py3-none-any.whl", - "sha256": "c3a9c4211ff4c309edb8b8c4f1cbfa7ae324c4ba9f91ff254e3d305b9fd54561", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__tomli": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", - "sha256": "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__wheel": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/7d/cd/d7460c9a869b16c3dd4e1e403cce337df165368c71d6af229a74699622ce/wheel-0.43.0-py3-none-any.whl", - "sha256": "55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__zipp": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/da/55/a03fd7240714916507e1fcf7ae355bd9d9ed2e6db492595f1a67f61681be/zipp-3.18.2-py3-none-any.whl", - "sha256": "dce197b859eb796242b0622af1b8beb0a722d52aa2f57133ead08edd5bf5374e", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_python+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_python+", - "pypi__build", - "rules_python++config+pypi__build" - ], - [ - "rules_python+", - "pypi__click", - "rules_python++config+pypi__click" - ], - [ - "rules_python+", - "pypi__colorama", - "rules_python++config+pypi__colorama" - ], - [ - "rules_python+", - "pypi__importlib_metadata", - "rules_python++config+pypi__importlib_metadata" - ], - [ - "rules_python+", - "pypi__installer", - "rules_python++config+pypi__installer" - ], - [ - "rules_python+", - "pypi__more_itertools", - "rules_python++config+pypi__more_itertools" - ], - [ - "rules_python+", - "pypi__packaging", - "rules_python++config+pypi__packaging" - ], - [ - "rules_python+", - "pypi__pep517", - "rules_python++config+pypi__pep517" - ], - [ - "rules_python+", - "pypi__pip", - "rules_python++config+pypi__pip" - ], - [ - "rules_python+", - "pypi__pip_tools", - "rules_python++config+pypi__pip_tools" - ], - [ - "rules_python+", - "pypi__pyproject_hooks", - "rules_python++config+pypi__pyproject_hooks" - ], - [ - "rules_python+", - "pypi__setuptools", - "rules_python++config+pypi__setuptools" - ], - [ - "rules_python+", - "pypi__tomli", - "rules_python++config+pypi__tomli" - ], - [ - "rules_python+", - "pypi__wheel", - "rules_python++config+pypi__wheel" - ], - [ - "rules_python+", - "pypi__zipp", - "rules_python++config+pypi__zipp" - ] - ] - } - }, - "@@rules_python+//python/uv:uv.bzl%uv": { - "general": { - "bzlTransitiveDigest": "bb4QpQxvjqUS7bhlSKPbiLJgYNlcA0T5b9vYziRFobM=", - "usagesDigest": "gx0mA10eprg2o8cg9R+m/jpqCuVjnTnNzD17TPQdFu0=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "uv_0_6_5_aarch64_apple_darwin": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", - "attributes": { - "version": "0.6.5", - "platform": "aarch64-apple-darwin", - "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-aarch64-apple-darwin.tar.gz" - ], - "sha256": "4952c6275778fbc7e5cf6b5739ffd944c47063134d6ea43a7037b0a028ef771e" - } - }, - "uv_0_6_5_aarch64_unknown_linux_gnu": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", - "attributes": { - "version": "0.6.5", - "platform": "aarch64-unknown-linux-gnu", - "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-aarch64-unknown-linux-gnu.tar.gz" - ], - "sha256": "ebf0707ab949b948ad14250ed93f968fec954e20d12ce6bc9fce74b0ce7e4900" - } - }, - "uv_0_6_5_powerpc64_unknown_linux_gnu": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", - "attributes": { - "version": "0.6.5", - "platform": "powerpc64-unknown-linux-gnu", - "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-powerpc64-unknown-linux-gnu.tar.gz" - ], - "sha256": "1f8fd8dde91258380dbcd4b9fc65f15a7dc3af6cfffefb9a4f99d32c4c67436d" - } - }, - "uv_0_6_5_powerpc64le_unknown_linux_gnu": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", - "attributes": { - "version": "0.6.5", - "platform": "powerpc64le-unknown-linux-gnu", - "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-powerpc64le-unknown-linux-gnu.tar.gz" - ], - "sha256": "d2b6e19ca7ecd8d7e0d1ddced965aa91a3c822d07a7f7657743945245c1f627a" - } - }, - "uv_0_6_5_s390x_unknown_linux_gnu": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", - "attributes": { - "version": "0.6.5", - "platform": "s390x-unknown-linux-gnu", - "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-s390x-unknown-linux-gnu.tar.gz" - ], - "sha256": "43a3e39f19fd5d098bb8c896b6c0f99a146253c271d78409cf31c2f55f2763d0" - } - }, - "uv_0_6_5_x86_64_apple_darwin": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", - "attributes": { - "version": "0.6.5", - "platform": "x86_64-apple-darwin", - "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-x86_64-apple-darwin.tar.gz" - ], - "sha256": "d336d862694e1c36afea55eb15f5df02aa77c04a168b781df515d641e23285f0" - } - }, - "uv_0_6_5_x86_64_pc_windows_msvc": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", - "attributes": { - "version": "0.6.5", - "platform": "x86_64-pc-windows-msvc", - "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-x86_64-pc-windows-msvc.zip" - ], - "sha256": "a3e9f0913189c38b804bf6bb3d348af305a8ceabc690be92b6e111245a821640" - } - }, - "uv_0_6_5_x86_64_unknown_linux_gnu": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", - "attributes": { - "version": "0.6.5", - "platform": "x86_64-unknown-linux-gnu", - "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-x86_64-unknown-linux-gnu.tar.gz" - ], - "sha256": "8fc9895719a1291ecd193cb86f9282ff3649cef797d29eacc74c4f573aab1e2f" - } - }, - "uv": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_toolchains_repo.bzl%uv_toolchains_repo", - "attributes": { - "toolchain_type": "'@@rules_python+//python/uv:uv_toolchain_type'", - "toolchain_names": [ - "0_6_5_aarch64_apple_darwin", - "0_6_5_aarch64_unknown_linux_gnu", - "0_6_5_powerpc64_unknown_linux_gnu", - "0_6_5_powerpc64le_unknown_linux_gnu", - "0_6_5_s390x_unknown_linux_gnu", - "0_6_5_x86_64_apple_darwin", - "0_6_5_x86_64_pc_windows_msvc", - "0_6_5_x86_64_unknown_linux_gnu" - ], - "toolchain_implementations": { - "0_6_5_aarch64_apple_darwin": "@uv_0_6_5_aarch64_apple_darwin//:uv_toolchain", - "0_6_5_aarch64_unknown_linux_gnu": "@uv_0_6_5_aarch64_unknown_linux_gnu//:uv_toolchain", - "0_6_5_powerpc64_unknown_linux_gnu": "@uv_0_6_5_powerpc64_unknown_linux_gnu//:uv_toolchain", - "0_6_5_powerpc64le_unknown_linux_gnu": "@uv_0_6_5_powerpc64le_unknown_linux_gnu//:uv_toolchain", - "0_6_5_s390x_unknown_linux_gnu": "@uv_0_6_5_s390x_unknown_linux_gnu//:uv_toolchain", - "0_6_5_x86_64_apple_darwin": "@uv_0_6_5_x86_64_apple_darwin//:uv_toolchain", - "0_6_5_x86_64_pc_windows_msvc": "@uv_0_6_5_x86_64_pc_windows_msvc//:uv_toolchain", - "0_6_5_x86_64_unknown_linux_gnu": "@uv_0_6_5_x86_64_unknown_linux_gnu//:uv_toolchain" - }, - "toolchain_compatible_with": { - "0_6_5_aarch64_apple_darwin": [ - "'@@platforms//os:macos'", - "'@@platforms//cpu:aarch64'" - ], - "0_6_5_aarch64_unknown_linux_gnu": [ - "'@@platforms//os:linux'", - "'@@platforms//cpu:aarch64'" - ], - "0_6_5_powerpc64_unknown_linux_gnu": [ - "'@@platforms//os:linux'", - "'@@platforms//cpu:ppc'" - ], - "0_6_5_powerpc64le_unknown_linux_gnu": [ - "'@@platforms//os:linux'", - "'@@platforms//cpu:ppc64le'" - ], - "0_6_5_s390x_unknown_linux_gnu": [ - "'@@platforms//os:linux'", - "'@@platforms//cpu:s390x'" - ], - "0_6_5_x86_64_apple_darwin": [ - "'@@platforms//os:macos'", - "'@@platforms//cpu:x86_64'" - ], - "0_6_5_x86_64_pc_windows_msvc": [ - "'@@platforms//os:windows'", - "'@@platforms//cpu:x86_64'" - ], - "0_6_5_x86_64_unknown_linux_gnu": [ - "'@@platforms//os:linux'", - "'@@platforms//cpu:x86_64'" - ] - }, - "toolchain_target_settings": {} - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_python+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_python+", - "platforms", - "platforms" - ] - ] - } - }, - "@@rules_rust+//crate_universe:extensions.bzl%crate": { - "general": { - "bzlTransitiveDigest": "NvjljZ+5m2nc1Qpf7ekjzs+FJrjYYY9DwXvwuXnippI=", - "usagesDigest": "mup6mgSFatk34Q0znMZPpIEOFZyk9v3xf6jRIXEFTxc=", - "recordedFileInputs": { - "@@//zml/tokenizer/hftokenizers/Cargo.lock": "7fb683b6a36181a92d7564c8552b4fc5bad00229a41d520a4536c4e577c2b888", - "@@//zml/tokenizer/hftokenizers/Cargo.toml": "e46e67c2ae6792917dc1259d6ba5db389becc8bafa9fa1911bb3b6cec6017620" - }, - "recordedDirentsInputs": {}, - "envVariables": { - "CARGO_BAZEL_DEBUG": null, - "CARGO_BAZEL_GENERATOR_SHA256": null, - "CARGO_BAZEL_GENERATOR_URL": null, - "CARGO_BAZEL_ISOLATED": null, - "CARGO_BAZEL_REPIN": null, - "CARGO_BAZEL_REPIN_ONLY": null, - "CARGO_BAZEL_TIMEOUT": null, - "REPIN": null - }, - "generatedRepoSpecs": { - "crates": { - "repoRuleId": "@@rules_rust+//crate_universe:extensions.bzl%_generate_repo", - "attributes": { - "contents": { - "BUILD.bazel": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files(\n [\n \"cargo-bazel.json\",\n \"crates.bzl\",\n \"defs.bzl\",\n ] + glob(\n allow_empty = True,\n include = [\"*.bazel\"],\n ),\n)\n\nfilegroup(\n name = \"srcs\",\n srcs = glob(\n allow_empty = True,\n include = [\n \"*.bazel\",\n \"*.bzl\",\n ],\n ),\n)\n\n# Workspace Member Dependencies\nalias(\n name = \"tokenizers-0.22.2\",\n actual = \"@crates__tokenizers-0.22.2//:tokenizers\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"tokenizers\",\n actual = \"@crates__tokenizers-0.22.2//:tokenizers\",\n tags = [\"manual\"],\n)\n", - "alias_rules.bzl": "\"\"\"Alias that transitions its target to `compilation_mode=opt`. Use `transition_alias=\"opt\"` to enable.\"\"\"\n\nload(\"@rules_cc//cc:defs.bzl\", \"CcInfo\")\nload(\"@rules_rust//rust:rust_common.bzl\", \"COMMON_PROVIDERS\")\n\ndef _transition_alias_impl(ctx):\n # `ctx.attr.actual` is a list of 1 item due to the transition\n providers = [ctx.attr.actual[0][provider] for provider in COMMON_PROVIDERS]\n if CcInfo in ctx.attr.actual[0]:\n providers.append(ctx.attr.actual[0][CcInfo])\n return providers\n\ndef _change_compilation_mode(compilation_mode):\n def _change_compilation_mode_impl(_settings, _attr):\n return {\n \"//command_line_option:compilation_mode\": compilation_mode,\n }\n\n return transition(\n implementation = _change_compilation_mode_impl,\n inputs = [],\n outputs = [\n \"//command_line_option:compilation_mode\",\n ],\n )\n\ndef _transition_alias_rule(compilation_mode):\n return rule(\n implementation = _transition_alias_impl,\n provides = COMMON_PROVIDERS,\n attrs = {\n \"actual\": attr.label(\n mandatory = True,\n doc = \"`rust_library()` target to transition to `compilation_mode=opt`.\",\n providers = COMMON_PROVIDERS,\n cfg = _change_compilation_mode(compilation_mode),\n ),\n \"_allowlist_function_transition\": attr.label(\n default = \"@bazel_tools//tools/allowlists/function_transition_allowlist\",\n ),\n },\n doc = \"Transitions a Rust library crate to the `compilation_mode=opt`.\",\n )\n\ntransition_alias_dbg = _transition_alias_rule(\"dbg\")\ntransition_alias_fastbuild = _transition_alias_rule(\"fastbuild\")\ntransition_alias_opt = _transition_alias_rule(\"opt\")\n", - "defs.bzl": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\"\"\"\n# `crates_repository` API\n\n- [aliases](#aliases)\n- [crate_deps](#crate_deps)\n- [all_crate_deps](#all_crate_deps)\n- [crate_repositories](#crate_repositories)\n\n\"\"\"\n\nload(\"@bazel_tools//tools/build_defs/repo:git.bzl\", \"new_git_repository\")\nload(\"@bazel_tools//tools/build_defs/repo:http.bzl\", \"http_archive\")\nload(\"@bazel_tools//tools/build_defs/repo:utils.bzl\", \"maybe\")\nload(\"@bazel_skylib//lib:selects.bzl\", \"selects\")\nload(\"@rules_rust//crate_universe/private:local_crate_mirror.bzl\", \"local_crate_mirror\")\n\n###############################################################################\n# MACROS API\n###############################################################################\n\n# An identifier that represent common dependencies (unconditional).\n_COMMON_CONDITION = \"\"\n\ndef _flatten_dependency_maps(all_dependency_maps):\n \"\"\"Flatten a list of dependency maps into one dictionary.\n\n Dependency maps have the following structure:\n\n ```python\n DEPENDENCIES_MAP = {\n # The first key in the map is a Bazel package\n # name of the workspace this file is defined in.\n \"workspace_member_package\": {\n\n # Not all dependencies are supported for all platforms.\n # the condition key is the condition required to be true\n # on the host platform.\n \"condition\": {\n\n # An alias to a crate target. # The label of the crate target the\n # Aliases are only crate names. # package name refers to.\n \"package_name\": \"@full//:label\",\n }\n }\n }\n ```\n\n Args:\n all_dependency_maps (list): A list of dicts as described above\n\n Returns:\n dict: A dictionary as described above\n \"\"\"\n dependencies = {}\n\n for workspace_deps_map in all_dependency_maps:\n for pkg_name, conditional_deps_map in workspace_deps_map.items():\n if pkg_name not in dependencies:\n non_frozen_map = dict()\n for key, values in conditional_deps_map.items():\n non_frozen_map.update({key: dict(values.items())})\n dependencies.setdefault(pkg_name, non_frozen_map)\n continue\n\n for condition, deps_map in conditional_deps_map.items():\n # If the condition has not been recorded, do so and continue\n if condition not in dependencies[pkg_name]:\n dependencies[pkg_name].setdefault(condition, dict(deps_map.items()))\n continue\n\n # Alert on any miss-matched dependencies\n inconsistent_entries = []\n for crate_name, crate_label in deps_map.items():\n existing = dependencies[pkg_name][condition].get(crate_name)\n if existing and existing != crate_label:\n inconsistent_entries.append((crate_name, existing, crate_label))\n dependencies[pkg_name][condition].update({crate_name: crate_label})\n\n return dependencies\n\ndef crate_deps(deps, package_name = None):\n \"\"\"Finds the fully qualified label of the requested crates for the package where this macro is called.\n\n Args:\n deps (list): The desired list of crate targets.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()`.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if not deps:\n return []\n\n if package_name == None:\n package_name = native.package_name()\n\n # Join both sets of dependencies\n dependencies = _flatten_dependency_maps([\n _NORMAL_DEPENDENCIES,\n _NORMAL_DEV_DEPENDENCIES,\n _PROC_MACRO_DEPENDENCIES,\n _PROC_MACRO_DEV_DEPENDENCIES,\n _BUILD_DEPENDENCIES,\n _BUILD_PROC_MACRO_DEPENDENCIES,\n ]).pop(package_name, {})\n\n # Combine all conditional packages so we can easily index over a flat list\n # TODO: Perhaps this should actually return select statements and maintain\n # the conditionals of the dependencies\n flat_deps = {}\n for deps_set in dependencies.values():\n for crate_name, crate_label in deps_set.items():\n flat_deps.update({crate_name: crate_label})\n\n missing_crates = []\n crate_targets = []\n for crate_target in deps:\n if crate_target not in flat_deps:\n missing_crates.append(crate_target)\n else:\n crate_targets.append(flat_deps[crate_target])\n\n if missing_crates:\n fail(\"Could not find crates `{}` among dependencies of `{}`. Available dependencies were `{}`\".format(\n missing_crates,\n package_name,\n dependencies,\n ))\n\n return crate_targets\n\ndef all_crate_deps(\n normal = False, \n normal_dev = False, \n proc_macro = False, \n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Finds the fully qualified label of all requested direct crate dependencies \\\n for the package where this macro is called.\n\n If no parameters are set, all normal dependencies are returned. Setting any one flag will\n otherwise impact the contents of the returned list.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_dependency_maps = []\n if normal:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n if normal_dev:\n all_dependency_maps.append(_NORMAL_DEV_DEPENDENCIES)\n if proc_macro:\n all_dependency_maps.append(_PROC_MACRO_DEPENDENCIES)\n if proc_macro_dev:\n all_dependency_maps.append(_PROC_MACRO_DEV_DEPENDENCIES)\n if build:\n all_dependency_maps.append(_BUILD_DEPENDENCIES)\n if build_proc_macro:\n all_dependency_maps.append(_BUILD_PROC_MACRO_DEPENDENCIES)\n\n # Default to always using normal dependencies\n if not all_dependency_maps:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n\n dependencies = _flatten_dependency_maps(all_dependency_maps).pop(package_name, None)\n\n if not dependencies:\n if dependencies == None:\n fail(\"Tried to get all_crate_deps for package \" + package_name + \" but that package had no Cargo.toml file\")\n else:\n return []\n\n crate_deps = list(dependencies.pop(_COMMON_CONDITION, {}).values())\n for condition, deps in dependencies.items():\n crate_deps += selects.with_or({\n tuple(_CONDITIONS[condition]): deps.values(),\n \"//conditions:default\": [],\n })\n\n return crate_deps\n\ndef aliases(\n normal = False,\n normal_dev = False,\n proc_macro = False,\n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Produces a map of Crate alias names to their original label\n\n If no dependency kinds are specified, `normal` and `proc_macro` are used by default.\n Setting any one flag will otherwise determine the contents of the returned dict.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n dict: The aliases of all associated packages\n \"\"\"\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_aliases_maps = []\n if normal:\n all_aliases_maps.append(_NORMAL_ALIASES)\n if normal_dev:\n all_aliases_maps.append(_NORMAL_DEV_ALIASES)\n if proc_macro:\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n if proc_macro_dev:\n all_aliases_maps.append(_PROC_MACRO_DEV_ALIASES)\n if build:\n all_aliases_maps.append(_BUILD_ALIASES)\n if build_proc_macro:\n all_aliases_maps.append(_BUILD_PROC_MACRO_ALIASES)\n\n # Default to always using normal aliases\n if not all_aliases_maps:\n all_aliases_maps.append(_NORMAL_ALIASES)\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n\n aliases = _flatten_dependency_maps(all_aliases_maps).pop(package_name, None)\n\n if not aliases:\n return dict()\n\n common_items = aliases.pop(_COMMON_CONDITION, {}).items()\n\n # If there are only common items in the dictionary, immediately return them\n if not len(aliases.keys()) == 1:\n return dict(common_items)\n\n # Build a single select statement where each conditional has accounted for the\n # common set of aliases.\n crate_aliases = {\"//conditions:default\": dict(common_items)}\n for condition, deps in aliases.items():\n condition_triples = _CONDITIONS[condition]\n for triple in condition_triples:\n if triple in crate_aliases:\n crate_aliases[triple].update(deps)\n else:\n crate_aliases.update({triple: dict(deps.items() + common_items)})\n\n return select(crate_aliases)\n\n###############################################################################\n# WORKSPACE MEMBER DEPS AND ALIASES\n###############################################################################\n\n_NORMAL_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n _COMMON_CONDITION: {\n \"tokenizers\": Label(\"@crates//:tokenizers-0.22.2\"),\n },\n },\n}\n\n\n_NORMAL_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n _COMMON_CONDITION: {\n },\n },\n}\n\n\n_NORMAL_DEV_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_NORMAL_DEV_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEV_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEV_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_CONDITIONS = {\n \"aarch64-apple-darwin\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\"],\n \"aarch64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\"],\n \"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\",\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\": [],\n \"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\": [],\n \"cfg(any())\": [],\n \"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\": [],\n \"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\": [],\n \"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\": [],\n \"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\",\"@rules_rust//rust/platform:x86_64-apple-darwin\"],\n \"cfg(not(all(target_arch = \\\"arm\\\", target_os = \\\"none\\\")))\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\",\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\",\"@rules_rust//rust/platform:x86_64-apple-darwin\",\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"cfg(target_os = \\\"netbsd\\\")\": [],\n \"cfg(target_os = \\\"solaris\\\")\": [],\n \"cfg(target_os = \\\"vxworks\\\")\": [],\n \"cfg(windows)\": [],\n \"x86_64-apple-darwin\": [\"@rules_rust//rust/platform:x86_64-apple-darwin\"],\n \"x86_64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n}\n\n###############################################################################\n\ndef crate_repositories():\n \"\"\"A macro for defining repositories for all generated crates.\n\n Returns:\n A list of repos visible to the module through the module extension.\n \"\"\"\n maybe(\n http_archive,\n name = \"crates__ahash-0.8.12\",\n sha256 = \"5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ahash/0.8.12/download\"],\n strip_prefix = \"ahash-0.8.12\",\n build_file = Label(\"@crates//crates:BUILD.ahash-0.8.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__aho-corasick-1.1.3\",\n sha256 = \"8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/aho-corasick/1.1.3/download\"],\n strip_prefix = \"aho-corasick-1.1.3\",\n build_file = Label(\"@crates//crates:BUILD.aho-corasick-1.1.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__base64-0.13.1\",\n sha256 = \"9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/base64/0.13.1/download\"],\n strip_prefix = \"base64-0.13.1\",\n build_file = Label(\"@crates//crates:BUILD.base64-0.13.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__bitflags-2.9.1\",\n sha256 = \"1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/bitflags/2.9.1/download\"],\n strip_prefix = \"bitflags-2.9.1\",\n build_file = Label(\"@crates//crates:BUILD.bitflags-2.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__byteorder-1.5.0\",\n sha256 = \"1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/byteorder/1.5.0/download\"],\n strip_prefix = \"byteorder-1.5.0\",\n build_file = Label(\"@crates//crates:BUILD.byteorder-1.5.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__castaway-0.2.4\",\n sha256 = \"dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/castaway/0.2.4/download\"],\n strip_prefix = \"castaway-0.2.4\",\n build_file = Label(\"@crates//crates:BUILD.castaway-0.2.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__cc-1.2.33\",\n sha256 = \"3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/cc/1.2.33/download\"],\n strip_prefix = \"cc-1.2.33\",\n build_file = Label(\"@crates//crates:BUILD.cc-1.2.33.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__cfg-if-1.0.0\",\n sha256 = \"baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/cfg-if/1.0.0/download\"],\n strip_prefix = \"cfg-if-1.0.0\",\n build_file = Label(\"@crates//crates:BUILD.cfg-if-1.0.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__compact_str-0.9.0\",\n sha256 = \"3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/compact_str/0.9.0/download\"],\n strip_prefix = \"compact_str-0.9.0\",\n build_file = Label(\"@crates//crates:BUILD.compact_str-0.9.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-deque-0.8.6\",\n sha256 = \"9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-deque/0.8.6/download\"],\n strip_prefix = \"crossbeam-deque-0.8.6\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-deque-0.8.6.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-epoch-0.9.18\",\n sha256 = \"5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-epoch/0.9.18/download\"],\n strip_prefix = \"crossbeam-epoch-0.9.18\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-epoch-0.9.18.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-utils-0.8.21\",\n sha256 = \"d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-utils/0.8.21/download\"],\n strip_prefix = \"crossbeam-utils-0.8.21\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-utils-0.8.21.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling-0.20.10\",\n sha256 = \"6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling/0.20.10/download\"],\n strip_prefix = \"darling-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling_core-0.20.10\",\n sha256 = \"95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling_core/0.20.10/download\"],\n strip_prefix = \"darling_core-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling_core-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling_macro-0.20.10\",\n sha256 = \"d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling_macro/0.20.10/download\"],\n strip_prefix = \"darling_macro-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling_macro-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__dary_heap-0.3.7\",\n sha256 = \"04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/dary_heap/0.3.7/download\"],\n strip_prefix = \"dary_heap-0.3.7\",\n build_file = Label(\"@crates//crates:BUILD.dary_heap-0.3.7.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder-0.20.2\",\n sha256 = \"507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder/0.20.2/download\"],\n strip_prefix = \"derive_builder-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder_core-0.20.2\",\n sha256 = \"2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder_core/0.20.2/download\"],\n strip_prefix = \"derive_builder_core-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder_core-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder_macro-0.20.2\",\n sha256 = \"ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder_macro/0.20.2/download\"],\n strip_prefix = \"derive_builder_macro-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder_macro-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__either-1.13.0\",\n sha256 = \"60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/either/1.13.0/download\"],\n strip_prefix = \"either-1.13.0\",\n build_file = Label(\"@crates//crates:BUILD.either-1.13.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__esaxx-rs-0.1.10\",\n sha256 = \"d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/esaxx-rs/0.1.10/download\"],\n strip_prefix = \"esaxx-rs-0.1.10\",\n build_file = Label(\"@crates//crates:BUILD.esaxx-rs-0.1.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__fnv-1.0.7\",\n sha256 = \"3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/fnv/1.0.7/download\"],\n strip_prefix = \"fnv-1.0.7\",\n build_file = Label(\"@crates//crates:BUILD.fnv-1.0.7.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__getrandom-0.3.3\",\n sha256 = \"26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/getrandom/0.3.3/download\"],\n strip_prefix = \"getrandom-0.3.3\",\n build_file = Label(\"@crates//crates:BUILD.getrandom-0.3.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ident_case-1.0.1\",\n sha256 = \"b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ident_case/1.0.1/download\"],\n strip_prefix = \"ident_case-1.0.1\",\n build_file = Label(\"@crates//crates:BUILD.ident_case-1.0.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__itertools-0.14.0\",\n sha256 = \"2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/itertools/0.14.0/download\"],\n strip_prefix = \"itertools-0.14.0\",\n build_file = Label(\"@crates//crates:BUILD.itertools-0.14.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__itoa-1.0.14\",\n sha256 = \"d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/itoa/1.0.14/download\"],\n strip_prefix = \"itoa-1.0.14\",\n build_file = Label(\"@crates//crates:BUILD.itoa-1.0.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__libc-0.2.169\",\n sha256 = \"b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/libc/0.2.169/download\"],\n strip_prefix = \"libc-0.2.169\",\n build_file = Label(\"@crates//crates:BUILD.libc-0.2.169.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__log-0.4.22\",\n sha256 = \"a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/log/0.4.22/download\"],\n strip_prefix = \"log-0.4.22\",\n build_file = Label(\"@crates//crates:BUILD.log-0.4.22.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__macro_rules_attribute-0.2.0\",\n sha256 = \"8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/macro_rules_attribute/0.2.0/download\"],\n strip_prefix = \"macro_rules_attribute-0.2.0\",\n build_file = Label(\"@crates//crates:BUILD.macro_rules_attribute-0.2.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__macro_rules_attribute-proc_macro-0.2.0\",\n sha256 = \"b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/macro_rules_attribute-proc_macro/0.2.0/download\"],\n strip_prefix = \"macro_rules_attribute-proc_macro-0.2.0\",\n build_file = Label(\"@crates//crates:BUILD.macro_rules_attribute-proc_macro-0.2.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__memchr-2.7.4\",\n sha256 = \"78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/memchr/2.7.4/download\"],\n strip_prefix = \"memchr-2.7.4\",\n build_file = Label(\"@crates//crates:BUILD.memchr-2.7.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__minimal-lexical-0.2.1\",\n sha256 = \"68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/minimal-lexical/0.2.1/download\"],\n strip_prefix = \"minimal-lexical-0.2.1\",\n build_file = Label(\"@crates//crates:BUILD.minimal-lexical-0.2.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__monostate-0.1.13\",\n sha256 = \"0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/monostate/0.1.13/download\"],\n strip_prefix = \"monostate-0.1.13\",\n build_file = Label(\"@crates//crates:BUILD.monostate-0.1.13.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__monostate-impl-0.1.13\",\n sha256 = \"a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/monostate-impl/0.1.13/download\"],\n strip_prefix = \"monostate-impl-0.1.13\",\n build_file = Label(\"@crates//crates:BUILD.monostate-impl-0.1.13.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__nom-7.1.3\",\n sha256 = \"d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/nom/7.1.3/download\"],\n strip_prefix = \"nom-7.1.3\",\n build_file = Label(\"@crates//crates:BUILD.nom-7.1.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__once_cell-1.20.2\",\n sha256 = \"1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/once_cell/1.20.2/download\"],\n strip_prefix = \"once_cell-1.20.2\",\n build_file = Label(\"@crates//crates:BUILD.once_cell-1.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__onig-6.5.1\",\n sha256 = \"336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/onig/6.5.1/download\"],\n strip_prefix = \"onig-6.5.1\",\n build_file = Label(\"@crates//crates:BUILD.onig-6.5.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__onig_sys-69.9.1\",\n sha256 = \"c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/onig_sys/69.9.1/download\"],\n strip_prefix = \"onig_sys-69.9.1\",\n build_file = Label(\"@crates//crates:BUILD.onig_sys-69.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__paste-1.0.15\",\n sha256 = \"57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/paste/1.0.15/download\"],\n strip_prefix = \"paste-1.0.15\",\n build_file = Label(\"@crates//crates:BUILD.paste-1.0.15.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__pkg-config-0.3.32\",\n sha256 = \"7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/pkg-config/0.3.32/download\"],\n strip_prefix = \"pkg-config-0.3.32\",\n build_file = Label(\"@crates//crates:BUILD.pkg-config-0.3.32.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ppv-lite86-0.2.20\",\n sha256 = \"77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ppv-lite86/0.2.20/download\"],\n strip_prefix = \"ppv-lite86-0.2.20\",\n build_file = Label(\"@crates//crates:BUILD.ppv-lite86-0.2.20.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__proc-macro2-1.0.92\",\n sha256 = \"37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/proc-macro2/1.0.92/download\"],\n strip_prefix = \"proc-macro2-1.0.92\",\n build_file = Label(\"@crates//crates:BUILD.proc-macro2-1.0.92.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__quote-1.0.37\",\n sha256 = \"b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/quote/1.0.37/download\"],\n strip_prefix = \"quote-1.0.37\",\n build_file = Label(\"@crates//crates:BUILD.quote-1.0.37.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__r-efi-5.3.0\",\n sha256 = \"69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/r-efi/5.3.0/download\"],\n strip_prefix = \"r-efi-5.3.0\",\n build_file = Label(\"@crates//crates:BUILD.r-efi-5.3.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand-0.9.1\",\n sha256 = \"9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand/0.9.1/download\"],\n strip_prefix = \"rand-0.9.1\",\n build_file = Label(\"@crates//crates:BUILD.rand-0.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand_chacha-0.9.0\",\n sha256 = \"d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand_chacha/0.9.0/download\"],\n strip_prefix = \"rand_chacha-0.9.0\",\n build_file = Label(\"@crates//crates:BUILD.rand_chacha-0.9.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand_core-0.9.3\",\n sha256 = \"99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand_core/0.9.3/download\"],\n strip_prefix = \"rand_core-0.9.3\",\n build_file = Label(\"@crates//crates:BUILD.rand_core-0.9.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-1.10.0\",\n sha256 = \"b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon/1.10.0/download\"],\n strip_prefix = \"rayon-1.10.0\",\n build_file = Label(\"@crates//crates:BUILD.rayon-1.10.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-cond-0.4.0\",\n sha256 = \"2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon-cond/0.4.0/download\"],\n strip_prefix = \"rayon-cond-0.4.0\",\n build_file = Label(\"@crates//crates:BUILD.rayon-cond-0.4.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-core-1.12.1\",\n sha256 = \"1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon-core/1.12.1/download\"],\n strip_prefix = \"rayon-core-1.12.1\",\n build_file = Label(\"@crates//crates:BUILD.rayon-core-1.12.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-1.11.1\",\n sha256 = \"b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex/1.11.1/download\"],\n strip_prefix = \"regex-1.11.1\",\n build_file = Label(\"@crates//crates:BUILD.regex-1.11.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-automata-0.4.9\",\n sha256 = \"809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-automata/0.4.9/download\"],\n strip_prefix = \"regex-automata-0.4.9\",\n build_file = Label(\"@crates//crates:BUILD.regex-automata-0.4.9.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-syntax-0.8.5\",\n sha256 = \"2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-syntax/0.8.5/download\"],\n strip_prefix = \"regex-syntax-0.8.5\",\n build_file = Label(\"@crates//crates:BUILD.regex-syntax-0.8.5.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rustversion-1.0.21\",\n sha256 = \"8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rustversion/1.0.21/download\"],\n strip_prefix = \"rustversion-1.0.21\",\n build_file = Label(\"@crates//crates:BUILD.rustversion-1.0.21.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ryu-1.0.18\",\n sha256 = \"f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ryu/1.0.18/download\"],\n strip_prefix = \"ryu-1.0.18\",\n build_file = Label(\"@crates//crates:BUILD.ryu-1.0.18.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde-1.0.216\",\n sha256 = \"0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde/1.0.216/download\"],\n strip_prefix = \"serde-1.0.216\",\n build_file = Label(\"@crates//crates:BUILD.serde-1.0.216.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde_derive-1.0.216\",\n sha256 = \"46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde_derive/1.0.216/download\"],\n strip_prefix = \"serde_derive-1.0.216\",\n build_file = Label(\"@crates//crates:BUILD.serde_derive-1.0.216.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde_json-1.0.134\",\n sha256 = \"d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde_json/1.0.134/download\"],\n strip_prefix = \"serde_json-1.0.134\",\n build_file = Label(\"@crates//crates:BUILD.serde_json-1.0.134.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__shlex-1.3.0\",\n sha256 = \"0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/shlex/1.3.0/download\"],\n strip_prefix = \"shlex-1.3.0\",\n build_file = Label(\"@crates//crates:BUILD.shlex-1.3.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__smallvec-1.13.2\",\n sha256 = \"3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/smallvec/1.13.2/download\"],\n strip_prefix = \"smallvec-1.13.2\",\n build_file = Label(\"@crates//crates:BUILD.smallvec-1.13.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__spm_precompiled-0.1.4\",\n sha256 = \"5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/spm_precompiled/0.1.4/download\"],\n strip_prefix = \"spm_precompiled-0.1.4\",\n build_file = Label(\"@crates//crates:BUILD.spm_precompiled-0.1.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__static_assertions-1.1.0\",\n sha256 = \"a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/static_assertions/1.1.0/download\"],\n strip_prefix = \"static_assertions-1.1.0\",\n build_file = Label(\"@crates//crates:BUILD.static_assertions-1.1.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__strsim-0.11.1\",\n sha256 = \"7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/strsim/0.11.1/download\"],\n strip_prefix = \"strsim-0.11.1\",\n build_file = Label(\"@crates//crates:BUILD.strsim-0.11.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__syn-2.0.91\",\n sha256 = \"d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/syn/2.0.91/download\"],\n strip_prefix = \"syn-2.0.91\",\n build_file = Label(\"@crates//crates:BUILD.syn-2.0.91.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__thiserror-2.0.12\",\n sha256 = \"567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/thiserror/2.0.12/download\"],\n strip_prefix = \"thiserror-2.0.12\",\n build_file = Label(\"@crates//crates:BUILD.thiserror-2.0.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__thiserror-impl-2.0.12\",\n sha256 = \"7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/thiserror-impl/2.0.12/download\"],\n strip_prefix = \"thiserror-impl-2.0.12\",\n build_file = Label(\"@crates//crates:BUILD.thiserror-impl-2.0.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__tokenizers-0.22.2\",\n sha256 = \"b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/tokenizers/0.22.2/download\"],\n strip_prefix = \"tokenizers-0.22.2\",\n build_file = Label(\"@crates//crates:BUILD.tokenizers-0.22.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-ident-1.0.14\",\n sha256 = \"adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-ident/1.0.14/download\"],\n strip_prefix = \"unicode-ident-1.0.14\",\n build_file = Label(\"@crates//crates:BUILD.unicode-ident-1.0.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-normalization-alignments-0.1.12\",\n sha256 = \"43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-normalization-alignments/0.1.12/download\"],\n strip_prefix = \"unicode-normalization-alignments-0.1.12\",\n build_file = Label(\"@crates//crates:BUILD.unicode-normalization-alignments-0.1.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-segmentation-1.12.0\",\n sha256 = \"f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-segmentation/1.12.0/download\"],\n strip_prefix = \"unicode-segmentation-1.12.0\",\n build_file = Label(\"@crates//crates:BUILD.unicode-segmentation-1.12.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode_categories-0.1.1\",\n sha256 = \"39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode_categories/0.1.1/download\"],\n strip_prefix = \"unicode_categories-0.1.1\",\n build_file = Label(\"@crates//crates:BUILD.unicode_categories-0.1.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__version_check-0.9.5\",\n sha256 = \"0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/version_check/0.9.5/download\"],\n strip_prefix = \"version_check-0.9.5\",\n build_file = Label(\"@crates//crates:BUILD.version_check-0.9.5.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasi-0.14.2-wasi-0.2.4\",\n sha256 = \"9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasi/0.14.2+wasi-0.2.4/download\"],\n strip_prefix = \"wasi-0.14.2+wasi-0.2.4\",\n build_file = Label(\"@crates//crates:BUILD.wasi-0.14.2+wasi-0.2.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wit-bindgen-rt-0.39.0\",\n sha256 = \"6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wit-bindgen-rt/0.39.0/download\"],\n strip_prefix = \"wit-bindgen-rt-0.39.0\",\n build_file = Label(\"@crates//crates:BUILD.wit-bindgen-rt-0.39.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-0.7.35\",\n sha256 = \"1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy/0.7.35/download\"],\n strip_prefix = \"zerocopy-0.7.35\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-0.7.35.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-0.8.26\",\n sha256 = \"1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy/0.8.26/download\"],\n strip_prefix = \"zerocopy-0.8.26\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-0.8.26.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-derive-0.7.35\",\n sha256 = \"fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy-derive/0.7.35/download\"],\n strip_prefix = \"zerocopy-derive-0.7.35\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-derive-0.7.35.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-derive-0.8.26\",\n sha256 = \"9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy-derive/0.8.26/download\"],\n strip_prefix = \"zerocopy-derive-0.8.26\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-derive-0.8.26.bazel\"),\n )\n\n return [\n struct(repo=\"crates__tokenizers-0.22.2\", is_dev_dep = False),\n ]\n" - } - } - }, - "crates__ahash-0.8.12": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/ahash/0.8.12/download" - ], - "strip_prefix": "ahash-0.8.12", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"ahash\",\n deps = [\n \"@crates__ahash-0.8.12//:build_script_build\",\n \"@crates__cfg-if-1.0.0//:cfg_if\",\n \"@crates__getrandom-0.3.3//:getrandom\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__zerocopy-0.8.26//:zerocopy\",\n ] + select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [\n \"@crates__once_cell-1.20.2//:once_cell\", # cfg(not(all(target_arch = \"arm\", target_os = \"none\")))\n ],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [\n \"@crates__once_cell-1.20.2//:once_cell\", # cfg(not(all(target_arch = \"arm\", target_os = \"none\")))\n ],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [\n \"@crates__once_cell-1.20.2//:once_cell\", # cfg(not(all(target_arch = \"arm\", target_os = \"none\")))\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [\n \"@crates__once_cell-1.20.2//:once_cell\", # cfg(not(all(target_arch = \"arm\", target_os = \"none\")))\n ],\n \"//conditions:default\": [],\n }),\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"getrandom\",\n \"runtime-rng\",\n \"serde\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=ahash\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.12\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"getrandom\",\n \"runtime-rng\",\n \"serde\",\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n deps = [\n \"@crates__version_check-0.9.5//:version_check\",\n ],\n edition = \"2018\",\n pkg_name = \"ahash\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=ahash\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.8.12\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__aho-corasick-1.1.3": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/aho-corasick/1.1.3/download" - ], - "strip_prefix": "aho-corasick-1.1.3", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"aho_corasick\",\n deps = [\n \"@crates__memchr-2.7.4//:memchr\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"perf-literal\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=aho-corasick\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.1.3\",\n)\n" - } - }, - "crates__base64-0.13.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/base64/0.13.1/download" - ], - "strip_prefix": "base64-0.13.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"base64\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=base64\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.13.1\",\n)\n" - } - }, - "crates__bitflags-2.9.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/bitflags/2.9.1/download" - ], - "strip_prefix": "bitflags-2.9.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"bitflags\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=bitflags\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.9.1\",\n)\n" - } - }, - "crates__byteorder-1.5.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/byteorder/1.5.0/download" - ], - "strip_prefix": "byteorder-1.5.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"byteorder\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=byteorder\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.5.0\",\n)\n" - } - }, - "crates__castaway-0.2.4": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/castaway/0.2.4/download" - ], - "strip_prefix": "castaway-0.2.4", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"castaway\",\n proc_macro_deps = [\n \"@crates__rustversion-1.0.21//:rustversion\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=castaway\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.4\",\n)\n" - } - }, - "crates__cc-1.2.33": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/cc/1.2.33/download" - ], - "strip_prefix": "cc-1.2.33", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"cc\",\n deps = [\n \"@crates__shlex-1.3.0//:shlex\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=cc\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.2.33\",\n)\n" - } - }, - "crates__cfg-if-1.0.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/cfg-if/1.0.0/download" - ], - "strip_prefix": "cfg-if-1.0.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"cfg_if\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=cfg-if\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.0\",\n)\n" - } - }, - "crates__compact_str-0.9.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/compact_str/0.9.0/download" - ], - "strip_prefix": "compact_str-0.9.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"compact_str\",\n deps = [\n \"@crates__castaway-0.2.4//:castaway\",\n \"@crates__cfg-if-1.0.0//:cfg_if\",\n \"@crates__itoa-1.0.14//:itoa\",\n \"@crates__ryu-1.0.18//:ryu\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__static_assertions-1.1.0//:static_assertions\",\n ],\n proc_macro_deps = [\n \"@crates__rustversion-1.0.21//:rustversion\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"serde\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=compact_str\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.9.0\",\n)\n" - } - }, - "crates__crossbeam-deque-0.8.6": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/crossbeam-deque/0.8.6/download" - ], - "strip_prefix": "crossbeam-deque-0.8.6", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"crossbeam_deque\",\n deps = [\n \"@crates__crossbeam-epoch-0.9.18//:crossbeam_epoch\",\n \"@crates__crossbeam-utils-0.8.21//:crossbeam_utils\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=crossbeam-deque\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.6\",\n)\n" - } - }, - "crates__crossbeam-epoch-0.9.18": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/crossbeam-epoch/0.9.18/download" - ], - "strip_prefix": "crossbeam-epoch-0.9.18", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"crossbeam_epoch\",\n deps = [\n \"@crates__crossbeam-utils-0.8.21//:crossbeam_utils\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=crossbeam-epoch\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.9.18\",\n)\n" - } - }, - "crates__crossbeam-utils-0.8.21": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/crossbeam-utils/0.8.21/download" - ], - "strip_prefix": "crossbeam-utils-0.8.21", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"crossbeam_utils\",\n deps = [\n \"@crates__crossbeam-utils-0.8.21//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=crossbeam-utils\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.21\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"crossbeam-utils\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=crossbeam-utils\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.8.21\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__darling-0.20.10": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/darling/0.20.10/download" - ], - "strip_prefix": "darling-0.20.10", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"darling\",\n deps = [\n \"@crates__darling_core-0.20.10//:darling_core\",\n ],\n proc_macro_deps = [\n \"@crates__darling_macro-0.20.10//:darling_macro\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"suggestions\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=darling\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.20.10\",\n)\n" - } - }, - "crates__darling_core-0.20.10": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/darling_core/0.20.10/download" - ], - "strip_prefix": "darling_core-0.20.10", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"darling_core\",\n deps = [\n \"@crates__fnv-1.0.7//:fnv\",\n \"@crates__ident_case-1.0.1//:ident_case\",\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__strsim-0.11.1//:strsim\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"strsim\",\n \"suggestions\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=darling_core\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.20.10\",\n)\n" - } - }, - "crates__darling_macro-0.20.10": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/darling_macro/0.20.10/download" - ], - "strip_prefix": "darling_macro-0.20.10", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"darling_macro\",\n deps = [\n \"@crates__darling_core-0.20.10//:darling_core\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=darling_macro\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.20.10\",\n)\n" - } - }, - "crates__dary_heap-0.3.7": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/dary_heap/0.3.7/download" - ], - "strip_prefix": "dary_heap-0.3.7", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"dary_heap\",\n deps = [\n \"@crates__serde-1.0.216//:serde\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"serde\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=dary_heap\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.7\",\n)\n" - } - }, - "crates__derive_builder-0.20.2": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/derive_builder/0.20.2/download" - ], - "strip_prefix": "derive_builder-0.20.2", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"derive_builder\",\n proc_macro_deps = [\n \"@crates__derive_builder_macro-0.20.2//:derive_builder_macro\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=derive_builder\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.20.2\",\n)\n" - } - }, - "crates__derive_builder_core-0.20.2": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/derive_builder_core/0.20.2/download" - ], - "strip_prefix": "derive_builder_core-0.20.2", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"derive_builder_core\",\n deps = [\n \"@crates__darling-0.20.10//:darling\",\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"lib_has_std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=derive_builder_core\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.20.2\",\n)\n" - } - }, - "crates__derive_builder_macro-0.20.2": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/derive_builder_macro/0.20.2/download" - ], - "strip_prefix": "derive_builder_macro-0.20.2", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"derive_builder_macro\",\n deps = [\n \"@crates__derive_builder_core-0.20.2//:derive_builder_core\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"lib_has_std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=derive_builder_macro\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.20.2\",\n)\n" - } - }, - "crates__either-1.13.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/either/1.13.0/download" - ], - "strip_prefix": "either-1.13.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"either\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"use_std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=either\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.13.0\",\n)\n" - } - }, - "crates__esaxx-rs-0.1.10": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/esaxx-rs/0.1.10/download" - ], - "strip_prefix": "esaxx-rs-0.1.10", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"esaxx_rs\",\n deps = [\n \"@crates__esaxx-rs-0.1.10//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"cc\",\n \"cpp\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=esaxx-rs\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.10\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"cc\",\n \"cpp\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n deps = [\n \"@crates__cc-1.2.33//:cc\",\n ],\n edition = \"2018\",\n pkg_name = \"esaxx-rs\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=esaxx-rs\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.1.10\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__fnv-1.0.7": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/fnv/1.0.7/download" - ], - "strip_prefix": "fnv-1.0.7", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"fnv\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=fnv\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.7\",\n)\n" - } - }, - "crates__getrandom-0.3.3": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/getrandom/0.3.3/download" - ], - "strip_prefix": "getrandom-0.3.3", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"getrandom\",\n deps = [\n \"@crates__cfg-if-1.0.0//:cfg_if\",\n \"@crates__getrandom-0.3.3//:build_script_build\",\n ] + select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(any(target_os = \"macos\", target_os = \"openbsd\", target_os = \"vita\", target_os = \"emscripten\"))\n ],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(all(any(target_os = \"linux\", target_os = \"android\"), not(any(all(target_os = \"linux\", target_env = \"\"), getrandom_backend = \"custom\", getrandom_backend = \"linux_raw\", getrandom_backend = \"rdrand\", getrandom_backend = \"rndr\"))))\n ],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(any(target_os = \"macos\", target_os = \"openbsd\", target_os = \"vita\", target_os = \"emscripten\"))\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(all(any(target_os = \"linux\", target_os = \"android\"), not(any(all(target_os = \"linux\", target_env = \"\"), getrandom_backend = \"custom\", getrandom_backend = \"linux_raw\", getrandom_backend = \"rdrand\", getrandom_backend = \"rndr\"))))\n ],\n \"//conditions:default\": [],\n }),\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=getrandom\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.3\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"getrandom\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=getrandom\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.3.3\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__ident_case-1.0.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/ident_case/1.0.1/download" - ], - "strip_prefix": "ident_case-1.0.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"ident_case\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=ident_case\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.1\",\n)\n" - } - }, - "crates__itertools-0.14.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/itertools/0.14.0/download" - ], - "strip_prefix": "itertools-0.14.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"itertools\",\n deps = [\n \"@crates__either-1.13.0//:either\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"use_alloc\",\n \"use_std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=itertools\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.14.0\",\n)\n" - } - }, - "crates__itoa-1.0.14": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/itoa/1.0.14/download" - ], - "strip_prefix": "itoa-1.0.14", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"itoa\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=itoa\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.14\",\n)\n" - } - }, - "crates__libc-0.2.169": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/libc/0.2.169/download" - ], - "strip_prefix": "libc-0.2.169", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"libc\",\n deps = [\n \"@crates__libc-0.2.169//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=libc\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.169\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"libc\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=libc\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.2.169\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__log-0.4.22": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/log/0.4.22/download" - ], - "strip_prefix": "log-0.4.22", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"log\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=log\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.4.22\",\n)\n" - } - }, - "crates__macro_rules_attribute-0.2.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/macro_rules_attribute/0.2.0/download" - ], - "strip_prefix": "macro_rules_attribute-0.2.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"macro_rules_attribute\",\n proc_macro_deps = [\n \"@crates__macro_rules_attribute-proc_macro-0.2.0//:macro_rules_attribute_proc_macro\",\n \"@crates__paste-1.0.15//:paste\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=macro_rules_attribute\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.0\",\n)\n" - } - }, - "crates__macro_rules_attribute-proc_macro-0.2.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/macro_rules_attribute-proc_macro/0.2.0/download" - ], - "strip_prefix": "macro_rules_attribute-proc_macro-0.2.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"macro_rules_attribute_proc_macro\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"mod.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=macro_rules_attribute-proc_macro\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.0\",\n)\n" - } - }, - "crates__memchr-2.7.4": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/memchr/2.7.4/download" - ], - "strip_prefix": "memchr-2.7.4", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"memchr\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=memchr\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.7.4\",\n)\n" - } - }, - "crates__minimal-lexical-0.2.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/minimal-lexical/0.2.1/download" - ], - "strip_prefix": "minimal-lexical-0.2.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"minimal_lexical\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=minimal-lexical\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.1\",\n)\n" - } - }, - "crates__monostate-0.1.13": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/monostate/0.1.13/download" - ], - "strip_prefix": "monostate-0.1.13", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"monostate\",\n deps = [\n \"@crates__serde-1.0.216//:serde\",\n ],\n proc_macro_deps = [\n \"@crates__monostate-impl-0.1.13//:monostate_impl\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=monostate\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.13\",\n)\n" - } - }, - "crates__monostate-impl-0.1.13": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/monostate-impl/0.1.13/download" - ], - "strip_prefix": "monostate-impl-0.1.13", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"monostate_impl\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=monostate-impl\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.13\",\n)\n" - } - }, - "crates__nom-7.1.3": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/nom/7.1.3/download" - ], - "strip_prefix": "nom-7.1.3", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"nom\",\n deps = [\n \"@crates__memchr-2.7.4//:memchr\",\n \"@crates__minimal-lexical-0.2.1//:minimal_lexical\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=nom\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"7.1.3\",\n)\n" - } - }, - "crates__once_cell-1.20.2": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/once_cell/1.20.2/download" - ], - "strip_prefix": "once_cell-1.20.2", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"once_cell\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"default\",\n \"race\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=once_cell\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.20.2\",\n)\n" - } - }, - "crates__onig-6.5.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/onig/6.5.1/download" - ], - "strip_prefix": "onig-6.5.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"onig\",\n deps = [\n \"@crates__bitflags-2.9.1//:bitflags\",\n \"@crates__once_cell-1.20.2//:once_cell\",\n \"@crates__onig_sys-69.9.1//:onig_sys\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=onig\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"6.5.1\",\n)\n" - } - }, - "crates__onig_sys-69.9.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/onig_sys/69.9.1/download" - ], - "strip_prefix": "onig_sys-69.9.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"onig_sys\",\n deps = [\n \"@crates__onig_sys-69.9.1//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=onig_sys\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"69.9.1\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n deps = [\n \"@crates__cc-1.2.33//:cc\",\n \"@crates__pkg-config-0.3.32//:pkg_config\",\n ],\n edition = \"2021\",\n links = \"onig\",\n pkg_name = \"onig_sys\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=onig_sys\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"69.9.1\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__paste-1.0.15": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/paste/1.0.15/download" - ], - "strip_prefix": "paste-1.0.15", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"paste\",\n deps = [\n \"@crates__paste-1.0.15//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=paste\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.15\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2018\",\n pkg_name = \"paste\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=paste\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.15\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__pkg-config-0.3.32": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/pkg-config/0.3.32/download" - ], - "strip_prefix": "pkg-config-0.3.32", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"pkg_config\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=pkg-config\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.32\",\n)\n" - } - }, - "crates__ppv-lite86-0.2.20": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/ppv-lite86/0.2.20/download" - ], - "strip_prefix": "ppv-lite86-0.2.20", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"ppv_lite86\",\n deps = [\n \"@crates__zerocopy-0.7.35//:zerocopy\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"simd\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=ppv-lite86\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.20\",\n)\n" - } - }, - "crates__proc-macro2-1.0.92": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/proc-macro2/1.0.92/download" - ], - "strip_prefix": "proc-macro2-1.0.92", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"proc_macro2\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:build_script_build\",\n \"@crates__unicode-ident-1.0.14//:unicode_ident\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"proc-macro\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=proc-macro2\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.92\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"proc-macro\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"proc-macro2\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=proc-macro2\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.92\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__quote-1.0.37": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/quote/1.0.37/download" - ], - "strip_prefix": "quote-1.0.37", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"quote\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"proc-macro\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=quote\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.37\",\n)\n" - } - }, - "crates__r-efi-5.3.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/r-efi/5.3.0/download" - ], - "strip_prefix": "r-efi-5.3.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"r_efi\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=r-efi\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"5.3.0\",\n)\n" - } - }, - "crates__rand-0.9.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/rand/0.9.1/download" - ], - "strip_prefix": "rand-0.9.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"rand\",\n deps = [\n \"@crates__rand_chacha-0.9.0//:rand_chacha\",\n \"@crates__rand_core-0.9.3//:rand_core\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"default\",\n \"os_rng\",\n \"small_rng\",\n \"std\",\n \"std_rng\",\n \"thread_rng\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rand\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.9.1\",\n)\n" - } - }, - "crates__rand_chacha-0.9.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/rand_chacha/0.9.0/download" - ], - "strip_prefix": "rand_chacha-0.9.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"rand_chacha\",\n deps = [\n \"@crates__ppv-lite86-0.2.20//:ppv_lite86\",\n \"@crates__rand_core-0.9.3//:rand_core\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rand_chacha\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.9.0\",\n)\n" - } - }, - "crates__rand_core-0.9.3": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/rand_core/0.9.3/download" - ], - "strip_prefix": "rand_core-0.9.3", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"rand_core\",\n deps = [\n \"@crates__getrandom-0.3.3//:getrandom\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"os_rng\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rand_core\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.9.3\",\n)\n" - } - }, - "crates__rayon-1.10.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/rayon/1.10.0/download" - ], - "strip_prefix": "rayon-1.10.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"rayon\",\n deps = [\n \"@crates__either-1.13.0//:either\",\n \"@crates__rayon-core-1.12.1//:rayon_core\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rayon\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.10.0\",\n)\n" - } - }, - "crates__rayon-cond-0.4.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/rayon-cond/0.4.0/download" - ], - "strip_prefix": "rayon-cond-0.4.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"rayon_cond\",\n deps = [\n \"@crates__either-1.13.0//:either\",\n \"@crates__itertools-0.14.0//:itertools\",\n \"@crates__rayon-1.10.0//:rayon\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rayon-cond\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.4.0\",\n)\n" - } - }, - "crates__rayon-core-1.12.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/rayon-core/1.12.1/download" - ], - "strip_prefix": "rayon-core-1.12.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"rayon_core\",\n deps = [\n \"@crates__crossbeam-deque-0.8.6//:crossbeam_deque\",\n \"@crates__crossbeam-utils-0.8.21//:crossbeam_utils\",\n \"@crates__rayon-core-1.12.1//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rayon-core\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.12.1\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n links = \"rayon-core\",\n pkg_name = \"rayon-core\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rayon-core\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.12.1\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__regex-1.11.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/regex/1.11.1/download" - ], - "strip_prefix": "regex-1.11.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"regex\",\n deps = [\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__memchr-2.7.4//:memchr\",\n \"@crates__regex-automata-0.4.9//:regex_automata\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"perf\",\n \"perf-backtrack\",\n \"perf-cache\",\n \"perf-dfa\",\n \"perf-inline\",\n \"perf-literal\",\n \"perf-onepass\",\n \"std\",\n \"unicode\",\n \"unicode-age\",\n \"unicode-bool\",\n \"unicode-case\",\n \"unicode-gencat\",\n \"unicode-perl\",\n \"unicode-script\",\n \"unicode-segment\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=regex\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.11.1\",\n)\n" - } - }, - "crates__regex-automata-0.4.9": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/regex-automata/0.4.9/download" - ], - "strip_prefix": "regex-automata-0.4.9", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"regex_automata\",\n deps = [\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__memchr-2.7.4//:memchr\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"dfa-onepass\",\n \"hybrid\",\n \"meta\",\n \"nfa-backtrack\",\n \"nfa-pikevm\",\n \"nfa-thompson\",\n \"perf-inline\",\n \"perf-literal\",\n \"perf-literal-multisubstring\",\n \"perf-literal-substring\",\n \"std\",\n \"syntax\",\n \"unicode\",\n \"unicode-age\",\n \"unicode-bool\",\n \"unicode-case\",\n \"unicode-gencat\",\n \"unicode-perl\",\n \"unicode-script\",\n \"unicode-segment\",\n \"unicode-word-boundary\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=regex-automata\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.4.9\",\n)\n" - } - }, - "crates__regex-syntax-0.8.5": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/regex-syntax/0.8.5/download" - ], - "strip_prefix": "regex-syntax-0.8.5", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"regex_syntax\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n \"unicode\",\n \"unicode-age\",\n \"unicode-bool\",\n \"unicode-case\",\n \"unicode-gencat\",\n \"unicode-perl\",\n \"unicode-script\",\n \"unicode-segment\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=regex-syntax\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.5\",\n)\n" - } - }, - "crates__rustversion-1.0.21": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/rustversion/1.0.21/download" - ], - "strip_prefix": "rustversion-1.0.21", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"rustversion\",\n deps = [\n \"@crates__rustversion-1.0.21//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rustversion\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.21\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build/build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2018\",\n pkg_name = \"rustversion\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rustversion\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.21\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__ryu-1.0.18": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/ryu/1.0.18/download" - ], - "strip_prefix": "ryu-1.0.18", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"ryu\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=ryu\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.18\",\n)\n" - } - }, - "crates__serde-1.0.216": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/serde/1.0.216/download" - ], - "strip_prefix": "serde-1.0.216", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"serde\",\n deps = [\n \"@crates__serde-1.0.216//:build_script_build\",\n ],\n proc_macro_deps = [\n \"@crates__serde_derive-1.0.216//:serde_derive\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"default\",\n \"derive\",\n \"serde_derive\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.216\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"default\",\n \"derive\",\n \"serde_derive\",\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2018\",\n pkg_name = \"serde\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.216\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__serde_derive-1.0.216": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/serde_derive/1.0.216/download" - ], - "strip_prefix": "serde_derive-1.0.216", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"serde_derive\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde_derive\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.216\",\n)\n" - } - }, - "crates__serde_json-1.0.134": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/serde_json/1.0.134/download" - ], - "strip_prefix": "serde_json-1.0.134", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"serde_json\",\n deps = [\n \"@crates__itoa-1.0.14//:itoa\",\n \"@crates__memchr-2.7.4//:memchr\",\n \"@crates__ryu-1.0.18//:ryu\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__serde_json-1.0.134//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde_json\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.134\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"serde_json\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde_json\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.134\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__shlex-1.3.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/shlex/1.3.0/download" - ], - "strip_prefix": "shlex-1.3.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"shlex\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=shlex\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.3.0\",\n)\n" - } - }, - "crates__smallvec-1.13.2": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/smallvec/1.13.2/download" - ], - "strip_prefix": "smallvec-1.13.2", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"smallvec\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=smallvec\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.13.2\",\n)\n" - } - }, - "crates__spm_precompiled-0.1.4": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/spm_precompiled/0.1.4/download" - ], - "strip_prefix": "spm_precompiled-0.1.4", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"spm_precompiled\",\n deps = [\n \"@crates__base64-0.13.1//:base64\",\n \"@crates__nom-7.1.3//:nom\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__unicode-segmentation-1.12.0//:unicode_segmentation\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=spm_precompiled\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.4\",\n)\n" - } - }, - "crates__static_assertions-1.1.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/static_assertions/1.1.0/download" - ], - "strip_prefix": "static_assertions-1.1.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"static_assertions\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=static_assertions\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.1.0\",\n)\n" - } - }, - "crates__strsim-0.11.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/strsim/0.11.1/download" - ], - "strip_prefix": "strsim-0.11.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"strsim\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=strsim\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.11.1\",\n)\n" - } - }, - "crates__syn-2.0.91": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/syn/2.0.91/download" - ], - "strip_prefix": "syn-2.0.91", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"syn\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__unicode-ident-1.0.14//:unicode_ident\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"clone-impls\",\n \"default\",\n \"derive\",\n \"extra-traits\",\n \"full\",\n \"parsing\",\n \"printing\",\n \"proc-macro\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=syn\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.0.91\",\n)\n" - } - }, - "crates__thiserror-2.0.12": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/thiserror/2.0.12/download" - ], - "strip_prefix": "thiserror-2.0.12", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"thiserror\",\n deps = [\n \"@crates__thiserror-2.0.12//:build_script_build\",\n ],\n proc_macro_deps = [\n \"@crates__thiserror-impl-2.0.12//:thiserror_impl\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=thiserror\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.0.12\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"thiserror\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=thiserror\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"2.0.12\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__thiserror-impl-2.0.12": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/thiserror-impl/2.0.12/download" - ], - "strip_prefix": "thiserror-impl-2.0.12", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"thiserror_impl\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=thiserror-impl\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.0.12\",\n)\n" - } - }, - "crates__tokenizers-0.22.2": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/tokenizers/0.22.2/download" - ], - "strip_prefix": "tokenizers-0.22.2", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"tokenizers\",\n deps = [\n \"@crates__ahash-0.8.12//:ahash\",\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__compact_str-0.9.0//:compact_str\",\n \"@crates__dary_heap-0.3.7//:dary_heap\",\n \"@crates__derive_builder-0.20.2//:derive_builder\",\n \"@crates__esaxx-rs-0.1.10//:esaxx_rs\",\n \"@crates__getrandom-0.3.3//:getrandom\",\n \"@crates__itertools-0.14.0//:itertools\",\n \"@crates__log-0.4.22//:log\",\n \"@crates__macro_rules_attribute-0.2.0//:macro_rules_attribute\",\n \"@crates__monostate-0.1.13//:monostate\",\n \"@crates__onig-6.5.1//:onig\",\n \"@crates__rand-0.9.1//:rand\",\n \"@crates__rayon-1.10.0//:rayon\",\n \"@crates__rayon-cond-0.4.0//:rayon_cond\",\n \"@crates__regex-1.11.1//:regex\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__serde_json-1.0.134//:serde_json\",\n \"@crates__spm_precompiled-0.1.4//:spm_precompiled\",\n \"@crates__thiserror-2.0.12//:thiserror\",\n \"@crates__unicode-normalization-alignments-0.1.12//:unicode_normalization_alignments\",\n \"@crates__unicode-segmentation-1.12.0//:unicode_segmentation\",\n \"@crates__unicode_categories-0.1.1//:unicode_categories\",\n ],\n proc_macro_deps = [\n \"@crates__paste-1.0.15//:paste\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"esaxx_fast\",\n \"onig\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=tokenizers\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.22.2\",\n)\n" - } - }, - "crates__unicode-ident-1.0.14": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/unicode-ident/1.0.14/download" - ], - "strip_prefix": "unicode-ident-1.0.14", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"unicode_ident\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=unicode-ident\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.14\",\n)\n" - } - }, - "crates__unicode-normalization-alignments-0.1.12": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/unicode-normalization-alignments/0.1.12/download" - ], - "strip_prefix": "unicode-normalization-alignments-0.1.12", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"unicode_normalization_alignments\",\n deps = [\n \"@crates__smallvec-1.13.2//:smallvec\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=unicode-normalization-alignments\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.12\",\n)\n" - } - }, - "crates__unicode-segmentation-1.12.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/unicode-segmentation/1.12.0/download" - ], - "strip_prefix": "unicode-segmentation-1.12.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"unicode_segmentation\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=unicode-segmentation\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.12.0\",\n)\n" - } - }, - "crates__unicode_categories-0.1.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/unicode_categories/0.1.1/download" - ], - "strip_prefix": "unicode_categories-0.1.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"unicode_categories\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=unicode_categories\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.1\",\n)\n" - } - }, - "crates__version_check-0.9.5": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/version_check/0.9.5/download" - ], - "strip_prefix": "version_check-0.9.5", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"version_check\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=version_check\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.9.5\",\n)\n" - } - }, - "crates__wasi-0.14.2-wasi-0.2.4": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/wasi/0.14.2+wasi-0.2.4/download" - ], - "strip_prefix": "wasi-0.14.2+wasi-0.2.4", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wasi\",\n deps = [\n \"@crates__wit-bindgen-rt-0.39.0//:wit_bindgen_rt\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasi\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.14.2+wasi-0.2.4\",\n)\n" - } - }, - "crates__wit-bindgen-rt-0.39.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/wit-bindgen-rt/0.39.0/download" - ], - "strip_prefix": "wit-bindgen-rt-0.39.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wit_bindgen_rt\",\n deps = [\n \"@crates__wit-bindgen-rt-0.39.0//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wit-bindgen-rt\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.39.0\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"wit-bindgen-rt\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wit-bindgen-rt\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.39.0\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__zerocopy-0.7.35": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/zerocopy/0.7.35/download" - ], - "strip_prefix": "zerocopy-0.7.35", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"zerocopy\",\n deps = [\n \"@crates__byteorder-1.5.0//:byteorder\",\n ],\n proc_macro_deps = [\n \"@crates__zerocopy-derive-0.7.35//:zerocopy_derive\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"byteorder\",\n \"default\",\n \"derive\",\n \"simd\",\n \"zerocopy-derive\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=zerocopy\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.7.35\",\n)\n" - } - }, - "crates__zerocopy-0.8.26": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/zerocopy/0.8.26/download" - ], - "strip_prefix": "zerocopy-0.8.26", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"zerocopy\",\n deps = [\n \"@crates__zerocopy-0.8.26//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"simd\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=zerocopy\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.26\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"simd\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"zerocopy\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=zerocopy\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.8.26\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__zerocopy-derive-0.7.35": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/zerocopy-derive/0.7.35/download" - ], - "strip_prefix": "zerocopy-derive-0.7.35", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"zerocopy_derive\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=zerocopy-derive\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.7.35\",\n)\n" - } - }, - "crates__zerocopy-derive-0.8.26": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/zerocopy-derive/0.8.26/download" - ], - "strip_prefix": "zerocopy-derive-0.8.26", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"zerocopy_derive\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=zerocopy-derive\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.26\",\n)\n" - } - } - }, - "moduleExtensionMetadata": { - "useAllRepos": "NO", - "reproducible": false - }, - "recordedRepoMappingEntries": [ - [ - "bazel_features+", - "bazel_features_globals", - "bazel_features++version_extension+bazel_features_globals" - ], - [ - "bazel_features+", - "bazel_features_version", - "bazel_features++version_extension+bazel_features_version" - ], - [ - "rules_cc+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_cc+", - "cc_compatibility_proxy", - "rules_cc++compatibility_proxy+cc_compatibility_proxy" - ], - [ - "rules_cc+", - "rules_cc", - "rules_cc+" - ], - [ - "rules_cc++compatibility_proxy+cc_compatibility_proxy", - "rules_cc", - "rules_cc+" - ], - [ - "rules_rust+", - "bazel_features", - "bazel_features+" - ], - [ - "rules_rust+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "rules_rust+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_rust+", - "rules_cc", - "rules_cc+" - ], - [ - "rules_rust+", - "rules_rust", - "rules_rust+" - ] - ] - } - }, - "@@rules_rust+//crate_universe/private:internal_extensions.bzl%cu_nr": { - "general": { - "bzlTransitiveDigest": "ANGVT5r/yDVCawqIISn+1Gh62Yw8VCMEqxi/JAAJCzw=", - "usagesDigest": "3vKI8uvqTpJCf+t8aU6UD5d5cUWinWhtMjKkRpCLR+A=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "cargo_bazel_bootstrap": { - "repoRuleId": "@@rules_rust+//cargo/private:cargo_bootstrap.bzl%cargo_bootstrap_repository", - "attributes": { - "srcs": [ - "@@rules_rust+//crate_universe:src/api.rs", - "@@rules_rust+//crate_universe:src/api/lockfile.rs", - "@@rules_rust+//crate_universe:src/cli.rs", - "@@rules_rust+//crate_universe:src/cli/generate.rs", - "@@rules_rust+//crate_universe:src/cli/query.rs", - "@@rules_rust+//crate_universe:src/cli/render.rs", - "@@rules_rust+//crate_universe:src/cli/splice.rs", - "@@rules_rust+//crate_universe:src/cli/vendor.rs", - "@@rules_rust+//crate_universe:src/config.rs", - "@@rules_rust+//crate_universe:src/context.rs", - "@@rules_rust+//crate_universe:src/context/crate_context.rs", - "@@rules_rust+//crate_universe:src/context/platforms.rs", - "@@rules_rust+//crate_universe:src/lib.rs", - "@@rules_rust+//crate_universe:src/lockfile.rs", - "@@rules_rust+//crate_universe:src/main.rs", - "@@rules_rust+//crate_universe:src/metadata.rs", - "@@rules_rust+//crate_universe:src/metadata/cargo_bin.rs", - "@@rules_rust+//crate_universe:src/metadata/cargo_tree_resolver.rs", - "@@rules_rust+//crate_universe:src/metadata/cargo_tree_rustc_wrapper.bat", - "@@rules_rust+//crate_universe:src/metadata/cargo_tree_rustc_wrapper.sh", - "@@rules_rust+//crate_universe:src/metadata/dependency.rs", - "@@rules_rust+//crate_universe:src/metadata/metadata_annotation.rs", - "@@rules_rust+//crate_universe:src/metadata/workspace_discoverer.rs", - "@@rules_rust+//crate_universe:src/rendering.rs", - "@@rules_rust+//crate_universe:src/rendering/template_engine.rs", - "@@rules_rust+//crate_universe:src/rendering/templates/module_bzl.j2", - "@@rules_rust+//crate_universe:src/rendering/templates/partials/header.j2", - "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/aliases_map.j2", - "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/deps_map.j2", - "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/repo_git.j2", - "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/repo_http.j2", - "@@rules_rust+//crate_universe:src/rendering/templates/vendor_module.j2", - "@@rules_rust+//crate_universe:src/rendering/verbatim/alias_rules.bzl", - "@@rules_rust+//crate_universe:src/select.rs", - "@@rules_rust+//crate_universe:src/splicing.rs", - "@@rules_rust+//crate_universe:src/splicing/cargo_config.rs", - "@@rules_rust+//crate_universe:src/splicing/crate_index_lookup.rs", - "@@rules_rust+//crate_universe:src/splicing/splicer.rs", - "@@rules_rust+//crate_universe:src/test.rs", - "@@rules_rust+//crate_universe:src/utils.rs", - "@@rules_rust+//crate_universe:src/utils/starlark.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/glob.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/label.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/select.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/select_dict.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/select_list.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/select_scalar.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/select_set.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/serialize.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/target_compatible_with.rs", - "@@rules_rust+//crate_universe:src/utils/symlink.rs", - "@@rules_rust+//crate_universe:src/utils/target_triple.rs" - ], - "binary": "cargo-bazel", - "cargo_lockfile": "@@rules_rust+//crate_universe:Cargo.lock", - "cargo_toml": "@@rules_rust+//crate_universe:Cargo.toml", - "version": "1.86.0", - "timeout": 900, - "rust_toolchain_cargo_template": "@rust_host_tools//:bin/{tool}", - "rust_toolchain_rustc_template": "@rust_host_tools//:bin/{tool}", - "compressed_windows_toolchain_names": false - } - } - }, - "moduleExtensionMetadata": { - "explicitRootModuleDirectDeps": [ - "cargo_bazel_bootstrap" - ], - "explicitRootModuleDirectDevDeps": [], - "useAllRepos": "NO", - "reproducible": false - }, - "recordedRepoMappingEntries": [ - [ - "bazel_features+", - "bazel_features_globals", - "bazel_features++version_extension+bazel_features_globals" - ], - [ - "bazel_features+", - "bazel_features_version", - "bazel_features++version_extension+bazel_features_version" - ], - [ - "rules_cc+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_cc+", - "cc_compatibility_proxy", - "rules_cc++compatibility_proxy+cc_compatibility_proxy" - ], - [ - "rules_cc+", - "rules_cc", - "rules_cc+" - ], - [ - "rules_cc++compatibility_proxy+cc_compatibility_proxy", - "rules_cc", - "rules_cc+" - ], - [ - "rules_rust+", - "bazel_features", - "bazel_features+" - ], - [ - "rules_rust+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "rules_rust+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_rust+", - "cui", - "rules_rust++cu+cui" - ], - [ - "rules_rust+", - "rules_cc", - "rules_cc+" - ], - [ - "rules_rust+", - "rules_rust", - "rules_rust+" - ], - [ - "rules_rust+", - "rules_rust_ctve", - "rules_rust++i2+rules_rust_ctve" - ] - ] - } - }, - "@@rules_swift+//swift:extensions.bzl%non_module_deps": { - "general": { - "bzlTransitiveDigest": "zLT+pQh+AVbyxn970hzqSTfnjUqwnk0PUnP/REyVSMY=", - "usagesDigest": "mhACFnrdMv9Wi0Mt67bxocJqviRkDSV+Ee5Mqdj5akA=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "com_github_apple_swift_protobuf": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-protobuf/archive/1.20.2.tar.gz" - ], - "sha256": "3fb50bd4d293337f202d917b6ada22f9548a0a0aed9d9a4d791e6fbd8a246ebb", - "strip_prefix": "swift-protobuf-1.20.2/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_protobuf/BUILD.overlay" - } - }, - "com_github_grpc_grpc_swift": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/grpc/grpc-swift/archive/1.16.0.tar.gz" - ], - "sha256": "58b60431d0064969f9679411264b82e40a217ae6bd34e17096d92cc4e47556a5", - "strip_prefix": "grpc-swift-1.16.0/", - "build_file": "@@rules_swift+//third_party:com_github_grpc_grpc_swift/BUILD.overlay" - } - }, - "com_github_apple_swift_docc_symbolkit": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-docc-symbolkit/archive/refs/tags/swift-5.10-RELEASE.tar.gz" - ], - "sha256": "de1d4b6940468ddb53b89df7aa1a81323b9712775b0e33e8254fa0f6f7469a97", - "strip_prefix": "swift-docc-symbolkit-swift-5.10-RELEASE", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_docc_symbolkit/BUILD.overlay" - } - }, - "com_github_apple_swift_nio": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio/archive/2.42.0.tar.gz" - ], - "sha256": "e3304bc3fb53aea74a3e54bd005ede11f6dc357117d9b1db642d03aea87194a0", - "strip_prefix": "swift-nio-2.42.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_http2": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-http2/archive/1.26.0.tar.gz" - ], - "sha256": "f0edfc9d6a7be1d587e5b403f2d04264bdfae59aac1d74f7d974a9022c6d2b25", - "strip_prefix": "swift-nio-http2-1.26.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_http2/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_transport_services": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-transport-services/archive/1.15.0.tar.gz" - ], - "sha256": "f3498dafa633751a52b9b7f741f7ac30c42bcbeb3b9edca6d447e0da8e693262", - "strip_prefix": "swift-nio-transport-services-1.15.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_transport_services/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_extras": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-extras/archive/1.4.0.tar.gz" - ], - "sha256": "4684b52951d9d9937bb3e8ccd6b5daedd777021ef2519ea2f18c4c922843b52b", - "strip_prefix": "swift-nio-extras-1.4.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_extras/BUILD.overlay" - } - }, - "com_github_apple_swift_log": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-log/archive/1.4.4.tar.gz" - ], - "sha256": "48fe66426c784c0c20031f15dc17faf9f4c9037c192bfac2f643f65cb2321ba0", - "strip_prefix": "swift-log-1.4.4/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_log/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_ssl": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-ssl/archive/2.23.0.tar.gz" - ], - "sha256": "4787c63f61dd04d99e498adc3d1a628193387e41efddf8de19b8db04544d016d", - "strip_prefix": "swift-nio-ssl-2.23.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_ssl/BUILD.overlay" - } - }, - "com_github_apple_swift_collections": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-collections/archive/1.0.4.tar.gz" - ], - "sha256": "d9e4c8a91c60fb9c92a04caccbb10ded42f4cb47b26a212bc6b39cc390a4b096", - "strip_prefix": "swift-collections-1.0.4/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_collections/BUILD.overlay" - } - }, - "com_github_apple_swift_atomics": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-atomics/archive/1.1.0.tar.gz" - ], - "sha256": "1bee7f469f7e8dc49f11cfa4da07182fbc79eab000ec2c17bfdce468c5d276fb", - "strip_prefix": "swift-atomics-1.1.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_atomics/BUILD.overlay" - } - }, - "build_bazel_rules_swift_index_import": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "build_file": "@@rules_swift+//third_party:build_bazel_rules_swift_index_import/BUILD.overlay", - "canonical_id": "index-import-5.8", - "urls": [ - "https://github.com/MobileNativeFoundation/index-import/releases/download/5.8.0.1/index-import.tar.gz" - ], - "sha256": "28c1ffa39d99e74ed70623899b207b41f79214c498c603915aef55972a851a15" - } - }, - "build_bazel_rules_swift_local_config": { - "repoRuleId": "@@rules_swift+//swift/internal:swift_autoconfiguration.bzl%swift_autoconfiguration", - "attributes": {} - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_swift+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_swift+", - "build_bazel_rules_swift", - "rules_swift+" - ] - ] - } - }, - "@@rules_zig+//zig:extensions.bzl%zig": { - "general": { - "bzlTransitiveDigest": "wV3euhBZSI0D2ZlqbY1YW592mghFY88d9H6zBMQRaJY=", - "usagesDigest": "MUggfXR9Y92m2LJD1coQy9DezZvA1s0RkFDA7YzrxBw=", - "recordedFileInputs": { - "@@//bazel/zig_index.json": "d38c409f2a4af04ad52d52f817ff71b77bb919194f5be6c34f827c68296d5795", - "@@rules_zig+//zig/private/versions.json": "6eb85ebaee72c4e6fbc82d8ad73f69e0899d839b70a7e19caa792ab6a5fa0c36" - }, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-linux": { - "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", - "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "mirrors": [ - "https://mirror.zml.ai/zig", - "https://ziglang.org/builds/" - ], - "sha256": "de4eddf26f6f5e95d1bd055577cba7e9ebe9fbe8b1425bd4d40f073303ccc65b", - "zig_version": "0.16.0-dev.1912+0cbaaa5eb", - "platform": "aarch64-linux" - } - }, - "zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-macos": { - "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", - "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "mirrors": [ - "https://mirror.zml.ai/zig", - "https://ziglang.org/builds/" - ], - "sha256": "241fa43a37803a9e50e7f4a2d2c436882ee2215e6c420c454cb4de57adae84d3", - "zig_version": "0.16.0-dev.1912+0cbaaa5eb", - "platform": "aarch64-macos" - } - }, - "zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-windows": { - "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", - "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1912+0cbaaa5eb.zip", - "mirrors": [ - "https://mirror.zml.ai/zig", - "https://ziglang.org/builds/" - ], - "sha256": "824044c2dd76d4fdf9dd3d3fc7e8ed9bd39a1a03a3eb3acf1fecde25cee640cc", - "zig_version": "0.16.0-dev.1912+0cbaaa5eb", - "platform": "aarch64-windows" - } - }, - "zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-linux": { - "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", - "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "mirrors": [ - "https://mirror.zml.ai/zig", - "https://ziglang.org/builds/" - ], - "sha256": "d0bfbe4c6e6871a36ad7c9be577df2abda37c69af437fedb6865d8729d60cbdc", - "zig_version": "0.16.0-dev.1912+0cbaaa5eb", - "platform": "x86_64-linux" - } - }, - "zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-macos": { - "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", - "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "mirrors": [ - "https://mirror.zml.ai/zig", - "https://ziglang.org/builds/" - ], - "sha256": "9bfa7e5237c5e5b132229e3686eb849ca2afb23f7b6f6b72cbae0e5e8766aade", - "zig_version": "0.16.0-dev.1912+0cbaaa5eb", - "platform": "x86_64-macos" - } - }, - "zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-windows": { - "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", - "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1912+0cbaaa5eb.zip", - "mirrors": [ - "https://mirror.zml.ai/zig", - "https://ziglang.org/builds/" - ], - "sha256": "719859787d0be77a8dc19f5ecc37d8cc7802ddca06f616d81e9ebde34e610dd4", - "zig_version": "0.16.0-dev.1912+0cbaaa5eb", - "platform": "x86_64-windows" - } - }, - "zig_toolchains": { - "repoRuleId": "@@rules_zig+//zig/private/repo:toolchains_repo.bzl%toolchains_repo", - "attributes": { - "names": [ - "zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-linux", - "zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-macos", - "zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-windows", - "zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-linux", - "zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-macos", - "zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-windows" - ], - "labels": [ - "@zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-linux//:zig_toolchain", - "@zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-macos//:zig_toolchain", - "@zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-windows//:zig_toolchain", - "@zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-linux//:zig_toolchain", - "@zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-macos//:zig_toolchain", - "@zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-windows//:zig_toolchain" - ], - "zig_versions": [ - "0.16.0-dev.1912+0cbaaa5eb", - "0.16.0-dev.1912+0cbaaa5eb", - "0.16.0-dev.1912+0cbaaa5eb", - "0.16.0-dev.1912+0cbaaa5eb", - "0.16.0-dev.1912+0cbaaa5eb", - "0.16.0-dev.1912+0cbaaa5eb" - ], - "exec_lengths": [ - 2, - 2, - 2, - 2, - 2, - 2 - ], - "exec_constraints": [ - "@platforms//os:linux", - "@platforms//cpu:aarch64", - "@platforms//os:macos", - "@platforms//cpu:aarch64", - "@platforms//os:windows", - "@platforms//cpu:aarch64", - "@platforms//os:linux", - "@platforms//cpu:x86_64", - "@platforms//os:macos", - "@platforms//cpu:x86_64", - "@platforms//os:windows", - "@platforms//cpu:x86_64" - ] - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_zig+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "rules_zig+", - "bazel_tools", - "bazel_tools" - ] - ] - } - } - } + "moduleExtensions": {}, + "facts": {} } diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 9a2a9937..3a26bb06 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -9,6 +9,13 @@ const stdx = zml.stdx; const log = std.log.scoped(.llama); +pub const TransferCtx = struct { + read_pool: *zml.io.ConcurrentBufferPool, + write_pool: *zml.io.ConcurrentBufferPool, + transferred_bytes: *usize, + progress: *std.Progress.Node, +}; + /// Llama architecture, using huggingface transformers naming. /// Dimensions of activations: {.b, .s, .d} pub const LlamaLM = struct { @@ -61,15 +68,25 @@ pub const LlamaLM = struct { self.model.deinit(allocator); } - pub fn loadBuffers(self: *const LlamaLM, allocator: std.mem.Allocator, io: std.Io, store: zml.io.TensorStore.View, platform: zml.Platform) !zml.Bufferized(LlamaLM) { - const lm_head = if (self.lm_head) |lm_head| try zml.io.loadBuffersFromId(allocator, io, lm_head, store.withPrefix("lm_head"), platform) else null; - // TODO(Corentin): errdefer + pub fn loadBuffers( + self: *const LlamaLM, + bufferize_ctx: zml.io.BufferizeContext(TransferCtx), + group: *zml.stdx.Io.AllocatingLimitedConcurrentGroup, + store: zml.io.TensorStore.View, + cb: zml.io.CallbackTensorBufferTransfer(TransferCtx), + ) !zml.Bufferized(LlamaLM) { + var lm_head_bufferized: ?zml.Bufferized(zml.nn.Linear) = null; + if (self.lm_head) |lm_head| { + lm_head_bufferized = undefined; - const model = try self.model.loadBuffers(allocator, io, store.withPrefix("model"), platform); + const transfers = try zml.io.bufferize(TransferCtx, bufferize_ctx, &lm_head, &lm_head_bufferized.?, store.withPrefix("lm_head")); + for (transfers) |t| try group.concurrent(bufferize_ctx.io, cb, .{t}); + } + const model_bufferized = try self.model.loadBuffers(bufferize_ctx, group, store.withPrefix("model"), cb); return .{ - .lm_head = lm_head, - .model = model, + .lm_head = lm_head_bufferized, + .model = model_bufferized, }; } @@ -154,19 +171,29 @@ pub const Llama = struct { allocator.free(self.layers); } - pub fn loadBuffers(self: *const Llama, allocator: std.mem.Allocator, io: std.Io, store: zml.io.TensorStore.View, platform: zml.Platform) !zml.Bufferized(Llama) { - const layers = try allocator.alloc(zml.Bufferized(TransformerLayer), self.layers.len); - errdefer allocator.free(layers); + pub fn loadBuffers( + self: *const Llama, + bufferize_ctx: zml.io.BufferizeContext(TransferCtx), + group: *zml.stdx.Io.AllocatingLimitedConcurrentGroup, + store: zml.io.TensorStore.View, + cb: zml.io.CallbackTensorBufferTransfer(TransferCtx), + ) !zml.Bufferized(Llama) { + const bufferized_layers = try bufferize_ctx.allocator.alloc(zml.Bufferized(TransformerLayer), self.layers.len); - for (layers, 0..) |*layer, i| { - layer.* = try zml.io.loadBuffersFromId(allocator, io, self.layers[i], store.withLayer(i), platform); + var bufferized: zml.Bufferized(Llama) = .{ .embed_tokens = undefined, .norm = undefined, .layers = bufferized_layers }; + + const transfers = try zml.io.bufferize(TransferCtx, bufferize_ctx, &self.embed_tokens, &bufferized.embed_tokens, store.withPrefix("embed_tokens")); + for (transfers) |t| try group.concurrent(bufferize_ctx.io, cb, .{t}); + + const norm_transfers = try zml.io.bufferize(TransferCtx, bufferize_ctx, &self.norm, &bufferized.norm, store.withPrefix("norm")); + for (norm_transfers) |t| try group.concurrent(bufferize_ctx.io, cb, .{t}); + + for (self.layers, bufferized_layers, 0..) |layer, *buf_layer, i| { + const layer_transfers = try zml.io.bufferize(TransferCtx, bufferize_ctx, &layer, buf_layer, store.withPrefix("layers").withLayer(i)); + for (layer_transfers) |t| try group.concurrent(bufferize_ctx.io, cb, .{t}); } - return .{ - .embed_tokens = try zml.io.loadBuffersFromId(allocator, io, self.embed_tokens, store.withPrefix("embed_tokens"), platform), - .layers = layers, - .norm = try zml.io.loadBuffersFromId(allocator, io, self.norm, store.withPrefix("norm"), platform), - }; + return bufferized; } pub fn unloadBuffers(self: *zml.Bufferized(Llama), allocator: std.mem.Allocator) void { diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 91b218dc..30ce7929 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -21,19 +21,26 @@ pub const std_options: std.Options = .{ pub const Args = struct { model: ?[]const u8 = null, + prompt: ?[]const u8 = null, seqlen: u32 = 512, + reader_buffer_size_mb: u32 = 32, + writer_buffer_size_mb: u32 = 32, + async_limit: ?usize = null, }; pub fn main() !void { log.info("LLama was compiled with {}", .{@import("builtin").mode}); - const allocator = std.heap.c_allocator; + var debug_allocator: ?std.heap.DebugAllocator(.{}) = null; + const allocator = if (@import("builtin").mode == .Debug) blk: { + debug_allocator = .init; + break :blk debug_allocator.?.allocator(); + } else std.heap.c_allocator; + defer if (debug_allocator) |*da| std.debug.assert(da.deinit() == .ok); var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); - const io = threaded.io(); - const args: Args = blk: { var ret: Args = .{}; var it = std.process.args(); @@ -43,6 +50,14 @@ pub fn main() !void { ret.model = arg["--model=".len..]; } else if (std.mem.startsWith(u8, arg, "--seqlen=")) { ret.seqlen = try std.fmt.parseUnsigned(u32, arg["--seqlen=".len..], 10); + } else if (std.mem.startsWith(u8, arg, "--prompt=")) { + ret.prompt = arg["--prompt=".len..]; + } else if (std.mem.startsWith(u8, arg, "--reader-buffer-size-mb=")) { + ret.reader_buffer_size_mb = try std.fmt.parseUnsigned(u32, arg["--reader-buffer-size-mb=".len..], 10); + } else if (std.mem.startsWith(u8, arg, "--writer-buffer-size-mb=")) { + ret.writer_buffer_size_mb = try std.fmt.parseUnsigned(u32, arg["--writer-buffer-size-mb=".len..], 10); + } else if (std.mem.startsWith(u8, arg, "--async-limit=")) { + ret.async_limit = try std.fmt.parseUnsigned(usize, arg["--async-limit=".len..], 10); } } @@ -54,11 +69,66 @@ pub fn main() !void { break :blk ret; }; - log.info("Resolving Model repo", .{}); - var repo = try zml.safetensors.resolveModelRepo(allocator, io, args.model.?); - defer repo.deinit(allocator, io); + if (args.async_limit) |limit| threaded.setAsyncLimit(.limited(limit)); - const parsed_config = try parseConfig(allocator, io, repo.dir); + log.info("Running with threaded io async limit set to {?d}", .{threaded.async_limit.toInt()}); + + var http_client: std.http.Client = .{ + .allocator = allocator, + .io = threaded.io(), + .connection_pool = .{ + .free_size = threaded.async_limit.toInt() orelse 16, + }, + }; + + try http_client.initDefaultProxies(allocator); + defer http_client.deinit(); + + var vfs_file: zml.io.VFS.File = .init( + allocator, + threaded.io(), + .{ + .direct_io = true, + .direct_io_alignment = .fromByteUnits(4 * 1024), + }, + ); + defer vfs_file.deinit(); + + var vfs_https: zml.io.VFS.HTTP = try .init(allocator, threaded.io(), .{ + .http_client = &http_client, + .protocol = .https, + }); + defer vfs_https.deinit(); + + var hf_auth: zml.io.VFS.HF.Auth = try .auto(allocator, threaded.io()); + defer hf_auth.deinit(allocator); + + var hf_vfs: zml.io.VFS.HF = try .init( + allocator, + threaded.io(), + .{ + .http_client = &http_client, + .auth = hf_auth, + }, + ); + defer hf_vfs.deinit(); + + var vfs: zml.io.VFS = try .init(allocator, threaded.io()); + defer vfs.deinit(); + + try vfs.register("file", vfs_file.io()); + try vfs.register("https", vfs_https.io()); + try vfs.register("hf", hf_vfs.io()); + + const io = vfs.io(); + + zml.init(); + defer zml.deinit(); + + log.info("Resolving model repo", .{}); + const repo = try zml.safetensors.resolveModelRepo(io, args.model.?); + + const parsed_config = try parseConfig(allocator, io, repo); defer parsed_config.deinit(); const config = parsed_config.value; @@ -96,7 +166,7 @@ pub fn main() !void { .rng = .init(), }; - var tokenizer_future = io.async(loadTokenizer, .{ allocator, io, repo.dir }); + var tokenizer_future = io.async(loadTokenizer, .{ allocator, io, repo }); errdefer blk: { var v = tokenizer_future.cancel(io) catch break :blk; v.deinit(); @@ -116,7 +186,33 @@ pub fn main() !void { defer v.decode_exe.deinit(); } else |_| {}; - var llama_buffers_future = io.async(loadModelBuffers, .{ allocator, io, platform, &store, llama_model }); + var load_model_buffers: zml.stdx.Io.AllocatingLimitedConcurrentGroup = try .init(allocator, threaded.async_limit.toInt() orelse 16); + defer { + load_model_buffers.cancel(io); + load_model_buffers.deinit(); + } + + var read_pool, var write_pool = try zml.io.ConcurrentBufferPool.initRW( + allocator, + platform, + .{ + .size = args.reader_buffer_size_mb * 1024 * 1024, + .concurrency = load_model_buffers.limit, + .dma = true, + }, + .{ + .size = args.writer_buffer_size_mb * 1024 * 1024, + .concurrency = load_model_buffers.limit, + .dma = true, + }, + ); + defer { + read_pool.deinit(); + write_pool.deinit(); + } + + var progress = std.Progress.start(io, .{ .root_name = args.model.?, .estimated_total_items = store.view().count() }); + var llama_buffers_future = io.async(loadModelBuffers, .{ allocator, io, &progress, &load_model_buffers, &read_pool, &write_pool, platform, &store, llama_model }); errdefer b: { var v = llama_buffers_future.cancel(io) catch break :b; LlamaLM.unloadBuffers(&v, allocator); @@ -128,6 +224,7 @@ pub fn main() !void { var llama_buffers = try llama_buffers_future.await(io); defer LlamaLM.unloadBuffers(&llama_buffers, allocator); + progress.end(); log.info("Creating KvCache", .{}); var kv_cache_buffers = try llama_parameters.kv_cache.initBuffer(io, platform); @@ -136,13 +233,11 @@ pub fn main() !void { var tokenizer = try tokenizer_future.await(io); defer tokenizer.deinit(); - const prompt = blk: { + const prompt = if (args.prompt) |p| p else blk: { var reader = std.Io.File.stdin().reader(io, &.{}); break :blk try reader.interface.allocRemaining(allocator, .unlimited); }; - defer allocator.free(prompt); - // const prompt = cli.args.prompt orelse "What is the capital of France?"; log.info("✅\tPrompt: {s}", .{prompt}); // Unbuffered writing of the tokens to stdout. @@ -245,11 +340,77 @@ fn compileModel(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform return .{ .prefill_exe = prefill_exe, .decode_exe = decode_exe }; } -fn loadModelBuffers(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform, store: *zml.io.TensorStore, llama_model: llama.LlamaLM) !zml.Bufferized(llama.LlamaLM) { - var timer = try std.time.Timer.start(); +fn loadModelBuffers( + allocator: std.mem.Allocator, + io: std.Io, + progress: *std.Progress.Node, + group: *zml.stdx.Io.AllocatingLimitedConcurrentGroup, + read_pool: *zml.io.ConcurrentBufferPool, + write_pool: *zml.io.ConcurrentBufferPool, + platform: zml.Platform, + store: *zml.io.TensorStore, + llama_model: llama.LlamaLM, +) !zml.Bufferized(llama.LlamaLM) { + var transferred_bytes: usize = 0; + var timer = try stdx.time.Timer.start(); log.info("Loading model", .{}); - defer log.info("Loaded model [{D}]", .{timer.read()}); - return llama_model.loadBuffers(allocator, io, store.view(), platform); + defer { + const duration = timer.read(); + const seconds = @as(f64, @floatFromInt(duration.ns)) / 1e9; + const gb_per_sec = @as(f64, @floatFromInt(transferred_bytes)) / (1024.0 * 1024.0 * 1024.0) / seconds; + const gbps = gb_per_sec * 8.0; + log.info("Loaded model [{D} {d:.3} GB/s {d:.3} gbps]", .{ duration, gb_per_sec, gbps }); + } + + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + const bufferize_ctx: zml.io.BufferizeContext(llama.TransferCtx) = .{ + .allocator = allocator, + .arena = &arena, + .io = io, + .platform = platform, + .cb_ctx = .{ .read_pool = read_pool, .write_pool = write_pool, .transferred_bytes = &transferred_bytes, .progress = progress }, + }; + + const bufferized = llama_model.loadBuffers( + bufferize_ctx, + group, + store.view(), + transferBuffer, + ); + + try group.await(io); + + return bufferized; +} + +fn transferBuffer(ctx: zml.io.TensorBufferTransfer(llama.TransferCtx)) !void { + const read_pool = ctx.cb_ctx.read_pool; + const write_pool = ctx.cb_ctx.write_pool; + const progress = ctx.cb_ctx.progress; + + const read_buffer = read_pool.acquire(ctx.io) catch unreachable; + defer read_pool.release(ctx.io, read_buffer) catch {}; + + const write_buffer = write_pool.acquire(ctx.io) catch unreachable; + defer write_pool.release(ctx.io, write_buffer) catch {}; + + var transfer_progress = progress.start(ctx.tensor.name, ctx.tensor.byteSize()); + + var reader = zml.safetensors.TensorReader.init(ctx.io, ctx.tensor, read_buffer) catch unreachable; + defer reader.deinit(); + + var writer = zml.io.DeviceWriter.init(ctx.io, &transfer_progress, ctx.platform, ctx.buffer, .device, write_buffer) catch unreachable; + defer { + writer.interface.flush() catch unreachable; + writer.deinit(); + } + + _ = reader.interface.streamRemaining(&writer.interface) catch unreachable; + + transfer_progress.end(); + ctx.cb_ctx.transferred_bytes.* += ctx.tensor.byteSize(); } pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: LlamaLM.Config, prompt: []const u8, skip_llama3_encoding: bool) ![]u32 { diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 49633891..b6888dec 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -37,8 +37,16 @@ const Mnist = struct { }; } - pub fn loadBuffers(self: Mnist, allocator: std.mem.Allocator, io: std.Io, store: zml.io.TensorStore.View, platform: zml.Platform) !zml.Bufferized(Mnist) { - return zml.io.loadBuffersFromId(allocator, io, self, store, platform); + pub fn loadBuffers( + self: Mnist, + allocator: std.mem.Allocator, + io: std.Io, + platform: zml.Platform, + store: zml.io.TensorStore.View, + read_pool_config: zml.io.ConcurrentBufferPool.Config, + write_pool_config: zml.io.ConcurrentBufferPool.Config, + ) !zml.Bufferized(Mnist) { + return zml.io.loadBuffersFromId(allocator, io, platform, self, store, read_pool_config, write_pool_config); } pub fn unloadBuffers(self: *zml.Bufferized(Mnist)) void { @@ -50,7 +58,6 @@ const Mnist = struct { /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { - // std.log.info("Compiling for target: {s}", .{@tagName(input.getContext().target())}); var x = input.flatten().convert(.f32).withTags(.{.d}); const layers: []const Layer = &.{ self.fc1, self.fc2 }; for (layers) |layer| { @@ -62,24 +69,18 @@ const Mnist = struct { pub fn main() !void { const allocator = std.heap.c_allocator; - //var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - //defer _ = gpa.deinit(); - - //const allocator = gpa.allocator(); var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); - // var vfs_file: zml.io.VFS.File = .init(allocator, threaded.io(), .{}); + var vfs_file: zml.io.VFS.File = .init(allocator, threaded.io(), .{}); - // var vfs: zml.io.VFS = .init(allocator, threaded.io()); - // defer vfs.deinit(); + var vfs: zml.io.VFS = try .init(allocator, threaded.io()); + defer vfs.deinit(); - // try vfs.register("file", vfs_file.io()); + try vfs.register("file", vfs_file.io()); - // const io = vfs.io(); - // - const io = threaded.io(); + const io = vfs.io(); // Parse program args const process_args = try std.process.argsAlloc(allocator); @@ -114,7 +115,11 @@ pub fn main() !void { log.info("Transfering weights....", .{}); var timer = try std.time.Timer.start(); defer log.info("✅ Transferred weights in {D}", .{timer.read()}); - break :blk try mnist_model.loadBuffers(allocator, io, store.view(), platform); + + const read_pool_config: zml.io.ConcurrentBufferPool.Config = .{ .size = 64 * 1024 * 1024, .concurrency = 10, .dma = true }; + const write_pool_config: zml.io.ConcurrentBufferPool.Config = .{ .size = 32 * 1024 * 1024, .concurrency = 10, .dma = true }; + + break :blk try mnist_model.loadBuffers(allocator, io, platform, store.view(), read_pool_config, write_pool_config); }; defer Mnist.unloadBuffers(&mnist_buffers); diff --git a/examples/vfs/BUILD.bazel b/examples/vfs/BUILD.bazel new file mode 100644 index 00000000..aa3a533f --- /dev/null +++ b/examples/vfs/BUILD.bazel @@ -0,0 +1,15 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load("@rules_zig//zig:defs.bzl", "zig_binary") + +zig_binary( + name = "playground", + args = [], + data = [], + main = "playground.zig", + deps = ["//zml"], +) + +build_test( + name = "test", + targets = [":mnist"], +) diff --git a/examples/vfs/playground.zig b/examples/vfs/playground.zig new file mode 100644 index 00000000..3633845d --- /dev/null +++ b/examples/vfs/playground.zig @@ -0,0 +1,209 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const log = std.log.scoped(.vfs); + +pub const std_options: std.Options = .{ + .log_level = .info, +}; + +pub const Command = union(enum) { + cat, + tree, + stat, +}; + +pub const Args = struct { + command: ?Command = null, + path: ?[]const u8 = null, + reader_buffer_size_mb: u32 = 32, + async_limit: ?usize = null, +}; + +// -- cat https://iprs.fly.dev +pub fn main() !void { + log.info("VFS playground was compiled with {}", .{@import("builtin").mode}); + + var debug_allocator: ?std.heap.DebugAllocator(.{}) = null; + const allocator = if (@import("builtin").mode == .Debug) blk: { + debug_allocator = .init; + break :blk debug_allocator.?.allocator(); + } else std.heap.c_allocator; + defer if (debug_allocator) |*da| std.debug.assert(da.deinit() == .ok); + + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + + const args: Args = blk: { + var ret: Args = .{}; + var it = std.process.args(); + defer it.deinit(); + var index: usize = 0; + while (it.next()) |arg| { + if (std.mem.startsWith(u8, arg, "--reader-buffer-size-mb=")) { + ret.reader_buffer_size_mb = try std.fmt.parseUnsigned(u32, arg["--reader-buffer-size-mb=".len..], 10); + } else if (std.mem.startsWith(u8, arg, "--async-limit=")) { + ret.async_limit = try std.fmt.parseUnsigned(usize, arg["--async-limit=".len..], 10); + } else { + if (index == 0) { + // skip program name + } else if (index == 1) { + if (std.mem.eql(u8, arg, "cat")) { + ret.command = .cat; + } else if (std.mem.eql(u8, arg, "stat")) { + ret.command = .stat; + } else if (std.mem.eql(u8, arg, "tree")) { + ret.command = .tree; + } else { + log.err("Unknown command: {s}", .{arg}); + return; + } + } else if (index == 2) { + ret.path = arg; + } else { + log.err("Unknown argument: {s}", .{arg}); + return; + } + index += 1; + } + } + + if (ret.command == null) { + log.err("Missing command as argument", .{}); + return; + } + + if (ret.path == null) { + log.err("Missing path as argument", .{}); + return; + } + + break :blk ret; + }; + + if (args.async_limit) |limit| threaded.setAsyncLimit(.limited(limit)); + + log.info("Running with threaded io async limit set to {?d}", .{threaded.async_limit.toInt()}); + + var http_client: std.http.Client = .{ + .allocator = allocator, + .io = threaded.io(), + .connection_pool = .{ + .free_size = threaded.async_limit.toInt() orelse 16, + }, + }; + + try http_client.initDefaultProxies(allocator); + defer http_client.deinit(); + + var vfs_file: zml.io.VFS.File = .init( + allocator, + threaded.io(), + .{ + .direct_io = true, + .direct_io_alignment = .fromByteUnits(4 * 1024), + }, + ); + defer vfs_file.deinit(); + + var vfs_https: zml.io.VFS.HTTP = try .init(allocator, threaded.io(), .{ + .http_client = &http_client, + .protocol = .https, + }); + defer vfs_https.deinit(); + + var hf_auth: zml.io.VFS.HF.Auth = try .auto(allocator, threaded.io()); + defer hf_auth.deinit(allocator); + + var hf_vfs: zml.io.VFS.HF = try .init( + allocator, + threaded.io(), + .{ + .http_client = &http_client, + .auth = hf_auth, + }, + ); + defer hf_vfs.deinit(); + + var vfs: zml.io.VFS = try .init(allocator, threaded.io()); + defer vfs.deinit(); + + try vfs.register("file", vfs_file.io()); + try vfs.register("https", vfs_https.io()); + try vfs.register("hf", hf_vfs.io()); + + const io = vfs.io(); + + const path = args.path.?; + + const buffer = try allocator.alloc(u8, args.reader_buffer_size_mb * 1024 * 1024); + defer allocator.free(buffer); + + var stdout_writer = std.Io.File.stdout().writer(io, buffer); + defer stdout_writer.interface.flush() catch {}; + + switch (args.command.?) { + .cat => { + var file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); + defer file.close(io); + + var reader = file.reader(io, &.{}); + const read = try reader.interface.streamRemaining(&stdout_writer.interface); + _ = try stdout_writer.interface.write("\n"); + + log.info("Wrote {d} bytes to stdout from {s}", .{ read, path }); + }, + .tree => { + var dir = try std.Io.Dir.openDir(.cwd(), io, path, .{ .iterate = true }); + defer dir.close(io); + + try printTree(io, &stdout_writer.interface, dir, 0, 10); + }, + .stat => { + const stat = std.Io.Dir.statFile(.cwd(), io, path, .{}) catch |err| blk: { + if (err == error.IsDir) { + var dir = try std.Io.Dir.openDir(.cwd(), io, path, .{}); + defer dir.close(io); + + break :blk try dir.stat(io); + } + return err; + }; + + log.info("Stat for {s}: size={d} bytes, kind={s}", .{ path, stat.size, @tagName(stat.kind) }); + }, + } +} + +fn printTree(io: std.Io, writer: *std.Io.Writer, dir: std.Io.Dir, depth: usize, max_depth: usize) !void { + if (depth > max_depth) return; + + var it = dir.iterate(); + while (try it.next(io)) |entry| { + for (0..depth) |_| { + try writer.writeAll(" "); + } + + const size = if (entry.kind == .file) blk: { + const stat = try dir.statFile(io, entry.name, .{ .follow_symlinks = true }); + break :blk stat.size; + } else if (entry.kind == .directory) blk: { + const stat = try dir.stat(io); + break :blk stat.size; + } else { + return error.UnexpectedEntryKind; + }; + + var buf: [256]u8 = undefined; + const slice = try std.fmt.bufPrint(&buf, "- {s} kind={s} size={d}\n", .{ entry.name, @tagName(entry.kind), size }); + try writer.writeAll(slice); + + if (entry.kind == .directory) { + var sub_dir = try dir.openDir(io, entry.name, .{ .iterate = true }); + defer sub_dir.close(io); + + try printTree(io, writer, sub_dir, depth + 1, max_depth); + } + } +} diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 7b07b3a9..b6cbcaea 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -524,7 +524,7 @@ pub const Client = opaque { } pub fn dmaMap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { - try api.call(.PJRT_Client_DmaMap, .{ + _ = try api.call(.PJRT_Client_DmaMap, .{ .client = self.inner(), .data = @ptrCast(@constCast(data)), .size = @intCast(data.len), @@ -532,7 +532,7 @@ pub const Client = opaque { } pub fn dmaUnmap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { - try api.call(.PJRT_Client_DmaUnmap, .{ + _ = try api.call(.PJRT_Client_DmaUnmap, .{ .client = self.inner(), .data = @ptrCast(@constCast(data)), }); diff --git a/stdx/BUILD.bazel b/stdx/BUILD.bazel index 5f44c3ed..bf5b5607 100644 --- a/stdx/BUILD.bazel +++ b/stdx/BUILD.bazel @@ -13,14 +13,15 @@ zig_library( "meta.zig", "process.zig", "queue.zig", + "segmented_list.zig", "signature.zig", "time.zig", "unicode.zig", ], - zigopts = ["-lc"], main = "stdx.zig", visibility = ["//visibility:public"], zigopts = ["-lc"], + zigopts = ["-lc"], ) zig_test( diff --git a/stdx/Io.zig b/stdx/Io.zig index 52319906..5f73bd05 100644 --- a/stdx/Io.zig +++ b/stdx/Io.zig @@ -21,3 +21,104 @@ pub const Dir = struct { return buffer; } }; + +pub fn LimitedConcurrentGroup(comptime limit: usize) type { + return struct { + const Self = @This(); + + queue_buffer: [limit]bool = undefined, + + group: std.Io.Group = .init, + queue: std.Io.Queue(bool) = undefined, + + pub fn init() Self { + var s: Self = .{}; + s.queue = std.Io.Queue(bool).init(&s.queue_buffer); + return s; + } + + pub fn concurrent( + self: *Self, + io: std.Io, + comptime function: anytype, + args: std.meta.ArgsTuple(@TypeOf(function)), + ) !void { + self.queue.putOneUncancelable(io, true) catch unreachable; + + try self.group.concurrent(io, struct { + fn wrapper( + io_: std.Io, + queue: *std.Io.Queue(bool), + inner_args: std.meta.ArgsTuple(@TypeOf(function)), + ) !void { + defer _ = queue.getOneUncancelable(io_) catch unreachable; + return @call(.auto, function, inner_args); + } + }.wrapper, .{ io, &self.queue, args }); + } + + pub fn await(self: *AllocatingLimitedConcurrentGroup, io: std.Io) std.Io.Cancelable!void { + try self.group.await(io); + } + + pub fn cancel(self: *Self, io: std.Io) void { + self.group.cancel(io); + } + }; +} + +pub const AllocatingLimitedConcurrentGroup = struct { + allocator: std.mem.Allocator, + + queue_buffer: []bool = &[_]bool{}, + limit: usize, + + group: std.Io.Group = .init, + queue: std.Io.Queue(bool) = undefined, + + pub fn init(allocator: std.mem.Allocator, limit: usize) std.mem.Allocator.Error!AllocatingLimitedConcurrentGroup { + const buffer = try allocator.alloc(bool, limit); + @memset(buffer, false); + var self: AllocatingLimitedConcurrentGroup = .{ + .allocator = allocator, + .queue_buffer = buffer, + .limit = limit, + .group = .init, + .queue = undefined, + }; + self.queue = std.Io.Queue(bool).init(self.queue_buffer); + return self; + } + + pub fn deinit(self: *AllocatingLimitedConcurrentGroup) void { + self.allocator.free(self.queue_buffer); + } + + pub fn concurrent( + self: *AllocatingLimitedConcurrentGroup, + io: std.Io, + comptime function: anytype, + args: std.meta.ArgsTuple(@TypeOf(function)), + ) !void { + self.queue.putOneUncancelable(io, true) catch unreachable; + + try self.group.concurrent(io, struct { + fn wrapper( + io_: std.Io, + queue: *std.Io.Queue(bool), + inner_args: std.meta.ArgsTuple(@TypeOf(function)), + ) !void { + defer _ = queue.getOneUncancelable(io_) catch unreachable; + return @call(.auto, function, inner_args); + } + }.wrapper, .{ io, &self.queue, args }); + } + + pub fn await(self: *AllocatingLimitedConcurrentGroup, io: std.Io) std.Io.Cancelable!void { + try self.group.await(io); + } + + pub fn cancel(self: *AllocatingLimitedConcurrentGroup, io: std.Io) void { + self.group.cancel(io); + } +}; diff --git a/stdx/segmented_list.zig b/stdx/segmented_list.zig new file mode 100644 index 00000000..ee115ce2 --- /dev/null +++ b/stdx/segmented_list.zig @@ -0,0 +1,539 @@ +// Origin: https://github.com/ziglang/std-lib-orphanage/ + +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. + +const std = @import("std"); +const assert = std.debug.assert; +const testing = std.testing; +const mem = std.mem; +const Allocator = std.mem.Allocator; + +// Imagine that `fn at(self: *Self, index: usize) &T` is a customer asking for a box +// from a warehouse, based on a flat array, boxes ordered from 0 to N - 1. +// But the warehouse actually stores boxes in shelves of increasing powers of 2 sizes. +// So when the customer requests a box index, we have to translate it to shelf index +// and box index within that shelf. Illustration: +// +// customer indexes: +// shelf 0: 0 +// shelf 1: 1 2 +// shelf 2: 3 4 5 6 +// shelf 3: 7 8 9 10 11 12 13 14 +// shelf 4: 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 +// shelf 5: 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 +// ... +// +// warehouse indexes: +// shelf 0: 0 +// shelf 1: 0 1 +// shelf 2: 0 1 2 3 +// shelf 3: 0 1 2 3 4 5 6 7 +// shelf 4: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 +// shelf 5: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 +// ... +// +// With this arrangement, here are the equations to get the shelf index and +// box index based on customer box index: +// +// shelf_index = floor(log2(customer_index + 1)) +// shelf_count = ceil(log2(box_count + 1)) +// box_index = customer_index + 1 - 2 ** shelf +// shelf_size = 2 ** shelf_index +// +// Now we complicate it a little bit further by adding a preallocated shelf, which must be +// a power of 2: +// prealloc=4 +// +// customer indexes: +// prealloc: 0 1 2 3 +// shelf 0: 4 5 6 7 8 9 10 11 +// shelf 1: 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 +// shelf 2: 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 +// ... +// +// warehouse indexes: +// prealloc: 0 1 2 3 +// shelf 0: 0 1 2 3 4 5 6 7 +// shelf 1: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 +// shelf 2: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 +// ... +// +// Now the equations are: +// +// shelf_index = floor(log2(customer_index + prealloc)) - log2(prealloc) - 1 +// shelf_count = ceil(log2(box_count + prealloc)) - log2(prealloc) - 1 +// box_index = customer_index + prealloc - 2 ** (log2(prealloc) + 1 + shelf) +// shelf_size = prealloc * 2 ** (shelf_index + 1) + +/// This is a stack data structure where pointers to indexes have the same lifetime as the data structure +/// itself, unlike ArrayList where append() invalidates all existing element pointers. +/// The tradeoff is that elements are not guaranteed to be contiguous. For that, use ArrayList. +/// Note however that most elements are contiguous, making this data structure cache-friendly. +/// +/// Because it never has to copy elements from an old location to a new location, it does not require +/// its elements to be copyable, and it avoids wasting memory when backed by an ArenaAllocator. +/// Note that the append() and pop() convenience methods perform a copy, but you can instead use +/// addOne(), at(), setCapacity(), and shrinkCapacity() to avoid copying items. +/// +/// This data structure has O(1) append and O(1) pop. +/// +/// It supports preallocated elements, making it especially well suited when the expected maximum +/// size is small. `prealloc_item_count` must be 0, or a power of 2. +pub fn SegmentedList(comptime T: type, comptime prealloc_item_count: usize) type { + return struct { + const Self = @This(); + const ShelfIndex = std.math.Log2Int(usize); + + const prealloc_exp: ShelfIndex = blk: { + // we don't use the prealloc_exp constant when prealloc_item_count is 0 + // but lazy-init may still be triggered by other code so supply a value + if (prealloc_item_count == 0) { + break :blk 0; + } else { + assert(std.math.isPowerOfTwo(prealloc_item_count)); + const value = std.math.log2_int(usize, prealloc_item_count); + break :blk value; + } + }; + + prealloc_segment: [prealloc_item_count]T = undefined, + dynamic_segments: [][*]T = &[_][*]T{}, + len: usize = 0, + + pub const prealloc_count = prealloc_item_count; + + fn AtType(comptime SelfType: type) type { + if (@typeInfo(SelfType).pointer.is_const) { + return *const T; + } else { + return *T; + } + } + + pub fn deinit(self: *Self, allocator: Allocator) void { + self.freeShelves(allocator, @as(ShelfIndex, @intCast(self.dynamic_segments.len)), 0); + allocator.free(self.dynamic_segments); + self.* = undefined; + } + + pub fn at(self: anytype, i: usize) AtType(@TypeOf(self)) { + assert(i < self.len); + return self.uncheckedAt(i); + } + + pub fn count(self: Self) usize { + return self.len; + } + + pub fn append(self: *Self, allocator: Allocator, item: T) Allocator.Error!void { + const new_item_ptr = try self.addOne(allocator); + new_item_ptr.* = item; + } + + pub fn appendSlice(self: *Self, allocator: Allocator, items: []const T) Allocator.Error!void { + for (items) |item| { + try self.append(allocator, item); + } + } + + pub fn pop(self: *Self) ?T { + if (self.len == 0) return null; + + const index = self.len - 1; + const result = uncheckedAt(self, index).*; + self.len = index; + return result; + } + + pub fn addOne(self: *Self, allocator: Allocator) Allocator.Error!*T { + const new_length = self.len + 1; + try self.growCapacity(allocator, new_length); + const result = uncheckedAt(self, self.len); + self.len = new_length; + return result; + } + + /// Reduce length to `new_len`. + /// Invalidates pointers for the elements at index new_len and beyond. + pub fn shrinkRetainingCapacity(self: *Self, new_len: usize) void { + assert(new_len <= self.len); + self.len = new_len; + } + + /// Invalidates all element pointers. + pub fn clearRetainingCapacity(self: *Self) void { + self.len = 0; + } + + /// Invalidates all element pointers. + pub fn clearAndFree(self: *Self, allocator: Allocator) void { + self.setCapacity(allocator, 0) catch unreachable; + self.len = 0; + } + + /// Grows or shrinks capacity to match usage. + /// TODO update this and related methods to match the conventions set by ArrayList + pub fn setCapacity(self: *Self, allocator: Allocator, new_capacity: usize) Allocator.Error!void { + if (prealloc_item_count != 0) { + if (new_capacity <= @as(usize, 1) << (prealloc_exp + @as(ShelfIndex, @intCast(self.dynamic_segments.len)))) { + return self.shrinkCapacity(allocator, new_capacity); + } + } + return self.growCapacity(allocator, new_capacity); + } + + /// Only grows capacity, or retains current capacity. + pub fn growCapacity(self: *Self, allocator: Allocator, new_capacity: usize) Allocator.Error!void { + const new_cap_shelf_count = shelfCount(new_capacity); + const old_shelf_count = @as(ShelfIndex, @intCast(self.dynamic_segments.len)); + if (new_cap_shelf_count <= old_shelf_count) return; + + const new_dynamic_segments = try allocator.alloc([*]T, new_cap_shelf_count); + errdefer allocator.free(new_dynamic_segments); + + var i: ShelfIndex = 0; + while (i < old_shelf_count) : (i += 1) { + new_dynamic_segments[i] = self.dynamic_segments[i]; + } + errdefer while (i > old_shelf_count) : (i -= 1) { + allocator.free(new_dynamic_segments[i][0..shelfSize(i)]); + }; + while (i < new_cap_shelf_count) : (i += 1) { + new_dynamic_segments[i] = (try allocator.alloc(T, shelfSize(i))).ptr; + } + + allocator.free(self.dynamic_segments); + self.dynamic_segments = new_dynamic_segments; + } + + /// Only shrinks capacity or retains current capacity. + /// It may fail to reduce the capacity in which case the capacity will remain unchanged. + pub fn shrinkCapacity(self: *Self, allocator: Allocator, new_capacity: usize) void { + if (new_capacity <= prealloc_item_count) { + const len = @as(ShelfIndex, @intCast(self.dynamic_segments.len)); + self.freeShelves(allocator, len, 0); + allocator.free(self.dynamic_segments); + self.dynamic_segments = &[_][*]T{}; + return; + } + + const new_cap_shelf_count = shelfCount(new_capacity); + const old_shelf_count = @as(ShelfIndex, @intCast(self.dynamic_segments.len)); + assert(new_cap_shelf_count <= old_shelf_count); + if (new_cap_shelf_count == old_shelf_count) return; + + // freeShelves() must be called before resizing the dynamic + // segments, but we don't know if resizing the dynamic segments + // will work until we try it. So we must allocate a fresh memory + // buffer in order to reduce capacity. + const new_dynamic_segments = allocator.alloc([*]T, new_cap_shelf_count) catch return; + self.freeShelves(allocator, old_shelf_count, new_cap_shelf_count); + if (allocator.resize(self.dynamic_segments, new_cap_shelf_count)) { + // We didn't need the new memory allocation after all. + self.dynamic_segments = self.dynamic_segments[0..new_cap_shelf_count]; + allocator.free(new_dynamic_segments); + } else { + // Good thing we allocated that new memory slice. + @memcpy(new_dynamic_segments, self.dynamic_segments[0..new_cap_shelf_count]); + allocator.free(self.dynamic_segments); + self.dynamic_segments = new_dynamic_segments; + } + } + + pub fn shrink(self: *Self, new_len: usize) void { + assert(new_len <= self.len); + // TODO take advantage of the new realloc semantics + self.len = new_len; + } + + pub fn writeToSlice(self: *Self, dest: []T, start: usize) void { + const end = start + dest.len; + assert(end <= self.len); + + var i = start; + if (end <= prealloc_item_count) { + const src = self.prealloc_segment[i..end]; + @memcpy(dest[i - start ..][0..src.len], src); + return; + } else if (i < prealloc_item_count) { + const src = self.prealloc_segment[i..]; + @memcpy(dest[i - start ..][0..src.len], src); + i = prealloc_item_count; + } + + while (i < end) { + const shelf_index = shelfIndex(i); + const copy_start = boxIndex(i, shelf_index); + const copy_end = @min(shelfSize(shelf_index), copy_start + end - i); + const src = self.dynamic_segments[shelf_index][copy_start..copy_end]; + @memcpy(dest[i - start ..][0..src.len], src); + i += (copy_end - copy_start); + } + } + + pub fn uncheckedAt(self: anytype, index: usize) AtType(@TypeOf(self)) { + if (index < prealloc_item_count) { + return &self.prealloc_segment[index]; + } + const shelf_index = shelfIndex(index); + const box_index = boxIndex(index, shelf_index); + return &self.dynamic_segments[shelf_index][box_index]; + } + + fn shelfCount(box_count: usize) ShelfIndex { + if (prealloc_item_count == 0) { + return log2_int_ceil(usize, box_count + 1); + } + return log2_int_ceil(usize, box_count + prealloc_item_count) - prealloc_exp - 1; + } + + fn shelfSize(shelf_index: ShelfIndex) usize { + if (prealloc_item_count == 0) { + return @as(usize, 1) << shelf_index; + } + return @as(usize, 1) << (shelf_index + (prealloc_exp + 1)); + } + + fn shelfIndex(list_index: usize) ShelfIndex { + if (prealloc_item_count == 0) { + return std.math.log2_int(usize, list_index + 1); + } + return std.math.log2_int(usize, list_index + prealloc_item_count) - prealloc_exp - 1; + } + + fn boxIndex(list_index: usize, shelf_index: ShelfIndex) usize { + if (prealloc_item_count == 0) { + return (list_index + 1) - (@as(usize, 1) << shelf_index); + } + return list_index + prealloc_item_count - (@as(usize, 1) << ((prealloc_exp + 1) + shelf_index)); + } + + fn freeShelves(self: *Self, allocator: Allocator, from_count: ShelfIndex, to_count: ShelfIndex) void { + var i = from_count; + while (i != to_count) { + i -= 1; + allocator.free(self.dynamic_segments[i][0..shelfSize(i)]); + } + } + + pub const Iterator = BaseIterator(*Self, *T); + pub const ConstIterator = BaseIterator(*const Self, *const T); + fn BaseIterator(comptime SelfType: type, comptime ElementPtr: type) type { + return struct { + list: SelfType, + index: usize, + box_index: usize, + shelf_index: ShelfIndex, + shelf_size: usize, + + pub fn next(it: *@This()) ?ElementPtr { + if (it.index >= it.list.len) return null; + if (it.index < prealloc_item_count) { + const ptr = &it.list.prealloc_segment[it.index]; + it.index += 1; + if (it.index == prealloc_item_count) { + it.box_index = 0; + it.shelf_index = 0; + it.shelf_size = prealloc_item_count * 2; + } + return ptr; + } + + const ptr = &it.list.dynamic_segments[it.shelf_index][it.box_index]; + it.index += 1; + it.box_index += 1; + if (it.box_index == it.shelf_size) { + it.shelf_index += 1; + it.box_index = 0; + it.shelf_size *= 2; + } + return ptr; + } + + pub fn prev(it: *@This()) ?ElementPtr { + if (it.index == 0) return null; + + it.index -= 1; + if (it.index < prealloc_item_count) return &it.list.prealloc_segment[it.index]; + + if (it.box_index == 0) { + it.shelf_index -= 1; + it.shelf_size /= 2; + it.box_index = it.shelf_size - 1; + } else { + it.box_index -= 1; + } + + return &it.list.dynamic_segments[it.shelf_index][it.box_index]; + } + + pub fn peek(it: *@This()) ?ElementPtr { + if (it.index >= it.list.len) + return null; + if (it.index < prealloc_item_count) + return &it.list.prealloc_segment[it.index]; + + return &it.list.dynamic_segments[it.shelf_index][it.box_index]; + } + + pub fn set(it: *@This(), index: usize) void { + it.index = index; + if (index < prealloc_item_count) return; + it.shelf_index = shelfIndex(index); + it.box_index = boxIndex(index, it.shelf_index); + it.shelf_size = shelfSize(it.shelf_index); + } + }; + } + + pub fn iterator(self: *Self, start_index: usize) Iterator { + var it = Iterator{ + .list = self, + .index = undefined, + .shelf_index = undefined, + .box_index = undefined, + .shelf_size = undefined, + }; + it.set(start_index); + return it; + } + + pub fn constIterator(self: *const Self, start_index: usize) ConstIterator { + var it = ConstIterator{ + .list = self, + .index = undefined, + .shelf_index = undefined, + .box_index = undefined, + .shelf_size = undefined, + }; + it.set(start_index); + return it; + } + }; +} + +test "basic usage" { + try testSegmentedList(0); + try testSegmentedList(1); + try testSegmentedList(2); + try testSegmentedList(4); + try testSegmentedList(8); + try testSegmentedList(16); +} + +fn testSegmentedList(comptime prealloc: usize) !void { + var list = SegmentedList(i32, prealloc){}; + defer list.deinit(testing.allocator); + + { + var i: usize = 0; + while (i < 100) : (i += 1) { + try list.append(testing.allocator, @as(i32, @intCast(i + 1))); + try testing.expect(list.len == i + 1); + } + } + + { + var i: usize = 0; + while (i < 100) : (i += 1) { + try testing.expect(list.at(i).* == @as(i32, @intCast(i + 1))); + } + } + + { + var it = list.iterator(0); + var x: i32 = 0; + while (it.next()) |item| { + x += 1; + try testing.expect(item.* == x); + } + try testing.expect(x == 100); + while (it.prev()) |item| : (x -= 1) { + try testing.expect(item.* == x); + } + try testing.expect(x == 0); + } + + { + var it = list.constIterator(0); + var x: i32 = 0; + while (it.next()) |item| { + x += 1; + try testing.expect(item.* == x); + } + try testing.expect(x == 100); + while (it.prev()) |item| : (x -= 1) { + try testing.expect(item.* == x); + } + try testing.expect(x == 0); + } + + try testing.expect(list.pop().? == 100); + try testing.expect(list.len == 99); + + try list.appendSlice(testing.allocator, &[_]i32{ 1, 2, 3 }); + try testing.expect(list.len == 102); + try testing.expect(list.pop().? == 3); + try testing.expect(list.pop().? == 2); + try testing.expect(list.pop().? == 1); + try testing.expect(list.len == 99); + + try list.appendSlice(testing.allocator, &[_]i32{}); + try testing.expect(list.len == 99); + + { + var i: i32 = 99; + while (list.pop()) |item| : (i -= 1) { + try testing.expect(item == i); + list.shrinkCapacity(testing.allocator, list.len); + } + } + + { + var control: [100]i32 = undefined; + var dest: [100]i32 = undefined; + + var i: i32 = 0; + while (i < 100) : (i += 1) { + try list.append(testing.allocator, i + 1); + control[@as(usize, @intCast(i))] = i + 1; + } + + @memset(dest[0..], 0); + list.writeToSlice(dest[0..], 0); + try testing.expect(mem.eql(i32, control[0..], dest[0..])); + + @memset(dest[0..], 0); + list.writeToSlice(dest[50..], 50); + try testing.expect(mem.eql(i32, control[50..], dest[50..])); + } + + try list.setCapacity(testing.allocator, 0); +} + +test "clearRetainingCapacity" { + var list = SegmentedList(i32, 1){}; + defer list.deinit(testing.allocator); + + try list.appendSlice(testing.allocator, &[_]i32{ 4, 5 }); + list.clearRetainingCapacity(); + try list.append(testing.allocator, 6); + try testing.expect(list.at(0).* == 6); + try testing.expect(list.len == 1); + list.clearRetainingCapacity(); + try testing.expect(list.len == 0); +} + +/// TODO look into why this std.math function was changed in +/// fc9430f56798a53f9393a697f4ccd6bf9981b970. +fn log2_int_ceil(comptime T: type, x: T) std.math.Log2Int(T) { + assert(x != 0); + const log2_val = std.math.log2_int(T, x); + if (@as(T, 1) << log2_val == x) + return log2_val; + return log2_val + 1; +} diff --git a/stdx/stdx.zig b/stdx/stdx.zig index a149f013..8cb35de2 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -12,6 +12,7 @@ pub const math = @import("math.zig"); pub const meta = @import("meta.zig"); pub const process = @import("process.zig"); pub const queue = @import("queue.zig"); +pub const SegmentedList = @import("segmented_list.zig").SegmentedList; pub const time = @import("time.zig"); pub const unicode = @import("unicode.zig"); diff --git a/zml/buffer.zig b/zml/buffer.zig index 43067706..af591f45 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -218,6 +218,15 @@ pub const Buffer = struct { }; } + pub fn fromIoBufferize(platform: Platform, shape_: Shape) Buffer { + return .{ + ._api = platform.pjrt_api, + ._target = platform.target, + ._shape = shape_, + ._shards = .{}, + }; + } + pub fn devicePtr(self: Buffer) *anyopaque { //stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer", .{}); return self._shards.get(0).opaqueDeviceMemoryDataPointer(self._api) catch unreachable; diff --git a/zml/io.zig b/zml/io.zig index a962edaf..879d8abb 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -175,115 +175,327 @@ pub const TensorStore = struct { const key = std.fmt.bufPrint(&key_buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; return self.store.getReader(key, io, buffer); } + + pub fn count(self: View) usize { + var count_: usize = 0; + const prefix_ = self.prefix() orelse ""; + var it = self.store.registry.tensors.iterator(); + while (it.next()) |item| { + const key = item.key_ptr.*; + if (std.mem.startsWith(u8, key, prefix_)) { + count_ += 1; + } + } + return count_; + } }; }; -fn collectShapes(allocator: std.mem.Allocator, v: anytype) ![]Shape { - const LocalContext = struct { - list: *std.array_list.Managed(Shape), +pub fn TensorBufferTransfer(comptime UserData: type) type { + return struct { + io: std.Io, + platform: Platform, + tensor: safetensors.Tensor, + buffer: *Buffer, + cb_ctx: UserData, }; - var list = std.array_list.Managed(Shape).init(allocator); - errdefer list.deinit(); - - var context: LocalContext = .{ .list = &list }; - try meta.visit(struct { - fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { - try ctx_.list.append(tensor.shape()); - } - }.cb, &context, v); - - return try list.toOwnedSlice(); } -fn collectTensorDesc(allocator: std.mem.Allocator, store: TensorStore.View, v: anytype) ![]safetensors.Tensor { - const LocalContext = struct { - list: *std.array_list.Managed(safetensors.Tensor), - store: TensorStore.View, +pub fn CallbackTensorBufferTransfer(comptime UserData: type) type { + return fn (t: TensorBufferTransfer(UserData)) std.Io.Cancelable!void; +} + +pub fn BufferizeContext(comptime UserData: type) type { + return struct { + allocator: std.mem.Allocator, + arena: *std.heap.ArenaAllocator, + io: std.Io, + platform: Platform, + cb_ctx: UserData, }; - var list = std.array_list.Managed(safetensors.Tensor).init(allocator); - var context: LocalContext = .{ .list = &list, .store = store }; +} + +pub fn bufferize( + comptime UserData: type, + bufferize_ctx: BufferizeContext(UserData), + model: anytype, + bufferized: *Bufferized(@TypeOf(model.*)), + store: TensorStore.View, +) ![]TensorBufferTransfer(UserData) { + comptime { + switch (@typeInfo(@TypeOf(model))) { + .pointer => {}, + else => @compileError("zml.io.bufferize expects a pointer to the model (pass &model or &self.field)"), + } + } + + var tensor_count: usize = 0; meta.visit(struct { - fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { - const tensor_desc = ctx_.store.store.getPtrFromId(tensor.id).?.*; - ctx_.list.append(tensor_desc) catch unreachable; + fn cb(ctx: *usize, _: *const Tensor) void { + ctx.* += 1; } - }.cb, &context, v); + }.cb, &tensor_count, model); - return try list.toOwnedSlice(); + initBufferizedFrom(model.*, bufferized); + + const arena_allocator = bufferize_ctx.arena.allocator(); + + const transfers = try arena_allocator.alloc(TensorBufferTransfer(UserData), tensor_count); + const buffers = try bufferize_ctx.allocator.alloc(*Buffer, tensor_count); + defer bufferize_ctx.allocator.free(buffers); + + const VisitBuffersCtx = struct { + index: usize = 0, + buffers: []*Buffer, + }; + + var visit_buffers_ctx: VisitBuffersCtx = .{ .buffers = buffers }; + meta.visit(struct { + fn cb(ctx: *VisitBuffersCtx, buffer: *Buffer) void { + ctx.buffers[ctx.index] = buffer; + ctx.index += 1; + } + }.cb, &visit_buffers_ctx, bufferized); + + const FillBuffersCtx = struct { + bufferize_ctx: BufferizeContext(UserData), + store: TensorStore.View, + buffers: []*Buffer, + cb_ctx: UserData, + transfers: []TensorBufferTransfer(UserData), + idx: usize = 0, + }; + var fill_buffers_ctx: FillBuffersCtx = .{ + .bufferize_ctx = bufferize_ctx, + .store = store, + .buffers = buffers, + .cb_ctx = bufferize_ctx.cb_ctx, + .transfers = transfers, + }; + + meta.visit(struct { + fn cb(ctx: *FillBuffersCtx, tensor: *const Tensor) void { + const ptr = ctx.store.store.getPtrFromId(tensor.id) orelse return; + const buffer = ctx.buffers[ctx.idx]; + buffer.* = .fromIoBufferize(ctx.bufferize_ctx.platform, tensor.shape()); + + ctx.transfers[ctx.idx] = .{ + .io = ctx.bufferize_ctx.io, + .platform = ctx.bufferize_ctx.platform, + .tensor = ptr.*, + .buffer = buffer, + .cb_ctx = ctx.cb_ctx, + }; + ctx.idx += 1; + } + }.cb, &fill_buffers_ctx, model); + + return transfers; } -pub fn loadBuffersFromId(allocator: std.mem.Allocator, io: std.Io, model: anytype, store: TensorStore.View, platform: Platform) !Bufferized(@TypeOf(model)) { - const Model = @TypeOf(model); - var result: Bufferized(Model) = undefined; - initBufferizedFrom(model, &result); +pub const Writer = union(enum) { device: DeviceWriter, buffered_device: BufferedDeviceWriter }; - var arena: std.heap.ArenaAllocator = .init(allocator); +pub fn loadBuffersFromId( + allocator: std.mem.Allocator, + io: std.Io, + platform: Platform, + model: anytype, + store: TensorStore.View, + read_pool_config: ConcurrentBufferPool.Config, + write_pool_config: ConcurrentBufferPool.Config, +) !Bufferized(@TypeOf(model)) { + var bufferized: Bufferized(@TypeOf(model)) = undefined; + var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); - const shapes = try collectShapes(arena.allocator(), &model); + var read_pool, var write_pool = try ConcurrentBufferPool.initRW( + allocator, + platform, + read_pool_config, + write_pool_config, + ); + defer { + read_pool.deinit(); + write_pool.deinit(); + } - // var transfer: Transfer = try .init(arena.allocator(), shapes, platform); - // defer transfer.deinit(platform); + const LoadCtx = struct { allocator: std.mem.Allocator, read_pool: *ConcurrentBufferPool, write_pool: *ConcurrentBufferPool }; - const tensor_descs = try collectTensorDesc(arena.allocator(), store, &model); - - // TODO(Corentin): Find a way to inject that - const buffer_reader = try allocator.alloc(u8, 16 * 1024 * 1024); - defer allocator.free(buffer_reader); - const buffer_writer = try allocator.alloc(u8, 16 * 1024 * 1024); - defer allocator.free(buffer_writer); - - const LocalContext = struct { - tensor_descs: []safetensors.Tensor, - shapes: []const Shape, - platform: Platform, - // transfer: *Transfer, - index: usize = 0, - buffer_reader: []u8, - buffer_writer: []u8, - store: TensorStore.View, - allocator: std.mem.Allocator, - io: std.Io, - }; - var context: LocalContext = .{ - .tensor_descs = tensor_descs, - .shapes = shapes, - .platform = platform, - // .transfer = &transfer, - .buffer_reader = buffer_reader, - .buffer_writer = buffer_writer, - .store = store, + const bufferize_ctx: BufferizeContext(LoadCtx) = .{ .allocator = allocator, + .arena = &arena, .io = io, + .platform = platform, + .cb_ctx = .{ .allocator = allocator, .read_pool = &read_pool, .write_pool = &write_pool }, }; - try meta.visit(struct { - fn cb(context_: *LocalContext, buffer: *Buffer) !void { - const tensor_desc = context_.tensor_descs[context_.index]; - var reader = try safetensors.TensorReader.init(context_.io, tensor_desc, context_.buffer_reader); - defer reader.deinit(); + const transfers = try bufferize(LoadCtx, bufferize_ctx, &model, &bufferized, store); - // var writer = try context_.transfer.getWriter(context_.io, context_.index, context_.buffer_writer); - var writer = try BufferedDeviceWriter.init( - context_.allocator, - context_.io, - context_.platform, - context_.shapes[context_.index], - ); + var group = try stdx.Io.AllocatingLimitedConcurrentGroup.init(allocator, read_pool.config.concurrency); + defer { + group.cancel(io); + group.deinit(); + } - _ = try reader.interface.streamRemaining(&writer.interface); - try writer.interface.flush(); + for (transfers) |t| { + try group.concurrent(io, struct { + fn run(ctx: TensorBufferTransfer(LoadCtx)) !void { + const allocator_ = ctx.cb_ctx.allocator; + const read_pool_ = ctx.cb_ctx.read_pool; + const write_pool_ = ctx.cb_ctx.write_pool; - // buffer.* = try context_.transfer.getBuffer(context_.index); - buffer.* = writer.buffer.?; - context_.index += 1; - } - }.cb, &context, &result); + const read_buffer = read_pool_.acquire(ctx.io) catch unreachable; + defer read_pool_.release(ctx.io, read_buffer) catch {}; - return result; + const write_buffer = write_pool_.acquire(ctx.io) catch unreachable; + defer write_pool_.release(ctx.io, write_buffer) catch {}; + + var reader = safetensors.TensorReader.init(ctx.io, ctx.tensor, read_buffer) catch unreachable; + defer reader.deinit(); + + var writer: Writer = if (ctx.platform.target == .neuron) blk: { + break :blk .{ .buffered_device = BufferedDeviceWriter.init(allocator_, ctx.io, null, ctx.platform, ctx.buffer.shape(), ctx.buffer) catch unreachable }; + } else blk: { + break :blk .{ .device = DeviceWriter.init(ctx.io, null, ctx.platform, ctx.buffer, .device, write_buffer) catch unreachable }; + }; + + switch (writer) { + .device => |*d| { + defer { + d.interface.flush() catch unreachable; + d.deinit(); + } + _ = reader.interface.streamRemaining(&d.interface) catch unreachable; + }, + .buffered_device => |*d| { + defer { + d.interface.flush() catch unreachable; + d.deinit(); + } + _ = reader.interface.streamRemaining(&d.interface) catch unreachable; + }, + } + } + }.run, .{t}); + } + + try group.await(io); + + return bufferized; } -pub fn initBufferizedFrom(model: anytype, bufferized_: *Bufferized(@TypeOf(model))) void { +pub const DeviceWriter = struct { + io: std.Io, + progress: ?*std.Progress.Node, + platform: Platform, + buffer: *Buffer, + transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, + offset: usize = 0, + interface: std.Io.Writer, + + pub fn init(io: std.Io, progress: ?*std.Progress.Node, platform: Platform, buffer: *Buffer, memory: Buffer.Memory, buf: []u8) !DeviceWriter { + const shape = buffer.shape(); + const memories = platform.getDevices()[0].addressableMemories(platform.pjrt_api); + + const mem = blk: { + for (memories) |mem_| { + if (mem_.kind(platform.pjrt_api) == memory) { + break :blk mem_; + } + } + return error.MemoryNotFound; + }; + + const shape_spec: pjrt.ShapeSpec = .init(shape.dims(), pjrtx.bufferTypeFromDtype(shape.dtype())); + const transfer_manager = try platform.pjrt_client.createBuffersForAsyncHostToDevice(platform.pjrt_api, .{ + .shape_specs = &.{shape_spec}, + .memory = mem, + }); + try buffer._shards.append(transfer_manager.retrieveBuffer(platform.pjrt_api, 0) catch unreachable); + + return .{ + .io = io, + .progress = progress, + .platform = platform, + .buffer = buffer, + .transfer_manager = transfer_manager, + .interface = .{ + .buffer = buf, + .end = 0, + .vtable = &.{ + .drain = drain, + }, + }, + }; + } + + pub fn deinit(self: *const DeviceWriter) void { + self.transfer_manager.deinit(self.platform.pjrt_api); + } + + pub fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + _ = data; + _ = splat; + const self: *DeviceWriter = @alignCast(@fieldParentPtr("interface", w)); + + stdx.debug.assert(self.offset + w.end <= self.buffer.shape().byteSize(), "Can't write more data than required", .{}); + + const is_last_transfer = self.offset + w.end >= self.buffer.shape().byteSize(); + const event = self.transfer_manager.transferData(self.platform.pjrt_api, 0, w.buffer[0..w.end], @intCast(self.offset), is_last_transfer) catch return error.WriteFailed; + event.await(self.platform.pjrt_api, self.io) catch return error.WriteFailed; + + self.offset += w.end; + if (self.progress) |p| p.setCompletedItems(self.offset); + w.end = 0; + + return 0; + } +}; + +pub const BufferedDeviceWriter = struct { + allocator: std.mem.Allocator, + io: std.Io, + progress: ?*std.Progress.Node, + transfer_buffer: []u8, + platform: Platform, + shape: Shape, + buffer: *Buffer, + interface: std.Io.Writer, + + pub fn init(allocator: std.mem.Allocator, io: std.Io, progress: ?*std.Progress.Node, platform: Platform, shape: Shape, buffer: *Buffer) !BufferedDeviceWriter { + const transfer_buffer = try allocator.alloc(u8, shape.byteSize()); + return .{ + .allocator = allocator, + .io = io, + .progress = progress, + .transfer_buffer = transfer_buffer, + .shape = shape, + .platform = platform, + .buffer = buffer, + .interface = .{ + .buffer = transfer_buffer, + .vtable = &.{ + .drain = std.Io.Writer.fixedDrain, + .flush = flush, + .rebase = std.Io.Writer.failingRebase, + }, + }, + }; + } + + pub fn deinit(self: *BufferedDeviceWriter) void { + self.allocator.free(self.transfer_buffer); + } + + pub fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { + const self: *BufferedDeviceWriter = @alignCast(@fieldParentPtr("interface", w)); + self.buffer.* = Buffer.fromBytes(self.io, self.platform, self.shape, self.transfer_buffer) catch return std.Io.Writer.Error.WriteFailed; + if (self.progress) |p| p.setCompletedItems(self.shape.byteSize()); + _ = self.buffer.await(self.io) catch return std.Io.Writer.Error.WriteFailed; + } +}; + +fn initBufferizedFrom(model: anytype, bufferized_: *Bufferized(@TypeOf(model))) void { const Model = @TypeOf(model); const type_info = @typeInfo(Bufferized(Model)); switch (type_info) { @@ -314,133 +526,93 @@ pub fn initBufferizedFrom(model: anytype, bufferized_: *Bufferized(@TypeOf(model } } -pub const SimpleWriter = struct { - offset: usize = 0, - io: std.Io, - transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, - shape: Shape, - buffer_index: usize, +pub const ConcurrentBufferPool = struct { + pub const Error = error{ OutOfMemory, Notfound, NotAcquired }; + + const alignment: std.mem.Alignment = .fromByteUnits(4 * 1024); + + allocator: std.mem.Allocator, + mutex: std.Io.Mutex = .init, + platform: Platform, - interface: std.Io.Writer, + locked: []bool, + buffers: []u8, + config: Config, - pub fn init(buffer: []u8, io: std.Io, transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, shape: Shape, buffer_index: usize, platform: Platform) SimpleWriter { - return .{ - .io = io, - .transfer_manager = transfer_manager, - .shape = shape, - .buffer_index = buffer_index, - .platform = platform, - .interface = .{ - .buffer = buffer, - .end = 0, - .vtable = &.{ - .drain = drain, - }, - }, - }; - } + pub const Config = struct { + size: usize, + concurrency: usize, + dma: bool = true, + }; - pub fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { - _ = data; - _ = splat; - const writer: *SimpleWriter = @alignCast(@fieldParentPtr("interface", w)); - stdx.debug.assert(writer.offset + w.end <= writer.shape.byteSize(), "Can't write more data than required", .{}); - const is_last_transfer = writer.offset + w.end >= writer.shape.byteSize(); - log.debug("Writing {} bytes", .{w.end}); - const event = writer.transfer_manager.transferData(writer.platform.pjrt_api, writer.buffer_index, w.buffer[0..w.end], @intCast(writer.offset), is_last_transfer) catch return error.WriteFailed; - event.await(writer.platform.pjrt_api, writer.io) catch return error.WriteFailed; - const written = w.end; - writer.offset += written; - w.end = 0; - return 0; - } -}; + pub fn init( + allocator: std.mem.Allocator, + platform: Platform, + config: Config, + ) !ConcurrentBufferPool { + const buffers = try allocator.alignedAlloc(u8, alignment, config.concurrency * config.size); + errdefer allocator.free(buffers); -pub const Transfer = struct { - shapes: []Shape, - transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, - arena: std.heap.ArenaAllocator, - platform: Platform, - - pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, platform: Platform) !Transfer { - const shape_specs = try allocator.alloc(pjrt.ShapeSpec, shapes.len); - defer allocator.free(shape_specs); - - var temp_arena = std.heap.ArenaAllocator.init(allocator); - defer temp_arena.deinit(); - - var arena = std.heap.ArenaAllocator.init(allocator); - errdefer arena.deinit(); - - for (shape_specs, shapes) |*spec, shape| { - const dims = try temp_arena.allocator().dupe(i64, shape.dims()); - spec.* = pjrt.ShapeSpec.init(dims, pjrtx.bufferTypeFromDtype(shape.dtype())); + if (config.dma and platform.target != .cpu) { + try platform.pjrt_client.dmaMap(platform.pjrt_api, buffers); } - const memory = platform.pjrt_client.addressableMemory(platform.pjrt_api, .device).?; + const locked = try allocator.alloc(bool, config.concurrency); + @memset(locked, false); - const transfer_manager = try platform.pjrt_client.createBuffersForAsyncHostToDevice(platform.pjrt_api, .{ .shape_specs = shape_specs, .memory = memory }); - errdefer transfer_manager.deinit(platform.pjrt_api); - - return .{ - .shapes = try arena.allocator().dupe(Shape, shapes), - .transfer_manager = transfer_manager, - .arena = arena, - .platform = platform, - }; - } - - pub fn deinit(self: Transfer, platform: Platform) void { - self.arena.deinit(); - self.transfer_manager.deinit(platform.pjrt_api); - } - - pub fn getBuffer(self: *const Transfer, index: usize) !Buffer { - const pjrt_buffer = self.transfer_manager.retrieveBuffer(self.platform.pjrt_api, index) catch return error.NotFound; - return .fromPjrtBuffers(self.platform, self.shapes[index], &.{pjrt_buffer}); - } - - pub fn getWriter(self: *const Transfer, io: std.Io, index: usize, buffer: []u8) !SimpleWriter { - const writer: SimpleWriter = .init(buffer, io, self.transfer_manager, self.shapes[index], index, self.platform); - return writer; - } -}; - -pub const BufferedDeviceWriter = struct { - allocator: std.mem.Allocator, - io: std.Io, - transfer_buffer: []u8, - platform: Platform, - shape: Shape, - buffer: ?Buffer = null, - interface: std.Io.Writer, - - pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: Platform, shape: Shape) !BufferedDeviceWriter { - const transfer_buffer = try allocator.alloc(u8, shape.byteSize()); return .{ .allocator = allocator, - .io = io, - .transfer_buffer = transfer_buffer, - .shape = shape, + .mutex = .init, .platform = platform, - .interface = .{ - .buffer = transfer_buffer, - .vtable = &.{ - .drain = std.Io.Writer.fixedDrain, - .flush = flush, - .rebase = std.Io.Writer.failingRebase, - }, - }, + .locked = locked, + .buffers = buffers, + .config = config, }; } - pub fn deinit(self: *BufferedDeviceWriter) void { - self.allocator.free(self.transfer_buffer); + pub fn initRW(allocator: std.mem.Allocator, platform: Platform, read_config: Config, write_config: Config) !struct { ConcurrentBufferPool, ConcurrentBufferPool } { + return .{ try .init(allocator, platform, read_config), try .init(allocator, platform, write_config) }; } - pub fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { - const self: *BufferedDeviceWriter = @alignCast(@fieldParentPtr("interface", w)); - self.buffer = Buffer.fromBytes(self.io, self.platform, self.shape, self.transfer_buffer) catch return std.Io.Writer.Error.WriteFailed; - _ = self.buffer.?.await(self.io) catch return std.Io.Writer.Error.WriteFailed; + pub fn deinit(self: *ConcurrentBufferPool) void { + if (self.config.dma and self.platform.target != .cpu) { + self.platform.pjrt_client.dmaUnmap(self.platform.pjrt_api, self.buffers) catch {}; + } + + self.allocator.free(self.buffers); + self.allocator.free(self.locked); + } + + pub fn acquire(self: *ConcurrentBufferPool, io: std.Io) Error![]u8 { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + + for (self.locked, 0..) |locked, i| { + if (!locked) { + self.locked[i] = true; + return self.buffers[i * self.config.size .. (i + 1) * self.config.size]; + } + } + + return Error.OutOfMemory; + } + + pub fn release(self: *ConcurrentBufferPool, io: std.Io, buffer: []u8) Error!void { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + + for (self.locked, 0..) |locked, i| { + const slice = self.buffers[i * self.config.size .. (i + 1) * self.config.size]; + if (slice.ptr == buffer.ptr) { + if (locked) { + self.locked[i] = false; + return; + } else { + return Error.NotAcquired; + } + } + } + + return Error.Notfound; } }; diff --git a/zml/io/vfs.zig b/zml/io/vfs.zig index 413761a7..0fd827bf 100644 --- a/zml/io/vfs.zig +++ b/zml/io/vfs.zig @@ -1,859 +1,349 @@ const std = @import("std"); + const vfs = @import("vfs"); const log = std.log.scoped(.@"zml/io/vfs"); pub const VFS = struct { - pub const Error = FileEntryError || DirEntryError || ResolveAbsolutePathError; - - pub const FileEntryError = error{ - MissingVirtualFileHandle, - } || ResolveAbsolutePathError; - - pub const DirEntryError = error{ - MissingVirtualDirHandle, - } || ResolveAbsolutePathError; - - pub const ResolveAbsolutePathError = error{ - SchemeNotRegistered, - InvalidURI, - }; - pub const File = vfs.File; pub const HTTP = vfs.HTTP; pub const HF = vfs.HF; + const VFSBase = vfs.VFSBase; - const FsFile = struct { - backend: std.Io, - handle: std.Io.File.Handle, - }; + const Handles = std.ArrayList(Handle); + const ClosedHandles = std.ArrayList(u32); - const FsDir = struct { - backend: std.Io, - handle: std.Io.Dir.Handle, + const CWD_HANDLE: u32 = 0; + + const Handle = struct { + pub const Type = enum { + file, + directory, + }; + + scheme: ?[]const u8, + type: Type, + handle: u32, + + pub fn isRootFs(self: *Handle) bool { + return self.scheme == null; + } }; allocator: std.mem.Allocator, - mutex: std.Io.Mutex, + mutex: std.Io.Mutex = .init, - backends: std.StringHashMapUnmanaged(std.Io), - // Maps virtual file and dir handles to backend handles - files: std.AutoHashMapUnmanaged(std.Io.File.Handle, *FsFile), - dirs: std.AutoHashMapUnmanaged(std.Io.Dir.Handle, *FsDir), - next_file_handle: std.Io.File.Handle, - next_dir_handle: std.Io.Dir.Handle, + backends: std.StringHashMapUnmanaged(std.Io) = .{}, + handles: Handles = .{}, + closed_handles: ClosedHandles = .{}, - vtable: std.Io.VTable, - base_io: std.Io, + base: VFSBase, - pub fn init(allocator: std.mem.Allocator, base_io: std.Io) VFS { - const vtable: std.Io.VTable = .{ - .async = async, - .concurrent = concurrent, - .await = await, - .cancel = cancel, - .cancelRequested = cancelRequested, - .groupAsync = groupAsync, - .groupConcurrent = groupConcurrent, - .groupWait = groupWait, - .groupCancel = groupCancel, - .select = select, - .mutexLock = mutexLock, - .mutexLockUncancelable = mutexLockUncancelable, - .mutexUnlock = mutexUnlock, - .conditionWait = conditionWait, - .conditionWaitUncancelable = conditionWaitUncancelable, - .conditionWake = conditionWake, - .now = now, - .sleep = sleep, - .netListenIp = netListenIp, - .netAccept = netAccept, - .netBindIp = netBindIp, - .netConnectIp = netConnectIp, - .netListenUnix = netListenUnix, - .netConnectUnix = netConnectUnix, - .netSend = netSend, - .netReceive = netReceive, - .netRead = netRead, - .netWrite = netWrite, - .netClose = netClose, - .netInterfaceNameResolve = netInterfaceNameResolve, - .netInterfaceName = netInterfaceName, - .netLookup = netLookup, - .dirMake = dirMake, - .dirMakePath = dirMakePath, - .dirMakeOpenPath = dirMakeOpenPath, - .dirStat = dirStat, - .dirStatPath = dirStatPath, - .dirAccess = dirAccess, - .dirCreateFile = dirCreateFile, - .dirOpenFile = dirOpenFile, - .dirOpenDir = dirOpenDir, - .dirClose = dirClose, - .fileStat = fileStat, - .fileClose = fileClose, - .fileWriteStreaming = fileWriteStreaming, - .fileWritePositional = fileWritePositional, - .fileReadStreaming = fileReadStreaming, - .fileReadPositional = fileReadPositional, - .fileSeekBy = fileSeekBy, - .fileSeekTo = fileSeekTo, - .openSelfExe = openSelfExe, - }; + pub fn init(allocator: std.mem.Allocator, base_io: std.Io) !VFS { + const base = VFSBase.init(base_io); + + var handles: Handles = .{}; + try handles.append(allocator, .{ + .scheme = null, + .type = .directory, + .handle = CWD_HANDLE, + }); return .{ .allocator = allocator, - .mutex = .init, - .backends = .{}, - .files = .{}, - .dirs = .{}, - .next_file_handle = 0, - .next_dir_handle = 0, - .vtable = vtable, - .base_io = base_io, + .handles = handles, + .base = base, }; } pub fn deinit(self: *VFS) void { - var file_it = self.files.valueIterator(); - while (file_it.next()) |entry_ptr| { - self.allocator.destroy(entry_ptr.*); - } - self.files.deinit(self.allocator); - - var dir_it = self.dirs.valueIterator(); - while (dir_it.next()) |entry_ptr| { - self.allocator.destroy(entry_ptr.*); - } - self.dirs.deinit(self.allocator); - + self.handles.deinit(self.allocator); + self.closed_handles.deinit(self.allocator); self.backends.deinit(self.allocator); } pub fn register(self: *VFS, scheme: []const u8, backend: std.Io) std.mem.Allocator.Error!void { - self.mutex.lockUncancelable(self.base_io); - defer self.mutex.unlock(self.base_io); + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); try self.backends.put(self.allocator, scheme, backend); } pub fn unregister(self: *VFS, scheme: []const u8) bool { - self.mutex.lockUncancelable(self.base_io); - defer self.mutex.unlock(self.base_io); + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); return self.backends.remove(scheme); } pub fn io(self: *VFS) std.Io { return .{ - .vtable = &self.vtable, - .userdata = self, + .userdata = &self.base, + .vtable = &comptime VFSBase.vtable(.{ + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRead = dirRead, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileReadStreaming = fileReadStreaming, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileRealPath = fileRealPath, + }), }; } - fn backendFromAbsolutePath(self: *VFS, path: []const u8) ResolveAbsolutePathError!std.Io { - if (std.fs.path.isAbsolutePosix(path)) return self.base_io; - const uri = std.Uri.parse(path) catch return ResolveAbsolutePathError.InvalidURI; - return self.backends.get(uri.scheme) orelse return ResolveAbsolutePathError.SchemeNotRegistered; - } + fn openHandle(self: *VFS) !struct { u32, *Handle } { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); - fn registerFile(self: *VFS, backend: std.Io, handle: std.Io.File.Handle) std.mem.Allocator.Error!std.Io.File.Handle { - const entry = try self.allocator.create(FsFile); - errdefer self.allocator.destroy(entry); - - entry.* = .{ - .backend = backend, - .handle = handle, - }; - - self.mutex.lockUncancelable(self.base_io); - defer self.mutex.unlock(self.base_io); - - const virtual_handle = self.next_file_handle; - self.next_file_handle += 1; - - try self.files.put(self.allocator, virtual_handle, entry); - - return virtual_handle; - } - - fn closeFile(self: *VFS, virtual_handle: std.Io.File.Handle) void { - self.mutex.lockUncancelable(self.base_io); - const entry = self.files.fetchRemove(virtual_handle); - self.mutex.unlock(self.base_io); - - if (entry) |kv| { - const fs_file = kv.value; - fs_file.backend.vtable.fileClose(fs_file.backend.userdata, .{ .handle = fs_file.handle }); - self.allocator.destroy(fs_file); + if (self.closed_handles.pop()) |idx| { + return .{ idx, &self.handles.items[idx] }; } + return .{ @intCast(self.handles.items.len), try self.handles.addOne(self.allocator) }; } - fn fsFile(self: *VFS, virtual_handle: std.Io.File.Handle) FileEntryError!FsFile { - self.mutex.lockUncancelable(self.base_io); - defer self.mutex.unlock(self.base_io); + fn closeHandle(self: *VFS, idx: u32) !void { + if (idx == CWD_HANDLE) return; - const entry = self.files.get(virtual_handle) orelse return FileEntryError.MissingVirtualFileHandle; - return entry.*; + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + try self.closed_handles.append(self.allocator, idx); } - fn registerDir(self: *VFS, backend: std.Io, handle: std.Io.Dir.Handle) std.mem.Allocator.Error!std.Io.Dir.Handle { - const entry = try self.allocator.create(FsDir); - errdefer self.allocator.destroy(entry); + fn getFileHandle(self: *VFS, file: std.Io.File) struct { *Handle, std.Io } { + self.mutex.lockUncancelable(self.base.inner); + const handle = &self.handles.items[@intCast(file.handle)]; + self.mutex.unlock(self.base.inner); - entry.* = .{ - .backend = backend, - .handle = handle, - }; - - self.mutex.lockUncancelable(self.base_io); - defer self.mutex.unlock(self.base_io); - - const virtual_handle = self.next_dir_handle; - self.next_dir_handle += 1; - - try self.dirs.put(self.allocator, virtual_handle, entry); - return virtual_handle; + const backend = if (handle.scheme) |s| + self.backends.get(s).? + else + self.base.inner; + return .{ handle, backend }; } - fn closeDir(self: *VFS, virtual_handle: std.Io.Dir.Handle) void { - self.mutex.lockUncancelable(self.base_io); - const entry = self.dirs.fetchRemove(virtual_handle); - self.mutex.unlock(self.base_io); + fn getDirHandle(self: *VFS, dir: std.Io.Dir) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); - if (entry) |kv| { - const fs_dir = kv.value; - fs_dir.backend.vtable.dirClose(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }); - self.allocator.destroy(fs_dir); - } - } - - fn fsDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) DirEntryError!FsDir { if (std.meta.eql(dir, std.Io.Dir.cwd())) { - std.debug.assert(sub_path != null); - const backend = self.backendFromAbsolutePath(sub_path.?) catch blk: { - log.warn("Unable to resolve backend for relative path {s} in cwd, falling back to base_io", .{sub_path.?}); - break :blk self.base_io; - }; - return .{ .backend = backend, .handle = dir.handle }; + return &self.handles.items[CWD_HANDLE]; } - self.mutex.lockUncancelable(self.base_io); - defer self.mutex.unlock(self.base_io); - - const entry = self.dirs.get(dir.handle) orelse return DirEntryError.MissingVirtualDirHandle; - return entry.*; + return &self.handles.items[@intCast(dir.handle)]; } - // VTable implementations + fn lookup(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?[]const u8, std.Io.Dir, std.Io } { + if (std.meta.eql(dir, std.Io.Dir.cwd())) { + if (sub_path == null) return .{ null, dir, self.base.inner }; + if (std.fs.path.isAbsolutePosix(sub_path.?)) return .{ null, dir, self.base.inner }; - fn dirMake( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - mode: std.Io.Dir.Mode, - ) std.Io.Dir.MakeError!void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + const uri = std.Uri.parse(sub_path.?) catch null; + if (uri) |u| { + const backend = self.backends.get(u.scheme) orelse return error.VFSNotRegistered; + return .{ u.scheme, std.Io.Dir.cwd(), backend }; + } else { + return .{ null, std.Io.Dir.cwd(), self.base.inner }; + } + } else { + const handle = self.getDirHandle(dir); + if (handle.scheme) |s| { + const backend = self.backends.get(s) orelse return error.VFSNotRegistered; + return .{ s, .{ .handle = @intCast(handle.handle) }, backend }; + } else { + return .{ null, .{ .handle = @intCast(handle.handle) }, self.base.inner }; + } + } + } - const fs_dir = self.fsDir(dir, sub_path) catch |err| { - return switch (err) { - DirEntryError.SchemeNotRegistered => std.Io.Dir.MakeError.BadPathName, - DirEntryError.InvalidURI => std.Io.Dir.MakeError.BadPathName, - DirEntryError.MissingVirtualDirHandle => std.Io.Dir.MakeError.FileNotFound, - }; + fn stripScheme(path: []const u8) []const u8 { + const uri = std.Uri.parse(path) catch return path; + return path[uri.scheme.len + 3 ..]; + } + + fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const scheme, const dir_, const backend = self.lookup(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for opening dir '{s}' : {any}", .{ sub_path, err }); + return std.Io.Dir.OpenError.Unexpected; + }; + const fs_dir = try backend.vtable.dirOpenDir(backend.userdata, dir_, stripScheme(sub_path), options); + const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; + handle.* = .{ + .scheme = scheme, + .type = .directory, + .handle = @intCast(fs_dir.handle), + }; + return .{ .handle = @intCast(idx) }; + } + + fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + _, const dir_, const backend = self.lookup(dir, null) catch |err| { + log.err("Failed to lookup backend for dir stat : {any}", .{err}); + return std.Io.Dir.StatError.Unexpected; + }; + return backend.vtable.dirStat(backend.userdata, dir_); + } + + pub fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + _, const dir_, const backend = self.lookup(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for dir stat file '{s}' : {any}", .{ sub_path, err }); + return std.Io.Dir.StatFileError.Unexpected; + }; + return backend.vtable.dirStatFile(backend.userdata, dir_, stripScheme(sub_path), options); + } + + pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + _, const dir_, const backend = self.lookup(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for dir access '{s}' : {any}", .{ sub_path, err }); + return std.Io.Dir.AccessError.Unexpected; + }; + return backend.vtable.dirAccess(backend.userdata, dir_, stripScheme(sub_path), options); + } + + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const scheme, const dir_, const backend = self.lookup(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for opening file '{s}' : {any}", .{ sub_path, err }); + return std.Io.File.OpenError.Unexpected; + }; + const file = try backend.vtable.dirOpenFile(backend.userdata, dir_, stripScheme(sub_path), flags); + const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; + handle.* = .{ + .scheme = scheme, + .type = .file, + .handle = @intCast(file.handle), }; - return fs_dir.backend.vtable.dirMake(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }, sub_path, mode); + return .{ .handle = @intCast(idx) }; } - fn dirMakePath( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - mode: std.Io.Dir.Mode, - ) std.Io.Dir.MakeError!void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_dir = self.fsDir(dir, sub_path) catch |err| { - return switch (err) { - DirEntryError.SchemeNotRegistered => std.Io.Dir.MakeError.BadPathName, - DirEntryError.InvalidURI => std.Io.Dir.MakeError.BadPathName, - DirEntryError.MissingVirtualDirHandle => std.Io.Dir.MakeError.FileNotFound, + fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + for (dirs) |dir| { + _, const dir_, const backend = self.lookup(dir, null) catch |err| { + log.err("Failed to lookup backend for closing dir : {any}", .{err}); + continue; }; + backend.vtable.dirClose(backend.userdata, &.{dir_}); + self.closeHandle(@intCast(dir_.handle)) catch unreachable; + } + } + + fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + _, const dir_, const backend = self.lookup(reader.dir, null) catch |err| { + log.err("Failed to lookup backend for dir real path : {any}", .{err}); + return std.Io.Dir.RealPathError.Unexpected; }; - return fs_dir.backend.vtable.dirMakePath(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }, sub_path, mode); + const original_dir = reader.dir; + reader.dir = dir_; + defer reader.dir = original_dir; + + return backend.vtable.dirRead(backend.userdata, reader, entries); } - fn dirMakeOpenPath( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - options: std.Io.Dir.OpenOptions, - ) std.Io.Dir.MakeOpenPathError!std.Io.Dir { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_dir = self.fsDir(dir, sub_path) catch |err| { - return switch (err) { - DirEntryError.SchemeNotRegistered => std.Io.Dir.MakeOpenPathError.BadPathName, - DirEntryError.InvalidURI => std.Io.Dir.MakeOpenPathError.BadPathName, - DirEntryError.MissingVirtualDirHandle => std.Io.Dir.MakeOpenPathError.FileNotFound, - }; + fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const scheme, const dir_, const backend = self.lookup(dir, null) catch |err| { + log.err("Failed to lookup backend for dir real path : {any}", .{err}); + return std.Io.Dir.RealPathError.Unexpected; }; - const fs_open_dir = try fs_dir.backend.vtable.dirMakeOpenPath(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }, sub_path, options); + if (scheme) |s| { + const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); + const path_len = try backend.vtable.dirRealPath(backend.userdata, dir_, out_buffer[prefix.len..]); + return prefix.len + path_len; + } else { + return try backend.vtable.dirRealPath(backend.userdata, dir_, out_buffer); + } + } - const vfs_handle = self.registerDir(fs_dir.backend, fs_open_dir.handle) catch |err| { - return switch (err) { - std.mem.Allocator.Error.OutOfMemory => std.Io.Dir.OpenError.SystemFdQuotaExceeded, - }; + fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const scheme, const dir_, const backend = self.lookup(dir, null) catch |err| { + log.err("Failed to lookup backend for dir real path file '{s}' : {any}", .{ path_name, err }); + return std.Io.Dir.RealPathFileError.Unexpected; }; - return .{ .handle = vfs_handle }; + if (scheme) |s| { + const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); + const path_len = try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer[prefix.len..]); + return prefix.len + path_len; + } else { + return try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer); + } } - fn dirStat( - userdata: ?*anyopaque, - dir: std.Io.Dir, - ) std.Io.Dir.StatError!std.Io.Dir.Stat { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_dir = self.fsDir(dir, null) catch |err| { - return switch (err) { - DirEntryError.SchemeNotRegistered => std.Io.Dir.StatError.Unexpected, - DirEntryError.InvalidURI => std.Io.Dir.StatError.Unexpected, - DirEntryError.MissingVirtualDirHandle => std.Io.Dir.StatError.Unexpected, - }; - }; - - return fs_dir.backend.vtable.dirStat(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }); + fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileStat(backend.userdata, .{ .handle = @intCast(handle.handle) }); } - fn dirStatPath( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - options: std.Io.Dir.StatPathOptions, - ) std.Io.Dir.StatPathError!std.Io.Dir.Stat { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_dir = self.fsDir(dir, sub_path) catch |err| { - return switch (err) { - DirEntryError.SchemeNotRegistered => std.Io.Dir.StatPathError.BadPathName, - DirEntryError.InvalidURI => std.Io.Dir.StatPathError.BadPathName, - DirEntryError.MissingVirtualDirHandle => std.Io.Dir.StatPathError.FileNotFound, - }; - }; - - return fs_dir.backend.vtable.dirStatPath(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }, sub_path, options); + fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileLength(backend.userdata, .{ .handle = @intCast(handle.handle) }); } - fn dirAccess( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - options: std.Io.Dir.AccessOptions, - ) std.Io.Dir.AccessError!void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_dir = self.fsDir(dir, sub_path) catch |err| { - return switch (err) { - DirEntryError.SchemeNotRegistered => std.Io.Dir.AccessError.BadPathName, - DirEntryError.InvalidURI => std.Io.Dir.AccessError.BadPathName, - DirEntryError.MissingVirtualDirHandle => std.Io.Dir.AccessError.FileNotFound, - }; - }; - - return fs_dir.backend.vtable.dirAccess(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }, sub_path, options); + fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + for (files) |file| { + const handle, const backend = self.getFileHandle(file); + backend.vtable.fileClose(backend.userdata, &.{.{ .handle = @intCast(handle.handle) }}); + self.closeHandle(@intCast(file.handle)) catch unreachable; + } } - fn dirCreateFile( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - flags: std.Io.File.CreateFlags, - ) std.Io.File.OpenError!std.Io.File { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_dir = self.fsDir(dir, sub_path) catch |err| { - return switch (err) { - DirEntryError.SchemeNotRegistered => std.Io.File.OpenError.BadPathName, - DirEntryError.InvalidURI => std.Io.File.OpenError.BadPathName, - DirEntryError.MissingVirtualDirHandle => std.Io.File.OpenError.FileNotFound, - }; - }; - - const fs_file = try fs_dir.backend.vtable.dirCreateFile(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }, sub_path, flags); - - const vfs_handle = self.registerFile(fs_dir.backend, fs_file.handle) catch |err| { - return switch (err) { - std.mem.Allocator.Error.OutOfMemory => std.Io.File.OpenError.SystemFdQuotaExceeded, - }; - }; - - return .{ .handle = vfs_handle }; + fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileReadStreaming(backend.userdata, .{ .handle = @intCast(handle.handle) }, data); } - fn dirOpenFile( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - flags: std.Io.File.OpenFlags, - ) std.Io.File.OpenError!std.Io.File { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_dir = self.fsDir(dir, sub_path) catch |err| { - return switch (err) { - DirEntryError.SchemeNotRegistered => std.Io.File.OpenError.BadPathName, - DirEntryError.InvalidURI => std.Io.File.OpenError.BadPathName, - DirEntryError.MissingVirtualDirHandle => std.Io.File.OpenError.FileNotFound, - }; - }; - - const fs_file = try fs_dir.backend.vtable.dirOpenFile(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }, sub_path, flags); - - const vfs_handle = self.registerFile(fs_dir.backend, fs_file.handle) catch |err| { - return switch (err) { - std.mem.Allocator.Error.OutOfMemory => std.Io.File.OpenError.SystemFdQuotaExceeded, - }; - }; - - return .{ .handle = vfs_handle }; + fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileReadPositional(backend.userdata, .{ .handle = @intCast(handle.handle) }, data, offset); } - fn dirOpenDir( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - options: std.Io.Dir.OpenOptions, - ) std.Io.Dir.OpenError!std.Io.Dir { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_dir = self.fsDir(dir, sub_path) catch |err| { - return switch (err) { - DirEntryError.SchemeNotRegistered => std.Io.Dir.OpenError.BadPathName, - DirEntryError.InvalidURI => std.Io.Dir.OpenError.BadPathName, - DirEntryError.MissingVirtualDirHandle => std.Io.Dir.OpenError.FileNotFound, - }; - }; - - const fs_sub_dir = try fs_dir.backend.vtable.dirOpenDir(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }, sub_path, options); - - const vfs_handle = self.registerDir(fs_dir.backend, fs_sub_dir.handle) catch |err| { - return switch (err) { - std.mem.Allocator.Error.OutOfMemory => std.Io.Dir.OpenError.SystemFdQuotaExceeded, - }; - }; - - return .{ .handle = vfs_handle }; + fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileSeekBy(backend.userdata, .{ .handle = @intCast(handle.handle) }, relative_offset); } - fn dirClose( - userdata: ?*anyopaque, - dir: std.Io.Dir, - ) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - self.closeDir(dir.handle); + fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileSeekTo(backend.userdata, .{ .handle = @intCast(handle.handle) }, absolute_offset); } - fn fileStat( - userdata: ?*anyopaque, - file: std.Io.File, - ) std.Io.File.StatError!std.Io.File.Stat { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); - const fs_file = self.fsFile(file.handle) catch |err| { - return switch (err) { - FileEntryError.SchemeNotRegistered => std.Io.File.StatError.Unexpected, - FileEntryError.InvalidURI => std.Io.File.StatError.Unexpected, - FileEntryError.MissingVirtualFileHandle => std.Io.File.StatError.Unexpected, - }; - }; - - return fs_file.backend.vtable.fileStat(fs_file.backend.userdata, .{ .handle = fs_file.handle }); - } - - fn fileClose( - userdata: ?*anyopaque, - file: std.Io.File, - ) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - self.closeFile(file.handle); - } - - fn fileWriteStreaming( - userdata: ?*anyopaque, - file: std.Io.File, - buffer: [][]const u8, - ) std.Io.File.WriteStreamingError!usize { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_file = self.fsFile(file.handle) catch |err| { - return switch (err) { - FileEntryError.SchemeNotRegistered => std.Io.File.WriteStreamingError.Unexpected, - FileEntryError.InvalidURI => std.Io.File.WriteStreamingError.Unexpected, - FileEntryError.MissingVirtualFileHandle => std.Io.File.WriteStreamingError.Unexpected, - }; - }; - - return fs_file.backend.vtable.fileWriteStreaming(fs_file.backend.userdata, .{ .handle = fs_file.handle }, buffer); - } - - fn fileWritePositional( - userdata: ?*anyopaque, - file: std.Io.File, - buffer: [][]const u8, - offset: u64, - ) std.Io.File.WritePositionalError!usize { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_file = self.fsFile(file.handle) catch |err| { - return switch (err) { - FileEntryError.SchemeNotRegistered => std.Io.File.WritePositionalError.Unexpected, - FileEntryError.InvalidURI => std.Io.File.WritePositionalError.Unexpected, - FileEntryError.MissingVirtualFileHandle => std.Io.File.WritePositionalError.Unexpected, - }; - }; - - return fs_file.backend.vtable.fileWritePositional(fs_file.backend.userdata, .{ .handle = fs_file.handle }, buffer, offset); - } - - fn fileReadStreaming( - userdata: ?*anyopaque, - file: std.Io.File, - data: [][]u8, - ) std.Io.File.Reader.Error!usize { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_file = self.fsFile(file.handle) catch |err| { - return switch (err) { - FileEntryError.SchemeNotRegistered => std.Io.File.Reader.Error.Unexpected, - FileEntryError.InvalidURI => std.Io.File.Reader.Error.Unexpected, - FileEntryError.MissingVirtualFileHandle => std.Io.File.Reader.Error.Unexpected, - }; - }; - - return fs_file.backend.vtable.fileReadStreaming(fs_file.backend.userdata, .{ .handle = fs_file.handle }, data); - } - - fn fileReadPositional( - userdata: ?*anyopaque, - file: std.Io.File, - data: [][]u8, - offset: u64, - ) std.Io.File.ReadPositionalError!usize { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_file = self.fsFile(file.handle) catch |err| { - return switch (err) { - FileEntryError.SchemeNotRegistered => std.Io.File.ReadPositionalError.Unexpected, - FileEntryError.InvalidURI => std.Io.File.ReadPositionalError.Unexpected, - FileEntryError.MissingVirtualFileHandle => std.Io.File.ReadPositionalError.Unexpected, - }; - }; - - return fs_file.backend.vtable.fileReadPositional(fs_file.backend.userdata, .{ .handle = fs_file.handle }, data, offset); - } - - fn fileSeekBy( - userdata: ?*anyopaque, - file: std.Io.File, - relative_offset: i64, - ) std.Io.File.SeekError!void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_file = self.fsFile(file.handle) catch |err| { - return switch (err) { - FileEntryError.SchemeNotRegistered => std.Io.File.SeekError.Unexpected, - FileEntryError.InvalidURI => std.Io.File.SeekError.Unexpected, - FileEntryError.MissingVirtualFileHandle => std.Io.File.SeekError.Unexpected, - }; - }; - - return fs_file.backend.vtable.fileSeekBy(fs_file.backend.userdata, .{ .handle = fs_file.handle }, relative_offset); - } - - fn fileSeekTo( - userdata: ?*anyopaque, - file: std.Io.File, - absolute_offset: u64, - ) std.Io.File.SeekError!void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_file = self.fsFile(file.handle) catch |err| { - return switch (err) { - FileEntryError.SchemeNotRegistered => std.Io.File.SeekError.Unexpected, - FileEntryError.InvalidURI => std.Io.File.SeekError.Unexpected, - FileEntryError.MissingVirtualFileHandle => std.Io.File.SeekError.Unexpected, - }; - }; - - return fs_file.backend.vtable.fileSeekTo(fs_file.backend.userdata, .{ .handle = fs_file.handle }, absolute_offset); - } - - fn openSelfExe(userdata: ?*anyopaque, flags: std.Io.File.OpenFlags) std.Io.File.OpenSelfExeError!std.Io.File { - _ = userdata; - _ = flags; - return std.Io.File.OpenSelfExeError.NotSupported; - } - - // Forwarding wrappers - - fn async( - userdata: ?*anyopaque, - result: []u8, - result_alignment: std.mem.Alignment, - context: []const u8, - context_alignment: std.mem.Alignment, - start: *const fn (context: *const anyopaque, result: *anyopaque) void, - ) ?*std.Io.AnyFuture { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.async; - return f(self.base_io.userdata, result, result_alignment, context, context_alignment, start); - } - - fn concurrent( - userdata: ?*anyopaque, - result_len: usize, - result_alignment: std.mem.Alignment, - context: []const u8, - context_alignment: std.mem.Alignment, - start: *const fn (context: *const anyopaque, result: *anyopaque) void, - ) std.Io.ConcurrentError!*std.Io.AnyFuture { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.concurrent; - return f(self.base_io.userdata, result_len, result_alignment, context, context_alignment, start); - } - - fn await( - userdata: ?*anyopaque, - any_future: *std.Io.AnyFuture, - result: []u8, - result_alignment: std.mem.Alignment, - ) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.await; - f(self.base_io.userdata, any_future, result, result_alignment); - } - - fn cancel( - userdata: ?*anyopaque, - any_future: *std.Io.AnyFuture, - result: []u8, - result_alignment: std.mem.Alignment, - ) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.cancel; - f(self.base_io.userdata, any_future, result, result_alignment); - } - - fn cancelRequested(userdata: ?*anyopaque) bool { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.cancelRequested; - return f(self.base_io.userdata); - } - - fn groupAsync( - userdata: ?*anyopaque, - group: *std.Io.Group, - context: []const u8, - context_alignment: std.mem.Alignment, - start: *const fn (*std.Io.Group, context: *const anyopaque) void, - ) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.groupAsync; - f(self.base_io.userdata, group, context, context_alignment, start); - } - - fn groupConcurrent( - userdata: ?*anyopaque, - group: *std.Io.Group, - context: []const u8, - context_alignment: std.mem.Alignment, - start: *const fn (*std.Io.Group, context: *const anyopaque) void, - ) std.Io.ConcurrentError!void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.groupConcurrent(self.base_io.userdata, group, context, context_alignment, start); - } - - fn groupWait(userdata: ?*anyopaque, group: *std.Io.Group, token: *anyopaque) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.groupWait; - f(self.base_io.userdata, group, token); - } - - fn groupCancel(userdata: ?*anyopaque, group: *std.Io.Group, token: *anyopaque) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.groupCancel; - f(self.base_io.userdata, group, token); - } - - fn select(userdata: ?*anyopaque, futures: []const *std.Io.AnyFuture) std.Io.Cancelable!usize { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.select(self.base_io.userdata, futures); - } - - fn mutexLock(userdata: ?*anyopaque, prev_state: std.Io.Mutex.State, mutex: *std.Io.Mutex) std.Io.Cancelable!void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.mutexLock(self.base_io.userdata, prev_state, mutex); - } - - fn mutexLockUncancelable(userdata: ?*anyopaque, prev_state: std.Io.Mutex.State, mutex: *std.Io.Mutex) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.mutexLockUncancelable; - f(self.base_io.userdata, prev_state, mutex); - } - - fn mutexUnlock(userdata: ?*anyopaque, prev_state: std.Io.Mutex.State, mutex: *std.Io.Mutex) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.mutexUnlock; - f(self.base_io.userdata, prev_state, mutex); - } - - fn conditionWait(userdata: ?*anyopaque, cond: *std.Io.Condition, mutex: *std.Io.Mutex) std.Io.Cancelable!void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.conditionWait(self.base_io.userdata, cond, mutex); - } - - fn conditionWaitUncancelable(userdata: ?*anyopaque, cond: *std.Io.Condition, mutex: *std.Io.Mutex) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.conditionWaitUncancelable; - f(self.base_io.userdata, cond, mutex); - } - - fn conditionWake(userdata: ?*anyopaque, cond: *std.Io.Condition, wake: std.Io.Condition.Wake) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.conditionWake; - f(self.base_io.userdata, cond, wake); - } - - fn now(userdata: ?*anyopaque, clock: std.Io.Clock) std.Io.Clock.Error!std.Io.Timestamp { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.now(self.base_io.userdata, clock); - } - - fn sleep(userdata: ?*anyopaque, timeout: std.Io.Timeout) std.Io.SleepError!void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.sleep(self.base_io.userdata, timeout); - } - - fn netListenIp( - userdata: ?*anyopaque, - address: std.Io.net.IpAddress, - options: std.Io.net.IpAddress.ListenOptions, - ) std.Io.net.IpAddress.ListenError!std.Io.net.Server { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.netListenIp(self.base_io.userdata, address, options); - } - - fn netAccept(userdata: ?*anyopaque, server: std.Io.net.Socket.Handle) std.Io.net.Server.AcceptError!std.Io.net.Stream { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.netAccept(self.base_io.userdata, server); - } - - fn netBindIp( - userdata: ?*anyopaque, - address: *const std.Io.net.IpAddress, - options: std.Io.net.IpAddress.BindOptions, - ) std.Io.net.IpAddress.BindError!std.Io.net.Socket { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.netBindIp(self.base_io.userdata, address, options); - } - - fn netConnectIp( - userdata: ?*anyopaque, - address: *const std.Io.net.IpAddress, - options: std.Io.net.IpAddress.ConnectOptions, - ) std.Io.net.IpAddress.ConnectError!std.Io.net.Stream { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.netConnectIp(self.base_io.userdata, address, options); - } - - fn netListenUnix( - userdata: ?*anyopaque, - address: *const std.Io.net.UnixAddress, - options: std.Io.net.UnixAddress.ListenOptions, - ) std.Io.net.UnixAddress.ListenError!std.Io.net.Socket.Handle { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.netListenUnix(self.base_io.userdata, address, options); - } - - fn netConnectUnix( - userdata: ?*anyopaque, - address: *const std.Io.net.UnixAddress, - ) std.Io.net.UnixAddress.ConnectError!std.Io.net.Socket.Handle { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.netConnectUnix(self.base_io.userdata, address); - } - - fn netSend( - userdata: ?*anyopaque, - handle: std.Io.net.Socket.Handle, - msgs: []std.Io.net.OutgoingMessage, - flags: std.Io.net.SendFlags, - ) struct { ?std.Io.net.Socket.SendError, usize } { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.netSend(self.base_io.userdata, handle, msgs, flags); - } - - fn netReceive( - userdata: ?*anyopaque, - handle: std.Io.net.Socket.Handle, - message_buffer: []std.Io.net.IncomingMessage, - data_buffer: []u8, - flags: std.Io.net.ReceiveFlags, - timeout: std.Io.Timeout, - ) struct { ?std.Io.net.Socket.ReceiveTimeoutError, usize } { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.netReceive(self.base_io.userdata, handle, message_buffer, data_buffer, flags, timeout); - } - - fn netRead(userdata: ?*anyopaque, src: std.Io.net.Socket.Handle, data: [][]u8) std.Io.net.Stream.Reader.Error!usize { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.netRead(self.base_io.userdata, src, data); - } - - fn netWrite( - userdata: ?*anyopaque, - dest: std.Io.net.Socket.Handle, - header: []const u8, - data: []const []const u8, - splat: usize, - ) std.Io.net.Stream.Writer.Error!usize { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.netWrite(self.base_io.userdata, dest, header, data, splat); - } - - fn netClose(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.netClose; - f(self.base_io.userdata, handle); - } - - fn netInterfaceNameResolve( - userdata: ?*anyopaque, - name: *const std.Io.net.Interface.Name, - ) std.Io.net.Interface.Name.ResolveError!std.Io.net.Interface { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.netInterfaceNameResolve(self.base_io.userdata, name); - } - - fn netInterfaceName( - userdata: ?*anyopaque, - iface: std.Io.net.Interface, - ) std.Io.net.Interface.NameError!std.Io.net.Interface.Name { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.netInterfaceName(self.base_io.userdata, iface); - } - - fn netLookup( - userdata: ?*anyopaque, - host: std.Io.net.HostName, - q: *std.Io.Queue(std.Io.net.HostName.LookupResult), - options: std.Io.net.HostName.LookupOptions, - ) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.netLookup; - f(self.base_io.userdata, host, q, options); + if (handle.scheme) |s| { + const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); + const path_len = try backend.vtable.fileRealPath(backend.userdata, .{ .handle = @intCast(handle.handle) }, out_buffer[prefix.len..]); + return prefix.len + path_len; + } else { + return try backend.vtable.fileRealPath(backend.userdata, .{ .handle = @intCast(handle.handle) }, out_buffer); + } } }; diff --git a/zml/io/vfs/BUILD.bazel b/zml/io/vfs/BUILD.bazel index 0a0bc3c4..922a69c7 100644 --- a/zml/io/vfs/BUILD.bazel +++ b/zml/io/vfs/BUILD.bazel @@ -6,17 +6,11 @@ zig_library( "base.zig", "file.zig", "hf.zig", - "proxy.zig", "http.zig", ], + deps = [ + "//stdx", + ], main = "index.zig", visibility = ["//visibility:public"], ) - -zig_binary( - name = "hf2", - main = "hf2.zig", - srcs = ["base.zig"], - deps = ["//stdx"], - visibility = ["//visibility:public"], -) diff --git a/zml/io/vfs/base.zig b/zml/io/vfs/base.zig index 6b032562..e586d10b 100644 --- a/zml/io/vfs/base.zig +++ b/zml/io/vfs/base.zig @@ -3,49 +3,6 @@ const std = @import("std"); const stdx = @import("stdx"); pub const VFSBase = struct { - pub fn HandleRegistry(comptime InnerType: type) type { - return struct { - const Self = @This(); - - const Node = struct { - value: InnerType, - next_closed: ?usize = null, - }; - - mutex: std.Io.Mutex = .init, - handles: stdx.SegmentedList(InnerType, 0) = .{}, - next_closed: ?usize = null, - - pub fn open(self: *Self, allocator: std.mem.Allocator, io: std.Io) !struct { usize, *InnerType } { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - - if (self.next_closed) |handle| { - const node = self.handles.at(handle); - self.next_closed = node.next_closed; - return .{ handle, node.value }; - } - return .{ self.handles.count(), try self.handles.addOne(allocator) }; - } - - pub fn close(self: *Self, io: std.Io, handle: usize) void { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - - const node = self.handles.at(handle); - node.next_closed = self.next_closed; - self.next_closed = handle; - } - - pub fn get(self: *Self, io: std.Io, handle: usize) *InnerType { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - - return &self.handles.at(handle).value; - } - }; - } - inner: std.Io, pub fn init(io: std.Io) VFSBase { @@ -378,7 +335,6 @@ pub const VFSBase = struct { } pub fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { - std.log.info("cocou bg", .{}); const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileWriteFilePositional(self.inner.userdata, file, header, reader, limit, offset); } diff --git a/zml/io/vfs/file.zig b/zml/io/vfs/file.zig index b0f3d757..31fe0e80 100644 --- a/zml/io/vfs/file.zig +++ b/zml/io/vfs/file.zig @@ -1,5 +1,9 @@ -const builtin = @import("builtin"); const std = @import("std"); +const builtin = @import("builtin"); + +const stdx = @import("stdx"); + +const VFSBase = @import("base.zig").VFSBase; const log = std.log.scoped(.@"zml/io/vfs/file"); @@ -58,176 +62,145 @@ pub const File = struct { direct_io_alignment: std.mem.Alignment = .fromByteUnits(4 * 1024), }; - allocator: std.mem.Allocator, - direct_io_map: std.AutoHashMapUnmanaged(std.Io.File.Handle, std.Io.File.Handle), - mutex: std.Io.Mutex, - config: Config, + const Handle = struct { + inner_handle: std.Io.File.Handle, + direct_io_handle: ?std.Io.File.Handle, + }; - inner: std.Io, - vtable: std.Io.VTable, + allocator: std.mem.Allocator, + mutex: std.Io.Mutex = .init, + handles: stdx.SegmentedList(Handle, 0), + closed_handles: std.ArrayList(u32) = .{}, + config: Config, + base: VFSBase, pub fn init(allocator: std.mem.Allocator, inner: std.Io, config: Config) File { - var vtable = inner.vtable.*; - vtable.dirMake = dirMake; - vtable.dirMakePath = dirMakePath; - vtable.dirMakeOpenPath = dirMakeOpenPath; - vtable.dirStatPath = dirStatPath; - vtable.dirAccess = dirAccess; - vtable.dirCreateFile = dirCreateFile; - vtable.dirOpenFile = dirOpenFile; - vtable.fileClose = fileClose; - vtable.dirOpenDir = dirOpenDir; - vtable.fileReadPositional = fileReadPositional; - vtable.fileReadStreaming = fileReadStreaming; - return .{ .allocator = allocator, - .direct_io_map = .{}, - .mutex = .init, + .handles = .{}, .config = config, - .inner = inner, - .vtable = vtable, + .base = .init(inner), }; } pub fn deinit(self: *File) void { - self.direct_io_map.deinit(self.allocator); + self.handles.deinit(self.allocator); + self.closed_handles.deinit(self.allocator); } pub fn io(self: *File) std.Io { return .{ - .userdata = self, - .vtable = &self.vtable, + .userdata = &self.base, + .vtable = &comptime VFSBase.vtable(.{ + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRead = dirRead, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileReadStreaming = fileReadStreaming, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileRealPath = fileRealPath, + }), }; } - fn stripScheme(path: []const u8) []const u8 { - if (std.mem.startsWith(u8, path, "file://")) { - return path[7..]; + fn openHandle(self: *File) !struct { u32, *Handle } { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + if (self.closed_handles.pop()) |idx| { + return .{ idx, self.handles.at(idx) }; } - return path; + return .{ @intCast(self.handles.len), try self.handles.addOne(self.allocator) }; + } + + fn closeHandle(self: *File, idx: u32) !void { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + try self.closed_handles.append(self.allocator, idx); + } + + fn getFileHandle(self: *File, file: std.Io.File) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + return self.handles.at(@intCast(file.handle)); } fn canOpenWithDirectIO(self: *File, file: std.Io.File, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!bool { - const file_stat = self.inner.vtable.fileStat(self.inner.userdata, file) catch |err| switch (err) { + const file_stat = self.base.inner.vtable.fileStat(self.base.inner.userdata, file) catch |err| switch (err) { else => return std.Io.File.OpenError.Unexpected, }; return self.config.direct_io and flags.mode == .read_only and file_stat.size >= self.config.direct_io_alignment.toByteUnits(); } - fn innerFile( - self: *File, - file: std.Io.File, - buffers: [][]u8, - position: u64, - ) std.Io.File { - const alignment_bytes: usize = self.config.direct_io_alignment.toByteUnits(); - var buffers_aligned = true; + fn innerFile(self: *File, file: std.Io.File, data: []const []u8, position: u64) std.Io.File { + const handle = self.getFileHandle(file); + const direct_io_handle = handle.direct_io_handle orelse return .{ .handle = handle.inner_handle }; - for (buffers) |buf| { - const ptr_addr = @intFromPtr(buf.ptr); - if (!std.mem.isAligned(ptr_addr, alignment_bytes) or - !std.mem.isAligned(buf.len, alignment_bytes) or - (buf.len < alignment_bytes)) - { - buffers_aligned = false; - break; - } + const alignment_bytes: usize = self.config.direct_io_alignment.toByteUnits(); + + if (!std.mem.isAligned(@as(usize, position), alignment_bytes)) { + return .{ .handle = handle.inner_handle }; } - const pos_aligned = std.mem.isAligned(@as(usize, position), alignment_bytes); - - self.mutex.lockUncancelable(self.inner); - const direct_io_handle = self.direct_io_map.get(file.handle); - self.mutex.unlock(self.inner); - - return if (direct_io_handle) |handle| blk: { - if (buffers_aligned and pos_aligned) { - break :blk std.Io.File{ .handle = handle }; - } else { - break :blk file; + var total_size: usize = 0; + for (data) |buf| { + if (!std.mem.isAligned(@intFromPtr(buf.ptr), alignment_bytes)) { + return .{ .handle = handle.inner_handle }; } - } else blk: { - break :blk file; - }; + total_size += buf.len; + } + + if (!std.mem.isAligned(total_size, alignment_bytes)) { + return .{ .handle = handle.inner_handle }; + } + + return .{ .handle = direct_io_handle }; } - fn dirMake( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - mode: std.Io.Dir.Mode, - ) std.Io.Dir.MakeError!void { - const self: *File = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirMake(self.inner.userdata, dir, stripScheme(sub_path), mode); + fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.base.inner.vtable.dirOpenDir(self.base.inner.userdata, dir, sub_path, options); } - fn dirMakePath( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - mode: std.Io.Dir.Mode, - ) std.Io.Dir.MakeError!void { - const self: *File = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirMakePath(self.inner.userdata, dir, stripScheme(sub_path), mode); + fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.base.inner.vtable.dirStat(self.base.inner.userdata, dir); } - fn dirMakeOpenPath( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - options: std.Io.Dir.OpenOptions, - ) std.Io.Dir.MakeOpenPathError!std.Io.Dir { - const self: *File = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirMakeOpenPath(self.inner.userdata, dir, stripScheme(sub_path), options); + fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.base.inner.vtable.dirStatFile(self.base.inner.userdata, dir, sub_path, options); } - fn dirStatPath( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - options: std.Io.Dir.StatPathOptions, - ) std.Io.Dir.StatPathError!std.Io.File.Stat { - const self: *File = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirStatPath(self.inner.userdata, dir, stripScheme(sub_path), options); + fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.base.inner.vtable.dirAccess(self.base.inner.userdata, dir, sub_path, options); } - fn dirAccess( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - options: std.Io.Dir.AccessOptions, - ) std.Io.Dir.AccessError!void { - const self: *File = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirAccess(self.inner.userdata, dir, stripScheme(sub_path), options); - } + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const inner_file = try self.base.inner.vtable.dirOpenFile(self.base.inner.userdata, dir, sub_path, flags); + errdefer inner_file.close(self.base.inner); - fn dirCreateFile( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - flags: std.Io.File.CreateFlags, - ) std.Io.File.OpenError!std.Io.File { - const self: *File = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirCreateFile(self.inner.userdata, dir, stripScheme(sub_path), flags); - } - - fn dirOpenFile( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - flags: std.Io.File.OpenFlags, - ) std.Io.File.OpenError!std.Io.File { - const self: *File = @ptrCast(@alignCast(userdata orelse unreachable)); - const file = try self.inner.vtable.dirOpenFile(self.inner.userdata, dir, stripScheme(sub_path), flags); - - const use_direct_io = try self.canOpenWithDirectIO(file, flags); - errdefer file.close(self.inner); + const use_direct_io = try self.canOpenWithDirectIO(inner_file, flags); + var direct_io_handle: ?std.Io.File.Handle = null; if (use_direct_io) { - const direct_io_file = try self.inner.vtable.dirOpenFile(self.inner.userdata, dir, stripScheme(sub_path), flags); - errdefer direct_io_file.close(self.inner); + const direct_io_file = try self.base.inner.vtable.dirOpenFile(self.base.inner.userdata, dir, sub_path, flags); + defer if (direct_io_handle == null) direct_io_file.close(self.base.inner); switchToDirectIO(direct_io_file) catch |err| switch (err) { DirectIoError.UnsupportedPlatform => { @@ -239,67 +212,103 @@ pub const File = struct { }, }; - self.mutex.lockUncancelable(self.inner); - defer self.mutex.unlock(self.inner); - - self.direct_io_map.put(self.allocator, file.handle, direct_io_file.handle) catch |err| { - log.err("Failed to insert Direct I/O file into map: {any}", .{err}); - return std.Io.File.OpenError.Unexpected; - }; + if (useDirectIO(direct_io_file) catch false) { + direct_io_handle = direct_io_file.handle; + } } - return file; + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; + handle.* = .{ + .inner_handle = inner_file.handle, + .direct_io_handle = direct_io_handle, + }; + + return .{ .handle = @intCast(idx) }; } - fn fileClose(userdata: ?*anyopaque, file: std.Io.File) void { - const self: *File = @ptrCast(@alignCast(userdata orelse unreachable)); + fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.base.inner.vtable.dirClose(self.base.inner.userdata, dirs); + } - self.mutex.lockUncancelable(self.inner); - const direct_io_handle = self.direct_io_map.fetchRemove(file.handle); - self.mutex.unlock(self.inner); + fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.base.inner.vtable.dirRead(self.base.inner.userdata, reader, entries); + } - if (direct_io_handle) |handle| { - const direct_io_file: std.Io.File = .{ - .handle = handle.value, - }; + fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.base.inner.vtable.dirRealPath(self.base.inner.userdata, dir, out_buffer); + } - self.inner.vtable.fileClose(self.inner.userdata, direct_io_file); + fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.base.inner.vtable.dirRealPathFile(self.base.inner.userdata, dir, path_name, out_buffer); + } + + fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + const inner_file: std.Io.File = .{ .handle = handle.inner_handle }; + return self.base.inner.vtable.fileStat(self.base.inner.userdata, inner_file); + } + + fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + const inner_file: std.Io.File = .{ .handle = handle.inner_handle }; + return self.base.inner.vtable.fileLength(self.base.inner.userdata, inner_file); + } + + fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + for (files) |file| { + const handle = self.getFileHandle(file); + + if (handle.direct_io_handle) |dio_handle| { + const direct_io_file: std.Io.File = .{ .handle = dio_handle }; + self.base.inner.vtable.fileClose(self.base.inner.userdata, &.{direct_io_file}); + } + + const inner_file: std.Io.File = .{ .handle = handle.inner_handle }; + self.base.inner.vtable.fileClose(self.base.inner.userdata, &.{inner_file}); + + self.closeHandle(@intCast(file.handle)) catch unreachable; } - - return self.inner.vtable.fileClose(self.inner.userdata, file); } - fn dirOpenDir( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - options: std.Io.Dir.OpenOptions, - ) std.Io.Dir.OpenError!std.Io.Dir { - const self: *File = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirOpenDir(self.inner.userdata, dir, stripScheme(sub_path), options); + fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + const position: u64 = @intCast(std.posix.system.lseek(handle.inner_handle, 0, std.posix.SEEK.CUR)); + const inner_file = self.innerFile(file, data, position); + return self.base.inner.vtable.fileReadStreaming(self.base.inner.userdata, inner_file, data); } - fn fileReadPositional( - userdata: ?*anyopaque, - file: std.Io.File, - buffer: [][]u8, - position: u64, - ) std.Io.File.ReadPositionalError!usize { - const self: *File = @ptrCast(@alignCast(userdata orelse unreachable)); - const inner_file = self.innerFile(file, buffer, position); - - return self.inner.vtable.fileReadPositional(self.inner.userdata, inner_file, buffer, position); + fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const inner_file = self.innerFile(file, data, offset); + return self.base.inner.vtable.fileReadPositional(self.base.inner.userdata, inner_file, data, offset); } - fn fileReadStreaming( - userdata: ?*anyopaque, - file: std.Io.File, - buffer: [][]u8, - ) std.Io.File.Reader.Error!usize { - const self: *File = @ptrCast(@alignCast(userdata orelse unreachable)); - const position = std.posix.lseek_CUR_get(file.handle) catch return std.Io.File.Reader.Error.Unexpected; - const inner_file = self.innerFile(file, buffer, position); + fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + const inner_file: std.Io.File = .{ .handle = handle.inner_handle }; + return self.base.inner.vtable.fileSeekBy(self.base.inner.userdata, inner_file, relative_offset); + } - return self.inner.vtable.fileReadStreaming(self.inner.userdata, inner_file, buffer); + fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + const inner_file: std.Io.File = .{ .handle = handle.inner_handle }; + return self.base.inner.vtable.fileSeekTo(self.base.inner.userdata, inner_file, absolute_offset); + } + + fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + const inner_file: std.Io.File = .{ .handle = handle.inner_handle }; + return self.base.inner.vtable.fileRealPath(self.base.inner.userdata, inner_file, out_buffer); } }; diff --git a/zml/io/vfs/hf.zig b/zml/io/vfs/hf.zig index 7cb488e8..e821906e 100644 --- a/zml/io/vfs/hf.zig +++ b/zml/io/vfs/hf.zig @@ -1,420 +1,54 @@ -const json = @import("std").json; const std = @import("std"); +const stdx = @import("stdx"); + +const VFSBase = @import("base.zig").VFSBase; + const log = std.log.scoped(.@"zml/io/vfs/hf"); -const HFApi = struct { - pub const Error = error{ - MalformedURI, - RequestFailed, - ResponseParsingFailed, - } || std.mem.Allocator.Error; +pub const API = struct { + const TREE_URL_TEMPLATE = "https://huggingface.co/api/models/{[repo]s}/{[model]s}/tree/{[rev]s}/{[path]s}?expand=false&recursive=true&limit=1000"; + const LFS_FILE_URL_TEMPLATE = "https://huggingface.co/{[repo]s}/{[model]s}/resolve/{[rev]s}/{[path]s}"; - const PickleImport = struct { - module: []const u8, - name: []const u8, - safety: []const u8, - }; - - const Scan = struct { - status: ?[]const u8 = null, - message: ?[]const u8 = null, - reportLink: ?[]const u8 = null, - reportLabel: ?[]const u8 = null, - pickleImports: ?[]PickleImport = null, - version: ?[]const u8 = null, - }; - - const SecurityFileStatus = struct { - status: ?[]const u8 = null, - jFrogScan: ?Scan = null, - protectAiScan: ?Scan = null, - avScan: ?Scan = null, - pickleImportScan: ?Scan = null, - virusTotalScan: ?Scan = null, - }; - - const LastCommit = struct { - id: []const u8, - title: []const u8, - date: []const u8, - }; - - const Lfs = struct { + pub const Tree = struct { oid: []const u8, - size: u64, - pointerSize: u64, - }; - - pub const TreeEntry = struct { + path: []const u8, type: []const u8, - oid: []const u8, - path: []const u8, - size: u64, - lfs: ?Lfs = null, - lastCommit: ?LastCommit = null, - xetHash: ?[]const u8 = null, - securityFileStatus: ?SecurityFileStatus = null, - }; - - pub const TreeSize = struct { - path: []const u8, size: u64, }; +}; - pub const Model = struct { - namespace: []const u8, - repo: []const u8, - revision: ?[]const u8, - }; +const ReadState = struct { + children: []const TreeNode, + index: usize, +}; - pub fn modelFromUri(uri: []const u8) Error!Model { - const parsed = std.Uri.parse(uri) catch { - log.err("Failed to parse HF model URI: {s}", .{uri}); - return Error.MalformedURI; - }; +pub const TreeNode = struct { + name: []const u8, + kind: std.Io.File.Kind, + size: u64, + children: ?std.ArrayList(TreeNode), - const namespace = parsed.host.?.percent_encoded; - - var path = parsed.path.percent_encoded; - // strip leading slash - if (path.len > 0 and path[0] == '/') path = path[1..]; - // strip trailing slash - if (path.len > 0 and path[path.len - 1] == '/') path = path[0 .. path.len - 1]; - - var it = std.mem.splitSequence(u8, path, "/"); - const repo_and_rev = it.next(); - - if (repo_and_rev == null) { - log.err("Malformed HF model URI, missing repo: {s}", .{uri}); - return Error.MalformedURI; + fn deinit(self: *TreeNode, allocator: std.mem.Allocator) void { + allocator.free(self.name); + if (self.children) |*children| { + for (children.items) |*child| child.deinit(allocator); + children.deinit(allocator); } - - // repo may contain an optional @revision suffix - var rev: ?[]const u8 = null; - var repo = repo_and_rev.?; - const at_index = std.mem.indexOf(u8, repo_and_rev.?, "@"); - if (at_index) |idx| { - repo = repo_and_rev.?[0..idx]; - rev = repo_and_rev.?[idx + 1 ..]; - if (rev.?.len == 0) rev = null; - } - - return .{ - .namespace = namespace, - .repo = repo, - .revision = rev, - }; - } - - pub fn filePathFromUri(uri: []const u8) Error![]const u8 { - const parsed = std.Uri.parse(uri) catch { - log.err("Failed to parse HF file URI: {s}", .{uri}); - return Error.MalformedURI; - }; - - var path = parsed.path.percent_encoded; - // strip leading slash - if (path.len > 0 and path[0] == '/') path = path[1..]; - // strip trailing slash - if (path.len > 0 and path[path.len - 1] == '/') path = path[0 .. path.len - 1]; - - // Remove the first segment (repo[@rev]) - var it = std.mem.splitSequence(u8, path, "/"); - _ = it.next(); // skip repo[@rev] - const rest = it.rest(); - path = rest; - - return path; - } - - pub fn isModelPath(uri: []const u8) bool { - // Determine whether the directory URI represents an HF repo root. - // Valid root examples: - // hf://user/repo - // hf://user/repo/ - // hf://user/repo@rev - // hf://user/repo@rev/ - // Invalid examples: - // hf:// - // hf://user - // hf://user/ - // hf://user/repo/some/path - - const parsed_uri = std.Uri.parse(uri) catch return false; - - // Must be hf scheme with a host - if (!std.mem.eql(u8, parsed_uri.scheme, "hf")) return false; - if (parsed_uri.host == null) return false; - - var path = parsed_uri.path.percent_encoded; - - // No path or root-only path ("/") are not repo roots - if (path.len == 0) return false; - if (path.len == 1 and path[0] == '/') return false; - - // Strip leading slash if present - if (path[0] == '/') path = path[1..]; - - // Strip trailing slash if present - if (path.len > 0 and path[path.len - 1] == '/') path = path[0 .. path.len - 1]; - - // After trimming, there must be exactly one segment (no additional '/') - if (path.len == 0) return false; - - // Check for additional '/' in the path - var i: usize = 0; - while (i < path.len) : (i += 1) { - if (path[i] == '/') return false; - } - - return true; - } - - pub fn fetchModelSize( - allocator: std.mem.Allocator, - client: *std.http.Client, - extra_headers: []std.http.Header, - repo_uri: []const u8, - ) Error!json.Parsed(TreeSize) { - const model = try HFApi.modelFromUri(repo_uri); - const rev = model.revision orelse blk: { - log.debug("No revision specified in repo URI; defaulting to 'main'", .{}); - break :blk "main"; - }; - - var redirect_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; - var url_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const api_url = std.fmt.bufPrint( - &url_buf, - "https://huggingface.co/api/models/{s}/{s}/treesize/{s}/", - .{ model.namespace, model.repo, rev }, - ) catch { - log.err("Failed to format HF model size API URL for repo URI {s}", .{repo_uri}); - return Error.OutOfMemory; - }; - - var req = try allocator.create(std.http.Client.Request); - defer allocator.destroy(req); - - const api_uri = std.Uri.parse(api_url) catch { - log.err("Failed to parse HF model size API URL {s}", .{api_url}); - return Error.MalformedURI; - }; - - req.* = client.request(.GET, api_uri, .{ - .headers = .{ .accept_encoding = .{ .override = "identity" } }, - .extra_headers = extra_headers, - }) catch { - log.err("Failed to create HTTP request for HF model size API URL {s}", .{api_url}); - return Error.RequestFailed; - }; - defer req.deinit(); - - req.sendBodiless() catch { - log.err("Failed to send HTTP request for HF model size API URL {s}", .{api_url}); - return Error.RequestFailed; - }; - - var resp = req.receiveHead(&redirect_buffer) catch { - log.err("Failed to receive HTTP response for HF model size API URL {s}", .{api_url}); - return Error.RequestFailed; - }; - - if (resp.head.status != .ok) { - log.err("Failed to fetch HF model size, got {s} for {s}", .{ @tagName(resp.head.status), api_url }); - return Error.RequestFailed; - } - - const content_length = resp.head.content_length orelse { - log.err("Missing Content-Length in HF model size response for {s}", .{api_url}); - return Error.ResponseParsingFailed; - }; - - const body = resp.reader(&.{}).readAlloc(allocator, content_length) catch { - log.err("Failed to read HF model size response body for {s} of content length {d}", .{ api_url, content_length }); - return Error.ResponseParsingFailed; - }; - defer allocator.free(body); - - const parsed = json.parseFromSlice( - HFApi.TreeSize, - allocator, - body, - .{ .ignore_unknown_fields = true }, - ) catch { - log.err("Failed to parse HF model size response JSON for {s}", .{api_url}); - return Error.ResponseParsingFailed; - }; - - return parsed; } }; -// todo: ref counters -const HFRecords = struct { - pub const Error = HFApi.Error || std.mem.Allocator.Error; +const RepoKey = struct { + repo: []const u8, + model: []const u8, + rev: []const u8, - const Mapping = std.StringArrayHashMapUnmanaged(HFApi.TreeEntry); - - arena: std.heap.ArenaAllocator, - mapping: Mapping, - - pub fn init(allocator: std.mem.Allocator) HFRecords { - return .{ - .arena = std.heap.ArenaAllocator.init(allocator), - .mapping = .{}, - }; + fn format(self: RepoKey, buffer: []u8) ![]u8 { + return std.fmt.bufPrint(buffer, "{s}/{s}@{s}", .{ self.repo, self.model, self.rev }); } - pub fn get(self: *HFRecords, key: []const u8) ?HFApi.TreeEntry { - // Normalize key by stripping directory trailing slash if present - const normalized_key = if (key.len > 0 and key[key.len - 1] == '/') key[0 .. key.len - 1] else key; - - return self.mapping.get(normalized_key); - } - - pub fn containsPathPrefix(self: *HFRecords, prefix: []const u8) bool { - var it = self.mapping.iterator(); - while (it.next()) |kv| { - const key = kv.key_ptr.*; - if (key.len >= prefix.len and std.mem.startsWith(u8, key, prefix)) { - return true; - } - } - - return false; - } - - pub fn fetch( - self: *HFRecords, - client: *std.http.Client, - limit: usize, - extra_headers: []std.http.Header, - repo_uri: []const u8, - url: ?[]const u8, // used for pagination - ) Error!void { - const allocator = self.arena.allocator(); - - var url_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - var redirect_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; - - const api_url = url orelse blk: { - const model = try HFApi.modelFromUri(repo_uri); - const rev = model.revision orelse blk_: { - log.debug("No revision specified in repo URI; defaulting to 'main'", .{}); - break :blk_ "main"; - }; - - break :blk std.fmt.bufPrint( - &url_buf, - "https://huggingface.co/api/models/{s}/{s}/tree/{s}/?expand=true&recursive=true&limit={d}", - .{ model.namespace, model.repo, rev, limit }, - ) catch { - log.err("Failed to format HF repo tree API URL for repo URI {s}", .{repo_uri}); - return Error.OutOfMemory; - }; - }; - - const api_uri = std.Uri.parse(api_url) catch { - log.err("Failed to parse HF repo tree API URL {s}", .{api_url}); - return Error.MalformedURI; - }; - - var req = try allocator.create(std.http.Client.Request); - defer allocator.destroy(req); - - req.* = client.request(.GET, api_uri, .{ - .headers = .{ .accept_encoding = .{ .override = "identity" } }, - .extra_headers = extra_headers, - }) catch { - log.err("Failed to create HTTP request for HF repo tree API URL {s}", .{api_url}); - return Error.RequestFailed; - }; - defer req.deinit(); - - req.sendBodiless() catch { - log.err("Failed to send HTTP request for HF repo tree API URL {s}", .{api_url}); - return Error.RequestFailed; - }; - - var resp = req.receiveHead(&redirect_buffer) catch { - log.err("Failed to receive HTTP response for HF repo tree API URL {s}", .{api_url}); - return Error.RequestFailed; - }; - if (resp.head.status != .ok) { - log.err("Failed to fetch HF repo tree, got {s} for {s}", .{ @tagName(resp.head.status), api_url }); - return Error.RequestFailed; - } - - var next_url: ?[]const u8 = null; - var it = resp.head.iterateHeaders(); - while (it.next()) |hdr| { - if (std.mem.eql(u8, hdr.name, "Link") or std.mem.eql(u8, hdr.name, "link")) { - const hv = hdr.value; - var i: usize = 0; - while (i < hv.len) : (i += 1) { - const lt = std.mem.indexOf(u8, hv[i..], "<"); - if (lt == null) break; - const start = i + lt.? + 1; - const gt = std.mem.indexOf(u8, hv[start..], ">"); - if (gt == null) break; - const end = start + gt.?; - const url_slice = hv[start..end]; - const after = hv[end..]; - if (std.mem.indexOf(u8, after, "rel=\"next\"") != null) { - next_url = url_slice; - break; - } - i = end + 1; - } - } - } - - const content_length = resp.head.content_length orelse { - log.err("Missing Content-Length in HF repo tree response for {s}", .{api_url}); - return Error.ResponseParsingFailed; - }; - const body = resp.reader(&.{}).readAlloc(allocator, content_length) catch { - log.err("Failed to read HF repo tree response body for {s} of content length {d}", .{ api_url, content_length }); - return Error.ResponseParsingFailed; - }; - - const entries = json.parseFromSliceLeaky( - []HFApi.TreeEntry, - allocator, - body, - .{ .ignore_unknown_fields = true }, - ) catch { - log.err("Failed to parse HF repo tree response JSON for {s}", .{api_url}); - return Error.ResponseParsingFailed; - }; - - for (entries) |entry| { - try self.add(repo_uri, entry); - } - - if (next_url) |url_| { - try self.fetch(client, limit, extra_headers, repo_uri, url_); - } - } - - fn add(self: *HFRecords, repo_uri: []const u8, entry: HFApi.TreeEntry) Error!void { - const allocator = self.arena.allocator(); - - // Ensure keys are built from the canonical repo root (hf://namespace/repo[@rev]/) - const model = try HFApi.modelFromUri(repo_uri); - const base = if (model.revision) |rev| blk: { - break :blk try std.fmt.allocPrint(allocator, "hf://{s}/{s}@{s}/", .{ model.namespace, model.repo, rev }); - } else blk: { - break :blk try std.fmt.allocPrint(allocator, "hf://{s}/{s}/", .{ model.namespace, model.repo }); - }; - const key = try std.fmt.allocPrint(allocator, "{s}{s}", .{ base, entry.path }); - try self.mapping.put(allocator, key, entry); - } - - pub fn deinit(self: *HFRecords) void { - self.arena.deinit(); + fn allocFormat(self: RepoKey, allocator: std.mem.Allocator) ![]u8 { + return std.fmt.allocPrint(allocator, "{s}/{s}@{s}", .{ self.repo, self.model, self.rev }); } }; @@ -428,7 +62,8 @@ pub const HF = struct { InvalidHFToken, MissingHomePath, MissingConfigFile, - } || std.process.GetEnvVarOwnedError || std.fs.File.OpenError || std.fs.File.Reader.Error || std.Io.Reader.ReadAllocError || std.mem.Allocator.Error; + Unexpected, + }; pub fn deinit(self: *Auth, allocator: std.mem.Allocator) void { switch (self.*) { @@ -446,902 +81,737 @@ pub const HF = struct { log.warn("No Hugging Face authentication token found in environment or home config; proceeding without authentication.", .{}); return .none; }, - else => return e, + else => return AuthError.Unexpected, }, - else => |e| return e, + else => return AuthError.Unexpected, }; } pub fn fromEnv(allocator: std.mem.Allocator) AuthError!Auth { const token = std.process.getEnvVarOwned(allocator, "HF_TOKEN") catch |err| switch (err) { error.EnvironmentVariableNotFound => return AuthError.MissingHFToken, - else => |e| return e, + else => return AuthError.Unexpected, }; defer allocator.free(token); - return .{ .hf_token = try std.fmt.allocPrint(allocator, "Bearer {s}", .{token}) }; + return .{ .hf_token = std.fmt.allocPrint(allocator, "Bearer {s}", .{token}) catch return AuthError.Unexpected }; } - pub fn fromHomeConfig(allocator: std.mem.Allocator, io_: std.Io) AuthError!Auth { + pub fn fromHomeConfig(allocator: std.mem.Allocator, base_io: std.Io) AuthError!Auth { const home_path = std.process.getEnvVarOwned(allocator, "HOME") catch |err| switch (err) { error.EnvironmentVariableNotFound => return AuthError.MissingHomePath, - else => |e| return e, + else => return AuthError.Unexpected, }; defer allocator.free(home_path); - const token_path = try std.fmt.allocPrint(allocator, "{s}/.cache/huggingface/token", .{home_path}); + const token_path = std.fmt.allocPrint(allocator, "{s}/.cache/huggingface/token", .{home_path}) catch return AuthError.Unexpected; defer allocator.free(token_path); - const file = std.Io.File.openAbsolute(io_, token_path, .{ .mode = .read_only }) catch |err| switch (err) { + const file = std.Io.Dir.openFileAbsolute(base_io, token_path, .{ .mode = .read_only }) catch |err| switch (err) { error.FileNotFound => return AuthError.MissingConfigFile, - else => |e| return e, + else => return AuthError.Unexpected, }; - var reader = file.reader(io_, &.{}); - const size = reader.getSize() catch unreachable; + var reader = file.reader(base_io, &.{}); + const size = reader.getSize() catch return AuthError.Unexpected; - const token = try reader.interface.readAlloc(allocator, size); + const token = reader.interface.readAlloc(allocator, size) catch return AuthError.Unexpected; defer allocator.free(token); - return .{ .hf_token = try std.fmt.allocPrint(allocator, "Bearer {s}", .{token}) }; + return .{ .hf_token = std.fmt.allocPrint(allocator, "Bearer {s}", .{std.mem.trim(u8, token, "\n")}) catch return AuthError.Unexpected }; } }; - const HandleId = i32; + pub const Repo = struct { + repo: []const u8, + model: []const u8, + rev: []const u8, + path: []const u8, - pub const InitHandleError = error{ - PathTooLong, - PathUnresolvable, - FailedToFetchRemoteResource, - MissingRessource, - } || std.mem.Allocator.Error; + pub fn parse(uri: []const u8) !Repo { + var parts = std.mem.splitScalar(u8, uri, '/'); + var repo: Repo = .{ + .repo = parts.first(), + .model = parts.next() orelse return error.InvalidURI, + .rev = "main", + .path = std.mem.trimEnd(u8, parts.rest(), "/"), + }; + if (std.mem.findScalar(u8, repo.model, '@')) |at_index| { + repo.rev = repo.model[at_index + 1 ..]; + repo.model = repo.model[0..at_index]; + } + return repo; + } - pub const FileHandle = struct { - allocator: std.mem.Allocator, + fn toKey(self: Repo) RepoKey { + return .{ .repo = self.repo, .model = self.model, .rev = self.rev }; + } + }; + const Handle = struct { + pub const Type = enum { + file, + directory, + }; + + type: Type, uri: []const u8, pos: u64, size: u64, - redirect_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined, - request: ?*std.http.Client.Request = null, - response: ?std.http.Client.Response = null, - body_reader: ?*std.Io.Reader = null, - total_requests: u64 = 0, - total_latency_ns: u64 = 0, - min_latency_ns: u64 = std.math.maxInt(u64), - max_latency_ns: u64 = 0, - - pub fn deinit(self: *FileHandle) void { - self.releaseRequest(); - self.allocator.free(self.uri); - self.allocator.destroy(self); + pub fn init(allocator: std.mem.Allocator, handle_type: Type, uri: []const u8, size: u64) !Handle { + return .{ + .type = handle_type, + .uri = try allocator.dupe(u8, uri), + .pos = 0, + .size = size, + }; } - pub fn releaseRequest(self: *FileHandle) void { - if (self.request) |req| { - req.deinit(); - self.allocator.destroy(req); - self.request = null; - self.response = null; - self.body_reader = null; - } - } - - pub fn recordLatency(self: *FileHandle, latency_ns: u64) void { - self.total_requests += 1; - self.total_latency_ns += latency_ns; - self.min_latency_ns = @min(self.min_latency_ns, latency_ns); - self.max_latency_ns = @max(self.max_latency_ns, latency_ns); - } - - pub fn avgLatencyMs(self: *FileHandle) u64 { - if (self.total_requests == 0) return 0; - return (self.total_latency_ns / self.total_requests) / std.time.ns_per_ms; + pub fn deinit(self: *Handle, allocator: std.mem.Allocator) void { + allocator.free(self.uri); } }; - pub const DirHandle = struct { - allocator: std.mem.Allocator, - uri: []const u8, - size: u64, - - pub fn deinit(self: *DirHandle) void { - self.allocator.free(self.uri); - self.allocator.destroy(self); - } - }; - - pub const Config = struct { + const Config = struct { + http_client: *std.http.Client, auth: Auth = .none, - request_range_min: u64 = 8 * 1024, - request_range_max: u64 = 128 * 1024 * 1024, - hf_pagination_limit: usize = 100, }; allocator: std.mem.Allocator, - mutex: std.Io.Mutex, - - file_handles: std.AutoHashMapUnmanaged(HandleId, *FileHandle), - dir_handles: std.AutoHashMapUnmanaged(HandleId, *DirHandle), - next_file_handle_id: HandleId, - next_dir_handle_id: HandleId, - - hf_records: HFRecords, - + mutex: std.Io.Mutex = .init, client: *std.http.Client, - http_headers: []std.http.Header, - config: Config, + authorization: std.http.Client.Request.Headers.Value = .default, + handles: stdx.SegmentedList(Handle, 0), + closed_handles: std.ArrayList(u32) = .{}, + base: VFSBase, + trees: std.StringHashMapUnmanaged(std.ArrayList(TreeNode)) = .{}, + dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, - vtable: std.Io.VTable, - - pub fn init(allocator: std.mem.Allocator, base_io: std.Io, http_client: *std.http.Client, config: Config) std.mem.Allocator.Error!HF { - var self: HF = .{ - .allocator = allocator, - .mutex = .init, - .file_handles = .{}, - .dir_handles = .{}, - .next_file_handle_id = 0, - .next_dir_handle_id = 0, - .hf_records = .init(allocator), - .client = http_client, - .http_headers = try allocator.alloc(std.http.Header, 2), - .config = config, - .vtable = makeVTable(base_io), + pub fn init(allocator: std.mem.Allocator, base_io: std.Io, config: Config) !HF { + var handles: stdx.SegmentedList(Handle, 0) = .{}; + // cwd + const handle = try handles.addOne(allocator); + handle.* = .{ + .type = .directory, + .uri = "", + .pos = 0, + .size = 0, }; - switch (self.config.auth) { + var hf: HF = .{ + .allocator = allocator, + .handles = handles, + .base = .init(base_io), + .client = config.http_client, + }; + + switch (config.auth) { .hf_token => |token| { - self.http_headers[0] = .{ .name = "Authorization", .value = token }; + hf.authorization = .{ .override = token }; }, - else => {}, + .none => {}, } - return self; + return hf; } pub fn deinit(self: *HF) void { - var file_it = self.file_handles.valueIterator(); - while (file_it.next()) |handle_ptr| { - handle_ptr.*.deinit(); - } - self.file_handles.deinit(self.allocator); + var idx: usize = 0; + while (idx < self.handles.len) : (idx += 1) { + const is_closed = for (self.closed_handles.items) |closed_idx| { + if (closed_idx == idx) break true; + } else false; - var dir_it = self.dir_handles.valueIterator(); - while (dir_it.next()) |handle_ptr| { - handle_ptr.*.deinit(); + if (!is_closed) { + self.handles.at(idx).deinit(self.allocator); + } } - self.dir_handles.deinit(self.allocator); - self.hf_records.deinit(); - self.allocator.free(self.http_headers); + self.handles.deinit(self.allocator); + self.closed_handles.deinit(self.allocator); + + var it = self.trees.iterator(); + while (it.next()) |entry| { + for (entry.value_ptr.items) |*node| { + node.deinit(self.allocator); + } + entry.value_ptr.deinit(self.allocator); + self.allocator.free(entry.key_ptr.*); + } + self.trees.deinit(self.allocator); + self.dir_read_states.deinit(self.allocator); } pub fn io(self: *HF) std.Io { return .{ - .vtable = &self.vtable, - .userdata = self, + .userdata = &self.base, + .vtable = &comptime VFSBase.vtable(.{ + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRead = dirRead, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileReadStreaming = fileReadStreaming, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileRealPath = fileRealPath, + }), }; } - const URIKind = union(enum) { - dir, - file, - }; + fn openHandle(self: *HF) !struct { u32, *Handle } { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); - fn resolveUri(self: *HF, parent_path: []const u8, sub_path: []const u8, kind: URIKind) InitHandleError![]const u8 { - if (std.mem.startsWith(u8, sub_path, "hf://")) { - if (kind == .dir and !std.mem.endsWith(u8, sub_path, "/")) { - return std.fmt.allocPrint(self.allocator, "{s}/", .{sub_path}); - } else { - return try self.allocator.dupe(u8, sub_path); - } - } - - const base_uri = std.Uri.parse(parent_path) catch return InitHandleError.PathUnresolvable; - var aux_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - - if (sub_path.len > aux_buf.len) return InitHandleError.PathTooLong; - - @memcpy(aux_buf[0..sub_path.len], sub_path); - var aux_slice: []u8 = aux_buf[0..]; - const resolved = std.Uri.resolveInPlace(base_uri, sub_path.len, &aux_slice) catch return InitHandleError.PathUnresolvable; - if (kind == .dir and !std.mem.endsWith(u8, resolved.path.percent_encoded, "/")) { - return std.fmt.allocPrint(self.allocator, "{f}/", .{resolved.fmt(.{ - .scheme = true, - .authority = true, - .path = true, - .query = false, - .fragment = false, - })}); - } else { - return std.fmt.allocPrint(self.allocator, "{f}", .{resolved.fmt(.{ - .scheme = true, - .authority = true, - .path = true, - .query = false, - .fragment = false, - })}); + if (self.closed_handles.pop()) |idx| { + return .{ idx, self.handles.at(idx) }; } + const len: u32 = @intCast(self.handles.len); + return .{ len, try self.handles.addOne(self.allocator) }; } - fn fetchHFRecordsIfNeeded( - self: *HF, - uri: []const u8, - ) HFRecords.Error!void { - var extra_headers: []std.http.Header = &.{}; - switch (self.config.auth) { - .hf_token => |_| { - extra_headers = self.http_headers[0..1]; - }, - else => {}, - } + fn closeHandle(self: *HF, idx: u32) !void { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); - if (!self.hf_records.containsPathPrefix(uri)) { - log.debug("Fetching HF records for URI {s}", .{uri}); - try self.hf_records.fetch(self.client, self.config.hf_pagination_limit, extra_headers, uri, null); - } + self.handles.at(idx).deinit(self.allocator); + try self.closed_handles.append(self.allocator, idx); } - fn dirSize(self: *HF, uri: []const u8) HFApi.Error!u64 { - var extra_headers: []std.http.Header = &.{}; - switch (self.config.auth) { - .hf_token => |_| { - extra_headers = self.http_headers[0..1]; - }, - else => {}, - } - - if (HFApi.isModelPath(uri)) { - const model_size = HFApi.fetchModelSize( - self.allocator, - self.client, - extra_headers, - uri, - ) catch |err| { - log.err("Failed to fetch model size for dirSize with URI {s}", .{uri}); - return err; - }; - defer model_size.deinit(); - - return model_size.value.size; - } else { - const entry = self.hf_records.get(uri) orelse { - log.err("Directory path not found in repo tree entries with URI {s}", .{uri}); - return HFApi.Error.ResponseParsingFailed; - }; - - return entry.size; - } + fn getFileHandle(self: *HF, file: std.Io.File) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + return self.handles.at(@intCast(file.handle)); } - fn initFileHandle( - self: *HF, - dir: DirHandle, - sub_path: []const u8, - ) InitHandleError!*FileHandle { - const handle = try self.allocator.create(FileHandle); - errdefer self.allocator.destroy(handle); - - const uri = try self.resolveUri(dir.uri, sub_path, .file); - errdefer self.allocator.free(uri); - - self.fetchHFRecordsIfNeeded(uri) catch { - log.err("Failed to fetch records in dirOpenFile for URI {s}", .{uri}); - return InitHandleError.FailedToFetchRemoteResource; - }; - - const entry = self.hf_records.get(uri) orelse { - log.err("Directory path not found in repo tree entries with URI {s}", .{uri}); - return InitHandleError.MissingRessource; - }; - - handle.* = .{ - .allocator = self.allocator, - .uri = uri, - .pos = 0, - .size = entry.size, - }; - - return handle; + fn getDirHandle(self: *HF, dir: std.Io.Dir) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + return self.handles.at(@intCast(dir.handle)); } - fn registerFileHandle(self: *HF, handle: *FileHandle) std.mem.Allocator.Error!HandleId { - self.mutex.lockUncancelable(self.io()); - defer self.mutex.unlock(self.io()); - - const handle_id = self.next_file_handle_id; - self.next_file_handle_id += 1; - - try self.file_handles.put(self.allocator, handle_id, handle); - - return handle_id; - } - - fn fileHandle(self: *HF, file: std.Io.File) ?*FileHandle { - self.mutex.lockUncancelable(self.io()); - defer self.mutex.unlock(self.io()); - - return self.file_handles.get(file.handle); - } - - fn initDirHandle( - self: *HF, - parent_path: []const u8, - sub_path: []const u8, - ) InitHandleError!*DirHandle { - const dir_handle = try self.allocator.create(DirHandle); - errdefer self.allocator.destroy(dir_handle); - - const uri = try self.resolveUri(parent_path, sub_path, .dir); - errdefer self.allocator.free(uri); - - self.fetchHFRecordsIfNeeded(uri) catch { - log.err("Failed to fetch HF records in dirOpenFile for URI {s}", .{uri}); - return InitHandleError.FailedToFetchRemoteResource; - }; - - const dir_size = self.dirSize(uri) catch { - log.err("Failed to get directory size in dirOpenFile for URI {s}", .{uri}); - return InitHandleError.FailedToFetchRemoteResource; - }; - - dir_handle.* = .{ - .allocator = self.allocator, - .uri = uri, - .size = dir_size, - }; - - return dir_handle; - } - - fn registerDirHandle(self: *HF, handle: *DirHandle) std.mem.Allocator.Error!HandleId { - self.mutex.lockUncancelable(self.io()); - defer self.mutex.unlock(self.io()); - - const handle_id = self.next_dir_handle_id; - self.next_dir_handle_id += 1; - - try self.dir_handles.put(self.allocator, handle_id, handle); - - return handle_id; - } - - fn dirHandle(self: *HF, dir: std.Io.Dir) ?DirHandle { - self.mutex.lockUncancelable(self.io()); - defer self.mutex.unlock(self.io()); - + fn resolvePath(self: *HF, dir: std.Io.Dir, sub_path: []const u8, out_buffer: []u8) ![]u8 { if (std.meta.eql(dir, std.Io.Dir.cwd())) { - return .{ - .allocator = self.allocator, - .uri = "", - .size = 0, - }; + return try std.fmt.bufPrint(out_buffer, "{s}", .{sub_path}); } - if (self.dir_handles.get(dir.handle)) |handle_ptr| { - return handle_ptr.*; - } else { - return null; - } + const handle = self.getDirHandle(dir); + + const trimmed_uri = std.mem.trimEnd(u8, handle.uri, "/"); + const trimmed_sub_path = std.mem.trimStart(u8, sub_path, "/"); + + if (trimmed_uri.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_sub_path}); + if (trimmed_sub_path.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_uri}); + return try std.fmt.bufPrint(out_buffer, "{s}/{s}", .{ trimmed_uri, trimmed_sub_path }); } - fn openFile( - self: *HF, - dir: std.Io.Dir, - sub_path: []const u8, - ) std.Io.File.OpenError!std.Io.File { - const dir_handle = self.dirHandle(dir) orelse { - log.warn("Directory handle not found for dirOpenFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); - return std.Io.File.OpenError.FileNotFound; - }; + fn getOrFetchTree(self: *HF, repo: Repo) !*std.ArrayList(TreeNode) { + const key = repo.toKey(); - const handle = self.initFileHandle(dir_handle, sub_path) catch { - log.warn("Failed to init FileHandle in openFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); - return std.Io.File.OpenError.SystemResources; - }; + var cache_key_buffer: [512]u8 = undefined; - const handle_id = self.registerFileHandle(handle) catch { - log.warn("Failed to register FileHandle in openFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); - return std.Io.File.OpenError.SystemResources; - }; + const cache_key_lookup = try key.format(&cache_key_buffer); - return .{ .handle = @intCast(handle_id) }; - } + if (self.trees.getPtr(cache_key_lookup)) |tree_ptr| return tree_ptr; - fn openDir( - self: *HF, - dir: std.Io.Dir, - sub_path: []const u8, - ) std.Io.Dir.OpenError!std.Io.Dir { - const dir_handle = self.dirHandle(dir) orelse { - log.warn("Directory handle not found for dirOpenFile", .{}); - return std.Io.Dir.OpenError.FileNotFound; - }; + const cache_key = try key.allocFormat(self.allocator); + errdefer self.allocator.free(cache_key); - const handle = self.initDirHandle(dir_handle.uri, sub_path) catch { - log.warn("Failed to init DirHandle in openDir with dir={any} sub_path={s}", .{ dir.handle, sub_path }); - return std.Io.Dir.OpenError.SystemResources; - }; + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); - const handle_id = self.registerDirHandle(handle) catch { - log.warn("Failed to register DirHandle in openDir with dir={any} sub_path={s}", .{ dir.handle, sub_path }); - return std.Io.Dir.OpenError.SystemResources; - }; + const gop = try self.trees.getOrPut(self.allocator, cache_key); + errdefer _ = self.trees.remove(cache_key); - return .{ .handle = @intCast(handle_id) }; - } - - fn closeDir(self: *HF, dir: std.Io.Dir) void { - self.mutex.lockUncancelable(self.io()); - defer self.mutex.unlock(self.io()); - - const handle_kv = self.dir_handles.fetchRemove(dir.handle) orelse { - log.warn("Attempted to close non-existent dir handle: {d}", .{dir.handle}); - return; - }; - - handle_kv.value.deinit(); - } - - fn closeFile(self: *HF, file: std.Io.File) void { - self.mutex.lockUncancelable(self.io()); - defer self.mutex.unlock(self.io()); - - const handle_kv = self.file_handles.fetchRemove(file.handle) orelse { - log.warn("Attempted to close non-existent file handle: {d}", .{file.handle}); - return; - }; - - handle_kv.value.deinit(); - } - - fn setAbsolutePos(self: *HF, file: std.Io.File, pos: u64) std.Io.File.SeekError!void { - const handle = self.fileHandle(file) orelse return std.Io.File.SeekError.Unexpected; - handle.releaseRequest(); - handle.pos = pos; - } - - fn setRelativeOffset(self: *HF, file: std.Io.File, offset: i64) std.Io.File.SeekError!void { - const handle = self.fileHandle(file) orelse return std.Io.File.SeekError.Unexpected; - handle.releaseRequest(); - - const new_pos = @as(i64, @intCast(handle.pos)) + offset; - handle.pos = @intCast(new_pos); - } - - fn performPositionalRead(self: *HF, file: std.Io.File, data: [][]u8, offset: u64) std.Io.File.ReadPositionalError!usize { - const handle = self.fileHandle(file) orelse return std.Io.File.ReadPositionalError.Unexpected; - - const saved_pos = handle.pos; - - if (handle.pos != offset) { - handle.releaseRequest(); - handle.pos = offset; + if (gop.found_existing) { + self.allocator.free(cache_key); + return gop.value_ptr; } - const bytes_read = try self.performRead(file, data); + const tree_root = try self.fetchTreeFromAPI(repo); + gop.value_ptr.* = tree_root; - handle.pos = saved_pos; - handle.releaseRequest(); - - return bytes_read; + return gop.value_ptr; } - fn performRead(self: *HF, file: std.Io.File, data: [][]u8) std.Io.File.Reader.Error!usize { - const handle = self.fileHandle(file) orelse { - log.err("Invalid file handle in VFS read", .{}); - return std.Io.File.Reader.Error.Unexpected; - }; + fn fetchTreeFromAPI(self: *HF, repo: Repo) !std.ArrayList(TreeNode) { + var url_buffer: [8 * 1024]u8 = undefined; + var redirect_buffer: [8 * 1024]u8 = undefined; - if (data.len == 0) return 0; - - var total_read: usize = 0; - - for (data) |buf| { - var offset: usize = 0; - - while (offset < buf.len) { - // Check EOF against known file size - if (handle.pos >= handle.size) { - return total_read; - } - - const start = handle.pos; - var desired_read_len: u64 = @max(@as(u64, @intCast(buf.len)), self.config.request_range_min); - if (desired_read_len > self.config.request_range_max) desired_read_len = self.config.request_range_max; - - const end = if (handle.size > 0 and start < handle.size) @min(start + desired_read_len - 1, handle.size - 1) else start + desired_read_len - 1; - - try self.requestRead(file, start, end); - const reader = handle.body_reader.?; - - const dest = buf[offset..]; - var iov = [_][]u8{dest}; - - const n = reader.readVec(&iov) catch |err| switch (err) { - error.EndOfStream => { - log.debug("Reached EndOfStream for handle={d} pos={d}, will re-request remaining", .{ file.handle, handle.pos }); - handle.releaseRequest(); - continue; - }, - else => { - return std.Io.File.Reader.Error.Unexpected; - }, - }; - - offset += n; - handle.pos += n; - total_read += n; - } - } - return total_read; - } - - fn requestRead(self: *HF, file: std.Io.File, start: u64, end: u64) std.Io.File.Reader.Error!void { - const handle = self.fileHandle(file) orelse return std.Io.File.Reader.Error.Unexpected; - - if (handle.request) |_| return; - - std.debug.assert(start <= end); - - var range_buf: [64]u8 = undefined; - const range_header = std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ start, end }) catch unreachable; - - var timer = std.time.Timer.start() catch { - log.err("Failed to start timer for request", .{}); - return std.Io.File.Reader.Error.SystemResources; - }; - - const request = self.allocator.create(std.http.Client.Request) catch { - log.err("Failed to allocate request", .{}); - return std.Io.File.Reader.Error.SystemResources; - }; - errdefer self.allocator.destroy(request); - - var url_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - - const path = HFApi.filePathFromUri(handle.uri) catch |err| { - log.err("Failed to extract file path from URI {s}: {}", .{ handle.uri, err }); - return std.Io.File.Reader.Error.Unexpected; - }; - - const api_url = blk: { - const model = HFApi.modelFromUri(handle.uri) catch |err| { - log.err("Failed to parse model from URI {s}: {}", .{ handle.uri, err }); - return std.Io.File.Reader.Error.Unexpected; - }; - const rev = model.revision orelse blk_: { - log.debug("No revision specified in repo URI; defaulting to 'main'", .{}); - break :blk_ "main"; - }; - - break :blk std.fmt.bufPrint( - &url_buf, - "https://huggingface.co/{s}/{s}/resolve/{s}/{s}", - .{ model.namespace, model.repo, rev, path }, - ) catch { - log.err("Failed to format HF repo tree API URL for repo URI {s}", .{handle.uri}); - return std.Io.File.Reader.Error.SystemResources; - }; - }; - - const api_uri = std.Uri.parse(api_url) catch { - log.err("Failed to parse URL during read request", .{}); - return std.Io.File.Reader.Error.Unexpected; - }; - - self.http_headers[1] = .{ .name = "Range", .value = range_header }; - - var extra_headers: []std.http.Header = &.{}; - switch (self.config.auth) { - .hf_token => |_| { - extra_headers = self.http_headers[0..2]; - }, - else => { - extra_headers = self.http_headers[1..2]; - }, - } - - request.* = self.client.request(.GET, api_uri, .{ - .headers = .{ .accept_encoding = .{ .override = "identity" } }, - .extra_headers = extra_headers, - }) catch |err| { - log.err("Failed to create GET request: {}", .{err}); - return std.Io.File.Reader.Error.SystemResources; - }; - errdefer request.deinit(); - - request.sendBodiless() catch |err| { - log.err("Failed to send GET request: {}", .{err}); - return std.Io.File.Reader.Error.SystemResources; - }; - - const response = request.receiveHead(&handle.redirect_buffer) catch { - log.err("Failed to receive header response for URL {s} ", .{api_url}); - return std.Io.File.Reader.Error.Unexpected; - }; - - const latency = timer.read(); - handle.recordLatency(latency); - - log.debug("GET call: handle={d} status={s} content_length={?} range={d}-{d} ({d}) latency={d}ms avg={d}ms min={d}ms max={d}ms for {s}", .{ - file.handle, - @tagName(response.head.status), - response.head.content_length, - start, - end, - end - start + 1, - latency / std.time.ns_per_ms, - handle.avgLatencyMs(), - handle.min_latency_ns / std.time.ns_per_ms, - handle.max_latency_ns / std.time.ns_per_ms, - api_url, + const url = try std.fmt.bufPrint(&url_buffer, API.TREE_URL_TEMPLATE, .{ + .repo = repo.repo, + .model = repo.model, + .rev = repo.rev, + .path = "", }); + const uri = try std.Uri.parse(url); - if (response.head.status != .partial_content and response.head.status != .ok) { - log.err("Unexpected HTTP status ({s}) for GET {s}", .{ @tagName(response.head.status), api_url }); - var it = response.head.iterateHeaders(); - while (it.next()) |hdr| { - log.err("Header: {s}: {s}", .{ hdr.name, hdr.value }); + var req = try self.client.request(.GET, uri, .{ + .headers = .{ .authorization = self.authorization }, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var res = try req.receiveHead(&redirect_buffer); + + if (res.head.status != .ok) { + log.err("Failed to fetch tree: status={}", .{res.head.status}); + return error.RequestFailed; + } + + const body = try res.reader(&.{}).readAlloc(self.allocator, res.head.content_length.?); + defer self.allocator.free(body); + + const parsed = try std.json.parseFromSlice( + []API.Tree, + self.allocator, + body, + .{ .ignore_unknown_fields = true }, + ); + defer parsed.deinit(); + + var tree_root: std.ArrayList(TreeNode) = .{}; + for (parsed.value) |item| { + try insertTreeNode(self.allocator, &tree_root, item); + } + + return tree_root; + } + + fn findDirChildren(tree_items: []const TreeNode, dir_path: []const u8) ?[]const TreeNode { + if (dir_path.len == 0) { + return tree_items; + } + + var parts = std.mem.tokenizeScalar(u8, dir_path, '/'); + var current = tree_items; + + while (parts.next()) |part| { + var found: ?*const TreeNode = null; + for (current) |*node| { + if (std.mem.eql(u8, node.name, part)) { + found = node; + break; + } } - if (response.head.status == .unauthorized) { - return std.Io.File.Reader.Error.AccessDenied; + if (found) |node| { + if (node.children) |children| { + current = children.items; + } else { + return null; + } } else { - return std.Io.File.Reader.Error.Unexpected; + return null; } } - handle.request = request; - handle.response = response; - handle.body_reader = handle.response.?.reader(&.{}); + return current; } - fn makeVTable(base_io: std.Io) std.Io.VTable { - var vtable = base_io.vtable.*; - vtable.dirMake = dirMake; - vtable.dirMakePath = dirMakePath; - vtable.dirMakeOpenPath = dirMakeOpenPath; - vtable.dirStat = dirStat; - vtable.dirStatPath = dirStatPath; - vtable.dirAccess = dirAccess; - vtable.dirCreateFile = dirCreateFile; - vtable.dirOpenFile = dirOpenFile; - vtable.dirOpenDir = dirOpenDir; - vtable.dirClose = dirClose; - vtable.fileStat = fileStat; - vtable.fileClose = fileClose; - vtable.fileWriteStreaming = fileWriteStreaming; - vtable.fileWritePositional = fileWritePositional; - vtable.fileReadStreaming = fileReadStreaming; - vtable.fileReadPositional = fileReadPositional; - vtable.fileSeekBy = fileSeekBy; - vtable.fileSeekTo = fileSeekTo; - vtable.openSelfExe = openSelfExe; + fn findNode(tree_items: []const TreeNode, path: []const u8) ?*const TreeNode { + if (path.len == 0) return null; - return vtable; - } + var parts = std.mem.tokenizeScalar(u8, path, '/'); + var current = tree_items; - fn dirMake( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - mode: std.Io.Dir.Mode, - ) std.Io.Dir.MakeError!void { - _ = userdata; - _ = dir; - _ = sub_path; - _ = mode; - log.err("VFS is read-only, dirMake not supported", .{}); - return std.Io.Dir.MakeError.ReadOnlyFileSystem; - } + while (parts.next()) |part| { + const is_last = parts.peek() == null; - fn dirMakePath( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - mode: std.Io.Dir.Mode, - ) std.Io.Dir.MakeError!void { - _ = userdata; - _ = dir; - _ = sub_path; - _ = mode; - log.err("VFS is read-only, dirMakePath not supported", .{}); - return std.Io.Dir.MakeError.ReadOnlyFileSystem; - } + var found: ?*const TreeNode = null; + for (current) |*node| { + if (std.mem.eql(u8, node.name, part)) { + found = node; + break; + } + } - fn dirMakeOpenPath( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - options: std.Io.Dir.OpenOptions, - ) std.Io.Dir.MakeOpenPathError!std.Io.Dir { - _ = userdata; - _ = dir; - _ = sub_path; - _ = options; - log.err("VFS is read-only, dirMakeOpenPath not supported", .{}); - return std.Io.Dir.MakeOpenPathError.Unexpected; - } - - fn dirStat( - userdata: ?*anyopaque, - dir: std.Io.Dir, - ) std.Io.Dir.StatError!std.Io.Dir.Stat { - _ = userdata; - _ = dir; - log.err("VFS does not support dirStat", .{}); - return std.Io.Dir.StatError.AccessDenied; - } - - fn dirStatPath( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - options: std.Io.Dir.StatPathOptions, - ) std.Io.Dir.StatPathError!std.Io.File.Stat { - _ = userdata; - _ = dir; - _ = sub_path; - _ = options; - @panic("TODO: implement VFS dirStatPath"); - } - - fn dirAccess( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - options: std.Io.Dir.AccessOptions, - ) std.Io.Dir.AccessError!void { - _ = userdata; - _ = dir; - _ = sub_path; - - if (options.write) { - return std.Io.Dir.AccessError.ReadOnlyFileSystem; + if (found) |node| { + if (is_last) return node; + if (node.children) |children| { + current = children.items; + } else { + return null; + } + } else { + return null; + } } - log.err("VFS does not support dirAccess", .{}); - return std.Io.Dir.AccessError.AccessDenied; + return null; } - fn dirCreateFile( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - flags: std.Io.File.CreateFlags, - ) std.Io.File.OpenError!std.Io.File { - _ = userdata; - _ = dir; - _ = sub_path; - _ = flags; - log.err("VFS is read-only, dirCreateFile not supported", .{}); - return std.Io.File.OpenError.Unexpected; + fn getSizeFromTree(tree: *std.ArrayList(TreeNode), path: []const u8) ?u64 { + if (path.len == 0) { + var total: u64 = 0; + for (tree.items) |*node| { + if (node.kind == .file) total += node.size; + } + return total; + } + + const node = findNode(tree.items, path) orelse return null; + return node.size; } - fn dirOpenFile( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - flags: std.Io.File.OpenFlags, - ) std.Io.File.OpenError!std.Io.File { - _ = flags; - const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); - return try self.openFile(dir, sub_path); + fn insertTreeNode(allocator: std.mem.Allocator, root: *std.ArrayList(TreeNode), item: API.Tree) !void { + var parts = std.mem.tokenizeScalar(u8, item.path, '/'); + var current_list = root; + + var parents: std.ArrayList(*TreeNode) = .{}; + defer parents.deinit(allocator); + + while (true) { + const part = parts.next() orelse break; + const is_last = parts.peek() == null; + + var found: ?*TreeNode = null; + for (current_list.items) |*node| { + if (std.mem.eql(u8, node.name, part)) { + found = node; + break; + } + } + + if (found) |f| { + if (!is_last) { + try parents.append(allocator, f); + if (f.children == null) { + f.children = .{}; + } + current_list = &f.children.?; + } else if (std.mem.eql(u8, item.type, "file")) { + f.size = item.size; + } + } else { + const is_file = is_last and std.mem.eql(u8, item.type, "file"); + const new_node: TreeNode = .{ + .name = try allocator.dupe(u8, part), + .kind = if (is_file) .file else .directory, + .size = if (is_last) item.size else 0, + .children = if (is_file) null else .{}, + }; + try current_list.append(allocator, new_node); + + if (!is_last) { + const new_node_ptr = ¤t_list.items[current_list.items.len - 1]; + try parents.append(allocator, new_node_ptr); + current_list = &new_node_ptr.children.?; + } + } + } + + for (parents.items) |parent| { + parent.size += item.size; + } } - fn dirOpenDir( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - options: std.Io.Dir.OpenOptions, - ) std.Io.Dir.OpenError!std.Io.Dir { - _ = options; - const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); - return try self.openDir(dir, sub_path); + fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + + var path_buffer: [8 * 1024]u8 = undefined; + const full_path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.OpenError.SystemResources; + + const repo = Repo.parse(full_path) catch return std.Io.Dir.OpenError.BadPathName; + const tree = self.getOrFetchTree(repo) catch return std.Io.Dir.OpenError.Unexpected; + const size = getSizeFromTree(tree, repo.path) orelse return std.Io.Dir.OpenError.FileNotFound; + + const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .directory, full_path, size) catch return std.Io.Dir.OpenError.Unexpected; + + return .{ .handle = @intCast(idx) }; } - fn dirClose(userdata: ?*anyopaque, dir: std.Io.Dir) void { - const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); - self.closeDir(dir); - } + fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getDirHandle(dir); - fn fileStat( - userdata: ?*anyopaque, - file: std.Io.File, - ) std.Io.File.StatError!std.Io.File.Stat { - const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); - const handle = self.fileHandle(file) orelse { - log.err("Invalid file handle in VFS stat", .{}); - return std.Io.File.StatError.Unexpected; + return .{ + .inode = @intCast(dir.handle), + .nlink = 0, + .size = handle.size, + .permissions = .fromMode(0o444), + .kind = .directory, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, }; + } + + fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + + var path_buffer: [8 * 1024]u8 = undefined; + const full_path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.StatFileError.SystemResources; + + const repo = Repo.parse(full_path) catch return std.Io.Dir.StatFileError.Unexpected; + const tree = self.getOrFetchTree(repo) catch return std.Io.Dir.StatFileError.Unexpected; + const node = findNode(tree.items, repo.path) orelse return std.Io.Dir.StatFileError.IsDir; + + if (node.kind == .directory) return std.Io.Dir.StatFileError.IsDir; + const size = getSizeFromTree(tree, repo.path) orelse return std.Io.Dir.StatFileError.FileNotFound; + + return .{ + .inode = @intCast(0), + .nlink = 0, + .size = size, + .permissions = .fromMode(0o444), + .kind = .file, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + }; + } + + fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + + var path_buffer: [8 * 1024]u8 = undefined; + const full_path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.AccessError.SystemResources; + + const repo = Repo.parse(full_path) catch return std.Io.Dir.AccessError.FileNotFound; + const tree = self.getOrFetchTree(repo) catch return std.Io.Dir.AccessError.Unexpected; + + if (repo.path.len == 0) return; + + const node = findNode(tree.items, repo.path); + if (node == null) return std.Io.Dir.AccessError.FileNotFound; + } + + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + + var path_buffer: [8 * 1024]u8 = undefined; + const full_path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; + + const repo = Repo.parse(full_path) catch return std.Io.File.OpenError.BadPathName; + const tree = self.getOrFetchTree(repo) catch return std.Io.File.OpenError.Unexpected; + const size = getSizeFromTree(tree, repo.path) orelse return std.Io.File.OpenError.FileNotFound; + + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .file, full_path, size) catch return std.Io.File.OpenError.Unexpected; + + return .{ .handle = @intCast(idx) }; + } + + fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + for (dirs) |d| { + self.closeHandle(@intCast(d.handle)) catch {}; + } + } + + fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + + if (reader.state == .finished) return 0; + + if (reader.state == .reset) { + _ = self.dir_read_states.remove(reader); + + const handle = self.getDirHandle(reader.dir); + const repo = Repo.parse(handle.uri) catch return std.Io.Dir.Reader.Error.Unexpected; + const tree = self.getOrFetchTree(repo) catch return std.Io.Dir.Reader.Error.Unexpected; + + const children = findDirChildren(tree.items, repo.path) orelse &.{}; + + self.dir_read_states.put(self.allocator, reader, .{ + .children = children, + .index = 0, + }) catch return std.Io.Dir.Reader.Error.Unexpected; + + reader.state = if (children.len > 0) .reading else .finished; + if (children.len == 0) return 0; + } + + const state = self.dir_read_states.getPtr(reader) orelse return std.Io.Dir.Reader.Error.Unexpected; + + var count: usize = 0; + while (count < entries.len and state.index < state.children.len) { + const node = &state.children[state.index]; + entries[count] = .{ + .name = node.name, + .kind = node.kind, + .inode = state.index, + }; + count += 1; + state.index += 1; + } + + if (state.index >= state.children.len) { + reader.state = .finished; + _ = self.dir_read_states.remove(reader); + } + + return count; + } + + fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getDirHandle(dir); + const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.Dir.RealPathError.SystemResources; + return path.len; + } + + fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const real_path = self.resolvePath(dir, path_name, out_buffer) catch return std.Io.Dir.RealPathFileError.NameTooLong; + return real_path.len; + } + + fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); return .{ .inode = @intCast(file.handle), - .kind = .file, + .nlink = 0, .size = handle.size, - .mode = 0o444, - .atime = .{ .nanoseconds = 0 }, - .mtime = .{ .nanoseconds = 0 }, - .ctime = .{ .nanoseconds = 0 }, + .permissions = .fromMode(0o444), + .kind = .file, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, }; } - fn fileClose(userdata: ?*anyopaque, file: std.Io.File) void { - const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); - self.closeFile(file); + fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.getFileHandle(file).size; } - fn fileWriteStreaming( - userdata: ?*anyopaque, - file: std.Io.File, - buffer: [][]const u8, - ) std.Io.File.WriteStreamingError!usize { - _ = userdata; - _ = file; - _ = buffer; - log.err("VFS is read-only, fileWriteStreaming not supported", .{}); - return std.Io.File.WriteStreamingError.Unexpected; + fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + for (files) |file| { + self.closeHandle(@intCast(file.handle)) catch unreachable; + } } - fn fileWritePositional( - userdata: ?*anyopaque, - file: std.Io.File, - buffer: [][]const u8, - offset: u64, - ) std.Io.File.WritePositionalError!usize { - _ = userdata; - _ = file; - _ = buffer; - _ = offset; - log.err("VFS is read-only, fileWritePositional not supported", .{}); - return std.Io.File.WritePositionalError.Unexpected; + fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + const total = self.performRead(handle, data, handle.pos) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); + return std.Io.File.Reader.Error.Unexpected; + }; + handle.pos += @intCast(total); + return total; } - fn fileReadStreaming( - userdata: ?*anyopaque, - file: std.Io.File, - data: [][]u8, - ) std.Io.File.Reader.Error!usize { - const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); - return try self.performRead(file, data); + fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return self.performRead(handle, data, offset) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); + return std.Io.File.Reader.Error.Unexpected; + }; } - fn fileReadPositional( - userdata: ?*anyopaque, - file: std.Io.File, - data: [][]u8, - offset: u64, - ) std.Io.File.ReadPositionalError!usize { - const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); - return try self.performPositionalRead(file, data, offset); + fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + + handle.pos = if (relative_offset >= 0) + handle.pos + @as(u64, @intCast(relative_offset)) + else + handle.pos - @as(u64, @intCast(-relative_offset)); } - fn fileSeekBy( - userdata: ?*anyopaque, - file: std.Io.File, - relative_offset: i64, - ) std.Io.File.SeekError!void { - const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); - try self.setRelativeOffset(file, relative_offset); + fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + handle.pos = absolute_offset; } - fn fileSeekTo( - userdata: ?*anyopaque, - file: std.Io.File, - absolute_offset: u64, - ) std.Io.File.SeekError!void { - const self: *HF = @ptrCast(@alignCast(userdata orelse unreachable)); - try self.setAbsolutePos(file, absolute_offset); + fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.File.RealPathError.SystemResources; + return path.len; } - fn openSelfExe( - userdata: ?*anyopaque, - flags: std.Io.File.OpenFlags, - ) std.Io.File.OpenSelfExeError!std.Io.File { - _ = userdata; - _ = flags; - log.err("VFS does not support openSelfExe", .{}); - return std.Io.File.OpenSelfExeError.NotSupported; + fn performRead(self: *HF, handle: *Handle, data: []const []u8, offset: u64) !usize { + if (offset >= handle.size) return 0; + + var range_buf: [64]u8 = undefined; + const range_header = blk: { + var total_bytes: u64 = 0; + for (data) |buf| { + total_bytes += @as(u64, buf.len); + } + const remaining = handle.size - offset; + const take = @min(remaining, total_bytes); + const end = offset + take - 1; + break :blk std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ offset, end }) catch unreachable; + }; + + const repo: Repo = try .parse(handle.uri); + var url_buffer: [8 * 1024]u8 = undefined; + const url: []const u8 = try std.fmt.bufPrint(&url_buffer, API.LFS_FILE_URL_TEMPLATE, repo); + + const uri: std.Uri = try .parse(url); + var req = try self.client.request(.GET, uri, .{ + .headers = .{ .authorization = self.authorization }, + .extra_headers = &.{.{ .name = "Range", .value = range_header }}, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var redirect_buffer: [8 * 1024]u8 = undefined; + var res = try req.receiveHead(&redirect_buffer); + + if (res.head.status != .partial_content) { + log.err("Failed to perform read for {s}", .{url}); + log.err("{s}", .{res.head.bytes}); + return error.RequestFailed; + } + + const content_range = blk: { + var it = res.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { + break :blk parseContentRange(header.value); + } + } + break :blk null; + }; + + const reader = res.reader(&.{}); + + if (content_range) |cr| { + if (cr.start < offset) { + try reader.discardAll(offset - cr.start); + } + } + + return try reader.readSliceShort(data[0]); + } + + const ContentRange = struct { + start: u64, + end: u64, + total: u64, + }; + + fn parseContentRange(value: []const u8) ?ContentRange { + const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; + const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; + const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; + + return .{ + .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, + .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, + .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, + }; } }; diff --git a/zml/io/vfs/hf2.zig b/zml/io/vfs/hf2.zig deleted file mode 100644 index 1092ab98..00000000 --- a/zml/io/vfs/hf2.zig +++ /dev/null @@ -1,316 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -const VFSBase = @import("base.zig").VFSBase; - -const log = std.log.scoped(.@"zml/io/vfs/hf"); - -pub const API = struct { - const TREE_URL_TEMPLATE = "https://huggingface.co/api/models/{[repo]s}/{[model]s}/tree/{[rev]s}/{[path]s}"; - const TREESIZE_URL_TEMPLATE = "https://huggingface.co/api/models/{[repo]s}/{[model]s}/treesize/{[rev]s}/{[path]s}"; - const RAW_FILE_URL_REMPLATE = "https://huggingface.co/{model}/raw/{rev}/{path}"; - const LFS_FILE_URL_TEMPLATE = "https://huggingface.co/{[repo]s}/{[model]s}/resolve/{[rev]s}/{[path]s}"; - - pub const Entry = struct { - pub const Type = enum { - file, - directory, - }; - - pub const LFS = struct { - oid: []const u8, - size: u64, - pointerSize: u64, - }; - - type: Type, - oid: []const u8, - size: u64, - path: []const u8, - lfs: ?LFS = null, - }; -}; - -pub const HF = struct { - pub const Repo = struct { - repo: []const u8, - model: []const u8, - rev: []const u8, - path: []const u8, - - pub fn parse(uri: []const u8) !Repo { - var parts = std.mem.splitScalar(u8, uri, '/'); - var repo: Repo = .{ - .repo = parts.first(), - .model = parts.next() orelse return error.InvalidURI, - .rev = "main", - .path = std.mem.trimEnd(u8, parts.rest(), "/"), - }; - if (std.mem.findScalar(u8, repo.model, '@')) |at_index| { - repo.rev = repo.model[at_index + 1 ..]; - repo.model = repo.model[0..at_index]; - } - return repo; - } - }; - - pub const Handle = struct { - pub const Type = enum { - file, - directory, - }; - - type: Type, - url: []const u8, - uri: std.Uri, - pos: u64, - size: u64, - - pub fn init(allocator: std.mem.Allocator, type_: Type, sub_path: []const u8, size: u64) Handle { - const repo = Repo.parse(sub_path) catch unreachable; - const lfs_url = std.fmt.allocPrint(allocator, API.LFS_FILE_URL_TEMPLATE, repo) catch unreachable; - errdefer allocator.free(lfs_url); - - return .{ - .type = type_, - .url = lfs_url, - .uri = std.Uri.parse(lfs_url) catch unreachable, - .pos = 0, - .size = size, - }; - } - - pub fn deinit(self: *Handle, allocator: std.mem.Allocator) void { - allocator.free(self.url); - } - }; - - base: VFSBase, - authorization: std.http.Client.Request.Headers.Value = .default, - allocator: std.mem.Allocator, - client: *std.http.Client, - handles: std.ArrayList(Handle) = .{}, - closed_handles: std.ArrayList(u32) = .{}, - - pub fn init(allocator: std.mem.Allocator, io_: std.Io, http_client_: *std.http.Client) !HF { - return .{ - .allocator = allocator, - .base = .init(io_), - .client = http_client_, - .authorization = .{ - .override = "Bearer xxxx", - }, - }; - } - - pub fn deinit(self: *HF) void { - _ = self; // autofix - } - - pub fn io(self: *HF) std.Io { - return .{ - .userdata = &self.base, - .vtable = &comptime VFSBase.vtable(.{ - .dirOpenFile = dirOpenFile, - .fileReadPositional = fileReadPositional, - .fileReadStreaming = fileReadStreaming, - .fileClose = fileClose, - }), - }; - } - - pub fn openHandle(self: *HF) !struct { u32, *Handle } { - if (self.closed_handles.pop()) |idx| { - return .{ idx, &self.handles.items[idx] }; - } - return .{ @intCast(self.handles.items.len), try self.handles.addOne(self.allocator) }; - } - - pub fn closeHandle(self: *HF, idx: u32) !void { - try self.closed_handles.append(self.allocator, idx); - } - - fn getFileHandle(self: *HF, file: std.Io.File) *Handle { - return &self.handles.items[@intCast(file.handle)]; - } - - fn parseContentRange(value: []const u8) struct { u64, u64, u64 } { - const space = std.mem.indexOfScalar(u8, value, ' ').?; - const range = std.mem.indexOfScalar(u8, value, '-').?; - const slash = std.mem.indexOfScalar(u8, value, '/').?; - - return .{ - std.fmt.parseInt(u64, value[space + 1 .. range], 10) catch 0, - std.fmt.parseInt(u64, value[range + 1 .. slash], 10) catch 0, - std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch 0, - }; - } - - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { - _ = dir; // autofix - _ = flags; // autofix - // _ = flags; // autofix - const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); - - var url_buffer: [1024]u8 = undefined; - var redirect_buffer: [8 * 1024]u8 = undefined; - var aux_buffer: []u8 = &redirect_buffer; - - const repo = Repo.parse(sub_path) catch return std.Io.File.OpenError.BadPathName; - const url: []const u8 = std.fmt.bufPrint(&url_buffer, API.LFS_FILE_URL_TEMPLATE, repo) catch return std.Io.File.OpenError.BadPathName; - - log.info("Opening url: {s}", .{url}); - - var uri = std.Uri.parse(url) catch return std.Io.File.OpenError.BadPathName; - while (true) { - var req = self.client.request(.HEAD, uri, .{ - .headers = .{ .authorization = self.authorization }, - .redirect_behavior = .not_allowed, - }) catch return std.Io.File.Reader.Error.Unexpected; - defer req.deinit(); - - req.sendBodiless() catch { - log.err("Failed to send HTTP request for HF model size API URL", .{}); - return std.Io.File.Reader.Error.Unexpected; - }; - - var resp = req.receiveHead(&.{}) catch { - log.err("Failed to receive HTTP response for HF model size API URL", .{}); - return std.Io.File.Reader.Error.Unexpected; - }; - - if (resp.head.status.class() == .redirect) { - const location = resp.head.location.?; - @memcpy(aux_buffer[0..location.len], location); - uri = uri.resolveInPlace(location.len, &aux_buffer) catch unreachable; - log.info("Redirect {f}", .{uri}); - continue; - } - - const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; - handle.* = .init(self.allocator, .file, sub_path, resp.head.content_length.?); - - return .{ .handle = @intCast(idx) }; - } - } - - pub fn performRead(self: *HF, handle: *Handle, data: []const []u8, offset: u64) std.Io.File.Reader.Error!usize { - if (offset >= handle.size) { - log.info("EOF", .{}); - return 0; - } - - var range_buf: [64]u8 = undefined; - const range_header = blk: { - var total_bytes: u64 = 0; - for (data) |buf| { - total_bytes += buf.len; - } - break :blk std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ offset, @min(handle.size - offset, offset + total_bytes) }) catch unreachable; - }; - - var req = self.client.request(.GET, handle.uri, .{ - .headers = .{ .authorization = self.authorization }, - .extra_headers = &.{ - .{ .name = "Range", .value = range_header }, - }, - }) catch return std.Io.File.Reader.Error.Unexpected; - defer req.deinit(); - - req.sendBodiless() catch { - log.err("Failed to send HTTP request for HF model size API URL", .{}); - return std.Io.File.Reader.Error.Unexpected; - }; - - var redirect_buffer: [8 * 1024]u8 = undefined; - var resp = req.receiveHead(&redirect_buffer) catch { - log.err("Failed to receive HTTP response for HF model size API URL", .{}); - return std.Io.File.Reader.Error.Unexpected; - }; - - const range = blk: { - var it = resp.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { - break :blk parseContentRange(header.value); - } - } else break :blk .{ 0, 0, 0 }; - }; - - log.info(">>>>> {any}", .{range}); - - const reader = resp.reader(&.{}); - - if (range.@"0" < offset) { - reader.discardAll(offset - range.@"0") catch return std.Io.File.Reader.Error.Unexpected; - } - - return reader.readSliceShort(data[0]) catch return std.Io.File.Reader.Error.Unexpected; - } - - pub fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { - const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - return try self.performRead(handle, data, offset); - } - - fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { - const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - const total = try self.performRead(handle, data, handle.pos); - handle.pos += @intCast(total); - return total; - } - - fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { - const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); - for (files) |file| { - self.closeHandle(@intCast(file.handle)) catch unreachable; - } - } -}; - -pub fn main() !void { - const allocator = std.heap.smp_allocator; - - var threaded: std.Io.Threaded = .init(allocator, .{}); - defer threaded.deinit(); - - var client: std.http.Client = .{ - .allocator = allocator, - .io = threaded.io(), - }; - - try client.initDefaultProxies(allocator); - defer client.deinit(); - - var hf: HF = try .init(allocator, threaded.io(), &client); - defer hf.deinit(); - - const io = hf.io(); - - const file = std.Io.Dir.cwd().openFile(io, "Qwen/Qwen3-8B/model-00001-of-00005.safetensors", .{}) catch |err| { - log.err("Failed to open HF file: {any}", .{err}); - unreachable; - }; - - log.info(">>>>>>> {any}", .{file}); - - var reader = file.reader(io, &.{}); - reader.mode = reader.mode.toSimple(); - defer log.info("DONE", .{}); - - // const data = try reader.interface.readAlloc(allocator, 1024); - // defer allocator.free(data); - // log.info("READ {s}", .{data}); - - const buffer = try allocator.alloc(u8, 256 * 1024 * 1024); - var stdout_writer = std.Io.File.stdout().writer(io, buffer); - defer stdout_writer.interface.flush() catch {}; - - _ = reader.interface.streamRemaining(&stdout_writer.interface) catch |err| { - log.err("Failed to stream HF file: {any}", .{err}); - unreachable; - }; -} diff --git a/zml/io/vfs/http.zig b/zml/io/vfs/http.zig index e8168c9c..fe10767b 100644 --- a/zml/io/vfs/http.zig +++ b/zml/io/vfs/http.zig @@ -1,841 +1,431 @@ const std = @import("std"); +const stdx = @import("stdx"); + +const VFSBase = @import("base.zig").VFSBase; + const log = std.log.scoped(.@"zml/io/vfs/http"); pub const HTTP = struct { - const HandleId = i32; - - pub const InitHandleError = error{ - PathTooLong, - PathUnresolvable, - FailedToFetchRemoteResource, - MissingRessource, - } || std.mem.Allocator.Error; - - pub const FileHandle = struct { - allocator: std.mem.Allocator, + const Handle = struct { + pub const Type = enum { + file, + directory, + }; + type: Type, uri: []const u8, pos: u64, size: u64, - redirect_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined, - request: ?*std.http.Client.Request = null, - response: ?std.http.Client.Response = null, - body_reader: ?*std.Io.Reader = null, - total_requests: u64 = 0, - total_latency_ns: u64 = 0, - min_latency_ns: u64 = std.math.maxInt(u64), - max_latency_ns: u64 = 0, + pub fn init(allocator: std.mem.Allocator, type_: Type, path: []const u8, size: u64) !Handle { + const uri = try allocator.dupe(u8, path); + errdefer allocator.free(uri); - pub fn deinit(self: *FileHandle) void { - self.releaseRequest(); - self.allocator.free(self.uri); - self.allocator.destroy(self); + return .{ + .type = type_, + .uri = uri, + .pos = 0, + .size = size, + }; } - pub fn releaseRequest(self: *FileHandle) void { - if (self.request) |req| { - req.deinit(); - self.allocator.destroy(req); - self.request = null; - self.response = null; - self.body_reader = null; - } - } - - pub fn recordLatency(self: *FileHandle, latency_ns: u64) void { - self.total_requests += 1; - self.total_latency_ns += latency_ns; - self.min_latency_ns = @min(self.min_latency_ns, latency_ns); - self.max_latency_ns = @max(self.max_latency_ns, latency_ns); - } - - pub fn avgLatencyMs(self: *FileHandle) u64 { - if (self.total_requests == 0) return 0; - return (self.total_latency_ns / self.total_requests) / std.time.ns_per_ms; + pub fn deinit(self: *Handle, allocator: std.mem.Allocator) void { + allocator.free(self.uri); } }; - pub const DirHandle = struct { - allocator: std.mem.Allocator, - uri: []const u8, - - pub fn deinit(self: *DirHandle) void { - self.allocator.free(self.uri); - self.allocator.destroy(self); - } + const Protocol = union(enum) { + http, + https, }; - pub const Config = struct { - max_fetch_attempts: u32 = 10, - request_range_min: u64 = 8 * 1024, - request_range_max: u64 = 128 * 1024 * 1024, + const Config = struct { + http_client: *std.http.Client, + protocol: Protocol, }; allocator: std.mem.Allocator, - mutex: std.Io.Mutex, - - file_handles: std.AutoHashMapUnmanaged(HandleId, *FileHandle), - dir_handles: std.AutoHashMapUnmanaged(HandleId, *DirHandle), - next_file_handle_id: HandleId, - next_dir_handle_id: HandleId, - + mutex: std.Io.Mutex = .init, client: *std.http.Client, - config: Config, + protocol: Protocol, + handles: stdx.SegmentedList(Handle, 0), + closed_handles: std.ArrayList(u32) = .{}, + base: VFSBase, - vtable: std.Io.VTable, + pub fn init(allocator: std.mem.Allocator, base_io: std.Io, config: Config) !HTTP { + var handles: stdx.SegmentedList(Handle, 0) = .{}; + // cwd + const handle = try handles.addOne(allocator); + handle.* = .{ + .type = .directory, + .uri = "", + .pos = 0, + .size = 0, + }; - pub fn init(allocator: std.mem.Allocator, base_io: std.Io, http_client: *std.http.Client, config: Config) std.mem.Allocator.Error!HTTP { return .{ .allocator = allocator, - .mutex = .init, - .file_handles = .{}, - .dir_handles = .{}, - .next_file_handle_id = 0, - .next_dir_handle_id = 0, - .client = http_client, - .config = config, - .vtable = makeVTable(base_io), + .base = .init(base_io), + .handles = handles, + .client = config.http_client, + .protocol = config.protocol, }; } pub fn deinit(self: *HTTP) void { - var file_it = self.file_handles.valueIterator(); - while (file_it.next()) |handle_ptr| { - handle_ptr.*.deinit(); - } - self.file_handles.deinit(self.allocator); + var idx: usize = 0; + while (idx < self.handles.len) : (idx += 1) { + const is_closed = for (self.closed_handles.items) |closed_idx| { + if (closed_idx == idx) break true; + } else false; - var dir_it = self.dir_handles.valueIterator(); - while (dir_it.next()) |handle_ptr| { - handle_ptr.*.deinit(); + if (!is_closed) { + self.handles.at(idx).deinit(self.allocator); + } } - self.dir_handles.deinit(self.allocator); + self.handles.deinit(self.allocator); + self.closed_handles.deinit(self.allocator); } pub fn io(self: *HTTP) std.Io { return .{ - .vtable = &self.vtable, - .userdata = self, + .userdata = &self.base, + .vtable = &comptime VFSBase.vtable(.{ + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRead = dirRead, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileReadStreaming = fileReadStreaming, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileRealPath = fileRealPath, + }), }; } - const URIKind = union(enum) { - dir, - file, - }; + fn openHandle(self: *HTTP) !struct { u32, *Handle } { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); - fn resolveUri(self: *HTTP, parent_path: []const u8, sub_path: []const u8, kind: URIKind) InitHandleError![]const u8 { - if (std.mem.startsWith(u8, sub_path, "http://") or std.mem.startsWith(u8, sub_path, "https://")) { - if (kind == .dir and !std.mem.endsWith(u8, sub_path, "/")) { - return std.fmt.allocPrint(self.allocator, "{s}/", .{sub_path}); - } else { - return try self.allocator.dupe(u8, sub_path); - } - } - - const base_uri = std.Uri.parse(parent_path) catch return InitHandleError.PathUnresolvable; - var aux_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - - if (sub_path.len > aux_buf.len) return InitHandleError.PathTooLong; - - @memcpy(aux_buf[0..sub_path.len], sub_path); - var aux_slice: []u8 = aux_buf[0..]; - const resolved = std.Uri.resolveInPlace(base_uri, sub_path.len, &aux_slice) catch return InitHandleError.PathUnresolvable; - if (kind == .dir and !std.mem.endsWith(u8, resolved.path.percent_encoded, "/")) { - return std.fmt.allocPrint(self.allocator, "{f}/", .{resolved.fmt(.{ - .scheme = true, - .authority = true, - .path = true, - .query = false, - .fragment = false, - })}); - } else { - return std.fmt.allocPrint(self.allocator, "{f}", .{resolved.fmt(.{ - .scheme = true, - .authority = true, - .path = true, - .query = false, - .fragment = false, - })}); + if (self.closed_handles.pop()) |idx| { + return .{ idx, self.handles.at(idx) }; } + return .{ @intCast(self.handles.len), try self.handles.addOne(self.allocator) }; } - fn initFileHandle( - self: *HTTP, - dir: DirHandle, - sub_path: []const u8, - ) InitHandleError!*FileHandle { - const handle = try self.allocator.create(FileHandle); - errdefer self.allocator.destroy(handle); + fn closeHandle(self: *HTTP, idx: u32) !void { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); - const uri = try self.resolveUri(dir.uri, sub_path, .file); - errdefer self.allocator.free(uri); - - handle.* = .{ - .allocator = self.allocator, - .uri = uri, - .pos = 0, - .size = undefined, - }; - - const stat = self.fetchFileStat(handle) catch { - log.err("Failed to fetch file stat for URI {s}", .{uri}); - return InitHandleError.FailedToFetchRemoteResource; - }; - - handle.size = stat.size; - - return handle; + self.handles.at(idx).deinit(self.allocator); + try self.closed_handles.append(self.allocator, idx); } - fn registerFileHandle(self: *HTTP, handle: *FileHandle) std.mem.Allocator.Error!HandleId { - self.mutex.lockUncancelable(self.io()); - defer self.mutex.unlock(self.io()); - - const handle_id = self.next_file_handle_id; - self.next_file_handle_id += 1; - - try self.file_handles.put(self.allocator, handle_id, handle); - - return handle_id; + fn getFileHandle(self: *HTTP, file: std.Io.File) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + return self.handles.at(@intCast(file.handle)); } - fn fileHandle(self: *HTTP, file: std.Io.File) ?*FileHandle { - self.mutex.lockUncancelable(self.io()); - defer self.mutex.unlock(self.io()); - - return self.file_handles.get(file.handle); + fn getDirHandle(self: *HTTP, dir: std.Io.Dir) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + return self.handles.at(@intCast(dir.handle)); } - fn initDirHandle( - self: *HTTP, - parent_path: []const u8, - sub_path: []const u8, - ) InitHandleError!*DirHandle { - const dir_handle = try self.allocator.create(DirHandle); - errdefer self.allocator.destroy(dir_handle); - - const uri = try self.resolveUri(parent_path, sub_path, .dir); - errdefer self.allocator.free(uri); - - dir_handle.* = .{ - .allocator = self.allocator, - .uri = uri, - }; - - return dir_handle; - } - - fn registerDirHandle(self: *HTTP, handle: *DirHandle) std.mem.Allocator.Error!HandleId { - self.mutex.lockUncancelable(self.io()); - defer self.mutex.unlock(self.io()); - - const handle_id = self.next_dir_handle_id; - self.next_dir_handle_id += 1; - - try self.dir_handles.put(self.allocator, handle_id, handle); - - return handle_id; - } - - fn dirHandle(self: *HTTP, dir: std.Io.Dir) ?DirHandle { - self.mutex.lockUncancelable(self.io()); - defer self.mutex.unlock(self.io()); - + fn resolvePath(self: *HTTP, dir: std.Io.Dir, sub_path: []const u8, out_buffer: []u8) ![]u8 { if (std.meta.eql(dir, std.Io.Dir.cwd())) { - return .{ - .allocator = self.allocator, - .uri = "", - }; + return try std.fmt.bufPrint(out_buffer, "{s}", .{sub_path}); } - if (self.dir_handles.get(dir.handle)) |handle_ptr| { - return handle_ptr.*; - } else { - return null; + const handle = self.getDirHandle(dir); + + const trimmed_uri = std.mem.trimEnd(u8, handle.uri, "/"); + const trimmed_sub_path = std.mem.trimStart(u8, sub_path, "/"); + + if (trimmed_uri.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_sub_path}); + if (trimmed_sub_path.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_uri}); + return try std.fmt.bufPrint(out_buffer, "{s}/{s}", .{ trimmed_uri, trimmed_sub_path }); + } + + fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.OpenError.SystemResources; + + const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .directory, path, 0) catch return std.Io.Dir.OpenError.Unexpected; + + return .{ .handle = @intCast(idx) }; + } + + fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getDirHandle(dir); + + return .{ + .inode = @intCast(dir.handle), + .nlink = 0, + .size = handle.size, + .permissions = .fromMode(0o444), + .kind = .directory, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + }; + } + + fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + const size = self.fetchSize(dir, sub_path) catch return std.Io.Dir.StatFileError.Unexpected; + + return .{ + .inode = @intCast(dir.handle), + .nlink = 0, + .size = size, + .permissions = .fromMode(0o444), + .kind = .file, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + }; + } + + fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} + + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + + const size = self.fetchSize(dir, sub_path) catch return std.Io.File.OpenError.Unexpected; + + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; + + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; + + return .{ .handle = @intCast(idx) }; + } + + fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + for (dirs) |dir| { + self.closeHandle(@intCast(dir.handle)) catch unreachable; } } - fn openFile( - self: *HTTP, - dir: std.Io.Dir, - sub_path: []const u8, - ) std.Io.File.OpenError!std.Io.File { - const dir_handle = self.dirHandle(dir) orelse { - log.warn("Directory handle not found for dirOpenFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); - return std.Io.File.OpenError.FileNotFound; - }; - - const handle = self.initFileHandle(dir_handle, sub_path) catch { - log.warn("Failed to init FileHandle in openFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); - return std.Io.File.OpenError.SystemResources; - }; - - const handle_id = self.registerFileHandle(handle) catch { - log.warn("Failed to register FileHandle in openFile with dir={any} sub_path={s}", .{ dir.handle, sub_path }); - return std.Io.File.OpenError.SystemResources; - }; - - return .{ .handle = @intCast(handle_id) }; + fn dirRead(_: ?*anyopaque, _: *std.Io.Dir.Reader, _: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { + log.err("dirRead unsupported", .{}); + return std.Io.Dir.Reader.Error.Unexpected; } - fn openDir( - self: *HTTP, - dir: std.Io.Dir, - sub_path: []const u8, - ) std.Io.Dir.OpenError!std.Io.Dir { - const dir_handle = self.dirHandle(dir) orelse { - log.warn("Directory handle not found for dirOpenFile", .{}); - return std.Io.Dir.OpenError.FileNotFound; - }; - - const handle = self.initDirHandle(dir_handle.uri, sub_path) catch { - log.warn("Failed to init DirHandle in openDir with dir={any} sub_path={s}", .{ dir.handle, sub_path }); - return std.Io.Dir.OpenError.SystemResources; - }; - - const handle_id = self.registerDirHandle(handle) catch { - log.warn("Failed to register DirHandle in openDir with dir={any} sub_path={s}", .{ dir.handle, sub_path }); - return std.Io.Dir.OpenError.SystemResources; - }; - - return .{ .handle = @intCast(handle_id) }; + fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getDirHandle(dir); + const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.Dir.RealPathError.SystemResources; + return path.len; } - fn closeDir(self: *HTTP, dir: std.Io.Dir) void { - self.mutex.lockUncancelable(self.io()); - defer self.mutex.unlock(self.io()); - - const handle_kv = self.dir_handles.fetchRemove(dir.handle) orelse { - log.warn("Attempted to close non-existent dir handle: {d}", .{dir.handle}); - return; - }; - - handle_kv.value.deinit(); + fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + const real_path = self.resolvePath(dir, path_name, out_buffer) catch return std.Io.Dir.RealPathFileError.NameTooLong; + return real_path.len; } - fn closeFile(self: *HTTP, file: std.Io.File) void { - self.mutex.lockUncancelable(self.io()); - defer self.mutex.unlock(self.io()); + fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle_kv = self.file_handles.fetchRemove(file.handle) orelse { - log.warn("Attempted to close non-existent file handle: {d}", .{file.handle}); - return; - }; - - handle_kv.value.deinit(); - } - - fn setAbsolutePos(self: *HTTP, file: std.Io.File, pos: u64) std.Io.File.SeekError!void { - const handle = self.fileHandle(file) orelse return std.Io.File.SeekError.Unexpected; - handle.releaseRequest(); - handle.pos = pos; - } - - fn setRelativeOffset(self: *HTTP, file: std.Io.File, offset: i64) std.Io.File.SeekError!void { - const handle = self.fileHandle(file) orelse return std.Io.File.SeekError.Unexpected; - handle.releaseRequest(); - - const new_pos = @as(i64, @intCast(handle.pos)) + offset; - handle.pos = @intCast(new_pos); - } - - fn performPositionalRead(self: *HTTP, file: std.Io.File, data: [][]u8, offset: u64) std.Io.File.ReadPositionalError!usize { - const handle = self.fileHandle(file) orelse return std.Io.File.ReadPositionalError.Unexpected; - - const saved_pos = handle.pos; - - if (handle.pos != offset) { - handle.releaseRequest(); - handle.pos = offset; - } - - const bytes_read = try self.performRead(file, data); - - handle.pos = saved_pos; - handle.releaseRequest(); - - return bytes_read; - } - - fn performRead(self: *HTTP, file: std.Io.File, data: [][]u8) std.Io.File.Reader.Error!usize { - const handle = self.fileHandle(file) orelse { - log.err("Invalid file handle in VFS read", .{}); - return std.Io.File.Reader.Error.Unexpected; - }; - - if (data.len == 0) return 0; - - var total_read: usize = 0; - - for (data) |buf| { - var offset: usize = 0; - - while (offset < buf.len) { - // Check EOF against known file size - if (handle.pos >= handle.size) { - return total_read; - } - - const start = handle.pos; - var desired_read_len: u64 = @max(@as(u64, @intCast(buf.len)), self.config.request_range_min); - if (desired_read_len > self.config.request_range_max) desired_read_len = self.config.request_range_max; - - const end = if (handle.size > 0 and start < handle.size) @min(start + desired_read_len - 1, handle.size - 1) else start + desired_read_len - 1; - - try self.requestRead(file, start, end); - const reader = handle.body_reader.?; - - const dest = buf[offset..]; - var iov = [_][]u8{dest}; - - const n = reader.readVec(&iov) catch |err| switch (err) { - error.EndOfStream => { - log.debug("Reached EndOfStream for handle={d} pos={d}, will re-request remaining", .{ file.handle, handle.pos }); - handle.releaseRequest(); - continue; - }, - else => { - return std.Io.File.Reader.Error.Unexpected; - }, - }; - - offset += n; - handle.pos += n; - total_read += n; - } - } - return total_read; - } - - fn requestRead(self: *HTTP, file: std.Io.File, start: u64, end: u64) std.Io.File.Reader.Error!void { - const handle = self.fileHandle(file) orelse return std.Io.File.Reader.Error.Unexpected; - - if (handle.request) |_| return; - - std.debug.assert(start <= end); - - var range_buf: [64]u8 = undefined; - const range_header = std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ start, end }) catch unreachable; - - var timer = std.time.Timer.start() catch { - log.err("Failed to start timer for request", .{}); - return std.Io.File.Reader.Error.SystemResources; - }; - - const request = self.allocator.create(std.http.Client.Request) catch { - log.err("Failed to allocate request", .{}); - return std.Io.File.Reader.Error.SystemResources; - }; - errdefer self.allocator.destroy(request); - - const uri = std.Uri.parse(handle.uri) catch |err| { - log.err("Failed to parse URI during read request: {}", .{err}); - return std.Io.File.Reader.Error.Unexpected; - }; - - request.* = self.client.request(.GET, uri, .{ - .headers = .{ .accept_encoding = .{ .override = "identity" } }, - .extra_headers = &.{.{ .name = "Range", .value = range_header }}, - }) catch |err| { - log.err("Failed to create GET request: {}", .{err}); - return std.Io.File.Reader.Error.SystemResources; - }; - errdefer request.deinit(); - - request.sendBodiless() catch |err| { - log.err("Failed to send GET request: {}", .{err}); - return std.Io.File.Reader.Error.SystemResources; - }; - - const response = request.receiveHead(&handle.redirect_buffer) catch |err| { - log.err("Failed to receive HTTP response head for handle={d} uri={s}: {}", .{ file.handle, handle.uri, err }); - return std.Io.File.Reader.Error.Unexpected; - }; - - const latency = timer.read(); - handle.recordLatency(latency); - - log.debug("GET call: handle={d} status={s} content_length={?} range={d}-{d} ({d}) latency={d}ms avg={d}ms min={d}ms max={d}ms for {s}", .{ - file.handle, - @tagName(response.head.status), - response.head.content_length, - start, - end, - end - start + 1, - latency / std.time.ns_per_ms, - handle.avgLatencyMs(), - handle.min_latency_ns / std.time.ns_per_ms, - handle.max_latency_ns / std.time.ns_per_ms, - handle.uri, - }); - - if (response.head.status != .partial_content and response.head.status != .ok) { - log.err("Unexpected HTTP status ({s}) for GET {s}", .{ @tagName(response.head.status), handle.uri }); - var it = response.head.iterateHeaders(); - while (it.next()) |hdr| { - log.err("Header: {s}: {s}", .{ hdr.name, hdr.value }); - } - - if (response.head.status == .unauthorized) { - return std.Io.File.Reader.Error.AccessDenied; - } else { - return std.Io.File.Reader.Error.Unexpected; - } - } - - handle.request = request; - handle.response = response; - handle.body_reader = handle.response.?.reader(&.{}); - } - - pub const FileStat = struct { - size: u64, - }; - - fn fetchFileStat(self: *HTTP, handle: *FileHandle) std.Io.File.StatError!FileStat { - var attempt: u32 = 0; - while (attempt < self.config.max_fetch_attempts) : (attempt += 1) { - const req = self.allocator.create(std.http.Client.Request) catch { - log.err("Failed to allocate HTTP HEAD request", .{}); - return std.Io.File.StatError.SystemResources; - }; - - req.* = self.client.request(.HEAD, std.Uri.parse(handle.uri) catch return std.Io.File.StatError.Unexpected, .{ - .headers = .{ .accept_encoding = .{ .override = "identity" } }, - }) catch { - self.allocator.destroy(req); - continue; - }; - - req.sendBodiless() catch { - req.deinit(); - self.allocator.destroy(req); - continue; - }; - - var response = req.receiveHead(&handle.redirect_buffer) catch { - req.deinit(); - self.allocator.destroy(req); - continue; - }; - - defer { - req.deinit(); - self.allocator.destroy(req); - } - - switch (response.head.status) { - .moved_permanently, .found, .see_other, .temporary_redirect, .permanent_redirect => { - if (response.head.location) |location| { - var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - - if (location.len > buf.len) return std.Io.File.StatError.Unexpected; - @memcpy(buf[0..location.len], location); - - var aux_buf: []u8 = buf[0..]; - - const uri = std.Uri.parse(handle.uri) catch |err| { - log.err("Failed to parse URL during redirect: {}", .{err}); - return std.Io.File.StatError.Unexpected; - }; - - const resolved = std.Uri.resolveInPlace(uri, location.len, &aux_buf) catch |err| { - log.err("Failed to resolve redirect URL: {}", .{err}); - return std.Io.File.StatError.Unexpected; - }; - const new_uri = std.fmt.allocPrint(self.allocator, "{f}", .{resolved.fmt(.{ - .scheme = true, - .authority = true, - .path = true, - .query = true, - .fragment = true, - })}) catch |err| { - log.err("Failed to allocate new URL during redirect: {}", .{err}); - return std.Io.File.StatError.SystemResources; - }; - - self.allocator.free(handle.uri); - handle.uri = new_uri; - - continue; - } - }, - .ok, .partial_content => { - const size = response.head.content_length orelse { - log.err("HTTP stat response missing Content-Length", .{}); - return std.Io.File.StatError.SystemResources; - }; - - return .{ - .size = size, - }; - }, - .internal_server_error, .bad_gateway, .service_unavailable, .gateway_timeout => { - std.Io.sleep(self.io(), .{ .nanoseconds = 50 * std.time.ns_per_ms }, .real) catch |err| { - log.err("Sleep failed during HTTP stat retry: {}", .{err}); - }; - continue; - }, - else => { - log.err("HTTP stat failed with status: {s}", .{@tagName(response.head.status)}); - return std.Io.File.StatError.Unexpected; - }, - } - } - - log.err("HTTP stat failed after {d} attempts", .{self.config.max_fetch_attempts}); - return std.Io.File.StatError.Unexpected; - } - - fn makeVTable(base_io: std.Io) std.Io.VTable { - var vtable = base_io.vtable.*; - vtable.dirMake = dirMake; - vtable.dirMakePath = dirMakePath; - vtable.dirMakeOpenPath = dirMakeOpenPath; - vtable.dirStat = dirStat; - vtable.dirStatPath = dirStatPath; - vtable.dirAccess = dirAccess; - vtable.dirCreateFile = dirCreateFile; - vtable.dirOpenFile = dirOpenFile; - vtable.dirOpenDir = dirOpenDir; - vtable.dirClose = dirClose; - vtable.fileStat = fileStat; - vtable.fileClose = fileClose; - vtable.fileWriteStreaming = fileWriteStreaming; - vtable.fileWritePositional = fileWritePositional; - vtable.fileReadStreaming = fileReadStreaming; - vtable.fileReadPositional = fileReadPositional; - vtable.fileSeekBy = fileSeekBy; - vtable.fileSeekTo = fileSeekTo; - vtable.openSelfExe = openSelfExe; - - return vtable; - } - - fn dirMake( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - mode: std.Io.Dir.Mode, - ) std.Io.Dir.MakeError!void { - _ = userdata; - _ = dir; - _ = sub_path; - _ = mode; - log.err("VFS is read-only, dirMake not supported", .{}); - return std.Io.Dir.MakeError.ReadOnlyFileSystem; - } - - fn dirMakePath( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - mode: std.Io.Dir.Mode, - ) std.Io.Dir.MakeError!void { - _ = userdata; - _ = dir; - _ = sub_path; - _ = mode; - log.err("VFS is read-only, dirMakePath not supported", .{}); - return std.Io.Dir.MakeError.ReadOnlyFileSystem; - } - - fn dirMakeOpenPath( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - options: std.Io.Dir.OpenOptions, - ) std.Io.Dir.MakeOpenPathError!std.Io.Dir { - _ = userdata; - _ = dir; - _ = sub_path; - _ = options; - log.err("VFS is read-only, dirMakeOpenPath not supported", .{}); - return std.Io.Dir.MakeOpenPathError.Unexpected; - } - - fn dirStat( - userdata: ?*anyopaque, - dir: std.Io.Dir, - ) std.Io.Dir.StatError!std.Io.Dir.Stat { - _ = userdata; - _ = dir; - log.err("VFS does not support dirStat", .{}); - return std.Io.Dir.StatError.AccessDenied; - } - - fn dirStatPath( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - options: std.Io.Dir.StatPathOptions, - ) std.Io.Dir.StatPathError!std.Io.File.Stat { - _ = userdata; - _ = dir; - _ = sub_path; - _ = options; - @panic("TODO: implement VFS dirStatPath"); - } - - fn dirAccess( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - options: std.Io.Dir.AccessOptions, - ) std.Io.Dir.AccessError!void { - _ = userdata; - _ = dir; - _ = sub_path; - - if (options.write) { - return std.Io.Dir.AccessError.ReadOnlyFileSystem; - } - - log.err("VFS does not support dirAccess", .{}); - return std.Io.Dir.AccessError.AccessDenied; - } - - fn dirCreateFile( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - flags: std.Io.File.CreateFlags, - ) std.Io.File.OpenError!std.Io.File { - _ = userdata; - _ = dir; - _ = sub_path; - _ = flags; - log.err("VFS is read-only, dirCreateFile not supported", .{}); - return std.Io.File.OpenError.Unexpected; - } - - fn dirOpenFile( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - flags: std.Io.File.OpenFlags, - ) std.Io.File.OpenError!std.Io.File { - _ = flags; - const self: *HTTP = @ptrCast(@alignCast(userdata orelse unreachable)); - return try self.openFile(dir, sub_path); - } - - fn dirOpenDir( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - options: std.Io.Dir.OpenOptions, - ) std.Io.Dir.OpenError!std.Io.Dir { - _ = options; - const self: *HTTP = @ptrCast(@alignCast(userdata orelse unreachable)); - return try self.openDir(dir, sub_path); - } - - fn dirClose(userdata: ?*anyopaque, dir: std.Io.Dir) void { - const self: *HTTP = @ptrCast(@alignCast(userdata orelse unreachable)); - self.closeDir(dir); - } - - fn fileStat( - userdata: ?*anyopaque, - file: std.Io.File, - ) std.Io.File.StatError!std.Io.File.Stat { - const self: *HTTP = @ptrCast(@alignCast(userdata orelse unreachable)); - const handle = self.fileHandle(file) orelse { - log.err("Invalid file handle in VFS stat", .{}); - - return std.Io.File.StatError.Unexpected; - }; + const handle = self.getFileHandle(file); return .{ .inode = @intCast(file.handle), - .kind = .file, + .nlink = 0, .size = handle.size, - .mode = 0o444, - .atime = .{ .nanoseconds = 0 }, - .mtime = .{ .nanoseconds = 0 }, - .ctime = .{ .nanoseconds = 0 }, + .permissions = .fromMode(0o444), + .kind = .file, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, }; } - fn fileClose(userdata: ?*anyopaque, file: std.Io.File) void { - const self: *HTTP = @ptrCast(@alignCast(userdata orelse unreachable)); - self.closeFile(file); + fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.getFileHandle(file).size; } - fn fileWriteStreaming( - userdata: ?*anyopaque, - file: std.Io.File, - buffer: [][]const u8, - ) std.Io.File.WriteStreamingError!usize { - _ = userdata; - _ = file; - _ = buffer; - log.err("VFS is read-only, fileWriteStreaming not supported", .{}); - return std.Io.File.WriteStreamingError.Unexpected; + fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + for (files) |file| { + self.closeHandle(@intCast(file.handle)) catch unreachable; + } } - fn fileWritePositional( - userdata: ?*anyopaque, - file: std.Io.File, - buffer: [][]const u8, - offset: u64, - ) std.Io.File.WritePositionalError!usize { - _ = userdata; - _ = file; - _ = buffer; - _ = offset; - log.err("VFS is read-only, fileWritePositional not supported", .{}); - return std.Io.File.WritePositionalError.Unexpected; + fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + const total = self.performRead(handle, data, handle.pos) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); + return std.Io.File.Reader.Error.Unexpected; + }; + handle.pos += @intCast(total); + return total; } - fn fileReadStreaming( - userdata: ?*anyopaque, - file: std.Io.File, - data: [][]u8, - ) std.Io.File.Reader.Error!usize { - const self: *HTTP = @ptrCast(@alignCast(userdata orelse unreachable)); - return try self.performRead(file, data); + fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return self.performRead(handle, data, offset) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); + return std.Io.File.Reader.Error.Unexpected; + }; } - fn fileReadPositional( - userdata: ?*anyopaque, - file: std.Io.File, - data: [][]u8, - offset: u64, - ) std.Io.File.ReadPositionalError!usize { - const self: *HTTP = @ptrCast(@alignCast(userdata orelse unreachable)); - return try self.performPositionalRead(file, data, offset); + fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + + handle.pos = if (relative_offset >= 0) + handle.pos + @as(u64, @intCast(relative_offset)) + else + handle.pos - @as(u64, @intCast(-relative_offset)); } - fn fileSeekBy( - userdata: ?*anyopaque, - file: std.Io.File, - relative_offset: i64, - ) std.Io.File.SeekError!void { - const self: *HTTP = @ptrCast(@alignCast(userdata orelse unreachable)); - try self.setRelativeOffset(file, relative_offset); + fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + handle.pos = absolute_offset; } - fn fileSeekTo( - userdata: ?*anyopaque, - file: std.Io.File, - absolute_offset: u64, - ) std.Io.File.SeekError!void { - const self: *HTTP = @ptrCast(@alignCast(userdata orelse unreachable)); - try self.setAbsolutePos(file, absolute_offset); + fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.File.RealPathError.SystemResources; + return path.len; } - fn openSelfExe( - userdata: ?*anyopaque, - flags: std.Io.File.OpenFlags, - ) std.Io.File.OpenSelfExeError!std.Io.File { - _ = userdata; - _ = flags; - log.err("VFS does not support openSelfExe", .{}); - return std.Io.File.OpenSelfExeError.NotSupported; + fn fetchSize(self: *HTTP, dir: std.Io.Dir, sub_path: []const u8) !u64 { + var path_buffer: [8 * 1024]u8 = undefined; + var redirect_buffer: [8 * 1024]u8 = undefined; + var aux_buffer: []u8 = &redirect_buffer; + + const scheme = @tagName(self.protocol); + const url = try self.resolvePath(dir, sub_path, &path_buffer); + const full_url = try std.fmt.bufPrint(aux_buffer, "{s}://{s}", .{ scheme, url }); + + var uri = std.Uri.parse(full_url) catch return std.Io.File.OpenError.BadPathName; + while (true) { + var req = try self.client.request(.HEAD, uri, .{ + .redirect_behavior = .not_allowed, + .headers = .{ .accept_encoding = .{ .override = "identity" } }, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var res = try req.receiveHead(&redirect_buffer); + + switch (res.head.status.class()) { + .server_error, .client_error => { + log.err("Failed to fetch tree size for {s}", .{url}); + log.err("{s}", .{res.head.bytes}); + return error.ServerError; + }, + .informational => return error.UnexpectedStatus, + .success => return res.head.content_length.?, + .redirect => { + const location = res.head.location.?; + @memcpy(aux_buffer[0..location.len], location); + uri = uri.resolveInPlace(location.len, &aux_buffer) catch unreachable; + continue; + }, + } + } + } + + fn performRead(self: *HTTP, handle: *Handle, data: []const []u8, offset: u64) !usize { + if (offset >= handle.size) return 0; + + var range_buf: [64]u8 = undefined; + const range_header = blk: { + var total_bytes: u64 = 0; + for (data) |buf| { + total_bytes += @as(u64, buf.len); + } + const remaining = handle.size - offset; + const take = @min(remaining, total_bytes); + const end = offset + take - 1; + break :blk std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ offset, end }) catch unreachable; + }; + + var url_buffer: [8 * 1024]u8 = undefined; + const url = try std.fmt.bufPrint(&url_buffer, "{s}://{s}", .{ @tagName(self.protocol), handle.uri }); + const uri: std.Uri = try .parse(url); + + var req = try self.client.request(.GET, uri, .{ + .headers = .{ .accept_encoding = .{ .override = "identity" } }, + .extra_headers = &.{.{ .name = "Range", .value = range_header }}, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var redirect_buffer: [8 * 1024]u8 = undefined; + var res = try req.receiveHead(&redirect_buffer); + + if (res.head.status != .partial_content and res.head.status != .ok) { + log.err("Failed to perform read for {s}", .{handle.uri}); + log.err("{s}", .{res.head.bytes}); + return error.RequestFailed; + } + + const content_range = blk: { + var it = res.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { + break :blk parseContentRange(header.value); + } + } + break :blk null; + }; + + const reader = res.reader(&.{}); + + if (content_range) |cr| { + if (cr.start < offset) { + try reader.discardAll(offset - cr.start); + } + } + + return try reader.readSliceShort(data[0]); + } + + const ContentRange = struct { + start: u64, + end: u64, + total: u64, + }; + + fn parseContentRange(value: []const u8) ?ContentRange { + const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; + const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; + const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; + + return .{ + .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, + .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, + .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, + }; } }; diff --git a/zml/io/vfs/index.zig b/zml/io/vfs/index.zig index fc44f6b6..91c1d2e4 100644 --- a/zml/io/vfs/index.zig +++ b/zml/io/vfs/index.zig @@ -2,6 +2,7 @@ const std = @import("std"); const log = std.log.scoped(.@"zml/io/vfs/vfs"); +pub const VFSBase = @import("base.zig").VFSBase; pub const File = @import("file.zig").File; pub const HTTP = @import("http.zig").HTTP; pub const HF = @import("hf.zig").HF; diff --git a/zml/io/vfs/proxy.zig b/zml/io/vfs/proxy.zig deleted file mode 100644 index 0382d5ac..00000000 --- a/zml/io/vfs/proxy.zig +++ /dev/null @@ -1,546 +0,0 @@ -const std = @import("std"); - -pub const IoProxy = struct { - inner: std.Io, - - pub fn init(io: std.Io) IoProxy { - return .{ .inner = io }; - } - - pub fn partial(overrides: anytype) std.Io.VTable { - var new_vtable = vtable; - for (std.meta.fieldNames(@TypeOf(overrides))) |field_name| { - @field(new_vtable, field_name) = @field(overrides, field_name); - } - return new_vtable; - } - - pub fn as(userdata: ?*anyopaque) *IoProxy { - return @ptrCast(@alignCast(userdata orelse unreachable)); - } - - pub const vtable: std.Io.VTable = .{ - .async = async, - .concurrent = concurrent, - .await = await, - .cancel = cancel, - .groupAsync = groupAsync, - .groupConcurrent = groupConcurrent, - .groupAwait = groupAwait, - .groupCancel = groupCancel, - .recancel = recancel, - .swapCancelProtection = swapCancelProtection, - .checkCancel = checkCancel, - .select = select, - .futexWait = futexWait, - .futexWaitUncancelable = futexWaitUncancelable, - .futexWake = futexWake, - .dirCreateDir = dirCreateDir, - .dirCreateDirPath = dirCreateDirPath, - .dirCreateDirPathOpen = dirCreateDirPathOpen, - .dirOpenDir = dirOpenDir, - .dirStat = dirStat, - .dirStatFile = dirStatFile, - .dirAccess = dirAccess, - .dirCreateFile = dirCreateFile, - .dirOpenFile = dirOpenFile, - .dirClose = dirClose, - .dirRead = dirRead, - .dirRealPath = dirRealPath, - .dirRealPathFile = dirRealPathFile, - .dirDeleteFile = dirDeleteFile, - .dirDeleteDir = dirDeleteDir, - .dirRename = dirRename, - .dirSymLink = dirSymLink, - .dirReadLink = dirReadLink, - .dirSetOwner = dirSetOwner, - .dirSetFileOwner = dirSetFileOwner, - .dirSetPermissions = dirSetPermissions, - .dirSetFilePermissions = dirSetFilePermissions, - .dirSetTimestamps = dirSetTimestamps, - .dirHardLink = dirHardLink, - .fileStat = fileStat, - .fileLength = fileLength, - .fileClose = fileClose, - .fileWriteStreaming = fileWriteStreaming, - .fileWritePositional = fileWritePositional, - .fileWriteFileStreaming = fileWriteFileStreaming, - .fileWriteFilePositional = fileWriteFilePositional, - .fileReadStreaming = fileReadStreaming, - .fileReadPositional = fileReadPositional, - .fileSeekBy = fileSeekBy, - .fileSeekTo = fileSeekTo, - .fileSync = fileSync, - .fileIsTty = fileIsTty, - .fileEnableAnsiEscapeCodes = fileEnableAnsiEscapeCodes, - .fileSupportsAnsiEscapeCodes = fileSupportsAnsiEscapeCodes, - .fileSetLength = fileSetLength, - .fileSetOwner = fileSetOwner, - .fileSetPermissions = fileSetPermissions, - .fileSetTimestamps = fileSetTimestamps, - .fileLock = fileLock, - .fileTryLock = fileTryLock, - .fileUnlock = fileUnlock, - .fileDowngradeLock = fileDowngradeLock, - .fileRealPath = fileRealPath, - .processExecutableOpen = processExecutableOpen, - .processExecutablePath = processExecutablePath, - .lockStderr = lockStderr, - .tryLockStderr = tryLockStderr, - .unlockStderr = unlockStderr, - .processSetCurrentDir = processSetCurrentDir, - .now = now, - .sleep = sleep, - .netListenIp = netListenIp, - .netAccept = netAccept, - .netBindIp = netBindIp, - .netConnectIp = netConnectIp, - .netListenUnix = netListenUnix, - .netConnectUnix = netConnectUnix, - .netSend = netSend, - .netReceive = netReceive, - .netRead = netRead, - .netWrite = netWrite, - .netWriteFile = netWriteFile, - .netClose = netClose, - .netShutdown = netShutdown, - .netInterfaceNameResolve = netInterfaceNameResolve, - .netInterfaceName = netInterfaceName, - .netLookup = netLookup, - }; - - pub fn async(userdata: ?*anyopaque, result: []u8, result_alignment: std.mem.Alignment, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque, result: *anyopaque) void) ?*std.Io.AnyFuture { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.async(self.inner.userdata, result, result_alignment, context, context_alignment, start); - } - - pub fn concurrent(userdata: ?*anyopaque, result_len: usize, result_alignment: std.mem.Alignment, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque, result: *anyopaque) void) std.Io.ConcurrentError!*std.Io.AnyFuture { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.concurrent(self.inner.userdata, result_len, result_alignment, context, context_alignment, start); - } - - pub fn await(userdata: ?*anyopaque, any_future: *std.Io.AnyFuture, result: []u8, result_alignment: std.mem.Alignment) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.await(self.inner.userdata, any_future, result, result_alignment); - } - - pub fn cancel(userdata: ?*anyopaque, any_future: *std.Io.AnyFuture, result: []u8, result_alignment: std.mem.Alignment) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.cancel(self.inner.userdata, any_future, result, result_alignment); - } - - pub fn groupAsync(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (*std.Io.Group, context: *const anyopaque) std.Io.Cancelable!void) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.groupAsync(self.inner.userdata, group, context, context_alignment, start); - } - - pub fn groupConcurrent(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (*std.Io.Group, context: *const anyopaque) std.Io.Cancelable!void) std.Io.ConcurrentError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.groupConcurrent(self.inner.userdata, group, context, context_alignment, start); - } - - pub fn groupAwait(userdata: ?*anyopaque, group: *std.Io.Group, token: *anyopaque) std.Io.Cancelable!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.groupAwait(self.inner.userdata, group, token); - } - - pub fn groupCancel(userdata: ?*anyopaque, group: *std.Io.Group, token: *anyopaque) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.groupCancel(self.inner.userdata, group, token); - } - - pub fn recancel(userdata: ?*anyopaque) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.recancel(self.inner.userdata); - } - - pub fn swapCancelProtection(userdata: ?*anyopaque, new: std.Io.CancelProtection) std.Io.CancelProtection { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.swapCancelProtection(self.inner.userdata, new); - } - - pub fn checkCancel(userdata: ?*anyopaque) std.Io.Cancelable!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.checkCancel(self.inner.userdata); - } - - pub fn select(userdata: ?*anyopaque, futures: []const *std.Io.AnyFuture) std.Io.Cancelable!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.select(self.inner.userdata, futures); - } - - pub fn futexWait(userdata: ?*anyopaque, ptr: *const u32, expected: u32, timeout: std.Io.Timeout) std.Io.Cancelable!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.futexWait(self.inner.userdata, ptr, expected, timeout); - } - - pub fn futexWaitUncancelable(userdata: ?*anyopaque, ptr: *const u32, expected: u32) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.futexWaitUncancelable(self.inner.userdata, ptr, expected); - } - - pub fn futexWake(userdata: ?*anyopaque, ptr: *const u32, max_waiters: u32) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.futexWake(self.inner.userdata, ptr, max_waiters); - } - - pub fn dirCreateDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions) std.Io.Dir.CreateDirError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirCreateDir(self.inner.userdata, dir, sub_path, permissions); - } - - pub fn dirCreateDirPath(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions) std.Io.Dir.CreateDirPathError!std.Io.Dir.CreatePathStatus { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirCreateDirPath(self.inner.userdata, dir, sub_path, permissions); - } - - pub fn dirCreateDirPathOpen(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions, options: std.Io.Dir.OpenOptions) std.Io.Dir.CreateDirPathOpenError!std.Io.Dir { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirCreateDirPathOpen(self.inner.userdata, dir, sub_path, permissions, options); - } - - pub fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirOpenDir(self.inner.userdata, dir, sub_path, options); - } - - pub fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirStat(self.inner.userdata, dir); - } - - pub fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirStatFile(self.inner.userdata, dir, sub_path, options); - } - - pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirAccess(self.inner.userdata, dir, sub_path, options); - } - - pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.CreateFlags) std.Io.File.OpenError!std.Io.File { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirCreateFile(self.inner.userdata, dir, sub_path, flags); - } - - pub fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirOpenFile(self.inner.userdata, dir, sub_path, flags); - } - - pub fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - self.inner.vtable.dirClose(self.inner.userdata, dirs); - } - - pub fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirRead(self.inner.userdata, reader, entries); - } - - pub fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirRealPath(self.inner.userdata, dir, out_buffer); - } - - pub fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirRealPathFile(self.inner.userdata, dir, path_name, out_buffer); - } - - pub fn dirDeleteFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8) std.Io.Dir.DeleteFileError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirDeleteFile(self.inner.userdata, dir, sub_path); - } - - pub fn dirDeleteDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8) std.Io.Dir.DeleteDirError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirDeleteDir(self.inner.userdata, dir, sub_path); - } - - pub fn dirRename(userdata: ?*anyopaque, old_dir: std.Io.Dir, old_sub_path: []const u8, new_dir: std.Io.Dir, new_sub_path: []const u8) std.Io.Dir.RenameError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirRename(self.inner.userdata, old_dir, old_sub_path, new_dir, new_sub_path); - } - - pub fn dirSymLink(userdata: ?*anyopaque, dir: std.Io.Dir, target_path: []const u8, sym_link_path: []const u8, flags: std.Io.Dir.SymLinkFlags) std.Io.Dir.SymLinkError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirSymLink(self.inner.userdata, dir, target_path, sym_link_path, flags); - } - - pub fn dirReadLink(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, buffer: []u8) std.Io.Dir.ReadLinkError!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirReadLink(self.inner.userdata, dir, sub_path, buffer); - } - - pub fn dirSetOwner(userdata: ?*anyopaque, dir: std.Io.Dir, uid: ?std.Io.File.Uid, gid: ?std.Io.File.Gid) std.Io.Dir.SetOwnerError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirSetOwner(self.inner.userdata, dir, uid, gid); - } - - pub fn dirSetFileOwner(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, uid: ?std.Io.File.Uid, gid: ?std.Io.File.Gid, options: std.Io.Dir.SetFileOwnerOptions) std.Io.Dir.SetFileOwnerError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirSetFileOwner(self.inner.userdata, dir, sub_path, uid, gid, options); - } - - pub fn dirSetPermissions(userdata: ?*anyopaque, dir: std.Io.Dir, permissions: std.Io.Dir.Permissions) std.Io.Dir.SetPermissionsError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirSetPermissions(self.inner.userdata, dir, permissions); - } - - pub fn dirSetFilePermissions(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.File.Permissions, options: std.Io.Dir.SetFilePermissionsOptions) std.Io.Dir.SetFilePermissionsError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirSetFilePermissions(self.inner.userdata, dir, sub_path, permissions, options); - } - - pub fn dirSetTimestamps(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.SetTimestampsOptions) std.Io.Dir.SetTimestampsError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirSetTimestamps(self.inner.userdata, dir, sub_path, options); - } - - pub fn dirHardLink(userdata: ?*anyopaque, old_dir: std.Io.Dir, old_sub_path: []const u8, new_dir: std.Io.Dir, new_sub_path: []const u8, options: std.Io.Dir.HardLinkOptions) std.Io.Dir.HardLinkError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirHardLink(self.inner.userdata, old_dir, old_sub_path, new_dir, new_sub_path, options); - } - - pub fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileStat(self.inner.userdata, file); - } - - pub fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileLength(self.inner.userdata, file); - } - - pub fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - self.inner.vtable.fileClose(self.inner.userdata, files); - } - - pub fn fileWriteStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize) std.Io.File.Writer.Error!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileWriteStreaming(self.inner.userdata, file, header, data, splat); - } - - pub fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) std.Io.File.WritePositionalError!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileWritePositional(self.inner.userdata, file, header, data, splat, offset); - } - - pub fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileWriteFileStreaming(self.inner.userdata, file, header, reader, limit); - } - - pub fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileWriteFilePositional(self.inner.userdata, file, header, reader, limit, offset); - } - - pub fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileReadStreaming(self.inner.userdata, file, data); - } - - pub fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileReadPositional(self.inner.userdata, file, data, offset); - } - - pub fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileSeekBy(self.inner.userdata, file, relative_offset); - } - - pub fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileSeekTo(self.inner.userdata, file, absolute_offset); - } - - pub fn fileSync(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.SyncError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileSync(self.inner.userdata, file); - } - - pub fn fileIsTty(userdata: ?*anyopaque, file: std.Io.File) std.Io.Cancelable!bool { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileIsTty(self.inner.userdata, file); - } - - pub fn fileEnableAnsiEscapeCodes(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.EnableAnsiEscapeCodesError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileEnableAnsiEscapeCodes(self.inner.userdata, file); - } - - pub fn fileSupportsAnsiEscapeCodes(userdata: ?*anyopaque, file: std.Io.File) std.Io.Cancelable!bool { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileSupportsAnsiEscapeCodes(self.inner.userdata, file); - } - - pub fn fileSetLength(userdata: ?*anyopaque, file: std.Io.File, length: u64) std.Io.File.SetLengthError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileSetLength(self.inner.userdata, file, length); - } - - pub fn fileSetOwner(userdata: ?*anyopaque, file: std.Io.File, uid: ?std.Io.File.Uid, gid: ?std.Io.File.Gid) std.Io.File.SetOwnerError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileSetOwner(self.inner.userdata, file, uid, gid); - } - - pub fn fileSetPermissions(userdata: ?*anyopaque, file: std.Io.File, permissions: std.Io.File.Permissions) std.Io.File.SetPermissionsError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileSetPermissions(self.inner.userdata, file, permissions); - } - - pub fn fileSetTimestamps(userdata: ?*anyopaque, file: std.Io.File, options: std.Io.File.SetTimestampsOptions) std.Io.File.SetTimestampsError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileSetTimestamps(self.inner.userdata, file, options); - } - - pub fn fileLock(userdata: ?*anyopaque, file: std.Io.File, lock: std.Io.File.Lock) std.Io.File.LockError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileLock(self.inner.userdata, file, lock); - } - - pub fn fileTryLock(userdata: ?*anyopaque, file: std.Io.File, lock: std.Io.File.Lock) std.Io.File.LockError!bool { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileTryLock(self.inner.userdata, file, lock); - } - - pub fn fileUnlock(userdata: ?*anyopaque, file: std.Io.File) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileUnlock(self.inner.userdata, file); - } - - pub fn fileDowngradeLock(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.DowngradeLockError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileDowngradeLock(self.inner.userdata, file); - } - - pub fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileRealPath(self.inner.userdata, file, out_buffer); - } - - pub fn processExecutableOpen(userdata: ?*anyopaque, flags: std.Io.File.OpenFlags) std.process.OpenExecutableError!std.Io.File { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.processExecutableOpen(self.inner.userdata, flags); - } - - pub fn processExecutablePath(userdata: ?*anyopaque, buffer: []u8) std.process.ExecutablePathError!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.processExecutablePath(self.inner.userdata, buffer); - } - - pub fn lockStderr(userdata: ?*anyopaque, mode: ?std.Io.Terminal.Mode) std.Io.Cancelable!std.Io.LockedStderr { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.lockStderr(self.inner.userdata, mode); - } - - pub fn tryLockStderr(userdata: ?*anyopaque, mode: ?std.Io.Terminal.Mode) std.Io.Cancelable!?std.Io.LockedStderr { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.tryLockStderr(self.inner.userdata, mode); - } - - pub fn unlockStderr(userdata: ?*anyopaque) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.unlockStderr(self.inner.userdata); - } - - pub fn processSetCurrentDir(userdata: ?*anyopaque, dir: std.Io.Dir) std.process.SetCurrentDirError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.processSetCurrentDir(self.inner.userdata, dir); - } - - pub fn now(userdata: ?*anyopaque, clock: std.Io.Clock) std.Io.Clock.Error!std.Io.Timestamp { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.now(self.inner.userdata, clock); - } - - pub fn sleep(userdata: ?*anyopaque, timeout: std.Io.Timeout) std.Io.SleepError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.sleep(self.inner.userdata, timeout); - } - - pub fn netListenIp(userdata: ?*anyopaque, address: std.Io.net.IpAddress, options: std.Io.net.IpAddress.ListenOptions) std.Io.net.IpAddress.ListenError!std.Io.net.Server { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netListenIp(self.inner.userdata, address, options); - } - - pub fn netAccept(userdata: ?*anyopaque, server: std.Io.net.Socket.Handle) std.Io.net.Server.AcceptError!std.Io.net.Stream { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netAccept(self.inner.userdata, server); - } - - pub fn netBindIp(userdata: ?*anyopaque, address: *const std.Io.net.IpAddress, options: std.Io.net.IpAddress.BindOptions) std.Io.net.IpAddress.BindError!std.Io.net.Socket { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netBindIp(self.inner.userdata, address, options); - } - - pub fn netConnectIp(userdata: ?*anyopaque, address: *const std.Io.net.IpAddress, options: std.Io.net.IpAddress.ConnectOptions) std.Io.net.IpAddress.ConnectError!std.Io.net.Stream { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netConnectIp(self.inner.userdata, address, options); - } - - pub fn netListenUnix(userdata: ?*anyopaque, address: *const std.Io.net.UnixAddress, options: std.Io.net.UnixAddress.ListenOptions) std.Io.net.UnixAddress.ListenError!std.Io.net.Socket.Handle { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netListenUnix(self.inner.userdata, address, options); - } - - pub fn netConnectUnix(userdata: ?*anyopaque, address: *const std.Io.net.UnixAddress) std.Io.net.UnixAddress.ConnectError!std.Io.net.Socket.Handle { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netConnectUnix(self.inner.userdata, address); - } - - pub fn netSend(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, msgs: []std.Io.net.OutgoingMessage, flags: std.Io.net.SendFlags) struct { ?std.Io.net.Socket.SendError, usize } { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netSend(self.inner.userdata, handle, msgs, flags); - } - - pub fn netReceive(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, message_buffer: []std.Io.net.IncomingMessage, data_buffer: []u8, flags: std.Io.net.ReceiveFlags, timeout: std.Io.Timeout) struct { ?std.Io.net.Socket.ReceiveTimeoutError, usize } { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netReceive(self.inner.userdata, handle, message_buffer, data_buffer, flags, timeout); - } - - pub fn netRead(userdata: ?*anyopaque, src: std.Io.net.Socket.Handle, data: [][]u8) std.Io.net.Stream.Reader.Error!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netRead(self.inner.userdata, src, data); - } - - pub fn netWrite(userdata: ?*anyopaque, dest: std.Io.net.Socket.Handle, header: []const u8, data: []const []const u8, splat: usize) std.Io.net.Stream.Writer.Error!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netWrite(self.inner.userdata, dest, header, data, splat); - } - - pub fn netWriteFile(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.net.Stream.Writer.WriteFileError!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netWriteFile(self.inner.userdata, handle, header, reader, limit); - } - - pub fn netClose(userdata: ?*anyopaque, handles: []const std.Io.net.Socket.Handle) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - self.inner.vtable.netClose(self.inner.userdata, handles); - } - - pub fn netShutdown(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, how: std.Io.net.ShutdownHow) std.Io.net.ShutdownError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netShutdown(self.inner.userdata, handle, how); - } - - pub fn netInterfaceNameResolve(userdata: ?*anyopaque, name: *const std.Io.net.Interface.Name) std.Io.net.Interface.Name.ResolveError!std.Io.net.Interface { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netInterfaceNameResolve(self.inner.userdata, name); - } - - pub fn netInterfaceName(userdata: ?*anyopaque, iface: std.Io.net.Interface) std.Io.net.Interface.NameError!std.Io.net.Interface.Name { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netInterfaceName(self.inner.userdata, iface); - } - - pub fn netLookup(userdata: ?*anyopaque, host: std.Io.net.HostName, q: *std.Io.Queue(std.Io.net.HostName.LookupResult), options: std.Io.net.HostName.LookupOptions) std.Io.net.HostName.LookupError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netLookup(self.inner.userdata, host, q, options); - } -}; diff --git a/zml/safetensors.zig b/zml/safetensors.zig index d7653216..b1b791a1 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -22,21 +22,17 @@ pub const FileType = enum { pub fn fetchRegistry( allocator: std.mem.Allocator, io: std.Io, - repo: ModelRepo, - entrypoint: ModelEntrypoint, + repo: std.Io.Dir, + model: std.Io.File, ) !TensorRegistry { - const file_type = resolveFiletype(entrypoint.path); - log.debug("Resolved file type: {any}", .{file_type}); + const file_type = try resolveFiletype(io, model); return switch (file_type) { .index => blk: { - const index_file = try repo.dir.openFile(io, entrypoint.path, .{ .mode = .read_only }); - defer index_file.close(io); - const file_reader_buf = try allocator.alloc(u8, 64 * 1024); defer allocator.free(file_reader_buf); - var index_reader = index_file.reader(io, file_reader_buf); + var index_reader = model.reader(io, file_reader_buf); var safetensors_index = try parseSafetensorsIndex( allocator, @@ -49,8 +45,8 @@ pub fn fetchRegistry( try parseSafetensorsIndexFiles( allocator, io, - safetensors_index, ®istry, + safetensors_index, repo, ); @@ -59,15 +55,7 @@ pub fn fetchRegistry( .safetensors => blk: { var registry: TensorRegistry = .init(allocator); - const file = try repo.dir.openFile(io, entrypoint.path, .{}); - defer file.close(io); - - const header_buf = try allocator.alloc(u8, 16 * 1024); - defer allocator.free(header_buf); - - var reader = file.reader(io, header_buf); - - try parseSafetensors(allocator, entrypoint.path, &reader.interface, ®istry); + try parseSafetensors(allocator, io, ®istry, model); break :blk registry; }, @@ -194,12 +182,9 @@ pub const TensorRegistry = struct { pub fn fromRepo( allocator: std.mem.Allocator, io: std.Io, - repo: ModelRepo, + repo: std.Io.Dir, ) !TensorRegistry { - var entrypoint = try resolveModelEntrypoint(allocator, io, repo); - defer entrypoint.deinit(allocator, io); - log.debug("Resolved model path entrypoint: {s}", .{entrypoint.path}); - + const entrypoint = try resolveModelEntrypoint(io, repo); return try fetchRegistry(allocator, io, repo, entrypoint); } @@ -208,25 +193,14 @@ pub const TensorRegistry = struct { io: std.Io, path: []const u8, ) !TensorRegistry { - var repo = try resolveModelRepo(allocator, io, path); - defer repo.deinit(allocator, io); + var repo = try resolveModelRepo(io, path); if (std.mem.endsWith(u8, path, ".safetensors.index.json") or std.mem.endsWith(u8, path, ".safetensors")) { - var entrypoint: ModelEntrypoint = .{ - .file = try repo.dir.openFile(io, path, .{ .mode = .read_only }), - .path = try allocator.dupe(u8, path), - }; - defer entrypoint.deinit(allocator, io); - log.debug("Resolved model path entrypoint: {s}", .{entrypoint.path}); - - return try fetchRegistry(allocator, io, repo, entrypoint); + return try fetchRegistry(allocator, io, repo, try repo.openFile(io, path, .{ .mode = .read_only })); } else { - var entrypoint = try resolveModelEntrypoint(allocator, io, repo); - defer entrypoint.deinit(allocator, io); - log.debug("Resolved model path entrypoint: {s}", .{entrypoint.path}); - + const entrypoint = try resolveModelEntrypoint(io, repo); return try fetchRegistry(allocator, io, repo, entrypoint); } } @@ -313,20 +287,27 @@ pub const TensorRegistry = struct { pub fn parseSafetensors( allocator: std.mem.Allocator, - file_uri: []const u8, - reader: *std.Io.Reader, + io: std.Io, registry: *TensorRegistry, + file: std.Io.File, ) !void { var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); - const json_header_length: u64 = try reader.takeInt(u64, .little); + const header_buf = try allocator.alloc(u8, 64 * 1024); + defer allocator.free(header_buf); + var reader = file.reader(io, header_buf); + + var file_uri_buf: [1024]u8 = undefined; + const file_uri_len = try file.realPath(io, &file_uri_buf); + const file_uri = file_uri_buf[0..file_uri_len]; + + const json_header_length: u64 = try reader.interface.takeInt(u64, .little); const data_start_offset = BYTES_HEADER + json_header_length; - const json_data = try arena.allocator().alloc(u8, @intCast(json_header_length)); - try reader.readSliceAll(json_data); + try reader.interface.readSliceAll(json_data); const headers: std.json.Value = try std.json.parseFromSliceLeaky( std.json.Value, @@ -379,7 +360,7 @@ pub fn parseSafetensors( } } -pub const SafetensorsIndex = struct { +const SafetensorsIndex = struct { pub const Map = std.StringArrayHashMapUnmanaged(std.ArrayList([]const u8)); arena: std.heap.ArenaAllocator, @@ -407,7 +388,7 @@ pub const SafetensorsIndex = struct { } }; -pub fn parseSafetensorsIndex( +fn parseSafetensorsIndex( allocator: std.mem.Allocator, reader: *std.Io.Reader, ) !SafetensorsIndex { @@ -452,12 +433,12 @@ pub fn parseSafetensorsIndex( return safetensors_index; } -pub fn parseSafetensorsIndexFiles( +fn parseSafetensorsIndexFiles( allocator: std.mem.Allocator, io: std.Io, - safetensors_index: SafetensorsIndex, registry: *TensorRegistry, - repo: ModelRepo, + safetensors_index: SafetensorsIndex, + repo: std.Io.Dir, ) !void { var group: std.Io.Group = .init; defer group.cancel(io); @@ -490,7 +471,7 @@ const AsyncParseSafetensorsIndexFile = struct { allocator: std.mem.Allocator, io: std.Io, registry: *TensorRegistry, - repo: ModelRepo, + repo: std.Io.Dir, filename: []const u8, err_ptr: *?anyerror, ) void { @@ -503,28 +484,23 @@ const AsyncParseSafetensorsIndexFile = struct { allocator: std.mem.Allocator, io: std.Io, registry: *TensorRegistry, - repo: ModelRepo, + repo: std.Io.Dir, filename: []const u8, ) !void { - const file_uri = try std.fs.path.join(allocator, &.{ repo.path, filename }); - defer allocator.free(file_uri); - - const file = try repo.dir.openFile(io, filename, .{}); + const file = try repo.openFile(io, filename, .{}); defer file.close(io); - const header_buf = try allocator.alloc(u8, 64 * 1024); - defer allocator.free(header_buf); - - var reader = file.reader(io, header_buf); - - try parseSafetensors(allocator, file_uri, &reader.interface, registry); + try parseSafetensors(allocator, io, registry, file); } }; -pub fn resolveFiletype(path: []const u8) FileType { - if (std.mem.endsWith(u8, path, ".safetensors.index.json")) { +pub fn resolveFiletype(io: std.Io, file: std.Io.File) !FileType { + var path_buf: [1024]u8 = undefined; + const path_len = try file.realPath(io, &path_buf); + + if (std.mem.endsWith(u8, path_buf[0..path_len], ".safetensors.index.json")) { return .index; - } else if (std.mem.endsWith(u8, path, ".safetensors")) { + } else if (std.mem.endsWith(u8, path_buf[0..path_len], ".safetensors")) { return .safetensors; } else { return .unknown; @@ -536,108 +512,54 @@ pub const ModelPathResolutionError = error{ InvalidPath, } || std.mem.Allocator.Error; -pub const ModelRepo = struct { - dir: std.Io.Dir, - path: []const u8, - - pub fn deinit(self: *ModelRepo, allocator: std.mem.Allocator, io: std.Io) void { - self.dir.close(io); - allocator.free(self.path); - } -}; - -// Helper to ensure trailing slash from the given user input path. -fn ensureTrailingSlash(allocator: std.mem.Allocator, input: []const u8) ![]u8 { - if (input.len == 0 or input[input.len - 1] == std.fs.path.sep) { - return try allocator.dupe(u8, input); - } else { - var out = try allocator.alloc(u8, input.len + 1); - @memcpy(out[0..input.len], input); - out[input.len] = std.fs.path.sep; - return out; - } -} - pub fn resolveModelRepo( - allocator: std.mem.Allocator, io: std.Io, path: []const u8, -) ModelPathResolutionError!ModelRepo { - const resolved_path = try allocator.dupe(u8, path); - - if (std.mem.endsWith(u8, resolved_path, ".safetensors.index.json") or - std.mem.endsWith(u8, resolved_path, ".safetensors")) +) ModelPathResolutionError!std.Io.Dir { + if (std.mem.endsWith(u8, path, ".safetensors.index.json") or + std.mem.endsWith(u8, path, ".safetensors")) { - const dir_path = std.fs.path.dirname(resolved_path) orelse { - allocator.free(resolved_path); - return ModelPathResolutionError.InvalidPath; - }; + const dir_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; - const dir_path_with_slash = try ensureTrailingSlash(allocator, dir_path); - allocator.free(resolved_path); - - const dir = std.Io.Dir.openDir(.cwd(), io, dir_path_with_slash, .{}) catch |e| { - allocator.free(dir_path_with_slash); + const dir = std.Io.Dir.openDir(.cwd(), io, dir_path, .{}) catch |e| { return switch (e) { error.FileNotFound => ModelPathResolutionError.FileNotFound, else => ModelPathResolutionError.InvalidPath, }; }; - return .{ - .dir = dir, - .path = dir_path_with_slash, - }; + return dir; } - const path_with_slash = try ensureTrailingSlash(allocator, resolved_path); - allocator.free(resolved_path); - - const dir = std.Io.Dir.openDir(.cwd(), io, path_with_slash, .{}) catch |e| { - allocator.free(path_with_slash); + const dir = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { return switch (e) { error.FileNotFound => ModelPathResolutionError.FileNotFound, else => ModelPathResolutionError.InvalidPath, }; }; - return .{ - .dir = dir, - .path = path_with_slash, - }; + return dir; } -pub const ModelEntrypoint = struct { - file: std.Io.File, - path: []const u8, - - pub fn deinit(self: *ModelEntrypoint, allocator: std.mem.Allocator, io: std.Io) void { - self.file.close(io); - allocator.free(self.path); - } -}; - -pub fn resolveModelEntrypoint(allocator: std.mem.Allocator, io: std.Io, repo: ModelRepo) ModelPathResolutionError!ModelEntrypoint { +pub fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!std.Io.File { { const index_name = "model.safetensors.index.json"; - if (repo.dir.openFile(io, index_name, .{ .mode = .read_only })) |index_file| { - const index_path = try std.fmt.allocPrint(allocator, "{s}{s}", .{ repo.path, index_name }); - return .{ .file = index_file, .path = index_path }; + if (repo.openFile(io, index_name, .{ .mode = .read_only })) |index_file| { + return index_file; } else |_| {} } { const model_name = "model.safetensors"; - if (repo.dir.openFile(io, model_name, .{ .mode = .read_only })) |model_file| { - const model_path = try std.fmt.allocPrint(allocator, "{s}{s}", .{ repo.path, model_name }); - return .{ .file = model_file, .path = model_path }; + if (repo.openFile(io, model_name, .{ .mode = .read_only })) |model_file| { + return model_file; } else |_| {} } return ModelPathResolutionError.FileNotFound; } -pub const Metadata = union(enum) { +const Metadata = union(enum) { null: void, int: i64, float: f64, @@ -774,7 +696,7 @@ pub const Metadata = union(enum) { } }; -pub fn stringToDtype(safetensor_type: []const u8) !DataType { +fn stringToDtype(safetensor_type: []const u8) !DataType { const map = std.StaticStringMap(DataType).initComptime(.{ .{ "F64", .f64 }, .{ "F32", .f32 }, From 42633d133d14dfcf7b0d10db1b39c4a40a4f2c7c Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 14 Jan 2026 16:43:54 +0000 Subject: [PATCH 234/881] use model name as progress --- examples/llama/main.zig | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 30ce7929..67dff8c0 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -211,7 +211,13 @@ pub fn main() !void { write_pool.deinit(); } - var progress = std.Progress.start(io, .{ .root_name = args.model.?, .estimated_total_items = store.view().count() }); + var model_name_it = std.mem.splitScalar(u8, args.model.?, '/'); + var model_name = model_name_it.next() orelse args.model.?; + while (model_name_it.next()) |part| { + if (!std.mem.eql(u8, part, "")) model_name = part; + } + + var progress = std.Progress.start(io, .{ .root_name = model_name, .estimated_total_items = store.view().count() }); var llama_buffers_future = io.async(loadModelBuffers, .{ allocator, io, &progress, &load_model_buffers, &read_pool, &write_pool, platform, &store, llama_model }); errdefer b: { var v = llama_buffers_future.cancel(io) catch break :b; From 6ee968e033efe0c5ce0a5d4a2dcd52b30d6618f4 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Wed, 14 Jan 2026 18:08:56 +0100 Subject: [PATCH 235/881] WIP --- MODULE.bazel.lock | 2339 ++++++++++++++++++++++++++++++++++++++- examples/llama/main.zig | 3 - stdx/BUILD.bazel | 1 - 3 files changed, 2338 insertions(+), 5 deletions(-) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index fc687128..b0b9673b 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -435,6 +435,2343 @@ "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198" }, "selectedYankedVersions": {}, - "moduleExtensions": {}, + "moduleExtensions": { + "@@apple_support+//crosstool:setup.bzl%apple_cc_configure_extension": { + "general": { + "bzlTransitiveDigest": "cw6mKbsar0LsMbAePOLZdW+szZNKRE8/CrhomQFNbNE=", + "usagesDigest": "GXVjAqzvGUnh20kBPs+f0GSdAtaqLPet4sogeCfejZY=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "local_config_apple_cc_toolchains": { + "repoRuleId": "@@apple_support+//crosstool:setup.bzl%_apple_cc_autoconf_toolchains", + "attributes": {} + }, + "local_config_apple_cc": { + "repoRuleId": "@@apple_support+//crosstool:setup.bzl%_apple_cc_autoconf", + "attributes": {} + } + }, + "recordedRepoMappingEntries": [ + [ + "apple_support+", + "bazel_tools", + "bazel_tools" + ], + [ + "bazel_tools", + "rules_cc", + "rules_cc+" + ] + ] + } + }, + "@@aspect_rules_py+//py:extensions.bzl%py_tools": { + "general": { + "bzlTransitiveDigest": "G/1q87gmIpapGBPovxgsd2S/VIcXuqJ4H0PnAtwkKWc=", + "usagesDigest": "tVlELaqQFX7HPl1uFzRY9m7nVV7tY9roq1qfK75ApwE=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "bsd_tar_darwin_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", + "attributes": { + "platform": "darwin_amd64" + } + }, + "bsd_tar_darwin_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", + "attributes": { + "platform": "darwin_arm64" + } + }, + "bsd_tar_linux_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", + "attributes": { + "platform": "linux_amd64" + } + }, + "bsd_tar_linux_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", + "attributes": { + "platform": "linux_arm64" + } + }, + "bsd_tar_windows_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", + "attributes": { + "platform": "windows_amd64" + } + }, + "bsd_tar_windows_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", + "attributes": { + "platform": "windows_arm64" + } + }, + "bsd_tar_toolchains": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%tar_toolchains_repo", + "attributes": { + "user_repository_name": "bsd_tar" + } + }, + "rules_py_tools.darwin_amd64": { + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", + "attributes": { + "platform": "darwin_amd64" + } + }, + "rules_py_tools.darwin_arm64": { + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", + "attributes": { + "platform": "darwin_arm64" + } + }, + "rules_py_tools.linux_amd64": { + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", + "attributes": { + "platform": "linux_amd64" + } + }, + "rules_py_tools.linux_arm64": { + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", + "attributes": { + "platform": "linux_arm64" + } + }, + "rules_py_tools": { + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%toolchains_repo", + "attributes": { + "user_repository_name": "rules_py_tools" + } + }, + "rules_py_pex_2_3_1": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "urls": [ + "https://files.pythonhosted.org/packages/e7/d0/fbda2a4d41d62d86ce53f5ae4fbaaee8c34070f75bb7ca009090510ae874/pex-2.3.1-py2.py3-none-any.whl" + ], + "sha256": "64692a5bf6f298403aab930d22f0d836ae4736c5bc820e262e9092fe8c56f830", + "downloaded_file_path": "pex-2.3.1-py2.py3-none-any.whl" + } + } + }, + "recordedRepoMappingEntries": [ + [ + "aspect_bazel_lib+", + "bazel_tools", + "bazel_tools" + ], + [ + "aspect_rules_py+", + "aspect_bazel_lib", + "aspect_bazel_lib+" + ], + [ + "aspect_rules_py+", + "aspect_tools_telemetry_report", + "aspect_tools_telemetry++telemetry+aspect_tools_telemetry_report" + ], + [ + "aspect_rules_py+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "aspect_rules_py+", + "bazel_tools", + "bazel_tools" + ] + ] + } + }, + "@@aspect_tools_telemetry+//:extension.bzl%telemetry": { + "general": { + "bzlTransitiveDigest": "aVNc9R1NtH1OtynCXvZ9j7WtILv+X93Qxzmkyn4x++8=", + "usagesDigest": "bLd5DJLF7d7awlSuBaWvY0qEqBLja5XVKriiQmSNEi8=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "aspect_tools_telemetry_report": { + "repoRuleId": "@@aspect_tools_telemetry+//:extension.bzl%tel_repository", + "attributes": { + "deps": { + "aspect_rules_py": "1.6.3", + "aspect_tools_telemetry": "0.2.6" + } + } + } + }, + "recordedRepoMappingEntries": [ + [ + "aspect_tools_telemetry+", + "aspect_bazel_lib", + "aspect_bazel_lib+" + ], + [ + "aspect_tools_telemetry+", + "bazel_skylib", + "bazel_skylib+" + ] + ] + } + }, + "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { + "general": { + "bzlTransitiveDigest": "AEsQvKwdstptzt98AO1bJYfoWpPUn2upx53ra6o/Rc8=", + "usagesDigest": "QI2z8ZUR+mqtbwsf2fLqYdJAkPOHdOV+tF2yVAUgRzw=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "com_github_jetbrains_kotlin_git": { + "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:compiler.bzl%kotlin_compiler_git_repository", + "attributes": { + "urls": [ + "https://github.com/JetBrains/kotlin/releases/download/v1.9.23/kotlin-compiler-1.9.23.zip" + ], + "sha256": "93137d3aab9afa9b27cb06a824c2324195c6b6f6179d8a8653f440f5bd58be88" + } + }, + "com_github_jetbrains_kotlin": { + "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:compiler.bzl%kotlin_capabilities_repository", + "attributes": { + "git_repository_name": "com_github_jetbrains_kotlin_git", + "compiler_version": "1.9.23" + } + }, + "com_github_google_ksp": { + "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:ksp.bzl%ksp_compiler_plugin_repository", + "attributes": { + "urls": [ + "https://github.com/google/ksp/releases/download/1.9.23-1.0.20/artifacts.zip" + ], + "sha256": "ee0618755913ef7fd6511288a232e8fad24838b9af6ea73972a76e81053c8c2d", + "strip_version": "1.9.23-1.0.20" + } + }, + "com_github_pinterest_ktlint": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "sha256": "01b2e0ef893383a50dbeb13970fe7fa3be36ca3e83259e01649945b09d736985", + "urls": [ + "https://github.com/pinterest/ktlint/releases/download/1.3.0/ktlint" + ], + "executable": true + } + }, + "rules_android": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "sha256": "cd06d15dd8bb59926e4d65f9003bfc20f9da4b2519985c27e190cddc8b7a7806", + "strip_prefix": "rules_android-0.1.1", + "urls": [ + "https://github.com/bazelbuild/rules_android/archive/v0.1.1.zip" + ] + } + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_kotlin+", + "bazel_tools", + "bazel_tools" + ] + ] + } + }, + "@@rules_oci+//oci:extensions.bzl%oci": { + "general": { + "bzlTransitiveDigest": "Uiu9QIgPhgfbEC1lExLocKf2lZ0a9XSSmq3OrI8IL8I=", + "usagesDigest": "vK9qUa3pyGJik2hb1NI2fKVJrjIszor+wN/44XMkKi0=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "distroless_cc_debian12_linux_amd64": { + "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_pull", + "attributes": { + "www_authenticate_challenges": {}, + "scheme": "https", + "registry": "gcr.io", + "repository": "distroless/cc-debian12", + "identifier": "sha256:eccec5274132c1be0ce5d2c8e6fe41033e64af5e987ccee9007826e4c012069d", + "platform": "linux/amd64", + "target_name": "distroless_cc_debian12_linux_amd64", + "bazel_tags": [] + } + }, + "distroless_cc_debian12": { + "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_alias", + "attributes": { + "target_name": "distroless_cc_debian12", + "www_authenticate_challenges": {}, + "scheme": "https", + "registry": "gcr.io", + "repository": "distroless/cc-debian12", + "identifier": "sha256:eccec5274132c1be0ce5d2c8e6fe41033e64af5e987ccee9007826e4c012069d", + "platforms": { + "@@platforms//cpu:x86_64": "@distroless_cc_debian12_linux_amd64" + }, + "bzlmod_repository": "distroless_cc_debian12", + "reproducible": true + } + }, + "oci_crane_darwin_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "darwin_amd64", + "crane_version": "v0.18.0" + } + }, + "oci_crane_darwin_arm64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "darwin_arm64", + "crane_version": "v0.18.0" + } + }, + "oci_crane_linux_arm64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "linux_arm64", + "crane_version": "v0.18.0" + } + }, + "oci_crane_linux_armv6": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "linux_armv6", + "crane_version": "v0.18.0" + } + }, + "oci_crane_linux_i386": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "linux_i386", + "crane_version": "v0.18.0" + } + }, + "oci_crane_linux_s390x": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "linux_s390x", + "crane_version": "v0.18.0" + } + }, + "oci_crane_linux_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "linux_amd64", + "crane_version": "v0.18.0" + } + }, + "oci_crane_windows_armv6": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "windows_armv6", + "crane_version": "v0.18.0" + } + }, + "oci_crane_windows_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "windows_amd64", + "crane_version": "v0.18.0" + } + }, + "oci_crane_toolchains": { + "repoRuleId": "@@rules_oci+//oci/private:toolchains_repo.bzl%toolchains_repo", + "attributes": { + "toolchain_type": "@rules_oci//oci:crane_toolchain_type", + "toolchain": "@oci_crane_{platform}//:crane_toolchain" + } + }, + "oci_regctl_darwin_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "darwin_amd64" + } + }, + "oci_regctl_darwin_arm64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "darwin_arm64" + } + }, + "oci_regctl_linux_arm64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "linux_arm64" + } + }, + "oci_regctl_linux_s390x": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "linux_s390x" + } + }, + "oci_regctl_linux_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "linux_amd64" + } + }, + "oci_regctl_windows_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "windows_amd64" + } + }, + "oci_regctl_toolchains": { + "repoRuleId": "@@rules_oci+//oci/private:toolchains_repo.bzl%toolchains_repo", + "attributes": { + "toolchain_type": "@rules_oci//oci:regctl_toolchain_type", + "toolchain": "@oci_regctl_{platform}//:regctl_toolchain" + } + } + }, + "moduleExtensionMetadata": { + "explicitRootModuleDirectDeps": [ + "distroless_cc_debian12", + "distroless_cc_debian12_linux_amd64" + ], + "explicitRootModuleDirectDevDeps": [], + "useAllRepos": "NO", + "reproducible": false + }, + "recordedRepoMappingEntries": [ + [ + "aspect_bazel_lib+", + "bazel_tools", + "bazel_tools" + ], + [ + "bazel_features+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_oci+", + "aspect_bazel_lib", + "aspect_bazel_lib+" + ], + [ + "rules_oci+", + "bazel_features", + "bazel_features+" + ], + [ + "rules_oci+", + "bazel_skylib", + "bazel_skylib+" + ] + ] + } + }, + "@@rules_python+//python/extensions:config.bzl%config": { + "general": { + "bzlTransitiveDigest": "X49Lbeli5+XwNYRk+isOvNKFMIF8g/rnxL3Fdlwrffc=", + "usagesDigest": "tIEieEA/gbsjNF3L/Oouyg6UdqGOVxFsPqBjFxkTAKM=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "rules_python_internal": { + "repoRuleId": "@@rules_python+//python/private:internal_config_repo.bzl%internal_config_repo", + "attributes": { + "transition_setting_generators": {}, + "transition_settings": [] + } + }, + "pypi__build": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/e2/03/f3c8ba0a6b6e30d7d18c40faab90807c9bb5e9a1e3b2fe2008af624a9c97/build-1.2.1-py3-none-any.whl", + "sha256": "75e10f767a433d9a86e50d83f418e83efc18ede923ee5ff7df93b6cb0306c5d4", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__click": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl", + "sha256": "ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__colorama": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", + "sha256": "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__importlib_metadata": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/2d/0a/679461c511447ffaf176567d5c496d1de27cbe34a87df6677d7171b2fbd4/importlib_metadata-7.1.0-py3-none-any.whl", + "sha256": "30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__installer": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl", + "sha256": "05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__more_itertools": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/50/e2/8e10e465ee3987bb7c9ab69efb91d867d93959095f4807db102d07995d94/more_itertools-10.2.0-py3-none-any.whl", + "sha256": "686b06abe565edfab151cb8fd385a05651e1fdf8f0a14191e4439283421f8684", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__packaging": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/49/df/1fceb2f8900f8639e278b056416d49134fb8d84c5942ffaa01ad34782422/packaging-24.0-py3-none-any.whl", + "sha256": "2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__pep517": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/25/6e/ca4a5434eb0e502210f591b97537d322546e4833dcb4d470a48c375c5540/pep517-0.13.1-py3-none-any.whl", + "sha256": "31b206f67165b3536dd577c5c3f1518e8fbaf38cbc57efff8369a392feff1721", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__pip": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/8a/6a/19e9fe04fca059ccf770861c7d5721ab4c2aebc539889e97c7977528a53b/pip-24.0-py3-none-any.whl", + "sha256": "ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__pip_tools": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/0d/dc/38f4ce065e92c66f058ea7a368a9c5de4e702272b479c0992059f7693941/pip_tools-7.4.1-py3-none-any.whl", + "sha256": "4c690e5fbae2f21e87843e89c26191f0d9454f362d8acdbd695716493ec8b3a9", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__pyproject_hooks": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/ae/f3/431b9d5fe7d14af7a32340792ef43b8a714e7726f1d7b69cc4e8e7a3f1d7/pyproject_hooks-1.1.0-py3-none-any.whl", + "sha256": "7ceeefe9aec63a1064c18d939bdc3adf2d8aa1988a510afec15151578b232aa2", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__setuptools": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/90/99/158ad0609729111163fc1f674a5a42f2605371a4cf036d0441070e2f7455/setuptools-78.1.1-py3-none-any.whl", + "sha256": "c3a9c4211ff4c309edb8b8c4f1cbfa7ae324c4ba9f91ff254e3d305b9fd54561", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__tomli": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", + "sha256": "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__wheel": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/7d/cd/d7460c9a869b16c3dd4e1e403cce337df165368c71d6af229a74699622ce/wheel-0.43.0-py3-none-any.whl", + "sha256": "55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__zipp": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/da/55/a03fd7240714916507e1fcf7ae355bd9d9ed2e6db492595f1a67f61681be/zipp-3.18.2-py3-none-any.whl", + "sha256": "dce197b859eb796242b0622af1b8beb0a722d52aa2f57133ead08edd5bf5374e", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_python+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_python+", + "pypi__build", + "rules_python++config+pypi__build" + ], + [ + "rules_python+", + "pypi__click", + "rules_python++config+pypi__click" + ], + [ + "rules_python+", + "pypi__colorama", + "rules_python++config+pypi__colorama" + ], + [ + "rules_python+", + "pypi__importlib_metadata", + "rules_python++config+pypi__importlib_metadata" + ], + [ + "rules_python+", + "pypi__installer", + "rules_python++config+pypi__installer" + ], + [ + "rules_python+", + "pypi__more_itertools", + "rules_python++config+pypi__more_itertools" + ], + [ + "rules_python+", + "pypi__packaging", + "rules_python++config+pypi__packaging" + ], + [ + "rules_python+", + "pypi__pep517", + "rules_python++config+pypi__pep517" + ], + [ + "rules_python+", + "pypi__pip", + "rules_python++config+pypi__pip" + ], + [ + "rules_python+", + "pypi__pip_tools", + "rules_python++config+pypi__pip_tools" + ], + [ + "rules_python+", + "pypi__pyproject_hooks", + "rules_python++config+pypi__pyproject_hooks" + ], + [ + "rules_python+", + "pypi__setuptools", + "rules_python++config+pypi__setuptools" + ], + [ + "rules_python+", + "pypi__tomli", + "rules_python++config+pypi__tomli" + ], + [ + "rules_python+", + "pypi__wheel", + "rules_python++config+pypi__wheel" + ], + [ + "rules_python+", + "pypi__zipp", + "rules_python++config+pypi__zipp" + ] + ] + } + }, + "@@rules_python+//python/uv:uv.bzl%uv": { + "general": { + "bzlTransitiveDigest": "CJBWuaTerWIqLpLWl77vjWFD1RRpXfzC2UGMwEk6eVg=", + "usagesDigest": "gx0mA10eprg2o8cg9R+m/jpqCuVjnTnNzD17TPQdFu0=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "uv_0_6_5_aarch64_apple_darwin": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.6.5", + "platform": "aarch64-apple-darwin", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-aarch64-apple-darwin.tar.gz" + ], + "sha256": "4952c6275778fbc7e5cf6b5739ffd944c47063134d6ea43a7037b0a028ef771e" + } + }, + "uv_0_6_5_aarch64_unknown_linux_gnu": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.6.5", + "platform": "aarch64-unknown-linux-gnu", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-aarch64-unknown-linux-gnu.tar.gz" + ], + "sha256": "ebf0707ab949b948ad14250ed93f968fec954e20d12ce6bc9fce74b0ce7e4900" + } + }, + "uv_0_6_5_powerpc64_unknown_linux_gnu": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.6.5", + "platform": "powerpc64-unknown-linux-gnu", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-powerpc64-unknown-linux-gnu.tar.gz" + ], + "sha256": "1f8fd8dde91258380dbcd4b9fc65f15a7dc3af6cfffefb9a4f99d32c4c67436d" + } + }, + "uv_0_6_5_powerpc64le_unknown_linux_gnu": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.6.5", + "platform": "powerpc64le-unknown-linux-gnu", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-powerpc64le-unknown-linux-gnu.tar.gz" + ], + "sha256": "d2b6e19ca7ecd8d7e0d1ddced965aa91a3c822d07a7f7657743945245c1f627a" + } + }, + "uv_0_6_5_s390x_unknown_linux_gnu": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.6.5", + "platform": "s390x-unknown-linux-gnu", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-s390x-unknown-linux-gnu.tar.gz" + ], + "sha256": "43a3e39f19fd5d098bb8c896b6c0f99a146253c271d78409cf31c2f55f2763d0" + } + }, + "uv_0_6_5_x86_64_apple_darwin": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.6.5", + "platform": "x86_64-apple-darwin", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-x86_64-apple-darwin.tar.gz" + ], + "sha256": "d336d862694e1c36afea55eb15f5df02aa77c04a168b781df515d641e23285f0" + } + }, + "uv_0_6_5_x86_64_pc_windows_msvc": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.6.5", + "platform": "x86_64-pc-windows-msvc", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-x86_64-pc-windows-msvc.zip" + ], + "sha256": "a3e9f0913189c38b804bf6bb3d348af305a8ceabc690be92b6e111245a821640" + } + }, + "uv_0_6_5_x86_64_unknown_linux_gnu": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.6.5", + "platform": "x86_64-unknown-linux-gnu", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-x86_64-unknown-linux-gnu.tar.gz" + ], + "sha256": "8fc9895719a1291ecd193cb86f9282ff3649cef797d29eacc74c4f573aab1e2f" + } + }, + "uv": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_toolchains_repo.bzl%uv_toolchains_repo", + "attributes": { + "toolchain_type": "'@@rules_python+//python/uv:uv_toolchain_type'", + "toolchain_names": [ + "0_6_5_aarch64_apple_darwin", + "0_6_5_aarch64_unknown_linux_gnu", + "0_6_5_powerpc64_unknown_linux_gnu", + "0_6_5_powerpc64le_unknown_linux_gnu", + "0_6_5_s390x_unknown_linux_gnu", + "0_6_5_x86_64_apple_darwin", + "0_6_5_x86_64_pc_windows_msvc", + "0_6_5_x86_64_unknown_linux_gnu" + ], + "toolchain_implementations": { + "0_6_5_aarch64_apple_darwin": "@uv_0_6_5_aarch64_apple_darwin//:uv_toolchain", + "0_6_5_aarch64_unknown_linux_gnu": "@uv_0_6_5_aarch64_unknown_linux_gnu//:uv_toolchain", + "0_6_5_powerpc64_unknown_linux_gnu": "@uv_0_6_5_powerpc64_unknown_linux_gnu//:uv_toolchain", + "0_6_5_powerpc64le_unknown_linux_gnu": "@uv_0_6_5_powerpc64le_unknown_linux_gnu//:uv_toolchain", + "0_6_5_s390x_unknown_linux_gnu": "@uv_0_6_5_s390x_unknown_linux_gnu//:uv_toolchain", + "0_6_5_x86_64_apple_darwin": "@uv_0_6_5_x86_64_apple_darwin//:uv_toolchain", + "0_6_5_x86_64_pc_windows_msvc": "@uv_0_6_5_x86_64_pc_windows_msvc//:uv_toolchain", + "0_6_5_x86_64_unknown_linux_gnu": "@uv_0_6_5_x86_64_unknown_linux_gnu//:uv_toolchain" + }, + "toolchain_compatible_with": { + "0_6_5_aarch64_apple_darwin": [ + "'@@platforms//os:macos'", + "'@@platforms//cpu:aarch64'" + ], + "0_6_5_aarch64_unknown_linux_gnu": [ + "'@@platforms//os:linux'", + "'@@platforms//cpu:aarch64'" + ], + "0_6_5_powerpc64_unknown_linux_gnu": [ + "'@@platforms//os:linux'", + "'@@platforms//cpu:ppc'" + ], + "0_6_5_powerpc64le_unknown_linux_gnu": [ + "'@@platforms//os:linux'", + "'@@platforms//cpu:ppc64le'" + ], + "0_6_5_s390x_unknown_linux_gnu": [ + "'@@platforms//os:linux'", + "'@@platforms//cpu:s390x'" + ], + "0_6_5_x86_64_apple_darwin": [ + "'@@platforms//os:macos'", + "'@@platforms//cpu:x86_64'" + ], + "0_6_5_x86_64_pc_windows_msvc": [ + "'@@platforms//os:windows'", + "'@@platforms//cpu:x86_64'" + ], + "0_6_5_x86_64_unknown_linux_gnu": [ + "'@@platforms//os:linux'", + "'@@platforms//cpu:x86_64'" + ] + }, + "toolchain_target_settings": {} + } + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_python+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_python+", + "platforms", + "platforms" + ] + ] + } + }, + "@@rules_rust+//crate_universe:extensions.bzl%crate": { + "general": { + "bzlTransitiveDigest": "NRu94346iEpwiU1qJJMclq8aGKHipA+LB4vVn84/gec=", + "usagesDigest": "mup6mgSFatk34Q0znMZPpIEOFZyk9v3xf6jRIXEFTxc=", + "recordedFileInputs": { + "@@//zml/tokenizer/hftokenizers/Cargo.lock": "7fb683b6a36181a92d7564c8552b4fc5bad00229a41d520a4536c4e577c2b888", + "@@//zml/tokenizer/hftokenizers/Cargo.toml": "e46e67c2ae6792917dc1259d6ba5db389becc8bafa9fa1911bb3b6cec6017620" + }, + "recordedDirentsInputs": {}, + "envVariables": { + "CARGO_BAZEL_DEBUG": null, + "CARGO_BAZEL_GENERATOR_SHA256": null, + "CARGO_BAZEL_GENERATOR_URL": null, + "CARGO_BAZEL_ISOLATED": null, + "CARGO_BAZEL_REPIN": null, + "CARGO_BAZEL_REPIN_ONLY": null, + "CARGO_BAZEL_TIMEOUT": null, + "REPIN": null + }, + "generatedRepoSpecs": { + "crates": { + "repoRuleId": "@@rules_rust+//crate_universe:extensions.bzl%_generate_repo", + "attributes": { + "contents": { + "BUILD.bazel": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files(\n [\n \"cargo-bazel.json\",\n \"crates.bzl\",\n \"defs.bzl\",\n ] + glob(\n allow_empty = True,\n include = [\"*.bazel\"],\n ),\n)\n\nfilegroup(\n name = \"srcs\",\n srcs = glob(\n allow_empty = True,\n include = [\n \"*.bazel\",\n \"*.bzl\",\n ],\n ),\n)\n\n# Workspace Member Dependencies\nalias(\n name = \"tokenizers-0.22.2\",\n actual = \"@crates__tokenizers-0.22.2//:tokenizers\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"tokenizers\",\n actual = \"@crates__tokenizers-0.22.2//:tokenizers\",\n tags = [\"manual\"],\n)\n", + "alias_rules.bzl": "\"\"\"Alias that transitions its target to `compilation_mode=opt`. Use `transition_alias=\"opt\"` to enable.\"\"\"\n\nload(\"@rules_cc//cc:defs.bzl\", \"CcInfo\")\nload(\"@rules_rust//rust:rust_common.bzl\", \"COMMON_PROVIDERS\")\n\ndef _transition_alias_impl(ctx):\n # `ctx.attr.actual` is a list of 1 item due to the transition\n providers = [ctx.attr.actual[0][provider] for provider in COMMON_PROVIDERS]\n if CcInfo in ctx.attr.actual[0]:\n providers.append(ctx.attr.actual[0][CcInfo])\n return providers\n\ndef _change_compilation_mode(compilation_mode):\n def _change_compilation_mode_impl(_settings, _attr):\n return {\n \"//command_line_option:compilation_mode\": compilation_mode,\n }\n\n return transition(\n implementation = _change_compilation_mode_impl,\n inputs = [],\n outputs = [\n \"//command_line_option:compilation_mode\",\n ],\n )\n\ndef _transition_alias_rule(compilation_mode):\n return rule(\n implementation = _transition_alias_impl,\n provides = COMMON_PROVIDERS,\n attrs = {\n \"actual\": attr.label(\n mandatory = True,\n doc = \"`rust_library()` target to transition to `compilation_mode=opt`.\",\n providers = COMMON_PROVIDERS,\n cfg = _change_compilation_mode(compilation_mode),\n ),\n \"_allowlist_function_transition\": attr.label(\n default = \"@bazel_tools//tools/allowlists/function_transition_allowlist\",\n ),\n },\n doc = \"Transitions a Rust library crate to the `compilation_mode=opt`.\",\n )\n\ntransition_alias_dbg = _transition_alias_rule(\"dbg\")\ntransition_alias_fastbuild = _transition_alias_rule(\"fastbuild\")\ntransition_alias_opt = _transition_alias_rule(\"opt\")\n", + "defs.bzl": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\"\"\"\n# `crates_repository` API\n\n- [aliases](#aliases)\n- [crate_deps](#crate_deps)\n- [all_crate_deps](#all_crate_deps)\n- [crate_repositories](#crate_repositories)\n\n\"\"\"\n\nload(\"@bazel_tools//tools/build_defs/repo:git.bzl\", \"new_git_repository\")\nload(\"@bazel_tools//tools/build_defs/repo:http.bzl\", \"http_archive\")\nload(\"@bazel_tools//tools/build_defs/repo:utils.bzl\", \"maybe\")\nload(\"@bazel_skylib//lib:selects.bzl\", \"selects\")\nload(\"@rules_rust//crate_universe/private:local_crate_mirror.bzl\", \"local_crate_mirror\")\n\n###############################################################################\n# MACROS API\n###############################################################################\n\n# An identifier that represent common dependencies (unconditional).\n_COMMON_CONDITION = \"\"\n\ndef _flatten_dependency_maps(all_dependency_maps):\n \"\"\"Flatten a list of dependency maps into one dictionary.\n\n Dependency maps have the following structure:\n\n ```python\n DEPENDENCIES_MAP = {\n # The first key in the map is a Bazel package\n # name of the workspace this file is defined in.\n \"workspace_member_package\": {\n\n # Not all dependencies are supported for all platforms.\n # the condition key is the condition required to be true\n # on the host platform.\n \"condition\": {\n\n # An alias to a crate target. # The label of the crate target the\n # Aliases are only crate names. # package name refers to.\n \"package_name\": \"@full//:label\",\n }\n }\n }\n ```\n\n Args:\n all_dependency_maps (list): A list of dicts as described above\n\n Returns:\n dict: A dictionary as described above\n \"\"\"\n dependencies = {}\n\n for workspace_deps_map in all_dependency_maps:\n for pkg_name, conditional_deps_map in workspace_deps_map.items():\n if pkg_name not in dependencies:\n non_frozen_map = dict()\n for key, values in conditional_deps_map.items():\n non_frozen_map.update({key: dict(values.items())})\n dependencies.setdefault(pkg_name, non_frozen_map)\n continue\n\n for condition, deps_map in conditional_deps_map.items():\n # If the condition has not been recorded, do so and continue\n if condition not in dependencies[pkg_name]:\n dependencies[pkg_name].setdefault(condition, dict(deps_map.items()))\n continue\n\n # Alert on any miss-matched dependencies\n inconsistent_entries = []\n for crate_name, crate_label in deps_map.items():\n existing = dependencies[pkg_name][condition].get(crate_name)\n if existing and existing != crate_label:\n inconsistent_entries.append((crate_name, existing, crate_label))\n dependencies[pkg_name][condition].update({crate_name: crate_label})\n\n return dependencies\n\ndef crate_deps(deps, package_name = None):\n \"\"\"Finds the fully qualified label of the requested crates for the package where this macro is called.\n\n Args:\n deps (list): The desired list of crate targets.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()`.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if not deps:\n return []\n\n if package_name == None:\n package_name = native.package_name()\n\n # Join both sets of dependencies\n dependencies = _flatten_dependency_maps([\n _NORMAL_DEPENDENCIES,\n _NORMAL_DEV_DEPENDENCIES,\n _PROC_MACRO_DEPENDENCIES,\n _PROC_MACRO_DEV_DEPENDENCIES,\n _BUILD_DEPENDENCIES,\n _BUILD_PROC_MACRO_DEPENDENCIES,\n ]).pop(package_name, {})\n\n # Combine all conditional packages so we can easily index over a flat list\n # TODO: Perhaps this should actually return select statements and maintain\n # the conditionals of the dependencies\n flat_deps = {}\n for deps_set in dependencies.values():\n for crate_name, crate_label in deps_set.items():\n flat_deps.update({crate_name: crate_label})\n\n missing_crates = []\n crate_targets = []\n for crate_target in deps:\n if crate_target not in flat_deps:\n missing_crates.append(crate_target)\n else:\n crate_targets.append(flat_deps[crate_target])\n\n if missing_crates:\n fail(\"Could not find crates `{}` among dependencies of `{}`. Available dependencies were `{}`\".format(\n missing_crates,\n package_name,\n dependencies,\n ))\n\n return crate_targets\n\ndef all_crate_deps(\n normal = False, \n normal_dev = False, \n proc_macro = False, \n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Finds the fully qualified label of all requested direct crate dependencies \\\n for the package where this macro is called.\n\n If no parameters are set, all normal dependencies are returned. Setting any one flag will\n otherwise impact the contents of the returned list.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_dependency_maps = []\n if normal:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n if normal_dev:\n all_dependency_maps.append(_NORMAL_DEV_DEPENDENCIES)\n if proc_macro:\n all_dependency_maps.append(_PROC_MACRO_DEPENDENCIES)\n if proc_macro_dev:\n all_dependency_maps.append(_PROC_MACRO_DEV_DEPENDENCIES)\n if build:\n all_dependency_maps.append(_BUILD_DEPENDENCIES)\n if build_proc_macro:\n all_dependency_maps.append(_BUILD_PROC_MACRO_DEPENDENCIES)\n\n # Default to always using normal dependencies\n if not all_dependency_maps:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n\n dependencies = _flatten_dependency_maps(all_dependency_maps).pop(package_name, None)\n\n if not dependencies:\n if dependencies == None:\n fail(\"Tried to get all_crate_deps for package \" + package_name + \" but that package had no Cargo.toml file\")\n else:\n return []\n\n crate_deps = list(dependencies.pop(_COMMON_CONDITION, {}).values())\n for condition, deps in dependencies.items():\n crate_deps += selects.with_or({\n tuple(_CONDITIONS[condition]): deps.values(),\n \"//conditions:default\": [],\n })\n\n return crate_deps\n\ndef aliases(\n normal = False,\n normal_dev = False,\n proc_macro = False,\n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Produces a map of Crate alias names to their original label\n\n If no dependency kinds are specified, `normal` and `proc_macro` are used by default.\n Setting any one flag will otherwise determine the contents of the returned dict.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n dict: The aliases of all associated packages\n \"\"\"\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_aliases_maps = []\n if normal:\n all_aliases_maps.append(_NORMAL_ALIASES)\n if normal_dev:\n all_aliases_maps.append(_NORMAL_DEV_ALIASES)\n if proc_macro:\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n if proc_macro_dev:\n all_aliases_maps.append(_PROC_MACRO_DEV_ALIASES)\n if build:\n all_aliases_maps.append(_BUILD_ALIASES)\n if build_proc_macro:\n all_aliases_maps.append(_BUILD_PROC_MACRO_ALIASES)\n\n # Default to always using normal aliases\n if not all_aliases_maps:\n all_aliases_maps.append(_NORMAL_ALIASES)\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n\n aliases = _flatten_dependency_maps(all_aliases_maps).pop(package_name, None)\n\n if not aliases:\n return dict()\n\n common_items = aliases.pop(_COMMON_CONDITION, {}).items()\n\n # If there are only common items in the dictionary, immediately return them\n if not len(aliases.keys()) == 1:\n return dict(common_items)\n\n # Build a single select statement where each conditional has accounted for the\n # common set of aliases.\n crate_aliases = {\"//conditions:default\": dict(common_items)}\n for condition, deps in aliases.items():\n condition_triples = _CONDITIONS[condition]\n for triple in condition_triples:\n if triple in crate_aliases:\n crate_aliases[triple].update(deps)\n else:\n crate_aliases.update({triple: dict(deps.items() + common_items)})\n\n return select(crate_aliases)\n\n###############################################################################\n# WORKSPACE MEMBER DEPS AND ALIASES\n###############################################################################\n\n_NORMAL_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n _COMMON_CONDITION: {\n \"tokenizers\": Label(\"@crates//:tokenizers-0.22.2\"),\n },\n },\n}\n\n\n_NORMAL_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n _COMMON_CONDITION: {\n },\n },\n}\n\n\n_NORMAL_DEV_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_NORMAL_DEV_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEV_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEV_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_CONDITIONS = {\n \"aarch64-apple-darwin\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\"],\n \"aarch64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\"],\n \"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\",\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\": [],\n \"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\": [],\n \"cfg(any())\": [],\n \"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\": [],\n \"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\": [],\n \"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\": [],\n \"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\",\"@rules_rust//rust/platform:x86_64-apple-darwin\"],\n \"cfg(not(all(target_arch = \\\"arm\\\", target_os = \\\"none\\\")))\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\",\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\",\"@rules_rust//rust/platform:x86_64-apple-darwin\",\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"cfg(target_os = \\\"netbsd\\\")\": [],\n \"cfg(target_os = \\\"solaris\\\")\": [],\n \"cfg(target_os = \\\"vxworks\\\")\": [],\n \"cfg(windows)\": [],\n \"x86_64-apple-darwin\": [\"@rules_rust//rust/platform:x86_64-apple-darwin\"],\n \"x86_64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n}\n\n###############################################################################\n\ndef crate_repositories():\n \"\"\"A macro for defining repositories for all generated crates.\n\n Returns:\n A list of repos visible to the module through the module extension.\n \"\"\"\n maybe(\n http_archive,\n name = \"crates__ahash-0.8.12\",\n sha256 = \"5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ahash/0.8.12/download\"],\n strip_prefix = \"ahash-0.8.12\",\n build_file = Label(\"@crates//crates:BUILD.ahash-0.8.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__aho-corasick-1.1.3\",\n sha256 = \"8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/aho-corasick/1.1.3/download\"],\n strip_prefix = \"aho-corasick-1.1.3\",\n build_file = Label(\"@crates//crates:BUILD.aho-corasick-1.1.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__base64-0.13.1\",\n sha256 = \"9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/base64/0.13.1/download\"],\n strip_prefix = \"base64-0.13.1\",\n build_file = Label(\"@crates//crates:BUILD.base64-0.13.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__bitflags-2.9.1\",\n sha256 = \"1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/bitflags/2.9.1/download\"],\n strip_prefix = \"bitflags-2.9.1\",\n build_file = Label(\"@crates//crates:BUILD.bitflags-2.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__byteorder-1.5.0\",\n sha256 = \"1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/byteorder/1.5.0/download\"],\n strip_prefix = \"byteorder-1.5.0\",\n build_file = Label(\"@crates//crates:BUILD.byteorder-1.5.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__castaway-0.2.4\",\n sha256 = \"dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/castaway/0.2.4/download\"],\n strip_prefix = \"castaway-0.2.4\",\n build_file = Label(\"@crates//crates:BUILD.castaway-0.2.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__cc-1.2.33\",\n sha256 = \"3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/cc/1.2.33/download\"],\n strip_prefix = \"cc-1.2.33\",\n build_file = Label(\"@crates//crates:BUILD.cc-1.2.33.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__cfg-if-1.0.0\",\n sha256 = \"baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/cfg-if/1.0.0/download\"],\n strip_prefix = \"cfg-if-1.0.0\",\n build_file = Label(\"@crates//crates:BUILD.cfg-if-1.0.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__compact_str-0.9.0\",\n sha256 = \"3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/compact_str/0.9.0/download\"],\n strip_prefix = \"compact_str-0.9.0\",\n build_file = Label(\"@crates//crates:BUILD.compact_str-0.9.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-deque-0.8.6\",\n sha256 = \"9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-deque/0.8.6/download\"],\n strip_prefix = \"crossbeam-deque-0.8.6\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-deque-0.8.6.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-epoch-0.9.18\",\n sha256 = \"5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-epoch/0.9.18/download\"],\n strip_prefix = \"crossbeam-epoch-0.9.18\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-epoch-0.9.18.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-utils-0.8.21\",\n sha256 = \"d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-utils/0.8.21/download\"],\n strip_prefix = \"crossbeam-utils-0.8.21\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-utils-0.8.21.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling-0.20.10\",\n sha256 = \"6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling/0.20.10/download\"],\n strip_prefix = \"darling-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling_core-0.20.10\",\n sha256 = \"95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling_core/0.20.10/download\"],\n strip_prefix = \"darling_core-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling_core-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling_macro-0.20.10\",\n sha256 = \"d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling_macro/0.20.10/download\"],\n strip_prefix = \"darling_macro-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling_macro-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__dary_heap-0.3.7\",\n sha256 = \"04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/dary_heap/0.3.7/download\"],\n strip_prefix = \"dary_heap-0.3.7\",\n build_file = Label(\"@crates//crates:BUILD.dary_heap-0.3.7.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder-0.20.2\",\n sha256 = \"507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder/0.20.2/download\"],\n strip_prefix = \"derive_builder-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder_core-0.20.2\",\n sha256 = \"2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder_core/0.20.2/download\"],\n strip_prefix = \"derive_builder_core-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder_core-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder_macro-0.20.2\",\n sha256 = \"ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder_macro/0.20.2/download\"],\n strip_prefix = \"derive_builder_macro-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder_macro-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__either-1.13.0\",\n sha256 = \"60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/either/1.13.0/download\"],\n strip_prefix = \"either-1.13.0\",\n build_file = Label(\"@crates//crates:BUILD.either-1.13.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__esaxx-rs-0.1.10\",\n sha256 = \"d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/esaxx-rs/0.1.10/download\"],\n strip_prefix = \"esaxx-rs-0.1.10\",\n build_file = Label(\"@crates//crates:BUILD.esaxx-rs-0.1.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__fnv-1.0.7\",\n sha256 = \"3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/fnv/1.0.7/download\"],\n strip_prefix = \"fnv-1.0.7\",\n build_file = Label(\"@crates//crates:BUILD.fnv-1.0.7.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__getrandom-0.3.3\",\n sha256 = \"26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/getrandom/0.3.3/download\"],\n strip_prefix = \"getrandom-0.3.3\",\n build_file = Label(\"@crates//crates:BUILD.getrandom-0.3.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ident_case-1.0.1\",\n sha256 = \"b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ident_case/1.0.1/download\"],\n strip_prefix = \"ident_case-1.0.1\",\n build_file = Label(\"@crates//crates:BUILD.ident_case-1.0.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__itertools-0.14.0\",\n sha256 = \"2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/itertools/0.14.0/download\"],\n strip_prefix = \"itertools-0.14.0\",\n build_file = Label(\"@crates//crates:BUILD.itertools-0.14.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__itoa-1.0.14\",\n sha256 = \"d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/itoa/1.0.14/download\"],\n strip_prefix = \"itoa-1.0.14\",\n build_file = Label(\"@crates//crates:BUILD.itoa-1.0.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__libc-0.2.169\",\n sha256 = \"b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/libc/0.2.169/download\"],\n strip_prefix = \"libc-0.2.169\",\n build_file = Label(\"@crates//crates:BUILD.libc-0.2.169.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__log-0.4.22\",\n sha256 = \"a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/log/0.4.22/download\"],\n strip_prefix = \"log-0.4.22\",\n build_file = Label(\"@crates//crates:BUILD.log-0.4.22.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__macro_rules_attribute-0.2.0\",\n sha256 = \"8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/macro_rules_attribute/0.2.0/download\"],\n strip_prefix = \"macro_rules_attribute-0.2.0\",\n build_file = Label(\"@crates//crates:BUILD.macro_rules_attribute-0.2.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__macro_rules_attribute-proc_macro-0.2.0\",\n sha256 = \"b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/macro_rules_attribute-proc_macro/0.2.0/download\"],\n strip_prefix = \"macro_rules_attribute-proc_macro-0.2.0\",\n build_file = Label(\"@crates//crates:BUILD.macro_rules_attribute-proc_macro-0.2.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__memchr-2.7.4\",\n sha256 = \"78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/memchr/2.7.4/download\"],\n strip_prefix = \"memchr-2.7.4\",\n build_file = Label(\"@crates//crates:BUILD.memchr-2.7.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__minimal-lexical-0.2.1\",\n sha256 = \"68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/minimal-lexical/0.2.1/download\"],\n strip_prefix = \"minimal-lexical-0.2.1\",\n build_file = Label(\"@crates//crates:BUILD.minimal-lexical-0.2.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__monostate-0.1.13\",\n sha256 = \"0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/monostate/0.1.13/download\"],\n strip_prefix = \"monostate-0.1.13\",\n build_file = Label(\"@crates//crates:BUILD.monostate-0.1.13.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__monostate-impl-0.1.13\",\n sha256 = \"a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/monostate-impl/0.1.13/download\"],\n strip_prefix = \"monostate-impl-0.1.13\",\n build_file = Label(\"@crates//crates:BUILD.monostate-impl-0.1.13.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__nom-7.1.3\",\n sha256 = \"d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/nom/7.1.3/download\"],\n strip_prefix = \"nom-7.1.3\",\n build_file = Label(\"@crates//crates:BUILD.nom-7.1.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__once_cell-1.20.2\",\n sha256 = \"1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/once_cell/1.20.2/download\"],\n strip_prefix = \"once_cell-1.20.2\",\n build_file = Label(\"@crates//crates:BUILD.once_cell-1.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__onig-6.5.1\",\n sha256 = \"336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/onig/6.5.1/download\"],\n strip_prefix = \"onig-6.5.1\",\n build_file = Label(\"@crates//crates:BUILD.onig-6.5.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__onig_sys-69.9.1\",\n sha256 = \"c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/onig_sys/69.9.1/download\"],\n strip_prefix = \"onig_sys-69.9.1\",\n build_file = Label(\"@crates//crates:BUILD.onig_sys-69.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__paste-1.0.15\",\n sha256 = \"57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/paste/1.0.15/download\"],\n strip_prefix = \"paste-1.0.15\",\n build_file = Label(\"@crates//crates:BUILD.paste-1.0.15.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__pkg-config-0.3.32\",\n sha256 = \"7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/pkg-config/0.3.32/download\"],\n strip_prefix = \"pkg-config-0.3.32\",\n build_file = Label(\"@crates//crates:BUILD.pkg-config-0.3.32.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ppv-lite86-0.2.20\",\n sha256 = \"77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ppv-lite86/0.2.20/download\"],\n strip_prefix = \"ppv-lite86-0.2.20\",\n build_file = Label(\"@crates//crates:BUILD.ppv-lite86-0.2.20.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__proc-macro2-1.0.92\",\n sha256 = \"37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/proc-macro2/1.0.92/download\"],\n strip_prefix = \"proc-macro2-1.0.92\",\n build_file = Label(\"@crates//crates:BUILD.proc-macro2-1.0.92.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__quote-1.0.37\",\n sha256 = \"b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/quote/1.0.37/download\"],\n strip_prefix = \"quote-1.0.37\",\n build_file = Label(\"@crates//crates:BUILD.quote-1.0.37.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__r-efi-5.3.0\",\n sha256 = \"69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/r-efi/5.3.0/download\"],\n strip_prefix = \"r-efi-5.3.0\",\n build_file = Label(\"@crates//crates:BUILD.r-efi-5.3.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand-0.9.1\",\n sha256 = \"9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand/0.9.1/download\"],\n strip_prefix = \"rand-0.9.1\",\n build_file = Label(\"@crates//crates:BUILD.rand-0.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand_chacha-0.9.0\",\n sha256 = \"d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand_chacha/0.9.0/download\"],\n strip_prefix = \"rand_chacha-0.9.0\",\n build_file = Label(\"@crates//crates:BUILD.rand_chacha-0.9.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand_core-0.9.3\",\n sha256 = \"99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand_core/0.9.3/download\"],\n strip_prefix = \"rand_core-0.9.3\",\n build_file = Label(\"@crates//crates:BUILD.rand_core-0.9.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-1.10.0\",\n sha256 = \"b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon/1.10.0/download\"],\n strip_prefix = \"rayon-1.10.0\",\n build_file = Label(\"@crates//crates:BUILD.rayon-1.10.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-cond-0.4.0\",\n sha256 = \"2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon-cond/0.4.0/download\"],\n strip_prefix = \"rayon-cond-0.4.0\",\n build_file = Label(\"@crates//crates:BUILD.rayon-cond-0.4.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-core-1.12.1\",\n sha256 = \"1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon-core/1.12.1/download\"],\n strip_prefix = \"rayon-core-1.12.1\",\n build_file = Label(\"@crates//crates:BUILD.rayon-core-1.12.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-1.11.1\",\n sha256 = \"b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex/1.11.1/download\"],\n strip_prefix = \"regex-1.11.1\",\n build_file = Label(\"@crates//crates:BUILD.regex-1.11.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-automata-0.4.9\",\n sha256 = \"809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-automata/0.4.9/download\"],\n strip_prefix = \"regex-automata-0.4.9\",\n build_file = Label(\"@crates//crates:BUILD.regex-automata-0.4.9.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-syntax-0.8.5\",\n sha256 = \"2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-syntax/0.8.5/download\"],\n strip_prefix = \"regex-syntax-0.8.5\",\n build_file = Label(\"@crates//crates:BUILD.regex-syntax-0.8.5.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rustversion-1.0.21\",\n sha256 = \"8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rustversion/1.0.21/download\"],\n strip_prefix = \"rustversion-1.0.21\",\n build_file = Label(\"@crates//crates:BUILD.rustversion-1.0.21.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ryu-1.0.18\",\n sha256 = \"f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ryu/1.0.18/download\"],\n strip_prefix = \"ryu-1.0.18\",\n build_file = Label(\"@crates//crates:BUILD.ryu-1.0.18.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde-1.0.216\",\n sha256 = \"0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde/1.0.216/download\"],\n strip_prefix = \"serde-1.0.216\",\n build_file = Label(\"@crates//crates:BUILD.serde-1.0.216.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde_derive-1.0.216\",\n sha256 = \"46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde_derive/1.0.216/download\"],\n strip_prefix = \"serde_derive-1.0.216\",\n build_file = Label(\"@crates//crates:BUILD.serde_derive-1.0.216.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde_json-1.0.134\",\n sha256 = \"d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde_json/1.0.134/download\"],\n strip_prefix = \"serde_json-1.0.134\",\n build_file = Label(\"@crates//crates:BUILD.serde_json-1.0.134.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__shlex-1.3.0\",\n sha256 = \"0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/shlex/1.3.0/download\"],\n strip_prefix = \"shlex-1.3.0\",\n build_file = Label(\"@crates//crates:BUILD.shlex-1.3.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__smallvec-1.13.2\",\n sha256 = \"3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/smallvec/1.13.2/download\"],\n strip_prefix = \"smallvec-1.13.2\",\n build_file = Label(\"@crates//crates:BUILD.smallvec-1.13.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__spm_precompiled-0.1.4\",\n sha256 = \"5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/spm_precompiled/0.1.4/download\"],\n strip_prefix = \"spm_precompiled-0.1.4\",\n build_file = Label(\"@crates//crates:BUILD.spm_precompiled-0.1.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__static_assertions-1.1.0\",\n sha256 = \"a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/static_assertions/1.1.0/download\"],\n strip_prefix = \"static_assertions-1.1.0\",\n build_file = Label(\"@crates//crates:BUILD.static_assertions-1.1.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__strsim-0.11.1\",\n sha256 = \"7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/strsim/0.11.1/download\"],\n strip_prefix = \"strsim-0.11.1\",\n build_file = Label(\"@crates//crates:BUILD.strsim-0.11.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__syn-2.0.91\",\n sha256 = \"d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/syn/2.0.91/download\"],\n strip_prefix = \"syn-2.0.91\",\n build_file = Label(\"@crates//crates:BUILD.syn-2.0.91.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__thiserror-2.0.12\",\n sha256 = \"567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/thiserror/2.0.12/download\"],\n strip_prefix = \"thiserror-2.0.12\",\n build_file = Label(\"@crates//crates:BUILD.thiserror-2.0.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__thiserror-impl-2.0.12\",\n sha256 = \"7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/thiserror-impl/2.0.12/download\"],\n strip_prefix = \"thiserror-impl-2.0.12\",\n build_file = Label(\"@crates//crates:BUILD.thiserror-impl-2.0.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__tokenizers-0.22.2\",\n sha256 = \"b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/tokenizers/0.22.2/download\"],\n strip_prefix = \"tokenizers-0.22.2\",\n build_file = Label(\"@crates//crates:BUILD.tokenizers-0.22.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-ident-1.0.14\",\n sha256 = \"adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-ident/1.0.14/download\"],\n strip_prefix = \"unicode-ident-1.0.14\",\n build_file = Label(\"@crates//crates:BUILD.unicode-ident-1.0.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-normalization-alignments-0.1.12\",\n sha256 = \"43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-normalization-alignments/0.1.12/download\"],\n strip_prefix = \"unicode-normalization-alignments-0.1.12\",\n build_file = Label(\"@crates//crates:BUILD.unicode-normalization-alignments-0.1.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-segmentation-1.12.0\",\n sha256 = \"f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-segmentation/1.12.0/download\"],\n strip_prefix = \"unicode-segmentation-1.12.0\",\n build_file = Label(\"@crates//crates:BUILD.unicode-segmentation-1.12.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode_categories-0.1.1\",\n sha256 = \"39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode_categories/0.1.1/download\"],\n strip_prefix = \"unicode_categories-0.1.1\",\n build_file = Label(\"@crates//crates:BUILD.unicode_categories-0.1.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__version_check-0.9.5\",\n sha256 = \"0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/version_check/0.9.5/download\"],\n strip_prefix = \"version_check-0.9.5\",\n build_file = Label(\"@crates//crates:BUILD.version_check-0.9.5.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasi-0.14.2-wasi-0.2.4\",\n sha256 = \"9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasi/0.14.2+wasi-0.2.4/download\"],\n strip_prefix = \"wasi-0.14.2+wasi-0.2.4\",\n build_file = Label(\"@crates//crates:BUILD.wasi-0.14.2+wasi-0.2.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wit-bindgen-rt-0.39.0\",\n sha256 = \"6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wit-bindgen-rt/0.39.0/download\"],\n strip_prefix = \"wit-bindgen-rt-0.39.0\",\n build_file = Label(\"@crates//crates:BUILD.wit-bindgen-rt-0.39.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-0.7.35\",\n sha256 = \"1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy/0.7.35/download\"],\n strip_prefix = \"zerocopy-0.7.35\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-0.7.35.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-0.8.26\",\n sha256 = \"1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy/0.8.26/download\"],\n strip_prefix = \"zerocopy-0.8.26\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-0.8.26.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-derive-0.7.35\",\n sha256 = \"fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy-derive/0.7.35/download\"],\n strip_prefix = \"zerocopy-derive-0.7.35\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-derive-0.7.35.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-derive-0.8.26\",\n sha256 = \"9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy-derive/0.8.26/download\"],\n strip_prefix = \"zerocopy-derive-0.8.26\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-derive-0.8.26.bazel\"),\n )\n\n return [\n struct(repo=\"crates__tokenizers-0.22.2\", is_dev_dep = False),\n ]\n" + } + } + }, + "crates__ahash-0.8.12": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/ahash/0.8.12/download" + ], + "strip_prefix": "ahash-0.8.12", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"ahash\",\n deps = [\n \"@crates__ahash-0.8.12//:build_script_build\",\n \"@crates__cfg-if-1.0.0//:cfg_if\",\n \"@crates__getrandom-0.3.3//:getrandom\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__zerocopy-0.8.26//:zerocopy\",\n ] + select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [\n \"@crates__once_cell-1.20.2//:once_cell\", # cfg(not(all(target_arch = \"arm\", target_os = \"none\")))\n ],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [\n \"@crates__once_cell-1.20.2//:once_cell\", # cfg(not(all(target_arch = \"arm\", target_os = \"none\")))\n ],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [\n \"@crates__once_cell-1.20.2//:once_cell\", # cfg(not(all(target_arch = \"arm\", target_os = \"none\")))\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [\n \"@crates__once_cell-1.20.2//:once_cell\", # cfg(not(all(target_arch = \"arm\", target_os = \"none\")))\n ],\n \"//conditions:default\": [],\n }),\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"getrandom\",\n \"runtime-rng\",\n \"serde\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=ahash\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.12\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"getrandom\",\n \"runtime-rng\",\n \"serde\",\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n deps = [\n \"@crates__version_check-0.9.5//:version_check\",\n ],\n edition = \"2018\",\n pkg_name = \"ahash\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=ahash\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.8.12\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + } + }, + "crates__aho-corasick-1.1.3": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/aho-corasick/1.1.3/download" + ], + "strip_prefix": "aho-corasick-1.1.3", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"aho_corasick\",\n deps = [\n \"@crates__memchr-2.7.4//:memchr\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"perf-literal\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=aho-corasick\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.1.3\",\n)\n" + } + }, + "crates__base64-0.13.1": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/base64/0.13.1/download" + ], + "strip_prefix": "base64-0.13.1", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"base64\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=base64\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.13.1\",\n)\n" + } + }, + "crates__bitflags-2.9.1": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/bitflags/2.9.1/download" + ], + "strip_prefix": "bitflags-2.9.1", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"bitflags\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=bitflags\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.9.1\",\n)\n" + } + }, + "crates__byteorder-1.5.0": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/byteorder/1.5.0/download" + ], + "strip_prefix": "byteorder-1.5.0", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"byteorder\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=byteorder\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.5.0\",\n)\n" + } + }, + "crates__castaway-0.2.4": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/castaway/0.2.4/download" + ], + "strip_prefix": "castaway-0.2.4", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"castaway\",\n proc_macro_deps = [\n \"@crates__rustversion-1.0.21//:rustversion\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=castaway\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.4\",\n)\n" + } + }, + "crates__cc-1.2.33": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/cc/1.2.33/download" + ], + "strip_prefix": "cc-1.2.33", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"cc\",\n deps = [\n \"@crates__shlex-1.3.0//:shlex\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=cc\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.2.33\",\n)\n" + } + }, + "crates__cfg-if-1.0.0": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/cfg-if/1.0.0/download" + ], + "strip_prefix": "cfg-if-1.0.0", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"cfg_if\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=cfg-if\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.0\",\n)\n" + } + }, + "crates__compact_str-0.9.0": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/compact_str/0.9.0/download" + ], + "strip_prefix": "compact_str-0.9.0", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"compact_str\",\n deps = [\n \"@crates__castaway-0.2.4//:castaway\",\n \"@crates__cfg-if-1.0.0//:cfg_if\",\n \"@crates__itoa-1.0.14//:itoa\",\n \"@crates__ryu-1.0.18//:ryu\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__static_assertions-1.1.0//:static_assertions\",\n ],\n proc_macro_deps = [\n \"@crates__rustversion-1.0.21//:rustversion\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"serde\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=compact_str\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.9.0\",\n)\n" + } + }, + "crates__crossbeam-deque-0.8.6": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/crossbeam-deque/0.8.6/download" + ], + "strip_prefix": "crossbeam-deque-0.8.6", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"crossbeam_deque\",\n deps = [\n \"@crates__crossbeam-epoch-0.9.18//:crossbeam_epoch\",\n \"@crates__crossbeam-utils-0.8.21//:crossbeam_utils\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=crossbeam-deque\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.6\",\n)\n" + } + }, + "crates__crossbeam-epoch-0.9.18": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/crossbeam-epoch/0.9.18/download" + ], + "strip_prefix": "crossbeam-epoch-0.9.18", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"crossbeam_epoch\",\n deps = [\n \"@crates__crossbeam-utils-0.8.21//:crossbeam_utils\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=crossbeam-epoch\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.9.18\",\n)\n" + } + }, + "crates__crossbeam-utils-0.8.21": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/crossbeam-utils/0.8.21/download" + ], + "strip_prefix": "crossbeam-utils-0.8.21", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"crossbeam_utils\",\n deps = [\n \"@crates__crossbeam-utils-0.8.21//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=crossbeam-utils\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.21\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"crossbeam-utils\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=crossbeam-utils\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.8.21\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + } + }, + "crates__darling-0.20.10": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/darling/0.20.10/download" + ], + "strip_prefix": "darling-0.20.10", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"darling\",\n deps = [\n \"@crates__darling_core-0.20.10//:darling_core\",\n ],\n proc_macro_deps = [\n \"@crates__darling_macro-0.20.10//:darling_macro\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"suggestions\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=darling\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.20.10\",\n)\n" + } + }, + "crates__darling_core-0.20.10": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/darling_core/0.20.10/download" + ], + "strip_prefix": "darling_core-0.20.10", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"darling_core\",\n deps = [\n \"@crates__fnv-1.0.7//:fnv\",\n \"@crates__ident_case-1.0.1//:ident_case\",\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__strsim-0.11.1//:strsim\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"strsim\",\n \"suggestions\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=darling_core\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.20.10\",\n)\n" + } + }, + "crates__darling_macro-0.20.10": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/darling_macro/0.20.10/download" + ], + "strip_prefix": "darling_macro-0.20.10", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"darling_macro\",\n deps = [\n \"@crates__darling_core-0.20.10//:darling_core\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=darling_macro\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.20.10\",\n)\n" + } + }, + "crates__dary_heap-0.3.7": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/dary_heap/0.3.7/download" + ], + "strip_prefix": "dary_heap-0.3.7", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"dary_heap\",\n deps = [\n \"@crates__serde-1.0.216//:serde\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"serde\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=dary_heap\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.7\",\n)\n" + } + }, + "crates__derive_builder-0.20.2": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/derive_builder/0.20.2/download" + ], + "strip_prefix": "derive_builder-0.20.2", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"derive_builder\",\n proc_macro_deps = [\n \"@crates__derive_builder_macro-0.20.2//:derive_builder_macro\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=derive_builder\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.20.2\",\n)\n" + } + }, + "crates__derive_builder_core-0.20.2": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/derive_builder_core/0.20.2/download" + ], + "strip_prefix": "derive_builder_core-0.20.2", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"derive_builder_core\",\n deps = [\n \"@crates__darling-0.20.10//:darling\",\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"lib_has_std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=derive_builder_core\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.20.2\",\n)\n" + } + }, + "crates__derive_builder_macro-0.20.2": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/derive_builder_macro/0.20.2/download" + ], + "strip_prefix": "derive_builder_macro-0.20.2", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"derive_builder_macro\",\n deps = [\n \"@crates__derive_builder_core-0.20.2//:derive_builder_core\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"lib_has_std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=derive_builder_macro\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.20.2\",\n)\n" + } + }, + "crates__either-1.13.0": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/either/1.13.0/download" + ], + "strip_prefix": "either-1.13.0", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"either\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"use_std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=either\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.13.0\",\n)\n" + } + }, + "crates__esaxx-rs-0.1.10": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/esaxx-rs/0.1.10/download" + ], + "strip_prefix": "esaxx-rs-0.1.10", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"esaxx_rs\",\n deps = [\n \"@crates__esaxx-rs-0.1.10//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"cc\",\n \"cpp\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=esaxx-rs\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.10\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"cc\",\n \"cpp\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n deps = [\n \"@crates__cc-1.2.33//:cc\",\n ],\n edition = \"2018\",\n pkg_name = \"esaxx-rs\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=esaxx-rs\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.1.10\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + } + }, + "crates__fnv-1.0.7": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/fnv/1.0.7/download" + ], + "strip_prefix": "fnv-1.0.7", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"fnv\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=fnv\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.7\",\n)\n" + } + }, + "crates__getrandom-0.3.3": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/getrandom/0.3.3/download" + ], + "strip_prefix": "getrandom-0.3.3", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"getrandom\",\n deps = [\n \"@crates__cfg-if-1.0.0//:cfg_if\",\n \"@crates__getrandom-0.3.3//:build_script_build\",\n ] + select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(any(target_os = \"macos\", target_os = \"openbsd\", target_os = \"vita\", target_os = \"emscripten\"))\n ],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(all(any(target_os = \"linux\", target_os = \"android\"), not(any(all(target_os = \"linux\", target_env = \"\"), getrandom_backend = \"custom\", getrandom_backend = \"linux_raw\", getrandom_backend = \"rdrand\", getrandom_backend = \"rndr\"))))\n ],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(any(target_os = \"macos\", target_os = \"openbsd\", target_os = \"vita\", target_os = \"emscripten\"))\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(all(any(target_os = \"linux\", target_os = \"android\"), not(any(all(target_os = \"linux\", target_env = \"\"), getrandom_backend = \"custom\", getrandom_backend = \"linux_raw\", getrandom_backend = \"rdrand\", getrandom_backend = \"rndr\"))))\n ],\n \"//conditions:default\": [],\n }),\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=getrandom\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.3\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"getrandom\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=getrandom\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.3.3\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + } + }, + "crates__ident_case-1.0.1": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/ident_case/1.0.1/download" + ], + "strip_prefix": "ident_case-1.0.1", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"ident_case\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=ident_case\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.1\",\n)\n" + } + }, + "crates__itertools-0.14.0": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/itertools/0.14.0/download" + ], + "strip_prefix": "itertools-0.14.0", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"itertools\",\n deps = [\n \"@crates__either-1.13.0//:either\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"use_alloc\",\n \"use_std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=itertools\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.14.0\",\n)\n" + } + }, + "crates__itoa-1.0.14": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/itoa/1.0.14/download" + ], + "strip_prefix": "itoa-1.0.14", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"itoa\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=itoa\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.14\",\n)\n" + } + }, + "crates__libc-0.2.169": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/libc/0.2.169/download" + ], + "strip_prefix": "libc-0.2.169", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"libc\",\n deps = [\n \"@crates__libc-0.2.169//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=libc\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.169\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"libc\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=libc\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.2.169\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + } + }, + "crates__log-0.4.22": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/log/0.4.22/download" + ], + "strip_prefix": "log-0.4.22", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"log\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=log\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.4.22\",\n)\n" + } + }, + "crates__macro_rules_attribute-0.2.0": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/macro_rules_attribute/0.2.0/download" + ], + "strip_prefix": "macro_rules_attribute-0.2.0", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"macro_rules_attribute\",\n proc_macro_deps = [\n \"@crates__macro_rules_attribute-proc_macro-0.2.0//:macro_rules_attribute_proc_macro\",\n \"@crates__paste-1.0.15//:paste\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=macro_rules_attribute\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.0\",\n)\n" + } + }, + "crates__macro_rules_attribute-proc_macro-0.2.0": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/macro_rules_attribute-proc_macro/0.2.0/download" + ], + "strip_prefix": "macro_rules_attribute-proc_macro-0.2.0", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"macro_rules_attribute_proc_macro\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"mod.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=macro_rules_attribute-proc_macro\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.0\",\n)\n" + } + }, + "crates__memchr-2.7.4": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/memchr/2.7.4/download" + ], + "strip_prefix": "memchr-2.7.4", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"memchr\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=memchr\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.7.4\",\n)\n" + } + }, + "crates__minimal-lexical-0.2.1": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/minimal-lexical/0.2.1/download" + ], + "strip_prefix": "minimal-lexical-0.2.1", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"minimal_lexical\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=minimal-lexical\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.1\",\n)\n" + } + }, + "crates__monostate-0.1.13": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/monostate/0.1.13/download" + ], + "strip_prefix": "monostate-0.1.13", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"monostate\",\n deps = [\n \"@crates__serde-1.0.216//:serde\",\n ],\n proc_macro_deps = [\n \"@crates__monostate-impl-0.1.13//:monostate_impl\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=monostate\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.13\",\n)\n" + } + }, + "crates__monostate-impl-0.1.13": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/monostate-impl/0.1.13/download" + ], + "strip_prefix": "monostate-impl-0.1.13", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"monostate_impl\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=monostate-impl\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.13\",\n)\n" + } + }, + "crates__nom-7.1.3": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/nom/7.1.3/download" + ], + "strip_prefix": "nom-7.1.3", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"nom\",\n deps = [\n \"@crates__memchr-2.7.4//:memchr\",\n \"@crates__minimal-lexical-0.2.1//:minimal_lexical\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=nom\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"7.1.3\",\n)\n" + } + }, + "crates__once_cell-1.20.2": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/once_cell/1.20.2/download" + ], + "strip_prefix": "once_cell-1.20.2", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"once_cell\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"default\",\n \"race\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=once_cell\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.20.2\",\n)\n" + } + }, + "crates__onig-6.5.1": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/onig/6.5.1/download" + ], + "strip_prefix": "onig-6.5.1", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"onig\",\n deps = [\n \"@crates__bitflags-2.9.1//:bitflags\",\n \"@crates__once_cell-1.20.2//:once_cell\",\n \"@crates__onig_sys-69.9.1//:onig_sys\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=onig\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"6.5.1\",\n)\n" + } + }, + "crates__onig_sys-69.9.1": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/onig_sys/69.9.1/download" + ], + "strip_prefix": "onig_sys-69.9.1", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"onig_sys\",\n deps = [\n \"@crates__onig_sys-69.9.1//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=onig_sys\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"69.9.1\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n deps = [\n \"@crates__cc-1.2.33//:cc\",\n \"@crates__pkg-config-0.3.32//:pkg_config\",\n ],\n edition = \"2021\",\n links = \"onig\",\n pkg_name = \"onig_sys\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=onig_sys\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"69.9.1\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + } + }, + "crates__paste-1.0.15": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/paste/1.0.15/download" + ], + "strip_prefix": "paste-1.0.15", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"paste\",\n deps = [\n \"@crates__paste-1.0.15//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=paste\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.15\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2018\",\n pkg_name = \"paste\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=paste\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.15\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + } + }, + "crates__pkg-config-0.3.32": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/pkg-config/0.3.32/download" + ], + "strip_prefix": "pkg-config-0.3.32", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"pkg_config\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=pkg-config\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.32\",\n)\n" + } + }, + "crates__ppv-lite86-0.2.20": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/ppv-lite86/0.2.20/download" + ], + "strip_prefix": "ppv-lite86-0.2.20", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"ppv_lite86\",\n deps = [\n \"@crates__zerocopy-0.7.35//:zerocopy\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"simd\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=ppv-lite86\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.20\",\n)\n" + } + }, + "crates__proc-macro2-1.0.92": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/proc-macro2/1.0.92/download" + ], + "strip_prefix": "proc-macro2-1.0.92", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"proc_macro2\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:build_script_build\",\n \"@crates__unicode-ident-1.0.14//:unicode_ident\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"proc-macro\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=proc-macro2\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.92\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"proc-macro\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"proc-macro2\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=proc-macro2\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.92\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + } + }, + "crates__quote-1.0.37": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/quote/1.0.37/download" + ], + "strip_prefix": "quote-1.0.37", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"quote\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"proc-macro\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=quote\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.37\",\n)\n" + } + }, + "crates__r-efi-5.3.0": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/r-efi/5.3.0/download" + ], + "strip_prefix": "r-efi-5.3.0", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"r_efi\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=r-efi\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"5.3.0\",\n)\n" + } + }, + "crates__rand-0.9.1": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/rand/0.9.1/download" + ], + "strip_prefix": "rand-0.9.1", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"rand\",\n deps = [\n \"@crates__rand_chacha-0.9.0//:rand_chacha\",\n \"@crates__rand_core-0.9.3//:rand_core\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"default\",\n \"os_rng\",\n \"small_rng\",\n \"std\",\n \"std_rng\",\n \"thread_rng\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rand\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.9.1\",\n)\n" + } + }, + "crates__rand_chacha-0.9.0": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/rand_chacha/0.9.0/download" + ], + "strip_prefix": "rand_chacha-0.9.0", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"rand_chacha\",\n deps = [\n \"@crates__ppv-lite86-0.2.20//:ppv_lite86\",\n \"@crates__rand_core-0.9.3//:rand_core\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rand_chacha\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.9.0\",\n)\n" + } + }, + "crates__rand_core-0.9.3": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/rand_core/0.9.3/download" + ], + "strip_prefix": "rand_core-0.9.3", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"rand_core\",\n deps = [\n \"@crates__getrandom-0.3.3//:getrandom\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"os_rng\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rand_core\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.9.3\",\n)\n" + } + }, + "crates__rayon-1.10.0": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/rayon/1.10.0/download" + ], + "strip_prefix": "rayon-1.10.0", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"rayon\",\n deps = [\n \"@crates__either-1.13.0//:either\",\n \"@crates__rayon-core-1.12.1//:rayon_core\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rayon\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.10.0\",\n)\n" + } + }, + "crates__rayon-cond-0.4.0": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/rayon-cond/0.4.0/download" + ], + "strip_prefix": "rayon-cond-0.4.0", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"rayon_cond\",\n deps = [\n \"@crates__either-1.13.0//:either\",\n \"@crates__itertools-0.14.0//:itertools\",\n \"@crates__rayon-1.10.0//:rayon\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rayon-cond\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.4.0\",\n)\n" + } + }, + "crates__rayon-core-1.12.1": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/rayon-core/1.12.1/download" + ], + "strip_prefix": "rayon-core-1.12.1", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"rayon_core\",\n deps = [\n \"@crates__crossbeam-deque-0.8.6//:crossbeam_deque\",\n \"@crates__crossbeam-utils-0.8.21//:crossbeam_utils\",\n \"@crates__rayon-core-1.12.1//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rayon-core\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.12.1\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n links = \"rayon-core\",\n pkg_name = \"rayon-core\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rayon-core\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.12.1\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + } + }, + "crates__regex-1.11.1": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/regex/1.11.1/download" + ], + "strip_prefix": "regex-1.11.1", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"regex\",\n deps = [\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__memchr-2.7.4//:memchr\",\n \"@crates__regex-automata-0.4.9//:regex_automata\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"perf\",\n \"perf-backtrack\",\n \"perf-cache\",\n \"perf-dfa\",\n \"perf-inline\",\n \"perf-literal\",\n \"perf-onepass\",\n \"std\",\n \"unicode\",\n \"unicode-age\",\n \"unicode-bool\",\n \"unicode-case\",\n \"unicode-gencat\",\n \"unicode-perl\",\n \"unicode-script\",\n \"unicode-segment\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=regex\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.11.1\",\n)\n" + } + }, + "crates__regex-automata-0.4.9": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/regex-automata/0.4.9/download" + ], + "strip_prefix": "regex-automata-0.4.9", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"regex_automata\",\n deps = [\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__memchr-2.7.4//:memchr\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"dfa-onepass\",\n \"hybrid\",\n \"meta\",\n \"nfa-backtrack\",\n \"nfa-pikevm\",\n \"nfa-thompson\",\n \"perf-inline\",\n \"perf-literal\",\n \"perf-literal-multisubstring\",\n \"perf-literal-substring\",\n \"std\",\n \"syntax\",\n \"unicode\",\n \"unicode-age\",\n \"unicode-bool\",\n \"unicode-case\",\n \"unicode-gencat\",\n \"unicode-perl\",\n \"unicode-script\",\n \"unicode-segment\",\n \"unicode-word-boundary\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=regex-automata\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.4.9\",\n)\n" + } + }, + "crates__regex-syntax-0.8.5": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/regex-syntax/0.8.5/download" + ], + "strip_prefix": "regex-syntax-0.8.5", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"regex_syntax\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n \"unicode\",\n \"unicode-age\",\n \"unicode-bool\",\n \"unicode-case\",\n \"unicode-gencat\",\n \"unicode-perl\",\n \"unicode-script\",\n \"unicode-segment\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=regex-syntax\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.5\",\n)\n" + } + }, + "crates__rustversion-1.0.21": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/rustversion/1.0.21/download" + ], + "strip_prefix": "rustversion-1.0.21", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"rustversion\",\n deps = [\n \"@crates__rustversion-1.0.21//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rustversion\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.21\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build/build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2018\",\n pkg_name = \"rustversion\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rustversion\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.21\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + } + }, + "crates__ryu-1.0.18": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/ryu/1.0.18/download" + ], + "strip_prefix": "ryu-1.0.18", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"ryu\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=ryu\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.18\",\n)\n" + } + }, + "crates__serde-1.0.216": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/serde/1.0.216/download" + ], + "strip_prefix": "serde-1.0.216", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"serde\",\n deps = [\n \"@crates__serde-1.0.216//:build_script_build\",\n ],\n proc_macro_deps = [\n \"@crates__serde_derive-1.0.216//:serde_derive\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"default\",\n \"derive\",\n \"serde_derive\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.216\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"default\",\n \"derive\",\n \"serde_derive\",\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2018\",\n pkg_name = \"serde\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.216\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + } + }, + "crates__serde_derive-1.0.216": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/serde_derive/1.0.216/download" + ], + "strip_prefix": "serde_derive-1.0.216", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"serde_derive\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde_derive\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.216\",\n)\n" + } + }, + "crates__serde_json-1.0.134": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/serde_json/1.0.134/download" + ], + "strip_prefix": "serde_json-1.0.134", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"serde_json\",\n deps = [\n \"@crates__itoa-1.0.14//:itoa\",\n \"@crates__memchr-2.7.4//:memchr\",\n \"@crates__ryu-1.0.18//:ryu\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__serde_json-1.0.134//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde_json\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.134\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"serde_json\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde_json\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.134\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + } + }, + "crates__shlex-1.3.0": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/shlex/1.3.0/download" + ], + "strip_prefix": "shlex-1.3.0", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"shlex\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=shlex\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.3.0\",\n)\n" + } + }, + "crates__smallvec-1.13.2": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/smallvec/1.13.2/download" + ], + "strip_prefix": "smallvec-1.13.2", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"smallvec\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=smallvec\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.13.2\",\n)\n" + } + }, + "crates__spm_precompiled-0.1.4": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/spm_precompiled/0.1.4/download" + ], + "strip_prefix": "spm_precompiled-0.1.4", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"spm_precompiled\",\n deps = [\n \"@crates__base64-0.13.1//:base64\",\n \"@crates__nom-7.1.3//:nom\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__unicode-segmentation-1.12.0//:unicode_segmentation\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=spm_precompiled\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.4\",\n)\n" + } + }, + "crates__static_assertions-1.1.0": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/static_assertions/1.1.0/download" + ], + "strip_prefix": "static_assertions-1.1.0", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"static_assertions\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=static_assertions\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.1.0\",\n)\n" + } + }, + "crates__strsim-0.11.1": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/strsim/0.11.1/download" + ], + "strip_prefix": "strsim-0.11.1", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"strsim\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=strsim\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.11.1\",\n)\n" + } + }, + "crates__syn-2.0.91": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/syn/2.0.91/download" + ], + "strip_prefix": "syn-2.0.91", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"syn\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__unicode-ident-1.0.14//:unicode_ident\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"clone-impls\",\n \"default\",\n \"derive\",\n \"extra-traits\",\n \"full\",\n \"parsing\",\n \"printing\",\n \"proc-macro\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=syn\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.0.91\",\n)\n" + } + }, + "crates__thiserror-2.0.12": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/thiserror/2.0.12/download" + ], + "strip_prefix": "thiserror-2.0.12", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"thiserror\",\n deps = [\n \"@crates__thiserror-2.0.12//:build_script_build\",\n ],\n proc_macro_deps = [\n \"@crates__thiserror-impl-2.0.12//:thiserror_impl\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=thiserror\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.0.12\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"thiserror\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=thiserror\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"2.0.12\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + } + }, + "crates__thiserror-impl-2.0.12": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/thiserror-impl/2.0.12/download" + ], + "strip_prefix": "thiserror-impl-2.0.12", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"thiserror_impl\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=thiserror-impl\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.0.12\",\n)\n" + } + }, + "crates__tokenizers-0.22.2": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/tokenizers/0.22.2/download" + ], + "strip_prefix": "tokenizers-0.22.2", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"tokenizers\",\n deps = [\n \"@crates__ahash-0.8.12//:ahash\",\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__compact_str-0.9.0//:compact_str\",\n \"@crates__dary_heap-0.3.7//:dary_heap\",\n \"@crates__derive_builder-0.20.2//:derive_builder\",\n \"@crates__esaxx-rs-0.1.10//:esaxx_rs\",\n \"@crates__getrandom-0.3.3//:getrandom\",\n \"@crates__itertools-0.14.0//:itertools\",\n \"@crates__log-0.4.22//:log\",\n \"@crates__macro_rules_attribute-0.2.0//:macro_rules_attribute\",\n \"@crates__monostate-0.1.13//:monostate\",\n \"@crates__onig-6.5.1//:onig\",\n \"@crates__rand-0.9.1//:rand\",\n \"@crates__rayon-1.10.0//:rayon\",\n \"@crates__rayon-cond-0.4.0//:rayon_cond\",\n \"@crates__regex-1.11.1//:regex\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__serde_json-1.0.134//:serde_json\",\n \"@crates__spm_precompiled-0.1.4//:spm_precompiled\",\n \"@crates__thiserror-2.0.12//:thiserror\",\n \"@crates__unicode-normalization-alignments-0.1.12//:unicode_normalization_alignments\",\n \"@crates__unicode-segmentation-1.12.0//:unicode_segmentation\",\n \"@crates__unicode_categories-0.1.1//:unicode_categories\",\n ],\n proc_macro_deps = [\n \"@crates__paste-1.0.15//:paste\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"esaxx_fast\",\n \"onig\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=tokenizers\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.22.2\",\n)\n" + } + }, + "crates__unicode-ident-1.0.14": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/unicode-ident/1.0.14/download" + ], + "strip_prefix": "unicode-ident-1.0.14", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"unicode_ident\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=unicode-ident\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.14\",\n)\n" + } + }, + "crates__unicode-normalization-alignments-0.1.12": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/unicode-normalization-alignments/0.1.12/download" + ], + "strip_prefix": "unicode-normalization-alignments-0.1.12", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"unicode_normalization_alignments\",\n deps = [\n \"@crates__smallvec-1.13.2//:smallvec\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=unicode-normalization-alignments\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.12\",\n)\n" + } + }, + "crates__unicode-segmentation-1.12.0": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/unicode-segmentation/1.12.0/download" + ], + "strip_prefix": "unicode-segmentation-1.12.0", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"unicode_segmentation\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=unicode-segmentation\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.12.0\",\n)\n" + } + }, + "crates__unicode_categories-0.1.1": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/unicode_categories/0.1.1/download" + ], + "strip_prefix": "unicode_categories-0.1.1", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"unicode_categories\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=unicode_categories\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.1\",\n)\n" + } + }, + "crates__version_check-0.9.5": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/version_check/0.9.5/download" + ], + "strip_prefix": "version_check-0.9.5", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"version_check\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=version_check\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.9.5\",\n)\n" + } + }, + "crates__wasi-0.14.2-wasi-0.2.4": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/wasi/0.14.2+wasi-0.2.4/download" + ], + "strip_prefix": "wasi-0.14.2+wasi-0.2.4", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wasi\",\n deps = [\n \"@crates__wit-bindgen-rt-0.39.0//:wit_bindgen_rt\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasi\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.14.2+wasi-0.2.4\",\n)\n" + } + }, + "crates__wit-bindgen-rt-0.39.0": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/wit-bindgen-rt/0.39.0/download" + ], + "strip_prefix": "wit-bindgen-rt-0.39.0", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wit_bindgen_rt\",\n deps = [\n \"@crates__wit-bindgen-rt-0.39.0//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wit-bindgen-rt\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.39.0\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"wit-bindgen-rt\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wit-bindgen-rt\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.39.0\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + } + }, + "crates__zerocopy-0.7.35": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/zerocopy/0.7.35/download" + ], + "strip_prefix": "zerocopy-0.7.35", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"zerocopy\",\n deps = [\n \"@crates__byteorder-1.5.0//:byteorder\",\n ],\n proc_macro_deps = [\n \"@crates__zerocopy-derive-0.7.35//:zerocopy_derive\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"byteorder\",\n \"default\",\n \"derive\",\n \"simd\",\n \"zerocopy-derive\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=zerocopy\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.7.35\",\n)\n" + } + }, + "crates__zerocopy-0.8.26": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/zerocopy/0.8.26/download" + ], + "strip_prefix": "zerocopy-0.8.26", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"zerocopy\",\n deps = [\n \"@crates__zerocopy-0.8.26//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"simd\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=zerocopy\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.26\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"simd\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"zerocopy\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=zerocopy\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.8.26\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + } + }, + "crates__zerocopy-derive-0.7.35": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/zerocopy-derive/0.7.35/download" + ], + "strip_prefix": "zerocopy-derive-0.7.35", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"zerocopy_derive\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=zerocopy-derive\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.7.35\",\n)\n" + } + }, + "crates__zerocopy-derive-0.8.26": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "patch_args": [], + "patch_tool": "", + "patches": [], + "remote_patch_strip": 1, + "sha256": "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/zerocopy-derive/0.8.26/download" + ], + "strip_prefix": "zerocopy-derive-0.8.26", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"zerocopy_derive\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=zerocopy-derive\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.26\",\n)\n" + } + } + }, + "recordedRepoMappingEntries": [ + [ + "bazel_features+", + "bazel_features_globals", + "bazel_features++version_extension+bazel_features_globals" + ], + [ + "bazel_features+", + "bazel_features_version", + "bazel_features++version_extension+bazel_features_version" + ], + [ + "rules_cc+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_cc+", + "cc_compatibility_proxy", + "rules_cc++compatibility_proxy+cc_compatibility_proxy" + ], + [ + "rules_cc+", + "rules_cc", + "rules_cc+" + ], + [ + "rules_cc++compatibility_proxy+cc_compatibility_proxy", + "rules_cc", + "rules_cc+" + ], + [ + "rules_rust+", + "bazel_features", + "bazel_features+" + ], + [ + "rules_rust+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "rules_rust+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_rust+", + "rules_cc", + "rules_cc+" + ], + [ + "rules_rust+", + "rules_rust", + "rules_rust+" + ] + ] + } + }, + "@@rules_zig+//zig:extensions.bzl%zig": { + "general": { + "bzlTransitiveDigest": "AqBp0Gbe2OObIJWxY2tUasJpYX8sJRaKBfRgA1KUQN4=", + "usagesDigest": "MUggfXR9Y92m2LJD1coQy9DezZvA1s0RkFDA7YzrxBw=", + "recordedFileInputs": { + "@@//bazel/zig_index.json": "d38c409f2a4af04ad52d52f817ff71b77bb919194f5be6c34f827c68296d5795", + "@@rules_zig+//zig/private/versions.json": "6eb85ebaee72c4e6fbc82d8ad73f69e0899d839b70a7e19caa792ab6a5fa0c36" + }, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-linux": { + "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", + "attributes": { + "url": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "mirrors": [ + "https://mirror.zml.ai/zig", + "https://ziglang.org/builds/" + ], + "sha256": "de4eddf26f6f5e95d1bd055577cba7e9ebe9fbe8b1425bd4d40f073303ccc65b", + "zig_version": "0.16.0-dev.1912+0cbaaa5eb", + "platform": "aarch64-linux" + } + }, + "zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-macos": { + "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", + "attributes": { + "url": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "mirrors": [ + "https://mirror.zml.ai/zig", + "https://ziglang.org/builds/" + ], + "sha256": "241fa43a37803a9e50e7f4a2d2c436882ee2215e6c420c454cb4de57adae84d3", + "zig_version": "0.16.0-dev.1912+0cbaaa5eb", + "platform": "aarch64-macos" + } + }, + "zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-windows": { + "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", + "attributes": { + "url": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1912+0cbaaa5eb.zip", + "mirrors": [ + "https://mirror.zml.ai/zig", + "https://ziglang.org/builds/" + ], + "sha256": "824044c2dd76d4fdf9dd3d3fc7e8ed9bd39a1a03a3eb3acf1fecde25cee640cc", + "zig_version": "0.16.0-dev.1912+0cbaaa5eb", + "platform": "aarch64-windows" + } + }, + "zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-linux": { + "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", + "attributes": { + "url": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "mirrors": [ + "https://mirror.zml.ai/zig", + "https://ziglang.org/builds/" + ], + "sha256": "d0bfbe4c6e6871a36ad7c9be577df2abda37c69af437fedb6865d8729d60cbdc", + "zig_version": "0.16.0-dev.1912+0cbaaa5eb", + "platform": "x86_64-linux" + } + }, + "zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-macos": { + "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", + "attributes": { + "url": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "mirrors": [ + "https://mirror.zml.ai/zig", + "https://ziglang.org/builds/" + ], + "sha256": "9bfa7e5237c5e5b132229e3686eb849ca2afb23f7b6f6b72cbae0e5e8766aade", + "zig_version": "0.16.0-dev.1912+0cbaaa5eb", + "platform": "x86_64-macos" + } + }, + "zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-windows": { + "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", + "attributes": { + "url": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1912+0cbaaa5eb.zip", + "mirrors": [ + "https://mirror.zml.ai/zig", + "https://ziglang.org/builds/" + ], + "sha256": "719859787d0be77a8dc19f5ecc37d8cc7802ddca06f616d81e9ebde34e610dd4", + "zig_version": "0.16.0-dev.1912+0cbaaa5eb", + "platform": "x86_64-windows" + } + }, + "zig_toolchains": { + "repoRuleId": "@@rules_zig+//zig/private/repo:toolchains_repo.bzl%toolchains_repo", + "attributes": { + "names": [ + "zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-linux", + "zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-macos", + "zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-windows", + "zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-linux", + "zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-macos", + "zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-windows" + ], + "labels": [ + "@zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-linux//:zig_toolchain", + "@zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-macos//:zig_toolchain", + "@zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-windows//:zig_toolchain", + "@zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-linux//:zig_toolchain", + "@zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-macos//:zig_toolchain", + "@zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-windows//:zig_toolchain" + ], + "zig_versions": [ + "0.16.0-dev.1912+0cbaaa5eb", + "0.16.0-dev.1912+0cbaaa5eb", + "0.16.0-dev.1912+0cbaaa5eb", + "0.16.0-dev.1912+0cbaaa5eb", + "0.16.0-dev.1912+0cbaaa5eb", + "0.16.0-dev.1912+0cbaaa5eb" + ], + "exec_lengths": [ + 2, + 2, + 2, + 2, + 2, + 2 + ], + "exec_constraints": [ + "@platforms//os:linux", + "@platforms//cpu:aarch64", + "@platforms//os:macos", + "@platforms//cpu:aarch64", + "@platforms//os:windows", + "@platforms//cpu:aarch64", + "@platforms//os:linux", + "@platforms//cpu:x86_64", + "@platforms//os:macos", + "@platforms//cpu:x86_64", + "@platforms//os:windows", + "@platforms//cpu:x86_64" + ] + } + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_zig+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "rules_zig+", + "bazel_tools", + "bazel_tools" + ] + ] + } + } + }, "facts": {} } diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 67dff8c0..44531ad6 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -122,9 +122,6 @@ pub fn main() !void { const io = vfs.io(); - zml.init(); - defer zml.deinit(); - log.info("Resolving model repo", .{}); const repo = try zml.safetensors.resolveModelRepo(io, args.model.?); diff --git a/stdx/BUILD.bazel b/stdx/BUILD.bazel index bf5b5607..1dcfc56d 100644 --- a/stdx/BUILD.bazel +++ b/stdx/BUILD.bazel @@ -21,7 +21,6 @@ zig_library( main = "stdx.zig", visibility = ["//visibility:public"], zigopts = ["-lc"], - zigopts = ["-lc"], ) zig_test( From 7a798bb057efbbdcc6719d100e56d6e583056243 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Wed, 14 Jan 2026 19:19:09 +0100 Subject: [PATCH 236/881] zml/io/vfs: use indices to address backends, switch to StringArrayHashMapUnmanaged --- zml/io/vfs.zig | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/zml/io/vfs.zig b/zml/io/vfs.zig index 0fd827bf..32d3fad1 100644 --- a/zml/io/vfs.zig +++ b/zml/io/vfs.zig @@ -21,7 +21,7 @@ pub const VFS = struct { directory, }; - scheme: ?[]const u8, + idx: ?usize, type: Type, handle: u32, @@ -33,7 +33,7 @@ pub const VFS = struct { allocator: std.mem.Allocator, mutex: std.Io.Mutex = .init, - backends: std.StringHashMapUnmanaged(std.Io) = .{}, + backends: std.StringArrayHashMapUnmanaged(std.Io) = .empty, handles: Handles = .{}, closed_handles: ClosedHandles = .{}, @@ -44,7 +44,7 @@ pub const VFS = struct { var handles: Handles = .{}; try handles.append(allocator, .{ - .scheme = null, + .idx = null, .type = .directory, .handle = CWD_HANDLE, }); @@ -125,8 +125,8 @@ pub const VFS = struct { const handle = &self.handles.items[@intCast(file.handle)]; self.mutex.unlock(self.base.inner); - const backend = if (handle.scheme) |s| - self.backends.get(s).? + const backend = if (handle.idx) |idx| + self.backends.entries.items(.value)[idx] else self.base.inner; return .{ handle, backend }; @@ -157,9 +157,10 @@ pub const VFS = struct { } } else { const handle = self.getDirHandle(dir); - if (handle.scheme) |s| { - const backend = self.backends.get(s) orelse return error.VFSNotRegistered; - return .{ s, .{ .handle = @intCast(handle.handle) }, backend }; + if (handle.idx) |idx| { + const backend = self.backends.entries.items(.value)[idx]; + const scheme = self.backends.entries.items(.key)[idx]; + return .{ scheme, .{ .handle = @intCast(handle.handle) }, backend }; } else { return .{ null, .{ .handle = @intCast(handle.handle) }, self.base.inner }; } @@ -180,7 +181,7 @@ pub const VFS = struct { const fs_dir = try backend.vtable.dirOpenDir(backend.userdata, dir_, stripScheme(sub_path), options); const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; handle.* = .{ - .scheme = scheme, + .idx = if (scheme) |s| self.backends.getIndex(s) else null, .type = .directory, .handle = @intCast(fs_dir.handle), }; @@ -223,7 +224,7 @@ pub const VFS = struct { const file = try backend.vtable.dirOpenFile(backend.userdata, dir_, stripScheme(sub_path), flags); const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; handle.* = .{ - .scheme = scheme, + .idx = if (scheme) |s| self.backends.getIndex(s) else null, .type = .file, .handle = @intCast(file.handle), }; @@ -338,8 +339,9 @@ pub const VFS = struct { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const handle, const backend = self.getFileHandle(file); - if (handle.scheme) |s| { - const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); + if (handle.idx) |idx| { + const scheme = self.backends.entries.items(.key)[idx]; + const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{scheme}); const path_len = try backend.vtable.fileRealPath(backend.userdata, .{ .handle = @intCast(handle.handle) }, out_buffer[prefix.len..]); return prefix.len + path_len; } else { From 3c96903568440514f3c3fa7138f4cdf9f567740a Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 15 Jan 2026 15:07:07 +0100 Subject: [PATCH 237/881] Refactor llama and VFS to new CLI and IO APIs --- examples/llama/main.zig | 99 +++++++------------- examples/vfs/playground.zig | 181 ++++++++++++++---------------------- stdx/flags.zig | 6 +- zml/io/vfs.zig | 91 +++++++----------- zml/io/vfs/file.zig | 10 +- zml/io/vfs/hf.zig | 152 +++++++++--------------------- zml/io/vfs/http.zig | 35 ++----- 7 files changed, 199 insertions(+), 375 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 44531ad6..cf07e7ff 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -19,13 +19,34 @@ pub const std_options: std.Options = .{ .log_level = .info, }; -pub const Args = struct { - model: ?[]const u8 = null, +const CliArgs = struct { + model: []const u8, prompt: ?[]const u8 = null, seqlen: u32 = 512, - reader_buffer_size_mb: u32 = 32, - writer_buffer_size_mb: u32 = 32, - async_limit: ?usize = null, + reader_buffer_size: stdx.flags.ByteSize = .{ .value = 32, .unit = .mib }, + writer_buffer_size: stdx.flags.ByteSize = .{ .value = 32, .unit = .mib }, + async_limit: ?u32 = null, + + pub const help = + \\Usage: llama --model= [options] + \\ + \\Run LLaMA inference on a model. + \\ + \\Options: + \\ --model= Path to model (local path or HuggingFace repo) + \\ --prompt= Input prompt (reads from stdin if not provided) + \\ --seqlen= Maximum sequence length (default: 512) + \\ --reader-buffer-size= Reader buffer size (default: 32MiB) + \\ --writer-buffer-size= Writer buffer size (default: 32MiB) + \\ --async-limit= Async I/O concurrency limit + \\ -h, --help Show this help message + \\ + \\Examples: + \\ llama --model=hf://meta-llama/Llama-3.1-8B-Instruct --prompt="Hello, world!" + \\ llama --model=$(realpath ../Llama-3.1-8B-Instruct) --seqlen=1024 + \\ echo "What is 2+2?" | llama --model=meta-llama/Llama-3.1-8B-Instruct + \\ + ; }; pub fn main() !void { @@ -41,33 +62,7 @@ pub fn main() !void { var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); - const args: Args = blk: { - var ret: Args = .{}; - var it = std.process.args(); - defer it.deinit(); - while (it.next()) |arg| { - if (std.mem.startsWith(u8, arg, "--model=")) { - ret.model = arg["--model=".len..]; - } else if (std.mem.startsWith(u8, arg, "--seqlen=")) { - ret.seqlen = try std.fmt.parseUnsigned(u32, arg["--seqlen=".len..], 10); - } else if (std.mem.startsWith(u8, arg, "--prompt=")) { - ret.prompt = arg["--prompt=".len..]; - } else if (std.mem.startsWith(u8, arg, "--reader-buffer-size-mb=")) { - ret.reader_buffer_size_mb = try std.fmt.parseUnsigned(u32, arg["--reader-buffer-size-mb=".len..], 10); - } else if (std.mem.startsWith(u8, arg, "--writer-buffer-size-mb=")) { - ret.writer_buffer_size_mb = try std.fmt.parseUnsigned(u32, arg["--writer-buffer-size-mb=".len..], 10); - } else if (std.mem.startsWith(u8, arg, "--async-limit=")) { - ret.async_limit = try std.fmt.parseUnsigned(usize, arg["--async-limit=".len..], 10); - } - } - - if (ret.model == null) { - log.err("Missing --model", .{}); - return; - } - - break :blk ret; - }; + const args = stdx.flags.parseProcessArgs(CliArgs); if (args.async_limit) |limit| threaded.setAsyncLimit(.limited(limit)); @@ -76,41 +71,19 @@ pub fn main() !void { var http_client: std.http.Client = .{ .allocator = allocator, .io = threaded.io(), - .connection_pool = .{ - .free_size = threaded.async_limit.toInt() orelse 16, - }, + .connection_pool = .{ .free_size = threaded.async_limit.toInt() orelse 16 }, }; try http_client.initDefaultProxies(allocator); defer http_client.deinit(); - var vfs_file: zml.io.VFS.File = .init( - allocator, - threaded.io(), - .{ - .direct_io = true, - .direct_io_alignment = .fromByteUnits(4 * 1024), - }, - ); + var vfs_file: zml.io.VFS.File = .init(allocator, threaded.io(), .{}); defer vfs_file.deinit(); - var vfs_https: zml.io.VFS.HTTP = try .init(allocator, threaded.io(), .{ - .http_client = &http_client, - .protocol = .https, - }); + var vfs_https: zml.io.VFS.HTTP = try .init(allocator, threaded.io(), &http_client, .https); defer vfs_https.deinit(); - var hf_auth: zml.io.VFS.HF.Auth = try .auto(allocator, threaded.io()); - defer hf_auth.deinit(allocator); - - var hf_vfs: zml.io.VFS.HF = try .init( - allocator, - threaded.io(), - .{ - .http_client = &http_client, - .auth = hf_auth, - }, - ); + var hf_vfs: zml.io.VFS.HF = try .auto(allocator, threaded.io(), &http_client); defer hf_vfs.deinit(); var vfs: zml.io.VFS = try .init(allocator, threaded.io()); @@ -123,7 +96,7 @@ pub fn main() !void { const io = vfs.io(); log.info("Resolving model repo", .{}); - const repo = try zml.safetensors.resolveModelRepo(io, args.model.?); + const repo = try zml.safetensors.resolveModelRepo(io, args.model); const parsed_config = try parseConfig(allocator, io, repo); defer parsed_config.deinit(); @@ -193,12 +166,12 @@ pub fn main() !void { allocator, platform, .{ - .size = args.reader_buffer_size_mb * 1024 * 1024, + .size = args.reader_buffer_size.bytes() * 1024 * 1024, .concurrency = load_model_buffers.limit, .dma = true, }, .{ - .size = args.writer_buffer_size_mb * 1024 * 1024, + .size = args.writer_buffer_size.bytes() * 1024 * 1024, .concurrency = load_model_buffers.limit, .dma = true, }, @@ -208,8 +181,8 @@ pub fn main() !void { write_pool.deinit(); } - var model_name_it = std.mem.splitScalar(u8, args.model.?, '/'); - var model_name = model_name_it.next() orelse args.model.?; + var model_name_it = std.mem.splitScalar(u8, args.model, '/'); + var model_name = model_name_it.next() orelse args.model; while (model_name_it.next()) |part| { if (!std.mem.eql(u8, part, "")) model_name = part; } diff --git a/examples/vfs/playground.zig b/examples/vfs/playground.zig index 3633845d..a453101b 100644 --- a/examples/vfs/playground.zig +++ b/examples/vfs/playground.zig @@ -1,6 +1,7 @@ const std = @import("std"); const zml = @import("zml"); +const stdx = zml.stdx; const log = std.log.scoped(.vfs); @@ -8,23 +9,11 @@ pub const std_options: std.Options = .{ .log_level = .info, }; -pub const Command = union(enum) { - cat, - tree, - stat, -}; - -pub const Args = struct { - command: ?Command = null, - path: ?[]const u8 = null, - reader_buffer_size_mb: u32 = 32, - async_limit: ?usize = null, -}; - +// -- ls hf://openai/gpt-oss-20b@6cee5e8 +// -- ls hf://Qwen/Qwen3-235B-A22B-Instruct-2507 +// -- ls hf://meta-llama/Llama-3.1-8B-Instruct@0e9e39f // -- cat https://iprs.fly.dev pub fn main() !void { - log.info("VFS playground was compiled with {}", .{@import("builtin").mode}); - var debug_allocator: ?std.heap.DebugAllocator(.{}) = null; const allocator = if (@import("builtin").mode == .Debug) blk: { debug_allocator = .init; @@ -35,56 +24,12 @@ pub fn main() !void { var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); - const args: Args = blk: { - var ret: Args = .{}; - var it = std.process.args(); - defer it.deinit(); - var index: usize = 0; - while (it.next()) |arg| { - if (std.mem.startsWith(u8, arg, "--reader-buffer-size-mb=")) { - ret.reader_buffer_size_mb = try std.fmt.parseUnsigned(u32, arg["--reader-buffer-size-mb=".len..], 10); - } else if (std.mem.startsWith(u8, arg, "--async-limit=")) { - ret.async_limit = try std.fmt.parseUnsigned(usize, arg["--async-limit=".len..], 10); - } else { - if (index == 0) { - // skip program name - } else if (index == 1) { - if (std.mem.eql(u8, arg, "cat")) { - ret.command = .cat; - } else if (std.mem.eql(u8, arg, "stat")) { - ret.command = .stat; - } else if (std.mem.eql(u8, arg, "tree")) { - ret.command = .tree; - } else { - log.err("Unknown command: {s}", .{arg}); - return; - } - } else if (index == 2) { - ret.path = arg; - } else { - log.err("Unknown argument: {s}", .{arg}); - return; - } - index += 1; - } - } + const Command = enum { cat, ls, stat, realpath }; - if (ret.command == null) { - log.err("Missing command as argument", .{}); - return; - } - - if (ret.path == null) { - log.err("Missing path as argument", .{}); - return; - } - - break :blk ret; - }; - - if (args.async_limit) |limit| threaded.setAsyncLimit(.limited(limit)); - - log.info("Running with threaded io async limit set to {?d}", .{threaded.async_limit.toInt()}); + var it = std.process.args(); + _ = it.next(); // skip program name + const command: Command = std.meta.stringToEnum(Command, it.next() orelse return error.MissingCommand) orelse return error.CommandInvalid; + const path = it.next() orelse return error.MissingPath; var http_client: std.http.Client = .{ .allocator = allocator, @@ -97,33 +42,13 @@ pub fn main() !void { try http_client.initDefaultProxies(allocator); defer http_client.deinit(); - var vfs_file: zml.io.VFS.File = .init( - allocator, - threaded.io(), - .{ - .direct_io = true, - .direct_io_alignment = .fromByteUnits(4 * 1024), - }, - ); + var vfs_file: zml.io.VFS.File = .init(allocator, threaded.io(), .{}); defer vfs_file.deinit(); - var vfs_https: zml.io.VFS.HTTP = try .init(allocator, threaded.io(), .{ - .http_client = &http_client, - .protocol = .https, - }); + var vfs_https: zml.io.VFS.HTTP = try .init(allocator, threaded.io(), &http_client, .https); defer vfs_https.deinit(); - var hf_auth: zml.io.VFS.HF.Auth = try .auto(allocator, threaded.io()); - defer hf_auth.deinit(allocator); - - var hf_vfs: zml.io.VFS.HF = try .init( - allocator, - threaded.io(), - .{ - .http_client = &http_client, - .auth = hf_auth, - }, - ); + var hf_vfs: zml.io.VFS.HF = try .auto(allocator, threaded.io(), &http_client); defer hf_vfs.deinit(); var vfs: zml.io.VFS = try .init(allocator, threaded.io()); @@ -135,15 +60,13 @@ pub fn main() !void { const io = vfs.io(); - const path = args.path.?; - - const buffer = try allocator.alloc(u8, args.reader_buffer_size_mb * 1024 * 1024); + const buffer = try allocator.alloc(u8, 16 * 1024 * 1024); defer allocator.free(buffer); var stdout_writer = std.Io.File.stdout().writer(io, buffer); defer stdout_writer.interface.flush() catch {}; - switch (args.command.?) { + switch (command) { .cat => { var file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); defer file.close(io); @@ -152,13 +75,18 @@ pub fn main() !void { const read = try reader.interface.streamRemaining(&stdout_writer.interface); _ = try stdout_writer.interface.write("\n"); - log.info("Wrote {d} bytes to stdout from {s}", .{ read, path }); + try stdout_writer.interface.print("Wrote {B:.2} to stdout from {s}\n", .{ read, path }); }, - .tree => { + .ls => { var dir = try std.Io.Dir.openDir(.cwd(), io, path, .{ .iterate = true }); defer dir.close(io); - try printTree(io, &stdout_writer.interface, dir, 0, 10); + const dir_stat = try dir.stat(io); + try stdout_writer.interface.print("{s} - {B:.2}\n", .{ path, dir_stat.size }); + + var counts: TreeCounts = .{}; + try printTree(io, &stdout_writer.interface, dir, "", 10, &counts); + try stdout_writer.interface.print("\n{d} directories, {d} files\n", .{ counts.dirs, counts.files }); }, .stat => { const stat = std.Io.Dir.statFile(.cwd(), io, path, .{}) catch |err| blk: { @@ -171,39 +99,66 @@ pub fn main() !void { return err; }; - log.info("Stat for {s}: size={d} bytes, kind={s}", .{ path, stat.size, @tagName(stat.kind) }); + try stdout_writer.interface.print("{s}: {B:.2} ({s})\n", .{ path, stat.size, @tagName(stat.kind) }); + }, + .realpath => { + var dir = try std.Io.Dir.openDir(.cwd(), io, path, .{}); + defer dir.close(io); + + var real_path_buf: [256]u8 = undefined; + const len = try dir.realPath(io, &real_path_buf); + + try stdout_writer.interface.print("{s}\n", .{real_path_buf[0..len]}); }, } } -fn printTree(io: std.Io, writer: *std.Io.Writer, dir: std.Io.Dir, depth: usize, max_depth: usize) !void { - if (depth > max_depth) return; +const TreeCounts = struct { + dirs: usize = 0, + files: usize = 0, +}; +fn printTree(io: std.Io, writer: *std.Io.Writer, dir: std.Io.Dir, prefix: []const u8, max_depth: usize, counts: *TreeCounts) !void { + if (max_depth == 0) return; + + var entries: stdx.BoundedArray(std.Io.Dir.Entry, 1024) = .{}; var it = dir.iterate(); while (try it.next(io)) |entry| { - for (0..depth) |_| { - try writer.writeAll(" "); - } + entries.appendAssumeCapacity(entry); + } - const size = if (entry.kind == .file) blk: { - const stat = try dir.statFile(io, entry.name, .{ .follow_symlinks = true }); - break :blk stat.size; - } else if (entry.kind == .directory) blk: { - const stat = try dir.stat(io); - break :blk stat.size; - } else { - return error.UnexpectedEntryKind; + for (entries.constSlice(), 0..) |entry, idx| { + const is_last = (idx == entries.len - 1); + const connector = if (is_last) "└── " else "├── "; + const extension = if (is_last) " " else "│ "; + + const size: u64 = switch (entry.kind) { + .file => blk: { + const stat = try dir.statFile(io, entry.name, .{}); + break :blk stat.size; + }, + .directory => blk: { + var sub_dir = dir.openDir(io, entry.name, .{}) catch break :blk 0; + defer sub_dir.close(io); + + const stat = try sub_dir.stat(io); + break :blk stat.size; + }, + else => 0, }; - var buf: [256]u8 = undefined; - const slice = try std.fmt.bufPrint(&buf, "- {s} kind={s} size={d}\n", .{ entry.name, @tagName(entry.kind), size }); - try writer.writeAll(slice); + try writer.print("{s}{s}{s} - {B:.2}\n", .{ prefix, connector, entry.name, size }); if (entry.kind == .directory) { - var sub_dir = try dir.openDir(io, entry.name, .{ .iterate = true }); + counts.dirs += 1; + var sub_dir = dir.openDir(io, entry.name, .{ .iterate = true }) catch continue; defer sub_dir.close(io); - try printTree(io, writer, sub_dir, depth + 1, max_depth); + var new_prefix_buf: [4096]u8 = undefined; + const new_prefix = std.fmt.bufPrint(&new_prefix_buf, "{s}{s}", .{ prefix, extension }) catch continue; + try printTree(io, writer, sub_dir, new_prefix, max_depth - 1, counts); + } else { + counts.files += 1; } } } diff --git a/stdx/flags.zig b/stdx/flags.zig index 3c5213ee..a0654839 100644 --- a/stdx/flags.zig +++ b/stdx/flags.zig @@ -49,7 +49,7 @@ const debug = @import("debug.zig"); /// Format and print an error message to stderr, then exit with an exit code of 1. pub fn fatal(comptime fmt_string: []const u8, args: anytype) noreturn { std.debug.print("error: " ++ fmt_string ++ "\n", args); - std.posix.exit(1); + std.process.exit(1); } /// Parse CLI arguments for subcommands specified as Zig `struct` or `union(enum)`: @@ -104,8 +104,8 @@ fn parse_commands(args: *std.process.ArgIterator, comptime Commands: type) Comma // NB: help must be declared as *pub* const to be visible here. if (@hasDecl(Commands, "help")) { if (std.mem.eql(u8, first_arg, "-h") or std.mem.eql(u8, first_arg, "--help")) { - std.io.getStdOut().writeAll(Commands.help) catch std.posix.exit(1); - std.posix.exit(0); + std.io.getStdOut().writeAll(Commands.help) catch std.process.exit(1); + std.process.exit(0); } } diff --git a/zml/io/vfs.zig b/zml/io/vfs.zig index 32d3fad1..28377a7c 100644 --- a/zml/io/vfs.zig +++ b/zml/io/vfs.zig @@ -15,20 +15,7 @@ pub const VFS = struct { const CWD_HANDLE: u32 = 0; - const Handle = struct { - pub const Type = enum { - file, - directory, - }; - - idx: ?usize, - type: Type, - handle: u32, - - pub fn isRootFs(self: *Handle) bool { - return self.scheme == null; - } - }; + const Handle = struct { handle: u32, backend_idx: ?usize }; allocator: std.mem.Allocator, mutex: std.Io.Mutex = .init, @@ -43,11 +30,7 @@ pub const VFS = struct { const base = VFSBase.init(base_io); var handles: Handles = .{}; - try handles.append(allocator, .{ - .idx = null, - .type = .directory, - .handle = CWD_HANDLE, - }); + try handles.append(allocator, .{ .handle = CWD_HANDLE, .backend_idx = null }); return .{ .allocator = allocator, @@ -125,42 +108,44 @@ pub const VFS = struct { const handle = &self.handles.items[@intCast(file.handle)]; self.mutex.unlock(self.base.inner); - const backend = if (handle.idx) |idx| - self.backends.entries.items(.value)[idx] - else - self.base.inner; - return .{ handle, backend }; + return .{ handle, self.getBackend(handle.backend_idx) }; } fn getDirHandle(self: *VFS, dir: std.Io.Dir) *Handle { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); - if (std.meta.eql(dir, std.Io.Dir.cwd())) { - return &self.handles.items[CWD_HANDLE]; - } - + if (std.meta.eql(dir, std.Io.Dir.cwd())) return &self.handles.items[CWD_HANDLE]; return &self.handles.items[@intCast(dir.handle)]; } - fn lookup(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?[]const u8, std.Io.Dir, std.Io } { + fn getScheme(self: *VFS, backend_idx: ?usize) ?[]const u8 { + if (backend_idx) |idx| return self.backends.entries.items(.key)[idx] else return null; + } + + fn getBackend(self: *VFS, backend_idx: ?usize) std.Io { + if (backend_idx) |idx| return self.backends.entries.items(.value)[idx] else return self.base.inner; + } + + fn lookupDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?usize, std.Io.Dir, std.Io } { if (std.meta.eql(dir, std.Io.Dir.cwd())) { if (sub_path == null) return .{ null, dir, self.base.inner }; if (std.fs.path.isAbsolutePosix(sub_path.?)) return .{ null, dir, self.base.inner }; const uri = std.Uri.parse(sub_path.?) catch null; if (uri) |u| { - const backend = self.backends.get(u.scheme) orelse return error.VFSNotRegistered; - return .{ u.scheme, std.Io.Dir.cwd(), backend }; + const backend_idx: ?usize = for (self.backends.entries.items(.key), 0..) |s, idx| { + if (std.mem.eql(u8, u.scheme, s)) break idx; + } else null; + if (backend_idx == null) return error.VFSNotRegistered; + return .{ backend_idx, std.Io.Dir.cwd(), self.getBackend(backend_idx) }; } else { return .{ null, std.Io.Dir.cwd(), self.base.inner }; } } else { const handle = self.getDirHandle(dir); - if (handle.idx) |idx| { - const backend = self.backends.entries.items(.value)[idx]; - const scheme = self.backends.entries.items(.key)[idx]; - return .{ scheme, .{ .handle = @intCast(handle.handle) }, backend }; + if (handle.backend_idx) |backend_idx| { + return .{ backend_idx, .{ .handle = @intCast(handle.handle) }, self.getBackend(backend_idx) }; } else { return .{ null, .{ .handle = @intCast(handle.handle) }, self.base.inner }; } @@ -174,23 +159,22 @@ pub const VFS = struct { fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const scheme, const dir_, const backend = self.lookup(dir, sub_path) catch |err| { + const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { log.err("Failed to lookup backend for opening dir '{s}' : {any}", .{ sub_path, err }); return std.Io.Dir.OpenError.Unexpected; }; const fs_dir = try backend.vtable.dirOpenDir(backend.userdata, dir_, stripScheme(sub_path), options); const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; handle.* = .{ - .idx = if (scheme) |s| self.backends.getIndex(s) else null, - .type = .directory, .handle = @intCast(fs_dir.handle), + .backend_idx = backend_idx, }; return .{ .handle = @intCast(idx) }; } fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - _, const dir_, const backend = self.lookup(dir, null) catch |err| { + _, const dir_, const backend = self.lookupDir(dir, null) catch |err| { log.err("Failed to lookup backend for dir stat : {any}", .{err}); return std.Io.Dir.StatError.Unexpected; }; @@ -199,7 +183,7 @@ pub const VFS = struct { pub fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - _, const dir_, const backend = self.lookup(dir, sub_path) catch |err| { + _, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { log.err("Failed to lookup backend for dir stat file '{s}' : {any}", .{ sub_path, err }); return std.Io.Dir.StatFileError.Unexpected; }; @@ -208,7 +192,7 @@ pub const VFS = struct { pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - _, const dir_, const backend = self.lookup(dir, sub_path) catch |err| { + _, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { log.err("Failed to lookup backend for dir access '{s}' : {any}", .{ sub_path, err }); return std.Io.Dir.AccessError.Unexpected; }; @@ -217,36 +201,34 @@ pub const VFS = struct { fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const scheme, const dir_, const backend = self.lookup(dir, sub_path) catch |err| { + const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { log.err("Failed to lookup backend for opening file '{s}' : {any}", .{ sub_path, err }); return std.Io.File.OpenError.Unexpected; }; const file = try backend.vtable.dirOpenFile(backend.userdata, dir_, stripScheme(sub_path), flags); const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; handle.* = .{ - .idx = if (scheme) |s| self.backends.getIndex(s) else null, - .type = .file, .handle = @intCast(file.handle), + .backend_idx = backend_idx, }; - return .{ .handle = @intCast(idx) }; } fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); for (dirs) |dir| { - _, const dir_, const backend = self.lookup(dir, null) catch |err| { + _, const dir_, const backend = self.lookupDir(dir, null) catch |err| { log.err("Failed to lookup backend for closing dir : {any}", .{err}); continue; }; backend.vtable.dirClose(backend.userdata, &.{dir_}); - self.closeHandle(@intCast(dir_.handle)) catch unreachable; + self.closeHandle(@intCast(dir.handle)) catch unreachable; } } fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - _, const dir_, const backend = self.lookup(reader.dir, null) catch |err| { + _, const dir_, const backend = self.lookupDir(reader.dir, null) catch |err| { log.err("Failed to lookup backend for dir real path : {any}", .{err}); return std.Io.Dir.RealPathError.Unexpected; }; @@ -260,12 +242,12 @@ pub const VFS = struct { fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const scheme, const dir_, const backend = self.lookup(dir, null) catch |err| { + const backend_idx, const dir_, const backend = self.lookupDir(dir, null) catch |err| { log.err("Failed to lookup backend for dir real path : {any}", .{err}); return std.Io.Dir.RealPathError.Unexpected; }; - if (scheme) |s| { + if (self.getScheme(backend_idx)) |s| { const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); const path_len = try backend.vtable.dirRealPath(backend.userdata, dir_, out_buffer[prefix.len..]); return prefix.len + path_len; @@ -276,12 +258,12 @@ pub const VFS = struct { fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const scheme, const dir_, const backend = self.lookup(dir, null) catch |err| { + const backend_idx, const dir_, const backend = self.lookupDir(dir, path_name) catch |err| { log.err("Failed to lookup backend for dir real path file '{s}' : {any}", .{ path_name, err }); return std.Io.Dir.RealPathFileError.Unexpected; }; - if (scheme) |s| { + if (self.getScheme(backend_idx)) |s| { const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); const path_len = try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer[prefix.len..]); return prefix.len + path_len; @@ -339,9 +321,8 @@ pub const VFS = struct { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const handle, const backend = self.getFileHandle(file); - if (handle.idx) |idx| { - const scheme = self.backends.entries.items(.key)[idx]; - const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{scheme}); + if (self.getScheme(handle.backend_idx)) |s| { + const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); const path_len = try backend.vtable.fileRealPath(backend.userdata, .{ .handle = @intCast(handle.handle) }, out_buffer[prefix.len..]); return prefix.len + path_len; } else { diff --git a/zml/io/vfs/file.zig b/zml/io/vfs/file.zig index 31fe0e80..0c52eada 100644 --- a/zml/io/vfs/file.zig +++ b/zml/io/vfs/file.zig @@ -58,18 +58,15 @@ fn switchToDirectIO(file: std.Io.File) DirectIoError!void { pub const File = struct { pub const Config = struct { - direct_io: bool = false, + direct_io: bool = true, direct_io_alignment: std.mem.Alignment = .fromByteUnits(4 * 1024), }; - const Handle = struct { - inner_handle: std.Io.File.Handle, - direct_io_handle: ?std.Io.File.Handle, - }; + const Handle = struct { inner_handle: std.Io.File.Handle, direct_io_handle: ?std.Io.File.Handle }; allocator: std.mem.Allocator, mutex: std.Io.Mutex = .init, - handles: stdx.SegmentedList(Handle, 0), + handles: stdx.SegmentedList(Handle, 0) = .{}, closed_handles: std.ArrayList(u32) = .{}, config: Config, base: VFSBase, @@ -77,7 +74,6 @@ pub const File = struct { pub fn init(allocator: std.mem.Allocator, inner: std.Io, config: Config) File { return .{ .allocator = allocator, - .handles = .{}, .config = config, .base = .init(inner), }; diff --git a/zml/io/vfs/hf.zig b/zml/io/vfs/hf.zig index e821906e..50e2811b 100644 --- a/zml/io/vfs/hf.zig +++ b/zml/io/vfs/hf.zig @@ -18,10 +18,7 @@ pub const API = struct { }; }; -const ReadState = struct { - children: []const TreeNode, - index: usize, -}; +const ReadState = struct { children: []const TreeNode, index: usize }; pub const TreeNode = struct { name: []const u8, @@ -53,75 +50,6 @@ const RepoKey = struct { }; pub const HF = struct { - pub const Auth = union(enum) { - none, - hf_token: []const u8, - - pub const AuthError = error{ - MissingHFToken, - InvalidHFToken, - MissingHomePath, - MissingConfigFile, - Unexpected, - }; - - pub fn deinit(self: *Auth, allocator: std.mem.Allocator) void { - switch (self.*) { - .none => {}, - .hf_token => |token| { - allocator.free(token); - }, - } - } - - pub fn auto(allocator: std.mem.Allocator, io_: std.Io) AuthError!Auth { - return fromHomeConfig(allocator, io_) catch |err| switch (err) { - AuthError.MissingHomePath, AuthError.MissingConfigFile => return fromEnv(allocator) catch |e| switch (e) { - AuthError.MissingHFToken => { - log.warn("No Hugging Face authentication token found in environment or home config; proceeding without authentication.", .{}); - return .none; - }, - else => return AuthError.Unexpected, - }, - else => return AuthError.Unexpected, - }; - } - - pub fn fromEnv(allocator: std.mem.Allocator) AuthError!Auth { - const token = std.process.getEnvVarOwned(allocator, "HF_TOKEN") catch |err| switch (err) { - error.EnvironmentVariableNotFound => return AuthError.MissingHFToken, - else => return AuthError.Unexpected, - }; - defer allocator.free(token); - - return .{ .hf_token = std.fmt.allocPrint(allocator, "Bearer {s}", .{token}) catch return AuthError.Unexpected }; - } - - pub fn fromHomeConfig(allocator: std.mem.Allocator, base_io: std.Io) AuthError!Auth { - const home_path = std.process.getEnvVarOwned(allocator, "HOME") catch |err| switch (err) { - error.EnvironmentVariableNotFound => return AuthError.MissingHomePath, - else => return AuthError.Unexpected, - }; - defer allocator.free(home_path); - - const token_path = std.fmt.allocPrint(allocator, "{s}/.cache/huggingface/token", .{home_path}) catch return AuthError.Unexpected; - defer allocator.free(token_path); - - const file = std.Io.Dir.openFileAbsolute(base_io, token_path, .{ .mode = .read_only }) catch |err| switch (err) { - error.FileNotFound => return AuthError.MissingConfigFile, - else => return AuthError.Unexpected, - }; - - var reader = file.reader(base_io, &.{}); - const size = reader.getSize() catch return AuthError.Unexpected; - - const token = reader.interface.readAlloc(allocator, size) catch return AuthError.Unexpected; - defer allocator.free(token); - - return .{ .hf_token = std.fmt.allocPrint(allocator, "Bearer {s}", .{std.mem.trim(u8, token, "\n")}) catch return AuthError.Unexpected }; - } - }; - pub const Repo = struct { repo: []const u8, model: []const u8, @@ -149,10 +77,7 @@ pub const HF = struct { }; const Handle = struct { - pub const Type = enum { - file, - directory, - }; + pub const Type = enum { file, directory }; type: Type, uri: []const u8, @@ -173,47 +98,55 @@ pub const HF = struct { } }; - const Config = struct { - http_client: *std.http.Client, - auth: Auth = .none, - }; - allocator: std.mem.Allocator, mutex: std.Io.Mutex = .init, client: *std.http.Client, - authorization: std.http.Client.Request.Headers.Value = .default, - handles: stdx.SegmentedList(Handle, 0), + authorization: std.http.Client.Request.Headers.Value, + handles: stdx.SegmentedList(Handle, 0) = .{}, closed_handles: std.ArrayList(u32) = .{}, base: VFSBase, trees: std.StringHashMapUnmanaged(std.ArrayList(TreeNode)) = .{}, dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, - pub fn init(allocator: std.mem.Allocator, base_io: std.Io, config: Config) !HF { - var handles: stdx.SegmentedList(Handle, 0) = .{}; - // cwd - const handle = try handles.addOne(allocator); - handle.* = .{ - .type = .directory, - .uri = "", - .pos = 0, - .size = 0, - }; - - var hf: HF = .{ + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, hf_token: ?[]const u8) !HF { + return .{ .allocator = allocator, - .handles = handles, - .base = .init(base_io), - .client = config.http_client, - }; - - switch (config.auth) { - .hf_token => |token| { - hf.authorization = .{ .override = token }; + .base = .init(inner), + .client = http_client, + .authorization = if (hf_token) |token| blk: { + break :blk .{ + .override = try std.fmt.allocPrint(allocator, "Bearer {s}", .{std.mem.trim(u8, token, " \t\n\r")}), + }; + } else blk: { + break :blk .default; }, - .none => {}, - } + }; + } - return hf; + pub fn auto(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client) !HF { + const hf_token = if (std.process.getEnvVarOwned(allocator, "HF_TOKEN")) |token| blk: { + break :blk token; + } else |_| blk: { + const home_path = std.process.getEnvVarOwned(allocator, "HOME") catch break :blk null; + defer allocator.free(home_path); + + var path_buf: [256]u8 = undefined; + const token_path = std.fmt.bufPrint(&path_buf, "{s}/.cache/huggingface/token", .{home_path}) catch break :blk null; + + var file = std.Io.Dir.openFileAbsolute(inner, token_path, .{ .mode = .read_only }) catch break :blk null; + defer file.close(inner); + + const size = file.stat(inner) catch break :blk null; + var reader = file.reader(inner, &.{}); + const token = reader.interface.readAlloc(allocator, size.size) catch break :blk null; + + break :blk token; + }; + defer if (hf_token) |token| allocator.free(token); + + if (hf_token == null) log.warn("No Hugging Face authentication token found in environment or home config; proceeding without authentication.", .{}); + + return init(allocator, inner, http_client, hf_token); } pub fn deinit(self: *HF) void { @@ -240,6 +173,11 @@ pub const HF = struct { } self.trees.deinit(self.allocator); self.dir_read_states.deinit(self.allocator); + + switch (self.authorization) { + .default, .omit => {}, + .override => |t| self.allocator.free(t), + } } pub fn io(self: *HF) std.Io { diff --git a/zml/io/vfs/http.zig b/zml/io/vfs/http.zig index fe10767b..6106ff08 100644 --- a/zml/io/vfs/http.zig +++ b/zml/io/vfs/http.zig @@ -35,41 +35,22 @@ pub const HTTP = struct { } }; - const Protocol = union(enum) { - http, - https, - }; - - const Config = struct { - http_client: *std.http.Client, - protocol: Protocol, - }; + const Protocol = enum { http, https }; allocator: std.mem.Allocator, mutex: std.Io.Mutex = .init, client: *std.http.Client, protocol: Protocol, - handles: stdx.SegmentedList(Handle, 0), + handles: stdx.SegmentedList(Handle, 0) = .{}, closed_handles: std.ArrayList(u32) = .{}, base: VFSBase, - pub fn init(allocator: std.mem.Allocator, base_io: std.Io, config: Config) !HTTP { - var handles: stdx.SegmentedList(Handle, 0) = .{}; - // cwd - const handle = try handles.addOne(allocator); - handle.* = .{ - .type = .directory, - .uri = "", - .pos = 0, - .size = 0, - }; - + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, protocol: Protocol) !HTTP { return .{ .allocator = allocator, - .base = .init(base_io), - .handles = handles, - .client = config.http_client, - .protocol = config.protocol, + .base = .init(inner), + .client = http_client, + .protocol = protocol, }; } @@ -175,7 +156,7 @@ pub const HTTP = struct { const handle = self.getDirHandle(dir); return .{ - .inode = @intCast(dir.handle), + .inode = @intCast(0), .nlink = 0, .size = handle.size, .permissions = .fromMode(0o444), @@ -191,7 +172,7 @@ pub const HTTP = struct { const size = self.fetchSize(dir, sub_path) catch return std.Io.Dir.StatFileError.Unexpected; return .{ - .inode = @intCast(dir.handle), + .inode = @intCast(0), .nlink = 0, .size = size, .permissions = .fromMode(0o444), From 015ed2245fa1ebd438020a9431e412782d103c5e Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 15 Jan 2026 16:12:43 +0000 Subject: [PATCH 238/881] zml/io/vfs: add s3 vfs --- examples/llama/main.zig | 4 + examples/vfs/playground.zig | 4 + zml/io/vfs.zig | 1 + zml/io/vfs/BUILD.bazel | 1 + zml/io/vfs/index.zig | 9 +- zml/io/vfs/s3.zig | 684 ++++++++++++++++++++++++++++++++++++ 6 files changed, 699 insertions(+), 4 deletions(-) create mode 100644 zml/io/vfs/s3.zig diff --git a/examples/llama/main.zig b/examples/llama/main.zig index cf07e7ff..0370ceb7 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -86,12 +86,16 @@ pub fn main() !void { var hf_vfs: zml.io.VFS.HF = try .auto(allocator, threaded.io(), &http_client); defer hf_vfs.deinit(); + var s3_vfs: zml.io.VFS.S3 = try .fromEnv(allocator, threaded.io(), &http_client); + defer s3_vfs.deinit(); + var vfs: zml.io.VFS = try .init(allocator, threaded.io()); defer vfs.deinit(); try vfs.register("file", vfs_file.io()); try vfs.register("https", vfs_https.io()); try vfs.register("hf", hf_vfs.io()); + try vfs.register("s3", s3_vfs.io()); const io = vfs.io(); diff --git a/examples/vfs/playground.zig b/examples/vfs/playground.zig index a453101b..1c5a1e47 100644 --- a/examples/vfs/playground.zig +++ b/examples/vfs/playground.zig @@ -51,12 +51,16 @@ pub fn main() !void { var hf_vfs: zml.io.VFS.HF = try .auto(allocator, threaded.io(), &http_client); defer hf_vfs.deinit(); + var s3_vfs: zml.io.VFS.S3 = try .fromEnv(allocator, threaded.io(), &http_client); + defer s3_vfs.deinit(); + var vfs: zml.io.VFS = try .init(allocator, threaded.io()); defer vfs.deinit(); try vfs.register("file", vfs_file.io()); try vfs.register("https", vfs_https.io()); try vfs.register("hf", hf_vfs.io()); + try vfs.register("s3", s3_vfs.io()); const io = vfs.io(); diff --git a/zml/io/vfs.zig b/zml/io/vfs.zig index 28377a7c..b9f61550 100644 --- a/zml/io/vfs.zig +++ b/zml/io/vfs.zig @@ -8,6 +8,7 @@ pub const VFS = struct { pub const File = vfs.File; pub const HTTP = vfs.HTTP; pub const HF = vfs.HF; + pub const S3 = vfs.S3; const VFSBase = vfs.VFSBase; const Handles = std.ArrayList(Handle); diff --git a/zml/io/vfs/BUILD.bazel b/zml/io/vfs/BUILD.bazel index 922a69c7..9f79dd24 100644 --- a/zml/io/vfs/BUILD.bazel +++ b/zml/io/vfs/BUILD.bazel @@ -7,6 +7,7 @@ zig_library( "file.zig", "hf.zig", "http.zig", + "s3.zig", ], deps = [ "//stdx", diff --git a/zml/io/vfs/index.zig b/zml/io/vfs/index.zig index 91c1d2e4..605fc8c6 100644 --- a/zml/io/vfs/index.zig +++ b/zml/io/vfs/index.zig @@ -1,8 +1,9 @@ -const std = @import("std"); - const log = std.log.scoped(.@"zml/io/vfs/vfs"); -pub const VFSBase = @import("base.zig").VFSBase; +const std = @import("std"); + pub const File = @import("file.zig").File; -pub const HTTP = @import("http.zig").HTTP; pub const HF = @import("hf.zig").HF; +pub const HTTP = @import("http.zig").HTTP; +pub const S3 = @import("s3.zig").S3; +pub const VFSBase = @import("base.zig").VFSBase; diff --git a/zml/io/vfs/s3.zig b/zml/io/vfs/s3.zig new file mode 100644 index 00000000..288ab054 --- /dev/null +++ b/zml/io/vfs/s3.zig @@ -0,0 +1,684 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const VFSBase = @import("base.zig").VFSBase; + +const log = std.log.scoped(.@"zml/io/vfs/s3"); + +pub const AwsSigV4 = struct { + access_key: []const u8, + secret_key: []const u8, + region: []const u8, + service: []const u8 = "s3", + + const ALGORITHM = "AWS4-HMAC-SHA256"; + pub const UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD"; + + pub fn generateAuthHeader( + self: AwsSigV4, + allocator: std.mem.Allocator, + method: std.http.Method, + endpoint: []const u8, + uri: std.Uri, + timestamp: []const u8, + extra_headers: []const std.http.Header, + ) ![]const u8 { + const date = timestamp[0..8]; + + var buf: [256]u8 = undefined; + const host = getHostForEndpoint(endpoint, &buf); + + // Build sorted headers list + var headers: std.ArrayList(std.http.Header) = .{}; + defer headers.deinit(allocator); + + try headers.append(allocator, .{ .name = "host", .value = host }); + try headers.append(allocator, .{ .name = "x-amz-content-sha256", .value = UNSIGNED_PAYLOAD }); + try headers.append(allocator, .{ .name = "x-amz-date", .value = timestamp }); + + for (extra_headers) |h| { + try headers.append(allocator, .{ .name = h.name, .value = h.value }); + } + + // Important: sort by header name + std.mem.sort( + std.http.Header, + headers.items, + {}, + struct { + fn lessThan(_: void, a: std.http.Header, b: std.http.Header) bool { + return std.mem.order(u8, a.name, b.name) == .lt; + } + }.lessThan, + ); + + // Build canonical headers and signed headers + var canonical_headers_list: std.ArrayList(u8) = .{}; + defer canonical_headers_list.deinit(allocator); + + var signed_headers_list: std.ArrayList(u8) = .{}; + defer signed_headers_list.deinit(allocator); + + for (headers.items, 0..) |header, i| { + // Add to canonical headers: "name:value\n" + try canonical_headers_list.appendSlice(allocator, header.name); + try canonical_headers_list.append(allocator, ':'); + try canonical_headers_list.appendSlice(allocator, header.value); + try canonical_headers_list.append(allocator, '\n'); + + // Add to signed headers list: "name1;name2;name3" + if (i > 0) try signed_headers_list.append(allocator, ';'); + try signed_headers_list.appendSlice(allocator, header.name); + } + + const canonical_headers = canonical_headers_list.items; + const signed_headers = signed_headers_list.items; + + // Build canonical request + var canonical_request_list: std.ArrayList(u8) = .{}; + defer canonical_request_list.deinit(allocator); + + const canonical_uri = if (uri.path.percent_encoded.len > 0) uri.path.percent_encoded else "/"; + const canonical_query = ""; // todo + + // METHOD\nURI\nQUERY\nHEADERS\nSIGNED_HEADERS\nPAYLOAD_HASH + try canonical_request_list.appendSlice(allocator, @tagName(method)); + try canonical_request_list.append(allocator, '\n'); + try canonical_request_list.appendSlice(allocator, canonical_uri); + try canonical_request_list.append(allocator, '\n'); + try canonical_request_list.appendSlice(allocator, canonical_query); + try canonical_request_list.append(allocator, '\n'); + try canonical_request_list.appendSlice(allocator, canonical_headers); + try canonical_request_list.append(allocator, '\n'); + try canonical_request_list.appendSlice(allocator, signed_headers); + try canonical_request_list.append(allocator, '\n'); + try canonical_request_list.appendSlice(allocator, UNSIGNED_PAYLOAD); + + // Hash canonical request + const cr_hash = sha256(canonical_request_list.items); + var cr_hash_hex: [64]u8 = undefined; + _ = std.fmt.bufPrint(&cr_hash_hex, "{x}", .{&cr_hash}) catch unreachable; + + // Build credential scope: date/region/service/aws4_request + const credential_scope = try std.fmt.allocPrint(allocator, "{s}/{s}/{s}/aws4_request", .{ date, self.region, self.service }); + defer allocator.free(credential_scope); + + // Build string to sign + var string_to_sign_list: std.ArrayList(u8) = .{}; + defer string_to_sign_list.deinit(allocator); + + try string_to_sign_list.appendSlice(allocator, ALGORITHM); + try string_to_sign_list.append(allocator, '\n'); + try string_to_sign_list.appendSlice(allocator, timestamp); + try string_to_sign_list.append(allocator, '\n'); + try string_to_sign_list.appendSlice(allocator, credential_scope); + try string_to_sign_list.append(allocator, '\n'); + try string_to_sign_list.appendSlice(allocator, &cr_hash_hex); + + // Calculate signing key + const k_date = hmacSha256WithPrefix("AWS4", self.secret_key, date); + const k_region = hmacSha256(&k_date, self.region); + const k_service = hmacSha256(&k_region, self.service); + const k_signing = hmacSha256(&k_service, "aws4_request"); + + // Calculate signature + const signature = hmacSha256(&k_signing, string_to_sign_list.items); + var signature_hex: [64]u8 = undefined; + _ = std.fmt.bufPrint(&signature_hex, "{x}", .{&signature}) catch unreachable; + + // Build final Authorization header + return try std.fmt.allocPrint(allocator, "{s} Credential={s}/{s}, SignedHeaders={s}, Signature={s}", .{ + ALGORITHM, + self.access_key, + credential_scope, + signed_headers, + signature_hex[0..64], + }); + } + + fn getHostForEndpoint(endpoint: []const u8, buf: []u8) []const u8 { + const uri = std.Uri.parse(endpoint) catch return endpoint; + const host_raw = uri.host orelse return endpoint; + const host = switch (host_raw) { + .raw => |r| r, + .percent_encoded => |p| p, + }; + if (uri.port) |port| { + const len = std.fmt.bufPrint(buf, "{s}:{d}", .{ host, port }) catch return endpoint; + return buf[0..len.len]; + } + return host; + } + + fn hmacSha256WithPrefix(prefix: []const u8, secret: []const u8, data: []const u8) [32]u8 { + var key_buf: [128]u8 = undefined; + @memcpy(key_buf[0..prefix.len], prefix); + @memcpy(key_buf[prefix.len..][0..secret.len], secret); + const key = key_buf[0 .. prefix.len + secret.len]; + + var out: [32]u8 = undefined; + std.crypto.auth.hmac.sha2.HmacSha256.create(&out, data, key); + return out; + } + + fn sha256(data: []const u8) [32]u8 { + var hash: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(data, &hash, .{}); + return hash; + } + + fn hmacSha256(key: []const u8, data: []const u8) [32]u8 { + var out: [32]u8 = undefined; + std.crypto.auth.hmac.sha2.HmacSha256.create(&out, data, key); + return out; + } +}; + +pub const S3 = struct { + const Handle = struct { + pub const Type = enum { file, directory }; + + type: Type, + uri: []const u8, + pos: u64, + size: u64, + + pub fn init(allocator: std.mem.Allocator, handle_type: Type, path: []const u8, size: u64) !Handle { + return .{ .type = handle_type, .uri = try allocator.dupe(u8, path), .pos = 0, .size = size }; + } + + pub fn deinit(self: *Handle, allocator: std.mem.Allocator) void { + allocator.free(self.uri); + } + }; + + pub const Credentials = struct { + access_key: []const u8, + secret_key: []const u8, + endpoint_url: []const u8, + region: []const u8 = "us-east-1", // todo: no default? + + pub fn fromEnv(allocator: std.mem.Allocator) !Credentials { + const access_key = std.process.getEnvVarOwned(allocator, "AWS_ACCESS_KEY_ID") catch |err| { + log.err("AWS_ACCESS_KEY_ID environment variable not set", .{}); + return err; + }; + errdefer allocator.free(access_key); + + const secret_key = std.process.getEnvVarOwned(allocator, "AWS_SECRET_ACCESS_KEY") catch |err| { + log.err("AWS_SECRET_ACCESS_KEY environment variable not set", .{}); + return err; + }; + errdefer allocator.free(secret_key); + + const endpoint_url = std.process.getEnvVarOwned(allocator, "AWS_ENDPOINT_URL") catch + std.process.getEnvVarOwned(allocator, "AWS_ENDPOINT_URL_S3") catch |err| { + log.err("Neither AWS_ENDPOINT_URL nor AWS_ENDPOINT_URL_S3 environment variable is set", .{}); + return err; + }; + errdefer allocator.free(endpoint_url); + + const region = std.process.getEnvVarOwned(allocator, "AWS_REGION") catch + std.process.getEnvVarOwned(allocator, "AWS_DEFAULT_REGION") catch + try allocator.dupe(u8, "us-east-1"); + + return .{ .access_key = access_key, .secret_key = secret_key, .endpoint_url = endpoint_url, .region = region }; + } + + pub fn deinit(self: *Credentials, allocator: std.mem.Allocator) void { + allocator.free(self.access_key); + allocator.free(self.secret_key); + allocator.free(self.endpoint_url); + allocator.free(self.region); + } + }; + + allocator: std.mem.Allocator, + mutex: std.Io.Mutex = .init, + client: *std.http.Client, + access_key: []const u8, + secret_key: []const u8, + endpoint_url: []const u8, + region: []const u8, + handles: stdx.SegmentedList(Handle, 0) = .{}, + closed_handles: std.ArrayList(u32) = .{}, + base: VFSBase, + + pub fn init( + allocator: std.mem.Allocator, + inner: std.Io, + http_client: *std.http.Client, + credentials: Credentials, + ) !S3 { + return .{ + .allocator = allocator, + .base = .init(inner), + .client = http_client, + .access_key = try allocator.dupe(u8, credentials.access_key), + .secret_key = try allocator.dupe(u8, credentials.secret_key), + .endpoint_url = try allocator.dupe(u8, credentials.endpoint_url), + .region = try allocator.dupe(u8, credentials.region), + }; + } + + pub fn fromEnv(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client) !S3 { + var credentials = try Credentials.fromEnv(allocator); + defer credentials.deinit(allocator); + + return init(allocator, inner, http_client, credentials); + } + + pub fn deinit(self: *S3) void { + var idx: usize = 0; + while (idx < self.handles.len) : (idx += 1) { + const is_closed = for (self.closed_handles.items) |closed_idx| { + if (closed_idx == idx) break true; + } else false; + if (!is_closed) self.handles.at(idx).deinit(self.allocator); + } + self.handles.deinit(self.allocator); + self.closed_handles.deinit(self.allocator); + self.allocator.free(self.access_key); + self.allocator.free(self.secret_key); + self.allocator.free(self.endpoint_url); + self.allocator.free(self.region); + } + + pub fn io(self: *S3) std.Io { + return .{ + .userdata = &self.base, + .vtable = &comptime VFSBase.vtable(.{ + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRead = dirRead, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileReadStreaming = fileReadStreaming, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileRealPath = fileRealPath, + }), + }; + } + + fn openHandle(self: *S3) !struct { u32, *Handle } { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + if (self.closed_handles.pop()) |idx| return .{ idx, self.handles.at(idx) }; + return .{ @intCast(self.handles.len), try self.handles.addOne(self.allocator) }; + } + + fn closeHandle(self: *S3, idx: u32) !void { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + self.handles.at(idx).deinit(self.allocator); + try self.closed_handles.append(self.allocator, idx); + } + + fn getFileHandle(self: *S3, file: std.Io.File) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + return self.handles.at(@intCast(file.handle)); + } + + fn getDirHandle(self: *S3, dir: std.Io.Dir) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + return self.handles.at(@intCast(dir.handle)); + } + + fn resolvePath(self: *S3, dir: std.Io.Dir, sub_path: []const u8, out_buffer: []u8) ![]u8 { + if (std.meta.eql(dir, std.Io.Dir.cwd())) { + return try std.fmt.bufPrint(out_buffer, "{s}", .{sub_path}); + } + const handle = self.getDirHandle(dir); + const trimmed_uri = std.mem.trimEnd(u8, handle.uri, "/"); + const trimmed_sub_path = std.mem.trimStart(u8, sub_path, "/"); + + if (trimmed_uri.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_sub_path}); + if (trimmed_sub_path.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_uri}); + return try std.fmt.bufPrint(out_buffer, "{s}/{s}", .{ trimmed_uri, trimmed_sub_path }); + } + + // todo + fn parseBucketKey(path: []const u8) struct { bucket: []const u8, key: []const u8 } { + const trimmed = std.mem.trim(u8, path, "/"); + if (std.mem.indexOfScalar(u8, trimmed, '/')) |idx| { + return .{ .bucket = trimmed[0..idx], .key = if (idx + 1 < trimmed.len) trimmed[idx + 1 ..] else "" }; + } + return .{ .bucket = trimmed, .key = "" }; + } + + // todo + const UrlStyle = enum { + virtual_hosted, // AWS + path_style, // MinIO + }; + + // todo + fn buildS3Url(self: *S3, allocator: std.mem.Allocator, bucket: []const u8, key: []const u8) ![]const u8 { + const trimmed_endpoint = std.mem.trimEnd(u8, self.endpoint_url, "/"); + + if (key.len > 0) { + return try std.fmt.allocPrint(allocator, "{s}/{s}/{s}", .{ trimmed_endpoint, bucket, key }); + } + return try std.fmt.allocPrint(allocator, "{s}/{s}/", .{ trimmed_endpoint, bucket }); + } + + // todo + fn getTimestamp(allocator: std.mem.Allocator, io_: std.Io) ![]const u8 { + const now_ts = try std.Io.Clock.now(.real, io_); + + const now: u64 = @intCast(@divFloor(now_ts.nanoseconds, std.time.ns_per_s)); + const epoch_secs: std.time.epoch.EpochSeconds = .{ .secs = now }; + const year_day = epoch_secs.getEpochDay().calculateYearDay(); + const month_day = year_day.calculateMonthDay(); + const day_seconds = epoch_secs.getDaySeconds(); + + return try std.fmt.allocPrint(allocator, "{d:0>4}{d:0>2}{d:0>2}T{d:0>2}{d:0>2}{d:0>2}Z", .{ + year_day.year, + @intFromEnum(month_day.month), + month_day.day_index + 1, + day_seconds.getHoursIntoDay(), + day_seconds.getMinutesIntoHour(), + day_seconds.getSecondsIntoMinute(), + }); + } + + fn fetchSize(self: *S3, dir: std.Io.Dir, sub_path: []const u8) !u64 { + var path_buffer: [8 * 1024]u8 = undefined; + const path = try self.resolvePath(dir, sub_path, &path_buffer); + const parsed = parseBucketKey(path); + + const url = try self.buildS3Url(self.allocator, parsed.bucket, parsed.key); + defer self.allocator.free(url); + + const uri = std.Uri.parse(url) catch return error.BadPathName; + + const timestamp = try getTimestamp(self.allocator, self.base.inner); + defer self.allocator.free(timestamp); + + const signer: AwsSigV4 = .{ + .access_key = self.access_key, + .secret_key = self.secret_key, + .region = self.region, + }; + + const auth_header = try signer.generateAuthHeader(self.allocator, .HEAD, self.endpoint_url, uri, timestamp, &.{}); + defer self.allocator.free(auth_header); + + var req = try self.client.request(.HEAD, uri, .{ + .redirect_behavior = .not_allowed, + .headers = .{ .accept_encoding = .{ .override = "identity" } }, + .extra_headers = &.{ + .{ .name = "Authorization", .value = auth_header }, + .{ .name = "x-amz-date", .value = timestamp }, + .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, + }, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var redirect_buffer: [8 * 1024]u8 = undefined; + const res = try req.receiveHead(&redirect_buffer); + + switch (res.head.status.class()) { + .success => return res.head.content_length.?, + else => { + log.err("Failed to fetch size for {s}: {s}", .{ url, res.head.bytes }); + return error.ServerError; + }, + } + } + + fn performRead(self: *S3, handle: *Handle, data: []const []u8, offset: u64) !usize { + if (offset >= handle.size) return 0; + + const parsed = parseBucketKey(handle.uri); + + const url = try self.buildS3Url(self.allocator, parsed.bucket, parsed.key); + defer self.allocator.free(url); + + const uri = std.Uri.parse(url) catch return error.BadPathName; + + // Calculate range + var total_bytes: u64 = 0; + for (data) |buf| total_bytes += buf.len; + const remaining = handle.size - offset; + const take = @min(remaining, total_bytes); + const end = offset + take - 1; + + const range_header = try std.fmt.allocPrint(self.allocator, "bytes={d}-{d}", .{ offset, end }); + defer self.allocator.free(range_header); + + const timestamp = try getTimestamp(self.allocator, self.base.inner); + defer self.allocator.free(timestamp); + + const signer: AwsSigV4 = .{ + .access_key = self.access_key, + .secret_key = self.secret_key, + .region = self.region, + }; + + const auth_header = try signer.generateAuthHeader(self.allocator, .GET, self.endpoint_url, uri, timestamp, &.{}); + defer self.allocator.free(auth_header); + + var req = try self.client.request(.GET, uri, .{ + .headers = .{ .accept_encoding = .{ .override = "identity" } }, + .extra_headers = &.{ + .{ .name = "Authorization", .value = auth_header }, + .{ .name = "x-amz-date", .value = timestamp }, + .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, + .{ .name = "Range", .value = range_header }, + }, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var redirect_buffer: [8 * 1024]u8 = undefined; + var res = try req.receiveHead(&redirect_buffer); + + if (res.head.status != .partial_content and res.head.status != .ok) { + log.err("Failed to read {s}: {s}", .{ handle.uri, res.head.bytes }); + return error.RequestFailed; + } + + const content_range = blk: { + var it = res.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { + break :blk parseContentRange(header.value); + } + } + break :blk null; + }; + + const reader = res.reader(&.{}); + + if (content_range) |cr| { + if (cr.start < offset) { + try reader.discardAll(offset - cr.start); + } + } + + return try reader.readSliceShort(data[0]); + } + + const ContentRange = struct { + start: u64, + end: u64, + total: u64, + }; + + fn parseContentRange(value: []const u8) ?ContentRange { + // Format: "bytes start-end/total" + const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; + const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; + const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; + + return .{ + .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, + .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, + .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, + }; + } + + fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.OpenError.SystemResources; + const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .directory, path, 0) catch return std.Io.Dir.OpenError.Unexpected; + return .{ .handle = @intCast(idx) }; + } + + fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getDirHandle(dir); + return .{ + .inode = 0, + .nlink = 0, + .size = handle.size, + .permissions = .fromMode(0o444), + .kind = .directory, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + }; + } + + fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const size = self.fetchSize(dir, sub_path) catch return std.Io.Dir.StatFileError.Unexpected; + return .{ + .inode = 0, + .nlink = 0, + .size = size, + .permissions = .fromMode(0o444), + .kind = .file, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + }; + } + + fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} + + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const size = self.fetchSize(dir, sub_path) catch return std.Io.File.OpenError.Unexpected; + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; + return .{ .handle = @intCast(idx) }; + } + + fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + for (dirs) |dir| self.closeHandle(@intCast(dir.handle)) catch unreachable; + } + + fn dirRead(_: ?*anyopaque, _: *std.Io.Dir.Reader, _: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { + log.err("dirRead unsupported for S3 VFS", .{}); + return std.Io.Dir.Reader.Error.Unexpected; + } + + fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getDirHandle(dir); + const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.Dir.RealPathError.SystemResources; + return path.len; + } + + fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const real_path = self.resolvePath(dir, path_name, out_buffer) catch return std.Io.Dir.RealPathFileError.NameTooLong; + return real_path.len; + } + + fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return .{ + .inode = @intCast(file.handle), + .nlink = 0, + .size = handle.size, + .permissions = .fromMode(0o444), + .kind = .file, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + }; + } + + fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.getFileHandle(file).size; + } + + fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + for (files) |file| self.closeHandle(@intCast(file.handle)) catch unreachable; + } + + fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + const total = self.performRead(handle, data, handle.pos) catch |err| { + log.err("Failed to read {s} at {d}: {any}", .{ handle.uri, handle.pos, err }); + return std.Io.File.Reader.Error.Unexpected; + }; + handle.pos += @intCast(total); + return total; + } + + fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return self.performRead(handle, data, offset) catch |err| { + log.err("Failed to read {s} at {d}: {any}", .{ handle.uri, offset, err }); + return std.Io.File.ReadPositionalError.Unexpected; + }; + } + + fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + + if (relative_offset >= 0) { + handle.pos = handle.pos +| @as(u64, @intCast(relative_offset)); + } else { + const abs_offset: u64 = @intCast(-relative_offset); + if (abs_offset > handle.pos) { + handle.pos = 0; + } else { + handle.pos -= abs_offset; + } + } + } + + fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + handle.pos = absolute_offset; + } + + fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.File.RealPathError.SystemResources; + return path.len; + } +}; From de41199c6888f17f9940c9fe3e8018022a497627 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 15 Jan 2026 18:23:53 +0000 Subject: [PATCH 239/881] examples/llama: fix buffer size allocation --- examples/llama/main.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 0370ceb7..f3f73512 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -170,12 +170,12 @@ pub fn main() !void { allocator, platform, .{ - .size = args.reader_buffer_size.bytes() * 1024 * 1024, + .size = args.reader_buffer_size.bytes(), .concurrency = load_model_buffers.limit, .dma = true, }, .{ - .size = args.writer_buffer_size.bytes() * 1024 * 1024, + .size = args.writer_buffer_size.bytes(), .concurrency = load_model_buffers.limit, .dma = true, }, From 2ea598a90fd5ba53d6f24da50725e94a25ed308f Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 15 Jan 2026 18:20:15 +0000 Subject: [PATCH 240/881] pjrt: missing fixes --- pjrt/ffi.zig | 63 ++++++++++++++++++++++++++------------------------- pjrt/pjrt.zig | 8 +++---- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index 8d28bd66..b8d2a241 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -82,10 +82,11 @@ pub const Api = opaque { pub const inner = TransmuteMixin(Api, c.XLA_FFI_Api).to; pub fn stream(self: *const Api, context: *const ExecutionContext) *pjrt.Stream { - var ret = pjrt.pjrtStruct2(c.XLA_FFI_Stream_Get_Args, .{ + var ret: pjrt.meta.Struct(c.XLA_FFI_Stream_Get_Args) = .{ .ctx = @constCast(context.inner()), - }); - const result = self.inner().XLA_FFI_Stream_Get.?(&ret); + }; + + const result = self.inner().XLA_FFI_Stream_Get.?(@ptrCast(&ret)); if (result) |ffi_error| { const err = Error.fromInner(ffi_error); defer err.destroy(self); @@ -98,12 +99,12 @@ pub const Api = opaque { } pub fn allocateDeviceMemory(self: *const Api, context: *const ExecutionContext, size: usize, alignment: usize) pjrt.ApiError!*anyopaque { - var ret = pjrt.pjrtStruct2(c.XLA_FFI_DeviceMemory_Allocate_Args, .{ + var ret: pjrt.meta.Struct(c.XLA_FFI_DeviceMemory_Allocate_Args) = .{ .ctx = @constCast(context.inner()), .size = size, .alignment = alignment, - }); - const result = self.inner().XLA_FFI_DeviceMemory_Allocate.?(&ret); + }; + const result = self.inner().XLA_FFI_DeviceMemory_Allocate.?(@ptrCast(&ret)); if (result) |ffi_error| { const err = Error.fromInner(ffi_error); @@ -118,12 +119,12 @@ pub const Api = opaque { } pub fn freeDeviceMemory(self: *const Api, context: *const ExecutionContext, data: *anyopaque, size: usize) pjrt.ApiError!void { - var ret = pjrt.pjrtStruct2(c.XLA_FFI_DeviceMemory_Free_Args, .{ + var ret: pjrt.meta.Struct(c.XLA_FFI_DeviceMemory_Free_Args) = .{ .ctx = @constCast(context.inner()), .size = size, .data = data, - }); - const result = self.inner().XLA_FFI_DeviceMemory_Free.?(&ret); + }; + const result = self.inner().XLA_FFI_DeviceMemory_Free.?(@ptrCast(&ret)); if (result) |ffi_error| { const err = Error.fromInner(ffi_error); @@ -177,10 +178,10 @@ pub const ExecutionContext = opaque { } pub fn getDeviceOrdinal(self: *const ExecutionContext, api: *const Api) pjrt.ApiError!i32 { - var ret = pjrt.pjrtStruct2(c.XLA_FFI_DeviceOrdinal_Get_Args, .{ + var ret: pjrt.meta.Struct(c.XLA_FFI_DeviceOrdinal_Get_Args) = .{ .ctx = @constCast(self.inner()), - }); - const result = api.inner().XLA_FFI_DeviceOrdinal_Get.?(&ret); + }; + const result = api.inner().XLA_FFI_DeviceOrdinal_Get.?(@ptrCast(&ret)); if (result) |ffi_error| { const err = Error.fromInner(ffi_error); @@ -197,13 +198,13 @@ pub const ExecutionContext = opaque { const Task = fn (*anyopaque) void; pub fn scheduleTask(self: *const ExecutionContext, api: *const Api, task: *const Task, data: *anyopaque) pjrt.ApiError!void { - var ret = pjrt.pjrtStruct2(c.XLA_FFI_ThreadPool_Schedule_Args, .{ + var ret: pjrt.meta.Struct(c.XLA_FFI_ThreadPool_Schedule_Args) = .{ .ctx = @constCast(self.inner()), .task = @as(?*const c.XLA_FFI_Task, @ptrCast(@alignCast(task))), .data = data, - }); + }; - const result = api.inner().XLA_FFI_ThreadPool_Schedule.?(&ret); + const result = api.inner().XLA_FFI_ThreadPool_Schedule.?(@ptrCast(&ret)); if (result) |ffi_error| { const err = Error.fromInner(ffi_error); @@ -445,23 +446,23 @@ pub const Error = opaque { pub const ok: ?*Error = null; pub fn create(api: *const Api, error_code: ErrorCode, message: []const u8) *Error { - var ret = pjrt.pjrtStruct2(c.XLA_FFI_Error_Create_Args, .{ + var ret: pjrt.meta.Struct(c.XLA_FFI_Error_Create_Args) = .{ .message = message.ptr, .errc = @intFromEnum(error_code), - }); - return fromInner(api.inner().XLA_FFI_Error_Create.?(&ret).?); + }; + return fromInner(api.inner().XLA_FFI_Error_Create.?(@ptrCast(&ret)).?); } pub fn destroy(err: *Error, api: *const Api) void { - var ret = pjrt.pjrtStruct2(c.XLA_FFI_Error_Destroy_Args, .{ .@"error" = err.inner() }); - api.inner().XLA_FFI_Error_Destroy.?(&ret); + var ret: pjrt.meta.Struct(c.XLA_FFI_Error_Destroy_Args) = .{ .@"error" = err.inner() }; + api.inner().XLA_FFI_Error_Destroy.?(@ptrCast(&ret)); } pub fn getMessage(err: *Error, api: *const Api) [:0]const u8 { - var ret = pjrt.pjrtStruct2(c.XLA_FFI_Error_GetMessage_Args, .{ + var ret: pjrt.meta.Struct(c.XLA_FFI_Error_GetMessage_Args) = .{ .@"error" = err.inner(), - }); - api.inner().XLA_FFI_Error_GetMessage.?(&ret); + }; + api.inner().XLA_FFI_Error_GetMessage.?(@ptrCast(&ret)); return std.mem.span(ret.message); } }; @@ -471,8 +472,8 @@ pub const Future = opaque { pub const fromInner = TransmuteMixin(Future, c.XLA_FFI_Future).from; pub fn create(api: *const Api) pjrt.ApiError!*Future { - var ret = pjrt.pjrtStruct2(c.XLA_FFI_Future_Create_Args, .{}); - const result = api.inner().XLA_FFI_Future_Create.?(&ret); + var ret: pjrt.meta.Struct(c.XLA_FFI_Future_Create_Args) = .{}; + const result = api.inner().XLA_FFI_Future_Create.?(@ptrCast(&ret)); if (result) |ffi_error| { const err = Error.fromInner(ffi_error); @@ -487,11 +488,11 @@ pub const Future = opaque { } pub fn setAvailable(self: *Future, api: *const Api) pjrt.ApiError!void { - var ret = pjrt.pjrtStruct2(c.XLA_FFI_Future_SetAvailable_Args, .{ + var ret: pjrt.meta.Struct(c.XLA_FFI_Future_SetAvailable_Args) = .{ .future = self.inner(), - }); + }; - const result = api.inner().XLA_FFI_Future_SetAvailable.?(&ret); + const result = api.inner().XLA_FFI_Future_SetAvailable.?(@ptrCast(&ret)); if (result) |ffi_error| { const err = Error.fromInner(ffi_error); @@ -504,12 +505,12 @@ pub const Future = opaque { } pub fn setError(self: *Future, api: *const Api, err: *Error) pjrt.ApiError!void { - var ret = pjrt.pjrtStruct2(c.XLA_FFI_Future_SetError_Args, .{ + var ret: pjrt.meta.Struct(c.XLA_FFI_Future_SetError_Args) = .{ .future = self.inner(), .@"error" = err.inner(), - }); + }; - const result = api.inner().XLA_FFI_Future_SetError.?(&ret); + const result = api.inner().XLA_FFI_Future_SetError.?(@ptrCast(&ret)); if (result) |ffi_error| { const err2 = Error.fromInner(ffi_error); diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index b6cbcaea..0a1c5003 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -11,7 +11,7 @@ test { std.testing.refAllDecls(@This()); } -const meta = struct { +pub const meta = struct { // We could calculate it like PJRT does, but it turns out that some of those // were wrong in PJRT itself [1], which gets propagated to binary plugins. In // order to mirror that, we just the value as computed by PJRT itself, through @@ -252,8 +252,8 @@ pub const Api = struct { } pub fn ffi(api: *const Api) ?Ffi { - if (api.extension(c.PJRT_FFI_Extension, c.PJRT_Extension_Type_FFI)) |ext| { - return .{ .inner = ext }; + if (api.extension(.ffi)) |ext| { + return .{ .inner = ext.ffi }; } return null; } @@ -1441,7 +1441,7 @@ pub const Ffi = extern struct { .platform_name_size = platform_name.len, .traits = @bitCast(traits), }; - const result = self.inner.register_handler.?(&ret); + const result = self.inner.register_handler.?(@ptrCast(&ret)); if (result) |pjrt_c_error| { const pjrt_error: *Error = @ptrCast(pjrt_c_error); log.err("registerFfi error: {s}", .{pjrt_error.getMessage(api)}); From 7bbce6c3d0f42b8d5b8a252e87e8f9ba547e84e3 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 16 Jan 2026 10:59:12 +0100 Subject: [PATCH 241/881] Attention Backend on ZML V2 (#373) --- MODULE.bazel | 2 +- examples/llama/llama.zig | 33 +- examples/llama/main.zig | 55 +++- mlir/mlir.zig | 21 ++ pjrt/pjrt.zig | 2 +- runtimes/cuda/BUILD.bazel | 6 + third_party/flashattn/BUILD.bazel | 11 + third_party/flashattn/repo.bzl | 13 + third_party/non_module_deps.bzl | 2 + zml/BUILD.bazel | 4 + zml/attention/attention.zig | 120 ++++++++ zml/attention/flashattn/bindings.zig | 417 ++++++++++++++++++++++++++ zml/attention/flashattn/flashattn.zig | 195 ++++++++++++ zml/ops.zig | 27 +- zml/platform.zig | 33 +- zml/testing.zig | 2 +- zml/zml.zig | 2 + 17 files changed, 913 insertions(+), 32 deletions(-) create mode 100644 third_party/flashattn/BUILD.bazel create mode 100644 third_party/flashattn/repo.bzl create mode 100644 zml/attention/attention.zig create mode 100644 zml/attention/flashattn/bindings.zig create mode 100644 zml/attention/flashattn/flashattn.zig diff --git a/MODULE.bazel b/MODULE.bazel index 266c86b2..14218ae3 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -113,7 +113,7 @@ use_repo(zls, "zls_toolchains") register_toolchains("@zls_toolchains//:all") non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") -use_repo(non_module_deps, "arocc", "com_google_sentencepiece", "mnist", "org_swig_swig", "translate-c", "xla") +use_repo(non_module_deps, "arocc", "com_google_sentencepiece", "mnist", "org_swig_swig", "translate-c", "xla", "flashattn") xla = use_extension("//third_party/xla:xla.bzl", "xla") use_repo( diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 3a26bb06..a1ba1016 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -106,10 +106,12 @@ pub const LlamaLM = struct { token_index: Tensor, kv_cache: KvCache, rng: Tensor.Rng, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, ) struct { Tensor, KvCache, Tensor.Rng } { stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Llama ! Expected >=1d tokens and 0d token_index, got: {f} and {f}", .{ tokens_, token_index }); const tokens = tokens_.withPartialTags(.{.s}); - const out, const updated_kv_cache = self.model.forward(tokens, token_index, kv_cache); + const out, const updated_kv_cache = self.model.forward(tokens, token_index, kv_cache, attention_metadata, attention_parameters); const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; } @@ -207,13 +209,20 @@ pub const Llama = struct { /// Forward one token, using KV cache for previous tokens. /// Returns result and updated KV cache. - pub fn forward(self: Llama, tokens: Tensor, token_index: Tensor, kv_cache: KvCache) struct { Tensor, KvCache } { + pub fn forward( + self: Llama, + tokens: Tensor, + token_index: Tensor, + kv_cache: KvCache, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, + ) struct { Tensor, KvCache } { const embeds = embed(self.embed_tokens, tokens); var hidden = embeds; var updated_kv_cache = kv_cache; for (self.layers, 0..) |layer, i| { - hidden, updated_kv_cache = layer.forward(hidden, token_index, updated_kv_cache.atLayer(i)); + hidden, updated_kv_cache = layer.forward(hidden, token_index, updated_kv_cache.atLayer(i), attention_metadata, attention_parameters); } const output = self.norm.forward(hidden); @@ -252,13 +261,15 @@ pub const TransformerLayer = struct { x0: Tensor, token_index: Tensor, kv_cache: KvCache, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, ) struct { Tensor, KvCache } { // Self Attention //log.debug("TransformerLayer({f}) -> {f}", .{ x0, self.input_layernorm.forward(x0) }); stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {f}", .{x0}); const x0_normalized = self.input_layernorm.forward(x0); - const delta0, const updated_kv_cache = self.self_attn.forward(x0_normalized, token_index, kv_cache); + const delta0, const updated_kv_cache = self.self_attn.forward(x0_normalized, token_index, kv_cache, attention_metadata, attention_parameters); const x1 = x0.add(delta0); // Fully Connected @@ -385,20 +396,14 @@ pub const SelfAttn = struct { x: Tensor, token_index: Tensor, kv_cache: KvCache, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, ) struct { Tensor, KvCache } { const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; var q = self.q_proj.forward(x).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); var k = self.k_proj.forward(x).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); var v = self.v_proj.forward(x).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); - // Generate the attention mask. - const seq_len = kv_cache.k.dim(.k); - var attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, x.dtype(), null); - - // Note: in Pytorch it would be very inefficient to generate the full attn_mask, - // then slice into it, but XLA is able to optimize this correctly. - attn_mask = attn_mask.gatherSlices(zml.Shape.init(.{ .q = x.dim(.s) }, attn_mask.dtype()), token_index.reshape(.{ .coord = 1 }), .{}); - // In self-attention, .s axis is used both for keys and queries. const pos_index = b: { const temp = Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()).withTags(.{.s}).broad(zml.Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); @@ -418,8 +423,8 @@ pub const SelfAttn = struct { k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); - const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask, .allow_cudnn = true }); - // const attn_output = zml.nn.sdpaMemEfficient(q, k, v, .{ .attn_mask = attn_mask }, .{ .q_chunk_size = 4096, .k_chunk_size = 1024 }); + const attn_output = zml.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters); + const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); return .{ self.o_proj.forward(attn), new_kv_cache }; } diff --git a/examples/llama/main.zig b/examples/llama/main.zig index f3f73512..b8aa5bea 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -26,6 +26,7 @@ const CliArgs = struct { reader_buffer_size: stdx.flags.ByteSize = .{ .value = 32, .unit = .mib }, writer_buffer_size: stdx.flags.ByteSize = .{ .value = 32, .unit = .mib }, async_limit: ?u32 = null, + backend: ?zml.attention.Backend = null, pub const help = \\Usage: llama --model= [options] @@ -36,6 +37,7 @@ const CliArgs = struct { \\ --model= Path to model (local path or HuggingFace repo) \\ --prompt= Input prompt (reads from stdin if not provided) \\ --seqlen= Maximum sequence length (default: 512) + \\ --backend= Attention backend to use ([vanilla, cuda_fa2, cuda_fa3], default: auto-selection) \\ --reader-buffer-size= Reader buffer size (default: 32MiB) \\ --writer-buffer-size= Writer buffer size (default: 32MiB) \\ --async-limit= Async I/O concurrency limit @@ -121,10 +123,24 @@ pub fn main() !void { }, }; + var platform: zml.Platform = try .auto(io, .{}); + defer platform.deinit(); + var it = platform.devicesIterator(); + log.info("Devices:", .{}); + while (it.next()) |device| { + log.info("\t- {f}", .{device}); + } + // Initialize the Llama struct and map the content of the .safetensors to the model tensors. const llama_model: llama.LlamaLM = try .init(allocator, store.view(), config, llama_options); defer llama_model.deinit(allocator); + const backend = args.backend orelse b: { + const selected = zml.attention.Backend.auto(platform); + log.info("Selected backend: {}", .{selected}); + break :b selected; + }; + // Specify shapes of input arguments const dtype = llama_model.model.embed_tokens.weight.dtype(); const llama_parameters: LlamaParameters = .{ @@ -138,6 +154,8 @@ pub fn main() !void { .hd = config.head_dim orelse @divExact(config.hidden_size, config.num_attention_heads), }, dtype)), .rng = .init(), + .attention_metadata = .init(.fromBackend(backend, @intCast(args.seqlen))), + .attention_parameters = .init(.fromBackend(backend)), }; var tokenizer_future = io.async(loadTokenizer, .{ allocator, io, repo }); @@ -146,14 +164,6 @@ pub fn main() !void { v.deinit(); } - var platform: zml.Platform = try .auto(io, .{}); - defer platform.deinit(); - var it = platform.devicesIterator(); - log.info("Devices:", .{}); - while (it.next()) |device| { - log.info("\t- {f}", .{device}); - } - var compiled_model_result_future = io.async(compileModel, .{ allocator, io, platform, llama_model, llama_parameters }); errdefer if (compiled_model_result_future.cancel(io)) |v| { defer v.prefill_exe.deinit(); @@ -210,6 +220,9 @@ pub fn main() !void { var kv_cache_buffers = try llama_parameters.kv_cache.initBuffer(io, platform); defer llama.KvCache.deinitBuffer(&kv_cache_buffers); + var attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = try llama_parameters.attention_metadata.initBuffer(io, platform); + defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); + var tokenizer = try tokenizer_future.await(io); defer tokenizer.deinit(); @@ -230,6 +243,7 @@ pub fn main() !void { compiled_model_result.prefill_exe, compiled_model_result.decode_exe, &kv_cache_buffers, + &attention_metadata_buffers, tokenizer, config, llama_options, @@ -281,6 +295,8 @@ const LlamaParameters = struct { token_index: zml.Tensor, kv_cache: llama.KvCache, rng: zml.Tensor.Rng, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, }; const CompileModelResult = struct { @@ -299,7 +315,14 @@ fn compileModel(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform var timer_ = try std.time.Timer.start(); log.info("Compiling prefill", .{}); defer log.info("Compiled prefill [{D}]", .{timer_.read()}); - return platform_.compile(allocator_, io_, llama_model_, .forward, .{ parameters_.prefill_tokens, parameters_.token_index, parameters_.kv_cache, parameters_.rng }); + return platform_.compile(allocator_, io_, llama_model_, .forward, .{ + parameters_.prefill_tokens, + parameters_.token_index, + parameters_.kv_cache, + parameters_.rng, + parameters_.attention_metadata, + parameters_.attention_parameters, + }); } }.call, .{ allocator, io, platform, llama_model, parameters }); errdefer if (prefill_future.cancel(io)) |v| v.deinit() else |_| {}; @@ -309,7 +332,14 @@ fn compileModel(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform var timer_ = try std.time.Timer.start(); log.info("Compiling decode", .{}); defer log.info("Compiled decode [{D}]", .{timer_.read()}); - return platform_.compile(allocator_, io_, llama_model_, .forward, .{ parameters_.decode_tokens, parameters_.token_index, parameters_.kv_cache, parameters_.rng }); + return platform_.compile(allocator_, io_, llama_model_, .forward, .{ + parameters_.decode_tokens, + parameters_.token_index, + parameters_.kv_cache, + parameters_.rng, + parameters_.attention_metadata, + parameters_.attention_parameters, + }); } }.call, .{ allocator, io, platform, llama_model, parameters }); errdefer if (decode_future.cancel(io)) |v| v.deinit() else |_| {}; @@ -427,6 +457,7 @@ pub fn generateText( prefill_exe: zml.exe.Exe, decode_exe: zml.exe.Exe, kv_cache_buffers: *zml.Bufferized(llama.KvCache), + attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), tokenizer: zml.tokenizer.Tokenizer, config: LlamaLM.Config, options: LlamaLM.Options, @@ -467,7 +498,7 @@ pub fn generateText( var prefill_token_pos_buffer = try zml.Buffer.scalar(io, platform, 0, .u32); defer prefill_token_pos_buffer.deinit(); - prefill_args.set(.{ llama_buffers, prefill_tokens_buffer, prefill_token_pos_buffer, kv_cache_buffers, rng_buffers }); + prefill_args.set(.{ llama_buffers, prefill_tokens_buffer, prefill_token_pos_buffer, kv_cache_buffers, rng_buffers, attention_metadata_buffers }); prefill_exe.call(prefill_args, &prefill_results); @@ -520,7 +551,7 @@ pub fn generateText( defer token_pos_buffer.deinit(); // call to generate the next token - decode_args.set(.{ llama_buffers, current_token_buffer, token_pos_buffer, kv_cache_buffers, rng_buffers }); + decode_args.set(.{ llama_buffers, current_token_buffer, token_pos_buffer, kv_cache_buffers, rng_buffers, attention_metadata_buffers }); decode_exe.call(decode_args, &decode_results); diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 79177cf7..6f4bef3b 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -583,6 +583,27 @@ pub fn integerAttribute(ctx: *Context, it: IntegerTypes, value_: anytype) *const return @ptrCast(IntegerAttribute.init(ctx, it, value_)); } +pub const FloatAttribute = opaque { + const M = Methods(FloatAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAFloat; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, comptime ft: FloatTypes, value_: f64) *const FloatAttribute { + return @ptrCast(c.mlirFloatAttrDoubleGet( + ctx.ptr(), + FloatType(ft).get(ctx).ptr(), + value_, + ).ptr); + } +}; + +pub fn floatAttribute(ctx: *Context, comptime ft: FloatTypes, value_: anytype) *const Attribute { + return @ptrCast(FloatAttribute.init(ctx, ft, value_)); +} + pub const BoolAttribute = opaque { const M = Methods(BoolAttribute, c.MlirAttribute); diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 0a1c5003..000c1989 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -1436,7 +1436,7 @@ pub const Ffi = extern struct { var ret: meta.Struct(c.PJRT_FFI_Register_Handler_Args) = .{ .target_name = target_name.ptr, .target_name_size = target_name.len, - .handler = @ptrCast(@constCast(func)), + .handler = @as(?*anyopaque, @ptrCast(@constCast(func))), .platform_name = platform_name.ptr, .platform_name_size = platform_name.len, .traits = @bitCast(traits), diff --git a/runtimes/cuda/BUILD.bazel b/runtimes/cuda/BUILD.bazel index 6009eb72..a789a24e 100644 --- a/runtimes/cuda/BUILD.bazel +++ b/runtimes/cuda/BUILD.bazel @@ -39,6 +39,12 @@ zig_library( ], "//conditions:default": [":empty"], }), + data = select({ + "//runtimes:cuda.enabled": [ + "@flashattn//:libflashattn" + ], + "//conditions:default": [":empty"], + }), ) filegroup( diff --git a/third_party/flashattn/BUILD.bazel b/third_party/flashattn/BUILD.bazel new file mode 100644 index 00000000..025e0666 --- /dev/null +++ b/third_party/flashattn/BUILD.bazel @@ -0,0 +1,11 @@ +filegroup( + name = "libflashattn", + visibility = ["//visibility:public"], + srcs = ["flashattn/lib/libflashattn.so"], +) + +cc_library( + name = "c", + hdrs = ["flashattn/include/capi.h"], + visibility = ["//visibility:public"], +) diff --git a/third_party/flashattn/repo.bzl b/third_party/flashattn/repo.bzl new file mode 100644 index 00000000..84781cb6 --- /dev/null +++ b/third_party/flashattn/repo.bzl @@ -0,0 +1,13 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") + +def repo(): + http_archive( + name = "flashattn", + url = "https://github.com/zml/flash-attention/releases/download/v0.0.6-rc6/flashattn.tar.gz", + sha256 = "ff44c0657c1e0d41883b6bc034340f68afe9c4f8cddfd9232c039f4c6b5e1da6", + build_file = "//:third_party/flashattn/BUILD.bazel", + ) + + #new_local_repository(name = "flashattn", build_file="//:third_party/flashattn/BUILD.bazel", path="/home/corendos/flashattn/") + diff --git a/third_party/non_module_deps.bzl b/third_party/non_module_deps.bzl index 36791eb3..5790cc3e 100644 --- a/third_party/non_module_deps.bzl +++ b/third_party/non_module_deps.bzl @@ -4,6 +4,7 @@ load("//third_party/mnist:repo.bzl", mnist = "repo") load("//third_party/org_swig_swig:repo.bzl", org_swig_swig = "repo") load("//third_party/translate-c:repo.bzl", translate_c = "repo") load("//third_party/xla:repo.bzl", xla = "repo") +load("//third_party/flashattn:repo.bzl", flashattn = "repo") def _non_module_deps_impl(mctx): com_google_sentencepiece() @@ -12,6 +13,7 @@ def _non_module_deps_impl(mctx): xla() arocc() translate_c() + flashattn() return mctx.extension_metadata( reproducible = True, diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index aa5774c0..f064a956 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -39,6 +39,9 @@ zig_library( "tensor.zig", "testing.zig", "zml.zig", + "attention/attention.zig", + "attention/flashattn/flashattn.zig", + "attention/flashattn/bindings.zig", ], main = "zml.zig", visibility = ["//visibility:public"], @@ -59,6 +62,7 @@ zig_library( "//zml/tokenizer", "//zml/io", "@rules_zig//zig/runfiles", + "@flashattn//:c", ], ) diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig new file mode 100644 index 00000000..b9dd80a9 --- /dev/null +++ b/zml/attention/attention.zig @@ -0,0 +1,120 @@ +const std = @import("std"); + +const zml = @import("../zml.zig"); +pub const flashattn = @import("flashattn/flashattn.zig"); + +const Attention = @This(); + +pub const Backend = enum { + vanilla, + cuda_fa2, + cuda_fa3, + + pub fn auto(platform: zml.Platform) Backend { + return switch (platform.target) { + .cuda => b: { + const first_device = platform.pjrt_client.devices(platform.pjrt_api)[0]; + + if (zml.platform.cuda.tryGetComputeCapabilities(platform, first_device)) |cc| { + break :b if (std.mem.eql(u8, cc, "9.0")) + .cuda_fa3 + else + .cuda_fa2; + } + + break :b .vanilla; + }, + else => .vanilla, + }; + } +}; + +pub const Parameters = union(Backend) { + vanilla: void, + cuda_fa2: flashattn.fa2.Parameters, + cuda_fa3: flashattn.fa3.Parameters, + + pub const InitOptions = union(Backend) { + vanilla: void, + cuda_fa2: flashattn.fa2.Parameters.InitOptions, + cuda_fa3: flashattn.fa3.Parameters.InitOptions, + + pub fn fromBackend(backend: Backend) InitOptions { + return switch (backend) { + .vanilla => .{ .vanilla = {} }, + .cuda_fa2 => .{ .cuda_fa2 = .{} }, + .cuda_fa3 => .{ .cuda_fa3 = .{} }, + }; + } + }; + + pub fn init(opts: InitOptions) Parameters { + return switch (opts) { + .vanilla => .{ .vanilla = {} }, + .cuda_fa2 => |v| .{ .cuda_fa2 = .init(v) }, + .cuda_fa3 => |v| .{ .cuda_fa3 = .init(v) }, + }; + } +}; + +pub const Metadata = union(Backend) { + vanilla: void, + cuda_fa2: flashattn.fa2.Metadata, + cuda_fa3: flashattn.fa3.Metadata, + + pub const InitOptions = union(Backend) { + vanilla: void, + cuda_fa2: flashattn.fa2.Metadata.InitOptions, + cuda_fa3: flashattn.fa3.Metadata.InitOptions, + + pub fn fromBackend(backend: Backend, seqlen: i64) InitOptions { + return switch (backend) { + .vanilla => .{ .vanilla = {} }, + .cuda_fa2 => .{ .cuda_fa2 = .{ .seqlen = seqlen } }, + .cuda_fa3 => .{ .cuda_fa3 = .{ .seqlen = seqlen } }, + }; + } + }; + + pub fn init(opts: InitOptions) Metadata { + return switch (opts) { + .vanilla => .{ .vanilla = {} }, + .cuda_fa2 => |o| .{ .cuda_fa2 = flashattn.fa2.Metadata.init(o) }, + .cuda_fa3 => |o| .{ .cuda_fa3 = flashattn.fa3.Metadata.init(o) }, + }; + } + + pub fn initBuffer(self: Metadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(Metadata) { + return switch (self) { + .vanilla => .{ .vanilla = {} }, + .cuda_fa2 => |v| .{ .cuda_fa2 = try v.initBuffer(io, platform) }, + .cuda_fa3 => |v| .{ .cuda_fa3 = try v.initBuffer(io, platform) }, + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { + switch (self.*) { + .vanilla => {}, + .cuda_fa2 => |*v| flashattn.fa2.Metadata.deinitBuffer(v), + .cuda_fa3 => |*v| flashattn.fa3.Metadata.deinitBuffer(v), + } + } +}; + +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { + return switch (parameters) { + .vanilla => b: { + // Generate the attention mask. + const seq_len = k.dim(.k); + var attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, q.dtype(), null); + + // Note: in Pytorch it would be very inefficient to generate the full attn_mask, + // then slice into it, but XLA is able to optimize this correctly. + attn_mask = attn_mask.gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, attn_mask.dtype()), token_index.reshape(.{ .coord = 1 }), .{}); + const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask, .allow_cudnn = true }); + break :b attn_output; + }, + .cuda_fa2 => flashattn.fa2.attention(q, k, v, token_index, metadata.cuda_fa2, parameters.cuda_fa2), + .cuda_fa3 => flashattn.fa3.attention(q, k, v, token_index, metadata.cuda_fa3, parameters.cuda_fa3), + }; +} diff --git a/zml/attention/flashattn/bindings.zig b/zml/attention/flashattn/bindings.zig new file mode 100644 index 00000000..63ab1c7d --- /dev/null +++ b/zml/attention/flashattn/bindings.zig @@ -0,0 +1,417 @@ +const std = @import("std"); + +const bazel_builtin = @import("bazel_builtin"); +const c = @import("c"); +const runfiles = @import("runfiles"); + +const zml = @import("../../zml.zig"); +const ffi = zml.pjrt.ffi; + +fn flashattnDataTypeFromZmlDataType(dtype: zml.DataType) c.DataType { + return switch (dtype) { + .f32 => c.CAPI_FLOAT, + .f16 => c.CAPI_FLOAT16, + .bf16 => c.CAPI_BFLOAT16, + .i32 => c.CAPI_INT32, + .i8 => c.CAPI_INT8, + else => unreachable, + }; +} + +pub fn tensorFromBuffer(buffer: zml.Buffer, platform: zml.Platform) !c.FlashattnTensor { + const rank: usize = buffer.rank(); + var dims: [8]i64 = undefined; + @memcpy(dims[0..rank], buffer.shape().dims()); + var strides: [8]i64 = undefined; + @memcpy(strides[0..rank], buffer.shape().withDtype(.u8).computeStrides().constSlice()); + + const tensor: c.FlashattnTensor = .{ + .ptr = try buffer._shards.get(0).getOpaqueDeviceMemoryDataPointer(platform.pjrt_api), + .rank = buffer.rank(), + .dims = dims, + .strides = strides, + .dtype = flashattnDataTypeFromZmlDataType(buffer.dtype()), + }; + + return tensor; +} + +pub const Buffer = struct { + shape: zml.Shape, + ptr: ?*anyopaque, + + pub fn toFlashattnTensor(buffer: Buffer) c.FlashattnTensor { + const rank: usize = buffer.shape.rank(); + var dims: [8]i64 = undefined; + @memcpy(dims[0..rank], buffer.shape.dims()); + var strides: [8]i64 = undefined; + @memcpy(strides[0..rank], buffer.shape.withDtype(.u8).computeByteStrides().constSlice()); + + const tensor: c.FlashattnTensor = .{ + .ptr = buffer.ptr, + .rank = buffer.shape.rank(), + .dims = dims, + .strides = strides, + .dtype = flashattnDataTypeFromZmlDataType(buffer.shape.dtype()), + }; + + return tensor; + } +}; + +fn getPlatform(call_frame: *ffi.CallFrame) zml.Platform { + const pjrt_api_ptr = call_frame.attrs.getByName(.scalar, "pjrt_api") orelse unreachable; + std.debug.assert(pjrt_api_ptr.dtype == .u64); + const pjrt_api: ?*zml.pjrt.Api = @ptrFromInt(pjrt_api_ptr.get(usize)); + + const pjrt_client_ptr = call_frame.attrs.getByName(.scalar, "pjrt_client") orelse unreachable; + std.debug.assert(pjrt_client_ptr.dtype == .u64); + const pjrt_client: ?*zml.pjrt.Client = @ptrFromInt(pjrt_client_ptr.get(usize)); + + return .{ .target = .cuda, .pjrt_api = pjrt_api.?, .pjrt_client = pjrt_client.? }; +} + +fn dataTypeFromFfiDataType(ffi_dt: ffi.DataType) zml.DataType { + return switch (ffi_dt) { + .bool => .bool, + .i8 => .i8, + .i16 => .i16, + .i32 => .i32, + .i64 => .i64, + .u8 => .u8, + .u16 => .u16, + .u32 => .u32, + .u64 => .u64, + .f16 => .f16, + .f32 => .f32, + .f64 => .f64, + .bf16 => .bf16, + .c64 => .c64, + .c128 => .c128, + .f8e5m2 => .f8e5m2, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3b11fnuz => .f8e4m3b11fnuz, + .f8e5m2fnuz => .f8e5m2fnuz, + .f8e4m3fnuz => .f8e4m3fnuz, + else => unreachable, + }; +} + +fn shapeFromFfiBuffer(buffer: *const ffi.Buffer) zml.Shape { + const dtype = dataTypeFromFfiDataType(buffer.dtype); + return zml.Shape.init(buffer.dims(), dtype); +} + +fn bufferFromFfiBuffer(ffi_buffer: *const ffi.Buffer) Buffer { + return .{ + .shape = shapeFromFfiBuffer(ffi_buffer), + .ptr = ffi_buffer.data, + }; +} + +fn getScalarAttributeAs(comptime T: type, call_frame: *ffi.CallFrame, attribute_name: []const u8) ?T { + const attribute = call_frame.attrs.getByName(.scalar, attribute_name) orelse return null; + return attribute.get(T); +} + +fn fixupKvCacheBuffer(buffer: Buffer, layer_index: i64) Buffer { + var shape = buffer.shape; + const layer_stride = shape.computeByteStrides().get(0); + shape = shape.remove(0); + const ptr = @as([*]u8, @ptrCast(buffer.ptr)); + return .{ + .shape = shape, + .ptr = ptr + @as(usize, @intCast(layer_stride * layer_index)), + }; +} + +pub fn Wrapper(comptime T: type, run_func: std.meta.DeclEnum(T)) type { + return struct { + pub fn register(platform: zml.Platform) !void { + try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "cuda", T.run, .{ .command_buffer_compatible = true }); + } + + pub fn run(call_frame: *ffi.CallFrame) callconv(.c) ?*ffi.Error { + return @field(T, @tagName(run_func))(call_frame) catch b: { + break :b ffi.Error.create(call_frame.api.?, .unknown, "Unknown"); + }; + } + }; +} + +pub const Fa2 = struct { + pub const custom_call_name = "fa2_mha_fwd"; + const Wrapped = Wrapper(@This(), .runInner); + + pub const register = Wrapped.register; + pub const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const k = bufferFromFfiBuffer(call_frame.args.buffers()[1]); + const v = bufferFromFfiBuffer(call_frame.args.buffers()[2]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(3); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: c.FA2MhaFwdParams = .{ + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + }; + + fa2_mha_fwd.?( + &q.toFlashattnTensor(), + &k.toFlashattnTensor(), + &v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &softmax_lse.toFlashattnTensor(), + null, + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + ¶ms, + stream, + ); + + return null; + } +}; + +pub const Fa2Varlen = struct { + pub const custom_call_name = "fa2_mha_varlen_fwd"; + const Wrapped = Wrapper(@This(), .runInner); + + pub const register = Wrapped.register; + pub const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const k = bufferFromFfiBuffer(call_frame.args.buffers()[1]); + const v = bufferFromFfiBuffer(call_frame.args.buffers()[2]); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_q: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_q").?; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: c.FA2MhaVarlenFwdParams = .{ + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .num_splits = 0, + .num_heads = num_heads, + }; + + fa2_mha_varlen_fwd.?( + &q.toFlashattnTensor(), + &k.toFlashattnTensor(), + &v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + &cu_seqlens_k.toFlashattnTensor(), + null, + null, + &softmax_lse.toFlashattnTensor(), + null, + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + ¶ms, + stream, + ); + + return null; + } +}; + +pub const Fa3 = struct { + pub const custom_call_name = "fa3_mha_fwd"; + const Wrapped = Wrapper(@This(), .runInner); + + pub const register = Wrapped.register; + pub const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const k = bufferFromFfiBuffer(call_frame.args.buffers()[1]); + const v = bufferFromFfiBuffer(call_frame.args.buffers()[2]); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const scheduler_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_q: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_q").?; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: c.FA3MhaFwdParams = .{ + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + .softcap = 0.0, + .is_rotary_interleaved = false, + .num_splits = 0, + .sm_margin = 0, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .cp_world_size = 1, + .cp_rank = 0, + }; + + fa3_mha_fwd.?( + &q.toFlashattnTensor(), + &k.toFlashattnTensor(), + &v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + &cu_seqlens_k.toFlashattnTensor(), + null, + null, + null, + null, + null, + null, + &softmax_lse.toFlashattnTensor(), + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + &scheduler_metadata.toFlashattnTensor(), + null, + null, + ¶ms, + stream, + ); + + return null; + } +}; + +const Fa2MhaFwdFunc = *const fn ( + q: ?*const c.FlashattnTensor, + k: ?*const c.FlashattnTensor, + v: ?*const c.FlashattnTensor, + out: ?*const c.FlashattnTensor, + softmax_lse: ?*const c.FlashattnTensor, + alibi_slopes_: ?*const c.FlashattnTensor, + softmax_lse_accum: ?*const c.FlashattnTensor, + out_accum: ?*const c.FlashattnTensor, + params: ?*const c.FA2MhaFwdParams, + stream: ?*anyopaque, +) callconv(.c) void; + +const Fa2MhaVarlenFwdFunc = *const fn ( + q: ?*const c.FlashattnTensor, + k: ?*const c.FlashattnTensor, + v: ?*const c.FlashattnTensor, + out: ?*const c.FlashattnTensor, + cu_seqlens_q: ?*const c.FlashattnTensor, + cu_seqlens_k: ?*const c.FlashattnTensor, + seqused_k: ?*const c.FlashattnTensor, + block_table: ?*const c.FlashattnTensor, + softmax_lse: ?*const c.FlashattnTensor, + alibi_slopes_: ?*const c.FlashattnTensor, + softmax_lse_accum: ?*const c.FlashattnTensor, + out_accum: ?*const c.FlashattnTensor, + params: ?*const c.FA2MhaVarlenFwdParams, + stream: ?*anyopaque, +) callconv(.c) void; + +const Fa3MhaFwdFunc = *const fn ( + q: ?*const c.FlashattnTensor, + k: ?*const c.FlashattnTensor, + v: ?*const c.FlashattnTensor, + out: ?*const c.FlashattnTensor, + cu_seqlens_q: ?*const c.FlashattnTensor, + cu_seqlens_k: ?*const c.FlashattnTensor, + seqused_q: ?*const c.FlashattnTensor, + seqused_k: ?*const c.FlashattnTensor, + page_table: ?*const c.FlashattnTensor, + q_descale: ?*const c.FlashattnTensor, + k_descale: ?*const c.FlashattnTensor, + v_descale: ?*const c.FlashattnTensor, + softmax_lse: ?*const c.FlashattnTensor, + softmax_lse_accum: ?*const c.FlashattnTensor, + out_accum: ?*const c.FlashattnTensor, + scheduler_metadata: ?*const c.FlashattnTensor, + s_aux: ?*const c.FlashattnTensor, + cp_tot_seqused_k: ?*const c.FlashattnTensor, + params: ?*const c.FA3MhaFwdParams, + stream: ?*anyopaque, +) callconv(.c) void; + +var fa2_mha_fwd: ?Fa2MhaFwdFunc = null; +var fa2_mha_varlen_fwd: ?Fa2MhaVarlenFwdFunc = null; +var fa3_mha_fwd: ?Fa3MhaFwdFunc = null; + +pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { + var r_ = (try runfiles.Runfiles.create(.{ .allocator = allocator, .io = io })).?; + defer r_.deinit(allocator); + + const source_repo = bazel_builtin.current_repository; + const r = r_.withSourceRepo(source_repo); + var buffer: [std.posix.PATH_MAX]u8 = undefined; + const library = (try r.rlocation("flashattn/flashattn/lib/libflashattn.so", &buffer)) orelse return error.NotFound; + const path = try std.posix.toPosixPath(library); + const handle = std.c.dlopen(&path, .{ .NOW = true, .GLOBAL = true }) orelse { + std.log.err("Failed to open libflashattn.so", .{}); + return; + }; + + fa2_mha_varlen_fwd = @ptrCast(@alignCast(std.c.dlsym(handle, "fa2_mha_varlen_fwd") orelse return error.NotFound)); + fa2_mha_fwd = @ptrCast(std.c.dlsym(handle, "fa2_mha_fwd") orelse return error.NotFound); + fa3_mha_fwd = @ptrCast(@alignCast(std.c.dlsym(handle, "fa3_mha_fwd") orelse return error.NotFound)); +} + +pub fn register(platform: zml.Platform) !void { + try Fa2.register(platform); + try Fa2Varlen.register(platform); + try Fa3.register(platform); +} diff --git a/zml/attention/flashattn/flashattn.zig b/zml/attention/flashattn/flashattn.zig new file mode 100644 index 00000000..28cff850 --- /dev/null +++ b/zml/attention/flashattn/flashattn.zig @@ -0,0 +1,195 @@ +const std = @import("std"); +const stdx = @import("stdx"); + +const zml = @import("../../zml.zig"); +const bindings = @import("bindings.zig"); + +pub const load = bindings.load; +pub const register = bindings.register; + +pub const fa2 = struct { + pub const Parameters = struct { + pub const InitOptions = struct {}; + + pub fn init(opts: InitOptions) fa2.Parameters { + _ = opts; // autofix + return .{}; + } + }; + + pub const Metadata = struct { + softmax_lse: zml.Tensor, + softmax_lse_accum: zml.Tensor, + out_accum: zml.Tensor, + + pub const InitOptions = struct { + seqlen: i64, + }; + + pub fn init(opts: InitOptions) fa2.Metadata { + return .{ + .softmax_lse = .init(.{1 * 32 * opts.seqlen * 4}, .i8), + .softmax_lse_accum = .init(.{32 * 1 * 32 * 4 * 4}, .i8), + .out_accum = .init(.{32 * 1 * 32 * 4 * 128 * 4}, .i8), + }; + } + + pub fn initBuffer(self: fa2.Metadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(fa2.Metadata) { + return .{ + .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), .{}), + .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), .{}), + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(fa2.Metadata)) void { + self.softmax_lse.deinit(); + self.softmax_lse_accum.deinit(); + self.out_accum.deinit(); + } + }; + + pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: fa2.Metadata, parameters: fa2.Parameters) zml.Tensor { + _ = parameters; // autofix + stdx.debug.assert(q_.shape().hasTag(.b) == null or q_.dim(.b) == 1, "fa2.attention support for batch size != 1 is not supported yet.", .{}); + const seqused_k = token_index.addConstant(q_.dim(.q)).reshape(.{1}); + // TODO(Corendos): replace with cumsum + const cu_seqlens_k = b: { + const zero = zml.Tensor.constant(token_index.dtype().zero()).reshape(.{1}); + break :b zml.Tensor.concatenate(&.{ zero, seqused_k }, 0).convert(.i32); + }; + const max_seqlen_q: i32 = @intCast(q_.dim(.q)); + const max_seqlen_k: i32 = @intCast(k_.dim(.k)); + var q = q_.insertAxes(.q, .{.b}).merge(.{ .tot = .{ .b, .q } }); + const k = k_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); + const v = v_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); + // TODO(Corendos): replace with cumsum + const cu_seqlens_q = zml.Tensor.constantTensor(zml.Shape.init(.{2}, .i32), std.mem.sliceAsBytes(&[2]i32{ 0, max_seqlen_q })); + + const original_tot = q.dim(.tot); + const num_heads = q_.dim(.h); + const num_heads_k = k_.dim(.h); + const head_size = q_.dim(.hd); + const ngroups = @divExact(num_heads, num_heads_k); + const seqlenq_ngroups_swapped = max_seqlen_q == 1 and num_heads > num_heads_k and @mod(head_size, 8) == 0; + if (seqlenq_ngroups_swapped) { + q = q.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .tot, .ngroups, .h, .hd }).merge(.{ .tot = .{ .tot, .ngroups } }); + } + + var o = zml.ops.customCall(bindings.Fa2Varlen.custom_call_name, .{ + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + metadata.softmax_lse, + metadata.softmax_lse_accum, + metadata.out_accum, + }, .{q.shape()}, .{ + //.softmax_scale = @as(f32, 1.0), + .is_causal = true, + .window_size_left = @as(i32, -1), + .window_size_right = @as(i32, -1), + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + .num_heads = @as(i32, @intCast(num_heads)), + }, .{ + .output_operand_aliases = &.{0}, + .has_side_effect = true, + }); + + if (seqlenq_ngroups_swapped) { + o = o.splitAxis(.tot, .{ .tot = original_tot, .ngroups = ngroups }).transpose(.{ .tot, .h, .ngroups, .hd }).merge(.{ .h = .{ .h, .ngroups } }); + } + + return o.splitAxis(.tot, .{ .b = 1, .q = q_.dim(.q) }).squeeze(.b); + } +}; + +pub const fa3 = struct { + pub const Parameters = struct { + pub const InitOptions = struct {}; + + pub fn init(opts: InitOptions) fa3.Parameters { + _ = opts; // autofix + return .{}; + } + }; + + pub const Metadata = struct { + softmax_lse: zml.Tensor, + softmax_lse_accum: zml.Tensor, + out_accum: zml.Tensor, + scheduler_metadata: zml.Tensor, + + pub const InitOptions = struct { + seqlen: i64, + }; + + pub fn init(opts: InitOptions) fa3.Metadata { + return .{ + .softmax_lse = .init(.{1 * 32 * opts.seqlen * 4}, .i8), + .softmax_lse_accum = .init(.{32 * 1 * 32 * 4 * 4}, .i8), + .out_accum = .init(.{32 * 1 * 32 * 4 * 128 * 4}, .i8), + .scheduler_metadata = .init(.{2}, .i32), + }; + } + + pub fn initBuffer(self: fa3.Metadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(fa3.Metadata) { + return .{ + .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), .{}), + .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), .{}), + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + .scheduler_metadata = try zml.Buffer.uninitialized(io, platform, self.scheduler_metadata.shape(), .{}), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(fa3.Metadata)) void { + self.softmax_lse.deinit(); + self.softmax_lse_accum.deinit(); + self.out_accum.deinit(); + self.scheduler_metadata.deinit(); + } + }; + + pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: fa3.Metadata, parameters: fa3.Parameters) zml.Tensor { + _ = parameters; // autofix + stdx.debug.assert(q_.shape().hasTag(.b) == null or q_.dim(.b) == 1, "fa3.attention support for batch size != 1 is not supported yet.", .{}); + const seqused_k = token_index.addConstant(q_.dim(.q)).reshape(.{1}); + // TODO(Corendos): replace with cumsum + const cu_seqlens_k = b: { + const zero = zml.Tensor.constant(token_index.dtype().zero()).reshape(.{1}); + break :b zml.Tensor.concatenate(&.{ zero, seqused_k }, 0).convert(.i32); + }; + const max_seqlen_q: i32 = @intCast(q_.dim(.q)); + const max_seqlen_k: i32 = @intCast(k_.dim(.k)); + var q = q_.insertAxes(.q, .{.b}).merge(.{ .tot = .{ .b, .q } }); + const k = k_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); + const v = v_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); + // TODO(Corendos): replace with cumsum + const cu_seqlens_q = zml.Tensor.constantTensor(zml.Shape.init(.{2}, .i32), std.mem.sliceAsBytes(&[2]i32{ 0, max_seqlen_q })); + + var o = zml.ops.customCall(bindings.Fa3.custom_call_name, .{ + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + metadata.softmax_lse, + metadata.softmax_lse_accum, + metadata.out_accum, + metadata.scheduler_metadata, + }, .{q.shape()}, .{ + .is_causal = true, + .window_size_left = @as(i32, -1), + .window_size_right = @as(i32, -1), + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + }, .{ + .output_operand_aliases = &.{0}, + .has_side_effect = true, + }); + + return o.splitAxis(.tot, .{ .b = 1, .q = q_.dim(.q) }).squeeze(.b); + } +}; diff --git a/zml/ops.zig b/zml/ops.zig index 637e0947..bc082812 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -962,7 +962,30 @@ fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs var metadata_attributes: [metadata_type_info.@"struct".fields.len]mlir.NamedAttribute = undefined; inline for (metadata_type_info.@"struct".fields, 0..) |field, i| { const attribute: *const mlir.Attribute = switch (@typeInfo(field.type)) { - .int, .comptime_int => mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@field(metadata, field.name)))), + .comptime_int => mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @field(metadata, field.name))), + .int => |int_field| switch (int_field.signedness) { + .signed => switch (int_field.bits) { + 8 => mlir.integerAttribute(ctx.mlir_ctx, .i8, @field(metadata, field.name)), + 16 => mlir.integerAttribute(ctx.mlir_ctx, .i16, @field(metadata, field.name)), + 32 => mlir.integerAttribute(ctx.mlir_ctx, .i32, @field(metadata, field.name)), + 64 => mlir.integerAttribute(ctx.mlir_ctx, .i64, @field(metadata, field.name)), + else => @panic("Unsupported DataType"), + }, + .unsigned => switch (int_field.bits) { + 8 => mlir.integerAttribute(ctx.mlir_ctx, .u8, @field(metadata, field.name)), + 16 => mlir.integerAttribute(ctx.mlir_ctx, .u16, @field(metadata, field.name)), + 32 => mlir.integerAttribute(ctx.mlir_ctx, .u32, @field(metadata, field.name)), + 64 => mlir.integerAttribute(ctx.mlir_ctx, .u64, @field(metadata, field.name)), + else => @panic("Unsupported DataType"), + }, + }, + .float => |float_field| switch (float_field.bits) { + 16 => mlir.floatAttribute(ctx.mlir_ctx, .f16, @as(f64, @field(metadata, field.name))), + 32 => mlir.floatAttribute(ctx.mlir_ctx, .f32, @as(f64, @field(metadata, field.name))), + 64 => mlir.floatAttribute(ctx.mlir_ctx, .f64, @as(f64, @field(metadata, field.name))), + else => @panic("Unsupported DataType"), + }, + .bool => mlir.boolAttribute(ctx.mlir_ctx, @field(metadata, field.name)), else => @compileError("Unsupported metadata type: " ++ @typeName(field.type)), }; metadata_attributes[i] = mlir.NamedAttribute.named(ctx.mlir_ctx, field.name, attribute); @@ -996,7 +1019,7 @@ fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs .output_operand_aliases = opts.output_operand_aliases orelse &.{}, }, .unknown(ctx.mlir_ctx), - ); + ).appendTo(CompilationContext.current().currentScope().block); const outputs_ = ctx.arena.allocator().alloc(Tensor, outputs.len) catch unreachable; for (outputs, 0..) |output, i| { diff --git a/zml/platform.zig b/zml/platform.zig index 00578890..61e697d3 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -109,12 +109,28 @@ pub const Platform = struct { if (true_num_devices > MAX_NUM_DEVICES) { log.warn("platform {} got {} devices, but ZML only support up to {} devices. Some devices won't be used.", .{ target, true_num_devices, MAX_NUM_DEVICES }); } - return .{ + + const platform: Platform = .{ .target = target, .pjrt_api = api, .pjrt_client = pjrt_client, .compilation_options = .{}, }; + + switch (target) { + .cuda => { + // TODO(Corentin): provide a better allocator + zml.attention.flashattn.load(std.heap.c_allocator, io) catch { + log.warn("Failed to load flashattn", .{}); + }; + zml.attention.flashattn.register(platform) catch { + log.warn("Failed to register flashattn custom call", .{}); + }; + }, + else => {}, + } + + return platform; } pub fn auto(io: std.Io, options: CreateOptions) !Platform { @@ -340,3 +356,18 @@ pub const CreateOptions = struct { return values.items; } }; + +// TODO(Corendos): Consider moving that in its own file if its size increase too much. +pub const cuda = struct { + pub fn tryGetComputeCapabilities(platform: zml.Platform, device: *const pjrt.Device) ?[]const u8 { + stdx.debug.assert(platform.target == .cuda, "tryGetComputeCapabilities expects .cuda platform, got {}", .{platform.target}); + const description = device.getDescription(platform.pjrt_api); + + const attributes = description.attributes(platform.pjrt_api); + return for (attributes) |attr| { + if (std.mem.eql(u8, attr.name(), "compute_capability")) { + break attr.value().string; + } + } else null; + } +}; diff --git a/zml/testing.zig b/zml/testing.zig index bf968551..f96dfad9 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -49,7 +49,7 @@ pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, tolerance: f32) if (should_free_left) left.free(allocator); if (should_free_right) right.free(allocator); } - errdefer log.err("\n--> Left: {0f}{0d:24.3}\n--> Right: {1f}{1d:24.3}", .{ left, right }); + errdefer log.err("\n--> Left: {0d:24.3}\n--> Right: {1d:24.3}", .{ left, right }); if (!std.mem.eql(i64, left.shape.dims(), right.shape.dims())) { log.err("left.shape() {f} != right.shape() {f}", .{ left.shape, right.shape }); return error.TestUnexpectedResult; diff --git a/zml/zml.zig b/zml/zml.zig index a68b9dc3..3ea5c22b 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -12,6 +12,7 @@ const runfiles = @import("runfiles"); pub const stdx = @import("stdx"); pub const tokenizer = @import("zml/tokenizer"); +pub const attention = @import("attention/attention.zig"); pub const Buffer = @import("buffer.zig").Buffer; pub const constants = @import("constants.zig"); pub const dtype = @import("dtype.zig"); @@ -25,6 +26,7 @@ pub const meta = @import("meta.zig"); pub const mlir = @import("mlirx.zig"); pub const module = @import("module.zig"); pub const nn = @import("nn.zig"); +pub const ops = @import("ops.zig"); pub const platform = @import("platform.zig"); pub const Platform = platform.Platform; pub const Target = platform.Target; From 27cf49cbe4ede22c8af3facb297bf2541fdef510 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 16 Jan 2026 18:15:07 +0100 Subject: [PATCH 242/881] zml: added missing @alignCast It was removed for a test and wasn't added back --- zml/attention/flashattn/bindings.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/attention/flashattn/bindings.zig b/zml/attention/flashattn/bindings.zig index 63ab1c7d..a8c48409 100644 --- a/zml/attention/flashattn/bindings.zig +++ b/zml/attention/flashattn/bindings.zig @@ -406,7 +406,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { }; fa2_mha_varlen_fwd = @ptrCast(@alignCast(std.c.dlsym(handle, "fa2_mha_varlen_fwd") orelse return error.NotFound)); - fa2_mha_fwd = @ptrCast(std.c.dlsym(handle, "fa2_mha_fwd") orelse return error.NotFound); + fa2_mha_fwd = @ptrCast(@alignCast(std.c.dlsym(handle, "fa2_mha_fwd") orelse return error.NotFound)); fa3_mha_fwd = @ptrCast(@alignCast(std.c.dlsym(handle, "fa3_mha_fwd") orelse return error.NotFound)); } From 32ca342f2cfd54acece2f94d4096d52f6e83360c Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 16 Jan 2026 21:32:39 +0100 Subject: [PATCH 243/881] examples/llama: remove S3 vfs init --- examples/llama/main.zig | 4 ---- 1 file changed, 4 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index b8aa5bea..07ae0ae4 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -88,16 +88,12 @@ pub fn main() !void { var hf_vfs: zml.io.VFS.HF = try .auto(allocator, threaded.io(), &http_client); defer hf_vfs.deinit(); - var s3_vfs: zml.io.VFS.S3 = try .fromEnv(allocator, threaded.io(), &http_client); - defer s3_vfs.deinit(); - var vfs: zml.io.VFS = try .init(allocator, threaded.io()); defer vfs.deinit(); try vfs.register("file", vfs_file.io()); try vfs.register("https", vfs_https.io()); try vfs.register("hf", hf_vfs.io()); - try vfs.register("s3", s3_vfs.io()); const io = vfs.io(); From dc2e876b21edbf4646a6d203cfc265aececf4d7b Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Mon, 19 Jan 2026 14:24:43 +0000 Subject: [PATCH 244/881] Fix llama stack corruption and remove Buffer.fromIoBufferize --- examples/llama/llama.zig | 6 ++---- examples/llama/main.zig | 44 ++++++++++++++++------------------------ zml/buffer.zig | 9 -------- zml/io.zig | 19 +++++++++-------- 4 files changed, 29 insertions(+), 49 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index a1ba1016..45c580fb 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -76,10 +76,8 @@ pub const LlamaLM = struct { cb: zml.io.CallbackTensorBufferTransfer(TransferCtx), ) !zml.Bufferized(LlamaLM) { var lm_head_bufferized: ?zml.Bufferized(zml.nn.Linear) = null; - if (self.lm_head) |lm_head| { - lm_head_bufferized = undefined; - - const transfers = try zml.io.bufferize(TransferCtx, bufferize_ctx, &lm_head, &lm_head_bufferized.?, store.withPrefix("lm_head")); + if (self.lm_head) |_| { + const transfers = try zml.io.bufferize(TransferCtx, bufferize_ctx, &self.lm_head, &lm_head_bufferized, store.withPrefix("lm_head")); for (transfers) |t| try group.concurrent(bufferize_ctx.io, cb, .{t}); } diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 07ae0ae4..014b9bbc 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -166,29 +166,10 @@ pub fn main() !void { defer v.decode_exe.deinit(); } else |_| {}; - var load_model_buffers: zml.stdx.Io.AllocatingLimitedConcurrentGroup = try .init(allocator, threaded.async_limit.toInt() orelse 16); + var load_model_buffers_group: zml.stdx.Io.AllocatingLimitedConcurrentGroup = try .init(allocator, threaded.async_limit.toInt() orelse 16); defer { - load_model_buffers.cancel(io); - load_model_buffers.deinit(); - } - - var read_pool, var write_pool = try zml.io.ConcurrentBufferPool.initRW( - allocator, - platform, - .{ - .size = args.reader_buffer_size.bytes(), - .concurrency = load_model_buffers.limit, - .dma = true, - }, - .{ - .size = args.writer_buffer_size.bytes(), - .concurrency = load_model_buffers.limit, - .dma = true, - }, - ); - defer { - read_pool.deinit(); - write_pool.deinit(); + load_model_buffers_group.cancel(io); + load_model_buffers_group.deinit(); } var model_name_it = std.mem.splitScalar(u8, args.model, '/'); @@ -197,8 +178,11 @@ pub fn main() !void { if (!std.mem.eql(u8, part, "")) model_name = part; } + const read_pool_config: zml.io.ConcurrentBufferPool.Config = .{ .size = args.reader_buffer_size.bytes(), .concurrency = load_model_buffers_group.limit, .dma = true }; + const write_pool_config: zml.io.ConcurrentBufferPool.Config = .{ .size = args.writer_buffer_size.bytes(), .concurrency = load_model_buffers_group.limit, .dma = true }; + var progress = std.Progress.start(io, .{ .root_name = model_name, .estimated_total_items = store.view().count() }); - var llama_buffers_future = io.async(loadModelBuffers, .{ allocator, io, &progress, &load_model_buffers, &read_pool, &write_pool, platform, &store, llama_model }); + var llama_buffers_future = io.async(loadModelBuffers, .{ allocator, io, &progress, &load_model_buffers_group, read_pool_config, write_pool_config, platform, &store, llama_model }); errdefer b: { var v = llama_buffers_future.cancel(io) catch break :b; LlamaLM.unloadBuffers(&v, allocator); @@ -351,8 +335,8 @@ fn loadModelBuffers( io: std.Io, progress: *std.Progress.Node, group: *zml.stdx.Io.AllocatingLimitedConcurrentGroup, - read_pool: *zml.io.ConcurrentBufferPool, - write_pool: *zml.io.ConcurrentBufferPool, + read_pool_config: zml.io.ConcurrentBufferPool.Config, + write_pool_config: zml.io.ConcurrentBufferPool.Config, platform: zml.Platform, store: *zml.io.TensorStore, llama_model: llama.LlamaLM, @@ -371,12 +355,18 @@ fn loadModelBuffers( var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); + var read_pool = try zml.io.ConcurrentBufferPool.init(allocator, platform, read_pool_config); + defer read_pool.deinit(); + + var write_pool = try zml.io.ConcurrentBufferPool.init(allocator, platform, write_pool_config); + defer write_pool.deinit(); + const bufferize_ctx: zml.io.BufferizeContext(llama.TransferCtx) = .{ .allocator = allocator, .arena = &arena, .io = io, .platform = platform, - .cb_ctx = .{ .read_pool = read_pool, .write_pool = write_pool, .transferred_bytes = &transferred_bytes, .progress = progress }, + .cb_ctx = .{ .read_pool = &read_pool, .write_pool = &write_pool, .transferred_bytes = &transferred_bytes, .progress = progress }, }; const bufferized = llama_model.loadBuffers( @@ -407,7 +397,7 @@ fn transferBuffer(ctx: zml.io.TensorBufferTransfer(llama.TransferCtx)) !void { var reader = zml.safetensors.TensorReader.init(ctx.io, ctx.tensor, read_buffer) catch unreachable; defer reader.deinit(); - var writer = zml.io.DeviceWriter.init(ctx.io, &transfer_progress, ctx.platform, ctx.buffer, .device, write_buffer) catch unreachable; + var writer = zml.io.DeviceWriter.init(ctx.io, &transfer_progress, ctx.platform, ctx.tensor.shape, ctx.buffer, .device, write_buffer) catch unreachable; defer { writer.interface.flush() catch unreachable; writer.deinit(); diff --git a/zml/buffer.zig b/zml/buffer.zig index af591f45..43067706 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -218,15 +218,6 @@ pub const Buffer = struct { }; } - pub fn fromIoBufferize(platform: Platform, shape_: Shape) Buffer { - return .{ - ._api = platform.pjrt_api, - ._target = platform.target, - ._shape = shape_, - ._shards = .{}, - }; - } - pub fn devicePtr(self: Buffer) *anyopaque { //stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer", .{}); return self._shards.get(0).opaqueDeviceMemoryDataPointer(self._api) catch unreachable; diff --git a/zml/io.zig b/zml/io.zig index 879d8abb..0f386f42 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -1,5 +1,4 @@ const std = @import("std"); -const builtin = @import("builtin"); const pjrt = @import("pjrt"); const stdx = @import("stdx"); @@ -277,7 +276,6 @@ pub fn bufferize( fn cb(ctx: *FillBuffersCtx, tensor: *const Tensor) void { const ptr = ctx.store.store.getPtrFromId(tensor.id) orelse return; const buffer = ctx.buffers[ctx.idx]; - buffer.* = .fromIoBufferize(ctx.bufferize_ctx.platform, tensor.shape()); ctx.transfers[ctx.idx] = .{ .io = ctx.bufferize_ctx.io, @@ -388,13 +386,13 @@ pub const DeviceWriter = struct { io: std.Io, progress: ?*std.Progress.Node, platform: Platform, + shape: Shape, buffer: *Buffer, transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, offset: usize = 0, interface: std.Io.Writer, - pub fn init(io: std.Io, progress: ?*std.Progress.Node, platform: Platform, buffer: *Buffer, memory: Buffer.Memory, buf: []u8) !DeviceWriter { - const shape = buffer.shape(); + pub fn init(io: std.Io, progress: ?*std.Progress.Node, platform: Platform, shape: Shape, buffer: *Buffer, memory: Buffer.Memory, buf: []u8) !DeviceWriter { const memories = platform.getDevices()[0].addressableMemories(platform.pjrt_api); const mem = blk: { @@ -411,12 +409,15 @@ pub const DeviceWriter = struct { .shape_specs = &.{shape_spec}, .memory = mem, }); - try buffer._shards.append(transfer_manager.retrieveBuffer(platform.pjrt_api, 0) catch unreachable); + + const pjrt_buffer = transfer_manager.retrieveBuffer(platform.pjrt_api, 0) catch unreachable; + buffer.* = Buffer.fromPjrtBuffers(platform, shape, &.{pjrt_buffer}); return .{ .io = io, .progress = progress, .platform = platform, + .shape = shape, .buffer = buffer, .transfer_manager = transfer_manager, .interface = .{ @@ -438,9 +439,9 @@ pub const DeviceWriter = struct { _ = splat; const self: *DeviceWriter = @alignCast(@fieldParentPtr("interface", w)); - stdx.debug.assert(self.offset + w.end <= self.buffer.shape().byteSize(), "Can't write more data than required", .{}); + stdx.debug.assert(self.offset + w.end <= self.shape.byteSize(), "Can't write more data than required", .{}); - const is_last_transfer = self.offset + w.end >= self.buffer.shape().byteSize(); + const is_last_transfer = self.offset + w.end >= self.shape.byteSize(); const event = self.transfer_manager.transferData(self.platform.pjrt_api, 0, w.buffer[0..w.end], @intCast(self.offset), is_last_transfer) catch return error.WriteFailed; event.await(self.platform.pjrt_api, self.io) catch return error.WriteFailed; @@ -513,11 +514,11 @@ fn initBufferizedFrom(model: anytype, bufferized_: *Bufferized(@TypeOf(model))) }, } }, - .optional => { + .optional => |optional_type_info| { if (model == null) { bufferized_.* = null; } else { - bufferized_.* = undefined; + bufferized_.* = @as(optional_type_info.child, undefined); initBufferizedFrom(model.?, &bufferized_.*.?); } }, From ba57994cac9824faa0715fc5767d459f30091d97 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 19 Jan 2026 16:10:59 +0100 Subject: [PATCH 245/881] runtimes: remove layers targets it is now handled in the runtimes themselves --- runtimes/BUILD.bazel | 35 ----------------------------------- runtimes/cpu/BUILD.bazel | 6 ------ runtimes/cuda/BUILD.bazel | 24 +++++++++--------------- runtimes/neuron/BUILD.bazel | 6 ------ runtimes/rocm/BUILD.bazel | 12 +++--------- runtimes/tpu/BUILD.bazel | 6 ------ 6 files changed, 12 insertions(+), 77 deletions(-) diff --git a/runtimes/BUILD.bazel b/runtimes/BUILD.bazel index d0a1bcdb..4f887480 100644 --- a/runtimes/BUILD.bazel +++ b/runtimes/BUILD.bazel @@ -1,6 +1,4 @@ -load("@aspect_bazel_lib//lib:tar.bzl", "tar") load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") -load("@bazel_skylib//rules:write_file.bzl", "write_file") load("@rules_zig//zig:defs.bzl", "zig_library") RUNTIMES = { @@ -28,39 +26,6 @@ RUNTIMES = { for runtime in RUNTIMES.keys() ] -write_file( - name = "zml_txt", - out = "zml.txt", - content = ["ZML loves you <3"], -) - -tar( - name = "empty", - srcs = [":zml_txt"], - mtree = "auto", -) - -filegroup( - name = "layers", - srcs = [":empty"] + select({ - ":cpu.enabled": ["//runtimes/cpu:layers"], - "//conditions:default": [], - }) + select({ - ":cuda.enabled": ["//runtimes/cuda:layers"], - "//conditions:default": [], - }) + select({ - ":neuron.enabled": ["//runtimes/neuron:layers"], - "//conditions:default": [], - }) + select({ - ":rocm.enabled": ["//runtimes/rocm:layers"], - "//conditions:default": [], - }) + select({ - ":tpu.enabled": ["//runtimes/tpu:layers"], - "//conditions:default": [], - }), - visibility = ["//visibility:public"], -) - zig_library( name = "runtimes", main = "runtimes.zig", diff --git a/runtimes/cpu/BUILD.bazel b/runtimes/cpu/BUILD.bazel index 0c016593..1c502881 100644 --- a/runtimes/cpu/BUILD.bazel +++ b/runtimes/cpu/BUILD.bazel @@ -56,9 +56,3 @@ zig_library( "//conditions:default": [":empty"], }), ) - -filegroup( - name = "layers", - srcs = [], - visibility = ["//visibility:public"], -) diff --git a/runtimes/cuda/BUILD.bazel b/runtimes/cuda/BUILD.bazel index a789a24e..34d976a2 100644 --- a/runtimes/cuda/BUILD.bazel +++ b/runtimes/cuda/BUILD.bazel @@ -3,12 +3,12 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_shared_library") zig_shared_library( name = "zmlxcuda", - # Use Clang's compiler-rt, but disable stack checking - # to avoid requiring on the _zig_probe_stack symbol. - zigopts = ["-fno-stack-check"], main = "zmlxcuda.zig", shared_lib_name = "libzmlxcuda.so.0", visibility = ["@libpjrt_cuda//:__subpackages__"], + # Use Clang's compiler-rt, but disable stack checking + # to avoid requiring on the _zig_probe_stack symbol. + zigopts = ["-fno-stack-check"], deps = ["//stdx"], ) @@ -26,6 +26,12 @@ cc_library( zig_library( name = "cuda", + data = select({ + "//runtimes:cuda.enabled": [ + "@flashattn//:libflashattn", + ], + "//conditions:default": [":empty"], + }), import_name = "runtimes/cuda", main = "cuda.zig", visibility = ["//visibility:public"], @@ -39,16 +45,4 @@ zig_library( ], "//conditions:default": [":empty"], }), - data = select({ - "//runtimes:cuda.enabled": [ - "@flashattn//:libflashattn" - ], - "//conditions:default": [":empty"], - }), -) - -filegroup( - name = "layers", - srcs = [], - visibility = ["//visibility:public"], ) diff --git a/runtimes/neuron/BUILD.bazel b/runtimes/neuron/BUILD.bazel index c9eb2f84..88ea3fb2 100644 --- a/runtimes/neuron/BUILD.bazel +++ b/runtimes/neuron/BUILD.bazel @@ -62,12 +62,6 @@ cc_library( deps = ["@libpjrt_neuron//:libnrt_headers"], ) -filegroup( - name = "layers", - srcs = [], - visibility = ["//visibility:public"], -) - upb_c_proto_library( name = "xla_data_upb", deps = ["@xla//xla:xla_data_proto"], diff --git a/runtimes/rocm/BUILD.bazel b/runtimes/rocm/BUILD.bazel index 6e0cd373..28f3438f 100644 --- a/runtimes/rocm/BUILD.bazel +++ b/runtimes/rocm/BUILD.bazel @@ -3,12 +3,12 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_shared_library") zig_shared_library( name = "zmlxrocm", - # Use Clang's compiler-rt, but disable stack checking - # to avoid requiring on the _zig_probe_stack symbol. - zigopts = ["-fno-stack-check"], main = "zmlxrocm.zig", shared_lib_name = "libzmlxrocm.so.0", visibility = ["@libpjrt_rocm//:__subpackages__"], + # Use Clang's compiler-rt, but disable stack checking + # to avoid requiring on the _zig_probe_stack symbol. + zigopts = ["-fno-stack-check"], deps = ["//stdx"], ) @@ -49,9 +49,3 @@ zig_library( "//conditions:default": [":empty"], }), ) - -filegroup( - name = "layers", - srcs = [], - visibility = ["//visibility:public"], -) diff --git a/runtimes/tpu/BUILD.bazel b/runtimes/tpu/BUILD.bazel index bc1e3a99..22b56558 100644 --- a/runtimes/tpu/BUILD.bazel +++ b/runtimes/tpu/BUILD.bazel @@ -28,9 +28,3 @@ zig_library( "//conditions:default": [":empty"], }), ) - -filegroup( - name = "layers", - srcs = [], - visibility = ["//visibility:public"], -) From 9e2c09066bde8024a6e384e4ea1d4d3447c303bd Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 19 Jan 2026 16:18:17 +0100 Subject: [PATCH 246/881] runtimes: rename to platforms --- .github/workflows/ci.yaml | 6 +-- MODULE.bazel | 26 ++++++------ README.md | 12 +++--- docs/howtos/deploy_on_server.md | 20 +++++----- docs/howtos/dockerize_models.md | 14 +++---- docs/tutorials/getting_started.md | 12 +++--- platforms/BUILD.bazel | 40 +++++++++++++++++++ {runtimes => platforms}/cpu/BUILD.bazel | 4 +- {runtimes => platforms}/cpu/cpu.bzl | 6 +-- {runtimes => platforms}/cpu/cpu.zig | 4 +- {runtimes => platforms}/cuda/BUILD.bazel | 6 +-- {runtimes => platforms}/cuda/cuda.bzl | 4 +- {runtimes => platforms}/cuda/cuda.zig | 2 +- .../cuda/libpjrt_cuda.BUILD.bazel | 4 +- {runtimes => platforms}/cuda/libpjrt_cuda.h | 0 .../cuda/packages.lock.json | 0 {runtimes => platforms}/cuda/packages.yaml | 0 {runtimes => platforms}/cuda/zmlxcuda.zig | 0 {runtimes => platforms}/neuron/BUILD.bazel | 4 +- .../neuron/libneuronxla.zig | 2 +- .../neuron/libpjrt_neuron.BUILD.bazel | 19 ++++----- .../neuron/libpjrt_neuron.zig | 2 +- {runtimes => platforms}/neuron/libpython.h | 0 {runtimes => platforms}/neuron/neuron.bzl | 4 +- {runtimes => platforms}/neuron/neuron.zig | 2 +- {runtimes => platforms}/neuron/neuronx-cc.zig | 0 {runtimes => platforms}/neuron/nrt.h | 0 .../neuron/packages.lock.json | 0 {runtimes => platforms}/neuron/packages.yaml | 0 .../neuron/requirements.in | 0 .../neuron/requirements.lock.txt | 2 +- {runtimes/common => platforms}/packages.bzl | 0 .../runtimes.zig => platforms/platforms.zig | 10 ++--- {runtimes => platforms}/rocm/BUILD.bazel | 4 +- {runtimes => platforms}/rocm/gfx.bzl | 2 +- .../rocm/libpjrt_rocm.BUILD.bazel | 4 +- .../rocm/packages.lock.json | 0 {runtimes => platforms}/rocm/packages.yaml | 0 {runtimes => platforms}/rocm/rocm.bzl | 8 ++-- {runtimes => platforms}/rocm/rocm.zig | 2 +- {runtimes => platforms}/rocm/zmlxrocm.zig | 0 {runtimes => platforms}/tpu/BUILD.bazel | 4 +- .../tpu/libpjrt_tpu.BUILD.bazel | 2 +- {runtimes => platforms}/tpu/tpu.bzl | 0 {runtimes => platforms}/tpu/tpu.zig | 2 +- runtimes/BUILD.bazel | 39 ------------------ runtimes/common/BUILD.bazel | 0 zml/BUILD.bazel | 14 +++---- zml/platform.zig | 8 ++-- 49 files changed, 146 insertions(+), 148 deletions(-) rename {runtimes => platforms}/cpu/BUILD.bazel (94%) rename {runtimes => platforms}/cpu/cpu.bzl (90%) rename {runtimes => platforms}/cpu/cpu.zig (91%) rename {runtimes => platforms}/cuda/BUILD.bazel (90%) rename {runtimes => platforms}/cuda/cuda.bzl (98%) rename {runtimes => platforms}/cuda/cuda.zig (98%) rename {runtimes => platforms}/cuda/libpjrt_cuda.BUILD.bazel (94%) rename {runtimes => platforms}/cuda/libpjrt_cuda.h (100%) rename {runtimes => platforms}/cuda/packages.lock.json (100%) rename {runtimes => platforms}/cuda/packages.yaml (100%) rename {runtimes => platforms}/cuda/zmlxcuda.zig (100%) rename {runtimes => platforms}/neuron/BUILD.bazel (97%) rename {runtimes => platforms}/neuron/libneuronxla.zig (99%) rename {runtimes => platforms}/neuron/libpjrt_neuron.BUILD.bazel (83%) rename {runtimes => platforms}/neuron/libpjrt_neuron.zig (99%) rename {runtimes => platforms}/neuron/libpython.h (100%) rename {runtimes => platforms}/neuron/neuron.bzl (96%) rename {runtimes => platforms}/neuron/neuron.zig (97%) rename {runtimes => platforms}/neuron/neuronx-cc.zig (100%) rename {runtimes => platforms}/neuron/nrt.h (100%) rename {runtimes => platforms}/neuron/packages.lock.json (100%) rename {runtimes => platforms}/neuron/packages.yaml (100%) rename {runtimes => platforms}/neuron/requirements.in (100%) rename {runtimes => platforms}/neuron/requirements.lock.txt (99%) rename {runtimes/common => platforms}/packages.bzl (100%) rename runtimes/runtimes.zig => platforms/platforms.zig (76%) rename {runtimes => platforms}/rocm/BUILD.bazel (93%) rename {runtimes => platforms}/rocm/gfx.bzl (96%) rename {runtimes => platforms}/rocm/libpjrt_rocm.BUILD.bazel (96%) rename {runtimes => platforms}/rocm/packages.lock.json (100%) rename {runtimes => platforms}/rocm/packages.yaml (100%) rename {runtimes => platforms}/rocm/rocm.bzl (96%) rename {runtimes => platforms}/rocm/rocm.zig (97%) rename {runtimes => platforms}/rocm/zmlxrocm.zig (100%) rename {runtimes => platforms}/tpu/BUILD.bazel (88%) rename {runtimes => platforms}/tpu/libpjrt_tpu.BUILD.bazel (87%) rename {runtimes => platforms}/tpu/tpu.bzl (100%) rename {runtimes => platforms}/tpu/tpu.zig (97%) delete mode 100644 runtimes/BUILD.bazel delete mode 100644 runtimes/common/BUILD.bazel diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 67e7ba75..48bc9320 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,17 +17,17 @@ jobs: matrix: runtimes: - target: cuda - bazel_opts: "--@zml//runtimes:cuda=true --@zml//runtimes:cpu=false" + bazel_opts: "--@zml//platforms:cuda=true --@zml//platforms:cpu=false" enable_cache: false user: actions runs-on: ["self-hosted", "platform=cuda"] - target: rocm - bazel_opts: "--@zml//runtimes:rocm=true --@zml//runtimes:cpu=false" + bazel_opts: "--@zml//platforms:rocm=true --@zml//platforms:cpu=false" enable_cache: false user: actions runs-on: ["self-hosted", "platform=rocm"] - target: cpu - bazel_opts: "--@zml//runtimes:cpu=true" + bazel_opts: "--@zml//platforms:cpu=true" enable_cache: true user: runner runs-on: diff --git a/MODULE.bazel b/MODULE.bazel index 14218ae3..972e2686 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -79,7 +79,7 @@ pip.parse( ], hub_name = "neuron_py_deps", python_version = "3.12", - requirements_lock = "//runtimes/neuron:requirements.lock.txt", + requirements_lock = "//platforms/neuron:requirements.lock.txt", ) use_repo(pip, "neuron_py_deps") @@ -92,19 +92,19 @@ pip.parse( ) use_repo(pip, "huggingface_hub") -cpu = use_extension("//runtimes/cpu:cpu.bzl", "cpu_pjrt_plugin") +cpu = use_extension("//platforms/cpu:cpu.bzl", "cpu_pjrt_plugin") use_repo(cpu, "libpjrt_cpu_darwin_amd64", "libpjrt_cpu_darwin_arm64", "libpjrt_cpu_linux_amd64") -cuda = use_extension("//runtimes/cuda:cuda.bzl", "cuda_packages") +cuda = use_extension("//platforms/cuda:cuda.bzl", "cuda_packages") use_repo(cuda, "libpjrt_cuda") -rocm = use_extension("//runtimes/rocm:rocm.bzl", "rocm_packages") +rocm = use_extension("//platforms/rocm:rocm.bzl", "rocm_packages") use_repo(rocm, "hipblaslt", "libpjrt_rocm", "rocblas") -tpu = use_extension("//runtimes/tpu:tpu.bzl", "tpu_packages") +tpu = use_extension("//platforms/tpu:tpu.bzl", "tpu_packages") use_repo(tpu, "libpjrt_tpu") -neuron = use_extension("//runtimes/neuron:neuron.bzl", "neuron_packages") +neuron = use_extension("//platforms/neuron:neuron.bzl", "neuron_packages") use_repo(neuron, "aws-neuronx-collectives", "aws-neuronx-runtime-lib", "libgomp1", "libpjrt_neuron", "zlib1g") zls = use_extension("//third_party/zls:zls_toolchain.bzl", "zls_toolchains") @@ -113,7 +113,7 @@ use_repo(zls, "zls_toolchains") register_toolchains("@zls_toolchains//:all") non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") -use_repo(non_module_deps, "arocc", "com_google_sentencepiece", "mnist", "org_swig_swig", "translate-c", "xla", "flashattn") +use_repo(non_module_deps, "arocc", "com_google_sentencepiece", "flashattn", "mnist", "org_swig_swig", "translate-c", "xla") xla = use_extension("//third_party/xla:xla.bzl", "xla") use_repo( @@ -170,22 +170,22 @@ use_repo(crate, "crates") apt = use_extension("@rules_distroless//apt:extensions.bzl", "apt") apt.install( name = "apt_cuda", - lock = "//runtimes/cuda:packages.lock.json", - manifest = "//runtimes/cuda:packages.yaml", + lock = "//platforms/cuda:packages.lock.json", + manifest = "//platforms/cuda:packages.yaml", ) use_repo(apt, "apt_cuda") apt.install( name = "apt_rocm", - lock = "//runtimes/rocm:packages.lock.json", - manifest = "//runtimes/rocm:packages.yaml", + lock = "//platforms/rocm:packages.lock.json", + manifest = "//platforms/rocm:packages.yaml", ) use_repo(apt, "apt_rocm") apt.install( name = "apt_neuron", - lock = "//runtimes/neuron:packages.lock.json", - manifest = "//runtimes/neuron:packages.yaml", + lock = "//platforms/neuron:packages.lock.json", + manifest = "//platforms/neuron:packages.yaml", ) use_repo(apt, "apt_neuron") diff --git a/README.md b/README.md index dbfc2ae8..b511fc1d 100644 --- a/README.md +++ b/README.md @@ -155,11 +155,11 @@ For a larger 3.2 model, you can also try `Llama-3.2-3B-Instruct`. You can compile models for accelerator runtimes by appending one or more of the following arguments to the command line when compiling / running a model: -- NVIDIA CUDA: `--@zml//runtimes:cuda=true` -- AMD RoCM: `--@zml//runtimes:rocm=true` -- Google TPU: `--@zml//runtimes:tpu=true` -- AWS Trainium/Inferentia 2: `--@zml//runtimes:neuron=true` -- **AVOID CPU:** `--@zml//runtimes:cpu=false` +- NVIDIA CUDA: `--@zml//platforms:cuda=true` +- AMD RoCM: `--@zml//platforms:rocm=true` +- Google TPU: `--@zml//platforms:tpu=true` +- AWS Trainium/Inferentia 2: `--@zml//platforms:neuron=true` +- **AVOID CPU:** `--@zml//platforms:cpu=false` The latter, avoiding compilation for CPU, cuts down compilation time. @@ -168,7 +168,7 @@ run the following: ``` bazel run --config=release //examples/llama \ - --@zml//runtimes:cuda=true \ + --@zml//platforms:cuda=true \ -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct \ --prompt="What is the capital of France?" ``` diff --git a/docs/howtos/deploy_on_server.md b/docs/howtos/deploy_on_server.md index 4c5b8f25..05a88889 100644 --- a/docs/howtos/deploy_on_server.md +++ b/docs/howtos/deploy_on_server.md @@ -14,17 +14,17 @@ information on how to compile a model. You can compile models for accelerator runtimes by appending one or more of the following arguments to the command line when compiling / running a model: -- NVIDIA CUDA: `--@zml//runtimes:cuda=true` -- AMD RoCM: `--@zml//runtimes:rocm=true` -- Google TPU: `--@zml//runtimes:tpu=true` -- AWS Trainium/Inferentia 2: `--@zml//runtimes:neuron=true` -- **AVOID CPU:** `--@zml//runtimes:cpu=false` +- NVIDIA CUDA: `--@zml//platforms:cuda=true` +- AMD RoCM: `--@zml//platforms:rocm=true` +- Google TPU: `--@zml//platforms:tpu=true` +- AWS Trainium/Inferentia 2: `--@zml//platforms:neuron=true` +- **AVOID CPU:** `--@zml//platforms:cpu=false` So, to run the Llama model from above **on your development machine** housing an NVIDIA GPU, run the following: ``` -bazel run --config=release //examples/llama --@zml//runtimes:cuda=true -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct +bazel run --config=release //examples/llama --@zml//platforms:cuda=true -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct ``` @@ -41,8 +41,8 @@ As an example, here is how you build above Llama for CUDA on Linux X86_64: ``` bazel build --config=release //examples/llama \ - --@zml//runtimes:cuda=true \ - --@zml//runtimes:cpu=false \ + --@zml//platforms:cuda=true \ + --@zml//platforms:cpu=false \ --platforms=@zml//platforms:linux_amd64 ``` @@ -82,8 +82,8 @@ tar( ``` bazel build --config=release //mnist:archive \ - --@zml//runtimes:cuda=true \ - --@zml//runtimes:cpu=false \ + --@zml//platforms:cuda=true \ + --@zml//platforms:cpu=false \ --platforms=@zml//platforms:linux_amd64 ``` diff --git a/docs/howtos/dockerize_models.md b/docs/howtos/dockerize_models.md index 0d2d8451..ac35990c 100644 --- a/docs/howtos/dockerize_models.md +++ b/docs/howtos/dockerize_models.md @@ -222,16 +222,16 @@ it out. **Note: Please add one more of the following parameters to specify all the platforms your containerized model should support.** -- NVIDIA CUDA: `--@zml//runtimes:cuda=true` -- AMD RoCM: `--@zml//runtimes:rocm=true` -- Google TPU: `--@zml//runtimes:tpu=true` -- AWS Trainium/Inferentia 2: `--@zml//runtimes:neuron=true` -- **AVOID CPU:** `--@zml//runtimes:cpu=false` +- NVIDIA CUDA: `--@zml//platforms:cuda=true` +- AMD RoCM: `--@zml//platforms:rocm=true` +- Google TPU: `--@zml//platforms:tpu=true` +- AWS Trainium/Inferentia 2: `--@zml//platforms:neuron=true` +- **AVOID CPU:** `--@zml//platforms:cpu=false` **Example:** ``` -bazel run //mnist:push --config=release --@zml//runtimes:cuda=true -- --repository index.docker.io/my_org/zml_mnist +bazel run //mnist:push --config=release --@zml//platforms:cuda=true -- --repository index.docker.io/my_org/zml_mnist ``` @@ -372,5 +372,5 @@ And that's it! With one simple bazel command, you can push a neatly packaged MNIST model, including weights and dataset, to the docker registry: ``` -bazel run //mnist:push --@zml//runtimes:cuda=true -- --repository index.docker.io/my_org/zml_mnist +bazel run //mnist:push --@zml//platforms:cuda=true -- --repository index.docker.io/my_org/zml_mnist ``` diff --git a/docs/tutorials/getting_started.md b/docs/tutorials/getting_started.md index d5debcbc..31f8b4db 100644 --- a/docs/tutorials/getting_started.md +++ b/docs/tutorials/getting_started.md @@ -105,11 +105,11 @@ bazel test //zml:test You can compile models for accelerator runtimes by appending one or more of the following arguments to the command line when compiling or running a model: -- NVIDIA CUDA: `--@zml//runtimes:cuda=true` -- AMD RoCM: `--@zml//runtimes:rocm=true` -- Google TPU: `--@zml//runtimes:tpu=true` -- AWS Trainium/Inferentia 2: `--@zml//runtimes:neuron=true` -- **AVOID CPU:** `--@zml//runtimes:cpu=false` +- NVIDIA CUDA: `--@zml//platforms:cuda=true` +- AMD RoCM: `--@zml//platforms:rocm=true` +- Google TPU: `--@zml//platforms:tpu=true` +- AWS Trainium/Inferentia 2: `--@zml//platforms:neuron=true` +- **AVOID CPU:** `--@zml//platforms:cpu=false` The latter, avoiding compilation for CPU, cuts down compilation time. @@ -119,7 +119,7 @@ run the following: ``` bazel run --config=release //examples/llama:Llama-3.2-1B-Instruct \ - --@zml//runtimes:cuda=true \ + --@zml//platforms:cuda=true \ -- --prompt="What is the capital of France?" ``` diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index 62a36b81..3fccf1ad 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -1,3 +1,43 @@ +load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") +load("@rules_zig//zig:defs.bzl", "zig_library") + +PLATFORMS = { + "cpu": True, + "cuda": False, + "rocm": False, + "tpu": False, + "neuron": False, +} + +[ + bool_flag( + name = runtime, + build_setting_default = default, + ) + for runtime, default in PLATFORMS.items() +] + +[ + config_setting( + name = "{}.enabled".format(runtime), + flag_values = {":{}".format(runtime): "True"}, + visibility = ["//platforms:__subpackages__"], + ) + for runtime in PLATFORMS.keys() +] + +zig_library( + name = "platforms", + main = "platforms.zig", + visibility = ["//visibility:public"], + deps = [ + "//pjrt", + ] + [ + "//platforms/{}".format(runtime) + for runtime in PLATFORMS.keys() + ], +) + platform( name = "linux_amd64", constraint_values = [ diff --git a/runtimes/cpu/BUILD.bazel b/platforms/cpu/BUILD.bazel similarity index 94% rename from runtimes/cpu/BUILD.bazel rename to platforms/cpu/BUILD.bazel index 1c502881..9f9295c0 100644 --- a/runtimes/cpu/BUILD.bazel +++ b/platforms/cpu/BUILD.bazel @@ -42,13 +42,13 @@ cc_library( zig_library( name = "cpu", - import_name = "runtimes/cpu", + import_name = "platforms/cpu", main = "cpu.zig", visibility = ["//visibility:public"], deps = [ "//pjrt", ] + select({ - "//runtimes:cpu.enabled": [ + "//platforms:cpu.enabled": [ ":libpjrt_cpu", "//stdx", "@rules_zig//zig/runfiles", diff --git a/runtimes/cpu/cpu.bzl b/platforms/cpu/cpu.bzl similarity index 90% rename from runtimes/cpu/cpu.bzl rename to platforms/cpu/cpu.bzl index 40dc257d..35d0746e 100644 --- a/runtimes/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -1,5 +1,5 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load("//runtimes/common:packages.bzl", "packages") +load("//platforms:packages.bzl", "packages") _BUILD_FILE_DEFAULT_VISIBILITY = """\ package(default_visibility = ["//visibility:public"]) @@ -9,14 +9,14 @@ _BUILD_LINUX = "\n".join([ packages.filegroup( name = "libpjrt_cpu", srcs = ["libpjrt_cpu.so"], - visibility = ["@zml//runtimes/cpu:__subpackages__"], + visibility = ["@zml//platforms/cpu:__subpackages__"], ), ]) _BUILD_DARWIN = packages.filegroup( name = "libpjrt_cpu", srcs = ["libpjrt_cpu.dylib"], - visibility = ["@zml//runtimes/cpu:__subpackages__"], + visibility = ["@zml//platforms/cpu:__subpackages__"], ) def _cpu_pjrt_plugin_impl(mctx): diff --git a/runtimes/cpu/cpu.zig b/platforms/cpu/cpu.zig similarity index 91% rename from runtimes/cpu/cpu.zig rename to platforms/cpu/cpu.zig index 35abd5bf..980c7cda 100644 --- a/runtimes/cpu/cpu.zig +++ b/platforms/cpu/cpu.zig @@ -7,7 +7,7 @@ const pjrt = @import("pjrt"); const runfiles = @import("runfiles"); const stdx = @import("stdx"); -const log = std.log.scoped(.@"zml/runtime/cpu"); +const log = std.log.scoped(.@"zml/platforms/cpu"); pub fn isEnabled() bool { return @hasDecl(c, "ZML_RUNTIME_CPU"); @@ -29,7 +29,7 @@ pub fn load(io: std.Io) !*const pjrt.Api { const r = r_.withSourceRepo(source_repo); var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = try r.rlocation("zml/runtimes/cpu/sandbox/lib", &path_buf) orelse { + const sandbox_path = try r.rlocation("zml/platforms/cpu/sandbox/lib", &path_buf) orelse { log.err("Failed to find sandbox path for CPU runtime", .{}); return error.FileNotFound; }; diff --git a/runtimes/cuda/BUILD.bazel b/platforms/cuda/BUILD.bazel similarity index 90% rename from runtimes/cuda/BUILD.bazel rename to platforms/cuda/BUILD.bazel index 34d976a2..15bb10e6 100644 --- a/runtimes/cuda/BUILD.bazel +++ b/platforms/cuda/BUILD.bazel @@ -27,18 +27,18 @@ cc_library( zig_library( name = "cuda", data = select({ - "//runtimes:cuda.enabled": [ + "//platforms:cuda.enabled": [ "@flashattn//:libflashattn", ], "//conditions:default": [":empty"], }), - import_name = "runtimes/cuda", + import_name = "platforms/cuda", main = "cuda.zig", visibility = ["//visibility:public"], deps = [ "//pjrt", ] + select({ - "//runtimes:cuda.enabled": [ + "//platforms:cuda.enabled": [ ":libpjrt_cuda", "//stdx", "@rules_zig//zig/runfiles", diff --git a/runtimes/cuda/cuda.bzl b/platforms/cuda/cuda.bzl similarity index 98% rename from runtimes/cuda/cuda.bzl rename to platforms/cuda/cuda.bzl index ea2e0a95..fd95bf77 100644 --- a/runtimes/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -1,7 +1,7 @@ load("@bazel_skylib//lib:paths.bzl", "paths") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") -load("//runtimes/common:packages.bzl", "packages") +load("//platforms:packages.bzl", "packages") _BUILD_FILE_DEFAULT_VISIBILITY = """\ package(default_visibility = ["//visibility:public"]) @@ -151,7 +151,7 @@ def _read_redist_json(mctx, url, sha256): def _cuda_impl(mctx): loaded_packages = packages.read(mctx, [ - "@zml//runtimes/cuda:packages.lock.json", + "@zml//platforms/cuda:packages.lock.json", ]) CUDA_REDIST = _read_redist_json( mctx, diff --git a/runtimes/cuda/cuda.zig b/platforms/cuda/cuda.zig similarity index 98% rename from runtimes/cuda/cuda.zig rename to platforms/cuda/cuda.zig index 26796e37..244c97f4 100644 --- a/runtimes/cuda/cuda.zig +++ b/platforms/cuda/cuda.zig @@ -9,7 +9,7 @@ const stdx = @import("stdx"); const nvidiaLibsPath = "/cuda/"; -const log = std.log.scoped(.@"zml/runtime/cuda"); +const log = std.log.scoped(.@"zml/platforms/cuda"); pub fn isEnabled() bool { return @hasDecl(c, "ZML_RUNTIME_CUDA"); diff --git a/runtimes/cuda/libpjrt_cuda.BUILD.bazel b/platforms/cuda/libpjrt_cuda.BUILD.bazel similarity index 94% rename from runtimes/cuda/libpjrt_cuda.BUILD.bazel rename to platforms/cuda/libpjrt_cuda.BUILD.bazel index 92c3ca31..8354809f 100644 --- a/runtimes/cuda/libpjrt_cuda.BUILD.bazel +++ b/platforms/cuda/libpjrt_cuda.BUILD.bazel @@ -32,7 +32,7 @@ copy_to_directory( "@libnvjitlink", "@nccl", "@zlib1g", - "@zml//runtimes/cuda:zmlxcuda", + "@zml//platforms/cuda:zmlxcuda", ], replace_prefixes = { "nvidia/nccl/lib": "lib", @@ -61,5 +61,5 @@ cc_library( # is the default in most bazel CC toolchains as well as in certain linkers. "-Wl,-z,lazy", ], - visibility = ["@zml//runtimes/cuda:__subpackages__"], + visibility = ["@zml//platforms/cuda:__subpackages__"], ) diff --git a/runtimes/cuda/libpjrt_cuda.h b/platforms/cuda/libpjrt_cuda.h similarity index 100% rename from runtimes/cuda/libpjrt_cuda.h rename to platforms/cuda/libpjrt_cuda.h diff --git a/runtimes/cuda/packages.lock.json b/platforms/cuda/packages.lock.json similarity index 100% rename from runtimes/cuda/packages.lock.json rename to platforms/cuda/packages.lock.json diff --git a/runtimes/cuda/packages.yaml b/platforms/cuda/packages.yaml similarity index 100% rename from runtimes/cuda/packages.yaml rename to platforms/cuda/packages.yaml diff --git a/runtimes/cuda/zmlxcuda.zig b/platforms/cuda/zmlxcuda.zig similarity index 100% rename from runtimes/cuda/zmlxcuda.zig rename to platforms/cuda/zmlxcuda.zig diff --git a/runtimes/neuron/BUILD.bazel b/platforms/neuron/BUILD.bazel similarity index 97% rename from runtimes/neuron/BUILD.bazel rename to platforms/neuron/BUILD.bazel index 88ea3fb2..104e70d2 100644 --- a/runtimes/neuron/BUILD.bazel +++ b/platforms/neuron/BUILD.bazel @@ -107,13 +107,13 @@ uv_lock( zig_library( name = "neuron", - import_name = "runtimes/neuron", + import_name = "platforms/neuron", main = "neuron.zig", visibility = ["//visibility:public"], deps = [ "//pjrt", ] + select({ - "//runtimes:neuron.enabled": [ + "//platforms:neuron.enabled": [ ":libnrt_headers", ":libpython", ":zmlxneuron", diff --git a/runtimes/neuron/libneuronxla.zig b/platforms/neuron/libneuronxla.zig similarity index 99% rename from runtimes/neuron/libneuronxla.zig rename to platforms/neuron/libneuronxla.zig index 57d0dea1..8185cf56 100644 --- a/runtimes/neuron/libneuronxla.zig +++ b/platforms/neuron/libneuronxla.zig @@ -4,7 +4,7 @@ const c = @import("c"); const stdx = @import("stdx"); const upb = @import("upb"); -const log = std.log.scoped(.@"zml/runtimes/neuron/libneuronxla"); +const log = std.log.scoped(.@"zml/platforms/neuron/libneuronxla"); pub fn makeTempDir(io: std.Io, buf: []u8, prefix: []const u8) ![]const u8 { const tmp_dir = std.posix.getenv("TMPDIR") orelse "/tmp"; diff --git a/runtimes/neuron/libpjrt_neuron.BUILD.bazel b/platforms/neuron/libpjrt_neuron.BUILD.bazel similarity index 83% rename from runtimes/neuron/libpjrt_neuron.BUILD.bazel rename to platforms/neuron/libpjrt_neuron.BUILD.bazel index 876d9d9b..174e2a39 100644 --- a/runtimes/neuron/libpjrt_neuron.BUILD.bazel +++ b/platforms/neuron/libpjrt_neuron.BUILD.bazel @@ -1,10 +1,7 @@ load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") -load("@zml//bazel:patchelf.bzl", "patchelf") -load("@bazel_skylib//rules:copy_file.bzl", "copy_file") load("@bazel_skylib//rules:select_file.bzl", "select_file") -load("@rules_cc//cc:cc_shared_library.bzl", "cc_shared_library") +load("@zml//bazel:patchelf.bzl", "patchelf") load("@zml//bazel:runfiles.bzl", "runfiles_to_default") -load("@rules_python//python:py_binary.bzl", "py_binary") runfiles_to_default( name = "libneuronxla_files", @@ -19,15 +16,15 @@ select_file( patchelf( name = "libneuronpjrt_so", - set_rpath = '$ORIGIN', + set_rpath = "$ORIGIN", src = ":libneuronpjrt_so_orig", soname = "libneuronpjrt.so", ) patchelf( name = "libpjrt_neuron_so", - set_rpath = '$ORIGIN', - src = "@zml//runtimes/neuron:libpjrt_neuron", + set_rpath = "$ORIGIN", + src = "@zml//platforms/neuron:libpjrt_neuron", soname = "libpjrt_neuron.so", ) @@ -36,8 +33,8 @@ copy_to_directory( srcs = [ ":libneuronpjrt_so", ":libpjrt_neuron_so", - "@zml//runtimes/neuron:neuronx-cc_files", - "@zml//runtimes/neuron:libneuronxla", + "@zml//platforms/neuron:neuronx-cc_files", + "@zml//platforms/neuron:libneuronxla", "@rules_python//python:current_py_toolchain", "@aws-neuronx-runtime-lib//:libnrt.patchelf", "@aws-neuronx-runtime-lib//:libncfw.patchelf", @@ -90,11 +87,11 @@ cc_library( # is the default in most bazel CC toolchains as well as in certain linkers. "-Wl,-z,lazy", ], - visibility = ["@zml//runtimes/neuron:__subpackages__"], + visibility = ["@zml//platforms/neuron:__subpackages__"], ) cc_library( name = "libpjrt_neuron", data = [":sandbox"], - visibility = ["@zml//runtimes/neuron:__subpackages__"], + visibility = ["@zml//platforms/neuron:__subpackages__"], ) diff --git a/runtimes/neuron/libpjrt_neuron.zig b/platforms/neuron/libpjrt_neuron.zig similarity index 99% rename from runtimes/neuron/libpjrt_neuron.zig rename to platforms/neuron/libpjrt_neuron.zig index e87d1f23..1d25cde6 100644 --- a/runtimes/neuron/libpjrt_neuron.zig +++ b/platforms/neuron/libpjrt_neuron.zig @@ -5,7 +5,7 @@ const c = @import("c"); const runfiles = @import("runfiles"); const stdx = @import("stdx"); -const log = std.log.scoped(.@"zml/runtimes/neuron"); +const log = std.log.scoped(.@"zml/platforms/neuron"); fn findFreeTcpPort(io: std.Io) !u16 { var server = try std.Io.net.IpAddress.listen( diff --git a/runtimes/neuron/libpython.h b/platforms/neuron/libpython.h similarity index 100% rename from runtimes/neuron/libpython.h rename to platforms/neuron/libpython.h diff --git a/runtimes/neuron/neuron.bzl b/platforms/neuron/neuron.bzl similarity index 96% rename from runtimes/neuron/neuron.bzl rename to platforms/neuron/neuron.bzl index d329c319..409904ca 100644 --- a/runtimes/neuron/neuron.bzl +++ b/platforms/neuron/neuron.bzl @@ -3,7 +3,7 @@ load("@rules_python//python:defs.bzl", "PyInfo") load("@with_cfg.bzl", "with_cfg") load("//bazel:http_deb_archive.bzl", "http_deb_archive") load("//bazel:simple_repository.bzl", "simple_repository") -load("//runtimes/common:packages.bzl", "packages") +load("//platforms:packages.bzl", "packages") BASE_URL = "https://apt.repos.neuron.amazonaws.com" STRIP_PREFIX = "opt/aws/neuron" @@ -61,7 +61,7 @@ _NEURON_PACKAGES = { def _neuron_impl(mctx): loaded_packages = packages.read(mctx, [ - "@zml//runtimes/neuron:packages.lock.json", + "@zml//platforms/neuron:packages.lock.json", ]) simple_repository( diff --git a/runtimes/neuron/neuron.zig b/platforms/neuron/neuron.zig similarity index 97% rename from runtimes/neuron/neuron.zig rename to platforms/neuron/neuron.zig index 5ea729c3..7ffcc1c4 100644 --- a/runtimes/neuron/neuron.zig +++ b/platforms/neuron/neuron.zig @@ -7,7 +7,7 @@ const pjrt = @import("pjrt"); const runfiles = @import("runfiles"); const stdx = @import("stdx"); -const log = std.log.scoped(.@"zml/runtime/neuron"); +const log = std.log.scoped(.@"zml/platforms/neuron"); pub fn isEnabled() bool { return @hasDecl(c, "ZML_RUNTIME_NEURON"); diff --git a/runtimes/neuron/neuronx-cc.zig b/platforms/neuron/neuronx-cc.zig similarity index 100% rename from runtimes/neuron/neuronx-cc.zig rename to platforms/neuron/neuronx-cc.zig diff --git a/runtimes/neuron/nrt.h b/platforms/neuron/nrt.h similarity index 100% rename from runtimes/neuron/nrt.h rename to platforms/neuron/nrt.h diff --git a/runtimes/neuron/packages.lock.json b/platforms/neuron/packages.lock.json similarity index 100% rename from runtimes/neuron/packages.lock.json rename to platforms/neuron/packages.lock.json diff --git a/runtimes/neuron/packages.yaml b/platforms/neuron/packages.yaml similarity index 100% rename from runtimes/neuron/packages.yaml rename to platforms/neuron/packages.yaml diff --git a/runtimes/neuron/requirements.in b/platforms/neuron/requirements.in similarity index 100% rename from runtimes/neuron/requirements.in rename to platforms/neuron/requirements.in diff --git a/runtimes/neuron/requirements.lock.txt b/platforms/neuron/requirements.lock.txt similarity index 99% rename from runtimes/neuron/requirements.lock.txt rename to platforms/neuron/requirements.lock.txt index cf1301fa..19c7bb90 100644 --- a/runtimes/neuron/requirements.lock.txt +++ b/platforms/neuron/requirements.lock.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# bazel run //runtimes/neuron:requirements.update +# bazel run //platforms/neuron:requirements.update --index-url https://pypi.org/simple --extra-index-url https://pip.repos.neuron.amazonaws.com --find-links https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html diff --git a/runtimes/common/packages.bzl b/platforms/packages.bzl similarity index 100% rename from runtimes/common/packages.bzl rename to platforms/packages.bzl diff --git a/runtimes/runtimes.zig b/platforms/platforms.zig similarity index 76% rename from runtimes/runtimes.zig rename to platforms/platforms.zig index 76311ee6..d89f77f8 100644 --- a/runtimes/runtimes.zig +++ b/platforms/platforms.zig @@ -1,11 +1,11 @@ const std = @import("std"); +const cpu = @import("platforms/cpu"); +const cuda = @import("platforms/cuda"); +const neuron = @import("platforms/neuron"); const pjrt = @import("pjrt"); -const cpu = @import("runtimes/cpu"); -const cuda = @import("runtimes/cuda"); -const rocm = @import("runtimes/rocm"); -const tpu = @import("runtimes/tpu"); -const neuron = @import("runtimes/neuron"); +const rocm = @import("platforms/rocm"); +const tpu = @import("platforms/tpu"); pub const Platform = enum { cpu, diff --git a/runtimes/rocm/BUILD.bazel b/platforms/rocm/BUILD.bazel similarity index 93% rename from runtimes/rocm/BUILD.bazel rename to platforms/rocm/BUILD.bazel index 28f3438f..2b625ad6 100644 --- a/runtimes/rocm/BUILD.bazel +++ b/platforms/rocm/BUILD.bazel @@ -35,13 +35,13 @@ cc_library( zig_library( name = "rocm", - import_name = "runtimes/rocm", + import_name = "platforms/rocm", main = "rocm.zig", visibility = ["//visibility:public"], deps = [ "//pjrt", ] + select({ - "//runtimes:rocm.enabled": [ + "//platforms:rocm.enabled": [ ":libpjrt_rocm", "//stdx", "@rules_zig//zig/runfiles", diff --git a/runtimes/rocm/gfx.bzl b/platforms/rocm/gfx.bzl similarity index 96% rename from runtimes/rocm/gfx.bzl rename to platforms/rocm/gfx.bzl index fd47cb8c..aa2d325d 100644 --- a/runtimes/rocm/gfx.bzl +++ b/platforms/rocm/gfx.bzl @@ -42,6 +42,6 @@ bytecode_select = rule( def if_gfx(gfx, value): return select({ - "@zml//runtimes/rocm:_{}".format(gfx): value, + "@zml//platforms/rocm:_{}".format(gfx): value, "//conditions:default": [], }) diff --git a/runtimes/rocm/libpjrt_rocm.BUILD.bazel b/platforms/rocm/libpjrt_rocm.BUILD.bazel similarity index 96% rename from runtimes/rocm/libpjrt_rocm.BUILD.bazel rename to platforms/rocm/libpjrt_rocm.BUILD.bazel index 72fd2a12..fbd1dd54 100644 --- a/runtimes/rocm/libpjrt_rocm.BUILD.bazel +++ b/platforms/rocm/libpjrt_rocm.BUILD.bazel @@ -68,7 +68,7 @@ copy_to_directory( "@roctracer", "@roctracer//:roctx", "@zlib1g", - "@zml//runtimes/rocm:zmlxrocm", + "@zml//platforms/rocm:zmlxrocm", ] + select({ ":_hipblaslt": ["@hipblaslt//:runfiles"], "//conditions:default": [], @@ -100,5 +100,5 @@ tar( cc_library( name = "libpjrt_rocm", data = [":sandbox"], - visibility = ["@zml//runtimes/rocm:__subpackages__"], + visibility = ["@zml//platforms/rocm:__subpackages__"], ) diff --git a/runtimes/rocm/packages.lock.json b/platforms/rocm/packages.lock.json similarity index 100% rename from runtimes/rocm/packages.lock.json rename to platforms/rocm/packages.lock.json diff --git a/runtimes/rocm/packages.yaml b/platforms/rocm/packages.yaml similarity index 100% rename from runtimes/rocm/packages.yaml rename to platforms/rocm/packages.yaml diff --git a/runtimes/rocm/rocm.bzl b/platforms/rocm/rocm.bzl similarity index 96% rename from runtimes/rocm/rocm.bzl rename to platforms/rocm/rocm.bzl index 0d392389..a055767a 100644 --- a/runtimes/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -1,6 +1,6 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") -load("//runtimes/common:packages.bzl", "packages") +load("//platforms:packages.bzl", "packages") _BUILD_FILE_DEFAULT_VISIBILITY = """\ package(default_visibility = ["//visibility:public"]) @@ -61,7 +61,7 @@ _ROCM_PACKAGES = { "hip-dev": """filegroup(name = "runfiles", srcs = glob(["share/**"]))""", "rocblas": "\n".join([ packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), - packages.load_("@zml//runtimes/rocm:gfx.bzl", "bytecode_select"), + packages.load_("@zml//platforms/rocm:gfx.bzl", "bytecode_select"), packages.patchelf( name = "rocblas", src = "lib/librocblas.so.5", @@ -92,7 +92,7 @@ _ROCM_PACKAGES = { ]), "hipblaslt": "\n".join([ packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), - packages.load_("@zml//runtimes/rocm:gfx.bzl", "bytecode_select"), + packages.load_("@zml//platforms/rocm:gfx.bzl", "bytecode_select"), packages.patchelf( name = "hipblaslt", src = "lib/libhipblaslt.so.1", @@ -145,7 +145,7 @@ _ROCM_PACKAGES = { def _rocm_impl(mctx): loaded_packages = packages.read(mctx, [ - "@zml//runtimes/rocm:packages.lock.json", + "@zml//platforms/rocm:packages.lock.json", ]) for pkg_name, build_file_content in _UBUNTU_PACKAGES.items(): diff --git a/runtimes/rocm/rocm.zig b/platforms/rocm/rocm.zig similarity index 97% rename from runtimes/rocm/rocm.zig rename to platforms/rocm/rocm.zig index 95ac8d91..6eaa32d0 100644 --- a/runtimes/rocm/rocm.zig +++ b/platforms/rocm/rocm.zig @@ -7,7 +7,7 @@ const pjrt = @import("pjrt"); const runfiles = @import("runfiles"); const stdx = @import("stdx"); -const log = std.log.scoped(.@"zml/runtime/rocm"); +const log = std.log.scoped(.@"zml/platforms/rocm"); pub fn isEnabled() bool { return @hasDecl(c, "ZML_RUNTIME_ROCM"); diff --git a/runtimes/rocm/zmlxrocm.zig b/platforms/rocm/zmlxrocm.zig similarity index 100% rename from runtimes/rocm/zmlxrocm.zig rename to platforms/rocm/zmlxrocm.zig diff --git a/runtimes/tpu/BUILD.bazel b/platforms/tpu/BUILD.bazel similarity index 88% rename from runtimes/tpu/BUILD.bazel rename to platforms/tpu/BUILD.bazel index 22b56558..876255c7 100644 --- a/runtimes/tpu/BUILD.bazel +++ b/platforms/tpu/BUILD.bazel @@ -14,13 +14,13 @@ cc_library( zig_library( name = "tpu", - import_name = "runtimes/tpu", + import_name = "platforms/tpu", main = "tpu.zig", visibility = ["//visibility:public"], deps = [ "//pjrt", ] + select({ - "//runtimes:tpu.enabled": [ + "//platforms:tpu.enabled": [ ":libpjrt_tpu", "//stdx", "@rules_zig//zig/runfiles", diff --git a/runtimes/tpu/libpjrt_tpu.BUILD.bazel b/platforms/tpu/libpjrt_tpu.BUILD.bazel similarity index 87% rename from runtimes/tpu/libpjrt_tpu.BUILD.bazel rename to platforms/tpu/libpjrt_tpu.BUILD.bazel index 338867be..767cd004 100644 --- a/runtimes/tpu/libpjrt_tpu.BUILD.bazel +++ b/platforms/tpu/libpjrt_tpu.BUILD.bazel @@ -17,5 +17,5 @@ copy_to_directory( cc_library( name = "libpjrt_tpu", data = [":sandbox"], - visibility = ["@zml//runtimes/tpu:__subpackages__"], + visibility = ["@zml//platforms/tpu:__subpackages__"], ) diff --git a/runtimes/tpu/tpu.bzl b/platforms/tpu/tpu.bzl similarity index 100% rename from runtimes/tpu/tpu.bzl rename to platforms/tpu/tpu.bzl diff --git a/runtimes/tpu/tpu.zig b/platforms/tpu/tpu.zig similarity index 97% rename from runtimes/tpu/tpu.zig rename to platforms/tpu/tpu.zig index 83d6c4a9..511f420c 100644 --- a/runtimes/tpu/tpu.zig +++ b/platforms/tpu/tpu.zig @@ -7,7 +7,7 @@ const pjrt = @import("pjrt"); const runfiles = @import("runfiles"); const stdx = @import("stdx"); -const log = std.log.scoped(.@"zml/runtime/tpu"); +const log = std.log.scoped(.@"zml/platforms/tpu"); pub fn isEnabled() bool { return @hasDecl(c, "ZML_RUNTIME_TPU"); diff --git a/runtimes/BUILD.bazel b/runtimes/BUILD.bazel deleted file mode 100644 index 4f887480..00000000 --- a/runtimes/BUILD.bazel +++ /dev/null @@ -1,39 +0,0 @@ -load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") -load("@rules_zig//zig:defs.bzl", "zig_library") - -RUNTIMES = { - "cpu": True, - "cuda": False, - "rocm": False, - "tpu": False, - "neuron": False, -} - -[ - bool_flag( - name = runtime, - build_setting_default = default, - ) - for runtime, default in RUNTIMES.items() -] - -[ - config_setting( - name = "{}.enabled".format(runtime), - flag_values = {":{}".format(runtime): "True"}, - visibility = ["//runtimes:__subpackages__"], - ) - for runtime in RUNTIMES.keys() -] - -zig_library( - name = "runtimes", - main = "runtimes.zig", - visibility = ["//visibility:public"], - deps = [ - "//pjrt", - ] + [ - "//runtimes/{}".format(runtime) - for runtime in RUNTIMES.keys() - ], -) diff --git a/runtimes/common/BUILD.bazel b/runtimes/common/BUILD.bazel deleted file mode 100644 index e69de29b..00000000 diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index f064a956..1206c00a 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -1,6 +1,6 @@ load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_c_proto_library") load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_test") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") upb_c_proto_library( name = "xla_data_upb", @@ -20,6 +20,9 @@ cc_library( zig_library( name = "zml", srcs = [ + "attention/attention.zig", + "attention/flashattn/bindings.zig", + "attention/flashattn/flashattn.zig", "buffer.zig", "constants.zig", "dtype.zig", @@ -39,9 +42,6 @@ zig_library( "tensor.zig", "testing.zig", "zml.zig", - "attention/attention.zig", - "attention/flashattn/flashattn.zig", - "attention/flashattn/bindings.zig", ], main = "zml.zig", visibility = ["//visibility:public"], @@ -56,13 +56,13 @@ zig_library( "//mlir", "//mlir/dialects", "//pjrt", - "//runtimes", + "//platforms", "//stdx", "//upb", - "//zml/tokenizer", "//zml/io", - "@rules_zig//zig/runfiles", + "//zml/tokenizer", "@flashattn//:c", + "@rules_zig//zig/runfiles", ], ) diff --git a/zml/platform.zig b/zml/platform.zig index 61e697d3..d2366fd9 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -3,8 +3,8 @@ const std = @import("std"); const c = @import("c"); const pjrt = @import("pjrt"); const pjrtx = @import("pjrt"); -const runtimes = @import("runtimes"); -pub const Target = runtimes.Platform; +const platforms = @import("platforms"); +pub const Target = platforms.Platform; const stdx = @import("stdx"); const Exe = @import("exe.zig").Exe; @@ -53,7 +53,7 @@ fn loadOrGetApi(target: Target, io: std.Io) !*const pjrt.Api { return switch (target) { inline else => |tag| @field(api_map, @tagName(tag)) orelse b: { disableXlaLogs(); - const api = try runtimes.load(tag, io); + const api = try platforms.load(tag, io); @field(api_map, @tagName(tag)) = api; break :b api; }, @@ -71,7 +71,7 @@ pub const Platform = struct { // `const comp = platform.compiler(compile_opts); const exe = comp.compile(...);` compilation_options: CompilationOptions = .{}, - pub const MAX_NUM_DEVICES: u8 = if (runtimes.isEnabled(.tpu)) 32 else 8; + pub const MAX_NUM_DEVICES: u8 = if (platforms.isEnabled(.tpu)) 32 else 8; pub const Device = struct { pub const Iterator = struct { From 21a7c5fec8fde7c115297db6c498caa4e9adb7c7 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 19 Jan 2026 17:41:57 +0100 Subject: [PATCH 247/881] platforms: pass allocator to load --- examples/benchmark/main.zig | 2 +- examples/llama/main.zig | 2 +- examples/mnist/mnist.zig | 2 +- platforms/cpu/cpu.zig | 4 ++-- platforms/cuda/cuda.zig | 4 ++-- platforms/neuron/neuron.zig | 4 ++-- platforms/platforms.zig | 12 ++++++------ platforms/rocm/rocm.zig | 4 ++-- platforms/tpu/tpu.zig | 4 ++-- zml/platform.zig | 12 ++++++------ 10 files changed, 25 insertions(+), 25 deletions(-) diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index 06d4f047..1fc50d90 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -25,7 +25,7 @@ pub fn main() !void { const io = threaded.io(); // Auto-select platform - const platform: zml.Platform = try .auto(io, .{}); + const platform: zml.Platform = try .auto(allocator, io, .{}); const cli_args: CliArgs = blk: { var ret: CliArgs = .{}; diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 014b9bbc..4cf73bab 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -119,7 +119,7 @@ pub fn main() !void { }, }; - var platform: zml.Platform = try .auto(io, .{}); + var platform: zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(); var it = platform.devicesIterator(); log.info("Devices:", .{}); diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index b6888dec..1c6178e6 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -98,7 +98,7 @@ pub fn main() !void { const mnist_model: Mnist = .init(store.view()); // Auto-select platform - const platform: zml.Platform = try .auto(io, .{}); + const platform: zml.Platform = try .auto(allocator, io, .{}); // Compile model const input: zml.Tensor = .init(.{ 28, 28 }, .u8); diff --git a/platforms/cpu/cpu.zig b/platforms/cpu/cpu.zig index 980c7cda..923ece24 100644 --- a/platforms/cpu/cpu.zig +++ b/platforms/cpu/cpu.zig @@ -13,12 +13,12 @@ pub fn isEnabled() bool { return @hasDecl(c, "ZML_RUNTIME_CPU"); } -pub fn load(io: std.Io) !*const pjrt.Api { +pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { if (comptime !isEnabled()) { return error.Unavailable; } - var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); + var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator(), .io = io }) orelse { diff --git a/platforms/cuda/cuda.zig b/platforms/cuda/cuda.zig index 244c97f4..9bcf07bb 100644 --- a/platforms/cuda/cuda.zig +++ b/platforms/cuda/cuda.zig @@ -32,7 +32,7 @@ fn setupXlaGpuCudaDirFlag(allocator: std.mem.Allocator, sandbox: []const u8) !vo _ = c.setenv("XLA_FLAGS", new_xla_flagsZ, 1); } -pub fn load(io: std.Io) !*const pjrt.Api { +pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { if (comptime !isEnabled()) { return error.Unavailable; } @@ -46,7 +46,7 @@ pub fn load(io: std.Io) !*const pjrt.Api { log.warn("Detected {s} in LD_LIBRARY_PATH. This can lead to undefined behaviors and crashes", .{nvidiaLibsPath}); } - var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); + var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator(), .io = io }) orelse { diff --git a/platforms/neuron/neuron.zig b/platforms/neuron/neuron.zig index 7ffcc1c4..89eb8c4b 100644 --- a/platforms/neuron/neuron.zig +++ b/platforms/neuron/neuron.zig @@ -33,7 +33,7 @@ fn isRunningOnEC2(io: std.Io) !bool { ); } -pub fn load(io: std.Io) !*const pjrt.Api { +pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { if (comptime !isEnabled()) { return error.Unavailable; } @@ -47,7 +47,7 @@ pub fn load(io: std.Io) !*const pjrt.Api { return error.Unavailable; } - var arena = std.heap.ArenaAllocator.init(std.heap.smp_allocator); + var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator(), .io = io }) orelse { diff --git a/platforms/platforms.zig b/platforms/platforms.zig index d89f77f8..4033415c 100644 --- a/platforms/platforms.zig +++ b/platforms/platforms.zig @@ -15,13 +15,13 @@ pub const Platform = enum { neuron, }; -pub fn load(tag: Platform, io: std.Io) !*const pjrt.Api { +pub fn load(allocator: std.mem.Allocator, io: std.Io, tag: Platform) !*const pjrt.Api { return switch (tag) { - .cpu => try cpu.load(io), - .cuda => try cuda.load(io), - .rocm => try rocm.load(io), - .tpu => try tpu.load(io), - .neuron => try neuron.load(io), + .cpu => try cpu.load(allocator, io), + .cuda => try cuda.load(allocator, io), + .rocm => try rocm.load(allocator, io), + .tpu => try tpu.load(allocator, io), + .neuron => try neuron.load(allocator, io), }; } diff --git a/platforms/rocm/rocm.zig b/platforms/rocm/rocm.zig index 6eaa32d0..616be6da 100644 --- a/platforms/rocm/rocm.zig +++ b/platforms/rocm/rocm.zig @@ -25,7 +25,7 @@ fn setupRocmEnv(rocm_data_dir: []const u8) !void { _ = c.setenv("ROCM_PATH", try stdx.Io.Dir.path.bufJoinZ(&buf, &.{rocm_data_dir}), 1); // must be zero terminated } -pub fn load(io: std.Io) !*const pjrt.Api { +pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { if (comptime !isEnabled()) { return error.Unavailable; } @@ -36,7 +36,7 @@ pub fn load(io: std.Io) !*const pjrt.Api { return error.Unavailable; } - var arena = std.heap.ArenaAllocator.init(std.heap.smp_allocator); + var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator(), .io = io }) orelse { diff --git a/platforms/tpu/tpu.zig b/platforms/tpu/tpu.zig index 511f420c..af2395ff 100644 --- a/platforms/tpu/tpu.zig +++ b/platforms/tpu/tpu.zig @@ -32,7 +32,7 @@ fn isOnGCP(io: std.Io) !bool { ); } -pub fn load(io: std.Io) !*const pjrt.Api { +pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { if (comptime !isEnabled()) { return error.Unavailable; } @@ -43,7 +43,7 @@ pub fn load(io: std.Io) !*const pjrt.Api { return error.Unavailable; } - var arena = std.heap.ArenaAllocator.init(std.heap.smp_allocator); + var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator() }) orelse { diff --git a/zml/platform.zig b/zml/platform.zig index d2366fd9..2f70a978 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -49,11 +49,11 @@ fn disableXlaLogs() void { ); } -fn loadOrGetApi(target: Target, io: std.Io) !*const pjrt.Api { +fn loadOrGetApi(allocator: std.mem.Allocator, io: std.Io, target: Target) !*const pjrt.Api { return switch (target) { inline else => |tag| @field(api_map, @tagName(tag)) orelse b: { disableXlaLogs(); - const api = try platforms.load(tag, io); + const api = try platforms.load(allocator, io, tag); @field(api_map, @tagName(tag)) = api; break :b api; }, @@ -100,8 +100,8 @@ pub const Platform = struct { } }; - pub fn init(target: Target, io: std.Io, options: CreateOptions) !Platform { - const api = try loadOrGetApi(target, io); + pub fn init(allocator: std.mem.Allocator, io: std.Io, target: Target, options: CreateOptions) !Platform { + const api = try loadOrGetApi(allocator, io, target); var named_values_buf: [16]pjrt.NamedValue = undefined; const pjrt_client = try pjrt.Client.init(api, options.toNamedValues(target, &named_values_buf)); @@ -133,7 +133,7 @@ pub const Platform = struct { return platform; } - pub fn auto(io: std.Io, options: CreateOptions) !Platform { + pub fn auto(allocator: std.mem.Allocator, io: std.Io, options: CreateOptions) !Platform { const ordered_targets: []const Target = &.{ .tpu, .neuron, @@ -142,7 +142,7 @@ pub const Platform = struct { .cpu, }; return for (ordered_targets) |target| { - break init(target, io, options) catch continue; + break init(allocator, io, target, options) catch continue; } else error.Unavailable; } From 1b644887b7752715baeee9de489b450b8c03d920 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 19 Jan 2026 18:10:35 +0100 Subject: [PATCH 248/881] bazel,platforms: centralize runfiles loading runfiles are process-wide in space and time. so it makes sense to keep them around because they are bound to the process, we use the smp_allocator and except the memory to be kept around during the lifetime of the process. --- bazel/BUILD.bazel | 6 +----- bazel/bazel.zig | 18 ++++++++++++++++ platforms/cpu/BUILD.bazel | 1 + platforms/cpu/cpu.zig | 12 +++-------- platforms/cuda/BUILD.bazel | 1 + platforms/cuda/cuda.zig | 9 ++------ platforms/neuron/BUILD.bazel | 1 + platforms/neuron/neuron.zig | 13 +++--------- platforms/rocm/BUILD.bazel | 1 + platforms/rocm/rocm.zig | 12 +++-------- platforms/tpu/BUILD.bazel | 1 + platforms/tpu/tpu.zig | 12 +++-------- stdx/stdx.zig | 41 ++++++++++++++++++++++++++++++++++++ 13 files changed, 79 insertions(+), 49 deletions(-) create mode 100644 bazel/bazel.zig diff --git a/bazel/BUILD.bazel b/bazel/BUILD.bazel index 27870cab..bb4a1781 100644 --- a/bazel/BUILD.bazel +++ b/bazel/BUILD.bazel @@ -1,10 +1,6 @@ -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") -load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") -load("@bazel_skylib//rules:build_test.bzl", "build_test") -load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") load("@rules_zig//zig:defs.bzl", "zig_library") -zig_binary( +zig_library( name = "bazel", main = "bazel.zig", visibility = ["//visibility:public"], diff --git a/bazel/bazel.zig b/bazel/bazel.zig new file mode 100644 index 00000000..02a49e0d --- /dev/null +++ b/bazel/bazel.zig @@ -0,0 +1,18 @@ +const std = @import("std"); + +const bazel_runfiles = @import("runfiles"); +const stdx = @import("stdx"); + +var runfiles_once = stdx.onceWithArgs(struct { + fn call(allocator: std.mem.Allocator, io: std.Io) !*bazel_runfiles.Runfiles { + const r = try allocator.create(bazel_runfiles.Runfiles); + r.* = try bazel_runfiles.Runfiles.create(.{ .allocator = allocator, .io = io }) orelse { + stdx.debug.panic("Unable to find runfiles", .{}); + }; + return r; + } +}.call); + +pub fn runfiles(io: std.Io, source_repository: []const u8) !bazel_runfiles.Runfiles.WithSourceRepo { + return (try runfiles_once.call(.{ std.heap.smp_allocator, io })).withSourceRepo(source_repository); +} diff --git a/platforms/cpu/BUILD.bazel b/platforms/cpu/BUILD.bazel index 9f9295c0..16e059d5 100644 --- a/platforms/cpu/BUILD.bazel +++ b/platforms/cpu/BUILD.bazel @@ -50,6 +50,7 @@ zig_library( ] + select({ "//platforms:cpu.enabled": [ ":libpjrt_cpu", + "//bazel", "//stdx", "@rules_zig//zig/runfiles", ], diff --git a/platforms/cpu/cpu.zig b/platforms/cpu/cpu.zig index 923ece24..866087ce 100644 --- a/platforms/cpu/cpu.zig +++ b/platforms/cpu/cpu.zig @@ -1,6 +1,7 @@ const std = @import("std"); const builtin = @import("builtin"); +const bazel = @import("bazel"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); @@ -14,19 +15,12 @@ pub fn isEnabled() bool { } pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { + _ = allocator; // autofix if (comptime !isEnabled()) { return error.Unavailable; } - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - - var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator(), .io = io }) orelse { - stdx.debug.panic("Unable to find runfiles", .{}); - }; - - const source_repo = bazel_builtin.current_repository; - const r = r_.withSourceRepo(source_repo); + const r = try bazel.runfiles(io, bazel_builtin.current_repository); var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const sandbox_path = try r.rlocation("zml/platforms/cpu/sandbox/lib", &path_buf) orelse { diff --git a/platforms/cuda/BUILD.bazel b/platforms/cuda/BUILD.bazel index 15bb10e6..3bb7cfc0 100644 --- a/platforms/cuda/BUILD.bazel +++ b/platforms/cuda/BUILD.bazel @@ -40,6 +40,7 @@ zig_library( ] + select({ "//platforms:cuda.enabled": [ ":libpjrt_cuda", + "//bazel", "//stdx", "@rules_zig//zig/runfiles", ], diff --git a/platforms/cuda/cuda.zig b/platforms/cuda/cuda.zig index 9bcf07bb..4a235e05 100644 --- a/platforms/cuda/cuda.zig +++ b/platforms/cuda/cuda.zig @@ -1,10 +1,10 @@ const std = @import("std"); const builtin = @import("builtin"); +const bazel = @import("bazel"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); -const runfiles = @import("runfiles"); const stdx = @import("stdx"); const nvidiaLibsPath = "/cuda/"; @@ -49,12 +49,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); - var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator(), .io = io }) orelse { - stdx.debug.panic("Unable to find runfiles", .{}); - }; - - const source_repo = bazel_builtin.current_repository; - const r = r_.withSourceRepo(source_repo); + const r = try bazel.runfiles(io, bazel_builtin.current_repository); var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const sandbox_path = try r.rlocation("libpjrt_cuda/sandbox", &path_buf) orelse { diff --git a/platforms/neuron/BUILD.bazel b/platforms/neuron/BUILD.bazel index 104e70d2..e095d025 100644 --- a/platforms/neuron/BUILD.bazel +++ b/platforms/neuron/BUILD.bazel @@ -117,6 +117,7 @@ zig_library( ":libnrt_headers", ":libpython", ":zmlxneuron", + "//bazel", "//stdx", "@libpjrt_neuron", "@rules_zig//zig/runfiles", diff --git a/platforms/neuron/neuron.zig b/platforms/neuron/neuron.zig index 89eb8c4b..3caa7258 100644 --- a/platforms/neuron/neuron.zig +++ b/platforms/neuron/neuron.zig @@ -1,10 +1,10 @@ const std = @import("std"); const builtin = @import("builtin"); +const bazel = @import("bazel"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); -const runfiles = @import("runfiles"); const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/platforms/neuron"); @@ -34,6 +34,7 @@ fn isRunningOnEC2(io: std.Io) !bool { } pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { + _ = allocator; // autofix if (comptime !isEnabled()) { return error.Unavailable; } @@ -47,15 +48,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { return error.Unavailable; } - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - - var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator(), .io = io }) orelse { - stdx.debug.panic("Unable to find runfiles", .{}); - }; - - const source_repo = bazel_builtin.current_repository; - const r = r_.withSourceRepo(source_repo); + const r = try bazel.runfiles(io, bazel_builtin.current_repository); var sandbox_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const sandbox_path = try r.rlocation("libpjrt_neuron/sandbox", &sandbox_path_buf) orelse { diff --git a/platforms/rocm/BUILD.bazel b/platforms/rocm/BUILD.bazel index 2b625ad6..ff764fb0 100644 --- a/platforms/rocm/BUILD.bazel +++ b/platforms/rocm/BUILD.bazel @@ -43,6 +43,7 @@ zig_library( ] + select({ "//platforms:rocm.enabled": [ ":libpjrt_rocm", + "//bazel", "//stdx", "@rules_zig//zig/runfiles", ], diff --git a/platforms/rocm/rocm.zig b/platforms/rocm/rocm.zig index 616be6da..6073f697 100644 --- a/platforms/rocm/rocm.zig +++ b/platforms/rocm/rocm.zig @@ -1,6 +1,7 @@ const std = @import("std"); const builtin = @import("builtin"); +const bazel = @import("bazel"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); @@ -26,6 +27,7 @@ fn setupRocmEnv(rocm_data_dir: []const u8) !void { } pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { + _ = allocator; // autofix if (comptime !isEnabled()) { return error.Unavailable; } @@ -36,15 +38,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { return error.Unavailable; } - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - - var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator(), .io = io }) orelse { - stdx.debug.panic("Unable to find runfiles", .{}); - }; - - const source_repo = bazel_builtin.current_repository; - const r = r_.withSourceRepo(source_repo); + const r = try bazel.runfiles(io, bazel_builtin.current_repository); var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const sandbox_path = try r.rlocation("libpjrt_rocm/sandbox", &path_buf) orelse { diff --git a/platforms/tpu/BUILD.bazel b/platforms/tpu/BUILD.bazel index 876255c7..af25cd2f 100644 --- a/platforms/tpu/BUILD.bazel +++ b/platforms/tpu/BUILD.bazel @@ -22,6 +22,7 @@ zig_library( ] + select({ "//platforms:tpu.enabled": [ ":libpjrt_tpu", + "//bazel", "//stdx", "@rules_zig//zig/runfiles", ], diff --git a/platforms/tpu/tpu.zig b/platforms/tpu/tpu.zig index af2395ff..6ce71097 100644 --- a/platforms/tpu/tpu.zig +++ b/platforms/tpu/tpu.zig @@ -1,6 +1,7 @@ const std = @import("std"); const builtin = @import("builtin"); +const bazel = @import("bazel"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); @@ -33,6 +34,7 @@ fn isOnGCP(io: std.Io) !bool { } pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { + _ = allocator; // autofix if (comptime !isEnabled()) { return error.Unavailable; } @@ -43,15 +45,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { return error.Unavailable; } - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - - var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator() }) orelse { - stdx.debug.panic("Unable to find runfiles", .{}); - }; - - const source_repo = bazel_builtin.current_repository; - const r = r_.withSourceRepo(source_repo); + const r = try bazel.runfiles(io, bazel_builtin.current_repository); var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const sandbox_path = try r.rlocation("libpjrt_tpu/sandbox", &path_buf) orelse { diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 8cb35de2..9401bdd4 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -37,3 +37,44 @@ pub fn pinToCore(core_id: usize) void { std.os.linux.sched_setaffinity(0, @ptrCast(&set.masks)) catch {}; } } + +pub fn onceWithArgs(comptime f: anytype) OnceWithArgs(f) { + return .{}; +} + +/// An object that executes the function `f` just once. +/// It is undefined behavior if `f` re-enters the same Once instance. +pub fn OnceWithArgs(comptime f: anytype) type { + const Args = std.meta.ArgsTuple(@TypeOf(f)); + const Result = meta.FnResult(f); + return struct { + done: bool = false, + mutex: std.Thread.Mutex = .{}, + result: Result = undefined, + + /// Call the function `f`. + /// If `call` is invoked multiple times `f` will be executed only the + /// first time. + /// The invocations are thread-safe. + pub fn call(self: *@This(), args: Args) Result { + if (@atomicLoad(bool, &self.done, .acquire)) + return self.result; + + return self.callSlow(args); + } + + fn callSlow(self: *@This(), args: Args) Result { + @branchHint(.cold); + + self.mutex.lock(); + defer self.mutex.unlock(); + + // The first thread to acquire the mutex gets to run the initializer + if (!self.done) { + self.result = @call(.auto, f, args); + @atomicStore(bool, &self.done, true, .release); + } + return self.result; + } + }; +} From 0aff9cf4dc7862011f6e9e1958f99130c0dd6baa Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 19 Jan 2026 18:42:05 +0100 Subject: [PATCH 249/881] examples/llama: plug std.Progress to the compilation pipeline --- examples/llama/main.zig | 53 ++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 4cf73bab..000322dd 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -70,6 +70,9 @@ pub fn main() !void { log.info("Running with threaded io async limit set to {?d}", .{threaded.async_limit.toInt()}); + var vfs: zml.io.VFS = try .init(allocator, threaded.io()); + defer vfs.deinit(); + var http_client: std.http.Client = .{ .allocator = allocator, .io = threaded.io(), @@ -81,18 +84,14 @@ pub fn main() !void { var vfs_file: zml.io.VFS.File = .init(allocator, threaded.io(), .{}); defer vfs_file.deinit(); + try vfs.register("file", vfs_file.io()); var vfs_https: zml.io.VFS.HTTP = try .init(allocator, threaded.io(), &http_client, .https); defer vfs_https.deinit(); + try vfs.register("https", vfs_https.io()); var hf_vfs: zml.io.VFS.HF = try .auto(allocator, threaded.io(), &http_client); defer hf_vfs.deinit(); - - var vfs: zml.io.VFS = try .init(allocator, threaded.io()); - defer vfs.deinit(); - - try vfs.register("file", vfs_file.io()); - try vfs.register("https", vfs_https.io()); try vfs.register("hf", hf_vfs.io()); const io = vfs.io(); @@ -154,34 +153,29 @@ pub fn main() !void { .attention_parameters = .init(.fromBackend(backend)), }; - var tokenizer_future = io.async(loadTokenizer, .{ allocator, io, repo }); + var progress = std.Progress.start(io, .{ .root_name = args.model, .estimated_total_items = store.view().count() }); + + var tokenizer_future = io.async(loadTokenizer, .{ allocator, io, repo, &progress }); errdefer blk: { var v = tokenizer_future.cancel(io) catch break :blk; v.deinit(); } - var compiled_model_result_future = io.async(compileModel, .{ allocator, io, platform, llama_model, llama_parameters }); + var compiled_model_result_future = io.async(compileModel, .{ allocator, io, platform, llama_model, llama_parameters, &progress }); errdefer if (compiled_model_result_future.cancel(io)) |v| { defer v.prefill_exe.deinit(); defer v.decode_exe.deinit(); } else |_| {}; - var load_model_buffers_group: zml.stdx.Io.AllocatingLimitedConcurrentGroup = try .init(allocator, threaded.async_limit.toInt() orelse 16); + var load_model_buffers_group: zml.stdx.Io.AllocatingLimitedConcurrentGroup = try .init(allocator, threaded.async_limit.toInt() orelse 32); defer { load_model_buffers_group.cancel(io); load_model_buffers_group.deinit(); } - var model_name_it = std.mem.splitScalar(u8, args.model, '/'); - var model_name = model_name_it.next() orelse args.model; - while (model_name_it.next()) |part| { - if (!std.mem.eql(u8, part, "")) model_name = part; - } - const read_pool_config: zml.io.ConcurrentBufferPool.Config = .{ .size = args.reader_buffer_size.bytes(), .concurrency = load_model_buffers_group.limit, .dma = true }; const write_pool_config: zml.io.ConcurrentBufferPool.Config = .{ .size = args.writer_buffer_size.bytes(), .concurrency = load_model_buffers_group.limit, .dma = true }; - var progress = std.Progress.start(io, .{ .root_name = model_name, .estimated_total_items = store.view().count() }); var llama_buffers_future = io.async(loadModelBuffers, .{ allocator, io, &progress, &load_model_buffers_group, read_pool_config, write_pool_config, platform, &store, llama_model }); errdefer b: { var v = llama_buffers_future.cancel(io) catch break :b; @@ -254,8 +248,10 @@ fn parseConfig(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !std.j return parsed_config; } -fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !zml.tokenizer.Tokenizer { - log.info("Loading tokenizer", .{}); +fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, progress: *std.Progress.Node) !zml.tokenizer.Tokenizer { + // log.info("Loading tokenizer", .{}); + var node = progress.start("Loading tokenizer...", 1); + defer node.end(); var timer = try std.time.Timer.start(); defer log.info("Loaded tokenizer [{D}]", .{timer.read()}); const bytes = b: { @@ -284,16 +280,19 @@ const CompileModelResult = struct { decode_exe: zml.Exe, }; -fn compileModel(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform, llama_model: llama.LlamaLM, parameters: LlamaParameters) !CompileModelResult { +fn compileModel(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform, llama_model: llama.LlamaLM, parameters: LlamaParameters, progress: *std.Progress.Node) !CompileModelResult { + var node = progress.start("Compiling...", 2); + defer node.end(); + var timer = try std.time.Timer.start(); - log.info("Compiling model", .{}); defer log.info("Compiled model [{D}]", .{timer.read()}); // Compile the model twice, one for prefill, one for generation. var prefill_future = io.async(struct { - fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: zml.Platform, llama_model_: llama.LlamaLM, parameters_: LlamaParameters) !zml.Exe { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: zml.Platform, llama_model_: llama.LlamaLM, parameters_: LlamaParameters, progress_: *std.Progress.Node) !zml.Exe { + var node_ = progress_.start("Compiling prefill...", 1); + defer node_.end(); var timer_ = try std.time.Timer.start(); - log.info("Compiling prefill", .{}); defer log.info("Compiled prefill [{D}]", .{timer_.read()}); return platform_.compile(allocator_, io_, llama_model_, .forward, .{ parameters_.prefill_tokens, @@ -304,13 +303,14 @@ fn compileModel(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform parameters_.attention_parameters, }); } - }.call, .{ allocator, io, platform, llama_model, parameters }); + }.call, .{ allocator, io, platform, llama_model, parameters, &node }); errdefer if (prefill_future.cancel(io)) |v| v.deinit() else |_| {}; var decode_future = io.async(struct { - fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: zml.Platform, llama_model_: llama.LlamaLM, parameters_: LlamaParameters) !zml.Exe { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: zml.Platform, llama_model_: llama.LlamaLM, parameters_: LlamaParameters, progress_: *std.Progress.Node) !zml.Exe { + var node_ = progress_.start("Compiling decode...", 1); + defer node_.end(); var timer_ = try std.time.Timer.start(); - log.info("Compiling decode", .{}); defer log.info("Compiled decode [{D}]", .{timer_.read()}); return platform_.compile(allocator_, io_, llama_model_, .forward, .{ parameters_.decode_tokens, @@ -321,7 +321,7 @@ fn compileModel(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform parameters_.attention_parameters, }); } - }.call, .{ allocator, io, platform, llama_model, parameters }); + }.call, .{ allocator, io, platform, llama_model, parameters, &node }); errdefer if (decode_future.cancel(io)) |v| v.deinit() else |_| {}; const prefill_exe = try prefill_future.await(io); @@ -343,7 +343,6 @@ fn loadModelBuffers( ) !zml.Bufferized(llama.LlamaLM) { var transferred_bytes: usize = 0; var timer = try stdx.time.Timer.start(); - log.info("Loading model", .{}); defer { const duration = timer.read(); const seconds = @as(f64, @floatFromInt(duration.ns)) / 1e9; From d26386d206ab9fbab612677428aca324db5ebc62 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 19 Jan 2026 22:16:47 +0100 Subject: [PATCH 250/881] platforms: fix cuda, rocm and neuron sandboxes --- platforms/cuda/libpjrt_cuda.BUILD.bazel | 2 +- platforms/neuron/libpjrt_neuron.BUILD.bazel | 6 +++--- platforms/rocm/libpjrt_rocm.BUILD.bazel | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/platforms/cuda/libpjrt_cuda.BUILD.bazel b/platforms/cuda/libpjrt_cuda.BUILD.bazel index 8354809f..f30962fc 100644 --- a/platforms/cuda/libpjrt_cuda.BUILD.bazel +++ b/platforms/cuda/libpjrt_cuda.BUILD.bazel @@ -39,7 +39,7 @@ copy_to_directory( "nvvm/lib64": "lib", "libpjrt_cuda_so": "lib", "lib/x86_64-linux-gnu": "lib", - "runtimes/cuda": "lib", + "platforms/cuda": "lib", }, add_directory_to_runfiles = False, include_external_repositories = ["**"], diff --git a/platforms/neuron/libpjrt_neuron.BUILD.bazel b/platforms/neuron/libpjrt_neuron.BUILD.bazel index 174e2a39..df831ad4 100644 --- a/platforms/neuron/libpjrt_neuron.BUILD.bazel +++ b/platforms/neuron/libpjrt_neuron.BUILD.bazel @@ -44,8 +44,8 @@ copy_to_directory( "@libgomp1", ], replace_prefixes = { - "runtimes/neuron/neuronx-cc": "bin/neuronx-cc", - "runtimes/neuron/libneuronxla.so": "site-packages/libneuronxla.so", + "platforms/neuron/neuronx-cc": "bin/neuronx-cc", + "platforms/neuron/libneuronxla.so": "site-packages/libneuronxla.so", "libneuronpjrt_so": "lib", "libpjrt_neuron_so": "lib", "libnrt.patchelf": "lib", @@ -61,7 +61,7 @@ copy_to_directory( "**/_solib_*/**", ], include_srcs_patterns = [ - "runtimes/neuron/neuronx-cc", + "platforms/neuron/neuronx-cc", "**/lib**", "lib/python*/**", "site-packages/**", diff --git a/platforms/rocm/libpjrt_rocm.BUILD.bazel b/platforms/rocm/libpjrt_rocm.BUILD.bazel index fbd1dd54..3147d09e 100644 --- a/platforms/rocm/libpjrt_rocm.BUILD.bazel +++ b/platforms/rocm/libpjrt_rocm.BUILD.bazel @@ -84,7 +84,7 @@ copy_to_directory( "opt/amdgpu/share": "share", "rocblas": "lib", "rocroller": "lib", - "runtimes/rocm": "lib", + "platforms/rocm": "lib", "usr/lib/x86_64-linux-gnu": "lib", }, add_directory_to_runfiles = True, From ed709599f652d304817205f6597b841888bf2fb3 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 20 Jan 2026 18:55:21 +0000 Subject: [PATCH 251/881] cuda/flashattn: package bindings in cuda platform --- platforms/cuda/flashattn/BUILD.bazel | 13 + platforms/cuda/flashattn/flashattn.zig | 111 ++++++ zml/BUILD.bazel | 5 +- zml/attention/attention.zig | 2 +- zml/attention/flashattn.zig | 463 +++++++++++++++++++++++++ zml/attention/flashattn/bindings.zig | 417 ---------------------- zml/attention/flashattn/flashattn.zig | 195 ----------- 7 files changed, 590 insertions(+), 616 deletions(-) create mode 100644 platforms/cuda/flashattn/BUILD.bazel create mode 100644 platforms/cuda/flashattn/flashattn.zig create mode 100644 zml/attention/flashattn.zig delete mode 100644 zml/attention/flashattn/bindings.zig delete mode 100644 zml/attention/flashattn/flashattn.zig diff --git a/platforms/cuda/flashattn/BUILD.bazel b/platforms/cuda/flashattn/BUILD.bazel new file mode 100644 index 00000000..9ef2bbf8 --- /dev/null +++ b/platforms/cuda/flashattn/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "flashattn", + import_name = "platforms/cuda/flashattn", + main = "flashattn.zig", + visibility = ["//visibility:public"], + deps = [ + "//bazel", + "@flashattn//:c", + "@rules_zig//zig/runfiles", + ], +) diff --git a/platforms/cuda/flashattn/flashattn.zig b/platforms/cuda/flashattn/flashattn.zig new file mode 100644 index 00000000..c2d8ab75 --- /dev/null +++ b/platforms/cuda/flashattn/flashattn.zig @@ -0,0 +1,111 @@ +const std = @import("std"); + +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); +const c = @import("c"); +pub const FA2MhaFwdParams = c.FA2MhaFwdParams; +pub const FA2MhaVarlenFwdParams = c.FA2MhaVarlenFwdParams; +pub const FA3MhaFwdParams = c.FA3MhaFwdParams; +const runfiles = @import("runfiles"); + +pub const Tensor = struct { + inner: c.FlashattnTensor, + + pub fn init(ptr: *anyopaque, dims: []const i64, strides: []const i64, dtype: DataType) Tensor { + const rank = dims.len; + var ret: Tensor = .{ + .inner = .{ + .ptr = ptr, + .rank = @intCast(rank), + .dims = undefined, + .strides = undefined, + .dtype = @intCast(@intFromEnum(dtype)), + }, + }; + @memcpy(ret.inner.dims[0..rank], dims); + @memcpy(ret.inner.strides[0..rank], strides); + return ret; + } +}; + +pub const DataType = enum(c_int) { + f32 = c.CAPI_FLOAT, + f16 = c.CAPI_FLOAT16, + bf16 = c.CAPI_BFLOAT16, + i32 = c.CAPI_INT32, + i8 = c.CAPI_INT8, +}; + +const Fa2MhaFwdFunc = *const fn ( + q: *const Tensor, + k: *const Tensor, + v: *const Tensor, + out: *const Tensor, + softmax_lse: ?*const Tensor, + alibi_slopes_: ?*const Tensor, + softmax_lse_accum: ?*const Tensor, + out_accum: ?*const Tensor, + params: ?*const FA2MhaFwdParams, + stream: ?*anyopaque, +) callconv(.c) void; + +const Fa2MhaVarlenFwdFunc = *const fn ( + q: *const Tensor, + k: *const Tensor, + v: *const Tensor, + out: *const Tensor, + cu_seqlens_q: ?*const Tensor, + cu_seqlens_k: ?*const Tensor, + seqused_k: ?*const Tensor, + block_table: ?*const Tensor, + softmax_lse: ?*const Tensor, + alibi_slopes_: ?*const Tensor, + softmax_lse_accum: ?*const Tensor, + out_accum: ?*const Tensor, + params: ?*const FA2MhaVarlenFwdParams, + stream: ?*anyopaque, +) callconv(.c) void; + +const Fa3MhaFwdFunc = *const fn ( + q: ?*const Tensor, + k: ?*const Tensor, + v: ?*const Tensor, + out: ?*const Tensor, + cu_seqlens_q: ?*const Tensor, + cu_seqlens_k: ?*const Tensor, + seqused_q: ?*const Tensor, + seqused_k: ?*const Tensor, + page_table: ?*const Tensor, + q_descale: ?*const Tensor, + k_descale: ?*const Tensor, + v_descale: ?*const Tensor, + softmax_lse: ?*const Tensor, + softmax_lse_accum: ?*const Tensor, + out_accum: ?*const Tensor, + scheduler_metadata: ?*const Tensor, + s_aux: ?*const Tensor, + cp_tot_seqused_k: ?*const Tensor, + params: ?*const FA3MhaFwdParams, + stream: ?*anyopaque, +) callconv(.c) void; + +pub var fa2_mha_fwd: Fa2MhaFwdFunc = undefined; +pub var fa2_mha_varlen_fwd: Fa2MhaVarlenFwdFunc = undefined; +pub var fa3_mha_fwd: Fa3MhaFwdFunc = undefined; + +pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { + _ = allocator; // autofix + + const r = try bazel.runfiles(io, bazel_builtin.current_repository); + + var buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; + const library = (try r.rlocation("flashattn/flashattn/lib/libflashattn.so", &buffer)) orelse return error.NotFound; + var lib = std.DynLib.open(library) catch |err| { + std.log.err("Failed to open libflashattn.so: {any}", .{err}); + return err; + }; + + fa2_mha_varlen_fwd = lib.lookup(Fa2MhaVarlenFwdFunc, "fa2_mha_varlen_fwd") orelse return error.NotFound; + fa2_mha_fwd = lib.lookup(Fa2MhaFwdFunc, "fa2_mha_fwd") orelse return error.NotFound; + fa3_mha_fwd = lib.lookup(Fa3MhaFwdFunc, "fa3_mha_fwd") orelse return error.NotFound; +} diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 1206c00a..9c95f090 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -21,8 +21,7 @@ zig_library( name = "zml", srcs = [ "attention/attention.zig", - "attention/flashattn/bindings.zig", - "attention/flashattn/flashattn.zig", + "attention/flashattn.zig", "buffer.zig", "constants.zig", "dtype.zig", @@ -61,7 +60,7 @@ zig_library( "//upb", "//zml/io", "//zml/tokenizer", - "@flashattn//:c", + "//platforms/cuda/flashattn", "@rules_zig//zig/runfiles", ], ) diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index b9dd80a9..6bdce1fa 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -1,7 +1,7 @@ const std = @import("std"); const zml = @import("../zml.zig"); -pub const flashattn = @import("flashattn/flashattn.zig"); +pub const flashattn = @import("flashattn.zig"); const Attention = @This(); diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig new file mode 100644 index 00000000..a941d4d1 --- /dev/null +++ b/zml/attention/flashattn.zig @@ -0,0 +1,463 @@ +const std = @import("std"); + +const flashattn = @import("platforms/cuda/flashattn"); +pub const load = flashattn.load; +const stdx = @import("stdx"); + +const zml = @import("../zml.zig"); +const ffi = zml.pjrt.ffi; + +fn flashattnDataTypeFromZmlDataType(dtype: zml.DataType) flashattn.DataType { + return switch (dtype) { + .f32 => .f32, + .f16 => .f16, + .bf16 => .bf16, + .i32 => .i32, + .i8 => .i8, + else => unreachable, + }; +} + +fn tensorFromBuffer(buffer: zml.Buffer, platform: zml.Platform) !flashattn.Tensor { + return .init( + try buffer._shards.get(0).getOpaqueDeviceMemoryDataPointer(platform.pjrt_api), + buffer.shape().dims(), + buffer.shape().withDtype(.u8).computeStrides().constSlice(), + flashattnDataTypeFromZmlDataType(buffer.dtype()), + ); +} + +const Buffer = struct { + shape: zml.Shape, + ptr: *anyopaque, + + fn toFlashattnTensor(buffer: Buffer) flashattn.Tensor { + return .init( + buffer.ptr, + buffer.shape.dims(), + buffer.shape.withDtype(.u8).computeByteStrides().constSlice(), + flashattnDataTypeFromZmlDataType(buffer.shape.dtype()), + ); + } +}; + +fn getPlatform(call_frame: *ffi.CallFrame) zml.Platform { + const pjrt_api_ptr = call_frame.attrs.getByName(.scalar, "pjrt_api") orelse unreachable; + std.debug.assert(pjrt_api_ptr.dtype == .u64); + const pjrt_api: ?*zml.pjrt.Api = @ptrFromInt(pjrt_api_ptr.get(usize)); + + const pjrt_client_ptr = call_frame.attrs.getByName(.scalar, "pjrt_client") orelse unreachable; + std.debug.assert(pjrt_client_ptr.dtype == .u64); + const pjrt_client: ?*zml.pjrt.Client = @ptrFromInt(pjrt_client_ptr.get(usize)); + + return .{ .target = .cuda, .pjrt_api = pjrt_api.?, .pjrt_client = pjrt_client.? }; +} + +fn dataTypeFromFfiDataType(ffi_dt: ffi.DataType) zml.DataType { + return switch (ffi_dt) { + .bool => .bool, + .i8 => .i8, + .i16 => .i16, + .i32 => .i32, + .i64 => .i64, + .u8 => .u8, + .u16 => .u16, + .u32 => .u32, + .u64 => .u64, + .f16 => .f16, + .f32 => .f32, + .f64 => .f64, + .bf16 => .bf16, + .c64 => .c64, + .c128 => .c128, + .f8e5m2 => .f8e5m2, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3b11fnuz => .f8e4m3b11fnuz, + .f8e5m2fnuz => .f8e5m2fnuz, + .f8e4m3fnuz => .f8e4m3fnuz, + else => unreachable, + }; +} + +fn shapeFromFfiBuffer(buffer: *const ffi.Buffer) zml.Shape { + return .init(buffer.dims(), dataTypeFromFfiDataType(buffer.dtype)); +} + +fn bufferFromFfiBuffer(ffi_buffer: *const ffi.Buffer) Buffer { + return .{ + .shape = shapeFromFfiBuffer(ffi_buffer), + .ptr = ffi_buffer.data, + }; +} + +fn getScalarAttributeAs(comptime T: type, call_frame: *ffi.CallFrame, attribute_name: []const u8) ?T { + const attribute = call_frame.attrs.getByName(.scalar, attribute_name) orelse return null; + return attribute.get(T); +} + +fn fixupKvCacheBuffer(buffer: Buffer, layer_index: i64) Buffer { + var shape = buffer.shape; + const layer_stride = shape.computeByteStrides().get(0); + shape = shape.remove(0); + const ptr = @as([*]u8, @ptrCast(buffer.ptr)); + return .{ + .shape = shape, + .ptr = ptr + @as(usize, @intCast(layer_stride * layer_index)), + }; +} + +pub fn Wrapper(comptime T: type, run_func: std.meta.DeclEnum(T)) type { + return struct { + pub fn register(platform: zml.Platform) !void { + try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "cuda", T.run, .{ .command_buffer_compatible = true }); + } + + pub fn run(call_frame: *ffi.CallFrame) callconv(.c) ?*ffi.Error { + return @field(T, @tagName(run_func))(call_frame) catch b: { + break :b ffi.Error.create(call_frame.api.?, .unknown, "Unknown"); + }; + } + }; +} + +pub const fa2 = struct { + const custom_call_name = "fa2_mha_varlen_fwd"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const k = bufferFromFfiBuffer(call_frame.args.buffers()[1]); + const v = bufferFromFfiBuffer(call_frame.args.buffers()[2]); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_q: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_q").?; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA2MhaVarlenFwdParams = .{ + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .num_splits = 0, + .num_heads = num_heads, + }; + + flashattn.fa2_mha_varlen_fwd( + &q.toFlashattnTensor(), + &k.toFlashattnTensor(), + &v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + &cu_seqlens_k.toFlashattnTensor(), + null, + null, + &softmax_lse.toFlashattnTensor(), + null, + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + ¶ms, + stream, + ); + + return null; + } + + pub const Parameters = struct { + pub const InitOptions = struct {}; + + pub fn init(opts: InitOptions) fa2.Parameters { + _ = opts; // autofix + return .{}; + } + }; + + pub const Metadata = struct { + softmax_lse: zml.Tensor, + softmax_lse_accum: zml.Tensor, + out_accum: zml.Tensor, + + pub const InitOptions = struct { + seqlen: i64, + }; + + pub fn init(opts: InitOptions) Metadata { + return .{ + .softmax_lse = .init(.{1 * 32 * opts.seqlen * 4}, .i8), + .softmax_lse_accum = .init(.{32 * 1 * 32 * 4 * 4}, .i8), + .out_accum = .init(.{32 * 1 * 32 * 4 * 128 * 4}, .i8), + }; + } + + pub fn initBuffer(self: Metadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(Metadata) { + return .{ + .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), .{}), + .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), .{}), + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { + self.softmax_lse.deinit(); + self.softmax_lse_accum.deinit(); + self.out_accum.deinit(); + } + }; + + pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { + _ = parameters; // autofix + stdx.debug.assert(q_.shape().hasTag(.b) == null or q_.dim(.b) == 1, "fa2.attention support for batch size != 1 is not supported yet.", .{}); + const seqused_k = token_index.addConstant(q_.dim(.q)).reshape(.{1}); + // TODO(Corendos): replace with cumsum + const cu_seqlens_k = b: { + const zero = zml.Tensor.constant(token_index.dtype().zero()).reshape(.{1}); + break :b zml.Tensor.concatenate(&.{ zero, seqused_k }, 0).convert(.i32); + }; + const max_seqlen_q: i32 = @intCast(q_.dim(.q)); + const max_seqlen_k: i32 = @intCast(k_.dim(.k)); + var q = q_.insertAxes(.q, .{.b}).merge(.{ .tot = .{ .b, .q } }); + const k = k_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); + const v = v_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); + // TODO(Corendos): replace with cumsum + const cu_seqlens_q = zml.Tensor.constantTensor(zml.Shape.init(.{2}, .i32), std.mem.sliceAsBytes(&[2]i32{ 0, max_seqlen_q })); + + const original_tot = q.dim(.tot); + const num_heads = q_.dim(.h); + const num_heads_k = k_.dim(.h); + const head_size = q_.dim(.hd); + const ngroups = @divExact(num_heads, num_heads_k); + const seqlenq_ngroups_swapped = max_seqlen_q == 1 and num_heads > num_heads_k and @mod(head_size, 8) == 0; + if (seqlenq_ngroups_swapped) { + q = q.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .tot, .ngroups, .h, .hd }).merge(.{ .tot = .{ .tot, .ngroups } }); + } + + var o = zml.ops.customCall( + custom_call_name, + .{ + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + metadata.softmax_lse, + metadata.softmax_lse_accum, + metadata.out_accum, + }, + .{q.shape()}, + .{ + //.softmax_scale = @as(f32, 1.0), + .is_causal = true, + .window_size_left = @as(i32, -1), + .window_size_right = @as(i32, -1), + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + .num_heads = @as(i32, @intCast(num_heads)), + }, + .{ + .output_operand_aliases = &.{0}, + .has_side_effect = true, + }, + ); + + if (seqlenq_ngroups_swapped) { + o = o.splitAxis(.tot, .{ .tot = original_tot, .ngroups = ngroups }).transpose(.{ .tot, .h, .ngroups, .hd }).merge(.{ .h = .{ .h, .ngroups } }); + } + + return o.splitAxis(.tot, .{ .b = 1, .q = q_.dim(.q) }).squeeze(.b); + } +}; + +pub const fa3 = struct { + const custom_call_name = "fa3_mha_fwd"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const k = bufferFromFfiBuffer(call_frame.args.buffers()[1]); + const v = bufferFromFfiBuffer(call_frame.args.buffers()[2]); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const scheduler_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_q: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_q").?; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA3MhaFwdParams = .{ + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + .softcap = 0.0, + .is_rotary_interleaved = false, + .num_splits = 0, + .sm_margin = 0, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .cp_world_size = 1, + .cp_rank = 0, + }; + + flashattn.fa3_mha_fwd( + &q.toFlashattnTensor(), + &k.toFlashattnTensor(), + &v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + &cu_seqlens_k.toFlashattnTensor(), + null, + null, + null, + null, + null, + null, + &softmax_lse.toFlashattnTensor(), + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + &scheduler_metadata.toFlashattnTensor(), + null, + null, + ¶ms, + stream, + ); + + return null; + } + + pub const Parameters = struct { + pub const InitOptions = struct {}; + + pub fn init(opts: InitOptions) fa3.Parameters { + _ = opts; // autofix + return .{}; + } + }; + + pub const Metadata = struct { + softmax_lse: zml.Tensor, + softmax_lse_accum: zml.Tensor, + out_accum: zml.Tensor, + scheduler_metadata: zml.Tensor, + + pub const InitOptions = struct { + seqlen: i64, + }; + + pub fn init(opts: InitOptions) Metadata { + return .{ + .softmax_lse = .init(.{1 * 32 * opts.seqlen * 4}, .i8), + .softmax_lse_accum = .init(.{32 * 1 * 32 * 4 * 4}, .i8), + .out_accum = .init(.{32 * 1 * 32 * 4 * 128 * 4}, .i8), + .scheduler_metadata = .init(.{2}, .i32), + }; + } + + pub fn initBuffer(self: Metadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(Metadata) { + return .{ + .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), .{}), + .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), .{}), + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + .scheduler_metadata = try zml.Buffer.uninitialized(io, platform, self.scheduler_metadata.shape(), .{}), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { + self.softmax_lse.deinit(); + self.softmax_lse_accum.deinit(); + self.out_accum.deinit(); + self.scheduler_metadata.deinit(); + } + }; + + pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { + _ = parameters; // autofix + stdx.debug.assert(q_.shape().hasTag(.b) == null or q_.dim(.b) == 1, "fa3.attention support for batch size != 1 is not supported yet.", .{}); + const seqused_k = token_index.addConstant(q_.dim(.q)).reshape(.{1}); + // TODO(Corendos): replace with cumsum + const cu_seqlens_k = b: { + const zero = zml.Tensor.constant(token_index.dtype().zero()).reshape(.{1}); + break :b zml.Tensor.concatenate(&.{ zero, seqused_k }, 0).convert(.i32); + }; + const max_seqlen_q: i32 = @intCast(q_.dim(.q)); + const max_seqlen_k: i32 = @intCast(k_.dim(.k)); + var q = q_.insertAxes(.q, .{.b}).merge(.{ .tot = .{ .b, .q } }); + const k = k_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); + const v = v_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); + // TODO(Corendos): replace with cumsum + const cu_seqlens_q = zml.Tensor.constantTensor(zml.Shape.init(.{2}, .i32), std.mem.sliceAsBytes(&[2]i32{ 0, max_seqlen_q })); + + var o = zml.ops.customCall( + custom_call_name, + .{ + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + metadata.softmax_lse, + metadata.softmax_lse_accum, + metadata.out_accum, + metadata.scheduler_metadata, + }, + .{q.shape()}, + .{ + .is_causal = true, + .window_size_left = @as(i32, -1), + .window_size_right = @as(i32, -1), + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + }, + .{ + .output_operand_aliases = &.{0}, + .has_side_effect = true, + }, + ); + + return o.splitAxis(.tot, .{ .b = 1, .q = q_.dim(.q) }).squeeze(.b); + } +}; + +pub fn register(platform: zml.Platform) !void { + try fa2.register(platform); + try fa3.register(platform); +} diff --git a/zml/attention/flashattn/bindings.zig b/zml/attention/flashattn/bindings.zig deleted file mode 100644 index a8c48409..00000000 --- a/zml/attention/flashattn/bindings.zig +++ /dev/null @@ -1,417 +0,0 @@ -const std = @import("std"); - -const bazel_builtin = @import("bazel_builtin"); -const c = @import("c"); -const runfiles = @import("runfiles"); - -const zml = @import("../../zml.zig"); -const ffi = zml.pjrt.ffi; - -fn flashattnDataTypeFromZmlDataType(dtype: zml.DataType) c.DataType { - return switch (dtype) { - .f32 => c.CAPI_FLOAT, - .f16 => c.CAPI_FLOAT16, - .bf16 => c.CAPI_BFLOAT16, - .i32 => c.CAPI_INT32, - .i8 => c.CAPI_INT8, - else => unreachable, - }; -} - -pub fn tensorFromBuffer(buffer: zml.Buffer, platform: zml.Platform) !c.FlashattnTensor { - const rank: usize = buffer.rank(); - var dims: [8]i64 = undefined; - @memcpy(dims[0..rank], buffer.shape().dims()); - var strides: [8]i64 = undefined; - @memcpy(strides[0..rank], buffer.shape().withDtype(.u8).computeStrides().constSlice()); - - const tensor: c.FlashattnTensor = .{ - .ptr = try buffer._shards.get(0).getOpaqueDeviceMemoryDataPointer(platform.pjrt_api), - .rank = buffer.rank(), - .dims = dims, - .strides = strides, - .dtype = flashattnDataTypeFromZmlDataType(buffer.dtype()), - }; - - return tensor; -} - -pub const Buffer = struct { - shape: zml.Shape, - ptr: ?*anyopaque, - - pub fn toFlashattnTensor(buffer: Buffer) c.FlashattnTensor { - const rank: usize = buffer.shape.rank(); - var dims: [8]i64 = undefined; - @memcpy(dims[0..rank], buffer.shape.dims()); - var strides: [8]i64 = undefined; - @memcpy(strides[0..rank], buffer.shape.withDtype(.u8).computeByteStrides().constSlice()); - - const tensor: c.FlashattnTensor = .{ - .ptr = buffer.ptr, - .rank = buffer.shape.rank(), - .dims = dims, - .strides = strides, - .dtype = flashattnDataTypeFromZmlDataType(buffer.shape.dtype()), - }; - - return tensor; - } -}; - -fn getPlatform(call_frame: *ffi.CallFrame) zml.Platform { - const pjrt_api_ptr = call_frame.attrs.getByName(.scalar, "pjrt_api") orelse unreachable; - std.debug.assert(pjrt_api_ptr.dtype == .u64); - const pjrt_api: ?*zml.pjrt.Api = @ptrFromInt(pjrt_api_ptr.get(usize)); - - const pjrt_client_ptr = call_frame.attrs.getByName(.scalar, "pjrt_client") orelse unreachable; - std.debug.assert(pjrt_client_ptr.dtype == .u64); - const pjrt_client: ?*zml.pjrt.Client = @ptrFromInt(pjrt_client_ptr.get(usize)); - - return .{ .target = .cuda, .pjrt_api = pjrt_api.?, .pjrt_client = pjrt_client.? }; -} - -fn dataTypeFromFfiDataType(ffi_dt: ffi.DataType) zml.DataType { - return switch (ffi_dt) { - .bool => .bool, - .i8 => .i8, - .i16 => .i16, - .i32 => .i32, - .i64 => .i64, - .u8 => .u8, - .u16 => .u16, - .u32 => .u32, - .u64 => .u64, - .f16 => .f16, - .f32 => .f32, - .f64 => .f64, - .bf16 => .bf16, - .c64 => .c64, - .c128 => .c128, - .f8e5m2 => .f8e5m2, - .f8e4m3fn => .f8e4m3fn, - .f8e4m3b11fnuz => .f8e4m3b11fnuz, - .f8e5m2fnuz => .f8e5m2fnuz, - .f8e4m3fnuz => .f8e4m3fnuz, - else => unreachable, - }; -} - -fn shapeFromFfiBuffer(buffer: *const ffi.Buffer) zml.Shape { - const dtype = dataTypeFromFfiDataType(buffer.dtype); - return zml.Shape.init(buffer.dims(), dtype); -} - -fn bufferFromFfiBuffer(ffi_buffer: *const ffi.Buffer) Buffer { - return .{ - .shape = shapeFromFfiBuffer(ffi_buffer), - .ptr = ffi_buffer.data, - }; -} - -fn getScalarAttributeAs(comptime T: type, call_frame: *ffi.CallFrame, attribute_name: []const u8) ?T { - const attribute = call_frame.attrs.getByName(.scalar, attribute_name) orelse return null; - return attribute.get(T); -} - -fn fixupKvCacheBuffer(buffer: Buffer, layer_index: i64) Buffer { - var shape = buffer.shape; - const layer_stride = shape.computeByteStrides().get(0); - shape = shape.remove(0); - const ptr = @as([*]u8, @ptrCast(buffer.ptr)); - return .{ - .shape = shape, - .ptr = ptr + @as(usize, @intCast(layer_stride * layer_index)), - }; -} - -pub fn Wrapper(comptime T: type, run_func: std.meta.DeclEnum(T)) type { - return struct { - pub fn register(platform: zml.Platform) !void { - try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "cuda", T.run, .{ .command_buffer_compatible = true }); - } - - pub fn run(call_frame: *ffi.CallFrame) callconv(.c) ?*ffi.Error { - return @field(T, @tagName(run_func))(call_frame) catch b: { - break :b ffi.Error.create(call_frame.api.?, .unknown, "Unknown"); - }; - } - }; -} - -pub const Fa2 = struct { - pub const custom_call_name = "fa2_mha_fwd"; - const Wrapped = Wrapper(@This(), .runInner); - - pub const register = Wrapped.register; - pub const run = Wrapped.run; - - pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { - if (call_frame.registeringHook()) return null; - - const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); - const k = bufferFromFfiBuffer(call_frame.args.buffers()[1]); - const v = bufferFromFfiBuffer(call_frame.args.buffers()[2]); - const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[3]); - const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[4]); - const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[5]); - const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); - - const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { - const head_dim = q.shape.dim(3); - break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); - }; - const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; - const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; - const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; - - const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); - const stream = call_frame.api.stream(ctx); - - const params: c.FA2MhaFwdParams = .{ - .is_causal = is_causal, - .softmax_scale = softmax_scale, - .window_size_left = window_size_left, - .window_size_right = window_size_right, - }; - - fa2_mha_fwd.?( - &q.toFlashattnTensor(), - &k.toFlashattnTensor(), - &v.toFlashattnTensor(), - &o.toFlashattnTensor(), - &softmax_lse.toFlashattnTensor(), - null, - &softmax_lse_accum.toFlashattnTensor(), - &out_accum.toFlashattnTensor(), - ¶ms, - stream, - ); - - return null; - } -}; - -pub const Fa2Varlen = struct { - pub const custom_call_name = "fa2_mha_varlen_fwd"; - const Wrapped = Wrapper(@This(), .runInner); - - pub const register = Wrapped.register; - pub const run = Wrapped.run; - - pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { - if (call_frame.registeringHook()) return null; - - const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); - const k = bufferFromFfiBuffer(call_frame.args.buffers()[1]); - const v = bufferFromFfiBuffer(call_frame.args.buffers()[2]); - const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); - const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); - const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[5]); - const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[6]); - const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); - const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); - - const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { - const head_dim = q.shape.dim(2); - break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); - }; - const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; - const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; - const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; - const max_seqlen_q: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_q").?; - const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; - const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; - - const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); - const stream = call_frame.api.stream(ctx); - - const params: c.FA2MhaVarlenFwdParams = .{ - .max_seqlen_q = max_seqlen_q, - .max_seqlen_k = max_seqlen_k, - .is_causal = is_causal, - .softmax_scale = softmax_scale, - .window_size_left = window_size_left, - .window_size_right = window_size_right, - .num_splits = 0, - .num_heads = num_heads, - }; - - fa2_mha_varlen_fwd.?( - &q.toFlashattnTensor(), - &k.toFlashattnTensor(), - &v.toFlashattnTensor(), - &o.toFlashattnTensor(), - &cu_seqlens_q.toFlashattnTensor(), - &cu_seqlens_k.toFlashattnTensor(), - null, - null, - &softmax_lse.toFlashattnTensor(), - null, - &softmax_lse_accum.toFlashattnTensor(), - &out_accum.toFlashattnTensor(), - ¶ms, - stream, - ); - - return null; - } -}; - -pub const Fa3 = struct { - pub const custom_call_name = "fa3_mha_fwd"; - const Wrapped = Wrapper(@This(), .runInner); - - pub const register = Wrapped.register; - pub const run = Wrapped.run; - - pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { - if (call_frame.registeringHook()) return null; - - const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); - const k = bufferFromFfiBuffer(call_frame.args.buffers()[1]); - const v = bufferFromFfiBuffer(call_frame.args.buffers()[2]); - const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); - const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); - const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[5]); - const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[6]); - const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); - const scheduler_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[8]); - const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); - - const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { - const head_dim = q.shape.dim(2); - break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); - }; - const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; - const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; - const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; - const max_seqlen_q: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_q").?; - const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; - - const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); - const stream = call_frame.api.stream(ctx); - - const params: c.FA3MhaFwdParams = .{ - .max_seqlen_q = max_seqlen_q, - .max_seqlen_k = max_seqlen_k, - .softcap = 0.0, - .is_rotary_interleaved = false, - .num_splits = 0, - .sm_margin = 0, - .is_causal = is_causal, - .softmax_scale = softmax_scale, - .window_size_left = window_size_left, - .window_size_right = window_size_right, - .cp_world_size = 1, - .cp_rank = 0, - }; - - fa3_mha_fwd.?( - &q.toFlashattnTensor(), - &k.toFlashattnTensor(), - &v.toFlashattnTensor(), - &o.toFlashattnTensor(), - &cu_seqlens_q.toFlashattnTensor(), - &cu_seqlens_k.toFlashattnTensor(), - null, - null, - null, - null, - null, - null, - &softmax_lse.toFlashattnTensor(), - &softmax_lse_accum.toFlashattnTensor(), - &out_accum.toFlashattnTensor(), - &scheduler_metadata.toFlashattnTensor(), - null, - null, - ¶ms, - stream, - ); - - return null; - } -}; - -const Fa2MhaFwdFunc = *const fn ( - q: ?*const c.FlashattnTensor, - k: ?*const c.FlashattnTensor, - v: ?*const c.FlashattnTensor, - out: ?*const c.FlashattnTensor, - softmax_lse: ?*const c.FlashattnTensor, - alibi_slopes_: ?*const c.FlashattnTensor, - softmax_lse_accum: ?*const c.FlashattnTensor, - out_accum: ?*const c.FlashattnTensor, - params: ?*const c.FA2MhaFwdParams, - stream: ?*anyopaque, -) callconv(.c) void; - -const Fa2MhaVarlenFwdFunc = *const fn ( - q: ?*const c.FlashattnTensor, - k: ?*const c.FlashattnTensor, - v: ?*const c.FlashattnTensor, - out: ?*const c.FlashattnTensor, - cu_seqlens_q: ?*const c.FlashattnTensor, - cu_seqlens_k: ?*const c.FlashattnTensor, - seqused_k: ?*const c.FlashattnTensor, - block_table: ?*const c.FlashattnTensor, - softmax_lse: ?*const c.FlashattnTensor, - alibi_slopes_: ?*const c.FlashattnTensor, - softmax_lse_accum: ?*const c.FlashattnTensor, - out_accum: ?*const c.FlashattnTensor, - params: ?*const c.FA2MhaVarlenFwdParams, - stream: ?*anyopaque, -) callconv(.c) void; - -const Fa3MhaFwdFunc = *const fn ( - q: ?*const c.FlashattnTensor, - k: ?*const c.FlashattnTensor, - v: ?*const c.FlashattnTensor, - out: ?*const c.FlashattnTensor, - cu_seqlens_q: ?*const c.FlashattnTensor, - cu_seqlens_k: ?*const c.FlashattnTensor, - seqused_q: ?*const c.FlashattnTensor, - seqused_k: ?*const c.FlashattnTensor, - page_table: ?*const c.FlashattnTensor, - q_descale: ?*const c.FlashattnTensor, - k_descale: ?*const c.FlashattnTensor, - v_descale: ?*const c.FlashattnTensor, - softmax_lse: ?*const c.FlashattnTensor, - softmax_lse_accum: ?*const c.FlashattnTensor, - out_accum: ?*const c.FlashattnTensor, - scheduler_metadata: ?*const c.FlashattnTensor, - s_aux: ?*const c.FlashattnTensor, - cp_tot_seqused_k: ?*const c.FlashattnTensor, - params: ?*const c.FA3MhaFwdParams, - stream: ?*anyopaque, -) callconv(.c) void; - -var fa2_mha_fwd: ?Fa2MhaFwdFunc = null; -var fa2_mha_varlen_fwd: ?Fa2MhaVarlenFwdFunc = null; -var fa3_mha_fwd: ?Fa3MhaFwdFunc = null; - -pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { - var r_ = (try runfiles.Runfiles.create(.{ .allocator = allocator, .io = io })).?; - defer r_.deinit(allocator); - - const source_repo = bazel_builtin.current_repository; - const r = r_.withSourceRepo(source_repo); - var buffer: [std.posix.PATH_MAX]u8 = undefined; - const library = (try r.rlocation("flashattn/flashattn/lib/libflashattn.so", &buffer)) orelse return error.NotFound; - const path = try std.posix.toPosixPath(library); - const handle = std.c.dlopen(&path, .{ .NOW = true, .GLOBAL = true }) orelse { - std.log.err("Failed to open libflashattn.so", .{}); - return; - }; - - fa2_mha_varlen_fwd = @ptrCast(@alignCast(std.c.dlsym(handle, "fa2_mha_varlen_fwd") orelse return error.NotFound)); - fa2_mha_fwd = @ptrCast(@alignCast(std.c.dlsym(handle, "fa2_mha_fwd") orelse return error.NotFound)); - fa3_mha_fwd = @ptrCast(@alignCast(std.c.dlsym(handle, "fa3_mha_fwd") orelse return error.NotFound)); -} - -pub fn register(platform: zml.Platform) !void { - try Fa2.register(platform); - try Fa2Varlen.register(platform); - try Fa3.register(platform); -} diff --git a/zml/attention/flashattn/flashattn.zig b/zml/attention/flashattn/flashattn.zig deleted file mode 100644 index 28cff850..00000000 --- a/zml/attention/flashattn/flashattn.zig +++ /dev/null @@ -1,195 +0,0 @@ -const std = @import("std"); -const stdx = @import("stdx"); - -const zml = @import("../../zml.zig"); -const bindings = @import("bindings.zig"); - -pub const load = bindings.load; -pub const register = bindings.register; - -pub const fa2 = struct { - pub const Parameters = struct { - pub const InitOptions = struct {}; - - pub fn init(opts: InitOptions) fa2.Parameters { - _ = opts; // autofix - return .{}; - } - }; - - pub const Metadata = struct { - softmax_lse: zml.Tensor, - softmax_lse_accum: zml.Tensor, - out_accum: zml.Tensor, - - pub const InitOptions = struct { - seqlen: i64, - }; - - pub fn init(opts: InitOptions) fa2.Metadata { - return .{ - .softmax_lse = .init(.{1 * 32 * opts.seqlen * 4}, .i8), - .softmax_lse_accum = .init(.{32 * 1 * 32 * 4 * 4}, .i8), - .out_accum = .init(.{32 * 1 * 32 * 4 * 128 * 4}, .i8), - }; - } - - pub fn initBuffer(self: fa2.Metadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(fa2.Metadata) { - return .{ - .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), .{}), - .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), .{}), - .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), - }; - } - - pub fn deinitBuffer(self: *zml.Bufferized(fa2.Metadata)) void { - self.softmax_lse.deinit(); - self.softmax_lse_accum.deinit(); - self.out_accum.deinit(); - } - }; - - pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: fa2.Metadata, parameters: fa2.Parameters) zml.Tensor { - _ = parameters; // autofix - stdx.debug.assert(q_.shape().hasTag(.b) == null or q_.dim(.b) == 1, "fa2.attention support for batch size != 1 is not supported yet.", .{}); - const seqused_k = token_index.addConstant(q_.dim(.q)).reshape(.{1}); - // TODO(Corendos): replace with cumsum - const cu_seqlens_k = b: { - const zero = zml.Tensor.constant(token_index.dtype().zero()).reshape(.{1}); - break :b zml.Tensor.concatenate(&.{ zero, seqused_k }, 0).convert(.i32); - }; - const max_seqlen_q: i32 = @intCast(q_.dim(.q)); - const max_seqlen_k: i32 = @intCast(k_.dim(.k)); - var q = q_.insertAxes(.q, .{.b}).merge(.{ .tot = .{ .b, .q } }); - const k = k_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); - const v = v_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); - // TODO(Corendos): replace with cumsum - const cu_seqlens_q = zml.Tensor.constantTensor(zml.Shape.init(.{2}, .i32), std.mem.sliceAsBytes(&[2]i32{ 0, max_seqlen_q })); - - const original_tot = q.dim(.tot); - const num_heads = q_.dim(.h); - const num_heads_k = k_.dim(.h); - const head_size = q_.dim(.hd); - const ngroups = @divExact(num_heads, num_heads_k); - const seqlenq_ngroups_swapped = max_seqlen_q == 1 and num_heads > num_heads_k and @mod(head_size, 8) == 0; - if (seqlenq_ngroups_swapped) { - q = q.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .tot, .ngroups, .h, .hd }).merge(.{ .tot = .{ .tot, .ngroups } }); - } - - var o = zml.ops.customCall(bindings.Fa2Varlen.custom_call_name, .{ - q, - k, - v, - cu_seqlens_q, - cu_seqlens_k, - metadata.softmax_lse, - metadata.softmax_lse_accum, - metadata.out_accum, - }, .{q.shape()}, .{ - //.softmax_scale = @as(f32, 1.0), - .is_causal = true, - .window_size_left = @as(i32, -1), - .window_size_right = @as(i32, -1), - .max_seqlen_q = max_seqlen_q, - .max_seqlen_k = max_seqlen_k, - .num_heads = @as(i32, @intCast(num_heads)), - }, .{ - .output_operand_aliases = &.{0}, - .has_side_effect = true, - }); - - if (seqlenq_ngroups_swapped) { - o = o.splitAxis(.tot, .{ .tot = original_tot, .ngroups = ngroups }).transpose(.{ .tot, .h, .ngroups, .hd }).merge(.{ .h = .{ .h, .ngroups } }); - } - - return o.splitAxis(.tot, .{ .b = 1, .q = q_.dim(.q) }).squeeze(.b); - } -}; - -pub const fa3 = struct { - pub const Parameters = struct { - pub const InitOptions = struct {}; - - pub fn init(opts: InitOptions) fa3.Parameters { - _ = opts; // autofix - return .{}; - } - }; - - pub const Metadata = struct { - softmax_lse: zml.Tensor, - softmax_lse_accum: zml.Tensor, - out_accum: zml.Tensor, - scheduler_metadata: zml.Tensor, - - pub const InitOptions = struct { - seqlen: i64, - }; - - pub fn init(opts: InitOptions) fa3.Metadata { - return .{ - .softmax_lse = .init(.{1 * 32 * opts.seqlen * 4}, .i8), - .softmax_lse_accum = .init(.{32 * 1 * 32 * 4 * 4}, .i8), - .out_accum = .init(.{32 * 1 * 32 * 4 * 128 * 4}, .i8), - .scheduler_metadata = .init(.{2}, .i32), - }; - } - - pub fn initBuffer(self: fa3.Metadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(fa3.Metadata) { - return .{ - .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), .{}), - .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), .{}), - .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), - .scheduler_metadata = try zml.Buffer.uninitialized(io, platform, self.scheduler_metadata.shape(), .{}), - }; - } - - pub fn deinitBuffer(self: *zml.Bufferized(fa3.Metadata)) void { - self.softmax_lse.deinit(); - self.softmax_lse_accum.deinit(); - self.out_accum.deinit(); - self.scheduler_metadata.deinit(); - } - }; - - pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: fa3.Metadata, parameters: fa3.Parameters) zml.Tensor { - _ = parameters; // autofix - stdx.debug.assert(q_.shape().hasTag(.b) == null or q_.dim(.b) == 1, "fa3.attention support for batch size != 1 is not supported yet.", .{}); - const seqused_k = token_index.addConstant(q_.dim(.q)).reshape(.{1}); - // TODO(Corendos): replace with cumsum - const cu_seqlens_k = b: { - const zero = zml.Tensor.constant(token_index.dtype().zero()).reshape(.{1}); - break :b zml.Tensor.concatenate(&.{ zero, seqused_k }, 0).convert(.i32); - }; - const max_seqlen_q: i32 = @intCast(q_.dim(.q)); - const max_seqlen_k: i32 = @intCast(k_.dim(.k)); - var q = q_.insertAxes(.q, .{.b}).merge(.{ .tot = .{ .b, .q } }); - const k = k_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); - const v = v_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); - // TODO(Corendos): replace with cumsum - const cu_seqlens_q = zml.Tensor.constantTensor(zml.Shape.init(.{2}, .i32), std.mem.sliceAsBytes(&[2]i32{ 0, max_seqlen_q })); - - var o = zml.ops.customCall(bindings.Fa3.custom_call_name, .{ - q, - k, - v, - cu_seqlens_q, - cu_seqlens_k, - metadata.softmax_lse, - metadata.softmax_lse_accum, - metadata.out_accum, - metadata.scheduler_metadata, - }, .{q.shape()}, .{ - .is_causal = true, - .window_size_left = @as(i32, -1), - .window_size_right = @as(i32, -1), - .max_seqlen_q = max_seqlen_q, - .max_seqlen_k = max_seqlen_k, - }, .{ - .output_operand_aliases = &.{0}, - .has_side_effect = true, - }); - - return o.splitAxis(.tot, .{ .b = 1, .q = q_.dim(.q) }).squeeze(.b); - } -}; From 0b25fbc6f55a1b8ce61efac41a502780c976d3af Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 20 Jan 2026 19:02:55 +0000 Subject: [PATCH 252/881] platforms/cuda/flashattn: proper bazel dep chain this won't depend on flashattn is cuda is disabled --- platforms/BUILD.bazel | 2 +- zml/BUILD.bazel | 8 ++++++-- zml/attention/flashattn.zig | 20 ++++++++++++++------ 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index 3fccf1ad..a2a023b1 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -21,7 +21,7 @@ PLATFORMS = { config_setting( name = "{}.enabled".format(runtime), flag_values = {":{}".format(runtime): "True"}, - visibility = ["//platforms:__subpackages__"], + visibility = ["//visibility:public"], ) for runtime in PLATFORMS.keys() ] diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 9c95f090..8a81b8d8 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -60,9 +60,13 @@ zig_library( "//upb", "//zml/io", "//zml/tokenizer", - "//platforms/cuda/flashattn", "@rules_zig//zig/runfiles", - ], + ] + select({ + "//platforms:cuda.enabled": [ + "//platforms/cuda/flashattn", + ], + "//conditions:default": [], + }), ) # All ZML Tests diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index a941d4d1..10ac36b3 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -1,12 +1,25 @@ const std = @import("std"); const flashattn = @import("platforms/cuda/flashattn"); -pub const load = flashattn.load; +const platforms = @import("platforms"); const stdx = @import("stdx"); const zml = @import("../zml.zig"); const ffi = zml.pjrt.ffi; +pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { + if (comptime platforms.isEnabled(.cuda)) { + try flashattn.load(allocator, io); + } +} + +pub fn register(platform: zml.Platform) !void { + if (comptime platforms.isEnabled(.cuda)) { + try fa2.register(platform); + try fa3.register(platform); + } +} + fn flashattnDataTypeFromZmlDataType(dtype: zml.DataType) flashattn.DataType { return switch (dtype) { .f32 => .f32, @@ -456,8 +469,3 @@ pub const fa3 = struct { return o.splitAxis(.tot, .{ .b = 1, .q = q_.dim(.q) }).squeeze(.b); } }; - -pub fn register(platform: zml.Platform) !void { - try fa2.register(platform); - try fa3.register(platform); -} From 16f877060208640f5d6b77e119f574975301baad Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 21 Jan 2026 11:00:02 +0100 Subject: [PATCH 253/881] zml: fix test_runner --- zml/test_runner.zig | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/zml/test_runner.zig b/zml/test_runner.zig index e3c801f3..121b9be4 100644 --- a/zml/test_runner.zig +++ b/zml/test_runner.zig @@ -22,11 +22,12 @@ pub fn main() !void { var ok_count: usize = 0; var skip_count: usize = 0; var fail_count: usize = 0; - const root_node = std.Progress.start(.{ + const root_node = std.Progress.start(std.testing.io, .{ .root_name = "Test", .estimated_total_items = test_fn_list.len, }); - const have_tty = std.fs.File.stderr().isTty(); + + const have_tty = try std.Io.File.stderr().isTty(std.testing.io); var args = std.process.args(); // Skip executable path @@ -53,7 +54,7 @@ pub fn main() !void { } } - testing.io_instance = .init(testing.allocator_instance.allocator()); + testing.io_instance = .init(testing.allocator_instance.allocator(), .{}); defer testing.io_instance.deinit(); const test_node = root_node.start(test_fn.name, 0); From e04091b5af33e9740fc33e6215e1b9ce0ccd7857 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 21 Jan 2026 11:00:19 +0100 Subject: [PATCH 254/881] zml: Removed ConstSlice constness is now handled at runtime via a union in `zml.Slice` --- examples/llama/main.zig | 2 +- zml/buffer.zig | 13 ++-- zml/nn.zig | 14 ++-- zml/slice.zig | 153 ++++++++++++++++++++-------------------- zml/tensor.zig | 7 +- zml/testing.zig | 20 +++--- zml/zml.zig | 1 - 7 files changed, 102 insertions(+), 108 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 000322dd..97b72f8c 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -531,7 +531,7 @@ pub fn generateText( } // current token pos needs to go into a zml.Buffer - const token_pos_slice: zml.ConstSlice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(prompt_tok.len + i)})); + const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(prompt_tok.len + i)})); const token_pos_buffer: zml.Buffer = try .fromSlice(io, platform, token_pos_slice); defer token_pos_buffer.deinit(); diff --git a/zml/buffer.zig b/zml/buffer.zig index 43067706..80509a76 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -4,7 +4,6 @@ const pjrt = @import("pjrt"); const stdx = @import("stdx"); const constants = @import("constants.zig"); -const ConstSlice = @import("slice.zig").ConstSlice; const DataType = @import("dtype.zig").DataType; const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; @@ -116,16 +115,14 @@ pub const Buffer = struct { return from(io, platform, sh, data, .{}); } - /// Copies the given zml.Slice or zml.ConstSlice to the accelerator memory and + /// Copies the given zml.Slice to the accelerator memory and /// return a Buffer. - pub fn fromSlice(io: std.Io, platform: Platform, slice: anytype) !Buffer { + pub fn fromSlice(io: std.Io, platform: Platform, slice: Slice) !Buffer { return fromSliceOpts(io, platform, slice, .{}); } - pub fn fromSliceOpts(io: std.Io, platform: Platform, slice: anytype, opts: FromOptions) !Buffer { - if (@TypeOf(slice) != Slice and @TypeOf(slice) != ConstSlice) @compileError("Buffer.fromSlice expects a Slice or ConstSlice, got " ++ @typeName(@TypeOf(slice))); - const real_slice: ConstSlice = if (@TypeOf(slice) == Slice) slice.constSlice() else slice; - return from(io, platform, real_slice.shape, std.mem.sliceAsBytes(real_slice.data), opts); + pub fn fromSliceOpts(io: std.Io, platform: Platform, slice: Slice, opts: FromOptions) !Buffer { + return from(io, platform, slice.shape, std.mem.sliceAsBytes(slice.constData()), opts); } /// Creates a Buffer with a single element. @@ -236,7 +233,7 @@ pub const Buffer = struct { /// Copies the content of the Buffer to the provided slice. pub fn toSlice(self: Buffer, io: std.Io, slice: Slice) !void { //stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, slice.data); + const maybe_event = try self._shards.get(0).toHostBuffer(self._api, slice.data()); if (maybe_event) |event| { try event.await(self._api, io); } diff --git a/zml/nn.zig b/zml/nn.zig index d244f235..d4abcd89 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -3,7 +3,7 @@ const std = @import("std"); const stdx = @import("stdx"); -const ConstSlice = @import("slice.zig").ConstSlice; +const Slice = @import("slice.zig").Slice; const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); const ops = @import("ops.zig"); @@ -136,7 +136,7 @@ test normalizeL2 { var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, normalizeL2, .{input_buffer}); defer res.deinit(); - const expectation: ConstSlice = .init(input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6937, -0.7203, -0.9360, 0.3520 })); + const expectation: Slice = .init(input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6937, -0.7203, -0.9360, 0.3520 })); try zml.testing.expectClose(std.testing.io, expectation, res, 1e-4); } @@ -606,7 +606,7 @@ test nearest { defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 1, 1, 6 }, result.shape().dims()); - const expected: ConstSlice = .init(Shape.init(.{ 1, 1, 6 }, .i32), std.mem.sliceAsBytes(&[1][1][6]i32{.{.{ 1, 1, 1, 2, 2, 2 }}})); + const expected: Slice = .init(Shape.init(.{ 1, 1, 6 }, .i32), std.mem.sliceAsBytes(&[1][1][6]i32{.{.{ 1, 1, 1, 2, 2, 2 }}})); try zml.testing.expectClose(std.testing.io, expected, result, 0); } @@ -626,7 +626,7 @@ test nearest { defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 2, 3, 8 }, result.shape().dims()); - const expected: ConstSlice = .init(Shape.init(.{ 2, 3, 8 }, .i32), std.mem.sliceAsBytes(&[2][3][8]i32{ + const expected: Slice = .init(Shape.init(.{ 2, 3, 8 }, .i32), std.mem.sliceAsBytes(&[2][3][8]i32{ .{ .{ 1, 1, 2, 2, 3, 3, 4, 4 }, .{ 5, 5, 6, 6, 7, 7, 8, 8 }, @@ -654,7 +654,7 @@ test nearest { defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 1, 1, 6, 6 }, result.shape().dims()); - const expected: ConstSlice = .init(Shape.init(.{ 1, 1, 6, 6 }, .i32), std.mem.sliceAsBytes(&[1][1][6][6]i32{.{.{ + const expected: Slice = .init(Shape.init(.{ 1, 1, 6, 6 }, .i32), std.mem.sliceAsBytes(&[1][1][6][6]i32{.{.{ .{ 1, 1, 1, 2, 2, 2 }, .{ 1, 1, 1, 2, 2, 2 }, .{ 1, 1, 1, 2, 2, 2 }, @@ -683,7 +683,7 @@ test nearest { defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 2, 2, 4, 4 }, result.shape().dims()); - const expected: ConstSlice = .init(Shape.init(.{ 2, 2, 4, 4 }, .i32), std.mem.sliceAsBytes(&[2][2][4][4]i32{ + const expected: Slice = .init(Shape.init(.{ 2, 2, 4, 4 }, .i32), std.mem.sliceAsBytes(&[2][2][4][4]i32{ .{ .{ .{ 1, 1, 2, 2 }, @@ -728,7 +728,7 @@ test nearest { defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 1, 1, 2, 4, 4 }, result.shape().dims()); - const expected: ConstSlice = .init(Shape.init(.{ 1, 1, 2, 4, 4 }, .i32), std.mem.sliceAsBytes(&[1][1][2][4][4]i32{ + const expected: Slice = .init(Shape.init(.{ 1, 1, 2, 4, 4 }, .i32), std.mem.sliceAsBytes(&[1][1][2][4][4]i32{ .{ .{ .{ diff --git a/zml/slice.zig b/zml/slice.zig index ff13f6bf..3a26a44c 100644 --- a/zml/slice.zig +++ b/zml/slice.zig @@ -6,12 +6,30 @@ const floats = @import("floats.zig"); const DataType = @import("dtype.zig").DataType; const Shape = @import("shape.zig").Shape; +pub fn isBytes(comptime T: type) bool { + const type_info = @typeInfo(T); + if (type_info != .pointer) return false; + if (type_info.pointer.child == u8) return true; + const child_type_info = @typeInfo(type_info.pointer.child); + if (child_type_info != .array) return false; + if (child_type_info.array.child != u8) return false; + return true; +} + pub const Slice = struct { - data: []u8, + inner_data: union(enum) { + mutable: []u8, + immutable: []const u8, + }, shape: Shape, - pub fn init(shape: Shape, data: []u8) Slice { - return .{ .data = data, .shape = shape }; + pub fn init(shape: Shape, bytes: anytype) Slice { + stdx.debug.assertComptime(isBytes(@TypeOf(bytes)), "Expected \"bytes\" to be a []u8 or []const u8, got {s}", .{@typeName(@TypeOf(bytes))}); + const type_info = @typeInfo(@TypeOf(bytes)); + return if (type_info.pointer.is_const) + .{ .inner_data = .{ .immutable = bytes }, .shape = shape } + else + .{ .inner_data = .{ .mutable = bytes }, .shape = shape }; } pub fn alloc(allocator: std.mem.Allocator, shape_: Shape) !Slice { @@ -27,87 +45,70 @@ pub const Slice = struct { else => |v| stdx.debug.panic("Unsupported alignment: {}", .{v}), }; - return .{ .data = bytes, .shape = shape_ }; + return .{ .inner_data = .{ .mutable = bytes }, .shape = shape_ }; } pub fn free(slice: Slice, allocator: std.mem.Allocator) void { - slice.constSlice().free(allocator); - } + const d = slice.constData(); - pub fn constSlice(self: Slice) ConstSlice { - return .{ .data = self.data, .shape = self.shape }; - } - - pub fn dtype(self: Slice) DataType { - return self.shape.dtype(); - } - - pub fn items(self: Slice, comptime T: type) []T { - return @alignCast(std.mem.bytesAsSlice(T, self.data)); - } - - pub fn format( - self: @This(), - writer: *std.Io.Writer, - ) std.Io.Writer.Error!void { - return writer.print("{any}", .{self}); - } - - pub fn formatNumber(self: @This(), writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { - return self.constSlice().formatNumber(writer, n); - } -}; - -pub const ConstSlice = struct { - data: []const u8, - shape: Shape, - - pub fn init(shape: Shape, data: []const u8) ConstSlice { - return .{ .data = data, .shape = shape }; - } - - pub fn free(slice: ConstSlice, allocator: std.mem.Allocator) void { switch (slice.shape.dtype().alignOf()) { - 1 => allocator.free(@as([]align(1) const u8, @alignCast(slice.data))), - 2 => allocator.free(@as([]align(2) const u8, @alignCast(slice.data))), - 4 => allocator.free(@as([]align(4) const u8, @alignCast(slice.data))), - 8 => allocator.free(@as([]align(8) const u8, @alignCast(slice.data))), - 16 => allocator.free(@as([]align(16) const u8, @alignCast(slice.data))), - 32 => allocator.free(@as([]align(32) const u8, @alignCast(slice.data))), - 64 => allocator.free(@as([]align(64) const u8, @alignCast(slice.data))), + 1 => allocator.free(@as([]align(1) const u8, @alignCast(d))), + 2 => allocator.free(@as([]align(2) const u8, @alignCast(d))), + 4 => allocator.free(@as([]align(4) const u8, @alignCast(d))), + 8 => allocator.free(@as([]align(8) const u8, @alignCast(d))), + 16 => allocator.free(@as([]align(16) const u8, @alignCast(d))), + 32 => allocator.free(@as([]align(32) const u8, @alignCast(d))), + 64 => allocator.free(@as([]align(64) const u8, @alignCast(d))), else => |v| stdx.debug.panic("Unsupported alignment: {}", .{v}), } } - pub fn dtype(self: ConstSlice) DataType { - return self.shape.dtype(); + pub fn dtype(slice: Slice) DataType { + return slice.shape.dtype(); } - pub fn items(self: ConstSlice, comptime T: type) []const T { - return @alignCast(std.mem.bytesAsSlice(T, self.data)); + pub fn data(slice: Slice) []u8 { + return switch (slice.inner_data) { + .mutable => |d| d, + else => stdx.debug.panic("Expected slice to be mutable but it's immutable", .{}), + }; + } + + pub fn constData(slice: Slice) []const u8 { + return switch (slice.inner_data) { + inline else => |d| d, + }; + } + + pub fn items(slice: Slice, comptime T: type) []T { + return @alignCast(std.mem.bytesAsSlice(T, slice.data())); + } + + pub fn constItems(slice: Slice, comptime T: type) []const T { + return @alignCast(std.mem.bytesAsSlice(T, slice.constData())); } pub fn format( - self: @This(), + slice: @This(), writer: *std.Io.Writer, ) std.Io.Writer.Error!void { - return writer.print("{any}", .{self}); + return writer.print("{any}", .{slice}); } - pub fn formatNumber(self: @This(), writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { - return self.prettyPrintIndented(writer, 4, 0, n); + pub fn formatNumber(slice: Slice, writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + return slice.prettyPrintIndented(writer, 4, 0, n); } - pub fn prettyPrint(self: ConstSlice, writer: *std.Io.Writer, options: std.fmt.Number) !void { - return self.prettyPrintIndented(writer, 4, 0, options); + pub fn prettyPrint(slice: Slice, writer: *std.Io.Writer, options: std.fmt.Number) !void { + return slice.prettyPrintIndented(writer, 4, 0, options); } - fn prettyPrintIndented(self: ConstSlice, writer: *std.Io.Writer, num_rows: u8, indent_level: u8, options: std.fmt.Number) !void { - if (self.shape.rank() == 0) { + fn prettyPrintIndented(slice: Slice, writer: *std.Io.Writer, num_rows: u8, indent_level: u8, options: std.fmt.Number) !void { + if (slice.shape.rank() == 0) { // Special case input tensor is a scalar - return switch (self.dtype()) { + return switch (slice.dtype()) { inline else => |dt| { - const val: dt.toZigType() = self.items(dt.toZigType())[0]; + const val: dt.toZigType() = slice.constItems(dt.toZigType())[0]; return switch (comptime dt.class()) { // Since we have custom floats, we need to explicitly convert to float32 ourselves. .float => stdx.fmt.formatFloat(floats.floatCast(f32, val), options, writer), @@ -119,16 +120,16 @@ pub const ConstSlice = struct { }; } - if (self.shape.rank() == 1) { + if (slice.shape.rank() == 1) { // Print a contiguous slice of items from the buffer in one line. // The number of items printed is controlled by the user through format syntax. try writer.splatByteAll(' ', indent_level); - switch (self.dtype()) { + switch (slice.dtype()) { inline else => |dt| { const T = dt.toZigType(); // TODO: handle negative strides - const elem_strides: i64 = self.shape.computeElementStrides().get(0); - const values = self.items(T); + const elem_strides: i64 = slice.shape.computeElementStrides().get(0); + const values = slice.constItems(T); switch (comptime dt.class()) { .float => try stdx.fmt.formatFloatSlice(values, options, elem_strides, writer), .integer => try stdx.fmt.formatIntSlice(values, options, elem_strides, writer), @@ -144,12 +145,12 @@ pub const ConstSlice = struct { _ = try writer.write("{\n"); // Write first rows - const n: u64 = @intCast(self.shape.dim(0)); + const n: u64 = @intCast(slice.shape.dim(0)); for (0..@min(num_rows, n)) |d| { - const byte_stride: usize = @intCast(self.shape.computeByteStrides().get(0)); - const sub_slice: ConstSlice = .init(self.shape.drop(0), self.data[d * byte_stride ..][0..byte_stride]); + const byte_stride: usize = @intCast(slice.shape.computeByteStrides().get(0)); + const sub_slice: Slice = .init(slice.shape.drop(0), slice.constData()[d * byte_stride ..][0..byte_stride]); try sub_slice.prettyPrintIndented(writer, num_rows, indent_level + 2, options); - if (self.shape.rank() > 1) { + if (slice.shape.rank() > 1) { try writer.writeAll(",\n"); } } @@ -162,10 +163,10 @@ pub const ConstSlice = struct { } // Write last rows for (@max(n - num_rows, num_rows)..n) |d| { - const byte_stride: usize = @intCast(self.shape.computeByteStrides().get(0)); - const sub_slice: ConstSlice = .init(self.shape.drop(0), self.data[d * byte_stride ..][0..byte_stride]); + const byte_stride: usize = @intCast(slice.shape.computeByteStrides().get(0)); + const sub_slice: Slice = .init(slice.shape.drop(0), slice.constData()[d * byte_stride ..][0..byte_stride]); try sub_slice.prettyPrintIndented(writer, num_rows, indent_level + 2, options); - if (self.shape.rank() > 1) { + if (slice.shape.rank() > 1) { try writer.writeAll(",\n"); } } @@ -202,7 +203,7 @@ test "slice pretty print rank 3" { .{ 60, 61, 62, 63 }, }, }; - const slice = ConstSlice.init(.init(.{ 4, 4, 4 }, .i32), std.mem.asBytes(&data)); + const slice = Slice.init(.init(.{ 4, 4, 4 }, .i32), std.mem.asBytes(&data)); const expected = \\{ \\ { @@ -237,21 +238,21 @@ test "slice pretty print rank 3" { test "slice pretty print rank 1" { const data: [4]i32 = .{ 0, 1, 2, 3 }; - const slice = ConstSlice.init(.init(.{4}, .i32), std.mem.asBytes(&data)); + const slice = Slice.init(.init(.{4}, .i32), std.mem.asBytes(&data)); const expected = "{0,1,2,3}"; try std.testing.expectFmt(expected, "{d}", .{slice}); } test "slice pretty print rank 0" { - const data: [1]i32 = .{ 0 }; - const slice = ConstSlice.init(.init(.{}, .i32), std.mem.asBytes(&data)); + const data: [1]i32 = .{0}; + const slice = Slice.init(.init(.{}, .i32), std.mem.asBytes(&data)); const expected = "0"; try std.testing.expectFmt(expected, "{d}", .{slice}); } test "slice pretty print ellipsis" { const data: [9][1]i32 = .{ .{0}, .{1}, .{2}, .{3}, .{4}, .{5}, .{6}, .{7}, .{8} }; - const slice = ConstSlice.init(.init(.{ 9, 1 }, .i32), std.mem.asBytes(&data)); + const slice = Slice.init(.init(.{ 9, 1 }, .i32), std.mem.asBytes(&data)); const expected = \\{ \\ {0}, diff --git a/zml/tensor.zig b/zml/tensor.zig index 776af877..d82f0373 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -9,7 +9,6 @@ const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("zml.zig").Bufferized; const CompilationContext = @import("module.zig").CompilationContext; const constants = @import("constants.zig"); -const ConstSlice = @import("slice.zig").ConstSlice; const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); @@ -321,7 +320,7 @@ pub const Tensor = struct { const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.fmod, .{input_buffer}); - try zml.testing.expectClose(std.testing.io, ConstSlice.init(zml.Shape.init(.{6}, .f32), std.mem.sliceAsBytes(&e)), output, 1e-4); + try zml.testing.expectClose(std.testing.io, zml.Slice.init(zml.Shape.init(.{6}, .f32), std.mem.sliceAsBytes(&e)), output, 1e-4); } } @@ -1239,7 +1238,7 @@ pub const Tensor = struct { const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.leakyReLU, .{input_buffer}); defer res.deinit(); - const expectation: ConstSlice = .init(input.shape(), std.mem.sliceAsBytes(&[2]f32{ -0.0688, 1.6795 })); + const expectation: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[2]f32{ -0.0688, 1.6795 })); try zml.testing.expectClose(std.testing.io, expectation, res, 1e-4); } @@ -2443,7 +2442,7 @@ pub const Tensor = struct { const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._gatherSlices, .{ operand_buffer, indices_buffer }); defer result.deinit(); - const expected: ConstSlice = .init(Shape.init(.{ 2, 2, 2, 3 }, .u16), std.mem.sliceAsBytes(&[2][2][2][3]u16{ + const expected: zml.Slice = .init(Shape.init(.{ 2, 2, 2, 3 }, .u16), std.mem.sliceAsBytes(&[2][2][2][3]u16{ .{ .{ .{ 13, 14, 15 }, .{ 19, 20, 21 } }, .{ .{ 37, 38, 39 }, .{ 43, 44, 45 } }, diff --git a/zml/testing.zig b/zml/testing.zig index f96dfad9..c78efea4 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -3,7 +3,7 @@ const builtin = @import("builtin"); const stdx = @import("stdx"); -const ConstSlice = @import("slice.zig").ConstSlice; +const Slice = @import("slice.zig").Slice; const Platform = @import("platform.zig").Platform; const zml = @import("zml.zig"); @@ -14,9 +14,7 @@ var _platform: ?Platform = null; pub fn env() Platform { if (!builtin.is_test) @compileError("Cannot use zml.testing.env outside of a test block"); if (_platform == null) { - zml.init(); - - _platform = Platform.auto(std.testing.io, .{}) catch unreachable; + _platform = Platform.auto(std.testing.allocator, std.testing.io, .{}) catch unreachable; } return _platform.?; @@ -35,14 +33,14 @@ pub fn approxEq(comptime Float: type, l: Float, r: Float, tolerance: Float) bool /// host for comparison ! pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, tolerance: f32) !void { const allocator = if (builtin.is_test) std.testing.allocator else std.heap.smp_allocator; - var left: ConstSlice, const should_free_left = if (@TypeOf(left_) == zml.Buffer) b: { + var left: Slice, const should_free_left = if (@TypeOf(left_) == zml.Buffer) b: { const slice = try left_.toSliceAlloc(allocator, io); - break :b .{ slice.constSlice(), true }; + break :b .{ slice, true }; } else .{ left_, false }; - var right: ConstSlice, const should_free_right = if (@TypeOf(right_) == zml.Buffer) b: { + var right: Slice, const should_free_right = if (@TypeOf(right_) == zml.Buffer) b: { const slice = try right_.toSliceAlloc(allocator, io); - break :b .{ slice.constSlice(), true }; + break :b .{ slice, true }; } else .{ right_, false }; defer { @@ -75,7 +73,7 @@ pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, tolerance: f32) .f8e8m0, => |t| { const L = t.toZigType(); - const left_data = left.items(L); + const left_data = left.constItems(L); switch (right.dtype()) { inline .bf16, .f16, @@ -88,7 +86,7 @@ pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, tolerance: f32) .f8e5m2fnuz, => |rt| { const R = rt.toZigType(); - const right_data = right.items(R); + const right_data = right.constItems(R); for (left_data, right_data, 0..) |l, r, i| { if (!approxEq(f32, zml.floats.floatCast(f32, l), zml.floats.floatCast(f32, r), tolerance)) { log.err("left.data != right_data.\n < {d:40.3} \n > {d:40.3}\n error at idx {d}: {d:.3} != {d:.3}", .{ stdx.fmt.slice(center(left_data, i)), stdx.fmt.slice(center(right_data, i)), i, left_data[i], right_data[i] }); @@ -101,7 +99,7 @@ pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, tolerance: f32) }, inline .bool, .u2, .u4, .u8, .u16, .u32, .u64, .i2, .i4, .i8, .i16, .i32, .i64 => |t| { const T = t.toZigType(); - return std.testing.expectEqualSlices(T, left.items(T), right.items(T)); + return std.testing.expectEqualSlices(T, left.constItems(T), right.constItems(T)); }, .c64, .c128 => @panic("TODO: support comparison of complex"), } diff --git a/zml/zml.zig b/zml/zml.zig index 3ea5c22b..a494b298 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -36,7 +36,6 @@ pub const shape = @import("shape.zig"); pub const Shape = shape.Shape; pub const slice = @import("slice.zig"); pub const Slice = slice.Slice; -pub const ConstSlice = slice.ConstSlice; pub const tensor = @import("tensor.zig"); pub const Tensor = tensor.Tensor; pub const testing = @import("testing.zig"); From f89f4d1cac783d8dfc886f9484d8d92cac6a581f Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 21 Jan 2026 17:10:44 +0100 Subject: [PATCH 255/881] zml/io: quickfix for loadBuffersFromId This fix should be properly addressed in the future. --- zml/io.zig | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/zml/io.zig b/zml/io.zig index 0f386f42..e524f78a 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -197,6 +197,7 @@ pub fn TensorBufferTransfer(comptime UserData: type) type { tensor: safetensors.Tensor, buffer: *Buffer, cb_ctx: UserData, + shape: Shape, }; } @@ -283,6 +284,7 @@ pub fn bufferize( .tensor = ptr.*, .buffer = buffer, .cb_ctx = ctx.cb_ctx, + .shape = tensor.shape(), }; ctx.idx += 1; } @@ -354,7 +356,7 @@ pub fn loadBuffersFromId( var writer: Writer = if (ctx.platform.target == .neuron) blk: { break :blk .{ .buffered_device = BufferedDeviceWriter.init(allocator_, ctx.io, null, ctx.platform, ctx.buffer.shape(), ctx.buffer) catch unreachable }; } else blk: { - break :blk .{ .device = DeviceWriter.init(ctx.io, null, ctx.platform, ctx.buffer, .device, write_buffer) catch unreachable }; + break :blk .{ .device = DeviceWriter.init(ctx.io, null, ctx.platform, ctx.shape, ctx.buffer, .device, write_buffer) catch unreachable }; }; switch (writer) { @@ -496,7 +498,7 @@ pub const BufferedDeviceWriter = struct { } }; -fn initBufferizedFrom(model: anytype, bufferized_: *Bufferized(@TypeOf(model))) void { +pub fn initBufferizedFrom(model: anytype, bufferized_: *Bufferized(@TypeOf(model))) void { const Model = @TypeOf(model); const type_info = @typeInfo(Bufferized(Model)); switch (type_info) { From d56e5ce673d2724c0b1c1a04ac9c46d2e2073fef Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 21 Jan 2026 17:11:09 +0100 Subject: [PATCH 256/881] zml/testing: Re-introduced `testLayer` convenience This function helps when porting models from torch. --- zml/testing.zig | 124 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/zml/testing.zig b/zml/testing.zig index c78efea4..fc0e3e6f 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -135,6 +135,130 @@ pub fn autoCall(allocator: std.mem.Allocator, io: std.Io, exe: *const zml.exe.Ex return output; } +fn countWithPrefix(store: zml.io.TensorStore.View, prefix: []const u8) usize { + var count: usize = 0; + var it = store.store.registry.iterator(); + while (it.next()) |entry| { + if (std.mem.startsWith(u8, entry.key_ptr.*, prefix)) { + count += 1; + } + } + + return count; +} + +/// Test a layer implementation against reference activations stored in a TensorStore. +/// The layer is expected to have a method with the name given by `func`. +/// The reference activations are expected to be stored under the keys: +/// - `{name}.in.0`, `{name}.in.1`, ... for inputs +/// - `{name}.out.0`, `{name}.out.1`, ... for outputs +/// +/// The layer weights are expected to be provided in `layer_weights`. +/// The comparison tolerance can be adjusted with `tolerance`. +/// +/// The function arguments need to be valid when initialized to undefined, meaning no slices, +/// no unions, no pointers. +pub fn testlayer( + allocator: std.mem.Allocator, + io: std.Io, + platform: zml.Platform, + layer: anytype, + comptime func: std.meta.DeclEnum(@TypeOf(layer)), + activation_store: zml.io.TensorStore.View, + comptime name: []const u8, + layer_weights: zml.Bufferized(@TypeOf(layer)), + tolerance: f32, +) !void { + const forward = @field(@TypeOf(layer), @tagName(func)); + const ArgsT = stdx.meta.Tail(stdx.meta.FnArgs(forward)); + + var args: ArgsT = undefined; + + const LocalContext = struct { + activation_store: zml.io.TensorStore.View, + index: usize = 0, + }; + + const input_count = zml.meta.count(zml.Tensor, &args); + const expected_input_count = countWithPrefix(activation_store, name ++ ".in"); + if (input_count != expected_input_count) { + log.warn("Reference models uses {d} inputs, but implementation uses {d}", .{ expected_input_count, input_count }); + } + + const store_input = activation_store.withPrefix(name ++ ".in"); + var ctx = LocalContext{ .activation_store = store_input }; + try zml.meta.visit(struct { + fn cb(ctx_: *const LocalContext, tensor: *zml.Tensor) !void { + var buffer: [256]u8 = undefined; + const subkey = std.fmt.bufPrint(&buffer, "{d}", .{ctx_.index}) catch unreachable; + + tensor.* = ctx_.activation_store.createTensor(subkey); + } + }.cb, &ctx, &args); + + const exe = try platform.compile(allocator, io, layer, func, args); + defer exe.deinit(); + + const output_count = exe.output_shapes.len; + const store_output = activation_store.withPrefix(name ++ ".out"); + const expected_output_count = countWithPrefix(store_output, name ++ ".out"); + if (output_count != expected_output_count) { + log.warn("Reference models produces {d} outputs, but implementation produces {d}", .{ expected_output_count, output_count }); + } + + var exe_args = try exe.args(allocator); + defer exe_args.deinit(allocator); + + var exe_results = try exe.results(allocator); + defer exe_results.deinit(allocator); + + var args_buffers = try zml.io.loadBuffersFromId(allocator, io, platform, args, activation_store, .{ .size = 4096, .concurrency = 1 }, .{ .size = 4096, .concurrency = 1 }); + defer zml.meta.visit(struct { + fn cb(_: void, b: *zml.Buffer) void { + b.deinit(); + } + }.cb, {}, &args_buffers); + + exe_args.set(.{ layer_weights, args_buffers }); + + exe.callOpts(io, exe_args, &exe_results, .{ .wait = true }); + + var results = try allocator.alloc(zml.Buffer, output_count); + defer allocator.free(results); + + exe_results.fill(.{results}); + + var failed: bool = false; + var reader_buffer: [4096]u8 = undefined; + for (0..output_count) |i| { + const expected_slice = b: { + var buffer: [256]u8 = undefined; + const subkey = try std.fmt.bufPrint(&buffer, "{d}", .{i}); + var reader = try store_output.getReader(subkey, io, &reader_buffer); + defer reader.deinit(); + const shape = store_output.getShape(subkey).?; + const expected_bytes = try reader.interface.readAlloc(allocator, shape.byteSize()); + break :b zml.Slice.init(shape, expected_bytes); + }; + defer expected_slice.free(allocator); + + const output_slice = try results[i].toSliceAlloc(allocator, io); + defer output_slice.free(allocator); + + zml.testing.expectClose(io, expected_slice, output_slice, tolerance) catch |err| switch (err) { + error.TestUnexpectedResult => { + log.err("{s}.{d} doesn't match !", .{ name ++ ".out", i }); + failed = true; + continue; + }, + else => return err, + }; + } + + if (failed) return error.TestUnexpectedResult; + log.info("all good for {s} !", .{name}); +} + fn center(slice: anytype, i: usize) @TypeOf(slice) { const c = 20; const start = if (i < c) 0 else i - c; From b48c122756268072861e7723ba562c70ad7e3ba7 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 27 Jan 2026 22:48:27 +0000 Subject: [PATCH 257/881] workspace: import cc_* rules from rules_cc This becomes mandatory starting bazel 9 Also fixed in patchelf=0.18.0.bcr.1 --- BUILD.bazel | 2 +- MODULE.bazel | 2 +- bazel/swig.bzl | 4 +++- platforms/cuda/cuda.bzl | 3 +++ platforms/cuda/libpjrt_cuda.BUILD.bazel | 1 + third_party/com_google_sentencepiece/BUILD.bazel | 1 + third_party/com_google_sentencepiece/sentencepiece.bazel | 1 + third_party/flashattn/BUILD.bazel | 2 ++ third_party/org_swig_swig/swig.bazel | 2 ++ zml/tokenizer/sentencepiece/BUILD.bazel | 3 ++- 10 files changed, 17 insertions(+), 4 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index ec4bcf23..4fdc68d5 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -5,6 +5,6 @@ zls_completion( visibility = ["//visibility:public"], deps = [ "//stdx", - "//zml:zml", + "//zml", ], ) diff --git a/MODULE.bazel b/MODULE.bazel index 972e2686..b2be2a3f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -8,7 +8,7 @@ bazel_dep(name = "aspect_bazel_lib", version = "2.21.2") bazel_dep(name = "aspect_rules_py", version = "1.6.3") bazel_dep(name = "bazel_skylib", version = "1.8.2") bazel_dep(name = "libxev", version = "0.0.0-20251014.0-9f785d2") -bazel_dep(name = "patchelf", version = "0.18.0") +bazel_dep(name = "patchelf", version = "0.18.0.bcr.1") bazel_dep(name = "pcre2", version = "10.45") bazel_dep(name = "platforms", version = "1.0.0") bazel_dep(name = "protobuf", version = "33.1", repo_name = "com_google_protobuf") diff --git a/bazel/swig.bzl b/bazel/swig.bzl index 757e2311..d81310fb 100644 --- a/bazel/swig.bzl +++ b/bazel/swig.bzl @@ -1,4 +1,6 @@ load("@rules_cc//cc:action_names.bzl", "C_COMPILE_ACTION_NAME") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_cc//cc:defs.bzl", "CcInfo", "cc_common") load("@rules_cc//cc:find_cc_toolchain.bzl", "find_cc_toolchain", "use_cc_toolchain") def _swig_cc_library_impl(ctx): @@ -141,7 +143,7 @@ def swig_cc_library(name, deps = [], **kwargs): srcs = [":{}.swig".format(name)], output_group = "srcs", ) - native.cc_library( + cc_library( name = name, hdrs = [":{}.hdrs".format(name)], srcs = [":{}.srcs".format(name)], diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index fd95bf77..c786ce3c 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -5,6 +5,9 @@ load("//platforms:packages.bzl", "packages") _BUILD_FILE_DEFAULT_VISIBILITY = """\ package(default_visibility = ["//visibility:public"]) + +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_cc//cc:cc_import.bzl", "cc_import") """ ARCH = "linux-x86_64" diff --git a/platforms/cuda/libpjrt_cuda.BUILD.bazel b/platforms/cuda/libpjrt_cuda.BUILD.bazel index f30962fc..6d7f17e5 100644 --- a/platforms/cuda/libpjrt_cuda.BUILD.bazel +++ b/platforms/cuda/libpjrt_cuda.BUILD.bazel @@ -1,5 +1,6 @@ load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@zml//bazel:patchelf.bzl", "patchelf") +load("@rules_cc//cc:cc_library.bzl", "cc_library") patchelf( name = "libpjrt_cuda_so", diff --git a/third_party/com_google_sentencepiece/BUILD.bazel b/third_party/com_google_sentencepiece/BUILD.bazel index 7314ec78..53e12d3a 100644 --- a/third_party/com_google_sentencepiece/BUILD.bazel +++ b/third_party/com_google_sentencepiece/BUILD.bazel @@ -1,3 +1,4 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") load(":fwd.bzl", "include_fwd") include_fwd( diff --git a/third_party/com_google_sentencepiece/sentencepiece.bazel b/third_party/com_google_sentencepiece/sentencepiece.bazel index f73793b3..9d200b0e 100644 --- a/third_party/com_google_sentencepiece/sentencepiece.bazel +++ b/third_party/com_google_sentencepiece/sentencepiece.bazel @@ -1,6 +1,7 @@ load("@bazel_skylib//rules:copy_file.bzl", "copy_file") load("@com_google_protobuf//bazel:cc_proto_library.bzl", "cc_proto_library") load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") +load("@rules_cc//cc:cc_library.bzl", "cc_library") package( default_visibility = ["//visibility:public"], diff --git a/third_party/flashattn/BUILD.bazel b/third_party/flashattn/BUILD.bazel index 025e0666..3f32c61c 100644 --- a/third_party/flashattn/BUILD.bazel +++ b/third_party/flashattn/BUILD.bazel @@ -1,3 +1,5 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") + filegroup( name = "libflashattn", visibility = ["//visibility:public"], diff --git a/third_party/org_swig_swig/swig.bazel b/third_party/org_swig_swig/swig.bazel index 44d0f9f1..c47ed381 100644 --- a/third_party/org_swig_swig/swig.bazel +++ b/third_party/org_swig_swig/swig.bazel @@ -1,3 +1,5 @@ +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") + licenses(["restricted"]) # GPLv3 exports_files(["LICENSE"]) diff --git a/zml/tokenizer/sentencepiece/BUILD.bazel b/zml/tokenizer/sentencepiece/BUILD.bazel index 10ac080b..0fe8e061 100644 --- a/zml/tokenizer/sentencepiece/BUILD.bazel +++ b/zml/tokenizer/sentencepiece/BUILD.bazel @@ -1,6 +1,7 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") load("//bazel:swig.bzl", "swig_cc_library") load("//bazel:zig_srcs.bzl", "zig_srcs") +load("@rules_cc//cc:cc_static_library.bzl", "cc_static_library") +load("@rules_zig//zig:defs.bzl", "zig_library") swig_cc_library( name = "sentencepiece_swig", From a89c1a16b26a55a27199e9b18c2aa4ecef4d87e9 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 27 Jan 2026 22:53:04 +0000 Subject: [PATCH 258/881] zml: introduce mem subpackage This implement various host pinned allocators as well as static and dynamic buffer pools --- zml/BUILD.bazel | 1 + zml/mem.zig | 379 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 380 insertions(+) create mode 100644 zml/mem.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 8a81b8d8..e8da1a27 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -29,6 +29,7 @@ zig_library( "floats.zig", "io.zig", "meta.zig", + "mem.zig", "mlirx.zig", "module.zig", "nn.zig", diff --git a/zml/mem.zig b/zml/mem.zig new file mode 100644 index 00000000..0912209b --- /dev/null +++ b/zml/mem.zig @@ -0,0 +1,379 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Alignment = std.mem.Alignment; +const assert = std.debug.assert; + +const pjrt = @import("pjrt"); + +const Buffer = @import("buffer.zig").Buffer; +const meta = @import("meta.zig"); +const Platform = @import("platform.zig").Platform; +const Tensor = @import("tensor.zig").Tensor; + +const log = std.log.scoped(.@"zml/mem"); + +pub const HostPinnedAllocator = union(enum) { + passthrough: std.mem.Allocator, + uib: UnitializedBufferHostPinnedAllocator, + dmam: DmaMapAllocator, + + pub fn init(parent: std.mem.Allocator, platform: *const Platform) HostPinnedAllocator { + return switch (platform.target) { + .cuda, .rocm => .{ .dmam = .init(parent, platform) }, + else => .{ .passthrough = parent }, + }; + } + + pub fn allocator(self: *const HostPinnedAllocator) std.mem.Allocator { + return switch (self.*) { + .passthrough => |a| a, + inline else => |*a| a.allocator(), + }; + } +}; + +pub const UnitializedBufferHostPinnedAllocator = struct { + platform: *const Platform, + host_pinned_memory: *const pjrt.Memory, + + const Header = struct { + buffer: *pjrt.Buffer, + }; + + pub fn init(platform: *const Platform) UnitializedBufferHostPinnedAllocator { + const devices = platform.getDevices(); + const mem = platform.memoryForDevice(.host_pinned, devices[0]); + + return .{ + .platform = platform, + .host_pinned_memory = mem, + }; + } + + pub fn allocator(self: *const UnitializedBufferHostPinnedAllocator) std.mem.Allocator { + return .{ + .ptr = @constCast(self), + .vtable = &.{ + .alloc = alloc, + .resize = resize, + .remap = remap, + .free = free, + }, + }; + } + + fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, _: usize) ?[*]u8 { + const self: *UnitializedBufferHostPinnedAllocator = @ptrCast(@alignCast(ctx)); + + const total_len = std.mem.alignForward(usize, @sizeOf(Header) + len, alignment.toByteUnits()); + + const pjrt_buffer = self.platform.pjrt_client.createUninitializedBuffer(self.platform.pjrt_api, .{ + .dims = &.{@intCast(total_len)}, + .element_type = .u8, + .layout = .{ + .tiled = .{ + .minor_to_major = &.{0}, + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }, + .dst = .{ .memory = self.host_pinned_memory }, + }) catch return null; + + const opaque_ptr: [*]u8 = @ptrCast(pjrt_buffer.opaqueDeviceMemoryDataPointer(self.platform.pjrt_api) catch unreachable); + const data_with_header: []u8 = opaque_ptr[0..total_len]; + + const header = std.mem.bytesAsValue(Header, opaque_ptr); + header.* = .{ + .buffer = pjrt_buffer, + }; + const offset = std.mem.alignForward(usize, @sizeOf(Header), alignment.toByteUnits()); + return @ptrCast(data_with_header[offset..]); + } + + fn free(ctx: *anyopaque, buf: []u8, alignment: Alignment, ret_addr: usize) void { + _ = ret_addr; + const self: *UnitializedBufferHostPinnedAllocator = @ptrCast(@alignCast(ctx)); + const header: *Header = @ptrFromInt(std.mem.alignBackward(usize, @intFromPtr(buf.ptr) - @sizeOf(Header), alignment.toByteUnits())); + header.buffer.deinit(self.platform.pjrt_api); + } + + fn resize(ctx: *anyopaque, buf: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) bool { + _ = ctx; + _ = buf; + _ = alignment; + _ = new_len; + _ = ret_addr; + return false; + } + + fn remap(ctx: *anyopaque, buf: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { + _ = ctx; + _ = buf; + _ = alignment; + _ = new_len; + _ = ret_addr; + return null; + } +}; + +pub const DmaMapAllocator = struct { + parent: std.mem.Allocator, + platform: *const Platform, + + pub fn init(parent: std.mem.Allocator, platform: *const Platform) DmaMapAllocator { + return .{ + .parent = parent, + .platform = platform, + }; + } + + pub fn allocator(self: *const DmaMapAllocator) Allocator { + return .{ + .ptr = @constCast(self), + .vtable = &.{ + .alloc = alloc, + .resize = resize, + .remap = remap, + .free = free, + }, + }; + } + + fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[*]u8 { + const self: *const DmaMapAllocator = @ptrCast(@alignCast(ctx)); + const allocation = self.parent.rawAlloc(len, alignment, ret_addr); + if (allocation) |loc| { + const data = loc[0..len]; + self.platform.pjrt_client.dmaMap(self.platform.pjrt_api, @ptrCast(data)) catch { + self.parent.rawFree(data, alignment, ret_addr); + return null; + }; + } + return allocation; + } + + fn resize(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) bool { + _ = ctx; + _ = buf; + _ = alignment; + _ = new_len; + _ = ret_addr; + return false; + } + + fn remap(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { + _ = ctx; + _ = buf; + _ = alignment; + _ = new_len; + _ = ret_addr; + return null; + } + + fn free(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, ret_addr: usize) void { + const self: *const DmaMapAllocator = @ptrCast(@alignCast(ctx)); + self.platform.pjrt_client.dmaUnmap(self.platform.pjrt_api, @ptrCast(buf[0..buf.len])) catch unreachable; + self.parent.rawFree(buf, alignment, ret_addr); + } +}; + +pub const FixedBufferPool = struct { + buffer: []u8, + block_size: usize, + q_buf: []const u16, + q: std.Io.Queue(u16), + + pub fn init(allocator: std.mem.Allocator, buffer_: []u8, blocks_: u16) !FixedBufferPool { + const block_size = @divExact(buffer_.len, blocks_); + const q_buf = try allocator.alloc(u16, blocks_); + for (q_buf, 0..) |*idx, i| { + idx.* = @intCast(i); + } + var q: std.Io.Queue(u16) = .init(q_buf); + q.type_erased.len = q.type_erased.buffer.len; // make the queue full + return .{ + .buffer = buffer_, + .block_size = block_size, + .q = q, + .q_buf = q_buf, + }; + } + + pub fn deinit(self: *FixedBufferPool, allocator: std.mem.Allocator) void { + allocator.free(self.q_buf); + } + + pub fn get(self: *FixedBufferPool, io: std.Io) ![]u8 { + const idx = try self.q.getOneUncancelable(io); + return self.buffer[idx * self.block_size ..][0..self.block_size]; + } + + fn inRange(sub_buffer: []const u8, buffer: []const u8) bool { + return @intFromPtr(sub_buffer.ptr) >= @intFromPtr(buffer.ptr) and + @intFromPtr(sub_buffer[sub_buffer.len - 1 ..].ptr) <= @intFromPtr(buffer[buffer.len - 1 ..].ptr); + } + + pub fn put(self: *FixedBufferPool, io: std.Io, buf: []u8) void { + // is the pointer in range ? + std.debug.assert(inRange(buf, self.buffer)); + const idx = @divExact(@intFromPtr(buf.ptr) - @intFromPtr(self.buffer.ptr), self.block_size); + self.q.putOneUncancelable(io, @intCast(idx)) catch unreachable; + } +}; + +pub const DynamicBufferPool = struct { + const Node = struct { next: ?*Node }; + const alignment: std.mem.Alignment = .max(.of(u8), .of(Node)); + const ItemPtr = [*]align(alignment.toByteUnits()) u8; + const Slice = []align(alignment.toByteUnits()) u8; + + block_size: usize, + max_blocks: usize, + + arena_state: std.heap.ArenaAllocator.State = .{}, + free_stack: std.atomic.Value(?*Node) = .init(null), + in_flight: std.atomic.Value(usize) = .init(0), + mutex: std.Io.Mutex = .init, + condition: std.Io.Condition = .init, + + pub fn init(max_blocks: usize, block_size: usize) DynamicBufferPool { + return .{ + .block_size = block_size, + .max_blocks = max_blocks, + }; + } + + pub fn deinit(self: *DynamicBufferPool, allocator: std.mem.Allocator) void { + self.arena_state.promote(allocator).deinit(); + } + + pub fn get(self: *DynamicBufferPool, allocator: std.mem.Allocator, io: std.Io) ![]u8 { + while (true) { + var in_flight = self.in_flight.load(.acquire); + while (in_flight < self.max_blocks) { + if (self.in_flight.cmpxchgWeak(in_flight, in_flight + 1, .release, .acquire)) |actual| { + in_flight = actual; + continue; + } + + if (self.pop()) |node| { + return self.nodeToSlice(node); + } + + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + + var arena = self.arena_state.promote(allocator); + defer self.arena_state = arena.state; + + errdefer _ = self.in_flight.fetchSub(1, .release); + const buffer = try arena.allocator().alignedAlloc(u8, alignment, self.block_size); + return @alignCast(buffer); + } + + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + while (self.in_flight.load(.acquire) >= self.max_blocks) { + try self.condition.wait(io, &self.mutex); + } + } + } + + pub fn put(self: *DynamicBufferPool, io: std.Io, buf: []u8) void { + const node: *Node = @ptrCast(@alignCast(buf.ptr)); + var head = self.free_stack.load(.acquire); + while (true) { + node.next = head; + if (self.free_stack.cmpxchgWeak(head, node, .release, .acquire)) |actual_head| { + head = actual_head; + } else { + break; + } + } + + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + _ = self.in_flight.fetchSub(1, .release); + self.condition.signal(io); + } + + fn pop(self: *DynamicBufferPool) ?*Node { + var head = self.free_stack.load(.acquire); + while (true) { + if (self.free_stack.cmpxchgWeak(head, if (head) |h| h.next else null, .acquire, .acquire)) |actual| { + head = actual; + continue; + } + + return head; + } + } + + fn nodeToSlice(self: *DynamicBufferPool, node: *Node) Slice { + const ptr: ItemPtr = @ptrCast(@alignCast(node)); + return ptr[0..self.block_size]; + } +}; + +/// Return a clone of a type with Tensors replaced by Buffer. +/// Non-Tensor metadata is stripped out of the resulting struct. +/// Recursively descends into the type. +pub fn Bufferized(comptime T: type) type { + @setEvalBranchQuota(10_000); + return meta.MapRestrict(Tensor, Buffer).map(T); +} + +fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bufferized(@TypeOf(model))) !void { + @setEvalBranchQuota(10_000); + const Model = @TypeOf(model); + const ModelBufferized = Bufferized(Model); + + if (ModelBufferized == Buffer) { + return; + } + + const type_info = @typeInfo(Bufferized); + switch (type_info) { + .@"struct" => |struct_type_info| { + inline for (struct_type_info.fields) |field| { + try bufferizeInner(allocator, @field(model, field.name), &@field(bufferized_, field.name)); + } + }, + .@"union" => { + switch (model) { + inline else => |v, tag| { + bufferized_.* = @unionInit(ModelBufferized, @tagName(tag), undefined); + try bufferizeInner(allocator, v, @field(bufferized_, @tagName(tag))); + }, + } + }, + .optional => |optional_type_info| { + if (model == null) { + bufferized_.* = null; + } else { + bufferized_.* = @as(optional_type_info.child, undefined); + try bufferizeInner(allocator, model.?, &bufferized_.*.?); + } + }, + .pointer => |p| { + switch (p.size) { + .slice => { + bufferized_.* = try allocator.alignedAlloc(p.child, .fromByteUnits(p.alignment), model.len); + for (model, bufferized_.*) |src, *dst| { + try bufferizeInner(allocator, src, dst); + } + }, + else => unreachable, + } + }, + .void, .int, .@"enum", .bool, .enum_literal, .float, .vector => {}, + else => unreachable, + } +} + +pub inline fn bufferize(allocator: std.mem.Allocator, model: anytype) !Bufferized(@TypeOf(model)) { + var bufferized: Bufferized(@TypeOf(model)) = undefined; + try bufferizeInner(allocator, model, &bufferized); + return bufferized; +} From 24137cdc114b089e110cabfec4cb0862f2cbe992 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 27 Jan 2026 23:19:10 +0000 Subject: [PATCH 259/881] pjrt: sugarize Event.onReady Dont't expose callconv stuff, just pass a comptimed function --- pjrt/pjrt.zig | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 000c1989..98caf5ff 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -1139,15 +1139,14 @@ pub const Event = opaque { return; } - var ctx = struct { + const Ctx = struct { err: ?*Error = null, event: std.Io.Event = .unset, io: std.Io, - }{ .io = io }; - - try self.onReady(api, struct { - fn call(err: ?*Error, user_arg: ?*anyopaque) callconv(.c) void { - const ctx_: *@TypeOf(ctx) = @ptrCast(@alignCast(user_arg.?)); + }; + var ctx: Ctx = .{ .io = io }; + try self.onReady(api, Ctx, struct { + fn call(err: ?*Error, ctx_: *Ctx) void { ctx_.err = err; ctx_.event.set(ctx_.io); } @@ -1168,10 +1167,14 @@ pub const Event = opaque { }); } - pub fn onReady(self: *Event, api: *const Api, func: *const fn (err: ?*Error, user_arg: ?*anyopaque) callconv(.c) void, user_arg: ?*anyopaque) ApiError!void { + pub fn onReady(self: *Event, api: *const Api, comptime T: type, func: fn (err: ?*Error, user_arg: *T) void, user_arg: *T) ApiError!void { _ = try api.call(.PJRT_Event_OnReady, .{ .event = self.inner(), - .callback = @ptrCast(func), + .callback = @ptrCast(&struct { + fn call(err: ?*Error, user_arg_: ?*anyopaque) callconv(.c) void { + func(err, @ptrCast(@alignCast(user_arg_.?))); + } + }.call), .user_arg = user_arg, }); } From a8154fb5def076b1b0b47780091a9a487fe061e4 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Wed, 28 Jan 2026 18:07:06 +0000 Subject: [PATCH 260/881] zml: upgrade meta.TupleRange to Zig 0.16 --- stdx/meta.zig | 34 ++++++++++------------------------ 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/stdx/meta.zig b/stdx/meta.zig index 0d724a56..3bfc8895 100644 --- a/stdx/meta.zig +++ b/stdx/meta.zig @@ -123,33 +123,19 @@ pub fn asSlice(comptime T: type) type { } pub fn TupleRange(comptime T: type, comptime start: ?usize, comptime end: ?usize) type { - return TupleRangeX(T, start orelse 0, end orelse std.meta.fields(T).len); -} - -pub fn TupleRangeX(comptime T: type, comptime start: usize, comptime end: usize) type { const fields = std.meta.fields(T); + const start_ = start orelse 0; + const end_ = end orelse fields.len; - var field_names: [end - start][]const u8 = undefined; - var field_types: [end - start]type = undefined; - var field_attrs: [end - start]std.builtin.Type.StructField.Attributes = undefined; - - inline for (start..end, 0..) |i, j| { - var new_field = fields[i]; - var num_buf: [32]u8 = undefined; - new_field.name = blk: { - const s = std.fmt.formatIntBuf(&num_buf, j, 10, .lower, .{}); - num_buf[s] = 0; - break :blk num_buf[0..s :0]; - }; - field_names[j] = new_field.name; - field_types[j] = new_field.type; - field_attrs[j] = .{ - .@"comptime" = new_field.is_comptime, - .@"align" = new_field.alignment, - .default_value_ptr = new_field.default_value_ptr, - }; + if (start_ == end_) { + return @Tuple(&.{}); } - return @Struct(.auto, null, &field_names, &field_types, &field_attrs); + + var field_types: [end_ - start_]type = undefined; + inline for (start_..end_, 0..) |i, j| { + field_types[j] = fields[i].type; + } + return @Tuple(&field_types); } pub fn FnParam(comptime func: anytype, comptime n: comptime_int) type { From 7f936ba7b333e7d89d1ea58b068d81f1ca4e87ae Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Wed, 28 Jan 2026 18:11:35 +0000 Subject: [PATCH 261/881] zml: create and pass Platform via const pointer Platform is destined to hold more things, since it's effectively an object that is instanciated once per program. Make it a pointer and pass it via const pointers around. --- zml/attention/flashattn.zig | 4 +- zml/buffer.zig | 16 ++-- zml/exe.zig | 8 +- zml/module.zig | 8 +- zml/platform.zig | 145 ++++++++++++++++++++---------------- 5 files changed, 97 insertions(+), 84 deletions(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 10ac36b3..a9a77300 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -13,7 +13,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { } } -pub fn register(platform: zml.Platform) !void { +pub fn register(platform: *zml.Platform) !void { if (comptime platforms.isEnabled(.cuda)) { try fa2.register(platform); try fa3.register(platform); @@ -121,7 +121,7 @@ fn fixupKvCacheBuffer(buffer: Buffer, layer_index: i64) Buffer { pub fn Wrapper(comptime T: type, run_func: std.meta.DeclEnum(T)) type { return struct { - pub fn register(platform: zml.Platform) !void { + pub fn register(platform: *zml.Platform) !void { try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "cuda", T.run, .{ .command_buffer_compatible = true }); } diff --git a/zml/buffer.zig b/zml/buffer.zig index 80509a76..a5dc31cb 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -34,7 +34,7 @@ pub const Buffer = struct { /// Frees the accelerator memory. /// Depending on the platform, the memory is typically not released to the OS /// but just marked as available in the memory pool. - pub fn deinit(self: *const Buffer) void { + pub fn deinit(self: *Buffer) void { // log.warn("Unloading {f} {d} bytes", .{ self._shape, self._shape.byteSize() }); for (self._shards.constSlice()) |buffer| { buffer.deinit(self._api); @@ -51,7 +51,7 @@ pub const Buffer = struct { } /// Copies the content of the given buffer from host memory to the accelerator memory. - pub fn from(io: std.Io, platform: Platform, shape_: Shape, data_: []const u8, opts: FromOptions) !Buffer { + pub fn from(io: std.Io, platform: *const Platform, shape_: Shape, data_: []const u8, opts: FromOptions) !Buffer { var res: Buffer = .{ ._api = platform.pjrt_api, ._target = platform.target, @@ -111,22 +111,22 @@ pub const Buffer = struct { /// Copies the given Zig bytes to the accelerator memory and /// return a Buffer with the given dimensions. - pub fn fromBytes(io: std.Io, platform: Platform, sh: Shape, data: []const u8) !Buffer { + pub fn fromBytes(io: std.Io, platform: *const Platform, sh: Shape, data: []const u8) !Buffer { return from(io, platform, sh, data, .{}); } /// Copies the given zml.Slice to the accelerator memory and /// return a Buffer. - pub fn fromSlice(io: std.Io, platform: Platform, slice: Slice) !Buffer { + pub fn fromSlice(io: std.Io, platform: *const Platform, slice: Slice) !Buffer { return fromSliceOpts(io, platform, slice, .{}); } - pub fn fromSliceOpts(io: std.Io, platform: Platform, slice: Slice, opts: FromOptions) !Buffer { + pub fn fromSliceOpts(io: std.Io, platform: *const Platform, slice: Slice, opts: FromOptions) !Buffer { return from(io, platform, slice.shape, std.mem.sliceAsBytes(slice.constData()), opts); } /// Creates a Buffer with a single element. - pub fn scalar(io: std.Io, platform: Platform, val: anytype, dtype_: DataType) !Buffer { + pub fn scalar(io: std.Io, platform: *const Platform, val: anytype, dtype_: DataType) !Buffer { const x = dtype_.constant(val); return fromBytes(io, platform, Shape.init(.{}, dtype_), x.asBytes()); } @@ -143,7 +143,7 @@ pub const Buffer = struct { pub const UnitializedOptions = struct { memory: Memory = .device }; - pub fn uninitialized(io: std.Io, platform: Platform, shape_: Shape, opts: UnitializedOptions) !Buffer { + pub fn uninitialized(io: std.Io, platform: *const Platform, shape_: Shape, opts: UnitializedOptions) !Buffer { if (opts.memory != .device) { // XLA uninitialized doesn't respect memory see https://github.com/openxla/xla/pull/31292 // TODO: use uninitialized when it works again. @@ -202,7 +202,7 @@ pub const Buffer = struct { } /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. - pub fn fromPjrtBuffers(platform: Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { + pub fn fromPjrtBuffers(platform: *const Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); stdx.debug.assert(pjrt_buffers.len > 0, "fromPjrtBuffers expects at least one buffer, got 0.", .{}); var shards: Shards = .{}; diff --git a/zml/exe.zig b/zml/exe.zig index 49361e0b..f5503d65 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -9,7 +9,7 @@ const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; pub const Exe = struct { - platform: Platform, + platform: *const Platform, exe: *pjrt.LoadedExecutable, context: ?*pjrt.ExecuteContext = null, @@ -23,7 +23,7 @@ pub const Exe = struct { pub fn init( allocator: std.mem.Allocator, - platform: Platform, + platform: *const Platform, exe: *pjrt.LoadedExecutable, input_shapes: []const Shape, output_shapes: []const Shape, @@ -133,12 +133,12 @@ pub const Exe = struct { }; pub const Results = struct { - platform: Platform, + platform: *const Platform, flat_buffers: FlatBuffers, expected_shapes: []const Shape, - pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize, platform: Platform) !Results { + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize, platform: *const Platform) !Results { const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); errdefer flat_buffers.deinit(allocator); diff --git a/zml/module.zig b/zml/module.zig index 25082857..7a2a43af 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -52,7 +52,7 @@ pub const CompilationContext = struct { mlir_pass_manager: *mlir.PassManager, //mlir_op_pass_manager: *mlir.OpPassManager, module: *mlir.Module, - platform: Platform, + platform: *const Platform, scopes: stdx.BoundedArray(Scope, 16) = .{}, @@ -64,7 +64,7 @@ pub const CompilationContext = struct { } }.call); - pub fn init(allocator: std.mem.Allocator, platform: Platform) CompilationContext { + pub fn init(allocator: std.mem.Allocator, platform: *const Platform) CompilationContext { mlir_once.call(); const mlir_registry = mlir.DialectRegistry.init() catch unreachable; inline for (.{ "func", "stablehlo" }) |d| { @@ -175,7 +175,7 @@ pub const CompilationContext = struct { //} }; -pub fn compile(allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, args: stdx.meta.FnArgs(func), platform: Platform) !Exe { +pub fn compile(allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, args: stdx.meta.FnArgs(func), platform: *const Platform) !Exe { var compilation_context: CompilationContext = .init(allocator, platform); defer compilation_context.deinit(); @@ -425,7 +425,7 @@ fn setXlaOverrideFlag(map: *c.upb_Map, flag: []const u8, value: anytype, upb_are } } -fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: Platform, module: *const mlir.Module, xla_dump_to_: ?[]const u8) !*pjrt.LoadedExecutable { +fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: *const Platform, module: *const mlir.Module, xla_dump_to_: ?[]const u8) !*pjrt.LoadedExecutable { //const tracer = Tracer.init("ai.zml.compilation"); //const compile_frame = tracer.frameStart("pjrt compilation"); //defer tracer.frameEnd(compile_frame, "pjrt compilation"); diff --git a/zml/platform.zig b/zml/platform.zig index 2f70a978..eb2f3b15 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -60,10 +60,56 @@ fn loadOrGetApi(allocator: std.mem.Allocator, io: std.Io, target: Target) !*cons }; } +pub const Device = struct { + platform: *const Platform, + device: *const pjrt.Device, + desc: *const pjrt.DeviceDescription, + + fn init(device_: *const pjrt.Device, platform: *const Platform) Device { + return .{ + .platform = platform, + .device = device_, + .desc = device_.getDescription(platform.pjrt_api), + }; + } + + pub fn id(self: Device) i32 { + return self.desc.id(self.platform.pjrt_api); + } + + pub fn processIndex(self: Device) i32 { + return self.desc.processIndex(self.platform.pjrt_api); + } + + pub fn localHardwareId(self: Device) i32 { + return self.device.localHardwareId(self.platform.pjrt_api); + } + + pub fn kind(self: Device) []const u8 { + return self.desc.kind(self.platform.pjrt_api); + } + + pub fn debugString(self: Device) []const u8 { + return self.desc.debugString(self.platform.pjrt_api); + } + + pub fn toString(self: Device) []const u8 { + return self.desc.toString(self.platform.pjrt_api); + } + + pub fn format(self: Device, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("{s} ({s})", .{ + self.desc.kind(self.platform.pjrt_api), + self.desc.debugString(self.platform.pjrt_api), + }); + } +}; + pub const Platform = struct { target: Target, pjrt_api: *const pjrt.Api, pjrt_client: *pjrt.Client, + _devices: []Device, // This make the pjrt struct quite fat, but is only used during compilation. // TODO: Reconsider having it here, and maybe pass explicitly to compile, @@ -73,54 +119,33 @@ pub const Platform = struct { pub const MAX_NUM_DEVICES: u8 = if (platforms.isEnabled(.tpu)) 32 else 8; - pub const Device = struct { - pub const Iterator = struct { - api: *const pjrt.Api, - devices: []const *const pjrt.Device, - current: usize = 0, - - pub fn next(self: *Iterator) ?Device { - if (self.current >= self.devices.len) { - return null; - } - defer self.current += 1; - return .{ - .api = self.api, - .device = self.devices[self.current], - }; - } - }; - - api: *const pjrt.Api, - device: *const pjrt.Device, - - pub fn format(self: Device, writer: *std.Io.Writer) std.Io.Writer.Error!void { - const description = self.device.getDescription(self.api); - try writer.print("{s} ({s})", .{ description.kind(self.api), description.debugString(self.api) }); - } - }; - - pub fn init(allocator: std.mem.Allocator, io: std.Io, target: Target, options: CreateOptions) !Platform { + pub fn init(allocator: std.mem.Allocator, io: std.Io, target: Target, options: CreateOptions) !*Platform { const api = try loadOrGetApi(allocator, io, target); var named_values_buf: [16]pjrt.NamedValue = undefined; const pjrt_client = try pjrt.Client.init(api, options.toNamedValues(target, &named_values_buf)); - const true_num_devices = pjrt_client.addressableDevices(api).len; - if (true_num_devices > MAX_NUM_DEVICES) { - log.warn("platform {} got {} devices, but ZML only support up to {} devices. Some devices won't be used.", .{ target, true_num_devices, MAX_NUM_DEVICES }); + const pjrt_devices = pjrt_client.addressableDevices(api); + if (pjrt_devices.len > MAX_NUM_DEVICES) { + log.warn("platform {} got {} devices, but ZML only support up to {} devices. Some devices won't be used.", .{ target, pjrt_devices.len, MAX_NUM_DEVICES }); } - const platform: Platform = .{ + var platform = try allocator.create(Platform); + platform.* = .{ .target = target, .pjrt_api = api, .pjrt_client = pjrt_client, .compilation_options = .{}, + ._devices = try allocator.alloc(Device, pjrt_devices.len), }; + for (pjrt_devices, platform._devices) |pjrt_device, *platform_device| { + platform_device.* = .init(pjrt_device, platform); + } + switch (target) { .cuda => { // TODO(Corentin): provide a better allocator - zml.attention.flashattn.load(std.heap.c_allocator, io) catch { + zml.attention.flashattn.load(allocator, io) catch { log.warn("Failed to load flashattn", .{}); }; zml.attention.flashattn.register(platform) catch { @@ -133,7 +158,7 @@ pub const Platform = struct { return platform; } - pub fn auto(allocator: std.mem.Allocator, io: std.Io, options: CreateOptions) !Platform { + pub fn auto(allocator: std.mem.Allocator, io: std.Io, options: CreateOptions) !*Platform { const ordered_targets: []const Target = &.{ .tpu, .neuron, @@ -146,28 +171,23 @@ pub const Platform = struct { } else error.Unavailable; } - pub fn devicesIterator(self: Platform) Device.Iterator { - return .{ - .api = self.pjrt_api, - .devices = self.getDevices(), - }; + pub fn devices(self: *const Platform) []const Device { + return self._devices; } - pub fn formatWithDevices(self: Platform, writer: *std.Io.Writer) std.Io.Writer.Error!void { + pub fn formatWithDevices(self: *const Platform, writer: *std.Io.Writer) std.Io.Writer.Error!void { try writer.print("Platform: {s}\n", .{@tagName(self.target)}); - const devices = self.getDevices(); - try writer.print("Devices total={d}\n", .{devices.len}); - for (devices, 0..) |device, i| { - const description = device.getDescription(self.pjrt_api); - try writer.print("\t#{d}: {s}\n", .{ i, description.kind(self.pjrt_api) }); + try writer.print("Devices total={d}\n", .{devices().len}); + for (devices(), 0..) |device, i| { + try writer.print("\t#{d}: {s}\n", .{ i, device.kind() }); } } - pub fn fmtVerbose(self: Platform) std.fmt.Alt(Platform, formatWithDevices) { + pub fn fmtVerbose(self: *const Platform) std.fmt.Alt(*const Platform, formatWithDevices) { return .{ .data = self }; } - pub fn getDevices(self: Platform) []const *const pjrt.Device { + pub fn getDevices(self: *const Platform) []const *const pjrt.Device { const all_devices = self.pjrt_client.addressableDevices(self.pjrt_api); if (all_devices.len > MAX_NUM_DEVICES) { return all_devices[0..MAX_NUM_DEVICES]; @@ -177,7 +197,7 @@ pub const Platform = struct { pub const Sharding = struct { num_replicas: u8, num_partitions: u8 }; - pub fn sharding(self: Platform) Sharding { + pub fn sharding(self: *const Platform) Sharding { // replicas run the same function but with different inputs, // while partitions contribute to one evaluation over a shared input. // Inside an inference process, we generally don't want replicas, @@ -190,17 +210,12 @@ pub const Platform = struct { .{ .num_replicas = 1, .num_partitions = 1 }; } - pub fn withCompilationOptions(self: Platform, opts: CompilationOptions) Platform { - var res = self; - res.compilation_options = opts; - return res; - } - - pub fn deinit(self: *Platform) void { + pub fn deinit(self: *Platform, allocator: std.mem.Allocator) void { + allocator.free(self._devices); self.pjrt_client.deinit(self.pjrt_api); } - pub fn memoryForDevice(platform: Platform, memory: pjrt.Memory.Kind, device: *const pjrt.Device) *const pjrt.Memory { + pub fn memoryForDevice(platform: *const Platform, memory: pjrt.Memory.Kind, device: *const pjrt.Device) *const pjrt.Memory { const memory_target: pjrt.Memory.Kind = switch (memory) { .host_unpinned => switch (platform.target) { // Cuda doesn't have host_unpinned. @@ -232,7 +247,7 @@ pub const Platform = struct { } } - pub fn memoryStats(platform: Platform, device_id: usize) pjrt.MemoryStats { + pub fn memoryStats(platform: *const Platform, device_id: usize) pjrt.MemoryStats { if (platform.target == .cpu) return .zeroes; const device = platform.getDevices()[device_id]; @@ -240,7 +255,7 @@ pub const Platform = struct { } pub fn compile( - self: Platform, + self: *const Platform, allocator: std.mem.Allocator, io: std.Io, model_: anytype, @@ -255,21 +270,19 @@ pub const Platform = struct { ); } - pub fn compileModel(self: Platform, allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, model: stdx.meta.Head(stdx.meta.FnArgs(func)), args: stdx.meta.Tail(stdx.meta.FnArgs(func))) !Exe { + pub fn compileModel(self: *const Platform, allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, model: stdx.meta.Head(stdx.meta.FnArgs(func)), args: stdx.meta.Tail(stdx.meta.FnArgs(func))) !Exe { return self.compileFn(allocator, io, func, .{model} ++ args); } - pub fn compileFn(self: Platform, allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, args: stdx.meta.FnArgs(func)) !Exe { + pub fn compileFn(self: *const Platform, allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, args: stdx.meta.FnArgs(func)) !Exe { return zml.module.compile(allocator, io, func, args, self); } - pub fn format(self: @This(), writer: *std.Io.Writer) std.Io.Writer.Error!void { + pub fn format(self: *const Platform, writer: *std.Io.Writer) std.Io.Writer.Error!void { try writer.print("{s} {{ ", .{@tagName(self.target)}); - const devices = self.getDevices(); - for (0..devices.len) |i| { - const description = devices[i].getDescription(self.pjrt_api); - try writer.print("{s}(\"{s}\")", .{ description.toString(self.pjrt_api), description.kind(self.pjrt_api) }); - if (i < devices.len - 1) try writer.writeAll(", "); + for (self.devices(), 0..) |device, i| { + try writer.print("{s}(\"{s}\")", .{ device.toString(), device.kind() }); + if (i < devices().len - 1) try writer.writeAll(", "); } try writer.writeAll(" }"); } From 7832fd26ac5cb3d555c78eedd7f78d8ffa3e9089 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Wed, 28 Jan 2026 18:12:15 +0000 Subject: [PATCH 262/881] stdx: rework stdx.Io.LimitedGroup to use atomics and conditions --- stdx/Io.zig | 89 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 38 deletions(-) diff --git a/stdx/Io.zig b/stdx/Io.zig index 5f73bd05..a7a1fe3e 100644 --- a/stdx/Io.zig +++ b/stdx/Io.zig @@ -22,50 +22,63 @@ pub const Dir = struct { } }; -pub fn LimitedConcurrentGroup(comptime limit: usize) type { - return struct { - const Self = @This(); +pub const LimitedGroup = struct { + limit: usize, + in_flight: std.atomic.Value(usize) = .init(0), + group: std.Io.Group = .init, + cond: std.Io.Condition = .init, + mutex: std.Io.Mutex = .init, - queue_buffer: [limit]bool = undefined, + fn Wrapper(comptime function: anytype) type { + return struct { + fn wrapper(self: *LimitedGroup, io: std.Io, args: std.meta.ArgsTuple(@TypeOf(function))) std.Io.Cancelable!void { + while (true) { + var in_flight = self.in_flight.load(.acquire); + while (in_flight < self.limit) { + if (self.in_flight.cmpxchgWeak(in_flight, in_flight + 1, .release, .acquire)) |actual| { + in_flight = actual; + continue; + } - group: std.Io.Group = .init, - queue: std.Io.Queue(bool) = undefined, + defer { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + _ = self.in_flight.fetchSub(1, .release); + self.cond.signal(io); + } + return @call(.auto, function, args); + } - pub fn init() Self { - var s: Self = .{}; - s.queue = std.Io.Queue(bool).init(&s.queue_buffer); - return s; - } - - pub fn concurrent( - self: *Self, - io: std.Io, - comptime function: anytype, - args: std.meta.ArgsTuple(@TypeOf(function)), - ) !void { - self.queue.putOneUncancelable(io, true) catch unreachable; - - try self.group.concurrent(io, struct { - fn wrapper( - io_: std.Io, - queue: *std.Io.Queue(bool), - inner_args: std.meta.ArgsTuple(@TypeOf(function)), - ) !void { - defer _ = queue.getOneUncancelable(io_) catch unreachable; - return @call(.auto, function, inner_args); + try self.mutex.lock(io); + defer self.mutex.unlock(io); + while (self.in_flight.load(.acquire) >= self.limit) { + try self.cond.wait(io, &self.mutex); + } } - }.wrapper, .{ io, &self.queue, args }); - } + } + }; + } - pub fn await(self: *AllocatingLimitedConcurrentGroup, io: std.Io) std.Io.Cancelable!void { - try self.group.await(io); - } + pub fn init(limit: usize) LimitedGroup { + return .{ .limit = limit }; + } - pub fn cancel(self: *Self, io: std.Io) void { - self.group.cancel(io); - } - }; -} + pub fn async(self: *LimitedGroup, io: std.Io, comptime function: anytype, args: std.meta.ArgsTuple(@TypeOf(function))) void { + self.group.async(io, Wrapper(function).wrapper, .{ self, io, args }); + } + + pub fn concurrent(self: *LimitedGroup, io: std.Io, comptime function: anytype, args: std.meta.ArgsTuple(@TypeOf(function))) std.Io.ConcurrentError!void { + try self.group.concurrent(io, Wrapper(function).wrapper, .{ self, io, args }); + } + + pub fn await(self: *LimitedGroup, io: std.Io) std.Io.Cancelable!void { + try self.group.await(io); + } + + pub fn cancel(self: *LimitedGroup, io: std.Io) void { + self.group.cancel(io); + } +}; pub const AllocatingLimitedConcurrentGroup = struct { allocator: std.mem.Allocator, From f76f02984e777f106e3f4db2e6b4b3c664951d1b Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Wed, 28 Jan 2026 18:13:17 +0000 Subject: [PATCH 263/881] zml: add io.BufferedDeviceWriter, io.DirectDeviceWriter and io.ProgressWriter --- zml/io.zig | 553 +++++++++++++++++++---------------------------------- 1 file changed, 198 insertions(+), 355 deletions(-) diff --git a/zml/io.zig b/zml/io.zig index e524f78a..9edfa958 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -6,6 +6,7 @@ pub const VFS = @import("io").VFS; const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("zml.zig").Bufferized; +const mem = @import("mem.zig"); const meta = @import("meta.zig"); const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; @@ -190,293 +191,123 @@ pub const TensorStore = struct { }; }; -pub fn TensorBufferTransfer(comptime UserData: type) type { - return struct { - io: std.Io, - platform: Platform, - tensor: safetensors.Tensor, - buffer: *Buffer, - cb_ctx: UserData, - shape: Shape, - }; -} - -pub fn CallbackTensorBufferTransfer(comptime UserData: type) type { - return fn (t: TensorBufferTransfer(UserData)) std.Io.Cancelable!void; -} - -pub fn BufferizeContext(comptime UserData: type) type { - return struct { - allocator: std.mem.Allocator, - arena: *std.heap.ArenaAllocator, - io: std.Io, - platform: Platform, - cb_ctx: UserData, - }; -} - -pub fn bufferize( - comptime UserData: type, - bufferize_ctx: BufferizeContext(UserData), - model: anytype, - bufferized: *Bufferized(@TypeOf(model.*)), - store: TensorStore.View, -) ![]TensorBufferTransfer(UserData) { - comptime { - switch (@typeInfo(@TypeOf(model))) { - .pointer => {}, - else => @compileError("zml.io.bufferize expects a pointer to the model (pass &model or &self.field)"), - } - } - - var tensor_count: usize = 0; - meta.visit(struct { - fn cb(ctx: *usize, _: *const Tensor) void { - ctx.* += 1; - } - }.cb, &tensor_count, model); - - initBufferizedFrom(model.*, bufferized); - - const arena_allocator = bufferize_ctx.arena.allocator(); - - const transfers = try arena_allocator.alloc(TensorBufferTransfer(UserData), tensor_count); - const buffers = try bufferize_ctx.allocator.alloc(*Buffer, tensor_count); - defer bufferize_ctx.allocator.free(buffers); - - const VisitBuffersCtx = struct { - index: usize = 0, - buffers: []*Buffer, - }; - - var visit_buffers_ctx: VisitBuffersCtx = .{ .buffers = buffers }; - meta.visit(struct { - fn cb(ctx: *VisitBuffersCtx, buffer: *Buffer) void { - ctx.buffers[ctx.index] = buffer; - ctx.index += 1; - } - }.cb, &visit_buffers_ctx, bufferized); - - const FillBuffersCtx = struct { - bufferize_ctx: BufferizeContext(UserData), - store: TensorStore.View, - buffers: []*Buffer, - cb_ctx: UserData, - transfers: []TensorBufferTransfer(UserData), - idx: usize = 0, - }; - var fill_buffers_ctx: FillBuffersCtx = .{ - .bufferize_ctx = bufferize_ctx, - .store = store, - .buffers = buffers, - .cb_ctx = bufferize_ctx.cb_ctx, - .transfers = transfers, - }; - - meta.visit(struct { - fn cb(ctx: *FillBuffersCtx, tensor: *const Tensor) void { - const ptr = ctx.store.store.getPtrFromId(tensor.id) orelse return; - const buffer = ctx.buffers[ctx.idx]; - - ctx.transfers[ctx.idx] = .{ - .io = ctx.bufferize_ctx.io, - .platform = ctx.bufferize_ctx.platform, - .tensor = ptr.*, - .buffer = buffer, - .cb_ctx = ctx.cb_ctx, - .shape = tensor.shape(), - }; - ctx.idx += 1; - } - }.cb, &fill_buffers_ctx, model); - - return transfers; -} - -pub const Writer = union(enum) { device: DeviceWriter, buffered_device: BufferedDeviceWriter }; - -pub fn loadBuffersFromId( - allocator: std.mem.Allocator, - io: std.Io, - platform: Platform, - model: anytype, - store: TensorStore.View, - read_pool_config: ConcurrentBufferPool.Config, - write_pool_config: ConcurrentBufferPool.Config, -) !Bufferized(@TypeOf(model)) { - var bufferized: Bufferized(@TypeOf(model)) = undefined; - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - - var read_pool, var write_pool = try ConcurrentBufferPool.initRW( - allocator, - platform, - read_pool_config, - write_pool_config, - ); - defer { - read_pool.deinit(); - write_pool.deinit(); - } - - const LoadCtx = struct { allocator: std.mem.Allocator, read_pool: *ConcurrentBufferPool, write_pool: *ConcurrentBufferPool }; - - const bufferize_ctx: BufferizeContext(LoadCtx) = .{ - .allocator = allocator, - .arena = &arena, - .io = io, - .platform = platform, - .cb_ctx = .{ .allocator = allocator, .read_pool = &read_pool, .write_pool = &write_pool }, - }; - - const transfers = try bufferize(LoadCtx, bufferize_ctx, &model, &bufferized, store); - - var group = try stdx.Io.AllocatingLimitedConcurrentGroup.init(allocator, read_pool.config.concurrency); - defer { - group.cancel(io); - group.deinit(); - } - - for (transfers) |t| { - try group.concurrent(io, struct { - fn run(ctx: TensorBufferTransfer(LoadCtx)) !void { - const allocator_ = ctx.cb_ctx.allocator; - const read_pool_ = ctx.cb_ctx.read_pool; - const write_pool_ = ctx.cb_ctx.write_pool; - - const read_buffer = read_pool_.acquire(ctx.io) catch unreachable; - defer read_pool_.release(ctx.io, read_buffer) catch {}; - - const write_buffer = write_pool_.acquire(ctx.io) catch unreachable; - defer write_pool_.release(ctx.io, write_buffer) catch {}; - - var reader = safetensors.TensorReader.init(ctx.io, ctx.tensor, read_buffer) catch unreachable; - defer reader.deinit(); - - var writer: Writer = if (ctx.platform.target == .neuron) blk: { - break :blk .{ .buffered_device = BufferedDeviceWriter.init(allocator_, ctx.io, null, ctx.platform, ctx.buffer.shape(), ctx.buffer) catch unreachable }; - } else blk: { - break :blk .{ .device = DeviceWriter.init(ctx.io, null, ctx.platform, ctx.shape, ctx.buffer, .device, write_buffer) catch unreachable }; - }; - - switch (writer) { - .device => |*d| { - defer { - d.interface.flush() catch unreachable; - d.deinit(); - } - _ = reader.interface.streamRemaining(&d.interface) catch unreachable; - }, - .buffered_device => |*d| { - defer { - d.interface.flush() catch unreachable; - d.deinit(); - } - _ = reader.interface.streamRemaining(&d.interface) catch unreachable; - }, - } - } - }.run, .{t}); - } - - try group.await(io); - - return bufferized; -} - -pub const DeviceWriter = struct { - io: std.Io, - progress: ?*std.Progress.Node, - platform: Platform, - shape: Shape, - buffer: *Buffer, - transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, - offset: usize = 0, +pub const ProgressWriter = struct { + inner: *std.Io.Writer, + progress: *std.Progress.Node, interface: std.Io.Writer, + total: usize = 0, + scale: usize, - pub fn init(io: std.Io, progress: ?*std.Progress.Node, platform: Platform, shape: Shape, buffer: *Buffer, memory: Buffer.Memory, buf: []u8) !DeviceWriter { - const memories = platform.getDevices()[0].addressableMemories(platform.pjrt_api); - - const mem = blk: { - for (memories) |mem_| { - if (mem_.kind(platform.pjrt_api) == memory) { - break :blk mem_; - } - } - return error.MemoryNotFound; - }; - - const shape_spec: pjrt.ShapeSpec = .init(shape.dims(), pjrtx.bufferTypeFromDtype(shape.dtype())); - const transfer_manager = try platform.pjrt_client.createBuffersForAsyncHostToDevice(platform.pjrt_api, .{ - .shape_specs = &.{shape_spec}, - .memory = mem, - }); - - const pjrt_buffer = transfer_manager.retrieveBuffer(platform.pjrt_api, 0) catch unreachable; - buffer.* = Buffer.fromPjrtBuffers(platform, shape, &.{pjrt_buffer}); + pub const InitOpts = struct { + scale: usize = 1, + }; + pub fn init(inner_: *std.Io.Writer, progress_: *std.Progress.Node, opts: InitOpts) ProgressWriter { return .{ - .io = io, - .progress = progress, - .platform = platform, - .shape = shape, - .buffer = buffer, - .transfer_manager = transfer_manager, + .inner = inner_, + .progress = progress_, + .scale = opts.scale, .interface = .{ - .buffer = buf, - .end = 0, + .buffer = inner_.buffer, + .end = inner_.end, .vtable = &.{ .drain = drain, + .flush = flush, + .rebase = rebase, + .sendFile = sendFile, }, }, }; } - pub fn deinit(self: *const DeviceWriter) void { - self.transfer_manager.deinit(self.platform.pjrt_api); + pub fn pre(self: *ProgressWriter) usize { + self.inner.buffer = self.interface.buffer; + self.inner.end = self.interface.end; + return self.inner.end; + } + + pub fn post(self: *ProgressWriter, len_pre: usize, total: usize) void { + self.interface.buffer = self.inner.buffer; + self.interface.end = self.inner.end; + self.total += (len_pre - self.interface.end) + total; + self.progress.setCompletedItems(self.total / self.scale); } pub fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { - _ = data; - _ = splat; - const self: *DeviceWriter = @alignCast(@fieldParentPtr("interface", w)); + const self: *ProgressWriter = @alignCast(@fieldParentPtr("interface", w)); + const len_pre = self.pre(); + errdefer self.post(len_pre, 0); + const total = try self.inner.vtable.drain(self.inner, data, splat); + self.post(len_pre, total); + return total; + } - stdx.debug.assert(self.offset + w.end <= self.shape.byteSize(), "Can't write more data than required", .{}); + pub fn sendFile(w: *std.Io.Writer, file_reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.Writer.FileError!usize { + const self: *ProgressWriter = @alignCast(@fieldParentPtr("interface", w)); + const len_pre = self.pre(); + errdefer self.post(len_pre, 0); + const total = try self.inner.vtable.sendFile(self.inner, file_reader, limit); + self.post(len_pre, total); + return total; + } - const is_last_transfer = self.offset + w.end >= self.shape.byteSize(); - const event = self.transfer_manager.transferData(self.platform.pjrt_api, 0, w.buffer[0..w.end], @intCast(self.offset), is_last_transfer) catch return error.WriteFailed; - event.await(self.platform.pjrt_api, self.io) catch return error.WriteFailed; + pub fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { + const self: *ProgressWriter = @alignCast(@fieldParentPtr("interface", w)); + const len_pre = self.pre(); + defer self.post(len_pre, 0); + try self.inner.vtable.flush(self.inner); + } - self.offset += w.end; - if (self.progress) |p| p.setCompletedItems(self.offset); - w.end = 0; + pub fn rebase(w: *std.Io.Writer, preserve: usize, capacity: usize) std.Io.Writer.Error!void { + const self: *ProgressWriter = @alignCast(@fieldParentPtr("interface", w)); + const len_pre = self.pre(); + defer self.post(len_pre, 0); + try self.inner.vtable.rebase(self.inner, preserve, capacity); + } +}; - return 0; +pub const DeviceWriter = union(enum) { + direct: DirectDeviceWriter, + buffered: BufferedDeviceWriter, + + pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, pool: *mem.DynamicBufferPool, shape: Shape, buffer: *Buffer, memory_kind: Buffer.Memory) !DeviceWriter { + return switch (platform.target) { + .cuda, .rocm => .{ .direct = try .init(allocator, io, platform, pool, shape, buffer, memory_kind) }, + .tpu, .neuron => .{ .buffered = try .init(allocator, io, platform, shape, buffer) }, + else => unreachable, + }; + } + + pub fn interface(self: *DeviceWriter) *std.Io.Writer { + return switch (self.*) { + .direct => &self.direct.interface, + .buffered => &self.buffered.interface, + }; + } + + pub fn deinit(self: *DeviceWriter, allocator: std.mem.Allocator) void { + switch (self.*) { + .direct => self.direct.deinit(), + .buffered => self.buffered.deinit(allocator), + } } }; pub const BufferedDeviceWriter = struct { - allocator: std.mem.Allocator, io: std.Io, - progress: ?*std.Progress.Node, - transfer_buffer: []u8, - platform: Platform, + platform: *const Platform, shape: Shape, buffer: *Buffer, interface: std.Io.Writer, - pub fn init(allocator: std.mem.Allocator, io: std.Io, progress: ?*std.Progress.Node, platform: Platform, shape: Shape, buffer: *Buffer) !BufferedDeviceWriter { - const transfer_buffer = try allocator.alloc(u8, shape.byteSize()); + pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, shape: Shape, buffer: *Buffer) !BufferedDeviceWriter { return .{ - .allocator = allocator, .io = io, - .progress = progress, - .transfer_buffer = transfer_buffer, .shape = shape, .platform = platform, .buffer = buffer, .interface = .{ - .buffer = transfer_buffer, + .buffer = try allocator.alloc(u8, shape.byteSize()), .vtable = &.{ .drain = std.Io.Writer.fixedDrain, .flush = flush, @@ -486,136 +317,148 @@ pub const BufferedDeviceWriter = struct { }; } - pub fn deinit(self: *BufferedDeviceWriter) void { - self.allocator.free(self.transfer_buffer); + pub fn deinit(self: *BufferedDeviceWriter, allocator: std.mem.Allocator) void { + if (self.interface.buffer.len > 0) { + allocator.free(self.interface.buffer); + } } pub fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { const self: *BufferedDeviceWriter = @alignCast(@fieldParentPtr("interface", w)); - self.buffer.* = Buffer.fromBytes(self.io, self.platform, self.shape, self.transfer_buffer) catch return std.Io.Writer.Error.WriteFailed; - if (self.progress) |p| p.setCompletedItems(self.shape.byteSize()); + self.buffer.* = Buffer.fromBytes(self.io, self.platform, self.shape, self.interface.buffer) catch return std.Io.Writer.Error.WriteFailed; _ = self.buffer.await(self.io) catch return std.Io.Writer.Error.WriteFailed; } }; -pub fn initBufferizedFrom(model: anytype, bufferized_: *Bufferized(@TypeOf(model))) void { - const Model = @TypeOf(model); - const type_info = @typeInfo(Bufferized(Model)); - switch (type_info) { - .@"struct" => |struct_type_info| { - if (Bufferized(Model) == Buffer) return; - inline for (struct_type_info.fields) |field| { - initBufferizedFrom(@field(model, field.name), &@field(bufferized_, field.name)); - } - }, - .@"union" => { - switch (model) { - inline else => |v, tag| { - bufferized_.* = @unionInit(Bufferized(Model), @tagName(tag), undefined); - initBufferizedFrom(v, @field(bufferized_, @tagName(tag))); - }, - } - }, - .optional => |optional_type_info| { - if (model == null) { - bufferized_.* = null; - } else { - bufferized_.* = @as(optional_type_info.child, undefined); - initBufferizedFrom(model.?, &bufferized_.*.?); - } - }, - .void, .int, .@"enum", .bool, .enum_literal, .float, .pointer, .vector => {}, - else => unreachable, - } -} - -pub const ConcurrentBufferPool = struct { - pub const Error = error{ OutOfMemory, Notfound, NotAcquired }; - - const alignment: std.mem.Alignment = .fromByteUnits(4 * 1024); - - allocator: std.mem.Allocator, - mutex: std.Io.Mutex = .init, - - platform: Platform, - locked: []bool, - buffers: []u8, - config: Config, - - pub const Config = struct { - size: usize, - concurrency: usize, - dma: bool = true, +pub const DirectDeviceWriter = struct { + const EventContext = struct { + self: *DirectDeviceWriter, + err: ?*pjrt.Error = null, + pjrt_event: *pjrt.Event, + event: std.Io.Event = .unset, + buffer: []u8, }; - pub fn init( - allocator: std.mem.Allocator, - platform: Platform, - config: Config, - ) !ConcurrentBufferPool { - const buffers = try allocator.alignedAlloc(u8, alignment, config.concurrency * config.size); - errdefer allocator.free(buffers); + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + pool: *mem.DynamicBufferPool, + total: usize, + buffer: *Buffer, - if (config.dma and platform.target != .cpu) { - try platform.pjrt_client.dmaMap(platform.pjrt_api, buffers); - } + transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, + offset: usize = 0, - const locked = try allocator.alloc(bool, config.concurrency); - @memset(locked, false); + interface: std.Io.Writer, + flip_flop: u1 = 0, + events_contexts: [2]?EventContext = @splat(null), + + pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, pool: *mem.DynamicBufferPool, shape: Shape, buffer: *Buffer, memory_kind: Buffer.Memory) !DirectDeviceWriter { + const shape_spec: pjrt.ShapeSpec = .init(shape.dims(), pjrtx.bufferTypeFromDtype(shape.dtype())); + const transfer_manager = try platform.pjrt_client.createBuffersForAsyncHostToDevice(platform.pjrt_api, .{ + .shape_specs = &.{shape_spec}, + .memory = blk: { + const devices = platform.getDevices(); + for (devices[0].addressableMemories(platform.pjrt_api)) |m| { + if (m.kind(platform.pjrt_api) == memory_kind) break :blk m; + } + return error.MemoryNotFound; + }, + }); + + const pjrt_buffer = transfer_manager.retrieveBuffer(platform.pjrt_api, 0) catch unreachable; + buffer.* = Buffer.fromPjrtBuffers(platform, shape, &.{pjrt_buffer}); return .{ .allocator = allocator, - .mutex = .init, + .io = io, .platform = platform, - .locked = locked, - .buffers = buffers, - .config = config, + .pool = pool, + .total = shape.byteSize(), + .buffer = buffer, + .transfer_manager = transfer_manager, + .interface = .{ + .buffer = try pool.get(allocator, io), + .end = 0, + .vtable = &.{ + .drain = drain, + }, + }, }; } - pub fn initRW(allocator: std.mem.Allocator, platform: Platform, read_config: Config, write_config: Config) !struct { ConcurrentBufferPool, ConcurrentBufferPool } { - return .{ try .init(allocator, platform, read_config), try .init(allocator, platform, write_config) }; + pub fn deinit(self: *DirectDeviceWriter) void { + self.transfer_manager.deinit(self.platform.pjrt_api); } - pub fn deinit(self: *ConcurrentBufferPool) void { - if (self.config.dma and self.platform.target != .cpu) { - self.platform.pjrt_client.dmaUnmap(self.platform.pjrt_api, self.buffers) catch {}; + fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + _ = data; // autofix + _ = splat; // autofix + const self: *DirectDeviceWriter = @alignCast(@fieldParentPtr("interface", w)); + if (self.offset >= self.total) { + return std.Io.Writer.Error.WriteFailed; } - self.allocator.free(self.buffers); - self.allocator.free(self.locked); - } + const current_buffer = self.interface.buffer; + const buffered = self.interface.buffered(); + const sliced_buffer = self.interface.buffered()[0..@min(buffered.len, self.total - self.offset)]; + const written = sliced_buffer.len; + const is_last = (self.offset + sliced_buffer.len) >= self.total; - pub fn acquire(self: *ConcurrentBufferPool, io: std.Io) Error![]u8 { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); + const transfer_event = self.transfer_manager.transferData( + self.platform.pjrt_api, + 0, + sliced_buffer, + @intCast(self.offset), + is_last, + ) catch |err| { + log.err("error when transferring data to device: {any}", .{err}); + return std.Io.Writer.Error.WriteFailed; + }; + const ctx = &self.events_contexts[@intCast(self.flip_flop)]; + ctx.* = .{ + .self = self, + .buffer = current_buffer, + .pjrt_event = transfer_event, + }; + transfer_event.onReady(self.platform.pjrt_api, EventContext, struct { + fn call(err: ?*pjrt.Error, ctx_: *EventContext) void { + ctx_.self.pool.put(ctx_.self.io, ctx_.buffer); + ctx_.err = err; + ctx_.event.set(ctx_.self.io); + } + }.call, &(ctx.*.?)) catch |err| { + log.err("error when setting up transfer completion callback: {any}", .{err}); + return std.Io.Writer.Error.WriteFailed; + }; - for (self.locked, 0..) |locked, i| { - if (!locked) { - self.locked[i] = true; - return self.buffers[i * self.config.size .. (i + 1) * self.config.size]; + if (self.events_contexts[@intCast(self.flip_flop ^ 1)]) |*ctx_previous| { + defer self.events_contexts[@intCast(self.flip_flop ^ 1)] = null; + // std.log.info("[{*} WAITING", .{self}); + ctx_previous.event.waitUncancelable(self.io); + // std.log.info("[{*} WAITED", .{self}); + defer ctx_previous.pjrt_event.deinit(self.platform.pjrt_api); + if (ctx_previous.err) |e| { + defer e.deinit(self.platform.pjrt_api); + log.err("error while awaiting: {s}: {s}", .{ + @tagName(e.getCode(self.platform.pjrt_api)), + e.getMessage(self.platform.pjrt_api), + }); + return std.Io.Writer.Error.WriteFailed; } } - return Error.OutOfMemory; - } - - pub fn release(self: *ConcurrentBufferPool, io: std.Io, buffer: []u8) Error!void { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - - for (self.locked, 0..) |locked, i| { - const slice = self.buffers[i * self.config.size .. (i + 1) * self.config.size]; - if (slice.ptr == buffer.ptr) { - if (locked) { - self.locked[i] = false; - return; - } else { - return Error.NotAcquired; - } - } + self.flip_flop ^= 1; + self.offset += written; + if (is_last) { + self.interface = .failing; + } else { + self.interface.end = 0; + self.interface.buffer = self.pool.get(self.allocator, self.io) catch |err| { + log.err("unable to get a new buffer from the pool: {any}", .{err}); + return std.Io.Writer.Error.WriteFailed; + }; } - - return Error.Notfound; + return 0; } }; From 0f460fb6361b88407fba51e17d59bf1184c43960 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 3 Feb 2026 10:59:39 +0000 Subject: [PATCH 264/881] zml: expose Memory, pass Platform as *const, add zml.io.load --- MODULE.bazel | 4 +- MODULE.bazel.lock | 10 +- examples/llama/BUILD.bazel | 9 ++ examples/llama/llama.zig | 59 +++++---- examples/llama/main.zig | 154 ++++++++++++------------ examples/mnist/mnist.zig | 79 ++++++------ pjrt/pjrt.zig | 25 +++- zml/attention/attention.zig | 4 +- zml/attention/flashattn.zig | 4 +- zml/buffer.zig | 64 ++++------ zml/io.zig | 205 ++++++++++++++++++++++++------- zml/mem.zig | 57 ++++----- zml/meta.zig | 17 +++ zml/module.zig | 18 ++- zml/platform.zig | 233 ++++++++++++++++++++++++------------ zml/safetensors.zig | 11 +- zml/tensor.zig | 11 +- zml/zml.zig | 15 +-- 18 files changed, 605 insertions(+), 374 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index b2be2a3f..309bb188 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -15,7 +15,7 @@ bazel_dep(name = "protobuf", version = "33.1", repo_name = "com_google_protobuf" # Needs to be added before rules_cc so that the cc toolchain declared by # apple_support wins over the one in rules_cc. -bazel_dep(name = "rules_cc", version = "0.2.12") +bazel_dep(name = "rules_cc", version = "0.2.16") bazel_dep(name = "rules_distroless", version = "0.5.1") bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep(name = "rules_oci", version = "2.2.6") @@ -233,7 +233,7 @@ git_override( git_override( module_name = "toolchains_llvm_bootstrapped", - commit = "4902b6cadc4ca840645c97d52f985c6f67967b6f", + commit = "023f62a94eb38d2991432a766a1d4c05e90414d7", remote = "https://github.com/cerisier/toolchains_llvm_bootstrapped.git", ) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index b0b9673b..764564fa 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -72,7 +72,7 @@ "https://bazel-registry.zml.ai/modules/libxev/0.0.0-20251014.0-9f785d2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/nlohmann_json/3.6.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/package_metadata/0.0.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/patchelf/0.18.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/patchelf/0.18.0.bcr.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/pcre2/10.45/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/platforms/0.0.10/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/platforms/0.0.11/MODULE.bazel": "not found", @@ -113,8 +113,6 @@ "https://bazel-registry.zml.ai/modules/rules_cc/0.1.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.1.5/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.2.12/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.2.14/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.16/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.2.4/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_distroless/0.5.1/MODULE.bazel": "not found", @@ -283,8 +281,8 @@ "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/source.json": "f448c6e8963fdfa7eb831457df83ad63d3d6355018f6574fb017e8169deb43a9", "https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92", "https://bcr.bazel.build/modules/package_metadata/0.0.2/source.json": "e53a759a72488d2c0576f57491ef2da0cf4aab05ac0997314012495935531b73", - "https://bcr.bazel.build/modules/patchelf/0.18.0/MODULE.bazel": "15a6beff7e828d585c5bd0f9f93589df117b5594e9d19e43096c77de58b9ae5f", - "https://bcr.bazel.build/modules/patchelf/0.18.0/source.json": "57caf6bcaa5ba515c6fb1c2eacee00735afbeb1ffacb34a57553fb139c8e4333", + "https://bcr.bazel.build/modules/patchelf/0.18.0.bcr.1/MODULE.bazel": "35320668481400503dea7c75313473b9ffd6dea4e7ba583b96a896199b0088d5", + "https://bcr.bazel.build/modules/patchelf/0.18.0.bcr.1/source.json": "d877d271ef5d531407885322054c99c7f0617a9a85ff99c0a70f4ced7f13e13a", "https://bcr.bazel.build/modules/pcre2/10.45/MODULE.bazel": "9bdc838aa2e21be030d7181385e45a5c87b14575bb7ff35d10b5be3d56913bed", "https://bcr.bazel.build/modules/pcre2/10.45/source.json": "c0b10fd1afbfc978aad9a83e84533880b6c533026fa930d73bb95df34291bc4b", "https://bcr.bazel.build/modules/platforms/0.0.10/MODULE.bazel": "8cb8efaf200bdeb2150d93e162c40f388529a25852b332cec879373771e48ed5", @@ -332,8 +330,6 @@ "https://bcr.bazel.build/modules/rules_cc/0.1.1/MODULE.bazel": "2f0222a6f229f0bf44cd711dc13c858dad98c62d52bd51d8fc3a764a83125513", "https://bcr.bazel.build/modules/rules_cc/0.1.5/MODULE.bazel": "88dfc9361e8b5ae1008ac38f7cdfd45ad738e4fa676a3ad67d19204f045a1fd8", "https://bcr.bazel.build/modules/rules_cc/0.2.0/MODULE.bazel": "b5c17f90458caae90d2ccd114c81970062946f49f355610ed89bebf954f5783c", - "https://bcr.bazel.build/modules/rules_cc/0.2.12/MODULE.bazel": "4216c383ce3223c7dfbd3afcb09c7056265156150b2a3bd3fd9d02949a7ee1bc", - "https://bcr.bazel.build/modules/rules_cc/0.2.14/MODULE.bazel": "353c99ed148887ee89c54a17d4100ae7e7e436593d104b668476019023b58df8", "https://bcr.bazel.build/modules/rules_cc/0.2.16/MODULE.bazel": "9242fa89f950c6ef7702801ab53922e99c69b02310c39fb6e62b2bd30df2a1d4", "https://bcr.bazel.build/modules/rules_cc/0.2.16/source.json": "d03d5cde49376d87e14ec14b666c56075e5e3926930327fd5d0484a1ff2ac1cc", "https://bcr.bazel.build/modules/rules_cc/0.2.4/MODULE.bazel": "1ff1223dfd24f3ecf8f028446d4a27608aa43c3f41e346d22838a4223980b8cc", diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel index 39d3cd21..c8fce3bd 100644 --- a/examples/llama/BUILD.bazel +++ b/examples/llama/BUILD.bazel @@ -12,6 +12,15 @@ zig_binary( deps = ["//zml"], ) +zig_binary( + name = "loader", + srcs = ["llama.zig"], + main = "main2.zig", + visibility = ["//visibility:public"], + deps = ["//zml"], +) + + mtree_spec( name = "mtree", srcs = [":llama"], diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 45c580fb..cdc0f1be 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -10,8 +10,8 @@ const stdx = zml.stdx; const log = std.log.scoped(.llama); pub const TransferCtx = struct { - read_pool: *zml.io.ConcurrentBufferPool, - write_pool: *zml.io.ConcurrentBufferPool, + allocator: std.mem.Allocator, + pool: *zml.mem.DynamicBufferPool, transferred_bytes: *usize, progress: *std.Progress.Node, }; @@ -68,24 +68,33 @@ pub const LlamaLM = struct { self.model.deinit(allocator); } - pub fn loadBuffers( + pub fn load( self: *const LlamaLM, - bufferize_ctx: zml.io.BufferizeContext(TransferCtx), - group: *zml.stdx.Io.AllocatingLimitedConcurrentGroup, - store: zml.io.TensorStore.View, - cb: zml.io.CallbackTensorBufferTransfer(TransferCtx), + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, ) !zml.Bufferized(LlamaLM) { - var lm_head_bufferized: ?zml.Bufferized(zml.nn.Linear) = null; - if (self.lm_head) |_| { - const transfers = try zml.io.bufferize(TransferCtx, bufferize_ctx, &self.lm_head, &lm_head_bufferized, store.withPrefix("lm_head")); - for (transfers) |t| try group.concurrent(bufferize_ctx.io, cb, .{t}); + progress.increaseEstimatedTotalItems(store.view().count()); + var timer: std.time.Timer = try .start(); + var total_bytes: usize = 0; + defer { + const took = timer.read(); + log.info("Loaded weights [{Bi:.2}, {D}, {Bi:.2}/s]", .{ + total_bytes, + took, + total_bytes / took * std.time.ns_per_s, + }); } - - const model_bufferized = try self.model.loadBuffers(bufferize_ctx, group, store.withPrefix("model"), cb); - return .{ - .lm_head = lm_head_bufferized, - .model = model_bufferized, - }; + return zml.io.load(LlamaLM, self, allocator, io, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 128 * zml.MiB, + .progress = progress, + .store = store, + .parallelism = 16, + .total_bytes = &total_bytes, + }); } pub fn unloadBuffers(self: *zml.Bufferized(LlamaLM), allocator: std.mem.Allocator) void { @@ -174,7 +183,7 @@ pub const Llama = struct { pub fn loadBuffers( self: *const Llama, bufferize_ctx: zml.io.BufferizeContext(TransferCtx), - group: *zml.stdx.Io.AllocatingLimitedConcurrentGroup, + group: *stdx.Io.LimitedGroup, store: zml.io.TensorStore.View, cb: zml.io.CallbackTensorBufferTransfer(TransferCtx), ) !zml.Bufferized(Llama) { @@ -183,14 +192,20 @@ pub const Llama = struct { var bufferized: zml.Bufferized(Llama) = .{ .embed_tokens = undefined, .norm = undefined, .layers = bufferized_layers }; const transfers = try zml.io.bufferize(TransferCtx, bufferize_ctx, &self.embed_tokens, &bufferized.embed_tokens, store.withPrefix("embed_tokens")); - for (transfers) |t| try group.concurrent(bufferize_ctx.io, cb, .{t}); + for (transfers) |t| { + try group.concurrent(bufferize_ctx.io, cb, .{t}); + } const norm_transfers = try zml.io.bufferize(TransferCtx, bufferize_ctx, &self.norm, &bufferized.norm, store.withPrefix("norm")); - for (norm_transfers) |t| try group.concurrent(bufferize_ctx.io, cb, .{t}); + for (norm_transfers) |t| { + try group.concurrent(bufferize_ctx.io, cb, .{t}); + } for (self.layers, bufferized_layers, 0..) |layer, *buf_layer, i| { const layer_transfers = try zml.io.bufferize(TransferCtx, bufferize_ctx, &layer, buf_layer, store.withPrefix("layers").withLayer(i)); - for (layer_transfers) |t| try group.concurrent(bufferize_ctx.io, cb, .{t}); + for (layer_transfers) |t| { + try group.concurrent(bufferize_ctx.io, cb, .{t}); + } } return bufferized; @@ -449,7 +464,7 @@ pub const KvCache = struct { }; } - pub fn initBuffer(self: KvCache, io: std.Io, platform: zml.Platform) !zml.Bufferized(KvCache) { + pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { return .{ .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}), diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 97b72f8c..09d1f861 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const log = std.log; const zml = @import("zml"); const Buffer = zml.Buffer; @@ -13,8 +14,6 @@ const KvCache = llama.KvCache; const TransformerLayer = llama.TransformerLayer; const SelfAttn = llama.SelfAttn; -const log = std.log.scoped(.llama); - pub const std_options: std.Options = .{ .log_level = .info, }; @@ -54,29 +53,25 @@ const CliArgs = struct { pub fn main() !void { log.info("LLama was compiled with {}", .{@import("builtin").mode}); - var debug_allocator: ?std.heap.DebugAllocator(.{}) = null; - const allocator = if (@import("builtin").mode == .Debug) blk: { - debug_allocator = .init; - break :blk debug_allocator.?.allocator(); - } else std.heap.c_allocator; - defer if (debug_allocator) |*da| std.debug.assert(da.deinit() == .ok); + // var debug_allocator: ?std.heap.DebugAllocator(.{}) = null; + // const allocator = if (@import("builtin").mode == .Debug) blk: { + // debug_allocator = .init; + // break :blk debug_allocator.?.allocator(); + // } else std.heap.c_allocator; + // defer if (debug_allocator) |*da| std.debug.assert(da.deinit() == .ok); + const allocator = std.heap.c_allocator; var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); const args = stdx.flags.parseProcessArgs(CliArgs); - if (args.async_limit) |limit| threaded.setAsyncLimit(.limited(limit)); - - log.info("Running with threaded io async limit set to {?d}", .{threaded.async_limit.toInt()}); - var vfs: zml.io.VFS = try .init(allocator, threaded.io()); defer vfs.deinit(); var http_client: std.http.Client = .{ .allocator = allocator, .io = threaded.io(), - .connection_pool = .{ .free_size = threaded.async_limit.toInt() orelse 16 }, }; try http_client.initDefaultProxies(allocator); @@ -118,16 +113,16 @@ pub fn main() !void { }, }; - var platform: zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(); - var it = platform.devicesIterator(); - log.info("Devices:", .{}); - while (it.next()) |device| { - log.info("\t- {f}", .{device}); - } + var platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator); + log.info("\n{f}", .{platform.fmtVerbose()}); + // log.info("Devices:", .{}); + // for (platform.devices) |device| { + // log.info("\t- {f}", .{device}); + // } // Initialize the Llama struct and map the content of the .safetensors to the model tensors. - const llama_model: llama.LlamaLM = try .init(allocator, store.view(), config, llama_options); + var llama_model: llama.LlamaLM = try .init(allocator, store.view(), config, llama_options); defer llama_model.deinit(allocator); const backend = args.backend orelse b: { @@ -153,33 +148,35 @@ pub fn main() !void { .attention_parameters = .init(.fromBackend(backend)), }; - var progress = std.Progress.start(io, .{ .root_name = args.model, .estimated_total_items = store.view().count() }); + var progress = std.Progress.start(io, .{ .root_name = args.model }); - var tokenizer_future = io.async(loadTokenizer, .{ allocator, io, repo, &progress }); + var tokenizer_future = try io.concurrent(loadTokenizer, .{ allocator, io, repo, &progress }); errdefer blk: { var v = tokenizer_future.cancel(io) catch break :blk; v.deinit(); } - var compiled_model_result_future = io.async(compileModel, .{ allocator, io, platform, llama_model, llama_parameters, &progress }); + var compiled_model_result_future = try io.concurrent(compileModel, .{ allocator, io, platform, llama_model, llama_parameters, &progress }); errdefer if (compiled_model_result_future.cancel(io)) |v| { defer v.prefill_exe.deinit(); defer v.decode_exe.deinit(); } else |_| {}; - var load_model_buffers_group: zml.stdx.Io.AllocatingLimitedConcurrentGroup = try .init(allocator, threaded.async_limit.toInt() orelse 32); - defer { - load_model_buffers_group.cancel(io); - load_model_buffers_group.deinit(); - } + var load_group: stdx.Io.LimitedGroup = .init(16); + defer load_group.cancel(io); - const read_pool_config: zml.io.ConcurrentBufferPool.Config = .{ .size = args.reader_buffer_size.bytes(), .concurrency = load_model_buffers_group.limit, .dma = true }; - const write_pool_config: zml.io.ConcurrentBufferPool.Config = .{ .size = args.writer_buffer_size.bytes(), .concurrency = load_model_buffers_group.limit, .dma = true }; - - var llama_buffers_future = io.async(loadModelBuffers, .{ allocator, io, &progress, &load_model_buffers_group, read_pool_config, write_pool_config, platform, &store, llama_model }); + var llama_buffers_future = try io.concurrent(LlamaLM.load, .{ + &llama_model, + allocator, + io, + platform, + &store, + &progress, + }); errdefer b: { var v = llama_buffers_future.cancel(io) catch break :b; - LlamaLM.unloadBuffers(&v, allocator); + v = v; // autofix + // LlamaLM.unloadBuffers(&v, allocator); } const compiled_model_result = try compiled_model_result_future.await(io); @@ -187,7 +184,8 @@ pub fn main() !void { defer compiled_model_result.decode_exe.deinit(); var llama_buffers = try llama_buffers_future.await(io); - defer LlamaLM.unloadBuffers(&llama_buffers, allocator); + llama_buffers = llama_buffers; // autofix + // defer LlamaLM.unloadBuffers(&llama_buffers, allocator); progress.end(); log.info("Creating KvCache", .{}); @@ -205,7 +203,7 @@ pub fn main() !void { break :blk try reader.interface.allocRemaining(allocator, .unlimited); }; - log.info("✅\tPrompt: {s}", .{prompt}); + log.info("✅ Prompt: {s}", .{prompt}); // Unbuffered writing of the tokens to stdout. var stdout = std.Io.File.stdout().writer(io, &.{}); @@ -213,7 +211,7 @@ pub fn main() !void { try generateText( allocator, io, - llama_buffers, + &llama_buffers, compiled_model_result.prefill_exe, compiled_model_result.decode_exe, &kv_cache_buffers, @@ -250,6 +248,7 @@ fn parseConfig(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !std.j fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, progress: *std.Progress.Node) !zml.tokenizer.Tokenizer { // log.info("Loading tokenizer", .{}); + progress.increaseEstimatedTotalItems(1); var node = progress.start("Loading tokenizer...", 1); defer node.end(); var timer = try std.time.Timer.start(); @@ -280,16 +279,21 @@ const CompileModelResult = struct { decode_exe: zml.Exe, }; -fn compileModel(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform, llama_model: llama.LlamaLM, parameters: LlamaParameters, progress: *std.Progress.Node) !CompileModelResult { - var node = progress.start("Compiling...", 2); - defer node.end(); - +fn compileModel( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + llama_model: llama.LlamaLM, + parameters: LlamaParameters, + progress: *std.Progress.Node, +) !CompileModelResult { var timer = try std.time.Timer.start(); defer log.info("Compiled model [{D}]", .{timer.read()}); // Compile the model twice, one for prefill, one for generation. - var prefill_future = io.async(struct { - fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: zml.Platform, llama_model_: llama.LlamaLM, parameters_: LlamaParameters, progress_: *std.Progress.Node) !zml.Exe { + var prefill_future = try io.concurrent(struct { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, llama_model_: llama.LlamaLM, parameters_: LlamaParameters, progress_: *std.Progress.Node) !zml.Exe { + progress_.increaseEstimatedTotalItems(1); var node_ = progress_.start("Compiling prefill...", 1); defer node_.end(); var timer_ = try std.time.Timer.start(); @@ -303,11 +307,12 @@ fn compileModel(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform parameters_.attention_parameters, }); } - }.call, .{ allocator, io, platform, llama_model, parameters, &node }); + }.call, .{ allocator, io, platform, llama_model, parameters, progress }); errdefer if (prefill_future.cancel(io)) |v| v.deinit() else |_| {}; - var decode_future = io.async(struct { - fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: zml.Platform, llama_model_: llama.LlamaLM, parameters_: LlamaParameters, progress_: *std.Progress.Node) !zml.Exe { + var decode_future = try io.concurrent(struct { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, llama_model_: llama.LlamaLM, parameters_: LlamaParameters, progress_: *std.Progress.Node) !zml.Exe { + progress_.increaseEstimatedTotalItems(1); var node_ = progress_.start("Compiling decode...", 1); defer node_.end(); var timer_ = try std.time.Timer.start(); @@ -321,7 +326,7 @@ fn compileModel(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform parameters_.attention_parameters, }); } - }.call, .{ allocator, io, platform, llama_model, parameters, &node }); + }.call, .{ allocator, io, platform, llama_model, parameters, progress }); errdefer if (decode_future.cancel(io)) |v| v.deinit() else |_| {}; const prefill_exe = try prefill_future.await(io); @@ -332,16 +337,17 @@ fn compileModel(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform fn loadModelBuffers( allocator: std.mem.Allocator, + buffers_allocator: std.mem.Allocator, io: std.Io, progress: *std.Progress.Node, - group: *zml.stdx.Io.AllocatingLimitedConcurrentGroup, - read_pool_config: zml.io.ConcurrentBufferPool.Config, - write_pool_config: zml.io.ConcurrentBufferPool.Config, - platform: zml.Platform, + platform: *const zml.Platform, store: *zml.io.TensorStore, llama_model: llama.LlamaLM, + pool: *zml.mem.DynamicBufferPool, + async_limit: usize, ) !zml.Bufferized(llama.LlamaLM) { var transferred_bytes: usize = 0; + var timer = try stdx.time.Timer.start(); defer { const duration = timer.read(); @@ -354,57 +360,45 @@ fn loadModelBuffers( var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); - var read_pool = try zml.io.ConcurrentBufferPool.init(allocator, platform, read_pool_config); - defer read_pool.deinit(); - - var write_pool = try zml.io.ConcurrentBufferPool.init(allocator, platform, write_pool_config); - defer write_pool.deinit(); + var group: stdx.Io.LimitedGroup = .init(async_limit); + defer group.cancel(io); const bufferize_ctx: zml.io.BufferizeContext(llama.TransferCtx) = .{ .allocator = allocator, .arena = &arena, .io = io, .platform = platform, - .cb_ctx = .{ .read_pool = &read_pool, .write_pool = &write_pool, .transferred_bytes = &transferred_bytes, .progress = progress }, + .cb_ctx = .{ + .allocator = buffers_allocator, + .pool = pool, + .transferred_bytes = &transferred_bytes, + .progress = progress, + }, }; - const bufferized = llama_model.loadBuffers( - bufferize_ctx, - group, - store.view(), - transferBuffer, - ); - + const bufferized = llama_model.loadBuffers(bufferize_ctx, &group, store.view(), transferBuffer); try group.await(io); return bufferized; } fn transferBuffer(ctx: zml.io.TensorBufferTransfer(llama.TransferCtx)) !void { - const read_pool = ctx.cb_ctx.read_pool; - const write_pool = ctx.cb_ctx.write_pool; - const progress = ctx.cb_ctx.progress; + var transfer_progress = ctx.cb_ctx.progress.start(ctx.tensor.name, ctx.tensor.byteSize()); - const read_buffer = read_pool.acquire(ctx.io) catch unreachable; - defer read_pool.release(ctx.io, read_buffer) catch {}; - - const write_buffer = write_pool.acquire(ctx.io) catch unreachable; - defer write_pool.release(ctx.io, write_buffer) catch {}; - - var transfer_progress = progress.start(ctx.tensor.name, ctx.tensor.byteSize()); - - var reader = zml.safetensors.TensorReader.init(ctx.io, ctx.tensor, read_buffer) catch unreachable; + var reader = zml.safetensors.TensorReader.init(ctx.io, ctx.tensor, &.{}) catch unreachable; defer reader.deinit(); - var writer = zml.io.DeviceWriter.init(ctx.io, &transfer_progress, ctx.platform, ctx.tensor.shape, ctx.buffer, .device, write_buffer) catch unreachable; + var writer = zml.io.DeviceWriter2.init(ctx.cb_ctx.allocator, ctx.io, ctx.platform, ctx.cb_ctx.pool, ctx.tensor.shape, ctx.buffer, .device) catch unreachable; defer { writer.interface.flush() catch unreachable; writer.deinit(); } _ = reader.interface.streamRemaining(&writer.interface) catch unreachable; + writer.interface.flush() catch unreachable; transfer_progress.end(); + ctx.cb_ctx.transferred_bytes.* += ctx.tensor.byteSize(); } @@ -438,7 +432,7 @@ pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tok pub fn generateText( allocator: std.mem.Allocator, io: std.Io, - llama_buffers: zml.Bufferized(LlamaLM), + llama_buffers: *zml.Bufferized(LlamaLM), prefill_exe: zml.exe.Exe, decode_exe: zml.exe.Exe, kv_cache_buffers: *zml.Bufferized(llama.KvCache), @@ -450,7 +444,7 @@ pub fn generateText( prompt: []const u8, skip_llama3_encoding: bool, writer: *std.Io.Writer, - platform: zml.Platform, + platform: *const zml.Platform, ) !void { const prompt_tok: []const u32 = try tokenizePrompt(allocator, tokenizer, config, prompt, skip_llama3_encoding); defer allocator.free(prompt_tok); @@ -532,7 +526,7 @@ pub fn generateText( // current token pos needs to go into a zml.Buffer const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(prompt_tok.len + i)})); - const token_pos_buffer: zml.Buffer = try .fromSlice(io, platform, token_pos_slice); + var token_pos_buffer: zml.Buffer = try .fromSlice(io, platform, token_pos_slice); defer token_pos_buffer.deinit(); // call to generate the next token diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 1c6178e6..fdaa6900 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -1,10 +1,9 @@ const std = @import("std"); +const log = std.log; const zml = @import("zml"); const stdx = zml.stdx; -const log = std.log.scoped(.mnist); - pub const std_options: std.Options = .{ .log_level = .info, }; @@ -37,16 +36,19 @@ const Mnist = struct { }; } - pub fn loadBuffers( - self: Mnist, + pub fn load( + self: *const Mnist, allocator: std.mem.Allocator, io: std.Io, - platform: zml.Platform, - store: zml.io.TensorStore.View, - read_pool_config: zml.io.ConcurrentBufferPool.Config, - write_pool_config: zml.io.ConcurrentBufferPool.Config, + platform: *const zml.Platform, + store: *const zml.io.TensorStore, ) !zml.Bufferized(Mnist) { - return zml.io.loadBuffersFromId(allocator, io, platform, self, store, read_pool_config, write_pool_config); + return zml.io.load(Mnist, self, allocator, io, platform, .{ + .store = store, + .parallelism = 1, + .dma_chunks = 1, + .dma_chunk_size = 16 * 1024 * 1024, + }); } pub fn unloadBuffers(self: *zml.Bufferized(Mnist)) void { @@ -68,19 +70,12 @@ const Mnist = struct { }; pub fn main() !void { - const allocator = std.heap.c_allocator; + const allocator = std.heap.smp_allocator; var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); - var vfs_file: zml.io.VFS.File = .init(allocator, threaded.io(), .{}); - - var vfs: zml.io.VFS = try .init(allocator, threaded.io()); - defer vfs.deinit(); - - try vfs.register("file", vfs_file.io()); - - const io = vfs.io(); + const io = threaded.io(); // Parse program args const process_args = try std.process.argsAlloc(allocator); @@ -98,14 +93,14 @@ pub fn main() !void { const mnist_model: Mnist = .init(store.view()); // Auto-select platform - const platform: zml.Platform = try .auto(allocator, io, .{}); + const platform: *zml.Platform = try .auto(allocator, io, .{}); // Compile model const input: zml.Tensor = .init(.{ 28, 28 }, .u8); var exe = blk: { log.info("Compiling model....", .{}); var timer = try std.time.Timer.start(); - defer log.info("✅ Compiled model in {D}", .{timer.read()}); + defer log.info("✅ Compiled model [{D}]", .{timer.read()}); break :blk try platform.compile(allocator, io, mnist_model, .forward, .{input}); }; defer exe.deinit(); @@ -114,12 +109,8 @@ pub fn main() !void { var mnist_buffers = blk: { log.info("Transfering weights....", .{}); var timer = try std.time.Timer.start(); - defer log.info("✅ Transferred weights in {D}", .{timer.read()}); - - const read_pool_config: zml.io.ConcurrentBufferPool.Config = .{ .size = 64 * 1024 * 1024, .concurrency = 10, .dma = true }; - const write_pool_config: zml.io.ConcurrentBufferPool.Config = .{ .size = 32 * 1024 * 1024, .concurrency = 10, .dma = true }; - - break :blk try mnist_model.loadBuffers(allocator, io, platform, store.view(), read_pool_config, write_pool_config); + defer log.info("✅ Transferred weights [{D}]", .{timer.read()}); + break :blk try mnist_model.load(allocator, io, platform, &store); }; defer Mnist.unloadBuffers(&mnist_buffers); @@ -139,29 +130,27 @@ pub fn main() !void { }; // inference - can be looped - while (true) { - const idx = rng.random().uintLessThan(u64, 10000); - var sample: [28 * 28]u8 align(16) = undefined; - _ = try dataset.readPositionalAll(io, &sample, 16 + (idx * 28 * 28)); + const idx = rng.random().uintLessThan(u64, 10000); + var sample: [28 * 28]u8 align(16) = undefined; + _ = try dataset.readPositionalAll(io, &sample, 16 + (idx * 28 * 28)); - var input_buffer: zml.Buffer = try .fromSlice(io, platform, zml.Slice.init(input.shape(), &sample)); - defer input_buffer.deinit(); + var input_buffer: zml.Buffer = try .fromSlice(io, platform, zml.Slice.init(input.shape(), &sample)); + defer input_buffer.deinit(); - printDigit(sample); + printDigit(sample); - args.set(.{ mnist_buffers, input_buffer }); - exe.call(args, &results); - var result: zml.Buffer = results.get(zml.Buffer); - defer result.deinit(); + args.set(.{ mnist_buffers, input_buffer }); + exe.call(args, &results); + var result: zml.Buffer = results.get(zml.Buffer); + defer result.deinit(); - log.info( - \\✅ RECOGNIZED DIGIT: - \\ +-------------+ - \\{s} - \\ +-------------+ - \\ - , .{digits[try result.getValue(u8, io)]}); - } + log.info( + \\✅ RECOGNIZED DIGIT: + \\ +-------------+ + \\{s} + \\ +-------------+ + \\ + , .{digits[try result.getValue(u8, io)]}); } fn printDigit(digit: [28 * 28]u8) void { diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 98caf5ff..d68cff1b 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -86,6 +86,10 @@ pub const Api = struct { pub const Version = struct { major: i64, minor: i64, + + pub fn format(self: Version, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("{d}.{d}", .{ self.major, self.minor }); + } }; const Funcs = std.meta.FieldEnum(c.PJRT_Api); @@ -93,7 +97,11 @@ pub const Api = struct { inner: c.PJRT_Api, pub fn loadFrom(library: [:0]const u8) !*const Api { - log.info("Loading library {s}...", .{library}); + const basename = std.Io.Dir.path.basename(library); + log.info("Loading: {s}...", .{basename}); + var timer: std.time.Timer = try .start(); + defer log.info("Loaded: {s} [{D}]", .{ basename, timer.read() }); + var lib: std.DynLib = switch (builtin.os.tag) { .linux => blk: { // We use RTLD_GLOBAL so that symbols from NEEDED libraries are available in the global namespace. @@ -113,7 +121,6 @@ pub const Api = struct { }; const api = DynGetPjrtApi(); - log.info("Loaded library: {s}", .{library}); _ = api.call(.PJRT_Plugin_Initialize, .{}) catch unreachable; return api; @@ -622,6 +629,13 @@ pub const Device = opaque { return null; } + pub fn defaultMemory(self: *const Device, api: *const Api) *const Memory { + const ret = api.call(.PJRT_Device_DefaultMemory, .{ + .device = self.inner(), + }) catch unreachable; + return @ptrCast(ret.memory); + } + pub const MemoryStats = struct { /// Number of bytes in use. bytes_in_use: u64, @@ -669,7 +683,7 @@ pub const Device = opaque { const ret = try api.call(.PJRT_Device_MemoryStats, .{ .device = self.inner(), }); - return .fromCStruct(ret); + return .fromCStruct(@bitCast(ret)); } }; @@ -1212,6 +1226,11 @@ pub const Memory = opaque { }; } + pub fn kind_(self: *const Memory, api: *const Api) []const u8 { + const ret = api.call(.PJRT_Memory_Kind, .{ .memory = self.inner() }) catch unreachable; + return ret.kind[0..ret.kind_size]; + } + pub fn kindId(self: *const Memory, api: *const Api) u32 { const ret = api.call(.PJRT_Memory_Kind_Id, .{ .memory = self.inner(), diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index 6bdce1fa..34bf52e0 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -10,7 +10,7 @@ pub const Backend = enum { cuda_fa2, cuda_fa3, - pub fn auto(platform: zml.Platform) Backend { + pub fn auto(platform: *const zml.Platform) Backend { return switch (platform.target) { .cuda => b: { const first_device = platform.pjrt_client.devices(platform.pjrt_api)[0]; @@ -84,7 +84,7 @@ pub const Metadata = union(Backend) { }; } - pub fn initBuffer(self: Metadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(Metadata) { + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { return switch (self) { .vanilla => .{ .vanilla = {} }, .cuda_fa2 => |v| .{ .cuda_fa2 = try v.initBuffer(io, platform) }, diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index a9a77300..8673bc4e 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -224,7 +224,7 @@ pub const fa2 = struct { }; } - pub fn initBuffer(self: Metadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(Metadata) { + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { return .{ .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), .{}), .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), .{}), @@ -405,7 +405,7 @@ pub const fa3 = struct { }; } - pub fn initBuffer(self: Metadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(Metadata) { + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { return .{ .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), .{}), .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), .{}), diff --git a/zml/buffer.zig b/zml/buffer.zig index a5dc31cb..7b0a88ac 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -5,6 +5,7 @@ const stdx = @import("stdx"); const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; +const Memory = @import("platform.zig").Memory; const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; @@ -21,15 +22,13 @@ const log = std.log.scoped(.zml); /// * can be created by calling `HostBuffer.toDevice(platform)`. pub const Buffer = struct { _shape: Shape, - _api: *const pjrt.Api, - _target: Target, + platform: *const Platform, _shards: Shards, pub const MAX_NUM_SHARDS: u8 = Platform.MAX_NUM_DEVICES; pub const Shards = stdx.BoundedArray(*pjrt.Buffer, MAX_NUM_SHARDS); - pub const Memory = pjrt.Memory.Kind; - pub const FromOptions = struct { wait: bool = true, memory: Memory = .device }; + pub const FromOptions = struct { wait: bool = true, memory: Memory.Kind = .default }; /// Frees the accelerator memory. /// Depending on the platform, the memory is typically not released to the OS @@ -37,7 +36,7 @@ pub const Buffer = struct { pub fn deinit(self: *Buffer) void { // log.warn("Unloading {f} {d} bytes", .{ self._shape, self._shape.byteSize() }); for (self._shards.constSlice()) |buffer| { - buffer.deinit(self._api); + buffer.deinit(self.platform.pjrt_api); } } @@ -53,8 +52,7 @@ pub const Buffer = struct { /// Copies the content of the given buffer from host memory to the accelerator memory. pub fn from(io: std.Io, platform: *const Platform, shape_: Shape, data_: []const u8, opts: FromOptions) !Buffer { var res: Buffer = .{ - ._api = platform.pjrt_api, - ._target = platform.target, + .platform = platform, ._shape = shape_, ._shards = .{}, }; @@ -74,7 +72,7 @@ pub const Buffer = struct { const buffer_type = pjrtx.bufferTypeFromDtype(shape_.dtype()); const byte_strides = shape_.computeByteStrides(); - const devices = platform.getDevices(); + const devices = platform.devices; for (0..n_partitions) |i| { // If no sharding if found, the given buffer is replicated on all devices. //const buf = if (sharding_ax) |ax| buf: { @@ -89,11 +87,7 @@ pub const Buffer = struct { .dims = shape_.dims(), .byte_strides = byte_strides.slice(), .host_buffer_semantics = .ImmutableUntilTransferCompletes, - // CPU has no distinctions between memories. - .dst = if (platform.target == .cpu or opts.memory == .device) - .{ .device = devices[i] } - else - .{ .memory = platform.memoryForDevice(opts.memory, devices[i]) }, + .dst = .{ .memory = devices[i].memory(opts.memory).pjrt_memory }, }; const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); @@ -103,7 +97,7 @@ pub const Buffer = struct { } if (opts.wait) { - res = try res.await(io); + try res.await(io); } return res; @@ -131,32 +125,22 @@ pub const Buffer = struct { return fromBytes(io, platform, Shape.init(.{}, dtype_), x.asBytes()); } - pub fn await(self: Buffer, io: std.Io) !Buffer { + pub fn await(self: Buffer, io: std.Io) !void { for (self._shards.constSlice()) |buffer| { - const ev = buffer.readyEvent(self._api); - defer ev.deinit(self._api); - try ev.await(self._api, io); + const ev = buffer.readyEvent(self.platform.pjrt_api); + defer ev.deinit(self.platform.pjrt_api); + try ev.await(self.platform.pjrt_api, io); } - - return self; } - pub const UnitializedOptions = struct { memory: Memory = .device }; + pub const UnitializedOptions = struct { memory: Memory.Kind = .default }; pub fn uninitialized(io: std.Io, platform: *const Platform, shape_: Shape, opts: UnitializedOptions) !Buffer { - if (opts.memory != .device) { - // XLA uninitialized doesn't respect memory see https://github.com/openxla/xla/pull/31292 - // TODO: use uninitialized when it works again. - const slice: Slice = try .alloc(std.heap.smp_allocator, shape_); - defer slice.free(std.heap.smp_allocator); - return .fromSliceOpts(io, platform, slice, .{ .memory = opts.memory }); - } - + _ = io; // autofix var res: Buffer = .{ - ._api = platform.pjrt_api, + .platform = platform, ._shape = shape_, ._shards = .{}, - ._target = platform.target, }; errdefer for (res._shards.slice()) |shard| { shard.deinit(platform.pjrt_api); @@ -187,13 +171,8 @@ pub const Buffer = struct { .dst = undefined, }; - const devices = platform.getDevices(); - for (0..n_partitions) |i| { - args.dst = if (platform.target == .cpu or opts.memory == .device) - .{ .device = devices[i] } - else - .{ .memory = platform.memoryForDevice(opts.memory, devices[i]) }; - + for (platform.devices[0..n_partitions]) |*device| { + args.dst = .{ .memory = device.memory(opts.memory).pjrt_memory }; const shard = try platform.pjrt_client.createUninitializedBuffer(platform.pjrt_api, args); res._shards.appendAssumeCapacity(shard); } @@ -208,8 +187,7 @@ pub const Buffer = struct { var shards: Shards = .{}; shards.appendSliceAssumeCapacity(pjrt_buffers); return .{ - ._api = platform.pjrt_api, - ._target = platform.target, + .platform = platform, ._shape = shape_, ._shards = shards, }; @@ -217,7 +195,7 @@ pub const Buffer = struct { pub fn devicePtr(self: Buffer) *anyopaque { //stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer", .{}); - return self._shards.get(0).opaqueDeviceMemoryDataPointer(self._api) catch unreachable; + return self._shards.get(0).opaqueDeviceMemoryDataPointer(self.platform.pjrt_api) catch unreachable; } /// Fetches the content of the given buffer into a stack variable of the given type. @@ -233,9 +211,9 @@ pub const Buffer = struct { /// Copies the content of the Buffer to the provided slice. pub fn toSlice(self: Buffer, io: std.Io, slice: Slice) !void { //stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self._api, slice.data()); + const maybe_event = try self._shards.get(0).toHostBuffer(self.platform.pjrt_api, slice.data()); if (maybe_event) |event| { - try event.await(self._api, io); + try event.await(self.platform.pjrt_api, io); } } diff --git a/zml/io.zig b/zml/io.zig index 9edfa958..814b4e6a 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -7,6 +7,7 @@ pub const VFS = @import("io").VFS; const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("zml.zig").Bufferized; const mem = @import("mem.zig"); +const Memory = @import("platform.zig").Memory; const meta = @import("meta.zig"); const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; @@ -266,26 +267,25 @@ pub const ProgressWriter = struct { } }; -pub const DeviceWriter = union(enum) { - direct: DirectDeviceWriter, - buffered: BufferedDeviceWriter, +pub const MemoryWriter = union(enum) { + direct: DirectMemoryWriter, + buffered: BufferedMemoryWriter, - pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, pool: *mem.DynamicBufferPool, shape: Shape, buffer: *Buffer, memory_kind: Buffer.Memory) !DeviceWriter { - return switch (platform.target) { - .cuda, .rocm => .{ .direct = try .init(allocator, io, platform, pool, shape, buffer, memory_kind) }, - .tpu, .neuron => .{ .buffered = try .init(allocator, io, platform, shape, buffer) }, - else => unreachable, + pub fn init(allocator: std.mem.Allocator, io: std.Io, memory: *const Memory, pool: *mem.DynamicBufferPool, shape: Shape, buffer: *Buffer) !MemoryWriter { + return switch (memory.platform.target) { + .cuda, .rocm => .{ .direct = try .init(allocator, io, memory, pool, shape, buffer) }, + .tpu, .neuron, .cpu => .{ .buffered = try .init(allocator, io, memory, shape, buffer) }, }; } - pub fn interface(self: *DeviceWriter) *std.Io.Writer { + pub fn interface(self: *MemoryWriter) *std.Io.Writer { return switch (self.*) { .direct => &self.direct.interface, .buffered => &self.buffered.interface, }; } - pub fn deinit(self: *DeviceWriter, allocator: std.mem.Allocator) void { + pub fn deinit(self: *MemoryWriter, allocator: std.mem.Allocator) void { switch (self.*) { .direct => self.direct.deinit(), .buffered => self.buffered.deinit(allocator), @@ -293,18 +293,18 @@ pub const DeviceWriter = union(enum) { } }; -pub const BufferedDeviceWriter = struct { +pub const BufferedMemoryWriter = struct { io: std.Io, - platform: *const Platform, + memory: *const Memory, shape: Shape, buffer: *Buffer, interface: std.Io.Writer, - pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, shape: Shape, buffer: *Buffer) !BufferedDeviceWriter { + pub fn init(allocator: std.mem.Allocator, io: std.Io, memory: *const Memory, shape: Shape, buffer: *Buffer) !BufferedMemoryWriter { return .{ .io = io, .shape = shape, - .platform = platform, + .memory = memory, .buffer = buffer, .interface = .{ .buffer = try allocator.alloc(u8, shape.byteSize()), @@ -317,22 +317,42 @@ pub const BufferedDeviceWriter = struct { }; } - pub fn deinit(self: *BufferedDeviceWriter, allocator: std.mem.Allocator) void { + pub fn deinit(self: *BufferedMemoryWriter, allocator: std.mem.Allocator) void { if (self.interface.buffer.len > 0) { allocator.free(self.interface.buffer); } } + fn flush_(w: *std.Io.Writer) !void { + const self: *BufferedMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); + const pjrt_api = self.memory.platform.pjrt_api; + + const args: pjrt.Client.BufferFromHostBufferArgs = .{ + .data = @ptrCast(self.interface.buffer), + .buffer_type = pjrtx.bufferTypeFromDtype(self.shape.dtype()), + .dims = self.shape.dims(), + .byte_strides = self.shape.computeByteStrides().constSlice(), + .host_buffer_semantics = .ImmutableUntilTransferCompletes, + .dst = .{ .memory = self.memory.pjrt_memory }, + }; + const pjrt_buffer, const event = try self.memory.platform.pjrt_client.bufferFromHostBuffer(pjrt_api, args); + self.buffer.* = Buffer.fromPjrtBuffers(self.memory.platform, self.shape, &.{pjrt_buffer}); + if (event) |ev| { + defer ev.deinit(pjrt_api); + try ev.await(pjrt_api, self.io); + } else { + try self.buffer.await(self.io); + } + } + pub fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { - const self: *BufferedDeviceWriter = @alignCast(@fieldParentPtr("interface", w)); - self.buffer.* = Buffer.fromBytes(self.io, self.platform, self.shape, self.interface.buffer) catch return std.Io.Writer.Error.WriteFailed; - _ = self.buffer.await(self.io) catch return std.Io.Writer.Error.WriteFailed; + flush_(w) catch return std.Io.Writer.Error.WriteFailed; } }; -pub const DirectDeviceWriter = struct { +pub const DirectMemoryWriter = struct { const EventContext = struct { - self: *DirectDeviceWriter, + self: *DirectMemoryWriter, err: ?*pjrt.Error = null, pjrt_event: *pjrt.Event, event: std.Io.Event = .unset, @@ -341,7 +361,7 @@ pub const DirectDeviceWriter = struct { allocator: std.mem.Allocator, io: std.Io, - platform: *const Platform, + memory: *const Memory, pool: *mem.DynamicBufferPool, total: usize, buffer: *Buffer, @@ -353,26 +373,23 @@ pub const DirectDeviceWriter = struct { flip_flop: u1 = 0, events_contexts: [2]?EventContext = @splat(null), - pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, pool: *mem.DynamicBufferPool, shape: Shape, buffer: *Buffer, memory_kind: Buffer.Memory) !DirectDeviceWriter { + pub fn init(allocator: std.mem.Allocator, io: std.Io, memory: *const Memory, pool: *mem.DynamicBufferPool, shape: Shape, buffer: *Buffer) !DirectMemoryWriter { const shape_spec: pjrt.ShapeSpec = .init(shape.dims(), pjrtx.bufferTypeFromDtype(shape.dtype())); - const transfer_manager = try platform.pjrt_client.createBuffersForAsyncHostToDevice(platform.pjrt_api, .{ - .shape_specs = &.{shape_spec}, - .memory = blk: { - const devices = platform.getDevices(); - for (devices[0].addressableMemories(platform.pjrt_api)) |m| { - if (m.kind(platform.pjrt_api) == memory_kind) break :blk m; - } - return error.MemoryNotFound; + const transfer_manager = try memory.platform.pjrt_client.createBuffersForAsyncHostToDevice( + memory.platform.pjrt_api, + .{ + .shape_specs = &.{shape_spec}, + .memory = memory.pjrt_memory, }, - }); + ); - const pjrt_buffer = transfer_manager.retrieveBuffer(platform.pjrt_api, 0) catch unreachable; - buffer.* = Buffer.fromPjrtBuffers(platform, shape, &.{pjrt_buffer}); + const pjrt_buffer = transfer_manager.retrieveBuffer(memory.platform.pjrt_api, 0) catch unreachable; + buffer.* = Buffer.fromPjrtBuffers(memory.platform, shape, &.{pjrt_buffer}); return .{ .allocator = allocator, .io = io, - .platform = platform, + .memory = memory, .pool = pool, .total = shape.byteSize(), .buffer = buffer, @@ -387,18 +404,20 @@ pub const DirectDeviceWriter = struct { }; } - pub fn deinit(self: *DirectDeviceWriter) void { - self.transfer_manager.deinit(self.platform.pjrt_api); + pub fn deinit(self: *DirectMemoryWriter) void { + self.transfer_manager.deinit(self.memory.platform.pjrt_api); } fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { _ = data; // autofix _ = splat; // autofix - const self: *DirectDeviceWriter = @alignCast(@fieldParentPtr("interface", w)); + const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); if (self.offset >= self.total) { return std.Io.Writer.Error.WriteFailed; } + const pjrt_api = self.memory.platform.pjrt_api; + const current_buffer = self.interface.buffer; const buffered = self.interface.buffered(); const sliced_buffer = self.interface.buffered()[0..@min(buffered.len, self.total - self.offset)]; @@ -406,7 +425,7 @@ pub const DirectDeviceWriter = struct { const is_last = (self.offset + sliced_buffer.len) >= self.total; const transfer_event = self.transfer_manager.transferData( - self.platform.pjrt_api, + pjrt_api, 0, sliced_buffer, @intCast(self.offset), @@ -421,7 +440,7 @@ pub const DirectDeviceWriter = struct { .buffer = current_buffer, .pjrt_event = transfer_event, }; - transfer_event.onReady(self.platform.pjrt_api, EventContext, struct { + transfer_event.onReady(pjrt_api, EventContext, struct { fn call(err: ?*pjrt.Error, ctx_: *EventContext) void { ctx_.self.pool.put(ctx_.self.io, ctx_.buffer); ctx_.err = err; @@ -434,15 +453,13 @@ pub const DirectDeviceWriter = struct { if (self.events_contexts[@intCast(self.flip_flop ^ 1)]) |*ctx_previous| { defer self.events_contexts[@intCast(self.flip_flop ^ 1)] = null; - // std.log.info("[{*} WAITING", .{self}); ctx_previous.event.waitUncancelable(self.io); - // std.log.info("[{*} WAITED", .{self}); - defer ctx_previous.pjrt_event.deinit(self.platform.pjrt_api); + defer ctx_previous.pjrt_event.deinit(pjrt_api); if (ctx_previous.err) |e| { - defer e.deinit(self.platform.pjrt_api); + defer e.deinit(pjrt_api); log.err("error while awaiting: {s}: {s}", .{ - @tagName(e.getCode(self.platform.pjrt_api)), - e.getMessage(self.platform.pjrt_api), + @tagName(e.getCode(pjrt_api)), + e.getMessage(pjrt_api), }); return std.Io.Writer.Error.WriteFailed; } @@ -462,3 +479,101 @@ pub const DirectDeviceWriter = struct { return 0; } }; + +pub const LoadOpts = struct { + parallelism: usize, + store: *const TensorStore, + progress: ?*std.Progress.Node = null, + dma_chunks: usize, + dma_chunk_size: usize, + total_bytes: ?*usize = null, +}; + +pub fn load( + comptime ModelType: type, + model: *const ModelType, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + opts: LoadOpts, +) !Bufferized(ModelType) { + var bufferized = try mem.bufferize(allocator, ModelType, model); + + const first_device = platform.devices[0]; // Temporary until sharding is re-exposed + const dma_alloc: mem.DmaAllocator = .init(allocator, &first_device); + var buffer_pool: mem.DynamicBufferPool = .init(opts.dma_chunks, opts.dma_chunk_size); + defer buffer_pool.deinit(dma_alloc.allocator()); + + const Ctx = struct { + allocator: std.mem.Allocator, + dma_allocator: std.mem.Allocator, + pinned_buffer_pool: *mem.DynamicBufferPool, + io: std.Io, + buffers: []*Buffer, + store: *const TensorStore, + memory: *const Memory, + group: stdx.Io.LimitedGroup, + total: std.atomic.Value(usize) = .init(0), + progress: ?*std.Progress.Node, + }; + var walk_ctx: Ctx = .{ + .buffers = try allocator.alloc(*Buffer, meta.count(Tensor, model)), + .store = opts.store, + .allocator = allocator, + .dma_allocator = dma_alloc.allocator(), + .pinned_buffer_pool = &buffer_pool, + .io = io, + .memory = first_device.memory(.default), + .progress = opts.progress, + .group = .init(opts.parallelism), + }; + + defer if (opts.total_bytes) |total_bytes_ptr| { + total_bytes_ptr.* = walk_ctx.total.load(.monotonic); + }; + + meta.forEachVisit(&bufferized, *Buffer, struct { + fn call(i: usize, buffer: *Buffer, ctx: *Ctx) void { + ctx.buffers[i] = buffer; + } + }.call, .{&walk_ctx}); + + meta.forEachVisit(model, *const Tensor, struct { + fn call(i: usize, tensor: *const Tensor, ctx: *Ctx) void { + ctx.group.concurrent(ctx.io, struct { + fn call(i_: usize, tensor_: *const Tensor, ctx_: *Ctx) !void { + var reader = ctx_.store.getReaderById(tensor_.id, ctx_.io, &.{}) catch unreachable; + defer reader.deinit(); + + var memory_writer = MemoryWriter.init( + ctx_.dma_allocator, + ctx_.io, + ctx_.memory, + ctx_.pinned_buffer_pool, + reader.tensor.shape, + ctx_.buffers[i_], + ) catch unreachable; + defer memory_writer.deinit(ctx_.dma_allocator); + + const scale = 1024; + + if (ctx_.progress) |progress| { + var node = progress.start(reader.tensor.name, reader.tensor.shape.byteSize() / scale); + defer node.end(); + var progress_writer: ProgressWriter = .init(memory_writer.interface(), &node, .{ .scale = scale }); + const total = reader.interface.streamRemaining(&progress_writer.interface) catch unreachable; + progress_writer.interface.flush() catch unreachable; + _ = ctx_.total.fetchAdd(total, .monotonic); + } else { + const total = reader.interface.streamRemaining(memory_writer.interface()) catch unreachable; + memory_writer.interface().flush() catch unreachable; + _ = ctx_.total.fetchAdd(total, .monotonic); + } + } + }.call, .{ i, tensor, ctx }) catch unreachable; + } + }.call, .{&walk_ctx}); + walk_ctx.group.await(io) catch unreachable; + + return bufferized; +} diff --git a/zml/mem.zig b/zml/mem.zig index 0912209b..68c7bc74 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -6,25 +6,28 @@ const assert = std.debug.assert; const pjrt = @import("pjrt"); const Buffer = @import("buffer.zig").Buffer; +const Device = @import("platform.zig").Device; +const Memory = @import("platform.zig").Memory; const meta = @import("meta.zig"); const Platform = @import("platform.zig").Platform; const Tensor = @import("tensor.zig").Tensor; const log = std.log.scoped(.@"zml/mem"); -pub const HostPinnedAllocator = union(enum) { +pub const DmaAllocator = union(enum) { passthrough: std.mem.Allocator, - uib: UnitializedBufferHostPinnedAllocator, + uib: UninitializedBufferAllocator, dmam: DmaMapAllocator, - pub fn init(parent: std.mem.Allocator, platform: *const Platform) HostPinnedAllocator { - return switch (platform.target) { - .cuda, .rocm => .{ .dmam = .init(parent, platform) }, - else => .{ .passthrough = parent }, + pub fn init(parent: std.mem.Allocator, device: *const Device) DmaAllocator { + return switch (device.platform.target) { + .cuda, .rocm => .{ .dmam = .init(parent, device.platform) }, + .tpu, .neuron => .{ .uib = .init(device.memory(.host_pinned)) }, + .cpu => .{ .passthrough = parent }, }; } - pub fn allocator(self: *const HostPinnedAllocator) std.mem.Allocator { + pub fn allocator(self: *const DmaAllocator) std.mem.Allocator { return switch (self.*) { .passthrough => |a| a, inline else => |*a| a.allocator(), @@ -32,25 +35,20 @@ pub const HostPinnedAllocator = union(enum) { } }; -pub const UnitializedBufferHostPinnedAllocator = struct { - platform: *const Platform, - host_pinned_memory: *const pjrt.Memory, +pub const UninitializedBufferAllocator = struct { + memory: *const Memory, const Header = struct { buffer: *pjrt.Buffer, }; - pub fn init(platform: *const Platform) UnitializedBufferHostPinnedAllocator { - const devices = platform.getDevices(); - const mem = platform.memoryForDevice(.host_pinned, devices[0]); - + pub fn init(memory: *const Memory) UninitializedBufferAllocator { return .{ - .platform = platform, - .host_pinned_memory = mem, + .memory = memory, }; } - pub fn allocator(self: *const UnitializedBufferHostPinnedAllocator) std.mem.Allocator { + pub fn allocator(self: *const UninitializedBufferAllocator) std.mem.Allocator { return .{ .ptr = @constCast(self), .vtable = &.{ @@ -63,11 +61,13 @@ pub const UnitializedBufferHostPinnedAllocator = struct { } fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, _: usize) ?[*]u8 { - const self: *UnitializedBufferHostPinnedAllocator = @ptrCast(@alignCast(ctx)); + const self: *UninitializedBufferAllocator = @ptrCast(@alignCast(ctx)); + const pjrt_api = self.memory.platform.pjrt_api; + const pjrt_client = self.memory.platform.pjrt_client; const total_len = std.mem.alignForward(usize, @sizeOf(Header) + len, alignment.toByteUnits()); - const pjrt_buffer = self.platform.pjrt_client.createUninitializedBuffer(self.platform.pjrt_api, .{ + const pjrt_buffer = pjrt_client.createUninitializedBuffer(pjrt_api, .{ .dims = &.{@intCast(total_len)}, .element_type = .u8, .layout = .{ @@ -77,10 +77,10 @@ pub const UnitializedBufferHostPinnedAllocator = struct { .tile_dims_sizes = &.{}, }, }, - .dst = .{ .memory = self.host_pinned_memory }, + .dst = .{ .memory = self.memory.pjrt_memory }, }) catch return null; - const opaque_ptr: [*]u8 = @ptrCast(pjrt_buffer.opaqueDeviceMemoryDataPointer(self.platform.pjrt_api) catch unreachable); + const opaque_ptr: [*]u8 = @ptrCast(pjrt_buffer.opaqueDeviceMemoryDataPointer(pjrt_api) catch unreachable); const data_with_header: []u8 = opaque_ptr[0..total_len]; const header = std.mem.bytesAsValue(Header, opaque_ptr); @@ -93,9 +93,10 @@ pub const UnitializedBufferHostPinnedAllocator = struct { fn free(ctx: *anyopaque, buf: []u8, alignment: Alignment, ret_addr: usize) void { _ = ret_addr; - const self: *UnitializedBufferHostPinnedAllocator = @ptrCast(@alignCast(ctx)); + const self: *UninitializedBufferAllocator = @ptrCast(@alignCast(ctx)); + const pjrt_api = self.memory.platform.pjrt_api; const header: *Header = @ptrFromInt(std.mem.alignBackward(usize, @intFromPtr(buf.ptr) - @sizeOf(Header), alignment.toByteUnits())); - header.buffer.deinit(self.platform.pjrt_api); + header.buffer.deinit(pjrt_api); } fn resize(ctx: *anyopaque, buf: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) bool { @@ -333,7 +334,7 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu return; } - const type_info = @typeInfo(Bufferized); + const type_info = @typeInfo(ModelBufferized); switch (type_info) { .@"struct" => |struct_type_info| { inline for (struct_type_info.fields) |field| { @@ -372,8 +373,10 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu } } -pub inline fn bufferize(allocator: std.mem.Allocator, model: anytype) !Bufferized(@TypeOf(model)) { - var bufferized: Bufferized(@TypeOf(model)) = undefined; - try bufferizeInner(allocator, model, &bufferized); +/// Convert a model to its bufferized form by replacing Tensor fields with Buffer +/// and allocating any required slices using the provided allocator. +pub inline fn bufferize(allocator: std.mem.Allocator, comptime ModelType: type, model: *const ModelType) !Bufferized(ModelType) { + var bufferized: Bufferized(ModelType) = undefined; + try bufferizeInner(allocator, model.*, &bufferized); return bufferized; } diff --git a/zml/meta.zig b/zml/meta.zig index e26fa014..17196752 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -791,3 +791,20 @@ pub fn Contains(Haystack: type, T: type) bool { else => false, }; } + +pub fn forEachVisit(v: anytype, comptime T: type, f: anytype, args: stdx.meta.TupleRange(std.meta.ArgsTuple(@TypeOf(f)), 2, null)) void { + const Ctx = struct { + i: usize, + args_: @TypeOf(args), + }; + var ctx: Ctx = .{ + .i = 0, + .args_ = args, + }; + visit(struct { + fn cb(ctx_: *Ctx, elem: T) void { + @call(.auto, f, .{ ctx_.i, elem } ++ ctx_.args_); + ctx_.i += 1; + } + }.cb, &ctx, v); +} diff --git a/zml/module.zig b/zml/module.zig index 7a2a43af..f717b093 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -9,6 +9,7 @@ const upb = @import("upb"); const Buffer = @import("buffer.zig").Buffer; const Exe = @import("exe.zig").Exe; +const Memory = @import("platform.zig").Memory; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); const pjrtx = @import("pjrtx.zig"); @@ -25,7 +26,7 @@ pub const CompilationContext = struct { block: *mlir.Block, id_to_argument: std.AutoArrayHashMapUnmanaged(usize, usize), id_to_donation: std.AutoArrayHashMapUnmanaged(usize, usize), - id_to_output_memory_kind: std.AutoArrayHashMapUnmanaged(usize, Buffer.Memory), + id_to_output_memory_kind: std.AutoArrayHashMapUnmanaged(usize, Memory.Kind), arena: std.heap.ArenaAllocator, pub fn initFromBlock(allocator: std.mem.Allocator, block: *mlir.Block) Scope { @@ -223,7 +224,7 @@ pub const OutputInfo = struct { shapes: []Shape, values: []*const mlir.Value, donations: []?usize, - output_memory_kinds: []Buffer.Memory, + output_memory_kinds: []Memory.Kind, pub fn deinit(self: OutputInfo, allocator: std.mem.Allocator) void { allocator.free(self.shapes); @@ -238,7 +239,7 @@ fn collectOutputInfo(allocator: std.mem.Allocator, v: anytype) !OutputInfo { shape_list: *std.array_list.Managed(Shape), value_list: *std.array_list.Managed(*const mlir.Value), donation_list: *std.array_list.Managed(?usize), - output_memory_kind_list: *std.array_list.Managed(Buffer.Memory), + output_memory_kind_list: *std.array_list.Managed(Memory.Kind), }; var shape_list = std.array_list.Managed(Shape).init(allocator); @@ -247,7 +248,7 @@ fn collectOutputInfo(allocator: std.mem.Allocator, v: anytype) !OutputInfo { errdefer value_list.deinit(); var donation_list = std.array_list.Managed(?usize).init(allocator); errdefer donation_list.deinit(); - var output_memory_kind_list = std.array_list.Managed(Buffer.Memory).init(allocator); + var output_memory_kind_list = std.array_list.Managed(Memory.Kind).init(allocator); errdefer output_memory_kind_list.deinit(); var context: LocalContext = .{ @@ -376,7 +377,14 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar }; for (output_info.output_memory_kinds, 0..) |output_memory_kind, index| { if (output_memory_kind == .device) continue; - output_attributes[index].appendAssumeCapacity(.named(compilation_context.mlir_ctx, "mhlo.memory_kind", mlir.stringAttribute(compilation_context.mlir_ctx, output_memory_kind.pjrtName()))); + output_attributes[index].appendAssumeCapacity(.named( + compilation_context.mlir_ctx, + "mhlo.memory_kind", + mlir.stringAttribute( + compilation_context.mlir_ctx, + compilation_context.platform.memoryKind(output_memory_kind), + ), + )); } _ = dialects.func.returns(compilation_context.mlir_ctx, output_info.values, .unknown(compilation_context.mlir_ctx)).appendTo(compilation_context.currentScope().block); diff --git a/zml/platform.zig b/zml/platform.zig index eb2f3b15..346c95e1 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -60,25 +60,87 @@ fn loadOrGetApi(allocator: std.mem.Allocator, io: std.Io, target: Target) !*cons }; } -pub const Device = struct { +pub const Memory = struct { + pub const Kind = enum { + default, + host_unpinned, + host_pinned, + device, + }; + + pjrt_memory: *const pjrt.Memory, platform: *const Platform, - device: *const pjrt.Device, - desc: *const pjrt.DeviceDescription, + addressable_by_devices: []*const Device, + + fn init(allocator: std.mem.Allocator, pjrt_memory: *const pjrt.Memory, platform: *Platform, all_devices: []const Device) !Memory { + const pjrt_addressable_by_devices = pjrt_memory.addressableByDevices(platform.pjrt_api); + const addressable_by_devices = try allocator.alloc(*const Device, pjrt_addressable_by_devices.len); + for (pjrt_addressable_by_devices, addressable_by_devices) |pjrt_device, *addressable_by_device| { + addressable_by_device.* = &all_devices[pjrt_device.localHardwareId(platform.pjrt_api)]; + } - fn init(device_: *const pjrt.Device, platform: *const Platform) Device { return .{ + .pjrt_memory = pjrt_memory, .platform = platform, - .device = device_, - .desc = device_.getDescription(platform.pjrt_api), + .addressable_by_devices = addressable_by_devices, }; } + pub fn kind(self: Memory) []const u8 { + return self.pjrt_memory.kind_(self.platform.pjrt_api); + } + + pub fn isOfKind(self: Memory, kind_: Kind) bool { + switch (self.platform.target) { + .cuda, .rocm => { + const zml_kind: Memory.Kind = switch (self.kind().len) { + "device".len => .device, + "pinned_host".len => .host_pinned, + "unpinned_host".len => .host_unpinned, + else => std.debug.panic("unknown memory {s}", .{self.kind()}), + }; + return zml_kind == kind_; + }, + .cpu, .tpu, .neuron => return true, + } + } + + fn deinit(self: *Memory, allocator: std.mem.Allocator) void { + allocator.free(self.addressable_devices); + } +}; + +pub const Device = struct { + platform: *const Platform, + pjrt_device: *const pjrt.Device, + pjrt_desc: *const pjrt.DeviceDescription, + addressable_memories: []*const Memory, + + fn init(allocator: std.mem.Allocator, pjrt_device_: *const pjrt.Device, platform: *const Platform, all_addressable_memories: []const Memory) !Device { + const pjrt_addressable_memories = pjrt_device_.addressableMemories(platform.pjrt_api); + const addressable_memories = try allocator.alloc(*const Memory, pjrt_addressable_memories.len); + for (pjrt_addressable_memories, addressable_memories) |pjrt_memory, *addressable_memory| { + addressable_memory.* = &all_addressable_memories[pjrt_memory.id(platform.pjrt_api)]; + } + + return .{ + .platform = platform, + .pjrt_device = pjrt_device_, + .pjrt_desc = pjrt_device_.getDescription(platform.pjrt_api), + .addressable_memories = addressable_memories, + }; + } + + fn deinit(self: *Device, allocator: std.mem.Allocator) void { + allocator.free(self.addressable_memories); + } + pub fn id(self: Device) i32 { - return self.desc.id(self.platform.pjrt_api); + return self.pjrt_desc.id(self.platform.pjrt_api); } pub fn processIndex(self: Device) i32 { - return self.desc.processIndex(self.platform.pjrt_api); + return self.pjrt_desc.processIndex(self.platform.pjrt_api); } pub fn localHardwareId(self: Device) i32 { @@ -86,30 +148,50 @@ pub const Device = struct { } pub fn kind(self: Device) []const u8 { - return self.desc.kind(self.platform.pjrt_api); + return self.pjrt_desc.kind(self.platform.pjrt_api); } pub fn debugString(self: Device) []const u8 { - return self.desc.debugString(self.platform.pjrt_api); + return self.pjrt_desc.debugString(self.platform.pjrt_api); } pub fn toString(self: Device) []const u8 { - return self.desc.toString(self.platform.pjrt_api); + return self.pjrt_desc.toString(self.platform.pjrt_api); + } + + pub fn memoryStats(self: Device) pjrt.Device.MemoryStats { + return self.pjrt_device.memoryStats(self.platform.pjrt_api) catch .zeroes; } pub fn format(self: Device, writer: *std.Io.Writer) std.Io.Writer.Error!void { try writer.print("{s} ({s})", .{ - self.desc.kind(self.platform.pjrt_api), - self.desc.debugString(self.platform.pjrt_api), + self.pjrt_desc.kind(self.platform.pjrt_api), + self.pjrt_desc.debugString(self.platform.pjrt_api), }); } + + pub fn memory(self: Device, memory_kind: Memory.Kind) *const Memory { + if (memory_kind == .default) { + const mem = self.pjrt_device.defaultMemory(self.platform.pjrt_api); + return &self.platform.memories[mem.id(self.platform.pjrt_api)]; + } + + for (self.addressable_memories) |mem| { + if (mem.isOfKind(memory_kind)) { + return mem; + } + } + unreachable; + } }; pub const Platform = struct { + arena_state: std.heap.ArenaAllocator.State = .{}, target: Target, pjrt_api: *const pjrt.Api, pjrt_client: *pjrt.Client, - _devices: []Device, + devices: []const Device, + memories: []const Memory, // This make the pjrt struct quite fat, but is only used during compilation. // TODO: Reconsider having it here, and maybe pass explicitly to compile, @@ -129,17 +211,32 @@ pub const Platform = struct { log.warn("platform {} got {} devices, but ZML only support up to {} devices. Some devices won't be used.", .{ target, pjrt_devices.len, MAX_NUM_DEVICES }); } - var platform = try allocator.create(Platform); + var arena = std.heap.ArenaAllocator.init(allocator); + errdefer arena.deinit(); + + const devices = try arena.allocator().alloc(Device, pjrt_devices.len); + + const pjrt_memories = pjrt_client.addressableMemories(api); + const memories = try arena.allocator().alloc(Memory, pjrt_memories.len); + + const platform = try arena.allocator().create(Platform); platform.* = .{ .target = target, .pjrt_api = api, .pjrt_client = pjrt_client, .compilation_options = .{}, - ._devices = try allocator.alloc(Device, pjrt_devices.len), + .devices = devices, + .memories = memories, }; + defer platform.arena_state = arena.state; - for (pjrt_devices, platform._devices) |pjrt_device, *platform_device| { - platform_device.* = .init(pjrt_device, platform); + { + for (pjrt_devices, devices) |pjrt_device, *platform_device| { + platform_device.* = try .init(allocator, pjrt_device, platform, memories); + } + for (pjrt_memories, memories) |pjrt_memory, *platform_memory| { + platform_memory.* = try .init(allocator, pjrt_memory, platform, devices); + } } switch (target) { @@ -171,15 +268,33 @@ pub const Platform = struct { } else error.Unavailable; } - pub fn devices(self: *const Platform) []const Device { - return self._devices; - } - pub fn formatWithDevices(self: *const Platform, writer: *std.Io.Writer) std.Io.Writer.Error!void { - try writer.print("Platform: {s}\n", .{@tagName(self.target)}); - try writer.print("Devices total={d}\n", .{devices().len}); - for (devices(), 0..) |device, i| { - try writer.print("\t#{d}: {s}\n", .{ i, device.kind() }); + const tee = "├─ "; + const line = "│ "; + const langle = "└─ "; + + try writer.print("platform: {s}\n", .{@tagName(self.target)}); + try writer.print("version: {f}\n", .{self.pjrt_api.version()}); + try writer.print("stablehlo_version: {?s}\n", .{self.pjrt_api.stablehloCurrentVersion()}); + try writer.print("extensions:\n", .{}); + { + var it = self.pjrt_api.extensions(); + while (it.next()) |ext| { + try writer.print("{s}{s}\n", .{ if (it.current != null) tee else langle, @tagName(ext) }); + } + } + + try writer.print("devices:\n", .{}); + for (self.devices, 0..) |device, i| { + const is_last_device = i < self.devices.len - 1; + try writer.print("{s}{f}\n", .{ if (is_last_device) tee else langle, device }); + for (device.addressable_memories, 0..) |mem, j| { + try writer.print("{s}{s}memory: {s}\n", .{ + if (is_last_device) line else " ", + if (j < device.addressable_memories.len - 1) tee else langle, + mem.kind(), + }); + } } } @@ -187,14 +302,6 @@ pub const Platform = struct { return .{ .data = self }; } - pub fn getDevices(self: *const Platform) []const *const pjrt.Device { - const all_devices = self.pjrt_client.addressableDevices(self.pjrt_api); - if (all_devices.len > MAX_NUM_DEVICES) { - return all_devices[0..MAX_NUM_DEVICES]; - } - return all_devices; - } - pub const Sharding = struct { num_replicas: u8, num_partitions: u8 }; pub fn sharding(self: *const Platform) Sharding { @@ -203,7 +310,7 @@ pub const Platform = struct { // Inside an inference process, we generally don't want replicas, // as it's best to fully isolate replicas on different processes. // For now we hardcode num_replicas = 1. - const num_devices: u8 = @intCast(self.getDevices().len); + const num_devices: u8 = @intCast(self.devices.len); return if (self.compilation_options.sharding_enabled) .{ .num_replicas = 1, .num_partitions = num_devices } else @@ -211,47 +318,8 @@ pub const Platform = struct { } pub fn deinit(self: *Platform, allocator: std.mem.Allocator) void { - allocator.free(self._devices); self.pjrt_client.deinit(self.pjrt_api); - } - - pub fn memoryForDevice(platform: *const Platform, memory: pjrt.Memory.Kind, device: *const pjrt.Device) *const pjrt.Memory { - const memory_target: pjrt.Memory.Kind = switch (memory) { - .host_unpinned => switch (platform.target) { - // Cuda doesn't have host_unpinned. - // ROCm doesn't seem to have it either. - // TODO(gwenzek): investigate why it was not forced before. - .cuda, .rocm => .host_pinned, - else => .host_unpinned, - }, - inline else => |t| t, - }; - // TODO measure the cost of this and consider caching. - const device_memories = device.addressableMemories(platform.pjrt_api); - for (device_memories) |m| { - if (memory_target == m.kind(platform.pjrt_api)) { - return m; - } - } - log.err("Platform {t} doesn't have memory {t}", .{ platform.target, memory }); - @panic("Memory kind not found"); - } - - test memoryForDevice { - const platform = zml.testing.env(); - const memory_fields = @typeInfo(pjrt.Memory.Kind).@"enum".fields; - inline for (memory_fields) |field| { - for (platform.getDevices()) |dev| { - _ = platform.memoryForDevice(@field(pjrt.Memory.Kind, field.name), dev); - } - } - } - - pub fn memoryStats(platform: *const Platform, device_id: usize) pjrt.MemoryStats { - if (platform.target == .cpu) return .zeroes; - - const device = platform.getDevices()[device_id]; - return device.memoryStats(platform.pjrt_api) catch .zeroes; + self.arena_state.promote(allocator).deinit(); } pub fn compile( @@ -282,10 +350,19 @@ pub const Platform = struct { try writer.print("{s} {{ ", .{@tagName(self.target)}); for (self.devices(), 0..) |device, i| { try writer.print("{s}(\"{s}\")", .{ device.toString(), device.kind() }); - if (i < devices().len - 1) try writer.writeAll(", "); + if (i < self.devices.len - 1) try writer.writeAll(", "); } try writer.writeAll(" }"); } + + pub fn memoryKind(self: *const Platform, kind: Memory.Kind) []const u8 { + for (self.memories) |mem| { + if (mem.isOfKind(kind)) { + return mem.kind(); + } + } + unreachable; + } }; pub const CreateOptions = struct { @@ -294,7 +371,7 @@ pub const CreateOptions = struct { // bump memory fraction from XLA defaults of 75% to 90%. // Even on a 8GB GPU it should leave enough space for the Cuda driver // https://github.com/openxla/xla/blob/3e87afa11a865cf91137522492918ad18bfe5b7c/xla/pjrt/plugin/xla_gpu/xla_gpu_allocator_config.h#L25-L60 - cuda: Cuda = .{ .allocator = .{ .async = .{ .preallocate = true, .memory_fraction = 0.90 } } }, + cuda: Cuda = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.95 } } }, rocm: struct {} = .{}, tpu: struct {} = .{}, neuron: struct {} = .{}, @@ -372,7 +449,7 @@ pub const CreateOptions = struct { // TODO(Corendos): Consider moving that in its own file if its size increase too much. pub const cuda = struct { - pub fn tryGetComputeCapabilities(platform: zml.Platform, device: *const pjrt.Device) ?[]const u8 { + pub fn tryGetComputeCapabilities(platform: *const zml.Platform, device: *const pjrt.Device) ?[]const u8 { stdx.debug.assert(platform.target == .cuda, "tryGetComputeCapabilities expects .cuda platform, got {}", .{platform.target}); const description = device.getDescription(platform.pjrt_api); diff --git a/zml/safetensors.zig b/zml/safetensors.zig index b1b791a1..aa3585d8 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -69,6 +69,7 @@ pub const TensorReader = struct { remaining: u64, io: std.Io, interface: std.Io.Reader, + tensor: Tensor, pub const Error = error{TensorNotFound} || std.Io.File.OpenError || std.Io.File.Reader.SeekError || std.mem.Allocator.Error; @@ -87,6 +88,7 @@ pub const TensorReader = struct { .file = file, .file_reader = file_reader, .remaining = tensor.byteSize(), + .tensor = tensor, .io = io, .interface = .{ .vtable = &.{ @@ -106,7 +108,9 @@ pub const TensorReader = struct { fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { const self: *TensorReader = @fieldParentPtr("interface", r); - if (self.remaining == 0) return error.EndOfStream; + if (self.remaining == 0) { + return error.EndOfStream; + } const combined_limit = limit.min(.limited64(self.remaining)); const n = try self.file_reader.interface.stream(w, combined_limit); @@ -116,7 +120,10 @@ pub const TensorReader = struct { fn discard(r: *std.Io.Reader, limit: std.Io.Limit) std.Io.Reader.Error!usize { const self: *TensorReader = @fieldParentPtr("interface", r); - if (self.remaining == 0) return error.EndOfStream; + if (self.remaining == 0) { + std.log.info("EOS2 !", .{}); + return error.EndOfStream; + } const combined_limit = limit.min(.limited64(self.remaining)); const n = try self.file_reader.interface.discard(combined_limit); diff --git a/zml/tensor.zig b/zml/tensor.zig index d82f0373..f101ed5e 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -10,12 +10,12 @@ const Bufferized = @import("zml.zig").Bufferized; const CompilationContext = @import("module.zig").CompilationContext; const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; +const Memory = @import("platform.zig").Memory; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; -const Memory = Buffer.Memory; pub const Tensor = struct { var current_id: std.atomic.Value(usize) = .{ .raw = 1 }; @@ -154,7 +154,10 @@ pub const Tensor = struct { if (ctx.platform.target == .cpu) return self; const frontend_attributes = mlir.dictionaryAttribute(ctx.mlir_ctx, &.{ - .named(ctx.mlir_ctx, "_xla_buffer_placement", mlir.stringAttribute(ctx.mlir_ctx, kind.pjrtName())), + .named(ctx.mlir_ctx, "_xla_buffer_placement", mlir.stringAttribute( + ctx.mlir_ctx, + ctx.platform.memoryKind(kind), + )), }); const op = dialects.stablehlo.custom_call( @@ -474,7 +477,7 @@ pub const Tensor = struct { return .{ ._state = .init(.{2}, .u64) }; } - pub fn initBuffer(platform: Platform, seed: u128, io: std.Io) !Bufferized(Rng) { + pub fn initBuffer(platform: *const Platform, seed: u128, io: std.Io) !Bufferized(Rng) { return .{ ._state = try .fromBytes(io, platform, Shape.init(.{2}, .u64), std.mem.asBytes(&seed)), }; @@ -3988,7 +3991,7 @@ pub const Tensor = struct { } /// Returns the output memory kind of the tensor. - pub fn outputMemoryKind(self: Tensor) Buffer.Memory { + pub fn outputMemoryKind(self: Tensor) Memory.Kind { return CompilationContext.current().currentScope().id_to_output_memory_kind.get(self.id) orelse .device; } }; diff --git a/zml/zml.zig b/zml/zml.zig index a494b298..37252e74 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -22,12 +22,15 @@ pub const exe = @import("exe.zig"); pub const Exe = exe.Exe; pub const floats = @import("floats.zig"); pub const io = @import("io.zig"); +pub const mem = @import("mem.zig"); +pub const Bufferized = mem.Bufferized; pub const meta = @import("meta.zig"); pub const mlir = @import("mlirx.zig"); pub const module = @import("module.zig"); pub const nn = @import("nn.zig"); pub const ops = @import("ops.zig"); pub const platform = @import("platform.zig"); +pub const Memory = platform.Memory; pub const Platform = platform.Platform; pub const Target = platform.Target; pub const CompilationOptions = platform.CompilationOptions; @@ -40,13 +43,11 @@ pub const tensor = @import("tensor.zig"); pub const Tensor = tensor.Tensor; pub const testing = @import("testing.zig"); -/// Return a clone of a type with Tensors replaced by Buffer. -/// Non-Tensor metadata is stripped out of the resulting struct. -/// Recursively descends into the type. -pub fn Bufferized(comptime T: type) type { - return meta.MapRestrict(Tensor, Buffer).map(T); -} - test "zml" { std.testing.refAllDecls(@This()); } + +pub const KiB = 1024; +pub const MiB = 1024 * KiB; +pub const GiB = 1024 * MiB; +pub const TiB = 1024 * GiB; From 5a1d5041d9aa916333858d901e2ce47bb894e49a Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 4 Feb 2026 16:57:12 +0100 Subject: [PATCH 265/881] zml/platform: small memory leak fix (#377) devices and memories are not allocated within the arena when the rest of the platform is. --- zml/platform.zig | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/zml/platform.zig b/zml/platform.zig index 346c95e1..303e24b2 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -232,17 +232,16 @@ pub const Platform = struct { { for (pjrt_devices, devices) |pjrt_device, *platform_device| { - platform_device.* = try .init(allocator, pjrt_device, platform, memories); + platform_device.* = try .init(arena.allocator(), pjrt_device, platform, memories); } for (pjrt_memories, memories) |pjrt_memory, *platform_memory| { - platform_memory.* = try .init(allocator, pjrt_memory, platform, devices); + platform_memory.* = try .init(arena.allocator(), pjrt_memory, platform, devices); } } switch (target) { .cuda => { - // TODO(Corentin): provide a better allocator - zml.attention.flashattn.load(allocator, io) catch { + zml.attention.flashattn.load(arena.allocator(), io) catch { log.warn("Failed to load flashattn", .{}); }; zml.attention.flashattn.register(platform) catch { From eb28352018afb7d304405ad122ed36bec89324dc Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 4 Feb 2026 18:03:24 +0100 Subject: [PATCH 266/881] zml/io: memory leak in load (#378) --- zml/io.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/zml/io.zig b/zml/io.zig index 814b4e6a..c39486bc 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -527,6 +527,7 @@ pub fn load( .progress = opts.progress, .group = .init(opts.parallelism), }; + defer allocator.free(walk_ctx.buffers); defer if (opts.total_bytes) |total_bytes_ptr| { total_bytes_ptr.* = walk_ctx.total.load(.monotonic); From 1fd90f1e458e8cc845a2b5ae8a3fe82b21d6431c Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 9 Feb 2026 12:01:22 +0100 Subject: [PATCH 267/881] Various improvements (#379) This pull request includes several improvements and bug fixes across the codebase, focusing on memory handling, correctness in tensor testing, and API consistency. The most significant changes are grouped below: ### Tensor Testing and Memory Management Improvements * Fixed a bug in `zml/testing.zig`'s `testlayer` by updating the callback signature to take a mutable context and increment the index correctly, ensuring tensors are correctly initialized. * Improved memory management for expected output slices in `testlayer` by switching to `zml.Slice.alloc` for allocation, using `readSliceAll`, and freeing only the data instead of the whole slice, removing the potential alignment problems. * Enhanced test logging to include the tolerance value in success messages for better diagnostics. ### API and Code Quality Enhancements * Increased the evaluation branch quota in `MapRestrict` to support more complex compile-time type mapping, preventing quota exceeded errors. * Updated the `fromCStruct` function in `pjrt/pjrt.zig` to accept a `meta.Struct`. ### StableHLO Output Operand Alias Handling * Improved handling of output operand aliases in `mlir/dialects/stablehlo/stablehlo.zig::custom_call` by passing `output_tuple_indices` for multi-result functions, increasing correctness in custom call attribute construction. --- mlir/dialects/stablehlo/stablehlo.zig | 5 +++-- pjrt/pjrt.zig | 2 +- zml/meta.zig | 1 + zml/testing.zig | 13 ++++++++----- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index a3b4b5d2..96979112 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -761,9 +761,10 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result if (opts.output_operand_aliases) |output_operand_aliases| { var buffer: stdx.BoundedArray(*const mlir.Attribute, MAX_RESULTS) = .{}; - for (output_operand_aliases) |alias| { + for (output_operand_aliases, 0..) |alias, output_index| { + const output_tuple_indices = if (result_types.len > 1) &[1]i64{@intCast(output_index)} else &.{}; buffer.appendAssumeCapacity( - outputOperandAliasAttribute(ctx, .{ .operand_index = alias }), + outputOperandAliasAttribute(ctx, .{ .operand_index = alias, .output_tuple_indices = output_tuple_indices }), ); } attrs.appendAssumeCapacity( diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index d68cff1b..37f75c12 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -662,7 +662,7 @@ pub const Device = opaque { pub const zeroes = std.mem.zeroes(MemoryStats); - fn fromCStruct(v: c.PJRT_Device_MemoryStats_Args) MemoryStats { + fn fromCStruct(v: meta.Struct(c.PJRT_Device_MemoryStats_Args)) MemoryStats { return .{ .bytes_in_use = @intCast(v.bytes_in_use), .peak_bytes_in_use = if (v.peak_bytes_in_use_is_set) @intCast(v.peak_bytes_in_use) else null, diff --git a/zml/meta.zig b/zml/meta.zig index 17196752..bfd001d2 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -305,6 +305,7 @@ test mapAlloc { /// The returned type will contains only `To` making it easy for the compiler to produce compact layout. /// Used by `zml.Bufferized` to strip compile time arguments from a model struct. pub fn MapRestrict(From: type, To: type) type { + @setEvalBranchQuota(100000); return struct { pub fn map(T: type) type { switch (T) { diff --git a/zml/testing.zig b/zml/testing.zig index fc0e3e6f..a1b8076e 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -188,11 +188,12 @@ pub fn testlayer( const store_input = activation_store.withPrefix(name ++ ".in"); var ctx = LocalContext{ .activation_store = store_input }; try zml.meta.visit(struct { - fn cb(ctx_: *const LocalContext, tensor: *zml.Tensor) !void { + fn cb(ctx_: *LocalContext, tensor: *zml.Tensor) !void { var buffer: [256]u8 = undefined; const subkey = std.fmt.bufPrint(&buffer, "{d}", .{ctx_.index}) catch unreachable; tensor.* = ctx_.activation_store.createTensor(subkey); + ctx_.index += 1; } }.cb, &ctx, &args); @@ -237,10 +238,12 @@ pub fn testlayer( var reader = try store_output.getReader(subkey, io, &reader_buffer); defer reader.deinit(); const shape = store_output.getShape(subkey).?; - const expected_bytes = try reader.interface.readAlloc(allocator, shape.byteSize()); - break :b zml.Slice.init(shape, expected_bytes); + const expected_slice: zml.Slice = try .alloc(allocator, shape); + errdefer expected_slice.free(allocator); + try reader.interface.readSliceAll(expected_slice.data()); + break :b expected_slice; }; - defer expected_slice.free(allocator); + defer allocator.free(expected_slice.data()); const output_slice = try results[i].toSliceAlloc(allocator, io); defer output_slice.free(allocator); @@ -256,7 +259,7 @@ pub fn testlayer( } if (failed) return error.TestUnexpectedResult; - log.info("all good for {s} !", .{name}); + log.info("all good for {s} ! (tolerance: {e})", .{ name, tolerance }); } fn center(slice: anytype, i: usize) @TypeOf(slice) { From 24ca08692dc6db0daa942bd12fafa073faf5ab00 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 9 Feb 2026 18:02:48 +0000 Subject: [PATCH 268/881] zml: fix tests --- zml/nn.zig | 14 +++++------ zml/ops.zig | 2 +- zml/tensor.zig | 66 ++++++++++++++++++++++++------------------------- zml/testing.zig | 8 +++--- 4 files changed, 46 insertions(+), 44 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index d4abcd89..cdc2b66b 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -602,7 +602,7 @@ test nearest { var input_3d_basic_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_3d_basic.shape(), std.mem.sliceAsBytes(&[1][1][2]i32{.{.{ 1, 2 }}})); defer input_3d_basic_buffer.deinit(); - const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_3d_basic_buffer}); + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_3d_basic_buffer}); defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 1, 1, 6 }, result.shape().dims()); @@ -622,7 +622,7 @@ test nearest { })); defer input_3d_advanced_buffer.deinit(); - const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_3d_advanced_buffer}); + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_3d_advanced_buffer}); defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 2, 3, 8 }, result.shape().dims()); @@ -650,7 +650,7 @@ test nearest { var input_4d_basic_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_basic.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); defer input_4d_basic_buffer.deinit(); - const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_4d_basic_buffer}); + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_4d_basic_buffer}); defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 1, 1, 6, 6 }, result.shape().dims()); @@ -679,7 +679,7 @@ test nearest { } })); defer input_4d_advanced_buffer.deinit(); - const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_4d_advanced_buffer}); + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_4d_advanced_buffer}); defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 2, 2, 4, 4 }, result.shape().dims()); @@ -724,7 +724,7 @@ test nearest { var input_5d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_5d.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); defer input_5d_buffer.deinit(); - const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_5d_buffer}); + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_5d_buffer}); defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 1, 1, 2, 4, 4 }, result.shape().dims()); @@ -1077,7 +1077,7 @@ test sampleTokens { }) |logits_expected| { const logits, const expected: i32 = logits_expected; - const activations_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, activations.shape(), std.mem.sliceAsBytes(&logits)); + var activations_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, activations.shape(), std.mem.sliceAsBytes(&logits)); defer activations_buffer.deinit(); var sampled, rng_buffer = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, sampleTokens, .{ activations_buffer, rng_buffer }); @@ -1115,7 +1115,7 @@ pub const DynamicSamplingStrategy = struct { pub fn makeBuffers( io: std.Io, - platform: zml.Platform, + platform: *const zml.Platform, dtype: zml.DataType, opts: Opts, ) !zml.Bufferized(DynamicSamplingStrategy) { diff --git a/zml/ops.zig b/zml/ops.zig index bc082812..ff1bc93b 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -374,7 +374,7 @@ test "triton" { var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), std.mem.sliceAsBytes(&[1]f32{3})); defer b_buffer.deinit(); - const results = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, TritonMod.forward, .{ a_buffer, b_buffer }); + var results = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, TritonMod.forward, .{ a_buffer, b_buffer }); defer results[0].deinit(); defer results[1].deinit(); diff --git a/zml/tensor.zig b/zml/tensor.zig index f101ed5e..15173b96 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -318,7 +318,7 @@ pub const Tensor = struct { const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.fmod, .{ input, d }, platform); defer exe.deinit(); - const input_buffer = try zml.Buffer.fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&i)); + var input_buffer = try zml.Buffer.fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&i)); defer input_buffer.deinit(); const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.fmod, .{input_buffer}); @@ -582,10 +582,10 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.uniformStats, .{ Rng.init(), Shape.init(.{1024}, .f32), .{ .min = -2, .max = 10 } }, platform); defer exe.deinit(); - const rng_buffer = try Rng.initBuffer(platform, 1234, std.testing.io); + var rng_buffer = try Rng.initBuffer(platform, 1234, std.testing.io); defer rng_buffer._state.deinit(); - const rand, const stats = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Stats.uniformStats, .{rng_buffer}); + var rand, var stats = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Stats.uniformStats, .{rng_buffer}); defer rand._state.deinit(); defer stats.mean.deinit(); defer stats.variance.deinit(); @@ -689,12 +689,12 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.gumbelStats, .{ Rng.init(), tgt_dist }, platform); defer exe.deinit(); - const rng_buffer = try Rng.initBuffer(platform, 1234, std.testing.io); + var rng_buffer = try Rng.initBuffer(platform, 1234, std.testing.io); defer rng_buffer._state.deinit(); - const tgt_dist_buffer: Buffer = try .fromBytes(std.testing.io, platform, tgt_dist.shape(), std.mem.sliceAsBytes(&tgt_dist_data)); + var tgt_dist_buffer: Buffer = try .fromBytes(std.testing.io, platform, tgt_dist.shape(), std.mem.sliceAsBytes(&tgt_dist_data)); defer tgt_dist_buffer.deinit(); - const rand, const stats = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Stats.gumbelStats, .{ rng_buffer, tgt_dist_buffer }); + var rand, var stats = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Stats.gumbelStats, .{ rng_buffer, tgt_dist_buffer }); defer rand._state.deinit(); defer stats.mean.deinit(); defer stats.variance.deinit(); @@ -1021,10 +1021,10 @@ pub const Tensor = struct { const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.convert, .{ x_d, .f4e2m1 }, platform); defer exe.deinit(); - const x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), std.mem.sliceAsBytes(&x)); + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), std.mem.sliceAsBytes(&x)); defer x_d_buffer.deinit(); - const x_f4_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); + var x_f4_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); const x_f4_xla = try x_f4_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); defer x_f4_xla.free(std.testing.allocator); @@ -1042,10 +1042,10 @@ pub const Tensor = struct { const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.convert, .{ x_d, .f8e3m4 }, platform); defer exe.deinit(); - const x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), std.mem.sliceAsBytes(&x)); + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), std.mem.sliceAsBytes(&x)); defer x_d_buffer.deinit(); - const x_f8e3_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); + var x_f8e3_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); const x_f8e3_xla = try x_f8e3_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); defer x_f8e3_xla.free(std.testing.allocator); @@ -1235,10 +1235,10 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.leakyReLU, .{ input, 0.1 }, platform); defer exe.deinit(); - const input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6884, 1.6795 })); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6884, 1.6795 })); defer input_buffer.deinit(); - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.leakyReLU, .{input_buffer}); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.leakyReLU, .{input_buffer}); defer res.deinit(); const expectation: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[2]f32{ -0.0688, 1.6795 })); @@ -1391,10 +1391,10 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cumsum, .{x}, platform); defer exe.deinit(); - const x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } })); defer x_buffer.deinit(); - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._cumsum, .{x_buffer}); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._cumsum, .{x_buffer}); defer res.deinit(); try std.testing.expectEqual([2][5]f32{ .{ 0, 1, 2, 2, 3 }, .{ 3, 4, 4, 6, 7 } }, try res.getValue([2][5]f32, std.testing.io)); @@ -1591,7 +1591,7 @@ pub const Tensor = struct { const x: Tensor = .init(.{ 2, 5 }, .f32); - const x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); defer x_buffer.deinit(); // Wrap slice1d to hide the anytype in the signature. @@ -1610,7 +1610,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._slice1dAxis, .{ x, ax, slice_ }, platform); defer exe.deinit(); - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._slice1dAxis, .{x_buffer}); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._slice1dAxis, .{x_buffer}); defer res.deinit(); try std.testing.expectEqual(expectation, try res.getValue(@TypeOf(expectation), std.testing.io)); @@ -2427,9 +2427,9 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._gatherSlices, .{ operand, Shape.init(.{ .b = 2, .c = 3 }, .u16), indices, .{} }, platform); defer exe.deinit(); - const indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, indices.shape(), std.mem.sliceAsBytes(&[2][2]i32{ .{ 2, 1 }, .{ 0, 3 } })); + var indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, indices.shape(), std.mem.sliceAsBytes(&[2][2]i32{ .{ 2, 1 }, .{ 0, 3 } })); defer indices_buffer.deinit(); - const operand_buffer: zml.Buffer = b: { + var operand_buffer: zml.Buffer = b: { const temp_func = struct { fn forward() Tensor { return Tensor.arange(.{ .end = 2 * 4 * 6 }, .u16).reshape(.{ .a = 2, .b = 4, .c = 6 }); @@ -2442,7 +2442,7 @@ pub const Tensor = struct { break :b buffer; }; defer operand_buffer.deinit(); - const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._gatherSlices, .{ operand_buffer, indices_buffer }); + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._gatherSlices, .{ operand_buffer, indices_buffer }); defer result.deinit(); const expected: zml.Slice = .init(Shape.init(.{ 2, 2, 2, 3 }, .u16), std.mem.sliceAsBytes(&[2][2][2][3]u16{ @@ -2814,7 +2814,7 @@ pub const Tensor = struct { var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, 4.1, 7.9, 0, 7.9 }})); defer x_buffer.deinit(); - const res = try zml.testing.autoCall(allocator, std.testing.io, &exe, ArgMaxTest._fwd, .{x_buffer}); + var res = try zml.testing.autoCall(allocator, std.testing.io, &exe, ArgMaxTest._fwd, .{x_buffer}); defer res.values.deinit(); defer res.indices.deinit(); const max_ = res.values.getValue(f32, std.testing.io); @@ -2828,7 +2828,7 @@ pub const Tensor = struct { var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, std.math.nan(f32), 7.9, 0, 7.9 }})); defer x_buffer.deinit(); - const res = try zml.testing.autoCall(allocator, std.testing.io, &exe, ArgMaxTest._fwd, .{x_buffer}); + var res = try zml.testing.autoCall(allocator, std.testing.io, &exe, ArgMaxTest._fwd, .{x_buffer}); defer res.values.deinit(); defer res.indices.deinit(); const max_ = try res.values.getValue(f32, std.testing.io); @@ -3307,7 +3307,7 @@ pub const Tensor = struct { var z_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, z.shape(), std.mem.asBytes(&z_value)); defer z_buffer.deinit(); - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.dynamicSlice1d, .{ x_buffer, .{ .start = z_buffer } }); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.dynamicSlice1d, .{ x_buffer, .{ .start = z_buffer } }); defer res.deinit(); try std.testing.expectEqual(expectation, try res.getValue(@TypeOf(expectation), std.testing.io)); @@ -3433,7 +3433,7 @@ pub const Tensor = struct { var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{4})); defer ids_buffer.deinit(); - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ .a = ids_buffer }, y_buffer }); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ .a = ids_buffer }, y_buffer }); defer res.deinit(); try std.testing.expectEqual([10]f32{ 0, 1, 2, 3, -1, -1, 6, 7, 8, 9 }, try res.getValue([10]f32, std.testing.io)); @@ -3459,7 +3459,7 @@ pub const Tensor = struct { var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer ids_buffer.deinit(); - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, ids_buffer, y_buffer }); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, ids_buffer, y_buffer }); defer res.deinit(); try std.testing.expectEqualDeep( @@ -3488,7 +3488,7 @@ pub const Tensor = struct { var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer ids_buffer.deinit(); - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, ids_buffer, y_buffer }); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, ids_buffer, y_buffer }); defer res.deinit(); try std.testing.expectEqualDeep( @@ -3520,7 +3520,7 @@ pub const Tensor = struct { var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer idx_b_buffer.deinit(); - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ .a = idx_a_buffer, .b = idx_b_buffer }, y_buffer }); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ .a = idx_a_buffer, .b = idx_b_buffer }, y_buffer }); defer res.deinit(); try std.testing.expectEqualDeep( @@ -3552,7 +3552,7 @@ pub const Tensor = struct { var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer idx_b_buffer.deinit(); - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ idx_a_buffer, idx_b_buffer }, y_buffer }); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ idx_a_buffer, idx_b_buffer }, y_buffer }); defer res.deinit(); try std.testing.expectEqualDeep( @@ -3627,7 +3627,7 @@ pub const Tensor = struct { var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][2]u8{ .{ 1, 2 }, .{ 3, 4 } })); defer x_buffer.deinit(); - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._toDiag, .{x_buffer}); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._toDiag, .{x_buffer}); defer res.deinit(); try std.testing.expectEqual( [2][2][2]u8{ .{ @@ -3685,7 +3685,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 0 }, platform); defer exe.deinit(); - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); defer res.deinit(); try std.testing.expectEqual( @@ -3701,7 +3701,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 1 }, platform); defer exe.deinit(); - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); defer res.deinit(); try std.testing.expectEqual( @@ -3717,7 +3717,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, -1 }, platform); defer exe.deinit(); - const res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); defer res.deinit(); try std.testing.expectEqual( @@ -4061,7 +4061,7 @@ test "Tensor.maxPool1d" { var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&data)); defer x_buffer.deinit(); - const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, MaxPool._fwd, .{x_buffer}); + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, MaxPool._fwd, .{x_buffer}); defer result.values.deinit(); defer result.indices.deinit(); @@ -4100,7 +4100,7 @@ test "Tensor.maxPool2d" { var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&data)); defer x_buffer.deinit(); - const result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, MaxPool._fwd, .{x_buffer}); + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, MaxPool._fwd, .{x_buffer}); defer result.values.deinit(); defer result.indices.deinit(); diff --git a/zml/testing.zig b/zml/testing.zig index a1b8076e..f4fdd461 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -9,12 +9,14 @@ const zml = @import("zml.zig"); const log = std.log.scoped(.@"zml/testing"); -var _platform: ?Platform = null; +var _platform: ?*const Platform = null; -pub fn env() Platform { +pub fn env() *const Platform { if (!builtin.is_test) @compileError("Cannot use zml.testing.env outside of a test block"); if (_platform == null) { - _platform = Platform.auto(std.testing.allocator, std.testing.io, .{}) catch unreachable; + _platform = Platform.auto(std.heap.c_allocator, std.testing.io, .{ + .cuda = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.85 } } }, + }) catch unreachable; } return _platform.?; From ef60536186707ccc31cb4bdb117277daa4df1005 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 9 Feb 2026 18:03:03 +0000 Subject: [PATCH 269/881] zml/slice: fix formatting Also added a test --- zml/slice.zig | 53 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/zml/slice.zig b/zml/slice.zig index 3a26a44c..d9fb9956 100644 --- a/zml/slice.zig +++ b/zml/slice.zig @@ -144,29 +144,33 @@ pub const Slice = struct { try writer.splatByteAll(' ', indent_level); _ = try writer.write("{\n"); - // Write first rows const n: u64 = @intCast(slice.shape.dim(0)); - for (0..@min(num_rows, n)) |d| { - const byte_stride: usize = @intCast(slice.shape.computeByteStrides().get(0)); - const sub_slice: Slice = .init(slice.shape.drop(0), slice.constData()[d * byte_stride ..][0..byte_stride]); - try sub_slice.prettyPrintIndented(writer, num_rows, indent_level + 2, options); - if (slice.shape.rank() > 1) { + if (n >= num_rows) { + // Write first rows + for (0..@min(num_rows, n)) |d| { + const byte_stride: usize = @intCast(slice.shape.computeByteStrides().get(0)); + const sub_slice: Slice = .init(slice.shape.drop(0), slice.constData()[d * byte_stride ..][0..byte_stride]); + try sub_slice.prettyPrintIndented(writer, num_rows, indent_level + 2, options); try writer.writeAll(",\n"); } - } - if (n < num_rows) return; - // Skip middle rows - if (n > 2 * num_rows) { - try writer.splatByteAll(' ', indent_level + 2); - _ = try writer.write("...\n"); - } - // Write last rows - for (@max(n - num_rows, num_rows)..n) |d| { - const byte_stride: usize = @intCast(slice.shape.computeByteStrides().get(0)); - const sub_slice: Slice = .init(slice.shape.drop(0), slice.constData()[d * byte_stride ..][0..byte_stride]); - try sub_slice.prettyPrintIndented(writer, num_rows, indent_level + 2, options); - if (slice.shape.rank() > 1) { + // Skip middle rows + if (n > 2 * num_rows) { + try writer.splatByteAll(' ', indent_level + 2); + _ = try writer.write("...\n"); + } + // Write last rows + for (@max(n - num_rows, num_rows)..n) |d| { + const byte_stride: usize = @intCast(slice.shape.computeByteStrides().get(0)); + const sub_slice: Slice = .init(slice.shape.drop(0), slice.constData()[d * byte_stride ..][0..byte_stride]); + try sub_slice.prettyPrintIndented(writer, num_rows, indent_level + 2, options); + try writer.writeAll(",\n"); + } + } else { + for (0..n) |d| { + const byte_stride: usize = @intCast(slice.shape.computeByteStrides().get(0)); + const sub_slice: Slice = .init(slice.shape.drop(0), slice.constData()[d * byte_stride ..][0..byte_stride]); + try sub_slice.prettyPrintIndented(writer, num_rows, indent_level + 2, options); try writer.writeAll(",\n"); } } @@ -236,6 +240,17 @@ test "slice pretty print rank 3" { try std.testing.expectFmt(expected, "{d}", .{slice}); } +test "slice pretty print rank 2" { + const data: [4]i32 = .{ 0, 1, 2, 3 }; + const slice = Slice.init(.init(.{ 1, 4 }, .i32), std.mem.asBytes(&data)); + const expected = + \\{ + \\ {0,1,2,3}, + \\} + ; + try std.testing.expectFmt(expected, "{d}", .{slice}); +} + test "slice pretty print rank 1" { const data: [4]i32 = .{ 0, 1, 2, 3 }; const slice = Slice.init(.init(.{4}, .i32), std.mem.asBytes(&data)); From 2218ec0f3f3a87abceffdaf0e943a1847845281d Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 10 Feb 2026 10:11:48 +0100 Subject: [PATCH 270/881] V2: `Tensor.print()` (#380) Added `Tensor.print()` back --- pjrt/ffi.zig | 1 + pjrt/pjrt.zig | 2 +- zml/ops.zig | 7 ++++ zml/platform.zig | 102 ++++++++++++++++++++++++++++++++++++++++++++++- zml/tensor.zig | 7 ++-- 5 files changed, 113 insertions(+), 6 deletions(-) diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index b8d2a241..83fad4d2 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -321,6 +321,7 @@ pub const Attrs = extern struct { const Attr = extern union { scalar: Scalar, array: Array, + string: ByteSpan, }; pub const Scalar = extern struct { diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 37f75c12..47964582 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -495,7 +495,7 @@ pub const Client = opaque { dims: []const i64, element_type: BufferType, layout: MemoryLayout, - device: *const Device, + device: ?*const Device = null, on_delete_callback: *const fn (device_buffer_ptr: ?*anyopaque, ctx: ?*anyopaque) callconv(.c) void = &struct { fn call(_: ?*anyopaque, _: ?*anyopaque) callconv(.c) void {} }.call, diff --git a/zml/ops.zig b/zml/ops.zig index ff1bc93b..757949cd 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -986,6 +986,13 @@ fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs else => @panic("Unsupported DataType"), }, .bool => mlir.boolAttribute(ctx.mlir_ctx, @field(metadata, field.name)), + .pointer => |pointer_info| switch (pointer_info.size) { + .slice => if (pointer_info.child == u8) + mlir.stringAttribute(ctx.mlir_ctx, @field(metadata, field.name)) + else + @panic("Unsupported pointer type in metadata"), + else => @panic("Unsupported pointer type in metadata"), + }, else => @compileError("Unsupported metadata type: " ++ @typeName(field.type)), }; metadata_attributes[i] = mlir.NamedAttribute.named(ctx.mlir_ctx, field.name, attribute); diff --git a/zml/platform.zig b/zml/platform.zig index 303e24b2..f291267d 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -2,7 +2,7 @@ const std = @import("std"); const c = @import("c"); const pjrt = @import("pjrt"); -const pjrtx = @import("pjrt"); +const pjrtx = @import("pjrtx.zig"); const platforms = @import("platforms"); pub const Target = platforms.Platform; const stdx = @import("stdx"); @@ -251,6 +251,18 @@ pub const Platform = struct { else => {}, } + if (platform.pjrt_api.ffi()) |ffi| b: { + const platform_name = switch (target) { + .cuda => "cuda", + .rocm => "rocm", + .cpu => "host", + else => break :b, + }; + ffi.register(platform.pjrt_api, "zml$print", platform_name, printCallback, .{ .command_buffer_compatible = false }) catch |e| { + log.warn("Failed to register \"print\" callback, error: {}", .{e}); + }; + } + return platform; } @@ -460,3 +472,91 @@ pub const cuda = struct { } else null; } }; + +fn dataTypeFromFfiDataType(ffi_dt: pjrt.ffi.DataType) zml.DataType { + return switch (ffi_dt) { + .bool => .bool, + .i8 => .i8, + .i16 => .i16, + .i32 => .i32, + .i64 => .i64, + .u8 => .u8, + .u16 => .u16, + .u32 => .u32, + .u64 => .u64, + .f16 => .f16, + .f32 => .f32, + .f64 => .f64, + .bf16 => .bf16, + .c64 => .c64, + .c128 => .c128, + .f8e5m2 => .f8e5m2, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3b11fnuz => .f8e4m3b11fnuz, + .f8e5m2fnuz => .f8e5m2fnuz, + .f8e4m3fnuz => .f8e4m3fnuz, + else => unreachable, + }; +} + +fn shapeFromFfiBuffer(buffer: *const pjrt.ffi.Buffer) zml.Shape { + return .init(buffer.dims(), dataTypeFromFfiDataType(buffer.dtype)); +} + +fn getScalarAttributeAs(comptime T: type, call_frame: *pjrt.ffi.CallFrame, attribute_name: []const u8) ?T { + const attribute = call_frame.attrs.getByName(.scalar, attribute_name) orelse return null; + return attribute.get(T); +} + +fn printCallback(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { + return printCallbackInner(call_frame) catch |e| b: { + log.err("Error in print callback: {}", .{e}); + break :b pjrt.ffi.Error.create(call_frame.api, .unknown, "Unknown"); + }; +} + +fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { + if (call_frame.registeringHook()) return null; + + const pjrt_api: *pjrt.Api = @ptrFromInt(getScalarAttributeAs(u64, call_frame, "pjrt_api").?); + const pjrt_client: *pjrt.Client = @ptrFromInt(getScalarAttributeAs(u64, call_frame, "pjrt_client").?); + + const device_ordinal: usize = @intCast(try call_frame.ctx.getDeviceOrdinal(call_frame.api)); + + const buffer = call_frame.args.buffers()[0]; + const shape = shapeFromFfiBuffer(buffer); + + // NOTE(Corentin): This is a hack. We take the first non device memory, hoping that it's host visible, + // and copy the buffer there to read it on the CPU and print it. + const device = pjrt_client.devices(pjrt_api)[device_ordinal]; + const addressable_memories = device.addressableMemories(pjrt_api); + const first_non_device_memory = for (addressable_memories) |memory| { + if (!std.mem.eql(u8, memory.kind_(pjrt_api), "device")) break memory; + } else return error.MemoryNotFound; + + var pjrt_buffer = try pjrt_client.createViewOfDeviceBuffer(pjrt_api, .{ + .data = buffer.data, + .dims = shape.dims(), + .element_type = pjrtx.bufferTypeFromDtype(shape.dtype()), + .device = device, + .layout = .{ + .tiled = .{ + .minor_to_major = zml.constants.minorToMajor(shape.rank()), + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }, + }); + + pjrt_buffer = try pjrt_buffer.copyToMemory(pjrt_api, first_non_device_memory); + try pjrt_buffer.readyEvent(pjrt_api).awaitRaw(pjrt_api); + + const host_visible_data: [*]u8 = @ptrCast(@alignCast(try pjrt_buffer.opaqueDeviceMemoryDataPointer(pjrt_api))); + + const slice: zml.Slice = .init(shape, host_visible_data[0..shape.byteSize()]); + const name = call_frame.attrs.getByName(.string, "name").?.slice(); + + log.info("{s}: {d}", .{ name, slice }); + + return null; +} diff --git a/zml/tensor.zig b/zml/tensor.zig index 15173b96..6d27f606 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -3969,12 +3969,11 @@ pub const Tensor = struct { } /// Insert code that will print the content of the given buffer at runtime. + /// Use the name parameter to differentiate different print calls in the output. /// Only for debug purpose, it inserts device to host synchronization /// so it will slow down the program execution. - pub fn print(input: Tensor) Tensor { - // TODO(Corentin) - _ = input; - @panic("Unimplemented"); + pub fn print(input: Tensor, name: []const u8) void { + _ = ops.customCall("zml$print", .{input}, .{}, .{ .name = name }, .{ .has_side_effect = true }); } fn mlirCtx() *mlir.Context { From 2f9f01e9b9b63ae015ca6f365b4f654eb3452027 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 10 Feb 2026 17:25:13 +0000 Subject: [PATCH 271/881] zml/tensor: Added checks on `Tensor.dot()` --- zml/tensor.zig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zml/tensor.zig b/zml/tensor.zig index 6d27f606..957b4a44 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1071,6 +1071,9 @@ pub const Tensor = struct { /// Axes with the same tag on both sides, and which aren't contracting, /// are considered "batching axes". pub fn dot(lhs: Tensor, rhs: Tensor, args: anytype) Tensor { + stdx.debug.assert(lhs.shape().hasTag(args) != null, "Expected lhs to have {} tag, got {f}", .{ args, lhs.shape() }); + stdx.debug.assert(rhs.shape().hasTag(args) != null, "Expected rhs to have {} tag, got {f}", .{ args, rhs.shape() }); + const lhs_contracting_dim: i8 = @intCast(lhs.shape().hasTag(args).?); const rhs_contracting_dim: i8 = @intCast(rhs.shape().hasTag(args).?); From 7d497907efee406413c7c53b7b629cb241b73dda Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 11 Feb 2026 14:14:37 +0100 Subject: [PATCH 272/881] zml/testing: updated testLayer to new API Also renamed to testLayer --- zml/testing.zig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zml/testing.zig b/zml/testing.zig index f4fdd461..a7b91f90 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -160,10 +160,10 @@ fn countWithPrefix(store: zml.io.TensorStore.View, prefix: []const u8) usize { /// /// The function arguments need to be valid when initialized to undefined, meaning no slices, /// no unions, no pointers. -pub fn testlayer( +pub fn testLayer( allocator: std.mem.Allocator, io: std.Io, - platform: zml.Platform, + platform: *const zml.Platform, layer: anytype, comptime func: std.meta.DeclEnum(@TypeOf(layer)), activation_store: zml.io.TensorStore.View, @@ -215,7 +215,7 @@ pub fn testlayer( var exe_results = try exe.results(allocator); defer exe_results.deinit(allocator); - var args_buffers = try zml.io.loadBuffersFromId(allocator, io, platform, args, activation_store, .{ .size = 4096, .concurrency = 1 }, .{ .size = 4096, .concurrency = 1 }); + var args_buffers = try zml.io.load(ArgsT, &args, allocator, io, platform, .{ .dma_chunks = 1, .dma_chunk_size = 4096, .store = activation_store.store, .parallelism = 1 }); defer zml.meta.visit(struct { fn cb(_: void, b: *zml.Buffer) void { b.deinit(); From d5cfd726acf3275121b5a34e2cd61bd5602137aa Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 12 Feb 2026 11:59:09 +0100 Subject: [PATCH 273/881] Better `testLayer` (#383) --- zml/testing.zig | 180 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 153 insertions(+), 27 deletions(-) diff --git a/zml/testing.zig b/zml/testing.zig index a7b91f90..f84d9f62 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -31,31 +31,35 @@ pub fn approxEq(comptime Float: type, l: Float, r: Float, tolerance: Float) bool return closeRel or closeAbs; } +fn toSlice(allocator: std.mem.Allocator, io: std.Io, slice_or_buffer: anytype) !struct { Slice, bool } { + return if (@TypeOf(slice_or_buffer) == zml.Buffer) b: { + const slice = try slice_or_buffer.toSliceAlloc(allocator, io); + break :b .{ slice, true }; + } else .{ slice_or_buffer, false }; +} + /// Testing utility. Accepts both zml.Buffer and zml.HostBuffer but zml.Buffer will be copied to the /// host for comparison ! -pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, tolerance: f32) !void { +pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, opts: CompareOpts) !void { const allocator = if (builtin.is_test) std.testing.allocator else std.heap.smp_allocator; - var left: Slice, const should_free_left = if (@TypeOf(left_) == zml.Buffer) b: { - const slice = try left_.toSliceAlloc(allocator, io); - break :b .{ slice, true }; - } else .{ left_, false }; + var left: Slice, const should_free_left = try toSlice(allocator, io, left_); + defer if (should_free_left) left.free(allocator); - var right: Slice, const should_free_right = if (@TypeOf(right_) == zml.Buffer) b: { - const slice = try right_.toSliceAlloc(allocator, io); - break :b .{ slice, true }; - } else .{ right_, false }; + var right: Slice, const should_free_right = try toSlice(allocator, io, right_); + defer if (should_free_right) right.free(allocator); - defer { - if (should_free_left) left.free(allocator); - if (should_free_right) right.free(allocator); - } - errdefer log.err("\n--> Left: {0d:24.3}\n--> Right: {1d:24.3}", .{ left, right }); + const stderr = std.debug.lockStderr(&.{}); + defer std.debug.unlockStderr(); + const w = &stderr.file_writer.interface; + + //errdefer log.err("\n--> Left: {0d:24.3}\n--> Right: {1d:24.3}", .{ left, right }); if (!std.mem.eql(i64, left.shape.dims(), right.shape.dims())) { - log.err("left.shape() {f} != right.shape() {f}", .{ left.shape, right.shape }); + try w.print("left.shape() {f} != right.shape() {f}\n", .{ left.shape, right.shape }); return error.TestUnexpectedResult; } - if (left.dtype() != right.dtype() and !(left.dtype() == .f16 and right.dtype() == .bf16)) { - log.err("left.dtype ({}) != right.dtype ({})", .{ left.dtype(), right.dtype() }); + + if (left.dtype() != right.dtype()) { + try w.print("left.dtype ({}) != right.dtype ({})\n", .{ left.dtype(), right.dtype() }); return error.TestUnexpectedResult; } @@ -89,11 +93,13 @@ pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, tolerance: f32) => |rt| { const R = rt.toZigType(); const right_data = right.constItems(R); - for (left_data, right_data, 0..) |l, r, i| { - if (!approxEq(f32, zml.floats.floatCast(f32, l), zml.floats.floatCast(f32, r), tolerance)) { - log.err("left.data != right_data.\n < {d:40.3} \n > {d:40.3}\n error at idx {d}: {d:.3} != {d:.3}", .{ stdx.fmt.slice(center(left_data, i)), stdx.fmt.slice(center(right_data, i)), i, left_data[i], right_data[i] }); - return error.TestUnexpectedResult; - } + const compare_report = try compareSlices(allocator, L, R, left_data, right_data, opts); + const ok = !compare_report.nan_or_inf and compare_report.close_fraction >= opts.minimum_close_fraction; + if (ok) { + return; + } else { + try w.print("{f}\n", .{compare_report}); + return error.TestUnexpectedResult; } }, else => unreachable, @@ -107,6 +113,120 @@ pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, tolerance: f32) } } +pub const CompareOpts = struct { + absolute_tolerance: f32 = 1e-3, + relative_tolerance: f32 = 1e-2, + epsilon_relative: f32 = 1e-6, + minimum_close_fraction: f32 = 0.999, +}; + +pub const CompareReport = struct { + nan_or_inf: bool, + max_absolute_error: f32, + mean_absolute_error: f32, + rmse: f32, + + close_fraction: f32, + + p50_absolute_error: f32, + p90_absolute_error: f32, + p99_absolute_error: f32, + p999_absolute_error: f32, + + pub fn format( + self: @This(), + writer: *std.Io.Writer, + ) std.Io.Writer.Error!void { + try writer.print( + \\ nan_or_inf: {} + \\ max_absolute_error: {d} + \\ mean_absolute_error: {d} + \\ rmse: {d} + \\ close_fraction: {d} + \\ p50_absolute_error: {d} + \\ p90_absolute_error: {d} + \\ p99_absolute_error: {d} + \\ p999_absolute_error: {d} + , .{ + self.nan_or_inf, + self.max_absolute_error, + self.mean_absolute_error, + self.rmse, + self.close_fraction, + self.p50_absolute_error, + self.p90_absolute_error, + self.p99_absolute_error, + self.p999_absolute_error, + }); + } +}; + +pub fn compareSlices(allocator: std.mem.Allocator, comptime L: type, comptime R: type, left: []const L, right: []const R, opts: CompareOpts) !CompareReport { + var nan_or_inf: bool = false; + var max_absolute_error: f32 = 0; + var sum_absolute_error: f64 = 0; + var sum_squared_error: f64 = 0; + var count_close: usize = 0; + var absolute_errors = try allocator.alloc(f32, left.len); + defer allocator.free(absolute_errors); + var relative_errors = try allocator.alloc(f32, left.len); + defer allocator.free(relative_errors); + + for (left, right, 0..) |l, r, i| { + const l_f32 = zml.floats.floatCast(f32, l); + const r_f32 = zml.floats.floatCast(f32, r); + if (!std.math.isFinite(l_f32) or !std.math.isFinite(r_f32)) { + nan_or_inf = true; + continue; + } + const absolute_error = @abs(l_f32 - r_f32); + max_absolute_error = @max(max_absolute_error, absolute_error); + sum_absolute_error += @as(f64, @floatCast(absolute_error)); + sum_squared_error += @as(f64, @floatCast(absolute_error)) * @as(f64, @floatCast(absolute_error)); + + const scale = @max(@abs(l_f32), @abs(r_f32)); + const tolerance = opts.absolute_tolerance + opts.relative_tolerance * scale; + if (absolute_error <= tolerance) { + count_close += 1; + } + + absolute_errors[i] = absolute_error; + const denom = @max(opts.epsilon_relative, scale); + relative_errors[i] = absolute_error / denom; + } + + std.sort.heap(f32, absolute_errors, {}, std.sort.asc(f32)); + std.sort.heap(f32, relative_errors, {}, std.sort.asc(f32)); + + const q = struct { + pub fn q(values: []const f32, frac: f32) f32 { + if (values.len == 0) return 0; + const idx: usize = @intFromFloat(std.math.round(@as(f32, @floatFromInt(values.len - 1)) * frac)); + return values[idx]; + } + }.q; + + const mean_absolute_error: f32 = @floatCast(sum_absolute_error / @as(f64, @floatFromInt(left.len))); + const rmse: f32 = @floatCast(std.math.sqrt(sum_squared_error / @as(f64, @floatFromInt(left.len)))); + const close_fraction = @as(f32, @floatFromInt(count_close)) / @as(f32, @floatFromInt(left.len)); + + const report: CompareReport = .{ + .nan_or_inf = nan_or_inf, + .max_absolute_error = max_absolute_error, + .mean_absolute_error = mean_absolute_error, + .rmse = rmse, + .close_fraction = close_fraction, + .p50_absolute_error = q(absolute_errors, 0.5), + .p90_absolute_error = q(absolute_errors, 0.9), + .p99_absolute_error = q(absolute_errors, 0.99), + .p999_absolute_error = q(absolute_errors, 0.999), + }; + + // TODO: Move + //const ok = !report.nan_or_inf and report.close_fraction >= opts.minimum_close_fraction; + return report; +} + pub fn expectEqualShapes(expected: zml.Shape, actual: zml.Shape) error{TestExpectedEqual}!void { if (expected.eqlWithTags(actual)) return; @@ -169,7 +289,7 @@ pub fn testLayer( activation_store: zml.io.TensorStore.View, comptime name: []const u8, layer_weights: zml.Bufferized(@TypeOf(layer)), - tolerance: f32, + opts: CompareOpts, ) !void { const forward = @field(@TypeOf(layer), @tagName(func)); const ArgsT = stdx.meta.Tail(stdx.meta.FnArgs(forward)); @@ -250,9 +370,12 @@ pub fn testLayer( const output_slice = try results[i].toSliceAlloc(allocator, io); defer output_slice.free(allocator); - zml.testing.expectClose(io, expected_slice, output_slice, tolerance) catch |err| switch (err) { + zml.testing.expectClose(io, expected_slice, output_slice, opts) catch |err| switch (err) { error.TestUnexpectedResult => { - log.err("{s}.{d} doesn't match !", .{ name ++ ".out", i }); + const stderr = std.debug.lockStderr(&.{}); + defer std.debug.unlockStderr(); + const w = &stderr.file_writer.interface; + try w.print("{s}.{d} doesn't match !", .{ name ++ ".out", i }); failed = true; continue; }, @@ -260,8 +383,11 @@ pub fn testLayer( }; } - if (failed) return error.TestUnexpectedResult; - log.info("all good for {s} ! (tolerance: {e})", .{ name, tolerance }); + if (failed) { + log.info("❌ check failed for {s} ! (absolute tolerance: {e} - relative tolerance: {e} - minimum_close_fraction: {d:0>3})", .{ name, opts.absolute_tolerance, opts.relative_tolerance, opts.minimum_close_fraction }); + } else { + log.info("✅ all good for {s} ! (absolute tolerance: {e} - relative tolerance: {e} - minimum_close_fraction: {d:0>3})", .{ name, opts.absolute_tolerance, opts.relative_tolerance, opts.minimum_close_fraction }); + } } fn center(slice: anytype, i: usize) @TypeOf(slice) { From 7bf0f66024b2161ffd88ba17818d4214b26ca6aa Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Thu, 12 Feb 2026 18:31:09 +0100 Subject: [PATCH 274/881] Add linenoise as bazel third party dependency (#384) Co-authored-by: Benjamin Rabier --- MODULE.bazel | 2 +- third_party/linenoise/BUILD.bazel | 1 + third_party/linenoise/linenoise.bazel | 11 +++++++++++ third_party/linenoise/repo.bzl | 9 +++++++++ third_party/non_module_deps.bzl | 2 ++ 5 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 third_party/linenoise/BUILD.bazel create mode 100644 third_party/linenoise/linenoise.bazel create mode 100644 third_party/linenoise/repo.bzl diff --git a/MODULE.bazel b/MODULE.bazel index 309bb188..0f6bc745 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -113,7 +113,7 @@ use_repo(zls, "zls_toolchains") register_toolchains("@zls_toolchains//:all") non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") -use_repo(non_module_deps, "arocc", "com_google_sentencepiece", "flashattn", "mnist", "org_swig_swig", "translate-c", "xla") +use_repo(non_module_deps, "arocc", "com_google_sentencepiece", "flashattn", "linenoise", "mnist", "org_swig_swig", "translate-c", "xla") xla = use_extension("//third_party/xla:xla.bzl", "xla") use_repo( diff --git a/third_party/linenoise/BUILD.bazel b/third_party/linenoise/BUILD.bazel new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/third_party/linenoise/BUILD.bazel @@ -0,0 +1 @@ + diff --git a/third_party/linenoise/linenoise.bazel b/third_party/linenoise/linenoise.bazel new file mode 100644 index 00000000..cd4dbbc9 --- /dev/null +++ b/third_party/linenoise/linenoise.bazel @@ -0,0 +1,11 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # BSD-2-Clause + +cc_library( + name = "linenoise", + srcs = ["linenoise.c"], + hdrs = ["linenoise.h"], +) diff --git a/third_party/linenoise/repo.bzl b/third_party/linenoise/repo.bzl new file mode 100644 index 00000000..7837ac67 --- /dev/null +++ b/third_party/linenoise/repo.bzl @@ -0,0 +1,9 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") + +def repo(): + new_git_repository( + name = "linenoise", + remote = "https://github.com/antirez/linenoise.git", + commit = "dc83cc373ac2058030eb3cf5e404959a26fef112", + build_file = "//third_party/linenoise:linenoise.bazel", + ) diff --git a/third_party/non_module_deps.bzl b/third_party/non_module_deps.bzl index 5790cc3e..fbeee624 100644 --- a/third_party/non_module_deps.bzl +++ b/third_party/non_module_deps.bzl @@ -5,6 +5,7 @@ load("//third_party/org_swig_swig:repo.bzl", org_swig_swig = "repo") load("//third_party/translate-c:repo.bzl", translate_c = "repo") load("//third_party/xla:repo.bzl", xla = "repo") load("//third_party/flashattn:repo.bzl", flashattn = "repo") +load("//third_party/linenoise:repo.bzl", linenoise = "repo") def _non_module_deps_impl(mctx): com_google_sentencepiece() @@ -14,6 +15,7 @@ def _non_module_deps_impl(mctx): arocc() translate_c() flashattn() + linenoise() return mctx.extension_metadata( reproducible = True, From d7655bc0299c5748739bb944ebaf68b2b4927f15 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 13 Feb 2026 11:59:30 +0100 Subject: [PATCH 275/881] zml/tensor: fix assert formatting --- zml/tensor.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zml/tensor.zig b/zml/tensor.zig index 957b4a44..a379db22 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1071,8 +1071,8 @@ pub const Tensor = struct { /// Axes with the same tag on both sides, and which aren't contracting, /// are considered "batching axes". pub fn dot(lhs: Tensor, rhs: Tensor, args: anytype) Tensor { - stdx.debug.assert(lhs.shape().hasTag(args) != null, "Expected lhs to have {} tag, got {f}", .{ args, lhs.shape() }); - stdx.debug.assert(rhs.shape().hasTag(args) != null, "Expected rhs to have {} tag, got {f}", .{ args, rhs.shape() }); + stdx.debug.assert(lhs.shape().hasTag(args) != null, "Expected lhs to have {any} tag, got {f}", .{ args, lhs.shape() }); + stdx.debug.assert(rhs.shape().hasTag(args) != null, "Expected rhs to have {any} tag, got {f}", .{ args, rhs.shape() }); const lhs_contracting_dim: i8 = @intCast(lhs.shape().hasTag(args).?); const rhs_contracting_dim: i8 = @intCast(rhs.shape().hasTag(args).?); From 24354d34b3bc1b791cd6d643b18756992813e275 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 12 Feb 2026 21:00:05 +0100 Subject: [PATCH 276/881] pjrt: add from DynLib --- pjrt/pjrt.zig | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 47964582..3556aefd 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -116,8 +116,13 @@ pub const Api = struct { return err; }, }; + + return fromDynLib(&lib); + } + + pub fn fromDynLib(lib: *std.DynLib) !*const Api { const DynGetPjrtApi = lib.lookup(*const fn () callconv(.c) *const Api, "GetPjrtApi") orelse { - std.debug.panic("Unable to find GetPjrtApi symbol in library: {s}", .{library}); + std.debug.panic("Unable to find GetPjrtApi symbol in library", .{}); }; const api = DynGetPjrtApi(); From 639850347e0b8d28307301e19c33cbd62b873e33 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 12 Feb 2026 21:08:46 +0100 Subject: [PATCH 277/881] pjrt: use dlopen on macos too to keep symbols private --- pjrt/pjrt.zig | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 3556aefd..b9f000ae 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -103,13 +103,21 @@ pub const Api = struct { defer log.info("Loaded: {s} [{D}]", .{ basename, timer.read() }); var lib: std.DynLib = switch (builtin.os.tag) { - .linux => blk: { - // We use RTLD_GLOBAL so that symbols from NEEDED libraries are available in the global namespace. - const handle = std.c.dlopen(library, .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { - log.err("Unable to dlopen plugin: {s}", .{library}); - return error.FileNotFound; + .linux, .macos => blk: { + const rtld: std.c.RTLD = switch (builtin.os.tag) { + // We use RTLD_GLOBAL so that symbols from NEEDED libraries are available in the global namespace. + .linux => .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }, + .macos => .{ .LAZY = true, .LOCAL = true }, + else => unreachable, + }; + break :blk .{ + .inner = .{ + .handle = std.c.dlopen(library, rtld) orelse { + log.err("Unable to dlopen plugin: {s}", .{library}); + return error.FileNotFound; + }, + }, }; - break :blk .{ .inner = .{ .handle = handle } }; }, else => std.DynLib.open(library) catch |err| { log.err("Unable to dlopen plugin: {s}", .{library}); From 2d65af067fdf6f8ab2062b7130da1803fc03ff84 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 13 Feb 2026 12:22:31 +0100 Subject: [PATCH 278/881] examples/benchmark: platform is not a const pointer --- examples/benchmark/main.zig | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index 1fc50d90..5927ad6e 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -25,7 +25,8 @@ pub fn main() !void { const io = threaded.io(); // Auto-select platform - const platform: zml.Platform = try .auto(allocator, io, .{}); + const platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator); const cli_args: CliArgs = blk: { var ret: CliArgs = .{}; @@ -99,7 +100,7 @@ pub fn main() !void { }); } -fn createRandomBuffer(allocator: std.mem.Allocator, io: std.Io, platform: zml.Platform, shape: zml.Shape, random: std.Random) !zml.Buffer { +fn createRandomBuffer(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, shape: zml.Shape, random: std.Random) !zml.Buffer { const slice = try zml.Slice.alloc(allocator, shape); defer slice.free(allocator); From dcbf5f2e2797581a725f9889d6b9b59f7e3bebe4 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 16 Feb 2026 12:04:05 +0100 Subject: [PATCH 279/881] workspace: update to 0.16.0-dev.2611+f996d2866, add remote exec also: - use toolchains_llvm_bootstrapped for macOS builds --- .bazelrc | 14 +- .bazelversion | 2 +- .zed/settings.json | 20 +- BUILD.bazel | 19 + MODULE.bazel | 68 +- MODULE.bazel.lock | 1897 +++----------------- bazel/bazel.zig | 56 +- bazel/zig_index.json | 154 +- examples/llama/BUILD.bazel | 6 +- examples/llama/llama.zig | 8 +- examples/llama/main.zig | 91 +- examples/mnist/BUILD.bazel | 6 +- examples/mnist/mnist.zig | 29 +- mlir/dialects/stablehlo/stablehlo.zig | 20 +- pjrt/pjrt.zig | 3 +- platforms/cpu/cpu.zig | 10 +- stdx/Io.zig | 14 + stdx/flags.zig | 18 +- stdx/fmt.zig | 12 + stdx/meta.zig | 6 +- stdx/stdx.zig | 23 +- third_party/arocc/repo.bzl | 2 +- third_party/translate-c/repo.bzl | 4 +- third_party/zls/BUILD.bazel | 5 + third_party/zls/workspace_printer.zig | 141 +- third_party/zls/zls.completion.tpl | 4 +- third_party/zls/zls_build_runner.zig | 70 +- third_party/zls/zls_completion.bzl | 4 +- third_party/zls/zls_runner.zig | 49 +- third_party/zls/zls_toolchain.bzl | 10 +- third_party/zls/zls_write_build_config.bzl | 41 +- zml/BUILD.bazel | 3 +- zml/meta.zig | 4 +- zml/module.zig | 21 +- zml/platform.zig | 25 +- zml/safetensors.zig | 12 +- zml/zml.zig | 5 + 37 files changed, 743 insertions(+), 2133 deletions(-) diff --git a/.bazelrc b/.bazelrc index e3c17bb7..e386fc58 100644 --- a/.bazelrc +++ b/.bazelrc @@ -14,11 +14,6 @@ common --experimental_remote_cache_eviction_retries=5 # Self explanatory common --enable_platform_specific_config -# Use local and ZML registries -common --registry=file://%workspace%/third_party/ -common --registry=https://bazel-registry.zml.ai -common --registry=https://bcr.bazel.build - # Ensure host tools are built optimized common --host_compilation_mode=opt @@ -82,12 +77,11 @@ common --@rules_zig//zig/settings:zigopt=-fllvm common --@toolchains_llvm_bootstrapped//config:experimental_stub_libgcc_s=True -common:macos --action_env=MACOSX_DEPLOYMENT_TARGET=13.0 -common:macos --macos_sdk_version=13.0 +common --macos_minimum_os=13.0 -# Don't detect the local C++ toolchain, be fully hermetic -# This is done only on Linux for now since we use the local toolchain on macOS -common:linux --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 +common --@rules_cc//cc/toolchains/args/archiver_flags:use_libtool_on_macos=false +common --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 +common --repo_env=BAZEL_NO_APPLE_CPP_TOOLCHAIN=1 # Required for protobuf upb on Linux common:linux --copt=-fPIC diff --git a/.bazelversion b/.bazelversion index 6d289079..f9c71a52 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -8.5.0 +8.5.1 diff --git a/.zed/settings.json b/.zed/settings.json index 55eff7c8..141b19f4 100644 --- a/.zed/settings.json +++ b/.zed/settings.json @@ -8,12 +8,12 @@ "formatter": "language_server", "code_actions_on_format": { "source.fixAll": true, - "source.organizeImports": true - } + "source.organizeImports": true, + }, }, "YAML": { - "tab_size": 2 - } + "tab_size": 2, + }, }, "file_scan_exclusions": [ "**/.git", @@ -25,17 +25,17 @@ "**/Thumbs.db", "**/.classpath", "**/.settings", - "**/bazel-*" + "**/bazel-*", ], "tab_size": 4, "inlay_hints": { - "enabled": true + "enabled": true, }, "lsp": { "zls": { "binary": { - "path": "tools/zls.sh" - } - } - } + "path": "tools/zls.sh", + }, + }, + }, } diff --git a/BUILD.bazel b/BUILD.bazel index 4fdc68d5..5dbfe326 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -4,7 +4,26 @@ zls_completion( name = "completion", visibility = ["//visibility:public"], deps = [ + "//examples/llama", + "//examples/mnist", "//stdx", "//zml", ], ) + +platform( + name = "rbe_platform", + constraint_values = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + # ubuntu:22.04 has GNU C Library (Ubuntu GLIBC 2.35-0ubuntu3.10) + # But apparently not in this docker://ubuntu:22.04 image? + "@toolchains_llvm_bootstrapped//constraints/libc:gnu.2.31", + "@bazel_tools//tools/cpp:clang", + ], + exec_properties = { + "container-image": "docker://ubuntu:22.04", + "OSFamily": "Linux", + }, + visibility = ["//visibility:public"], +) diff --git a/MODULE.bazel b/MODULE.bazel index 0f6bc745..3fcc9a16 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -22,9 +22,10 @@ bazel_dep(name = "rules_oci", version = "2.2.6") bazel_dep(name = "rules_proto", version = "7.1.0") bazel_dep(name = "rules_python", version = "1.8.0-rc1") bazel_dep(name = "rules_rust", version = "0.68.1") +bazel_dep(name = "rules_rs", version = "0.0.28") bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "rules_zig", version = "0.12.2") -bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.3.1") +bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.5.5") bazel_dep(name = "with_cfg.bzl", version = "0.11.0") bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2", dev_dependency = True) @@ -32,9 +33,21 @@ bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2", dev_dependency = Tr ### Zig toolchain zig = use_extension("@rules_zig//zig:extensions.bzl", "zig") zig.index(file = "//bazel:zig_index.json") -zig.toolchain(zig_version = "0.16.0-dev.1912+0cbaaa5eb") +zig.toolchain(zig_version = "0.16.0-dev.2611+f996d2866") zig.mirrors(urls = [ - "https://mirror.zml.ai/zig", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds", "https://ziglang.org/builds/", ]) use_repo(zig, "zig_toolchains") @@ -153,12 +166,12 @@ use_repo(rust, "rust_toolchains") register_toolchains("@rust_toolchains//:all") -crate = use_extension("@rules_rust//crate_universe:extensions.bzl", "crate") +crate = use_extension("@rules_rs//rs:extensions.bzl", "crate") crate.from_cargo( name = "crates", - cargo_lockfile = "//zml/tokenizer/hftokenizers:Cargo.lock", - manifests = ["//zml/tokenizer/hftokenizers:Cargo.toml"], - supported_platform_triples = [ + cargo_lock = "//zml/tokenizer/hftokenizers:Cargo.lock", + cargo_toml = "//zml/tokenizer/hftokenizers:Cargo.toml", + platform_triples = [ "aarch64-apple-darwin", "x86_64-apple-darwin", "aarch64-unknown-linux-gnu", @@ -198,47 +211,8 @@ oci.pull( ) use_repo(oci, "distroless_cc_debian12", "distroless_cc_debian12_linux_amd64") -huggingface = use_extension("@zml//bazel:huggingface.bzl", "huggingface") - -################################################################################ -# ModernBERT -huggingface.model( - name = "ModernBERT-base", - build_file_content = """exports_files(glob(["**"]), visibility = ["//visibility:public"])""", - commit = "94032bb66234a691cf6248265170006a7ced4970", - includes = [ - "model.safetensors", - "tokenizer.json", - ], - model = "answerdotai/ModernBERT-base", -) -huggingface.model( - name = "ModernBERT-large", - build_file_content = """exports_files(glob(["**"]), visibility = ["//visibility:public"])""", - commit = "4bbcbf40bed02ce487125bcb3c897ea9bdc88340", - includes = [ - "model.safetensors", - "tokenizer.json", - ], - model = "answerdotai/ModernBERT-large", -) -use_repo(huggingface, "ModernBERT-base", "ModernBERT-large") -################################################################################ - git_override( module_name = "rules_zig", - commit = "b4ae496abfd61f880927dd01ea59823df7b42c29", + commit = "6c75cd2e0ef95dec8c8dbcbecb4586d0b313cf06", remote = "https://github.com/zml/rules_zig.git", ) - -git_override( - module_name = "toolchains_llvm_bootstrapped", - commit = "023f62a94eb38d2991432a766a1d4c05e90414d7", - remote = "https://github.com/cerisier/toolchains_llvm_bootstrapped.git", -) - -git_override( - module_name = "apple_support", - commit = "6ddcf9c1c46be1a7eb8767c7dbec7714dd47efb8", - remote = "https://github.com/cerisier/apple_support.git", -) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 764564fa..64278448 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1,198 +1,6 @@ { "lockFileVersion": 24, "registryFileHashes": { - "https://bazel-registry.zml.ai/modules/abseil-cpp/20210324.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20211102.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20230125.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20230802.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20230802.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20250127.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20250512.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20250814.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.0.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.16.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.19.3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.21.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.9.4/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/aspect_rules_py/1.6.3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/aspect_tools_telemetry/0.2.6/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.10.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.11.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.15.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.17.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.18.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.19.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.20.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.21.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.24.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.27.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.28.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.3.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.30.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.32.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.33.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.34.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.4.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.9.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.9.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_lib/3.0.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.0.3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.2.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.2.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.3.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.4.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.4.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.5.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.6.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.7.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.7.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.8.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.8.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/8.2.0.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/buildozer/7.1.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/gawk/5.3.2.bcr.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/google_benchmark/1.8.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/googletest/1.11.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/googletest/1.14.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/googletest/1.15.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/googletest/1.17.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/jq.bzl/0.1.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/jsoncpp/1.9.5/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/jsoncpp/1.9.6/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/libpfm/4.11.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/libxev/0.0.0-20251014.0-9f785d2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/nlohmann_json/3.6.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/package_metadata/0.0.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/patchelf/0.18.0.bcr.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/pcre2/10.45/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.10/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.11/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.4/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.5/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.6/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.7/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.8/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.9/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/1.0.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/21.7/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/27.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/27.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/29.0-rc2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/29.0-rc3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/29.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/29.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/3.19.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/33.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/pybind11_bazel/2.11.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/pybind11_bazel/2.12.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/re2/2023-09-01/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/re2/2024-07-02/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_android/0.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_apple/3.16.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.10/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.13/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.14/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.15/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.16/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.17/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.6/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.8/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.9/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.1.5/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.2.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.2.16/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.2.4/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_distroless/0.5.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_fuzzing/0.5.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/4.0.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/5.3.5/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/6.0.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/6.3.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/6.4.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/6.5.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/7.10.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/7.12.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/7.2.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/7.3.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/7.6.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/8.14.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/8.3.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/8.5.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/8.6.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/8.6.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/8.8.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_jvm_external/4.4.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_jvm_external/5.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_jvm_external/5.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_jvm_external/5.3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_jvm_external/6.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_jvm_external/6.3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_jvm_external/6.7/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_kotlin/1.9.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_kotlin/1.9.6/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_license/0.0.3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_license/0.0.7/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_license/1.0.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_oci/2.2.6/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_pkg/0.7.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_pkg/1.0.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_proto/4.0.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_proto/6.0.0-rc1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_proto/6.0.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_proto/7.0.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_proto/7.1.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.10.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.23.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.25.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.28.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.29.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.31.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.33.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.4.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.40.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/1.6.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/1.8.0-rc1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_rust/0.68.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_shell/0.2.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_shell/0.3.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_shell/0.4.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_shell/0.6.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/stardoc/0.5.3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/stardoc/0.5.4/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/stardoc/0.5.6/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/stardoc/0.6.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/stardoc/0.7.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/stardoc/0.7.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/stardoc/0.7.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/tar.bzl/0.2.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/tar.bzl/0.5.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/tar.bzl/0.6.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/with_cfg.bzl/0.11.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/with_cfg.bzl/0.12.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/yq.bzl/0.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/zlib/1.2.11/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/zlib/1.3.1/MODULE.bazel": "not found", "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2", "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/MODULE.bazel": "70390338f7a5106231d20620712f7cccb659cd0e9d073d1991c038eb9fc57589", @@ -206,6 +14,14 @@ "https://bcr.bazel.build/modules/abseil-cpp/20250512.1/MODULE.bazel": "d209fdb6f36ffaf61c509fcc81b19e81b411a999a934a032e10cd009a0226215", "https://bcr.bazel.build/modules/abseil-cpp/20250814.0/MODULE.bazel": "c43c16ca2c432566cdb78913964497259903ebe8fb7d9b57b38e9f1425b427b8", "https://bcr.bazel.build/modules/abseil-cpp/20250814.0/source.json": "b88bff599ceaf0f56c264c749b1606f8485cec3b8c38ba30f88a4df9af142861", + "https://bcr.bazel.build/modules/apple_support/1.11.1/MODULE.bazel": "1843d7cd8a58369a444fc6000e7304425fba600ff641592161d9f15b179fb896", + "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", + "https://bcr.bazel.build/modules/apple_support/1.23.0/MODULE.bazel": "317d47e3f65b580e7fb4221c160797fda48e32f07d2dfff63d754ef2316dcd25", + "https://bcr.bazel.build/modules/apple_support/1.23.1/MODULE.bazel": "53763fed456a968cf919b3240427cf3a9d5481ec5466abc9d5dc51bc70087442", + "https://bcr.bazel.build/modules/apple_support/1.24.1/MODULE.bazel": "f46e8ddad60aef170ee92b2f3d00ef66c147ceafea68b6877cb45bd91737f5f8", + "https://bcr.bazel.build/modules/apple_support/1.24.5/MODULE.bazel": "692784f32ee3175cc1ced78f4262a0d4b7110c89793cd0b63f950af2d9cc6bdc", + "https://bcr.bazel.build/modules/apple_support/2.2.0/MODULE.bazel": "5e32bc42e413a4544df7afee7fa4c7aff73e670a44e56e8e45ce1954332034ad", + "https://bcr.bazel.build/modules/apple_support/2.2.0/source.json": "368bf1de5dace2a411e792e6df2ea5384eb8db3899680ed51b37eb48b88fd8ac", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.0.0/MODULE.bazel": "e118477db5c49419a88d78ebc7a2c2cea9d49600fe0f490c1903324a2c16ecd9", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.16.0/MODULE.bazel": "852f9ebbda017572a7c113a2434592dd3b2f55cd9a0faea3d4be5a09a59e4900", @@ -218,7 +34,8 @@ "https://bcr.bazel.build/modules/aspect_rules_py/1.6.3/MODULE.bazel": "a34d989f9a258f478b40f69a0c1dd0a810b65173681b459c296ffeebabd997b0", "https://bcr.bazel.build/modules/aspect_rules_py/1.6.3/source.json": "aac06d362804a5a4c1424efc37fc1ded6c6ac7d8be84ef28f735944f8a1f884d", "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.2.6/MODULE.bazel": "cafb8781ad591bc57cc765dca5fefab08cf9f65af363d162b79d49205c7f8af7", - "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.2.6/source.json": "4d98137d5f74f01e00c6efa8bf591c02718e6c5f31f0bcc73983ea514dd02a12", + "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.2/MODULE.bazel": "598e7fe3b54f5fa64fdbeead1027653963a359cc23561d43680006f3b463d5a4", + "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.2/source.json": "c6f5c39e6f32eb395f8fdaea63031a233bbe96d49a3bfb9f75f6fce9b74bec6c", "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", "https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101", "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", @@ -241,7 +58,8 @@ "https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b", "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", "https://bcr.bazel.build/modules/bazel_lib/3.0.0/MODULE.bazel": "22b70b80ac89ad3f3772526cd9feee2fa412c2b01933fea7ed13238a448d370d", - "https://bcr.bazel.build/modules/bazel_lib/3.0.0/source.json": "895f21909c6fba01d7c17914bb6c8e135982275a1b18cdaa4e62272217ef1751", + "https://bcr.bazel.build/modules/bazel_lib/3.2.0/MODULE.bazel": "39b50d94b9be6bda507862254e20c263f9b950e3160112348d10a938be9ce2c2", + "https://bcr.bazel.build/modules/bazel_lib/3.2.0/source.json": "a6f45a903134bebbf33a6166dd42b4c7ab45169de094b37a85f348ca41170a84", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", @@ -280,7 +98,8 @@ "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/source.json": "f448c6e8963fdfa7eb831457df83ad63d3d6355018f6574fb017e8169deb43a9", "https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92", - "https://bcr.bazel.build/modules/package_metadata/0.0.2/source.json": "e53a759a72488d2c0576f57491ef2da0cf4aab05ac0997314012495935531b73", + "https://bcr.bazel.build/modules/package_metadata/0.0.5/MODULE.bazel": "ef4f9439e3270fdd6b9fd4dbc3d2f29d13888e44c529a1b243f7a31dfbc2e8e4", + "https://bcr.bazel.build/modules/package_metadata/0.0.5/source.json": "2326db2f6592578177751c3e1f74786b79382cd6008834c9d01ec865b9126a85", "https://bcr.bazel.build/modules/patchelf/0.18.0.bcr.1/MODULE.bazel": "35320668481400503dea7c75313473b9ffd6dea4e7ba583b96a896199b0088d5", "https://bcr.bazel.build/modules/patchelf/0.18.0.bcr.1/source.json": "d877d271ef5d531407885322054c99c7f0617a9a85ff99c0a70f4ced7f13e13a", "https://bcr.bazel.build/modules/pcre2/10.45/MODULE.bazel": "9bdc838aa2e21be030d7181385e45a5c87b14575bb7ff35d10b5be3d56913bed", @@ -330,9 +149,12 @@ "https://bcr.bazel.build/modules/rules_cc/0.1.1/MODULE.bazel": "2f0222a6f229f0bf44cd711dc13c858dad98c62d52bd51d8fc3a764a83125513", "https://bcr.bazel.build/modules/rules_cc/0.1.5/MODULE.bazel": "88dfc9361e8b5ae1008ac38f7cdfd45ad738e4fa676a3ad67d19204f045a1fd8", "https://bcr.bazel.build/modules/rules_cc/0.2.0/MODULE.bazel": "b5c17f90458caae90d2ccd114c81970062946f49f355610ed89bebf954f5783c", + "https://bcr.bazel.build/modules/rules_cc/0.2.13/MODULE.bazel": "eecdd666eda6be16a8d9dc15e44b5c75133405e820f620a234acc4b1fdc5aa37", + "https://bcr.bazel.build/modules/rules_cc/0.2.15/MODULE.bazel": "6a0a4a75a57aa6dc888300d848053a58c6b12a29f89d4304e1c41448514ec6e8", "https://bcr.bazel.build/modules/rules_cc/0.2.16/MODULE.bazel": "9242fa89f950c6ef7702801ab53922e99c69b02310c39fb6e62b2bd30df2a1d4", "https://bcr.bazel.build/modules/rules_cc/0.2.16/source.json": "d03d5cde49376d87e14ec14b666c56075e5e3926930327fd5d0484a1ff2ac1cc", "https://bcr.bazel.build/modules/rules_cc/0.2.4/MODULE.bazel": "1ff1223dfd24f3ecf8f028446d4a27608aa43c3f41e346d22838a4223980b8cc", + "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", "https://bcr.bazel.build/modules/rules_distroless/0.5.1/MODULE.bazel": "2a63f4744d30749128105da5f96adf7caf5628e37548293f89e7fa39c3b3f2c2", "https://bcr.bazel.build/modules/rules_distroless/0.5.1/source.json": "c6b9ff7f325bfed89c3671757f14c1d1bc6077d0fcef809b8aa2d007cac7dd1d", "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", @@ -394,10 +216,14 @@ "https://bcr.bazel.build/modules/rules_python/1.6.0/MODULE.bazel": "7e04ad8f8d5bea40451cf80b1bd8262552aa73f841415d20db96b7241bd027d8", "https://bcr.bazel.build/modules/rules_python/1.8.0-rc1/MODULE.bazel": "d5348333fd8be9589c3ea8d9110fd00b1c7d84bc7b505668307cbe9f105e9c8f", "https://bcr.bazel.build/modules/rules_python/1.8.0-rc1/source.json": "d26719d5b92a569eaa387bad9667a042d4810193f279ea64590c9d76ae38f3ba", + "https://bcr.bazel.build/modules/rules_rs/0.0.28/MODULE.bazel": "3c3830eda824d0476818443dd4a3a2fa0765484589db874d18e94fb6ee50a60f", + "https://bcr.bazel.build/modules/rules_rs/0.0.28/source.json": "3480135f6ef8111ec7abe2b7cc164af40354b4eaf8f885170567922a70a2dc99", + "https://bcr.bazel.build/modules/rules_rust/0.66.0/MODULE.bazel": "86ef763a582f4739a27029bdcc6c562258ed0ea6f8d58294b049e215ceb251b3", "https://bcr.bazel.build/modules/rules_rust/0.68.1/MODULE.bazel": "8d3332ef4079673385eb81f8bd68b012decc04ac00c9d5a01a40eff90301732c", "https://bcr.bazel.build/modules/rules_rust/0.68.1/source.json": "3378e746f81b62457fdfd37391244fa8ff075ba85c05931ee4f3a20ac1efe963", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", + "https://bcr.bazel.build/modules/rules_shell/0.4.0/MODULE.bazel": "0f8f11bb3cd11755f0b48c1de0bbcf62b4b34421023aa41a2fc74ef68d9584f0", "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", "https://bcr.bazel.build/modules/rules_shell/0.6.1/MODULE.bazel": "72e76b0eea4e81611ef5452aa82b3da34caca0c8b7b5c0c9584338aa93bae26b", "https://bcr.bazel.build/modules/rules_shell/0.6.1/source.json": "20ec05cd5e592055e214b2da8ccb283c7f2a421ea0dc2acbf1aa792e11c03d0c", @@ -419,6 +245,8 @@ "https://bcr.bazel.build/modules/tar.bzl/0.5.1/MODULE.bazel": "7c2eb3dcfc53b0f3d6f9acdfd911ca803eaf92aadf54f8ca6e4c1f3aee288351", "https://bcr.bazel.build/modules/tar.bzl/0.6.0/MODULE.bazel": "a3584b4edcfafcabd9b0ef9819808f05b372957bbdff41601429d5fd0aac2e7c", "https://bcr.bazel.build/modules/tar.bzl/0.6.0/source.json": "4a620381df075a16cb3a7ed57bd1d05f7480222394c64a20fa51bdb636fda658", + "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.5.5/MODULE.bazel": "a2f1469b2920fcc9c043c87301b7a207d320231615abecd7771d89e516fc3d0b", + "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.5.5/source.json": "8498c094703da13a859d19dd2a200152f75870fb7f22a9770df99d230a4f50d1", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", "https://bcr.bazel.build/modules/with_cfg.bzl/0.11.0/MODULE.bazel": "bcb1dba64b74542648a93d6313e160b2670f22395581edd8c4ffadb811618ef7", "https://bcr.bazel.build/modules/with_cfg.bzl/0.12.0/MODULE.bazel": "b573395fe63aef4299ba095173e2f62ccfee5ad9bbf7acaa95dba73af9fc2b38", @@ -432,37 +260,6 @@ }, "selectedYankedVersions": {}, "moduleExtensions": { - "@@apple_support+//crosstool:setup.bzl%apple_cc_configure_extension": { - "general": { - "bzlTransitiveDigest": "cw6mKbsar0LsMbAePOLZdW+szZNKRE8/CrhomQFNbNE=", - "usagesDigest": "GXVjAqzvGUnh20kBPs+f0GSdAtaqLPet4sogeCfejZY=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "local_config_apple_cc_toolchains": { - "repoRuleId": "@@apple_support+//crosstool:setup.bzl%_apple_cc_autoconf_toolchains", - "attributes": {} - }, - "local_config_apple_cc": { - "repoRuleId": "@@apple_support+//crosstool:setup.bzl%_apple_cc_autoconf", - "attributes": {} - } - }, - "recordedRepoMappingEntries": [ - [ - "apple_support+", - "bazel_tools", - "bazel_tools" - ], - [ - "bazel_tools", - "rules_cc", - "rules_cc+" - ] - ] - } - }, "@@aspect_rules_py+//py:extensions.bzl%py_tools": { "general": { "bzlTransitiveDigest": "G/1q87gmIpapGBPovxgsd2S/VIcXuqJ4H0PnAtwkKWc=", @@ -585,8 +382,8 @@ }, "@@aspect_tools_telemetry+//:extension.bzl%telemetry": { "general": { - "bzlTransitiveDigest": "aVNc9R1NtH1OtynCXvZ9j7WtILv+X93Qxzmkyn4x++8=", - "usagesDigest": "bLd5DJLF7d7awlSuBaWvY0qEqBLja5XVKriiQmSNEi8=", + "bzlTransitiveDigest": "BXySuBSW3dtagvU27mRMChwOLh8Z8/Iyds20LjMajuE=", + "usagesDigest": "dcbNgiRoyIPXA1ORP5Y4jD1mG/tTZiyOfFASmpRxpfY=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -595,8 +392,51 @@ "repoRuleId": "@@aspect_tools_telemetry+//:extension.bzl%tel_repository", "attributes": { "deps": { + "abseil-cpp": "20250814.0", + "aspect_bazel_lib": "2.21.2", "aspect_rules_py": "1.6.3", - "aspect_tools_telemetry": "0.2.6" + "aspect_tools_telemetry": "0.3.2", + "bazel_features": "1.34.0", + "bazel_lib": "3.2.0", + "bazel_skylib": "1.8.2", + "buildifier_prebuilt": "8.2.0.2", + "buildozer": "7.1.2", + "gawk": "5.3.2.bcr.1", + "googletest": "1.17.0", + "jq.bzl": "0.1.0", + "jsoncpp": "1.9.6", + "libxev": "0.0.0-20251014.0-9f785d2", + "nlohmann_json": "3.6.1", + "package_metadata": "0.0.5", + "patchelf": "0.18.0.bcr.1", + "pcre2": "10.45", + "platforms": "1.0.0", + "protobuf": "33.1", + "pybind11_bazel": "2.12.0", + "re2": "2024-07-02.bcr.1", + "rules_android": "0.1.1", + "rules_apple": "3.16.0", + "rules_cc": "0.2.16", + "rules_distroless": "0.5.1", + "rules_java": "8.14.0", + "rules_jvm_external": "6.7", + "rules_kotlin": "1.9.6", + "rules_license": "1.0.0", + "rules_oci": "2.2.6", + "rules_pkg": "1.0.1", + "rules_proto": "7.1.0", + "rules_python": "1.8.0-rc1", + "rules_rs": "0.0.28", + "rules_rust": "0.66.0", + "rules_shell": "0.6.1", + "rules_swift": "2.1.1", + "stardoc": "0.7.2", + "swift_argument_parser": "1.3.1.1", + "tar.bzl": "0.6.0", + "toolchains_llvm_bootstrapped": "0.5.5", + "with_cfg.bzl": "0.12.0", + "yq.bzl": "0.1.1", + "zlib": "1.3.1.bcr.5" } } } @@ -604,8 +444,8 @@ "recordedRepoMappingEntries": [ [ "aspect_tools_telemetry+", - "aspect_bazel_lib", - "aspect_bazel_lib+" + "bazel_lib", + "bazel_lib+" ], [ "aspect_tools_telemetry+", @@ -1275,1430 +1115,164 @@ ] } }, - "@@rules_rust+//crate_universe:extensions.bzl%crate": { - "general": { - "bzlTransitiveDigest": "NRu94346iEpwiU1qJJMclq8aGKHipA+LB4vVn84/gec=", - "usagesDigest": "mup6mgSFatk34Q0znMZPpIEOFZyk9v3xf6jRIXEFTxc=", - "recordedFileInputs": { - "@@//zml/tokenizer/hftokenizers/Cargo.lock": "7fb683b6a36181a92d7564c8552b4fc5bad00229a41d520a4536c4e577c2b888", - "@@//zml/tokenizer/hftokenizers/Cargo.toml": "e46e67c2ae6792917dc1259d6ba5db389becc8bafa9fa1911bb3b6cec6017620" - }, - "recordedDirentsInputs": {}, - "envVariables": { - "CARGO_BAZEL_DEBUG": null, - "CARGO_BAZEL_GENERATOR_SHA256": null, - "CARGO_BAZEL_GENERATOR_URL": null, - "CARGO_BAZEL_ISOLATED": null, - "CARGO_BAZEL_REPIN": null, - "CARGO_BAZEL_REPIN_ONLY": null, - "CARGO_BAZEL_TIMEOUT": null, - "REPIN": null - }, - "generatedRepoSpecs": { - "crates": { - "repoRuleId": "@@rules_rust+//crate_universe:extensions.bzl%_generate_repo", - "attributes": { - "contents": { - "BUILD.bazel": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files(\n [\n \"cargo-bazel.json\",\n \"crates.bzl\",\n \"defs.bzl\",\n ] + glob(\n allow_empty = True,\n include = [\"*.bazel\"],\n ),\n)\n\nfilegroup(\n name = \"srcs\",\n srcs = glob(\n allow_empty = True,\n include = [\n \"*.bazel\",\n \"*.bzl\",\n ],\n ),\n)\n\n# Workspace Member Dependencies\nalias(\n name = \"tokenizers-0.22.2\",\n actual = \"@crates__tokenizers-0.22.2//:tokenizers\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"tokenizers\",\n actual = \"@crates__tokenizers-0.22.2//:tokenizers\",\n tags = [\"manual\"],\n)\n", - "alias_rules.bzl": "\"\"\"Alias that transitions its target to `compilation_mode=opt`. Use `transition_alias=\"opt\"` to enable.\"\"\"\n\nload(\"@rules_cc//cc:defs.bzl\", \"CcInfo\")\nload(\"@rules_rust//rust:rust_common.bzl\", \"COMMON_PROVIDERS\")\n\ndef _transition_alias_impl(ctx):\n # `ctx.attr.actual` is a list of 1 item due to the transition\n providers = [ctx.attr.actual[0][provider] for provider in COMMON_PROVIDERS]\n if CcInfo in ctx.attr.actual[0]:\n providers.append(ctx.attr.actual[0][CcInfo])\n return providers\n\ndef _change_compilation_mode(compilation_mode):\n def _change_compilation_mode_impl(_settings, _attr):\n return {\n \"//command_line_option:compilation_mode\": compilation_mode,\n }\n\n return transition(\n implementation = _change_compilation_mode_impl,\n inputs = [],\n outputs = [\n \"//command_line_option:compilation_mode\",\n ],\n )\n\ndef _transition_alias_rule(compilation_mode):\n return rule(\n implementation = _transition_alias_impl,\n provides = COMMON_PROVIDERS,\n attrs = {\n \"actual\": attr.label(\n mandatory = True,\n doc = \"`rust_library()` target to transition to `compilation_mode=opt`.\",\n providers = COMMON_PROVIDERS,\n cfg = _change_compilation_mode(compilation_mode),\n ),\n \"_allowlist_function_transition\": attr.label(\n default = \"@bazel_tools//tools/allowlists/function_transition_allowlist\",\n ),\n },\n doc = \"Transitions a Rust library crate to the `compilation_mode=opt`.\",\n )\n\ntransition_alias_dbg = _transition_alias_rule(\"dbg\")\ntransition_alias_fastbuild = _transition_alias_rule(\"fastbuild\")\ntransition_alias_opt = _transition_alias_rule(\"opt\")\n", - "defs.bzl": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\"\"\"\n# `crates_repository` API\n\n- [aliases](#aliases)\n- [crate_deps](#crate_deps)\n- [all_crate_deps](#all_crate_deps)\n- [crate_repositories](#crate_repositories)\n\n\"\"\"\n\nload(\"@bazel_tools//tools/build_defs/repo:git.bzl\", \"new_git_repository\")\nload(\"@bazel_tools//tools/build_defs/repo:http.bzl\", \"http_archive\")\nload(\"@bazel_tools//tools/build_defs/repo:utils.bzl\", \"maybe\")\nload(\"@bazel_skylib//lib:selects.bzl\", \"selects\")\nload(\"@rules_rust//crate_universe/private:local_crate_mirror.bzl\", \"local_crate_mirror\")\n\n###############################################################################\n# MACROS API\n###############################################################################\n\n# An identifier that represent common dependencies (unconditional).\n_COMMON_CONDITION = \"\"\n\ndef _flatten_dependency_maps(all_dependency_maps):\n \"\"\"Flatten a list of dependency maps into one dictionary.\n\n Dependency maps have the following structure:\n\n ```python\n DEPENDENCIES_MAP = {\n # The first key in the map is a Bazel package\n # name of the workspace this file is defined in.\n \"workspace_member_package\": {\n\n # Not all dependencies are supported for all platforms.\n # the condition key is the condition required to be true\n # on the host platform.\n \"condition\": {\n\n # An alias to a crate target. # The label of the crate target the\n # Aliases are only crate names. # package name refers to.\n \"package_name\": \"@full//:label\",\n }\n }\n }\n ```\n\n Args:\n all_dependency_maps (list): A list of dicts as described above\n\n Returns:\n dict: A dictionary as described above\n \"\"\"\n dependencies = {}\n\n for workspace_deps_map in all_dependency_maps:\n for pkg_name, conditional_deps_map in workspace_deps_map.items():\n if pkg_name not in dependencies:\n non_frozen_map = dict()\n for key, values in conditional_deps_map.items():\n non_frozen_map.update({key: dict(values.items())})\n dependencies.setdefault(pkg_name, non_frozen_map)\n continue\n\n for condition, deps_map in conditional_deps_map.items():\n # If the condition has not been recorded, do so and continue\n if condition not in dependencies[pkg_name]:\n dependencies[pkg_name].setdefault(condition, dict(deps_map.items()))\n continue\n\n # Alert on any miss-matched dependencies\n inconsistent_entries = []\n for crate_name, crate_label in deps_map.items():\n existing = dependencies[pkg_name][condition].get(crate_name)\n if existing and existing != crate_label:\n inconsistent_entries.append((crate_name, existing, crate_label))\n dependencies[pkg_name][condition].update({crate_name: crate_label})\n\n return dependencies\n\ndef crate_deps(deps, package_name = None):\n \"\"\"Finds the fully qualified label of the requested crates for the package where this macro is called.\n\n Args:\n deps (list): The desired list of crate targets.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()`.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if not deps:\n return []\n\n if package_name == None:\n package_name = native.package_name()\n\n # Join both sets of dependencies\n dependencies = _flatten_dependency_maps([\n _NORMAL_DEPENDENCIES,\n _NORMAL_DEV_DEPENDENCIES,\n _PROC_MACRO_DEPENDENCIES,\n _PROC_MACRO_DEV_DEPENDENCIES,\n _BUILD_DEPENDENCIES,\n _BUILD_PROC_MACRO_DEPENDENCIES,\n ]).pop(package_name, {})\n\n # Combine all conditional packages so we can easily index over a flat list\n # TODO: Perhaps this should actually return select statements and maintain\n # the conditionals of the dependencies\n flat_deps = {}\n for deps_set in dependencies.values():\n for crate_name, crate_label in deps_set.items():\n flat_deps.update({crate_name: crate_label})\n\n missing_crates = []\n crate_targets = []\n for crate_target in deps:\n if crate_target not in flat_deps:\n missing_crates.append(crate_target)\n else:\n crate_targets.append(flat_deps[crate_target])\n\n if missing_crates:\n fail(\"Could not find crates `{}` among dependencies of `{}`. Available dependencies were `{}`\".format(\n missing_crates,\n package_name,\n dependencies,\n ))\n\n return crate_targets\n\ndef all_crate_deps(\n normal = False, \n normal_dev = False, \n proc_macro = False, \n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Finds the fully qualified label of all requested direct crate dependencies \\\n for the package where this macro is called.\n\n If no parameters are set, all normal dependencies are returned. Setting any one flag will\n otherwise impact the contents of the returned list.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_dependency_maps = []\n if normal:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n if normal_dev:\n all_dependency_maps.append(_NORMAL_DEV_DEPENDENCIES)\n if proc_macro:\n all_dependency_maps.append(_PROC_MACRO_DEPENDENCIES)\n if proc_macro_dev:\n all_dependency_maps.append(_PROC_MACRO_DEV_DEPENDENCIES)\n if build:\n all_dependency_maps.append(_BUILD_DEPENDENCIES)\n if build_proc_macro:\n all_dependency_maps.append(_BUILD_PROC_MACRO_DEPENDENCIES)\n\n # Default to always using normal dependencies\n if not all_dependency_maps:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n\n dependencies = _flatten_dependency_maps(all_dependency_maps).pop(package_name, None)\n\n if not dependencies:\n if dependencies == None:\n fail(\"Tried to get all_crate_deps for package \" + package_name + \" but that package had no Cargo.toml file\")\n else:\n return []\n\n crate_deps = list(dependencies.pop(_COMMON_CONDITION, {}).values())\n for condition, deps in dependencies.items():\n crate_deps += selects.with_or({\n tuple(_CONDITIONS[condition]): deps.values(),\n \"//conditions:default\": [],\n })\n\n return crate_deps\n\ndef aliases(\n normal = False,\n normal_dev = False,\n proc_macro = False,\n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Produces a map of Crate alias names to their original label\n\n If no dependency kinds are specified, `normal` and `proc_macro` are used by default.\n Setting any one flag will otherwise determine the contents of the returned dict.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n dict: The aliases of all associated packages\n \"\"\"\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_aliases_maps = []\n if normal:\n all_aliases_maps.append(_NORMAL_ALIASES)\n if normal_dev:\n all_aliases_maps.append(_NORMAL_DEV_ALIASES)\n if proc_macro:\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n if proc_macro_dev:\n all_aliases_maps.append(_PROC_MACRO_DEV_ALIASES)\n if build:\n all_aliases_maps.append(_BUILD_ALIASES)\n if build_proc_macro:\n all_aliases_maps.append(_BUILD_PROC_MACRO_ALIASES)\n\n # Default to always using normal aliases\n if not all_aliases_maps:\n all_aliases_maps.append(_NORMAL_ALIASES)\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n\n aliases = _flatten_dependency_maps(all_aliases_maps).pop(package_name, None)\n\n if not aliases:\n return dict()\n\n common_items = aliases.pop(_COMMON_CONDITION, {}).items()\n\n # If there are only common items in the dictionary, immediately return them\n if not len(aliases.keys()) == 1:\n return dict(common_items)\n\n # Build a single select statement where each conditional has accounted for the\n # common set of aliases.\n crate_aliases = {\"//conditions:default\": dict(common_items)}\n for condition, deps in aliases.items():\n condition_triples = _CONDITIONS[condition]\n for triple in condition_triples:\n if triple in crate_aliases:\n crate_aliases[triple].update(deps)\n else:\n crate_aliases.update({triple: dict(deps.items() + common_items)})\n\n return select(crate_aliases)\n\n###############################################################################\n# WORKSPACE MEMBER DEPS AND ALIASES\n###############################################################################\n\n_NORMAL_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n _COMMON_CONDITION: {\n \"tokenizers\": Label(\"@crates//:tokenizers-0.22.2\"),\n },\n },\n}\n\n\n_NORMAL_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n _COMMON_CONDITION: {\n },\n },\n}\n\n\n_NORMAL_DEV_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_NORMAL_DEV_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEV_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_PROC_MACRO_DEV_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_DEPENDENCIES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_ALIASES = {\n \"zml/tokenizer/hftokenizers\": {\n },\n}\n\n\n_CONDITIONS = {\n \"aarch64-apple-darwin\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\"],\n \"aarch64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\"],\n \"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\",\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\": [],\n \"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\": [],\n \"cfg(any())\": [],\n \"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\": [],\n \"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\": [],\n \"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\": [],\n \"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\",\"@rules_rust//rust/platform:x86_64-apple-darwin\"],\n \"cfg(not(all(target_arch = \\\"arm\\\", target_os = \\\"none\\\")))\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\",\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\",\"@rules_rust//rust/platform:x86_64-apple-darwin\",\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"cfg(target_os = \\\"netbsd\\\")\": [],\n \"cfg(target_os = \\\"solaris\\\")\": [],\n \"cfg(target_os = \\\"vxworks\\\")\": [],\n \"cfg(windows)\": [],\n \"x86_64-apple-darwin\": [\"@rules_rust//rust/platform:x86_64-apple-darwin\"],\n \"x86_64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n}\n\n###############################################################################\n\ndef crate_repositories():\n \"\"\"A macro for defining repositories for all generated crates.\n\n Returns:\n A list of repos visible to the module through the module extension.\n \"\"\"\n maybe(\n http_archive,\n name = \"crates__ahash-0.8.12\",\n sha256 = \"5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ahash/0.8.12/download\"],\n strip_prefix = \"ahash-0.8.12\",\n build_file = Label(\"@crates//crates:BUILD.ahash-0.8.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__aho-corasick-1.1.3\",\n sha256 = \"8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/aho-corasick/1.1.3/download\"],\n strip_prefix = \"aho-corasick-1.1.3\",\n build_file = Label(\"@crates//crates:BUILD.aho-corasick-1.1.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__base64-0.13.1\",\n sha256 = \"9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/base64/0.13.1/download\"],\n strip_prefix = \"base64-0.13.1\",\n build_file = Label(\"@crates//crates:BUILD.base64-0.13.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__bitflags-2.9.1\",\n sha256 = \"1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/bitflags/2.9.1/download\"],\n strip_prefix = \"bitflags-2.9.1\",\n build_file = Label(\"@crates//crates:BUILD.bitflags-2.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__byteorder-1.5.0\",\n sha256 = \"1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/byteorder/1.5.0/download\"],\n strip_prefix = \"byteorder-1.5.0\",\n build_file = Label(\"@crates//crates:BUILD.byteorder-1.5.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__castaway-0.2.4\",\n sha256 = \"dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/castaway/0.2.4/download\"],\n strip_prefix = \"castaway-0.2.4\",\n build_file = Label(\"@crates//crates:BUILD.castaway-0.2.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__cc-1.2.33\",\n sha256 = \"3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/cc/1.2.33/download\"],\n strip_prefix = \"cc-1.2.33\",\n build_file = Label(\"@crates//crates:BUILD.cc-1.2.33.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__cfg-if-1.0.0\",\n sha256 = \"baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/cfg-if/1.0.0/download\"],\n strip_prefix = \"cfg-if-1.0.0\",\n build_file = Label(\"@crates//crates:BUILD.cfg-if-1.0.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__compact_str-0.9.0\",\n sha256 = \"3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/compact_str/0.9.0/download\"],\n strip_prefix = \"compact_str-0.9.0\",\n build_file = Label(\"@crates//crates:BUILD.compact_str-0.9.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-deque-0.8.6\",\n sha256 = \"9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-deque/0.8.6/download\"],\n strip_prefix = \"crossbeam-deque-0.8.6\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-deque-0.8.6.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-epoch-0.9.18\",\n sha256 = \"5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-epoch/0.9.18/download\"],\n strip_prefix = \"crossbeam-epoch-0.9.18\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-epoch-0.9.18.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__crossbeam-utils-0.8.21\",\n sha256 = \"d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/crossbeam-utils/0.8.21/download\"],\n strip_prefix = \"crossbeam-utils-0.8.21\",\n build_file = Label(\"@crates//crates:BUILD.crossbeam-utils-0.8.21.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling-0.20.10\",\n sha256 = \"6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling/0.20.10/download\"],\n strip_prefix = \"darling-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling_core-0.20.10\",\n sha256 = \"95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling_core/0.20.10/download\"],\n strip_prefix = \"darling_core-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling_core-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__darling_macro-0.20.10\",\n sha256 = \"d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/darling_macro/0.20.10/download\"],\n strip_prefix = \"darling_macro-0.20.10\",\n build_file = Label(\"@crates//crates:BUILD.darling_macro-0.20.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__dary_heap-0.3.7\",\n sha256 = \"04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/dary_heap/0.3.7/download\"],\n strip_prefix = \"dary_heap-0.3.7\",\n build_file = Label(\"@crates//crates:BUILD.dary_heap-0.3.7.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder-0.20.2\",\n sha256 = \"507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder/0.20.2/download\"],\n strip_prefix = \"derive_builder-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder_core-0.20.2\",\n sha256 = \"2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder_core/0.20.2/download\"],\n strip_prefix = \"derive_builder_core-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder_core-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__derive_builder_macro-0.20.2\",\n sha256 = \"ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/derive_builder_macro/0.20.2/download\"],\n strip_prefix = \"derive_builder_macro-0.20.2\",\n build_file = Label(\"@crates//crates:BUILD.derive_builder_macro-0.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__either-1.13.0\",\n sha256 = \"60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/either/1.13.0/download\"],\n strip_prefix = \"either-1.13.0\",\n build_file = Label(\"@crates//crates:BUILD.either-1.13.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__esaxx-rs-0.1.10\",\n sha256 = \"d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/esaxx-rs/0.1.10/download\"],\n strip_prefix = \"esaxx-rs-0.1.10\",\n build_file = Label(\"@crates//crates:BUILD.esaxx-rs-0.1.10.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__fnv-1.0.7\",\n sha256 = \"3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/fnv/1.0.7/download\"],\n strip_prefix = \"fnv-1.0.7\",\n build_file = Label(\"@crates//crates:BUILD.fnv-1.0.7.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__getrandom-0.3.3\",\n sha256 = \"26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/getrandom/0.3.3/download\"],\n strip_prefix = \"getrandom-0.3.3\",\n build_file = Label(\"@crates//crates:BUILD.getrandom-0.3.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ident_case-1.0.1\",\n sha256 = \"b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ident_case/1.0.1/download\"],\n strip_prefix = \"ident_case-1.0.1\",\n build_file = Label(\"@crates//crates:BUILD.ident_case-1.0.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__itertools-0.14.0\",\n sha256 = \"2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/itertools/0.14.0/download\"],\n strip_prefix = \"itertools-0.14.0\",\n build_file = Label(\"@crates//crates:BUILD.itertools-0.14.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__itoa-1.0.14\",\n sha256 = \"d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/itoa/1.0.14/download\"],\n strip_prefix = \"itoa-1.0.14\",\n build_file = Label(\"@crates//crates:BUILD.itoa-1.0.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__libc-0.2.169\",\n sha256 = \"b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/libc/0.2.169/download\"],\n strip_prefix = \"libc-0.2.169\",\n build_file = Label(\"@crates//crates:BUILD.libc-0.2.169.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__log-0.4.22\",\n sha256 = \"a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/log/0.4.22/download\"],\n strip_prefix = \"log-0.4.22\",\n build_file = Label(\"@crates//crates:BUILD.log-0.4.22.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__macro_rules_attribute-0.2.0\",\n sha256 = \"8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/macro_rules_attribute/0.2.0/download\"],\n strip_prefix = \"macro_rules_attribute-0.2.0\",\n build_file = Label(\"@crates//crates:BUILD.macro_rules_attribute-0.2.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__macro_rules_attribute-proc_macro-0.2.0\",\n sha256 = \"b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/macro_rules_attribute-proc_macro/0.2.0/download\"],\n strip_prefix = \"macro_rules_attribute-proc_macro-0.2.0\",\n build_file = Label(\"@crates//crates:BUILD.macro_rules_attribute-proc_macro-0.2.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__memchr-2.7.4\",\n sha256 = \"78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/memchr/2.7.4/download\"],\n strip_prefix = \"memchr-2.7.4\",\n build_file = Label(\"@crates//crates:BUILD.memchr-2.7.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__minimal-lexical-0.2.1\",\n sha256 = \"68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/minimal-lexical/0.2.1/download\"],\n strip_prefix = \"minimal-lexical-0.2.1\",\n build_file = Label(\"@crates//crates:BUILD.minimal-lexical-0.2.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__monostate-0.1.13\",\n sha256 = \"0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/monostate/0.1.13/download\"],\n strip_prefix = \"monostate-0.1.13\",\n build_file = Label(\"@crates//crates:BUILD.monostate-0.1.13.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__monostate-impl-0.1.13\",\n sha256 = \"a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/monostate-impl/0.1.13/download\"],\n strip_prefix = \"monostate-impl-0.1.13\",\n build_file = Label(\"@crates//crates:BUILD.monostate-impl-0.1.13.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__nom-7.1.3\",\n sha256 = \"d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/nom/7.1.3/download\"],\n strip_prefix = \"nom-7.1.3\",\n build_file = Label(\"@crates//crates:BUILD.nom-7.1.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__once_cell-1.20.2\",\n sha256 = \"1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/once_cell/1.20.2/download\"],\n strip_prefix = \"once_cell-1.20.2\",\n build_file = Label(\"@crates//crates:BUILD.once_cell-1.20.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__onig-6.5.1\",\n sha256 = \"336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/onig/6.5.1/download\"],\n strip_prefix = \"onig-6.5.1\",\n build_file = Label(\"@crates//crates:BUILD.onig-6.5.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__onig_sys-69.9.1\",\n sha256 = \"c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/onig_sys/69.9.1/download\"],\n strip_prefix = \"onig_sys-69.9.1\",\n build_file = Label(\"@crates//crates:BUILD.onig_sys-69.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__paste-1.0.15\",\n sha256 = \"57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/paste/1.0.15/download\"],\n strip_prefix = \"paste-1.0.15\",\n build_file = Label(\"@crates//crates:BUILD.paste-1.0.15.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__pkg-config-0.3.32\",\n sha256 = \"7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/pkg-config/0.3.32/download\"],\n strip_prefix = \"pkg-config-0.3.32\",\n build_file = Label(\"@crates//crates:BUILD.pkg-config-0.3.32.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ppv-lite86-0.2.20\",\n sha256 = \"77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ppv-lite86/0.2.20/download\"],\n strip_prefix = \"ppv-lite86-0.2.20\",\n build_file = Label(\"@crates//crates:BUILD.ppv-lite86-0.2.20.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__proc-macro2-1.0.92\",\n sha256 = \"37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/proc-macro2/1.0.92/download\"],\n strip_prefix = \"proc-macro2-1.0.92\",\n build_file = Label(\"@crates//crates:BUILD.proc-macro2-1.0.92.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__quote-1.0.37\",\n sha256 = \"b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/quote/1.0.37/download\"],\n strip_prefix = \"quote-1.0.37\",\n build_file = Label(\"@crates//crates:BUILD.quote-1.0.37.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__r-efi-5.3.0\",\n sha256 = \"69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/r-efi/5.3.0/download\"],\n strip_prefix = \"r-efi-5.3.0\",\n build_file = Label(\"@crates//crates:BUILD.r-efi-5.3.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand-0.9.1\",\n sha256 = \"9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand/0.9.1/download\"],\n strip_prefix = \"rand-0.9.1\",\n build_file = Label(\"@crates//crates:BUILD.rand-0.9.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand_chacha-0.9.0\",\n sha256 = \"d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand_chacha/0.9.0/download\"],\n strip_prefix = \"rand_chacha-0.9.0\",\n build_file = Label(\"@crates//crates:BUILD.rand_chacha-0.9.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rand_core-0.9.3\",\n sha256 = \"99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rand_core/0.9.3/download\"],\n strip_prefix = \"rand_core-0.9.3\",\n build_file = Label(\"@crates//crates:BUILD.rand_core-0.9.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-1.10.0\",\n sha256 = \"b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon/1.10.0/download\"],\n strip_prefix = \"rayon-1.10.0\",\n build_file = Label(\"@crates//crates:BUILD.rayon-1.10.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-cond-0.4.0\",\n sha256 = \"2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon-cond/0.4.0/download\"],\n strip_prefix = \"rayon-cond-0.4.0\",\n build_file = Label(\"@crates//crates:BUILD.rayon-cond-0.4.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rayon-core-1.12.1\",\n sha256 = \"1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rayon-core/1.12.1/download\"],\n strip_prefix = \"rayon-core-1.12.1\",\n build_file = Label(\"@crates//crates:BUILD.rayon-core-1.12.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-1.11.1\",\n sha256 = \"b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex/1.11.1/download\"],\n strip_prefix = \"regex-1.11.1\",\n build_file = Label(\"@crates//crates:BUILD.regex-1.11.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-automata-0.4.9\",\n sha256 = \"809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-automata/0.4.9/download\"],\n strip_prefix = \"regex-automata-0.4.9\",\n build_file = Label(\"@crates//crates:BUILD.regex-automata-0.4.9.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-syntax-0.8.5\",\n sha256 = \"2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-syntax/0.8.5/download\"],\n strip_prefix = \"regex-syntax-0.8.5\",\n build_file = Label(\"@crates//crates:BUILD.regex-syntax-0.8.5.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rustversion-1.0.21\",\n sha256 = \"8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rustversion/1.0.21/download\"],\n strip_prefix = \"rustversion-1.0.21\",\n build_file = Label(\"@crates//crates:BUILD.rustversion-1.0.21.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__ryu-1.0.18\",\n sha256 = \"f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/ryu/1.0.18/download\"],\n strip_prefix = \"ryu-1.0.18\",\n build_file = Label(\"@crates//crates:BUILD.ryu-1.0.18.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde-1.0.216\",\n sha256 = \"0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde/1.0.216/download\"],\n strip_prefix = \"serde-1.0.216\",\n build_file = Label(\"@crates//crates:BUILD.serde-1.0.216.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde_derive-1.0.216\",\n sha256 = \"46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde_derive/1.0.216/download\"],\n strip_prefix = \"serde_derive-1.0.216\",\n build_file = Label(\"@crates//crates:BUILD.serde_derive-1.0.216.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__serde_json-1.0.134\",\n sha256 = \"d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/serde_json/1.0.134/download\"],\n strip_prefix = \"serde_json-1.0.134\",\n build_file = Label(\"@crates//crates:BUILD.serde_json-1.0.134.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__shlex-1.3.0\",\n sha256 = \"0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/shlex/1.3.0/download\"],\n strip_prefix = \"shlex-1.3.0\",\n build_file = Label(\"@crates//crates:BUILD.shlex-1.3.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__smallvec-1.13.2\",\n sha256 = \"3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/smallvec/1.13.2/download\"],\n strip_prefix = \"smallvec-1.13.2\",\n build_file = Label(\"@crates//crates:BUILD.smallvec-1.13.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__spm_precompiled-0.1.4\",\n sha256 = \"5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/spm_precompiled/0.1.4/download\"],\n strip_prefix = \"spm_precompiled-0.1.4\",\n build_file = Label(\"@crates//crates:BUILD.spm_precompiled-0.1.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__static_assertions-1.1.0\",\n sha256 = \"a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/static_assertions/1.1.0/download\"],\n strip_prefix = \"static_assertions-1.1.0\",\n build_file = Label(\"@crates//crates:BUILD.static_assertions-1.1.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__strsim-0.11.1\",\n sha256 = \"7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/strsim/0.11.1/download\"],\n strip_prefix = \"strsim-0.11.1\",\n build_file = Label(\"@crates//crates:BUILD.strsim-0.11.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__syn-2.0.91\",\n sha256 = \"d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/syn/2.0.91/download\"],\n strip_prefix = \"syn-2.0.91\",\n build_file = Label(\"@crates//crates:BUILD.syn-2.0.91.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__thiserror-2.0.12\",\n sha256 = \"567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/thiserror/2.0.12/download\"],\n strip_prefix = \"thiserror-2.0.12\",\n build_file = Label(\"@crates//crates:BUILD.thiserror-2.0.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__thiserror-impl-2.0.12\",\n sha256 = \"7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/thiserror-impl/2.0.12/download\"],\n strip_prefix = \"thiserror-impl-2.0.12\",\n build_file = Label(\"@crates//crates:BUILD.thiserror-impl-2.0.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__tokenizers-0.22.2\",\n sha256 = \"b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/tokenizers/0.22.2/download\"],\n strip_prefix = \"tokenizers-0.22.2\",\n build_file = Label(\"@crates//crates:BUILD.tokenizers-0.22.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-ident-1.0.14\",\n sha256 = \"adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-ident/1.0.14/download\"],\n strip_prefix = \"unicode-ident-1.0.14\",\n build_file = Label(\"@crates//crates:BUILD.unicode-ident-1.0.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-normalization-alignments-0.1.12\",\n sha256 = \"43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-normalization-alignments/0.1.12/download\"],\n strip_prefix = \"unicode-normalization-alignments-0.1.12\",\n build_file = Label(\"@crates//crates:BUILD.unicode-normalization-alignments-0.1.12.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-segmentation-1.12.0\",\n sha256 = \"f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-segmentation/1.12.0/download\"],\n strip_prefix = \"unicode-segmentation-1.12.0\",\n build_file = Label(\"@crates//crates:BUILD.unicode-segmentation-1.12.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode_categories-0.1.1\",\n sha256 = \"39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode_categories/0.1.1/download\"],\n strip_prefix = \"unicode_categories-0.1.1\",\n build_file = Label(\"@crates//crates:BUILD.unicode_categories-0.1.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__version_check-0.9.5\",\n sha256 = \"0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/version_check/0.9.5/download\"],\n strip_prefix = \"version_check-0.9.5\",\n build_file = Label(\"@crates//crates:BUILD.version_check-0.9.5.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wasi-0.14.2-wasi-0.2.4\",\n sha256 = \"9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wasi/0.14.2+wasi-0.2.4/download\"],\n strip_prefix = \"wasi-0.14.2+wasi-0.2.4\",\n build_file = Label(\"@crates//crates:BUILD.wasi-0.14.2+wasi-0.2.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__wit-bindgen-rt-0.39.0\",\n sha256 = \"6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/wit-bindgen-rt/0.39.0/download\"],\n strip_prefix = \"wit-bindgen-rt-0.39.0\",\n build_file = Label(\"@crates//crates:BUILD.wit-bindgen-rt-0.39.0.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-0.7.35\",\n sha256 = \"1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy/0.7.35/download\"],\n strip_prefix = \"zerocopy-0.7.35\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-0.7.35.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-0.8.26\",\n sha256 = \"1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy/0.8.26/download\"],\n strip_prefix = \"zerocopy-0.8.26\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-0.8.26.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-derive-0.7.35\",\n sha256 = \"fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy-derive/0.7.35/download\"],\n strip_prefix = \"zerocopy-derive-0.7.35\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-derive-0.7.35.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__zerocopy-derive-0.8.26\",\n sha256 = \"9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/zerocopy-derive/0.8.26/download\"],\n strip_prefix = \"zerocopy-derive-0.8.26\",\n build_file = Label(\"@crates//crates:BUILD.zerocopy-derive-0.8.26.bazel\"),\n )\n\n return [\n struct(repo=\"crates__tokenizers-0.22.2\", is_dev_dep = False),\n ]\n" - } - } - }, - "crates__ahash-0.8.12": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/ahash/0.8.12/download" - ], - "strip_prefix": "ahash-0.8.12", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"ahash\",\n deps = [\n \"@crates__ahash-0.8.12//:build_script_build\",\n \"@crates__cfg-if-1.0.0//:cfg_if\",\n \"@crates__getrandom-0.3.3//:getrandom\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__zerocopy-0.8.26//:zerocopy\",\n ] + select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [\n \"@crates__once_cell-1.20.2//:once_cell\", # cfg(not(all(target_arch = \"arm\", target_os = \"none\")))\n ],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [\n \"@crates__once_cell-1.20.2//:once_cell\", # cfg(not(all(target_arch = \"arm\", target_os = \"none\")))\n ],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [\n \"@crates__once_cell-1.20.2//:once_cell\", # cfg(not(all(target_arch = \"arm\", target_os = \"none\")))\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [\n \"@crates__once_cell-1.20.2//:once_cell\", # cfg(not(all(target_arch = \"arm\", target_os = \"none\")))\n ],\n \"//conditions:default\": [],\n }),\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"getrandom\",\n \"runtime-rng\",\n \"serde\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=ahash\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.12\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"getrandom\",\n \"runtime-rng\",\n \"serde\",\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n deps = [\n \"@crates__version_check-0.9.5//:version_check\",\n ],\n edition = \"2018\",\n pkg_name = \"ahash\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=ahash\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.8.12\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__aho-corasick-1.1.3": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/aho-corasick/1.1.3/download" - ], - "strip_prefix": "aho-corasick-1.1.3", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"aho_corasick\",\n deps = [\n \"@crates__memchr-2.7.4//:memchr\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"perf-literal\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=aho-corasick\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.1.3\",\n)\n" - } - }, - "crates__base64-0.13.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/base64/0.13.1/download" - ], - "strip_prefix": "base64-0.13.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"base64\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=base64\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.13.1\",\n)\n" - } - }, - "crates__bitflags-2.9.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/bitflags/2.9.1/download" - ], - "strip_prefix": "bitflags-2.9.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"bitflags\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=bitflags\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.9.1\",\n)\n" - } - }, - "crates__byteorder-1.5.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/byteorder/1.5.0/download" - ], - "strip_prefix": "byteorder-1.5.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"byteorder\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=byteorder\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.5.0\",\n)\n" - } - }, - "crates__castaway-0.2.4": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/castaway/0.2.4/download" - ], - "strip_prefix": "castaway-0.2.4", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"castaway\",\n proc_macro_deps = [\n \"@crates__rustversion-1.0.21//:rustversion\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=castaway\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.4\",\n)\n" - } - }, - "crates__cc-1.2.33": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/cc/1.2.33/download" - ], - "strip_prefix": "cc-1.2.33", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"cc\",\n deps = [\n \"@crates__shlex-1.3.0//:shlex\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=cc\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.2.33\",\n)\n" - } - }, - "crates__cfg-if-1.0.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/cfg-if/1.0.0/download" - ], - "strip_prefix": "cfg-if-1.0.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"cfg_if\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=cfg-if\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.0\",\n)\n" - } - }, - "crates__compact_str-0.9.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/compact_str/0.9.0/download" - ], - "strip_prefix": "compact_str-0.9.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"compact_str\",\n deps = [\n \"@crates__castaway-0.2.4//:castaway\",\n \"@crates__cfg-if-1.0.0//:cfg_if\",\n \"@crates__itoa-1.0.14//:itoa\",\n \"@crates__ryu-1.0.18//:ryu\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__static_assertions-1.1.0//:static_assertions\",\n ],\n proc_macro_deps = [\n \"@crates__rustversion-1.0.21//:rustversion\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"serde\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=compact_str\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.9.0\",\n)\n" - } - }, - "crates__crossbeam-deque-0.8.6": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/crossbeam-deque/0.8.6/download" - ], - "strip_prefix": "crossbeam-deque-0.8.6", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"crossbeam_deque\",\n deps = [\n \"@crates__crossbeam-epoch-0.9.18//:crossbeam_epoch\",\n \"@crates__crossbeam-utils-0.8.21//:crossbeam_utils\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=crossbeam-deque\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.6\",\n)\n" - } - }, - "crates__crossbeam-epoch-0.9.18": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/crossbeam-epoch/0.9.18/download" - ], - "strip_prefix": "crossbeam-epoch-0.9.18", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"crossbeam_epoch\",\n deps = [\n \"@crates__crossbeam-utils-0.8.21//:crossbeam_utils\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=crossbeam-epoch\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.9.18\",\n)\n" - } - }, - "crates__crossbeam-utils-0.8.21": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/crossbeam-utils/0.8.21/download" - ], - "strip_prefix": "crossbeam-utils-0.8.21", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"crossbeam_utils\",\n deps = [\n \"@crates__crossbeam-utils-0.8.21//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=crossbeam-utils\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.21\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"crossbeam-utils\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=crossbeam-utils\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.8.21\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__darling-0.20.10": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/darling/0.20.10/download" - ], - "strip_prefix": "darling-0.20.10", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"darling\",\n deps = [\n \"@crates__darling_core-0.20.10//:darling_core\",\n ],\n proc_macro_deps = [\n \"@crates__darling_macro-0.20.10//:darling_macro\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"suggestions\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=darling\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.20.10\",\n)\n" - } - }, - "crates__darling_core-0.20.10": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/darling_core/0.20.10/download" - ], - "strip_prefix": "darling_core-0.20.10", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"darling_core\",\n deps = [\n \"@crates__fnv-1.0.7//:fnv\",\n \"@crates__ident_case-1.0.1//:ident_case\",\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__strsim-0.11.1//:strsim\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"strsim\",\n \"suggestions\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=darling_core\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.20.10\",\n)\n" - } - }, - "crates__darling_macro-0.20.10": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/darling_macro/0.20.10/download" - ], - "strip_prefix": "darling_macro-0.20.10", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"darling_macro\",\n deps = [\n \"@crates__darling_core-0.20.10//:darling_core\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=darling_macro\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.20.10\",\n)\n" - } - }, - "crates__dary_heap-0.3.7": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/dary_heap/0.3.7/download" - ], - "strip_prefix": "dary_heap-0.3.7", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"dary_heap\",\n deps = [\n \"@crates__serde-1.0.216//:serde\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"serde\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=dary_heap\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.7\",\n)\n" - } - }, - "crates__derive_builder-0.20.2": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/derive_builder/0.20.2/download" - ], - "strip_prefix": "derive_builder-0.20.2", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"derive_builder\",\n proc_macro_deps = [\n \"@crates__derive_builder_macro-0.20.2//:derive_builder_macro\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=derive_builder\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.20.2\",\n)\n" - } - }, - "crates__derive_builder_core-0.20.2": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/derive_builder_core/0.20.2/download" - ], - "strip_prefix": "derive_builder_core-0.20.2", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"derive_builder_core\",\n deps = [\n \"@crates__darling-0.20.10//:darling\",\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"lib_has_std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=derive_builder_core\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.20.2\",\n)\n" - } - }, - "crates__derive_builder_macro-0.20.2": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/derive_builder_macro/0.20.2/download" - ], - "strip_prefix": "derive_builder_macro-0.20.2", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"derive_builder_macro\",\n deps = [\n \"@crates__derive_builder_core-0.20.2//:derive_builder_core\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"lib_has_std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=derive_builder_macro\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.20.2\",\n)\n" - } - }, - "crates__either-1.13.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/either/1.13.0/download" - ], - "strip_prefix": "either-1.13.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"either\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"use_std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=either\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.13.0\",\n)\n" - } - }, - "crates__esaxx-rs-0.1.10": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/esaxx-rs/0.1.10/download" - ], - "strip_prefix": "esaxx-rs-0.1.10", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"esaxx_rs\",\n deps = [\n \"@crates__esaxx-rs-0.1.10//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"cc\",\n \"cpp\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=esaxx-rs\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.10\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"cc\",\n \"cpp\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n deps = [\n \"@crates__cc-1.2.33//:cc\",\n ],\n edition = \"2018\",\n pkg_name = \"esaxx-rs\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=esaxx-rs\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.1.10\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__fnv-1.0.7": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/fnv/1.0.7/download" - ], - "strip_prefix": "fnv-1.0.7", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"fnv\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=fnv\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.7\",\n)\n" - } - }, - "crates__getrandom-0.3.3": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/getrandom/0.3.3/download" - ], - "strip_prefix": "getrandom-0.3.3", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"getrandom\",\n deps = [\n \"@crates__cfg-if-1.0.0//:cfg_if\",\n \"@crates__getrandom-0.3.3//:build_script_build\",\n ] + select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(any(target_os = \"macos\", target_os = \"openbsd\", target_os = \"vita\", target_os = \"emscripten\"))\n ],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(all(any(target_os = \"linux\", target_os = \"android\"), not(any(all(target_os = \"linux\", target_env = \"\"), getrandom_backend = \"custom\", getrandom_backend = \"linux_raw\", getrandom_backend = \"rdrand\", getrandom_backend = \"rndr\"))))\n ],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(any(target_os = \"macos\", target_os = \"openbsd\", target_os = \"vita\", target_os = \"emscripten\"))\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [\n \"@crates__libc-0.2.169//:libc\", # cfg(all(any(target_os = \"linux\", target_os = \"android\"), not(any(all(target_os = \"linux\", target_env = \"\"), getrandom_backend = \"custom\", getrandom_backend = \"linux_raw\", getrandom_backend = \"rdrand\", getrandom_backend = \"rndr\"))))\n ],\n \"//conditions:default\": [],\n }),\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=getrandom\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.3\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"getrandom\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=getrandom\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.3.3\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__ident_case-1.0.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/ident_case/1.0.1/download" - ], - "strip_prefix": "ident_case-1.0.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"ident_case\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=ident_case\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.1\",\n)\n" - } - }, - "crates__itertools-0.14.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/itertools/0.14.0/download" - ], - "strip_prefix": "itertools-0.14.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"itertools\",\n deps = [\n \"@crates__either-1.13.0//:either\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"use_alloc\",\n \"use_std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=itertools\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.14.0\",\n)\n" - } - }, - "crates__itoa-1.0.14": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/itoa/1.0.14/download" - ], - "strip_prefix": "itoa-1.0.14", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"itoa\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=itoa\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.14\",\n)\n" - } - }, - "crates__libc-0.2.169": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/libc/0.2.169/download" - ], - "strip_prefix": "libc-0.2.169", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"libc\",\n deps = [\n \"@crates__libc-0.2.169//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=libc\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.169\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"libc\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=libc\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.2.169\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__log-0.4.22": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/log/0.4.22/download" - ], - "strip_prefix": "log-0.4.22", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"log\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=log\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.4.22\",\n)\n" - } - }, - "crates__macro_rules_attribute-0.2.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/macro_rules_attribute/0.2.0/download" - ], - "strip_prefix": "macro_rules_attribute-0.2.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"macro_rules_attribute\",\n proc_macro_deps = [\n \"@crates__macro_rules_attribute-proc_macro-0.2.0//:macro_rules_attribute_proc_macro\",\n \"@crates__paste-1.0.15//:paste\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=macro_rules_attribute\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.0\",\n)\n" - } - }, - "crates__macro_rules_attribute-proc_macro-0.2.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/macro_rules_attribute-proc_macro/0.2.0/download" - ], - "strip_prefix": "macro_rules_attribute-proc_macro-0.2.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"macro_rules_attribute_proc_macro\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"mod.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=macro_rules_attribute-proc_macro\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.0\",\n)\n" - } - }, - "crates__memchr-2.7.4": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/memchr/2.7.4/download" - ], - "strip_prefix": "memchr-2.7.4", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"memchr\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=memchr\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.7.4\",\n)\n" - } - }, - "crates__minimal-lexical-0.2.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/minimal-lexical/0.2.1/download" - ], - "strip_prefix": "minimal-lexical-0.2.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"minimal_lexical\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=minimal-lexical\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.1\",\n)\n" - } - }, - "crates__monostate-0.1.13": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/monostate/0.1.13/download" - ], - "strip_prefix": "monostate-0.1.13", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"monostate\",\n deps = [\n \"@crates__serde-1.0.216//:serde\",\n ],\n proc_macro_deps = [\n \"@crates__monostate-impl-0.1.13//:monostate_impl\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=monostate\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.13\",\n)\n" - } - }, - "crates__monostate-impl-0.1.13": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/monostate-impl/0.1.13/download" - ], - "strip_prefix": "monostate-impl-0.1.13", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"monostate_impl\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=monostate-impl\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.13\",\n)\n" - } - }, - "crates__nom-7.1.3": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/nom/7.1.3/download" - ], - "strip_prefix": "nom-7.1.3", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"nom\",\n deps = [\n \"@crates__memchr-2.7.4//:memchr\",\n \"@crates__minimal-lexical-0.2.1//:minimal_lexical\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=nom\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"7.1.3\",\n)\n" - } - }, - "crates__once_cell-1.20.2": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/once_cell/1.20.2/download" - ], - "strip_prefix": "once_cell-1.20.2", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"once_cell\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"default\",\n \"race\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=once_cell\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.20.2\",\n)\n" - } - }, - "crates__onig-6.5.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/onig/6.5.1/download" - ], - "strip_prefix": "onig-6.5.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"onig\",\n deps = [\n \"@crates__bitflags-2.9.1//:bitflags\",\n \"@crates__once_cell-1.20.2//:once_cell\",\n \"@crates__onig_sys-69.9.1//:onig_sys\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=onig\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"6.5.1\",\n)\n" - } - }, - "crates__onig_sys-69.9.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/onig_sys/69.9.1/download" - ], - "strip_prefix": "onig_sys-69.9.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"onig_sys\",\n deps = [\n \"@crates__onig_sys-69.9.1//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=onig_sys\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"69.9.1\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n deps = [\n \"@crates__cc-1.2.33//:cc\",\n \"@crates__pkg-config-0.3.32//:pkg_config\",\n ],\n edition = \"2021\",\n links = \"onig\",\n pkg_name = \"onig_sys\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=onig_sys\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"69.9.1\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__paste-1.0.15": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/paste/1.0.15/download" - ], - "strip_prefix": "paste-1.0.15", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"paste\",\n deps = [\n \"@crates__paste-1.0.15//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=paste\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.15\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2018\",\n pkg_name = \"paste\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=paste\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.15\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__pkg-config-0.3.32": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/pkg-config/0.3.32/download" - ], - "strip_prefix": "pkg-config-0.3.32", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"pkg_config\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=pkg-config\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.32\",\n)\n" - } - }, - "crates__ppv-lite86-0.2.20": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/ppv-lite86/0.2.20/download" - ], - "strip_prefix": "ppv-lite86-0.2.20", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"ppv_lite86\",\n deps = [\n \"@crates__zerocopy-0.7.35//:zerocopy\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"simd\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=ppv-lite86\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.20\",\n)\n" - } - }, - "crates__proc-macro2-1.0.92": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/proc-macro2/1.0.92/download" - ], - "strip_prefix": "proc-macro2-1.0.92", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"proc_macro2\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:build_script_build\",\n \"@crates__unicode-ident-1.0.14//:unicode_ident\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"proc-macro\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=proc-macro2\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.92\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"proc-macro\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"proc-macro2\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=proc-macro2\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.92\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__quote-1.0.37": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/quote/1.0.37/download" - ], - "strip_prefix": "quote-1.0.37", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"quote\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"proc-macro\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=quote\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.37\",\n)\n" - } - }, - "crates__r-efi-5.3.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/r-efi/5.3.0/download" - ], - "strip_prefix": "r-efi-5.3.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"r_efi\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=r-efi\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"5.3.0\",\n)\n" - } - }, - "crates__rand-0.9.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/rand/0.9.1/download" - ], - "strip_prefix": "rand-0.9.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"rand\",\n deps = [\n \"@crates__rand_chacha-0.9.0//:rand_chacha\",\n \"@crates__rand_core-0.9.3//:rand_core\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"default\",\n \"os_rng\",\n \"small_rng\",\n \"std\",\n \"std_rng\",\n \"thread_rng\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rand\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.9.1\",\n)\n" - } - }, - "crates__rand_chacha-0.9.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/rand_chacha/0.9.0/download" - ], - "strip_prefix": "rand_chacha-0.9.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"rand_chacha\",\n deps = [\n \"@crates__ppv-lite86-0.2.20//:ppv_lite86\",\n \"@crates__rand_core-0.9.3//:rand_core\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rand_chacha\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.9.0\",\n)\n" - } - }, - "crates__rand_core-0.9.3": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/rand_core/0.9.3/download" - ], - "strip_prefix": "rand_core-0.9.3", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"rand_core\",\n deps = [\n \"@crates__getrandom-0.3.3//:getrandom\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"os_rng\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rand_core\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.9.3\",\n)\n" - } - }, - "crates__rayon-1.10.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/rayon/1.10.0/download" - ], - "strip_prefix": "rayon-1.10.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"rayon\",\n deps = [\n \"@crates__either-1.13.0//:either\",\n \"@crates__rayon-core-1.12.1//:rayon_core\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rayon\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.10.0\",\n)\n" - } - }, - "crates__rayon-cond-0.4.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/rayon-cond/0.4.0/download" - ], - "strip_prefix": "rayon-cond-0.4.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"rayon_cond\",\n deps = [\n \"@crates__either-1.13.0//:either\",\n \"@crates__itertools-0.14.0//:itertools\",\n \"@crates__rayon-1.10.0//:rayon\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rayon-cond\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.4.0\",\n)\n" - } - }, - "crates__rayon-core-1.12.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/rayon-core/1.12.1/download" - ], - "strip_prefix": "rayon-core-1.12.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"rayon_core\",\n deps = [\n \"@crates__crossbeam-deque-0.8.6//:crossbeam_deque\",\n \"@crates__crossbeam-utils-0.8.21//:crossbeam_utils\",\n \"@crates__rayon-core-1.12.1//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rayon-core\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.12.1\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n links = \"rayon-core\",\n pkg_name = \"rayon-core\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rayon-core\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.12.1\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__regex-1.11.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/regex/1.11.1/download" - ], - "strip_prefix": "regex-1.11.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"regex\",\n deps = [\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__memchr-2.7.4//:memchr\",\n \"@crates__regex-automata-0.4.9//:regex_automata\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"perf\",\n \"perf-backtrack\",\n \"perf-cache\",\n \"perf-dfa\",\n \"perf-inline\",\n \"perf-literal\",\n \"perf-onepass\",\n \"std\",\n \"unicode\",\n \"unicode-age\",\n \"unicode-bool\",\n \"unicode-case\",\n \"unicode-gencat\",\n \"unicode-perl\",\n \"unicode-script\",\n \"unicode-segment\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=regex\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.11.1\",\n)\n" - } - }, - "crates__regex-automata-0.4.9": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/regex-automata/0.4.9/download" - ], - "strip_prefix": "regex-automata-0.4.9", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"regex_automata\",\n deps = [\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__memchr-2.7.4//:memchr\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"dfa-onepass\",\n \"hybrid\",\n \"meta\",\n \"nfa-backtrack\",\n \"nfa-pikevm\",\n \"nfa-thompson\",\n \"perf-inline\",\n \"perf-literal\",\n \"perf-literal-multisubstring\",\n \"perf-literal-substring\",\n \"std\",\n \"syntax\",\n \"unicode\",\n \"unicode-age\",\n \"unicode-bool\",\n \"unicode-case\",\n \"unicode-gencat\",\n \"unicode-perl\",\n \"unicode-script\",\n \"unicode-segment\",\n \"unicode-word-boundary\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=regex-automata\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.4.9\",\n)\n" - } - }, - "crates__regex-syntax-0.8.5": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/regex-syntax/0.8.5/download" - ], - "strip_prefix": "regex-syntax-0.8.5", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"regex_syntax\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n \"unicode\",\n \"unicode-age\",\n \"unicode-bool\",\n \"unicode-case\",\n \"unicode-gencat\",\n \"unicode-perl\",\n \"unicode-script\",\n \"unicode-segment\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=regex-syntax\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.5\",\n)\n" - } - }, - "crates__rustversion-1.0.21": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/rustversion/1.0.21/download" - ], - "strip_prefix": "rustversion-1.0.21", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"rustversion\",\n deps = [\n \"@crates__rustversion-1.0.21//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rustversion\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.21\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build/build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2018\",\n pkg_name = \"rustversion\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rustversion\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.21\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__ryu-1.0.18": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/ryu/1.0.18/download" - ], - "strip_prefix": "ryu-1.0.18", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"ryu\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=ryu\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.18\",\n)\n" - } - }, - "crates__serde-1.0.216": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/serde/1.0.216/download" - ], - "strip_prefix": "serde-1.0.216", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"serde\",\n deps = [\n \"@crates__serde-1.0.216//:build_script_build\",\n ],\n proc_macro_deps = [\n \"@crates__serde_derive-1.0.216//:serde_derive\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"default\",\n \"derive\",\n \"serde_derive\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.216\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"default\",\n \"derive\",\n \"serde_derive\",\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2018\",\n pkg_name = \"serde\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.216\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__serde_derive-1.0.216": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/serde_derive/1.0.216/download" - ], - "strip_prefix": "serde_derive-1.0.216", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"serde_derive\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde_derive\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.216\",\n)\n" - } - }, - "crates__serde_json-1.0.134": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/serde_json/1.0.134/download" - ], - "strip_prefix": "serde_json-1.0.134", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"serde_json\",\n deps = [\n \"@crates__itoa-1.0.14//:itoa\",\n \"@crates__memchr-2.7.4//:memchr\",\n \"@crates__ryu-1.0.18//:ryu\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__serde_json-1.0.134//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde_json\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.134\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"serde_json\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=serde_json\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.134\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__shlex-1.3.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/shlex/1.3.0/download" - ], - "strip_prefix": "shlex-1.3.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"shlex\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=shlex\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.3.0\",\n)\n" - } - }, - "crates__smallvec-1.13.2": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/smallvec/1.13.2/download" - ], - "strip_prefix": "smallvec-1.13.2", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"smallvec\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=smallvec\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.13.2\",\n)\n" - } - }, - "crates__spm_precompiled-0.1.4": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/spm_precompiled/0.1.4/download" - ], - "strip_prefix": "spm_precompiled-0.1.4", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"spm_precompiled\",\n deps = [\n \"@crates__base64-0.13.1//:base64\",\n \"@crates__nom-7.1.3//:nom\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__unicode-segmentation-1.12.0//:unicode_segmentation\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=spm_precompiled\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.4\",\n)\n" - } - }, - "crates__static_assertions-1.1.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/static_assertions/1.1.0/download" - ], - "strip_prefix": "static_assertions-1.1.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"static_assertions\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=static_assertions\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.1.0\",\n)\n" - } - }, - "crates__strsim-0.11.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/strsim/0.11.1/download" - ], - "strip_prefix": "strsim-0.11.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"strsim\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=strsim\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.11.1\",\n)\n" - } - }, - "crates__syn-2.0.91": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/syn/2.0.91/download" - ], - "strip_prefix": "syn-2.0.91", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"syn\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__unicode-ident-1.0.14//:unicode_ident\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"clone-impls\",\n \"default\",\n \"derive\",\n \"extra-traits\",\n \"full\",\n \"parsing\",\n \"printing\",\n \"proc-macro\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=syn\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.0.91\",\n)\n" - } - }, - "crates__thiserror-2.0.12": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/thiserror/2.0.12/download" - ], - "strip_prefix": "thiserror-2.0.12", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"thiserror\",\n deps = [\n \"@crates__thiserror-2.0.12//:build_script_build\",\n ],\n proc_macro_deps = [\n \"@crates__thiserror-impl-2.0.12//:thiserror_impl\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=thiserror\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.0.12\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"thiserror\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=thiserror\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"2.0.12\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__thiserror-impl-2.0.12": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/thiserror-impl/2.0.12/download" - ], - "strip_prefix": "thiserror-impl-2.0.12", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"thiserror_impl\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=thiserror-impl\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.0.12\",\n)\n" - } - }, - "crates__tokenizers-0.22.2": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/tokenizers/0.22.2/download" - ], - "strip_prefix": "tokenizers-0.22.2", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"tokenizers\",\n deps = [\n \"@crates__ahash-0.8.12//:ahash\",\n \"@crates__aho-corasick-1.1.3//:aho_corasick\",\n \"@crates__compact_str-0.9.0//:compact_str\",\n \"@crates__dary_heap-0.3.7//:dary_heap\",\n \"@crates__derive_builder-0.20.2//:derive_builder\",\n \"@crates__esaxx-rs-0.1.10//:esaxx_rs\",\n \"@crates__getrandom-0.3.3//:getrandom\",\n \"@crates__itertools-0.14.0//:itertools\",\n \"@crates__log-0.4.22//:log\",\n \"@crates__macro_rules_attribute-0.2.0//:macro_rules_attribute\",\n \"@crates__monostate-0.1.13//:monostate\",\n \"@crates__onig-6.5.1//:onig\",\n \"@crates__rand-0.9.1//:rand\",\n \"@crates__rayon-1.10.0//:rayon\",\n \"@crates__rayon-cond-0.4.0//:rayon_cond\",\n \"@crates__regex-1.11.1//:regex\",\n \"@crates__regex-syntax-0.8.5//:regex_syntax\",\n \"@crates__serde-1.0.216//:serde\",\n \"@crates__serde_json-1.0.134//:serde_json\",\n \"@crates__spm_precompiled-0.1.4//:spm_precompiled\",\n \"@crates__thiserror-2.0.12//:thiserror\",\n \"@crates__unicode-normalization-alignments-0.1.12//:unicode_normalization_alignments\",\n \"@crates__unicode-segmentation-1.12.0//:unicode_segmentation\",\n \"@crates__unicode_categories-0.1.1//:unicode_categories\",\n ],\n proc_macro_deps = [\n \"@crates__paste-1.0.15//:paste\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"esaxx_fast\",\n \"onig\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=tokenizers\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.22.2\",\n)\n" - } - }, - "crates__unicode-ident-1.0.14": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/unicode-ident/1.0.14/download" - ], - "strip_prefix": "unicode-ident-1.0.14", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"unicode_ident\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=unicode-ident\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.14\",\n)\n" - } - }, - "crates__unicode-normalization-alignments-0.1.12": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/unicode-normalization-alignments/0.1.12/download" - ], - "strip_prefix": "unicode-normalization-alignments-0.1.12", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"unicode_normalization_alignments\",\n deps = [\n \"@crates__smallvec-1.13.2//:smallvec\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=unicode-normalization-alignments\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.12\",\n)\n" - } - }, - "crates__unicode-segmentation-1.12.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/unicode-segmentation/1.12.0/download" - ], - "strip_prefix": "unicode-segmentation-1.12.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"unicode_segmentation\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=unicode-segmentation\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.12.0\",\n)\n" - } - }, - "crates__unicode_categories-0.1.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/unicode_categories/0.1.1/download" - ], - "strip_prefix": "unicode_categories-0.1.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"unicode_categories\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=unicode_categories\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.1.1\",\n)\n" - } - }, - "crates__version_check-0.9.5": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/version_check/0.9.5/download" - ], - "strip_prefix": "version_check-0.9.5", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"version_check\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=version_check\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.9.5\",\n)\n" - } - }, - "crates__wasi-0.14.2-wasi-0.2.4": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/wasi/0.14.2+wasi-0.2.4/download" - ], - "strip_prefix": "wasi-0.14.2+wasi-0.2.4", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wasi\",\n deps = [\n \"@crates__wit-bindgen-rt-0.39.0//:wit_bindgen_rt\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wasi\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.14.2+wasi-0.2.4\",\n)\n" - } - }, - "crates__wit-bindgen-rt-0.39.0": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/wit-bindgen-rt/0.39.0/download" - ], - "strip_prefix": "wit-bindgen-rt-0.39.0", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"wit_bindgen_rt\",\n deps = [\n \"@crates__wit-bindgen-rt-0.39.0//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wit-bindgen-rt\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.39.0\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"wit-bindgen-rt\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=wit-bindgen-rt\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.39.0\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__zerocopy-0.7.35": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/zerocopy/0.7.35/download" - ], - "strip_prefix": "zerocopy-0.7.35", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"zerocopy\",\n deps = [\n \"@crates__byteorder-1.5.0//:byteorder\",\n ],\n proc_macro_deps = [\n \"@crates__zerocopy-derive-0.7.35//:zerocopy_derive\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"byteorder\",\n \"default\",\n \"derive\",\n \"simd\",\n \"zerocopy-derive\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=zerocopy\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.7.35\",\n)\n" - } - }, - "crates__zerocopy-0.8.26": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/zerocopy/0.8.26/download" - ], - "strip_prefix": "zerocopy-0.8.26", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"zerocopy\",\n deps = [\n \"@crates__zerocopy-0.8.26//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"simd\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=zerocopy\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.26\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"simd\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"zerocopy\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=zerocopy\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.8.26\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__zerocopy-derive-0.7.35": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/zerocopy-derive/0.7.35/download" - ], - "strip_prefix": "zerocopy-derive-0.7.35", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"zerocopy_derive\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=zerocopy-derive\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.7.35\",\n)\n" - } - }, - "crates__zerocopy-derive-0.8.26": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "patch_args": [], - "patch_tool": "", - "patches": [], - "remote_patch_strip": 1, - "sha256": "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/zerocopy-derive/0.8.26/download" - ], - "strip_prefix": "zerocopy-derive-0.8.26", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'zml'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"zerocopy_derive\",\n deps = [\n \"@crates__proc-macro2-1.0.92//:proc_macro2\",\n \"@crates__quote-1.0.37//:quote\",\n \"@crates__syn-2.0.91//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=zerocopy-derive\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-apple-darwin\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.26\",\n)\n" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "bazel_features+", - "bazel_features_globals", - "bazel_features++version_extension+bazel_features_globals" - ], - [ - "bazel_features+", - "bazel_features_version", - "bazel_features++version_extension+bazel_features_version" - ], - [ - "rules_cc+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_cc+", - "cc_compatibility_proxy", - "rules_cc++compatibility_proxy+cc_compatibility_proxy" - ], - [ - "rules_cc+", - "rules_cc", - "rules_cc+" - ], - [ - "rules_cc++compatibility_proxy+cc_compatibility_proxy", - "rules_cc", - "rules_cc+" - ], - [ - "rules_rust+", - "bazel_features", - "bazel_features+" - ], - [ - "rules_rust+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "rules_rust+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_rust+", - "rules_cc", - "rules_cc+" - ], - [ - "rules_rust+", - "rules_rust", - "rules_rust+" - ] - ] - } - }, "@@rules_zig+//zig:extensions.bzl%zig": { "general": { - "bzlTransitiveDigest": "AqBp0Gbe2OObIJWxY2tUasJpYX8sJRaKBfRgA1KUQN4=", - "usagesDigest": "MUggfXR9Y92m2LJD1coQy9DezZvA1s0RkFDA7YzrxBw=", + "bzlTransitiveDigest": "p1+tPOh3UlhcIaGGf1ycZCCC6I4NIHhiiHS0nAhhgHA=", + "usagesDigest": "8TQ1JYfn2xQJIQya4EwCvLiuZAdGIocYzOMc1au4GNs=", "recordedFileInputs": { - "@@//bazel/zig_index.json": "d38c409f2a4af04ad52d52f817ff71b77bb919194f5be6c34f827c68296d5795", + "@@//bazel/zig_index.json": "0b2647eea9809f05c33dac461c6572e85e0401221bca32ebb6353100132f29c5", "@@rules_zig+//zig/private/versions.json": "6eb85ebaee72c4e6fbc82d8ad73f69e0899d839b70a7e19caa792ab6a5fa0c36" }, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { - "zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-linux": { + "zig_0.16.0-dev.2611_Pf996d2866_aarch64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "url": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.2611+f996d2866.tar.xz", "mirrors": [ - "https://mirror.zml.ai/zig", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds", "https://ziglang.org/builds/" ], - "sha256": "de4eddf26f6f5e95d1bd055577cba7e9ebe9fbe8b1425bd4d40f073303ccc65b", - "zig_version": "0.16.0-dev.1912+0cbaaa5eb", + "sha256": "d4219908cded6efb97a7326e4159f82607e3956d6d56efb1580187bf0872603e", + "zig_version": "0.16.0-dev.2611+f996d2866", "platform": "aarch64-linux" } }, - "zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-macos": { + "zig_0.16.0-dev.2611_Pf996d2866_aarch64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "url": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.2611+f996d2866.tar.xz", "mirrors": [ - "https://mirror.zml.ai/zig", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds", "https://ziglang.org/builds/" ], - "sha256": "241fa43a37803a9e50e7f4a2d2c436882ee2215e6c420c454cb4de57adae84d3", - "zig_version": "0.16.0-dev.1912+0cbaaa5eb", + "sha256": "1eb526bb948c1c1bffdb5be318c044fc0216457be28b0d92aa2d92294b9d32c9", + "zig_version": "0.16.0-dev.2611+f996d2866", "platform": "aarch64-macos" } }, - "zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-windows": { + "zig_0.16.0-dev.2611_Pf996d2866_aarch64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1912+0cbaaa5eb.zip", + "url": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.2611+f996d2866.zip", "mirrors": [ - "https://mirror.zml.ai/zig", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds", "https://ziglang.org/builds/" ], - "sha256": "824044c2dd76d4fdf9dd3d3fc7e8ed9bd39a1a03a3eb3acf1fecde25cee640cc", - "zig_version": "0.16.0-dev.1912+0cbaaa5eb", + "sha256": "42a612662453e47ad4001eb19c1d219dec6f254913988e2ebe6c95e4b5da3e82", + "zig_version": "0.16.0-dev.2611+f996d2866", "platform": "aarch64-windows" } }, - "zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-linux": { + "zig_0.16.0-dev.2611_Pf996d2866_x86_64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "url": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.2611+f996d2866.tar.xz", "mirrors": [ - "https://mirror.zml.ai/zig", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds", "https://ziglang.org/builds/" ], - "sha256": "d0bfbe4c6e6871a36ad7c9be577df2abda37c69af437fedb6865d8729d60cbdc", - "zig_version": "0.16.0-dev.1912+0cbaaa5eb", + "sha256": "0188a70a3f84fe1918ac391a8b64bd241eb4c028c44ea023e853743cbecd3009", + "zig_version": "0.16.0-dev.2611+f996d2866", "platform": "x86_64-linux" } }, - "zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-macos": { + "zig_0.16.0-dev.2611_Pf996d2866_x86_64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1912+0cbaaa5eb.tar.xz", + "url": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.2611+f996d2866.tar.xz", "mirrors": [ - "https://mirror.zml.ai/zig", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds", "https://ziglang.org/builds/" ], - "sha256": "9bfa7e5237c5e5b132229e3686eb849ca2afb23f7b6f6b72cbae0e5e8766aade", - "zig_version": "0.16.0-dev.1912+0cbaaa5eb", + "sha256": "5ba0c08bd2e9c74d18c6ea25483226a8f8922f4e50d09d9219be842894b41334", + "zig_version": "0.16.0-dev.2611+f996d2866", "platform": "x86_64-macos" } }, - "zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-windows": { + "zig_0.16.0-dev.2611_Pf996d2866_x86_64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1912+0cbaaa5eb.zip", + "url": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.2611+f996d2866.zip", "mirrors": [ - "https://mirror.zml.ai/zig", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds", "https://ziglang.org/builds/" ], - "sha256": "719859787d0be77a8dc19f5ecc37d8cc7802ddca06f616d81e9ebde34e610dd4", - "zig_version": "0.16.0-dev.1912+0cbaaa5eb", + "sha256": "3ec5cac4b74b5fe6e04f092ffb6e025993538ecf07970a428919d638efafa542", + "zig_version": "0.16.0-dev.2611+f996d2866", "platform": "x86_64-windows" } }, @@ -2706,28 +1280,28 @@ "repoRuleId": "@@rules_zig+//zig/private/repo:toolchains_repo.bzl%toolchains_repo", "attributes": { "names": [ - "zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-linux", - "zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-macos", - "zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-windows", - "zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-linux", - "zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-macos", - "zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-windows" + "zig_0.16.0-dev.2611_Pf996d2866_aarch64-linux", + "zig_0.16.0-dev.2611_Pf996d2866_aarch64-macos", + "zig_0.16.0-dev.2611_Pf996d2866_aarch64-windows", + "zig_0.16.0-dev.2611_Pf996d2866_x86_64-linux", + "zig_0.16.0-dev.2611_Pf996d2866_x86_64-macos", + "zig_0.16.0-dev.2611_Pf996d2866_x86_64-windows" ], "labels": [ - "@zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-linux//:zig_toolchain", - "@zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-macos//:zig_toolchain", - "@zig_0.16.0-dev.1912_P0cbaaa5eb_aarch64-windows//:zig_toolchain", - "@zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-linux//:zig_toolchain", - "@zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-macos//:zig_toolchain", - "@zig_0.16.0-dev.1912_P0cbaaa5eb_x86_64-windows//:zig_toolchain" + "@zig_0.16.0-dev.2611_Pf996d2866_aarch64-linux//:zig_toolchain", + "@zig_0.16.0-dev.2611_Pf996d2866_aarch64-macos//:zig_toolchain", + "@zig_0.16.0-dev.2611_Pf996d2866_aarch64-windows//:zig_toolchain", + "@zig_0.16.0-dev.2611_Pf996d2866_x86_64-linux//:zig_toolchain", + "@zig_0.16.0-dev.2611_Pf996d2866_x86_64-macos//:zig_toolchain", + "@zig_0.16.0-dev.2611_Pf996d2866_x86_64-windows//:zig_toolchain" ], "zig_versions": [ - "0.16.0-dev.1912+0cbaaa5eb", - "0.16.0-dev.1912+0cbaaa5eb", - "0.16.0-dev.1912+0cbaaa5eb", - "0.16.0-dev.1912+0cbaaa5eb", - "0.16.0-dev.1912+0cbaaa5eb", - "0.16.0-dev.1912+0cbaaa5eb" + "0.16.0-dev.2611+f996d2866", + "0.16.0-dev.2611+f996d2866", + "0.16.0-dev.2611+f996d2866", + "0.16.0-dev.2611+f996d2866", + "0.16.0-dev.2611+f996d2866", + "0.16.0-dev.2611+f996d2866" ], "exec_lengths": [ 2, @@ -2769,5 +1343,86 @@ } } }, - "facts": {} + "facts": { + "@@rules_rs+//rs:extensions.bzl%crate": { + "ahash_0.8.12": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1.0\"},{\"name\":\"const-random\",\"optional\":true,\"req\":\"^0.1.17\"},{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.2\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.5\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2.1\"},{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"hashbrown\",\"req\":\"^0.14.3\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.2\"},{\"kind\":\"dev\",\"name\":\"no-panic\",\"req\":\"^0.1.10\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"once_cell\",\"req\":\"^1.18.0\",\"target\":\"cfg(not(all(target_arch = \\\"arm\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"pcg-mwc\",\"req\":\"^0.2.1\"},{\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"seahash\",\"req\":\"^4.0\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.117\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.59\"},{\"kind\":\"dev\",\"name\":\"smallvec\",\"req\":\"^1.13.1\"},{\"kind\":\"build\",\"name\":\"version_check\",\"req\":\"^0.9.4\"},{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"zerocopy\",\"req\":\"^0.8.24\"}],\"features\":{\"atomic-polyfill\":[\"dep:portable-atomic\",\"once_cell/critical-section\"],\"compile-time-rng\":[\"const-random\"],\"default\":[\"std\",\"runtime-rng\"],\"nightly-arm-aes\":[],\"no-rng\":[],\"runtime-rng\":[\"getrandom\"],\"std\":[]}}", + "aho-corasick_1.1.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.17\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.4.0\"}],\"features\":{\"default\":[\"std\",\"perf-literal\"],\"logging\":[\"dep:log\"],\"perf-literal\":[\"dep:memchr\"],\"std\":[\"memchr?/std\"]}}", + "base64_0.13.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"=0.3.2\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.6.1\"},{\"kind\":\"dev\",\"name\":\"structopt\",\"req\":\"^0.3\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[]}}", + "bitflags_2.9.1": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"arbitrary\",\"req\":\"^1.0\"},{\"name\":\"bytemuck\",\"optional\":true,\"req\":\"^1.12\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"bytemuck\",\"req\":\"^1.12.2\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.19\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.18\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"zerocopy\",\"req\":\"^0.8\"}],\"features\":{\"example_generated\":[],\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"],\"std\":[]}}", + "byteorder_1.5.0": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9.2\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"std\"],\"i128\":[],\"std\":[]}}", + "castaway_0.2.4": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1\"},{\"name\":\"rustversion\",\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", + "cc_1.2.33": "{\"dependencies\":[{\"default_features\":false,\"name\":\"jobserver\",\"optional\":true,\"req\":\"^0.1.30\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.62\",\"target\":\"cfg(unix)\"},{\"name\":\"shlex\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{\"jobserver\":[],\"parallel\":[\"dep:libc\",\"dep:jobserver\"]}}", + "cfg-if_1.0.0": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"]}}", + "compact_str_0.9.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"borsh\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"castaway\",\"req\":\"^0.2.3\"},{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"cfg-if\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"diesel\",\"optional\":true,\"req\":\"^2\"},{\"name\":\"itoa\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"markup\",\"optional\":true,\"req\":\"^0.15\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"proptest\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"rkyv\",\"optional\":true,\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rkyv\",\"req\":\"^0.8.8\"},{\"name\":\"rustversion\",\"req\":\"^1\"},{\"name\":\"ryu\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"derive\",\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"features\":[\"union\"],\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"name\":\"sqlx\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"static_assertions\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"test-case\",\"req\":\"^3\"},{\"kind\":\"dev\",\"name\":\"test-strategy\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"borsh\":[\"dep:borsh\"],\"bytes\":[\"dep:bytes\"],\"default\":[\"std\"],\"diesel\":[\"dep:diesel\"],\"markup\":[\"dep:markup\"],\"proptest\":[\"dep:proptest\"],\"quickcheck\":[\"dep:quickcheck\"],\"rkyv\":[\"dep:rkyv\"],\"serde\":[\"dep:serde\"],\"smallvec\":[\"dep:smallvec\"],\"sqlx\":[\"dep:sqlx\",\"std\"],\"sqlx-mysql\":[\"sqlx\",\"sqlx/mysql\"],\"sqlx-postgres\":[\"sqlx\",\"sqlx/postgres\"],\"sqlx-sqlite\":[\"sqlx\",\"sqlx/sqlite\"],\"std\":[],\"zeroize\":[\"dep:zeroize\"]}}", + "crossbeam-deque_0.8.6": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-epoch\",\"req\":\"^0.9.17\"},{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"crossbeam-epoch/std\",\"crossbeam-utils/std\"]}}", + "crossbeam-epoch_0.9.18": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"name\":\"loom-crate\",\"optional\":true,\"package\":\"loom\",\"req\":\"^0.7.1\",\"target\":\"cfg(crossbeam_loom)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"loom\":[\"loom-crate\",\"crossbeam-utils/loom\"],\"nightly\":[\"crossbeam-utils/nightly\"],\"std\":[\"alloc\",\"crossbeam-utils/std\"]}}", + "crossbeam-utils_0.8.21": "{\"dependencies\":[{\"name\":\"loom\",\"optional\":true,\"req\":\"^0.7.1\",\"target\":\"cfg(crossbeam_loom)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"std\"],\"nightly\":[],\"std\":[]}}", + "darling_0.20.10": "{\"dependencies\":[{\"name\":\"darling_core\",\"req\":\"=0.20.10\"},{\"name\":\"darling_macro\",\"req\":\"=0.20.10\"},{\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.37\"},{\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.9\",\"target\":\"cfg(compiletests)\"},{\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.15\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.38\",\"target\":\"cfg(compiletests)\"}],\"features\":{\"default\":[\"suggestions\"],\"diagnostics\":[\"darling_core/diagnostics\"],\"suggestions\":[\"darling_core/suggestions\"]}}", + "darling_core_0.20.10": "{\"dependencies\":[{\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"name\":\"ident_case\",\"req\":\"^1.0.1\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.37\"},{\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"name\":\"strsim\",\"optional\":true,\"req\":\"^0.11.1\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"diagnostics\":[],\"suggestions\":[\"strsim\"]}}", + "darling_macro_0.20.10": "{\"dependencies\":[{\"name\":\"darling_core\",\"req\":\"=0.20.10\"},{\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{}}", + "dary_heap_0.3.7": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1\"}],\"features\":{\"extra\":[],\"unstable\":[],\"unstable_nightly\":[]}}", + "derive_builder_0.20.2": "{\"dependencies\":[{\"name\":\"derive_builder_macro\",\"req\":\"=0.20.2\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.4\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.38\",\"target\":\"cfg(compiletests)\"}],\"features\":{\"alloc\":[\"derive_builder_macro/alloc\"],\"clippy\":[\"derive_builder_macro/clippy\"],\"default\":[\"std\"],\"std\":[\"derive_builder_macro/lib_has_std\"]}}", + "derive_builder_core_0.20.2": "{\"dependencies\":[{\"name\":\"darling\",\"req\":\"^0.20.10\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.37\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"alloc\":[],\"clippy\":[],\"lib_has_std\":[]}}", + "derive_builder_macro_0.20.2": "{\"dependencies\":[{\"name\":\"derive_builder_core\",\"req\":\"=0.20.2\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"alloc\":[\"derive_builder_core/alloc\"],\"clippy\":[\"derive_builder_core/clippy\"],\"lib_has_std\":[\"derive_builder_core/lib_has_std\"]}}", + "either_1.13.0": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"use_std\"],\"use_std\":[]}}", + "esaxx-rs_0.1.10": "{\"dependencies\":[{\"kind\":\"build\",\"name\":\"cc\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"}],\"features\":{\"cpp\":[\"cc\"],\"default\":[\"cpp\"]}}", + "fnv_1.0.7": "{\"dependencies\":[],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "getrandom_0.3.3": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3.77\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"), target_feature = \\\"atomics\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"netbsd\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"solaris\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"vxworks\\\")\"},{\"default_features\":false,\"name\":\"r-efi\",\"req\":\"^5.1\",\"target\":\"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\"},{\"default_features\":false,\"name\":\"wasi\",\"req\":\"^0.14\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.98\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"}],\"features\":{\"rustc-dep-of-std\":[\"dep:compiler_builtins\",\"dep:core\"],\"std\":[],\"wasm_js\":[\"dep:wasm-bindgen\",\"dep:js-sys\"]}}", + "ident_case_1.0.1": "{\"dependencies\":[],\"features\":{}}", + "itertools_0.14.0": "{\"dependencies\":[{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"permutohedron\",\"req\":\"^0.2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"use_std\"],\"use_alloc\":[],\"use_std\":[\"use_alloc\",\"either/use_std\"]}}", + "itoa_1.0.14": "{\"dependencies\":[{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"}],\"features\":{}}", + "libc_0.2.169": "{\"dependencies\":[{\"name\":\"rustc-std-workspace-core\",\"optional\":true,\"req\":\"^1.0.0\"}],\"features\":{\"align\":[],\"const-extern-fn\":[],\"default\":[\"std\"],\"extra_traits\":[],\"rustc-dep-of-std\":[\"align\",\"rustc-std-workspace-core\"],\"std\":[],\"use_std\":[\"std\"]}}", + "log_0.4.22": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.63\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"sval\",\"optional\":true,\"req\":\"^2.1\"},{\"kind\":\"dev\",\"name\":\"sval\",\"req\":\"^2.1\"},{\"kind\":\"dev\",\"name\":\"sval_derive\",\"req\":\"^2.1\"},{\"default_features\":false,\"name\":\"sval_ref\",\"optional\":true,\"req\":\"^2.1\"},{\"default_features\":false,\"features\":[\"inline-i128\"],\"name\":\"value-bag\",\"optional\":true,\"req\":\"^1.7\"},{\"features\":[\"test\"],\"kind\":\"dev\",\"name\":\"value-bag\",\"req\":\"^1.7\"}],\"features\":{\"kv\":[],\"kv_serde\":[\"kv_std\",\"value-bag/serde\",\"serde\"],\"kv_std\":[\"std\",\"kv\",\"value-bag/error\"],\"kv_sval\":[\"kv\",\"value-bag/sval\",\"sval\",\"sval_ref\"],\"kv_unstable\":[\"kv\",\"value-bag\"],\"kv_unstable_serde\":[\"kv_serde\",\"kv_unstable_std\"],\"kv_unstable_std\":[\"kv_std\",\"kv_unstable\"],\"kv_unstable_sval\":[\"kv_sval\",\"kv_unstable\"],\"max_level_debug\":[],\"max_level_error\":[],\"max_level_info\":[],\"max_level_off\":[],\"max_level_trace\":[],\"max_level_warn\":[],\"release_max_level_debug\":[],\"release_max_level_error\":[],\"release_max_level_info\":[],\"release_max_level_off\":[],\"release_max_level_trace\":[],\"release_max_level_warn\":[],\"std\":[]}}", + "macro_rules_attribute-proc_macro_0.2.0": "{\"dependencies\":[],\"features\":{\"verbose-expansions\":[]}}", + "macro_rules_attribute_0.2.0": "{\"dependencies\":[{\"name\":\"macro_rules_attribute-proc_macro\",\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.10.0\"},{\"name\":\"paste\",\"req\":\"^1.0.7\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.8\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.136\"}],\"features\":{\"better-docs\":[],\"default\":[],\"verbose-expansions\":[\"macro_rules_attribute-proc_macro/verbose-expansions\"]}}", + "memchr_2.7.4": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.20\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"libc\":[],\"logging\":[\"dep:log\"],\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"],\"std\":[\"alloc\"],\"use_std\":[\"std\"]}}", + "minimal-lexical_0.2.1": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"compact\":[],\"default\":[\"std\"],\"lint\":[],\"nightly\":[],\"std\":[]}}", + "monostate-impl_0.1.13": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"default_features\":false,\"features\":[\"parsing\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0.59\"}],\"features\":{}}", + "monostate_0.1.13": "{\"dependencies\":[{\"name\":\"monostate-impl\",\"req\":\"=0.1.13\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.166\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.99\"}],\"features\":{}}", + "nom_7.1.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2.3\"},{\"default_features\":false,\"name\":\"minimal-lexical\",\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.0.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"docsrs\":[],\"std\":[\"alloc\",\"memchr/std\",\"minimal-lexical/std\"]}}", + "once_cell_1.20.2": "{\"dependencies\":[{\"name\":\"critical-section\",\"optional\":true,\"req\":\"^1.1.3\"},{\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"critical-section\",\"req\":\"^1.1.3\"},{\"default_features\":false,\"name\":\"parking_lot_core\",\"optional\":true,\"req\":\"^0.9.10\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.10.6\"}],\"features\":{\"alloc\":[\"race\"],\"atomic-polyfill\":[\"critical-section\"],\"critical-section\":[\"dep:critical-section\",\"portable-atomic\"],\"default\":[\"std\"],\"parking_lot\":[\"dep:parking_lot_core\"],\"portable-atomic\":[\"dep:portable-atomic\"],\"race\":[],\"std\":[\"alloc\"],\"unstable\":[]}}", + "onig_6.5.1": "{\"dependencies\":[{\"name\":\"bitflags\",\"req\":\"^2.4.0\"},{\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(windows)\"},{\"name\":\"once_cell\",\"req\":\"^1.12\"},{\"default_features\":false,\"name\":\"onig_sys\",\"req\":\"^69.9.1\"}],\"features\":{\"default\":[\"generate\"],\"generate\":[\"onig_sys/generate\"],\"posix-api\":[\"onig_sys/posix-api\"],\"print-debug\":[\"onig_sys/print-debug\"],\"std-pattern\":[]}}", + "onig_sys_69.9.1": "{\"dependencies\":[{\"features\":[\"runtime\"],\"kind\":\"build\",\"name\":\"bindgen\",\"optional\":true,\"req\":\"^0.71\"},{\"kind\":\"build\",\"name\":\"cc\",\"req\":\"^1.0\"},{\"kind\":\"build\",\"name\":\"pkg-config\",\"req\":\"^0.3.16\"}],\"features\":{\"default\":[\"generate\"],\"generate\":[\"bindgen\"],\"posix-api\":[],\"print-debug\":[]}}", + "paste_1.0.15": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"paste-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", + "pkg-config_0.3.32": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1\"}],\"features\":{}}", + "ppv-lite86_0.2.20": "{\"dependencies\":[{\"features\":[\"simd\",\"derive\"],\"name\":\"zerocopy\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"std\"],\"no_simd\":[],\"simd\":[],\"std\":[]}}", + "proc-macro2_1.0.92": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4\"},{\"name\":\"unicode-ident\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"proc-macro\"],\"nightly\":[],\"proc-macro\":[],\"span-locations\":[]}}", + "quote_1.0.37": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.66\"}],\"features\":{\"default\":[\"proc-macro\"],\"proc-macro\":[\"proc-macro2/proc-macro\"]}}", + "r-efi_5.3.0": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"efiapi\":[],\"examples\":[\"native\"],\"native\":[],\"rustc-dep-of-std\":[\"core\"]}}", + "rand_0.9.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.2.1\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.4\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"optional\":true,\"req\":\"^0.9.0\"},{\"default_features\":false,\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rand_pcg\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.7\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"std_rng\",\"os_rng\",\"small_rng\",\"thread_rng\"],\"log\":[\"dep:log\"],\"nightly\":[],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\",\"rand_core/serde\"],\"simd_support\":[],\"small_rng\":[],\"std\":[\"rand_core/std\",\"rand_chacha?/std\",\"alloc\"],\"std_rng\":[\"dep:rand_chacha\"],\"thread_rng\":[\"std\",\"std_rng\",\"os_rng\"],\"unbiased\":[]}}", + "rand_chacha_0.9.0": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"ppv-lite86\",\"req\":\"^0.2.14\"},{\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"features\":[\"os_rng\"],\"kind\":\"dev\",\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\"],\"std\":[\"ppv-lite86/std\",\"rand_core/std\"]}}", + "rand_core_0.9.3": "{\"dependencies\":[{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.3.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"os_rng\":[\"dep:getrandom\"],\"serde\":[\"dep:serde\"],\"std\":[\"getrandom?/std\"]}}", + "rayon-cond_0.4.0": "{\"dependencies\":[{\"name\":\"either\",\"req\":\"^1.6\"},{\"name\":\"itertools\",\"req\":\"^0.14\"},{\"name\":\"rayon\",\"req\":\"^1.7\"}],\"features\":{}}", + "rayon-core_1.12.1": "{\"dependencies\":[{\"name\":\"crossbeam-deque\",\"req\":\"^0.8.1\"},{\"name\":\"crossbeam-utils\",\"req\":\"^0.8.0\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"scoped-tls\",\"req\":\"^1.0\"},{\"name\":\"wasm_sync\",\"optional\":true,\"req\":\"^0.1.0\"}],\"features\":{\"web_spin_lock\":[\"dep:wasm_sync\"]}}", + "rayon_1.10.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"name\":\"rayon-core\",\"req\":\"^1.12.1\"},{\"name\":\"wasm_sync\",\"optional\":true,\"req\":\"^0.1.0\"}],\"features\":{\"web_spin_lock\":[\"dep:wasm_sync\",\"rayon-core/web_spin_lock\"]}}", + "regex-automata_0.4.9": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"bstr\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.14\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"optional\":true,\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"syntax\",\"perf\",\"unicode\",\"meta\",\"nfa\",\"dfa\",\"hybrid\"],\"dfa\":[\"dfa-build\",\"dfa-search\",\"dfa-onepass\"],\"dfa-build\":[\"nfa-thompson\",\"dfa-search\"],\"dfa-onepass\":[\"nfa-thompson\"],\"dfa-search\":[],\"hybrid\":[\"alloc\",\"nfa-thompson\"],\"internal-instrument\":[\"internal-instrument-pikevm\"],\"internal-instrument-pikevm\":[\"logging\",\"std\"],\"logging\":[\"dep:log\",\"aho-corasick?/logging\",\"memchr?/logging\"],\"meta\":[\"syntax\",\"nfa-pikevm\"],\"nfa\":[\"nfa-thompson\",\"nfa-pikevm\",\"nfa-backtrack\"],\"nfa-backtrack\":[\"nfa-thompson\"],\"nfa-pikevm\":[\"nfa-thompson\"],\"nfa-thompson\":[\"alloc\"],\"perf\":[\"perf-inline\",\"perf-literal\"],\"perf-inline\":[],\"perf-literal\":[\"perf-literal-substring\",\"perf-literal-multisubstring\"],\"perf-literal-multisubstring\":[\"std\",\"dep:aho-corasick\"],\"perf-literal-substring\":[\"aho-corasick?/perf-literal\",\"dep:memchr\"],\"std\":[\"regex-syntax?/std\",\"memchr?/std\",\"aho-corasick?/std\",\"alloc\"],\"syntax\":[\"dep:regex-syntax\",\"alloc\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"unicode-word-boundary\",\"regex-syntax?/unicode\"],\"unicode-age\":[\"regex-syntax?/unicode-age\"],\"unicode-bool\":[\"regex-syntax?/unicode-bool\"],\"unicode-case\":[\"regex-syntax?/unicode-case\"],\"unicode-gencat\":[\"regex-syntax?/unicode-gencat\"],\"unicode-perl\":[\"regex-syntax?/unicode-perl\"],\"unicode-script\":[\"regex-syntax?/unicode-script\"],\"unicode-segment\":[\"regex-syntax?/unicode-segment\"],\"unicode-word-boundary\":[]}}", + "regex-syntax_0.8.5": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.3.0\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"default\":[\"std\",\"unicode\"],\"std\":[],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\"],\"unicode-age\":[],\"unicode-bool\":[],\"unicode-case\":[],\"unicode-gencat\":[],\"unicode-perl\":[],\"unicode-script\":[],\"unicode-segment\":[]}}", + "regex_1.11.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.17.1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"features\":[\"alloc\",\"syntax\",\"meta\",\"nfa-pikevm\"],\"name\":\"regex-automata\",\"req\":\"^0.4.8\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"default\":[\"std\",\"perf\",\"unicode\",\"regex-syntax/default\"],\"logging\":[\"aho-corasick?/logging\",\"memchr?/logging\",\"regex-automata/logging\"],\"pattern\":[],\"perf\":[\"perf-cache\",\"perf-dfa\",\"perf-onepass\",\"perf-backtrack\",\"perf-inline\",\"perf-literal\"],\"perf-backtrack\":[\"regex-automata/nfa-backtrack\"],\"perf-cache\":[],\"perf-dfa\":[\"regex-automata/hybrid\"],\"perf-dfa-full\":[\"regex-automata/dfa-build\",\"regex-automata/dfa-search\"],\"perf-inline\":[\"regex-automata/perf-inline\"],\"perf-literal\":[\"dep:aho-corasick\",\"dep:memchr\",\"regex-automata/perf-literal\"],\"perf-onepass\":[\"regex-automata/dfa-onepass\"],\"std\":[\"aho-corasick?/std\",\"memchr?/std\",\"regex-automata/std\",\"regex-syntax/std\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"regex-automata/unicode\",\"regex-syntax/unicode\"],\"unicode-age\":[\"regex-automata/unicode-age\",\"regex-syntax/unicode-age\"],\"unicode-bool\":[\"regex-automata/unicode-bool\",\"regex-syntax/unicode-bool\"],\"unicode-case\":[\"regex-automata/unicode-case\",\"regex-syntax/unicode-case\"],\"unicode-gencat\":[\"regex-automata/unicode-gencat\",\"regex-syntax/unicode-gencat\"],\"unicode-perl\":[\"regex-automata/unicode-perl\",\"regex-automata/unicode-word-boundary\",\"regex-syntax/unicode-perl\"],\"unicode-script\":[\"regex-automata/unicode-script\",\"regex-syntax/unicode-script\"],\"unicode-segment\":[\"regex-automata/unicode-segment\",\"regex-syntax/unicode-segment\"],\"unstable\":[\"pattern\"],\"use_std\":[\"std\"]}}", + "rustversion_1.0.21": "{\"dependencies\":[{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", + "ryu_1.0.18": "{\"dependencies\":[{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"num_cpus\",\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"}],\"features\":{\"small\":[]}}", + "serde_1.0.216": "{\"dependencies\":[{\"name\":\"serde_derive\",\"req\":\"=1.0.216\",\"target\":\"cfg(any())\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"derive\":[\"serde_derive\"],\"rc\":[],\"std\":[],\"unstable\":[]}}", + "serde_derive_1.0.216": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"derive\",\"parsing\",\"printing\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0.81\"}],\"features\":{\"default\":[],\"deserialize_in_place\":[]}}", + "serde_json_1.0.134": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0.11\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.2.3\"},{\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2.0.2\"},{\"name\":\"itoa\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"name\":\"ryu\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"kind\":\"dev\",\"name\":\"serde_bytes\",\"req\":\"^0.11.10\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_stacker\",\"req\":\"^0.1.8\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{\"alloc\":[\"serde/alloc\"],\"arbitrary_precision\":[],\"default\":[\"std\"],\"float_roundtrip\":[],\"preserve_order\":[\"indexmap\",\"std\"],\"raw_value\":[],\"std\":[\"memchr/std\",\"serde/std\"],\"unbounded_depth\":[]}}", + "shlex_1.3.0": "{\"dependencies\":[],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "smallvec_1.13.2": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.0.1\"},{\"kind\":\"dev\",\"name\":\"debugger_test\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"debugger_test_parser\",\"req\":\"^0.1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"const_generics\":[],\"const_new\":[\"const_generics\"],\"debugger_visualizer\":[],\"drain_filter\":[],\"drain_keep_rest\":[\"drain_filter\"],\"may_dangle\":[],\"specialization\":[],\"union\":[],\"write\":[]}}", + "spm_precompiled_0.1.4": "{\"dependencies\":[{\"name\":\"base64\",\"req\":\"^0.13\"},{\"name\":\"nom\",\"req\":\"^7.1.1\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"unicode-segmentation\",\"req\":\"^1.9\"}],\"features\":{}}", + "static_assertions_1.1.0": "{\"dependencies\":[],\"features\":{\"nightly\":[]}}", + "strsim_0.11.1": "{\"dependencies\":[],\"features\":{}}", + "syn_2.0.91": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"insta\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.91\"},{\"default_features\":false,\"name\":\"quote\",\"optional\":true,\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1\"},{\"features\":[\"blocking\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.12\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"syn-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4.16\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"termcolor\",\"req\":\"^1\"},{\"name\":\"unicode-ident\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\",\"target\":\"cfg(not(miri))\"}],\"features\":{\"clone-impls\":[],\"default\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\"],\"derive\":[],\"extra-traits\":[],\"fold\":[],\"full\":[],\"parsing\":[],\"printing\":[\"dep:quote\"],\"proc-macro\":[\"proc-macro2/proc-macro\",\"quote?/proc-macro\"],\"test\":[\"syn-test-suite/all-features\"],\"visit\":[],\"visit-mut\":[]}}", + "thiserror-impl_2.0.12": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"name\":\"syn\",\"req\":\"^2.0.87\"}],\"features\":{}}", + "thiserror_2.0.12": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.73\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"name\":\"thiserror-impl\",\"req\":\"=2.0.12\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "tokenizers_0.22.2": "{\"dependencies\":[{\"features\":[\"serde\"],\"name\":\"ahash\",\"req\":\"^0.8.11\"},{\"name\":\"aho-corasick\",\"req\":\"^1.1\"},{\"kind\":\"dev\",\"name\":\"assert_approx_eq\",\"req\":\"^1.1\"},{\"features\":[\"serde\"],\"name\":\"compact_str\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.6\"},{\"features\":[\"serde\"],\"name\":\"dary_heap\",\"req\":\"^0.3.6\"},{\"name\":\"derive_builder\",\"req\":\"^0.20\"},{\"default_features\":false,\"name\":\"esaxx-rs\",\"req\":\"^0.1.10\"},{\"name\":\"fancy-regex\",\"optional\":true,\"req\":\"^0.14\"},{\"name\":\"getrandom\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"ureq\"],\"name\":\"hf-hub\",\"optional\":true,\"req\":\"^0.4.1\"},{\"name\":\"indicatif\",\"optional\":true,\"req\":\"^0.18\"},{\"name\":\"itertools\",\"req\":\"^0.14\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"macro_rules_attribute\",\"req\":\"^0.2.0\"},{\"name\":\"monostate\",\"req\":\"^0.1.12\"},{\"default_features\":false,\"name\":\"onig\",\"optional\":true,\"req\":\"^6.5.1\"},{\"name\":\"paste\",\"req\":\"^1.0.14\"},{\"name\":\"rand\",\"req\":\"^0.9\"},{\"name\":\"rayon\",\"req\":\"^1.10\"},{\"name\":\"rayon-cond\",\"req\":\"^0.4\"},{\"name\":\"regex\",\"req\":\"^1.10\"},{\"name\":\"regex-syntax\",\"req\":\"^0.8\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"spm_precompiled\",\"req\":\"^0.1.3\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.10\"},{\"name\":\"thiserror\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3.18\"},{\"name\":\"unicode-normalization-alignments\",\"req\":\"^0.1\"},{\"name\":\"unicode-segmentation\",\"req\":\"^1.11\"},{\"name\":\"unicode_categories\",\"req\":\"^0.1\"}],\"features\":{\"default\":[\"progressbar\",\"onig\",\"esaxx_fast\"],\"esaxx_fast\":[\"esaxx-rs/cpp\"],\"http\":[\"hf-hub\"],\"progressbar\":[\"indicatif\"],\"rustls-tls\":[\"hf-hub?/rustls-tls\"],\"unstable_wasm\":[\"fancy-regex\",\"getrandom/wasm_js\"]}}", + "unicode-ident_1.0.14": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"fst\",\"req\":\"^0.4\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"roaring\",\"req\":\"^0.10\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"ucd-trie\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"unicode-xid\",\"req\":\"^0.2.6\"}],\"features\":{}}", + "unicode-normalization-alignments_0.1.12": "{\"dependencies\":[{\"name\":\"smallvec\",\"req\":\"^1.0\"}],\"features\":{}}", + "unicode-segmentation_1.12.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.7\"}],\"features\":{\"no_std\":[]}}", + "unicode_categories_0.1.1": "{\"dependencies\":[],\"features\":{}}", + "version_check_0.9.5": "{\"dependencies\":[],\"features\":{}}", + "wasi_0.14.2+wasi-0.2.4": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"bitflags\"],\"name\":\"wit-bindgen-rt\",\"req\":\"^0.39.0\"}],\"features\":{\"default\":[\"std\"],\"rustc-dep-of-std\":[\"compiler_builtins\",\"core\",\"rustc-std-workspace-alloc\"],\"std\":[]}}", + "wit-bindgen-rt_0.39.0": "{\"dependencies\":[{\"name\":\"bitflags\",\"optional\":true,\"req\":\"^2.3.3\"},{\"name\":\"futures\",\"optional\":true,\"req\":\"^0.3.30\"},{\"name\":\"once_cell\",\"optional\":true,\"req\":\"^1.19.0\"}],\"features\":{\"async\":[\"dep:futures\",\"dep:once_cell\"]}}", + "zerocopy-derive_0.7.35": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.10\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"name\":\"syn\",\"req\":\"^2.0.31\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.85\"}],\"features\":{}}", + "zerocopy-derive_0.8.26": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"dissimilar\",\"req\":\"^1.0.9\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"=0.2.163\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"=1.9\"},{\"kind\":\"dev\",\"name\":\"prettyplease\",\"req\":\"=0.2.17\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.10\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.89\"}],\"features\":{}}", + "zerocopy_0.7.35": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"assert_matches\",\"req\":\"^1.5\"},{\"default_features\":false,\"name\":\"byteorder\",\"optional\":true,\"req\":\"^1.3\"},{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.85\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.7.35\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.7.35\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.7.35\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\"],\"alloc\":[],\"default\":[\"byteorder\"],\"derive\":[\"zerocopy-derive\"],\"simd\":[],\"simd-nightly\":[\"simd\"]}}", + "zerocopy_0.8.26": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"either\",\"req\":\"=1.13.0\"},{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.89\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.8.26\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\",\"std\"],\"alloc\":[],\"derive\":[\"zerocopy-derive\"],\"float-nightly\":[],\"simd\":[],\"simd-nightly\":[\"simd\"],\"std\":[\"alloc\"]}}" + } + } } diff --git a/bazel/bazel.zig b/bazel/bazel.zig index 02a49e0d..6d5540b0 100644 --- a/bazel/bazel.zig +++ b/bazel/bazel.zig @@ -1,18 +1,52 @@ const std = @import("std"); +const builtin = @import("builtin"); const bazel_runfiles = @import("runfiles"); const stdx = @import("stdx"); -var runfiles_once = stdx.onceWithArgs(struct { - fn call(allocator: std.mem.Allocator, io: std.Io) !*bazel_runfiles.Runfiles { - const r = try allocator.create(bazel_runfiles.Runfiles); - r.* = try bazel_runfiles.Runfiles.create(.{ .allocator = allocator, .io = io }) orelse { - stdx.debug.panic("Unable to find runfiles", .{}); - }; - return r; - } -}.call); +var runfiles_global: bazel_runfiles.Runfiles = undefined; -pub fn runfiles(io: std.Io, source_repository: []const u8) !bazel_runfiles.Runfiles.WithSourceRepo { - return (try runfiles_once.call(.{ std.heap.smp_allocator, io })).withSourceRepo(source_repository); +const init_array_section = switch (builtin.object_format) { + .macho => "__DATA,__mod_init_func", + .elf => ".init_array", + else => "", +}; + +export const _ linksection(init_array_section) = &struct { + fn call(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { + _ = argc; // autofix + _ = envp; // autofix + var threaded: std.Io.Threaded = .init_single_threaded; + runfiles_global = bazel_runfiles.Runfiles.create(.{ + .allocator = std.heap.c_allocator, + .io = threaded.io(), + .argv0 = std.mem.span(argv[0]), + .directory = if (std.c.getenv("RUNFILES_DIRECTORY")) |directory| std.mem.span(directory) else null, + .manifest = if (std.c.getenv("RUNFILES_MANIFEST_FILE")) |manifest| std.mem.span(manifest) else null, + }) catch { + std.debug.panic("Unable to find runfiles", .{}); + } orelse { + std.debug.panic("Unable to initialize runfiles", .{}); + }; + } +}.call; + +pub fn init(allocator: std.mem.Allocator, io: std.Io, argv: std.process.Args, environ_map: *std.process.Environ.Map) !void { + runfiles_global = try allocator.create(bazel_runfiles.Runfiles); + runfiles_global.* = try bazel_runfiles.Runfiles.create(.{ + .allocator = allocator, + .io = io, + .argv = argv, + .environ_map = environ_map, + }) orelse { + std.debug.panic("Unable to find runfiles", .{}); + }; +} + +pub fn initSimple(init_: std.process.Init) !void { + try init(init_.arena.allocator(), init_.io, init_.minimal.args, init_.environ_map); +} + +pub fn runfiles(source_repository: []const u8) !bazel_runfiles.Runfiles.WithSourceRepo { + return runfiles_global.withSourceRepo(source_repository); } diff --git a/bazel/zig_index.json b/bazel/zig_index.json index 4720c0e1..da8c0455 100644 --- a/bazel/zig_index.json +++ b/bazel/zig_index.json @@ -1,133 +1,133 @@ { "master": { - "version": "0.16.0-dev.1912+0cbaaa5eb", - "date": "2026-01-04", + "version": "0.16.0-dev.2611+f996d2866", + "date": "2026-02-14", "docs": "https://ziglang.org/documentation/master/", "stdDocs": "https://ziglang.org/documentation/master/std/", "src": { - "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "shasum": "4730e30cf16fb088b706bf2834cd973a54a1ff06d56cba2dc845b28408d7e8bd", - "size": "21866640" + "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "cf085b0b63b3520cda55aa7d98a8c86b28a5c9739bffe4a0253c58880ded4c7b", + "size": "22442124" }, "bootstrap": { - "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "shasum": "e928729be6613ba78dc0dc01773380148a8b1e94a4d13b0e0b7a36bfbaa3823f", - "size": "54630236" + "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "b87fcb4aa966c9b7f07ceca302b15eb587936938a65b5861188200d5b3616c0d", + "size": "55205844" }, "x86_64-macos": { - "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "shasum": "9bfa7e5237c5e5b132229e3686eb849ca2afb23f7b6f6b72cbae0e5e8766aade", - "size": "57011520" + "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "5ba0c08bd2e9c74d18c6ea25483226a8f8922f4e50d09d9219be842894b41334", + "size": "57619112" }, "aarch64-macos": { - "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "shasum": "241fa43a37803a9e50e7f4a2d2c436882ee2215e6c420c454cb4de57adae84d3", - "size": "51849528" + "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "1eb526bb948c1c1bffdb5be318c044fc0216457be28b0d92aa2d92294b9d32c9", + "size": "52470216" }, "x86_64-linux": { - "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "shasum": "d0bfbe4c6e6871a36ad7c9be577df2abda37c69af437fedb6865d8729d60cbdc", - "size": "54905880" + "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "0188a70a3f84fe1918ac391a8b64bd241eb4c028c44ea023e853743cbecd3009", + "size": "55505692" }, "aarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "shasum": "de4eddf26f6f5e95d1bd055577cba7e9ebe9fbe8b1425bd4d40f073303ccc65b", - "size": "50663184" + "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "d4219908cded6efb97a7326e4159f82607e3956d6d56efb1580187bf0872603e", + "size": "51276904" }, "arm-linux": { - "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "shasum": "d3bc2a7973a89e58018244be6020ab5594ac50b31075f62b191bcb59a4394161", - "size": "51546152" + "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "b0b809695bee67798ce9c956be24d850beab5b2ca4f3bf3fc65552baea0214bb", + "size": "52142540" }, "riscv64-linux": { - "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "shasum": "4b492fc3b85fc71c9f89ecdce9b1a40cc74d3c5460fd1aeacca15cfcf75e39f2", - "size": "54724024" + "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "9244e44969386bd35477079b42db7c7dd5b636ed6ddd21cbcb35debf1db5f285", + "size": "55318440" }, "powerpc64le-linux": { - "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "shasum": "f9d8a4b1747042cdd8845ddf6c89f44efdb3c8d09306321b86efa1241d340bad", - "size": "54736740" + "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "fd9f8b35403ac5c78d40c979a0f467413ddbb0294c2157590bb8e48635c18df5", + "size": "55350924" }, "x86-linux": { - "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "shasum": "4b47620480328f5f82925022ace2c98e0b13d0730defcd3af74c82e9da1c2b2d", - "size": "57621640" + "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "24d2fbc05b7d92a63e228292c693f7fdfa974655510e8c7089e09ce8b9f41fd4", + "size": "58210984" }, "loongarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "shasum": "897d10a9f0cd6b8ab8454adc5c0fda9dfcecc65df97316400bf71396f78d7fe9", - "size": "51956176" + "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "1448d4e780a1257d042c836c237807c6ecaaba36c03961fef1940a842c22cd6d", + "size": "52566124" }, "s390x-linux": { - "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "shasum": "07501ac4c472d652d8f6128cc4026a68fd461c08a8a2006bc6fe66a9a8f70415", - "size": "54434044" + "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "d5bd0131aa1e1a40434acc703afd48f0d235ba9d1e988c944148d03df4dce967", + "size": "54992952" }, "x86_64-windows": { - "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.1912+0cbaaa5eb.zip", - "shasum": "719859787d0be77a8dc19f5ecc37d8cc7802ddca06f616d81e9ebde34e610dd4", - "size": "94983652" + "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.2611+f996d2866.zip", + "shasum": "3ec5cac4b74b5fe6e04f092ffb6e025993538ecf07970a428919d638efafa542", + "size": "98104739" }, "aarch64-windows": { - "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.1912+0cbaaa5eb.zip", - "shasum": "824044c2dd76d4fdf9dd3d3fc7e8ed9bd39a1a03a3eb3acf1fecde25cee640cc", - "size": "90939398" + "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.2611+f996d2866.zip", + "shasum": "42a612662453e47ad4001eb19c1d219dec6f254913988e2ebe6c95e4b5da3e82", + "size": "94105952" }, "x86-windows": { - "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.1912+0cbaaa5eb.zip", - "shasum": "aa8048f1ddc6c5dbd14d2d04665d2d9b82b6c8fc07d61813607a16e371e98149", - "size": "96824007" + "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.2611+f996d2866.zip", + "shasum": "ccc288a50431d732b4b8dcc55c6e76ec79b406cbb73c794864cd3c538a77ee0a", + "size": "99920751" }, "aarch64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "shasum": "e5d6ad13df1b22d0a9706c9df248890dcbf864a42daf1a307634816523f0134d", - "size": "50596788" + "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "c0c4b7bee7d67c4fb44692a96068bde05ba88e4bbe431928540db5bb29a8ab34", + "size": "51220656" }, "arm-freebsd": { - "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "shasum": "903745269c221596c4d9c59d727b89810f95498c86f015fd1627f30d00d830f7", - "size": "52216472" + "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "9f470b7304d70e59febfb23e14cacdbf1f81c2839e181f290009c3ccb04bbd4e", + "size": "52814796" }, "powerpc64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-powerpc64-freebsd-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "shasum": "de9f318029687e616118a44e4d8b3da5d3d3ffb05a22fc6a14f0d90f32b13b3b", - "size": "53314532" + "tarball": "https://ziglang.org/builds/zig-powerpc64-freebsd-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "7caa39c909cee4342876f0623a40ad91f364b509a14bdcdcdd86629071a1dfbb", + "size": "53925668" }, "powerpc64le-freebsd": { - "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "shasum": "8e2ec39e098f83de0408d40f58b201f8734e07f3763a3c6048b3ba76b8ca68ab", - "size": "54750456" + "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "4a0ea164b83b6bac83764c5b34c64b85ecbae1b685dc07a0fa59410efdde32c5", + "size": "55367852" }, "riscv64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "shasum": "b4aa67fc5b08f68dca234ba671934512765421e46a84500c57b239a9c3e1b308", - "size": "54857948" + "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "49f2e4403aad2256b921a2952a7e4e262c1077f3eb3bb0bd7c3e69a3154fbf68", + "size": "55478468" }, "x86_64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "shasum": "8dc6475b7b9b68c0322513b727da1f168d4e3d8b29f215cdf0e236e7650fa383", - "size": "55013180" + "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "169314c88df8e98c4b5c2db9805d160dcf332013fcd1f8a0df09f81a5eb3425a", + "size": "55617364" }, "aarch64-netbsd": { - "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "shasum": "9ecfc5b13dc620ccdb37452b59f8123d723db848d9e038413514a43715403f5f", - "size": "50554668" + "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "a091fd4f7128e0098915d6693eb720b1a4d1740dcb552e99a0957d99c3fda3c4", + "size": "51174248" }, "arm-netbsd": { - "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "shasum": "66e0ef652963906fccd9b2e4ff0af4d9d6968093c25dd1fff4b3de1f56e29f0e", - "size": "53254492" + "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "e399f1751fc25e3e04c2b7411a5fd9b9c316233b9370fc2fdf7fb45e8b020922", + "size": "53840612" }, "x86-netbsd": { - "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "shasum": "a2f092bfe2c34fa0193263fa3df18318886fda437767f1cc2a2d589baa65f6da", - "size": "58215316" + "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "8fae56f0683a8f9d2767ad89adc156971f0a872fb485b4c70966e6b113bcad68", + "size": "58800484" }, "x86_64-netbsd": { - "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.1912+0cbaaa5eb.tar.xz", - "shasum": "612f5e287a482b32eeb62e16f44ef41ce64b6ba5a9cc296f88a06e41014c3783", - "size": "54954084" + "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "0063013d3d306b4979b16f87db42fa1c7d94d5eac9275979cf40afa944328a56", + "size": "55551904" } }, "0.15.2": { diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel index c8fce3bd..c7c91eb4 100644 --- a/examples/llama/BUILD.bazel +++ b/examples/llama/BUILD.bazel @@ -9,7 +9,10 @@ zig_binary( srcs = ["llama.zig"], main = "main.zig", visibility = ["//visibility:public"], - deps = ["//zml"], + deps = [ + "//bazel", + "//zml", + ], ) zig_binary( @@ -20,7 +23,6 @@ zig_binary( deps = ["//zml"], ) - mtree_spec( name = "mtree", srcs = [":llama"], diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index cdc0f1be..fa567455 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -77,14 +77,14 @@ pub const LlamaLM = struct { progress: *std.Progress.Node, ) !zml.Bufferized(LlamaLM) { progress.increaseEstimatedTotalItems(store.view().count()); - var timer: std.time.Timer = try .start(); + const now: std.Io.Timestamp = .now(io, .awake); var total_bytes: usize = 0; defer { - const took = timer.read(); + const took = now.untilNow(io, .awake); log.info("Loaded weights [{Bi:.2}, {D}, {Bi:.2}/s]", .{ total_bytes, - took, - total_bytes / took * std.time.ns_per_s, + stdx.fmt.fmtDuration(took), + total_bytes / (@as(usize, @intCast(took.toNanoseconds())) * std.time.ns_per_s), }); } return zml.io.load(LlamaLM, self, allocator, io, platform, .{ diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 09d1f861..4ab3c51e 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -50,46 +50,41 @@ const CliArgs = struct { ; }; -pub fn main() !void { +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const arena = init.arena; + log.info("LLama was compiled with {}", .{@import("builtin").mode}); - // var debug_allocator: ?std.heap.DebugAllocator(.{}) = null; - // const allocator = if (@import("builtin").mode == .Debug) blk: { - // debug_allocator = .init; - // break :blk debug_allocator.?.allocator(); - // } else std.heap.c_allocator; - // defer if (debug_allocator) |*da| std.debug.assert(da.deinit() == .ok); - const allocator = std.heap.c_allocator; + const args = stdx.flags.parse(init.minimal.args, CliArgs); - var threaded: std.Io.Threaded = .init(allocator, .{}); - defer threaded.deinit(); - - const args = stdx.flags.parseProcessArgs(CliArgs); - - var vfs: zml.io.VFS = try .init(allocator, threaded.io()); + var vfs: zml.io.VFS = try .init(allocator, init.io); defer vfs.deinit(); + // var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); + // defer vfs_file.deinit(); + // try vfs.register("file", vfs_file.io()); + + // var vfs_https: zml.io.VFS.HTTP = try .init(allocator, init.io, &http_client, .https); + // defer vfs_https.deinit(); + // try vfs.register("https", vfs_https.io()); + var http_client: std.http.Client = .{ .allocator = allocator, - .io = threaded.io(), + .io = init.io, }; - - try http_client.initDefaultProxies(allocator); + try http_client.initDefaultProxies(arena.allocator(), init.environ_map); defer http_client.deinit(); - var vfs_file: zml.io.VFS.File = .init(allocator, threaded.io(), .{}); - defer vfs_file.deinit(); - try vfs.register("file", vfs_file.io()); - - var vfs_https: zml.io.VFS.HTTP = try .init(allocator, threaded.io(), &http_client, .https); - defer vfs_https.deinit(); - try vfs.register("https", vfs_https.io()); - - var hf_vfs: zml.io.VFS.HF = try .auto(allocator, threaded.io(), &http_client); + var hf_vfs: zml.io.VFS.HF = try .auto( + allocator, + init.io, + &http_client, + ); defer hf_vfs.deinit(); try vfs.register("hf", hf_vfs.io()); - const io = vfs.io(); + const io = vfs.io; log.info("Resolving model repo", .{}); const repo = try zml.safetensors.resolveModelRepo(io, args.model); @@ -219,7 +214,7 @@ pub fn main() !void { tokenizer, config, llama_options, - @intCast((try std.Io.Clock.now(.real, io)).toNanoseconds()), + @intCast(std.Io.Clock.now(.awake, io).toNanoseconds()), prompt[0..], false, &stdout.interface, @@ -228,9 +223,11 @@ pub fn main() !void { } fn parseConfig(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !std.json.Parsed(LlamaLM.Config) { - var timer = try std.time.Timer.start(); + const now: std.Io.Timestamp = .now(io, .awake); log.info("Loading model config", .{}); - defer log.info("Loaded model config [{D}]", .{timer.read()}); + defer log.info("Loaded model config [{D}]", .{ + stdx.fmt.fmtDuration(now.untilNow(io, .awake)), + }); const parsed_config = blk: { const config_json_file = try dir.openFile(io, "config.json", .{}); @@ -251,8 +248,10 @@ fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, prog progress.increaseEstimatedTotalItems(1); var node = progress.start("Loading tokenizer...", 1); defer node.end(); - var timer = try std.time.Timer.start(); - defer log.info("Loaded tokenizer [{D}]", .{timer.read()}); + const now: std.Io.Timestamp = .now(io, .awake); + defer log.info("Loaded tokenizer [{D}]", .{ + stdx.fmt.fmtDuration(now.untilNow(io, .awake)), + }); const bytes = b: { const file = try dir.openFile(io, "tokenizer.json", .{}); defer file.close(io); @@ -287,8 +286,8 @@ fn compileModel( parameters: LlamaParameters, progress: *std.Progress.Node, ) !CompileModelResult { - var timer = try std.time.Timer.start(); - defer log.info("Compiled model [{D}]", .{timer.read()}); + const now: std.Io.Timestamp = .now(io, .awake); + defer log.info("Compiled model [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); // Compile the model twice, one for prefill, one for generation. var prefill_future = try io.concurrent(struct { @@ -296,8 +295,8 @@ fn compileModel( progress_.increaseEstimatedTotalItems(1); var node_ = progress_.start("Compiling prefill...", 1); defer node_.end(); - var timer_ = try std.time.Timer.start(); - defer log.info("Compiled prefill [{D}]", .{timer_.read()}); + const now_: std.Io.Timestamp = .now(io_, .awake); + defer log.info("Compiled prefill [{D}]", .{stdx.fmt.fmtDuration(now_.untilNow(io_, .awake))}); return platform_.compile(allocator_, io_, llama_model_, .forward, .{ parameters_.prefill_tokens, parameters_.token_index, @@ -315,8 +314,8 @@ fn compileModel( progress_.increaseEstimatedTotalItems(1); var node_ = progress_.start("Compiling decode...", 1); defer node_.end(); - var timer_ = try std.time.Timer.start(); - defer log.info("Compiled decode [{D}]", .{timer_.read()}); + const now_: std.Io.Timestamp = .now(io_, .awake); + defer log.info("Compiled decode [{D}]", .{stdx.fmt.fmtDuration(now_.untilNow(io_, .awake))}); return platform_.compile(allocator_, io_, llama_model_, .forward, .{ parameters_.decode_tokens, parameters_.token_index, @@ -348,13 +347,13 @@ fn loadModelBuffers( ) !zml.Bufferized(llama.LlamaLM) { var transferred_bytes: usize = 0; - var timer = try stdx.time.Timer.start(); + const now: std.Io.Timestamp = .now(io, .awake); defer { - const duration = timer.read(); + const duration = now.untilNow(io, .awake); const seconds = @as(f64, @floatFromInt(duration.ns)) / 1e9; const gb_per_sec = @as(f64, @floatFromInt(transferred_bytes)) / (1024.0 * 1024.0 * 1024.0) / seconds; const gbps = gb_per_sec * 8.0; - log.info("Loaded model [{D} {d:.3} GB/s {d:.3} gbps]", .{ duration, gb_per_sec, gbps }); + log.info("Loaded model [{D} {d:.3} GB/s {d:.3} gbps]", .{ stdx.fmt.fmtDuration(duration), gb_per_sec, gbps }); } var arena = std.heap.ArenaAllocator.init(allocator); @@ -499,7 +498,7 @@ pub fn generateText( defer current_token_buffer.deinit(); const output_tokens_len = max_seq_len - prompt_tok.len - 1; - var timer = try stdx.time.Timer.start(); + const now: std.Io.Timestamp = .now(io, .awake); // One token has alreadyh been generated by the prefill. var num_tokens_generated: usize = 1; @@ -539,7 +538,11 @@ pub fn generateText( // extract the generated token from the buffer try current_token_buffer.toSlice(io, generated_token_slice); } - const duration = timer.read(); + const duration = now.untilNow(io, .awake); std.debug.print("\n", .{}); - log.info("✅ Generated {} tokens in {D}: {:.3}tok/s", .{ num_tokens_generated, duration, duration.div(num_tokens_generated).hzFloat() }); + log.info("✅ Generated {} tokens in {D}: {:.3}tok/s", .{ + num_tokens_generated, + stdx.fmt.fmtDuration(duration), + stdx.Io.Duration.hzFloat(stdx.Io.Duration.div(duration, num_tokens_generated)), + }); } diff --git a/examples/mnist/BUILD.bazel b/examples/mnist/BUILD.bazel index 787d30f4..ad10f9f1 100644 --- a/examples/mnist/BUILD.bazel +++ b/examples/mnist/BUILD.bazel @@ -12,7 +12,11 @@ zig_binary( "@mnist//:t10k-images.idx3-ubyte", ], main = "mnist.zig", - deps = ["//zml"], + visibility = ["//visibility:public"], + deps = [ + "//bazel", + "//zml", + ], ) build_test( diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index fdaa6900..696de3af 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -69,17 +69,13 @@ const Mnist = struct { } }; -pub fn main() !void { - const allocator = std.heap.smp_allocator; - - var threaded: std.Io.Threaded = .init(allocator, .{}); - defer threaded.deinit(); - - const io = threaded.io(); +pub fn main(init: std.process.Init) !void { + const arena = init.arena; + const allocator = init.gpa; + const io = init.io; // Parse program args - const process_args = try std.process.argsAlloc(allocator); - defer std.process.argsFree(allocator, process_args); + const process_args = try init.minimal.args.toSlice(arena.allocator()); const model_path = process_args[1]; const t10kfilename = process_args[2]; @@ -94,13 +90,14 @@ pub fn main() !void { // Auto-select platform const platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator); - // Compile model + // // Compile model const input: zml.Tensor = .init(.{ 28, 28 }, .u8); var exe = blk: { log.info("Compiling model....", .{}); - var timer = try std.time.Timer.start(); - defer log.info("✅ Compiled model [{D}]", .{timer.read()}); + const start: std.Io.Timestamp = .now(io, .awake); + defer log.info("✅ Compiled model [{D}]", .{stdx.fmt.fmtDuration(start.untilNow(io, .awake))}); break :blk try platform.compile(allocator, io, mnist_model, .forward, .{input}); }; defer exe.deinit(); @@ -108,8 +105,10 @@ pub fn main() !void { // Load buffers var mnist_buffers = blk: { log.info("Transfering weights....", .{}); - var timer = try std.time.Timer.start(); - defer log.info("✅ Transferred weights [{D}]", .{timer.read()}); + const start: std.Io.Timestamp = .now(io, .awake); + defer log.info("✅ Transferred weights [{D}]", .{ + stdx.fmt.fmtDuration(start.untilNow(io, .awake)), + }); break :blk try mnist_model.load(allocator, io, platform, &store); }; defer Mnist.unloadBuffers(&mnist_buffers); @@ -125,7 +124,7 @@ pub fn main() !void { defer dataset.close(io); var rng: std.Random.DefaultPrng = blk: { - const now = try std.Io.Clock.now(.awake, io); + const now: std.Io.Timestamp = .now(io, .awake); break :blk .init(@intCast(now.toMilliseconds())); }; diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 96979112..a21eec76 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -1326,31 +1326,15 @@ pub fn smallerVersion(version1: []const u8, version2: []const u8) []const u8 { pub fn currentVersion() []const u8 { const state = struct { var buf: [32]u8 = undefined; - var str: []const u8 = undefined; - var once = std.once(call); - - fn call() void { - str = stringFromStream(&buf, c.stablehloGetCurrentVersion, .{}); - } }; - - state.once.call(); - return state.str; + return stringFromStream(&state.buf, c.stablehloGetCurrentVersion, .{}); } pub fn minimumVersion() []const u8 { const state = struct { var buf: [32]u8 = undefined; - var str: []const u8 = undefined; - var once = std.once(call); - - fn call() void { - str = stringFromStream(&buf, c.stablehloGetMinimumVersion, .{}); - } }; - - state.once.call(); - return state.str; + return stringFromStream(&state.buf, c.stablehloGetMinimumVersion, .{}); } pub fn serializePortableArtifact(bytecode: []const u8, target_version: []const u8, writer: *std.Io.Writer) !void { diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index b9f000ae..073691e5 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -99,8 +99,7 @@ pub const Api = struct { pub fn loadFrom(library: [:0]const u8) !*const Api { const basename = std.Io.Dir.path.basename(library); log.info("Loading: {s}...", .{basename}); - var timer: std.time.Timer = try .start(); - defer log.info("Loaded: {s} [{D}]", .{ basename, timer.read() }); + defer log.info("Loaded: {s}", .{basename}); var lib: std.DynLib = switch (builtin.os.tag) { .linux, .macos => blk: { diff --git a/platforms/cpu/cpu.zig b/platforms/cpu/cpu.zig index 866087ce..37131347 100644 --- a/platforms/cpu/cpu.zig +++ b/platforms/cpu/cpu.zig @@ -15,12 +15,13 @@ pub fn isEnabled() bool { } pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { + _ = io; // autofix _ = allocator; // autofix if (comptime !isEnabled()) { return error.Unavailable; } - const r = try bazel.runfiles(io, bazel_builtin.current_repository); + const r = try bazel.runfiles(bazel_builtin.current_repository); var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const sandbox_path = try r.rlocation("zml/platforms/cpu/sandbox/lib", &path_buf) orelse { @@ -37,11 +38,6 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "libpjrt_cpu" ++ ext }); - var future = io.async(struct { - fn call(path_: [:0]const u8) !*const pjrt.Api { - return pjrt.Api.loadFrom(path_); - } - }.call, .{path}); - break :blk future.await(io); + break :blk .loadFrom(path); }; } diff --git a/stdx/Io.zig b/stdx/Io.zig index a7a1fe3e..5b5a35d8 100644 --- a/stdx/Io.zig +++ b/stdx/Io.zig @@ -135,3 +135,17 @@ pub const AllocatingLimitedConcurrentGroup = struct { self.group.cancel(io); } }; + +pub const Duration = struct { + pub fn div(self: std.Io.Duration, rhs: u64) std.Io.Duration { + return .fromNanoseconds(@divTrunc(self.nanoseconds, @as(i96, @intCast(rhs)))); + } + + pub fn hz(self: std.Io.Duration) u64 { + return @intCast(std.time.ns_per_s / self.nanoseconds); + } + + pub fn hzFloat(self: std.Io.Duration) f64 { + return (1 * std.time.ns_per_s) / @as(f64, @floatFromInt(self.nanoseconds)); + } +}; diff --git a/stdx/flags.zig b/stdx/flags.zig index a0654839..9a0f32ae 100644 --- a/stdx/flags.zig +++ b/stdx/flags.zig @@ -75,24 +75,24 @@ pub fn fatal(comptime fmt_string: []const u8, args: anytype) noreturn { /// `positional` field is treated specially, it designates positional arguments. /// /// If `pub const help` declaration is present, it is used to implement `-h/--help` argument. -pub fn parse(args: *std.process.ArgIterator, comptime CliArgs: type) CliArgs { - assert(args.skip()); // Discard executable name. +pub fn parse(args: std.process.Args, comptime CliArgs: type) CliArgs { + var it = args.iterate(); + assert(it.skip()); // Discard executable name. return switch (@typeInfo(CliArgs)) { - .@"union" => parse_commands(args, CliArgs), - .@"struct" => parse_flags(args, CliArgs), + .@"union" => parse_commands(&it, CliArgs), + .@"struct" => parse_flags(&it, CliArgs), else => unreachable, }; } /// Parse CLI arguments for current process. /// See `stdx.flags.parse` documentation for more. -pub fn parseProcessArgs(comptime CliArgs: type) CliArgs { - var args = std.process.args(); - return parse(&args, CliArgs); +pub fn parseProcessArgs(init_: std.process.Init.Minimal, comptime CliArgs: type) CliArgs { + return parse(init_.args, CliArgs); } -fn parse_commands(args: *std.process.ArgIterator, comptime Commands: type) Commands { +fn parse_commands(args: *std.process.Args.Iterator, comptime Commands: type) Commands { comptime assert(@typeInfo(Commands) == .Union); comptime assert(std.meta.fields(Commands).len >= 2); @@ -118,7 +118,7 @@ fn parse_commands(args: *std.process.ArgIterator, comptime Commands: type) Comma fatal("unknown subcommand: '{s}'", .{first_arg}); } -fn parse_flags(args: *std.process.ArgIterator, comptime Flags: type) Flags { +fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { @setEvalBranchQuota(5_000); if (Flags == void) { diff --git a/stdx/fmt.zig b/stdx/fmt.zig index 16dd843a..ec3a0343 100644 --- a/stdx/fmt.zig +++ b/stdx/fmt.zig @@ -176,3 +176,15 @@ fn printValue( inline else => try w.printValue("any", options, value, max_depth - 1), } } + +pub const FormatDuration = struct { + inner: std.Io.Duration, + + pub fn formatDuration(self: FormatDuration, w: *std.Io.Writer) std.Io.Writer.Error!void { + try w.printDurationSigned(@intCast(self.inner.toNanoseconds())); + } +}; + +pub fn fmtDuration(ts: std.Io.Duration) FormatDuration { + return .{ .inner = ts }; +} diff --git a/stdx/meta.zig b/stdx/meta.zig index 3bfc8895..9a096122 100644 --- a/stdx/meta.zig +++ b/stdx/meta.zig @@ -171,8 +171,10 @@ pub fn Tail(Tuple: type) type { .@"struct" => |struct_info| { if (struct_info.fields.len == 0) @compileError("Can't tail empty tuple"); var types: [struct_info.fields.len - 1]type = undefined; - for (struct_info.fields[1..], 0..) |field, i| types[i] = field.type; - return std.meta.Tuple(&types); + for (struct_info.fields[1..], &types) |field, *type_| { + type_.* = field.type; + } + return @Tuple(&types); }, else => @compileError("Tail works on tuple type"), }; diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 9401bdd4..83e14277 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -38,43 +38,40 @@ pub fn pinToCore(core_id: usize) void { } } -pub fn onceWithArgs(comptime f: anytype) OnceWithArgs(f) { +pub fn once(comptime f: anytype) once(f) { return .{}; } /// An object that executes the function `f` just once. /// It is undefined behavior if `f` re-enters the same Once instance. -pub fn OnceWithArgs(comptime f: anytype) type { +pub fn Once(comptime f: anytype) type { const Args = std.meta.ArgsTuple(@TypeOf(f)); - const Result = meta.FnResult(f); return struct { done: bool = false, - mutex: std.Thread.Mutex = .{}, - result: Result = undefined, + mutex: std.Io.Mutex = .init, /// Call the function `f`. /// If `call` is invoked multiple times `f` will be executed only the /// first time. /// The invocations are thread-safe. - pub fn call(self: *@This(), args: Args) Result { + pub fn call(self: *@This(), io: std.Io, args: Args) void { if (@atomicLoad(bool, &self.done, .acquire)) - return self.result; + return; - return self.callSlow(args); + self.callSlow(io, args); } - fn callSlow(self: *@This(), args: Args) Result { + fn callSlow(self: *@This(), io: std.Io, args: Args) void { @branchHint(.cold); - self.mutex.lock(); - defer self.mutex.unlock(); + self.mutex.lock(io); + defer self.mutex.unlock(io); // The first thread to acquire the mutex gets to run the initializer if (!self.done) { - self.result = @call(.auto, f, args); + @call(.auto, f, args); @atomicStore(bool, &self.done, true, .release); } - return self.result; } }; } diff --git a/third_party/arocc/repo.bzl b/third_party/arocc/repo.bzl index ac5f8573..510445d1 100644 --- a/third_party/arocc/repo.bzl +++ b/third_party/arocc/repo.bzl @@ -4,6 +4,6 @@ def repo(): new_git_repository( name = "arocc", remote = "https://github.com/zml/arocc.git", - commit = "7e60c78a9660016e46d8be8907591b143ba2e700", + commit = "73acf3aa8164cc8fee16b0f79036b92b07dab053", build_file = "//third_party/arocc:arocc.bazel", ) diff --git a/third_party/translate-c/repo.bzl b/third_party/translate-c/repo.bzl index 2b3facf1..ab5e9860 100644 --- a/third_party/translate-c/repo.bzl +++ b/third_party/translate-c/repo.bzl @@ -3,7 +3,7 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") def repo(): new_git_repository( name = "translate-c", - remote = "https://github.com/zml/translate-c.git", - commit = "5308ec6eba13a96dcaec452bffd0cc946384909a", + remote = "https://codeberg.org/ziglang/translate-c.git", + commit = "d2be2f19ef7c9caa1561d38c96581ac79dd4c654", build_file = "//third_party/translate-c:translate-c.bazel", ) diff --git a/third_party/zls/BUILD.bazel b/third_party/zls/BUILD.bazel index 00b78c07..b47cae3e 100644 --- a/third_party/zls/BUILD.bazel +++ b/third_party/zls/BUILD.bazel @@ -44,3 +44,8 @@ zig_binary( name = "workspace_printer", main = "workspace_printer.zig", ) + +zig_binary( + name = "build_runner", + main = "zls_build_runner.zig", +) diff --git a/third_party/zls/workspace_printer.zig b/third_party/zls/workspace_printer.zig index d55785ac..cbd8e130 100644 --- a/third_party/zls/workspace_printer.zig +++ b/third_party/zls/workspace_printer.zig @@ -1,114 +1,39 @@ const std = @import("std"); -const builtin = @import("builtin"); -const PATTERN = "@@__BUILD_WORKSPACE_DIRECTORY__@@"; +pub fn main(init: std.process.Init) !void { + const arena = init.arena; + const io = init.io; + const args = try init.minimal.args.toSlice(arena.allocator()); -pub const StringReplaceReader = struct { - interface: std.Io.Reader = .{ - .buffer = &.{}, - .vtable = &.{ - .stream = stream, + const build_workspace_directory = init.environ_map.get("BUILD_WORKSPACE_DIRECTORY").?; + var output_base_result = try std.process.run(arena.allocator(), init.io, .{ + .argv = &.{ + "bazel", + "info", + "execution_root", }, - .seek = 0, - .end = 0, - }, - in: *std.Io.Reader, - pattern: []const u8, - replace_by: []const u8, + .cwd = .{ .path = build_workspace_directory }, + }); - pub fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { - const self: *StringReplaceReader = @fieldParentPtr("interface", r); - if (std.mem.eql(u8, try self.in.peek(self.pattern.len), self.pattern)) { - self.in.toss(self.pattern.len); - try w.writeAll(self.replace_by); - return self.replace_by.len; - } - return self.in.streamDelimiterLimit(w, self.pattern[0], limit) catch |err| switch (err) { - std.Io.Reader.StreamDelimiterLimitError.StreamTooLong => return 0, - else => |e| return e, - }; - } -}; - -pub fn main_pre_015() !void { - var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); - defer arena.deinit(); - - const args = try std.process.argsAlloc(arena.allocator()); - defer std.process.argsFree(arena.allocator(), args); - - const file_path = args[1]; - const content = try std.fs.cwd().readFileAlloc(arena.allocator(), file_path, std.math.maxInt(usize)); - const build_workspace_directory = try std.process.getEnvVarOwned(arena.allocator(), "BUILD_WORKSPACE_DIRECTORY"); - - const need = std.mem.replacementSize(u8, content, PATTERN, build_workspace_directory); - const replaced = try arena.allocator().alloc(u8, need); - _ = std.mem.replace(u8, content, PATTERN, build_workspace_directory, replaced); - try std.io.getStdOut().writeAll(replaced); + var output = try std.Io.Dir.cwd().readFileAlloc( + io, + args[1], + arena.allocator(), + .unlimited, + ); + output = try std.mem.replaceOwned( + u8, + arena.allocator(), + output, + "@@__BUILD_WORKSPACE_DIRECTORY__@@", + build_workspace_directory, + ); + output = try std.mem.replaceOwned( + u8, + arena.allocator(), + output, + "@@__BAZEL_EXECUTION_ROOT__@@", + std.mem.trimEnd(u8, output_base_result.stdout, "\n"), + ); + try std.Io.File.stdout().writeStreamingAll(io, output); } - -pub fn main_015() !void { - var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); - defer arena.deinit(); - - const args = try std.process.argsAlloc(arena.allocator()); - defer std.process.argsFree(arena.allocator(), args); - - const file_path = args[1]; - var file = try std.fs.cwd().openFile(file_path, .{}); - defer file.close(); - - var read_buffer: [8192]u8 = undefined; - var reader = file.reader(&read_buffer); - - const build_workspace_directory = try std.process.getEnvVarOwned(arena.allocator(), "BUILD_WORKSPACE_DIRECTORY"); - var replacer: StringReplaceReader = .{ - .in = &reader.interface, - .pattern = "@@__BUILD_WORKSPACE_DIRECTORY__@@", - .replace_by = build_workspace_directory, - }; - - var write_buffer: [8192]u8 = undefined; - var stdout_writer = std.fs.File.stdout().writer(&write_buffer); - - _ = try replacer.interface.streamRemaining(&stdout_writer.interface); - try stdout_writer.interface.flush(); -} - -pub fn main_016() !void { - var arena = std.heap.ArenaAllocator.init(std.heap.smp_allocator); - defer arena.deinit(); - - var threaded: std.Io.Threaded = .init(arena.allocator(), .{}); - defer threaded.deinit(); - const io = threaded.io(); - - const args = try std.process.argsAlloc(arena.allocator()); - defer std.process.argsFree(arena.allocator(), args); - - const file_path = args[1]; - var file = try std.Io.Dir.cwd().openFile(io, file_path, .{ .mode = .read_only }); - defer file.close(io); - - var read_buffer: [8192]u8 = undefined; - var reader = file.reader(io, &read_buffer); - - const build_workspace_directory = try std.process.getEnvVarOwned(arena.allocator(), "BUILD_WORKSPACE_DIRECTORY"); - var replacer: StringReplaceReader = .{ - .in = &reader.interface, - .pattern = "@@__BUILD_WORKSPACE_DIRECTORY__@@", - .replace_by = build_workspace_directory, - }; - - var write_buffer: [8192]u8 = undefined; - var writer = std.Io.File.stdout().writer(io, &write_buffer); - _ = try replacer.interface.streamRemaining(&writer.interface); - try writer.interface.flush(); -} - -pub const main = if (builtin.zig_version.major == 0 and builtin.zig_version.minor >= 16) - main_016 -else if (builtin.zig_version.major == 0 and builtin.zig_version.minor == 15) - main_015 -else - main_pre_015; diff --git a/third_party/zls/zls.completion.tpl b/third_party/zls/zls.completion.tpl index 154b4c17..b3fb688e 100644 --- a/third_party/zls/zls.completion.tpl +++ b/third_party/zls/zls.completion.tpl @@ -1,6 +1,6 @@ { - "deps_build_roots": [], - "packages": [ + "dependencies": {}, + "modules": [ @@PACKAGES@@ ], "include_dirs": [], diff --git a/third_party/zls/zls_build_runner.zig b/third_party/zls/zls_build_runner.zig index a2ce8dad..ce82a66d 100644 --- a/third_party/zls/zls_build_runner.zig +++ b/third_party/zls/zls_build_runner.zig @@ -1,62 +1,14 @@ const std = @import("std"); -const builtin = @import("builtin"); -pub fn main() !void { - var arena_ = std.heap.ArenaAllocator.init(std.heap.smp_allocator); - defer arena_.deinit(); - const arena = arena_.allocator(); - - var threaded: std.Io.Threaded = .init(std.heap.smp_allocator, .{}); - defer threaded.deinit(); - - const io = threaded.io(); - - const build_workspace_directory = try std.process.getEnvVarOwned(arena, "BUILD_WORKSPACE_DIRECTORY"); - var child = std.process.Child.init(&.{ - "bazel", - "run", - "@@__TARGET__@@", - }, arena); - child.stdin_behavior = .Ignore; - child.stdout_behavior = .Inherit; - child.stderr_behavior = .Inherit; - child.cwd = build_workspace_directory; - - try child.spawn(io); - - // var out = child.stdout.?; - // _ = out; // autofix - - // const file = if (builtin.zig_version.major == 0 and builtin.zig_version.minor >= 16) blk: { - // var threaded = std.Io.Threaded.init(arena); - // defer threaded.deinit(); - // const io = threaded.io(); - - // var read_buffer: [8192]u8 = undefined; - // var reader = out.reader(io, &read_buffer); - // break :blk try reader.interface.allocRemaining(arena, .unlimited); - // } else try out.readToEndAlloc(arena, 16 * 1024 * 1024); - - // const need = std.mem.replacementSize( - // u8, - // file, - // "@@__BUILD_WORKSPACE_DIRECTORY__@@", - // build_workspace_directory, - // ); - - // const replaced = try arena.alloc(u8, need); - - // _ = std.mem.replace( - // u8, - // file, - // "@@__BUILD_WORKSPACE_DIRECTORY__@@", - // build_workspace_directory, - // replaced, - // ); - - // if (comptime builtin.zig_version.order(.{ .major = 0, .minor = 15, .patch = 0 }).compare(.gte)) { - // _ = try std.fs.File.stdout().writeAll(replaced); - // } else { - // _ = try std.io.getStdOut().writeAll(replaced); - // } +pub fn main(init: std.process.Init) !void { + const build_workspace_directory = init.environ_map.get("BUILD_WORKSPACE_DIRECTORY").?; + var child = try std.process.spawn(init.io, .{ + .argv = &.{ + "bazel", + "run", + "@@__TARGET__@@", + }, + .cwd = .{ .path = build_workspace_directory }, + }); + _ = try child.wait(init.io); } diff --git a/third_party/zls/zls_completion.bzl b/third_party/zls/zls_completion.bzl index ffb7cbf8..eac7dbd3 100644 --- a/third_party/zls/zls_completion.bzl +++ b/third_party/zls/zls_completion.bzl @@ -1,4 +1,5 @@ """Implementation of the zls_completion macro.""" + load("@aspect_bazel_lib//lib:utils.bzl", "utils") load("@bazel_skylib//rules:expand_template.bzl", "expand_template") load("@rules_zig//zig:defs.bzl", "zig_binary") @@ -13,6 +14,7 @@ def zls_completion(name, deps, **kwargs): deps: The List of Zig modules to include for completion. **kwargs: Additional keyword arguments passed to the `zig_binary` """ + # Generate the ZLS BuildConfig file. # It contains the list of Zig packages alongside their main Zig file paths. build_config = name + ".build_config" @@ -59,7 +61,7 @@ def zls_completion(name, deps, **kwargs): main = name + ".runner", data = [ "@rules_zig//zig:resolved_toolchain", - "@zml//third_party/zls:resolved_toolchain", + "@@//third_party/zls:resolved_toolchain", name + ".build_runner", ], deps = [ diff --git a/third_party/zls/zls_runner.zig b/third_party/zls/zls_runner.zig index 5ca4f695..c0808d9e 100644 --- a/third_party/zls/zls_runner.zig +++ b/third_party/zls/zls_runner.zig @@ -4,11 +4,12 @@ /// /// This file is used as a template by `zls_write_runner_zig_src.bzl`. const std = @import("std"); -const runfiles = @import("runfiles"); + const bazel_builtin = @import("bazel_builtin"); +const runfiles = @import("runfiles"); fn getRandomFilename(io: std.Io, buf: []u8, extension: []const u8) ![]const u8 { - const now = try std.Io.Clock.real.now(io); + const now = std.Io.Clock.real.now(io); return std.fmt.bufPrint(buf, "/tmp/{d}{s}", .{ now.nanoseconds, extension }) catch @panic("OOM"); } @@ -26,29 +27,21 @@ const Config = struct { global_cache_path: ?[]const u8 = null, }; -pub fn main() !void { - const gpa = std.heap.page_allocator; - var arena = std.heap.ArenaAllocator.init(gpa); - const allocator = arena.allocator(); +pub fn main(init: std.process.Init) !void { + const arena = init.arena; + const io = init.io; - var threaded: std.Io.Threaded = .init(gpa, .{}); - defer threaded.deinit(); - - const io = threaded.io(); - - var r_ = try runfiles.Runfiles.create(.{ .allocator = allocator, .io = io }) orelse + var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator(), .io = io, .environ_map = init.environ_map, .argv = init.minimal.args }) orelse return error.RunfilesNotFound; - defer r_.deinit(allocator); - const r = r_.withSourceRepo(bazel_builtin.current_repository); const zls_bin_rpath = "@@__ZLS_BIN_RPATH__@@"; - var zls_bin_path_buf: [std.fs.max_path_bytes]u8 = undefined; + var zls_bin_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const zls_bin_path = try r.rlocation(zls_bin_rpath, &zls_bin_path_buf) orelse return error.RLocationNotFound; const zig_exe_rpath = "@@__ZIG_EXE_RPATH__@@"; - var zig_exe_path_buf: [std.fs.max_path_bytes]u8 = undefined; + var zig_exe_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const zig_exe_path = blk2: { if (zig_exe_rpath[0] == '/') { break :blk2 zig_exe_rpath; @@ -68,7 +61,7 @@ pub fn main() !void { }; const zls_build_runner_rpath = "@@__ZLS_BUILD_RUNNER_RPATH__@@"; - var zls_build_runner_path_buf: [std.fs.max_path_bytes]u8 = undefined; + var zls_build_runner_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const zls_build_runner_path = try r.rlocation(zls_build_runner_rpath, &zls_build_runner_path_buf) orelse return error.RLocationNotFound; @@ -97,20 +90,20 @@ pub fn main() !void { try tmp_file_writer.interface.flush(); } - const args = std.process.argsAlloc(allocator) catch return error.ArgsAllocFailed; - defer std.process.argsFree(allocator, args); + const args = init.minimal.args.toSlice(arena.allocator()) catch return error.ArgsAllocFailed; const exec_args_len = args.len - 1 + 3; // Skip args[0] and add "zls" + --config-path + tmp_file_path - var exec_args = try allocator.alloc([]const u8, exec_args_len); + var exec_args = try arena.allocator().alloc([]const u8, exec_args_len); exec_args[0] = zls_bin_path; // "zls" - // ${@} - for (args[1..], 0..) |arg, i| { - exec_args[1 + i] = arg; - } - exec_args[exec_args.len - 2] = "--config-path"; - exec_args[exec_args.len - 1] = tmp_file_path; - - return std.process.execv(arena.allocator(), exec_args); + @memcpy(exec_args[1 .. exec_args.len - 2], args[1..]); + @memcpy(exec_args[exec_args.len - 2 ..], &[_][]const u8{ + "--config-path", + tmp_file_path, + }); + var child = try std.process.spawn(io, .{ + .argv = exec_args, + }); + _ = try child.wait(io); } diff --git a/third_party/zls/zls_toolchain.bzl b/third_party/zls/zls_toolchain.bzl index e487f8bc..6c1e20dc 100644 --- a/third_party/zls/zls_toolchain.bzl +++ b/third_party/zls/zls_toolchain.bzl @@ -2,7 +2,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -_VERSION = "0.16.0-dev.82+cbb62284" +_VERSION = "0.16.0-dev.215+0e289f5c" _ZLS_PLATFORMS = { "x86_64-linux": struct( @@ -28,7 +28,7 @@ _ZLS_PLATFORMS = { _ZLS_VERSIONS = { "x86_64-linux": "27277ee215687fda4a80fa518fb38061f4abb56a469349853ac3e5671e005421", "x86_64-macos": "98e3618d41532fcb1b6148bed75f081fc2f42e74fa6a23827870eb328dfbf87c", - "aarch64-macos": "083bc9b5fd9f70c7cd380766bffa7ce72a626237c62d7cf26ae3ac0395530ee0", + "aarch64-macos": "fb711e068e2ec77b3979458ff72f18311fa7df74644a8cf43985c0f88cae5c66", } ZlsToolchainInfo = provider( @@ -80,7 +80,7 @@ def _zls_toolchains_repos_impl(mctx): ), sha256 = _ZLS_VERSIONS[platform], build_file_content = """\ -load("@zml//third_party/zls:zls_toolchain.bzl", "zls_toolchain") +load("@@//third_party/zls:zls_toolchain.bzl", "zls_toolchain") zls_toolchain(name = "toolchain", bin = "zls") """, ) @@ -98,7 +98,7 @@ zls_toolchains = module_extension( ) def _zls_toolchains_repo_impl(rctx): - build_content = """# @generated by @zml//third_party/zls:zls_toolchain.bzl""" + build_content = """# @generated by @@//third_party/zls:zls_toolchain.bzl""" for platform, config in _ZLS_PLATFORMS.items(): build_content += """ @@ -107,7 +107,7 @@ toolchain( exec_compatible_with = {compatible_with}, target_compatible_with = {compatible_with}, toolchain = "@zls_{platform}//:toolchain", - toolchain_type = "@zml//third_party/zls:toolchain_type", + toolchain_type = "@@//third_party/zls:toolchain_type", ) """.format( platform = platform, diff --git a/third_party/zls/zls_write_build_config.bzl b/third_party/zls/zls_write_build_config.bzl index 1a758b67..861198e2 100644 --- a/third_party/zls/zls_write_build_config.bzl +++ b/third_party/zls/zls_write_build_config.bzl @@ -53,8 +53,11 @@ zls_construct_zig_module_info = aspect( implementation = _zls_construct_zig_module_info_impl, ) -def _add_context(context): - return json.encode(struct(name = context.name, path = "@@__BUILD_WORKSPACE_DIRECTORY__@@/" + context.main)) +def format_main_file(main): + prefix = "@@__BUILD_WORKSPACE_DIRECTORY__@@/" + if (main.startswith("bazel-out/") or main.startswith("external/")): + prefix = "@@__BAZEL_EXECUTION_ROOT__@@/" + return prefix + main def _zls_write_build_config_impl(ctx): zigtoolchaininfo = ctx.toolchains["@rules_zig//zig:toolchain_type"].zigtoolchaininfo @@ -91,7 +94,10 @@ def _zls_write_build_config_impl(ctx): cc_infos = [cc_info], ) - c_module_contexts = [depset(direct = [c_module.module_context])] + c_module_contexts = [depset( + direct = [c_module.module_context], + transitive = [c_module.transitive_module_contexts], + )] c_module_inputs = [c_module.transitive_inputs] contexts = depset( @@ -99,14 +105,33 @@ def _zls_write_build_config_impl(ctx): transitive = [dep[ZigModuleInfo].transitive_module_contexts for dep in ctx.attr.deps] + c_module_contexts, ) - template_dict = ctx.actions.template_dict() - template_dict.add_joined("@@PACKAGES@@", contexts, map_each = _add_context, join_with = ",\n ") + modules = { + mod.canonical_name: mod + for mod in contexts.to_list() + } + + output = { + "dependencies": {}, + "modules": { + format_main_file(mod.main): { + "import_table": { + dep.name: format_main_file(modules.get(dep.canonical_name, "").main) + for dep in mod.dependency_mappings + }, + "c_macros": [], + "include_dirs": [], + } + for mod in modules.values() + }, + "compilations": [], + "top_level_steps": [], + "available_options": {}, + } config = ctx.outputs.out - ctx.actions.expand_template( + ctx.actions.write( output = config, - template = ctx.file._zls_completion_tpl, - computed_substitutions = template_dict, + content = json.encode(output), ) return [ diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index e8da1a27..497bc834 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -28,8 +28,8 @@ zig_library( "exe.zig", "floats.zig", "io.zig", - "meta.zig", "mem.zig", + "meta.zig", "mlirx.zig", "module.zig", "nn.zig", @@ -53,6 +53,7 @@ zig_library( ":posix", ":xla_compile_options_upb", ":xla_data_upb", + "//bazel", "//mlir", "//mlir/dialects", "//pjrt", diff --git a/zml/meta.zig b/zml/meta.zig index bfd001d2..1274761b 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -491,7 +491,7 @@ pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) VisitReturn( else => {}, } }, - .array => |_| for (v) |*elem| if (can_error) try visit(cb, ctx, elem) else visit(cb, ctx, elem), + .array => for (v) |*elem| if (can_error) try visit(cb, ctx, elem) else visit(cb, ctx, elem), .optional => if (v.* != null) if (can_error) try visit(cb, ctx, &v.*.?) else visit(cb, ctx, &v.*.?), .@"union" => switch (v.*) { inline else => |*v_field| if (can_error) try visit(cb, ctx, v_field) else visit(cb, ctx, v_field), @@ -511,7 +511,7 @@ pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) VisitReturn( if (can_error) try visit(cb, ctx, &@field(v_elem, field.name)) else visit(cb, ctx, &@field(v_elem, field.name)); } }, - .array => |_| for (v) |*elem| if (can_error) try visit(cb, ctx, elem) else visit(cb, ctx, elem), + .array => for (v) |*elem| if (can_error) try visit(cb, ctx, elem) else visit(cb, ctx, elem), .optional => if (v.* != null) if (can_error) try visit(cb, ctx, &v.*.?) else visit(cb, ctx, &v.*.?), .@"union" => switch (v_elem.*) { inline else => |*v_field| if (can_error) try visit(cb, ctx, v_field) else visit(cb, ctx, v_field), diff --git a/zml/module.zig b/zml/module.zig index f717b093..8da65be4 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -59,14 +59,9 @@ pub const CompilationContext = struct { threadlocal var _current: ?*CompilationContext = null; - var mlir_once = std.once(struct { - fn call() void { - mlir.registerPasses("Transforms"); - } - }.call); - - pub fn init(allocator: std.mem.Allocator, platform: *const Platform) CompilationContext { - mlir_once.call(); + pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform) CompilationContext { + _ = io; // autofix + mlir.registerPasses("Transforms"); const mlir_registry = mlir.DialectRegistry.init() catch unreachable; inline for (.{ "func", "stablehlo" }) |d| { mlir.DialectHandle.fromString(d).insertDialect(mlir_registry); @@ -176,8 +171,14 @@ pub const CompilationContext = struct { //} }; -pub fn compile(allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, args: stdx.meta.FnArgs(func), platform: *const Platform) !Exe { - var compilation_context: CompilationContext = .init(allocator, platform); +pub fn compile( + allocator: std.mem.Allocator, + io: std.Io, + comptime func: anytype, + args: std.meta.ArgsTuple(@TypeOf(func)), + platform: *const Platform, +) !Exe { + var compilation_context: CompilationContext = .init(allocator, io, platform); defer compilation_context.deinit(); const result = emitMlir(&compilation_context, func, args) catch unreachable; diff --git a/zml/platform.zig b/zml/platform.zig index f291267d..2daf43d7 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -2,12 +2,12 @@ const std = @import("std"); const c = @import("c"); const pjrt = @import("pjrt"); -const pjrtx = @import("pjrtx.zig"); const platforms = @import("platforms"); pub const Target = platforms.Platform; const stdx = @import("stdx"); const Exe = @import("exe.zig").Exe; +const pjrtx = @import("pjrtx.zig"); const zml = @import("zml.zig"); const log = std.log.scoped(.zml); @@ -44,7 +44,7 @@ fn disableXlaLogs() void { }; _ = c.setenv( "TF_CPP_MIN_LOG_LEVEL", - std.posix.getenv("TF_CPP_MIN_LOG_LEVEL") orelse TF_CPP_LOG_LEVEL.ERROR, + std.c.getenv("TF_CPP_MIN_LOG_LEVEL") orelse TF_CPP_LOG_LEVEL.ERROR, 1, ); } @@ -339,7 +339,9 @@ pub const Platform = struct { io: std.Io, model_: anytype, comptime func: std.meta.DeclEnum(@TypeOf(model_)), - args: stdx.meta.Tail(stdx.meta.FnArgs(@field(@TypeOf(model_), @tagName(func)))), + args: stdx.meta.Tail( + std.meta.ArgsTuple(@TypeOf(@field(@TypeOf(model_), @tagName(func)))), + ), ) !Exe { return self.compileFn( allocator, @@ -349,11 +351,24 @@ pub const Platform = struct { ); } - pub fn compileModel(self: *const Platform, allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, model: stdx.meta.Head(stdx.meta.FnArgs(func)), args: stdx.meta.Tail(stdx.meta.FnArgs(func))) !Exe { + pub fn compileModel( + self: *const Platform, + allocator: std.mem.Allocator, + io: std.Io, + comptime func: anytype, + model: stdx.meta.Head(std.meta.ArgsTuple(@TypeOf(func))), + args: stdx.meta.Tail(std.meta.ArgsTuple(@TypeOf(func))), + ) !Exe { return self.compileFn(allocator, io, func, .{model} ++ args); } - pub fn compileFn(self: *const Platform, allocator: std.mem.Allocator, io: std.Io, comptime func: anytype, args: stdx.meta.FnArgs(func)) !Exe { + pub fn compileFn( + self: *const Platform, + allocator: std.mem.Allocator, + io: std.Io, + comptime func: anytype, + args: std.meta.ArgsTuple(@TypeOf(func)), + ) !Exe { return zml.module.compile(allocator, io, func, args, self); } diff --git a/zml/safetensors.zig b/zml/safetensors.zig index aa3585d8..f6f9b363 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -159,14 +159,13 @@ pub const TensorRegistry = struct { tensors: Tensors, metadata: Metadatas, - mutex: std.Thread.Mutex, + mutex: std.Io.Mutex = .init, pub fn init(allocator: std.mem.Allocator) TensorRegistry { return .{ .arena = std.heap.ArenaAllocator.init(allocator), .tensors = .{}, .metadata = .{}, - .mutex = .{}, }; } @@ -178,7 +177,6 @@ pub const TensorRegistry = struct { .arena = std.heap.ArenaAllocator.init(allocator), .tensors = .{}, .metadata = .empty, - .mutex = .{}, }; try self.mergeMetadata(metadata); @@ -223,8 +221,8 @@ pub const TensorRegistry = struct { self: *TensorRegistry, other: Metadatas, ) !void { - self.mutex.lock(); - defer self.mutex.unlock(); + // self.mutex.lock(); + // defer self.mutex.unlock(); const allocator = self.arena.allocator(); @@ -251,8 +249,8 @@ pub const TensorRegistry = struct { ) !void { const allocator = self.arena.allocator(); - self.mutex.lock(); - defer self.mutex.unlock(); + // self.mutex.lock(); + // defer self.mutex.unlock(); var tensor_copy = tensor; diff --git a/zml/zml.zig b/zml/zml.zig index 37252e74..4d576110 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -6,6 +6,7 @@ // Namespaces const std = @import("std"); +pub const bazel = @import("bazel"); const c = @import("c"); pub const pjrt = @import("pjrt"); const runfiles = @import("runfiles"); @@ -51,3 +52,7 @@ pub const KiB = 1024; pub const MiB = 1024 * KiB; pub const GiB = 1024 * MiB; pub const TiB = 1024 * GiB; + +pub fn init(init_: std.process.Init) !void { + try bazel.initSimple(init_); +} From d511605d5b7d7d7e5bbf13838c4bc1cd12dfcd9d Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 16 Feb 2026 13:45:51 +0000 Subject: [PATCH 280/881] platforms: port to zig 0.16.0-dev.2611+f996d2866 --- platforms/cuda/cuda.zig | 8 +++----- platforms/cuda/flashattn/flashattn.zig | 3 ++- platforms/neuron/neuron.zig | 7 +------ platforms/rocm/rocm.zig | 9 +++++---- platforms/tpu/tpu.zig | 9 ++------- 5 files changed, 13 insertions(+), 23 deletions(-) diff --git a/platforms/cuda/cuda.zig b/platforms/cuda/cuda.zig index 4a235e05..b0ff4425 100644 --- a/platforms/cuda/cuda.zig +++ b/platforms/cuda/cuda.zig @@ -26,9 +26,8 @@ fn hasCudaPathInLDPath() bool { } fn setupXlaGpuCudaDirFlag(allocator: std.mem.Allocator, sandbox: []const u8) !void { - const xla_flags = std.process.getEnvVarOwned(allocator, "XLA_FLAGS") catch ""; + const xla_flags = std.c.getenv("XLA_FLAGS") orelse ""; const new_xla_flagsZ = try std.fmt.allocPrintSentinel(allocator, "{s} --xla_gpu_cuda_data_dir={s}", .{ xla_flags, sandbox }, 0); - _ = c.setenv("XLA_FLAGS", new_xla_flagsZ, 1); } @@ -49,7 +48,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); - const r = try bazel.runfiles(io, bazel_builtin.current_repository); + const r = try bazel.runfiles(bazel_builtin.current_repository); var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const sandbox_path = try r.rlocation("libpjrt_cuda/sandbox", &path_buf) orelse { @@ -73,7 +72,6 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_cuda.so" }); - var future = io.async(pjrt.Api.loadFrom, .{path}); - break :blk future.await(io); + break :blk .loadFrom(path); }; } diff --git a/platforms/cuda/flashattn/flashattn.zig b/platforms/cuda/flashattn/flashattn.zig index c2d8ab75..2a743d33 100644 --- a/platforms/cuda/flashattn/flashattn.zig +++ b/platforms/cuda/flashattn/flashattn.zig @@ -94,9 +94,10 @@ pub var fa2_mha_varlen_fwd: Fa2MhaVarlenFwdFunc = undefined; pub var fa3_mha_fwd: Fa3MhaFwdFunc = undefined; pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { + _ = io; // autofix _ = allocator; // autofix - const r = try bazel.runfiles(io, bazel_builtin.current_repository); + const r = try bazel.runfiles(bazel_builtin.current_repository); var buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; const library = (try r.rlocation("flashattn/flashattn/lib/libflashattn.so", &buffer)) orelse return error.NotFound; diff --git a/platforms/neuron/neuron.zig b/platforms/neuron/neuron.zig index 3caa7258..8751f5ac 100644 --- a/platforms/neuron/neuron.zig +++ b/platforms/neuron/neuron.zig @@ -59,11 +59,6 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_neuron.so" }); - var future = io.async(struct { - fn call(path_: [:0]const u8) !*const pjrt.Api { - return pjrt.Api.loadFrom(path_); - } - }.call, .{path}); - break :blk future.await(io); + break :blk .loadFrom(path); }; } diff --git a/platforms/rocm/rocm.zig b/platforms/rocm/rocm.zig index 6073f697..617cc63c 100644 --- a/platforms/rocm/rocm.zig +++ b/platforms/rocm/rocm.zig @@ -48,9 +48,6 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { try setupRocmEnv(sandbox_path); - var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const lib_path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_rocm.so" }); - // We must load the PJRT plugin from the main thread. // // This is because libamdhip64.so use thread local storage as part of the static destructors... @@ -60,5 +57,9 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { // on the library, the thread-local storage (TLS) offset may be resolved // relative to the TLS base of the main thread, rather than the thread actually // executing the destructor. Accessing this variable results in a segmentation fault... - return try pjrt.Api.loadFrom(lib_path); + return blk: { + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const lib_path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_rocm.so" }); + break :blk .loadFrom(lib_path); + }; } diff --git a/platforms/tpu/tpu.zig b/platforms/tpu/tpu.zig index 6ce71097..a9316699 100644 --- a/platforms/tpu/tpu.zig +++ b/platforms/tpu/tpu.zig @@ -45,7 +45,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { return error.Unavailable; } - const r = try bazel.runfiles(io, bazel_builtin.current_repository); + const r = try bazel.runfiles(bazel_builtin.current_repository); var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const sandbox_path = try r.rlocation("libpjrt_tpu/sandbox", &path_buf) orelse { @@ -56,11 +56,6 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_tpu.so" }); - var future = io.async(struct { - fn call(path_: [:0]const u8) !*const pjrt.Api { - return pjrt.Api.loadFrom(path_); - } - }.call, .{path}); - break :blk future.await(io); + break :blk .loadFrom(path); }; } From aaab6f7a86a5e58142993fabdb3fcb966aba457a Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 16 Feb 2026 14:12:40 +0000 Subject: [PATCH 281/881] zml: load safetensors sequencially --- zml/safetensors.zig | 48 +-------------------------------------------- 1 file changed, 1 insertion(+), 47 deletions(-) diff --git a/zml/safetensors.zig b/zml/safetensors.zig index f6f9b363..5e4ff6ea 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -445,59 +445,13 @@ fn parseSafetensorsIndexFiles( safetensors_index: SafetensorsIndex, repo: std.Io.Dir, ) !void { - var group: std.Io.Group = .init; - defer group.cancel(io); - - var err: ?anyerror = null; - const filenames = safetensors_index.map.keys(); - for (filenames) |filename| { - group.async(io, AsyncParseSafetensorsIndexFile.run, .{ - allocator, - io, - registry, - repo, - filename, - &err, - }); - } - - try group.await(io); - - if (err) |e| { - log.err("Error parsing safetensors index files: {any}", .{e}); - return e; - } -} - -const AsyncParseSafetensorsIndexFile = struct { - pub fn run( - allocator: std.mem.Allocator, - io: std.Io, - registry: *TensorRegistry, - repo: std.Io.Dir, - filename: []const u8, - err_ptr: *?anyerror, - ) void { - parse(allocator, io, registry, repo, filename) catch |err| { - err_ptr.* = err; - }; - } - - fn parse( - allocator: std.mem.Allocator, - io: std.Io, - registry: *TensorRegistry, - repo: std.Io.Dir, - filename: []const u8, - ) !void { const file = try repo.openFile(io, filename, .{}); defer file.close(io); - try parseSafetensors(allocator, io, registry, file); } -}; +} pub fn resolveFiletype(io: std.Io, file: std.Io.File) !FileType { var path_buf: [1024]u8 = undefined; From 51a36fa8d86d83f07b83f954fb40d5f1453020b0 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 16 Feb 2026 14:13:14 +0000 Subject: [PATCH 282/881] third_party/zls: update hashes --- third_party/zls/zls_toolchain.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/third_party/zls/zls_toolchain.bzl b/third_party/zls/zls_toolchain.bzl index 6c1e20dc..5ca5fde0 100644 --- a/third_party/zls/zls_toolchain.bzl +++ b/third_party/zls/zls_toolchain.bzl @@ -26,8 +26,8 @@ _ZLS_PLATFORMS = { } _ZLS_VERSIONS = { - "x86_64-linux": "27277ee215687fda4a80fa518fb38061f4abb56a469349853ac3e5671e005421", - "x86_64-macos": "98e3618d41532fcb1b6148bed75f081fc2f42e74fa6a23827870eb328dfbf87c", + "x86_64-linux": "026a0ce9ea06b276319210509fa2a2edd7acf4e68da83b1a704c06ae26671548", + "x86_64-macos": "061845ec346620b914063763e8e39f9a374d2817c4ae976ea3eeb577a6e1cf32", "aarch64-macos": "fb711e068e2ec77b3979458ff72f18311fa7df74644a8cf43985c0f88cae5c66", } From f781bfdde3f403256fbb7fc7a80d253bc401fe40 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 16 Feb 2026 14:20:24 +0000 Subject: [PATCH 283/881] workspace: make rules_zig work on linux again --- MODULE.bazel | 2 +- zml/io.zig | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 3fcc9a16..bbddad6a 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -213,6 +213,6 @@ use_repo(oci, "distroless_cc_debian12", "distroless_cc_debian12_linux_amd64") git_override( module_name = "rules_zig", - commit = "6c75cd2e0ef95dec8c8dbcbecb4586d0b313cf06", + commit = "6c5d6a07b4ef48be894e7418bcae37bac54e1c2e", remote = "https://github.com/zml/rules_zig.git", ) diff --git a/zml/io.zig b/zml/io.zig index c39486bc..b3738a9e 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -465,10 +465,20 @@ pub const DirectMemoryWriter = struct { } } - self.flip_flop ^= 1; - self.offset += written; if (is_last) { - self.interface = .failing; + defer ctx.* = null; + defer self.interface = .failing; + const ctx_ = &ctx.*.?; + ctx_.event.waitUncancelable(self.io); + defer ctx_.pjrt_event.deinit(pjrt_api); + if (ctx_.err) |e| { + defer e.deinit(pjrt_api); + log.err("error while awaiting: {s}: {s}", .{ + @tagName(e.getCode(pjrt_api)), + e.getMessage(pjrt_api), + }); + return std.Io.Writer.Error.WriteFailed; + } } else { self.interface.end = 0; self.interface.buffer = self.pool.get(self.allocator, self.io) catch |err| { @@ -476,6 +486,9 @@ pub const DirectMemoryWriter = struct { return std.Io.Writer.Error.WriteFailed; }; } + self.flip_flop ^= 1; + self.offset += written; + return 0; } }; From b72124af505a1b267cf0d7ad049029b187d66b52 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 16 Feb 2026 14:20:48 +0000 Subject: [PATCH 284/881] examples/llama: disable VFS (until reenabling it later) --- examples/llama/main.zig | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 4ab3c51e..94dafe36 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -53,13 +53,14 @@ const CliArgs = struct { pub fn main(init: std.process.Init) !void { const allocator = init.gpa; const arena = init.arena; + _ = arena; // autofix log.info("LLama was compiled with {}", .{@import("builtin").mode}); const args = stdx.flags.parse(init.minimal.args, CliArgs); - var vfs: zml.io.VFS = try .init(allocator, init.io); - defer vfs.deinit(); + // var vfs: zml.io.VFS = try .init(allocator, init.io); + // defer vfs.deinit(); // var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); // defer vfs_file.deinit(); @@ -69,22 +70,22 @@ pub fn main(init: std.process.Init) !void { // defer vfs_https.deinit(); // try vfs.register("https", vfs_https.io()); - var http_client: std.http.Client = .{ - .allocator = allocator, - .io = init.io, - }; - try http_client.initDefaultProxies(arena.allocator(), init.environ_map); - defer http_client.deinit(); + // var http_client: std.http.Client = .{ + // .allocator = allocator, + // .io = init.io, + // }; + // try http_client.initDefaultProxies(arena.allocator(), init.environ_map); + // defer http_client.deinit(); - var hf_vfs: zml.io.VFS.HF = try .auto( - allocator, - init.io, - &http_client, - ); - defer hf_vfs.deinit(); - try vfs.register("hf", hf_vfs.io()); + // var hf_vfs: zml.io.VFS.HF = try .auto( + // allocator, + // init.io, + // &http_client, + // ); + // defer hf_vfs.deinit(); + // try vfs.register("hf", hf_vfs.io()); - const io = vfs.io; + const io = init.io; log.info("Resolving model repo", .{}); const repo = try zml.safetensors.resolveModelRepo(io, args.model); From 1323d02f4bfd91abe33871591f8b0331727cfd7a Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 16 Feb 2026 14:37:37 +0000 Subject: [PATCH 285/881] platforms/cuda: bump to CUDA 13.1 --- platforms/cuda/cuda.bzl | 24 ++++++++++++------------ platforms/cuda/libpjrt_cuda.BUILD.bazel | 4 ++++ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index c786ce3c..14e4867e 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -13,16 +13,16 @@ load("@rules_cc//cc:cc_import.bzl", "cc_import") ARCH = "linux-x86_64" CUDA_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cuda/redist/" -CUDA_VERSION = "13.0.2" -CUDA_REDIST_JSON_SHA256 = "" +CUDA_VERSION = "13.1.1" +CUDA_REDIST_JSON_SHA256 = "97cf605ccc4751825b1865f4af571c9b50dd29ffd13e9a38b296a9ecb1f0d422" CUDNN_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cudnn/redist/" -CUDNN_VERSION = "9.14.0" -CUDNN_REDIST_JSON_SHA256 = "fe58e8e9559ef5c61ab7a9954472d16acdcbad3b099004296ae410d25982830d" +CUDNN_VERSION = "9.19.0" +CUDNN_REDIST_JSON_SHA256 = "6ad4f2c047ee03131d1efb95db56f78c24953cb30880a2edcf99407ff9362a04" NVSHMEM_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/nvshmem/redist/" -NVSHMEM_VERSION = "3.4.5" -NVSHMEM_REDIST_JSON_SHA256 = "a656614a6ec638d85922bc816e5e26063308c3905273a72a863cf0f24e188f38" +NVSHMEM_VERSION = "3.5.19" +NVSHMEM_REDIST_JSON_SHA256 = "6dced4193eb728542504b346cfb768da6e3de2abca0cded95fda3a69729994d2" _UBUNTU_PACKAGES = { "zlib1g": packages.filegroup(name = "zlib1g", srcs = ["lib/x86_64-linux-gnu/libz.so.1"]), @@ -99,7 +99,7 @@ CUDA_PACKAGES = { name = "cuda_nvrtc", srcs = [ "lib/libnvrtc.so.13", - "lib/libnvrtc-builtins.so.13.0", + "lib/libnvrtc-builtins.so.13.1", ], ), ]), @@ -138,7 +138,7 @@ NVSHMEM_PACKAGES = { srcs = [ "lib/libnvshmem_host.so.3", "lib/nvshmem_bootstrap_uid.so.3", - "lib/nvshmem_transport_ibrc.so.3", + "lib/nvshmem_transport_ibrc.so.4", ], ), } @@ -201,7 +201,7 @@ def _cuda_impl(mctx): arch_data = pkg_data.get(ARCH) if not arch_data: continue - arch_data = arch_data.get("cuda12", arch_data) + arch_data = arch_data.get("cuda13", arch_data) http_archive( name = pkg, build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, @@ -215,7 +215,7 @@ def _cuda_impl(mctx): arch_data = pkg_data.get(ARCH) if not arch_data: continue - arch_data = arch_data.get("cuda12", arch_data) + arch_data = arch_data.get("cuda13", arch_data) http_archive( name = pkg, build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, @@ -226,9 +226,9 @@ def _cuda_impl(mctx): http_archive( name = "nccl", - urls = ["https://pypi.nvidia.com/nvidia-nccl-cu13/nvidia_nccl_cu13-2.27.7-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl"], + urls = ["https://pypi.nvidia.com/nvidia-nccl-cu13/nvidia_nccl_cu13-2.29.3-py3-none-manylinux_2_18_x86_64.whl"], type = "zip", - sha256 = "b28a524abd8389b76a4a3f133c76a7aaa7005e47fcaa9d9603b90103927a3f93", + sha256 = "2a321629f49490e4e0122ecb578a4b4a6f89e72740dd988e04dfa4758fab7fc3", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + packages.filegroup( name = "nccl", srcs = ["nvidia/nccl/lib/libnccl.so.2"], diff --git a/platforms/cuda/libpjrt_cuda.BUILD.bazel b/platforms/cuda/libpjrt_cuda.BUILD.bazel index 6d7f17e5..372874a9 100644 --- a/platforms/cuda/libpjrt_cuda.BUILD.bazel +++ b/platforms/cuda/libpjrt_cuda.BUILD.bazel @@ -11,6 +11,10 @@ patchelf( rename_dynamic_symbols = { "dlopen": "zmlxcuda_dlopen", }, + replace_needed = { + "nvshmem_transport_ibrc.so.3": "nvshmem_transport_ibrc.so.4", + "libnvrtc-builtins.so.13.0": "libnvrtc-builtins.so.13.1", + }, set_rpath = "$ORIGIN", ) From 18896ce10c08ee52feb9e98e7bbf39877d19947a Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 16 Feb 2026 18:25:03 +0000 Subject: [PATCH 286/881] platforms/rocm: bump to ROCm 7.2.0 --- platforms/rocm/BUILD.bazel | 5 - platforms/rocm/gfx.bzl | 47 -- platforms/rocm/libpjrt_rocm.BUILD.bazel | 8 +- platforms/rocm/packages.lock.json | 897 ++++++++++-------------- platforms/rocm/packages.yaml | 64 +- platforms/rocm/rocm.bzl | 39 +- platforms/rocm/rocm.zig | 2 +- zml/io.zig | 4 +- zml/mem.zig | 4 +- zml/module.zig | 4 +- 10 files changed, 403 insertions(+), 671 deletions(-) delete mode 100644 platforms/rocm/gfx.bzl diff --git a/platforms/rocm/BUILD.bazel b/platforms/rocm/BUILD.bazel index ff764fb0..76ee4e9f 100644 --- a/platforms/rocm/BUILD.bazel +++ b/platforms/rocm/BUILD.bazel @@ -17,11 +17,6 @@ alias( actual = "@libpjrt_rocm//:hipblaslt", ) -alias( - name = "gfx", - actual = "@libpjrt_rocm//:gfx", -) - cc_library( name = "empty", defines = ["ZML_RUNTIME_ROCM_DISABLED"], diff --git a/platforms/rocm/gfx.bzl b/platforms/rocm/gfx.bzl deleted file mode 100644 index aa2d325d..00000000 --- a/platforms/rocm/gfx.bzl +++ /dev/null @@ -1,47 +0,0 @@ -load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") - -_ALL_GFX = ["gfx900", "gfx906", "gfx908", "gfx90a", "gfx940", "gfx941", "gfx942", "gfx1010", "gfx1012", "gfx1030", "gfx1100", "gfx1101", "gfx1102"] - -def _compute_enabled_gfx(values): - ret = {} - for v in values: - if (v == "all"): - ret = {gfx: True for gfx in _ALL_GFX} - elif (v == "none"): - ret = {} - else: - ret[v] = True - return ret - -def _gfx_from_file(file): - return file.basename[:-len(file.extension) - 1].rpartition("_")[-1].partition("-")[0] - -def _is_file_enabled(file, enabled_gfx): - gfx = _gfx_from_file(file) - return gfx in enabled_gfx or gfx == "fallback" - -def _bytecode_select_impl(ctx): - enabled_gfx = _compute_enabled_gfx(ctx.attr.enabled_gfx[BuildSettingInfo].value) - return [ - DefaultInfo( - files = depset([ - file - for file in ctx.files.srcs - if _is_file_enabled(file, enabled_gfx) - ]), - ), - ] - -bytecode_select = rule( - implementation = _bytecode_select_impl, - attrs = { - "srcs": attr.label_list(allow_files = True), - "enabled_gfx": attr.label(mandatory = True), - }, -) - -def if_gfx(gfx, value): - return select({ - "@zml//platforms/rocm:_{}".format(gfx): value, - "//conditions:default": [], - }) diff --git a/platforms/rocm/libpjrt_rocm.BUILD.bazel b/platforms/rocm/libpjrt_rocm.BUILD.bazel index 3147d09e..4aaabacf 100644 --- a/platforms/rocm/libpjrt_rocm.BUILD.bazel +++ b/platforms/rocm/libpjrt_rocm.BUILD.bazel @@ -3,15 +3,9 @@ load("@aspect_bazel_lib//lib:tar.bzl", "tar") load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "string_list_flag") load("@zml//bazel:patchelf.bzl", "patchelf") -string_list_flag( - name = "gfx", - build_setting_default = ["all"], - visibility = ["//visibility:public"], -) - bool_flag( name = "hipblaslt", - build_setting_default = True, + build_setting_default = False, visibility = ["//visibility:public"], ) diff --git a/platforms/rocm/packages.lock.json b/platforms/rocm/packages.lock.json index 4737457d..67139f24 100755 --- a/platforms/rocm/packages.lock.json +++ b/platforms/rocm/packages.lock.json @@ -4,37 +4,37 @@ "arch": "amd64", "dependencies": [ { - "key": "amdgpu-core_1-7.1.70101-2255337.22.04_amd64", + "key": "amdgpu-core_1-7.2.70200-2278374.22.04_amd64", "name": "amdgpu-core", - "version": "1:7.1.70101-2255337.22.04" + "version": "1:7.2.70200-2278374.22.04" } ], - "key": "libdrm-amdgpu-common_1.0.0.70101-2255337.22.04_amd64", + "key": "libdrm-amdgpu-common_1.0.0.70200-2278374.22.04_amd64", "name": "libdrm-amdgpu-common", - "sha256": "19330c218a1643f8b683b6c68c7bb0754932d839948e8a1d3f78e64ac5e111ff", + "sha256": "baa11bc7e3f8e6931a69dea7f1485c2f982da184811ac7545342953ffb64abc4", "urls": [ - "https://repo.radeon.com/graphics/7.1.1/ubuntu/pool/main/libd/libdrm-amdgpu-common/libdrm-amdgpu-common_1.0.0.70101-2255337.22.04_all.deb" + "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/libd/libdrm-amdgpu-common/libdrm-amdgpu-common_1.0.0.70200-2278374.22.04_all.deb" ], - "version": "1.0.0.70101-2255337.22.04" + "version": "1.0.0.70200-2278374.22.04" }, { "arch": "amd64", "dependencies": [], - "key": "amdgpu-core_1-7.1.70101-2255337.22.04_amd64", + "key": "amdgpu-core_1-7.2.70200-2278374.22.04_amd64", "name": "amdgpu-core", - "sha256": "2b1b81ab71327771fa7ec0dad3673441a97d5c20348128fe71d8ef6b6ce7c9ca", + "sha256": "916fdee8e388c36a67abda89e0868e5789f4b77a3c351f945db4218fbcbe226c", "urls": [ - "https://repo.radeon.com/graphics/7.1.1/ubuntu/pool/main/a/amdgpu-core/amdgpu-core_7.1.70101-2255337.22.04_all.deb" + "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/a/amdgpu-core/amdgpu-core_7.2.70200-2278374.22.04_all.deb" ], - "version": "1:7.1.70101-2255337.22.04" + "version": "1:7.2.70200-2278374.22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocprofiler-register_0.6.0.70101-38_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70101-38~22.04" + "version": "0.6.0.70200-43~22.04" }, { "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", @@ -62,9 +62,9 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", "name": "rocm-core", - "version": "7.1.1.70101-38~22.04" + "version": "7.2.0.70200-43~22.04" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -272,33 +272,33 @@ "version": "2.0.14-3ubuntu2" }, { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70101-2255337.22.04_amd64", + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70200-2278374.22.04_amd64", "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.125.70101-2255337.22.04" + "version": "1:2.4.125.70200-2278374.22.04" }, { - "key": "libdrm-amdgpu-common_1.0.0.70101-2255337.22.04_amd64", + "key": "libdrm-amdgpu-common_1.0.0.70200-2278374.22.04_amd64", "name": "libdrm-amdgpu-common", - "version": "1.0.0.70101-2255337.22.04" + "version": "1.0.0.70200-2278374.22.04" }, { - "key": "amdgpu-core_1-7.1.70101-2255337.22.04_amd64", + "key": "amdgpu-core_1-7.2.70200-2278374.22.04_amd64", "name": "amdgpu-core", - "version": "1:7.1.70101-2255337.22.04" + "version": "1:7.2.70200-2278374.22.04" }, { - "key": "libdrm2-amdgpu_1-2.4.125.70101-2255337.22.04_amd64", + "key": "libdrm2-amdgpu_1-2.4.125.70200-2278374.22.04_amd64", "name": "libdrm2-amdgpu", - "version": "1:2.4.125.70101-2255337.22.04" + "version": "1:2.4.125.70200-2278374.22.04" } ], - "key": "hsa-rocr_1.18.0.70101-38_22.04_amd64", + "key": "hsa-rocr_1.18.0.70200-43_22.04_amd64", "name": "hsa-rocr", - "sha256": "34cb8d18abbad9c9bfada312e5e72b80cdd8d6e4b3410b4090cf5697ce06e1f0", + "sha256": "c2ee2205b61a0c6b00478e70fa0cde4441fc9658cddcedb8bff07ab07569719c", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/h/hsa-rocr/hsa-rocr_1.18.0.70101-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hsa-rocr/hsa-rocr_1.18.0.70200-43~22.04_amd64.deb" ], - "version": "1.18.0.70101-38~22.04" + "version": "1.18.0.70200-43~22.04" }, { "arch": "amd64", @@ -329,9 +329,9 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", "name": "rocm-core", - "version": "7.1.1.70101-38~22.04" + "version": "7.2.0.70200-43~22.04" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -529,13 +529,13 @@ "version": "3.10.6-1~22.04.1" } ], - "key": "rocprofiler-register_0.6.0.70101-38_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", "name": "rocprofiler-register", - "sha256": "379c4129de4f0087c142e75b3891822569d1bde6abfee3cad9d7e83bd8054b63", + "sha256": "144d516c55f368e82c231df469c1d3097ee4f44302ef3b71fe3eeded4199f5d3", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocprofiler-register/rocprofiler-register_0.6.0.70101-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocprofiler-register/rocprofiler-register_0.6.0.70200-43~22.04_amd64.deb" ], - "version": "0.6.0.70101-38~22.04" + "version": "0.6.0.70200-43~22.04" }, { "arch": "amd64", @@ -595,13 +595,13 @@ { "arch": "amd64", "dependencies": [], - "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", "name": "rocm-core", - "sha256": "7332c2d6101392fae6b83889e25208e6a261f1152407a5d332aa6ad4da711931", + "sha256": "9e8c61b3b260547f88b795f521d82249dfae50f9f82522c6ba9a4fad8fe10f0a", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocm-core/rocm-core_7.1.1.70101-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocm-core/rocm-core_7.2.0.70200-43~22.04_amd64.deb" ], - "version": "7.1.1.70101-38~22.04" + "version": "7.2.0.70200-43~22.04" }, { "arch": "amd64", @@ -1057,32 +1057,32 @@ { "arch": "amd64", "dependencies": [], - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70101-2255337.22.04_amd64", + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70200-2278374.22.04_amd64", "name": "libdrm-amdgpu-amdgpu1", - "sha256": "c97887b0d1198d3e572ead52ff3061d088d3821430c0066ae489133389ef7117", + "sha256": "460f6dd499dbff0d4bc368d40b0c3f943dbf981df77f362ac05cce7befcf7779", "urls": [ - "https://repo.radeon.com/graphics/7.1.1/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-amdgpu1_2.4.125.70101-2255337.22.04_amd64.deb" + "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-amdgpu1_2.4.125.70200-2278374.22.04_amd64.deb" ], - "version": "1:2.4.125.70101-2255337.22.04" + "version": "1:2.4.125.70200-2278374.22.04" }, { "arch": "amd64", "dependencies": [], - "key": "libdrm2-amdgpu_1-2.4.125.70101-2255337.22.04_amd64", + "key": "libdrm2-amdgpu_1-2.4.125.70200-2278374.22.04_amd64", "name": "libdrm2-amdgpu", - "sha256": "5314dc876a0a84f60575af8bdc6773cd9fe9cc8072089adac72ca04304d0411c", + "sha256": "a5a620541bcd852d6feb8299dda804c41cb979a3870cf82a783112b58ad9d7e4", "urls": [ - "https://repo.radeon.com/graphics/7.1.1/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm2-amdgpu_2.4.125.70101-2255337.22.04_amd64.deb" + "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm2-amdgpu_2.4.125.70200-2278374.22.04_amd64.deb" ], - "version": "1:2.4.125.70101-2255337.22.04" + "version": "1:2.4.125.70200-2278374.22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", "name": "rocm-core", - "version": "7.1.1.70101-38~22.04" + "version": "7.2.0.70200-43~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -1305,53 +1305,34 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "hsa-rocr_1.18.0.70101-38_22.04_amd64", - "name": "hsa-rocr", - "version": "1.18.0.70101-38~22.04" - }, - { - "key": "rocprofiler-register_0.6.0.70101-38_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.6.0.70101-38~22.04" + "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", + "name": "hsa-runtime-rocr4wsl-amdgpu", + "version": "25.30.13-2281980.22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", "name": "libelf1", "version": "0.186-1ubuntu0.1" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70101-2255337.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.125.70101-2255337.22.04" - }, - { - "key": "libdrm-amdgpu-common_1.0.0.70101-2255337.22.04_amd64", - "name": "libdrm-amdgpu-common", - "version": "1.0.0.70101-2255337.22.04" - }, - { - "key": "amdgpu-core_1-7.1.70101-2255337.22.04_amd64", - "name": "amdgpu-core", - "version": "1:7.1.70101-2255337.22.04" - }, - { - "key": "libdrm2-amdgpu_1-2.4.125.70101-2255337.22.04_amd64", - "name": "libdrm2-amdgpu", - "version": "1:2.4.125.70101-2255337.22.04" } ], - "key": "hsa-amd-aqlprofile_1.0.0.70101-38_22.04_amd64", + "key": "hsa-amd-aqlprofile_1.0.0.70200-43_22.04_amd64", "name": "hsa-amd-aqlprofile", - "sha256": "d55331fdf6e4e27dce1b02a2f8ae8b5df811357bb0632df921df176044e6a36f", + "sha256": "c4eabbbac5adb81d30442fe4172660591b4b11f70596035c0cd6b5dc80bd27be", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/h/hsa-amd-aqlprofile/hsa-amd-aqlprofile_1.0.0.70101-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hsa-amd-aqlprofile/hsa-amd-aqlprofile_1.0.0.70200-43~22.04_amd64.deb" ], - "version": "1.0.0.70101-38~22.04" + "version": "1.0.0.70200-43~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", + "name": "hsa-runtime-rocr4wsl-amdgpu", + "sha256": "a62cc2bbf9cfc46b263b17ff90bea66ae01267c03014b55a9764eab6895db4a7", + "urls": [ + "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/h/hsa-runtime-rocr4wsl-amdgpu/hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64.deb" + ], + "version": "25.30.13-2281980.22.04" }, { "arch": "amd64", @@ -1392,9 +1373,9 @@ "version": "1.4.8+dfsg-3build1" }, { - "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", "name": "rocm-core", - "version": "7.1.1.70101-38~22.04" + "version": "7.2.0.70200-43~22.04" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -1637,13 +1618,13 @@ "version": "6.3-2ubuntu0.1" } ], - "key": "comgr_3.0.0.70101-38_22.04_amd64", + "key": "comgr_3.0.0.70200-43_22.04_amd64", "name": "comgr", - "sha256": "0b3e5d433a512a7ee07b076fbcf1d3867d0061160d23a741ab37d2be4af141b4", + "sha256": "8f58b1958811430ba19426489608b3cbf685e5a78c1d6d9414c7f23cc8311a13", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/c/comgr/comgr_3.0.0.70101-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/c/comgr/comgr_3.0.0.70200-43~22.04_amd64.deb" ], - "version": "3.0.0.70101-38~22.04" + "version": "3.0.0.70200-43~22.04" }, { "arch": "amd64", @@ -1770,14 +1751,14 @@ "arch": "amd64", "dependencies": [ { - "key": "rocprofiler-sdk-rocpd_1.0.0-38_22.04_amd64", + "key": "rocprofiler-sdk-rocpd_1.1.0-43_22.04_amd64", "name": "rocprofiler-sdk-rocpd", - "version": "1.0.0-38~22.04" + "version": "1.1.0-43~22.04" }, { - "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", "name": "rocm-core", - "version": "7.1.1.70101-38~22.04" + "version": "7.2.0.70200-43~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -2000,53 +1981,53 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "rocprofiler-sdk-roctx_1.0.0-38_22.04_amd64", + "key": "rocprofiler-sdk-roctx_1.1.0-43_22.04_amd64", "name": "rocprofiler-sdk-roctx", - "version": "1.0.0-38~22.04" + "version": "1.1.0-43~22.04" }, { - "key": "rocprofiler-register_0.6.0.70101-38_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70101-38~22.04" + "version": "0.6.0.70200-43~22.04" } ], - "key": "rocprofiler-sdk_1.0.0-38_22.04_amd64", + "key": "rocprofiler-sdk_1.1.0-43_22.04_amd64", "name": "rocprofiler-sdk", - "sha256": "49268825761685f71efd3a07b538b4b9f4038e4a00b90968877b0c7d451f944e", + "sha256": "adaebb45a2b713fb7677cca10d50afbce33b025f6bbf5a84cf37707fc413127d", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocprofiler-sdk/rocprofiler-sdk_1.0.0-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocprofiler-sdk/rocprofiler-sdk_1.1.0-43~22.04_amd64.deb" ], - "version": "1.0.0-38~22.04" + "version": "1.1.0-43~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocprofiler-sdk-rocpd_1.0.0-38_22.04_amd64", + "key": "rocprofiler-sdk-rocpd_1.1.0-43_22.04_amd64", "name": "rocprofiler-sdk-rocpd", - "sha256": "6be1beb6fcbdf4e01a65806ed7b759b2499a708a294a29889f3788d47e2ed0e8", + "sha256": "2dbfad4c5c1e55bb65771bb3aa34460c52e6c7952767f26756d8aa7d98c47414", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocprofiler-sdk-rocpd/rocprofiler-sdk-rocpd_1.0.0-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocprofiler-sdk-rocpd/rocprofiler-sdk-rocpd_1.1.0-43~22.04_amd64.deb" ], - "version": "1.0.0-38~22.04" + "version": "1.1.0-43~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocprofiler-sdk-roctx_1.0.0-38_22.04_amd64", + "key": "rocprofiler-sdk-roctx_1.1.0-43_22.04_amd64", "name": "rocprofiler-sdk-roctx", - "sha256": "35112e33ca1cce1b86eb7e31a703320e022c6825c5c8c0f45f9eec01ac58e9ba", + "sha256": "4c74438154cd91d81768c5ecbbed24859f787db638dcacce6b70445abf8577c7", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocprofiler-sdk-roctx/rocprofiler-sdk-roctx_1.0.0-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocprofiler-sdk-roctx/rocprofiler-sdk-roctx_1.1.0-43~22.04_amd64.deb" ], - "version": "1.0.0-38~22.04" + "version": "1.1.0-43~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", "name": "rocm-core", - "version": "7.1.1.70101-38~22.04" + "version": "7.2.0.70200-43~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -2269,29 +2250,29 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "rocrand_4.1.0.70101-38_22.04_amd64", + "key": "rocrand_4.2.0.70200-43_22.04_amd64", "name": "rocrand", - "version": "4.1.0.70101-38~22.04" + "version": "4.2.0.70200-43~22.04" }, { - "key": "hipblaslt_1.1.0.70101-38_22.04_amd64", + "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", "name": "hipblaslt", - "version": "1.1.0.70101-38~22.04" + "version": "1.2.1.70200-43~22.04" }, { - "key": "roctracer_4.1.70101.70101-38_22.04_amd64", + "key": "roctracer_4.1.70200.70200-43_22.04_amd64", "name": "roctracer", - "version": "4.1.70101.70101-38~22.04" + "version": "4.1.70200.70200-43~22.04" }, { - "key": "rocblas_5.1.1.70101-38_22.04_amd64", + "key": "rocblas_5.2.0.70200-43_22.04_amd64", "name": "rocblas", - "version": "5.1.1.70101-38~22.04" + "version": "5.2.0.70200-43~22.04" }, { - "key": "hip-runtime-amd_7.1.52802.70101-38_22.04_amd64", + "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", "name": "hip-runtime-amd", - "version": "7.1.52802.70101-38~22.04" + "version": "7.2.26015.70200-43~22.04" }, { "key": "libnuma1_2.0.14-3ubuntu2_amd64", @@ -2299,14 +2280,14 @@ "version": "2.0.14-3ubuntu2" }, { - "key": "rocprofiler-register_0.6.0.70101-38_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70101-38~22.04" + "version": "0.6.0.70200-43~22.04" }, { - "key": "comgr_3.0.0.70101-38_22.04_amd64", + "key": "comgr_3.0.0.70200-43_22.04_amd64", "name": "comgr", - "version": "3.0.0.70101-38~22.04" + "version": "3.0.0.70200-43~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -2364,9 +2345,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70101-38_22.04_amd64", + "key": "rocminfo_1.0.0.70200-43_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.70101-38~22.04" + "version": "1.0.0.70200-43~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -2394,62 +2375,42 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-rocr_1.18.0.70101-38_22.04_amd64", - "name": "hsa-rocr", - "version": "1.18.0.70101-38~22.04" + "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", + "name": "hsa-runtime-rocr4wsl-amdgpu", + "version": "25.30.13-2281980.22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", "name": "libelf1", "version": "0.186-1ubuntu0.1" - }, - { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70101-2255337.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.125.70101-2255337.22.04" - }, - { - "key": "libdrm-amdgpu-common_1.0.0.70101-2255337.22.04_amd64", - "name": "libdrm-amdgpu-common", - "version": "1.0.0.70101-2255337.22.04" - }, - { - "key": "amdgpu-core_1-7.1.70101-2255337.22.04_amd64", - "name": "amdgpu-core", - "version": "1:7.1.70101-2255337.22.04" - }, - { - "key": "libdrm2-amdgpu_1-2.4.125.70101-2255337.22.04_amd64", - "name": "libdrm2-amdgpu", - "version": "1:2.4.125.70101-2255337.22.04" } ], - "key": "miopen-hip_3.5.1.70101-38_22.04_amd64", + "key": "miopen-hip_3.5.1.70200-43_22.04_amd64", "name": "miopen-hip", - "sha256": "0ec588284420c27a7b2ef1e0c2578d669a9455a5ee14aa6917675cb765c47f35", + "sha256": "c9b6ea5b4f07aa6ee21f95f69ac3dd88ef87449e52c41d751b01b071ab121656", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/m/miopen-hip/miopen-hip_3.5.1.70101-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/m/miopen-hip/miopen-hip_3.5.1.70200-43~22.04_amd64.deb" ], - "version": "3.5.1.70101-38~22.04" + "version": "3.5.1.70200-43~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocrand_4.1.0.70101-38_22.04_amd64", + "key": "rocrand_4.2.0.70200-43_22.04_amd64", "name": "rocrand", - "sha256": "0c8ed29a197a1444cf11bd52cc20dcf981ea0310e0db83f404eec14edd9d0520", + "sha256": "2b22a1cc2861eff622c6756b7cba160fb0b9211cbd9452b89b7d3ab3ce19e6c8", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocrand/rocrand_4.1.0.70101-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocrand/rocrand_4.2.0.70200-43~22.04_amd64.deb" ], - "version": "4.1.0.70101-38~22.04" + "version": "4.2.0.70200-43~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", "name": "rocm-core", - "version": "7.1.1.70101-38~22.04" + "version": "7.2.0.70200-43~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -2672,26 +2633,26 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "roctracer_4.1.70101.70101-38_22.04_amd64", + "key": "roctracer_4.1.70200.70200-43_22.04_amd64", "name": "roctracer", - "version": "4.1.70101.70101-38~22.04" + "version": "4.1.70200.70200-43~22.04" } ], - "key": "hipblaslt_1.1.0.70101-38_22.04_amd64", + "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", "name": "hipblaslt", - "sha256": "c2689d9a1ec1a236ac6545cc0b60ca99704e5ebfc6d536a0f2a28f061817fd7f", + "sha256": "a7baff52908f4a25b134d7e2613c1eb2b51b369509465d2a502b457f3ce3c1b2", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/h/hipblaslt/hipblaslt_1.1.0.70101-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hipblaslt/hipblaslt_1.2.1.70200-43~22.04_amd64.deb" ], - "version": "1.1.0.70101-38~22.04" + "version": "1.2.1.70200-43~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", "name": "rocm-core", - "version": "7.1.1.70101-38~22.04" + "version": "7.2.0.70200-43~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -2914,21 +2875,21 @@ "version": "3.10.6-1~22.04.1" } ], - "key": "roctracer_4.1.70101.70101-38_22.04_amd64", + "key": "roctracer_4.1.70200.70200-43_22.04_amd64", "name": "roctracer", - "sha256": "621441c073c1a85ce22a9d9df9b6b2f40cbd0f4ecb1c6647eef1db57df02b6b2", + "sha256": "4b40ae9c1981979f3fb69e9713c5b9fc1a17f868a83732f38f8fb5e53be3a612", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/roctracer/roctracer_4.1.70101.70101-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/roctracer/roctracer_4.1.70200.70200-43~22.04_amd64.deb" ], - "version": "4.1.70101.70101-38~22.04" + "version": "4.1.70200.70200-43~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", "name": "rocm-core", - "version": "7.1.1.70101-38~22.04" + "version": "7.2.0.70200-43~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -3151,9 +3112,9 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "hip-runtime-amd_7.1.52802.70101-38_22.04_amd64", + "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", "name": "hip-runtime-amd", - "version": "7.1.52802.70101-38~22.04" + "version": "7.2.26015.70200-43~22.04" }, { "key": "libnuma1_2.0.14-3ubuntu2_amd64", @@ -3161,14 +3122,14 @@ "version": "2.0.14-3ubuntu2" }, { - "key": "rocprofiler-register_0.6.0.70101-38_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70101-38~22.04" + "version": "0.6.0.70200-43~22.04" }, { - "key": "comgr_3.0.0.70101-38_22.04_amd64", + "key": "comgr_3.0.0.70200-43_22.04_amd64", "name": "comgr", - "version": "3.0.0.70101-38~22.04" + "version": "3.0.0.70200-43~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -3226,9 +3187,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70101-38_22.04_amd64", + "key": "rocminfo_1.0.0.70200-43_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.70101-38~22.04" + "version": "1.0.0.70200-43~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -3256,9 +3217,9 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-rocr_1.18.0.70101-38_22.04_amd64", - "name": "hsa-rocr", - "version": "1.18.0.70101-38~22.04" + "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", + "name": "hsa-runtime-rocr4wsl-amdgpu", + "version": "25.30.13-2281980.22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", @@ -3266,43 +3227,23 @@ "version": "0.186-1ubuntu0.1" }, { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70101-2255337.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.125.70101-2255337.22.04" - }, - { - "key": "libdrm-amdgpu-common_1.0.0.70101-2255337.22.04_amd64", - "name": "libdrm-amdgpu-common", - "version": "1.0.0.70101-2255337.22.04" - }, - { - "key": "amdgpu-core_1-7.1.70101-2255337.22.04_amd64", - "name": "amdgpu-core", - "version": "1:7.1.70101-2255337.22.04" - }, - { - "key": "libdrm2-amdgpu_1-2.4.125.70101-2255337.22.04_amd64", - "name": "libdrm2-amdgpu", - "version": "1:2.4.125.70101-2255337.22.04" - }, - { - "key": "hipblaslt_1.1.0.70101-38_22.04_amd64", + "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", "name": "hipblaslt", - "version": "1.1.0.70101-38~22.04" + "version": "1.2.1.70200-43~22.04" }, { - "key": "roctracer_4.1.70101.70101-38_22.04_amd64", + "key": "roctracer_4.1.70200.70200-43_22.04_amd64", "name": "roctracer", - "version": "4.1.70101.70101-38~22.04" + "version": "4.1.70200.70200-43~22.04" } ], - "key": "rocblas_5.1.1.70101-38_22.04_amd64", + "key": "rocblas_5.2.0.70200-43_22.04_amd64", "name": "rocblas", - "sha256": "46c414775f070518afba3326611fd577059a5bb277182356a0bae720e13086a7", + "sha256": "0581ada80350b0d959e1863af4ff4be3ef31fe9838e941d09b8f0693dda7d23d", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocblas/rocblas_5.1.1.70101-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocblas/rocblas_5.2.0.70200-43~22.04_amd64.deb" ], - "version": "5.1.1.70101-38~22.04" + "version": "5.2.0.70200-43~22.04" }, { "arch": "amd64", @@ -3338,14 +3279,14 @@ "version": "2.0.14-3ubuntu2" }, { - "key": "rocprofiler-register_0.6.0.70101-38_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70101-38~22.04" + "version": "0.6.0.70200-43~22.04" }, { - "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", "name": "rocm-core", - "version": "7.1.1.70101-38~22.04" + "version": "7.2.0.70200-43~22.04" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -3543,9 +3484,9 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "comgr_3.0.0.70101-38_22.04_amd64", + "key": "comgr_3.0.0.70200-43_22.04_amd64", "name": "comgr", - "version": "3.0.0.70101-38~22.04" + "version": "3.0.0.70200-43~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -3603,9 +3544,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70101-38_22.04_amd64", + "key": "rocminfo_1.0.0.70200-43_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.70101-38~22.04" + "version": "1.0.0.70200-43~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -3633,54 +3574,34 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-rocr_1.18.0.70101-38_22.04_amd64", - "name": "hsa-rocr", - "version": "1.18.0.70101-38~22.04" + "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", + "name": "hsa-runtime-rocr4wsl-amdgpu", + "version": "25.30.13-2281980.22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", "name": "libelf1", "version": "0.186-1ubuntu0.1" - }, - { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70101-2255337.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.125.70101-2255337.22.04" - }, - { - "key": "libdrm-amdgpu-common_1.0.0.70101-2255337.22.04_amd64", - "name": "libdrm-amdgpu-common", - "version": "1.0.0.70101-2255337.22.04" - }, - { - "key": "amdgpu-core_1-7.1.70101-2255337.22.04_amd64", - "name": "amdgpu-core", - "version": "1:7.1.70101-2255337.22.04" - }, - { - "key": "libdrm2-amdgpu_1-2.4.125.70101-2255337.22.04_amd64", - "name": "libdrm2-amdgpu", - "version": "1:2.4.125.70101-2255337.22.04" } ], - "key": "hip-runtime-amd_7.1.52802.70101-38_22.04_amd64", + "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", "name": "hip-runtime-amd", - "sha256": "5f4e042d2cc08c1acc40a710e619dc6fc4e98ff1285329d0a2981b4bbcd6436c", + "sha256": "97f477244025fb3bd8762d8d730d18545cafe080cb938f3f6520aeaff6e05c42", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/h/hip-runtime-amd/hip-runtime-amd_7.1.52802.70101-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hip-runtime-amd/hip-runtime-amd_7.2.26015.70200-43~22.04_amd64.deb" ], - "version": "7.1.52802.70101-38~22.04" + "version": "7.2.26015.70200-43~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocminfo_1.0.0.70101-38_22.04_amd64", + "key": "rocminfo_1.0.0.70200-43_22.04_amd64", "name": "rocminfo", - "sha256": "996f635e8bb55e6ea37e4910c44dffd461ffa6ee5189c4bdb57906bc57dcbd0e", + "sha256": "6b8bcd3f5d0589a3b6447353c140b4ef676c0fbe319050a0f584fde4aa83f16f", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocminfo/rocminfo_1.0.0.70101-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocminfo/rocminfo_1.0.0.70200-43~22.04_amd64.deb" ], - "version": "1.0.0.70101-38~22.04" + "version": "1.0.0.70200-43~22.04" }, { "arch": "amd64", @@ -3766,9 +3687,9 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", "name": "rocm-core", - "version": "7.1.1.70101-38~22.04" + "version": "7.2.0.70200-43~22.04" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -3966,14 +3887,14 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "rocm-smi-lib_7.8.0.70101-38_22.04_amd64", + "key": "rocm-smi-lib_7.8.0.70200-43_22.04_amd64", "name": "rocm-smi-lib", - "version": "7.8.0.70101-38~22.04" + "version": "7.8.0.70200-43~22.04" }, { - "key": "hip-runtime-amd_7.1.52802.70101-38_22.04_amd64", + "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", "name": "hip-runtime-amd", - "version": "7.1.52802.70101-38~22.04" + "version": "7.2.26015.70200-43~22.04" }, { "key": "libnuma1_2.0.14-3ubuntu2_amd64", @@ -3981,14 +3902,14 @@ "version": "2.0.14-3ubuntu2" }, { - "key": "rocprofiler-register_0.6.0.70101-38_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70101-38~22.04" + "version": "0.6.0.70200-43~22.04" }, { - "key": "comgr_3.0.0.70101-38_22.04_amd64", + "key": "comgr_3.0.0.70200-43_22.04_amd64", "name": "comgr", - "version": "3.0.0.70101-38~22.04" + "version": "3.0.0.70200-43~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -4046,9 +3967,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70101-38_22.04_amd64", + "key": "rocminfo_1.0.0.70200-43_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.70101-38~22.04" + "version": "1.0.0.70200-43~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -4076,62 +3997,42 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-rocr_1.18.0.70101-38_22.04_amd64", - "name": "hsa-rocr", - "version": "1.18.0.70101-38~22.04" + "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", + "name": "hsa-runtime-rocr4wsl-amdgpu", + "version": "25.30.13-2281980.22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", "name": "libelf1", "version": "0.186-1ubuntu0.1" - }, - { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70101-2255337.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.125.70101-2255337.22.04" - }, - { - "key": "libdrm-amdgpu-common_1.0.0.70101-2255337.22.04_amd64", - "name": "libdrm-amdgpu-common", - "version": "1.0.0.70101-2255337.22.04" - }, - { - "key": "amdgpu-core_1-7.1.70101-2255337.22.04_amd64", - "name": "amdgpu-core", - "version": "1:7.1.70101-2255337.22.04" - }, - { - "key": "libdrm2-amdgpu_1-2.4.125.70101-2255337.22.04_amd64", - "name": "libdrm2-amdgpu", - "version": "1:2.4.125.70101-2255337.22.04" } ], - "key": "rccl_2.27.7.70101-38_22.04_amd64", + "key": "rccl_2.27.7.70200-43_22.04_amd64", "name": "rccl", - "sha256": "4fa485a696d6450e528c81c1f9dc0b8afba6f464ab43f6560b8594906c2de5f6", + "sha256": "bb450a46ee36578a2b171b9af891f5e94ef04e9f743f75ec16e99246ddaff878", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rccl/rccl_2.27.7.70101-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rccl/rccl_2.27.7.70200-43~22.04_amd64.deb" ], - "version": "2.27.7.70101-38~22.04" + "version": "2.27.7.70200-43~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocm-smi-lib_7.8.0.70101-38_22.04_amd64", + "key": "rocm-smi-lib_7.8.0.70200-43_22.04_amd64", "name": "rocm-smi-lib", - "sha256": "0daa96b94592fc1c6886115b7ec0d530ad53bc2fe5b5bb4ece05915ef8db479c", + "sha256": "bfd4ef1c3382eab7685c92c218ef14593ea1bd06836bb8cc903e4100e9125d79", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocm-smi-lib/rocm-smi-lib_7.8.0.70101-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocm-smi-lib/rocm-smi-lib_7.8.0.70200-43~22.04_amd64.deb" ], - "version": "7.8.0.70101-38~22.04" + "version": "7.8.0.70200-43~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", "name": "rocm-core", - "version": "7.1.1.70101-38~22.04" + "version": "7.2.0.70200-43~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -4354,31 +4255,31 @@ "version": "3.10.6-1~22.04.1" } ], - "key": "rocm-device-libs_1.0.0.70101-38_22.04_amd64", + "key": "rocm-device-libs_1.0.0.70200-43_22.04_amd64", "name": "rocm-device-libs", - "sha256": "e042075ac40c261efe7bcff48ad784d8ea5f3b701dfb1692320cd93fa1805df0", + "sha256": "f1fe8f8fea614c3f734d002f6ae688929156bcfde0f27457412a7c3074d18d51", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocm-device-libs/rocm-device-libs_1.0.0.70101-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocm-device-libs/rocm-device-libs_1.0.0.70200-43~22.04_amd64.deb" ], - "version": "1.0.0.70101-38~22.04" + "version": "1.0.0.70200-43~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "hipcc_1.1.1.70101-38_22.04_amd64", + "key": "hipcc_1.1.1.70200-43_22.04_amd64", "name": "hipcc", - "version": "1.1.1.70101-38~22.04" + "version": "1.1.1.70200-43~22.04" }, { - "key": "rocm-llvm_20.0.0.25444.70101-38_22.04_amd64", + "key": "rocm-llvm_22.0.0.26014.70200-43_22.04_amd64", "name": "rocm-llvm", - "version": "20.0.0.25444.70101-38~22.04" + "version": "22.0.0.26014.70200-43~22.04" }, { - "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", "name": "rocm-core", - "version": "7.1.1.70101-38~22.04" + "version": "7.2.0.70200-43~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -4691,84 +4592,69 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "hsa-rocr-dev_1.18.0.70101-38_22.04_amd64", + "key": "hsa-rocr-dev_1.18.0.70200-43_22.04_amd64", "name": "hsa-rocr-dev", - "version": "1.18.0.70101-38~22.04" + "version": "1.18.0.70200-43~22.04" }, { - "key": "hsa-rocr_1.18.0.70101-38_22.04_amd64", - "name": "hsa-rocr", - "version": "1.18.0.70101-38~22.04" - }, - { - "key": "rocprofiler-register_0.6.0.70101-38_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.6.0.70101-38~22.04" + "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", + "name": "hsa-runtime-rocr4wsl-amdgpu", + "version": "25.30.13-2281980.22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", "name": "libelf1", "version": "0.186-1ubuntu0.1" }, + { + "key": "libdrm-amdgpu-dev_1-2.4.125.70200-2278374.22.04_amd64", + "name": "libdrm-amdgpu-dev", + "version": "1:2.4.125.70200-2278374.22.04" + }, + { + "key": "libdrm-amdgpu-radeon1_1-2.4.125.70200-2278374.22.04_amd64", + "name": "libdrm-amdgpu-radeon1", + "version": "1:2.4.125.70200-2278374.22.04" + }, + { + "key": "libdrm2-amdgpu_1-2.4.125.70200-2278374.22.04_amd64", + "name": "libdrm2-amdgpu", + "version": "1:2.4.125.70200-2278374.22.04" + }, + { + "key": "amdgpu-core_1-7.2.70200-2278374.22.04_amd64", + "name": "amdgpu-core", + "version": "1:7.2.70200-2278374.22.04" + }, + { + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70200-2278374.22.04_amd64", + "name": "libdrm-amdgpu-amdgpu1", + "version": "1:2.4.125.70200-2278374.22.04" + }, + { + "key": "libdrm-amdgpu-common_1.0.0.70200-2278374.22.04_amd64", + "name": "libdrm-amdgpu-common", + "version": "1.0.0.70200-2278374.22.04" + }, + { + "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", + "name": "hip-runtime-amd", + "version": "7.2.26015.70200-43~22.04" + }, { "key": "libnuma1_2.0.14-3ubuntu2_amd64", "name": "libnuma1", "version": "2.0.14-3ubuntu2" }, { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70101-2255337.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.125.70101-2255337.22.04" + "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70200-43~22.04" }, { - "key": "libdrm-amdgpu-common_1.0.0.70101-2255337.22.04_amd64", - "name": "libdrm-amdgpu-common", - "version": "1.0.0.70101-2255337.22.04" - }, - { - "key": "amdgpu-core_1-7.1.70101-2255337.22.04_amd64", - "name": "amdgpu-core", - "version": "1:7.1.70101-2255337.22.04" - }, - { - "key": "libdrm2-amdgpu_1-2.4.125.70101-2255337.22.04_amd64", - "name": "libdrm2-amdgpu", - "version": "1:2.4.125.70101-2255337.22.04" - }, - { - "key": "libdrm-amdgpu-dev_1-2.4.125.70101-2255337.22.04_amd64", - "name": "libdrm-amdgpu-dev", - "version": "1:2.4.125.70101-2255337.22.04" - }, - { - "key": "valgrind_1-3.18.1-1ubuntu2_amd64", - "name": "valgrind", - "version": "1:3.18.1-1ubuntu2" - }, - { - "key": "libc6-dbg_2.35-0ubuntu3.10_amd64", - "name": "libc6-dbg", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libc6-i386_2.35-0ubuntu3.10_amd64", - "name": "libc6-i386", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libdrm-amdgpu-radeon1_1-2.4.125.70101-2255337.22.04_amd64", - "name": "libdrm-amdgpu-radeon1", - "version": "1:2.4.125.70101-2255337.22.04" - }, - { - "key": "hip-runtime-amd_7.1.52802.70101-38_22.04_amd64", - "name": "hip-runtime-amd", - "version": "7.1.52802.70101-38~22.04" - }, - { - "key": "comgr_3.0.0.70101-38_22.04_amd64", + "key": "comgr_3.0.0.70200-43_22.04_amd64", "name": "comgr", - "version": "3.0.0.70101-38~22.04" + "version": "3.0.0.70200-43~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -4791,9 +4677,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70101-38_22.04_amd64", + "key": "rocminfo_1.0.0.70200-43_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.70101-38~22.04" + "version": "1.0.0.70200-43~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -4836,35 +4722,35 @@ "version": "1:5.41-3ubuntu0.1" } ], - "key": "hip-dev_7.1.52802.70101-38_22.04_amd64", + "key": "hip-dev_7.2.26015.70200-43_22.04_amd64", "name": "hip-dev", - "sha256": "9f33ea2f9c62bad88cd2942f15a82eb59082ce6604feede4b662fc12f13042cf", + "sha256": "19d62d6f520b4e521fe90246d9ae872b30528e490f80c6e265f83a583302f0ad", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/h/hip-dev/hip-dev_7.1.52802.70101-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hip-dev/hip-dev_7.2.26015.70200-43~22.04_amd64.deb" ], - "version": "7.1.52802.70101-38~22.04" + "version": "7.2.26015.70200-43~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "hipcc_1.1.1.70101-38_22.04_amd64", + "key": "hipcc_1.1.1.70200-43_22.04_amd64", "name": "hipcc", - "sha256": "49355c960eef059dcbcc12134ac6b621505f429b6a9d20ac0902dd9546963593", + "sha256": "3c037af2114bd9058fb1420c00586c5d1c01a63d1d34500d8e61b39802ac570b", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/h/hipcc/hipcc_1.1.1.70101-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hipcc/hipcc_1.1.1.70200-43~22.04_amd64.deb" ], - "version": "1.1.1.70101-38~22.04" + "version": "1.1.1.70200-43~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocm-llvm_20.0.0.25444.70101-38_22.04_amd64", + "key": "rocm-llvm_22.0.0.26014.70200-43_22.04_amd64", "name": "rocm-llvm", - "sha256": "23a2682b3a1e718c04ca04b39ed10f52fc8a3b524b12e1de2245ad7120988e6a", + "sha256": "88d604fe0ab4a2502f0c7e0f8d3b7b49e829ae3057971de681a68b2fdb2505d2", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocm-llvm/rocm-llvm_20.0.0.25444.70101-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocm-llvm/rocm-llvm_22.0.0.26014.70200-43~22.04_amd64.deb" ], - "version": "20.0.0.25444.70101-38~22.04" + "version": "22.0.0.26014.70200-43~22.04" }, { "arch": "amd64", @@ -4990,68 +4876,35 @@ { "arch": "amd64", "dependencies": [], - "key": "hsa-rocr-dev_1.18.0.70101-38_22.04_amd64", + "key": "hsa-rocr-dev_1.18.0.70200-43_22.04_amd64", "name": "hsa-rocr-dev", - "sha256": "92135d3d9775fc07a446641272e39f9022830afb9cb35042b7ed3cfd4bd504e4", + "sha256": "71233ffaac8f28550c35bcf3e287a377a3c5c938a1e815e65d4155139cf4bdff", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/h/hsa-rocr-dev/hsa-rocr-dev_1.18.0.70101-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hsa-rocr-dev/hsa-rocr-dev_1.18.0.70200-43~22.04_amd64.deb" ], - "version": "1.18.0.70101-38~22.04" + "version": "1.18.0.70200-43~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "libdrm-amdgpu-dev_1-2.4.125.70101-2255337.22.04_amd64", + "key": "libdrm-amdgpu-dev_1-2.4.125.70200-2278374.22.04_amd64", "name": "libdrm-amdgpu-dev", - "sha256": "af20b1033129e0797efc2fcc071cf213f2739483674b1a1a312d31aad3166aef", + "sha256": "fec583602169ae32d3c9313dcb24c00732911234712495cbb6bb10f2194d2aee", "urls": [ - "https://repo.radeon.com/graphics/7.1.1/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-dev_2.4.125.70101-2255337.22.04_amd64.deb" + "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-dev_2.4.125.70200-2278374.22.04_amd64.deb" ], - "version": "1:2.4.125.70101-2255337.22.04" + "version": "1:2.4.125.70200-2278374.22.04" }, { "arch": "amd64", "dependencies": [], - "key": "valgrind_1-3.18.1-1ubuntu2_amd64", - "name": "valgrind", - "sha256": "8376c1524d187246eb09b15531c83d83567bdbc609880622a05bc8126558a221", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/v/valgrind/valgrind_3.18.1-1ubuntu2_amd64.deb" - ], - "version": "1:3.18.1-1ubuntu2" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libc6-dbg_2.35-0ubuntu3.10_amd64", - "name": "libc6-dbg", - "sha256": "a69e6e8bf94a714b3a4686bb6161d0ce6fb1fc863e97c82b40dae83a650f0ade", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc6-dbg_2.35-0ubuntu3.10_amd64.deb" - ], - "version": "2.35-0ubuntu3.10" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libc6-i386_2.35-0ubuntu3.10_amd64", - "name": "libc6-i386", - "sha256": "1a4822354c437e51d47cde7b3db268b3d68ccdc6c2dd340926cdc31c5d8f5e73", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc6-i386_2.35-0ubuntu3.10_amd64.deb" - ], - "version": "2.35-0ubuntu3.10" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libdrm-amdgpu-radeon1_1-2.4.125.70101-2255337.22.04_amd64", + "key": "libdrm-amdgpu-radeon1_1-2.4.125.70200-2278374.22.04_amd64", "name": "libdrm-amdgpu-radeon1", - "sha256": "d864ae2ec21b164d20b48807f9343ee6e57a4b937e7c218b96099479af53e643", + "sha256": "70049c919cff1b728107a03cd893b69106de8c00ac344e2d4019922d6bdbcd8b", "urls": [ - "https://repo.radeon.com/graphics/7.1.1/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-radeon1_2.4.125.70101-2255337.22.04_amd64.deb" + "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-radeon1_2.4.125.70200-2278374.22.04_amd64.deb" ], - "version": "1:2.4.125.70101-2255337.22.04" + "version": "1:2.4.125.70200-2278374.22.04" }, { "arch": "amd64", @@ -5090,9 +4943,9 @@ "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", "name": "rocm-core", - "version": "7.1.1.70101-38~22.04" + "version": "7.2.0.70200-43~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -5315,14 +5168,14 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "rocblas_5.1.1.70101-38_22.04_amd64", + "key": "rocblas_5.2.0.70200-43_22.04_amd64", "name": "rocblas", - "version": "5.1.1.70101-38~22.04" + "version": "5.2.0.70200-43~22.04" }, { - "key": "hip-runtime-amd_7.1.52802.70101-38_22.04_amd64", + "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", "name": "hip-runtime-amd", - "version": "7.1.52802.70101-38~22.04" + "version": "7.2.26015.70200-43~22.04" }, { "key": "libnuma1_2.0.14-3ubuntu2_amd64", @@ -5330,14 +5183,14 @@ "version": "2.0.14-3ubuntu2" }, { - "key": "rocprofiler-register_0.6.0.70101-38_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70101-38~22.04" + "version": "0.6.0.70200-43~22.04" }, { - "key": "comgr_3.0.0.70101-38_22.04_amd64", + "key": "comgr_3.0.0.70200-43_22.04_amd64", "name": "comgr", - "version": "3.0.0.70101-38~22.04" + "version": "3.0.0.70200-43~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -5395,9 +5248,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70101-38_22.04_amd64", + "key": "rocminfo_1.0.0.70200-43_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.70101-38~22.04" + "version": "1.0.0.70200-43~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -5425,9 +5278,9 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-rocr_1.18.0.70101-38_22.04_amd64", - "name": "hsa-rocr", - "version": "1.18.0.70101-38~22.04" + "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", + "name": "hsa-runtime-rocr4wsl-amdgpu", + "version": "25.30.13-2281980.22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", @@ -5435,51 +5288,31 @@ "version": "0.186-1ubuntu0.1" }, { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70101-2255337.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.125.70101-2255337.22.04" - }, - { - "key": "libdrm-amdgpu-common_1.0.0.70101-2255337.22.04_amd64", - "name": "libdrm-amdgpu-common", - "version": "1.0.0.70101-2255337.22.04" - }, - { - "key": "amdgpu-core_1-7.1.70101-2255337.22.04_amd64", - "name": "amdgpu-core", - "version": "1:7.1.70101-2255337.22.04" - }, - { - "key": "libdrm2-amdgpu_1-2.4.125.70101-2255337.22.04_amd64", - "name": "libdrm2-amdgpu", - "version": "1:2.4.125.70101-2255337.22.04" - }, - { - "key": "hipblaslt_1.1.0.70101-38_22.04_amd64", + "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", "name": "hipblaslt", - "version": "1.1.0.70101-38~22.04" + "version": "1.2.1.70200-43~22.04" }, { - "key": "roctracer_4.1.70101.70101-38_22.04_amd64", + "key": "roctracer_4.1.70200.70200-43_22.04_amd64", "name": "roctracer", - "version": "4.1.70101.70101-38~22.04" + "version": "4.1.70200.70200-43~22.04" } ], - "key": "rocsolver_3.31.0.70101-38_22.04_amd64", + "key": "rocsolver_3.32.0.70200-43_22.04_amd64", "name": "rocsolver", - "sha256": "0fdf9d402336e592f80d41358eb324b21b08ae9b8b6e1a312bcd2e0da0fdf8da", + "sha256": "f18b57d3ebeba075b2b91f9e71d92da3efba73f34c5cff697cea1259b829547a", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocsolver/rocsolver_3.31.0.70101-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocsolver/rocsolver_3.32.0.70200-43~22.04_amd64.deb" ], - "version": "3.31.0.70101-38~22.04" + "version": "3.32.0.70200-43~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", "name": "rocm-core", - "version": "7.1.1.70101-38~22.04" + "version": "7.2.0.70200-43~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -5702,19 +5535,19 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "rocsolver_3.31.0.70101-38_22.04_amd64", + "key": "rocsolver_3.32.0.70200-43_22.04_amd64", "name": "rocsolver", - "version": "3.31.0.70101-38~22.04" + "version": "3.32.0.70200-43~22.04" }, { - "key": "rocblas_5.1.1.70101-38_22.04_amd64", + "key": "rocblas_5.2.0.70200-43_22.04_amd64", "name": "rocblas", - "version": "5.1.1.70101-38~22.04" + "version": "5.2.0.70200-43~22.04" }, { - "key": "hip-runtime-amd_7.1.52802.70101-38_22.04_amd64", + "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", "name": "hip-runtime-amd", - "version": "7.1.52802.70101-38~22.04" + "version": "7.2.26015.70200-43~22.04" }, { "key": "libnuma1_2.0.14-3ubuntu2_amd64", @@ -5722,14 +5555,14 @@ "version": "2.0.14-3ubuntu2" }, { - "key": "rocprofiler-register_0.6.0.70101-38_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70101-38~22.04" + "version": "0.6.0.70200-43~22.04" }, { - "key": "comgr_3.0.0.70101-38_22.04_amd64", + "key": "comgr_3.0.0.70200-43_22.04_amd64", "name": "comgr", - "version": "3.0.0.70101-38~22.04" + "version": "3.0.0.70200-43~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -5787,9 +5620,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70101-38_22.04_amd64", + "key": "rocminfo_1.0.0.70200-43_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.70101-38~22.04" + "version": "1.0.0.70200-43~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -5817,9 +5650,9 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-rocr_1.18.0.70101-38_22.04_amd64", - "name": "hsa-rocr", - "version": "1.18.0.70101-38~22.04" + "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", + "name": "hsa-runtime-rocr4wsl-amdgpu", + "version": "25.30.13-2281980.22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", @@ -5827,51 +5660,31 @@ "version": "0.186-1ubuntu0.1" }, { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70101-2255337.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.125.70101-2255337.22.04" - }, - { - "key": "libdrm-amdgpu-common_1.0.0.70101-2255337.22.04_amd64", - "name": "libdrm-amdgpu-common", - "version": "1.0.0.70101-2255337.22.04" - }, - { - "key": "amdgpu-core_1-7.1.70101-2255337.22.04_amd64", - "name": "amdgpu-core", - "version": "1:7.1.70101-2255337.22.04" - }, - { - "key": "libdrm2-amdgpu_1-2.4.125.70101-2255337.22.04_amd64", - "name": "libdrm2-amdgpu", - "version": "1:2.4.125.70101-2255337.22.04" - }, - { - "key": "hipblaslt_1.1.0.70101-38_22.04_amd64", + "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", "name": "hipblaslt", - "version": "1.1.0.70101-38~22.04" + "version": "1.2.1.70200-43~22.04" }, { - "key": "roctracer_4.1.70101.70101-38_22.04_amd64", + "key": "roctracer_4.1.70200.70200-43_22.04_amd64", "name": "roctracer", - "version": "4.1.70101.70101-38~22.04" + "version": "4.1.70200.70200-43~22.04" } ], - "key": "hipsolver_3.1.0.70101-38_22.04_amd64", + "key": "hipsolver_3.2.0.70200-43_22.04_amd64", "name": "hipsolver", - "sha256": "1ebd0afa7bb9b52f341755603fc432ab1ede8218c014b1df1bff03d7a6d11c0a", + "sha256": "0fbff8007eada3d952db641c0653df26854854ee9e7058b4bd63202db66f2af8", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/h/hipsolver/hipsolver_3.1.0.70101-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hipsolver/hipsolver_3.2.0.70200-43~22.04_amd64.deb" ], - "version": "3.1.0.70101-38~22.04" + "version": "3.2.0.70200-43~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", "name": "rocm-core", - "version": "7.1.1.70101-38~22.04" + "version": "7.2.0.70200-43~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -6094,42 +5907,42 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "rocfft_1.0.35.70101-38_22.04_amd64", + "key": "rocfft_1.0.36.70200-43_22.04_amd64", "name": "rocfft", - "version": "1.0.35.70101-38~22.04" + "version": "1.0.36.70200-43~22.04" } ], - "key": "hipfft_1.0.21.70101-38_22.04_amd64", + "key": "hipfft_1.0.22.70200-43_22.04_amd64", "name": "hipfft", - "sha256": "14adaf6995d9b26295d5ae20d6eb3d76357521294349796f88bb49becbb7717a", + "sha256": "55a791b56dbeadcc3a1f1d0d770afc71a7fbb06aed4cd190b75ee27c19c18c61", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/h/hipfft/hipfft_1.0.21.70101-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hipfft/hipfft_1.0.22.70200-43~22.04_amd64.deb" ], - "version": "1.0.21.70101-38~22.04" + "version": "1.0.22.70200-43~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocfft_1.0.35.70101-38_22.04_amd64", + "key": "rocfft_1.0.36.70200-43_22.04_amd64", "name": "rocfft", - "sha256": "c9cb19ce041ab9694355cf6fc3e58514c6ccfdbd1af409829e03aaa7093555cb", + "sha256": "8dbfef90c0055ef0d572451cb4a7a3380c3394485ed33cab4c116283d2fea7fe", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/r/rocfft/rocfft_1.0.35.70101-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocfft/rocfft_1.0.36.70200-43~22.04_amd64.deb" ], - "version": "1.0.35.70101-38~22.04" + "version": "1.0.36.70200-43~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "hipblaslt_1.1.0.70101-38_22.04_amd64", + "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", "name": "hipblaslt", - "version": "1.1.0.70101-38~22.04" + "version": "1.2.1.70200-43~22.04" }, { - "key": "rocm-core_7.1.1.70101-38_22.04_amd64", + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", "name": "rocm-core", - "version": "7.1.1.70101-38~22.04" + "version": "7.2.0.70200-43~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -6352,34 +6165,34 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "roctracer_4.1.70101.70101-38_22.04_amd64", + "key": "roctracer_4.1.70200.70200-43_22.04_amd64", "name": "roctracer", - "version": "4.1.70101.70101-38~22.04" + "version": "4.1.70200.70200-43~22.04" }, { - "key": "hipblas-common-dev_1.3.0.70101-38_22.04_amd64", + "key": "hipblas-common-dev_1.4.0.70200-43_22.04_amd64", "name": "hipblas-common-dev", - "version": "1.3.0.70101-38~22.04" + "version": "1.4.0.70200-43~22.04" } ], - "key": "hipblaslt-dev_1.1.0.70101-38_22.04_amd64", + "key": "hipblaslt-dev_1.2.1.70200-43_22.04_amd64", "name": "hipblaslt-dev", - "sha256": "a54d9203412a4b2ecfe446c23358d87d2446bab4c49b874706ac8ec420321a47", + "sha256": "7bb01f62e51be15495ea4a124db7b6c4e600d27d401ce636fb9b3daaf6b4b958", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/h/hipblaslt-dev/hipblaslt-dev_1.1.0.70101-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hipblaslt-dev/hipblaslt-dev_1.2.1.70200-43~22.04_amd64.deb" ], - "version": "1.1.0.70101-38~22.04" + "version": "1.2.1.70200-43~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "hipblas-common-dev_1.3.0.70101-38_22.04_amd64", + "key": "hipblas-common-dev_1.4.0.70200-43_22.04_amd64", "name": "hipblas-common-dev", - "sha256": "e6a3e7dd89519d3a0ce0875a722d74e4ec125264c99195231f27b9a59a3c2bed", + "sha256": "1eed050f8367b58d697db452c3928780ae408ee055694f1676326d54f4caa739", "urls": [ - "https://repo.radeon.com/rocm/apt/7.1.1/pool/main/h/hipblas-common-dev/hipblas-common-dev_1.3.0.70101-38~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hipblas-common-dev/hipblas-common-dev_1.4.0.70200-43~22.04_amd64.deb" ], - "version": "1.3.0.70101-38~22.04" + "version": "1.4.0.70200-43~22.04" } ], "version": 1 diff --git a/platforms/rocm/packages.yaml b/platforms/rocm/packages.yaml index 19b45fca..b9b96902 100644 --- a/platforms/rocm/packages.yaml +++ b/platforms/rocm/packages.yaml @@ -4,39 +4,39 @@ version: 1 sources: - - channel: jammy main - url: https://repo.radeon.com/rocm/apt/7.1.1 - - channel: jammy main - url: https://repo.radeon.com/graphics/7.1.1/ubuntu - - channel: jammy main - url: https://snapshot.ubuntu.com/ubuntu/20250711T030400Z - - channel: jammy-security main - url: https://snapshot.ubuntu.com/ubuntu/20250711T030400Z - - channel: jammy-updates main - url: https://snapshot.ubuntu.com/ubuntu/20250711T030400Z + - channel: jammy main + url: https://repo.radeon.com/rocm/apt/7.2 + - channel: jammy main + url: https://repo.radeon.com/graphics/7.2/ubuntu + - channel: jammy main + url: https://snapshot.ubuntu.com/ubuntu/20250711T030400Z + - channel: jammy-security main + url: https://snapshot.ubuntu.com/ubuntu/20250711T030400Z + - channel: jammy-updates main + url: https://snapshot.ubuntu.com/ubuntu/20250711T030400Z archs: - - "amd64" + - "amd64" -# readelf -d libpjrt_rosm.so | grep NEEDED +# readelf -d libpjrt_rocm.so | grep NEEDED packages: - # - "rocm-smi-lib" - - "libdrm-amdgpu-common" - # Do not use hsa-rocr from amdgpu repo because it is overriden by hsa-runtime-rocr4wsl-amdgpu - - "hsa-rocr (<< 20)" - - "hsa-amd-aqlprofile" - - "comgr" - - "rocprofiler-register" - - "rocprofiler-sdk" - - "miopen-hip" - - "rccl" - - "rocm-device-libs" - - "hip-dev" - - "rocblas" - - "rocsolver" - - "hipsolver" - - "hipfft" - - "roctracer" - - "hipblaslt" - - "hipblaslt-dev" - - "hip-runtime-amd" + # - "rocm-smi-lib" + - "libdrm-amdgpu-common" + # Do not use hsa-rocr from amdgpu repo because it is overriden by hsa-runtime-rocr4wsl-amdgpu + - "hsa-rocr (<< 20)" + - "hsa-amd-aqlprofile" + - "comgr" + - "rocprofiler-register" + - "rocprofiler-sdk" + - "miopen-hip" + - "rccl" + - "rocm-device-libs" + - "hip-dev" + - "rocblas" + - "rocsolver" + - "hipsolver" + - "hipfft" + - "roctracer" + - "hipblaslt" + - "hipblaslt-dev" + - "hip-runtime-amd" diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index a055767a..78171a3b 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -6,7 +6,7 @@ _BUILD_FILE_DEFAULT_VISIBILITY = """\ package(default_visibility = ["//visibility:public"]) """ -_ROCM_STRIP_PREFIX = "opt/rocm-7.1.1" +_ROCM_STRIP_PREFIX = "opt/rocm-7.2.0" _UBUNTU_PACKAGES = { "libdrm2-amdgpu": packages.filegroup(name = "libdrm2-amdgpu", srcs = ["opt/amdgpu/lib/x86_64-linux-gnu/libdrm.so.2"]), @@ -61,7 +61,6 @@ _ROCM_PACKAGES = { "hip-dev": """filegroup(name = "runfiles", srcs = glob(["share/**"]))""", "rocblas": "\n".join([ packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), - packages.load_("@zml//platforms/rocm:gfx.bzl", "bytecode_select"), packages.patchelf( name = "rocblas", src = "lib/librocblas.so.5", @@ -70,19 +69,11 @@ _ROCM_PACKAGES = { "dlopen": "zmlxrocm_dlopen", }, ), - """bytecode_select( - name = "bytecodes", - srcs = glob(["lib/rocblas/library/*"]), - enabled_gfx = "@libpjrt_rocm//:gfx", - ) - """, - packages.filegroup( + """filegroup( name = "runfiles", - srcs = [ - "lib/rocblas/library/TensileManifest.txt", - ":bytecodes", - ], - ), + srcs = glob(["lib/rocblas/library/**"]), + ) + """ ]), "rocfft": packages.filegroup(name = "rocfft", srcs = ["lib/librocfft.so.0"]), "rocsolver": packages.filegroup(name = "rocsolver", srcs = ["lib/librocsolver.so.0"]), @@ -92,7 +83,6 @@ _ROCM_PACKAGES = { ]), "hipblaslt": "\n".join([ packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), - packages.load_("@zml//platforms/rocm:gfx.bzl", "bytecode_select"), packages.patchelf( name = "hipblaslt", src = "lib/libhipblaslt.so.1", @@ -110,22 +100,11 @@ _ROCM_PACKAGES = { "dlopen": "zmlxrocm_dlopen", }, ), - """bytecode_select( - name = "bytecodes", - srcs = glob( - include = ["lib/hipblaslt/library/*"], - exclude = ["lib/hipblaslt/library/hipblasltExtOpLibrary.dat"], - ), - enabled_gfx = "@libpjrt_rocm//:gfx", - ) - """, - packages.filegroup( + """filegroup( name = "runfiles", - srcs = [ - "lib/hipblaslt/library/hipblasltExtOpLibrary.dat", - ":bytecodes", - ], - ), + srcs = glob(["lib/hipblaslt/library/**"]), + ) + """ ]), "hipfft": packages.filegroup(name = "hipfft", srcs = ["lib/libhipfft.so.0"]), "hip-runtime-amd": "\n".join([ diff --git a/platforms/rocm/rocm.zig b/platforms/rocm/rocm.zig index 617cc63c..79053208 100644 --- a/platforms/rocm/rocm.zig +++ b/platforms/rocm/rocm.zig @@ -38,7 +38,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { return error.Unavailable; } - const r = try bazel.runfiles(io, bazel_builtin.current_repository); + const r = try bazel.runfiles(bazel_builtin.current_repository); var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const sandbox_path = try r.rlocation("libpjrt_rocm/sandbox", &path_buf) orelse { diff --git a/zml/io.zig b/zml/io.zig index b3738a9e..8b8703cf 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -273,8 +273,8 @@ pub const MemoryWriter = union(enum) { pub fn init(allocator: std.mem.Allocator, io: std.Io, memory: *const Memory, pool: *mem.DynamicBufferPool, shape: Shape, buffer: *Buffer) !MemoryWriter { return switch (memory.platform.target) { - .cuda, .rocm => .{ .direct = try .init(allocator, io, memory, pool, shape, buffer) }, - .tpu, .neuron, .cpu => .{ .buffered = try .init(allocator, io, memory, shape, buffer) }, + .cuda => .{ .direct = try .init(allocator, io, memory, pool, shape, buffer) }, + .rocm, .tpu, .neuron, .cpu => .{ .buffered = try .init(allocator, io, memory, shape, buffer) }, }; } diff --git a/zml/mem.zig b/zml/mem.zig index 68c7bc74..6a5c475b 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -21,9 +21,9 @@ pub const DmaAllocator = union(enum) { pub fn init(parent: std.mem.Allocator, device: *const Device) DmaAllocator { return switch (device.platform.target) { - .cuda, .rocm => .{ .dmam = .init(parent, device.platform) }, + .cuda => .{ .dmam = .init(parent, device.platform) }, .tpu, .neuron => .{ .uib = .init(device.memory(.host_pinned)) }, - .cpu => .{ .passthrough = parent }, + .rocm, .cpu => .{ .passthrough = parent }, }; } diff --git a/zml/module.zig b/zml/module.zig index 8da65be4..31637852 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -496,9 +496,7 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: .{ "gfx906", {} }, .{ "gfx900", {} }, }); - const first_device = platform.pjrt_client.devices(platform.pjrt_api)[0]; - const description = first_device.getDescription(platform.pjrt_api); - break :gemm_blk supported.has(description.attribute(platform.pjrt_api, "compute_capability").?.string); + break :gemm_blk supported.has(platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "compute_capability").?.string); }, upb_arena); // Use lld from libllvm instead of invoking the ld.lld binary. // This saves us from having to sandbox it. From 8fd3fec244a320431bceb9ef40f428b226d58817 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 16 Feb 2026 19:58:58 +0000 Subject: [PATCH 287/881] platforms/cuda: fix cuda 13 loader names --- platforms/cuda/zmlxcuda.zig | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/platforms/cuda/zmlxcuda.zig b/platforms/cuda/zmlxcuda.zig index dc9be25f..5834cb6b 100644 --- a/platforms/cuda/zmlxcuda.zig +++ b/platforms/cuda/zmlxcuda.zig @@ -4,13 +4,13 @@ const stdx = @import("stdx"); pub export fn zmlxcuda_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque { const replacements: std.StaticStringMap([:0]const u8) = .initComptime(.{ - .{ "libcublas.so", "libcublas.so.12" }, - .{ "libcublasLt.so", "libcublasLt.so.12" }, - .{ "libcudart.so", "libcudart.so.12" }, + .{ "libcublas.so", "libcublas.so.13" }, + .{ "libcublasLt.so", "libcublasLt.so.13" }, + .{ "libcudart.so", "libcudart.so.13" }, .{ "libcudnn.so", "libcudnn.so.9" }, - .{ "libcufft.so", "libcufft.so.11" }, - .{ "libcupti.so", "libcupti.so.12" }, - .{ "libcusolver.so", "libcusolver.so.11" }, + .{ "libcufft.so", "libcufft.so.12" }, + .{ "libcupti.so", "libcupti.so.13" }, + .{ "libcusolver.so", "libcusolver.so.12" }, .{ "libcusparse.so", "libcusparse.so.12" }, .{ "libnccl.so", "libnccl.so.2" }, }); From f64ad1ef7929cf5c53848b98f7cd56ae8f4473d9 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 16 Feb 2026 19:59:38 +0000 Subject: [PATCH 288/881] workspace: remove apple_support --- MODULE.bazel | 28 +-- MODULE.bazel.lock | 245 ++++++++++----------- third_party/zls/zls_write_build_config.bzl | 3 +- 3 files changed, 122 insertions(+), 154 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index bbddad6a..b0d54b40 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -2,31 +2,26 @@ module( name = "zml", ) -bazel_dep(name = "apple_support", version = "1.24.5") -bazel_dep(name = "abseil-cpp", version = "20250814.0") -bazel_dep(name = "aspect_bazel_lib", version = "2.21.2") -bazel_dep(name = "aspect_rules_py", version = "1.6.3") -bazel_dep(name = "bazel_skylib", version = "1.8.2") -bazel_dep(name = "libxev", version = "0.0.0-20251014.0-9f785d2") +bazel_dep(name = "abseil-cpp", version = "20260107.1") +bazel_dep(name = "aspect_bazel_lib", version = "2.22.5") +bazel_dep(name = "aspect_rules_py", version = "1.8.4") +bazel_dep(name = "bazel_skylib", version = "1.9.0") bazel_dep(name = "patchelf", version = "0.18.0.bcr.1") bazel_dep(name = "pcre2", version = "10.45") bazel_dep(name = "platforms", version = "1.0.0") bazel_dep(name = "protobuf", version = "33.1", repo_name = "com_google_protobuf") - -# Needs to be added before rules_cc so that the cc toolchain declared by -# apple_support wins over the one in rules_cc. bazel_dep(name = "rules_cc", version = "0.2.16") -bazel_dep(name = "rules_distroless", version = "0.5.1") +bazel_dep(name = "rules_distroless", version = "0.6.2") bazel_dep(name = "rules_license", version = "1.0.0") -bazel_dep(name = "rules_oci", version = "2.2.6") +bazel_dep(name = "rules_oci", version = "2.2.7") bazel_dep(name = "rules_proto", version = "7.1.0") -bazel_dep(name = "rules_python", version = "1.8.0-rc1") +bazel_dep(name = "rules_python", version = "1.8.4") bazel_dep(name = "rules_rust", version = "0.68.1") bazel_dep(name = "rules_rs", version = "0.0.28") bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "rules_zig", version = "0.12.2") bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.5.5") -bazel_dep(name = "with_cfg.bzl", version = "0.11.0") +bazel_dep(name = "with_cfg.bzl", version = "0.14.1") bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2", dev_dependency = True) @@ -57,13 +52,6 @@ register_toolchains("@rules_zig//zig/target:all") register_toolchains("@zig_toolchains//:all") ### -### Apple C/C++ toolchain (needs to be first) -apple_cc_configure = use_extension("@apple_support//crosstool:setup.bzl", "apple_cc_configure_extension") -use_repo(apple_cc_configure, "local_config_apple_cc", "local_config_apple_cc_toolchains") - -register_toolchains("@local_config_apple_cc_toolchains//:all") -### - ### LLVM Bootstrapped toolchains register_toolchains( "@toolchains_llvm_bootstrapped//toolchain:all", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 64278448..cc67df7a 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -12,30 +12,27 @@ "https://bcr.bazel.build/modules/abseil-cpp/20240116.2/MODULE.bazel": "73939767a4686cd9a520d16af5ab440071ed75cec1a876bf2fcfaf1f71987a16", "https://bcr.bazel.build/modules/abseil-cpp/20250127.1/MODULE.bazel": "c4a89e7ceb9bf1e25cf84a9f830ff6b817b72874088bf5141b314726e46a57c1", "https://bcr.bazel.build/modules/abseil-cpp/20250512.1/MODULE.bazel": "d209fdb6f36ffaf61c509fcc81b19e81b411a999a934a032e10cd009a0226215", - "https://bcr.bazel.build/modules/abseil-cpp/20250814.0/MODULE.bazel": "c43c16ca2c432566cdb78913964497259903ebe8fb7d9b57b38e9f1425b427b8", - "https://bcr.bazel.build/modules/abseil-cpp/20250814.0/source.json": "b88bff599ceaf0f56c264c749b1606f8485cec3b8c38ba30f88a4df9af142861", + "https://bcr.bazel.build/modules/abseil-cpp/20260107.1/MODULE.bazel": "e33b3801443f5fd64465262084534115db76363df13d2168a42bbfacc747be81", + "https://bcr.bazel.build/modules/abseil-cpp/20260107.1/source.json": "7a9a88969b1e79268cf613728ca8ff8fa4bc4b1a9abee9ec1fb5f113ca751971", "https://bcr.bazel.build/modules/apple_support/1.11.1/MODULE.bazel": "1843d7cd8a58369a444fc6000e7304425fba600ff641592161d9f15b179fb896", "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", "https://bcr.bazel.build/modules/apple_support/1.23.0/MODULE.bazel": "317d47e3f65b580e7fb4221c160797fda48e32f07d2dfff63d754ef2316dcd25", "https://bcr.bazel.build/modules/apple_support/1.23.1/MODULE.bazel": "53763fed456a968cf919b3240427cf3a9d5481ec5466abc9d5dc51bc70087442", "https://bcr.bazel.build/modules/apple_support/1.24.1/MODULE.bazel": "f46e8ddad60aef170ee92b2f3d00ef66c147ceafea68b6877cb45bd91737f5f8", - "https://bcr.bazel.build/modules/apple_support/1.24.5/MODULE.bazel": "692784f32ee3175cc1ced78f4262a0d4b7110c89793cd0b63f950af2d9cc6bdc", "https://bcr.bazel.build/modules/apple_support/2.2.0/MODULE.bazel": "5e32bc42e413a4544df7afee7fa4c7aff73e670a44e56e8e45ce1954332034ad", "https://bcr.bazel.build/modules/apple_support/2.2.0/source.json": "368bf1de5dace2a411e792e6df2ea5384eb8db3899680ed51b37eb48b88fd8ac", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.0.0/MODULE.bazel": "e118477db5c49419a88d78ebc7a2c2cea9d49600fe0f490c1903324a2c16ecd9", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.16.0/MODULE.bazel": "852f9ebbda017572a7c113a2434592dd3b2f55cd9a0faea3d4be5a09a59e4900", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.3/MODULE.bazel": "253d739ba126f62a5767d832765b12b59e9f8d2bc88cc1572f4a73e46eb298ca", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.21.2/MODULE.bazel": "276347663a25b0d5bd6cad869252bea3e160c4d980e764b15f3bae7f80b30624", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.21.2/source.json": "f42051fa42629f0e59b7ac2adf0a55749144b11f1efcd8c697f0ee247181e526", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.22.5/MODULE.bazel": "004ba890363d05372a97248c37205ae64b6fa31047629cd2c0895a9d0c7779e8", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.22.5/source.json": "ac2c3213df8f985785f1d0aeb7f0f73d5324e6e67d593d9b9470fb74a25d4a9b", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "780d1a6522b28f5edb7ea09630748720721dfe27690d65a2d33aa7509de77e07", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "812d2dd42f65dca362152101fbec418029cc8fd34cbad1a2fde905383d705838", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.9.4/MODULE.bazel": "ccc41028429f894b02fde7ef67d416cba3ba5084ed9ddb9bb6107aa82d118776", - "https://bcr.bazel.build/modules/aspect_rules_py/1.6.3/MODULE.bazel": "a34d989f9a258f478b40f69a0c1dd0a810b65173681b459c296ffeebabd997b0", - "https://bcr.bazel.build/modules/aspect_rules_py/1.6.3/source.json": "aac06d362804a5a4c1424efc37fc1ded6c6ac7d8be84ef28f735944f8a1f884d", - "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.2.6/MODULE.bazel": "cafb8781ad591bc57cc765dca5fefab08cf9f65af363d162b79d49205c7f8af7", + "https://bcr.bazel.build/modules/aspect_rules_py/1.8.4/MODULE.bazel": "4ab4a78e9d65cf15e755807f7ae504655e1771cefd7c3343ad34dbbb625b9564", + "https://bcr.bazel.build/modules/aspect_rules_py/1.8.4/source.json": "7cd80ad811f0c2b3aad3c878506ae602fce2fe53b78487ecfbcc8bcde07d220e", "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.2/MODULE.bazel": "598e7fe3b54f5fa64fdbeead1027653963a359cc23561d43680006f3b463d5a4", - "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.2/source.json": "c6f5c39e6f32eb395f8fdaea63031a233bbe96d49a3bfb9f75f6fce9b74bec6c", + "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.3/MODULE.bazel": "37c764292861c2f70314efa9846bb6dbb44fc0308903b3285da6528305450183", + "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.3/source.json": "605086bbc197743a0d360f7ddc550a1d4dfa0441bc807236e17170f636153348", "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", "https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101", "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", @@ -53,10 +50,13 @@ "https://bcr.bazel.build/modules/bazel_features/1.32.0/MODULE.bazel": "095d67022a58cb20f7e20e1aefecfa65257a222c18a938e2914fd257b5f1ccdc", "https://bcr.bazel.build/modules/bazel_features/1.33.0/MODULE.bazel": "8b8dc9d2a4c88609409c3191165bccec0e4cb044cd7a72ccbe826583303459f6", "https://bcr.bazel.build/modules/bazel_features/1.34.0/MODULE.bazel": "e8475ad7c8965542e0c7aac8af68eb48c4af904be3d614b6aa6274c092c2ea1e", - "https://bcr.bazel.build/modules/bazel_features/1.34.0/source.json": "dfa5c4b01110313153b484a735764d247fee5624bbab63d25289e43b151a657a", + "https://bcr.bazel.build/modules/bazel_features/1.38.0/MODULE.bazel": "f9b8a9c890ebd216b4049fd12a31d3c2602e3403c7af636b04fbbd7453edc9c9", + "https://bcr.bazel.build/modules/bazel_features/1.38.0/source.json": "31ba776c122b54a2885e23651642e32f087a87bf025465f8040751894b571277", "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", "https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b", "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", + "https://bcr.bazel.build/modules/bazel_lib/3.0.0-beta.1/MODULE.bazel": "407729e232f611c3270005b016b437005daa7b1505826798ea584169a476e878", + "https://bcr.bazel.build/modules/bazel_lib/3.0.0-rc.0/MODULE.bazel": "d6e00979a98ac14ada5e31c8794708b41434d461e7e7ca39b59b765e6d233b18", "https://bcr.bazel.build/modules/bazel_lib/3.0.0/MODULE.bazel": "22b70b80ac89ad3f3772526cd9feee2fa412c2b01933fea7ed13238a448d370d", "https://bcr.bazel.build/modules/bazel_lib/3.2.0/MODULE.bazel": "39b50d94b9be6bda507862254e20c263f9b950e3160112348d10a938be9ce2c2", "https://bcr.bazel.build/modules/bazel_lib/3.2.0/source.json": "a6f45a903134bebbf33a6166dd42b4c7ab45169de094b37a85f348ca41170a84", @@ -73,13 +73,15 @@ "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/MODULE.bazel": "3120d80c5861aa616222ec015332e5f8d3171e062e3e804a2a0253e1be26e59b", "https://bcr.bazel.build/modules/bazel_skylib/1.8.1/MODULE.bazel": "88ade7293becda963e0e3ea33e7d54d3425127e0a326e0d17da085a5f1f03ff6", "https://bcr.bazel.build/modules/bazel_skylib/1.8.2/MODULE.bazel": "69ad6927098316848b34a9142bcc975e018ba27f08c4ff403f50c1b6e646ca67", - "https://bcr.bazel.build/modules/bazel_skylib/1.8.2/source.json": "34a3c8bcf233b835eb74be9d628899bb32999d3e0eadef1947a0a562a2b16ffb", + "https://bcr.bazel.build/modules/bazel_skylib/1.9.0/MODULE.bazel": "72997b29dfd95c3fa0d0c48322d05590418edef451f8db8db5509c57875fb4b7", + "https://bcr.bazel.build/modules/bazel_skylib/1.9.0/source.json": "7ad77c1e8c1b84222d9b3f3cae016a76639435744c19330b0b37c0a3c9da7dc0", "https://bcr.bazel.build/modules/buildifier_prebuilt/8.2.0.2/MODULE.bazel": "a9b689711d5b69f9db741649b218c119b9fdf82924ba390415037e09798edd03", "https://bcr.bazel.build/modules/buildifier_prebuilt/8.2.0.2/source.json": "51eb0a4b38aaaeab7fa64361576d616c4d8bfd0f17a0a10184aeab7084d79f8e", "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.1/MODULE.bazel": "cdf8cbe5ee750db04b78878c9633cc76e80dcf4416cbe982ac3a9222f80713c8", - "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.1/source.json": "fa7b512dfcb5eafd90ce3959cf42a2a6fe96144ebbb4b3b3928054895f2afac2", + "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.3/MODULE.bazel": "f1b7bb2dd53e8f2ef984b39485ec8a44e9076dda5c4b8efd2fb4c6a6e856a31d", + "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.3/source.json": "ebe931bfe362e4b41e59ee00a528db6074157ff2ced92eb9e970acab2e1089c9", "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb", "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6", @@ -93,8 +95,6 @@ "https://bcr.bazel.build/modules/jsoncpp/1.9.6/MODULE.bazel": "2f8d20d3b7d54143213c4dfc3d98225c42de7d666011528dc8fe91591e2e17b0", "https://bcr.bazel.build/modules/jsoncpp/1.9.6/source.json": "a04756d367a2126c3541682864ecec52f92cdee80a35735a3cb249ce015ca000", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", - "https://bcr.bazel.build/modules/libxev/0.0.0-20251014.0-9f785d2/MODULE.bazel": "d8bda6be48f8c0613d3393cfe15143467a7ee17d7e702fa15719ddc3e9fb646a", - "https://bcr.bazel.build/modules/libxev/0.0.0-20251014.0-9f785d2/source.json": "52ac2605138fd00de1413a13b2b8c5ceb4adc0d8e052133db888400f6768746f", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/source.json": "f448c6e8963fdfa7eb831457df83ad63d3d6355018f6574fb017e8169deb43a9", "https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92", @@ -148,15 +148,14 @@ "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", "https://bcr.bazel.build/modules/rules_cc/0.1.1/MODULE.bazel": "2f0222a6f229f0bf44cd711dc13c858dad98c62d52bd51d8fc3a764a83125513", "https://bcr.bazel.build/modules/rules_cc/0.1.5/MODULE.bazel": "88dfc9361e8b5ae1008ac38f7cdfd45ad738e4fa676a3ad67d19204f045a1fd8", - "https://bcr.bazel.build/modules/rules_cc/0.2.0/MODULE.bazel": "b5c17f90458caae90d2ccd114c81970062946f49f355610ed89bebf954f5783c", - "https://bcr.bazel.build/modules/rules_cc/0.2.13/MODULE.bazel": "eecdd666eda6be16a8d9dc15e44b5c75133405e820f620a234acc4b1fdc5aa37", "https://bcr.bazel.build/modules/rules_cc/0.2.15/MODULE.bazel": "6a0a4a75a57aa6dc888300d848053a58c6b12a29f89d4304e1c41448514ec6e8", "https://bcr.bazel.build/modules/rules_cc/0.2.16/MODULE.bazel": "9242fa89f950c6ef7702801ab53922e99c69b02310c39fb6e62b2bd30df2a1d4", "https://bcr.bazel.build/modules/rules_cc/0.2.16/source.json": "d03d5cde49376d87e14ec14b666c56075e5e3926930327fd5d0484a1ff2ac1cc", "https://bcr.bazel.build/modules/rules_cc/0.2.4/MODULE.bazel": "1ff1223dfd24f3ecf8f028446d4a27608aa43c3f41e346d22838a4223980b8cc", "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", - "https://bcr.bazel.build/modules/rules_distroless/0.5.1/MODULE.bazel": "2a63f4744d30749128105da5f96adf7caf5628e37548293f89e7fa39c3b3f2c2", - "https://bcr.bazel.build/modules/rules_distroless/0.5.1/source.json": "c6b9ff7f325bfed89c3671757f14c1d1bc6077d0fcef809b8aa2d007cac7dd1d", + "https://bcr.bazel.build/modules/rules_cc/0.2.9/MODULE.bazel": "34263f1dca62ea664265438cef714d7db124c03e1ed55ebb4f1dc860164308d1", + "https://bcr.bazel.build/modules/rules_distroless/0.6.2/MODULE.bazel": "eafe8b473c0c78038d9a6e1dbd9278b7bf83457ddc78207297483bfbe1012d56", + "https://bcr.bazel.build/modules/rules_distroless/0.6.2/source.json": "830d7a25ce382e571e58791fbed3b47e6da3237805624bab8997ae59d763bdc2", "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", "https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/MODULE.bazel": "40c97d1144356f52905566c55811f13b299453a14ac7769dfba2ac38192337a8", "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", @@ -192,8 +191,8 @@ "https://bcr.bazel.build/modules/rules_license/0.0.7/MODULE.bazel": "088fbeb0b6a419005b89cf93fe62d9517c0a2b8bb56af3244af65ecfe37e7d5d", "https://bcr.bazel.build/modules/rules_license/1.0.0/MODULE.bazel": "a7fda60eefdf3d8c827262ba499957e4df06f659330bbe6cdbdb975b768bb65c", "https://bcr.bazel.build/modules/rules_license/1.0.0/source.json": "a52c89e54cc311196e478f8382df91c15f7a2bfdf4c6cd0e2675cc2ff0b56efb", - "https://bcr.bazel.build/modules/rules_oci/2.2.6/MODULE.bazel": "2ba6ddd679269e00aeffe9ca04faa2d0ca4129650982c9246d0d459fe2da47d9", - "https://bcr.bazel.build/modules/rules_oci/2.2.6/source.json": "94e7decb8f95d9465b0bbea71c65064cd16083be1350c7468f131818641dc4a5", + "https://bcr.bazel.build/modules/rules_oci/2.2.7/MODULE.bazel": "f6150e4b224d459f7f6523ef65967464ca4efdd266c7fbf2f5a2a51011957e0c", + "https://bcr.bazel.build/modules/rules_oci/2.2.7/source.json": "b099f02af330f47f19dc67fc9300ef6e1937a8c86882690db0e7a2fcea8c7f6b", "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc", "https://bcr.bazel.build/modules/rules_pkg/1.0.1/MODULE.bazel": "5b1df97dbc29623bccdf2b0dcd0f5cb08e2f2c9050aab1092fd39a41e82686ff", "https://bcr.bazel.build/modules/rules_pkg/1.0.1/source.json": "bd82e5d7b9ce2d31e380dd9f50c111d678c3bdaca190cb76b0e1c71b05e1ba8a", @@ -208,14 +207,14 @@ "https://bcr.bazel.build/modules/rules_python/0.23.1/MODULE.bazel": "49ffccf0511cb8414de28321f5fcf2a31312b47c40cc21577144b7447f2bf300", "https://bcr.bazel.build/modules/rules_python/0.25.0/MODULE.bazel": "72f1506841c920a1afec76975b35312410eea3aa7b63267436bfb1dd91d2d382", "https://bcr.bazel.build/modules/rules_python/0.28.0/MODULE.bazel": "cba2573d870babc976664a912539b320cbaa7114cd3e8f053c720171cde331ed", - "https://bcr.bazel.build/modules/rules_python/0.29.0/MODULE.bazel": "2ac8cd70524b4b9ec49a0b8284c79e4cd86199296f82f6e0d5da3f783d660c82", "https://bcr.bazel.build/modules/rules_python/0.31.0/MODULE.bazel": "93a43dc47ee570e6ec9f5779b2e64c1476a6ce921c48cc9a1678a91dd5f8fd58", "https://bcr.bazel.build/modules/rules_python/0.33.2/MODULE.bazel": "3e036c4ad8d804a4dad897d333d8dce200d943df4827cb849840055be8d2e937", "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", "https://bcr.bazel.build/modules/rules_python/0.40.0/MODULE.bazel": "9d1a3cd88ed7d8e39583d9ffe56ae8a244f67783ae89b60caafc9f5cf318ada7", + "https://bcr.bazel.build/modules/rules_python/1.0.0/MODULE.bazel": "898a3d999c22caa585eb062b600f88654bf92efb204fa346fb55f6f8edffca43", "https://bcr.bazel.build/modules/rules_python/1.6.0/MODULE.bazel": "7e04ad8f8d5bea40451cf80b1bd8262552aa73f841415d20db96b7241bd027d8", - "https://bcr.bazel.build/modules/rules_python/1.8.0-rc1/MODULE.bazel": "d5348333fd8be9589c3ea8d9110fd00b1c7d84bc7b505668307cbe9f105e9c8f", - "https://bcr.bazel.build/modules/rules_python/1.8.0-rc1/source.json": "d26719d5b92a569eaa387bad9667a042d4810193f279ea64590c9d76ae38f3ba", + "https://bcr.bazel.build/modules/rules_python/1.8.4/MODULE.bazel": "33e3971e66161a3e955f7a0d411a8d1f291c4ce4c561851512466f3c77ff8ece", + "https://bcr.bazel.build/modules/rules_python/1.8.4/source.json": "9fbc0e57bae52cddcc3831d668bce87a47e0c655104a85098d4459dd9a3b0a10", "https://bcr.bazel.build/modules/rules_rs/0.0.28/MODULE.bazel": "3c3830eda824d0476818443dd4a3a2fa0765484589db874d18e94fb6ee50a60f", "https://bcr.bazel.build/modules/rules_rs/0.0.28/source.json": "3480135f6ef8111ec7abe2b7cc164af40354b4eaf8f885170567922a70a2dc99", "https://bcr.bazel.build/modules/rules_rust/0.66.0/MODULE.bazel": "86ef763a582f4739a27029bdcc6c562258ed0ea6f8d58294b049e215ceb251b3", @@ -243,16 +242,18 @@ "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/source.json": "32bd87e5f4d7acc57c5b2ff7c325ae3061d5e242c0c4c214ae87e0f1c13e54cb", "https://bcr.bazel.build/modules/tar.bzl/0.2.1/MODULE.bazel": "52d1c00a80a8cc67acbd01649e83d8dd6a9dc426a6c0b754a04fe8c219c76468", "https://bcr.bazel.build/modules/tar.bzl/0.5.1/MODULE.bazel": "7c2eb3dcfc53b0f3d6f9acdfd911ca803eaf92aadf54f8ca6e4c1f3aee288351", + "https://bcr.bazel.build/modules/tar.bzl/0.5.5/MODULE.bazel": "4bfab9bbc7a1966c2c5f7371f5848f5e2d27c465951b4435adc9aaf00ed681da", "https://bcr.bazel.build/modules/tar.bzl/0.6.0/MODULE.bazel": "a3584b4edcfafcabd9b0ef9819808f05b372957bbdff41601429d5fd0aac2e7c", "https://bcr.bazel.build/modules/tar.bzl/0.6.0/source.json": "4a620381df075a16cb3a7ed57bd1d05f7480222394c64a20fa51bdb636fda658", "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.5.5/MODULE.bazel": "a2f1469b2920fcc9c043c87301b7a207d320231615abecd7771d89e516fc3d0b", "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.5.5/source.json": "8498c094703da13a859d19dd2a200152f75870fb7f22a9770df99d230a4f50d1", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", - "https://bcr.bazel.build/modules/with_cfg.bzl/0.11.0/MODULE.bazel": "bcb1dba64b74542648a93d6313e160b2670f22395581edd8c4ffadb811618ef7", "https://bcr.bazel.build/modules/with_cfg.bzl/0.12.0/MODULE.bazel": "b573395fe63aef4299ba095173e2f62ccfee5ad9bbf7acaa95dba73af9fc2b38", - "https://bcr.bazel.build/modules/with_cfg.bzl/0.12.0/source.json": "3f3fbaeafecaf629877ad152a2c9def21f8d330d91aa94c5dc75bbb98c10b8b8", + "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.1/MODULE.bazel": "aa0ef3f6c67dd35db7ac38c76785dc02f48b53f34e16b20487e70f51b5324d0a", + "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.1/source.json": "4666d3035f69063ecd136f3f0c68e424f1b1dd97ffc9cceb679522c0be38761b", "https://bcr.bazel.build/modules/yq.bzl/0.1.1/MODULE.bazel": "9039681f9bcb8958ee2c87ffc74bdafba9f4369096a2b5634b88abc0eaefa072", - "https://bcr.bazel.build/modules/yq.bzl/0.1.1/source.json": "2d2bad780a9f2b9195a4a370314d2c17ae95eaa745cefc2e12fbc49759b15aa3", + "https://bcr.bazel.build/modules/yq.bzl/0.3.1/MODULE.bazel": "9bcb7151b3cd4681b89d350530eaf7b45e32a44dda94843b8932b0cb1cd4594a", + "https://bcr.bazel.build/modules/yq.bzl/0.3.1/source.json": "f0b0f204a2a6b0e34b4c9541efe8c04f2ef1af65948daa784eccea738b21dbd2", "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "eec517b5bbe5492629466e11dae908d043364302283de25581e3eb944326c4ca", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/source.json": "22bc55c47af97246cfc093d0acf683a7869377de362b5d1c552c2c2e16b7a806", @@ -262,54 +263,12 @@ "moduleExtensions": { "@@aspect_rules_py+//py:extensions.bzl%py_tools": { "general": { - "bzlTransitiveDigest": "G/1q87gmIpapGBPovxgsd2S/VIcXuqJ4H0PnAtwkKWc=", - "usagesDigest": "tVlELaqQFX7HPl1uFzRY9m7nVV7tY9roq1qfK75ApwE=", + "bzlTransitiveDigest": "GLVIk55uU2jb460gpL1FFbE/tBFqzMxRfta5QXYfjQo=", + "usagesDigest": "9BiP3ASHlGl4doGhzXJwK17taNzv/q6x/IZ4Hb6LxNY=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { - "bsd_tar_darwin_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "bsd_tar_darwin_arm64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "bsd_tar_linux_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "bsd_tar_linux_arm64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "bsd_tar_windows_amd64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "windows_amd64" - } - }, - "bsd_tar_windows_arm64": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "windows_arm64" - } - }, - "bsd_tar_toolchains": { - "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%tar_toolchains_repo", - "attributes": { - "user_repository_name": "bsd_tar" - } - }, "rules_py_tools.darwin_amd64": { "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", "attributes": { @@ -352,16 +311,6 @@ } }, "recordedRepoMappingEntries": [ - [ - "aspect_bazel_lib+", - "bazel_tools", - "bazel_tools" - ], - [ - "aspect_rules_py+", - "aspect_bazel_lib", - "aspect_bazel_lib+" - ], [ "aspect_rules_py+", "aspect_tools_telemetry_report", @@ -382,8 +331,8 @@ }, "@@aspect_tools_telemetry+//:extension.bzl%telemetry": { "general": { - "bzlTransitiveDigest": "BXySuBSW3dtagvU27mRMChwOLh8Z8/Iyds20LjMajuE=", - "usagesDigest": "dcbNgiRoyIPXA1ORP5Y4jD1mG/tTZiyOfFASmpRxpfY=", + "bzlTransitiveDigest": "MePriaXmQNSqUfE+YQvEtzBc2bU1mjITINIffFiZYgo=", + "usagesDigest": "73l8h4f8amAmbKge5Ixr/4s4BE7L7fqDBTTAMrNSHHg=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -392,51 +341,9 @@ "repoRuleId": "@@aspect_tools_telemetry+//:extension.bzl%tel_repository", "attributes": { "deps": { - "abseil-cpp": "20250814.0", - "aspect_bazel_lib": "2.21.2", - "aspect_rules_py": "1.6.3", - "aspect_tools_telemetry": "0.3.2", - "bazel_features": "1.34.0", - "bazel_lib": "3.2.0", - "bazel_skylib": "1.8.2", - "buildifier_prebuilt": "8.2.0.2", - "buildozer": "7.1.2", - "gawk": "5.3.2.bcr.1", - "googletest": "1.17.0", - "jq.bzl": "0.1.0", - "jsoncpp": "1.9.6", - "libxev": "0.0.0-20251014.0-9f785d2", - "nlohmann_json": "3.6.1", - "package_metadata": "0.0.5", - "patchelf": "0.18.0.bcr.1", - "pcre2": "10.45", - "platforms": "1.0.0", - "protobuf": "33.1", - "pybind11_bazel": "2.12.0", - "re2": "2024-07-02.bcr.1", - "rules_android": "0.1.1", - "rules_apple": "3.16.0", - "rules_cc": "0.2.16", - "rules_distroless": "0.5.1", - "rules_java": "8.14.0", - "rules_jvm_external": "6.7", - "rules_kotlin": "1.9.6", - "rules_license": "1.0.0", - "rules_oci": "2.2.6", - "rules_pkg": "1.0.1", - "rules_proto": "7.1.0", - "rules_python": "1.8.0-rc1", + "aspect_rules_py": "1.8.4", "rules_rs": "0.0.28", - "rules_rust": "0.66.0", - "rules_shell": "0.6.1", - "rules_swift": "2.1.1", - "stardoc": "0.7.2", - "swift_argument_parser": "1.3.1.1", - "tar.bzl": "0.6.0", - "toolchains_llvm_bootstrapped": "0.5.5", - "with_cfg.bzl": "0.12.0", - "yq.bzl": "0.1.1", - "zlib": "1.3.1.bcr.5" + "aspect_tools_telemetry": "0.3.3" } } } @@ -521,8 +428,8 @@ }, "@@rules_oci+//oci:extensions.bzl%oci": { "general": { - "bzlTransitiveDigest": "Uiu9QIgPhgfbEC1lExLocKf2lZ0a9XSSmq3OrI8IL8I=", - "usagesDigest": "vK9qUa3pyGJik2hb1NI2fKVJrjIszor+wN/44XMkKi0=", + "bzlTransitiveDigest": "nCIRDeZJX85AXhvovlzr0Sx1f3ZKx8IS8xvTHWlUlqM=", + "usagesDigest": "IIGPr69MT/mSUo6fNUffrIiU31EqGjlEYm5qFmNjmUc=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -711,7 +618,7 @@ "@@rules_python+//python/extensions:config.bzl%config": { "general": { "bzlTransitiveDigest": "X49Lbeli5+XwNYRk+isOvNKFMIF8g/rnxL3Fdlwrffc=", - "usagesDigest": "tIEieEA/gbsjNF3L/Oouyg6UdqGOVxFsPqBjFxkTAKM=", + "usagesDigest": "p2al+dDKI5UlCyNvheMVynbWSGbdiji/jMz53fMNfJA=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -946,7 +853,7 @@ "@@rules_python+//python/uv:uv.bzl%uv": { "general": { "bzlTransitiveDigest": "CJBWuaTerWIqLpLWl77vjWFD1RRpXfzC2UGMwEk6eVg=", - "usagesDigest": "gx0mA10eprg2o8cg9R+m/jpqCuVjnTnNzD17TPQdFu0=", + "usagesDigest": "lPQI4kxB6q1JLGHa7OFxCNrWxnLjeMKg5ogOIB5OF6U=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -1341,6 +1248,80 @@ ] ] } + }, + "@@yq.bzl+//yq:extensions.bzl%yq": { + "general": { + "bzlTransitiveDigest": "iWlctm1cJYZLESuQEBygCQc6adq+oIRMb4Dmk0rYxkA=", + "usagesDigest": "J3TR36exv3ePI9O2y+uqrzTDmT6s6ADkTsROB++nm6I=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "yq_darwin_amd64": { + "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", + "attributes": { + "platform": "darwin_amd64", + "version": "4.45.1" + } + }, + "yq_darwin_arm64": { + "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", + "attributes": { + "platform": "darwin_arm64", + "version": "4.45.1" + } + }, + "yq_linux_amd64": { + "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", + "attributes": { + "platform": "linux_amd64", + "version": "4.45.1" + } + }, + "yq_linux_arm64": { + "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", + "attributes": { + "platform": "linux_arm64", + "version": "4.45.1" + } + }, + "yq_linux_s390x": { + "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", + "attributes": { + "platform": "linux_s390x", + "version": "4.45.1" + } + }, + "yq_linux_riscv64": { + "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", + "attributes": { + "platform": "linux_riscv64", + "version": "4.45.1" + } + }, + "yq_linux_ppc64le": { + "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", + "attributes": { + "platform": "linux_ppc64le", + "version": "4.45.1" + } + }, + "yq_windows_amd64": { + "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", + "attributes": { + "platform": "windows_amd64", + "version": "4.45.1" + } + }, + "yq_toolchains": { + "repoRuleId": "@@yq.bzl+//yq/toolchain:toolchain.bzl%yq_toolchains_repo", + "attributes": { + "user_repository_name": "yq" + } + } + }, + "recordedRepoMappingEntries": [] + } } }, "facts": { diff --git a/third_party/zls/zls_write_build_config.bzl b/third_party/zls/zls_write_build_config.bzl index 861198e2..fdd20322 100644 --- a/third_party/zls/zls_write_build_config.bzl +++ b/third_party/zls/zls_write_build_config.bzl @@ -1,6 +1,5 @@ """Generates the ZLS build config file.""" -load("@apple_support//lib:apple_support.bzl", "apple_support") load("@rules_cc//cc:find_cc_toolchain.bzl", "use_cc_toolchain") load("@rules_cc//cc/common:cc_common.bzl", "cc_common") load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") @@ -168,7 +167,7 @@ zls_write_build_config = rule( "_c_builtins": attr.label( default = Label("@rules_zig//zig/private/common:c_builtins"), ), - } | apple_support.action_required_attrs() | apple_support.platform_constraint_attrs(), + }, toolchains = [ "@rules_zig//zig:toolchain_type", "@rules_zig//zig/target:toolchain_type", From ee9155b7bb381956ec671cb0224b06e1f133962e Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 16 Feb 2026 20:00:03 +0000 Subject: [PATCH 289/881] examples/llama: remove bufferized leak --- examples/llama/main.zig | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 94dafe36..b23e6601 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -53,7 +53,6 @@ const CliArgs = struct { pub fn main(init: std.process.Init) !void { const allocator = init.gpa; const arena = init.arena; - _ = arena; // autofix log.info("LLama was compiled with {}", .{@import("builtin").mode}); @@ -163,7 +162,7 @@ pub fn main(init: std.process.Init) !void { var llama_buffers_future = try io.concurrent(LlamaLM.load, .{ &llama_model, - allocator, + arena.allocator(), io, platform, &store, From 2bafd654583f80bb1c1daf13dda34b3af9206bc4 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 9 Feb 2026 12:01:22 +0100 Subject: [PATCH 290/881] Various improvements (#379) This pull request includes several improvements and bug fixes across the codebase, focusing on memory handling, correctness in tensor testing, and API consistency. The most significant changes are grouped below: * Fixed a bug in `zml/testing.zig`'s `testlayer` by updating the callback signature to take a mutable context and increment the index correctly, ensuring tensors are correctly initialized. * Improved memory management for expected output slices in `testlayer` by switching to `zml.Slice.alloc` for allocation, using `readSliceAll`, and freeing only the data instead of the whole slice, removing the potential alignment problems. * Enhanced test logging to include the tolerance value in success messages for better diagnostics. * Increased the evaluation branch quota in `MapRestrict` to support more complex compile-time type mapping, preventing quota exceeded errors. * Updated the `fromCStruct` function in `pjrt/pjrt.zig` to accept a `meta.Struct`. * Improved handling of output operand aliases in `mlir/dialects/stablehlo/stablehlo.zig::custom_call` by passing `output_tuple_indices` for multi-result functions, increasing correctness in custom call attribute construction. --- zml/testing.zig | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/zml/testing.zig b/zml/testing.zig index f84d9f62..5e200864 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -383,11 +383,8 @@ pub fn testLayer( }; } - if (failed) { - log.info("❌ check failed for {s} ! (absolute tolerance: {e} - relative tolerance: {e} - minimum_close_fraction: {d:0>3})", .{ name, opts.absolute_tolerance, opts.relative_tolerance, opts.minimum_close_fraction }); - } else { - log.info("✅ all good for {s} ! (absolute tolerance: {e} - relative tolerance: {e} - minimum_close_fraction: {d:0>3})", .{ name, opts.absolute_tolerance, opts.relative_tolerance, opts.minimum_close_fraction }); - } + if (failed) return error.TestUnexpectedResult; + log.info("all good for {s} ! (tolerance: {e})", .{ name, tolerance }); } fn center(slice: anytype, i: usize) @TypeOf(slice) { From 79cec4f9a4b422a01170c1419940b772c4e35e1e Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 10 Feb 2026 10:11:48 +0100 Subject: [PATCH 291/881] V2: `Tensor.print()` (#380) Added `Tensor.print()` back --- zml/platform.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/platform.zig b/zml/platform.zig index 2daf43d7..daa2f9fe 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -397,7 +397,7 @@ pub const CreateOptions = struct { // bump memory fraction from XLA defaults of 75% to 90%. // Even on a 8GB GPU it should leave enough space for the Cuda driver // https://github.com/openxla/xla/blob/3e87afa11a865cf91137522492918ad18bfe5b7c/xla/pjrt/plugin/xla_gpu/xla_gpu_allocator_config.h#L25-L60 - cuda: Cuda = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.95 } } }, + cuda: Cuda = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.85 } } }, rocm: struct {} = .{}, tpu: struct {} = .{}, neuron: struct {} = .{}, From 692825cd72ad015e45ca02a16bd0b3c7702b48c1 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 12 Feb 2026 11:59:09 +0100 Subject: [PATCH 292/881] Better `testLayer` (#383) --- zml/testing.zig | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/zml/testing.zig b/zml/testing.zig index 5e200864..f84d9f62 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -383,8 +383,11 @@ pub fn testLayer( }; } - if (failed) return error.TestUnexpectedResult; - log.info("all good for {s} ! (tolerance: {e})", .{ name, tolerance }); + if (failed) { + log.info("❌ check failed for {s} ! (absolute tolerance: {e} - relative tolerance: {e} - minimum_close_fraction: {d:0>3})", .{ name, opts.absolute_tolerance, opts.relative_tolerance, opts.minimum_close_fraction }); + } else { + log.info("✅ all good for {s} ! (absolute tolerance: {e} - relative tolerance: {e} - minimum_close_fraction: {d:0>3})", .{ name, opts.absolute_tolerance, opts.relative_tolerance, opts.minimum_close_fraction }); + } } fn center(slice: anytype, i: usize) @TypeOf(slice) { From ed27b2722d607fb88f6bad8c06b2ffcf6937d635 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 22 Jan 2026 14:28:01 +0100 Subject: [PATCH 293/881] zml: added some paged attention bindings --- zml/BUILD.bazel | 2 + zml/attention.zig | 3 + zml/attention/attention.zig | 2 +- zml/attention/flashattn.zig | 486 +++++++++++++++++++++++++++++- zml/attention/paged_attention.zig | 55 ++++ zml/zml.zig | 2 +- 6 files changed, 546 insertions(+), 4 deletions(-) create mode 100644 zml/attention.zig create mode 100644 zml/attention/paged_attention.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 497bc834..7f0f3caf 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -20,8 +20,10 @@ cc_library( zig_library( name = "zml", srcs = [ + "attention.zig", "attention/attention.zig", "attention/flashattn.zig", + "attention/paged_attention.zig", "buffer.zig", "constants.zig", "dtype.zig", diff --git a/zml/attention.zig b/zml/attention.zig new file mode 100644 index 00000000..1aa8001b --- /dev/null +++ b/zml/attention.zig @@ -0,0 +1,3 @@ +pub const attention = @import("attention/attention.zig"); +pub const paged_attention = @import("attention/paged_attention.zig"); +pub const flashattn = @import("attention/flashattn.zig"); diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index 34bf52e0..ad747f04 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -1,7 +1,7 @@ const std = @import("std"); const zml = @import("../zml.zig"); -pub const flashattn = @import("flashattn.zig"); +const flashattn = @import("flashattn.zig"); const Attention = @This(); diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 8673bc4e..ebf87ca9 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -17,6 +17,8 @@ pub fn register(platform: *zml.Platform) !void { if (comptime platforms.isEnabled(.cuda)) { try fa2.register(platform); try fa3.register(platform); + try paged_fa2.Decode.register(platform); + try paged_fa2.Prefill.register(platform); } } @@ -121,8 +123,8 @@ fn fixupKvCacheBuffer(buffer: Buffer, layer_index: i64) Buffer { pub fn Wrapper(comptime T: type, run_func: std.meta.DeclEnum(T)) type { return struct { - pub fn register(platform: *zml.Platform) !void { - try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "cuda", T.run, .{ .command_buffer_compatible = true }); + pub fn register(platform: zml.Platform) !void { + try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "host", T.run, .{ .command_buffer_compatible = true }); } pub fn run(call_frame: *ffi.CallFrame) callconv(.c) ?*ffi.Error { @@ -469,3 +471,483 @@ pub const fa3 = struct { return o.splitAxis(.tot, .{ .b = 1, .q = q_.dim(.q) }).squeeze(.b); } }; + +pub const paged_fa2 = struct { + pub const Variant = enum { + decode, + mixed, + }; + + pub const Options = union(Variant) { + decode: DecodeOptions, + mixed: MixedOptions, + + pub fn isPrefill(self: Options) bool { + return self == .mixed; + } + }; + + pub const Parameters = union(Variant) { + decode: DecodeParameters, + mixed: MixedParameters, + + pub fn init(options_: Options) Parameters { + return switch (options_) { + .decode => |decode_options| .{ .decode = DecodeParameters.init(decode_options) }, + .mixed => |mixed_options| .{ .mixed = MixedParameters.init(mixed_options) }, + }; + } + + pub fn allocationSize(options_: Options) usize { + return switch (options_) { + .decode => |decode_options| DecodeParameters.allocationSize(decode_options), + .mixed => |mixed_options| MixedParameters.allocationSize(mixed_options), + }; + } + + pub fn options(self: Parameters) Options { + return switch (self) { + .decode => |v| .{ .decode = v.options }, + .mixed => |v| .{ .mixed = v.options }, + }; + } + }; + + pub const DecodeOptions = struct { + batch_size: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_token_count: usize, + num_heads: usize, + head_dim: usize, + }; + + pub const DecodeParameters = struct { + block_table: zml.Tensor, + cu_seqlens_q: zml.Tensor, + seqused_k: zml.Tensor, + + metadata: DecodeMetadata, + options: DecodeOptions, + + pub fn init(options: DecodeOptions) DecodeParameters { + return .{ + .block_table = .init(.{ options.batch_size, options.max_num_pages }, .i32), + .cu_seqlens_q = .init(.{options.batch_size + 1}, .i32), + .seqused_k = .init(.{options.batch_size}, .i32), + .metadata = DecodeMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .options = options, + }; + } + + pub fn allocationSize(options: DecodeOptions) usize { + const shapes_ = DecodeParameters.shapes(options); + var allocation_size: usize = 0; + allocation_size += shapes_.block_table.byteSize(); + allocation_size += shapes_.cu_seqlens_q.byteSize(); + allocation_size += shapes_.seqused_k.byteSize(); + allocation_size += DecodeMetadata.allocationSize(options.max_token_count, options.num_heads, options.head_dim); + + return allocation_size; + } + }; + + pub const DecodeMetadata = struct { + out_accum: zml.Tensor, + + pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) DecodeMetadata { + return .{ + .out_accum = .init(.{8 * max_token_count * num_heads * head_dim * 4}, .i8), + }; + } + + pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { + const shapes_ = DecodeMetadata.shapes(max_token_count, num_heads, head_dim); + var allocation_size: usize = 0; + allocation_size += shapes_.out_accum.byteSize(); + return allocation_size; + } + }; + + pub const MixedOptions = struct { + batch_size_prefill: usize, + batch_size_decode: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_seqlen_q: usize, + max_token_count: usize, + num_heads: usize, + head_dim: usize, + }; + + pub const MixedParameters = struct { + block_table_prefill: zml.Tensor, + cu_seqlens_q_prefill: zml.Tensor, + seqused_k_prefill: zml.Tensor, + + block_table_decode: zml.Tensor, + cu_seqlens_q_decode: zml.Tensor, + seqused_k_decode: zml.Tensor, + + metadata: MixedMetadata, + options: MixedOptions, + + pub fn init(options: MixedOptions) MixedParameters { + return .{ + .block_table_prefill = .init(.{ options.batch_size_prefill, options.max_num_pages }, .i32), + .cu_seqlens_q_prefill = .init(.{options.batch_size_prefill + 1}, .i32), + .seqused_k_prefill = .init(.{options.batch_size_prefill}, .i32), + .block_table_decode = .init(.{ options.batch_size_decode, options.max_num_pages }, .i32), + .cu_seqlens_q_decode = .init(.{options.batch_size_decode + 1}, .i32), + .seqused_k_decode = .init(.{options.batch_size_decode}, .i32), + .metadata = MixedMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .options = options, + }; + } + + pub fn allocationSize(options: MixedOptions) usize { + const shapes_ = MixedParameters.shapes(options); + var allocation_size: usize = 0; + allocation_size += shapes_.block_table_decode.byteSize(); + allocation_size += shapes_.cu_seqlens_q_prefill.byteSize(); + allocation_size += shapes_.seqused_k_prefill.byteSize(); + + allocation_size += shapes_.block_table_decode.byteSize(); + allocation_size += shapes_.cu_seqlens_q_decode.byteSize(); + allocation_size += shapes_.seqused_k_decode.byteSize(); + allocation_size += MixedMetadata.allocationSize(options.max_token_count, options.num_heads, options.head_dim); + + return allocation_size; + } + }; + + pub const MixedMetadata = struct { + out_accum: zml.Tensor, + host_metadata: zml.Tensor, + + pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) MixedMetadata { + return .{ + .out_accum = .init(.{8 * max_token_count * num_heads * head_dim * 4}, .i8), + .host_metadata = .init(.{2}, .i32), + }; + } + + pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { + const shapes_ = MixedMetadata.shapes(max_token_count, num_heads, head_dim); + var allocation_size: usize = 0; + allocation_size += shapes_.out_accum.byteSize(); + allocation_size += shapes_.host_metadata.byteSize(); + return allocation_size; + } + }; + + pub const Context = struct { + max_seqlen_k: usize, + max_seqlen_q: usize, + decode_offset: ?zml.Tensor = null, + + //pub fn init(options: Options) zml.ShapeOf(Context) { + // const max_seqlen_k = switch (options) { + // inline else => |v| v.max_seqlen_k, + // }; + // const decode_offset: ?zml.Shape = switch (options) { + // .mixed => zml.Shape.init(.{1}, .i32), + // .decode => null, + // }; + // return .{ + // .max_seqlen_k = max_seqlen_k, + // .decode_offset = decode_offset, + // }; + //} + + pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { + _ = num_heads; // autofix + _ = num_kv_heads; // autofix + _ = head_dim; // autofix + _ = page_size; // autofix + const max_seqlen_k = switch (parameters) { + inline else => |v| v.options.max_seqlen_k, + }; + const max_seqlen_q = switch (parameters) { + .mixed => |mixed_parameters| mixed_parameters.options.max_seqlen_q, + else => 1, + }; + const decode_offset = switch (parameters) { + .mixed => |mixed_parameters| mixed_parameters.metadata.host_metadata.slice1d(0, .{ .end = 1 }).squeeze(0), + .decode => null, + }; + return .{ .max_seqlen_k = max_seqlen_k, .max_seqlen_q = max_seqlen_q, .decode_offset = decode_offset }; + } + }; + + pub const Prefill = struct { + pub const custom_call_name = "paged_fa2_prefill"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); + const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_q: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_q").?; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA2MhaVarlenFwdParams = .{ + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .num_splits = 0, + .num_heads = num_heads, + }; + + flashattn.fa2_mha_varlen_fwd( + &q.toFlashattnTensor(), + &paged_k.toFlashattnTensor(), + &paged_v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + null, + &seqused_k.toFlashattnTensor(), + &block_table.toFlashattnTensor(), + &softmax_lse.toFlashattnTensor(), + null, + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + ¶ms, + stream, + ); + + return null; + } + }; + + pub const Decode = struct { + pub const custom_call_name = "paged_fa2_decode"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); + const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_q: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_q").?; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA2MhaVarlenFwdParams = .{ + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .num_splits = 0, + .num_heads = num_heads, + }; + + flashattn.fa2_mha_varlen_fwd( + &q.toFlashattnTensor(), + &paged_k.toFlashattnTensor(), + &paged_v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + null, + &seqused_k.toFlashattnTensor(), + &block_table.toFlashattnTensor(), + &softmax_lse.toFlashattnTensor(), + null, + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + ¶ms, + stream, + ); + + return null; + } + }; + + pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor) zml.Tensor { + stdx.debug.assert(q.shape().hasTags(.{ .b, .h, .hd }), "Expected q to have tags .b, .h, .hd", .{}); + stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); + stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); + + const o = switch (parameters) { + .decode => |decode_parameters| b: { + const softmax_lse = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.dim(.h), .q = q.dim(.b) }, .f32)); + const softmax_lse_accum = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * decode_parameters.block_table.dim(0) * q.dim(.h) * 4}, .i8)); + + const original_tot = q.dim(.b); + const num_heads = q.dim(.h); + const num_heads_k = k_cache.dim(.h); + const head_size = q.dim(.hd); + const ngroups = @divExact(num_heads, num_heads_k); + const seqlenq_ngroups_swapped = context.max_seqlen_q == 1 and num_heads > num_heads_k and @mod(head_size, 8) == 0; + var q2 = q; + if (seqlenq_ngroups_swapped) { + q2 = q2.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .b, .ngroups, .h, .hd }).merge(.{ .b = .{ .b, .ngroups } }); + } + + var o = zml.ops.customCall( + Decode.custom_call_name, + .{ + q2, + k_cache, + v_cache, + decode_parameters.block_table, + decode_parameters.cu_seqlens_q, + decode_parameters.seqused_k, + softmax_lse, + softmax_lse_accum, + decode_parameters.metadata.out_accum, + layer_index, + }, + .{q2.shape()}, + .{ + .is_causal = true, + .max_seqlen_k = context.max_seqlen_k, + .max_seqlen_q = context.max_seqlen_q, + .num_heads = num_heads, + }, + .{ .has_side_effect = false }, + ); + + if (seqlenq_ngroups_swapped) { + o = o.splitAxis(.b, .{ .b = original_tot, .ngroups = ngroups }).transpose(.{ .b, .h, .ngroups, .hd }).merge(.{ .h = .{ .h, .ngroups } }); + } + + break :b o; + }, + .mixed => |mixed_parameters| b: { + const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = q.dim(.b) }, .f32)); + const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * q.dim(.b) * q.dim(.h) * 4}, .i8)); + var o = zml.ops.customCall( + Prefill.custom_call_name, + .{ + q, + k_cache, + v_cache, + mixed_parameters.block_table_prefill, + mixed_parameters.cu_seqlens_q_prefill, + mixed_parameters.seqused_k_prefill, + softmax_lse_prefill, + softmax_lse_accum_prefill, + mixed_parameters.metadata.out_accum, + layer_index, + }, + .{q.shape()}, + .{ + .is_causal = true, + .max_seqlen_k = context.max_seqlen_k, + .max_seqlen_q = context.max_seqlen_q, + .num_heads = q.dim(.h), + }, + .{ .has_side_effect = false }, + ); + + const batch_size_decode = mixed_parameters.block_table_prefill.dim(0); + const softmax_lse_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = batch_size_decode }, .f32)); + const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * mixed_parameters.block_table_decode.dim(0) * q.dim(.h) * 4}, .i8)); + var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_size_decode }); + + const original_tot = q_decode.dim(.b); + const num_heads = q_decode.dim(.h); + const num_heads_k = k_cache.dim(.h); + const head_size = q_decode.dim(.hd); + const ngroups = @divExact(num_heads, num_heads_k); + const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0; + if (seqlenq_ngroups_swapped) { + q_decode = q_decode.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .b, .ngroups, .h, .hd }).merge(.{ .b = .{ .b, .ngroups } }); + } + + var o_decode = zml.ops.customCall( + Decode.custom_call_name, + .{ + q_decode, + k_cache, + v_cache, + mixed_parameters.block_table_decode, + mixed_parameters.cu_seqlens_q_decode, + mixed_parameters.seqused_k_decode, + softmax_lse_decode, + softmax_lse_accum_decode, + mixed_parameters.metadata.out_accum, + layer_index, + }, + .{q_decode.shape()}, + .{ + .is_causal = true, + .max_seqlen_k = context.max_seqlen_k, + .max_seqlen_q = 1, + .num_heads = q.dim(.h), + }, + .{ .has_side_effect = false }, + ); + + if (seqlenq_ngroups_swapped) { + o_decode = o_decode.splitAxis(.b, .{ .b = original_tot, .ngroups = ngroups }).transpose(.{ .b, .h, .ngroups, .hd }).merge(.{ .h = .{ .h, .ngroups } }); + } + + o = o.dynamicUpdateSlice1d(o_decode, 0, context.decode_offset.?); + break :b o; + }, + }; + + return o; + } +}; + +pub const paged_fa3 = struct {}; diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig new file mode 100644 index 00000000..acf99b5a --- /dev/null +++ b/zml/attention/paged_attention.zig @@ -0,0 +1,55 @@ +const std = @import("std"); + +const zml = @import("../zml.zig"); +const flashattn = @import("flashattn.zig"); + +const PagedAttention = @This(); + +pub const Backend = enum { + cuda_fa2, +}; + +pub const Options = union(Backend) { + cuda_fa2: flashattn.paged_fa2.Options, + + pub fn isPrefill(self: Options) bool { + return switch (self) { + inline else => |v| v.isPrefill(), + }; + } +}; + +pub const Parameters = union(Backend) { + cuda_fa2: flashattn.paged_fa2.Parameters, + + pub fn init(options_: Options) Parameters { + return switch (options_) { + .cuda_fa2 => |cuda_fa2_options| .{ .cuda_fa2 = flashattn.paged_fa2.Parameters.init(cuda_fa2_options) }, + }; + } + + pub fn options(self: Parameters) Options { + return switch (self) { + .cuda_fa2 => |v| .{ .cuda_fa2 = v.options() }, + }; + } +}; + +/// Internal state that can be used inside the model code. It's derived from Parameters and Option. +pub const Context = union(Backend) { + cuda_fa2: flashattn.paged_fa2.Context, + + pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { + return switch (parameters) { + .cuda_fa2 => |cuda_fa2_parameters| .{ .cuda_fa2 = flashattn.paged_fa2.Context.init(cuda_fa2_parameters, num_heads, num_kv_heads, head_dim, page_size) }, + }; + } +}; + +pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor) zml.Tensor { + _ = k; // autofix + _ = v; // autofix + return switch (parameters) { + .cuda_fa2 => |cuda_fa2_parameters| flashattn.paged_fa2.pagedAttention(cuda_fa2_parameters, context.cuda_fa2, q, k_cache, v_cache, layer_index), + }; +} diff --git a/zml/zml.zig b/zml/zml.zig index 4d576110..045d5072 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -13,7 +13,7 @@ const runfiles = @import("runfiles"); pub const stdx = @import("stdx"); pub const tokenizer = @import("zml/tokenizer"); -pub const attention = @import("attention/attention.zig"); +pub const attention = @import("attention.zig"); pub const Buffer = @import("buffer.zig").Buffer; pub const constants = @import("constants.zig"); pub const dtype = @import("dtype.zig"); From ea69b89eee571660e3da03f0f9c8347de5190613 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 23 Jan 2026 20:30:30 +0000 Subject: [PATCH 294/881] wip it works --- examples/llama/llama.zig | 18 +++---- examples/llama/main.zig | 20 +++++--- zml/attention/flashattn.zig | 100 +++++++++++++++++++++++------------- zml/slice.zig | 3 +- 4 files changed, 87 insertions(+), 54 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index fa567455..352dc01c 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -113,8 +113,8 @@ pub const LlamaLM = struct { token_index: Tensor, kv_cache: KvCache, rng: Tensor.Rng, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, ) struct { Tensor, KvCache, Tensor.Rng } { stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Llama ! Expected >=1d tokens and 0d token_index, got: {f} and {f}", .{ tokens_, token_index }); const tokens = tokens_.withPartialTags(.{.s}); @@ -227,8 +227,8 @@ pub const Llama = struct { tokens: Tensor, token_index: Tensor, kv_cache: KvCache, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, ) struct { Tensor, KvCache } { const embeds = embed(self.embed_tokens, tokens); var hidden = embeds; @@ -274,8 +274,8 @@ pub const TransformerLayer = struct { x0: Tensor, token_index: Tensor, kv_cache: KvCache, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, ) struct { Tensor, KvCache } { // Self Attention //log.debug("TransformerLayer({f}) -> {f}", .{ x0, self.input_layernorm.forward(x0) }); @@ -409,8 +409,8 @@ pub const SelfAttn = struct { x: Tensor, token_index: Tensor, kv_cache: KvCache, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, ) struct { Tensor, KvCache } { const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; var q = self.q_proj.forward(x).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); @@ -436,7 +436,7 @@ pub const SelfAttn = struct { k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); - const attn_output = zml.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters); + const attn_output = zml.attention.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters); const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); return .{ self.o_proj.forward(attn), new_kv_cache }; diff --git a/examples/llama/main.zig b/examples/llama/main.zig index b23e6601..aa048103 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -25,7 +25,7 @@ const CliArgs = struct { reader_buffer_size: stdx.flags.ByteSize = .{ .value = 32, .unit = .mib }, writer_buffer_size: stdx.flags.ByteSize = .{ .value = 32, .unit = .mib }, async_limit: ?u32 = null, - backend: ?zml.attention.Backend = null, + backend: ?zml.attention.attention.Backend = null, pub const help = \\Usage: llama --model= [options] @@ -103,7 +103,7 @@ pub fn main(init: std.process.Init) !void { const llama_options: llama.LlamaLM.Options = .{ .max_seq_len = args.seqlen, .sampling_strategy = .{ - .topk = 2, + .topk = 1, .temperature = 1.0, }, }; @@ -121,7 +121,7 @@ pub fn main(init: std.process.Init) !void { defer llama_model.deinit(allocator); const backend = args.backend orelse b: { - const selected = zml.attention.Backend.auto(platform); + const selected = zml.attention.attention.Backend.auto(platform); log.info("Selected backend: {}", .{selected}); break :b selected; }; @@ -187,8 +187,8 @@ pub fn main(init: std.process.Init) !void { var kv_cache_buffers = try llama_parameters.kv_cache.initBuffer(io, platform); defer llama.KvCache.deinitBuffer(&kv_cache_buffers); - var attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = try llama_parameters.attention_metadata.initBuffer(io, platform); - defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); + var attention_metadata_buffers: zml.Bufferized(zml.attention.attention.Metadata) = try llama_parameters.attention_metadata.initBuffer(io, platform); + defer zml.attention.attention.Metadata.deinitBuffer(&attention_metadata_buffers); var tokenizer = try tokenizer_future.await(io); defer tokenizer.deinit(); @@ -269,8 +269,8 @@ const LlamaParameters = struct { token_index: zml.Tensor, kv_cache: llama.KvCache, rng: zml.Tensor.Rng, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, }; const CompileModelResult = struct { @@ -435,7 +435,7 @@ pub fn generateText( prefill_exe: zml.exe.Exe, decode_exe: zml.exe.Exe, kv_cache_buffers: *zml.Bufferized(llama.KvCache), - attention_metadata_buffers: *zml.Bufferized(zml.attention.Metadata), + attention_metadata_buffers: *zml.Bufferized(zml.attention.attention.Metadata), tokenizer: zml.tokenizer.Tokenizer, config: LlamaLM.Config, options: LlamaLM.Options, @@ -483,6 +483,10 @@ pub fn generateText( prefill_results.fill(.{ &prefill_tokens_buffer, kv_cache_buffers, &rng_buffers }); try prefill_tokens_buffer.toSlice(io, prefill_tokens_slice); generated_token_slice.items(u32)[0] = prefill_tokens_slice.items(u32)[prompt_tok.len - 1]; + + std.log.info("\n\ngenerated tokens: {any}\n\n", .{prefill_tokens_slice.constItems(u32)[0..prompt_tok.len]}); + const out = try tokenizer_decoder.decode(prefill_tokens_slice.constItems(u32)[0..prompt_tok.len]); + std.log.info("\n\ngenerated: {s}\n\n", .{out}); } // Prepare for token-by-token generation, diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index ebf87ca9..fa758b2b 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -54,6 +54,13 @@ const Buffer = struct { flashattnDataTypeFromZmlDataType(buffer.shape.dtype()), ); } + + pub fn format( + self: @This(), + writer: *std.Io.Writer, + ) std.Io.Writer.Error!void { + try writer.print("{f}@{*}", .{ self.shape, self.ptr }); + } }; fn getPlatform(call_frame: *ffi.CallFrame) zml.Platform { @@ -124,7 +131,7 @@ fn fixupKvCacheBuffer(buffer: Buffer, layer_index: i64) Buffer { pub fn Wrapper(comptime T: type, run_func: std.meta.DeclEnum(T)) type { return struct { pub fn register(platform: zml.Platform) !void { - try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "host", T.run, .{ .command_buffer_compatible = true }); + try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "cuda", T.run, .{ .command_buffer_compatible = false }); } pub fn run(call_frame: *ffi.CallFrame) callconv(.c) ?*ffi.Error { @@ -561,6 +568,16 @@ pub const paged_fa2 = struct { }; } + pub fn initBuffers(self: *const DecodeMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(DecodeMetadata) { + return .{ + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + }; + } + + pub fn deinitBuffers(self: *const zml.Bufferized(DecodeMetadata)) void { + self.out_accum.deinit(); + } + pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { const shapes_ = DecodeMetadata.shapes(max_token_count, num_heads, head_dim); var allocation_size: usize = 0; @@ -574,7 +591,6 @@ pub const paged_fa2 = struct { batch_size_decode: usize, max_num_pages: usize, max_seqlen_k: usize, - max_seqlen_q: usize, max_token_count: usize, num_heads: usize, head_dim: usize, @@ -632,6 +648,17 @@ pub const paged_fa2 = struct { }; } + pub fn initBuffers(self: *const MixedMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(MixedMetadata) { + return .{ + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + .host_metadata = undefined, + }; + } + + pub fn deinitBuffers(self: *const zml.Bufferized(MixedMetadata)) void { + self.out_accum.deinit(); + } + pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { const shapes_ = MixedMetadata.shapes(max_token_count, num_heads, head_dim); var allocation_size: usize = 0; @@ -643,7 +670,6 @@ pub const paged_fa2 = struct { pub const Context = struct { max_seqlen_k: usize, - max_seqlen_q: usize, decode_offset: ?zml.Tensor = null, //pub fn init(options: Options) zml.ShapeOf(Context) { @@ -668,15 +694,11 @@ pub const paged_fa2 = struct { const max_seqlen_k = switch (parameters) { inline else => |v| v.options.max_seqlen_k, }; - const max_seqlen_q = switch (parameters) { - .mixed => |mixed_parameters| mixed_parameters.options.max_seqlen_q, - else => 1, - }; const decode_offset = switch (parameters) { .mixed => |mixed_parameters| mixed_parameters.metadata.host_metadata.slice1d(0, .{ .end = 1 }).squeeze(0), .decode => null, }; - return .{ .max_seqlen_k = max_seqlen_k, .max_seqlen_q = max_seqlen_q, .decode_offset = decode_offset }; + return .{ .max_seqlen_k = max_seqlen_k, .decode_offset = decode_offset }; } }; @@ -687,23 +709,26 @@ pub const paged_fa2 = struct { const register = Wrapped.register; const run = Wrapped.run; - fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { if (call_frame.registeringHook()) return null; - const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[11]); const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); - const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[3]); - const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); - const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[5]); - const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[6]); - const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); - const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const host_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[10]); const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + const max_seqlen_q = @as([*]i32, @ptrCast(@alignCast(host_metadata.ptr)))[1]; const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { const head_dim = q.shape.dim(2); break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); @@ -711,7 +736,6 @@ pub const paged_fa2 = struct { const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; - const max_seqlen_q: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_q").?; const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; @@ -735,7 +759,7 @@ pub const paged_fa2 = struct { &paged_v.toFlashattnTensor(), &o.toFlashattnTensor(), &cu_seqlens_q.toFlashattnTensor(), - null, + &cu_seqlens_k.toFlashattnTensor(), &seqused_k.toFlashattnTensor(), &block_table.toFlashattnTensor(), &softmax_lse.toFlashattnTensor(), @@ -757,21 +781,22 @@ pub const paged_fa2 = struct { const register = Wrapped.register; const run = Wrapped.run; - fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { if (call_frame.registeringHook()) return null; - const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[10]); const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); - const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); - const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[5]); - const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[6]); - const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); - const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[9]); const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { @@ -781,7 +806,6 @@ pub const paged_fa2 = struct { const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; - const max_seqlen_q: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_q").?; const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; @@ -789,7 +813,7 @@ pub const paged_fa2 = struct { const stream = call_frame.api.stream(ctx); const params: flashattn.FA2MhaVarlenFwdParams = .{ - .max_seqlen_q = max_seqlen_q, + .max_seqlen_q = 1, .max_seqlen_k = max_seqlen_k, .is_causal = is_causal, .softmax_scale = softmax_scale, @@ -805,7 +829,7 @@ pub const paged_fa2 = struct { &paged_v.toFlashattnTensor(), &o.toFlashattnTensor(), &cu_seqlens_q.toFlashattnTensor(), - null, + &cu_seqlens_k.toFlashattnTensor(), &seqused_k.toFlashattnTensor(), &block_table.toFlashattnTensor(), &softmax_lse.toFlashattnTensor(), @@ -829,13 +853,14 @@ pub const paged_fa2 = struct { .decode => |decode_parameters| b: { const softmax_lse = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.dim(.h), .q = q.dim(.b) }, .f32)); const softmax_lse_accum = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * decode_parameters.block_table.dim(0) * q.dim(.h) * 4}, .i8)); + const dummy_cu_seqlens_k = zml.Tensor.constant(zml.DataType.i32.zero()).broad(decode_parameters.cu_seqlens_q.shape()); const original_tot = q.dim(.b); const num_heads = q.dim(.h); const num_heads_k = k_cache.dim(.h); const head_size = q.dim(.hd); const ngroups = @divExact(num_heads, num_heads_k); - const seqlenq_ngroups_swapped = context.max_seqlen_q == 1 and num_heads > num_heads_k and @mod(head_size, 8) == 0; + const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0; var q2 = q; if (seqlenq_ngroups_swapped) { q2 = q2.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .b, .ngroups, .h, .hd }).merge(.{ .b = .{ .b, .ngroups } }); @@ -847,9 +872,10 @@ pub const paged_fa2 = struct { q2, k_cache, v_cache, - decode_parameters.block_table, decode_parameters.cu_seqlens_q, + dummy_cu_seqlens_k, decode_parameters.seqused_k, + decode_parameters.block_table, softmax_lse, softmax_lse_accum, decode_parameters.metadata.out_accum, @@ -859,7 +885,6 @@ pub const paged_fa2 = struct { .{ .is_causal = true, .max_seqlen_k = context.max_seqlen_k, - .max_seqlen_q = context.max_seqlen_q, .num_heads = num_heads, }, .{ .has_side_effect = false }, @@ -874,25 +899,27 @@ pub const paged_fa2 = struct { .mixed => |mixed_parameters| b: { const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = q.dim(.b) }, .f32)); const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * q.dim(.b) * q.dim(.h) * 4}, .i8)); + const dummy_cu_seqlens_k_prefill = zml.Tensor.constant(zml.DataType.i32.zero()).broad(mixed_parameters.cu_seqlens_q_prefill.shape()); var o = zml.ops.customCall( Prefill.custom_call_name, .{ q, k_cache, v_cache, - mixed_parameters.block_table_prefill, mixed_parameters.cu_seqlens_q_prefill, + dummy_cu_seqlens_k_prefill, mixed_parameters.seqused_k_prefill, + mixed_parameters.block_table_prefill, softmax_lse_prefill, softmax_lse_accum_prefill, mixed_parameters.metadata.out_accum, + mixed_parameters.metadata.host_metadata, layer_index, }, .{q.shape()}, .{ - .is_causal = true, + .is_causal = false, .max_seqlen_k = context.max_seqlen_k, - .max_seqlen_q = context.max_seqlen_q, .num_heads = q.dim(.h), }, .{ .has_side_effect = false }, @@ -901,6 +928,7 @@ pub const paged_fa2 = struct { const batch_size_decode = mixed_parameters.block_table_prefill.dim(0); const softmax_lse_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = batch_size_decode }, .f32)); const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * mixed_parameters.block_table_decode.dim(0) * q.dim(.h) * 4}, .i8)); + const dummy_cu_seqlens_k_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(mixed_parameters.cu_seqlens_q_decode.shape()); var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_size_decode }); const original_tot = q_decode.dim(.b); @@ -919,9 +947,10 @@ pub const paged_fa2 = struct { q_decode, k_cache, v_cache, - mixed_parameters.block_table_decode, mixed_parameters.cu_seqlens_q_decode, + dummy_cu_seqlens_k_decode, mixed_parameters.seqused_k_decode, + mixed_parameters.block_table_decode, softmax_lse_decode, softmax_lse_accum_decode, mixed_parameters.metadata.out_accum, @@ -931,7 +960,6 @@ pub const paged_fa2 = struct { .{ .is_causal = true, .max_seqlen_k = context.max_seqlen_k, - .max_seqlen_q = 1, .num_heads = q.dim(.h), }, .{ .has_side_effect = false }, diff --git a/zml/slice.zig b/zml/slice.zig index d9fb9956..e33316db 100644 --- a/zml/slice.zig +++ b/zml/slice.zig @@ -1,9 +1,9 @@ const std = @import("std"); const stdx = @import("stdx"); -const floats = @import("floats.zig"); const DataType = @import("dtype.zig").DataType; +const floats = @import("floats.zig"); const Shape = @import("shape.zig").Shape; pub fn isBytes(comptime T: type) bool { @@ -25,6 +25,7 @@ pub const Slice = struct { pub fn init(shape: Shape, bytes: anytype) Slice { stdx.debug.assertComptime(isBytes(@TypeOf(bytes)), "Expected \"bytes\" to be a []u8 or []const u8, got {s}", .{@typeName(@TypeOf(bytes))}); + stdx.debug.assert(bytes.len == shape.byteSize(), "Expected \"bytes\" to have the same length as shape.byteSize() ({}), got {}", .{ shape.byteSize(), bytes.len }); const type_info = @typeInfo(@TypeOf(bytes)); return if (type_info.pointer.is_const) .{ .inner_data = .{ .immutable = bytes }, .shape = shape } From 1e2ace0ced97befd53007c1b844b7373eee507cf Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 27 Jan 2026 09:51:15 +0000 Subject: [PATCH 295/881] wip: argument baking --- zml/exe.zig | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/zml/exe.zig b/zml/exe.zig index f5503d65..67493887 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -90,6 +90,7 @@ pub const Exe = struct { pub const Arguments = struct { flat_buffers: FlatBuffers, expected_shapes: []const Shape, + baked_count: usize = 0, pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize) !Arguments { const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); @@ -118,7 +119,7 @@ pub const Exe = struct { self: *Arguments, current_index: usize = 0, }; - var context: LocalContext = .{ .self = self, .current_index = offset }; + var context: LocalContext = .{ .self = self, .current_index = offset + self.baked_count }; meta.visit(struct { fn cb(context_: *LocalContext, buffer: *const Buffer) void { stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected argument {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); @@ -130,6 +131,30 @@ pub const Exe = struct { } }.cb, &context, &v); } + + pub fn bake(self: *Arguments, v: anytype) void { + const LocalContext = struct { + self: *Arguments, + + current_index: usize = 0, + }; + + var context: LocalContext = .{ .self = self, .current_index = self.baked_count }; + + meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *const Buffer) void { + stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected argument {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); + + for (0..context_.self.flat_buffers.num_devices) |device_index| { + context_.self.flat_buffers.buffers[device_index][context_.current_index] = buffer._shards.get(device_index); + } + + context_.current_index += 1; + } + }.cb, &context, &v); + + self.baked_count = context.current_index; + } }; pub const Results = struct { From 80692454e72255120c9a0a4da90e7a9ba350aaae Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 27 Jan 2026 09:51:54 +0000 Subject: [PATCH 296/881] Added paged attention FA3 --- zml/attention/flashattn.zig | 489 +++++++++++++++++++++++++++++- zml/attention/paged_attention.zig | 8 + 2 files changed, 481 insertions(+), 16 deletions(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index fa758b2b..ec402dc7 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -19,6 +19,8 @@ pub fn register(platform: *zml.Platform) !void { try fa3.register(platform); try paged_fa2.Decode.register(platform); try paged_fa2.Prefill.register(platform); + try paged_fa3.Decode.register(platform); + try paged_fa3.Prefill.register(platform); } } @@ -131,7 +133,7 @@ fn fixupKvCacheBuffer(buffer: Buffer, layer_index: i64) Buffer { pub fn Wrapper(comptime T: type, run_func: std.meta.DeclEnum(T)) type { return struct { pub fn register(platform: zml.Platform) !void { - try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "cuda", T.run, .{ .command_buffer_compatible = false }); + try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "cuda", T.run, .{ .command_buffer_compatible = true }); } pub fn run(call_frame: *ffi.CallFrame) callconv(.c) ?*ffi.Error { @@ -672,20 +674,6 @@ pub const paged_fa2 = struct { max_seqlen_k: usize, decode_offset: ?zml.Tensor = null, - //pub fn init(options: Options) zml.ShapeOf(Context) { - // const max_seqlen_k = switch (options) { - // inline else => |v| v.max_seqlen_k, - // }; - // const decode_offset: ?zml.Shape = switch (options) { - // .mixed => zml.Shape.init(.{1}, .i32), - // .decode => null, - // }; - // return .{ - // .max_seqlen_k = max_seqlen_k, - // .decode_offset = decode_offset, - // }; - //} - pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { _ = num_heads; // autofix _ = num_kv_heads; // autofix @@ -978,4 +966,473 @@ pub const paged_fa2 = struct { } }; -pub const paged_fa3 = struct {}; +pub const paged_fa3 = struct { + pub const Variant = enum { + decode, + mixed, + }; + + pub const Options = union(Variant) { + decode: DecodeOptions, + mixed: MixedOptions, + + pub fn isPrefill(self: Options) bool { + return self == .mixed; + } + }; + + pub const Parameters = union(Variant) { + decode: DecodeParameters, + mixed: MixedParameters, + + pub fn init(options_: Options) Parameters { + return switch (options_) { + .decode => |decode_options| .{ .decode = DecodeParameters.init(decode_options) }, + .mixed => |mixed_options| .{ .mixed = MixedParameters.init(mixed_options) }, + }; + } + + pub fn allocationSize(options_: Options) usize { + return switch (options_) { + .decode => |decode_options| DecodeParameters.allocationSize(decode_options), + .mixed => |mixed_options| MixedParameters.allocationSize(mixed_options), + }; + } + + pub fn options(self: Parameters) Options { + return switch (self) { + .decode => |v| .{ .decode = v.options }, + .mixed => |v| .{ .mixed = v.options }, + }; + } + }; + + pub const DecodeOptions = struct { + batch_size: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_token_count: usize, + num_heads: usize, + head_dim: usize, + }; + + pub const DecodeParameters = struct { + block_table: zml.Tensor, + cu_seqlens_q: zml.Tensor, + seqused_k: zml.Tensor, + + metadata: DecodeMetadata, + options: DecodeOptions, + + pub fn init(options: DecodeOptions) DecodeParameters { + return .{ + .block_table = .init(.{ options.batch_size, options.max_num_pages }, .i32), + .cu_seqlens_q = .init(.{options.batch_size + 1}, .i32), + .seqused_k = .init(.{options.batch_size}, .i32), + .metadata = DecodeMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .options = options, + }; + } + + pub fn allocationSize(options: DecodeOptions) usize { + const shapes_ = DecodeParameters.shapes(options); + var allocation_size: usize = 0; + allocation_size += shapes_.block_table.byteSize(); + allocation_size += shapes_.cu_seqlens_q.byteSize(); + allocation_size += shapes_.seqused_k.byteSize(); + allocation_size += DecodeMetadata.allocationSize(options.max_token_count, options.num_heads, options.head_dim); + + return allocation_size; + } + }; + + pub const DecodeMetadata = struct { + out_accum: zml.Tensor, + + pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) DecodeMetadata { + return .{ + .out_accum = .init(.{16 * max_token_count * num_heads * head_dim * 4}, .i8), + }; + } + + pub fn initBuffers(self: *const DecodeMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(DecodeMetadata) { + return .{ + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + }; + } + + pub fn deinitBuffers(self: *const zml.Bufferized(DecodeMetadata)) void { + self.out_accum.deinit(); + } + + pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { + const shapes_ = DecodeMetadata.shapes(max_token_count, num_heads, head_dim); + var allocation_size: usize = 0; + allocation_size += shapes_.out_accum.byteSize(); + return allocation_size; + } + }; + + pub const MixedOptions = struct { + batch_size_prefill: usize, + batch_size_decode: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_token_count: usize, + num_heads: usize, + head_dim: usize, + }; + + pub const MixedParameters = struct { + block_table_prefill: zml.Tensor, + cu_seqlens_q_prefill: zml.Tensor, + seqused_k_prefill: zml.Tensor, + + block_table_decode: zml.Tensor, + cu_seqlens_q_decode: zml.Tensor, + seqused_k_decode: zml.Tensor, + + metadata: MixedMetadata, + options: MixedOptions, + + pub fn init(options: MixedOptions) MixedParameters { + return .{ + .block_table_prefill = .init(.{ options.batch_size_prefill, options.max_num_pages }, .i32), + .cu_seqlens_q_prefill = .init(.{options.batch_size_prefill + 1}, .i32), + .seqused_k_prefill = .init(.{options.batch_size_prefill}, .i32), + .block_table_decode = .init(.{ options.batch_size_decode, options.max_num_pages }, .i32), + .cu_seqlens_q_decode = .init(.{options.batch_size_decode + 1}, .i32), + .seqused_k_decode = .init(.{options.batch_size_decode}, .i32), + .metadata = MixedMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .options = options, + }; + } + + pub fn allocationSize(options: MixedOptions) usize { + const shapes_ = MixedParameters.shapes(options); + var allocation_size: usize = 0; + allocation_size += shapes_.block_table_decode.byteSize(); + allocation_size += shapes_.cu_seqlens_q_prefill.byteSize(); + allocation_size += shapes_.seqused_k_prefill.byteSize(); + + allocation_size += shapes_.block_table_decode.byteSize(); + allocation_size += shapes_.cu_seqlens_q_decode.byteSize(); + allocation_size += shapes_.seqused_k_decode.byteSize(); + allocation_size += MixedMetadata.allocationSize(options.max_token_count, options.num_heads, options.head_dim); + + return allocation_size; + } + }; + + pub const MixedMetadata = struct { + out_accum: zml.Tensor, + host_metadata: zml.Tensor, + + pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) MixedMetadata { + return .{ + .out_accum = .init(.{32 * max_token_count * num_heads * head_dim * 4}, .i8), + .host_metadata = .init(.{2}, .i32), + }; + } + + pub fn initBuffers(self: *const MixedMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(MixedMetadata) { + return .{ + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + .host_metadata = undefined, + }; + } + + pub fn deinitBuffers(self: *const zml.Bufferized(MixedMetadata)) void { + self.out_accum.deinit(); + } + + pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { + const shapes_ = MixedMetadata.shapes(max_token_count, num_heads, head_dim); + var allocation_size: usize = 0; + allocation_size += shapes_.out_accum.byteSize(); + allocation_size += shapes_.host_metadata.byteSize(); + return allocation_size; + } + }; + + pub const Context = struct { + max_seqlen_k: usize, + decode_offset: ?zml.Tensor = null, + + pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { + _ = num_heads; // autofix + _ = num_kv_heads; // autofix + _ = head_dim; // autofix + _ = page_size; // autofix + const max_seqlen_k = switch (parameters) { + inline else => |v| v.options.max_seqlen_k, + }; + const decode_offset = switch (parameters) { + .mixed => |mixed_parameters| mixed_parameters.metadata.host_metadata.slice1d(0, .{ .end = 1 }).squeeze(0), + .decode => null, + }; + return .{ .max_seqlen_k = max_seqlen_k, .decode_offset = decode_offset }; + } + }; + + pub const Prefill = struct { + pub const custom_call_name = "paged_fa3_prefill"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[11]); + const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); + const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const scheduler_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const host_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[10]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const max_seqlen_q = @as([*]i32, @ptrCast(@alignCast(host_metadata.ptr)))[1]; + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA3MhaFwdParams = .{ + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .softcap = 0.0, + .is_rotary_interleaved = false, + .num_splits = 0, + .sm_margin = 0, + .cp_world_size = 1, + .cp_rank = 0, + }; + + flashattn.fa3_mha_fwd( + &q.toFlashattnTensor(), + &paged_k.toFlashattnTensor(), + &paged_v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + null, + null, + &seqused_k.toFlashattnTensor(), + &block_table.toFlashattnTensor(), + null, + null, + null, + &softmax_lse.toFlashattnTensor(), + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + &scheduler_metadata.toFlashattnTensor(), + null, + null, + ¶ms, + stream, + ); + + return null; + } + }; + + pub const Decode = struct { + pub const custom_call_name = "paged_fa3_decode"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[10]); + const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); + const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const scheduler_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA3MhaFwdParams = .{ + .max_seqlen_q = 1, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .softcap = 0.0, + .is_rotary_interleaved = false, + .num_splits = 0, + .sm_margin = 0, + .cp_world_size = 1, + .cp_rank = 0, + }; + + flashattn.fa3_mha_fwd( + &q.toFlashattnTensor(), + &paged_k.toFlashattnTensor(), + &paged_v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + null, + null, + &seqused_k.toFlashattnTensor(), + &block_table.toFlashattnTensor(), + null, + null, + null, + &softmax_lse.toFlashattnTensor(), + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + &scheduler_metadata.toFlashattnTensor(), + null, + null, + ¶ms, + stream, + ); + + return null; + } + }; + + pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor) zml.Tensor { + stdx.debug.assert(q.shape().hasTags(.{ .b, .h, .hd }), "Expected q to have tags .b, .h, .hd", .{}); + stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); + stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); + + const o = switch (parameters) { + .decode => |decode_parameters| b: { + const softmax_lse = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.dim(.h), .q = q.dim(.b) }, .f32)); + const softmax_lse_accum = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * decode_parameters.block_table.dim(0) * q.dim(.h) * 4}, .i8)); + const scheduler_metadata = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = decode_parameters.block_table.dim(0) + 1 }, .i32)); + + const o = zml.ops.customCall( + Decode.custom_call_name, + .{ + q, + k_cache, + v_cache, + decode_parameters.cu_seqlens_q, + decode_parameters.seqused_k, + decode_parameters.block_table, + softmax_lse, + softmax_lse_accum, + decode_parameters.metadata.out_accum, + scheduler_metadata, + layer_index, + }, + .{q.shape()}, + .{ + .is_causal = true, + .max_seqlen_k = context.max_seqlen_k, + }, + .{ .has_side_effect = false }, + ); + + break :b o; + }, + .mixed => |mixed_parameters| b: { + const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = q.dim(.b) }, .f32)); + const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * q.dim(.b) * q.dim(.h) * 4}, .i8)); + const scheduler_metadata_prefill = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = mixed_parameters.block_table_prefill.dim(0) + 1 }, .i32)); + var o = zml.ops.customCall( + Prefill.custom_call_name, + .{ + q, + k_cache, + v_cache, + mixed_parameters.cu_seqlens_q_prefill, + mixed_parameters.seqused_k_prefill, + mixed_parameters.block_table_prefill, + softmax_lse_prefill, + softmax_lse_accum_prefill, + mixed_parameters.metadata.out_accum, + scheduler_metadata_prefill, + mixed_parameters.metadata.host_metadata, + layer_index, + }, + .{q.shape()}, + .{ + .is_causal = false, + .max_seqlen_k = context.max_seqlen_k, + }, + .{ .has_side_effect = false }, + ); + + const batch_size_decode = mixed_parameters.block_table_prefill.dim(0); + const softmax_lse_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = batch_size_decode }, .f32)); + const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * mixed_parameters.block_table_decode.dim(0) * q.dim(.h) * 4}, .i8)); + const scheduler_metadata_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = mixed_parameters.block_table_decode.dim(0) + 1 }, .i32)); + var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_size_decode }); + + const o_decode = zml.ops.customCall( + Decode.custom_call_name, + .{ + q_decode, + k_cache, + v_cache, + mixed_parameters.cu_seqlens_q_decode, + mixed_parameters.seqused_k_decode, + mixed_parameters.block_table_decode, + softmax_lse_decode, + softmax_lse_accum_decode, + scheduler_metadata_decode, + mixed_parameters.metadata.out_accum, + layer_index, + }, + .{q_decode.shape()}, + .{ + .is_causal = true, + .max_seqlen_k = context.max_seqlen_k, + }, + .{ .has_side_effect = false }, + ); + + o = o.dynamicUpdateSlice1d(o_decode, 0, context.decode_offset.?); + break :b o; + }, + }; + + return o; + } +}; diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index acf99b5a..767ac8ee 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -7,10 +7,12 @@ const PagedAttention = @This(); pub const Backend = enum { cuda_fa2, + cuda_fa3, }; pub const Options = union(Backend) { cuda_fa2: flashattn.paged_fa2.Options, + cuda_fa3: flashattn.paged_fa3.Options, pub fn isPrefill(self: Options) bool { return switch (self) { @@ -21,16 +23,19 @@ pub const Options = union(Backend) { pub const Parameters = union(Backend) { cuda_fa2: flashattn.paged_fa2.Parameters, + cuda_fa3: flashattn.paged_fa3.Parameters, pub fn init(options_: Options) Parameters { return switch (options_) { .cuda_fa2 => |cuda_fa2_options| .{ .cuda_fa2 = flashattn.paged_fa2.Parameters.init(cuda_fa2_options) }, + .cuda_fa3 => |cuda_fa3_options| .{ .cuda_fa3 = flashattn.paged_fa3.Parameters.init(cuda_fa3_options) }, }; } pub fn options(self: Parameters) Options { return switch (self) { .cuda_fa2 => |v| .{ .cuda_fa2 = v.options() }, + .cuda_fa3 => |v| .{ .cuda_fa3 = v.options() }, }; } }; @@ -38,10 +43,12 @@ pub const Parameters = union(Backend) { /// Internal state that can be used inside the model code. It's derived from Parameters and Option. pub const Context = union(Backend) { cuda_fa2: flashattn.paged_fa2.Context, + cuda_fa3: flashattn.paged_fa3.Context, pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { return switch (parameters) { .cuda_fa2 => |cuda_fa2_parameters| .{ .cuda_fa2 = flashattn.paged_fa2.Context.init(cuda_fa2_parameters, num_heads, num_kv_heads, head_dim, page_size) }, + .cuda_fa3 => |cuda_fa3_parameters| .{ .cuda_fa3 = flashattn.paged_fa3.Context.init(cuda_fa3_parameters, num_heads, num_kv_heads, head_dim, page_size) }, }; } }; @@ -51,5 +58,6 @@ pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k _ = v; // autofix return switch (parameters) { .cuda_fa2 => |cuda_fa2_parameters| flashattn.paged_fa2.pagedAttention(cuda_fa2_parameters, context.cuda_fa2, q, k_cache, v_cache, layer_index), + .cuda_fa3 => |cuda_fa3_parameters| flashattn.paged_fa3.pagedAttention(cuda_fa3_parameters, context.cuda_fa3, q, k_cache, v_cache, layer_index), }; } From 6b351b9b4060b25ae63a0584f2b006b7b2333dee Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 27 Jan 2026 09:52:14 +0000 Subject: [PATCH 297/881] zml: added simple tracer support back --- zml/BUILD.bazel | 1 + zml/tracer.zig | 77 +++++++++++++++++++++++++++++++++++++++++++++++++ zml/zml.zig | 1 + 3 files changed, 79 insertions(+) create mode 100644 zml/tracer.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 7f0f3caf..f037d6b2 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -43,6 +43,7 @@ zig_library( "slice.zig", "tensor.zig", "testing.zig", + "tracer.zig", "zml.zig", ], main = "zml.zig", diff --git a/zml/tracer.zig b/zml/tracer.zig new file mode 100644 index 00000000..e2e173ef --- /dev/null +++ b/zml/tracer.zig @@ -0,0 +1,77 @@ +const builtin = @import("builtin"); + +const c = @import("c"); + +pub const Tracer = switch (builtin.os.tag) { + // TODO(cerisier): fix MacOsTracer + // .macos => MacOsTracer, + .linux => if (@hasDecl(c, "ZML_RUNTIME_CUDA")) CudaTracer else FakeTracer, + else => FakeTracer, +}; + +const CudaTracer = struct { + + // Those symbols are defined in cudaProfiler.h but their implementation is in libcuda.so + // They will be bound at call time after libcuda.so is loaded (as a needed dependency of libpjrt_cuda.so). + const cuProfilerStart = @extern(*const fn () callconv(.c) c_int, .{ .name = "cuProfilerStart", .linkage = .weak }) orelse unreachable; + const cuProfilerStop = @extern(*const fn () callconv(.c) c_int, .{ .name = "cuProfilerStop", .linkage = .weak }) orelse unreachable; + + // Those symbols are defined in nvToolsExt.h which we don't want to provide. + // However, we link with libnvToolsExt.so which provides them. + // They will be bound at call time after libnvToolsExt.so is loaded (manually dlopen'ed by us). + const nvtxMarkA = @extern(*const fn ([*:0]const u8) callconv(.c) void, .{ .name = "nvtxMarkA", .linkage = .weak }) orelse unreachable; + const nvtxRangeStartA = @extern(*const fn ([*:0]const u8) callconv(.c) c_int, .{ .name = "nvtxRangeStartA", .linkage = .weak }) orelse unreachable; + const nvtxRangeEnd = @extern(*const fn (c_int) callconv(.c) void, .{ .name = "nvtxRangeEnd", .linkage = .weak }) orelse unreachable; + + pub fn event(message: [:0]const u8) void { + nvtxMarkA(message.ptr); + } + + pub fn frameStart(message: [:0]const u8) u64 { + return @intCast(nvtxRangeStartA(message.ptr)); + } + + pub fn frameEnd(interval_id: u64, message: [:0]const u8) void { + _ = message; + nvtxRangeEnd(@intCast(interval_id)); + return; + } +}; + +const MacOsTracer = struct { + logger: c.os_log_t, + + pub fn event(self: *const MacOsTracer, message: [:0]const u8) void { + const interval_id = c.os_signpost_id_generate(self.logger); + c.zml_os_signpost_event(self.logger, interval_id, message); + } + + pub fn frameStart(self: *const MacOsTracer, message: [:0]const u8) c.os_signpost_id_t { + const interval_id = c.os_signpost_id_generate(self.logger); + c.zml_os_signpost_interval_begin(self.logger, interval_id, message); + return interval_id; + } + + pub fn frameEnd(self: *const MacOsTracer, interval_id: c.os_signpost_id_t, message: [:0]const u8) void { + c.zml_os_signpost_interval_end(self.logger, interval_id, message); + } +}; + +/// Mock tracer for OS which don't have an impl. +const FakeTracer = struct { + pub fn event(message: [:0]const u8) void { + _ = message; + return; + } + + pub fn frameStart(message: [:0]const u8) u64 { + _ = message; + return 0; + } + + pub fn frameEnd(interval_id: u64, message: [:0]const u8) void { + _ = interval_id; + _ = message; + return; + } +}; diff --git a/zml/zml.zig b/zml/zml.zig index 045d5072..22faac3c 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -43,6 +43,7 @@ pub const Slice = slice.Slice; pub const tensor = @import("tensor.zig"); pub const Tensor = tensor.Tensor; pub const testing = @import("testing.zig"); +pub const tracer = @import("tracer.zig"); test "zml" { std.testing.refAllDecls(@This()); From 64223d935bc5ffc4ef24fc7e2d0539e0da67a6bb Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 27 Jan 2026 09:52:36 +0000 Subject: [PATCH 298/881] flashattn lib override --- third_party/flashattn/repo.bzl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/third_party/flashattn/repo.bzl b/third_party/flashattn/repo.bzl index 84781cb6..a500ec9f 100644 --- a/third_party/flashattn/repo.bzl +++ b/third_party/flashattn/repo.bzl @@ -2,12 +2,12 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") def repo(): - http_archive( - name = "flashattn", - url = "https://github.com/zml/flash-attention/releases/download/v0.0.6-rc6/flashattn.tar.gz", - sha256 = "ff44c0657c1e0d41883b6bc034340f68afe9c4f8cddfd9232c039f4c6b5e1da6", - build_file = "//:third_party/flashattn/BUILD.bazel", - ) + #http_archive( + # name = "flashattn", + # url = "https://github.com/zml/flash-attention/releases/download/v0.0.6-rc6/flashattn.tar.gz", + # sha256 = "ff44c0657c1e0d41883b6bc034340f68afe9c4f8cddfd9232c039f4c6b5e1da6", + # build_file = "//:third_party/flashattn/BUILD.bazel", + #) - #new_local_repository(name = "flashattn", build_file="//:third_party/flashattn/BUILD.bazel", path="/home/corendos/flashattn/") + new_local_repository(name = "flashattn", build_file="//:third_party/flashattn/BUILD.bazel", path="/home/corendos/flashattn/") From 9e29f446423a81e3dc239076170ce08e71fc3033 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 27 Jan 2026 10:41:10 +0000 Subject: [PATCH 299/881] small fixes --- zml/attention/flashattn.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index ec402dc7..4f4aa26e 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -1051,7 +1051,7 @@ pub const paged_fa3 = struct { pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) DecodeMetadata { return .{ - .out_accum = .init(.{16 * max_token_count * num_heads * head_dim * 4}, .i8), + .out_accum = .init(.{32 * max_token_count * num_heads * head_dim * 4}, .i8), }; } @@ -1416,8 +1416,8 @@ pub const paged_fa3 = struct { mixed_parameters.block_table_decode, softmax_lse_decode, softmax_lse_accum_decode, - scheduler_metadata_decode, mixed_parameters.metadata.out_accum, + scheduler_metadata_decode, layer_index, }, .{q_decode.shape()}, From 7b3474ab89ac9911802670e0aa77a73ab67cf005 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 27 Jan 2026 16:17:36 +0000 Subject: [PATCH 300/881] third_party/flashattn: bump libflashattn.so --- third_party/flashattn/repo.bzl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/third_party/flashattn/repo.bzl b/third_party/flashattn/repo.bzl index a500ec9f..654620cc 100644 --- a/third_party/flashattn/repo.bzl +++ b/third_party/flashattn/repo.bzl @@ -2,12 +2,12 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") def repo(): - #http_archive( - # name = "flashattn", - # url = "https://github.com/zml/flash-attention/releases/download/v0.0.6-rc6/flashattn.tar.gz", - # sha256 = "ff44c0657c1e0d41883b6bc034340f68afe9c4f8cddfd9232c039f4c6b5e1da6", - # build_file = "//:third_party/flashattn/BUILD.bazel", - #) + http_archive( + name = "flashattn", + url = "https://github.com/zml/flash-attention/releases/download/v0.0.6-rc7/flashattn.tar.gz", + sha256 = "6b8195a5ab285404fcb084122c1c32f342ff695040b8652c9144afd2289b71b3", + build_file = "//:third_party/flashattn/BUILD.bazel", + ) - new_local_repository(name = "flashattn", build_file="//:third_party/flashattn/BUILD.bazel", path="/home/corendos/flashattn/") + #new_local_repository(name = "flashattn", build_file="//:third_party/flashattn/BUILD.bazel", path="/home/corendos/flashattn/") From 1200262da023d05ec773e3d71245573c0dc139cc Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 5 Feb 2026 17:07:07 +0000 Subject: [PATCH 301/881] zml/attention: fix wrong causal option --- zml/attention/flashattn.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 4f4aa26e..f7d1f38b 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -906,7 +906,7 @@ pub const paged_fa2 = struct { }, .{q.shape()}, .{ - .is_causal = false, + .is_causal = true, .max_seqlen_k = context.max_seqlen_k, .num_heads = q.dim(.h), }, From 44c249de07a8041c62ecea5fb234d0d2e8bf9088 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 5 Feb 2026 17:07:33 +0000 Subject: [PATCH 302/881] zml/nn: removed commented code --- zml/nn.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/zml/nn.zig b/zml/nn.zig index cdc2b66b..c53b840b 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -29,7 +29,6 @@ pub const Linear = struct { pub fn forward(self: Linear, x: Tensor) Tensor { var y = x.dot(self.weight.convert(x.dtype()), self.tag); - // log.debug("Linear({*}): {d} -> {d} -> {d}", .{ self, x.dims(), y.dims(), if (self.bias) |bias| y.add(bias).dims() else y.dims() }); return if (self.bias) |bias| y.add(bias.broad(y.shape())) else y; } }; From f81763a11a40a689f6f851f5e48061c9a37f74ce Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 5 Feb 2026 17:08:05 +0000 Subject: [PATCH 303/881] zml/nn: Use higher precision for rmsNorm --- zml/nn.zig | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index c53b840b..0a53de0e 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -90,12 +90,13 @@ pub const LayerNorm = struct { } }; -pub fn rmsNorm(x: Tensor, axis: anytype, eps: f32) Tensor { - const ax = x.axis(axis); +pub fn rmsNorm(x_: Tensor, axis: anytype, eps: f32) Tensor { + const ax = x_.axis(axis); // upcast to improve precision - const variance = x.convert(.f32).powByConst(2).mean(ax); - const rsqrt = Tensor.rsqrt(variance.addConstant(eps)).convert(x.dtype()); - return x.mul(rsqrt.broad(x.shape())); + var x = x_.convert(.f32); + const variance = x.powByConst(2).mean(ax); + const rsqrt = Tensor.rsqrt(variance.addConstant(eps)); + return x.mul(rsqrt.broad(x.shape())).convert(x_.dtype()); } /// Center and scale by the variance. From e4f831cc462ee86f2f8a2d1c7a8893583d173e87 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 5 Feb 2026 17:18:41 +0000 Subject: [PATCH 304/881] zml/io: added TensorStore.hasKey() to check key existence It's a bit hacky since it does an array traversal to see if there is a match. --- zml/io.zig | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/zml/io.zig b/zml/io.zig index 8b8703cf..404843d2 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -120,6 +120,14 @@ pub const TensorStore = struct { return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; } + pub fn hasKey(self: *const View, subkey: []const u8) bool { + var buffer: [256]u8 = undefined; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + return for (self.store.registry.tensors.keys()) |k| { + if (std.mem.startsWith(u8, k, key)) break true; + } else false; + } + pub fn maybeCreateTensor(self: View, subkey: []const u8) ?Tensor { var buffer: [256]u8 = undefined; const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; From e11c68792941b5499aeec9897a41c3818e2e152e Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 5 Feb 2026 17:21:53 +0000 Subject: [PATCH 305/881] zml/attention: Re-introduced allocationSize --- zml/attention/flashattn.zig | 98 +++++++++++++++---------------- zml/attention/paged_attention.zig | 6 ++ 2 files changed, 52 insertions(+), 52 deletions(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index f7d1f38b..354252e4 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -507,10 +507,10 @@ pub const paged_fa2 = struct { }; } - pub fn allocationSize(options_: Options) usize { - return switch (options_) { - .decode => |decode_options| DecodeParameters.allocationSize(decode_options), - .mixed => |mixed_options| MixedParameters.allocationSize(mixed_options), + pub fn allocationSize(self: Parameters) usize { + return switch (self) { + .decode => |decode| decode.allocationSize(), + .mixed => |mixed| mixed.allocationSize(), }; } @@ -549,13 +549,13 @@ pub const paged_fa2 = struct { }; } - pub fn allocationSize(options: DecodeOptions) usize { - const shapes_ = DecodeParameters.shapes(options); + pub fn allocationSize(self: DecodeParameters) usize { var allocation_size: usize = 0; - allocation_size += shapes_.block_table.byteSize(); - allocation_size += shapes_.cu_seqlens_q.byteSize(); - allocation_size += shapes_.seqused_k.byteSize(); - allocation_size += DecodeMetadata.allocationSize(options.max_token_count, options.num_heads, options.head_dim); + + allocation_size += self.block_table.byteSize(); + allocation_size += self.cu_seqlens_q.byteSize(); + allocation_size += self.seqused_k.byteSize(); + allocation_size += self.metadata.allocationSize(); return allocation_size; } @@ -580,10 +580,9 @@ pub const paged_fa2 = struct { self.out_accum.deinit(); } - pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { - const shapes_ = DecodeMetadata.shapes(max_token_count, num_heads, head_dim); + pub fn allocationSize(self: DecodeMetadata) usize { var allocation_size: usize = 0; - allocation_size += shapes_.out_accum.byteSize(); + allocation_size += self.out_accum.byteSize(); return allocation_size; } }; @@ -623,17 +622,16 @@ pub const paged_fa2 = struct { }; } - pub fn allocationSize(options: MixedOptions) usize { - const shapes_ = MixedParameters.shapes(options); + pub fn allocationSize(self: MixedParameters) usize { var allocation_size: usize = 0; - allocation_size += shapes_.block_table_decode.byteSize(); - allocation_size += shapes_.cu_seqlens_q_prefill.byteSize(); - allocation_size += shapes_.seqused_k_prefill.byteSize(); + allocation_size += self.block_table_decode.byteSize(); + allocation_size += self.cu_seqlens_q_prefill.byteSize(); + allocation_size += self.seqused_k_prefill.byteSize(); - allocation_size += shapes_.block_table_decode.byteSize(); - allocation_size += shapes_.cu_seqlens_q_decode.byteSize(); - allocation_size += shapes_.seqused_k_decode.byteSize(); - allocation_size += MixedMetadata.allocationSize(options.max_token_count, options.num_heads, options.head_dim); + allocation_size += self.block_table_decode.byteSize(); + allocation_size += self.cu_seqlens_q_decode.byteSize(); + allocation_size += self.seqused_k_decode.byteSize(); + allocation_size += self.metadata.allocationSize(); return allocation_size; } @@ -661,11 +659,10 @@ pub const paged_fa2 = struct { self.out_accum.deinit(); } - pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { - const shapes_ = MixedMetadata.shapes(max_token_count, num_heads, head_dim); + pub fn allocationSize(self: MixedMetadata) usize { var allocation_size: usize = 0; - allocation_size += shapes_.out_accum.byteSize(); - allocation_size += shapes_.host_metadata.byteSize(); + allocation_size += self.out_accum.byteSize(); + allocation_size += self.host_metadata.byteSize(); return allocation_size; } }; @@ -992,10 +989,10 @@ pub const paged_fa3 = struct { }; } - pub fn allocationSize(options_: Options) usize { - return switch (options_) { - .decode => |decode_options| DecodeParameters.allocationSize(decode_options), - .mixed => |mixed_options| MixedParameters.allocationSize(mixed_options), + pub fn allocationSize(self: Parameters) usize { + return switch (self) { + .decode => |decode| decode.allocationSize(), + .mixed => |mixed| mixed.allocationSize(), }; } @@ -1034,13 +1031,13 @@ pub const paged_fa3 = struct { }; } - pub fn allocationSize(options: DecodeOptions) usize { - const shapes_ = DecodeParameters.shapes(options); + pub fn allocationSize(self: DecodeParameters) usize { var allocation_size: usize = 0; - allocation_size += shapes_.block_table.byteSize(); - allocation_size += shapes_.cu_seqlens_q.byteSize(); - allocation_size += shapes_.seqused_k.byteSize(); - allocation_size += DecodeMetadata.allocationSize(options.max_token_count, options.num_heads, options.head_dim); + + allocation_size += self.block_table.byteSize(); + allocation_size += self.cu_seqlens_q.byteSize(); + allocation_size += self.seqused_k.byteSize(); + allocation_size += self.metadata.allocationSize(); return allocation_size; } @@ -1065,10 +1062,9 @@ pub const paged_fa3 = struct { self.out_accum.deinit(); } - pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { - const shapes_ = DecodeMetadata.shapes(max_token_count, num_heads, head_dim); + pub fn allocationSize(self: DecodeMetadata) usize { var allocation_size: usize = 0; - allocation_size += shapes_.out_accum.byteSize(); + allocation_size += self.out_accum.byteSize(); return allocation_size; } }; @@ -1108,17 +1104,16 @@ pub const paged_fa3 = struct { }; } - pub fn allocationSize(options: MixedOptions) usize { - const shapes_ = MixedParameters.shapes(options); + pub fn allocationSize(self: MixedParameters) usize { var allocation_size: usize = 0; - allocation_size += shapes_.block_table_decode.byteSize(); - allocation_size += shapes_.cu_seqlens_q_prefill.byteSize(); - allocation_size += shapes_.seqused_k_prefill.byteSize(); + allocation_size += self.block_table_decode.byteSize(); + allocation_size += self.cu_seqlens_q_prefill.byteSize(); + allocation_size += self.seqused_k_prefill.byteSize(); - allocation_size += shapes_.block_table_decode.byteSize(); - allocation_size += shapes_.cu_seqlens_q_decode.byteSize(); - allocation_size += shapes_.seqused_k_decode.byteSize(); - allocation_size += MixedMetadata.allocationSize(options.max_token_count, options.num_heads, options.head_dim); + allocation_size += self.block_table_decode.byteSize(); + allocation_size += self.cu_seqlens_q_decode.byteSize(); + allocation_size += self.seqused_k_decode.byteSize(); + allocation_size += self.metadata.allocationSize(); return allocation_size; } @@ -1146,11 +1141,10 @@ pub const paged_fa3 = struct { self.out_accum.deinit(); } - pub fn allocationSize(max_token_count: usize, num_heads: usize, head_dim: usize) usize { - const shapes_ = MixedMetadata.shapes(max_token_count, num_heads, head_dim); + pub fn allocationSize(self: MixedMetadata) usize { var allocation_size: usize = 0; - allocation_size += shapes_.out_accum.byteSize(); - allocation_size += shapes_.host_metadata.byteSize(); + allocation_size += self.out_accum.byteSize(); + allocation_size += self.host_metadata.byteSize(); return allocation_size; } }; diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 767ac8ee..f07eb289 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -38,6 +38,12 @@ pub const Parameters = union(Backend) { .cuda_fa3 => |v| .{ .cuda_fa3 = v.options() }, }; } + + pub fn allocationSize(self: Parameters) usize { + return switch (self) { + inline else => |v| v.allocationSize(), + }; + } }; /// Internal state that can be used inside the model code. It's derived from Parameters and Option. From 1302b0c166668c00a6992f3e8c8fecb377ef5ac4 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 5 Feb 2026 17:22:36 +0000 Subject: [PATCH 306/881] zml/nn: Improved Yarn logic --- zml/nn.zig | 47 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index 0a53de0e..f6a9617f 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -169,35 +169,64 @@ pub const RopeOpts = struct { pub const Yarn = struct { beta_fast: f32 = 32.0, beta_slow: f32 = 1.0, - factor: f32, + factor: ?f32 = null, + mscale: ?f32 = null, + mscale_all_dim: ?f32 = null, truncate: bool = true, original_max_position_embeddings: u32, + rope_theta: f32, + llama_4_scaling_beta: ?f32 = null, + attention_factor: ?f32 = null, }; /// Read a Rope scaling config from HF config.json format. pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !Scaling { - const content = try std.json.Value.jsonParse(allocator, source, options); - if (content == .null) return .default; + return jsonParseFromValue( + allocator, + try std.json.innerParse(std.json.Value, allocator, source, options), + options, + ); + } - if (content != .object) return error.InvalidEnumTag; + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) std.json.ParseFromValueError!Scaling { + _ = allocator; // autofix + _ = options; // autofix + if (source == .null) return .default; - const obj = content.object; + if (source != .object) return error.InvalidEnumTag; + + const obj = source.object; const impl = obj.get("rope_type") orelse return error.MissingField; if (impl != .string) return error.InvalidEnumTag; if (std.mem.eql(u8, impl.string, "llama3")) { // Note: leaky is fine here cause Llama3 struct don't need to allocate memory. - return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, stdx.noalloc, content, .{ .ignore_unknown_fields = true }) }; + return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; } else if (std.mem.eql(u8, impl.string, "yarn")) { - return .{ .yarn = try std.json.parseFromValueLeaky(Yarn, stdx.noalloc, content, .{ .ignore_unknown_fields = true }) }; + return .{ .yarn = try std.json.parseFromValueLeaky(Yarn, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; } else { log.warn("Unsupported Rope implementation: {s}, will use the default one which will produce altered results", .{impl.string}); return .{ .default = {} }; } } + fn getMScale(factor: f32, mscale: f32) f32 { + if (mscale < 1.0) { + return 1.0; + } + return 0.1 * mscale * @log(factor) + 1.0; + } + pub fn attentionScaling(scaling: Scaling) f32 { return switch (scaling) { - .yarn => |yarn| 0.1 * @log(yarn.factor) + 1.0, + .yarn => |yarn| yarn_scaling: { + const attention_factor = if (yarn.attention_factor) |af| af else b: { + break :b if (yarn.mscale != null and yarn.mscale_all_dim != null) + getMScale(yarn.factor.?, yarn.mscale.?) / getMScale(yarn.factor.?, yarn.mscale_all_dim.?) + else + getMScale(yarn.factor.?, 1.0); + }; + break :yarn_scaling attention_factor; + }, else => 1.0, }; } @@ -317,7 +346,7 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { const M: f64 = @floatFromInt(s.original_max_position_embeddings); const f_high = s.beta_fast * (2 * std.math.pi) / M; const f_low = s.beta_slow * (2 * std.math.pi) / M; - const downscaling = 1.0 / s.factor; + const downscaling = 1.0 / s.factor.?; // This isn't a typo: low n have a high frequency, high n have a low frequency. var n_low: f64 = -@log(f_high) / @log(opts.freq_base) * N_f; From 13a5ad77ccbd4ebd2ca4ad19eb678a8bc455da74 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 9 Feb 2026 17:05:11 +0000 Subject: [PATCH 307/881] zml/testing: Made testlayer take a runtime name --- zml/testing.zig | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/zml/testing.zig b/zml/testing.zig index f84d9f62..6fda9241 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -287,10 +287,13 @@ pub fn testLayer( layer: anytype, comptime func: std.meta.DeclEnum(@TypeOf(layer)), activation_store: zml.io.TensorStore.View, - comptime name: []const u8, + name: []const u8, layer_weights: zml.Bufferized(@TypeOf(layer)), opts: CompareOpts, ) !void { + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + const forward = @field(@TypeOf(layer), @tagName(func)); const ArgsT = stdx.meta.Tail(stdx.meta.FnArgs(forward)); @@ -301,13 +304,14 @@ pub fn testLayer( index: usize = 0, }; + const input_name = try std.fmt.allocPrint(arena.allocator(), "{s}.in", .{name}); const input_count = zml.meta.count(zml.Tensor, &args); - const expected_input_count = countWithPrefix(activation_store, name ++ ".in"); + const expected_input_count = countWithPrefix(activation_store, input_name); if (input_count != expected_input_count) { log.warn("Reference models uses {d} inputs, but implementation uses {d}", .{ expected_input_count, input_count }); } - const store_input = activation_store.withPrefix(name ++ ".in"); + const store_input = activation_store.withPrefix(input_name); var ctx = LocalContext{ .activation_store = store_input }; try zml.meta.visit(struct { fn cb(ctx_: *LocalContext, tensor: *zml.Tensor) !void { @@ -322,9 +326,10 @@ pub fn testLayer( const exe = try platform.compile(allocator, io, layer, func, args); defer exe.deinit(); + const output_name = try std.fmt.allocPrint(arena.allocator(), "{s}.out", .{name}); const output_count = exe.output_shapes.len; - const store_output = activation_store.withPrefix(name ++ ".out"); - const expected_output_count = countWithPrefix(store_output, name ++ ".out"); + const store_output = activation_store.withPrefix(output_name); + const expected_output_count = countWithPrefix(store_output, output_name); if (output_count != expected_output_count) { log.warn("Reference models produces {d} outputs, but implementation produces {d}", .{ expected_output_count, output_count }); } @@ -365,7 +370,7 @@ pub fn testLayer( try reader.interface.readSliceAll(expected_slice.data()); break :b expected_slice; }; - defer allocator.free(expected_slice.data()); + defer expected_slice.free(allocator); const output_slice = try results[i].toSliceAlloc(allocator, io); defer output_slice.free(allocator); @@ -375,7 +380,7 @@ pub fn testLayer( const stderr = std.debug.lockStderr(&.{}); defer std.debug.unlockStderr(); const w = &stderr.file_writer.interface; - try w.print("{s}.{d} doesn't match !", .{ name ++ ".out", i }); + try w.print("{s}.{d} doesn't match !", .{ output_name, i }); failed = true; continue; }, From bbe3af7c67461017a7fafa2bfb36d0b9bc41758f Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 9 Feb 2026 17:10:11 +0000 Subject: [PATCH 308/881] zml: added ops.scaledDot() It implements a scaled dot product --- zml/ops.zig | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/zml/ops.zig b/zml/ops.zig index 757949cd..81b5022a 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1063,3 +1063,10 @@ fn toUsize(values: anytype) stdx.BoundedArray(usize, constants.MAX_RANK) { for (values) |val| res.appendAssumeCapacity(@intCast(val)); return res; } + +pub fn scaled_dot(lhs: Tensor, rhs: Tensor, lhs_scale: Tensor, rhs_scale: Tensor, args: anytype) Tensor { + const real_lhs = lhs.convert(.bf16).mul(lhs_scale.convert(.bf16)); + const real_rhs = rhs.convert(.bf16).mul(rhs_scale.convert(.bf16)); + + return real_lhs.dot(real_rhs, args).convert(.bf16); +} From 0f694a89033c388b1a8c2c144cd6b9d42fda2d15 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 9 Feb 2026 17:29:31 +0000 Subject: [PATCH 309/881] zml: added some convenience to paged_attention --- zml/attention/paged_attention.zig | 71 +++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index f07eb289..77f49125 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const stdx = @import("stdx"); const zml = @import("../zml.zig"); const flashattn = @import("flashattn.zig"); @@ -8,12 +9,82 @@ const PagedAttention = @This(); pub const Backend = enum { cuda_fa2, cuda_fa3, + + pub fn auto(platform: zml.Platform) Backend { + return switch (platform.target) { + .cuda => b: { + const first_device = platform.pjrt_client.devices(platform.pjrt_api)[0]; + + if (zml.platform.cuda.tryGetComputeCapabilities(platform, first_device)) |cc| { + if (std.mem.eql(u8, cc, "9.0")) { + break :b .cuda_fa3; + } + } + + break :b .cuda_fa2; + }, + else => stdx.debug.panic("Paged attention is not supported on {s} yet", .{@tagName(platform.target)}), + }; + } }; pub const Options = union(Backend) { cuda_fa2: flashattn.paged_fa2.Options, cuda_fa3: flashattn.paged_fa3.Options, + pub fn fromBackend(backend: Backend, is_prefill: bool, batch_size: u32, seq_len: u32, page_chunk_size: u32, max_token_count: u32, num_attention_heads: u32, head_dim: u32) Options { + return switch (backend) { + .cuda_fa2 => if (is_prefill) .{ + .cuda_fa2 = .{ + .mixed = .{ + .batch_size_decode = batch_size, + .batch_size_prefill = batch_size, + .max_num_pages = seq_len / page_chunk_size, + .max_seqlen_k = seq_len, + .max_token_count = max_token_count, + .num_heads = num_attention_heads, + .head_dim = head_dim, + }, + }, + } else .{ + .cuda_fa2 = .{ + .decode = .{ + .batch_size = batch_size, + .max_num_pages = seq_len / page_chunk_size, + .max_seqlen_k = seq_len, + .max_token_count = max_token_count, + .num_heads = num_attention_heads, + .head_dim = head_dim, + }, + }, + }, + .cuda_fa3 => if (is_prefill) .{ + .cuda_fa3 = .{ + .mixed = .{ + .batch_size_decode = batch_size, + .batch_size_prefill = batch_size, + .max_num_pages = seq_len / page_chunk_size, + .max_seqlen_k = seq_len, + .max_token_count = max_token_count, + .num_heads = num_attention_heads, + .head_dim = head_dim, + }, + }, + } else .{ + .cuda_fa3 = .{ + .decode = .{ + .batch_size = batch_size, + .max_num_pages = seq_len / page_chunk_size, + .max_seqlen_k = seq_len, + .max_token_count = max_token_count, + .num_heads = num_attention_heads, + .head_dim = head_dim, + }, + }, + }, + }; + } + pub fn isPrefill(self: Options) bool { return switch (self) { inline else => |v| v.isPrefill(), From 87d3eb83ed9d3fe970a3dba1a0ae6a8ca8918f01 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 12 Feb 2026 14:50:17 +0000 Subject: [PATCH 310/881] zml/attention: update to latest zml/v2 changes --- zml/attention/flashattn.zig | 4 ++-- zml/attention/paged_attention.zig | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 354252e4..0a11629b 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -13,7 +13,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { } } -pub fn register(platform: *zml.Platform) !void { +pub fn register(platform: *const zml.Platform) !void { if (comptime platforms.isEnabled(.cuda)) { try fa2.register(platform); try fa3.register(platform); @@ -132,7 +132,7 @@ fn fixupKvCacheBuffer(buffer: Buffer, layer_index: i64) Buffer { pub fn Wrapper(comptime T: type, run_func: std.meta.DeclEnum(T)) type { return struct { - pub fn register(platform: zml.Platform) !void { + pub fn register(platform: *const zml.Platform) !void { try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "cuda", T.run, .{ .command_buffer_compatible = true }); } diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 77f49125..29f05876 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -10,7 +10,7 @@ pub const Backend = enum { cuda_fa2, cuda_fa3, - pub fn auto(platform: zml.Platform) Backend { + pub fn auto(platform: *const zml.Platform) Backend { return switch (platform.target) { .cuda => b: { const first_device = platform.pjrt_client.devices(platform.pjrt_api)[0]; From 2bd356e0f91265e11e941461326126d426499e3e Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 12 Feb 2026 14:54:11 +0000 Subject: [PATCH 311/881] zml/nn: added common `rope_theta` parameter to rope variants --- examples/llama/llama.zig | 2 +- zml/nn.zig | 34 +++++++++++++++++++++++++--------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 352dc01c..0968b83e 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -35,7 +35,7 @@ pub const LlamaLM = struct { rms_norm_eps: f32, hf_rope_impl: bool = true, tie_word_embeddings: bool = false, - rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = {} }, + rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = .{} }, }; pub const Options = struct { diff --git a/zml/nn.zig b/zml/nn.zig index f6a9617f..167925c6 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -143,7 +143,7 @@ test normalizeL2 { pub const RopeOpts = struct { layout: Layout = .sequential, freq_base: f32 = 10_000, - scaling: Scaling = .default, + scaling: Scaling = .{ .default = .{} }, /// There are two layouts corresponding to how to split `x` in real/imag parts. /// * Interleaved means that the real/imag of each scalar is contiguous. @@ -153,10 +153,14 @@ pub const RopeOpts = struct { /// There are several ways to init the scaling aka "inv_freq" pub const Scaling = union(enum) { - default: void, - custom: []const f32, + default: Default, llama3: Llama3, yarn: Yarn, + linear: Scaling.Linear, + + pub const Default = struct { + rope_theta: f32 = 10000, + }; pub const Llama3 = struct { factor: f32, @@ -164,6 +168,7 @@ pub const RopeOpts = struct { low_freq_factor: f32, original_max_position_embeddings: u32, truncate: bool = true, + rope_theta: f32 = 10000, }; pub const Yarn = struct { @@ -179,6 +184,11 @@ pub const RopeOpts = struct { attention_factor: ?f32 = null, }; + pub const Linear = struct { + factor: f32, + rope_theta: f32, + }; + /// Read a Rope scaling config from HF config.json format. pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !Scaling { return jsonParseFromValue( @@ -191,7 +201,7 @@ pub const RopeOpts = struct { pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) std.json.ParseFromValueError!Scaling { _ = allocator; // autofix _ = options; // autofix - if (source == .null) return .default; + if (source == .null) return .{ .default = .{} }; if (source != .object) return error.InvalidEnumTag; @@ -203,9 +213,11 @@ pub const RopeOpts = struct { return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; } else if (std.mem.eql(u8, impl.string, "yarn")) { return .{ .yarn = try std.json.parseFromValueLeaky(Yarn, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; + } else if (std.mem.eql(u8, impl.string, "linear")) { + return .{ .linear = try std.json.parseFromValueLeaky(Scaling.Linear, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; } else { log.warn("Unsupported Rope implementation: {s}, will use the default one which will produce altered results", .{impl.string}); - return .{ .default = {} }; + return .{ .default = .{} }; } } @@ -230,6 +242,13 @@ pub const RopeOpts = struct { else => 1.0, }; } + + pub fn setRopeTheta(self: *Scaling, rope_theta: f32) void { + // NOTE(Corendos): We can use a "inline else" because `rope_theta` is mandatory in each variant. + return switch (self.*) { + inline else => |*s| s.rope_theta = rope_theta, + }; + } }; }; @@ -314,10 +333,7 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { switch (opts.scaling) { .default => {}, - .custom => { - stdx.debug.assert(opts.scaling.custom.len == N, "rope expected custom inv_freq to match half head dimension {d}, got {d}", .{ N, opts.scaling.custom.len }); - @memcpy(inv_freq, opts.scaling.custom); - }, + .linear => @panic("Not implemented"), .llama3 => |s| { // https://arxiv.org/pdf/2309.16039 // After Llama2 they observed that the rope frequencies where too sharp and hurting long distance attention. From 5234081d59b7a59adcc97b2441cd0ce4d3ec51ff Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 12 Feb 2026 14:54:56 +0000 Subject: [PATCH 312/881] zml/testing: formatting fix --- zml/testing.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/testing.zig b/zml/testing.zig index 6fda9241..8e33ed58 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -380,7 +380,7 @@ pub fn testLayer( const stderr = std.debug.lockStderr(&.{}); defer std.debug.unlockStderr(); const w = &stderr.file_writer.interface; - try w.print("{s}.{d} doesn't match !", .{ output_name, i }); + try w.print("{s}.{d} doesn't match !\n", .{ output_name, i }); failed = true; continue; }, From 41f793fd39ec3d496c647b01060e7f9fae206ab1 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 16 Feb 2026 09:05:21 +0000 Subject: [PATCH 313/881] zml/attention: added options for paged_attention --- zml/attention/flashattn.zig | 26 ++++++++++++++++---------- zml/attention/paged_attention.zig | 13 +++++++++---- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 0a11629b..111b7b1c 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -7,6 +7,8 @@ const stdx = @import("stdx"); const zml = @import("../zml.zig"); const ffi = zml.pjrt.ffi; +const AttentionOptions = @import("paged_attention.zig").AttentionOptions; + pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { if (comptime platforms.isEnabled(.cuda)) { try flashattn.load(allocator, io); @@ -829,7 +831,7 @@ pub const paged_fa2 = struct { } }; - pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor) zml.Tensor { + pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { stdx.debug.assert(q.shape().hasTags(.{ .b, .h, .hd }), "Expected q to have tags .b, .h, .hd", .{}); stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); @@ -837,7 +839,7 @@ pub const paged_fa2 = struct { const o = switch (parameters) { .decode => |decode_parameters| b: { const softmax_lse = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.dim(.h), .q = q.dim(.b) }, .f32)); - const softmax_lse_accum = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * decode_parameters.block_table.dim(0) * q.dim(.h) * 4}, .i8)); + const softmax_lse_accum = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{64 * decode_parameters.block_table.dim(0) * q.dim(.h) * 4}, .i8)); const dummy_cu_seqlens_k = zml.Tensor.constant(zml.DataType.i32.zero()).broad(decode_parameters.cu_seqlens_q.shape()); const original_tot = q.dim(.b); @@ -845,7 +847,7 @@ pub const paged_fa2 = struct { const num_heads_k = k_cache.dim(.h); const head_size = q.dim(.hd); const ngroups = @divExact(num_heads, num_heads_k); - const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0; + const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0 and opts.sliding_window < 0; var q2 = q; if (seqlenq_ngroups_swapped) { q2 = q2.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .b, .ngroups, .h, .hd }).merge(.{ .b = .{ .b, .ngroups } }); @@ -868,9 +870,10 @@ pub const paged_fa2 = struct { }, .{q2.shape()}, .{ - .is_causal = true, + .is_causal = opts.is_causal, .max_seqlen_k = context.max_seqlen_k, .num_heads = num_heads, + .window_size_left = opts.sliding_window, }, .{ .has_side_effect = false }, ); @@ -883,7 +886,7 @@ pub const paged_fa2 = struct { }, .mixed => |mixed_parameters| b: { const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = q.dim(.b) }, .f32)); - const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * q.dim(.b) * q.dim(.h) * 4}, .i8)); + const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{64 * q.dim(.b) * q.dim(.h) * 4}, .i8)); const dummy_cu_seqlens_k_prefill = zml.Tensor.constant(zml.DataType.i32.zero()).broad(mixed_parameters.cu_seqlens_q_prefill.shape()); var o = zml.ops.customCall( Prefill.custom_call_name, @@ -903,16 +906,17 @@ pub const paged_fa2 = struct { }, .{q.shape()}, .{ - .is_causal = true, + .is_causal = opts.is_causal, .max_seqlen_k = context.max_seqlen_k, .num_heads = q.dim(.h), + .window_size_left = opts.sliding_window, }, .{ .has_side_effect = false }, ); const batch_size_decode = mixed_parameters.block_table_prefill.dim(0); const softmax_lse_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = batch_size_decode }, .f32)); - const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * mixed_parameters.block_table_decode.dim(0) * q.dim(.h) * 4}, .i8)); + const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{64 * mixed_parameters.block_table_decode.dim(0) * q.dim(.h) * 4}, .i8)); const dummy_cu_seqlens_k_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(mixed_parameters.cu_seqlens_q_decode.shape()); var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_size_decode }); @@ -921,7 +925,7 @@ pub const paged_fa2 = struct { const num_heads_k = k_cache.dim(.h); const head_size = q_decode.dim(.hd); const ngroups = @divExact(num_heads, num_heads_k); - const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0; + const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0 and opts.sliding_window < 0; if (seqlenq_ngroups_swapped) { q_decode = q_decode.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .b, .ngroups, .h, .hd }).merge(.{ .b = .{ .b, .ngroups } }); } @@ -943,9 +947,10 @@ pub const paged_fa2 = struct { }, .{q_decode.shape()}, .{ - .is_causal = true, + .is_causal = opts.is_causal, .max_seqlen_k = context.max_seqlen_k, .num_heads = q.dim(.h), + .window_size_left = opts.sliding_window, }, .{ .has_side_effect = false }, ); @@ -1329,7 +1334,8 @@ pub const paged_fa3 = struct { } }; - pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor) zml.Tensor { + pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { + _ = opts; // autofix stdx.debug.assert(q.shape().hasTags(.{ .b, .h, .hd }), "Expected q to have tags .b, .h, .hd", .{}); stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 29f05876..deb63ec7 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -13,7 +13,7 @@ pub const Backend = enum { pub fn auto(platform: *const zml.Platform) Backend { return switch (platform.target) { .cuda => b: { - const first_device = platform.pjrt_client.devices(platform.pjrt_api)[0]; + const first_device = platform.devices[0].pjrt_device; if (zml.platform.cuda.tryGetComputeCapabilities(platform, first_device)) |cc| { if (std.mem.eql(u8, cc, "9.0")) { @@ -130,11 +130,16 @@ pub const Context = union(Backend) { } }; -pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor) zml.Tensor { +pub const AttentionOptions = struct { + is_causal: bool = true, + sliding_window: i32 = -1, +}; + +pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { _ = k; // autofix _ = v; // autofix return switch (parameters) { - .cuda_fa2 => |cuda_fa2_parameters| flashattn.paged_fa2.pagedAttention(cuda_fa2_parameters, context.cuda_fa2, q, k_cache, v_cache, layer_index), - .cuda_fa3 => |cuda_fa3_parameters| flashattn.paged_fa3.pagedAttention(cuda_fa3_parameters, context.cuda_fa3, q, k_cache, v_cache, layer_index), + .cuda_fa2 => |cuda_fa2_parameters| flashattn.paged_fa2.pagedAttention(cuda_fa2_parameters, context.cuda_fa2, q, k_cache, v_cache, layer_index, opts), + .cuda_fa3 => |cuda_fa3_parameters| flashattn.paged_fa3.pagedAttention(cuda_fa3_parameters, context.cuda_fa3, q, k_cache, v_cache, layer_index, opts), }; } From 73d734cc27fb695138d359a453ad6c3df7772f87 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 16 Feb 2026 09:05:48 +0000 Subject: [PATCH 314/881] zml/nn: added default value for rope_theta + linear scaling --- zml/nn.zig | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index 167925c6..ee79e9a9 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -186,7 +186,7 @@ pub const RopeOpts = struct { pub const Linear = struct { factor: f32, - rope_theta: f32, + rope_theta: f32 = 10000, }; /// Read a Rope scaling config from HF config.json format. @@ -249,6 +249,13 @@ pub const RopeOpts = struct { inline else => |*s| s.rope_theta = rope_theta, }; } + + pub fn getRopeTheta(self: *const Scaling) f32 { + // NOTE(Corendos): We can use a "inline else" because `rope_theta` is mandatory in each variant. + return switch (self.*) { + inline else => |s| s.rope_theta, + }; + } }; }; @@ -333,7 +340,9 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { switch (opts.scaling) { .default => {}, - .linear => @panic("Not implemented"), + .linear => |l| { + for (inv_freq) |*f| f.* /= l.factor; + }, .llama3 => |s| { // https://arxiv.org/pdf/2309.16039 // After Llama2 they observed that the rope frequencies where too sharp and hurting long distance attention. From aa0b4924d67bf1e5eba6cb7b122ad87888030369 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 16 Feb 2026 09:06:06 +0000 Subject: [PATCH 315/881] zml/nn: remove useless convert --- zml/nn.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/nn.zig b/zml/nn.zig index ee79e9a9..a903c712 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -27,7 +27,7 @@ pub const Linear = struct { } pub fn forward(self: Linear, x: Tensor) Tensor { - var y = x.dot(self.weight.convert(x.dtype()), self.tag); + var y = x.dot(self.weight, self.tag); return if (self.bias) |bias| y.add(bias.broad(y.shape())) else y; } From 44f28b0335316c1da3a094c8cb979e3a27d36bb4 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 16 Feb 2026 20:28:59 +0000 Subject: [PATCH 316/881] third_party/zls: allow using outside of ZML repo --- third_party/zls/zls_completion.bzl | 2 +- third_party/zls/zls_toolchain.bzl | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/third_party/zls/zls_completion.bzl b/third_party/zls/zls_completion.bzl index eac7dbd3..9028c386 100644 --- a/third_party/zls/zls_completion.bzl +++ b/third_party/zls/zls_completion.bzl @@ -61,7 +61,7 @@ def zls_completion(name, deps, **kwargs): main = name + ".runner", data = [ "@rules_zig//zig:resolved_toolchain", - "@@//third_party/zls:resolved_toolchain", + "@zml//third_party/zls:resolved_toolchain", name + ".build_runner", ], deps = [ diff --git a/third_party/zls/zls_toolchain.bzl b/third_party/zls/zls_toolchain.bzl index 5ca5fde0..d83e8f1a 100644 --- a/third_party/zls/zls_toolchain.bzl +++ b/third_party/zls/zls_toolchain.bzl @@ -80,7 +80,7 @@ def _zls_toolchains_repos_impl(mctx): ), sha256 = _ZLS_VERSIONS[platform], build_file_content = """\ -load("@@//third_party/zls:zls_toolchain.bzl", "zls_toolchain") +load("@zml//third_party/zls:zls_toolchain.bzl", "zls_toolchain") zls_toolchain(name = "toolchain", bin = "zls") """, ) @@ -98,7 +98,7 @@ zls_toolchains = module_extension( ) def _zls_toolchains_repo_impl(rctx): - build_content = """# @generated by @@//third_party/zls:zls_toolchain.bzl""" + build_content = """# @generated by @zml//third_party/zls:zls_toolchain.bzl""" for platform, config in _ZLS_PLATFORMS.items(): build_content += """ @@ -107,7 +107,7 @@ toolchain( exec_compatible_with = {compatible_with}, target_compatible_with = {compatible_with}, toolchain = "@zls_{platform}//:toolchain", - toolchain_type = "@@//third_party/zls:toolchain_type", + toolchain_type = "@zml//third_party/zls:toolchain_type", ) """.format( platform = platform, From 82c9c258361a1bd94aaa046a98580c6d02e5cd6a Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Thu, 19 Feb 2026 09:27:10 +0100 Subject: [PATCH 317/881] examples/lfm: Add LiquidAI LFM2.5 (#376) --- AGENTS.md | 157 +++++++++ BUILD.bazel | 1 + examples/lfm/BUILD.bazel | 10 + examples/lfm/config.zig | 83 +++++ examples/lfm/discussion.zig | 296 ++++++++++++++++ examples/lfm/inference.zig | 382 +++++++++++++++++++++ examples/lfm/main.zig | 172 ++++++++++ examples/lfm/model.zig | 634 +++++++++++++++++++++++++++++++++++ examples/lfm/test_runner.zig | 233 +++++++++++++ tools/zls.sh | 5 +- zml/attention/flashattn.zig | 10 +- zml/mem.zig | 2 +- zml/tensor.zig | 17 + 13 files changed, 1997 insertions(+), 5 deletions(-) create mode 100644 AGENTS.md create mode 100644 examples/lfm/BUILD.bazel create mode 100644 examples/lfm/config.zig create mode 100644 examples/lfm/discussion.zig create mode 100644 examples/lfm/inference.zig create mode 100644 examples/lfm/main.zig create mode 100644 examples/lfm/model.zig create mode 100644 examples/lfm/test_runner.zig diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..834e379e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,157 @@ +# AGENTS.md - ZML Agent Guide + +This guide is for autonomous coding agents working in this repository. + +## Scope and Environment + +- Primary language: Zig (dev snapshot), built and tested with Bazel. +- Main subsystems: `zml/`, `stdx/`, `mlir/`, `pjrt/`, `platforms/`, `examples/`. +- Bazel version is pinned in `.bazelversion` (`8.5.1`). +- Build mode defaults to debug via `.bazelrc` (`--compilation_mode=dbg`). + +## External Instruction Files + +- Cursor rules: `.cursor/rules/**` and `.cursorrules` are not present. +- Copilot rules: `.github/copilot-instructions.md` is not present. +- If any of these files appear later, treat them as required constraints. + +## Core Build Commands + +```bash +# Build all targets +bazel build //... + +# Build specific targets/packages +bazel build //zml +bazel build //examples/llama +bazel build //examples/mnist + +# Alternate build configs +bazel build --config=debug //... +bazel build --config=alldebug //... +bazel build --config=release //... +bazel build --config=native //... +``` + +### Useful `.bazelrc` config meanings + +- `--config=debug`: optimized backend, Zig frontend debug mode. +- `--config=release`: optimized (`-Ofast`, Zig `release_safe`). +- `--config=alldebug` / `--config=native` / `--config=silent`: debug-all, local CPU tuning, reduced UI noise. + +## Test Commands + +```bash +# Run everything +bazel test //... + +# Common package tests +bazel test //zml:test +bazel test //stdx:test +bazel test //mlir:test +bazel test //zml/tokenizer:test + +# Discover Zig test targets +bazel query 'kind(zig_test, //...)' +``` + +### Run a single test (inside a `zig_test` target) + +`zml/test_runner.zig` supports substring filtering from positional args. +Pass filter with `--test_arg`: + +```bash +# Runs tests whose name contains "normalizeL2" +bazel test //zml:test --test_arg=normalizeL2 + +# Example on another target +bazel test //stdx:test --test_arg=json +``` + +- Matching is substring-based on test function identifier; `.bazelrc` already sets `test --test_output=errors`. + +## Lint / Formatting Commands + +```bash +# Format Bazel/Starlark files +./tools/buildifier.sh + +# Run ZLS through Bazel (editor and language tooling) +./tools/zls.sh +``` + +## Platform / Accelerator Flags + +Append these to `bazel build|test|run` as needed: + +```bash +--@zml//platforms:cuda=true +--@zml//platforms:rocm=true +--@zml//platforms:tpu=true +--@zml//platforms:neuron=true +--@zml//platforms:cpu=false +``` + +## Code Style (Repository Conventions) + +### Imports and namespacing + +- Prefer module imports, then namespaced access. +- Prefer: `const foo = @import("foo.zig"); foo.bar()`. +- Avoid: `const bar = @import("foo.zig").bar` as default style. +- Import specific types directly only when heavily used. +- Keep local imports relative within a package. + +### Formatting and structure + +- Follow Zig style and existing local patterns first. +- Prefer typed anonymous initialization where clear: `const x: Foo = .{ .bar = 1 };`. +- Keep functions focused; pair resource acquisition with nearby `defer`/`errdefer`. +- Add comments only for non-obvious invariants, ownership, or shape semantics. + +### Types, naming, and tensor semantics + +- Types/structs: `PascalCase`. +- Functions/vars/fields: Zig lower camel case. +- Keep names descriptive; avoid short opaque identifiers unless very local. +- Use semantic tensor tags (`.b`, `.s`, `.d`, `.h`, `.hd`, etc.) over raw index assumptions. + +### Errors, assertions, and cleanup + +- Prefer `try`-based propagation. +- Use `catch |err|` when you add useful recovery or context. +- Use `errdefer` on partial initialization paths. +- Prefer `stdx.debug.assert` over bare `std.debug.assert` in this codebase. +- Avoid panic-style control flow except for unrecoverable programmer errors. + +### Logging and CLI entrypoints + +- Use scoped logs (for example `std.log.scoped(.@"zml/io")`). +- CLI binaries commonly define `pub const std_options`. +- Argument parsing usually follows `stdx.flags` conventions. + +## Testing Patterns + +- Tests are usually inline Zig `test "..." {}` blocks near implementation. +- Common helpers: `zml.testing.env`, `zml.testing.expectClose`, `zml.testing.approxEq`. +- `zig_test` targets often use `zml/test_runner.zig` for progress/filter/leak checks. + +## Critical Architecture Guardrails + +- `Tensor` ops in `forward` build an MLIR graph; they do not execute immediately. +- Preserve distinctions: + - `Shape` = metadata only + - `Tensor` = compile-time graph value + - `Buffer` = runtime device allocation + - `Slice` = host-side data view +- `zml.Bufferized(T)` maps tensor-typed model structures to runtime buffer structures. +- Weight loading paths are centered on safetensors + TensorStore/VFS. + +## Practical Agent Workflow + +- Read the target file, nearby tests, and owning `BUILD.bazel`. +- Check whether behavior lives in compile-time graph construction or runtime execution. + +- Run the narrowest relevant tests first (`--test_arg` when possible). +- Run broader package tests if shared code changed. +- Run `./tools/buildifier.sh` when any Bazel files changed. diff --git a/BUILD.bazel b/BUILD.bazel index 5dbfe326..57873f06 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -4,6 +4,7 @@ zls_completion( name = "completion", visibility = ["//visibility:public"], deps = [ + "//examples/lfm", "//examples/llama", "//examples/mnist", "//stdx", diff --git a/examples/lfm/BUILD.bazel b/examples/lfm/BUILD.bazel new file mode 100644 index 00000000..a49cea2a --- /dev/null +++ b/examples/lfm/BUILD.bazel @@ -0,0 +1,10 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_test") + +zig_binary( + name = "lfm", + srcs = ["model.zig", "config.zig", "inference.zig", "discussion.zig", "test_runner.zig"], + main = "main.zig", + visibility = ["//visibility:public"], + deps = ["//zml", "@linenoise"], +) diff --git a/examples/lfm/config.zig b/examples/lfm/config.zig new file mode 100644 index 00000000..82ea9762 --- /dev/null +++ b/examples/lfm/config.zig @@ -0,0 +1,83 @@ +/// Copied from https://github.com/huggingface/transformers/blob/0997c2f2ab08c32c8e2f90aaad06e29a7108535b/src/transformers/models/lfm2/configuration_lfm2.py +/// +/// Configuration parsed from LFM2-style `config.json`. +/// Notes: +/// - `layer_types` is expected to be fully specified (HF can derive it from `full_attn_idxs`). +/// - HF may use `block_ff_dim` as the effective MLP size; keep `intermediate_size` and `block_ff_dim` +/// in sync in the JSON if you rely on both. +/// - HF uses `tie_embedding` as an alias for `tie_word_embeddings`. +/// - `eos_token_id` can differ from the HF default (e.g. 7 in LFM2 configs). +pub const Config = struct { + /// Model class names in the source config. + architectures: []const []const u8, + /// Whether to auto-adjust the FFN dimension. + block_auto_adjust_ff_dim: bool, + /// Base block hidden dimension. + block_dim: u32, + /// Block FFN dimension (HF uses this as the effective `intermediate_size` when present). + block_ff_dim: u32, + /// Multiplier for FFN dimension. + block_ffn_dim_multiplier: f32, + /// MLP init scale in block config. + block_mlp_init_scale: f32, + /// Multiple for intermediate size rounding. + block_multiple_of: u32, + /// RMS norm epsilon within blocks. + block_norm_eps: f32, + /// Output init scale in block config. + block_out_init_scale: f32, + /// Whether to use SwiGLU in blocks. + block_use_swiglu: bool, + /// Whether to use Xavier init in blocks. + block_use_xavier_init: bool, + /// BOS token id. + bos_token_id: u32, + /// Convolution cache length. + conv_L_cache: u32, + /// Whether conv layers use bias. + conv_bias: bool, + /// Convolution dimension. + conv_dim: u32, + /// Whether to use Xavier init in conv layers. + conv_use_xavier_init: bool, + /// Serialized dtype (e.g. "bfloat16"). + dtype: []const u8, + /// EOS token id. + eos_token_id: u32, + /// Hidden size of the model. + hidden_size: u32, + /// Weight init stddev. + initializer_range: f32, + /// FFN hidden dimension (HF may override with `block_ff_dim`). + intermediate_size: u32, + /// Per-layer operator types (e.g. "conv" or "full_attention"). + layer_types: []const []const u8, + /// Max sequence length supported by RoPE. + max_position_embeddings: u32, + /// Model type tag, expected "lfm2". + model_type: []const u8, + /// RMS norm epsilon. + norm_eps: f32, + /// Number of attention heads. + num_attention_heads: u32, + /// Total number of heads (may duplicate `num_attention_heads`). + num_heads: u32, + /// Number of transformer layers. + num_hidden_layers: u32, + /// Number of key/value heads for GQA/MQA. + num_key_value_heads: u32, + /// PAD token id. + pad_token_id: u32, + /// RoPE theta. + rope_theta: f32, + /// Whether to tie embeddings (aka `tie_word_embeddings`). + tie_embedding: bool, + /// Transformers version string. + transformers_version: []const u8, + /// Whether to use KV cache. + use_cache: bool, + /// Whether to use positional encodings. + use_pos_enc: bool, + /// Vocabulary size. + vocab_size: u32, +}; diff --git a/examples/lfm/discussion.zig b/examples/lfm/discussion.zig new file mode 100644 index 00000000..7f6fcd81 --- /dev/null +++ b/examples/lfm/discussion.zig @@ -0,0 +1,296 @@ +const std = @import("std"); + +const c = @import("c"); +const zml = @import("zml"); +const stdx = zml.stdx; +const attention = zml.attention.attention; + +const inference = @import("inference.zig"); +const model = @import("model.zig"); + +const log = std.log.scoped(.lfm); + +pub const DEFAULT_PROMPT: [:0]const u8 = "> "; + +pub const Context = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + model_buffers: *zml.Bufferized(model.Model), + exe: inference.Inference, + tokenizer: zml.tokenizer.Tokenizer, + config: model.Config, + seqlen: u32, + cache_buffers: zml.Bufferized(model.Cache), + attention_metadata_buffers: zml.Bufferized(attention.Metadata), + rng_buf: zml.Bufferized(zml.Tensor.Rng), + generated_token_slice: zml.Slice, + token_pos: u32, + all_tokens: std.ArrayList(u32), + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + model_buffers: *zml.Bufferized(model.Model), + cache: model.Cache, + attention_metadata: attention.Metadata, + exe: inference.Inference, + tokenizer: zml.tokenizer.Tokenizer, + config: model.Config, + seqlen: u32, + ) !Context { + const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); + return .{ + .allocator = allocator, + .io = io, + .platform = platform, + .model_buffers = model_buffers, + .exe = exe, + .tokenizer = tokenizer, + .config = config, + .seqlen = seqlen, + .cache_buffers = try cache.initBuffers(allocator, io, platform), + .attention_metadata_buffers = try attention_metadata.initBuffer(io, platform), + .rng_buf = try zml.Tensor.Rng.initBuffer(platform, seed, io), + .generated_token_slice = try .alloc(allocator, zml.Shape.init(.{ .batch = 1, .seq = 1 }, .u32)), + .token_pos = 0, + .all_tokens = try .initCapacity(allocator, seqlen), + }; + } + + pub fn deinit(self: *Context) void { + model.Cache.unloadBuffers(&self.cache_buffers); + attention.Metadata.deinitBuffer(&self.attention_metadata_buffers); + zml.Tensor.Rng.deinitBuffer(&self.rng_buf); + self.generated_token_slice.free(self.allocator); + self.all_tokens.deinit(self.allocator); + } + + pub fn runOnce(self: *Context, prompt: []const u8) !void { + const prompt_tokens: []const u32 = try tokenizePrompt( + self.allocator, + self.tokenizer, + self.config, + prompt, + ); + defer self.allocator.free(prompt_tokens); + + var stdout = std.Io.File.stdout().writer(self.io, &.{}); + + log.info("Running prefill...", .{}); + try self.runPrefill(prompt_tokens); + + log.info("Running decode...", .{}); + try self.runDecode(&stdout.interface); + } + + pub fn start(self: *Context, prompt: []const u8) !void { + var turn_tokens: []const u32 = try tokenizePrompt( + self.allocator, + self.tokenizer, + self.config, + prompt, + ); + defer self.allocator.free(turn_tokens); + + var stdout = std.Io.File.stdout().writer(self.io, &.{}); + + while (self.remainingTokens() > 0) { + try stdout.interface.writeAll("\x1b[2mprefill...\x1b[0m"); + try stdout.interface.flush(); + + const prefill_start: std.Io.Timestamp = .now(self.io, .awake); + const prefill_pos_before = self.token_pos; + try self.runPrefill(turn_tokens); + const prefill_duration = prefill_start.untilNow(self.io, .awake); + const prefill_tokens: u64 = self.token_pos - prefill_pos_before; + + try stdout.interface.writeAll("\r \r"); + try stdout.interface.flush(); + + const decode_start: std.Io.Timestamp = .now(self.io, .awake); + const decode_pos_before = self.token_pos; + try self.runDecode(&stdout.interface); + const decode_duration = decode_start.untilNow(self.io, .awake); + const decode_tokens: u64 = self.token_pos - decode_pos_before; + + try stdout.interface.writeAll("\n\n"); + + try stdout.interface.print("\x1b[36mprefill\x1b[0m \x1b[2m{D} \xc2\xb7 {:.1}tok/s\x1b[0m \x1b[2m\xe2\x94\x82\x1b[0m \x1b[36mdecode\x1b[0m \x1b[2m{D} \xc2\xb7 {:.1}tok/s\x1b[0m\n\n\n", .{ + stdx.fmt.fmtDuration(prefill_duration), + tokensPerSecond(prefill_duration, prefill_tokens), + stdx.fmt.fmtDuration(decode_duration), + tokensPerSecond(decode_duration, decode_tokens), + }); + try stdout.interface.flush(); + + while (true) { + const line = c.linenoise(DEFAULT_PROMPT.ptr) orelse { + try stdout.interface.print("\x1b[2m{}/{} tokens used\x1b[0m\n\n", .{ self.token_pos, self.seqlen }); + try stdout.interface.flush(); + return; + }; + defer c.linenoiseFree(line); + const input = std.mem.sliceTo(line, 0); + if (input.len == 0) continue; + + self.allocator.free(turn_tokens); + turn_tokens = try tokenizeTurn(self.allocator, self.tokenizer, input); + break; + } + + if (turn_tokens.len > self.remainingTokens()) { + log.warn("Not enough tokens remaining ({} available, {} needed). Consider using a higher seqlen, for example: `--seqlen={d}`.", .{ self.remainingTokens(), turn_tokens.len, std.math.ceilPowerOfTwo(u32, self.seqlen + 1) catch std.math.maxInt(u32) }); + break; + } + } + } + + fn remainingTokens(self: Context) u32 { + return self.seqlen -| self.token_pos; + } + + fn runPrefill(self: *Context, prompt_tokens: []const u32) !void { + try self.all_tokens.appendSlice(self.allocator, prompt_tokens); + + if (self.all_tokens.items.len > self.seqlen) { + return error.PromptTooLong; + } + + const tokens_slice: zml.Slice = try .alloc(self.allocator, .init(.{ .batch = 1, .seq = self.seqlen }, .u32)); + defer tokens_slice.free(self.allocator); + const tokens = tokens_slice.items(u32); + @memset(tokens, self.config.pad_token_id); + @memcpy(tokens[0..self.all_tokens.items.len], self.all_tokens.items); + + var tokens_buf: zml.Buffer = try .fromSlice(self.io, self.platform, tokens_slice); + defer tokens_buf.deinit(); + + // Always prefill from position 0 with all accumulated tokens. + const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{0})); + var tokens_pos_buf: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice); + defer tokens_pos_buf.deinit(); + + const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(self.all_tokens.items.len)})); + var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice); + defer actual_seq_len_buf.deinit(); + + try self.exe.prefill.run(.{ + .allocator = self.allocator, + .io = self.io, + .platform = self.platform, + .model_buffers = self.model_buffers, + .tokens_buf = &tokens_buf, + .tokens_pos_buf = &tokens_pos_buf, + .actual_seq_len_buf = &actual_seq_len_buf, + .rng_buf = &self.rng_buf, + .cache_buffers = &self.cache_buffers, + .attention_metadata_buffers = self.attention_metadata_buffers, + }); + + try tokens_buf.toSlice(self.io, tokens_slice); + self.generated_token_slice.items(u32)[0] = tokens_slice.items(u32)[self.all_tokens.items.len - 1]; + self.token_pos = @intCast(self.all_tokens.items.len); + } + + fn runDecode(self: *Context, writer: *std.Io.Writer) !void { + var tokenizer_decoder = try self.tokenizer.decoder(); + defer tokenizer_decoder.deinit(); + + var current_token_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, self.generated_token_slice); + defer current_token_buffer.deinit(); + + const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{0})); + var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice); + defer actual_seq_len_buf.deinit(); + + const think_start = self.tokenizer.tokenToId("") orelse unreachable; + const think_end = self.tokenizer.tokenToId("") orelse unreachable; + + generation: while (true) { + const generated_token = self.generated_token_slice.items(u32)[0]; + + if (generated_token == self.config.eos_token_id) break :generation; + if (try tokenizer_decoder.next(generated_token)) |chunk| { + if (generated_token == think_start) { + try writer.writeAll("\x1b[2m"); + } + try writer.writeAll(chunk); + if (generated_token == think_end) { + try writer.writeAll("\x1b[0m"); + } + try writer.flush(); + } + + // Accumulate the generated token. + try self.all_tokens.append(self.allocator, generated_token); + + if (self.remainingTokens() == 0) break :generation; + const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{self.token_pos})); + var token_pos_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice); + defer token_pos_buffer.deinit(); + + try self.exe.decode.run(.{ + .allocator = self.allocator, + .io = self.io, + .platform = self.platform, + .model_buffers = self.model_buffers, + .tokens_buf = ¤t_token_buffer, + .tokens_pos_buf = &token_pos_buffer, + .actual_seq_len_buf = &actual_seq_len_buf, + .rng_buf = &self.rng_buf, + .cache_buffers = &self.cache_buffers, + .attention_metadata_buffers = self.attention_metadata_buffers, + }); + + try current_token_buffer.toSlice(self.io, self.generated_token_slice); + self.token_pos += 1; + } + } +}; + +fn tokensPerSecond(duration: std.Io.Duration, tokens: u64) f64 { + if (tokens == 0) return 0; + const seconds = @as(f64, @floatFromInt(duration.toNanoseconds())) / 1e9; + if (seconds <= 0) return 0; + return @as(f64, @floatFromInt(tokens)) / seconds; +} + +fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: model.Config, prompt: []const u8) ![]u32 { + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const im_start = tokenizer.tokenToId("<|im_start|>") orelse return error.NoSuchToken; + const im_end = tokenizer.tokenToId("<|im_end|>") orelse return error.NoSuchToken; + const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; + const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; + const newline = (try encoder.encode("\n"))[0]; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); + try tokens.appendSlice(allocator, &.{ config.bos_token_id, im_start, user, newline }); + try tokens.appendSlice(allocator, try encoder.encode(prompt)); + try tokens.appendSlice(allocator, &.{ im_end, newline }); + try tokens.appendSlice(allocator, &.{ im_start, assistant, newline }); + + return tokens.toOwnedSlice(allocator); +} + +fn tokenizeTurn(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]u32 { + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const im_start = tokenizer.tokenToId("<|im_start|>") orelse return error.NoSuchToken; + const im_end = tokenizer.tokenToId("<|im_end|>") orelse return error.NoSuchToken; + const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; + const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; + const newline = (try encoder.encode("\n"))[0]; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); + try tokens.appendSlice(allocator, &.{ im_end, newline, im_start, user, newline }); + try tokens.appendSlice(allocator, try encoder.encode(prompt)); + try tokens.appendSlice(allocator, &.{ im_end, newline }); + try tokens.appendSlice(allocator, &.{ im_start, assistant, newline }); + + return tokens.toOwnedSlice(allocator); +} diff --git a/examples/lfm/inference.zig b/examples/lfm/inference.zig new file mode 100644 index 00000000..9feabfaf --- /dev/null +++ b/examples/lfm/inference.zig @@ -0,0 +1,382 @@ +const std = @import("std"); + +const zml = @import("zml"); +const stdx = zml.stdx; +const attention = zml.attention.attention; + +const model = @import("model.zig"); + +const log = std.log.scoped(.lfm); + +pub const CompilationOptions = struct { + hidden_dim: usize, + batch_dim: usize, + single: bool, + rng: zml.Tensor.Rng, + cache: model.Cache, + attention_metadata: attention.Metadata, + attention_parameters: attention.Parameters, + + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: attention.Backend, single: bool) CompilationOptions { + stdx.debug.assert(seqlen >= config.conv_L_cache, "seqlen ({}) must be at least conv_L_cache ({})", .{ seqlen, config.conv_L_cache }); + const cache: model.Cache = .{ + .kv = .init(.init(.{ + .layer = mdl.num_attention_layers, + .batch = 1, + .k = seqlen, + .h = config.num_key_value_heads, + .hd = config.hidden_size / config.num_attention_heads, + }, mdl.embed_tokens.weight.dtype())), + .conv = .init(.init(.{ + .layer = mdl.num_conv_layers, + .batch = 1, + .seq = config.conv_L_cache, + .d = config.hidden_size, + }, mdl.embed_tokens.weight.dtype())), + }; + + return .{ + .single = single, + .hidden_dim = config.hidden_size, + .batch_dim = 1, + .rng = .init(), + .cache = cache, + .attention_metadata = .init(.fromBackend(backend, seqlen)), + .attention_parameters = .init(.fromBackend(backend)), + }; + } +}; + +pub const Args = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + model_buffers: *zml.Bufferized(model.Model), + tokens_buf: *zml.Buffer, + tokens_pos_buf: *zml.Buffer, + actual_seq_len_buf: *zml.Buffer, + rng_buf: *zml.Bufferized(zml.Tensor.Rng), + cache_buffers: *zml.Bufferized(model.Cache), + attention_metadata_buffers: zml.Bufferized(attention.Metadata), +}; + +pub const Inference = struct { + prefill: KernelExe, + decode: KernelExe, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + mdl: model.Model, + opts: CompilationOptions, + seqlen: u32, + progress: *std.Progress.Node, + ) !Inference { + if (opts.single) { + const prefill_exe = try compileSingleKernelExe(allocator, io, platform, mdl, opts, seqlen, true, progress); + const decode_exe = try compileSingleKernelExe(allocator, io, platform, mdl, opts, 1, false, progress); + + return .{ + .prefill = .{ .single = prefill_exe }, + .decode = .{ .single = decode_exe }, + }; + } else { + const prefill_exe = try compileComposedKernelExe(allocator, io, platform, mdl, opts, seqlen, true, progress); + const decode_exe = try compileComposedKernelExe(allocator, io, platform, mdl, opts, 1, false, progress); + return .{ + .prefill = .{ .composed = prefill_exe }, + .decode = .{ .composed = decode_exe }, + }; + } + } + + pub fn deinit(self: Inference) void { + self.prefill.deinit(); + self.decode.deinit(); + } +}; + +pub const KernelExe = union(enum) { + single: SingleKernelExe, + composed: ComposedKernelExe, + + pub fn deinit(self: KernelExe) void { + switch (self) { + .single => |exe| exe.deinit(), + .composed => |exe| exe.deinit(), + } + } + + pub fn run(self: *const KernelExe, args: Args) !void { + switch (self.*) { + .single => |*exe| try exe.run(args), + .composed => |*exe| try exe.run(args), + } + } +}; + +pub const SingleKernelExe = struct { + exe: zml.Exe, + + fn deinit(self: SingleKernelExe) void { + self.exe.deinit(); + } + + pub fn run(self: *const SingleKernelExe, args: Args) !void { + _ = args.io; // autofix + _ = args.platform; // autofix + var exe_args = try self.exe.args(args.allocator); + defer exe_args.deinit(args.allocator); + + var results = try self.exe.results(args.allocator); + defer results.deinit(args.allocator); + + exe_args.set(.{ + args.model_buffers, + args.tokens_buf, + args.tokens_pos_buf, + args.actual_seq_len_buf, + args.rng_buf, + args.cache_buffers, + args.attention_metadata_buffers, + }); + self.exe.call(exe_args, &results); + + var new_tokens, var new_cache, var new_rng = results.get(struct { + zml.Buffer, + zml.Bufferized(model.Cache), + zml.Bufferized(zml.Tensor.Rng), + }); + replaceBuffer(args.tokens_buf, &new_tokens); + replaceCacheBuffers(args.cache_buffers, &new_cache); + replaceBuffer(&args.rng_buf._state, &new_rng._state); + } +}; + +fn compileSingleKernelExe(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, mdl: model.Model, opts: CompilationOptions, seqlen: u32, is_prefill: bool, progress: *std.Progress.Node) !SingleKernelExe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start("Compiling single kernel...", 1); + defer node.end(); + const now: std.Io.Timestamp = .now(io, .awake); + defer log.info("Compiled single kernel [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); + + const actual_seq_len: zml.Tensor = .init(.{}, .u32); + const tokens: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32); + const token_position_offset: zml.Tensor = .init(.{ .batch = opts.batch_dim }, .u32); + + const exe = try platform.compile(allocator, io, mdl, .forward, .{ tokens, token_position_offset, actual_seq_len, opts.rng, opts.cache, opts.attention_metadata, opts.attention_parameters, model.ConvParameters{ .is_prefill = is_prefill } }); + + return .{ .exe = exe }; +} + +pub const ComposedKernelExe = struct { + embed_tokens: zml.Exe, + conv_layer: zml.Exe, + attn_layer: zml.Exe, + lm_head: zml.Exe, + + fn deinit(self: ComposedKernelExe) void { + self.embed_tokens.deinit(); + self.conv_layer.deinit(); + self.attn_layer.deinit(); + self.lm_head.deinit(); + } + + /// Orchestrates the 4 sub-executables to replicate Model.forward behavior. + /// + /// Inputs and outputs mirror the monolithic Model.forward signature: + /// - Inputs: model_buffers, tokens, position_offset, actual_seq_len, rng, cache, attention_metadata + /// - Outputs: tokens (mutated), cache (mutated), rng (mutated) + pub fn run(self: *const ComposedKernelExe, args: Args) !void { + // Step 1: embed_tokens — inputs: [weight, tokens] → output: [hidden] + var hidden_buf: zml.Buffer = b: { + var exe_args = try self.embed_tokens.args(args.allocator); + defer exe_args.deinit(args.allocator); + var results = try self.embed_tokens.results(args.allocator); + defer results.deinit(args.allocator); + + exe_args.set(.{ args.model_buffers.embed_tokens, args.tokens_buf }); + self.embed_tokens.call(exe_args, &results); + break :b results.get(zml.Buffer); + }; + defer hidden_buf.deinit(); + + // Step 2: iterate over layers, dispatching to conv_layer or attn_layer exe. + var conv_cache_index_buf: zml.Buffer = try .scalar(args.io, args.platform, @as(u32, 0), .u32); + defer conv_cache_index_buf.deinit(); + var kv_cache_index_buf: zml.Buffer = try .scalar(args.io, args.platform, @as(u32, 0), .u32); + defer kv_cache_index_buf.deinit(); + + for (args.model_buffers.layers) |layer_bufs| { + const exe = switch (layer_bufs.operator) { + .conv => &self.conv_layer, + .self_attn => &self.attn_layer, + }; + + var exe_args = try exe.args(args.allocator); + defer exe_args.deinit(args.allocator); + var results = try exe.results(args.allocator); + defer results.deinit(args.allocator); + + exe_args.set(.{ layer_bufs, &hidden_buf, args.tokens_pos_buf, args.actual_seq_len_buf, args.cache_buffers, &conv_cache_index_buf, &kv_cache_index_buf, args.attention_metadata_buffers }); + exe.call(exe_args, &results); + + // Outputs: { hidden, cache, conv_cache_index, kv_cache_index } + var new_hidden, var new_cache, var new_conv_cache_index, var new_kv_cache_index = results.get(struct { + zml.Buffer, + zml.Bufferized(model.Cache), + zml.Buffer, + zml.Buffer, + }); + replaceBuffer(&hidden_buf, &new_hidden); + replaceCacheBuffers(args.cache_buffers, &new_cache); + replaceBuffer(&conv_cache_index_buf, &new_conv_cache_index); + replaceBuffer(&kv_cache_index_buf, &new_kv_cache_index); + } + + // Step 3: lm_head — inputs: [lm_head buffers, hidden, embed_tokens.weight, tokens, rng] → outputs: [tokens, rng] + { + var exe_args = try self.lm_head.args(args.allocator); + defer exe_args.deinit(args.allocator); + var results = try self.lm_head.results(args.allocator); + defer results.deinit(args.allocator); + + exe_args.set(.{ args.model_buffers.lm_head, hidden_buf, args.model_buffers.embed_tokens, args.tokens_buf, args.rng_buf }); + self.lm_head.call(exe_args, &results); + var new_tokens, var new_rng = results.get(struct { zml.Buffer, zml.Bufferized(zml.Tensor.Rng) }); + replaceBuffer(args.tokens_buf, &new_tokens); + replaceBuffer(&args.rng_buf._state, &new_rng._state); + } + } +}; + +fn replaceCacheBuffers(dst: *zml.Bufferized(model.Cache), src: *zml.Bufferized(model.Cache)) void { + replaceBuffer(&dst.conv.state, &src.conv.state); + replaceBuffer(&dst.kv.k, &src.kv.k); + replaceBuffer(&dst.kv.v, &src.kv.v); +} + +fn replaceBuffer(dst: *zml.Buffer, src: *zml.Buffer) void { + if (!sameBufferHandle(dst.*, src.*)) { + dst.deinit(); + } + dst.* = src.*; +} + +fn sameBufferHandle(a: zml.Buffer, b: zml.Buffer) bool { + if (a._shards.len != b._shards.len) return false; + for (a._shards.constSlice(), b._shards.constSlice()) |a_shard, b_shard| { + if (a_shard != b_shard) return false; + } + return true; +} + +pub fn compileComposedKernelExe( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + mdl: model.Model, + opts: CompilationOptions, + seqlen: u32, + is_prefill: bool, + progress: *std.Progress.Node, +) !ComposedKernelExe { + const embed_tokens_exe = try compileEmbedTokens(allocator, io, platform, mdl.embed_tokens, opts, seqlen, progress); + const conv_layer_exe = try compileConvLayer(allocator, io, platform, mdl, opts, seqlen, is_prefill, progress); + const attn_layer_exe = try compileAttnLayer(allocator, io, platform, mdl, opts, seqlen, is_prefill, progress); + const lm_head_exe = try compileLmHead(allocator, io, platform, mdl, opts, seqlen, progress); + + return .{ + .embed_tokens = embed_tokens_exe, + .conv_layer = conv_layer_exe, + .attn_layer = attn_layer_exe, + .lm_head = lm_head_exe, + }; +} + +fn compileEmbedTokens(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, embed_tokens: model.TokenEmbedding, opts: CompilationOptions, seqlen: u32, progress: *std.Progress.Node) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start("Compiling embed_tokens...", 1); + defer node.end(); + const now: std.Io.Timestamp = .now(io, .awake); + defer log.info("Compiled embed_tokens [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); + + const tokens: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32); + return platform.compile(allocator, io, embed_tokens, .forward, .{tokens}); +} + +fn compileConvLayer(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, mdl: model.Model, opts: CompilationOptions, seqlen: u32, is_prefill: bool, progress: *std.Progress.Node) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start("Compiling conv layer...", 1); + defer node.end(); + const now: std.Io.Timestamp = .now(io, .awake); + defer log.info("Compiled conv layer [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); + + // Find the first conv layer as a representative for compilation. + const conv_layer = for (mdl.layers) |layer| { + if (layer.operator == .conv) break layer; + } else unreachable; + + const hidden: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()); + const token_position_offset: zml.Tensor = .init(.{ .batch = opts.batch_dim }, .u32); + const actual_seq_len: zml.Tensor = .init(.{}, .u32); + const conv_cache_index: zml.Tensor = .init(.{}, .u32); + const kv_cache_index: zml.Tensor = .init(.{}, .u32); + + return platform.compile(allocator, io, conv_layer, .forward, .{ + hidden, + token_position_offset, + actual_seq_len, + opts.cache, + conv_cache_index, + kv_cache_index, + opts.attention_metadata, + opts.attention_parameters, + model.ConvParameters{ .is_prefill = is_prefill }, + }); +} + +fn compileAttnLayer(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, mdl: model.Model, opts: CompilationOptions, seqlen: u32, is_prefill: bool, progress: *std.Progress.Node) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start("Compiling attn layer...", 1); + defer node.end(); + const now: std.Io.Timestamp = .now(io, .awake); + defer log.info("Compiled attn layer [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); + + // Find the first attention layer as a representative for compilation. + const attn_layer = for (mdl.layers) |layer| { + if (layer.operator == .self_attn) break layer; + } else unreachable; + + const hidden: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()); + const token_position_offset: zml.Tensor = .init(.{ .batch = opts.batch_dim }, .u32); + const actual_seq_len: zml.Tensor = .init(.{}, .u32); + const conv_cache_index: zml.Tensor = .init(.{}, .u32); + const kv_cache_index: zml.Tensor = .init(.{}, .u32); + + return platform.compile(allocator, io, attn_layer, .forward, .{ + hidden, + token_position_offset, + actual_seq_len, + opts.cache, + conv_cache_index, + kv_cache_index, + opts.attention_metadata, + opts.attention_parameters, + model.ConvParameters{ .is_prefill = is_prefill }, + }); +} + +fn compileLmHead(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, mdl: model.Model, opts: CompilationOptions, seqlen: u32, progress: *std.Progress.Node) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start("Compiling lm_head...", 1); + defer node.end(); + const now: std.Io.Timestamp = .now(io, .awake); + defer log.info("Compiled lm_head [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); + + const hidden: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()); + const tokens: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32); + + return platform.compile(allocator, io, mdl.lm_head, .forward, .{ hidden, mdl.embed_tokens, tokens, opts.rng }); +} diff --git a/examples/lfm/main.zig b/examples/lfm/main.zig new file mode 100644 index 00000000..6f93fb01 --- /dev/null +++ b/examples/lfm/main.zig @@ -0,0 +1,172 @@ +const std = @import("std"); + +const c = @import("c"); +const zml = @import("zml"); +const attention = zml.attention.attention; +const stdx = zml.stdx; + +const discussion = @import("discussion.zig"); +const inference = @import("inference.zig"); +const model = @import("model.zig"); +const test_runner = @import("test_runner.zig"); + +pub const std_options: std.Options = .{ + .log_level = .info, +}; + +const log = std.log.scoped(.lfm); +const Gpa = std.heap.GeneralPurposeAllocator(.{ .thread_safe = true }); + +const Args = struct { + model: []const u8, + prompt: ?[]const u8 = null, + seqlen: u32 = 2048, + backend: ?attention.Backend = null, + run_tests: bool = false, + single: bool = false, + + pub const help = + \\Use lfm --model= [options] + \\ + \\ Run LFM model inference. + \\ + \\ Options: + \\ --model= Path to the model repository (required) + \\ --prompt= Prompt to use for generation (default: none) + \\ --seqlen= Sequence length (default: 1024) + \\ --backend= Attention backend to use ([vanilla, cuda_fa2, cuda_fa3], default: auto-selection) + \\ --run-tests Run tests instead of inference + \\ --single Create a single kernel encompassing all the layers. + \\ + ; +}; + +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const io = init.io; + + const platform: *zml.Platform = try .auto(allocator, io, .{ .cuda = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.8 } } } }); + defer platform.deinit(allocator); + + log.info("\n{f}", .{platform.fmtVerbose()}); + + const args = stdx.flags.parse(init.minimal.args, Args); + + log.info("Resolving model repository..", .{}); + const repo = try zml.safetensors.resolveModelRepo(io, args.model); + + log.info("Loading configuration...", .{}); + const parsed_config = try loadConfig(model.Config, allocator, io, repo); + defer parsed_config.deinit(); + const config = parsed_config.value; + + log.info("Initializing model...", .{}); + var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); + defer registry.deinit(); + + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + defer store.deinit(); + + var mdl: model.Model = .init(allocator, store.view(), config); + defer mdl.deinit(allocator); + + const attention_backend = args.backend orelse b: { + const selected = attention.Backend.auto(platform); + log.info("Selected backend: {}", .{selected}); + break :b selected; + }; + + var progress = std.Progress.start(io, .{ .root_name = args.model }); + errdefer progress.end(); + + var tokenizer_fut = try io.concurrent(loadTokenizer, .{ allocator, io, repo, &progress }); + defer if (tokenizer_fut.cancel(io)) |tokenizer| { + var tk = tokenizer; + tk.deinit(); + } else |_| {}; + + const params: inference.CompilationOptions = .init(mdl, config, args.seqlen, attention_backend, args.single); + + var compilation_fut = try io.concurrent(inference.Inference.init, .{ allocator, io, platform, mdl, params, args.seqlen, &progress }); + defer if (compilation_fut.cancel(io)) |exe| { + exe.deinit(); + } else |_| {}; + + var load_model_fut = try io.concurrent(model.Model.loadBuffers, .{ + &mdl, + allocator, + io, + platform, + &store, + &progress, + }); + defer if (load_model_fut.cancel(io)) |model_buffers| { + var mb = model_buffers; + model.Model.unloadBuffers(&mb, allocator); + } else |_| {}; + + const exe: inference.Inference = try compilation_fut.await(io); + var model_buffers: zml.Bufferized(model.Model) = try load_model_fut.await(io); + const tokenizer = try tokenizer_fut.await(io); + progress.end(); + + if (args.run_tests) { + return test_runner.run( + allocator, + io, + platform, + config, + mdl, + &model_buffers, + &store, + params.attention_metadata, + params.attention_parameters, + ); + } + + const interactive = args.prompt == null; + const prompt = if (args.prompt) |p| p else blk: { + const line = c.linenoise(discussion.DEFAULT_PROMPT) orelse return; + defer c.linenoiseFree(line); + break :blk try allocator.dupe(u8, std.mem.sliceTo(line, 0)); + }; + defer allocator.free(prompt); + + var ctx = try discussion.Context.init(allocator, io, platform, &model_buffers, params.cache, params.attention_metadata, exe, tokenizer, config, args.seqlen); + defer ctx.deinit(); + + if (interactive) { + return ctx.start(prompt); + } else { + return ctx.runOnce(prompt); + } +} + +pub fn loadConfig(comptime T: type, allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !std.json.Parsed(T) { + const file = try dir.openFile(io, "config.json", .{}); + defer file.close(io); + + var buffer: [256]u8 = undefined; + var file_reader = file.reader(io, &buffer); + var reader: std.json.Reader = .init(allocator, &file_reader.interface); + defer reader.deinit(); + + return try std.json.parseFromTokenSource(model.Config, allocator, &reader, .{ .ignore_unknown_fields = true }); +} + +pub fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, progress: *std.Progress.Node) !zml.tokenizer.Tokenizer { + progress.increaseEstimatedTotalItems(1); + var node = progress.start("Loading tokenizer...", 1); + defer node.end(); + const now: std.Io.Timestamp = .now(io, .awake); + defer log.info("Loaded tokenizer [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); + const bytes = b: { + const file = try dir.openFile(io, "tokenizer.json", .{}); + defer file.close(io); + var reader = file.reader(io, &.{}); + break :b try reader.interface.readAlloc(allocator, try file.length(io)); + }; + defer allocator.free(bytes); + + return try .fromBytes(allocator, io, bytes); +} diff --git a/examples/lfm/model.zig b/examples/lfm/model.zig new file mode 100644 index 00000000..c8925f0d --- /dev/null +++ b/examples/lfm/model.zig @@ -0,0 +1,634 @@ +const std = @import("std"); + +const zml = @import("zml"); +const stdx = zml.stdx; +const Tensor = zml.Tensor; +const Shape = zml.Shape; +const attention = zml.attention.attention; + +const cfg = @import("config.zig"); +pub const Config = cfg.Config; + +const log = std.log.scoped(.lfm); + +pub const Model = struct { + embed_tokens: TokenEmbedding, + lm_head: LmHead, + layers: []DecoderLayer, + num_attention_layers: usize, + num_conv_layers: usize, + + pub fn init(allocator: std.mem.Allocator, root_store: zml.io.TensorStore.View, config: Config) Model { + const store = root_store.withPrefix("model"); + stdx.debug.assert(config.layer_types.len == config.num_hidden_layers, "Expected layer_types len {} to match num_hidden_layers {}", .{ config.layer_types.len, config.num_hidden_layers }); + + const layers = allocator.alloc(DecoderLayer, config.num_hidden_layers) catch unreachable; + var num_attention_layers: usize = 0; + var num_conv_layers: usize = 0; + for (layers, 0..) |*layer, i| { + const layer_store = store.withPrefix("layers").withLayer(i); + const kind = DecoderLayer.parseOperatorKind(config.layer_types[i]); + switch (kind) { + .conv => num_conv_layers += 1, + .full_attention => num_attention_layers += 1, + } + layer.* = DecoderLayer.init(config, layer_store, kind); + } + + return .{ + .embed_tokens = .init(store.withPrefix("embed_tokens")), + .lm_head = LmHead.init(store, config), + .layers = layers, + .num_attention_layers = num_attention_layers, + .num_conv_layers = num_conv_layers, + }; + } + + pub fn loadBuffers( + self: *const Model, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + ) !zml.Bufferized(Model) { + progress.increaseEstimatedTotalItems(store.view().count()); + + const now: std.Io.Timestamp = .now(io, .awake); + var total_bytes: usize = 0; + + defer { + const took = now.untilNow(io, .awake); + const took_ns: usize = @max(1, @as(usize, @intCast(took.toNanoseconds()))); + log.info("Loaded weights [{Bi:.2}, {D}, {Bi:.2}/s]", .{ + total_bytes, + stdx.fmt.fmtDuration(took), + total_bytes * std.time.ns_per_s / took_ns, + }); + } + + return zml.io.load(Model, self, allocator, io, platform, .{ + .dma_chunks = 8, + .dma_chunk_size = 128 * zml.MiB, + .progress = progress, + .store = store, + .parallelism = 16, + .total_bytes = &total_bytes, + }); + } + + pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { + self.embed_tokens.weight.deinit(); + LmHead.unloadBuffers(&self.lm_head); + for (self.layers) |*layer| { + DecoderLayer.unloadBuffers(layer); + } + allocator.free(self.layers); + } + + pub fn forward( + // + self: Model, + tokens: Tensor, + tokens_position_offset: Tensor, + actual_seq_len: Tensor, + rng: Tensor.Rng, + cache_: Cache, + attention_metadata: attention.Metadata, + attention_parameters: attention.Parameters, + conv_parameters: ConvParameters, + ) struct { Tensor, Cache, Tensor.Rng } { + stdx.debug.assert(tokens.shape().hasTags(.{ .batch, .seq }), "Tokens should have tags {{.batch, .seq}}, got {f}", .{tokens.shape()}); + + const embeds = self.embed_tokens.forward(tokens); + + var hidden = embeds; + var cache = cache_; + var conv_cache_index = Tensor.scalar(@as(u32, 0), .u32); + var kv_cache_index = Tensor.scalar(@as(u32, 0), .u32); + for (self.layers) |layer| { + hidden, cache, conv_cache_index, kv_cache_index = layer.forward(hidden, tokens_position_offset, actual_seq_len, cache, conv_cache_index, kv_cache_index, attention_metadata, attention_parameters, conv_parameters); + } + + const new_tokens, const new_rng = self.lm_head.forward(hidden, self.embed_tokens, tokens, rng); + return .{ new_tokens, cache.reuseBuffer(cache_), new_rng }; + } + + pub fn deinit(self: Model, allocator: std.mem.Allocator) void { + allocator.free(self.layers); + } +}; + +pub const TokenEmbedding = struct { + weight: Tensor, + + pub fn init(store: zml.io.TensorStore.View) TokenEmbedding { + return .{ .weight = store.createTensor("weight").withTags(.{ .voc, .d }) }; + } + + pub fn forward(self: TokenEmbedding, tokens: Tensor) Tensor { + stdx.debug.assert(tokens.dtype().isInteger(), "TokenEmbedding expects an integer input, received: {f}", .{tokens}); + stdx.debug.assert(self.weight.rank() == 2, "TokenEmbedding expects it's weight Tensor to be a 2D matrix, got {f}", .{self.weight}); + return self.weight.gather(.{ .voc = tokens }, .{}); + } + + pub fn unembed(self: TokenEmbedding, embeds: Tensor) Tensor { + stdx.debug.assert(embeds.shape().hasTags(.{.d}), "TokenEmbedding expects the input embeds to have a .d tag, got {f}", .{embeds.shape()}); + return self.weight.dot(embeds, .d); + } +}; + +pub const LmHead = struct { + embedding_norm: RmsNorm, + + pub fn init(store: zml.io.TensorStore.View, config: Config) LmHead { + return .{ + .embedding_norm = RmsNorm.init(store.withPrefix("embedding_norm"), config.norm_eps, .d), + }; + } + + pub fn forward(self: LmHead, hidden: Tensor, embed_tokens: TokenEmbedding, tokens: Tensor, rng: Tensor.Rng) struct { Tensor, Tensor.Rng } { + const logits = embed_tokens.unembed(self.embedding_norm.forward(hidden)); + const new_tokens, const new_rng = zml.nn.sampleTokens(logits, .{}, rng); + return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), new_rng }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(LmHead)) void { + RmsNorm.unloadBuffers(&self.embedding_norm); + } +}; + +pub const OperatorKind = enum { conv, full_attention }; +const Operator = union(enum) { + conv: ShortConv, + self_attn: Attention, +}; + +pub const DecoderLayer = struct { + operator: Operator, + operator_norm: RmsNorm, + ffn_norm: RmsNorm, + feed_forward: Mlp, + + pub fn parseOperatorKind(layer_type: []const u8) OperatorKind { + return std.meta.stringToEnum(OperatorKind, layer_type) orelse { + stdx.debug.assert(false, "Unsupported layer type {s}", .{layer_type}); + unreachable; + }; + } + + pub fn init(config: Config, store: zml.io.TensorStore.View, kind: OperatorKind) DecoderLayer { + const operator_norm = RmsNorm.init(store.withPrefix("operator_norm"), config.norm_eps, .d); + const ffn_norm = RmsNorm.init(store.withPrefix("ffn_norm"), config.norm_eps, .d); + const feed_forward = Mlp.init(store.withPrefix("feed_forward")); + const operator: Operator = switch (kind) { + .conv => .{ .conv = ShortConv.init(config, store.withPrefix("conv")) }, + .full_attention => .{ .self_attn = Attention.init(config, store.withPrefix("self_attn")) }, + }; + + return .{ + .operator = operator, + .operator_norm = operator_norm, + .ffn_norm = ffn_norm, + .feed_forward = feed_forward, + }; + } + + pub fn forward( + self: DecoderLayer, + input: Tensor, + tokens_position_offset: Tensor, + actual_seq_len: Tensor, + cache_: Cache, + conv_cache_index_: Tensor, + kv_cache_index_: Tensor, + attention_metadata: attention.Metadata, + attention_parameters: attention.Parameters, + conv_parameters: ConvParameters, + ) struct { Tensor, Cache, Tensor, Tensor } { + var cache = cache_; + var conv_cache_index = conv_cache_index_; + var kv_cache_index = kv_cache_index_; + const residual = switch (self.operator) { + .conv => |operator| b: { + const residual, const updated_conv_cache = operator.forward( + self.operator_norm.forward(input), + tokens_position_offset, + actual_seq_len, + cache.conv, + conv_cache_index, + conv_parameters, + ); + cache.conv = updated_conv_cache; + conv_cache_index = conv_cache_index.add(Tensor.scalar(@as(u32, 1), .u32)); + break :b residual; + }, + .self_attn => |operator| b: { + const residual, const updated_kv_cache = operator.forward( + self.operator_norm.forward(input), + tokens_position_offset, + cache.kv, + kv_cache_index, + attention_metadata, + attention_parameters, + ); + cache.kv = updated_kv_cache; + kv_cache_index = kv_cache_index.add(Tensor.scalar(@as(u32, 1), .u32)); + break :b residual; + }, + }; + const x = input.add(residual); + return .{ + x.add(self.feed_forward.forward(self.ffn_norm.forward(x))).reuseBuffer(input), + cache.reuseBuffer(cache_), + conv_cache_index.reuseBuffer(conv_cache_index_), + kv_cache_index.reuseBuffer(kv_cache_index_), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(DecoderLayer)) void { + switch (self.operator) { + .conv => |*operator| ShortConv.unloadBuffers(operator), + .self_attn => |*operator| Attention.unloadBuffers(operator), + } + RmsNorm.unloadBuffers(&self.operator_norm); + RmsNorm.unloadBuffers(&self.ffn_norm); + Mlp.unloadBuffers(&self.feed_forward); + } +}; + +pub const ConvParameters = struct { is_prefill: bool }; + +pub const ShortConv = struct { + in_proj: Linear, + out_proj: Linear, + kernel: zml.Tensor, + config: Config, + + pub fn init(config: Config, store: zml.io.TensorStore.View) ShortConv { + stdx.debug.assert(!config.conv_bias, "conv_bias is not supported.", .{}); + return ShortConv{ .in_proj = initLinear(store.withPrefix("in_proj"), .d), .out_proj = initLinear(store.withPrefix("out_proj"), .d), .kernel = store.createTensorWithTags("conv.weight", .{ .out, .in, .kernel_size }), .config = config }; + } + + pub fn forward( + self: ShortConv, + input: Tensor, + tokens_position_offset: Tensor, + actual_seq_len: Tensor, + cache_: ConvCache, + cache_index: Tensor, + parameters: ConvParameters, + ) struct { Tensor, ConvCache } { + var cache = cache_; + // Allows us to have a consistent API for the generic DecoderLayer + const BCx = self.in_proj.forward(input); + + const B, const C, const x = BCx.chunkExact(.d, 3); + const Bx = B.mul(x); + + const conv_out = if (parameters.is_prefill) b: { + const actual_seq_len_i32 = actual_seq_len.convert(.i32); + const start = actual_seq_len_i32.sub(Tensor.scalar(@as(i32, @intCast(self.config.conv_L_cache)), .i32)).maximum(Tensor.scalar(@as(i32, 0), .i32)); + const cache_seq_indices = lkp: { + const n = actual_seq_len_i32.sub(start); + const left_pad = Tensor.scalar(@as(i32, @intCast(self.config.conv_L_cache)), .i32).sub(n); + const sh = tokens_position_offset.shape().insert(.last, .{ .seq = self.config.conv_L_cache }); + break :lkp Tensor.iota(sh, .seq).convert(.u32).add(left_pad.convert(.u32).broad(sh)).broad(sh); + }; + const scatter_data = Bx.dynamicSlice(.{ .seq = @as(Tensor.DynSlice, .{ .start = start.convert(.u32), .len = @intCast(self.config.conv_L_cache) }) }); + cache.state = cache.state.scatterSlices( + .{ .layer = cache_index, .seq = cache_seq_indices }, + scatter_data, + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(cache.state); + const padding = self.config.conv_L_cache - 1; + const conv_out = Tensor.conv1d(Bx, self.kernel, .{ + .padding = &.{ padding, 0 }, + .input_batch_dimension = Bx.axis(.batch), + .input_feature_dimension = B.axis(.d), + .input_spatial_dimensions = B.axis(.seq), + .kernel_output_feature_dimension = self.kernel.axis(.out), + .kernel_input_feature_dimension = self.kernel.axis(.in), + .kernel_spatial_dimensions = self.kernel.axis(.kernel_size), + .output_batch_dimension = Bx.axis(.batch), + .output_feature_dimension = Bx.axis(.d), + .output_spatial_dimensions = Bx.axis(.seq), + .feature_group_count = Bx.dim(.d), + }); + break :b conv_out.slice1d(.seq, .{ .end = Bx.dim(.seq) }); + } else b: { + cache.state = cache.state.rollRight1d(.seq, -1) + .scatterSlices( + .{ .layer = cache_index, .seq = tokens_position_offset.convert(.u32).clamp(.scalar(@as(u32, @intCast(0)), .u32), .scalar(self.config.conv_L_cache - 1, .u32)) }, + Bx, + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(cache.state); + const kernel = self.kernel.squeeze(.in).rename(.{ .out = .d, .kernel_size = .seq }); + const sh = kernel.shape().insert(.last, .{ .batch = tokens_position_offset.dim(.batch) }); + + break :b cache.state.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = cache_index, .len = 1 } }).squeeze(.layer) + .mul(kernel.broad(sh).transpose(.{ .batch, .seq, .d })) + .sum(.seq); + }; + + const y = C.mul(conv_out); + + const output = self.out_proj.forward(y); + return .{ output.reuseBuffer(input), cache.reuseBuffer(cache_) }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(ShortConv)) void { + Linear.unloadBuffers(&self.in_proj); + Linear.unloadBuffers(&self.out_proj); + self.kernel.deinit(); + } +}; + +pub const Attention = struct { + q_proj: Linear, + k_proj: Linear, + v_proj: Linear, + out_proj: Linear, + q_layernorm: RmsNorm, + k_layernorm: RmsNorm, + + head_dim: usize, + num_key_value_groups: usize, + rope_opts: zml.nn.RopeOpts, + + pub fn init(config: Config, store: zml.io.TensorStore.View) Attention { + const head_dim = config.hidden_size / config.num_attention_heads; + const num_key_value_groups = config.num_attention_heads / config.num_key_value_heads; + + return .{ + // + .q_proj = initLinear(store.withPrefix("q_proj"), .d), + .k_proj = initLinear(store.withPrefix("k_proj"), .d), + .v_proj = initLinear(store.withPrefix("v_proj"), .d), + .out_proj = initLinear(store.withPrefix("out_proj"), .d), + .q_layernorm = RmsNorm.init(store.withPrefix("q_layernorm"), config.norm_eps, .hd), + .k_layernorm = RmsNorm.init(store.withPrefix("k_layernorm"), config.norm_eps, .hd), + .head_dim = head_dim, + .num_key_value_groups = num_key_value_groups, + .rope_opts = .{ + .layout = .sequential, + .freq_base = config.rope_theta, + .scaling = .{ .default = .{} }, + }, + }; + } + + pub fn forward( + self: Attention, + x: Tensor, + tokens_position_offset: Tensor, + kv_cache: KvCache, + cache_index: Tensor, + attention_metadata: attention.Metadata, + attention_parameters: attention.Parameters, + ) struct { Tensor, KvCache } { + var q = self.q_proj.forward(x).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); + var k = self.k_proj.forward(x).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); + var v = self.v_proj.forward(x).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); + + q = self.q_layernorm.forward(q); + k = self.k_layernorm.forward(k); + + const token_positions = b: { + const sh = tokens_position_offset.shape().insert(.last, .{ .seq = x.dim(.seq) }); + break :b Tensor.iota(sh, .seq).convert(.u32).add(tokens_position_offset.broad(sh)); + }; + + q = zml.nn.rope(q, token_positions, self.rope_opts); + k = zml.nn.rope(k, token_positions, self.rope_opts); + + q = q.rename(.{ .seq = .q }); + k = k.rename(.{ .seq = .k }); + v = v.rename(.{ .seq = .k }); + + const new_kv_cache = kv_cache.update(k, v, tokens_position_offset, cache_index); + k = new_kv_cache.keys(cache_index); + v = new_kv_cache.values(cache_index); + + stdx.debug.assert(q.dim(.batch) == 1, "LFM attention currently expects batch size 1 for flash attention backend, got {}", .{q.dim(.batch)}); + const attn = switch (attention_parameters) { + .vanilla => attention.attention( + q, + k, + v, + tokens_position_offset, + attention_metadata, + attention_parameters, + ).merge(.{ .d = .{ .h, .hd } }), + // flash attention doesn't support batch dimension at all in practice. + else => attention.attention( + q.squeeze(.batch), + k.squeeze(.batch), + v.squeeze(.batch), + tokens_position_offset.squeeze(.batch), + attention_metadata, + attention_parameters, + ).merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .seq }).insertAxes(.seq, .{.batch}), + }; + + return .{ self.out_proj.forward(attn).reuseBuffer(x), new_kv_cache.reuseBuffer(kv_cache) }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Attention)) void { + Linear.unloadBuffers(&self.q_proj); + Linear.unloadBuffers(&self.k_proj); + Linear.unloadBuffers(&self.v_proj); + Linear.unloadBuffers(&self.out_proj); + RmsNorm.unloadBuffers(&self.q_layernorm); + RmsNorm.unloadBuffers(&self.k_layernorm); + } +}; + +pub const Cache = struct { + conv: ConvCache, + kv: KvCache, + + pub fn initBuffers(self: Cache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Cache) { + return .{ + .conv = try self.conv.initBuffers(allocator, io, platform), + .kv = try self.kv.initBuffers(io, platform), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Cache)) void { + ConvCache.unloadBuffers(&self.conv); + KvCache.unloadBuffers(&self.kv); + } + + pub fn reuseBuffer(self: Cache, other: Cache) Cache { + return .{ + .conv = self.conv.reuseBuffer(other.conv), + .kv = self.kv.reuseBuffer(other.kv), + }; + } +}; + +pub const ConvCache = struct { + state: Tensor, + + pub fn init(shape: zml.Shape) ConvCache { + return .{ + .state = .fromShape(shape), + }; + } + + pub fn initBuffers(self: ConvCache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(ConvCache) { + const sh = self.state.shape(); + const host = try allocator.alloc(u8, sh.byteSize()); + defer allocator.free(host); + @memset(host, 0); + + return .{ .state = try zml.Buffer.fromBytes(io, platform, sh, host) }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(ConvCache)) void { + self.state.deinit(); + } + + pub fn reuseBuffer(self: ConvCache, other: ConvCache) ConvCache { + return .{ + .state = self.state.reuseBuffer(other.state), + }; + } +}; + +pub const KvCache = struct { + k: Tensor, + v: Tensor, + + pub fn init(kv_shape: zml.Shape) KvCache { + return .{ + .k = .fromShape(kv_shape), + .v = .fromShape(kv_shape), + }; + } + + pub fn initBuffers(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { + return .{ + .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), + .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(KvCache)) void { + self.k.deinit(); + self.v.deinit(); + } + + pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { + return .{ + .k = self.k.reuseBuffer(other.k), + .v = self.v.reuseBuffer(other.v), + }; + } + + pub fn keys(self: KvCache, cache_index: Tensor) Tensor { + return self.k.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = cache_index, .len = 1 } }).squeeze(.layer); + } + + pub fn values(self: KvCache, cache_index: Tensor) Tensor { + return self.v.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = cache_index, .len = 1 } }).squeeze(.layer); + } + + pub fn update(self: KvCache, new_k: Tensor, new_v: Tensor, token_position: Tensor, cache_index: Tensor) KvCache { + const k_shape = self.k.shape().drop(.layer); + const layer = cache_index.broad(token_position.shape()); + + return .{ + .k = self.k.scatterSlices( + .{ .layer = layer, .k = token_position }, + new_k.transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.k), + .v = self.v.scatterSlices( + .{ .layer = layer, .k = token_position }, + new_v.transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.v), + }; + } +}; + +fn initLinear(store: zml.io.TensorStore.View, tag: anytype) Linear { + return .init(store.createTensorWithTags("weight", .{ .out, tag }), null, tag); +} + +pub const Linear = struct { + weight: Tensor, + bias: ?Tensor = null, + in_tag: Shape.Tag, + out_tag: Shape.Tag, + + /// Assumes that the weights are tagged `.in` and `.out`. + pub fn init(weight: Tensor, bias: ?Tensor, tag: anytype) Linear { + stdx.debug.guard(weight.shape().hasTag(tag) != null, @src()); + const axis = weight.shape().axis(tag); + const out_tag = weight.shape().tag(1 - axis); + return .{ + .weight = weight, + .bias = bias, + .in_tag = zml.Shape.toTag(tag), + .out_tag = out_tag, + }; + } + + pub fn unloadBuffers(linear: *zml.Bufferized(Linear)) void { + linear.weight.deinit(); + if (linear.bias) |*bias| bias.deinit(); + } + + pub fn forward(self: Linear, x: Tensor) Tensor { + var y = x.dot(self.weight, self.in_tag).renameTag(self.out_tag, self.in_tag); + + return if (self.bias) |bias| y.add(bias.broad(y.shape())).reuseBuffer(x) else y; + } +}; + +const Mlp = struct { + w1: Linear, + w2: Linear, + w3: Linear, + + pub fn init(store: zml.io.TensorStore.View) Mlp { + return .{ + .w1 = initLinear(store.withPrefix("w1"), .d), + .w2 = initLinear(store.withPrefix("w2"), .d), + .w3 = initLinear(store.withPrefix("w3"), .d), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { + Linear.unloadBuffers(&self.w1); + Linear.unloadBuffers(&self.w2); + Linear.unloadBuffers(&self.w3); + } + + pub fn forward(self: Mlp, x: Tensor) Tensor { + return self.w2.forward(self.w1.forward(x).silu().mul(self.w3.forward(x))).reuseBuffer(x); + } +}; + +const RmsNorm = struct { + weight: Tensor, + eps: f32, + tag: Shape.Tag, + + pub fn init(store: zml.io.TensorStore.View, eps: f32, tag: anytype) RmsNorm { + return .{ .weight = store.createTensorWithTags("weight", .{tag}), .eps = eps, .tag = zml.Shape.toTag(tag) }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { + self.weight.deinit(); + } + + /// L2 normalization of input tensor along `.d` axis. + pub fn forward(self: RmsNorm, input: Tensor) Tensor { + const normalized = zml.nn.rmsNorm(input, self.tag, self.eps); + return normalized.mul(self.weight.broad(input.shape())).reuseBuffer(input); + } +}; diff --git a/examples/lfm/test_runner.zig b/examples/lfm/test_runner.zig new file mode 100644 index 00000000..0df68635 --- /dev/null +++ b/examples/lfm/test_runner.zig @@ -0,0 +1,233 @@ +const std = @import("std"); + +const zml = @import("zml"); +const stdx = zml.stdx; +const attention = zml.attention.attention; + +const model = @import("model.zig"); + +pub fn run(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, config: model.Config, mdl: model.Model, model_buffers: *zml.Bufferized(model.Model), store: *zml.io.TensorStore, attention_metadata: attention.Metadata, attention_parameters: attention.Parameters) !void { + _ = store; + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, "/Users/brabier/github/huggingface/transformers/lfm_activations.safetensors"); + defer registry.deinit(); + + var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + defer activation_store.deinit(); + + var ctx = TestContext{ .allocator = allocator, .io = io, .platform = platform, .activations_store = &activation_store, .attention_metadata = attention_metadata, .attention_parameters = attention_parameters }; + + try ctx.testLayer("embed_tokens", .{ .batch, .seq }, mdl.embed_tokens, model_buffers.embed_tokens, .{}); + + var num_attn_layers: usize = 0; + var num_conv_layers: usize = 0; + for (0..config.num_hidden_layers) |i| { + const kind = std.meta.stringToEnum(model.OperatorKind, config.layer_types[i]) orelse { + stdx.debug.assert(false, "Unsupported layer type {s}", .{config.layer_types[i]}); + unreachable; + }; + const layer = mdl.layers[i]; + const layer_buffers = model_buffers.layers[i]; + switch (kind) { + .conv => { + try ctx.testLayerPrint("layers.{d}.conv.in_proj", .{i}, .{ .batch, .seq, .d }, layer.operator.conv.in_proj, layer_buffers.operator.conv.in_proj, .{}); + try ctx.testLayerPrint("layers.{d}.conv.out_proj", .{i}, .{ .batch, .seq, .d }, layer.operator.conv.out_proj, layer_buffers.operator.conv.out_proj, .{}); + try ctx.testLayerPrint("layers.{d}.feed_forward.w1", .{i}, .{ .batch, .seq, .d }, layer.feed_forward.w1, layer_buffers.feed_forward.w1, .{}); + try ctx.testLayerPrint("layers.{d}.feed_forward.w2", .{i}, .{ .batch, .seq, .d }, layer.feed_forward.w2, layer_buffers.feed_forward.w2, .{}); + try ctx.testLayerPrint("layers.{d}.feed_forward.w3", .{i}, .{ .batch, .seq, .d }, layer.feed_forward.w3, layer_buffers.feed_forward.w3, .{}); + try ctx.testLayerPrint("layers.{d}.feed_forward", .{i}, .{ .batch, .seq, .d }, layer.feed_forward, layer_buffers.feed_forward, .{}); + try ctx.testConvLayer(i, num_conv_layers, mdl.layers[i].operator.conv, model_buffers.layers[i].operator.conv, .{}); + num_conv_layers += 1; + }, + .full_attention => { + try ctx.testLayerPrint("layers.{d}.self_attn.k_proj", .{i}, .{ .batch, .seq, .d }, layer.operator.self_attn.k_proj, layer_buffers.operator.self_attn.k_proj, .{}); + try ctx.testLayerPrint("layers.{d}.self_attn.q_proj", .{i}, .{ .batch, .seq, .d }, layer.operator.self_attn.q_proj, layer_buffers.operator.self_attn.q_proj, .{}); + try ctx.testLayerPrint("layers.{d}.self_attn.out_proj", .{i}, .{ .batch, .seq, .d }, layer.operator.self_attn.out_proj, layer_buffers.operator.self_attn.out_proj, .{}); + try ctx.testLayerPrint("layers.{d}.self_attn.v_proj", .{i}, .{ .batch, .seq, .d }, layer.operator.self_attn.v_proj, layer_buffers.operator.self_attn.v_proj, .{}); + try ctx.testLayerPrint("layers.{d}.feed_forward.w1", .{i}, .{ .batch, .seq, .d }, layer.feed_forward.w1, layer_buffers.feed_forward.w1, .{}); + try ctx.testLayerPrint("layers.{d}.feed_forward.w2", .{i}, .{ .batch, .seq, .d }, layer.feed_forward.w2, layer_buffers.feed_forward.w2, .{}); + try ctx.testLayerPrint("layers.{d}.feed_forward.w3", .{i}, .{ .batch, .seq, .d }, layer.feed_forward.w3, layer_buffers.feed_forward.w3, .{}); + try ctx.testLayerPrint("layers.{d}.feed_forward", .{i}, .{ .batch, .seq, .d }, layer.feed_forward, layer_buffers.feed_forward, .{}); + try ctx.testAttnLayer(i, num_attn_layers, mdl.layers[i].operator.self_attn, model_buffers.layers[i].operator.self_attn, .{}); + num_attn_layers += 1; + }, + } + } +} + +const TestContext = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + activations_store: *zml.io.TensorStore, + attention_metadata: attention.Metadata, + attention_parameters: attention.Parameters, + + fn testLayerPrint( + self: *TestContext, + comptime name_fmt: []const u8, + name_args: anytype, + tagz: anytype, + layer: anytype, + layer_buffers: anytype, + opts: zml.testing.CompareOpts, + ) !void { + const name = try std.fmt.allocPrint(self.allocator, name_fmt, name_args); + defer self.allocator.free(name); + try self.testLayer(name, tagz, layer, layer_buffers, opts); + } + + fn testLayer(self: *TestContext, name: []const u8, tagz: anytype, layer: anytype, layer_buffers: anytype, opts: zml.testing.CompareOpts) !void { + std.log.info("Testing layer: {s}", .{name}); + + const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); + defer self.allocator.free(in_key); + const in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); + const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(tagz); + + const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); + defer self.allocator.free(out_key); + const out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); + + const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor }); + defer exe.deinit(); + + var args = try exe.args(self.allocator); + defer args.deinit(self.allocator); + args.set(.{ layer_buffers, in_buffer }); + + var res = try exe.results(self.allocator); + defer res.deinit(self.allocator); + + exe.call(args, &res); + + const out_result = res.get(zml.Buffer); + + try zml.testing.expectClose(self.io, out_result, out_buffer_expected, opts); + std.log.info("Layer {s} passed!", .{name}); + } + + fn testConvLayer(self: *TestContext, ix: usize, cache_ix: usize, layer: model.ShortConv, layer_buffers: zml.Bufferized(model.ShortConv), opts: zml.testing.CompareOpts) !void { + const name = try std.fmt.allocPrint(self.allocator, "layers.{d}.conv", .{ix}); + defer self.allocator.free(name); + std.log.info("Testing layer: {s}", .{name}); + + const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); + defer self.allocator.free(in_key); + const in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); + const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(.{ .batch, .seq, .d }); + + const cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache", .{name}); + defer self.allocator.free(cache_key); + const cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_key); + const cache_tensor = zml.Tensor.fromShape(cache_buffer.shape()).withTags(.{ .layer, .batch, .seq, .d }); + + const cache_pos_key = try std.fmt.allocPrint(self.allocator, "{s}.cache_position", .{name}); + defer self.allocator.free(cache_pos_key); + const cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key); + const cache_pos_tensor = zml.Tensor.fromShape(cache_pos_buffer.shape()).withTags(.{.batch}); + + const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); + defer self.allocator.free(out_key); + const out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); + + const actual_seq_len_tensor: zml.Tensor = .init(.{}, .u32); + const actual_seq_len: u32 = @intCast(in_tensor.dim(.seq)); + + const cache_index_tensor: zml.Tensor = .init(.{}, .u32); + + const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, actual_seq_len_tensor, model.ConvCache{ .state = cache_tensor }, cache_index_tensor, model.ConvParameters{ .is_prefill = false } }); + defer exe.deinit(); + + var args = try exe.args(self.allocator); + defer args.deinit(self.allocator); + const conv_cache: zml.Bufferized(model.ConvCache) = .{ .state = cache_buffer }; + const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{actual_seq_len})); + var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice); + defer actual_seq_len_buf.deinit(); + const cache_index_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(cache_ix)})); + var cache_index_buf: zml.Buffer = try .fromSlice(self.io, self.platform, cache_index_slice); + defer cache_index_buf.deinit(); + args.set(.{ layer_buffers, in_buffer, cache_pos_buffer, actual_seq_len_buf, conv_cache, cache_index_buf }); + + var res = try exe.results(self.allocator); + defer res.deinit(self.allocator); + + exe.call(args, &res); + + const out_result = res.get(zml.Buffer); + + try zml.testing.expectClose(self.io, out_result, out_buffer_expected, opts); + std.log.info("Layer {s} passed!", .{name}); + } + + fn testAttnLayer(self: *TestContext, ix: usize, cache_ix: usize, layer: model.Attention, layer_buffers: zml.Bufferized(model.Attention), opts: zml.testing.CompareOpts) !void { + const name = try std.fmt.allocPrint(self.allocator, "layers.{d}.self_attn", .{ix}); + defer self.allocator.free(name); + std.log.info("Testing layer: {s}", .{name}); + + const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); + defer self.allocator.free(in_key); + const in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); + const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(.{ .batch, .seq, .d }); + + const key_cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache.key", .{name}); + defer self.allocator.free(key_cache_key); + const key_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, key_cache_key); + const key_cache_tensor = zml.Tensor.fromShape(key_cache_buffer.shape()).withTags(.{ .layer, .batch, .h, .k, .hd }); + + const value_cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache.value", .{name}); + defer self.allocator.free(value_cache_key); + const value_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, value_cache_key); + const value_cache_tensor = zml.Tensor.fromShape(value_cache_buffer.shape()).withTags(.{ .layer, .batch, .h, .k, .hd }); + + const cache_pos_key = try std.fmt.allocPrint(self.allocator, "{s}.cache_position", .{name}); + defer self.allocator.free(cache_pos_key); + const cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key); + const cache_pos_tensor = zml.Tensor.fromShape(cache_pos_buffer.shape()).withTags(.{.batch}); + + const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); + defer self.allocator.free(out_key); + const out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); + + const cache_index_tensor: zml.Tensor = .init(.{}, .u32); + + const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, model.KvCache{ .k = key_cache_tensor, .v = value_cache_tensor }, cache_index_tensor, self.attention_metadata, self.attention_parameters }); + defer exe.deinit(); + + var args = try exe.args(self.allocator); + defer args.deinit(self.allocator); + const kv_cache: zml.Bufferized(model.KvCache) = .{ .k = key_cache_buffer, .v = value_cache_buffer }; + var attention_metadata_buffers = try self.attention_metadata.initBuffer(self.io, self.platform); + defer attention.Metadata.deinitBuffer(&attention_metadata_buffers); + const cache_index_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(cache_ix)})); + var cache_index_buf: zml.Buffer = try .fromSlice(self.io, self.platform, cache_index_slice); + defer cache_index_buf.deinit(); + args.set(.{ layer_buffers, in_buffer, cache_pos_buffer, kv_cache, cache_index_buf, attention_metadata_buffers }); + + var res = try exe.results(self.allocator); + defer res.deinit(self.allocator); + + exe.call(args, &res); + + var out_result, var updated_kv = res.get(struct { zml.Buffer, zml.Bufferized(model.KvCache) }); + defer out_result.deinit(); + defer model.KvCache.unloadBuffers(&updated_kv); + + try zml.testing.expectClose(self.io, out_result, out_buffer_expected, opts); + std.log.info("Layer {s} passed!", .{name}); + } +}; + +fn loadBufferFromStore(allocator: std.mem.Allocator, io: anytype, platform: *zml.Platform, store: *zml.io.TensorStore, key: []const u8) !zml.Buffer { + const shape = store.view().getShape(key) orelse return error.NotFound; + + const host_bytes = try allocator.alloc(u8, shape.byteSize()); + defer allocator.free(host_bytes); + + var io_buffer: [8 * 1024]u8 = undefined; + var reader = try store.view().getReader(key, io, &io_buffer); + defer reader.deinit(); + _ = try reader.interface.readSliceAll(host_bytes); + + return zml.Buffer.fromBytes(io, platform, shape, host_bytes); +} diff --git a/tools/zls.sh b/tools/zls.sh index 8428ee01..4b59ccaf 100755 --- a/tools/zls.sh +++ b/tools/zls.sh @@ -1,4 +1,7 @@ -#!/bin/bash +#!/usr/bin/env bash + +set -euo pipefail + cd "$(dirname "${BASH_SOURCE[0]}")" cd "$(bazel info workspace)" exec bazel run -- @zml//:completion "${@}" diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 111b7b1c..e77b27bd 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -263,9 +263,13 @@ pub const fa2 = struct { }; const max_seqlen_q: i32 = @intCast(q_.dim(.q)); const max_seqlen_k: i32 = @intCast(k_.dim(.k)); - var q = q_.insertAxes(.q, .{.b}).merge(.{ .tot = .{ .b, .q } }); - const k = k_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); - const v = v_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); + var q, const k, const v = if (q_.shape().hasTag(.b) != null) b: { + break :b [_]zml.Tensor{ + q_.merge(.{ .tot = .{ .b, .q } }), k_.merge(.{ .tot = .{ .b, .k } }), v_.merge(.{ .tot = .{ .b, .k } }), + }; + } else b: { + break :b [_]zml.Tensor{ q_.rename(.{ .q = .tot }), k_.rename(.{ .k = .tot }), v_.rename(.{ .k = .tot }) }; + }; // TODO(Corendos): replace with cumsum const cu_seqlens_q = zml.Tensor.constantTensor(zml.Shape.init(.{2}, .i32), std.mem.sliceAsBytes(&[2]i32{ 0, max_seqlen_q })); diff --git a/zml/mem.zig b/zml/mem.zig index 6a5c475b..c12a2f61 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -345,7 +345,7 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu switch (model) { inline else => |v, tag| { bufferized_.* = @unionInit(ModelBufferized, @tagName(tag), undefined); - try bufferizeInner(allocator, v, @field(bufferized_, @tagName(tag))); + try bufferizeInner(allocator, v, &@field(bufferized_, @tagName(tag))); }, } }, diff --git a/zml/tensor.zig b/zml/tensor.zig index a379db22..1b2f8736 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -187,6 +187,12 @@ pub const Tensor = struct { return res; } + pub fn renameTag(self: Tensor, old: anytype, new: anytype) Tensor { + var res = self; + res._shape = self._shape.setTag(old, new); + return res; + } + pub fn renameAxis(self: Tensor, ax: i8, name: @EnumLiteral()) Tensor { var res = self; res._shape._tags.set(self.axis(ax), @tagName(name).ptr); @@ -3890,6 +3896,17 @@ pub const Tensor = struct { ); } + pub fn rollRight1d(self: Tensor, axis_: anytype, shift: i64) Tensor { + const a = self.axis(axis_); + const n = self.dim(a); + const k = @mod(shift, n); // normalize shift to [0, n) + if (k == 0) return self; + return Tensor.concatenate(&.{ + self.slice1d(a, .{ .start = n - k, .end = n }), + self.slice1d(a, .{ .start = 0, .end = n - k }), + }, a); + } + /// Given a set of N vectors of lengths A, B, C, D, /// returns 1 tensors of rank N+1, and shape (A, B, C, D, N). /// For any coordinate (a, b, c, d), we have: From 48df9c85e9f89b071627c535df5cc599906db43a Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Thu, 19 Feb 2026 09:27:53 +0100 Subject: [PATCH 318/881] examples/benchmark: Add transfer-benchmark & fix benchmark (#387) Co-authored-by: Benjamin Rabier --- examples/benchmark/main.zig | 35 +++-------- examples/transfer-benchmark/BUILD.bazel | 13 ++++ examples/transfer-benchmark/main.zig | 84 +++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 25 deletions(-) create mode 100644 examples/transfer-benchmark/BUILD.bazel create mode 100644 examples/transfer-benchmark/main.zig diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index 5927ad6e..1f655b55 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -8,7 +8,7 @@ pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { return a.dot(b, .k); } -pub fn main() !void { +pub fn main(init: std.process.Init) !void { const CliArgs = struct { pub const help = \\ benchmark --size=4096 --dtype=f16 @@ -17,38 +17,22 @@ pub fn main() !void { dtype: zml.DataType = .f16, }; - const allocator = std.heap.smp_allocator; - - var threaded: std.Io.Threaded = .init(allocator, .{}); - defer threaded.deinit(); - - const io = threaded.io(); + const allocator = init.gpa; + const io = init.io; // Auto-select platform const platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator); - const cli_args: CliArgs = blk: { - var ret: CliArgs = .{}; - var it = std.process.args(); - defer it.deinit(); - while (it.next()) |arg| { - if (std.mem.startsWith(u8, arg, "--dtype=")) { - ret.dtype = std.meta.stringToEnum(zml.DataType, arg["--dtype=".len..]) orelse unreachable; - } else if (std.mem.startsWith(u8, arg, "--size=")) { - ret.size = try std.fmt.parseUnsigned(usize, arg["--size=".len..], 10); - } - } - break :blk ret; - }; + const cli_args: CliArgs = stdx.flags.parse(init.minimal.args, CliArgs); const a: zml.Tensor = .init(.{ .m = cli_args.size, .k = cli_args.size }, cli_args.dtype); const b: zml.Tensor = .init(.{ .k = cli_args.size, .n = cli_args.size }, cli_args.dtype); var exe = blk: { log.info("⏱️ Compiling benchmark...", .{}); - var timer = try std.time.Timer.start(); - defer log.info("✅ Compiled benchmark [{D}]", .{timer.read()}); + const now: std.Io.Timestamp = .now(io, .awake); + defer log.info("✅ Compiled benchmark [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); break :blk try platform.compileFn(allocator, io, benchmark, .{ a, b }); }; defer exe.deinit(); @@ -79,12 +63,13 @@ pub fn main() !void { } // call our executable module - var timer = try std.time.Timer.start(); + const run_start: std.Io.Timestamp = .now(io, .awake); exe.call(exe_args, &exe_results); var result = exe_results.get(zml.Buffer); _ = try result.await(io); defer result.deinit(); - const elapsed_ns = timer.read(); + const elapsed = run_start.untilNow(io, .awake); + const elapsed_ns = elapsed.toNanoseconds(); const elapsed_s = @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_s; log.info("✅ Benchmark done!", .{}); @@ -95,7 +80,7 @@ pub fn main() !void { cli_args.size, cli_args.size, @tagName(cli_args.dtype), - elapsed_ns, + stdx.fmt.fmtDuration(elapsed), flops / 1_000_000_000, }); } diff --git a/examples/transfer-benchmark/BUILD.bazel b/examples/transfer-benchmark/BUILD.bazel new file mode 100644 index 00000000..a3931095 --- /dev/null +++ b/examples/transfer-benchmark/BUILD.bazel @@ -0,0 +1,13 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load("@rules_zig//zig:defs.bzl", "zig_binary") + +zig_binary( + name = "transfer-benchmark", + main = "main.zig", + deps = ["//zml"], +) + +build_test( + name = "test", + targets = [":transfer-benchmark"], +) diff --git a/examples/transfer-benchmark/main.zig b/examples/transfer-benchmark/main.zig new file mode 100644 index 00000000..d1d9fd99 --- /dev/null +++ b/examples/transfer-benchmark/main.zig @@ -0,0 +1,84 @@ +const std = @import("std"); +const log = std.log; + +const zml = @import("zml"); +const stdx = zml.stdx; + +pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + return a.dot(b, .k); +} + +const Args = struct { + pub const help = + \\ benchmark --model=./model --tensor="model.embed_tokens.weight" + ; + model: []const u8, + tensor: []const u8 = "model.embed_tokens.weight", +}; + +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const io = init.io; + + const platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator); + std.debug.print("{f}", .{platform.fmtVerbose()}); + const args = stdx.flags.parseProcessArgs(init.minimal, Args); + + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, args.model); + defer registry.deinit(); + + var reader = try registry.reader(io, args.tensor, &.{}); + + var buffer: zml.Buffer = undefined; + defer buffer.deinit(); + + log.info("⏱️ Running benchmark...", .{}); + + const now: std.Io.Timestamp = .now(io, .awake); + + try transfer(allocator, io, platform, reader.tensor.shape, &reader.interface, &buffer, 4, 32 * zml.MiB); + + const elapsed = now.untilNow(io, .awake); + const elapsed_ns = elapsed.toNanoseconds(); + const elapsed_s = @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_s; + + log.info("✅ Benchmark done!", .{}); + + const size = reader.tensor.shape.byteSize(); + const flops = @as(f64, @floatFromInt(size >> 20)) / elapsed_s; + log.info("Transfer {d} MiB - Elapsed: {D} - {d:.3} MiB/s", .{ + size >> 20, + stdx.fmt.fmtDuration(elapsed), + flops, + }); +} + +pub fn transfer( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + shape: zml.Shape, + reader: *std.Io.Reader, + buffer: *zml.Buffer, + dma_chunks: usize, + dma_chunk_size: usize, +) !void { + const first_device = platform.devices[0]; // Temporary until sharding is re-exposed + const dma_allocator: zml.mem.DmaAllocator = .init(allocator, &first_device); + var buffer_pool: zml.mem.DynamicBufferPool = .init(dma_chunks, dma_chunk_size); + defer buffer_pool.deinit(dma_allocator.allocator()); + + var memory_writer = zml.io.MemoryWriter.init( + dma_allocator.allocator(), + io, + first_device.memory(.default), + &buffer_pool, + shape, + buffer, + ) catch unreachable; + defer memory_writer.deinit(dma_allocator.allocator()); + + _ = reader.streamRemaining(memory_writer.interface()) catch unreachable; + memory_writer.interface().flush() catch unreachable; +} From a175578eba00db816c94e5198903a47a4017a1fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20Gauti=C3=A9?= <51361266+tgautie@users.noreply.github.com> Date: Thu, 19 Feb 2026 09:28:36 +0100 Subject: [PATCH 319/881] examples/llama: improve defer pattern (#386) We improve the defer pattern in the Llama example to: - avoid double deinit of `prefill_exe` and `decode_exe` by cleaning at future-definition time, - using defer instead of errdefer for future-allocated resources cleanup in the main function scope, - unload buffers from the arena allocator. --- examples/llama/main.zig | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/examples/llama/main.zig b/examples/llama/main.zig index aa048103..6285bde9 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -152,9 +152,9 @@ pub fn main(init: std.process.Init) !void { } var compiled_model_result_future = try io.concurrent(compileModel, .{ allocator, io, platform, llama_model, llama_parameters, &progress }); - errdefer if (compiled_model_result_future.cancel(io)) |v| { - defer v.prefill_exe.deinit(); - defer v.decode_exe.deinit(); + defer if (compiled_model_result_future.cancel(io)) |v| { + v.prefill_exe.deinit(); + v.decode_exe.deinit(); } else |_| {}; var load_group: stdx.Io.LimitedGroup = .init(16); @@ -168,19 +168,14 @@ pub fn main(init: std.process.Init) !void { &store, &progress, }); - errdefer b: { - var v = llama_buffers_future.cancel(io) catch break :b; - v = v; // autofix - // LlamaLM.unloadBuffers(&v, allocator); + defer blk: { + var v = llama_buffers_future.cancel(io) catch break :blk; + LlamaLM.unloadBuffers(&v, arena.allocator()); } const compiled_model_result = try compiled_model_result_future.await(io); - defer compiled_model_result.prefill_exe.deinit(); - defer compiled_model_result.decode_exe.deinit(); var llama_buffers = try llama_buffers_future.await(io); - llama_buffers = llama_buffers; // autofix - // defer LlamaLM.unloadBuffers(&llama_buffers, allocator); progress.end(); log.info("Creating KvCache", .{}); @@ -190,8 +185,7 @@ pub fn main(init: std.process.Init) !void { var attention_metadata_buffers: zml.Bufferized(zml.attention.attention.Metadata) = try llama_parameters.attention_metadata.initBuffer(io, platform); defer zml.attention.attention.Metadata.deinitBuffer(&attention_metadata_buffers); - var tokenizer = try tokenizer_future.await(io); - defer tokenizer.deinit(); + const tokenizer = try tokenizer_future.await(io); const prompt = if (args.prompt) |p| p else blk: { var reader = std.Io.File.stdin().reader(io, &.{}); @@ -307,7 +301,9 @@ fn compileModel( }); } }.call, .{ allocator, io, platform, llama_model, parameters, progress }); - errdefer if (prefill_future.cancel(io)) |v| v.deinit() else |_| {}; + errdefer if (prefill_future.cancel(io)) |v| { + v.deinit(); + } else |_| {}; var decode_future = try io.concurrent(struct { fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, llama_model_: llama.LlamaLM, parameters_: LlamaParameters, progress_: *std.Progress.Node) !zml.Exe { @@ -326,7 +322,9 @@ fn compileModel( }); } }.call, .{ allocator, io, platform, llama_model, parameters, progress }); - errdefer if (decode_future.cancel(io)) |v| v.deinit() else |_| {}; + errdefer if (decode_future.cancel(io)) |v| { + v.deinit(); + } else |_| {}; const prefill_exe = try prefill_future.await(io); const decode_exe = try decode_future.await(io); From a9f26872379c7524dcffc27dfcdcd6c3edd0c679 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 19 Feb 2026 10:48:15 +0100 Subject: [PATCH 320/881] zml/io/vfs: fixes for Zig 0.16.0-dev.2611+f996d2866 + new S3 VFS (#388) --- examples/llama/llama.zig | 38 +- examples/llama/main.zig | 114 +---- examples/vfs/playground.zig | 177 ++++++- stdx/BUILD.bazel | 1 + stdx/crypto.zig | 41 ++ stdx/stdx.zig | 1 + zml/io.zig | 2 +- zml/io/vfs.zig | 91 +++- zml/io/vfs/base.zig | 166 ++++++- zml/io/vfs/file.zig | 99 ++-- zml/io/vfs/hf.zig | 51 +- zml/io/vfs/http.zig | 41 +- zml/io/vfs/s3.zig | 958 +++++++++++++++++++++--------------- zml/safetensors.zig | 105 +++- 14 files changed, 1226 insertions(+), 659 deletions(-) create mode 100644 stdx/crypto.zig diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 0968b83e..5073a140 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -81,11 +81,8 @@ pub const LlamaLM = struct { var total_bytes: usize = 0; defer { const took = now.untilNow(io, .awake); - log.info("Loaded weights [{Bi:.2}, {D}, {Bi:.2}/s]", .{ - total_bytes, - stdx.fmt.fmtDuration(took), - total_bytes / (@as(usize, @intCast(took.toNanoseconds())) * std.time.ns_per_s), - }); + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {D}, {Bi:.2}/s]", .{ total_bytes, stdx.fmt.fmtDuration(took), bytes_per_sec }); } return zml.io.load(LlamaLM, self, allocator, io, platform, .{ .dma_chunks = 32, @@ -180,37 +177,6 @@ pub const Llama = struct { allocator.free(self.layers); } - pub fn loadBuffers( - self: *const Llama, - bufferize_ctx: zml.io.BufferizeContext(TransferCtx), - group: *stdx.Io.LimitedGroup, - store: zml.io.TensorStore.View, - cb: zml.io.CallbackTensorBufferTransfer(TransferCtx), - ) !zml.Bufferized(Llama) { - const bufferized_layers = try bufferize_ctx.allocator.alloc(zml.Bufferized(TransformerLayer), self.layers.len); - - var bufferized: zml.Bufferized(Llama) = .{ .embed_tokens = undefined, .norm = undefined, .layers = bufferized_layers }; - - const transfers = try zml.io.bufferize(TransferCtx, bufferize_ctx, &self.embed_tokens, &bufferized.embed_tokens, store.withPrefix("embed_tokens")); - for (transfers) |t| { - try group.concurrent(bufferize_ctx.io, cb, .{t}); - } - - const norm_transfers = try zml.io.bufferize(TransferCtx, bufferize_ctx, &self.norm, &bufferized.norm, store.withPrefix("norm")); - for (norm_transfers) |t| { - try group.concurrent(bufferize_ctx.io, cb, .{t}); - } - - for (self.layers, bufferized_layers, 0..) |layer, *buf_layer, i| { - const layer_transfers = try zml.io.bufferize(TransferCtx, bufferize_ctx, &layer, buf_layer, store.withPrefix("layers").withLayer(i)); - for (layer_transfers) |t| { - try group.concurrent(bufferize_ctx.io, cb, .{t}); - } - } - - return bufferized; - } - pub fn unloadBuffers(self: *zml.Bufferized(Llama), allocator: std.mem.Allocator) void { self.embed_tokens.weight.deinit(); RmsNorm.unloadBuffers(&self.norm); diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 6285bde9..e16ee303 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -58,33 +58,32 @@ pub fn main(init: std.process.Init) !void { const args = stdx.flags.parse(init.minimal.args, CliArgs); - // var vfs: zml.io.VFS = try .init(allocator, init.io); - // defer vfs.deinit(); + var http_client: std.http.Client = .{ .allocator = allocator, .io = init.io }; - // var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); - // defer vfs_file.deinit(); - // try vfs.register("file", vfs_file.io()); + try http_client.initDefaultProxies(allocator, init.environ_map); + defer http_client.deinit(); - // var vfs_https: zml.io.VFS.HTTP = try .init(allocator, init.io, &http_client, .https); - // defer vfs_https.deinit(); - // try vfs.register("https", vfs_https.io()); + var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); + defer vfs_file.deinit(); - // var http_client: std.http.Client = .{ - // .allocator = allocator, - // .io = init.io, - // }; - // try http_client.initDefaultProxies(arena.allocator(), init.environ_map); - // defer http_client.deinit(); + var vfs_https: zml.io.VFS.HTTP = try .init(allocator, init.io, &http_client, .https); + defer vfs_https.deinit(); - // var hf_vfs: zml.io.VFS.HF = try .auto( - // allocator, - // init.io, - // &http_client, - // ); - // defer hf_vfs.deinit(); - // try vfs.register("hf", hf_vfs.io()); + var hf_vfs: zml.io.VFS.HF = try .auto(allocator, init.io, &http_client, init.environ_map); + defer hf_vfs.deinit(); - const io = init.io; + var s3_vfs: zml.io.VFS.S3 = try .auto(allocator, init.io, &http_client, init.environ_map); + defer s3_vfs.deinit(); + + var vfs: zml.io.VFS = try .init(allocator, init.io); + defer vfs.deinit(); + + try vfs.register("file", vfs_file.io()); + try vfs.register("https", vfs_https.io()); + try vfs.register("hf", hf_vfs.io()); + try vfs.register("s3", s3_vfs.io()); + + const io = vfs.io(); log.info("Resolving model repo", .{}); const repo = try zml.safetensors.resolveModelRepo(io, args.model); @@ -111,10 +110,6 @@ pub fn main(init: std.process.Init) !void { var platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator); log.info("\n{f}", .{platform.fmtVerbose()}); - // log.info("Devices:", .{}); - // for (platform.devices) |device| { - // log.info("\t- {f}", .{device}); - // } // Initialize the Llama struct and map the content of the .safetensors to the model tensors. var llama_model: llama.LlamaLM = try .init(allocator, store.view(), config, llama_options); @@ -332,73 +327,6 @@ fn compileModel( return .{ .prefill_exe = prefill_exe, .decode_exe = decode_exe }; } -fn loadModelBuffers( - allocator: std.mem.Allocator, - buffers_allocator: std.mem.Allocator, - io: std.Io, - progress: *std.Progress.Node, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - llama_model: llama.LlamaLM, - pool: *zml.mem.DynamicBufferPool, - async_limit: usize, -) !zml.Bufferized(llama.LlamaLM) { - var transferred_bytes: usize = 0; - - const now: std.Io.Timestamp = .now(io, .awake); - defer { - const duration = now.untilNow(io, .awake); - const seconds = @as(f64, @floatFromInt(duration.ns)) / 1e9; - const gb_per_sec = @as(f64, @floatFromInt(transferred_bytes)) / (1024.0 * 1024.0 * 1024.0) / seconds; - const gbps = gb_per_sec * 8.0; - log.info("Loaded model [{D} {d:.3} GB/s {d:.3} gbps]", .{ stdx.fmt.fmtDuration(duration), gb_per_sec, gbps }); - } - - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - - var group: stdx.Io.LimitedGroup = .init(async_limit); - defer group.cancel(io); - - const bufferize_ctx: zml.io.BufferizeContext(llama.TransferCtx) = .{ - .allocator = allocator, - .arena = &arena, - .io = io, - .platform = platform, - .cb_ctx = .{ - .allocator = buffers_allocator, - .pool = pool, - .transferred_bytes = &transferred_bytes, - .progress = progress, - }, - }; - - const bufferized = llama_model.loadBuffers(bufferize_ctx, &group, store.view(), transferBuffer); - try group.await(io); - - return bufferized; -} - -fn transferBuffer(ctx: zml.io.TensorBufferTransfer(llama.TransferCtx)) !void { - var transfer_progress = ctx.cb_ctx.progress.start(ctx.tensor.name, ctx.tensor.byteSize()); - - var reader = zml.safetensors.TensorReader.init(ctx.io, ctx.tensor, &.{}) catch unreachable; - defer reader.deinit(); - - var writer = zml.io.DeviceWriter2.init(ctx.cb_ctx.allocator, ctx.io, ctx.platform, ctx.cb_ctx.pool, ctx.tensor.shape, ctx.buffer, .device) catch unreachable; - defer { - writer.interface.flush() catch unreachable; - writer.deinit(); - } - - _ = reader.interface.streamRemaining(&writer.interface) catch unreachable; - writer.interface.flush() catch unreachable; - - transfer_progress.end(); - - ctx.cb_ctx.transferred_bytes.* += ctx.tensor.byteSize(); -} - pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: LlamaLM.Config, prompt: []const u8, skip_llama3_encoding: bool) ![]u32 { var encoder = try tokenizer.encoder(); defer encoder.deinit(); diff --git a/examples/vfs/playground.zig b/examples/vfs/playground.zig index 1c5a1e47..64b2f084 100644 --- a/examples/vfs/playground.zig +++ b/examples/vfs/playground.zig @@ -13,48 +13,34 @@ pub const std_options: std.Options = .{ // -- ls hf://Qwen/Qwen3-235B-A22B-Instruct-2507 // -- ls hf://meta-llama/Llama-3.1-8B-Instruct@0e9e39f // -- cat https://iprs.fly.dev -pub fn main() !void { - var debug_allocator: ?std.heap.DebugAllocator(.{}) = null; - const allocator = if (@import("builtin").mode == .Debug) blk: { - debug_allocator = .init; - break :blk debug_allocator.?.allocator(); - } else std.heap.c_allocator; - defer if (debug_allocator) |*da| std.debug.assert(da.deinit() == .ok); +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; - var threaded: std.Io.Threaded = .init(allocator, .{}); - defer threaded.deinit(); + const Command = enum { cat, ls, cp, stat, realpath, safetensors }; - const Command = enum { cat, ls, stat, realpath }; - - var it = std.process.args(); + var it = init.minimal.args.iterate(); _ = it.next(); // skip program name const command: Command = std.meta.stringToEnum(Command, it.next() orelse return error.MissingCommand) orelse return error.CommandInvalid; const path = it.next() orelse return error.MissingPath; - var http_client: std.http.Client = .{ - .allocator = allocator, - .io = threaded.io(), - .connection_pool = .{ - .free_size = threaded.async_limit.toInt() orelse 16, - }, - }; + var http_client: std.http.Client = .{ .allocator = allocator, .io = init.io }; - try http_client.initDefaultProxies(allocator); + try http_client.initDefaultProxies(allocator, init.environ_map); defer http_client.deinit(); - var vfs_file: zml.io.VFS.File = .init(allocator, threaded.io(), .{}); + var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); defer vfs_file.deinit(); - var vfs_https: zml.io.VFS.HTTP = try .init(allocator, threaded.io(), &http_client, .https); + var vfs_https: zml.io.VFS.HTTP = try .init(allocator, init.io, &http_client, .https); defer vfs_https.deinit(); - var hf_vfs: zml.io.VFS.HF = try .auto(allocator, threaded.io(), &http_client); + var hf_vfs: zml.io.VFS.HF = try .auto(allocator, init.io, &http_client, init.environ_map); defer hf_vfs.deinit(); - var s3_vfs: zml.io.VFS.S3 = try .fromEnv(allocator, threaded.io(), &http_client); + var s3_vfs: zml.io.VFS.S3 = try .auto(allocator, init.io, &http_client, init.environ_map); defer s3_vfs.deinit(); - var vfs: zml.io.VFS = try .init(allocator, threaded.io()); + var vfs: zml.io.VFS = try .init(allocator, init.io); defer vfs.deinit(); try vfs.register("file", vfs_file.io()); @@ -64,7 +50,7 @@ pub fn main() !void { const io = vfs.io(); - const buffer = try allocator.alloc(u8, 16 * 1024 * 1024); + const buffer = try allocator.alignedAlloc(u8, .fromByteUnits(4 * 1024), 16 * 1024 * 1024); defer allocator.free(buffer); var stdout_writer = std.Io.File.stdout().writer(io, buffer); @@ -75,7 +61,8 @@ pub fn main() !void { var file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); defer file.close(io); - var reader = file.reader(io, &.{}); + var reader: std.Io.File.Reader = .initStreaming(file, io, &.{}); + const read = try reader.interface.streamRemaining(&stdout_writer.interface); _ = try stdout_writer.interface.write("\n"); @@ -92,6 +79,26 @@ pub fn main() !void { try printTree(io, &stdout_writer.interface, dir, "", 10, &counts); try stdout_writer.interface.print("\n{d} directories, {d} files\n", .{ counts.dirs, counts.files }); }, + .cp => { + const destination_path = it.next() orelse { + try stdout_writer.interface.print("Usage: cp \n", .{}); + return error.InvalidArgument; + }; + + const source = try std.Io.Dir.openFile(.cwd(), io, path, .{}); + defer source.close(io); + + const destination = try std.Io.Dir.createFile(.cwd(), io, destination_path, .{}); + defer destination.close(io); + + var reader: std.Io.File.Reader = .initStreaming(source, io, &.{}); + var writer = destination.writer(io, buffer); + + const read = try reader.interface.streamRemaining(&writer.interface); + try writer.interface.flush(); + + try stdout_writer.interface.print("Copied {B:.2} from {s} to {s}\n", .{ read, path, destination_path }); + }, .stat => { const stat = std.Io.Dir.statFile(.cwd(), io, path, .{}) catch |err| blk: { if (err == error.IsDir) { @@ -114,6 +121,34 @@ pub fn main() !void { try stdout_writer.interface.print("{s}\n", .{real_path_buf[0..len]}); }, + .safetensors => { + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); + defer registry.deinit(); + + const root = try TensorNode.init(init.arena.allocator(), ""); + + var registry_it = registry.iterator(); + while (registry_it.next()) |kv| { + const name = kv.key_ptr.*; + const tensor = kv.value_ptr.*; + + var current = root; + var parts = std.mem.tokenizeScalar(u8, name, '.'); + while (parts.next()) |part| { + const gop = try current.children.getOrPut(part); + if (!gop.found_existing) { + gop.value_ptr.* = try TensorNode.init(init.arena.allocator(), part); + } + current = gop.value_ptr.*; + } + + current.tensor = tensor; + } + + try stdout_writer.interface.print("{s}\n", .{path}); + try printTensorTree(&stdout_writer.interface, root, "", true, true); + try stdout_writer.interface.flush(); + }, } } @@ -166,3 +201,91 @@ fn printTree(io: std.Io, writer: *std.Io.Writer, dir: std.Io.Dir, prefix: []cons } } } + +const TensorNode = struct { + name: []const u8, + children: std.StringArrayHashMap(*TensorNode), + tensor: ?zml.safetensors.Tensor = null, + + fn init(allocator: std.mem.Allocator, name: []const u8) !*TensorNode { + const node = try allocator.create(TensorNode); + node.* = .{ + .name = name, + .children = std.StringArrayHashMap(*TensorNode).init(allocator), + .tensor = null, + }; + return node; + } +}; + +fn getSortedChildren(allocator: std.mem.Allocator, node: *TensorNode) ![]*TensorNode { + const children_nodes = try allocator.alloc(*TensorNode, node.children.count()); + var it = node.children.iterator(); + var idx: usize = 0; + while (it.next()) |entry| : (idx += 1) { + children_nodes[idx] = entry.value_ptr.*; + } + + std.mem.sort(*TensorNode, children_nodes, {}, struct { + fn lessThan(_: void, a: *TensorNode, b: *TensorNode) bool { + const a_num = std.fmt.parseInt(usize, a.name, 10) catch null; + const b_num = std.fmt.parseInt(usize, b.name, 10) catch null; + if (a_num != null and b_num != null) return a_num.? < b_num.?; + return std.mem.lessThan(u8, a.name, b.name); + } + }.lessThan); + return children_nodes; +} + +fn printTensorTree( + writer: anytype, + node: *TensorNode, + prefix: []const u8, + is_last: bool, + is_root: bool, +) !void { + const allocator = node.children.allocator; + + if (is_root) { + const children_nodes = try getSortedChildren(allocator, node); + for (children_nodes, 0..) |child, i| { + try printTensorTree(writer, child, "", i == children_nodes.len - 1, false); + } + return; + } + + var compacted_name = node.name; + var walk = node; + while (walk.children.count() == 1 and walk.tensor == null) { + const next = walk.children.values()[0]; + compacted_name = try std.fmt.allocPrint(allocator, "{s}.{s}", .{ compacted_name, next.name }); + walk = next; + } + + const connector = if (is_last) "└── " else "├── "; + try writer.print("{s}{s}{s}", .{ prefix, connector, compacted_name }); + + if (walk.tensor) |t| { + try writer.print(" [shape={f} size={B:.2}]", .{ t.shape, t.byteSize() }); + } + try writer.print("\n", .{}); + + const extension = if (is_last) " " else "│ "; + const child_prefix = try std.fmt.allocPrint(allocator, "{s}{s}", .{ prefix, extension }); + const children_nodes = try getSortedChildren(allocator, walk); + + const show_count = 2; + const skip_threshold = 8; + + for (children_nodes, 0..) |child, i| { + if (children_nodes.len > skip_threshold) { + if (i >= show_count and i < children_nodes.len - show_count) { + if (i == show_count) { + try writer.print("{s}├── ...\n", .{child_prefix}); + } + continue; + } + } + try printTensorTree(writer, child, child_prefix, i == children_nodes.len - 1, false); + } +} diff --git a/stdx/BUILD.bazel b/stdx/BUILD.bazel index 1dcfc56d..f42a73fa 100644 --- a/stdx/BUILD.bazel +++ b/stdx/BUILD.bazel @@ -5,6 +5,7 @@ zig_library( srcs = [ "Io.zig", "bounded_array.zig", + "crypto.zig", "debug.zig", "flags.zig", "fmt.zig", diff --git a/stdx/crypto.zig b/stdx/crypto.zig new file mode 100644 index 00000000..eb5729b6 --- /dev/null +++ b/stdx/crypto.zig @@ -0,0 +1,41 @@ +const std = @import("std"); + +pub fn MacWriter(comptime T: type) type { + return struct { + const Self = @This(); + + mac: *T, + interface: std.Io.Writer, + + pub fn init(mac_: *T) MacWriter(T) { + return .{ + .mac = mac_, + .interface = .{ + .buffer = &.{}, + .vtable = &.{ + .drain = drain, + }, + }, + }; + } + + pub fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + const self: *Self = @alignCast(@fieldParentPtr("interface", w)); + var total: usize = 0; + for (data) |chunk| { + self.mac.update(chunk); + total += chunk.len; + } + const last = data[data.len - 1]; + for (0..splat - 1) |_| { + self.mac.update(last); + total += last.len; + } + return total; + } + }; +} + +pub fn hmacWriter(hmac: anytype) MacWriter(std.meta.Child(@TypeOf(hmac))) { + return .init(hmac); +} diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 83e14277..5ccf1cb4 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -3,6 +3,7 @@ const builtin = @import("builtin"); pub const BoundedArray = @import("bounded_array.zig").BoundedArray; pub const BoundedArrayAligned = @import("bounded_array.zig").BoundedArrayAligned; +pub const crypto = @import("crypto.zig"); pub const debug = @import("debug.zig"); pub const flags = @import("flags.zig"); pub const fmt = @import("fmt.zig"); diff --git a/zml/io.zig b/zml/io.zig index 404843d2..9538987b 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -63,7 +63,7 @@ pub const TensorStore = struct { pub fn getReaderById(self: *const TensorStore, id: usize, io: std.Io, buffer: []u8) !safetensors.TensorReader { const tensor_desc = self.id_map.get(id) orelse return error.NotFound; - return safetensors.TensorReader.init(io, tensor_desc.*, buffer); + return safetensors.TensorReader.init(io, tensor_desc.*, buffer, .{}); } pub fn view(self: *TensorStore) View { diff --git a/zml/io/vfs.zig b/zml/io/vfs.zig index b9f61550..a282371c 100644 --- a/zml/io/vfs.zig +++ b/zml/io/vfs.zig @@ -16,7 +16,7 @@ pub const VFS = struct { const CWD_HANDLE: u32 = 0; - const Handle = struct { handle: u32, backend_idx: ?usize }; + const Handle = struct { handle: u32, backend_idx: ?usize, flags: std.Io.File.Flags = .{ .nonblocking = false } }; allocator: std.mem.Allocator, mutex: std.Io.Mutex = .init, @@ -32,6 +32,9 @@ pub const VFS = struct { var handles: Handles = .{}; try handles.append(allocator, .{ .handle = CWD_HANDLE, .backend_idx = null }); + try handles.append(allocator, .{ .handle = std.posix.STDIN_FILENO, .backend_idx = null }); + try handles.append(allocator, .{ .handle = std.posix.STDOUT_FILENO, .backend_idx = null }); + try handles.append(allocator, .{ .handle = std.posix.STDERR_FILENO, .backend_idx = null }); return .{ .allocator = allocator, @@ -64,10 +67,12 @@ pub const VFS = struct { return .{ .userdata = &self.base, .vtable = &comptime VFSBase.vtable(.{ + .operate = operate, .dirOpenDir = dirOpenDir, .dirStat = dirStat, .dirStatFile = dirStatFile, .dirAccess = dirAccess, + .dirCreateFile = dirCreateFile, .dirOpenFile = dirOpenFile, .dirClose = dirClose, .dirRead = dirRead, @@ -76,7 +81,9 @@ pub const VFS = struct { .fileStat = fileStat, .fileLength = fileLength, .fileClose = fileClose, - .fileReadStreaming = fileReadStreaming, + .fileWritePositional = fileWritePositional, + .fileWriteFileStreaming = fileWriteFileStreaming, + .fileWriteFilePositional = fileWriteFilePositional, .fileReadPositional = fileReadPositional, .fileSeekBy = fileSeekBy, .fileSeekTo = fileSeekTo, @@ -158,6 +165,22 @@ pub const VFS = struct { return path[uri.scheme.len + 3 ..]; } + fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + switch (operation) { + .file_read_streaming => |o| { + const handle, const backend = self.getFileHandle(o.file); + return backend.vtable.operate(backend.userdata, .{ .file_read_streaming = .{ + .file = .{ .handle = @intCast(handle.handle), .flags = handle.flags }, + .data = o.data, + } }); + }, + .device_io_control, .file_write_streaming => { + return self.base.inner.vtable.operate(self.base.inner.userdata, operation); + }, + } + } + fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { @@ -200,6 +223,22 @@ pub const VFS = struct { return backend.vtable.dirAccess(backend.userdata, dir_, stripScheme(sub_path), options); } + pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.CreateFlags) std.Io.File.OpenError!std.Io.File { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for dir create file '{s}' : {any}", .{ sub_path, err }); + return std.Io.Dir.AccessError.Unexpected; + }; + + const file = try backend.vtable.dirCreateFile(backend.userdata, dir_, stripScheme(sub_path), flags); + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; + handle.* = .{ + .handle = @intCast(file.handle), + .backend_idx = backend_idx, + }; + return .{ .handle = @intCast(idx), .flags = .{ .nonblocking = false } }; + } + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { @@ -212,7 +251,7 @@ pub const VFS = struct { .handle = @intCast(file.handle), .backend_idx = backend_idx, }; - return .{ .handle = @intCast(idx) }; + return .{ .handle = @intCast(idx), .flags = handle.flags }; } fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { @@ -276,46 +315,72 @@ pub const VFS = struct { fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const handle, const backend = self.getFileHandle(file); - return backend.vtable.fileStat(backend.userdata, .{ .handle = @intCast(handle.handle) }); + return backend.vtable.fileStat(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }); } fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const handle, const backend = self.getFileHandle(file); - return backend.vtable.fileLength(backend.userdata, .{ .handle = @intCast(handle.handle) }); + return backend.vtable.fileLength(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }); } fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); for (files) |file| { const handle, const backend = self.getFileHandle(file); - backend.vtable.fileClose(backend.userdata, &.{.{ .handle = @intCast(handle.handle) }}); + backend.vtable.fileClose(backend.userdata, &.{.{ .handle = @intCast(handle.handle), .flags = handle.flags }}); self.closeHandle(@intCast(file.handle)) catch unreachable; } } - fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { + fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) std.Io.File.WritePositionalError!usize { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const handle, const backend = self.getFileHandle(file); - return backend.vtable.fileReadStreaming(backend.userdata, .{ .handle = @intCast(handle.handle) }, data); + return backend.vtable.fileWritePositional(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, header, data, splat, offset); + } + + fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + + const dst_handle, const dst_backend = self.getFileHandle(file); + const src_handle, _ = self.getFileHandle(reader.file); + + const original_src_handle = reader.file; + reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; + defer reader.file = original_src_handle; + + return dst_backend.vtable.fileWriteFileStreaming(dst_backend.userdata, .{ .handle = @intCast(dst_handle.handle), .flags = dst_handle.flags }, header, reader, limit); + } + + fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + + const dst_handle, const dst_backend = self.getFileHandle(file); + const src_handle, _ = self.getFileHandle(reader.file); + + const original_src_handle = reader.file; + reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; + defer reader.file = original_src_handle; + + return dst_backend.vtable.fileWriteFilePositional(dst_backend.userdata, .{ .handle = @intCast(dst_handle.handle), .flags = dst_handle.flags }, header, reader, limit, offset); } fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const handle, const backend = self.getFileHandle(file); - return backend.vtable.fileReadPositional(backend.userdata, .{ .handle = @intCast(handle.handle) }, data, offset); + return backend.vtable.fileReadPositional(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, data, offset); } fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const handle, const backend = self.getFileHandle(file); - return backend.vtable.fileSeekBy(backend.userdata, .{ .handle = @intCast(handle.handle) }, relative_offset); + return backend.vtable.fileSeekBy(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, relative_offset); } fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const handle, const backend = self.getFileHandle(file); - return backend.vtable.fileSeekTo(backend.userdata, .{ .handle = @intCast(handle.handle) }, absolute_offset); + return backend.vtable.fileSeekTo(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, absolute_offset); } fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { @@ -324,10 +389,10 @@ pub const VFS = struct { if (self.getScheme(handle.backend_idx)) |s| { const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); - const path_len = try backend.vtable.fileRealPath(backend.userdata, .{ .handle = @intCast(handle.handle) }, out_buffer[prefix.len..]); + const path_len = try backend.vtable.fileRealPath(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, out_buffer[prefix.len..]); return prefix.len + path_len; } else { - return try backend.vtable.fileRealPath(backend.userdata, .{ .handle = @intCast(handle.handle) }, out_buffer); + return try backend.vtable.fileRealPath(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, out_buffer); } } }; diff --git a/zml/io/vfs/base.zig b/zml/io/vfs/base.zig index e586d10b..041d4689 100644 --- a/zml/io/vfs/base.zig +++ b/zml/io/vfs/base.zig @@ -11,6 +11,7 @@ pub const VFSBase = struct { pub fn vtable(overrides: anytype) std.Io.VTable { var new_vtable: std.Io.VTable = .{ + .crashHandler = crashHandler, .async = async, .concurrent = concurrent, .await = await, @@ -26,6 +27,10 @@ pub const VFSBase = struct { .futexWait = futexWait, .futexWaitUncancelable = futexWaitUncancelable, .futexWake = futexWake, + .operate = operate, + .batchAwaitAsync = batchAwaitAsync, + .batchAwaitConcurrent = batchAwaitConcurrent, + .batchCancel = batchCancel, .dirCreateDir = dirCreateDir, .dirCreateDirPath = dirCreateDirPath, .dirCreateDirPathOpen = dirCreateDirPathOpen, @@ -34,6 +39,7 @@ pub const VFSBase = struct { .dirStatFile = dirStatFile, .dirAccess = dirAccess, .dirCreateFile = dirCreateFile, + .dirCreateFileAtomic = dirCreateFileAtomic, .dirOpenFile = dirOpenFile, .dirClose = dirClose, .dirRead = dirRead, @@ -42,6 +48,7 @@ pub const VFSBase = struct { .dirDeleteFile = dirDeleteFile, .dirDeleteDir = dirDeleteDir, .dirRename = dirRename, + .dirRenamePreserve = dirRenamePreserve, .dirSymLink = dirSymLink, .dirReadLink = dirReadLink, .dirSetOwner = dirSetOwner, @@ -53,11 +60,9 @@ pub const VFSBase = struct { .fileStat = fileStat, .fileLength = fileLength, .fileClose = fileClose, - .fileWriteStreaming = fileWriteStreaming, .fileWritePositional = fileWritePositional, .fileWriteFileStreaming = fileWriteFileStreaming, .fileWriteFilePositional = fileWriteFilePositional, - .fileReadStreaming = fileReadStreaming, .fileReadPositional = fileReadPositional, .fileSeekBy = fileSeekBy, .fileSeekTo = fileSeekTo, @@ -74,20 +79,38 @@ pub const VFSBase = struct { .fileUnlock = fileUnlock, .fileDowngradeLock = fileDowngradeLock, .fileRealPath = fileRealPath, + .fileHardLink = fileHardLink, + .fileMemoryMapCreate = fileMemoryMapCreate, + .fileMemoryMapDestroy = fileMemoryMapDestroy, + .fileMemoryMapSetLength = fileMemoryMapSetLength, + .fileMemoryMapRead = fileMemoryMapRead, + .fileMemoryMapWrite = fileMemoryMapWrite, .processExecutableOpen = processExecutableOpen, .processExecutablePath = processExecutablePath, .lockStderr = lockStderr, .tryLockStderr = tryLockStderr, .unlockStderr = unlockStderr, + .processCurrentPath = processCurrentPath, .processSetCurrentDir = processSetCurrentDir, + .processReplace = processReplace, + .processReplacePath = processReplacePath, + .processSpawn = processSpawn, + .processSpawnPath = processSpawnPath, + .childWait = childWait, + .childKill = childKill, + .progressParentFile = progressParentFile, .now = now, + .clockResolution = clockResolution, .sleep = sleep, + .random = random, + .randomSecure = randomSecure, .netListenIp = netListenIp, .netAccept = netAccept, .netBindIp = netBindIp, .netConnectIp = netConnectIp, .netListenUnix = netListenUnix, .netConnectUnix = netConnectUnix, + .netSocketCreatePair = netSocketCreatePair, .netSend = netSend, .netReceive = netReceive, .netRead = netRead, @@ -109,6 +132,11 @@ pub const VFSBase = struct { return @ptrCast(@alignCast(userdata.?)); } + pub fn crashHandler(userdata: ?*anyopaque) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + self.inner.vtable.crashHandler(self.inner.userdata); + } + pub fn async(userdata: ?*anyopaque, result: []u8, result_alignment: std.mem.Alignment, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque, result: *anyopaque) void) ?*std.Io.AnyFuture { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.async(self.inner.userdata, result, result_alignment, context, context_alignment, start); @@ -184,6 +212,26 @@ pub const VFSBase = struct { return self.inner.vtable.futexWake(self.inner.userdata, ptr, max_waiters); } + pub fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.operate(self.inner.userdata, operation); + } + + pub fn batchAwaitAsync(userdata: ?*anyopaque, batch: *std.Io.Batch) std.Io.Cancelable!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.batchAwaitAsync(self.inner.userdata, batch); + } + + pub fn batchAwaitConcurrent(userdata: ?*anyopaque, batch: *std.Io.Batch, timeout: std.Io.Timeout) std.Io.Batch.AwaitConcurrentError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.batchAwaitConcurrent(self.inner.userdata, batch, timeout); + } + + pub fn batchCancel(userdata: ?*anyopaque, batch: *std.Io.Batch) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + self.inner.vtable.batchCancel(self.inner.userdata, batch); + } + pub fn dirCreateDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions) std.Io.Dir.CreateDirError!void { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirCreateDir(self.inner.userdata, dir, sub_path, permissions); @@ -224,6 +272,11 @@ pub const VFSBase = struct { return self.inner.vtable.dirCreateFile(self.inner.userdata, dir, sub_path, flags); } + pub fn dirCreateFileAtomic(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.CreateFileAtomicOptions) std.Io.Dir.CreateFileAtomicError!std.Io.File.Atomic { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirCreateFileAtomic(self.inner.userdata, dir, sub_path, options); + } + pub fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirOpenFile(self.inner.userdata, dir, sub_path, flags); @@ -264,6 +317,11 @@ pub const VFSBase = struct { return self.inner.vtable.dirRename(self.inner.userdata, old_dir, old_sub_path, new_dir, new_sub_path); } + pub fn dirRenamePreserve(userdata: ?*anyopaque, old_dir: std.Io.Dir, old_sub_path: []const u8, new_dir: std.Io.Dir, new_sub_path: []const u8) std.Io.Dir.RenamePreserveError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirRenamePreserve(self.inner.userdata, old_dir, old_sub_path, new_dir, new_sub_path); + } + pub fn dirSymLink(userdata: ?*anyopaque, dir: std.Io.Dir, target_path: []const u8, sym_link_path: []const u8, flags: std.Io.Dir.SymLinkFlags) std.Io.Dir.SymLinkError!void { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirSymLink(self.inner.userdata, dir, target_path, sym_link_path, flags); @@ -319,11 +377,6 @@ pub const VFSBase = struct { self.inner.vtable.fileClose(self.inner.userdata, files); } - pub fn fileWriteStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize) std.Io.File.Writer.Error!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.fileWriteStreaming(self.inner.userdata, file, header, data, splat); - } - pub fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) std.Io.File.WritePositionalError!usize { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileWritePositional(self.inner.userdata, file, header, data, splat, offset); @@ -339,11 +392,6 @@ pub const VFSBase = struct { return self.inner.vtable.fileWriteFilePositional(self.inner.userdata, file, header, reader, limit, offset); } - pub fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.fileReadStreaming(self.inner.userdata, file, data); - } - pub fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileReadPositional(self.inner.userdata, file, data, offset); @@ -424,6 +472,36 @@ pub const VFSBase = struct { return self.inner.vtable.fileRealPath(self.inner.userdata, file, out_buffer); } + pub fn fileHardLink(userdata: ?*anyopaque, file: std.Io.File, dir: std.Io.Dir, path: []const u8, options: std.Io.File.HardLinkOptions) std.Io.File.HardLinkError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileHardLink(self.inner.userdata, file, dir, path, options); + } + + pub fn fileMemoryMapCreate(userdata: ?*anyopaque, file: std.Io.File, options: std.Io.File.MemoryMap.CreateOptions) std.Io.File.MemoryMap.CreateError!std.Io.File.MemoryMap { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileMemoryMapCreate(self.inner.userdata, file, options); + } + + pub fn fileMemoryMapDestroy(userdata: ?*anyopaque, memory_map: *std.Io.File.MemoryMap) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + self.inner.vtable.fileMemoryMapDestroy(self.inner.userdata, memory_map); + } + + pub fn fileMemoryMapSetLength(userdata: ?*anyopaque, memory_map: *std.Io.File.MemoryMap, length: usize) std.Io.File.MemoryMap.SetLengthError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileMemoryMapSetLength(self.inner.userdata, memory_map, length); + } + + pub fn fileMemoryMapRead(userdata: ?*anyopaque, memory_map: *std.Io.File.MemoryMap) std.Io.File.ReadPositionalError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileMemoryMapRead(self.inner.userdata, memory_map); + } + + pub fn fileMemoryMapWrite(userdata: ?*anyopaque, memory_map: *std.Io.File.MemoryMap) std.Io.File.WritePositionalError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileMemoryMapWrite(self.inner.userdata, memory_map); + } + pub fn processExecutableOpen(userdata: ?*anyopaque, flags: std.Io.File.OpenFlags) std.process.OpenExecutableError!std.Io.File { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.processExecutableOpen(self.inner.userdata, flags); @@ -449,21 +527,76 @@ pub const VFSBase = struct { return self.inner.vtable.unlockStderr(self.inner.userdata); } + pub fn processCurrentPath(userdata: ?*anyopaque, buffer: []u8) std.process.CurrentPathError!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.processCurrentPath(self.inner.userdata, buffer); + } + pub fn processSetCurrentDir(userdata: ?*anyopaque, dir: std.Io.Dir) std.process.SetCurrentDirError!void { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.processSetCurrentDir(self.inner.userdata, dir); } - pub fn now(userdata: ?*anyopaque, clock: std.Io.Clock) std.Io.Clock.Error!std.Io.Timestamp { + pub fn processReplace(userdata: ?*anyopaque, options: std.process.ReplaceOptions) std.process.ReplaceError { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.processReplace(self.inner.userdata, options); + } + + pub fn processReplacePath(userdata: ?*anyopaque, dir: std.Io.Dir, options: std.process.ReplaceOptions) std.process.ReplaceError { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.processReplacePath(self.inner.userdata, dir, options); + } + + pub fn processSpawn(userdata: ?*anyopaque, options: std.process.SpawnOptions) std.process.SpawnError!std.process.Child { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.processSpawn(self.inner.userdata, options); + } + + pub fn processSpawnPath(userdata: ?*anyopaque, dir: std.Io.Dir, options: std.process.SpawnOptions) std.process.SpawnError!std.process.Child { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.processSpawnPath(self.inner.userdata, dir, options); + } + + pub fn childWait(userdata: ?*anyopaque, child: *std.process.Child) std.process.Child.WaitError!std.process.Child.Term { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.childWait(self.inner.userdata, child); + } + + pub fn childKill(userdata: ?*anyopaque, child: *std.process.Child) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + self.inner.vtable.childKill(self.inner.userdata, child); + } + + pub fn progressParentFile(userdata: ?*anyopaque) std.Progress.ParentFileError!std.Io.File { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.progressParentFile(self.inner.userdata); + } + + pub fn now(userdata: ?*anyopaque, clock: std.Io.Clock) std.Io.Timestamp { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.now(self.inner.userdata, clock); } - pub fn sleep(userdata: ?*anyopaque, timeout: std.Io.Timeout) std.Io.SleepError!void { + pub fn clockResolution(userdata: ?*anyopaque, clock: std.Io.Clock) std.Io.Clock.ResolutionError!std.Io.Duration { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.clockResolution(self.inner.userdata, clock); + } + + pub fn sleep(userdata: ?*anyopaque, timeout: std.Io.Timeout) std.Io.Cancelable!void { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.sleep(self.inner.userdata, timeout); } + pub fn random(userdata: ?*anyopaque, buffer: []u8) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + self.inner.vtable.random(self.inner.userdata, buffer); + } + + pub fn randomSecure(userdata: ?*anyopaque, buffer: []u8) std.Io.RandomSecureError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.randomSecure(self.inner.userdata, buffer); + } + pub fn netListenIp(userdata: ?*anyopaque, address: std.Io.net.IpAddress, options: std.Io.net.IpAddress.ListenOptions) std.Io.net.IpAddress.ListenError!std.Io.net.Server { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netListenIp(self.inner.userdata, address, options); @@ -494,6 +627,11 @@ pub const VFSBase = struct { return self.inner.vtable.netConnectUnix(self.inner.userdata, address); } + pub fn netSocketCreatePair(userdata: ?*anyopaque, options: std.Io.net.Socket.CreatePairOptions) std.Io.net.Socket.CreatePairError![2]std.Io.net.Socket { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.netSocketCreatePair(self.inner.userdata, options); + } + pub fn netSend(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, msgs: []std.Io.net.OutgoingMessage, flags: std.Io.net.SendFlags) struct { ?std.Io.net.Socket.SendError, usize } { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netSend(self.inner.userdata, handle, msgs, flags); diff --git a/zml/io/vfs/file.zig b/zml/io/vfs/file.zig index 0c52eada..ff8df678 100644 --- a/zml/io/vfs/file.zig +++ b/zml/io/vfs/file.zig @@ -12,7 +12,7 @@ pub const DirectIoError = error{ MisalignedBuffer, BufferTooSmall, UnexpectedFcntlResult, -} || std.posix.FcntlError; +}; fn canUseDirectIO() bool { if (builtin.target.os.tag == .linux) { @@ -23,7 +23,7 @@ fn canUseDirectIO() bool { fn useDirectIO(file: std.Io.File) DirectIoError!bool { if (comptime canUseDirectIO()) { - const flags = try std.posix.fcntl(file.handle, std.posix.F.GETFL, 0); + const flags = std.posix.system.fcntl(file.handle, std.posix.F.GETFL); const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); return (flags & direct_flag) != 0; } else { @@ -33,7 +33,7 @@ fn useDirectIO(file: std.Io.File) DirectIoError!bool { fn switchToBufferedIO(file: std.fs.File) DirectIoError!void { if (comptime canUseDirectIO()) { - const flags = try std.posix.fcntl(file.handle, std.posix.F.GETFL, 0); + const flags = std.posix.system.fcntl(file.handle, std.posix.F.GETFL); const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); if ((flags & direct_flag) == 0) return; @@ -46,10 +46,10 @@ fn switchToBufferedIO(file: std.fs.File) DirectIoError!void { fn switchToDirectIO(file: std.Io.File) DirectIoError!void { if (builtin.os.tag == .linux and canUseDirectIO()) { - const flags = try std.posix.fcntl(file.handle, std.posix.F.GETFL, 0); + const flags = std.posix.system.fcntl(file.handle, std.posix.F.GETFL); const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); - const result = try std.posix.fcntl(file.handle, std.posix.F.SETFL, flags | direct_flag); + const result = std.posix.system.fcntl(file.handle, std.posix.F.SETFL, flags | direct_flag); if (result != 0) return DirectIoError.UnexpectedFcntlResult; } else { return DirectIoError.UnsupportedPlatform; @@ -62,7 +62,25 @@ pub const File = struct { direct_io_alignment: std.mem.Alignment = .fromByteUnits(4 * 1024), }; - const Handle = struct { inner_handle: std.Io.File.Handle, direct_io_handle: ?std.Io.File.Handle }; + const Handle = struct { + inner_handle: std.Io.File.Handle, + inner_flags: std.Io.File.Flags, + direct_io_handle: ?std.Io.File.Handle, + + pub fn innerFile(self: *Handle) std.Io.File { + return .{ + .handle = self.inner_handle, + .flags = self.inner_flags, + }; + } + + pub fn directIoFile(self: *Handle) std.Io.File { + return .{ + .handle = self.direct_io_handle.?, + .flags = self.inner_flags, + }; + } + }; allocator: std.mem.Allocator, mutex: std.Io.Mutex = .init, @@ -88,6 +106,7 @@ pub const File = struct { return .{ .userdata = &self.base, .vtable = &comptime VFSBase.vtable(.{ + .operate = operate, .dirOpenDir = dirOpenDir, .dirStat = dirStat, .dirStatFile = dirStatFile, @@ -100,7 +119,6 @@ pub const File = struct { .fileStat = fileStat, .fileLength = fileLength, .fileClose = fileClose, - .fileReadStreaming = fileReadStreaming, .fileReadPositional = fileReadPositional, .fileSeekBy = fileSeekBy, .fileSeekTo = fileSeekTo, @@ -143,27 +161,50 @@ pub const File = struct { fn innerFile(self: *File, file: std.Io.File, data: []const []u8, position: u64) std.Io.File { const handle = self.getFileHandle(file); - const direct_io_handle = handle.direct_io_handle orelse return .{ .handle = handle.inner_handle }; + + if (handle.direct_io_handle == null) { + return handle.innerFile(); + } const alignment_bytes: usize = self.config.direct_io_alignment.toByteUnits(); if (!std.mem.isAligned(@as(usize, position), alignment_bytes)) { - return .{ .handle = handle.inner_handle }; + return handle.innerFile(); } var total_size: usize = 0; for (data) |buf| { if (!std.mem.isAligned(@intFromPtr(buf.ptr), alignment_bytes)) { - return .{ .handle = handle.inner_handle }; + return handle.innerFile(); } total_size += buf.len; } if (!std.mem.isAligned(total_size, alignment_bytes)) { - return .{ .handle = handle.inner_handle }; + return handle.innerFile(); } - return .{ .handle = direct_io_handle }; + return handle.directIoFile(); + } + + fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + switch (operation) { + .file_read_streaming => |o| { + const handle = self.getFileHandle(o.file); + const position: u64 = @intCast(std.posix.system.lseek(handle.inner_handle, 0, std.posix.SEEK.CUR)); + const inner_file = self.innerFile(o.file, o.data, position); + return self.base.inner.vtable.operate(self.base.inner.userdata, .{ + .file_read_streaming = .{ + .file = inner_file, + .data = o.data, + }, + }); + }, + .file_write_streaming, .device_io_control => { + return self.base.inner.vtable.operate(self.base.inner.userdata, operation); + }, + } } fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { @@ -216,10 +257,11 @@ pub const File = struct { const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; handle.* = .{ .inner_handle = inner_file.handle, + .inner_flags = inner_file.flags, .direct_io_handle = direct_io_handle, }; - return .{ .handle = @intCast(idx) }; + return .{ .handle = @intCast(idx), .flags = inner_file.flags }; } fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { @@ -245,15 +287,13 @@ pub const File = struct { fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); - const inner_file: std.Io.File = .{ .handle = handle.inner_handle }; - return self.base.inner.vtable.fileStat(self.base.inner.userdata, inner_file); + return self.base.inner.vtable.fileStat(self.base.inner.userdata, handle.innerFile()); } fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); - const inner_file: std.Io.File = .{ .handle = handle.inner_handle }; - return self.base.inner.vtable.fileLength(self.base.inner.userdata, inner_file); + return self.base.inner.vtable.fileLength(self.base.inner.userdata, handle.innerFile()); } fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { @@ -261,26 +301,16 @@ pub const File = struct { for (files) |file| { const handle = self.getFileHandle(file); - if (handle.direct_io_handle) |dio_handle| { - const direct_io_file: std.Io.File = .{ .handle = dio_handle }; - self.base.inner.vtable.fileClose(self.base.inner.userdata, &.{direct_io_file}); + if (handle.direct_io_handle) |_| { + self.base.inner.vtable.fileClose(self.base.inner.userdata, &.{handle.directIoFile()}); } - const inner_file: std.Io.File = .{ .handle = handle.inner_handle }; - self.base.inner.vtable.fileClose(self.base.inner.userdata, &.{inner_file}); + self.base.inner.vtable.fileClose(self.base.inner.userdata, &.{handle.innerFile()}); self.closeHandle(@intCast(file.handle)) catch unreachable; } } - fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - const position: u64 = @intCast(std.posix.system.lseek(handle.inner_handle, 0, std.posix.SEEK.CUR)); - const inner_file = self.innerFile(file, data, position); - return self.base.inner.vtable.fileReadStreaming(self.base.inner.userdata, inner_file, data); - } - fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); const inner_file = self.innerFile(file, data, offset); @@ -290,21 +320,18 @@ pub const File = struct { fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); - const inner_file: std.Io.File = .{ .handle = handle.inner_handle }; - return self.base.inner.vtable.fileSeekBy(self.base.inner.userdata, inner_file, relative_offset); + return self.base.inner.vtable.fileSeekBy(self.base.inner.userdata, handle.innerFile(), relative_offset); } fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); - const inner_file: std.Io.File = .{ .handle = handle.inner_handle }; - return self.base.inner.vtable.fileSeekTo(self.base.inner.userdata, inner_file, absolute_offset); + return self.base.inner.vtable.fileSeekTo(self.base.inner.userdata, handle.innerFile(), absolute_offset); } fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); - const inner_file: std.Io.File = .{ .handle = handle.inner_handle }; - return self.base.inner.vtable.fileRealPath(self.base.inner.userdata, inner_file, out_buffer); + return self.base.inner.vtable.fileRealPath(self.base.inner.userdata, handle.innerFile(), out_buffer); } }; diff --git a/zml/io/vfs/hf.zig b/zml/io/vfs/hf.zig index 50e2811b..651f9d5f 100644 --- a/zml/io/vfs/hf.zig +++ b/zml/io/vfs/hf.zig @@ -123,12 +123,11 @@ pub const HF = struct { }; } - pub fn auto(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client) !HF { - const hf_token = if (std.process.getEnvVarOwned(allocator, "HF_TOKEN")) |token| blk: { + pub fn auto(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, environ_map: *std.process.Environ.Map) !HF { + const hf_token = if (environ_map.get("HF_TOKEN")) |token| blk: { break :blk token; - } else |_| blk: { - const home_path = std.process.getEnvVarOwned(allocator, "HOME") catch break :blk null; - defer allocator.free(home_path); + } else blk: { + const home_path = environ_map.get("HOME") orelse break :blk null; var path_buf: [256]u8 = undefined; const token_path = std.fmt.bufPrint(&path_buf, "{s}/.cache/huggingface/token", .{home_path}) catch break :blk null; @@ -142,7 +141,6 @@ pub const HF = struct { break :blk token; }; - defer if (hf_token) |token| allocator.free(token); if (hf_token == null) log.warn("No Hugging Face authentication token found in environment or home config; proceeding without authentication.", .{}); @@ -184,6 +182,7 @@ pub const HF = struct { return .{ .userdata = &self.base, .vtable = &comptime VFSBase.vtable(.{ + .operate = operate, .dirOpenDir = dirOpenDir, .dirStat = dirStat, .dirStatFile = dirStatFile, @@ -196,7 +195,6 @@ pub const HF = struct { .fileStat = fileStat, .fileLength = fileLength, .fileClose = fileClose, - .fileReadStreaming = fileReadStreaming, .fileReadPositional = fileReadPositional, .fileSeekBy = fileSeekBy, .fileSeekTo = fileSeekTo, @@ -453,6 +451,29 @@ pub const HF = struct { } } + fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + switch (operation) { + .file_read_streaming => |o| { + const handle = self.getFileHandle(o.file); + const total = self.performRead(handle, o.data, handle.pos) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); + return .{ .file_read_streaming = std.Io.File.ReadStreamingError.EndOfStream }; + }; + + if (total == 0) { + return .{ .file_read_streaming = std.Io.File.ReadStreamingError.EndOfStream }; + } + + handle.pos += @intCast(total); + return .{ .file_read_streaming = total }; + }, + .file_write_streaming, .device_io_control => { + return self.base.inner.vtable.operate(self.base.inner.userdata, operation); + }, + } + } + fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); @@ -482,6 +503,7 @@ pub const HF = struct { .atime = null, .mtime = std.Io.Timestamp.zero, .ctime = std.Io.Timestamp.zero, + .block_size = 1, }; } @@ -507,6 +529,7 @@ pub const HF = struct { .atime = null, .mtime = std.Io.Timestamp.zero, .ctime = std.Io.Timestamp.zero, + .block_size = 1, }; } @@ -538,7 +561,7 @@ pub const HF = struct { const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; handle.* = Handle.init(self.allocator, .file, full_path, size) catch return std.Io.File.OpenError.Unexpected; - return .{ .handle = @intCast(idx) }; + return .{ .handle = @intCast(idx), .flags = .{ .nonblocking = false } }; } fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { @@ -619,6 +642,7 @@ pub const HF = struct { .atime = null, .mtime = std.Io.Timestamp.zero, .ctime = std.Io.Timestamp.zero, + .block_size = 1, }; } @@ -634,17 +658,6 @@ pub const HF = struct { } } - fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { - const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - const total = self.performRead(handle, data, handle.pos) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return std.Io.File.Reader.Error.Unexpected; - }; - handle.pos += @intCast(total); - return total; - } - fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); diff --git a/zml/io/vfs/http.zig b/zml/io/vfs/http.zig index 6106ff08..c2bfa332 100644 --- a/zml/io/vfs/http.zig +++ b/zml/io/vfs/http.zig @@ -73,6 +73,7 @@ pub const HTTP = struct { return .{ .userdata = &self.base, .vtable = &comptime VFSBase.vtable(.{ + .operate = operate, .dirOpenDir = dirOpenDir, .dirStat = dirStat, .dirStatFile = dirStatFile, @@ -85,7 +86,6 @@ pub const HTTP = struct { .fileStat = fileStat, .fileLength = fileLength, .fileClose = fileClose, - .fileReadStreaming = fileReadStreaming, .fileReadPositional = fileReadPositional, .fileSeekBy = fileSeekBy, .fileSeekTo = fileSeekTo, @@ -139,6 +139,29 @@ pub const HTTP = struct { return try std.fmt.bufPrint(out_buffer, "{s}/{s}", .{ trimmed_uri, trimmed_sub_path }); } + fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + switch (operation) { + .file_read_streaming => |o| { + const handle = self.getFileHandle(o.file); + const total = self.performRead(handle, o.data, handle.pos) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); + return .{ .file_read_streaming = std.Io.File.ReadStreamingError.EndOfStream }; + }; + + if (total == 0) { + return .{ .file_read_streaming = std.Io.File.ReadStreamingError.EndOfStream }; + } + + handle.pos += @intCast(total); + return .{ .file_read_streaming = total }; + }, + .file_write_streaming, .device_io_control => { + return self.base.inner.vtable.operate(self.base.inner.userdata, operation); + }, + } + } + fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); @@ -164,6 +187,7 @@ pub const HTTP = struct { .atime = null, .mtime = std.Io.Timestamp.zero, .ctime = std.Io.Timestamp.zero, + .block_size = 0, }; } @@ -180,6 +204,7 @@ pub const HTTP = struct { .atime = null, .mtime = std.Io.Timestamp.zero, .ctime = std.Io.Timestamp.zero, + .block_size = 1, }; } @@ -196,7 +221,7 @@ pub const HTTP = struct { const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; - return .{ .handle = @intCast(idx) }; + return .{ .handle = @intCast(idx), .flags = .{ .nonblocking = false } }; } fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { @@ -238,6 +263,7 @@ pub const HTTP = struct { .atime = null, .mtime = std.Io.Timestamp.zero, .ctime = std.Io.Timestamp.zero, + .block_size = 1, }; } @@ -253,17 +279,6 @@ pub const HTTP = struct { } } - fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { - const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - const total = self.performRead(handle, data, handle.pos) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return std.Io.File.Reader.Error.Unexpected; - }; - handle.pos += @intCast(total); - return total; - } - fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); diff --git a/zml/io/vfs/s3.zig b/zml/io/vfs/s3.zig index 288ab054..193009e0 100644 --- a/zml/io/vfs/s3.zig +++ b/zml/io/vfs/s3.zig @@ -7,175 +7,147 @@ const VFSBase = @import("base.zig").VFSBase; const log = std.log.scoped(.@"zml/io/vfs/s3"); pub const AwsSigV4 = struct { - access_key: []const u8, - secret_key: []const u8, + access_key: ?[]const u8, + secret_key: ?[]const u8, region: []const u8, - service: []const u8 = "s3", + service: []const u8, const ALGORITHM = "AWS4-HMAC-SHA256"; pub const UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD"; pub fn generateAuthHeader( self: AwsSigV4, - allocator: std.mem.Allocator, + output_buffer: []u8, method: std.http.Method, - endpoint: []const u8, uri: std.Uri, timestamp: []const u8, extra_headers: []const std.http.Header, - ) ![]const u8 { + ) !?[]const u8 { + const access_key = self.access_key orelse return null; + _ = self.secret_key orelse return null; + const date = timestamp[0..8]; - var buf: [256]u8 = undefined; - const host = getHostForEndpoint(endpoint, &buf); + var header_buf: [32]std.http.Header = undefined; + var header_count: usize = 0; - // Build sorted headers list - var headers: std.ArrayList(std.http.Header) = .{}; - defer headers.deinit(allocator); + const host = uri.host orelse return error.MissingHost; + const host_header: std.http.Header = blk: { + const host_ = switch (host) { + .raw, .percent_encoded => |h| h, + }; + if (uri.port) |port| { + var host_buf: [128]u8 = undefined; + break :blk .{ .name = "host", .value = try std.fmt.bufPrint(&host_buf, "{s}:{d}", .{ host_, port }) }; + } else { + break :blk .{ .name = "host", .value = host_ }; + } + }; - try headers.append(allocator, .{ .name = "host", .value = host }); - try headers.append(allocator, .{ .name = "x-amz-content-sha256", .value = UNSIGNED_PAYLOAD }); - try headers.append(allocator, .{ .name = "x-amz-date", .value = timestamp }); + header_buf[0] = host_header; + header_buf[1] = .{ .name = "x-amz-content-sha256", .value = UNSIGNED_PAYLOAD }; + header_buf[2] = .{ .name = "x-amz-date", .value = timestamp }; + header_count = 3; for (extra_headers) |h| { - try headers.append(allocator, .{ .name = h.name, .value = h.value }); + if (header_count >= header_buf.len) return error.TooManyHeaders; + header_buf[header_count] = h; + header_count += 1; } - // Important: sort by header name - std.mem.sort( - std.http.Header, - headers.items, - {}, - struct { - fn lessThan(_: void, a: std.http.Header, b: std.http.Header) bool { - return std.mem.order(u8, a.name, b.name) == .lt; - } - }.lessThan, - ); + const active_headers = header_buf[0..header_count]; - // Build canonical headers and signed headers - var canonical_headers_list: std.ArrayList(u8) = .{}; - defer canonical_headers_list.deinit(allocator); + std.mem.sort(std.http.Header, active_headers, {}, struct { + fn less(_: void, a: std.http.Header, b: std.http.Header) bool { + return std.ascii.lessThanIgnoreCase(a.name, b.name); + } + }.less); - var signed_headers_list: std.ArrayList(u8) = .{}; - defer signed_headers_list.deinit(allocator); + var sha = std.crypto.hash.sha2.Sha256.init(.{}); + var sha_writer = stdx.crypto.hmacWriter(&sha); + const sw = &sha_writer.interface; - for (headers.items, 0..) |header, i| { - // Add to canonical headers: "name:value\n" - try canonical_headers_list.appendSlice(allocator, header.name); - try canonical_headers_list.append(allocator, ':'); - try canonical_headers_list.appendSlice(allocator, header.value); - try canonical_headers_list.append(allocator, '\n'); - - // Add to signed headers list: "name1;name2;name3" - if (i > 0) try signed_headers_list.append(allocator, ';'); - try signed_headers_list.appendSlice(allocator, header.name); + try sw.print("{s}\n", .{@tagName(method)}); + if (uri.path.isEmpty()) { + try sw.writeAll("/\n"); + } else { + try uri.path.formatPath(sw); + try sw.writeByte('\n'); } - const canonical_headers = canonical_headers_list.items; - const signed_headers = signed_headers_list.items; + if (uri.query) |query| { + try query.formatEscaped(sw); + try sw.writeByte('\n'); + } else { + try sw.writeAll("\n"); + } - // Build canonical request - var canonical_request_list: std.ArrayList(u8) = .{}; - defer canonical_request_list.deinit(allocator); + var signed_headers_buf: [512]u8 = undefined; + var sh_writer = std.Io.Writer.fixed(&signed_headers_buf); + for (active_headers, 0..) |h, i| { + for (h.name) |char| try sw.writeByte(std.ascii.toLower(char)); + try sw.print(":{s}\n", .{h.value}); + if (i > 0) _ = try sh_writer.write(";"); + for (h.name) |char| try sh_writer.writeByte(std.ascii.toLower(char)); + } + const signed_headers = sh_writer.buffered(); - const canonical_uri = if (uri.path.percent_encoded.len > 0) uri.path.percent_encoded else "/"; - const canonical_query = ""; // todo + try sw.print("\n{s}\n{s}", .{ signed_headers, UNSIGNED_PAYLOAD }); - // METHOD\nURI\nQUERY\nHEADERS\nSIGNED_HEADERS\nPAYLOAD_HASH - try canonical_request_list.appendSlice(allocator, @tagName(method)); - try canonical_request_list.append(allocator, '\n'); - try canonical_request_list.appendSlice(allocator, canonical_uri); - try canonical_request_list.append(allocator, '\n'); - try canonical_request_list.appendSlice(allocator, canonical_query); - try canonical_request_list.append(allocator, '\n'); - try canonical_request_list.appendSlice(allocator, canonical_headers); - try canonical_request_list.append(allocator, '\n'); - try canonical_request_list.appendSlice(allocator, signed_headers); - try canonical_request_list.append(allocator, '\n'); - try canonical_request_list.appendSlice(allocator, UNSIGNED_PAYLOAD); + var canonical_request_hash: [32]u8 = undefined; + sha.final(&canonical_request_hash); - // Hash canonical request - const cr_hash = sha256(canonical_request_list.items); - var cr_hash_hex: [64]u8 = undefined; - _ = std.fmt.bufPrint(&cr_hash_hex, "{x}", .{&cr_hash}) catch unreachable; + var scope_buf: [128]u8 = undefined; + const scope = try std.fmt.bufPrint(&scope_buf, "{s}/{s}/{s}/aws4_request", .{ date, self.region, self.service }); - // Build credential scope: date/region/service/aws4_request - const credential_scope = try std.fmt.allocPrint(allocator, "{s}/{s}/{s}/aws4_request", .{ date, self.region, self.service }); - defer allocator.free(credential_scope); + const signing_key = try self.deriveSigningKey(date); - // Build string to sign - var string_to_sign_list: std.ArrayList(u8) = .{}; - defer string_to_sign_list.deinit(allocator); + var h_state = std.crypto.auth.hmac.sha2.HmacSha256.init(&signing_key); + var h_writer = stdx.crypto.hmacWriter(&h_state); + const hw = &h_writer.interface; - try string_to_sign_list.appendSlice(allocator, ALGORITHM); - try string_to_sign_list.append(allocator, '\n'); - try string_to_sign_list.appendSlice(allocator, timestamp); - try string_to_sign_list.append(allocator, '\n'); - try string_to_sign_list.appendSlice(allocator, credential_scope); - try string_to_sign_list.append(allocator, '\n'); - try string_to_sign_list.appendSlice(allocator, &cr_hash_hex); + try hw.print("{s}\n{s}\n{s}\n{x}", .{ ALGORITHM, timestamp, scope, &canonical_request_hash }); - // Calculate signing key - const k_date = hmacSha256WithPrefix("AWS4", self.secret_key, date); - const k_region = hmacSha256(&k_date, self.region); - const k_service = hmacSha256(&k_region, self.service); - const k_signing = hmacSha256(&k_service, "aws4_request"); + var signature: [32]u8 = undefined; + h_state.final(&signature); - // Calculate signature - const signature = hmacSha256(&k_signing, string_to_sign_list.items); - var signature_hex: [64]u8 = undefined; - _ = std.fmt.bufPrint(&signature_hex, "{x}", .{&signature}) catch unreachable; - - // Build final Authorization header - return try std.fmt.allocPrint(allocator, "{s} Credential={s}/{s}, SignedHeaders={s}, Signature={s}", .{ + return try std.fmt.bufPrint(output_buffer, "{s} Credential={s}/{s}, SignedHeaders={s}, Signature={x}", .{ ALGORITHM, - self.access_key, - credential_scope, + access_key, + scope, signed_headers, - signature_hex[0..64], + &signature, }); } - fn getHostForEndpoint(endpoint: []const u8, buf: []u8) []const u8 { - const uri = std.Uri.parse(endpoint) catch return endpoint; - const host_raw = uri.host orelse return endpoint; - const host = switch (host_raw) { - .raw => |r| r, - .percent_encoded => |p| p, - }; - if (uri.port) |port| { - const len = std.fmt.bufPrint(buf, "{s}:{d}", .{ host, port }) catch return endpoint; - return buf[0..len.len]; - } - return host; - } - - fn hmacSha256WithPrefix(prefix: []const u8, secret: []const u8, data: []const u8) [32]u8 { + fn deriveSigningKey(self: AwsSigV4, date: []const u8) ![32]u8 { var key_buf: [128]u8 = undefined; - @memcpy(key_buf[0..prefix.len], prefix); - @memcpy(key_buf[prefix.len..][0..secret.len], secret); - const key = key_buf[0 .. prefix.len + secret.len]; + const k_init = try std.fmt.bufPrint(&key_buf, "AWS4{s}", .{self.secret_key.?}); - var out: [32]u8 = undefined; - std.crypto.auth.hmac.sha2.HmacSha256.create(&out, data, key); - return out; + const k_date = hmacStatic(k_init, date); + const k_region = hmacStatic(&k_date, self.region); + const k_service = hmacStatic(&k_region, self.service); + return hmacStatic(&k_service, "aws4_request"); } - fn sha256(data: []const u8) [32]u8 { - var hash: [32]u8 = undefined; - std.crypto.hash.sha2.Sha256.hash(data, &hash, .{}); - return hash; - } - - fn hmacSha256(key: []const u8, data: []const u8) [32]u8 { + fn hmacStatic(key: []const u8, data: []const u8) [32]u8 { var out: [32]u8 = undefined; std.crypto.auth.hmac.sha2.HmacSha256.create(&out, data, key); return out; } }; +const ReadState = struct { index: usize, objects: [][]const u8 }; + pub const S3 = struct { + pub const Config = struct { + access_key: ?[]const u8 = null, + secret_key: ?[]const u8 = null, + endpoint_url: []const u8, + region: []const u8, + auth_service: []const u8 = "s3", + }; + const Handle = struct { pub const Type = enum { file, directory }; @@ -184,8 +156,13 @@ pub const S3 = struct { pos: u64, size: u64, - pub fn init(allocator: std.mem.Allocator, handle_type: Type, path: []const u8, size: u64) !Handle { - return .{ .type = handle_type, .uri = try allocator.dupe(u8, path), .pos = 0, .size = size }; + pub fn init(allocator: std.mem.Allocator, type_: Type, path: []const u8, size: u64) !Handle { + return .{ + .type = type_, + .uri = try allocator.dupe(u8, path), + .pos = 0, + .size = size, + }; } pub fn deinit(self: *Handle, allocator: std.mem.Allocator) void { @@ -193,80 +170,78 @@ pub const S3 = struct { } }; - pub const Credentials = struct { - access_key: []const u8, - secret_key: []const u8, - endpoint_url: []const u8, - region: []const u8 = "us-east-1", // todo: no default? - - pub fn fromEnv(allocator: std.mem.Allocator) !Credentials { - const access_key = std.process.getEnvVarOwned(allocator, "AWS_ACCESS_KEY_ID") catch |err| { - log.err("AWS_ACCESS_KEY_ID environment variable not set", .{}); - return err; - }; - errdefer allocator.free(access_key); - - const secret_key = std.process.getEnvVarOwned(allocator, "AWS_SECRET_ACCESS_KEY") catch |err| { - log.err("AWS_SECRET_ACCESS_KEY environment variable not set", .{}); - return err; - }; - errdefer allocator.free(secret_key); - - const endpoint_url = std.process.getEnvVarOwned(allocator, "AWS_ENDPOINT_URL") catch - std.process.getEnvVarOwned(allocator, "AWS_ENDPOINT_URL_S3") catch |err| { - log.err("Neither AWS_ENDPOINT_URL nor AWS_ENDPOINT_URL_S3 environment variable is set", .{}); - return err; - }; - errdefer allocator.free(endpoint_url); - - const region = std.process.getEnvVarOwned(allocator, "AWS_REGION") catch - std.process.getEnvVarOwned(allocator, "AWS_DEFAULT_REGION") catch - try allocator.dupe(u8, "us-east-1"); - - return .{ .access_key = access_key, .secret_key = secret_key, .endpoint_url = endpoint_url, .region = region }; - } - - pub fn deinit(self: *Credentials, allocator: std.mem.Allocator) void { - allocator.free(self.access_key); - allocator.free(self.secret_key); - allocator.free(self.endpoint_url); - allocator.free(self.region); - } - }; - allocator: std.mem.Allocator, mutex: std.Io.Mutex = .init, client: *std.http.Client, - access_key: []const u8, - secret_key: []const u8, - endpoint_url: []const u8, - region: []const u8, + config: Config, handles: stdx.SegmentedList(Handle, 0) = .{}, closed_handles: std.ArrayList(u32) = .{}, + dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, base: VFSBase, pub fn init( allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, - credentials: Credentials, + config: Config, ) !S3 { return .{ .allocator = allocator, .base = .init(inner), .client = http_client, - .access_key = try allocator.dupe(u8, credentials.access_key), - .secret_key = try allocator.dupe(u8, credentials.secret_key), - .endpoint_url = try allocator.dupe(u8, credentials.endpoint_url), - .region = try allocator.dupe(u8, credentials.region), + .config = .{ + .access_key = if (config.access_key) |k| try allocator.dupe(u8, k) else null, + .secret_key = if (config.secret_key) |k| try allocator.dupe(u8, k) else null, + .endpoint_url = try allocator.dupe(u8, config.endpoint_url), + .region = try allocator.dupe(u8, config.region), + .auth_service = try allocator.dupe(u8, config.auth_service), + }, }; } - pub fn fromEnv(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client) !S3 { - var credentials = try Credentials.fromEnv(allocator); - defer credentials.deinit(allocator); + pub fn auto(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, environ_map: *std.process.Environ.Map) !S3 { + const access_key = environ_map.get("AWS_ACCESS_KEY_ID"); + const secret_key = environ_map.get("AWS_SECRET_ACCESS_KEY"); - return init(allocator, inner, http_client, credentials); + var endpoint_is_default = false; + const endpoint = environ_map.get("AWS_ENDPOINT_URL") orelse environ_map.get("AWS_ENDPOINT_URL_S3") orelse blk: { + endpoint_is_default = true; + break :blk "https://s3.amazonaws.com"; + }; + + var region_is_default = false; + const region = environ_map.get("AWS_REGION") orelse environ_map.get("AWS_DEFAULT_REGION") orelse blk: { + region_is_default = true; + break :blk "us-east-1"; + }; + + if (access_key == null or secret_key == null or endpoint_is_default or region_is_default) { + var buf: [512]u8 = undefined; + var writer = std.Io.Writer.fixed(&buf); + + if (access_key == null or secret_key == null) { + try writer.writeAll("S3 initialized without full credentials (AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY). Access will be unauthenticated."); + } else { + try writer.writeAll("S3 initialized with credentials."); + } + + if (endpoint_is_default) { + try writer.print(" Using default endpoint: {s} (set AWS_ENDPOINT_URL or AWS_ENDPOINT_URL_S3).", .{endpoint}); + } + + if (region_is_default) { + try writer.print(" Using default region: {s} (set AWS_REGION or AWS_DEFAULT_REGION).", .{region}); + } + + log.warn("{s}", .{writer.buffered()}); + } + + return init(allocator, inner, http_client, .{ + .access_key = access_key, + .secret_key = secret_key, + .endpoint_url = endpoint, + .region = region, + }); } pub fn deinit(self: *S3) void { @@ -275,20 +250,35 @@ pub const S3 = struct { const is_closed = for (self.closed_handles.items) |closed_idx| { if (closed_idx == idx) break true; } else false; - if (!is_closed) self.handles.at(idx).deinit(self.allocator); + + if (!is_closed) { + self.handles.at(idx).deinit(self.allocator); + } } self.handles.deinit(self.allocator); self.closed_handles.deinit(self.allocator); - self.allocator.free(self.access_key); - self.allocator.free(self.secret_key); - self.allocator.free(self.endpoint_url); - self.allocator.free(self.region); + + var it = self.dir_read_states.iterator(); + while (it.next()) |entry| { + for (entry.value_ptr.objects) |obj| { + self.allocator.free(obj); + } + self.allocator.free(entry.value_ptr.objects); + } + self.dir_read_states.deinit(self.allocator); + + if (self.config.access_key) |access_key| self.allocator.free(access_key); + if (self.config.secret_key) |secret_key| self.allocator.free(secret_key); + self.allocator.free(self.config.endpoint_url); + self.allocator.free(self.config.region); + self.allocator.free(self.config.auth_service); } pub fn io(self: *S3) std.Io { return .{ .userdata = &self.base, .vtable = &comptime VFSBase.vtable(.{ + .operate = operate, .dirOpenDir = dirOpenDir, .dirStat = dirStat, .dirStatFile = dirStatFile, @@ -301,7 +291,6 @@ pub const S3 = struct { .fileStat = fileStat, .fileLength = fileLength, .fileClose = fileClose, - .fileReadStreaming = fileReadStreaming, .fileReadPositional = fileReadPositional, .fileSeekBy = fileSeekBy, .fileSeekTo = fileSeekTo, @@ -313,7 +302,9 @@ pub const S3 = struct { fn openHandle(self: *S3) !struct { u32, *Handle } { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); + if (self.closed_handles.pop()) |idx| return .{ idx, self.handles.at(idx) }; + return .{ @intCast(self.handles.len), try self.handles.addOne(self.allocator) }; } @@ -340,43 +331,241 @@ pub const S3 = struct { if (std.meta.eql(dir, std.Io.Dir.cwd())) { return try std.fmt.bufPrint(out_buffer, "{s}", .{sub_path}); } + const handle = self.getDirHandle(dir); + const trimmed_uri = std.mem.trimEnd(u8, handle.uri, "/"); const trimmed_sub_path = std.mem.trimStart(u8, sub_path, "/"); if (trimmed_uri.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_sub_path}); if (trimmed_sub_path.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_uri}); + return try std.fmt.bufPrint(out_buffer, "{s}/{s}", .{ trimmed_uri, trimmed_sub_path }); } - // todo - fn parseBucketKey(path: []const u8) struct { bucket: []const u8, key: []const u8 } { - const trimmed = std.mem.trim(u8, path, "/"); - if (std.mem.indexOfScalar(u8, trimmed, '/')) |idx| { - return .{ .bucket = trimmed[0..idx], .key = if (idx + 1 < trimmed.len) trimmed[idx + 1 ..] else "" }; + fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + switch (operation) { + .file_read_streaming => |o| { + const handle = self.getFileHandle(o.file); + const total = self.performRead(handle, o.data, handle.pos) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); + return .{ .file_read_streaming = std.Io.File.ReadStreamingError.EndOfStream }; + }; + + if (total == 0) { + return .{ .file_read_streaming = std.Io.File.ReadStreamingError.EndOfStream }; + } + + handle.pos += @intCast(total); + return .{ .file_read_streaming = total }; + }, + .file_write_streaming, .device_io_control => { + return self.base.inner.vtable.operate(self.base.inner.userdata, operation); + }, } - return .{ .bucket = trimmed, .key = "" }; } - // todo - const UrlStyle = enum { - virtual_hosted, // AWS - path_style, // MinIO - }; + fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.OpenError.SystemResources; - // todo - fn buildS3Url(self: *S3, allocator: std.mem.Allocator, bucket: []const u8, key: []const u8) ![]const u8 { - const trimmed_endpoint = std.mem.trimEnd(u8, self.endpoint_url, "/"); + const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .directory, path, 0) catch return std.Io.Dir.OpenError.Unexpected; - if (key.len > 0) { - return try std.fmt.allocPrint(allocator, "{s}/{s}/{s}", .{ trimmed_endpoint, bucket, key }); - } - return try std.fmt.allocPrint(allocator, "{s}/{s}/", .{ trimmed_endpoint, bucket }); + return .{ .handle = @intCast(idx) }; } - // todo - fn getTimestamp(allocator: std.mem.Allocator, io_: std.Io) ![]const u8 { - const now_ts = try std.Io.Clock.now(.real, io_); + fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getDirHandle(dir); + + return .{ + .inode = @intCast(@intFromPtr(handle)), + .nlink = 0, + .size = handle.size, + .permissions = .fromMode(0o444), + .kind = .directory, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + .block_size = 1, + }; + } + + fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const size = self.fetchSize(dir, sub_path) catch return std.Io.Dir.StatFileError.Unexpected; + + return .{ + .inode = 0, + .nlink = 0, + .size = size, + .permissions = .fromMode(0o444), + .kind = .file, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + .block_size = 1, + }; + } + + fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} + + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const size = self.fetchSize(dir, sub_path) catch return std.Io.File.OpenError.Unexpected; + + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; + + return .{ .handle = @intCast(idx), .flags = .{ .nonblocking = false } }; + } + + fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + for (dirs) |dir| { + self.closeHandle(@intCast(dir.handle)) catch unreachable; + } + } + + fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + + if (reader.state == .finished) return 0; + + if (reader.state == .reset) { + if (self.dir_read_states.fetchRemove(reader)) |kv| { + for (kv.value.objects) |obj| self.allocator.free(obj); + self.allocator.free(kv.value.objects); + } + + const handle = self.getDirHandle(reader.dir); + const objects = self.listObjects(handle.uri) catch return std.Io.Dir.Reader.Error.Unexpected; + + self.dir_read_states.put(self.allocator, reader, .{ + .index = 0, + .objects = objects, + }) catch return std.Io.Dir.Reader.Error.Unexpected; + + reader.state = if (objects.len > 0) .reading else .finished; + + if (objects.len == 0) return 0; + } + + const state = self.dir_read_states.getPtr(reader) orelse return std.Io.Dir.Reader.Error.Unexpected; + + var count: usize = 0; + while (count < entries.len and state.index < state.objects.len) { + const obj_key = state.objects[state.index]; + const kind: std.Io.File.Kind = if (std.mem.endsWith(u8, obj_key, "/")) .directory else .file; + + const name = if (std.mem.lastIndexOfScalar(u8, std.mem.trimEnd(u8, obj_key, "/"), '/')) |idx| + obj_key[idx + 1 ..] + else + obj_key; + + entries[count] = .{ + .name = std.mem.trimEnd(u8, name, "/"), + .kind = kind, + .inode = state.index, + }; + count += 1; + state.index += 1; + } + + if (state.index >= state.objects.len) { + reader.state = .finished; + } + + return count; + } + + fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getDirHandle(dir); + const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.Dir.RealPathError.SystemResources; + return path.len; + } + + fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const real_path = self.resolvePath(dir, path_name, out_buffer) catch return std.Io.Dir.RealPathFileError.NameTooLong; + return real_path.len; + } + + fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + + return .{ + .inode = @intCast(file.handle), + .nlink = 0, + .size = handle.size, + .permissions = .fromMode(0o444), + .kind = .file, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + .block_size = 1, + }; + } + + fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.getFileHandle(file).size; + } + + fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + for (files) |file| { + self.closeHandle(@intCast(file.handle)) catch unreachable; + } + } + + fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return self.performRead(handle, data, offset) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); + return std.Io.File.Reader.Error.Unexpected; + }; + } + + fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + + if (relative_offset >= 0) { + handle.pos = handle.pos +| @as(u64, @intCast(relative_offset)); + } else { + const abs_offset: u64 = @intCast(-relative_offset); + if (abs_offset > handle.pos) { + handle.pos = 0; + } else { + handle.pos -= abs_offset; + } + } + } + + fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + handle.pos = absolute_offset; + } + + fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.File.RealPathError.SystemResources; + return path.len; + } + + fn getTimestamp(self: *S3, buf: []u8) ![]const u8 { + const now_ts = std.Io.Clock.now(.real, self.base.inner); const now: u64 = @intCast(@divFloor(now_ts.nanoseconds, std.time.ns_per_s)); const epoch_secs: std.time.epoch.EpochSeconds = .{ .secs = now }; @@ -384,7 +573,7 @@ pub const S3 = struct { const month_day = year_day.calculateMonthDay(); const day_seconds = epoch_secs.getDaySeconds(); - return try std.fmt.allocPrint(allocator, "{d:0>4}{d:0>2}{d:0>2}T{d:0>2}{d:0>2}{d:0>2}Z", .{ + return try std.fmt.bufPrint(buf, "{d:0>4}{d:0>2}{d:0>2}T{d:0>2}{d:0>2}{d:0>2}Z", .{ year_day.year, @intFromEnum(month_day.month), month_day.day_index + 1, @@ -394,33 +583,156 @@ pub const S3 = struct { }); } + fn pathComponents(self: *S3, path_: []const u8) struct { []const u8, []const u8, []const u8 } { + const endpoint = std.mem.trimEnd(u8, self.config.endpoint_url, "/"); + const path = std.mem.trim(u8, path_, "/"); + + if (std.mem.findScalar(u8, path, '/')) |idx| { + return .{ endpoint, path[0..idx], if (idx + 1 < path.len) path[idx + 1 ..] else "" }; + } else { + return .{ endpoint, path, "" }; + } + } + + fn s3Url(self: *S3, path: []const u8, buf: []u8) ![]const u8 { + const endpoint, const bucket, const key = self.pathComponents(path); + return try std.fmt.bufPrint(buf, "{s}/{s}/{s}", .{ endpoint, bucket, key }); + } + + fn listObjects(self: *S3, prefix: []const u8) ![][]const u8 { + const endpoint, const bucket, const key_prefix = self.pathComponents(prefix); + + var query_buf: [4096]u8 = undefined; + var query_writer = std.Io.Writer.fixed(&query_buf); + + try query_writer.writeAll("delimiter="); + try std.Uri.Component.percentEncode(&query_writer, "/", s3EncodeIsValid); + try query_writer.writeAll("&list-type=2&max-keys=1000"); + + if (key_prefix.len > 0) { + try query_writer.writeAll("&prefix="); + try std.Uri.Component.percentEncode(&query_writer, key_prefix, s3EncodeIsValid); + try std.Uri.Component.percentEncode(&query_writer, "/", s3EncodeIsValid); + } + + const endpoint_uri = try std.Uri.parse(endpoint); + + var path_buf: [256]u8 = undefined; + + const uri: std.Uri = .{ + .scheme = endpoint_uri.scheme, + .host = endpoint_uri.host, + .port = endpoint_uri.port, + .path = .{ .percent_encoded = try std.fmt.bufPrint(&path_buf, "/{s}", .{bucket}) }, + .query = .{ .percent_encoded = query_writer.buffered() }, + .fragment = null, + }; + + var timestamp_buf: [16]u8 = undefined; + const timestamp = try self.getTimestamp(×tamp_buf); + + const signer: AwsSigV4 = .{ + .access_key = self.config.access_key, + .secret_key = self.config.secret_key, + .region = self.config.region, + .service = self.config.auth_service, + }; + + var authorization_buffer: [512]u8 = undefined; + const authorization = try signer.generateAuthHeader(&authorization_buffer, .GET, uri, timestamp, &.{}); + + var req = try self.client.request(.GET, uri, .{ + .redirect_behavior = .not_allowed, + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = if (authorization) |auth| .{ .override = auth } else .omit, + }, + .extra_headers = &.{ + .{ .name = "x-amz-date", .value = timestamp }, + .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, + }, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var redirect_buffer: [2 * 1024]u8 = undefined; + var res = try req.receiveHead(&redirect_buffer); + + if (res.head.status != .ok) { + log.err("Failed to list object {f}", .{uri}); + log.err("{s}", .{res.head.bytes}); + return error.RequestFailed; + } + + const body = try res.reader(&.{}).readAlloc(self.allocator, res.head.content_length orelse 1024 * 1024); + defer self.allocator.free(body); + + return try self.parseListObjectsResponse(body, key_prefix); + } + + fn s3EncodeIsValid(c: u8) bool { + return switch (c) { + 'A'...'Z', 'a'...'z', '0'...'9', '-', '.', '_', '~' => true, + else => false, + }; + } + + fn parseListObjectsResponse(self: *S3, xml: []const u8, prefix: []const u8) ![][]const u8 { + var results = std.ArrayListUnmanaged([]const u8){}; + errdefer { + for (results.items) |item| self.allocator.free(item); + results.deinit(self.allocator); + } + + inline for (.{ "Key", "Prefix" }) |tag| { + var pos: usize = 0; + while (std.mem.indexOfPos(u8, xml, pos, "<" ++ tag ++ ">")) |start| { + const v_start = start + tag.len + 2; + const end = std.mem.indexOfPos(u8, xml, v_start, "") orelse break; + const val = xml[v_start..end]; + + // Filter out the directory itself (S3 often includes the prefix as a 0-byte key) + const is_self = std.mem.eql(u8, std.mem.trimEnd(u8, val, "/"), std.mem.trimEnd(u8, prefix, "/")); + if (val.len > 0 and !is_self) { + try results.append(self.allocator, try self.allocator.dupe(u8, val)); + } + pos = end + tag.len + 3; + } + } + + return try results.toOwnedSlice(self.allocator); + } + fn fetchSize(self: *S3, dir: std.Io.Dir, sub_path: []const u8) !u64 { var path_buffer: [8 * 1024]u8 = undefined; - const path = try self.resolvePath(dir, sub_path, &path_buffer); - const parsed = parseBucketKey(path); + var url_buf: [8 * 1024]u8 = undefined; - const url = try self.buildS3Url(self.allocator, parsed.bucket, parsed.key); - defer self.allocator.free(url); + const path = try self.resolvePath(dir, sub_path, &path_buffer); + const url = try self.s3Url(path, &url_buf); const uri = std.Uri.parse(url) catch return error.BadPathName; - const timestamp = try getTimestamp(self.allocator, self.base.inner); - defer self.allocator.free(timestamp); + var timestamp_buf: [16]u8 = undefined; + const timestamp = try self.getTimestamp(×tamp_buf); const signer: AwsSigV4 = .{ - .access_key = self.access_key, - .secret_key = self.secret_key, - .region = self.region, + .access_key = self.config.access_key, + .secret_key = self.config.secret_key, + .region = self.config.region, + .service = self.config.auth_service, }; - const auth_header = try signer.generateAuthHeader(self.allocator, .HEAD, self.endpoint_url, uri, timestamp, &.{}); - defer self.allocator.free(auth_header); + var authorization_buffer: [512]u8 = undefined; + const authorization = try signer.generateAuthHeader(&authorization_buffer, .HEAD, uri, timestamp, &.{}); var req = try self.client.request(.HEAD, uri, .{ .redirect_behavior = .not_allowed, - .headers = .{ .accept_encoding = .{ .override = "identity" } }, + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = if (authorization) |auth| .{ .override = auth } else .omit, + }, .extra_headers = &.{ - .{ .name = "Authorization", .value = auth_header }, .{ .name = "x-amz-date", .value = timestamp }, .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, }, @@ -444,44 +756,44 @@ pub const S3 = struct { fn performRead(self: *S3, handle: *Handle, data: []const []u8, offset: u64) !usize { if (offset >= handle.size) return 0; - const parsed = parseBucketKey(handle.uri); - - const url = try self.buildS3Url(self.allocator, parsed.bucket, parsed.key); - defer self.allocator.free(url); + var url_buf: [8 * 1024]u8 = undefined; + const url = try self.s3Url(handle.uri, &url_buf); const uri = std.Uri.parse(url) catch return error.BadPathName; - // Calculate range - var total_bytes: u64 = 0; - for (data) |buf| total_bytes += buf.len; - const remaining = handle.size - offset; - const take = @min(remaining, total_bytes); - const end = offset + take - 1; - - const range_header = try std.fmt.allocPrint(self.allocator, "bytes={d}-{d}", .{ offset, end }); - defer self.allocator.free(range_header); - - const timestamp = try getTimestamp(self.allocator, self.base.inner); - defer self.allocator.free(timestamp); - - const signer: AwsSigV4 = .{ - .access_key = self.access_key, - .secret_key = self.secret_key, - .region = self.region, + var range_buf: [64]u8 = undefined; + const range_header = blk: { + var total_bytes: u64 = 0; + for (data) |buf| { + total_bytes += @as(u64, buf.len); + } + const remaining = handle.size - offset; + const take = @min(remaining, total_bytes); + const end = offset + take - 1; + break :blk std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ offset, end }) catch unreachable; }; - const auth_header = try signer.generateAuthHeader(self.allocator, .GET, self.endpoint_url, uri, timestamp, &.{}); - defer self.allocator.free(auth_header); + var timestamp_buf: [16]u8 = undefined; + const timestamp = try self.getTimestamp(×tamp_buf); - var req = try self.client.request(.GET, uri, .{ - .headers = .{ .accept_encoding = .{ .override = "identity" } }, - .extra_headers = &.{ - .{ .name = "Authorization", .value = auth_header }, - .{ .name = "x-amz-date", .value = timestamp }, - .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, - .{ .name = "Range", .value = range_header }, - }, - }); + const signer: AwsSigV4 = .{ + .access_key = self.config.access_key, + .secret_key = self.config.secret_key, + .region = self.config.region, + .service = self.config.auth_service, + }; + + var authorization_buffer: [512]u8 = undefined; + const authorization = try signer.generateAuthHeader(&authorization_buffer, .GET, uri, timestamp, &.{}); + + var req = try self.client.request(.GET, uri, .{ .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = if (authorization) |auth| .{ .override = auth } else .omit, + }, .extra_headers = &.{ + .{ .name = "x-amz-date", .value = timestamp }, + .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, + .{ .name = "Range", .value = range_header }, + } }); defer req.deinit(); try req.sendBodiless(); @@ -533,152 +845,4 @@ pub const S3 = struct { .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, }; } - - fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - var path_buffer: [8 * 1024]u8 = undefined; - const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.OpenError.SystemResources; - const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; - handle.* = Handle.init(self.allocator, .directory, path, 0) catch return std.Io.Dir.OpenError.Unexpected; - return .{ .handle = @intCast(idx) }; - } - - fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getDirHandle(dir); - return .{ - .inode = 0, - .nlink = 0, - .size = handle.size, - .permissions = .fromMode(0o444), - .kind = .directory, - .atime = null, - .mtime = std.Io.Timestamp.zero, - .ctime = std.Io.Timestamp.zero, - }; - } - - fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const size = self.fetchSize(dir, sub_path) catch return std.Io.Dir.StatFileError.Unexpected; - return .{ - .inode = 0, - .nlink = 0, - .size = size, - .permissions = .fromMode(0o444), - .kind = .file, - .atime = null, - .mtime = std.Io.Timestamp.zero, - .ctime = std.Io.Timestamp.zero, - }; - } - - fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} - - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const size = self.fetchSize(dir, sub_path) catch return std.Io.File.OpenError.Unexpected; - var path_buffer: [8 * 1024]u8 = undefined; - const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; - const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; - handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; - return .{ .handle = @intCast(idx) }; - } - - fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - for (dirs) |dir| self.closeHandle(@intCast(dir.handle)) catch unreachable; - } - - fn dirRead(_: ?*anyopaque, _: *std.Io.Dir.Reader, _: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { - log.err("dirRead unsupported for S3 VFS", .{}); - return std.Io.Dir.Reader.Error.Unexpected; - } - - fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getDirHandle(dir); - const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.Dir.RealPathError.SystemResources; - return path.len; - } - - fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const real_path = self.resolvePath(dir, path_name, out_buffer) catch return std.Io.Dir.RealPathFileError.NameTooLong; - return real_path.len; - } - - fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - return .{ - .inode = @intCast(file.handle), - .nlink = 0, - .size = handle.size, - .permissions = .fromMode(0o444), - .kind = .file, - .atime = null, - .mtime = std.Io.Timestamp.zero, - .ctime = std.Io.Timestamp.zero, - }; - } - - fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.getFileHandle(file).size; - } - - fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - for (files) |file| self.closeHandle(@intCast(file.handle)) catch unreachable; - } - - fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - const total = self.performRead(handle, data, handle.pos) catch |err| { - log.err("Failed to read {s} at {d}: {any}", .{ handle.uri, handle.pos, err }); - return std.Io.File.Reader.Error.Unexpected; - }; - handle.pos += @intCast(total); - return total; - } - - fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - return self.performRead(handle, data, offset) catch |err| { - log.err("Failed to read {s} at {d}: {any}", .{ handle.uri, offset, err }); - return std.Io.File.ReadPositionalError.Unexpected; - }; - } - - fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - - if (relative_offset >= 0) { - handle.pos = handle.pos +| @as(u64, @intCast(relative_offset)); - } else { - const abs_offset: u64 = @intCast(-relative_offset); - if (abs_offset > handle.pos) { - handle.pos = 0; - } else { - handle.pos -= abs_offset; - } - } - } - - fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - handle.pos = absolute_offset; - } - - fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.File.RealPathError.SystemResources; - return path.len; - } }; diff --git a/zml/safetensors.zig b/zml/safetensors.zig index 5e4ff6ea..cd7677c7 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -64,36 +64,56 @@ pub fn fetchRegistry( } pub const TensorReader = struct { + tensor: Tensor, file: std.Io.File, file_reader: std.Io.File.Reader, + alignment: ?std.mem.Alignment, remaining: u64, + padding_remaining: u64, + io: std.Io, interface: std.Io.Reader, - tensor: Tensor, - - pub const Error = error{TensorNotFound} || std.Io.File.OpenError || std.Io.File.Reader.SeekError || std.mem.Allocator.Error; pub fn init( io: std.Io, tensor: Tensor, buffer: []u8, - ) Error!TensorReader { + opts: struct { + alignment: ?std.mem.Alignment = null, + }, + ) !TensorReader { const file = try std.Io.Dir.openFile(.cwd(), io, tensor.file_uri, .{ .mode = .read_only }); errdefer file.close(io); + const offset, const padding_remaining = if (opts.alignment) |alignment| blk: { + const aligned_offset = std.mem.alignBackward(u64, tensor.offset, alignment.toByteUnits()); + break :blk .{ aligned_offset, tensor.offset - aligned_offset }; + } else blk: { + break :blk .{ tensor.offset, 0 }; + }; + var file_reader = file.reader(io, buffer); - try file_reader.seekTo(tensor.offset); + try file_reader.seekTo(offset); return .{ + .tensor = tensor, .file = file, .file_reader = file_reader, + .alignment = opts.alignment, .remaining = tensor.byteSize(), - .tensor = tensor, + .padding_remaining = padding_remaining, .io = io, .interface = .{ - .vtable = &.{ - .stream = stream, - .discard = discard, + .vtable = if (opts.alignment) |_| blk: { + break :blk &.{ + .stream = streamAligned, + .discard = discardAligned, + }; + } else blk: { + break :blk &.{ + .stream = stream, + .discard = discard, + }; }, .buffer = &.{}, .seek = 0, @@ -121,7 +141,6 @@ pub const TensorReader = struct { fn discard(r: *std.Io.Reader, limit: std.Io.Limit) std.Io.Reader.Error!usize { const self: *TensorReader = @fieldParentPtr("interface", r); if (self.remaining == 0) { - std.log.info("EOS2 !", .{}); return error.EndOfStream; } @@ -130,6 +149,72 @@ pub const TensorReader = struct { self.remaining -= n; return n; } + + fn streamAligned(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { + const self: *TensorReader = @fieldParentPtr("interface", r); + if (self.remaining == 0) { + return error.EndOfStream; + } + + const requested_bytes = limit.minInt64(self.remaining); + + if (self.padding_remaining > 0) { + _ = try self.file_reader.interface.discard(.limited64(self.padding_remaining)); + self.padding_remaining = 0; + } + + const align_bytes = self.alignment.?.toByteUnits(); + const file_offset = self.file_reader.pos - std.mem.alignBackward(u64, self.file_reader.pos, align_bytes); + const writer_addr = @intFromPtr(w.buffer.ptr) + w.end; + const writer_offset = writer_addr - std.mem.alignBackward(usize, writer_addr, align_bytes); + + if (file_offset != writer_offset) { + const skip = if (file_offset >= writer_offset) blk: { + break :blk file_offset - writer_offset; + } else blk: { + break :blk align_bytes - writer_offset + file_offset; + }; + + w.advance(skip); + } + + if (w.writableSliceGreedy(align_bytes)) |dest| { + const phys_buf = dest.ptr - file_offset; + const phys_pos = self.file_reader.pos - file_offset; + const phys_size = std.mem.alignBackward(usize, @min(dest.len, requested_bytes) + file_offset, align_bytes); + + if (phys_size > file_offset) { + const phys_read = self.file.readPositional(self.io, &.{phys_buf[0..phys_size]}, phys_pos) catch unreachable; + if (phys_read > file_offset) { + const bytes_read = phys_read - file_offset; + w.advance(bytes_read); + self.file_reader.pos += bytes_read; + self.remaining -= bytes_read; + return bytes_read; + } + } + } else |_| {} + + const bytes_read = try self.file_reader.interface.stream(w, .limited64(requested_bytes)); + self.remaining -= bytes_read; + + return bytes_read; + } + + fn discardAligned(r: *std.Io.Reader, limit: std.Io.Limit) std.Io.Reader.Error!usize { + const self: *TensorReader = @fieldParentPtr("interface", r); + if (self.remaining == 0) return 0; + + if (self.padding_remaining > 0) { + _ = try self.file_reader.interface.discard(.limited64(self.padding_remaining)); + self.padding_remaining = 0; + } + + const combined_limit = limit.min(.limited64(self.remaining)); + const n = try self.file_reader.interface.discard(combined_limit); + self.remaining -= n; + return n; + } }; pub const Tensor = struct { From 32a8d69b065aebeb879568e6d6078a3a2fe5a5ab Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 19 Feb 2026 10:36:49 +0000 Subject: [PATCH 321/881] zml/io/vfs/hf: check performRead http status correctly (aligned with s3 and http) --- zml/io/vfs/hf.zig | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/zml/io/vfs/hf.zig b/zml/io/vfs/hf.zig index 651f9d5f..bc2a2bd1 100644 --- a/zml/io/vfs/hf.zig +++ b/zml/io/vfs/hf.zig @@ -711,7 +711,10 @@ pub const HF = struct { const uri: std.Uri = try .parse(url); var req = try self.client.request(.GET, uri, .{ - .headers = .{ .authorization = self.authorization }, + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = self.authorization, + }, .extra_headers = &.{.{ .name = "Range", .value = range_header }}, }); defer req.deinit(); @@ -721,7 +724,7 @@ pub const HF = struct { var redirect_buffer: [8 * 1024]u8 = undefined; var res = try req.receiveHead(&redirect_buffer); - if (res.head.status != .partial_content) { + if (res.head.status != .partial_content and res.head.status != .ok) { log.err("Failed to perform read for {s}", .{url}); log.err("{s}", .{res.head.bytes}); return error.RequestFailed; From a1506d3ed4a449bda5a86879a17364773fcbd859 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 19 Feb 2026 15:39:39 +0000 Subject: [PATCH 322/881] platforms/cuda: add cuda_compat functionality --- platforms/cuda/cuda.bzl | 17 +++++++++++++++++ platforms/cuda/libpjrt_cuda.BUILD.bazel | 2 ++ platforms/cuda/zmlxcuda.zig | 1 + 3 files changed, 20 insertions(+) diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 14e4867e..d2e2599f 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -14,6 +14,7 @@ ARCH = "linux-x86_64" CUDA_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cuda/redist/" CUDA_VERSION = "13.1.1" +CUDA_VARIANT = "cuda13.1" CUDA_REDIST_JSON_SHA256 = "97cf605ccc4751825b1865f4af571c9b50dd29ffd13e9a38b296a9ecb1f0d422" CUDNN_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cudnn/redist/" @@ -52,6 +53,21 @@ CUDA_PACKAGES = { srcs = ["lib/libnvtx3interop.so"], ), ]), + "cuda_compat": "\n".join([ + packages.filegroup( + name = "cuda_compat", + srcs = [ + "compat/libcuda.so.1", + "compat/libcudadebugger.so.1", + "compat/libnvidia-gpucomp.so.590.48.01", + "compat/libnvidia-nvvm.so.4", + "compat/libnvidia-nvvm70.so.4", + "compat/libnvidia-pkcs11-openssl3.so.590.48.01", + "compat/libnvidia-ptxjitcompiler.so.1", + "compat/libnvidia-tileiras.so.590.48.01", + ], + ), + ]), "libcufft": packages.filegroup( name = "libcufft", srcs = ["lib/libcufft.so.12"], @@ -188,6 +204,7 @@ def _cuda_impl(mctx): arch_data = pkg_data.get(ARCH) if not arch_data: continue + arch_data = arch_data.get(CUDA_VARIANT, None) or arch_data http_archive( name = pkg, build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, diff --git a/platforms/cuda/libpjrt_cuda.BUILD.bazel b/platforms/cuda/libpjrt_cuda.BUILD.bazel index 372874a9..9ae32efb 100644 --- a/platforms/cuda/libpjrt_cuda.BUILD.bazel +++ b/platforms/cuda/libpjrt_cuda.BUILD.bazel @@ -24,6 +24,7 @@ copy_to_directory( ":libpjrt_cuda_so", "@cuda_cudart", "@cuda_cupti", + "@cuda_compat", "@cuda_nvcc", "@libnvvm", "@cuda_nvrtc", @@ -40,6 +41,7 @@ copy_to_directory( "@zml//platforms/cuda:zmlxcuda", ], replace_prefixes = { + "compat": "lib", "nvidia/nccl/lib": "lib", "nvvm/lib64": "lib", "libpjrt_cuda_so": "lib", diff --git a/platforms/cuda/zmlxcuda.zig b/platforms/cuda/zmlxcuda.zig index 5834cb6b..2c2c52bb 100644 --- a/platforms/cuda/zmlxcuda.zig +++ b/platforms/cuda/zmlxcuda.zig @@ -4,6 +4,7 @@ const stdx = @import("stdx"); pub export fn zmlxcuda_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque { const replacements: std.StaticStringMap([:0]const u8) = .initComptime(.{ + .{ "libcuda.so.1", "libcuda.so.1" }, .{ "libcublas.so", "libcublas.so.13" }, .{ "libcublasLt.so", "libcublasLt.so.13" }, .{ "libcudart.so", "libcudart.so.13" }, From 6636093cb474d861601f572ce76b07f72001a3be Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 20 Feb 2026 10:17:56 +0000 Subject: [PATCH 323/881] platforms/cuda: prevent cuda-compat leaking to host --- platforms/cuda/cuda.bzl | 25 ++++++++++++++++++++----- platforms/cuda/libpjrt_cuda.BUILD.bazel | 5 +++++ platforms/cuda/zmlxcuda.zig | 4 +++- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index d2e2599f..9e78edaa 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -8,6 +8,7 @@ package(default_visibility = ["//visibility:public"]) load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_cc//cc:cc_import.bzl", "cc_import") +load("@zml//bazel:patchelf.bzl", "patchelf") """ ARCH = "linux-x86_64" @@ -25,6 +26,14 @@ NVSHMEM_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/nvshmem/r NVSHMEM_VERSION = "3.5.19" NVSHMEM_REDIST_JSON_SHA256 = "6dced4193eb728542504b346cfb768da6e3de2abca0cded95fda3a69729994d2" +CUDA_COMPAT_FILES = [ + "libcuda.so.1", + "libcudadebugger.so.1", + "libnvidia-nvvm.so.4", + "libnvidia-nvvm70.so.4", + "libnvidia-ptxjitcompiler.so.1", +] + _UBUNTU_PACKAGES = { "zlib1g": packages.filegroup(name = "zlib1g", srcs = ["lib/x86_64-linux-gnu/libz.so.1"]), } @@ -54,17 +63,23 @@ CUDA_PACKAGES = { ), ]), "cuda_compat": "\n".join([ + packages.patchelf( + name = "{}.patchelf".format(file), + set_rpath = "$ORIGIN", + src = "compat/{}".format(file), + soname = file, + ) + for file in CUDA_COMPAT_FILES + ] + [ packages.filegroup( name = "cuda_compat", srcs = [ - "compat/libcuda.so.1", - "compat/libcudadebugger.so.1", "compat/libnvidia-gpucomp.so.590.48.01", - "compat/libnvidia-nvvm.so.4", - "compat/libnvidia-nvvm70.so.4", "compat/libnvidia-pkcs11-openssl3.so.590.48.01", - "compat/libnvidia-ptxjitcompiler.so.1", "compat/libnvidia-tileiras.so.590.48.01", + ] + [ + ":{}.patchelf".format(file) + for file in CUDA_COMPAT_FILES ], ), ]), diff --git a/platforms/cuda/libpjrt_cuda.BUILD.bazel b/platforms/cuda/libpjrt_cuda.BUILD.bazel index 9ae32efb..a4a70699 100644 --- a/platforms/cuda/libpjrt_cuda.BUILD.bazel +++ b/platforms/cuda/libpjrt_cuda.BUILD.bazel @@ -1,6 +1,7 @@ load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@zml//bazel:patchelf.bzl", "patchelf") load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@zml//platforms/cuda:cuda.bzl", "CUDA_COMPAT_FILES") patchelf( name = "libpjrt_cuda_so", @@ -42,11 +43,15 @@ copy_to_directory( ], replace_prefixes = { "compat": "lib", + "libcuda": "lib", "nvidia/nccl/lib": "lib", "nvvm/lib64": "lib", "libpjrt_cuda_so": "lib", "lib/x86_64-linux-gnu": "lib", "platforms/cuda": "lib", + } | { + "{}.patchelf".format(file): "lib" + for file in CUDA_COMPAT_FILES }, add_directory_to_runfiles = False, include_external_repositories = ["**"], diff --git a/platforms/cuda/zmlxcuda.zig b/platforms/cuda/zmlxcuda.zig index 2c2c52bb..922fffaa 100644 --- a/platforms/cuda/zmlxcuda.zig +++ b/platforms/cuda/zmlxcuda.zig @@ -4,7 +4,6 @@ const stdx = @import("stdx"); pub export fn zmlxcuda_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque { const replacements: std.StaticStringMap([:0]const u8) = .initComptime(.{ - .{ "libcuda.so.1", "libcuda.so.1" }, .{ "libcublas.so", "libcublas.so.13" }, .{ "libcublasLt.so", "libcublasLt.so.13" }, .{ "libcudart.so", "libcudart.so.13" }, @@ -14,6 +13,9 @@ pub export fn zmlxcuda_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque .{ "libcusolver.so", "libcusolver.so.12" }, .{ "libcusparse.so", "libcusparse.so.12" }, .{ "libnccl.so", "libnccl.so.2" }, + + // CUDA Compat + .{ "libcuda.so.1", "libcuda.so.1" }, }); var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; From 45b21db58927a098c7adee9a93e2cb9bb8ee7c06 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Sun, 22 Feb 2026 23:36:30 +0100 Subject: [PATCH 324/881] Replace rules_rust by rules_rs --- MODULE.bazel | 19 +- MODULE.bazel.lock | 465 ++++++++++++++++++++++++- zml/tokenizer/hftokenizers/BUILD.bazel | 2 +- 3 files changed, 462 insertions(+), 24 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index b0d54b40..81f6833c 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -16,8 +16,7 @@ bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep(name = "rules_oci", version = "2.2.7") bazel_dep(name = "rules_proto", version = "7.1.0") bazel_dep(name = "rules_python", version = "1.8.4") -bazel_dep(name = "rules_rust", version = "0.68.1") -bazel_dep(name = "rules_rs", version = "0.0.28") +bazel_dep(name = "rules_rs", version = "0.0.33") bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "rules_zig", version = "0.12.2") bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.5.5") @@ -139,20 +138,14 @@ llvm = use_extension("//third_party/xla:llvm.bzl", "llvm") llvm.configure() use_repo(llvm, "llvm-project") -rust = use_extension("@rules_rust//rust:extensions.bzl", "rust") -rust.toolchain( +rs_toolchains = use_extension("@rules_rs//rs/experimental/toolchains:module_extension.bzl", "toolchains") +rs_toolchains.toolchain( edition = "2024", - extra_target_triples = [ - "aarch64-apple-darwin", - "x86_64-apple-darwin", - "aarch64-unknown-linux-gnu", - "x86_64-unknown-linux-gnu", - ], - versions = ["1.92.0"], + version = "1.92.0", ) -use_repo(rust, "rust_toolchains") +use_repo(rs_toolchains, "default_rust_toolchains") -register_toolchains("@rust_toolchains//:all") +register_toolchains("@default_rust_toolchains//:all") crate = use_extension("@rules_rs//rs:extensions.bzl", "crate") crate.from_cargo( diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index cc67df7a..9238451c 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -16,7 +16,6 @@ "https://bcr.bazel.build/modules/abseil-cpp/20260107.1/source.json": "7a9a88969b1e79268cf613728ca8ff8fa4bc4b1a9abee9ec1fb5f113ca751971", "https://bcr.bazel.build/modules/apple_support/1.11.1/MODULE.bazel": "1843d7cd8a58369a444fc6000e7304425fba600ff641592161d9f15b179fb896", "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", - "https://bcr.bazel.build/modules/apple_support/1.23.0/MODULE.bazel": "317d47e3f65b580e7fb4221c160797fda48e32f07d2dfff63d754ef2316dcd25", "https://bcr.bazel.build/modules/apple_support/1.23.1/MODULE.bazel": "53763fed456a968cf919b3240427cf3a9d5481ec5466abc9d5dc51bc70087442", "https://bcr.bazel.build/modules/apple_support/1.24.1/MODULE.bazel": "f46e8ddad60aef170ee92b2f3d00ef66c147ceafea68b6877cb45bd91737f5f8", "https://bcr.bazel.build/modules/apple_support/2.2.0/MODULE.bazel": "5e32bc42e413a4544df7afee7fa4c7aff73e670a44e56e8e45ce1954332034ad", @@ -47,7 +46,6 @@ "https://bcr.bazel.build/modules/bazel_features/1.28.0/MODULE.bazel": "4b4200e6cbf8fa335b2c3f43e1d6ef3e240319c33d43d60cc0fbd4b87ece299d", "https://bcr.bazel.build/modules/bazel_features/1.3.0/MODULE.bazel": "cdcafe83ec318cda34e02948e81d790aab8df7a929cec6f6969f13a489ccecd9", "https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87", - "https://bcr.bazel.build/modules/bazel_features/1.32.0/MODULE.bazel": "095d67022a58cb20f7e20e1aefecfa65257a222c18a938e2914fd257b5f1ccdc", "https://bcr.bazel.build/modules/bazel_features/1.33.0/MODULE.bazel": "8b8dc9d2a4c88609409c3191165bccec0e4cb044cd7a72ccbe826583303459f6", "https://bcr.bazel.build/modules/bazel_features/1.34.0/MODULE.bazel": "e8475ad7c8965542e0c7aac8af68eb48c4af904be3d614b6aa6274c092c2ea1e", "https://bcr.bazel.build/modules/bazel_features/1.38.0/MODULE.bazel": "f9b8a9c890ebd216b4049fd12a31d3c2602e3403c7af636b04fbbd7453edc9c9", @@ -215,14 +213,10 @@ "https://bcr.bazel.build/modules/rules_python/1.6.0/MODULE.bazel": "7e04ad8f8d5bea40451cf80b1bd8262552aa73f841415d20db96b7241bd027d8", "https://bcr.bazel.build/modules/rules_python/1.8.4/MODULE.bazel": "33e3971e66161a3e955f7a0d411a8d1f291c4ce4c561851512466f3c77ff8ece", "https://bcr.bazel.build/modules/rules_python/1.8.4/source.json": "9fbc0e57bae52cddcc3831d668bce87a47e0c655104a85098d4459dd9a3b0a10", - "https://bcr.bazel.build/modules/rules_rs/0.0.28/MODULE.bazel": "3c3830eda824d0476818443dd4a3a2fa0765484589db874d18e94fb6ee50a60f", - "https://bcr.bazel.build/modules/rules_rs/0.0.28/source.json": "3480135f6ef8111ec7abe2b7cc164af40354b4eaf8f885170567922a70a2dc99", - "https://bcr.bazel.build/modules/rules_rust/0.66.0/MODULE.bazel": "86ef763a582f4739a27029bdcc6c562258ed0ea6f8d58294b049e215ceb251b3", - "https://bcr.bazel.build/modules/rules_rust/0.68.1/MODULE.bazel": "8d3332ef4079673385eb81f8bd68b012decc04ac00c9d5a01a40eff90301732c", - "https://bcr.bazel.build/modules/rules_rust/0.68.1/source.json": "3378e746f81b62457fdfd37391244fa8ff075ba85c05931ee4f3a20ac1efe963", + "https://bcr.bazel.build/modules/rules_rs/0.0.33/MODULE.bazel": "a4b5cc28e2325f75c15055d29905b896af942f157e8347cccfe7226ea03639ff", + "https://bcr.bazel.build/modules/rules_rs/0.0.33/source.json": "d7dbddeb334e57a91be80b0e11b3df37ca885e38f44e11ce430cf4c5be438cfb", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", - "https://bcr.bazel.build/modules/rules_shell/0.4.0/MODULE.bazel": "0f8f11bb3cd11755f0b48c1de0bbcf62b4b34421023aa41a2fc74ef68d9584f0", "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", "https://bcr.bazel.build/modules/rules_shell/0.6.1/MODULE.bazel": "72e76b0eea4e81611ef5452aa82b3da34caca0c8b7b5c0c9584338aa93bae26b", "https://bcr.bazel.build/modules/rules_shell/0.6.1/source.json": "20ec05cd5e592055e214b2da8ccb283c7f2a421ea0dc2acbf1aa792e11c03d0c", @@ -245,6 +239,7 @@ "https://bcr.bazel.build/modules/tar.bzl/0.5.5/MODULE.bazel": "4bfab9bbc7a1966c2c5f7371f5848f5e2d27c465951b4435adc9aaf00ed681da", "https://bcr.bazel.build/modules/tar.bzl/0.6.0/MODULE.bazel": "a3584b4edcfafcabd9b0ef9819808f05b372957bbdff41601429d5fd0aac2e7c", "https://bcr.bazel.build/modules/tar.bzl/0.6.0/source.json": "4a620381df075a16cb3a7ed57bd1d05f7480222394c64a20fa51bdb636fda658", + "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.5.2/MODULE.bazel": "f7c822cea99caef928d7cbe695498096e53c4b2c0ea45997e9a64bf6b77b43b0", "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.5.5/MODULE.bazel": "a2f1469b2920fcc9c043c87301b7a207d320231615abecd7771d89e516fc3d0b", "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.5.5/source.json": "8498c094703da13a859d19dd2a200152f75870fb7f22a9770df99d230a4f50d1", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", @@ -329,10 +324,60 @@ ] } }, + "@@aspect_rules_py+//uv/private/tomltool:extension.bzl%tomltool": { + "general": { + "bzlTransitiveDigest": "9BC6q6vdlG+lqF9L13wVxxakVkZBx8buKPFIndN6mHk=", + "usagesDigest": "tQ76lH+Nn5OlAFLvBbnfvnCzNIiwGgY6tOumYZXH44w=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "toml2json_aarch64_osx_libsystem": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_darwin_arm64", + "sha256": "9adc44d976e8f5baf5a6d613ceb3db6e5f56b2e22e75ac56521fdce62f227d88", + "executable": true + } + }, + "toml2json_x86_64_osx_libsystem": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_darwin_amd64", + "sha256": "425bac4015394fd9840fabcd52ac60fc796a9c10655b36c57e36ffb7dc9f3dd4", + "executable": true + } + }, + "toml2json_aarch64_linux_gnu": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_linux_arm64", + "sha256": "be8376c8e3232a242eae0d187f741a475498a949b942361a9af6e95072ef5670", + "executable": true + } + }, + "toml2json_x86_64_linux_gnu": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_linux_amd64", + "sha256": "f3dd54fabf2d27d0c027b0421860e5d9d909080be1613f0ffd87057633b65e9a", + "executable": true + } + } + }, + "recordedRepoMappingEntries": [ + [ + "aspect_rules_py+", + "bazel_tools", + "bazel_tools" + ] + ] + } + }, "@@aspect_tools_telemetry+//:extension.bzl%telemetry": { "general": { "bzlTransitiveDigest": "MePriaXmQNSqUfE+YQvEtzBc2bU1mjITINIffFiZYgo=", - "usagesDigest": "73l8h4f8amAmbKge5Ixr/4s4BE7L7fqDBTTAMrNSHHg=", + "usagesDigest": "xzgqWAkG6mzgCQKHijLhIVslGHP1WV0cvuQ6y2vaxIw=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -342,7 +387,7 @@ "attributes": { "deps": { "aspect_rules_py": "1.8.4", - "rules_rs": "0.0.28", + "rules_rs": "0.0.33", "aspect_tools_telemetry": "0.3.3" } } @@ -362,6 +407,156 @@ ] } }, + "@@pybind11_bazel+//:internal_configure.bzl%internal_configure_extension": { + "general": { + "bzlTransitiveDigest": "jTw8kZYoegX0+gtXU1FE+mVEDLV9nUqeeuVWSrz01Lw=", + "usagesDigest": "D1r3lfzMuUBFxgG8V6o0bQTLMk3GkaGOaPzw53wrwyw=", + "recordedFileInputs": { + "@@pybind11_bazel+//MODULE.bazel": "45d98c531db5f8430f847b195357dca36665d9c926cbbee77a85105fc6865d27" + }, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "pybind11": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file": "@@pybind11_bazel+//:pybind11-BUILD.bazel", + "strip_prefix": "pybind11-2.12.0", + "urls": [ + "https://github.com/pybind/pybind11/archive/v2.12.0.zip" + ] + } + } + }, + "recordedRepoMappingEntries": [ + [ + "pybind11_bazel+", + "bazel_tools", + "bazel_tools" + ] + ] + } + }, + "@@rules_apple+//apple:apple.bzl%provisioning_profile_repository_extension": { + "general": { + "bzlTransitiveDigest": "uY1CHD/Su1yr/BcqlNp/BNNojNdV0IX9TkUkISbvHzU=", + "usagesDigest": "vsJl8Rw5NL+5Ag2wdUDoTeRF/5klkXO8545Iy7U1Q08=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "local_provisioning_profiles": { + "repoRuleId": "@@rules_apple+//apple/internal:local_provisioning_profiles.bzl%provisioning_profile_repository", + "attributes": {} + } + }, + "recordedRepoMappingEntries": [ + [ + "apple_support+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "bazel_tools", + "rules_cc", + "rules_cc+" + ], + [ + "rules_apple+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "rules_apple+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_apple+", + "build_bazel_apple_support", + "apple_support+" + ], + [ + "rules_apple+", + "build_bazel_rules_swift", + "rules_swift+" + ], + [ + "rules_cc+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_cc+", + "cc_compatibility_proxy", + "rules_cc++compatibility_proxy+cc_compatibility_proxy" + ], + [ + "rules_cc+", + "rules_cc", + "rules_cc+" + ], + [ + "rules_cc++compatibility_proxy+cc_compatibility_proxy", + "rules_cc", + "rules_cc+" + ], + [ + "rules_swift+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "rules_swift+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_swift+", + "build_bazel_apple_support", + "apple_support+" + ], + [ + "rules_swift+", + "build_bazel_rules_swift", + "rules_swift+" + ], + [ + "rules_swift+", + "build_bazel_rules_swift_local_config", + "rules_swift++non_module_deps+build_bazel_rules_swift_local_config" + ] + ] + } + }, + "@@rules_apple+//apple:extensions.bzl%non_module_deps": { + "general": { + "bzlTransitiveDigest": "fe3Htkk5VXHdsIrdLtxsT4oUgADmqgmxwx83bja4UqY=", + "usagesDigest": "M3VqFpeTCo4qmrNKGZw0dxBHvTYDrfV3cscGzlSAhQ4=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "xctestrunner": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/google/xctestrunner/archive/b7698df3d435b6491b4b4c0f9fc7a63fbed5e3a6.tar.gz" + ], + "strip_prefix": "xctestrunner-b7698df3d435b6491b4b4c0f9fc7a63fbed5e3a6", + "sha256": "ae3a063c985a8633cb7eb566db21656f8db8eb9a0edb8c182312c7f0db53730d" + } + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_apple+", + "bazel_tools", + "bazel_tools" + ] + ] + } + }, "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { "general": { "bzlTransitiveDigest": "AEsQvKwdstptzt98AO1bJYfoWpPUn2upx53ra6o/Rc8=", @@ -1022,6 +1217,165 @@ ] } }, + "@@rules_swift+//swift:extensions.bzl%non_module_deps": { + "general": { + "bzlTransitiveDigest": "vwewHB1JdoOb38MyyeRQPIdGOkEKD52YNUYgGwmeOy8=", + "usagesDigest": "mhACFnrdMv9Wi0Mt67bxocJqviRkDSV+Ee5Mqdj5akA=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "com_github_apple_swift_protobuf": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-protobuf/archive/1.20.2.tar.gz" + ], + "sha256": "3fb50bd4d293337f202d917b6ada22f9548a0a0aed9d9a4d791e6fbd8a246ebb", + "strip_prefix": "swift-protobuf-1.20.2/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_protobuf/BUILD.overlay" + } + }, + "com_github_grpc_grpc_swift": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/grpc/grpc-swift/archive/1.16.0.tar.gz" + ], + "sha256": "58b60431d0064969f9679411264b82e40a217ae6bd34e17096d92cc4e47556a5", + "strip_prefix": "grpc-swift-1.16.0/", + "build_file": "@@rules_swift+//third_party:com_github_grpc_grpc_swift/BUILD.overlay" + } + }, + "com_github_apple_swift_docc_symbolkit": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-docc-symbolkit/archive/refs/tags/swift-5.10-RELEASE.tar.gz" + ], + "sha256": "de1d4b6940468ddb53b89df7aa1a81323b9712775b0e33e8254fa0f6f7469a97", + "strip_prefix": "swift-docc-symbolkit-swift-5.10-RELEASE", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_docc_symbolkit/BUILD.overlay" + } + }, + "com_github_apple_swift_nio": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio/archive/2.42.0.tar.gz" + ], + "sha256": "e3304bc3fb53aea74a3e54bd005ede11f6dc357117d9b1db642d03aea87194a0", + "strip_prefix": "swift-nio-2.42.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_http2": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-http2/archive/1.26.0.tar.gz" + ], + "sha256": "f0edfc9d6a7be1d587e5b403f2d04264bdfae59aac1d74f7d974a9022c6d2b25", + "strip_prefix": "swift-nio-http2-1.26.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_http2/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_transport_services": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-transport-services/archive/1.15.0.tar.gz" + ], + "sha256": "f3498dafa633751a52b9b7f741f7ac30c42bcbeb3b9edca6d447e0da8e693262", + "strip_prefix": "swift-nio-transport-services-1.15.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_transport_services/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_extras": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-extras/archive/1.4.0.tar.gz" + ], + "sha256": "4684b52951d9d9937bb3e8ccd6b5daedd777021ef2519ea2f18c4c922843b52b", + "strip_prefix": "swift-nio-extras-1.4.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_extras/BUILD.overlay" + } + }, + "com_github_apple_swift_log": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-log/archive/1.4.4.tar.gz" + ], + "sha256": "48fe66426c784c0c20031f15dc17faf9f4c9037c192bfac2f643f65cb2321ba0", + "strip_prefix": "swift-log-1.4.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_log/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_ssl": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-ssl/archive/2.23.0.tar.gz" + ], + "sha256": "4787c63f61dd04d99e498adc3d1a628193387e41efddf8de19b8db04544d016d", + "strip_prefix": "swift-nio-ssl-2.23.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_ssl/BUILD.overlay" + } + }, + "com_github_apple_swift_collections": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-collections/archive/1.0.4.tar.gz" + ], + "sha256": "d9e4c8a91c60fb9c92a04caccbb10ded42f4cb47b26a212bc6b39cc390a4b096", + "strip_prefix": "swift-collections-1.0.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_collections/BUILD.overlay" + } + }, + "com_github_apple_swift_atomics": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-atomics/archive/1.1.0.tar.gz" + ], + "sha256": "1bee7f469f7e8dc49f11cfa4da07182fbc79eab000ec2c17bfdce468c5d276fb", + "strip_prefix": "swift-atomics-1.1.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_atomics/BUILD.overlay" + } + }, + "build_bazel_rules_swift_index_import": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file": "@@rules_swift+//third_party:build_bazel_rules_swift_index_import/BUILD.overlay", + "canonical_id": "index-import-5.8", + "urls": [ + "https://github.com/MobileNativeFoundation/index-import/releases/download/5.8.0.1/index-import.tar.gz" + ], + "sha256": "28c1ffa39d99e74ed70623899b207b41f79214c498c603915aef55972a851a15" + } + }, + "build_bazel_rules_swift_local_config": { + "repoRuleId": "@@rules_swift+//swift/internal:swift_autoconfiguration.bzl%swift_autoconfiguration", + "attributes": {} + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_swift+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_swift+", + "build_bazel_rules_swift", + "rules_swift+" + ] + ] + } + }, "@@rules_zig+//zig:extensions.bzl%zig": { "general": { "bzlTransitiveDigest": "p1+tPOh3UlhcIaGGf1ycZCCC6I4NIHhiiHS0nAhhgHA=", @@ -1404,6 +1758,97 @@ "zerocopy-derive_0.8.26": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"dissimilar\",\"req\":\"^1.0.9\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"=0.2.163\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"=1.9\"},{\"kind\":\"dev\",\"name\":\"prettyplease\",\"req\":\"=0.2.17\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.10\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.89\"}],\"features\":{}}", "zerocopy_0.7.35": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"assert_matches\",\"req\":\"^1.5\"},{\"default_features\":false,\"name\":\"byteorder\",\"optional\":true,\"req\":\"^1.3\"},{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.85\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.7.35\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.7.35\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.7.35\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\"],\"alloc\":[],\"default\":[\"byteorder\"],\"derive\":[\"zerocopy-derive\"],\"simd\":[],\"simd-nightly\":[\"simd\"]}}", "zerocopy_0.8.26": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"either\",\"req\":\"=1.13.0\"},{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.89\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.8.26\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\",\"std\"],\"alloc\":[],\"derive\":[\"zerocopy-derive\"],\"float-nightly\":[],\"simd\":[],\"simd-nightly\":[\"simd\"],\"std\":[\"alloc\"]}}" + }, + "@@rules_rs+//rs/experimental/toolchains:module_extension.bzl%toolchains": { + "cargo-1.92.0-aarch64-apple-darwin.tar.xz": "bce6e7def37240c5a63115828017a9fc0ebcb31e64115382f5943b62b71aa34a", + "cargo-1.92.0-aarch64-pc-windows-msvc.tar.xz": "3fc47cc29781190c5075f807836c4b98d411aaf27c59598fbfb2aa781ccf6190", + "cargo-1.92.0-aarch64-unknown-linux-gnu.tar.xz": "cb2ce6be6411b986e25c71ad8a813f9dfbe3461738136fd684e3644f8dd75df4", + "cargo-1.92.0-x86_64-apple-darwin.tar.xz": "b033a7c33aba8af947c9d0ab2785f9696347cded228ffe731897f1c627466262", + "cargo-1.92.0-x86_64-pc-windows-msvc.tar.xz": "f4057534ec58708ed3554957582a6c8e8c6abfee011b5c8ad45c3ab27c0b3076", + "cargo-1.92.0-x86_64-unknown-linux-gnu.tar.xz": "e5e12be2c7126a7036c8adf573078a28b92611f5767cc9bd0a6f7c83081df103", + "clippy-1.92.0-aarch64-apple-darwin.tar.xz": "08c65b6cf8faae3861706f8c97acf2aa6b784ed9455354c3b13495a7cfe5cb84", + "clippy-1.92.0-aarch64-pc-windows-msvc.tar.xz": "2aa2f1288072ea1f870d6d860d09f5911c1305d506f344b65cf5e27b5070e262", + "clippy-1.92.0-aarch64-unknown-linux-gnu.tar.xz": "333ab38c673b589468b8293b525e5704fb52515d9d516ee28d3d34dd5a63d3c3", + "clippy-1.92.0-x86_64-apple-darwin.tar.xz": "39cce87aab3d8b71350edcb3f943fba7bc59581ce1e65e158ee01e64cf0f1cf5", + "clippy-1.92.0-x86_64-pc-windows-msvc.tar.xz": "51a9a67931a3e50eb0c26b4c6a7ec2ef6ddd633319343a2d4767710c1e33b21f", + "clippy-1.92.0-x86_64-unknown-linux-gnu.tar.xz": "2c1bf6e7da8ec50feba03fe188fc9a744ba59e2c6ece7970c13e201d08defa9a", + "rust-std-1.92.0-aarch64-apple-darwin.tar.xz": "ea619984fcb8e24b05dbd568d599b8e10d904435ab458dfba6469e03e0fd69aa", + "rust-std-1.92.0-aarch64-apple-ios-macabi.tar.xz": "d0453906db0abe9efb595e1ed59feb131a94c0312c0bc72da6482936667ff8da", + "rust-std-1.92.0-aarch64-apple-ios-sim.tar.xz": "9362b66fbaf2503276ea34f34b41b171269db9ba4dce05dde3472ea9d1852601", + "rust-std-1.92.0-aarch64-apple-ios.tar.xz": "81fb496f94a3f52ec2818a76a7107905b13b37d490ca849d22c0a7a7d7b0125e", + "rust-std-1.92.0-aarch64-linux-android.tar.xz": "ce6350bd43856c630773c93e40310989c6cb98a1178233c44e512a31761943d0", + "rust-std-1.92.0-aarch64-pc-windows-gnullvm.tar.xz": "4e79f57b80040757a3425568b5978986f026daf771649a64021c74bcc138214b", + "rust-std-1.92.0-aarch64-pc-windows-msvc.tar.xz": "b20c5c696af4ecfb683370ca0ee3c76ab8726fe6470ce9f1368d41a5b06ea065", + "rust-std-1.92.0-aarch64-unknown-fuchsia.tar.xz": "6021246b6c0d9d6104c0b350f7cd48a31d5707edaa8063f77f5636fe07bdb79d", + "rust-std-1.92.0-aarch64-unknown-linux-gnu.tar.xz": "ce2ab42c09d633b0a8b4b65a297c700ae0fad47aae890f75894782f95be7e36d", + "rust-std-1.92.0-aarch64-unknown-linux-musl.tar.xz": "94b9f84f21d29825c55a27fbb6b4b9fb9296a4a841aa54d85b95c42445623363", + "rust-std-1.92.0-aarch64-unknown-none-softfloat.tar.xz": "0dc46fafaaa36f53eec49e14a69e1d6d9ac6f0b9624a01081ad311d8139a2be0", + "rust-std-1.92.0-aarch64-unknown-none.tar.xz": "ab6a2edab924739fc2c86e9f8fd8068b379e881a6261a177d66608d3ea4cacb1", + "rust-std-1.92.0-aarch64-unknown-uefi.tar.xz": "f98001222bf23743598382c232b08d3137035b53645a420a1425288d501808af", + "rust-std-1.92.0-arm-linux-androideabi.tar.xz": "d41ec7255556b605dda04201a23e4445b5b86bc6786c703f91eb985bddc9f4ca", + "rust-std-1.92.0-arm-unknown-linux-gnueabi.tar.xz": "35478e20f8cc13912b31f2905b313a2820ddae564b363a66ab7a5da39d12787f", + "rust-std-1.92.0-arm-unknown-linux-gnueabihf.tar.xz": "836ada282b65c57d71f9b7e6fb490832410c954aac905c5437fb0bf851b53c83", + "rust-std-1.92.0-arm-unknown-linux-musleabi.tar.xz": "91d3c5fdbda9ba2e778bb638e3a5d060f3021bbc7a60edf22b0035be4e611b30", + "rust-std-1.92.0-arm-unknown-linux-musleabihf.tar.xz": "20411e3858308add0dadde9ce2059d82cdd6c5339881fa93ac995e733df0f832", + "rust-std-1.92.0-armv7-linux-androideabi.tar.xz": "06ac2f08dcf5c480e7767c0541c9bd7460754ec3a369a4b7097b328eca28fab9", + "rust-std-1.92.0-armv7-unknown-linux-gnueabi.tar.xz": "3d5e3fb14441ea8e6fc6307cbd89efd69be42143ff7f2b8dfb19818ddca993c0", + "rust-std-1.92.0-armv7-unknown-linux-gnueabihf.tar.xz": "e3ac810db43067d8af9e17679d47534e871f1daad8cd0762e67da935681e9e19", + "rust-std-1.92.0-armv7-unknown-linux-musleabi.tar.xz": "89cbf7db934d543754b446a52398081ec40ee6b98ed5bca93ac1dbd5faf48c16", + "rust-std-1.92.0-armv7-unknown-linux-musleabihf.tar.xz": "c7c08a389bd351226fd52266bfe2d2c444597e1bbb5d0307da44bdfa4df62c99", + "rust-std-1.92.0-i686-linux-android.tar.xz": "e45832b005556f65c6d26f05f34bd4ab5ceea8d9b9fefc4175d52b0780ca89db", + "rust-std-1.92.0-i686-pc-windows-gnu.tar.xz": "b3eefe10d4aed3ccbeaff3ae9cd2e0e0a36894c0635d0e69af1c9a698679d75f", + "rust-std-1.92.0-i686-pc-windows-gnullvm.tar.xz": "e6d709f85dea51d81f2f1a030845b732b9f7761d95d93c57c7276b0a451c2993", + "rust-std-1.92.0-i686-pc-windows-msvc.tar.xz": "e5671b276047647e994a7cab99c90ee70c46787572fbe4e266a13c6edb84d5ce", + "rust-std-1.92.0-i686-unknown-freebsd.tar.xz": "e008a0506ec4d5eff30abdf376c7933e235670bd6c5e1131c52bcda097a21116", + "rust-std-1.92.0-i686-unknown-linux-gnu.tar.xz": "abc840631a4462a4c8ec61341110ff653ab2ef86ef3b10f84489d00cc8a9310d", + "rust-std-1.92.0-i686-unknown-linux-musl.tar.xz": "c796874b1343721f575203fa179dc258e09ac45cd95dd6c35c4d5979a3870494", + "rust-std-1.92.0-i686-unknown-uefi.tar.xz": "90da7759e28e62fb82454d4eb4e02ac14320b8da3372e02f9ca4f47f7afbfe40", + "rust-std-1.92.0-powerpc-unknown-linux-gnu.tar.xz": "c3e809a324b00eb53096c58df38645bb496c6560de334dfe04ed0b77c0605aaa", + "rust-std-1.92.0-powerpc64-unknown-linux-gnu.tar.xz": "2ce706afa4a46b6773340854de877fc63618a40e351298a4e3da8eb482619863", + "rust-std-1.92.0-powerpc64le-unknown-linux-gnu.tar.xz": "eba59766c2d9805c0a1fc82fd723acbb36569e1bec1088c037bba84d965f70ba", + "rust-std-1.92.0-powerpc64le-unknown-linux-musl.tar.xz": "8b515e18b6ac8f8d37ea3cabe644b7f571984333c3b4192b7f5877e79eae7893", + "rust-std-1.92.0-riscv32imc-unknown-none-elf.tar.xz": "e1c6968ce25ab78f2c5b5460691763a2d39cb71b01d6d54c208d4a8b735b0584", + "rust-std-1.92.0-riscv64gc-unknown-linux-gnu.tar.xz": "8ee20dcf3b1063fa6069b3ce85e1fcf42794dfa783263314865cb53fff42d9e4", + "rust-std-1.92.0-riscv64gc-unknown-linux-musl.tar.xz": "f20d822309900fd6c7230688694baf91e900154e44e6247feca49b7b7a203a57", + "rust-std-1.92.0-riscv64gc-unknown-none-elf.tar.xz": "7eacf6a98786d58ef912643fd5aedc35111712e96a47b7e5d3ccddcdf9fb166d", + "rust-std-1.92.0-s390x-unknown-linux-gnu.tar.xz": "ebf944dc95015498d322504a54e4f9cdb28590f7790aa3a9eb86d6cf4b6c93ff", + "rust-std-1.92.0-thumbv6m-none-eabi.tar.xz": "f55de77126b60e1da38f8a5cdd127db872155ce9fbb53d4459502dd47b1dd476", + "rust-std-1.92.0-thumbv7em-none-eabi.tar.xz": "fdee017dcebfa8675220c20ca65b945b6eaee744d5d19b92cb0ca5480dd19ba6", + "rust-std-1.92.0-thumbv7em-none-eabihf.tar.xz": "6bfd083ce4917440fb4b04ca39ab4dadc9f40d9dc2775056899857bfa8911fd0", + "rust-std-1.92.0-thumbv7m-none-eabi.tar.xz": "7dc4c92e97db5ce1ddd9fbb7fbb1ad2d367f1c2e20d296a6d6427480c900c315", + "rust-std-1.92.0-thumbv8m.main-none-eabi.tar.xz": "61a6a80d03ebdb80ab06a044d4ec60e3c2bd8dc4d6011e920cf41592df4f0646", + "rust-std-1.92.0-thumbv8m.main-none-eabihf.tar.xz": "24c2f65371a2a5c6a40b51ae0e276ea9afff0479255f180f5e1549a0a4d58fb3", + "rust-std-1.92.0-wasm32-unknown-emscripten.tar.xz": "59b7adf18c0cc416a005fad7f704203b965905eb1c0ed9c556daa3c14048b9f4", + "rust-std-1.92.0-wasm32-unknown-unknown.tar.xz": "6c73f053ccd6adc886f802270ba960fd854e5e1111e4b5cfb875f1fdcfc0eb60", + "rust-std-1.92.0-wasm32-wasip1-threads.tar.xz": "feea056dd657a26560dfddfe4b53daeef3ded83c140b453b6dbdebaabd2a664c", + "rust-std-1.92.0-wasm32-wasip1.tar.xz": "8107dc35f0b6b744d998e766351419b4e0d27cddd6456c461337753a25f29fcd", + "rust-std-1.92.0-wasm32-wasip2.tar.xz": "e69ce601b6b24eea08b0d9e1fb0d9bf2a4188b3109353b272be4d995f687c943", + "rust-std-1.92.0-x86_64-apple-darwin.tar.xz": "6ce143bf9e83c71e200f4180e8774ab22c8c8c2351c88484b13ff13be82c8d57", + "rust-std-1.92.0-x86_64-apple-ios-macabi.tar.xz": "6a292d774653f2deaac743060ec68bfd5d7ff01d9b364e1a7d1e57a679650b47", + "rust-std-1.92.0-x86_64-apple-ios.tar.xz": "b6e38e5f8c9e6fb294681a7951301856b8f9424af4589e14493c0c939338814c", + "rust-std-1.92.0-x86_64-linux-android.tar.xz": "ffd39429435ff2f0763b940dd6afb4b9ccb1ed443eeef4fff9f1e9b3c5730026", + "rust-std-1.92.0-x86_64-pc-windows-gnu.tar.xz": "d4043304ef0e4792fb79a1153cbeca41308aac37cb1af3aa6bc3f0bb6d2276e1", + "rust-std-1.92.0-x86_64-pc-windows-gnullvm.tar.xz": "6169605b3073a7c2d6960bc1c74cb9cd6b65f45ee34e7ede02368e07ce4564cf", + "rust-std-1.92.0-x86_64-pc-windows-msvc.tar.xz": "b4e53a9c9b96a1a0618364791b7728a1c8978101ec6d1ee78fe930e1ef061994", + "rust-std-1.92.0-x86_64-unknown-freebsd.tar.xz": "151929a4255175d14b2cbcb69ef46d9217add23be268b9cd1446f8eab16616f2", + "rust-std-1.92.0-x86_64-unknown-fuchsia.tar.xz": "51ec26391d166e2f190a329919e3c7bd793861f02284aba461a086268725d9f1", + "rust-std-1.92.0-x86_64-unknown-linux-gnu.tar.xz": "5f106805ed86ebf8df287039e53a45cf974391ef4d088c2760776b05b8e48b5d", + "rust-std-1.92.0-x86_64-unknown-linux-musl.tar.xz": "11f0b7efccdb5e7972e3c0fc23693a487abc28b624675c08161d055a016d527e", + "rust-std-1.92.0-x86_64-unknown-netbsd.tar.xz": "db6a8d3a091551701b12e40cc58d4a541adfb63f250074aae90d250329beb8de", + "rust-std-1.92.0-x86_64-unknown-none.tar.xz": "1d8420ab8eb241a35e38b76470277c722bd5a7aa4ac0c7a565ad6f30b37cb852", + "rust-std-1.92.0-x86_64-unknown-uefi.tar.xz": "1b849250cf095269f3a2c7bc2087a919386da7da28e80dc289e6268bc705142d", + "rustc-1.92.0-aarch64-apple-darwin.tar.xz": "15dee753c9217dff4cf45d734b29dc13ce6017d8a55fe34eed75022b39a63ff0", + "rustc-1.92.0-aarch64-pc-windows-msvc.tar.xz": "07ba5606143de3bc916455714f67732fca05805591396a8190d5e88b8863eea3", + "rustc-1.92.0-aarch64-unknown-linux-gnu.tar.xz": "7c8706fad4c038b5eacab0092e15db54d2b365d5f3323ca046fe987f814e7826", + "rustc-1.92.0-x86_64-apple-darwin.tar.xz": "0facbd5d2742c8e97c53d59c9b5b81db6088cfc285d9ecb99523a50d6765fc5c", + "rustc-1.92.0-x86_64-pc-windows-msvc.tar.xz": "6aec5aba5384ce1041538e71ebebf671d82e29f4fe697bc7844626d94b0dbfe6", + "rustc-1.92.0-x86_64-unknown-linux-gnu.tar.xz": "78b2dd9c6b1fcd2621fa81c611cf5e2d6950690775038b585c64f364422886e0", + "rustfmt-1.92.0-aarch64-apple-darwin.tar.xz": "5d8ea865a7999dc9141603be8a9352745bf8440da051eb1c43f0fcaaf6845441", + "rustfmt-1.92.0-aarch64-pc-windows-msvc.tar.xz": "e626e08df361c5a9127e0e4b5d73f87fd17ed30cc471887ebc033bebe97c0ff2", + "rustfmt-1.92.0-aarch64-unknown-linux-gnu.tar.xz": "1dce37aea2a7cb801f1756ffc531d7140428315a3d2c2f836272547eb7b9dacd", + "rustfmt-1.92.0-x86_64-apple-darwin.tar.xz": "e038bda323ed7f4d417efc5be44c4245d74b8394f9f8393b9d464d662c3a9499", + "rustfmt-1.92.0-x86_64-pc-windows-msvc.tar.xz": "3761b2e8a18d672b2804b0d1864dc122b7bc54343d6a109135c7843de7fa2ace", + "rustfmt-1.92.0-x86_64-unknown-linux-gnu.tar.xz": "38951ee55f21e9170236fc98c8ba373ae4338d863087c6b0a5fa8c4e797d52c4" } } } diff --git a/zml/tokenizer/hftokenizers/BUILD.bazel b/zml/tokenizer/hftokenizers/BUILD.bazel index a07d855b..ebd30da0 100644 --- a/zml/tokenizer/hftokenizers/BUILD.bazel +++ b/zml/tokenizer/hftokenizers/BUILD.bazel @@ -1,5 +1,5 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_rust//rust:defs.bzl", "rust_static_library") +load("@rules_rs//rs:rust_static_library.bzl", "rust_static_library") load("@rules_zig//zig:defs.bzl", "zig_library") rust_static_library( From c53d2275b41477e38f71d55b5d4409fd610e3bbb Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Mon, 23 Feb 2026 14:10:37 +0100 Subject: [PATCH 325/881] bazel: Revert cuda_compat & rules_rs (#392) Co-authored-by: Benjamin Rabier --- MODULE.bazel | 19 +- MODULE.bazel.lock | 465 +----------------------- platforms/cuda/cuda.bzl | 32 -- platforms/cuda/libpjrt_cuda.BUILD.bazel | 7 - platforms/cuda/zmlxcuda.zig | 3 - zml/tokenizer/hftokenizers/BUILD.bazel | 2 +- 6 files changed, 24 insertions(+), 504 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 81f6833c..b0d54b40 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -16,7 +16,8 @@ bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep(name = "rules_oci", version = "2.2.7") bazel_dep(name = "rules_proto", version = "7.1.0") bazel_dep(name = "rules_python", version = "1.8.4") -bazel_dep(name = "rules_rs", version = "0.0.33") +bazel_dep(name = "rules_rust", version = "0.68.1") +bazel_dep(name = "rules_rs", version = "0.0.28") bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "rules_zig", version = "0.12.2") bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.5.5") @@ -138,14 +139,20 @@ llvm = use_extension("//third_party/xla:llvm.bzl", "llvm") llvm.configure() use_repo(llvm, "llvm-project") -rs_toolchains = use_extension("@rules_rs//rs/experimental/toolchains:module_extension.bzl", "toolchains") -rs_toolchains.toolchain( +rust = use_extension("@rules_rust//rust:extensions.bzl", "rust") +rust.toolchain( edition = "2024", - version = "1.92.0", + extra_target_triples = [ + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-gnu", + ], + versions = ["1.92.0"], ) -use_repo(rs_toolchains, "default_rust_toolchains") +use_repo(rust, "rust_toolchains") -register_toolchains("@default_rust_toolchains//:all") +register_toolchains("@rust_toolchains//:all") crate = use_extension("@rules_rs//rs:extensions.bzl", "crate") crate.from_cargo( diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 9238451c..cc67df7a 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -16,6 +16,7 @@ "https://bcr.bazel.build/modules/abseil-cpp/20260107.1/source.json": "7a9a88969b1e79268cf613728ca8ff8fa4bc4b1a9abee9ec1fb5f113ca751971", "https://bcr.bazel.build/modules/apple_support/1.11.1/MODULE.bazel": "1843d7cd8a58369a444fc6000e7304425fba600ff641592161d9f15b179fb896", "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", + "https://bcr.bazel.build/modules/apple_support/1.23.0/MODULE.bazel": "317d47e3f65b580e7fb4221c160797fda48e32f07d2dfff63d754ef2316dcd25", "https://bcr.bazel.build/modules/apple_support/1.23.1/MODULE.bazel": "53763fed456a968cf919b3240427cf3a9d5481ec5466abc9d5dc51bc70087442", "https://bcr.bazel.build/modules/apple_support/1.24.1/MODULE.bazel": "f46e8ddad60aef170ee92b2f3d00ef66c147ceafea68b6877cb45bd91737f5f8", "https://bcr.bazel.build/modules/apple_support/2.2.0/MODULE.bazel": "5e32bc42e413a4544df7afee7fa4c7aff73e670a44e56e8e45ce1954332034ad", @@ -46,6 +47,7 @@ "https://bcr.bazel.build/modules/bazel_features/1.28.0/MODULE.bazel": "4b4200e6cbf8fa335b2c3f43e1d6ef3e240319c33d43d60cc0fbd4b87ece299d", "https://bcr.bazel.build/modules/bazel_features/1.3.0/MODULE.bazel": "cdcafe83ec318cda34e02948e81d790aab8df7a929cec6f6969f13a489ccecd9", "https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87", + "https://bcr.bazel.build/modules/bazel_features/1.32.0/MODULE.bazel": "095d67022a58cb20f7e20e1aefecfa65257a222c18a938e2914fd257b5f1ccdc", "https://bcr.bazel.build/modules/bazel_features/1.33.0/MODULE.bazel": "8b8dc9d2a4c88609409c3191165bccec0e4cb044cd7a72ccbe826583303459f6", "https://bcr.bazel.build/modules/bazel_features/1.34.0/MODULE.bazel": "e8475ad7c8965542e0c7aac8af68eb48c4af904be3d614b6aa6274c092c2ea1e", "https://bcr.bazel.build/modules/bazel_features/1.38.0/MODULE.bazel": "f9b8a9c890ebd216b4049fd12a31d3c2602e3403c7af636b04fbbd7453edc9c9", @@ -213,10 +215,14 @@ "https://bcr.bazel.build/modules/rules_python/1.6.0/MODULE.bazel": "7e04ad8f8d5bea40451cf80b1bd8262552aa73f841415d20db96b7241bd027d8", "https://bcr.bazel.build/modules/rules_python/1.8.4/MODULE.bazel": "33e3971e66161a3e955f7a0d411a8d1f291c4ce4c561851512466f3c77ff8ece", "https://bcr.bazel.build/modules/rules_python/1.8.4/source.json": "9fbc0e57bae52cddcc3831d668bce87a47e0c655104a85098d4459dd9a3b0a10", - "https://bcr.bazel.build/modules/rules_rs/0.0.33/MODULE.bazel": "a4b5cc28e2325f75c15055d29905b896af942f157e8347cccfe7226ea03639ff", - "https://bcr.bazel.build/modules/rules_rs/0.0.33/source.json": "d7dbddeb334e57a91be80b0e11b3df37ca885e38f44e11ce430cf4c5be438cfb", + "https://bcr.bazel.build/modules/rules_rs/0.0.28/MODULE.bazel": "3c3830eda824d0476818443dd4a3a2fa0765484589db874d18e94fb6ee50a60f", + "https://bcr.bazel.build/modules/rules_rs/0.0.28/source.json": "3480135f6ef8111ec7abe2b7cc164af40354b4eaf8f885170567922a70a2dc99", + "https://bcr.bazel.build/modules/rules_rust/0.66.0/MODULE.bazel": "86ef763a582f4739a27029bdcc6c562258ed0ea6f8d58294b049e215ceb251b3", + "https://bcr.bazel.build/modules/rules_rust/0.68.1/MODULE.bazel": "8d3332ef4079673385eb81f8bd68b012decc04ac00c9d5a01a40eff90301732c", + "https://bcr.bazel.build/modules/rules_rust/0.68.1/source.json": "3378e746f81b62457fdfd37391244fa8ff075ba85c05931ee4f3a20ac1efe963", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", + "https://bcr.bazel.build/modules/rules_shell/0.4.0/MODULE.bazel": "0f8f11bb3cd11755f0b48c1de0bbcf62b4b34421023aa41a2fc74ef68d9584f0", "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", "https://bcr.bazel.build/modules/rules_shell/0.6.1/MODULE.bazel": "72e76b0eea4e81611ef5452aa82b3da34caca0c8b7b5c0c9584338aa93bae26b", "https://bcr.bazel.build/modules/rules_shell/0.6.1/source.json": "20ec05cd5e592055e214b2da8ccb283c7f2a421ea0dc2acbf1aa792e11c03d0c", @@ -239,7 +245,6 @@ "https://bcr.bazel.build/modules/tar.bzl/0.5.5/MODULE.bazel": "4bfab9bbc7a1966c2c5f7371f5848f5e2d27c465951b4435adc9aaf00ed681da", "https://bcr.bazel.build/modules/tar.bzl/0.6.0/MODULE.bazel": "a3584b4edcfafcabd9b0ef9819808f05b372957bbdff41601429d5fd0aac2e7c", "https://bcr.bazel.build/modules/tar.bzl/0.6.0/source.json": "4a620381df075a16cb3a7ed57bd1d05f7480222394c64a20fa51bdb636fda658", - "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.5.2/MODULE.bazel": "f7c822cea99caef928d7cbe695498096e53c4b2c0ea45997e9a64bf6b77b43b0", "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.5.5/MODULE.bazel": "a2f1469b2920fcc9c043c87301b7a207d320231615abecd7771d89e516fc3d0b", "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.5.5/source.json": "8498c094703da13a859d19dd2a200152f75870fb7f22a9770df99d230a4f50d1", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", @@ -324,60 +329,10 @@ ] } }, - "@@aspect_rules_py+//uv/private/tomltool:extension.bzl%tomltool": { - "general": { - "bzlTransitiveDigest": "9BC6q6vdlG+lqF9L13wVxxakVkZBx8buKPFIndN6mHk=", - "usagesDigest": "tQ76lH+Nn5OlAFLvBbnfvnCzNIiwGgY6tOumYZXH44w=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "toml2json_aarch64_osx_libsystem": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_darwin_arm64", - "sha256": "9adc44d976e8f5baf5a6d613ceb3db6e5f56b2e22e75ac56521fdce62f227d88", - "executable": true - } - }, - "toml2json_x86_64_osx_libsystem": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_darwin_amd64", - "sha256": "425bac4015394fd9840fabcd52ac60fc796a9c10655b36c57e36ffb7dc9f3dd4", - "executable": true - } - }, - "toml2json_aarch64_linux_gnu": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_linux_arm64", - "sha256": "be8376c8e3232a242eae0d187f741a475498a949b942361a9af6e95072ef5670", - "executable": true - } - }, - "toml2json_x86_64_linux_gnu": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_linux_amd64", - "sha256": "f3dd54fabf2d27d0c027b0421860e5d9d909080be1613f0ffd87057633b65e9a", - "executable": true - } - } - }, - "recordedRepoMappingEntries": [ - [ - "aspect_rules_py+", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, "@@aspect_tools_telemetry+//:extension.bzl%telemetry": { "general": { "bzlTransitiveDigest": "MePriaXmQNSqUfE+YQvEtzBc2bU1mjITINIffFiZYgo=", - "usagesDigest": "xzgqWAkG6mzgCQKHijLhIVslGHP1WV0cvuQ6y2vaxIw=", + "usagesDigest": "73l8h4f8amAmbKge5Ixr/4s4BE7L7fqDBTTAMrNSHHg=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -387,7 +342,7 @@ "attributes": { "deps": { "aspect_rules_py": "1.8.4", - "rules_rs": "0.0.33", + "rules_rs": "0.0.28", "aspect_tools_telemetry": "0.3.3" } } @@ -407,156 +362,6 @@ ] } }, - "@@pybind11_bazel+//:internal_configure.bzl%internal_configure_extension": { - "general": { - "bzlTransitiveDigest": "jTw8kZYoegX0+gtXU1FE+mVEDLV9nUqeeuVWSrz01Lw=", - "usagesDigest": "D1r3lfzMuUBFxgG8V6o0bQTLMk3GkaGOaPzw53wrwyw=", - "recordedFileInputs": { - "@@pybind11_bazel+//MODULE.bazel": "45d98c531db5f8430f847b195357dca36665d9c926cbbee77a85105fc6865d27" - }, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "pybind11": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "build_file": "@@pybind11_bazel+//:pybind11-BUILD.bazel", - "strip_prefix": "pybind11-2.12.0", - "urls": [ - "https://github.com/pybind/pybind11/archive/v2.12.0.zip" - ] - } - } - }, - "recordedRepoMappingEntries": [ - [ - "pybind11_bazel+", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@rules_apple+//apple:apple.bzl%provisioning_profile_repository_extension": { - "general": { - "bzlTransitiveDigest": "uY1CHD/Su1yr/BcqlNp/BNNojNdV0IX9TkUkISbvHzU=", - "usagesDigest": "vsJl8Rw5NL+5Ag2wdUDoTeRF/5klkXO8545Iy7U1Q08=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "local_provisioning_profiles": { - "repoRuleId": "@@rules_apple+//apple/internal:local_provisioning_profiles.bzl%provisioning_profile_repository", - "attributes": {} - } - }, - "recordedRepoMappingEntries": [ - [ - "apple_support+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "bazel_tools", - "rules_cc", - "rules_cc+" - ], - [ - "rules_apple+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "rules_apple+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_apple+", - "build_bazel_apple_support", - "apple_support+" - ], - [ - "rules_apple+", - "build_bazel_rules_swift", - "rules_swift+" - ], - [ - "rules_cc+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_cc+", - "cc_compatibility_proxy", - "rules_cc++compatibility_proxy+cc_compatibility_proxy" - ], - [ - "rules_cc+", - "rules_cc", - "rules_cc+" - ], - [ - "rules_cc++compatibility_proxy+cc_compatibility_proxy", - "rules_cc", - "rules_cc+" - ], - [ - "rules_swift+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "rules_swift+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_swift+", - "build_bazel_apple_support", - "apple_support+" - ], - [ - "rules_swift+", - "build_bazel_rules_swift", - "rules_swift+" - ], - [ - "rules_swift+", - "build_bazel_rules_swift_local_config", - "rules_swift++non_module_deps+build_bazel_rules_swift_local_config" - ] - ] - } - }, - "@@rules_apple+//apple:extensions.bzl%non_module_deps": { - "general": { - "bzlTransitiveDigest": "fe3Htkk5VXHdsIrdLtxsT4oUgADmqgmxwx83bja4UqY=", - "usagesDigest": "M3VqFpeTCo4qmrNKGZw0dxBHvTYDrfV3cscGzlSAhQ4=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "xctestrunner": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/google/xctestrunner/archive/b7698df3d435b6491b4b4c0f9fc7a63fbed5e3a6.tar.gz" - ], - "strip_prefix": "xctestrunner-b7698df3d435b6491b4b4c0f9fc7a63fbed5e3a6", - "sha256": "ae3a063c985a8633cb7eb566db21656f8db8eb9a0edb8c182312c7f0db53730d" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_apple+", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { "general": { "bzlTransitiveDigest": "AEsQvKwdstptzt98AO1bJYfoWpPUn2upx53ra6o/Rc8=", @@ -1217,165 +1022,6 @@ ] } }, - "@@rules_swift+//swift:extensions.bzl%non_module_deps": { - "general": { - "bzlTransitiveDigest": "vwewHB1JdoOb38MyyeRQPIdGOkEKD52YNUYgGwmeOy8=", - "usagesDigest": "mhACFnrdMv9Wi0Mt67bxocJqviRkDSV+Ee5Mqdj5akA=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "com_github_apple_swift_protobuf": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-protobuf/archive/1.20.2.tar.gz" - ], - "sha256": "3fb50bd4d293337f202d917b6ada22f9548a0a0aed9d9a4d791e6fbd8a246ebb", - "strip_prefix": "swift-protobuf-1.20.2/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_protobuf/BUILD.overlay" - } - }, - "com_github_grpc_grpc_swift": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/grpc/grpc-swift/archive/1.16.0.tar.gz" - ], - "sha256": "58b60431d0064969f9679411264b82e40a217ae6bd34e17096d92cc4e47556a5", - "strip_prefix": "grpc-swift-1.16.0/", - "build_file": "@@rules_swift+//third_party:com_github_grpc_grpc_swift/BUILD.overlay" - } - }, - "com_github_apple_swift_docc_symbolkit": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-docc-symbolkit/archive/refs/tags/swift-5.10-RELEASE.tar.gz" - ], - "sha256": "de1d4b6940468ddb53b89df7aa1a81323b9712775b0e33e8254fa0f6f7469a97", - "strip_prefix": "swift-docc-symbolkit-swift-5.10-RELEASE", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_docc_symbolkit/BUILD.overlay" - } - }, - "com_github_apple_swift_nio": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio/archive/2.42.0.tar.gz" - ], - "sha256": "e3304bc3fb53aea74a3e54bd005ede11f6dc357117d9b1db642d03aea87194a0", - "strip_prefix": "swift-nio-2.42.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_http2": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-http2/archive/1.26.0.tar.gz" - ], - "sha256": "f0edfc9d6a7be1d587e5b403f2d04264bdfae59aac1d74f7d974a9022c6d2b25", - "strip_prefix": "swift-nio-http2-1.26.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_http2/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_transport_services": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-transport-services/archive/1.15.0.tar.gz" - ], - "sha256": "f3498dafa633751a52b9b7f741f7ac30c42bcbeb3b9edca6d447e0da8e693262", - "strip_prefix": "swift-nio-transport-services-1.15.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_transport_services/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_extras": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-extras/archive/1.4.0.tar.gz" - ], - "sha256": "4684b52951d9d9937bb3e8ccd6b5daedd777021ef2519ea2f18c4c922843b52b", - "strip_prefix": "swift-nio-extras-1.4.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_extras/BUILD.overlay" - } - }, - "com_github_apple_swift_log": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-log/archive/1.4.4.tar.gz" - ], - "sha256": "48fe66426c784c0c20031f15dc17faf9f4c9037c192bfac2f643f65cb2321ba0", - "strip_prefix": "swift-log-1.4.4/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_log/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_ssl": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-ssl/archive/2.23.0.tar.gz" - ], - "sha256": "4787c63f61dd04d99e498adc3d1a628193387e41efddf8de19b8db04544d016d", - "strip_prefix": "swift-nio-ssl-2.23.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_ssl/BUILD.overlay" - } - }, - "com_github_apple_swift_collections": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-collections/archive/1.0.4.tar.gz" - ], - "sha256": "d9e4c8a91c60fb9c92a04caccbb10ded42f4cb47b26a212bc6b39cc390a4b096", - "strip_prefix": "swift-collections-1.0.4/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_collections/BUILD.overlay" - } - }, - "com_github_apple_swift_atomics": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-atomics/archive/1.1.0.tar.gz" - ], - "sha256": "1bee7f469f7e8dc49f11cfa4da07182fbc79eab000ec2c17bfdce468c5d276fb", - "strip_prefix": "swift-atomics-1.1.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_atomics/BUILD.overlay" - } - }, - "build_bazel_rules_swift_index_import": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "build_file": "@@rules_swift+//third_party:build_bazel_rules_swift_index_import/BUILD.overlay", - "canonical_id": "index-import-5.8", - "urls": [ - "https://github.com/MobileNativeFoundation/index-import/releases/download/5.8.0.1/index-import.tar.gz" - ], - "sha256": "28c1ffa39d99e74ed70623899b207b41f79214c498c603915aef55972a851a15" - } - }, - "build_bazel_rules_swift_local_config": { - "repoRuleId": "@@rules_swift+//swift/internal:swift_autoconfiguration.bzl%swift_autoconfiguration", - "attributes": {} - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_swift+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_swift+", - "build_bazel_rules_swift", - "rules_swift+" - ] - ] - } - }, "@@rules_zig+//zig:extensions.bzl%zig": { "general": { "bzlTransitiveDigest": "p1+tPOh3UlhcIaGGf1ycZCCC6I4NIHhiiHS0nAhhgHA=", @@ -1758,97 +1404,6 @@ "zerocopy-derive_0.8.26": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"dissimilar\",\"req\":\"^1.0.9\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"=0.2.163\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"=1.9\"},{\"kind\":\"dev\",\"name\":\"prettyplease\",\"req\":\"=0.2.17\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.10\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.89\"}],\"features\":{}}", "zerocopy_0.7.35": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"assert_matches\",\"req\":\"^1.5\"},{\"default_features\":false,\"name\":\"byteorder\",\"optional\":true,\"req\":\"^1.3\"},{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.85\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.7.35\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.7.35\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.7.35\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\"],\"alloc\":[],\"default\":[\"byteorder\"],\"derive\":[\"zerocopy-derive\"],\"simd\":[],\"simd-nightly\":[\"simd\"]}}", "zerocopy_0.8.26": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"either\",\"req\":\"=1.13.0\"},{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.89\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.8.26\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\",\"std\"],\"alloc\":[],\"derive\":[\"zerocopy-derive\"],\"float-nightly\":[],\"simd\":[],\"simd-nightly\":[\"simd\"],\"std\":[\"alloc\"]}}" - }, - "@@rules_rs+//rs/experimental/toolchains:module_extension.bzl%toolchains": { - "cargo-1.92.0-aarch64-apple-darwin.tar.xz": "bce6e7def37240c5a63115828017a9fc0ebcb31e64115382f5943b62b71aa34a", - "cargo-1.92.0-aarch64-pc-windows-msvc.tar.xz": "3fc47cc29781190c5075f807836c4b98d411aaf27c59598fbfb2aa781ccf6190", - "cargo-1.92.0-aarch64-unknown-linux-gnu.tar.xz": "cb2ce6be6411b986e25c71ad8a813f9dfbe3461738136fd684e3644f8dd75df4", - "cargo-1.92.0-x86_64-apple-darwin.tar.xz": "b033a7c33aba8af947c9d0ab2785f9696347cded228ffe731897f1c627466262", - "cargo-1.92.0-x86_64-pc-windows-msvc.tar.xz": "f4057534ec58708ed3554957582a6c8e8c6abfee011b5c8ad45c3ab27c0b3076", - "cargo-1.92.0-x86_64-unknown-linux-gnu.tar.xz": "e5e12be2c7126a7036c8adf573078a28b92611f5767cc9bd0a6f7c83081df103", - "clippy-1.92.0-aarch64-apple-darwin.tar.xz": "08c65b6cf8faae3861706f8c97acf2aa6b784ed9455354c3b13495a7cfe5cb84", - "clippy-1.92.0-aarch64-pc-windows-msvc.tar.xz": "2aa2f1288072ea1f870d6d860d09f5911c1305d506f344b65cf5e27b5070e262", - "clippy-1.92.0-aarch64-unknown-linux-gnu.tar.xz": "333ab38c673b589468b8293b525e5704fb52515d9d516ee28d3d34dd5a63d3c3", - "clippy-1.92.0-x86_64-apple-darwin.tar.xz": "39cce87aab3d8b71350edcb3f943fba7bc59581ce1e65e158ee01e64cf0f1cf5", - "clippy-1.92.0-x86_64-pc-windows-msvc.tar.xz": "51a9a67931a3e50eb0c26b4c6a7ec2ef6ddd633319343a2d4767710c1e33b21f", - "clippy-1.92.0-x86_64-unknown-linux-gnu.tar.xz": "2c1bf6e7da8ec50feba03fe188fc9a744ba59e2c6ece7970c13e201d08defa9a", - "rust-std-1.92.0-aarch64-apple-darwin.tar.xz": "ea619984fcb8e24b05dbd568d599b8e10d904435ab458dfba6469e03e0fd69aa", - "rust-std-1.92.0-aarch64-apple-ios-macabi.tar.xz": "d0453906db0abe9efb595e1ed59feb131a94c0312c0bc72da6482936667ff8da", - "rust-std-1.92.0-aarch64-apple-ios-sim.tar.xz": "9362b66fbaf2503276ea34f34b41b171269db9ba4dce05dde3472ea9d1852601", - "rust-std-1.92.0-aarch64-apple-ios.tar.xz": "81fb496f94a3f52ec2818a76a7107905b13b37d490ca849d22c0a7a7d7b0125e", - "rust-std-1.92.0-aarch64-linux-android.tar.xz": "ce6350bd43856c630773c93e40310989c6cb98a1178233c44e512a31761943d0", - "rust-std-1.92.0-aarch64-pc-windows-gnullvm.tar.xz": "4e79f57b80040757a3425568b5978986f026daf771649a64021c74bcc138214b", - "rust-std-1.92.0-aarch64-pc-windows-msvc.tar.xz": "b20c5c696af4ecfb683370ca0ee3c76ab8726fe6470ce9f1368d41a5b06ea065", - "rust-std-1.92.0-aarch64-unknown-fuchsia.tar.xz": "6021246b6c0d9d6104c0b350f7cd48a31d5707edaa8063f77f5636fe07bdb79d", - "rust-std-1.92.0-aarch64-unknown-linux-gnu.tar.xz": "ce2ab42c09d633b0a8b4b65a297c700ae0fad47aae890f75894782f95be7e36d", - "rust-std-1.92.0-aarch64-unknown-linux-musl.tar.xz": "94b9f84f21d29825c55a27fbb6b4b9fb9296a4a841aa54d85b95c42445623363", - "rust-std-1.92.0-aarch64-unknown-none-softfloat.tar.xz": "0dc46fafaaa36f53eec49e14a69e1d6d9ac6f0b9624a01081ad311d8139a2be0", - "rust-std-1.92.0-aarch64-unknown-none.tar.xz": "ab6a2edab924739fc2c86e9f8fd8068b379e881a6261a177d66608d3ea4cacb1", - "rust-std-1.92.0-aarch64-unknown-uefi.tar.xz": "f98001222bf23743598382c232b08d3137035b53645a420a1425288d501808af", - "rust-std-1.92.0-arm-linux-androideabi.tar.xz": "d41ec7255556b605dda04201a23e4445b5b86bc6786c703f91eb985bddc9f4ca", - "rust-std-1.92.0-arm-unknown-linux-gnueabi.tar.xz": "35478e20f8cc13912b31f2905b313a2820ddae564b363a66ab7a5da39d12787f", - "rust-std-1.92.0-arm-unknown-linux-gnueabihf.tar.xz": "836ada282b65c57d71f9b7e6fb490832410c954aac905c5437fb0bf851b53c83", - "rust-std-1.92.0-arm-unknown-linux-musleabi.tar.xz": "91d3c5fdbda9ba2e778bb638e3a5d060f3021bbc7a60edf22b0035be4e611b30", - "rust-std-1.92.0-arm-unknown-linux-musleabihf.tar.xz": "20411e3858308add0dadde9ce2059d82cdd6c5339881fa93ac995e733df0f832", - "rust-std-1.92.0-armv7-linux-androideabi.tar.xz": "06ac2f08dcf5c480e7767c0541c9bd7460754ec3a369a4b7097b328eca28fab9", - "rust-std-1.92.0-armv7-unknown-linux-gnueabi.tar.xz": "3d5e3fb14441ea8e6fc6307cbd89efd69be42143ff7f2b8dfb19818ddca993c0", - "rust-std-1.92.0-armv7-unknown-linux-gnueabihf.tar.xz": "e3ac810db43067d8af9e17679d47534e871f1daad8cd0762e67da935681e9e19", - "rust-std-1.92.0-armv7-unknown-linux-musleabi.tar.xz": "89cbf7db934d543754b446a52398081ec40ee6b98ed5bca93ac1dbd5faf48c16", - "rust-std-1.92.0-armv7-unknown-linux-musleabihf.tar.xz": "c7c08a389bd351226fd52266bfe2d2c444597e1bbb5d0307da44bdfa4df62c99", - "rust-std-1.92.0-i686-linux-android.tar.xz": "e45832b005556f65c6d26f05f34bd4ab5ceea8d9b9fefc4175d52b0780ca89db", - "rust-std-1.92.0-i686-pc-windows-gnu.tar.xz": "b3eefe10d4aed3ccbeaff3ae9cd2e0e0a36894c0635d0e69af1c9a698679d75f", - "rust-std-1.92.0-i686-pc-windows-gnullvm.tar.xz": "e6d709f85dea51d81f2f1a030845b732b9f7761d95d93c57c7276b0a451c2993", - "rust-std-1.92.0-i686-pc-windows-msvc.tar.xz": "e5671b276047647e994a7cab99c90ee70c46787572fbe4e266a13c6edb84d5ce", - "rust-std-1.92.0-i686-unknown-freebsd.tar.xz": "e008a0506ec4d5eff30abdf376c7933e235670bd6c5e1131c52bcda097a21116", - "rust-std-1.92.0-i686-unknown-linux-gnu.tar.xz": "abc840631a4462a4c8ec61341110ff653ab2ef86ef3b10f84489d00cc8a9310d", - "rust-std-1.92.0-i686-unknown-linux-musl.tar.xz": "c796874b1343721f575203fa179dc258e09ac45cd95dd6c35c4d5979a3870494", - "rust-std-1.92.0-i686-unknown-uefi.tar.xz": "90da7759e28e62fb82454d4eb4e02ac14320b8da3372e02f9ca4f47f7afbfe40", - "rust-std-1.92.0-powerpc-unknown-linux-gnu.tar.xz": "c3e809a324b00eb53096c58df38645bb496c6560de334dfe04ed0b77c0605aaa", - "rust-std-1.92.0-powerpc64-unknown-linux-gnu.tar.xz": "2ce706afa4a46b6773340854de877fc63618a40e351298a4e3da8eb482619863", - "rust-std-1.92.0-powerpc64le-unknown-linux-gnu.tar.xz": "eba59766c2d9805c0a1fc82fd723acbb36569e1bec1088c037bba84d965f70ba", - "rust-std-1.92.0-powerpc64le-unknown-linux-musl.tar.xz": "8b515e18b6ac8f8d37ea3cabe644b7f571984333c3b4192b7f5877e79eae7893", - "rust-std-1.92.0-riscv32imc-unknown-none-elf.tar.xz": "e1c6968ce25ab78f2c5b5460691763a2d39cb71b01d6d54c208d4a8b735b0584", - "rust-std-1.92.0-riscv64gc-unknown-linux-gnu.tar.xz": "8ee20dcf3b1063fa6069b3ce85e1fcf42794dfa783263314865cb53fff42d9e4", - "rust-std-1.92.0-riscv64gc-unknown-linux-musl.tar.xz": "f20d822309900fd6c7230688694baf91e900154e44e6247feca49b7b7a203a57", - "rust-std-1.92.0-riscv64gc-unknown-none-elf.tar.xz": "7eacf6a98786d58ef912643fd5aedc35111712e96a47b7e5d3ccddcdf9fb166d", - "rust-std-1.92.0-s390x-unknown-linux-gnu.tar.xz": "ebf944dc95015498d322504a54e4f9cdb28590f7790aa3a9eb86d6cf4b6c93ff", - "rust-std-1.92.0-thumbv6m-none-eabi.tar.xz": "f55de77126b60e1da38f8a5cdd127db872155ce9fbb53d4459502dd47b1dd476", - "rust-std-1.92.0-thumbv7em-none-eabi.tar.xz": "fdee017dcebfa8675220c20ca65b945b6eaee744d5d19b92cb0ca5480dd19ba6", - "rust-std-1.92.0-thumbv7em-none-eabihf.tar.xz": "6bfd083ce4917440fb4b04ca39ab4dadc9f40d9dc2775056899857bfa8911fd0", - "rust-std-1.92.0-thumbv7m-none-eabi.tar.xz": "7dc4c92e97db5ce1ddd9fbb7fbb1ad2d367f1c2e20d296a6d6427480c900c315", - "rust-std-1.92.0-thumbv8m.main-none-eabi.tar.xz": "61a6a80d03ebdb80ab06a044d4ec60e3c2bd8dc4d6011e920cf41592df4f0646", - "rust-std-1.92.0-thumbv8m.main-none-eabihf.tar.xz": "24c2f65371a2a5c6a40b51ae0e276ea9afff0479255f180f5e1549a0a4d58fb3", - "rust-std-1.92.0-wasm32-unknown-emscripten.tar.xz": "59b7adf18c0cc416a005fad7f704203b965905eb1c0ed9c556daa3c14048b9f4", - "rust-std-1.92.0-wasm32-unknown-unknown.tar.xz": "6c73f053ccd6adc886f802270ba960fd854e5e1111e4b5cfb875f1fdcfc0eb60", - "rust-std-1.92.0-wasm32-wasip1-threads.tar.xz": "feea056dd657a26560dfddfe4b53daeef3ded83c140b453b6dbdebaabd2a664c", - "rust-std-1.92.0-wasm32-wasip1.tar.xz": "8107dc35f0b6b744d998e766351419b4e0d27cddd6456c461337753a25f29fcd", - "rust-std-1.92.0-wasm32-wasip2.tar.xz": "e69ce601b6b24eea08b0d9e1fb0d9bf2a4188b3109353b272be4d995f687c943", - "rust-std-1.92.0-x86_64-apple-darwin.tar.xz": "6ce143bf9e83c71e200f4180e8774ab22c8c8c2351c88484b13ff13be82c8d57", - "rust-std-1.92.0-x86_64-apple-ios-macabi.tar.xz": "6a292d774653f2deaac743060ec68bfd5d7ff01d9b364e1a7d1e57a679650b47", - "rust-std-1.92.0-x86_64-apple-ios.tar.xz": "b6e38e5f8c9e6fb294681a7951301856b8f9424af4589e14493c0c939338814c", - "rust-std-1.92.0-x86_64-linux-android.tar.xz": "ffd39429435ff2f0763b940dd6afb4b9ccb1ed443eeef4fff9f1e9b3c5730026", - "rust-std-1.92.0-x86_64-pc-windows-gnu.tar.xz": "d4043304ef0e4792fb79a1153cbeca41308aac37cb1af3aa6bc3f0bb6d2276e1", - "rust-std-1.92.0-x86_64-pc-windows-gnullvm.tar.xz": "6169605b3073a7c2d6960bc1c74cb9cd6b65f45ee34e7ede02368e07ce4564cf", - "rust-std-1.92.0-x86_64-pc-windows-msvc.tar.xz": "b4e53a9c9b96a1a0618364791b7728a1c8978101ec6d1ee78fe930e1ef061994", - "rust-std-1.92.0-x86_64-unknown-freebsd.tar.xz": "151929a4255175d14b2cbcb69ef46d9217add23be268b9cd1446f8eab16616f2", - "rust-std-1.92.0-x86_64-unknown-fuchsia.tar.xz": "51ec26391d166e2f190a329919e3c7bd793861f02284aba461a086268725d9f1", - "rust-std-1.92.0-x86_64-unknown-linux-gnu.tar.xz": "5f106805ed86ebf8df287039e53a45cf974391ef4d088c2760776b05b8e48b5d", - "rust-std-1.92.0-x86_64-unknown-linux-musl.tar.xz": "11f0b7efccdb5e7972e3c0fc23693a487abc28b624675c08161d055a016d527e", - "rust-std-1.92.0-x86_64-unknown-netbsd.tar.xz": "db6a8d3a091551701b12e40cc58d4a541adfb63f250074aae90d250329beb8de", - "rust-std-1.92.0-x86_64-unknown-none.tar.xz": "1d8420ab8eb241a35e38b76470277c722bd5a7aa4ac0c7a565ad6f30b37cb852", - "rust-std-1.92.0-x86_64-unknown-uefi.tar.xz": "1b849250cf095269f3a2c7bc2087a919386da7da28e80dc289e6268bc705142d", - "rustc-1.92.0-aarch64-apple-darwin.tar.xz": "15dee753c9217dff4cf45d734b29dc13ce6017d8a55fe34eed75022b39a63ff0", - "rustc-1.92.0-aarch64-pc-windows-msvc.tar.xz": "07ba5606143de3bc916455714f67732fca05805591396a8190d5e88b8863eea3", - "rustc-1.92.0-aarch64-unknown-linux-gnu.tar.xz": "7c8706fad4c038b5eacab0092e15db54d2b365d5f3323ca046fe987f814e7826", - "rustc-1.92.0-x86_64-apple-darwin.tar.xz": "0facbd5d2742c8e97c53d59c9b5b81db6088cfc285d9ecb99523a50d6765fc5c", - "rustc-1.92.0-x86_64-pc-windows-msvc.tar.xz": "6aec5aba5384ce1041538e71ebebf671d82e29f4fe697bc7844626d94b0dbfe6", - "rustc-1.92.0-x86_64-unknown-linux-gnu.tar.xz": "78b2dd9c6b1fcd2621fa81c611cf5e2d6950690775038b585c64f364422886e0", - "rustfmt-1.92.0-aarch64-apple-darwin.tar.xz": "5d8ea865a7999dc9141603be8a9352745bf8440da051eb1c43f0fcaaf6845441", - "rustfmt-1.92.0-aarch64-pc-windows-msvc.tar.xz": "e626e08df361c5a9127e0e4b5d73f87fd17ed30cc471887ebc033bebe97c0ff2", - "rustfmt-1.92.0-aarch64-unknown-linux-gnu.tar.xz": "1dce37aea2a7cb801f1756ffc531d7140428315a3d2c2f836272547eb7b9dacd", - "rustfmt-1.92.0-x86_64-apple-darwin.tar.xz": "e038bda323ed7f4d417efc5be44c4245d74b8394f9f8393b9d464d662c3a9499", - "rustfmt-1.92.0-x86_64-pc-windows-msvc.tar.xz": "3761b2e8a18d672b2804b0d1864dc122b7bc54343d6a109135c7843de7fa2ace", - "rustfmt-1.92.0-x86_64-unknown-linux-gnu.tar.xz": "38951ee55f21e9170236fc98c8ba373ae4338d863087c6b0a5fa8c4e797d52c4" } } } diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 9e78edaa..14e4867e 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -8,14 +8,12 @@ package(default_visibility = ["//visibility:public"]) load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_cc//cc:cc_import.bzl", "cc_import") -load("@zml//bazel:patchelf.bzl", "patchelf") """ ARCH = "linux-x86_64" CUDA_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cuda/redist/" CUDA_VERSION = "13.1.1" -CUDA_VARIANT = "cuda13.1" CUDA_REDIST_JSON_SHA256 = "97cf605ccc4751825b1865f4af571c9b50dd29ffd13e9a38b296a9ecb1f0d422" CUDNN_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cudnn/redist/" @@ -26,14 +24,6 @@ NVSHMEM_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/nvshmem/r NVSHMEM_VERSION = "3.5.19" NVSHMEM_REDIST_JSON_SHA256 = "6dced4193eb728542504b346cfb768da6e3de2abca0cded95fda3a69729994d2" -CUDA_COMPAT_FILES = [ - "libcuda.so.1", - "libcudadebugger.so.1", - "libnvidia-nvvm.so.4", - "libnvidia-nvvm70.so.4", - "libnvidia-ptxjitcompiler.so.1", -] - _UBUNTU_PACKAGES = { "zlib1g": packages.filegroup(name = "zlib1g", srcs = ["lib/x86_64-linux-gnu/libz.so.1"]), } @@ -62,27 +52,6 @@ CUDA_PACKAGES = { srcs = ["lib/libnvtx3interop.so"], ), ]), - "cuda_compat": "\n".join([ - packages.patchelf( - name = "{}.patchelf".format(file), - set_rpath = "$ORIGIN", - src = "compat/{}".format(file), - soname = file, - ) - for file in CUDA_COMPAT_FILES - ] + [ - packages.filegroup( - name = "cuda_compat", - srcs = [ - "compat/libnvidia-gpucomp.so.590.48.01", - "compat/libnvidia-pkcs11-openssl3.so.590.48.01", - "compat/libnvidia-tileiras.so.590.48.01", - ] + [ - ":{}.patchelf".format(file) - for file in CUDA_COMPAT_FILES - ], - ), - ]), "libcufft": packages.filegroup( name = "libcufft", srcs = ["lib/libcufft.so.12"], @@ -219,7 +188,6 @@ def _cuda_impl(mctx): arch_data = pkg_data.get(ARCH) if not arch_data: continue - arch_data = arch_data.get(CUDA_VARIANT, None) or arch_data http_archive( name = pkg, build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, diff --git a/platforms/cuda/libpjrt_cuda.BUILD.bazel b/platforms/cuda/libpjrt_cuda.BUILD.bazel index a4a70699..372874a9 100644 --- a/platforms/cuda/libpjrt_cuda.BUILD.bazel +++ b/platforms/cuda/libpjrt_cuda.BUILD.bazel @@ -1,7 +1,6 @@ load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@zml//bazel:patchelf.bzl", "patchelf") load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@zml//platforms/cuda:cuda.bzl", "CUDA_COMPAT_FILES") patchelf( name = "libpjrt_cuda_so", @@ -25,7 +24,6 @@ copy_to_directory( ":libpjrt_cuda_so", "@cuda_cudart", "@cuda_cupti", - "@cuda_compat", "@cuda_nvcc", "@libnvvm", "@cuda_nvrtc", @@ -42,16 +40,11 @@ copy_to_directory( "@zml//platforms/cuda:zmlxcuda", ], replace_prefixes = { - "compat": "lib", - "libcuda": "lib", "nvidia/nccl/lib": "lib", "nvvm/lib64": "lib", "libpjrt_cuda_so": "lib", "lib/x86_64-linux-gnu": "lib", "platforms/cuda": "lib", - } | { - "{}.patchelf".format(file): "lib" - for file in CUDA_COMPAT_FILES }, add_directory_to_runfiles = False, include_external_repositories = ["**"], diff --git a/platforms/cuda/zmlxcuda.zig b/platforms/cuda/zmlxcuda.zig index 922fffaa..5834cb6b 100644 --- a/platforms/cuda/zmlxcuda.zig +++ b/platforms/cuda/zmlxcuda.zig @@ -13,9 +13,6 @@ pub export fn zmlxcuda_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque .{ "libcusolver.so", "libcusolver.so.12" }, .{ "libcusparse.so", "libcusparse.so.12" }, .{ "libnccl.so", "libnccl.so.2" }, - - // CUDA Compat - .{ "libcuda.so.1", "libcuda.so.1" }, }); var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; diff --git a/zml/tokenizer/hftokenizers/BUILD.bazel b/zml/tokenizer/hftokenizers/BUILD.bazel index ebd30da0..a07d855b 100644 --- a/zml/tokenizer/hftokenizers/BUILD.bazel +++ b/zml/tokenizer/hftokenizers/BUILD.bazel @@ -1,5 +1,5 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_rs//rs:rust_static_library.bzl", "rust_static_library") +load("@rules_rust//rust:defs.bzl", "rust_static_library") load("@rules_zig//zig:defs.bzl", "zig_library") rust_static_library( From de284c05e0b27817239814509350ba952ff5ace2 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Mon, 23 Feb 2026 14:11:38 +0100 Subject: [PATCH 326/881] bazel: Use target_platforms for neuron instead of pip args to force x86 (#391) Co-authored-by: Benjamin Rabier --- MODULE.bazel | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index b0d54b40..0faba184 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -68,19 +68,10 @@ use_repo(python, "python_versions") pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") pip.parse( download_only = True, - extra_pip_args = [ - "--abi=cp312", - "--implementation=cp", - "--python-version=312", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64", - "--platform=manylinux_2_27_x86_64", - "--platform=manylinux_2_28_x86_64", - "--platform=manylinux_2_29_x86_64", - ], hub_name = "neuron_py_deps", python_version = "3.12", requirements_lock = "//platforms/neuron:requirements.lock.txt", + target_platforms = ["linux_x86_64"], ) use_repo(pip, "neuron_py_deps") From 8f85fc1c062ebeb9fc99213ab190a2847f4020ee Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 24 Feb 2026 09:52:19 +0100 Subject: [PATCH 327/881] bazel: fix zls completion for zig_test (#389) Co-authored-by: Benjamin Rabier --- third_party/zls/zls_completion.bzl | 7 ++++++- third_party/zls/zls_write_build_config.bzl | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/third_party/zls/zls_completion.bzl b/third_party/zls/zls_completion.bzl index 9028c386..3b84dbad 100644 --- a/third_party/zls/zls_completion.bzl +++ b/third_party/zls/zls_completion.bzl @@ -6,7 +6,7 @@ load("@rules_zig//zig:defs.bzl", "zig_binary") load(":zls_write_build_config.bzl", "zls_write_build_config") load(":zls_write_runner_zig_src.bzl", "zls_write_runner_zig_src") -def zls_completion(name, deps, **kwargs): +def zls_completion(name, deps, testonly = False, **kwargs): """Entry point for ZLS completion. Args: @@ -22,6 +22,7 @@ def zls_completion(name, deps, **kwargs): name = build_config, out = name + ".build_config.json", deps = deps, + testonly = testonly, ) # Create a target that will be invoked by ZLS using our customer build_runner. @@ -36,6 +37,7 @@ def zls_completion(name, deps, **kwargs): "$(location :{})".format(build_config), ], visibility = ["//visibility:private"], + testonly = testonly, ) # Generate the Zig build runner that will be used by ZLS to query the build config. @@ -46,6 +48,7 @@ def zls_completion(name, deps, **kwargs): "@@__TARGET__@@": str(utils.to_label(build_config_printer)), }, template = Label(":zls_build_runner.zig"), + testonly = testonly, ) # Generate the Zig source file for the ZLS runner binary which embeds the @@ -54,6 +57,7 @@ def zls_completion(name, deps, **kwargs): name = name + ".runner", out = name + ".runner.zig", build_runner = ":" + name + ".build_runner.zig", + testonly = testonly, ) zig_binary( @@ -67,5 +71,6 @@ def zls_completion(name, deps, **kwargs): deps = [ "@rules_zig//zig/runfiles", ], + testonly = testonly, **kwargs ) diff --git a/third_party/zls/zls_write_build_config.bzl b/third_party/zls/zls_write_build_config.bzl index fdd20322..d7260de2 100644 --- a/third_party/zls/zls_write_build_config.bzl +++ b/third_party/zls/zls_write_build_config.bzl @@ -18,7 +18,7 @@ def _zls_construct_zig_module_info_impl(target, ctx): if ZigModuleInfo in target: return [] - if ctx.rule.kind not in ("zig_binary", "zig_static_library", "zig_shared_library"): + if ctx.rule.kind not in ("zig_binary", "zig_static_library", "zig_shared_library", "zig_test"): return [] cdeps = [] From 569bd1dfd78f9c8e18d4c4ca369c5437ee8d21cb Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 24 Feb 2026 09:52:40 +0100 Subject: [PATCH 328/881] examples/transfer-benchmark: Use VFS & full model for transfer-benchmark (#390) Co-authored-by: Benjamin Rabier --- BUILD.bazel | 1 + examples/transfer-benchmark/BUILD.bazel | 3 +- examples/transfer-benchmark/main.zig | 178 +++++++++++++++++++----- zml/io/vfs/hf.zig | 3 +- zml/safetensors.zig | 6 +- 5 files changed, 150 insertions(+), 41 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 57873f06..b3dd9f68 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -5,6 +5,7 @@ zls_completion( visibility = ["//visibility:public"], deps = [ "//examples/lfm", + "//examples/transfer-benchmark", "//examples/llama", "//examples/mnist", "//stdx", diff --git a/examples/transfer-benchmark/BUILD.bazel b/examples/transfer-benchmark/BUILD.bazel index a3931095..20082520 100644 --- a/examples/transfer-benchmark/BUILD.bazel +++ b/examples/transfer-benchmark/BUILD.bazel @@ -4,7 +4,8 @@ load("@rules_zig//zig:defs.bzl", "zig_binary") zig_binary( name = "transfer-benchmark", main = "main.zig", - deps = ["//zml"], + deps = ["//zml", "//bazel"], + visibility = ["//visibility:public"], ) build_test( diff --git a/examples/transfer-benchmark/main.zig b/examples/transfer-benchmark/main.zig index d1d9fd99..52f5633b 100644 --- a/examples/transfer-benchmark/main.zig +++ b/examples/transfer-benchmark/main.zig @@ -4,40 +4,76 @@ const log = std.log; const zml = @import("zml"); const stdx = zml.stdx; -pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { - return a.dot(b, .k); -} - const Args = struct { pub const help = - \\ benchmark --model=./model --tensor="model.embed_tokens.weight" + \\ benchmark --model=./model ; model: []const u8, - tensor: []const u8 = "model.embed_tokens.weight", }; pub fn main(init: std.process.Init) !void { const allocator = init.gpa; - const io = init.io; + + var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); + defer vfs_file.deinit(); + + var http_client: std.http.Client = .{ .allocator = allocator, .io = init.io }; + + try http_client.initDefaultProxies(allocator, init.environ_map); + defer http_client.deinit(); + + var vfs_https: zml.io.VFS.HTTP = try .init(allocator, init.io, &http_client, .https); + defer vfs_https.deinit(); + + var hf_vfs: zml.io.VFS.HF = try .auto(allocator, init.io, &http_client, init.environ_map); + defer hf_vfs.deinit(); + + var s3_vfs: zml.io.VFS.S3 = try .auto(allocator, init.io, &http_client, init.environ_map); + defer s3_vfs.deinit(); + + var vfs: zml.io.VFS = try .init(allocator, init.io); + defer vfs.deinit(); + + try vfs.register("file", vfs_file.io()); + try vfs.register("https", vfs_https.io()); + try vfs.register("hf", hf_vfs.io()); + try vfs.register("s3", s3_vfs.io()); + + const io = vfs.io(); const platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator); - std.debug.print("{f}", .{platform.fmtVerbose()}); const args = stdx.flags.parseProcessArgs(init.minimal, Args); var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, args.model); defer registry.deinit(); - var reader = try registry.reader(io, args.tensor, &.{}); + var store = zml.io.TensorStore.fromRegistry(allocator, ®istry); + defer store.deinit(); - var buffer: zml.Buffer = undefined; - defer buffer.deinit(); + var progress = std.Progress.start(io, .{ .root_name = args.model }); + defer progress.end(); + var transfer_progress = progress.start("Transferring tensors (MiB)...", @intCast(registry.totalBytes() / zml.MiB)); + defer transfer_progress.end(); log.info("⏱️ Running benchmark...", .{}); const now: std.Io.Timestamp = .now(io, .awake); - try transfer(allocator, io, platform, reader.tensor.shape, &reader.interface, &buffer, 4, 32 * zml.MiB); + var total_bytes: usize = 0; + const tensor_count = try transferAll( + allocator, + io, + platform, + .{ + .parallelism = 16, + .store = &store, + .dma_chunks = 4, + .dma_chunk_size = 32 * zml.MiB, + .progress = &transfer_progress, + .total_bytes = &total_bytes, + }, + ); const elapsed = now.untilNow(io, .awake); const elapsed_ns = elapsed.toNanoseconds(); @@ -45,40 +81,110 @@ pub fn main(init: std.process.Init) !void { log.info("✅ Benchmark done!", .{}); - const size = reader.tensor.shape.byteSize(); - const flops = @as(f64, @floatFromInt(size >> 20)) / elapsed_s; - log.info("Transfer {d} MiB - Elapsed: {D} - {d:.3} MiB/s", .{ - size >> 20, + const transfer_mib = @as(f64, @floatFromInt(total_bytes)) / @as(f64, zml.MiB); + const mib_per_s = transfer_mib / elapsed_s; + log.info("Transferred {d} tensors ({d} MiB) - Elapsed: {D} - {d:.3} MiB/s", .{ + tensor_count, + total_bytes >> 20, stdx.fmt.fmtDuration(elapsed), - flops, + mib_per_s, }); } -pub fn transfer( +fn transferAll( allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, - shape: zml.Shape, - reader: *std.Io.Reader, - buffer: *zml.Buffer, - dma_chunks: usize, - dma_chunk_size: usize, -) !void { + opts: zml.io.LoadOpts, +) !usize { const first_device = platform.devices[0]; // Temporary until sharding is re-exposed const dma_allocator: zml.mem.DmaAllocator = .init(allocator, &first_device); - var buffer_pool: zml.mem.DynamicBufferPool = .init(dma_chunks, dma_chunk_size); + var buffer_pool: zml.mem.DynamicBufferPool = .init(opts.dma_chunks, opts.dma_chunk_size); defer buffer_pool.deinit(dma_allocator.allocator()); - var memory_writer = zml.io.MemoryWriter.init( - dma_allocator.allocator(), - io, - first_device.memory(.default), - &buffer_pool, - shape, - buffer, - ) catch unreachable; - defer memory_writer.deinit(dma_allocator.allocator()); + const keys = opts.store.registry.tensors.keys(); + const buffers = try allocator.alloc(zml.Buffer, keys.len); + const buffers_initialized = try allocator.alloc(bool, keys.len); + @memset(buffers_initialized, false); + defer { + for (buffers, buffers_initialized) |*buffer, is_initialized| { + if (is_initialized) buffer.deinit(); + } + allocator.free(buffers_initialized); + allocator.free(buffers); + } - _ = reader.streamRemaining(memory_writer.interface()) catch unreachable; - memory_writer.interface().flush() catch unreachable; + var ctx: TransferCtx = .{ + .dma_allocator = dma_allocator.allocator(), + .pinned_buffer_pool = &buffer_pool, + .io = io, + .memory = first_device.memory(.default), + .group = .init(opts.parallelism), + .progress = opts.progress, + .store = opts.store, + .buffers = buffers, + .buffers_initialized = buffers_initialized, + }; + + for (keys, 0..) |key, i| { + ctx.group.concurrent(io, transferOne, .{ i, key, &ctx }) catch unreachable; + } + + ctx.group.await(io) catch unreachable; + + if (opts.total_bytes) |total_bytes_ptr| { + total_bytes_ptr.* = ctx.total.load(.monotonic); + } + + return keys.len; +} + +const TransferCtx = struct { + dma_allocator: std.mem.Allocator, + pinned_buffer_pool: *zml.mem.DynamicBufferPool, + io: std.Io, + memory: *const zml.Memory, + group: stdx.Io.LimitedGroup, + total: std.atomic.Value(usize) = .init(0), + progress: ?*std.Progress.Node, + store: *const zml.io.TensorStore, + buffers: []zml.Buffer, + buffers_initialized: []bool, +}; + +fn transferOne(i: usize, key: []const u8, ctx: *TransferCtx) !void { + var reader = ctx.store.getReader(key, ctx.io, &.{}) catch unreachable; + defer reader.deinit(); + + var memory_writer = zml.io.MemoryWriter.init( + ctx.dma_allocator, + ctx.io, + ctx.memory, + ctx.pinned_buffer_pool, + reader.tensor.shape, + &ctx.buffers[i], + ) catch unreachable; + defer memory_writer.deinit(ctx.dma_allocator); + ctx.buffers_initialized[i] = true; + + const scale = zml.MiB; + const total = if (ctx.progress) |progress| blk: { + const child_total = @max(@as(u64, 1), reader.tensor.shape.byteSize() / scale); + var child = progress.start(reader.tensor.name, child_total); + defer child.end(); + + var progress_writer: zml.io.ProgressWriter = .init(memory_writer.interface(), &child, .{ .scale = scale }); + const written = reader.interface.streamRemaining(&progress_writer.interface) catch unreachable; + progress_writer.interface.flush() catch unreachable; + break :blk written; + } else blk: { + const written = reader.interface.streamRemaining(memory_writer.interface()) catch unreachable; + memory_writer.interface().flush() catch unreachable; + break :blk written; + }; + + const previous = ctx.total.fetchAdd(total, .monotonic); + if (ctx.progress) |progress| { + progress.setCompletedItems((previous + total) / scale); + } } diff --git a/zml/io/vfs/hf.zig b/zml/io/vfs/hf.zig index bc2a2bd1..ca9a0bc8 100644 --- a/zml/io/vfs/hf.zig +++ b/zml/io/vfs/hf.zig @@ -125,7 +125,7 @@ pub const HF = struct { pub fn auto(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, environ_map: *std.process.Environ.Map) !HF { const hf_token = if (environ_map.get("HF_TOKEN")) |token| blk: { - break :blk token; + break :blk try allocator.dupe(u8, token); } else blk: { const home_path = environ_map.get("HOME") orelse break :blk null; @@ -141,6 +141,7 @@ pub const HF = struct { break :blk token; }; + defer if (hf_token) |tk| allocator.free(tk); if (hf_token == null) log.warn("No Hugging Face authentication token found in environment or home config; proceeding without authentication.", .{}); diff --git a/zml/safetensors.zig b/zml/safetensors.zig index cd7677c7..bb14c334 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -350,13 +350,13 @@ pub const TensorRegistry = struct { io: std.Io, tensor_name: []const u8, buffer: []u8, - ) TensorReader.Error!TensorReader { + ) !TensorReader { const tensor = self.tensors.get(tensor_name) orelse { log.err("Tensor {s} not found in registry", .{tensor_name}); - return TensorReader.Error.TensorNotFound; + return error.TensorNotFound; }; - return try .init(io, tensor, buffer); + return try .init(io, tensor, buffer, .{}); } pub fn iterator(self: *TensorRegistry) Tensors.Iterator { From 6be028f873dba5683f131f549c153e8081c9db39 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 25 Feb 2026 12:42:26 +0000 Subject: [PATCH 329/881] runtimes: update tpu plugin and fix mem --- platforms/tpu/tpu.bzl | 4 ++-- zml/platform.zig | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/platforms/tpu/tpu.bzl b/platforms/tpu/tpu.bzl index fe111015..ef187b37 100644 --- a/platforms/tpu/tpu.bzl +++ b/platforms/tpu/tpu.bzl @@ -4,9 +4,9 @@ def _tpu_impl(mctx): # https://storage.googleapis.com/jax-releases/libtpu_releases.html http_archive( name = "libpjrt_tpu", - url = "https://storage.googleapis.com/libtpu-nightly-releases/wheels/libtpu/libtpu-0.0.33.dev20251218+nightly-cp314-cp314t-manylinux_2_31_x86_64.whl", + url = "https://files.pythonhosted.org/packages/5e/ea/6271a8906d5509349dde55da1f516ac232e73b3cd8e1fba095f4132142bf/libtpu-0.0.36-cp314-cp314t-manylinux_2_31_x86_64.whl", type = "zip", - sha256 = "b7d9855c74a7fc56d11c4c7a2c55d5395c80594d43caec1549c69d5f908583cf", + sha256 = "6d0e6a2ec26b851f5a00c74933738b2b185af47c4eacbd161e23954b1d911ae4", build_file = "libpjrt_tpu.BUILD.bazel", ) return mctx.extension_metadata( diff --git a/zml/platform.zig b/zml/platform.zig index daa2f9fe..8b46ae90 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -92,7 +92,7 @@ pub const Memory = struct { pub fn isOfKind(self: Memory, kind_: Kind) bool { switch (self.platform.target) { - .cuda, .rocm => { + .cuda, .rocm, .tpu => { const zml_kind: Memory.Kind = switch (self.kind().len) { "device".len => .device, "pinned_host".len => .host_pinned, @@ -101,7 +101,7 @@ pub const Memory = struct { }; return zml_kind == kind_; }, - .cpu, .tpu, .neuron => return true, + .cpu,.neuron => return true, } } From e258ecb1473109f279c0f5b58190b1fac1b00770 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Mar 2026 17:40:23 +0100 Subject: [PATCH 330/881] Add devenv files It defines my nix environment for this project, and it's a lot easier to manage by committing it. --- .gitignore | 11 +++++ devenv.lock | 123 ++++++++++++++++++++++++++++++++++++++++++++++++++++ devenv.nix | 19 ++++++++ devenv.yaml | 15 +++++++ 4 files changed, 168 insertions(+) create mode 100644 devenv.lock create mode 100644 devenv.nix create mode 100644 devenv.yaml diff --git a/.gitignore b/.gitignore index a4b52b95..0ec34365 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,14 @@ venv # Editor specific ## Neovim .nvim.local.lua + +# Devenv +.devenv* +devenv.local.nix +devenv.local.yaml + +# direnv +.direnv + +# pre-commit +.pre-commit-config.yaml diff --git a/devenv.lock b/devenv.lock new file mode 100644 index 00000000..78eabc08 --- /dev/null +++ b/devenv.lock @@ -0,0 +1,123 @@ +{ + "nodes": { + "devenv": { + "locked": { + "dir": "src/modules", + "lastModified": 1772143911, + "owner": "cachix", + "repo": "devenv", + "rev": "83c4b5682b84fe04c049970792f38d5879d3c0f1", + "type": "github" + }, + "original": { + "dir": "src/modules", + "owner": "cachix", + "repo": "devenv", + "type": "github" + } + }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1767039857, + "owner": "NixOS", + "repo": "flake-compat", + "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "flake-compat", + "type": "github" + } + }, + "git-hooks": { + "inputs": { + "flake-compat": "flake-compat", + "gitignore": "gitignore", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1772024342, + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "6e34e97ed9788b17796ee43ccdbaf871a5c2b476", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, + "gitignore": { + "inputs": { + "nixpkgs": [ + "git-hooks", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1762808025, + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "cb5e3fdca1de58ccbc3ef53de65bd372b48f567c", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, + "nixpkgs": { + "inputs": { + "nixpkgs-src": "nixpkgs-src" + }, + "locked": { + "lastModified": 1770434727, + "owner": "cachix", + "repo": "devenv-nixpkgs", + "rev": "8430f16a39c27bdeef236f1eeb56f0b51b33d348", + "type": "github" + }, + "original": { + "owner": "cachix", + "ref": "rolling", + "repo": "devenv-nixpkgs", + "type": "github" + } + }, + "nixpkgs-src": { + "flake": false, + "locked": { + "lastModified": 1769922788, + "narHash": "sha256-H3AfG4ObMDTkTJYkd8cz1/RbY9LatN5Mk4UF48VuSXc=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "207d15f1a6603226e1e223dc79ac29c7846da32e", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "devenv": "devenv", + "git-hooks": "git-hooks", + "nixpkgs": "nixpkgs", + "pre-commit-hooks": [ + "git-hooks" + ] + } + } + }, + "root": "root", + "version": 7 +} diff --git a/devenv.nix b/devenv.nix new file mode 100644 index 00000000..6b77a707 --- /dev/null +++ b/devenv.nix @@ -0,0 +1,19 @@ +{ + pkgs, + config, + ... +}: { + packages = with pkgs; [ + bazelisk + python3 + uv + ]; + env.ZLS_CMD = "${config.devenv.root}/tools/zls.sh"; + scripts.bazel.exec = ''bazelisk "$@"''; + + enterShell = pkgs.lib.optionalString pkgs.stdenv.isDarwin '' + # Use the "real" XCode rather than Nixpkg's one. This does assume you have it installed. + export DEVELOPER_DIR="/Applications/Xcode.app/Contents/Developer" + export SDKROOT="/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk"; + ''; +} diff --git a/devenv.yaml b/devenv.yaml new file mode 100644 index 00000000..116a2adb --- /dev/null +++ b/devenv.yaml @@ -0,0 +1,15 @@ +# yaml-language-server: $schema=https://devenv.sh/devenv.schema.json +inputs: + nixpkgs: + url: github:cachix/devenv-nixpkgs/rolling + +# If you're using non-OSS software, you can set allowUnfree to true. +# allowUnfree: true + +# If you're willing to use a package that's vulnerable +# permittedInsecurePackages: +# - "openssl-1.1.1w" + +# If you have more than one devenv you can merge them +#imports: +# - ./backend From 1df41a8ee1b373c3d57033e1516be41b34a78e63 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 4 Mar 2026 08:05:37 +0100 Subject: [PATCH 331/881] zls: neovim fix canonicalizing paths (#399) - Fix incorrect build config path by using runfiles. - Canonicalize paths like neovim does for ZLS (doesn't work otherwise) --- third_party/zls/workspace_printer.zig | 116 ++++++++++++++++++++++++-- third_party/zls/zls_completion.bzl | 5 +- 2 files changed, 113 insertions(+), 8 deletions(-) diff --git a/third_party/zls/workspace_printer.zig b/third_party/zls/workspace_printer.zig index cbd8e130..38bdc5ce 100644 --- a/third_party/zls/workspace_printer.zig +++ b/third_party/zls/workspace_printer.zig @@ -1,12 +1,107 @@ const std = @import("std"); +const bazel_builtin = @import("bazel_builtin"); +const runfiles = @import("runfiles"); + +// Neovim canonicalizes the path it sends to ZLS, we need to do the same as ZLS will match against the config we generate. +// We canonicalize the following module paths: +// +// { +// "modules": { +// "": { +// "import_table": { +// "": "" +// }, +// }, +// }, +// } +fn canonicalizeModules(allocator: std.mem.Allocator, io: std.Io, root: *std.json.Value) !void { + if (root.* != .object) return; + const modules_value = root.object.getPtr("modules") orelse return; + if (modules_value.* != .object) return; + + var canonical_modules = std.json.ObjectMap.init(allocator); + var modules_it = modules_value.object.iterator(); + while (modules_it.next()) |module_entry| { + const module_path = module_entry.key_ptr.*; + var module_info = module_entry.value_ptr.*; + + if (module_info == .object) { + if (module_info.object.getPtr("import_table")) |imports_value| { + if (imports_value.* == .object) { + var canonical_imports = std.json.ObjectMap.init(allocator); + var imports_it = imports_value.object.iterator(); + while (imports_it.next()) |import_entry| { + const import_name = import_entry.key_ptr.*; + const import_value = import_entry.value_ptr.*; + + const canonical_import_value = if (import_value == .string) + std.json.Value{ .string = try std.Io.Dir.realPathFileAbsoluteAlloc(io, import_value.string, allocator) } + else + import_value; + + try canonical_imports.put(import_name, canonical_import_value); + } + imports_value.* = .{ .object = canonical_imports }; + } + } + } + + const canonical_module_path = try std.Io.Dir.realPathFileAbsoluteAlloc(io, module_path, allocator); + try canonical_modules.put(canonical_module_path, module_info); + } + + modules_value.* = .{ .object = canonical_modules }; +} + +fn readBuildConfig( + allocator: std.mem.Allocator, + io: std.Io, + init: std.process.Init, + arg_path: []const u8, +) ![]u8 { + if (std.fs.path.isAbsolutePosix(arg_path)) { + return std.Io.Dir.cwd().readFileAlloc(io, arg_path, allocator, .unlimited); + } + + var r_ = try runfiles.Runfiles.create(.{ + .allocator = allocator, + .io = io, + .environ_map = init.environ_map, + .argv = init.minimal.args, + }) orelse return error.RunfilesNotFound; + + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const r = r_.withSourceRepo(bazel_builtin.current_repository); + + if (try r.rlocation(arg_path, &path_buf)) |config_path| { + return std.Io.Dir.cwd().readFileAlloc(io, config_path, allocator, .unlimited); + } + + const stripped = if (std.mem.startsWith(u8, arg_path, "./")) arg_path[2..] else arg_path; + if (try r.rlocation(stripped, &path_buf)) |config_path| { + return std.Io.Dir.cwd().readFileAlloc(io, config_path, allocator, .unlimited); + } + + // Convenience for testing. + const output = std.Io.Dir.cwd().readFileAlloc(io, arg_path, allocator, .unlimited) catch |err| switch (err) { + error.FileNotFound => null, + else => return err, + }; + if (output) |content| { + return content; + } + + return error.BuildConfigNotFound; +} + pub fn main(init: std.process.Init) !void { const arena = init.arena; const io = init.io; const args = try init.minimal.args.toSlice(arena.allocator()); const build_workspace_directory = init.environ_map.get("BUILD_WORKSPACE_DIRECTORY").?; - var output_base_result = try std.process.run(arena.allocator(), init.io, .{ + const output_base_result = try std.process.run(arena.allocator(), init.io, .{ .argv = &.{ "bazel", "info", @@ -15,11 +110,11 @@ pub fn main(init: std.process.Init) !void { .cwd = .{ .path = build_workspace_directory }, }); - var output = try std.Io.Dir.cwd().readFileAlloc( - io, - args[1], + var output = try readBuildConfig( arena.allocator(), - .unlimited, + io, + init, + args[1], ); output = try std.mem.replaceOwned( u8, @@ -28,12 +123,19 @@ pub fn main(init: std.process.Init) !void { "@@__BUILD_WORKSPACE_DIRECTORY__@@", build_workspace_directory, ); + const execution_root = std.mem.trimEnd(u8, output_base_result.stdout, "\n"); output = try std.mem.replaceOwned( u8, arena.allocator(), output, "@@__BAZEL_EXECUTION_ROOT__@@", - std.mem.trimEnd(u8, output_base_result.stdout, "\n"), + execution_root, ); - try std.Io.File.stdout().writeStreamingAll(io, output); + var root = try std.json.parseFromSliceLeaky(std.json.Value, arena.allocator(), output, .{}); + try canonicalizeModules(arena.allocator(), io, &root); + + var stdout_buf: [4096]u8 = undefined; + var writer = std.Io.File.stdout().writer(io, &stdout_buf); + try std.json.Stringify.value(root, .{}, &writer.interface); + try writer.flush(); } diff --git a/third_party/zls/zls_completion.bzl b/third_party/zls/zls_completion.bzl index 3b84dbad..2e949c7e 100644 --- a/third_party/zls/zls_completion.bzl +++ b/third_party/zls/zls_completion.bzl @@ -34,7 +34,10 @@ def zls_completion(name, deps, testonly = False, **kwargs): ":{}".format(build_config), ], args = [ - "$(location :{})".format(build_config), + "$(rlocationpath :{})".format(build_config), + ], + deps = [ + "@rules_zig//zig/runfiles", ], visibility = ["//visibility:private"], testonly = testonly, From cadf6f5759acb00fe1438f7c298fed1ba71dd499 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 4 Mar 2026 10:47:32 +0100 Subject: [PATCH 332/881] zml/attention: Use divCeil form max_num_page and expose it (#400) --- zml/attention/flashattn.zig | 14 ++++++++++++++ zml/attention/paged_attention.zig | 15 +++++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index e77b27bd..bd1979fc 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -500,6 +500,13 @@ pub const paged_fa2 = struct { pub fn isPrefill(self: Options) bool { return self == .mixed; } + + pub fn maxNumPages(self: Options) usize { + return switch (self) { + .decode => |decode_options| decode_options.max_num_pages, + .mixed => |mixed_options| mixed_options.max_num_pages, + }; + } }; pub const Parameters = union(Variant) { @@ -985,6 +992,13 @@ pub const paged_fa3 = struct { pub fn isPrefill(self: Options) bool { return self == .mixed; } + + pub fn maxNumPages(self: Options) usize { + return switch (self) { + .decode => |decode_options| decode_options.max_num_pages, + .mixed => |mixed_options| mixed_options.max_num_pages, + }; + } }; pub const Parameters = union(Variant) { diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index deb63ec7..10048820 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -33,13 +33,14 @@ pub const Options = union(Backend) { cuda_fa3: flashattn.paged_fa3.Options, pub fn fromBackend(backend: Backend, is_prefill: bool, batch_size: u32, seq_len: u32, page_chunk_size: u32, max_token_count: u32, num_attention_heads: u32, head_dim: u32) Options { + const max_num_pages = @divFloor(seq_len + page_chunk_size - 1, page_chunk_size); // @divCeil return switch (backend) { .cuda_fa2 => if (is_prefill) .{ .cuda_fa2 = .{ .mixed = .{ .batch_size_decode = batch_size, .batch_size_prefill = batch_size, - .max_num_pages = seq_len / page_chunk_size, + .max_num_pages = max_num_pages, .max_seqlen_k = seq_len, .max_token_count = max_token_count, .num_heads = num_attention_heads, @@ -50,7 +51,7 @@ pub const Options = union(Backend) { .cuda_fa2 = .{ .decode = .{ .batch_size = batch_size, - .max_num_pages = seq_len / page_chunk_size, + .max_num_pages = max_num_pages, .max_seqlen_k = seq_len, .max_token_count = max_token_count, .num_heads = num_attention_heads, @@ -63,7 +64,7 @@ pub const Options = union(Backend) { .mixed = .{ .batch_size_decode = batch_size, .batch_size_prefill = batch_size, - .max_num_pages = seq_len / page_chunk_size, + .max_num_pages = max_num_pages, .max_seqlen_k = seq_len, .max_token_count = max_token_count, .num_heads = num_attention_heads, @@ -74,7 +75,7 @@ pub const Options = union(Backend) { .cuda_fa3 = .{ .decode = .{ .batch_size = batch_size, - .max_num_pages = seq_len / page_chunk_size, + .max_num_pages = max_num_pages, .max_seqlen_k = seq_len, .max_token_count = max_token_count, .num_heads = num_attention_heads, @@ -90,6 +91,12 @@ pub const Options = union(Backend) { inline else => |v| v.isPrefill(), }; } + + pub fn maxNumPages(self: Options) usize { + return switch (self) { + inline else => |v| v.maxNumPages(), + }; + } }; pub const Parameters = union(Backend) { From 8995f3a8dc73e7b0967b6d0cfee5ff20b9314658 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Thu, 5 Mar 2026 09:23:35 +0100 Subject: [PATCH 333/881] bazel: update llvm toolchain to 0.6.1 (#401) --- .bazelrc | 2 +- BUILD.bazel | 2 +- MODULE.bazel | 6 ++---- MODULE.bazel.lock | 9 +++++---- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.bazelrc b/.bazelrc index e386fc58..e0c4d194 100644 --- a/.bazelrc +++ b/.bazelrc @@ -75,7 +75,7 @@ common --@rules_zig//zig/settings:use_cc_common_link=True # Zig self hosted backend is too fragile for now common --@rules_zig//zig/settings:zigopt=-fllvm -common --@toolchains_llvm_bootstrapped//config:experimental_stub_libgcc_s=True +common --@llvm//config:experimental_stub_libgcc_s=True common --macos_minimum_os=13.0 diff --git a/BUILD.bazel b/BUILD.bazel index b3dd9f68..e4e5dd65 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -20,7 +20,7 @@ platform( "@platforms//os:linux", # ubuntu:22.04 has GNU C Library (Ubuntu GLIBC 2.35-0ubuntu3.10) # But apparently not in this docker://ubuntu:22.04 image? - "@toolchains_llvm_bootstrapped//constraints/libc:gnu.2.31", + "@llvm//constraints/libc:gnu.2.31", "@bazel_tools//tools/cpp:clang", ], exec_properties = { diff --git a/MODULE.bazel b/MODULE.bazel index 0faba184..2e5006ff 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -20,7 +20,7 @@ bazel_dep(name = "rules_rust", version = "0.68.1") bazel_dep(name = "rules_rs", version = "0.0.28") bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "rules_zig", version = "0.12.2") -bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.5.5") +bazel_dep(name = "llvm", version = "0.6.1") bazel_dep(name = "with_cfg.bzl", version = "0.14.1") bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2", dev_dependency = True) @@ -53,9 +53,7 @@ register_toolchains("@zig_toolchains//:all") ### ### LLVM Bootstrapped toolchains -register_toolchains( - "@toolchains_llvm_bootstrapped//toolchain:all", -) +register_toolchains("@llvm//toolchain:all") ### uv = use_extension("@rules_python//python/uv:uv.bzl", "uv", dev_dependency = True) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index cc67df7a..c725862f 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -51,8 +51,9 @@ "https://bcr.bazel.build/modules/bazel_features/1.33.0/MODULE.bazel": "8b8dc9d2a4c88609409c3191165bccec0e4cb044cd7a72ccbe826583303459f6", "https://bcr.bazel.build/modules/bazel_features/1.34.0/MODULE.bazel": "e8475ad7c8965542e0c7aac8af68eb48c4af904be3d614b6aa6274c092c2ea1e", "https://bcr.bazel.build/modules/bazel_features/1.38.0/MODULE.bazel": "f9b8a9c890ebd216b4049fd12a31d3c2602e3403c7af636b04fbbd7453edc9c9", - "https://bcr.bazel.build/modules/bazel_features/1.38.0/source.json": "31ba776c122b54a2885e23651642e32f087a87bf025465f8040751894b571277", "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", + "https://bcr.bazel.build/modules/bazel_features/1.42.0/MODULE.bazel": "e8ca15cb2639c5f12183db6dcb678735555d0cdd739b32a0418b6532b5e565f8", + "https://bcr.bazel.build/modules/bazel_features/1.42.0/source.json": "f2ea90e5dd0322481147114c7d5e4608c4b3fae2eeccae655e4d76a382389f6f", "https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b", "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", "https://bcr.bazel.build/modules/bazel_lib/3.0.0-beta.1/MODULE.bazel": "407729e232f611c3270005b016b437005daa7b1505826798ea584169a476e878", @@ -95,6 +96,8 @@ "https://bcr.bazel.build/modules/jsoncpp/1.9.6/MODULE.bazel": "2f8d20d3b7d54143213c4dfc3d98225c42de7d666011528dc8fe91591e2e17b0", "https://bcr.bazel.build/modules/jsoncpp/1.9.6/source.json": "a04756d367a2126c3541682864ecec52f92cdee80a35735a3cb249ce015ca000", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", + "https://bcr.bazel.build/modules/llvm/0.6.1/MODULE.bazel": "29170ab19f4e2dc9b6bbf9b3d101738e84142f63ba29a13cc33e0d40f74c79b0", + "https://bcr.bazel.build/modules/llvm/0.6.1/source.json": "2d8cdd3a5f8e1d16132dbbe97250133101e4863c0376d23273d9afd7363cc331", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/source.json": "f448c6e8963fdfa7eb831457df83ad63d3d6355018f6574fb017e8169deb43a9", "https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92", @@ -245,8 +248,6 @@ "https://bcr.bazel.build/modules/tar.bzl/0.5.5/MODULE.bazel": "4bfab9bbc7a1966c2c5f7371f5848f5e2d27c465951b4435adc9aaf00ed681da", "https://bcr.bazel.build/modules/tar.bzl/0.6.0/MODULE.bazel": "a3584b4edcfafcabd9b0ef9819808f05b372957bbdff41601429d5fd0aac2e7c", "https://bcr.bazel.build/modules/tar.bzl/0.6.0/source.json": "4a620381df075a16cb3a7ed57bd1d05f7480222394c64a20fa51bdb636fda658", - "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.5.5/MODULE.bazel": "a2f1469b2920fcc9c043c87301b7a207d320231615abecd7771d89e516fc3d0b", - "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.5.5/source.json": "8498c094703da13a859d19dd2a200152f75870fb7f22a9770df99d230a4f50d1", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", "https://bcr.bazel.build/modules/with_cfg.bzl/0.12.0/MODULE.bazel": "b573395fe63aef4299ba095173e2f62ccfee5ad9bbf7acaa95dba73af9fc2b38", "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.1/MODULE.bazel": "aa0ef3f6c67dd35db7ac38c76785dc02f48b53f34e16b20487e70f51b5324d0a", @@ -428,7 +429,7 @@ }, "@@rules_oci+//oci:extensions.bzl%oci": { "general": { - "bzlTransitiveDigest": "nCIRDeZJX85AXhvovlzr0Sx1f3ZKx8IS8xvTHWlUlqM=", + "bzlTransitiveDigest": "+eLA5tKLmYYsk3vwQ5V6L8BLyV5uoQPm6tbs+jLOFNk=", "usagesDigest": "IIGPr69MT/mSUo6fNUffrIiU31EqGjlEYm5qFmNjmUc=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, From 0535a5b056eb180b68d9ee624bfabd5ef283eab1 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 5 Mar 2026 10:52:11 +0000 Subject: [PATCH 334/881] zml/module: make MLIR init one-time and reuse global dialect registry --- zml/module.zig | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/zml/module.zig b/zml/module.zig index 31637852..fda035dd 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -19,6 +19,27 @@ const Tensor = @import("tensor.zig").Tensor; const log = std.log.scoped(.@"zml/module"); +var mlir_global_init_mutex: std.Io.Mutex = .init; +var mlir_global_registry: ?*mlir.DialectRegistry = null; + +fn mlirRegistry(io: std.Io) *mlir.DialectRegistry { + mlir_global_init_mutex.lockUncancelable(io); + defer mlir_global_init_mutex.unlock(io); + + if (mlir_global_registry == null) { + mlir.registerPasses("Transforms"); + + const mlir_registry = mlir.DialectRegistry.init() catch unreachable; + inline for (.{ "func", "stablehlo" }) |d| { + mlir.DialectHandle.fromString(d).insertDialect(mlir_registry); + } + + mlir_global_registry = mlir_registry; + } + + return mlir_global_registry.?; +} + const AttributeList = stdx.BoundedArray(mlir.NamedAttribute, 3); pub const CompilationContext = struct { @@ -60,12 +81,7 @@ pub const CompilationContext = struct { threadlocal var _current: ?*CompilationContext = null; pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform) CompilationContext { - _ = io; // autofix - mlir.registerPasses("Transforms"); - const mlir_registry = mlir.DialectRegistry.init() catch unreachable; - inline for (.{ "func", "stablehlo" }) |d| { - mlir.DialectHandle.fromString(d).insertDialect(mlir_registry); - } + const mlir_registry = mlirRegistry(io); var mlir_ctx = mlir.Context.init(.{ .registry = mlir_registry, .threading = false }) catch unreachable; mlir_ctx.loadAllAvailableDialects(); @@ -101,7 +117,6 @@ pub const CompilationContext = struct { self.mlir_pass_manager.deinit(); self.module.deinit(); self.mlir_ctx.deinit(); - self.mlir_registry.deinit(); self.arena.deinit(); } From 9f1f46f686ee80119e100a530b3cd58a3ff9cda0 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 5 Mar 2026 11:24:55 +0000 Subject: [PATCH 335/881] mlir: introduce func extension registration via C++ bridge --- mlir/BUILD.bazel | 2 ++ mlir/mlir.h | 10 ++++++++++ mlir/mlir.zig | 4 ++++ mlir/register.cc | 6 ++++++ zml/module.zig | 1 + 5 files changed, 23 insertions(+) create mode 100644 mlir/register.cc diff --git a/mlir/BUILD.bazel b/mlir/BUILD.bazel index 87982a7e..9605a38b 100644 --- a/mlir/BUILD.bazel +++ b/mlir/BUILD.bazel @@ -3,6 +3,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") cc_library( name = "c", + srcs = ["register.cc"], hdrs = ["mlir.h"], visibility = ["//mlir:__subpackages__"], deps = [ @@ -11,6 +12,7 @@ cc_library( "@llvm-project//mlir:CAPIMath", "@llvm-project//mlir:CAPISCF", "@llvm-project//mlir:CAPITransforms", + "@llvm-project//mlir:FuncExtensions", ], ) diff --git a/mlir/mlir.h b/mlir/mlir.h index 4ad1523e..d3e2719e 100644 --- a/mlir/mlir.h +++ b/mlir/mlir.h @@ -7,3 +7,13 @@ #include #include #include + +#ifdef __cplusplus +extern "C" { +#endif + +void mlirRegisterFuncExtensions(MlirDialectRegistry registry); + +#ifdef __cplusplus +} +#endif diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 6f4bef3b..e451bdb2 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -50,6 +50,10 @@ pub fn registerPasses(comptime passes: []const u8) void { @field(c, "mlirRegister" ++ passes ++ "Passes")(); } +pub fn registerFuncExtensions(registry: *DialectRegistry) void { + c.mlirRegisterFuncExtensions(registry.ptr()); +} + fn isPtrConst(comptime T: type) bool { return @typeInfo(T).pointer.is_const; } diff --git a/mlir/register.cc b/mlir/register.cc new file mode 100644 index 00000000..0b5ef061 --- /dev/null +++ b/mlir/register.cc @@ -0,0 +1,6 @@ +#include +#include + +extern "C" void mlirRegisterFuncExtensions(MlirDialectRegistry registry) { + mlir::func::registerAllExtensions(*unwrap(registry)); +} diff --git a/zml/module.zig b/zml/module.zig index fda035dd..b3d429f5 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -33,6 +33,7 @@ fn mlirRegistry(io: std.Io) *mlir.DialectRegistry { inline for (.{ "func", "stablehlo" }) |d| { mlir.DialectHandle.fromString(d).insertDialect(mlir_registry); } + mlir.registerFuncExtensions(mlir_registry); mlir_global_registry = mlir_registry; } From 8bd1676e0f9930ab5896de1188fe3582b5ec9c06 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 6 Mar 2026 20:01:46 +0000 Subject: [PATCH 336/881] Revert "bazel: Revert cuda_compat & rules_rs (#392)" This reverts commit c53d2275b41477e38f71d55b5d4409fd610e3bbb. --- MODULE.bazel | 19 +++++---------- platforms/cuda/cuda.bzl | 32 +++++++++++++++++++++++++ platforms/cuda/libpjrt_cuda.BUILD.bazel | 7 ++++++ platforms/cuda/zmlxcuda.zig | 3 +++ zml/tokenizer/hftokenizers/BUILD.bazel | 2 +- 5 files changed, 49 insertions(+), 14 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 2e5006ff..69553d54 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -16,8 +16,7 @@ bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep(name = "rules_oci", version = "2.2.7") bazel_dep(name = "rules_proto", version = "7.1.0") bazel_dep(name = "rules_python", version = "1.8.4") -bazel_dep(name = "rules_rust", version = "0.68.1") -bazel_dep(name = "rules_rs", version = "0.0.28") +bazel_dep(name = "rules_rs", version = "0.0.33") bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "rules_zig", version = "0.12.2") bazel_dep(name = "llvm", version = "0.6.1") @@ -128,20 +127,14 @@ llvm = use_extension("//third_party/xla:llvm.bzl", "llvm") llvm.configure() use_repo(llvm, "llvm-project") -rust = use_extension("@rules_rust//rust:extensions.bzl", "rust") -rust.toolchain( +rs_toolchains = use_extension("@rules_rs//rs/experimental/toolchains:module_extension.bzl", "toolchains") +rs_toolchains.toolchain( edition = "2024", - extra_target_triples = [ - "aarch64-apple-darwin", - "x86_64-apple-darwin", - "aarch64-unknown-linux-gnu", - "x86_64-unknown-linux-gnu", - ], - versions = ["1.92.0"], + version = "1.92.0", ) -use_repo(rust, "rust_toolchains") +use_repo(rs_toolchains, "default_rust_toolchains") -register_toolchains("@rust_toolchains//:all") +register_toolchains("@default_rust_toolchains//:all") crate = use_extension("@rules_rs//rs:extensions.bzl", "crate") crate.from_cargo( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 14e4867e..9e78edaa 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -8,12 +8,14 @@ package(default_visibility = ["//visibility:public"]) load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_cc//cc:cc_import.bzl", "cc_import") +load("@zml//bazel:patchelf.bzl", "patchelf") """ ARCH = "linux-x86_64" CUDA_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cuda/redist/" CUDA_VERSION = "13.1.1" +CUDA_VARIANT = "cuda13.1" CUDA_REDIST_JSON_SHA256 = "97cf605ccc4751825b1865f4af571c9b50dd29ffd13e9a38b296a9ecb1f0d422" CUDNN_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cudnn/redist/" @@ -24,6 +26,14 @@ NVSHMEM_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/nvshmem/r NVSHMEM_VERSION = "3.5.19" NVSHMEM_REDIST_JSON_SHA256 = "6dced4193eb728542504b346cfb768da6e3de2abca0cded95fda3a69729994d2" +CUDA_COMPAT_FILES = [ + "libcuda.so.1", + "libcudadebugger.so.1", + "libnvidia-nvvm.so.4", + "libnvidia-nvvm70.so.4", + "libnvidia-ptxjitcompiler.so.1", +] + _UBUNTU_PACKAGES = { "zlib1g": packages.filegroup(name = "zlib1g", srcs = ["lib/x86_64-linux-gnu/libz.so.1"]), } @@ -52,6 +62,27 @@ CUDA_PACKAGES = { srcs = ["lib/libnvtx3interop.so"], ), ]), + "cuda_compat": "\n".join([ + packages.patchelf( + name = "{}.patchelf".format(file), + set_rpath = "$ORIGIN", + src = "compat/{}".format(file), + soname = file, + ) + for file in CUDA_COMPAT_FILES + ] + [ + packages.filegroup( + name = "cuda_compat", + srcs = [ + "compat/libnvidia-gpucomp.so.590.48.01", + "compat/libnvidia-pkcs11-openssl3.so.590.48.01", + "compat/libnvidia-tileiras.so.590.48.01", + ] + [ + ":{}.patchelf".format(file) + for file in CUDA_COMPAT_FILES + ], + ), + ]), "libcufft": packages.filegroup( name = "libcufft", srcs = ["lib/libcufft.so.12"], @@ -188,6 +219,7 @@ def _cuda_impl(mctx): arch_data = pkg_data.get(ARCH) if not arch_data: continue + arch_data = arch_data.get(CUDA_VARIANT, None) or arch_data http_archive( name = pkg, build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, diff --git a/platforms/cuda/libpjrt_cuda.BUILD.bazel b/platforms/cuda/libpjrt_cuda.BUILD.bazel index 372874a9..a4a70699 100644 --- a/platforms/cuda/libpjrt_cuda.BUILD.bazel +++ b/platforms/cuda/libpjrt_cuda.BUILD.bazel @@ -1,6 +1,7 @@ load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@zml//bazel:patchelf.bzl", "patchelf") load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@zml//platforms/cuda:cuda.bzl", "CUDA_COMPAT_FILES") patchelf( name = "libpjrt_cuda_so", @@ -24,6 +25,7 @@ copy_to_directory( ":libpjrt_cuda_so", "@cuda_cudart", "@cuda_cupti", + "@cuda_compat", "@cuda_nvcc", "@libnvvm", "@cuda_nvrtc", @@ -40,11 +42,16 @@ copy_to_directory( "@zml//platforms/cuda:zmlxcuda", ], replace_prefixes = { + "compat": "lib", + "libcuda": "lib", "nvidia/nccl/lib": "lib", "nvvm/lib64": "lib", "libpjrt_cuda_so": "lib", "lib/x86_64-linux-gnu": "lib", "platforms/cuda": "lib", + } | { + "{}.patchelf".format(file): "lib" + for file in CUDA_COMPAT_FILES }, add_directory_to_runfiles = False, include_external_repositories = ["**"], diff --git a/platforms/cuda/zmlxcuda.zig b/platforms/cuda/zmlxcuda.zig index 5834cb6b..922fffaa 100644 --- a/platforms/cuda/zmlxcuda.zig +++ b/platforms/cuda/zmlxcuda.zig @@ -13,6 +13,9 @@ pub export fn zmlxcuda_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque .{ "libcusolver.so", "libcusolver.so.12" }, .{ "libcusparse.so", "libcusparse.so.12" }, .{ "libnccl.so", "libnccl.so.2" }, + + // CUDA Compat + .{ "libcuda.so.1", "libcuda.so.1" }, }); var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; diff --git a/zml/tokenizer/hftokenizers/BUILD.bazel b/zml/tokenizer/hftokenizers/BUILD.bazel index a07d855b..ebd30da0 100644 --- a/zml/tokenizer/hftokenizers/BUILD.bazel +++ b/zml/tokenizer/hftokenizers/BUILD.bazel @@ -1,5 +1,5 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_rust//rust:defs.bzl", "rust_static_library") +load("@rules_rs//rs:rust_static_library.bzl", "rust_static_library") load("@rules_zig//zig:defs.bzl", "zig_library") rust_static_library( From 0fda597649d4bca504f4ef2cbe14ae0e88e03e4e Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 6 Mar 2026 21:00:30 +0000 Subject: [PATCH 337/881] platforms/cuda: add WSL support --- platforms/cuda/cuda.zig | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/platforms/cuda/cuda.zig b/platforms/cuda/cuda.zig index b0ff4425..3765454d 100644 --- a/platforms/cuda/cuda.zig +++ b/platforms/cuda/cuda.zig @@ -16,8 +16,11 @@ pub fn isEnabled() bool { } fn hasNvidiaDevice(io: std.Io) bool { - std.Io.Dir.accessAbsolute(io, "/dev/nvidiactl", .{ .read = true }) catch return false; - return true; + for (&[_][]const u8{ "/dev/nvidiactl", "/dev/dxg" }) |dev| { + std.Io.Dir.accessAbsolute(io, dev, .{ .read = true }) catch continue; + return true; + } + return false; } fn hasCudaPathInLDPath() bool { From 2780770da70178a31e25f2fff0e48627fb19c5b4 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 6 Mar 2026 23:03:14 +0000 Subject: [PATCH 338/881] workspace: re-enable rules_rs --- .bazelrc | 6 +- MODULE.bazel | 5 +- MODULE.bazel.lock | 105 ++++++++++++++++++++++--- platform_mappings | 29 ------- platforms/BUILD.bazel | 8 ++ zml/tokenizer/hftokenizers/BUILD.bazel | 1 + 6 files changed, 112 insertions(+), 42 deletions(-) delete mode 100644 platform_mappings diff --git a/.bazelrc b/.bazelrc index e0c4d194..cbbd8cda 100644 --- a/.bazelrc +++ b/.bazelrc @@ -71,11 +71,11 @@ common --incompatible_enable_proto_toolchain_resolution common --incompatible_disallow_empty_glob=false # Use the C linker to link Zig code -common --@rules_zig//zig/settings:use_cc_common_link=True +common --@rules_zig//zig/settings:use_cc_common_link=true # Zig self hosted backend is too fragile for now common --@rules_zig//zig/settings:zigopt=-fllvm -common --@llvm//config:experimental_stub_libgcc_s=True +common --@llvm//config:experimental_stub_libgcc_s=true common --macos_minimum_os=13.0 @@ -114,3 +114,5 @@ common:release --@rules_zig//zig/settings:mode=release_safe common:native --copt=-march=native common:native --@rules_zig//zig/settings:zigopt=-mcpu=native common:native --@rules_rust//rust/settings:extra_rustc_flag=-Ctarget-cpu=native + +common:linux --host_platform=//platforms:host_glibc diff --git a/MODULE.bazel b/MODULE.bazel index 69553d54..42214dc7 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -16,7 +16,7 @@ bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep(name = "rules_oci", version = "2.2.7") bazel_dep(name = "rules_proto", version = "7.1.0") bazel_dep(name = "rules_python", version = "1.8.4") -bazel_dep(name = "rules_rs", version = "0.0.33") +bazel_dep(name = "rules_rs", version = "0.0.41") bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "rules_zig", version = "0.12.2") bazel_dep(name = "llvm", version = "0.6.1") @@ -130,7 +130,7 @@ use_repo(llvm, "llvm-project") rs_toolchains = use_extension("@rules_rs//rs/experimental/toolchains:module_extension.bzl", "toolchains") rs_toolchains.toolchain( edition = "2024", - version = "1.92.0", + version = "1.94.0", ) use_repo(rs_toolchains, "default_rust_toolchains") @@ -147,6 +147,7 @@ crate.from_cargo( "aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu", ], + use_experimental_platforms = True, ) use_repo(crate, "crates") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index c725862f..0996516a 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -16,7 +16,6 @@ "https://bcr.bazel.build/modules/abseil-cpp/20260107.1/source.json": "7a9a88969b1e79268cf613728ca8ff8fa4bc4b1a9abee9ec1fb5f113ca751971", "https://bcr.bazel.build/modules/apple_support/1.11.1/MODULE.bazel": "1843d7cd8a58369a444fc6000e7304425fba600ff641592161d9f15b179fb896", "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", - "https://bcr.bazel.build/modules/apple_support/1.23.0/MODULE.bazel": "317d47e3f65b580e7fb4221c160797fda48e32f07d2dfff63d754ef2316dcd25", "https://bcr.bazel.build/modules/apple_support/1.23.1/MODULE.bazel": "53763fed456a968cf919b3240427cf3a9d5481ec5466abc9d5dc51bc70087442", "https://bcr.bazel.build/modules/apple_support/1.24.1/MODULE.bazel": "f46e8ddad60aef170ee92b2f3d00ef66c147ceafea68b6877cb45bd91737f5f8", "https://bcr.bazel.build/modules/apple_support/2.2.0/MODULE.bazel": "5e32bc42e413a4544df7afee7fa4c7aff73e670a44e56e8e45ce1954332034ad", @@ -96,6 +95,7 @@ "https://bcr.bazel.build/modules/jsoncpp/1.9.6/MODULE.bazel": "2f8d20d3b7d54143213c4dfc3d98225c42de7d666011528dc8fe91591e2e17b0", "https://bcr.bazel.build/modules/jsoncpp/1.9.6/source.json": "a04756d367a2126c3541682864ecec52f92cdee80a35735a3cb249ce015ca000", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", + "https://bcr.bazel.build/modules/llvm/0.6.0/MODULE.bazel": "42c2182c49f13d2df83a4a4a95ab55d31efda47b2d67acf419bf6b31522b2a30", "https://bcr.bazel.build/modules/llvm/0.6.1/MODULE.bazel": "29170ab19f4e2dc9b6bbf9b3d101738e84142f63ba29a13cc33e0d40f74c79b0", "https://bcr.bazel.build/modules/llvm/0.6.1/source.json": "2d8cdd3a5f8e1d16132dbbe97250133101e4863c0376d23273d9afd7363cc331", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", @@ -218,14 +218,10 @@ "https://bcr.bazel.build/modules/rules_python/1.6.0/MODULE.bazel": "7e04ad8f8d5bea40451cf80b1bd8262552aa73f841415d20db96b7241bd027d8", "https://bcr.bazel.build/modules/rules_python/1.8.4/MODULE.bazel": "33e3971e66161a3e955f7a0d411a8d1f291c4ce4c561851512466f3c77ff8ece", "https://bcr.bazel.build/modules/rules_python/1.8.4/source.json": "9fbc0e57bae52cddcc3831d668bce87a47e0c655104a85098d4459dd9a3b0a10", - "https://bcr.bazel.build/modules/rules_rs/0.0.28/MODULE.bazel": "3c3830eda824d0476818443dd4a3a2fa0765484589db874d18e94fb6ee50a60f", - "https://bcr.bazel.build/modules/rules_rs/0.0.28/source.json": "3480135f6ef8111ec7abe2b7cc164af40354b4eaf8f885170567922a70a2dc99", - "https://bcr.bazel.build/modules/rules_rust/0.66.0/MODULE.bazel": "86ef763a582f4739a27029bdcc6c562258ed0ea6f8d58294b049e215ceb251b3", - "https://bcr.bazel.build/modules/rules_rust/0.68.1/MODULE.bazel": "8d3332ef4079673385eb81f8bd68b012decc04ac00c9d5a01a40eff90301732c", - "https://bcr.bazel.build/modules/rules_rust/0.68.1/source.json": "3378e746f81b62457fdfd37391244fa8ff075ba85c05931ee4f3a20ac1efe963", + "https://bcr.bazel.build/modules/rules_rs/0.0.41/MODULE.bazel": "3f4f7e4cf87c7e8f8e32e8e7a49e1fb6caa5529b9b52757ec25d0b4d0272e380", + "https://bcr.bazel.build/modules/rules_rs/0.0.41/source.json": "19cf5bd6235a7e2609c5a181c52fa3111d0ef70ba19edee752489e276024c34b", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", - "https://bcr.bazel.build/modules/rules_shell/0.4.0/MODULE.bazel": "0f8f11bb3cd11755f0b48c1de0bbcf62b4b34421023aa41a2fc74ef68d9584f0", "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", "https://bcr.bazel.build/modules/rules_shell/0.6.1/MODULE.bazel": "72e76b0eea4e81611ef5452aa82b3da34caca0c8b7b5c0c9584338aa93bae26b", "https://bcr.bazel.build/modules/rules_shell/0.6.1/source.json": "20ec05cd5e592055e214b2da8ccb283c7f2a421ea0dc2acbf1aa792e11c03d0c", @@ -333,7 +329,7 @@ "@@aspect_tools_telemetry+//:extension.bzl%telemetry": { "general": { "bzlTransitiveDigest": "MePriaXmQNSqUfE+YQvEtzBc2bU1mjITINIffFiZYgo=", - "usagesDigest": "73l8h4f8amAmbKge5Ixr/4s4BE7L7fqDBTTAMrNSHHg=", + "usagesDigest": "qri4Ekw3JtqdQ7Lg8s2Zpc+NTUMzWNCysFG3DPfQ3RY=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -343,7 +339,7 @@ "attributes": { "deps": { "aspect_rules_py": "1.8.4", - "rules_rs": "0.0.28", + "rules_rs": "0.0.41", "aspect_tools_telemetry": "0.3.3" } } @@ -1405,6 +1401,97 @@ "zerocopy-derive_0.8.26": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"dissimilar\",\"req\":\"^1.0.9\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"=0.2.163\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"=1.9\"},{\"kind\":\"dev\",\"name\":\"prettyplease\",\"req\":\"=0.2.17\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.10\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.89\"}],\"features\":{}}", "zerocopy_0.7.35": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"assert_matches\",\"req\":\"^1.5\"},{\"default_features\":false,\"name\":\"byteorder\",\"optional\":true,\"req\":\"^1.3\"},{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.85\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.7.35\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.7.35\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.7.35\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\"],\"alloc\":[],\"default\":[\"byteorder\"],\"derive\":[\"zerocopy-derive\"],\"simd\":[],\"simd-nightly\":[\"simd\"]}}", "zerocopy_0.8.26": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"either\",\"req\":\"=1.13.0\"},{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.89\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.8.26\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\",\"std\"],\"alloc\":[],\"derive\":[\"zerocopy-derive\"],\"float-nightly\":[],\"simd\":[],\"simd-nightly\":[\"simd\"],\"std\":[\"alloc\"]}}" + }, + "@@rules_rs+//rs/experimental/toolchains:module_extension.bzl%toolchains": { + "cargo-1.94.0-aarch64-apple-darwin.tar.xz": "7497ab873993cf38b806548c85bf818ffd71d1c3cf77b5bb4c3d3f1769588a08", + "cargo-1.94.0-aarch64-pc-windows-msvc.tar.xz": "0eacb6af29cacb850af4b390e698b2c8060f639a70e6b9f0575cdcc82b0822b2", + "cargo-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "d57ab7b23b52cba46e58c78327904d0c7db19892988677b0d59495c8af4bb790", + "cargo-1.94.0-x86_64-apple-darwin.tar.xz": "3beb7874010ebd757780284c0c9da47fc9f0b18091d7415d7b7231a8c19ec44f", + "cargo-1.94.0-x86_64-pc-windows-msvc.tar.xz": "6cf7113e1d5633721739a3bd6fee7148d6394bbab748fbede941210100374741", + "cargo-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "8e17624f3de39e079845bfb25ed15a042f4b50ceca78e37c56c4b9b15949b9f7", + "clippy-1.94.0-aarch64-apple-darwin.tar.xz": "42a683cbac0b81b96ad3b887cdade36a57ef3a131c203583ca7e3ce0be98f415", + "clippy-1.94.0-aarch64-pc-windows-msvc.tar.xz": "44468016e27457640c7ab62b7ceaddb44308e8ec77ee4344c1bd36113448c8eb", + "clippy-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "f7450c414e394450b53be602d37c374e2a2c5a2f51be2b54faf3d1e3272160b1", + "clippy-1.94.0-x86_64-apple-darwin.tar.xz": "1393cc688bc4d784a9d66619d2eec683a1ddb30b263f03ab615c0a2b18053126", + "clippy-1.94.0-x86_64-pc-windows-msvc.tar.xz": "e4dcc5a796857589e364064493abb3afb469e0e36e90155e7b0a2c5ce4bb7503", + "clippy-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "935529b0b9150084fa9cc1734c6598efe86b6e7d76f31ad3f8d4bb79ebf6031a", + "rust-std-1.94.0-aarch64-apple-darwin.tar.xz": "d3293668128b0a7838ef2544ae341eadd4437460ca22098af723349885a10ba2", + "rust-std-1.94.0-aarch64-apple-ios-macabi.tar.xz": "914f085065539ea1038d2832a5af1c22852cd5e9a063356b20e46698e72e770e", + "rust-std-1.94.0-aarch64-apple-ios-sim.tar.xz": "b4d16842b8f98ff2f431d6ab4aced298f4028ca00f0f5d588bc6efc94faa0680", + "rust-std-1.94.0-aarch64-apple-ios.tar.xz": "41ec1d18f4762f85231a19cf8aba3385da78fece2ff900a435c62881a3a3b44b", + "rust-std-1.94.0-aarch64-linux-android.tar.xz": "402c80df672a4b9fc948b6dc8bde21a63e1093eb1c8a2c2bd42221c309d9eba2", + "rust-std-1.94.0-aarch64-pc-windows-gnullvm.tar.xz": "8c89521f6826a1d2bf07b27c95405f4d6bf7c24f3de832c02758c41d4c1a511f", + "rust-std-1.94.0-aarch64-pc-windows-msvc.tar.xz": "d7964debd444dca92e771c953c3cd582fb927031b536c1439e1a831014d5f0d1", + "rust-std-1.94.0-aarch64-unknown-fuchsia.tar.xz": "87e11d97055540a0675bdc3dacf3d3d8cca5f806171ad0b51d0427f7cb9510c9", + "rust-std-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "c781b3ef4fefa5508fbe05820eddc95e46351d905a30921cc020febd9c596a2e", + "rust-std-1.94.0-aarch64-unknown-linux-musl.tar.xz": "3513032b2953d795bd3ac1489de4e8174e72500a31b4e7d1b349a23ab78a7a7b", + "rust-std-1.94.0-aarch64-unknown-none-softfloat.tar.xz": "375c496af31cf208ce67613c0aa1bde1894b5d674b00050cd46f7f954ed4a668", + "rust-std-1.94.0-aarch64-unknown-none.tar.xz": "854a9f8cea0ba889f553a7ea664c2a6479fc1fc49a4b56f84f99fc37001c4322", + "rust-std-1.94.0-aarch64-unknown-uefi.tar.xz": "f038d9e3df557d4aecdcf65989896179bcf0ecbc4d62bde71d33d7b914d3c253", + "rust-std-1.94.0-arm-linux-androideabi.tar.xz": "d2113ef50ed0b974f33b41102b961588c26abaf34f45a289a293b3d31ba01113", + "rust-std-1.94.0-arm-unknown-linux-gnueabi.tar.xz": "ad364b54b668e866315900ebff0d225616c556ab5228599792470aede05ba42c", + "rust-std-1.94.0-arm-unknown-linux-gnueabihf.tar.xz": "d5db943974fe05306ae43d100b0b7f3540108b93ef52de670675ce966af8a0fd", + "rust-std-1.94.0-arm-unknown-linux-musleabi.tar.xz": "c724b1a1b5dddeca563cda9b9695610bb56e681727b3af0a352174960087069a", + "rust-std-1.94.0-arm-unknown-linux-musleabihf.tar.xz": "9f8b707760a632b80864e91ea81a1b19c3f5452ff5e18b96da75792be1cf0a3c", + "rust-std-1.94.0-armv7-linux-androideabi.tar.xz": "e0b6883364f939300c281d96d6778ecf7fd97fe867b11936f0aa5c9a29f488ad", + "rust-std-1.94.0-armv7-unknown-linux-gnueabi.tar.xz": "9b1d8143881d510785d89413ea09b9fe910b64e3f300ab30e52e7fdd652ea66a", + "rust-std-1.94.0-armv7-unknown-linux-gnueabihf.tar.xz": "4b1744a8e48f58c2741afc1e076d0629a86051d785bd36e2c97f329eab6a5b49", + "rust-std-1.94.0-armv7-unknown-linux-musleabi.tar.xz": "fc2754f50ece0c7d3d32b2e0034b19c8b8bba9b471e0eb00f8ece7fa0bcb2837", + "rust-std-1.94.0-armv7-unknown-linux-musleabihf.tar.xz": "abfc0e7c1eaff1e3332b5c6262d12da309035a228261d07155add44464ed52f4", + "rust-std-1.94.0-i686-linux-android.tar.xz": "9093a894d87568a4b5eb4a144758bf49d4b62743677f747848afc3807c23f126", + "rust-std-1.94.0-i686-pc-windows-gnu.tar.xz": "188490bde6649163347050a70fd81c6d9e2e972ef12ba646f7f4fa106e21a956", + "rust-std-1.94.0-i686-pc-windows-gnullvm.tar.xz": "ba61dc038c3f4102a832a3381935192e88b0a1f79ff50b6d5590d476bdfa6879", + "rust-std-1.94.0-i686-pc-windows-msvc.tar.xz": "388cc2bb17ef9f12864c08e462cdbe3222d5ddafd84969dcf4ed944da27efcbd", + "rust-std-1.94.0-i686-unknown-freebsd.tar.xz": "3ce634092bcecb000c793c9d4bb0ccc6fd45c66fa8259fbb4f18c59844c4b13b", + "rust-std-1.94.0-i686-unknown-linux-gnu.tar.xz": "528c1dda491b25bb855426f48757773f4740c75d3c7271efffe7412c2fd5730d", + "rust-std-1.94.0-i686-unknown-linux-musl.tar.xz": "90ff16d879f774048c9b81d07188af4b63ed73a31848759cf560a9f51baa175f", + "rust-std-1.94.0-i686-unknown-uefi.tar.xz": "8c095069ec09d6e1d0b7e00f340fd7ea925a31acbad845d5956e1bc4a7d47733", + "rust-std-1.94.0-powerpc-unknown-linux-gnu.tar.xz": "23e32cab484d7787b7966a0508c49baf231bcdb4a79a421ee85ca69e81f2a06f", + "rust-std-1.94.0-powerpc64-unknown-linux-gnu.tar.xz": "7713b676a37fbd0a52a711c2e3c205a917f8b05614e17cd9f7f37874a88469a3", + "rust-std-1.94.0-powerpc64le-unknown-linux-gnu.tar.xz": "094f3955828d6630b70a9c7c71e3165c3ffc7a301929601dde08501def623347", + "rust-std-1.94.0-powerpc64le-unknown-linux-musl.tar.xz": "f3956b29463ec58d730f054ff2334755f929c602181ad67a25af9725c8500f68", + "rust-std-1.94.0-riscv32imc-unknown-none-elf.tar.xz": "5d10c7ab74cd5d10451480ec894ac5b98940a8527c333465eeec4a45b1125d43", + "rust-std-1.94.0-riscv64gc-unknown-linux-gnu.tar.xz": "32e9b7006f1269e48fb65af6298f6e8a222e1a8a2a20b112528bfb3dad2f65a8", + "rust-std-1.94.0-riscv64gc-unknown-linux-musl.tar.xz": "2c63d21143c7f1d44f3e3b1c9a82f3101576344c3fd1feb2d43ffc9d035e94f1", + "rust-std-1.94.0-riscv64gc-unknown-none-elf.tar.xz": "071a1ae54f7dead1b8147b9bae7d831a5554a37a1133f8cd52b265a765595a12", + "rust-std-1.94.0-s390x-unknown-linux-gnu.tar.xz": "ea69833addbc9521664464418c7468871f7a80b16fd2c385f6213a6c99cedfe9", + "rust-std-1.94.0-thumbv6m-none-eabi.tar.xz": "b022df94f6efa3bd2a6c4042472e114d2325bf058983b3ac7d586e24c8d77f25", + "rust-std-1.94.0-thumbv7em-none-eabi.tar.xz": "41894db05c4d8f421bf0dba7d0bed995685032393779f92b4ea1f3e6468badfb", + "rust-std-1.94.0-thumbv7em-none-eabihf.tar.xz": "be7e81535e7118fbad58a5dc31a1acfa7f2a36a2348aed23595169418559d67f", + "rust-std-1.94.0-thumbv7m-none-eabi.tar.xz": "4702e615be264df97b120fd075bed4e46710cc9a03c48115ce2ee3abfbf3822e", + "rust-std-1.94.0-thumbv8m.main-none-eabi.tar.xz": "5c1fb5ed8ad31580241b6e095583a2cac0ef90ea29dad7705c8a0ce0933a7797", + "rust-std-1.94.0-thumbv8m.main-none-eabihf.tar.xz": "573e9570bea039316ae3b55e323cbc741617ace1e0b87df8f3ecbbdf34b3de8a", + "rust-std-1.94.0-wasm32-unknown-emscripten.tar.xz": "0a607ba5d78afb47e9f5f134633ed5a954ece5558298e081084abe44f5265463", + "rust-std-1.94.0-wasm32-unknown-unknown.tar.xz": "bc65af3c0bfa2e5b20e299358710a26ce33a3674c762a2a010bc383c5b7fc202", + "rust-std-1.94.0-wasm32-wasip1-threads.tar.xz": "311b9fe1e94c90c16695899e670f0d76a366d493cdda29008191ce2d436cd22c", + "rust-std-1.94.0-wasm32-wasip1.tar.xz": "acbacacdca3e0ec0d93ff46d5f3579ae8cd5133b86e31a893da22cd94e28d0bf", + "rust-std-1.94.0-wasm32-wasip2.tar.xz": "e947e730da3fea212453b2835765802d9b0c80e19405cab188a6854834196e6c", + "rust-std-1.94.0-x86_64-apple-darwin.tar.xz": "cf716b5a5fab007398ce76536a85a3449df074b66e7c762c0f5de1d6c56a1342", + "rust-std-1.94.0-x86_64-apple-ios-macabi.tar.xz": "f39f4c1121dafcf8ea70a21c91be9552e30ea76debcad9bff28be945aaa79282", + "rust-std-1.94.0-x86_64-apple-ios.tar.xz": "60e70b80469dd3900dd8b1705831f5f02ac44fa97088b8a81fdda110ed421091", + "rust-std-1.94.0-x86_64-linux-android.tar.xz": "488c8a581612ac4779a05f05967b76a96e0ff0bdd1ab8d99c13a859c19056b01", + "rust-std-1.94.0-x86_64-pc-windows-gnu.tar.xz": "c9f98d62213d0559c8d2ee6d7eafbafbf5dfe04921c9e7cb15b66a770903bc33", + "rust-std-1.94.0-x86_64-pc-windows-gnullvm.tar.xz": "54c28d0030fab585457a431c555e5d23502132364d0483e5b3b2dfbfc3929cc6", + "rust-std-1.94.0-x86_64-pc-windows-msvc.tar.xz": "f8e2b43e52957e8224d97de68d6ec5da01613eb98bce4199de80071678b5fda0", + "rust-std-1.94.0-x86_64-unknown-freebsd.tar.xz": "081972f2f386ee859c15ed619695a9440896a80d1814d0e29d7620b7ef96207c", + "rust-std-1.94.0-x86_64-unknown-fuchsia.tar.xz": "6c18112553cf72cf991352208cce2bc96c712ca7b209acdd098abac894d674ef", + "rust-std-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "dd33653107c36e040082050d9e547e64dac5b456ba74069430d838c00c189a05", + "rust-std-1.94.0-x86_64-unknown-linux-musl.tar.xz": "b6786b5c490df2dcef76829039177799058f94e17725b42f912a057c0f6224f2", + "rust-std-1.94.0-x86_64-unknown-netbsd.tar.xz": "f763e56ca1ce3f819fb2136638d774777dd696f836cc52e6564431859a0f5159", + "rust-std-1.94.0-x86_64-unknown-none.tar.xz": "8509489165a8aa60a0590aeaa668f63423b3e7a33bb93e8e5719ff5404b6f6aa", + "rust-std-1.94.0-x86_64-unknown-uefi.tar.xz": "31b56e8178dd2319e51ccc474959d247db2767cda7149fe3704d636df9aee1c9", + "rustc-1.94.0-aarch64-apple-darwin.tar.xz": "6ba00a4f486af11826335cd7fe405592d959f50afb9489176fc1463cbe75dbf2", + "rustc-1.94.0-aarch64-pc-windows-msvc.tar.xz": "6d88d89c200a147357ded65e9bc426e14ebb6f5046d9772f3916b6a1ac60cdd5", + "rustc-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "7261b2fcdb88aef8b21f4b9f915d9a75c959674193f5b38e13fab4569684fc5b", + "rustc-1.94.0-x86_64-apple-darwin.tar.xz": "b1374b0fb63f5abcd8eb65607e43a499bfdc5e8160746d1f6f993aee9313f320", + "rustc-1.94.0-x86_64-pc-windows-msvc.tar.xz": "e0f59503778b563d3ee6d4edecd52b44056779ffe456181bad1fb4dc069802e6", + "rustc-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "31a0d3ac9383dfdeb4fce86eeed5ade3230131c635264c0eab7252dbf235f28e", + "rustfmt-1.94.0-aarch64-apple-darwin.tar.xz": "0eb716d201dc0d6d24ef8207d3ef27611d9d15d2212e3b27aa11e97286ed02ac", + "rustfmt-1.94.0-aarch64-pc-windows-msvc.tar.xz": "705ac3e797c4abed9e722e0864902f7a03872a8a201dcd7b70ff1b749eed2327", + "rustfmt-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "1ac17dbe7f8f6c98db5b62cb149fcd10c7565d624a59471115fc40055db9af28", + "rustfmt-1.94.0-x86_64-apple-darwin.tar.xz": "e58601a10ee85e956611ea2814bf26de88934a8224c1663bb11aa23f727663ec", + "rustfmt-1.94.0-x86_64-pc-windows-msvc.tar.xz": "7b32d8b0dd99f9bf6ddaa4866d39b7b96e8c2db46644d8527096f7f34b4fbe26", + "rustfmt-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "903e768f7626f78ed2d6cb82c1820b93bd339afec8575102e847fd3a379d33e5" } } } diff --git a/platform_mappings b/platform_mappings deleted file mode 100644 index ed02329d..00000000 --- a/platform_mappings +++ /dev/null @@ -1,29 +0,0 @@ -platforms: - @zml//platforms:linux_amd64 - --cpu=k8 - - @zml//platforms:linux_arm64 - --cpu=aarch64 - - @zml//platforms:macos_arm64 - --cpu=darwin_arm64 - --apple_platform_type=macos - - @zml//platforms:macos_amd64 - --cpu=darwin_amd64 - --apple_platform_type=macos - -flags: - --cpu=darwin_arm64 - --apple_platform_type=macos - @zml//platforms:macos_arm64 - - --cpu=darwin_amd64 - --apple_platform_type=macos - @zml//platforms:macos_amd64 - - --cpu=k8 - @zml//platforms:linux_amd64 - - --cpu=aarch64 - @zml//platforms:linux_arm64 diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index a2a023b1..a4acc4e8 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -79,3 +79,11 @@ platform( ], visibility = ["//visibility:public"], ) + +platform( + name = "host_glibc", + parents = ["@platforms//host"], + constraint_values = [ + "@llvm//constraints/libc:gnu.2.28", + ], +) diff --git a/zml/tokenizer/hftokenizers/BUILD.bazel b/zml/tokenizer/hftokenizers/BUILD.bazel index ebd30da0..02a49495 100644 --- a/zml/tokenizer/hftokenizers/BUILD.bazel +++ b/zml/tokenizer/hftokenizers/BUILD.bazel @@ -7,6 +7,7 @@ rust_static_library( srcs = ["hftokenizers.rs"], crate_name = "zml_tokenizer_hftokenizers", edition = "2021", + proc_macro_deps = ["@crates//:tokenizers"], deps = ["@crates//:tokenizers"], ) From 54c2153389857ca8caa85c094ae2d5b2ad0ce2d4 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 6 Mar 2026 23:05:52 +0000 Subject: [PATCH 339/881] platforms/cuda: enable cuda-compat only on supported GPUs It turns out it's only supported on enterprise GPUs. The trick is to set a soname on the compat/libcuda.so.1 and preload it so that later resolution uses this one. See https://docs.nvidia.com/deploy/cuda-compatibility/forward-compatibility.html --- platforms/cuda/cuda.zig | 68 +++++++++++++++++++++++-- platforms/cuda/libpjrt_cuda.BUILD.bazel | 4 +- platforms/cuda/zmlxcuda.zig | 3 -- 3 files changed, 65 insertions(+), 10 deletions(-) diff --git a/platforms/cuda/cuda.zig b/platforms/cuda/cuda.zig index 3765454d..a8e51434 100644 --- a/platforms/cuda/cuda.zig +++ b/platforms/cuda/cuda.zig @@ -15,6 +15,56 @@ pub fn isEnabled() bool { return @hasDecl(c, "ZML_RUNTIME_CUDA"); } +pub fn modelNameNeedsCompat(model_line: []const u8) bool { + const names = &[_][]const u8{ + " A100", + " A6000", + " H100", + " H200", + " B100", + " B200", + " B300", + }; + for (names) |name| { + if (std.ascii.findIgnoreCase(model_line, name) != null) { + return true; + } + } + return false; +} + +pub fn needsCudaCompat(io: std.Io) !bool { + var dir = try std.Io.Dir.openDirAbsolute(io, "/proc/driver/nvidia/gpus", .{ .iterate = true }); + defer dir.close(io); + + var it = dir.iterate(); + while (try it.next(io)) |entry| { + if (entry.kind != .directory) { + continue; + } + + var gpu_dir = try dir.openDir(io, entry.name, .{}); + defer gpu_dir.close(io); + + var information_file: std.Io.File = try gpu_dir.openFile(io, "information", .{}); + defer information_file.close(io); + + var buffer: [4096]u8 = undefined; + var file_reader = information_file.reader(io, &buffer); + while (true) { + const line = file_reader.interface.takeDelimiterInclusive('\n') catch |err| switch (err) { + std.Io.Reader.DelimiterError.EndOfStream => break, + else => return err, + }; + if (std.ascii.startsWithIgnoreCase(line, "Model:")) { + return modelNameNeedsCompat(line); + } + } + } + + return false; +} + fn hasNvidiaDevice(io: std.Io) bool { for (&[_][]const u8{ "/dev/nvidiactl", "/dev/dxg" }) |dev| { std.Io.Dir.accessAbsolute(io, dev, .{ .read = true }) catch continue; @@ -64,12 +114,20 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { try setupXlaGpuCudaDirFlag(arena.allocator(), sandbox_path); { - var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libnvtx3interop.so" }); - _ = std.c.dlopen(path, .{ .NOW = true, .GLOBAL = true }) orelse { - log.err("Unable to dlopen libnvtx3interop.so: {s}", .{std.c.dlerror().?}); - return error.DlError; + const cudaCompat = needsCudaCompat(io) catch |err| blk: { + log.err("Unable to determine wether or not to use CUDA Compat, disabling: {any}", .{err}); + break :blk false; }; + + if (cudaCompat) { + log.warn("Detected NVIDIA GPU that requires CUDA compatibility libraries.", .{}); + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "compat", "libcuda.so.1" }); + _ = std.c.dlopen(path, .{ .NOW = true }) orelse { + log.warn("Failed to load CUDA compatibility library from {s}: {any}", .{ path, std.mem.span(std.c.dlerror()) }); + }; + log.info("Loaded CUDA compatibility libraries.", .{}); + } } return blk: { diff --git a/platforms/cuda/libpjrt_cuda.BUILD.bazel b/platforms/cuda/libpjrt_cuda.BUILD.bazel index a4a70699..91048a66 100644 --- a/platforms/cuda/libpjrt_cuda.BUILD.bazel +++ b/platforms/cuda/libpjrt_cuda.BUILD.bazel @@ -42,7 +42,7 @@ copy_to_directory( "@zml//platforms/cuda:zmlxcuda", ], replace_prefixes = { - "compat": "lib", + "compat": "lib/compat", "libcuda": "lib", "nvidia/nccl/lib": "lib", "nvvm/lib64": "lib", @@ -50,7 +50,7 @@ copy_to_directory( "lib/x86_64-linux-gnu": "lib", "platforms/cuda": "lib", } | { - "{}.patchelf".format(file): "lib" + "{}.patchelf".format(file): "lib/compat" for file in CUDA_COMPAT_FILES }, add_directory_to_runfiles = False, diff --git a/platforms/cuda/zmlxcuda.zig b/platforms/cuda/zmlxcuda.zig index 922fffaa..5834cb6b 100644 --- a/platforms/cuda/zmlxcuda.zig +++ b/platforms/cuda/zmlxcuda.zig @@ -13,9 +13,6 @@ pub export fn zmlxcuda_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque .{ "libcusolver.so", "libcusolver.so.12" }, .{ "libcusparse.so", "libcusparse.so.12" }, .{ "libnccl.so", "libnccl.so.2" }, - - // CUDA Compat - .{ "libcuda.so.1", "libcuda.so.1" }, }); var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; From e17da6892f7781b8578aed10cf26c94df8cb98ee Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sat, 7 Mar 2026 10:53:14 +0000 Subject: [PATCH 340/881] platforms/cuda: probe cuda-compat via an external binary Detect CUDA Compat by spawning a probe binary that will try to initialize it. If it succeeds, the main binary can use it too. --- platforms/cuda/BUILD.bazel | 9 ++- platforms/cuda/compat_probe.zig | 30 ++++++++++ platforms/cuda/cuda.zig | 75 +++++++++---------------- platforms/cuda/libpjrt_cuda.BUILD.bazel | 5 +- 4 files changed, 69 insertions(+), 50 deletions(-) create mode 100644 platforms/cuda/compat_probe.zig diff --git a/platforms/cuda/BUILD.bazel b/platforms/cuda/BUILD.bazel index 3bb7cfc0..8386c10f 100644 --- a/platforms/cuda/BUILD.bazel +++ b/platforms/cuda/BUILD.bazel @@ -1,5 +1,11 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_shared_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_shared_library", "zig_binary"); + +zig_binary( + name = "compat_probe", + main = "compat_probe.zig", + visibility = ["@libpjrt_cuda//:__subpackages__"], +) zig_shared_library( name = "zmlxcuda", @@ -32,6 +38,7 @@ zig_library( ], "//conditions:default": [":empty"], }), + srcs = ["compat_probe.zig"], import_name = "platforms/cuda", main = "cuda.zig", visibility = ["//visibility:public"], diff --git a/platforms/cuda/compat_probe.zig b/platforms/cuda/compat_probe.zig new file mode 100644 index 00000000..06e7e82c --- /dev/null +++ b/platforms/cuda/compat_probe.zig @@ -0,0 +1,30 @@ +const std = @import("std"); + +pub const ExitCode = enum(u8) { + Success = 0, + CompatNotSupportedOnDevice = 1, + UnexpectedError = 2, +}; + +const CUDA_SUCCESS = 0; +const CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE = 804; + +pub fn main(init: std.process.Init) !u8 { + const arena = init.arena; + const io = init.io; + + const self_exe_dir = try std.process.executableDirPathAlloc(io, arena.allocator()); + const libcuda_path = try std.Io.Dir.path.joinZ(arena.allocator(), &.{ self_exe_dir, "..", "lib", "compat", "libcuda.so.1" }); + + var libcuda = try std.DynLib.open(libcuda_path); + const cuInit = libcuda.lookup(*const fn (c_uint) callconv(.c) c_int, "cuInit").?; + + return @intFromEnum(switch (cuInit(0)) { + CUDA_SUCCESS => ExitCode.Success, + CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE => ExitCode.CompatNotSupportedOnDevice, + else => |v| blk: { + std.log.err("cuInit returned unexpected error code: {d}", .{v}); + break :blk ExitCode.UnexpectedError; + }, + }); +} diff --git a/platforms/cuda/cuda.zig b/platforms/cuda/cuda.zig index a8e51434..88e5f1be 100644 --- a/platforms/cuda/cuda.zig +++ b/platforms/cuda/cuda.zig @@ -7,6 +7,8 @@ const c = @import("c"); const pjrt = @import("pjrt"); const stdx = @import("stdx"); +const compat_probe = @import("compat_probe.zig"); + const nvidiaLibsPath = "/cuda/"; const log = std.log.scoped(.@"zml/platforms/cuda"); @@ -15,54 +17,33 @@ pub fn isEnabled() bool { return @hasDecl(c, "ZML_RUNTIME_CUDA"); } -pub fn modelNameNeedsCompat(model_line: []const u8) bool { - const names = &[_][]const u8{ - " A100", - " A6000", - " H100", - " H200", - " B100", - " B200", - " B300", +pub fn needsCudaCompat(io: std.Io, sandbox_path: []const u8) !bool { + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const nvidia_compat_path = try stdx.Io.Dir.path.bufJoinZ(&buf, &.{ sandbox_path, "bin", "compat_probe" }); + + var child = try std.process.spawn(io, .{ + .argv = &[_][]const u8{nvidia_compat_path}, + .cwd = .{ .path = sandbox_path }, + .stdin = .ignore, + .stdout = .ignore, + .stderr = .ignore, + }); + defer child.kill(io); + + const res = child.wait(io) catch |err| { + log.err("Failed to run CUDA compatibility probe: {any}", .{err}); + return err; }; - for (names) |name| { - if (std.ascii.findIgnoreCase(model_line, name) != null) { - return true; - } - } - return false; -} + const result: compat_probe.ExitCode = @enumFromInt(res.exited); -pub fn needsCudaCompat(io: std.Io) !bool { - var dir = try std.Io.Dir.openDirAbsolute(io, "/proc/driver/nvidia/gpus", .{ .iterate = true }); - defer dir.close(io); - - var it = dir.iterate(); - while (try it.next(io)) |entry| { - if (entry.kind != .directory) { - continue; - } - - var gpu_dir = try dir.openDir(io, entry.name, .{}); - defer gpu_dir.close(io); - - var information_file: std.Io.File = try gpu_dir.openFile(io, "information", .{}); - defer information_file.close(io); - - var buffer: [4096]u8 = undefined; - var file_reader = information_file.reader(io, &buffer); - while (true) { - const line = file_reader.interface.takeDelimiterInclusive('\n') catch |err| switch (err) { - std.Io.Reader.DelimiterError.EndOfStream => break, - else => return err, - }; - if (std.ascii.startsWithIgnoreCase(line, "Model:")) { - return modelNameNeedsCompat(line); - } - } - } - - return false; + return switch (result) { + .Success => true, + .CompatNotSupportedOnDevice => false, + .UnexpectedError => blk: { + log.err("CUDA compatibility probe returned unexpected error code", .{}); + break :blk false; + }, + }; } fn hasNvidiaDevice(io: std.Io) bool { @@ -114,7 +95,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { try setupXlaGpuCudaDirFlag(arena.allocator(), sandbox_path); { - const cudaCompat = needsCudaCompat(io) catch |err| blk: { + const cudaCompat = needsCudaCompat(io, sandbox_path) catch |err| blk: { log.err("Unable to determine wether or not to use CUDA Compat, disabling: {any}", .{err}); break :blk false; }; diff --git a/platforms/cuda/libpjrt_cuda.BUILD.bazel b/platforms/cuda/libpjrt_cuda.BUILD.bazel index 91048a66..4d8a06af 100644 --- a/platforms/cuda/libpjrt_cuda.BUILD.bazel +++ b/platforms/cuda/libpjrt_cuda.BUILD.bazel @@ -39,16 +39,17 @@ copy_to_directory( "@libnvjitlink", "@nccl", "@zlib1g", + "@zml//platforms/cuda:compat_probe", "@zml//platforms/cuda:zmlxcuda", ], replace_prefixes = { "compat": "lib/compat", - "libcuda": "lib", "nvidia/nccl/lib": "lib", "nvvm/lib64": "lib", "libpjrt_cuda_so": "lib", "lib/x86_64-linux-gnu": "lib", - "platforms/cuda": "lib", + "platforms/cuda/compat_probe": "bin/compat_probe", + "platforms/cuda/libzmlxcuda": "lib/libzmlxcuda", } | { "{}.patchelf".format(file): "lib/compat" for file in CUDA_COMPAT_FILES From fd53bfcf31a9cdb864d05058640d53ee149f3f99 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sat, 7 Mar 2026 10:53:59 +0000 Subject: [PATCH 341/881] platforms: set glibc version for linux platform --- platforms/BUILD.bazel | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index a4acc4e8..64f3c911 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -43,6 +43,7 @@ platform( constraint_values = [ "@platforms//cpu:x86_64", "@platforms//os:linux", + "@llvm//constraints/libc:gnu.2.28", ], flags = [ "--force_pic=true", # required for protobuf upb on Linux @@ -55,6 +56,7 @@ platform( constraint_values = [ "@platforms//cpu:aarch64", "@platforms//os:linux", + "@llvm//constraints/libc:gnu.2.28", ], flags = [ "--force_pic=true", # required for protobuf upb on Linux From 26c832dc386137c2db48f6e72cd3a7ed16e0728a Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Mon, 9 Mar 2026 11:22:45 +0000 Subject: [PATCH 342/881] workspace/bazelrc: switch config native from rules_rust to rules_rs --- .bazelrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index cbbd8cda..5bfaf909 100644 --- a/.bazelrc +++ b/.bazelrc @@ -113,6 +113,6 @@ common:release --@rules_zig//zig/settings:mode=release_safe # --config=native builds everything optimized for the local CPU common:native --copt=-march=native common:native --@rules_zig//zig/settings:zigopt=-mcpu=native -common:native --@rules_rust//rust/settings:extra_rustc_flag=-Ctarget-cpu=native +common:native --@@rules_rs++rules_rust+rules_rust//rust/settings:extra_rustc_flag=-Ctarget-cpu=native common:linux --host_platform=//platforms:host_glibc From c7ca488f3bc29e766d95c10c4e7ad5f325812eae Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 10 Mar 2026 09:34:22 +0100 Subject: [PATCH 343/881] third_party/zls: faillible canonicalization of modules (#406) Files provided by bazel may not exist when zls starts, so canonicalization would fail and break zls. Now we fall back to the provided path instead. Furthermore, to make it a bit more robust, if any part of the post process/canonicalization fails, we return the provided input string instead of failing. I'm now also using the same struct definition as ZLS instead of manipulating `std.json.Value`. --- third_party/zls/workspace_printer.zig | 96 +++++++++++++++++---------- 1 file changed, 60 insertions(+), 36 deletions(-) diff --git a/third_party/zls/workspace_printer.zig b/third_party/zls/workspace_printer.zig index 38bdc5ce..94329a8f 100644 --- a/third_party/zls/workspace_printer.zig +++ b/third_party/zls/workspace_printer.zig @@ -3,6 +3,42 @@ const std = @import("std"); const bazel_builtin = @import("bazel_builtin"); const runfiles = @import("runfiles"); +/// https://github.com/zigtools/zls/blob/master/src/build_runner/shared.zig#L6 (2026-03-10) +pub const BuildConfig = struct { + /// The `dependencies` in `build.zig.zon`. + dependencies: std.json.ArrayHashMap([]const u8), + /// The key is the `root_source_file`. + /// All modules with the same root source file are merged. This limitation may be lifted in the future. + modules: std.json.ArrayHashMap(Module), + /// List of all compilations units. + compilations: []Compile, + /// The names of all top level steps. + top_level_steps: []const []const u8, + available_options: std.json.ArrayHashMap(AvailableOption), + + pub const Module = struct { + import_table: std.json.ArrayHashMap([]const u8), + c_macros: []const []const u8, + include_dirs: []const []const u8, + }; + + pub const Compile = struct { + /// Key in `BuildConfig.modules`. + root_module: []const u8, + + // may contain additional information in the future like `target` or `link_libc`. + }; + + /// Equivalent to `std.Build.AvailableOption` which is not accessible because it non-pub. + pub const AvailableOption = @FieldType(@FieldType(std.Build, "available_options_map").KV, "value"); +}; + +fn processConfigLeaky(allocator: std.mem.Allocator, io: std.Io, config_string: []const u8) !BuildConfig { + var config = try std.json.parseFromSliceLeaky(BuildConfig, allocator, config_string, .{}); + try canonicalizeModulesLeaky(allocator, io, &config); + return config; +} + // Neovim canonicalizes the path it sends to ZLS, we need to do the same as ZLS will match against the config we generate. // We canonicalize the following module paths: // @@ -15,43 +51,29 @@ const runfiles = @import("runfiles"); // }, // }, // } -fn canonicalizeModules(allocator: std.mem.Allocator, io: std.Io, root: *std.json.Value) !void { - if (root.* != .object) return; - const modules_value = root.object.getPtr("modules") orelse return; - if (modules_value.* != .object) return; +fn canonicalizeModulesLeaky(allocator: std.mem.Allocator, io: std.Io, config: *BuildConfig) !void { + var old_modules = config.modules.map.iterator(); + var new_modules: std.StringArrayHashMapUnmanaged(BuildConfig.Module) = .empty; + try new_modules.ensureTotalCapacity(allocator, old_modules.len); - var canonical_modules = std.json.ObjectMap.init(allocator); - var modules_it = modules_value.object.iterator(); - while (modules_it.next()) |module_entry| { - const module_path = module_entry.key_ptr.*; - var module_info = module_entry.value_ptr.*; - - if (module_info == .object) { - if (module_info.object.getPtr("import_table")) |imports_value| { - if (imports_value.* == .object) { - var canonical_imports = std.json.ObjectMap.init(allocator); - var imports_it = imports_value.object.iterator(); - while (imports_it.next()) |import_entry| { - const import_name = import_entry.key_ptr.*; - const import_value = import_entry.value_ptr.*; - - const canonical_import_value = if (import_value == .string) - std.json.Value{ .string = try std.Io.Dir.realPathFileAbsoluteAlloc(io, import_value.string, allocator) } - else - import_value; - - try canonical_imports.put(import_name, canonical_import_value); - } - imports_value.* = .{ .object = canonical_imports }; - } - } + while (old_modules.next()) |entry| { + const module_name = canonicalizePath(allocator, io, entry.key_ptr.*); + var module: BuildConfig.Module = entry.value_ptr.*; + for (module.import_table.map.values()) |*imported_module_path| { + imported_module_path.* = canonicalizePath(allocator, io, imported_module_path.*); } - - const canonical_module_path = try std.Io.Dir.realPathFileAbsoluteAlloc(io, module_path, allocator); - try canonical_modules.put(canonical_module_path, module_info); + new_modules.put(allocator, module_name, entry.value_ptr.*) catch return; } - modules_value.* = .{ .object = canonical_modules }; + for (config.compilations) |*compile| { + compile.root_module = canonicalizePath(allocator, io, compile.root_module); + } + + config.modules.map = new_modules; +} + +fn canonicalizePath(allocator: std.mem.Allocator, io: std.Io, path: []const u8) []const u8 { + return std.Io.Dir.realPathFileAbsoluteAlloc(io, path, allocator) catch path; } fn readBuildConfig( @@ -131,11 +153,13 @@ pub fn main(init: std.process.Init) !void { "@@__BAZEL_EXECUTION_ROOT__@@", execution_root, ); - var root = try std.json.parseFromSliceLeaky(std.json.Value, arena.allocator(), output, .{}); - try canonicalizeModules(arena.allocator(), io, &root); var stdout_buf: [4096]u8 = undefined; var writer = std.Io.File.stdout().writer(io, &stdout_buf); - try std.json.Stringify.value(root, .{}, &writer.interface); + if (processConfigLeaky(arena.allocator(), io, output)) |config| { + try std.json.Stringify.value(config, .{}, &writer.interface); + } else |_| { + try writer.interface.writeAll(output); + } try writer.flush(); } From 9df955adad74836ad3692a168bd6f14cd84c3f8a Mon Sep 17 00:00:00 2001 From: KB Date: Tue, 10 Mar 2026 11:09:07 +0100 Subject: [PATCH 344/881] feat: add stb library integration with BUILD and repo files (#408) Co-authored-by: Kevin Barre --- MODULE.bazel | 2 +- third_party/stb/BUILD.bazel | 1 + third_party/stb/repo.bzl | 9 +++++++++ third_party/stb/stb.BUILD | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 third_party/stb/BUILD.bazel create mode 100644 third_party/stb/repo.bzl create mode 100644 third_party/stb/stb.BUILD diff --git a/MODULE.bazel b/MODULE.bazel index 42214dc7..af85e93e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -102,7 +102,7 @@ use_repo(zls, "zls_toolchains") register_toolchains("@zls_toolchains//:all") non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") -use_repo(non_module_deps, "arocc", "com_google_sentencepiece", "flashattn", "linenoise", "mnist", "org_swig_swig", "translate-c", "xla") +use_repo(non_module_deps, "arocc", "com_google_sentencepiece", "flashattn", "linenoise", "mnist", "org_swig_swig", "translate-c", "xla", "stb") xla = use_extension("//third_party/xla:xla.bzl", "xla") use_repo( diff --git a/third_party/stb/BUILD.bazel b/third_party/stb/BUILD.bazel new file mode 100644 index 00000000..f91653e8 --- /dev/null +++ b/third_party/stb/BUILD.bazel @@ -0,0 +1 @@ +exports_files(["stb.BUILD"]) diff --git a/third_party/stb/repo.bzl b/third_party/stb/repo.bzl new file mode 100644 index 00000000..44645c46 --- /dev/null +++ b/third_party/stb/repo.bzl @@ -0,0 +1,9 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + git_repository( + name = "stb", + remote = "https://github.com/nothings/stb.git", + build_file = "//third_party/stb:stb.BUILD", + commit = "f1c79c02822848a9bed4315b12c8c8f3761e1296", + ) diff --git a/third_party/stb/stb.BUILD b/third_party/stb/stb.BUILD new file mode 100644 index 00000000..b59ef43f --- /dev/null +++ b/third_party/stb/stb.BUILD @@ -0,0 +1,35 @@ +cc_import( + name = "stb_image_write", + hdrs = ["stb_image_write.h"], + includes = ["."], + visibility = ["//visibility:public"], +) + +genrule( + name = "impl", + outs = ["impl.c"], + cmd = """\ +cat < $(@) +#include "stb_image.h" +#include "stb_image_resize2.h" +#include "stb_image_write.h" +EOF +""", +) + +cc_library( + name = "stb", + local_defines = [ + "STB_IMAGE_IMPLEMENTATION", + "STB_IMAGE_RESIZE_IMPLEMENTATION", + "STB_IMAGE_WRITE_IMPLEMENTATION", + ], + srcs = [":impl"], + hdrs = [ + "stb_image.h", + "stb_image_resize2.h", + "stb_image_write.h", + ], + includes = ["."], + visibility = ["//visibility:public"], +) From abde0a04ddffad0a2e0b05160ff74b215af835c2 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 10 Mar 2026 11:17:57 +0100 Subject: [PATCH 345/881] zml: sharding (#407) Co-authored-by: Benjamin Rabier --- .zed/settings.json | 20 +- BUILD.bazel | 1 + MODULE.bazel | 1 + bazel/bazel.zig | 5 +- examples/benchmark/main.zig | 28 +- examples/{vfs => io}/BUILD.bazel | 4 +- examples/{vfs/playground.zig => io/main.zig} | 69 +- examples/llama/llama.zig | 84 +- examples/llama/main.zig | 39 +- examples/llama/test.zig | 2 +- examples/mnist/mnist.zig | 14 +- .../BUILD.bazel | 8 +- examples/sharding/main.zig | 255 ++ examples/transfer-benchmark/main.zig | 190 -- mlir/dialects/BUILD.bazel | 1 + mlir/dialects/dialects.zig | 1 + mlir/dialects/func.zig | 3 + mlir/mlir.zig | 6 +- pjrt/pjrt.zig | 268 +- stdx/bounded_array.zig | 3 +- third_party/xla/xla.bzl | 2 + zml/BUILD.bazel | 19 +- zml/attention/attention.zig | 12 +- zml/attention/flashattn.zig | 446 ++-- zml/attention/paged_attention.zig | 69 +- zml/buffer.zig | 278 ++- zml/exe.zig | 59 +- zml/io.zig | 1080 +++++++- zml/module.zig | 264 +- zml/nn.zig | 133 +- zml/ops.zig | 422 +++- zml/pjrtx.zig | 17 +- zml/platform.zig | 227 +- zml/sharding.zig | 2184 +++++++++++++++++ zml/slice.zig | 119 +- zml/tensor.zig | 370 ++- zml/test_runner.zig | 112 - zml/testing.zig | 26 +- zml/tokenizer/BUILD.bazel | 9 +- zml/tokenizer/sentencepiece/BUILD.bazel | 9 +- zml/zml.zig | 4 +- 41 files changed, 5701 insertions(+), 1162 deletions(-) rename examples/{vfs => io}/BUILD.bazel (80%) rename examples/{vfs/playground.zig => io/main.zig} (76%) rename examples/{transfer-benchmark => sharding}/BUILD.bazel (67%) create mode 100644 examples/sharding/main.zig delete mode 100644 examples/transfer-benchmark/main.zig create mode 100644 zml/sharding.zig delete mode 100644 zml/test_runner.zig diff --git a/.zed/settings.json b/.zed/settings.json index 141b19f4..55eff7c8 100644 --- a/.zed/settings.json +++ b/.zed/settings.json @@ -8,12 +8,12 @@ "formatter": "language_server", "code_actions_on_format": { "source.fixAll": true, - "source.organizeImports": true, - }, + "source.organizeImports": true + } }, "YAML": { - "tab_size": 2, - }, + "tab_size": 2 + } }, "file_scan_exclusions": [ "**/.git", @@ -25,17 +25,17 @@ "**/Thumbs.db", "**/.classpath", "**/.settings", - "**/bazel-*", + "**/bazel-*" ], "tab_size": 4, "inlay_hints": { - "enabled": true, + "enabled": true }, "lsp": { "zls": { "binary": { - "path": "tools/zls.sh", - }, - }, - }, + "path": "tools/zls.sh" + } + } + } } diff --git a/BUILD.bazel b/BUILD.bazel index e4e5dd65..872030c6 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -8,6 +8,7 @@ zls_completion( "//examples/transfer-benchmark", "//examples/llama", "//examples/mnist", + "//examples/sharding", "//stdx", "//zml", ], diff --git a/MODULE.bazel b/MODULE.bazel index af85e93e..2f27a5fa 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -117,6 +117,7 @@ use_repo( "python_version_repo", "rules_ml_toolchain", "stablehlo", + "shardy", "sycl_configure", "sycl_configure_ext", "triton", diff --git a/bazel/bazel.zig b/bazel/bazel.zig index 6d5540b0..acbb4356 100644 --- a/bazel/bazel.zig +++ b/bazel/bazel.zig @@ -21,7 +21,7 @@ export const _ linksection(init_array_section) = &struct { .allocator = std.heap.c_allocator, .io = threaded.io(), .argv0 = std.mem.span(argv[0]), - .directory = if (std.c.getenv("RUNFILES_DIRECTORY")) |directory| std.mem.span(directory) else null, + .directory = if (std.c.getenv("RUNFILES_DIR")) |directory| std.mem.span(directory) else null, .manifest = if (std.c.getenv("RUNFILES_MANIFEST_FILE")) |manifest| std.mem.span(manifest) else null, }) catch { std.debug.panic("Unable to find runfiles", .{}); @@ -32,8 +32,7 @@ export const _ linksection(init_array_section) = &struct { }.call; pub fn init(allocator: std.mem.Allocator, io: std.Io, argv: std.process.Args, environ_map: *std.process.Environ.Map) !void { - runfiles_global = try allocator.create(bazel_runfiles.Runfiles); - runfiles_global.* = try bazel_runfiles.Runfiles.create(.{ + runfiles_global = try bazel_runfiles.Runfiles.create(.{ .allocator = allocator, .io = io, .argv = argv, diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index 1f655b55..549a592c 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -5,7 +5,7 @@ const zml = @import("zml"); const stdx = zml.stdx; pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { - return a.dot(b, .k); + return a.dot(b, .k).withPartitioning(.{ .m = .m, .n = .replicated }); } pub fn main(init: std.process.Init) !void { @@ -24,25 +24,37 @@ pub fn main(init: std.process.Init) !void { const platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator); + const logical_mesh: zml.sharding.LogicalMesh = try .init("benchmark_mesh", .{ + .m = .low_bandwidth, + .n = .high_bandwidth, + }); + const strategy: zml.sharding.Strategy = try .suggest(logical_mesh, platform.physical_mesh); + const benchmark_sharding: zml.sharding.Sharding = try .initFromStrategy(platform, logical_mesh, strategy); + const cli_args: CliArgs = stdx.flags.parse(init.minimal.args, CliArgs); - const a: zml.Tensor = .init(.{ .m = cli_args.size, .k = cli_args.size }, cli_args.dtype); - const b: zml.Tensor = .init(.{ .k = cli_args.size, .n = cli_args.size }, cli_args.dtype); + const a_shape = zml.Shape.init(.{ .m = cli_args.size, .k = cli_args.size }, cli_args.dtype) + .withPartitioning(.{ .m = .m, .k = .replicated }); + const b_shape = zml.Shape.init(.{ .k = cli_args.size, .n = cli_args.size }, cli_args.dtype) + .withPartitioning(.{ .k = .replicated, .n = .n }); + + const a: zml.Tensor = .fromShape(a_shape); + const b: zml.Tensor = .fromShape(b_shape); var exe = blk: { log.info("⏱️ Compiling benchmark...", .{}); const now: std.Io.Timestamp = .now(io, .awake); defer log.info("✅ Compiled benchmark [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); - break :blk try platform.compileFn(allocator, io, benchmark, .{ a, b }); + break :blk try platform.compileFn(allocator, io, benchmark, .{ a, b }, .{ .shardings = &.{benchmark_sharding} }); }; defer exe.deinit(); var rng = std.Random.DefaultPrng.init(0); const random = rng.random(); - var a_buffer = try createRandomBuffer(allocator, io, platform, a.shape(), random); + var a_buffer = try createRandomBuffer(allocator, io, platform, a.shape(), benchmark_sharding, random); defer a_buffer.deinit(); - var b_buffer = try createRandomBuffer(allocator, io, platform, b.shape(), random); + var b_buffer = try createRandomBuffer(allocator, io, platform, b.shape(), benchmark_sharding, random); defer b_buffer.deinit(); var exe_args = try exe.args(allocator); @@ -85,7 +97,7 @@ pub fn main(init: std.process.Init) !void { }); } -fn createRandomBuffer(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, shape: zml.Shape, random: std.Random) !zml.Buffer { +fn createRandomBuffer(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, shape: zml.Shape, sharding: zml.sharding.Sharding, random: std.Random) !zml.Buffer { const slice = try zml.Slice.alloc(allocator, shape); defer slice.free(allocator); @@ -110,5 +122,5 @@ fn createRandomBuffer(allocator: std.mem.Allocator, io: std.Io, platform: *const }, } - return .fromSlice(io, platform, slice); + return .fromSlice(io, platform, slice, sharding); } diff --git a/examples/vfs/BUILD.bazel b/examples/io/BUILD.bazel similarity index 80% rename from examples/vfs/BUILD.bazel rename to examples/io/BUILD.bazel index aa3a533f..7d854ac7 100644 --- a/examples/vfs/BUILD.bazel +++ b/examples/io/BUILD.bazel @@ -5,11 +5,11 @@ zig_binary( name = "playground", args = [], data = [], - main = "playground.zig", + main = "main.zig", deps = ["//zml"], ) build_test( name = "test", - targets = [":mnist"], + targets = [":io"], ) diff --git a/examples/vfs/playground.zig b/examples/io/main.zig similarity index 76% rename from examples/vfs/playground.zig rename to examples/io/main.zig index 64b2f084..977b744a 100644 --- a/examples/vfs/playground.zig +++ b/examples/io/main.zig @@ -16,7 +16,7 @@ pub const std_options: std.Options = .{ pub fn main(init: std.process.Init) !void { const allocator = init.gpa; - const Command = enum { cat, ls, cp, stat, realpath, safetensors }; + const Command = enum { cat, ls, cp, stat, realpath, safetensors, load }; var it = init.minimal.args.iterate(); _ = it.next(); // skip program name @@ -149,6 +149,73 @@ pub fn main(init: std.process.Init) !void { try printTensorTree(&stdout_writer.interface, root, "", true, true); try stdout_writer.interface.flush(); }, + .load => { + const ShardingType = enum { replicated, sharded }; + + const sharding_type: ShardingType = std.meta.stringToEnum(ShardingType, it.next() orelse "sharded") orelse return error.InvalidShardingKind; + + const platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator); + + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); + defer registry.deinit(); + + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + defer store.deinit(); + + const AllTensorsModel = struct { + tensors: []zml.Tensor, + }; + + const tensor_count = registry.tensors.count(); + + const tensors = try allocator.alloc(zml.Tensor, tensor_count); + defer allocator.free(tensors); + + var registry_it = registry.iterator(); + var load_count: usize = 0; + while (registry_it.next()) |entry| : (load_count += 1) { + tensors[load_count] = switch (sharding_type) { + .replicated => store.view().createTensor(entry.key_ptr.*, null, null), + .sharded => if (entry.value_ptr.shape.rank() > 0) + store.view().createTensor(entry.key_ptr.*, null, .{ ._0 = .model }) + else + store.view().createTensor(entry.key_ptr.*, null, null), + }; + } + + const model: AllTensorsModel = .{ .tensors = tensors }; + + const replicated_sharding = try zml.sharding.replicatedSharding(platform); + + const sharded_sharding: zml.sharding.Sharding = blk: { + const model_logical_mesh: zml.sharding.LogicalMesh = try .init("playground_model", .{ .model = .high_bandwidth }); + const model_strategy: zml.sharding.Strategy = try .suggest(model_logical_mesh, platform.physical_mesh); + break :blk try .initFromStrategy(platform, model_logical_mesh, model_strategy); + }; + + var progress = std.Progress.start(io, .{ .root_name = "zml.examples.load" }); + progress.increaseEstimatedTotalItems(load_count); + defer progress.end(); + + const now: std.Io.Timestamp = .now(io, .awake); + var total_bytes: usize = 0; + defer { + const took = now.untilNow(io, .awake); + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {D}, {Bi:.2}/s]", .{ total_bytes, stdx.fmt.fmtDuration(took), bytes_per_sec }); + } + + _ = try zml.io.load(AllTensorsModel, &model, init.arena.allocator(), io, platform, .{ + .store = &store, + .shardings = &.{ replicated_sharding, sharded_sharding }, + .parallelism = 16, + .dma_chunks = 16, + .dma_chunk_size = 64 * zml.MiB, + .progress = &progress, + .total_bytes = &total_bytes, + }); + }, } } diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 5073a140..55aac505 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -51,7 +51,7 @@ pub const LlamaLM = struct { config: Config, pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config, options: Options) !LlamaLM { - const lm_head: ?zml.nn.Linear = if (store.withPrefix("lm_head").maybeCreateTensorWithTags("weight", .{ .dout, .d })) |weight| + const lm_head: ?zml.nn.Linear = if (store.withPrefix("lm_head").maybeCreateTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated })) |weight| .init(weight, null, .d) else null; @@ -74,6 +74,7 @@ pub const LlamaLM = struct { io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, + shardings: []const zml.sharding.Sharding, progress: *std.Progress.Node, ) !zml.Bufferized(LlamaLM) { progress.increaseEstimatedTotalItems(store.view().count()); @@ -89,6 +90,7 @@ pub const LlamaLM = struct { .dma_chunk_size = 128 * zml.MiB, .progress = progress, .store = store, + .shardings = shardings, .parallelism = 16, .total_bytes = &total_bytes, }); @@ -167,8 +169,8 @@ pub const Llama = struct { } return .{ - .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight") }, - .norm = .{ .weight = store.withPrefix("norm").createTensorWithTags("weight", .{.d}), .eps = config.rms_norm_eps }, + .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight", .{ .voc, .d }, .{ .voc = .replicated, .d = .model }) }, + .norm = .{ .weight = store.withPrefix("norm").createTensor("weight", .{.d}, .{ .d = .replicated }), .eps = config.rms_norm_eps }, .layers = layers, }; } @@ -247,13 +249,19 @@ pub const TransformerLayer = struct { //log.debug("TransformerLayer({f}) -> {f}", .{ x0, self.input_layernorm.forward(x0) }); stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {f}", .{x0}); - const x0_normalized = self.input_layernorm.forward(x0); + // Keep the residual stream replicated to avoid repeated gathers before q/k/v. + const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const x0_normalized = self.input_layernorm.forward(x0_replicated); const delta0, const updated_kv_cache = self.self_attn.forward(x0_normalized, token_index, kv_cache, attention_metadata, attention_parameters); - const x1 = x0.add(delta0); + const x1 = x0_replicated.add(delta0).withPartitioning(.{ .d = .replicated }); // Fully Connected const x1_normalized = self.post_attention_layernorm.forward(x1); - const x2 = self.mlp.forward(x1_normalized).add(x1).rename(.{ .dout = .d }); + const x2 = self.mlp.forward(x1_normalized) + .rename(.{ .dout = .d }) + .withPartitioning(.{ .d = .replicated }) + .add(x1) + .withPartitioning(.{ .d = .replicated }); return .{ x2.reuseBuffer(x0), updated_kv_cache }; } @@ -264,7 +272,7 @@ const RmsNorm = struct { eps: f32 = 1e-5, pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { - return .{ .weight = store.createTensorWithTags("weight", .{.d}), .eps = eps }; + return .{ .weight = store.createTensor("weight", .{.d}, .{ .d = .replicated }), .eps = eps }; } pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { @@ -284,15 +292,11 @@ const Mlp = struct { gate_proj: zml.nn.Linear, // (dim -> hidden_dim) down_proj: zml.nn.Linear, // (hidden_dim -> dim) - fn initLinear(store: zml.io.TensorStore.View) zml.nn.Linear { - return .init(store.createTensorWithTags("weight", .{ .dout, .d }), store.maybeCreateTensorWithTags("bias", .{.dout}), .d); - } - pub fn init(store: zml.io.TensorStore.View) Mlp { return .{ - .up_proj = initLinear(store.withPrefix("up_proj")), - .gate_proj = initLinear(store.withPrefix("gate_proj")), - .down_proj = initLinear(store.withPrefix("down_proj")), + .up_proj = .init(store.createTensor("up_proj.weight", .{ .dout, .d }, .{ .dout = .model }), null, .d), + .gate_proj = .init(store.createTensor("gate_proj.weight", .{ .dout, .d }, .{ .dout = .model }), null, .d), + .down_proj = .init(store.createTensor("down_proj.weight", .{ .dout, .d }, .{ .d = .model }), null, .d), }; } @@ -326,16 +330,12 @@ pub const SelfAttn = struct { num_kv_heads: i64 = 0, rope_opts: zml.nn.RopeOpts = undefined, - fn initProj(store: zml.io.TensorStore.View) zml.nn.Linear { - return .init(store.createTensorWithTags("weight", .{ .dout, .d }), store.maybeCreateTensorWithTags("bias", .{.dout}), .d); - } - pub fn init(store: zml.io.TensorStore.View, config: LlamaLM.Config) !SelfAttn { return .{ - .q_proj = initProj(store.withPrefix("q_proj")), - .k_proj = initProj(store.withPrefix("k_proj")), - .v_proj = initProj(store.withPrefix("v_proj")), - .o_proj = initProj(store.withPrefix("o_proj")), + .q_proj = .init(store.createTensor("q_proj.weight", .{ .dout, .d }, .{ .dout = .model }), null, .d), + .k_proj = .init(store.createTensor("k_proj.weight", .{ .dout, .d }, .{ .dout = .model }), null, .d), + .v_proj = .init(store.createTensor("v_proj.weight", .{ .dout, .d }, .{ .dout = .model }), null, .d), + .o_proj = .init(store.createTensor("o_proj.weight", .{ .dout, .d }, .{ .d = .model }), null, .d), // TODO(Corentin): fix that .q_norm = null, .k_norm = null, @@ -379,9 +379,16 @@ pub const SelfAttn = struct { attention_parameters: zml.attention.attention.Parameters, ) struct { Tensor, KvCache } { const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; - var q = self.q_proj.forward(x).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); - var k = self.k_proj.forward(x).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); - var v = self.v_proj.forward(x).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); + // Make hidden state replicated once and reuse it across q/k/v projections. + // This avoids paying gather-style collectives independently for each projection. + const x_qkv = x.withPartitioning(.{ .d = .replicated }); + + var q = self.q_proj.forward(x_qkv).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); + var k = self.k_proj.forward(x_qkv).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); + var v = self.v_proj.forward(x_qkv).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); + q = q.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); + k = k.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); + v = v.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); // In self-attention, .s axis is used both for keys and queries. const pos_index = b: { @@ -396,16 +403,25 @@ pub const SelfAttn = struct { q = q.rename(.{ .s = .q }); k = k.rename(.{ .s = .k }); v = v.rename(.{ .s = .k }); + q = q.withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }); + k = k.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); + v = v.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); const dtype = q.dtype(); const new_kv_cache = kv_cache.update(k, v, token_index); k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); + k = k.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); + v = v.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); - const attn_output = zml.attention.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters); + const attn_output = zml.attention.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters) + .withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }); const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); - return .{ self.o_proj.forward(attn), new_kv_cache }; + const delta = self.o_proj.forward(attn) + .rename(.{ .dout = .d }) + .withPartitioning(.{ .d = .replicated }); + return .{ delta, new_kv_cache }; } }; @@ -415,9 +431,11 @@ pub const KvCache = struct { layer_index: Tensor, pub fn init(kv_shape: zml.Shape) KvCache { + const sharded_shape = kv_shape.withPartitioning(.{ .h = .model }); + return .{ - .k = .fromShape(kv_shape), - .v = .fromShape(kv_shape), + .k = .fromShape(sharded_shape), + .v = .fromShape(sharded_shape), .layer_index = .init(.{}, .u32), }; } @@ -430,11 +448,11 @@ pub const KvCache = struct { }; } - pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { + pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !zml.Bufferized(KvCache) { return .{ - .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), - .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}), - .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), + .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), sharding, .{}), + .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), sharding, .{}), + .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32, sharding), }; } diff --git a/examples/llama/main.zig b/examples/llama/main.zig index e16ee303..662466f4 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -134,10 +134,15 @@ pub fn main(init: std.process.Init) !void { .hd = config.head_dim orelse @divExact(config.hidden_size, config.num_attention_heads), }, dtype)), .rng = .init(), - .attention_metadata = .init(.fromBackend(backend, @intCast(args.seqlen))), + .attention_metadata = .init(.fromBackend(backend, @intCast(args.seqlen), @intCast(config.num_attention_heads))), .attention_parameters = .init(.fromBackend(backend)), }; + const tp_mesh: zml.sharding.LogicalMesh = try .init("tp_mesh", .{ .model = .high_bandwidth }); + const tp_strategy: zml.sharding.Strategy = try .suggest(tp_mesh, platform.physical_mesh); + const sharding_tp: zml.sharding.Sharding = try .initFromStrategy(platform, tp_mesh, tp_strategy); + const sharding_replicated = try zml.sharding.replicatedSharding(platform); + var progress = std.Progress.start(io, .{ .root_name = args.model }); var tokenizer_future = try io.concurrent(loadTokenizer, .{ allocator, io, repo, &progress }); @@ -146,7 +151,7 @@ pub fn main(init: std.process.Init) !void { v.deinit(); } - var compiled_model_result_future = try io.concurrent(compileModel, .{ allocator, io, platform, llama_model, llama_parameters, &progress }); + var compiled_model_result_future = try io.concurrent(compileModel, .{ allocator, io, platform, llama_model, llama_parameters, &.{sharding_tp}, &progress }); defer if (compiled_model_result_future.cancel(io)) |v| { v.prefill_exe.deinit(); v.decode_exe.deinit(); @@ -161,6 +166,7 @@ pub fn main(init: std.process.Init) !void { io, platform, &store, + &.{sharding_tp}, &progress, }); defer blk: { @@ -174,10 +180,10 @@ pub fn main(init: std.process.Init) !void { progress.end(); log.info("Creating KvCache", .{}); - var kv_cache_buffers = try llama_parameters.kv_cache.initBuffer(io, platform); + var kv_cache_buffers = try llama_parameters.kv_cache.initBuffer(io, platform, sharding_tp); defer llama.KvCache.deinitBuffer(&kv_cache_buffers); - var attention_metadata_buffers: zml.Bufferized(zml.attention.attention.Metadata) = try llama_parameters.attention_metadata.initBuffer(io, platform); + var attention_metadata_buffers: zml.Bufferized(zml.attention.attention.Metadata) = try llama_parameters.attention_metadata.initBuffer(io, platform, sharding_tp); defer zml.attention.attention.Metadata.deinitBuffer(&attention_metadata_buffers); const tokenizer = try tokenizer_future.await(io); @@ -208,6 +214,7 @@ pub fn main(init: std.process.Init) !void { false, &stdout.interface, platform, + sharding_replicated, ); } @@ -273,6 +280,7 @@ fn compileModel( platform: *const zml.Platform, llama_model: llama.LlamaLM, parameters: LlamaParameters, + shardings: []const zml.sharding.Sharding, progress: *std.Progress.Node, ) !CompileModelResult { const now: std.Io.Timestamp = .now(io, .awake); @@ -280,7 +288,7 @@ fn compileModel( // Compile the model twice, one for prefill, one for generation. var prefill_future = try io.concurrent(struct { - fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, llama_model_: llama.LlamaLM, parameters_: LlamaParameters, progress_: *std.Progress.Node) !zml.Exe { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, llama_model_: llama.LlamaLM, parameters_: LlamaParameters, shardings_: []const zml.sharding.Sharding, progress_: *std.Progress.Node) !zml.Exe { progress_.increaseEstimatedTotalItems(1); var node_ = progress_.start("Compiling prefill...", 1); defer node_.end(); @@ -293,15 +301,15 @@ fn compileModel( parameters_.rng, parameters_.attention_metadata, parameters_.attention_parameters, - }); + }, .{ .shardings = shardings_ }); } - }.call, .{ allocator, io, platform, llama_model, parameters, progress }); + }.call, .{ allocator, io, platform, llama_model, parameters, shardings, progress }); errdefer if (prefill_future.cancel(io)) |v| { v.deinit(); } else |_| {}; var decode_future = try io.concurrent(struct { - fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, llama_model_: llama.LlamaLM, parameters_: LlamaParameters, progress_: *std.Progress.Node) !zml.Exe { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, llama_model_: llama.LlamaLM, parameters_: LlamaParameters, shardings_: []const zml.sharding.Sharding, progress_: *std.Progress.Node) !zml.Exe { progress_.increaseEstimatedTotalItems(1); var node_ = progress_.start("Compiling decode...", 1); defer node_.end(); @@ -314,9 +322,9 @@ fn compileModel( parameters_.rng, parameters_.attention_metadata, parameters_.attention_parameters, - }); + }, .{ .shardings = shardings_ }); } - }.call, .{ allocator, io, platform, llama_model, parameters, progress }); + }.call, .{ allocator, io, platform, llama_model, parameters, shardings, progress }); errdefer if (decode_future.cancel(io)) |v| { v.deinit(); } else |_| {}; @@ -370,6 +378,7 @@ pub fn generateText( skip_llama3_encoding: bool, writer: *std.Io.Writer, platform: *const zml.Platform, + replicated_sharding: zml.sharding.Sharding, ) !void { const prompt_tok: []const u32 = try tokenizePrompt(allocator, tokenizer, config, prompt, skip_llama3_encoding); defer allocator.free(prompt_tok); @@ -380,7 +389,7 @@ pub fn generateText( const max_seq_len = options.max_seq_len; // init RNG and buffers - var rng_buffers = try zml.Tensor.Rng.initBuffer(platform, seed, io); + var rng_buffers = try zml.Tensor.Rng.initBuffer(platform, seed, io, replicated_sharding); defer zml.Tensor.Rng.deinitBuffer(&rng_buffers); var generated_token_slice: zml.Slice = try .alloc(allocator, zml.Shape.init(.{ .s = 1 }, .u32)); defer generated_token_slice.free(allocator); @@ -397,9 +406,9 @@ pub fn generateText( defer prefill_tokens_slice.free(allocator); @memcpy(prefill_tokens_slice.items(u32)[0..prompt_tok.len], prompt_tok); - var prefill_tokens_buffer: zml.Buffer = try .fromSlice(io, platform, prefill_tokens_slice); + var prefill_tokens_buffer: zml.Buffer = try .fromSlice(io, platform, prefill_tokens_slice, replicated_sharding); defer prefill_tokens_buffer.deinit(); - var prefill_token_pos_buffer = try zml.Buffer.scalar(io, platform, 0, .u32); + var prefill_token_pos_buffer = try zml.Buffer.scalar(io, platform, 0, .u32, replicated_sharding); defer prefill_token_pos_buffer.deinit(); prefill_args.set(.{ llama_buffers, prefill_tokens_buffer, prefill_token_pos_buffer, kv_cache_buffers, rng_buffers, attention_metadata_buffers }); @@ -424,7 +433,7 @@ pub fn generateText( defer decode_results.deinit(allocator); // start with the token generated based on the full prompt. - var current_token_buffer: zml.Buffer = try .fromSlice(io, platform, generated_token_slice); + var current_token_buffer: zml.Buffer = try .fromSlice(io, platform, generated_token_slice, replicated_sharding); defer current_token_buffer.deinit(); const output_tokens_len = max_seq_len - prompt_tok.len - 1; @@ -455,7 +464,7 @@ pub fn generateText( // current token pos needs to go into a zml.Buffer const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(prompt_tok.len + i)})); - var token_pos_buffer: zml.Buffer = try .fromSlice(io, platform, token_pos_slice); + var token_pos_buffer: zml.Buffer = try .fromSlice(io, platform, token_pos_slice, replicated_sharding); defer token_pos_buffer.deinit(); // call to generate the next token diff --git a/examples/llama/test.zig b/examples/llama/test.zig index 6c4ebc84..892c76f4 100644 --- a/examples/llama/test.zig +++ b/examples/llama/test.zig @@ -103,7 +103,7 @@ pub fn asyncMain() !void { } fn testImplementation( - platform: zml.Platform, + platform: *const zml.Platform, llama: LlamaLM, llama_weights: zml.Bufferized(LlamaLM), buffer_store: zml.aio.BufferStore, diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 696de3af..6ec5cf9e 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -19,8 +19,8 @@ const Mnist = struct { pub fn init(store: zml.io.TensorStore.View) Layer { return .{ - .weight = store.createTensorWithTags("weight", .{ .d_out, .d }), - .bias = store.createTensorWithTags("bias", .{.d_out}), + .weight = store.createTensor("weight", .{ .d_out, .d }, null), + .bias = store.createTensor("bias", .{.d_out}, null), }; } @@ -42,9 +42,11 @@ const Mnist = struct { io: std.Io, platform: *const zml.Platform, store: *const zml.io.TensorStore, + shardings: []const zml.sharding.Sharding, ) !zml.Bufferized(Mnist) { return zml.io.load(Mnist, self, allocator, io, platform, .{ .store = store, + .shardings = shardings, .parallelism = 1, .dma_chunks = 1, .dma_chunk_size = 16 * 1024 * 1024, @@ -92,13 +94,15 @@ pub fn main(init: std.process.Init) !void { const platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator); + const replicated_sharding = try zml.sharding.replicatedSharding(platform); + // // Compile model const input: zml.Tensor = .init(.{ 28, 28 }, .u8); var exe = blk: { log.info("Compiling model....", .{}); const start: std.Io.Timestamp = .now(io, .awake); defer log.info("✅ Compiled model [{D}]", .{stdx.fmt.fmtDuration(start.untilNow(io, .awake))}); - break :blk try platform.compile(allocator, io, mnist_model, .forward, .{input}); + break :blk try platform.compile(allocator, io, mnist_model, .forward, .{input}, .{ .shardings = &.{replicated_sharding} }); }; defer exe.deinit(); @@ -109,7 +113,7 @@ pub fn main(init: std.process.Init) !void { defer log.info("✅ Transferred weights [{D}]", .{ stdx.fmt.fmtDuration(start.untilNow(io, .awake)), }); - break :blk try mnist_model.load(allocator, io, platform, &store); + break :blk try mnist_model.load(init.arena.allocator(), io, platform, &store, &.{replicated_sharding}); }; defer Mnist.unloadBuffers(&mnist_buffers); @@ -133,7 +137,7 @@ pub fn main(init: std.process.Init) !void { var sample: [28 * 28]u8 align(16) = undefined; _ = try dataset.readPositionalAll(io, &sample, 16 + (idx * 28 * 28)); - var input_buffer: zml.Buffer = try .fromSlice(io, platform, zml.Slice.init(input.shape(), &sample)); + var input_buffer: zml.Buffer = try .fromSlice(io, platform, zml.Slice.init(input.shape(), &sample), replicated_sharding); defer input_buffer.deinit(); printDigit(sample); diff --git a/examples/transfer-benchmark/BUILD.bazel b/examples/sharding/BUILD.bazel similarity index 67% rename from examples/transfer-benchmark/BUILD.bazel rename to examples/sharding/BUILD.bazel index 20082520..81f6de89 100644 --- a/examples/transfer-benchmark/BUILD.bazel +++ b/examples/sharding/BUILD.bazel @@ -2,13 +2,15 @@ load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@rules_zig//zig:defs.bzl", "zig_binary") zig_binary( - name = "transfer-benchmark", + name = "sharding", + args = [], + data = [], main = "main.zig", - deps = ["//zml", "//bazel"], + deps = ["//zml"], visibility = ["//visibility:public"], ) build_test( name = "test", - targets = [":transfer-benchmark"], + targets = [":sharding"], ) diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig new file mode 100644 index 00000000..7e674b5b --- /dev/null +++ b/examples/sharding/main.zig @@ -0,0 +1,255 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const log = std.log.scoped(.sharding); + +pub const std_options: std.Options = .{ + .log_level = .info, + .log_scope_levels = &.{ + .{ .scope = .@"zml/module", .level = .debug }, + }, +}; + +const CliMesh = enum { + auto, + mock, +}; + +const CliArgs = struct { + partitioner: zml.sharding.Partitioning.Partitioner = .shardy, + mesh: CliMesh = .auto, +}; + +const DemoModel = struct { + w: zml.Tensor, // {feature, hidden} + b: zml.Tensor, // {hidden} + + pub fn init(w: zml.Tensor, b: zml.Tensor) DemoModel { + return .{ .w = w, .b = b }; + } + + pub fn forward(self: DemoModel, input: zml.Tensor) zml.Tensor { + const x = input.convert(.f32); + + var y = x.dot(self.w, .feature); + y = y.add(self.b.broad(y.shape())); + y = y.withPartitioning(.{ .batch = .data, .hidden = .model }); + y.print("dense_out"); + + const gate = y.scale(0.01).sigmoid(); + return zml.ops.manualComputation( + .{ y, gate }, + y.shape(), + {}, + (struct { + fn body(_: void, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output_sharded: zml.Shape) zml.Tensor { + const local_result = sharded_inputs[0].relu().mul(sharded_inputs[1]); + return local_result.reshape(output_sharded); + } + }).body, + ); + } +}; + +fn usage() void { + std.debug.print( + \\Usage: sharding [--partitioner=shardy|gspmd] [--mesh=auto|mock] + \\ + \\Simple sharding demo: + \\ - explicit partitioning on data/model axes + \\ - shard-local tensor print + \\ - manualComputation with multiple inputs and one output + \\ + \\Mesh modes: + \\ - auto: physical_mesh=auto, inferred cpu device_count=2 + \\ - mock: physical_mesh=mock topology, inferred cpu device_count=9 + \\ + , .{}); +} + +fn parseArgs(init: std.process.Init) !CliArgs { + var args: CliArgs = .{}; + var it = init.minimal.args.iterate(); + _ = it.next(); // program name + + while (it.next()) |arg| { + if (std.mem.eql(u8, arg, "-h") or std.mem.eql(u8, arg, "--help")) { + usage(); + std.process.exit(0); + } else if (std.mem.startsWith(u8, arg, "--partitioner=")) { + const value = arg["--partitioner=".len..]; + if (std.mem.eql(u8, value, "shardy")) { + args.partitioner = .shardy; + } else if (std.mem.eql(u8, value, "gspmd")) { + args.partitioner = .gspmd; + } else { + std.debug.print("error: unknown partitioner '{s}'\n\n", .{value}); + usage(); + return error.InvalidPartitioner; + } + } else if (std.mem.startsWith(u8, arg, "--mesh=")) { + const value = arg["--mesh=".len..]; + if (std.mem.eql(u8, value, "auto")) { + args.mesh = .auto; + } else if (std.mem.eql(u8, value, "mock")) { + args.mesh = .mock; + } else { + std.debug.print("error: unknown mesh mode '{s}'\n\n", .{value}); + usage(); + return error.InvalidMeshMode; + } + } else { + std.debug.print("error: unknown argument '{s}'\n\n", .{arg}); + usage(); + return error.InvalidArgument; + } + } + return args; +} + +fn buildMockMesh( + allocator: std.mem.Allocator, + target: zml.Target, + devices: []const zml.platform.Device, +) !zml.sharding.PhysicalMesh { + if (devices.len < 8) return error.NotEnoughDevicesForMockMesh; + const topology: zml.sharding.PhysicalMesh.Tree = .axis(.link_x, .{ .mesh = .torus }, &.{ + .axis(.link_y, .{ .mesh = .torus }, &.{ + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[3]), .device(devices[1]), + }), + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[2]), .device(devices[0]), + }), + }), + .axis(.link_y, .{ .mesh = .torus }, &.{ + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[4]), .device(devices[5]), + }), + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[6]), .device(devices[7]), + }), + }), + }); + return zml.sharding.PhysicalMesh.fromTree(allocator, target, topology); +} + +fn createSequenceBuffer( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + shape: zml.Shape, + sharding: zml.sharding.Sharding, + start: f32, +) !zml.Buffer { + const slice = try zml.Slice.alloc(allocator, shape); + defer slice.free(allocator); + + for (slice.items(f32), 0..) |*e, i| { + e.* = start + @as(f32, @floatFromInt(i)); + } + + return zml.Buffer.fromSlice(io, platform, slice, sharding); +} + +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const io = init.io; + const args = try parseArgs(init); + + const create_options: zml.platform.CreateOptions = switch (args.mesh) { + .auto => .{}, + .mock => .{ + .cpu = .{ .device_count = 8 }, + .physical_mesh = .{ .custom = buildMockMesh }, + }, + }; + + var platform: *zml.Platform = try .auto(allocator, io, create_options); + defer platform.deinit(allocator); + + log.info("\n{f}", .{platform.fmtVerbose()}); + + var profiler = try platform.profiler(init.arena.allocator()); + if (profiler) |*p| { + try p.start(); + } + + defer { + if (profiler) |*p| { + p.stop() catch {}; + const pb = p.collectData(allocator) catch unreachable; + const file: std.Io.File = std.Io.Dir.createFile(.cwd(), io, "/tmp/profiling.pb", .{}) catch unreachable; + file.writePositionalAll(io, pb, 0) catch unreachable; + allocator.free(pb); + } + } + + log.info("Partitioner: {s}", .{@tagName(args.partitioner)}); + log.info("{f}", .{platform.physical_mesh}); + + const physical_mesh = platform.physical_mesh; + const mesh: zml.sharding.LogicalMesh = try .init("demo_mesh", .{ + .data = .low_bandwidth, + .model = .high_bandwidth, + }); + const strategy: zml.sharding.Strategy = try .suggest(mesh, physical_mesh); + const sharding: zml.sharding.Sharding = try .initFromStrategy(platform, mesh, strategy); + + log.info("{f}", .{mesh}); + log.info("{f}", .{sharding}); + + const input_shape = zml.Shape.init(.{ .batch = 16, .feature = 32 }, .f32) + .withPartitioning(.{ .batch = .data, .feature = .replicated }); + const w_shape = zml.Shape.init(.{ .feature = 32, .hidden = 64 }, .f32) + .withPartitioning(.{ .feature = .replicated, .hidden = .model }); + const b_shape = zml.Shape.init(.{ .hidden = 64 }, .f32) + .withPartitioning(.{ .hidden = .model }); + + const input: zml.Tensor = .fromShape(input_shape); + const w: zml.Tensor = .fromShape(w_shape); + const b: zml.Tensor = .fromShape(b_shape); + const model: DemoModel = .init(w, b); + + var exe = try platform.compile( + allocator, + io, + model, + .forward, + .{input}, + .{ + .partitioner = args.partitioner, + .shardings = &.{sharding}, + }, + ); + defer exe.deinit(); + + var w_buf = try createSequenceBuffer(allocator, io, platform, w_shape, sharding, 0.0); + defer w_buf.deinit(); + var b_buf = try createSequenceBuffer(allocator, io, platform, b_shape, sharding, 100.0); + defer b_buf.deinit(); + var input_buf = try createSequenceBuffer(allocator, io, platform, input_shape, sharding, 1000.0); + defer input_buf.deinit(); + + log.info("input placement: {f}", .{input_buf.placement()}); + log.info("weight placement: {f}", .{w_buf.placement()}); + log.info("bias placement: {f}", .{b_buf.placement()}); + + var exe_args = try exe.args(allocator); + defer exe_args.deinit(allocator); + var exe_results = try exe.results(allocator); + defer exe_results.deinit(allocator); + exe_args.set(.{ w_buf, b_buf, input_buf }); + + exe.call(exe_args, &exe_results); + var out = exe_results.get(zml.Buffer); + defer out.deinit(); + _ = try out.await(io); + + const out_slice = try out.toSliceAlloc(allocator, io); + defer out_slice.free(allocator); + const out_items = out_slice.items(f32); + const preview_len = @min(out_items.len, @as(usize, 16)); + log.info("output preview: {any}", .{out_items[0..preview_len]}); +} diff --git a/examples/transfer-benchmark/main.zig b/examples/transfer-benchmark/main.zig deleted file mode 100644 index 52f5633b..00000000 --- a/examples/transfer-benchmark/main.zig +++ /dev/null @@ -1,190 +0,0 @@ -const std = @import("std"); -const log = std.log; - -const zml = @import("zml"); -const stdx = zml.stdx; - -const Args = struct { - pub const help = - \\ benchmark --model=./model - ; - model: []const u8, -}; - -pub fn main(init: std.process.Init) !void { - const allocator = init.gpa; - - var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); - defer vfs_file.deinit(); - - var http_client: std.http.Client = .{ .allocator = allocator, .io = init.io }; - - try http_client.initDefaultProxies(allocator, init.environ_map); - defer http_client.deinit(); - - var vfs_https: zml.io.VFS.HTTP = try .init(allocator, init.io, &http_client, .https); - defer vfs_https.deinit(); - - var hf_vfs: zml.io.VFS.HF = try .auto(allocator, init.io, &http_client, init.environ_map); - defer hf_vfs.deinit(); - - var s3_vfs: zml.io.VFS.S3 = try .auto(allocator, init.io, &http_client, init.environ_map); - defer s3_vfs.deinit(); - - var vfs: zml.io.VFS = try .init(allocator, init.io); - defer vfs.deinit(); - - try vfs.register("file", vfs_file.io()); - try vfs.register("https", vfs_https.io()); - try vfs.register("hf", hf_vfs.io()); - try vfs.register("s3", s3_vfs.io()); - - const io = vfs.io(); - - const platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator); - const args = stdx.flags.parseProcessArgs(init.minimal, Args); - - var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, args.model); - defer registry.deinit(); - - var store = zml.io.TensorStore.fromRegistry(allocator, ®istry); - defer store.deinit(); - - var progress = std.Progress.start(io, .{ .root_name = args.model }); - defer progress.end(); - var transfer_progress = progress.start("Transferring tensors (MiB)...", @intCast(registry.totalBytes() / zml.MiB)); - defer transfer_progress.end(); - - log.info("⏱️ Running benchmark...", .{}); - - const now: std.Io.Timestamp = .now(io, .awake); - - var total_bytes: usize = 0; - const tensor_count = try transferAll( - allocator, - io, - platform, - .{ - .parallelism = 16, - .store = &store, - .dma_chunks = 4, - .dma_chunk_size = 32 * zml.MiB, - .progress = &transfer_progress, - .total_bytes = &total_bytes, - }, - ); - - const elapsed = now.untilNow(io, .awake); - const elapsed_ns = elapsed.toNanoseconds(); - const elapsed_s = @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_s; - - log.info("✅ Benchmark done!", .{}); - - const transfer_mib = @as(f64, @floatFromInt(total_bytes)) / @as(f64, zml.MiB); - const mib_per_s = transfer_mib / elapsed_s; - log.info("Transferred {d} tensors ({d} MiB) - Elapsed: {D} - {d:.3} MiB/s", .{ - tensor_count, - total_bytes >> 20, - stdx.fmt.fmtDuration(elapsed), - mib_per_s, - }); -} - -fn transferAll( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - opts: zml.io.LoadOpts, -) !usize { - const first_device = platform.devices[0]; // Temporary until sharding is re-exposed - const dma_allocator: zml.mem.DmaAllocator = .init(allocator, &first_device); - var buffer_pool: zml.mem.DynamicBufferPool = .init(opts.dma_chunks, opts.dma_chunk_size); - defer buffer_pool.deinit(dma_allocator.allocator()); - - const keys = opts.store.registry.tensors.keys(); - const buffers = try allocator.alloc(zml.Buffer, keys.len); - const buffers_initialized = try allocator.alloc(bool, keys.len); - @memset(buffers_initialized, false); - defer { - for (buffers, buffers_initialized) |*buffer, is_initialized| { - if (is_initialized) buffer.deinit(); - } - allocator.free(buffers_initialized); - allocator.free(buffers); - } - - var ctx: TransferCtx = .{ - .dma_allocator = dma_allocator.allocator(), - .pinned_buffer_pool = &buffer_pool, - .io = io, - .memory = first_device.memory(.default), - .group = .init(opts.parallelism), - .progress = opts.progress, - .store = opts.store, - .buffers = buffers, - .buffers_initialized = buffers_initialized, - }; - - for (keys, 0..) |key, i| { - ctx.group.concurrent(io, transferOne, .{ i, key, &ctx }) catch unreachable; - } - - ctx.group.await(io) catch unreachable; - - if (opts.total_bytes) |total_bytes_ptr| { - total_bytes_ptr.* = ctx.total.load(.monotonic); - } - - return keys.len; -} - -const TransferCtx = struct { - dma_allocator: std.mem.Allocator, - pinned_buffer_pool: *zml.mem.DynamicBufferPool, - io: std.Io, - memory: *const zml.Memory, - group: stdx.Io.LimitedGroup, - total: std.atomic.Value(usize) = .init(0), - progress: ?*std.Progress.Node, - store: *const zml.io.TensorStore, - buffers: []zml.Buffer, - buffers_initialized: []bool, -}; - -fn transferOne(i: usize, key: []const u8, ctx: *TransferCtx) !void { - var reader = ctx.store.getReader(key, ctx.io, &.{}) catch unreachable; - defer reader.deinit(); - - var memory_writer = zml.io.MemoryWriter.init( - ctx.dma_allocator, - ctx.io, - ctx.memory, - ctx.pinned_buffer_pool, - reader.tensor.shape, - &ctx.buffers[i], - ) catch unreachable; - defer memory_writer.deinit(ctx.dma_allocator); - ctx.buffers_initialized[i] = true; - - const scale = zml.MiB; - const total = if (ctx.progress) |progress| blk: { - const child_total = @max(@as(u64, 1), reader.tensor.shape.byteSize() / scale); - var child = progress.start(reader.tensor.name, child_total); - defer child.end(); - - var progress_writer: zml.io.ProgressWriter = .init(memory_writer.interface(), &child, .{ .scale = scale }); - const written = reader.interface.streamRemaining(&progress_writer.interface) catch unreachable; - progress_writer.interface.flush() catch unreachable; - break :blk written; - } else blk: { - const written = reader.interface.streamRemaining(memory_writer.interface()) catch unreachable; - memory_writer.interface().flush() catch unreachable; - break :blk written; - }; - - const previous = ctx.total.fetchAdd(total, .monotonic); - if (ctx.progress) |progress| { - progress.setCompletedItems((previous + total) / scale); - } -} diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/BUILD.bazel index 26cca662..5c5dd67a 100644 --- a/mlir/dialects/BUILD.bazel +++ b/mlir/dialects/BUILD.bazel @@ -12,6 +12,7 @@ zig_library( "//mlir", "//mlir/dialects/stablehlo", "//stdx", + "@shardy//shardy/integrations/c:sdy_capi", ], ) diff --git a/mlir/dialects/dialects.zig b/mlir/dialects/dialects.zig index 3deac104..4f4780d4 100644 --- a/mlir/dialects/dialects.zig +++ b/mlir/dialects/dialects.zig @@ -1,6 +1,7 @@ const std = @import("std"); pub const stablehlo = @import("mlir/dialects/stablehlo"); + pub const func = @import("func.zig"); test { diff --git a/mlir/dialects/func.zig b/mlir/dialects/func.zig index b9f8b7f5..dd881648 100644 --- a/mlir/dialects/func.zig +++ b/mlir/dialects/func.zig @@ -18,6 +18,7 @@ const FuncOpArgs = struct { location: *const mlir.Location, no_inline: bool = false, visibility: Visibility = .public, + verify: bool = true, }; pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { @@ -62,9 +63,11 @@ pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { .blocks = &.{args.block}, .attributes = attr_tuples_buffer.constSlice(), .location = args.location, + .verify = args.verify, }); } + pub fn call(ctx: *mlir.Context, name: []const u8, values: []const *const mlir.Value, results: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "func.call", .{ .operands = .{ .variadic = &.{values} }, diff --git a/mlir/mlir.zig b/mlir/mlir.zig index e451bdb2..14376974 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -1030,7 +1030,7 @@ pub const Operation = opaque { } pub fn name(self: *const Operation) []const u8 { - return string(c.mlirOperationGetName(self.ptr())); + return string(c.mlirIdentifierStr(c.mlirOperationGetName(self.ptr()))); } pub fn clone(self: *const Operation) *Operation { @@ -1612,8 +1612,8 @@ pub const MemRefType = opaque { element_type.ptr(), @intCast(shape.len), @ptrCast(shape), - if (layout) |l| l.ptr() else .{}, - if (memory_space) |m| m.ptr() else .{}, + if (layout) |l| l.ptr() else .{ .ptr = null }, + if (memory_space) |m| m.ptr() else .{ .ptr = null }, ).ptr); } diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 073691e5..81b5ccd3 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -180,7 +180,6 @@ pub const Api = struct { pub const Type = enum(c.PJRT_Extension_Type) { custom_call = c.PJRT_Extension_Type_Gpu_Custom_Call, profiler = c.PJRT_Extension_Type_Profiler, - custom_partitioner = c.PJRT_Extension_Type_Custom_Partitioner, stream = c.PJRT_Extension_Type_Stream, layouts = c.PJRT_Extension_Type_Layouts, ffi = c.PJRT_Extension_Type_FFI, @@ -194,7 +193,6 @@ pub const Api = struct { pub const Extension = union(Type) { custom_call: *const c.PJRT_Gpu_Custom_Call, profiler: *const c.PJRT_Profiler_Extension, - custom_partitioner: *const c.PJRT_Custom_Partitioner_Extension, stream: *const c.PJRT_Stream_Extension, layouts: *const c.PJRT_Layouts_Extension, ffi: *const c.PJRT_FFI_Extension, @@ -254,7 +252,7 @@ pub const Api = struct { if (state.str) |str| { return str; } - if (self.getPluginAttribute("stablehlo_current_version")) |nv| { + if (self.pluginAttribute("stablehlo_current_version")) |nv| { switch (nv.value()) { .int64list => |v| { state.str = std.fmt.bufPrintZ(&state.buf, "{d}.{d}.{d}", .{ v[0], v[1], v[2] }) catch unreachable; @@ -277,8 +275,16 @@ pub const Api = struct { return null; } - fn getPluginAttribute(api: *const Api, key: []const u8) ?NamedValue { - const attributes = api.getPluginAttributes(); + pub fn profiler(self: *const Api, options_pb: []const u8) ApiError!?Profiler { + if (self.extension(.profiler)) |ext| { + return try Profiler.init(self, ext.profiler, options_pb); + } + + return null; + } + + pub fn pluginAttribute(api: *const Api, key: []const u8) ?NamedValue { + const attributes = api.pluginAttributes(); for (attributes) |attr| { if (std.mem.eql(u8, attr.name(), key)) { return attr; @@ -287,7 +293,7 @@ pub const Api = struct { return null; } - fn getPluginAttributes(api: *const Api) []const NamedValue { + pub fn pluginAttributes(api: *const Api) []const NamedValue { const ret = api.call(.PJRT_Plugin_Attributes, .{ .extension_start = null, }) catch unreachable; @@ -466,6 +472,7 @@ pub const Client = opaque { buffer_type: BufferType, dims: []const i64, byte_strides: ?[]const i64, + layout: MemoryLayout, host_buffer_semantics: HostBufferSemantics, dst: union(enum) { device: *const Device, @@ -482,6 +489,7 @@ pub const Client = opaque { .num_dims = args.dims.len, .byte_strides = if (args.byte_strides) |bs| @ptrCast(@constCast(bs)) else null, .num_byte_strides = if (args.byte_strides) |bs| bs.len else 0, + .device_layout = @ptrCast(@constCast(&args.layout.toCStruct())), .host_buffer_semantics = @intFromEnum(args.host_buffer_semantics), .device = if (args.dst == .device) @ptrCast(@constCast(args.dst.device)) else null, .memory = if (args.dst == .memory) @ptrCast(@constCast(args.dst.memory)) else null, @@ -575,6 +583,63 @@ pub const Client = opaque { return @ptrCast(ret.transfer_manager.?); } + pub fn defaultMemoryLayout( + self: *const Client, + api: *const Api, + element_type: BufferType, + dims: []const i64, + ) !DefaultMemoryLayout { + const ext = if (api.extension(.layouts)) |e| e.layouts else return error.LayoutsExtensionUnavailable; + + var get_args: c.PJRT_Layouts_PJRT_Client_GetDefaultLayout_Args = .{ + .struct_size = c.PJRT_Layouts_PJRT_Client_GetDefaultLayout_Args_STRUCT_SIZE, + .extension_start = null, + .client = self.inner(), + .type = @intFromEnum(element_type), + .dims = dims.ptr, + .num_dims = dims.len, + .layout = null, + }; + if (ext.PJRT_Layouts_PJRT_Client_GetDefaultLayout.?(&get_args)) |pjrt_c_error| { + const pjrt_error: *Error = @ptrCast(pjrt_c_error); + defer pjrt_error.deinit(api); + return pjrt_error.getCode(api).toApiError(); + } + defer { + var destroy_args: c.PJRT_Layouts_MemoryLayout_Destroy_Args = .{ + .struct_size = c.PJRT_Layouts_MemoryLayout_Destroy_Args_STRUCT_SIZE, + .extension_start = null, + .layout = get_args.layout, + }; + if (ext.PJRT_Layouts_MemoryLayout_Destroy.?(&destroy_args)) |pjrt_c_error| { + const pjrt_error: *Error = @ptrCast(pjrt_c_error); + defer pjrt_error.deinit(api); + log.err("[PJRT_Layouts_MemoryLayout_Destroy] {s}", .{pjrt_error.getMessage(api)}); + } + } + + var serialize_args: c.PJRT_Layouts_MemoryLayout_Serialize_Args = .{ + .struct_size = c.PJRT_Layouts_MemoryLayout_Serialize_Args_STRUCT_SIZE, + .extension_start = null, + .layout = get_args.layout, + .serialized_bytes = null, + .serialized_bytes_size = 0, + .serialized_layout = null, + .serialized_layout_deleter = null, + }; + if (ext.PJRT_Layouts_MemoryLayout_Serialize.?(&serialize_args)) |pjrt_c_error| { + const pjrt_error: *Error = @ptrCast(pjrt_c_error); + defer pjrt_error.deinit(api); + return pjrt_error.getCode(api).toApiError(); + } + defer if (serialize_args.serialized_layout_deleter) |deleter| { + deleter(serialize_args.serialized_layout); + }; + + const serialized = serialize_args.serialized_bytes[0..serialize_args.serialized_bytes_size]; + return try DefaultMemoryLayout.parseSerialized(serialized); + } + pub const CreateUninitializedBufferArgs = struct { dims: []const i64, element_type: BufferType, @@ -741,6 +806,9 @@ pub const DeviceDescription = opaque { const ret = api.call(.PJRT_DeviceDescription_Attributes, .{ .device_description = self.inner(), }) catch unreachable; + + if (ret.attributes == null) return &.{}; + return @ptrCast(ret.attributes[0..ret.num_attributes]); } @@ -993,6 +1061,99 @@ pub const MemoryLayout = union(MemoryLayoutType) { } }; +pub const DefaultMemoryLayout = struct { + pub const MAX_MINOR_TO_MAJOR: usize = 16; + pub const MAX_TILE_DIMS: usize = 128; + pub const MAX_NUM_TILES: usize = 16; + + minor_to_major_storage: [MAX_MINOR_TO_MAJOR]i64 = undefined, + minor_to_major_len: usize = 0, + + tile_dims_storage: [MAX_TILE_DIMS]i64 = undefined, + tile_dims_len: usize = 0, + + tile_dims_sizes_storage: [MAX_NUM_TILES]usize = undefined, + tile_dims_sizes_len: usize = 0, + + pub fn toMemoryLayout(self: *const DefaultMemoryLayout) MemoryLayout { + return .{ + .tiled = .{ + .minor_to_major = self.minor_to_major_storage[0..self.minor_to_major_len], + .tile_dims = self.tile_dims_storage[0..self.tile_dims_len], + .tile_dims_sizes = self.tile_dims_sizes_storage[0..self.tile_dims_sizes_len], + }, + }; + } + + pub fn parseSerialized(serialized: []const u8) !DefaultMemoryLayout { + var parsed: DefaultMemoryLayout = .{}; + try parseSerializedInto(serialized, &parsed); + return parsed; + } + + fn parseSerializedInto(serialized: []const u8, out: *DefaultMemoryLayout) !void { + out.* = .{}; + const raw = std.mem.trim(u8, serialized, " \t\n\r\x00"); + + if (raw.len < 2 or raw[0] != '{' or raw[raw.len - 1] != '}') { + return error.InvalidLayoutString; + } + + const body = raw[1 .. raw.len - 1]; + const first_colon = std.mem.indexOfScalar(u8, body, ':'); + + const minor_to_major_str = std.mem.trim(u8, if (first_colon) |index| body[0..index] else body, " \t\n\r"); + + if (minor_to_major_str.len > 0) { + var mtm_iter = std.mem.splitScalar(u8, minor_to_major_str, ','); + while (mtm_iter.next()) |part_raw| { + if (out.minor_to_major_len >= MAX_MINOR_TO_MAJOR) return error.LayoutTooLarge; + out.minor_to_major_storage[out.minor_to_major_len] = try parseI64Token(part_raw); + out.minor_to_major_len += 1; + } + } + + if (first_colon) |index| { + const attrs = body[index + 1 ..]; + var cursor: usize = 0; + while (cursor < attrs.len) { + const t_pos = std.mem.indexOfScalarPos(u8, attrs, cursor, 'T') orelse break; + cursor = t_pos + 1; + while (cursor < attrs.len and std.ascii.isWhitespace(attrs[cursor])) : (cursor += 1) {} + if (cursor >= attrs.len or attrs[cursor] != '(') continue; + + while (cursor < attrs.len and attrs[cursor] == '(') { + const tuple_start = cursor + 1; + const tuple_end = std.mem.indexOfScalarPos(u8, attrs, tuple_start, ')') orelse return error.InvalidLayoutString; + const tuple = attrs[tuple_start..tuple_end]; + + var dims_in_tile: usize = 0; + var tuple_iter = std.mem.splitScalar(u8, tuple, ','); + while (tuple_iter.next()) |dim_raw| { + if (out.tile_dims_len >= MAX_TILE_DIMS) return error.LayoutTooLarge; + out.tile_dims_storage[out.tile_dims_len] = try parseI64Token(dim_raw); + out.tile_dims_len += 1; + dims_in_tile += 1; + } + if (dims_in_tile == 0) return error.InvalidLayoutString; + + if (out.tile_dims_sizes_len >= MAX_NUM_TILES) return error.LayoutTooLarge; + out.tile_dims_sizes_storage[out.tile_dims_sizes_len] = dims_in_tile; + out.tile_dims_sizes_len += 1; + + cursor = tuple_end + 1; + } + } + } + } + + fn parseI64Token(token_raw: []const u8) !i64 { + const token = std.mem.trim(u8, token_raw, " \t\n\r"); + if (token.len == 0) return error.InvalidLayoutString; + return std.fmt.parseInt(i64, token, 10); + } +}; + pub const HostBufferSemantics = enum(c.PJRT_HostBufferSemantics) { ImmutableOnlyDuringCall = c.PJRT_HostBufferSemantics_kImmutableOnlyDuringCall, ImmutableUntilTransferCompletes = c.PJRT_HostBufferSemantics_kImmutableUntilTransferCompletes, @@ -1512,3 +1673,98 @@ pub const Ffi = extern struct { } } }; + +pub const Profiler = struct { + pjrt_api: *const Api, + api: *const c.PLUGIN_Profiler_Api, + inner: *c.PLUGIN_Profiler, + + pub fn init(api_: *const Api, prof_ext: *const c.PJRT_Profiler_Extension, options_pb: []const u8) ApiError!Profiler { + var args: c.PLUGIN_Profiler_Create_Args = .{ + .struct_size = meta.structSize(c.PLUGIN_Profiler_Create_Args), + .options = options_pb.ptr, + .options_size = options_pb.len, + .profiler = null, + }; + + const profiler_api: *const c.PLUGIN_Profiler_Api = @ptrCast(prof_ext.profiler_api); + if (profiler_api.create.?(&args)) |err| { + const pjrt_err: *Error = @ptrCast(err); + return pjrt_err.getCode(api_).toApiError(); + } + + return .{ + .pjrt_api = api_, + .api = profiler_api, + .inner = args.profiler.?, + }; + } + + pub fn start(self: *Profiler) ApiError!void { + var args: c.PLUGIN_Profiler_Start_Args = .{ + .struct_size = meta.structSize(c.PLUGIN_Profiler_Start_Args), + .profiler = self.inner, + }; + + if (self.api.start.?(&args)) |err| { + const pjrt_err: *Error = @ptrCast(err); + return pjrt_err.getCode(self.pjrt_api).toApiError(); + } + } + + pub fn stop(self: *Profiler) ApiError!void { + var args: c.PLUGIN_Profiler_Stop_Args = .{ + .struct_size = meta.structSize(c.PLUGIN_Profiler_Stop_Args), + .profiler = self.inner, + }; + + if (self.api.stop.?(&args)) |err| { + const pjrt_err: *Error = @ptrCast(err); + return pjrt_err.getCode(self.pjrt_api).toApiError(); + } + } + + pub fn collectData(self: *Profiler, allocator: std.mem.Allocator) ![]u8 { + var args: c.PLUGIN_Profiler_CollectData_Args = .{ + .struct_size = meta.structSize(c.PLUGIN_Profiler_CollectData_Args), + .profiler = self.inner, + .buffer = null, + .buffer_size_in_bytes = 0, + }; + + if (self.api.collect_data.?(&args)) |err| { + const pjrt_err: *Error = @ptrCast(err); + return pjrt_err.getCode(self.pjrt_api).toApiError(); + } + + if (args.buffer != null) { + var data = args.buffer[0..args.buffer_size_in_bytes]; + if (data.len > 0 and data[data.len - 1] == 0) { + data = data[0 .. data.len - 1]; + } + return allocator.dupe(u8, data); + } + + if (args.buffer_size_in_bytes == 0) return &[_]u8{}; + + const buffer = try allocator.alloc(u8, args.buffer_size_in_bytes); + errdefer allocator.free(buffer); + + args.buffer = buffer.ptr; + if (self.api.collect_data.?(&args)) |err| { + const pjrt_err: *Error = @ptrCast(err); + return pjrt_err.getCode(self.pjrt_api).toApiError(); + } + + // Trim trailing null byte logic + var data = buffer[0..args.buffer_size_in_bytes]; + if (data.len > 0 and data[data.len - 1] == 0) { + data = data[0 .. data.len - 1]; + } + + const pb = try allocator.dupe(u8, data); + allocator.free(buffer); + + return pb; + } +}; diff --git a/stdx/bounded_array.zig b/stdx/bounded_array.zig index bdee5bee..81871615 100644 --- a/stdx/bounded_array.zig +++ b/stdx/bounded_array.zig @@ -393,9 +393,8 @@ test BoundedArray { try testing.expectEqual(a.len, 36); while (a.pop()) |_| {} - const w = a.writer(); const s = "hello, this is a test string"; - try w.writeAll(s); + _ = try a.appendWrite(s); try testing.expectEqualStrings(s, a.constSlice()); } diff --git a/third_party/xla/xla.bzl b/third_party/xla/xla.bzl index 43a6629b..9b8e4303 100644 --- a/third_party/xla/xla.bzl +++ b/third_party/xla/xla.bzl @@ -1,6 +1,7 @@ load("@xla//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls", "tf_vendored") load("@xla//third_party/llvm:workspace.bzl", llvm = "repo") load("@xla//third_party/stablehlo:workspace.bzl", stablehlo = "repo") +load("@xla//third_party/shardy:workspace.bzl", shardy = "repo") load("@xla//third_party/triton:workspace.bzl", triton = "repo") _BZL_HELPERS = """\ @@ -74,6 +75,7 @@ if_sycl_is_configured = always_if_false def _xla_impl(mctx): llvm("llvm-raw") stablehlo() + shardy() triton() tf_http_archive( diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index f037d6b2..fbf6e861 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -12,6 +12,11 @@ upb_c_proto_library( deps = ["@xla//xla/pjrt/proto:compile_options_proto"], ) +upb_c_proto_library( + name = "profiler_options", + deps = ["@tsl//tsl/profiler/protobuf:profiler_options_proto"], +) + cc_library( name = "posix", hdrs = ["posix.h"], @@ -35,6 +40,7 @@ zig_library( "mlirx.zig", "module.zig", "nn.zig", + "sharding.zig", "ops.zig", "pjrtx.zig", "platform.zig", @@ -57,6 +63,7 @@ zig_library( ":xla_compile_options_upb", ":xla_data_upb", "//bazel", + ":profiler_options", "//mlir", "//mlir/dialects", "//pjrt", @@ -74,20 +81,8 @@ zig_library( }), ) -# All ZML Tests zig_test( name = "test", - data = [ - #"aio/torch/simple.pt", - #"aio/torch/simple_test_4.pickle", - ], - test_runner = ":test_runner", deps = ["//zml"], ) - -filegroup( - name = "test_runner", - srcs = ["test_runner.zig"], - visibility = ["//visibility:public"], -) diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index ad747f04..c50756ba 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -67,11 +67,11 @@ pub const Metadata = union(Backend) { cuda_fa2: flashattn.fa2.Metadata.InitOptions, cuda_fa3: flashattn.fa3.Metadata.InitOptions, - pub fn fromBackend(backend: Backend, seqlen: i64) InitOptions { + pub fn fromBackend(backend: Backend, seqlen: i64, num_heads: i64) InitOptions { return switch (backend) { .vanilla => .{ .vanilla = {} }, - .cuda_fa2 => .{ .cuda_fa2 = .{ .seqlen = seqlen } }, - .cuda_fa3 => .{ .cuda_fa3 = .{ .seqlen = seqlen } }, + .cuda_fa2 => .{ .cuda_fa2 = .{ .seqlen = seqlen, .num_heads = num_heads } }, + .cuda_fa3 => .{ .cuda_fa3 = .{ .seqlen = seqlen, .num_heads = num_heads } }, }; } }; @@ -84,11 +84,11 @@ pub const Metadata = union(Backend) { }; } - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !zml.Bufferized(Metadata) { return switch (self) { .vanilla => .{ .vanilla = {} }, - .cuda_fa2 => |v| .{ .cuda_fa2 = try v.initBuffer(io, platform) }, - .cuda_fa3 => |v| .{ .cuda_fa3 = try v.initBuffer(io, platform) }, + .cuda_fa2 => |v| .{ .cuda_fa2 = try v.initBuffer(io, platform, sharding) }, + .cuda_fa3 => |v| .{ .cuda_fa3 = try v.initBuffer(io, platform, sharding) }, }; } diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index bd1979fc..5e232bd3 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -4,11 +4,13 @@ const flashattn = @import("platforms/cuda/flashattn"); const platforms = @import("platforms"); const stdx = @import("stdx"); +const CompilationContext = @import("../module.zig").CompilationContext; const zml = @import("../zml.zig"); const ffi = zml.pjrt.ffi; - const AttentionOptions = @import("paged_attention.zig").AttentionOptions; +const log = std.log.scoped(.@"zml/attention/flashattn"); + pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { if (comptime platforms.isEnabled(.cuda)) { try flashattn.load(allocator, io); @@ -37,15 +39,6 @@ fn flashattnDataTypeFromZmlDataType(dtype: zml.DataType) flashattn.DataType { }; } -fn tensorFromBuffer(buffer: zml.Buffer, platform: zml.Platform) !flashattn.Tensor { - return .init( - try buffer._shards.get(0).getOpaqueDeviceMemoryDataPointer(platform.pjrt_api), - buffer.shape().dims(), - buffer.shape().withDtype(.u8).computeStrides().constSlice(), - flashattnDataTypeFromZmlDataType(buffer.dtype()), - ); -} - const Buffer = struct { shape: zml.Shape, ptr: *anyopaque, @@ -135,7 +128,12 @@ fn fixupKvCacheBuffer(buffer: Buffer, layer_index: i64) Buffer { pub fn Wrapper(comptime T: type, run_func: std.meta.DeclEnum(T)) type { return struct { pub fn register(platform: *const zml.Platform) !void { - try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "cuda", T.run, .{ .command_buffer_compatible = true }); + try platform.registerFfi(.{ + .name = T.custom_call_name, + .platform_name = "cuda", + .handler = T.run, + .traits = .{ .command_buffer_compatible = true }, + }); } pub fn run(call_frame: *ffi.CallFrame) callconv(.c) ?*ffi.Error { @@ -214,8 +212,7 @@ pub const fa2 = struct { pub const Parameters = struct { pub const InitOptions = struct {}; - pub fn init(opts: InitOptions) fa2.Parameters { - _ = opts; // autofix + pub fn init(_: InitOptions) fa2.Parameters { return .{}; } }; @@ -227,21 +224,30 @@ pub const fa2 = struct { pub const InitOptions = struct { seqlen: i64, + num_heads: i64, }; pub fn init(opts: InitOptions) Metadata { return .{ - .softmax_lse = .init(.{1 * 32 * opts.seqlen * 4}, .i8), - .softmax_lse_accum = .init(.{32 * 1 * 32 * 4 * 4}, .i8), - .out_accum = .init(.{32 * 1 * 32 * 4 * 128 * 4}, .i8), + .softmax_lse = .fromShape(zml.Shape.init(.{ opts.seqlen, opts.num_heads, 1 }, .f32) + .withTags(.{ .s, .h, .dummy }) + .withPartitioning(.{ .h = .model })), + + .softmax_lse_accum = .fromShape(zml.Shape.init(.{ 1, opts.num_heads, 128 }, .f32) + .withTags(.{ .dummy, .h, .hd }) + .withPartitioning(.{ .h = .model })), + + .out_accum = .fromShape(zml.Shape.init(.{ opts.seqlen, opts.num_heads, 128 }, .f32) + .withTags(.{ .s, .h, .hd }) + .withPartitioning(.{ .h = .model })), }; } - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !zml.Bufferized(Metadata) { return .{ - .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), .{}), - .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), .{}), - .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), sharding, .{}), + .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), sharding, .{}), + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), sharding, .{}), }; } @@ -252,8 +258,9 @@ pub const fa2 = struct { } }; - pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { - _ = parameters; // autofix + pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, _: Parameters) zml.Tensor { + const ctx = CompilationContext.current(); + stdx.debug.assert(q_.shape().hasTag(.b) == null or q_.dim(.b) == 1, "fa2.attention support for batch size != 1 is not supported yet.", .{}); const seqused_k = token_index.addConstant(q_.dim(.q)).reshape(.{1}); // TODO(Corendos): replace with cumsum @@ -271,7 +278,8 @@ pub const fa2 = struct { break :b [_]zml.Tensor{ q_.rename(.{ .q = .tot }), k_.rename(.{ .k = .tot }), v_.rename(.{ .k = .tot }) }; }; // TODO(Corendos): replace with cumsum - const cu_seqlens_q = zml.Tensor.constantTensor(zml.Shape.init(.{2}, .i32), std.mem.sliceAsBytes(&[2]i32{ 0, max_seqlen_q })); + const cu_seqlens_q = zml.Tensor.constantTensor(zml.Shape.init(.{2}, .i32), std.mem.sliceAsBytes(&[2]i32{ 0, max_seqlen_q })) + .withPartitioning(.{ ._0 = .replicated }); const original_tot = q.dim(.tot); const num_heads = q_.dim(.h); @@ -283,32 +291,42 @@ pub const fa2 = struct { q = q.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .tot, .ngroups, .h, .hd }).merge(.{ .tot = .{ .tot, .ngroups } }); } - var o = zml.ops.customCall( - custom_call_name, + const q_sharded = q.withPartitioning(.{ .h = .model }); + const model_partitions = ctx.partitioning.numPartitionsForLogicalAxis(q_sharded.shape(), .model) catch unreachable; + + const output_shape = q_sharded.shape(); + var o = zml.ops.manualComputation( .{ - q, + q_sharded, k, v, cu_seqlens_q, - cu_seqlens_k, - metadata.softmax_lse, - metadata.softmax_lse_accum, - metadata.out_accum, + cu_seqlens_k.withTags(.{.i}).withPartitioning(.{ .i = .replicated }), + metadata.softmax_lse.withPartitioning(.{ .h = .model }), + metadata.softmax_lse_accum.withPartitioning(.{ .h = .model }), + metadata.out_accum.withPartitioning(.{ .h = .model }), }, - .{q.shape()}, + output_shape, .{ - //.softmax_scale = @as(f32, 1.0), - .is_causal = true, - .window_size_left = @as(i32, -1), - .window_size_right = @as(i32, -1), - .max_seqlen_q = max_seqlen_q, - .max_seqlen_k = max_seqlen_k, - .num_heads = @as(i32, @intCast(num_heads)), - }, - .{ - .output_operand_aliases = &.{0}, - .has_side_effect = true, + .metadata = .{ + //.softmax_scale = @as(f32, 1.0), + .is_causal = true, + .window_size_left = @as(i32, -1), + .window_size_right = @as(i32, -1), + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + .num_heads = @as(i32, @intCast(@divExact(num_heads, model_partitions))), + }, + .opts = zml.ops.CustomCallOptions{ + .output_operand_aliases = &.{0}, + .has_side_effect = false, + }, }, + (struct { + fn body(ctx_: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output: zml.Shape) zml.Tensor { + return zml.ops.customCall(custom_call_name, sharded_inputs, output, ctx_.metadata, ctx_.opts); + } + }).body, ); if (seqlenq_ngroups_swapped) { @@ -411,23 +429,28 @@ pub const fa3 = struct { pub const InitOptions = struct { seqlen: i64, + num_heads: i64, }; pub fn init(opts: InitOptions) Metadata { return .{ - .softmax_lse = .init(.{1 * 32 * opts.seqlen * 4}, .i8), - .softmax_lse_accum = .init(.{32 * 1 * 32 * 4 * 4}, .i8), - .out_accum = .init(.{32 * 1 * 32 * 4 * 128 * 4}, .i8), - .scheduler_metadata = .init(.{2}, .i32), + .softmax_lse = .fromShape(zml.Shape.init(.{opts.num_heads * opts.seqlen * 4}, .i8) + .withTags(.{.h}).withPartitioning(.{ .h = .model })), + .softmax_lse_accum = .fromShape(zml.Shape.init(.{opts.num_heads * 128 * 4}, .i8) + .withTags(.{.h}).withPartitioning(.{ .h = .model })), + .out_accum = .fromShape(zml.Shape.init(.{opts.num_heads * opts.seqlen * 128 * 4}, .i8) + .withTags(.{.h}).withPartitioning(.{ .h = .model })), + .scheduler_metadata = .fromShape(zml.Shape.init(.{2}, .i32) + .withTags(.{.meta}).withPartitioning(.{ .meta = .replicated })), }; } - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !zml.Bufferized(Metadata) { return .{ - .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), .{}), - .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), .{}), - .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), - .scheduler_metadata = try zml.Buffer.uninitialized(io, platform, self.scheduler_metadata.shape(), .{}), + .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), sharding, .{}), + .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), sharding, .{}), + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), sharding, .{}), + .scheduler_metadata = try zml.Buffer.uninitialized(io, platform, self.scheduler_metadata.shape(), sharding, .{}), }; } @@ -439,8 +462,7 @@ pub const fa3 = struct { } }; - pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { - _ = parameters; // autofix + pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, _: Parameters) zml.Tensor { stdx.debug.assert(q_.shape().hasTag(.b) == null or q_.dim(.b) == 1, "fa3.attention support for batch size != 1 is not supported yet.", .{}); const seqused_k = token_index.addConstant(q_.dim(.q)).reshape(.{1}); // TODO(Corendos): replace with cumsum @@ -454,7 +476,8 @@ pub const fa3 = struct { const k = k_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); const v = v_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); // TODO(Corendos): replace with cumsum - const cu_seqlens_q = zml.Tensor.constantTensor(zml.Shape.init(.{2}, .i32), std.mem.sliceAsBytes(&[2]i32{ 0, max_seqlen_q })); + const cu_seqlens_q = zml.Tensor.constantTensor(zml.Shape.init(.{2}, .i32), std.mem.sliceAsBytes(&[2]i32{ 0, max_seqlen_q })) + .withPartitioning(.{ ._0 = .replicated }); var o = zml.ops.customCall( custom_call_name, @@ -479,7 +502,7 @@ pub const fa3 = struct { }, .{ .output_operand_aliases = &.{0}, - .has_side_effect = true, + .has_side_effect = false, }, ); @@ -488,6 +511,9 @@ pub const fa3 = struct { }; pub const paged_fa2 = struct { + // God knows why flash attention uses this number and not something else. + const MAX_NUM_SPLITS = 8; + pub const Variant = enum { decode, mixed, @@ -541,6 +567,7 @@ pub const paged_fa2 = struct { max_seqlen_k: usize, max_token_count: usize, num_heads: usize, + num_kv_heads: usize, head_dim: usize, }; @@ -554,10 +581,10 @@ pub const paged_fa2 = struct { pub fn init(options: DecodeOptions) DecodeParameters { return .{ - .block_table = .init(.{ options.batch_size, options.max_num_pages }, .i32), - .cu_seqlens_q = .init(.{options.batch_size + 1}, .i32), - .seqused_k = .init(.{options.batch_size}, .i32), - .metadata = DecodeMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .block_table = zml.Tensor.init(.{ .b = options.batch_size, .p = options.max_num_pages }, .i32), + .cu_seqlens_q = zml.Tensor.init(.{ .b = options.batch_size + 1 }, .i32), + .seqused_k = zml.Tensor.init(.{ .b = options.batch_size }, .i32), + .metadata = DecodeMetadata.init(options), .options = options, }; } @@ -577,15 +604,22 @@ pub const paged_fa2 = struct { pub const DecodeMetadata = struct { out_accum: zml.Tensor, - pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) DecodeMetadata { + pub fn init(opts: DecodeOptions) DecodeMetadata { + const shape = zml.Shape.init(.{ + .splits = MAX_NUM_SPLITS, + .hkv = opts.num_kv_heads, + .hg = @divExact(opts.num_heads, opts.num_kv_heads), + .b = opts.max_token_count, + .hd = opts.head_dim, + }, .f32).withPartitioning(.{ .hkv = .model }); return .{ - .out_accum = .init(.{8 * max_token_count * num_heads * head_dim * 4}, .i8), + .out_accum = .fromShape(shape), }; } - pub fn initBuffers(self: *const DecodeMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(DecodeMetadata) { + pub fn initBuffers(self: *const DecodeMetadata, io: std.Io, platform: zml.Platform, out_accum_sharding: zml.sharding.Sharding) !zml.Bufferized(DecodeMetadata) { return .{ - .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + .out_accum = try zml.Buffer.uninitialized(io, &platform, self.out_accum.shape(), out_accum_sharding, .{}), }; } @@ -607,6 +641,7 @@ pub const paged_fa2 = struct { max_seqlen_k: usize, max_token_count: usize, num_heads: usize, + num_kv_heads: usize, head_dim: usize, }; @@ -624,13 +659,16 @@ pub const paged_fa2 = struct { pub fn init(options: MixedOptions) MixedParameters { return .{ - .block_table_prefill = .init(.{ options.batch_size_prefill, options.max_num_pages }, .i32), - .cu_seqlens_q_prefill = .init(.{options.batch_size_prefill + 1}, .i32), - .seqused_k_prefill = .init(.{options.batch_size_prefill}, .i32), - .block_table_decode = .init(.{ options.batch_size_decode, options.max_num_pages }, .i32), - .cu_seqlens_q_decode = .init(.{options.batch_size_decode + 1}, .i32), - .seqused_k_decode = .init(.{options.batch_size_decode}, .i32), - .metadata = MixedMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .block_table_prefill = zml.Tensor.init( + .{ .b = options.batch_size_prefill, .p = options.max_num_pages }, + .i32, + ), + .cu_seqlens_q_prefill = zml.Tensor.init(.{ .b = options.batch_size_prefill + 1 }, .i32), + .seqused_k_prefill = zml.Tensor.init(.{ .b = options.batch_size_prefill }, .i32), + .block_table_decode = zml.Tensor.init(.{ .b = options.batch_size_decode, .p = options.max_num_pages }, .i32), + .cu_seqlens_q_decode = zml.Tensor.init(.{ .b = options.batch_size_decode + 1 }, .i32), + .seqused_k_decode = zml.Tensor.init(.{ .b = options.batch_size_decode }, .i32), + .metadata = MixedMetadata.init(options), .options = options, }; } @@ -654,16 +692,23 @@ pub const paged_fa2 = struct { out_accum: zml.Tensor, host_metadata: zml.Tensor, - pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) MixedMetadata { + pub fn init(opts: MixedOptions) MixedMetadata { + const shape = zml.Shape.init(.{ + .splits = MAX_NUM_SPLITS, + .hkv = opts.num_kv_heads, + .hg = @divExact(opts.num_heads, opts.num_kv_heads), + .b = opts.max_token_count, + .hd = opts.head_dim, + }, .f32).withPartitioning(.{ .hkv = .model }); return .{ - .out_accum = .init(.{8 * max_token_count * num_heads * head_dim * 4}, .i8), + .out_accum = .fromShape(shape), .host_metadata = .init(.{2}, .i32), }; } - pub fn initBuffers(self: *const MixedMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(MixedMetadata) { + pub fn initBuffers(self: *const MixedMetadata, io: std.Io, platform: zml.Platform, out_accum_sharding: zml.sharding.Sharding) !zml.Bufferized(MixedMetadata) { return .{ - .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + .out_accum = try zml.Buffer.uninitialized(io, &platform, self.out_accum.shape(), out_accum_sharding, .{}), .host_metadata = undefined, }; } @@ -747,7 +792,7 @@ pub const paged_fa2 = struct { .softmax_scale = softmax_scale, .window_size_left = window_size_left, .window_size_right = window_size_right, - .num_splits = 0, + .num_splits = MAX_NUM_SPLITS, .num_heads = num_heads, }; @@ -817,7 +862,7 @@ pub const paged_fa2 = struct { .softmax_scale = softmax_scale, .window_size_left = window_size_left, .window_size_right = window_size_right, - .num_splits = 0, + .num_splits = MAX_NUM_SPLITS, .num_heads = num_heads, }; @@ -842,132 +887,235 @@ pub const paged_fa2 = struct { } }; - pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { - stdx.debug.assert(q.shape().hasTags(.{ .b, .h, .hd }), "Expected q to have tags .b, .h, .hd", .{}); - stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); - stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); + pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index_: zml.Tensor, opts: AttentionOptions) zml.Tensor { + stdx.debug.assert(q.shape().hasTags(.{ .b, .hg, .hkv, .hd }), "Expected q to have tags .b, .h, .hd", .{}); + stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); + stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); + const ctx = CompilationContext.current(); + + const num_head_groups = q.dim(.hg); + const num_kv_heads = q.dim(.hkv); + const head_dim = q.dim(.hd); + const num_heads = num_head_groups * num_kv_heads; + const layer_index = layer_index_.reshape(.{1}).withPartitioning(.{ ._0 = .replicated }); + // FIXME: remove unreachable and propagate error correctly. + const num_heads_per_shard = @divExact(num_heads, ctx.partitioning.numPartitionsForLogicalAxis(q.shape(), .model) catch unreachable); const o = switch (parameters) { .decode => |decode_parameters| b: { - const softmax_lse = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.dim(.h), .q = q.dim(.b) }, .f32)); - const softmax_lse_accum = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{64 * decode_parameters.block_table.dim(0) * q.dim(.h) * 4}, .i8)); - const dummy_cu_seqlens_k = zml.Tensor.constant(zml.DataType.i32.zero()).broad(decode_parameters.cu_seqlens_q.shape()); + const seqlenq_ngroups_swapped = num_heads > num_kv_heads and @mod(head_dim, 8) == 0 and opts.sliding_window < 0; - const original_tot = q.dim(.b); - const num_heads = q.dim(.h); - const num_heads_k = k_cache.dim(.h); - const head_size = q.dim(.hd); - const ngroups = @divExact(num_heads, num_heads_k); - const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0 and opts.sliding_window < 0; + const block_table = decode_parameters.block_table.withPartitioning(.{ .b = .replicated }); + const cu_seqlens_q = decode_parameters.cu_seqlens_q.withPartitioning(.{ .b = .replicated }); + const seqused_k = decode_parameters.seqused_k.withPartitioning(.{ .b = .replicated }); + + const out_accum = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + .splits = MAX_NUM_SPLITS, + .hkv = num_kv_heads, + .hg = num_head_groups, + .b = q.dim(.b), + .hd = head_dim, + }, .f32)).withPartitioning(.{ .hkv = .model }); + const softmax_lse = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + .hkv = num_kv_heads, + .hg = num_head_groups, + .b = q.dim(.b), + }, .f32)).withPartitioning(.{ .hkv = .model }); + const softmax_lse_accum = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + .splits = MAX_NUM_SPLITS, + .hkv = num_kv_heads, + .hg = num_head_groups, + .b = q.dim(.b), + }, .f32)).withPartitioning(.{ .hkv = .model }); + const dummy_cu_seqlens_k = zml.Tensor.constant(zml.DataType.i32.zero()).broad(cu_seqlens_q.shape()); + + const batch_dim = q.dim(.b); var q2 = q; if (seqlenq_ngroups_swapped) { - q2 = q2.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .b, .ngroups, .h, .hd }).merge(.{ .b = .{ .b, .ngroups } }); + q2 = q2.transpose(.{ .b, .hg, .hkv, .hd }).merge(.{ .b = .{ .b, .hg } }).withPartitioning(.{ .hkv = .model }); + } else { + q2 = q2.transpose(.{ .b, .hkv, .hg, .hd }).merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); } - var o = zml.ops.customCall( - Decode.custom_call_name, + const output_shape = q2.shape(); + var o = zml.ops.manualComputation( .{ q2, k_cache, v_cache, - decode_parameters.cu_seqlens_q, + cu_seqlens_q, dummy_cu_seqlens_k, - decode_parameters.seqused_k, - decode_parameters.block_table, + seqused_k, + block_table, softmax_lse, softmax_lse_accum, - decode_parameters.metadata.out_accum, + out_accum, layer_index, }, - .{q2.shape()}, + output_shape, .{ - .is_causal = opts.is_causal, - .max_seqlen_k = context.max_seqlen_k, - .num_heads = num_heads, - .window_size_left = opts.sliding_window, + .metadata = .{ + .is_causal = opts.is_causal, + .max_seqlen_k = context.max_seqlen_k, + .num_heads = num_heads_per_shard, + .window_size_left = opts.sliding_window, + }, + .opts = zml.ops.CustomCallOptions{ + .has_side_effect = false, + }, }, - .{ .has_side_effect = false }, + (struct { + fn body(ctx_: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output: zml.Shape) zml.Tensor { + return zml.ops.customCall(Decode.custom_call_name, sharded_inputs, output, ctx_.metadata, ctx_.opts); + } + }).body, ); if (seqlenq_ngroups_swapped) { - o = o.splitAxis(.b, .{ .b = original_tot, .ngroups = ngroups }).transpose(.{ .b, .h, .ngroups, .hd }).merge(.{ .h = .{ .h, .ngroups } }); + o = o.splitAxis(.b, .{ .b = batch_dim, .hg = num_head_groups }).transpose(.{ .b, .hkv, .hg, .hd }); + } else { + o = o.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = num_head_groups }); } break :b o; }, .mixed => |mixed_parameters| b: { - const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = q.dim(.b) }, .f32)); - const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{64 * q.dim(.b) * q.dim(.h) * 4}, .i8)); - const dummy_cu_seqlens_k_prefill = zml.Tensor.constant(zml.DataType.i32.zero()).broad(mixed_parameters.cu_seqlens_q_prefill.shape()); - var o = zml.ops.customCall( - Prefill.custom_call_name, + const seqlenq_ngroups_swapped = num_heads > num_kv_heads and @mod(head_dim, 8) == 0 and opts.sliding_window < 0; + + const block_table_prefill = mixed_parameters.block_table_prefill.withPartitioning(.{ .b = .replicated }); + const cu_seqlens_q_prefill = mixed_parameters.cu_seqlens_q_prefill.withPartitioning(.{ .b = .replicated }); + const seqused_k_prefill = mixed_parameters.seqused_k_prefill.withPartitioning(.{ .b = .replicated }); + const block_table_decode = mixed_parameters.block_table_decode.withPartitioning(.{ .b = .replicated }); + const cu_seqlens_q_decode = mixed_parameters.cu_seqlens_q_decode.withPartitioning(.{ .b = .replicated }); + const seqused_k_decode = mixed_parameters.seqused_k_decode.withPartitioning(.{ .b = .replicated }); + const host_metadata = mixed_parameters.metadata.host_metadata.withTags(.{.x}).withPartitioning(.{ .x = .replicated }); + + const out_accum_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + .splits = MAX_NUM_SPLITS, + .hkv = num_kv_heads, + .hg = num_head_groups, + .b = q.dim(.b), + .hd = head_dim, + }, .f32)).withPartitioning(.{ .hkv = .model }); + const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + .hkv = num_kv_heads, + .hg = num_head_groups, + .b = q.dim(.b), + }, .f32)).withPartitioning(.{ .hkv = .model }); + const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + .splits = MAX_NUM_SPLITS, + .hkv = num_kv_heads, + .hg = num_head_groups, + .b = q.dim(.b), + }, .f32)).withPartitioning(.{ .hkv = .model }); + const dummy_cu_seqlens_k_prefill = zml.Tensor.constant(zml.DataType.i32.zero()).broad(cu_seqlens_q_prefill.shape()); + + var q2 = q; + q2 = q2.transpose(.{ .b, .hkv, .hg, .hd }).merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + + const output_shape = q2.shape(); + var o = zml.ops.manualComputation( .{ - q, + q2, k_cache, v_cache, - mixed_parameters.cu_seqlens_q_prefill, + cu_seqlens_q_prefill, dummy_cu_seqlens_k_prefill, - mixed_parameters.seqused_k_prefill, - mixed_parameters.block_table_prefill, + seqused_k_prefill, + block_table_prefill, softmax_lse_prefill, softmax_lse_accum_prefill, - mixed_parameters.metadata.out_accum, - mixed_parameters.metadata.host_metadata, + out_accum_prefill, + host_metadata, layer_index, }, - .{q.shape()}, + output_shape, .{ - .is_causal = opts.is_causal, - .max_seqlen_k = context.max_seqlen_k, - .num_heads = q.dim(.h), - .window_size_left = opts.sliding_window, + .metadata = .{ + .is_causal = opts.is_causal, + .max_seqlen_k = context.max_seqlen_k, + .num_heads = num_heads_per_shard, + .window_size_left = opts.sliding_window, + }, + .opts = zml.ops.CustomCallOptions{ + .has_side_effect = false, + }, }, - .{ .has_side_effect = false }, + (struct { + fn body(ctx_: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output: zml.Shape) zml.Tensor { + return zml.ops.customCall(Prefill.custom_call_name, sharded_inputs, output, ctx_.metadata, ctx_.opts); + } + }).body, ); - const batch_size_decode = mixed_parameters.block_table_prefill.dim(0); - const softmax_lse_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = batch_size_decode }, .f32)); - const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{64 * mixed_parameters.block_table_decode.dim(0) * q.dim(.h) * 4}, .i8)); - const dummy_cu_seqlens_k_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(mixed_parameters.cu_seqlens_q_decode.shape()); - var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_size_decode }); + o = o.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = num_head_groups }); + + const batch_dim_decode = block_table_decode.dim(0); + const out_accum_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + .splits = MAX_NUM_SPLITS, + .hkv = num_kv_heads, + .hg = num_head_groups, + .b = batch_dim_decode, + .hd = head_dim, + }, .f32)).withPartitioning(.{ .hkv = .model }); + const softmax_lse_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + .hkv = num_kv_heads, + .hg = num_head_groups, + .b = batch_dim_decode, + }, .f32)).withPartitioning(.{ .hkv = .model }); + const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + .splits = MAX_NUM_SPLITS, + .hkv = num_kv_heads, + .hg = num_head_groups, + .b = batch_dim_decode, + }, .f32)).withPartitioning(.{ .hkv = .model }); + const dummy_cu_seqlens_k_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(cu_seqlens_q_decode.shape()); + var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_dim_decode }); - const original_tot = q_decode.dim(.b); - const num_heads = q_decode.dim(.h); - const num_heads_k = k_cache.dim(.h); - const head_size = q_decode.dim(.hd); - const ngroups = @divExact(num_heads, num_heads_k); - const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0 and opts.sliding_window < 0; if (seqlenq_ngroups_swapped) { - q_decode = q_decode.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .b, .ngroups, .h, .hd }).merge(.{ .b = .{ .b, .ngroups } }); + q_decode = q_decode.transpose(.{ .b, .hg, .hkv, .hd }).merge(.{ .b = .{ .b, .hg } }).withPartitioning(.{ .hkv = .model }); + } else { + q_decode = q_decode.transpose(.{ .b, .hkv, .hg, .hd }).merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); } - var o_decode = zml.ops.customCall( - Decode.custom_call_name, + const output_shape_decode = q_decode.shape(); + var o_decode = zml.ops.manualComputation( .{ q_decode, k_cache, v_cache, - mixed_parameters.cu_seqlens_q_decode, + cu_seqlens_q_decode, dummy_cu_seqlens_k_decode, - mixed_parameters.seqused_k_decode, - mixed_parameters.block_table_decode, + seqused_k_decode, + block_table_decode, softmax_lse_decode, softmax_lse_accum_decode, - mixed_parameters.metadata.out_accum, + out_accum_decode, layer_index, }, - .{q_decode.shape()}, + output_shape_decode, .{ - .is_causal = opts.is_causal, - .max_seqlen_k = context.max_seqlen_k, - .num_heads = q.dim(.h), - .window_size_left = opts.sliding_window, + .metadata = .{ + .is_causal = opts.is_causal, + .max_seqlen_k = context.max_seqlen_k, + .num_heads = num_heads_per_shard, + .window_size_left = opts.sliding_window, + }, + .opts = zml.ops.CustomCallOptions{ + .has_side_effect = false, + }, }, - .{ .has_side_effect = false }, + (struct { + fn body(ctx_: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output: zml.Shape) zml.Tensor { + return zml.ops.customCall(Decode.custom_call_name, sharded_inputs, output, ctx_.metadata, ctx_.opts); + } + }).body, ); if (seqlenq_ngroups_swapped) { - o_decode = o_decode.splitAxis(.b, .{ .b = original_tot, .ngroups = ngroups }).transpose(.{ .b, .h, .ngroups, .hd }).merge(.{ .h = .{ .h, .ngroups } }); + o_decode = o_decode.splitAxis(.b, .{ .b = batch_dim_decode, .hg = num_head_groups }).transpose(.{ .b, .hkv, .hg, .hd }); + } else { + o_decode = o_decode.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = num_head_groups }); } o = o.dynamicUpdateSlice1d(o_decode, 0, context.decode_offset.?); @@ -1075,9 +1223,9 @@ pub const paged_fa3 = struct { }; } - pub fn initBuffers(self: *const DecodeMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(DecodeMetadata) { + pub fn initBuffers(self: *const DecodeMetadata, io: std.Io, platform: zml.Platform, out_accum_sharding: zml.sharding.Sharding) !zml.Bufferized(DecodeMetadata) { return .{ - .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + .out_accum = try zml.Buffer.uninitialized(io, &platform, self.out_accum.shape(), out_accum_sharding, .{}), }; } @@ -1153,9 +1301,9 @@ pub const paged_fa3 = struct { }; } - pub fn initBuffers(self: *const MixedMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(MixedMetadata) { + pub fn initBuffers(self: *const MixedMetadata, io: std.Io, platform: zml.Platform, out_accum_sharding: zml.sharding.Sharding) !zml.Bufferized(MixedMetadata) { return .{ - .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + .out_accum = try zml.Buffer.uninitialized(io, &platform, self.out_accum.shape(), out_accum_sharding, .{}), .host_metadata = undefined, }; } diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 10048820..44091fc5 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -1,4 +1,5 @@ const std = @import("std"); + const stdx = @import("stdx"); const zml = @import("../zml.zig"); @@ -32,54 +33,68 @@ pub const Options = union(Backend) { cuda_fa2: flashattn.paged_fa2.Options, cuda_fa3: flashattn.paged_fa3.Options, - pub fn fromBackend(backend: Backend, is_prefill: bool, batch_size: u32, seq_len: u32, page_chunk_size: u32, max_token_count: u32, num_attention_heads: u32, head_dim: u32) Options { - const max_num_pages = @divFloor(seq_len + page_chunk_size - 1, page_chunk_size); // @divCeil - return switch (backend) { - .cuda_fa2 => if (is_prefill) .{ + const Args = struct { + backend: Backend, + is_prefill: bool, + batch_size: u32, + seq_len: u32, + page_chunk_size: u32, + max_token_count: u32, + num_heads: u32, + num_kv_heads: u32, + head_dim: u32, + }; + + pub fn fromBackend(args: Args) Options { + const max_num_pages = @divFloor(args.seq_len + args.page_chunk_size - 1, args.page_chunk_size); // @divCeil + return switch (args.backend) { + .cuda_fa2 => if (args.is_prefill) .{ .cuda_fa2 = .{ .mixed = .{ - .batch_size_decode = batch_size, - .batch_size_prefill = batch_size, + .batch_size_decode = args.batch_size, + .batch_size_prefill = args.batch_size, .max_num_pages = max_num_pages, - .max_seqlen_k = seq_len, - .max_token_count = max_token_count, - .num_heads = num_attention_heads, - .head_dim = head_dim, + .max_seqlen_k = args.seq_len, + .max_token_count = args.max_token_count, + .num_heads = args.num_heads, + .num_kv_heads = args.num_kv_heads, + .head_dim = args.head_dim, }, }, } else .{ .cuda_fa2 = .{ .decode = .{ - .batch_size = batch_size, + .batch_size = args.batch_size, .max_num_pages = max_num_pages, - .max_seqlen_k = seq_len, - .max_token_count = max_token_count, - .num_heads = num_attention_heads, - .head_dim = head_dim, + .max_seqlen_k = args.seq_len, + .max_token_count = args.max_token_count, + .num_heads = args.num_heads, + .num_kv_heads = args.num_kv_heads, + .head_dim = args.head_dim, }, }, }, - .cuda_fa3 => if (is_prefill) .{ + .cuda_fa3 => if (args.is_prefill) .{ .cuda_fa3 = .{ .mixed = .{ - .batch_size_decode = batch_size, - .batch_size_prefill = batch_size, + .batch_size_decode = args.batch_size, + .batch_size_prefill = args.batch_size, .max_num_pages = max_num_pages, - .max_seqlen_k = seq_len, - .max_token_count = max_token_count, - .num_heads = num_attention_heads, - .head_dim = head_dim, + .max_seqlen_k = args.seq_len, + .max_token_count = args.max_token_count, + .num_heads = args.num_heads, + .head_dim = args.head_dim, }, }, } else .{ .cuda_fa3 = .{ .decode = .{ - .batch_size = batch_size, + .batch_size = args.batch_size, .max_num_pages = max_num_pages, - .max_seqlen_k = seq_len, - .max_token_count = max_token_count, - .num_heads = num_attention_heads, - .head_dim = head_dim, + .max_seqlen_k = args.seq_len, + .max_token_count = args.max_token_count, + .num_heads = args.num_heads, + .head_dim = args.head_dim, }, }, }, diff --git a/zml/buffer.zig b/zml/buffer.zig index 7b0a88ac..dd108ad0 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -9,6 +9,9 @@ const Memory = @import("platform.zig").Memory; const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; +const sharding_ = @import("sharding.zig"); +const Sharding = sharding_.Sharding; +const Placement = sharding_.Placement; const Slice = @import("slice.zig").Slice; const Target = @import("platform.zig").Target; const testing = @import("testing.zig"); @@ -21,22 +24,51 @@ const log = std.log.scoped(.zml); /// * loading weights from disk directly to the `device zml.aio.loadBuffers` /// * can be created by calling `HostBuffer.toDevice(platform)`. pub const Buffer = struct { + _platform: *const Platform, _shape: Shape, - platform: *const Platform, + _sharding: Sharding, _shards: Shards, - pub const MAX_NUM_SHARDS: u8 = Platform.MAX_NUM_DEVICES; + pub const MAX_NUM_SHARDS: u16 = Platform.MAX_NUM_DEVICES; pub const Shards = stdx.BoundedArray(*pjrt.Buffer, MAX_NUM_SHARDS); + pub const Shard = struct { + _platform: *const Platform, + _pjrt_buffer: *pjrt.Buffer, + + pub fn devicePtr(self: *const Shard) *anyopaque { + return self._pjrt_buffer.opaqueDeviceMemoryDataPointer(self._platform.pjrt_api) catch unreachable; + } + }; + + pub const ShardIterator = struct { + _platform: *const Platform, + _shards: []const *pjrt.Buffer, + _index: usize = 0, + + pub fn remaining(self: *ShardIterator) usize { + return self._shards.len -| self._index; + } + + pub fn next(self: *ShardIterator) ?Shard { + defer self._index += 1; + if (self._index >= self._shards.len) return null; + + return .{ + ._pjrt_buffer = self._shards[self._index], + ._platform = self._platform, + }; + } + }; + pub const FromOptions = struct { wait: bool = true, memory: Memory.Kind = .default }; /// Frees the accelerator memory. /// Depending on the platform, the memory is typically not released to the OS /// but just marked as available in the memory pool. pub fn deinit(self: *Buffer) void { - // log.warn("Unloading {f} {d} bytes", .{ self._shape, self._shape.byteSize() }); for (self._shards.constSlice()) |buffer| { - buffer.deinit(self.platform.pjrt_api); + buffer.deinit(self._platform.pjrt_api); } } @@ -45,54 +77,69 @@ pub const Buffer = struct { return self._shape; } + pub fn shards(self: *const Buffer) ShardIterator { + return .{ + ._platform = self._platform, + ._shards = self._shards.constSlice(), + }; + } + pub fn format(self: Buffer, writer: *std.Io.Writer) !void { - try writer.print("Buffer({f})@{x}", .{ self._shape, @intFromPtr(self.devicePtr()) }); + try writer.print("{f}", .{self.placement()}); } /// Copies the content of the given buffer from host memory to the accelerator memory. - pub fn from(io: std.Io, platform: *const Platform, shape_: Shape, data_: []const u8, opts: FromOptions) !Buffer { + pub fn from( + io: std.Io, + platform: *const Platform, + sh: Shape, + sharding: Sharding, + data_: []const u8, + opts: FromOptions, + ) !Buffer { var res: Buffer = .{ - .platform = platform, - ._shape = shape_, + ._platform = platform, + ._shape = sh, + ._sharding = sharding, ._shards = .{}, }; - // We shard only on the first axis so that the chunks are still contiguous. - // TODO: support more advanced sharding specs - stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); - //const sharding_ax: ?u3 = std.simd.firstTrue(shape_._sharding_info); - const n_partitions = platform.sharding().num_partitions; - //const chunk_size = if (sharding_ax) |ax| cs: { - // // This kind of sharding error should be detected earlier on. - // stdx.debug.assert(@rem(shape_.dim(ax), n_partitions) == 0, "Buffer.from({f}) expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ shape_, ax, n_partitions }); - // break :cs @divExact(shape_.dim(ax), n_partitions); - //} else 0; - //_ = chunk_size; + const buffer_type = pjrtx.bufferTypeFromDtype(sh.dtype()); + const slice = Slice.init(sh, data_); - const buffer_type = pjrtx.bufferTypeFromDtype(shape_.dtype()); - const byte_strides = shape_.computeByteStrides(); - - const devices = platform.devices; - for (0..n_partitions) |i| { - // If no sharding if found, the given buffer is replicated on all devices. - //const buf = if (sharding_ax) |ax| buf: { - // @panic("sharding not implemented"); - // //const start: i64 = @as(i64, @intCast(i)) * chunk_size; - // //break :buf host_buffer.slice1d(ax, .{ .start = start, .end = start + chunk_size }); - //} else host_buffer; - - const args = pjrt.Client.BufferFromHostBufferArgs{ - .data = data_.ptr, + for (res.placement().shards.constSlice()) |shard| { + const sub_slice = shard.shardSlice(slice); + var default_layout: ?pjrt.DefaultMemoryLayout = null; + const layout: pjrt.MemoryLayout = switch (platform.target) { + .tpu => blk: { + default_layout = try platform.pjrt_client.defaultMemoryLayout( + platform.pjrt_api, + pjrtx.bufferTypeFromDtype(shard.shape.dtype()), + shard.shape.dims(), + ); + break :blk default_layout.?.toMemoryLayout(); + }, + else => .{ + .tiled = .{ + .minor_to_major = constants.minorToMajor(shard.shape.rank()), + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }, + }; + const args: pjrt.Client.BufferFromHostBufferArgs = .{ + .data = sub_slice.constData().ptr, .buffer_type = buffer_type, - .dims = shape_.dims(), - .byte_strides = byte_strides.slice(), + .dims = shard.shape.dims(), + .byte_strides = sub_slice.byte_strides.constSlice(), + .layout = layout, .host_buffer_semantics = .ImmutableUntilTransferCompletes, - .dst = .{ .memory = devices[i].memory(opts.memory).pjrt_memory }, + .dst = .{ .memory = shard.memory(platform, opts.memory).pjrt_memory }, }; const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); - if (event) |ev| ev.deinit(platform.pjrt_api); + res._shards.appendAssumeCapacity(pjrt_buffer); } @@ -105,97 +152,97 @@ pub const Buffer = struct { /// Copies the given Zig bytes to the accelerator memory and /// return a Buffer with the given dimensions. - pub fn fromBytes(io: std.Io, platform: *const Platform, sh: Shape, data: []const u8) !Buffer { - return from(io, platform, sh, data, .{}); + pub fn fromBytes(io: std.Io, platform: *const Platform, sh: Shape, sharding: Sharding, data: []const u8) !Buffer { + return from(io, platform, sh, sharding, data, .{}); } /// Copies the given zml.Slice to the accelerator memory and /// return a Buffer. - pub fn fromSlice(io: std.Io, platform: *const Platform, slice: Slice) !Buffer { - return fromSliceOpts(io, platform, slice, .{}); + pub fn fromSlice(io: std.Io, platform: *const Platform, slice: Slice, sharding: Sharding) !Buffer { + return fromSliceOpts(io, platform, slice, sharding, .{}); } - pub fn fromSliceOpts(io: std.Io, platform: *const Platform, slice: Slice, opts: FromOptions) !Buffer { - return from(io, platform, slice.shape, std.mem.sliceAsBytes(slice.constData()), opts); + pub fn fromSliceOpts(io: std.Io, platform: *const Platform, slice: Slice, sharding: Sharding, opts: FromOptions) !Buffer { + return from(io, platform, slice.shape, sharding, std.mem.sliceAsBytes(slice.constData()), opts); } /// Creates a Buffer with a single element. - pub fn scalar(io: std.Io, platform: *const Platform, val: anytype, dtype_: DataType) !Buffer { + pub fn scalar(io: std.Io, platform: *const Platform, val: anytype, dtype_: DataType, sharding: Sharding) !Buffer { const x = dtype_.constant(val); - return fromBytes(io, platform, Shape.init(.{}, dtype_), x.asBytes()); + return fromBytes(io, platform, Shape.init(.{}, dtype_), sharding, x.asBytes()); } pub fn await(self: Buffer, io: std.Io) !void { for (self._shards.constSlice()) |buffer| { - const ev = buffer.readyEvent(self.platform.pjrt_api); - defer ev.deinit(self.platform.pjrt_api); - try ev.await(self.platform.pjrt_api, io); + const ev = buffer.readyEvent(self._platform.pjrt_api); + defer ev.deinit(self._platform.pjrt_api); + try ev.await(self._platform.pjrt_api, io); } } pub const UnitializedOptions = struct { memory: Memory.Kind = .default }; - pub fn uninitialized(io: std.Io, platform: *const Platform, shape_: Shape, opts: UnitializedOptions) !Buffer { - _ = io; // autofix + pub fn uninitialized( + _: std.Io, + platform: *const Platform, + sh: Shape, + sharding: Sharding, + opts: UnitializedOptions, + ) !Buffer { var res: Buffer = .{ - .platform = platform, - ._shape = shape_, + ._platform = platform, + ._shape = sh, + ._sharding = sharding, ._shards = .{}, }; errdefer for (res._shards.slice()) |shard| { shard.deinit(platform.pjrt_api); }; - stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); - //const sharding_ax: ?u3 = std.simd.firstTrue(shape_._sharding_info); - const n_partitions = platform.sharding().num_partitions; - - //const shard_shape = if (sharding_ax) |ax| s: { - // // This kind of sharding error should be detected earlier on. - // stdx.debug.assert(@rem(shape_.dim(ax), n_partitions) == 0, "Buffer.uninitialized() expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ ax, n_partitions }); - // const shard_shape = shape_.set(ax, @divExact(shape_.dim(ax), n_partitions)); - // break :s shard_shape; - //} else shape_; - - var args = pjrt.Client.CreateUninitializedBufferArgs{ - .dims = shape_.dims(), - .element_type = pjrtx.bufferTypeFromDtype(shape_.dtype()), - .layout = .{ - .tiled = .{ - .minor_to_major = constants.minorToMajor(shape_.rank()), - .tile_dims = &.{}, - .tile_dims_sizes = &.{}, + for (res.placement().shards.constSlice()) |shard| { + var default_layout: ?pjrt.DefaultMemoryLayout = null; + const layout: pjrt.MemoryLayout = switch (platform.target) { + .tpu => blk: { + default_layout = try platform.pjrt_client.defaultMemoryLayout( + platform.pjrt_api, + pjrtx.bufferTypeFromDtype(shard.shape.dtype()), + shard.shape.dims(), + ); + break :blk default_layout.?.toMemoryLayout(); }, - }, - // set per device, see below. - .dst = undefined, - }; + else => .{ + .tiled = .{ + .minor_to_major = constants.minorToMajor(shard.shape.rank()), + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }, + }; + const args: pjrt.Client.CreateUninitializedBufferArgs = .{ + .dims = shard.shape.dims(), + .element_type = pjrtx.bufferTypeFromDtype(shard.shape.dtype()), + .layout = layout, + .dst = .{ .memory = shard.memory(platform, opts.memory).pjrt_memory }, + }; - for (platform.devices[0..n_partitions]) |*device| { - args.dst = .{ .memory = device.memory(opts.memory).pjrt_memory }; - const shard = try platform.pjrt_client.createUninitializedBuffer(platform.pjrt_api, args); - res._shards.appendAssumeCapacity(shard); + const shard_buffer = try platform.pjrt_client.createUninitializedBuffer(platform.pjrt_api, args); + res._shards.appendAssumeCapacity(shard_buffer); } return res; } /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. - pub fn fromPjrtBuffers(platform: *const Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { + pub fn fromPjrtBuffers(platform: *const Platform, sh: Shape, sharding: Sharding, pjrt_buffers: []const *pjrt.Buffer) Buffer { stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); stdx.debug.assert(pjrt_buffers.len > 0, "fromPjrtBuffers expects at least one buffer, got 0.", .{}); - var shards: Shards = .{}; - shards.appendSliceAssumeCapacity(pjrt_buffers); - return .{ - .platform = platform, - ._shape = shape_, - ._shards = shards, - }; - } - pub fn devicePtr(self: Buffer) *anyopaque { - //stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer", .{}); - return self._shards.get(0).opaqueDeviceMemoryDataPointer(self.platform.pjrt_api) catch unreachable; + return .{ + ._platform = platform, + ._shape = sh, + ._sharding = sharding, + ._shards = Shards.fromSlice(pjrt_buffers) catch unreachable, + }; } /// Fetches the content of the given buffer into a stack variable of the given type. @@ -210,10 +257,23 @@ pub const Buffer = struct { /// Copies the content of the Buffer to the provided slice. pub fn toSlice(self: Buffer, io: std.Io, slice: Slice) !void { - //stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self.platform.pjrt_api, slice.data()); - if (maybe_event) |event| { - try event.await(self.platform.pjrt_api, io); + stdx.debug.assert(self._shape.eql(slice.shape), "Buffer shape {f} doesn't match destination slice {f}", .{ self._shape, slice.shape }); + + for (self.placement().shards.constSlice(), 0..) |shard, shard_index| { + const sub_slice = shard.shardSlice(slice); + if (!sub_slice.isContiguous()) return error.NonContiguousShardRead; + + const size_bytes = shard.shape.byteSize(); + const start = sub_slice.offset_bytes; + const end = start + size_bytes; + stdx.debug.assert(end <= slice.constData().len, "Shard sub_slice exceeds destination slice", .{}); + + const destination = slice.data()[start..end]; + const maybe_event = try self._shards.get(shard_index).toHostBuffer(self._platform.pjrt_api, destination); + + if (maybe_event) |event| { + try event.await(self._platform.pjrt_api, io); + } } } @@ -223,8 +283,34 @@ pub const Buffer = struct { const slice = try Slice.alloc(allocator, self.shape()); errdefer slice.free(allocator); - try self.toSlice(io, slice); + for (self.placement().shards.constSlice(), 0..) |shard, shard_index| { + const sub_slice = shard.shardSlice(slice); + + var shard_slice = try Slice.alloc(allocator, sub_slice.shape); + defer shard_slice.free(allocator); + + const maybe_event = try self._shards.get(shard_index).toHostBuffer(self._platform.pjrt_api, shard_slice.data()); + if (maybe_event) |event| { + try event.await(self._platform.pjrt_api, io); + } + + sub_slice.copy(shard_slice.constData()); + } return slice; } + + pub fn byteSize(self: Buffer) usize { + var byte_size: usize = 0; + + for (self.placement().shards.constSlice()) |shard| { + byte_size += shard.shape.byteSize(); + } + + return byte_size; + } + + pub fn placement(self: Buffer) Placement { + return Placement.init(self._sharding, self._shape) catch unreachable; + } }; diff --git a/zml/exe.zig b/zml/exe.zig index 67493887..9d93f66d 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -7,6 +7,7 @@ const Buffer = @import("buffer.zig").Buffer; const meta = @import("meta.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; +const Sharding = @import("sharding.zig").Sharding; pub const Exe = struct { platform: *const Platform, @@ -17,7 +18,11 @@ pub const Exe = struct { input_shapes: []const Shape, output_shapes: []const Shape, - num_devices: u8, + input_shardings: []const Sharding, + output_shardings: []const Sharding, + + num_devices: usize, + num_partitions: i32, arena: std.heap.ArenaAllocator, @@ -25,9 +30,12 @@ pub const Exe = struct { allocator: std.mem.Allocator, platform: *const Platform, exe: *pjrt.LoadedExecutable, + num_devices: usize, + num_partitions: i32, input_shapes: []const Shape, output_shapes: []const Shape, - num_devices: u8, + input_shardings: []const Sharding, + output_shardings: []const Sharding, ) !Exe { var arena = std.heap.ArenaAllocator.init(allocator); errdefer arena.deinit(); @@ -35,12 +43,18 @@ pub const Exe = struct { const input_shapes_copy = try arena.allocator().dupe(Shape, input_shapes); const output_shapes_copy = try arena.allocator().dupe(Shape, output_shapes); + const input_shardings_copy = try arena.allocator().dupe(Sharding, input_shardings); + const output_shardings_copy = try arena.allocator().dupe(Sharding, output_shardings); + return .{ .platform = platform, .exe = exe, .input_shapes = input_shapes_copy, .output_shapes = output_shapes_copy, + .input_shardings = input_shardings_copy, + .output_shardings = output_shardings_copy, .num_devices = num_devices, + .num_partitions = num_partitions, .arena = arena, }; } @@ -50,11 +64,11 @@ pub const Exe = struct { } pub fn args(self: *const Exe, allocator: std.mem.Allocator) !Arguments { - return Arguments.init(allocator, self.input_shapes, self.num_devices); + return Arguments.init(allocator, self.input_shapes, self.input_shardings, self.num_devices); } pub fn results(self: *const Exe, allocator: std.mem.Allocator) !Results { - return Results.init(allocator, self.output_shapes, self.num_devices, self.platform); + return Results.init(allocator, self.output_shapes, self.output_shardings, self.platform, self.num_devices); } pub const FlatBuffers = struct { @@ -91,8 +105,9 @@ pub const Exe = struct { flat_buffers: FlatBuffers, expected_shapes: []const Shape, baked_count: usize = 0, + shardings: []const Sharding, - pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize) !Arguments { + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, shardings: []const Sharding, num_devices: usize) !Arguments { const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); errdefer flat_buffers.deinit(allocator); @@ -102,6 +117,7 @@ pub const Exe = struct { return .{ .flat_buffers = flat_buffers, .expected_shapes = expected_shapes, + .shardings = shardings, }; } @@ -122,9 +138,24 @@ pub const Exe = struct { var context: LocalContext = .{ .self = self, .current_index = offset + self.baked_count }; meta.visit(struct { fn cb(context_: *LocalContext, buffer: *const Buffer) void { - stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected argument {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); - for (0..context_.self.flat_buffers.num_devices) |device_index| { - context_.self.flat_buffers.buffers[device_index][context_.current_index] = buffer._shards.get(device_index); + stdx.debug.assert( + context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), + "Expected argument {} to have shape {f}, got {f}", + .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }, + ); + + const expected = context_.self.flat_buffers.num_devices; + const shard_count = buffer._shards.len; + + stdx.debug.assert( + shard_count == expected, + "Argument {} has {d} shards but executable expects {d}", + .{ context_.current_index, shard_count, expected }, + ); + + for (0..expected) |device_index| { + context_.self.flat_buffers.buffers[device_index][context_.current_index] = + buffer._shards.get(device_index); } context_.current_index += 1; @@ -162,8 +193,9 @@ pub const Exe = struct { flat_buffers: FlatBuffers, expected_shapes: []const Shape, + shardings: []const Sharding, - pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize, platform: *const Platform) !Results { + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, shardings: []const Sharding, platform: *const Platform, num_devices: usize) !Results { const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); errdefer flat_buffers.deinit(allocator); @@ -174,6 +206,7 @@ pub const Exe = struct { .platform = platform, .flat_buffers = flat_buffers, .expected_shapes = expected_shapes, + .shardings = shardings, }; } @@ -195,7 +228,7 @@ pub const Exe = struct { for (0..context_.self.flat_buffers.num_devices) |device_index| { shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); } - buffer.* = Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], shards.constSlice()); + buffer.* = Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], context_.self.shardings[context_.current_index], shards.constSlice()); context_.current_index += 1; } }.cb, &context, &result); @@ -215,7 +248,7 @@ pub const Exe = struct { for (0..context_.self.flat_buffers.num_devices) |device_index| { shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); } - buffer.* = Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], shards.constSlice()); + buffer.* = Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], context_.self.shardings[context_.current_index], shards.constSlice()); context_.current_index += 1; } }.cb, &context, &v); @@ -225,8 +258,8 @@ pub const Exe = struct { pub fn internalCall(self: *const Exe, io: ?std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { stdx.debug.assert(opts.wait == false or io != null, "io should not be null when waiting for execution completion", .{}); var events = [_]?*pjrt.Event{null} ** Platform.MAX_NUM_DEVICES; - const sharding = self.platform.sharding(); - const events_slice: ?[]?*pjrt.Event = if (opts.wait) events[0..sharding.num_partitions] else null; + + const events_slice: ?[]?*pjrt.Event = if (opts.wait) events[0..@intCast(self.num_partitions)] else null; self.exe.execute(self.platform.pjrt_api, .{ .arguments = arguments.flat_buffers.buffers, diff --git a/zml/io.zig b/zml/io.zig index 9538987b..7d2f2d01 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -6,6 +6,7 @@ pub const VFS = @import("io").VFS; const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("zml.zig").Bufferized; +const CreateOptions = @import("platform.zig").CreateOptions; const mem = @import("mem.zig"); const Memory = @import("platform.zig").Memory; const meta = @import("meta.zig"); @@ -13,6 +14,12 @@ const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const safetensors = @import("safetensors.zig"); const Shape = @import("shape.zig").Shape; +const sharding_ = @import("sharding.zig"); +const Partitioner = sharding_.Partitioner; +const Partitioning = sharding_.Partitioning; +const Placement = sharding_.Placement; +const Sharding = sharding_.Sharding; +const Slice = @import("slice.zig").Slice; const Tensor = @import("tensor.zig").Tensor; const log = std.log.scoped(.@"zml/io"); @@ -128,11 +135,28 @@ pub const TensorStore = struct { } else false; } - pub fn maybeCreateTensor(self: View, subkey: []const u8) ?Tensor { + pub fn maybeCreateTensor(self: View, subkey: []const u8, tagz: anytype, partitioning: anytype) ?Tensor { var buffer: [256]u8 = undefined; const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; const ptr = self.store.getPtrFromKey(key) orelse return null; + if (@TypeOf(tagz) != @TypeOf(null)) { + switch (@typeInfo(@TypeOf(tagz))) { + .optional => if (tagz) |t| { + ptr.shape = ptr.shape.withTags(t); + }, + else => ptr.shape = ptr.shape.withTags(tagz), + } + } + + if (@TypeOf(partitioning) != @TypeOf(null)) { + switch (@typeInfo(@TypeOf(partitioning))) { + .optional => if (partitioning) |p| { + ptr.shape = ptr.shape.withPartitioning(p); + }, + else => ptr.shape = ptr.shape.withPartitioning(partitioning), + } + } const tensor: Tensor = .fromShape(ptr.shape); self.store.bindIdToKey(key, tensor.id) catch unreachable; @@ -140,25 +164,8 @@ pub const TensorStore = struct { return tensor; } - pub fn createTensor(self: View, subkey: []const u8) Tensor { - return self.maybeCreateTensor(subkey).?; - } - - pub fn maybeCreateTensorWithTags(self: View, subkey: []const u8, tagz: anytype) ?Tensor { - var buffer: [256]u8 = undefined; - const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; - - const ptr = self.store.getPtrFromKey(key) orelse return null; - ptr.shape = ptr.shape.withTags(tagz); - - const tensor: Tensor = .fromShape(ptr.shape); - self.store.bindIdToKey(key, tensor.id) catch unreachable; - - return tensor; - } - - pub fn createTensorWithTags(self: View, subkey: []const u8, tagz: anytype) Tensor { - return self.maybeCreateTensorWithTags(subkey, tagz).?; + pub fn createTensor(self: View, subkey: []const u8, tagz: anytype, partitioning: anytype) Tensor { + return self.maybeCreateTensor(subkey, tagz, partitioning).?; } pub fn getShape(self: View, subkey: []const u8) ?Shape { @@ -238,7 +245,8 @@ pub const ProgressWriter = struct { pub fn post(self: *ProgressWriter, len_pre: usize, total: usize) void { self.interface.buffer = self.inner.buffer; self.interface.end = self.inner.end; - self.total += (len_pre - self.interface.end) + total; + const drained_pre = len_pre -| self.interface.end; + self.total += drained_pre + total; self.progress.setCompletedItems(self.total / self.scale); } @@ -279,10 +287,19 @@ pub const MemoryWriter = union(enum) { direct: DirectMemoryWriter, buffered: BufferedMemoryWriter, - pub fn init(allocator: std.mem.Allocator, io: std.Io, memory: *const Memory, pool: *mem.DynamicBufferPool, shape: Shape, buffer: *Buffer) !MemoryWriter { - return switch (memory.platform.target) { - .cuda => .{ .direct = try .init(allocator, io, memory, pool, shape, buffer) }, - .rocm, .tpu, .neuron, .cpu => .{ .buffered = try .init(allocator, io, memory, shape, buffer) }, + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + pools: []mem.DynamicBufferPool, + dma_allocators: []const mem.DmaAllocator, + shape: Shape, + sharding: Sharding, + buffer: *Buffer, + ) !MemoryWriter { + return switch (platform.target) { + .cuda => .{ .direct = try DirectMemoryWriter.init(allocator, io, platform, pools, dma_allocators, shape, sharding, buffer) }, + .rocm, .tpu, .neuron, .cpu => .{ .buffered = try BufferedMemoryWriter.init(allocator, io, platform, shape, sharding, buffer) }, }; } @@ -299,20 +316,29 @@ pub const MemoryWriter = union(enum) { .buffered => self.buffered.deinit(allocator), } } + + pub fn setProgress(self: *MemoryWriter, progress: ?*std.Progress.Node) void { + switch (self.*) { + .direct => self.direct.setProgress(progress), + .buffered => {}, + } + } }; pub const BufferedMemoryWriter = struct { io: std.Io, - memory: *const Memory, + platform: *const Platform, shape: Shape, + sharding: Sharding, buffer: *Buffer, interface: std.Io.Writer, - pub fn init(allocator: std.mem.Allocator, io: std.Io, memory: *const Memory, shape: Shape, buffer: *Buffer) !BufferedMemoryWriter { + pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, shape: Shape, sharding: Sharding, buffer: *Buffer) !BufferedMemoryWriter { return .{ .io = io, + .platform = platform, .shape = shape, - .memory = memory, + .sharding = sharding, .buffer = buffer, .interface = .{ .buffer = try allocator.alloc(u8, shape.byteSize()), @@ -331,36 +357,23 @@ pub const BufferedMemoryWriter = struct { } } - fn flush_(w: *std.Io.Writer) !void { - const self: *BufferedMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); - const pjrt_api = self.memory.platform.pjrt_api; - - const args: pjrt.Client.BufferFromHostBufferArgs = .{ - .data = @ptrCast(self.interface.buffer), - .buffer_type = pjrtx.bufferTypeFromDtype(self.shape.dtype()), - .dims = self.shape.dims(), - .byte_strides = self.shape.computeByteStrides().constSlice(), - .host_buffer_semantics = .ImmutableUntilTransferCompletes, - .dst = .{ .memory = self.memory.pjrt_memory }, - }; - const pjrt_buffer, const event = try self.memory.platform.pjrt_client.bufferFromHostBuffer(pjrt_api, args); - self.buffer.* = Buffer.fromPjrtBuffers(self.memory.platform, self.shape, &.{pjrt_buffer}); - if (event) |ev| { - defer ev.deinit(pjrt_api); - try ev.await(pjrt_api, self.io); - } else { - try self.buffer.await(self.io); - } - } - pub fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { - flush_(w) catch return std.Io.Writer.Error.WriteFailed; + const self: *BufferedMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); + + self.buffer.* = Buffer.from( + self.io, + self.platform, + self.shape, + self.sharding, + @ptrCast(self.interface.buffer), + .{ .wait = true }, + ) catch return std.Io.Writer.Error.WriteFailed; } }; -pub const DirectMemoryWriter = struct { +const DirectShardWriter = struct { const EventContext = struct { - self: *DirectMemoryWriter, + self: *DirectShardWriter, err: ?*pjrt.Error = null, pjrt_event: *pjrt.Event, event: std.Io.Event = .unset, @@ -372,7 +385,7 @@ pub const DirectMemoryWriter = struct { memory: *const Memory, pool: *mem.DynamicBufferPool, total: usize, - buffer: *Buffer, + pjrt_buffer: *pjrt.Buffer, transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, offset: usize = 0, @@ -381,7 +394,7 @@ pub const DirectMemoryWriter = struct { flip_flop: u1 = 0, events_contexts: [2]?EventContext = @splat(null), - pub fn init(allocator: std.mem.Allocator, io: std.Io, memory: *const Memory, pool: *mem.DynamicBufferPool, shape: Shape, buffer: *Buffer) !DirectMemoryWriter { + pub fn init(allocator: std.mem.Allocator, io: std.Io, memory: *const Memory, pool: *mem.DynamicBufferPool, shape: Shape) !DirectShardWriter { const shape_spec: pjrt.ShapeSpec = .init(shape.dims(), pjrtx.bufferTypeFromDtype(shape.dtype())); const transfer_manager = try memory.platform.pjrt_client.createBuffersForAsyncHostToDevice( memory.platform.pjrt_api, @@ -392,7 +405,6 @@ pub const DirectMemoryWriter = struct { ); const pjrt_buffer = transfer_manager.retrieveBuffer(memory.platform.pjrt_api, 0) catch unreachable; - buffer.* = Buffer.fromPjrtBuffers(memory.platform, shape, &.{pjrt_buffer}); return .{ .allocator = allocator, @@ -400,29 +412,99 @@ pub const DirectMemoryWriter = struct { .memory = memory, .pool = pool, .total = shape.byteSize(), - .buffer = buffer, + .pjrt_buffer = pjrt_buffer, .transfer_manager = transfer_manager, .interface = .{ .buffer = try pool.get(allocator, io), .end = 0, .vtable = &.{ .drain = drain, + .flush = flush, + .rebase = rebase, }, }, }; } - pub fn deinit(self: *DirectMemoryWriter) void { + pub fn deinit(self: *DirectShardWriter) void { self.transfer_manager.deinit(self.memory.platform.pjrt_api); } fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { - _ = data; // autofix - _ = splat; // autofix - const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); - if (self.offset >= self.total) { + const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); + var consumed: usize = 0; + + const pushChunk = struct { + fn call(self_: *DirectShardWriter, chunk: []const u8, consumed_: *usize) std.Io.Writer.Error!void { + var remaining = chunk; + while (remaining.len > 0) { + if (self_.offset + self_.interface.end >= self_.total) { + return std.Io.Writer.Error.WriteFailed; + } + + if (self_.interface.end == self_.interface.buffer.len) { + try self_.submitBuffered(); + } + + const writable_now = self_.interface.buffer.len - self_.interface.end; + const writable_to_total = self_.total - (self_.offset + self_.interface.end); + const to_copy = @min(remaining.len, @min(writable_now, writable_to_total)); + + @memcpy( + self_.interface.buffer[self_.interface.end .. self_.interface.end + to_copy], + remaining[0..to_copy], + ); + self_.interface.end += to_copy; + consumed_.* += to_copy; + remaining = remaining[to_copy..]; + + if (self_.interface.end == self_.interface.buffer.len or self_.offset + self_.interface.end == self_.total) { + try self_.submitBuffered(); + } + } + } + }.call; + + for (data) |chunk| { + try pushChunk(self, chunk, &consumed); + } + if (data.len > 0 and splat > 1) { + const last = data[data.len - 1]; + for (0..splat - 1) |_| { + try pushChunk(self, last, &consumed); + } + } + + return consumed; + } + + fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { + const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); + try self.flushBuffered(); + } + + fn rebase(w: *std.Io.Writer, preserve: usize, capacity: usize) std.Io.Writer.Error!void { + const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); + if (self.interface.buffer.len - self.interface.end >= capacity) return; + if (preserve != 0) return std.Io.Writer.Error.WriteFailed; + + if (self.interface.end > 0) { + try self.submitBuffered(); + } + + if (self.interface.buffer.len - self.interface.end < capacity) { return std.Io.Writer.Error.WriteFailed; } + } + + fn flushBuffered(self: *DirectShardWriter) std.Io.Writer.Error!void { + if (self.offset >= self.total) return; + if (self.interface.end == 0) return; + try self.submitBuffered(); + } + + fn submitBuffered(self: *DirectShardWriter) std.Io.Writer.Error!void { + if (self.offset >= self.total) return; const pjrt_api = self.memory.platform.pjrt_api; @@ -496,14 +578,489 @@ pub const DirectMemoryWriter = struct { } self.flip_flop ^= 1; self.offset += written; + } +}; - return 0; +pub const DirectMemoryWriter = struct { + const StreamPlanner = struct { + const AxisRange = struct { + start: i64, + size: i64, + }; + + allocator: std.mem.Allocator, + shape: Shape, + placement: Placement, + raw_segments: std.ArrayListUnmanaged(RawSegment) = .{}, + segments: std.ArrayListUnmanaged(StreamSegment) = .{}, + mirrors: std.ArrayListUnmanaged(usize) = .{}, + + fn init(allocator: std.mem.Allocator, shape: Shape, placement: Placement) StreamPlanner { + return .{ + .allocator = allocator, + .shape = shape, + .placement = placement, + }; + } + + fn deinit(self: *StreamPlanner) void { + self.raw_segments.deinit(self.allocator); + self.segments.deinit(self.allocator); + self.mirrors.deinit(self.allocator); + } + + fn build(self: *StreamPlanner) !StreamPlan { + try self.collectRawSegments(); + self.sortRawSegments(); + try self.compactSegments(); + + const owned_segments = try self.segments.toOwnedSlice(self.allocator); + errdefer self.allocator.free(owned_segments); + const owned_mirrors = try self.mirrors.toOwnedSlice(self.allocator); + + return .{ + .segments = owned_segments, + .mirrors = owned_mirrors, + }; + } + + fn collectRawSegments(self: *StreamPlanner) !void { + for (self.placement.shards.constSlice(), 0..) |shard, writer_index| { + try self.appendShardSegments(shard, writer_index); + } + } + + fn sortRawSegments(self: *StreamPlanner) void { + std.mem.sort(RawSegment, self.raw_segments.items, {}, struct { + fn lessThan(_: void, lhs: RawSegment, rhs: RawSegment) bool { + if (lhs.start != rhs.start) return lhs.start < rhs.start; + if (lhs.len != rhs.len) return lhs.len < rhs.len; + return lhs.writer_index < rhs.writer_index; + } + }.lessThan); + } + + fn compactSegments(self: *StreamPlanner) !void { + var i: usize = 0; + var cursor: usize = 0; + while (i < self.raw_segments.items.len) { + const first = self.raw_segments.items[i]; + if (first.start != cursor) return error.NonContiguousShardPlacement; + + const mirror_start = self.mirrors.items.len; + var j = i + 1; + while (j < self.raw_segments.items.len and + self.raw_segments.items[j].start == first.start and + self.raw_segments.items[j].len == first.len) : (j += 1) + { + try self.mirrors.append(self.allocator, self.raw_segments.items[j].writer_index); + } + + try self.segments.append(self.allocator, .{ + .len = first.len, + .primary_writer = first.writer_index, + .mirror_start = mirror_start, + .mirror_len = j - i - 1, + }); + cursor += first.len; + i = j; + } + + if (cursor != self.shape.byteSize()) return error.NonContiguousShardPlacement; + } + + fn appendRawSegment(self: *StreamPlanner, writer_index: usize, start: usize, len: usize) !void { + if (len == 0) return; + + if (self.raw_segments.items.len > 0) { + const last = &self.raw_segments.items[self.raw_segments.items.len - 1]; + if (last.writer_index == writer_index and last.start + last.len == start) { + last.len += len; + return; + } + } + + try self.raw_segments.append(self.allocator, .{ + .writer_index = writer_index, + .start = start, + .len = len, + }); + } + + fn axisRange(self: *const StreamPlanner, shard: Placement.Shard, axis: usize) AxisRange { + for (shard.slices.constSlice()) |slice| { + if (slice.axis == axis) { + return .{ + .start = slice.start, + .size = slice.size, + }; + } + } + + return .{ + .start = 0, + .size = self.shape.dim(axis), + }; + } + + fn appendShardSegmentsAtAxis( + self: *StreamPlanner, + shard: Placement.Shard, + writer_index: usize, + rank: usize, + axis: usize, + base_start: i64, + strides: []const i64, + ) !void { + const range = self.axisRange(shard, axis); + if (range.size == 0) return; + + if (axis + 1 == rank) { + const segment_start: usize = @intCast(base_start + range.start * strides[axis]); + const segment_len: usize = @intCast(range.size * strides[axis]); + try self.appendRawSegment(writer_index, segment_start, segment_len); + return; + } + + var i: i64 = 0; + while (i < range.size) : (i += 1) { + const child_start = base_start + (range.start + i) * strides[axis]; + try self.appendShardSegmentsAtAxis(shard, writer_index, rank, axis + 1, child_start, strides); + } + } + + fn appendShardSegments(self: *StreamPlanner, shard: Placement.Shard, writer_index: usize) !void { + const rank = self.shape.rank(); + if (rank == 0) { + try self.appendRawSegment(writer_index, 0, self.shape.byteSize()); + return; + } + + const strides = self.shape.computeByteStrides(); + try self.appendShardSegmentsAtAxis(shard, writer_index, rank, 0, 0, strides.constSlice()); + } + }; + + const StreamSegment = struct { + len: usize, + primary_writer: usize, + mirror_start: usize, + mirror_len: usize, + }; + + const RawSegment = struct { + writer_index: usize, + start: usize, + len: usize, + }; + + const StreamPlan = struct { + segments: []StreamSegment, + mirrors: []usize, + }; + + const ShardProgress = struct { + node: std.Progress.Node, + label: [32]u8 = undefined, + scale: usize = 1024, + }; + + allocator: std.mem.Allocator, + shard_writers: []DirectShardWriter, + stream_segments: []StreamSegment, + segment_mirrors: []usize, + segment_index: usize = 0, + segment_offset: usize = 0, + active_writer_index: ?usize = null, + shard_progress: ?[]ShardProgress = null, + interface: std.Io.Writer, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + pools: []mem.DynamicBufferPool, + dma_allocators: []const mem.DmaAllocator, + shape: Shape, + sharding: Sharding, + buffer: *Buffer, + ) !DirectMemoryWriter { + const placement = try Placement.init(sharding, shape); + var shard_writers = try allocator.alloc(DirectShardWriter, placement.shards.len); + errdefer allocator.free(shard_writers); + + var initialized: usize = 0; + errdefer for (shard_writers[0..initialized]) |*writer| { + writer.deinit(); + }; + + var pjrt_buffers: Buffer.Shards = .{}; + for (placement.shards.constSlice(), 0..) |shard, i| { + defer initialized += 1; + + const device_index = shard.platformDeviceIndex(platform) orelse return error.UnknownShardDevice; + const pool = &pools[device_index]; + const shard_dma_allocator = dma_allocators[device_index].allocator(); + + shard_writers[i] = try DirectShardWriter.init(shard_dma_allocator, io, shard.memory(platform, .default), pool, shard.shape); + + pjrt_buffers.appendAssumeCapacity(shard_writers[i].pjrt_buffer); + } + + buffer.* = Buffer.fromPjrtBuffers(platform, shape, sharding, pjrt_buffers.constSlice()); + + var planner = StreamPlanner.init(allocator, shape, placement); + defer planner.deinit(); + + const stream_plan = try planner.build(); + errdefer { + allocator.free(stream_plan.segments); + allocator.free(stream_plan.mirrors); + } + + const first_segment = stream_plan.segments[0]; + const initial_writer_index = first_segment.primary_writer; + const initial_writer = &shard_writers[initial_writer_index]; + const shard_remaining = initial_writer.total - (initial_writer.offset + initial_writer.interface.end); + const initial_visible_len: usize = @min( + initial_writer.interface.buffer.len, + initial_writer.interface.end + @min(first_segment.len, shard_remaining), + ); + + return .{ + .allocator = allocator, + .shard_writers = shard_writers, + .stream_segments = stream_plan.segments, + .segment_mirrors = stream_plan.mirrors, + .active_writer_index = initial_writer_index, + .interface = .{ + .buffer = initial_writer.interface.buffer[0..initial_visible_len], + .end = initial_writer.interface.end, + .vtable = &.{ + .drain = drain, + .flush = flush, + .rebase = rebase, + }, + }, + }; + } + + pub fn deinit(self: *DirectMemoryWriter) void { + self.setProgress(null); + for (self.shard_writers) |*writer| { + writer.deinit(); + } + self.allocator.free(self.shard_writers); + self.allocator.free(self.stream_segments); + self.allocator.free(self.segment_mirrors); + } + + pub fn setProgress(self: *DirectMemoryWriter, progress: ?*std.Progress.Node) void { + if (self.shard_progress) |states| { + for (states) |*s| { + s.node.end(); + } + self.allocator.free(states); + self.shard_progress = null; + } + + const parent = progress orelse return; + const states = self.allocator.alloc(ShardProgress, self.shard_writers.len) catch return; + for (states, self.shard_writers, 0..) |*state, writer, i| { + const scale: usize = 1024; + const total_items = std.math.divCeil(usize, writer.total, scale) catch unreachable; + const label = std.fmt.bufPrint(&state.label, "shard[{d}]", .{i}) catch unreachable; + state.scale = scale; + state.node = parent.start(label, total_items); + state.node.setCompletedItems(0); + } + self.shard_progress = states; + } + + fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); + var consumed: usize = 0; + + for (data) |chunk| { + consumed += try self.writeChunk(chunk); + } + if (data.len > 0 and splat > 1) { + const last = data[data.len - 1]; + for (0..splat - 1) |_| { + consumed += try self.writeChunk(last); + } + } + + return consumed; + } + + fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { + const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); + + try self.syncProxyIntoActive(); + self.refreshExposedWindow(); + + if (self.currentSegment() != null) return std.Io.Writer.Error.WriteFailed; + + for (self.shard_writers, 0..) |*shard_writer, i| { + if (shard_writer.interface.end > 0) { + try shard_writer.flushBuffered(); + } + if (shard_writer.offset < shard_writer.total) return std.Io.Writer.Error.WriteFailed; + self.updateShardProgress(i); + } + + self.interface = .failing; + } + + fn rebase(w: *std.Io.Writer, preserve: usize, capacity: usize) std.Io.Writer.Error!void { + const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); + if (preserve != 0) return std.Io.Writer.Error.WriteFailed; + try self.ensureWritable(capacity); + } + + fn ensureWritable(self: *DirectMemoryWriter, capacity: usize) std.Io.Writer.Error!void { + if (capacity == 0) return; + while (self.interface.buffer.len - self.interface.end < capacity) { + try self.syncProxyIntoActive(); + self.refreshExposedWindow(); + + const segment = self.currentSegment() orelse { + self.interface = .failing; + return std.Io.Writer.Error.WriteFailed; + }; + + if (self.interface.buffer.len - self.interface.end >= capacity) return; + if (capacity > self.interface.buffer.len and self.interface.buffer.len != 0) { + return std.Io.Writer.Error.WriteFailed; + } + + const shard_writer = &self.shard_writers[segment.primary_writer]; + if (shard_writer.interface.end == 0) return std.Io.Writer.Error.WriteFailed; + try shard_writer.flushBuffered(); + + self.refreshExposedWindow(); + if (self.interface.buffer.len - self.interface.end >= capacity) return; + + if (self.currentSegmentRemaining() == 0) continue; + return std.Io.Writer.Error.WriteFailed; + } + } + + fn currentSegment(self: *const DirectMemoryWriter) ?StreamSegment { + if (self.segment_index >= self.stream_segments.len) return null; + return self.stream_segments[self.segment_index]; + } + + fn currentSegmentRemaining(self: *const DirectMemoryWriter) usize { + const segment = self.currentSegment() orelse return 0; + return segment.len -| self.segment_offset; + } + + fn writeChunk(self: *DirectMemoryWriter, chunk: []const u8) std.Io.Writer.Error!usize { + var consumed: usize = 0; + while (consumed < chunk.len) { + try self.ensureWritable(1); + + const segment_remaining = self.currentSegmentRemaining(); + const writable = self.interface.buffer.len - self.interface.end; + const to_write = @min(chunk.len - consumed, @min(segment_remaining, writable)); + if (to_write == 0) return std.Io.Writer.Error.WriteFailed; + + @memcpy( + self.interface.buffer[self.interface.end .. self.interface.end + to_write], + chunk[consumed..][0..to_write], + ); + self.interface.end += to_write; + try self.syncProxyIntoActive(); + consumed += to_write; + } + + return consumed; + } + + fn refreshExposedWindow(self: *DirectMemoryWriter) void { + self.skipCompletedSegments(); + if (self.currentSegment()) |segment| { + self.exposeShardWindow(segment.primary_writer, self.currentSegmentRemaining()); + } else { + self.hideExposedWindow(); + } + } + + fn skipCompletedSegments(self: *DirectMemoryWriter) void { + while (self.currentSegment()) |_| { + if (self.currentSegmentRemaining() != 0) return; + self.segment_index += 1; + self.segment_offset = 0; + } + } + + fn hideExposedWindow(self: *DirectMemoryWriter) void { + self.active_writer_index = null; + stdx.debug.assert(self.shard_writers.len > 0, "DirectMemoryWriter requires at least one shard writer", .{}); + self.interface.buffer = self.shard_writers[0].interface.buffer[0..0]; + self.interface.end = 0; + } + + fn syncProxyIntoActive(self: *DirectMemoryWriter) std.Io.Writer.Error!void { + const writer_index = self.active_writer_index orelse return; + const segment = self.currentSegment() orelse return; + if (segment.primary_writer != writer_index) return; + + const shard_writer = &self.shard_writers[writer_index]; + if (self.interface.end <= shard_writer.interface.end) { + self.interface.end = shard_writer.interface.end; + return; + } + + const old_end = shard_writer.interface.end; + const new_end = self.interface.end; + if (new_end > shard_writer.interface.buffer.len) return std.Io.Writer.Error.WriteFailed; + const chunk = shard_writer.interface.buffer[old_end..new_end]; + if (chunk.len > self.currentSegmentRemaining()) return std.Io.Writer.Error.WriteFailed; + + shard_writer.interface.end = new_end; + self.segment_offset += chunk.len; + try self.mirrorChunk(segment, chunk); + self.updateShardProgress(writer_index); + } + + fn exposeShardWindow(self: *DirectMemoryWriter, writer_index: usize, segment_remaining: usize) void { + const shard_writer = &self.shard_writers[writer_index]; + self.active_writer_index = writer_index; + const shard_remaining = shard_writer.total - (shard_writer.offset + shard_writer.interface.end); + const visible_remaining = @min(shard_remaining, segment_remaining); + const visible_len = @min(shard_writer.interface.buffer.len, shard_writer.interface.end + visible_remaining); + self.interface.buffer = shard_writer.interface.buffer[0..visible_len]; + self.interface.end = shard_writer.interface.end; + } + + fn mirrorChunk(self: *DirectMemoryWriter, segment: StreamSegment, chunk: []const u8) std.Io.Writer.Error!void { + if (chunk.len == 0) return; + if (segment.mirror_len == 0) return; + + for (self.segment_mirrors[segment.mirror_start .. segment.mirror_start + segment.mirror_len]) |writer_index| { + const shard_writer = &self.shard_writers[writer_index]; + const wrote = try shard_writer.interface.vtable.drain(&shard_writer.interface, &.{chunk}, 1); + if (wrote != chunk.len) return std.Io.Writer.Error.WriteFailed; + self.updateShardProgress(writer_index); + } + } + + fn updateShardProgress(self: *DirectMemoryWriter, writer_index: usize) void { + const states = self.shard_progress orelse return; + const state = &states[writer_index]; + const writer = &self.shard_writers[writer_index]; + const completed_bytes = writer.offset + writer.interface.end; + state.node.setCompletedItems(completed_bytes / state.scale); } }; pub const LoadOpts = struct { parallelism: usize, store: *const TensorStore, + shardings: []const Sharding, progress: ?*std.Progress.Node = null, dma_chunks: usize, dma_chunk_size: usize, @@ -520,31 +1077,55 @@ pub fn load( ) !Bufferized(ModelType) { var bufferized = try mem.bufferize(allocator, ModelType, model); - const first_device = platform.devices[0]; // Temporary until sharding is re-exposed - const dma_alloc: mem.DmaAllocator = .init(allocator, &first_device); - var buffer_pool: mem.DynamicBufferPool = .init(opts.dma_chunks, opts.dma_chunk_size); - defer buffer_pool.deinit(dma_alloc.allocator()); + // todo: remove when https://codeberg.org/ziglang/zig/pulls/31320 is merged + var thread_safe_allocator: std.heap.ThreadSafeAllocator = .{ + .child_allocator = allocator, + .io = io, + }; + const concurrent_allocator = thread_safe_allocator.allocator(); + + const pool_count = platform.devices.len; + const dma_allocators = try allocator.alloc(mem.DmaAllocator, pool_count); + defer allocator.free(dma_allocators); + for (platform.devices, 0..) |*device, i| { + dma_allocators[i] = .init(concurrent_allocator, device); + } + + const buffer_pools = try allocator.alloc(mem.DynamicBufferPool, pool_count); + defer allocator.free(buffer_pools); + for (buffer_pools) |*pool_| { + pool_.* = .init(opts.dma_chunks, opts.dma_chunk_size); + } + defer for (buffer_pools, 0..) |*pool_, i| { + pool_.deinit(dma_allocators[i].allocator()); + }; + + const replicated_sharding = try sharding_.replicatedSharding(platform); const Ctx = struct { allocator: std.mem.Allocator, - dma_allocator: std.mem.Allocator, - pinned_buffer_pool: *mem.DynamicBufferPool, + dma_allocators: []const mem.DmaAllocator, + pinned_buffer_pools: []mem.DynamicBufferPool, io: std.Io, + platform: *const Platform, buffers: []*Buffer, + shardings: []const Sharding, + replicated_sharding: Sharding, store: *const TensorStore, - memory: *const Memory, group: stdx.Io.LimitedGroup, total: std.atomic.Value(usize) = .init(0), progress: ?*std.Progress.Node, }; var walk_ctx: Ctx = .{ + .platform = platform, .buffers = try allocator.alloc(*Buffer, meta.count(Tensor, model)), .store = opts.store, - .allocator = allocator, - .dma_allocator = dma_alloc.allocator(), - .pinned_buffer_pool = &buffer_pool, + .allocator = concurrent_allocator, + .dma_allocators = dma_allocators, + .pinned_buffer_pools = buffer_pools, .io = io, - .memory = first_device.memory(.default), + .shardings = opts.shardings, + .replicated_sharding = replicated_sharding, .progress = opts.progress, .group = .init(opts.parallelism), }; @@ -567,28 +1148,39 @@ pub fn load( var reader = ctx_.store.getReaderById(tensor_.id, ctx_.io, &.{}) catch unreachable; defer reader.deinit(); - var memory_writer = MemoryWriter.init( - ctx_.dma_allocator, + const shape = reader.tensor.shape; + const select_sharding = sharding_.pickSharding(ctx_.shardings, shape, .explicit_axis_binding); + const sharding = if (select_sharding) |s| s else blk: { + log.debug("No sharding strategy found for tensor {s} with shape {f}, using replicated sharding:\n {f}", .{ reader.tensor.name, shape, ctx_.replicated_sharding }); + break :blk ctx_.replicated_sharding; + }; + + var writer = MemoryWriter.init( + ctx_.allocator, ctx_.io, - ctx_.memory, - ctx_.pinned_buffer_pool, - reader.tensor.shape, + ctx_.platform, + ctx_.pinned_buffer_pools, + ctx_.dma_allocators, + shape, + sharding, ctx_.buffers[i_], ) catch unreachable; - defer memory_writer.deinit(ctx_.dma_allocator); + defer writer.deinit(ctx_.allocator); const scale = 1024; if (ctx_.progress) |progress| { var node = progress.start(reader.tensor.name, reader.tensor.shape.byteSize() / scale); defer node.end(); - var progress_writer: ProgressWriter = .init(memory_writer.interface(), &node, .{ .scale = scale }); + writer.setProgress(&node); + defer writer.setProgress(null); + var progress_writer: ProgressWriter = .init(writer.interface(), &node, .{ .scale = scale }); const total = reader.interface.streamRemaining(&progress_writer.interface) catch unreachable; progress_writer.interface.flush() catch unreachable; _ = ctx_.total.fetchAdd(total, .monotonic); } else { - const total = reader.interface.streamRemaining(memory_writer.interface()) catch unreachable; - memory_writer.interface().flush() catch unreachable; + const total = reader.interface.streamRemaining(writer.interface()) catch unreachable; + writer.interface().flush() catch unreachable; _ = ctx_.total.fetchAdd(total, .monotonic); } } @@ -599,3 +1191,325 @@ pub fn load( return bufferized; } + +fn buildMesh2x2( + allocator: std.mem.Allocator, + target: @import("platform.zig").Target, + devices: []const @import("platform.zig").Device, +) !sharding_.PhysicalMesh { + if (devices.len < 4) return error.NotEnoughDevices; + const topology: sharding_.PhysicalMesh.Tree = .axis(.link_x, .{ .mesh = .torus }, &.{ + .axis(.link_y, .{ .mesh = .torus }, &.{ + .device(devices[0]), + .device(devices[1]), + }), + .axis(.link_y, .{ .mesh = .torus }, &.{ + .device(devices[2]), + .device(devices[3]), + }), + }); + + return sharding_.PhysicalMesh.fromTree(allocator, target, topology); +} + +fn buildMesh2x2x2( + allocator: std.mem.Allocator, + target: @import("platform.zig").Target, + devices: []const @import("platform.zig").Device, +) !sharding_.PhysicalMesh { + if (devices.len < 8) return error.NotEnoughDevices; + const topology: sharding_.PhysicalMesh.Tree = .axis(.link_x, .{ .mesh = .torus }, &.{ + .axis(.link_y, .{ .mesh = .torus }, &.{ + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[0]), + .device(devices[1]), + }), + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[2]), + .device(devices[3]), + }), + }), + .axis(.link_y, .{ .mesh = .torus }, &.{ + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[4]), + .device(devices[5]), + }), + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[6]), + .device(devices[7]), + }), + }), + }); + + return sharding_.PhysicalMesh.fromTree(allocator, target, topology); +} + +const DirectMemoryWriterDeviceTest = struct { + const WriteMode = enum { + stream_remaining, + writable_slice_greedy, + }; + + pub const Scenario = struct { + name: []const u8, + create_options: CreateOptions, + shape: Shape, + logical_mesh: sharding_.LogicalMesh, + strategy: sharding_.Strategy, + write_mode: WriteMode = .stream_remaining, + writable_slice_min_len: usize = 128, + pool_chunks: usize = 4, + pool_chunk_size: usize = 1 << 20, + }; + + allocator: std.mem.Allocator, + io: std.Io, + + fn run(self: DirectMemoryWriterDeviceTest, scenario: Scenario) !void { + _ = scenario.name; + + var platform = Platform.auto(self.allocator, self.io, scenario.create_options) catch return error.SkipZigTest; + defer platform.deinit(self.allocator); + + const sharding: Sharding = try .initFromStrategy(platform, scenario.logical_mesh, scenario.strategy); + try self.runDirectMemoryWriter( + platform, + scenario.shape, + sharding, + scenario.write_mode, + scenario.writable_slice_min_len, + scenario.pool_chunks, + scenario.pool_chunk_size, + ); + } + + fn runDirectMemoryWriter( + self: DirectMemoryWriterDeviceTest, + platform: *const Platform, + shape: Shape, + sharding: Sharding, + write_mode: WriteMode, + writable_slice_min_len: usize, + pool_chunks: usize, + pool_chunk_size: usize, + ) !void { + const slice = try Slice.alloc(self.allocator, shape); + defer slice.free(self.allocator); + + for (slice.items(f32), 0..) |*e, i| { + e.* = @as(f32, @floatFromInt(i)); + } + + const pool_count = platform.devices.len; + const dma_allocators = try self.allocator.alloc(mem.DmaAllocator, pool_count); + defer self.allocator.free(dma_allocators); + for (platform.devices, 0..) |*device, i| { + dma_allocators[i] = .init(self.allocator, device); + } + + const pools = try self.allocator.alloc(mem.DynamicBufferPool, pool_count); + defer self.allocator.free(pools); + for (pools) |*pool| { + pool.* = .init(pool_chunks, pool_chunk_size); + } + defer for (pools, 0..) |*pool, i| { + pool.deinit(dma_allocators[i].allocator()); + }; + + var written_buffer: Buffer = undefined; + var writer: DirectMemoryWriter = try .init( + self.allocator, + self.io, + platform, + pools, + dma_allocators, + shape, + sharding, + &written_buffer, + ); + defer writer.deinit(); + defer written_buffer.deinit(); + + switch (write_mode) { + .stream_remaining => { + var reader: std.Io.Reader = .fixed(slice.constData()); + const streamed = try reader.streamRemaining(&writer.interface); + try std.testing.expectEqual(slice.constData().len, streamed); + }, + .writable_slice_greedy => { + var offset: usize = 0; + while (offset < slice.constData().len) { + const min_len = @max(@as(usize, 1), writable_slice_min_len); + const dest = try writer.interface.writableSliceGreedy(min_len); + const to_write = @min(dest.len, slice.constData().len - offset); + if (to_write == 0) return std.Io.Writer.Error.WriteFailed; + @memcpy(dest[0..to_write], slice.constData()[offset..][0..to_write]); + writer.interface.advance(to_write); + offset += to_write; + } + }, + } + + try writer.interface.flush(); + try written_buffer.await(self.io); + + var written_slice = try written_buffer.toSliceAlloc(self.allocator, self.io); + defer written_slice.free(self.allocator); + try std.testing.expectEqualSlices(u8, slice.constData(), written_slice.constData()); + } +}; + +test "DirectMemoryWriter: replicated with auto topology" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "replicated_auto", + .create_options = .{ + .physical_mesh = .auto, + .cpu = .{ .device_count = 4 }, + }, + .shape = Shape.init(.{ .rows = 8, .cols = 128 }, .f32) + .withPartitioning(.{ .rows = .replicated, .cols = .replicated }), + .logical_mesh = try .init("replicated_cpu", .{ .x = .high_bandwidth }), + .strategy = blk: { + var strategy: sharding_.Strategy = .init; + try strategy.addBinding(.x, .link_x); + + break :blk strategy; + }, + }); +} + +test "DirectMemoryWriter: 1D model split with 2x2 physical mesh" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "model_auto", + .create_options = .{ + .physical_mesh = .{ .custom = buildMesh2x2 }, + .cpu = .{ .device_count = 4 }, + }, + .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32) + .withPartitioning(.{ .rows = .replicated, .cols = .model }), + .logical_mesh = try .init("model_cpu", .{ .model = .high_bandwidth }), + .strategy = blk: { + var strategy: sharding_.Strategy = .init; + try strategy.addBinding(.model, .link_x); + + break :blk strategy; + }, + }); +} + +test "DirectMemoryWriter: 2D batch/model split with 2x2 physical mesh" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "batch_model_2d_torus", + .create_options = .{ + .physical_mesh = .{ .custom = buildMesh2x2 }, + .cpu = .{ .device_count = 4 }, + }, + .shape = Shape.init(.{ .batch = 8, .model = 1024 }, .f32) + .withPartitioning(.{ .batch = .batch, .model = .model }), + .logical_mesh = try .init("batch_model_cpu", .{ + .batch = .low_bandwidth, + .model = .high_bandwidth, + }), + .strategy = blk: { + var strategy: sharding_.Strategy = .init; + try strategy.addBinding(.batch, .link_x); + try strategy.addBinding(.model, .link_y); + + break :blk strategy; + }, + }); +} + +test "DirectMemoryWriter: folded model sharding with 2x2 physical mesh" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "model_folded_2d_torus", + .create_options = .{ + .physical_mesh = .{ .custom = buildMesh2x2 }, + .cpu = .{ .device_count = 4 }, + }, + .shape = Shape.init(.{ .model = 4096 }, .f32).withPartitioning(.{ .model = .model }), + .logical_mesh = try .init("model_folded_cpu", .{ .model = .high_bandwidth }), + .strategy = blk: { + var strategy: sharding_.Strategy = .init; + try strategy.addBinding(.model, .link_x); + try strategy.addFold(.link_x, &.{ .link_x, .link_y }); + + break :blk strategy; + }, + }); +} + +test "DirectMemoryWriter: writableSliceGreedy with mirrored shards" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "model_auto_writable_slice", + .create_options = .{ + .physical_mesh = .{ .custom = buildMesh2x2 }, + .cpu = .{ .device_count = 4 }, + }, + .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32) + .withPartitioning(.{ .rows = .replicated, .cols = .model }), + .logical_mesh = try .init("model_cpu_greedy", .{ .model = .high_bandwidth }), + .strategy = blk: { + var strategy: sharding_.Strategy = .init; + try strategy.addBinding(.model, .link_x); + + break :blk strategy; + }, + .write_mode = .writable_slice_greedy, + .writable_slice_min_len = 64, + .pool_chunk_size = 1024, + }); +} + +test "DirectMemoryWriter: 3D topology folded model + replicated batch" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "topology_3d_folded_model", + .create_options = .{ + .physical_mesh = .{ .custom = buildMesh2x2x2 }, + .cpu = .{ .device_count = 8 }, + }, + .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32) + .withPartitioning(.{ .batch = .replicated, .model = .model }), + .logical_mesh = try .init("topology_3d_folded_model_mesh", .{ + .batch = .low_bandwidth, + .model = .high_bandwidth, + }), + .strategy = blk: { + var strategy: sharding_.Strategy = .init; + try strategy.addBinding(.model, .link_x); + try strategy.addFold(.link_x, &.{ .link_x, .link_z }); + + break :blk strategy; + }, + }); +} diff --git a/zml/module.zig b/zml/module.zig index b3d429f5..c65017ad 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -15,6 +15,9 @@ const mlirx = @import("mlirx.zig"); const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; +const sharding_ = @import("sharding.zig"); +const Sharding = sharding_.Sharding; +const Partitioning = sharding_.Partitioning; const Tensor = @import("tensor.zig").Tensor; const log = std.log.scoped(.@"zml/module"); @@ -30,7 +33,7 @@ fn mlirRegistry(io: std.Io) *mlir.DialectRegistry { mlir.registerPasses("Transforms"); const mlir_registry = mlir.DialectRegistry.init() catch unreachable; - inline for (.{ "func", "stablehlo" }) |d| { + inline for (.{ "func", "stablehlo", "sdy" }) |d| { mlir.DialectHandle.fromString(d).insertDialect(mlir_registry); } mlir.registerFuncExtensions(mlir_registry); @@ -40,6 +43,16 @@ fn mlirRegistry(io: std.Io) *mlir.DialectRegistry { return mlir_global_registry.?; } +pub const CompilationOptions = struct { + shardings: []const Sharding, + // If null, will be initialized from the target + partitioner: ?Partitioning.Partitioner = null, + // Debugging options + program_name: []const u8 = "zml", + xla_dump_to: ?[]const u8 = null, + xla_dump_fusion_visualization: bool = false, + xla_dump_hlo_pass_re: ?[]const u8 = null, +}; const AttributeList = stdx.BoundedArray(mlir.NamedAttribute, 3); @@ -73,22 +86,22 @@ pub const CompilationContext = struct { mlir_registry: *mlir.DialectRegistry, mlir_ctx: *mlir.Context, mlir_pass_manager: *mlir.PassManager, - //mlir_op_pass_manager: *mlir.OpPassManager, module: *mlir.Module, platform: *const Platform, + partitioning: Partitioning, scopes: stdx.BoundedArray(Scope, 16) = .{}, + manual_computation_depth: usize = 0, threadlocal var _current: ?*CompilationContext = null; - pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform) CompilationContext { + pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: CompilationOptions) CompilationContext { const mlir_registry = mlirRegistry(io); var mlir_ctx = mlir.Context.init(.{ .registry = mlir_registry, .threading = false }) catch unreachable; mlir_ctx.loadAllAvailableDialects(); - //const loc = mlir.Location.fromSrc(mlir_ctx, @src()).named(mlir_ctx, "main"); const module = mlir.Module.init(.unknown(mlir_ctx)); - module.operation().setAttributeByName("sym_name", mlir.stringAttribute(mlir_ctx, "zml")); + module.operation().setAttributeByName("sym_name", mlir.stringAttribute(mlir_ctx, opts.program_name)); const pass_manager = mlir.PassManager.init(mlir_ctx); { @@ -103,6 +116,8 @@ pub const CompilationContext = struct { } } + const partitioning = Partitioning.init(opts.partitioner orelse Partitioning.Partitioner.fromTarget(platform.target), opts.shardings) catch unreachable; + return .{ .allocator = allocator, .arena = std.heap.ArenaAllocator.init(allocator), @@ -111,6 +126,7 @@ pub const CompilationContext = struct { .mlir_pass_manager = pass_manager, .module = module, .platform = platform, + .partitioning = partitioning, }; } @@ -150,41 +166,6 @@ pub const CompilationContext = struct { popped.deinit(); } } - - //pub fn addShardingAttributes( - // self: *const CompilationContext, - // mesh: Mesh, - // arg_attrs: []AttributeList, - // res_attrs: []AttributeList, - // input_shapes: []const Shape, - // input_meshes: []const ?Mesh, - // output_info: OutputInfo, - //) void { - // if (mesh.isSinglePartition()) return; // No sharding attributes for single partition. - - // const default_layout = mlir.NamedAttribute.named(self.mlir_ctx, "mhlo.layout_mode", mlir.stringAttribute(self.mlir_ctx, "default")); - // for (arg_attrs, input_shapes, input_meshes) |*attr, shape, input_mesh| { - // attr.appendAssumeCapacity(default_layout); - // if (input_mesh) |m| { - // // log.warn("addShardingAttributes for {} with input mesh: {}", .{ shape, m }); - // attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(m, shape)))); - // } else { - // // log.warn("addShardingAttributes for {} with progran mesh: {}", .{ shape, mesh }); - // attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(mesh, shape)))); - // } - // } - - // for (res_attrs, output_info.shapes) |*attr, shape| { - // attr.appendAssumeCapacity(default_layout); - // attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(mesh, shape)))); - // } - //} - - //pub fn getShardingAttr(self: *const CompilationContext, sharding: Sharding) *const mlir.Attribute { - // var sharding_str: stdx.BoundedArray(u8, 128) = .{}; - // sharding.writeShardingRepresentation(sharding_str.writer()) catch unreachable; - // return mlir.stringAttribute(self.mlir_ctx, sharding_str.constSlice()); - //} }; pub fn compile( @@ -193,74 +174,121 @@ pub fn compile( comptime func: anytype, args: std.meta.ArgsTuple(@TypeOf(func)), platform: *const Platform, + opts: CompilationOptions, ) !Exe { - var compilation_context: CompilationContext = .init(allocator, io, platform); + var compilation_context: CompilationContext = .init(allocator, io, platform, opts); defer compilation_context.deinit(); const result = emitMlir(&compilation_context, func, args) catch unreachable; defer result.output_info.deinit(compilation_context.allocator); defer result.input_info.deinit(compilation_context.allocator); + try addPartitionerOperations(&compilation_context); + _ = result.func.appendTo(compilation_context.module.body()); - const sharding = platform.sharding(); - compilation_context.module.operation().setAttributeByName("mhlo.num_replicas", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, sharding.num_replicas)); - compilation_context.module.operation().setAttributeByName("mhlo.num_partitions", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, sharding.num_partitions)); + + const num_partitions = compilation_context.partitioning.numPartitions(); + const num_replicas = compilation_context.partitioning.numReplicas(); + const num_devices = compilation_context.partitioning.numDevices(); + + compilation_context.module.operation().setAttributeByName( + "mhlo.num_partitions", + mlir.integerAttribute(compilation_context.mlir_ctx, .i32, num_partitions), + ); + compilation_context.module.operation().setAttributeByName( + "mhlo.num_replicas", + mlir.integerAttribute(compilation_context.mlir_ctx, .i32, num_replicas), + ); + + compilation_context.mlir_pass_manager.runOnOp(compilation_context.module.operation()) catch |err| switch (err) { + error.MlirUnexpected => { + std.log.err("Failed to canonicalize invalid mlir: \n {f} \n ", .{compilation_context.module.operation()}); + @panic("ZML generated invalid mlir. Please open a bug report"); + }, + }; var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); - const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), io, platform, compilation_context.module, null) catch unreachable; - + const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), io, platform, compilation_context.module, compilation_context.partitioning, opts) catch unreachable; log.debug("\n******** ZML generated MLIR ********\n{f}", .{compilation_context.module.operation()}); - const num_devices = sharding.num_partitions * sharding.num_replicas; - const exe = try Exe.init(allocator, platform, loaded_executable, result.input_info.shapes, result.output_info.shapes, num_devices); + const exe = try Exe.init( + allocator, + platform, + loaded_executable, + @intCast(num_devices), + num_partitions, + result.input_info.shapes, + result.output_info.shapes, + result.input_info.shardings, + result.output_info.shardings, + ); errdefer exe.deinit(); return exe; } -fn collectShapes(allocator: std.mem.Allocator, v: anytype) ![]Shape { - const LocalContext = struct { - list: *std.array_list.Managed(Shape), - }; - var list = std.array_list.Managed(Shape).init(allocator); - errdefer list.deinit(); +fn addPartitionerOperations(compilation_context: *CompilationContext) !void { + const allocator = compilation_context.arena.allocator(); + const mlir_ctx = compilation_context.mlir_ctx; + const module = compilation_context.module; + const partitioning = compilation_context.partitioning; - var context: LocalContext = .{ .list = &list }; - try meta.visit(struct { - fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { - try ctx_.list.append(tensor.shape()); - } - }.cb, &context, v); + switch (partitioning.partitioner) { + .gspmd => {}, + .shardy => { + for (partitioning.shardings) |sharding| { + const attr_str = try sharding.sdyMeshAttr(allocator); + defer allocator.free(attr_str); - return try list.toOwnedSlice(); + const mesh_attr = try mlir.Attribute.parse(mlir_ctx, attr_str); + + const mesh_op = mlir.Operation.make(mlir_ctx, "sdy.mesh", .{ + .attributes = &.{ + .named(mlir_ctx, "sym_name", mlir.stringAttribute(mlir_ctx, sharding.name())), + .named(mlir_ctx, "mesh", @ptrCast(mesh_attr)), + }, + .location = .unknown(mlir_ctx), + .verify = false, + }); + + _ = mesh_op.appendTo(module.body()); + } + }, + } } pub const OutputInfo = struct { shapes: []Shape, + shardings: []Sharding, values: []*const mlir.Value, donations: []?usize, output_memory_kinds: []Memory.Kind, pub fn deinit(self: OutputInfo, allocator: std.mem.Allocator) void { allocator.free(self.shapes); + allocator.free(self.shardings); allocator.free(self.values); allocator.free(self.donations); allocator.free(self.output_memory_kinds); } }; -fn collectOutputInfo(allocator: std.mem.Allocator, v: anytype) !OutputInfo { +fn collectOutputInfo(allocator: std.mem.Allocator, partitioning: Partitioning, v: anytype) !OutputInfo { const LocalContext = struct { shape_list: *std.array_list.Managed(Shape), + sharding_list: *std.array_list.Managed(Sharding), value_list: *std.array_list.Managed(*const mlir.Value), donation_list: *std.array_list.Managed(?usize), output_memory_kind_list: *std.array_list.Managed(Memory.Kind), + partitioning: Partitioning, }; var shape_list = std.array_list.Managed(Shape).init(allocator); errdefer shape_list.deinit(); + var sharding_list = std.array_list.Managed(Sharding).init(allocator); + errdefer sharding_list.deinit(); var value_list = std.array_list.Managed(*const mlir.Value).init(allocator); errdefer value_list.deinit(); var donation_list = std.array_list.Managed(?usize).init(allocator); @@ -270,14 +298,17 @@ fn collectOutputInfo(allocator: std.mem.Allocator, v: anytype) !OutputInfo { var context: LocalContext = .{ .shape_list = &shape_list, + .sharding_list = &sharding_list, .value_list = &value_list, .donation_list = &donation_list, .output_memory_kind_list = &output_memory_kind_list, + .partitioning = partitioning, }; try meta.visit(struct { fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { try ctx_.shape_list.append(tensor.shape()); + try ctx_.sharding_list.append(try ctx_.partitioning.selectSharding(tensor.shape())); try ctx_.value_list.append(tensor.value()); try ctx_.donation_list.append(tensor.donation()); try ctx_.output_memory_kind_list.append(tensor.outputMemoryKind()); @@ -286,6 +317,7 @@ fn collectOutputInfo(allocator: std.mem.Allocator, v: anytype) !OutputInfo { return .{ .shapes = try shape_list.toOwnedSlice(), + .shardings = try sharding_list.toOwnedSlice(), .values = try value_list.toOwnedSlice(), .donations = try donation_list.toOwnedSlice(), .output_memory_kinds = try output_memory_kind_list.toOwnedSlice(), @@ -294,30 +326,43 @@ fn collectOutputInfo(allocator: std.mem.Allocator, v: anytype) !OutputInfo { pub const InputInfo = struct { shapes: []Shape, + shardings: []Sharding, pub fn deinit(self: InputInfo, allocator: std.mem.Allocator) void { allocator.free(self.shapes); + allocator.free(self.shardings); } }; -fn collectInputInfo(allocator: std.mem.Allocator, v: anytype) !InputInfo { +fn collectInputInfo(allocator: std.mem.Allocator, partitioning: Partitioning, v: anytype) !InputInfo { const LocalContext = struct { shape_list: *std.array_list.Managed(Shape), + sharding_list: *std.array_list.Managed(Sharding), + partitioning: Partitioning, }; var shape_list = std.array_list.Managed(Shape).init(allocator); errdefer shape_list.deinit(); - var context: LocalContext = .{ .shape_list = &shape_list }; + var sharding_list = std.array_list.Managed(Sharding).init(allocator); + errdefer sharding_list.deinit(); + + var context: LocalContext = .{ + .shape_list = &shape_list, + .sharding_list = &sharding_list, + .partitioning = partitioning, + }; try meta.visit(struct { fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { try ctx_.shape_list.append(tensor.shape()); + try ctx_.sharding_list.append(try ctx_.partitioning.selectSharding(tensor.shape())); } }.cb, &context, v); return .{ .shapes = try shape_list.toOwnedSlice(), + .shardings = try sharding_list.toOwnedSlice(), }; } @@ -367,7 +412,7 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar } }.cb, &context, &args); - const input_info = try collectInputInfo(compilation_context.allocator, &args); + const input_info = try collectInputInfo(compilation_context.allocator, compilation_context.partitioning, &args); errdefer input_info.deinit(compilation_context.allocator); const input_attributes = try arena.allocator().alloc(AttributeList, input_info.shapes.len); @@ -379,7 +424,7 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar const result = @call(.auto, func, args); - const output_info = try collectOutputInfo(compilation_context.allocator, &result); + const output_info = try collectOutputInfo(compilation_context.allocator, compilation_context.partitioning, &result); errdefer output_info.deinit(compilation_context.allocator); break :b output_info; @@ -405,22 +450,37 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar } _ = dialects.func.returns(compilation_context.mlir_ctx, output_info.values, .unknown(compilation_context.mlir_ctx)).appendTo(compilation_context.currentScope().block); + for (input_info.shapes, input_info.shardings, 0..) |shape, sharding, i| { + const attr_str = try compilation_context.partitioning.tensorShardingAttr(compilation_context.arena.allocator(), shape, sharding); + + const name, const attr = switch (compilation_context.partitioning.partitioner) { + .gspmd => .{ "mhlo.sharding", mlir.stringAttribute(compilation_context.mlir_ctx, attr_str) }, + .shardy => .{ "sdy.sharding", try mlir.Attribute.parse(compilation_context.mlir_ctx, attr_str) }, + }; + + input_attributes[i].appendAssumeCapacity(.named(compilation_context.mlir_ctx, name, attr)); + } + + for (output_info.shapes, output_info.shardings, 0..) |shape, sharding, i| { + const attr_str = try compilation_context.partitioning.tensorShardingAttr(compilation_context.arena.allocator(), shape, sharding); + + const name, const attr = switch (compilation_context.partitioning.partitioner) { + .gspmd => .{ "mhlo.sharding", mlir.stringAttribute(compilation_context.mlir_ctx, attr_str) }, + .shardy => .{ "sdy.sharding", try mlir.Attribute.parse(compilation_context.mlir_ctx, attr_str) }, + }; + + output_attributes[i].appendAssumeCapacity(.named(compilation_context.mlir_ctx, name, attr)); + } + const mlir_func = dialects.func.func(compilation_context.mlir_ctx, .{ .name = "main", .block = compilation_context.currentScope().block, .location = .unknown(compilation_context.mlir_ctx), .args_attributes = try finalizeAttributeList(arena.allocator(), compilation_context.mlir_ctx, input_attributes), .results_attributes = try finalizeAttributeList(arena.allocator(), compilation_context.mlir_ctx, output_attributes), + .verify = false, }); - compilation_context.mlir_pass_manager.runOnOp(mlir_func) catch |err| switch (err) { - error.MlirUnexpected => { - std.log.err("Failed to canonicalize invalid mlir: {f}", .{mlir_func}); - // user errors should have triggered a panic before we reach this. - @panic("ZML generated invalid mlir. Please open a bug report"); - }, - }; - return .{ .func = mlir_func, .input_info = input_info, @@ -450,39 +510,53 @@ fn setXlaOverrideFlag(map: *c.upb_Map, flag: []const u8, value: anytype, upb_are } } -fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: *const Platform, module: *const mlir.Module, xla_dump_to_: ?[]const u8) !*pjrt.LoadedExecutable { - //const tracer = Tracer.init("ai.zml.compilation"); - //const compile_frame = tracer.frameStart("pjrt compilation"); - //defer tracer.frameEnd(compile_frame, "pjrt compilation"); - - const sharding = platform.sharding(); - +fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: *const Platform, module: *const mlir.Module, partitioning: Partitioning, opts: CompilationOptions) !*pjrt.LoadedExecutable { var upb_alloc: upb.Allocator = .init(arena); const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); defer c.upb_Arena_Free(upb_arena); + const use_shardy_partitioner = switch (partitioning.partitioner) { + .shardy => true, + .gspmd => false, + }; + + const num_partitions = partitioning.numPartitions(); + const num_replicas = partitioning.numReplicas(); + + const device_assignment = try partitioning.deviceAssignment(arena); + const options = blk: { const options = try upb.new(c.xla_CompileOptionsProto, upb_arena); c.xla_CompileOptionsProto_set_executable_build_options(options, executable_build_options_blk: { const exec_build_options = try upb.new(c.xla_ExecutableBuildOptionsProto, upb_arena); - c.xla_ExecutableBuildOptionsProto_set_device_ordinal(exec_build_options, -1); - c.xla_ExecutableBuildOptionsProto_set_num_replicas(exec_build_options, sharding.num_replicas); - c.xla_ExecutableBuildOptionsProto_set_num_partitions(exec_build_options, sharding.num_partitions); - c.xla_ExecutableBuildOptionsProto_set_use_spmd_partitioning(exec_build_options, sharding.num_partitions > 1 or sharding.num_replicas > 1); + c.xla_ExecutableBuildOptionsProto_set_num_replicas(exec_build_options, num_replicas); + c.xla_ExecutableBuildOptionsProto_set_num_partitions(exec_build_options, num_partitions); + c.xla_ExecutableBuildOptionsProto_set_use_spmd_partitioning(exec_build_options, true); + c.xla_ExecutableBuildOptionsProto_set_use_shardy_partitioner(exec_build_options, use_shardy_partitioner); c.xla_ExecutableBuildOptionsProto_set_device_assignment(exec_build_options, device_assignment_blk: { - const device_assignment = try upb.new(c.xla_DeviceAssignmentProto, upb_arena); + const device_assignment_proto = try upb.new(c.xla_DeviceAssignmentProto, upb_arena); - c.xla_DeviceAssignmentProto_set_replica_count(device_assignment, sharding.num_replicas); - c.xla_DeviceAssignmentProto_set_computation_count(device_assignment, sharding.num_partitions); + c.xla_DeviceAssignmentProto_set_replica_count(device_assignment_proto, num_replicas); + c.xla_DeviceAssignmentProto_set_computation_count(device_assignment_proto, num_partitions); - const computation_devices = c.xla_DeviceAssignmentProto_resize_computation_devices(device_assignment, sharding.num_partitions, upb_arena); - for (computation_devices[0..sharding.num_partitions], 0..) |*computation_device, i| { + const computation_devices = c.xla_DeviceAssignmentProto_resize_computation_devices( + device_assignment_proto, + @intCast(num_partitions), + upb_arena, + ); + + for (computation_devices[0..@intCast(num_partitions)], 0..) |*computation_device, i| { computation_device.* = try upb.new(c.xla_DeviceAssignmentProto_ComputationDevice, upb_arena); - _ = c.xla_DeviceAssignmentProto_ComputationDevice_add_replica_device_ids(computation_device.*, @intCast(i), upb_arena); + _ = c.xla_DeviceAssignmentProto_ComputationDevice_add_replica_device_ids( + computation_device.*, + @intCast(device_assignment[@intCast(i)]), + upb_arena, + ); } - break :device_assignment_blk device_assignment; + + break :device_assignment_blk device_assignment_proto; }); break :executable_build_options_blk exec_build_options; @@ -524,13 +598,13 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: else => {}, } - if (xla_dump_to_ orelse platform.compilation_options.xla_dump_to) |xla_dump_to| { + if (opts.xla_dump_to) |xla_dump_to| { try setXlaOverrideFlag(overrides_map, "xla_dump_to", xla_dump_to, upb_arena); try setXlaOverrideFlag(overrides_map, "xla_dump_hlo_as_proto", true, upb_arena); - if (platform.compilation_options.xla_dump_fusion_visualization) { + if (opts.xla_dump_fusion_visualization) { try setXlaOverrideFlag(overrides_map, "xla_dump_fusion_visualization", true, upb_arena); } - if (platform.compilation_options.xla_dump_hlo_pass_re) |re| { + if (opts.xla_dump_hlo_pass_re) |re| { try setXlaOverrideFlag(overrides_map, "xla_dump_hlo_pass_re", re, upb_arena); } } diff --git a/zml/nn.zig b/zml/nn.zig index a903c712..4df96114 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -3,11 +3,11 @@ const std = @import("std"); const stdx = @import("stdx"); -const Slice = @import("slice.zig").Slice; const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); const ops = @import("ops.zig"); const Shape = @import("shape.zig").Shape; +const Slice = @import("slice.zig").Slice; const Tensor = @import("tensor.zig").Tensor; const zml = @import("zml.zig"); @@ -127,17 +127,17 @@ test normalizeL2 { const input: zml.Tensor = .init(.{ 2, 2 }, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, normalizeL2, .{ input, 1e-12 }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, normalizeL2, .{ input, 1e-12 }, platform, .{ .shardings = &.{zml.testing.replicatedSharding()} }); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.9686, -1.0058, -1.7808, 0.6698 })); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), zml.testing.replicatedSharding(), std.mem.sliceAsBytes(&[_]f32{ -0.9686, -1.0058, -1.7808, 0.6698 })); defer input_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, normalizeL2, .{input_buffer}); defer res.deinit(); const expectation: Slice = .init(input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6937, -0.7203, -0.9360, 0.3520 })); - try zml.testing.expectClose(std.testing.io, expectation, res, 1e-4); + try zml.testing.expectClose(std.testing.io, expectation, res, .{}); } pub const RopeOpts = struct { @@ -427,6 +427,7 @@ test "invFreq Yarn" { .beta_slow = 1.0, .original_max_position_embeddings = 4096, .truncate = true, + .rope_theta = 10000, } }, }; const yarn_freq = [_]f32{ 1.000000000000e+00, 6.890442967415e-01, 4.747820496559e-01, 3.271458745003e-01, 2.254180014133e-01, 1.553229838610e-01, 1.070244237781e-01, 7.374456524849e-02, 5.081327259541e-02, 3.162075206637e-02, 1.945096626878e-02, 1.179219130427e-02, 7.015713956207e-03, 4.069554619491e-03, 2.277272054926e-03, 1.206130953506e-03, 5.809474969283e-04, 2.279478358105e-04, 3.830881178146e-05, 2.639646845637e-05, 1.818833698053e-05, 1.253256959899e-05, 8.635495760245e-06, 5.950239483354e-06, 4.099978468730e-06, 2.825066758305e-06, 1.946596285052e-06, 1.341290953860e-06, 9.242089618056e-07, 6.368209142238e-07, 4.387978549403e-07, 3.023511396805e-07 }; @@ -500,7 +501,7 @@ test "real/img" { } }; { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.interleaved}, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.interleaved}, platform, .{ .shardings = &.{zml.testing.replicatedSharding()} }); defer exe.deinit(); var d_interleaved = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitMergeIsId, {}); @@ -508,7 +509,7 @@ test "real/img" { try std.testing.expectEqual(20, try d_interleaved.getValue(i32, std.testing.io)); } { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.sequential}, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.sequential}, platform, .{ .shardings = &.{zml.testing.replicatedSharding()} }); defer exe.deinit(); var d_sequential = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitMergeIsId, {}); @@ -518,7 +519,7 @@ test "real/img" { // test the function that accepts 1 void argument { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitSeqVoid, .{{}}, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitSeqVoid, .{{}}, platform, .{ .shardings = &.{zml.testing.replicatedSharding()} }); defer exe.deinit(); var d_split_seq_void = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitSeqVoid, {}); @@ -528,7 +529,7 @@ test "real/img" { // test the function that takes NO arguments { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitSeq, .{}, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitSeq, .{}, platform, .{ .shardings = &.{zml.testing.replicatedSharding()} }); defer exe.deinit(); var d_split_seq = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitSeq, {}); @@ -537,7 +538,7 @@ test "real/img" { } { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitInterleaved, .{}, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitInterleaved, .{}, platform, .{ .shardings = &.{zml.testing.replicatedSharding()} }); defer exe.deinit(); var d_split_seq = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitInterleaved, {}); @@ -571,13 +572,13 @@ test rope { // x is made such as the interleaved and sequential reps are the same. // So the two implementations should give the same results. const x: zml.Tensor = .init(.{ .b = 1, .s = 5, .hd = 4 }, .f32); - var exe_interleaved = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .interleaved } }, platform); + var exe_interleaved = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .interleaved } }, platform, .{ .shardings = &.{zml.testing.replicatedSharding()} }); defer exe_interleaved.deinit(); - var exe_sequential = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .sequential } }, platform); + var exe_sequential = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .sequential } }, platform, .{ .shardings = &.{zml.testing.replicatedSharding()} }); defer exe_sequential.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ 1.0, 0.1, -1.0, -0.5 } ** 5)); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), zml.testing.replicatedSharding(), std.mem.sliceAsBytes(&[_]f32{ 1.0, 0.1, -1.0, -0.5 } ** 5)); defer x_buffer.deinit(); var res1 = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_interleaved, Local._fwd, .{x_buffer}); @@ -585,7 +586,7 @@ test rope { var res2 = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_sequential, Local._fwd, .{x_buffer}); defer res2.deinit(); - try zml.testing.expectClose(std.testing.io, res1, res2, 1e-4); + try zml.testing.expectClose(std.testing.io, res1, res2, .{}); } pub const UpsampleMode = enum { @@ -650,10 +651,23 @@ test nearest { // 3D Tensor (basic) { const input_3d_basic: zml.Tensor = .init(.{ 1, 1, 2 }, .i32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_3d_basic, .{ .scale_factor = &.{3}, .mode = .nearest } }, platform); + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + upsample, + .{ input_3d_basic, .{ .scale_factor = &.{3}, .mode = .nearest } }, + platform, + .{ .shardings = &.{zml.testing.replicatedSharding()} }, + ); defer exe.deinit(); - var input_3d_basic_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_3d_basic.shape(), std.mem.sliceAsBytes(&[1][1][2]i32{.{.{ 1, 2 }}})); + var input_3d_basic_buffer: zml.Buffer = try .fromBytes( + std.testing.io, + platform, + input_3d_basic.shape(), + zml.testing.replicatedSharding(), + std.mem.sliceAsBytes(&[1][1][2]i32{.{.{ 1, 2 }}}), + ); defer input_3d_basic_buffer.deinit(); var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_3d_basic_buffer}); @@ -661,16 +675,23 @@ test nearest { try std.testing.expectEqualSlices(i64, &.{ 1, 1, 6 }, result.shape().dims()); const expected: Slice = .init(Shape.init(.{ 1, 1, 6 }, .i32), std.mem.sliceAsBytes(&[1][1][6]i32{.{.{ 1, 1, 1, 2, 2, 2 }}})); - try zml.testing.expectClose(std.testing.io, expected, result, 0); + try zml.testing.expectClose(std.testing.io, expected, result, .{}); } // 3D Tensor (advanced) { const input_3d_advanced: zml.Tensor = .init(.{ 2, 3, 4 }, .i32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_3d_advanced, .{ .scale_factor = &.{2}, .mode = .nearest } }, platform); + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + upsample, + .{ input_3d_advanced, .{ .scale_factor = &.{2}, .mode = .nearest } }, + platform, + .{ .shardings = &.{zml.testing.replicatedSharding()} }, + ); defer exe.deinit(); - var input_3d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_3d_advanced.shape(), std.mem.sliceAsBytes(&[2][3][4]i32{ + var input_3d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_3d_advanced.shape(), zml.testing.replicatedSharding(), std.mem.sliceAsBytes(&[2][3][4]i32{ .{ .{ 1, 2, 3, 4 }, .{ 5, 6, 7, 8 }, .{ 9, 10, 11, 12 } }, .{ .{ 13, 14, 15, 16 }, .{ 17, 18, 19, 20 }, .{ 21, 22, 23, 24 } }, })); @@ -692,16 +713,23 @@ test nearest { .{ 21, 21, 22, 22, 23, 23, 24, 24 }, }, })); - try zml.testing.expectClose(std.testing.io, expected, result, 0); + try zml.testing.expectClose(std.testing.io, expected, result, .{}); } // 4D Tensor (basic) { const input_4d_basic: zml.Tensor = .init(.{ 1, 1, 2, 2 }, .i32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_4d_basic, .{ .scale_factor = &.{ 3, 3 }, .mode = .nearest } }, platform); + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + upsample, + .{ input_4d_basic, .{ .scale_factor = &.{ 3, 3 }, .mode = .nearest } }, + platform, + .{ .shardings = &.{zml.testing.replicatedSharding()} }, + ); defer exe.deinit(); - var input_4d_basic_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_basic.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); + var input_4d_basic_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_basic.shape(), zml.testing.replicatedSharding(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); defer input_4d_basic_buffer.deinit(); var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_4d_basic_buffer}); @@ -716,15 +744,22 @@ test nearest { .{ 3, 3, 3, 4, 4, 4 }, .{ 3, 3, 3, 4, 4, 4 }, }}})); - try zml.testing.expectClose(std.testing.io, expected, result, 0); + try zml.testing.expectClose(std.testing.io, expected, result, .{}); } // 4D Tensor (advanced) { const input_4d_advanced: zml.Tensor = .init(.{ 2, 2, 2, 2 }, .i32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_4d_advanced, .{ .scale_factor = &.{ 2, 2 }, .mode = .nearest } }, platform); + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + upsample, + .{ input_4d_advanced, .{ .scale_factor = &.{ 2, 2 }, .mode = .nearest } }, + platform, + .{ .shardings = &.{zml.testing.replicatedSharding()} }, + ); defer exe.deinit(); - var input_4d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_advanced.shape(), std.mem.sliceAsBytes(&[2][2][2][2]i32{ .{ + var input_4d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_advanced.shape(), zml.testing.replicatedSharding(), std.mem.sliceAsBytes(&[2][2][2][2]i32{ .{ .{ .{ 1, 2 }, .{ 3, 4 } }, .{ .{ 5, 6 }, .{ 7, 8 } }, }, .{ @@ -767,15 +802,22 @@ test nearest { }, }, })); - try zml.testing.expectClose(std.testing.io, expected, result, 0); + try zml.testing.expectClose(std.testing.io, expected, result, .{}); } // 5D Tensor (basic) { const input_5d: zml.Tensor = .init(.{ 1, 1, 1, 2, 2 }, .i32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_5d, .{ .scale_factor = &.{2}, .mode = .nearest } }, platform); + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + upsample, + .{ input_5d, .{ .scale_factor = &.{2}, .mode = .nearest } }, + platform, + .{ .shardings = &.{zml.testing.replicatedSharding()} }, + ); defer exe.deinit(); - var input_5d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_5d.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); + var input_5d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_5d.shape(), zml.testing.replicatedSharding(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); defer input_5d_buffer.deinit(); var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_5d_buffer}); @@ -800,7 +842,7 @@ test nearest { }, }, })); - try zml.testing.expectClose(std.testing.io, expected, result, 0); + try zml.testing.expectClose(std.testing.io, expected, result, .{}); } } @@ -832,8 +874,9 @@ pub fn resizeBilinear(image: Tensor, resized_axes: anytype, opt: ResizeOpts) Ten test resizeBilinear { const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); + var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{ .shardings = &.{replicated_sharding} }); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -898,8 +941,9 @@ pub fn resizeBicubic(image: Tensor, resized_axes: anytype, opt: ResizeOpts) Tens test resizeBicubic { const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); + var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{ .shardings = &.{replicated_sharding} }); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -1112,14 +1156,15 @@ pub fn sampleTokens(activations: Tensor, opts: SamplingStrategy, rng: Tensor.Rng test sampleTokens { const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); const rng: zml.Tensor.Rng = .init(); const activations: zml.Tensor = .init(.{ .voc = 4 }, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, sampleTokens, .{ activations, .{ .topk = 4, .temperature = 2.0 }, rng }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, sampleTokens, .{ activations, .{ .topk = 4, .temperature = 2.0 }, rng }, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); - var rng_buffer = try zml.Tensor.Rng.initBuffer(platform, 0xdeadbeef, std.testing.io); + var rng_buffer = try zml.Tensor.Rng.initBuffer(platform, 0xdeadbeef, std.testing.io, replicated_sharding); defer rng_buffer._state.deinit(); const inf = std.math.inf(f32); @@ -1131,7 +1176,7 @@ test sampleTokens { }) |logits_expected| { const logits, const expected: i32 = logits_expected; - var activations_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, activations.shape(), std.mem.sliceAsBytes(&logits)); + var activations_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, activations.shape(), replicated_sharding, std.mem.sliceAsBytes(&logits)); defer activations_buffer.deinit(); var sampled, rng_buffer = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, sampleTokens, .{ activations_buffer, rng_buffer }); @@ -1170,14 +1215,15 @@ pub const DynamicSamplingStrategy = struct { pub fn makeBuffers( io: std.Io, platform: *const zml.Platform, + sharding: zml.sharding.Sharding, dtype: zml.DataType, opts: Opts, ) !zml.Bufferized(DynamicSamplingStrategy) { return .{ - .top_k = try zml.Buffer.scalar(io, platform, opts.top_k, .i32), - .temperature = try zml.Buffer.scalar(io, platform, opts.temperature, dtype), - .top_p = try zml.Buffer.scalar(io, platform, opts.top_p, dtype), - .min_p = try zml.Buffer.scalar(io, platform, opts.min_p, dtype), + .top_k = try zml.Buffer.scalar(io, platform, opts.top_k, .i32, sharding), + .temperature = try zml.Buffer.scalar(io, platform, opts.temperature, dtype, sharding), + .top_p = try zml.Buffer.scalar(io, platform, opts.top_p, dtype, sharding), + .min_p = try zml.Buffer.scalar(io, platform, opts.min_p, dtype, sharding), }; } @@ -1248,6 +1294,7 @@ fn fixupLogits(logits: Tensor, opts: DynamicSamplingStrategy) [2]Tensor { test sampleTokensDynamic { const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); const ___ = -std.math.inf(f32); const logits_data = [_]f32{ @log(2.0), @log(1.0), @log(4.0), @log(3.0) }; @@ -1256,10 +1303,10 @@ test sampleTokensDynamic { const logits: zml.Tensor = .init(.{ .voc = logits_data.len }, .f32); const dynamic_sampling_strategy = DynamicSamplingStrategy.init(.f32, 0); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, fixupLogits, .{ logits, dynamic_sampling_strategy }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, fixupLogits, .{ logits, dynamic_sampling_strategy }, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); - var logits_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits.shape(), std.mem.sliceAsBytes(&logits_data)); + var logits_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits.shape(), replicated_sharding, std.mem.sliceAsBytes(&logits_data)); defer logits_buffer.deinit(); const Args = struct { DynamicSamplingStrategy.Opts, [4]f32 }; @@ -1278,7 +1325,7 @@ test sampleTokensDynamic { .{ .{ .top_k = 4, .top_p = 0.901, .min_p = 0.6 }, [_]f32{ @log(4.0), @log(3.0), ___, ___ } }, }) |args_expected| { const args, const expected = args_expected; - var dynamic_sampling_strategy_buffers = try DynamicSamplingStrategy.makeBuffers(std.testing.io, platform, .f32, args); + var dynamic_sampling_strategy_buffers = try DynamicSamplingStrategy.makeBuffers(std.testing.io, platform, replicated_sharding, .f32, args); defer DynamicSamplingStrategy.deinitBuffers(&dynamic_sampling_strategy_buffers); var new_logits, var indices = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, fixupLogits, .{ logits_buffer, dynamic_sampling_strategy_buffers }); defer new_logits.deinit(); @@ -1293,15 +1340,15 @@ test sampleTokensDynamic { const logits_bf16: zml.Tensor = .init(.{ .voc = logits_data.len }, .bf16); const dynamic_sampling_strategy_bf16 = DynamicSamplingStrategy.init(.bf16, 0); - var exe_bf16 = try zml.module.compile(std.testing.allocator, std.testing.io, fixupLogits, .{ logits_bf16, dynamic_sampling_strategy_bf16 }, platform); + var exe_bf16 = try zml.module.compile(std.testing.allocator, std.testing.io, fixupLogits, .{ logits_bf16, dynamic_sampling_strategy_bf16 }, platform, .{ .shardings = &.{replicated_sharding} }); defer exe_bf16.deinit(); const boost = bf16.inf; const nerf = bf16.minus_inf; - var logits_bf16_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits_bf16.shape(), std.mem.sliceAsBytes(&[4]bf16{ boost, boost, bf16.fromF32(2), nerf })); + var logits_bf16_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits_bf16.shape(), replicated_sharding, std.mem.sliceAsBytes(&[4]bf16{ boost, boost, bf16.fromF32(2), nerf })); defer logits_bf16_buffer.deinit(); - var dynamic_sampling_strategy_bf16_buffers = try DynamicSamplingStrategy.makeBuffers(std.testing.io, platform, .bf16, .{ .top_k = 4, .top_p = 0.9, .min_p = 0.1 }); + var dynamic_sampling_strategy_bf16_buffers = try DynamicSamplingStrategy.makeBuffers(std.testing.io, platform, replicated_sharding, .bf16, .{ .top_k = 4, .top_p = 0.9, .min_p = 0.1 }); defer DynamicSamplingStrategy.deinitBuffers(&dynamic_sampling_strategy_bf16_buffers); var new_logits, var indices = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_bf16, fixupLogits, .{ logits_bf16_buffer, dynamic_sampling_strategy_bf16_buffers }); diff --git a/zml/ops.zig b/zml/ops.zig index 81b5022a..a83157cc 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -366,12 +366,12 @@ test "triton" { const a: zml.Tensor = .init(.{}, .f32); const b: zml.Tensor = .init(.{}, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, TritonMod.forward, .{ a, b }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, TritonMod.forward, .{ a, b }, platform, .{ .shardings = &.{zml.testing.replicatedSharding()} }); defer exe.deinit(); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[1]f32{1})); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), zml.testing.replicatedSharding(), std.mem.sliceAsBytes(&[1]f32{1})); defer a_buffer.deinit(); - var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), std.mem.sliceAsBytes(&[1]f32{3})); + var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), zml.testing.replicatedSharding(), std.mem.sliceAsBytes(&[1]f32{3})); defer b_buffer.deinit(); var results = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, TritonMod.forward, .{ a_buffer, b_buffer }); @@ -631,7 +631,7 @@ test scatterConfig { const zml = @import("zml.zig"); const platform = zml.testing.env(); - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); + var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{ .shardings = &.{zml.testing.replicatedSharding()} }); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -840,12 +840,13 @@ pub fn _collectAxes(T: type, bounded_array: stdx.BoundedArray(T, constants.MAX_R return res; } -fn TensorOrTensorArray(comptime T: type) type { - const type_info = @typeInfo(T); +fn CustomCallResultTypeFromOutputSpec(comptime OutputSpecT: type) type { + const type_info = @typeInfo(OutputSpecT); return switch (type_info) { + .void => void, .@"struct" => |struct_info| b: { - if (T == Tensor) break :b Tensor; - if (!struct_info.is_tuple) @compileError("Expected tuple"); + if (OutputSpecT == Shape) break :b Tensor; + if (!struct_info.is_tuple) @compileError("Expected tuple output shape spec"); break :b if (struct_info.fields.len == 1) Tensor else @@ -858,10 +859,10 @@ fn TensorOrTensorArray(comptime T: type) type { [array_info.len]Tensor; }, .pointer => |pointer_info| b: { - if (pointer_info.size != .slice) @compileError("Expected slice"); + if (pointer_info.size != .slice) @compileError("Expected output shape slice"); break :b []Tensor; }, - else => @compileError("Unsupported type: " ++ @typeName(T)), + else => @compileError("Unsupported customCall output shape spec type: " ++ @typeName(OutputSpecT)), }; } @@ -870,7 +871,7 @@ pub const CustomCallOptions = struct { output_operand_aliases: ?[]const i64 = null, }; -pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, metadata: anytype, opts: CustomCallOptions) TensorOrTensorArray(@TypeOf(outputs)) { +pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, metadata: anytype, opts: CustomCallOptions) CustomCallResultTypeFromOutputSpec(@TypeOf(outputs)) { // Transform generic inputs to flat slice. const inputs_: []const Tensor = switch (@typeInfo(@TypeOf(inputs))) { .@"struct" => |struct_info| b: { @@ -896,6 +897,7 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, // Transform generic outputs to flat slice. const output_shapes: []const Shape = switch (@typeInfo(@TypeOf(outputs))) { + .void => &.{}, .@"struct" => |struct_info| b: { if (@TypeOf(outputs) == Shape) { break :b &[1]Shape{outputs}; @@ -918,6 +920,10 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, }; const outputs_flat = customCallInternal(target_name, inputs_, output_shapes, metadata, opts); + if (comptime CustomCallResultTypeFromOutputSpec(@TypeOf(outputs)) == void) { + stdx.debug.assert(outputs_flat.len == 0, "customCall expected zero outputs for void return, got {}", .{outputs_flat.len}); + return; + } // Transform flat slice to generic outputs. return switch (@typeInfo(@TypeOf(outputs))) { @@ -943,19 +949,357 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, }; } -fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs: []const Shape, metadata: anytype, opts: CustomCallOptions) []Tensor { - const ctx = CompilationContext.current(); - var arena = std.heap.ArenaAllocator.init(ctx.allocator); - defer arena.deinit(); +pub fn manualComputation( + inputs: anytype, + outputs: anytype, + body_context: anytype, + comptime body_fn: anytype, +) manualComputationReturnType(body_fn) { + const inputs_: []const Tensor = switch (@typeInfo(@TypeOf(inputs))) { + .@"struct" => |struct_info| b: { + if (@TypeOf(inputs) == Tensor) { + break :b &[1]Tensor{inputs}; + } + if (!struct_info.is_tuple) @compileError("Expected tuple inputs"); + var inputs_flat: [struct_info.fields.len]Tensor = undefined; + meta.collectBuf((struct { + pub fn func(t: Tensor) Tensor { + return t; + } + }).func, {}, &inputs, &inputs_flat); + break :b &inputs_flat; + }, + .array => &inputs, + .pointer => |pointer_info| b: { + if (pointer_info.size != .slice) @compileError("Expected input slice"); + break :b inputs; + }, + else => @compileError("Unsupported manualComputation input type: " ++ @typeName(@TypeOf(inputs))), + }; - const values = arena.allocator().alloc(*const mlir.Value, inputs.len) catch unreachable; - for (0..inputs.len) |i| { - values[i] = inputs[i].value(); + const output_shapes: []const Shape = switch (@typeInfo(@TypeOf(outputs))) { + .void => &.{}, + .@"struct" => |struct_info| b: { + if (@TypeOf(outputs) == Shape) { + break :b &[1]Shape{outputs}; + } + if (!struct_info.is_tuple) @compileError("Expected tuple output shapes"); + var output_shapes_flat: [struct_info.fields.len]Shape = undefined; + meta.collectBuf((struct { + pub fn func(t: Shape) Shape { + return t; + } + }).func, {}, &outputs, &output_shapes_flat); + break :b &output_shapes_flat; + }, + .array => &outputs, + .pointer => |pointer_info| b: { + if (pointer_info.size != .slice) @compileError("Expected output shape slice"); + break :b outputs; + }, + else => @compileError("Unsupported manualComputation output type: " ++ @typeName(@TypeOf(outputs))), + }; + + const ReturnT = manualComputationReturnType(body_fn); + return manualComputationSliceToReturn(ReturnT, manualComputationInternal(inputs_, output_shapes, body_context, body_fn)); +} + +fn manualComputationInternal( + inputs: []const Tensor, + outputs: []const Shape, + body_context: anytype, + comptime body_fn: anytype, +) []Tensor { + const BodyReturnT = manualComputationReturnType(body_fn); + const BodyInputsT = stdx.meta.FnParam(body_fn, 2); + const BodyOutputShapesT = stdx.meta.FnParam(body_fn, 3); + + const ctx = CompilationContext.current(); + const allocator = ctx.arena.allocator(); + + const input_shapes = allocator.alloc(Shape, inputs.len) catch unreachable; + const input_values = allocator.alloc(*const mlir.Value, inputs.len) catch unreachable; + + for (inputs, 0..) |input, i| { + input_shapes[i] = input.shape(); + input_values[i] = input.value(); } - const res_types = arena.allocator().alloc(*const mlir.Type, outputs.len) catch unreachable; - for (outputs, 0..) |output, i| { - res_types[i] = mlir.rankedTensorType(output.dims(), mlirx.Type.fromDType(ctx.mlir_ctx, output.dtype())); + return switch (ctx.partitioning.partitioner) { + .shardy => blk: { + const local_input_shapes = allocator.alloc(Shape, inputs.len) catch unreachable; + const local_output_shapes = allocator.alloc(Shape, outputs.len) catch unreachable; + + for (input_shapes, 0..) |input_shape, i| { + local_input_shapes[i] = ctx.partitioning.localShapeForShape(input_shape) catch unreachable; + } + for (outputs, 0..) |output_shape, i| { + local_output_shapes[i] = ctx.partitioning.localShapeForShape(output_shape) catch unreachable; + } + + const in_shardings_str = ctx.partitioning.sdyPerValueShardingAttr(allocator, input_shapes) catch unreachable; + const out_shardings_str = ctx.partitioning.sdyPerValueShardingAttr(allocator, outputs) catch unreachable; + const manual_axes_str = ctx.partitioning.sdyManualAxesAttr(allocator, input_shapes, outputs) catch unreachable; + const in_shardings_attr = mlir.Attribute.parse(ctx.mlir_ctx, in_shardings_str) catch unreachable; + const out_shardings_attr = mlir.Attribute.parse(ctx.mlir_ctx, out_shardings_str) catch unreachable; + const manual_axes_attr = mlir.Attribute.parse(ctx.mlir_ctx, manual_axes_str) catch unreachable; + + const block_types = allocator.alloc(*const mlir.Type, inputs.len) catch unreachable; + const block_locs = allocator.alloc(*const mlir.Location, inputs.len) catch unreachable; + for (local_input_shapes, 0..) |input_shape, i| { + block_types[i] = mlir.rankedTensorType(input_shape.dims(), mlirx.Type.fromDType(ctx.mlir_ctx, input_shape.dtype())); + block_locs[i] = mlir.Location.unknown(ctx.mlir_ctx); + } + + const parent_block = ctx.currentScope().block; + const manual_block = mlir.Block.init(block_types, block_locs); + errdefer manual_block.deinit(); + + ctx.pushBlock(manual_block); + defer ctx.popBlock(); + + const local_inputs = allocator.alloc(Tensor, inputs.len) catch unreachable; + for (0..inputs.len) |i| { + local_inputs[i] = Tensor._result(local_input_shapes[i], manual_block.argument(i)); + } + + ctx.manual_computation_depth += 1; + defer ctx.manual_computation_depth -= 1; + + const body_inputs = manualComputationInputsArg(BodyInputsT, local_inputs); + const body_output_shapes = manualComputationOutputShapesArg(BodyOutputShapesT, local_output_shapes); + const body_result = @call(.auto, body_fn, .{ body_context, allocator, body_inputs, body_output_shapes }); + const local_outputs = manualComputationBodyToSlice(BodyReturnT, allocator, body_result); + stdx.debug.assert(local_outputs.len == outputs.len, "manualComputation body returned {} values, expected {}", .{ local_outputs.len, outputs.len }); + + const local_output_values = allocator.alloc(*const mlir.Value, outputs.len) catch unreachable; + for (0..outputs.len) |i| { + stdx.debug.assert(local_outputs[i].shape().eql(local_output_shapes[i]), "manualComputation body returned shape {f}, expected {f}", .{ local_outputs[i].shape(), local_output_shapes[i] }); + local_output_values[i] = local_outputs[i].value(); + } + + _ = mlir.Operation.make(ctx.mlir_ctx, "sdy.return", .{ + .operands = .{ .flat = local_output_values }, + .verify = false, + .location = .unknown(ctx.mlir_ctx), + }).appendTo(manual_block); + + const global_result_types = allocator.alloc(*const mlir.Type, outputs.len) catch unreachable; + for (outputs, 0..) |output_shape, i| { + global_result_types[i] = mlir.rankedTensorType(output_shape.dims(), mlirx.Type.fromDType(ctx.mlir_ctx, output_shape.dtype())); + } + + const op = mlir.Operation.make(ctx.mlir_ctx, "sdy.manual_computation", .{ + .operands = .{ .flat = input_values }, + .results = .{ .flat = global_result_types }, + .blocks = &.{manual_block}, + .attributes = &.{ + .named(ctx.mlir_ctx, "in_shardings", in_shardings_attr), + .named(ctx.mlir_ctx, "out_shardings", out_shardings_attr), + .named(ctx.mlir_ctx, "manual_axes", manual_axes_attr), + }, + .verify = true, + .location = .unknown(ctx.mlir_ctx), + }).appendTo(parent_block); + + const outputs_ = allocator.alloc(Tensor, outputs.len) catch unreachable; + for (outputs, 0..) |output, i| { + outputs_[i] = Tensor._result(output, op.result(i)); + } + break :blk outputs_; + }, + .gspmd => blk: { + const manual_sharding = "{manual}"; + const output_shardings = allocator.alloc([]const u8, outputs.len) catch unreachable; + const local_input_shapes = allocator.alloc(Shape, inputs.len) catch unreachable; + const local_output_shapes = allocator.alloc(Shape, outputs.len) catch unreachable; + + for (input_shapes, 0..) |input_shape, i| { + local_input_shapes[i] = ctx.partitioning.localShapeForShape(input_shape) catch unreachable; + } + for (outputs, 0..) |output_shape, i| { + local_output_shapes[i] = ctx.partitioning.localShapeForShape(output_shape) catch unreachable; + output_shardings[i] = ctx.partitioning.tensorShardingAttr(allocator, output_shape, null) catch unreachable; + } + + const local_input_values = allocator.alloc(*const mlir.Value, inputs.len) catch unreachable; + for (0..inputs.len) |i| { + const local_type = mlir.rankedTensorType(local_input_shapes[i].dims(), mlirx.Type.fromDType(ctx.mlir_ctx, local_input_shapes[i].dtype())); + const full_to_shard = dialects.stablehlo.custom_call( + ctx.mlir_ctx, + &.{input_values[i]}, + &.{local_type}, + .{ + .call_target_name = "SPMDFullToShardShape", + .has_side_effect = false, + .backend_config = .{ .original = "" }, + .additional_attributes = &.{ + .named(ctx.mlir_ctx, "mhlo.sharding", mlir.stringAttribute(ctx.mlir_ctx, manual_sharding)), + }, + }, + .unknown(ctx.mlir_ctx), + ).appendTo(ctx.currentScope().block); + local_input_values[i] = full_to_shard.result(0); + } + + const local_inputs = allocator.alloc(Tensor, inputs.len) catch unreachable; + for (0..inputs.len) |i| { + local_inputs[i] = Tensor._result(local_input_shapes[i], local_input_values[i]); + } + + ctx.manual_computation_depth += 1; + defer ctx.manual_computation_depth -= 1; + const body_inputs = manualComputationInputsArg(BodyInputsT, local_inputs); + const body_output_shapes = manualComputationOutputShapesArg(BodyOutputShapesT, local_output_shapes); + const body_result = @call(.auto, body_fn, .{ body_context, allocator, body_inputs, body_output_shapes }); + const local_outputs = manualComputationBodyToSlice(BodyReturnT, allocator, body_result); + stdx.debug.assert(local_outputs.len == outputs.len, "manualComputation body returned {} values, expected {}", .{ local_outputs.len, outputs.len }); + for (0..outputs.len) |i| { + stdx.debug.assert(local_outputs[i].shape().eql(local_output_shapes[i]), "manualComputation body returned shape {f}, expected {f}", .{ local_outputs[i].shape(), local_output_shapes[i] }); + } + + if (outputs.len == 0) { + break :blk allocator.alloc(Tensor, 0) catch unreachable; + } + + const global_values = allocator.alloc(*const mlir.Value, outputs.len) catch unreachable; + const global_types = allocator.alloc(*const mlir.Type, outputs.len) catch unreachable; + for (outputs, 0..) |output_shape, i| { + global_types[i] = mlir.rankedTensorType(output_shape.dims(), mlirx.Type.fromDType(ctx.mlir_ctx, output_shape.dtype())); + const shard_to_full = dialects.stablehlo.custom_call( + ctx.mlir_ctx, + &.{local_outputs[i].value()}, + &.{global_types[i]}, + .{ + .call_target_name = "SPMDShardToFullShape", + .has_side_effect = false, + .backend_config = .{ .original = "" }, + .additional_attributes = &.{ + .named(ctx.mlir_ctx, "mhlo.sharding", mlir.stringAttribute(ctx.mlir_ctx, output_shardings[i])), + }, + }, + .unknown(ctx.mlir_ctx), + ).appendTo(ctx.currentScope().block); + global_values[i] = shard_to_full.result(0); + } + + const barrier = mlir.Operation.make(ctx.mlir_ctx, "stablehlo.optimization_barrier", .{ + .operands = .{ .flat = global_values }, + .results = .{ .flat = global_types }, + .location = .unknown(ctx.mlir_ctx), + }).appendTo(ctx.currentScope().block); + + const outputs_ = allocator.alloc(Tensor, outputs.len) catch unreachable; + for (outputs, 0..) |output_shape, i| { + outputs_[i] = Tensor._result(output_shape, barrier.result(i)); + } + break :blk outputs_; + }, + }; +} + +fn manualComputationReturnType(comptime body_fn: anytype) type { + const ReturnT = stdx.meta.FnResult(body_fn); + if (ReturnT == void or ReturnT == Tensor) return ReturnT; + + return switch (@typeInfo(ReturnT)) { + .pointer => |pointer_info| b: { + if (pointer_info.size != .slice or pointer_info.child != Tensor) { + @compileError("manualComputation body must return void, Tensor, []Tensor, or []const Tensor; got " ++ @typeName(ReturnT)); + } + break :b ReturnT; + }, + else => @compileError("manualComputation body must return void, Tensor, []Tensor, or []const Tensor; got " ++ @typeName(ReturnT)), + }; +} + +fn manualComputationBodyToSlice(comptime ReturnT: type, allocator: std.mem.Allocator, result: ReturnT) []const Tensor { + if (ReturnT == void) return &.{}; + if (ReturnT == Tensor) { + const out = allocator.alloc(Tensor, 1) catch unreachable; + out[0] = result; + return out; + } + + return result; +} + +fn manualComputationInputsArg(comptime InputsT: type, local_inputs: []const Tensor) InputsT { + if (InputsT == void) { + stdx.debug.assert(local_inputs.len == 0, "manualComputation body expects no inputs, got {}", .{local_inputs.len}); + return; + } + if (InputsT == Tensor) { + stdx.debug.assert(local_inputs.len == 1, "manualComputation body expects one input, got {}", .{local_inputs.len}); + return local_inputs[0]; + } + + return switch (@typeInfo(InputsT)) { + .pointer => |pointer_info| b: { + if (pointer_info.size != .slice or pointer_info.child != Tensor) { + @compileError("manualComputation body inputs must be void, Tensor, []Tensor, or []const Tensor; got " ++ @typeName(InputsT)); + } + break :b if (pointer_info.is_const) + local_inputs + else + @constCast(local_inputs); + }, + else => @compileError("manualComputation body inputs must be void, Tensor, []Tensor, or []const Tensor; got " ++ @typeName(InputsT)), + }; +} + +fn manualComputationOutputShapesArg(comptime OutputShapesT: type, local_output_shapes: []const Shape) OutputShapesT { + if (OutputShapesT == void) { + stdx.debug.assert(local_output_shapes.len == 0, "manualComputation body expects no output shapes, got {}", .{local_output_shapes.len}); + return; + } + if (OutputShapesT == Shape) { + stdx.debug.assert(local_output_shapes.len == 1, "manualComputation body expects one output shape, got {}", .{local_output_shapes.len}); + return local_output_shapes[0]; + } + + return switch (@typeInfo(OutputShapesT)) { + .pointer => |pointer_info| b: { + if (pointer_info.size != .slice or pointer_info.child != Shape) { + @compileError("manualComputation body output shapes must be void, Shape, []Shape, or []const Shape; got " ++ @typeName(OutputShapesT)); + } + break :b if (pointer_info.is_const) + local_output_shapes + else + @constCast(local_output_shapes); + }, + else => @compileError("manualComputation body output shapes must be void, Shape, []Shape, or []const Shape; got " ++ @typeName(OutputShapesT)), + }; +} + +fn manualComputationSliceToReturn(comptime ReturnT: type, outputs: []Tensor) ReturnT { + if (ReturnT == void) { + stdx.debug.assert(outputs.len == 0, "manualComputation expected no outputs, got {}", .{outputs.len}); + return; + } + if (ReturnT == Tensor) { + stdx.debug.assert(outputs.len == 1, "manualComputation expected one output, got {}", .{outputs.len}); + return outputs[0]; + } + return outputs; +} + +fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs: []const Shape, metadata: anytype, opts: CustomCallOptions) []Tensor { + const ctx = CompilationContext.current(); + const allocator = ctx.arena.allocator(); + + stdx.debug.assert(!opts.has_side_effect or ctx.manual_computation_depth > 0, "side-effect customCall '{s}' must be emitted inside manualComputation", .{target_name}); + + const values = allocator.alloc(*const mlir.Value, inputs.len) catch unreachable; + const input_shapes = allocator.alloc(Shape, inputs.len) catch unreachable; + for (inputs, 0..) |input, i| { + values[i] = input.value(); + input_shapes[i] = input.shape(); + } + + const result_types = allocator.alloc(*const mlir.Type, outputs.len) catch unreachable; + for (outputs, 0..) |shape, i| { + result_types[i] = mlir.rankedTensorType(shape.dims(), mlirx.Type.fromDType(ctx.mlir_ctx, shape.dtype())); } const metadata_type_info = @typeInfo(@TypeOf(metadata)); @@ -1003,34 +1347,39 @@ fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs .named(ctx.mlir_ctx, "pjrt_client", mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_client))))), })); - const operands_layouts = arena.allocator().alloc([]const usize, inputs.len) catch unreachable; - for (inputs, 0..) |input, i| { - operands_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(input.rank())).constSlice()) catch unreachable; + const operand_layouts = allocator.alloc([]const usize, input_shapes.len) catch unreachable; + for (input_shapes, 0..) |shape, i| { + operand_layouts[i] = allocator.dupe(usize, toUsize(constants.minorToMajor(shape.rank())).constSlice()) catch unreachable; } - const results_layouts = arena.allocator().alloc([]const usize, outputs.len) catch unreachable; - for (outputs, 0..) |output, i| { - results_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(output.rank())).constSlice()) catch unreachable; + const result_layouts = allocator.alloc([]const usize, outputs.len) catch unreachable; + for (outputs, 0..) |shape, i| { + result_layouts[i] = allocator.dupe(usize, toUsize(constants.minorToMajor(shape.rank())).constSlice()) catch unreachable; } const op = dialects.stablehlo.custom_call( ctx.mlir_ctx, values, - res_types, + result_types, .{ .call_target_name = target_name, .backend_config = .{ .typed_ffi = backend_config }, .has_side_effect = opts.has_side_effect, - .operand_layouts = operands_layouts, - .result_layouts = results_layouts, + .operand_layouts = operand_layouts, + .result_layouts = result_layouts, .output_operand_aliases = opts.output_operand_aliases orelse &.{}, + .additional_attributes = &.{}, }, .unknown(ctx.mlir_ctx), - ).appendTo(CompilationContext.current().currentScope().block); + ).appendTo(ctx.currentScope().block); - const outputs_ = ctx.arena.allocator().alloc(Tensor, outputs.len) catch unreachable; - for (outputs, 0..) |output, i| { - outputs_[i] = Tensor._result(output, op.result(i)); + if (ctx.manual_computation_depth > 0 and ctx.partitioning.partitioner == .gspmd) { + op.setAttributeByName("mhlo.sharding", mlir.stringAttribute(ctx.mlir_ctx, "{manual}")); + } + + const outputs_ = allocator.alloc(Tensor, outputs.len) catch unreachable; + for (outputs, 0..) |output_shape, i| { + outputs_[i] = Tensor._result(output_shape, op.result(i)); } return outputs_; @@ -1040,7 +1389,7 @@ test customCall { const zml = @import("zml.zig"); const platform = zml.testing.env(); - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); + var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{ .shardings = &.{zml.testing.replicatedSharding()} }); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -1049,7 +1398,8 @@ test customCall { comp.pushBlock(block); defer comp.popBlock(); - const input = Tensor.constant(zml.DataType.bf16.constant(0)).broad(Shape.init(.{128}, .bf16)); + const shape = zml.Shape.init(.{128}, .bf16).withPartitioning(.{ ._0 = .x }); + const input = Tensor.constant(zml.DataType.bf16.constant(0)).broad(shape); const output = customCall("my_custom_call", .{input}, .{zml.Shape.init(.{128}, .bf16)}, .{}, .{ .has_side_effect = false, .output_operand_aliases = &.{0}, diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 6cb4ca3b..36a4ef30 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -12,16 +12,6 @@ pub const Client = opaque { pub const CompileError = std.mem.Allocator.Error || error{InvalidMlirBytecodeVersion} || pjrt.ApiError; pub fn compile(client: *const pjrt.Client, api: *const pjrt.Api, allocator: std.mem.Allocator, io: std.Io, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*pjrt.LoadedExecutable { - var bytecode: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); - defer bytecode.deinit(); - module.operation().writeBytecode(.{ .desired_emit_version = 1 }, &bytecode.writer) catch |err| { - log.err("failed to write module bytecode: {}", .{err}); - return switch (err) { - error.WriteFailed => error.OutOfMemory, - else => |e| e, - }; - }; - const stablehlo_version = blk: { if (api.stablehloCurrentVersion()) |requested_version| { break :blk dialects.stablehlo.smallerVersion(requested_version, dialects.stablehlo.currentVersion()); @@ -29,9 +19,14 @@ pub const Client = opaque { break :blk dialects.stablehlo.minimumVersion(); }; + const cloned_op = module.operation().clone(); + var cloned_module = mlir.Module.fromOperation(cloned_op); + defer cloned_module.deinit(); + var serialized_buffer: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); defer serialized_buffer.deinit(); - dialects.stablehlo.serializePortableArtifact(bytecode.written(), stablehlo_version, &serialized_buffer.writer) catch |err| { + + dialects.stablehlo.serializePortableArtifact2(cloned_module, stablehlo_version, &serialized_buffer.writer) catch |err| { log.err("failed to serialize to portable artifact: {}", .{err}); return switch (err) { std.Io.Writer.Error.WriteFailed => error.OutOfMemory, diff --git a/zml/platform.zig b/zml/platform.zig index 8b46ae90..9305baa9 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -5,6 +5,7 @@ const pjrt = @import("pjrt"); const platforms = @import("platforms"); pub const Target = platforms.Platform; const stdx = @import("stdx"); +const upb = @import("upb"); const Exe = @import("exe.zig").Exe; const pjrtx = @import("pjrtx.zig"); @@ -12,15 +13,6 @@ const zml = @import("zml.zig"); const log = std.log.scoped(.zml); -pub const CompilationOptions = struct { - xla_dump_to: ?[]const u8 = null, - xla_dump_fusion_visualization: bool = false, - xla_dump_hlo_pass_re: ?[]const u8 = null, - sharding_enabled: bool = false, - sharding_axes: stdx.BoundedArray([*:0]const u8, 8) = .{}, - device_memory_size: u64 = 0, -}; - fn StaticPlatformMap(comptime E: type, comptime T: type) type { const tag_count = @typeInfo(E).@"enum".fields.len; var struct_field_names: [tag_count][]const u8 = undefined; @@ -101,7 +93,7 @@ pub const Memory = struct { }; return zml_kind == kind_; }, - .cpu,.neuron => return true, + .cpu, .neuron => return true, } } @@ -135,7 +127,7 @@ pub const Device = struct { allocator.free(self.addressable_memories); } - pub fn id(self: Device) i32 { + pub fn id(self: Device) usize { return self.pjrt_desc.id(self.platform.pjrt_api); } @@ -192,14 +184,9 @@ pub const Platform = struct { pjrt_client: *pjrt.Client, devices: []const Device, memories: []const Memory, + physical_mesh: zml.sharding.PhysicalMesh, - // This make the pjrt struct quite fat, but is only used during compilation. - // TODO: Reconsider having it here, and maybe pass explicitly to compile, - // or create an intermediary struct: - // `const comp = platform.compiler(compile_opts); const exe = comp.compile(...);` - compilation_options: CompilationOptions = .{}, - - pub const MAX_NUM_DEVICES: u8 = if (platforms.isEnabled(.tpu)) 32 else 8; + pub const MAX_NUM_DEVICES: u16 = if (platforms.isEnabled(.tpu)) 64 else 32; pub fn init(allocator: std.mem.Allocator, io: std.Io, target: Target, options: CreateOptions) !*Platform { const api = try loadOrGetApi(allocator, io, target); @@ -224,9 +211,9 @@ pub const Platform = struct { .target = target, .pjrt_api = api, .pjrt_client = pjrt_client, - .compilation_options = .{}, .devices = devices, .memories = memories, + .physical_mesh = undefined, }; defer platform.arena_state = arena.state; @@ -237,6 +224,11 @@ pub const Platform = struct { for (pjrt_memories, memories) |pjrt_memory, *platform_memory| { platform_memory.* = try .init(arena.allocator(), pjrt_memory, platform, devices); } + + platform.physical_mesh = try switch (options.physical_mesh) { + .auto => zml.sharding.PhysicalMesh.auto(arena.allocator(), target, devices), + .custom => |builder| builder(arena.allocator(), target, devices), + }; } switch (target) { @@ -251,17 +243,15 @@ pub const Platform = struct { else => {}, } - if (platform.pjrt_api.ffi()) |ffi| b: { - const platform_name = switch (target) { - .cuda => "cuda", - .rocm => "rocm", - .cpu => "host", - else => break :b, - }; - ffi.register(platform.pjrt_api, "zml$print", platform_name, printCallback, .{ .command_buffer_compatible = false }) catch |e| { - log.warn("Failed to register \"print\" callback, error: {}", .{e}); - }; - } + platform.registerFfi( + .{ + .name = "zml$print", + .handler = printCallback, + .traits = .{ .command_buffer_compatible = false }, + }, + ) catch |err| { + log.warn("Failed to register FFI custom call \"zml$print\", error: {}", .{err}); + }; return platform; } @@ -279,14 +269,14 @@ pub const Platform = struct { } else error.Unavailable; } - pub fn formatWithDevices(self: *const Platform, writer: *std.Io.Writer) std.Io.Writer.Error!void { + pub fn formatWithAttributes(self: *const Platform, writer: *std.Io.Writer) std.Io.Writer.Error!void { const tee = "├─ "; const line = "│ "; const langle = "└─ "; try writer.print("platform: {s}\n", .{@tagName(self.target)}); try writer.print("version: {f}\n", .{self.pjrt_api.version()}); - try writer.print("stablehlo_version: {?s}\n", .{self.pjrt_api.stablehloCurrentVersion()}); + try writer.print("extensions:\n", .{}); { var it = self.pjrt_api.extensions(); @@ -295,39 +285,106 @@ pub const Platform = struct { } } + try writer.print("plugin attributes:\n", .{}); + { + const attributes = self.pjrt_api.pluginAttributes(); + if (attributes.len == 0) { + try writer.print("{s}(none)\n", .{langle}); + } else { + for (attributes, 0..) |attr, i| { + const is_last_attr = i == attributes.len - 1; + try writer.print("{s}{s}", .{ if (is_last_attr) langle else tee, attr.name() }); + switch (attr.value()) { + .string => |v| try writer.print(": \"{s}\"", .{v}), + .int64 => |v| try writer.print(": {d}", .{v}), + .int64list => |v| { + for (v, 0..) |item, j| { + if (j == 0) { + try writer.print(": {d}", .{item}); + } else { + try writer.print(".{d}", .{item}); + } + } + }, + .float => |v| try writer.print(": {d}", .{v}), + .bool => |v| try writer.print(": {}", .{v}), + } + try writer.writeAll("\n"); + } + } + } + try writer.print("devices:\n", .{}); for (self.devices, 0..) |device, i| { - const is_last_device = i < self.devices.len - 1; - try writer.print("{s}{f}\n", .{ if (is_last_device) tee else langle, device }); - for (device.addressable_memories, 0..) |mem, j| { - try writer.print("{s}{s}memory: {s}\n", .{ - if (is_last_device) line else " ", - if (j < device.addressable_memories.len - 1) tee else langle, - mem.kind(), - }); + const is_last_device = i == self.devices.len - 1; + const child_indent = if (is_last_device) " " else line; + + try writer.print("{s}{f}\n", .{ if (is_last_device) langle else tee, device }); + + { + const device_attrs = device.pjrt_desc.attributes(self.pjrt_api); + if (device_attrs.len > 0) { + var last_name: ?[]const u8 = null; + var remaining: usize = device_attrs.len; + + while (remaining > 0) : (remaining -= 1) { + var next_index: ?usize = null; + for (device_attrs, 0..) |attr, j| { + const name = attr.name(); + if (last_name) |last| { + if (std.mem.order(u8, name, last) != .gt) continue; + } + if (next_index) |ni| { + if (std.mem.order(u8, name, device_attrs[ni].name()) == .lt) { + next_index = j; + } + } else { + next_index = j; + } + } + + if (next_index == null) break; + const attr = device_attrs[next_index.?]; + + try writer.print("{s}{s}{s}", .{ child_indent, tee, attr.name() }); + switch (attr.value()) { + .string => |v| try writer.print(": \"{s}\"", .{v}), + .int64 => |v| try writer.print(": {d}", .{v}), + .int64list => |v| try writer.print(": {any}", .{v}), + .float => |v| try writer.print(": {d}", .{v}), + .bool => |v| try writer.print(": {}", .{v}), + } + try writer.writeAll("\n"); + + last_name = attr.name(); + } + } + } + + try writer.print("{s}{s}memories:\n", .{ child_indent, langle }); + { + const memory_indent = " "; + if (device.addressable_memories.len == 0) { + try writer.print("{s}{s}{s}(none)\n", .{ child_indent, memory_indent, langle }); + } else { + for (device.addressable_memories, 0..) |mem, j| { + const is_last_mem = j == device.addressable_memories.len - 1; + try writer.print("{s}{s}{s}memory: {s}\n", .{ + child_indent, + memory_indent, + if (is_last_mem) langle else tee, + mem.pjrt_memory.debugString(self.pjrt_api), + }); + } + } } } } - pub fn fmtVerbose(self: *const Platform) std.fmt.Alt(*const Platform, formatWithDevices) { + pub fn fmtVerbose(self: *const Platform) std.fmt.Alt(*const Platform, formatWithAttributes) { return .{ .data = self }; } - pub const Sharding = struct { num_replicas: u8, num_partitions: u8 }; - - pub fn sharding(self: *const Platform) Sharding { - // replicas run the same function but with different inputs, - // while partitions contribute to one evaluation over a shared input. - // Inside an inference process, we generally don't want replicas, - // as it's best to fully isolate replicas on different processes. - // For now we hardcode num_replicas = 1. - const num_devices: u8 = @intCast(self.devices.len); - return if (self.compilation_options.sharding_enabled) - .{ .num_replicas = 1, .num_partitions = num_devices } - else - .{ .num_replicas = 1, .num_partitions = 1 }; - } - pub fn deinit(self: *Platform, allocator: std.mem.Allocator) void { self.pjrt_client.deinit(self.pjrt_api); self.arena_state.promote(allocator).deinit(); @@ -342,12 +399,14 @@ pub const Platform = struct { args: stdx.meta.Tail( std.meta.ArgsTuple(@TypeOf(@field(@TypeOf(model_), @tagName(func)))), ), + opts: zml.module.CompilationOptions, ) !Exe { return self.compileFn( allocator, io, @field(@TypeOf(model_), @tagName(func)), .{model_} ++ args, + opts, ); } @@ -358,8 +417,9 @@ pub const Platform = struct { comptime func: anytype, model: stdx.meta.Head(std.meta.ArgsTuple(@TypeOf(func))), args: stdx.meta.Tail(std.meta.ArgsTuple(@TypeOf(func))), + opts: zml.module.CompilationOptions, ) !Exe { - return self.compileFn(allocator, io, func, .{model} ++ args); + return self.compileFn(allocator, io, func, .{model} ++ args, opts); } pub fn compileFn( @@ -368,8 +428,9 @@ pub const Platform = struct { io: std.Io, comptime func: anytype, args: std.meta.ArgsTuple(@TypeOf(func)), + opts: zml.module.CompilationOptions, ) !Exe { - return zml.module.compile(allocator, io, func, args, self); + return zml.module.compile(allocator, io, func, args, self, opts); } pub fn format(self: *const Platform, writer: *std.Io.Writer) std.Io.Writer.Error!void { @@ -389,9 +450,57 @@ pub const Platform = struct { } unreachable; } + + pub const FfiRegistration = struct { + name: []const u8, + handler: *const pjrt.ffi.Handler, + traits: pjrt.ffi.HandlerTraits = .{ .command_buffer_compatible = false }, + platform_name: ?[]const u8 = null, + }; + + pub fn registerFfi(self: *const zml.Platform, registration: FfiRegistration) !void { + const platform_name = registration.platform_name orelse self.pjrt_client.platformName(self.pjrt_api); + if (self.pjrt_api.ffi()) |ffi| { + try ffi.register(self.pjrt_api, registration.name, platform_name, registration.handler, registration.traits); + } else { + log.warn("PJRT FFI extension not available for {s}", .{@tagName(self.target)}); + } + } + + pub fn profiler(self: *const Platform, allocator: std.mem.Allocator) !?pjrt.Profiler { + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + var upb_alloc: upb.Allocator = .init(arena.allocator()); + const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); + defer c.upb_Arena_Free(upb_arena); + + const options = try upb.new(c.tensorflow_ProfileOptions, upb_arena); + + c.tensorflow_ProfileOptions_set_device_type(options, 0); + c.tensorflow_ProfileOptions_set_host_tracer_level(options, 3); + c.tensorflow_ProfileOptions_set_device_tracer_level(options, 3); + c.tensorflow_ProfileOptions_set_enable_hlo_proto(options, true); + + const serialized_options = try upb.serialize(options, upb_arena); + + return try self.pjrt_api.profiler(serialized_options); + } }; pub const CreateOptions = struct { + pub const CreatePhysicalMeshFn = *const fn ( + allocator: std.mem.Allocator, + target: Target, + devices: []const Device, + ) anyerror!zml.sharding.PhysicalMesh; + + pub const PhysicalMesh = union(enum) { + auto, + custom: CreatePhysicalMeshFn, + }; + + physical_mesh: PhysicalMesh = .auto, cpu: Cpu = .{ .device_count = 4 }, // bump memory fraction from XLA defaults of 75% to 90%. @@ -571,7 +680,7 @@ fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { const slice: zml.Slice = .init(shape, host_visible_data[0..shape.byteSize()]); const name = call_frame.attrs.getByName(.string, "name").?.slice(); - log.info("{s}: {d}", .{ name, slice }); + log.info("{s} {f} [device={d}]: {d}", .{ name, slice.shape, device_ordinal, slice }); return null; } diff --git a/zml/sharding.zig b/zml/sharding.zig new file mode 100644 index 00000000..af4ad8db --- /dev/null +++ b/zml/sharding.zig @@ -0,0 +1,2184 @@ +const std = @import("std"); + +const pjrt = @import("pjrt"); +const stdx = @import("stdx"); + +const Memory = @import("platform.zig").Memory; +const Platform = @import("platform.zig").Platform; +const PlatformDevice = @import("platform.zig").Device; +const Shape = @import("shape.zig").Shape; +const Slice = @import("slice.zig").Slice; +const Target = @import("platform.zig").Target; + +const log = std.log.scoped(.@"zml/sharding"); + +pub const Partitioning = struct { + pub const Partitioner = union(enum) { + shardy, + gspmd, + + pub fn fromTarget(target: Target) Partitioner { + return switch (target) { + .cpu, .cuda, .rocm, .tpu => .shardy, + .neuron => .gspmd, + }; + } + }; + + partitioner: Partitioner, + shardings: []const Sharding, + + pub fn init(partitioner: Partitioner, shardings: []const Sharding) !Partitioning { + stdx.debug.assert(shardings.len >= 1, "Waiting at leat 1 sharding strategy to be implemented", .{}); + var plan: Partitioning = .{ .partitioner = partitioner, .shardings = shardings }; + + const first = plan.primarySharding(); + const partitions = first.numPartitions(); + const replicas = first.numReplicas(); + + for (shardings[1..]) |s| { + if (s.numPartitions() != partitions or s.numReplicas() != replicas) { + // todo: deviceAssignments should also be checked for consistency here, but for simplicity we just check the cardinality numbers + return error.InconsistentShardingCardinality; + } + } + + return plan; + } + + pub fn numPartitions(self: Partitioning) i32 { + const primary = self.primarySharding(); + return primary.numPartitions(); + } + + pub fn numReplicas(self: Partitioning) i32 { + const primary = self.primarySharding(); + return primary.numReplicas(); + } + + pub fn numDevices(self: Partitioning) i32 { + return self.numPartitions() * self.numReplicas(); + } + + pub fn deviceAssignment(self: Partitioning, allocator: std.mem.Allocator) ![]usize { + return switch (self.partitioner) { + .shardy, .gspmd => blk: { + const sharding = self.primarySharding(); + break :blk sharding.deviceAssignment(allocator); + }, + }; + } + + pub fn tensorShardingAttr(self: Partitioning, allocator: std.mem.Allocator, shape: Shape, sharding: ?Sharding) ![]const u8 { + const selected_sharding = sharding orelse try self.selectSharding(shape); + return switch (self.partitioner) { + .shardy => try selected_sharding.sdyShardingAttrForShape(allocator, shape), + .gspmd => try selected_sharding.gspmdShardingAttrForShape(allocator, shape), + }; + } + + pub fn localShapeForShape(self: Partitioning, shape: Shape) !Shape { + const sharding = try self.selectSharding(shape); + const placement = try Placement.init(sharding, shape); + return placement.shards.get(0).shape; + } + + pub fn numPartitionsForLogicalAxis(self: Partitioning, shape: Shape, logical_axis: anytype) !i64 { + const sharding = try self.selectSharding(shape); + return sharding.numPartitionsForLogicalAxis(logical_axis); + } + + pub fn sdyPerValueShardingAttr(self: Partitioning, allocator: std.mem.Allocator, shapes: []const Shape) ![]const u8 { + stdx.debug.assert(self.partitioner == .shardy, "sdyPerValueShardingAttr requires shardy partitioner", .{}); + + var out: std.Io.Writer.Allocating = .init(allocator); + out.writer.writeAll("#sdy.sharding_per_value<[") catch unreachable; + for (shapes, 0..) |shape, i| { + const attr_str = try self.tensorShardingAttr(allocator, shape, null); + if (i > 0) out.writer.writeAll(", ") catch unreachable; + out.writer.writeAll(attr_str["#sdy.sharding".len..]) catch unreachable; + } + out.writer.writeAll("]>") catch unreachable; + return out.toOwnedSlice() catch unreachable; + } + + pub fn sdyManualAxesAttr(self: Partitioning, allocator: std.mem.Allocator, in_shapes: []const Shape, out_shapes: []const Shape) ![]const u8 { + stdx.debug.assert(self.partitioner == .shardy, "sdyManualAxesAttr requires shardy partitioner", .{}); + + var axis_names = std.ArrayList([]const u8).empty; + defer axis_names.deinit(allocator); + + const Collect = struct { + fn appendUnique(list: *std.ArrayList([]const u8), allocator_: std.mem.Allocator, axis_name: []const u8) void { + for (list.items) |existing| { + if (std.mem.eql(u8, existing, axis_name)) return; + } + list.append(allocator_, axis_name) catch unreachable; + } + + fn scan(list: *std.ArrayList([]const u8), allocator_: std.mem.Allocator, sharding: []const u8) void { + var i: usize = 0; + while (i < sharding.len) : (i += 1) { + if (sharding[i] != '"') continue; + const start = i + 1; + i = start; + while (i < sharding.len and sharding[i] != '"') : (i += 1) {} + if (i > start) appendUnique(list, allocator_, sharding[start..i]); + } + } + }; + + for (in_shapes) |shape| { + const attr_str = try self.tensorShardingAttr(allocator, shape, null); + Collect.scan(&axis_names, allocator, attr_str); + } + for (out_shapes) |shape| { + const attr_str = try self.tensorShardingAttr(allocator, shape, null); + Collect.scan(&axis_names, allocator, attr_str); + } + + var out: std.Io.Writer.Allocating = .init(allocator); + out.writer.writeAll("#sdy 0) out.writer.writeAll(", ") catch unreachable; + out.writer.print("\"{s}\"", .{axis_name}) catch unreachable; + } + out.writer.writeAll("}>") catch unreachable; + return out.toOwnedSlice() catch unreachable; + } + + pub fn selectSharding(self: Partitioning, shape: Shape) !Sharding { + return pickSharding(self.shardings, shape, .any_covering) orelse error.NoSuitableSharding; + } + + fn primarySharding(self: Partitioning) Sharding { + return self.shardings[0]; + } +}; + +pub const SelectShardingMode = enum { + any_covering, + explicit_axis_binding, +}; + +pub fn pickSharding(shardings: []const Sharding, shape: Shape, mode: SelectShardingMode) ?Sharding { + if (mode == .explicit_axis_binding and !shapeHasAxisPartition(shape)) return null; + + for (shardings) |sharding| { + if (shardingCoversShape(sharding, shape)) return sharding; + } + return null; +} + +fn shapeHasAxisPartition(shape: Shape) bool { + for (0..shape.rank()) |ax| { + if (shape.partition(ax) == .axis) return true; + } + return false; +} + +fn shardingCoversShape(sharding: Sharding, shape: Shape) bool { + for (0..shape.rank()) |ax| { + switch (shape.partition(ax)) { + .axis => |tag| if (sharding.binding(tag) == null) return false, + else => {}, + } + } + return true; +} + +/// Device is the leaf representation in a PhysicalMesh. +/// It carries identity, optional coordinates, and PJRT handle. +pub const Device = struct { + /// Unique device identifier in the mesh + id: usize, + + /// Coordinates in the physical mesh + coords: stdx.BoundedArray(usize, Shape.MAX_RANK) = .{}, + + /// Placeholder + pjrt_device: ?*const pjrt.Device = null, + + pub fn coordsSlice(self: *const Device) ?[]const usize { + if (self.coords.len == 0) return null; + return self.coords.constSlice(); + } + + pub fn format(self: Device, writer: *std.Io.Writer) !void { + try writer.print( + "Device(id={d} coords={any})", + .{ self.id, self.coords.constSlice() }, + ); + } +}; + +pub const Devices = stdx.BoundedArray(Device, Platform.MAX_NUM_DEVICES); + +/// Physical axis tags represent hardware interconnect tiers. +pub const PhysicalAxisTag = enum { + link, // 1D Interconnect (Inf1, Inf2 Islands) + link_x, // Torus/Mesh Dim X (TPU, Trainium) + link_y, // Torus/Mesh Dim Y (TPU, Trainium) + link_z, // Torus/Mesh Dim Z (TPU v3/v4/v5) + bus, // PCIe / NUMA (Potential isolation boundary) +}; + +/// Ring geometry style for 1D neighbor-only interconnects. +pub const RingKind = enum { + linear, // Neighbors only, open ends + closed_ring, // Neighbors only, circular (wrap-around) +}; + +/// Mesh geometry style for 2D/3D neighbor-only interconnects. +pub const MeshKind = enum { + grid, // 2D/3D Neighbors only, open ends + torus, // 2D/3D Neighbors only, circular wrap-around +}; + +/// AxisGeometry describes the communication pattern for an axis. +pub const AxisGeometry = union(enum) { + point_to_point: void, // Full all-to-all (NVLink Switch, TPU v5p) + ring: RingKind, // Neighbor-only 1D + mesh: MeshKind, // Neighbor-only 2D/3D + tree: void, // Hierarchical (Standard PCIe) + isolated: void, // No direct communication (AWS Inf2 islands) + + pub fn format(self: AxisGeometry, writer: *std.Io.Writer) !void { + switch (self) { + .point_to_point => try writer.writeAll("P2P"), + .ring => |k| try writer.print("Ring({s})", .{@tagName(k)}), + .mesh => |k| try writer.print("Mesh({s})", .{@tagName(k)}), + .tree => try writer.writeAll("Tree"), + .isolated => try writer.writeAll("Isolated"), + } + } +}; + +/// Represents the hierarchical physical topology of the hardware devices. +pub const PhysicalNode = union(enum) { + /// A branch represents a physical axis + branch: struct { + tag: PhysicalAxisTag, + geometry: AxisGeometry, + children: []PhysicalNode, + }, + /// A leaf represents the actual hardware device + leaf: Device, + + pub fn axis(tag: PhysicalAxisTag, geometry_: AxisGeometry, children: []const PhysicalNode) PhysicalNode { + return .{ + .branch = .{ + .tag = tag, + .geometry = geometry_, + .children = @constCast(children), + }, + }; + } + + pub fn device(device_: PlatformDevice) PhysicalNode { + return .{ + .leaf = .{ + .id = @intCast(device_.id()), + .coords = .{}, + .pjrt_device = device_.pjrt_device, + }, + }; + } + + pub fn countDevices(self: PhysicalNode) usize { + return switch (self) { + .leaf => 1, + .branch => |b| { + var sum: usize = 0; + for (b.children) |child| sum += child.countDevices(); + return sum; + }, + }; + } +}; + +/// PhysicalMesh models hardware as a tree of axes. +/// Each branch = one physical axis (link_x/link_y/bus/etc), +/// each leaf = one device. +/// +/// Diagram: +/// link_x +/// ├─ link_y +/// │ └─ device +/// └─ link_y +/// └─ device +/// +/// The DFS axis order defines canonical coordinate ordering. +pub const PhysicalMesh = struct { + /// Explicit topology tree API (canonical form) + pub const Tree = PhysicalNode; + + /// AxisTraversal captures canonical axis order and depth mapping. + pub const AxisTraversal = struct { + pub const Order = stdx.BoundedArray(PhysicalAxisTag, Shape.MAX_RANK); + pub const DepthByTag = std.EnumArray(PhysicalAxisTag, ?u8); + + /// Order of axes (DFS, first-child descent). + order: Order, + /// Depth (index) of each axis tag in the order. + depth_by_tag: DepthByTag, + + /// Build axis order and depth mapping from a topology tree. + pub fn init(root: PhysicalNode) !AxisTraversal { + var order: Order = try .init(0); + axisOrderNode(root, &order); + + var depth_by_tag: DepthByTag = .initFill(null); + for (order.constSlice(), 0..) |t, i| { + depth_by_tag.set(t, @intCast(i)); + } + + return .{ + .order = order, + .depth_by_tag = depth_by_tag, + }; + } + + pub fn depth(self: AxisTraversal, tag: PhysicalAxisTag) ?u8 { + return self.depth_by_tag.get(tag); + } + + pub fn format(self: AxisTraversal, writer: *std.Io.Writer) !void { + try writer.writeAll("AxisTraversal(order=["); + + for (self.order.constSlice(), 0..) |tag, i| { + if (i > 0) try writer.writeAll(", "); + try writer.writeAll(@tagName(tag)); + } + + try writer.writeAll("], depth_by_tag={"); + + const fields = std.meta.fields(PhysicalAxisTag); + inline for (fields, 0..) |field, i| { + if (i > 0) try writer.writeAll(", "); + const tag = @field(PhysicalAxisTag, field.name); + try writer.writeAll(field.name); + try writer.writeAll("="); + if (self.depth_by_tag.get(tag)) |d| { + try writer.print("{d}", .{d}); + } else { + try writer.writeAll("null"); + } + } + + try writer.writeAll("})"); + } + + /// Canonical DFS axis order: + /// + /// Tree: + /// link_x + /// └─ link_y + /// └─ leaf + /// + /// Order = [link_x, link_y] + /// + /// We descend only the first child to keep a stable axis order that + /// depends on structure (axes), not on device count. + fn axisOrderNode(node: PhysicalNode, out: *Order) void { + switch (node) { + .leaf => {}, + .branch => |b| { + out.appendAssumeCapacity(b.tag); + if (b.children.len > 0) axisOrderNode(b.children[0], out); + }, + } + } + }; + + /// AxisInfo exposes the size and geometry for a tag. + pub const AxisInfo = struct { + tag: PhysicalAxisTag, + size: i64, + geometry: AxisGeometry, + }; + + target: Target, + root: PhysicalNode, + axis_traversal: AxisTraversal, + + /// Build a PhysicalMesh from a tree: + /// - clone the tree into `allocator` + /// - validate geometry invariants + /// - assign coordinates to leaves + /// - compute axis traversal + pub fn fromTree(allocator: std.mem.Allocator, target: Target, root: PhysicalNode) !PhysicalMesh { + const cloned = try cloneNode(allocator, root); + errdefer freeNode(allocator, cloned); + + return try fromOwnedTree(target, cloned); + } + + fn fromOwnedTree(target: Target, root: PhysicalNode) !PhysicalMesh { + var owned_root = root; + + try validateGeometry(owned_root); + + var path = [_]usize{0} ** Shape.MAX_RANK; + try assignCoords(&owned_root, &path, 0); + + return .{ + .target = target, + .root = owned_root, + .axis_traversal = try .init(owned_root), + }; + } + + fn cloneNode(allocator: std.mem.Allocator, node: PhysicalNode) !PhysicalNode { + return switch (node) { + .leaf => |d| .{ + .leaf = .{ + .id = d.id, + .coords = d.coords, + .pjrt_device = d.pjrt_device, + }, + }, + .branch => |b| blk: { + const children = try allocator.alloc(PhysicalNode, b.children.len); + errdefer allocator.free(children); + for (b.children, 0..) |child, i| { + children[i] = try cloneNode(allocator, child); + } + break :blk .{ + .branch = .{ + .tag = b.tag, + .geometry = b.geometry, + .children = children, + }, + }; + }, + }; + } + + fn freeNode(allocator: std.mem.Allocator, node: PhysicalNode) void { + switch (node) { + .leaf => {}, + .branch => |b| { + for (b.children) |child| freeNode(allocator, child); + allocator.free(b.children); + }, + } + } + + fn validateGeometry(node: PhysicalNode) !void { + switch (node) { + .leaf => return, + .branch => |b| { + if (b.children.len == 0) return error.InvalidPhysicalMesh; + + switch (b.geometry) { + .mesh => { + var has_leaf = false; + var has_branch = false; + var expected_tag: ?PhysicalAxisTag = null; + + for (b.children) |child| { + switch (child) { + .leaf => has_leaf = true, + .branch => |cb| { + has_branch = true; + if (expected_tag == null) { + expected_tag = cb.tag; + } else if (cb.tag != expected_tag.?) { + return error.IncompatibleGeometry; + } + }, + } + } + + if (has_leaf and has_branch) return error.IncompatibleGeometry; + }, + .ring => { + if (b.children.len < 2) return error.IncompatibleGeometry; + }, + .point_to_point, .tree, .isolated => {}, + } + + for (b.children) |child| try validateGeometry(child); + }, + } + } + + /// Assign coords to each leaf using DFS path indices. + /// + /// Example (2x2): + /// link_x + /// └─ link_y + /// + /// coords: + /// (0,0) (0,1) + /// (1,0) (1,1) + fn assignCoords(node: *PhysicalNode, path: *[Shape.MAX_RANK]usize, depth: usize) !void { + switch (node.*) { + .leaf => |*d| { + // Coords can already been set by caller + if (d.coords.len > 0) return; + + var coords = stdx.BoundedArray(usize, Shape.MAX_RANK).init(0) catch unreachable; + for (path[0..depth]) |c| coords.appendAssumeCapacity(c); + d.coords = coords; + }, + .branch => |*b| { + for (b.children, 0..) |*child, i| { + path[depth] = i; + try assignCoords(child, path, depth + 1); + } + }, + } + } + + pub fn countDevices(self: PhysicalMesh) usize { + return self.root.countDevices(); + } + + pub fn devices(self: PhysicalMesh, allocator: std.mem.Allocator) ![]Device { + var list: Devices = .init(0) catch unreachable; + try self.devicesInto(&list); + + const out = try allocator.alloc(Device, list.len); + @memcpy(out, list.constSlice()); + return out; + } + + pub fn devicesInto(self: PhysicalMesh, out: *Devices) !void { + try devicesNodeInto(self.root, out); + } + + fn devicesNodeInto(node: PhysicalNode, out: *Devices) !void { + switch (node) { + .leaf => |d| out.appendAssumeCapacity(d), + .branch => |b| { + for (b.children) |child| { + try devicesNodeInto(child, out); + } + }, + } + } + + /// Canonical DFS linearization for coordinate consensus. + /// + /// sizes: S0..Sk, coords: C0..Ck + /// linear = (((C0 * S1) + C1) * S2 + C2) ... + pub fn linearIndexFromCoords(self: PhysicalMesh, coords: []const usize) usize { + const order = self.axisOrder(); + var idx: usize = 0; + for (order.constSlice()) |tag| { + const depth = self.axis_traversal.depth(tag); + if (depth) |d| { + const coord = coords[@intCast(d)]; + const size = self.axis(tag); + idx = idx * @as(usize, @intCast(size)) + coord; + } + } + return idx; + } + + pub fn axisInfo(self: PhysicalMesh, tag: PhysicalAxisTag) ?AxisInfo { + return axisInfoNode(self.root, tag); + } + + fn axisInfoNode(node: PhysicalNode, tag: PhysicalAxisTag) ?AxisInfo { + return switch (node) { + .leaf => null, + .branch => |b| { + if (b.tag == tag) { + return .{ + .tag = b.tag, + .size = @intCast(b.children.len), + .geometry = b.geometry, + }; + } + + for (b.children) |child| { + if (axisInfoNode(child, tag)) |info| return info; + } + + return null; + }, + }; + } + + pub fn hasAxis(self: PhysicalMesh, tag: PhysicalAxisTag) bool { + return self.axisInfo(tag) != null; + } + + pub fn axis(self: PhysicalMesh, tag: PhysicalAxisTag) i64 { + if (self.axisInfo(tag)) |info| return info.size; + + return 1; + } + + pub fn axisOrder(self: PhysicalMesh) AxisTraversal.Order { + return self.axis_traversal.order; + } + + pub fn geometry(self: PhysicalMesh, tag: PhysicalAxisTag) ?AxisGeometry { + if (self.axisInfo(tag)) |info| return info.geometry; + + return null; + } + + /// Preferred axis ordering by speed: fastest -> slowest. + /// This is used by Strategy.suggest to match intent to bandwidth. + pub fn shardableAxes(self: PhysicalMesh) []const PhysicalAxisTag { + return switch (self.target) { + .tpu => &.{ .link_x, .link_y, .link_z }, + .neuron => &.{ .link_x, .link_y, .link_z, .link }, + .cuda, .rocm => &.{.link}, + .cpu => &.{.bus}, + }; + } + + pub fn isShardable(self: PhysicalMesh, tag: PhysicalAxisTag) bool { + for (self.shardableAxes()) |t| { + if (t == tag) return true; + } + + return false; + } + + pub fn format(self: *const PhysicalMesh, writer: *std.Io.Writer) !void { + try writer.print("\nPhysicalMesh(platform={s} shardable_axes={any} num_devices={d})\n", .{ @tagName(self.target), self.shardableAxes(), self.countDevices() }); + try writer.print("├── {f}\n", .{self.axis_traversal}); + try writer.print("│ \n", .{}); + + try formatNode(self.root, writer, "", true); + } + + fn formatNode(node: PhysicalNode, writer: *std.Io.Writer, prefix: []const u8, is_last: bool) !void { + try writer.print("{s}{s}", .{ prefix, if (is_last) "└── " else "├── " }); + + switch (node) { + .leaf => |d| { + try writer.print("{f}\n", .{d}); + }, + .branch => |b| { + try writer.print("[{s}] x{d} ({s})\n", .{ @tagName(b.tag), b.children.len, @tagName(b.geometry) }); + + var buf: [256]u8 = undefined; + const new_prefix = std.fmt.bufPrint(&buf, "{s}{s}", .{ prefix, if (is_last) " " else "│ " }) catch unreachable; + + for (b.children, 0..) |child, i| { + const child_is_last = (i == b.children.len - 1); + try formatNode(child, writer, new_prefix, child_is_last); + } + }, + } + } + + /// CoordsTopology builds a mesh from PJRT `coords` attributes. + const CoordsTopology = struct { + const CoordPlacement = struct { + device_id: usize, + pjrt_device: ?*const pjrt.Device, + coords: stdx.BoundedArray(usize, Shape.MAX_RANK), + }; + + const CoordLayout = struct { + rank: usize, + axis_sizes: [Shape.MAX_RANK]usize, + }; + + const IndexedCoord = struct { + placement: CoordPlacement, + linear: usize, + }; + + fn parseDevice(device: PlatformDevice) !CoordPlacement { + const api = device.platform.pjrt_api; + const coords_attr = device.pjrt_desc.attribute(api, "coords") orelse return error.MissingDeviceCoords; + var coords: stdx.BoundedArray(usize, Shape.MAX_RANK) = try .init(0); + + switch (coords_attr) { + .int64list => |values| { + if (values.len == 0 or values.len > Shape.MAX_RANK) return error.InvalidDeviceCoords; + for (values) |value| { + if (value < 0) return error.InvalidDeviceCoords; + coords.appendAssumeCapacity(@intCast(value)); + } + }, + else => return error.InvalidDeviceCoords, + } + + return .{ + .device_id = device.id(), + .pjrt_device = device.pjrt_device, + .coords = coords, + }; + } + + pub fn collect(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) ![]CoordPlacement { + const placements = try allocator.alloc(CoordPlacement, platform_devices.len); + for (platform_devices, 0..) |device, i| { + placements[i] = try parseDevice(device); + } + return placements; + } + + pub fn layout(placements: []const CoordPlacement, max_rank: usize) !CoordLayout { + if (placements.len == 0) return error.InvalidPhysicalMesh; + + const rank = placements[0].coords.len; + if (rank == 0) return error.InvalidDeviceCoords; + if (rank > max_rank) return error.UnsupportedDeviceCoordsRank; + + for (placements[1..]) |placement| { + if (placement.coords.len != rank) return error.InvalidDeviceCoordsRank; + } + + var axis_sizes = [_]usize{1} ** Shape.MAX_RANK; + for (0..rank) |ax_i| { + var max_coord: usize = 0; + for (placements) |placement| { + max_coord = @max(max_coord, placement.coords.constSlice()[ax_i]); + } + axis_sizes[ax_i] = max_coord + 1; + } + + return .{ + .rank = rank, + .axis_sizes = axis_sizes, + }; + } + + fn linearIndex(coords: []const usize, rank: usize, axis_sizes: []const usize) usize { + var linear: usize = 0; + var stride: usize = 1; + var ax = rank; + while (ax > 0) { + ax -= 1; + linear += coords[ax] * stride; + stride *= axis_sizes[ax]; + } + return linear; + } + + pub fn sorted(allocator: std.mem.Allocator, placements: []const CoordPlacement, rank: usize, axis_sizes: []const usize) ![]IndexedCoord { + const indexed = try allocator.alloc(IndexedCoord, placements.len); + for (placements, indexed) |placement, *out| { + out.* = .{ + .placement = placement, + .linear = linearIndex(placement.coords.constSlice(), rank, axis_sizes), + }; + } + + const SortCtx = struct { + fn lessThan(_: @This(), a: IndexedCoord, b: IndexedCoord) bool { + return a.linear < b.linear; + } + }; + std.mem.sort(IndexedCoord, indexed, SortCtx{}, SortCtx.lessThan); + + for (indexed[1..], 1..) |entry, i| { + if (entry.linear == indexed[i - 1].linear) return error.InvalidDeviceTopology; + } + + return indexed; + } + + pub fn leaf(placement: *const CoordPlacement, coords_slice: []const usize) !PhysicalNode { + var coords: stdx.BoundedArray(usize, Shape.MAX_RANK) = try .init(0); + for (coords_slice) |coord| coords.appendAssumeCapacity(coord); + + return .{ + .leaf = .{ + .id = placement.device_id, + .coords = coords, + .pjrt_device = placement.pjrt_device, + }, + }; + } + + pub fn axisStrides(axis_sizes: []const usize) ![Shape.MAX_RANK]usize { + var strides = [_]usize{0} ** Shape.MAX_RANK; + var stride: usize = 1; + + var i = axis_sizes.len; + while (i > 0) { + i -= 1; + strides[i] = stride; + stride = std.math.mul(usize, stride, axis_sizes[i]) catch return error.InvalidDeviceTopology; + } + + return strides; + } + + pub fn buildMeshNode( + allocator: std.mem.Allocator, + indexed: []const IndexedCoord, + axis_sizes: []const usize, + axis_tags: []const PhysicalAxisTag, + axis_strides: []const usize, + depth: usize, + base_offset: usize, + ) !PhysicalNode { + if (depth == axis_sizes.len) { + const placement = &indexed[base_offset].placement; + return leaf(placement, placement.coords.constSlice()[0..axis_sizes.len]); + } + + const children = try allocator.alloc(PhysicalNode, axis_sizes[depth]); + var built: usize = 0; + errdefer { + for (children[0..built]) |child| freeNode(allocator, child); + allocator.free(children); + } + + for (children, 0..) |*child, i| { + child.* = try buildMeshNode( + allocator, + indexed, + axis_sizes, + axis_tags, + axis_strides, + depth + 1, + base_offset + i * axis_strides[depth], + ); + built += 1; + } + + return .{ + .branch = .{ + .tag = axis_tags[depth], + .geometry = .{ .mesh = .torus }, + .children = children, + }, + }; + } + }; + + pub fn auto(allocator: std.mem.Allocator, target: Target, platform_devices: []const PlatformDevice) !PhysicalMesh { + const root = try switch (target) { + .cpu => cpu(allocator, platform_devices), + .cuda, .rocm => gpu(allocator, platform_devices), + .tpu => tpu(allocator, platform_devices), + .neuron => neuron(allocator, platform_devices), + }; + errdefer freeNode(allocator, root); + + return try fromOwnedTree(target, root); + } + + pub fn cpu(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) !Tree { + const nodes = try allocator.alloc(PhysicalNode, platform_devices.len); + + for (nodes, platform_devices) |*n, d| n.* = .device(d); + + return .{ + .branch = .{ + .tag = .bus, + .geometry = .tree, + .children = nodes, + }, + }; + } + + pub fn gpu(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) !Tree { + const placements = try CoordsTopology.collect(allocator, platform_devices); + defer allocator.free(placements); + + const layout = try CoordsTopology.layout(placements, Shape.MAX_RANK); + const indexed = try CoordsTopology.sorted(allocator, placements, layout.rank, layout.axis_sizes[0..layout.rank]); + defer allocator.free(indexed); + + const nodes = try allocator.alloc(PhysicalNode, platform_devices.len); + for (nodes, indexed, 0..) |*n, entry, i| { + const linear_coord: [1]usize = .{i}; + n.* = try CoordsTopology.leaf(&entry.placement, linear_coord[0..]); + } + + return .{ + .branch = .{ + .tag = .link, + .geometry = .point_to_point, + .children = nodes, + }, + }; + } + + pub fn tpu(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) !Tree { + if (platform_devices.len == 0) return error.InvalidPhysicalMesh; + + const placements = try CoordsTopology.collect(allocator, platform_devices); + defer allocator.free(placements); + + const layout = try CoordsTopology.layout(placements, 3); + var rank = layout.rank; + var axis_sizes = layout.axis_sizes; + + while (rank > 1 and axis_sizes[rank - 1] == 1) : (rank -= 1) {} + const indexed = try CoordsTopology.sorted(allocator, placements, rank, axis_sizes[0..rank]); + defer allocator.free(indexed); + + var total_devices: usize = 1; + for (axis_sizes[0..rank]) |axis_size| { + total_devices = std.math.mul(usize, total_devices, axis_size) catch return error.InvalidDeviceTopology; + } + if (total_devices != indexed.len) return error.InvalidDeviceTopology; + for (indexed, 0..) |entry, i| { + if (entry.linear != i) return error.InvalidDeviceTopology; + } + + const axis_tags: [3]PhysicalAxisTag = .{ .link_x, .link_y, .link_z }; + const axis_strides = try CoordsTopology.axisStrides(axis_sizes[0..rank]); + + return CoordsTopology.buildMeshNode(allocator, indexed, axis_sizes[0..rank], axis_tags[0..rank], axis_strides[0..rank], 0, 0); + } + + pub fn neuron(_: std.mem.Allocator, _: []const PlatformDevice) !PhysicalNode { + stdx.debug.panic("neuron topology not implemented", .{}); + } +}; + +pub const LogicalAxisIntent = enum { + /// Needs the fastest possible path (e.g. Weight sharding / All-Reduce). + /// Requires high-bandwidth, low-latency interconnects (Cores/NVLink). + high_bandwidth, + + /// Needs reliable throughput but can handle some latency (e.g. Shuffling / All-to-All). + /// Suitable for chip-to-chip interconnects (NeuronLink/ICI). + balanced, + + /// Can tolerate lower bandwidth and higher latency (e.g. Batch sharding). + /// Suitable for system bus or network (PCIe/Ethernet). + low_bandwidth, +}; + +// Defines the logical mesh used for sharding the model and data. +// E.g., batch, model, context, experts, etc. +// Each axis represents a dimension along which the workload can be partitioned. The value is the intent of the axis. +/// LogicalMesh defines *semantic* axes only (batch/model/context/...). +/// It stores intent, not size. +/// +/// Why: +/// - Logical axes represent model meaning, not hardware. +/// - Sizes come from tensors later, during assignment. +/// +/// How it works: +/// - init() takes a struct of axis intents. +/// - each field name becomes a logical axis tag. +/// +/// Limitations: +/// - No sizes here; no divisibility checks here. +pub const LogicalMesh = struct { + pub const Axes = stdx.BoundedArray(Shape.Tag, Shape.MAX_RANK); + pub const Intents = stdx.BoundedArray(LogicalAxisIntent, Shape.MAX_RANK); + + name: []const u8, + axes: Axes, + intents: Intents, + + pub fn init(name: []const u8, axes_: anytype) !LogicalMesh { + const T = @TypeOf(axes_); + + var axes: Axes = try .init(0); + var intents: Intents = try .init(0); + + inline for (std.meta.fields(T)) |field| { + const value = @field(axes_, field.name); + axes.appendAssumeCapacity(Shape.toTag(field)); + intents.appendAssumeCapacity(intentFromValue(value)); + } + + if (axes.len == 0) { + stdx.debug.panic("LogicalMesh must have at least one axis defined", .{}); + } + + return .{ + .name = name, + .axes = axes, + .intents = intents, + }; + } + + pub fn intent(self: LogicalMesh, tag: Shape.Tag) ?LogicalAxisIntent { + const target = std.mem.span(tag); + for (self.axes.constSlice(), self.intents.constSlice()) |t, i| { + if (std.mem.eql(u8, std.mem.span(t), target)) return i; + } + + return null; + } + + fn intentFromValue(v: anytype) LogicalAxisIntent { + const V = @TypeOf(v); + if (V == LogicalAxisIntent) return v; + + if (V == @EnumLiteral()) { + return @field(LogicalAxisIntent, @tagName(v)); + } + + stdx.debug.compileError("LogicalMesh intent must be LogicalAxisIntent, got {}", .{V}); + } + + pub fn format(self: LogicalMesh, writer: *std.Io.Writer) !void { + try writer.print("LogicalMesh(name={s}", .{self.name}); + for (self.axes.constSlice(), self.intents.constSlice()) |axis, int| { + try writer.print(" {s}={s}", .{ axis, @tagName(int) }); + } + try writer.print(")", .{}); + } +}; + +pub const Sharding = struct { + pub const AxisList = stdx.BoundedArray(PhysicalAxisTag, Shape.MAX_RANK); + pub const Binding = struct { + logical: Shape.Tag, + physical: AxisList, + }; + pub const Bindings = stdx.BoundedArray(Binding, Shape.MAX_RANK); + + pub const Fold = struct { + target: PhysicalAxisTag, + sources: AxisList, + }; + pub const Folds = stdx.BoundedArray(Fold, Shape.MAX_RANK); + + pub const Axis = struct { + tag: PhysicalAxisTag, + size: i64, + geometry: ?AxisGeometry, + folded: stdx.BoundedArray(PhysicalAxisTag, Shape.MAX_RANK), + + pub fn contains(self: *const Axis, tag: PhysicalAxisTag) bool { + if (self.tag == tag) return true; + for (self.folded.constSlice()) |t| if (t == tag) return true; + return false; + } + }; + + pub const PhysicalView = struct { + axes: stdx.BoundedArray(Axis, Shape.MAX_RANK), + total_devices: i64, + + pub fn axisCoordFromLinearShard(self: *const PhysicalView, axis_index: usize, linear_idx: usize) usize { + const stride = self.axisStrideForLinear(axis_index); + const axis_size: usize = @intCast(self.axes.constSlice()[axis_index].size); + return (linear_idx / stride) % axis_size; + } + + pub fn axisStrideForLinear(self: *const PhysicalView, axis_index: usize) usize { + var stride: usize = 1; + var j = axis_index + 1; + while (j < self.axes.len) : (j += 1) { + stride *= @intCast(self.axes.constSlice()[j].size); + } + return stride; + } + }; + + platform: *const Platform, + logical: LogicalMesh, + + /// Compact binding table: logical axis -> physical axes + bindings: Bindings, + + /// Explicit folding rules: kept axis -> ordered source axes. + folds: Folds, + folds_consumed: std.EnumSet(PhysicalAxisTag), + + pub fn binding(self: *const Sharding, tag: Shape.Tag) ?[]const PhysicalAxisTag { + const target = std.mem.span(tag); + for (self.bindings.constSlice()) |*b| { + if (std.mem.eql(u8, std.mem.span(b.logical), target)) return b.physical.constSlice(); + } + return null; + } + + pub fn initFromStrategy( + platform: *const Platform, + logical: LogicalMesh, + strategy: Strategy, + ) !Sharding { + const physical = platform.physical_mesh; + const axis_order = physical.axisOrder().constSlice(); + if (axis_order.len == 0) return error.InvalidPhysicalMesh; + + var bindings: Bindings = try .init(0); + for (strategy.bindings.constSlice()) |bind| { + var list: AxisList = try .init(0); + for (bind.physical.constSlice()) |p_tag| { + list.appendAssumeCapacity(p_tag); + } + bindings.appendAssumeCapacity(.{ .logical = bind.logical, .physical = list }); + } + + var folds: Folds = try .init(0); + var folds_consumed = std.EnumSet(PhysicalAxisTag).initEmpty(); + for (strategy.folding.constSlice()) |entry| { + for (entry.sources.constSlice()) |src| { + if (!physical.hasAxis(src)) return error.InvalidPhysicalAxis; + folds_consumed.insert(src); + } + folds.appendAssumeCapacity(.{ .target = entry.target, .sources = entry.sources }); + } + + return .{ + .logical = logical, + .platform = platform, + .bindings = bindings, + .folds = folds, + .folds_consumed = folds_consumed, + }; + } + + fn foldSources(self: *const Sharding, tag: PhysicalAxisTag) ?[]const PhysicalAxisTag { + for (self.folds.constSlice()) |*f| { + if (f.target == tag) return f.sources.constSlice(); + } + return null; + } + + pub fn physicalView(self: Sharding) PhysicalView { + const physical = self.platform.physical_mesh; + var view: PhysicalView = .{ + .axes = stdx.BoundedArray(Axis, Shape.MAX_RANK).init(0) catch unreachable, + .total_devices = 1, + }; + + const axis_order = physical.axisOrder().constSlice(); + + for (axis_order) |tag| { + if (!physical.hasAxis(tag)) continue; + if (self.folds_consumed.contains(tag) and self.foldSources(tag) == null) continue; + + var folded = stdx.BoundedArray(PhysicalAxisTag, Shape.MAX_RANK).init(0) catch unreachable; + var size: i64 = 1; + + if (self.foldSources(tag)) |sources| { + for (sources) |src| { + folded.appendAssumeCapacity(src); + size *= physical.axis(src); + } + } else { + folded.appendAssumeCapacity(tag); + size = physical.axis(tag); + } + + view.total_devices *= size; + view.axes.appendAssumeCapacity(.{ + .tag = tag, + .size = size, + .geometry = physical.geometry(tag), + .folded = folded, + }); + } + + return view; + } + + pub fn name(self: Sharding) []const u8 { + return self.logical.name; + } + + pub fn logicalIndexFromCoords(self: Sharding, coords: []const usize) usize { + return self.platform.physical_mesh.linearIndexFromCoords(coords); + } + + pub fn numPartitions(self: Sharding) i32 { + return @intCast(self.physicalView().total_devices); + } + + pub fn numPartitionsForLogicalAxis(self: Sharding, logical_axis: anytype) i64 { + const logical_tag = Shape.toTag(logical_axis); + const bound_axes = self.binding(logical_tag) orelse return 1; + + var physical_axes = std.EnumSet(PhysicalAxisTag).initEmpty(); + for (bound_axes) |bound_axis| { + if (self.foldSources(bound_axis)) |sources| { + for (sources) |source| physical_axes.insert(source); + } else { + physical_axes.insert(bound_axis); + } + } + + var partitions: i64 = 1; + for (std.enums.values(PhysicalAxisTag)) |physical_axis| { + if (physical_axes.contains(physical_axis)) { + partitions *= self.platform.physical_mesh.axis(physical_axis); + } + } + return partitions; + } + + pub fn numReplicas(_: Sharding) i32 { + return 1; + } + + pub fn numDevices(self: Sharding) i32 { + return self.numPartitions() * self.numReplicas(); + } + + pub fn sdyMeshAttr(self: Sharding, allocator: std.mem.Allocator) ![]const u8 { + var out: std.Io.Writer.Allocating = .init(allocator); + errdefer out.deinit(); + + const view = self.physicalView(); + try out.writer.writeAll("#sdy.mesh<["); + for (view.axes.constSlice(), 0..) |p, i| { + if (i > 0) try out.writer.writeAll(", "); + try out.writer.print("\"{s}\"={d}", .{ @tagName(p.tag), p.size }); + } + try out.writer.writeAll("]>"); + return try out.toOwnedSlice(); + } + + const DimMapping = struct { + /// For each tensor dimension, the indices of physical axes (in PhysicalView) sharding it. + axes_per_dim: stdx.BoundedArray(stdx.BoundedArray(usize, Shape.MAX_RANK), Shape.MAX_RANK), + /// Indices of physical axes used for replication. + replicated_axes: stdx.BoundedArray(usize, Shape.MAX_RANK), + /// Reference to the view for sizes. + view: PhysicalView, + }; + + /// Common logic to map tensor dimensions to physical mesh indices. + fn getDimMapping(self: *const Sharding, shape: Shape) DimMapping { + const view = self.physicalView(); + var axes_per_dim = stdx.BoundedArray(stdx.BoundedArray(usize, Shape.MAX_RANK), Shape.MAX_RANK).init(0) catch unreachable; + var used_mask = [_]bool{false} ** Shape.MAX_RANK; + var globally_used = std.EnumSet(PhysicalAxisTag).initEmpty(); + + for (0..shape.rank()) |ax| { + var dim_axes = stdx.BoundedArray(usize, Shape.MAX_RANK).init(0) catch unreachable; + const spec = shape.partition(ax); + + if (spec == .axis) { + if (self.binding(spec.axis)) |binding_| { + for (binding_) |p_tag| { + for (view.axes.constSlice(), 0..) |v_ax, i| { + // Only use the axis if it's bound and hasn't been consumed by a previous dimension + if (v_ax.contains(p_tag) and !globally_used.contains(v_ax.tag)) { + dim_axes.appendAssumeCapacity(i); + globally_used.insert(v_ax.tag); + used_mask[i] = true; + } + } + } + } + } + axes_per_dim.appendAssumeCapacity(dim_axes); + } + + var replicated_axes = stdx.BoundedArray(usize, Shape.MAX_RANK).init(0) catch unreachable; + for (0..view.axes.len) |i| { + if (!used_mask[i]) replicated_axes.appendAssumeCapacity(i); + } + + return .{ + .axes_per_dim = axes_per_dim, + .replicated_axes = replicated_axes, + .view = view, + }; + } + + pub fn sdyShardingAttrForShape(self: Sharding, allocator: std.mem.Allocator, shape: Shape) ![]const u8 { + var any_explicit = false; + for (0..shape.rank()) |ax| { + if (shape.partition(ax) != .unknown) { + any_explicit = true; + break; + } + } + const all_replicated = !any_explicit; + + const mapping = self.getDimMapping(shape); + var out: std.Io.Writer.Allocating = .init(allocator); + errdefer out.deinit(); + + try out.writer.print("#sdy.sharding<@{s}, [", .{self.name()}); + for (0..shape.rank()) |ax| { + if (ax > 0) try out.writer.writeAll(", "); + const spec = shape.partition(ax); + + switch (spec) { + .axis => |logical_tag| { + if (self.binding(logical_tag)) |_| { + const dim_phys_indices = mapping.axes_per_dim.get(ax); + if (dim_phys_indices.len == 0) { + try out.writer.writeAll("{}"); + } else { + try out.writer.writeAll("{"); + for (dim_phys_indices.constSlice(), 0..) |p_idx, i| { + if (i > 0) try out.writer.writeAll(", "); + try out.writer.print("\"{s}\"", .{@tagName(mapping.view.axes.get(p_idx).tag)}); + } + try out.writer.writeAll("}"); + } + } else { + try out.writer.writeAll("{?}"); + } + }, + .replicated => try out.writer.writeAll("{}"), + .open, .unknown => { + if (all_replicated) { + try out.writer.writeAll("{}"); + } else { + try out.writer.writeAll("{?}"); + } + }, + } + } + try out.writer.writeAll("]"); + + if (mapping.replicated_axes.len > 0) { + try out.writer.writeAll(", replicated={"); + for (mapping.replicated_axes.constSlice(), 0..) |p_idx, i| { + if (i > 0) try out.writer.writeAll(", "); + try out.writer.print("\"{s}\"", .{@tagName(mapping.view.axes.get(p_idx).tag)}); + } + try out.writer.writeAll("}"); + } + try out.writer.writeAll(">"); + return try out.toOwnedSlice(); + } + + pub fn gspmdShardingAttrForShape(self: Sharding, allocator: std.mem.Allocator, shape: Shape) ![]const u8 { + var has_sharding = false; + for (0..shape.rank()) |ax| { + if (shape.partition(ax) == .axis) { + has_sharding = true; + break; + } + } + if (!has_sharding) return try allocator.dupe(u8, "{replicated}"); + + const mapping = self.getDimMapping(shape); + var tile_shape = stdx.BoundedArray(i64, Shape.MAX_RANK + 1).init(0) catch unreachable; + + // Calculate tile sizes per tensor dimension + for (mapping.axes_per_dim.constSlice()) |dim_axes| { + var combined: i64 = 1; + for (dim_axes.constSlice()) |idx| { + combined *= mapping.view.axes.get(idx).size; + } + tile_shape.appendAssumeCapacity(combined); + } + + // Add replication dimension if physical axes are left over. + const has_replication = mapping.replicated_axes.len > 0; + if (has_replication) { + var repl_size: i64 = 1; + for (mapping.replicated_axes.constSlice()) |idx| { + repl_size *= mapping.view.axes.get(idx).size; + } + tile_shape.appendAssumeCapacity(repl_size); + } + + var out: std.Io.Writer.Allocating = .init(allocator); + errdefer out.deinit(); + + // Emit explicit device assignment list to avoid XLA requiring + // a sharding attribute on the custom-call instruction. + const ids = try self.deviceAssignment(allocator); + defer allocator.free(ids); + + try out.writer.writeAll("{devices=["); + for (tile_shape.constSlice(), 0..) |s, i| { + if (i > 0) try out.writer.writeAll(","); + try out.writer.print("{d}", .{s}); + } + try out.writer.writeAll("]"); + + for (ids, 0..) |id, i| { + if (i == 0) { + try out.writer.print("{d}", .{id}); + } else { + try out.writer.print(",{d}", .{id}); + } + } + + if (has_replication) { + try out.writer.writeAll(" last_tile_dim_replicate"); + } + + try out.writer.writeAll("}"); + + return try out.toOwnedSlice(); + } + + pub fn deviceAssignment(self: Sharding, allocator: std.mem.Allocator) ![]usize { + const view = self.physicalView(); + const count: usize = @intCast(view.total_devices); + + var ids = try allocator.alloc(usize, count); + @memset(ids, std.math.maxInt(usize)); + + const ordered_devices = try self.devicesInCanonicalOrder(); + for (ordered_devices.constSlice()) |d| { + const coords = d.coordsSlice() orelse return error.MissingDeviceCoords; + const idx = self.platform.physical_mesh.linearIndexFromCoords(coords); + ids[idx] = d.id; + } + + for (ids) |id| { + if (id == std.math.maxInt(usize)) return error.MissingDeviceInTile; + } + + return ids; + } + + fn devicesInCanonicalOrder(self: Sharding) !Devices { + const physical = self.platform.physical_mesh; + var devices: Devices = try .init(0); + try physical.devicesInto(&devices); + + const Order = struct { + mesh: PhysicalMesh, + fn lessThan(ctx: @This(), a: Device, b: Device) bool { + const ca = a.coordsSlice() orelse unreachable; + const cb = b.coordsSlice() orelse unreachable; + const ia = ctx.mesh.linearIndexFromCoords(ca); + const ib = ctx.mesh.linearIndexFromCoords(cb); + return ia < ib; + } + }; + + std.mem.sort(Device, devices.slice(), Order{ .mesh = physical }, Order.lessThan); + return devices; + } + + pub fn format(self: Sharding, writer: *std.Io.Writer) !void { + try writer.print("Sharding(name={s})\n", .{self.logical.name}); + + try writer.writeAll("Bindings:\n"); + for (self.logical.axes.constSlice(), self.logical.intents.constSlice()) |l_tag, l_intent| { + try writer.print(" - {s} ({s}) -> ", .{ l_tag, @tagName(l_intent) }); + + if (self.binding(l_tag)) |axes| { + if (axes.len == 0) { + try writer.writeAll("replicated\n"); + } else { + for (axes, 0..) |p, i| { + if (i > 0) try writer.writeAll(", "); + try writer.writeAll(@tagName(p)); + } + try writer.writeAll("\n"); + } + } else { + try writer.writeAll("unbound\n"); + } + } + + const view = self.physicalView(); + + try writer.print("Physical capacity: {d}\n", .{view.total_devices}); + + try writer.writeAll("Physical axes: "); + if (view.axes.len == 0) { + try writer.writeAll("(none)\n"); + } else { + for (view.axes.constSlice(), 0..) |axis, i| { + if (i > 0) try writer.writeAll(" × "); + try writer.print("{s}[{d}]", .{ @tagName(axis.tag), axis.size }); + } + try writer.writeAll("\n"); + } + + try writer.writeAll("Geometries: "); + if (view.axes.len == 0) { + try writer.writeAll("(none)\n"); + } else { + for (view.axes.constSlice(), 0..) |axis, i| { + if (i > 0) try writer.writeAll(", "); + try writer.print("{s}={s}", .{ + @tagName(axis.tag), + @tagName(axis.geometry orelse .point_to_point), + }); + } + try writer.writeAll("\n"); + } + } +}; + +pub fn replicatedSharding(platform: *const Platform) !Sharding { + const physical_mesh = platform.physical_mesh; + const logical_mesh: LogicalMesh = try .init("replicated", .{ .x = .high_bandwidth }); + const strategy: Strategy = try .suggest(logical_mesh, physical_mesh); + return try .initFromStrategy(platform, logical_mesh, strategy); +} + +pub const Strategy = struct { + pub const PhysicalList = stdx.BoundedArray(PhysicalAxisTag, Shape.MAX_RANK); + pub const Binding = struct { + logical: Shape.Tag, + physical: PhysicalList, + }; + pub const Bindings = stdx.BoundedArray(Binding, Shape.MAX_RANK); + pub const Fold = struct { + target: PhysicalAxisTag, + sources: PhysicalList, + }; + pub const Folding = stdx.BoundedArray(Fold, Shape.MAX_RANK); + + bindings: Bindings, + folding: Folding, + + pub const init: Strategy = .{ + .bindings = .{}, + .folding = .{}, + }; + + pub fn addBinding(self: *Strategy, logical: anytype, physical: PhysicalAxisTag) !void { + const raw_tag = Shape.toTag(logical); + const logical_tag: []const u8 = std.mem.span(raw_tag); + + for (self.bindings.slice()) |*b| { + if (std.mem.eql(u8, std.mem.span(b.logical), logical_tag)) { + b.physical.appendAssumeCapacity(physical); + return; + } + } + + var list: PhysicalList = try .init(0); + list.appendAssumeCapacity(physical); + self.bindings.appendAssumeCapacity(.{ .logical = raw_tag, .physical = list }); + } + + /// Explicitly fold axes: `target` is the kept axis, `sources` define order. + /// If `target` is missing from `sources`, it is prepended. + pub fn addFold(self: *Strategy, target: PhysicalAxisTag, sources: []const PhysicalAxisTag) !void { + var list: PhysicalList = try .init(0); + + var has_target = false; + for (sources) |s| { + if (s == target) { + has_target = true; + break; + } + } + + if (!has_target) list.appendAssumeCapacity(target); + for (sources) |s| list.appendAssumeCapacity(s); + + for (self.folding.slice()) |*f| { + if (f.target == target) { + f.sources = list; + return; + } + } + + self.folding.appendAssumeCapacity(.{ .target = target, .sources = list }); + } + + /// suggest builds a Strategy from logical intents. + pub fn suggest(logical: LogicalMesh, physical: PhysicalMesh) !Strategy { + var strategy: Strategy = .init; + + const base_order = physical.shardableAxes(); + var available = stdx.BoundedArray(PhysicalAxisTag, Shape.MAX_RANK).init(0) catch unreachable; + for (base_order) |tag| { + if (physical.hasAxis(tag)) available.appendAssumeCapacity(tag); + } + + if (available.len == 0) { + for (physical.axisOrder().constSlice()) |tag| available.appendAssumeCapacity(tag); + } + + const avail = available.constSlice(); + var counters = std.EnumArray(LogicalAxisIntent, usize).initFill(0); + + inline for (std.meta.tags(LogicalAxisIntent)) |target_intent| { + for (logical.axes.constSlice(), logical.intents.constSlice()) |l_tag, l_intent| { + if (l_intent != target_intent) continue; + + const i = counters.get(target_intent); + const p_tag = switch (target_intent) { + .high_bandwidth => avail[i % avail.len], + .balanced => avail[(avail.len / 2 + i) % avail.len], + .low_bandwidth => avail[avail.len - 1 - (i % avail.len)], + }; + + try strategy.addBinding(l_tag, p_tag); + counters.set(target_intent, i + 1); + } + } + + return strategy; + } +}; + +pub const Placement = struct { + pub const Slice1d = struct { + axis: u8, + start: i64, + size: i64, + }; + + pub const Shard = struct { + device_id: usize, + device_coords: stdx.BoundedArray(usize, Shape.MAX_RANK), + shape: Shape, + global_shape: Shape, + slices: stdx.BoundedArray(Slice1d, Shape.MAX_RANK), + + pub fn platformDeviceIndex(self: *const Shard, platform: *const Platform) ?usize { + for (platform.devices, 0..) |d, device_index| { + if (d.id() == self.device_id) return device_index; + } + + return null; + } + + pub fn device(self: *const Shard, platform: *const Platform) PlatformDevice { + const device_index = self.platformDeviceIndex(platform) orelse unreachable; + return platform.devices[device_index]; + } + + pub fn memory(self: *const Shard, platform: *const Platform, kind: Memory.Kind) *const Memory { + return self.device(platform).memory(kind); + } + + pub fn shardSlice(self: *const Shard, slice: Slice) Slice { + stdx.debug.assert( + self.global_shape.eql(slice.shape), + "Placement global shape {f} doesn't match slice shape {f}", + .{ self.global_shape, slice.shape }, + ); + + var res = slice; + for (self.slices.constSlice()) |s| { + res = res.subSlice(s.axis, s.start, s.size); + } + return res; + } + + pub fn format(self: Shard, writer: *std.Io.Writer) !void { + try writer.print("device_id={d} coords={any}", .{ self.device_id, self.device_coords.constSlice() }); + + try writer.writeAll(" slices=["); + for (self.slices.constSlice(), 0..) |s, i| { + if (i > 0) try writer.writeAll(", "); + const axis_label = self.global_shape.debugTag(s.axis); + try writer.print("{s}:[{d}:{d}]", .{ axis_label, s.start, s.start + s.size }); + } + try writer.writeAll("]"); + } + }; + pub const Shards = stdx.BoundedArray(Shard, Platform.MAX_NUM_DEVICES); + + const AxisSplit = struct { + product: i64, + counts: stdx.BoundedArray(i64, Shape.MAX_RANK), + indices: stdx.BoundedArray(i64, Shape.MAX_RANK), + + pub fn init() !AxisSplit { + return .{ + .product = 1, + .counts = try .init(0), + .indices = try .init(0), + }; + } + + pub fn linearIndex(self: AxisSplit) i64 { + var linear: i64 = 0; + for (self.counts.constSlice(), self.indices.constSlice()) |c, iidx| { + linear = linear * c + iidx; + } + return linear; + } + }; + + sharding: Sharding, + shape: Shape, + shards: Shards, + + pub fn init( + sharding: Sharding, + shape: Shape, + ) !Placement { + const ordered_devices = try sharding.devicesInCanonicalOrder(); + + var shards: Shards = try .init(0); + + for (ordered_devices.constSlice()) |d| { + if (d.coords.len == 0) return error.MissingDeviceCoords; + + shards.appendAssumeCapacity(.{ + .device_id = d.id, + .device_coords = d.coords, + .shape = undefined, // Calculated below + .global_shape = shape, + .slices = try .init(0), + }); + } + + var self: Placement = .{ + .sharding = sharding, + .shape = shape, + .shards = shards, + }; + + for (self.shards.slice()) |*s| { + var used_axes = std.EnumSet(PhysicalAxisTag).initEmpty(); + + for (0..shape.rank()) |ax| { + const axis_index: u8 = @intCast(ax); + const slice = try self.sliceForDevice(s.device_coords.constSlice(), &used_axes, axis_index); + s.slices.appendAssumeCapacity(slice); + } + + s.shape = blk: { + var sh = shape; + for (s.slices.constSlice()) |slice| { + sh = sh.set(slice.axis, slice.size); + } + break :blk sh; + }; + } + + return self; + } + + fn sliceForDevice( + self: *Placement, + device_coords: []const usize, + used_axes: *std.EnumSet(PhysicalAxisTag), + axis_index: u8, + ) !Slice1d { + const dim = self.shape.dim(axis_index); + const spec = self.shape.partition(axis_index); + + switch (spec) { + .axis => |logical_tag| { + const binding = self.sharding.binding(logical_tag) orelse return error.MissingLogicalBinding; + + // Calculate the split based on the physical coordinates of the current device + const plan = try self.calculateSplit(dim, binding, device_coords, used_axes); + + const size = @divExact(dim, plan.product); + const start = plan.linearIndex() * size; + + return .{ + .axis = axis_index, + .start = start, + .size = size, + }; + }, + else => return .{ .axis = axis_index, .start = 0, .size = dim }, + } + } + + fn calculateSplit( + self: *Placement, + dim: i64, + binding: []const PhysicalAxisTag, + device_coords: []const usize, + used_axes: *std.EnumSet(PhysicalAxisTag), + ) !AxisSplit { + var plan: AxisSplit = try .init(); + + for (binding) |p_tag| { + // Expand the physical tag: check if it's a target for folded source axes + if (self.sharding.foldSources(p_tag)) |sources| { + for (sources) |src| { + if (used_axes.contains(src)) continue; + try self.addPhysicalToSplit(&plan, src, device_coords); + used_axes.insert(src); + } + } else { + // Standard case: directly bound, not part of an explicit fold + if (used_axes.contains(p_tag)) continue; + try self.addPhysicalToSplit(&plan, p_tag, device_coords); + used_axes.insert(p_tag); + } + } + + // Validate divisibility + if (plan.product > 0 and @rem(dim, plan.product) != 0) { + return error.IncompatibleSharding; + } + return plan; + } + + /// Extract coordinate data from the physical mesh for a specific axis. + fn addPhysicalToSplit(self: *Placement, plan: *AxisSplit, tag: PhysicalAxisTag, device_coords: []const usize) !void { + const physical = self.sharding.platform.physical_mesh; + const info = physical.axisInfo(tag) orelse return; + const depth = physical.axis_traversal.depth(tag) orelse return; + const coord = device_coords[depth]; + + plan.counts.appendAssumeCapacity(@intCast(info.size)); + plan.indices.appendAssumeCapacity(@intCast(coord)); + plan.product *= info.size; + } + + pub fn format(self: Placement, writer: *std.Io.Writer) !void { + try writer.print("Placement(shape={f} shards={d})\n", .{ self.shape, self.shards.len }); + + try writer.writeAll("Axis partitioning summary:\n"); + for (0..self.shape.rank()) |ax| { + const dim = self.shape.dim(ax); + const spec = self.shape.partition(ax); + const axis_label = self.shape.debugTag(ax); + + var shard_size: i64 = dim; + if (self.shards.len > 0) { + for (self.shards.constSlice()[0].slices.constSlice()) |s| { + if (s.axis == ax) { + shard_size = s.size; + break; + } + } + } + + const shards_count: i64 = if (shard_size > 0 and @rem(dim, shard_size) == 0) + @divExact(dim, shard_size) + else + 0; + + try writer.print( + "└─ {s}: dim={d}, spec={s}, shard_size={d}, shards={d}\n", + .{ axis_label, dim, @tagName(spec), shard_size, shards_count }, + ); + } + + try writer.writeAll("Per-shard placement:\n"); + for (self.shards.constSlice(), 0..) |shard, i| { + try writer.print("└─ Shard[{d}] {f}\n", .{ i, shard }); + } + } +}; + +const ShardingTest = struct { + pub const ExpectedShard = struct { + device_id: usize, + slices: []const [2]i64, + }; + + pub const Scenario = struct { + platform: Platform, + logical: LogicalMesh, + strategy: Strategy, + shape: Shape, + + expected_sdy: ?[]const u8 = null, + expected_shards: []const ExpectedShard = &.{}, + + expect_error: ?anyerror = null, + }; + + allocator: std.mem.Allocator, + + pub fn init(allocator: std.mem.Allocator) ShardingTest { + return .{ .allocator = allocator }; + } + + /// Creates a 1D-3D PhysicalMesh from simple dimensions. + pub fn physical(self: ShardingTest, dims: anytype, geometry: AxisGeometry) !PhysicalMesh { + const info = @typeInfo(@TypeOf(dims)).@"struct"; + var tags = try stdx.BoundedArray(PhysicalAxisTag, 3).init(0); + var sizes = try stdx.BoundedArray(usize, 3).init(0); + + const tag_names = [_]PhysicalAxisTag{ .link_x, .link_y, .link_z }; + inline for (info.fields, 0..) |field, i| { + tags.appendAssumeCapacity(tag_names[i]); + sizes.appendAssumeCapacity(@intCast(@field(dims, field.name))); + } + + var next_id: usize = 0; + const root = try self.buildNode(tags.constSlice(), sizes.constSlice(), 0, &next_id, geometry); + return try .fromTree(self.allocator, .tpu, root); + } + + fn buildNode(self: ShardingTest, tags: []const PhysicalAxisTag, sizes: []const usize, depth: usize, next_id: *usize, geometry: AxisGeometry) !PhysicalNode { + if (depth == tags.len) { + const id = next_id.*; + next_id.* += 1; + return .{ .leaf = .{ .id = id } }; + } + const count = sizes[depth]; + const children = try self.allocator.alloc(PhysicalNode, count); + for (children) |*child| { + child.* = try self.buildNode(tags, sizes, depth + 1, next_id, geometry); + } + return .{ .branch = .{ .tag = tags[depth], .geometry = geometry, .children = children } }; + } + + fn platformForMesh(self: ShardingTest, mesh: PhysicalMesh) Platform { + _ = self; + return .{ + .arena_state = .{}, + .target = mesh.target, + .pjrt_api = undefined, + .pjrt_client = undefined, + .devices = &.{}, + .memories = &.{}, + .physical_mesh = mesh, + }; + } + + pub fn run(self: ShardingTest, s: Scenario) !void { + const sharding = try Sharding.initFromStrategy(&s.platform, s.logical, s.strategy); + + // Verify MLIR String + if (s.expected_sdy) |expected_attr| { + const actual_attr = try sharding.sdyShardingAttrForShape(self.allocator, s.shape); + defer self.allocator.free(actual_attr); + try std.testing.expectEqualStrings(expected_attr, actual_attr); + } + + // Verify Placement logic / Error + if (s.expect_error) |err| { + try std.testing.expectError(err, Placement.init(sharding, s.shape)); + return; + } + + // Verify Shard Slices (Math) + if (s.expected_shards.len > 0) { + const placement = try Placement.init(sharding, s.shape); + try std.testing.expectEqual(s.expected_shards.len, placement.shards.len); + for (s.expected_shards, 0..) |expected, i| { + const actual = placement.shards.constSlice()[i]; + try std.testing.expectEqual(expected.device_id, actual.device_id); + for (expected.slices, 0..) |exp_slice, dim| { + const act_slice = actual.slices.constSlice()[dim]; + try std.testing.expectEqual(exp_slice[0], act_slice.start); + try std.testing.expectEqual(exp_slice[1], act_slice.size); + } + } + } + } +}; + +test "sharding: unknown partitioning implies replication" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + + const physical: PhysicalMesh = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); + const logical: LogicalMesh = try .init("folded_mesh", .{ .batch = .low_bandwidth }); + + var strategy: Strategy = .init; + try strategy.addBinding(.batch, .link_x); + + const scenario: ShardingTest.Scenario = .{ + .platform = runner.platformForMesh(physical), + .logical = logical, + .strategy = strategy, + .shape = Shape.init(.{ .batch = 8 }, .f32), + .expected_sdy = "#sdy.sharding<@folded_mesh, [{}], replicated={\"link_x\", \"link_y\"}>", + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{.{ 0, 8 }} }, + .{ .device_id = 1, .slices = &.{.{ 0, 8 }} }, + .{ .device_id = 2, .slices = &.{.{ 0, 8 }} }, + .{ .device_id = 3, .slices = &.{.{ 0, 8 }} }, + }, + }; + try runner.run(scenario); +} + +test "sharding: suggest strategy realization" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + + const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + + const logical: LogicalMesh = try .init("suggested_mesh", .{ + .batch = .low_bandwidth, + .model = .high_bandwidth, + }); + + const strategy: Strategy = try .suggest(logical, physical); + + const scenario: ShardingTest.Scenario = .{ + .platform = runner.platformForMesh(physical), + .logical = logical, + .strategy = strategy, + .shape = Shape.init(.{ .batch = 4, .model = 4 }, .f32) + .withPartitioning(.{ .batch = .batch, .model = .model }), + .expected_sdy = "#sdy.sharding<@suggested_mesh, [{\"link_z\"}, {\"link_x\"}], replicated={\"link_y\"}>", + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{ .{ 0, 2 }, .{ 0, 2 } } }, // x0, y0, z0 + .{ .device_id = 1, .slices = &.{ .{ 2, 2 }, .{ 0, 2 } } }, // x0, y0, z1 (batch split) + .{ .device_id = 2, .slices = &.{ .{ 0, 2 }, .{ 0, 2 } } }, // x0, y1, z0 (replicated y) + .{ .device_id = 3, .slices = &.{ .{ 2, 2 }, .{ 0, 2 } } }, // x0, y1, z1 + .{ .device_id = 4, .slices = &.{ .{ 0, 2 }, .{ 2, 2 } } }, // x1, y0, z0 (model split) + .{ .device_id = 5, .slices = &.{ .{ 2, 2 }, .{ 2, 2 } } }, + .{ .device_id = 6, .slices = &.{ .{ 0, 2 }, .{ 2, 2 } } }, + .{ .device_id = 7, .slices = &.{ .{ 2, 2 }, .{ 2, 2 } } }, + }, + }; + try runner.run(scenario); +} + +test "sharding: suggest folds logical axes with same intent" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + + const physical = try runner.physical(.{4}, .point_to_point); + + const logical: LogicalMesh = try .init("fold_mesh", .{ + .model = .high_bandwidth, + .experts = .high_bandwidth, + }); + + const strategy: Strategy = try .suggest(logical, physical); + + const scenario: ShardingTest.Scenario = .{ + .platform = runner.platformForMesh(physical), + .logical = logical, + .strategy = strategy, + .shape = Shape.init(.{ .model = 4, .experts = 4 }, .f32) + .withPartitioning(.{ .model = .model, .experts = .experts }), + .expected_sdy = "#sdy.sharding<@fold_mesh, [{\"link_x\"}, {}]>", + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{ .{ 0, 1 }, .{ 0, 4 } } }, + .{ .device_id = 1, .slices = &.{ .{ 1, 1 }, .{ 0, 4 } } }, + .{ .device_id = 2, .slices = &.{ .{ 2, 1 }, .{ 0, 4 } } }, + .{ .device_id = 3, .slices = &.{ .{ 3, 1 }, .{ 0, 4 } } }, + }, + }; + try runner.run(scenario); +} + +test "sharding: multiple physical axes on one logical dimension (folding)" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + + const physical: PhysicalMesh = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); + const logical: LogicalMesh = try .init("folded_mesh", .{ .model = .high_bandwidth }); + + var strategy: Strategy = .init; + try strategy.addBinding(.model, .link_x); + try strategy.addBinding(.model, .link_y); + + const scenario: ShardingTest.Scenario = .{ + .platform = runner.platformForMesh(physical), + .logical = logical, + .strategy = strategy, + .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(.{ .model = .model }), + .expected_sdy = "#sdy.sharding<@folded_mesh, [{\"link_x\", \"link_y\"}]>", + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, + .{ .device_id = 1, .slices = &.{.{ 4, 4 }} }, + .{ .device_id = 2, .slices = &.{.{ 8, 4 }} }, + .{ .device_id = 3, .slices = &.{.{ 12, 4 }} }, + }, + }; + try runner.run(scenario); +} + +test "sharding: explicit strategy folding" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + + const physical: PhysicalMesh = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + const logical: LogicalMesh = try .init("strategy_fold", .{ .model = .high_bandwidth }); + + var strategy: Strategy = .init; + try strategy.addBinding(.model, .link_x); + try strategy.addFold(.link_x, &.{ .link_x, .link_z }); + + const scenario: ShardingTest.Scenario = .{ + .platform = runner.platformForMesh(physical), + .logical = logical, + .strategy = strategy, + .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(.{ .model = .model }), + .expected_sdy = "#sdy.sharding<@strategy_fold, [{\"link_x\"}], replicated={\"link_y\"}>", + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, + .{ .device_id = 1, .slices = &.{.{ 4, 4 }} }, + .{ .device_id = 2, .slices = &.{.{ 0, 4 }} }, + .{ .device_id = 3, .slices = &.{.{ 4, 4 }} }, + .{ .device_id = 4, .slices = &.{.{ 8, 4 }} }, + .{ .device_id = 5, .slices = &.{.{ 12, 4 }} }, + .{ .device_id = 6, .slices = &.{.{ 8, 4 }} }, + .{ .device_id = 7, .slices = &.{.{ 12, 4 }} }, + }, + }; + try runner.run(scenario); +} + +test "sharding: open and replicated dimension mix" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + + const physical: PhysicalMesh = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); + const logical: LogicalMesh = try .init("mix_mesh", .{ .batch = .low_bandwidth, .model = .high_bandwidth }); + + var strategy: Strategy = .init; + try strategy.addBinding(.batch, .link_x); + try strategy.addBinding(.model, .link_y); + + const scenario: ShardingTest.Scenario = .{ + .platform = runner.platformForMesh(physical), + .logical = logical, + .strategy = strategy, + .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(.{ .batch = .open, .model = .replicated }), + .expected_sdy = "#sdy.sharding<@mix_mesh, [{?}, {}], replicated={\"link_x\", \"link_y\"}>", + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, + .{ .device_id = 1, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, + .{ .device_id = 2, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, + .{ .device_id = 3, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, + }, + }; + try runner.run(scenario); +} + +test "sharding: full 3D cluster sharding" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + + const physical: PhysicalMesh = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + const logical: LogicalMesh = try .init("3d_mesh", .{ .batch = .low_bandwidth, .model = .high_bandwidth, .context = .balanced }); + + var strategy: Strategy = .init; + try strategy.addBinding(.batch, .link_x); + try strategy.addBinding(.model, .link_y); + try strategy.addBinding(.context, .link_z); + + const scenario: ShardingTest.Scenario = .{ + .platform = runner.platformForMesh(physical), + .logical = logical, + .strategy = strategy, + .shape = Shape.init(.{ .batch = 4, .model = 4, .context = 4 }, .f32) + .withPartitioning(.{ .batch = .batch, .model = .model, .context = .context }), + .expected_sdy = "#sdy.sharding<@3d_mesh, [{\"link_x\"}, {\"link_y\"}, {\"link_z\"}]>", + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{ .{ 0, 2 }, .{ 0, 2 }, .{ 0, 2 } } }, + .{ .device_id = 1, .slices = &.{ .{ 0, 2 }, .{ 0, 2 }, .{ 2, 2 } } }, + .{ .device_id = 2, .slices = &.{ .{ 0, 2 }, .{ 2, 2 }, .{ 0, 2 } } }, + .{ .device_id = 3, .slices = &.{ .{ 0, 2 }, .{ 2, 2 }, .{ 2, 2 } } }, + .{ .device_id = 4, .slices = &.{ .{ 2, 2 }, .{ 0, 2 }, .{ 0, 2 } } }, + .{ .device_id = 5, .slices = &.{ .{ 2, 2 }, .{ 0, 2 }, .{ 2, 2 } } }, + .{ .device_id = 6, .slices = &.{ .{ 2, 2 }, .{ 2, 2 }, .{ 0, 2 } } }, + .{ .device_id = 7, .slices = &.{ .{ 2, 2 }, .{ 2, 2 }, .{ 2, 2 } } }, + }, + }; + try runner.run(scenario); +} + +test "sharding: num partitions for logical axis" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + + const physical: PhysicalMesh = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + const logical: LogicalMesh = try .init("axis_parts_mesh", .{ + .model = .high_bandwidth, + .batch = .low_bandwidth, + }); + + var strategy: Strategy = .init; + try strategy.addBinding(.model, .link_x); + try strategy.addFold(.link_x, &.{ .link_x, .link_z }); + + var platform = runner.platformForMesh(physical); + const sharding: Sharding = try .initFromStrategy(&platform, logical, strategy); + + try std.testing.expectEqual(4, sharding.numPartitionsForLogicalAxis(.model)); + try std.testing.expectEqual(1, sharding.numPartitionsForLogicalAxis(.batch)); +} diff --git a/zml/slice.zig b/zml/slice.zig index e33316db..be604b0e 100644 --- a/zml/slice.zig +++ b/zml/slice.zig @@ -2,6 +2,7 @@ const std = @import("std"); const stdx = @import("stdx"); +const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const floats = @import("floats.zig"); const Shape = @import("shape.zig").Shape; @@ -22,15 +23,18 @@ pub const Slice = struct { immutable: []const u8, }, shape: Shape, + offset_bytes: usize = 0, + byte_strides: stdx.BoundedArray(i64, constants.MAX_RANK), pub fn init(shape: Shape, bytes: anytype) Slice { stdx.debug.assertComptime(isBytes(@TypeOf(bytes)), "Expected \"bytes\" to be a []u8 or []const u8, got {s}", .{@typeName(@TypeOf(bytes))}); stdx.debug.assert(bytes.len == shape.byteSize(), "Expected \"bytes\" to have the same length as shape.byteSize() ({}), got {}", .{ shape.byteSize(), bytes.len }); const type_info = @typeInfo(@TypeOf(bytes)); + const byte_strides = shape.computeByteStrides(); return if (type_info.pointer.is_const) - .{ .inner_data = .{ .immutable = bytes }, .shape = shape } + .{ .inner_data = .{ .immutable = bytes }, .shape = shape, .offset_bytes = 0, .byte_strides = byte_strides } else - .{ .inner_data = .{ .mutable = bytes }, .shape = shape }; + .{ .inner_data = .{ .mutable = bytes }, .shape = shape, .offset_bytes = 0, .byte_strides = byte_strides }; } pub fn alloc(allocator: std.mem.Allocator, shape_: Shape) !Slice { @@ -46,11 +50,16 @@ pub const Slice = struct { else => |v| stdx.debug.panic("Unsupported alignment: {}", .{v}), }; - return .{ .inner_data = .{ .mutable = bytes }, .shape = shape_ }; + return .{ + .inner_data = .{ .mutable = bytes }, + .shape = shape_, + .offset_bytes = 0, + .byte_strides = shape_.computeByteStrides(), + }; } pub fn free(slice: Slice, allocator: std.mem.Allocator) void { - const d = slice.constData(); + const d = slice.constData_(); switch (slice.shape.dtype().alignOf()) { 1 => allocator.free(@as([]align(1) const u8, @alignCast(d))), @@ -68,7 +77,57 @@ pub const Slice = struct { return slice.shape.dtype(); } + pub fn isContiguous(slice: Slice) bool { + const expected = slice.shape.computeByteStrides(); + const rank = slice.shape.rank(); + for (0..rank) |ax| { + if (slice.byte_strides.get(ax) != expected.get(ax)) return false; + } + return true; + } + + /// Returns the actual element-wise strides for this specific slice. + /// This respects transpositions and non-contiguous views. + pub fn elementStrides(self: Slice) stdx.BoundedArray(i64, constants.MAX_RANK) { + const element_size = @as(i64, @intCast(self.dtype().sizeOf())); + var res = self.byte_strides; + for (res.slice()) |*stride| { + // We divide the actual byte strides by the size of the data type + stride.* = @divExact(stride.*, element_size); + } + return res; + } + + pub fn subSlice(slice: Slice, axis: anytype, start: i64, size: i64) Slice { + const dim = slice.shape.dim(axis); + + stdx.debug.assert(start >= 0 and size >= 0 and start + size <= dim, "subSlice out of bounds: axis {d} start {d} size {d} dim {d}", .{ axis, start, size, dim }); + + const stride_bytes = slice.byte_strides.get(axis); + const new_offset = @as(i64, @intCast(slice.offset_bytes)) + start * stride_bytes; + stdx.debug.assert(new_offset >= 0, "subSlice produced negative offset: {d}", .{new_offset}); + + var res = slice; + res.offset_bytes = @intCast(new_offset); + res.shape = slice.shape.set(axis, size); + return res; + } + + fn dropAxis(slice: Slice, axis_: anytype) Slice { + const axis = slice.shape.axis(axis_); + var res = slice; + res.shape = slice.shape.drop(axis); + _ = res.byte_strides.orderedRemove(@intCast(axis)); + return res; + } + pub fn data(slice: Slice) []u8 { + const base = slice.data_(); + stdx.debug.assert(slice.offset_bytes <= base.len, "Slice offset exceeds data length", .{}); + return base[slice.offset_bytes..]; + } + + fn data_(slice: Slice) []u8 { return switch (slice.inner_data) { .mutable => |d| d, else => stdx.debug.panic("Expected slice to be mutable but it's immutable", .{}), @@ -76,6 +135,12 @@ pub const Slice = struct { } pub fn constData(slice: Slice) []const u8 { + const base = slice.constData_(); + stdx.debug.assert(slice.offset_bytes <= base.len, "Slice offset exceeds data length", .{}); + return base[slice.offset_bytes..]; + } + + fn constData_(slice: Slice) []const u8 { return switch (slice.inner_data) { inline else => |d| d, }; @@ -128,14 +193,18 @@ pub const Slice = struct { switch (slice.dtype()) { inline else => |dt| { const T = dt.toZigType(); - // TODO: handle negative strides - const elem_strides: i64 = slice.shape.computeElementStrides().get(0); - const values = slice.constItems(T); + const n = slice.shape.dim(0); + + const stride = @divExact(slice.byte_strides.get(0), @as(i64, @intCast(@sizeOf(T)))); + + const needed_len: usize = if (n == 0) 0 else @intCast(@abs((n - 1) * stride) + 1); + const values = slice.constItems(T)[0..needed_len]; + switch (comptime dt.class()) { - .float => try stdx.fmt.formatFloatSlice(values, options, elem_strides, writer), - .integer => try stdx.fmt.formatIntSlice(values, options, elem_strides, writer), - .complex => try stdx.fmt.formatComplexSlice(values, options, elem_strides, writer), - .bool => try stdx.fmt.formatBoolSlice(values, options, elem_strides, writer), + .float => try stdx.fmt.formatFloatSlice(values, options, stride, writer), + .integer => try stdx.fmt.formatIntSlice(values, options, stride, writer), + .complex => try stdx.fmt.formatComplexSlice(values, options, stride, writer), + .bool => try stdx.fmt.formatBoolSlice(values, options, stride, writer), } }, } @@ -179,6 +248,34 @@ pub const Slice = struct { try writer.splatByteAll(' ', indent_level); _ = try writer.write("}"); } + + pub fn copy(destination: Slice, source: []const u8) void { + const total_bytes = destination.shape.byteSize(); + stdx.debug.assert(source.len >= total_bytes, "Source buffer size ({}) smaller than destination shape ({})", .{ source.len, total_bytes }); + + if (total_bytes == 0) return; + + if (destination.isContiguous()) { + @memcpy(destination.data()[0..total_bytes], source[0..total_bytes]); + return; + } + + const rank = destination.shape.rank(); + stdx.debug.assert(rank > 0, "Non-contiguous slice with rank 0 is unexpected", .{}); + + const outer_len: usize = @intCast(destination.shape.dim(0)); + stdx.debug.assert(outer_len > 0, "Invalid dimension size: {d}", .{outer_len}); + + stdx.debug.assert(total_bytes % outer_len == 0, "Non-even split for contiguous source", .{}); + const outer_span_bytes = total_bytes / outer_len; + + for (0..outer_len) |i| { + const destination_outer = destination.subSlice(0, @intCast(i), 1).dropAxis(0); + const source_outer_start = i * outer_span_bytes; + const source_outer_end = source_outer_start + outer_span_bytes; + destination_outer.copy(source[source_outer_start..source_outer_end]); + } + } }; test "slice pretty print rank 3" { diff --git a/zml/tensor.zig b/zml/tensor.zig index 1b2f8736..edbadf89 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -16,6 +16,7 @@ const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; +const Sharding = @import("sharding.zig").Sharding; pub const Tensor = struct { var current_id: std.atomic.Value(usize) = .{ .raw = 1 }; @@ -94,8 +95,8 @@ pub const Tensor = struct { for (0..n) |i| { sh._dims.appendAssumeCapacity(ranked_tensor.dimension(i)); } - sh._tags.resize(n) catch unreachable; - sh._partitioning.resize(n) catch unreachable; + sh._tags.appendNTimes(Shape.TagUnknown, n) catch unreachable; + sh._partitioning.appendNTimes(.unknown, n) catch unreachable; return .{ ._shape = sh, ._value = val, .id = Tensor.current_id.fetchAdd(1, .seq_cst) }; } @@ -142,11 +143,44 @@ pub const Tensor = struct { return res; } - // TODO(Corentin) - pub fn withSharding(self: Tensor, axes_: anytype) Tensor { - _ = self; - _ = axes_; - @panic("Unimplemented"); + pub fn withPartitioning(self: Tensor, axes_: anytype) Tensor { + const ctx = CompilationContext.current(); + const partitioned_shape = self._shape.withPartitioning(axes_); + + const attr = ctx.partitioning.tensorShardingAttr(ctx.allocator, partitioned_shape, null) catch unreachable; + defer ctx.allocator.free(attr); + + const op_result = switch (ctx.partitioning.partitioner) { + .shardy => blk: { + const op = mlir.Operation.make(ctx.mlir_ctx, "sdy.sharding_constraint", .{ + .operands = .{ .flat = &.{self.value()} }, + .results = .{ .flat = &.{self.value().type_()} }, + .attributes = &.{ + .named(ctx.mlir_ctx, "sharding", mlir.Attribute.parse(ctx.mlir_ctx, attr) catch unreachable), + }, + }).appendTo(currentBlock()); + break :blk op.result(0); + }, + .gspmd => blk: { + const op = dialects.stablehlo.custom_call( + ctx.mlir_ctx, + &.{self.value()}, + &.{self.value().type_()}, + .{ + .call_target_name = "Sharding", + .has_side_effect = false, + .backend_config = .{ .original = "" }, + .additional_attributes = &.{ + .named(ctx.mlir_ctx, "mhlo.sharding", mlir.stringAttribute(ctx.mlir_ctx, attr)), + }, + }, + .unknown(ctx.mlir_ctx), + ).appendTo(currentBlock()); + break :blk op.result(0); + }, + }; + + return _result(partitioned_shape, op_result); } pub fn toMemory(self: Tensor, kind: Memory) Tensor { @@ -313,6 +347,7 @@ pub const Tensor = struct { test fmod { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); const inputs: [2][6]f32 = .{ .{ -3.0, -2, -1, 1, 2, 3 }, .{ 1, 2, 3, 4, 5, -5 } }; const expectations: [2][6]f32 = .{ .{ -1.0, -0.0, -1.0, 1.0, 0.0, 1.0 }, .{ 1.0000, 0.5000, 0.0000, 1.0000, 0.5000, -0.5000 } }; @@ -321,15 +356,22 @@ pub const Tensor = struct { inline for (inputs, expectations, divisors) |i, e, d| { const input: zml.Tensor = .init(.{6}, .f32); - const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.fmod, .{ input, d }, platform); + const exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.fmod, + .{ input, d }, + platform, + .{ .shardings = &.{replicated_sharding} }, + ); defer exe.deinit(); - var input_buffer = try zml.Buffer.fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&i)); + var input_buffer = try zml.Buffer.fromBytes(std.testing.io, platform, input.shape(), replicated_sharding, std.mem.sliceAsBytes(&i)); defer input_buffer.deinit(); const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.fmod, .{input_buffer}); - try zml.testing.expectClose(std.testing.io, zml.Slice.init(zml.Shape.init(.{6}, .f32), std.mem.sliceAsBytes(&e)), output, 1e-4); + try zml.testing.expectClose(std.testing.io, zml.Slice.init(zml.Shape.init(.{6}, .f32), std.mem.sliceAsBytes(&e)), output, .{}); } } @@ -483,9 +525,9 @@ pub const Tensor = struct { return .{ ._state = .init(.{2}, .u64) }; } - pub fn initBuffer(platform: *const Platform, seed: u128, io: std.Io) !Bufferized(Rng) { + pub fn initBuffer(platform: *const Platform, seed: u128, io: std.Io, sharding: Sharding) !Bufferized(Rng) { return .{ - ._state = try .fromBytes(io, platform, Shape.init(.{2}, .u64), std.mem.asBytes(&seed)), + ._state = try .fromBytes(io, platform, Shape.init(.{2}, .u64), sharding, std.mem.asBytes(&seed)), }; } @@ -584,11 +626,12 @@ pub const Tensor = struct { }; const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); // Compute stats over a uniform distribution on [-2, 10]. - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.uniformStats, .{ Rng.init(), Shape.init(.{1024}, .f32), .{ .min = -2, .max = 10 } }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.uniformStats, .{ Rng.init(), Shape.init(.{1024}, .f32), .{ .min = -2, .max = 10 } }, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); - var rng_buffer = try Rng.initBuffer(platform, 1234, std.testing.io); + var rng_buffer = try Rng.initBuffer(platform, 1234, std.testing.io, replicated_sharding); defer rng_buffer._state.deinit(); var rand, var stats = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Stats.uniformStats, .{rng_buffer}); @@ -689,15 +732,17 @@ pub const Tensor = struct { }; const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); + const tgt_dist_data = [_]f32{ 2.0, 1.0, 4.0, 3.0 }; const tgt_dist: Tensor = .init(.{tgt_dist_data.len}, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.gumbelStats, .{ Rng.init(), tgt_dist }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.gumbelStats, .{ Rng.init(), tgt_dist }, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); - var rng_buffer = try Rng.initBuffer(platform, 1234, std.testing.io); + var rng_buffer = try Rng.initBuffer(platform, 1234, std.testing.io, replicated_sharding); defer rng_buffer._state.deinit(); - var tgt_dist_buffer: Buffer = try .fromBytes(std.testing.io, platform, tgt_dist.shape(), std.mem.sliceAsBytes(&tgt_dist_data)); + var tgt_dist_buffer: Buffer = try .fromBytes(std.testing.io, platform, tgt_dist.shape(), replicated_sharding, std.mem.sliceAsBytes(&tgt_dist_data)); defer tgt_dist_buffer.deinit(); var rand, var stats = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Stats.gumbelStats, .{ rng_buffer, tgt_dist_buffer }); @@ -1016,6 +1061,7 @@ pub const Tensor = struct { const floats = @import("floats.zig"); const zml = @import("zml.zig"); const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); // f4e2m1 { @@ -1024,10 +1070,17 @@ pub const Tensor = struct { for (&x_f4, &x) |*xi_f4, xi| xi_f4.* = .fromF32(xi); const x_d: Tensor = .init(.{x.len}, .f32); - const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.convert, .{ x_d, .f4e2m1 }, platform); + const exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.convert, + .{ x_d, .f4e2m1 }, + platform, + .{ .shardings = &.{replicated_sharding} }, + ); defer exe.deinit(); - var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), std.mem.sliceAsBytes(&x)); + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), replicated_sharding, std.mem.sliceAsBytes(&x)); defer x_d_buffer.deinit(); var x_f4_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); @@ -1045,10 +1098,17 @@ pub const Tensor = struct { for (&x_f8e3, &x) |*xi_f8e3, xi| xi_f8e3.* = .fromF32(xi); const x_d: Tensor = .init(.{x.len}, .f32); - const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.convert, .{ x_d, .f8e3m4 }, platform); + const exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.convert, + .{ x_d, .f8e3m4 }, + platform, + .{ .shardings = &.{replicated_sharding} }, + ); defer exe.deinit(); - var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), std.mem.sliceAsBytes(&x)); + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), replicated_sharding, std.mem.sliceAsBytes(&x)); defer x_d_buffer.deinit(); var x_f8e3_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); @@ -1097,6 +1157,7 @@ pub const Tensor = struct { test dot { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); inline for (.{ .{ .{ .c = 20 }, .{ .c = 20 }, .c, .{} }, @@ -1130,7 +1191,14 @@ pub const Tensor = struct { } }.forward; - const exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, y, ctr }, platform); + const exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + forward, + .{ x, y, ctr }, + platform, + .{ .shardings = &.{replicated_sharding} }, + ); defer exe.deinit(); try zml.testing.expectEqualShapes(Shape.init(z_shape, .f32), exe.output_shapes[0]); @@ -1239,19 +1307,27 @@ pub const Tensor = struct { test leakyReLU { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); const input: Tensor = .init(.{2}, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.leakyReLU, .{ input, 0.1 }, platform); + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.leakyReLU, + .{ input, 0.1 }, + platform, + .{ .shardings = &.{replicated_sharding} }, + ); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6884, 1.6795 })); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), replicated_sharding, std.mem.sliceAsBytes(&[_]f32{ -0.6884, 1.6795 })); defer input_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.leakyReLU, .{input_buffer}); defer res.deinit(); const expectation: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[2]f32{ -0.0688, 1.6795 })); - try zml.testing.expectClose(std.testing.io, expectation, res, 1e-4); + try zml.testing.expectClose(std.testing.io, expectation, res, .{}); } /// Returns a Tensor containing the SwiGLU activation function applied to the input Tensor. @@ -1384,6 +1460,7 @@ pub const Tensor = struct { test cumulativeSum { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); const Local = struct { pub fn _cumsum(input: Tensor) Tensor { @@ -1397,10 +1474,17 @@ pub const Tensor = struct { const x: Tensor = .init(.{ 2, 5 }, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cumsum, .{x}, platform); + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Local._cumsum, + .{x}, + platform, + .{ .shardings = &.{replicated_sharding} }, + ); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } })); defer x_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._cumsum, .{x_buffer}); @@ -1597,10 +1681,11 @@ pub const Tensor = struct { test slice { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); const x: Tensor = .init(.{ 2, 5 }, .f32); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); defer x_buffer.deinit(); // Wrap slice1d to hide the anytype in the signature. @@ -1616,7 +1701,14 @@ pub const Tensor = struct { .{ -1, Tensor.Slice{ .start = -2 }, [4]f32{ 3, 4, 8, 9 } }, }) |testcase| { const ax, const slice_, const expectation = testcase; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._slice1dAxis, .{ x, ax, slice_ }, platform); + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Local._slice1dAxis, + .{ x, ax, slice_ }, + platform, + .{ .shardings = &.{replicated_sharding} }, + ); defer exe.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._slice1dAxis, .{x_buffer}); @@ -1732,6 +1824,7 @@ pub const Tensor = struct { test repeat1d { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); const Local = struct { fn repeat1d(x: Tensor, axis_: u3, n_reps: u32) Tensor { @@ -1746,10 +1839,17 @@ pub const Tensor = struct { const input_data, const shape_, const expectation, const ax, const reps = testcase; const input: Tensor = .fromShape(shape_); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.repeat1d, .{ input, ax, reps }, platform); + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Local.repeat1d, + .{ input, ax, reps }, + platform, + .{ .shardings = &.{replicated_sharding} }, + ); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&input_data)); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), replicated_sharding, std.mem.sliceAsBytes(&input_data)); defer input_buffer.deinit(); const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local.repeat1d, .{input_buffer}); @@ -1786,6 +1886,7 @@ pub const Tensor = struct { test stutter1d { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); const Local = struct { fn stutter1d(x: Tensor, axis_: u3, n_reps: u32) Tensor { @@ -1801,10 +1902,17 @@ pub const Tensor = struct { const input_data, const shape_, const expectation, const ax, const reps = testcase; const input: Tensor = .fromShape(shape_); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.stutter1d, .{ input, ax, reps }, platform); + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Local.stutter1d, + .{ input, ax, reps }, + platform, + .{ .shardings = &.{replicated_sharding} }, + ); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&input_data)); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), replicated_sharding, std.mem.sliceAsBytes(&input_data)); defer input_buffer.deinit(); const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local.stutter1d, .{input_buffer}); @@ -1962,6 +2070,7 @@ pub const Tensor = struct { test scalar { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); const Local = struct { pub fn _fwd() [6]Tensor { @@ -1972,7 +2081,7 @@ pub const Tensor = struct { } }; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{}, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{}, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); } @@ -2227,6 +2336,7 @@ pub const Tensor = struct { test gather { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); const Local = struct { pub fn _idx(idx_shape: anytype) Tensor { @@ -2238,7 +2348,7 @@ pub const Tensor = struct { { // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); + var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{ .shardings = &.{replicated_sharding} }); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -2381,6 +2491,7 @@ pub const Tensor = struct { test gatherSlices { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); const Local = struct { pub fn _gatherSlices(self: Tensor, slice_shape: Shape, indices: Tensor, opts: GatherOpts) Tensor { @@ -2390,7 +2501,7 @@ pub const Tensor = struct { { // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); + var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{ .shardings = &.{replicated_sharding} }); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -2433,10 +2544,17 @@ pub const Tensor = struct { // Test with actual values. const operand: Tensor = .init(.{ .a = 2, .b = 4, .c = 6 }, .u16); const indices: Tensor = .init(.{ .n = 2, ._ = 2 }, .i32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._gatherSlices, .{ operand, Shape.init(.{ .b = 2, .c = 3 }, .u16), indices, .{} }, platform); + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Local._gatherSlices, + .{ operand, Shape.init(.{ .b = 2, .c = 3 }, .u16), indices, .{} }, + platform, + .{ .shardings = &.{replicated_sharding} }, + ); defer exe.deinit(); - var indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, indices.shape(), std.mem.sliceAsBytes(&[2][2]i32{ .{ 2, 1 }, .{ 0, 3 } })); + var indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, indices.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2][2]i32{ .{ 2, 1 }, .{ 0, 3 } })); defer indices_buffer.deinit(); var operand_buffer: zml.Buffer = b: { const temp_func = struct { @@ -2444,7 +2562,7 @@ pub const Tensor = struct { return Tensor.arange(.{ .end = 2 * 4 * 6 }, .u16).reshape(.{ .a = 2, .b = 4, .c = 6 }); } }.forward; - var temp_exe = try zml.module.compile(std.testing.allocator, std.testing.io, temp_func, .{}, platform); + var temp_exe = try zml.module.compile(std.testing.allocator, std.testing.io, temp_func, .{}, platform, .{ .shardings = &.{replicated_sharding} }); defer temp_exe.deinit(); const buffer = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &temp_exe, temp_func, {}); @@ -2464,7 +2582,7 @@ pub const Tensor = struct { .{ .{ 27, 28, 29 }, .{ 33, 34, 35 } }, }, })); - try zml.testing.expectClose(std.testing.io, expected, result, 0); + try zml.testing.expectClose(std.testing.io, expected, result, .{}); } pub const ScatterOpts = struct { @@ -2583,6 +2701,7 @@ pub const Tensor = struct { test scatterSlices { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); const Local = struct { pub fn _scatter(self: Tensor, indices: []const Tensor, updates: Tensor) Tensor { @@ -2608,7 +2727,7 @@ pub const Tensor = struct { { // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); + var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{ .shardings = &.{replicated_sharding} }); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -2651,14 +2770,14 @@ pub const Tensor = struct { const scatter_indices = Tensor.init(.{2}, .i32).withTags(.{.n}); const updates = Tensor.init(.{ 2, 3 }, .i32).withTags(.{ .n, .b }); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), replicated_sharding, std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); defer a_buffer.deinit(); - var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), std.mem.sliceAsBytes(&[2]i32{ 0, 2 })); + var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2]i32{ 0, 2 })); defer scatter_indices_buffer.deinit(); - var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), std.mem.sliceAsBytes(&[2][3]i32{ .{ 10, 20, 30 }, .{ 70, 80, 90 } })); + var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2][3]i32{ .{ 10, 20, 30 }, .{ 70, 80, 90 } })); defer updates_buffer.deinit(); var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatter, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); @@ -2675,14 +2794,14 @@ pub const Tensor = struct { const scatter_indices = Tensor.init(.{2}, .i32).withTags(.{.n}); const updates = Tensor.init(.{2}, .i32).withTags(.{.n}); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), replicated_sharding, std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); defer a_buffer.deinit(); - var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), std.mem.sliceAsBytes(&[2]i32{ 2, 7 })); + var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2]i32{ 2, 7 })); defer scatter_indices_buffer.deinit(); - var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), std.mem.sliceAsBytes(&[2]i32{ 20, 70 })); + var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2]i32{ 20, 70 })); defer updates_buffer.deinit(); var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatter, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); @@ -2698,12 +2817,12 @@ pub const Tensor = struct { const start_indices: Tensor = .init(.{ .n = 2, .a = 2, .m = 3, .coord = 2 }, .i32); const values: Tensor = .init(.{ .n = 2, .a = 2, .m = 3, .c = 2, .d = 2 }, .u16); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatterCB, .{ operand, start_indices, values }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatterCB, .{ operand, start_indices, values }, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); - var operand_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, operand.shape(), std.mem.sliceAsBytes(&@as([2 * 3 * 4 * 2]u16, @splat(0)))); + var operand_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, operand.shape(), replicated_sharding, std.mem.sliceAsBytes(&@as([2 * 3 * 4 * 2]u16, @splat(0)))); defer operand_buffer.deinit(); - var start_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, start_indices.shape(), std.mem.sliceAsBytes(&[2][2][3][2]i32{ + var start_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, start_indices.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2][2][3][2]i32{ .{ .{ .{ 0, 0 }, .{ 1, 0 }, .{ 2, 1 } }, .{ .{ 0, 1 }, .{ 1, 1 }, .{ 0, 9 } }, @@ -2714,7 +2833,7 @@ pub const Tensor = struct { }, })); defer start_indices_buffer.deinit(); - var values_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, values.shape(), std.mem.sliceAsBytes(&@as([2 * 2 * 3 * 2 * 2]u16, @splat(1)))); + var values_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, values.shape(), replicated_sharding, std.mem.sliceAsBytes(&@as([2 * 2 * 3 * 2 * 2]u16, @splat(1)))); defer values_buffer.deinit(); var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatterCB, .{ operand_buffer, start_indices_buffer, values_buffer }); @@ -2808,6 +2927,7 @@ pub const Tensor = struct { test argMax { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); const allocator = std.testing.allocator; const ArgMaxTest = struct { pub fn _fwd(x: Tensor) Tensor.ArgMaxRes { @@ -2816,11 +2936,11 @@ pub const Tensor = struct { }; const x: Tensor = .init(.{ 1, 5 }, .f32); - var exe = try zml.module.compile(allocator, std.testing.io, ArgMaxTest._fwd, .{x}, platform); + var exe = try zml.module.compile(allocator, std.testing.io, ArgMaxTest._fwd, .{x}, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); { - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, 4.1, 7.9, 0, 7.9 }})); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, 4.1, 7.9, 0, 7.9 }})); defer x_buffer.deinit(); var res = try zml.testing.autoCall(allocator, std.testing.io, &exe, ArgMaxTest._fwd, .{x_buffer}); @@ -2834,7 +2954,7 @@ pub const Tensor = struct { } { - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, std.math.nan(f32), 7.9, 0, 7.9 }})); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, std.math.nan(f32), 7.9, 0, 7.9 }})); defer x_buffer.deinit(); var res = try zml.testing.autoCall(allocator, std.testing.io, &exe, ArgMaxTest._fwd, .{x_buffer}); @@ -2874,6 +2994,7 @@ pub const Tensor = struct { test argsort { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); const Local = struct { pub fn _argsort(x: Tensor, axis_: u3, opts: ArgSortOpts) Tensor { @@ -2885,10 +3006,10 @@ pub const Tensor = struct { { const x: Tensor = .init(.{ 2, 5 }, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{} }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{} }, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 })); defer x_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); @@ -2904,10 +3025,10 @@ pub const Tensor = struct { { const x: Tensor = .init(.{ 1, 5, 10 }, .f16); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{ .descending = true } }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{ .descending = true } }, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f16{ + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[_]f16{ -0.2505, 1.2520, -0.7041, 0.1066, 1.2773, -1.7246, 0.8389, 1.1094, 0.0601, 1.0684, 0.9619, 1.3916, 1.2246, -0.1406, 0.3674, -1.2480, -1.7051, -0.0934, 0.3435, 0.4373, 1.3809, 0.5444, -0.6079, 1.2031, -0.6880, 1.2979, -0.1869, 0.2991, 0.0156, 0.1847, @@ -2935,10 +3056,10 @@ pub const Tensor = struct { { const x: Tensor = .init(.{ 4, 2, 1, 4 }, .i32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 3, .{} }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 3, .{} }, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[_]i32{ 89, 31, 22, 42, 64, 39, 0, 30, 64, 71, 46, 31, @@ -3101,9 +3222,10 @@ pub const Tensor = struct { test chunkExact { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); + var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{ .shardings = &.{replicated_sharding} }); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -3162,9 +3284,10 @@ pub const Tensor = struct { test chunkAllowTrailing { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); + var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{ .shardings = &.{replicated_sharding} }); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -3299,6 +3422,7 @@ pub const Tensor = struct { test dynamicSlice { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); inline for (.{ .{ [10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, Shape.init(.{10}, .f32), [2]f32{ 4, 5 }, 4, 0 }, @@ -3308,12 +3432,12 @@ pub const Tensor = struct { const x: Tensor = .fromShape(x_shape); const z: Tensor = .init(.{}, .i32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.dynamicSlice1d, .{ x, ax, .{ .len = 2, .start = z } }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.dynamicSlice1d, .{ x, ax, .{ .len = 2, .start = z } }, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&x_data)); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&x_data)); defer x_buffer.deinit(); - var z_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, z.shape(), std.mem.asBytes(&z_value)); + var z_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, z.shape(), replicated_sharding, std.mem.asBytes(&z_value)); defer z_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.dynamicSlice1d, .{ x_buffer, .{ .start = z_buffer } }); @@ -3422,6 +3546,7 @@ pub const Tensor = struct { test dynamicUpdateSlice { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); { const x = Tensor.init(.{10}, .f32).withTags(.{.a}); @@ -3432,14 +3557,14 @@ pub const Tensor = struct { return x_.dynamicUpdateSlice(idx_, y_); } }._fwd; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = ids }, y }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = ids }, y }, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); defer y_buffer.deinit(); - var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{4})); + var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1]i32{4})); defer ids_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ .a = ids_buffer }, y_buffer }); @@ -3458,14 +3583,14 @@ pub const Tensor = struct { return x_.dynamicUpdateSlice(.{ .b = idx_ }, y_); } }._fwd; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); defer y_buffer.deinit(); - var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3})); + var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1]i32{3})); defer ids_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, ids_buffer, y_buffer }); @@ -3487,14 +3612,14 @@ pub const Tensor = struct { return x_.dynamicUpdateSlice(.{ zml.Tensor.scalar(0, .i32), idx_ }, y_); } }._fwd; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[2][1]f32{ .{-1}, .{-1} })); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2][1]f32{ .{-1}, .{-1} })); defer y_buffer.deinit(); - var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3})); + var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1]i32{3})); defer ids_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, ids_buffer, y_buffer }); @@ -3517,16 +3642,16 @@ pub const Tensor = struct { return x_.dynamicUpdateSlice(idx_, y_); } }._fwd; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = idx_a, .b = idx_b }, y }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = idx_a, .b = idx_b }, y }, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[1]f32{-1})); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1]f32{-1})); defer y_buffer.deinit(); - var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), std.mem.sliceAsBytes(&[1]i32{1})); + var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1]i32{1})); defer idx_a_buffer.deinit(); - var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3})); + var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1]i32{3})); defer idx_b_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ .a = idx_a_buffer, .b = idx_b_buffer }, y_buffer }); @@ -3549,16 +3674,16 @@ pub const Tensor = struct { return x_.dynamicUpdateSlice(&idx_, y_); } }._fwd; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ idx_a, idx_b }, y }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ idx_a, idx_b }, y }, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[1][1]f32{.{-1}})); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1][1]f32{.{-1}})); defer y_buffer.deinit(); - var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), std.mem.sliceAsBytes(&[1]i32{1})); + var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1]i32{1})); defer idx_a_buffer.deinit(); - var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3})); + var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1]i32{3})); defer idx_b_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ idx_a_buffer, idx_b_buffer }, y_buffer }); @@ -3618,6 +3743,7 @@ pub const Tensor = struct { test toDiagonal { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); const Local = struct { pub fn _toDiag(input: Tensor) Tensor { @@ -3629,23 +3755,20 @@ pub const Tensor = struct { }; const x: Tensor = .init(.{ 2, 2 }, .u8); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._toDiag, .{x}, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._toDiag, .{x}, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][2]u8{ .{ 1, 2 }, .{ 3, 4 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2][2]u8{ .{ 1, 2 }, .{ 3, 4 } })); defer x_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._toDiag, .{x_buffer}); defer res.deinit(); + try std.testing.expectEqual( - [2][2][2]u8{ .{ - .{ 1, 0 }, - .{ 0, 2 }, - }, .{ - .{ 3, 0 }, - .{ 0, 4 }, - } }, + [2][2][2]u8{ + .{ .{ 1, 0 }, .{ 0, 2 } }, + .{ .{ 3, 0 }, .{ 0, 4 } }, + }, try res.getValue([2][2][2]u8, std.testing.io), ); } @@ -3674,6 +3797,7 @@ pub const Tensor = struct { test triangular { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); const Local = struct { pub fn _tri(input: Tensor, num_diagonals: i32) Tensor { @@ -3683,7 +3807,7 @@ pub const Tensor = struct { const x: Tensor = .init(.{ 3, 3 }, .u8); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[3][3]u8{ + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[3][3]u8{ .{ 1, 1, 1 }, .{ 1, 1, 1 }, .{ 1, 1, 1 }, @@ -3691,7 +3815,7 @@ pub const Tensor = struct { defer x_buffer.deinit(); { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 0 }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 0 }, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); @@ -3707,7 +3831,7 @@ pub const Tensor = struct { ); } { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 1 }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 1 }, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); @@ -3723,7 +3847,7 @@ pub const Tensor = struct { ); } { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, -1 }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, -1 }, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); @@ -3848,6 +3972,7 @@ pub const Tensor = struct { test cartesianProduct { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); const x: Tensor = .init(.{6}, .i32); const y: Tensor = .init(.{4}, .i32); @@ -3858,12 +3983,12 @@ pub const Tensor = struct { } }; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cartesianProduct2, .{ x, y }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cartesianProduct2, .{ x, y }, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), replicated_sharding, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); defer y_buffer.deinit(); var xs, var ys = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._cartesianProduct2, .{ x_buffer, y_buffer }); @@ -3922,6 +4047,7 @@ pub const Tensor = struct { test cartesianProductStacked { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); const x: Tensor = .init(.{6}, .i32); const y: Tensor = .init(.{4}, .i32); @@ -3931,12 +4057,12 @@ pub const Tensor = struct { } }; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, y }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, y }, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), replicated_sharding, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); defer y_buffer.deinit(); var z = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._fwd, .{ x_buffer, y_buffer }); @@ -3993,7 +4119,11 @@ pub const Tensor = struct { /// Only for debug purpose, it inserts device to host synchronization /// so it will slow down the program execution. pub fn print(input: Tensor, name: []const u8) void { - _ = ops.customCall("zml$print", .{input}, .{}, .{ .name = name }, .{ .has_side_effect = true }); + ops.manualComputation(input, {}, .{ .name = name }, (struct { + fn body(ctx_: anytype, _: std.mem.Allocator, sharded_input: Tensor, _: void) void { + ops.customCall("zml$print", sharded_input, {}, .{ .name = ctx_.name }, .{ .has_side_effect = true }); + } + }).body); } fn mlirCtx() *mlir.Context { @@ -4059,6 +4189,7 @@ fn getComparisonType(ctx: *mlir.Context, dtype: DataType) *const dialects.stable test "Tensor.maxPool1d" { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); const MaxPool = struct { pub fn _fwd(x: zml.Tensor) Tensor.ArgMaxRes { @@ -4074,10 +4205,10 @@ test "Tensor.maxPool1d" { const x: Tensor = .init(.{ 2, 2, 5 }, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&data)); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&data)); defer x_buffer.deinit(); var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, MaxPool._fwd, .{x_buffer}); @@ -4098,6 +4229,7 @@ test "Tensor.maxPool1d" { test "Tensor.maxPool2d" { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); const MaxPool = struct { pub fn _fwd(x: Tensor) Tensor.ArgMaxRes { @@ -4113,10 +4245,10 @@ test "Tensor.maxPool2d" { const x: Tensor = .init(.{ 2, 2, 5, 5 }, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&data)); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&data)); defer x_buffer.deinit(); var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, MaxPool._fwd, .{x_buffer}); @@ -4204,6 +4336,7 @@ test transposeIsJustAReshape { test "unused tensor" { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); const Local = struct { pub fn _fwd(x: Tensor) Tensor { @@ -4213,6 +4346,13 @@ test "unused tensor" { } }; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{Tensor.init(.{10}, .f32)}, platform); + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Local._fwd, + .{Tensor.init(.{10}, .f32)}, + platform, + .{ .shardings = &.{replicated_sharding} }, + ); defer exe.deinit(); } diff --git a/zml/test_runner.zig b/zml/test_runner.zig deleted file mode 100644 index 121b9be4..00000000 --- a/zml/test_runner.zig +++ /dev/null @@ -1,112 +0,0 @@ -//! Test runner for unit test based on https://github.com/ziglang/zig/blob/master/lib/compiler/test_runner.zig with async -const std = @import("std"); -const testing = std.testing; -const builtin = @import("builtin"); - -// note: std_options.log_level does not respect testing.log_level -// ref: https://github.com/ziglang/zig/issues/5738 -const log_level: std.log.Level = .warn; - -pub const std_options: std.Options = .{ - .log_level = log_level, -}; - -var log_err_count: usize = 0; -var fba_buffer: [8192]u8 = undefined; -var fba = std.heap.FixedBufferAllocator.init(&fba_buffer); - -pub fn main() !void { - testing.log_level = log_level; - - const test_fn_list: []const std.builtin.TestFn = builtin.test_functions; - var ok_count: usize = 0; - var skip_count: usize = 0; - var fail_count: usize = 0; - const root_node = std.Progress.start(std.testing.io, .{ - .root_name = "Test", - .estimated_total_items = test_fn_list.len, - }); - - const have_tty = try std.Io.File.stderr().isTty(std.testing.io); - - var args = std.process.args(); - // Skip executable path - _ = args.next().?; - - const identifier_query = if (args.next()) |arg| blk: { - std.debug.print("Only tests with identifiers that includes `{s}` will be run\n", .{arg}); - break :blk arg; - } else blk: { - break :blk ""; - }; - - var leaks: usize = 0; - - for (test_fn_list, 0..) |test_fn, i| { - if (std.mem.indexOf(u8, test_fn.name, identifier_query) == null) { - continue; - } - - testing.allocator_instance = .{}; - defer { - if (testing.allocator_instance.deinit() == .leak) { - leaks += 1; - } - } - - testing.io_instance = .init(testing.allocator_instance.allocator(), .{}); - defer testing.io_instance.deinit(); - - const test_node = root_node.start(test_fn.name, 0); - if (!have_tty) { - std.debug.print("{d}/{d} {s}...", .{ i + 1, test_fn_list.len, test_fn.name }); - } - if (test_fn.func()) |_| { - ok_count += 1; - test_node.end(); - if (!have_tty) std.debug.print("OK\n", .{}); - } else |err| switch (err) { - error.SkipZigTest => { - skip_count += 1; - if (have_tty) { - std.debug.print("{d}/{d} {s}...SKIP\n", .{ i + 1, test_fn_list.len, test_fn.name }); - } else { - std.debug.print("SKIP\n", .{}); - } - test_node.end(); - }, - else => { - fail_count += 1; - if (have_tty) { - std.debug.print("{d}/{d} {s}...FAIL ({s})\n", .{ - i + 1, test_fn_list.len, test_fn.name, @errorName(err), - }); - } else { - std.debug.print("FAIL ({s})\n", .{@errorName(err)}); - } - if (@errorReturnTrace()) |trace| { - std.debug.dumpStackTrace(trace); - } - test_node.end(); - }, - } - } - root_node.end(); - if (ok_count == test_fn_list.len) { - std.debug.print("All {d} tests passed.\n", .{ok_count}); - } else { - std.debug.print("{d} passed; {d} skipped; {d} failed.\n", .{ ok_count, skip_count, fail_count }); - } - if (log_err_count != 0) { - std.debug.print("{d} errors were logged.\n", .{log_err_count}); - } - if (leaks != 0) { - std.debug.print("{d} tests leaked memory.\n", .{leaks}); - } - if (leaks != 0 or log_err_count != 0 or fail_count != 0) { - std.process.exit(1); - } - - // Explicit exit 0 to exit from async main thread and main properly - std.process.exit(0); -} diff --git a/zml/testing.zig b/zml/testing.zig index 8e33ed58..b6a7d363 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -3,13 +3,15 @@ const builtin = @import("builtin"); const stdx = @import("stdx"); -const Slice = @import("slice.zig").Slice; const Platform = @import("platform.zig").Platform; +const sharding = @import("sharding.zig"); +const Slice = @import("slice.zig").Slice; const zml = @import("zml.zig"); const log = std.log.scoped(.@"zml/testing"); var _platform: ?*const Platform = null; +var _replicated_sharding: ?sharding.Sharding = null; pub fn env() *const Platform { if (!builtin.is_test) @compileError("Cannot use zml.testing.env outside of a test block"); @@ -22,6 +24,17 @@ pub fn env() *const Platform { return _platform.?; } +pub fn physicalMesh() sharding.PhysicalMesh { + return env().physical_mesh; +} + +pub fn replicatedSharding() sharding.Sharding { + if (_replicated_sharding == null) { + _replicated_sharding = sharding.replicatedSharding(env()) catch unreachable; + } + return _replicated_sharding.?; +} + /// In neural network we generally care about the relative precision, /// but on a given dimension, if the output is close to 0, then the precision /// don't matter as much. @@ -289,6 +302,7 @@ pub fn testLayer( activation_store: zml.io.TensorStore.View, name: []const u8, layer_weights: zml.Bufferized(@TypeOf(layer)), + shardings: []const zml.Sharding, opts: CompareOpts, ) !void { var arena = std.heap.ArenaAllocator.init(allocator); @@ -323,7 +337,7 @@ pub fn testLayer( } }.cb, &ctx, &args); - const exe = try platform.compile(allocator, io, layer, func, args); + const exe = try platform.compile(allocator, io, layer, func, args, .{ .shardings = shardings }); defer exe.deinit(); const output_name = try std.fmt.allocPrint(arena.allocator(), "{s}.out", .{name}); @@ -340,7 +354,13 @@ pub fn testLayer( var exe_results = try exe.results(allocator); defer exe_results.deinit(allocator); - var args_buffers = try zml.io.load(ArgsT, &args, allocator, io, platform, .{ .dma_chunks = 1, .dma_chunk_size = 4096, .store = activation_store.store, .parallelism = 1 }); + var args_buffers = try zml.io.load(ArgsT, &args, allocator, io, platform, .{ + .dma_chunks = 1, + .dma_chunk_size = 4096, + .store = activation_store.store, + .parallelism = 1, + .shardings = shardings, + }); defer zml.meta.visit(struct { fn cb(_: void, b: *zml.Buffer) void { b.deinit(); diff --git a/zml/tokenizer/BUILD.bazel b/zml/tokenizer/BUILD.bazel index 691bd0ee..96360191 100644 --- a/zml/tokenizer/BUILD.bazel +++ b/zml/tokenizer/BUILD.bazel @@ -35,7 +35,8 @@ zig_test( ], ) -zig_srcs( - name = "sources", - zig_lib = ":tokenizer", -) +# todo: reenable after fix +# zig_srcs( +# name = "sources", +# zig_lib = ":tokenizer", +# ) diff --git a/zml/tokenizer/sentencepiece/BUILD.bazel b/zml/tokenizer/sentencepiece/BUILD.bazel index 0fe8e061..f8e571e2 100644 --- a/zml/tokenizer/sentencepiece/BUILD.bazel +++ b/zml/tokenizer/sentencepiece/BUILD.bazel @@ -25,10 +25,11 @@ zig_library( ], ) -zig_srcs( - name = "sources", - zig_lib = ":sentencepiece", -) +# todo: reenable after fix +# zig_srcs( +# name = "sources", +# zig_lib = ":sentencepiece", +# ) cc_static_library( name="sentencepiece_static", diff --git a/zml/zml.zig b/zml/zml.zig index 22faac3c..b582466f 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -28,16 +28,18 @@ pub const Bufferized = mem.Bufferized; pub const meta = @import("meta.zig"); pub const mlir = @import("mlirx.zig"); pub const module = @import("module.zig"); +pub const CompilationOptions = module.CompilationOptions; pub const nn = @import("nn.zig"); pub const ops = @import("ops.zig"); +pub const pjrtx = @import("pjrtx.zig"); pub const platform = @import("platform.zig"); pub const Memory = platform.Memory; pub const Platform = platform.Platform; pub const Target = platform.Target; -pub const CompilationOptions = platform.CompilationOptions; pub const safetensors = @import("safetensors.zig"); pub const shape = @import("shape.zig"); pub const Shape = shape.Shape; +pub const sharding = @import("sharding.zig"); pub const slice = @import("slice.zig"); pub const Slice = slice.Slice; pub const tensor = @import("tensor.zig"); From a6058eb354870aeadb6419cb58bca99ad9453d14 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 10 Mar 2026 12:49:33 +0000 Subject: [PATCH 346/881] third_party: stb is a non module dep --- third_party/non_module_deps.bzl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/third_party/non_module_deps.bzl b/third_party/non_module_deps.bzl index fbeee624..af773459 100644 --- a/third_party/non_module_deps.bzl +++ b/third_party/non_module_deps.bzl @@ -6,6 +6,7 @@ load("//third_party/translate-c:repo.bzl", translate_c = "repo") load("//third_party/xla:repo.bzl", xla = "repo") load("//third_party/flashattn:repo.bzl", flashattn = "repo") load("//third_party/linenoise:repo.bzl", linenoise = "repo") +load("//third_party/stb:repo.bzl", stb = "repo") def _non_module_deps_impl(mctx): com_google_sentencepiece() @@ -16,6 +17,7 @@ def _non_module_deps_impl(mctx): translate_c() flashattn() linenoise() + stb() return mctx.extension_metadata( reproducible = True, From 7b048b7e510bdb4807b787c54a77552971988186 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 10 Mar 2026 12:51:08 +0000 Subject: [PATCH 347/881] zml/io/hf: fix request using identity header for tree api call --- zml/io/vfs/hf.zig | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/zml/io/vfs/hf.zig b/zml/io/vfs/hf.zig index ca9a0bc8..3fc38e01 100644 --- a/zml/io/vfs/hf.zig +++ b/zml/io/vfs/hf.zig @@ -292,7 +292,10 @@ pub const HF = struct { const uri = try std.Uri.parse(url); var req = try self.client.request(.GET, uri, .{ - .headers = .{ .authorization = self.authorization }, + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = self.authorization, + }, }); defer req.deinit(); From 70e4c9953d758d7a5dd7c54c179423b28072c415 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 10 Mar 2026 15:58:48 +0100 Subject: [PATCH 348/881] Bump zig to `0.16.0-dev.2722+f16eb18ce` (#409) Co-authored-by: Hugo Mano --- MODULE.bazel | 4 +- MODULE.bazel.lock | 202 +++++++++++++------------- bazel/bazel.zig | 15 -- bazel/zig_index.json | 154 ++++++++++---------- examples/benchmark/main.zig | 6 +- examples/io/main.zig | 2 +- examples/lfm/discussion.zig | 6 +- examples/lfm/inference.zig | 10 +- examples/lfm/main.zig | 2 +- examples/lfm/model.zig | 4 +- examples/llama/llama.zig | 9 +- examples/llama/main.zig | 18 +-- examples/mnist/mnist.zig | 6 +- mlir/dialects/stablehlo/stablehlo.zig | 4 +- stdx/fmt.zig | 12 -- zml/io.zig | 11 +- zml/io/vfs/base.zig | 10 +- zml/nn.zig | 15 +- zml/zml.zig | 4 - 19 files changed, 224 insertions(+), 270 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 2f27a5fa..dc63e32f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -27,7 +27,7 @@ bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2", dev_dependency = Tr ### Zig toolchain zig = use_extension("@rules_zig//zig:extensions.bzl", "zig") zig.index(file = "//bazel:zig_index.json") -zig.toolchain(zig_version = "0.16.0-dev.2611+f996d2866") +zig.toolchain(zig_version = "0.16.0-dev.2722+f16eb18ce") zig.mirrors(urls = [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -56,7 +56,7 @@ register_toolchains("@llvm//toolchain:all") ### uv = use_extension("@rules_python//python/uv:uv.bzl", "uv", dev_dependency = True) -uv.configure(version = "0.6.5") +uv.configure(version = "0.9.30") python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain(python_version = "3.12") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 0996516a..995dd985 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -850,97 +850,97 @@ "@@rules_python+//python/uv:uv.bzl%uv": { "general": { "bzlTransitiveDigest": "CJBWuaTerWIqLpLWl77vjWFD1RRpXfzC2UGMwEk6eVg=", - "usagesDigest": "lPQI4kxB6q1JLGHa7OFxCNrWxnLjeMKg5ogOIB5OF6U=", + "usagesDigest": "zIGk4Tqt5ZS3xJ783QnGoVqaEhMIIzUMmK0DHMRGugg=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { - "uv_0_6_5_aarch64_apple_darwin": { + "uv_0_9_30_aarch64_apple_darwin": { "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", "attributes": { - "version": "0.6.5", + "version": "0.9.30", "platform": "aarch64-apple-darwin", "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-aarch64-apple-darwin.tar.gz" + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-aarch64-apple-darwin.tar.gz" ], - "sha256": "4952c6275778fbc7e5cf6b5739ffd944c47063134d6ea43a7037b0a028ef771e" + "sha256": "03a5d9ec7f7d588446b2ec226d13ff6300055e55365eca8f3fab39f342b0e805" } }, - "uv_0_6_5_aarch64_unknown_linux_gnu": { + "uv_0_9_30_aarch64_unknown_linux_gnu": { "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", "attributes": { - "version": "0.6.5", + "version": "0.9.30", "platform": "aarch64-unknown-linux-gnu", "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-aarch64-unknown-linux-gnu.tar.gz" + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-aarch64-unknown-linux-gnu.tar.gz" ], - "sha256": "ebf0707ab949b948ad14250ed93f968fec954e20d12ce6bc9fce74b0ce7e4900" + "sha256": "6aadf3c71600d594e16dabf382cc15282ead4c5ca768599b6bcb43c5004d9aa8" } }, - "uv_0_6_5_powerpc64_unknown_linux_gnu": { + "uv_0_9_30_powerpc64_unknown_linux_gnu": { "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", "attributes": { - "version": "0.6.5", + "version": "0.9.30", "platform": "powerpc64-unknown-linux-gnu", "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-powerpc64-unknown-linux-gnu.tar.gz" + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-powerpc64-unknown-linux-gnu.tar.gz" ], - "sha256": "1f8fd8dde91258380dbcd4b9fc65f15a7dc3af6cfffefb9a4f99d32c4c67436d" + "sha256": "ac4cd1a021462885932f6023b005a4835cca4c72bb60dec186ee2be4b60dca6f" } }, - "uv_0_6_5_powerpc64le_unknown_linux_gnu": { + "uv_0_9_30_powerpc64le_unknown_linux_gnu": { "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", "attributes": { - "version": "0.6.5", + "version": "0.9.30", "platform": "powerpc64le-unknown-linux-gnu", "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-powerpc64le-unknown-linux-gnu.tar.gz" + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-powerpc64le-unknown-linux-gnu.tar.gz" ], - "sha256": "d2b6e19ca7ecd8d7e0d1ddced965aa91a3c822d07a7f7657743945245c1f627a" + "sha256": "73b8cbc560c6b2fa205358365d4e174abdf50cfcf57dc36a447572c56eba5ae4" } }, - "uv_0_6_5_s390x_unknown_linux_gnu": { + "uv_0_9_30_s390x_unknown_linux_gnu": { "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", "attributes": { - "version": "0.6.5", + "version": "0.9.30", "platform": "s390x-unknown-linux-gnu", "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-s390x-unknown-linux-gnu.tar.gz" + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-s390x-unknown-linux-gnu.tar.gz" ], - "sha256": "43a3e39f19fd5d098bb8c896b6c0f99a146253c271d78409cf31c2f55f2763d0" + "sha256": "b35975bb9e5c2c418b428d0316cc6e3c7a6eff710c69212be14005c192f54516" } }, - "uv_0_6_5_x86_64_apple_darwin": { + "uv_0_9_30_x86_64_apple_darwin": { "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", "attributes": { - "version": "0.6.5", + "version": "0.9.30", "platform": "x86_64-apple-darwin", "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-x86_64-apple-darwin.tar.gz" + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-x86_64-apple-darwin.tar.gz" ], - "sha256": "d336d862694e1c36afea55eb15f5df02aa77c04a168b781df515d641e23285f0" + "sha256": "ce069bf750567e9a4a31d6e285d1eae75d444d8a281409b641235903943b7681" } }, - "uv_0_6_5_x86_64_pc_windows_msvc": { + "uv_0_9_30_x86_64_pc_windows_msvc": { "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", "attributes": { - "version": "0.6.5", + "version": "0.9.30", "platform": "x86_64-pc-windows-msvc", "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-x86_64-pc-windows-msvc.zip" + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-x86_64-pc-windows-msvc.zip" ], - "sha256": "a3e9f0913189c38b804bf6bb3d348af305a8ceabc690be92b6e111245a821640" + "sha256": "875981be7908295937dee09532bb66d576986d4f223259e171b0c767c885897a" } }, - "uv_0_6_5_x86_64_unknown_linux_gnu": { + "uv_0_9_30_x86_64_unknown_linux_gnu": { "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", "attributes": { - "version": "0.6.5", + "version": "0.9.30", "platform": "x86_64-unknown-linux-gnu", "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-x86_64-unknown-linux-gnu.tar.gz" + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-x86_64-unknown-linux-gnu.tar.gz" ], - "sha256": "8fc9895719a1291ecd193cb86f9282ff3649cef797d29eacc74c4f573aab1e2f" + "sha256": "8b3762374972daa7a74bbc6896cc73229ca69a07403dd9f9ea3805a51ffd7582" } }, "uv": { @@ -948,55 +948,55 @@ "attributes": { "toolchain_type": "'@@rules_python+//python/uv:uv_toolchain_type'", "toolchain_names": [ - "0_6_5_aarch64_apple_darwin", - "0_6_5_aarch64_unknown_linux_gnu", - "0_6_5_powerpc64_unknown_linux_gnu", - "0_6_5_powerpc64le_unknown_linux_gnu", - "0_6_5_s390x_unknown_linux_gnu", - "0_6_5_x86_64_apple_darwin", - "0_6_5_x86_64_pc_windows_msvc", - "0_6_5_x86_64_unknown_linux_gnu" + "0_9_30_aarch64_apple_darwin", + "0_9_30_aarch64_unknown_linux_gnu", + "0_9_30_powerpc64_unknown_linux_gnu", + "0_9_30_powerpc64le_unknown_linux_gnu", + "0_9_30_s390x_unknown_linux_gnu", + "0_9_30_x86_64_apple_darwin", + "0_9_30_x86_64_pc_windows_msvc", + "0_9_30_x86_64_unknown_linux_gnu" ], "toolchain_implementations": { - "0_6_5_aarch64_apple_darwin": "@uv_0_6_5_aarch64_apple_darwin//:uv_toolchain", - "0_6_5_aarch64_unknown_linux_gnu": "@uv_0_6_5_aarch64_unknown_linux_gnu//:uv_toolchain", - "0_6_5_powerpc64_unknown_linux_gnu": "@uv_0_6_5_powerpc64_unknown_linux_gnu//:uv_toolchain", - "0_6_5_powerpc64le_unknown_linux_gnu": "@uv_0_6_5_powerpc64le_unknown_linux_gnu//:uv_toolchain", - "0_6_5_s390x_unknown_linux_gnu": "@uv_0_6_5_s390x_unknown_linux_gnu//:uv_toolchain", - "0_6_5_x86_64_apple_darwin": "@uv_0_6_5_x86_64_apple_darwin//:uv_toolchain", - "0_6_5_x86_64_pc_windows_msvc": "@uv_0_6_5_x86_64_pc_windows_msvc//:uv_toolchain", - "0_6_5_x86_64_unknown_linux_gnu": "@uv_0_6_5_x86_64_unknown_linux_gnu//:uv_toolchain" + "0_9_30_aarch64_apple_darwin": "@uv_0_9_30_aarch64_apple_darwin//:uv_toolchain", + "0_9_30_aarch64_unknown_linux_gnu": "@uv_0_9_30_aarch64_unknown_linux_gnu//:uv_toolchain", + "0_9_30_powerpc64_unknown_linux_gnu": "@uv_0_9_30_powerpc64_unknown_linux_gnu//:uv_toolchain", + "0_9_30_powerpc64le_unknown_linux_gnu": "@uv_0_9_30_powerpc64le_unknown_linux_gnu//:uv_toolchain", + "0_9_30_s390x_unknown_linux_gnu": "@uv_0_9_30_s390x_unknown_linux_gnu//:uv_toolchain", + "0_9_30_x86_64_apple_darwin": "@uv_0_9_30_x86_64_apple_darwin//:uv_toolchain", + "0_9_30_x86_64_pc_windows_msvc": "@uv_0_9_30_x86_64_pc_windows_msvc//:uv_toolchain", + "0_9_30_x86_64_unknown_linux_gnu": "@uv_0_9_30_x86_64_unknown_linux_gnu//:uv_toolchain" }, "toolchain_compatible_with": { - "0_6_5_aarch64_apple_darwin": [ + "0_9_30_aarch64_apple_darwin": [ "'@@platforms//os:macos'", "'@@platforms//cpu:aarch64'" ], - "0_6_5_aarch64_unknown_linux_gnu": [ + "0_9_30_aarch64_unknown_linux_gnu": [ "'@@platforms//os:linux'", "'@@platforms//cpu:aarch64'" ], - "0_6_5_powerpc64_unknown_linux_gnu": [ + "0_9_30_powerpc64_unknown_linux_gnu": [ "'@@platforms//os:linux'", "'@@platforms//cpu:ppc'" ], - "0_6_5_powerpc64le_unknown_linux_gnu": [ + "0_9_30_powerpc64le_unknown_linux_gnu": [ "'@@platforms//os:linux'", "'@@platforms//cpu:ppc64le'" ], - "0_6_5_s390x_unknown_linux_gnu": [ + "0_9_30_s390x_unknown_linux_gnu": [ "'@@platforms//os:linux'", "'@@platforms//cpu:s390x'" ], - "0_6_5_x86_64_apple_darwin": [ + "0_9_30_x86_64_apple_darwin": [ "'@@platforms//os:macos'", "'@@platforms//cpu:x86_64'" ], - "0_6_5_x86_64_pc_windows_msvc": [ + "0_9_30_x86_64_pc_windows_msvc": [ "'@@platforms//os:windows'", "'@@platforms//cpu:x86_64'" ], - "0_6_5_x86_64_unknown_linux_gnu": [ + "0_9_30_x86_64_unknown_linux_gnu": [ "'@@platforms//os:linux'", "'@@platforms//cpu:x86_64'" ] @@ -1022,18 +1022,18 @@ "@@rules_zig+//zig:extensions.bzl%zig": { "general": { "bzlTransitiveDigest": "p1+tPOh3UlhcIaGGf1ycZCCC6I4NIHhiiHS0nAhhgHA=", - "usagesDigest": "8TQ1JYfn2xQJIQya4EwCvLiuZAdGIocYzOMc1au4GNs=", + "usagesDigest": "aLO2dhznavzbCoReZv2wJRz7GoTgcrHljpLeLgEjBkY=", "recordedFileInputs": { - "@@//bazel/zig_index.json": "0b2647eea9809f05c33dac461c6572e85e0401221bca32ebb6353100132f29c5", + "@@//bazel/zig_index.json": "809b545c21f13c11c4587dccbfe81de2b767a292325c25ffb785b03c601bbc2e", "@@rules_zig+//zig/private/versions.json": "6eb85ebaee72c4e6fbc82d8ad73f69e0899d839b70a7e19caa792ab6a5fa0c36" }, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { - "zig_0.16.0-dev.2611_Pf996d2866_aarch64-linux": { + "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.2611+f996d2866.tar.xz", + "url": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.2722+f16eb18ce.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1050,15 +1050,15 @@ "https://fs.liujiacai.net/zigbuilds", "https://ziglang.org/builds/" ], - "sha256": "d4219908cded6efb97a7326e4159f82607e3956d6d56efb1580187bf0872603e", - "zig_version": "0.16.0-dev.2611+f996d2866", + "sha256": "03725d297a2e2a8331d774eb601163af42ec61979b82753d41b2c460eefab1be", + "zig_version": "0.16.0-dev.2722+f16eb18ce", "platform": "aarch64-linux" } }, - "zig_0.16.0-dev.2611_Pf996d2866_aarch64-macos": { + "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.2611+f996d2866.tar.xz", + "url": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.2722+f16eb18ce.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1075,15 +1075,15 @@ "https://fs.liujiacai.net/zigbuilds", "https://ziglang.org/builds/" ], - "sha256": "1eb526bb948c1c1bffdb5be318c044fc0216457be28b0d92aa2d92294b9d32c9", - "zig_version": "0.16.0-dev.2611+f996d2866", + "sha256": "268aa8342e8ef539693e88e1d0982361b1d5dc4a080949264f2d5a14e5eb83e0", + "zig_version": "0.16.0-dev.2722+f16eb18ce", "platform": "aarch64-macos" } }, - "zig_0.16.0-dev.2611_Pf996d2866_aarch64-windows": { + "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.2611+f996d2866.zip", + "url": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.2722+f16eb18ce.zip", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1100,15 +1100,15 @@ "https://fs.liujiacai.net/zigbuilds", "https://ziglang.org/builds/" ], - "sha256": "42a612662453e47ad4001eb19c1d219dec6f254913988e2ebe6c95e4b5da3e82", - "zig_version": "0.16.0-dev.2611+f996d2866", + "sha256": "43daaad311bc3b06a733a8a87d2c3180214d62db2067df1591b921826de453ea", + "zig_version": "0.16.0-dev.2722+f16eb18ce", "platform": "aarch64-windows" } }, - "zig_0.16.0-dev.2611_Pf996d2866_x86_64-linux": { + "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.2611+f996d2866.tar.xz", + "url": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.2722+f16eb18ce.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1125,15 +1125,15 @@ "https://fs.liujiacai.net/zigbuilds", "https://ziglang.org/builds/" ], - "sha256": "0188a70a3f84fe1918ac391a8b64bd241eb4c028c44ea023e853743cbecd3009", - "zig_version": "0.16.0-dev.2611+f996d2866", + "sha256": "b0fe8fd996b4974cd80fc0967460b13e3a6e3105c4bec19ea0566bd47d73ad81", + "zig_version": "0.16.0-dev.2722+f16eb18ce", "platform": "x86_64-linux" } }, - "zig_0.16.0-dev.2611_Pf996d2866_x86_64-macos": { + "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.2611+f996d2866.tar.xz", + "url": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.2722+f16eb18ce.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1150,15 +1150,15 @@ "https://fs.liujiacai.net/zigbuilds", "https://ziglang.org/builds/" ], - "sha256": "5ba0c08bd2e9c74d18c6ea25483226a8f8922f4e50d09d9219be842894b41334", - "zig_version": "0.16.0-dev.2611+f996d2866", + "sha256": "8e09fb9e90c0566629dfa52bce433e886f4cf7e2cd9985a358d7b8c473c4e9ce", + "zig_version": "0.16.0-dev.2722+f16eb18ce", "platform": "x86_64-macos" } }, - "zig_0.16.0-dev.2611_Pf996d2866_x86_64-windows": { + "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.2611+f996d2866.zip", + "url": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.2722+f16eb18ce.zip", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1175,8 +1175,8 @@ "https://fs.liujiacai.net/zigbuilds", "https://ziglang.org/builds/" ], - "sha256": "3ec5cac4b74b5fe6e04f092ffb6e025993538ecf07970a428919d638efafa542", - "zig_version": "0.16.0-dev.2611+f996d2866", + "sha256": "0f92e7234ed1d9608444487ca8fb928bd38da79dea744acf6c7a8fa0c440da6a", + "zig_version": "0.16.0-dev.2722+f16eb18ce", "platform": "x86_64-windows" } }, @@ -1184,28 +1184,28 @@ "repoRuleId": "@@rules_zig+//zig/private/repo:toolchains_repo.bzl%toolchains_repo", "attributes": { "names": [ - "zig_0.16.0-dev.2611_Pf996d2866_aarch64-linux", - "zig_0.16.0-dev.2611_Pf996d2866_aarch64-macos", - "zig_0.16.0-dev.2611_Pf996d2866_aarch64-windows", - "zig_0.16.0-dev.2611_Pf996d2866_x86_64-linux", - "zig_0.16.0-dev.2611_Pf996d2866_x86_64-macos", - "zig_0.16.0-dev.2611_Pf996d2866_x86_64-windows" + "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-linux", + "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-macos", + "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-windows", + "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-linux", + "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-macos", + "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-windows" ], "labels": [ - "@zig_0.16.0-dev.2611_Pf996d2866_aarch64-linux//:zig_toolchain", - "@zig_0.16.0-dev.2611_Pf996d2866_aarch64-macos//:zig_toolchain", - "@zig_0.16.0-dev.2611_Pf996d2866_aarch64-windows//:zig_toolchain", - "@zig_0.16.0-dev.2611_Pf996d2866_x86_64-linux//:zig_toolchain", - "@zig_0.16.0-dev.2611_Pf996d2866_x86_64-macos//:zig_toolchain", - "@zig_0.16.0-dev.2611_Pf996d2866_x86_64-windows//:zig_toolchain" + "@zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-linux//:zig_toolchain", + "@zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-macos//:zig_toolchain", + "@zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-windows//:zig_toolchain", + "@zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-linux//:zig_toolchain", + "@zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-macos//:zig_toolchain", + "@zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-windows//:zig_toolchain" ], "zig_versions": [ - "0.16.0-dev.2611+f996d2866", - "0.16.0-dev.2611+f996d2866", - "0.16.0-dev.2611+f996d2866", - "0.16.0-dev.2611+f996d2866", - "0.16.0-dev.2611+f996d2866", - "0.16.0-dev.2611+f996d2866" + "0.16.0-dev.2722+f16eb18ce", + "0.16.0-dev.2722+f16eb18ce", + "0.16.0-dev.2722+f16eb18ce", + "0.16.0-dev.2722+f16eb18ce", + "0.16.0-dev.2722+f16eb18ce", + "0.16.0-dev.2722+f16eb18ce" ], "exec_lengths": [ 2, diff --git a/bazel/bazel.zig b/bazel/bazel.zig index acbb4356..97ab63b2 100644 --- a/bazel/bazel.zig +++ b/bazel/bazel.zig @@ -31,21 +31,6 @@ export const _ linksection(init_array_section) = &struct { } }.call; -pub fn init(allocator: std.mem.Allocator, io: std.Io, argv: std.process.Args, environ_map: *std.process.Environ.Map) !void { - runfiles_global = try bazel_runfiles.Runfiles.create(.{ - .allocator = allocator, - .io = io, - .argv = argv, - .environ_map = environ_map, - }) orelse { - std.debug.panic("Unable to find runfiles", .{}); - }; -} - -pub fn initSimple(init_: std.process.Init) !void { - try init(init_.arena.allocator(), init_.io, init_.minimal.args, init_.environ_map); -} - pub fn runfiles(source_repository: []const u8) !bazel_runfiles.Runfiles.WithSourceRepo { return runfiles_global.withSourceRepo(source_repository); } diff --git a/bazel/zig_index.json b/bazel/zig_index.json index da8c0455..ff679151 100644 --- a/bazel/zig_index.json +++ b/bazel/zig_index.json @@ -1,133 +1,133 @@ { "master": { - "version": "0.16.0-dev.2611+f996d2866", - "date": "2026-02-14", + "version": "0.16.0-dev.2722+f16eb18ce", + "date": "2026-03-08", "docs": "https://ziglang.org/documentation/master/", "stdDocs": "https://ziglang.org/documentation/master/std/", "src": { - "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "cf085b0b63b3520cda55aa7d98a8c86b28a5c9739bffe4a0253c58880ded4c7b", - "size": "22442124" + "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.2722+f16eb18ce.tar.xz", + "shasum": "1880ddb5502206d8f7322ca1e0138b67b3928fb88dbab3defff80d1dd8a311bb", + "size": "22460744" }, "bootstrap": { - "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "b87fcb4aa966c9b7f07ceca302b15eb587936938a65b5861188200d5b3616c0d", - "size": "55205844" + "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.2722+f16eb18ce.tar.xz", + "shasum": "4e3f14dfdf73f9448fcb3eb7bf61c4e0b26f4e0949e572824fbd47ea62adf97e", + "size": "55225064" }, "x86_64-macos": { - "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "5ba0c08bd2e9c74d18c6ea25483226a8f8922f4e50d09d9219be842894b41334", - "size": "57619112" + "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.2722+f16eb18ce.tar.xz", + "shasum": "8e09fb9e90c0566629dfa52bce433e886f4cf7e2cd9985a358d7b8c473c4e9ce", + "size": "57648992" }, "aarch64-macos": { - "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "1eb526bb948c1c1bffdb5be318c044fc0216457be28b0d92aa2d92294b9d32c9", - "size": "52470216" + "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.2722+f16eb18ce.tar.xz", + "shasum": "268aa8342e8ef539693e88e1d0982361b1d5dc4a080949264f2d5a14e5eb83e0", + "size": "52489044" }, "x86_64-linux": { - "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "0188a70a3f84fe1918ac391a8b64bd241eb4c028c44ea023e853743cbecd3009", - "size": "55505692" + "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.2722+f16eb18ce.tar.xz", + "shasum": "b0fe8fd996b4974cd80fc0967460b13e3a6e3105c4bec19ea0566bd47d73ad81", + "size": "55512692" }, "aarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "d4219908cded6efb97a7326e4159f82607e3956d6d56efb1580187bf0872603e", - "size": "51276904" + "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.2722+f16eb18ce.tar.xz", + "shasum": "03725d297a2e2a8331d774eb601163af42ec61979b82753d41b2c460eefab1be", + "size": "51286740" }, "arm-linux": { - "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "b0b809695bee67798ce9c956be24d850beab5b2ca4f3bf3fc65552baea0214bb", - "size": "52142540" + "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.2722+f16eb18ce.tar.xz", + "shasum": "b4407543ef223d459d5925b2f1b8c700e61da3193a79fae3b6104c5116e092e7", + "size": "52149112" }, "riscv64-linux": { - "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "9244e44969386bd35477079b42db7c7dd5b636ed6ddd21cbcb35debf1db5f285", - "size": "55318440" + "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.2722+f16eb18ce.tar.xz", + "shasum": "473e13775ef2eefe715b727b933462d2ee84cbdebb606cc8988aa70cb6ae72f1", + "size": "55352592" }, "powerpc64le-linux": { - "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "fd9f8b35403ac5c78d40c979a0f467413ddbb0294c2157590bb8e48635c18df5", - "size": "55350924" + "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.2722+f16eb18ce.tar.xz", + "shasum": "2382d40afa44218a961dddd935cb5ce1f6800d5c5dd9df39a27a25026e0b09de", + "size": "55387192" }, "x86-linux": { - "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "24d2fbc05b7d92a63e228292c693f7fdfa974655510e8c7089e09ce8b9f41fd4", - "size": "58210984" + "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.2722+f16eb18ce.tar.xz", + "shasum": "6cb242bd3e0a709a70c1b0e45a81afe1253b6d8367d411e7e25e464344f815b1", + "size": "58228456" }, "loongarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "1448d4e780a1257d042c836c237807c6ecaaba36c03961fef1940a842c22cd6d", - "size": "52566124" + "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.2722+f16eb18ce.tar.xz", + "shasum": "5564d7f8323386df38327e72d0b515834487f72d2e7a34673d5195019b552a4d", + "size": "52590788" }, "s390x-linux": { - "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "d5bd0131aa1e1a40434acc703afd48f0d235ba9d1e988c944148d03df4dce967", - "size": "54992952" + "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.2722+f16eb18ce.tar.xz", + "shasum": "c7d74bcbbe8b6edb54e7498281b8efe76e7ddbdde9f7bef7699f39d020045bba", + "size": "55040152" }, "x86_64-windows": { - "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.2611+f996d2866.zip", - "shasum": "3ec5cac4b74b5fe6e04f092ffb6e025993538ecf07970a428919d638efafa542", - "size": "98104739" + "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.2722+f16eb18ce.zip", + "shasum": "0f92e7234ed1d9608444487ca8fb928bd38da79dea744acf6c7a8fa0c440da6a", + "size": "98134892" }, "aarch64-windows": { - "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.2611+f996d2866.zip", - "shasum": "42a612662453e47ad4001eb19c1d219dec6f254913988e2ebe6c95e4b5da3e82", - "size": "94105952" + "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.2722+f16eb18ce.zip", + "shasum": "43daaad311bc3b06a733a8a87d2c3180214d62db2067df1591b921826de453ea", + "size": "94131162" }, "x86-windows": { - "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.2611+f996d2866.zip", - "shasum": "ccc288a50431d732b4b8dcc55c6e76ec79b406cbb73c794864cd3c538a77ee0a", - "size": "99920751" + "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.2722+f16eb18ce.zip", + "shasum": "3e3c3224040329bdc7120a1d16a68117f82572a26985234cb6e68bd800eed11b", + "size": "99937580" }, "aarch64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "c0c4b7bee7d67c4fb44692a96068bde05ba88e4bbe431928540db5bb29a8ab34", - "size": "51220656" + "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.2722+f16eb18ce.tar.xz", + "shasum": "fd84224ef223721ce7ee45d31296ec520c0de34ae97c035a4927370d67e955a1", + "size": "51224920" }, "arm-freebsd": { - "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "9f470b7304d70e59febfb23e14cacdbf1f81c2839e181f290009c3ccb04bbd4e", - "size": "52814796" + "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.2722+f16eb18ce.tar.xz", + "shasum": "01446e45de800ef17f1e8076a3765a51be91c8aa3630166bf8722104e7bf9db2", + "size": "52823280" }, "powerpc64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-powerpc64-freebsd-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "7caa39c909cee4342876f0623a40ad91f364b509a14bdcdcdd86629071a1dfbb", - "size": "53925668" + "tarball": "https://ziglang.org/builds/zig-powerpc64-freebsd-0.16.0-dev.2722+f16eb18ce.tar.xz", + "shasum": "7b68f44fb289651b09318b45e754413f2473ae5956736307f5b55a6d89745ebf", + "size": "53947616" }, "powerpc64le-freebsd": { - "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "4a0ea164b83b6bac83764c5b34c64b85ecbae1b685dc07a0fa59410efdde32c5", - "size": "55367852" + "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.2722+f16eb18ce.tar.xz", + "shasum": "a77432752e3f9d2746a99b973563a6439bf1908b512a6ad6b74ed8b0bd02c999", + "size": "55402492" }, "riscv64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "49f2e4403aad2256b921a2952a7e4e262c1077f3eb3bb0bd7c3e69a3154fbf68", - "size": "55478468" + "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.2722+f16eb18ce.tar.xz", + "shasum": "d75e1c261075c64a295ae92c28a4871c6ea13541f2604e9399effb31a1473e3a", + "size": "55496140" }, "x86_64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "169314c88df8e98c4b5c2db9805d160dcf332013fcd1f8a0df09f81a5eb3425a", - "size": "55617364" + "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.2722+f16eb18ce.tar.xz", + "shasum": "edd6ba3241cc0475f177ac2e4bcba692193259c30d0b5da288a4d76f427e0737", + "size": "55622060" }, "aarch64-netbsd": { - "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "a091fd4f7128e0098915d6693eb720b1a4d1740dcb552e99a0957d99c3fda3c4", - "size": "51174248" + "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.2722+f16eb18ce.tar.xz", + "shasum": "0499caf08e5bd91bb4201b4cad68faa71be016edab9dbbd25454721549d61bfb", + "size": "51189128" }, "arm-netbsd": { - "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "e399f1751fc25e3e04c2b7411a5fd9b9c316233b9370fc2fdf7fb45e8b020922", - "size": "53840612" + "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.2722+f16eb18ce.tar.xz", + "shasum": "46a90dc70bc59384adf9e5f2d70da6119dd4414c8a7e696309313d3b8f9e3022", + "size": "53863744" }, "x86-netbsd": { - "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "8fae56f0683a8f9d2767ad89adc156971f0a872fb485b4c70966e6b113bcad68", - "size": "58800484" + "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.2722+f16eb18ce.tar.xz", + "shasum": "92c420f1291b9f8cd4bb423000cca24403ef6dea03ec9f468e81fc20554d0db3", + "size": "58827296" }, "x86_64-netbsd": { - "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "0063013d3d306b4979b16f87db42fa1c7d94d5eac9275979cf40afa944328a56", - "size": "55551904" + "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.2722+f16eb18ce.tar.xz", + "shasum": "5b0940ac45f96718ef697de9f5edcb66bd7985928aad2de91004d807a8bca07a", + "size": "55576856" } }, "0.15.2": { diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index 549a592c..304cea91 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -44,7 +44,7 @@ pub fn main(init: std.process.Init) !void { var exe = blk: { log.info("⏱️ Compiling benchmark...", .{}); const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("✅ Compiled benchmark [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); + defer log.info("✅ Compiled benchmark [{f}]", .{now.untilNow(io, .awake)}); break :blk try platform.compileFn(allocator, io, benchmark, .{ a, b }, .{ .shardings = &.{benchmark_sharding} }); }; defer exe.deinit(); @@ -88,11 +88,11 @@ pub fn main(init: std.process.Init) !void { const floating_op_count = 2 * cli_args.size * cli_args.size * cli_args.size; const flops = @as(f64, @floatFromInt(floating_op_count)) / elapsed_s; - log.info("Dot product size: {d}x{d} - Datatype: {s} - Elapsed: {D} - {d:.3} GFLOP/s", .{ + log.info("Dot product size: {d}x{d} - Datatype: {s} - Elapsed: {f} - {d:.3} GFLOP/s", .{ cli_args.size, cli_args.size, @tagName(cli_args.dtype), - stdx.fmt.fmtDuration(elapsed), + elapsed, flops / 1_000_000_000, }); } diff --git a/examples/io/main.zig b/examples/io/main.zig index 977b744a..8b4d65bc 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -203,7 +203,7 @@ pub fn main(init: std.process.Init) !void { defer { const took = now.untilNow(io, .awake); const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {D}, {Bi:.2}/s]", .{ total_bytes, stdx.fmt.fmtDuration(took), bytes_per_sec }); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); } _ = try zml.io.load(AllTensorsModel, &model, init.arena.allocator(), io, platform, .{ diff --git a/examples/lfm/discussion.zig b/examples/lfm/discussion.zig index 7f6fcd81..8a5852e1 100644 --- a/examples/lfm/discussion.zig +++ b/examples/lfm/discussion.zig @@ -117,10 +117,10 @@ pub const Context = struct { try stdout.interface.writeAll("\n\n"); - try stdout.interface.print("\x1b[36mprefill\x1b[0m \x1b[2m{D} \xc2\xb7 {:.1}tok/s\x1b[0m \x1b[2m\xe2\x94\x82\x1b[0m \x1b[36mdecode\x1b[0m \x1b[2m{D} \xc2\xb7 {:.1}tok/s\x1b[0m\n\n\n", .{ - stdx.fmt.fmtDuration(prefill_duration), + try stdout.interface.print("\x1b[36mprefill\x1b[0m \x1b[2m{f} \xc2\xb7 {:.1}tok/s\x1b[0m \x1b[2m\xe2\x94\x82\x1b[0m \x1b[36mdecode\x1b[0m \x1b[2m{f} \xc2\xb7 {:.1}tok/s\x1b[0m\n\n\n", .{ + prefill_duration, tokensPerSecond(prefill_duration, prefill_tokens), - stdx.fmt.fmtDuration(decode_duration), + decode_duration, tokensPerSecond(decode_duration, decode_tokens), }); try stdout.interface.flush(); diff --git a/examples/lfm/inference.zig b/examples/lfm/inference.zig index 9feabfaf..01479c03 100644 --- a/examples/lfm/inference.zig +++ b/examples/lfm/inference.zig @@ -159,7 +159,7 @@ fn compileSingleKernelExe(allocator: std.mem.Allocator, io: std.Io, platform: *z var node = progress.start("Compiling single kernel...", 1); defer node.end(); const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Compiled single kernel [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); + defer log.info("Compiled single kernel [{f}]", .{now.untilNow(io, .awake)}); const actual_seq_len: zml.Tensor = .init(.{}, .u32); const tokens: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32); @@ -300,7 +300,7 @@ fn compileEmbedTokens(allocator: std.mem.Allocator, io: std.Io, platform: *zml.P var node = progress.start("Compiling embed_tokens...", 1); defer node.end(); const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Compiled embed_tokens [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); + defer log.info("Compiled embed_tokens [{f}]", .{now.untilNow(io, .awake)}); const tokens: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32); return platform.compile(allocator, io, embed_tokens, .forward, .{tokens}); @@ -311,7 +311,7 @@ fn compileConvLayer(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Pla var node = progress.start("Compiling conv layer...", 1); defer node.end(); const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Compiled conv layer [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); + defer log.info("Compiled conv layer [{f}]", .{now.untilNow(io, .awake)}); // Find the first conv layer as a representative for compilation. const conv_layer = for (mdl.layers) |layer| { @@ -342,7 +342,7 @@ fn compileAttnLayer(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Pla var node = progress.start("Compiling attn layer...", 1); defer node.end(); const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Compiled attn layer [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); + defer log.info("Compiled attn layer [{f}]", .{now.untilNow(io, .awake)}); // Find the first attention layer as a representative for compilation. const attn_layer = for (mdl.layers) |layer| { @@ -373,7 +373,7 @@ fn compileLmHead(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platfo var node = progress.start("Compiling lm_head...", 1); defer node.end(); const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Compiled lm_head [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); + defer log.info("Compiled lm_head [{f}]", .{now.untilNow(io, .awake)}); const hidden: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()); const tokens: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32); diff --git a/examples/lfm/main.zig b/examples/lfm/main.zig index 6f93fb01..dc555500 100644 --- a/examples/lfm/main.zig +++ b/examples/lfm/main.zig @@ -159,7 +159,7 @@ pub fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, var node = progress.start("Loading tokenizer...", 1); defer node.end(); const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Loaded tokenizer [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); + defer log.info("Loaded tokenizer [{f}]", .{now.untilNow(io, .awake)}); const bytes = b: { const file = try dir.openFile(io, "tokenizer.json", .{}); defer file.close(io); diff --git a/examples/lfm/model.zig b/examples/lfm/model.zig index c8925f0d..9eb61912 100644 --- a/examples/lfm/model.zig +++ b/examples/lfm/model.zig @@ -60,9 +60,9 @@ pub const Model = struct { defer { const took = now.untilNow(io, .awake); const took_ns: usize = @max(1, @as(usize, @intCast(took.toNanoseconds()))); - log.info("Loaded weights [{Bi:.2}, {D}, {Bi:.2}/s]", .{ + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, - stdx.fmt.fmtDuration(took), + took, total_bytes * std.time.ns_per_s / took_ns, }); } diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 55aac505..d4f0ce80 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -83,7 +83,7 @@ pub const LlamaLM = struct { defer { const took = now.untilNow(io, .awake); const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {D}, {Bi:.2}/s]", .{ total_bytes, stdx.fmt.fmtDuration(took), bytes_per_sec }); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); } return zml.io.load(LlamaLM, self, allocator, io, platform, .{ .dma_chunks = 32, @@ -157,7 +157,7 @@ pub const Llama = struct { num_kv_heads: u32 = 32, rope_opts: zml.nn.RopeOpts = .{ .layout = .interleaved, - .freq_base = 10_000, + .scaling = .{ .default = .{ .rope_theta = 10_000 } }, }, pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: LlamaLM.Config) !Llama { @@ -331,6 +331,8 @@ pub const SelfAttn = struct { rope_opts: zml.nn.RopeOpts = undefined, pub fn init(store: zml.io.TensorStore.View, config: LlamaLM.Config) !SelfAttn { + var rope_scaling = config.rope_scaling; + rope_scaling.setRopeTheta(config.rope_theta); return .{ .q_proj = .init(store.createTensor("q_proj.weight", .{ .dout, .d }, .{ .dout = .model }), null, .d), .k_proj = .init(store.createTensor("k_proj.weight", .{ .dout, .d }, .{ .dout = .model }), null, .d), @@ -343,8 +345,7 @@ pub const SelfAttn = struct { .num_kv_heads = @intCast(config.num_key_value_heads), .rope_opts = .{ .layout = if (config.hf_rope_impl) .sequential else .interleaved, - .freq_base = config.rope_theta, - .scaling = config.rope_scaling, + .scaling = rope_scaling, }, }; } diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 662466f4..fdd49ef4 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -221,9 +221,7 @@ pub fn main(init: std.process.Init) !void { fn parseConfig(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !std.json.Parsed(LlamaLM.Config) { const now: std.Io.Timestamp = .now(io, .awake); log.info("Loading model config", .{}); - defer log.info("Loaded model config [{D}]", .{ - stdx.fmt.fmtDuration(now.untilNow(io, .awake)), - }); + defer log.info("Loaded model config [{f}]", .{now.untilNow(io, .awake)}); const parsed_config = blk: { const config_json_file = try dir.openFile(io, "config.json", .{}); @@ -245,9 +243,7 @@ fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, prog var node = progress.start("Loading tokenizer...", 1); defer node.end(); const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Loaded tokenizer [{D}]", .{ - stdx.fmt.fmtDuration(now.untilNow(io, .awake)), - }); + defer log.info("Loaded tokenizer [{f}]", .{now.untilNow(io, .awake)}); const bytes = b: { const file = try dir.openFile(io, "tokenizer.json", .{}); defer file.close(io); @@ -284,7 +280,7 @@ fn compileModel( progress: *std.Progress.Node, ) !CompileModelResult { const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Compiled model [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); + defer log.info("Compiled model [{f}]", .{now.untilNow(io, .awake)}); // Compile the model twice, one for prefill, one for generation. var prefill_future = try io.concurrent(struct { @@ -293,7 +289,7 @@ fn compileModel( var node_ = progress_.start("Compiling prefill...", 1); defer node_.end(); const now_: std.Io.Timestamp = .now(io_, .awake); - defer log.info("Compiled prefill [{D}]", .{stdx.fmt.fmtDuration(now_.untilNow(io_, .awake))}); + defer log.info("Compiled prefill [{f}]", .{now_.untilNow(io_, .awake)}); return platform_.compile(allocator_, io_, llama_model_, .forward, .{ parameters_.prefill_tokens, parameters_.token_index, @@ -314,7 +310,7 @@ fn compileModel( var node_ = progress_.start("Compiling decode...", 1); defer node_.end(); const now_: std.Io.Timestamp = .now(io_, .awake); - defer log.info("Compiled decode [{D}]", .{stdx.fmt.fmtDuration(now_.untilNow(io_, .awake))}); + defer log.info("Compiled decode [{f}]", .{now_.untilNow(io_, .awake)}); return platform_.compile(allocator_, io_, llama_model_, .forward, .{ parameters_.decode_tokens, parameters_.token_index, @@ -479,9 +475,9 @@ pub fn generateText( } const duration = now.untilNow(io, .awake); std.debug.print("\n", .{}); - log.info("✅ Generated {} tokens in {D}: {:.3}tok/s", .{ + log.info("✅ Generated {} tokens in {f}: {:.3}tok/s", .{ num_tokens_generated, - stdx.fmt.fmtDuration(duration), + duration, stdx.Io.Duration.hzFloat(stdx.Io.Duration.div(duration, num_tokens_generated)), }); } diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 6ec5cf9e..16488b43 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -101,7 +101,7 @@ pub fn main(init: std.process.Init) !void { var exe = blk: { log.info("Compiling model....", .{}); const start: std.Io.Timestamp = .now(io, .awake); - defer log.info("✅ Compiled model [{D}]", .{stdx.fmt.fmtDuration(start.untilNow(io, .awake))}); + defer log.info("✅ Compiled model [{f}]", .{start.untilNow(io, .awake)}); break :blk try platform.compile(allocator, io, mnist_model, .forward, .{input}, .{ .shardings = &.{replicated_sharding} }); }; defer exe.deinit(); @@ -110,8 +110,8 @@ pub fn main(init: std.process.Init) !void { var mnist_buffers = blk: { log.info("Transfering weights....", .{}); const start: std.Io.Timestamp = .now(io, .awake); - defer log.info("✅ Transferred weights [{D}]", .{ - stdx.fmt.fmtDuration(start.untilNow(io, .awake)), + defer log.info("✅ Transferred weights [{f}]", .{ + start.untilNow(io, .awake), }); break :blk try mnist_model.load(init.arena.allocator(), io, platform, &store, &.{replicated_sharding}); }; diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index a21eec76..127de962 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -1365,7 +1365,9 @@ pub fn serializePortableArtifact2(module: *mlir.Module, target_version: []const if (sctx.err) |err| { return err; } - if (c.mlirLogicalResultIsFailure(result)) { + // TODO(Corentin): Find out why this crashes on macOS + //if (c.mlirLogicalResultIsFailure(result)) { + if (@as(c_int, result.value) == 0) { return error.InvalidMlirBytecodeVersion; } } diff --git a/stdx/fmt.zig b/stdx/fmt.zig index ec3a0343..16dd843a 100644 --- a/stdx/fmt.zig +++ b/stdx/fmt.zig @@ -176,15 +176,3 @@ fn printValue( inline else => try w.printValue("any", options, value, max_depth - 1), } } - -pub const FormatDuration = struct { - inner: std.Io.Duration, - - pub fn formatDuration(self: FormatDuration, w: *std.Io.Writer) std.Io.Writer.Error!void { - try w.printDurationSigned(@intCast(self.inner.toNanoseconds())); - } -}; - -pub fn fmtDuration(ts: std.Io.Duration) FormatDuration { - return .{ .inner = ts }; -} diff --git a/zml/io.zig b/zml/io.zig index 7d2f2d01..c1f87de9 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -1077,18 +1077,11 @@ pub fn load( ) !Bufferized(ModelType) { var bufferized = try mem.bufferize(allocator, ModelType, model); - // todo: remove when https://codeberg.org/ziglang/zig/pulls/31320 is merged - var thread_safe_allocator: std.heap.ThreadSafeAllocator = .{ - .child_allocator = allocator, - .io = io, - }; - const concurrent_allocator = thread_safe_allocator.allocator(); - const pool_count = platform.devices.len; const dma_allocators = try allocator.alloc(mem.DmaAllocator, pool_count); defer allocator.free(dma_allocators); for (platform.devices, 0..) |*device, i| { - dma_allocators[i] = .init(concurrent_allocator, device); + dma_allocators[i] = .init(allocator, device); } const buffer_pools = try allocator.alloc(mem.DynamicBufferPool, pool_count); @@ -1120,7 +1113,7 @@ pub fn load( .platform = platform, .buffers = try allocator.alloc(*Buffer, meta.count(Tensor, model)), .store = opts.store, - .allocator = concurrent_allocator, + .allocator = allocator, .dma_allocators = dma_allocators, .pinned_buffer_pools = buffer_pools, .io = io, diff --git a/zml/io/vfs/base.zig b/zml/io/vfs/base.zig index 041d4689..7042c883 100644 --- a/zml/io/vfs/base.zig +++ b/zml/io/vfs/base.zig @@ -23,7 +23,6 @@ pub const VFSBase = struct { .recancel = recancel, .swapCancelProtection = swapCancelProtection, .checkCancel = checkCancel, - .select = select, .futexWait = futexWait, .futexWaitUncancelable = futexWaitUncancelable, .futexWake = futexWake, @@ -157,12 +156,12 @@ pub const VFSBase = struct { return self.inner.vtable.cancel(self.inner.userdata, any_future, result, result_alignment); } - pub fn groupAsync(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque) std.Io.Cancelable!void) void { + pub fn groupAsync(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque) void) void { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.groupAsync(self.inner.userdata, group, context, context_alignment, start); } - pub fn groupConcurrent(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque) std.Io.Cancelable!void) std.Io.ConcurrentError!void { + pub fn groupConcurrent(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque) void) std.Io.ConcurrentError!void { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.groupConcurrent(self.inner.userdata, group, context, context_alignment, start); } @@ -192,11 +191,6 @@ pub const VFSBase = struct { return self.inner.vtable.checkCancel(self.inner.userdata); } - pub fn select(userdata: ?*anyopaque, futures: []const *std.Io.AnyFuture) std.Io.Cancelable!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.select(self.inner.userdata, futures); - } - pub fn futexWait(userdata: ?*anyopaque, ptr: *const u32, expected: u32, timeout: std.Io.Timeout) std.Io.Cancelable!void { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.futexWait(self.inner.userdata, ptr, expected, timeout); diff --git a/zml/nn.zig b/zml/nn.zig index 4df96114..e5ae129b 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -125,6 +125,7 @@ pub fn normalizeL2(input: Tensor, eps: f32) Tensor { test normalizeL2 { const platform = zml.testing.env(); + const input: zml.Tensor = .init(.{ 2, 2 }, .f32); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, normalizeL2, .{ input, 1e-12 }, platform, .{ .shardings = &.{zml.testing.replicatedSharding()} }); @@ -142,7 +143,6 @@ test normalizeL2 { pub const RopeOpts = struct { layout: Layout = .sequential, - freq_base: f32 = 10_000, scaling: Scaling = .{ .default = .{} }, /// There are two layouts corresponding to how to split `x` in real/imag parts. @@ -179,7 +179,7 @@ pub const RopeOpts = struct { mscale_all_dim: ?f32 = null, truncate: bool = true, original_max_position_embeddings: u32, - rope_theta: f32, + rope_theta: f32 = 10000, llama_4_scaling_beta: ?f32 = null, attention_factor: ?f32 = null, }; @@ -335,7 +335,7 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { const N = inv_freq.len; // Default frequencies for (0.., inv_freq) |n, *f| { - f.* = @exp(-@log(opts.freq_base) * stdx.math.divFloat(f32, n, N)); + f.* = @exp(-@log(opts.scaling.getRopeTheta()) * stdx.math.divFloat(f32, n, N)); } switch (opts.scaling) { @@ -374,8 +374,8 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { const downscaling = 1.0 / s.factor.?; // This isn't a typo: low n have a high frequency, high n have a low frequency. - var n_low: f64 = -@log(f_high) / @log(opts.freq_base) * N_f; - var n_high: f64 = -@log(f_low) / @log(opts.freq_base) * N_f; + var n_low: f64 = -@log(f_high) / @log(opts.scaling.getRopeTheta()) * N_f; + var n_high: f64 = -@log(f_low) / @log(opts.scaling.getRopeTheta()) * N_f; if (s.truncate) { n_high = std.math.ceil(n_high); n_low = std.math.floor(n_low); @@ -400,12 +400,12 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { test "invFreq Llama3" { // Llama 3.2-1B config const llama_conf: RopeOpts = .{ - .freq_base = 500_000, .scaling = .{ .llama3 = .{ .factor = 32, .high_freq_factor = 4, .low_freq_factor = 1, .original_max_position_embeddings = 8192, + .rope_theta = 500_000, } }, }; const llama_freq = [_]f32{ 1.000000000000e+00, 6.636012792587e-01, 4.403666257858e-01, 2.922278344631e-01, 1.939227581024e-01, 1.286873817444e-01, 8.539710193872e-02, 5.666961893439e-02, 3.760603070259e-02, 2.495540864766e-02, 1.656044088304e-02, 1.098952908069e-02, 7.292665075511e-03, 4.839421249926e-03, 3.211446106434e-03, 1.290548010729e-03, 4.295567050576e-04, 9.708286233945e-05, 1.946163865796e-05, 1.291476746701e-05, 8.570255886298e-06, 5.687232260243e-06, 3.774054448513e-06, 2.504467147446e-06, 1.661967417022e-06, 1.102883629756e-06, 7.318749339902e-07, 4.856731266045e-07, 3.222932889457e-07, 2.138742303259e-07, 1.419272024350e-07, 9.418306490261e-08 }; @@ -420,14 +420,13 @@ test "invFreq Llama3" { test "invFreq Yarn" { const yarn_conf: RopeOpts = .{ - .freq_base = 150_000, .scaling = .{ .yarn = .{ .factor = 32.0, .beta_fast = 32.0, .beta_slow = 1.0, .original_max_position_embeddings = 4096, .truncate = true, - .rope_theta = 10000, + .rope_theta = 150_000, } }, }; const yarn_freq = [_]f32{ 1.000000000000e+00, 6.890442967415e-01, 4.747820496559e-01, 3.271458745003e-01, 2.254180014133e-01, 1.553229838610e-01, 1.070244237781e-01, 7.374456524849e-02, 5.081327259541e-02, 3.162075206637e-02, 1.945096626878e-02, 1.179219130427e-02, 7.015713956207e-03, 4.069554619491e-03, 2.277272054926e-03, 1.206130953506e-03, 5.809474969283e-04, 2.279478358105e-04, 3.830881178146e-05, 2.639646845637e-05, 1.818833698053e-05, 1.253256959899e-05, 8.635495760245e-06, 5.950239483354e-06, 4.099978468730e-06, 2.825066758305e-06, 1.946596285052e-06, 1.341290953860e-06, 9.242089618056e-07, 6.368209142238e-07, 4.387978549403e-07, 3.023511396805e-07 }; diff --git a/zml/zml.zig b/zml/zml.zig index b582466f..4be94092 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -55,7 +55,3 @@ pub const KiB = 1024; pub const MiB = 1024 * KiB; pub const GiB = 1024 * MiB; pub const TiB = 1024 * GiB; - -pub fn init(init_: std.process.Init) !void { - try bazel.initSimple(init_); -} From 1b50eb526cba0bba1e6d1999093fa83455b2a64c Mon Sep 17 00:00:00 2001 From: Olivier Boulant Date: Wed, 11 Mar 2026 16:49:58 +0100 Subject: [PATCH 349/881] chore(zls): remove transfer-benchmark from zls completion (#410) This allows the ZLS server to work. Before this fix, `bazel run //:completion.print_build_config` would fail into ```bash ERROR: no such package 'examples/transfer-benchmark': BUILD file not found in any of the following directories. Add a BUILD file to a directory to mark it as a package. - examples/transfer-benchmark ``` --- BUILD.bazel | 1 - 1 file changed, 1 deletion(-) diff --git a/BUILD.bazel b/BUILD.bazel index 872030c6..943fbd3c 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -5,7 +5,6 @@ zls_completion( visibility = ["//visibility:public"], deps = [ "//examples/lfm", - "//examples/transfer-benchmark", "//examples/llama", "//examples/mnist", "//examples/sharding", From 9fef9b85ef513c9407a41290c63f35b0bb183a2b Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 12 Mar 2026 11:01:21 +0000 Subject: [PATCH 350/881] zml/sharding: remove partitionning for shard shape --- zml/sharding.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/sharding.zig b/zml/sharding.zig index af4ad8db..42894f90 100644 --- a/zml/sharding.zig +++ b/zml/sharding.zig @@ -80,7 +80,7 @@ pub const Partitioning = struct { pub fn localShapeForShape(self: Partitioning, shape: Shape) !Shape { const sharding = try self.selectSharding(shape); const placement = try Placement.init(sharding, shape); - return placement.shards.get(0).shape; + return placement.shards.get(0).shape.withDefaultPartitioning(); } pub fn numPartitionsForLogicalAxis(self: Partitioning, shape: Shape, logical_axis: anytype) !i64 { From 9385ffbcfe10bb6c35a29f62e2b3f2cfd559477b Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 17 Mar 2026 11:10:14 +0100 Subject: [PATCH 351/881] workspace: update rules_zig translate-c handling --- MODULE.bazel | 15 +++++++++------ third_party/translate-c/repo.bzl | 2 +- third_party/translate-c/translate-c.bazel | 15 +++------------ translate-c/BUILD.bazel | 17 ----------------- 4 files changed, 13 insertions(+), 36 deletions(-) delete mode 100644 translate-c/BUILD.bazel diff --git a/MODULE.bazel b/MODULE.bazel index dc63e32f..bba9eba4 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -19,15 +19,18 @@ bazel_dep(name = "rules_python", version = "1.8.4") bazel_dep(name = "rules_rs", version = "0.0.41") bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "rules_zig", version = "0.12.2") -bazel_dep(name = "llvm", version = "0.6.1") -bazel_dep(name = "with_cfg.bzl", version = "0.14.1") +bazel_dep(name = "llvm", version = "0.6.8") +bazel_dep(name = "with_cfg.bzl", version = "0.14.4") bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2", dev_dependency = True) ### Zig toolchain zig = use_extension("@rules_zig//zig:extensions.bzl", "zig") zig.index(file = "//bazel:zig_index.json") -zig.toolchain(zig_version = "0.16.0-dev.2722+f16eb18ce") +zig.toolchain( + translate_c = "@translate-c", + zig_version = "0.16.0-dev.2722+f16eb18ce", +) zig.mirrors(urls = [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -102,7 +105,7 @@ use_repo(zls, "zls_toolchains") register_toolchains("@zls_toolchains//:all") non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") -use_repo(non_module_deps, "arocc", "com_google_sentencepiece", "flashattn", "linenoise", "mnist", "org_swig_swig", "translate-c", "xla", "stb") +use_repo(non_module_deps, "arocc", "com_google_sentencepiece", "flashattn", "linenoise", "mnist", "org_swig_swig", "stb", "translate-c", "xla") xla = use_extension("//third_party/xla:xla.bzl", "xla") use_repo( @@ -116,8 +119,8 @@ use_repo( "local_config_tensorrt", "python_version_repo", "rules_ml_toolchain", - "stablehlo", "shardy", + "stablehlo", "sycl_configure", "sycl_configure_ext", "triton", @@ -185,6 +188,6 @@ use_repo(oci, "distroless_cc_debian12", "distroless_cc_debian12_linux_amd64") git_override( module_name = "rules_zig", - commit = "6c5d6a07b4ef48be894e7418bcae37bac54e1c2e", + commit = "dbc3d75d4fce20d998eea343c2cc0fadc51aac0e", remote = "https://github.com/zml/rules_zig.git", ) diff --git a/third_party/translate-c/repo.bzl b/third_party/translate-c/repo.bzl index ab5e9860..46a43d68 100644 --- a/third_party/translate-c/repo.bzl +++ b/third_party/translate-c/repo.bzl @@ -3,7 +3,7 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") def repo(): new_git_repository( name = "translate-c", - remote = "https://codeberg.org/ziglang/translate-c.git", + remote = "git@codeberg.org:ziglang/translate-c.git", commit = "d2be2f19ef7c9caa1561d38c96581ac79dd4c654", build_file = "//third_party/translate-c:translate-c.bazel", ) diff --git a/third_party/translate-c/translate-c.bazel b/third_party/translate-c/translate-c.bazel index f71603e3..84a0e059 100644 --- a/third_party/translate-c/translate-c.bazel +++ b/third_party/translate-c/translate-c.bazel @@ -1,34 +1,25 @@ -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_configure_binary", "zig_library") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") zig_library( name = "c_builtins", main = "lib/c_builtins.zig", - visibility = ["//visibility:public"], ) zig_library( name = "helpers", main = "lib/helpers.zig", - visibility = ["//visibility:public"], ) zig_binary( - name = "translate-c_raw", + name = "translate-c", + data = glob(["lib/**/*.zig"]), srcs = glob(["src/**/*.zig"]), main = "src/main.zig", linkopts = ["-lc"], - visibility = ["//visibility:public"], deps = [ ":c_builtins", ":helpers", "@arocc//:aro", ], -) - -zig_configure_binary( - name = "translate-c", - actual = ":translate-c_raw", - no_translate_c = 1, - use_cc_common_link = 0, visibility = ["//visibility:public"], ) diff --git a/translate-c/BUILD.bazel b/translate-c/BUILD.bazel deleted file mode 100644 index 91f3fc51..00000000 --- a/translate-c/BUILD.bazel +++ /dev/null @@ -1,17 +0,0 @@ -alias( - name = "translate-c", - actual = "@translate-c", - visibility = ["//visibility:public"], -) - -alias( - name = "helpers", - actual = "@translate-c//:helpers", - visibility = ["//visibility:public"], -) - -alias( - name = "c_builtins", - actual = "@translate-c//:c_builtins", - visibility = ["//visibility:public"], -) From 880aeaf101a07c6c97d0801525c37b2f86f3b638 Mon Sep 17 00:00:00 2001 From: Tristan Gautie Date: Tue, 17 Mar 2026 11:29:43 +0100 Subject: [PATCH 352/881] Use http archive for translate-c repo --- MODULE.bazel.lock | 41 ++++++++++++++++++++++---------- third_party/translate-c/repo.bzl | 9 +++---- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 995dd985..18c240b9 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -59,7 +59,8 @@ "https://bcr.bazel.build/modules/bazel_lib/3.0.0-rc.0/MODULE.bazel": "d6e00979a98ac14ada5e31c8794708b41434d461e7e7ca39b59b765e6d233b18", "https://bcr.bazel.build/modules/bazel_lib/3.0.0/MODULE.bazel": "22b70b80ac89ad3f3772526cd9feee2fa412c2b01933fea7ed13238a448d370d", "https://bcr.bazel.build/modules/bazel_lib/3.2.0/MODULE.bazel": "39b50d94b9be6bda507862254e20c263f9b950e3160112348d10a938be9ce2c2", - "https://bcr.bazel.build/modules/bazel_lib/3.2.0/source.json": "a6f45a903134bebbf33a6166dd42b4c7ab45169de094b37a85f348ca41170a84", + "https://bcr.bazel.build/modules/bazel_lib/3.2.2/MODULE.bazel": "e2c890c8a515d6bca9c66d47718aa9e44b458fde64ec7204b8030bf2d349058c", + "https://bcr.bazel.build/modules/bazel_lib/3.2.2/source.json": "9e84e115c20e14652c5c21401ae85ff4daa8702e265b5c0b3bf89353f17aa212", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", @@ -96,8 +97,8 @@ "https://bcr.bazel.build/modules/jsoncpp/1.9.6/source.json": "a04756d367a2126c3541682864ecec52f92cdee80a35735a3cb249ce015ca000", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", "https://bcr.bazel.build/modules/llvm/0.6.0/MODULE.bazel": "42c2182c49f13d2df83a4a4a95ab55d31efda47b2d67acf419bf6b31522b2a30", - "https://bcr.bazel.build/modules/llvm/0.6.1/MODULE.bazel": "29170ab19f4e2dc9b6bbf9b3d101738e84142f63ba29a13cc33e0d40f74c79b0", - "https://bcr.bazel.build/modules/llvm/0.6.1/source.json": "2d8cdd3a5f8e1d16132dbbe97250133101e4863c0376d23273d9afd7363cc331", + "https://bcr.bazel.build/modules/llvm/0.6.8/MODULE.bazel": "53468e4a4be409c2d34e5b7331d2e1fef982151b777655ca3c0047225b333629", + "https://bcr.bazel.build/modules/llvm/0.6.8/source.json": "b673af466f716e01d6243f59e47729e99f37dc5e17026d2bf18c98206f09b6c5", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/source.json": "f448c6e8963fdfa7eb831457df83ad63d3d6355018f6574fb017e8169deb43a9", "https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92", @@ -216,6 +217,7 @@ "https://bcr.bazel.build/modules/rules_python/0.40.0/MODULE.bazel": "9d1a3cd88ed7d8e39583d9ffe56ae8a244f67783ae89b60caafc9f5cf318ada7", "https://bcr.bazel.build/modules/rules_python/1.0.0/MODULE.bazel": "898a3d999c22caa585eb062b600f88654bf92efb204fa346fb55f6f8edffca43", "https://bcr.bazel.build/modules/rules_python/1.6.0/MODULE.bazel": "7e04ad8f8d5bea40451cf80b1bd8262552aa73f841415d20db96b7241bd027d8", + "https://bcr.bazel.build/modules/rules_python/1.7.0/MODULE.bazel": "d01f995ecd137abf30238ad9ce97f8fc3ac57289c8b24bd0bf53324d937a14f8", "https://bcr.bazel.build/modules/rules_python/1.8.4/MODULE.bazel": "33e3971e66161a3e955f7a0d411a8d1f291c4ce4c561851512466f3c77ff8ece", "https://bcr.bazel.build/modules/rules_python/1.8.4/source.json": "9fbc0e57bae52cddcc3831d668bce87a47e0c655104a85098d4459dd9a3b0a10", "https://bcr.bazel.build/modules/rules_rs/0.0.41/MODULE.bazel": "3f4f7e4cf87c7e8f8e32e8e7a49e1fb6caa5529b9b52757ec25d0b4d0272e380", @@ -243,11 +245,13 @@ "https://bcr.bazel.build/modules/tar.bzl/0.5.1/MODULE.bazel": "7c2eb3dcfc53b0f3d6f9acdfd911ca803eaf92aadf54f8ca6e4c1f3aee288351", "https://bcr.bazel.build/modules/tar.bzl/0.5.5/MODULE.bazel": "4bfab9bbc7a1966c2c5f7371f5848f5e2d27c465951b4435adc9aaf00ed681da", "https://bcr.bazel.build/modules/tar.bzl/0.6.0/MODULE.bazel": "a3584b4edcfafcabd9b0ef9819808f05b372957bbdff41601429d5fd0aac2e7c", - "https://bcr.bazel.build/modules/tar.bzl/0.6.0/source.json": "4a620381df075a16cb3a7ed57bd1d05f7480222394c64a20fa51bdb636fda658", + "https://bcr.bazel.build/modules/tar.bzl/0.9.0/MODULE.bazel": "452a22d7f02b1c9d7a22ab25edf20f46f3e1101f0f67dc4bfbf9a474ddf02445", + "https://bcr.bazel.build/modules/tar.bzl/0.9.0/source.json": "c732760a374831a2cf5b08839e4be75017196b4d796a5aa55235272ee17cd839", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", "https://bcr.bazel.build/modules/with_cfg.bzl/0.12.0/MODULE.bazel": "b573395fe63aef4299ba095173e2f62ccfee5ad9bbf7acaa95dba73af9fc2b38", "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.1/MODULE.bazel": "aa0ef3f6c67dd35db7ac38c76785dc02f48b53f34e16b20487e70f51b5324d0a", - "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.1/source.json": "4666d3035f69063ecd136f3f0c68e424f1b1dd97ffc9cceb679522c0be38761b", + "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.4/MODULE.bazel": "be2276f83878525d8f228b9ec777daee5eb7baba6100dad45081510d0cec35e3", + "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.4/source.json": "63d661a0c144ca74fd8b0aa61e458c7627554c7d4a8e0a4d1ce00975437c776a", "https://bcr.bazel.build/modules/yq.bzl/0.1.1/MODULE.bazel": "9039681f9bcb8958ee2c87ffc74bdafba9f4369096a2b5634b88abc0eaefa072", "https://bcr.bazel.build/modules/yq.bzl/0.3.1/MODULE.bazel": "9bcb7151b3cd4681b89d350530eaf7b45e32a44dda94843b8932b0cb1cd4594a", "https://bcr.bazel.build/modules/yq.bzl/0.3.1/source.json": "f0b0f204a2a6b0e34b4c9541efe8c04f2ef1af65948daa784eccea738b21dbd2", @@ -1021,8 +1025,8 @@ }, "@@rules_zig+//zig:extensions.bzl%zig": { "general": { - "bzlTransitiveDigest": "p1+tPOh3UlhcIaGGf1ycZCCC6I4NIHhiiHS0nAhhgHA=", - "usagesDigest": "aLO2dhznavzbCoReZv2wJRz7GoTgcrHljpLeLgEjBkY=", + "bzlTransitiveDigest": "r7tCqbjlVKc7XEXV8Hiak2ElFfkjvDdMf8Ym+7zxukU=", + "usagesDigest": "lqvJAPUaAfUjthqXXAa4LwhcuZ2d3CNerCqKcLkkNPo=", "recordedFileInputs": { "@@//bazel/zig_index.json": "809b545c21f13c11c4587dccbfe81de2b767a292325c25ffb785b03c601bbc2e", "@@rules_zig+//zig/private/versions.json": "6eb85ebaee72c4e6fbc82d8ad73f69e0899d839b70a7e19caa792ab6a5fa0c36" @@ -1052,7 +1056,8 @@ ], "sha256": "03725d297a2e2a8331d774eb601163af42ec61979b82753d41b2c460eefab1be", "zig_version": "0.16.0-dev.2722+f16eb18ce", - "platform": "aarch64-linux" + "platform": "aarch64-linux", + "translate_c": "@@+non_module_deps+translate-c//:translate-c" } }, "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-macos": { @@ -1077,7 +1082,8 @@ ], "sha256": "268aa8342e8ef539693e88e1d0982361b1d5dc4a080949264f2d5a14e5eb83e0", "zig_version": "0.16.0-dev.2722+f16eb18ce", - "platform": "aarch64-macos" + "platform": "aarch64-macos", + "translate_c": "@@+non_module_deps+translate-c//:translate-c" } }, "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-windows": { @@ -1102,7 +1108,8 @@ ], "sha256": "43daaad311bc3b06a733a8a87d2c3180214d62db2067df1591b921826de453ea", "zig_version": "0.16.0-dev.2722+f16eb18ce", - "platform": "aarch64-windows" + "platform": "aarch64-windows", + "translate_c": "@@+non_module_deps+translate-c//:translate-c" } }, "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-linux": { @@ -1127,7 +1134,8 @@ ], "sha256": "b0fe8fd996b4974cd80fc0967460b13e3a6e3105c4bec19ea0566bd47d73ad81", "zig_version": "0.16.0-dev.2722+f16eb18ce", - "platform": "x86_64-linux" + "platform": "x86_64-linux", + "translate_c": "@@+non_module_deps+translate-c//:translate-c" } }, "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-macos": { @@ -1152,7 +1160,8 @@ ], "sha256": "8e09fb9e90c0566629dfa52bce433e886f4cf7e2cd9985a358d7b8c473c4e9ce", "zig_version": "0.16.0-dev.2722+f16eb18ce", - "platform": "x86_64-macos" + "platform": "x86_64-macos", + "translate_c": "@@+non_module_deps+translate-c//:translate-c" } }, "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-windows": { @@ -1177,7 +1186,8 @@ ], "sha256": "0f92e7234ed1d9608444487ca8fb928bd38da79dea744acf6c7a8fa0c440da6a", "zig_version": "0.16.0-dev.2722+f16eb18ce", - "platform": "x86_64-windows" + "platform": "x86_64-windows", + "translate_c": "@@+non_module_deps+translate-c//:translate-c" } }, "zig_toolchains": { @@ -1233,6 +1243,11 @@ } }, "recordedRepoMappingEntries": [ + [ + "", + "translate-c", + "+non_module_deps+translate-c" + ], [ "rules_zig+", "bazel_skylib", diff --git a/third_party/translate-c/repo.bzl b/third_party/translate-c/repo.bzl index 46a43d68..2f502b95 100644 --- a/third_party/translate-c/repo.bzl +++ b/third_party/translate-c/repo.bzl @@ -1,9 +1,10 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") def repo(): - new_git_repository( + http_archive( name = "translate-c", - remote = "git@codeberg.org:ziglang/translate-c.git", - commit = "d2be2f19ef7c9caa1561d38c96581ac79dd4c654", build_file = "//third_party/translate-c:translate-c.bazel", + sha256 = "506643ec817620025d341efd87169d90013aa369db28780e853cf18bff174326", + url = "https://codeberg.org/ziglang/translate-c/archive/d2be2f19ef7c9caa1561d38c96581ac79dd4c654.tar.gz", + strip_prefix = "translate-c", ) From ba8134f4a22601f09cddebbcc66abac252fb14c8 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 17 Mar 2026 13:58:31 +0100 Subject: [PATCH 353/881] stdx/json: consistent serialization of Union, TaggedUnion and NeverNull (#412) --- devenv.lock | 80 ++++----------------------- stdx/json.zig | 146 +++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 150 insertions(+), 76 deletions(-) diff --git a/devenv.lock b/devenv.lock index 78eabc08..ccdf5c5d 100644 --- a/devenv.lock +++ b/devenv.lock @@ -3,10 +3,11 @@ "devenv": { "locked": { "dir": "src/modules", - "lastModified": 1772143911, + "lastModified": 1773672671, + "narHash": "sha256-TcU3EYJ6vVNRE6Kdhwh7wI3X6UTqyX+K84PnP03YK7Y=", "owner": "cachix", "repo": "devenv", - "rev": "83c4b5682b84fe04c049970792f38d5879d3c0f1", + "rev": "9dfa16743f60733f62634ae2826c8c8a4762e255", "type": "github" }, "original": { @@ -16,71 +17,16 @@ "type": "github" } }, - "flake-compat": { - "flake": false, - "locked": { - "lastModified": 1767039857, - "owner": "NixOS", - "repo": "flake-compat", - "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", - "type": "github" - }, - "original": { - "owner": "NixOS", - "repo": "flake-compat", - "type": "github" - } - }, - "git-hooks": { - "inputs": { - "flake-compat": "flake-compat", - "gitignore": "gitignore", - "nixpkgs": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1772024342, - "owner": "cachix", - "repo": "git-hooks.nix", - "rev": "6e34e97ed9788b17796ee43ccdbaf871a5c2b476", - "type": "github" - }, - "original": { - "owner": "cachix", - "repo": "git-hooks.nix", - "type": "github" - } - }, - "gitignore": { - "inputs": { - "nixpkgs": [ - "git-hooks", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1762808025, - "owner": "hercules-ci", - "repo": "gitignore.nix", - "rev": "cb5e3fdca1de58ccbc3ef53de65bd372b48f567c", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "gitignore.nix", - "type": "github" - } - }, "nixpkgs": { "inputs": { "nixpkgs-src": "nixpkgs-src" }, "locked": { - "lastModified": 1770434727, + "lastModified": 1772749504, + "narHash": "sha256-eqtQIz0alxkQPym+Zh/33gdDjkkch9o6eHnMPnXFXN0=", "owner": "cachix", "repo": "devenv-nixpkgs", - "rev": "8430f16a39c27bdeef236f1eeb56f0b51b33d348", + "rev": "08543693199362c1fddb8f52126030d0d374ba2e", "type": "github" }, "original": { @@ -93,11 +39,11 @@ "nixpkgs-src": { "flake": false, "locked": { - "lastModified": 1769922788, - "narHash": "sha256-H3AfG4ObMDTkTJYkd8cz1/RbY9LatN5Mk4UF48VuSXc=", + "lastModified": 1772173633, + "narHash": "sha256-MOH58F4AIbCkh6qlQcwMycyk5SWvsqnS/TCfnqDlpj4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "207d15f1a6603226e1e223dc79ac29c7846da32e", + "rev": "c0f3d81a7ddbc2b1332be0d8481a672b4f6004d6", "type": "github" }, "original": { @@ -110,14 +56,10 @@ "root": { "inputs": { "devenv": "devenv", - "git-hooks": "git-hooks", - "nixpkgs": "nixpkgs", - "pre-commit-hooks": [ - "git-hooks" - ] + "nixpkgs": "nixpkgs" } } }, "root": "root", "version": 7 -} +} \ No newline at end of file diff --git a/stdx/json.zig b/stdx/json.zig index c99cfb00..a8bff835 100644 --- a/stdx/json.zig +++ b/stdx/json.zig @@ -62,6 +62,12 @@ pub fn Union(comptime T: type) type { } return error.UnknownField; } + + pub fn jsonStringify(self: Self, jw: anytype) !void { + return switch (self.value) { + inline else => |v| jw.write(v), + }; + } }; } @@ -88,6 +94,17 @@ pub fn Union(comptime T: type) type { /// const Message = []const stdx.json.TaggedUnion(Entry, "type"); /// ``` pub fn TaggedUnion(comptime T: type, comptime tag_name: [:0]const u8) type { + comptime { + if (@typeInfo(T) != .@"union") { + @compileError("TaggedUnion expects a union type, found " ++ @typeName(T)); + } + for (std.meta.fields(T)) |field| { + if (@typeInfo(field.type) != .@"struct") { + @compileError("TaggedUnion member '" ++ field.name ++ "' must be a struct, found " ++ @typeName(field.type)); + } + } + } + return struct { const Self = @This(); @@ -104,16 +121,12 @@ pub fn TaggedUnion(comptime T: type, comptime tag_name: [:0]const u8) type { pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!Self { errdefer std.log.warn("failed to parse: {} as {s}", .{ source, @typeName(T) }); if (source != .object) return error.UnexpectedToken; - const o = source.object; - const tag = o.get(tag_name) orelse return error.MissingField; - for (o.keys(), o.values()) |k, v| { - std.log.warn("object['{s}'] = {}", .{ k, v }); - } + var o = source.object; + const tag = (o.fetchSwapRemove(tag_name) orelse return error.MissingField).value; if (tag != .string) return error.LengthMismatch; inline for (std.meta.fields(T)) |field| { if (std.mem.eql(u8, field.name, tag.string)) { - const inner_source = o.get(field.name) orelse return error.MissingField; - const inner: field.type = std.json.innerParseFromValue(field.type, allocator, inner_source, options) catch |err| { + const inner: field.type = std.json.parseFromValueLeaky(field.type, allocator, .{ .object = o }, options) catch |err| { std.log.warn("failed to interpret {s} as a {s}: {}", .{ tag.string, @typeName(field.type), err }); return err; }; @@ -122,6 +135,25 @@ pub fn TaggedUnion(comptime T: type, comptime tag_name: [:0]const u8) type { } return error.InvalidEnumTag; } + + pub fn jsonStringify(self: Self, jw: anytype) !void { + try jw.beginObject(); + switch (self.value) { + inline else => |v, tag| { + const field_name = @tagName(tag); + try jw.objectField(tag_name); + try jw.write(field_name); + switch (@typeInfo(@TypeOf(v))) { + .@"struct" => inline for (std.meta.fields(@TypeOf(v))) |field| { + try jw.objectField(field.name); + try jw.write(@field(v, field.name)); + }, + else => unreachable, // Would have failed with comptime check. + } + }, + } + try jw.endObject(); + } }; } @@ -138,6 +170,10 @@ pub fn NeverNull(comptime T: type, comptime default_value: T) type { pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !Self { return .{ .value = (try std.json.innerParseFromValue(?T, allocator, source, options)) orelse default_value }; } + + pub fn jsonStringify(self: Self, jw: anytype) !void { + try jw.write(self.value); + } }; } @@ -151,3 +187,99 @@ pub fn fillDefaultStructValues(comptime T: type, r: *T) !void { } } } + +test "union" { + const Answer = union(enum) { + number: i32, + numbers: []const i32, + flag: bool, + ratio: f64, + text: []const u8, + }; + + const Message = struct { + question: []const u8, + answer: Union(Answer), + }; + + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + + const number_case = "{\"question\":\"How old are you?\",\"answer\":5}"; + const parsed_number = try std.json.parseFromSliceLeaky(Message, allocator, number_case, .{}); + try std.testing.expectEqualStrings("How old are you?", parsed_number.question); + try std.testing.expectEqual(@as(i32, 5), parsed_number.answer.value.number); + try std.testing.expectFmt(number_case, "{f}", .{std.json.fmt(parsed_number, .{})}); + + const numbers_case = "{\"question\":\"Count to three.\",\"answer\":[1,2,3]}"; + const parsed_numbers = try std.json.parseFromSliceLeaky(Message, allocator, numbers_case, .{}); + try std.testing.expectEqual(@as(usize, 3), parsed_numbers.answer.value.numbers.len); + try std.testing.expectEqual(@as(i32, 1), parsed_numbers.answer.value.numbers[0]); + try std.testing.expectEqual(@as(i32, 2), parsed_numbers.answer.value.numbers[1]); + try std.testing.expectEqual(@as(i32, 3), parsed_numbers.answer.value.numbers[2]); + try std.testing.expectFmt(numbers_case, "{f}", .{std.json.fmt(parsed_numbers, .{})}); + + const flag_case = "{\"question\":\"Are you ready?\",\"answer\":true}"; + const parsed_flag = try std.json.parseFromSliceLeaky(Message, allocator, flag_case, .{}); + try std.testing.expectEqual(true, parsed_flag.answer.value.flag); + try std.testing.expectFmt(flag_case, "{f}", .{std.json.fmt(parsed_flag, .{})}); + + const ratio_case = "{\"question\":\"What is pi?\",\"answer\":3.5}"; + const parsed_ratio = try std.json.parseFromSliceLeaky(Message, allocator, ratio_case, .{}); + try std.testing.expectEqual(@as(f64, 3.5), parsed_ratio.answer.value.ratio); + try std.testing.expectFmt(ratio_case, "{f}", .{std.json.fmt(parsed_ratio, .{})}); + + const text_case = "{\"question\":\"Your name?\",\"answer\":\"zml\"}"; + const parsed_text = try std.json.parseFromSliceLeaky(Message, allocator, text_case, .{}); + try std.testing.expectEqualStrings("zml", parsed_text.answer.value.text); + try std.testing.expectFmt(text_case, "{f}", .{std.json.fmt(parsed_text, .{})}); + + const null_value = try std.json.parseFromSliceLeaky(std.json.Value, allocator, "null", .{}); + try std.testing.expectError(error.UnknownField, Union(Answer).jsonParseFromValue(allocator, null_value, .{})); +} + +test "tagged union" { + const Entry = union(enum) { + faq: struct { + question: []const u8, + answer: u32, + }, + address: struct { + city: []const u8, + zipcode: []const u8, + }, + }; + + const Message = TaggedUnion(Entry, "type"); + + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + + const faq_case = "{\"type\":\"faq\",\"question\":\"How old are you?\",\"answer\":5}"; + const parsed_faq = try std.json.parseFromSliceLeaky(Message, allocator, faq_case, .{}); + try std.testing.expectEqualStrings("How old are you?", parsed_faq.value.faq.question); + try std.testing.expectEqual(@as(u32, 5), parsed_faq.value.faq.answer); + try std.testing.expectFmt(faq_case, "{f}", .{std.json.fmt(parsed_faq, .{})}); + + const address_case = "{\"type\":\"address\",\"city\":\"NYC\",\"zipcode\":\"49130\"}"; + const parsed_address = try std.json.parseFromSliceLeaky(Message, allocator, address_case, .{}); + try std.testing.expectEqualStrings("NYC", parsed_address.value.address.city); + try std.testing.expectEqualStrings("49130", parsed_address.value.address.zipcode); + try std.testing.expectFmt(address_case, "{f}", .{std.json.fmt(parsed_address, .{})}); +} + +test "never null" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + + const non_null = try std.json.parseFromSliceLeaky(NeverNull(i32, 10), allocator, "3", .{}); + try std.testing.expectEqual(@as(i32, 3), non_null.value); + try std.testing.expectFmt("3", "{f}", .{std.json.fmt(non_null, .{})}); + + const was_null = try std.json.parseFromSliceLeaky(NeverNull(i32, 10), allocator, "null", .{}); + try std.testing.expectEqual(@as(i32, 10), was_null.value); + try std.testing.expectFmt("10", "{f}", .{std.json.fmt(was_null, .{})}); +} From 94228368844b3aac69c6604e14b921c8d3637f9b Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 17 Mar 2026 18:06:31 +0100 Subject: [PATCH 354/881] third_party/zls: Remove absent attributes from latest rules_zig (#414) --- third_party/zls/zls_write_build_config.bzl | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/third_party/zls/zls_write_build_config.bzl b/third_party/zls/zls_write_build_config.bzl index d7260de2..3f3ed65a 100644 --- a/third_party/zls/zls_write_build_config.bzl +++ b/third_party/zls/zls_write_build_config.bzl @@ -156,17 +156,6 @@ zls_write_build_config = rule( default = ":zls.completion.tpl", allow_single_file = True, ), - "_translate_c": attr.label( - default = Label("@rules_zig//zig/private/common:translate-c"), - cfg = "exec", - executable = True, - ), - "_c_helpers": attr.label( - default = Label("@rules_zig//zig/private/common:helpers"), - ), - "_c_builtins": attr.label( - default = Label("@rules_zig//zig/private/common:c_builtins"), - ), }, toolchains = [ "@rules_zig//zig:toolchain_type", From 619fea8b1825610d3f09afe438ea6f90f4d24b03 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 17 Mar 2026 18:14:24 +0100 Subject: [PATCH 355/881] zml: introduce profiler (#413) Move profiling behind a zml-owned primitive instead of exposing PJRT details in user code. Add the XLA/TSL Bazel deps needed to convert collected XSpace protobufs into Perfetto traces with XLA/TLS APIs instead of having a custom fallible converter. --- MODULE.bazel | 17 +- MODULE.bazel.lock | 13 +- examples/sharding/main.zig | 19 +- .../com_google_sentencepiece/BUILD.bazel | 14 +- third_party/xla/BUILD.bazel | 1 + third_party/xla/repo.bzl | 2 + third_party/xla/xla.bzl | 78 +- third_party/xla/xspace_to_perfetto.patch | 24 + tools/BUILD.bazel | 22 + tools/xprof/BUILD.bazel | 19 + tools/xprof/main.py | 29 + tools/xprof/requirements.in | 1 + tools/xprof/requirements.lock.txt | 1335 +++++++++++++++++ tools/xspace_to_perfetto.cc | 33 + tools/xspace_to_perfetto.h | 15 + tools/xspace_to_perfetto_lib.cc | 125 ++ zml/BUILD.bazel | 3 + zml/platform.zig | 23 +- zml/profiler.zig | 224 +++ 19 files changed, 1956 insertions(+), 41 deletions(-) create mode 100644 third_party/xla/xspace_to_perfetto.patch create mode 100644 tools/xprof/BUILD.bazel create mode 100644 tools/xprof/main.py create mode 100644 tools/xprof/requirements.in create mode 100644 tools/xprof/requirements.lock.txt create mode 100644 tools/xspace_to_perfetto.cc create mode 100644 tools/xspace_to_perfetto.h create mode 100644 tools/xspace_to_perfetto_lib.cc create mode 100644 zml/profiler.zig diff --git a/MODULE.bazel b/MODULE.bazel index bba9eba4..68549aff 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -2,7 +2,7 @@ module( name = "zml", ) -bazel_dep(name = "abseil-cpp", version = "20260107.1") +bazel_dep(name = "abseil-cpp", version = "20260107.1", repo_name = "com_google_absl") bazel_dep(name = "aspect_bazel_lib", version = "2.22.5") bazel_dep(name = "aspect_rules_py", version = "1.8.4") bazel_dep(name = "bazel_skylib", version = "1.9.0") @@ -10,6 +10,8 @@ bazel_dep(name = "patchelf", version = "0.18.0.bcr.1") bazel_dep(name = "pcre2", version = "10.45") bazel_dep(name = "platforms", version = "1.0.0") bazel_dep(name = "protobuf", version = "33.1", repo_name = "com_google_protobuf") +bazel_dep(name = "pybind11_bazel", version = "2.13.6") +bazel_dep(name = "re2", version = "2025-11-05", repo_name = "com_googlesource_code_re2") bazel_dep(name = "rules_cc", version = "0.2.16") bazel_dep(name = "rules_distroless", version = "0.6.2") bazel_dep(name = "rules_license", version = "1.0.0") @@ -20,6 +22,7 @@ bazel_dep(name = "rules_rs", version = "0.0.41") bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "rules_zig", version = "0.12.2") bazel_dep(name = "llvm", version = "0.6.8") +bazel_dep(name = "jsoncpp", version = "1.9.6", repo_name = "jsoncpp_git") bazel_dep(name = "with_cfg.bzl", version = "0.14.4") bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2", dev_dependency = True) @@ -84,6 +87,14 @@ pip.parse( ) use_repo(pip, "huggingface_hub") +pip.parse( + download_only = True, + hub_name = "xprof_nightly", + python_version = "3.12", + requirements_lock = "//tools/xprof:requirements.lock.txt", +) +use_repo(pip, "xprof_nightly") + cpu = use_extension("//platforms/cpu:cpu.bzl", "cpu_pjrt_plugin") use_repo(cpu, "libpjrt_cpu_darwin_amd64", "libpjrt_cpu_darwin_arm64", "libpjrt_cpu_linux_amd64") @@ -111,12 +122,16 @@ xla = use_extension("//third_party/xla:xla.bzl", "xla") use_repo( xla, "com_github_grpc_grpc", + "eigen_archive", + "farmhash_archive", + "farmhash_gpu_archive", "llvm-raw", "local_config_cuda", "local_config_remote_execution", "local_config_rocm", "local_config_sycl", "local_config_tensorrt", + "ml_dtypes_py", "python_version_repo", "rules_ml_toolchain", "shardy", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 18c240b9..c6cd9cd5 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -119,6 +119,8 @@ "https://bcr.bazel.build/modules/platforms/1.0.0/MODULE.bazel": "f05feb42b48f1b3c225e4ccf351f367be0371411a803198ec34a389fb22aa580", "https://bcr.bazel.build/modules/platforms/1.0.0/source.json": "f4ff1fd412e0246fd38c82328eb209130ead81d62dcd5a9e40910f867f733d96", "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7", + "https://bcr.bazel.build/modules/protobuf/23.1/MODULE.bazel": "88b393b3eb4101d18129e5db51847cd40a5517a53e81216144a8c32dfeeca52a", + "https://bcr.bazel.build/modules/protobuf/24.4/MODULE.bazel": "7bc7ce5f2abf36b3b7b7c8218d3acdebb9426aeb35c2257c96445756f970eb12", "https://bcr.bazel.build/modules/protobuf/27.0/MODULE.bazel": "7873b60be88844a0a1d8f80b9d5d20cfbd8495a689b8763e76c6372998d3f64c", "https://bcr.bazel.build/modules/protobuf/27.1/MODULE.bazel": "703a7b614728bb06647f965264967a8ef1c39e09e8f167b3ca0bb1fd80449c0d", "https://bcr.bazel.build/modules/protobuf/29.0-rc2/MODULE.bazel": "6241d35983510143049943fc0d57937937122baf1b287862f9dc8590fc4c37df", @@ -130,11 +132,13 @@ "https://bcr.bazel.build/modules/protobuf/33.1/source.json": "992c237a40899425648213bf79b05f08c6e8dcd619f96cd944b4511b0276fbd8", "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e", "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/MODULE.bazel": "e6f4c20442eaa7c90d7190d8dc539d0ab422f95c65a57cc59562170c58ae3d34", - "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/source.json": "6900fdc8a9e95866b8c0d4ad4aba4d4236317b5c1cd04c502df3f0d33afed680", + "https://bcr.bazel.build/modules/pybind11_bazel/2.13.6/MODULE.bazel": "2d746fda559464b253b2b2e6073cb51643a2ac79009ca02100ebbc44b4548656", + "https://bcr.bazel.build/modules/pybind11_bazel/2.13.6/source.json": "6aa0703de8efb20cc897bbdbeb928582ee7beaf278bcd001ac253e1605bddfae", "https://bcr.bazel.build/modules/re2/2023-09-01/MODULE.bazel": "cb3d511531b16cfc78a225a9e2136007a48cf8a677e4264baeab57fe78a80206", "https://bcr.bazel.build/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "b4963dda9b31080be1905ef085ecd7dd6cd47c05c79b9cdf83ade83ab2ab271a", - "https://bcr.bazel.build/modules/re2/2024-07-02.bcr.1/source.json": "2ff292be6ef3340325ce8a045ecc326e92cbfab47c7cbab4bd85d28971b97ac4", "https://bcr.bazel.build/modules/re2/2024-07-02/MODULE.bazel": "0eadc4395959969297cbcf31a249ff457f2f1d456228c67719480205aa306daa", + "https://bcr.bazel.build/modules/re2/2025-11-05/MODULE.bazel": "d97e240364f132017500a3af816493af7efd4be40aab2bb403ed449cc96f70a3", + "https://bcr.bazel.build/modules/re2/2025-11-05/source.json": "7df1394aabda1c9bc188a302f5d54b1c657924edd04ebc57d2be29dbd7efd141", "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", "https://bcr.bazel.build/modules/rules_apple/3.16.0/MODULE.bazel": "0d1caf0b8375942ce98ea944be754a18874041e4e0459401d925577624d3a54a", @@ -151,6 +155,7 @@ "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", "https://bcr.bazel.build/modules/rules_cc/0.1.1/MODULE.bazel": "2f0222a6f229f0bf44cd711dc13c858dad98c62d52bd51d8fc3a764a83125513", + "https://bcr.bazel.build/modules/rules_cc/0.1.4/MODULE.bazel": "bb03a452a7527ac25a7518fb86a946ef63df860b9657d8323a0c50f8504fb0b9", "https://bcr.bazel.build/modules/rules_cc/0.1.5/MODULE.bazel": "88dfc9361e8b5ae1008ac38f7cdfd45ad738e4fa676a3ad67d19204f045a1fd8", "https://bcr.bazel.build/modules/rules_cc/0.2.15/MODULE.bazel": "6a0a4a75a57aa6dc888300d848053a58c6b12a29f89d4304e1c41448514ec6e8", "https://bcr.bazel.build/modules/rules_cc/0.2.16/MODULE.bazel": "9242fa89f950c6ef7702801ab53922e99c69b02310c39fb6e62b2bd30df2a1d4", @@ -168,6 +173,7 @@ "https://bcr.bazel.build/modules/rules_java/6.3.0/MODULE.bazel": "a97c7678c19f236a956ad260d59c86e10a463badb7eb2eda787490f4c969b963", "https://bcr.bazel.build/modules/rules_java/6.4.0/MODULE.bazel": "e986a9fe25aeaa84ac17ca093ef13a4637f6107375f64667a15999f77db6c8f6", "https://bcr.bazel.build/modules/rules_java/6.5.2/MODULE.bazel": "1d440d262d0e08453fa0c4d8f699ba81609ed0e9a9a0f02cd10b3e7942e61e31", + "https://bcr.bazel.build/modules/rules_java/7.1.0/MODULE.bazel": "30d9135a2b6561c761bd67bd4990da591e6bdc128790ce3e7afd6a3558b2fb64", "https://bcr.bazel.build/modules/rules_java/7.10.0/MODULE.bazel": "530c3beb3067e870561739f1144329a21c851ff771cd752a49e06e3dc9c2e71a", "https://bcr.bazel.build/modules/rules_java/7.12.2/MODULE.bazel": "579c505165ee757a4280ef83cda0150eea193eed3bef50b1004ba88b99da6de6", "https://bcr.bazel.build/modules/rules_java/7.2.0/MODULE.bazel": "06c0334c9be61e6cef2c8c84a7800cef502063269a5af25ceb100b192453d4ab", @@ -213,9 +219,11 @@ "https://bcr.bazel.build/modules/rules_python/0.28.0/MODULE.bazel": "cba2573d870babc976664a912539b320cbaa7114cd3e8f053c720171cde331ed", "https://bcr.bazel.build/modules/rules_python/0.31.0/MODULE.bazel": "93a43dc47ee570e6ec9f5779b2e64c1476a6ce921c48cc9a1678a91dd5f8fd58", "https://bcr.bazel.build/modules/rules_python/0.33.2/MODULE.bazel": "3e036c4ad8d804a4dad897d333d8dce200d943df4827cb849840055be8d2e937", + "https://bcr.bazel.build/modules/rules_python/0.34.0/MODULE.bazel": "1d623d026e075b78c9fde483a889cda7996f5da4f36dffb24c246ab30f06513a", "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", "https://bcr.bazel.build/modules/rules_python/0.40.0/MODULE.bazel": "9d1a3cd88ed7d8e39583d9ffe56ae8a244f67783ae89b60caafc9f5cf318ada7", "https://bcr.bazel.build/modules/rules_python/1.0.0/MODULE.bazel": "898a3d999c22caa585eb062b600f88654bf92efb204fa346fb55f6f8edffca43", + "https://bcr.bazel.build/modules/rules_python/1.5.1/MODULE.bazel": "acfe65880942d44a69129d4c5c3122d57baaf3edf58ae5a6bd4edea114906bf5", "https://bcr.bazel.build/modules/rules_python/1.6.0/MODULE.bazel": "7e04ad8f8d5bea40451cf80b1bd8262552aa73f841415d20db96b7241bd027d8", "https://bcr.bazel.build/modules/rules_python/1.7.0/MODULE.bazel": "d01f995ecd137abf30238ad9ce97f8fc3ac57289c8b24bd0bf53324d937a14f8", "https://bcr.bazel.build/modules/rules_python/1.8.4/MODULE.bazel": "33e3971e66161a3e955f7a0d411a8d1f291c4ce4c561851512466f3c77ff8ece", @@ -248,6 +256,7 @@ "https://bcr.bazel.build/modules/tar.bzl/0.9.0/MODULE.bazel": "452a22d7f02b1c9d7a22ab25edf20f46f3e1101f0f67dc4bfbf9a474ddf02445", "https://bcr.bazel.build/modules/tar.bzl/0.9.0/source.json": "c732760a374831a2cf5b08839e4be75017196b4d796a5aa55235272ee17cd839", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", + "https://bcr.bazel.build/modules/upb/0.0.0-20230516-61a97ef/MODULE.bazel": "c0df5e35ad55e264160417fd0875932ee3c9dda63d9fccace35ac62f45e1b6f9", "https://bcr.bazel.build/modules/with_cfg.bzl/0.12.0/MODULE.bazel": "b573395fe63aef4299ba095173e2f62ccfee5ad9bbf7acaa95dba73af9fc2b38", "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.1/MODULE.bazel": "aa0ef3f6c67dd35db7ac38c76785dc02f48b53f34e16b20487e70f51b5324d0a", "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.4/MODULE.bazel": "be2276f83878525d8f228b9ec777daee5eb7baba6100dad45081510d0cec35e3", diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index 7e674b5b..cfa67892 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -171,18 +171,17 @@ pub fn main(init: std.process.Init) !void { log.info("\n{f}", .{platform.fmtVerbose()}); - var profiler = try platform.profiler(init.arena.allocator()); - if (profiler) |*p| { - try p.start(); - } + var profiler_options: zml.Platform.ProfilerOptions = .defaults; + profiler_options.repository_path = "/tmp/xprof"; + profiler_options.session_id = "profiling"; + var profiler = try platform.profiler(allocator, io, profiler_options); + defer profiler.deinit(); + + try profiler.start(); defer { - if (profiler) |*p| { - p.stop() catch {}; - const pb = p.collectData(allocator) catch unreachable; - const file: std.Io.File = std.Io.Dir.createFile(.cwd(), io, "/tmp/profiling.pb", .{}) catch unreachable; - file.writePositionalAll(io, pb, 0) catch unreachable; - allocator.free(pb); + if ((profiler.stop() catch unreachable)) |profile| { + log.info("Profile dumped: {s} and {s}", .{ profile.protobuf_path, profile.perfetto_path }); } } diff --git a/third_party/com_google_sentencepiece/BUILD.bazel b/third_party/com_google_sentencepiece/BUILD.bazel index 53e12d3a..598dce0b 100644 --- a/third_party/com_google_sentencepiece/BUILD.bazel +++ b/third_party/com_google_sentencepiece/BUILD.bazel @@ -28,12 +28,12 @@ cc_library( includes = ["."], visibility = ["//visibility:public"], deps = [ - "@abseil-cpp//absl/container:flat_hash_map", - "@abseil-cpp//absl/container:flat_hash_set", - "@abseil-cpp//absl/flags:flag", - "@abseil-cpp//absl/flags:parse", - "@abseil-cpp//absl/flags:usage", - "@abseil-cpp//absl/strings", - "@abseil-cpp//absl/strings:string_view", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/flags:parse", + "@com_google_absl//absl/flags:usage", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", ], ) diff --git a/third_party/xla/BUILD.bazel b/third_party/xla/BUILD.bazel index e69de29b..8de71230 100644 --- a/third_party/xla/BUILD.bazel +++ b/third_party/xla/BUILD.bazel @@ -0,0 +1 @@ +exports_files(["xspace_to_perfetto.patch"]) diff --git a/third_party/xla/repo.bzl b/third_party/xla/repo.bzl index 67740474..7f3f0c1c 100644 --- a/third_party/xla/repo.bzl +++ b/third_party/xla/repo.bzl @@ -5,4 +5,6 @@ def repo(): name = "xla", remote = "https://github.com/openxla/xla.git", commit = "01da52b9afe3a2e694bd926323a649a6e63a3785", + patch_args = ["-p0"], + patches = ["//third_party/xla:xspace_to_perfetto.patch"], ) diff --git a/third_party/xla/xla.bzl b/third_party/xla/xla.bzl index 9b8e4303..701f2da0 100644 --- a/third_party/xla/xla.bzl +++ b/third_party/xla/xla.bzl @@ -1,5 +1,8 @@ load("@xla//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls", "tf_vendored") +load("@xla//third_party/farmhash:workspace.bzl", farmhash = "repo") +load("@xla//third_party/eigen3:workspace.bzl", eigen3 = "repo") load("@xla//third_party/llvm:workspace.bzl", llvm = "repo") +load("@xla//third_party/py/ml_dtypes:workspace.bzl", ml_dtypes = "repo") load("@xla//third_party/stablehlo:workspace.bzl", stablehlo = "repo") load("@xla//third_party/shardy:workspace.bzl", shardy = "repo") load("@xla//third_party/triton:workspace.bzl", triton = "repo") @@ -25,7 +28,39 @@ simple_files = repository_rule( def _dummy_repos(mctx): simple_files(name = "local_config_cuda", files = { - "cuda/BUILD.bazel": "", + "BUILD.bazel": """\ +package(default_visibility = ["//visibility:public"]) + +config_setting( + name = "is_cuda_enabled", + define_values = {"local_config_cuda_enabled": "true"}, +) + +config_setting( + name = "is_cuda_compiler_nvcc", + define_values = {"local_config_cuda_compiler_nvcc": "true"}, +) +""", + "cuda/BUILD.bazel": """\ +package(default_visibility = ["//visibility:public"]) + +exports_files(["build_defs.bzl"]) + +filegroup( + name = "build_defs_bzl", + srcs = ["build_defs.bzl"], +) + +config_setting( + name = "using_config_cuda", + define_values = {"local_config_cuda_using_config": "true"}, +) + +config_setting( + name = "FALSE", + define_values = {"local_config_cuda_false": "true"}, +) +""", "cuda/build_defs.bzl": _BZL_HELPERS + """\ cuda_library = always_false if_cuda = always_if_false @@ -35,7 +70,22 @@ is_cuda_configured = always_false """, }) simple_files(name = "local_config_rocm", files = { - "rocm/BUILD.bazel": "", + "BUILD.bazel": "", + "rocm/BUILD.bazel": """\ +package(default_visibility = ["//visibility:public"]) + +exports_files(["build_defs.bzl"]) + +filegroup( + name = "build_defs_bzl", + srcs = ["build_defs.bzl"], +) + +config_setting( + name = "using_hipcc", + define_values = {"local_config_rocm_using_hipcc": "true"}, +) +""", "rocm/build_defs.bzl": _BZL_HELPERS + """\ if_rocm = always_if_false if_rocm_is_configured = always_if_false @@ -48,7 +98,16 @@ get_rbe_amdgpu_pool = always_false }) simple_files(name = "local_config_sycl", files = { "BUILD.bazel": "", - "sycl/BUILD.bazel": "", + "sycl/BUILD.bazel": """\ +package(default_visibility = ["//visibility:public"]) + +exports_files(["build_defs.bzl"]) + +filegroup( + name = "build_defs_bzl", + srcs = ["build_defs.bzl"], +) +""", "crosstool/BUILD.bazel": "", "sycl/build_defs.bzl": _BZL_HELPERS + """\ if_sycl = always_if_false @@ -59,6 +118,16 @@ if_sycl_is_configured = always_if_false "remote_execution.bzl": """gpu_test_tags = lambda: []""", }) simple_files(name = "local_config_tensorrt", files = { + "BUILD.bazel": """\ +package(default_visibility = ["//visibility:public"]) + +exports_files(["build_defs.bzl"]) + +filegroup( + name = "build_defs_bzl", + srcs = ["build_defs.bzl"], +) +""", "build_defs.bzl": _BZL_HELPERS + """if_tensorrt = always_if_false""", }) simple_files(name = "python_version_repo", files = { @@ -77,6 +146,9 @@ def _xla_impl(mctx): stablehlo() shardy() triton() + farmhash() + eigen3() + ml_dtypes() tf_http_archive( name = "com_github_grpc_grpc", diff --git a/third_party/xla/xspace_to_perfetto.patch b/third_party/xla/xspace_to_perfetto.patch new file mode 100644 index 00000000..d42cb950 --- /dev/null +++ b/third_party/xla/xspace_to_perfetto.patch @@ -0,0 +1,24 @@ +--- xla/tsl/profiler/convert/BUILD ++++ xla/tsl/profiler/convert/BUILD +@@ -104,9 +104,7 @@ cc_library( + srcs = ["trace_events_to_json.cc"], + hdrs = ["trace_events_to_json.h"], + copts = tf_profiler_copts(), +- visibility = internal_visibility([ +- "//xla/tsl/profiler:internal", +- ]), ++ visibility = ["//visibility:public"], + deps = [ + ":trace_container", + "//xla/tsl/platform:types", +@@ -150,9 +148,7 @@ cc_library( + srcs = ["xplane_to_trace_events.cc"], + hdrs = ["xplane_to_trace_events.h"], + copts = tf_profiler_copts(), +- visibility = internal_visibility([ +- "//xla/tsl/profiler:internal", +- ]), ++ visibility = ["//visibility:public"], + deps = [ + ":trace_container", + "//xla/tsl/platform:types", diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 5dc3d219..513e9ac2 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -1,3 +1,5 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_cc//cc:cc_test.bzl", "cc_test") load("@rules_python//python:py_library.bzl", "py_library") py_library( @@ -5,3 +7,23 @@ py_library( srcs = ["zml_utils.py"], visibility = ["//visibility:public"], ) + +cc_library( + name = "xspace_to_perfetto_lib", + srcs = ["xspace_to_perfetto_lib.cc"], + hdrs = ["xspace_to_perfetto.h"], + visibility = ["//visibility:public"], + deps = [ + "//ffi:cc", + "@tsl//tsl/profiler/protobuf:xplane_proto_cc", + "@xla//xla/tsl/profiler/convert:trace_events_to_json", + "@xla//xla/tsl/profiler/convert:xplane_to_trace_events", + ], +) + +cc_binary( + name = "xspace_to_perfetto", + srcs = ["xspace_to_perfetto.cc"], + visibility = ["//visibility:public"], + deps = [":xspace_to_perfetto_lib"], +) diff --git a/tools/xprof/BUILD.bazel b/tools/xprof/BUILD.bazel new file mode 100644 index 00000000..3b4dc40e --- /dev/null +++ b/tools/xprof/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_python//python/uv:lock.bzl", uv_lock = "lock") +load("@rules_python//python:py_binary.bzl", "py_binary") + +py_binary( + name = "xprof", + srcs = ["main.py"], + main = "main.py", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = ["@xprof_nightly//xprof_nightly:pkg"], +) + +uv_lock( + name = "requirements", + srcs = ["requirements.in"], + out = "requirements.lock.txt", + args = ["--upgrade"], + tags = ["manual"], +) diff --git a/tools/xprof/main.py b/tools/xprof/main.py new file mode 100644 index 00000000..77da9b46 --- /dev/null +++ b/tools/xprof/main.py @@ -0,0 +1,29 @@ +import sys + +from xprof.server import main as xprof_main + + +def _argv() -> list[str]: + user_args = sys.argv[1:] + if user_args and not user_args[0].startswith("-"): + logdir = user_args[0] + trailing_args = user_args[1:] + else: + logdir = "." + trailing_args = user_args + return [ + sys.argv[0], + "--logdir", + logdir, + "--max_concurrent_worker_requests", + "32", + "--port", + "6006", + "--hide_capture_profile_button", + *trailing_args, + ] + + +if __name__ == "__main__": + sys.argv = _argv() + raise SystemExit(xprof_main()) diff --git a/tools/xprof/requirements.in b/tools/xprof/requirements.in new file mode 100644 index 00000000..666e8afb --- /dev/null +++ b/tools/xprof/requirements.in @@ -0,0 +1 @@ +xprof-nightly diff --git a/tools/xprof/requirements.lock.txt b/tools/xprof/requirements.lock.txt new file mode 100644 index 00000000..c62d58c0 --- /dev/null +++ b/tools/xprof/requirements.lock.txt @@ -0,0 +1,1335 @@ +# This file was autogenerated by uv via the following command: +# bazel run //tools/xprof:requirements.update +aiohappyeyeballs==2.6.1 \ + --hash=sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558 \ + --hash=sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 + # via aiohttp +aiohttp==3.13.3 \ + --hash=sha256:01ad2529d4b5035578f5081606a465f3b814c542882804e2e8cda61adf5c71bf \ + --hash=sha256:042e9e0bcb5fba81886c8b4fbb9a09d6b8a00245fd8d88e4d989c1f96c74164c \ + --hash=sha256:05861afbbec40650d8a07ea324367cb93e9e8cc7762e04dd4405df99fa65159c \ + --hash=sha256:084911a532763e9d3dd95adf78a78f4096cd5f58cdc18e6fdbc1b58417a45423 \ + --hash=sha256:0add0900ff220d1d5c5ebbf99ed88b0c1bbf87aa7e4262300ed1376a6b13414f \ + --hash=sha256:0db318f7a6f065d84cb1e02662c526294450b314a02bd9e2a8e67f0d8564ce40 \ + --hash=sha256:10b47b7ba335d2e9b1239fa571131a87e2d8ec96b333e68b2a305e7a98b0bae2 \ + --hash=sha256:1449ceddcdbcf2e0446957863af03ebaaa03f94c090f945411b61269e2cb5daf \ + --hash=sha256:147e422fd1223005c22b4fe080f5d93ced44460f5f9c105406b753612b587821 \ + --hash=sha256:1cb93e166e6c28716c8c6aeb5f99dfb6d5ccf482d29fe9bf9a794110e6d0ab64 \ + --hash=sha256:215a685b6fbbfcf71dfe96e3eba7a6f58f10da1dfdf4889c7dd856abe430dca7 \ + --hash=sha256:2712039939ec963c237286113c68dbad80a82a4281543f3abf766d9d73228998 \ + --hash=sha256:27234ef6d85c914f9efeb77ff616dbf4ad2380be0cda40b4db086ffc7ddd1b7d \ + --hash=sha256:28e027cf2f6b641693a09f631759b4d9ce9165099d2b5d92af9bd4e197690eea \ + --hash=sha256:2b8d8ddba8f95ba17582226f80e2de99c7a7948e66490ef8d947e272a93e9463 \ + --hash=sha256:2ba0eea45eb5cc3172dbfc497c066f19c41bac70963ea1a67d51fc92e4cf9a80 \ + --hash=sha256:2be0e9ccf23e8a94f6f0650ce06042cefc6ac703d0d7ab6c7a917289f2539ad4 \ + --hash=sha256:2e41b18a58da1e474a057b3d35248d8320029f61d70a37629535b16a0c8f3767 \ + --hash=sha256:2eb752b102b12a76ca02dff751a801f028b4ffbbc478840b473597fc91a9ed43 \ + --hash=sha256:2fc82186fadc4a8316768d61f3722c230e2c1dcab4200d52d2ebdf2482e47592 \ + --hash=sha256:2fff83cfc93f18f215896e3a190e8e5cb413ce01553901aca925176e7568963a \ + --hash=sha256:31a83ea4aead760dfcb6962efb1d861db48c34379f2ff72db9ddddd4cda9ea2e \ + --hash=sha256:34749271508078b261c4abb1767d42b8d0c0cc9449c73a4df494777dc55f0687 \ + --hash=sha256:34bac00a67a812570d4a460447e1e9e06fae622946955f939051e7cc895cfab8 \ + --hash=sha256:37239e9f9a7ea9ac5bf6b92b0260b01f8a22281996da609206a84df860bc1261 \ + --hash=sha256:37da61e244d1749798c151421602884db5270faf479cf0ef03af0ff68954c9dd \ + --hash=sha256:3b61b7169ababd7802f9568ed96142616a9118dd2be0d1866e920e77ec8fa92a \ + --hash=sha256:3d9908a48eb7416dc1f4524e69f1d32e5d90e3981e4e37eb0aa1cd18f9cfa2a4 \ + --hash=sha256:3dd4dce1c718e38081c8f35f323209d4c1df7d4db4bab1b5c88a6b4d12b74587 \ + --hash=sha256:4021b51936308aeea0367b8f006dc999ca02bc118a0cc78c303f50a2ff6afb91 \ + --hash=sha256:40c5e40ecc29ba010656c18052b877a1c28f84344825efa106705e835c28530f \ + --hash=sha256:425c126c0dc43861e22cb1c14ba4c8e45d09516d0a3ae0a3f7494b79f5f233a3 \ + --hash=sha256:44531a36aa2264a1860089ffd4dce7baf875ee5a6079d5fb42e261c704ef7344 \ + --hash=sha256:48e377758516d262bde50c2584fc6c578af272559c409eecbdd2bae1601184d6 \ + --hash=sha256:49a03727c1bba9a97d3e93c9f93ca03a57300f484b6e935463099841261195d3 \ + --hash=sha256:4ae5b5a0e1926e504c81c5b84353e7a5516d8778fbbff00429fe7b05bb25cbce \ + --hash=sha256:4e239d501f73d6db1522599e14b9b321a7e3b1de66ce33d53a765d975e9f4808 \ + --hash=sha256:56339a36b9f1fc708260c76c87e593e2afb30d26de9ae1eb445b5e051b98a7a1 \ + --hash=sha256:568f416a4072fbfae453dcf9a99194bbb8bdeab718e08ee13dfa2ba0e4bebf29 \ + --hash=sha256:5b179331a481cb5529fca8b432d8d3c7001cb217513c94cd72d668d1248688a3 \ + --hash=sha256:5b6073099fb654e0a068ae678b10feff95c5cae95bbfcbfa7af669d361a8aa6b \ + --hash=sha256:5d2d94f1f5fcbe40838ac51a6ab5704a6f9ea42e72ceda48de5e6b898521da51 \ + --hash=sha256:5dff64413671b0d3e7d5918ea490bdccb97a4ad29b3f311ed423200b2203e01c \ + --hash=sha256:5e1d8c8b8f1d91cd08d8f4a3c2b067bfca6ec043d3ff36de0f3a715feeedf926 \ + --hash=sha256:5f8ca7f2bb6ba8348a3614c7918cc4bb73268c5ac2a207576b7afea19d3d9f64 \ + --hash=sha256:642f752c3eb117b105acbd87e2c143de710987e09860d674e068c4c2c441034f \ + --hash=sha256:65d2ccb7eabee90ce0503c17716fc77226be026dcc3e65cce859a30db715025b \ + --hash=sha256:693781c45a4033d31d4187d2436f5ac701e7bbfe5df40d917736108c1cc7436e \ + --hash=sha256:694976222c711d1d00ba131904beb60534f93966562f64440d0c9d41b8cdb440 \ + --hash=sha256:697753042d57f4bf7122cab985bf15d0cef23c770864580f5af4f52023a56bd6 \ + --hash=sha256:69c56fbc1993fa17043e24a546959c0178fe2b5782405ad4559e6c13975c15e3 \ + --hash=sha256:6de499a1a44e7de70735d0b39f67c8f25eb3d91eb3103be99ca0fa882cdd987d \ + --hash=sha256:6fc0e2337d1a4c3e6acafda6a78a39d4c14caea625124817420abceed36e2415 \ + --hash=sha256:75ca857eba4e20ce9f546cd59c7007b33906a4cd48f2ff6ccf1ccfc3b646f279 \ + --hash=sha256:7a4a94eb787e606d0a09404b9c38c113d3b099d508021faa615d70a0131907ce \ + --hash=sha256:7b5e8fe4de30df199155baaf64f2fcd604f4c678ed20910db8e2c66dc4b11603 \ + --hash=sha256:7bfdc049127717581866fa4708791220970ce291c23e28ccf3922c700740fdc0 \ + --hash=sha256:7e63f210bc1b57ef699035f2b4b6d9ce096b5914414a49b0997c839b2bd2223c \ + --hash=sha256:7f9120f7093c2a32d9647abcaf21e6ad275b4fbec5b55969f978b1a97c7c86bf \ + --hash=sha256:8057c98e0c8472d8846b9c79f56766bcc57e3e8ac7bfd510482332366c56c591 \ + --hash=sha256:80dd4c21b0f6237676449c6baaa1039abae86b91636b6c91a7f8e61c87f89540 \ + --hash=sha256:81e97251d9298386c2b7dbeb490d3d1badbdc69107fb8c9299dd04eb39bddc0e \ + --hash=sha256:82611aeec80eb144416956ec85b6ca45a64d76429c1ed46ae1b5f86c6e0c9a26 \ + --hash=sha256:8542f41a62bcc58fc7f11cf7c90e0ec324ce44950003feb70640fc2a9092c32a \ + --hash=sha256:859bd3f2156e81dd01432f5849fc73e2243d4a487c4fd26609b1299534ee1845 \ + --hash=sha256:87797e645d9d8e222e04160ee32aa06bc5c163e8499f24db719e7852ec23093a \ + --hash=sha256:87b9aab6d6ed88235aa2970294f496ff1a1f9adcd724d800e9b952395a80ffd9 \ + --hash=sha256:8a60e60746623925eab7d25823329941aee7242d559baa119ca2b253c88a7bd6 \ + --hash=sha256:90455115e5da1c3c51ab619ac57f877da8fd6d73c05aacd125c5ae9819582aba \ + --hash=sha256:90751b8eed69435bac9ff4e3d2f6b3af1f57e37ecb0fbeee59c0174c9e2d41df \ + --hash=sha256:947c26539750deeaee933b000fb6517cc770bbd064bad6033f1cff4803881e43 \ + --hash=sha256:96d604498a7c782cb15a51c406acaea70d8c027ee6b90c569baa6e7b93073679 \ + --hash=sha256:988a8c5e317544fdf0d39871559e67b6341065b87fceac641108c2096d5506b7 \ + --hash=sha256:9a9dc347e5a3dc7dfdbc1f82da0ef29e388ddb2ed281bfce9dd8248a313e62b7 \ + --hash=sha256:9ae8dd55c8e6c4257eae3a20fd2c8f41edaea5992ed67156642493b8daf3cecc \ + --hash=sha256:9af5e68ee47d6534d36791bbe9b646d2a7c7deb6fc24d7943628edfbb3581f29 \ + --hash=sha256:9b174f267b5cfb9a7dba9ee6859cecd234e9a681841eb85068059bc867fb8f02 \ + --hash=sha256:9bf9f7a65e7aa20dd764151fb3d616c81088f91f8df39c3893a536e279b4b984 \ + --hash=sha256:9d4c940f02f49483b18b079d1c27ab948721852b281f8b015c058100e9421dd1 \ + --hash=sha256:9ebf57d09e131f5323464bd347135a88622d1c0976e88ce15b670e7ad57e4bd6 \ + --hash=sha256:a19884d2ee70b06d9204b2727a7b9f983d0c684c650254679e716b0b77920632 \ + --hash=sha256:a1e53262fd202e4b40b70c3aff944a8155059beedc8a89bba9dc1f9ef06a1b56 \ + --hash=sha256:a2212ad43c0833a873d0fb3c63fa1bacedd4cf6af2fee62bf4b739ceec3ab239 \ + --hash=sha256:a45530014d7a1e09f4a55f4f43097ba0fd155089372e105e4bff4ca76cb1b168 \ + --hash=sha256:a949eee43d3782f2daae4f4a2819b2cb9b0c5d3b7f7a927067cc84dafdbb9f88 \ + --hash=sha256:add1da70de90a2569c5e15249ff76a631ccacfe198375eead4aadf3b8dc849dc \ + --hash=sha256:af71fff7bac6bb7508956696dce8f6eec2bbb045eceb40343944b1ae62b5ef11 \ + --hash=sha256:b04be762396457bef43f3597c991e192ee7da460a4953d7e647ee4b1c28e7046 \ + --hash=sha256:b0d95340658b9d2f11d9697f59b3814a9d3bb4b7a7c20b131df4bcef464037c0 \ + --hash=sha256:b1a6102b4d3ebc07dad44fbf07b45bb600300f15b552ddf1851b5390202ea2e3 \ + --hash=sha256:b46020d11d23fe16551466c77823df9cc2f2c1e63cc965daf67fa5eec6ca1877 \ + --hash=sha256:b556c85915d8efaed322bf1bdae9486aa0f3f764195a0fb6ee962e5c71ef5ce1 \ + --hash=sha256:b903a4dfee7d347e2d87697d0713be59e0b87925be030c9178c5faa58ea58d5c \ + --hash=sha256:b928f30fe49574253644b1ca44b1b8adbd903aa0da4b9054a6c20fc7f4092a25 \ + --hash=sha256:b99281b0704c103d4e11e72a76f1b543d4946fea7dd10767e7e1b5f00d4e5704 \ + --hash=sha256:bae5c2ed2eae26cc382020edad80d01f36cb8e746da40b292e68fec40421dc6a \ + --hash=sha256:bb4f7475e359992b580559e008c598091c45b5088f28614e855e42d39c2f1033 \ + --hash=sha256:bbe7d4cecacb439e2e2a8a1a7b935c25b812af7a5fd26503a66dadf428e79ec1 \ + --hash=sha256:bfc1cc2fe31a6026a8a88e4ecfb98d7f6b1fec150cfd708adbfd1d2f42257c29 \ + --hash=sha256:c014c7ea7fb775dd015b2d3137378b7be0249a448a1612268b5a90c2d81de04d \ + --hash=sha256:c048058117fd649334d81b4b526e94bde3ccaddb20463a815ced6ecbb7d11160 \ + --hash=sha256:c0e2d366af265797506f0283487223146af57815b388623f0357ef7eac9b209d \ + --hash=sha256:c19b90316ad3b24c69cd78d5c9b4f3aa4497643685901185b65166293d36a00f \ + --hash=sha256:c685f2d80bb67ca8c3837823ad76196b3694b0159d232206d1e461d3d434666f \ + --hash=sha256:c6b8568a3bb5819a0ad087f16d40e5a3fb6099f39ea1d5625a3edc1e923fc538 \ + --hash=sha256:d32764c6c9aafb7fb55366a224756387cd50bfa720f32b88e0e6fa45b27dcf29 \ + --hash=sha256:d5a372fd5afd301b3a89582817fdcdb6c34124787c70dbcc616f259013e7eef7 \ + --hash=sha256:d60ac9663f44168038586cab2157e122e46bdef09e9368b37f2d82d354c23f72 \ + --hash=sha256:dca68018bf48c251ba17c72ed479f4dafe9dbd5a73707ad8d28a38d11f3d42af \ + --hash=sha256:de2c184bb1fe2cbd2cefba613e9db29a5ab559323f994b6737e370d3da0ac455 \ + --hash=sha256:e3531d63d3bdfa7e3ac5e9b27b2dd7ec9df3206a98e0b3445fa906f233264c57 \ + --hash=sha256:e50a2e1404f063427c9d027378472316201a2290959a295169bcf25992d04558 \ + --hash=sha256:e636b3c5f61da31a92bf0d91da83e58fdfa96f178ba682f11d24f31944cdd28c \ + --hash=sha256:ea37047c6b367fd4bd632bff8077449b8fa034b69e812a18e0132a00fae6e808 \ + --hash=sha256:f33ed1a2bf1997a36661874b017f5c4b760f41266341af36febaf271d179f6d7 \ + --hash=sha256:f76c1e3fe7d7c8afad7ed193f89a292e1999608170dcc9751a7462a87dfd5bc0 \ + --hash=sha256:f9444f105664c4ce47a2a7171a2418bce5b7bae45fb610f4e2c36045d85911d3 \ + --hash=sha256:fc290605db2a917f6e81b0e1e0796469871f5af381ce15c604a3c5c7e51cb730 \ + --hash=sha256:fc353029f176fd2b3ec6cfc71be166aba1936fe5d73dd1992ce289ca6647a9aa \ + --hash=sha256:fee0c6bc7db1de362252affec009707a17478a00ec69f797d23ca256e36d5940 + # via gcsfs +aiosignal==1.4.0 \ + --hash=sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e \ + --hash=sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7 + # via aiohttp +attrs==25.4.0 \ + --hash=sha256:16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11 \ + --hash=sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373 + # via aiohttp +certifi==2026.2.25 \ + --hash=sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa \ + --hash=sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7 + # via requests +cffi==2.0.0 \ + --hash=sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb \ + --hash=sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b \ + --hash=sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f \ + --hash=sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9 \ + --hash=sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44 \ + --hash=sha256:0f6084a0ea23d05d20c3edcda20c3d006f9b6f3fefeac38f59262e10cef47ee2 \ + --hash=sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c \ + --hash=sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75 \ + --hash=sha256:1cd13c99ce269b3ed80b417dcd591415d3372bcac067009b6e0f59c7d4015e65 \ + --hash=sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e \ + --hash=sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a \ + --hash=sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e \ + --hash=sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25 \ + --hash=sha256:2081580ebb843f759b9f617314a24ed5738c51d2aee65d31e02f6f7a2b97707a \ + --hash=sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe \ + --hash=sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b \ + --hash=sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91 \ + --hash=sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592 \ + --hash=sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187 \ + --hash=sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c \ + --hash=sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1 \ + --hash=sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94 \ + --hash=sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba \ + --hash=sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb \ + --hash=sha256:3f4d46d8b35698056ec29bca21546e1551a205058ae1a181d871e278b0b28165 \ + --hash=sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529 \ + --hash=sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca \ + --hash=sha256:4647afc2f90d1ddd33441e5b0e85b16b12ddec4fca55f0d9671fef036ecca27c \ + --hash=sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6 \ + --hash=sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c \ + --hash=sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0 \ + --hash=sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743 \ + --hash=sha256:61d028e90346df14fedc3d1e5441df818d095f3b87d286825dfcbd6459b7ef63 \ + --hash=sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5 \ + --hash=sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5 \ + --hash=sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4 \ + --hash=sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d \ + --hash=sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b \ + --hash=sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93 \ + --hash=sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205 \ + --hash=sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27 \ + --hash=sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512 \ + --hash=sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d \ + --hash=sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c \ + --hash=sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037 \ + --hash=sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26 \ + --hash=sha256:89472c9762729b5ae1ad974b777416bfda4ac5642423fa93bd57a09204712322 \ + --hash=sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb \ + --hash=sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c \ + --hash=sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8 \ + --hash=sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4 \ + --hash=sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414 \ + --hash=sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9 \ + --hash=sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664 \ + --hash=sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9 \ + --hash=sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775 \ + --hash=sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739 \ + --hash=sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc \ + --hash=sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062 \ + --hash=sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe \ + --hash=sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9 \ + --hash=sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92 \ + --hash=sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5 \ + --hash=sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13 \ + --hash=sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d \ + --hash=sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26 \ + --hash=sha256:cb527a79772e5ef98fb1d700678fe031e353e765d1ca2d409c92263c6d43e09f \ + --hash=sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495 \ + --hash=sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b \ + --hash=sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6 \ + --hash=sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c \ + --hash=sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef \ + --hash=sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5 \ + --hash=sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18 \ + --hash=sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad \ + --hash=sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3 \ + --hash=sha256:de8dad4425a6ca6e4e5e297b27b5c824ecc7581910bf9aee86cb6835e6812aa7 \ + --hash=sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5 \ + --hash=sha256:e6e73b9e02893c764e7e8d5bb5ce277f1a009cd5243f8228f75f842bf937c534 \ + --hash=sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49 \ + --hash=sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2 \ + --hash=sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5 \ + --hash=sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453 \ + --hash=sha256:fe562eb1a64e67dd297ccc4f5addea2501664954f2692b69a76449ec7913ecbf + # via cryptography +charset-normalizer==3.4.6 \ + --hash=sha256:06a7e86163334edfc5d20fe104db92fcd666e5a5df0977cb5680a506fe26cc8e \ + --hash=sha256:0c173ce3a681f309f31b87125fecec7a5d1347261ea11ebbb856fa6006b23c8c \ + --hash=sha256:0e28d62a8fc7a1fa411c43bd65e346f3bce9716dc51b897fbe930c5987b402d5 \ + --hash=sha256:0e901eb1049fdb80f5bd11ed5ea1e498ec423102f7a9b9e4645d5b8204ff2815 \ + --hash=sha256:11afb56037cbc4b1555a34dd69151e8e069bee82e613a73bef6e714ce733585f \ + --hash=sha256:150b8ce8e830eb7ccb029ec9ca36022f756986aaaa7956aad6d9ec90089338c0 \ + --hash=sha256:172985e4ff804a7ad08eebec0a1640ece87ba5041d565fff23c8f99c1f389484 \ + --hash=sha256:197c1a244a274bb016dd8b79204850144ef77fe81c5b797dc389327adb552407 \ + --hash=sha256:1ae6b62897110aa7c79ea2f5dd38d1abca6db663687c0b1ad9aed6f6bae3d9d6 \ + --hash=sha256:1cf0a70018692f85172348fe06d3a4b63f94ecb055e13a00c644d368eb82e5b8 \ + --hash=sha256:1ed80ff870ca6de33f4d953fda4d55654b9a2b340ff39ab32fa3adbcd718f264 \ + --hash=sha256:22c6f0c2fbc31e76c3b8a86fba1a56eda6166e238c29cdd3d14befdb4a4e4815 \ + --hash=sha256:231d4da14bcd9301310faf492051bee27df11f2bc7549bc0bb41fef11b82daa2 \ + --hash=sha256:259695e2ccc253feb2a016303543d691825e920917e31f894ca1a687982b1de4 \ + --hash=sha256:2a24157fa36980478dd1770b585c0f30d19e18f4fb0c47c13aa568f871718579 \ + --hash=sha256:2b1a63e8224e401cafe7739f77efd3f9e7f5f2026bda4aead8e59afab537784f \ + --hash=sha256:2bd9d128ef93637a5d7a6af25363cf5dec3fa21cf80e68055aad627f280e8afa \ + --hash=sha256:2e1d8ca8611099001949d1cdfaefc510cf0f212484fe7c565f735b68c78c3c95 \ + --hash=sha256:2ef7fedc7a6ecbe99969cd09632516738a97eeb8bd7258bf8a0f23114c057dab \ + --hash=sha256:2f7fdd9b6e6c529d6a2501a2d36b240109e78a8ceaef5687cfcfa2bbe671d297 \ + --hash=sha256:30f445ae60aad5e1f8bdbb3108e39f6fbc09f4ea16c815c66578878325f8f15a \ + --hash=sha256:31215157227939b4fb3d740cd23fe27be0439afef67b785a1eb78a3ae69cba9e \ + --hash=sha256:34315ff4fc374b285ad7f4a0bf7dcbfe769e1b104230d40f49f700d4ab6bbd84 \ + --hash=sha256:3516bbb8d42169de9e61b8520cbeeeb716f12f4ecfe3fd30a9919aa16c806ca8 \ + --hash=sha256:3778fd7d7cd04ae8f54651f4a7a0bd6e39a0cf20f801720a4c21d80e9b7ad6b0 \ + --hash=sha256:39f5068d35621da2881271e5c3205125cc456f54e9030d3f723288c873a71bf9 \ + --hash=sha256:404a1e552cf5b675a87f0651f8b79f5f1e6fd100ee88dc612f89aa16abd4486f \ + --hash=sha256:419a9d91bd238052642a51938af8ac05da5b3343becde08d5cdeab9046df9ee1 \ + --hash=sha256:423fb7e748a08f854a08a222b983f4df1912b1daedce51a72bd24fe8f26a1843 \ + --hash=sha256:4482481cb0572180b6fd976a4d5c72a30263e98564da68b86ec91f0fe35e8565 \ + --hash=sha256:461598cd852bfa5a61b09cae2b1c02e2efcd166ee5516e243d540ac24bfa68a7 \ + --hash=sha256:47955475ac79cc504ef2704b192364e51d0d473ad452caedd0002605f780101c \ + --hash=sha256:48696db7f18afb80a068821504296eb0787d9ce239b91ca15059d1d3eaacf13b \ + --hash=sha256:4be9f4830ba8741527693848403e2c457c16e499100963ec711b1c6f2049b7c7 \ + --hash=sha256:4d1d02209e06550bdaef34af58e041ad71b88e624f5d825519da3a3308e22687 \ + --hash=sha256:4f41da960b196ea355357285ad1316a00099f22d0929fe168343b99b254729c9 \ + --hash=sha256:517ad0e93394ac532745129ceabdf2696b609ec9f87863d337140317ebce1c14 \ + --hash=sha256:51fb3c322c81d20567019778cb5a4a6f2dc1c200b886bc0d636238e364848c89 \ + --hash=sha256:5273b9f0b5835ff0350c0828faea623c68bfa65b792720c453e22b25cc72930f \ + --hash=sha256:530d548084c4a9f7a16ed4a294d459b4f229db50df689bfe92027452452943a0 \ + --hash=sha256:530e8cebeea0d76bdcf93357aa5e41336f48c3dc709ac52da2bb167c5b8271d9 \ + --hash=sha256:54fae94be3d75f3e573c9a1b5402dc593de19377013c9a0e4285e3d402dd3a2a \ + --hash=sha256:572d7c822caf521f0525ba1bce1a622a0b85cf47ffbdae6c9c19e3b5ac3c4389 \ + --hash=sha256:58c948d0d086229efc484fe2f30c2d382c86720f55cd9bc33591774348ad44e0 \ + --hash=sha256:5d11595abf8dd942a77883a39d81433739b287b6aa71620f15164f8096221b30 \ + --hash=sha256:5f8ddd609f9e1af8c7bd6e2aca279c931aefecd148a14402d4e368f3171769fd \ + --hash=sha256:5feb91325bbceade6afab43eb3b508c63ee53579fe896c77137ded51c6b6958e \ + --hash=sha256:60c74963d8350241a79cb8feea80e54d518f72c26db618862a8f53e5023deaf9 \ + --hash=sha256:613f19aa6e082cf96e17e3ffd89383343d0d589abda756b7764cf78361fd41dc \ + --hash=sha256:659a1e1b500fac8f2779dd9e1570464e012f43e580371470b45277a27baa7532 \ + --hash=sha256:695f5c2823691a25f17bc5d5ffe79fa90972cc34b002ac6c843bb8a1720e950d \ + --hash=sha256:69dd852c2f0ad631b8b60cfbe25a28c0058a894de5abb566619c205ce0550eae \ + --hash=sha256:6cceb5473417d28edd20c6c984ab6fee6c6267d38d906823ebfe20b03d607dc2 \ + --hash=sha256:71be7e0e01753a89cf024abf7ecb6bca2c81738ead80d43004d9b5e3f1244e64 \ + --hash=sha256:74119174722c4349af9708993118581686f343adc1c8c9c007d59be90d077f3f \ + --hash=sha256:74a2e659c7ecbc73562e2a15e05039f1e22c75b7c7618b4b574a3ea9118d1557 \ + --hash=sha256:7504e9b7dc05f99a9bbb4525c67a2c155073b44d720470a148b34166a69c054e \ + --hash=sha256:79090741d842f564b1b2827c0b82d846405b744d31e84f18d7a7b41c20e473ff \ + --hash=sha256:7a6967aaf043bceabab5412ed6bd6bd26603dae84d5cb75bf8d9a74a4959d398 \ + --hash=sha256:7bda6eebafd42133efdca535b04ccb338ab29467b3f7bf79569883676fc628db \ + --hash=sha256:7edbed096e4a4798710ed6bc75dcaa2a21b68b6c356553ac4823c3658d53743a \ + --hash=sha256:7f9019c9cb613f084481bd6a100b12e1547cf2efe362d873c2e31e4035a6fa43 \ + --hash=sha256:802168e03fba8bbc5ce0d866d589e4b1ca751d06edee69f7f3a19c5a9fe6b597 \ + --hash=sha256:80d0a5615143c0b3225e5e3ef22c8d5d51f3f72ce0ea6fb84c943546c7b25b6c \ + --hash=sha256:82060f995ab5003a2d6e0f4ad29065b7672b6593c8c63559beefe5b443242c3e \ + --hash=sha256:836ab36280f21fc1a03c99cd05c6b7af70d2697e374c7af0b61ed271401a72a2 \ + --hash=sha256:8761ac29b6c81574724322a554605608a9960769ea83d2c73e396f3df896ad54 \ + --hash=sha256:87725cfb1a4f1f8c2fc9890ae2f42094120f4b44db9360be5d99a4c6b0e03a9e \ + --hash=sha256:899d28f422116b08be5118ef350c292b36fc15ec2daeb9ea987c89281c7bb5c4 \ + --hash=sha256:8bc5f0687d796c05b1e28ab0d38a50e6309906ee09375dd3aff6a9c09dd6e8f4 \ + --hash=sha256:8bea55c4eef25b0b19a0337dc4e3f9a15b00d569c77211fa8cde38684f234fb7 \ + --hash=sha256:8e5a94886bedca0f9b78fecd6afb6629142fd2605aa70a125d49f4edc6037ee6 \ + --hash=sha256:90ca27cd8da8118b18a52d5f547859cc1f8354a00cd1e8e5120df3e30d6279e5 \ + --hash=sha256:92734d4d8d187a354a556626c221cd1a892a4e0802ccb2af432a1d85ec012194 \ + --hash=sha256:947cf925bc916d90adba35a64c82aace04fa39b46b52d4630ece166655905a69 \ + --hash=sha256:95b52c68d64c1878818687a473a10547b3292e82b6f6fe483808fb1468e2f52f \ + --hash=sha256:97d0235baafca5f2b09cf332cc275f021e694e8362c6bb9c96fc9a0eb74fc316 \ + --hash=sha256:9ca4c0b502ab399ef89248a2c84c54954f77a070f28e546a85e91da627d1301e \ + --hash=sha256:9cc4fc6c196d6a8b76629a70ddfcd4635a6898756e2d9cac5565cf0654605d73 \ + --hash=sha256:9cc6e6d9e571d2f863fa77700701dae73ed5f78881efc8b3f9a4398772ff53e8 \ + --hash=sha256:a056d1ad2633548ca18ffa2f85c202cfb48b68615129143915b8dc72a806a923 \ + --hash=sha256:a26611d9987b230566f24a0a125f17fe0de6a6aff9f25c9f564aaa2721a5fb88 \ + --hash=sha256:a4474d924a47185a06411e0064b803c68be044be2d60e50e8bddcc2649957c1f \ + --hash=sha256:a4ea868bc28109052790eb2b52a9ab33f3aa7adc02f96673526ff47419490e21 \ + --hash=sha256:a9e68c9d88823b274cf1e72f28cb5dc89c990edf430b0bfd3e2fb0785bfeabf4 \ + --hash=sha256:aa9cccf4a44b9b62d8ba8b4dd06c649ba683e4bf04eea606d2e94cfc2d6ff4d6 \ + --hash=sha256:ab30e5e3e706e3063bc6de96b118688cb10396b70bb9864a430f67df98c61ecc \ + --hash=sha256:ac2393c73378fea4e52aa56285a3d64be50f1a12395afef9cce47772f60334c2 \ + --hash=sha256:ad8faf8df23f0378c6d527d8b0b15ea4a2e23c89376877c598c4870d1b2c7866 \ + --hash=sha256:b35b200d6a71b9839a46b9b7fff66b6638bb52fc9658aa58796b0326595d3021 \ + --hash=sha256:b3694e3f87f8ac7ce279d4355645b3c878d24d1424581b46282f24b92f5a4ae2 \ + --hash=sha256:b4ff1d35e8c5bd078be89349b6f3a845128e685e751b6ea1169cf2160b344c4d \ + --hash=sha256:bbc8c8650c6e51041ad1be191742b8b421d05bbd3410f43fa2a00c8db87678e8 \ + --hash=sha256:bc72863f4d9aba2e8fd9085e63548a324ba706d2ea2c83b260da08a59b9482de \ + --hash=sha256:bf625105bb9eef28a56a943fec8c8a98aeb80e7d7db99bd3c388137e6eb2d237 \ + --hash=sha256:c2274ca724536f173122f36c98ce188fd24ce3dad886ec2b7af859518ce008a4 \ + --hash=sha256:c45a03a4c69820a399f1dda9e1d8fbf3562eda46e7720458180302021b08f778 \ + --hash=sha256:c8ae56368f8cc97c7e40a7ee18e1cedaf8e780cd8bc5ed5ac8b81f238614facb \ + --hash=sha256:c907cdc8109f6c619e6254212e794d6548373cc40e1ec75e6e3823d9135d29cc \ + --hash=sha256:ca0276464d148c72defa8bb4390cce01b4a0e425f3b50d1435aa6d7a18107602 \ + --hash=sha256:cd5e2801c89992ed8c0a3f0293ae83c159a60d9a5d685005383ef4caca77f2c4 \ + --hash=sha256:d08ec48f0a1c48d75d0356cea971921848fb620fdeba805b28f937e90691209f \ + --hash=sha256:d1a2ee9c1499fc8f86f4521f27a973c914b211ffa87322f4ee33bb35392da2c5 \ + --hash=sha256:d5f5d1e9def3405f60e3ca8232d56f35c98fb7bf581efcc60051ebf53cb8b611 \ + --hash=sha256:d60377dce4511655582e300dc1e5a5f24ba0cb229005a1d5c8d0cb72bb758ab8 \ + --hash=sha256:d73beaac5e90173ac3deb9928a74763a6d230f494e4bfb422c217a0ad8e629bf \ + --hash=sha256:d7de2637729c67d67cf87614b566626057e95c303bc0a55ffe391f5205e7003d \ + --hash=sha256:dad6e0f2e481fffdcf776d10ebee25e0ef89f16d691f1e5dee4b586375fdc64b \ + --hash=sha256:dda86aba335c902b6149a02a55b38e96287157e609200811837678214ba2b1db \ + --hash=sha256:df01808ee470038c3f8dc4f48620df7225c49c2d6639e38f96e6d6ac6e6f7b0e \ + --hash=sha256:e1f6e2f00a6b8edb562826e4632e26d063ac10307e80f7461f7de3ad8ef3f077 \ + --hash=sha256:e25369dc110d58ddf29b949377a93e0716d72a24f62bad72b2b39f155949c1fd \ + --hash=sha256:e3c701e954abf6fc03a49f7c579cc80c2c6cc52525340ca3186c41d3f33482ef \ + --hash=sha256:e5bcc1a1ae744e0bb59641171ae53743760130600da8db48cbb6e4918e186e4e \ + --hash=sha256:e68c14b04827dd76dcbd1aeea9e604e3e4b78322d8faf2f8132c7138efa340a8 \ + --hash=sha256:e8aeb10fcbe92767f0fa69ad5a72deca50d0dca07fbde97848997d778a50c9fe \ + --hash=sha256:e985a16ff513596f217cee86c21371b8cd011c0f6f056d0920aa2d926c544058 \ + --hash=sha256:ecbbd45615a6885fe3240eb9db73b9e62518b611850fdf8ab08bd56de7ad2b17 \ + --hash=sha256:ee4ec14bc1680d6b0afab9aea2ef27e26d2024f18b24a2d7155a52b60da7e833 \ + --hash=sha256:ef5960d965e67165d75b7c7ffc60a83ec5abfc5c11b764ec13ea54fbef8b4421 \ + --hash=sha256:f0cdaecd4c953bfae0b6bb64910aaaca5a424ad9c72d85cb88417bb9814f7550 \ + --hash=sha256:f1ce721c8a7dfec21fcbdfe04e8f68174183cf4e8188e0645e92aa23985c57ff \ + --hash=sha256:f50498891691e0864dc3da965f340fada0771f6142a378083dc4608f4ea513e2 \ + --hash=sha256:f5ea69428fa1b49573eef0cc44a1d43bebd45ad0c611eb7d7eac760c7ae771bc \ + --hash=sha256:f61aa92e4aad0be58eb6eb4e0c21acf32cf8065f4b2cae5665da756c4ceef982 \ + --hash=sha256:f6e4333fb15c83f7d1482a76d45a0818897b3d33f00efd215528ff7c51b8e35d \ + --hash=sha256:f820f24b09e3e779fe84c3c456cb4108a7aa639b0d1f02c28046e11bfcd088ed \ + --hash=sha256:f98059e4fcd3e3e4e2d632b7cf81c2faae96c43c60b569e9c621468082f1d104 \ + --hash=sha256:fcce033e4021347d80ed9c66dcf1e7b1546319834b74445f561d2e2221de5659 + # via requests +cheroot==11.1.2 \ + --hash=sha256:0f6c0ba05c00fbc869fb46b1de4ec2384e1d85418ae963d3bc10ae83b688dbfa \ + --hash=sha256:bfb70c49663f63b0440f2b54dbc6b0d1650e56dfe4e2641f59b2c6f727b44aca + # via xprof-nightly +cryptography==46.0.5 \ + --hash=sha256:02f547fce831f5096c9a567fd41bc12ca8f11df260959ecc7c3202555cc47a72 \ + --hash=sha256:039917b0dc418bb9f6edce8a906572d69e74bd330b0b3fea4f79dab7f8ddd235 \ + --hash=sha256:1abfdb89b41c3be0365328a410baa9df3ff8a9110fb75e7b52e66803ddabc9a9 \ + --hash=sha256:2ae6971afd6246710480e3f15824ed3029a60fc16991db250034efd0b9fb4356 \ + --hash=sha256:2b7a67c9cd56372f3249b39699f2ad479f6991e62ea15800973b956f4b73e257 \ + --hash=sha256:351695ada9ea9618b3500b490ad54c739860883df6c1f555e088eaf25b1bbaad \ + --hash=sha256:38946c54b16c885c72c4f59846be9743d699eee2b69b6988e0a00a01f46a61a4 \ + --hash=sha256:3b4995dc971c9fb83c25aa44cf45f02ba86f71ee600d81091c2f0cbae116b06c \ + --hash=sha256:3ce58ba46e1bc2aac4f7d9290223cead56743fa6ab94a5d53292ffaac6a91614 \ + --hash=sha256:3ee190460e2fbe447175cda91b88b84ae8322a104fc27766ad09428754a618ed \ + --hash=sha256:4108d4c09fbbf2789d0c926eb4152ae1760d5a2d97612b92d508d96c861e4d31 \ + --hash=sha256:420d0e909050490d04359e7fdb5ed7e667ca5c3c402b809ae2563d7e66a92229 \ + --hash=sha256:47fb8a66058b80e509c47118ef8a75d14c455e81ac369050f20ba0d23e77fee0 \ + --hash=sha256:4c3341037c136030cb46e4b1e17b7418ea4cbd9dd207e4a6f3b2b24e0d4ac731 \ + --hash=sha256:4d7e3d356b8cd4ea5aff04f129d5f66ebdc7b6f8eae802b93739ed520c47c79b \ + --hash=sha256:4d8ae8659ab18c65ced284993c2265910f6c9e650189d4e3f68445ef82a810e4 \ + --hash=sha256:4e817a8920bfbcff8940ecfd60f23d01836408242b30f1a708d93198393a80b4 \ + --hash=sha256:50bfb6925eff619c9c023b967d5b77a54e04256c4281b0e21336a130cd7fc263 \ + --hash=sha256:556e106ee01aa13484ce9b0239bca667be5004efb0aabbed28d353df86445595 \ + --hash=sha256:582f5fcd2afa31622f317f80426a027f30dc792e9c80ffee87b993200ea115f1 \ + --hash=sha256:5be7bf2fb40769e05739dd0046e7b26f9d4670badc7b032d6ce4db64dddc0678 \ + --hash=sha256:60ee7e19e95104d4c03871d7d7dfb3d22ef8a9b9c6778c94e1c8fcc8365afd48 \ + --hash=sha256:61aa400dce22cb001a98014f647dc21cda08f7915ceb95df0c9eaf84b4b6af76 \ + --hash=sha256:68f68d13f2e1cb95163fa3b4db4bf9a159a418f5f6e7242564fc75fcae667fd0 \ + --hash=sha256:7d1f30a86d2757199cb2d56e48cce14deddf1f9c95f1ef1b64ee91ea43fe2e18 \ + --hash=sha256:7d731d4b107030987fd61a7f8ab512b25b53cef8f233a97379ede116f30eb67d \ + --hash=sha256:803812e111e75d1aa73690d2facc295eaefd4439be1023fefc4995eaea2af90d \ + --hash=sha256:80a8d7bfdf38f87ca30a5391c0c9ce4ed2926918e017c29ddf643d0ed2778ea1 \ + --hash=sha256:8293f3dea7fc929ef7240796ba231413afa7b68ce38fd21da2995549f5961981 \ + --hash=sha256:8456928655f856c6e1533ff59d5be76578a7157224dbd9ce6872f25055ab9ab7 \ + --hash=sha256:890bcb4abd5a2d3f852196437129eb3667d62630333aacc13dfd470fad3aaa82 \ + --hash=sha256:94a76daa32eb78d61339aff7952ea819b1734b46f73646a07decb40e5b3448e2 \ + --hash=sha256:9f16fbdf4da055efb21c22d81b89f155f02ba420558db21288b3d0035bafd5f4 \ + --hash=sha256:a3d1fae9863299076f05cb8a778c467578262fae09f9dc0ee9b12eb4268ce663 \ + --hash=sha256:a3d507bb6a513ca96ba84443226af944b0f7f47dcc9a399d110cd6146481d24c \ + --hash=sha256:abace499247268e3757271b2f1e244b36b06f8515cf27c4d49468fc9eb16e93d \ + --hash=sha256:ba2a27ff02f48193fc4daeadf8ad2590516fa3d0adeeb34336b96f7fa64c1e3a \ + --hash=sha256:bc84e875994c3b445871ea7181d424588171efec3e185dced958dad9e001950a \ + --hash=sha256:bfd56bb4b37ed4f330b82402f6f435845a5f5648edf1ad497da51a8452d5d62d \ + --hash=sha256:c18ff11e86df2e28854939acde2d003f7984f721eba450b56a200ad90eeb0e6b \ + --hash=sha256:c3bcce8521d785d510b2aad26ae2c966092b7daa8f45dd8f44734a104dc0bc1a \ + --hash=sha256:c4143987a42a2397f2fc3b4d7e3a7d313fbe684f67ff443999e803dd75a76826 \ + --hash=sha256:c69fd885df7d089548a42d5ec05be26050ebcd2283d89b3d30676eb32ff87dee \ + --hash=sha256:ced80795227d70549a411a4ab66e8ce307899fad2220ce5ab2f296e687eacde9 \ + --hash=sha256:d66e421495fdb797610a08f43b05269e0a5ea7f5e652a89bfd5a7d3c1dee3648 \ + --hash=sha256:d861ee9e76ace6cf36a6a89b959ec08e7bc2493ee39d07ffe5acb23ef46d27da \ + --hash=sha256:e9251e3be159d1020c4030bd2e5f84d6a43fe54b6c19c12f51cde9542a2817b2 \ + --hash=sha256:f145bba11b878005c496e93e257c1e88f154d278d2638e6450d17e0f31e558d2 \ + --hash=sha256:fe346b143ff9685e40192a4960938545c699054ba11d4f9029f94751e3f71d87 + # via google-auth +decorator==5.2.1 \ + --hash=sha256:65f266143752f734b0a7cc83c46f4618af75b8c5911b00ccb61d0ac9b6da0360 \ + --hash=sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a + # via gcsfs +etils[epath]==1.14.0 \ + --hash=sha256:8136e7f4c4173cd0af0ca5481c4475152f0b8686192951eefa60ee8711e1ede4 \ + --hash=sha256:b5df7341f54dbe1405a4450b2741207b4a8c279780402b45f87202b94dfc52b4 + # via xprof-nightly +frozenlist==1.8.0 \ + --hash=sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686 \ + --hash=sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0 \ + --hash=sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121 \ + --hash=sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd \ + --hash=sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7 \ + --hash=sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c \ + --hash=sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84 \ + --hash=sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d \ + --hash=sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b \ + --hash=sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79 \ + --hash=sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967 \ + --hash=sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f \ + --hash=sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4 \ + --hash=sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7 \ + --hash=sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef \ + --hash=sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9 \ + --hash=sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3 \ + --hash=sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd \ + --hash=sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087 \ + --hash=sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068 \ + --hash=sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7 \ + --hash=sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed \ + --hash=sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b \ + --hash=sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f \ + --hash=sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25 \ + --hash=sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe \ + --hash=sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143 \ + --hash=sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e \ + --hash=sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930 \ + --hash=sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37 \ + --hash=sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128 \ + --hash=sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2 \ + --hash=sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675 \ + --hash=sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f \ + --hash=sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746 \ + --hash=sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df \ + --hash=sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8 \ + --hash=sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c \ + --hash=sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0 \ + --hash=sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad \ + --hash=sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82 \ + --hash=sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29 \ + --hash=sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c \ + --hash=sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30 \ + --hash=sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf \ + --hash=sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62 \ + --hash=sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5 \ + --hash=sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383 \ + --hash=sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c \ + --hash=sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52 \ + --hash=sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d \ + --hash=sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1 \ + --hash=sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a \ + --hash=sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714 \ + --hash=sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65 \ + --hash=sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95 \ + --hash=sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1 \ + --hash=sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506 \ + --hash=sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888 \ + --hash=sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6 \ + --hash=sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41 \ + --hash=sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459 \ + --hash=sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a \ + --hash=sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608 \ + --hash=sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa \ + --hash=sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8 \ + --hash=sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1 \ + --hash=sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186 \ + --hash=sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6 \ + --hash=sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed \ + --hash=sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e \ + --hash=sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52 \ + --hash=sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231 \ + --hash=sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450 \ + --hash=sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496 \ + --hash=sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a \ + --hash=sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3 \ + --hash=sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24 \ + --hash=sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178 \ + --hash=sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695 \ + --hash=sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7 \ + --hash=sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4 \ + --hash=sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e \ + --hash=sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e \ + --hash=sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61 \ + --hash=sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca \ + --hash=sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad \ + --hash=sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b \ + --hash=sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a \ + --hash=sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8 \ + --hash=sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51 \ + --hash=sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011 \ + --hash=sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8 \ + --hash=sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103 \ + --hash=sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b \ + --hash=sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda \ + --hash=sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806 \ + --hash=sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042 \ + --hash=sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e \ + --hash=sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b \ + --hash=sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef \ + --hash=sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d \ + --hash=sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567 \ + --hash=sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a \ + --hash=sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2 \ + --hash=sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0 \ + --hash=sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e \ + --hash=sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b \ + --hash=sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d \ + --hash=sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a \ + --hash=sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52 \ + --hash=sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47 \ + --hash=sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1 \ + --hash=sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94 \ + --hash=sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f \ + --hash=sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff \ + --hash=sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822 \ + --hash=sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a \ + --hash=sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11 \ + --hash=sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581 \ + --hash=sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51 \ + --hash=sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565 \ + --hash=sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40 \ + --hash=sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92 \ + --hash=sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2 \ + --hash=sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5 \ + --hash=sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4 \ + --hash=sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93 \ + --hash=sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027 \ + --hash=sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd + # via + # aiohttp + # aiosignal +fsspec==2026.2.0 \ + --hash=sha256:6544e34b16869f5aacd5b90bdf1a71acb37792ea3ddf6125ee69a22a53fb8bff \ + --hash=sha256:98de475b5cb3bd66bedd5c4679e87b4fdfe1a3bf4d707b151b3c07e58c9a2437 + # via + # etils + # gcsfs + # xprof-nightly +gcsfs==2026.2.0 \ + --hash=sha256:407feaa2af0de81ebce44ea7e6f68598a3753e5e42257b61d6a9f8c0d6d4754e \ + --hash=sha256:d58a885d9e9c6227742b86da419c7a458e1f33c1de016e826ea2909f6338ed84 + # via xprof-nightly +google-api-core[grpc]==2.30.0 \ + --hash=sha256:02edfa9fab31e17fc0befb5f161b3bf93c9096d99aed584625f38065c511ad9b \ + --hash=sha256:80be49ee937ff9aba0fd79a6eddfde35fe658b9953ab9b79c57dd7061afa8df5 + # via + # google-cloud-core + # google-cloud-storage + # google-cloud-storage-control +google-auth==2.49.1 \ + --hash=sha256:16d40da1c3c5a0533f57d268fe72e0ebb0ae1cc3b567024122651c045d879b64 \ + --hash=sha256:195ebe3dca18eddd1b3db5edc5189b76c13e96f29e73043b923ebcf3f1a860f7 + # via + # gcsfs + # google-api-core + # google-auth-oauthlib + # google-cloud-core + # google-cloud-storage + # google-cloud-storage-control +google-auth-oauthlib==1.3.0 \ + --hash=sha256:386b3fb85cf4a5b819c6ad23e3128d975216b4cac76324de1d90b128aaf38f29 \ + --hash=sha256:cd39e807ac7229d6b8b9c1e297321d36fcc8a9e4857dff4301870985df51a528 + # via gcsfs +google-cloud-core==2.5.0 \ + --hash=sha256:67d977b41ae6c7211ee830c7912e41003ea8194bff15ae7d72fd6f51e57acabc \ + --hash=sha256:7c1b7ef5c92311717bd05301aa1a91ffbc565673d3b0b4163a52d8413a186963 + # via google-cloud-storage +google-cloud-storage==3.9.0 \ + --hash=sha256:2dce75a9e8b3387078cbbdad44757d410ecdb916101f8ba308abf202b6968066 \ + --hash=sha256:f2d8ca7db2f652be757e92573b2196e10fbc09649b5c016f8b422ad593c641cc + # via gcsfs +google-cloud-storage-control==1.10.0 \ + --hash=sha256:2bcbfa4ca6530d25a5baa8dbe80caf0eeabe4c6804798f4f107279719c316bdb \ + --hash=sha256:81d9dc6b50106836733adca868501f879f0d7a1c41503d887a1a1b9b9ddbf508 + # via gcsfs +google-crc32c==1.8.0 \ + --hash=sha256:014a7e68d623e9a4222d663931febc3033c5c7c9730785727de2a81f87d5bab8 \ + --hash=sha256:01f126a5cfddc378290de52095e2c7052be2ba7656a9f0caf4bcd1bfb1833f8a \ + --hash=sha256:0470b8c3d73b5f4e3300165498e4cf25221c7eb37f1159e221d1825b6df8a7ff \ + --hash=sha256:119fcd90c57c89f30040b47c211acee231b25a45d225e3225294386f5d258288 \ + --hash=sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411 \ + --hash=sha256:17446feb05abddc187e5441a45971b8394ea4c1b6efd88ab0af393fd9e0a156a \ + --hash=sha256:19b40d637a54cb71e0829179f6cb41835f0fbd9e8eb60552152a8b52c36cbe15 \ + --hash=sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb \ + --hash=sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa \ + --hash=sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962 \ + --hash=sha256:3d488e98b18809f5e322978d4506373599c0c13e6c5ad13e53bb44758e18d215 \ + --hash=sha256:3ebb04528e83b2634857f43f9bb8ef5b2bbe7f10f140daeb01b58f972d04736b \ + --hash=sha256:450dc98429d3e33ed2926fc99ee81001928d63460f8538f21a5d6060912a8e27 \ + --hash=sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113 \ + --hash=sha256:57a50a9035b75643996fbf224d6661e386c7162d1dfdab9bc4ca790947d1007f \ + --hash=sha256:61f58b28e0b21fcb249a8247ad0db2e64114e201e2e9b4200af020f3b6242c9f \ + --hash=sha256:6f35aaffc8ccd81ba3162443fabb920e65b1f20ab1952a31b13173a67811467d \ + --hash=sha256:71734788a88f551fbd6a97be9668a0020698e07b2bf5b3aa26a36c10cdfb27b2 \ + --hash=sha256:864abafe7d6e2c4c66395c1eb0fe12dc891879769b52a3d56499612ca93b6092 \ + --hash=sha256:86cfc00fe45a0ac7359e5214a1704e51a99e757d0272554874f419f79838c5f7 \ + --hash=sha256:87b0072c4ecc9505cfa16ee734b00cd7721d20a0f595be4d40d3d21b41f65ae2 \ + --hash=sha256:87fa445064e7db928226b2e6f0d5304ab4cd0339e664a4e9a25029f384d9bb93 \ + --hash=sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8 \ + --hash=sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21 \ + --hash=sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79 \ + --hash=sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2 \ + --hash=sha256:ba6aba18daf4d36ad4412feede6221414692f44d17e5428bdd81ad3fc1eee5dc \ + --hash=sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454 \ + --hash=sha256:d511b3153e7011a27ab6ee6bb3a5404a55b994dc1a7322c0b87b29606d9790e2 \ + --hash=sha256:db3fe8eaf0612fc8b20fa21a5f25bd785bc3cd5be69f8f3412b0ac2ffd49e733 \ + --hash=sha256:e6584b12cb06796d285d09e33f63309a09368b9d806a551d8036a4207ea43697 \ + --hash=sha256:f4b51844ef67d6cf2e9425983274da75f18b1597bb2c998e1c0a0e8d46f8f651 \ + --hash=sha256:f639065ea2042d5c034bf258a9f085eaa7af0cd250667c0635a3118e8f92c69c + # via + # google-cloud-storage + # google-resumable-media +google-resumable-media==2.8.0 \ + --hash=sha256:dd14a116af303845a8d932ddae161a26e86cc229645bc98b39f026f9b1717582 \ + --hash=sha256:f1157ed8b46994d60a1bc432544db62352043113684d4e030ee02e77ebe9a1ae + # via google-cloud-storage +googleapis-common-protos[grpc]==1.73.0 \ + --hash=sha256:778d07cd4fbeff84c6f7c72102f0daf98fa2bfd3fa8bea426edc545588da0b5a \ + --hash=sha256:dfdaaa2e860f242046be561e6d6cb5c5f1541ae02cfbcb034371aadb2942b4e8 + # via + # google-api-core + # grpc-google-iam-v1 + # grpcio-status +grpc-google-iam-v1==0.14.3 \ + --hash=sha256:7a7f697e017a067206a3dfef44e4c634a34d3dee135fe7d7a4613fe3e59217e6 \ + --hash=sha256:879ac4ef33136c5491a6300e27575a9ec760f6cdf9a2518798c1b8977a5dc389 + # via google-cloud-storage-control +grpcio==1.78.0 \ + --hash=sha256:082653eecbdf290e6e3e2c276ab2c54b9e7c299e07f4221872380312d8cf395e \ + --hash=sha256:10a9a644b5dd5aec3b82b5b0b90d41c0fa94c85ef42cb42cf78a23291ddb5e7d \ + --hash=sha256:12a771591ae40bc65ba67048fa52ef4f0e6db8279e595fd349f9dfddeef571f9 \ + --hash=sha256:185dea0d5260cbb2d224c507bf2a5444d5abbb1fa3594c1ed7e4c709d5eb8383 \ + --hash=sha256:1afa62af6e23f88629f2b29ec9e52ec7c65a7176c1e0a83292b93c76ca882558 \ + --hash=sha256:2045397e63a7a0ee7957c25f7dbb36ddc110e0cfb418403d110c0a7a68a844e9 \ + --hash=sha256:207db540302c884b8848036b80db352a832b99dfdf41db1eb554c2c2c7800f65 \ + --hash=sha256:271c73e6e5676afe4fc52907686670c7cea22ab2310b76a59b678403ed40d670 \ + --hash=sha256:2777b783f6c13b92bd7b716667452c329eefd646bfb3f2e9dabea2e05dbd34f6 \ + --hash=sha256:2bf5e2e163b356978b23652c4818ce4759d40f4712ee9ec5a83c4be6f8c23a3a \ + --hash=sha256:35eb275bf1751d2ffbd8f57cdbc46058e857cf3971041521b78b7db94bdaf127 \ + --hash=sha256:391e93548644e6b2726f1bb84ed60048d4bcc424ce5e4af0843d28ca0b754fec \ + --hash=sha256:3c586ac70e855c721bda8f548d38c3ca66ac791dc49b66a8281a1f99db85e452 \ + --hash=sha256:3f8904a8165ab21e07e58bf3e30a73f4dffc7a1e0dbc32d51c61b5360d26f43e \ + --hash=sha256:459ab414b35f4496138d0ecd735fed26f1318af5e52cb1efbc82a09f0d5aa911 \ + --hash=sha256:4c5533d03a6cbd7f56acfc9cfb44ea64f63d29091e40e44010d34178d392d7eb \ + --hash=sha256:51b13f9aed9d59ee389ad666b8c2214cc87b5de258fa712f9ab05f922e3896c6 \ + --hash=sha256:5361a0630a7fdb58a6a97638ab70e1dae2893c4d08d7aba64ded28bb9e7a29df \ + --hash=sha256:5397fff416b79e4b284959642a4e95ac4b0f1ece82c9993658e0e477d40551ec \ + --hash=sha256:57bab6deef2f4f1ca76cc04565df38dc5713ae6c17de690721bdf30cb1e0545c \ + --hash=sha256:6092beabe1966a3229f599d7088b38dfc8ffa1608b5b5cdda31e591e6500f856 \ + --hash=sha256:684083fd383e9dc04c794adb838d4faea08b291ce81f64ecd08e4577c7398adf \ + --hash=sha256:735e38e176a88ce41840c21bb49098ab66177c64c82426e24e0082500cc68af5 \ + --hash=sha256:7382b95189546f375c174f53a5fa873cef91c4b8005faa05cc5b3beea9c4f1c5 \ + --hash=sha256:748b6138585379c737adc08aeffd21222abbda1a86a0dca2a39682feb9196c20 \ + --hash=sha256:74be1268d1439eaaf552c698cdb11cd594f0c49295ae6bb72c34ee31abbe611b \ + --hash=sha256:7cc47943d524ee0096f973e1081cb8f4f17a4615f2116882a5f1416e4cfe92b5 \ + --hash=sha256:859b13906ce098c0b493af92142ad051bf64c7870fa58a123911c88606714996 \ + --hash=sha256:85f93781028ec63f383f6bc90db785a016319c561cc11151fbb7b34e0d012303 \ + --hash=sha256:86ce2371bfd7f212cf60d8517e5e854475c2c43ce14aa910e136ace72c6db6c1 \ + --hash=sha256:86f85dd7c947baa707078a236288a289044836d4b640962018ceb9cd1f899af5 \ + --hash=sha256:8dfffba826efcf366b1e3ccc37e67afe676f290e13a3b48d31a46739f80a8724 \ + --hash=sha256:8f2ac84905d12918e4e55a16da17939eb63e433dc11b677267c35568aa63fc84 \ + --hash=sha256:94309f498bcc07e5a7d16089ab984d42ad96af1d94b5a4eb966a266d9fcabf68 \ + --hash=sha256:94f95cf5d532d0e717eed4fc1810e8e6eded04621342ec54c89a7c2f14b581bf \ + --hash=sha256:9566fe4ababbb2610c39190791e5b829869351d14369603702e890ef3ad2d06e \ + --hash=sha256:9dca934f24c732750389ce49d638069c3892ad065df86cb465b3fa3012b70c9e \ + --hash=sha256:a9f136fbafe7ccf4ac7e8e0c28b31066e810be52d6e344ef954a3a70234e1702 \ + --hash=sha256:ab399ef5e3cd2a721b1038a0f3021001f19c5ab279f145e1146bb0b9f1b2b12c \ + --hash=sha256:b0c689c02947d636bc7fab3e30cc3a3445cca99c834dfb77cd4a6cabfc1c5597 \ + --hash=sha256:b2342d87af32790f934a79c3112641e7b27d63c261b8b4395350dad43eff1dc7 \ + --hash=sha256:b58f37edab4a3881bc6c9bca52670610e0c9ca14e2ea3cf9debf185b870457fb \ + --hash=sha256:bd8cb8026e5f5b50498a3c4f196f57f9db344dad829ffae16b82e4fdbaea2813 \ + --hash=sha256:be63c88b32e6c0f1429f1398ca5c09bc64b0d80950c8bb7807d7d7fb36fb84c7 \ + --hash=sha256:c3f293fdc675ccba4db5a561048cca627b5e7bd1c8a6973ffedabe7d116e22e2 \ + --hash=sha256:c41bc64626db62e72afec66b0c8a0da76491510015417c127bfc53b2fe6d7f7f \ + --hash=sha256:ce3a90455492bf8bfa38e56fbbe1dbd4f872a3d8eeaf7337dc3b1c8aa28c271b \ + --hash=sha256:ce7599575eeb25c0f4dc1be59cada6219f3b56176f799627f44088b21381a28a \ + --hash=sha256:dce09d6116df20a96acfdbf85e4866258c3758180e8c49845d6ba8248b6d0bbb \ + --hash=sha256:de8cb00d1483a412a06394b8303feec5dcb3b55f81d83aa216dbb6a0b86a94f5 \ + --hash=sha256:df2c8f3141f7cbd112a6ebbd760290b5849cda01884554f7c67acc14e7b1758a \ + --hash=sha256:e87cbc002b6f440482b3519e36e1313eb5443e9e9e73d6a52d43bd2004fcfd8e \ + --hash=sha256:e888474dee2f59ff68130f8a397792d8cb8e17e6b3434339657ba4ee90845a8c \ + --hash=sha256:f12857d24d98441af6a1d5c87442d624411db486f7ba12550b07788f74b67b04 \ + --hash=sha256:f2d4e43ee362adfc05994ed479334d5a451ab7bc3f3fee1b796b8ca66895acb4 \ + --hash=sha256:f3d6379493e18ad4d39537a82371c5281e153e963cecb13f953ebac155756525 \ + --hash=sha256:f8dff3d9777e5d2703a962ee5c286c239bf0ba173877cc68dc02c17d042e29de \ + --hash=sha256:f9ab915a267fc47c7e88c387a3a28325b58c898e23d4995f765728f4e3dedb97 \ + --hash=sha256:fbe6e89c7ffb48518384068321621b2a69cab509f58e40e4399fdd378fa6d074 \ + --hash=sha256:fd5f135b1bd58ab088930b3c613455796dfa0393626a6972663ccdda5b4ac6ce \ + --hash=sha256:ff870aebe9a93a85283837801d35cd5f8814fe2ad01e606861a7fb47c762a2b7 + # via + # google-api-core + # google-cloud-storage-control + # googleapis-common-protos + # grpc-google-iam-v1 + # grpcio-status +grpcio-status==1.78.0 \ + --hash=sha256:a34cfd28101bfea84b5aa0f936b4b423019e9213882907166af6b3bddc59e189 \ + --hash=sha256:b492b693d4bf27b47a6c32590701724f1d3b9444b36491878fb71f6208857f34 + # via google-api-core +gviz-api==1.10.0 \ + --hash=sha256:846692dd8cc73224fc31b18e41589bd934e1cc05090c6576af4b4b26c2e71b90 \ + --hash=sha256:a05055fed8c279f34f4b496eace7648c7fe9c1b06851e8a36e748541f1adbb05 + # via xprof-nightly +idna==3.11 \ + --hash=sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea \ + --hash=sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902 + # via + # requests + # yarl +jaraco-functools==4.4.0 \ + --hash=sha256:9eec1e36f45c818d9bf307c8948eb03b2b56cd44087b3cdc989abca1f20b9176 \ + --hash=sha256:da21933b0417b89515562656547a77b4931f98176eb173644c0d35032a33d6bb + # via cheroot +markupsafe==3.0.3 \ + --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f \ + --hash=sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a \ + --hash=sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf \ + --hash=sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19 \ + --hash=sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf \ + --hash=sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c \ + --hash=sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175 \ + --hash=sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219 \ + --hash=sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb \ + --hash=sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6 \ + --hash=sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab \ + --hash=sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26 \ + --hash=sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1 \ + --hash=sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce \ + --hash=sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218 \ + --hash=sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634 \ + --hash=sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695 \ + --hash=sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad \ + --hash=sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73 \ + --hash=sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c \ + --hash=sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe \ + --hash=sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa \ + --hash=sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559 \ + --hash=sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa \ + --hash=sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37 \ + --hash=sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758 \ + --hash=sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f \ + --hash=sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8 \ + --hash=sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d \ + --hash=sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c \ + --hash=sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97 \ + --hash=sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a \ + --hash=sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19 \ + --hash=sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9 \ + --hash=sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9 \ + --hash=sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc \ + --hash=sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2 \ + --hash=sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4 \ + --hash=sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354 \ + --hash=sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50 \ + --hash=sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698 \ + --hash=sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9 \ + --hash=sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b \ + --hash=sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc \ + --hash=sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115 \ + --hash=sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e \ + --hash=sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485 \ + --hash=sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f \ + --hash=sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12 \ + --hash=sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025 \ + --hash=sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009 \ + --hash=sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d \ + --hash=sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b \ + --hash=sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a \ + --hash=sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5 \ + --hash=sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f \ + --hash=sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d \ + --hash=sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1 \ + --hash=sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287 \ + --hash=sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6 \ + --hash=sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f \ + --hash=sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581 \ + --hash=sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed \ + --hash=sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b \ + --hash=sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c \ + --hash=sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026 \ + --hash=sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8 \ + --hash=sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676 \ + --hash=sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6 \ + --hash=sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e \ + --hash=sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d \ + --hash=sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d \ + --hash=sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01 \ + --hash=sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7 \ + --hash=sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419 \ + --hash=sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795 \ + --hash=sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1 \ + --hash=sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5 \ + --hash=sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d \ + --hash=sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42 \ + --hash=sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe \ + --hash=sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda \ + --hash=sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e \ + --hash=sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737 \ + --hash=sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523 \ + --hash=sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591 \ + --hash=sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc \ + --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \ + --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 + # via werkzeug +more-itertools==10.8.0 \ + --hash=sha256:52d4362373dcf7c52546bc4af9a86ee7c4579df9a8dc268be0a2f949d376cc9b \ + --hash=sha256:f638ddf8a1a0d134181275fb5d58b086ead7c6a72429ad725c67503f13ba30bd + # via + # cheroot + # jaraco-functools +multidict==6.7.1 \ + --hash=sha256:026d264228bcd637d4e060844e39cdc60f86c479e463d49075dedc21b18fbbe0 \ + --hash=sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9 \ + --hash=sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581 \ + --hash=sha256:067343c68cd6612d375710f895337b3a98a033c94f14b9a99eff902f205424e2 \ + --hash=sha256:08ccb2a6dc72009093ebe7f3f073e5ec5964cba9a706fa94b1a1484039b87941 \ + --hash=sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3 \ + --hash=sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43 \ + --hash=sha256:0d17522c37d03e85c8098ec8431636309b2682cf12e58f4dbc76121fb50e4962 \ + --hash=sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1 \ + --hash=sha256:0e697826df7eb63418ee190fd06ce9f1803593bb4b9517d08c60d9b9a7f69d8f \ + --hash=sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c \ + --hash=sha256:121a34e5bfa410cdf2c8c49716de160de3b1dbcd86b49656f5681e4543bcd1a8 \ + --hash=sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa \ + --hash=sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6 \ + --hash=sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c \ + --hash=sha256:17207077e29342fdc2c9a82e4b306f1127bf1ea91f8b71e02d4798a70bb99991 \ + --hash=sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262 \ + --hash=sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd \ + --hash=sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d \ + --hash=sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d \ + --hash=sha256:1fa6609d0364f4f6f58351b4659a1f3e0e898ba2a8c5cac04cb2c7bc556b0bc5 \ + --hash=sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3 \ + --hash=sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601 \ + --hash=sha256:24c0cf81544ca5e17cfcb6e482e7a82cd475925242b308b890c9452a074d4505 \ + --hash=sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0 \ + --hash=sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292 \ + --hash=sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed \ + --hash=sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362 \ + --hash=sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511 \ + --hash=sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23 \ + --hash=sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2 \ + --hash=sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb \ + --hash=sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e \ + --hash=sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582 \ + --hash=sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0 \ + --hash=sha256:3943debf0fbb57bdde5901695c11094a9a36723e5c03875f87718ee15ca2f4d2 \ + --hash=sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e \ + --hash=sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d \ + --hash=sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65 \ + --hash=sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a \ + --hash=sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd \ + --hash=sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d \ + --hash=sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108 \ + --hash=sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177 \ + --hash=sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144 \ + --hash=sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5 \ + --hash=sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd \ + --hash=sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5 \ + --hash=sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060 \ + --hash=sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37 \ + --hash=sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56 \ + --hash=sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df \ + --hash=sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963 \ + --hash=sha256:5884a04f4ff56c6120f6ccf703bdeb8b5079d808ba604d4d53aec0d55dc33568 \ + --hash=sha256:59bc83d3f66b41dac1e7460aac1d196edc70c9ba3094965c467715a70ecb46db \ + --hash=sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118 \ + --hash=sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84 \ + --hash=sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f \ + --hash=sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889 \ + --hash=sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71 \ + --hash=sha256:65573858d27cdeaca41893185677dc82395159aa28875a8867af66532d413a8f \ + --hash=sha256:6704fa2b7453b2fb121740555fa1ee20cd98c4d011120caf4d2b8d4e7c76eec0 \ + --hash=sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7 \ + --hash=sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048 \ + --hash=sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8 \ + --hash=sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49 \ + --hash=sha256:6f77ce314a29263e67adadc7e7c1bc699fcb3a305059ab973d038f87caa42ed0 \ + --hash=sha256:749aa54f578f2e5f439538706a475aa844bfa8ef75854b1401e6e528e4937cf9 \ + --hash=sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59 \ + --hash=sha256:7dfb78d966b2c906ae1d28ccf6e6712a3cd04407ee5088cd276fe8cb42186190 \ + --hash=sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709 \ + --hash=sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d \ + --hash=sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c \ + --hash=sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e \ + --hash=sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2 \ + --hash=sha256:8affcf1c98b82bc901702eb73b6947a1bfa170823c153fe8a47b5f5f02e48e40 \ + --hash=sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3 \ + --hash=sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee \ + --hash=sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609 \ + --hash=sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c \ + --hash=sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445 \ + --hash=sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1 \ + --hash=sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a \ + --hash=sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5 \ + --hash=sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31 \ + --hash=sha256:974e72a2474600827abaeda71af0c53d9ebbc3c2eb7da37b37d7829ae31232d8 \ + --hash=sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33 \ + --hash=sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7 \ + --hash=sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca \ + --hash=sha256:98c5787b0a0d9a41d9311eae44c3b76e6753def8d8870ab501320efe75a6a5f8 \ + --hash=sha256:9b0d9b91d1aa44db9c1f1ecd0d9d2ae610b2f4f856448664e01a3b35899f3f92 \ + --hash=sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733 \ + --hash=sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429 \ + --hash=sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9 \ + --hash=sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4 \ + --hash=sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6 \ + --hash=sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2 \ + --hash=sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172 \ + --hash=sha256:a9fc4caa29e2e6ae408d1c450ac8bf19892c5fca83ee634ecd88a53332c59981 \ + --hash=sha256:aa23b001d968faef416ff70dc0f1ab045517b9b42a90edd3e9bcdb06479e31d5 \ + --hash=sha256:ac1c665bad8b5d762f5f85ebe4d94130c26965f11de70c708c75671297c776de \ + --hash=sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52 \ + --hash=sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7 \ + --hash=sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c \ + --hash=sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2 \ + --hash=sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6 \ + --hash=sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf \ + --hash=sha256:bb08271280173720e9fea9ede98e5231defcbad90f1624bea26f32ec8a956e2f \ + --hash=sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b \ + --hash=sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961 \ + --hash=sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a \ + --hash=sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3 \ + --hash=sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b \ + --hash=sha256:c524c6fb8fc342793708ab111c4dbc90ff9abd568de220432500e47e990c0358 \ + --hash=sha256:c5f0c21549ab432b57dcc82130f388d84ad8179824cc3f223d5e7cfbfd4143f6 \ + --hash=sha256:c6b3228e1d80af737b72925ce5fb4daf5a335e49cd7ab77ed7b9fdfbf58c526e \ + --hash=sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1 \ + --hash=sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c \ + --hash=sha256:c93c3db7ea657dd4637d57e74ab73de31bccefe144d3d4ce370052035bc85fb5 \ + --hash=sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53 \ + --hash=sha256:cdea2e7b2456cfb6694fb113066fd0ec7ea4d67e3a35e1f4cbeea0b448bf5872 \ + --hash=sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e \ + --hash=sha256:cf37cbe5ced48d417ba045aca1b21bafca67489452debcde94778a576666a1df \ + --hash=sha256:d4f49cb5661344764e4c7c7973e92a47a59b8fc19b6523649ec9dc4960e58a03 \ + --hash=sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8 \ + --hash=sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a \ + --hash=sha256:d82dd730a95e6643802f4454b8fdecdf08667881a9c5670db85bc5a56693f122 \ + --hash=sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a \ + --hash=sha256:dd96c01a9dcd4889dcfcf9eb5544ca0c77603f239e3ffab0524ec17aea9a93ee \ + --hash=sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32 \ + --hash=sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3 \ + --hash=sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489 \ + --hash=sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23 \ + --hash=sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34 \ + --hash=sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75 \ + --hash=sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8 \ + --hash=sha256:eb351f72c26dc9abe338ca7294661aa22969ad8ffe7ef7d5541d19f368dc854a \ + --hash=sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d \ + --hash=sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855 \ + --hash=sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b \ + --hash=sha256:f537b55778cd3cbee430abe3131255d3a78202e0f9ea7ffc6ada893a4bcaeea4 \ + --hash=sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4 \ + --hash=sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d \ + --hash=sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0 \ + --hash=sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba \ + --hash=sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19 + # via + # aiohttp + # yarl +oauthlib==3.3.1 \ + --hash=sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9 \ + --hash=sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1 + # via requests-oauthlib +propcache==0.4.1 \ + --hash=sha256:0002004213ee1f36cfb3f9a42b5066100c44276b9b72b4e1504cddd3d692e86e \ + --hash=sha256:0013cb6f8dde4b2a2f66903b8ba740bdfe378c943c4377a200551ceb27f379e4 \ + --hash=sha256:005f08e6a0529984491e37d8dbc3dd86f84bd78a8ceb5fa9a021f4c48d4984be \ + --hash=sha256:031dce78b9dc099f4c29785d9cf5577a3faf9ebf74ecbd3c856a7b92768c3df3 \ + --hash=sha256:05674a162469f31358c30bcaa8883cb7829fa3110bf9c0991fe27d7896c42d85 \ + --hash=sha256:060b16ae65bc098da7f6d25bf359f1f31f688384858204fe5d652979e0015e5b \ + --hash=sha256:120c964da3fdc75e3731aa392527136d4ad35868cc556fd09bb6d09172d9a367 \ + --hash=sha256:15932ab57837c3368b024473a525e25d316d8353016e7cc0e5ba9eb343fbb1cf \ + --hash=sha256:17612831fda0138059cc5546f4d12a2aacfb9e47068c06af35c400ba58ba7393 \ + --hash=sha256:182b51b421f0501952d938dc0b0eb45246a5b5153c50d42b495ad5fb7517c888 \ + --hash=sha256:1cdb7988c4e5ac7f6d175a28a9aa0c94cb6f2ebe52756a3c0cda98d2809a9e37 \ + --hash=sha256:1eb2994229cc8ce7fe9b3db88f5465f5fd8651672840b2e426b88cdb1a30aac8 \ + --hash=sha256:1f0978529a418ebd1f49dad413a2b68af33f85d5c5ca5c6ca2a3bed375a7ac60 \ + --hash=sha256:204483131fb222bdaaeeea9f9e6c6ed0cac32731f75dfc1d4a567fc1926477c1 \ + --hash=sha256:296f4c8ed03ca7476813fe666c9ea97869a8d7aec972618671b33a38a5182ef4 \ + --hash=sha256:2ad890caa1d928c7c2965b48f3a3815c853180831d0e5503d35cf00c472f4717 \ + --hash=sha256:2b16ec437a8c8a965ecf95739448dd938b5c7f56e67ea009f4300d8df05f32b7 \ + --hash=sha256:2bb07ffd7eaad486576430c89f9b215f9e4be68c4866a96e97db9e97fead85dc \ + --hash=sha256:333ddb9031d2704a301ee3e506dc46b1fe5f294ec198ed6435ad5b6a085facfe \ + --hash=sha256:357f5bb5c377a82e105e44bd3d52ba22b616f7b9773714bff93573988ef0a5fb \ + --hash=sha256:35c3277624a080cc6ec6f847cbbbb5b49affa3598c4535a0a4682a697aaa5c75 \ + --hash=sha256:364426a62660f3f699949ac8c621aad6977be7126c5807ce48c0aeb8e7333ea6 \ + --hash=sha256:381914df18634f5494334d201e98245c0596067504b9372d8cf93f4bb23e025e \ + --hash=sha256:3d233076ccf9e450c8b3bc6720af226b898ef5d051a2d145f7d765e6e9f9bcff \ + --hash=sha256:3d902a36df4e5989763425a8ab9e98cd8ad5c52c823b34ee7ef307fd50582566 \ + --hash=sha256:3f7124c9d820ba5548d431afb4632301acf965db49e666aa21c305cbe8c6de12 \ + --hash=sha256:405aac25c6394ef275dee4c709be43745d36674b223ba4eb7144bf4d691b7367 \ + --hash=sha256:41a89040cb10bd345b3c1a873b2bf36413d48da1def52f268a055f7398514874 \ + --hash=sha256:43eedf29202c08550aac1d14e0ee619b0430aaef78f85864c1a892294fbc28cf \ + --hash=sha256:473c61b39e1460d386479b9b2f337da492042447c9b685f28be4f74d3529e566 \ + --hash=sha256:49a2dc67c154db2c1463013594c458881a069fcf98940e61a0569016a583020a \ + --hash=sha256:4b536b39c5199b96fc6245eb5fb796c497381d3942f169e44e8e392b29c9ebcc \ + --hash=sha256:4c3c70630930447f9ef1caac7728c8ad1c56bc5015338b20fed0d08ea2480b3a \ + --hash=sha256:4d3df5fa7e36b3225954fba85589da77a0fe6a53e3976de39caf04a0db4c36f1 \ + --hash=sha256:4d7af63f9f93fe593afbf104c21b3b15868efb2c21d07d8732c0c4287e66b6a6 \ + --hash=sha256:501d20b891688eb8e7aa903021f0b72d5a55db40ffaab27edefd1027caaafa61 \ + --hash=sha256:521a463429ef54143092c11a77e04056dd00636f72e8c45b70aaa3140d639726 \ + --hash=sha256:5558992a00dfd54ccbc64a32726a3357ec93825a418a401f5cc67df0ac5d9e49 \ + --hash=sha256:55c72fd6ea2da4c318e74ffdf93c4fe4e926051133657459131a95c846d16d44 \ + --hash=sha256:564d9f0d4d9509e1a870c920a89b2fec951b44bf5ba7d537a9e7c1ccec2c18af \ + --hash=sha256:580e97762b950f993ae618e167e7be9256b8353c2dcd8b99ec100eb50f5286aa \ + --hash=sha256:5a103c3eb905fcea0ab98be99c3a9a5ab2de60228aa5aceedc614c0281cf6153 \ + --hash=sha256:5c3310452e0d31390da9035c348633b43d7e7feb2e37be252be6da45abd1abcc \ + --hash=sha256:5d4e2366a9c7b837555cf02fb9be2e3167d333aff716332ef1b7c3a142ec40c5 \ + --hash=sha256:5fd37c406dd6dc85aa743e214cef35dc54bbdd1419baac4f6ae5e5b1a2976938 \ + --hash=sha256:60a8fda9644b7dfd5dece8c61d8a85e271cb958075bfc4e01083c148b61a7caf \ + --hash=sha256:66c1f011f45a3b33d7bcb22daed4b29c0c9e2224758b6be00686731e1b46f925 \ + --hash=sha256:671538c2262dadb5ba6395e26c1731e1d52534bfe9ae56d0b5573ce539266aa8 \ + --hash=sha256:678ae89ebc632c5c204c794f8dab2837c5f159aeb59e6ed0539500400577298c \ + --hash=sha256:67fad6162281e80e882fb3ec355398cf72864a54069d060321f6cd0ade95fe85 \ + --hash=sha256:6918ecbd897443087a3b7cd978d56546a812517dcaaca51b49526720571fa93e \ + --hash=sha256:6f6ff873ed40292cd4969ef5310179afd5db59fdf055897e282485043fc80ad0 \ + --hash=sha256:6f8b465489f927b0df505cbe26ffbeed4d6d8a2bbc61ce90eb074ff129ef0ab1 \ + --hash=sha256:71b749281b816793678ae7f3d0d84bd36e694953822eaad408d682efc5ca18e0 \ + --hash=sha256:74c1fb26515153e482e00177a1ad654721bf9207da8a494a0c05e797ad27b992 \ + --hash=sha256:7c2d1fa3201efaf55d730400d945b5b3ab6e672e100ba0f9a409d950ab25d7db \ + --hash=sha256:824e908bce90fb2743bd6b59db36eb4f45cd350a39637c9f73b1c1ea66f5b75f \ + --hash=sha256:8326e144341460402713f91df60ade3c999d601e7eb5ff8f6f7862d54de0610d \ + --hash=sha256:8873eb4460fd55333ea49b7d189749ecf6e55bf85080f11b1c4530ed3034cba1 \ + --hash=sha256:89eb3fa9524f7bec9de6e83cf3faed9d79bffa560672c118a96a171a6f55831e \ + --hash=sha256:8c9b3cbe4584636d72ff556d9036e0c9317fa27b3ac1f0f558e7e84d1c9c5900 \ + --hash=sha256:8e57061305815dfc910a3634dcf584f08168a8836e6999983569f51a8544cd89 \ + --hash=sha256:929d7cbe1f01bb7baffb33dc14eb5691c95831450a26354cd210a8155170c93a \ + --hash=sha256:92d1935ee1f8d7442da9c0c4fa7ac20d07e94064184811b685f5c4fada64553b \ + --hash=sha256:948dab269721ae9a87fd16c514a0a2c2a1bdb23a9a61b969b0f9d9ee2968546f \ + --hash=sha256:981333cb2f4c1896a12f4ab92a9cc8f09ea664e9b7dbdc4eff74627af3a11c0f \ + --hash=sha256:990f6b3e2a27d683cb7602ed6c86f15ee6b43b1194736f9baaeb93d0016633b1 \ + --hash=sha256:99d43339c83aaf4d32bda60928231848eee470c6bda8d02599cc4cebe872d183 \ + --hash=sha256:9a0bd56e5b100aef69bd8562b74b46254e7c8812918d3baa700c8a8009b0af66 \ + --hash=sha256:9a52009f2adffe195d0b605c25ec929d26b36ef986ba85244891dee3b294df21 \ + --hash=sha256:9d2b6caef873b4f09e26ea7e33d65f42b944837563a47a94719cc3544319a0db \ + --hash=sha256:9f302f4783709a78240ebc311b793f123328716a60911d667e0c036bc5dcbded \ + --hash=sha256:a0ee98db9c5f80785b266eb805016e36058ac72c51a064040f2bc43b61101cdb \ + --hash=sha256:a129e76735bc792794d5177069691c3217898b9f5cee2b2661471e52ffe13f19 \ + --hash=sha256:a78372c932c90ee474559c5ddfffd718238e8673c340dc21fe45c5b8b54559a0 \ + --hash=sha256:a9695397f85973bb40427dedddf70d8dc4a44b22f1650dd4af9eedf443d45165 \ + --hash=sha256:ab08df6c9a035bee56e31af99be621526bd237bea9f32def431c656b29e41778 \ + --hash=sha256:ab2943be7c652f09638800905ee1bab2c544e537edb57d527997a24c13dc1455 \ + --hash=sha256:ab4c29b49d560fe48b696cdcb127dd36e0bc2472548f3bf56cc5cb3da2b2984f \ + --hash=sha256:af223b406d6d000830c6f65f1e6431783fc3f713ba3e6cc8c024d5ee96170a4b \ + --hash=sha256:af2a6052aeb6cf17d3e46ee169099044fd8224cbaf75c76a2ef596e8163e2237 \ + --hash=sha256:bcc9aaa5d80322bc2fb24bb7accb4a30f81e90ab8d6ba187aec0744bc302ad81 \ + --hash=sha256:c07fda85708bc48578467e85099645167a955ba093be0a2dcba962195676e859 \ + --hash=sha256:c0d4b719b7da33599dfe3b22d3db1ef789210a0597bc650b7cee9c77c2be8c5c \ + --hash=sha256:c0ef0aaafc66fbd87842a3fe3902fd889825646bc21149eafe47be6072725835 \ + --hash=sha256:c2b5e7db5328427c57c8e8831abda175421b709672f6cfc3d630c3b7e2146393 \ + --hash=sha256:c30b53e7e6bda1d547cabb47c825f3843a0a1a42b0496087bb58d8fedf9f41b5 \ + --hash=sha256:c80ee5802e3fb9ea37938e7eecc307fb984837091d5fd262bb37238b1ae97641 \ + --hash=sha256:c9b822a577f560fbd9554812526831712c1436d2c046cedee4c3796d3543b144 \ + --hash=sha256:cae65ad55793da34db5f54e4029b89d3b9b9490d8abe1b4c7ab5d4b8ec7ebf74 \ + --hash=sha256:cb2d222e72399fcf5890d1d5cc1060857b9b236adff2792ff48ca2dfd46c81db \ + --hash=sha256:cbc3b6dfc728105b2a57c06791eb07a94229202ea75c59db644d7d496b698cac \ + --hash=sha256:cd547953428f7abb73c5ad82cbb32109566204260d98e41e5dfdc682eb7f8403 \ + --hash=sha256:cfc27c945f422e8b5071b6e93169679e4eb5bf73bbcbf1ba3ae3a83d2f78ebd9 \ + --hash=sha256:d472aeb4fbf9865e0c6d622d7f4d54a4e101a89715d8904282bb5f9a2f476c3f \ + --hash=sha256:d62cdfcfd89ccb8de04e0eda998535c406bf5e060ffd56be6c586cbcc05b3311 \ + --hash=sha256:d82ad62b19645419fe79dd63b3f9253e15b30e955c0170e5cebc350c1844e581 \ + --hash=sha256:d8f353eb14ee3441ee844ade4277d560cdd68288838673273b978e3d6d2c8f36 \ + --hash=sha256:daede9cd44e0f8bdd9e6cc9a607fc81feb80fae7a5fc6cecaff0e0bb32e42d00 \ + --hash=sha256:db65d2af507bbfbdcedb254a11149f894169d90488dd3e7190f7cdcb2d6cd57a \ + --hash=sha256:dee69d7015dc235f526fe80a9c90d65eb0039103fe565776250881731f06349f \ + --hash=sha256:e153e9cd40cc8945138822807139367f256f89c6810c2634a4f6902b52d3b4e2 \ + --hash=sha256:e35b88984e7fa64aacecea39236cee32dd9bd8c55f57ba8a75cf2399553f9bd7 \ + --hash=sha256:e53f3a38d3510c11953f3e6a33f205c6d1b001129f972805ca9b42fc308bc239 \ + --hash=sha256:e9b0d8d0845bbc4cfcdcbcdbf5086886bc8157aa963c31c777ceff7846c77757 \ + --hash=sha256:ec17c65562a827bba85e3872ead335f95405ea1674860d96483a02f5c698fa72 \ + --hash=sha256:ecef2343af4cc68e05131e45024ba34f6095821988a9d0a02aa7c73fcc448aa9 \ + --hash=sha256:ed5a841e8bb29a55fb8159ed526b26adc5bdd7e8bd7bf793ce647cb08656cdf4 \ + --hash=sha256:ee17f18d2498f2673e432faaa71698032b0127ebf23ae5974eeaf806c279df24 \ + --hash=sha256:f048da1b4f243fc44f205dfd320933a951b8d89e0afd4c7cacc762a8b9165207 \ + --hash=sha256:f10207adf04d08bec185bae14d9606a1444715bc99180f9331c9c02093e1959e \ + --hash=sha256:f1d2f90aeec838a52f1c1a32fe9a619fefd5e411721a9117fbf82aea638fe8a1 \ + --hash=sha256:f48107a8c637e80362555f37ecf49abe20370e557cc4ab374f04ec4423c97c3d \ + --hash=sha256:f7ee0e597f495cf415bcbd3da3caa3bd7e816b74d0d52b8145954c5e6fd3ff37 \ + --hash=sha256:f93243fdc5657247533273ac4f86ae106cc6445a0efacb9a1bfe982fcfefd90c \ + --hash=sha256:f95393b4d66bfae908c3ca8d169d5f79cd65636ae15b5e7a4f6e67af675adb0e \ + --hash=sha256:fc38cba02d1acba4e2869eef1a57a43dfbd3d49a59bf90dda7444ec2be6a5570 \ + --hash=sha256:fd0858c20f078a32cf55f7e81473d96dcf3b93fd2ccdb3d40fdf54b8573df3af \ + --hash=sha256:fd138803047fb4c062b1c1dd95462f5209456bfab55c734458f15d11da288f8f \ + --hash=sha256:fd2dbc472da1f772a4dae4fa24be938a6c544671a912e30529984dd80400cd88 \ + --hash=sha256:fd6f30fdcf9ae2a70abd34da54f18da086160e4d7d9251f81f3da0ff84fc5a48 \ + --hash=sha256:fe49d0a85038f36ba9e3ffafa1103e61170b28e95b16622e11be0a0ea07c6781 + # via + # aiohttp + # yarl +proto-plus==1.27.1 \ + --hash=sha256:912a7460446625b792f6448bade9e55cd4e41e6ac10e27009ef71a7f317fa147 \ + --hash=sha256:e4643061f3a4d0de092d62aa4ad09fa4756b2cbb89d4627f3985018216f9fefc + # via + # google-api-core + # google-cloud-storage-control +protobuf==6.33.5 \ + --hash=sha256:3093804752167bcab3998bec9f1048baae6e29505adaf1afd14a37bddede533c \ + --hash=sha256:69915a973dd0f60f31a08b8318b73eab2bd6a392c79184b3612226b0a3f8ec02 \ + --hash=sha256:6ddcac2a081f8b7b9642c09406bc6a4290128fce5f471cddd165960bb9119e5c \ + --hash=sha256:8afa18e1d6d20af15b417e728e9f60f3aa108ee76f23c3b2c07a2c3b546d3afd \ + --hash=sha256:8f04fa32763dcdb4973d537d6b54e615cc61108c7cb38fe59310c3192d29510a \ + --hash=sha256:9b71e0281f36f179d00cbcb119cb19dec4d14a81393e5ea220f64b286173e190 \ + --hash=sha256:a3157e62729aafb8df6da2c03aa5c0937c7266c626ce11a278b6eb7963c4e37c \ + --hash=sha256:a5cb85982d95d906df1e2210e58f8e4f1e3cdc088e52c921a041f9c9a0386de5 \ + --hash=sha256:cbf16ba3350fb7b889fca858fb215967792dc125b35c7976ca4818bee3521cf0 \ + --hash=sha256:d71b040839446bac0f4d162e758bea99c8251161dae9d0983a3b88dee345153b + # via + # google-api-core + # google-cloud-storage-control + # googleapis-common-protos + # grpc-google-iam-v1 + # grpcio-status + # proto-plus + # xprof-nightly +pyasn1==0.6.3 \ + --hash=sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf \ + --hash=sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde + # via pyasn1-modules +pyasn1-modules==0.4.2 \ + --hash=sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a \ + --hash=sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6 + # via google-auth +pycparser==3.0 \ + --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 + # via cffi +requests==2.32.5 \ + --hash=sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6 \ + --hash=sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf + # via + # gcsfs + # google-api-core + # google-cloud-storage + # requests-oauthlib +requests-oauthlib==2.0.0 \ + --hash=sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36 \ + --hash=sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9 + # via google-auth-oauthlib +setuptools==82.0.1 \ + --hash=sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9 \ + --hash=sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb + # via xprof-nightly +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via + # gviz-api + # xprof-nightly +typing-extensions==4.15.0 \ + --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ + --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 + # via + # aiosignal + # etils + # grpcio +urllib3==2.6.3 \ + --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ + --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 + # via requests +werkzeug==3.1.6 \ + --hash=sha256:210c6bede5a420a913956b4791a7f4d6843a43b6fcee4dfa08a65e93007d0d25 \ + --hash=sha256:7ddf3357bb9564e407607f988f683d72038551200c704012bb9a4c523d42f131 + # via xprof-nightly +xprof-nightly==2.23.0a20260317 \ + --hash=sha256:25d30efd382782f4e3689868e38cb963d0d152197c09fbde972c79ab4db06754 \ + --hash=sha256:295daa79f8ddf39e3575befa2ebf50b9b6f7356f19ec26f2a673dbf36f02e7d6 \ + --hash=sha256:3dd619c9aaf13b96c68b291e2857635407b37bdcb37bb943991fb42aaa2375c0 \ + --hash=sha256:5f49a4ebe94569316295fdbe501853b72a22c212d0d377dd9ff0dc96021b3fea \ + --hash=sha256:6364e51d1581d4d837be6b0b3783e4ba004200cde4266e5a28ad1f9c7049ddb0 \ + --hash=sha256:79743fae634e8cff896fd0be6c84bf726bd251a554a6b0c18e06b6c83e52953a \ + --hash=sha256:89a5ee026f670c409b5f880ebf02a24660736c2eef8d8f10da660d0b4438970f \ + --hash=sha256:a49a6f19d7505414196d6c06238161fbbfbc7bb2049095071e7427c79019f34e \ + --hash=sha256:a62a9c1a3670cb13446dcdc840a61370e19ceadc6f2d41307e2ab4677fd01d15 \ + --hash=sha256:ac61b96fb1a4accba93a4ede9a67ff2f6151466bcba37835c35a6b47501a19c7 \ + --hash=sha256:de1eff60d3164b06f9044f0c9ba12d69fc36f2abe55c81f7a76e6ec09e270c20 \ + --hash=sha256:ee7520f6ab3894ed08c4d77fe5fc1227b96aa8923452491d0104a2968c3d283e + # via -r tools/xprof/requirements.in +yarl==1.23.0 \ + --hash=sha256:03214408cfa590df47728b84c679ae4ef00be2428e11630277be0727eba2d7cc \ + --hash=sha256:041b1a4cefacf65840b4e295c6985f334ba83c30607441ae3cf206a0eed1a2e4 \ + --hash=sha256:0793e2bd0cf14234983bbb371591e6bea9e876ddf6896cdcc93450996b0b5c85 \ + --hash=sha256:0e1fdaa14ef51366d7757b45bde294e95f6c8c049194e793eedb8387c86d5993 \ + --hash=sha256:0e40111274f340d32ebcc0a5668d54d2b552a6cca84c9475859d364b380e3222 \ + --hash=sha256:115136c4a426f9da976187d238e84139ff6b51a20839aa6e3720cd1026d768de \ + --hash=sha256:13a563739ae600a631c36ce096615fe307f131344588b0bc0daec108cdb47b25 \ + --hash=sha256:16c6994ac35c3e74fb0ae93323bf8b9c2a9088d55946109489667c510a7d010e \ + --hash=sha256:170e26584b060879e29fac213e4228ef063f39128723807a312e5c7fec28eff2 \ + --hash=sha256:17235362f580149742739cc3828b80e24029d08cbb9c4bda0242c7b5bc610a8e \ + --hash=sha256:1932b6b8bba8d0160a9d1078aae5838a66039e8832d41d2992daa9a3a08f7860 \ + --hash=sha256:1b6b572edd95b4fa8df75de10b04bc81acc87c1c7d16bcdd2035b09d30acc957 \ + --hash=sha256:1c3a3598a832590c5a3ce56ab5576361b5688c12cb1d39429cf5dba30b510760 \ + --hash=sha256:1c57676bdedc94cd3bc37724cf6f8cd2779f02f6aba48de45feca073e714fe52 \ + --hash=sha256:1dc702e42d0684f42d6519c8d581e49c96cefaaab16691f03566d30658ee8788 \ + --hash=sha256:21d1b7305a71a15b4794b5ff22e8eef96ff4a6d7f9657155e5aa419444b28912 \ + --hash=sha256:23f371bd662cf44a7630d4d113101eafc0cfa7518a2760d20760b26021454719 \ + --hash=sha256:2569b67d616eab450d262ca7cb9f9e19d2f718c70a8b88712859359d0ab17035 \ + --hash=sha256:263cd4f47159c09b8b685890af949195b51d1aa82ba451c5847ca9bc6413c220 \ + --hash=sha256:2803ed8b21ca47a43da80a6fd1ed3019d30061f7061daa35ac54f63933409412 \ + --hash=sha256:2a6940a074fb3c48356ed0158a3ca5699c955ee4185b4d7d619be3c327143e05 \ + --hash=sha256:2e27c8841126e017dd2a054a95771569e6070b9ee1b133366d8b31beb5018a41 \ + --hash=sha256:31c9921eb8bd12633b41ad27686bbb0b1a2a9b8452bfdf221e34f311e9942ed4 \ + --hash=sha256:34b6cf500e61c90f305094911f9acc9c86da1a05a7a3f5be9f68817043f486e4 \ + --hash=sha256:3650dc2480f94f7116c364096bc84b1d602f44224ef7d5c7208425915c0475dd \ + --hash=sha256:389871e65468400d6283c0308e791a640b5ab5c83bcee02a2f51295f95e09748 \ + --hash=sha256:39004f0ad156da43e86aa71f44e033de68a44e5a31fc53507b36dd253970054a \ + --hash=sha256:394906945aa8b19fc14a61cf69743a868bb8c465efe85eee687109cc540b98f4 \ + --hash=sha256:3ceb13c5c858d01321b5d9bb65e4cf37a92169ea470b70fec6f236b2c9dd7e34 \ + --hash=sha256:411225bae281f114067578891bc75534cfb3d92a3b4dfef7a6ca78ba354e6069 \ + --hash=sha256:44bb7bef4ea409384e3f8bc36c063d77ea1b8d4a5b2706956c0d6695f07dcc25 \ + --hash=sha256:4503053d296bc6e4cbd1fad61cf3b6e33b939886c4f249ba7c78b602214fabe2 \ + --hash=sha256:4764a6a7588561a9aef92f65bda2c4fb58fe7c675c0883862e6df97559de0bfb \ + --hash=sha256:4966242ec68afc74c122f8459abd597afd7d8a60dc93d695c1334c5fd25f762f \ + --hash=sha256:4a42e651629dafb64fd5b0286a3580613702b5809ad3f24934ea87595804f2c5 \ + --hash=sha256:4a59ba56f340334766f3a4442e0efd0af895fae9e2b204741ef885c446b3a1a8 \ + --hash=sha256:4c41e021bc6d7affb3364dc1e1e5fa9582b470f283748784bd6ea0558f87f42c \ + --hash=sha256:5023346c4ee7992febc0068e7593de5fa2bf611848c08404b35ebbb76b1b0512 \ + --hash=sha256:50f9d8d531dfb767c565f348f33dd5139a6c43f5cbdf3f67da40d54241df93f6 \ + --hash=sha256:51430653db848d258336cfa0244427b17d12db63d42603a55f0d4546f50f25b5 \ + --hash=sha256:531ef597132086b6cf96faa7c6c1dcd0361dd5f1694e5cc30375907b9b7d3ea9 \ + --hash=sha256:53ad387048f6f09a8969631e4de3f1bf70c50e93545d64af4f751b2498755072 \ + --hash=sha256:53b1ea6ca88ebd4420379c330aea57e258408dd0df9af0992e5de2078dc9f5d5 \ + --hash=sha256:575aa4405a656e61a540f4a80eaa5260f2a38fff7bfdc4b5f611840d76e9e277 \ + --hash=sha256:578110dd426f0d209d1509244e6d4a3f1a3e9077655d98c5f22583d63252a08a \ + --hash=sha256:5ec2f42d41ccbd5df0270d7df31618a8ee267bfa50997f5d720ddba86c4a83a6 \ + --hash=sha256:5ee586fb17ff8f90c91cf73c6108a434b02d69925f44f5f8e0d7f2f260607eae \ + --hash=sha256:5f10fd85e4b75967468af655228fbfd212bdf66db1c0d135065ce288982eda26 \ + --hash=sha256:609d3614d78d74ebe35f54953c5bbd2ac647a7ddb9c30a5d877580f5e86b22f2 \ + --hash=sha256:62694e275c93d54f7ccedcfef57d42761b2aad5234b6be1f3e3026cae4001cd4 \ + --hash=sha256:63e92247f383c85ab00dd0091e8c3fa331a96e865459f5ee80353c70a4a42d70 \ + --hash=sha256:682bae25f0a0dd23a056739f23a134db9f52a63e2afd6bfb37ddc76292bbd723 \ + --hash=sha256:6b41389c19b07c760c7e427a3462e8ab83c4bb087d127f0e854c706ce1b9215c \ + --hash=sha256:6e87a6e8735b44816e7db0b2fbc9686932df473c826b0d9743148432e10bb9b9 \ + --hash=sha256:6f0fd84de0c957b2d280143522c4f91a73aada1923caee763e24a2b3fda9f8a5 \ + --hash=sha256:70efd20be968c76ece7baa8dafe04c5be06abc57f754d6f36f3741f7aa7a208e \ + --hash=sha256:71d006bee8397a4a89f469b8deb22469fe7508132d3c17fa6ed871e79832691c \ + --hash=sha256:73309162a6a571d4cbd3b6a1dcc703c7311843ae0d1578df6f09be4e98df38d4 \ + --hash=sha256:75e3026ab649bf48f9a10c0134512638725b521340293f202a69b567518d94e0 \ + --hash=sha256:76855800ac56f878847a09ce6dba727c93ca2d89c9e9d63002d26b916810b0a2 \ + --hash=sha256:7c6b9461a2a8b47c65eef63bb1c76a4f1c119618ffa99ea79bc5bb1e46c5821b \ + --hash=sha256:803a3c3ce4acc62eaf01eaca1208dcf0783025ef27572c3336502b9c232005e7 \ + --hash=sha256:80e6d33a3d42a7549b409f199857b4fb54e2103fc44fb87605b6663b7a7ff750 \ + --hash=sha256:8419ebd326430d1cbb7efb5292330a2cf39114e82df5cc3d83c9a0d5ebeaf2f2 \ + --hash=sha256:85610b4f27f69984932a7abbe52703688de3724d9f72bceb1cca667deff27474 \ + --hash=sha256:85e9beda1f591bc73e77ea1c51965c68e98dafd0fec72cdd745f77d727466716 \ + --hash=sha256:877b0738624280e34c55680d6054a307aa94f7d52fa0e3034a9cc6e790871da7 \ + --hash=sha256:88f9fb0116fbfcefcab70f85cf4b74a2b6ce5d199c41345296f49d974ddb4123 \ + --hash=sha256:8c4fe09e0780c6c3bf2b7d4af02ee2394439d11a523bbcf095cf4747c2932007 \ + --hash=sha256:93a784271881035ab4406a172edb0faecb6e7d00f4b53dc2f55919d6c9688595 \ + --hash=sha256:94f8575fbdf81749008d980c17796097e645574a3b8c28ee313931068dad14fe \ + --hash=sha256:95451e6ce06c3e104556d73b559f5da6c34a069b6b62946d3ad66afcd51642ea \ + --hash=sha256:99c8a9ed30f4164bc4c14b37a90208836cbf50d4ce2a57c71d0f52c7fb4f7598 \ + --hash=sha256:9a18d6f9359e45722c064c97464ec883eb0e0366d33eda61cb19a244bf222679 \ + --hash=sha256:9cbf44c5cb4a7633d078788e1b56387e3d3cf2b8139a3be38040b22d6c3221c8 \ + --hash=sha256:9ee33b875f0b390564c1fb7bc528abf18c8ee6073b201c6ae8524aca778e2d83 \ + --hash=sha256:a0e317df055958a0c1e79e5d2aa5a5eaa4a6d05a20d4b0c9c3f48918139c9fc6 \ + --hash=sha256:a2df6afe50dea8ae15fa34c9f824a3ee958d785fd5d089063d960bae1daa0a3f \ + --hash=sha256:a31de1613658308efdb21ada98cbc86a97c181aa050ba22a808120bb5be3ab94 \ + --hash=sha256:a3d2bff8f37f8d0f96c7ec554d16945050d54462d6e95414babaa18bfafc7f51 \ + --hash=sha256:a41bcf68efd19073376eb8cf948b8d9be0af26256403e512bb18f3966f1f9120 \ + --hash=sha256:a82836cab5f197a0514235aaf7ffccdc886ccdaa2324bc0aafdd4ae898103039 \ + --hash=sha256:a8d00f29b42f534cc8aa3931cfe773b13b23e561e10d2b26f27a8d309b0e82a1 \ + --hash=sha256:aafe5dcfda86c8af00386d7781d4c2181b5011b7be3f2add5e99899ea925df05 \ + --hash=sha256:ab5f043cb8a2d71c981c09c510da013bc79fd661f5c60139f00dd3c3cc4f2ffb \ + --hash=sha256:ac09d42f48f80c9ee1635b2fcaa819496a44502737660d3c0f2ade7526d29144 \ + --hash=sha256:aecfed0b41aa72b7881712c65cf764e39ce2ec352324f5e0837c7048d9e6daaa \ + --hash=sha256:b2c6b50c7b0464165472b56b42d4c76a7b864597007d9c085e8b63e185cf4a7a \ + --hash=sha256:b35d13d549077713e4414f927cdc388d62e543987c572baee613bf82f11a4b99 \ + --hash=sha256:b39cb32a6582750b6cc77bfb3c49c0f8760dc18dc96ec9fb55fbb0f04e08b928 \ + --hash=sha256:b5405bb8f0e783a988172993cfc627e4d9d00432d6bbac65a923041edacf997d \ + --hash=sha256:baaf55442359053c7d62f6f8413a62adba3205119bcb6f49594894d8be47e5e3 \ + --hash=sha256:bd654fad46d8d9e823afbb4f87c79160b5a374ed1ff5bde24e542e6ba8f41434 \ + --hash=sha256:be61f6fff406ca40e3b1d84716fde398fc08bc63dd96d15f3a14230a0973ed86 \ + --hash=sha256:bf49a3ae946a87083ef3a34c8f677ae4243f5b824bfc4c69672e72b3d6719d46 \ + --hash=sha256:c4a80f77dc1acaaa61f0934176fccca7096d9b1ff08c8ba9cddf5ae034a24319 \ + --hash=sha256:c75eb09e8d55bceb4367e83496ff8ef2bc7ea6960efb38e978e8073ea59ecb67 \ + --hash=sha256:c7f8dc16c498ff06497c015642333219871effba93e4a2e8604a06264aca5c5c \ + --hash=sha256:c8aa34a5c864db1087d911a0b902d60d203ea3607d91f615acd3f3108ac32169 \ + --hash=sha256:cbb0fef01f0c6b38cb0f39b1f78fc90b807e0e3c86a7ff3ce74ad77ce5c7880c \ + --hash=sha256:cde9a2ecd91668bcb7f077c4966d8ceddb60af01b52e6e3e2680e4cf00ad1a59 \ + --hash=sha256:cff6d44cb13d39db2663a22b22305d10855efa0fa8015ddeacc40bc59b9d8107 \ + --hash=sha256:d1009abedb49ae95b136a8904a3f71b342f849ffeced2d3747bf29caeda218c4 \ + --hash=sha256:d38c1e8231722c4ce40d7593f28d92b5fc72f3e9774fe73d7e800ec32299f63a \ + --hash=sha256:d53834e23c015ee83a99377db6e5e37d8484f333edb03bd15b4bc312cc7254fb \ + --hash=sha256:d7504f2b476d21653e4d143f44a175f7f751cd41233525312696c76aa3dbb23f \ + --hash=sha256:dbf507e9ef5688bada447a24d68b4b58dd389ba93b7afc065a2ba892bea54769 \ + --hash=sha256:dc52310451fc7c629e13c4e061cbe2dd01684d91f2f8ee2821b083c58bd72432 \ + --hash=sha256:dd00607bffbf30250fe108065f07453ec124dbf223420f57f5e749b04295e090 \ + --hash=sha256:dda608c88cf709b1d406bdfcd84d8d63cff7c9e577a403c6108ce8ce9dcc8764 \ + --hash=sha256:debe9c4f41c32990771be5c22b56f810659f9ddf3d63f67abfdcaa2c6c9c5c1d \ + --hash=sha256:e09fd068c2e169a7070d83d3bde728a4d48de0549f975290be3c108c02e499b4 \ + --hash=sha256:e0fd068364a6759bc794459f0a735ab151d11304346332489c7972bacbe9e72b \ + --hash=sha256:e4c53f8347cd4200f0d70a48ad059cabaf24f5adc6ba08622a23423bc7efa10d \ + --hash=sha256:e5723c01a56c5028c807c701aa66722916d2747ad737a046853f6c46f4875543 \ + --hash=sha256:e7b0460976dc75cb87ad9cc1f9899a4b97751e7d4e77ab840fc9b6d377b8fd24 \ + --hash=sha256:e9d9a4d06d3481eab79803beb4d9bd6f6a8e781ec078ac70d7ef2dcc29d1bea5 \ + --hash=sha256:ead11956716a940c1abc816b7df3fa2b84d06eaed8832ca32f5c5e058c65506b \ + --hash=sha256:ed5f69ce7be7902e5c70ea19eb72d20abf7d725ab5d49777d696e32d4fc1811d \ + --hash=sha256:f2af5c81a1f124609d5f33507082fc3f739959d4719b56877ab1ee7e7b3d602b \ + --hash=sha256:f40e782d49630ad384db66d4d8b73ff4f1b8955dc12e26b09a3e3af064b3b9d6 \ + --hash=sha256:f514f6474e04179d3d33175ed3f3e31434d3130d42ec153540d5b157deefd735 \ + --hash=sha256:f69f57305656a4852f2a7203efc661d8c042e6cc67f7acd97d8667fb448a426e \ + --hash=sha256:fb1e8b8d66c278b21d13b0a7ca22c41dd757a7c209c6b12c313e445c31dd3b28 \ + --hash=sha256:fb4948814a2a98e3912505f09c9e7493b1506226afb1f881825368d6fb776ee3 \ + --hash=sha256:fda207c815b253e34f7e1909840fd14299567b1c0eb4908f8c2ce01a41265401 \ + --hash=sha256:fe8f8f5e70e6dbdfca9882cd9deaac058729bcf323cf7a58660901e55c9c94f6 \ + --hash=sha256:fffc45637bcd6538de8b85f51e3df3223e4ad89bccbfca0481c08c7fc8b7ed7d + # via aiohttp +zipp==3.23.0 \ + --hash=sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e \ + --hash=sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166 + # via etils diff --git a/tools/xspace_to_perfetto.cc b/tools/xspace_to_perfetto.cc new file mode 100644 index 00000000..ce70cb6b --- /dev/null +++ b/tools/xspace_to_perfetto.cc @@ -0,0 +1,33 @@ +#include +#include +#include + +namespace zml::tools { +bool DumpXSpaceFileToTraceJsonFile(std::string_view input_path, + std::string_view output_path, + std::string* error); +} + +void PrintUsage(std::string_view argv0) { + std::cerr << "Usage: " << argv0 << " [output_trace.json]\n"; +} + +int main(int argc, char** argv) { + if (argc < 2 || argc > 3) { + PrintUsage(argv[0]); + return 1; + } + + const std::string input_path = argv[1]; + const std::string output_path = argc == 3 ? argv[2] : input_path + ".trace.json"; + + std::string error; + if (!zml::tools::DumpXSpaceFileToTraceJsonFile(input_path, output_path, + &error)) { + std::cerr << error << "\n"; + return 1; + } + + std::cout << output_path << "\n"; + return 0; +} diff --git a/tools/xspace_to_perfetto.h b/tools/xspace_to_perfetto.h new file mode 100644 index 00000000..48794bb7 --- /dev/null +++ b/tools/xspace_to_perfetto.h @@ -0,0 +1,15 @@ +#pragma once + +#include "ffi/zig_slice.h" + +#ifdef __cplusplus +extern "C" { +#endif + +zig_slice zml_xspace_to_perfetto_dump(zig_slice xspace_proto, + zig_slice output_path); +void zml_xspace_to_perfetto_str_free(zig_slice text); + +#ifdef __cplusplus +} +#endif diff --git a/tools/xspace_to_perfetto_lib.cc b/tools/xspace_to_perfetto_lib.cc new file mode 100644 index 00000000..42823dfd --- /dev/null +++ b/tools/xspace_to_perfetto_lib.cc @@ -0,0 +1,125 @@ +#include +#include +#include +#include +#include +#include + +#include "tools/xspace_to_perfetto.h" +#include "xla/tsl/profiler/convert/trace_events_to_json.h" +#include "xla/tsl/profiler/convert/xplane_to_trace_events.h" +#include "tsl/profiler/protobuf/xplane.pb.h" + +namespace { + +bool ParseXSpace(std::string_view xspace_proto, + tensorflow::profiler::XSpace* xspace, + std::string* error) { + if (xspace_proto.size() > static_cast(std::numeric_limits::max())) { + *error = "XSpace protobuf is too large to parse"; + return false; + } + if (!xspace->ParseFromArray(xspace_proto.data(), + static_cast(xspace_proto.size()))) { + *error = "Failed to parse XSpace protobuf"; + return false; + } + return true; +} + +bool WriteFile(std::string_view output_path, + std::string_view contents, + std::string* error) { + std::ofstream output(std::string(output_path), std::ios::binary); + if (!output.is_open()) { + *error = "Failed to open output file: " + std::string(output_path); + return false; + } + + output.write(contents.data(), static_cast(contents.size())); + if (!output) { + *error = "Failed to write output file: " + std::string(output_path); + return false; + } + return true; +} + +} + +namespace zml::tools { + +bool ConvertXSpaceProtoToTraceJson(std::string_view xspace_proto, + std::string* json, + std::string* error) { + tensorflow::profiler::XSpace xspace; + if (!ParseXSpace(xspace_proto, &xspace, error)) { + return false; + } + + const tsl::profiler::TraceContainer container = + tsl::profiler::ConvertXSpaceToTraceContainer(xspace); + *json = tsl::profiler::TraceContainerToJson(container); + return true; +} + +bool DumpXSpaceProtoToTraceJsonFile(std::string_view xspace_proto, + std::string_view output_path, + std::string* error) { + std::string json; + if (!ConvertXSpaceProtoToTraceJson(xspace_proto, &json, error)) { + return false; + } + return WriteFile(output_path, json, error); +} + +bool DumpXSpaceFileToTraceJsonFile(std::string_view input_path, + std::string_view output_path, + std::string* error) { + std::ifstream input(std::string(input_path), std::ios::binary); + if (!input.is_open()) { + *error = "Failed to open input file: " + std::string(input_path); + return false; + } + + tensorflow::profiler::XSpace xspace; + if (!xspace.ParseFromIstream(&input)) { + *error = "Failed to parse XSpace protobuf from: " + std::string(input_path); + return false; + } + + const tsl::profiler::TraceContainer container = + tsl::profiler::ConvertXSpaceToTraceContainer(xspace); + const std::string json = tsl::profiler::TraceContainerToJson(container); + return WriteFile(output_path, json, error); +} + +} + +extern "C" zig_slice zml_xspace_to_perfetto_dump(zig_slice xspace_proto, + zig_slice output_path) { + std::string error; + if (zml::tools::DumpXSpaceProtoToTraceJsonFile( + std::string_view(static_cast(xspace_proto.ptr), + xspace_proto.len), + std::string_view(static_cast(output_path.ptr), + output_path.len), + &error)) { + return {.ptr = nullptr, .len = 0}; + } + + if (error.empty()) { + return {.ptr = nullptr, .len = 0}; + } + + void* buffer = std::malloc(error.size()); + if (buffer == nullptr) { + return {.ptr = nullptr, .len = 0}; + } + + std::memcpy(buffer, error.data(), error.size()); + return {.ptr = buffer, .len = error.size()}; +} + +extern "C" void zml_xspace_to_perfetto_str_free(zig_slice text) { + std::free(text.ptr); +} diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index fbf6e861..ec635b7e 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -44,6 +44,7 @@ zig_library( "ops.zig", "pjrtx.zig", "platform.zig", + "profiler.zig", "safetensors.zig", "shape.zig", "slice.zig", @@ -63,12 +64,14 @@ zig_library( ":xla_compile_options_upb", ":xla_data_upb", "//bazel", + "//ffi:zig", ":profiler_options", "//mlir", "//mlir/dialects", "//pjrt", "//platforms", "//stdx", + "//tools:xspace_to_perfetto_lib", "//upb", "//zml/io", "//zml/tokenizer", diff --git a/zml/platform.zig b/zml/platform.zig index 9305baa9..42e3d648 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -5,10 +5,10 @@ const pjrt = @import("pjrt"); const platforms = @import("platforms"); pub const Target = platforms.Platform; const stdx = @import("stdx"); -const upb = @import("upb"); const Exe = @import("exe.zig").Exe; const pjrtx = @import("pjrtx.zig"); +const profiler_ = @import("profiler.zig"); const zml = @import("zml.zig"); const log = std.log.scoped(.zml); @@ -467,24 +467,11 @@ pub const Platform = struct { } } - pub fn profiler(self: *const Platform, allocator: std.mem.Allocator) !?pjrt.Profiler { - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); + pub const Profiler = profiler_.Profiler; + pub const ProfilerOptions = profiler_.ProfilerOptions; - var upb_alloc: upb.Allocator = .init(arena.allocator()); - const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); - defer c.upb_Arena_Free(upb_arena); - - const options = try upb.new(c.tensorflow_ProfileOptions, upb_arena); - - c.tensorflow_ProfileOptions_set_device_type(options, 0); - c.tensorflow_ProfileOptions_set_host_tracer_level(options, 3); - c.tensorflow_ProfileOptions_set_device_tracer_level(options, 3); - c.tensorflow_ProfileOptions_set_enable_hlo_proto(options, true); - - const serialized_options = try upb.serialize(options, upb_arena); - - return try self.pjrt_api.profiler(serialized_options); + pub fn profiler(self: *const Platform, allocator: std.mem.Allocator, io: std.Io, options: ProfilerOptions) !Profiler { + return try profiler_.profiler(self.pjrt_api, allocator, io, options); } }; diff --git a/zml/profiler.zig b/zml/profiler.zig new file mode 100644 index 00000000..5157a277 --- /dev/null +++ b/zml/profiler.zig @@ -0,0 +1,224 @@ +const std = @import("std"); +const log = std.log; + +const c = @import("c"); +const pjrt = @import("pjrt"); +const upb = @import("upb"); +const zffi = @import("ffi"); + +pub fn profiler(api: *const pjrt.Api, allocator: std.mem.Allocator, io: std.Io, options: ProfilerOptions) !Profiler { + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + var upb_alloc: upb.Allocator = .init(arena.allocator()); + const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); + defer c.upb_Arena_Free(upb_arena); + + const profile_options = try upb.new(c.tensorflow_ProfileOptions, upb_arena); + try options.writeProto(upb_arena, profile_options); + + const serialized_options = try upb.serialize(profile_options, upb_arena); + const inner = try api.profiler(serialized_options); + + return try .init(allocator, io, inner, options); +} + +pub const ProfilerOptions = struct { + pub const default_version: u32 = 1; + + pub const DeviceType = enum(i32) { + unspecified = c.tensorflow_ProfileOptions_UNSPECIFIED, + cpu = c.tensorflow_ProfileOptions_CPU, + gpu = c.tensorflow_ProfileOptions_GPU, + tpu = c.tensorflow_ProfileOptions_TPU, + pluggable_device = c.tensorflow_ProfileOptions_PLUGGABLE_DEVICE, + }; + + pub const TraceOptions = struct { + host_traceme_filter_mask: ?u64 = null, + + fn isEmpty(self: TraceOptions) bool { + return self.host_traceme_filter_mask == null; + } + + fn writeProto(self: TraceOptions, proto: *c.tensorflow_ProfileOptions_TraceOptions) void { + if (self.host_traceme_filter_mask) |value| { + c.tensorflow_ProfileOptions_TraceOptions_set_host_traceme_filter_mask(proto, value); + } + } + }; + + pub const AdvancedConfigValue = union(enum) { + string: []const u8, + boolean: bool, + int64: i64, + + fn writeProto(self: AdvancedConfigValue, proto: *c.tensorflow_ProfileOptions_AdvancedConfigValue) void { + switch (self) { + .string => |value| c.tensorflow_ProfileOptions_AdvancedConfigValue_set_string_value(proto, upb.stringView(value)), + .boolean => |value| c.tensorflow_ProfileOptions_AdvancedConfigValue_set_bool_value(proto, value), + .int64 => |value| c.tensorflow_ProfileOptions_AdvancedConfigValue_set_int64_value(proto, value), + } + } + }; + + pub const AdvancedConfiguration = struct { + key: []const u8, + value: AdvancedConfigValue, + }; + + pub const defaults: ProfilerOptions = .{}; + + version: u32 = default_version, + repository_path: []const u8 = "/tmp/xprof", + session_id: []const u8 = "profiling", + device_type: ?DeviceType = .unspecified, + include_dataset_ops: ?bool = true, + host_tracer_level: ?u32 = 3, + device_tracer_level: ?u32 = 3, + python_tracer_level: ?u32 = 1, + enable_hlo_proto: ?bool = true, + start_timestamp_ns: ?u64 = null, + duration_ms: ?u64 = null, + trace_options: ?TraceOptions = null, + advanced_configuration: []const AdvancedConfiguration = &.{}, + raise_error_on_start_failure: ?bool = null, + override_hostname: ?[]const u8 = null, + + pub fn writeProto(self: ProfilerOptions, upb_arena: *c.upb_Arena, proto: *c.tensorflow_ProfileOptions) !void { + c.tensorflow_ProfileOptions_set_version(proto, self.version); + c.tensorflow_ProfileOptions_set_session_id(proto, upb.stringView(self.session_id)); + c.tensorflow_ProfileOptions_set_repository_path(proto, upb.stringView(self.repository_path)); + + if (self.device_type) |value| { + c.tensorflow_ProfileOptions_set_device_type(proto, @intFromEnum(value)); + } + if (self.include_dataset_ops) |value| { + c.tensorflow_ProfileOptions_set_include_dataset_ops(proto, value); + } + if (self.host_tracer_level) |value| { + c.tensorflow_ProfileOptions_set_host_tracer_level(proto, value); + } + if (self.device_tracer_level) |value| { + c.tensorflow_ProfileOptions_set_device_tracer_level(proto, value); + } + if (self.python_tracer_level) |value| { + c.tensorflow_ProfileOptions_set_python_tracer_level(proto, value); + } + if (self.enable_hlo_proto) |value| { + c.tensorflow_ProfileOptions_set_enable_hlo_proto(proto, value); + } + if (self.start_timestamp_ns) |value| { + c.tensorflow_ProfileOptions_set_start_timestamp_ns(proto, value); + } + if (self.duration_ms) |value| { + c.tensorflow_ProfileOptions_set_duration_ms(proto, value); + } + if (self.trace_options) |trace_options| { + if (!trace_options.isEmpty()) { + const trace_proto = try upb.new(c.tensorflow_ProfileOptions_TraceOptions, upb_arena); + trace_options.writeProto(trace_proto); + c.tensorflow_ProfileOptions_set_trace_options(proto, trace_proto); + } + } + for (self.advanced_configuration) |entry| { + const value_proto = try upb.new(c.tensorflow_ProfileOptions_AdvancedConfigValue, upb_arena); + entry.value.writeProto(value_proto); + if (!c.tensorflow_ProfileOptions_advanced_configuration_set(proto, upb.stringView(entry.key), value_proto, upb_arena)) { + return error.OutOfMemory; + } + } + if (self.raise_error_on_start_failure) |value| { + c.tensorflow_ProfileOptions_set_raise_error_on_start_failure(proto, value); + } + if (self.override_hostname) |value| { + c.tensorflow_ProfileOptions_set_override_hostname(proto, upb.stringView(value)); + } + } +}; + +pub const Profiler = struct { + arena: std.heap.ArenaAllocator, + allocator: std.mem.Allocator, + io: std.Io, + inner: ?pjrt.Profiler, + session_dir: []const u8, + profile: Profile, + + pub const Profile = struct { + protobuf_path: []const u8, + perfetto_path: []const u8, + }; + + pub fn init(allocator: std.mem.Allocator, io: std.Io, inner: ?pjrt.Profiler, options: ProfilerOptions) !Profiler { + var arena = std.heap.ArenaAllocator.init(allocator); + errdefer arena.deinit(); + + const session_dir = try std.Io.Dir.path.join(allocator, &.{ options.repository_path, "plugins", "profile", options.session_id }); + errdefer allocator.free(session_dir); + + try std.Io.Dir.cwd().createDirPath(io, session_dir); + + const protobuf_path = try std.Io.Dir.path.join(allocator, &.{ session_dir, "profiling.xplane.pb" }); + errdefer allocator.free(protobuf_path); + + const perfetto_path = try std.Io.Dir.path.join(allocator, &.{ session_dir, "profiling.trace.json" }); + errdefer allocator.free(perfetto_path); + + return .{ + .arena = arena, + .allocator = allocator, + .io = io, + .inner = inner, + .session_dir = session_dir, + .profile = .{ + .protobuf_path = protobuf_path, + .perfetto_path = perfetto_path, + }, + }; + } + + pub fn deinit(self: *Profiler) void { + self.allocator.free(self.profile.protobuf_path); + self.allocator.free(self.profile.perfetto_path); + self.allocator.free(self.session_dir); + self.arena.deinit(); + } + + pub fn start(self: *Profiler) !void { + if (self.inner) |*inner| { + try inner.start(); + } + } + + pub fn stop(self: *Profiler) !?Profile { + var inner = self.inner orelse return null; + self.inner = null; + try inner.stop(); + + const protobuf = try inner.collectData(self.arena.allocator()); + + try self.writeFile(self.profile.protobuf_path, protobuf); + + const conversion_error = c.zml_xspace_to_perfetto_dump( + zffi.ZigSlice.from(protobuf), + zffi.ZigSlice.from(self.profile.perfetto_path), + ); + defer if (conversion_error.len != 0) { + c.zml_xspace_to_perfetto_str_free(conversion_error); + }; + if (conversion_error.len != 0) { + log.err("Failed to convert profile protobuf to Perfetto trace: {s}", .{zffi.ZigSlice.to(u8, conversion_error)}); + return error.ProfileTraceConversionFailed; + } + + return self.profile; + } + + fn writeFile(self: *const Profiler, path: []const u8, contents: []const u8) !void { + const file = try std.Io.Dir.createFile(.cwd(), self.io, path, .{}); + defer file.close(self.io); + + try file.writePositionalAll(self.io, contents, 0); + } +}; From ba44bf47e7667292768a3a3736b79aebdb74960a Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 18 Mar 2026 08:19:07 +0100 Subject: [PATCH 356/881] bazel: Fix rules_zig for tests (#415) --- MODULE.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index 68549aff..ac2ee800 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -203,6 +203,6 @@ use_repo(oci, "distroless_cc_debian12", "distroless_cc_debian12_linux_amd64") git_override( module_name = "rules_zig", - commit = "dbc3d75d4fce20d998eea343c2cc0fadc51aac0e", + commit = "3f4ab088be5f21a7409a03f7c19bf25a82cc7bde", remote = "https://github.com/zml/rules_zig.git", ) From a5f7e155e7959855c539e183bb0ad31cbba43c1a Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Wed, 18 Mar 2026 11:48:01 +0100 Subject: [PATCH 357/881] workspace: add remote build configuration --- .bazelrc | 17 +++++++++++++++++ BUILD.bazel | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index 5bfaf909..8e4ccba1 100644 --- a/.bazelrc +++ b/.bazelrc @@ -116,3 +116,20 @@ common:native --@rules_zig//zig/settings:zigopt=-mcpu=native common:native --@@rules_rs++rules_rust+rules_rust//rust/settings:extra_rustc_flag=-Ctarget-cpu=native common:linux --host_platform=//platforms:host_glibc + +# Remote execution and caching with BuildBuddy +common:remote --bes_results_url=https://app.buildbuddy.io/invocation/ +common:remote --bes_backend=grpcs://remote.buildbuddy.io +common:remote --remote_cache=grpcs://remote.buildbuddy.io +common:remote --experimental_remote_downloader=grpcs://remote.buildbuddy.io +common:remote --remote_executor=grpcs://remote.buildbuddy.io +common:remote --remote_download_toplevel +common:remote --nobuild_runfile_links +common:remote --remote_timeout=3600 +common:remote --noexperimental_throttle_remote_action_building +common:remote --experimental_remote_execution_keepalive +common:remote --remote_cache_compression +common:remote --grpc_keepalive_time=30s +common:remote --jobs=2000 +common:remote --extra_execution_platforms=@zml//:rbe_platform +common:remote --experimental_repo_remote_exec=true diff --git a/BUILD.bazel b/BUILD.bazel index 943fbd3c..671fd891 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -20,7 +20,7 @@ platform( "@platforms//os:linux", # ubuntu:22.04 has GNU C Library (Ubuntu GLIBC 2.35-0ubuntu3.10) # But apparently not in this docker://ubuntu:22.04 image? - "@llvm//constraints/libc:gnu.2.31", + "@llvm//constraints/libc:gnu.2.28", "@bazel_tools//tools/cpp:clang", ], exec_properties = { From 7d373133ff674336545a6ec3f870a60d35bb7c68 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Wed, 18 Mar 2026 11:49:39 +0100 Subject: [PATCH 358/881] workspace: bump protobuf and prefer prebuilt protoc --- .bazelrc | 3 +++ MODULE.bazel | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.bazelrc b/.bazelrc index 8e4ccba1..61ea8613 100644 --- a/.bazelrc +++ b/.bazelrc @@ -86,6 +86,9 @@ common --repo_env=BAZEL_NO_APPLE_CPP_TOOLCHAIN=1 # Required for protobuf upb on Linux common:linux --copt=-fPIC +# Prefer prebuilt protoc (finally!) +common --@com_google_protobuf//bazel/toolchains:prefer_prebuilt_protoc=true + # Print test errors in the console test --test_output=errors diff --git a/MODULE.bazel b/MODULE.bazel index ac2ee800..3094a6be 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -9,7 +9,7 @@ bazel_dep(name = "bazel_skylib", version = "1.9.0") bazel_dep(name = "patchelf", version = "0.18.0.bcr.1") bazel_dep(name = "pcre2", version = "10.45") bazel_dep(name = "platforms", version = "1.0.0") -bazel_dep(name = "protobuf", version = "33.1", repo_name = "com_google_protobuf") +bazel_dep(name = "protobuf", version = "34.0.bcr.1", repo_name = "com_google_protobuf") bazel_dep(name = "pybind11_bazel", version = "2.13.6") bazel_dep(name = "re2", version = "2025-11-05", repo_name = "com_googlesource_code_re2") bazel_dep(name = "rules_cc", version = "0.2.16") @@ -18,7 +18,7 @@ bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep(name = "rules_oci", version = "2.2.7") bazel_dep(name = "rules_proto", version = "7.1.0") bazel_dep(name = "rules_python", version = "1.8.4") -bazel_dep(name = "rules_rs", version = "0.0.41") +bazel_dep(name = "rules_rs", version = "0.0.44") bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "rules_zig", version = "0.12.2") bazel_dep(name = "llvm", version = "0.6.8") From ac971b4bc6f30c4f62679a98c64b66a185b338df Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Wed, 18 Mar 2026 13:03:04 +0100 Subject: [PATCH 359/881] workspace: fix upb rule loading --- platforms/neuron/BUILD.bazel | 2 +- zml/BUILD.bazel | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/platforms/neuron/BUILD.bazel b/platforms/neuron/BUILD.bazel index e095d025..a48cd25a 100644 --- a/platforms/neuron/BUILD.bazel +++ b/platforms/neuron/BUILD.bazel @@ -1,4 +1,4 @@ -load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_c_proto_library") +load("@com_google_protobuf//upb/bazel:upb_c_proto_library.bzl", "upb_c_proto_library") load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_python//python/uv:lock.bzl", uv_lock = "lock") load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_shared_library") diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index ec635b7e..da49f7de 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -1,4 +1,4 @@ -load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_c_proto_library") +load("@com_google_protobuf//upb/bazel:upb_c_proto_library.bzl", "upb_c_proto_library") load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") @@ -40,13 +40,13 @@ zig_library( "mlirx.zig", "module.zig", "nn.zig", - "sharding.zig", "ops.zig", "pjrtx.zig", "platform.zig", "profiler.zig", "safetensors.zig", "shape.zig", + "sharding.zig", "slice.zig", "tensor.zig", "testing.zig", @@ -61,11 +61,11 @@ zig_library( ], deps = [ ":posix", + ":profiler_options", ":xla_compile_options_upb", ":xla_data_upb", "//bazel", "//ffi:zig", - ":profiler_options", "//mlir", "//mlir/dialects", "//pjrt", @@ -84,7 +84,6 @@ zig_library( }), ) - zig_test( name = "test", deps = ["//zml"], From a0ef362b65a05788e7681ab3565f29abbdba912a Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 13 Mar 2026 15:53:44 +0100 Subject: [PATCH 360/881] zml/attention/paged_attention: replace custom divCeil with std.math.divCeil --- zml/attention/paged_attention.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 44091fc5..dcb82303 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -46,7 +46,7 @@ pub const Options = union(Backend) { }; pub fn fromBackend(args: Args) Options { - const max_num_pages = @divFloor(args.seq_len + args.page_chunk_size - 1, args.page_chunk_size); // @divCeil + const max_num_pages = std.math.divCeil(args.seq_len, args.page_chunk_size) catch unreachable; return switch (args.backend) { .cuda_fa2 => if (args.is_prefill) .{ .cuda_fa2 = .{ From 5ce4825ceee564c43dd498a50492aaca62f43d08 Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Thu, 19 Mar 2026 16:14:56 +0000 Subject: [PATCH 361/881] fix: Correct divCeil usage in fromBackend function for proper type handling --- zml/attention/paged_attention.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index dcb82303..88a94b28 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -46,7 +46,7 @@ pub const Options = union(Backend) { }; pub fn fromBackend(args: Args) Options { - const max_num_pages = std.math.divCeil(args.seq_len, args.page_chunk_size) catch unreachable; + const max_num_pages = std.math.divCeil(u32, args.seq_len, args.page_chunk_size) catch unreachable; return switch (args.backend) { .cuda_fa2 => if (args.is_prefill) .{ .cuda_fa2 = .{ From b56e86a8e5c08871a4b7ace9d0b61569c5129f95 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 19 Mar 2026 18:01:03 +0100 Subject: [PATCH 362/881] zml/attention/flashattn: enable sharding for Fa3 backend --- MODULE.bazel.lock | 26 ++++----------- zml/attention/flashattn.zig | 64 ++++++++++++++++++++++++++----------- 2 files changed, 52 insertions(+), 38 deletions(-) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index c6cd9cd5..7f7cdc5b 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -14,7 +14,6 @@ "https://bcr.bazel.build/modules/abseil-cpp/20250512.1/MODULE.bazel": "d209fdb6f36ffaf61c509fcc81b19e81b411a999a934a032e10cd009a0226215", "https://bcr.bazel.build/modules/abseil-cpp/20260107.1/MODULE.bazel": "e33b3801443f5fd64465262084534115db76363df13d2168a42bbfacc747be81", "https://bcr.bazel.build/modules/abseil-cpp/20260107.1/source.json": "7a9a88969b1e79268cf613728ca8ff8fa4bc4b1a9abee9ec1fb5f113ca751971", - "https://bcr.bazel.build/modules/apple_support/1.11.1/MODULE.bazel": "1843d7cd8a58369a444fc6000e7304425fba600ff641592161d9f15b179fb896", "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", "https://bcr.bazel.build/modules/apple_support/1.23.1/MODULE.bazel": "53763fed456a968cf919b3240427cf3a9d5481ec5466abc9d5dc51bc70087442", "https://bcr.bazel.build/modules/apple_support/1.24.1/MODULE.bazel": "f46e8ddad60aef170ee92b2f3d00ef66c147ceafea68b6877cb45bd91737f5f8", @@ -44,7 +43,6 @@ "https://bcr.bazel.build/modules/bazel_features/1.24.0/MODULE.bazel": "4796b4c25b47053e9bbffa792b3792d07e228ff66cd0405faef56a978708acd4", "https://bcr.bazel.build/modules/bazel_features/1.27.0/MODULE.bazel": "621eeee06c4458a9121d1f104efb80f39d34deff4984e778359c60eaf1a8cb65", "https://bcr.bazel.build/modules/bazel_features/1.28.0/MODULE.bazel": "4b4200e6cbf8fa335b2c3f43e1d6ef3e240319c33d43d60cc0fbd4b87ece299d", - "https://bcr.bazel.build/modules/bazel_features/1.3.0/MODULE.bazel": "cdcafe83ec318cda34e02948e81d790aab8df7a929cec6f6969f13a489ccecd9", "https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87", "https://bcr.bazel.build/modules/bazel_features/1.32.0/MODULE.bazel": "095d67022a58cb20f7e20e1aefecfa65257a222c18a938e2914fd257b5f1ccdc", "https://bcr.bazel.build/modules/bazel_features/1.33.0/MODULE.bazel": "8b8dc9d2a4c88609409c3191165bccec0e4cb044cd7a72ccbe826583303459f6", @@ -58,7 +56,6 @@ "https://bcr.bazel.build/modules/bazel_lib/3.0.0-beta.1/MODULE.bazel": "407729e232f611c3270005b016b437005daa7b1505826798ea584169a476e878", "https://bcr.bazel.build/modules/bazel_lib/3.0.0-rc.0/MODULE.bazel": "d6e00979a98ac14ada5e31c8794708b41434d461e7e7ca39b59b765e6d233b18", "https://bcr.bazel.build/modules/bazel_lib/3.0.0/MODULE.bazel": "22b70b80ac89ad3f3772526cd9feee2fa412c2b01933fea7ed13238a448d370d", - "https://bcr.bazel.build/modules/bazel_lib/3.2.0/MODULE.bazel": "39b50d94b9be6bda507862254e20c263f9b950e3160112348d10a938be9ce2c2", "https://bcr.bazel.build/modules/bazel_lib/3.2.2/MODULE.bazel": "e2c890c8a515d6bca9c66d47718aa9e44b458fde64ec7204b8030bf2d349058c", "https://bcr.bazel.build/modules/bazel_lib/3.2.2/source.json": "9e84e115c20e14652c5c21401ae85ff4daa8702e265b5c0b3bf89353f17aa212", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", @@ -96,11 +93,9 @@ "https://bcr.bazel.build/modules/jsoncpp/1.9.6/MODULE.bazel": "2f8d20d3b7d54143213c4dfc3d98225c42de7d666011528dc8fe91591e2e17b0", "https://bcr.bazel.build/modules/jsoncpp/1.9.6/source.json": "a04756d367a2126c3541682864ecec52f92cdee80a35735a3cb249ce015ca000", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", - "https://bcr.bazel.build/modules/llvm/0.6.0/MODULE.bazel": "42c2182c49f13d2df83a4a4a95ab55d31efda47b2d67acf419bf6b31522b2a30", + "https://bcr.bazel.build/modules/llvm/0.6.7/MODULE.bazel": "d37a2e10571864dc6a5bb53c29216d90b9400bbcadb422337f49107fd2eaf0d2", "https://bcr.bazel.build/modules/llvm/0.6.8/MODULE.bazel": "53468e4a4be409c2d34e5b7331d2e1fef982151b777655ca3c0047225b333629", "https://bcr.bazel.build/modules/llvm/0.6.8/source.json": "b673af466f716e01d6243f59e47729e99f37dc5e17026d2bf18c98206f09b6c5", - "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", - "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/source.json": "f448c6e8963fdfa7eb831457df83ad63d3d6355018f6574fb017e8169deb43a9", "https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92", "https://bcr.bazel.build/modules/package_metadata/0.0.5/MODULE.bazel": "ef4f9439e3270fdd6b9fd4dbc3d2f29d13888e44c529a1b243f7a31dfbc2e8e4", "https://bcr.bazel.build/modules/package_metadata/0.0.5/source.json": "2326db2f6592578177751c3e1f74786b79382cd6008834c9d01ec865b9126a85", @@ -128,8 +123,8 @@ "https://bcr.bazel.build/modules/protobuf/29.0/MODULE.bazel": "319dc8bf4c679ff87e71b1ccfb5a6e90a6dbc4693501d471f48662ac46d04e4e", "https://bcr.bazel.build/modules/protobuf/29.1/MODULE.bazel": "557c3457560ff49e122ed76c0bc3397a64af9574691cb8201b4e46d4ab2ecb95", "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0", - "https://bcr.bazel.build/modules/protobuf/33.1/MODULE.bazel": "982c8a0cceab4d790076f72b7677faf836b0dfadc2b66a34aab7232116c4ae39", - "https://bcr.bazel.build/modules/protobuf/33.1/source.json": "992c237a40899425648213bf79b05f08c6e8dcd619f96cd944b4511b0276fbd8", + "https://bcr.bazel.build/modules/protobuf/34.0.bcr.1/MODULE.bazel": "74e541b0ba877813da786a11707d4e394433c157841d5111a36be0d44b907931", + "https://bcr.bazel.build/modules/protobuf/34.0.bcr.1/source.json": "fc174b3d6215aa14197d1bd779f98bb72d9fd666ee5ec0d6bba6ae986baa4535", "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e", "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/MODULE.bazel": "e6f4c20442eaa7c90d7190d8dc539d0ab422f95c65a57cc59562170c58ae3d34", "https://bcr.bazel.build/modules/pybind11_bazel/2.13.6/MODULE.bazel": "2d746fda559464b253b2b2e6073cb51643a2ac79009ca02100ebbc44b4548656", @@ -141,8 +136,6 @@ "https://bcr.bazel.build/modules/re2/2025-11-05/source.json": "7df1394aabda1c9bc188a302f5d54b1c657924edd04ebc57d2be29dbd7efd141", "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", - "https://bcr.bazel.build/modules/rules_apple/3.16.0/MODULE.bazel": "0d1caf0b8375942ce98ea944be754a18874041e4e0459401d925577624d3a54a", - "https://bcr.bazel.build/modules/rules_apple/3.16.0/source.json": "d8b5fe461272018cc07cfafce11fe369c7525330804c37eec5a82f84cd475366", "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", "https://bcr.bazel.build/modules/rules_cc/0.0.10/MODULE.bazel": "ec1705118f7eaedd6e118508d3d26deba2a4e76476ada7e0e3965211be012002", "https://bcr.bazel.build/modules/rules_cc/0.0.13/MODULE.bazel": "0e8529ed7b323dad0775ff924d2ae5af7640b23553dfcd4d34344c7e7a867191", @@ -228,16 +221,13 @@ "https://bcr.bazel.build/modules/rules_python/1.7.0/MODULE.bazel": "d01f995ecd137abf30238ad9ce97f8fc3ac57289c8b24bd0bf53324d937a14f8", "https://bcr.bazel.build/modules/rules_python/1.8.4/MODULE.bazel": "33e3971e66161a3e955f7a0d411a8d1f291c4ce4c561851512466f3c77ff8ece", "https://bcr.bazel.build/modules/rules_python/1.8.4/source.json": "9fbc0e57bae52cddcc3831d668bce87a47e0c655104a85098d4459dd9a3b0a10", - "https://bcr.bazel.build/modules/rules_rs/0.0.41/MODULE.bazel": "3f4f7e4cf87c7e8f8e32e8e7a49e1fb6caa5529b9b52757ec25d0b4d0272e380", - "https://bcr.bazel.build/modules/rules_rs/0.0.41/source.json": "19cf5bd6235a7e2609c5a181c52fa3111d0ef70ba19edee752489e276024c34b", + "https://bcr.bazel.build/modules/rules_rs/0.0.44/MODULE.bazel": "aa151d9a96e5d9363732df8943c7fb5327738cc160732fa74c02e5de7822007b", + "https://bcr.bazel.build/modules/rules_rs/0.0.44/source.json": "1358c9418e1325fdb5d0caf53ef32f98d525fe2030668279cae2e97c34cd29eb", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", "https://bcr.bazel.build/modules/rules_shell/0.6.1/MODULE.bazel": "72e76b0eea4e81611ef5452aa82b3da34caca0c8b7b5c0c9584338aa93bae26b", "https://bcr.bazel.build/modules/rules_shell/0.6.1/source.json": "20ec05cd5e592055e214b2da8ccb283c7f2a421ea0dc2acbf1aa792e11c03d0c", - "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", - "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", - "https://bcr.bazel.build/modules/rules_swift/2.1.1/source.json": "40fc69dfaac64deddbb75bd99cdac55f4427d9ca0afbe408576a65428427a186", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", "https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4", @@ -247,8 +237,6 @@ "https://bcr.bazel.build/modules/stardoc/0.7.1/MODULE.bazel": "3548faea4ee5dda5580f9af150e79d0f6aea934fc60c1cc50f4efdd9420759e7", "https://bcr.bazel.build/modules/stardoc/0.7.2/MODULE.bazel": "fc152419aa2ea0f51c29583fab1e8c99ddefd5b3778421845606ee628629e0e5", "https://bcr.bazel.build/modules/stardoc/0.7.2/source.json": "58b029e5e901d6802967754adf0a9056747e8176f017cfe3607c0851f4d42216", - "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "5e463fbfba7b1701d957555ed45097d7f984211330106ccd1352c6e0af0dcf91", - "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/source.json": "32bd87e5f4d7acc57c5b2ff7c325ae3061d5e242c0c4c214ae87e0f1c13e54cb", "https://bcr.bazel.build/modules/tar.bzl/0.2.1/MODULE.bazel": "52d1c00a80a8cc67acbd01649e83d8dd6a9dc426a6c0b754a04fe8c219c76468", "https://bcr.bazel.build/modules/tar.bzl/0.5.1/MODULE.bazel": "7c2eb3dcfc53b0f3d6f9acdfd911ca803eaf92aadf54f8ca6e4c1f3aee288351", "https://bcr.bazel.build/modules/tar.bzl/0.5.5/MODULE.bazel": "4bfab9bbc7a1966c2c5f7371f5848f5e2d27c465951b4435adc9aaf00ed681da", @@ -342,7 +330,7 @@ "@@aspect_tools_telemetry+//:extension.bzl%telemetry": { "general": { "bzlTransitiveDigest": "MePriaXmQNSqUfE+YQvEtzBc2bU1mjITINIffFiZYgo=", - "usagesDigest": "qri4Ekw3JtqdQ7Lg8s2Zpc+NTUMzWNCysFG3DPfQ3RY=", + "usagesDigest": "TU1quwRKpvAEOEF35/IVh4NeoIS2hayv4fGp8f5RXoY=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -352,7 +340,7 @@ "attributes": { "deps": { "aspect_rules_py": "1.8.4", - "rules_rs": "0.0.41", + "rules_rs": "0.0.44", "aspect_tools_telemetry": "0.3.3" } } diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 5e232bd3..c68cc218 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -1501,7 +1501,6 @@ pub const paged_fa3 = struct { }; pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { - _ = opts; // autofix stdx.debug.assert(q.shape().hasTags(.{ .b, .h, .hd }), "Expected q to have tags .b, .h, .hd", .{}); stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); @@ -1512,8 +1511,8 @@ pub const paged_fa3 = struct { const softmax_lse_accum = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * decode_parameters.block_table.dim(0) * q.dim(.h) * 4}, .i8)); const scheduler_metadata = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = decode_parameters.block_table.dim(0) + 1 }, .i32)); - const o = zml.ops.customCall( - Decode.custom_call_name, + const output_shape = q.shape(); + const o = zml.ops.manualComputation( .{ q, k_cache, @@ -1527,12 +1526,22 @@ pub const paged_fa3 = struct { scheduler_metadata, layer_index, }, - .{q.shape()}, + output_shape, .{ - .is_causal = true, - .max_seqlen_k = context.max_seqlen_k, + .metadata = .{ + .is_causal = opts.is_causal, + .max_seqlen_k = context.max_seqlen_k, + .window_size_left = opts.sliding_window, + }, + .opts = zml.ops.CustomCallOptions{ + .has_side_effect = false, + }, }, - .{ .has_side_effect = false }, + (struct { + fn body(ctx_: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output: zml.Shape) zml.Tensor { + return zml.ops.customCall(Decode.custom_call_name, sharded_inputs, output, ctx_.metadata, ctx_.opts); + } + }).body, ); break :b o; @@ -1541,8 +1550,9 @@ pub const paged_fa3 = struct { const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = q.dim(.b) }, .f32)); const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * q.dim(.b) * q.dim(.h) * 4}, .i8)); const scheduler_metadata_prefill = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = mixed_parameters.block_table_prefill.dim(0) + 1 }, .i32)); - var o = zml.ops.customCall( - Prefill.custom_call_name, + + const output_shape = q.shape(); + var o = zml.ops.manualComputation( .{ q, k_cache, @@ -1557,12 +1567,20 @@ pub const paged_fa3 = struct { mixed_parameters.metadata.host_metadata, layer_index, }, - .{q.shape()}, + output_shape, .{ - .is_causal = false, - .max_seqlen_k = context.max_seqlen_k, + .metadata = .{ + .is_causal = opts.is_causal, + .max_seqlen_k = context.max_seqlen_k, + .window_size_left = opts.sliding_window, + }, + .opts = zml.ops.CustomCallOptions{ .has_side_effect = false }, }, - .{ .has_side_effect = false }, + (struct { + fn body(ctx_: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output: zml.Shape) zml.Tensor { + return zml.ops.customCall(Prefill.custom_call_name, sharded_inputs, output, ctx_.metadata, ctx_.opts); + } + }).body, ); const batch_size_decode = mixed_parameters.block_table_prefill.dim(0); @@ -1571,8 +1589,8 @@ pub const paged_fa3 = struct { const scheduler_metadata_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = mixed_parameters.block_table_decode.dim(0) + 1 }, .i32)); var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_size_decode }); - const o_decode = zml.ops.customCall( - Decode.custom_call_name, + const decode_output_shape = q_decode.shape(); + const o_decode = zml.ops.manualComputation( .{ q_decode, k_cache, @@ -1586,12 +1604,20 @@ pub const paged_fa3 = struct { scheduler_metadata_decode, layer_index, }, - .{q_decode.shape()}, + decode_output_shape, .{ - .is_causal = true, - .max_seqlen_k = context.max_seqlen_k, + .metadata = .{ + .is_causal = opts.is_causal, + .max_seqlen_k = context.max_seqlen_k, + .window_size_left = opts.sliding_window, + }, + .opts = zml.ops.CustomCallOptions{ .has_side_effect = false }, }, - .{ .has_side_effect = false }, + (struct { + fn body(ctx_: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output: zml.Shape) zml.Tensor { + return zml.ops.customCall(Decode.custom_call_name, sharded_inputs, output, ctx_.metadata, ctx_.opts); + } + }).body, ); o = o.dynamicUpdateSlice1d(o_decode, 0, context.decode_offset.?); From a011fb0e5230d2255b2292357e5db0921cc8a6a6 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Fri, 20 Mar 2026 10:31:57 +0100 Subject: [PATCH 363/881] Green CI (#419) Co-authored-by: Corentin Godeau --- .github/workflows/ci.yaml | 86 ++++++++------ MODULE.bazel.lock | 150 +++++++++++++++++++++++- bazel/bazel.zig | 4 +- examples/io/BUILD.bazel | 5 - examples/lfm/discussion.zig | 23 ++-- examples/lfm/inference.zig | 54 +++++---- examples/lfm/main.zig | 5 +- examples/lfm/model.zig | 29 ++--- examples/lfm/test_runner.zig | 53 +++++---- examples/llama/BUILD.bazel | 8 -- mlir/dialects/func.zig | 1 - platforms/cpu/cpu.zig | 4 +- platforms/cuda/flashattn/flashattn.zig | 4 +- platforms/neuron/neuron.zig | 2 +- platforms/rocm/rocm.zig | 2 +- platforms/tpu/tpu.zig | 2 +- zml/attention/flashattn.zig | 18 +-- zml/attention/paged_attention.zig | 4 +- zml/nn.zig | 5 +- zml/testing.zig | 4 +- zml/tokenizer/main.zig | 6 +- zml/tokenizer/sentencepiece/BUILD.bazel | 5 - zml/tokenizer/tokenizer.zig | 8 +- 23 files changed, 323 insertions(+), 159 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 48bc9320..0f483a89 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -58,9 +58,9 @@ jobs: if: ${{ !matrix.runtimes.enable_cache }} run: mkdir -p /tmp/zig-cache && mkdir -p /tmp/zml && sudo chmod -R 777 /tmp/zig-cache && sudo chmod -R 777 /tmp/zml - - uses: bazel-contrib/setup-bazel@0.15.0 + - uses: bazel-contrib/setup-bazel@0.18.0 with: - bazelisk-version: 1.26.0 + bazelisk-version: 1.28.1 bazelisk-cache: false # todo: use `--config=debug` instead of `-c opt` when static archive bug is fixed bazelrc: | @@ -73,11 +73,44 @@ jobs: common --disk_cache=/home/${{ matrix.runtimes.user }}/.cache/bazel-disk common --strip=always - - name: ZML / bazel build //... - run: bazel build ${{ matrix.runtimes.bazel_opts }} //... + - name: ZML / bazel build //zml/... + run: bazel build ${{ matrix.runtimes.bazel_opts }} //zml/... + + - name: ZML / bazel test //zml/... + run: bazel test ${{ matrix.runtimes.bazel_opts }} //zml/... + + - name: ZML / bazel test //mlir/... + run: bazel test ${{ matrix.runtimes.bazel_opts }} //mlir/... + + - name: ZML / bazel test //stdx/... + run: bazel test ${{ matrix.runtimes.bazel_opts }} //stdx/... + + zig-lint: + runs-on: + [ + "runs-on", + "runner=4cpu-linux-x64", + "image=ubuntu24-amd64", + "run-id=${{ github.run_id }}", + "extras=s3-cache", + ] + steps: + - uses: runs-on/action@v1 + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Zig + uses: mlugg/setup-zig@v2 + with: + version: 0.15.2 + mirror: https://mirror.zml.ai/zig + + - name: Zig Format + run: zig fmt --check $(git ls-files --exclude-standard -- ':!third_party/*' '*.zig' | xargs) + + - name: No autofix in Zig code + run: bash -c "! git grep -l '// autofix' -- '*.zig'" - - name: ZML / bazel test //... - run: bazel test ${{ matrix.runtimes.bazel_opts }} //... ci: runs-on: [ @@ -92,32 +125,14 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - uses: runs-on/snapshot@v1 + - uses: bazel-contrib/setup-bazel@0.18.0 with: - path: /home/runner/.cache/bazel-disk - volume_size: 350 - - - name: TMP fix for the mount user - run: sudo chown -R runner:runner /home/runner/.cache/ - - - name: Setup Zig - uses: mlugg/setup-zig@v2 - with: - version: 0.15.2 - mirror: https://mirror.zml.ai/zig - - - name: Zig Format - run: zig fmt --check $(git ls-files --exclude-standard -- ':!third_party/*' '*.zig' | xargs) - - - name: No autofix in Zig code - run: bash -c "! git grep -l '// autofix' -- '*.zig'" - - - uses: bazel-contrib/setup-bazel@0.15.0 - with: - bazelisk-version: 1.26.0 - bazelisk-cache: false - # todo: use `--config=debug` instead of `-c opt` when static archive bug is fixed + bazelisk-version: 1.28.1 + bazelisk-cache: true + repository-cache: true bazelrc: | + common --config=remote + common --remote_header=x-buildbuddy-api-key=${{ secrets.BUILDBUDDY_CI_API_KEY }} common -c opt common --verbose_failures common --keep_going @@ -127,11 +142,6 @@ jobs: common --disk_cache=/home/runner/.cache/bazel-disk common --strip=always - - name: ZML / bazel mod explain - run: bazel mod explain - - - name: ZML / bazel mod graph - run: bazel mod graph - - - name: ZML / bazel query //... - run: bazel query //... + - run: bazel mod explain + - run: bazel mod graph + - run: bazel query //... diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 7f7cdc5b..d36b5a20 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1335,14 +1335,30 @@ }, "facts": { "@@rules_rs+//rs:extensions.bzl%crate": { + "addr2line_0.25.1": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"backtrace\",\"req\":\"^0.3.13\"},{\"features\":[\"wrap_help\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4.3.21\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"cpp_demangle\",\"optional\":true,\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.7.0\"},{\"default_features\":false,\"name\":\"fallible-iterator\",\"optional\":true,\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"findshlibs\",\"req\":\"^0.10\"},{\"default_features\":false,\"features\":[\"read\"],\"name\":\"gimli\",\"req\":\"^0.32.0\"},{\"kind\":\"dev\",\"name\":\"libtest-mimic\",\"req\":\"^0.8.1\"},{\"name\":\"memmap2\",\"optional\":true,\"req\":\"^0.9.4\"},{\"default_features\":false,\"features\":[\"read\",\"compression\"],\"name\":\"object\",\"optional\":true,\"req\":\"^0.37.0\"},{\"name\":\"rustc-demangle\",\"optional\":true,\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"typed-arena\",\"optional\":true,\"req\":\"^2\"}],\"features\":{\"all\":[\"bin\",\"wasm\"],\"bin\":[\"loader\",\"rustc-demangle\",\"cpp_demangle\",\"fallible-iterator\",\"smallvec\",\"dep:clap\"],\"cargo-all\":[],\"default\":[\"rustc-demangle\",\"cpp_demangle\",\"loader\",\"fallible-iterator\",\"smallvec\"],\"loader\":[\"std\",\"dep:object\",\"dep:memmap2\",\"dep:typed-arena\"],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"gimli/rustc-dep-of-std\"],\"std\":[\"gimli/std\"],\"wasm\":[\"object/wasm\"]}}", + "adler2_2.0.1": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"std\"],\"rustc-dep-of-std\":[\"core\"],\"std\":[]}}", "ahash_0.8.12": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1.0\"},{\"name\":\"const-random\",\"optional\":true,\"req\":\"^0.1.17\"},{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.2\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.5\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2.1\"},{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"hashbrown\",\"req\":\"^0.14.3\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.2\"},{\"kind\":\"dev\",\"name\":\"no-panic\",\"req\":\"^0.1.10\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"once_cell\",\"req\":\"^1.18.0\",\"target\":\"cfg(not(all(target_arch = \\\"arm\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"pcg-mwc\",\"req\":\"^0.2.1\"},{\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"seahash\",\"req\":\"^4.0\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.117\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.59\"},{\"kind\":\"dev\",\"name\":\"smallvec\",\"req\":\"^1.13.1\"},{\"kind\":\"build\",\"name\":\"version_check\",\"req\":\"^0.9.4\"},{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"zerocopy\",\"req\":\"^0.8.24\"}],\"features\":{\"atomic-polyfill\":[\"dep:portable-atomic\",\"once_cell/critical-section\"],\"compile-time-rng\":[\"const-random\"],\"default\":[\"std\",\"runtime-rng\"],\"nightly-arm-aes\":[],\"no-rng\":[],\"runtime-rng\":[\"getrandom\"],\"std\":[]}}", "aho-corasick_1.1.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.17\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.4.0\"}],\"features\":{\"default\":[\"std\",\"perf-literal\"],\"logging\":[\"dep:log\"],\"perf-literal\":[\"dep:memchr\"],\"std\":[\"memchr?/std\"]}}", + "aho-corasick_1.1.4": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.17\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.4.0\"}],\"features\":{\"default\":[\"std\",\"perf-literal\"],\"logging\":[\"dep:log\"],\"perf-literal\":[\"dep:memchr\"],\"std\":[\"memchr?/std\"]}}", + "anyhow_1.0.102": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.6\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0\"},{\"kind\":\"dev\",\"name\":\"thiserror\",\"req\":\"^2\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"}],\"features\":{\"backtrace\":[],\"default\":[\"std\"],\"std\":[]}}", + "async-stream-impl_0.3.6": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"full\",\"visit-mut\"],\"name\":\"syn\",\"req\":\"^2.0.2\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"}],\"features\":{}}", + "async-stream_0.3.6": "{\"dependencies\":[{\"name\":\"async-stream-impl\",\"req\":\"=0.3.6\"},{\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1\"}],\"features\":{}}", + "async-trait_0.1.89": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.30\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"full\",\"parsing\",\"printing\",\"proc-macro\",\"visit-mut\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1.40\"},{\"kind\":\"dev\",\"name\":\"tracing-attributes\",\"req\":\"^0.1.27\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{}}", + "atomic-waker_1.1.2": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"cargo_bench_support\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.5\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.7.0\"}],\"features\":{}}", + "autocfg_1.5.0": "{\"dependencies\":[],\"features\":{}}", + "axum-core_0.4.5": "{\"dependencies\":[{\"name\":\"async-trait\",\"req\":\"^0.1.67\"},{\"kind\":\"dev\",\"name\":\"axum\",\"req\":\"^0.7.2\"},{\"name\":\"bytes\",\"req\":\"^1.2\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"http\",\"req\":\"^1.0.0\"},{\"name\":\"http-body\",\"req\":\"^1.0.0\"},{\"name\":\"http-body-util\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^1.0.0\"},{\"name\":\"mime\",\"req\":\"^0.3.16\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"},{\"name\":\"rustversion\",\"req\":\"^1.0.9\"},{\"name\":\"sync_wrapper\",\"req\":\"^1.0.0\"},{\"features\":[\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.25.0\"},{\"features\":[\"limit\"],\"name\":\"tower-http\",\"optional\":true,\"req\":\"^0.6.0\"},{\"features\":[\"limit\"],\"kind\":\"dev\",\"name\":\"tower-http\",\"req\":\"^0.6.0\"},{\"name\":\"tower-layer\",\"req\":\"^0.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.37\"}],\"features\":{\"__private_docs\":[\"dep:tower-http\"],\"tracing\":[\"dep:tracing\"]}}", + "axum_0.7.9": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0\"},{\"name\":\"async-trait\",\"req\":\"^0.1.67\"},{\"name\":\"axum-core\",\"req\":\"^0.4.5\"},{\"name\":\"axum-macros\",\"optional\":true,\"req\":\"^0.4.2\"},{\"features\":[\"__private\"],\"kind\":\"dev\",\"name\":\"axum-macros\",\"req\":\"^0.4.1\"},{\"name\":\"base64\",\"optional\":true,\"req\":\"^0.22.1\"},{\"name\":\"bytes\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"http\",\"req\":\"^1.0.0\"},{\"name\":\"http-body\",\"req\":\"^1.0.0\"},{\"name\":\"http-body-util\",\"req\":\"^0.1.0\"},{\"name\":\"hyper\",\"optional\":true,\"req\":\"^1.1.0\"},{\"features\":[\"tokio\",\"server\",\"service\"],\"name\":\"hyper-util\",\"optional\":true,\"req\":\"^0.1.3\"},{\"name\":\"itoa\",\"req\":\"^1.0.5\"},{\"name\":\"matchit\",\"req\":\"^0.7\"},{\"name\":\"memchr\",\"req\":\"^2.4.1\"},{\"name\":\"mime\",\"req\":\"^0.3.16\"},{\"name\":\"multer\",\"optional\":true,\"req\":\"^3.0.0\"},{\"name\":\"percent-encoding\",\"req\":\"^2.1\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"json\",\"stream\",\"multipart\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.12\"},{\"name\":\"rustversion\",\"req\":\"^1.0.9\"},{\"name\":\"serde\",\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"features\":[\"raw_value\"],\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"raw_value\"],\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"serde_path_to_error\",\"optional\":true,\"req\":\"^0.1.8\"},{\"name\":\"serde_urlencoded\",\"optional\":true,\"req\":\"^0.7\"},{\"name\":\"sha1\",\"optional\":true,\"req\":\"^0.10\"},{\"name\":\"sync_wrapper\",\"req\":\"^1.0.0\"},{\"features\":[\"serde-human-readable\"],\"kind\":\"dev\",\"name\":\"time\",\"req\":\"^0.3\"},{\"features\":[\"time\"],\"name\":\"tokio\",\"optional\":true,\"package\":\"tokio\",\"req\":\"^1.25.0\"},{\"features\":[\"macros\",\"rt\",\"rt-multi-thread\",\"net\",\"test-util\"],\"kind\":\"dev\",\"name\":\"tokio\",\"package\":\"tokio\",\"req\":\"^1.25.0\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"name\":\"tokio-tungstenite\",\"optional\":true,\"req\":\"^0.24.0\"},{\"kind\":\"dev\",\"name\":\"tokio-tungstenite\",\"req\":\"^0.24.0\"},{\"default_features\":false,\"features\":[\"util\"],\"name\":\"tower\",\"req\":\"^0.5.1\"},{\"features\":[\"util\",\"timeout\",\"limit\",\"load-shed\",\"steer\",\"filter\"],\"kind\":\"dev\",\"name\":\"tower\",\"package\":\"tower\",\"req\":\"^0.5.1\"},{\"features\":[\"add-extension\",\"auth\",\"catch-panic\",\"compression-br\",\"compression-deflate\",\"compression-gzip\",\"cors\",\"decompression-br\",\"decompression-deflate\",\"decompression-gzip\",\"follow-redirect\",\"fs\",\"limit\",\"map-request-body\",\"map-response-body\",\"metrics\",\"normalize-path\",\"propagate-header\",\"redirect\",\"request-id\",\"sensitive-headers\",\"set-header\",\"set-status\",\"timeout\",\"trace\",\"util\",\"validate-request\"],\"name\":\"tower-http\",\"optional\":true,\"req\":\"^0.6.0\"},{\"features\":[\"add-extension\",\"auth\",\"catch-panic\",\"compression-br\",\"compression-deflate\",\"compression-gzip\",\"cors\",\"decompression-br\",\"decompression-deflate\",\"decompression-gzip\",\"follow-redirect\",\"fs\",\"limit\",\"map-request-body\",\"map-response-body\",\"metrics\",\"normalize-path\",\"propagate-header\",\"redirect\",\"request-id\",\"sensitive-headers\",\"set-header\",\"set-status\",\"timeout\",\"trace\",\"util\",\"validate-request\"],\"kind\":\"dev\",\"name\":\"tower-http\",\"req\":\"^0.6.0\"},{\"name\":\"tower-layer\",\"req\":\"^0.3.2\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1\"},{\"features\":[\"json\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"},{\"features\":[\"serde\",\"v4\"],\"kind\":\"dev\",\"name\":\"uuid\",\"req\":\"^1.0\"}],\"features\":{\"__private_docs\":[\"axum-core/__private_docs\",\"tower/full\",\"dep:tower-http\"],\"default\":[\"form\",\"http1\",\"json\",\"matched-path\",\"original-uri\",\"query\",\"tokio\",\"tower-log\",\"tracing\"],\"form\":[\"dep:serde_urlencoded\"],\"http1\":[\"dep:hyper\",\"hyper?/http1\",\"hyper-util?/http1\"],\"http2\":[\"dep:hyper\",\"hyper?/http2\",\"hyper-util?/http2\"],\"json\":[\"dep:serde_json\",\"dep:serde_path_to_error\"],\"macros\":[\"dep:axum-macros\"],\"matched-path\":[],\"multipart\":[\"dep:multer\"],\"original-uri\":[],\"query\":[\"dep:serde_urlencoded\"],\"tokio\":[\"dep:hyper-util\",\"dep:tokio\",\"tokio/net\",\"tokio/rt\",\"tower/make\",\"tokio/macros\"],\"tower-log\":[\"tower/log\"],\"tracing\":[\"dep:tracing\",\"axum-core/tracing\"],\"ws\":[\"dep:hyper\",\"tokio\",\"dep:tokio-tungstenite\",\"dep:sha1\",\"dep:base64\"]}}", + "backtrace_0.3.76": "{\"dependencies\":[{\"default_features\":false,\"name\":\"addr2line\",\"req\":\"^0.25.0\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"name\":\"cfg-if\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"cpp_demangle\",\"optional\":true,\"req\":\"^0.5.0\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.156\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"kind\":\"dev\",\"name\":\"libloading\",\"req\":\"^0.8\"},{\"default_features\":false,\"name\":\"miniz_oxide\",\"req\":\"^0.8\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"default_features\":false,\"features\":[\"read_core\",\"elf\",\"macho\",\"pe\",\"xcoff\",\"unaligned\",\"archive\"],\"name\":\"object\",\"req\":\"^0.37.0\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"name\":\"rustc-demangle\",\"req\":\"^0.1.24\"},{\"default_features\":false,\"name\":\"ruzstd\",\"optional\":true,\"req\":\"^0.8.1\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"windows-link\",\"req\":\"^0.2\",\"target\":\"cfg(any(windows, target_os = \\\"cygwin\\\"))\"}],\"features\":{\"coresymbolication\":[],\"dbghelp\":[],\"default\":[\"std\"],\"dl_iterate_phdr\":[],\"dladdr\":[],\"kernel32\":[],\"libunwind\":[],\"ruzstd\":[\"dep:ruzstd\"],\"serialize-serde\":[\"serde\"],\"std\":[],\"unix-backtrace\":[]}}", "base64_0.13.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"=0.3.2\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.6.1\"},{\"kind\":\"dev\",\"name\":\"structopt\",\"req\":\"^0.3\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[]}}", + "base64_0.22.1": "{\"dependencies\":[{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"clap\",\"req\":\"^3.2.25\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rstest\",\"req\":\"^0.13.0\"},{\"kind\":\"dev\",\"name\":\"rstest_reuse\",\"req\":\"^0.6.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"strum\",\"req\":\"^0.25\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", + "bitflags_2.11.0": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"arbitrary\",\"req\":\"^1.0\"},{\"name\":\"bytemuck\",\"optional\":true,\"req\":\"^1.12\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"bytemuck\",\"req\":\"^1.12.2\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.228\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde_lib\",\"package\":\"serde\",\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.19\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.18\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"zerocopy\",\"req\":\"^0.8\"}],\"features\":{\"example_generated\":[],\"serde\":[\"serde_core\"],\"std\":[]}}", "bitflags_2.9.1": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"arbitrary\",\"req\":\"^1.0\"},{\"name\":\"bytemuck\",\"optional\":true,\"req\":\"^1.12\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"bytemuck\",\"req\":\"^1.12.2\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.19\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.18\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"zerocopy\",\"req\":\"^0.8\"}],\"features\":{\"example_generated\":[],\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"],\"std\":[]}}", "byteorder_1.5.0": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9.2\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"std\"],\"i128\":[],\"std\":[]}}", + "bytes_1.11.1": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"require-cas\"],\"name\":\"extra-platforms\",\"optional\":true,\"package\":\"portable-atomic\",\"req\":\"^1.3\"},{\"kind\":\"dev\",\"name\":\"loom\",\"req\":\"^0.7\",\"target\":\"cfg(loom)\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.60\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", "castaway_0.2.4": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1\"},{\"name\":\"rustversion\",\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", "cc_1.2.33": "{\"dependencies\":[{\"default_features\":false,\"name\":\"jobserver\",\"optional\":true,\"req\":\"^0.1.30\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.62\",\"target\":\"cfg(unix)\"},{\"name\":\"shlex\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{\"jobserver\":[],\"parallel\":[\"dep:libc\",\"dep:jobserver\"]}}", "cfg-if_1.0.0": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"]}}", + "cfg-if_1.0.4": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"rustc-dep-of-std\":[\"core\"]}}", "compact_str_0.9.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"borsh\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"castaway\",\"req\":\"^0.2.3\"},{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"cfg-if\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"diesel\",\"optional\":true,\"req\":\"^2\"},{\"name\":\"itoa\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"markup\",\"optional\":true,\"req\":\"^0.15\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"proptest\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"rkyv\",\"optional\":true,\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rkyv\",\"req\":\"^0.8.8\"},{\"name\":\"rustversion\",\"req\":\"^1\"},{\"name\":\"ryu\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"derive\",\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"features\":[\"union\"],\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"name\":\"sqlx\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"static_assertions\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"test-case\",\"req\":\"^3\"},{\"kind\":\"dev\",\"name\":\"test-strategy\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"borsh\":[\"dep:borsh\"],\"bytes\":[\"dep:bytes\"],\"default\":[\"std\"],\"diesel\":[\"dep:diesel\"],\"markup\":[\"dep:markup\"],\"proptest\":[\"dep:proptest\"],\"quickcheck\":[\"dep:quickcheck\"],\"rkyv\":[\"dep:rkyv\"],\"serde\":[\"dep:serde\"],\"smallvec\":[\"dep:smallvec\"],\"sqlx\":[\"dep:sqlx\",\"std\"],\"sqlx-mysql\":[\"sqlx\",\"sqlx/mysql\"],\"sqlx-postgres\":[\"sqlx\",\"sqlx/postgres\"],\"sqlx-sqlite\":[\"sqlx\",\"sqlx/sqlite\"],\"std\":[],\"zeroize\":[\"dep:zeroize\"]}}", "crossbeam-deque_0.8.6": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-epoch\",\"req\":\"^0.9.17\"},{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"crossbeam-epoch/std\",\"crossbeam-utils/std\"]}}", "crossbeam-epoch_0.9.18": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"name\":\"loom-crate\",\"optional\":true,\"package\":\"loom\",\"req\":\"^0.7.1\",\"target\":\"cfg(crossbeam_loom)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"loom\":[\"loom-crate\",\"crossbeam-utils/loom\"],\"nightly\":[\"crossbeam-utils/nightly\"],\"std\":[\"alloc\",\"crossbeam-utils/std\"]}}", @@ -1355,64 +1371,196 @@ "derive_builder_core_0.20.2": "{\"dependencies\":[{\"name\":\"darling\",\"req\":\"^0.20.10\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.37\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"alloc\":[],\"clippy\":[],\"lib_has_std\":[]}}", "derive_builder_macro_0.20.2": "{\"dependencies\":[{\"name\":\"derive_builder_core\",\"req\":\"=0.20.2\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"alloc\":[\"derive_builder_core/alloc\"],\"clippy\":[\"derive_builder_core/clippy\"],\"lib_has_std\":[\"derive_builder_core/lib_has_std\"]}}", "either_1.13.0": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"use_std\"],\"use_std\":[]}}", + "either_1.15.0": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"alloc\",\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.95\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[],\"use_std\":[\"std\"]}}", + "equivalent_1.0.2": "{\"dependencies\":[],\"features\":{}}", + "errno_0.3.14": "{\"dependencies\":[{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(target_os=\\\"hermit\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(target_os=\\\"wasi\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(unix)\"},{\"features\":[\"Win32_Foundation\",\"Win32_System_Diagnostics_Debug\"],\"name\":\"windows-sys\",\"req\":\">=0.52, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"libc/std\"]}}", "esaxx-rs_0.1.10": "{\"dependencies\":[{\"kind\":\"build\",\"name\":\"cc\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"}],\"features\":{\"cpp\":[\"cc\"],\"default\":[\"cpp\"]}}", + "fastrand_2.3.0": "{\"dependencies\":[{\"features\":[\"js\"],\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.2\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"getrandom\",\"req\":\"^0.2\"},{\"features\":[\"js\"],\"kind\":\"dev\",\"name\":\"getrandom\",\"req\":\"^0.2\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"wyhash\",\"req\":\"^0.5\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"js\":[\"std\",\"getrandom\"],\"std\":[\"alloc\"]}}", + "fixedbitset_0.4.2": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", "fnv_1.0.7": "{\"dependencies\":[],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "foldhash_0.1.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"ahash\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"chrono\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"hashbrown\",\"req\":\"^0.14\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"uuid\",\"req\":\"^1.8\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "futures-channel_0.3.32": "{\"dependencies\":[{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-sink\",\"optional\":true,\"req\":\"^0.3.32\"}],\"features\":{\"alloc\":[\"futures-core/alloc\"],\"cfg-target-has-atomic\":[],\"default\":[\"std\"],\"sink\":[\"futures-sink\"],\"std\":[\"alloc\",\"futures-core/std\"],\"unstable\":[]}}", + "futures-core_0.3.32": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"require-cas\"],\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.3\"}],\"features\":{\"alloc\":[],\"cfg-target-has-atomic\":[],\"default\":[\"std\"],\"std\":[\"alloc\"],\"unstable\":[]}}", + "futures-sink_0.3.32": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", + "futures-task_0.3.32": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"cfg-target-has-atomic\":[],\"default\":[\"std\"],\"std\":[\"alloc\"],\"unstable\":[]}}", + "futures-util_0.3.32": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"std\"],\"name\":\"futures-channel\",\"optional\":true,\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3.32\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"futures-io\",\"optional\":true,\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-macro\",\"optional\":true,\"req\":\"=0.3.32\"},{\"default_features\":false,\"name\":\"futures-sink\",\"optional\":true,\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-task\",\"req\":\"^0.3.32\"},{\"name\":\"futures_01\",\"optional\":true,\"package\":\"futures\",\"req\":\"^0.1.25\"},{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.26\"},{\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.2\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.6\"},{\"default_features\":false,\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4.7\"},{\"name\":\"spin\",\"optional\":true,\"req\":\"^0.10.0\"},{\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^0.1.11\"},{\"name\":\"tokio-io\",\"optional\":true,\"req\":\"^0.1.9\"}],\"features\":{\"alloc\":[\"futures-core/alloc\",\"futures-task/alloc\",\"slab\"],\"async-await\":[],\"async-await-macro\":[\"async-await\",\"futures-macro\"],\"bilock\":[],\"cfg-target-has-atomic\":[],\"channel\":[\"std\",\"futures-channel\"],\"compat\":[\"std\",\"futures_01\",\"libc\"],\"default\":[\"std\",\"async-await\",\"async-await-macro\"],\"io\":[\"std\",\"futures-io\",\"memchr\"],\"io-compat\":[\"io\",\"compat\",\"tokio-io\",\"libc\"],\"portable-atomic\":[\"futures-core/portable-atomic\"],\"sink\":[\"futures-sink\"],\"std\":[\"alloc\",\"futures-core/std\",\"futures-task/std\",\"slab/std\"],\"unstable\":[\"futures-core/unstable\",\"futures-task/unstable\"],\"write-all-vectored\":[\"io\"]}}", + "getrandom_0.2.17": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(unix)\"},{\"default_features\":false,\"name\":\"wasi\",\"req\":\"^0.11\",\"target\":\"cfg(target_os = \\\"wasi\\\")\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.62\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.18\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"}],\"features\":{\"custom\":[],\"js\":[\"wasm-bindgen\",\"js-sys\"],\"linux_disable_fallback\":[],\"rdrand\":[],\"rustc-dep-of-std\":[\"compiler_builtins\",\"core\",\"libc/rustc-dep-of-std\",\"wasi/rustc-dep-of-std\"],\"std\":[],\"test-in-browser\":[]}}", "getrandom_0.3.3": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3.77\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"), target_feature = \\\"atomics\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"netbsd\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"solaris\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"vxworks\\\")\"},{\"default_features\":false,\"name\":\"r-efi\",\"req\":\"^5.1\",\"target\":\"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\"},{\"default_features\":false,\"name\":\"wasi\",\"req\":\"^0.14\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.98\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"}],\"features\":{\"rustc-dep-of-std\":[\"dep:compiler_builtins\",\"dep:core\"],\"std\":[],\"wasm_js\":[\"dep:wasm-bindgen\",\"dep:js-sys\"]}}", + "getrandom_0.4.2": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3.77\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"), target_feature = \\\"atomics\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"netbsd\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"solaris\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"vxworks\\\")\"},{\"default_features\":false,\"name\":\"r-efi\",\"req\":\"^6\",\"target\":\"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\"},{\"name\":\"rand_core\",\"optional\":true,\"req\":\"^0.10.0\"},{\"default_features\":false,\"name\":\"wasip2\",\"req\":\"^1\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\"},{\"name\":\"wasip3\",\"req\":\"^0.4\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p3\\\"))\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.98\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"}],\"features\":{\"std\":[],\"sys_rng\":[\"dep:rand_core\"],\"wasm_js\":[\"dep:wasm-bindgen\",\"dep:js-sys\"]}}", + "gimli_0.32.3": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"fallible-iterator\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.0.0\"},{\"default_features\":false,\"name\":\"stable_deref_trait\",\"optional\":true,\"req\":\"^1.1.0\"},{\"kind\":\"dev\",\"name\":\"test-assembler\",\"req\":\"^0.1.3\"}],\"features\":{\"default\":[\"read-all\",\"write\"],\"endian-reader\":[\"read\",\"dep:stable_deref_trait\"],\"fallible-iterator\":[\"dep:fallible-iterator\"],\"read\":[\"read-core\"],\"read-all\":[\"read\",\"std\",\"fallible-iterator\",\"endian-reader\"],\"read-core\":[],\"rustc-dep-of-std\":[\"dep:core\",\"dep:alloc\"],\"std\":[\"fallible-iterator?/std\",\"stable_deref_trait?/std\"],\"write\":[\"dep:indexmap\"]}}", + "h2_0.4.6": "{\"dependencies\":[{\"name\":\"atomic-waker\",\"req\":\"^1.0.0\"},{\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.10\"},{\"name\":\"fnv\",\"req\":\"^1.0.5\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-sink\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.3\"},{\"name\":\"http\",\"req\":\"^1\"},{\"features\":[\"std\"],\"name\":\"indexmap\",\"req\":\"^2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.4\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.0\"},{\"name\":\"slab\",\"req\":\"^0.4.2\"},{\"features\":[\"io-util\"],\"name\":\"tokio\",\"req\":\"^1\"},{\"features\":[\"rt-multi-thread\",\"macros\",\"sync\",\"net\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tokio-rustls\",\"req\":\"^0.26\"},{\"features\":[\"codec\",\"io\"],\"name\":\"tokio-util\",\"req\":\"^0.7.1\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"req\":\"^0.1.35\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\"},{\"kind\":\"dev\",\"name\":\"webpki-roots\",\"req\":\"^0.26\"}],\"features\":{\"stream\":[],\"unstable\":[]}}", + "hashbrown_0.12.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"ahash\",\"optional\":true,\"req\":\"^0.7.0\"},{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"bumpalo\",\"optional\":true,\"req\":\"^3.5.0\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.3\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.25\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"ahash-compile-time-rng\":[\"ahash/compile-time-rng\"],\"default\":[\"ahash\",\"inline-more\"],\"inline-more\":[],\"nightly\":[],\"raw\":[],\"rustc-dep-of-std\":[\"nightly\",\"core\",\"compiler_builtins\",\"alloc\",\"rustc-internal-api\"],\"rustc-internal-api\":[]}}", + "hashbrown_0.15.5": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"allocator-api2\",\"optional\":true,\"req\":\"^0.2.9\"},{\"features\":[\"allocator-api2\"],\"kind\":\"dev\",\"name\":\"bumpalo\",\"req\":\"^3.13.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.1\"},{\"default_features\":false,\"name\":\"equivalent\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"default_features\":false,\"name\":\"foldhash\",\"optional\":true,\"req\":\"^0.1.2\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9.0\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.2\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.2\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.25\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"default-hasher\",\"inline-more\",\"allocator-api2\",\"equivalent\",\"raw-entry\"],\"default-hasher\":[\"dep:foldhash\"],\"inline-more\":[],\"nightly\":[\"bumpalo/allocator_api\"],\"raw-entry\":[],\"rustc-dep-of-std\":[\"nightly\",\"core\",\"alloc\",\"rustc-internal-api\"],\"rustc-internal-api\":[]}}", + "hashbrown_0.16.1": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"allocator-api2\",\"optional\":true,\"req\":\"^0.2.9\"},{\"features\":[\"allocator-api2\"],\"kind\":\"dev\",\"name\":\"bumpalo\",\"req\":\"^3.13.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"equivalent\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"default_features\":false,\"name\":\"foldhash\",\"optional\":true,\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.155\",\"target\":\"cfg(unix)\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9.0\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.9.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.2\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.221\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"default-hasher\",\"inline-more\",\"allocator-api2\",\"equivalent\",\"raw-entry\"],\"default-hasher\":[\"dep:foldhash\"],\"inline-more\":[],\"nightly\":[\"foldhash?/nightly\",\"bumpalo/allocator_api\"],\"raw-entry\":[],\"rustc-dep-of-std\":[\"nightly\",\"core\",\"alloc\",\"rustc-internal-api\"],\"rustc-internal-api\":[],\"serde\":[\"dep:serde_core\",\"dep:serde\"]}}", + "heck_0.5.0": "{\"dependencies\":[],\"features\":{}}", + "http-body-util_0.1.3": "{\"dependencies\":[{\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"http-body\",\"req\":\"^1\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"macros\",\"rt\",\"sync\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"}],\"features\":{\"channel\":[\"dep:tokio\"],\"default\":[],\"full\":[\"channel\"]}}", + "http-body_1.0.1": "{\"dependencies\":[{\"name\":\"bytes\",\"req\":\"^1\"},{\"name\":\"http\",\"req\":\"^1\"}],\"features\":{}}", + "http_1.4.0": "{\"dependencies\":[{\"name\":\"bytes\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"name\":\"itoa\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.0\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "httparse_1.10.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.5\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "httpdate_1.0.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"}],\"features\":{}}", + "hyper-timeout_0.5.2": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"name\":\"hyper\",\"req\":\"^1.1\"},{\"features\":[\"http1\"],\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^1.1\"},{\"kind\":\"dev\",\"name\":\"hyper-tls\",\"req\":\"^0.6\"},{\"features\":[\"client-legacy\",\"http1\"],\"name\":\"hyper-util\",\"req\":\"^0.1.10\"},{\"features\":[\"client-legacy\",\"http1\",\"server\",\"server-graceful\"],\"kind\":\"dev\",\"name\":\"hyper-util\",\"req\":\"^0.1.10\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"name\":\"tokio\",\"req\":\"^1.35\"},{\"features\":[\"io-std\",\"io-util\",\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.35\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"}],\"features\":{}}", + "hyper-util_0.1.20": "{\"dependencies\":[{\"name\":\"base64\",\"optional\":true,\"req\":\"^0.22\"},{\"name\":\"bytes\",\"req\":\"^1.7.1\"},{\"kind\":\"dev\",\"name\":\"bytes\",\"req\":\"^1\"},{\"name\":\"futures-channel\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.16\"},{\"default_features\":false,\"features\":[\"alloc\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3.16\"},{\"name\":\"http\",\"req\":\"^1.0\"},{\"name\":\"http-body\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1.0\"},{\"name\":\"hyper\",\"req\":\"^1.8.0\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^1.4.0\"},{\"name\":\"ipnet\",\"optional\":true,\"req\":\"^2.9\"},{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2\"},{\"name\":\"percent-encoding\",\"optional\":true,\"req\":\"^2.3\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.4\"},{\"kind\":\"dev\",\"name\":\"pnet_datalink\",\"req\":\"^0.35.0\",\"target\":\"cfg(any(target_os = \\\"linux\\\", target_os = \\\"macos\\\"))\"},{\"kind\":\"dev\",\"name\":\"pretty_env_logger\",\"req\":\"^0.5\"},{\"features\":[\"all\"],\"name\":\"socket2\",\"optional\":true,\"req\":\">=0.5.9, <0.7\"},{\"name\":\"system-configuration\",\"optional\":true,\"req\":\"^0.7\",\"target\":\"cfg(target_os = \\\"macos\\\")\"},{\"default_features\":false,\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"macros\",\"test-util\",\"signal\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"name\":\"tower-layer\",\"optional\":true,\"req\":\"^0.3\"},{\"name\":\"tower-service\",\"optional\":true,\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"tower-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"windows-registry\",\"optional\":true,\"req\":\">=0.3, <0.7\",\"target\":\"cfg(windows)\"}],\"features\":{\"__internal_happy_eyeballs_tests\":[],\"client\":[\"hyper/client\",\"tokio/net\",\"dep:tracing\",\"dep:futures-channel\",\"dep:tower-service\"],\"client-legacy\":[\"client\",\"dep:socket2\",\"tokio/sync\",\"dep:libc\",\"dep:futures-util\"],\"client-pool\":[\"client\",\"dep:futures-util\",\"dep:tower-layer\"],\"client-proxy\":[\"client\",\"dep:base64\",\"dep:ipnet\",\"dep:percent-encoding\"],\"client-proxy-system\":[\"dep:system-configuration\",\"dep:windows-registry\"],\"default\":[],\"full\":[\"client\",\"client-legacy\",\"client-pool\",\"client-proxy\",\"client-proxy-system\",\"server\",\"server-auto\",\"server-graceful\",\"service\",\"http1\",\"http2\",\"tokio\",\"tracing\"],\"http1\":[\"hyper/http1\"],\"http2\":[\"hyper/http2\"],\"server\":[\"hyper/server\"],\"server-auto\":[\"server\",\"http1\",\"http2\"],\"server-graceful\":[\"server\",\"tokio/sync\"],\"service\":[\"dep:tower-service\"],\"tokio\":[\"dep:tokio\",\"tokio/rt\",\"tokio/time\"],\"tracing\":[\"dep:tracing\"]}}", + "hyper_1.8.1": "{\"dependencies\":[{\"name\":\"atomic-waker\",\"optional\":true,\"req\":\"^1.1.2\"},{\"name\":\"bytes\",\"req\":\"^1.2\"},{\"kind\":\"dev\",\"name\":\"form_urlencoded\",\"req\":\"^1\"},{\"name\":\"futures-channel\",\"optional\":true,\"req\":\"^0.3\"},{\"features\":[\"sink\"],\"kind\":\"dev\",\"name\":\"futures-channel\",\"req\":\"^0.3\"},{\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3.31\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\",\"sink\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"h2\",\"optional\":true,\"req\":\"^0.4.2\"},{\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"http-body\",\"req\":\"^1\"},{\"name\":\"http-body-util\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"name\":\"httparse\",\"optional\":true,\"req\":\"^1.9\"},{\"name\":\"httpdate\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"itoa\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"pin-project-lite\",\"optional\":true,\"req\":\"^0.2.4\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.4\"},{\"name\":\"pin-utils\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"pretty_env_logger\",\"req\":\"^0.5\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"features\":[\"const_generics\",\"const_new\"],\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1.12\"},{\"kind\":\"dev\",\"name\":\"spmc\",\"req\":\"^0.3\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"req\":\"^1\"},{\"features\":[\"fs\",\"macros\",\"net\",\"io-std\",\"io-util\",\"rt\",\"rt-multi-thread\",\"sync\",\"time\",\"test-util\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"tokio-util\",\"req\":\"^0.7.10\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"want\",\"optional\":true,\"req\":\"^0.3\"}],\"features\":{\"capi\":[],\"client\":[\"dep:want\",\"dep:pin-project-lite\",\"dep:smallvec\"],\"default\":[],\"ffi\":[\"dep:http-body-util\",\"dep:futures-util\"],\"full\":[\"client\",\"http1\",\"http2\",\"server\"],\"http1\":[\"dep:atomic-waker\",\"dep:futures-channel\",\"dep:futures-core\",\"dep:httparse\",\"dep:itoa\",\"dep:pin-utils\"],\"http2\":[\"dep:futures-channel\",\"dep:futures-core\",\"dep:h2\"],\"nightly\":[],\"server\":[\"dep:httpdate\",\"dep:pin-project-lite\",\"dep:smallvec\"],\"tracing\":[\"dep:tracing\"]}}", + "id-arena_2.3.0": "{\"dependencies\":[{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.0.3\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", "ident_case_1.0.1": "{\"dependencies\":[],\"features\":{}}", + "indexmap_1.9.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"build\",\"name\":\"autocfg\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2.1\"},{\"default_features\":false,\"features\":[\"raw\"],\"name\":\"hashbrown\",\"req\":\"^0.12\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.3\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.4.1\"},{\"name\":\"rustc-rayon\",\"optional\":true,\"package\":\"rustc-rayon\",\"req\":\"^0.5\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0\"}],\"features\":{\"serde-1\":[\"serde\"],\"std\":[],\"test_debug\":[],\"test_low_transition_point\":[]}}", + "indexmap_2.13.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"borsh\",\"optional\":true,\"req\":\"^1.2\"},{\"default_features\":false,\"name\":\"equivalent\",\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"fastrand\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"hashbrown\",\"req\":\"^0.16.1\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.14\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.9\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"},{\"default_features\":false,\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.220\"},{\"default_features\":false,\"name\":\"sval\",\"optional\":true,\"req\":\"^2\"}],\"features\":{\"default\":[\"std\"],\"serde\":[\"dep:serde_core\",\"dep:serde\"],\"std\":[],\"test_debug\":[]}}", + "itertools_0.13.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"permutohedron\",\"req\":\"^0.2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"use_std\"],\"use_alloc\":[],\"use_std\":[\"use_alloc\",\"either/use_std\"]}}", "itertools_0.14.0": "{\"dependencies\":[{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"permutohedron\",\"req\":\"^0.2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"use_std\"],\"use_alloc\":[],\"use_std\":[\"use_alloc\",\"either/use_std\"]}}", "itoa_1.0.14": "{\"dependencies\":[{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"}],\"features\":{}}", + "itoa_1.0.17": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\",\"target\":\"cfg(not(miri))\"},{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"}],\"features\":{}}", + "leb128fmt_0.1.0": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[]}}", "libc_0.2.169": "{\"dependencies\":[{\"name\":\"rustc-std-workspace-core\",\"optional\":true,\"req\":\"^1.0.0\"}],\"features\":{\"align\":[],\"const-extern-fn\":[],\"default\":[\"std\"],\"extra_traits\":[],\"rustc-dep-of-std\":[\"align\",\"rustc-std-workspace-core\"],\"std\":[],\"use_std\":[\"std\"]}}", + "libc_0.2.183": "{\"dependencies\":[{\"name\":\"rustc-std-workspace-core\",\"optional\":true,\"req\":\"^1.0.1\"}],\"features\":{\"align\":[],\"const-extern-fn\":[],\"default\":[\"std\"],\"extra_traits\":[],\"rustc-dep-of-std\":[\"align\",\"rustc-std-workspace-core\"],\"std\":[],\"use_std\":[\"std\"]}}", + "linux-raw-sys_0.12.1": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.100\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1.0\"}],\"features\":{\"auxvec\":[],\"bootparam\":[],\"btrfs\":[],\"default\":[\"std\",\"general\",\"errno\"],\"elf\":[],\"elf_uapi\":[],\"errno\":[],\"general\":[],\"if_arp\":[],\"if_ether\":[],\"if_packet\":[],\"if_tun\":[],\"image\":[],\"io_uring\":[],\"ioctl\":[],\"landlock\":[],\"loop_device\":[],\"mempolicy\":[],\"net\":[],\"netlink\":[],\"no_std\":[],\"prctl\":[],\"ptrace\":[],\"rustc-dep-of-std\":[\"core\",\"no_std\"],\"std\":[],\"system\":[],\"vm_sockets\":[],\"xdp\":[]}}", + "lock_api_0.4.14": "{\"dependencies\":[{\"name\":\"owning_ref\",\"optional\":true,\"req\":\"^0.4.1\"},{\"default_features\":false,\"name\":\"scopeguard\",\"req\":\"^1.1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.126\"}],\"features\":{\"arc_lock\":[],\"atomic_usize\":[],\"default\":[\"atomic_usize\"],\"nightly\":[]}}", "log_0.4.22": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.63\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"sval\",\"optional\":true,\"req\":\"^2.1\"},{\"kind\":\"dev\",\"name\":\"sval\",\"req\":\"^2.1\"},{\"kind\":\"dev\",\"name\":\"sval_derive\",\"req\":\"^2.1\"},{\"default_features\":false,\"name\":\"sval_ref\",\"optional\":true,\"req\":\"^2.1\"},{\"default_features\":false,\"features\":[\"inline-i128\"],\"name\":\"value-bag\",\"optional\":true,\"req\":\"^1.7\"},{\"features\":[\"test\"],\"kind\":\"dev\",\"name\":\"value-bag\",\"req\":\"^1.7\"}],\"features\":{\"kv\":[],\"kv_serde\":[\"kv_std\",\"value-bag/serde\",\"serde\"],\"kv_std\":[\"std\",\"kv\",\"value-bag/error\"],\"kv_sval\":[\"kv\",\"value-bag/sval\",\"sval\",\"sval_ref\"],\"kv_unstable\":[\"kv\",\"value-bag\"],\"kv_unstable_serde\":[\"kv_serde\",\"kv_unstable_std\"],\"kv_unstable_std\":[\"kv_std\",\"kv_unstable\"],\"kv_unstable_sval\":[\"kv_sval\",\"kv_unstable\"],\"max_level_debug\":[],\"max_level_error\":[],\"max_level_info\":[],\"max_level_off\":[],\"max_level_trace\":[],\"max_level_warn\":[],\"release_max_level_debug\":[],\"release_max_level_error\":[],\"release_max_level_info\":[],\"release_max_level_off\":[],\"release_max_level_trace\":[],\"release_max_level_warn\":[],\"std\":[]}}", + "log_0.4.29": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.63\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"sval\",\"optional\":true,\"req\":\"^2.16\"},{\"kind\":\"dev\",\"name\":\"sval\",\"req\":\"^2.16\"},{\"kind\":\"dev\",\"name\":\"sval_derive\",\"req\":\"^2.16\"},{\"default_features\":false,\"name\":\"sval_ref\",\"optional\":true,\"req\":\"^2.16\"},{\"default_features\":false,\"features\":[\"inline-i128\"],\"name\":\"value-bag\",\"optional\":true,\"req\":\"^1.12\"},{\"features\":[\"test\"],\"kind\":\"dev\",\"name\":\"value-bag\",\"req\":\"^1.12\"}],\"features\":{\"kv\":[],\"kv_serde\":[\"kv_std\",\"value-bag/serde\",\"serde\"],\"kv_std\":[\"std\",\"kv\",\"value-bag/error\"],\"kv_sval\":[\"kv\",\"value-bag/sval\",\"sval\",\"sval_ref\"],\"kv_unstable\":[\"kv\",\"value-bag\"],\"kv_unstable_serde\":[\"kv_serde\",\"kv_unstable_std\"],\"kv_unstable_std\":[\"kv_std\",\"kv_unstable\"],\"kv_unstable_sval\":[\"kv_sval\",\"kv_unstable\"],\"max_level_debug\":[],\"max_level_error\":[],\"max_level_info\":[],\"max_level_off\":[],\"max_level_trace\":[],\"max_level_warn\":[],\"release_max_level_debug\":[],\"release_max_level_error\":[],\"release_max_level_info\":[],\"release_max_level_off\":[],\"release_max_level_trace\":[],\"release_max_level_warn\":[],\"serde\":[\"serde_core\"],\"std\":[]}}", "macro_rules_attribute-proc_macro_0.2.0": "{\"dependencies\":[],\"features\":{\"verbose-expansions\":[]}}", "macro_rules_attribute_0.2.0": "{\"dependencies\":[{\"name\":\"macro_rules_attribute-proc_macro\",\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.10.0\"},{\"name\":\"paste\",\"req\":\"^1.0.7\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.8\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.136\"}],\"features\":{\"better-docs\":[],\"default\":[],\"verbose-expansions\":[\"macro_rules_attribute-proc_macro/verbose-expansions\"]}}", + "matchit_0.7.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"actix-router\",\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.4\"},{\"kind\":\"dev\",\"name\":\"gonzales\",\"req\":\"^0.0.3-beta\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^0.14\"},{\"kind\":\"dev\",\"name\":\"path-tree\",\"req\":\"^0.2.2\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.5.4\"},{\"kind\":\"dev\",\"name\":\"route-recognizer\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"routefinder\",\"req\":\"^0.5.2\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"features\":[\"make\",\"util\"],\"kind\":\"dev\",\"name\":\"tower\",\"req\":\"^0.4\"}],\"features\":{\"__test_helpers\":[],\"default\":[]}}", "memchr_2.7.4": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.20\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"libc\":[],\"logging\":[\"dep:log\"],\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"],\"std\":[\"alloc\"],\"use_std\":[\"std\"]}}", + "memchr_2.8.0": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.20\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"libc\":[],\"logging\":[\"dep:log\"],\"rustc-dep-of-std\":[\"core\"],\"std\":[\"alloc\"],\"use_std\":[\"std\"]}}", + "mime_0.3.17": "{\"dependencies\":[],\"features\":{}}", "minimal-lexical_0.2.1": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"compact\":[],\"default\":[\"std\"],\"lint\":[],\"nightly\":[],\"std\":[]}}", + "miniz_oxide_0.8.9": "{\"dependencies\":[{\"default_features\":false,\"name\":\"adler2\",\"req\":\"^2.0\"},{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"simd-adler32\",\"optional\":true,\"req\":\"^0.3.3\"}],\"features\":{\"block-boundary\":[],\"default\":[\"with-alloc\"],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"adler2/rustc-dep-of-std\"],\"simd\":[\"simd-adler32\"],\"std\":[],\"with-alloc\":[]}}", + "mio_1.1.1": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"libc\",\"req\":\"^0.2.178\",\"target\":\"cfg(target_os = \\\"hermit\\\")\"},{\"name\":\"libc\",\"req\":\"^0.2.178\",\"target\":\"cfg(target_os = \\\"wasi\\\")\"},{\"name\":\"libc\",\"req\":\"^0.2.178\",\"target\":\"cfg(unix)\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.8\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9\"},{\"name\":\"wasi\",\"req\":\"^0.11.0\",\"target\":\"cfg(target_os = \\\"wasi\\\")\"},{\"features\":[\"Wdk_Foundation\",\"Wdk_Storage_FileSystem\",\"Wdk_System_IO\",\"Win32_Foundation\",\"Win32_Networking_WinSock\",\"Win32_Storage_FileSystem\",\"Win32_Security\",\"Win32_System_IO\",\"Win32_System_WindowsProgramming\"],\"name\":\"windows-sys\",\"req\":\"^0.61\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[\"log\"],\"net\":[],\"os-ext\":[\"os-poll\",\"windows-sys/Win32_System_Pipes\",\"windows-sys/Win32_Security\"],\"os-poll\":[]}}", "monostate-impl_0.1.13": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"default_features\":false,\"features\":[\"parsing\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0.59\"}],\"features\":{}}", "monostate_0.1.13": "{\"dependencies\":[{\"name\":\"monostate-impl\",\"req\":\"=0.1.13\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.166\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.99\"}],\"features\":{}}", + "multimap_0.10.1": "{\"dependencies\":[{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"serde_impl\"],\"serde_impl\":[\"serde\"]}}", "nom_7.1.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2.3\"},{\"default_features\":false,\"name\":\"minimal-lexical\",\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.0.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"docsrs\":[],\"std\":[\"alloc\",\"memchr/std\",\"minimal-lexical/std\"]}}", + "object_0.37.3": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"crc32fast\",\"optional\":true,\"req\":\"^1.2\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"default-hasher\"],\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.15.0\"},{\"default_features\":false,\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.0\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2.4.1\"},{\"name\":\"ruzstd\",\"optional\":true,\"req\":\"^0.8.1\"},{\"default_features\":false,\"name\":\"wasmparser\",\"optional\":true,\"req\":\"^0.236.0\"}],\"features\":{\"all\":[\"read\",\"write\",\"build\",\"std\",\"compression\",\"wasm\"],\"archive\":[],\"build\":[\"build_core\",\"write_std\",\"elf\"],\"build_core\":[\"read_core\",\"write_core\"],\"cargo-all\":[],\"coff\":[],\"compression\":[\"dep:flate2\",\"dep:ruzstd\",\"std\"],\"default\":[\"read\",\"compression\"],\"doc\":[\"read_core\",\"write_std\",\"build_core\",\"std\",\"compression\",\"archive\",\"coff\",\"elf\",\"macho\",\"pe\",\"wasm\",\"xcoff\"],\"elf\":[],\"macho\":[],\"pe\":[\"coff\"],\"read\":[\"read_core\",\"archive\",\"coff\",\"elf\",\"macho\",\"pe\",\"xcoff\",\"unaligned\"],\"read_core\":[],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"memchr/rustc-dep-of-std\"],\"std\":[\"memchr/std\"],\"unaligned\":[],\"unstable\":[],\"unstable-all\":[\"all\",\"unstable\"],\"wasm\":[\"dep:wasmparser\"],\"write\":[\"write_std\",\"coff\",\"elf\",\"macho\",\"pe\",\"xcoff\"],\"write_core\":[\"dep:crc32fast\",\"dep:indexmap\",\"dep:hashbrown\"],\"write_std\":[\"write_core\",\"std\",\"indexmap?/std\",\"crc32fast?/std\"],\"xcoff\":[]}}", "once_cell_1.20.2": "{\"dependencies\":[{\"name\":\"critical-section\",\"optional\":true,\"req\":\"^1.1.3\"},{\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"critical-section\",\"req\":\"^1.1.3\"},{\"default_features\":false,\"name\":\"parking_lot_core\",\"optional\":true,\"req\":\"^0.9.10\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.10.6\"}],\"features\":{\"alloc\":[\"race\"],\"atomic-polyfill\":[\"critical-section\"],\"critical-section\":[\"dep:critical-section\",\"portable-atomic\"],\"default\":[\"std\"],\"parking_lot\":[\"dep:parking_lot_core\"],\"portable-atomic\":[\"dep:portable-atomic\"],\"race\":[],\"std\":[\"alloc\"],\"unstable\":[]}}", + "once_cell_1.21.4": "{\"dependencies\":[{\"name\":\"critical-section\",\"optional\":true,\"req\":\"^1.1.3\"},{\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"critical-section\",\"req\":\"^1.1.3\"},{\"default_features\":false,\"name\":\"parking_lot_core\",\"optional\":true,\"req\":\"^0.9.10\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.10.6\"}],\"features\":{\"alloc\":[\"race\"],\"atomic-polyfill\":[\"critical-section\"],\"critical-section\":[\"dep:critical-section\",\"portable-atomic\"],\"default\":[\"std\"],\"parking_lot\":[\"dep:parking_lot_core\"],\"portable-atomic\":[\"dep:portable-atomic\"],\"race\":[],\"std\":[\"alloc\"],\"unstable\":[]}}", "onig_6.5.1": "{\"dependencies\":[{\"name\":\"bitflags\",\"req\":\"^2.4.0\"},{\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(windows)\"},{\"name\":\"once_cell\",\"req\":\"^1.12\"},{\"default_features\":false,\"name\":\"onig_sys\",\"req\":\"^69.9.1\"}],\"features\":{\"default\":[\"generate\"],\"generate\":[\"onig_sys/generate\"],\"posix-api\":[\"onig_sys/posix-api\"],\"print-debug\":[\"onig_sys/print-debug\"],\"std-pattern\":[]}}", "onig_sys_69.9.1": "{\"dependencies\":[{\"features\":[\"runtime\"],\"kind\":\"build\",\"name\":\"bindgen\",\"optional\":true,\"req\":\"^0.71\"},{\"kind\":\"build\",\"name\":\"cc\",\"req\":\"^1.0\"},{\"kind\":\"build\",\"name\":\"pkg-config\",\"req\":\"^0.3.16\"}],\"features\":{\"default\":[\"generate\"],\"generate\":[\"bindgen\"],\"posix-api\":[],\"print-debug\":[]}}", + "parking_lot_0.12.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.3.3\"},{\"name\":\"lock_api\",\"req\":\"^0.4.14\"},{\"name\":\"parking_lot_core\",\"req\":\"^0.9.12\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.3\"}],\"features\":{\"arc_lock\":[\"lock_api/arc_lock\"],\"deadlock_detection\":[\"parking_lot_core/deadlock_detection\"],\"default\":[],\"hardware-lock-elision\":[],\"nightly\":[\"parking_lot_core/nightly\",\"lock_api/nightly\"],\"owning_ref\":[\"lock_api/owning_ref\"],\"send_guard\":[],\"serde\":[\"lock_api/serde\"]}}", + "parking_lot_core_0.9.12": "{\"dependencies\":[{\"name\":\"backtrace\",\"optional\":true,\"req\":\"^0.3.60\"},{\"name\":\"cfg-if\",\"req\":\"^1.0.0\"},{\"name\":\"libc\",\"req\":\"^0.2.95\",\"target\":\"cfg(unix)\"},{\"name\":\"petgraph\",\"optional\":true,\"req\":\"^0.6.0\"},{\"name\":\"redox_syscall\",\"req\":\"^0.5\",\"target\":\"cfg(target_os = \\\"redox\\\")\"},{\"name\":\"smallvec\",\"req\":\"^1.6.1\"},{\"name\":\"windows-link\",\"req\":\"^0.2.0\",\"target\":\"cfg(windows)\"}],\"features\":{\"deadlock_detection\":[\"petgraph\",\"backtrace\"],\"nightly\":[]}}", "paste_1.0.15": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"paste-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", + "percent-encoding_2.3.2": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", + "petgraph_0.6.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"ahash\",\"req\":\"^0.7.2\"},{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.3.3\"},{\"kind\":\"dev\",\"name\":\"defmac\",\"req\":\"^0.2.1\"},{\"default_features\":false,\"name\":\"fixedbitset\",\"req\":\"^0.4.0\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2.1\"},{\"name\":\"indexmap\",\"req\":\"^2.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.12.1\"},{\"kind\":\"dev\",\"name\":\"odds\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.5.5\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.5.3\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1.0\"}],\"features\":{\"all\":[\"unstable\",\"quickcheck\",\"matrix_graph\",\"stable_graph\",\"graphmap\",\"rayon\"],\"default\":[\"graphmap\",\"stable_graph\",\"matrix_graph\"],\"generate\":[],\"graphmap\":[],\"matrix_graph\":[],\"rayon\":[\"dep:rayon\",\"indexmap/rayon\"],\"serde-1\":[\"serde\",\"serde_derive\"],\"stable_graph\":[],\"unstable\":[\"generate\"]}}", + "pin-project-internal_1.1.11": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1.0.25\"},{\"default_features\":false,\"features\":[\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\",\"full\",\"visit-mut\"],\"name\":\"syn\",\"req\":\"^2.0.1\"}],\"features\":{}}", + "pin-project-lite_0.2.17": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1\"}],\"features\":{}}", + "pin-project_1.1.11": "{\"dependencies\":[{\"name\":\"pin-project-internal\",\"req\":\"=1.1.11\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1\"}],\"features\":{}}", + "pin-utils_0.1.0": "{\"dependencies\":[],\"features\":{}}", "pkg-config_0.3.32": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1\"}],\"features\":{}}", "ppv-lite86_0.2.20": "{\"dependencies\":[{\"features\":[\"simd\",\"derive\"],\"name\":\"zerocopy\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"std\"],\"no_simd\":[],\"simd\":[],\"std\":[]}}", + "ppv-lite86_0.2.21": "{\"dependencies\":[{\"features\":[\"simd\"],\"name\":\"zerocopy\",\"req\":\"^0.8.23\"}],\"features\":{\"default\":[\"std\"],\"no_simd\":[],\"simd\":[],\"std\":[]}}", + "prettyplease_0.2.37": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"default_features\":false,\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.105\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"extra-traits\",\"parsing\",\"printing\",\"visit-mut\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.105\"}],\"features\":{\"verbatim\":[\"syn/parsing\"]}}", + "proc-macro2_1.0.106": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4\"},{\"name\":\"unicode-ident\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"proc-macro\"],\"nightly\":[],\"proc-macro\":[],\"span-locations\":[]}}", "proc-macro2_1.0.92": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4\"},{\"name\":\"unicode-ident\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"proc-macro\"],\"nightly\":[],\"proc-macro\":[],\"span-locations\":[]}}", + "prost-build_0.13.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"heck\",\"req\":\">=0.4, <=0.5\"},{\"default_features\":false,\"features\":[\"use_alloc\"],\"name\":\"itertools\",\"req\":\">=0.10, <=0.13\"},{\"name\":\"log\",\"req\":\"^0.4.4\"},{\"default_features\":false,\"name\":\"multimap\",\"req\":\">=0.8, <=0.10\"},{\"name\":\"once_cell\",\"req\":\"^1.17.1\"},{\"default_features\":false,\"name\":\"petgraph\",\"req\":\"^0.6\"},{\"name\":\"prettyplease\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"name\":\"prost\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-types\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"pulldown-cmark\",\"optional\":true,\"req\":\"^0.9.1\"},{\"name\":\"pulldown-cmark-to-cmark\",\"optional\":true,\"req\":\"^10.0.1\"},{\"default_features\":false,\"features\":[\"std\",\"unicode-bool\"],\"name\":\"regex\",\"req\":\"^1.8.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"optional\":true,\"req\":\"^2\"},{\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{\"cleanup-markdown\":[\"dep:pulldown-cmark\",\"dep:pulldown-cmark-to-cmark\"],\"default\":[\"format\"],\"format\":[\"dep:prettyplease\",\"dep:syn\"]}}", + "prost-derive_0.13.1": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.1\"},{\"name\":\"itertools\",\"req\":\">=0.10.1, <=0.13\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2\"}],\"features\":{}}", + "prost-types_0.13.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"prost-derive\"],\"name\":\"prost\",\"req\":\"^0.13.1\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"prost/std\"]}}", + "prost_0.13.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"name\":\"prost-derive\",\"optional\":true,\"req\":\"^0.13.1\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"derive\",\"std\"],\"derive\":[\"dep:prost-derive\"],\"no-recursion-limit\":[],\"prost-derive\":[\"derive\"],\"std\":[]}}", + "protoc-gen-prost_0.4.0": "{\"dependencies\":[{\"name\":\"once_cell\",\"req\":\"^1.10.0\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"prost\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-build\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-types\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"regex\",\"req\":\"^1.5.5\"}],\"features\":{}}", + "protoc-gen-tonic_0.4.1": "{\"dependencies\":[{\"name\":\"heck\",\"req\":\"^0.5.0\"},{\"name\":\"prettyplease\",\"req\":\"^0.2.9\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"prost\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-build\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-types\",\"req\":\"^0.13.1\"},{\"name\":\"protoc-gen-prost\",\"req\":\"^0.4.0\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"regex\",\"req\":\"^1.5.5\"},{\"features\":[\"parsing\",\"full\"],\"name\":\"syn\",\"req\":\"^2.0.22\"},{\"name\":\"tonic-build\",\"req\":\"^0.12.0\"}],\"features\":{}}", "quote_1.0.37": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.66\"}],\"features\":{\"default\":[\"proc-macro\"],\"proc-macro\":[\"proc-macro2/proc-macro\"]}}", + "quote_1.0.45": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"}],\"features\":{\"default\":[\"proc-macro\"],\"proc-macro\":[\"proc-macro2/proc-macro\"]}}", "r-efi_5.3.0": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"efiapi\":[],\"examples\":[\"native\"],\"native\":[],\"rustc-dep-of-std\":[\"core\"]}}", + "r-efi_6.0.0": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"native\":[],\"rustc-dep-of-std\":[\"core\"]}}", + "rand_0.8.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.2.1\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.22\",\"target\":\"cfg(unix)\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.4\"},{\"features\":[\"into_bits\"],\"name\":\"packed_simd\",\"optional\":true,\"package\":\"packed_simd_2\",\"req\":\"^0.3.7\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"rand_core\",\"req\":\"^0.6.0\"},{\"kind\":\"dev\",\"name\":\"rand_pcg\",\"req\":\"^0.3.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"}],\"features\":{\"alloc\":[\"rand_core/alloc\"],\"default\":[\"std\",\"std_rng\"],\"getrandom\":[\"rand_core/getrandom\"],\"min_const_gen\":[],\"nightly\":[],\"serde1\":[\"serde\",\"rand_core/serde1\"],\"simd_support\":[\"packed_simd\"],\"small_rng\":[],\"std\":[\"rand_core/std\",\"rand_chacha/std\",\"alloc\",\"getrandom\",\"libc\"],\"std_rng\":[\"rand_chacha\"]}}", "rand_0.9.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.2.1\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.4\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"optional\":true,\"req\":\"^0.9.0\"},{\"default_features\":false,\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rand_pcg\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.7\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"std_rng\",\"os_rng\",\"small_rng\",\"thread_rng\"],\"log\":[\"dep:log\"],\"nightly\":[],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\",\"rand_core/serde\"],\"simd_support\":[],\"small_rng\":[],\"std\":[\"rand_core/std\",\"rand_chacha?/std\",\"alloc\"],\"std_rng\":[\"dep:rand_chacha\"],\"thread_rng\":[\"std\",\"std_rng\",\"os_rng\"],\"unbiased\":[]}}", + "rand_chacha_0.3.1": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"ppv-lite86\",\"req\":\"^0.2.8\"},{\"name\":\"rand_core\",\"req\":\"^0.6.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"serde1\":[\"serde\"],\"simd\":[],\"std\":[\"ppv-lite86/std\"]}}", "rand_chacha_0.9.0": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"ppv-lite86\",\"req\":\"^0.2.14\"},{\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"features\":[\"os_rng\"],\"kind\":\"dev\",\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\"],\"std\":[\"ppv-lite86/std\",\"rand_core/std\"]}}", + "rand_core_0.6.4": "{\"dependencies\":[{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.2\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"serde1\":[\"serde\"],\"std\":[\"alloc\",\"getrandom\",\"getrandom/std\"]}}", "rand_core_0.9.3": "{\"dependencies\":[{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.3.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"os_rng\":[\"dep:getrandom\"],\"serde\":[\"dep:serde\"],\"std\":[\"getrandom?/std\"]}}", "rayon-cond_0.4.0": "{\"dependencies\":[{\"name\":\"either\",\"req\":\"^1.6\"},{\"name\":\"itertools\",\"req\":\"^0.14\"},{\"name\":\"rayon\",\"req\":\"^1.7\"}],\"features\":{}}", "rayon-core_1.12.1": "{\"dependencies\":[{\"name\":\"crossbeam-deque\",\"req\":\"^0.8.1\"},{\"name\":\"crossbeam-utils\",\"req\":\"^0.8.0\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"scoped-tls\",\"req\":\"^1.0\"},{\"name\":\"wasm_sync\",\"optional\":true,\"req\":\"^0.1.0\"}],\"features\":{\"web_spin_lock\":[\"dep:wasm_sync\"]}}", "rayon_1.10.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"name\":\"rayon-core\",\"req\":\"^1.12.1\"},{\"name\":\"wasm_sync\",\"optional\":true,\"req\":\"^0.1.0\"}],\"features\":{\"web_spin_lock\":[\"dep:wasm_sync\",\"rayon-core/web_spin_lock\"]}}", + "redox_syscall_0.5.18": "{\"dependencies\":[{\"name\":\"bitflags\",\"req\":\"^2.4\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"loom\",\"req\":\"^0.7\",\"target\":\"cfg(loom)\"}],\"features\":{\"default\":[\"userspace\"],\"rustc-dep-of-std\":[\"core\",\"bitflags/rustc-dep-of-std\"],\"std\":[],\"userspace\":[]}}", + "regex-automata_0.4.14": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"bstr\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.14\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"optional\":true,\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"syntax\",\"perf\",\"unicode\",\"meta\",\"nfa\",\"dfa\",\"hybrid\"],\"dfa\":[\"dfa-build\",\"dfa-search\",\"dfa-onepass\"],\"dfa-build\":[\"nfa-thompson\",\"dfa-search\"],\"dfa-onepass\":[\"nfa-thompson\"],\"dfa-search\":[],\"hybrid\":[\"alloc\",\"nfa-thompson\"],\"internal-instrument\":[\"internal-instrument-pikevm\"],\"internal-instrument-pikevm\":[\"logging\",\"std\"],\"logging\":[\"dep:log\",\"aho-corasick?/logging\",\"memchr?/logging\"],\"meta\":[\"syntax\",\"nfa-pikevm\"],\"nfa\":[\"nfa-thompson\",\"nfa-pikevm\",\"nfa-backtrack\"],\"nfa-backtrack\":[\"nfa-thompson\"],\"nfa-pikevm\":[\"nfa-thompson\"],\"nfa-thompson\":[\"alloc\"],\"perf\":[\"perf-inline\",\"perf-literal\"],\"perf-inline\":[],\"perf-literal\":[\"perf-literal-substring\",\"perf-literal-multisubstring\"],\"perf-literal-multisubstring\":[\"dep:aho-corasick\"],\"perf-literal-substring\":[\"aho-corasick?/perf-literal\",\"dep:memchr\"],\"std\":[\"regex-syntax?/std\",\"memchr?/std\",\"aho-corasick?/std\",\"alloc\"],\"syntax\":[\"dep:regex-syntax\",\"alloc\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"unicode-word-boundary\",\"regex-syntax?/unicode\"],\"unicode-age\":[\"regex-syntax?/unicode-age\"],\"unicode-bool\":[\"regex-syntax?/unicode-bool\"],\"unicode-case\":[\"regex-syntax?/unicode-case\"],\"unicode-gencat\":[\"regex-syntax?/unicode-gencat\"],\"unicode-perl\":[\"regex-syntax?/unicode-perl\"],\"unicode-script\":[\"regex-syntax?/unicode-script\"],\"unicode-segment\":[\"regex-syntax?/unicode-segment\"],\"unicode-word-boundary\":[]}}", "regex-automata_0.4.9": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"bstr\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.14\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"optional\":true,\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"syntax\",\"perf\",\"unicode\",\"meta\",\"nfa\",\"dfa\",\"hybrid\"],\"dfa\":[\"dfa-build\",\"dfa-search\",\"dfa-onepass\"],\"dfa-build\":[\"nfa-thompson\",\"dfa-search\"],\"dfa-onepass\":[\"nfa-thompson\"],\"dfa-search\":[],\"hybrid\":[\"alloc\",\"nfa-thompson\"],\"internal-instrument\":[\"internal-instrument-pikevm\"],\"internal-instrument-pikevm\":[\"logging\",\"std\"],\"logging\":[\"dep:log\",\"aho-corasick?/logging\",\"memchr?/logging\"],\"meta\":[\"syntax\",\"nfa-pikevm\"],\"nfa\":[\"nfa-thompson\",\"nfa-pikevm\",\"nfa-backtrack\"],\"nfa-backtrack\":[\"nfa-thompson\"],\"nfa-pikevm\":[\"nfa-thompson\"],\"nfa-thompson\":[\"alloc\"],\"perf\":[\"perf-inline\",\"perf-literal\"],\"perf-inline\":[],\"perf-literal\":[\"perf-literal-substring\",\"perf-literal-multisubstring\"],\"perf-literal-multisubstring\":[\"std\",\"dep:aho-corasick\"],\"perf-literal-substring\":[\"aho-corasick?/perf-literal\",\"dep:memchr\"],\"std\":[\"regex-syntax?/std\",\"memchr?/std\",\"aho-corasick?/std\",\"alloc\"],\"syntax\":[\"dep:regex-syntax\",\"alloc\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"unicode-word-boundary\",\"regex-syntax?/unicode\"],\"unicode-age\":[\"regex-syntax?/unicode-age\"],\"unicode-bool\":[\"regex-syntax?/unicode-bool\"],\"unicode-case\":[\"regex-syntax?/unicode-case\"],\"unicode-gencat\":[\"regex-syntax?/unicode-gencat\"],\"unicode-perl\":[\"regex-syntax?/unicode-perl\"],\"unicode-script\":[\"regex-syntax?/unicode-script\"],\"unicode-segment\":[\"regex-syntax?/unicode-segment\"],\"unicode-word-boundary\":[]}}", + "regex-syntax_0.8.10": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.3.0\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"default\":[\"std\",\"unicode\"],\"std\":[],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\"],\"unicode-age\":[],\"unicode-bool\":[],\"unicode-case\":[],\"unicode-gencat\":[],\"unicode-perl\":[],\"unicode-script\":[],\"unicode-segment\":[]}}", "regex-syntax_0.8.5": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.3.0\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"default\":[\"std\",\"unicode\"],\"std\":[],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\"],\"unicode-age\":[],\"unicode-bool\":[],\"unicode-case\":[],\"unicode-gencat\":[],\"unicode-perl\":[],\"unicode-script\":[],\"unicode-segment\":[]}}", "regex_1.11.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.17.1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"features\":[\"alloc\",\"syntax\",\"meta\",\"nfa-pikevm\"],\"name\":\"regex-automata\",\"req\":\"^0.4.8\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"default\":[\"std\",\"perf\",\"unicode\",\"regex-syntax/default\"],\"logging\":[\"aho-corasick?/logging\",\"memchr?/logging\",\"regex-automata/logging\"],\"pattern\":[],\"perf\":[\"perf-cache\",\"perf-dfa\",\"perf-onepass\",\"perf-backtrack\",\"perf-inline\",\"perf-literal\"],\"perf-backtrack\":[\"regex-automata/nfa-backtrack\"],\"perf-cache\":[],\"perf-dfa\":[\"regex-automata/hybrid\"],\"perf-dfa-full\":[\"regex-automata/dfa-build\",\"regex-automata/dfa-search\"],\"perf-inline\":[\"regex-automata/perf-inline\"],\"perf-literal\":[\"dep:aho-corasick\",\"dep:memchr\",\"regex-automata/perf-literal\"],\"perf-onepass\":[\"regex-automata/dfa-onepass\"],\"std\":[\"aho-corasick?/std\",\"memchr?/std\",\"regex-automata/std\",\"regex-syntax/std\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"regex-automata/unicode\",\"regex-syntax/unicode\"],\"unicode-age\":[\"regex-automata/unicode-age\",\"regex-syntax/unicode-age\"],\"unicode-bool\":[\"regex-automata/unicode-bool\",\"regex-syntax/unicode-bool\"],\"unicode-case\":[\"regex-automata/unicode-case\",\"regex-syntax/unicode-case\"],\"unicode-gencat\":[\"regex-automata/unicode-gencat\",\"regex-syntax/unicode-gencat\"],\"unicode-perl\":[\"regex-automata/unicode-perl\",\"regex-automata/unicode-word-boundary\",\"regex-syntax/unicode-perl\"],\"unicode-script\":[\"regex-automata/unicode-script\",\"regex-syntax/unicode-script\"],\"unicode-segment\":[\"regex-automata/unicode-segment\",\"regex-syntax/unicode-segment\"],\"unstable\":[\"pattern\"],\"use_std\":[\"std\"]}}", + "regex_1.12.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"features\":[\"alloc\",\"syntax\",\"meta\",\"nfa-pikevm\"],\"name\":\"regex-automata\",\"req\":\"^0.4.12\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"default\":[\"std\",\"perf\",\"unicode\",\"regex-syntax/default\"],\"logging\":[\"aho-corasick?/logging\",\"memchr?/logging\",\"regex-automata/logging\"],\"pattern\":[],\"perf\":[\"perf-cache\",\"perf-dfa\",\"perf-onepass\",\"perf-backtrack\",\"perf-inline\",\"perf-literal\"],\"perf-backtrack\":[\"regex-automata/nfa-backtrack\"],\"perf-cache\":[],\"perf-dfa\":[\"regex-automata/hybrid\"],\"perf-dfa-full\":[\"regex-automata/dfa-build\",\"regex-automata/dfa-search\"],\"perf-inline\":[\"regex-automata/perf-inline\"],\"perf-literal\":[\"dep:aho-corasick\",\"dep:memchr\",\"regex-automata/perf-literal\"],\"perf-onepass\":[\"regex-automata/dfa-onepass\"],\"std\":[\"aho-corasick?/std\",\"memchr?/std\",\"regex-automata/std\",\"regex-syntax/std\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"regex-automata/unicode\",\"regex-syntax/unicode\"],\"unicode-age\":[\"regex-automata/unicode-age\",\"regex-syntax/unicode-age\"],\"unicode-bool\":[\"regex-automata/unicode-bool\",\"regex-syntax/unicode-bool\"],\"unicode-case\":[\"regex-automata/unicode-case\",\"regex-syntax/unicode-case\"],\"unicode-gencat\":[\"regex-automata/unicode-gencat\",\"regex-syntax/unicode-gencat\"],\"unicode-perl\":[\"regex-automata/unicode-perl\",\"regex-automata/unicode-word-boundary\",\"regex-syntax/unicode-perl\"],\"unicode-script\":[\"regex-automata/unicode-script\",\"regex-syntax/unicode-script\"],\"unicode-segment\":[\"regex-automata/unicode-segment\",\"regex-syntax/unicode-segment\"],\"unstable\":[\"pattern\"],\"use_std\":[\"std\"]}}", + "rustc-demangle_0.1.27": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"compiler_builtins\":[],\"rustc-dep-of-std\":[\"core\"],\"std\":[]}}", + "rustix_1.1.4": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bitflags\",\"req\":\"^2.4.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4\",\"target\":\"cfg(all(criterion, not(any(target_os = \\\"emscripten\\\", target_os = \\\"wasi\\\"))))\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.182\",\"target\":\"cfg(all(not(windows), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.182\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.171\"},{\"default_features\":false,\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(all(not(windows), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(windows)\"},{\"default_features\":false,\"name\":\"libc_errno\",\"optional\":true,\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\"},{\"default_features\":false,\"features\":[\"general\",\"ioctl\",\"no_std\"],\"name\":\"linux-raw-sys\",\"req\":\"^0.12\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"features\":[\"auxvec\",\"general\",\"errno\",\"ioctl\",\"no_std\",\"elf\"],\"name\":\"linux-raw-sys\",\"req\":\"^0.12\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"kind\":\"dev\",\"name\":\"memoffset\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.20.3\",\"target\":\"cfg(windows)\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"serial_test\",\"req\":\"^2.0.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.5.0\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\"],\"name\":\"windows-sys\",\"req\":\">=0.52, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"all-apis\":[\"event\",\"fs\",\"io_uring\",\"mm\",\"mount\",\"net\",\"param\",\"pipe\",\"process\",\"pty\",\"rand\",\"runtime\",\"shm\",\"stdio\",\"system\",\"termios\",\"thread\",\"time\"],\"alloc\":[],\"default\":[\"std\"],\"event\":[],\"fs\":[],\"io_uring\":[\"event\",\"fs\",\"net\",\"thread\",\"linux-raw-sys/io_uring\"],\"linux_4_11\":[],\"linux_5_1\":[\"linux_4_11\"],\"linux_5_11\":[\"linux_5_1\"],\"linux_latest\":[\"linux_5_11\"],\"mm\":[],\"mount\":[],\"net\":[\"linux-raw-sys/net\",\"linux-raw-sys/netlink\",\"linux-raw-sys/if_ether\",\"linux-raw-sys/xdp\"],\"param\":[],\"pipe\":[],\"process\":[\"linux-raw-sys/prctl\"],\"pty\":[\"fs\"],\"rand\":[],\"runtime\":[\"linux-raw-sys/prctl\"],\"rustc-dep-of-std\":[\"core\",\"rustc-std-workspace-alloc\",\"linux-raw-sys/rustc-dep-of-std\",\"bitflags/rustc-dep-of-std\"],\"shm\":[\"fs\"],\"std\":[\"bitflags/std\",\"alloc\",\"libc?/std\",\"libc_errno?/std\"],\"stdio\":[],\"system\":[\"linux-raw-sys/system\"],\"termios\":[],\"thread\":[\"linux-raw-sys/prctl\"],\"time\":[],\"try_close\":[],\"use-explicitly-provided-auxv\":[],\"use-libc\":[\"libc_errno\",\"libc\"],\"use-libc-auxv\":[]}}", "rustversion_1.0.21": "{\"dependencies\":[{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", + "rustversion_1.0.22": "{\"dependencies\":[{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", "ryu_1.0.18": "{\"dependencies\":[{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"num_cpus\",\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"}],\"features\":{\"small\":[]}}", + "scopeguard_1.2.0": "{\"dependencies\":[],\"features\":{\"default\":[\"use_std\"],\"use_std\":[]}}", + "semver_1.0.27": "{\"dependencies\":[{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"package\":\"serde_core\",\"req\":\"^1.0.220\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"}],\"features\":{\"default\":[\"std\"],\"serde\":[\"dep:serde\"],\"std\":[]}}", "serde_1.0.216": "{\"dependencies\":[{\"name\":\"serde_derive\",\"req\":\"=1.0.216\",\"target\":\"cfg(any())\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"derive\":[\"serde_derive\"],\"rc\":[],\"std\":[],\"unstable\":[]}}", + "serde_1.0.228": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"result\"],\"name\":\"serde_core\",\"req\":\"=1.0.228\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[\"serde_core/alloc\"],\"default\":[\"std\"],\"derive\":[\"serde_derive\"],\"rc\":[\"serde_core/rc\"],\"std\":[\"serde_core/std\"],\"unstable\":[\"serde_core/unstable\"]}}", + "serde_core_1.0.228": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"name\":\"serde_derive\",\"req\":\"=1.0.228\",\"target\":\"cfg(any())\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"result\"],\"rc\":[],\"result\":[],\"std\":[],\"unstable\":[]}}", "serde_derive_1.0.216": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"derive\",\"parsing\",\"printing\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0.81\"}],\"features\":{\"default\":[],\"deserialize_in_place\":[]}}", + "serde_derive_1.0.228": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"derive\",\"parsing\",\"printing\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0.81\"}],\"features\":{\"default\":[],\"deserialize_in_place\":[]}}", "serde_json_1.0.134": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0.11\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.2.3\"},{\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2.0.2\"},{\"name\":\"itoa\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"name\":\"ryu\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"kind\":\"dev\",\"name\":\"serde_bytes\",\"req\":\"^0.11.10\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_stacker\",\"req\":\"^0.1.8\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{\"alloc\":[\"serde/alloc\"],\"arbitrary_precision\":[],\"default\":[\"std\"],\"float_roundtrip\":[],\"preserve_order\":[\"indexmap\",\"std\"],\"raw_value\":[],\"std\":[\"memchr/std\",\"serde/std\"],\"unbounded_depth\":[]}}", + "serde_json_1.0.149": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0.11\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.2.3\"},{\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2.0.2\"},{\"name\":\"itoa\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"kind\":\"dev\",\"name\":\"serde_bytes\",\"req\":\"^0.11.10\"},{\"default_features\":false,\"name\":\"serde_core\",\"req\":\"^1.0.220\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_stacker\",\"req\":\"^0.1.8\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"},{\"name\":\"zmij\",\"req\":\"^1.0\"}],\"features\":{\"alloc\":[\"serde_core/alloc\"],\"arbitrary_precision\":[],\"default\":[\"std\"],\"float_roundtrip\":[],\"preserve_order\":[\"indexmap\",\"std\"],\"raw_value\":[],\"std\":[\"memchr/std\",\"serde_core/std\"],\"unbounded_depth\":[]}}", "shlex_1.3.0": "{\"dependencies\":[],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "signal-hook-registry_1.4.8": "{\"dependencies\":[{\"name\":\"errno\",\"req\":\">=0.2, <0.4\"},{\"name\":\"libc\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"signal-hook\",\"req\":\"~0.3\"}],\"features\":{}}", + "slab_0.4.12": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.95\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", "smallvec_1.13.2": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.0.1\"},{\"kind\":\"dev\",\"name\":\"debugger_test\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"debugger_test_parser\",\"req\":\"^0.1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"const_generics\":[],\"const_new\":[\"const_generics\"],\"debugger_visualizer\":[],\"drain_filter\":[],\"drain_keep_rest\":[\"drain_filter\"],\"may_dangle\":[],\"specialization\":[],\"union\":[],\"write\":[]}}", + "smallvec_1.15.1": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"name\":\"bincode\",\"optional\":true,\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"bincode1\",\"package\":\"bincode\",\"req\":\"^1.0.1\"},{\"kind\":\"dev\",\"name\":\"debugger_test\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"debugger_test_parser\",\"req\":\"^0.1.0\"},{\"default_features\":false,\"name\":\"malloc_size_of\",\"optional\":true,\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"name\":\"unty\",\"optional\":true,\"req\":\"^0.0.4\"}],\"features\":{\"const_generics\":[],\"const_new\":[\"const_generics\"],\"debugger_visualizer\":[],\"drain_filter\":[],\"drain_keep_rest\":[\"drain_filter\"],\"impl_bincode\":[\"bincode\",\"unty\"],\"may_dangle\":[],\"specialization\":[],\"union\":[],\"write\":[]}}", + "socket2_0.5.10": "{\"dependencies\":[{\"name\":\"libc\",\"req\":\"^0.2.171\",\"target\":\"cfg(unix)\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\",\"Win32_System_IO\",\"Win32_System_Threading\",\"Win32_System_WindowsProgramming\"],\"name\":\"windows-sys\",\"req\":\"^0.52\",\"target\":\"cfg(windows)\"}],\"features\":{\"all\":[]}}", + "socket2_0.6.3": "{\"dependencies\":[{\"name\":\"libc\",\"req\":\"^0.2.172\",\"target\":\"cfg(any(unix, target_os = \\\"wasi\\\"))\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\",\"Win32_System_IO\",\"Win32_System_Threading\",\"Win32_System_WindowsProgramming\"],\"name\":\"windows-sys\",\"req\":\">=0.60, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"all\":[]}}", "spm_precompiled_0.1.4": "{\"dependencies\":[{\"name\":\"base64\",\"req\":\"^0.13\"},{\"name\":\"nom\",\"req\":\"^7.1.1\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"unicode-segmentation\",\"req\":\"^1.9\"}],\"features\":{}}", "static_assertions_1.1.0": "{\"dependencies\":[],\"features\":{\"nightly\":[]}}", "strsim_0.11.1": "{\"dependencies\":[],\"features\":{}}", + "syn_2.0.117": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"insta\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.91\"},{\"default_features\":false,\"name\":\"quote\",\"optional\":true,\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1\"},{\"features\":[\"blocking\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.13\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"syn-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4.16\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"termcolor\",\"req\":\"^1\"},{\"name\":\"unicode-ident\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\",\"target\":\"cfg(not(miri))\"}],\"features\":{\"clone-impls\":[],\"default\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\"],\"derive\":[],\"extra-traits\":[],\"fold\":[],\"full\":[],\"parsing\":[],\"printing\":[\"dep:quote\"],\"proc-macro\":[\"proc-macro2/proc-macro\",\"quote?/proc-macro\"],\"test\":[\"syn-test-suite/all-features\"],\"visit\":[],\"visit-mut\":[]}}", "syn_2.0.91": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"insta\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.91\"},{\"default_features\":false,\"name\":\"quote\",\"optional\":true,\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1\"},{\"features\":[\"blocking\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.12\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"syn-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4.16\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"termcolor\",\"req\":\"^1\"},{\"name\":\"unicode-ident\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\",\"target\":\"cfg(not(miri))\"}],\"features\":{\"clone-impls\":[],\"default\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\"],\"derive\":[],\"extra-traits\":[],\"fold\":[],\"full\":[],\"parsing\":[],\"printing\":[\"dep:quote\"],\"proc-macro\":[\"proc-macro2/proc-macro\",\"quote?/proc-macro\"],\"test\":[\"syn-test-suite/all-features\"],\"visit\":[],\"visit-mut\":[]}}", + "sync_wrapper_1.0.2": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"}],\"features\":{\"futures\":[\"futures-core\"]}}", + "tempfile_3.27.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"name\":\"fastrand\",\"req\":\"^2.1.1\"},{\"default_features\":false,\"name\":\"getrandom\",\"optional\":true,\"req\":\">=0.3.0, <0.5\",\"target\":\"cfg(any(unix, windows, target_os = \\\"wasi\\\"))\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"once_cell\",\"req\":\"^1.19.0\"},{\"features\":[\"fs\"],\"name\":\"rustix\",\"req\":\"^1.1.4\",\"target\":\"cfg(any(unix, target_os = \\\"wasi\\\"))\"},{\"features\":[\"Win32_Storage_FileSystem\",\"Win32_Foundation\"],\"name\":\"windows-sys\",\"req\":\">=0.52, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[\"getrandom\"],\"nightly\":[]}}", "thiserror-impl_2.0.12": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"name\":\"syn\",\"req\":\"^2.0.87\"}],\"features\":{}}", "thiserror_2.0.12": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.73\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"name\":\"thiserror-impl\",\"req\":\"=2.0.12\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", "tokenizers_0.22.2": "{\"dependencies\":[{\"features\":[\"serde\"],\"name\":\"ahash\",\"req\":\"^0.8.11\"},{\"name\":\"aho-corasick\",\"req\":\"^1.1\"},{\"kind\":\"dev\",\"name\":\"assert_approx_eq\",\"req\":\"^1.1\"},{\"features\":[\"serde\"],\"name\":\"compact_str\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.6\"},{\"features\":[\"serde\"],\"name\":\"dary_heap\",\"req\":\"^0.3.6\"},{\"name\":\"derive_builder\",\"req\":\"^0.20\"},{\"default_features\":false,\"name\":\"esaxx-rs\",\"req\":\"^0.1.10\"},{\"name\":\"fancy-regex\",\"optional\":true,\"req\":\"^0.14\"},{\"name\":\"getrandom\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"ureq\"],\"name\":\"hf-hub\",\"optional\":true,\"req\":\"^0.4.1\"},{\"name\":\"indicatif\",\"optional\":true,\"req\":\"^0.18\"},{\"name\":\"itertools\",\"req\":\"^0.14\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"macro_rules_attribute\",\"req\":\"^0.2.0\"},{\"name\":\"monostate\",\"req\":\"^0.1.12\"},{\"default_features\":false,\"name\":\"onig\",\"optional\":true,\"req\":\"^6.5.1\"},{\"name\":\"paste\",\"req\":\"^1.0.14\"},{\"name\":\"rand\",\"req\":\"^0.9\"},{\"name\":\"rayon\",\"req\":\"^1.10\"},{\"name\":\"rayon-cond\",\"req\":\"^0.4\"},{\"name\":\"regex\",\"req\":\"^1.10\"},{\"name\":\"regex-syntax\",\"req\":\"^0.8\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"spm_precompiled\",\"req\":\"^0.1.3\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.10\"},{\"name\":\"thiserror\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3.18\"},{\"name\":\"unicode-normalization-alignments\",\"req\":\"^0.1\"},{\"name\":\"unicode-segmentation\",\"req\":\"^1.11\"},{\"name\":\"unicode_categories\",\"req\":\"^0.1\"}],\"features\":{\"default\":[\"progressbar\",\"onig\",\"esaxx_fast\"],\"esaxx_fast\":[\"esaxx-rs/cpp\"],\"http\":[\"hf-hub\"],\"progressbar\":[\"indicatif\"],\"rustls-tls\":[\"hf-hub?/rustls-tls\"],\"unstable_wasm\":[\"fancy-regex\",\"getrandom/wasm_js\"]}}", + "tokio-macros_2.4.0": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0.0\"}],\"features\":{}}", + "tokio-stream_0.1.15": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-stream\",\"req\":\"^0.3\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"name\":\"futures-core\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"parking_lot\",\"req\":\"^0.12.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.11\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"req\":\"^1.15.0\"},{\"features\":[\"full\",\"test-util\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.2.0\"},{\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7.0\"}],\"features\":{\"default\":[\"time\"],\"fs\":[\"tokio/fs\"],\"full\":[\"time\",\"net\",\"io-util\",\"fs\",\"sync\",\"signal\"],\"io-util\":[\"tokio/io-util\"],\"net\":[\"tokio/net\"],\"signal\":[\"tokio/signal\"],\"sync\":[\"tokio/sync\",\"tokio-util\"],\"time\":[\"tokio/time\"]}}", + "tokio-util_0.7.17": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-stream\",\"req\":\"^0.3.0\"},{\"name\":\"bytes\",\"req\":\"^1.5.0\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.0\"},{\"name\":\"futures-core\",\"req\":\"^0.3.0\"},{\"name\":\"futures-io\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"futures-sink\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"futures-test\",\"req\":\"^0.3.5\"},{\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.0\"},{\"default_features\":false,\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.15.0\"},{\"kind\":\"dev\",\"name\":\"parking_lot\",\"req\":\"^0.12.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.11\"},{\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4.4\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.1.0\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"req\":\"^1.28.0\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.29\"}],\"features\":{\"__docs_rs\":[\"futures-util\"],\"codec\":[],\"compat\":[\"futures-io\"],\"default\":[],\"full\":[\"codec\",\"compat\",\"io-util\",\"time\",\"net\",\"rt\",\"join-map\"],\"io\":[],\"io-util\":[\"io\",\"tokio/rt\",\"tokio/io-util\"],\"join-map\":[\"rt\",\"hashbrown\"],\"net\":[\"tokio/net\"],\"rt\":[\"tokio/rt\",\"tokio/sync\",\"futures-util\"],\"time\":[\"tokio/time\",\"slab\"]}}", + "tokio_1.39.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-stream\",\"req\":\"^0.3\"},{\"name\":\"backtrace\",\"req\":\"^0.3.58\",\"target\":\"cfg(tokio_taskdump)\"},{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1.0.0\"},{\"features\":[\"async-await\"],\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.0\"},{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.149\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.149\",\"target\":\"cfg(unix)\"},{\"features\":[\"futures\",\"checkpoint\"],\"kind\":\"dev\",\"name\":\"loom\",\"req\":\"^0.7\",\"target\":\"cfg(loom)\"},{\"default_features\":false,\"name\":\"mio\",\"optional\":true,\"req\":\"^1.0.1\"},{\"features\":[\"tokio\"],\"kind\":\"dev\",\"name\":\"mio-aio\",\"req\":\"^0.9.0\",\"target\":\"cfg(target_os = \\\"freebsd\\\")\"},{\"kind\":\"dev\",\"name\":\"mockall\",\"req\":\"^0.11.1\"},{\"default_features\":false,\"features\":[\"aio\",\"fs\",\"socket\"],\"kind\":\"dev\",\"name\":\"nix\",\"req\":\"^0.29.0\",\"target\":\"cfg(unix)\"},{\"name\":\"parking_lot\",\"optional\":true,\"req\":\"^0.12.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.11\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.0\",\"target\":\"cfg(not(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\")))\"},{\"name\":\"signal-hook-registry\",\"optional\":true,\"req\":\"^1.1.1\",\"target\":\"cfg(unix)\"},{\"features\":[\"all\"],\"name\":\"socket2\",\"optional\":true,\"req\":\"^0.5.5\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"kind\":\"dev\",\"name\":\"socket2\",\"req\":\"^0.5.5\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.1.0\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"name\":\"tokio-macros\",\"optional\":true,\"req\":\"~2.4.0\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.25\",\"target\":\"cfg(tokio_unstable)\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.0\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", not(target_os = \\\"wasi\\\")))\"},{\"name\":\"windows-sys\",\"optional\":true,\"req\":\"^0.52\",\"target\":\"cfg(windows)\"},{\"features\":[\"Win32_Foundation\",\"Win32_Security_Authorization\"],\"kind\":\"dev\",\"name\":\"windows-sys\",\"req\":\"^0.52\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[],\"fs\":[],\"full\":[\"fs\",\"io-util\",\"io-std\",\"macros\",\"net\",\"parking_lot\",\"process\",\"rt\",\"rt-multi-thread\",\"signal\",\"sync\",\"time\"],\"io-std\":[],\"io-util\":[\"bytes\"],\"macros\":[\"tokio-macros\"],\"net\":[\"libc\",\"mio/os-poll\",\"mio/os-ext\",\"mio/net\",\"socket2\",\"windows-sys/Win32_Foundation\",\"windows-sys/Win32_Security\",\"windows-sys/Win32_Storage_FileSystem\",\"windows-sys/Win32_System_Pipes\",\"windows-sys/Win32_System_SystemServices\"],\"process\":[\"bytes\",\"libc\",\"mio/os-poll\",\"mio/os-ext\",\"mio/net\",\"signal-hook-registry\",\"windows-sys/Win32_Foundation\",\"windows-sys/Win32_System_Threading\",\"windows-sys/Win32_System_WindowsProgramming\"],\"rt\":[],\"rt-multi-thread\":[\"rt\"],\"signal\":[\"libc\",\"mio/os-poll\",\"mio/net\",\"mio/os-ext\",\"signal-hook-registry\",\"windows-sys/Win32_Foundation\",\"windows-sys/Win32_System_Console\"],\"sync\":[],\"test-util\":[\"rt\",\"sync\",\"time\"],\"time\":[]}}", + "tonic-build_0.12.3": "{\"dependencies\":[{\"name\":\"prettyplease\",\"req\":\"^0.2\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"name\":\"prost-build\",\"optional\":true,\"req\":\"^0.13\"},{\"name\":\"prost-types\",\"optional\":true,\"req\":\"^0.13\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"name\":\"syn\",\"req\":\"^2.0\"}],\"features\":{\"cleanup-markdown\":[\"prost\",\"prost-build/cleanup-markdown\"],\"default\":[\"transport\",\"prost\"],\"prost\":[\"prost-build\",\"dep:prost-types\"],\"transport\":[]}}", + "tonic_0.12.1": "{\"dependencies\":[{\"name\":\"async-stream\",\"optional\":true,\"req\":\"^0.3\"},{\"name\":\"async-trait\",\"optional\":true,\"req\":\"^0.1.13\"},{\"default_features\":false,\"name\":\"axum\",\"optional\":true,\"req\":\"^0.7\"},{\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.5\"},{\"name\":\"bytes\",\"req\":\"^1.0\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"h2\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"http-body\",\"req\":\"^1\"},{\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"features\":[\"http1\",\"http2\"],\"name\":\"hyper\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"hyper-timeout\",\"optional\":true,\"req\":\"^0.5\"},{\"features\":[\"tokio\"],\"name\":\"hyper-util\",\"optional\":true,\"req\":\"^0.1.4\"},{\"name\":\"percent-encoding\",\"req\":\"^2.1\"},{\"name\":\"pin-project\",\"req\":\"^1.0.11\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"prost\",\"optional\":true,\"req\":\"^0.13\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"name\":\"rustls-native-certs\",\"optional\":true,\"req\":\"^0.7\"},{\"name\":\"rustls-pemfile\",\"optional\":true,\"req\":\"^2.0\"},{\"features\":[\"all\"],\"name\":\"socket2\",\"optional\":true,\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"rt\",\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"logging\",\"tls12\",\"ring\"],\"name\":\"tokio-rustls\",\"optional\":true,\"req\":\"^0.26\"},{\"default_features\":false,\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"tower\",\"optional\":true,\"req\":\"^0.4.7\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tower\",\"req\":\"^0.4.7\"},{\"name\":\"tower-layer\",\"req\":\"^0.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"name\":\"tracing\",\"req\":\"^0.1\"},{\"name\":\"webpki-roots\",\"optional\":true,\"req\":\"^0.26\"},{\"name\":\"zstd\",\"optional\":true,\"req\":\"^0.13.0\"}],\"features\":{\"channel\":[\"dep:hyper\",\"hyper?/client\",\"dep:hyper-util\",\"hyper-util?/client-legacy\",\"dep:tower\",\"tower?/balance\",\"tower?/buffer\",\"tower?/discover\",\"tower?/limit\",\"dep:tokio\",\"tokio?/time\",\"dep:hyper-timeout\"],\"codegen\":[\"dep:async-trait\"],\"default\":[\"transport\",\"codegen\",\"prost\"],\"gzip\":[\"dep:flate2\"],\"prost\":[\"dep:prost\"],\"router\":[\"dep:axum\",\"dep:tower\",\"tower?/util\"],\"server\":[\"router\",\"dep:async-stream\",\"dep:h2\",\"dep:hyper\",\"hyper?/server\",\"dep:hyper-util\",\"hyper-util?/service\",\"hyper-util?/server-auto\",\"dep:socket2\",\"dep:tokio\",\"tokio?/macros\",\"tokio?/net\",\"tokio?/time\",\"tokio-stream/net\",\"dep:tower\",\"tower?/util\",\"tower?/limit\"],\"tls\":[\"dep:rustls-pemfile\",\"dep:tokio-rustls\",\"dep:tokio\",\"tokio?/rt\",\"tokio?/macros\"],\"tls-roots\":[\"tls\",\"channel\",\"dep:rustls-native-certs\"],\"tls-webpki-roots\":[\"tls\",\"channel\",\"dep:webpki-roots\"],\"transport\":[\"server\",\"channel\"],\"zstd\":[\"dep:zstd\"]}}", + "tower-layer_0.3.3": "{\"dependencies\":[],\"features\":{}}", + "tower-service_0.3.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.22\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^0.2\"},{\"features\":[\"macros\",\"time\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.6.2\"},{\"kind\":\"dev\",\"name\":\"tower-layer\",\"req\":\"^0.3\"}],\"features\":{}}", + "tower_0.4.13": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"hdrhistogram\",\"optional\":true,\"req\":\"^7.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"hdrhistogram\",\"req\":\"^7.0\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^0.2\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^1.0.2\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4.0\"},{\"name\":\"pin-project\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"pin-project-lite\",\"optional\":true,\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"},{\"features\":[\"small_rng\"],\"name\":\"rand\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.6\"},{\"features\":[\"macros\",\"sync\",\"test-util\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.6.2\"},{\"name\":\"tokio-stream\",\"optional\":true,\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7.0\"},{\"name\":\"tower-layer\",\"req\":\"^0.3.1\"},{\"name\":\"tower-service\",\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"tower-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.2\"},{\"default_features\":false,\"features\":[\"fmt\",\"ansi\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"}],\"features\":{\"__common\":[\"futures-core\",\"pin-project-lite\"],\"balance\":[\"discover\",\"load\",\"ready-cache\",\"make\",\"rand\",\"slab\"],\"buffer\":[\"__common\",\"tokio/sync\",\"tokio/rt\",\"tokio-util\",\"tracing\"],\"default\":[\"log\"],\"discover\":[\"__common\"],\"filter\":[\"__common\",\"futures-util\"],\"full\":[\"balance\",\"buffer\",\"discover\",\"filter\",\"hedge\",\"limit\",\"load\",\"load-shed\",\"make\",\"ready-cache\",\"reconnect\",\"retry\",\"spawn-ready\",\"steer\",\"timeout\",\"util\"],\"hedge\":[\"util\",\"filter\",\"futures-util\",\"hdrhistogram\",\"tokio/time\",\"tracing\"],\"limit\":[\"__common\",\"tokio/time\",\"tokio/sync\",\"tokio-util\",\"tracing\"],\"load\":[\"__common\",\"tokio/time\",\"tracing\"],\"load-shed\":[\"__common\"],\"log\":[\"tracing/log\"],\"make\":[\"futures-util\",\"pin-project-lite\",\"tokio/io-std\"],\"ready-cache\":[\"futures-core\",\"futures-util\",\"indexmap\",\"tokio/sync\",\"tracing\",\"pin-project-lite\"],\"reconnect\":[\"make\",\"tokio/io-std\",\"tracing\"],\"retry\":[\"__common\",\"tokio/time\"],\"spawn-ready\":[\"__common\",\"futures-util\",\"tokio/sync\",\"tokio/rt\",\"util\",\"tracing\"],\"steer\":[],\"timeout\":[\"pin-project-lite\",\"tokio/time\"],\"util\":[\"__common\",\"futures-util\",\"pin-project\"]}}", + "tower_0.5.3": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.22\"},{\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3.22\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.22\"},{\"default_features\":false,\"features\":[\"async-await-macro\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3.22\"},{\"default_features\":false,\"name\":\"hdrhistogram\",\"optional\":true,\"req\":\"^7.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"hdrhistogram\",\"req\":\"^7.0\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.0.2\"},{\"name\":\"pin-project-lite\",\"optional\":true,\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9\"},{\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4.9\"},{\"name\":\"sync_wrapper\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.6.2\"},{\"features\":[\"macros\",\"sync\",\"test-util\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.6.2\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1.1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7.0\"},{\"name\":\"tower-layer\",\"req\":\"^0.3.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3.3\"},{\"kind\":\"dev\",\"name\":\"tower-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.2\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1.2\"},{\"default_features\":false,\"features\":[\"fmt\",\"ansi\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"}],\"features\":{\"balance\":[\"discover\",\"load\",\"ready-cache\",\"make\",\"slab\",\"util\"],\"buffer\":[\"tokio/sync\",\"tokio/rt\",\"tokio-util\",\"tracing\",\"pin-project-lite\"],\"discover\":[\"futures-core\",\"pin-project-lite\"],\"filter\":[\"futures-util\",\"pin-project-lite\"],\"full\":[\"balance\",\"buffer\",\"discover\",\"filter\",\"hedge\",\"limit\",\"load\",\"load-shed\",\"make\",\"ready-cache\",\"reconnect\",\"retry\",\"spawn-ready\",\"steer\",\"timeout\",\"util\"],\"hedge\":[\"util\",\"filter\",\"futures-util\",\"hdrhistogram\",\"tokio/time\",\"tracing\"],\"limit\":[\"tokio/time\",\"tokio/sync\",\"tokio-util\",\"tracing\",\"pin-project-lite\"],\"load\":[\"tokio/time\",\"tracing\",\"pin-project-lite\"],\"load-shed\":[\"pin-project-lite\"],\"log\":[\"tracing/log\"],\"make\":[\"pin-project-lite\",\"tokio\"],\"ready-cache\":[\"futures-core\",\"futures-util\",\"indexmap\",\"tokio/sync\",\"tracing\",\"pin-project-lite\"],\"reconnect\":[\"make\",\"tracing\"],\"retry\":[\"tokio/time\",\"util\"],\"spawn-ready\":[\"futures-util\",\"tokio/sync\",\"tokio/rt\",\"util\",\"tracing\"],\"steer\":[],\"timeout\":[\"pin-project-lite\",\"tokio/time\"],\"tokio-stream\":[],\"util\":[\"futures-core\",\"futures-util\",\"pin-project-lite\",\"sync_wrapper\"]}}", + "tracing-attributes_0.1.31": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-trait\",\"req\":\"^0.1.67\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1.0.20\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.9\"},{\"default_features\":false,\"features\":[\"full\",\"parsing\",\"printing\",\"visit-mut\",\"clone-impls\",\"extra-traits\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4.2\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1.35\"},{\"features\":[\"env-filter\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.64\"}],\"features\":{\"async-await\":[]}}", + "tracing-core_0.1.36": "{\"dependencies\":[{\"name\":\"once_cell\",\"optional\":true,\"req\":\"^1.13.0\"},{\"default_features\":false,\"name\":\"valuable\",\"optional\":true,\"req\":\"^0.1.0\",\"target\":\"cfg(tracing_unstable)\"}],\"features\":{\"default\":[\"std\",\"valuable?/std\"],\"std\":[\"once_cell\"]}}", + "tracing_0.1.44": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.6\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.21\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4.17\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.9\"},{\"name\":\"tracing-attributes\",\"optional\":true,\"req\":\"^0.1.31\"},{\"default_features\":false,\"name\":\"tracing-core\",\"req\":\"^0.1.36\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.38\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"}],\"features\":{\"async-await\":[],\"attributes\":[\"tracing-attributes\"],\"default\":[\"std\",\"attributes\"],\"log-always\":[\"log\"],\"max_level_debug\":[],\"max_level_error\":[],\"max_level_info\":[],\"max_level_off\":[],\"max_level_trace\":[],\"max_level_warn\":[],\"release_max_level_debug\":[],\"release_max_level_error\":[],\"release_max_level_info\":[],\"release_max_level_off\":[],\"release_max_level_trace\":[],\"release_max_level_warn\":[],\"std\":[\"tracing-core/std\"],\"valuable\":[\"tracing-core/valuable\"]}}", + "try-lock_0.2.5": "{\"dependencies\":[],\"features\":{}}", "unicode-ident_1.0.14": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"fst\",\"req\":\"^0.4\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"roaring\",\"req\":\"^0.10\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"ucd-trie\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"unicode-xid\",\"req\":\"^0.2.6\"}],\"features\":{}}", + "unicode-ident_1.0.24": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"fst\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"roaring\",\"req\":\"^0.11\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"ucd-trie\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"unicode-xid\",\"req\":\"^0.2.6\"}],\"features\":{}}", "unicode-normalization-alignments_0.1.12": "{\"dependencies\":[{\"name\":\"smallvec\",\"req\":\"^1.0\"}],\"features\":{}}", "unicode-segmentation_1.12.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.7\"}],\"features\":{\"no_std\":[]}}", + "unicode-xid_0.2.6": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3\"}],\"features\":{\"bench\":[],\"default\":[],\"no_std\":[]}}", "unicode_categories_0.1.1": "{\"dependencies\":[],\"features\":{}}", "version_check_0.9.5": "{\"dependencies\":[],\"features\":{}}", + "want_0.3.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"tokio-executor\",\"req\":\"^0.2.0-alpha.2\"},{\"kind\":\"dev\",\"name\":\"tokio-sync\",\"req\":\"^0.2.0-alpha.2\"},{\"name\":\"try-lock\",\"req\":\"^0.2.4\"}],\"features\":{}}", + "wasi_0.11.1+wasi-snapshot-preview1": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"rustc-dep-of-std\":[\"core\",\"rustc-std-workspace-alloc\"],\"std\":[]}}", "wasi_0.14.2+wasi-0.2.4": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"bitflags\"],\"name\":\"wit-bindgen-rt\",\"req\":\"^0.39.0\"}],\"features\":{\"default\":[\"std\"],\"rustc-dep-of-std\":[\"compiler_builtins\",\"core\",\"rustc-std-workspace-alloc\"],\"std\":[]}}", + "wasip2_1.0.2+wasi-0.2.9": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"wit-bindgen\",\"req\":\"^0.51.0\"}],\"features\":{\"bitflags\":[\"wit-bindgen/bitflags\"],\"default\":[\"std\",\"bitflags\"],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"wit-bindgen/rustc-dep-of-std\"],\"std\":[]}}", + "wasip3_0.4.0+wasi-0.3.0-rc-2026-01-06": "{\"dependencies\":[{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1.10.1\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.31\"},{\"name\":\"http\",\"optional\":true,\"req\":\"^1.3.1\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^1.3.1\"},{\"name\":\"http-body\",\"optional\":true,\"req\":\"^1.0.1\"},{\"name\":\"thiserror\",\"optional\":true,\"req\":\"^2.0.17\"},{\"default_features\":false,\"features\":[\"async\"],\"name\":\"wit-bindgen\",\"req\":\"^0.51.0\"},{\"default_features\":false,\"features\":[\"async-spawn\"],\"kind\":\"dev\",\"name\":\"wit-bindgen\",\"req\":\"^0.51.0\"}],\"features\":{\"http-compat\":[\"dep:bytes\",\"dep:http-body\",\"dep:http\",\"dep:thiserror\",\"wit-bindgen/async-spawn\"]}}", + "wasm-encoder_0.244.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"default_features\":false,\"name\":\"leb128fmt\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.2.0\"},{\"default_features\":false,\"features\":[\"simd\",\"simd\"],\"name\":\"wasmparser\",\"optional\":true,\"req\":\"^0.244.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"wasmprinter\",\"req\":\"^0.244.0\"}],\"features\":{\"component-model\":[\"wasmparser?/component-model\"],\"default\":[\"std\",\"component-model\"],\"std\":[\"wasmparser?/std\"]}}", + "wasm-metadata_0.244.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"name\":\"auditable-serde\",\"optional\":true,\"req\":\"^0.8.0\"},{\"features\":[\"derive\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4.0.0\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1.1.0\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"indexmap\",\"req\":\"^2.7.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"spdx\",\"optional\":true,\"req\":\"^0.10.1\"},{\"name\":\"url\",\"optional\":true,\"req\":\"^2.0.0\"},{\"default_features\":false,\"features\":[\"std\",\"component-model\"],\"name\":\"wasm-encoder\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"simd\",\"std\",\"component-model\",\"hash-collections\"],\"name\":\"wasmparser\",\"req\":\"^0.244.0\"}],\"features\":{\"default\":[\"oci\",\"serde\"],\"oci\":[\"dep:auditable-serde\",\"dep:flate2\",\"dep:url\",\"dep:spdx\",\"dep:serde_json\",\"serde\"],\"serde\":[\"dep:serde_derive\",\"dep:serde\"]}}", + "wasmparser_0.244.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"name\":\"bitflags\",\"req\":\"^2.4.1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5.1\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"default-hasher\"],\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.15.2\"},{\"default_features\":false,\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.7.0\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.13.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.3\"},{\"default_features\":false,\"name\":\"semver\",\"optional\":true,\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.166\"}],\"features\":{\"component-model\":[\"dep:semver\"],\"default\":[\"std\",\"validate\",\"serde\",\"features\",\"component-model\",\"hash-collections\",\"simd\"],\"features\":[],\"hash-collections\":[\"dep:hashbrown\",\"dep:indexmap\"],\"prefer-btree-collections\":[],\"serde\":[\"dep:serde\",\"indexmap?/serde\",\"hashbrown?/serde\"],\"simd\":[],\"std\":[\"indexmap?/std\"],\"validate\":[]}}", + "windows-link_0.2.1": "{\"dependencies\":[],\"features\":{}}", + "windows-sys_0.52.0": "{\"dependencies\":[{\"name\":\"windows-targets\",\"req\":\"^0.52.0\"}],\"features\":{\"Wdk\":[],\"Wdk_Foundation\":[\"Wdk\"],\"Wdk_Graphics\":[\"Wdk\"],\"Wdk_Graphics_Direct3D\":[\"Wdk_Graphics\"],\"Wdk_Storage\":[\"Wdk\"],\"Wdk_Storage_FileSystem\":[\"Wdk_Storage\"],\"Wdk_Storage_FileSystem_Minifilters\":[\"Wdk_Storage_FileSystem\"],\"Wdk_System\":[\"Wdk\"],\"Wdk_System_IO\":[\"Wdk_System\"],\"Wdk_System_OfflineRegistry\":[\"Wdk_System\"],\"Wdk_System_Registry\":[\"Wdk_System\"],\"Wdk_System_SystemInformation\":[\"Wdk_System\"],\"Wdk_System_SystemServices\":[\"Wdk_System\"],\"Wdk_System_Threading\":[\"Wdk_System\"],\"Win32\":[],\"Win32_Data\":[\"Win32\"],\"Win32_Data_HtmlHelp\":[\"Win32_Data\"],\"Win32_Data_RightsManagement\":[\"Win32_Data\"],\"Win32_Devices\":[\"Win32\"],\"Win32_Devices_AllJoyn\":[\"Win32_Devices\"],\"Win32_Devices_BiometricFramework\":[\"Win32_Devices\"],\"Win32_Devices_Bluetooth\":[\"Win32_Devices\"],\"Win32_Devices_Communication\":[\"Win32_Devices\"],\"Win32_Devices_DeviceAndDriverInstallation\":[\"Win32_Devices\"],\"Win32_Devices_DeviceQuery\":[\"Win32_Devices\"],\"Win32_Devices_Display\":[\"Win32_Devices\"],\"Win32_Devices_Enumeration\":[\"Win32_Devices\"],\"Win32_Devices_Enumeration_Pnp\":[\"Win32_Devices_Enumeration\"],\"Win32_Devices_Fax\":[\"Win32_Devices\"],\"Win32_Devices_HumanInterfaceDevice\":[\"Win32_Devices\"],\"Win32_Devices_PortableDevices\":[\"Win32_Devices\"],\"Win32_Devices_Properties\":[\"Win32_Devices\"],\"Win32_Devices_Pwm\":[\"Win32_Devices\"],\"Win32_Devices_Sensors\":[\"Win32_Devices\"],\"Win32_Devices_SerialCommunication\":[\"Win32_Devices\"],\"Win32_Devices_Tapi\":[\"Win32_Devices\"],\"Win32_Devices_Usb\":[\"Win32_Devices\"],\"Win32_Devices_WebServicesOnDevices\":[\"Win32_Devices\"],\"Win32_Foundation\":[\"Win32\"],\"Win32_Gaming\":[\"Win32\"],\"Win32_Globalization\":[\"Win32\"],\"Win32_Graphics\":[\"Win32\"],\"Win32_Graphics_Dwm\":[\"Win32_Graphics\"],\"Win32_Graphics_Gdi\":[\"Win32_Graphics\"],\"Win32_Graphics_GdiPlus\":[\"Win32_Graphics\"],\"Win32_Graphics_Hlsl\":[\"Win32_Graphics\"],\"Win32_Graphics_OpenGL\":[\"Win32_Graphics\"],\"Win32_Graphics_Printing\":[\"Win32_Graphics\"],\"Win32_Graphics_Printing_PrintTicket\":[\"Win32_Graphics_Printing\"],\"Win32_Management\":[\"Win32\"],\"Win32_Management_MobileDeviceManagementRegistration\":[\"Win32_Management\"],\"Win32_Media\":[\"Win32\"],\"Win32_Media_Audio\":[\"Win32_Media\"],\"Win32_Media_DxMediaObjects\":[\"Win32_Media\"],\"Win32_Media_KernelStreaming\":[\"Win32_Media\"],\"Win32_Media_Multimedia\":[\"Win32_Media\"],\"Win32_Media_Streaming\":[\"Win32_Media\"],\"Win32_Media_WindowsMediaFormat\":[\"Win32_Media\"],\"Win32_NetworkManagement\":[\"Win32\"],\"Win32_NetworkManagement_Dhcp\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Dns\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_InternetConnectionWizard\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_IpHelper\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Multicast\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Ndis\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetBios\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetManagement\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetShell\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetworkDiagnosticsFramework\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_P2P\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_QoS\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Rras\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Snmp\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WNet\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WebDav\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WiFi\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsConnectionManager\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsFilteringPlatform\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsFirewall\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsNetworkVirtualization\":[\"Win32_NetworkManagement\"],\"Win32_Networking\":[\"Win32\"],\"Win32_Networking_ActiveDirectory\":[\"Win32_Networking\"],\"Win32_Networking_Clustering\":[\"Win32_Networking\"],\"Win32_Networking_HttpServer\":[\"Win32_Networking\"],\"Win32_Networking_Ldap\":[\"Win32_Networking\"],\"Win32_Networking_WebSocket\":[\"Win32_Networking\"],\"Win32_Networking_WinHttp\":[\"Win32_Networking\"],\"Win32_Networking_WinInet\":[\"Win32_Networking\"],\"Win32_Networking_WinSock\":[\"Win32_Networking\"],\"Win32_Networking_WindowsWebServices\":[\"Win32_Networking\"],\"Win32_Security\":[\"Win32\"],\"Win32_Security_AppLocker\":[\"Win32_Security\"],\"Win32_Security_Authentication\":[\"Win32_Security\"],\"Win32_Security_Authentication_Identity\":[\"Win32_Security_Authentication\"],\"Win32_Security_Authorization\":[\"Win32_Security\"],\"Win32_Security_Credentials\":[\"Win32_Security\"],\"Win32_Security_Cryptography\":[\"Win32_Security\"],\"Win32_Security_Cryptography_Catalog\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_Certificates\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_Sip\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_UI\":[\"Win32_Security_Cryptography\"],\"Win32_Security_DiagnosticDataQuery\":[\"Win32_Security\"],\"Win32_Security_DirectoryServices\":[\"Win32_Security\"],\"Win32_Security_EnterpriseData\":[\"Win32_Security\"],\"Win32_Security_ExtensibleAuthenticationProtocol\":[\"Win32_Security\"],\"Win32_Security_Isolation\":[\"Win32_Security\"],\"Win32_Security_LicenseProtection\":[\"Win32_Security\"],\"Win32_Security_NetworkAccessProtection\":[\"Win32_Security\"],\"Win32_Security_WinTrust\":[\"Win32_Security\"],\"Win32_Security_WinWlx\":[\"Win32_Security\"],\"Win32_Storage\":[\"Win32\"],\"Win32_Storage_Cabinets\":[\"Win32_Storage\"],\"Win32_Storage_CloudFilters\":[\"Win32_Storage\"],\"Win32_Storage_Compression\":[\"Win32_Storage\"],\"Win32_Storage_DistributedFileSystem\":[\"Win32_Storage\"],\"Win32_Storage_FileHistory\":[\"Win32_Storage\"],\"Win32_Storage_FileSystem\":[\"Win32_Storage\"],\"Win32_Storage_Imapi\":[\"Win32_Storage\"],\"Win32_Storage_IndexServer\":[\"Win32_Storage\"],\"Win32_Storage_InstallableFileSystems\":[\"Win32_Storage\"],\"Win32_Storage_IscsiDisc\":[\"Win32_Storage\"],\"Win32_Storage_Jet\":[\"Win32_Storage\"],\"Win32_Storage_Nvme\":[\"Win32_Storage\"],\"Win32_Storage_OfflineFiles\":[\"Win32_Storage\"],\"Win32_Storage_OperationRecorder\":[\"Win32_Storage\"],\"Win32_Storage_Packaging\":[\"Win32_Storage\"],\"Win32_Storage_Packaging_Appx\":[\"Win32_Storage_Packaging\"],\"Win32_Storage_ProjectedFileSystem\":[\"Win32_Storage\"],\"Win32_Storage_StructuredStorage\":[\"Win32_Storage\"],\"Win32_Storage_Vhd\":[\"Win32_Storage\"],\"Win32_Storage_Xps\":[\"Win32_Storage\"],\"Win32_System\":[\"Win32\"],\"Win32_System_AddressBook\":[\"Win32_System\"],\"Win32_System_Antimalware\":[\"Win32_System\"],\"Win32_System_ApplicationInstallationAndServicing\":[\"Win32_System\"],\"Win32_System_ApplicationVerifier\":[\"Win32_System\"],\"Win32_System_ClrHosting\":[\"Win32_System\"],\"Win32_System_Com\":[\"Win32_System\"],\"Win32_System_Com_Marshal\":[\"Win32_System_Com\"],\"Win32_System_Com_StructuredStorage\":[\"Win32_System_Com\"],\"Win32_System_Com_Urlmon\":[\"Win32_System_Com\"],\"Win32_System_ComponentServices\":[\"Win32_System\"],\"Win32_System_Console\":[\"Win32_System\"],\"Win32_System_CorrelationVector\":[\"Win32_System\"],\"Win32_System_DataExchange\":[\"Win32_System\"],\"Win32_System_DeploymentServices\":[\"Win32_System\"],\"Win32_System_DeveloperLicensing\":[\"Win32_System\"],\"Win32_System_Diagnostics\":[\"Win32_System\"],\"Win32_System_Diagnostics_Ceip\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_Debug\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_Debug_Extensions\":[\"Win32_System_Diagnostics_Debug\"],\"Win32_System_Diagnostics_Etw\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_ProcessSnapshotting\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_ToolHelp\":[\"Win32_System_Diagnostics\"],\"Win32_System_DistributedTransactionCoordinator\":[\"Win32_System\"],\"Win32_System_Environment\":[\"Win32_System\"],\"Win32_System_ErrorReporting\":[\"Win32_System\"],\"Win32_System_EventCollector\":[\"Win32_System\"],\"Win32_System_EventLog\":[\"Win32_System\"],\"Win32_System_EventNotificationService\":[\"Win32_System\"],\"Win32_System_GroupPolicy\":[\"Win32_System\"],\"Win32_System_HostCompute\":[\"Win32_System\"],\"Win32_System_HostComputeNetwork\":[\"Win32_System\"],\"Win32_System_HostComputeSystem\":[\"Win32_System\"],\"Win32_System_Hypervisor\":[\"Win32_System\"],\"Win32_System_IO\":[\"Win32_System\"],\"Win32_System_Iis\":[\"Win32_System\"],\"Win32_System_Ioctl\":[\"Win32_System\"],\"Win32_System_JobObjects\":[\"Win32_System\"],\"Win32_System_Js\":[\"Win32_System\"],\"Win32_System_Kernel\":[\"Win32_System\"],\"Win32_System_LibraryLoader\":[\"Win32_System\"],\"Win32_System_Mailslots\":[\"Win32_System\"],\"Win32_System_Mapi\":[\"Win32_System\"],\"Win32_System_Memory\":[\"Win32_System\"],\"Win32_System_Memory_NonVolatile\":[\"Win32_System_Memory\"],\"Win32_System_MessageQueuing\":[\"Win32_System\"],\"Win32_System_MixedReality\":[\"Win32_System\"],\"Win32_System_Ole\":[\"Win32_System\"],\"Win32_System_PasswordManagement\":[\"Win32_System\"],\"Win32_System_Performance\":[\"Win32_System\"],\"Win32_System_Performance_HardwareCounterProfiling\":[\"Win32_System_Performance\"],\"Win32_System_Pipes\":[\"Win32_System\"],\"Win32_System_Power\":[\"Win32_System\"],\"Win32_System_ProcessStatus\":[\"Win32_System\"],\"Win32_System_Recovery\":[\"Win32_System\"],\"Win32_System_Registry\":[\"Win32_System\"],\"Win32_System_RemoteDesktop\":[\"Win32_System\"],\"Win32_System_RemoteManagement\":[\"Win32_System\"],\"Win32_System_RestartManager\":[\"Win32_System\"],\"Win32_System_Restore\":[\"Win32_System\"],\"Win32_System_Rpc\":[\"Win32_System\"],\"Win32_System_Search\":[\"Win32_System\"],\"Win32_System_Search_Common\":[\"Win32_System_Search\"],\"Win32_System_SecurityCenter\":[\"Win32_System\"],\"Win32_System_Services\":[\"Win32_System\"],\"Win32_System_SetupAndMigration\":[\"Win32_System\"],\"Win32_System_Shutdown\":[\"Win32_System\"],\"Win32_System_StationsAndDesktops\":[\"Win32_System\"],\"Win32_System_SubsystemForLinux\":[\"Win32_System\"],\"Win32_System_SystemInformation\":[\"Win32_System\"],\"Win32_System_SystemServices\":[\"Win32_System\"],\"Win32_System_Threading\":[\"Win32_System\"],\"Win32_System_Time\":[\"Win32_System\"],\"Win32_System_TpmBaseServices\":[\"Win32_System\"],\"Win32_System_UserAccessLogging\":[\"Win32_System\"],\"Win32_System_Variant\":[\"Win32_System\"],\"Win32_System_VirtualDosMachines\":[\"Win32_System\"],\"Win32_System_WindowsProgramming\":[\"Win32_System\"],\"Win32_System_Wmi\":[\"Win32_System\"],\"Win32_UI\":[\"Win32\"],\"Win32_UI_Accessibility\":[\"Win32_UI\"],\"Win32_UI_ColorSystem\":[\"Win32_UI\"],\"Win32_UI_Controls\":[\"Win32_UI\"],\"Win32_UI_Controls_Dialogs\":[\"Win32_UI_Controls\"],\"Win32_UI_HiDpi\":[\"Win32_UI\"],\"Win32_UI_Input\":[\"Win32_UI\"],\"Win32_UI_Input_Ime\":[\"Win32_UI_Input\"],\"Win32_UI_Input_KeyboardAndMouse\":[\"Win32_UI_Input\"],\"Win32_UI_Input_Pointer\":[\"Win32_UI_Input\"],\"Win32_UI_Input_Touch\":[\"Win32_UI_Input\"],\"Win32_UI_Input_XboxController\":[\"Win32_UI_Input\"],\"Win32_UI_InteractionContext\":[\"Win32_UI\"],\"Win32_UI_Magnification\":[\"Win32_UI\"],\"Win32_UI_Shell\":[\"Win32_UI\"],\"Win32_UI_Shell_PropertiesSystem\":[\"Win32_UI_Shell\"],\"Win32_UI_TabletPC\":[\"Win32_UI\"],\"Win32_UI_TextServices\":[\"Win32_UI\"],\"Win32_UI_WindowsAndMessaging\":[\"Win32_UI\"],\"Win32_Web\":[\"Win32\"],\"Win32_Web_InternetExplorer\":[\"Win32_Web\"],\"default\":[],\"docs\":[]}}", + "windows-sys_0.61.2": "{\"dependencies\":[{\"default_features\":false,\"name\":\"windows-link\",\"req\":\"^0.2.1\"}],\"features\":{\"Wdk\":[\"Win32_Foundation\"],\"Wdk_Devices\":[\"Wdk\"],\"Wdk_Devices_Bluetooth\":[\"Wdk_Devices\"],\"Wdk_Devices_HumanInterfaceDevice\":[\"Wdk_Devices\"],\"Wdk_Foundation\":[\"Wdk\"],\"Wdk_Graphics\":[\"Wdk\"],\"Wdk_Graphics_Direct3D\":[\"Wdk_Graphics\"],\"Wdk_NetworkManagement\":[\"Wdk\"],\"Wdk_NetworkManagement_Ndis\":[\"Wdk_NetworkManagement\"],\"Wdk_NetworkManagement_WindowsFilteringPlatform\":[\"Wdk_NetworkManagement\"],\"Wdk_Storage\":[\"Wdk\"],\"Wdk_Storage_FileSystem\":[\"Wdk_Storage\"],\"Wdk_Storage_FileSystem_Minifilters\":[\"Wdk_Storage_FileSystem\"],\"Wdk_System\":[\"Wdk\"],\"Wdk_System_IO\":[\"Wdk_System\"],\"Wdk_System_Memory\":[\"Wdk_System\"],\"Wdk_System_OfflineRegistry\":[\"Wdk_System\"],\"Wdk_System_Registry\":[\"Wdk_System\"],\"Wdk_System_SystemInformation\":[\"Wdk_System\"],\"Wdk_System_SystemServices\":[\"Wdk_System\"],\"Wdk_System_Threading\":[\"Wdk_System\"],\"Win32\":[\"Win32_Foundation\"],\"Win32_Data\":[\"Win32\"],\"Win32_Data_HtmlHelp\":[\"Win32_Data\"],\"Win32_Data_RightsManagement\":[\"Win32_Data\"],\"Win32_Devices\":[\"Win32\"],\"Win32_Devices_AllJoyn\":[\"Win32_Devices\"],\"Win32_Devices_Beep\":[\"Win32_Devices\"],\"Win32_Devices_BiometricFramework\":[\"Win32_Devices\"],\"Win32_Devices_Bluetooth\":[\"Win32_Devices\"],\"Win32_Devices_Cdrom\":[\"Win32_Devices\"],\"Win32_Devices_Communication\":[\"Win32_Devices\"],\"Win32_Devices_DeviceAndDriverInstallation\":[\"Win32_Devices\"],\"Win32_Devices_DeviceQuery\":[\"Win32_Devices\"],\"Win32_Devices_Display\":[\"Win32_Devices\"],\"Win32_Devices_Dvd\":[\"Win32_Devices\"],\"Win32_Devices_Enumeration\":[\"Win32_Devices\"],\"Win32_Devices_Enumeration_Pnp\":[\"Win32_Devices_Enumeration\"],\"Win32_Devices_Fax\":[\"Win32_Devices\"],\"Win32_Devices_HumanInterfaceDevice\":[\"Win32_Devices\"],\"Win32_Devices_Nfc\":[\"Win32_Devices\"],\"Win32_Devices_Nfp\":[\"Win32_Devices\"],\"Win32_Devices_PortableDevices\":[\"Win32_Devices\"],\"Win32_Devices_Properties\":[\"Win32_Devices\"],\"Win32_Devices_Pwm\":[\"Win32_Devices\"],\"Win32_Devices_Sensors\":[\"Win32_Devices\"],\"Win32_Devices_SerialCommunication\":[\"Win32_Devices\"],\"Win32_Devices_Tapi\":[\"Win32_Devices\"],\"Win32_Devices_Usb\":[\"Win32_Devices\"],\"Win32_Devices_WebServicesOnDevices\":[\"Win32_Devices\"],\"Win32_Foundation\":[\"Win32\"],\"Win32_Gaming\":[\"Win32\"],\"Win32_Globalization\":[\"Win32\"],\"Win32_Graphics\":[\"Win32\"],\"Win32_Graphics_Dwm\":[\"Win32_Graphics\"],\"Win32_Graphics_Gdi\":[\"Win32_Graphics\"],\"Win32_Graphics_GdiPlus\":[\"Win32_Graphics\"],\"Win32_Graphics_Hlsl\":[\"Win32_Graphics\"],\"Win32_Graphics_OpenGL\":[\"Win32_Graphics\"],\"Win32_Graphics_Printing\":[\"Win32_Graphics\"],\"Win32_Graphics_Printing_PrintTicket\":[\"Win32_Graphics_Printing\"],\"Win32_Management\":[\"Win32\"],\"Win32_Management_MobileDeviceManagementRegistration\":[\"Win32_Management\"],\"Win32_Media\":[\"Win32\"],\"Win32_Media_Audio\":[\"Win32_Media\"],\"Win32_Media_DxMediaObjects\":[\"Win32_Media\"],\"Win32_Media_KernelStreaming\":[\"Win32_Media\"],\"Win32_Media_Multimedia\":[\"Win32_Media\"],\"Win32_Media_Streaming\":[\"Win32_Media\"],\"Win32_Media_WindowsMediaFormat\":[\"Win32_Media\"],\"Win32_NetworkManagement\":[\"Win32\"],\"Win32_NetworkManagement_Dhcp\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Dns\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_InternetConnectionWizard\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_IpHelper\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Multicast\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Ndis\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetBios\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetManagement\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetShell\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetworkDiagnosticsFramework\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_P2P\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_QoS\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Rras\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Snmp\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WNet\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WebDav\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WiFi\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsConnectionManager\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsFilteringPlatform\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsFirewall\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsNetworkVirtualization\":[\"Win32_NetworkManagement\"],\"Win32_Networking\":[\"Win32\"],\"Win32_Networking_ActiveDirectory\":[\"Win32_Networking\"],\"Win32_Networking_Clustering\":[\"Win32_Networking\"],\"Win32_Networking_HttpServer\":[\"Win32_Networking\"],\"Win32_Networking_Ldap\":[\"Win32_Networking\"],\"Win32_Networking_WebSocket\":[\"Win32_Networking\"],\"Win32_Networking_WinHttp\":[\"Win32_Networking\"],\"Win32_Networking_WinInet\":[\"Win32_Networking\"],\"Win32_Networking_WinSock\":[\"Win32_Networking\"],\"Win32_Networking_WindowsWebServices\":[\"Win32_Networking\"],\"Win32_Security\":[\"Win32\"],\"Win32_Security_AppLocker\":[\"Win32_Security\"],\"Win32_Security_Authentication\":[\"Win32_Security\"],\"Win32_Security_Authentication_Identity\":[\"Win32_Security_Authentication\"],\"Win32_Security_Authorization\":[\"Win32_Security\"],\"Win32_Security_Credentials\":[\"Win32_Security\"],\"Win32_Security_Cryptography\":[\"Win32_Security\"],\"Win32_Security_Cryptography_Catalog\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_Certificates\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_Sip\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_UI\":[\"Win32_Security_Cryptography\"],\"Win32_Security_DiagnosticDataQuery\":[\"Win32_Security\"],\"Win32_Security_DirectoryServices\":[\"Win32_Security\"],\"Win32_Security_EnterpriseData\":[\"Win32_Security\"],\"Win32_Security_ExtensibleAuthenticationProtocol\":[\"Win32_Security\"],\"Win32_Security_Isolation\":[\"Win32_Security\"],\"Win32_Security_LicenseProtection\":[\"Win32_Security\"],\"Win32_Security_NetworkAccessProtection\":[\"Win32_Security\"],\"Win32_Security_WinTrust\":[\"Win32_Security\"],\"Win32_Security_WinWlx\":[\"Win32_Security\"],\"Win32_Storage\":[\"Win32\"],\"Win32_Storage_Cabinets\":[\"Win32_Storage\"],\"Win32_Storage_CloudFilters\":[\"Win32_Storage\"],\"Win32_Storage_Compression\":[\"Win32_Storage\"],\"Win32_Storage_DistributedFileSystem\":[\"Win32_Storage\"],\"Win32_Storage_FileHistory\":[\"Win32_Storage\"],\"Win32_Storage_FileSystem\":[\"Win32_Storage\"],\"Win32_Storage_Imapi\":[\"Win32_Storage\"],\"Win32_Storage_IndexServer\":[\"Win32_Storage\"],\"Win32_Storage_InstallableFileSystems\":[\"Win32_Storage\"],\"Win32_Storage_IscsiDisc\":[\"Win32_Storage\"],\"Win32_Storage_Jet\":[\"Win32_Storage\"],\"Win32_Storage_Nvme\":[\"Win32_Storage\"],\"Win32_Storage_OfflineFiles\":[\"Win32_Storage\"],\"Win32_Storage_OperationRecorder\":[\"Win32_Storage\"],\"Win32_Storage_Packaging\":[\"Win32_Storage\"],\"Win32_Storage_Packaging_Appx\":[\"Win32_Storage_Packaging\"],\"Win32_Storage_ProjectedFileSystem\":[\"Win32_Storage\"],\"Win32_Storage_StructuredStorage\":[\"Win32_Storage\"],\"Win32_Storage_Vhd\":[\"Win32_Storage\"],\"Win32_Storage_Xps\":[\"Win32_Storage\"],\"Win32_System\":[\"Win32\"],\"Win32_System_AddressBook\":[\"Win32_System\"],\"Win32_System_Antimalware\":[\"Win32_System\"],\"Win32_System_ApplicationInstallationAndServicing\":[\"Win32_System\"],\"Win32_System_ApplicationVerifier\":[\"Win32_System\"],\"Win32_System_ClrHosting\":[\"Win32_System\"],\"Win32_System_Com\":[\"Win32_System\"],\"Win32_System_Com_Marshal\":[\"Win32_System_Com\"],\"Win32_System_Com_StructuredStorage\":[\"Win32_System_Com\"],\"Win32_System_Com_Urlmon\":[\"Win32_System_Com\"],\"Win32_System_ComponentServices\":[\"Win32_System\"],\"Win32_System_Console\":[\"Win32_System\"],\"Win32_System_CorrelationVector\":[\"Win32_System\"],\"Win32_System_DataExchange\":[\"Win32_System\"],\"Win32_System_DeploymentServices\":[\"Win32_System\"],\"Win32_System_DeveloperLicensing\":[\"Win32_System\"],\"Win32_System_Diagnostics\":[\"Win32_System\"],\"Win32_System_Diagnostics_Ceip\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_Debug\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_Debug_Extensions\":[\"Win32_System_Diagnostics_Debug\"],\"Win32_System_Diagnostics_Etw\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_ProcessSnapshotting\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_ToolHelp\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_TraceLogging\":[\"Win32_System_Diagnostics\"],\"Win32_System_DistributedTransactionCoordinator\":[\"Win32_System\"],\"Win32_System_Environment\":[\"Win32_System\"],\"Win32_System_ErrorReporting\":[\"Win32_System\"],\"Win32_System_EventCollector\":[\"Win32_System\"],\"Win32_System_EventLog\":[\"Win32_System\"],\"Win32_System_EventNotificationService\":[\"Win32_System\"],\"Win32_System_GroupPolicy\":[\"Win32_System\"],\"Win32_System_HostCompute\":[\"Win32_System\"],\"Win32_System_HostComputeNetwork\":[\"Win32_System\"],\"Win32_System_HostComputeSystem\":[\"Win32_System\"],\"Win32_System_Hypervisor\":[\"Win32_System\"],\"Win32_System_IO\":[\"Win32_System\"],\"Win32_System_Iis\":[\"Win32_System\"],\"Win32_System_Ioctl\":[\"Win32_System\"],\"Win32_System_JobObjects\":[\"Win32_System\"],\"Win32_System_Js\":[\"Win32_System\"],\"Win32_System_Kernel\":[\"Win32_System\"],\"Win32_System_LibraryLoader\":[\"Win32_System\"],\"Win32_System_Mailslots\":[\"Win32_System\"],\"Win32_System_Mapi\":[\"Win32_System\"],\"Win32_System_Memory\":[\"Win32_System\"],\"Win32_System_Memory_NonVolatile\":[\"Win32_System_Memory\"],\"Win32_System_MessageQueuing\":[\"Win32_System\"],\"Win32_System_MixedReality\":[\"Win32_System\"],\"Win32_System_Ole\":[\"Win32_System\"],\"Win32_System_PasswordManagement\":[\"Win32_System\"],\"Win32_System_Performance\":[\"Win32_System\"],\"Win32_System_Performance_HardwareCounterProfiling\":[\"Win32_System_Performance\"],\"Win32_System_Pipes\":[\"Win32_System\"],\"Win32_System_Power\":[\"Win32_System\"],\"Win32_System_ProcessStatus\":[\"Win32_System\"],\"Win32_System_Recovery\":[\"Win32_System\"],\"Win32_System_Registry\":[\"Win32_System\"],\"Win32_System_RemoteDesktop\":[\"Win32_System\"],\"Win32_System_RemoteManagement\":[\"Win32_System\"],\"Win32_System_RestartManager\":[\"Win32_System\"],\"Win32_System_Restore\":[\"Win32_System\"],\"Win32_System_Rpc\":[\"Win32_System\"],\"Win32_System_Search\":[\"Win32_System\"],\"Win32_System_Search_Common\":[\"Win32_System_Search\"],\"Win32_System_SecurityCenter\":[\"Win32_System\"],\"Win32_System_Services\":[\"Win32_System\"],\"Win32_System_SetupAndMigration\":[\"Win32_System\"],\"Win32_System_Shutdown\":[\"Win32_System\"],\"Win32_System_StationsAndDesktops\":[\"Win32_System\"],\"Win32_System_SubsystemForLinux\":[\"Win32_System\"],\"Win32_System_SystemInformation\":[\"Win32_System\"],\"Win32_System_SystemServices\":[\"Win32_System\"],\"Win32_System_Threading\":[\"Win32_System\"],\"Win32_System_Time\":[\"Win32_System\"],\"Win32_System_TpmBaseServices\":[\"Win32_System\"],\"Win32_System_UserAccessLogging\":[\"Win32_System\"],\"Win32_System_Variant\":[\"Win32_System\"],\"Win32_System_VirtualDosMachines\":[\"Win32_System\"],\"Win32_System_WindowsProgramming\":[\"Win32_System\"],\"Win32_System_Wmi\":[\"Win32_System\"],\"Win32_UI\":[\"Win32\"],\"Win32_UI_Accessibility\":[\"Win32_UI\"],\"Win32_UI_ColorSystem\":[\"Win32_UI\"],\"Win32_UI_Controls\":[\"Win32_UI\"],\"Win32_UI_Controls_Dialogs\":[\"Win32_UI_Controls\"],\"Win32_UI_HiDpi\":[\"Win32_UI\"],\"Win32_UI_Input\":[\"Win32_UI\"],\"Win32_UI_Input_Ime\":[\"Win32_UI_Input\"],\"Win32_UI_Input_KeyboardAndMouse\":[\"Win32_UI_Input\"],\"Win32_UI_Input_Pointer\":[\"Win32_UI_Input\"],\"Win32_UI_Input_Touch\":[\"Win32_UI_Input\"],\"Win32_UI_Input_XboxController\":[\"Win32_UI_Input\"],\"Win32_UI_InteractionContext\":[\"Win32_UI\"],\"Win32_UI_Magnification\":[\"Win32_UI\"],\"Win32_UI_Shell\":[\"Win32_UI\"],\"Win32_UI_Shell_Common\":[\"Win32_UI_Shell\"],\"Win32_UI_Shell_PropertiesSystem\":[\"Win32_UI_Shell\"],\"Win32_UI_TabletPC\":[\"Win32_UI\"],\"Win32_UI_TextServices\":[\"Win32_UI\"],\"Win32_UI_WindowsAndMessaging\":[\"Win32_UI\"],\"Win32_Web\":[\"Win32\"],\"Win32_Web_InternetExplorer\":[\"Win32_Web\"],\"default\":[],\"docs\":[]}}", + "windows-targets_0.52.6": "{\"dependencies\":[{\"name\":\"windows_aarch64_gnullvm\",\"req\":\"^0.52.6\",\"target\":\"aarch64-pc-windows-gnullvm\"},{\"name\":\"windows_aarch64_msvc\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(target_arch = \\\"aarch64\\\", target_env = \\\"msvc\\\", not(windows_raw_dylib)))\"},{\"name\":\"windows_i686_gnu\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(target_arch = \\\"x86\\\", target_env = \\\"gnu\\\", not(target_abi = \\\"llvm\\\"), not(windows_raw_dylib)))\"},{\"name\":\"windows_i686_gnullvm\",\"req\":\"^0.52.6\",\"target\":\"i686-pc-windows-gnullvm\"},{\"name\":\"windows_i686_msvc\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(target_arch = \\\"x86\\\", target_env = \\\"msvc\\\", not(windows_raw_dylib)))\"},{\"name\":\"windows_x86_64_gnu\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(target_arch = \\\"x86_64\\\", target_env = \\\"gnu\\\", not(target_abi = \\\"llvm\\\"), not(windows_raw_dylib)))\"},{\"name\":\"windows_x86_64_gnullvm\",\"req\":\"^0.52.6\",\"target\":\"x86_64-pc-windows-gnullvm\"},{\"name\":\"windows_x86_64_msvc\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(any(target_arch = \\\"x86_64\\\", target_arch = \\\"arm64ec\\\"), target_env = \\\"msvc\\\", not(windows_raw_dylib)))\"}],\"features\":{}}", + "windows_aarch64_gnullvm_0.52.6": "{\"dependencies\":[],\"features\":{}}", + "windows_aarch64_msvc_0.52.6": "{\"dependencies\":[],\"features\":{}}", + "windows_i686_gnu_0.52.6": "{\"dependencies\":[],\"features\":{}}", + "windows_i686_gnullvm_0.52.6": "{\"dependencies\":[],\"features\":{}}", + "windows_i686_msvc_0.52.6": "{\"dependencies\":[],\"features\":{}}", + "windows_x86_64_gnu_0.52.6": "{\"dependencies\":[],\"features\":{}}", + "windows_x86_64_gnullvm_0.52.6": "{\"dependencies\":[],\"features\":{}}", + "windows_x86_64_msvc_0.52.6": "{\"dependencies\":[],\"features\":{}}", + "wit-bindgen-core_0.51.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.72\"},{\"features\":[\"derive\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4.3.19\"},{\"name\":\"heck\",\"req\":\"^0.5\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.218\"},{\"name\":\"wit-parser\",\"req\":\"^0.244.0\"}],\"features\":{\"clap\":[\"dep:clap\"],\"serde\":[\"dep:serde\"]}}", "wit-bindgen-rt_0.39.0": "{\"dependencies\":[{\"name\":\"bitflags\",\"optional\":true,\"req\":\"^2.3.3\"},{\"name\":\"futures\",\"optional\":true,\"req\":\"^0.3.30\"},{\"name\":\"once_cell\",\"optional\":true,\"req\":\"^1.19.0\"}],\"features\":{\"async\":[\"dep:futures\",\"dep:once_cell\"]}}", + "wit-bindgen-rust-macro_0.51.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.72\"},{\"name\":\"prettyplease\",\"req\":\"^0.2.20\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"printing\"],\"name\":\"syn\",\"req\":\"^2.0.89\"},{\"name\":\"wit-bindgen-core\",\"req\":\"^0.51.0\"},{\"name\":\"wit-bindgen-rust\",\"req\":\"^0.51.0\"}],\"features\":{\"async\":[]}}", + "wit-bindgen-rust_0.51.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.72\"},{\"kind\":\"dev\",\"name\":\"bytes\",\"req\":\"^1\"},{\"features\":[\"derive\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4.3.19\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.31\"},{\"name\":\"heck\",\"req\":\"^0.5\"},{\"name\":\"indexmap\",\"req\":\"^2.0.0\"},{\"name\":\"prettyplease\",\"req\":\"^0.2.20\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.218\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"features\":[\"printing\"],\"name\":\"syn\",\"req\":\"^2.0.89\"},{\"default_features\":false,\"name\":\"wasm-metadata\",\"req\":\"^0.244.0\"},{\"name\":\"wit-bindgen-core\",\"req\":\"^0.51.0\"},{\"name\":\"wit-component\",\"req\":\"^0.244.0\"}],\"features\":{\"clap\":[\"dep:clap\",\"wit-bindgen-core/clap\"],\"serde\":[\"dep:serde\",\"wit-bindgen-core/serde\"]}}", + "wit-bindgen_0.51.0": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0\"},{\"name\":\"bitflags\",\"optional\":true,\"req\":\"^2.3.3\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"futures\",\"optional\":true,\"req\":\"^0.3.30\"},{\"name\":\"wit-bindgen-rust-macro\",\"optional\":true,\"req\":\"^0.51.0\"}],\"features\":{\"async\":[\"std\",\"wit-bindgen-rust-macro?/async\"],\"async-spawn\":[\"async\",\"dep:futures\"],\"bitflags\":[\"dep:bitflags\"],\"default\":[\"macros\",\"realloc\",\"async\",\"std\",\"bitflags\"],\"inter-task-wakeup\":[\"async\"],\"macros\":[\"dep:wit-bindgen-rust-macro\"],\"realloc\":[],\"rustc-dep-of-std\":[\"dep:core\",\"dep:alloc\"],\"std\":[]}}", + "wit-component_0.244.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"name\":\"bitflags\",\"req\":\"^2.3.3\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"kind\":\"dev\",\"name\":\"glob\",\"req\":\"^0.3.0\"},{\"default_features\":false,\"name\":\"indexmap\",\"req\":\"^2.7.0\"},{\"kind\":\"dev\",\"name\":\"libtest-mimic\",\"req\":\"^0.8.1\"},{\"name\":\"log\",\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.3.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"req\":\"^1.0.166\"},{\"name\":\"serde_derive\",\"req\":\"^1.0.166\"},{\"name\":\"serde_json\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"std\",\"wasmparser\"],\"name\":\"wasm-encoder\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"name\":\"wasm-metadata\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"oci\"],\"kind\":\"dev\",\"name\":\"wasm-metadata\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"simd\",\"std\",\"component-model\",\"simd\"],\"name\":\"wasmparser\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"simd\",\"std\",\"component-model\",\"features\"],\"kind\":\"dev\",\"name\":\"wasmparser\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"component-model\"],\"kind\":\"dev\",\"name\":\"wasmprinter\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"cranelift\",\"component-model\",\"runtime\",\"gc-drc\"],\"kind\":\"dev\",\"name\":\"wasmtime\",\"req\":\"^34.0.1\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"default_features\":false,\"name\":\"wast\",\"optional\":true,\"req\":\"^244.0.0\"},{\"default_features\":false,\"name\":\"wat\",\"optional\":true,\"req\":\"^1.244.0\"},{\"default_features\":false,\"features\":[\"component-model\"],\"kind\":\"dev\",\"name\":\"wat\",\"req\":\"^1.244.0\"},{\"features\":[\"decoding\",\"serde\"],\"name\":\"wit-parser\",\"req\":\"^0.244.0\"}],\"features\":{\"dummy-module\":[\"dep:wat\"],\"semver-check\":[\"dummy-module\"],\"wat\":[\"dep:wast\",\"dep:wat\"]}}", + "wit-parser_0.244.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"id-arena\",\"req\":\"^2\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"indexmap\",\"req\":\"^2.7.0\"},{\"kind\":\"dev\",\"name\":\"libtest-mimic\",\"req\":\"^0.8.1\"},{\"name\":\"log\",\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.3.0\"},{\"default_features\":false,\"name\":\"semver\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"name\":\"unicode-xid\",\"req\":\"^0.2.2\"},{\"default_features\":false,\"features\":[\"simd\",\"std\",\"validate\",\"component-model\",\"features\"],\"name\":\"wasmparser\",\"optional\":true,\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"component-model\"],\"name\":\"wat\",\"optional\":true,\"req\":\"^1.244.0\"}],\"features\":{\"decoding\":[\"dep:wasmparser\"],\"default\":[\"serde\",\"decoding\"],\"serde\":[\"dep:serde\",\"dep:serde_derive\",\"indexmap/serde\",\"serde_json\"],\"wat\":[\"decoding\",\"dep:wat\"]}}", "zerocopy-derive_0.7.35": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.10\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"name\":\"syn\",\"req\":\"^2.0.31\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.85\"}],\"features\":{}}", "zerocopy-derive_0.8.26": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"dissimilar\",\"req\":\"^1.0.9\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"=0.2.163\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"=1.9\"},{\"kind\":\"dev\",\"name\":\"prettyplease\",\"req\":\"=0.2.17\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.10\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.89\"}],\"features\":{}}", + "zerocopy-derive_0.8.42": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"dissimilar\",\"req\":\"^1.0.9\"},{\"kind\":\"dev\",\"name\":\"prettyplease\",\"req\":\"=0.2.17\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.40\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"features\":[\"visit\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.46\"}],\"features\":{}}", "zerocopy_0.7.35": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"assert_matches\",\"req\":\"^1.5\"},{\"default_features\":false,\"name\":\"byteorder\",\"optional\":true,\"req\":\"^1.3\"},{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.85\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.7.35\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.7.35\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.7.35\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\"],\"alloc\":[],\"default\":[\"byteorder\"],\"derive\":[\"zerocopy-derive\"],\"simd\":[],\"simd-nightly\":[\"simd\"]}}", - "zerocopy_0.8.26": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"either\",\"req\":\"=1.13.0\"},{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.89\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.8.26\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\",\"std\"],\"alloc\":[],\"derive\":[\"zerocopy-derive\"],\"float-nightly\":[],\"simd\":[],\"simd-nightly\":[\"simd\"],\"std\":[\"alloc\"]}}" + "zerocopy_0.8.26": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"either\",\"req\":\"=1.13.0\"},{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.89\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.8.26\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\",\"std\"],\"alloc\":[],\"derive\":[\"zerocopy-derive\"],\"float-nightly\":[],\"simd\":[],\"simd-nightly\":[\"simd\"],\"std\":[\"alloc\"]}}", + "zerocopy_0.8.42": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.8.42\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.8.42\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.8.42\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\",\"std\"],\"alloc\":[],\"derive\":[\"zerocopy-derive\"],\"float-nightly\":[],\"simd\":[],\"simd-nightly\":[\"simd\"],\"std\":[\"alloc\"]}}", + "zmij_1.0.21": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\",\"target\":\"cfg(not(miri))\"},{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1.36\"},{\"kind\":\"dev\",\"name\":\"num-bigint\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"num-integer\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"num_cpus\",\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"opt-level\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"ryu\",\"req\":\"^1\"}],\"features\":{}}" }, "@@rules_rs+//rs/experimental/toolchains:module_extension.bzl%toolchains": { "cargo-1.94.0-aarch64-apple-darwin.tar.xz": "7497ab873993cf38b806548c85bf818ffd71d1c3cf77b5bb4c3d3f1769588a08", diff --git a/bazel/bazel.zig b/bazel/bazel.zig index 97ab63b2..614d2fa7 100644 --- a/bazel/bazel.zig +++ b/bazel/bazel.zig @@ -14,8 +14,8 @@ const init_array_section = switch (builtin.object_format) { export const _ linksection(init_array_section) = &struct { fn call(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { - _ = argc; // autofix - _ = envp; // autofix + _ = argc; + _ = envp; var threaded: std.Io.Threaded = .init_single_threaded; runfiles_global = bazel_runfiles.Runfiles.create(.{ .allocator = std.heap.c_allocator, diff --git a/examples/io/BUILD.bazel b/examples/io/BUILD.bazel index 7d854ac7..64051f18 100644 --- a/examples/io/BUILD.bazel +++ b/examples/io/BUILD.bazel @@ -8,8 +8,3 @@ zig_binary( main = "main.zig", deps = ["//zml"], ) - -build_test( - name = "test", - targets = [":io"], -) diff --git a/examples/lfm/discussion.zig b/examples/lfm/discussion.zig index 8a5852e1..ae657805 100644 --- a/examples/lfm/discussion.zig +++ b/examples/lfm/discussion.zig @@ -27,6 +27,7 @@ pub const Context = struct { generated_token_slice: zml.Slice, token_pos: u32, all_tokens: std.ArrayList(u32), + sharding: zml.sharding.Sharding, pub fn init( allocator: std.mem.Allocator, @@ -39,6 +40,7 @@ pub const Context = struct { tokenizer: zml.tokenizer.Tokenizer, config: model.Config, seqlen: u32, + sharding: zml.sharding.Sharding, ) !Context { const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); return .{ @@ -50,12 +52,13 @@ pub const Context = struct { .tokenizer = tokenizer, .config = config, .seqlen = seqlen, - .cache_buffers = try cache.initBuffers(allocator, io, platform), - .attention_metadata_buffers = try attention_metadata.initBuffer(io, platform), - .rng_buf = try zml.Tensor.Rng.initBuffer(platform, seed, io), + .cache_buffers = try cache.initBuffers(allocator, io, platform, sharding), + .attention_metadata_buffers = try attention_metadata.initBuffer(io, platform, sharding), + .rng_buf = try zml.Tensor.Rng.initBuffer(platform, seed, io, sharding), .generated_token_slice = try .alloc(allocator, zml.Shape.init(.{ .batch = 1, .seq = 1 }, .u32)), .token_pos = 0, .all_tokens = try .initCapacity(allocator, seqlen), + .sharding = sharding, }; } @@ -164,16 +167,16 @@ pub const Context = struct { @memset(tokens, self.config.pad_token_id); @memcpy(tokens[0..self.all_tokens.items.len], self.all_tokens.items); - var tokens_buf: zml.Buffer = try .fromSlice(self.io, self.platform, tokens_slice); + var tokens_buf: zml.Buffer = try .fromSlice(self.io, self.platform, tokens_slice, self.sharding); defer tokens_buf.deinit(); // Always prefill from position 0 with all accumulated tokens. const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{0})); - var tokens_pos_buf: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice); + var tokens_pos_buf: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice, self.sharding); defer tokens_pos_buf.deinit(); const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(self.all_tokens.items.len)})); - var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice); + var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice, self.sharding); defer actual_seq_len_buf.deinit(); try self.exe.prefill.run(.{ @@ -187,6 +190,7 @@ pub const Context = struct { .rng_buf = &self.rng_buf, .cache_buffers = &self.cache_buffers, .attention_metadata_buffers = self.attention_metadata_buffers, + .sharding = self.sharding, }); try tokens_buf.toSlice(self.io, tokens_slice); @@ -198,11 +202,11 @@ pub const Context = struct { var tokenizer_decoder = try self.tokenizer.decoder(); defer tokenizer_decoder.deinit(); - var current_token_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, self.generated_token_slice); + var current_token_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, self.generated_token_slice, self.sharding); defer current_token_buffer.deinit(); const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{0})); - var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice); + var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice, self.sharding); defer actual_seq_len_buf.deinit(); const think_start = self.tokenizer.tokenToId("") orelse unreachable; @@ -228,7 +232,7 @@ pub const Context = struct { if (self.remainingTokens() == 0) break :generation; const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{self.token_pos})); - var token_pos_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice); + var token_pos_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice, self.sharding); defer token_pos_buffer.deinit(); try self.exe.decode.run(.{ @@ -242,6 +246,7 @@ pub const Context = struct { .rng_buf = &self.rng_buf, .cache_buffers = &self.cache_buffers, .attention_metadata_buffers = self.attention_metadata_buffers, + .sharding = self.sharding, }); try current_token_buffer.toSlice(self.io, self.generated_token_slice); diff --git a/examples/lfm/inference.zig b/examples/lfm/inference.zig index 01479c03..917e80ef 100644 --- a/examples/lfm/inference.zig +++ b/examples/lfm/inference.zig @@ -41,7 +41,7 @@ pub const CompilationOptions = struct { .batch_dim = 1, .rng = .init(), .cache = cache, - .attention_metadata = .init(.fromBackend(backend, seqlen)), + .attention_metadata = .init(.fromBackend(backend, seqlen, config.num_attention_heads)), .attention_parameters = .init(.fromBackend(backend)), }; } @@ -58,6 +58,7 @@ pub const Args = struct { rng_buf: *zml.Bufferized(zml.Tensor.Rng), cache_buffers: *zml.Bufferized(model.Cache), attention_metadata_buffers: zml.Bufferized(attention.Metadata), + sharding: zml.sharding.Sharding, }; pub const Inference = struct { @@ -72,18 +73,19 @@ pub const Inference = struct { opts: CompilationOptions, seqlen: u32, progress: *std.Progress.Node, + sharding: zml.sharding.Sharding, ) !Inference { if (opts.single) { - const prefill_exe = try compileSingleKernelExe(allocator, io, platform, mdl, opts, seqlen, true, progress); - const decode_exe = try compileSingleKernelExe(allocator, io, platform, mdl, opts, 1, false, progress); + const prefill_exe = try compileSingleKernelExe(allocator, io, platform, mdl, opts, seqlen, true, progress, sharding); + const decode_exe = try compileSingleKernelExe(allocator, io, platform, mdl, opts, 1, false, progress, sharding); return .{ .prefill = .{ .single = prefill_exe }, .decode = .{ .single = decode_exe }, }; } else { - const prefill_exe = try compileComposedKernelExe(allocator, io, platform, mdl, opts, seqlen, true, progress); - const decode_exe = try compileComposedKernelExe(allocator, io, platform, mdl, opts, 1, false, progress); + const prefill_exe = try compileComposedKernelExe(allocator, io, platform, mdl, opts, seqlen, true, progress, sharding); + const decode_exe = try compileComposedKernelExe(allocator, io, platform, mdl, opts, 1, false, progress, sharding); return .{ .prefill = .{ .composed = prefill_exe }, .decode = .{ .composed = decode_exe }, @@ -124,8 +126,6 @@ pub const SingleKernelExe = struct { } pub fn run(self: *const SingleKernelExe, args: Args) !void { - _ = args.io; // autofix - _ = args.platform; // autofix var exe_args = try self.exe.args(args.allocator); defer exe_args.deinit(args.allocator); @@ -154,7 +154,7 @@ pub const SingleKernelExe = struct { } }; -fn compileSingleKernelExe(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, mdl: model.Model, opts: CompilationOptions, seqlen: u32, is_prefill: bool, progress: *std.Progress.Node) !SingleKernelExe { +fn compileSingleKernelExe(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, mdl: model.Model, opts: CompilationOptions, seqlen: u32, is_prefill: bool, progress: *std.Progress.Node, sharding: zml.sharding.Sharding) !SingleKernelExe { progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling single kernel...", 1); defer node.end(); @@ -165,7 +165,14 @@ fn compileSingleKernelExe(allocator: std.mem.Allocator, io: std.Io, platform: *z const tokens: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32); const token_position_offset: zml.Tensor = .init(.{ .batch = opts.batch_dim }, .u32); - const exe = try platform.compile(allocator, io, mdl, .forward, .{ tokens, token_position_offset, actual_seq_len, opts.rng, opts.cache, opts.attention_metadata, opts.attention_parameters, model.ConvParameters{ .is_prefill = is_prefill } }); + const exe = try platform.compile( + allocator, + io, + mdl, + .forward, + .{ tokens, token_position_offset, actual_seq_len, opts.rng, opts.cache, opts.attention_metadata, opts.attention_parameters, model.ConvParameters{ .is_prefill = is_prefill } }, + .{ .shardings = &.{sharding} }, + ); return .{ .exe = exe }; } @@ -203,9 +210,9 @@ pub const ComposedKernelExe = struct { defer hidden_buf.deinit(); // Step 2: iterate over layers, dispatching to conv_layer or attn_layer exe. - var conv_cache_index_buf: zml.Buffer = try .scalar(args.io, args.platform, @as(u32, 0), .u32); + var conv_cache_index_buf: zml.Buffer = try .scalar(args.io, args.platform, @as(u32, 0), .u32, args.sharding); defer conv_cache_index_buf.deinit(); - var kv_cache_index_buf: zml.Buffer = try .scalar(args.io, args.platform, @as(u32, 0), .u32); + var kv_cache_index_buf: zml.Buffer = try .scalar(args.io, args.platform, @as(u32, 0), .u32, args.sharding); defer kv_cache_index_buf.deinit(); for (args.model_buffers.layers) |layer_bufs| { @@ -281,11 +288,12 @@ pub fn compileComposedKernelExe( seqlen: u32, is_prefill: bool, progress: *std.Progress.Node, + sharding: zml.sharding.Sharding, ) !ComposedKernelExe { - const embed_tokens_exe = try compileEmbedTokens(allocator, io, platform, mdl.embed_tokens, opts, seqlen, progress); - const conv_layer_exe = try compileConvLayer(allocator, io, platform, mdl, opts, seqlen, is_prefill, progress); - const attn_layer_exe = try compileAttnLayer(allocator, io, platform, mdl, opts, seqlen, is_prefill, progress); - const lm_head_exe = try compileLmHead(allocator, io, platform, mdl, opts, seqlen, progress); + const embed_tokens_exe = try compileEmbedTokens(allocator, io, platform, mdl.embed_tokens, opts, seqlen, progress, sharding); + const conv_layer_exe = try compileConvLayer(allocator, io, platform, mdl, opts, seqlen, is_prefill, progress, sharding); + const attn_layer_exe = try compileAttnLayer(allocator, io, platform, mdl, opts, seqlen, is_prefill, progress, sharding); + const lm_head_exe = try compileLmHead(allocator, io, platform, mdl, opts, seqlen, progress, sharding); return .{ .embed_tokens = embed_tokens_exe, @@ -295,7 +303,7 @@ pub fn compileComposedKernelExe( }; } -fn compileEmbedTokens(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, embed_tokens: model.TokenEmbedding, opts: CompilationOptions, seqlen: u32, progress: *std.Progress.Node) !zml.Exe { +fn compileEmbedTokens(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, embed_tokens: model.TokenEmbedding, opts: CompilationOptions, seqlen: u32, progress: *std.Progress.Node, sharding: zml.sharding.Sharding) !zml.Exe { progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling embed_tokens...", 1); defer node.end(); @@ -303,10 +311,10 @@ fn compileEmbedTokens(allocator: std.mem.Allocator, io: std.Io, platform: *zml.P defer log.info("Compiled embed_tokens [{f}]", .{now.untilNow(io, .awake)}); const tokens: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32); - return platform.compile(allocator, io, embed_tokens, .forward, .{tokens}); + return platform.compile(allocator, io, embed_tokens, .forward, .{tokens}, .{ .shardings = &.{sharding} }); } -fn compileConvLayer(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, mdl: model.Model, opts: CompilationOptions, seqlen: u32, is_prefill: bool, progress: *std.Progress.Node) !zml.Exe { +fn compileConvLayer(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, mdl: model.Model, opts: CompilationOptions, seqlen: u32, is_prefill: bool, progress: *std.Progress.Node, sharding: zml.sharding.Sharding) !zml.Exe { progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling conv layer...", 1); defer node.end(); @@ -334,10 +342,10 @@ fn compileConvLayer(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Pla opts.attention_metadata, opts.attention_parameters, model.ConvParameters{ .is_prefill = is_prefill }, - }); + }, .{ .shardings = &.{sharding} }); } -fn compileAttnLayer(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, mdl: model.Model, opts: CompilationOptions, seqlen: u32, is_prefill: bool, progress: *std.Progress.Node) !zml.Exe { +fn compileAttnLayer(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, mdl: model.Model, opts: CompilationOptions, seqlen: u32, is_prefill: bool, progress: *std.Progress.Node, sharding: zml.sharding.Sharding) !zml.Exe { progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling attn layer...", 1); defer node.end(); @@ -365,10 +373,10 @@ fn compileAttnLayer(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Pla opts.attention_metadata, opts.attention_parameters, model.ConvParameters{ .is_prefill = is_prefill }, - }); + }, .{ .shardings = &.{sharding} }); } -fn compileLmHead(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, mdl: model.Model, opts: CompilationOptions, seqlen: u32, progress: *std.Progress.Node) !zml.Exe { +fn compileLmHead(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, mdl: model.Model, opts: CompilationOptions, seqlen: u32, progress: *std.Progress.Node, sharding: zml.sharding.Sharding) !zml.Exe { progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling lm_head...", 1); defer node.end(); @@ -378,5 +386,5 @@ fn compileLmHead(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platfo const hidden: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()); const tokens: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32); - return platform.compile(allocator, io, mdl.lm_head, .forward, .{ hidden, mdl.embed_tokens, tokens, opts.rng }); + return platform.compile(allocator, io, mdl.lm_head, .forward, .{ hidden, mdl.embed_tokens, tokens, opts.rng }, .{ .shardings = &.{sharding} }); } diff --git a/examples/lfm/main.zig b/examples/lfm/main.zig index dc555500..c34144d6 100644 --- a/examples/lfm/main.zig +++ b/examples/lfm/main.zig @@ -87,7 +87,8 @@ pub fn main(init: std.process.Init) !void { const params: inference.CompilationOptions = .init(mdl, config, args.seqlen, attention_backend, args.single); - var compilation_fut = try io.concurrent(inference.Inference.init, .{ allocator, io, platform, mdl, params, args.seqlen, &progress }); + const replicated_sharding = try zml.sharding.replicatedSharding(platform); + var compilation_fut = try io.concurrent(inference.Inference.init, .{ allocator, io, platform, mdl, params, args.seqlen, &progress, replicated_sharding }); defer if (compilation_fut.cancel(io)) |exe| { exe.deinit(); } else |_| {}; @@ -132,7 +133,7 @@ pub fn main(init: std.process.Init) !void { }; defer allocator.free(prompt); - var ctx = try discussion.Context.init(allocator, io, platform, &model_buffers, params.cache, params.attention_metadata, exe, tokenizer, config, args.seqlen); + var ctx = try discussion.Context.init(allocator, io, platform, &model_buffers, params.cache, params.attention_metadata, exe, tokenizer, config, args.seqlen, replicated_sharding); defer ctx.deinit(); if (interactive) { diff --git a/examples/lfm/model.zig b/examples/lfm/model.zig index 9eb61912..417fd62d 100644 --- a/examples/lfm/model.zig +++ b/examples/lfm/model.zig @@ -67,6 +67,7 @@ pub const Model = struct { }); } + const sharding = try zml.sharding.replicatedSharding(platform); return zml.io.load(Model, self, allocator, io, platform, .{ .dma_chunks = 8, .dma_chunk_size = 128 * zml.MiB, @@ -74,6 +75,7 @@ pub const Model = struct { .store = store, .parallelism = 16, .total_bytes = &total_bytes, + .shardings = &.{sharding}, }); } @@ -123,7 +125,7 @@ pub const TokenEmbedding = struct { weight: Tensor, pub fn init(store: zml.io.TensorStore.View) TokenEmbedding { - return .{ .weight = store.createTensor("weight").withTags(.{ .voc, .d }) }; + return .{ .weight = store.createTensor("weight", .{ .voc, .d }, null) }; } pub fn forward(self: TokenEmbedding, tokens: Tensor) Tensor { @@ -267,7 +269,7 @@ pub const ShortConv = struct { pub fn init(config: Config, store: zml.io.TensorStore.View) ShortConv { stdx.debug.assert(!config.conv_bias, "conv_bias is not supported.", .{}); - return ShortConv{ .in_proj = initLinear(store.withPrefix("in_proj"), .d), .out_proj = initLinear(store.withPrefix("out_proj"), .d), .kernel = store.createTensorWithTags("conv.weight", .{ .out, .in, .kernel_size }), .config = config }; + return ShortConv{ .in_proj = initLinear(store.withPrefix("in_proj"), .d), .out_proj = initLinear(store.withPrefix("out_proj"), .d), .kernel = store.createTensor("conv.weight", .{ .out, .in, .kernel_size }, null), .config = config }; } pub fn forward( @@ -372,8 +374,7 @@ pub const Attention = struct { .num_key_value_groups = num_key_value_groups, .rope_opts = .{ .layout = .sequential, - .freq_base = config.rope_theta, - .scaling = .{ .default = .{} }, + .scaling = .{ .default = .{ .rope_theta = config.rope_theta } }, }, }; } @@ -448,10 +449,10 @@ pub const Cache = struct { conv: ConvCache, kv: KvCache, - pub fn initBuffers(self: Cache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Cache) { + pub fn initBuffers(self: Cache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !zml.Bufferized(Cache) { return .{ - .conv = try self.conv.initBuffers(allocator, io, platform), - .kv = try self.kv.initBuffers(io, platform), + .conv = try self.conv.initBuffers(allocator, io, platform, sharding), + .kv = try self.kv.initBuffers(io, platform, sharding), }; } @@ -477,13 +478,13 @@ pub const ConvCache = struct { }; } - pub fn initBuffers(self: ConvCache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(ConvCache) { + pub fn initBuffers(self: ConvCache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !zml.Bufferized(ConvCache) { const sh = self.state.shape(); const host = try allocator.alloc(u8, sh.byteSize()); defer allocator.free(host); @memset(host, 0); - return .{ .state = try zml.Buffer.fromBytes(io, platform, sh, host) }; + return .{ .state = try zml.Buffer.fromBytes(io, platform, sh, sharding, host) }; } pub fn unloadBuffers(self: *zml.Bufferized(ConvCache)) void { @@ -508,10 +509,10 @@ pub const KvCache = struct { }; } - pub fn initBuffers(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { + pub fn initBuffers(self: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !zml.Bufferized(KvCache) { return .{ - .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), - .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}), + .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), sharding, .{}), + .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), sharding, .{}), }; } @@ -555,7 +556,7 @@ pub const KvCache = struct { }; fn initLinear(store: zml.io.TensorStore.View, tag: anytype) Linear { - return .init(store.createTensorWithTags("weight", .{ .out, tag }), null, tag); + return .init(store.createTensor("weight", .{ .out, tag }, null), null, tag); } pub const Linear = struct { @@ -619,7 +620,7 @@ const RmsNorm = struct { tag: Shape.Tag, pub fn init(store: zml.io.TensorStore.View, eps: f32, tag: anytype) RmsNorm { - return .{ .weight = store.createTensorWithTags("weight", .{tag}), .eps = eps, .tag = zml.Shape.toTag(tag) }; + return .{ .weight = store.createTensor("weight", .{tag}, null), .eps = eps, .tag = zml.Shape.toTag(tag) }; } pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { diff --git a/examples/lfm/test_runner.zig b/examples/lfm/test_runner.zig index 0df68635..db508033 100644 --- a/examples/lfm/test_runner.zig +++ b/examples/lfm/test_runner.zig @@ -14,7 +14,17 @@ pub fn run(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, co var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); defer activation_store.deinit(); - var ctx = TestContext{ .allocator = allocator, .io = io, .platform = platform, .activations_store = &activation_store, .attention_metadata = attention_metadata, .attention_parameters = attention_parameters }; + const sharding = try zml.sharding.replicatedSharding(platform); + + var ctx = TestContext{ + .allocator = allocator, + .io = io, + .platform = platform, + .activations_store = &activation_store, + .attention_metadata = attention_metadata, + .attention_parameters = attention_parameters, + .sharding = sharding, + }; try ctx.testLayer("embed_tokens", .{ .batch, .seq }, mdl.embed_tokens, model_buffers.embed_tokens, .{}); @@ -61,6 +71,7 @@ const TestContext = struct { activations_store: *zml.io.TensorStore, attention_metadata: attention.Metadata, attention_parameters: attention.Parameters, + sharding: zml.sharding.Sharding, fn testLayerPrint( self: *TestContext, @@ -81,14 +92,14 @@ const TestContext = struct { const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); defer self.allocator.free(in_key); - const in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); + const in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key, self.sharding); const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(tagz); const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); defer self.allocator.free(out_key); - const out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); + const out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key, self.sharding); - const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor }); + const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor }, .{ .shardings = &.{self.sharding} }); defer exe.deinit(); var args = try exe.args(self.allocator); @@ -113,39 +124,39 @@ const TestContext = struct { const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); defer self.allocator.free(in_key); - const in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); + const in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key, self.sharding); const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(.{ .batch, .seq, .d }); const cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache", .{name}); defer self.allocator.free(cache_key); - const cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_key); + const cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_key, self.sharding); const cache_tensor = zml.Tensor.fromShape(cache_buffer.shape()).withTags(.{ .layer, .batch, .seq, .d }); const cache_pos_key = try std.fmt.allocPrint(self.allocator, "{s}.cache_position", .{name}); defer self.allocator.free(cache_pos_key); - const cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key); + const cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key, self.sharding); const cache_pos_tensor = zml.Tensor.fromShape(cache_pos_buffer.shape()).withTags(.{.batch}); const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); defer self.allocator.free(out_key); - const out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); + const out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key, self.sharding); const actual_seq_len_tensor: zml.Tensor = .init(.{}, .u32); const actual_seq_len: u32 = @intCast(in_tensor.dim(.seq)); const cache_index_tensor: zml.Tensor = .init(.{}, .u32); - const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, actual_seq_len_tensor, model.ConvCache{ .state = cache_tensor }, cache_index_tensor, model.ConvParameters{ .is_prefill = false } }); + const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, actual_seq_len_tensor, model.ConvCache{ .state = cache_tensor }, cache_index_tensor, model.ConvParameters{ .is_prefill = false } }, .{ .shardings = &.{self.sharding} }); defer exe.deinit(); var args = try exe.args(self.allocator); defer args.deinit(self.allocator); const conv_cache: zml.Bufferized(model.ConvCache) = .{ .state = cache_buffer }; const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{actual_seq_len})); - var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice); + var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice, self.sharding); defer actual_seq_len_buf.deinit(); const cache_index_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(cache_ix)})); - var cache_index_buf: zml.Buffer = try .fromSlice(self.io, self.platform, cache_index_slice); + var cache_index_buf: zml.Buffer = try .fromSlice(self.io, self.platform, cache_index_slice, self.sharding); defer cache_index_buf.deinit(); args.set(.{ layer_buffers, in_buffer, cache_pos_buffer, actual_seq_len_buf, conv_cache, cache_index_buf }); @@ -167,40 +178,40 @@ const TestContext = struct { const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); defer self.allocator.free(in_key); - const in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); + const in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key, self.sharding); const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(.{ .batch, .seq, .d }); const key_cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache.key", .{name}); defer self.allocator.free(key_cache_key); - const key_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, key_cache_key); + const key_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, key_cache_key, self.sharding); const key_cache_tensor = zml.Tensor.fromShape(key_cache_buffer.shape()).withTags(.{ .layer, .batch, .h, .k, .hd }); const value_cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache.value", .{name}); defer self.allocator.free(value_cache_key); - const value_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, value_cache_key); + const value_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, value_cache_key, self.sharding); const value_cache_tensor = zml.Tensor.fromShape(value_cache_buffer.shape()).withTags(.{ .layer, .batch, .h, .k, .hd }); const cache_pos_key = try std.fmt.allocPrint(self.allocator, "{s}.cache_position", .{name}); defer self.allocator.free(cache_pos_key); - const cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key); + const cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key, self.sharding); const cache_pos_tensor = zml.Tensor.fromShape(cache_pos_buffer.shape()).withTags(.{.batch}); const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); defer self.allocator.free(out_key); - const out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); + const out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key, self.sharding); const cache_index_tensor: zml.Tensor = .init(.{}, .u32); - const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, model.KvCache{ .k = key_cache_tensor, .v = value_cache_tensor }, cache_index_tensor, self.attention_metadata, self.attention_parameters }); + const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, model.KvCache{ .k = key_cache_tensor, .v = value_cache_tensor }, cache_index_tensor, self.attention_metadata, self.attention_parameters }, .{ .shardings = &.{self.sharding} }); defer exe.deinit(); var args = try exe.args(self.allocator); defer args.deinit(self.allocator); const kv_cache: zml.Bufferized(model.KvCache) = .{ .k = key_cache_buffer, .v = value_cache_buffer }; - var attention_metadata_buffers = try self.attention_metadata.initBuffer(self.io, self.platform); + var attention_metadata_buffers = try self.attention_metadata.initBuffer(self.io, self.platform, self.sharding); defer attention.Metadata.deinitBuffer(&attention_metadata_buffers); const cache_index_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(cache_ix)})); - var cache_index_buf: zml.Buffer = try .fromSlice(self.io, self.platform, cache_index_slice); + var cache_index_buf: zml.Buffer = try .fromSlice(self.io, self.platform, cache_index_slice, self.sharding); defer cache_index_buf.deinit(); args.set(.{ layer_buffers, in_buffer, cache_pos_buffer, kv_cache, cache_index_buf, attention_metadata_buffers }); @@ -218,7 +229,7 @@ const TestContext = struct { } }; -fn loadBufferFromStore(allocator: std.mem.Allocator, io: anytype, platform: *zml.Platform, store: *zml.io.TensorStore, key: []const u8) !zml.Buffer { +fn loadBufferFromStore(allocator: std.mem.Allocator, io: anytype, platform: *zml.Platform, store: *zml.io.TensorStore, key: []const u8, sharding: zml.sharding.Sharding) !zml.Buffer { const shape = store.view().getShape(key) orelse return error.NotFound; const host_bytes = try allocator.alloc(u8, shape.byteSize()); @@ -229,5 +240,5 @@ fn loadBufferFromStore(allocator: std.mem.Allocator, io: anytype, platform: *zml defer reader.deinit(); _ = try reader.interface.readSliceAll(host_bytes); - return zml.Buffer.fromBytes(io, platform, shape, host_bytes); + return zml.Buffer.fromBytes(io, platform, shape, sharding, host_bytes); } diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel index c7c91eb4..f6215502 100644 --- a/examples/llama/BUILD.bazel +++ b/examples/llama/BUILD.bazel @@ -15,14 +15,6 @@ zig_binary( ], ) -zig_binary( - name = "loader", - srcs = ["llama.zig"], - main = "main2.zig", - visibility = ["//visibility:public"], - deps = ["//zml"], -) - mtree_spec( name = "mtree", srcs = [":llama"], diff --git a/mlir/dialects/func.zig b/mlir/dialects/func.zig index dd881648..3a63eb01 100644 --- a/mlir/dialects/func.zig +++ b/mlir/dialects/func.zig @@ -67,7 +67,6 @@ pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { }); } - pub fn call(ctx: *mlir.Context, name: []const u8, values: []const *const mlir.Value, results: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "func.call", .{ .operands = .{ .variadic = &.{values} }, diff --git a/platforms/cpu/cpu.zig b/platforms/cpu/cpu.zig index 37131347..83d21246 100644 --- a/platforms/cpu/cpu.zig +++ b/platforms/cpu/cpu.zig @@ -15,8 +15,8 @@ pub fn isEnabled() bool { } pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { - _ = io; // autofix - _ = allocator; // autofix + _ = io; + _ = allocator; if (comptime !isEnabled()) { return error.Unavailable; } diff --git a/platforms/cuda/flashattn/flashattn.zig b/platforms/cuda/flashattn/flashattn.zig index 2a743d33..368d0ffb 100644 --- a/platforms/cuda/flashattn/flashattn.zig +++ b/platforms/cuda/flashattn/flashattn.zig @@ -94,8 +94,8 @@ pub var fa2_mha_varlen_fwd: Fa2MhaVarlenFwdFunc = undefined; pub var fa3_mha_fwd: Fa3MhaFwdFunc = undefined; pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { - _ = io; // autofix - _ = allocator; // autofix + _ = io; + _ = allocator; const r = try bazel.runfiles(bazel_builtin.current_repository); diff --git a/platforms/neuron/neuron.zig b/platforms/neuron/neuron.zig index 8751f5ac..ad65dc21 100644 --- a/platforms/neuron/neuron.zig +++ b/platforms/neuron/neuron.zig @@ -34,7 +34,7 @@ fn isRunningOnEC2(io: std.Io) !bool { } pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { - _ = allocator; // autofix + _ = allocator; if (comptime !isEnabled()) { return error.Unavailable; } diff --git a/platforms/rocm/rocm.zig b/platforms/rocm/rocm.zig index 79053208..c5820333 100644 --- a/platforms/rocm/rocm.zig +++ b/platforms/rocm/rocm.zig @@ -27,7 +27,7 @@ fn setupRocmEnv(rocm_data_dir: []const u8) !void { } pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { - _ = allocator; // autofix + _ = allocator; if (comptime !isEnabled()) { return error.Unavailable; } diff --git a/platforms/tpu/tpu.zig b/platforms/tpu/tpu.zig index a9316699..e19a7606 100644 --- a/platforms/tpu/tpu.zig +++ b/platforms/tpu/tpu.zig @@ -34,7 +34,7 @@ fn isOnGCP(io: std.Io) !bool { } pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { - _ = allocator; // autofix + _ = allocator; if (comptime !isEnabled()) { return error.Unavailable; } diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index c68cc218..3eca623e 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -416,7 +416,7 @@ pub const fa3 = struct { pub const InitOptions = struct {}; pub fn init(opts: InitOptions) fa3.Parameters { - _ = opts; // autofix + _ = opts; return .{}; } }; @@ -730,10 +730,10 @@ pub const paged_fa2 = struct { decode_offset: ?zml.Tensor = null, pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { - _ = num_heads; // autofix - _ = num_kv_heads; // autofix - _ = head_dim; // autofix - _ = page_size; // autofix + _ = num_heads; + _ = num_kv_heads; + _ = head_dim; + _ = page_size; const max_seqlen_k = switch (parameters) { inline else => |v| v.options.max_seqlen_k, }; @@ -1325,10 +1325,10 @@ pub const paged_fa3 = struct { decode_offset: ?zml.Tensor = null, pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { - _ = num_heads; // autofix - _ = num_kv_heads; // autofix - _ = head_dim; // autofix - _ = page_size; // autofix + _ = num_heads; + _ = num_kv_heads; + _ = head_dim; + _ = page_size; const max_seqlen_k = switch (parameters) { inline else => |v| v.options.max_seqlen_k, }; diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 88a94b28..a3126ff4 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -158,8 +158,8 @@ pub const AttentionOptions = struct { }; pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { - _ = k; // autofix - _ = v; // autofix + _ = k; + _ = v; return switch (parameters) { .cuda_fa2 => |cuda_fa2_parameters| flashattn.paged_fa2.pagedAttention(cuda_fa2_parameters, context.cuda_fa2, q, k_cache, v_cache, layer_index, opts), .cuda_fa3 => |cuda_fa3_parameters| flashattn.paged_fa3.pagedAttention(cuda_fa3_parameters, context.cuda_fa3, q, k_cache, v_cache, layer_index, opts), diff --git a/zml/nn.zig b/zml/nn.zig index e5ae129b..4a3d22e7 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -125,7 +125,6 @@ pub fn normalizeL2(input: Tensor, eps: f32) Tensor { test normalizeL2 { const platform = zml.testing.env(); - const input: zml.Tensor = .init(.{ 2, 2 }, .f32); var exe = try zml.module.compile(std.testing.allocator, std.testing.io, normalizeL2, .{ input, 1e-12 }, platform, .{ .shardings = &.{zml.testing.replicatedSharding()} }); @@ -199,8 +198,8 @@ pub const RopeOpts = struct { } pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) std.json.ParseFromValueError!Scaling { - _ = allocator; // autofix - _ = options; // autofix + _ = allocator; + _ = options; if (source == .null) return .{ .default = .{} }; if (source != .object) return error.InvalidEnumTag; diff --git a/zml/testing.zig b/zml/testing.zig index b6a7d363..8fbc9603 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -302,7 +302,7 @@ pub fn testLayer( activation_store: zml.io.TensorStore.View, name: []const u8, layer_weights: zml.Bufferized(@TypeOf(layer)), - shardings: []const zml.Sharding, + shardings: []const zml.sharding.Sharding, opts: CompareOpts, ) !void { var arena = std.heap.ArenaAllocator.init(allocator); @@ -332,7 +332,7 @@ pub fn testLayer( var buffer: [256]u8 = undefined; const subkey = std.fmt.bufPrint(&buffer, "{d}", .{ctx_.index}) catch unreachable; - tensor.* = ctx_.activation_store.createTensor(subkey); + tensor.* = ctx_.activation_store.createTensor(subkey, null, null); ctx_.index += 1; } }.cb, &ctx, &args); diff --git a/zml/tokenizer/main.zig b/zml/tokenizer/main.zig index a35de7d5..76039b89 100644 --- a/zml/tokenizer/main.zig +++ b/zml/tokenizer/main.zig @@ -12,16 +12,16 @@ const Flags = struct { verbose: bool = false, }; -pub fn main() !void { +pub fn main(init: std.process.Init) !void { var gpa: std.heap.GeneralPurposeAllocator(.{}) = .{}; const allocator = gpa.allocator(); - var threaded: std.Io.Threaded = .init(allocator); + var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); const io = threaded.io(); - const args = stdx.flags.parseProcessArgs(Flags); + const args = stdx.flags.parseProcessArgs(init.minimal, Flags); log.info("\tLoading tokenizer from {s}", .{args.tokenizer}); var tokenizer = try zml_tokenizer.Tokenizer.fromFile(allocator, io, args.tokenizer); diff --git a/zml/tokenizer/sentencepiece/BUILD.bazel b/zml/tokenizer/sentencepiece/BUILD.bazel index f8e571e2..9e46e373 100644 --- a/zml/tokenizer/sentencepiece/BUILD.bazel +++ b/zml/tokenizer/sentencepiece/BUILD.bazel @@ -30,8 +30,3 @@ zig_library( # name = "sources", # zig_lib = ":sentencepiece", # ) - -cc_static_library( - name="sentencepiece_static", - deps = [":sentencepiece_swig"] -) diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig index c72ec803..9c563daa 100644 --- a/zml/tokenizer/tokenizer.zig +++ b/zml/tokenizer/tokenizer.zig @@ -93,8 +93,8 @@ pub const Tokenizer = union(Tokenizers) { homemade: *homemade.Tokenizer, pub fn fromFile(allocator: std.mem.Allocator, io: std.Io, model: []const u8) !Tokenizer { - _ = allocator; // autofix - _ = io; // autofix + _ = allocator; + _ = io; if (std.mem.endsWith(u8, model, ".pb")) { return .{ .sentencepiece = try .fromFile(model) }; } @@ -106,8 +106,8 @@ pub const Tokenizer = union(Tokenizers) { } pub fn fromBytes(allocator: std.mem.Allocator, io: std.Io, bytes: []const u8) !Tokenizer { - _ = allocator; // autofix - _ = io; // autofix + _ = allocator; + _ = io; if (bytes[0] == '{') { return .{ .hftokenizers = try .fromBytes(bytes) }; } From 06f13f9a22ae79022383ef7fc4714fbc7ea9e459 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 20 Mar 2026 16:21:45 +0000 Subject: [PATCH 364/881] platforms: uncompress http_archives using http_bsdtar_archive Roughly a 2x speedup according to @cerisier. So take it for what it is. --- MODULE.bazel | 16 +- MODULE.bazel.lock | 5 +- bazel/http_deb_archive.bzl | 267 ++++++++++++++++--- bazel/zig_srcs.bzl | 3 +- platforms/cpu/BUILD.bazel | 2 +- platforms/cuda/cuda.bzl | 2 +- platforms/cuda/libpjrt_cuda.BUILD.bazel | 2 +- platforms/neuron/libpjrt_neuron.BUILD.bazel | 2 +- platforms/rocm/libpjrt_rocm.BUILD.bazel | 9 +- platforms/rocm/rocm.bzl | 2 +- platforms/tpu/libpjrt_tpu.BUILD.bazel | 2 +- platforms/tpu/tpu.bzl | 2 +- third_party/zls/zls_completion.bzl | 2 +- third_party/zls/zls_write_runner_zig_src.bzl | 2 +- 14 files changed, 266 insertions(+), 52 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 3094a6be..247533c8 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -3,7 +3,7 @@ module( ) bazel_dep(name = "abseil-cpp", version = "20260107.1", repo_name = "com_google_absl") -bazel_dep(name = "aspect_bazel_lib", version = "2.22.5") +bazel_dep(name = "bazel_lib", version = "3.2.2") bazel_dep(name = "aspect_rules_py", version = "1.8.4") bazel_dep(name = "bazel_skylib", version = "1.9.0") bazel_dep(name = "patchelf", version = "0.18.0.bcr.1") @@ -21,12 +21,26 @@ bazel_dep(name = "rules_python", version = "1.8.4") bazel_dep(name = "rules_rs", version = "0.0.44") bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "rules_zig", version = "0.12.2") +bazel_dep(name = "tar.bzl", version = "0.9.0") bazel_dep(name = "llvm", version = "0.6.8") bazel_dep(name = "jsoncpp", version = "1.9.6", repo_name = "jsoncpp_git") bazel_dep(name = "with_cfg.bzl", version = "0.14.4") bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2", dev_dependency = True) +# BSDTar toolchains +tar_toolchains = use_extension("@tar.bzl//tar:extensions.bzl", "toolchains") +use_repo( + tar_toolchains, + "bsd_tar_toolchains_darwin_amd64", + "bsd_tar_toolchains_darwin_arm64", + "bsd_tar_toolchains_linux_amd64", + "bsd_tar_toolchains_linux_arm64", + "bsd_tar_toolchains_windows_amd64", + "bsd_tar_toolchains_windows_arm64", +) +### + ### Zig toolchain zig = use_extension("@rules_zig//zig:extensions.bzl", "zig") zig.index(file = "//bazel:zig_index.json") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index d36b5a20..59b2af9c 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -22,8 +22,7 @@ "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.3/MODULE.bazel": "253d739ba126f62a5767d832765b12b59e9f8d2bc88cc1572f4a73e46eb298ca", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.21.2/MODULE.bazel": "276347663a25b0d5bd6cad869252bea3e160c4d980e764b15f3bae7f80b30624", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.22.5/MODULE.bazel": "004ba890363d05372a97248c37205ae64b6fa31047629cd2c0895a9d0c7779e8", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.22.5/source.json": "ac2c3213df8f985785f1d0aeb7f0f73d5324e6e67d593d9b9470fb74a25d4a9b", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.21.2/source.json": "f42051fa42629f0e59b7ac2adf0a55749144b11f1efcd8c697f0ee247181e526", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "780d1a6522b28f5edb7ea09630748720721dfe27690d65a2d33aa7509de77e07", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "812d2dd42f65dca362152101fbec418029cc8fd34cbad1a2fde905383d705838", "https://bcr.bazel.build/modules/aspect_rules_py/1.8.4/MODULE.bazel": "4ab4a78e9d65cf15e755807f7ae504655e1771cefd7c3343ad34dbbb625b9564", @@ -426,7 +425,7 @@ }, "@@rules_oci+//oci:extensions.bzl%oci": { "general": { - "bzlTransitiveDigest": "+eLA5tKLmYYsk3vwQ5V6L8BLyV5uoQPm6tbs+jLOFNk=", + "bzlTransitiveDigest": "twSeB3VPTVkayY0uwqEzDzal5MbnZX45iC6jdheZQRo=", "usagesDigest": "IIGPr69MT/mSUo6fNUffrIiU31EqGjlEYm5qFmNjmUc=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, diff --git a/bazel/http_deb_archive.bzl b/bazel/http_deb_archive.bzl index 91b52f5c..968001d6 100644 --- a/bazel/http_deb_archive.bzl +++ b/bazel/http_deb_archive.bzl @@ -1,46 +1,253 @@ +load("@bazel_lib//lib:repo_utils.bzl", "repo_utils") +load( + "@bazel_tools//tools/build_defs/repo:cache.bzl", + "DEFAULT_CANONICAL_ID_ENV", + "get_default_canonical_id", +) load( "@bazel_tools//tools/build_defs/repo:utils.bzl", "get_auth", "patch", + "update_attrs", "workspace_and_buildfile", ) -def _http_deb_archive_impl(rctx): - if rctx.attr.build_file and rctx.attr.build_file_content: - fail("Only one of build_file and build_file_content can be provided.") - download_info = rctx.download_and_extract( - url = rctx.attr.urls, - output = "tmp", - sha256 = rctx.attr.sha256, - type = "deb", - stripPrefix = "", - canonical_id = " ".join(rctx.attr.urls), - auth = get_auth(rctx, rctx.attr.urls), +def _get_source_urls(rctx): + if not rctx.attr.url and not rctx.attr.urls: + fail("At least one of url and urls must be provided") + + source_urls = [] + if rctx.attr.urls: + source_urls = rctx.attr.urls + if rctx.attr.url: + source_urls = [rctx.attr.url] + source_urls + return source_urls + +def _prefixed_pattern(strip_prefix, pattern): + if not strip_prefix: + return pattern + if not pattern: + return strip_prefix + if pattern == strip_prefix or pattern.startswith(strip_prefix + "/"): + return pattern + return strip_prefix + "/" + pattern + +def _downloaded_archive_name(rctx, source_urls): + if rctx.attr.type: + return ".downloaded.archive." + rctx.attr.type + + basename = source_urls[0].split("?", 1)[0].split("/")[-1] + if basename: + return ".downloaded." + basename + return ".downloaded.archive" + +def _placeholder_dir_from_exclude(pattern): + if pattern.endswith("/*"): + return pattern[:-2].strip("/") + return None + +def _ensure_placeholder_dirs(rctx, add_prefix): + for exclude in rctx.attr.excludes: + directory = _placeholder_dir_from_exclude(exclude) + if not directory: + continue + if add_prefix: + directory = add_prefix + "/" + directory + keep_file = directory + "/.http_deb_archive.keep" + rctx.file(keep_file, "") + rctx.delete(keep_file) + +def _host_bsdtar_label(rctx): + platform = repo_utils.platform(rctx) + binary = "tar.exe" if platform.startswith("windows_") else "tar" + return Label("@bsd_tar_toolchains_{}//:{}".format(platform, binary)) + +def _update_integrity_attrs(rctx, attrs, archive_info, remote_files_info, remote_patches_info): + integrity_override = {} + if not rctx.attr.sha256 and not rctx.attr.integrity: + integrity_override["integrity"] = archive_info.integrity + + remote_file_integrity = {path: info.integrity for path, info in remote_files_info.items()} + if rctx.attr.remote_file_integrity != remote_file_integrity: + integrity_override["remote_file_integrity"] = remote_file_integrity + + # TODO(cerisier): Remove this "if" when we no longer support bazel < 8.5.0 + if remote_patches_info: + remote_patch_integrity = {url: info.integrity for url, info in remote_patches_info.items()} + if rctx.attr.remote_patches != remote_patch_integrity: + integrity_override["remote_patches"] = remote_patch_integrity + + if not integrity_override: + return rctx.repo_metadata(reproducible = True) + + return rctx.repo_metadata( + attrs_for_reproducibility = update_attrs(rctx.attr, attrs.keys(), integrity_override), ) +# TODO(cerisier): Remove when we no longer support bazel < 8.5.0 +def symlink_files(ctx): + # type: (repository_ctx) -> None + """Utility function for symlinking local files. + + This is intended to be used in the implementation function of a repository rule. It assumes the + parameter `files` is present in `ctx.attr`. + + Existing files will be overwritten. + + Args: + ctx: The repository context of the repository rule calling this utility + function. + """ + for path, label in ctx.attr.files.items(): + src_path = ctx.path(label) + + # On Windows `ctx.symlink` may be implemented as a copy, so the file MUST be watched + ctx.watch(src_path) + if not src_path.exists: + fail("Input %s does not exist" % label) + if ctx.path(path).exists: + ctx.delete(path) + ctx.symlink(src_path, path) + +# TODO(cerisier): Remove when we no longer support bazel < 8.5.0 +def download_remote_files(ctx, auth = None): + """Utility function for downloading remote files. + + This rule is intended to be used in the implementation function of + a repository rule. It assumes the parameters `remote_file_urls` and + `remote_file_integrity` to be present in `ctx.attr`. + + Existing files will be overwritten. + + Args: + ctx: The repository context of the repository rule calling this utility + function. + auth: An optional dict specifying authentication information for some of the URLs. + + Returns: + dict mapping file paths to a download info. + """ + pending = { + path: ctx.download( + remote_file_urls, + path, + canonical_id = ctx.attr.canonical_id, + auth = get_auth(ctx, remote_file_urls) if auth == None else auth, + integrity = ctx.attr.remote_file_integrity.get(path, ""), + block = False, + # Overlaid files may be shell scripts. + executable = True, + ) + for path, remote_file_urls in ctx.attr.remote_file_urls.items() + } + + # Wait until the requests are done + return {path: token.wait() for path, token in pending.items()} + +def _http_deb_archive_impl(rctx): + source_urls = _get_source_urls(rctx) + + if rctx.attr.build_file and rctx.attr.build_file_content: + fail("Only one of build_file and build_file_content can be provided.") + + archive = _downloaded_archive_name(rctx, source_urls) + archive_info = rctx.download( + source_urls, + archive, + rctx.attr.sha256, + canonical_id = rctx.attr.canonical_id or get_default_canonical_id(rctx, source_urls), + auth = get_auth(rctx, source_urls), + integrity = rctx.attr.integrity, + ) + + host_bsdtar = _host_bsdtar_label(rctx) + res = rctx.execute([ + rctx.path(host_bsdtar), + "-xf", + archive, + "--include=data.tar.*", + ]) for ext in ["gz", "xz", "zst"]: - data = "tmp/data.tar.{}".format(ext) + data = "data.tar.{}".format(ext) if rctx.path(data).exists: - rctx.extract( - archive = data, - output = "", - stripPrefix = rctx.attr.strip_prefix, - ) - rctx.delete("tmp") + archive = data break + + strip_prefix = rctx.attr.strip_prefix.strip("/") + includes = [_prefixed_pattern(strip_prefix, include) for include in rctx.attr.includes] + excludes = [_prefixed_pattern(strip_prefix, exclude) for exclude in rctx.attr.excludes] + + if strip_prefix and not includes: + includes = [strip_prefix, strip_prefix + "/*"] + + args = [] + for include in includes: + args.extend(["--include", include]) + for exclude in excludes: + args.extend(["--exclude", exclude]) + if strip_prefix: + args.extend(["--strip-components", str(len(strip_prefix.split("/")))]) + args.extend(rctx.attr.bsdtar_extra_args) + + add_prefix = rctx.attr.add_prefix.strip("/") + keep_file = None + if add_prefix: + keep_file = add_prefix + "/.http_deb_archive.keep" + rctx.file(keep_file, "") + args.extend(["-C", add_prefix]) + + args.extend(["-xf", archive]) + res = rctx.execute([rctx.path(host_bsdtar)] + args) + if res.return_code != 0: + fail("Failed to extract archive: {}\n{}".format(res.stderr, res.stdout)) + + if keep_file: + rctx.delete(keep_file) + _ensure_placeholder_dirs(rctx, add_prefix) + rctx.delete(archive) + workspace_and_buildfile(rctx) - patch(rctx) + remote_files_info = download_remote_files(rctx) + remote_patches_info = patch(rctx) + symlink_files(rctx) + + return _update_integrity_attrs(rctx, _http_deb_archive_attrs, archive_info, remote_files_info, remote_patches_info) + +_http_deb_archive_attrs = { + "url": attr.string(), + "urls": attr.string_list(), + "sha256": attr.string(), + "integrity": attr.string(), + "netrc": attr.string(), + "auth_patterns": attr.string_dict(), + "canonical_id": attr.string(), + "strip_prefix": attr.string(), + "add_prefix": attr.string(default = ""), + "files": attr.string_keyed_label_dict(default = {}), + "type": attr.string(), + "patches": attr.label_list(default = []), + "remote_file_urls": attr.string_list_dict(default = {}), + "remote_file_integrity": attr.string_dict(default = {}), + "remote_module_file_urls": attr.string_list(default = []), + "remote_module_file_integrity": attr.string(default = ""), + "remote_patches": attr.string_dict(default = {}), + "remote_patch_strip": attr.int(default = 0), + "patch_tool": attr.string(default = ""), + "patch_args": attr.string_list(default = []), + "patch_strip": attr.int(default = 0), + "patch_cmds": attr.string_list(default = []), + "patch_cmds_win": attr.string_list(default = []), + "build_file": attr.label(allow_single_file = True), + "build_file_content": attr.string(), + "workspace_file": attr.label(), + "workspace_file_content": attr.string(), + "includes": attr.string_list(default = []), + "excludes": attr.string_list(default = []), + "bsdtar_extra_args": attr.string_list(default = []), +} http_deb_archive = repository_rule( - _http_deb_archive_impl, - attrs = { - "urls": attr.string_list(mandatory = True), - "sha256": attr.string(mandatory = True), - "strip_prefix": attr.string(), - "build_file": attr.label(allow_single_file = True), - "build_file_content": attr.string(), - "patches": attr.label_list(allow_files = True), - "workspace_file": attr.label(allow_single_file = True), - "workspace_file_content": attr.string(), - }, + implementation = _http_deb_archive_impl, + attrs = _http_deb_archive_attrs, + environ = [DEFAULT_CANONICAL_ID_ENV], ) diff --git a/bazel/zig_srcs.bzl b/bazel/zig_srcs.bzl index ee4adb36..96bebfab 100644 --- a/bazel/zig_srcs.bzl +++ b/bazel/zig_srcs.bzl @@ -1,4 +1,5 @@ -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") +load("@tar.bzl//tar:mtree.bzl", "mtree_spec") +load("@tar.bzl//tar:tar.bzl", "tar") load("@rules_zig//zig:defs.bzl", "zig_static_library") def zig_srcs(name, zig_bin = "", zig_lib = ""): diff --git a/platforms/cpu/BUILD.bazel b/platforms/cpu/BUILD.bazel index 16e059d5..baa31044 100644 --- a/platforms/cpu/BUILD.bazel +++ b/platforms/cpu/BUILD.bazel @@ -1,4 +1,4 @@ -load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library") diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 9e78edaa..a28eb0d6 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -1,5 +1,5 @@ load("@bazel_skylib//lib:paths.bzl", "paths") -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") load("//platforms:packages.bzl", "packages") diff --git a/platforms/cuda/libpjrt_cuda.BUILD.bazel b/platforms/cuda/libpjrt_cuda.BUILD.bazel index 4d8a06af..0cd7f7b6 100644 --- a/platforms/cuda/libpjrt_cuda.BUILD.bazel +++ b/platforms/cuda/libpjrt_cuda.BUILD.bazel @@ -1,4 +1,4 @@ -load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@zml//bazel:patchelf.bzl", "patchelf") load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@zml//platforms/cuda:cuda.bzl", "CUDA_COMPAT_FILES") diff --git a/platforms/neuron/libpjrt_neuron.BUILD.bazel b/platforms/neuron/libpjrt_neuron.BUILD.bazel index df831ad4..c95cb108 100644 --- a/platforms/neuron/libpjrt_neuron.BUILD.bazel +++ b/platforms/neuron/libpjrt_neuron.BUILD.bazel @@ -1,4 +1,4 @@ -load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@bazel_skylib//rules:select_file.bzl", "select_file") load("@zml//bazel:patchelf.bzl", "patchelf") load("@zml//bazel:runfiles.bzl", "runfiles_to_default") diff --git a/platforms/rocm/libpjrt_rocm.BUILD.bazel b/platforms/rocm/libpjrt_rocm.BUILD.bazel index 4aaabacf..c528f0d9 100644 --- a/platforms/rocm/libpjrt_rocm.BUILD.bazel +++ b/platforms/rocm/libpjrt_rocm.BUILD.bazel @@ -1,5 +1,4 @@ -load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") -load("@aspect_bazel_lib//lib:tar.bzl", "tar") +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "string_list_flag") load("@zml//bazel:patchelf.bzl", "patchelf") @@ -85,12 +84,6 @@ copy_to_directory( include_external_repositories = ["**"], ) -tar( - name = "amdgpu_ids_layer", - srcs = ["@libdrm-amdgpu-common//:amdgpu_ids"], - visibility = ["//visibility:public"], -) - cc_library( name = "libpjrt_rocm", data = [":sandbox"], diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 78171a3b..67402369 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -1,4 +1,4 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") load("//platforms:packages.bzl", "packages") diff --git a/platforms/tpu/libpjrt_tpu.BUILD.bazel b/platforms/tpu/libpjrt_tpu.BUILD.bazel index 767cd004..d85e167b 100644 --- a/platforms/tpu/libpjrt_tpu.BUILD.bazel +++ b/platforms/tpu/libpjrt_tpu.BUILD.bazel @@ -1,4 +1,4 @@ -load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@bazel_skylib//rules:copy_file.bzl", "copy_file") copy_file( diff --git a/platforms/tpu/tpu.bzl b/platforms/tpu/tpu.bzl index ef187b37..89d86e64 100644 --- a/platforms/tpu/tpu.bzl +++ b/platforms/tpu/tpu.bzl @@ -1,4 +1,4 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") def _tpu_impl(mctx): # https://storage.googleapis.com/jax-releases/libtpu_releases.html diff --git a/third_party/zls/zls_completion.bzl b/third_party/zls/zls_completion.bzl index 2e949c7e..a461f933 100644 --- a/third_party/zls/zls_completion.bzl +++ b/third_party/zls/zls_completion.bzl @@ -1,6 +1,6 @@ """Implementation of the zls_completion macro.""" -load("@aspect_bazel_lib//lib:utils.bzl", "utils") +load("@bazel_lib//lib:utils.bzl", "utils") load("@bazel_skylib//rules:expand_template.bzl", "expand_template") load("@rules_zig//zig:defs.bzl", "zig_binary") load(":zls_write_build_config.bzl", "zls_write_build_config") diff --git a/third_party/zls/zls_write_runner_zig_src.bzl b/third_party/zls/zls_write_runner_zig_src.bzl index ac89e08e..a4af5770 100644 --- a/third_party/zls/zls_write_runner_zig_src.bzl +++ b/third_party/zls/zls_write_runner_zig_src.bzl @@ -1,6 +1,6 @@ """Generates the ZLS runner source file.""" -load("@aspect_bazel_lib//lib:paths.bzl", "to_rlocation_path") +load("@bazel_lib//lib:paths.bzl", "to_rlocation_path") def _zls_write_runner_zig_src_impl(ctx): zigtoolchaininfo = ctx.toolchains["@rules_zig//zig:toolchain_type"].zigtoolchaininfo From cb5443e5da3559829fb362e1a5a573f8ce5aa2ae Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sat, 21 Mar 2026 12:15:14 +0000 Subject: [PATCH 365/881] platforms/cuda: bump PJRT plugin --- platforms/cuda/cuda.bzl | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index a28eb0d6..43947746 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -19,7 +19,7 @@ CUDA_VARIANT = "cuda13.1" CUDA_REDIST_JSON_SHA256 = "97cf605ccc4751825b1865f4af571c9b50dd29ffd13e9a38b296a9ecb1f0d422" CUDNN_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cudnn/redist/" -CUDNN_VERSION = "9.19.0" +CUDNN_VERSION = "9.19.1" CUDNN_REDIST_JSON_SHA256 = "6ad4f2c047ee03131d1efb95db56f78c24953cb30880a2edcf99407ff9362a04" NVSHMEM_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/nvshmem/redist/" @@ -120,10 +120,6 @@ CUDA_PACKAGES = { "nvvm/libdevice/libdevice.10.bc", ], ), - packages.cc_import( - name = "nvptxcompiler", - static_library = "lib/libnvptxcompiler_static.a", - ), ]), "cuda_nvrtc": "\n".join([ packages.filegroup( @@ -261,17 +257,20 @@ def _cuda_impl(mctx): urls = ["https://pypi.nvidia.com/nvidia-nccl-cu13/nvidia_nccl_cu13-2.29.3-py3-none-manylinux_2_18_x86_64.whl"], type = "zip", sha256 = "2a321629f49490e4e0122ecb578a4b4a6f89e72740dd988e04dfa4758fab7fc3", - build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + packages.filegroup( - name = "nccl", - srcs = ["nvidia/nccl/lib/libnccl.so.2"], - ), + build_file_content = "\n".join([ + _BUILD_FILE_DEFAULT_VISIBILITY, + packages.filegroup( + name = "nccl", + srcs = ["nvidia/nccl/lib/libnccl.so.2"], + ), + ]), ) http_archive( name = "libpjrt_cuda", build_file = "libpjrt_cuda.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v15.0.2/pjrt-cuda_linux-amd64.tar.gz", - sha256 = "ebc5f0fa54d38ec85346f0f02b29a6497ee394b9e5fa4832da5db16d3296ae84", + url = "https://github.com/zml/pjrt-artifacts/releases/download/nightly-2026-03-21/pjrt-cuda_linux-amd64.tar.gz", + sha256 = "854b2cabc162dd56ed38548062b5d2ce6bfc33cc21851789de8fcc346a82cb07", ) return mctx.extension_metadata( From 13c7c4bb33524ec724c586229a233254882b78c9 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sat, 21 Mar 2026 15:29:49 +0000 Subject: [PATCH 366/881] platforms/rocm: bump PJRT plugin, switch to hipBLASLt instead of triton faster compile times! but also +10tok/s on Llama 3.1 8B (125tok/s -> 136tok/s) --- platforms/rocm/libpjrt_rocm.BUILD.bazel | 3 +- platforms/rocm/packages.lock.json | 367 ++++++++++++++++++++++++ platforms/rocm/packages.yaml | 1 + platforms/rocm/rocm.bzl | 7 +- zml/module.zig | 29 +- 5 files changed, 383 insertions(+), 24 deletions(-) diff --git a/platforms/rocm/libpjrt_rocm.BUILD.bazel b/platforms/rocm/libpjrt_rocm.BUILD.bazel index c528f0d9..ee8200de 100644 --- a/platforms/rocm/libpjrt_rocm.BUILD.bazel +++ b/platforms/rocm/libpjrt_rocm.BUILD.bazel @@ -4,7 +4,7 @@ load("@zml//bazel:patchelf.bzl", "patchelf") bool_flag( name = "hipblaslt", - build_setting_default = False, + build_setting_default = True, visibility = ["//visibility:public"], ) @@ -57,6 +57,7 @@ copy_to_directory( "@rocprofiler-sdk//:rocprofiler-sdk", "@rocprofiler-sdk-rocpd//:rocprofiler-sdk-rocpd", "@rocprofiler-sdk-roctx//:rocprofiler-sdk-roctx", + "@rocsparse", "@rocsolver", "@roctracer", "@roctracer//:roctx", diff --git a/platforms/rocm/packages.lock.json b/platforms/rocm/packages.lock.json index 67139f24..a495ec2c 100755 --- a/platforms/rocm/packages.lock.json +++ b/platforms/rocm/packages.lock.json @@ -6193,6 +6193,373 @@ "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hipblas-common-dev/hipblas-common-dev_1.4.0.70200-43~22.04_amd64.deb" ], "version": "1.4.0.70200-43~22.04" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "name": "rocm-core", + "version": "7.2.0.70200-43~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "rocblas_5.2.0.70200-43_22.04_amd64", + "name": "rocblas", + "version": "5.2.0.70200-43~22.04" + }, + { + "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", + "name": "hip-runtime-amd", + "version": "7.2.26015.70200-43~22.04" + }, + { + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" + }, + { + "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70200-43~22.04" + }, + { + "key": "comgr_3.0.0.70200-43_22.04_amd64", + "name": "comgr", + "version": "3.0.0.70200-43~22.04" + }, + { + "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", + "name": "libtinfo-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", + "name": "libncurses-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", + "name": "ncurses-bin", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libncurses6_6.3-2ubuntu0.1_amd64", + "name": "libncurses6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "rocminfo_1.0.0.70200-43_22.04_amd64", + "name": "rocminfo", + "version": "1.0.0.70200-43~22.04" + }, + { + "key": "pciutils_1-3.7.0-6_amd64", + "name": "pciutils", + "version": "1:3.7.0-6" + }, + { + "key": "libpci3_1-3.7.0-6_amd64", + "name": "libpci3", + "version": "1:3.7.0-6" + }, + { + "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", + "name": "pci.ids", + "version": "0.0~2022.01.22-1ubuntu0.1" + }, + { + "key": "libudev1_249.11-0ubuntu3.16_amd64", + "name": "libudev1", + "version": "249.11-0ubuntu3.16" + }, + { + "key": "libkmod2_29-1ubuntu1_amd64", + "name": "libkmod2", + "version": "29-1ubuntu1" + }, + { + "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", + "name": "hsa-runtime-rocr4wsl-amdgpu", + "version": "25.30.13-2281980.22.04" + }, + { + "key": "libelf1_0.186-1ubuntu0.1_amd64", + "name": "libelf1", + "version": "0.186-1ubuntu0.1" + }, + { + "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", + "name": "hipblaslt", + "version": "1.2.1.70200-43~22.04" + }, + { + "key": "roctracer_4.1.70200.70200-43_22.04_amd64", + "name": "roctracer", + "version": "4.1.70200.70200-43~22.04" + } + ], + "key": "rocsparse_4.2.0.70200-43_22.04_amd64", + "name": "rocsparse", + "sha256": "f72489fc3632558c7142810090ecf01565aca6b9c702c751bec4492450de9a8a", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocsparse/rocsparse_4.2.0.70200-43~22.04_amd64.deb" + ], + "version": "4.2.0.70200-43~22.04" } ], "version": 1 diff --git a/platforms/rocm/packages.yaml b/platforms/rocm/packages.yaml index b9b96902..d98be620 100644 --- a/platforms/rocm/packages.yaml +++ b/platforms/rocm/packages.yaml @@ -40,3 +40,4 @@ packages: - "hipblaslt" - "hipblaslt-dev" - "hip-runtime-amd" + - "rocsparse" diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 67402369..2b49a128 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -6,7 +6,7 @@ _BUILD_FILE_DEFAULT_VISIBILITY = """\ package(default_visibility = ["//visibility:public"]) """ -_ROCM_STRIP_PREFIX = "opt/rocm-7.2.0" +_ROCM_STRIP_PREFIX = "./opt/rocm-7.2.0" _UBUNTU_PACKAGES = { "libdrm2-amdgpu": packages.filegroup(name = "libdrm2-amdgpu", srcs = ["opt/amdgpu/lib/x86_64-linux-gnu/libdrm.so.2"]), @@ -120,6 +120,7 @@ _ROCM_PACKAGES = { packages.filegroup(name = "hiprtc", srcs = ["lib/libhiprtc.so.7", "lib/libhiprtc-builtins.so.7"]), ]), "hipsolver": packages.filegroup(name = "hipsolver", srcs = ["lib/libhipsolver.so.1"]), + "rocsparse": packages.filegroup(name = "rocsparse", srcs = ["lib/librocsparse.so.1"]), } def _rocm_impl(mctx): @@ -149,8 +150,8 @@ def _rocm_impl(mctx): http_archive( name = "libpjrt_rocm", build_file = "libpjrt_rocm.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v16.0.0/pjrt-rocm_linux-amd64.tar.gz", - sha256 = "6a70cf6a5a8330809f173072dd5fae5332c3ec8fba80acb9b77ddd97bda6572b", + url = "https://github.com/zml/pjrt-artifacts/releases/download/nightly-2026-03-21/pjrt-rocm_linux-amd64.tar.gz", + sha256 = "fd324748258192fe9dd1b830f97b619365407362bf2bdb80b8d68bec7baba7ea", ) return mctx.extension_metadata( diff --git a/zml/module.zig b/zml/module.zig index c65017ad..f12e3292 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -571,29 +571,18 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_llvm_module_compilation_parallelism", true, upb_arena); }, .rocm => { - // enable only on CDNA - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_triton_gemm", gemm_blk: { - const supported: std.StaticStringMap(void) = .initComptime(.{ - .{ "gfx950", {} }, - .{ "gfx950", {} }, - .{ "gfx942", {} }, - .{ "gfx942", {} }, - .{ "gfx942", {} }, - .{ "gfx90a", {} }, - .{ "gfx90a", {} }, - .{ "gfx90a", {} }, - .{ "gfx908", {} }, - .{ "gfx906", {} }, - .{ "gfx900", {} }, - }); - break :gemm_blk supported.has(platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "compute_capability").?.string); - }, upb_arena); + // Use hipBLASLt only + try setXlaOverrideFlag(overrides_map, "xla_gpu_autotune_level", 0, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_triton_gemm", false, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_cublaslt", true, upb_arena); + // Use lld from libllvm instead of invoking the ld.lld binary. // This saves us from having to sandbox it. try setXlaOverrideFlag(overrides_map, "xla_gpu_use_inprocess_lld", true, upb_arena); - // Disable command buffer to avoid some weird crashes. - // This is what AMD recommended in the meantime. - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "", upb_arena); + + // Do not enable the FUSION command buffer to avoid some weird crashes. + // This is what AMD recommendeds in the meantime. + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "CUBLAS,CUBLASLT,CUSTOM_CALL,CUDNN,DYNAMIC_SLICE_FUSION", upb_arena); }, else => {}, } From 58b2a4b5524df05a3213b86145b8ff43eb679b9a Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sat, 21 Mar 2026 16:21:57 +0000 Subject: [PATCH 367/881] examples/llama: fix bazel_lib and tar imports --- examples/llama/BUILD.bazel | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel index f6215502..d02b86d2 100644 --- a/examples/llama/BUILD.bazel +++ b/examples/llama/BUILD.bazel @@ -1,8 +1,9 @@ -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") -load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") load("@rules_zig//zig:defs.bzl", "zig_binary") +load("@tar.bzl//tar:mtree.bzl", "mtree_spec") +load("@tar.bzl//tar:tar.bzl", "tar") zig_binary( name = "llama", From c8e8db75e6f5d34d0feae0c4d8fd87f00ba864a7 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 20 Mar 2026 11:16:34 +0100 Subject: [PATCH 368/881] third_party/translate-c: bump translate-c and arocc --- third_party/arocc/repo.bzl | 4 ++-- third_party/translate-c/repo.bzl | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/third_party/arocc/repo.bzl b/third_party/arocc/repo.bzl index 510445d1..627dca69 100644 --- a/third_party/arocc/repo.bzl +++ b/third_party/arocc/repo.bzl @@ -3,7 +3,7 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") def repo(): new_git_repository( name = "arocc", - remote = "https://github.com/zml/arocc.git", - commit = "73acf3aa8164cc8fee16b0f79036b92b07dab053", + remote = "https://github.com/Vexu/arocc.git", + commit = "a08e2ca4b0e83059c3892b8b3e026754467e0ed1", build_file = "//third_party/arocc:arocc.bazel", ) diff --git a/third_party/translate-c/repo.bzl b/third_party/translate-c/repo.bzl index 2f502b95..f601c6d3 100644 --- a/third_party/translate-c/repo.bzl +++ b/third_party/translate-c/repo.bzl @@ -1,10 +1,9 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") def repo(): - http_archive( + new_git_repository( name = "translate-c", + remote = "https://codeberg.org/ziglang/translate-c", + commit = "5ac39f77661a216b75b195fe74ce7d0a04b33b7d", build_file = "//third_party/translate-c:translate-c.bazel", - sha256 = "506643ec817620025d341efd87169d90013aa369db28780e853cf18bff174326", - url = "https://codeberg.org/ziglang/translate-c/archive/d2be2f19ef7c9caa1561d38c96581ac79dd4c654.tar.gz", - strip_prefix = "translate-c", ) From 78546350af68b6404d5302295dee88e26d68d51e Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 20 Mar 2026 14:48:35 +0100 Subject: [PATCH 369/881] stdx: remove meta.Signature has proven unstable on Zig nightlies and mostly needed since std.Io anyway. --- stdx/BUILD.bazel | 1 - stdx/meta.zig | 40 ++++++++++++++++++------------ stdx/signature.zig | 59 --------------------------------------------- tools/zls.sh | 3 --- zml/meta.zig | 15 ++++++------ zml/module.zig | 2 +- zml/ops.zig | 8 +++--- zml/safetensors.zig | 2 +- zml/testing.zig | 4 +-- 9 files changed, 40 insertions(+), 94 deletions(-) delete mode 100644 stdx/signature.zig diff --git a/stdx/BUILD.bazel b/stdx/BUILD.bazel index f42a73fa..01448459 100644 --- a/stdx/BUILD.bazel +++ b/stdx/BUILD.bazel @@ -15,7 +15,6 @@ zig_library( "process.zig", "queue.zig", "segmented_list.zig", - "signature.zig", "time.zig", "unicode.zig", ], diff --git a/stdx/meta.zig b/stdx/meta.zig index 9a096122..afaaa917 100644 --- a/stdx/meta.zig +++ b/stdx/meta.zig @@ -2,8 +2,6 @@ const std = @import("std"); const debug = @import("debug.zig"); const compileError = debug.compileError; -pub const FnSignature = @import("signature.zig").FnSignature; -pub const Signature = @import("signature.zig").Signature; pub fn isStruct(comptime T: type) bool { return switch (@typeInfo(T)) { @@ -139,24 +137,34 @@ pub fn TupleRange(comptime T: type, comptime start: ?usize, comptime end: ?usize } pub fn FnParam(comptime func: anytype, comptime n: comptime_int) type { - return @typeInfo(@TypeOf(func)).@"fn".params[n].type orelse @compileError("anytype is not supported"); + const params = @typeInfo(@TypeOf(func)).@"fn".params; + if (n >= params.len) { + @compileError("param doesn't exist in func"); + } + return params[n].type orelse @compileError("anytype is not supported"); } -pub fn FnArgs(comptime func: anytype) type { - debug.assertComptime(!@typeInfo(@TypeOf(func)).@"fn".is_generic, "FnArgs expects non generic function, got: {}", .{@TypeOf(func)}); - return FnSignature(func, null).ArgsT; -} - -pub fn FnArgsWithHint(comptime func: anytype, ArgsT: type) type { - debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".is_generic, "FnArgsWithHint expects a generic function, got: {}", .{@TypeOf(func)}); - return FnSignature(func, ArgsT).ArgsT; -} - -pub fn FnResult(comptime func: anytype) type { +pub fn FnReturn(comptime func: anytype) type { return @typeInfo(@TypeOf(func)).@"fn".return_type orelse @compileError("anytype is not supported"); } -pub fn Head(Tuple: type) type { +pub fn FnReturnPayload(comptime func: anytype) type { + const rt = @typeInfo(@TypeOf(func)).@"fn".return_type orelse @compileError("anytype is not supported"); + return switch (@typeInfo(rt)) { + .error_union => |u| u.payload, + else => rt, + }; +} + +pub fn FnReturnErrorSet(comptime func: anytype) ?type { + const rt = @typeInfo(@TypeOf(func)).@"fn".return_type orelse @compileError("anytype is not supported"); + return switch (@typeInfo(rt)) { + .error_union => |u| u.error_set, + else => null, + }; +} + +pub fn Head(comptime Tuple: type) type { return switch (@typeInfo(Tuple)) { .@"struct" => |struct_info| { if (struct_info.fields.len == 0) @compileError("Can't tail empty tuple"); @@ -166,7 +174,7 @@ pub fn Head(Tuple: type) type { }; } -pub fn Tail(Tuple: type) type { +pub fn Tail(comptime Tuple: type) type { return switch (@typeInfo(Tuple)) { .@"struct" => |struct_info| { if (struct_info.fields.len == 0) @compileError("Can't tail empty tuple"); diff --git a/stdx/signature.zig b/stdx/signature.zig deleted file mode 100644 index 955a1887..00000000 --- a/stdx/signature.zig +++ /dev/null @@ -1,59 +0,0 @@ -const std = @import("std"); - -const compileError = @import("debug.zig").compileError; - -pub fn ArgsTuple(comptime funcT: anytype, comptime ArgsT: ?type) type { - const params = @typeInfo(funcT).@"fn".params; - if (params.len == 0) { - return @TypeOf(.{}); - } - - if (@typeInfo(funcT).@"fn".is_generic == false) { - return std.meta.ArgsTuple(funcT); - } - - const args = std.meta.fields(ArgsT orelse @compileError("generic function requires an explicit ArgsTuple")); - var tuple_types: [params.len]type = undefined; - if (params.len != args.len) { - compileError("function {} expected {} args, got {}", .{ funcT, params.len, args.len }); - } - inline for (params, args, 0..) |param, arg, i| { - if (param.type == null) { - tuple_types[i] = arg.type; - continue; - } - tuple_types[i] = param.type.?; - } - - return @Tuple(tuple_types); -} - -pub const Signature = struct { - Func: type, - FuncT: type, - ArgsT: type, - ReturnT: type, - ReturnPayloadT: type, - ReturnErrorSet: ?type, -}; - -pub fn FnSignature(comptime func: anytype, comptime argsT_: ?type) Signature { - const argsT = ArgsTuple(@TypeOf(func), argsT_); - const return_type = @TypeOf(@call(.auto, func, @as(argsT, undefined))); - return Signature{ - .Func = struct { - pub const Value = func; - }, - .FuncT = @TypeOf(func), - .ArgsT = argsT, - .ReturnT = return_type, - .ReturnPayloadT = switch (@typeInfo(return_type)) { - .error_union => |u| u.payload, - else => return_type, - }, - .ReturnErrorSet = switch (@typeInfo(return_type)) { - .error_union => |u| u.error_set, - else => null, - }, - }; -} diff --git a/tools/zls.sh b/tools/zls.sh index 4b59ccaf..2b83f7b6 100755 --- a/tools/zls.sh +++ b/tools/zls.sh @@ -1,7 +1,4 @@ #!/usr/bin/env bash - -set -euo pipefail - cd "$(dirname "${BASH_SOURCE[0]}")" cd "$(bazel info workspace)" exec bazel run -- @zml//:completion "${@}" diff --git a/zml/meta.zig b/zml/meta.zig index 1274761b..8fcd07ef 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -148,7 +148,7 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam // TODO: handle tuple to slice conversion // TODO: handle error bubbling up const From = FnParam(cb, 1); - const To = stdx.meta.FnResult(cb); + const To = stdx.meta.FnReturn(cb); const FromStruct = @TypeOf(from); const type_info_to_ptr = @typeInfo(@TypeOf(to)); if (type_info_to_ptr != .pointer) { @@ -438,7 +438,7 @@ test MapRestrict { /// Recursively visit the given struct and calls the callback for each K found. /// The `v` parameter must me a pointer, and tensor data need to be mutable if callbacks needs it. pub fn VisitReturn(comptime cb: anytype) type { - return if (stdx.meta.FnSignature(cb, null).ReturnErrorSet) |error_set| + return if (stdx.meta.FnReturnErrorSet(cb)) |error_set| error_set!void else void; @@ -453,7 +453,7 @@ pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) VisitReturn( } const ptr_info = type_info_v.pointer; const Child = ptr_info.child; - const can_error = stdx.meta.FnSignature(cb, null).ReturnErrorSet != null; + const can_error = stdx.meta.FnReturnErrorSet(cb) != null; const K, const mutating_cb = switch (@typeInfo(FnParam(cb, 1))) { .pointer => |info| .{ info.child, !info.is_const }, @@ -702,13 +702,13 @@ pub fn collectAlloc( func_ctx: _CollectCtx(func), allocator: std.mem.Allocator, obj: anytype, -) std.mem.Allocator.Error![]stdx.meta.FnSignature(func, null).ReturnT { +) std.mem.Allocator.Error![]stdx.meta.FnReturn(func) { stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".params.len <= 2, "zml.meta.collect expects a func with two arguments, got: {}", .{@TypeOf(func)}); const CollectAllocCtx = struct { func_ctx: _CollectCtx(func), allocator: std.mem.Allocator, - out: std.ArrayList(stdx.meta.FnSignature(func, null).ReturnT) = .empty, + out: std.ArrayList(stdx.meta.FnReturn(func)) = .empty, oom: bool = false, fn cb(ctx: *@This(), val: *const _CollectArg(func)) void { @@ -729,7 +729,7 @@ pub fn collectAlloc( /// Given a func(X) -> Y or a func(Ctx, X) -> Y, /// finds all X in the given object, and write the result of func(X) into a given slice. /// Asserts that the number of X found is equal to the slice len. -pub fn collectBuf(func: anytype, func_ctx: _CollectCtx(func), obj: anytype, out: []stdx.meta.FnResult(func)) void { +pub fn collectBuf(func: anytype, func_ctx: _CollectCtx(func), obj: anytype, out: []stdx.meta.FnReturn(func)) void { stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".params.len <= 2, "zml.meta.collectBuf expects a func with one or two arguments, got: {}", .{@TypeOf(func)}); const LocalContext = struct { func_ctx: _CollectCtx(func), @@ -773,8 +773,9 @@ pub fn Contains(Haystack: type, T: type) bool { return switch (@typeInfo(Haystack)) { .@"struct" => |info| { inline for (info.fields) |field| { - if (!field.is_comptime and Contains(field.type, T)) + if (!field.is_comptime and Contains(field.type, T)) { return true; + } } return false; }, diff --git a/zml/module.zig b/zml/module.zig index f12e3292..6afc2254 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -380,7 +380,7 @@ fn finalizeAttributeList(allocator_: std.mem.Allocator, mlir_ctx: *mlir.Context, return res; } -fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, args: stdx.meta.FnArgs(func)) !EmitMlirResult { +fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, args: std.meta.ArgsTuple(@TypeOf(func))) !EmitMlirResult { var arena = std.heap.ArenaAllocator.init(compilation_context.allocator); defer arena.deinit(); diff --git a/zml/ops.zig b/zml/ops.zig index a83157cc..3c558655 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -16,7 +16,7 @@ pub const ReduceArgs = struct { right: Tensor, }; -pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { +pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func: anytype, context: anytype) stdx.meta.FnReturn(func) { var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); defer arena.deinit(); @@ -119,7 +119,7 @@ pub const ReduceWindowOpts = struct { padding: []const [2]i64, }; -pub fn reduceWindow(inputs: anytype, inits: anytype, opts: ReduceWindowOpts, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { +pub fn reduceWindow(inputs: anytype, inits: anytype, opts: ReduceWindowOpts, comptime func: anytype, context: anytype) stdx.meta.FnReturn(func) { var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); defer arena.deinit(); @@ -411,7 +411,7 @@ pub fn scatter( comptime func: anytype, context: anytype, opts: Tensor.ScatterOpts, -) stdx.meta.FnResult(func) { +) stdx.meta.FnReturn(func) { var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); defer arena.deinit(); @@ -1199,7 +1199,7 @@ fn manualComputationInternal( } fn manualComputationReturnType(comptime body_fn: anytype) type { - const ReturnT = stdx.meta.FnResult(body_fn); + const ReturnT = stdx.meta.FnReturn(body_fn); if (ReturnT == void or ReturnT == Tensor) return ReturnT; return switch (@typeInfo(ReturnT)) { diff --git a/zml/safetensors.zig b/zml/safetensors.zig index bb14c334..272a864c 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -507,7 +507,7 @@ fn parseSafetensorsIndex( const map_entry = try safetensors_index.map.getOrPut(arena_allocator, filename); if (!map_entry.found_existing) { - map_entry.value_ptr.* = .{}; + map_entry.value_ptr.* = .empty; } try map_entry.value_ptr.append(arena_allocator, try arena_allocator.dupe(u8, weight_name)); diff --git a/zml/testing.zig b/zml/testing.zig index 8fbc9603..d3895fcb 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -254,7 +254,7 @@ fn BufferizedWithArgs(comptime T: type) type { /// Automatically calls the executable with the given arguments, taking care of arguments and results allocation. /// This helper can be used in tests to make the code a bit less verbose. /// It doesn't handle pointers in inputs/outputs structs. -pub fn autoCall(allocator: std.mem.Allocator, io: std.Io, exe: *const zml.exe.Exe, func: anytype, inputs: zml.Bufferized(stdx.meta.FnArgs(func))) !zml.Bufferized(stdx.meta.FnResult(func)) { +pub fn autoCall(allocator: std.mem.Allocator, io: std.Io, exe: *const zml.exe.Exe, func: anytype, inputs: zml.Bufferized(std.meta.ArgsTuple(func))) !zml.Bufferized(stdx.meta.FnReturn(func)) { var args = try exe.args(allocator); defer args.deinit(allocator); @@ -264,7 +264,7 @@ pub fn autoCall(allocator: std.mem.Allocator, io: std.Io, exe: *const zml.exe.Ex args.set(inputs); exe.callOpts(io, args, &results, .{ .wait = true }); - var output: zml.Bufferized(stdx.meta.FnResult(func)) = undefined; + var output: zml.Bufferized(stdx.meta.FnReturn(func)) = undefined; results.fill(.{&output}); return output; From b065d976d31a3eb8f41b39e0a3add535cf23200e Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 20 Mar 2026 14:49:31 +0100 Subject: [PATCH 370/881] third_party/zls: fix completion on single dep targets take the opportunity to bump ZLS to a version compatible with our current nighly. --- third_party/zls/zls_toolchain.bzl | 8 ++++---- third_party/zls/zls_write_build_config.bzl | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/third_party/zls/zls_toolchain.bzl b/third_party/zls/zls_toolchain.bzl index d83e8f1a..f19bd0d1 100644 --- a/third_party/zls/zls_toolchain.bzl +++ b/third_party/zls/zls_toolchain.bzl @@ -2,7 +2,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -_VERSION = "0.16.0-dev.215+0e289f5c" +_VERSION = "0.16.0-dev.254+695a1d5c" _ZLS_PLATFORMS = { "x86_64-linux": struct( @@ -26,9 +26,9 @@ _ZLS_PLATFORMS = { } _ZLS_VERSIONS = { - "x86_64-linux": "026a0ce9ea06b276319210509fa2a2edd7acf4e68da83b1a704c06ae26671548", - "x86_64-macos": "061845ec346620b914063763e8e39f9a374d2817c4ae976ea3eeb577a6e1cf32", - "aarch64-macos": "fb711e068e2ec77b3979458ff72f18311fa7df74644a8cf43985c0f88cae5c66", + "x86_64-linux": "b02f9623a0cec2475c26268f12f49f48b81e9c41f2a8a6628806452685cd1013", + "x86_64-macos": "4cab46f56a9685822a5ad3665af3a9b11d2556313defa634b5e8b85cac8151cb", + "aarch64-macos": "8e4ef0603042a2081ae02672ad72f271d8c840746bbfa0ed9934c6b72ea10d7b", } ZlsToolchainInfo = provider( diff --git a/third_party/zls/zls_write_build_config.bzl b/third_party/zls/zls_write_build_config.bzl index 3f3ed65a..fc4a04f9 100644 --- a/third_party/zls/zls_write_build_config.bzl +++ b/third_party/zls/zls_write_build_config.bzl @@ -29,10 +29,8 @@ def _zls_construct_zig_module_info_impl(target, ctx): elif CcInfo in dep: cdeps.append(dep[CcInfo]) - root_module_is_only_dep = len(ctx.rule.attr.deps) == 1 and ZigModuleInfo in ctx.rule.attr.deps[0] - if root_module_is_only_dep: - root_module = ctx.rule.attr.deps[0][ZigModuleInfo] - else: + root_module = None + if ctx.rule.attr.main: root_module = zig_module_info( name = ctx.rule.attr.name, canonical_name = target.label.name, @@ -43,6 +41,8 @@ def _zls_construct_zig_module_info_impl(target, ctx): cdeps = cdeps, zigopts = [], ) + else: + root_module = zdeps[0] return [ root_module, From 5e2bf7fc1644982beeee00929cb760fa16319793 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 20 Mar 2026 15:02:24 +0100 Subject: [PATCH 371/881] workspace: bump buildifier prebuilt --- MODULE.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index 247533c8..fd201efd 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -26,7 +26,7 @@ bazel_dep(name = "llvm", version = "0.6.8") bazel_dep(name = "jsoncpp", version = "1.9.6", repo_name = "jsoncpp_git") bazel_dep(name = "with_cfg.bzl", version = "0.14.4") -bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2", dev_dependency = True) +bazel_dep(name = "buildifier_prebuilt", version = "8.5.1", dev_dependency = True) # BSDTar toolchains tar_toolchains = use_extension("@tar.bzl//tar:extensions.bzl", "toolchains") From 90fccf5f5a410808468d5cb7c3d75ddf98998fd6 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 23 Mar 2026 13:07:42 +0100 Subject: [PATCH 372/881] Proper sharding for paged `.cuda_fa3` (#423) --- MODULE.bazel.lock | 1164 +---------------------------- zml/attention/flashattn.zig | 175 +++-- zml/attention/paged_attention.zig | 2 + 3 files changed, 131 insertions(+), 1210 deletions(-) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 59b2af9c..33055bbb 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -72,8 +72,8 @@ "https://bcr.bazel.build/modules/bazel_skylib/1.8.2/MODULE.bazel": "69ad6927098316848b34a9142bcc975e018ba27f08c4ff403f50c1b6e646ca67", "https://bcr.bazel.build/modules/bazel_skylib/1.9.0/MODULE.bazel": "72997b29dfd95c3fa0d0c48322d05590418edef451f8db8db5509c57875fb4b7", "https://bcr.bazel.build/modules/bazel_skylib/1.9.0/source.json": "7ad77c1e8c1b84222d9b3f3cae016a76639435744c19330b0b37c0a3c9da7dc0", - "https://bcr.bazel.build/modules/buildifier_prebuilt/8.2.0.2/MODULE.bazel": "a9b689711d5b69f9db741649b218c119b9fdf82924ba390415037e09798edd03", - "https://bcr.bazel.build/modules/buildifier_prebuilt/8.2.0.2/source.json": "51eb0a4b38aaaeab7fa64361576d616c4d8bfd0f17a0a10184aeab7084d79f8e", + "https://bcr.bazel.build/modules/buildifier_prebuilt/8.5.1/MODULE.bazel": "77f2a1958d1d07376dd3ce3ae16540f2c1b01921c1fd21930827271260e75a66", + "https://bcr.bazel.build/modules/buildifier_prebuilt/8.5.1/source.json": "ae9f3d9dc7bec033976cf47165a78788bebad2b5c272241063ae31bad8664fd4", "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.1/MODULE.bazel": "cdf8cbe5ee750db04b78878c9633cc76e80dcf4416cbe982ac3a9222f80713c8", @@ -258,360 +258,6 @@ }, "selectedYankedVersions": {}, "moduleExtensions": { - "@@aspect_rules_py+//py:extensions.bzl%py_tools": { - "general": { - "bzlTransitiveDigest": "GLVIk55uU2jb460gpL1FFbE/tBFqzMxRfta5QXYfjQo=", - "usagesDigest": "9BiP3ASHlGl4doGhzXJwK17taNzv/q6x/IZ4Hb6LxNY=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "rules_py_tools.darwin_amd64": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "rules_py_tools.darwin_arm64": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "rules_py_tools.linux_amd64": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "rules_py_tools.linux_arm64": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "rules_py_tools": { - "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%toolchains_repo", - "attributes": { - "user_repository_name": "rules_py_tools" - } - }, - "rules_py_pex_2_3_1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "urls": [ - "https://files.pythonhosted.org/packages/e7/d0/fbda2a4d41d62d86ce53f5ae4fbaaee8c34070f75bb7ca009090510ae874/pex-2.3.1-py2.py3-none-any.whl" - ], - "sha256": "64692a5bf6f298403aab930d22f0d836ae4736c5bc820e262e9092fe8c56f830", - "downloaded_file_path": "pex-2.3.1-py2.py3-none-any.whl" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "aspect_rules_py+", - "aspect_tools_telemetry_report", - "aspect_tools_telemetry++telemetry+aspect_tools_telemetry_report" - ], - [ - "aspect_rules_py+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "aspect_rules_py+", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@aspect_tools_telemetry+//:extension.bzl%telemetry": { - "general": { - "bzlTransitiveDigest": "MePriaXmQNSqUfE+YQvEtzBc2bU1mjITINIffFiZYgo=", - "usagesDigest": "TU1quwRKpvAEOEF35/IVh4NeoIS2hayv4fGp8f5RXoY=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "aspect_tools_telemetry_report": { - "repoRuleId": "@@aspect_tools_telemetry+//:extension.bzl%tel_repository", - "attributes": { - "deps": { - "aspect_rules_py": "1.8.4", - "rules_rs": "0.0.44", - "aspect_tools_telemetry": "0.3.3" - } - } - } - }, - "recordedRepoMappingEntries": [ - [ - "aspect_tools_telemetry+", - "bazel_lib", - "bazel_lib+" - ], - [ - "aspect_tools_telemetry+", - "bazel_skylib", - "bazel_skylib+" - ] - ] - } - }, - "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { - "general": { - "bzlTransitiveDigest": "AEsQvKwdstptzt98AO1bJYfoWpPUn2upx53ra6o/Rc8=", - "usagesDigest": "QI2z8ZUR+mqtbwsf2fLqYdJAkPOHdOV+tF2yVAUgRzw=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "com_github_jetbrains_kotlin_git": { - "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:compiler.bzl%kotlin_compiler_git_repository", - "attributes": { - "urls": [ - "https://github.com/JetBrains/kotlin/releases/download/v1.9.23/kotlin-compiler-1.9.23.zip" - ], - "sha256": "93137d3aab9afa9b27cb06a824c2324195c6b6f6179d8a8653f440f5bd58be88" - } - }, - "com_github_jetbrains_kotlin": { - "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:compiler.bzl%kotlin_capabilities_repository", - "attributes": { - "git_repository_name": "com_github_jetbrains_kotlin_git", - "compiler_version": "1.9.23" - } - }, - "com_github_google_ksp": { - "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:ksp.bzl%ksp_compiler_plugin_repository", - "attributes": { - "urls": [ - "https://github.com/google/ksp/releases/download/1.9.23-1.0.20/artifacts.zip" - ], - "sha256": "ee0618755913ef7fd6511288a232e8fad24838b9af6ea73972a76e81053c8c2d", - "strip_version": "1.9.23-1.0.20" - } - }, - "com_github_pinterest_ktlint": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "sha256": "01b2e0ef893383a50dbeb13970fe7fa3be36ca3e83259e01649945b09d736985", - "urls": [ - "https://github.com/pinterest/ktlint/releases/download/1.3.0/ktlint" - ], - "executable": true - } - }, - "rules_android": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "sha256": "cd06d15dd8bb59926e4d65f9003bfc20f9da4b2519985c27e190cddc8b7a7806", - "strip_prefix": "rules_android-0.1.1", - "urls": [ - "https://github.com/bazelbuild/rules_android/archive/v0.1.1.zip" - ] - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_kotlin+", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@rules_oci+//oci:extensions.bzl%oci": { - "general": { - "bzlTransitiveDigest": "twSeB3VPTVkayY0uwqEzDzal5MbnZX45iC6jdheZQRo=", - "usagesDigest": "IIGPr69MT/mSUo6fNUffrIiU31EqGjlEYm5qFmNjmUc=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "distroless_cc_debian12_linux_amd64": { - "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_pull", - "attributes": { - "www_authenticate_challenges": {}, - "scheme": "https", - "registry": "gcr.io", - "repository": "distroless/cc-debian12", - "identifier": "sha256:eccec5274132c1be0ce5d2c8e6fe41033e64af5e987ccee9007826e4c012069d", - "platform": "linux/amd64", - "target_name": "distroless_cc_debian12_linux_amd64", - "bazel_tags": [] - } - }, - "distroless_cc_debian12": { - "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_alias", - "attributes": { - "target_name": "distroless_cc_debian12", - "www_authenticate_challenges": {}, - "scheme": "https", - "registry": "gcr.io", - "repository": "distroless/cc-debian12", - "identifier": "sha256:eccec5274132c1be0ce5d2c8e6fe41033e64af5e987ccee9007826e4c012069d", - "platforms": { - "@@platforms//cpu:x86_64": "@distroless_cc_debian12_linux_amd64" - }, - "bzlmod_repository": "distroless_cc_debian12", - "reproducible": true - } - }, - "oci_crane_darwin_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "darwin_amd64", - "crane_version": "v0.18.0" - } - }, - "oci_crane_darwin_arm64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "darwin_arm64", - "crane_version": "v0.18.0" - } - }, - "oci_crane_linux_arm64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "linux_arm64", - "crane_version": "v0.18.0" - } - }, - "oci_crane_linux_armv6": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "linux_armv6", - "crane_version": "v0.18.0" - } - }, - "oci_crane_linux_i386": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "linux_i386", - "crane_version": "v0.18.0" - } - }, - "oci_crane_linux_s390x": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "linux_s390x", - "crane_version": "v0.18.0" - } - }, - "oci_crane_linux_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "linux_amd64", - "crane_version": "v0.18.0" - } - }, - "oci_crane_windows_armv6": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "windows_armv6", - "crane_version": "v0.18.0" - } - }, - "oci_crane_windows_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "windows_amd64", - "crane_version": "v0.18.0" - } - }, - "oci_crane_toolchains": { - "repoRuleId": "@@rules_oci+//oci/private:toolchains_repo.bzl%toolchains_repo", - "attributes": { - "toolchain_type": "@rules_oci//oci:crane_toolchain_type", - "toolchain": "@oci_crane_{platform}//:crane_toolchain" - } - }, - "oci_regctl_darwin_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "darwin_amd64" - } - }, - "oci_regctl_darwin_arm64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "darwin_arm64" - } - }, - "oci_regctl_linux_arm64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "linux_arm64" - } - }, - "oci_regctl_linux_s390x": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "linux_s390x" - } - }, - "oci_regctl_linux_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "linux_amd64" - } - }, - "oci_regctl_windows_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "windows_amd64" - } - }, - "oci_regctl_toolchains": { - "repoRuleId": "@@rules_oci+//oci/private:toolchains_repo.bzl%toolchains_repo", - "attributes": { - "toolchain_type": "@rules_oci//oci:regctl_toolchain_type", - "toolchain": "@oci_regctl_{platform}//:regctl_toolchain" - } - } - }, - "moduleExtensionMetadata": { - "explicitRootModuleDirectDeps": [ - "distroless_cc_debian12", - "distroless_cc_debian12_linux_amd64" - ], - "explicitRootModuleDirectDevDeps": [], - "useAllRepos": "NO", - "reproducible": false - }, - "recordedRepoMappingEntries": [ - [ - "aspect_bazel_lib+", - "bazel_tools", - "bazel_tools" - ], - [ - "bazel_features+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_oci+", - "aspect_bazel_lib", - "aspect_bazel_lib+" - ], - [ - "rules_oci+", - "bazel_features", - "bazel_features+" - ], - [ - "rules_oci+", - "bazel_skylib", - "bazel_skylib+" - ] - ] - } - }, "@@rules_python+//python/extensions:config.bzl%config": { "general": { "bzlTransitiveDigest": "X49Lbeli5+XwNYRk+isOvNKFMIF8g/rnxL3Fdlwrffc=", @@ -846,811 +492,7 @@ ] ] } - }, - "@@rules_python+//python/uv:uv.bzl%uv": { - "general": { - "bzlTransitiveDigest": "CJBWuaTerWIqLpLWl77vjWFD1RRpXfzC2UGMwEk6eVg=", - "usagesDigest": "zIGk4Tqt5ZS3xJ783QnGoVqaEhMIIzUMmK0DHMRGugg=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "uv_0_9_30_aarch64_apple_darwin": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", - "attributes": { - "version": "0.9.30", - "platform": "aarch64-apple-darwin", - "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-aarch64-apple-darwin.tar.gz" - ], - "sha256": "03a5d9ec7f7d588446b2ec226d13ff6300055e55365eca8f3fab39f342b0e805" - } - }, - "uv_0_9_30_aarch64_unknown_linux_gnu": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", - "attributes": { - "version": "0.9.30", - "platform": "aarch64-unknown-linux-gnu", - "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-aarch64-unknown-linux-gnu.tar.gz" - ], - "sha256": "6aadf3c71600d594e16dabf382cc15282ead4c5ca768599b6bcb43c5004d9aa8" - } - }, - "uv_0_9_30_powerpc64_unknown_linux_gnu": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", - "attributes": { - "version": "0.9.30", - "platform": "powerpc64-unknown-linux-gnu", - "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-powerpc64-unknown-linux-gnu.tar.gz" - ], - "sha256": "ac4cd1a021462885932f6023b005a4835cca4c72bb60dec186ee2be4b60dca6f" - } - }, - "uv_0_9_30_powerpc64le_unknown_linux_gnu": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", - "attributes": { - "version": "0.9.30", - "platform": "powerpc64le-unknown-linux-gnu", - "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-powerpc64le-unknown-linux-gnu.tar.gz" - ], - "sha256": "73b8cbc560c6b2fa205358365d4e174abdf50cfcf57dc36a447572c56eba5ae4" - } - }, - "uv_0_9_30_s390x_unknown_linux_gnu": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", - "attributes": { - "version": "0.9.30", - "platform": "s390x-unknown-linux-gnu", - "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-s390x-unknown-linux-gnu.tar.gz" - ], - "sha256": "b35975bb9e5c2c418b428d0316cc6e3c7a6eff710c69212be14005c192f54516" - } - }, - "uv_0_9_30_x86_64_apple_darwin": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", - "attributes": { - "version": "0.9.30", - "platform": "x86_64-apple-darwin", - "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-x86_64-apple-darwin.tar.gz" - ], - "sha256": "ce069bf750567e9a4a31d6e285d1eae75d444d8a281409b641235903943b7681" - } - }, - "uv_0_9_30_x86_64_pc_windows_msvc": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", - "attributes": { - "version": "0.9.30", - "platform": "x86_64-pc-windows-msvc", - "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-x86_64-pc-windows-msvc.zip" - ], - "sha256": "875981be7908295937dee09532bb66d576986d4f223259e171b0c767c885897a" - } - }, - "uv_0_9_30_x86_64_unknown_linux_gnu": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", - "attributes": { - "version": "0.9.30", - "platform": "x86_64-unknown-linux-gnu", - "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-x86_64-unknown-linux-gnu.tar.gz" - ], - "sha256": "8b3762374972daa7a74bbc6896cc73229ca69a07403dd9f9ea3805a51ffd7582" - } - }, - "uv": { - "repoRuleId": "@@rules_python+//python/uv/private:uv_toolchains_repo.bzl%uv_toolchains_repo", - "attributes": { - "toolchain_type": "'@@rules_python+//python/uv:uv_toolchain_type'", - "toolchain_names": [ - "0_9_30_aarch64_apple_darwin", - "0_9_30_aarch64_unknown_linux_gnu", - "0_9_30_powerpc64_unknown_linux_gnu", - "0_9_30_powerpc64le_unknown_linux_gnu", - "0_9_30_s390x_unknown_linux_gnu", - "0_9_30_x86_64_apple_darwin", - "0_9_30_x86_64_pc_windows_msvc", - "0_9_30_x86_64_unknown_linux_gnu" - ], - "toolchain_implementations": { - "0_9_30_aarch64_apple_darwin": "@uv_0_9_30_aarch64_apple_darwin//:uv_toolchain", - "0_9_30_aarch64_unknown_linux_gnu": "@uv_0_9_30_aarch64_unknown_linux_gnu//:uv_toolchain", - "0_9_30_powerpc64_unknown_linux_gnu": "@uv_0_9_30_powerpc64_unknown_linux_gnu//:uv_toolchain", - "0_9_30_powerpc64le_unknown_linux_gnu": "@uv_0_9_30_powerpc64le_unknown_linux_gnu//:uv_toolchain", - "0_9_30_s390x_unknown_linux_gnu": "@uv_0_9_30_s390x_unknown_linux_gnu//:uv_toolchain", - "0_9_30_x86_64_apple_darwin": "@uv_0_9_30_x86_64_apple_darwin//:uv_toolchain", - "0_9_30_x86_64_pc_windows_msvc": "@uv_0_9_30_x86_64_pc_windows_msvc//:uv_toolchain", - "0_9_30_x86_64_unknown_linux_gnu": "@uv_0_9_30_x86_64_unknown_linux_gnu//:uv_toolchain" - }, - "toolchain_compatible_with": { - "0_9_30_aarch64_apple_darwin": [ - "'@@platforms//os:macos'", - "'@@platforms//cpu:aarch64'" - ], - "0_9_30_aarch64_unknown_linux_gnu": [ - "'@@platforms//os:linux'", - "'@@platforms//cpu:aarch64'" - ], - "0_9_30_powerpc64_unknown_linux_gnu": [ - "'@@platforms//os:linux'", - "'@@platforms//cpu:ppc'" - ], - "0_9_30_powerpc64le_unknown_linux_gnu": [ - "'@@platforms//os:linux'", - "'@@platforms//cpu:ppc64le'" - ], - "0_9_30_s390x_unknown_linux_gnu": [ - "'@@platforms//os:linux'", - "'@@platforms//cpu:s390x'" - ], - "0_9_30_x86_64_apple_darwin": [ - "'@@platforms//os:macos'", - "'@@platforms//cpu:x86_64'" - ], - "0_9_30_x86_64_pc_windows_msvc": [ - "'@@platforms//os:windows'", - "'@@platforms//cpu:x86_64'" - ], - "0_9_30_x86_64_unknown_linux_gnu": [ - "'@@platforms//os:linux'", - "'@@platforms//cpu:x86_64'" - ] - }, - "toolchain_target_settings": {} - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_python+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_python+", - "platforms", - "platforms" - ] - ] - } - }, - "@@rules_zig+//zig:extensions.bzl%zig": { - "general": { - "bzlTransitiveDigest": "r7tCqbjlVKc7XEXV8Hiak2ElFfkjvDdMf8Ym+7zxukU=", - "usagesDigest": "lqvJAPUaAfUjthqXXAa4LwhcuZ2d3CNerCqKcLkkNPo=", - "recordedFileInputs": { - "@@//bazel/zig_index.json": "809b545c21f13c11c4587dccbfe81de2b767a292325c25ffb785b03c601bbc2e", - "@@rules_zig+//zig/private/versions.json": "6eb85ebaee72c4e6fbc82d8ad73f69e0899d839b70a7e19caa792ab6a5fa0c36" - }, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-linux": { - "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", - "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.2722+f16eb18ce.tar.xz", - "mirrors": [ - "https://pkg.machengine.org/zig", - "https://zigmirror.hryx.net/zig", - "https://zig.linus.dev/zig", - "https://zig.squirl.dev", - "https://zig.florent.dev", - "https://zig.mirror.mschae23.de/zig", - "https://zigmirror.meox.dev", - "https://ziglang.freetls.fastly.net", - "https://zig.tilok.dev", - "https://zig-mirror.tsimnet.eu/zig", - "https://zig.karearl.com/zig", - "https://pkg.earth/zig", - "https://fs.liujiacai.net/zigbuilds", - "https://ziglang.org/builds/" - ], - "sha256": "03725d297a2e2a8331d774eb601163af42ec61979b82753d41b2c460eefab1be", - "zig_version": "0.16.0-dev.2722+f16eb18ce", - "platform": "aarch64-linux", - "translate_c": "@@+non_module_deps+translate-c//:translate-c" - } - }, - "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-macos": { - "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", - "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.2722+f16eb18ce.tar.xz", - "mirrors": [ - "https://pkg.machengine.org/zig", - "https://zigmirror.hryx.net/zig", - "https://zig.linus.dev/zig", - "https://zig.squirl.dev", - "https://zig.florent.dev", - "https://zig.mirror.mschae23.de/zig", - "https://zigmirror.meox.dev", - "https://ziglang.freetls.fastly.net", - "https://zig.tilok.dev", - "https://zig-mirror.tsimnet.eu/zig", - "https://zig.karearl.com/zig", - "https://pkg.earth/zig", - "https://fs.liujiacai.net/zigbuilds", - "https://ziglang.org/builds/" - ], - "sha256": "268aa8342e8ef539693e88e1d0982361b1d5dc4a080949264f2d5a14e5eb83e0", - "zig_version": "0.16.0-dev.2722+f16eb18ce", - "platform": "aarch64-macos", - "translate_c": "@@+non_module_deps+translate-c//:translate-c" - } - }, - "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-windows": { - "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", - "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.2722+f16eb18ce.zip", - "mirrors": [ - "https://pkg.machengine.org/zig", - "https://zigmirror.hryx.net/zig", - "https://zig.linus.dev/zig", - "https://zig.squirl.dev", - "https://zig.florent.dev", - "https://zig.mirror.mschae23.de/zig", - "https://zigmirror.meox.dev", - "https://ziglang.freetls.fastly.net", - "https://zig.tilok.dev", - "https://zig-mirror.tsimnet.eu/zig", - "https://zig.karearl.com/zig", - "https://pkg.earth/zig", - "https://fs.liujiacai.net/zigbuilds", - "https://ziglang.org/builds/" - ], - "sha256": "43daaad311bc3b06a733a8a87d2c3180214d62db2067df1591b921826de453ea", - "zig_version": "0.16.0-dev.2722+f16eb18ce", - "platform": "aarch64-windows", - "translate_c": "@@+non_module_deps+translate-c//:translate-c" - } - }, - "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-linux": { - "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", - "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.2722+f16eb18ce.tar.xz", - "mirrors": [ - "https://pkg.machengine.org/zig", - "https://zigmirror.hryx.net/zig", - "https://zig.linus.dev/zig", - "https://zig.squirl.dev", - "https://zig.florent.dev", - "https://zig.mirror.mschae23.de/zig", - "https://zigmirror.meox.dev", - "https://ziglang.freetls.fastly.net", - "https://zig.tilok.dev", - "https://zig-mirror.tsimnet.eu/zig", - "https://zig.karearl.com/zig", - "https://pkg.earth/zig", - "https://fs.liujiacai.net/zigbuilds", - "https://ziglang.org/builds/" - ], - "sha256": "b0fe8fd996b4974cd80fc0967460b13e3a6e3105c4bec19ea0566bd47d73ad81", - "zig_version": "0.16.0-dev.2722+f16eb18ce", - "platform": "x86_64-linux", - "translate_c": "@@+non_module_deps+translate-c//:translate-c" - } - }, - "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-macos": { - "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", - "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.2722+f16eb18ce.tar.xz", - "mirrors": [ - "https://pkg.machengine.org/zig", - "https://zigmirror.hryx.net/zig", - "https://zig.linus.dev/zig", - "https://zig.squirl.dev", - "https://zig.florent.dev", - "https://zig.mirror.mschae23.de/zig", - "https://zigmirror.meox.dev", - "https://ziglang.freetls.fastly.net", - "https://zig.tilok.dev", - "https://zig-mirror.tsimnet.eu/zig", - "https://zig.karearl.com/zig", - "https://pkg.earth/zig", - "https://fs.liujiacai.net/zigbuilds", - "https://ziglang.org/builds/" - ], - "sha256": "8e09fb9e90c0566629dfa52bce433e886f4cf7e2cd9985a358d7b8c473c4e9ce", - "zig_version": "0.16.0-dev.2722+f16eb18ce", - "platform": "x86_64-macos", - "translate_c": "@@+non_module_deps+translate-c//:translate-c" - } - }, - "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-windows": { - "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", - "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.2722+f16eb18ce.zip", - "mirrors": [ - "https://pkg.machengine.org/zig", - "https://zigmirror.hryx.net/zig", - "https://zig.linus.dev/zig", - "https://zig.squirl.dev", - "https://zig.florent.dev", - "https://zig.mirror.mschae23.de/zig", - "https://zigmirror.meox.dev", - "https://ziglang.freetls.fastly.net", - "https://zig.tilok.dev", - "https://zig-mirror.tsimnet.eu/zig", - "https://zig.karearl.com/zig", - "https://pkg.earth/zig", - "https://fs.liujiacai.net/zigbuilds", - "https://ziglang.org/builds/" - ], - "sha256": "0f92e7234ed1d9608444487ca8fb928bd38da79dea744acf6c7a8fa0c440da6a", - "zig_version": "0.16.0-dev.2722+f16eb18ce", - "platform": "x86_64-windows", - "translate_c": "@@+non_module_deps+translate-c//:translate-c" - } - }, - "zig_toolchains": { - "repoRuleId": "@@rules_zig+//zig/private/repo:toolchains_repo.bzl%toolchains_repo", - "attributes": { - "names": [ - "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-linux", - "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-macos", - "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-windows", - "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-linux", - "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-macos", - "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-windows" - ], - "labels": [ - "@zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-linux//:zig_toolchain", - "@zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-macos//:zig_toolchain", - "@zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-windows//:zig_toolchain", - "@zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-linux//:zig_toolchain", - "@zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-macos//:zig_toolchain", - "@zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-windows//:zig_toolchain" - ], - "zig_versions": [ - "0.16.0-dev.2722+f16eb18ce", - "0.16.0-dev.2722+f16eb18ce", - "0.16.0-dev.2722+f16eb18ce", - "0.16.0-dev.2722+f16eb18ce", - "0.16.0-dev.2722+f16eb18ce", - "0.16.0-dev.2722+f16eb18ce" - ], - "exec_lengths": [ - 2, - 2, - 2, - 2, - 2, - 2 - ], - "exec_constraints": [ - "@platforms//os:linux", - "@platforms//cpu:aarch64", - "@platforms//os:macos", - "@platforms//cpu:aarch64", - "@platforms//os:windows", - "@platforms//cpu:aarch64", - "@platforms//os:linux", - "@platforms//cpu:x86_64", - "@platforms//os:macos", - "@platforms//cpu:x86_64", - "@platforms//os:windows", - "@platforms//cpu:x86_64" - ] - } - } - }, - "recordedRepoMappingEntries": [ - [ - "", - "translate-c", - "+non_module_deps+translate-c" - ], - [ - "rules_zig+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "rules_zig+", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@yq.bzl+//yq:extensions.bzl%yq": { - "general": { - "bzlTransitiveDigest": "iWlctm1cJYZLESuQEBygCQc6adq+oIRMb4Dmk0rYxkA=", - "usagesDigest": "J3TR36exv3ePI9O2y+uqrzTDmT6s6ADkTsROB++nm6I=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "yq_darwin_amd64": { - "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", - "attributes": { - "platform": "darwin_amd64", - "version": "4.45.1" - } - }, - "yq_darwin_arm64": { - "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", - "attributes": { - "platform": "darwin_arm64", - "version": "4.45.1" - } - }, - "yq_linux_amd64": { - "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", - "attributes": { - "platform": "linux_amd64", - "version": "4.45.1" - } - }, - "yq_linux_arm64": { - "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", - "attributes": { - "platform": "linux_arm64", - "version": "4.45.1" - } - }, - "yq_linux_s390x": { - "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", - "attributes": { - "platform": "linux_s390x", - "version": "4.45.1" - } - }, - "yq_linux_riscv64": { - "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", - "attributes": { - "platform": "linux_riscv64", - "version": "4.45.1" - } - }, - "yq_linux_ppc64le": { - "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", - "attributes": { - "platform": "linux_ppc64le", - "version": "4.45.1" - } - }, - "yq_windows_amd64": { - "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", - "attributes": { - "platform": "windows_amd64", - "version": "4.45.1" - } - }, - "yq_toolchains": { - "repoRuleId": "@@yq.bzl+//yq/toolchain:toolchain.bzl%yq_toolchains_repo", - "attributes": { - "user_repository_name": "yq" - } - } - }, - "recordedRepoMappingEntries": [] - } } }, - "facts": { - "@@rules_rs+//rs:extensions.bzl%crate": { - "addr2line_0.25.1": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"backtrace\",\"req\":\"^0.3.13\"},{\"features\":[\"wrap_help\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4.3.21\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"cpp_demangle\",\"optional\":true,\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.7.0\"},{\"default_features\":false,\"name\":\"fallible-iterator\",\"optional\":true,\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"findshlibs\",\"req\":\"^0.10\"},{\"default_features\":false,\"features\":[\"read\"],\"name\":\"gimli\",\"req\":\"^0.32.0\"},{\"kind\":\"dev\",\"name\":\"libtest-mimic\",\"req\":\"^0.8.1\"},{\"name\":\"memmap2\",\"optional\":true,\"req\":\"^0.9.4\"},{\"default_features\":false,\"features\":[\"read\",\"compression\"],\"name\":\"object\",\"optional\":true,\"req\":\"^0.37.0\"},{\"name\":\"rustc-demangle\",\"optional\":true,\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"typed-arena\",\"optional\":true,\"req\":\"^2\"}],\"features\":{\"all\":[\"bin\",\"wasm\"],\"bin\":[\"loader\",\"rustc-demangle\",\"cpp_demangle\",\"fallible-iterator\",\"smallvec\",\"dep:clap\"],\"cargo-all\":[],\"default\":[\"rustc-demangle\",\"cpp_demangle\",\"loader\",\"fallible-iterator\",\"smallvec\"],\"loader\":[\"std\",\"dep:object\",\"dep:memmap2\",\"dep:typed-arena\"],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"gimli/rustc-dep-of-std\"],\"std\":[\"gimli/std\"],\"wasm\":[\"object/wasm\"]}}", - "adler2_2.0.1": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"std\"],\"rustc-dep-of-std\":[\"core\"],\"std\":[]}}", - "ahash_0.8.12": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1.0\"},{\"name\":\"const-random\",\"optional\":true,\"req\":\"^0.1.17\"},{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.2\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.5\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2.1\"},{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"hashbrown\",\"req\":\"^0.14.3\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.2\"},{\"kind\":\"dev\",\"name\":\"no-panic\",\"req\":\"^0.1.10\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"once_cell\",\"req\":\"^1.18.0\",\"target\":\"cfg(not(all(target_arch = \\\"arm\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"pcg-mwc\",\"req\":\"^0.2.1\"},{\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"seahash\",\"req\":\"^4.0\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.117\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.59\"},{\"kind\":\"dev\",\"name\":\"smallvec\",\"req\":\"^1.13.1\"},{\"kind\":\"build\",\"name\":\"version_check\",\"req\":\"^0.9.4\"},{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"zerocopy\",\"req\":\"^0.8.24\"}],\"features\":{\"atomic-polyfill\":[\"dep:portable-atomic\",\"once_cell/critical-section\"],\"compile-time-rng\":[\"const-random\"],\"default\":[\"std\",\"runtime-rng\"],\"nightly-arm-aes\":[],\"no-rng\":[],\"runtime-rng\":[\"getrandom\"],\"std\":[]}}", - "aho-corasick_1.1.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.17\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.4.0\"}],\"features\":{\"default\":[\"std\",\"perf-literal\"],\"logging\":[\"dep:log\"],\"perf-literal\":[\"dep:memchr\"],\"std\":[\"memchr?/std\"]}}", - "aho-corasick_1.1.4": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.17\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.4.0\"}],\"features\":{\"default\":[\"std\",\"perf-literal\"],\"logging\":[\"dep:log\"],\"perf-literal\":[\"dep:memchr\"],\"std\":[\"memchr?/std\"]}}", - "anyhow_1.0.102": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.6\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0\"},{\"kind\":\"dev\",\"name\":\"thiserror\",\"req\":\"^2\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"}],\"features\":{\"backtrace\":[],\"default\":[\"std\"],\"std\":[]}}", - "async-stream-impl_0.3.6": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"full\",\"visit-mut\"],\"name\":\"syn\",\"req\":\"^2.0.2\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"}],\"features\":{}}", - "async-stream_0.3.6": "{\"dependencies\":[{\"name\":\"async-stream-impl\",\"req\":\"=0.3.6\"},{\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1\"}],\"features\":{}}", - "async-trait_0.1.89": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.30\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"full\",\"parsing\",\"printing\",\"proc-macro\",\"visit-mut\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1.40\"},{\"kind\":\"dev\",\"name\":\"tracing-attributes\",\"req\":\"^0.1.27\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{}}", - "atomic-waker_1.1.2": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"cargo_bench_support\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.5\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.7.0\"}],\"features\":{}}", - "autocfg_1.5.0": "{\"dependencies\":[],\"features\":{}}", - "axum-core_0.4.5": "{\"dependencies\":[{\"name\":\"async-trait\",\"req\":\"^0.1.67\"},{\"kind\":\"dev\",\"name\":\"axum\",\"req\":\"^0.7.2\"},{\"name\":\"bytes\",\"req\":\"^1.2\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"http\",\"req\":\"^1.0.0\"},{\"name\":\"http-body\",\"req\":\"^1.0.0\"},{\"name\":\"http-body-util\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^1.0.0\"},{\"name\":\"mime\",\"req\":\"^0.3.16\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"},{\"name\":\"rustversion\",\"req\":\"^1.0.9\"},{\"name\":\"sync_wrapper\",\"req\":\"^1.0.0\"},{\"features\":[\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.25.0\"},{\"features\":[\"limit\"],\"name\":\"tower-http\",\"optional\":true,\"req\":\"^0.6.0\"},{\"features\":[\"limit\"],\"kind\":\"dev\",\"name\":\"tower-http\",\"req\":\"^0.6.0\"},{\"name\":\"tower-layer\",\"req\":\"^0.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.37\"}],\"features\":{\"__private_docs\":[\"dep:tower-http\"],\"tracing\":[\"dep:tracing\"]}}", - "axum_0.7.9": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0\"},{\"name\":\"async-trait\",\"req\":\"^0.1.67\"},{\"name\":\"axum-core\",\"req\":\"^0.4.5\"},{\"name\":\"axum-macros\",\"optional\":true,\"req\":\"^0.4.2\"},{\"features\":[\"__private\"],\"kind\":\"dev\",\"name\":\"axum-macros\",\"req\":\"^0.4.1\"},{\"name\":\"base64\",\"optional\":true,\"req\":\"^0.22.1\"},{\"name\":\"bytes\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"http\",\"req\":\"^1.0.0\"},{\"name\":\"http-body\",\"req\":\"^1.0.0\"},{\"name\":\"http-body-util\",\"req\":\"^0.1.0\"},{\"name\":\"hyper\",\"optional\":true,\"req\":\"^1.1.0\"},{\"features\":[\"tokio\",\"server\",\"service\"],\"name\":\"hyper-util\",\"optional\":true,\"req\":\"^0.1.3\"},{\"name\":\"itoa\",\"req\":\"^1.0.5\"},{\"name\":\"matchit\",\"req\":\"^0.7\"},{\"name\":\"memchr\",\"req\":\"^2.4.1\"},{\"name\":\"mime\",\"req\":\"^0.3.16\"},{\"name\":\"multer\",\"optional\":true,\"req\":\"^3.0.0\"},{\"name\":\"percent-encoding\",\"req\":\"^2.1\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"json\",\"stream\",\"multipart\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.12\"},{\"name\":\"rustversion\",\"req\":\"^1.0.9\"},{\"name\":\"serde\",\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"features\":[\"raw_value\"],\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"raw_value\"],\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"serde_path_to_error\",\"optional\":true,\"req\":\"^0.1.8\"},{\"name\":\"serde_urlencoded\",\"optional\":true,\"req\":\"^0.7\"},{\"name\":\"sha1\",\"optional\":true,\"req\":\"^0.10\"},{\"name\":\"sync_wrapper\",\"req\":\"^1.0.0\"},{\"features\":[\"serde-human-readable\"],\"kind\":\"dev\",\"name\":\"time\",\"req\":\"^0.3\"},{\"features\":[\"time\"],\"name\":\"tokio\",\"optional\":true,\"package\":\"tokio\",\"req\":\"^1.25.0\"},{\"features\":[\"macros\",\"rt\",\"rt-multi-thread\",\"net\",\"test-util\"],\"kind\":\"dev\",\"name\":\"tokio\",\"package\":\"tokio\",\"req\":\"^1.25.0\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"name\":\"tokio-tungstenite\",\"optional\":true,\"req\":\"^0.24.0\"},{\"kind\":\"dev\",\"name\":\"tokio-tungstenite\",\"req\":\"^0.24.0\"},{\"default_features\":false,\"features\":[\"util\"],\"name\":\"tower\",\"req\":\"^0.5.1\"},{\"features\":[\"util\",\"timeout\",\"limit\",\"load-shed\",\"steer\",\"filter\"],\"kind\":\"dev\",\"name\":\"tower\",\"package\":\"tower\",\"req\":\"^0.5.1\"},{\"features\":[\"add-extension\",\"auth\",\"catch-panic\",\"compression-br\",\"compression-deflate\",\"compression-gzip\",\"cors\",\"decompression-br\",\"decompression-deflate\",\"decompression-gzip\",\"follow-redirect\",\"fs\",\"limit\",\"map-request-body\",\"map-response-body\",\"metrics\",\"normalize-path\",\"propagate-header\",\"redirect\",\"request-id\",\"sensitive-headers\",\"set-header\",\"set-status\",\"timeout\",\"trace\",\"util\",\"validate-request\"],\"name\":\"tower-http\",\"optional\":true,\"req\":\"^0.6.0\"},{\"features\":[\"add-extension\",\"auth\",\"catch-panic\",\"compression-br\",\"compression-deflate\",\"compression-gzip\",\"cors\",\"decompression-br\",\"decompression-deflate\",\"decompression-gzip\",\"follow-redirect\",\"fs\",\"limit\",\"map-request-body\",\"map-response-body\",\"metrics\",\"normalize-path\",\"propagate-header\",\"redirect\",\"request-id\",\"sensitive-headers\",\"set-header\",\"set-status\",\"timeout\",\"trace\",\"util\",\"validate-request\"],\"kind\":\"dev\",\"name\":\"tower-http\",\"req\":\"^0.6.0\"},{\"name\":\"tower-layer\",\"req\":\"^0.3.2\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1\"},{\"features\":[\"json\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"},{\"features\":[\"serde\",\"v4\"],\"kind\":\"dev\",\"name\":\"uuid\",\"req\":\"^1.0\"}],\"features\":{\"__private_docs\":[\"axum-core/__private_docs\",\"tower/full\",\"dep:tower-http\"],\"default\":[\"form\",\"http1\",\"json\",\"matched-path\",\"original-uri\",\"query\",\"tokio\",\"tower-log\",\"tracing\"],\"form\":[\"dep:serde_urlencoded\"],\"http1\":[\"dep:hyper\",\"hyper?/http1\",\"hyper-util?/http1\"],\"http2\":[\"dep:hyper\",\"hyper?/http2\",\"hyper-util?/http2\"],\"json\":[\"dep:serde_json\",\"dep:serde_path_to_error\"],\"macros\":[\"dep:axum-macros\"],\"matched-path\":[],\"multipart\":[\"dep:multer\"],\"original-uri\":[],\"query\":[\"dep:serde_urlencoded\"],\"tokio\":[\"dep:hyper-util\",\"dep:tokio\",\"tokio/net\",\"tokio/rt\",\"tower/make\",\"tokio/macros\"],\"tower-log\":[\"tower/log\"],\"tracing\":[\"dep:tracing\",\"axum-core/tracing\"],\"ws\":[\"dep:hyper\",\"tokio\",\"dep:tokio-tungstenite\",\"dep:sha1\",\"dep:base64\"]}}", - "backtrace_0.3.76": "{\"dependencies\":[{\"default_features\":false,\"name\":\"addr2line\",\"req\":\"^0.25.0\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"name\":\"cfg-if\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"cpp_demangle\",\"optional\":true,\"req\":\"^0.5.0\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.156\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"kind\":\"dev\",\"name\":\"libloading\",\"req\":\"^0.8\"},{\"default_features\":false,\"name\":\"miniz_oxide\",\"req\":\"^0.8\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"default_features\":false,\"features\":[\"read_core\",\"elf\",\"macho\",\"pe\",\"xcoff\",\"unaligned\",\"archive\"],\"name\":\"object\",\"req\":\"^0.37.0\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"name\":\"rustc-demangle\",\"req\":\"^0.1.24\"},{\"default_features\":false,\"name\":\"ruzstd\",\"optional\":true,\"req\":\"^0.8.1\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"windows-link\",\"req\":\"^0.2\",\"target\":\"cfg(any(windows, target_os = \\\"cygwin\\\"))\"}],\"features\":{\"coresymbolication\":[],\"dbghelp\":[],\"default\":[\"std\"],\"dl_iterate_phdr\":[],\"dladdr\":[],\"kernel32\":[],\"libunwind\":[],\"ruzstd\":[\"dep:ruzstd\"],\"serialize-serde\":[\"serde\"],\"std\":[],\"unix-backtrace\":[]}}", - "base64_0.13.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"=0.3.2\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.6.1\"},{\"kind\":\"dev\",\"name\":\"structopt\",\"req\":\"^0.3\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[]}}", - "base64_0.22.1": "{\"dependencies\":[{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"clap\",\"req\":\"^3.2.25\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rstest\",\"req\":\"^0.13.0\"},{\"kind\":\"dev\",\"name\":\"rstest_reuse\",\"req\":\"^0.6.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"strum\",\"req\":\"^0.25\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", - "bitflags_2.11.0": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"arbitrary\",\"req\":\"^1.0\"},{\"name\":\"bytemuck\",\"optional\":true,\"req\":\"^1.12\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"bytemuck\",\"req\":\"^1.12.2\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.228\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde_lib\",\"package\":\"serde\",\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.19\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.18\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"zerocopy\",\"req\":\"^0.8\"}],\"features\":{\"example_generated\":[],\"serde\":[\"serde_core\"],\"std\":[]}}", - "bitflags_2.9.1": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"arbitrary\",\"req\":\"^1.0\"},{\"name\":\"bytemuck\",\"optional\":true,\"req\":\"^1.12\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"bytemuck\",\"req\":\"^1.12.2\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.19\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.18\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"zerocopy\",\"req\":\"^0.8\"}],\"features\":{\"example_generated\":[],\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"],\"std\":[]}}", - "byteorder_1.5.0": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9.2\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"std\"],\"i128\":[],\"std\":[]}}", - "bytes_1.11.1": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"require-cas\"],\"name\":\"extra-platforms\",\"optional\":true,\"package\":\"portable-atomic\",\"req\":\"^1.3\"},{\"kind\":\"dev\",\"name\":\"loom\",\"req\":\"^0.7\",\"target\":\"cfg(loom)\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.60\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "castaway_0.2.4": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1\"},{\"name\":\"rustversion\",\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", - "cc_1.2.33": "{\"dependencies\":[{\"default_features\":false,\"name\":\"jobserver\",\"optional\":true,\"req\":\"^0.1.30\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.62\",\"target\":\"cfg(unix)\"},{\"name\":\"shlex\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{\"jobserver\":[],\"parallel\":[\"dep:libc\",\"dep:jobserver\"]}}", - "cfg-if_1.0.0": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"]}}", - "cfg-if_1.0.4": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"rustc-dep-of-std\":[\"core\"]}}", - "compact_str_0.9.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"borsh\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"castaway\",\"req\":\"^0.2.3\"},{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"cfg-if\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"diesel\",\"optional\":true,\"req\":\"^2\"},{\"name\":\"itoa\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"markup\",\"optional\":true,\"req\":\"^0.15\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"proptest\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"rkyv\",\"optional\":true,\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rkyv\",\"req\":\"^0.8.8\"},{\"name\":\"rustversion\",\"req\":\"^1\"},{\"name\":\"ryu\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"derive\",\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"features\":[\"union\"],\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"name\":\"sqlx\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"static_assertions\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"test-case\",\"req\":\"^3\"},{\"kind\":\"dev\",\"name\":\"test-strategy\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"borsh\":[\"dep:borsh\"],\"bytes\":[\"dep:bytes\"],\"default\":[\"std\"],\"diesel\":[\"dep:diesel\"],\"markup\":[\"dep:markup\"],\"proptest\":[\"dep:proptest\"],\"quickcheck\":[\"dep:quickcheck\"],\"rkyv\":[\"dep:rkyv\"],\"serde\":[\"dep:serde\"],\"smallvec\":[\"dep:smallvec\"],\"sqlx\":[\"dep:sqlx\",\"std\"],\"sqlx-mysql\":[\"sqlx\",\"sqlx/mysql\"],\"sqlx-postgres\":[\"sqlx\",\"sqlx/postgres\"],\"sqlx-sqlite\":[\"sqlx\",\"sqlx/sqlite\"],\"std\":[],\"zeroize\":[\"dep:zeroize\"]}}", - "crossbeam-deque_0.8.6": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-epoch\",\"req\":\"^0.9.17\"},{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"crossbeam-epoch/std\",\"crossbeam-utils/std\"]}}", - "crossbeam-epoch_0.9.18": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"name\":\"loom-crate\",\"optional\":true,\"package\":\"loom\",\"req\":\"^0.7.1\",\"target\":\"cfg(crossbeam_loom)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"loom\":[\"loom-crate\",\"crossbeam-utils/loom\"],\"nightly\":[\"crossbeam-utils/nightly\"],\"std\":[\"alloc\",\"crossbeam-utils/std\"]}}", - "crossbeam-utils_0.8.21": "{\"dependencies\":[{\"name\":\"loom\",\"optional\":true,\"req\":\"^0.7.1\",\"target\":\"cfg(crossbeam_loom)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"std\"],\"nightly\":[],\"std\":[]}}", - "darling_0.20.10": "{\"dependencies\":[{\"name\":\"darling_core\",\"req\":\"=0.20.10\"},{\"name\":\"darling_macro\",\"req\":\"=0.20.10\"},{\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.37\"},{\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.9\",\"target\":\"cfg(compiletests)\"},{\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.15\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.38\",\"target\":\"cfg(compiletests)\"}],\"features\":{\"default\":[\"suggestions\"],\"diagnostics\":[\"darling_core/diagnostics\"],\"suggestions\":[\"darling_core/suggestions\"]}}", - "darling_core_0.20.10": "{\"dependencies\":[{\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"name\":\"ident_case\",\"req\":\"^1.0.1\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.37\"},{\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"name\":\"strsim\",\"optional\":true,\"req\":\"^0.11.1\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"diagnostics\":[],\"suggestions\":[\"strsim\"]}}", - "darling_macro_0.20.10": "{\"dependencies\":[{\"name\":\"darling_core\",\"req\":\"=0.20.10\"},{\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{}}", - "dary_heap_0.3.7": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1\"}],\"features\":{\"extra\":[],\"unstable\":[],\"unstable_nightly\":[]}}", - "derive_builder_0.20.2": "{\"dependencies\":[{\"name\":\"derive_builder_macro\",\"req\":\"=0.20.2\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.4\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.38\",\"target\":\"cfg(compiletests)\"}],\"features\":{\"alloc\":[\"derive_builder_macro/alloc\"],\"clippy\":[\"derive_builder_macro/clippy\"],\"default\":[\"std\"],\"std\":[\"derive_builder_macro/lib_has_std\"]}}", - "derive_builder_core_0.20.2": "{\"dependencies\":[{\"name\":\"darling\",\"req\":\"^0.20.10\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.37\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"alloc\":[],\"clippy\":[],\"lib_has_std\":[]}}", - "derive_builder_macro_0.20.2": "{\"dependencies\":[{\"name\":\"derive_builder_core\",\"req\":\"=0.20.2\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"alloc\":[\"derive_builder_core/alloc\"],\"clippy\":[\"derive_builder_core/clippy\"],\"lib_has_std\":[\"derive_builder_core/lib_has_std\"]}}", - "either_1.13.0": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"use_std\"],\"use_std\":[]}}", - "either_1.15.0": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"alloc\",\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.95\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[],\"use_std\":[\"std\"]}}", - "equivalent_1.0.2": "{\"dependencies\":[],\"features\":{}}", - "errno_0.3.14": "{\"dependencies\":[{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(target_os=\\\"hermit\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(target_os=\\\"wasi\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(unix)\"},{\"features\":[\"Win32_Foundation\",\"Win32_System_Diagnostics_Debug\"],\"name\":\"windows-sys\",\"req\":\">=0.52, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"libc/std\"]}}", - "esaxx-rs_0.1.10": "{\"dependencies\":[{\"kind\":\"build\",\"name\":\"cc\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"}],\"features\":{\"cpp\":[\"cc\"],\"default\":[\"cpp\"]}}", - "fastrand_2.3.0": "{\"dependencies\":[{\"features\":[\"js\"],\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.2\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"getrandom\",\"req\":\"^0.2\"},{\"features\":[\"js\"],\"kind\":\"dev\",\"name\":\"getrandom\",\"req\":\"^0.2\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"wyhash\",\"req\":\"^0.5\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"js\":[\"std\",\"getrandom\"],\"std\":[\"alloc\"]}}", - "fixedbitset_0.4.2": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "fnv_1.0.7": "{\"dependencies\":[],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "foldhash_0.1.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"ahash\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"chrono\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"hashbrown\",\"req\":\"^0.14\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"uuid\",\"req\":\"^1.8\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "futures-channel_0.3.32": "{\"dependencies\":[{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-sink\",\"optional\":true,\"req\":\"^0.3.32\"}],\"features\":{\"alloc\":[\"futures-core/alloc\"],\"cfg-target-has-atomic\":[],\"default\":[\"std\"],\"sink\":[\"futures-sink\"],\"std\":[\"alloc\",\"futures-core/std\"],\"unstable\":[]}}", - "futures-core_0.3.32": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"require-cas\"],\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.3\"}],\"features\":{\"alloc\":[],\"cfg-target-has-atomic\":[],\"default\":[\"std\"],\"std\":[\"alloc\"],\"unstable\":[]}}", - "futures-sink_0.3.32": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", - "futures-task_0.3.32": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"cfg-target-has-atomic\":[],\"default\":[\"std\"],\"std\":[\"alloc\"],\"unstable\":[]}}", - "futures-util_0.3.32": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"std\"],\"name\":\"futures-channel\",\"optional\":true,\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3.32\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"futures-io\",\"optional\":true,\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-macro\",\"optional\":true,\"req\":\"=0.3.32\"},{\"default_features\":false,\"name\":\"futures-sink\",\"optional\":true,\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-task\",\"req\":\"^0.3.32\"},{\"name\":\"futures_01\",\"optional\":true,\"package\":\"futures\",\"req\":\"^0.1.25\"},{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.26\"},{\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.2\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.6\"},{\"default_features\":false,\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4.7\"},{\"name\":\"spin\",\"optional\":true,\"req\":\"^0.10.0\"},{\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^0.1.11\"},{\"name\":\"tokio-io\",\"optional\":true,\"req\":\"^0.1.9\"}],\"features\":{\"alloc\":[\"futures-core/alloc\",\"futures-task/alloc\",\"slab\"],\"async-await\":[],\"async-await-macro\":[\"async-await\",\"futures-macro\"],\"bilock\":[],\"cfg-target-has-atomic\":[],\"channel\":[\"std\",\"futures-channel\"],\"compat\":[\"std\",\"futures_01\",\"libc\"],\"default\":[\"std\",\"async-await\",\"async-await-macro\"],\"io\":[\"std\",\"futures-io\",\"memchr\"],\"io-compat\":[\"io\",\"compat\",\"tokio-io\",\"libc\"],\"portable-atomic\":[\"futures-core/portable-atomic\"],\"sink\":[\"futures-sink\"],\"std\":[\"alloc\",\"futures-core/std\",\"futures-task/std\",\"slab/std\"],\"unstable\":[\"futures-core/unstable\",\"futures-task/unstable\"],\"write-all-vectored\":[\"io\"]}}", - "getrandom_0.2.17": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(unix)\"},{\"default_features\":false,\"name\":\"wasi\",\"req\":\"^0.11\",\"target\":\"cfg(target_os = \\\"wasi\\\")\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.62\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.18\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"}],\"features\":{\"custom\":[],\"js\":[\"wasm-bindgen\",\"js-sys\"],\"linux_disable_fallback\":[],\"rdrand\":[],\"rustc-dep-of-std\":[\"compiler_builtins\",\"core\",\"libc/rustc-dep-of-std\",\"wasi/rustc-dep-of-std\"],\"std\":[],\"test-in-browser\":[]}}", - "getrandom_0.3.3": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3.77\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"), target_feature = \\\"atomics\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"netbsd\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"solaris\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"vxworks\\\")\"},{\"default_features\":false,\"name\":\"r-efi\",\"req\":\"^5.1\",\"target\":\"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\"},{\"default_features\":false,\"name\":\"wasi\",\"req\":\"^0.14\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.98\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"}],\"features\":{\"rustc-dep-of-std\":[\"dep:compiler_builtins\",\"dep:core\"],\"std\":[],\"wasm_js\":[\"dep:wasm-bindgen\",\"dep:js-sys\"]}}", - "getrandom_0.4.2": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3.77\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"), target_feature = \\\"atomics\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"netbsd\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"solaris\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"vxworks\\\")\"},{\"default_features\":false,\"name\":\"r-efi\",\"req\":\"^6\",\"target\":\"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\"},{\"name\":\"rand_core\",\"optional\":true,\"req\":\"^0.10.0\"},{\"default_features\":false,\"name\":\"wasip2\",\"req\":\"^1\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\"},{\"name\":\"wasip3\",\"req\":\"^0.4\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p3\\\"))\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.98\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"}],\"features\":{\"std\":[],\"sys_rng\":[\"dep:rand_core\"],\"wasm_js\":[\"dep:wasm-bindgen\",\"dep:js-sys\"]}}", - "gimli_0.32.3": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"fallible-iterator\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.0.0\"},{\"default_features\":false,\"name\":\"stable_deref_trait\",\"optional\":true,\"req\":\"^1.1.0\"},{\"kind\":\"dev\",\"name\":\"test-assembler\",\"req\":\"^0.1.3\"}],\"features\":{\"default\":[\"read-all\",\"write\"],\"endian-reader\":[\"read\",\"dep:stable_deref_trait\"],\"fallible-iterator\":[\"dep:fallible-iterator\"],\"read\":[\"read-core\"],\"read-all\":[\"read\",\"std\",\"fallible-iterator\",\"endian-reader\"],\"read-core\":[],\"rustc-dep-of-std\":[\"dep:core\",\"dep:alloc\"],\"std\":[\"fallible-iterator?/std\",\"stable_deref_trait?/std\"],\"write\":[\"dep:indexmap\"]}}", - "h2_0.4.6": "{\"dependencies\":[{\"name\":\"atomic-waker\",\"req\":\"^1.0.0\"},{\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.10\"},{\"name\":\"fnv\",\"req\":\"^1.0.5\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-sink\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.3\"},{\"name\":\"http\",\"req\":\"^1\"},{\"features\":[\"std\"],\"name\":\"indexmap\",\"req\":\"^2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.4\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.0\"},{\"name\":\"slab\",\"req\":\"^0.4.2\"},{\"features\":[\"io-util\"],\"name\":\"tokio\",\"req\":\"^1\"},{\"features\":[\"rt-multi-thread\",\"macros\",\"sync\",\"net\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tokio-rustls\",\"req\":\"^0.26\"},{\"features\":[\"codec\",\"io\"],\"name\":\"tokio-util\",\"req\":\"^0.7.1\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"req\":\"^0.1.35\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\"},{\"kind\":\"dev\",\"name\":\"webpki-roots\",\"req\":\"^0.26\"}],\"features\":{\"stream\":[],\"unstable\":[]}}", - "hashbrown_0.12.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"ahash\",\"optional\":true,\"req\":\"^0.7.0\"},{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"bumpalo\",\"optional\":true,\"req\":\"^3.5.0\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.3\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.25\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"ahash-compile-time-rng\":[\"ahash/compile-time-rng\"],\"default\":[\"ahash\",\"inline-more\"],\"inline-more\":[],\"nightly\":[],\"raw\":[],\"rustc-dep-of-std\":[\"nightly\",\"core\",\"compiler_builtins\",\"alloc\",\"rustc-internal-api\"],\"rustc-internal-api\":[]}}", - "hashbrown_0.15.5": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"allocator-api2\",\"optional\":true,\"req\":\"^0.2.9\"},{\"features\":[\"allocator-api2\"],\"kind\":\"dev\",\"name\":\"bumpalo\",\"req\":\"^3.13.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.1\"},{\"default_features\":false,\"name\":\"equivalent\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"default_features\":false,\"name\":\"foldhash\",\"optional\":true,\"req\":\"^0.1.2\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9.0\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.2\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.2\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.25\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"default-hasher\",\"inline-more\",\"allocator-api2\",\"equivalent\",\"raw-entry\"],\"default-hasher\":[\"dep:foldhash\"],\"inline-more\":[],\"nightly\":[\"bumpalo/allocator_api\"],\"raw-entry\":[],\"rustc-dep-of-std\":[\"nightly\",\"core\",\"alloc\",\"rustc-internal-api\"],\"rustc-internal-api\":[]}}", - "hashbrown_0.16.1": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"allocator-api2\",\"optional\":true,\"req\":\"^0.2.9\"},{\"features\":[\"allocator-api2\"],\"kind\":\"dev\",\"name\":\"bumpalo\",\"req\":\"^3.13.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"equivalent\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"default_features\":false,\"name\":\"foldhash\",\"optional\":true,\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.155\",\"target\":\"cfg(unix)\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9.0\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.9.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.2\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.221\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"default-hasher\",\"inline-more\",\"allocator-api2\",\"equivalent\",\"raw-entry\"],\"default-hasher\":[\"dep:foldhash\"],\"inline-more\":[],\"nightly\":[\"foldhash?/nightly\",\"bumpalo/allocator_api\"],\"raw-entry\":[],\"rustc-dep-of-std\":[\"nightly\",\"core\",\"alloc\",\"rustc-internal-api\"],\"rustc-internal-api\":[],\"serde\":[\"dep:serde_core\",\"dep:serde\"]}}", - "heck_0.5.0": "{\"dependencies\":[],\"features\":{}}", - "http-body-util_0.1.3": "{\"dependencies\":[{\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"http-body\",\"req\":\"^1\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"macros\",\"rt\",\"sync\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"}],\"features\":{\"channel\":[\"dep:tokio\"],\"default\":[],\"full\":[\"channel\"]}}", - "http-body_1.0.1": "{\"dependencies\":[{\"name\":\"bytes\",\"req\":\"^1\"},{\"name\":\"http\",\"req\":\"^1\"}],\"features\":{}}", - "http_1.4.0": "{\"dependencies\":[{\"name\":\"bytes\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"name\":\"itoa\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.0\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "httparse_1.10.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.5\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "httpdate_1.0.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"}],\"features\":{}}", - "hyper-timeout_0.5.2": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"name\":\"hyper\",\"req\":\"^1.1\"},{\"features\":[\"http1\"],\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^1.1\"},{\"kind\":\"dev\",\"name\":\"hyper-tls\",\"req\":\"^0.6\"},{\"features\":[\"client-legacy\",\"http1\"],\"name\":\"hyper-util\",\"req\":\"^0.1.10\"},{\"features\":[\"client-legacy\",\"http1\",\"server\",\"server-graceful\"],\"kind\":\"dev\",\"name\":\"hyper-util\",\"req\":\"^0.1.10\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"name\":\"tokio\",\"req\":\"^1.35\"},{\"features\":[\"io-std\",\"io-util\",\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.35\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"}],\"features\":{}}", - "hyper-util_0.1.20": "{\"dependencies\":[{\"name\":\"base64\",\"optional\":true,\"req\":\"^0.22\"},{\"name\":\"bytes\",\"req\":\"^1.7.1\"},{\"kind\":\"dev\",\"name\":\"bytes\",\"req\":\"^1\"},{\"name\":\"futures-channel\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.16\"},{\"default_features\":false,\"features\":[\"alloc\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3.16\"},{\"name\":\"http\",\"req\":\"^1.0\"},{\"name\":\"http-body\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1.0\"},{\"name\":\"hyper\",\"req\":\"^1.8.0\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^1.4.0\"},{\"name\":\"ipnet\",\"optional\":true,\"req\":\"^2.9\"},{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2\"},{\"name\":\"percent-encoding\",\"optional\":true,\"req\":\"^2.3\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.4\"},{\"kind\":\"dev\",\"name\":\"pnet_datalink\",\"req\":\"^0.35.0\",\"target\":\"cfg(any(target_os = \\\"linux\\\", target_os = \\\"macos\\\"))\"},{\"kind\":\"dev\",\"name\":\"pretty_env_logger\",\"req\":\"^0.5\"},{\"features\":[\"all\"],\"name\":\"socket2\",\"optional\":true,\"req\":\">=0.5.9, <0.7\"},{\"name\":\"system-configuration\",\"optional\":true,\"req\":\"^0.7\",\"target\":\"cfg(target_os = \\\"macos\\\")\"},{\"default_features\":false,\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"macros\",\"test-util\",\"signal\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"name\":\"tower-layer\",\"optional\":true,\"req\":\"^0.3\"},{\"name\":\"tower-service\",\"optional\":true,\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"tower-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"windows-registry\",\"optional\":true,\"req\":\">=0.3, <0.7\",\"target\":\"cfg(windows)\"}],\"features\":{\"__internal_happy_eyeballs_tests\":[],\"client\":[\"hyper/client\",\"tokio/net\",\"dep:tracing\",\"dep:futures-channel\",\"dep:tower-service\"],\"client-legacy\":[\"client\",\"dep:socket2\",\"tokio/sync\",\"dep:libc\",\"dep:futures-util\"],\"client-pool\":[\"client\",\"dep:futures-util\",\"dep:tower-layer\"],\"client-proxy\":[\"client\",\"dep:base64\",\"dep:ipnet\",\"dep:percent-encoding\"],\"client-proxy-system\":[\"dep:system-configuration\",\"dep:windows-registry\"],\"default\":[],\"full\":[\"client\",\"client-legacy\",\"client-pool\",\"client-proxy\",\"client-proxy-system\",\"server\",\"server-auto\",\"server-graceful\",\"service\",\"http1\",\"http2\",\"tokio\",\"tracing\"],\"http1\":[\"hyper/http1\"],\"http2\":[\"hyper/http2\"],\"server\":[\"hyper/server\"],\"server-auto\":[\"server\",\"http1\",\"http2\"],\"server-graceful\":[\"server\",\"tokio/sync\"],\"service\":[\"dep:tower-service\"],\"tokio\":[\"dep:tokio\",\"tokio/rt\",\"tokio/time\"],\"tracing\":[\"dep:tracing\"]}}", - "hyper_1.8.1": "{\"dependencies\":[{\"name\":\"atomic-waker\",\"optional\":true,\"req\":\"^1.1.2\"},{\"name\":\"bytes\",\"req\":\"^1.2\"},{\"kind\":\"dev\",\"name\":\"form_urlencoded\",\"req\":\"^1\"},{\"name\":\"futures-channel\",\"optional\":true,\"req\":\"^0.3\"},{\"features\":[\"sink\"],\"kind\":\"dev\",\"name\":\"futures-channel\",\"req\":\"^0.3\"},{\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3.31\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\",\"sink\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"h2\",\"optional\":true,\"req\":\"^0.4.2\"},{\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"http-body\",\"req\":\"^1\"},{\"name\":\"http-body-util\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"name\":\"httparse\",\"optional\":true,\"req\":\"^1.9\"},{\"name\":\"httpdate\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"itoa\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"pin-project-lite\",\"optional\":true,\"req\":\"^0.2.4\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.4\"},{\"name\":\"pin-utils\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"pretty_env_logger\",\"req\":\"^0.5\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"features\":[\"const_generics\",\"const_new\"],\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1.12\"},{\"kind\":\"dev\",\"name\":\"spmc\",\"req\":\"^0.3\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"req\":\"^1\"},{\"features\":[\"fs\",\"macros\",\"net\",\"io-std\",\"io-util\",\"rt\",\"rt-multi-thread\",\"sync\",\"time\",\"test-util\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"tokio-util\",\"req\":\"^0.7.10\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"want\",\"optional\":true,\"req\":\"^0.3\"}],\"features\":{\"capi\":[],\"client\":[\"dep:want\",\"dep:pin-project-lite\",\"dep:smallvec\"],\"default\":[],\"ffi\":[\"dep:http-body-util\",\"dep:futures-util\"],\"full\":[\"client\",\"http1\",\"http2\",\"server\"],\"http1\":[\"dep:atomic-waker\",\"dep:futures-channel\",\"dep:futures-core\",\"dep:httparse\",\"dep:itoa\",\"dep:pin-utils\"],\"http2\":[\"dep:futures-channel\",\"dep:futures-core\",\"dep:h2\"],\"nightly\":[],\"server\":[\"dep:httpdate\",\"dep:pin-project-lite\",\"dep:smallvec\"],\"tracing\":[\"dep:tracing\"]}}", - "id-arena_2.3.0": "{\"dependencies\":[{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.0.3\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "ident_case_1.0.1": "{\"dependencies\":[],\"features\":{}}", - "indexmap_1.9.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"build\",\"name\":\"autocfg\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2.1\"},{\"default_features\":false,\"features\":[\"raw\"],\"name\":\"hashbrown\",\"req\":\"^0.12\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.3\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.4.1\"},{\"name\":\"rustc-rayon\",\"optional\":true,\"package\":\"rustc-rayon\",\"req\":\"^0.5\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0\"}],\"features\":{\"serde-1\":[\"serde\"],\"std\":[],\"test_debug\":[],\"test_low_transition_point\":[]}}", - "indexmap_2.13.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"borsh\",\"optional\":true,\"req\":\"^1.2\"},{\"default_features\":false,\"name\":\"equivalent\",\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"fastrand\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"hashbrown\",\"req\":\"^0.16.1\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.14\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.9\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"},{\"default_features\":false,\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.220\"},{\"default_features\":false,\"name\":\"sval\",\"optional\":true,\"req\":\"^2\"}],\"features\":{\"default\":[\"std\"],\"serde\":[\"dep:serde_core\",\"dep:serde\"],\"std\":[],\"test_debug\":[]}}", - "itertools_0.13.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"permutohedron\",\"req\":\"^0.2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"use_std\"],\"use_alloc\":[],\"use_std\":[\"use_alloc\",\"either/use_std\"]}}", - "itertools_0.14.0": "{\"dependencies\":[{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"permutohedron\",\"req\":\"^0.2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"use_std\"],\"use_alloc\":[],\"use_std\":[\"use_alloc\",\"either/use_std\"]}}", - "itoa_1.0.14": "{\"dependencies\":[{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"}],\"features\":{}}", - "itoa_1.0.17": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\",\"target\":\"cfg(not(miri))\"},{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"}],\"features\":{}}", - "leb128fmt_0.1.0": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[]}}", - "libc_0.2.169": "{\"dependencies\":[{\"name\":\"rustc-std-workspace-core\",\"optional\":true,\"req\":\"^1.0.0\"}],\"features\":{\"align\":[],\"const-extern-fn\":[],\"default\":[\"std\"],\"extra_traits\":[],\"rustc-dep-of-std\":[\"align\",\"rustc-std-workspace-core\"],\"std\":[],\"use_std\":[\"std\"]}}", - "libc_0.2.183": "{\"dependencies\":[{\"name\":\"rustc-std-workspace-core\",\"optional\":true,\"req\":\"^1.0.1\"}],\"features\":{\"align\":[],\"const-extern-fn\":[],\"default\":[\"std\"],\"extra_traits\":[],\"rustc-dep-of-std\":[\"align\",\"rustc-std-workspace-core\"],\"std\":[],\"use_std\":[\"std\"]}}", - "linux-raw-sys_0.12.1": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.100\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1.0\"}],\"features\":{\"auxvec\":[],\"bootparam\":[],\"btrfs\":[],\"default\":[\"std\",\"general\",\"errno\"],\"elf\":[],\"elf_uapi\":[],\"errno\":[],\"general\":[],\"if_arp\":[],\"if_ether\":[],\"if_packet\":[],\"if_tun\":[],\"image\":[],\"io_uring\":[],\"ioctl\":[],\"landlock\":[],\"loop_device\":[],\"mempolicy\":[],\"net\":[],\"netlink\":[],\"no_std\":[],\"prctl\":[],\"ptrace\":[],\"rustc-dep-of-std\":[\"core\",\"no_std\"],\"std\":[],\"system\":[],\"vm_sockets\":[],\"xdp\":[]}}", - "lock_api_0.4.14": "{\"dependencies\":[{\"name\":\"owning_ref\",\"optional\":true,\"req\":\"^0.4.1\"},{\"default_features\":false,\"name\":\"scopeguard\",\"req\":\"^1.1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.126\"}],\"features\":{\"arc_lock\":[],\"atomic_usize\":[],\"default\":[\"atomic_usize\"],\"nightly\":[]}}", - "log_0.4.22": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.63\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"sval\",\"optional\":true,\"req\":\"^2.1\"},{\"kind\":\"dev\",\"name\":\"sval\",\"req\":\"^2.1\"},{\"kind\":\"dev\",\"name\":\"sval_derive\",\"req\":\"^2.1\"},{\"default_features\":false,\"name\":\"sval_ref\",\"optional\":true,\"req\":\"^2.1\"},{\"default_features\":false,\"features\":[\"inline-i128\"],\"name\":\"value-bag\",\"optional\":true,\"req\":\"^1.7\"},{\"features\":[\"test\"],\"kind\":\"dev\",\"name\":\"value-bag\",\"req\":\"^1.7\"}],\"features\":{\"kv\":[],\"kv_serde\":[\"kv_std\",\"value-bag/serde\",\"serde\"],\"kv_std\":[\"std\",\"kv\",\"value-bag/error\"],\"kv_sval\":[\"kv\",\"value-bag/sval\",\"sval\",\"sval_ref\"],\"kv_unstable\":[\"kv\",\"value-bag\"],\"kv_unstable_serde\":[\"kv_serde\",\"kv_unstable_std\"],\"kv_unstable_std\":[\"kv_std\",\"kv_unstable\"],\"kv_unstable_sval\":[\"kv_sval\",\"kv_unstable\"],\"max_level_debug\":[],\"max_level_error\":[],\"max_level_info\":[],\"max_level_off\":[],\"max_level_trace\":[],\"max_level_warn\":[],\"release_max_level_debug\":[],\"release_max_level_error\":[],\"release_max_level_info\":[],\"release_max_level_off\":[],\"release_max_level_trace\":[],\"release_max_level_warn\":[],\"std\":[]}}", - "log_0.4.29": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.63\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"sval\",\"optional\":true,\"req\":\"^2.16\"},{\"kind\":\"dev\",\"name\":\"sval\",\"req\":\"^2.16\"},{\"kind\":\"dev\",\"name\":\"sval_derive\",\"req\":\"^2.16\"},{\"default_features\":false,\"name\":\"sval_ref\",\"optional\":true,\"req\":\"^2.16\"},{\"default_features\":false,\"features\":[\"inline-i128\"],\"name\":\"value-bag\",\"optional\":true,\"req\":\"^1.12\"},{\"features\":[\"test\"],\"kind\":\"dev\",\"name\":\"value-bag\",\"req\":\"^1.12\"}],\"features\":{\"kv\":[],\"kv_serde\":[\"kv_std\",\"value-bag/serde\",\"serde\"],\"kv_std\":[\"std\",\"kv\",\"value-bag/error\"],\"kv_sval\":[\"kv\",\"value-bag/sval\",\"sval\",\"sval_ref\"],\"kv_unstable\":[\"kv\",\"value-bag\"],\"kv_unstable_serde\":[\"kv_serde\",\"kv_unstable_std\"],\"kv_unstable_std\":[\"kv_std\",\"kv_unstable\"],\"kv_unstable_sval\":[\"kv_sval\",\"kv_unstable\"],\"max_level_debug\":[],\"max_level_error\":[],\"max_level_info\":[],\"max_level_off\":[],\"max_level_trace\":[],\"max_level_warn\":[],\"release_max_level_debug\":[],\"release_max_level_error\":[],\"release_max_level_info\":[],\"release_max_level_off\":[],\"release_max_level_trace\":[],\"release_max_level_warn\":[],\"serde\":[\"serde_core\"],\"std\":[]}}", - "macro_rules_attribute-proc_macro_0.2.0": "{\"dependencies\":[],\"features\":{\"verbose-expansions\":[]}}", - "macro_rules_attribute_0.2.0": "{\"dependencies\":[{\"name\":\"macro_rules_attribute-proc_macro\",\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.10.0\"},{\"name\":\"paste\",\"req\":\"^1.0.7\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.8\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.136\"}],\"features\":{\"better-docs\":[],\"default\":[],\"verbose-expansions\":[\"macro_rules_attribute-proc_macro/verbose-expansions\"]}}", - "matchit_0.7.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"actix-router\",\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.4\"},{\"kind\":\"dev\",\"name\":\"gonzales\",\"req\":\"^0.0.3-beta\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^0.14\"},{\"kind\":\"dev\",\"name\":\"path-tree\",\"req\":\"^0.2.2\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.5.4\"},{\"kind\":\"dev\",\"name\":\"route-recognizer\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"routefinder\",\"req\":\"^0.5.2\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"features\":[\"make\",\"util\"],\"kind\":\"dev\",\"name\":\"tower\",\"req\":\"^0.4\"}],\"features\":{\"__test_helpers\":[],\"default\":[]}}", - "memchr_2.7.4": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.20\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"libc\":[],\"logging\":[\"dep:log\"],\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"],\"std\":[\"alloc\"],\"use_std\":[\"std\"]}}", - "memchr_2.8.0": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.20\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"libc\":[],\"logging\":[\"dep:log\"],\"rustc-dep-of-std\":[\"core\"],\"std\":[\"alloc\"],\"use_std\":[\"std\"]}}", - "mime_0.3.17": "{\"dependencies\":[],\"features\":{}}", - "minimal-lexical_0.2.1": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"compact\":[],\"default\":[\"std\"],\"lint\":[],\"nightly\":[],\"std\":[]}}", - "miniz_oxide_0.8.9": "{\"dependencies\":[{\"default_features\":false,\"name\":\"adler2\",\"req\":\"^2.0\"},{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"simd-adler32\",\"optional\":true,\"req\":\"^0.3.3\"}],\"features\":{\"block-boundary\":[],\"default\":[\"with-alloc\"],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"adler2/rustc-dep-of-std\"],\"simd\":[\"simd-adler32\"],\"std\":[],\"with-alloc\":[]}}", - "mio_1.1.1": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"libc\",\"req\":\"^0.2.178\",\"target\":\"cfg(target_os = \\\"hermit\\\")\"},{\"name\":\"libc\",\"req\":\"^0.2.178\",\"target\":\"cfg(target_os = \\\"wasi\\\")\"},{\"name\":\"libc\",\"req\":\"^0.2.178\",\"target\":\"cfg(unix)\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.8\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9\"},{\"name\":\"wasi\",\"req\":\"^0.11.0\",\"target\":\"cfg(target_os = \\\"wasi\\\")\"},{\"features\":[\"Wdk_Foundation\",\"Wdk_Storage_FileSystem\",\"Wdk_System_IO\",\"Win32_Foundation\",\"Win32_Networking_WinSock\",\"Win32_Storage_FileSystem\",\"Win32_Security\",\"Win32_System_IO\",\"Win32_System_WindowsProgramming\"],\"name\":\"windows-sys\",\"req\":\"^0.61\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[\"log\"],\"net\":[],\"os-ext\":[\"os-poll\",\"windows-sys/Win32_System_Pipes\",\"windows-sys/Win32_Security\"],\"os-poll\":[]}}", - "monostate-impl_0.1.13": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"default_features\":false,\"features\":[\"parsing\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0.59\"}],\"features\":{}}", - "monostate_0.1.13": "{\"dependencies\":[{\"name\":\"monostate-impl\",\"req\":\"=0.1.13\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.166\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.99\"}],\"features\":{}}", - "multimap_0.10.1": "{\"dependencies\":[{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"serde_impl\"],\"serde_impl\":[\"serde\"]}}", - "nom_7.1.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2.3\"},{\"default_features\":false,\"name\":\"minimal-lexical\",\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.0.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"docsrs\":[],\"std\":[\"alloc\",\"memchr/std\",\"minimal-lexical/std\"]}}", - "object_0.37.3": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"crc32fast\",\"optional\":true,\"req\":\"^1.2\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"default-hasher\"],\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.15.0\"},{\"default_features\":false,\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.0\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2.4.1\"},{\"name\":\"ruzstd\",\"optional\":true,\"req\":\"^0.8.1\"},{\"default_features\":false,\"name\":\"wasmparser\",\"optional\":true,\"req\":\"^0.236.0\"}],\"features\":{\"all\":[\"read\",\"write\",\"build\",\"std\",\"compression\",\"wasm\"],\"archive\":[],\"build\":[\"build_core\",\"write_std\",\"elf\"],\"build_core\":[\"read_core\",\"write_core\"],\"cargo-all\":[],\"coff\":[],\"compression\":[\"dep:flate2\",\"dep:ruzstd\",\"std\"],\"default\":[\"read\",\"compression\"],\"doc\":[\"read_core\",\"write_std\",\"build_core\",\"std\",\"compression\",\"archive\",\"coff\",\"elf\",\"macho\",\"pe\",\"wasm\",\"xcoff\"],\"elf\":[],\"macho\":[],\"pe\":[\"coff\"],\"read\":[\"read_core\",\"archive\",\"coff\",\"elf\",\"macho\",\"pe\",\"xcoff\",\"unaligned\"],\"read_core\":[],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"memchr/rustc-dep-of-std\"],\"std\":[\"memchr/std\"],\"unaligned\":[],\"unstable\":[],\"unstable-all\":[\"all\",\"unstable\"],\"wasm\":[\"dep:wasmparser\"],\"write\":[\"write_std\",\"coff\",\"elf\",\"macho\",\"pe\",\"xcoff\"],\"write_core\":[\"dep:crc32fast\",\"dep:indexmap\",\"dep:hashbrown\"],\"write_std\":[\"write_core\",\"std\",\"indexmap?/std\",\"crc32fast?/std\"],\"xcoff\":[]}}", - "once_cell_1.20.2": "{\"dependencies\":[{\"name\":\"critical-section\",\"optional\":true,\"req\":\"^1.1.3\"},{\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"critical-section\",\"req\":\"^1.1.3\"},{\"default_features\":false,\"name\":\"parking_lot_core\",\"optional\":true,\"req\":\"^0.9.10\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.10.6\"}],\"features\":{\"alloc\":[\"race\"],\"atomic-polyfill\":[\"critical-section\"],\"critical-section\":[\"dep:critical-section\",\"portable-atomic\"],\"default\":[\"std\"],\"parking_lot\":[\"dep:parking_lot_core\"],\"portable-atomic\":[\"dep:portable-atomic\"],\"race\":[],\"std\":[\"alloc\"],\"unstable\":[]}}", - "once_cell_1.21.4": "{\"dependencies\":[{\"name\":\"critical-section\",\"optional\":true,\"req\":\"^1.1.3\"},{\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"critical-section\",\"req\":\"^1.1.3\"},{\"default_features\":false,\"name\":\"parking_lot_core\",\"optional\":true,\"req\":\"^0.9.10\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.10.6\"}],\"features\":{\"alloc\":[\"race\"],\"atomic-polyfill\":[\"critical-section\"],\"critical-section\":[\"dep:critical-section\",\"portable-atomic\"],\"default\":[\"std\"],\"parking_lot\":[\"dep:parking_lot_core\"],\"portable-atomic\":[\"dep:portable-atomic\"],\"race\":[],\"std\":[\"alloc\"],\"unstable\":[]}}", - "onig_6.5.1": "{\"dependencies\":[{\"name\":\"bitflags\",\"req\":\"^2.4.0\"},{\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(windows)\"},{\"name\":\"once_cell\",\"req\":\"^1.12\"},{\"default_features\":false,\"name\":\"onig_sys\",\"req\":\"^69.9.1\"}],\"features\":{\"default\":[\"generate\"],\"generate\":[\"onig_sys/generate\"],\"posix-api\":[\"onig_sys/posix-api\"],\"print-debug\":[\"onig_sys/print-debug\"],\"std-pattern\":[]}}", - "onig_sys_69.9.1": "{\"dependencies\":[{\"features\":[\"runtime\"],\"kind\":\"build\",\"name\":\"bindgen\",\"optional\":true,\"req\":\"^0.71\"},{\"kind\":\"build\",\"name\":\"cc\",\"req\":\"^1.0\"},{\"kind\":\"build\",\"name\":\"pkg-config\",\"req\":\"^0.3.16\"}],\"features\":{\"default\":[\"generate\"],\"generate\":[\"bindgen\"],\"posix-api\":[],\"print-debug\":[]}}", - "parking_lot_0.12.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.3.3\"},{\"name\":\"lock_api\",\"req\":\"^0.4.14\"},{\"name\":\"parking_lot_core\",\"req\":\"^0.9.12\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.3\"}],\"features\":{\"arc_lock\":[\"lock_api/arc_lock\"],\"deadlock_detection\":[\"parking_lot_core/deadlock_detection\"],\"default\":[],\"hardware-lock-elision\":[],\"nightly\":[\"parking_lot_core/nightly\",\"lock_api/nightly\"],\"owning_ref\":[\"lock_api/owning_ref\"],\"send_guard\":[],\"serde\":[\"lock_api/serde\"]}}", - "parking_lot_core_0.9.12": "{\"dependencies\":[{\"name\":\"backtrace\",\"optional\":true,\"req\":\"^0.3.60\"},{\"name\":\"cfg-if\",\"req\":\"^1.0.0\"},{\"name\":\"libc\",\"req\":\"^0.2.95\",\"target\":\"cfg(unix)\"},{\"name\":\"petgraph\",\"optional\":true,\"req\":\"^0.6.0\"},{\"name\":\"redox_syscall\",\"req\":\"^0.5\",\"target\":\"cfg(target_os = \\\"redox\\\")\"},{\"name\":\"smallvec\",\"req\":\"^1.6.1\"},{\"name\":\"windows-link\",\"req\":\"^0.2.0\",\"target\":\"cfg(windows)\"}],\"features\":{\"deadlock_detection\":[\"petgraph\",\"backtrace\"],\"nightly\":[]}}", - "paste_1.0.15": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"paste-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", - "percent-encoding_2.3.2": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", - "petgraph_0.6.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"ahash\",\"req\":\"^0.7.2\"},{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.3.3\"},{\"kind\":\"dev\",\"name\":\"defmac\",\"req\":\"^0.2.1\"},{\"default_features\":false,\"name\":\"fixedbitset\",\"req\":\"^0.4.0\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2.1\"},{\"name\":\"indexmap\",\"req\":\"^2.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.12.1\"},{\"kind\":\"dev\",\"name\":\"odds\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.5.5\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.5.3\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1.0\"}],\"features\":{\"all\":[\"unstable\",\"quickcheck\",\"matrix_graph\",\"stable_graph\",\"graphmap\",\"rayon\"],\"default\":[\"graphmap\",\"stable_graph\",\"matrix_graph\"],\"generate\":[],\"graphmap\":[],\"matrix_graph\":[],\"rayon\":[\"dep:rayon\",\"indexmap/rayon\"],\"serde-1\":[\"serde\",\"serde_derive\"],\"stable_graph\":[],\"unstable\":[\"generate\"]}}", - "pin-project-internal_1.1.11": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1.0.25\"},{\"default_features\":false,\"features\":[\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\",\"full\",\"visit-mut\"],\"name\":\"syn\",\"req\":\"^2.0.1\"}],\"features\":{}}", - "pin-project-lite_0.2.17": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1\"}],\"features\":{}}", - "pin-project_1.1.11": "{\"dependencies\":[{\"name\":\"pin-project-internal\",\"req\":\"=1.1.11\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1\"}],\"features\":{}}", - "pin-utils_0.1.0": "{\"dependencies\":[],\"features\":{}}", - "pkg-config_0.3.32": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1\"}],\"features\":{}}", - "ppv-lite86_0.2.20": "{\"dependencies\":[{\"features\":[\"simd\",\"derive\"],\"name\":\"zerocopy\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"std\"],\"no_simd\":[],\"simd\":[],\"std\":[]}}", - "ppv-lite86_0.2.21": "{\"dependencies\":[{\"features\":[\"simd\"],\"name\":\"zerocopy\",\"req\":\"^0.8.23\"}],\"features\":{\"default\":[\"std\"],\"no_simd\":[],\"simd\":[],\"std\":[]}}", - "prettyplease_0.2.37": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"default_features\":false,\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.105\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"extra-traits\",\"parsing\",\"printing\",\"visit-mut\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.105\"}],\"features\":{\"verbatim\":[\"syn/parsing\"]}}", - "proc-macro2_1.0.106": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4\"},{\"name\":\"unicode-ident\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"proc-macro\"],\"nightly\":[],\"proc-macro\":[],\"span-locations\":[]}}", - "proc-macro2_1.0.92": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4\"},{\"name\":\"unicode-ident\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"proc-macro\"],\"nightly\":[],\"proc-macro\":[],\"span-locations\":[]}}", - "prost-build_0.13.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"heck\",\"req\":\">=0.4, <=0.5\"},{\"default_features\":false,\"features\":[\"use_alloc\"],\"name\":\"itertools\",\"req\":\">=0.10, <=0.13\"},{\"name\":\"log\",\"req\":\"^0.4.4\"},{\"default_features\":false,\"name\":\"multimap\",\"req\":\">=0.8, <=0.10\"},{\"name\":\"once_cell\",\"req\":\"^1.17.1\"},{\"default_features\":false,\"name\":\"petgraph\",\"req\":\"^0.6\"},{\"name\":\"prettyplease\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"name\":\"prost\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-types\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"pulldown-cmark\",\"optional\":true,\"req\":\"^0.9.1\"},{\"name\":\"pulldown-cmark-to-cmark\",\"optional\":true,\"req\":\"^10.0.1\"},{\"default_features\":false,\"features\":[\"std\",\"unicode-bool\"],\"name\":\"regex\",\"req\":\"^1.8.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"optional\":true,\"req\":\"^2\"},{\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{\"cleanup-markdown\":[\"dep:pulldown-cmark\",\"dep:pulldown-cmark-to-cmark\"],\"default\":[\"format\"],\"format\":[\"dep:prettyplease\",\"dep:syn\"]}}", - "prost-derive_0.13.1": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.1\"},{\"name\":\"itertools\",\"req\":\">=0.10.1, <=0.13\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2\"}],\"features\":{}}", - "prost-types_0.13.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"prost-derive\"],\"name\":\"prost\",\"req\":\"^0.13.1\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"prost/std\"]}}", - "prost_0.13.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"name\":\"prost-derive\",\"optional\":true,\"req\":\"^0.13.1\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"derive\",\"std\"],\"derive\":[\"dep:prost-derive\"],\"no-recursion-limit\":[],\"prost-derive\":[\"derive\"],\"std\":[]}}", - "protoc-gen-prost_0.4.0": "{\"dependencies\":[{\"name\":\"once_cell\",\"req\":\"^1.10.0\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"prost\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-build\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-types\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"regex\",\"req\":\"^1.5.5\"}],\"features\":{}}", - "protoc-gen-tonic_0.4.1": "{\"dependencies\":[{\"name\":\"heck\",\"req\":\"^0.5.0\"},{\"name\":\"prettyplease\",\"req\":\"^0.2.9\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"prost\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-build\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-types\",\"req\":\"^0.13.1\"},{\"name\":\"protoc-gen-prost\",\"req\":\"^0.4.0\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"regex\",\"req\":\"^1.5.5\"},{\"features\":[\"parsing\",\"full\"],\"name\":\"syn\",\"req\":\"^2.0.22\"},{\"name\":\"tonic-build\",\"req\":\"^0.12.0\"}],\"features\":{}}", - "quote_1.0.37": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.66\"}],\"features\":{\"default\":[\"proc-macro\"],\"proc-macro\":[\"proc-macro2/proc-macro\"]}}", - "quote_1.0.45": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"}],\"features\":{\"default\":[\"proc-macro\"],\"proc-macro\":[\"proc-macro2/proc-macro\"]}}", - "r-efi_5.3.0": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"efiapi\":[],\"examples\":[\"native\"],\"native\":[],\"rustc-dep-of-std\":[\"core\"]}}", - "r-efi_6.0.0": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"native\":[],\"rustc-dep-of-std\":[\"core\"]}}", - "rand_0.8.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.2.1\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.22\",\"target\":\"cfg(unix)\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.4\"},{\"features\":[\"into_bits\"],\"name\":\"packed_simd\",\"optional\":true,\"package\":\"packed_simd_2\",\"req\":\"^0.3.7\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"rand_core\",\"req\":\"^0.6.0\"},{\"kind\":\"dev\",\"name\":\"rand_pcg\",\"req\":\"^0.3.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"}],\"features\":{\"alloc\":[\"rand_core/alloc\"],\"default\":[\"std\",\"std_rng\"],\"getrandom\":[\"rand_core/getrandom\"],\"min_const_gen\":[],\"nightly\":[],\"serde1\":[\"serde\",\"rand_core/serde1\"],\"simd_support\":[\"packed_simd\"],\"small_rng\":[],\"std\":[\"rand_core/std\",\"rand_chacha/std\",\"alloc\",\"getrandom\",\"libc\"],\"std_rng\":[\"rand_chacha\"]}}", - "rand_0.9.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.2.1\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.4\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"optional\":true,\"req\":\"^0.9.0\"},{\"default_features\":false,\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rand_pcg\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.7\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"std_rng\",\"os_rng\",\"small_rng\",\"thread_rng\"],\"log\":[\"dep:log\"],\"nightly\":[],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\",\"rand_core/serde\"],\"simd_support\":[],\"small_rng\":[],\"std\":[\"rand_core/std\",\"rand_chacha?/std\",\"alloc\"],\"std_rng\":[\"dep:rand_chacha\"],\"thread_rng\":[\"std\",\"std_rng\",\"os_rng\"],\"unbiased\":[]}}", - "rand_chacha_0.3.1": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"ppv-lite86\",\"req\":\"^0.2.8\"},{\"name\":\"rand_core\",\"req\":\"^0.6.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"serde1\":[\"serde\"],\"simd\":[],\"std\":[\"ppv-lite86/std\"]}}", - "rand_chacha_0.9.0": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"ppv-lite86\",\"req\":\"^0.2.14\"},{\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"features\":[\"os_rng\"],\"kind\":\"dev\",\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\"],\"std\":[\"ppv-lite86/std\",\"rand_core/std\"]}}", - "rand_core_0.6.4": "{\"dependencies\":[{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.2\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"serde1\":[\"serde\"],\"std\":[\"alloc\",\"getrandom\",\"getrandom/std\"]}}", - "rand_core_0.9.3": "{\"dependencies\":[{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.3.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"os_rng\":[\"dep:getrandom\"],\"serde\":[\"dep:serde\"],\"std\":[\"getrandom?/std\"]}}", - "rayon-cond_0.4.0": "{\"dependencies\":[{\"name\":\"either\",\"req\":\"^1.6\"},{\"name\":\"itertools\",\"req\":\"^0.14\"},{\"name\":\"rayon\",\"req\":\"^1.7\"}],\"features\":{}}", - "rayon-core_1.12.1": "{\"dependencies\":[{\"name\":\"crossbeam-deque\",\"req\":\"^0.8.1\"},{\"name\":\"crossbeam-utils\",\"req\":\"^0.8.0\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"scoped-tls\",\"req\":\"^1.0\"},{\"name\":\"wasm_sync\",\"optional\":true,\"req\":\"^0.1.0\"}],\"features\":{\"web_spin_lock\":[\"dep:wasm_sync\"]}}", - "rayon_1.10.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"name\":\"rayon-core\",\"req\":\"^1.12.1\"},{\"name\":\"wasm_sync\",\"optional\":true,\"req\":\"^0.1.0\"}],\"features\":{\"web_spin_lock\":[\"dep:wasm_sync\",\"rayon-core/web_spin_lock\"]}}", - "redox_syscall_0.5.18": "{\"dependencies\":[{\"name\":\"bitflags\",\"req\":\"^2.4\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"loom\",\"req\":\"^0.7\",\"target\":\"cfg(loom)\"}],\"features\":{\"default\":[\"userspace\"],\"rustc-dep-of-std\":[\"core\",\"bitflags/rustc-dep-of-std\"],\"std\":[],\"userspace\":[]}}", - "regex-automata_0.4.14": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"bstr\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.14\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"optional\":true,\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"syntax\",\"perf\",\"unicode\",\"meta\",\"nfa\",\"dfa\",\"hybrid\"],\"dfa\":[\"dfa-build\",\"dfa-search\",\"dfa-onepass\"],\"dfa-build\":[\"nfa-thompson\",\"dfa-search\"],\"dfa-onepass\":[\"nfa-thompson\"],\"dfa-search\":[],\"hybrid\":[\"alloc\",\"nfa-thompson\"],\"internal-instrument\":[\"internal-instrument-pikevm\"],\"internal-instrument-pikevm\":[\"logging\",\"std\"],\"logging\":[\"dep:log\",\"aho-corasick?/logging\",\"memchr?/logging\"],\"meta\":[\"syntax\",\"nfa-pikevm\"],\"nfa\":[\"nfa-thompson\",\"nfa-pikevm\",\"nfa-backtrack\"],\"nfa-backtrack\":[\"nfa-thompson\"],\"nfa-pikevm\":[\"nfa-thompson\"],\"nfa-thompson\":[\"alloc\"],\"perf\":[\"perf-inline\",\"perf-literal\"],\"perf-inline\":[],\"perf-literal\":[\"perf-literal-substring\",\"perf-literal-multisubstring\"],\"perf-literal-multisubstring\":[\"dep:aho-corasick\"],\"perf-literal-substring\":[\"aho-corasick?/perf-literal\",\"dep:memchr\"],\"std\":[\"regex-syntax?/std\",\"memchr?/std\",\"aho-corasick?/std\",\"alloc\"],\"syntax\":[\"dep:regex-syntax\",\"alloc\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"unicode-word-boundary\",\"regex-syntax?/unicode\"],\"unicode-age\":[\"regex-syntax?/unicode-age\"],\"unicode-bool\":[\"regex-syntax?/unicode-bool\"],\"unicode-case\":[\"regex-syntax?/unicode-case\"],\"unicode-gencat\":[\"regex-syntax?/unicode-gencat\"],\"unicode-perl\":[\"regex-syntax?/unicode-perl\"],\"unicode-script\":[\"regex-syntax?/unicode-script\"],\"unicode-segment\":[\"regex-syntax?/unicode-segment\"],\"unicode-word-boundary\":[]}}", - "regex-automata_0.4.9": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"bstr\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.14\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"optional\":true,\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"syntax\",\"perf\",\"unicode\",\"meta\",\"nfa\",\"dfa\",\"hybrid\"],\"dfa\":[\"dfa-build\",\"dfa-search\",\"dfa-onepass\"],\"dfa-build\":[\"nfa-thompson\",\"dfa-search\"],\"dfa-onepass\":[\"nfa-thompson\"],\"dfa-search\":[],\"hybrid\":[\"alloc\",\"nfa-thompson\"],\"internal-instrument\":[\"internal-instrument-pikevm\"],\"internal-instrument-pikevm\":[\"logging\",\"std\"],\"logging\":[\"dep:log\",\"aho-corasick?/logging\",\"memchr?/logging\"],\"meta\":[\"syntax\",\"nfa-pikevm\"],\"nfa\":[\"nfa-thompson\",\"nfa-pikevm\",\"nfa-backtrack\"],\"nfa-backtrack\":[\"nfa-thompson\"],\"nfa-pikevm\":[\"nfa-thompson\"],\"nfa-thompson\":[\"alloc\"],\"perf\":[\"perf-inline\",\"perf-literal\"],\"perf-inline\":[],\"perf-literal\":[\"perf-literal-substring\",\"perf-literal-multisubstring\"],\"perf-literal-multisubstring\":[\"std\",\"dep:aho-corasick\"],\"perf-literal-substring\":[\"aho-corasick?/perf-literal\",\"dep:memchr\"],\"std\":[\"regex-syntax?/std\",\"memchr?/std\",\"aho-corasick?/std\",\"alloc\"],\"syntax\":[\"dep:regex-syntax\",\"alloc\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"unicode-word-boundary\",\"regex-syntax?/unicode\"],\"unicode-age\":[\"regex-syntax?/unicode-age\"],\"unicode-bool\":[\"regex-syntax?/unicode-bool\"],\"unicode-case\":[\"regex-syntax?/unicode-case\"],\"unicode-gencat\":[\"regex-syntax?/unicode-gencat\"],\"unicode-perl\":[\"regex-syntax?/unicode-perl\"],\"unicode-script\":[\"regex-syntax?/unicode-script\"],\"unicode-segment\":[\"regex-syntax?/unicode-segment\"],\"unicode-word-boundary\":[]}}", - "regex-syntax_0.8.10": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.3.0\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"default\":[\"std\",\"unicode\"],\"std\":[],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\"],\"unicode-age\":[],\"unicode-bool\":[],\"unicode-case\":[],\"unicode-gencat\":[],\"unicode-perl\":[],\"unicode-script\":[],\"unicode-segment\":[]}}", - "regex-syntax_0.8.5": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.3.0\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"default\":[\"std\",\"unicode\"],\"std\":[],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\"],\"unicode-age\":[],\"unicode-bool\":[],\"unicode-case\":[],\"unicode-gencat\":[],\"unicode-perl\":[],\"unicode-script\":[],\"unicode-segment\":[]}}", - "regex_1.11.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.17.1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"features\":[\"alloc\",\"syntax\",\"meta\",\"nfa-pikevm\"],\"name\":\"regex-automata\",\"req\":\"^0.4.8\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"default\":[\"std\",\"perf\",\"unicode\",\"regex-syntax/default\"],\"logging\":[\"aho-corasick?/logging\",\"memchr?/logging\",\"regex-automata/logging\"],\"pattern\":[],\"perf\":[\"perf-cache\",\"perf-dfa\",\"perf-onepass\",\"perf-backtrack\",\"perf-inline\",\"perf-literal\"],\"perf-backtrack\":[\"regex-automata/nfa-backtrack\"],\"perf-cache\":[],\"perf-dfa\":[\"regex-automata/hybrid\"],\"perf-dfa-full\":[\"regex-automata/dfa-build\",\"regex-automata/dfa-search\"],\"perf-inline\":[\"regex-automata/perf-inline\"],\"perf-literal\":[\"dep:aho-corasick\",\"dep:memchr\",\"regex-automata/perf-literal\"],\"perf-onepass\":[\"regex-automata/dfa-onepass\"],\"std\":[\"aho-corasick?/std\",\"memchr?/std\",\"regex-automata/std\",\"regex-syntax/std\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"regex-automata/unicode\",\"regex-syntax/unicode\"],\"unicode-age\":[\"regex-automata/unicode-age\",\"regex-syntax/unicode-age\"],\"unicode-bool\":[\"regex-automata/unicode-bool\",\"regex-syntax/unicode-bool\"],\"unicode-case\":[\"regex-automata/unicode-case\",\"regex-syntax/unicode-case\"],\"unicode-gencat\":[\"regex-automata/unicode-gencat\",\"regex-syntax/unicode-gencat\"],\"unicode-perl\":[\"regex-automata/unicode-perl\",\"regex-automata/unicode-word-boundary\",\"regex-syntax/unicode-perl\"],\"unicode-script\":[\"regex-automata/unicode-script\",\"regex-syntax/unicode-script\"],\"unicode-segment\":[\"regex-automata/unicode-segment\",\"regex-syntax/unicode-segment\"],\"unstable\":[\"pattern\"],\"use_std\":[\"std\"]}}", - "regex_1.12.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"features\":[\"alloc\",\"syntax\",\"meta\",\"nfa-pikevm\"],\"name\":\"regex-automata\",\"req\":\"^0.4.12\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"default\":[\"std\",\"perf\",\"unicode\",\"regex-syntax/default\"],\"logging\":[\"aho-corasick?/logging\",\"memchr?/logging\",\"regex-automata/logging\"],\"pattern\":[],\"perf\":[\"perf-cache\",\"perf-dfa\",\"perf-onepass\",\"perf-backtrack\",\"perf-inline\",\"perf-literal\"],\"perf-backtrack\":[\"regex-automata/nfa-backtrack\"],\"perf-cache\":[],\"perf-dfa\":[\"regex-automata/hybrid\"],\"perf-dfa-full\":[\"regex-automata/dfa-build\",\"regex-automata/dfa-search\"],\"perf-inline\":[\"regex-automata/perf-inline\"],\"perf-literal\":[\"dep:aho-corasick\",\"dep:memchr\",\"regex-automata/perf-literal\"],\"perf-onepass\":[\"regex-automata/dfa-onepass\"],\"std\":[\"aho-corasick?/std\",\"memchr?/std\",\"regex-automata/std\",\"regex-syntax/std\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"regex-automata/unicode\",\"regex-syntax/unicode\"],\"unicode-age\":[\"regex-automata/unicode-age\",\"regex-syntax/unicode-age\"],\"unicode-bool\":[\"regex-automata/unicode-bool\",\"regex-syntax/unicode-bool\"],\"unicode-case\":[\"regex-automata/unicode-case\",\"regex-syntax/unicode-case\"],\"unicode-gencat\":[\"regex-automata/unicode-gencat\",\"regex-syntax/unicode-gencat\"],\"unicode-perl\":[\"regex-automata/unicode-perl\",\"regex-automata/unicode-word-boundary\",\"regex-syntax/unicode-perl\"],\"unicode-script\":[\"regex-automata/unicode-script\",\"regex-syntax/unicode-script\"],\"unicode-segment\":[\"regex-automata/unicode-segment\",\"regex-syntax/unicode-segment\"],\"unstable\":[\"pattern\"],\"use_std\":[\"std\"]}}", - "rustc-demangle_0.1.27": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"compiler_builtins\":[],\"rustc-dep-of-std\":[\"core\"],\"std\":[]}}", - "rustix_1.1.4": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bitflags\",\"req\":\"^2.4.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4\",\"target\":\"cfg(all(criterion, not(any(target_os = \\\"emscripten\\\", target_os = \\\"wasi\\\"))))\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.182\",\"target\":\"cfg(all(not(windows), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.182\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.171\"},{\"default_features\":false,\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(all(not(windows), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(windows)\"},{\"default_features\":false,\"name\":\"libc_errno\",\"optional\":true,\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\"},{\"default_features\":false,\"features\":[\"general\",\"ioctl\",\"no_std\"],\"name\":\"linux-raw-sys\",\"req\":\"^0.12\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"features\":[\"auxvec\",\"general\",\"errno\",\"ioctl\",\"no_std\",\"elf\"],\"name\":\"linux-raw-sys\",\"req\":\"^0.12\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"kind\":\"dev\",\"name\":\"memoffset\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.20.3\",\"target\":\"cfg(windows)\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"serial_test\",\"req\":\"^2.0.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.5.0\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\"],\"name\":\"windows-sys\",\"req\":\">=0.52, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"all-apis\":[\"event\",\"fs\",\"io_uring\",\"mm\",\"mount\",\"net\",\"param\",\"pipe\",\"process\",\"pty\",\"rand\",\"runtime\",\"shm\",\"stdio\",\"system\",\"termios\",\"thread\",\"time\"],\"alloc\":[],\"default\":[\"std\"],\"event\":[],\"fs\":[],\"io_uring\":[\"event\",\"fs\",\"net\",\"thread\",\"linux-raw-sys/io_uring\"],\"linux_4_11\":[],\"linux_5_1\":[\"linux_4_11\"],\"linux_5_11\":[\"linux_5_1\"],\"linux_latest\":[\"linux_5_11\"],\"mm\":[],\"mount\":[],\"net\":[\"linux-raw-sys/net\",\"linux-raw-sys/netlink\",\"linux-raw-sys/if_ether\",\"linux-raw-sys/xdp\"],\"param\":[],\"pipe\":[],\"process\":[\"linux-raw-sys/prctl\"],\"pty\":[\"fs\"],\"rand\":[],\"runtime\":[\"linux-raw-sys/prctl\"],\"rustc-dep-of-std\":[\"core\",\"rustc-std-workspace-alloc\",\"linux-raw-sys/rustc-dep-of-std\",\"bitflags/rustc-dep-of-std\"],\"shm\":[\"fs\"],\"std\":[\"bitflags/std\",\"alloc\",\"libc?/std\",\"libc_errno?/std\"],\"stdio\":[],\"system\":[\"linux-raw-sys/system\"],\"termios\":[],\"thread\":[\"linux-raw-sys/prctl\"],\"time\":[],\"try_close\":[],\"use-explicitly-provided-auxv\":[],\"use-libc\":[\"libc_errno\",\"libc\"],\"use-libc-auxv\":[]}}", - "rustversion_1.0.21": "{\"dependencies\":[{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", - "rustversion_1.0.22": "{\"dependencies\":[{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", - "ryu_1.0.18": "{\"dependencies\":[{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"num_cpus\",\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"}],\"features\":{\"small\":[]}}", - "scopeguard_1.2.0": "{\"dependencies\":[],\"features\":{\"default\":[\"use_std\"],\"use_std\":[]}}", - "semver_1.0.27": "{\"dependencies\":[{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"package\":\"serde_core\",\"req\":\"^1.0.220\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"}],\"features\":{\"default\":[\"std\"],\"serde\":[\"dep:serde\"],\"std\":[]}}", - "serde_1.0.216": "{\"dependencies\":[{\"name\":\"serde_derive\",\"req\":\"=1.0.216\",\"target\":\"cfg(any())\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"derive\":[\"serde_derive\"],\"rc\":[],\"std\":[],\"unstable\":[]}}", - "serde_1.0.228": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"result\"],\"name\":\"serde_core\",\"req\":\"=1.0.228\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[\"serde_core/alloc\"],\"default\":[\"std\"],\"derive\":[\"serde_derive\"],\"rc\":[\"serde_core/rc\"],\"std\":[\"serde_core/std\"],\"unstable\":[\"serde_core/unstable\"]}}", - "serde_core_1.0.228": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"name\":\"serde_derive\",\"req\":\"=1.0.228\",\"target\":\"cfg(any())\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"result\"],\"rc\":[],\"result\":[],\"std\":[],\"unstable\":[]}}", - "serde_derive_1.0.216": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"derive\",\"parsing\",\"printing\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0.81\"}],\"features\":{\"default\":[],\"deserialize_in_place\":[]}}", - "serde_derive_1.0.228": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"derive\",\"parsing\",\"printing\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0.81\"}],\"features\":{\"default\":[],\"deserialize_in_place\":[]}}", - "serde_json_1.0.134": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0.11\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.2.3\"},{\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2.0.2\"},{\"name\":\"itoa\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"name\":\"ryu\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"kind\":\"dev\",\"name\":\"serde_bytes\",\"req\":\"^0.11.10\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_stacker\",\"req\":\"^0.1.8\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{\"alloc\":[\"serde/alloc\"],\"arbitrary_precision\":[],\"default\":[\"std\"],\"float_roundtrip\":[],\"preserve_order\":[\"indexmap\",\"std\"],\"raw_value\":[],\"std\":[\"memchr/std\",\"serde/std\"],\"unbounded_depth\":[]}}", - "serde_json_1.0.149": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0.11\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.2.3\"},{\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2.0.2\"},{\"name\":\"itoa\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"kind\":\"dev\",\"name\":\"serde_bytes\",\"req\":\"^0.11.10\"},{\"default_features\":false,\"name\":\"serde_core\",\"req\":\"^1.0.220\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_stacker\",\"req\":\"^0.1.8\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"},{\"name\":\"zmij\",\"req\":\"^1.0\"}],\"features\":{\"alloc\":[\"serde_core/alloc\"],\"arbitrary_precision\":[],\"default\":[\"std\"],\"float_roundtrip\":[],\"preserve_order\":[\"indexmap\",\"std\"],\"raw_value\":[],\"std\":[\"memchr/std\",\"serde_core/std\"],\"unbounded_depth\":[]}}", - "shlex_1.3.0": "{\"dependencies\":[],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "signal-hook-registry_1.4.8": "{\"dependencies\":[{\"name\":\"errno\",\"req\":\">=0.2, <0.4\"},{\"name\":\"libc\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"signal-hook\",\"req\":\"~0.3\"}],\"features\":{}}", - "slab_0.4.12": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.95\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "smallvec_1.13.2": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.0.1\"},{\"kind\":\"dev\",\"name\":\"debugger_test\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"debugger_test_parser\",\"req\":\"^0.1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"const_generics\":[],\"const_new\":[\"const_generics\"],\"debugger_visualizer\":[],\"drain_filter\":[],\"drain_keep_rest\":[\"drain_filter\"],\"may_dangle\":[],\"specialization\":[],\"union\":[],\"write\":[]}}", - "smallvec_1.15.1": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"name\":\"bincode\",\"optional\":true,\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"bincode1\",\"package\":\"bincode\",\"req\":\"^1.0.1\"},{\"kind\":\"dev\",\"name\":\"debugger_test\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"debugger_test_parser\",\"req\":\"^0.1.0\"},{\"default_features\":false,\"name\":\"malloc_size_of\",\"optional\":true,\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"name\":\"unty\",\"optional\":true,\"req\":\"^0.0.4\"}],\"features\":{\"const_generics\":[],\"const_new\":[\"const_generics\"],\"debugger_visualizer\":[],\"drain_filter\":[],\"drain_keep_rest\":[\"drain_filter\"],\"impl_bincode\":[\"bincode\",\"unty\"],\"may_dangle\":[],\"specialization\":[],\"union\":[],\"write\":[]}}", - "socket2_0.5.10": "{\"dependencies\":[{\"name\":\"libc\",\"req\":\"^0.2.171\",\"target\":\"cfg(unix)\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\",\"Win32_System_IO\",\"Win32_System_Threading\",\"Win32_System_WindowsProgramming\"],\"name\":\"windows-sys\",\"req\":\"^0.52\",\"target\":\"cfg(windows)\"}],\"features\":{\"all\":[]}}", - "socket2_0.6.3": "{\"dependencies\":[{\"name\":\"libc\",\"req\":\"^0.2.172\",\"target\":\"cfg(any(unix, target_os = \\\"wasi\\\"))\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\",\"Win32_System_IO\",\"Win32_System_Threading\",\"Win32_System_WindowsProgramming\"],\"name\":\"windows-sys\",\"req\":\">=0.60, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"all\":[]}}", - "spm_precompiled_0.1.4": "{\"dependencies\":[{\"name\":\"base64\",\"req\":\"^0.13\"},{\"name\":\"nom\",\"req\":\"^7.1.1\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"unicode-segmentation\",\"req\":\"^1.9\"}],\"features\":{}}", - "static_assertions_1.1.0": "{\"dependencies\":[],\"features\":{\"nightly\":[]}}", - "strsim_0.11.1": "{\"dependencies\":[],\"features\":{}}", - "syn_2.0.117": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"insta\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.91\"},{\"default_features\":false,\"name\":\"quote\",\"optional\":true,\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1\"},{\"features\":[\"blocking\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.13\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"syn-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4.16\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"termcolor\",\"req\":\"^1\"},{\"name\":\"unicode-ident\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\",\"target\":\"cfg(not(miri))\"}],\"features\":{\"clone-impls\":[],\"default\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\"],\"derive\":[],\"extra-traits\":[],\"fold\":[],\"full\":[],\"parsing\":[],\"printing\":[\"dep:quote\"],\"proc-macro\":[\"proc-macro2/proc-macro\",\"quote?/proc-macro\"],\"test\":[\"syn-test-suite/all-features\"],\"visit\":[],\"visit-mut\":[]}}", - "syn_2.0.91": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"insta\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.91\"},{\"default_features\":false,\"name\":\"quote\",\"optional\":true,\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1\"},{\"features\":[\"blocking\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.12\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"syn-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4.16\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"termcolor\",\"req\":\"^1\"},{\"name\":\"unicode-ident\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\",\"target\":\"cfg(not(miri))\"}],\"features\":{\"clone-impls\":[],\"default\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\"],\"derive\":[],\"extra-traits\":[],\"fold\":[],\"full\":[],\"parsing\":[],\"printing\":[\"dep:quote\"],\"proc-macro\":[\"proc-macro2/proc-macro\",\"quote?/proc-macro\"],\"test\":[\"syn-test-suite/all-features\"],\"visit\":[],\"visit-mut\":[]}}", - "sync_wrapper_1.0.2": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"}],\"features\":{\"futures\":[\"futures-core\"]}}", - "tempfile_3.27.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"name\":\"fastrand\",\"req\":\"^2.1.1\"},{\"default_features\":false,\"name\":\"getrandom\",\"optional\":true,\"req\":\">=0.3.0, <0.5\",\"target\":\"cfg(any(unix, windows, target_os = \\\"wasi\\\"))\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"once_cell\",\"req\":\"^1.19.0\"},{\"features\":[\"fs\"],\"name\":\"rustix\",\"req\":\"^1.1.4\",\"target\":\"cfg(any(unix, target_os = \\\"wasi\\\"))\"},{\"features\":[\"Win32_Storage_FileSystem\",\"Win32_Foundation\"],\"name\":\"windows-sys\",\"req\":\">=0.52, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[\"getrandom\"],\"nightly\":[]}}", - "thiserror-impl_2.0.12": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"name\":\"syn\",\"req\":\"^2.0.87\"}],\"features\":{}}", - "thiserror_2.0.12": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.73\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"name\":\"thiserror-impl\",\"req\":\"=2.0.12\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "tokenizers_0.22.2": "{\"dependencies\":[{\"features\":[\"serde\"],\"name\":\"ahash\",\"req\":\"^0.8.11\"},{\"name\":\"aho-corasick\",\"req\":\"^1.1\"},{\"kind\":\"dev\",\"name\":\"assert_approx_eq\",\"req\":\"^1.1\"},{\"features\":[\"serde\"],\"name\":\"compact_str\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.6\"},{\"features\":[\"serde\"],\"name\":\"dary_heap\",\"req\":\"^0.3.6\"},{\"name\":\"derive_builder\",\"req\":\"^0.20\"},{\"default_features\":false,\"name\":\"esaxx-rs\",\"req\":\"^0.1.10\"},{\"name\":\"fancy-regex\",\"optional\":true,\"req\":\"^0.14\"},{\"name\":\"getrandom\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"ureq\"],\"name\":\"hf-hub\",\"optional\":true,\"req\":\"^0.4.1\"},{\"name\":\"indicatif\",\"optional\":true,\"req\":\"^0.18\"},{\"name\":\"itertools\",\"req\":\"^0.14\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"macro_rules_attribute\",\"req\":\"^0.2.0\"},{\"name\":\"monostate\",\"req\":\"^0.1.12\"},{\"default_features\":false,\"name\":\"onig\",\"optional\":true,\"req\":\"^6.5.1\"},{\"name\":\"paste\",\"req\":\"^1.0.14\"},{\"name\":\"rand\",\"req\":\"^0.9\"},{\"name\":\"rayon\",\"req\":\"^1.10\"},{\"name\":\"rayon-cond\",\"req\":\"^0.4\"},{\"name\":\"regex\",\"req\":\"^1.10\"},{\"name\":\"regex-syntax\",\"req\":\"^0.8\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"spm_precompiled\",\"req\":\"^0.1.3\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.10\"},{\"name\":\"thiserror\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3.18\"},{\"name\":\"unicode-normalization-alignments\",\"req\":\"^0.1\"},{\"name\":\"unicode-segmentation\",\"req\":\"^1.11\"},{\"name\":\"unicode_categories\",\"req\":\"^0.1\"}],\"features\":{\"default\":[\"progressbar\",\"onig\",\"esaxx_fast\"],\"esaxx_fast\":[\"esaxx-rs/cpp\"],\"http\":[\"hf-hub\"],\"progressbar\":[\"indicatif\"],\"rustls-tls\":[\"hf-hub?/rustls-tls\"],\"unstable_wasm\":[\"fancy-regex\",\"getrandom/wasm_js\"]}}", - "tokio-macros_2.4.0": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0.0\"}],\"features\":{}}", - "tokio-stream_0.1.15": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-stream\",\"req\":\"^0.3\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"name\":\"futures-core\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"parking_lot\",\"req\":\"^0.12.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.11\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"req\":\"^1.15.0\"},{\"features\":[\"full\",\"test-util\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.2.0\"},{\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7.0\"}],\"features\":{\"default\":[\"time\"],\"fs\":[\"tokio/fs\"],\"full\":[\"time\",\"net\",\"io-util\",\"fs\",\"sync\",\"signal\"],\"io-util\":[\"tokio/io-util\"],\"net\":[\"tokio/net\"],\"signal\":[\"tokio/signal\"],\"sync\":[\"tokio/sync\",\"tokio-util\"],\"time\":[\"tokio/time\"]}}", - "tokio-util_0.7.17": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-stream\",\"req\":\"^0.3.0\"},{\"name\":\"bytes\",\"req\":\"^1.5.0\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.0\"},{\"name\":\"futures-core\",\"req\":\"^0.3.0\"},{\"name\":\"futures-io\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"futures-sink\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"futures-test\",\"req\":\"^0.3.5\"},{\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.0\"},{\"default_features\":false,\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.15.0\"},{\"kind\":\"dev\",\"name\":\"parking_lot\",\"req\":\"^0.12.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.11\"},{\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4.4\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.1.0\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"req\":\"^1.28.0\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.29\"}],\"features\":{\"__docs_rs\":[\"futures-util\"],\"codec\":[],\"compat\":[\"futures-io\"],\"default\":[],\"full\":[\"codec\",\"compat\",\"io-util\",\"time\",\"net\",\"rt\",\"join-map\"],\"io\":[],\"io-util\":[\"io\",\"tokio/rt\",\"tokio/io-util\"],\"join-map\":[\"rt\",\"hashbrown\"],\"net\":[\"tokio/net\"],\"rt\":[\"tokio/rt\",\"tokio/sync\",\"futures-util\"],\"time\":[\"tokio/time\",\"slab\"]}}", - "tokio_1.39.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-stream\",\"req\":\"^0.3\"},{\"name\":\"backtrace\",\"req\":\"^0.3.58\",\"target\":\"cfg(tokio_taskdump)\"},{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1.0.0\"},{\"features\":[\"async-await\"],\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.0\"},{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.149\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.149\",\"target\":\"cfg(unix)\"},{\"features\":[\"futures\",\"checkpoint\"],\"kind\":\"dev\",\"name\":\"loom\",\"req\":\"^0.7\",\"target\":\"cfg(loom)\"},{\"default_features\":false,\"name\":\"mio\",\"optional\":true,\"req\":\"^1.0.1\"},{\"features\":[\"tokio\"],\"kind\":\"dev\",\"name\":\"mio-aio\",\"req\":\"^0.9.0\",\"target\":\"cfg(target_os = \\\"freebsd\\\")\"},{\"kind\":\"dev\",\"name\":\"mockall\",\"req\":\"^0.11.1\"},{\"default_features\":false,\"features\":[\"aio\",\"fs\",\"socket\"],\"kind\":\"dev\",\"name\":\"nix\",\"req\":\"^0.29.0\",\"target\":\"cfg(unix)\"},{\"name\":\"parking_lot\",\"optional\":true,\"req\":\"^0.12.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.11\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.0\",\"target\":\"cfg(not(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\")))\"},{\"name\":\"signal-hook-registry\",\"optional\":true,\"req\":\"^1.1.1\",\"target\":\"cfg(unix)\"},{\"features\":[\"all\"],\"name\":\"socket2\",\"optional\":true,\"req\":\"^0.5.5\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"kind\":\"dev\",\"name\":\"socket2\",\"req\":\"^0.5.5\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.1.0\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"name\":\"tokio-macros\",\"optional\":true,\"req\":\"~2.4.0\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.25\",\"target\":\"cfg(tokio_unstable)\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.0\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", not(target_os = \\\"wasi\\\")))\"},{\"name\":\"windows-sys\",\"optional\":true,\"req\":\"^0.52\",\"target\":\"cfg(windows)\"},{\"features\":[\"Win32_Foundation\",\"Win32_Security_Authorization\"],\"kind\":\"dev\",\"name\":\"windows-sys\",\"req\":\"^0.52\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[],\"fs\":[],\"full\":[\"fs\",\"io-util\",\"io-std\",\"macros\",\"net\",\"parking_lot\",\"process\",\"rt\",\"rt-multi-thread\",\"signal\",\"sync\",\"time\"],\"io-std\":[],\"io-util\":[\"bytes\"],\"macros\":[\"tokio-macros\"],\"net\":[\"libc\",\"mio/os-poll\",\"mio/os-ext\",\"mio/net\",\"socket2\",\"windows-sys/Win32_Foundation\",\"windows-sys/Win32_Security\",\"windows-sys/Win32_Storage_FileSystem\",\"windows-sys/Win32_System_Pipes\",\"windows-sys/Win32_System_SystemServices\"],\"process\":[\"bytes\",\"libc\",\"mio/os-poll\",\"mio/os-ext\",\"mio/net\",\"signal-hook-registry\",\"windows-sys/Win32_Foundation\",\"windows-sys/Win32_System_Threading\",\"windows-sys/Win32_System_WindowsProgramming\"],\"rt\":[],\"rt-multi-thread\":[\"rt\"],\"signal\":[\"libc\",\"mio/os-poll\",\"mio/net\",\"mio/os-ext\",\"signal-hook-registry\",\"windows-sys/Win32_Foundation\",\"windows-sys/Win32_System_Console\"],\"sync\":[],\"test-util\":[\"rt\",\"sync\",\"time\"],\"time\":[]}}", - "tonic-build_0.12.3": "{\"dependencies\":[{\"name\":\"prettyplease\",\"req\":\"^0.2\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"name\":\"prost-build\",\"optional\":true,\"req\":\"^0.13\"},{\"name\":\"prost-types\",\"optional\":true,\"req\":\"^0.13\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"name\":\"syn\",\"req\":\"^2.0\"}],\"features\":{\"cleanup-markdown\":[\"prost\",\"prost-build/cleanup-markdown\"],\"default\":[\"transport\",\"prost\"],\"prost\":[\"prost-build\",\"dep:prost-types\"],\"transport\":[]}}", - "tonic_0.12.1": "{\"dependencies\":[{\"name\":\"async-stream\",\"optional\":true,\"req\":\"^0.3\"},{\"name\":\"async-trait\",\"optional\":true,\"req\":\"^0.1.13\"},{\"default_features\":false,\"name\":\"axum\",\"optional\":true,\"req\":\"^0.7\"},{\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.5\"},{\"name\":\"bytes\",\"req\":\"^1.0\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"h2\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"http-body\",\"req\":\"^1\"},{\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"features\":[\"http1\",\"http2\"],\"name\":\"hyper\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"hyper-timeout\",\"optional\":true,\"req\":\"^0.5\"},{\"features\":[\"tokio\"],\"name\":\"hyper-util\",\"optional\":true,\"req\":\"^0.1.4\"},{\"name\":\"percent-encoding\",\"req\":\"^2.1\"},{\"name\":\"pin-project\",\"req\":\"^1.0.11\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"prost\",\"optional\":true,\"req\":\"^0.13\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"name\":\"rustls-native-certs\",\"optional\":true,\"req\":\"^0.7\"},{\"name\":\"rustls-pemfile\",\"optional\":true,\"req\":\"^2.0\"},{\"features\":[\"all\"],\"name\":\"socket2\",\"optional\":true,\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"rt\",\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"logging\",\"tls12\",\"ring\"],\"name\":\"tokio-rustls\",\"optional\":true,\"req\":\"^0.26\"},{\"default_features\":false,\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"tower\",\"optional\":true,\"req\":\"^0.4.7\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tower\",\"req\":\"^0.4.7\"},{\"name\":\"tower-layer\",\"req\":\"^0.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"name\":\"tracing\",\"req\":\"^0.1\"},{\"name\":\"webpki-roots\",\"optional\":true,\"req\":\"^0.26\"},{\"name\":\"zstd\",\"optional\":true,\"req\":\"^0.13.0\"}],\"features\":{\"channel\":[\"dep:hyper\",\"hyper?/client\",\"dep:hyper-util\",\"hyper-util?/client-legacy\",\"dep:tower\",\"tower?/balance\",\"tower?/buffer\",\"tower?/discover\",\"tower?/limit\",\"dep:tokio\",\"tokio?/time\",\"dep:hyper-timeout\"],\"codegen\":[\"dep:async-trait\"],\"default\":[\"transport\",\"codegen\",\"prost\"],\"gzip\":[\"dep:flate2\"],\"prost\":[\"dep:prost\"],\"router\":[\"dep:axum\",\"dep:tower\",\"tower?/util\"],\"server\":[\"router\",\"dep:async-stream\",\"dep:h2\",\"dep:hyper\",\"hyper?/server\",\"dep:hyper-util\",\"hyper-util?/service\",\"hyper-util?/server-auto\",\"dep:socket2\",\"dep:tokio\",\"tokio?/macros\",\"tokio?/net\",\"tokio?/time\",\"tokio-stream/net\",\"dep:tower\",\"tower?/util\",\"tower?/limit\"],\"tls\":[\"dep:rustls-pemfile\",\"dep:tokio-rustls\",\"dep:tokio\",\"tokio?/rt\",\"tokio?/macros\"],\"tls-roots\":[\"tls\",\"channel\",\"dep:rustls-native-certs\"],\"tls-webpki-roots\":[\"tls\",\"channel\",\"dep:webpki-roots\"],\"transport\":[\"server\",\"channel\"],\"zstd\":[\"dep:zstd\"]}}", - "tower-layer_0.3.3": "{\"dependencies\":[],\"features\":{}}", - "tower-service_0.3.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.22\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^0.2\"},{\"features\":[\"macros\",\"time\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.6.2\"},{\"kind\":\"dev\",\"name\":\"tower-layer\",\"req\":\"^0.3\"}],\"features\":{}}", - "tower_0.4.13": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"hdrhistogram\",\"optional\":true,\"req\":\"^7.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"hdrhistogram\",\"req\":\"^7.0\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^0.2\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^1.0.2\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4.0\"},{\"name\":\"pin-project\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"pin-project-lite\",\"optional\":true,\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"},{\"features\":[\"small_rng\"],\"name\":\"rand\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.6\"},{\"features\":[\"macros\",\"sync\",\"test-util\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.6.2\"},{\"name\":\"tokio-stream\",\"optional\":true,\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7.0\"},{\"name\":\"tower-layer\",\"req\":\"^0.3.1\"},{\"name\":\"tower-service\",\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"tower-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.2\"},{\"default_features\":false,\"features\":[\"fmt\",\"ansi\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"}],\"features\":{\"__common\":[\"futures-core\",\"pin-project-lite\"],\"balance\":[\"discover\",\"load\",\"ready-cache\",\"make\",\"rand\",\"slab\"],\"buffer\":[\"__common\",\"tokio/sync\",\"tokio/rt\",\"tokio-util\",\"tracing\"],\"default\":[\"log\"],\"discover\":[\"__common\"],\"filter\":[\"__common\",\"futures-util\"],\"full\":[\"balance\",\"buffer\",\"discover\",\"filter\",\"hedge\",\"limit\",\"load\",\"load-shed\",\"make\",\"ready-cache\",\"reconnect\",\"retry\",\"spawn-ready\",\"steer\",\"timeout\",\"util\"],\"hedge\":[\"util\",\"filter\",\"futures-util\",\"hdrhistogram\",\"tokio/time\",\"tracing\"],\"limit\":[\"__common\",\"tokio/time\",\"tokio/sync\",\"tokio-util\",\"tracing\"],\"load\":[\"__common\",\"tokio/time\",\"tracing\"],\"load-shed\":[\"__common\"],\"log\":[\"tracing/log\"],\"make\":[\"futures-util\",\"pin-project-lite\",\"tokio/io-std\"],\"ready-cache\":[\"futures-core\",\"futures-util\",\"indexmap\",\"tokio/sync\",\"tracing\",\"pin-project-lite\"],\"reconnect\":[\"make\",\"tokio/io-std\",\"tracing\"],\"retry\":[\"__common\",\"tokio/time\"],\"spawn-ready\":[\"__common\",\"futures-util\",\"tokio/sync\",\"tokio/rt\",\"util\",\"tracing\"],\"steer\":[],\"timeout\":[\"pin-project-lite\",\"tokio/time\"],\"util\":[\"__common\",\"futures-util\",\"pin-project\"]}}", - "tower_0.5.3": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.22\"},{\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3.22\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.22\"},{\"default_features\":false,\"features\":[\"async-await-macro\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3.22\"},{\"default_features\":false,\"name\":\"hdrhistogram\",\"optional\":true,\"req\":\"^7.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"hdrhistogram\",\"req\":\"^7.0\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.0.2\"},{\"name\":\"pin-project-lite\",\"optional\":true,\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9\"},{\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4.9\"},{\"name\":\"sync_wrapper\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.6.2\"},{\"features\":[\"macros\",\"sync\",\"test-util\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.6.2\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1.1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7.0\"},{\"name\":\"tower-layer\",\"req\":\"^0.3.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3.3\"},{\"kind\":\"dev\",\"name\":\"tower-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.2\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1.2\"},{\"default_features\":false,\"features\":[\"fmt\",\"ansi\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"}],\"features\":{\"balance\":[\"discover\",\"load\",\"ready-cache\",\"make\",\"slab\",\"util\"],\"buffer\":[\"tokio/sync\",\"tokio/rt\",\"tokio-util\",\"tracing\",\"pin-project-lite\"],\"discover\":[\"futures-core\",\"pin-project-lite\"],\"filter\":[\"futures-util\",\"pin-project-lite\"],\"full\":[\"balance\",\"buffer\",\"discover\",\"filter\",\"hedge\",\"limit\",\"load\",\"load-shed\",\"make\",\"ready-cache\",\"reconnect\",\"retry\",\"spawn-ready\",\"steer\",\"timeout\",\"util\"],\"hedge\":[\"util\",\"filter\",\"futures-util\",\"hdrhistogram\",\"tokio/time\",\"tracing\"],\"limit\":[\"tokio/time\",\"tokio/sync\",\"tokio-util\",\"tracing\",\"pin-project-lite\"],\"load\":[\"tokio/time\",\"tracing\",\"pin-project-lite\"],\"load-shed\":[\"pin-project-lite\"],\"log\":[\"tracing/log\"],\"make\":[\"pin-project-lite\",\"tokio\"],\"ready-cache\":[\"futures-core\",\"futures-util\",\"indexmap\",\"tokio/sync\",\"tracing\",\"pin-project-lite\"],\"reconnect\":[\"make\",\"tracing\"],\"retry\":[\"tokio/time\",\"util\"],\"spawn-ready\":[\"futures-util\",\"tokio/sync\",\"tokio/rt\",\"util\",\"tracing\"],\"steer\":[],\"timeout\":[\"pin-project-lite\",\"tokio/time\"],\"tokio-stream\":[],\"util\":[\"futures-core\",\"futures-util\",\"pin-project-lite\",\"sync_wrapper\"]}}", - "tracing-attributes_0.1.31": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-trait\",\"req\":\"^0.1.67\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1.0.20\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.9\"},{\"default_features\":false,\"features\":[\"full\",\"parsing\",\"printing\",\"visit-mut\",\"clone-impls\",\"extra-traits\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4.2\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1.35\"},{\"features\":[\"env-filter\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.64\"}],\"features\":{\"async-await\":[]}}", - "tracing-core_0.1.36": "{\"dependencies\":[{\"name\":\"once_cell\",\"optional\":true,\"req\":\"^1.13.0\"},{\"default_features\":false,\"name\":\"valuable\",\"optional\":true,\"req\":\"^0.1.0\",\"target\":\"cfg(tracing_unstable)\"}],\"features\":{\"default\":[\"std\",\"valuable?/std\"],\"std\":[\"once_cell\"]}}", - "tracing_0.1.44": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.6\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.21\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4.17\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.9\"},{\"name\":\"tracing-attributes\",\"optional\":true,\"req\":\"^0.1.31\"},{\"default_features\":false,\"name\":\"tracing-core\",\"req\":\"^0.1.36\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.38\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"}],\"features\":{\"async-await\":[],\"attributes\":[\"tracing-attributes\"],\"default\":[\"std\",\"attributes\"],\"log-always\":[\"log\"],\"max_level_debug\":[],\"max_level_error\":[],\"max_level_info\":[],\"max_level_off\":[],\"max_level_trace\":[],\"max_level_warn\":[],\"release_max_level_debug\":[],\"release_max_level_error\":[],\"release_max_level_info\":[],\"release_max_level_off\":[],\"release_max_level_trace\":[],\"release_max_level_warn\":[],\"std\":[\"tracing-core/std\"],\"valuable\":[\"tracing-core/valuable\"]}}", - "try-lock_0.2.5": "{\"dependencies\":[],\"features\":{}}", - "unicode-ident_1.0.14": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"fst\",\"req\":\"^0.4\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"roaring\",\"req\":\"^0.10\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"ucd-trie\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"unicode-xid\",\"req\":\"^0.2.6\"}],\"features\":{}}", - "unicode-ident_1.0.24": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"fst\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"roaring\",\"req\":\"^0.11\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"ucd-trie\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"unicode-xid\",\"req\":\"^0.2.6\"}],\"features\":{}}", - "unicode-normalization-alignments_0.1.12": "{\"dependencies\":[{\"name\":\"smallvec\",\"req\":\"^1.0\"}],\"features\":{}}", - "unicode-segmentation_1.12.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.7\"}],\"features\":{\"no_std\":[]}}", - "unicode-xid_0.2.6": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3\"}],\"features\":{\"bench\":[],\"default\":[],\"no_std\":[]}}", - "unicode_categories_0.1.1": "{\"dependencies\":[],\"features\":{}}", - "version_check_0.9.5": "{\"dependencies\":[],\"features\":{}}", - "want_0.3.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"tokio-executor\",\"req\":\"^0.2.0-alpha.2\"},{\"kind\":\"dev\",\"name\":\"tokio-sync\",\"req\":\"^0.2.0-alpha.2\"},{\"name\":\"try-lock\",\"req\":\"^0.2.4\"}],\"features\":{}}", - "wasi_0.11.1+wasi-snapshot-preview1": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"rustc-dep-of-std\":[\"core\",\"rustc-std-workspace-alloc\"],\"std\":[]}}", - "wasi_0.14.2+wasi-0.2.4": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"bitflags\"],\"name\":\"wit-bindgen-rt\",\"req\":\"^0.39.0\"}],\"features\":{\"default\":[\"std\"],\"rustc-dep-of-std\":[\"compiler_builtins\",\"core\",\"rustc-std-workspace-alloc\"],\"std\":[]}}", - "wasip2_1.0.2+wasi-0.2.9": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"wit-bindgen\",\"req\":\"^0.51.0\"}],\"features\":{\"bitflags\":[\"wit-bindgen/bitflags\"],\"default\":[\"std\",\"bitflags\"],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"wit-bindgen/rustc-dep-of-std\"],\"std\":[]}}", - "wasip3_0.4.0+wasi-0.3.0-rc-2026-01-06": "{\"dependencies\":[{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1.10.1\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.31\"},{\"name\":\"http\",\"optional\":true,\"req\":\"^1.3.1\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^1.3.1\"},{\"name\":\"http-body\",\"optional\":true,\"req\":\"^1.0.1\"},{\"name\":\"thiserror\",\"optional\":true,\"req\":\"^2.0.17\"},{\"default_features\":false,\"features\":[\"async\"],\"name\":\"wit-bindgen\",\"req\":\"^0.51.0\"},{\"default_features\":false,\"features\":[\"async-spawn\"],\"kind\":\"dev\",\"name\":\"wit-bindgen\",\"req\":\"^0.51.0\"}],\"features\":{\"http-compat\":[\"dep:bytes\",\"dep:http-body\",\"dep:http\",\"dep:thiserror\",\"wit-bindgen/async-spawn\"]}}", - "wasm-encoder_0.244.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"default_features\":false,\"name\":\"leb128fmt\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.2.0\"},{\"default_features\":false,\"features\":[\"simd\",\"simd\"],\"name\":\"wasmparser\",\"optional\":true,\"req\":\"^0.244.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"wasmprinter\",\"req\":\"^0.244.0\"}],\"features\":{\"component-model\":[\"wasmparser?/component-model\"],\"default\":[\"std\",\"component-model\"],\"std\":[\"wasmparser?/std\"]}}", - "wasm-metadata_0.244.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"name\":\"auditable-serde\",\"optional\":true,\"req\":\"^0.8.0\"},{\"features\":[\"derive\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4.0.0\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1.1.0\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"indexmap\",\"req\":\"^2.7.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"spdx\",\"optional\":true,\"req\":\"^0.10.1\"},{\"name\":\"url\",\"optional\":true,\"req\":\"^2.0.0\"},{\"default_features\":false,\"features\":[\"std\",\"component-model\"],\"name\":\"wasm-encoder\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"simd\",\"std\",\"component-model\",\"hash-collections\"],\"name\":\"wasmparser\",\"req\":\"^0.244.0\"}],\"features\":{\"default\":[\"oci\",\"serde\"],\"oci\":[\"dep:auditable-serde\",\"dep:flate2\",\"dep:url\",\"dep:spdx\",\"dep:serde_json\",\"serde\"],\"serde\":[\"dep:serde_derive\",\"dep:serde\"]}}", - "wasmparser_0.244.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"name\":\"bitflags\",\"req\":\"^2.4.1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5.1\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"default-hasher\"],\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.15.2\"},{\"default_features\":false,\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.7.0\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.13.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.3\"},{\"default_features\":false,\"name\":\"semver\",\"optional\":true,\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.166\"}],\"features\":{\"component-model\":[\"dep:semver\"],\"default\":[\"std\",\"validate\",\"serde\",\"features\",\"component-model\",\"hash-collections\",\"simd\"],\"features\":[],\"hash-collections\":[\"dep:hashbrown\",\"dep:indexmap\"],\"prefer-btree-collections\":[],\"serde\":[\"dep:serde\",\"indexmap?/serde\",\"hashbrown?/serde\"],\"simd\":[],\"std\":[\"indexmap?/std\"],\"validate\":[]}}", - "windows-link_0.2.1": "{\"dependencies\":[],\"features\":{}}", - "windows-sys_0.52.0": "{\"dependencies\":[{\"name\":\"windows-targets\",\"req\":\"^0.52.0\"}],\"features\":{\"Wdk\":[],\"Wdk_Foundation\":[\"Wdk\"],\"Wdk_Graphics\":[\"Wdk\"],\"Wdk_Graphics_Direct3D\":[\"Wdk_Graphics\"],\"Wdk_Storage\":[\"Wdk\"],\"Wdk_Storage_FileSystem\":[\"Wdk_Storage\"],\"Wdk_Storage_FileSystem_Minifilters\":[\"Wdk_Storage_FileSystem\"],\"Wdk_System\":[\"Wdk\"],\"Wdk_System_IO\":[\"Wdk_System\"],\"Wdk_System_OfflineRegistry\":[\"Wdk_System\"],\"Wdk_System_Registry\":[\"Wdk_System\"],\"Wdk_System_SystemInformation\":[\"Wdk_System\"],\"Wdk_System_SystemServices\":[\"Wdk_System\"],\"Wdk_System_Threading\":[\"Wdk_System\"],\"Win32\":[],\"Win32_Data\":[\"Win32\"],\"Win32_Data_HtmlHelp\":[\"Win32_Data\"],\"Win32_Data_RightsManagement\":[\"Win32_Data\"],\"Win32_Devices\":[\"Win32\"],\"Win32_Devices_AllJoyn\":[\"Win32_Devices\"],\"Win32_Devices_BiometricFramework\":[\"Win32_Devices\"],\"Win32_Devices_Bluetooth\":[\"Win32_Devices\"],\"Win32_Devices_Communication\":[\"Win32_Devices\"],\"Win32_Devices_DeviceAndDriverInstallation\":[\"Win32_Devices\"],\"Win32_Devices_DeviceQuery\":[\"Win32_Devices\"],\"Win32_Devices_Display\":[\"Win32_Devices\"],\"Win32_Devices_Enumeration\":[\"Win32_Devices\"],\"Win32_Devices_Enumeration_Pnp\":[\"Win32_Devices_Enumeration\"],\"Win32_Devices_Fax\":[\"Win32_Devices\"],\"Win32_Devices_HumanInterfaceDevice\":[\"Win32_Devices\"],\"Win32_Devices_PortableDevices\":[\"Win32_Devices\"],\"Win32_Devices_Properties\":[\"Win32_Devices\"],\"Win32_Devices_Pwm\":[\"Win32_Devices\"],\"Win32_Devices_Sensors\":[\"Win32_Devices\"],\"Win32_Devices_SerialCommunication\":[\"Win32_Devices\"],\"Win32_Devices_Tapi\":[\"Win32_Devices\"],\"Win32_Devices_Usb\":[\"Win32_Devices\"],\"Win32_Devices_WebServicesOnDevices\":[\"Win32_Devices\"],\"Win32_Foundation\":[\"Win32\"],\"Win32_Gaming\":[\"Win32\"],\"Win32_Globalization\":[\"Win32\"],\"Win32_Graphics\":[\"Win32\"],\"Win32_Graphics_Dwm\":[\"Win32_Graphics\"],\"Win32_Graphics_Gdi\":[\"Win32_Graphics\"],\"Win32_Graphics_GdiPlus\":[\"Win32_Graphics\"],\"Win32_Graphics_Hlsl\":[\"Win32_Graphics\"],\"Win32_Graphics_OpenGL\":[\"Win32_Graphics\"],\"Win32_Graphics_Printing\":[\"Win32_Graphics\"],\"Win32_Graphics_Printing_PrintTicket\":[\"Win32_Graphics_Printing\"],\"Win32_Management\":[\"Win32\"],\"Win32_Management_MobileDeviceManagementRegistration\":[\"Win32_Management\"],\"Win32_Media\":[\"Win32\"],\"Win32_Media_Audio\":[\"Win32_Media\"],\"Win32_Media_DxMediaObjects\":[\"Win32_Media\"],\"Win32_Media_KernelStreaming\":[\"Win32_Media\"],\"Win32_Media_Multimedia\":[\"Win32_Media\"],\"Win32_Media_Streaming\":[\"Win32_Media\"],\"Win32_Media_WindowsMediaFormat\":[\"Win32_Media\"],\"Win32_NetworkManagement\":[\"Win32\"],\"Win32_NetworkManagement_Dhcp\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Dns\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_InternetConnectionWizard\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_IpHelper\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Multicast\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Ndis\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetBios\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetManagement\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetShell\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetworkDiagnosticsFramework\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_P2P\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_QoS\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Rras\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Snmp\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WNet\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WebDav\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WiFi\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsConnectionManager\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsFilteringPlatform\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsFirewall\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsNetworkVirtualization\":[\"Win32_NetworkManagement\"],\"Win32_Networking\":[\"Win32\"],\"Win32_Networking_ActiveDirectory\":[\"Win32_Networking\"],\"Win32_Networking_Clustering\":[\"Win32_Networking\"],\"Win32_Networking_HttpServer\":[\"Win32_Networking\"],\"Win32_Networking_Ldap\":[\"Win32_Networking\"],\"Win32_Networking_WebSocket\":[\"Win32_Networking\"],\"Win32_Networking_WinHttp\":[\"Win32_Networking\"],\"Win32_Networking_WinInet\":[\"Win32_Networking\"],\"Win32_Networking_WinSock\":[\"Win32_Networking\"],\"Win32_Networking_WindowsWebServices\":[\"Win32_Networking\"],\"Win32_Security\":[\"Win32\"],\"Win32_Security_AppLocker\":[\"Win32_Security\"],\"Win32_Security_Authentication\":[\"Win32_Security\"],\"Win32_Security_Authentication_Identity\":[\"Win32_Security_Authentication\"],\"Win32_Security_Authorization\":[\"Win32_Security\"],\"Win32_Security_Credentials\":[\"Win32_Security\"],\"Win32_Security_Cryptography\":[\"Win32_Security\"],\"Win32_Security_Cryptography_Catalog\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_Certificates\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_Sip\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_UI\":[\"Win32_Security_Cryptography\"],\"Win32_Security_DiagnosticDataQuery\":[\"Win32_Security\"],\"Win32_Security_DirectoryServices\":[\"Win32_Security\"],\"Win32_Security_EnterpriseData\":[\"Win32_Security\"],\"Win32_Security_ExtensibleAuthenticationProtocol\":[\"Win32_Security\"],\"Win32_Security_Isolation\":[\"Win32_Security\"],\"Win32_Security_LicenseProtection\":[\"Win32_Security\"],\"Win32_Security_NetworkAccessProtection\":[\"Win32_Security\"],\"Win32_Security_WinTrust\":[\"Win32_Security\"],\"Win32_Security_WinWlx\":[\"Win32_Security\"],\"Win32_Storage\":[\"Win32\"],\"Win32_Storage_Cabinets\":[\"Win32_Storage\"],\"Win32_Storage_CloudFilters\":[\"Win32_Storage\"],\"Win32_Storage_Compression\":[\"Win32_Storage\"],\"Win32_Storage_DistributedFileSystem\":[\"Win32_Storage\"],\"Win32_Storage_FileHistory\":[\"Win32_Storage\"],\"Win32_Storage_FileSystem\":[\"Win32_Storage\"],\"Win32_Storage_Imapi\":[\"Win32_Storage\"],\"Win32_Storage_IndexServer\":[\"Win32_Storage\"],\"Win32_Storage_InstallableFileSystems\":[\"Win32_Storage\"],\"Win32_Storage_IscsiDisc\":[\"Win32_Storage\"],\"Win32_Storage_Jet\":[\"Win32_Storage\"],\"Win32_Storage_Nvme\":[\"Win32_Storage\"],\"Win32_Storage_OfflineFiles\":[\"Win32_Storage\"],\"Win32_Storage_OperationRecorder\":[\"Win32_Storage\"],\"Win32_Storage_Packaging\":[\"Win32_Storage\"],\"Win32_Storage_Packaging_Appx\":[\"Win32_Storage_Packaging\"],\"Win32_Storage_ProjectedFileSystem\":[\"Win32_Storage\"],\"Win32_Storage_StructuredStorage\":[\"Win32_Storage\"],\"Win32_Storage_Vhd\":[\"Win32_Storage\"],\"Win32_Storage_Xps\":[\"Win32_Storage\"],\"Win32_System\":[\"Win32\"],\"Win32_System_AddressBook\":[\"Win32_System\"],\"Win32_System_Antimalware\":[\"Win32_System\"],\"Win32_System_ApplicationInstallationAndServicing\":[\"Win32_System\"],\"Win32_System_ApplicationVerifier\":[\"Win32_System\"],\"Win32_System_ClrHosting\":[\"Win32_System\"],\"Win32_System_Com\":[\"Win32_System\"],\"Win32_System_Com_Marshal\":[\"Win32_System_Com\"],\"Win32_System_Com_StructuredStorage\":[\"Win32_System_Com\"],\"Win32_System_Com_Urlmon\":[\"Win32_System_Com\"],\"Win32_System_ComponentServices\":[\"Win32_System\"],\"Win32_System_Console\":[\"Win32_System\"],\"Win32_System_CorrelationVector\":[\"Win32_System\"],\"Win32_System_DataExchange\":[\"Win32_System\"],\"Win32_System_DeploymentServices\":[\"Win32_System\"],\"Win32_System_DeveloperLicensing\":[\"Win32_System\"],\"Win32_System_Diagnostics\":[\"Win32_System\"],\"Win32_System_Diagnostics_Ceip\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_Debug\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_Debug_Extensions\":[\"Win32_System_Diagnostics_Debug\"],\"Win32_System_Diagnostics_Etw\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_ProcessSnapshotting\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_ToolHelp\":[\"Win32_System_Diagnostics\"],\"Win32_System_DistributedTransactionCoordinator\":[\"Win32_System\"],\"Win32_System_Environment\":[\"Win32_System\"],\"Win32_System_ErrorReporting\":[\"Win32_System\"],\"Win32_System_EventCollector\":[\"Win32_System\"],\"Win32_System_EventLog\":[\"Win32_System\"],\"Win32_System_EventNotificationService\":[\"Win32_System\"],\"Win32_System_GroupPolicy\":[\"Win32_System\"],\"Win32_System_HostCompute\":[\"Win32_System\"],\"Win32_System_HostComputeNetwork\":[\"Win32_System\"],\"Win32_System_HostComputeSystem\":[\"Win32_System\"],\"Win32_System_Hypervisor\":[\"Win32_System\"],\"Win32_System_IO\":[\"Win32_System\"],\"Win32_System_Iis\":[\"Win32_System\"],\"Win32_System_Ioctl\":[\"Win32_System\"],\"Win32_System_JobObjects\":[\"Win32_System\"],\"Win32_System_Js\":[\"Win32_System\"],\"Win32_System_Kernel\":[\"Win32_System\"],\"Win32_System_LibraryLoader\":[\"Win32_System\"],\"Win32_System_Mailslots\":[\"Win32_System\"],\"Win32_System_Mapi\":[\"Win32_System\"],\"Win32_System_Memory\":[\"Win32_System\"],\"Win32_System_Memory_NonVolatile\":[\"Win32_System_Memory\"],\"Win32_System_MessageQueuing\":[\"Win32_System\"],\"Win32_System_MixedReality\":[\"Win32_System\"],\"Win32_System_Ole\":[\"Win32_System\"],\"Win32_System_PasswordManagement\":[\"Win32_System\"],\"Win32_System_Performance\":[\"Win32_System\"],\"Win32_System_Performance_HardwareCounterProfiling\":[\"Win32_System_Performance\"],\"Win32_System_Pipes\":[\"Win32_System\"],\"Win32_System_Power\":[\"Win32_System\"],\"Win32_System_ProcessStatus\":[\"Win32_System\"],\"Win32_System_Recovery\":[\"Win32_System\"],\"Win32_System_Registry\":[\"Win32_System\"],\"Win32_System_RemoteDesktop\":[\"Win32_System\"],\"Win32_System_RemoteManagement\":[\"Win32_System\"],\"Win32_System_RestartManager\":[\"Win32_System\"],\"Win32_System_Restore\":[\"Win32_System\"],\"Win32_System_Rpc\":[\"Win32_System\"],\"Win32_System_Search\":[\"Win32_System\"],\"Win32_System_Search_Common\":[\"Win32_System_Search\"],\"Win32_System_SecurityCenter\":[\"Win32_System\"],\"Win32_System_Services\":[\"Win32_System\"],\"Win32_System_SetupAndMigration\":[\"Win32_System\"],\"Win32_System_Shutdown\":[\"Win32_System\"],\"Win32_System_StationsAndDesktops\":[\"Win32_System\"],\"Win32_System_SubsystemForLinux\":[\"Win32_System\"],\"Win32_System_SystemInformation\":[\"Win32_System\"],\"Win32_System_SystemServices\":[\"Win32_System\"],\"Win32_System_Threading\":[\"Win32_System\"],\"Win32_System_Time\":[\"Win32_System\"],\"Win32_System_TpmBaseServices\":[\"Win32_System\"],\"Win32_System_UserAccessLogging\":[\"Win32_System\"],\"Win32_System_Variant\":[\"Win32_System\"],\"Win32_System_VirtualDosMachines\":[\"Win32_System\"],\"Win32_System_WindowsProgramming\":[\"Win32_System\"],\"Win32_System_Wmi\":[\"Win32_System\"],\"Win32_UI\":[\"Win32\"],\"Win32_UI_Accessibility\":[\"Win32_UI\"],\"Win32_UI_ColorSystem\":[\"Win32_UI\"],\"Win32_UI_Controls\":[\"Win32_UI\"],\"Win32_UI_Controls_Dialogs\":[\"Win32_UI_Controls\"],\"Win32_UI_HiDpi\":[\"Win32_UI\"],\"Win32_UI_Input\":[\"Win32_UI\"],\"Win32_UI_Input_Ime\":[\"Win32_UI_Input\"],\"Win32_UI_Input_KeyboardAndMouse\":[\"Win32_UI_Input\"],\"Win32_UI_Input_Pointer\":[\"Win32_UI_Input\"],\"Win32_UI_Input_Touch\":[\"Win32_UI_Input\"],\"Win32_UI_Input_XboxController\":[\"Win32_UI_Input\"],\"Win32_UI_InteractionContext\":[\"Win32_UI\"],\"Win32_UI_Magnification\":[\"Win32_UI\"],\"Win32_UI_Shell\":[\"Win32_UI\"],\"Win32_UI_Shell_PropertiesSystem\":[\"Win32_UI_Shell\"],\"Win32_UI_TabletPC\":[\"Win32_UI\"],\"Win32_UI_TextServices\":[\"Win32_UI\"],\"Win32_UI_WindowsAndMessaging\":[\"Win32_UI\"],\"Win32_Web\":[\"Win32\"],\"Win32_Web_InternetExplorer\":[\"Win32_Web\"],\"default\":[],\"docs\":[]}}", - "windows-sys_0.61.2": "{\"dependencies\":[{\"default_features\":false,\"name\":\"windows-link\",\"req\":\"^0.2.1\"}],\"features\":{\"Wdk\":[\"Win32_Foundation\"],\"Wdk_Devices\":[\"Wdk\"],\"Wdk_Devices_Bluetooth\":[\"Wdk_Devices\"],\"Wdk_Devices_HumanInterfaceDevice\":[\"Wdk_Devices\"],\"Wdk_Foundation\":[\"Wdk\"],\"Wdk_Graphics\":[\"Wdk\"],\"Wdk_Graphics_Direct3D\":[\"Wdk_Graphics\"],\"Wdk_NetworkManagement\":[\"Wdk\"],\"Wdk_NetworkManagement_Ndis\":[\"Wdk_NetworkManagement\"],\"Wdk_NetworkManagement_WindowsFilteringPlatform\":[\"Wdk_NetworkManagement\"],\"Wdk_Storage\":[\"Wdk\"],\"Wdk_Storage_FileSystem\":[\"Wdk_Storage\"],\"Wdk_Storage_FileSystem_Minifilters\":[\"Wdk_Storage_FileSystem\"],\"Wdk_System\":[\"Wdk\"],\"Wdk_System_IO\":[\"Wdk_System\"],\"Wdk_System_Memory\":[\"Wdk_System\"],\"Wdk_System_OfflineRegistry\":[\"Wdk_System\"],\"Wdk_System_Registry\":[\"Wdk_System\"],\"Wdk_System_SystemInformation\":[\"Wdk_System\"],\"Wdk_System_SystemServices\":[\"Wdk_System\"],\"Wdk_System_Threading\":[\"Wdk_System\"],\"Win32\":[\"Win32_Foundation\"],\"Win32_Data\":[\"Win32\"],\"Win32_Data_HtmlHelp\":[\"Win32_Data\"],\"Win32_Data_RightsManagement\":[\"Win32_Data\"],\"Win32_Devices\":[\"Win32\"],\"Win32_Devices_AllJoyn\":[\"Win32_Devices\"],\"Win32_Devices_Beep\":[\"Win32_Devices\"],\"Win32_Devices_BiometricFramework\":[\"Win32_Devices\"],\"Win32_Devices_Bluetooth\":[\"Win32_Devices\"],\"Win32_Devices_Cdrom\":[\"Win32_Devices\"],\"Win32_Devices_Communication\":[\"Win32_Devices\"],\"Win32_Devices_DeviceAndDriverInstallation\":[\"Win32_Devices\"],\"Win32_Devices_DeviceQuery\":[\"Win32_Devices\"],\"Win32_Devices_Display\":[\"Win32_Devices\"],\"Win32_Devices_Dvd\":[\"Win32_Devices\"],\"Win32_Devices_Enumeration\":[\"Win32_Devices\"],\"Win32_Devices_Enumeration_Pnp\":[\"Win32_Devices_Enumeration\"],\"Win32_Devices_Fax\":[\"Win32_Devices\"],\"Win32_Devices_HumanInterfaceDevice\":[\"Win32_Devices\"],\"Win32_Devices_Nfc\":[\"Win32_Devices\"],\"Win32_Devices_Nfp\":[\"Win32_Devices\"],\"Win32_Devices_PortableDevices\":[\"Win32_Devices\"],\"Win32_Devices_Properties\":[\"Win32_Devices\"],\"Win32_Devices_Pwm\":[\"Win32_Devices\"],\"Win32_Devices_Sensors\":[\"Win32_Devices\"],\"Win32_Devices_SerialCommunication\":[\"Win32_Devices\"],\"Win32_Devices_Tapi\":[\"Win32_Devices\"],\"Win32_Devices_Usb\":[\"Win32_Devices\"],\"Win32_Devices_WebServicesOnDevices\":[\"Win32_Devices\"],\"Win32_Foundation\":[\"Win32\"],\"Win32_Gaming\":[\"Win32\"],\"Win32_Globalization\":[\"Win32\"],\"Win32_Graphics\":[\"Win32\"],\"Win32_Graphics_Dwm\":[\"Win32_Graphics\"],\"Win32_Graphics_Gdi\":[\"Win32_Graphics\"],\"Win32_Graphics_GdiPlus\":[\"Win32_Graphics\"],\"Win32_Graphics_Hlsl\":[\"Win32_Graphics\"],\"Win32_Graphics_OpenGL\":[\"Win32_Graphics\"],\"Win32_Graphics_Printing\":[\"Win32_Graphics\"],\"Win32_Graphics_Printing_PrintTicket\":[\"Win32_Graphics_Printing\"],\"Win32_Management\":[\"Win32\"],\"Win32_Management_MobileDeviceManagementRegistration\":[\"Win32_Management\"],\"Win32_Media\":[\"Win32\"],\"Win32_Media_Audio\":[\"Win32_Media\"],\"Win32_Media_DxMediaObjects\":[\"Win32_Media\"],\"Win32_Media_KernelStreaming\":[\"Win32_Media\"],\"Win32_Media_Multimedia\":[\"Win32_Media\"],\"Win32_Media_Streaming\":[\"Win32_Media\"],\"Win32_Media_WindowsMediaFormat\":[\"Win32_Media\"],\"Win32_NetworkManagement\":[\"Win32\"],\"Win32_NetworkManagement_Dhcp\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Dns\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_InternetConnectionWizard\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_IpHelper\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Multicast\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Ndis\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetBios\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetManagement\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetShell\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetworkDiagnosticsFramework\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_P2P\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_QoS\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Rras\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Snmp\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WNet\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WebDav\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WiFi\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsConnectionManager\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsFilteringPlatform\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsFirewall\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsNetworkVirtualization\":[\"Win32_NetworkManagement\"],\"Win32_Networking\":[\"Win32\"],\"Win32_Networking_ActiveDirectory\":[\"Win32_Networking\"],\"Win32_Networking_Clustering\":[\"Win32_Networking\"],\"Win32_Networking_HttpServer\":[\"Win32_Networking\"],\"Win32_Networking_Ldap\":[\"Win32_Networking\"],\"Win32_Networking_WebSocket\":[\"Win32_Networking\"],\"Win32_Networking_WinHttp\":[\"Win32_Networking\"],\"Win32_Networking_WinInet\":[\"Win32_Networking\"],\"Win32_Networking_WinSock\":[\"Win32_Networking\"],\"Win32_Networking_WindowsWebServices\":[\"Win32_Networking\"],\"Win32_Security\":[\"Win32\"],\"Win32_Security_AppLocker\":[\"Win32_Security\"],\"Win32_Security_Authentication\":[\"Win32_Security\"],\"Win32_Security_Authentication_Identity\":[\"Win32_Security_Authentication\"],\"Win32_Security_Authorization\":[\"Win32_Security\"],\"Win32_Security_Credentials\":[\"Win32_Security\"],\"Win32_Security_Cryptography\":[\"Win32_Security\"],\"Win32_Security_Cryptography_Catalog\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_Certificates\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_Sip\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_UI\":[\"Win32_Security_Cryptography\"],\"Win32_Security_DiagnosticDataQuery\":[\"Win32_Security\"],\"Win32_Security_DirectoryServices\":[\"Win32_Security\"],\"Win32_Security_EnterpriseData\":[\"Win32_Security\"],\"Win32_Security_ExtensibleAuthenticationProtocol\":[\"Win32_Security\"],\"Win32_Security_Isolation\":[\"Win32_Security\"],\"Win32_Security_LicenseProtection\":[\"Win32_Security\"],\"Win32_Security_NetworkAccessProtection\":[\"Win32_Security\"],\"Win32_Security_WinTrust\":[\"Win32_Security\"],\"Win32_Security_WinWlx\":[\"Win32_Security\"],\"Win32_Storage\":[\"Win32\"],\"Win32_Storage_Cabinets\":[\"Win32_Storage\"],\"Win32_Storage_CloudFilters\":[\"Win32_Storage\"],\"Win32_Storage_Compression\":[\"Win32_Storage\"],\"Win32_Storage_DistributedFileSystem\":[\"Win32_Storage\"],\"Win32_Storage_FileHistory\":[\"Win32_Storage\"],\"Win32_Storage_FileSystem\":[\"Win32_Storage\"],\"Win32_Storage_Imapi\":[\"Win32_Storage\"],\"Win32_Storage_IndexServer\":[\"Win32_Storage\"],\"Win32_Storage_InstallableFileSystems\":[\"Win32_Storage\"],\"Win32_Storage_IscsiDisc\":[\"Win32_Storage\"],\"Win32_Storage_Jet\":[\"Win32_Storage\"],\"Win32_Storage_Nvme\":[\"Win32_Storage\"],\"Win32_Storage_OfflineFiles\":[\"Win32_Storage\"],\"Win32_Storage_OperationRecorder\":[\"Win32_Storage\"],\"Win32_Storage_Packaging\":[\"Win32_Storage\"],\"Win32_Storage_Packaging_Appx\":[\"Win32_Storage_Packaging\"],\"Win32_Storage_ProjectedFileSystem\":[\"Win32_Storage\"],\"Win32_Storage_StructuredStorage\":[\"Win32_Storage\"],\"Win32_Storage_Vhd\":[\"Win32_Storage\"],\"Win32_Storage_Xps\":[\"Win32_Storage\"],\"Win32_System\":[\"Win32\"],\"Win32_System_AddressBook\":[\"Win32_System\"],\"Win32_System_Antimalware\":[\"Win32_System\"],\"Win32_System_ApplicationInstallationAndServicing\":[\"Win32_System\"],\"Win32_System_ApplicationVerifier\":[\"Win32_System\"],\"Win32_System_ClrHosting\":[\"Win32_System\"],\"Win32_System_Com\":[\"Win32_System\"],\"Win32_System_Com_Marshal\":[\"Win32_System_Com\"],\"Win32_System_Com_StructuredStorage\":[\"Win32_System_Com\"],\"Win32_System_Com_Urlmon\":[\"Win32_System_Com\"],\"Win32_System_ComponentServices\":[\"Win32_System\"],\"Win32_System_Console\":[\"Win32_System\"],\"Win32_System_CorrelationVector\":[\"Win32_System\"],\"Win32_System_DataExchange\":[\"Win32_System\"],\"Win32_System_DeploymentServices\":[\"Win32_System\"],\"Win32_System_DeveloperLicensing\":[\"Win32_System\"],\"Win32_System_Diagnostics\":[\"Win32_System\"],\"Win32_System_Diagnostics_Ceip\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_Debug\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_Debug_Extensions\":[\"Win32_System_Diagnostics_Debug\"],\"Win32_System_Diagnostics_Etw\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_ProcessSnapshotting\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_ToolHelp\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_TraceLogging\":[\"Win32_System_Diagnostics\"],\"Win32_System_DistributedTransactionCoordinator\":[\"Win32_System\"],\"Win32_System_Environment\":[\"Win32_System\"],\"Win32_System_ErrorReporting\":[\"Win32_System\"],\"Win32_System_EventCollector\":[\"Win32_System\"],\"Win32_System_EventLog\":[\"Win32_System\"],\"Win32_System_EventNotificationService\":[\"Win32_System\"],\"Win32_System_GroupPolicy\":[\"Win32_System\"],\"Win32_System_HostCompute\":[\"Win32_System\"],\"Win32_System_HostComputeNetwork\":[\"Win32_System\"],\"Win32_System_HostComputeSystem\":[\"Win32_System\"],\"Win32_System_Hypervisor\":[\"Win32_System\"],\"Win32_System_IO\":[\"Win32_System\"],\"Win32_System_Iis\":[\"Win32_System\"],\"Win32_System_Ioctl\":[\"Win32_System\"],\"Win32_System_JobObjects\":[\"Win32_System\"],\"Win32_System_Js\":[\"Win32_System\"],\"Win32_System_Kernel\":[\"Win32_System\"],\"Win32_System_LibraryLoader\":[\"Win32_System\"],\"Win32_System_Mailslots\":[\"Win32_System\"],\"Win32_System_Mapi\":[\"Win32_System\"],\"Win32_System_Memory\":[\"Win32_System\"],\"Win32_System_Memory_NonVolatile\":[\"Win32_System_Memory\"],\"Win32_System_MessageQueuing\":[\"Win32_System\"],\"Win32_System_MixedReality\":[\"Win32_System\"],\"Win32_System_Ole\":[\"Win32_System\"],\"Win32_System_PasswordManagement\":[\"Win32_System\"],\"Win32_System_Performance\":[\"Win32_System\"],\"Win32_System_Performance_HardwareCounterProfiling\":[\"Win32_System_Performance\"],\"Win32_System_Pipes\":[\"Win32_System\"],\"Win32_System_Power\":[\"Win32_System\"],\"Win32_System_ProcessStatus\":[\"Win32_System\"],\"Win32_System_Recovery\":[\"Win32_System\"],\"Win32_System_Registry\":[\"Win32_System\"],\"Win32_System_RemoteDesktop\":[\"Win32_System\"],\"Win32_System_RemoteManagement\":[\"Win32_System\"],\"Win32_System_RestartManager\":[\"Win32_System\"],\"Win32_System_Restore\":[\"Win32_System\"],\"Win32_System_Rpc\":[\"Win32_System\"],\"Win32_System_Search\":[\"Win32_System\"],\"Win32_System_Search_Common\":[\"Win32_System_Search\"],\"Win32_System_SecurityCenter\":[\"Win32_System\"],\"Win32_System_Services\":[\"Win32_System\"],\"Win32_System_SetupAndMigration\":[\"Win32_System\"],\"Win32_System_Shutdown\":[\"Win32_System\"],\"Win32_System_StationsAndDesktops\":[\"Win32_System\"],\"Win32_System_SubsystemForLinux\":[\"Win32_System\"],\"Win32_System_SystemInformation\":[\"Win32_System\"],\"Win32_System_SystemServices\":[\"Win32_System\"],\"Win32_System_Threading\":[\"Win32_System\"],\"Win32_System_Time\":[\"Win32_System\"],\"Win32_System_TpmBaseServices\":[\"Win32_System\"],\"Win32_System_UserAccessLogging\":[\"Win32_System\"],\"Win32_System_Variant\":[\"Win32_System\"],\"Win32_System_VirtualDosMachines\":[\"Win32_System\"],\"Win32_System_WindowsProgramming\":[\"Win32_System\"],\"Win32_System_Wmi\":[\"Win32_System\"],\"Win32_UI\":[\"Win32\"],\"Win32_UI_Accessibility\":[\"Win32_UI\"],\"Win32_UI_ColorSystem\":[\"Win32_UI\"],\"Win32_UI_Controls\":[\"Win32_UI\"],\"Win32_UI_Controls_Dialogs\":[\"Win32_UI_Controls\"],\"Win32_UI_HiDpi\":[\"Win32_UI\"],\"Win32_UI_Input\":[\"Win32_UI\"],\"Win32_UI_Input_Ime\":[\"Win32_UI_Input\"],\"Win32_UI_Input_KeyboardAndMouse\":[\"Win32_UI_Input\"],\"Win32_UI_Input_Pointer\":[\"Win32_UI_Input\"],\"Win32_UI_Input_Touch\":[\"Win32_UI_Input\"],\"Win32_UI_Input_XboxController\":[\"Win32_UI_Input\"],\"Win32_UI_InteractionContext\":[\"Win32_UI\"],\"Win32_UI_Magnification\":[\"Win32_UI\"],\"Win32_UI_Shell\":[\"Win32_UI\"],\"Win32_UI_Shell_Common\":[\"Win32_UI_Shell\"],\"Win32_UI_Shell_PropertiesSystem\":[\"Win32_UI_Shell\"],\"Win32_UI_TabletPC\":[\"Win32_UI\"],\"Win32_UI_TextServices\":[\"Win32_UI\"],\"Win32_UI_WindowsAndMessaging\":[\"Win32_UI\"],\"Win32_Web\":[\"Win32\"],\"Win32_Web_InternetExplorer\":[\"Win32_Web\"],\"default\":[],\"docs\":[]}}", - "windows-targets_0.52.6": "{\"dependencies\":[{\"name\":\"windows_aarch64_gnullvm\",\"req\":\"^0.52.6\",\"target\":\"aarch64-pc-windows-gnullvm\"},{\"name\":\"windows_aarch64_msvc\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(target_arch = \\\"aarch64\\\", target_env = \\\"msvc\\\", not(windows_raw_dylib)))\"},{\"name\":\"windows_i686_gnu\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(target_arch = \\\"x86\\\", target_env = \\\"gnu\\\", not(target_abi = \\\"llvm\\\"), not(windows_raw_dylib)))\"},{\"name\":\"windows_i686_gnullvm\",\"req\":\"^0.52.6\",\"target\":\"i686-pc-windows-gnullvm\"},{\"name\":\"windows_i686_msvc\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(target_arch = \\\"x86\\\", target_env = \\\"msvc\\\", not(windows_raw_dylib)))\"},{\"name\":\"windows_x86_64_gnu\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(target_arch = \\\"x86_64\\\", target_env = \\\"gnu\\\", not(target_abi = \\\"llvm\\\"), not(windows_raw_dylib)))\"},{\"name\":\"windows_x86_64_gnullvm\",\"req\":\"^0.52.6\",\"target\":\"x86_64-pc-windows-gnullvm\"},{\"name\":\"windows_x86_64_msvc\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(any(target_arch = \\\"x86_64\\\", target_arch = \\\"arm64ec\\\"), target_env = \\\"msvc\\\", not(windows_raw_dylib)))\"}],\"features\":{}}", - "windows_aarch64_gnullvm_0.52.6": "{\"dependencies\":[],\"features\":{}}", - "windows_aarch64_msvc_0.52.6": "{\"dependencies\":[],\"features\":{}}", - "windows_i686_gnu_0.52.6": "{\"dependencies\":[],\"features\":{}}", - "windows_i686_gnullvm_0.52.6": "{\"dependencies\":[],\"features\":{}}", - "windows_i686_msvc_0.52.6": "{\"dependencies\":[],\"features\":{}}", - "windows_x86_64_gnu_0.52.6": "{\"dependencies\":[],\"features\":{}}", - "windows_x86_64_gnullvm_0.52.6": "{\"dependencies\":[],\"features\":{}}", - "windows_x86_64_msvc_0.52.6": "{\"dependencies\":[],\"features\":{}}", - "wit-bindgen-core_0.51.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.72\"},{\"features\":[\"derive\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4.3.19\"},{\"name\":\"heck\",\"req\":\"^0.5\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.218\"},{\"name\":\"wit-parser\",\"req\":\"^0.244.0\"}],\"features\":{\"clap\":[\"dep:clap\"],\"serde\":[\"dep:serde\"]}}", - "wit-bindgen-rt_0.39.0": "{\"dependencies\":[{\"name\":\"bitflags\",\"optional\":true,\"req\":\"^2.3.3\"},{\"name\":\"futures\",\"optional\":true,\"req\":\"^0.3.30\"},{\"name\":\"once_cell\",\"optional\":true,\"req\":\"^1.19.0\"}],\"features\":{\"async\":[\"dep:futures\",\"dep:once_cell\"]}}", - "wit-bindgen-rust-macro_0.51.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.72\"},{\"name\":\"prettyplease\",\"req\":\"^0.2.20\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"printing\"],\"name\":\"syn\",\"req\":\"^2.0.89\"},{\"name\":\"wit-bindgen-core\",\"req\":\"^0.51.0\"},{\"name\":\"wit-bindgen-rust\",\"req\":\"^0.51.0\"}],\"features\":{\"async\":[]}}", - "wit-bindgen-rust_0.51.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.72\"},{\"kind\":\"dev\",\"name\":\"bytes\",\"req\":\"^1\"},{\"features\":[\"derive\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4.3.19\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.31\"},{\"name\":\"heck\",\"req\":\"^0.5\"},{\"name\":\"indexmap\",\"req\":\"^2.0.0\"},{\"name\":\"prettyplease\",\"req\":\"^0.2.20\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.218\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"features\":[\"printing\"],\"name\":\"syn\",\"req\":\"^2.0.89\"},{\"default_features\":false,\"name\":\"wasm-metadata\",\"req\":\"^0.244.0\"},{\"name\":\"wit-bindgen-core\",\"req\":\"^0.51.0\"},{\"name\":\"wit-component\",\"req\":\"^0.244.0\"}],\"features\":{\"clap\":[\"dep:clap\",\"wit-bindgen-core/clap\"],\"serde\":[\"dep:serde\",\"wit-bindgen-core/serde\"]}}", - "wit-bindgen_0.51.0": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0\"},{\"name\":\"bitflags\",\"optional\":true,\"req\":\"^2.3.3\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"futures\",\"optional\":true,\"req\":\"^0.3.30\"},{\"name\":\"wit-bindgen-rust-macro\",\"optional\":true,\"req\":\"^0.51.0\"}],\"features\":{\"async\":[\"std\",\"wit-bindgen-rust-macro?/async\"],\"async-spawn\":[\"async\",\"dep:futures\"],\"bitflags\":[\"dep:bitflags\"],\"default\":[\"macros\",\"realloc\",\"async\",\"std\",\"bitflags\"],\"inter-task-wakeup\":[\"async\"],\"macros\":[\"dep:wit-bindgen-rust-macro\"],\"realloc\":[],\"rustc-dep-of-std\":[\"dep:core\",\"dep:alloc\"],\"std\":[]}}", - "wit-component_0.244.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"name\":\"bitflags\",\"req\":\"^2.3.3\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"kind\":\"dev\",\"name\":\"glob\",\"req\":\"^0.3.0\"},{\"default_features\":false,\"name\":\"indexmap\",\"req\":\"^2.7.0\"},{\"kind\":\"dev\",\"name\":\"libtest-mimic\",\"req\":\"^0.8.1\"},{\"name\":\"log\",\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.3.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"req\":\"^1.0.166\"},{\"name\":\"serde_derive\",\"req\":\"^1.0.166\"},{\"name\":\"serde_json\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"std\",\"wasmparser\"],\"name\":\"wasm-encoder\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"name\":\"wasm-metadata\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"oci\"],\"kind\":\"dev\",\"name\":\"wasm-metadata\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"simd\",\"std\",\"component-model\",\"simd\"],\"name\":\"wasmparser\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"simd\",\"std\",\"component-model\",\"features\"],\"kind\":\"dev\",\"name\":\"wasmparser\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"component-model\"],\"kind\":\"dev\",\"name\":\"wasmprinter\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"cranelift\",\"component-model\",\"runtime\",\"gc-drc\"],\"kind\":\"dev\",\"name\":\"wasmtime\",\"req\":\"^34.0.1\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"default_features\":false,\"name\":\"wast\",\"optional\":true,\"req\":\"^244.0.0\"},{\"default_features\":false,\"name\":\"wat\",\"optional\":true,\"req\":\"^1.244.0\"},{\"default_features\":false,\"features\":[\"component-model\"],\"kind\":\"dev\",\"name\":\"wat\",\"req\":\"^1.244.0\"},{\"features\":[\"decoding\",\"serde\"],\"name\":\"wit-parser\",\"req\":\"^0.244.0\"}],\"features\":{\"dummy-module\":[\"dep:wat\"],\"semver-check\":[\"dummy-module\"],\"wat\":[\"dep:wast\",\"dep:wat\"]}}", - "wit-parser_0.244.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"id-arena\",\"req\":\"^2\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"indexmap\",\"req\":\"^2.7.0\"},{\"kind\":\"dev\",\"name\":\"libtest-mimic\",\"req\":\"^0.8.1\"},{\"name\":\"log\",\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.3.0\"},{\"default_features\":false,\"name\":\"semver\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"name\":\"unicode-xid\",\"req\":\"^0.2.2\"},{\"default_features\":false,\"features\":[\"simd\",\"std\",\"validate\",\"component-model\",\"features\"],\"name\":\"wasmparser\",\"optional\":true,\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"component-model\"],\"name\":\"wat\",\"optional\":true,\"req\":\"^1.244.0\"}],\"features\":{\"decoding\":[\"dep:wasmparser\"],\"default\":[\"serde\",\"decoding\"],\"serde\":[\"dep:serde\",\"dep:serde_derive\",\"indexmap/serde\",\"serde_json\"],\"wat\":[\"decoding\",\"dep:wat\"]}}", - "zerocopy-derive_0.7.35": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.10\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"name\":\"syn\",\"req\":\"^2.0.31\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.85\"}],\"features\":{}}", - "zerocopy-derive_0.8.26": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"dissimilar\",\"req\":\"^1.0.9\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"=0.2.163\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"=1.9\"},{\"kind\":\"dev\",\"name\":\"prettyplease\",\"req\":\"=0.2.17\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.10\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.89\"}],\"features\":{}}", - "zerocopy-derive_0.8.42": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"dissimilar\",\"req\":\"^1.0.9\"},{\"kind\":\"dev\",\"name\":\"prettyplease\",\"req\":\"=0.2.17\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.40\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"features\":[\"visit\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.46\"}],\"features\":{}}", - "zerocopy_0.7.35": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"assert_matches\",\"req\":\"^1.5\"},{\"default_features\":false,\"name\":\"byteorder\",\"optional\":true,\"req\":\"^1.3\"},{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.85\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.7.35\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.7.35\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.7.35\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\"],\"alloc\":[],\"default\":[\"byteorder\"],\"derive\":[\"zerocopy-derive\"],\"simd\":[],\"simd-nightly\":[\"simd\"]}}", - "zerocopy_0.8.26": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"either\",\"req\":\"=1.13.0\"},{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.89\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.8.26\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\",\"std\"],\"alloc\":[],\"derive\":[\"zerocopy-derive\"],\"float-nightly\":[],\"simd\":[],\"simd-nightly\":[\"simd\"],\"std\":[\"alloc\"]}}", - "zerocopy_0.8.42": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.8.42\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.8.42\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.8.42\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\",\"std\"],\"alloc\":[],\"derive\":[\"zerocopy-derive\"],\"float-nightly\":[],\"simd\":[],\"simd-nightly\":[\"simd\"],\"std\":[\"alloc\"]}}", - "zmij_1.0.21": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\",\"target\":\"cfg(not(miri))\"},{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1.36\"},{\"kind\":\"dev\",\"name\":\"num-bigint\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"num-integer\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"num_cpus\",\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"opt-level\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"ryu\",\"req\":\"^1\"}],\"features\":{}}" - }, - "@@rules_rs+//rs/experimental/toolchains:module_extension.bzl%toolchains": { - "cargo-1.94.0-aarch64-apple-darwin.tar.xz": "7497ab873993cf38b806548c85bf818ffd71d1c3cf77b5bb4c3d3f1769588a08", - "cargo-1.94.0-aarch64-pc-windows-msvc.tar.xz": "0eacb6af29cacb850af4b390e698b2c8060f639a70e6b9f0575cdcc82b0822b2", - "cargo-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "d57ab7b23b52cba46e58c78327904d0c7db19892988677b0d59495c8af4bb790", - "cargo-1.94.0-x86_64-apple-darwin.tar.xz": "3beb7874010ebd757780284c0c9da47fc9f0b18091d7415d7b7231a8c19ec44f", - "cargo-1.94.0-x86_64-pc-windows-msvc.tar.xz": "6cf7113e1d5633721739a3bd6fee7148d6394bbab748fbede941210100374741", - "cargo-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "8e17624f3de39e079845bfb25ed15a042f4b50ceca78e37c56c4b9b15949b9f7", - "clippy-1.94.0-aarch64-apple-darwin.tar.xz": "42a683cbac0b81b96ad3b887cdade36a57ef3a131c203583ca7e3ce0be98f415", - "clippy-1.94.0-aarch64-pc-windows-msvc.tar.xz": "44468016e27457640c7ab62b7ceaddb44308e8ec77ee4344c1bd36113448c8eb", - "clippy-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "f7450c414e394450b53be602d37c374e2a2c5a2f51be2b54faf3d1e3272160b1", - "clippy-1.94.0-x86_64-apple-darwin.tar.xz": "1393cc688bc4d784a9d66619d2eec683a1ddb30b263f03ab615c0a2b18053126", - "clippy-1.94.0-x86_64-pc-windows-msvc.tar.xz": "e4dcc5a796857589e364064493abb3afb469e0e36e90155e7b0a2c5ce4bb7503", - "clippy-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "935529b0b9150084fa9cc1734c6598efe86b6e7d76f31ad3f8d4bb79ebf6031a", - "rust-std-1.94.0-aarch64-apple-darwin.tar.xz": "d3293668128b0a7838ef2544ae341eadd4437460ca22098af723349885a10ba2", - "rust-std-1.94.0-aarch64-apple-ios-macabi.tar.xz": "914f085065539ea1038d2832a5af1c22852cd5e9a063356b20e46698e72e770e", - "rust-std-1.94.0-aarch64-apple-ios-sim.tar.xz": "b4d16842b8f98ff2f431d6ab4aced298f4028ca00f0f5d588bc6efc94faa0680", - "rust-std-1.94.0-aarch64-apple-ios.tar.xz": "41ec1d18f4762f85231a19cf8aba3385da78fece2ff900a435c62881a3a3b44b", - "rust-std-1.94.0-aarch64-linux-android.tar.xz": "402c80df672a4b9fc948b6dc8bde21a63e1093eb1c8a2c2bd42221c309d9eba2", - "rust-std-1.94.0-aarch64-pc-windows-gnullvm.tar.xz": "8c89521f6826a1d2bf07b27c95405f4d6bf7c24f3de832c02758c41d4c1a511f", - "rust-std-1.94.0-aarch64-pc-windows-msvc.tar.xz": "d7964debd444dca92e771c953c3cd582fb927031b536c1439e1a831014d5f0d1", - "rust-std-1.94.0-aarch64-unknown-fuchsia.tar.xz": "87e11d97055540a0675bdc3dacf3d3d8cca5f806171ad0b51d0427f7cb9510c9", - "rust-std-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "c781b3ef4fefa5508fbe05820eddc95e46351d905a30921cc020febd9c596a2e", - "rust-std-1.94.0-aarch64-unknown-linux-musl.tar.xz": "3513032b2953d795bd3ac1489de4e8174e72500a31b4e7d1b349a23ab78a7a7b", - "rust-std-1.94.0-aarch64-unknown-none-softfloat.tar.xz": "375c496af31cf208ce67613c0aa1bde1894b5d674b00050cd46f7f954ed4a668", - "rust-std-1.94.0-aarch64-unknown-none.tar.xz": "854a9f8cea0ba889f553a7ea664c2a6479fc1fc49a4b56f84f99fc37001c4322", - "rust-std-1.94.0-aarch64-unknown-uefi.tar.xz": "f038d9e3df557d4aecdcf65989896179bcf0ecbc4d62bde71d33d7b914d3c253", - "rust-std-1.94.0-arm-linux-androideabi.tar.xz": "d2113ef50ed0b974f33b41102b961588c26abaf34f45a289a293b3d31ba01113", - "rust-std-1.94.0-arm-unknown-linux-gnueabi.tar.xz": "ad364b54b668e866315900ebff0d225616c556ab5228599792470aede05ba42c", - "rust-std-1.94.0-arm-unknown-linux-gnueabihf.tar.xz": "d5db943974fe05306ae43d100b0b7f3540108b93ef52de670675ce966af8a0fd", - "rust-std-1.94.0-arm-unknown-linux-musleabi.tar.xz": "c724b1a1b5dddeca563cda9b9695610bb56e681727b3af0a352174960087069a", - "rust-std-1.94.0-arm-unknown-linux-musleabihf.tar.xz": "9f8b707760a632b80864e91ea81a1b19c3f5452ff5e18b96da75792be1cf0a3c", - "rust-std-1.94.0-armv7-linux-androideabi.tar.xz": "e0b6883364f939300c281d96d6778ecf7fd97fe867b11936f0aa5c9a29f488ad", - "rust-std-1.94.0-armv7-unknown-linux-gnueabi.tar.xz": "9b1d8143881d510785d89413ea09b9fe910b64e3f300ab30e52e7fdd652ea66a", - "rust-std-1.94.0-armv7-unknown-linux-gnueabihf.tar.xz": "4b1744a8e48f58c2741afc1e076d0629a86051d785bd36e2c97f329eab6a5b49", - "rust-std-1.94.0-armv7-unknown-linux-musleabi.tar.xz": "fc2754f50ece0c7d3d32b2e0034b19c8b8bba9b471e0eb00f8ece7fa0bcb2837", - "rust-std-1.94.0-armv7-unknown-linux-musleabihf.tar.xz": "abfc0e7c1eaff1e3332b5c6262d12da309035a228261d07155add44464ed52f4", - "rust-std-1.94.0-i686-linux-android.tar.xz": "9093a894d87568a4b5eb4a144758bf49d4b62743677f747848afc3807c23f126", - "rust-std-1.94.0-i686-pc-windows-gnu.tar.xz": "188490bde6649163347050a70fd81c6d9e2e972ef12ba646f7f4fa106e21a956", - "rust-std-1.94.0-i686-pc-windows-gnullvm.tar.xz": "ba61dc038c3f4102a832a3381935192e88b0a1f79ff50b6d5590d476bdfa6879", - "rust-std-1.94.0-i686-pc-windows-msvc.tar.xz": "388cc2bb17ef9f12864c08e462cdbe3222d5ddafd84969dcf4ed944da27efcbd", - "rust-std-1.94.0-i686-unknown-freebsd.tar.xz": "3ce634092bcecb000c793c9d4bb0ccc6fd45c66fa8259fbb4f18c59844c4b13b", - "rust-std-1.94.0-i686-unknown-linux-gnu.tar.xz": "528c1dda491b25bb855426f48757773f4740c75d3c7271efffe7412c2fd5730d", - "rust-std-1.94.0-i686-unknown-linux-musl.tar.xz": "90ff16d879f774048c9b81d07188af4b63ed73a31848759cf560a9f51baa175f", - "rust-std-1.94.0-i686-unknown-uefi.tar.xz": "8c095069ec09d6e1d0b7e00f340fd7ea925a31acbad845d5956e1bc4a7d47733", - "rust-std-1.94.0-powerpc-unknown-linux-gnu.tar.xz": "23e32cab484d7787b7966a0508c49baf231bcdb4a79a421ee85ca69e81f2a06f", - "rust-std-1.94.0-powerpc64-unknown-linux-gnu.tar.xz": "7713b676a37fbd0a52a711c2e3c205a917f8b05614e17cd9f7f37874a88469a3", - "rust-std-1.94.0-powerpc64le-unknown-linux-gnu.tar.xz": "094f3955828d6630b70a9c7c71e3165c3ffc7a301929601dde08501def623347", - "rust-std-1.94.0-powerpc64le-unknown-linux-musl.tar.xz": "f3956b29463ec58d730f054ff2334755f929c602181ad67a25af9725c8500f68", - "rust-std-1.94.0-riscv32imc-unknown-none-elf.tar.xz": "5d10c7ab74cd5d10451480ec894ac5b98940a8527c333465eeec4a45b1125d43", - "rust-std-1.94.0-riscv64gc-unknown-linux-gnu.tar.xz": "32e9b7006f1269e48fb65af6298f6e8a222e1a8a2a20b112528bfb3dad2f65a8", - "rust-std-1.94.0-riscv64gc-unknown-linux-musl.tar.xz": "2c63d21143c7f1d44f3e3b1c9a82f3101576344c3fd1feb2d43ffc9d035e94f1", - "rust-std-1.94.0-riscv64gc-unknown-none-elf.tar.xz": "071a1ae54f7dead1b8147b9bae7d831a5554a37a1133f8cd52b265a765595a12", - "rust-std-1.94.0-s390x-unknown-linux-gnu.tar.xz": "ea69833addbc9521664464418c7468871f7a80b16fd2c385f6213a6c99cedfe9", - "rust-std-1.94.0-thumbv6m-none-eabi.tar.xz": "b022df94f6efa3bd2a6c4042472e114d2325bf058983b3ac7d586e24c8d77f25", - "rust-std-1.94.0-thumbv7em-none-eabi.tar.xz": "41894db05c4d8f421bf0dba7d0bed995685032393779f92b4ea1f3e6468badfb", - "rust-std-1.94.0-thumbv7em-none-eabihf.tar.xz": "be7e81535e7118fbad58a5dc31a1acfa7f2a36a2348aed23595169418559d67f", - "rust-std-1.94.0-thumbv7m-none-eabi.tar.xz": "4702e615be264df97b120fd075bed4e46710cc9a03c48115ce2ee3abfbf3822e", - "rust-std-1.94.0-thumbv8m.main-none-eabi.tar.xz": "5c1fb5ed8ad31580241b6e095583a2cac0ef90ea29dad7705c8a0ce0933a7797", - "rust-std-1.94.0-thumbv8m.main-none-eabihf.tar.xz": "573e9570bea039316ae3b55e323cbc741617ace1e0b87df8f3ecbbdf34b3de8a", - "rust-std-1.94.0-wasm32-unknown-emscripten.tar.xz": "0a607ba5d78afb47e9f5f134633ed5a954ece5558298e081084abe44f5265463", - "rust-std-1.94.0-wasm32-unknown-unknown.tar.xz": "bc65af3c0bfa2e5b20e299358710a26ce33a3674c762a2a010bc383c5b7fc202", - "rust-std-1.94.0-wasm32-wasip1-threads.tar.xz": "311b9fe1e94c90c16695899e670f0d76a366d493cdda29008191ce2d436cd22c", - "rust-std-1.94.0-wasm32-wasip1.tar.xz": "acbacacdca3e0ec0d93ff46d5f3579ae8cd5133b86e31a893da22cd94e28d0bf", - "rust-std-1.94.0-wasm32-wasip2.tar.xz": "e947e730da3fea212453b2835765802d9b0c80e19405cab188a6854834196e6c", - "rust-std-1.94.0-x86_64-apple-darwin.tar.xz": "cf716b5a5fab007398ce76536a85a3449df074b66e7c762c0f5de1d6c56a1342", - "rust-std-1.94.0-x86_64-apple-ios-macabi.tar.xz": "f39f4c1121dafcf8ea70a21c91be9552e30ea76debcad9bff28be945aaa79282", - "rust-std-1.94.0-x86_64-apple-ios.tar.xz": "60e70b80469dd3900dd8b1705831f5f02ac44fa97088b8a81fdda110ed421091", - "rust-std-1.94.0-x86_64-linux-android.tar.xz": "488c8a581612ac4779a05f05967b76a96e0ff0bdd1ab8d99c13a859c19056b01", - "rust-std-1.94.0-x86_64-pc-windows-gnu.tar.xz": "c9f98d62213d0559c8d2ee6d7eafbafbf5dfe04921c9e7cb15b66a770903bc33", - "rust-std-1.94.0-x86_64-pc-windows-gnullvm.tar.xz": "54c28d0030fab585457a431c555e5d23502132364d0483e5b3b2dfbfc3929cc6", - "rust-std-1.94.0-x86_64-pc-windows-msvc.tar.xz": "f8e2b43e52957e8224d97de68d6ec5da01613eb98bce4199de80071678b5fda0", - "rust-std-1.94.0-x86_64-unknown-freebsd.tar.xz": "081972f2f386ee859c15ed619695a9440896a80d1814d0e29d7620b7ef96207c", - "rust-std-1.94.0-x86_64-unknown-fuchsia.tar.xz": "6c18112553cf72cf991352208cce2bc96c712ca7b209acdd098abac894d674ef", - "rust-std-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "dd33653107c36e040082050d9e547e64dac5b456ba74069430d838c00c189a05", - "rust-std-1.94.0-x86_64-unknown-linux-musl.tar.xz": "b6786b5c490df2dcef76829039177799058f94e17725b42f912a057c0f6224f2", - "rust-std-1.94.0-x86_64-unknown-netbsd.tar.xz": "f763e56ca1ce3f819fb2136638d774777dd696f836cc52e6564431859a0f5159", - "rust-std-1.94.0-x86_64-unknown-none.tar.xz": "8509489165a8aa60a0590aeaa668f63423b3e7a33bb93e8e5719ff5404b6f6aa", - "rust-std-1.94.0-x86_64-unknown-uefi.tar.xz": "31b56e8178dd2319e51ccc474959d247db2767cda7149fe3704d636df9aee1c9", - "rustc-1.94.0-aarch64-apple-darwin.tar.xz": "6ba00a4f486af11826335cd7fe405592d959f50afb9489176fc1463cbe75dbf2", - "rustc-1.94.0-aarch64-pc-windows-msvc.tar.xz": "6d88d89c200a147357ded65e9bc426e14ebb6f5046d9772f3916b6a1ac60cdd5", - "rustc-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "7261b2fcdb88aef8b21f4b9f915d9a75c959674193f5b38e13fab4569684fc5b", - "rustc-1.94.0-x86_64-apple-darwin.tar.xz": "b1374b0fb63f5abcd8eb65607e43a499bfdc5e8160746d1f6f993aee9313f320", - "rustc-1.94.0-x86_64-pc-windows-msvc.tar.xz": "e0f59503778b563d3ee6d4edecd52b44056779ffe456181bad1fb4dc069802e6", - "rustc-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "31a0d3ac9383dfdeb4fce86eeed5ade3230131c635264c0eab7252dbf235f28e", - "rustfmt-1.94.0-aarch64-apple-darwin.tar.xz": "0eb716d201dc0d6d24ef8207d3ef27611d9d15d2212e3b27aa11e97286ed02ac", - "rustfmt-1.94.0-aarch64-pc-windows-msvc.tar.xz": "705ac3e797c4abed9e722e0864902f7a03872a8a201dcd7b70ff1b749eed2327", - "rustfmt-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "1ac17dbe7f8f6c98db5b62cb149fcd10c7565d624a59471115fc40055db9af28", - "rustfmt-1.94.0-x86_64-apple-darwin.tar.xz": "e58601a10ee85e956611ea2814bf26de88934a8224c1663bb11aa23f727663ec", - "rustfmt-1.94.0-x86_64-pc-windows-msvc.tar.xz": "7b32d8b0dd99f9bf6ddaa4866d39b7b96e8c2db46644d8527096f7f34b4fbe26", - "rustfmt-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "903e768f7626f78ed2d6cb82c1820b93bd339afec8575102e847fd3a379d33e5" - } - } + "facts": {} } diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 3eca623e..74aa5dc0 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -1128,6 +1128,9 @@ pub const paged_fa2 = struct { }; pub const paged_fa3 = struct { + // God knows why flash attention uses this number and not something else. + const MAX_NUM_SPLITS = 32; + pub const Variant = enum { decode, mixed, @@ -1181,6 +1184,7 @@ pub const paged_fa3 = struct { max_seqlen_k: usize, max_token_count: usize, num_heads: usize, + num_kv_heads: usize, head_dim: usize, }; @@ -1194,10 +1198,10 @@ pub const paged_fa3 = struct { pub fn init(options: DecodeOptions) DecodeParameters { return .{ - .block_table = .init(.{ options.batch_size, options.max_num_pages }, .i32), - .cu_seqlens_q = .init(.{options.batch_size + 1}, .i32), - .seqused_k = .init(.{options.batch_size}, .i32), - .metadata = DecodeMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .block_table = .init(.{ .b = options.batch_size, .p = options.max_num_pages }, .i32), + .cu_seqlens_q = .init(.{ .b = options.batch_size + 1 }, .i32), + .seqused_k = .init(.{ .b = options.batch_size }, .i32), + .metadata = DecodeMetadata.init(options), .options = options, }; } @@ -1217,9 +1221,16 @@ pub const paged_fa3 = struct { pub const DecodeMetadata = struct { out_accum: zml.Tensor, - pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) DecodeMetadata { + pub fn init(opts: DecodeOptions) DecodeMetadata { + const shape = zml.Shape.init(.{ + .splits = MAX_NUM_SPLITS, + .hkv = opts.num_kv_heads, + .hg = @divExact(opts.num_heads, opts.num_kv_heads), + .b = opts.max_token_count, + .hd = opts.head_dim, + }, .f32).withPartitioning(.{ .hkv = .model }); return .{ - .out_accum = .init(.{32 * max_token_count * num_heads * head_dim * 4}, .i8), + .out_accum = .fromShape(shape), }; } @@ -1247,6 +1258,7 @@ pub const paged_fa3 = struct { max_seqlen_k: usize, max_token_count: usize, num_heads: usize, + num_kv_heads: usize, head_dim: usize, }; @@ -1264,13 +1276,13 @@ pub const paged_fa3 = struct { pub fn init(options: MixedOptions) MixedParameters { return .{ - .block_table_prefill = .init(.{ options.batch_size_prefill, options.max_num_pages }, .i32), - .cu_seqlens_q_prefill = .init(.{options.batch_size_prefill + 1}, .i32), - .seqused_k_prefill = .init(.{options.batch_size_prefill}, .i32), - .block_table_decode = .init(.{ options.batch_size_decode, options.max_num_pages }, .i32), - .cu_seqlens_q_decode = .init(.{options.batch_size_decode + 1}, .i32), - .seqused_k_decode = .init(.{options.batch_size_decode}, .i32), - .metadata = MixedMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .block_table_prefill = .init(.{ .b = options.batch_size_prefill, .p = options.max_num_pages }, .i32), + .cu_seqlens_q_prefill = .init(.{ .b = options.batch_size_prefill + 1 }, .i32), + .seqused_k_prefill = .init(.{ .b = options.batch_size_prefill }, .i32), + .block_table_decode = .init(.{ .b = options.batch_size_decode, .p = options.max_num_pages }, .i32), + .cu_seqlens_q_decode = .init(.{ .b = options.batch_size_decode + 1 }, .i32), + .seqused_k_decode = .init(.{ .b = options.batch_size_decode }, .i32), + .metadata = MixedMetadata.init(options), .options = options, }; } @@ -1294,9 +1306,16 @@ pub const paged_fa3 = struct { out_accum: zml.Tensor, host_metadata: zml.Tensor, - pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) MixedMetadata { + pub fn init(opts: MixedOptions) MixedMetadata { + const shape = zml.Shape.init(.{ + .splits = MAX_NUM_SPLITS, + .hkv = opts.num_kv_heads, + .hg = @divExact(opts.num_heads, opts.num_kv_heads), + .b = opts.max_token_count, + .hd = opts.head_dim, + }, .f32).withPartitioning(.{ .hkv = .model }); return .{ - .out_accum = .init(.{32 * max_token_count * num_heads * head_dim * 4}, .i8), + .out_accum = .fromShape(shape), .host_metadata = .init(.{2}, .i32), }; } @@ -1388,7 +1407,7 @@ pub const paged_fa3 = struct { .window_size_right = window_size_right, .softcap = 0.0, .is_rotary_interleaved = false, - .num_splits = 0, + .num_splits = MAX_NUM_SPLITS, .sm_margin = 0, .cp_world_size = 1, .cp_rank = 0, @@ -1467,7 +1486,7 @@ pub const paged_fa3 = struct { .window_size_right = window_size_right, .softcap = 0.0, .is_rotary_interleaved = false, - .num_splits = 0, + .num_splits = MAX_NUM_SPLITS, .sm_margin = 0, .cp_world_size = 1, .cp_rank = 0, @@ -1501,28 +1520,48 @@ pub const paged_fa3 = struct { }; pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { - stdx.debug.assert(q.shape().hasTags(.{ .b, .h, .hd }), "Expected q to have tags .b, .h, .hd", .{}); - stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); - stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); + stdx.debug.assert(q.shape().hasTags(.{ .b, .hg, .hkv, .hd }), "Expected q to have tags .b, .h, .hd", .{}); + stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); + stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); + const num_head_groups = q.dim(.hg); + const num_kv_heads = q.dim(.hkv); const o = switch (parameters) { .decode => |decode_parameters| b: { - const softmax_lse = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.dim(.h), .q = q.dim(.b) }, .f32)); - const softmax_lse_accum = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * decode_parameters.block_table.dim(0) * q.dim(.h) * 4}, .i8)); - const scheduler_metadata = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = decode_parameters.block_table.dim(0) + 1 }, .i32)); + const batch_size = decode_parameters.block_table.dim(0); - const output_shape = q.shape(); - const o = zml.ops.manualComputation( + const block_table = decode_parameters.block_table.withPartitioning(.{ .b = .replicated }); + const cu_seqlens_q = decode_parameters.cu_seqlens_q.withPartitioning(.{ .b = .replicated }); + const seqused_k = decode_parameters.seqused_k.withPartitioning(.{ .b = .replicated }); + const out_accum = decode_parameters.metadata.out_accum.withPartitioning(.{ .hkv = .model }); + + const softmax_lse = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + .hkv = num_kv_heads, + .hg = num_head_groups, + .q = q.dim(.b), + }, .f32)).withPartitioning(.{ .hkv = .model }); + const softmax_lse_accum = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + .splits = MAX_NUM_SPLITS, + .q = q.dim(.b), + .hkv = num_kv_heads, + .hg = num_head_groups, + }, .f32)).withPartitioning(.{ .hkv = .model }); + const scheduler_metadata = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = batch_size + 1 }, .i32)).withPartitioning(.{ .b = .replicated }); + + var q2 = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + + const output_shape = q2.shape(); + var o = zml.ops.manualComputation( .{ - q, + q2, k_cache, v_cache, - decode_parameters.cu_seqlens_q, - decode_parameters.seqused_k, - decode_parameters.block_table, + cu_seqlens_q, + seqused_k, + block_table, softmax_lse, softmax_lse_accum, - decode_parameters.metadata.out_accum, + out_accum, scheduler_metadata, layer_index, }, @@ -1544,27 +1583,51 @@ pub const paged_fa3 = struct { }).body, ); + o = o.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = num_head_groups }); + break :b o; }, .mixed => |mixed_parameters| b: { - const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = q.dim(.b) }, .f32)); - const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * q.dim(.b) * q.dim(.h) * 4}, .i8)); - const scheduler_metadata_prefill = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = mixed_parameters.block_table_prefill.dim(0) + 1 }, .i32)); + const batch_size_prefill = mixed_parameters.block_table_prefill.dim(0); - const output_shape = q.shape(); + const block_table_prefill = mixed_parameters.block_table_prefill.withPartitioning(.{ .b = .replicated }); + const cu_seqlens_q_prefill = mixed_parameters.cu_seqlens_q_prefill.withPartitioning(.{ .b = .replicated }); + const seqused_k_prefill = mixed_parameters.seqused_k_prefill.withPartitioning(.{ .b = .replicated }); + const block_table_decode = mixed_parameters.block_table_decode.withPartitioning(.{ .b = .replicated }); + const cu_seqlens_q_decode = mixed_parameters.cu_seqlens_q_decode.withPartitioning(.{ .b = .replicated }); + const seqused_k_decode = mixed_parameters.seqused_k_decode.withPartitioning(.{ .b = .replicated }); + const out_accum = mixed_parameters.metadata.out_accum.withPartitioning(.{ .hkv = .model }); + const host_metadata = mixed_parameters.metadata.host_metadata.withTags(.{.x}).withPartitioning(.{ .x = .replicated }); + + const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + .hkv = num_kv_heads, + .hg = num_head_groups, + .q = q.dim(.b), + }, .f32)).withPartitioning(.{ .hkv = .model }); + const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + .splits = MAX_NUM_SPLITS, + .q = q.dim(.b), + .hkv = num_kv_heads, + .hg = num_head_groups, + }, .f32)).withPartitioning(.{ .hkv = .model }); + const scheduler_metadata_prefill = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = batch_size_prefill + 1 }, .i32)).withPartitioning(.{ .b = .replicated }); + + var q2 = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + + const output_shape = q2.shape(); var o = zml.ops.manualComputation( .{ - q, + q2, k_cache, v_cache, - mixed_parameters.cu_seqlens_q_prefill, - mixed_parameters.seqused_k_prefill, - mixed_parameters.block_table_prefill, + cu_seqlens_q_prefill, + seqused_k_prefill, + block_table_prefill, softmax_lse_prefill, softmax_lse_accum_prefill, - mixed_parameters.metadata.out_accum, + out_accum, scheduler_metadata_prefill, - mixed_parameters.metadata.host_metadata, + host_metadata, layer_index, }, output_shape, @@ -1583,24 +1646,37 @@ pub const paged_fa3 = struct { }).body, ); + o = o.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = num_head_groups }); + const batch_size_decode = mixed_parameters.block_table_prefill.dim(0); - const softmax_lse_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = batch_size_decode }, .f32)); - const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * mixed_parameters.block_table_decode.dim(0) * q.dim(.h) * 4}, .i8)); - const scheduler_metadata_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = mixed_parameters.block_table_decode.dim(0) + 1 }, .i32)); - var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_size_decode }); + const softmax_lse_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + .hkv = num_kv_heads, + .hg = num_head_groups, + .q = q.dim(.b), + }, .f32)); + const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + .splits = MAX_NUM_SPLITS, + .q = q.dim(.b), + .hkv = num_kv_heads, + .hg = num_head_groups, + }, .f32)); + const scheduler_metadata_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = batch_size_decode + 1 }, .i32)).withPartitioning(.{ .b = .replicated }); + var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_size_decode }).withPartitioning(.{ .hkv = .model }); + + q_decode = q_decode.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); const decode_output_shape = q_decode.shape(); - const o_decode = zml.ops.manualComputation( + var o_decode = zml.ops.manualComputation( .{ q_decode, k_cache, v_cache, - mixed_parameters.cu_seqlens_q_decode, - mixed_parameters.seqused_k_decode, - mixed_parameters.block_table_decode, + cu_seqlens_q_decode, + seqused_k_decode, + block_table_decode, softmax_lse_decode, softmax_lse_accum_decode, - mixed_parameters.metadata.out_accum, + out_accum, scheduler_metadata_decode, layer_index, }, @@ -1619,6 +1695,7 @@ pub const paged_fa3 = struct { } }).body, ); + o_decode = o_decode.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = num_head_groups }); o = o.dynamicUpdateSlice1d(o_decode, 0, context.decode_offset.?); break :b o; diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index a3126ff4..10a86939 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -83,6 +83,7 @@ pub const Options = union(Backend) { .max_seqlen_k = args.seq_len, .max_token_count = args.max_token_count, .num_heads = args.num_heads, + .num_kv_heads = args.num_kv_heads, .head_dim = args.head_dim, }, }, @@ -94,6 +95,7 @@ pub const Options = union(Backend) { .max_seqlen_k = args.seq_len, .max_token_count = args.max_token_count, .num_heads = args.num_heads, + .num_kv_heads = args.num_kv_heads, .head_dim = args.head_dim, }, }, From a96b8bd84f08d735b0606f2a8049ee615d1a7ab9 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Mon, 23 Mar 2026 13:22:59 +0000 Subject: [PATCH 373/881] platforms/cuda: fix CUDNN redist sha --- platforms/cuda/cuda.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 43947746..6785e80f 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -20,7 +20,7 @@ CUDA_REDIST_JSON_SHA256 = "97cf605ccc4751825b1865f4af571c9b50dd29ffd13e9a38b296a CUDNN_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cudnn/redist/" CUDNN_VERSION = "9.19.1" -CUDNN_REDIST_JSON_SHA256 = "6ad4f2c047ee03131d1efb95db56f78c24953cb30880a2edcf99407ff9362a04" +CUDNN_REDIST_JSON_SHA256 = "ee7bd6872b8611017bfc9ac99a4a71932652d1851b5917aa2c66bf29a12f8fd4" NVSHMEM_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/nvshmem/redist/" NVSHMEM_VERSION = "3.5.19" From 3b76703430d76c47401b19bd9cf78782123dac68 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Mon, 23 Mar 2026 14:55:48 +0100 Subject: [PATCH 374/881] workspace: fixes for build and test //... green Fixes: - Added `target_compatible_with = ["@platforms//os:linux"],` for neuron & rocm - Updated the zig in neuron - Updated neuron to use Python 3.12, it's at best a mess to handle two different python version with `build //...` - renamed `third_party/flashattn/BUILD.bazel` to `third_party/flashattn/flashattention.BUILD.bazel` as it's not part of this module. - Used HTTPS & fallback URL for swig, sometimes we would end up with an incorrect archive (wrong sha256) - Using `--@rules_python//python/config_settings:bootstrap_impl=script` for hermetic Python build, it relied on the system Python otherwise. - Used a separate pip extension for neuron to avoid the extra pip args to leak to other installations. - Fix `zml/testing` `fn autoCall` function --------- Co-authored-by: Hugo Mano --- .bazelignore | 1 + .bazelrc | 2 + .github/workflows/ci.yaml | 11 +- MODULE.bazel | 16 +- platforms/neuron/BUILD.bazel | 5 +- platforms/neuron/libneuronxla.zig | 62 +- platforms/neuron/neuron.bzl | 6 +- platforms/neuron/neuron.zig | 2 +- platforms/neuron/neuronx-cc.zig | 21 +- platforms/neuron/requirements.lock.txt | 632 +++++++++--------- platforms/rocm/BUILD.bazel | 1 + third_party/flashattn/BUILD.bazel | 13 - .../flashattn/flashattention.BUILD.bazel | 13 + third_party/flashattn/repo.bzl | 2 +- third_party/org_swig_swig/repo.bzl | 5 +- third_party/zls/BUILD.bazel | 3 + upb/upb.zig | 2 - zml/testing.zig | 2 +- 18 files changed, 424 insertions(+), 375 deletions(-) create mode 100644 third_party/flashattn/flashattention.BUILD.bazel diff --git a/.bazelignore b/.bazelignore index 1ef6fd4c..875340d8 100644 --- a/.bazelignore +++ b/.bazelignore @@ -1,3 +1,4 @@ .* third_party/modules .zig-cache +.devenv diff --git a/.bazelrc b/.bazelrc index 61ea8613..8286df2d 100644 --- a/.bazelrc +++ b/.bazelrc @@ -28,6 +28,8 @@ common --enable_runfiles # Ensure the local env doesn't leak into the build common --incompatible_strict_action_env +# Ensure rules_python doesn't rely on the system's Python for stage-1 bootstrap +common --@rules_python//python/config_settings:bootstrap_impl=script # We don't to check the built files in the sandbox or repositories common --experimental_check_output_files=false diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0f483a89..8b422d96 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -129,19 +129,24 @@ jobs: with: bazelisk-version: 1.28.1 bazelisk-cache: true + # Both cache are useful even with the remote execution. + # no cache 5min6s build + # repository-cache: 37s cache load + 4m3s build + # external-cache: 2m29s cache load + 1m25s build + # As of this comment, we're also limited by the runner's CPU with only 4 cores. repository-cache: true bazelrc: | common --config=remote common --remote_header=x-buildbuddy-api-key=${{ secrets.BUILDBUDDY_CI_API_KEY }} - common -c opt common --verbose_failures common --keep_going common --test_output=streamed common --color=yes common --show_timestamps - common --disk_cache=/home/runner/.cache/bazel-disk - common --strip=always + common --remote_download_outputs=minimal - run: bazel mod explain - run: bazel mod graph - run: bazel query //... + - run: bazel build //... + - run: bazel test //... diff --git a/MODULE.bazel b/MODULE.bazel index fd201efd..fa3f5f7f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -83,14 +83,26 @@ python.toolchain(python_version = "3.12") use_repo(python, "python_versions") pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") -pip.parse( +pip_neuron = use_extension("@rules_python//python/extensions:pip.bzl", "pip") + +pip_neuron.parse( download_only = True, hub_name = "neuron_py_deps", python_version = "3.12", + extra_pip_args = [ + "--abi=cp312", + "--implementation=cp", + "--python-version=312", + "--platform=linux_x86_64", + "--platform=manylinux2014_x86_64", + "--platform=manylinux_2_27_x86_64", + "--platform=manylinux_2_28_x86_64", + "--platform=manylinux_2_29_x86_64", + ], requirements_lock = "//platforms/neuron:requirements.lock.txt", target_platforms = ["linux_x86_64"], ) -use_repo(pip, "neuron_py_deps") +use_repo(pip_neuron, "neuron_py_deps") # For downloading models pip.parse( diff --git a/platforms/neuron/BUILD.bazel b/platforms/neuron/BUILD.bazel index a48cd25a..97baeeaa 100644 --- a/platforms/neuron/BUILD.bazel +++ b/platforms/neuron/BUILD.bazel @@ -12,6 +12,7 @@ load("@zml//bazel:runfiles.bzl", "runfiles_to_default") zig_shared_library( name = "libpjrt_neuron", main = "libpjrt_neuron.zig", + target_compatible_with = ["@platforms//os:linux"], visibility = ["@libpjrt_neuron//:__subpackages__"], zigopts = ["-fno-stack-check"], deps = [ @@ -24,6 +25,7 @@ zig_shared_library( zig_binary( name = "neuronx-cc", + target_compatible_with = ["@platforms//os:linux"], data = ["@neuron_py_deps//neuronx_cc"], linkopts = ["-Wl,-rpath,$$ORIGIN/../lib"], main = "neuronx-cc.zig", @@ -33,6 +35,7 @@ zig_binary( runfiles_to_default( name = "neuronx-cc_files", + target_compatible_with = ["@platforms//os:linux"], visibility = ["@libpjrt_neuron//:__subpackages__"], deps = [":neuronx-cc"], ) @@ -76,6 +79,7 @@ zig_shared_library( name = "libneuronxla", main = "libneuronxla.zig", shared_lib_name = "libneuronxla.so", + target_compatible_with = ["@platforms//os:linux"], visibility = ["@libpjrt_neuron//:__subpackages__"], zigopts = [ "-fno-stack-check", @@ -101,7 +105,6 @@ uv_lock( "--upgrade", "--python-platform=x86_64-unknown-linux-gnu", ], - python_version = "3.12", tags = ["manual"], ) diff --git a/platforms/neuron/libneuronxla.zig b/platforms/neuron/libneuronxla.zig index 8185cf56..58b70c1e 100644 --- a/platforms/neuron/libneuronxla.zig +++ b/platforms/neuron/libneuronxla.zig @@ -7,12 +7,12 @@ const upb = @import("upb"); const log = std.log.scoped(.@"zml/platforms/neuron/libneuronxla"); pub fn makeTempDir(io: std.Io, buf: []u8, prefix: []const u8) ![]const u8 { - const tmp_dir = std.posix.getenv("TMPDIR") orelse "/tmp"; + const tmp_dir = std.c.getenv("TMPDIR") orelse "/tmp"; const ret = try std.fmt.bufPrint(buf, "{s}{s}{s}{d}", .{ tmp_dir, std.Io.Dir.path.sep_str_posix, prefix, - (try std.Io.Clock.now(.real, io)).toNanoseconds(), + std.Io.Clock.now(.real, io).toNanoseconds(), }); try std.Io.Dir.createDir(.cwd(), io, ret, .fromMode(0o700)); return ret; @@ -192,34 +192,36 @@ fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) const neff_file = try std.Io.Dir.path.join(arena.allocator(), &.{ tmp_dir, "file.neff" }); var neuronx_cc_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - var child: std.process.Child = .init(&.{ - try stdx.Io.Dir.path.bufJoin(&neuronx_cc_buf, &.{ - stdx.process.selfSharedObjectDirPath(), - "..", - "bin", - "neuronx-cc", - }), - "compile", - "--framework=XLA", - "--target", - target, - "--verbose=user", - "--enable-internal-neff-wrapper", - "--output", - neff_file, - "--optlevel=1", - // generic is the default, but it fails on transformers, force it - "--model-type=transformer", - // disable it, we do our own - "--auto-cast=none", - "--enable-fast-loading-neuron-binaries", - code_file, - }, arena.allocator()); - child.stdin_behavior = .Ignore; - child.stdout_behavior = .Inherit; - child.stderr_behavior = .Inherit; - child.cwd = tmp_dir; - _ = try child.spawnAndWait(io); + var child = try std.process.spawn(io, .{ + .argv = &.{ + try stdx.Io.Dir.path.bufJoin(&neuronx_cc_buf, &.{ + stdx.process.selfSharedObjectDirPath(), + "..", + "bin", + "neuronx-cc", + }), + "compile", + "--framework=XLA", + "--target", + target, + "--verbose=user", + "--enable-internal-neff-wrapper", + "--output", + neff_file, + "--optlevel=1", + // generic is the default, but it fails on transformers, force it + "--model-type=transformer", + // disable it, we do our own + "--auto-cast=none", + "--enable-fast-loading-neuron-binaries", + code_file, + }, + .stdin = .ignore, + .stdout = .inherit, + .stderr = .inherit, + .cwd = .{ .path = tmp_dir }, + }); + _ = try child.wait(io); std.debug.print(">>>> {s}\n", .{tmp_dir}); diff --git a/platforms/neuron/neuron.bzl b/platforms/neuron/neuron.bzl index 409904ca..1c3f1f91 100644 --- a/platforms/neuron/neuron.bzl +++ b/platforms/neuron/neuron.bzl @@ -1,4 +1,4 @@ -load("@python_versions//3.11:defs.bzl", _py_binary = "py_binary") +load("@python_versions//3.12:defs.bzl", _py_binary = "py_binary") load("@rules_python//python:defs.bzl", "PyInfo") load("@with_cfg.bzl", "with_cfg") load("//bazel:http_deb_archive.bzl", "http_deb_archive") @@ -6,7 +6,7 @@ load("//bazel:simple_repository.bzl", "simple_repository") load("//platforms:packages.bzl", "packages") BASE_URL = "https://apt.repos.neuron.amazonaws.com" -STRIP_PREFIX = "opt/aws/neuron" +STRIP_PREFIX = "./opt/aws/neuron" _BUILD_FILE_PRELUDE = """\ package(default_visibility = ["//visibility:public"]) @@ -101,4 +101,4 @@ neuron_packages = module_extension( py_binary_with_script, _py_binary_internal = with_cfg( kind = _py_binary, extra_providers = [PyInfo], -).set(Label("@rules_python//python/config_settings:bootstrap_impl"), "script").build() +).build() diff --git a/platforms/neuron/neuron.zig b/platforms/neuron/neuron.zig index ad65dc21..e99fd991 100644 --- a/platforms/neuron/neuron.zig +++ b/platforms/neuron/neuron.zig @@ -48,7 +48,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { return error.Unavailable; } - const r = try bazel.runfiles(io, bazel_builtin.current_repository); + const r = try bazel.runfiles(bazel_builtin.current_repository); var sandbox_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const sandbox_path = try r.rlocation("libpjrt_neuron/sandbox", &sandbox_path_buf) orelse { diff --git a/platforms/neuron/neuronx-cc.zig b/platforms/neuron/neuronx-cc.zig index e77c4858..4efafc96 100644 --- a/platforms/neuron/neuronx-cc.zig +++ b/platforms/neuron/neuronx-cc.zig @@ -1,7 +1,6 @@ const std = @import("std"); const c = @import("c"); -const runfiles = @import("runfiles"); fn pyStatusCheck(status: c.PyStatus) void { if (c.PyStatus_Exception(status) != 0) { @@ -21,16 +20,9 @@ pub fn toPosixPathW(file_path: []const u8) error{NameTooLong}![std.posix.PATH_MA return path_with_null; } -pub fn main() !void { - const allocator = std.heap.c_allocator; - - var threaded: std.Io.Threaded = .init(allocator, .{}); - defer threaded.deinit(); - - const io = threaded.io(); - - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); +pub fn main(init: std.process.Init) !void { + const arena = init.arena; + const io = init.io; { var preconfig: c.PyPreConfig = undefined; @@ -47,7 +39,12 @@ pub fn main() !void { config.optimization_level = 2; config.write_bytecode = 0; - _ = c.PyConfig_SetBytesArgv(&config, @intCast(std.os.argv.len), @ptrCast(std.os.argv)); + const args = try init.minimal.args.toSlice(arena.allocator()); + const cArgs = try arena.allocator().alloc([*:0]const u8, args.len); + for (args, 0..) |arg, i| { + cArgs[i] = (try arena.allocator().dupeZ(u8, arg)).ptr; + } + pyStatusCheck(c.PyConfig_SetBytesArgv(&config, @intCast(cArgs.len), @ptrCast(cArgs.ptr))); var self_exe_dir_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const self_exe_dir = blk: { diff --git a/platforms/neuron/requirements.lock.txt b/platforms/neuron/requirements.lock.txt index 19c7bb90..4ff8e3aa 100644 --- a/platforms/neuron/requirements.lock.txt +++ b/platforms/neuron/requirements.lock.txt @@ -4,135 +4,151 @@ --extra-index-url https://pip.repos.neuron.amazonaws.com --find-links https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html -boto3==1.42.24 \ - --hash=sha256:8ed6ad670a5a2d7f66c1b0d3362791b48392c7a08f78479f5d8ab319a4d9118f \ - --hash=sha256:c47a2f40df933e3861fc66fd8d6b87ee36d4361663a7e7ba39a87f5a78b2eae1 +boto3==1.42.72 \ + --hash=sha256:2b5fdac4f202b2ccb9ed21f8b84229463b15573ea16941c2b1b8db1c69e08b63 \ + --hash=sha256:932f023ea3b54fd85df453dcfff7d8c5a0f8be144c0ad57568943505c72c1e6a # via libneuronxla -botocore==1.42.24 \ - --hash=sha256:8fca9781d7c84f7ad070fceffaff7179c4aa7a5ffb27b43df9d1d957801e0a8d \ - --hash=sha256:be8d1bea64fb91eea08254a1e5fea057e4428d08e61f4e11083a02cafc1f8cc6 +botocore==1.42.72 \ + --hash=sha256:038f34553da68df2ce70edc729f170cc198678daaad43f98649727c59f6404d4 \ + --hash=sha256:491b75eb41d46cf99cf8d6cab89f2e2c3602ce8e90d738a1da217a5fb6b2b7ef # via # boto3 # libneuronxla # s3transfer -certifi==2026.1.4 \ - --hash=sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c \ - --hash=sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120 +certifi==2026.2.25 \ + --hash=sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa \ + --hash=sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7 # via requests -charset-normalizer==3.4.4 \ - --hash=sha256:027f6de494925c0ab2a55eab46ae5129951638a49a34d87f4c3eda90f696b4ad \ - --hash=sha256:077fbb858e903c73f6c9db43374fd213b0b6a778106bc7032446a8e8b5b38b93 \ - --hash=sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394 \ - --hash=sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89 \ - --hash=sha256:0f04b14ffe5fdc8c4933862d8306109a2c51e0704acfa35d51598eb45a1e89fc \ - --hash=sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86 \ - --hash=sha256:194f08cbb32dc406d6e1aea671a68be0823673db2832b38405deba2fb0d88f63 \ - --hash=sha256:1bee1e43c28aa63cb16e5c14e582580546b08e535299b8b6158a7c9c768a1f3d \ - --hash=sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f \ - --hash=sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8 \ - --hash=sha256:244bfb999c71b35de57821b8ea746b24e863398194a4014e4c76adc2bbdfeff0 \ - --hash=sha256:2677acec1a2f8ef614c6888b5b4ae4060cc184174a938ed4e8ef690e15d3e505 \ - --hash=sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161 \ - --hash=sha256:2aaba3b0819274cc41757a1da876f810a3e4d7b6eb25699253a4effef9e8e4af \ - --hash=sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152 \ - --hash=sha256:2c9d3c380143a1fedbff95a312aa798578371eb29da42106a29019368a475318 \ - --hash=sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72 \ - --hash=sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4 \ - --hash=sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e \ - --hash=sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3 \ - --hash=sha256:44c2a8734b333e0578090c4cd6b16f275e07aa6614ca8715e6c038e865e70576 \ - --hash=sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c \ - --hash=sha256:4902828217069c3c5c71094537a8e623f5d097858ac6ca8252f7b4d10b7560f1 \ - --hash=sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8 \ - --hash=sha256:4fe7859a4e3e8457458e2ff592f15ccb02f3da787fcd31e0183879c3ad4692a1 \ - --hash=sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2 \ - --hash=sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44 \ - --hash=sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26 \ - --hash=sha256:5947809c8a2417be3267efc979c47d76a079758166f7d43ef5ae8e9f92751f88 \ - --hash=sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016 \ - --hash=sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede \ - --hash=sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf \ - --hash=sha256:5cb4d72eea50c8868f5288b7f7f33ed276118325c1dfd3957089f6b519e1382a \ - --hash=sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc \ - --hash=sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0 \ - --hash=sha256:64b55f9dce520635f018f907ff1b0df1fdc31f2795a922fb49dd14fbcdf48c84 \ - --hash=sha256:6515f3182dbe4ea06ced2d9e8666d97b46ef4c75e326b79bb624110f122551db \ - --hash=sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1 \ - --hash=sha256:6aee717dcfead04c6eb1ce3bd29ac1e22663cdea57f943c87d1eab9a025438d7 \ - --hash=sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed \ - --hash=sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8 \ - --hash=sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133 \ - --hash=sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e \ - --hash=sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef \ - --hash=sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14 \ - --hash=sha256:778d2e08eda00f4256d7f672ca9fef386071c9202f5e4607920b86d7803387f2 \ - --hash=sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0 \ - --hash=sha256:798d75d81754988d2565bff1b97ba5a44411867c0cf32b77a7e8f8d84796b10d \ - --hash=sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828 \ - --hash=sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f \ - --hash=sha256:7c308f7e26e4363d79df40ca5b2be1c6ba9f02bdbccfed5abddb7859a6ce72cf \ - --hash=sha256:7fa17817dc5625de8a027cb8b26d9fefa3ea28c8253929b8d6649e705d2835b6 \ - --hash=sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328 \ - --hash=sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090 \ - --hash=sha256:837c2ce8c5a65a2035be9b3569c684358dfbf109fd3b6969630a87535495ceaa \ - --hash=sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381 \ - --hash=sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c \ - --hash=sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb \ - --hash=sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc \ - --hash=sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a \ - --hash=sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec \ - --hash=sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc \ - --hash=sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac \ - --hash=sha256:9cd98cdc06614a2f768d2b7286d66805f94c48cde050acdbbb7db2600ab3197e \ - --hash=sha256:9d1bb833febdff5c8927f922386db610b49db6e0d4f4ee29601d71e7c2694313 \ - --hash=sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569 \ - --hash=sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3 \ - --hash=sha256:a61900df84c667873b292c3de315a786dd8dac506704dea57bc957bd31e22c7d \ - --hash=sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525 \ - --hash=sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894 \ - --hash=sha256:a8bf8d0f749c5757af2142fe7903a9df1d2e8aa3841559b2bad34b08d0e2bcf3 \ - --hash=sha256:a9768c477b9d7bd54bc0c86dbaebdec6f03306675526c9927c0e8a04e8f94af9 \ - --hash=sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a \ - --hash=sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9 \ - --hash=sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14 \ - --hash=sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25 \ - --hash=sha256:b5d84d37db046c5ca74ee7bb47dd6cbc13f80665fdde3e8040bdd3fb015ecb50 \ - --hash=sha256:b7cf1017d601aa35e6bb650b6ad28652c9cd78ee6caff19f3c28d03e1c80acbf \ - --hash=sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1 \ - --hash=sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3 \ - --hash=sha256:c4ef880e27901b6cc782f1b95f82da9313c0eb95c3af699103088fa0ac3ce9ac \ - --hash=sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e \ - --hash=sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815 \ - --hash=sha256:cb01158d8b88ee68f15949894ccc6712278243d95f344770fa7593fa2d94410c \ - --hash=sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6 \ - --hash=sha256:cc00f04ed596e9dc0da42ed17ac5e596c6ccba999ba6bd92b0e0aef2f170f2d6 \ - --hash=sha256:cd09d08005f958f370f539f186d10aec3377d55b9eeb0d796025d4886119d76e \ - --hash=sha256:cd4b7ca9984e5e7985c12bc60a6f173f3c958eae74f3ef6624bb6b26e2abbae4 \ - --hash=sha256:ce8a0633f41a967713a59c4139d29110c07e826d131a316b50ce11b1d79b4f84 \ - --hash=sha256:cead0978fc57397645f12578bfd2d5ea9138ea0fac82b2f63f7f7c6877986a69 \ - --hash=sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15 \ - --hash=sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191 \ - --hash=sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0 \ - --hash=sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897 \ - --hash=sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd \ - --hash=sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2 \ - --hash=sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794 \ - --hash=sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d \ - --hash=sha256:e912091979546adf63357d7e2ccff9b44f026c075aeaf25a52d0e95ad2281074 \ - --hash=sha256:eaabd426fe94daf8fd157c32e571c85cb12e66692f15516a83a03264b08d06c3 \ - --hash=sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224 \ - --hash=sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838 \ - --hash=sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a \ - --hash=sha256:f155a433c2ec037d4e8df17d18922c3a0d9b3232a396690f17175d2946f0218d \ - --hash=sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d \ - --hash=sha256:f34be2938726fc13801220747472850852fe6b1ea75869a048d6f896838c896f \ - --hash=sha256:f820802628d2694cb7e56db99213f930856014862f3fd943d290ea8438d07ca8 \ - --hash=sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490 \ - --hash=sha256:f8e160feb2aed042cd657a72acc0b481212ed28b1b9a95c0cee1621b524e1966 \ - --hash=sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9 \ - --hash=sha256:fa09f53c465e532f4d3db095e0c55b615f010ad81803d383195b6b5ca6cbf5f3 \ - --hash=sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e \ - --hash=sha256:fd44c878ea55ba351104cb93cc85e74916eb8fa440ca7903e57575e97394f608 +charset-normalizer==3.4.6 \ + --hash=sha256:06a7e86163334edfc5d20fe104db92fcd666e5a5df0977cb5680a506fe26cc8e \ + --hash=sha256:0c173ce3a681f309f31b87125fecec7a5d1347261ea11ebbb856fa6006b23c8c \ + --hash=sha256:0e28d62a8fc7a1fa411c43bd65e346f3bce9716dc51b897fbe930c5987b402d5 \ + --hash=sha256:0e901eb1049fdb80f5bd11ed5ea1e498ec423102f7a9b9e4645d5b8204ff2815 \ + --hash=sha256:11afb56037cbc4b1555a34dd69151e8e069bee82e613a73bef6e714ce733585f \ + --hash=sha256:150b8ce8e830eb7ccb029ec9ca36022f756986aaaa7956aad6d9ec90089338c0 \ + --hash=sha256:172985e4ff804a7ad08eebec0a1640ece87ba5041d565fff23c8f99c1f389484 \ + --hash=sha256:197c1a244a274bb016dd8b79204850144ef77fe81c5b797dc389327adb552407 \ + --hash=sha256:1ae6b62897110aa7c79ea2f5dd38d1abca6db663687c0b1ad9aed6f6bae3d9d6 \ + --hash=sha256:1cf0a70018692f85172348fe06d3a4b63f94ecb055e13a00c644d368eb82e5b8 \ + --hash=sha256:1ed80ff870ca6de33f4d953fda4d55654b9a2b340ff39ab32fa3adbcd718f264 \ + --hash=sha256:22c6f0c2fbc31e76c3b8a86fba1a56eda6166e238c29cdd3d14befdb4a4e4815 \ + --hash=sha256:231d4da14bcd9301310faf492051bee27df11f2bc7549bc0bb41fef11b82daa2 \ + --hash=sha256:259695e2ccc253feb2a016303543d691825e920917e31f894ca1a687982b1de4 \ + --hash=sha256:2a24157fa36980478dd1770b585c0f30d19e18f4fb0c47c13aa568f871718579 \ + --hash=sha256:2b1a63e8224e401cafe7739f77efd3f9e7f5f2026bda4aead8e59afab537784f \ + --hash=sha256:2bd9d128ef93637a5d7a6af25363cf5dec3fa21cf80e68055aad627f280e8afa \ + --hash=sha256:2e1d8ca8611099001949d1cdfaefc510cf0f212484fe7c565f735b68c78c3c95 \ + --hash=sha256:2ef7fedc7a6ecbe99969cd09632516738a97eeb8bd7258bf8a0f23114c057dab \ + --hash=sha256:2f7fdd9b6e6c529d6a2501a2d36b240109e78a8ceaef5687cfcfa2bbe671d297 \ + --hash=sha256:30f445ae60aad5e1f8bdbb3108e39f6fbc09f4ea16c815c66578878325f8f15a \ + --hash=sha256:31215157227939b4fb3d740cd23fe27be0439afef67b785a1eb78a3ae69cba9e \ + --hash=sha256:34315ff4fc374b285ad7f4a0bf7dcbfe769e1b104230d40f49f700d4ab6bbd84 \ + --hash=sha256:3516bbb8d42169de9e61b8520cbeeeb716f12f4ecfe3fd30a9919aa16c806ca8 \ + --hash=sha256:3778fd7d7cd04ae8f54651f4a7a0bd6e39a0cf20f801720a4c21d80e9b7ad6b0 \ + --hash=sha256:39f5068d35621da2881271e5c3205125cc456f54e9030d3f723288c873a71bf9 \ + --hash=sha256:404a1e552cf5b675a87f0651f8b79f5f1e6fd100ee88dc612f89aa16abd4486f \ + --hash=sha256:419a9d91bd238052642a51938af8ac05da5b3343becde08d5cdeab9046df9ee1 \ + --hash=sha256:423fb7e748a08f854a08a222b983f4df1912b1daedce51a72bd24fe8f26a1843 \ + --hash=sha256:4482481cb0572180b6fd976a4d5c72a30263e98564da68b86ec91f0fe35e8565 \ + --hash=sha256:461598cd852bfa5a61b09cae2b1c02e2efcd166ee5516e243d540ac24bfa68a7 \ + --hash=sha256:47955475ac79cc504ef2704b192364e51d0d473ad452caedd0002605f780101c \ + --hash=sha256:48696db7f18afb80a068821504296eb0787d9ce239b91ca15059d1d3eaacf13b \ + --hash=sha256:4be9f4830ba8741527693848403e2c457c16e499100963ec711b1c6f2049b7c7 \ + --hash=sha256:4d1d02209e06550bdaef34af58e041ad71b88e624f5d825519da3a3308e22687 \ + --hash=sha256:4f41da960b196ea355357285ad1316a00099f22d0929fe168343b99b254729c9 \ + --hash=sha256:517ad0e93394ac532745129ceabdf2696b609ec9f87863d337140317ebce1c14 \ + --hash=sha256:51fb3c322c81d20567019778cb5a4a6f2dc1c200b886bc0d636238e364848c89 \ + --hash=sha256:5273b9f0b5835ff0350c0828faea623c68bfa65b792720c453e22b25cc72930f \ + --hash=sha256:530d548084c4a9f7a16ed4a294d459b4f229db50df689bfe92027452452943a0 \ + --hash=sha256:530e8cebeea0d76bdcf93357aa5e41336f48c3dc709ac52da2bb167c5b8271d9 \ + --hash=sha256:54fae94be3d75f3e573c9a1b5402dc593de19377013c9a0e4285e3d402dd3a2a \ + --hash=sha256:572d7c822caf521f0525ba1bce1a622a0b85cf47ffbdae6c9c19e3b5ac3c4389 \ + --hash=sha256:58c948d0d086229efc484fe2f30c2d382c86720f55cd9bc33591774348ad44e0 \ + --hash=sha256:5d11595abf8dd942a77883a39d81433739b287b6aa71620f15164f8096221b30 \ + --hash=sha256:5f8ddd609f9e1af8c7bd6e2aca279c931aefecd148a14402d4e368f3171769fd \ + --hash=sha256:5feb91325bbceade6afab43eb3b508c63ee53579fe896c77137ded51c6b6958e \ + --hash=sha256:60c74963d8350241a79cb8feea80e54d518f72c26db618862a8f53e5023deaf9 \ + --hash=sha256:613f19aa6e082cf96e17e3ffd89383343d0d589abda756b7764cf78361fd41dc \ + --hash=sha256:659a1e1b500fac8f2779dd9e1570464e012f43e580371470b45277a27baa7532 \ + --hash=sha256:695f5c2823691a25f17bc5d5ffe79fa90972cc34b002ac6c843bb8a1720e950d \ + --hash=sha256:69dd852c2f0ad631b8b60cfbe25a28c0058a894de5abb566619c205ce0550eae \ + --hash=sha256:6cceb5473417d28edd20c6c984ab6fee6c6267d38d906823ebfe20b03d607dc2 \ + --hash=sha256:71be7e0e01753a89cf024abf7ecb6bca2c81738ead80d43004d9b5e3f1244e64 \ + --hash=sha256:74119174722c4349af9708993118581686f343adc1c8c9c007d59be90d077f3f \ + --hash=sha256:74a2e659c7ecbc73562e2a15e05039f1e22c75b7c7618b4b574a3ea9118d1557 \ + --hash=sha256:7504e9b7dc05f99a9bbb4525c67a2c155073b44d720470a148b34166a69c054e \ + --hash=sha256:79090741d842f564b1b2827c0b82d846405b744d31e84f18d7a7b41c20e473ff \ + --hash=sha256:7a6967aaf043bceabab5412ed6bd6bd26603dae84d5cb75bf8d9a74a4959d398 \ + --hash=sha256:7bda6eebafd42133efdca535b04ccb338ab29467b3f7bf79569883676fc628db \ + --hash=sha256:7edbed096e4a4798710ed6bc75dcaa2a21b68b6c356553ac4823c3658d53743a \ + --hash=sha256:7f9019c9cb613f084481bd6a100b12e1547cf2efe362d873c2e31e4035a6fa43 \ + --hash=sha256:802168e03fba8bbc5ce0d866d589e4b1ca751d06edee69f7f3a19c5a9fe6b597 \ + --hash=sha256:80d0a5615143c0b3225e5e3ef22c8d5d51f3f72ce0ea6fb84c943546c7b25b6c \ + --hash=sha256:82060f995ab5003a2d6e0f4ad29065b7672b6593c8c63559beefe5b443242c3e \ + --hash=sha256:836ab36280f21fc1a03c99cd05c6b7af70d2697e374c7af0b61ed271401a72a2 \ + --hash=sha256:8761ac29b6c81574724322a554605608a9960769ea83d2c73e396f3df896ad54 \ + --hash=sha256:87725cfb1a4f1f8c2fc9890ae2f42094120f4b44db9360be5d99a4c6b0e03a9e \ + --hash=sha256:899d28f422116b08be5118ef350c292b36fc15ec2daeb9ea987c89281c7bb5c4 \ + --hash=sha256:8bc5f0687d796c05b1e28ab0d38a50e6309906ee09375dd3aff6a9c09dd6e8f4 \ + --hash=sha256:8bea55c4eef25b0b19a0337dc4e3f9a15b00d569c77211fa8cde38684f234fb7 \ + --hash=sha256:8e5a94886bedca0f9b78fecd6afb6629142fd2605aa70a125d49f4edc6037ee6 \ + --hash=sha256:90ca27cd8da8118b18a52d5f547859cc1f8354a00cd1e8e5120df3e30d6279e5 \ + --hash=sha256:92734d4d8d187a354a556626c221cd1a892a4e0802ccb2af432a1d85ec012194 \ + --hash=sha256:947cf925bc916d90adba35a64c82aace04fa39b46b52d4630ece166655905a69 \ + --hash=sha256:95b52c68d64c1878818687a473a10547b3292e82b6f6fe483808fb1468e2f52f \ + --hash=sha256:97d0235baafca5f2b09cf332cc275f021e694e8362c6bb9c96fc9a0eb74fc316 \ + --hash=sha256:9ca4c0b502ab399ef89248a2c84c54954f77a070f28e546a85e91da627d1301e \ + --hash=sha256:9cc4fc6c196d6a8b76629a70ddfcd4635a6898756e2d9cac5565cf0654605d73 \ + --hash=sha256:9cc6e6d9e571d2f863fa77700701dae73ed5f78881efc8b3f9a4398772ff53e8 \ + --hash=sha256:a056d1ad2633548ca18ffa2f85c202cfb48b68615129143915b8dc72a806a923 \ + --hash=sha256:a26611d9987b230566f24a0a125f17fe0de6a6aff9f25c9f564aaa2721a5fb88 \ + --hash=sha256:a4474d924a47185a06411e0064b803c68be044be2d60e50e8bddcc2649957c1f \ + --hash=sha256:a4ea868bc28109052790eb2b52a9ab33f3aa7adc02f96673526ff47419490e21 \ + --hash=sha256:a9e68c9d88823b274cf1e72f28cb5dc89c990edf430b0bfd3e2fb0785bfeabf4 \ + --hash=sha256:aa9cccf4a44b9b62d8ba8b4dd06c649ba683e4bf04eea606d2e94cfc2d6ff4d6 \ + --hash=sha256:ab30e5e3e706e3063bc6de96b118688cb10396b70bb9864a430f67df98c61ecc \ + --hash=sha256:ac2393c73378fea4e52aa56285a3d64be50f1a12395afef9cce47772f60334c2 \ + --hash=sha256:ad8faf8df23f0378c6d527d8b0b15ea4a2e23c89376877c598c4870d1b2c7866 \ + --hash=sha256:b35b200d6a71b9839a46b9b7fff66b6638bb52fc9658aa58796b0326595d3021 \ + --hash=sha256:b3694e3f87f8ac7ce279d4355645b3c878d24d1424581b46282f24b92f5a4ae2 \ + --hash=sha256:b4ff1d35e8c5bd078be89349b6f3a845128e685e751b6ea1169cf2160b344c4d \ + --hash=sha256:bbc8c8650c6e51041ad1be191742b8b421d05bbd3410f43fa2a00c8db87678e8 \ + --hash=sha256:bc72863f4d9aba2e8fd9085e63548a324ba706d2ea2c83b260da08a59b9482de \ + --hash=sha256:bf625105bb9eef28a56a943fec8c8a98aeb80e7d7db99bd3c388137e6eb2d237 \ + --hash=sha256:c2274ca724536f173122f36c98ce188fd24ce3dad886ec2b7af859518ce008a4 \ + --hash=sha256:c45a03a4c69820a399f1dda9e1d8fbf3562eda46e7720458180302021b08f778 \ + --hash=sha256:c8ae56368f8cc97c7e40a7ee18e1cedaf8e780cd8bc5ed5ac8b81f238614facb \ + --hash=sha256:c907cdc8109f6c619e6254212e794d6548373cc40e1ec75e6e3823d9135d29cc \ + --hash=sha256:ca0276464d148c72defa8bb4390cce01b4a0e425f3b50d1435aa6d7a18107602 \ + --hash=sha256:cd5e2801c89992ed8c0a3f0293ae83c159a60d9a5d685005383ef4caca77f2c4 \ + --hash=sha256:d08ec48f0a1c48d75d0356cea971921848fb620fdeba805b28f937e90691209f \ + --hash=sha256:d1a2ee9c1499fc8f86f4521f27a973c914b211ffa87322f4ee33bb35392da2c5 \ + --hash=sha256:d5f5d1e9def3405f60e3ca8232d56f35c98fb7bf581efcc60051ebf53cb8b611 \ + --hash=sha256:d60377dce4511655582e300dc1e5a5f24ba0cb229005a1d5c8d0cb72bb758ab8 \ + --hash=sha256:d73beaac5e90173ac3deb9928a74763a6d230f494e4bfb422c217a0ad8e629bf \ + --hash=sha256:d7de2637729c67d67cf87614b566626057e95c303bc0a55ffe391f5205e7003d \ + --hash=sha256:dad6e0f2e481fffdcf776d10ebee25e0ef89f16d691f1e5dee4b586375fdc64b \ + --hash=sha256:dda86aba335c902b6149a02a55b38e96287157e609200811837678214ba2b1db \ + --hash=sha256:df01808ee470038c3f8dc4f48620df7225c49c2d6639e38f96e6d6ac6e6f7b0e \ + --hash=sha256:e1f6e2f00a6b8edb562826e4632e26d063ac10307e80f7461f7de3ad8ef3f077 \ + --hash=sha256:e25369dc110d58ddf29b949377a93e0716d72a24f62bad72b2b39f155949c1fd \ + --hash=sha256:e3c701e954abf6fc03a49f7c579cc80c2c6cc52525340ca3186c41d3f33482ef \ + --hash=sha256:e5bcc1a1ae744e0bb59641171ae53743760130600da8db48cbb6e4918e186e4e \ + --hash=sha256:e68c14b04827dd76dcbd1aeea9e604e3e4b78322d8faf2f8132c7138efa340a8 \ + --hash=sha256:e8aeb10fcbe92767f0fa69ad5a72deca50d0dca07fbde97848997d778a50c9fe \ + --hash=sha256:e985a16ff513596f217cee86c21371b8cd011c0f6f056d0920aa2d926c544058 \ + --hash=sha256:ecbbd45615a6885fe3240eb9db73b9e62518b611850fdf8ab08bd56de7ad2b17 \ + --hash=sha256:ee4ec14bc1680d6b0afab9aea2ef27e26d2024f18b24a2d7155a52b60da7e833 \ + --hash=sha256:ef5960d965e67165d75b7c7ffc60a83ec5abfc5c11b764ec13ea54fbef8b4421 \ + --hash=sha256:f0cdaecd4c953bfae0b6bb64910aaaca5a424ad9c72d85cb88417bb9814f7550 \ + --hash=sha256:f1ce721c8a7dfec21fcbdfe04e8f68174183cf4e8188e0645e92aa23985c57ff \ + --hash=sha256:f50498891691e0864dc3da965f340fada0771f6142a378083dc4608f4ea513e2 \ + --hash=sha256:f5ea69428fa1b49573eef0cc44a1d43bebd45ad0c611eb7d7eac760c7ae771bc \ + --hash=sha256:f61aa92e4aad0be58eb6eb4e0c21acf32cf8065f4b2cae5665da756c4ceef982 \ + --hash=sha256:f6e4333fb15c83f7d1482a76d45a0818897b3d33f00efd215528ff7c51b8e35d \ + --hash=sha256:f820f24b09e3e779fe84c3c456cb4108a7aa639b0d1f02c28046e11bfcd088ed \ + --hash=sha256:f98059e4fcd3e3e4e2d632b7cf81c2faae96c43c60b569e9c621468082f1d104 \ + --hash=sha256:fcce033e4021347d80ed9c66dcf1e7b1546319834b74445f561d2e2221de5659 # via requests ec2-metadata==2.17.0 \ --hash=sha256:a696a899cb8ee1f0101eb9ec7bc2050e1ffc8574ee4cd748a52d9c748fb19d9e \ @@ -184,15 +200,15 @@ islpy==2023.2.5 \ --hash=sha256:ed0c62e64d160735ed1a7fc314d4c0f501711c282bb84a00ceaf590b4c4bd61f \ --hash=sha256:eeec97588bd5282c983bd73f9010156e5e251e9e5307af547e2cb53b02d4639a # via neuronx-cc -jmespath==1.0.1 \ - --hash=sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 \ - --hash=sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe +jmespath==1.1.0 \ + --hash=sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d \ + --hash=sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64 # via # boto3 # botocore -libneuronxla==2.2.14584.0+06ac23d1 \ - --hash=sha256:d1e594b27716bc59b937ccd8f40e7f2b74f6c309643e83dcf511b7ea392924f2 - # via -r runtimes/neuron/requirements.in +libneuronxla==2.2.15515.0+50c26cbd \ + --hash=sha256:770d6ed89be152c2dc777b042ccf66c7af7c71e3204521f0b84d65705a6958e2 + # via -r platforms/neuron/requirements.in lockfile==0.12.2 \ --hash=sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799 \ --hash=sha256:6c3cb24f344923d30b2785d5ad75182c8ea7ac1b6171b08657258ec7429d50fa @@ -237,131 +253,135 @@ ml-dtypes==0.5.4 \ --hash=sha256:ce756d3a10d0c4067172804c9cc276ba9cc0ff47af9078ad439b075d1abdc29b \ --hash=sha256:d81fdb088defa30eb37bf390bb7dde35d3a83ec112ac8e33d75ab28cc29dd8b0 \ --hash=sha256:f21c9219ef48ca5ee78402d5cc831bd58ea27ce89beda894428bc67a52da5328 - # via neuronx-cc + # via + # neuronx-cc + # nki networkx==2.8.8 \ --hash=sha256:230d388117af870fce5647a3c52401fcf753e94720e6ea6b4197a5355648885e \ --hash=sha256:e435dfa75b1d7195c7b8378c3859f0445cd88c6b0375c181ed66823a9ceb7524 # via neuronx-cc -neuronx-cc==2.22.12471.0+b4a00d10 \ - --hash=sha256:19fcf633f6f453b73e791b6f23d0e3f741fd967a61b7b82208bd74d80908c81a +neuronx-cc==2.23.6484.0+3b612583 \ + --hash=sha256:91089eb2b745ac6022dde04e71f4e08f2ccd5b61c860403ab18df8d5058a5c97 # via - # -r runtimes/neuron/requirements.in + # -r platforms/neuron/requirements.in # libneuronxla -numpy==2.4.0 \ - --hash=sha256:009bd0ea12d3c784b6639a8457537016ce5172109e585338e11334f6a7bb88ee \ - --hash=sha256:02a2038eb27f9443a8b266a66911e926566b5a6ffd1a689b588f7f35b81e7dc3 \ - --hash=sha256:0804a8e4ab070d1d35496e65ffd3cf8114c136a2b81f61dfab0de4b218aacfd5 \ - --hash=sha256:0feafc9e03128074689183031181fac0897ff169692d8492066e949041096548 \ - --hash=sha256:1889b3a3f47a7b5bee16bc25a2145bd7cb91897f815ce3499db64c7458b6d91d \ - --hash=sha256:1d8f9fde5f6dc1b6fc34df8162f3b3079365468703fee7f31d4e0cc8c63baed9 \ - --hash=sha256:1ee32359fb7543b7b7bd0b2f46294db27e29e7bbdf70541e81b190836cd83ded \ - --hash=sha256:20c115517513831860c573996e395707aa9fb691eb179200125c250e895fcd93 \ - --hash=sha256:21555da4ec4a0c942520ead42c3b0dc9477441e085c42b0fbdd6a084869a6f6b \ - --hash=sha256:23a3e9d1a6f360267e8fbb38ba5db355a6a7e9be71d7fce7ab3125e88bb646c8 \ - --hash=sha256:2a8b6bb8369abefb8bd1801b054ad50e02b3275c8614dc6e5b0373c305291037 \ - --hash=sha256:2e284ca13d5a8367e43734148622caf0b261b275673823593e3e3634a6490f83 \ - --hash=sha256:2f585f52b2baf07ff3356158d9268ea095e221371f1074fadea2f42544d58b4d \ - --hash=sha256:316b2f2584682318539f0bcaca5a496ce9ca78c88066579ebd11fd06f8e4741e \ - --hash=sha256:32ed06d0fe9cae27d8fb5f400c63ccee72370599c75e683a6358dd3a4fb50aaf \ - --hash=sha256:355354388cba60f2132df297e2d53053d4063f79077b67b481d21276d61fc4df \ - --hash=sha256:35ddc8f4914466e6fc954c76527aa91aa763682a4f6d73249ef20b418fe6effb \ - --hash=sha256:36667db4d6c1cea79c8930ab72fadfb4060feb4bfe724141cd4bd064d2e5f8ce \ - --hash=sha256:36cbfb13c152b1c7c184ddac43765db8ad672567e7bafff2cc755a09917ed2e6 \ - --hash=sha256:39699233bc72dd482da1415dcb06076e32f60eddc796a796c5fb6c5efce94667 \ - --hash=sha256:39b19251dec4de8ff8496cd0806cbe27bf0684f765abb1f4809554de93785f2d \ - --hash=sha256:3d857f55e7fdf7c38ab96c4558c95b97d1c685be6b05c249f5fdafcbd6f9899e \ - --hash=sha256:3de19cfecd1465d0dcf8a5b5ea8b3155b42ed0b639dba4b71e323d74f2a3be5e \ - --hash=sha256:3f3096405acc48887458bbf9f6814d43785ac7ba2a57ea6442b581dedbc60ce6 \ - --hash=sha256:40483b2f2d3ba7aad426443767ff5632ec3156ef09742b96913787d13c336471 \ - --hash=sha256:413aa561266a4be2d06cd2b9665e89d9f54c543f418773076a76adcf2af08bc7 \ - --hash=sha256:44aba2f0cafd287871a495fb3163408b0bd25bbce135c6f621534a07f4f7875c \ - --hash=sha256:49ff32b09f5aa0cd30a20c2b39db3e669c845589f2b7fc910365210887e39344 \ - --hash=sha256:4d1cfce39e511069b11e67cd0bd78ceff31443b7c9e5c04db73c7a19f572967c \ - --hash=sha256:57c540ed8fb1f05cb997c6761cd56db72395b0d6985e90571ff660452ade4f98 \ - --hash=sha256:5f48cb3e88fbc294dc90e215d86fbaf1c852c63dbdb6c3a3e63f45c4b57f7344 \ - --hash=sha256:5fe44e277225fd3dff6882d86d3d447205d43532c3627313d17e754fb3905a0e \ - --hash=sha256:60e8c196cd82cbbd4f130b5290007e13e6de3eca79f0d4d38014769d96a7c475 \ - --hash=sha256:65167da969cd1ec3a1df31cb221ca3a19a8aaa25370ecb17d428415e93c1935e \ - --hash=sha256:680060061adb2d74ce352628cb798cfdec399068aa7f07ba9fb818b2b3305f98 \ - --hash=sha256:6bbaebf0d11567fa8926215ae731e1d58e6ec28a8a25235b8a47405d301332db \ - --hash=sha256:6c05483c3136ac4c91b4e81903cb53a8707d316f488124d0398499a4f8e8ef51 \ - --hash=sha256:6dc8b7e2f4eb184b37655195f421836cfae6f58197b67e3ffc501f1333d993fa \ - --hash=sha256:6e504f7b16118198f138ef31ba24d985b124c2c469fe8467007cf30fd992f934 \ - --hash=sha256:74623f2ab5cc3f7c886add4f735d1031a1d2be4a4ae63c0546cfd74e7a31ddf6 \ - --hash=sha256:77f0d13fa87036d7553bf81f0e1fe3ce68d14c9976c9851744e4d3e91127e95f \ - --hash=sha256:85eef4cb5625c47ee6425c58a3502555e10f45ee973da878ac8248ad58c136f3 \ - --hash=sha256:882567b7ae57c1b1a0250208cc21a7976d8cbcc49d5a322e607e6f09c9e0bd53 \ - --hash=sha256:8b986403023c8f3bf8f487c2e6186afda156174d31c175f747d8934dfddf3479 \ - --hash=sha256:8cfa5f29a695cb7438965e6c3e8d06e0416060cf0d709c1b1c1653a939bf5c2a \ - --hash=sha256:8eafe7c36c8430b7794edeab3087dec7bf31d634d92f2af9949434b9d1964cba \ - --hash=sha256:9198f447e1dc5647d07c9a6bbe2063cc0132728cc7175b39dbc796da5b54920d \ - --hash=sha256:92190db305a6f48734d3982f2c60fa30d6b5ee9bff10f2887b930d7b40119f4c \ - --hash=sha256:927ccf5cd17c48f801f4ed43a7e5673a2724bd2171460be3e3894e6e332ef83a \ - --hash=sha256:98e81648e0b36e325ab67e46b5400a7a6d4a22b8a7c8e8bbfe20e7db7906bf95 \ - --hash=sha256:9a818668b674047fd88c4cddada7ab8f1c298812783e8328e956b78dc4807f9f \ - --hash=sha256:a152d86a3ae00ba5f47b3acf3b827509fd0b6cb7d3259665e63dafbad22a75ea \ - --hash=sha256:a2718c1de8504121714234b6f8241d0019450353276c88b9453c9c3d92e101db \ - --hash=sha256:a39fb973a726e63223287adc6dafe444ce75af952d711e400f3bf2b36ef55a7b \ - --hash=sha256:a899699294f28f7be8992853c0c60741f16ff199205e2e6cdca155762cbaa59d \ - --hash=sha256:a8fdfed3deaf1928fb7667d96e0567cdf58c2b370ea2ee7e586aa383ec2cb346 \ - --hash=sha256:aabb081ca0ec5d39591fc33018cd4b3f96e1a2dd6756282029986d00a785fba4 \ - --hash=sha256:b1f5b45829ac1848893f0ddf5cb326110604d6df96cdc255b0bf9edd154104d4 \ - --hash=sha256:b48e35f4ab6f6a7597c46e301126ceba4c44cd3280e3750f85db48b082624fa4 \ - --hash=sha256:b54c83f1c0c0f1d748dca0af516062b8829d53d1f0c402be24b4257a9c48ada6 \ - --hash=sha256:ba0cb30acd3ef11c94dc27fbfba68940652492bc107075e7ffe23057f9425681 \ - --hash=sha256:bb50ce5fb202a26fd5404620e7ef820ad1ab3558b444cb0b55beb7ef66cd2d63 \ - --hash=sha256:c95eb6db2884917d86cde0b4d4cf31adf485c8ec36bf8696dd66fa70de96f36b \ - --hash=sha256:d0ffd9e2e4441c96a9c91ec1783285d80bf835b677853fc2770a89d50c1e48ac \ - --hash=sha256:d57b5046c120561ba8fa8e4030fbb8b822f3063910fa901ffadf16e2b7128ad6 \ - --hash=sha256:d9e6a7664ddd9746e20b7325351fe1a8408d0a2bf9c63b5e898290ddc8f09544 \ - --hash=sha256:dc578891de1db95b2a35001b695451767b580bb45753717498213c5ff3c41d63 \ - --hash=sha256:e0434aa22c821f44eeb4c650b81c7fbdd8c0122c6c4b5a576a76d5a35625ecd9 \ - --hash=sha256:e06a922a469cae9a57100864caf4f8a97a1026513793969f8ba5b63137a35d25 \ - --hash=sha256:e493962256a38f58283de033d8af176c5c91c084ea30f15834f7545451c42059 \ - --hash=sha256:ecb0019d44f4cdb50b676c5d0cb4b1eae8e15d1ed3d3e6639f986fc92b2ec52c \ - --hash=sha256:f935c4493eda9069851058fa0d9e39dbf6286be690066509305e52912714dbb2 +nki==0.2.0+g82fdb402 \ + --hash=sha256:42e2ffa5dfd9b5d22d4cca0f9bfb69a8ba8f2cac7f6eff801780798c05d07fb5 + # via neuronx-cc +numpy==2.4.3 \ + --hash=sha256:0200b25c687033316fb39f0ff4e3e690e8957a2c3c8d22499891ec58c37a3eb5 \ + --hash=sha256:0448e7f9caefb34b4b7dd2b77f21e8906e5d6f0365ad525f9f4f530b13df2afc \ + --hash=sha256:0a195f4216be9305a73c0e91c9b026a35f2161237cf1c6de9b681637772ea657 \ + --hash=sha256:0a60e17a14d640f49146cb38e3f105f571318db7826d9b6fef7e4dce758faecd \ + --hash=sha256:120df8c0a81ebbf5b9020c91439fccd85f5e018a927a39f624845be194a2be02 \ + --hash=sha256:148d59127ac95979d6f07e4d460f934ebdd6eed641db9c0db6c73026f2b2101a \ + --hash=sha256:1ec84fd7c8e652b0f4aaaf2e6e9cc8eaa9b1b80a537e06b2e3a2fb176eedcb26 \ + --hash=sha256:22654fe6be0e5206f553a9250762c653d3698e46686eee53b399ab90da59bd92 \ + --hash=sha256:22c31dc07025123aedf7f2db9e91783df13f1776dc52c6b22c620870dc0fab22 \ + --hash=sha256:23b46bb6d8ecb68b58c09944483c135ae5f0e9b8d8858ece5e4ead783771d2a9 \ + --hash=sha256:2629289168f4897a3c4e23dc98d6f1731f0fc0fe52fb9db19f974041e4cc12b9 \ + --hash=sha256:26952e18d82a1dbbc2f008d402021baa8d6fc8e84347a2072a25e08b46d698b9 \ + --hash=sha256:29363fbfa6f8ee855d7569c96ce524845e3d726d6c19b29eceec7dd555dab152 \ + --hash=sha256:297837823f5bc572c5f9379b0c9f3a3365f08492cbdc33bcc3af174372ebb168 \ + --hash=sha256:2abad5c7fef172b3377502bde47892439bae394a71bc329f31df0fd829b41a9e \ + --hash=sha256:2b3f8d2c4589b1a2028d2a770b0fc4d1f332fb5e01521f4de3199a896d158ddd \ + --hash=sha256:2ddb7919366ee468342b91dea2352824c25b55814a987847b6c52003a7c97f15 \ + --hash=sha256:2e03c05abaee1f672e9d67bc858f300b5ccba1c21397211e8d77d98350972093 \ + --hash=sha256:32e3bef222ad6b052280311d1d60db8e259e4947052c3ae7dd6817451fc8a4c5 \ + --hash=sha256:33b3bf58ee84b172c067f56aeadc7ee9ab6de69c5e800ab5b10295d54c581adb \ + --hash=sha256:45f003dbdffb997a03da2d1d0cb41fbd24a87507fb41605c0420a3db5bd4667b \ + --hash=sha256:483a201202b73495f00dbc83796c6ae63137a9bdade074f7648b3e32613412dd \ + --hash=sha256:48da3a4ee1336454b07497ff7ec83903efa5505792c4e6d9bf83d99dc07a1e18 \ + --hash=sha256:4b42639cdde6d24e732ff823a3fa5b701d8acad89c4142bc1d0bd6dc85200ba5 \ + --hash=sha256:4bd4741a6a676770e0e97fe9ab2e51de01183df3dcbcec591d26d331a40de950 \ + --hash=sha256:4d382735cecd7bcf090172489a525cd7d4087bc331f7df9f60ddc9a296cf208e \ + --hash=sha256:52077feedeff7c76ed7c9f1a0428558e50825347b7545bbb8523da2cd55c547a \ + --hash=sha256:54f29b877279d51e210e0c80709ee14ccbbad647810e8f3d375561c45ef613dd \ + --hash=sha256:5884ce5c7acfae1e4e1b6fde43797d10aa506074d25b531b4f54bde33c0c31d4 \ + --hash=sha256:5e10da9e93247e554bb1d22f8edc51847ddd7dde52d85ce31024c1b4312bfba0 \ + --hash=sha256:61b0cbabbb6126c8df63b9a3a0c4b1f44ebca5e12ff6997b80fcf267fb3150ef \ + --hash=sha256:65f3c2455188f09678355f5cae1f959a06b778bc66d535da07bf2ef20cd319d5 \ + --hash=sha256:679f2a834bae9020f81534671c56fd0cc76dd7e5182f57131478e23d0dc59e24 \ + --hash=sha256:6bd06731541f89cdc01b261ba2c9e037f1543df7472517836b78dfb15bd6e476 \ + --hash=sha256:715de7f82e192e8cae5a507a347d97ad17598f8e026152ca97233e3666daaa71 \ + --hash=sha256:737f630a337364665aba3b5a77e56a68cc42d350edd010c345d65a3efa3addcc \ + --hash=sha256:7395e69ff32526710748f92cd8c9849b361830968ea3e24a676f272653e8983e \ + --hash=sha256:76dbb9d4e43c16cf9aa711fcd8de1e2eeb27539dcefb60a1d5e9f12fae1d1ed8 \ + --hash=sha256:76f0f283506c28b12bba319c0fab98217e9f9b54e6160e9c79e9f7348ba32e9c \ + --hash=sha256:77e76d932c49a75617c6d13464e41203cd410956614d0a0e999b25e9e8d27eec \ + --hash=sha256:7aa4e54f6469300ebca1d9eb80acd5253cdfa36f2c03d79a35883687da430875 \ + --hash=sha256:7d1ce23cce91fcea443320a9d0ece9b9305d4368875bab09538f7a5b4131938a \ + --hash=sha256:7e58765ad74dcebd3ef0208a5078fba32dc8ec3578fe84a604432950cd043d79 \ + --hash=sha256:7f3408ff897f8ab07a07fbe2823d7aee6ff644c097cc1f90382511fe982f647f \ + --hash=sha256:8ba7b51e71c05aa1f9bc3641463cd82308eab40ce0d5c7e1fd4038cbf9938147 \ + --hash=sha256:8e236dbda4e1d319d681afcbb136c0c4a8e0f1a5c58ceec2adebb547357fe857 \ + --hash=sha256:94f3c4a151a2e529adf49c1d54f0f57ff8f9b233ee4d44af623a81553ab86368 \ + --hash=sha256:9684823a78a6cd6ad7511fc5e25b07947d1d5b5e2812c93fe99d7d4195130720 \ + --hash=sha256:a016db5c5dba78fa8fe9f5d80d6708f9c42ab087a739803c0ac83a43d686a470 \ + --hash=sha256:a111698b4a3f8dcbe54c64a7708f049355abd603e619013c346553c1fd4ca90b \ + --hash=sha256:a1988292870c7cb9d0ebb4cc96b4d447513a9644801de54606dc7aabf2b7d920 \ + --hash=sha256:a315e5234d88067f2d97e1f2ef670a7569df445d55400f1e33d117418d008d52 \ + --hash=sha256:a749547700de0a20a6718293396ec237bb38218049cfce788e08fcb716e8cf73 \ + --hash=sha256:a97cbf7e905c435865c2d939af3d93f99d18eaaa3cabe4256f4304fb51604349 \ + --hash=sha256:abdce0f71dcb4a00e4e77f3faf05e4616ceccfe72ccaa07f47ee79cda3b7b0f4 \ + --hash=sha256:b346845443716c8e542d54112966383b448f4a3ba5c66409771b8c0889485dd3 \ + --hash=sha256:b44fd60341c4d9783039598efadd03617fa28d041fc37d22b62d08f2027fa0e7 \ + --hash=sha256:bb2e3cf95854233799013779216c57e153c1ee67a0bf92138acca0e429aefaee \ + --hash=sha256:bc71942c789ef415a37f0d4eab90341425a00d538cd0642445d30b41023d3395 \ + --hash=sha256:be3b8487d725a77acccc9924f65fd8bce9af7fac8c9820df1049424a2115af6c \ + --hash=sha256:c59020932feb24ed49ffd03704fbab89f22aa9c0d4b180ff45542fe8918f5611 \ + --hash=sha256:c6b124bfcafb9e8d3ed09130dbee44848c20b3e758b6bbf006e641778927c028 \ + --hash=sha256:c9619741e9da2059cd9c3f206110b97583c7152c1dc9f8aafd4beb450ac1c89d \ + --hash=sha256:cd32fbacb9fd1bf041bf8e89e4576b6f00b895f06d00914820ae06a616bdfef7 \ + --hash=sha256:d1b90d840b25874cf5cd20c219af10bac3667db3876d9a495609273ebe679070 \ + --hash=sha256:d213c7e6e8d211888cc359bab7199670a00f5b82c0978b9d1c75baf1eddbeac0 \ + --hash=sha256:d5f51900414fc9204a0e0da158ba2ac52b75656e7dce7e77fb9f84bfa343b4cc \ + --hash=sha256:d71e379452a2f670ccb689ec801b1218cd3983e253105d6e83780967e899d687 \ + --hash=sha256:d84f0f881cb2225c2dfd7f78a10a5645d487a496c6668d6cc39f0f114164f3d0 \ + --hash=sha256:decb0eb8a53c3b009b0962378065589685d66b23467ef5dac16cbe818afde27f \ + --hash=sha256:e7dd01a46700b1967487141a66ac1a3cf0dd8ebf1f08db37d46389401512ca97 \ + --hash=sha256:eb610595dd91560905c132c709412b512135a60f1851ccbd2c959e136431ff67 # via # ml-dtypes # neuronx-cc + # nki # scipy pgzip==0.4.0 \ --hash=sha256:41e44c03a9f1bbbd390dd6b6e37a11484113f8c159a049fa81cd8a9ddde82a21 \ --hash=sha256:91f6526f0c6e1f6c2d3522707777e5843fad7c791401745593fd01bd869b7a05 # via neuronx-cc -protobuf==6.33.2 \ - --hash=sha256:1f8017c48c07ec5859106533b682260ba3d7c5567b1ca1f24297ce03384d1b4f \ - --hash=sha256:2981c58f582f44b6b13173e12bb8656711189c2a70250845f264b877f00b1913 \ - --hash=sha256:56dc370c91fbb8ac85bc13582c9e373569668a290aa2e66a590c2a0d35ddb9e4 \ - --hash=sha256:7109dcc38a680d033ffb8bf896727423528db9163be1b6a02d6a49606dcadbfe \ - --hash=sha256:7636aad9bb01768870266de5dc009de2d1b936771b38a793f73cbbf279c91c5c \ - --hash=sha256:87eb388bd2d0f78febd8f4c8779c79247b26a5befad525008e49a6955787ff3d \ - --hash=sha256:8cd7640aee0b7828b6d03ae518b5b4806fdfc1afe8de82f79c3454f8aef29872 \ - --hash=sha256:b5d3b5625192214066d99b2b605f5783483575656784de223f00a8d00754fc0e \ - --hash=sha256:d9b19771ca75935b3a4422957bc518b0cecb978b31d1dd12037b088f6bcc0e43 \ - --hash=sha256:fc2a0e8b05b180e5fc0dd1559fe8ebdae21a27e81ac77728fb6c42b12c7419b4 +protobuf==7.34.0 \ + --hash=sha256:3871a3df67c710aaf7bb8d214cc997342e63ceebd940c8c7fc65c9b3d697591a \ + --hash=sha256:4a72a8ec94e7a9f7ef7fe818ed26d073305f347f8b3b5ba31e22f81fd85fca02 \ + --hash=sha256:8e329966799f2c271d5e05e236459fe1cbfdb8755aaa3b0914fa60947ddea408 \ + --hash=sha256:964cf977e07f479c0697964e83deda72bcbc75c3badab506fb061b352d991b01 \ + --hash=sha256:9d7a5005fb96f3c1e64f397f91500b0eb371b28da81296ae73a6b08a5b76cdd6 \ + --hash=sha256:9f9079f1dde4e32342ecbd1c118d76367090d4aaa19da78230c38101c5b3dd40 \ + --hash=sha256:e3b914dd77fa33fa06ab2baa97937746ab25695f389869afdf03e81f34e45dc7 \ + --hash=sha256:f791ec509707a1d91bd02e07df157e75e4fb9fbdad12a81b7396201ec244e2e3 # via neuronx-cc -psutil==7.2.1 \ - --hash=sha256:05cc68dbb8c174828624062e73078e7e35406f4ca2d0866c272c2410d8ef06d1 \ - --hash=sha256:08a2f175e48a898c8eb8eace45ce01777f4785bc744c90aa2cc7f2fa5462a266 \ - --hash=sha256:0d67c1822c355aa6f7314d92018fb4268a76668a536f133599b91edd48759442 \ - --hash=sha256:2ceae842a78d1603753561132d5ad1b2f8a7979cb0c283f5b52fb4e6e14b1a79 \ - --hash=sha256:35630d5af80d5d0d49cfc4d64c1c13838baf6717a13effb35869a5919b854cdf \ - --hash=sha256:3fce5f92c22b00cdefd1645aa58ab4877a01679e901555067b1bd77039aa589f \ - --hash=sha256:494c513ccc53225ae23eec7fe6e1482f1b8a44674241b54561f755a898650679 \ - --hash=sha256:5e38404ca2bb30ed7267a46c02f06ff842e92da3bb8c5bfdadbd35a5722314d8 \ - --hash=sha256:81442dac7abfc2f4f4385ea9e12ddf5a796721c0f6133260687fec5c3780fa49 \ - --hash=sha256:923f8653416604e356073e6e0bccbe7c09990acef442def2f5640dd0faa9689f \ - --hash=sha256:93f3f7b0bb07711b49626e7940d6fe52aa9940ad86e8f7e74842e73189712129 \ - --hash=sha256:99a4cd17a5fdd1f3d014396502daa70b5ec21bf4ffe38393e152f8e449757d67 \ - --hash=sha256:ab2b98c9fc19f13f59628d94df5cc4cc4844bc572467d113a8b517d634e362c6 \ - --hash=sha256:b1b0671619343aa71c20ff9767eced0483e4fc9e1f489d50923738caf6a03c17 \ - --hash=sha256:b2e953fcfaedcfbc952b44744f22d16575d3aa78eb4f51ae74165b4e96e55f42 \ - --hash=sha256:ba9f33bb525b14c3ea563b2fd521a84d2fa214ec59e3e6a2858f78d0844dd60d \ - --hash=sha256:cfbe6b40ca48019a51827f20d830887b3107a74a79b01ceb8cc8de4ccb17b672 \ - --hash=sha256:d34d2ca888208eea2b5c68186841336a7f5e0b990edec929be909353a202768a \ - --hash=sha256:ea46c0d060491051d39f0d2cff4f98d5c72b288289f57a21556cc7d504db37fc \ - --hash=sha256:f7583aec590485b43ca601dd9cea0dcd65bd7bb21d30ef4ddbf4ea6b5ed1bdd3 \ - --hash=sha256:f78baafb38436d5a128f837fab2d92c276dfb48af01a240b861ae02b2413ada8 +psutil==7.2.2 \ + --hash=sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372 \ + --hash=sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9 \ + --hash=sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841 \ + --hash=sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63 \ + --hash=sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979 \ + --hash=sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a \ + --hash=sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b \ + --hash=sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9 \ + --hash=sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee \ + --hash=sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312 \ + --hash=sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b \ + --hash=sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9 \ + --hash=sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e \ + --hash=sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc \ + --hash=sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1 \ + --hash=sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf \ + --hash=sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea \ + --hash=sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988 \ + --hash=sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486 \ + --hash=sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00 \ + --hash=sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8 # via neuronx-cc python-daemon==3.1.2 \ --hash=sha256:b906833cef63502994ad48e2eab213259ed9bb18d54fa8774dcba2ff7864cec6 \ @@ -385,76 +405,78 @@ s3transfer==0.16.0 \ --hash=sha256:18e25d66fed509e3868dc1572b3f427ff947dd2c56f844a5bf09481ad3f3b2fe \ --hash=sha256:8e990f13268025792229cd52fa10cb7163744bf56e719e0b9cb925ab79abf920 # via boto3 -scipy==1.16.3 \ - --hash=sha256:0151a0749efeaaab78711c78422d413c583b8cdd2011a3c1d6c794938ee9fdb2 \ - --hash=sha256:01e87659402762f43bd2fee13370553a17ada367d42e7487800bf2916535aecb \ - --hash=sha256:03192a35e661470197556de24e7cb1330d84b35b94ead65c46ad6f16f6b28f2a \ - --hash=sha256:0553371015692a898e1aa858fed67a3576c34edefa6b7ebdb4e9dde49ce5c203 \ - --hash=sha256:062246acacbe9f8210de8e751b16fc37458213f124bef161a5a02c7a39284304 \ - --hash=sha256:0c3b4dd3d9b08dbce0f3440032c52e9e2ab9f96ade2d3943313dfe51a7056959 \ - --hash=sha256:0c623a54f7b79dd88ef56da19bc2873afec9673a48f3b85b18e4d402bdd29a5a \ - --hash=sha256:16b8bc35a4cc24db80a0ec836a9286d0e31b2503cb2fd7ff7fb0e0374a97081d \ - --hash=sha256:1fb2472e72e24d1530debe6ae078db70fb1605350c88a3d14bc401d6306dbffe \ - --hash=sha256:21d9d6b197227a12dcbf9633320a4e34c6b0e51c57268df255a0942983bac562 \ - --hash=sha256:2a207a6ce9c24f1951241f4693ede2d393f59c07abc159b2cb2be980820e01fb \ - --hash=sha256:2b71d93c8a9936046866acebc915e2af2e292b883ed6e2cbe5c34beb094b82d9 \ - --hash=sha256:2d1ae2cf0c350e7705168ff2429962a89ad90c2d49d1dd300686d8b2a5af22fc \ - --hash=sha256:3a4c460301fb2cffb7f88528f30b3127742cff583603aa7dc964a52c463b385d \ - --hash=sha256:3d4a07a8e785d80289dfe66b7c27d8634a773020742ec7187b85ccc4b0e7b686 \ - --hash=sha256:40be6cf99e68b6c4321e9f8782e7d5ff8265af28ef2cd56e9c9b2638fa08ad97 \ - --hash=sha256:4aff59800a3b7f786b70bfd6ab551001cb553244988d7d6b8299cb1ea653b353 \ - --hash=sha256:50a3dbf286dbc7d84f176f9a1574c705f277cb6565069f88f60db9eafdbe3ee2 \ - --hash=sha256:532fb5ad6a87e9e9cd9c959b106b73145a03f04c7d57ea3e6f6bb60b86ab0876 \ - --hash=sha256:53c3844d527213631e886621df5695d35e4f6a75f620dca412bcd292f6b87d78 \ - --hash=sha256:56edc65510d1331dae01ef9b658d428e33ed48b4f77b1d51caf479a0253f96dc \ - --hash=sha256:57d01cb6f85e34f0946b33caa66e892aae072b64b034183f3d87c4025802a119 \ - --hash=sha256:5803c5fadd29de0cf27fa08ccbfe7a9e5d741bf63e4ab1085437266f12460ff9 \ - --hash=sha256:6020470b9d00245926f2d5bb93b119ca0340f0d564eb6fbaad843eaebf9d690f \ - --hash=sha256:63d3cdacb8a824a295191a723ee5e4ea7768ca5ca5f2838532d9f2e2b3ce2135 \ - --hash=sha256:663b8d66a8748051c3ee9c96465fb417509315b99c71550fda2591d7dd634234 \ - --hash=sha256:72d1717fd3b5e6ec747327ce9bda32d5463f472c9dce9f54499e81fbd50245a1 \ - --hash=sha256:7dc1360c06535ea6116a2220f760ae572db9f661aba2d88074fe30ec2aa1ff88 \ - --hash=sha256:7f68154688c515cdb541a31ef8eb66d8cd1050605be9dcd74199cbd22ac739bc \ - --hash=sha256:81fc5827606858cf71446a5e98715ba0e11f0dbc83d71c7409d05486592a45d6 \ - --hash=sha256:875555ce62743e1d54f06cdf22c1e0bc47b91130ac40fe5d783b6dfa114beeb6 \ - --hash=sha256:8b3c820ddb80029fe9f43d61b81d8b488d3ef8ca010d15122b152db77dc94c22 \ - --hash=sha256:8be1ca9170fcb6223cc7c27f4305d680ded114a1567c0bd2bfcbf947d1b17511 \ - --hash=sha256:8d09d72dc92742988b0e7750bddb8060b0c7079606c0d24a8cc8e9c9c11f9079 \ - --hash=sha256:9452781bd879b14b6f055b26643703551320aa8d79ae064a71df55c00286a184 \ - --hash=sha256:96491a6a54e995f00a28a3c3badfff58fd093bf26cd5fb34a2188c8c756a3a2c \ - --hash=sha256:9b9c9c07b6d56a35777a1b4cc8966118fb16cfd8daf6743867d17d36cfad2d40 \ - --hash=sha256:a8a26c78ef223d3e30920ef759e25625a0ecdd0d60e5a8818b7513c3e5384cf2 \ - --hash=sha256:aadd23f98f9cb069b3bd64ddc900c4d277778242e961751f77a8cb5c4b946fb0 \ - --hash=sha256:b7180967113560cca57418a7bc719e30366b47959dd845a93206fbed693c867e \ - --hash=sha256:b7c5f1bda1354d6a19bc6af73a649f8285ca63ac6b52e64e658a5a11d4d69800 \ - --hash=sha256:b81c27fc41954319a943d43b20e07c40bdcd3ff7cf013f4fb86286faefe546c4 \ - --hash=sha256:bb61878c18a470021fb515a843dc7a76961a8daceaaaa8bad1332f1bf4b54657 \ - --hash=sha256:bea0a62734d20d67608660f69dcda23e7f90fb4ca20974ab80b6ed40df87a005 \ - --hash=sha256:c5192722cffe15f9329a3948c4b1db789fbb1f05c97899187dcf009b283aea70 \ - --hash=sha256:c97176013d404c7346bf57874eaac5187d969293bf40497140b0a2b2b7482e07 \ - --hash=sha256:cd13e354df9938598af2be05822c323e97132d5e6306b83a3b4ee6724c6e522e \ - --hash=sha256:d2ec56337675e61b312179a1ad124f5f570c00f920cc75e1000025451b88241c \ - --hash=sha256:d3837938ae715fc0fe3c39c0202de3a8853aff22ca66781ddc2ade7554b7e2cc \ - --hash=sha256:d9f48cafc7ce94cf9b15c6bffdc443a81a27bf7075cf2dcd5c8b40f85d10c4e7 \ - --hash=sha256:da7763f55885045036fabcebd80144b757d3db06ab0861415d1c3b7c69042146 \ - --hash=sha256:deb3841c925eeddb6afc1e4e4a45e418d19ec7b87c5df177695224078e8ec733 \ - --hash=sha256:e1d27cbcb4602680a49d787d90664fa4974063ac9d4134813332a8c53dbe667c \ - --hash=sha256:e5d42a9472e7579e473879a1990327830493a7047506d58d73fc429b84c1d49d \ - --hash=sha256:e7efa2681ea410b10dde31a52b18b0154d66f2485328830e45fdf183af5aefc6 \ - --hash=sha256:eab43fae33a0c39006a88096cd7b4f4ef545ea0447d250d5ac18202d40b6611d \ - --hash=sha256:f2622206f5559784fa5c4b53a950c3c7c1cf3e84ca1b9c4b6c03f062f289ca26 \ - --hash=sha256:f379b54b77a597aa7ee5e697df0d66903e41b9c85a6dd7946159e356319158e8 \ - --hash=sha256:f667a4542cc8917af1db06366d3f78a5c8e83badd56409f94d1eac8d8d9133fa \ - --hash=sha256:fb4b29f4cf8cc5a8d628bc8d8e26d12d7278cd1f219f22698a378c3d67db5e4b \ - --hash=sha256:ffa6eea95283b2b8079b821dc11f50a17d0571c92b43e2b5b12764dc5f9b285d - # via neuronx-cc +scipy==1.17.1 \ + --hash=sha256:010f4333c96c9bb1a4516269e33cb5917b08ef2166d5556ca2fd9f082a9e6ea0 \ + --hash=sha256:02ae3b274fde71c5e92ac4d54bc06c42d80e399fec704383dcd99b301df37458 \ + --hash=sha256:08b900519463543aa604a06bec02461558a6e1cef8fdbb8098f77a48a83c8118 \ + --hash=sha256:131f5aaea57602008f9822e2115029b55d4b5f7c070287699fe45c661d051e39 \ + --hash=sha256:158dd96d2207e21c966063e1635b1063cd7787b627b6f07305315dd73d9c679e \ + --hash=sha256:1cc682cea2ae55524432f3cdff9e9a3be743d52a7443d0cba9017c23c87ae2f6 \ + --hash=sha256:1f95b894f13729334fb990162e911c9e5dc1ab390c58aa6cbecb389c5b5e28ec \ + --hash=sha256:200e1050faffacc162be6a486a984a0497866ec54149a01270adc8a59b7c7d21 \ + --hash=sha256:2040ad4d1795a0ae89bfc7e8429677f365d45aa9fd5e4587cf1ea737f927b4a1 \ + --hash=sha256:2b64ca7d4aee0102a97f3ba22124052b4bd2152522355073580bf4845e2550b6 \ + --hash=sha256:2ceb2d3e01c5f1d83c4189737a42d9cb2fc38a6eeed225e7515eef71ad301dce \ + --hash=sha256:35c3a56d2ef83efc372eaec584314bd0ef2e2f0d2adb21c55e6ad5b344c0dcb8 \ + --hash=sha256:37425bc9175607b0268f493d79a292c39f9d001a357bebb6b88fdfaff13f6448 \ + --hash=sha256:3877ac408e14da24a6196de0ddcace62092bfc12a83823e92e49e40747e52c19 \ + --hash=sha256:3fd1fcdab3ea951b610dc4cef356d416d5802991e7e32b5254828d342f7b7e0b \ + --hash=sha256:41b71f4a3a4cab9d366cd9065b288efc4d4f3c0b37a91a8e0947fb5bd7f31d87 \ + --hash=sha256:43af8d1f3bea642559019edfe64e9b11192a8978efbd1539d7bc2aaa23d92de4 \ + --hash=sha256:45abad819184f07240d8a696117a7aacd39787af9e0b719d00285549ed19a1e9 \ + --hash=sha256:4b400bdc6f79fa02a4d86640310dde87a21fba0c979efff5248908c6f15fad1b \ + --hash=sha256:4eb6c25dd62ee8d5edf68a8e1c171dd71c292fdae95d8aeb3dd7d7de4c364082 \ + --hash=sha256:581b2264fc0aa555f3f435a5944da7504ea3a065d7029ad60e7c3d1ae09c5464 \ + --hash=sha256:5cf36e801231b6a2059bf354720274b7558746f3b1a4efb43fcf557ccd484a87 \ + --hash=sha256:5e3c5c011904115f88a39308379c17f91546f77c1667cea98739fe0fccea804c \ + --hash=sha256:6609bc224e9568f65064cfa72edc0f24ee6655b47575954ec6339534b2798369 \ + --hash=sha256:6e3dcd57ab780c741fde8dc68619de988b966db759a3c3152e8e9142c26295ad \ + --hash=sha256:6fac755ca3d2c3edcb22f479fceaa241704111414831ddd3bc6056e18516892f \ + --hash=sha256:744b2bf3640d907b79f3fd7874efe432d1cf171ee721243e350f55234b4cec4c \ + --hash=sha256:74cbb80d93260fe2ffa334efa24cb8f2f0f622a9b9febf8b483c0b865bfb3475 \ + --hash=sha256:766e0dc5a616d026a3a1cffa379af959671729083882f50307e18175797b3dfd \ + --hash=sha256:7bdf2da170b67fdf10bca777614b1c7d96ae3ca5794fd9587dce41eb2966e866 \ + --hash=sha256:7ff200bf9d24f2e4d5dc6ee8c3ac64d739d3a89e2326ba68aaf6c4a2b838fd7d \ + --hash=sha256:844e165636711ef41f80b4103ed234181646b98a53c8f05da12ca5ca289134f6 \ + --hash=sha256:8a604bae87c6195d8b1045eddece0514d041604b14f2727bbc2b3020172045eb \ + --hash=sha256:94055a11dfebe37c656e70317e1996dc197e1a15bbcc351bcdd4610e128fe1ca \ + --hash=sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0 \ + --hash=sha256:9cdc1a2fcfd5c52cfb3045feb399f7b3ce822abdde3a193a6b9a60b3cb5854ca \ + --hash=sha256:9ecb4efb1cd6e8c4afea0daa91a87fbddbce1b99d2895d151596716c0b2e859d \ + --hash=sha256:a3472cfbca0a54177d0faa68f697d8ba4c80bbdc19908c3465556d9f7efce9ee \ + --hash=sha256:a4328d245944d09fd639771de275701ccadf5f781ba0ff092ad141e017eccda4 \ + --hash=sha256:a48a72c77a310327f6a3a920092fa2b8fd03d7deaa60f093038f22d98e096717 \ + --hash=sha256:a720477885a9d2411f94a93d16f9d89bad0f28ca23c3f8daa521e2dcc3f44d49 \ + --hash=sha256:a77cbd07b940d326d39a1d1b37817e2ee4d79cb30e7338f3d0cddffae70fcaa2 \ + --hash=sha256:a9956e4d4f4a301ebf6cde39850333a6b6110799d470dbbb1e25326ac447f52a \ + --hash=sha256:adb2642e060a6549c343603a3851ba76ef0b74cc8c079a9a58121c7ec9fe2350 \ + --hash=sha256:beeda3d4ae615106d7094f7e7cef6218392e4465cc95d25f900bebabfded0950 \ + --hash=sha256:c80be5ede8f3f8eded4eff73cc99a25c388ce98e555b17d31da05287015ffa5b \ + --hash=sha256:cc90d2e9c7e5c7f1a482c9875007c095c3194b1cfedca3c2f3291cdc2bc7c086 \ + --hash=sha256:cd96a1898c0a47be4520327e01f874acfd61fb48a9420f8aa9f6483412ffa444 \ + --hash=sha256:d2650c1fb97e184d12d8ba010493ee7b322864f7d3d00d3f9bb97d9c21de4068 \ + --hash=sha256:d30e57c72013c2a4fe441c2fcb8e77b14e152ad48b5464858e07e2ad9fbfceff \ + --hash=sha256:d59c30000a16d8edc7e64152e30220bfbd724c9bbb08368c054e24c651314f0a \ + --hash=sha256:dbc12c9f3d185f5c737d801da555fb74b3dcfa1a50b66a1a93e09190f41fab50 \ + --hash=sha256:e18f12c6b0bc5a592ed23d3f7b891f68fd7f8241d69b7883769eb5d5dfb52696 \ + --hash=sha256:e19ebea31758fac5893a2ac360fedd00116cbb7628e650842a6691ba7ca28a21 \ + --hash=sha256:e30bdeaa5deed6bc27b4cc490823cd0347d7dae09119b8803ae576ea0ce52e4c \ + --hash=sha256:eb092099205ef62cd1782b006658db09e2fed75bffcae7cc0d44052d8aa0f484 \ + --hash=sha256:eee2cfda04c00a857206a4330f0c5e3e56535494e30ca445eb19ec624ae75118 \ + --hash=sha256:f4115102802df98b2b0db3cce5cb9b92572633a1197c77b7553e5203f284a5b3 \ + --hash=sha256:f590cd684941912d10becc07325a3eeb77886fe981415660d9265c4c418d0bea \ + --hash=sha256:f8885db0bc2bffa59d5c1b72fad7a6a92d3e80e7257f967dd81abb553a90d293 \ + --hash=sha256:fcb310ddb270a06114bb64bbe53c94926b943f5b7f0842194d585c65eb4edd76 + # via + # neuronx-cc + # nki six==1.17.0 \ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 # via python-dateutil -tqdm==4.67.1 \ - --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 \ - --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2 +tqdm==4.67.3 \ + --hash=sha256:7d825f03f89244ef73f1d4ce193cb1774a8179fd96f31d7e1dcde62092b960bb \ + --hash=sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf # via neuronx-cc typing-extensions==4.15.0 \ --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ diff --git a/platforms/rocm/BUILD.bazel b/platforms/rocm/BUILD.bazel index 76ee4e9f..6f55b279 100644 --- a/platforms/rocm/BUILD.bazel +++ b/platforms/rocm/BUILD.bazel @@ -10,6 +10,7 @@ zig_shared_library( # to avoid requiring on the _zig_probe_stack symbol. zigopts = ["-fno-stack-check"], deps = ["//stdx"], + target_compatible_with = ["@platforms//os:linux"] ) alias( diff --git a/third_party/flashattn/BUILD.bazel b/third_party/flashattn/BUILD.bazel index 3f32c61c..e69de29b 100644 --- a/third_party/flashattn/BUILD.bazel +++ b/third_party/flashattn/BUILD.bazel @@ -1,13 +0,0 @@ -load("@rules_cc//cc:cc_library.bzl", "cc_library") - -filegroup( - name = "libflashattn", - visibility = ["//visibility:public"], - srcs = ["flashattn/lib/libflashattn.so"], -) - -cc_library( - name = "c", - hdrs = ["flashattn/include/capi.h"], - visibility = ["//visibility:public"], -) diff --git a/third_party/flashattn/flashattention.BUILD.bazel b/third_party/flashattn/flashattention.BUILD.bazel new file mode 100644 index 00000000..3f32c61c --- /dev/null +++ b/third_party/flashattn/flashattention.BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +filegroup( + name = "libflashattn", + visibility = ["//visibility:public"], + srcs = ["flashattn/lib/libflashattn.so"], +) + +cc_library( + name = "c", + hdrs = ["flashattn/include/capi.h"], + visibility = ["//visibility:public"], +) diff --git a/third_party/flashattn/repo.bzl b/third_party/flashattn/repo.bzl index 654620cc..ced98a5b 100644 --- a/third_party/flashattn/repo.bzl +++ b/third_party/flashattn/repo.bzl @@ -6,7 +6,7 @@ def repo(): name = "flashattn", url = "https://github.com/zml/flash-attention/releases/download/v0.0.6-rc7/flashattn.tar.gz", sha256 = "6b8195a5ab285404fcb084122c1c32f342ff695040b8652c9144afd2289b71b3", - build_file = "//:third_party/flashattn/BUILD.bazel", + build_file = "//:third_party/flashattn/flashattention.BUILD.bazel", ) #new_local_repository(name = "flashattn", build_file="//:third_party/flashattn/BUILD.bazel", path="/home/corendos/flashattn/") diff --git a/third_party/org_swig_swig/repo.bzl b/third_party/org_swig_swig/repo.bzl index c35659bf..9a6c55bb 100644 --- a/third_party/org_swig_swig/repo.bzl +++ b/third_party/org_swig_swig/repo.bzl @@ -3,7 +3,10 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") def repo(): http_archive( name = "org_swig_swig", - url = "http://prdownloads.sourceforge.net/swig/swig-4.4.1.tar.gz", + urls = [ + "https://downloads.sourceforge.net/project/swig/swig/swig-4.4.1/swig-4.4.1.tar.gz", + "https://prdownloads.sourceforge.net/swig/swig-4.4.1.tar.gz", + ], sha256 = "40162a706c56f7592d08fd52ef5511cb7ac191f3593cf07306a0a554c6281fcf", strip_prefix = "swig-4.4.1", build_file = "//:third_party/org_swig_swig/swig.bazel", diff --git a/third_party/zls/BUILD.bazel b/third_party/zls/BUILD.bazel index b47cae3e..ec6f9df7 100644 --- a/third_party/zls/BUILD.bazel +++ b/third_party/zls/BUILD.bazel @@ -43,6 +43,9 @@ filegroup( zig_binary( name = "workspace_printer", main = "workspace_printer.zig", + deps = [ + "@rules_zig//zig/runfiles", + ], ) zig_binary( diff --git a/upb/upb.zig b/upb/upb.zig index b58252d7..f07d003a 100644 --- a/upb/upb.zig +++ b/upb/upb.zig @@ -34,7 +34,6 @@ pub const ParseError = error{ BadUtf8, MaxDepthExceeded, MissingRequired, - UnlinkedSubMessage, Unknown, } || std.mem.Allocator.Error; @@ -102,7 +101,6 @@ pub fn parseEx(comptime UpbType: type, arena: *c.upb_Arena, data: []const u8, op c.kUpb_DecodeStatus_BadUtf8 => ParseError.BadUtf8, c.kUpb_DecodeStatus_MaxDepthExceeded => ParseError.MaxDepthExceeded, c.kUpb_DecodeStatus_MissingRequired => ParseError.MissingRequired, - c.kUpb_DecodeStatus_UnlinkedSubMessage => ParseError.UnlinkedSubMessage, else => ParseError.Unknown, }; } diff --git a/zml/testing.zig b/zml/testing.zig index d3895fcb..265a19c6 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -254,7 +254,7 @@ fn BufferizedWithArgs(comptime T: type) type { /// Automatically calls the executable with the given arguments, taking care of arguments and results allocation. /// This helper can be used in tests to make the code a bit less verbose. /// It doesn't handle pointers in inputs/outputs structs. -pub fn autoCall(allocator: std.mem.Allocator, io: std.Io, exe: *const zml.exe.Exe, func: anytype, inputs: zml.Bufferized(std.meta.ArgsTuple(func))) !zml.Bufferized(stdx.meta.FnReturn(func)) { +pub fn autoCall(allocator: std.mem.Allocator, io: std.Io, exe: *const zml.exe.Exe, func: anytype, inputs: zml.Bufferized(std.meta.ArgsTuple(@TypeOf(func)))) !zml.Bufferized(stdx.meta.FnReturn(func)) { var args = try exe.args(allocator); defer args.deinit(allocator); From 2f175540bfbb504b528c3ae0d6132e36e6e86221 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Mon, 23 Mar 2026 16:07:40 +0100 Subject: [PATCH 375/881] ci: rework pipeline + add remote with buildbuddy Co-authored-by: Benjamin Rabier --- .github/workflows/ci.yaml | 147 +++++++++--------------- bazel/BUILD.bazel | 1 + examples/llama/BUILD.bazel | 3 + ffi/BUILD.bazel | 2 + mlir/BUILD.bazel | 2 + mlir/dialects/BUILD.bazel | 1 + mlir/dialects/stablehlo/BUILD.bazel | 1 + pjrt/BUILD.bazel | 1 + platforms/BUILD.bazel | 1 + platforms/cpu/BUILD.bazel | 3 + platforms/cuda/BUILD.bazel | 3 + platforms/cuda/flashattn/BUILD.bazel | 1 + platforms/neuron/BUILD.bazel | 7 ++ platforms/neuron/libpjrt_neuron.zig | 12 +- platforms/neuron/neuronx-cc.zig | 12 +- platforms/rocm/BUILD.bazel | 3 + platforms/tpu/BUILD.bazel | 3 + stdx/BUILD.bazel | 1 + tools/BUILD.bazel | 1 + upb/BUILD.bazel | 2 + zml/BUILD.bazel | 5 + zml/io/BUILD.bazel | 1 + zml/io/vfs/BUILD.bazel | 1 + zml/tokenizer/BUILD.bazel | 1 + zml/tokenizer/hftokenizers/BUILD.bazel | 2 + zml/tokenizer/sentencepiece/BUILD.bazel | 2 + 26 files changed, 123 insertions(+), 96 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8b422d96..c5238a56 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -12,80 +12,87 @@ concurrency: cancel-in-progress: ${{ !contains(github.ref, 'master')}} jobs: - platforms: + ci: + name: ci-${{ matrix.target }} strategy: + fail-fast: false matrix: - runtimes: + include: + - target: cpu + bazel_opts: "" + self_hosted: false + run_tests: true + runner_labels: + - runs-on + - runner=4cpu-linux-x64 + - image=ubuntu24-amd64 + - run-id=${{ github.run_id }} + - extras=s3-cache + - target: neuron + bazel_opts: "--@zml//platforms:neuron=true --@zml//platforms:cpu=false" + self_hosted: false + run_tests: false + runner_labels: + - runs-on + - runner=4cpu-linux-x64 + - image=ubuntu24-amd64 + - run-id=${{ github.run_id }} + - extras=s3-cache + - target: tpu + bazel_opts: "--@zml//platforms:tpu=true --@zml//platforms:cpu=false" + self_hosted: false + run_tests: false + runner_labels: + - runs-on + - runner=4cpu-linux-x64 + - image=ubuntu24-amd64 + - run-id=${{ github.run_id }} + - extras=s3-cache - target: cuda bazel_opts: "--@zml//platforms:cuda=true --@zml//platforms:cpu=false" - enable_cache: false - user: actions - runs-on: ["self-hosted", "platform=cuda"] + self_hosted: true + run_tests: true + runner_labels: ["self-hosted", "platform=cuda"] - target: rocm bazel_opts: "--@zml//platforms:rocm=true --@zml//platforms:cpu=false" - enable_cache: false - user: actions - runs-on: ["self-hosted", "platform=rocm"] - - target: cpu - bazel_opts: "--@zml//platforms:cpu=true" - enable_cache: true - user: runner - runs-on: - [ - "runs-on", - "runner=4cpu-linux-x64", - "image=ubuntu24-amd64", - "run-id=${{ github.run_id }}", - "extras=s3-cache", - ] - runs-on: ${{ matrix.runtimes.runs-on }} + self_hosted: true + run_tests: true + runner_labels: ["self-hosted", "platform=rocm"] + runs-on: ${{ matrix.runner_labels }} steps: - uses: runs-on/action@v1 - name: Checkout uses: actions/checkout@v4 - - uses: runs-on/snapshot@v1 - if: ${{ matrix.runtimes.enable_cache }} - with: - path: /home/${{ matrix.runtimes.user }}/.cache/bazel-disk - volume_size: 350 - - - name: TMP fix for the mount user - if: ${{ matrix.runtimes.enable_cache }} - run: sudo chown -R ${{ matrix.runtimes.user }}:${{ matrix.runtimes.user }} /home/${{ matrix.runtimes.user }}/.cache/ - - name: Ensure self hosted cache permissions - if: ${{ !matrix.runtimes.enable_cache }} + if: ${{ matrix.self_hosted }} run: mkdir -p /tmp/zig-cache && mkdir -p /tmp/zml && sudo chmod -R 777 /tmp/zig-cache && sudo chmod -R 777 /tmp/zml - - uses: bazel-contrib/setup-bazel@0.18.0 + - name: Setup Bazel + uses: bazel-contrib/setup-bazel@0.18.0 with: bazelisk-version: 1.28.1 - bazelisk-cache: false - # todo: use `--config=debug` instead of `-c opt` when static archive bug is fixed + bazelisk-cache: ${{ !matrix.self_hosted }} + repository-cache: ${{ !matrix.self_hosted }} + external-cache: ${{ !matrix.self_hosted }} bazelrc: | - common -c opt common --verbose_failures common --keep_going common --test_output=streamed common --color=yes common --show_timestamps - common --disk_cache=/home/${{ matrix.runtimes.user }}/.cache/bazel-disk - common --strip=always + ${{ !matrix.self_hosted && secrets.BUILDBUDDY_CI_API_KEY != '' && 'common --config=remote' || '' }} + ${{ !matrix.self_hosted && secrets.BUILDBUDDY_CI_API_KEY != '' && format('common:remote --remote_header=x-buildbuddy-api-key={0}', secrets.BUILDBUDDY_CI_API_KEY) || '' }} + ${{ !matrix.self_hosted && secrets.BUILDBUDDY_CI_API_KEY != '' && 'common:remote --remote_download_outputs=minimal' || '' }} + ${{ matrix.self_hosted && 'common --disk_cache=/home/actions/.cache/bazel-disk' || '' }} - - name: ZML / bazel build //zml/... - run: bazel build ${{ matrix.runtimes.bazel_opts }} //zml/... + - run: bazel mod graph + - run: bazel query ${{ matrix.bazel_opts }} //... + - run: bazel build ${{ matrix.bazel_opts }} //... + - if: ${{ matrix.run_tests }} + run: bazel test ${{ matrix.bazel_opts }} //... - - name: ZML / bazel test //zml/... - run: bazel test ${{ matrix.runtimes.bazel_opts }} //zml/... - - - name: ZML / bazel test //mlir/... - run: bazel test ${{ matrix.runtimes.bazel_opts }} //mlir/... - - - name: ZML / bazel test //stdx/... - run: bazel test ${{ matrix.runtimes.bazel_opts }} //stdx/... - - zig-lint: + lint: runs-on: [ "runs-on", @@ -110,43 +117,3 @@ jobs: - name: No autofix in Zig code run: bash -c "! git grep -l '// autofix' -- '*.zig'" - - ci: - runs-on: - [ - "runs-on", - "runner=4cpu-linux-x64", - "image=ubuntu24-amd64", - "run-id=${{ github.run_id }}", - "extras=s3-cache", - ] - steps: - - uses: runs-on/action@v1 - - name: Checkout - uses: actions/checkout@v4 - - - uses: bazel-contrib/setup-bazel@0.18.0 - with: - bazelisk-version: 1.28.1 - bazelisk-cache: true - # Both cache are useful even with the remote execution. - # no cache 5min6s build - # repository-cache: 37s cache load + 4m3s build - # external-cache: 2m29s cache load + 1m25s build - # As of this comment, we're also limited by the runner's CPU with only 4 cores. - repository-cache: true - bazelrc: | - common --config=remote - common --remote_header=x-buildbuddy-api-key=${{ secrets.BUILDBUDDY_CI_API_KEY }} - common --verbose_failures - common --keep_going - common --test_output=streamed - common --color=yes - common --show_timestamps - common --remote_download_outputs=minimal - - - run: bazel mod explain - - run: bazel mod graph - - run: bazel query //... - - run: bazel build //... - - run: bazel test //... diff --git a/bazel/BUILD.bazel b/bazel/BUILD.bazel index bb4a1781..6292f235 100644 --- a/bazel/BUILD.bazel +++ b/bazel/BUILD.bazel @@ -2,6 +2,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library") zig_library( name = "bazel", + tags = ["manual"], main = "bazel.zig", visibility = ["//visibility:public"], deps = [ diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel index d02b86d2..c2a6da49 100644 --- a/examples/llama/BUILD.bazel +++ b/examples/llama/BUILD.bazel @@ -19,6 +19,7 @@ zig_binary( mtree_spec( name = "mtree", srcs = [":llama"], + tags = ["manual"] ) tar( @@ -33,6 +34,7 @@ tar( ], compress = "zstd", mtree = ":mtree", + tags = ["manual"] ) oci_image( @@ -43,6 +45,7 @@ oci_image( "@platforms//os:linux", ], tars = [":archive"], + tags = ["manual"] ) platform_transition_filegroup( diff --git a/ffi/BUILD.bazel b/ffi/BUILD.bazel index 1bf8ca7f..f53c70ed 100644 --- a/ffi/BUILD.bazel +++ b/ffi/BUILD.bazel @@ -3,6 +3,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library") cc_library( name = "cc", + tags = ["manual"], hdrs = [ "zig_allocator.h", "zig_slice.h", @@ -12,6 +13,7 @@ cc_library( zig_library( name = "zig", + tags = ["manual"], srcs = [ "zig_allocator.zig", "zig_slice.zig", diff --git a/mlir/BUILD.bazel b/mlir/BUILD.bazel index 9605a38b..fe081091 100644 --- a/mlir/BUILD.bazel +++ b/mlir/BUILD.bazel @@ -3,6 +3,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") cc_library( name = "c", + tags = ["manual"], srcs = ["register.cc"], hdrs = ["mlir.h"], visibility = ["//mlir:__subpackages__"], @@ -18,6 +19,7 @@ cc_library( zig_library( name = "mlir", + tags = ["manual"], main = "mlir.zig", visibility = ["//visibility:public"], deps = [ diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/BUILD.bazel index 5c5dd67a..38e70da7 100644 --- a/mlir/dialects/BUILD.bazel +++ b/mlir/dialects/BUILD.bazel @@ -2,6 +2,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "dialects", + tags = ["manual"], srcs = [ "func.zig", ], diff --git a/mlir/dialects/stablehlo/BUILD.bazel b/mlir/dialects/stablehlo/BUILD.bazel index 45e69239..338cce95 100644 --- a/mlir/dialects/stablehlo/BUILD.bazel +++ b/mlir/dialects/stablehlo/BUILD.bazel @@ -2,6 +2,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "stablehlo", + tags = ["manual"], import_name = "mlir/dialects/stablehlo", main = "stablehlo.zig", visibility = ["//mlir/dialects:__subpackages__"], diff --git a/pjrt/BUILD.bazel b/pjrt/BUILD.bazel index e9f365d1..3d00cde7 100644 --- a/pjrt/BUILD.bazel +++ b/pjrt/BUILD.bazel @@ -2,6 +2,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library") zig_library( name = "pjrt", + tags = ["manual"], srcs = ["ffi.zig"], main = "pjrt.zig", visibility = ["//visibility:public"], diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index 64f3c911..873a6643 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -28,6 +28,7 @@ PLATFORMS = { zig_library( name = "platforms", + tags = ["manual"], main = "platforms.zig", visibility = ["//visibility:public"], deps = [ diff --git a/platforms/cpu/BUILD.bazel b/platforms/cpu/BUILD.bazel index baa31044..537c7a8d 100644 --- a/platforms/cpu/BUILD.bazel +++ b/platforms/cpu/BUILD.bazel @@ -20,6 +20,7 @@ config_setting( cc_library( name = "empty", + tags = ["manual"], defines = ["ZML_RUNTIME_CPU_DISABLED"], ) @@ -36,12 +37,14 @@ copy_to_directory( cc_library( name = "libpjrt_cpu", + tags = ["manual"], data = [":sandbox"], defines = ["ZML_RUNTIME_CPU"], ) zig_library( name = "cpu", + tags = ["manual"], import_name = "platforms/cpu", main = "cpu.zig", visibility = ["//visibility:public"], diff --git a/platforms/cuda/BUILD.bazel b/platforms/cuda/BUILD.bazel index 8386c10f..b0fead7e 100644 --- a/platforms/cuda/BUILD.bazel +++ b/platforms/cuda/BUILD.bazel @@ -20,11 +20,13 @@ zig_shared_library( cc_library( name = "empty", + tags = ["manual"], defines = ["ZML_RUNTIME_CUDA_DISABLED"], ) cc_library( name = "libpjrt_cuda", + tags = ["manual"], hdrs = ["libpjrt_cuda.h"], defines = ["ZML_RUNTIME_CUDA"], deps = ["@libpjrt_cuda"], @@ -32,6 +34,7 @@ cc_library( zig_library( name = "cuda", + tags = ["manual"], data = select({ "//platforms:cuda.enabled": [ "@flashattn//:libflashattn", diff --git a/platforms/cuda/flashattn/BUILD.bazel b/platforms/cuda/flashattn/BUILD.bazel index 9ef2bbf8..31fab169 100644 --- a/platforms/cuda/flashattn/BUILD.bazel +++ b/platforms/cuda/flashattn/BUILD.bazel @@ -2,6 +2,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library") zig_library( name = "flashattn", + tags = ["manual"], import_name = "platforms/cuda/flashattn", main = "flashattn.zig", visibility = ["//visibility:public"], diff --git a/platforms/neuron/BUILD.bazel b/platforms/neuron/BUILD.bazel index 97baeeaa..09ca892e 100644 --- a/platforms/neuron/BUILD.bazel +++ b/platforms/neuron/BUILD.bazel @@ -42,6 +42,7 @@ runfiles_to_default( cc_library( name = "libpython", + tags = ["manual"], hdrs = ["libpython.h"], deps = [ "@rules_python//python/cc:current_py_cc_headers", @@ -51,16 +52,19 @@ cc_library( cc_library( name = "empty", + tags = ["manual"], defines = ["ZML_RUNTIME_NEURON_DISABLED"], ) cc_library( name = "zmlxneuron", + tags = ["manual"], defines = ["ZML_RUNTIME_NEURON"], ) cc_library( name = "libnrt_headers", + tags = ["manual"], hdrs = ["nrt.h"], deps = ["@libpjrt_neuron//:libnrt_headers"], ) @@ -68,11 +72,13 @@ cc_library( upb_c_proto_library( name = "xla_data_upb", deps = ["@xla//xla:xla_data_proto"], + tags = ["manual"], ) upb_c_proto_library( name = "hlo_proto_upb", deps = ["@xla//xla/service:hlo_proto"], + tags = ["manual"], ) zig_shared_library( @@ -110,6 +116,7 @@ uv_lock( zig_library( name = "neuron", + tags = ["manual"], import_name = "platforms/neuron", main = "neuron.zig", visibility = ["//visibility:public"], diff --git a/platforms/neuron/libpjrt_neuron.zig b/platforms/neuron/libpjrt_neuron.zig index 1d25cde6..b132de2f 100644 --- a/platforms/neuron/libpjrt_neuron.zig +++ b/platforms/neuron/libpjrt_neuron.zig @@ -68,11 +68,17 @@ fn pyStatusCheck(status: c.PyStatus) void { } } -fn toPosixPathW(file_path: []const u8) error{NameTooLong}![std.posix.PATH_MAX - 1:0]c.wchar_t { - if (file_path.len >= std.posix.PATH_MAX) return error.NameTooLong; +fn toPosixPathW(file_path: []const u8) error{ InvalidUtf8, NameTooLong, CodepointTooLarge }![std.posix.PATH_MAX - 1:0]c.wchar_t { + var view = std.unicode.Utf8View.init(file_path) catch return error.InvalidUtf8; + var it = view.iterator(); var path_with_null: [std.posix.PATH_MAX - 1:0]c.wchar_t = undefined; - const len = c.mbstowcs(&path_with_null, file_path.ptr, file_path.len); + var len: usize = 0; + while (it.nextCodepoint()) |cp| { + if (len + 1 >= path_with_null.len) return error.NameTooLong; + path_with_null[len] = std.math.cast(c.wchar_t, cp) orelse return error.CodepointTooLarge; + len += 1; + } path_with_null[len] = 0; return path_with_null; } diff --git a/platforms/neuron/neuronx-cc.zig b/platforms/neuron/neuronx-cc.zig index 4efafc96..a0bbb526 100644 --- a/platforms/neuron/neuronx-cc.zig +++ b/platforms/neuron/neuronx-cc.zig @@ -11,11 +11,17 @@ fn pyStatusCheck(status: c.PyStatus) void { } } -pub fn toPosixPathW(file_path: []const u8) error{NameTooLong}![std.posix.PATH_MAX - 1:0]c.wchar_t { - if (file_path.len >= std.posix.PATH_MAX) return error.NameTooLong; +pub fn toPosixPathW(file_path: []const u8) error{ InvalidUtf8, NameTooLong, CodepointTooLarge }![std.posix.PATH_MAX - 1:0]c.wchar_t { + var view = std.unicode.Utf8View.init(file_path) catch return error.InvalidUtf8; + var it = view.iterator(); var path_with_null: [std.posix.PATH_MAX - 1:0]c.wchar_t = undefined; - const len = c.mbstowcs(&path_with_null, file_path.ptr, file_path.len); + var len: usize = 0; + while (it.nextCodepoint()) |cp| { + if (len + 1 >= path_with_null.len) return error.NameTooLong; + path_with_null[len] = std.math.cast(c.wchar_t, cp) orelse return error.CodepointTooLarge; + len += 1; + } path_with_null[len] = 0; return path_with_null; } diff --git a/platforms/rocm/BUILD.bazel b/platforms/rocm/BUILD.bazel index 6f55b279..5e1e8c05 100644 --- a/platforms/rocm/BUILD.bazel +++ b/platforms/rocm/BUILD.bazel @@ -20,17 +20,20 @@ alias( cc_library( name = "empty", + tags = ["manual"], defines = ["ZML_RUNTIME_ROCM_DISABLED"], ) cc_library( name = "libpjrt_rocm", + tags = ["manual"], defines = ["ZML_RUNTIME_ROCM"], deps = ["@libpjrt_rocm"], ) zig_library( name = "rocm", + tags = ["manual"], import_name = "platforms/rocm", main = "rocm.zig", visibility = ["//visibility:public"], diff --git a/platforms/tpu/BUILD.bazel b/platforms/tpu/BUILD.bazel index af25cd2f..7f9ae972 100644 --- a/platforms/tpu/BUILD.bazel +++ b/platforms/tpu/BUILD.bazel @@ -3,17 +3,20 @@ load("@rules_zig//zig:defs.bzl", "zig_library") cc_library( name = "empty", + tags = ["manual"], defines = ["ZML_RUNTIME_TPU_DISABLED"], ) cc_library( name = "libpjrt_tpu", + tags = ["manual"], defines = ["ZML_RUNTIME_TPU"], deps = ["@libpjrt_tpu"], ) zig_library( name = "tpu", + tags = ["manual"], import_name = "platforms/tpu", main = "tpu.zig", visibility = ["//visibility:public"], diff --git a/stdx/BUILD.bazel b/stdx/BUILD.bazel index 01448459..d4217b90 100644 --- a/stdx/BUILD.bazel +++ b/stdx/BUILD.bazel @@ -2,6 +2,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "stdx", + tags = ["manual"], srcs = [ "Io.zig", "bounded_array.zig", diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 513e9ac2..925a59db 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -10,6 +10,7 @@ py_library( cc_library( name = "xspace_to_perfetto_lib", + tags = ["manual"], srcs = ["xspace_to_perfetto_lib.cc"], hdrs = ["xspace_to_perfetto.h"], visibility = ["//visibility:public"], diff --git a/upb/BUILD.bazel b/upb/BUILD.bazel index ebf973fb..9189fc8f 100644 --- a/upb/BUILD.bazel +++ b/upb/BUILD.bazel @@ -3,6 +3,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library") cc_library( name = "upb_c", + tags = ["manual"], hdrs = ["upb.h"], deps = [ "@com_google_protobuf//upb/message:copy", @@ -11,6 +12,7 @@ cc_library( zig_library( name = "upb", + tags = ["manual"], main = "upb.zig", visibility = ["//visibility:public"], deps = [":upb_c"], diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index da49f7de..40be0245 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -5,25 +5,30 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") upb_c_proto_library( name = "xla_data_upb", deps = ["@xla//xla:xla_data_proto"], + tags = ["manual"], ) upb_c_proto_library( name = "xla_compile_options_upb", deps = ["@xla//xla/pjrt/proto:compile_options_proto"], + tags = ["manual"], ) upb_c_proto_library( name = "profiler_options", deps = ["@tsl//tsl/profiler/protobuf:profiler_options_proto"], + tags = ["manual"], ) cc_library( name = "posix", + tags = ["manual"], hdrs = ["posix.h"], ) zig_library( name = "zml", + tags = ["manual"], srcs = [ "attention.zig", "attention/attention.zig", diff --git a/zml/io/BUILD.bazel b/zml/io/BUILD.bazel index 8407a5c6..e12eaa01 100644 --- a/zml/io/BUILD.bazel +++ b/zml/io/BUILD.bazel @@ -2,6 +2,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library") zig_library( name = "io", + tags = ["manual"], srcs = [ "vfs.zig", ], diff --git a/zml/io/vfs/BUILD.bazel b/zml/io/vfs/BUILD.bazel index 9f79dd24..86221e9b 100644 --- a/zml/io/vfs/BUILD.bazel +++ b/zml/io/vfs/BUILD.bazel @@ -2,6 +2,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_binary") zig_library( name = "vfs", + tags = ["manual"], srcs = [ "base.zig", "file.zig", diff --git a/zml/tokenizer/BUILD.bazel b/zml/tokenizer/BUILD.bazel index 96360191..b00dd2ef 100644 --- a/zml/tokenizer/BUILD.bazel +++ b/zml/tokenizer/BUILD.bazel @@ -3,6 +3,7 @@ load("@zml//bazel:zig_srcs.bzl", "zig_srcs") zig_library( name = "tokenizer", + tags = ["manual"], srcs = ["homemade.zig"], import_name = "zml/tokenizer", main = "tokenizer.zig", diff --git a/zml/tokenizer/hftokenizers/BUILD.bazel b/zml/tokenizer/hftokenizers/BUILD.bazel index 02a49495..43ee1fbe 100644 --- a/zml/tokenizer/hftokenizers/BUILD.bazel +++ b/zml/tokenizer/hftokenizers/BUILD.bazel @@ -13,6 +13,7 @@ rust_static_library( cc_library( name = "hftokenizers_cc", + tags = ["manual"], hdrs = ["hftokenizers.h"], visibility = ["//visibility:public"], deps = [ @@ -23,6 +24,7 @@ cc_library( zig_library( name = "hftokenizers", + tags = ["manual"], main = "hftokenizers.zig", visibility = ["//visibility:public"], deps = [ diff --git a/zml/tokenizer/sentencepiece/BUILD.bazel b/zml/tokenizer/sentencepiece/BUILD.bazel index 9e46e373..dd4d2e76 100644 --- a/zml/tokenizer/sentencepiece/BUILD.bazel +++ b/zml/tokenizer/sentencepiece/BUILD.bazel @@ -5,6 +5,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library") swig_cc_library( name = "sentencepiece_swig", + tags = ["manual"], interface = "sentencepiece.i", module = "sentencepiece", deps = [ @@ -15,6 +16,7 @@ swig_cc_library( zig_library( name = "sentencepiece", + tags = ["manual"], import_name = "sentencepiece", main = "sentencepiece.zig", visibility = ["//visibility:public"], From a68f0f9c857891c3cb945512cadf7e48e009d6b1 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Mon, 23 Mar 2026 18:33:02 +0100 Subject: [PATCH 376/881] workspace/zls: add non regression test (#426) --- third_party/zls/BUILD.bazel | 2 + third_party/zls/workspace_printer.zig | 22 +++--- third_party/zls/workspace_printer_test.zig | 90 ++++++++++++++++++++++ third_party/zls/zls_completion.bzl | 30 +++++++- 4 files changed, 132 insertions(+), 12 deletions(-) create mode 100644 third_party/zls/workspace_printer_test.zig diff --git a/third_party/zls/BUILD.bazel b/third_party/zls/BUILD.bazel index ec6f9df7..c8bfd83d 100644 --- a/third_party/zls/BUILD.bazel +++ b/third_party/zls/BUILD.bazel @@ -20,6 +20,7 @@ resolved_toolchain( exports_files( [ "workspace_printer.zig", + "workspace_printer_test.zig", "zls_build_runner.zig", "zls_runner.zig", "zls.completion.tpl", @@ -31,6 +32,7 @@ filegroup( name = "all_files", srcs = [ ":BUILD.bazel", + ":workspace_printer_test.zig", ":zls.completion.tpl", ":zls_build_runner.zig", ":zls_completion.bzl", diff --git a/third_party/zls/workspace_printer.zig b/third_party/zls/workspace_printer.zig index 94329a8f..dd4b38e3 100644 --- a/third_party/zls/workspace_printer.zig +++ b/third_party/zls/workspace_printer.zig @@ -123,14 +123,19 @@ pub fn main(init: std.process.Init) !void { const args = try init.minimal.args.toSlice(arena.allocator()); const build_workspace_directory = init.environ_map.get("BUILD_WORKSPACE_DIRECTORY").?; - const output_base_result = try std.process.run(arena.allocator(), init.io, .{ - .argv = &.{ - "bazel", - "info", - "execution_root", - }, - .cwd = .{ .path = build_workspace_directory }, - }); + const execution_root = if (init.environ_map.get("BAZEL_EXECUTION_ROOT")) |value| + value + else blk: { + const output_base_result = try std.process.run(arena.allocator(), init.io, .{ + .argv = &.{ + "bazel", + "info", + "execution_root", + }, + .cwd = .{ .path = build_workspace_directory }, + }); + break :blk std.mem.trimEnd(u8, output_base_result.stdout, "\n"); + }; var output = try readBuildConfig( arena.allocator(), @@ -145,7 +150,6 @@ pub fn main(init: std.process.Init) !void { "@@__BUILD_WORKSPACE_DIRECTORY__@@", build_workspace_directory, ); - const execution_root = std.mem.trimEnd(u8, output_base_result.stdout, "\n"); output = try std.mem.replaceOwned( u8, arena.allocator(), diff --git a/third_party/zls/workspace_printer_test.zig b/third_party/zls/workspace_printer_test.zig new file mode 100644 index 00000000..71e81adb --- /dev/null +++ b/third_party/zls/workspace_printer_test.zig @@ -0,0 +1,90 @@ +const std = @import("std"); + +const bazel_builtin = @import("bazel_builtin"); +const runfiles = @import("runfiles"); + +const workspace_printer = @import("workspace_printer.zig"); + +fn requireEnv(name: [:0]const u8) ![]const u8 { + return std.mem.span(std.c.getenv(name) orelse return error.MissingEnvironmentVariable); +} + +fn workspaceDirFromConfigPath(config_path: []const u8, package_name: []const u8) ![]const u8 { + var workspace_dir = std.fs.path.dirname(config_path) orelse + return error.InvalidWorkspacePath; + if (package_name.len == 0) { + return workspace_dir; + } + + var it = std.mem.splitScalar(u8, package_name, '/'); + while (it.next()) |_| { + workspace_dir = std.fs.path.dirname(workspace_dir) orelse + return error.InvalidWorkspacePath; + } + return workspace_dir; +} + +test "completion print_build_config emits valid json" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + + var r_ = try runfiles.Runfiles.create(.{ + .allocator = allocator, + .io = std.testing.io, + .directory = if (std.c.getenv("RUNFILES_DIR")) |value| std.mem.span(value) else null, + .manifest = if (std.c.getenv("RUNFILES_MANIFEST_FILE")) |value| std.mem.span(value) else null, + }) orelse return error.RunfilesNotFound; + defer r_.deinit(allocator); + + const r = r_.withSourceRepo(bazel_builtin.current_repository); + + const printer_rpath = try requireEnv("COMPLETION_PRINTER_RLOCATION"); + const config_rpath = try requireEnv("COMPLETION_BUILD_CONFIG_RLOCATION"); + const package_name = try requireEnv("COMPLETION_PACKAGE"); + + const printer_path = try r.rlocationAlloc(allocator, printer_rpath) orelse + return error.RLocationNotFound; + const config_path = try r.rlocationAlloc(allocator, config_rpath) orelse + return error.RLocationNotFound; + const workspace_dir = try workspaceDirFromConfigPath(config_path, package_name); + const execution_root = workspace_dir; + + var child_env_map: std.process.Environ.Map = .init(std.testing.allocator); + defer child_env_map.deinit(); + try child_env_map.put("BUILD_WORKSPACE_DIRECTORY", workspace_dir); + try child_env_map.put("BAZEL_EXECUTION_ROOT", execution_root); + + const result = try std.process.run(std.testing.allocator, std.testing.io, .{ + .argv = &.{ printer_path, config_path }, + .environ_map = &child_env_map, + }); + defer std.testing.allocator.free(result.stdout); + defer std.testing.allocator.free(result.stderr); + + if (result.stderr.len > 0) { + std.log.warn("workspace_printer stderr: {s}", .{result.stderr}); + } + + try std.testing.expectEqual(@as(std.process.Child.Term, .{ .exited = 0 }), result.term); + + const config = try std.json.parseFromSliceLeaky(workspace_printer.BuildConfig, allocator, result.stdout, .{}); + try std.testing.expect(config.modules.map.count() > 0); + + var modules = config.modules.map.iterator(); + while (modules.next()) |entry| { + try std.testing.expect(entry.key_ptr.*.len > 0); + + const module = entry.value_ptr.*; + for (module.import_table.map.values()) |import_path| { + try std.testing.expect(import_path.len > 0); + } + for (module.include_dirs) |include_dir| { + try std.testing.expect(include_dir.len > 0); + } + } + + for (config.compilations) |compilation| { + try std.testing.expect(config.modules.map.contains(compilation.root_module)); + } +} diff --git a/third_party/zls/zls_completion.bzl b/third_party/zls/zls_completion.bzl index a461f933..c6ccadfe 100644 --- a/third_party/zls/zls_completion.bzl +++ b/third_party/zls/zls_completion.bzl @@ -2,7 +2,7 @@ load("@bazel_lib//lib:utils.bzl", "utils") load("@bazel_skylib//rules:expand_template.bzl", "expand_template") -load("@rules_zig//zig:defs.bzl", "zig_binary") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_test") load(":zls_write_build_config.bzl", "zls_write_build_config") load(":zls_write_runner_zig_src.bzl", "zls_write_runner_zig_src") @@ -18,9 +18,10 @@ def zls_completion(name, deps, testonly = False, **kwargs): # Generate the ZLS BuildConfig file. # It contains the list of Zig packages alongside their main Zig file paths. build_config = name + ".build_config" + build_config_file = name + ".build_config.json" zls_write_build_config( name = build_config, - out = name + ".build_config.json", + out = build_config_file, deps = deps, testonly = testonly, ) @@ -32,9 +33,10 @@ def zls_completion(name, deps, testonly = False, **kwargs): main = Label("//third_party/zls:workspace_printer.zig"), data = [ ":{}".format(build_config), + ":{}".format(build_config_file), ], args = [ - "$(rlocationpath :{})".format(build_config), + "$(rlocationpath :{})".format(build_config_file), ], deps = [ "@rules_zig//zig/runfiles", @@ -43,6 +45,28 @@ def zls_completion(name, deps, testonly = False, **kwargs): testonly = testonly, ) + # Used to verify that the build config printer works as expected. + # Prevent regressions. + zig_test( + name = "{}_test".format(build_config_printer), + main = Label("//third_party/zls:workspace_printer_test.zig"), + srcs = [Label("//third_party/zls:workspace_printer.zig")], + data = deps + [ + ":{}".format(build_config), + ":{}".format(build_config_file), + ":{}".format(build_config_printer), + ], + env = { + "COMPLETION_BUILD_CONFIG_RLOCATION": "$(rlocationpath :{})".format(build_config_file), + "COMPLETION_PACKAGE": native.package_name(), + "COMPLETION_PRINTER_RLOCATION": "$(rlocationpath :{})".format(build_config_printer), + }, + deps = [ + "@rules_zig//zig/runfiles", + ], + testonly = testonly, + ) + # Generate the Zig build runner that will be used by ZLS to query the build config. expand_template( name = name + ".build_runner", From 7f71fa58c2ead570be1d57678d1c0bb693cb80a4 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Mon, 23 Mar 2026 18:44:41 +0100 Subject: [PATCH 377/881] flashattn: update flash-attention to v2026.3.23 --- platforms/cuda/flashattn/flashattn.zig | 2 +- third_party/flashattn/flashattention.BUILD.bazel | 4 ++-- third_party/flashattn/repo.bzl | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/platforms/cuda/flashattn/flashattn.zig b/platforms/cuda/flashattn/flashattn.zig index 368d0ffb..eb4a8c72 100644 --- a/platforms/cuda/flashattn/flashattn.zig +++ b/platforms/cuda/flashattn/flashattn.zig @@ -100,7 +100,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { const r = try bazel.runfiles(bazel_builtin.current_repository); var buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; - const library = (try r.rlocation("flashattn/flashattn/lib/libflashattn.so", &buffer)) orelse return error.NotFound; + const library = (try r.rlocation("flashattn/flashattn/libflashattn.so", &buffer)) orelse return error.NotFound; var lib = std.DynLib.open(library) catch |err| { std.log.err("Failed to open libflashattn.so: {any}", .{err}); return err; diff --git a/third_party/flashattn/flashattention.BUILD.bazel b/third_party/flashattn/flashattention.BUILD.bazel index 3f32c61c..90f6aff5 100644 --- a/third_party/flashattn/flashattention.BUILD.bazel +++ b/third_party/flashattn/flashattention.BUILD.bazel @@ -3,11 +3,11 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") filegroup( name = "libflashattn", visibility = ["//visibility:public"], - srcs = ["flashattn/lib/libflashattn.so"], + srcs = ["flashattn/libflashattn.so"], ) cc_library( name = "c", - hdrs = ["flashattn/include/capi.h"], + hdrs = ["flashattn/flashattn.h"], visibility = ["//visibility:public"], ) diff --git a/third_party/flashattn/repo.bzl b/third_party/flashattn/repo.bzl index ced98a5b..146e5cc6 100644 --- a/third_party/flashattn/repo.bzl +++ b/third_party/flashattn/repo.bzl @@ -4,10 +4,9 @@ load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") def repo(): http_archive( name = "flashattn", - url = "https://github.com/zml/flash-attention/releases/download/v0.0.6-rc7/flashattn.tar.gz", - sha256 = "6b8195a5ab285404fcb084122c1c32f342ff695040b8652c9144afd2289b71b3", + url = "https://github.com/zml/flash-attention/releases/download/v2026.3.23/flashattn-linux-amd64.tar.zst", + sha256 = "79496eaa5a373170a201a31abe822c1774138fce8ffcd2aa39c6f0fc46c79e2d", build_file = "//:third_party/flashattn/flashattention.BUILD.bazel", ) #new_local_repository(name = "flashattn", build_file="//:third_party/flashattn/BUILD.bazel", path="/home/corendos/flashattn/") - From 463ca4a76263fa58239dfe46bab71acea60fc46f Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 24 Mar 2026 14:28:34 +0100 Subject: [PATCH 378/881] zml: add zml.ops.while and zml.nn.GatedDeltaNet (#427) --- mlir/dialects/stablehlo/stablehlo.zig | 22 ++ zml/nn.zig | 295 ++++++++++++++++++++++++++ zml/ops.zig | 178 ++++++++++++++++ 3 files changed, 495 insertions(+) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 127de962..51edfddf 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -620,6 +620,28 @@ pub fn dynamic_update_slice(ctx: *mlir.Context, operand: *const mlir.Value, upda }); } +pub fn while_( + ctx: *mlir.Context, + operands: []const *const mlir.Value, + result_types: []const *const mlir.Type, + cond_block: *mlir.Block, + body_block: *mlir.Block, + location: *const mlir.Location, +) *mlir.Operation { + _ = ctx; + var state = mlir.OperationState.init("stablehlo.while", location); + state.addOperands(operands); + state.addResults(result_types); + + const cond_region = mlir.Region.init(); + cond_region.appendOwnedBlock(cond_block); + const body_region = mlir.Region.init(); + body_region.appendOwnedBlock(body_block); + state.addOwnedRegions(&.{ cond_region, body_region }); + + return mlir.Operation.init(&state) catch @panic("Failed to create stablehlo.while operation"); +} + pub fn get_tuple_element(ctx: *mlir.Context, tuple_value: *const mlir.Value, index: i64, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.get_tuple_element", .{ .operands = .{ .flat = &.{tuple_value} }, diff --git a/zml/nn.zig b/zml/nn.zig index 4a3d22e7..a5673402 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -1115,6 +1115,301 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { return attn.transpose(q.shape()).merge(.{ .h = .{ .h, .hq } }); } +pub const GatedDeltaNet = struct { + pub const State = struct { + /// Per-head recurrent state with shape .{ .h, .v, .k }. + s: Tensor, + }; + + pub const StepInputs = struct { + /// Query tensor with shape .{ .h, .k }. + q: Tensor, + /// Key tensor with shape .{ .h, .k }. + k: Tensor, + /// Value tensor with shape .{ .h, .v }. + v: Tensor, + /// Forget gate with shape .{ .h }. + alpha: Tensor, + /// Delta gate with shape .{ .h }. + beta: Tensor, + }; + + pub const StepOutput = struct { + state: State, + output: Tensor, + }; + + pub const Output = struct { + /// Sequence of outputs with shape .{ .s, .h, .v }. + outputs: Tensor, + state: State, + }; + + fn sliceStep(input: Tensor, step_: Tensor) Tensor { + return input.dynamicSlice(.{ .s = Tensor.DynSlice{ .start = step_, .len = 1 } }).squeeze(.s); + } + + fn validateStep(state: State, inputs: StepInputs) void { + const err_template = "GatedDeltaNet.step(state: {f}, q: {f}, k: {f}, v: {f}, alpha: {f}, beta: {f}) is invalid ! "; + const err_args = .{ state.s, inputs.q, inputs.k, inputs.v, inputs.alpha, inputs.beta }; + + stdx.debug.assert(state.s.shape().hasTags(.{ .h, .v, .k }), err_template ++ "state.s is missing tags {{.h, .v, .k}}", err_args); + stdx.debug.assert(inputs.q.shape().hasTags(.{ .h, .k }), err_template ++ "q is missing tags {{.h, .k}}", err_args); + stdx.debug.assert(inputs.k.shape().hasTags(.{ .h, .k }), err_template ++ "k is missing tags {{.h, .k}}", err_args); + stdx.debug.assert(inputs.v.shape().hasTags(.{ .h, .v }), err_template ++ "v is missing tags {{.h, .v}}", err_args); + stdx.debug.assert(inputs.alpha.shape().hasTags(.{.h}), err_template ++ "alpha is missing tag {{.h}}", err_args); + stdx.debug.assert(inputs.beta.shape().hasTags(.{.h}), err_template ++ "beta is missing tag {{.h}}", err_args); + + _ = collectDims(.{.h}, &.{ state.s, inputs.q, inputs.k, inputs.v, inputs.alpha, inputs.beta }, .strict) catch { + stdx.debug.panic(err_template ++ "head dimensions are inconsistent.", err_args); + }; + _ = collectDims(.{.k}, &.{ state.s, inputs.q, inputs.k }, .strict) catch { + stdx.debug.panic(err_template ++ "key dimensions are inconsistent.", err_args); + }; + _ = collectDims(.{.v}, &.{ state.s, inputs.v }, .strict) catch { + stdx.debug.panic(err_template ++ "value dimensions are inconsistent.", err_args); + }; + } + + /// Single-step recurrent update for Gated Delta Net. + /// + /// Shapes: + /// - `state.s`: .{ .h, .v, .k } + /// - `inputs.q`, `inputs.k`: .{ .h, .k } + /// - `inputs.v`: .{ .h, .v } + /// - `inputs.alpha`, `inputs.beta`: .{ .h } + pub fn step(state: State, inputs: StepInputs) StepOutput { + validateStep(state, inputs); + + const v_hat = state.s.dot(inputs.k, .k).mul(inputs.alpha.insertAxes(.last, .{.v})); + const delta = inputs.v.sub(v_hat).mul(inputs.beta.insertAxes(.last, .{.v})); + const delta_k = delta.insertAxes(.last, .{.k}).broad(state.s.shape()).mul(inputs.k.insertAxes(.k, .{.v}).broad(state.s.shape())); + const s_new = state.s.mul(inputs.alpha.insertAxes(.last, .{ .v, .k })).add(delta_k); + const y = s_new.dot(inputs.q, .k); + + return .{ + .state = .{ .s = s_new }, + .output = y, + }; + } + + /// Processes a sequence with Gated Delta Net recurrence using `stablehlo.while`. + /// + /// Shapes: + /// - `queries`, `keys`: .{ .s, .h, .k } + /// - `values`, `outputs`: .{ .s, .h, .v } + /// - `alphas`, `betas`: .{ .s, .h } + /// - `initial_state.s`: .{ .h, .v, .k } + pub fn forward( + queries: Tensor, + keys: Tensor, + values: Tensor, + alphas: Tensor, + betas: Tensor, + initial_state: State, + ) Output { + const err_template = "GatedDeltaNet.forward(queries: {f}, keys: {f}, values: {f}, alphas: {f}, betas: {f}, state: {f}) is invalid ! "; + const err_args = .{ queries, keys, values, alphas, betas, initial_state.s }; + + stdx.debug.assert(queries.shape().hasTags(.{ .s, .h, .k }), err_template ++ "queries is missing tags {{.s, .h, .k}}", err_args); + stdx.debug.assert(keys.shape().hasTags(.{ .s, .h, .k }), err_template ++ "keys is missing tags {{.s, .h, .k}}", err_args); + stdx.debug.assert(values.shape().hasTags(.{ .s, .h, .v }), err_template ++ "values is missing tags {{.s, .h, .v}}", err_args); + stdx.debug.assert(alphas.shape().hasTags(.{ .s, .h }), err_template ++ "alphas is missing tags {{.s, .h}}", err_args); + stdx.debug.assert(betas.shape().hasTags(.{ .s, .h }), err_template ++ "betas is missing tags {{.s, .h}}", err_args); + stdx.debug.assert(initial_state.s.shape().hasTags(.{ .h, .v, .k }), err_template ++ "initial_state.s is missing tags {{.h, .v, .k}}", err_args); + + _ = collectDims(.{ .s, .h }, &.{ queries, keys, values, alphas, betas }, .strict) catch { + stdx.debug.panic(err_template ++ "sequence/head dimensions are inconsistent.", err_args); + }; + _ = collectDims(.{.k}, &.{ queries, keys, initial_state.s }, .strict) catch { + stdx.debug.panic(err_template ++ "key dimensions are inconsistent.", err_args); + }; + _ = collectDims(.{.v}, &.{ values, initial_state.s }, .strict) catch { + stdx.debug.panic(err_template ++ "value dimensions are inconsistent.", err_args); + }; + + const WhileContext = struct { + queries: Tensor, + keys: Tensor, + values: Tensor, + alphas: Tensor, + betas: Tensor, + seq_len: Tensor, + }; + const while_context: WhileContext = .{ + .queries = queries, + .keys = keys, + .values = values, + .alphas = alphas, + .betas = betas, + .seq_len = Tensor.scalar(queries.dim(.s), .i32), + }; + + const Local = struct { + fn cond(step_: Tensor, _: Tensor, _: Tensor, ctx: WhileContext) Tensor { + return step_.cmp(.LT, ctx.seq_len); + } + + fn body(step_: Tensor, s_prev: Tensor, outputs_prev: Tensor, ctx: WhileContext) [3]Tensor { + const step_out = step( + .{ .s = s_prev }, + .{ + .q = sliceStep(ctx.queries, step_), + .k = sliceStep(ctx.keys, step_), + .v = sliceStep(ctx.values, step_), + .alpha = sliceStep(ctx.alphas, step_), + .beta = sliceStep(ctx.betas, step_), + }, + ); + const outputs_new = outputs_prev.dynamicUpdateSlice(.{ .s = step_ }, step_out.output); + + return .{ + step_.add(Tensor.scalar(1, .i32)), + step_out.state.s, + outputs_new, + }; + } + }; + + const step0 = Tensor.scalar(0, .i32); + const outputs0 = Tensor.zeroes(values.shape()); + const loop_state = ops.@"while"( + .{ step0, initial_state.s, outputs0 }, + Local.cond, + Local.body, + .{while_context}, + ); + + return .{ + .outputs = loop_state[2], + .state = .{ .s = loop_state[1] }, + }; + } +}; + +test "gated delta net" { + const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); + + const queries: zml.Tensor = .init(.{ .s = 2, .h = 2, .k = 2 }, .f32); + const keys: zml.Tensor = .init(.{ .s = 2, .h = 2, .k = 2 }, .f32); + const values: zml.Tensor = .init(.{ .s = 2, .h = 2, .v = 2 }, .f32); + const alphas: zml.Tensor = .init(.{ .s = 2, .h = 2 }, .f32); + const betas: zml.Tensor = .init(.{ .s = 2, .h = 2 }, .f32); + const initial_s: zml.Tensor = .init(.{ .h = 2, .v = 2, .k = 2 }, .f32); + + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + GatedDeltaNet.forward, + .{ queries, keys, values, alphas, betas, GatedDeltaNet.State{ .s = initial_s } }, + platform, + .{ .shardings = &.{replicated_sharding} }, + ); + defer exe.deinit(); + + var queries_buffer: zml.Buffer = try .fromBytes( + std.testing.io, + platform, + queries.shape(), + replicated_sharding, + std.mem.sliceAsBytes(&[2][2][2]f32{ + .{ .{ 1.0, 0.0 }, .{ 0.0, 1.0 } }, + .{ .{ 1.0, 1.0 }, .{ 1.0, -1.0 } }, + }), + ); + defer queries_buffer.deinit(); + var keys_buffer: zml.Buffer = try .fromBytes( + std.testing.io, + platform, + keys.shape(), + replicated_sharding, + std.mem.sliceAsBytes(&[2][2][2]f32{ + .{ .{ 1.0, 2.0 }, .{ 0.0, 1.0 } }, + .{ .{ 2.0, 1.0 }, .{ 1.0, 0.0 } }, + }), + ); + defer keys_buffer.deinit(); + var values_buffer: zml.Buffer = try .fromBytes( + std.testing.io, + platform, + values.shape(), + replicated_sharding, + std.mem.sliceAsBytes(&[2][2][2]f32{ + .{ .{ 3.0, 1.0 }, .{ 2.0, 4.0 } }, + .{ .{ 1.0, 5.0 }, .{ 3.0, 0.0 } }, + }), + ); + defer values_buffer.deinit(); + var alphas_buffer: zml.Buffer = try .fromBytes( + std.testing.io, + platform, + alphas.shape(), + replicated_sharding, + std.mem.sliceAsBytes(&[2][2]f32{ + .{ 0.5, 0.25 }, + .{ 0.8, 0.6 }, + }), + ); + defer alphas_buffer.deinit(); + var betas_buffer: zml.Buffer = try .fromBytes( + std.testing.io, + platform, + betas.shape(), + replicated_sharding, + std.mem.sliceAsBytes(&[2][2]f32{ + .{ 1.0, 0.5 }, + .{ 0.75, 1.0 }, + }), + ); + defer betas_buffer.deinit(); + var initial_s_buffer: zml.Buffer = try .fromBytes( + std.testing.io, + platform, + initial_s.shape(), + replicated_sharding, + std.mem.sliceAsBytes(&[2][2][2]f32{ + .{ .{ 1.0, 0.0 }, .{ 0.0, 1.0 } }, + .{ .{ 2.0, 1.0 }, .{ 1.0, 0.0 } }, + }), + ); + defer initial_s_buffer.deinit(); + + var result = try zml.testing.autoCall( + std.testing.allocator, + std.testing.io, + &exe, + GatedDeltaNet.forward, + .{ queries_buffer, keys_buffer, values_buffer, alphas_buffer, betas_buffer, .{ .s = initial_s_buffer } }, + ); + defer result.outputs.deinit(); + defer result.state.s.deinit(); + + const expected_outputs: zml.Slice = .init( + zml.Shape.init(.{ 2, 2, 2 }, .f32), + std.mem.sliceAsBytes(&[2][2][2]f32{ + .{ .{ 3.0, 0.0 }, .{ 1.125, 2.0 } }, + .{ .{ -11.15, 10.75 }, .{ 2.325, -1.2 } }, + }), + ); + const expected_final_s: zml.Slice = .init( + zml.Shape.init(.{ 2, 2, 2 }, .f32), + std.mem.sliceAsBytes(&[2][2][2]f32{ + .{ .{ -9.3, -1.85 }, .{ 6.9, 3.85 } }, + .{ .{ 3.0, 0.675 }, .{ 0.0, 1.2 } }, + }), + ); + + try zml.testing.expectClose(std.testing.io, expected_outputs, result.outputs, .{ + .absolute_tolerance = 1e-4, + .relative_tolerance = 1e-4, + }); + try zml.testing.expectClose(std.testing.io, expected_final_s, result.state.s, .{ + .absolute_tolerance = 1e-4, + .relative_tolerance = 1e-4, + }); +} + /// Options controlling generation. The default values correspond to greedy decoding. pub const SamplingStrategy = struct { topk: u32 = 1, diff --git a/zml/ops.zig b/zml/ops.zig index 3c558655..d1ae194d 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -260,6 +260,184 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytyp return result; } +pub fn @"while"(operands: anytype, comptime cond: anytype, comptime body: anytype, context: anytype) stdx.meta.FnReturn(body) { + stdx.debug.assertComptime(stdx.meta.isTupleOf(@TypeOf(operands), Tensor), "zml.ops.while expects a tuple of Tensor operands, got {}", .{@TypeOf(operands)}); + stdx.debug.assertComptime(stdx.meta.isTuple(@TypeOf(context)), "zml.ops.while expects tuple context like .{{ ... }}, got {}", .{@TypeOf(context)}); + + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const mlir_ctx = CompilationContext.current().mlir_ctx; + const location = mlir.Location.unknown(mlir_ctx); + + var captured_context: @TypeOf(context) = undefined; + meta.mapAlloc(struct { + fn capture(_: void, tensor: Tensor) Tensor { + return Tensor._result(tensor.shape(), tensor.value()); + } + }.capture, arena.allocator(), {}, context, &captured_context) catch unreachable; + + var block_types: [operands.len]*const mlir.Type = undefined; + var block_locs: [operands.len]*const mlir.Location = @splat(location); + var operand_values: [operands.len]*const mlir.Value = undefined; + var operand_shapes: [operands.len]Shape = undefined; + + inline for (0..operands.len) |i| { + operand_shapes[i] = operands[i].shape(); + block_types[i] = mlir.rankedTensorType(operand_shapes[i].dims(), mlirx.Type.fromDType(mlir_ctx, operand_shapes[i].dtype())); + operand_values[i] = operands[i].value(); + } + + const cond_block = b: { + var cond_args: @TypeOf(operands) = undefined; + inline for (0..operands.len) |i| { + cond_args[i] = .fromShape(operand_shapes[i]); + } + + const block = mlir.Block.init(&block_types, &block_locs); + errdefer block.deinit(); + + CompilationContext.current().pushBlock(block); + defer CompilationContext.current().popBlock(); + + const scope = CompilationContext.current().currentScope(); + inline for (0..operands.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), cond_args[i].id, i) catch unreachable; + } + + const cond_result = @call(.auto, cond, cond_args ++ captured_context); + stdx.debug.assert(meta.count(Tensor, &cond_result) == 1, "zml.ops.while expects cond to return exactly one Tensor, got {}", .{@TypeOf(cond_result)}); + + const cond_tensor = meta.first(Tensor, cond_result); + stdx.debug.assert(cond_tensor.rank() == 0 and cond_tensor.dtype() == .bool, "zml.ops.while expects cond to return a scalar bool Tensor, got {f}", .{cond_tensor}); + + _ = dialects.stablehlo.return_(mlir_ctx, cond_tensor.value(), location).appendTo(block); + break :b block; + }; + + const body_block, var result = b: { + var body_args: @TypeOf(operands) = undefined; + inline for (0..operands.len) |i| { + body_args[i] = .fromShape(operand_shapes[i]); + } + + const block = mlir.Block.init(&block_types, &block_locs); + errdefer block.deinit(); + + CompilationContext.current().pushBlock(block); + defer CompilationContext.current().popBlock(); + + const scope = CompilationContext.current().currentScope(); + inline for (0..operands.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), body_args[i].id, i) catch unreachable; + } + + const result = @call(.auto, body, body_args ++ captured_context); + stdx.debug.assert(meta.count(Tensor, &result) == operands.len, "zml.ops.while expects body to return {d} Tensor values, got {}", .{ operands.len, @TypeOf(result) }); + + var result_tensors: [operands.len]Tensor = undefined; + meta.collectBuf(struct { + fn cb(tensor: Tensor) Tensor { + return tensor; + } + }.cb, {}, &result, &result_tensors); + + var result_values: [operands.len]*const mlir.Value = undefined; + inline for (0..operands.len) |i| { + stdx.debug.assert(result_tensors[i].shape().eql(operand_shapes[i]), "zml.ops.while expects body to preserve loop state shape, got {f} for operand {d} but expected {f}", .{ result_tensors[i], i, operands[i] }); + result_values[i] = result_tensors[i].value(); + } + + _ = dialects.stablehlo.returns(mlir_ctx, &result_values, location).appendTo(block); + break :b .{ block, result }; + }; + + const op = dialects.stablehlo.while_( + mlir_ctx, + &operand_values, + &block_types, + cond_block, + body_block, + location, + ).appendTo(CompilationContext.current().currentScope().block); + + const n_operands = operands.len; + const AssignResultCtx = struct { + op_: *mlir.Operation, + shapes: [n_operands]Shape, + idx: usize = 0, + }; + var assign_result_ctx: AssignResultCtx = .{ + .op_ = op, + .shapes = operand_shapes, + }; + meta.visit(struct { + fn cb(ctx: *AssignResultCtx, tensor: *Tensor) void { + tensor.* = Tensor._result(ctx.shapes[ctx.idx], ctx.op_.result(ctx.idx)); + ctx.idx += 1; + } + }.cb, &assign_result_ctx, &result); + + return result; +} + +pub const while_ = @"while"; + +test @"while" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const replicated_sharding = zml.testing.replicatedSharding(); + + const initial_i: zml.Tensor = .init(.{}, .i64); + const initial_sum: zml.Tensor = .init(.{}, .i64); + + const Local = struct { + fn cond(i: Tensor, sum: Tensor) Tensor { + _ = sum; + return i.cmp(.LT, Tensor.scalar(10, .i64)); + } + + fn body(i: Tensor, sum: Tensor) [2]Tensor { + const one = Tensor.scalar(1, .i64); + return .{ + i.add(one), + sum.add(one), + }; + } + + fn forward(i: Tensor, sum: Tensor) [2]Tensor { + return zml.ops.@"while"(.{ i, sum }, cond, body, .{}); + } + }; + + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Local.forward, + .{ initial_i, initial_sum }, + platform, + .{ .shardings = &.{replicated_sharding} }, + ); + defer exe.deinit(); + + var i_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_i.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1]i64{1})); + defer i_buffer.deinit(); + var sum_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_sum.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1]i64{0})); + defer sum_buffer.deinit(); + + var results = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local.forward, .{ i_buffer, sum_buffer }); + defer results[0].deinit(); + defer results[1].deinit(); + + var cpu_i = try results[0].toSliceAlloc(std.testing.allocator, std.testing.io); + defer cpu_i.free(std.testing.allocator); + var cpu_sum = try results[1].toSliceAlloc(std.testing.allocator, std.testing.io); + defer cpu_sum.free(std.testing.allocator); + + try std.testing.expectEqual(@as(i64, 10), cpu_i.items(i64)[0]); + try std.testing.expectEqual(@as(i64, 9), cpu_sum.items(i64)[0]); +} + pub const TritonOps = struct { debug: bool = false, name: [:0]const u8, From 47c0bdd94d6f0208a38085a6ba96404f01eb7fb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20Gauti=C3=A9?= <51361266+tgautie@users.noreply.github.com> Date: Tue, 24 Mar 2026 15:57:14 +0100 Subject: [PATCH 379/881] Set xla flags (#429) Fix XLA flags to avoid autotune crash on Nvidia 4090: - Enable CublasLT on Cuda - Disable autotune and triton gemm --- zml/module.zig | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/zml/module.zig b/zml/module.zig index 6afc2254..b71fbf36 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -565,9 +565,16 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: const overrides_map = c._xla_CompileOptionsProto_env_option_overrides_mutable_upb_map(options, upb_arena); switch (platform.target) { .cuda => { - // NVIDIA recommends these settings - // https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables + // Enable cuBLASLt, and disable autotuning and triton as it is unstable on some platforms (e.g. Nvidia 4090). + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_cublaslt", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_autotune_level", 0, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_triton_gemm", false, upb_arena); + + // NVIDIA recommends this setting, which allows XLA to move communication collectives to increase overlap with compute kernels. + // See https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_latency_hiding_scheduler", true, upb_arena); + + // Compile in parallel. try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_llvm_module_compilation_parallelism", true, upb_arena); }, .rocm => { From 942d6a1f0adc315b84c2cbeedf05536797759ad7 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 24 Mar 2026 15:11:03 +0000 Subject: [PATCH 380/881] Revert "Set xla flags (#429)" This reverts commit 47c0bdd94d6f0208a38085a6ba96404f01eb7fb3. --- zml/module.zig | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/zml/module.zig b/zml/module.zig index b71fbf36..6afc2254 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -565,16 +565,9 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: const overrides_map = c._xla_CompileOptionsProto_env_option_overrides_mutable_upb_map(options, upb_arena); switch (platform.target) { .cuda => { - // Enable cuBLASLt, and disable autotuning and triton as it is unstable on some platforms (e.g. Nvidia 4090). - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_cublaslt", true, upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_autotune_level", 0, upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_triton_gemm", false, upb_arena); - - // NVIDIA recommends this setting, which allows XLA to move communication collectives to increase overlap with compute kernels. - // See https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables + // NVIDIA recommends these settings + // https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_latency_hiding_scheduler", true, upb_arena); - - // Compile in parallel. try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_llvm_module_compilation_parallelism", true, upb_arena); }, .rocm => { From 140af6716e2240cf33bf449925ceb9faa1c9e36a Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 24 Mar 2026 15:21:47 +0000 Subject: [PATCH 381/881] platforms/cuda: bump PJRT plugin --- platforms/cuda/cuda.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 6785e80f..dc27fb96 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -269,8 +269,8 @@ def _cuda_impl(mctx): http_archive( name = "libpjrt_cuda", build_file = "libpjrt_cuda.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/nightly-2026-03-21/pjrt-cuda_linux-amd64.tar.gz", - sha256 = "854b2cabc162dd56ed38548062b5d2ce6bfc33cc21851789de8fcc346a82cb07", + url = "https://github.com/zml/pjrt-artifacts/releases/download/nightly-2026-03-24/pjrt-cuda_linux-amd64.tar.gz", + sha256 = "9ef9caf168974e71c8932361204739b739e4c61150637ab3c170ec4765ce9ac1", ) return mctx.extension_metadata( From 81299c67c841ba69b9cc5532a4cff08d484c8a90 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 24 Mar 2026 16:55:29 +0100 Subject: [PATCH 382/881] examples/llm: Common model example (#425) --- BUILD.bazel | 3 +- MODULE.bazel.lock | 1160 ++++++++++++++++- devenv.lock | 18 +- devenv.nix | 1 + examples/lfm/BUILD.bazel | 10 - examples/lfm/config.zig | 83 -- examples/lfm/discussion.zig | 301 ----- examples/lfm/main.zig | 173 --- examples/llama/BUILD.bazel | 76 -- examples/llama/main.zig | 483 ------- examples/llama/test.zig | 119 -- examples/llm/BUILD.bazel | 31 + examples/llm/README.md | 41 + examples/llm/chat.zig | 143 ++ examples/llm/main.zig | 186 +++ examples/llm/models.zig | 165 +++ examples/llm/models/common.zig | 30 + examples/llm/models/lfm2.zig | 10 + .../{lfm => llm/models/lfm2}/inference.zig | 145 ++- examples/{lfm => llm/models/lfm2}/model.zig | 320 ++--- examples/llm/models/lfm2/repository.zig | 76 ++ examples/llm/models/lfm2/session.zig | 160 +++ .../models/lfm2_tests.zig} | 142 +- examples/llm/models/llama.zig | 10 + examples/llm/models/llama/inference.zig | 150 +++ .../llama.zig => llm/models/llama/model.zig} | 295 +++-- examples/llm/models/llama/repository.zig | 81 ++ examples/llm/models/llama/session.zig | 167 +++ examples/llm/models/llama_tests.zig | 146 +++ examples/llm/models/qwen3_5.zig | 10 + examples/llm/models/qwen3_5/inference.zig | 143 ++ examples/llm/models/qwen3_5/model.zig | 984 ++++++++++++++ examples/llm/models/qwen3_5/repository.zig | 83 ++ examples/llm/models/qwen3_5/session.zig | 180 +++ zml/nn.zig | 2 +- 35 files changed, 4474 insertions(+), 1653 deletions(-) delete mode 100644 examples/lfm/BUILD.bazel delete mode 100644 examples/lfm/config.zig delete mode 100644 examples/lfm/discussion.zig delete mode 100644 examples/lfm/main.zig delete mode 100644 examples/llama/BUILD.bazel delete mode 100644 examples/llama/main.zig delete mode 100644 examples/llama/test.zig create mode 100644 examples/llm/BUILD.bazel create mode 100644 examples/llm/README.md create mode 100644 examples/llm/chat.zig create mode 100644 examples/llm/main.zig create mode 100644 examples/llm/models.zig create mode 100644 examples/llm/models/common.zig create mode 100644 examples/llm/models/lfm2.zig rename examples/{lfm => llm/models/lfm2}/inference.zig (78%) rename examples/{lfm => llm/models/lfm2}/model.zig (66%) create mode 100644 examples/llm/models/lfm2/repository.zig create mode 100644 examples/llm/models/lfm2/session.zig rename examples/{lfm/test_runner.zig => llm/models/lfm2_tests.zig} (69%) create mode 100644 examples/llm/models/llama.zig create mode 100644 examples/llm/models/llama/inference.zig rename examples/{llama/llama.zig => llm/models/llama/model.zig} (68%) create mode 100644 examples/llm/models/llama/repository.zig create mode 100644 examples/llm/models/llama/session.zig create mode 100644 examples/llm/models/llama_tests.zig create mode 100644 examples/llm/models/qwen3_5.zig create mode 100644 examples/llm/models/qwen3_5/inference.zig create mode 100644 examples/llm/models/qwen3_5/model.zig create mode 100644 examples/llm/models/qwen3_5/repository.zig create mode 100644 examples/llm/models/qwen3_5/session.zig diff --git a/BUILD.bazel b/BUILD.bazel index 671fd891..c133b6a0 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -4,8 +4,7 @@ zls_completion( name = "completion", visibility = ["//visibility:public"], deps = [ - "//examples/lfm", - "//examples/llama", + "//examples/llm", "//examples/mnist", "//examples/sharding", "//stdx", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 33055bbb..eeec762c 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -258,6 +258,360 @@ }, "selectedYankedVersions": {}, "moduleExtensions": { + "@@aspect_rules_py+//py:extensions.bzl%py_tools": { + "general": { + "bzlTransitiveDigest": "GLVIk55uU2jb460gpL1FFbE/tBFqzMxRfta5QXYfjQo=", + "usagesDigest": "9BiP3ASHlGl4doGhzXJwK17taNzv/q6x/IZ4Hb6LxNY=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "rules_py_tools.darwin_amd64": { + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", + "attributes": { + "platform": "darwin_amd64" + } + }, + "rules_py_tools.darwin_arm64": { + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", + "attributes": { + "platform": "darwin_arm64" + } + }, + "rules_py_tools.linux_amd64": { + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", + "attributes": { + "platform": "linux_amd64" + } + }, + "rules_py_tools.linux_arm64": { + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", + "attributes": { + "platform": "linux_arm64" + } + }, + "rules_py_tools": { + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%toolchains_repo", + "attributes": { + "user_repository_name": "rules_py_tools" + } + }, + "rules_py_pex_2_3_1": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "urls": [ + "https://files.pythonhosted.org/packages/e7/d0/fbda2a4d41d62d86ce53f5ae4fbaaee8c34070f75bb7ca009090510ae874/pex-2.3.1-py2.py3-none-any.whl" + ], + "sha256": "64692a5bf6f298403aab930d22f0d836ae4736c5bc820e262e9092fe8c56f830", + "downloaded_file_path": "pex-2.3.1-py2.py3-none-any.whl" + } + } + }, + "recordedRepoMappingEntries": [ + [ + "aspect_rules_py+", + "aspect_tools_telemetry_report", + "aspect_tools_telemetry++telemetry+aspect_tools_telemetry_report" + ], + [ + "aspect_rules_py+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "aspect_rules_py+", + "bazel_tools", + "bazel_tools" + ] + ] + } + }, + "@@aspect_tools_telemetry+//:extension.bzl%telemetry": { + "general": { + "bzlTransitiveDigest": "MePriaXmQNSqUfE+YQvEtzBc2bU1mjITINIffFiZYgo=", + "usagesDigest": "TU1quwRKpvAEOEF35/IVh4NeoIS2hayv4fGp8f5RXoY=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "aspect_tools_telemetry_report": { + "repoRuleId": "@@aspect_tools_telemetry+//:extension.bzl%tel_repository", + "attributes": { + "deps": { + "aspect_rules_py": "1.8.4", + "rules_rs": "0.0.44", + "aspect_tools_telemetry": "0.3.3" + } + } + } + }, + "recordedRepoMappingEntries": [ + [ + "aspect_tools_telemetry+", + "bazel_lib", + "bazel_lib+" + ], + [ + "aspect_tools_telemetry+", + "bazel_skylib", + "bazel_skylib+" + ] + ] + } + }, + "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { + "general": { + "bzlTransitiveDigest": "AEsQvKwdstptzt98AO1bJYfoWpPUn2upx53ra6o/Rc8=", + "usagesDigest": "QI2z8ZUR+mqtbwsf2fLqYdJAkPOHdOV+tF2yVAUgRzw=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "com_github_jetbrains_kotlin_git": { + "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:compiler.bzl%kotlin_compiler_git_repository", + "attributes": { + "urls": [ + "https://github.com/JetBrains/kotlin/releases/download/v1.9.23/kotlin-compiler-1.9.23.zip" + ], + "sha256": "93137d3aab9afa9b27cb06a824c2324195c6b6f6179d8a8653f440f5bd58be88" + } + }, + "com_github_jetbrains_kotlin": { + "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:compiler.bzl%kotlin_capabilities_repository", + "attributes": { + "git_repository_name": "com_github_jetbrains_kotlin_git", + "compiler_version": "1.9.23" + } + }, + "com_github_google_ksp": { + "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:ksp.bzl%ksp_compiler_plugin_repository", + "attributes": { + "urls": [ + "https://github.com/google/ksp/releases/download/1.9.23-1.0.20/artifacts.zip" + ], + "sha256": "ee0618755913ef7fd6511288a232e8fad24838b9af6ea73972a76e81053c8c2d", + "strip_version": "1.9.23-1.0.20" + } + }, + "com_github_pinterest_ktlint": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "sha256": "01b2e0ef893383a50dbeb13970fe7fa3be36ca3e83259e01649945b09d736985", + "urls": [ + "https://github.com/pinterest/ktlint/releases/download/1.3.0/ktlint" + ], + "executable": true + } + }, + "rules_android": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "sha256": "cd06d15dd8bb59926e4d65f9003bfc20f9da4b2519985c27e190cddc8b7a7806", + "strip_prefix": "rules_android-0.1.1", + "urls": [ + "https://github.com/bazelbuild/rules_android/archive/v0.1.1.zip" + ] + } + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_kotlin+", + "bazel_tools", + "bazel_tools" + ] + ] + } + }, + "@@rules_oci+//oci:extensions.bzl%oci": { + "general": { + "bzlTransitiveDigest": "twSeB3VPTVkayY0uwqEzDzal5MbnZX45iC6jdheZQRo=", + "usagesDigest": "IIGPr69MT/mSUo6fNUffrIiU31EqGjlEYm5qFmNjmUc=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "distroless_cc_debian12_linux_amd64": { + "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_pull", + "attributes": { + "www_authenticate_challenges": {}, + "scheme": "https", + "registry": "gcr.io", + "repository": "distroless/cc-debian12", + "identifier": "sha256:eccec5274132c1be0ce5d2c8e6fe41033e64af5e987ccee9007826e4c012069d", + "platform": "linux/amd64", + "target_name": "distroless_cc_debian12_linux_amd64", + "bazel_tags": [] + } + }, + "distroless_cc_debian12": { + "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_alias", + "attributes": { + "target_name": "distroless_cc_debian12", + "www_authenticate_challenges": {}, + "scheme": "https", + "registry": "gcr.io", + "repository": "distroless/cc-debian12", + "identifier": "sha256:eccec5274132c1be0ce5d2c8e6fe41033e64af5e987ccee9007826e4c012069d", + "platforms": { + "@@platforms//cpu:x86_64": "@distroless_cc_debian12_linux_amd64" + }, + "bzlmod_repository": "distroless_cc_debian12", + "reproducible": true + } + }, + "oci_crane_darwin_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "darwin_amd64", + "crane_version": "v0.18.0" + } + }, + "oci_crane_darwin_arm64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "darwin_arm64", + "crane_version": "v0.18.0" + } + }, + "oci_crane_linux_arm64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "linux_arm64", + "crane_version": "v0.18.0" + } + }, + "oci_crane_linux_armv6": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "linux_armv6", + "crane_version": "v0.18.0" + } + }, + "oci_crane_linux_i386": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "linux_i386", + "crane_version": "v0.18.0" + } + }, + "oci_crane_linux_s390x": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "linux_s390x", + "crane_version": "v0.18.0" + } + }, + "oci_crane_linux_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "linux_amd64", + "crane_version": "v0.18.0" + } + }, + "oci_crane_windows_armv6": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "windows_armv6", + "crane_version": "v0.18.0" + } + }, + "oci_crane_windows_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "windows_amd64", + "crane_version": "v0.18.0" + } + }, + "oci_crane_toolchains": { + "repoRuleId": "@@rules_oci+//oci/private:toolchains_repo.bzl%toolchains_repo", + "attributes": { + "toolchain_type": "@rules_oci//oci:crane_toolchain_type", + "toolchain": "@oci_crane_{platform}//:crane_toolchain" + } + }, + "oci_regctl_darwin_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "darwin_amd64" + } + }, + "oci_regctl_darwin_arm64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "darwin_arm64" + } + }, + "oci_regctl_linux_arm64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "linux_arm64" + } + }, + "oci_regctl_linux_s390x": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "linux_s390x" + } + }, + "oci_regctl_linux_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "linux_amd64" + } + }, + "oci_regctl_windows_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "windows_amd64" + } + }, + "oci_regctl_toolchains": { + "repoRuleId": "@@rules_oci+//oci/private:toolchains_repo.bzl%toolchains_repo", + "attributes": { + "toolchain_type": "@rules_oci//oci:regctl_toolchain_type", + "toolchain": "@oci_regctl_{platform}//:regctl_toolchain" + } + } + }, + "moduleExtensionMetadata": { + "explicitRootModuleDirectDeps": [ + "distroless_cc_debian12", + "distroless_cc_debian12_linux_amd64" + ], + "explicitRootModuleDirectDevDeps": [], + "useAllRepos": "NO", + "reproducible": false + }, + "recordedRepoMappingEntries": [ + [ + "aspect_bazel_lib+", + "bazel_tools", + "bazel_tools" + ], + [ + "bazel_features+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_oci+", + "aspect_bazel_lib", + "aspect_bazel_lib+" + ], + [ + "rules_oci+", + "bazel_features", + "bazel_features+" + ], + [ + "rules_oci+", + "bazel_skylib", + "bazel_skylib+" + ] + ] + } + }, "@@rules_python+//python/extensions:config.bzl%config": { "general": { "bzlTransitiveDigest": "X49Lbeli5+XwNYRk+isOvNKFMIF8g/rnxL3Fdlwrffc=", @@ -492,7 +846,811 @@ ] ] } + }, + "@@rules_python+//python/uv:uv.bzl%uv": { + "general": { + "bzlTransitiveDigest": "CJBWuaTerWIqLpLWl77vjWFD1RRpXfzC2UGMwEk6eVg=", + "usagesDigest": "zIGk4Tqt5ZS3xJ783QnGoVqaEhMIIzUMmK0DHMRGugg=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "uv_0_9_30_aarch64_apple_darwin": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.9.30", + "platform": "aarch64-apple-darwin", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-aarch64-apple-darwin.tar.gz" + ], + "sha256": "03a5d9ec7f7d588446b2ec226d13ff6300055e55365eca8f3fab39f342b0e805" + } + }, + "uv_0_9_30_aarch64_unknown_linux_gnu": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.9.30", + "platform": "aarch64-unknown-linux-gnu", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-aarch64-unknown-linux-gnu.tar.gz" + ], + "sha256": "6aadf3c71600d594e16dabf382cc15282ead4c5ca768599b6bcb43c5004d9aa8" + } + }, + "uv_0_9_30_powerpc64_unknown_linux_gnu": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.9.30", + "platform": "powerpc64-unknown-linux-gnu", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-powerpc64-unknown-linux-gnu.tar.gz" + ], + "sha256": "ac4cd1a021462885932f6023b005a4835cca4c72bb60dec186ee2be4b60dca6f" + } + }, + "uv_0_9_30_powerpc64le_unknown_linux_gnu": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.9.30", + "platform": "powerpc64le-unknown-linux-gnu", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-powerpc64le-unknown-linux-gnu.tar.gz" + ], + "sha256": "73b8cbc560c6b2fa205358365d4e174abdf50cfcf57dc36a447572c56eba5ae4" + } + }, + "uv_0_9_30_s390x_unknown_linux_gnu": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.9.30", + "platform": "s390x-unknown-linux-gnu", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-s390x-unknown-linux-gnu.tar.gz" + ], + "sha256": "b35975bb9e5c2c418b428d0316cc6e3c7a6eff710c69212be14005c192f54516" + } + }, + "uv_0_9_30_x86_64_apple_darwin": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.9.30", + "platform": "x86_64-apple-darwin", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-x86_64-apple-darwin.tar.gz" + ], + "sha256": "ce069bf750567e9a4a31d6e285d1eae75d444d8a281409b641235903943b7681" + } + }, + "uv_0_9_30_x86_64_pc_windows_msvc": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.9.30", + "platform": "x86_64-pc-windows-msvc", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-x86_64-pc-windows-msvc.zip" + ], + "sha256": "875981be7908295937dee09532bb66d576986d4f223259e171b0c767c885897a" + } + }, + "uv_0_9_30_x86_64_unknown_linux_gnu": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.9.30", + "platform": "x86_64-unknown-linux-gnu", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-x86_64-unknown-linux-gnu.tar.gz" + ], + "sha256": "8b3762374972daa7a74bbc6896cc73229ca69a07403dd9f9ea3805a51ffd7582" + } + }, + "uv": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_toolchains_repo.bzl%uv_toolchains_repo", + "attributes": { + "toolchain_type": "'@@rules_python+//python/uv:uv_toolchain_type'", + "toolchain_names": [ + "0_9_30_aarch64_apple_darwin", + "0_9_30_aarch64_unknown_linux_gnu", + "0_9_30_powerpc64_unknown_linux_gnu", + "0_9_30_powerpc64le_unknown_linux_gnu", + "0_9_30_s390x_unknown_linux_gnu", + "0_9_30_x86_64_apple_darwin", + "0_9_30_x86_64_pc_windows_msvc", + "0_9_30_x86_64_unknown_linux_gnu" + ], + "toolchain_implementations": { + "0_9_30_aarch64_apple_darwin": "@uv_0_9_30_aarch64_apple_darwin//:uv_toolchain", + "0_9_30_aarch64_unknown_linux_gnu": "@uv_0_9_30_aarch64_unknown_linux_gnu//:uv_toolchain", + "0_9_30_powerpc64_unknown_linux_gnu": "@uv_0_9_30_powerpc64_unknown_linux_gnu//:uv_toolchain", + "0_9_30_powerpc64le_unknown_linux_gnu": "@uv_0_9_30_powerpc64le_unknown_linux_gnu//:uv_toolchain", + "0_9_30_s390x_unknown_linux_gnu": "@uv_0_9_30_s390x_unknown_linux_gnu//:uv_toolchain", + "0_9_30_x86_64_apple_darwin": "@uv_0_9_30_x86_64_apple_darwin//:uv_toolchain", + "0_9_30_x86_64_pc_windows_msvc": "@uv_0_9_30_x86_64_pc_windows_msvc//:uv_toolchain", + "0_9_30_x86_64_unknown_linux_gnu": "@uv_0_9_30_x86_64_unknown_linux_gnu//:uv_toolchain" + }, + "toolchain_compatible_with": { + "0_9_30_aarch64_apple_darwin": [ + "'@@platforms//os:macos'", + "'@@platforms//cpu:aarch64'" + ], + "0_9_30_aarch64_unknown_linux_gnu": [ + "'@@platforms//os:linux'", + "'@@platforms//cpu:aarch64'" + ], + "0_9_30_powerpc64_unknown_linux_gnu": [ + "'@@platforms//os:linux'", + "'@@platforms//cpu:ppc'" + ], + "0_9_30_powerpc64le_unknown_linux_gnu": [ + "'@@platforms//os:linux'", + "'@@platforms//cpu:ppc64le'" + ], + "0_9_30_s390x_unknown_linux_gnu": [ + "'@@platforms//os:linux'", + "'@@platforms//cpu:s390x'" + ], + "0_9_30_x86_64_apple_darwin": [ + "'@@platforms//os:macos'", + "'@@platforms//cpu:x86_64'" + ], + "0_9_30_x86_64_pc_windows_msvc": [ + "'@@platforms//os:windows'", + "'@@platforms//cpu:x86_64'" + ], + "0_9_30_x86_64_unknown_linux_gnu": [ + "'@@platforms//os:linux'", + "'@@platforms//cpu:x86_64'" + ] + }, + "toolchain_target_settings": {} + } + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_python+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_python+", + "platforms", + "platforms" + ] + ] + } + }, + "@@rules_zig+//zig:extensions.bzl%zig": { + "general": { + "bzlTransitiveDigest": "r7tCqbjlVKc7XEXV8Hiak2ElFfkjvDdMf8Ym+7zxukU=", + "usagesDigest": "lqvJAPUaAfUjthqXXAa4LwhcuZ2d3CNerCqKcLkkNPo=", + "recordedFileInputs": { + "@@//bazel/zig_index.json": "809b545c21f13c11c4587dccbfe81de2b767a292325c25ffb785b03c601bbc2e", + "@@rules_zig+//zig/private/versions.json": "6eb85ebaee72c4e6fbc82d8ad73f69e0899d839b70a7e19caa792ab6a5fa0c36" + }, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-linux": { + "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", + "attributes": { + "url": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.2722+f16eb18ce.tar.xz", + "mirrors": [ + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds", + "https://ziglang.org/builds/" + ], + "sha256": "03725d297a2e2a8331d774eb601163af42ec61979b82753d41b2c460eefab1be", + "zig_version": "0.16.0-dev.2722+f16eb18ce", + "platform": "aarch64-linux", + "translate_c": "@@+non_module_deps+translate-c//:translate-c" + } + }, + "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-macos": { + "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", + "attributes": { + "url": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.2722+f16eb18ce.tar.xz", + "mirrors": [ + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds", + "https://ziglang.org/builds/" + ], + "sha256": "268aa8342e8ef539693e88e1d0982361b1d5dc4a080949264f2d5a14e5eb83e0", + "zig_version": "0.16.0-dev.2722+f16eb18ce", + "platform": "aarch64-macos", + "translate_c": "@@+non_module_deps+translate-c//:translate-c" + } + }, + "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-windows": { + "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", + "attributes": { + "url": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.2722+f16eb18ce.zip", + "mirrors": [ + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds", + "https://ziglang.org/builds/" + ], + "sha256": "43daaad311bc3b06a733a8a87d2c3180214d62db2067df1591b921826de453ea", + "zig_version": "0.16.0-dev.2722+f16eb18ce", + "platform": "aarch64-windows", + "translate_c": "@@+non_module_deps+translate-c//:translate-c" + } + }, + "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-linux": { + "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", + "attributes": { + "url": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.2722+f16eb18ce.tar.xz", + "mirrors": [ + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds", + "https://ziglang.org/builds/" + ], + "sha256": "b0fe8fd996b4974cd80fc0967460b13e3a6e3105c4bec19ea0566bd47d73ad81", + "zig_version": "0.16.0-dev.2722+f16eb18ce", + "platform": "x86_64-linux", + "translate_c": "@@+non_module_deps+translate-c//:translate-c" + } + }, + "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-macos": { + "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", + "attributes": { + "url": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.2722+f16eb18ce.tar.xz", + "mirrors": [ + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds", + "https://ziglang.org/builds/" + ], + "sha256": "8e09fb9e90c0566629dfa52bce433e886f4cf7e2cd9985a358d7b8c473c4e9ce", + "zig_version": "0.16.0-dev.2722+f16eb18ce", + "platform": "x86_64-macos", + "translate_c": "@@+non_module_deps+translate-c//:translate-c" + } + }, + "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-windows": { + "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", + "attributes": { + "url": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.2722+f16eb18ce.zip", + "mirrors": [ + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds", + "https://ziglang.org/builds/" + ], + "sha256": "0f92e7234ed1d9608444487ca8fb928bd38da79dea744acf6c7a8fa0c440da6a", + "zig_version": "0.16.0-dev.2722+f16eb18ce", + "platform": "x86_64-windows", + "translate_c": "@@+non_module_deps+translate-c//:translate-c" + } + }, + "zig_toolchains": { + "repoRuleId": "@@rules_zig+//zig/private/repo:toolchains_repo.bzl%toolchains_repo", + "attributes": { + "names": [ + "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-linux", + "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-macos", + "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-windows", + "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-linux", + "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-macos", + "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-windows" + ], + "labels": [ + "@zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-linux//:zig_toolchain", + "@zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-macos//:zig_toolchain", + "@zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-windows//:zig_toolchain", + "@zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-linux//:zig_toolchain", + "@zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-macos//:zig_toolchain", + "@zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-windows//:zig_toolchain" + ], + "zig_versions": [ + "0.16.0-dev.2722+f16eb18ce", + "0.16.0-dev.2722+f16eb18ce", + "0.16.0-dev.2722+f16eb18ce", + "0.16.0-dev.2722+f16eb18ce", + "0.16.0-dev.2722+f16eb18ce", + "0.16.0-dev.2722+f16eb18ce" + ], + "exec_lengths": [ + 2, + 2, + 2, + 2, + 2, + 2 + ], + "exec_constraints": [ + "@platforms//os:linux", + "@platforms//cpu:aarch64", + "@platforms//os:macos", + "@platforms//cpu:aarch64", + "@platforms//os:windows", + "@platforms//cpu:aarch64", + "@platforms//os:linux", + "@platforms//cpu:x86_64", + "@platforms//os:macos", + "@platforms//cpu:x86_64", + "@platforms//os:windows", + "@platforms//cpu:x86_64" + ] + } + } + }, + "recordedRepoMappingEntries": [ + [ + "", + "translate-c", + "+non_module_deps+translate-c" + ], + [ + "rules_zig+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "rules_zig+", + "bazel_tools", + "bazel_tools" + ] + ] + } + }, + "@@yq.bzl+//yq:extensions.bzl%yq": { + "general": { + "bzlTransitiveDigest": "iWlctm1cJYZLESuQEBygCQc6adq+oIRMb4Dmk0rYxkA=", + "usagesDigest": "J3TR36exv3ePI9O2y+uqrzTDmT6s6ADkTsROB++nm6I=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "yq_darwin_amd64": { + "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", + "attributes": { + "platform": "darwin_amd64", + "version": "4.45.1" + } + }, + "yq_darwin_arm64": { + "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", + "attributes": { + "platform": "darwin_arm64", + "version": "4.45.1" + } + }, + "yq_linux_amd64": { + "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", + "attributes": { + "platform": "linux_amd64", + "version": "4.45.1" + } + }, + "yq_linux_arm64": { + "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", + "attributes": { + "platform": "linux_arm64", + "version": "4.45.1" + } + }, + "yq_linux_s390x": { + "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", + "attributes": { + "platform": "linux_s390x", + "version": "4.45.1" + } + }, + "yq_linux_riscv64": { + "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", + "attributes": { + "platform": "linux_riscv64", + "version": "4.45.1" + } + }, + "yq_linux_ppc64le": { + "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", + "attributes": { + "platform": "linux_ppc64le", + "version": "4.45.1" + } + }, + "yq_windows_amd64": { + "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", + "attributes": { + "platform": "windows_amd64", + "version": "4.45.1" + } + }, + "yq_toolchains": { + "repoRuleId": "@@yq.bzl+//yq/toolchain:toolchain.bzl%yq_toolchains_repo", + "attributes": { + "user_repository_name": "yq" + } + } + }, + "recordedRepoMappingEntries": [] + } } }, - "facts": {} + "facts": { + "@@rules_rs+//rs:extensions.bzl%crate": { + "addr2line_0.25.1": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"backtrace\",\"req\":\"^0.3.13\"},{\"features\":[\"wrap_help\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4.3.21\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"cpp_demangle\",\"optional\":true,\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.7.0\"},{\"default_features\":false,\"name\":\"fallible-iterator\",\"optional\":true,\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"findshlibs\",\"req\":\"^0.10\"},{\"default_features\":false,\"features\":[\"read\"],\"name\":\"gimli\",\"req\":\"^0.32.0\"},{\"kind\":\"dev\",\"name\":\"libtest-mimic\",\"req\":\"^0.8.1\"},{\"name\":\"memmap2\",\"optional\":true,\"req\":\"^0.9.4\"},{\"default_features\":false,\"features\":[\"read\",\"compression\"],\"name\":\"object\",\"optional\":true,\"req\":\"^0.37.0\"},{\"name\":\"rustc-demangle\",\"optional\":true,\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"typed-arena\",\"optional\":true,\"req\":\"^2\"}],\"features\":{\"all\":[\"bin\",\"wasm\"],\"bin\":[\"loader\",\"rustc-demangle\",\"cpp_demangle\",\"fallible-iterator\",\"smallvec\",\"dep:clap\"],\"cargo-all\":[],\"default\":[\"rustc-demangle\",\"cpp_demangle\",\"loader\",\"fallible-iterator\",\"smallvec\"],\"loader\":[\"std\",\"dep:object\",\"dep:memmap2\",\"dep:typed-arena\"],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"gimli/rustc-dep-of-std\"],\"std\":[\"gimli/std\"],\"wasm\":[\"object/wasm\"]}}", + "adler2_2.0.1": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"std\"],\"rustc-dep-of-std\":[\"core\"],\"std\":[]}}", + "ahash_0.8.12": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1.0\"},{\"name\":\"const-random\",\"optional\":true,\"req\":\"^0.1.17\"},{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.2\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.5\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2.1\"},{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"hashbrown\",\"req\":\"^0.14.3\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.2\"},{\"kind\":\"dev\",\"name\":\"no-panic\",\"req\":\"^0.1.10\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"once_cell\",\"req\":\"^1.18.0\",\"target\":\"cfg(not(all(target_arch = \\\"arm\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"pcg-mwc\",\"req\":\"^0.2.1\"},{\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"seahash\",\"req\":\"^4.0\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.117\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.59\"},{\"kind\":\"dev\",\"name\":\"smallvec\",\"req\":\"^1.13.1\"},{\"kind\":\"build\",\"name\":\"version_check\",\"req\":\"^0.9.4\"},{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"zerocopy\",\"req\":\"^0.8.24\"}],\"features\":{\"atomic-polyfill\":[\"dep:portable-atomic\",\"once_cell/critical-section\"],\"compile-time-rng\":[\"const-random\"],\"default\":[\"std\",\"runtime-rng\"],\"nightly-arm-aes\":[],\"no-rng\":[],\"runtime-rng\":[\"getrandom\"],\"std\":[]}}", + "aho-corasick_1.1.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.17\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.4.0\"}],\"features\":{\"default\":[\"std\",\"perf-literal\"],\"logging\":[\"dep:log\"],\"perf-literal\":[\"dep:memchr\"],\"std\":[\"memchr?/std\"]}}", + "aho-corasick_1.1.4": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.17\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.4.0\"}],\"features\":{\"default\":[\"std\",\"perf-literal\"],\"logging\":[\"dep:log\"],\"perf-literal\":[\"dep:memchr\"],\"std\":[\"memchr?/std\"]}}", + "anyhow_1.0.102": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.6\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0\"},{\"kind\":\"dev\",\"name\":\"thiserror\",\"req\":\"^2\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"}],\"features\":{\"backtrace\":[],\"default\":[\"std\"],\"std\":[]}}", + "async-stream-impl_0.3.6": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"full\",\"visit-mut\"],\"name\":\"syn\",\"req\":\"^2.0.2\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"}],\"features\":{}}", + "async-stream_0.3.6": "{\"dependencies\":[{\"name\":\"async-stream-impl\",\"req\":\"=0.3.6\"},{\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1\"}],\"features\":{}}", + "async-trait_0.1.89": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.30\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"full\",\"parsing\",\"printing\",\"proc-macro\",\"visit-mut\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1.40\"},{\"kind\":\"dev\",\"name\":\"tracing-attributes\",\"req\":\"^0.1.27\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{}}", + "atomic-waker_1.1.2": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"cargo_bench_support\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.5\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.7.0\"}],\"features\":{}}", + "autocfg_1.5.0": "{\"dependencies\":[],\"features\":{}}", + "axum-core_0.4.5": "{\"dependencies\":[{\"name\":\"async-trait\",\"req\":\"^0.1.67\"},{\"kind\":\"dev\",\"name\":\"axum\",\"req\":\"^0.7.2\"},{\"name\":\"bytes\",\"req\":\"^1.2\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"http\",\"req\":\"^1.0.0\"},{\"name\":\"http-body\",\"req\":\"^1.0.0\"},{\"name\":\"http-body-util\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^1.0.0\"},{\"name\":\"mime\",\"req\":\"^0.3.16\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"},{\"name\":\"rustversion\",\"req\":\"^1.0.9\"},{\"name\":\"sync_wrapper\",\"req\":\"^1.0.0\"},{\"features\":[\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.25.0\"},{\"features\":[\"limit\"],\"name\":\"tower-http\",\"optional\":true,\"req\":\"^0.6.0\"},{\"features\":[\"limit\"],\"kind\":\"dev\",\"name\":\"tower-http\",\"req\":\"^0.6.0\"},{\"name\":\"tower-layer\",\"req\":\"^0.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.37\"}],\"features\":{\"__private_docs\":[\"dep:tower-http\"],\"tracing\":[\"dep:tracing\"]}}", + "axum_0.7.9": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0\"},{\"name\":\"async-trait\",\"req\":\"^0.1.67\"},{\"name\":\"axum-core\",\"req\":\"^0.4.5\"},{\"name\":\"axum-macros\",\"optional\":true,\"req\":\"^0.4.2\"},{\"features\":[\"__private\"],\"kind\":\"dev\",\"name\":\"axum-macros\",\"req\":\"^0.4.1\"},{\"name\":\"base64\",\"optional\":true,\"req\":\"^0.22.1\"},{\"name\":\"bytes\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"http\",\"req\":\"^1.0.0\"},{\"name\":\"http-body\",\"req\":\"^1.0.0\"},{\"name\":\"http-body-util\",\"req\":\"^0.1.0\"},{\"name\":\"hyper\",\"optional\":true,\"req\":\"^1.1.0\"},{\"features\":[\"tokio\",\"server\",\"service\"],\"name\":\"hyper-util\",\"optional\":true,\"req\":\"^0.1.3\"},{\"name\":\"itoa\",\"req\":\"^1.0.5\"},{\"name\":\"matchit\",\"req\":\"^0.7\"},{\"name\":\"memchr\",\"req\":\"^2.4.1\"},{\"name\":\"mime\",\"req\":\"^0.3.16\"},{\"name\":\"multer\",\"optional\":true,\"req\":\"^3.0.0\"},{\"name\":\"percent-encoding\",\"req\":\"^2.1\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"json\",\"stream\",\"multipart\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.12\"},{\"name\":\"rustversion\",\"req\":\"^1.0.9\"},{\"name\":\"serde\",\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"features\":[\"raw_value\"],\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"raw_value\"],\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"serde_path_to_error\",\"optional\":true,\"req\":\"^0.1.8\"},{\"name\":\"serde_urlencoded\",\"optional\":true,\"req\":\"^0.7\"},{\"name\":\"sha1\",\"optional\":true,\"req\":\"^0.10\"},{\"name\":\"sync_wrapper\",\"req\":\"^1.0.0\"},{\"features\":[\"serde-human-readable\"],\"kind\":\"dev\",\"name\":\"time\",\"req\":\"^0.3\"},{\"features\":[\"time\"],\"name\":\"tokio\",\"optional\":true,\"package\":\"tokio\",\"req\":\"^1.25.0\"},{\"features\":[\"macros\",\"rt\",\"rt-multi-thread\",\"net\",\"test-util\"],\"kind\":\"dev\",\"name\":\"tokio\",\"package\":\"tokio\",\"req\":\"^1.25.0\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"name\":\"tokio-tungstenite\",\"optional\":true,\"req\":\"^0.24.0\"},{\"kind\":\"dev\",\"name\":\"tokio-tungstenite\",\"req\":\"^0.24.0\"},{\"default_features\":false,\"features\":[\"util\"],\"name\":\"tower\",\"req\":\"^0.5.1\"},{\"features\":[\"util\",\"timeout\",\"limit\",\"load-shed\",\"steer\",\"filter\"],\"kind\":\"dev\",\"name\":\"tower\",\"package\":\"tower\",\"req\":\"^0.5.1\"},{\"features\":[\"add-extension\",\"auth\",\"catch-panic\",\"compression-br\",\"compression-deflate\",\"compression-gzip\",\"cors\",\"decompression-br\",\"decompression-deflate\",\"decompression-gzip\",\"follow-redirect\",\"fs\",\"limit\",\"map-request-body\",\"map-response-body\",\"metrics\",\"normalize-path\",\"propagate-header\",\"redirect\",\"request-id\",\"sensitive-headers\",\"set-header\",\"set-status\",\"timeout\",\"trace\",\"util\",\"validate-request\"],\"name\":\"tower-http\",\"optional\":true,\"req\":\"^0.6.0\"},{\"features\":[\"add-extension\",\"auth\",\"catch-panic\",\"compression-br\",\"compression-deflate\",\"compression-gzip\",\"cors\",\"decompression-br\",\"decompression-deflate\",\"decompression-gzip\",\"follow-redirect\",\"fs\",\"limit\",\"map-request-body\",\"map-response-body\",\"metrics\",\"normalize-path\",\"propagate-header\",\"redirect\",\"request-id\",\"sensitive-headers\",\"set-header\",\"set-status\",\"timeout\",\"trace\",\"util\",\"validate-request\"],\"kind\":\"dev\",\"name\":\"tower-http\",\"req\":\"^0.6.0\"},{\"name\":\"tower-layer\",\"req\":\"^0.3.2\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1\"},{\"features\":[\"json\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"},{\"features\":[\"serde\",\"v4\"],\"kind\":\"dev\",\"name\":\"uuid\",\"req\":\"^1.0\"}],\"features\":{\"__private_docs\":[\"axum-core/__private_docs\",\"tower/full\",\"dep:tower-http\"],\"default\":[\"form\",\"http1\",\"json\",\"matched-path\",\"original-uri\",\"query\",\"tokio\",\"tower-log\",\"tracing\"],\"form\":[\"dep:serde_urlencoded\"],\"http1\":[\"dep:hyper\",\"hyper?/http1\",\"hyper-util?/http1\"],\"http2\":[\"dep:hyper\",\"hyper?/http2\",\"hyper-util?/http2\"],\"json\":[\"dep:serde_json\",\"dep:serde_path_to_error\"],\"macros\":[\"dep:axum-macros\"],\"matched-path\":[],\"multipart\":[\"dep:multer\"],\"original-uri\":[],\"query\":[\"dep:serde_urlencoded\"],\"tokio\":[\"dep:hyper-util\",\"dep:tokio\",\"tokio/net\",\"tokio/rt\",\"tower/make\",\"tokio/macros\"],\"tower-log\":[\"tower/log\"],\"tracing\":[\"dep:tracing\",\"axum-core/tracing\"],\"ws\":[\"dep:hyper\",\"tokio\",\"dep:tokio-tungstenite\",\"dep:sha1\",\"dep:base64\"]}}", + "backtrace_0.3.76": "{\"dependencies\":[{\"default_features\":false,\"name\":\"addr2line\",\"req\":\"^0.25.0\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"name\":\"cfg-if\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"cpp_demangle\",\"optional\":true,\"req\":\"^0.5.0\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.156\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"kind\":\"dev\",\"name\":\"libloading\",\"req\":\"^0.8\"},{\"default_features\":false,\"name\":\"miniz_oxide\",\"req\":\"^0.8\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"default_features\":false,\"features\":[\"read_core\",\"elf\",\"macho\",\"pe\",\"xcoff\",\"unaligned\",\"archive\"],\"name\":\"object\",\"req\":\"^0.37.0\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"name\":\"rustc-demangle\",\"req\":\"^0.1.24\"},{\"default_features\":false,\"name\":\"ruzstd\",\"optional\":true,\"req\":\"^0.8.1\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"windows-link\",\"req\":\"^0.2\",\"target\":\"cfg(any(windows, target_os = \\\"cygwin\\\"))\"}],\"features\":{\"coresymbolication\":[],\"dbghelp\":[],\"default\":[\"std\"],\"dl_iterate_phdr\":[],\"dladdr\":[],\"kernel32\":[],\"libunwind\":[],\"ruzstd\":[\"dep:ruzstd\"],\"serialize-serde\":[\"serde\"],\"std\":[],\"unix-backtrace\":[]}}", + "base64_0.13.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"=0.3.2\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.6.1\"},{\"kind\":\"dev\",\"name\":\"structopt\",\"req\":\"^0.3\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[]}}", + "base64_0.22.1": "{\"dependencies\":[{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"clap\",\"req\":\"^3.2.25\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rstest\",\"req\":\"^0.13.0\"},{\"kind\":\"dev\",\"name\":\"rstest_reuse\",\"req\":\"^0.6.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"strum\",\"req\":\"^0.25\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", + "bitflags_2.11.0": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"arbitrary\",\"req\":\"^1.0\"},{\"name\":\"bytemuck\",\"optional\":true,\"req\":\"^1.12\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"bytemuck\",\"req\":\"^1.12.2\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.228\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde_lib\",\"package\":\"serde\",\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.19\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.18\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"zerocopy\",\"req\":\"^0.8\"}],\"features\":{\"example_generated\":[],\"serde\":[\"serde_core\"],\"std\":[]}}", + "bitflags_2.9.1": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"arbitrary\",\"req\":\"^1.0\"},{\"name\":\"bytemuck\",\"optional\":true,\"req\":\"^1.12\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"bytemuck\",\"req\":\"^1.12.2\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.19\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.18\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"zerocopy\",\"req\":\"^0.8\"}],\"features\":{\"example_generated\":[],\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"],\"std\":[]}}", + "byteorder_1.5.0": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9.2\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"std\"],\"i128\":[],\"std\":[]}}", + "bytes_1.11.1": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"require-cas\"],\"name\":\"extra-platforms\",\"optional\":true,\"package\":\"portable-atomic\",\"req\":\"^1.3\"},{\"kind\":\"dev\",\"name\":\"loom\",\"req\":\"^0.7\",\"target\":\"cfg(loom)\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.60\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "castaway_0.2.4": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1\"},{\"name\":\"rustversion\",\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", + "cc_1.2.33": "{\"dependencies\":[{\"default_features\":false,\"name\":\"jobserver\",\"optional\":true,\"req\":\"^0.1.30\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.62\",\"target\":\"cfg(unix)\"},{\"name\":\"shlex\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{\"jobserver\":[],\"parallel\":[\"dep:libc\",\"dep:jobserver\"]}}", + "cfg-if_1.0.0": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"]}}", + "cfg-if_1.0.4": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"rustc-dep-of-std\":[\"core\"]}}", + "compact_str_0.9.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"borsh\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"castaway\",\"req\":\"^0.2.3\"},{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"cfg-if\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"diesel\",\"optional\":true,\"req\":\"^2\"},{\"name\":\"itoa\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"markup\",\"optional\":true,\"req\":\"^0.15\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"proptest\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"rkyv\",\"optional\":true,\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rkyv\",\"req\":\"^0.8.8\"},{\"name\":\"rustversion\",\"req\":\"^1\"},{\"name\":\"ryu\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"derive\",\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"features\":[\"union\"],\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"name\":\"sqlx\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"static_assertions\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"test-case\",\"req\":\"^3\"},{\"kind\":\"dev\",\"name\":\"test-strategy\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"borsh\":[\"dep:borsh\"],\"bytes\":[\"dep:bytes\"],\"default\":[\"std\"],\"diesel\":[\"dep:diesel\"],\"markup\":[\"dep:markup\"],\"proptest\":[\"dep:proptest\"],\"quickcheck\":[\"dep:quickcheck\"],\"rkyv\":[\"dep:rkyv\"],\"serde\":[\"dep:serde\"],\"smallvec\":[\"dep:smallvec\"],\"sqlx\":[\"dep:sqlx\",\"std\"],\"sqlx-mysql\":[\"sqlx\",\"sqlx/mysql\"],\"sqlx-postgres\":[\"sqlx\",\"sqlx/postgres\"],\"sqlx-sqlite\":[\"sqlx\",\"sqlx/sqlite\"],\"std\":[],\"zeroize\":[\"dep:zeroize\"]}}", + "crossbeam-deque_0.8.6": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-epoch\",\"req\":\"^0.9.17\"},{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"crossbeam-epoch/std\",\"crossbeam-utils/std\"]}}", + "crossbeam-epoch_0.9.18": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"name\":\"loom-crate\",\"optional\":true,\"package\":\"loom\",\"req\":\"^0.7.1\",\"target\":\"cfg(crossbeam_loom)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"loom\":[\"loom-crate\",\"crossbeam-utils/loom\"],\"nightly\":[\"crossbeam-utils/nightly\"],\"std\":[\"alloc\",\"crossbeam-utils/std\"]}}", + "crossbeam-utils_0.8.21": "{\"dependencies\":[{\"name\":\"loom\",\"optional\":true,\"req\":\"^0.7.1\",\"target\":\"cfg(crossbeam_loom)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"std\"],\"nightly\":[],\"std\":[]}}", + "darling_0.20.10": "{\"dependencies\":[{\"name\":\"darling_core\",\"req\":\"=0.20.10\"},{\"name\":\"darling_macro\",\"req\":\"=0.20.10\"},{\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.37\"},{\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.9\",\"target\":\"cfg(compiletests)\"},{\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.15\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.38\",\"target\":\"cfg(compiletests)\"}],\"features\":{\"default\":[\"suggestions\"],\"diagnostics\":[\"darling_core/diagnostics\"],\"suggestions\":[\"darling_core/suggestions\"]}}", + "darling_core_0.20.10": "{\"dependencies\":[{\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"name\":\"ident_case\",\"req\":\"^1.0.1\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.37\"},{\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"name\":\"strsim\",\"optional\":true,\"req\":\"^0.11.1\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"diagnostics\":[],\"suggestions\":[\"strsim\"]}}", + "darling_macro_0.20.10": "{\"dependencies\":[{\"name\":\"darling_core\",\"req\":\"=0.20.10\"},{\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{}}", + "dary_heap_0.3.7": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1\"}],\"features\":{\"extra\":[],\"unstable\":[],\"unstable_nightly\":[]}}", + "derive_builder_0.20.2": "{\"dependencies\":[{\"name\":\"derive_builder_macro\",\"req\":\"=0.20.2\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.4\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.38\",\"target\":\"cfg(compiletests)\"}],\"features\":{\"alloc\":[\"derive_builder_macro/alloc\"],\"clippy\":[\"derive_builder_macro/clippy\"],\"default\":[\"std\"],\"std\":[\"derive_builder_macro/lib_has_std\"]}}", + "derive_builder_core_0.20.2": "{\"dependencies\":[{\"name\":\"darling\",\"req\":\"^0.20.10\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.37\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"alloc\":[],\"clippy\":[],\"lib_has_std\":[]}}", + "derive_builder_macro_0.20.2": "{\"dependencies\":[{\"name\":\"derive_builder_core\",\"req\":\"=0.20.2\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"alloc\":[\"derive_builder_core/alloc\"],\"clippy\":[\"derive_builder_core/clippy\"],\"lib_has_std\":[\"derive_builder_core/lib_has_std\"]}}", + "either_1.13.0": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"use_std\"],\"use_std\":[]}}", + "either_1.15.0": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"alloc\",\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.95\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[],\"use_std\":[\"std\"]}}", + "equivalent_1.0.2": "{\"dependencies\":[],\"features\":{}}", + "errno_0.3.14": "{\"dependencies\":[{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(target_os=\\\"hermit\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(target_os=\\\"wasi\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(unix)\"},{\"features\":[\"Win32_Foundation\",\"Win32_System_Diagnostics_Debug\"],\"name\":\"windows-sys\",\"req\":\">=0.52, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"libc/std\"]}}", + "esaxx-rs_0.1.10": "{\"dependencies\":[{\"kind\":\"build\",\"name\":\"cc\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"}],\"features\":{\"cpp\":[\"cc\"],\"default\":[\"cpp\"]}}", + "fastrand_2.3.0": "{\"dependencies\":[{\"features\":[\"js\"],\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.2\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"getrandom\",\"req\":\"^0.2\"},{\"features\":[\"js\"],\"kind\":\"dev\",\"name\":\"getrandom\",\"req\":\"^0.2\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"wyhash\",\"req\":\"^0.5\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"js\":[\"std\",\"getrandom\"],\"std\":[\"alloc\"]}}", + "fixedbitset_0.4.2": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "fnv_1.0.7": "{\"dependencies\":[],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "foldhash_0.1.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"ahash\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"chrono\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"hashbrown\",\"req\":\"^0.14\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"uuid\",\"req\":\"^1.8\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "futures-channel_0.3.32": "{\"dependencies\":[{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-sink\",\"optional\":true,\"req\":\"^0.3.32\"}],\"features\":{\"alloc\":[\"futures-core/alloc\"],\"cfg-target-has-atomic\":[],\"default\":[\"std\"],\"sink\":[\"futures-sink\"],\"std\":[\"alloc\",\"futures-core/std\"],\"unstable\":[]}}", + "futures-core_0.3.32": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"require-cas\"],\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.3\"}],\"features\":{\"alloc\":[],\"cfg-target-has-atomic\":[],\"default\":[\"std\"],\"std\":[\"alloc\"],\"unstable\":[]}}", + "futures-sink_0.3.32": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", + "futures-task_0.3.32": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"cfg-target-has-atomic\":[],\"default\":[\"std\"],\"std\":[\"alloc\"],\"unstable\":[]}}", + "futures-util_0.3.32": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"std\"],\"name\":\"futures-channel\",\"optional\":true,\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3.32\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"futures-io\",\"optional\":true,\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-macro\",\"optional\":true,\"req\":\"=0.3.32\"},{\"default_features\":false,\"name\":\"futures-sink\",\"optional\":true,\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-task\",\"req\":\"^0.3.32\"},{\"name\":\"futures_01\",\"optional\":true,\"package\":\"futures\",\"req\":\"^0.1.25\"},{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.26\"},{\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.2\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.6\"},{\"default_features\":false,\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4.7\"},{\"name\":\"spin\",\"optional\":true,\"req\":\"^0.10.0\"},{\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^0.1.11\"},{\"name\":\"tokio-io\",\"optional\":true,\"req\":\"^0.1.9\"}],\"features\":{\"alloc\":[\"futures-core/alloc\",\"futures-task/alloc\",\"slab\"],\"async-await\":[],\"async-await-macro\":[\"async-await\",\"futures-macro\"],\"bilock\":[],\"cfg-target-has-atomic\":[],\"channel\":[\"std\",\"futures-channel\"],\"compat\":[\"std\",\"futures_01\",\"libc\"],\"default\":[\"std\",\"async-await\",\"async-await-macro\"],\"io\":[\"std\",\"futures-io\",\"memchr\"],\"io-compat\":[\"io\",\"compat\",\"tokio-io\",\"libc\"],\"portable-atomic\":[\"futures-core/portable-atomic\"],\"sink\":[\"futures-sink\"],\"std\":[\"alloc\",\"futures-core/std\",\"futures-task/std\",\"slab/std\"],\"unstable\":[\"futures-core/unstable\",\"futures-task/unstable\"],\"write-all-vectored\":[\"io\"]}}", + "getrandom_0.2.17": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(unix)\"},{\"default_features\":false,\"name\":\"wasi\",\"req\":\"^0.11\",\"target\":\"cfg(target_os = \\\"wasi\\\")\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.62\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.18\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"}],\"features\":{\"custom\":[],\"js\":[\"wasm-bindgen\",\"js-sys\"],\"linux_disable_fallback\":[],\"rdrand\":[],\"rustc-dep-of-std\":[\"compiler_builtins\",\"core\",\"libc/rustc-dep-of-std\",\"wasi/rustc-dep-of-std\"],\"std\":[],\"test-in-browser\":[]}}", + "getrandom_0.3.3": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3.77\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"), target_feature = \\\"atomics\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"netbsd\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"solaris\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"vxworks\\\")\"},{\"default_features\":false,\"name\":\"r-efi\",\"req\":\"^5.1\",\"target\":\"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\"},{\"default_features\":false,\"name\":\"wasi\",\"req\":\"^0.14\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.98\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"}],\"features\":{\"rustc-dep-of-std\":[\"dep:compiler_builtins\",\"dep:core\"],\"std\":[],\"wasm_js\":[\"dep:wasm-bindgen\",\"dep:js-sys\"]}}", + "getrandom_0.4.2": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3.77\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"), target_feature = \\\"atomics\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"netbsd\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"solaris\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"vxworks\\\")\"},{\"default_features\":false,\"name\":\"r-efi\",\"req\":\"^6\",\"target\":\"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\"},{\"name\":\"rand_core\",\"optional\":true,\"req\":\"^0.10.0\"},{\"default_features\":false,\"name\":\"wasip2\",\"req\":\"^1\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\"},{\"name\":\"wasip3\",\"req\":\"^0.4\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p3\\\"))\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.98\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"}],\"features\":{\"std\":[],\"sys_rng\":[\"dep:rand_core\"],\"wasm_js\":[\"dep:wasm-bindgen\",\"dep:js-sys\"]}}", + "gimli_0.32.3": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"fallible-iterator\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.0.0\"},{\"default_features\":false,\"name\":\"stable_deref_trait\",\"optional\":true,\"req\":\"^1.1.0\"},{\"kind\":\"dev\",\"name\":\"test-assembler\",\"req\":\"^0.1.3\"}],\"features\":{\"default\":[\"read-all\",\"write\"],\"endian-reader\":[\"read\",\"dep:stable_deref_trait\"],\"fallible-iterator\":[\"dep:fallible-iterator\"],\"read\":[\"read-core\"],\"read-all\":[\"read\",\"std\",\"fallible-iterator\",\"endian-reader\"],\"read-core\":[],\"rustc-dep-of-std\":[\"dep:core\",\"dep:alloc\"],\"std\":[\"fallible-iterator?/std\",\"stable_deref_trait?/std\"],\"write\":[\"dep:indexmap\"]}}", + "h2_0.4.6": "{\"dependencies\":[{\"name\":\"atomic-waker\",\"req\":\"^1.0.0\"},{\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.10\"},{\"name\":\"fnv\",\"req\":\"^1.0.5\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-sink\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.3\"},{\"name\":\"http\",\"req\":\"^1\"},{\"features\":[\"std\"],\"name\":\"indexmap\",\"req\":\"^2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.4\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.0\"},{\"name\":\"slab\",\"req\":\"^0.4.2\"},{\"features\":[\"io-util\"],\"name\":\"tokio\",\"req\":\"^1\"},{\"features\":[\"rt-multi-thread\",\"macros\",\"sync\",\"net\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tokio-rustls\",\"req\":\"^0.26\"},{\"features\":[\"codec\",\"io\"],\"name\":\"tokio-util\",\"req\":\"^0.7.1\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"req\":\"^0.1.35\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\"},{\"kind\":\"dev\",\"name\":\"webpki-roots\",\"req\":\"^0.26\"}],\"features\":{\"stream\":[],\"unstable\":[]}}", + "hashbrown_0.12.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"ahash\",\"optional\":true,\"req\":\"^0.7.0\"},{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"bumpalo\",\"optional\":true,\"req\":\"^3.5.0\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.3\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.25\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"ahash-compile-time-rng\":[\"ahash/compile-time-rng\"],\"default\":[\"ahash\",\"inline-more\"],\"inline-more\":[],\"nightly\":[],\"raw\":[],\"rustc-dep-of-std\":[\"nightly\",\"core\",\"compiler_builtins\",\"alloc\",\"rustc-internal-api\"],\"rustc-internal-api\":[]}}", + "hashbrown_0.15.5": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"allocator-api2\",\"optional\":true,\"req\":\"^0.2.9\"},{\"features\":[\"allocator-api2\"],\"kind\":\"dev\",\"name\":\"bumpalo\",\"req\":\"^3.13.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.1\"},{\"default_features\":false,\"name\":\"equivalent\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"default_features\":false,\"name\":\"foldhash\",\"optional\":true,\"req\":\"^0.1.2\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9.0\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.2\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.2\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.25\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"default-hasher\",\"inline-more\",\"allocator-api2\",\"equivalent\",\"raw-entry\"],\"default-hasher\":[\"dep:foldhash\"],\"inline-more\":[],\"nightly\":[\"bumpalo/allocator_api\"],\"raw-entry\":[],\"rustc-dep-of-std\":[\"nightly\",\"core\",\"alloc\",\"rustc-internal-api\"],\"rustc-internal-api\":[]}}", + "hashbrown_0.16.1": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"allocator-api2\",\"optional\":true,\"req\":\"^0.2.9\"},{\"features\":[\"allocator-api2\"],\"kind\":\"dev\",\"name\":\"bumpalo\",\"req\":\"^3.13.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"equivalent\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"default_features\":false,\"name\":\"foldhash\",\"optional\":true,\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.155\",\"target\":\"cfg(unix)\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9.0\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.9.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.2\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.221\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"default-hasher\",\"inline-more\",\"allocator-api2\",\"equivalent\",\"raw-entry\"],\"default-hasher\":[\"dep:foldhash\"],\"inline-more\":[],\"nightly\":[\"foldhash?/nightly\",\"bumpalo/allocator_api\"],\"raw-entry\":[],\"rustc-dep-of-std\":[\"nightly\",\"core\",\"alloc\",\"rustc-internal-api\"],\"rustc-internal-api\":[],\"serde\":[\"dep:serde_core\",\"dep:serde\"]}}", + "heck_0.5.0": "{\"dependencies\":[],\"features\":{}}", + "http-body-util_0.1.3": "{\"dependencies\":[{\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"http-body\",\"req\":\"^1\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"macros\",\"rt\",\"sync\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"}],\"features\":{\"channel\":[\"dep:tokio\"],\"default\":[],\"full\":[\"channel\"]}}", + "http-body_1.0.1": "{\"dependencies\":[{\"name\":\"bytes\",\"req\":\"^1\"},{\"name\":\"http\",\"req\":\"^1\"}],\"features\":{}}", + "http_1.4.0": "{\"dependencies\":[{\"name\":\"bytes\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"name\":\"itoa\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.0\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "httparse_1.10.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.5\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "httpdate_1.0.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"}],\"features\":{}}", + "hyper-timeout_0.5.2": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"name\":\"hyper\",\"req\":\"^1.1\"},{\"features\":[\"http1\"],\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^1.1\"},{\"kind\":\"dev\",\"name\":\"hyper-tls\",\"req\":\"^0.6\"},{\"features\":[\"client-legacy\",\"http1\"],\"name\":\"hyper-util\",\"req\":\"^0.1.10\"},{\"features\":[\"client-legacy\",\"http1\",\"server\",\"server-graceful\"],\"kind\":\"dev\",\"name\":\"hyper-util\",\"req\":\"^0.1.10\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"name\":\"tokio\",\"req\":\"^1.35\"},{\"features\":[\"io-std\",\"io-util\",\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.35\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"}],\"features\":{}}", + "hyper-util_0.1.20": "{\"dependencies\":[{\"name\":\"base64\",\"optional\":true,\"req\":\"^0.22\"},{\"name\":\"bytes\",\"req\":\"^1.7.1\"},{\"kind\":\"dev\",\"name\":\"bytes\",\"req\":\"^1\"},{\"name\":\"futures-channel\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.16\"},{\"default_features\":false,\"features\":[\"alloc\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3.16\"},{\"name\":\"http\",\"req\":\"^1.0\"},{\"name\":\"http-body\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1.0\"},{\"name\":\"hyper\",\"req\":\"^1.8.0\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^1.4.0\"},{\"name\":\"ipnet\",\"optional\":true,\"req\":\"^2.9\"},{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2\"},{\"name\":\"percent-encoding\",\"optional\":true,\"req\":\"^2.3\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.4\"},{\"kind\":\"dev\",\"name\":\"pnet_datalink\",\"req\":\"^0.35.0\",\"target\":\"cfg(any(target_os = \\\"linux\\\", target_os = \\\"macos\\\"))\"},{\"kind\":\"dev\",\"name\":\"pretty_env_logger\",\"req\":\"^0.5\"},{\"features\":[\"all\"],\"name\":\"socket2\",\"optional\":true,\"req\":\">=0.5.9, <0.7\"},{\"name\":\"system-configuration\",\"optional\":true,\"req\":\"^0.7\",\"target\":\"cfg(target_os = \\\"macos\\\")\"},{\"default_features\":false,\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"macros\",\"test-util\",\"signal\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"name\":\"tower-layer\",\"optional\":true,\"req\":\"^0.3\"},{\"name\":\"tower-service\",\"optional\":true,\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"tower-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"windows-registry\",\"optional\":true,\"req\":\">=0.3, <0.7\",\"target\":\"cfg(windows)\"}],\"features\":{\"__internal_happy_eyeballs_tests\":[],\"client\":[\"hyper/client\",\"tokio/net\",\"dep:tracing\",\"dep:futures-channel\",\"dep:tower-service\"],\"client-legacy\":[\"client\",\"dep:socket2\",\"tokio/sync\",\"dep:libc\",\"dep:futures-util\"],\"client-pool\":[\"client\",\"dep:futures-util\",\"dep:tower-layer\"],\"client-proxy\":[\"client\",\"dep:base64\",\"dep:ipnet\",\"dep:percent-encoding\"],\"client-proxy-system\":[\"dep:system-configuration\",\"dep:windows-registry\"],\"default\":[],\"full\":[\"client\",\"client-legacy\",\"client-pool\",\"client-proxy\",\"client-proxy-system\",\"server\",\"server-auto\",\"server-graceful\",\"service\",\"http1\",\"http2\",\"tokio\",\"tracing\"],\"http1\":[\"hyper/http1\"],\"http2\":[\"hyper/http2\"],\"server\":[\"hyper/server\"],\"server-auto\":[\"server\",\"http1\",\"http2\"],\"server-graceful\":[\"server\",\"tokio/sync\"],\"service\":[\"dep:tower-service\"],\"tokio\":[\"dep:tokio\",\"tokio/rt\",\"tokio/time\"],\"tracing\":[\"dep:tracing\"]}}", + "hyper_1.8.1": "{\"dependencies\":[{\"name\":\"atomic-waker\",\"optional\":true,\"req\":\"^1.1.2\"},{\"name\":\"bytes\",\"req\":\"^1.2\"},{\"kind\":\"dev\",\"name\":\"form_urlencoded\",\"req\":\"^1\"},{\"name\":\"futures-channel\",\"optional\":true,\"req\":\"^0.3\"},{\"features\":[\"sink\"],\"kind\":\"dev\",\"name\":\"futures-channel\",\"req\":\"^0.3\"},{\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3.31\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\",\"sink\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"h2\",\"optional\":true,\"req\":\"^0.4.2\"},{\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"http-body\",\"req\":\"^1\"},{\"name\":\"http-body-util\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"name\":\"httparse\",\"optional\":true,\"req\":\"^1.9\"},{\"name\":\"httpdate\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"itoa\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"pin-project-lite\",\"optional\":true,\"req\":\"^0.2.4\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.4\"},{\"name\":\"pin-utils\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"pretty_env_logger\",\"req\":\"^0.5\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"features\":[\"const_generics\",\"const_new\"],\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1.12\"},{\"kind\":\"dev\",\"name\":\"spmc\",\"req\":\"^0.3\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"req\":\"^1\"},{\"features\":[\"fs\",\"macros\",\"net\",\"io-std\",\"io-util\",\"rt\",\"rt-multi-thread\",\"sync\",\"time\",\"test-util\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"tokio-util\",\"req\":\"^0.7.10\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"want\",\"optional\":true,\"req\":\"^0.3\"}],\"features\":{\"capi\":[],\"client\":[\"dep:want\",\"dep:pin-project-lite\",\"dep:smallvec\"],\"default\":[],\"ffi\":[\"dep:http-body-util\",\"dep:futures-util\"],\"full\":[\"client\",\"http1\",\"http2\",\"server\"],\"http1\":[\"dep:atomic-waker\",\"dep:futures-channel\",\"dep:futures-core\",\"dep:httparse\",\"dep:itoa\",\"dep:pin-utils\"],\"http2\":[\"dep:futures-channel\",\"dep:futures-core\",\"dep:h2\"],\"nightly\":[],\"server\":[\"dep:httpdate\",\"dep:pin-project-lite\",\"dep:smallvec\"],\"tracing\":[\"dep:tracing\"]}}", + "id-arena_2.3.0": "{\"dependencies\":[{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.0.3\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "ident_case_1.0.1": "{\"dependencies\":[],\"features\":{}}", + "indexmap_1.9.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"build\",\"name\":\"autocfg\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2.1\"},{\"default_features\":false,\"features\":[\"raw\"],\"name\":\"hashbrown\",\"req\":\"^0.12\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.3\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.4.1\"},{\"name\":\"rustc-rayon\",\"optional\":true,\"package\":\"rustc-rayon\",\"req\":\"^0.5\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0\"}],\"features\":{\"serde-1\":[\"serde\"],\"std\":[],\"test_debug\":[],\"test_low_transition_point\":[]}}", + "indexmap_2.13.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"borsh\",\"optional\":true,\"req\":\"^1.2\"},{\"default_features\":false,\"name\":\"equivalent\",\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"fastrand\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"hashbrown\",\"req\":\"^0.16.1\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.14\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.9\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"},{\"default_features\":false,\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.220\"},{\"default_features\":false,\"name\":\"sval\",\"optional\":true,\"req\":\"^2\"}],\"features\":{\"default\":[\"std\"],\"serde\":[\"dep:serde_core\",\"dep:serde\"],\"std\":[],\"test_debug\":[]}}", + "itertools_0.13.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"permutohedron\",\"req\":\"^0.2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"use_std\"],\"use_alloc\":[],\"use_std\":[\"use_alloc\",\"either/use_std\"]}}", + "itertools_0.14.0": "{\"dependencies\":[{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"permutohedron\",\"req\":\"^0.2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"use_std\"],\"use_alloc\":[],\"use_std\":[\"use_alloc\",\"either/use_std\"]}}", + "itoa_1.0.14": "{\"dependencies\":[{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"}],\"features\":{}}", + "itoa_1.0.17": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\",\"target\":\"cfg(not(miri))\"},{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"}],\"features\":{}}", + "leb128fmt_0.1.0": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[]}}", + "libc_0.2.169": "{\"dependencies\":[{\"name\":\"rustc-std-workspace-core\",\"optional\":true,\"req\":\"^1.0.0\"}],\"features\":{\"align\":[],\"const-extern-fn\":[],\"default\":[\"std\"],\"extra_traits\":[],\"rustc-dep-of-std\":[\"align\",\"rustc-std-workspace-core\"],\"std\":[],\"use_std\":[\"std\"]}}", + "libc_0.2.183": "{\"dependencies\":[{\"name\":\"rustc-std-workspace-core\",\"optional\":true,\"req\":\"^1.0.1\"}],\"features\":{\"align\":[],\"const-extern-fn\":[],\"default\":[\"std\"],\"extra_traits\":[],\"rustc-dep-of-std\":[\"align\",\"rustc-std-workspace-core\"],\"std\":[],\"use_std\":[\"std\"]}}", + "linux-raw-sys_0.12.1": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.100\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1.0\"}],\"features\":{\"auxvec\":[],\"bootparam\":[],\"btrfs\":[],\"default\":[\"std\",\"general\",\"errno\"],\"elf\":[],\"elf_uapi\":[],\"errno\":[],\"general\":[],\"if_arp\":[],\"if_ether\":[],\"if_packet\":[],\"if_tun\":[],\"image\":[],\"io_uring\":[],\"ioctl\":[],\"landlock\":[],\"loop_device\":[],\"mempolicy\":[],\"net\":[],\"netlink\":[],\"no_std\":[],\"prctl\":[],\"ptrace\":[],\"rustc-dep-of-std\":[\"core\",\"no_std\"],\"std\":[],\"system\":[],\"vm_sockets\":[],\"xdp\":[]}}", + "lock_api_0.4.14": "{\"dependencies\":[{\"name\":\"owning_ref\",\"optional\":true,\"req\":\"^0.4.1\"},{\"default_features\":false,\"name\":\"scopeguard\",\"req\":\"^1.1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.126\"}],\"features\":{\"arc_lock\":[],\"atomic_usize\":[],\"default\":[\"atomic_usize\"],\"nightly\":[]}}", + "log_0.4.22": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.63\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"sval\",\"optional\":true,\"req\":\"^2.1\"},{\"kind\":\"dev\",\"name\":\"sval\",\"req\":\"^2.1\"},{\"kind\":\"dev\",\"name\":\"sval_derive\",\"req\":\"^2.1\"},{\"default_features\":false,\"name\":\"sval_ref\",\"optional\":true,\"req\":\"^2.1\"},{\"default_features\":false,\"features\":[\"inline-i128\"],\"name\":\"value-bag\",\"optional\":true,\"req\":\"^1.7\"},{\"features\":[\"test\"],\"kind\":\"dev\",\"name\":\"value-bag\",\"req\":\"^1.7\"}],\"features\":{\"kv\":[],\"kv_serde\":[\"kv_std\",\"value-bag/serde\",\"serde\"],\"kv_std\":[\"std\",\"kv\",\"value-bag/error\"],\"kv_sval\":[\"kv\",\"value-bag/sval\",\"sval\",\"sval_ref\"],\"kv_unstable\":[\"kv\",\"value-bag\"],\"kv_unstable_serde\":[\"kv_serde\",\"kv_unstable_std\"],\"kv_unstable_std\":[\"kv_std\",\"kv_unstable\"],\"kv_unstable_sval\":[\"kv_sval\",\"kv_unstable\"],\"max_level_debug\":[],\"max_level_error\":[],\"max_level_info\":[],\"max_level_off\":[],\"max_level_trace\":[],\"max_level_warn\":[],\"release_max_level_debug\":[],\"release_max_level_error\":[],\"release_max_level_info\":[],\"release_max_level_off\":[],\"release_max_level_trace\":[],\"release_max_level_warn\":[],\"std\":[]}}", + "log_0.4.29": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.63\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"sval\",\"optional\":true,\"req\":\"^2.16\"},{\"kind\":\"dev\",\"name\":\"sval\",\"req\":\"^2.16\"},{\"kind\":\"dev\",\"name\":\"sval_derive\",\"req\":\"^2.16\"},{\"default_features\":false,\"name\":\"sval_ref\",\"optional\":true,\"req\":\"^2.16\"},{\"default_features\":false,\"features\":[\"inline-i128\"],\"name\":\"value-bag\",\"optional\":true,\"req\":\"^1.12\"},{\"features\":[\"test\"],\"kind\":\"dev\",\"name\":\"value-bag\",\"req\":\"^1.12\"}],\"features\":{\"kv\":[],\"kv_serde\":[\"kv_std\",\"value-bag/serde\",\"serde\"],\"kv_std\":[\"std\",\"kv\",\"value-bag/error\"],\"kv_sval\":[\"kv\",\"value-bag/sval\",\"sval\",\"sval_ref\"],\"kv_unstable\":[\"kv\",\"value-bag\"],\"kv_unstable_serde\":[\"kv_serde\",\"kv_unstable_std\"],\"kv_unstable_std\":[\"kv_std\",\"kv_unstable\"],\"kv_unstable_sval\":[\"kv_sval\",\"kv_unstable\"],\"max_level_debug\":[],\"max_level_error\":[],\"max_level_info\":[],\"max_level_off\":[],\"max_level_trace\":[],\"max_level_warn\":[],\"release_max_level_debug\":[],\"release_max_level_error\":[],\"release_max_level_info\":[],\"release_max_level_off\":[],\"release_max_level_trace\":[],\"release_max_level_warn\":[],\"serde\":[\"serde_core\"],\"std\":[]}}", + "macro_rules_attribute-proc_macro_0.2.0": "{\"dependencies\":[],\"features\":{\"verbose-expansions\":[]}}", + "macro_rules_attribute_0.2.0": "{\"dependencies\":[{\"name\":\"macro_rules_attribute-proc_macro\",\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.10.0\"},{\"name\":\"paste\",\"req\":\"^1.0.7\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.8\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.136\"}],\"features\":{\"better-docs\":[],\"default\":[],\"verbose-expansions\":[\"macro_rules_attribute-proc_macro/verbose-expansions\"]}}", + "matchit_0.7.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"actix-router\",\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.4\"},{\"kind\":\"dev\",\"name\":\"gonzales\",\"req\":\"^0.0.3-beta\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^0.14\"},{\"kind\":\"dev\",\"name\":\"path-tree\",\"req\":\"^0.2.2\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.5.4\"},{\"kind\":\"dev\",\"name\":\"route-recognizer\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"routefinder\",\"req\":\"^0.5.2\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"features\":[\"make\",\"util\"],\"kind\":\"dev\",\"name\":\"tower\",\"req\":\"^0.4\"}],\"features\":{\"__test_helpers\":[],\"default\":[]}}", + "memchr_2.7.4": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.20\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"libc\":[],\"logging\":[\"dep:log\"],\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"],\"std\":[\"alloc\"],\"use_std\":[\"std\"]}}", + "memchr_2.8.0": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.20\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"libc\":[],\"logging\":[\"dep:log\"],\"rustc-dep-of-std\":[\"core\"],\"std\":[\"alloc\"],\"use_std\":[\"std\"]}}", + "mime_0.3.17": "{\"dependencies\":[],\"features\":{}}", + "minimal-lexical_0.2.1": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"compact\":[],\"default\":[\"std\"],\"lint\":[],\"nightly\":[],\"std\":[]}}", + "miniz_oxide_0.8.9": "{\"dependencies\":[{\"default_features\":false,\"name\":\"adler2\",\"req\":\"^2.0\"},{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"simd-adler32\",\"optional\":true,\"req\":\"^0.3.3\"}],\"features\":{\"block-boundary\":[],\"default\":[\"with-alloc\"],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"adler2/rustc-dep-of-std\"],\"simd\":[\"simd-adler32\"],\"std\":[],\"with-alloc\":[]}}", + "mio_1.1.1": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"libc\",\"req\":\"^0.2.178\",\"target\":\"cfg(target_os = \\\"hermit\\\")\"},{\"name\":\"libc\",\"req\":\"^0.2.178\",\"target\":\"cfg(target_os = \\\"wasi\\\")\"},{\"name\":\"libc\",\"req\":\"^0.2.178\",\"target\":\"cfg(unix)\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.8\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9\"},{\"name\":\"wasi\",\"req\":\"^0.11.0\",\"target\":\"cfg(target_os = \\\"wasi\\\")\"},{\"features\":[\"Wdk_Foundation\",\"Wdk_Storage_FileSystem\",\"Wdk_System_IO\",\"Win32_Foundation\",\"Win32_Networking_WinSock\",\"Win32_Storage_FileSystem\",\"Win32_Security\",\"Win32_System_IO\",\"Win32_System_WindowsProgramming\"],\"name\":\"windows-sys\",\"req\":\"^0.61\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[\"log\"],\"net\":[],\"os-ext\":[\"os-poll\",\"windows-sys/Win32_System_Pipes\",\"windows-sys/Win32_Security\"],\"os-poll\":[]}}", + "monostate-impl_0.1.13": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"default_features\":false,\"features\":[\"parsing\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0.59\"}],\"features\":{}}", + "monostate_0.1.13": "{\"dependencies\":[{\"name\":\"monostate-impl\",\"req\":\"=0.1.13\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.166\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.99\"}],\"features\":{}}", + "multimap_0.10.1": "{\"dependencies\":[{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"serde_impl\"],\"serde_impl\":[\"serde\"]}}", + "nom_7.1.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2.3\"},{\"default_features\":false,\"name\":\"minimal-lexical\",\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.0.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"docsrs\":[],\"std\":[\"alloc\",\"memchr/std\",\"minimal-lexical/std\"]}}", + "object_0.37.3": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"crc32fast\",\"optional\":true,\"req\":\"^1.2\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"default-hasher\"],\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.15.0\"},{\"default_features\":false,\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.0\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2.4.1\"},{\"name\":\"ruzstd\",\"optional\":true,\"req\":\"^0.8.1\"},{\"default_features\":false,\"name\":\"wasmparser\",\"optional\":true,\"req\":\"^0.236.0\"}],\"features\":{\"all\":[\"read\",\"write\",\"build\",\"std\",\"compression\",\"wasm\"],\"archive\":[],\"build\":[\"build_core\",\"write_std\",\"elf\"],\"build_core\":[\"read_core\",\"write_core\"],\"cargo-all\":[],\"coff\":[],\"compression\":[\"dep:flate2\",\"dep:ruzstd\",\"std\"],\"default\":[\"read\",\"compression\"],\"doc\":[\"read_core\",\"write_std\",\"build_core\",\"std\",\"compression\",\"archive\",\"coff\",\"elf\",\"macho\",\"pe\",\"wasm\",\"xcoff\"],\"elf\":[],\"macho\":[],\"pe\":[\"coff\"],\"read\":[\"read_core\",\"archive\",\"coff\",\"elf\",\"macho\",\"pe\",\"xcoff\",\"unaligned\"],\"read_core\":[],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"memchr/rustc-dep-of-std\"],\"std\":[\"memchr/std\"],\"unaligned\":[],\"unstable\":[],\"unstable-all\":[\"all\",\"unstable\"],\"wasm\":[\"dep:wasmparser\"],\"write\":[\"write_std\",\"coff\",\"elf\",\"macho\",\"pe\",\"xcoff\"],\"write_core\":[\"dep:crc32fast\",\"dep:indexmap\",\"dep:hashbrown\"],\"write_std\":[\"write_core\",\"std\",\"indexmap?/std\",\"crc32fast?/std\"],\"xcoff\":[]}}", + "once_cell_1.20.2": "{\"dependencies\":[{\"name\":\"critical-section\",\"optional\":true,\"req\":\"^1.1.3\"},{\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"critical-section\",\"req\":\"^1.1.3\"},{\"default_features\":false,\"name\":\"parking_lot_core\",\"optional\":true,\"req\":\"^0.9.10\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.10.6\"}],\"features\":{\"alloc\":[\"race\"],\"atomic-polyfill\":[\"critical-section\"],\"critical-section\":[\"dep:critical-section\",\"portable-atomic\"],\"default\":[\"std\"],\"parking_lot\":[\"dep:parking_lot_core\"],\"portable-atomic\":[\"dep:portable-atomic\"],\"race\":[],\"std\":[\"alloc\"],\"unstable\":[]}}", + "once_cell_1.21.4": "{\"dependencies\":[{\"name\":\"critical-section\",\"optional\":true,\"req\":\"^1.1.3\"},{\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"critical-section\",\"req\":\"^1.1.3\"},{\"default_features\":false,\"name\":\"parking_lot_core\",\"optional\":true,\"req\":\"^0.9.10\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.10.6\"}],\"features\":{\"alloc\":[\"race\"],\"atomic-polyfill\":[\"critical-section\"],\"critical-section\":[\"dep:critical-section\",\"portable-atomic\"],\"default\":[\"std\"],\"parking_lot\":[\"dep:parking_lot_core\"],\"portable-atomic\":[\"dep:portable-atomic\"],\"race\":[],\"std\":[\"alloc\"],\"unstable\":[]}}", + "onig_6.5.1": "{\"dependencies\":[{\"name\":\"bitflags\",\"req\":\"^2.4.0\"},{\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(windows)\"},{\"name\":\"once_cell\",\"req\":\"^1.12\"},{\"default_features\":false,\"name\":\"onig_sys\",\"req\":\"^69.9.1\"}],\"features\":{\"default\":[\"generate\"],\"generate\":[\"onig_sys/generate\"],\"posix-api\":[\"onig_sys/posix-api\"],\"print-debug\":[\"onig_sys/print-debug\"],\"std-pattern\":[]}}", + "onig_sys_69.9.1": "{\"dependencies\":[{\"features\":[\"runtime\"],\"kind\":\"build\",\"name\":\"bindgen\",\"optional\":true,\"req\":\"^0.71\"},{\"kind\":\"build\",\"name\":\"cc\",\"req\":\"^1.0\"},{\"kind\":\"build\",\"name\":\"pkg-config\",\"req\":\"^0.3.16\"}],\"features\":{\"default\":[\"generate\"],\"generate\":[\"bindgen\"],\"posix-api\":[],\"print-debug\":[]}}", + "parking_lot_0.12.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.3.3\"},{\"name\":\"lock_api\",\"req\":\"^0.4.14\"},{\"name\":\"parking_lot_core\",\"req\":\"^0.9.12\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.3\"}],\"features\":{\"arc_lock\":[\"lock_api/arc_lock\"],\"deadlock_detection\":[\"parking_lot_core/deadlock_detection\"],\"default\":[],\"hardware-lock-elision\":[],\"nightly\":[\"parking_lot_core/nightly\",\"lock_api/nightly\"],\"owning_ref\":[\"lock_api/owning_ref\"],\"send_guard\":[],\"serde\":[\"lock_api/serde\"]}}", + "parking_lot_core_0.9.12": "{\"dependencies\":[{\"name\":\"backtrace\",\"optional\":true,\"req\":\"^0.3.60\"},{\"name\":\"cfg-if\",\"req\":\"^1.0.0\"},{\"name\":\"libc\",\"req\":\"^0.2.95\",\"target\":\"cfg(unix)\"},{\"name\":\"petgraph\",\"optional\":true,\"req\":\"^0.6.0\"},{\"name\":\"redox_syscall\",\"req\":\"^0.5\",\"target\":\"cfg(target_os = \\\"redox\\\")\"},{\"name\":\"smallvec\",\"req\":\"^1.6.1\"},{\"name\":\"windows-link\",\"req\":\"^0.2.0\",\"target\":\"cfg(windows)\"}],\"features\":{\"deadlock_detection\":[\"petgraph\",\"backtrace\"],\"nightly\":[]}}", + "paste_1.0.15": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"paste-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", + "percent-encoding_2.3.2": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", + "petgraph_0.6.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"ahash\",\"req\":\"^0.7.2\"},{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.3.3\"},{\"kind\":\"dev\",\"name\":\"defmac\",\"req\":\"^0.2.1\"},{\"default_features\":false,\"name\":\"fixedbitset\",\"req\":\"^0.4.0\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2.1\"},{\"name\":\"indexmap\",\"req\":\"^2.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.12.1\"},{\"kind\":\"dev\",\"name\":\"odds\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.5.5\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.5.3\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1.0\"}],\"features\":{\"all\":[\"unstable\",\"quickcheck\",\"matrix_graph\",\"stable_graph\",\"graphmap\",\"rayon\"],\"default\":[\"graphmap\",\"stable_graph\",\"matrix_graph\"],\"generate\":[],\"graphmap\":[],\"matrix_graph\":[],\"rayon\":[\"dep:rayon\",\"indexmap/rayon\"],\"serde-1\":[\"serde\",\"serde_derive\"],\"stable_graph\":[],\"unstable\":[\"generate\"]}}", + "pin-project-internal_1.1.11": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1.0.25\"},{\"default_features\":false,\"features\":[\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\",\"full\",\"visit-mut\"],\"name\":\"syn\",\"req\":\"^2.0.1\"}],\"features\":{}}", + "pin-project-lite_0.2.17": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1\"}],\"features\":{}}", + "pin-project_1.1.11": "{\"dependencies\":[{\"name\":\"pin-project-internal\",\"req\":\"=1.1.11\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1\"}],\"features\":{}}", + "pin-utils_0.1.0": "{\"dependencies\":[],\"features\":{}}", + "pkg-config_0.3.32": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1\"}],\"features\":{}}", + "ppv-lite86_0.2.20": "{\"dependencies\":[{\"features\":[\"simd\",\"derive\"],\"name\":\"zerocopy\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"std\"],\"no_simd\":[],\"simd\":[],\"std\":[]}}", + "ppv-lite86_0.2.21": "{\"dependencies\":[{\"features\":[\"simd\"],\"name\":\"zerocopy\",\"req\":\"^0.8.23\"}],\"features\":{\"default\":[\"std\"],\"no_simd\":[],\"simd\":[],\"std\":[]}}", + "prettyplease_0.2.37": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"default_features\":false,\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.105\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"extra-traits\",\"parsing\",\"printing\",\"visit-mut\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.105\"}],\"features\":{\"verbatim\":[\"syn/parsing\"]}}", + "proc-macro2_1.0.106": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4\"},{\"name\":\"unicode-ident\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"proc-macro\"],\"nightly\":[],\"proc-macro\":[],\"span-locations\":[]}}", + "proc-macro2_1.0.92": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4\"},{\"name\":\"unicode-ident\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"proc-macro\"],\"nightly\":[],\"proc-macro\":[],\"span-locations\":[]}}", + "prost-build_0.13.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"heck\",\"req\":\">=0.4, <=0.5\"},{\"default_features\":false,\"features\":[\"use_alloc\"],\"name\":\"itertools\",\"req\":\">=0.10, <=0.13\"},{\"name\":\"log\",\"req\":\"^0.4.4\"},{\"default_features\":false,\"name\":\"multimap\",\"req\":\">=0.8, <=0.10\"},{\"name\":\"once_cell\",\"req\":\"^1.17.1\"},{\"default_features\":false,\"name\":\"petgraph\",\"req\":\"^0.6\"},{\"name\":\"prettyplease\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"name\":\"prost\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-types\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"pulldown-cmark\",\"optional\":true,\"req\":\"^0.9.1\"},{\"name\":\"pulldown-cmark-to-cmark\",\"optional\":true,\"req\":\"^10.0.1\"},{\"default_features\":false,\"features\":[\"std\",\"unicode-bool\"],\"name\":\"regex\",\"req\":\"^1.8.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"optional\":true,\"req\":\"^2\"},{\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{\"cleanup-markdown\":[\"dep:pulldown-cmark\",\"dep:pulldown-cmark-to-cmark\"],\"default\":[\"format\"],\"format\":[\"dep:prettyplease\",\"dep:syn\"]}}", + "prost-derive_0.13.1": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.1\"},{\"name\":\"itertools\",\"req\":\">=0.10.1, <=0.13\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2\"}],\"features\":{}}", + "prost-types_0.13.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"prost-derive\"],\"name\":\"prost\",\"req\":\"^0.13.1\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"prost/std\"]}}", + "prost_0.13.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"name\":\"prost-derive\",\"optional\":true,\"req\":\"^0.13.1\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"derive\",\"std\"],\"derive\":[\"dep:prost-derive\"],\"no-recursion-limit\":[],\"prost-derive\":[\"derive\"],\"std\":[]}}", + "protoc-gen-prost_0.4.0": "{\"dependencies\":[{\"name\":\"once_cell\",\"req\":\"^1.10.0\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"prost\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-build\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-types\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"regex\",\"req\":\"^1.5.5\"}],\"features\":{}}", + "protoc-gen-tonic_0.4.1": "{\"dependencies\":[{\"name\":\"heck\",\"req\":\"^0.5.0\"},{\"name\":\"prettyplease\",\"req\":\"^0.2.9\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"prost\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-build\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-types\",\"req\":\"^0.13.1\"},{\"name\":\"protoc-gen-prost\",\"req\":\"^0.4.0\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"regex\",\"req\":\"^1.5.5\"},{\"features\":[\"parsing\",\"full\"],\"name\":\"syn\",\"req\":\"^2.0.22\"},{\"name\":\"tonic-build\",\"req\":\"^0.12.0\"}],\"features\":{}}", + "quote_1.0.37": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.66\"}],\"features\":{\"default\":[\"proc-macro\"],\"proc-macro\":[\"proc-macro2/proc-macro\"]}}", + "quote_1.0.45": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"}],\"features\":{\"default\":[\"proc-macro\"],\"proc-macro\":[\"proc-macro2/proc-macro\"]}}", + "r-efi_5.3.0": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"efiapi\":[],\"examples\":[\"native\"],\"native\":[],\"rustc-dep-of-std\":[\"core\"]}}", + "r-efi_6.0.0": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"native\":[],\"rustc-dep-of-std\":[\"core\"]}}", + "rand_0.8.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.2.1\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.22\",\"target\":\"cfg(unix)\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.4\"},{\"features\":[\"into_bits\"],\"name\":\"packed_simd\",\"optional\":true,\"package\":\"packed_simd_2\",\"req\":\"^0.3.7\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"rand_core\",\"req\":\"^0.6.0\"},{\"kind\":\"dev\",\"name\":\"rand_pcg\",\"req\":\"^0.3.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"}],\"features\":{\"alloc\":[\"rand_core/alloc\"],\"default\":[\"std\",\"std_rng\"],\"getrandom\":[\"rand_core/getrandom\"],\"min_const_gen\":[],\"nightly\":[],\"serde1\":[\"serde\",\"rand_core/serde1\"],\"simd_support\":[\"packed_simd\"],\"small_rng\":[],\"std\":[\"rand_core/std\",\"rand_chacha/std\",\"alloc\",\"getrandom\",\"libc\"],\"std_rng\":[\"rand_chacha\"]}}", + "rand_0.9.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.2.1\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.4\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"optional\":true,\"req\":\"^0.9.0\"},{\"default_features\":false,\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rand_pcg\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.7\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"std_rng\",\"os_rng\",\"small_rng\",\"thread_rng\"],\"log\":[\"dep:log\"],\"nightly\":[],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\",\"rand_core/serde\"],\"simd_support\":[],\"small_rng\":[],\"std\":[\"rand_core/std\",\"rand_chacha?/std\",\"alloc\"],\"std_rng\":[\"dep:rand_chacha\"],\"thread_rng\":[\"std\",\"std_rng\",\"os_rng\"],\"unbiased\":[]}}", + "rand_chacha_0.3.1": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"ppv-lite86\",\"req\":\"^0.2.8\"},{\"name\":\"rand_core\",\"req\":\"^0.6.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"serde1\":[\"serde\"],\"simd\":[],\"std\":[\"ppv-lite86/std\"]}}", + "rand_chacha_0.9.0": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"ppv-lite86\",\"req\":\"^0.2.14\"},{\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"features\":[\"os_rng\"],\"kind\":\"dev\",\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\"],\"std\":[\"ppv-lite86/std\",\"rand_core/std\"]}}", + "rand_core_0.6.4": "{\"dependencies\":[{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.2\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"serde1\":[\"serde\"],\"std\":[\"alloc\",\"getrandom\",\"getrandom/std\"]}}", + "rand_core_0.9.3": "{\"dependencies\":[{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.3.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"os_rng\":[\"dep:getrandom\"],\"serde\":[\"dep:serde\"],\"std\":[\"getrandom?/std\"]}}", + "rayon-cond_0.4.0": "{\"dependencies\":[{\"name\":\"either\",\"req\":\"^1.6\"},{\"name\":\"itertools\",\"req\":\"^0.14\"},{\"name\":\"rayon\",\"req\":\"^1.7\"}],\"features\":{}}", + "rayon-core_1.12.1": "{\"dependencies\":[{\"name\":\"crossbeam-deque\",\"req\":\"^0.8.1\"},{\"name\":\"crossbeam-utils\",\"req\":\"^0.8.0\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"scoped-tls\",\"req\":\"^1.0\"},{\"name\":\"wasm_sync\",\"optional\":true,\"req\":\"^0.1.0\"}],\"features\":{\"web_spin_lock\":[\"dep:wasm_sync\"]}}", + "rayon_1.10.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"name\":\"rayon-core\",\"req\":\"^1.12.1\"},{\"name\":\"wasm_sync\",\"optional\":true,\"req\":\"^0.1.0\"}],\"features\":{\"web_spin_lock\":[\"dep:wasm_sync\",\"rayon-core/web_spin_lock\"]}}", + "redox_syscall_0.5.18": "{\"dependencies\":[{\"name\":\"bitflags\",\"req\":\"^2.4\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"loom\",\"req\":\"^0.7\",\"target\":\"cfg(loom)\"}],\"features\":{\"default\":[\"userspace\"],\"rustc-dep-of-std\":[\"core\",\"bitflags/rustc-dep-of-std\"],\"std\":[],\"userspace\":[]}}", + "regex-automata_0.4.14": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"bstr\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.14\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"optional\":true,\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"syntax\",\"perf\",\"unicode\",\"meta\",\"nfa\",\"dfa\",\"hybrid\"],\"dfa\":[\"dfa-build\",\"dfa-search\",\"dfa-onepass\"],\"dfa-build\":[\"nfa-thompson\",\"dfa-search\"],\"dfa-onepass\":[\"nfa-thompson\"],\"dfa-search\":[],\"hybrid\":[\"alloc\",\"nfa-thompson\"],\"internal-instrument\":[\"internal-instrument-pikevm\"],\"internal-instrument-pikevm\":[\"logging\",\"std\"],\"logging\":[\"dep:log\",\"aho-corasick?/logging\",\"memchr?/logging\"],\"meta\":[\"syntax\",\"nfa-pikevm\"],\"nfa\":[\"nfa-thompson\",\"nfa-pikevm\",\"nfa-backtrack\"],\"nfa-backtrack\":[\"nfa-thompson\"],\"nfa-pikevm\":[\"nfa-thompson\"],\"nfa-thompson\":[\"alloc\"],\"perf\":[\"perf-inline\",\"perf-literal\"],\"perf-inline\":[],\"perf-literal\":[\"perf-literal-substring\",\"perf-literal-multisubstring\"],\"perf-literal-multisubstring\":[\"dep:aho-corasick\"],\"perf-literal-substring\":[\"aho-corasick?/perf-literal\",\"dep:memchr\"],\"std\":[\"regex-syntax?/std\",\"memchr?/std\",\"aho-corasick?/std\",\"alloc\"],\"syntax\":[\"dep:regex-syntax\",\"alloc\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"unicode-word-boundary\",\"regex-syntax?/unicode\"],\"unicode-age\":[\"regex-syntax?/unicode-age\"],\"unicode-bool\":[\"regex-syntax?/unicode-bool\"],\"unicode-case\":[\"regex-syntax?/unicode-case\"],\"unicode-gencat\":[\"regex-syntax?/unicode-gencat\"],\"unicode-perl\":[\"regex-syntax?/unicode-perl\"],\"unicode-script\":[\"regex-syntax?/unicode-script\"],\"unicode-segment\":[\"regex-syntax?/unicode-segment\"],\"unicode-word-boundary\":[]}}", + "regex-automata_0.4.9": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"bstr\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.14\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"optional\":true,\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"syntax\",\"perf\",\"unicode\",\"meta\",\"nfa\",\"dfa\",\"hybrid\"],\"dfa\":[\"dfa-build\",\"dfa-search\",\"dfa-onepass\"],\"dfa-build\":[\"nfa-thompson\",\"dfa-search\"],\"dfa-onepass\":[\"nfa-thompson\"],\"dfa-search\":[],\"hybrid\":[\"alloc\",\"nfa-thompson\"],\"internal-instrument\":[\"internal-instrument-pikevm\"],\"internal-instrument-pikevm\":[\"logging\",\"std\"],\"logging\":[\"dep:log\",\"aho-corasick?/logging\",\"memchr?/logging\"],\"meta\":[\"syntax\",\"nfa-pikevm\"],\"nfa\":[\"nfa-thompson\",\"nfa-pikevm\",\"nfa-backtrack\"],\"nfa-backtrack\":[\"nfa-thompson\"],\"nfa-pikevm\":[\"nfa-thompson\"],\"nfa-thompson\":[\"alloc\"],\"perf\":[\"perf-inline\",\"perf-literal\"],\"perf-inline\":[],\"perf-literal\":[\"perf-literal-substring\",\"perf-literal-multisubstring\"],\"perf-literal-multisubstring\":[\"std\",\"dep:aho-corasick\"],\"perf-literal-substring\":[\"aho-corasick?/perf-literal\",\"dep:memchr\"],\"std\":[\"regex-syntax?/std\",\"memchr?/std\",\"aho-corasick?/std\",\"alloc\"],\"syntax\":[\"dep:regex-syntax\",\"alloc\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"unicode-word-boundary\",\"regex-syntax?/unicode\"],\"unicode-age\":[\"regex-syntax?/unicode-age\"],\"unicode-bool\":[\"regex-syntax?/unicode-bool\"],\"unicode-case\":[\"regex-syntax?/unicode-case\"],\"unicode-gencat\":[\"regex-syntax?/unicode-gencat\"],\"unicode-perl\":[\"regex-syntax?/unicode-perl\"],\"unicode-script\":[\"regex-syntax?/unicode-script\"],\"unicode-segment\":[\"regex-syntax?/unicode-segment\"],\"unicode-word-boundary\":[]}}", + "regex-syntax_0.8.10": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.3.0\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"default\":[\"std\",\"unicode\"],\"std\":[],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\"],\"unicode-age\":[],\"unicode-bool\":[],\"unicode-case\":[],\"unicode-gencat\":[],\"unicode-perl\":[],\"unicode-script\":[],\"unicode-segment\":[]}}", + "regex-syntax_0.8.5": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.3.0\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"default\":[\"std\",\"unicode\"],\"std\":[],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\"],\"unicode-age\":[],\"unicode-bool\":[],\"unicode-case\":[],\"unicode-gencat\":[],\"unicode-perl\":[],\"unicode-script\":[],\"unicode-segment\":[]}}", + "regex_1.11.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.17.1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"features\":[\"alloc\",\"syntax\",\"meta\",\"nfa-pikevm\"],\"name\":\"regex-automata\",\"req\":\"^0.4.8\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"default\":[\"std\",\"perf\",\"unicode\",\"regex-syntax/default\"],\"logging\":[\"aho-corasick?/logging\",\"memchr?/logging\",\"regex-automata/logging\"],\"pattern\":[],\"perf\":[\"perf-cache\",\"perf-dfa\",\"perf-onepass\",\"perf-backtrack\",\"perf-inline\",\"perf-literal\"],\"perf-backtrack\":[\"regex-automata/nfa-backtrack\"],\"perf-cache\":[],\"perf-dfa\":[\"regex-automata/hybrid\"],\"perf-dfa-full\":[\"regex-automata/dfa-build\",\"regex-automata/dfa-search\"],\"perf-inline\":[\"regex-automata/perf-inline\"],\"perf-literal\":[\"dep:aho-corasick\",\"dep:memchr\",\"regex-automata/perf-literal\"],\"perf-onepass\":[\"regex-automata/dfa-onepass\"],\"std\":[\"aho-corasick?/std\",\"memchr?/std\",\"regex-automata/std\",\"regex-syntax/std\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"regex-automata/unicode\",\"regex-syntax/unicode\"],\"unicode-age\":[\"regex-automata/unicode-age\",\"regex-syntax/unicode-age\"],\"unicode-bool\":[\"regex-automata/unicode-bool\",\"regex-syntax/unicode-bool\"],\"unicode-case\":[\"regex-automata/unicode-case\",\"regex-syntax/unicode-case\"],\"unicode-gencat\":[\"regex-automata/unicode-gencat\",\"regex-syntax/unicode-gencat\"],\"unicode-perl\":[\"regex-automata/unicode-perl\",\"regex-automata/unicode-word-boundary\",\"regex-syntax/unicode-perl\"],\"unicode-script\":[\"regex-automata/unicode-script\",\"regex-syntax/unicode-script\"],\"unicode-segment\":[\"regex-automata/unicode-segment\",\"regex-syntax/unicode-segment\"],\"unstable\":[\"pattern\"],\"use_std\":[\"std\"]}}", + "regex_1.12.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"features\":[\"alloc\",\"syntax\",\"meta\",\"nfa-pikevm\"],\"name\":\"regex-automata\",\"req\":\"^0.4.12\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"default\":[\"std\",\"perf\",\"unicode\",\"regex-syntax/default\"],\"logging\":[\"aho-corasick?/logging\",\"memchr?/logging\",\"regex-automata/logging\"],\"pattern\":[],\"perf\":[\"perf-cache\",\"perf-dfa\",\"perf-onepass\",\"perf-backtrack\",\"perf-inline\",\"perf-literal\"],\"perf-backtrack\":[\"regex-automata/nfa-backtrack\"],\"perf-cache\":[],\"perf-dfa\":[\"regex-automata/hybrid\"],\"perf-dfa-full\":[\"regex-automata/dfa-build\",\"regex-automata/dfa-search\"],\"perf-inline\":[\"regex-automata/perf-inline\"],\"perf-literal\":[\"dep:aho-corasick\",\"dep:memchr\",\"regex-automata/perf-literal\"],\"perf-onepass\":[\"regex-automata/dfa-onepass\"],\"std\":[\"aho-corasick?/std\",\"memchr?/std\",\"regex-automata/std\",\"regex-syntax/std\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"regex-automata/unicode\",\"regex-syntax/unicode\"],\"unicode-age\":[\"regex-automata/unicode-age\",\"regex-syntax/unicode-age\"],\"unicode-bool\":[\"regex-automata/unicode-bool\",\"regex-syntax/unicode-bool\"],\"unicode-case\":[\"regex-automata/unicode-case\",\"regex-syntax/unicode-case\"],\"unicode-gencat\":[\"regex-automata/unicode-gencat\",\"regex-syntax/unicode-gencat\"],\"unicode-perl\":[\"regex-automata/unicode-perl\",\"regex-automata/unicode-word-boundary\",\"regex-syntax/unicode-perl\"],\"unicode-script\":[\"regex-automata/unicode-script\",\"regex-syntax/unicode-script\"],\"unicode-segment\":[\"regex-automata/unicode-segment\",\"regex-syntax/unicode-segment\"],\"unstable\":[\"pattern\"],\"use_std\":[\"std\"]}}", + "rustc-demangle_0.1.27": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"compiler_builtins\":[],\"rustc-dep-of-std\":[\"core\"],\"std\":[]}}", + "rustix_1.1.4": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bitflags\",\"req\":\"^2.4.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4\",\"target\":\"cfg(all(criterion, not(any(target_os = \\\"emscripten\\\", target_os = \\\"wasi\\\"))))\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.182\",\"target\":\"cfg(all(not(windows), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.182\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.171\"},{\"default_features\":false,\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(all(not(windows), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(windows)\"},{\"default_features\":false,\"name\":\"libc_errno\",\"optional\":true,\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\"},{\"default_features\":false,\"features\":[\"general\",\"ioctl\",\"no_std\"],\"name\":\"linux-raw-sys\",\"req\":\"^0.12\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"features\":[\"auxvec\",\"general\",\"errno\",\"ioctl\",\"no_std\",\"elf\"],\"name\":\"linux-raw-sys\",\"req\":\"^0.12\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"kind\":\"dev\",\"name\":\"memoffset\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.20.3\",\"target\":\"cfg(windows)\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"serial_test\",\"req\":\"^2.0.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.5.0\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\"],\"name\":\"windows-sys\",\"req\":\">=0.52, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"all-apis\":[\"event\",\"fs\",\"io_uring\",\"mm\",\"mount\",\"net\",\"param\",\"pipe\",\"process\",\"pty\",\"rand\",\"runtime\",\"shm\",\"stdio\",\"system\",\"termios\",\"thread\",\"time\"],\"alloc\":[],\"default\":[\"std\"],\"event\":[],\"fs\":[],\"io_uring\":[\"event\",\"fs\",\"net\",\"thread\",\"linux-raw-sys/io_uring\"],\"linux_4_11\":[],\"linux_5_1\":[\"linux_4_11\"],\"linux_5_11\":[\"linux_5_1\"],\"linux_latest\":[\"linux_5_11\"],\"mm\":[],\"mount\":[],\"net\":[\"linux-raw-sys/net\",\"linux-raw-sys/netlink\",\"linux-raw-sys/if_ether\",\"linux-raw-sys/xdp\"],\"param\":[],\"pipe\":[],\"process\":[\"linux-raw-sys/prctl\"],\"pty\":[\"fs\"],\"rand\":[],\"runtime\":[\"linux-raw-sys/prctl\"],\"rustc-dep-of-std\":[\"core\",\"rustc-std-workspace-alloc\",\"linux-raw-sys/rustc-dep-of-std\",\"bitflags/rustc-dep-of-std\"],\"shm\":[\"fs\"],\"std\":[\"bitflags/std\",\"alloc\",\"libc?/std\",\"libc_errno?/std\"],\"stdio\":[],\"system\":[\"linux-raw-sys/system\"],\"termios\":[],\"thread\":[\"linux-raw-sys/prctl\"],\"time\":[],\"try_close\":[],\"use-explicitly-provided-auxv\":[],\"use-libc\":[\"libc_errno\",\"libc\"],\"use-libc-auxv\":[]}}", + "rustversion_1.0.21": "{\"dependencies\":[{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", + "rustversion_1.0.22": "{\"dependencies\":[{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", + "ryu_1.0.18": "{\"dependencies\":[{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"num_cpus\",\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"}],\"features\":{\"small\":[]}}", + "scopeguard_1.2.0": "{\"dependencies\":[],\"features\":{\"default\":[\"use_std\"],\"use_std\":[]}}", + "semver_1.0.27": "{\"dependencies\":[{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"package\":\"serde_core\",\"req\":\"^1.0.220\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"}],\"features\":{\"default\":[\"std\"],\"serde\":[\"dep:serde\"],\"std\":[]}}", + "serde_1.0.216": "{\"dependencies\":[{\"name\":\"serde_derive\",\"req\":\"=1.0.216\",\"target\":\"cfg(any())\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"derive\":[\"serde_derive\"],\"rc\":[],\"std\":[],\"unstable\":[]}}", + "serde_1.0.228": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"result\"],\"name\":\"serde_core\",\"req\":\"=1.0.228\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[\"serde_core/alloc\"],\"default\":[\"std\"],\"derive\":[\"serde_derive\"],\"rc\":[\"serde_core/rc\"],\"std\":[\"serde_core/std\"],\"unstable\":[\"serde_core/unstable\"]}}", + "serde_core_1.0.228": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"name\":\"serde_derive\",\"req\":\"=1.0.228\",\"target\":\"cfg(any())\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"result\"],\"rc\":[],\"result\":[],\"std\":[],\"unstable\":[]}}", + "serde_derive_1.0.216": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"derive\",\"parsing\",\"printing\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0.81\"}],\"features\":{\"default\":[],\"deserialize_in_place\":[]}}", + "serde_derive_1.0.228": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"derive\",\"parsing\",\"printing\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0.81\"}],\"features\":{\"default\":[],\"deserialize_in_place\":[]}}", + "serde_json_1.0.134": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0.11\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.2.3\"},{\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2.0.2\"},{\"name\":\"itoa\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"name\":\"ryu\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"kind\":\"dev\",\"name\":\"serde_bytes\",\"req\":\"^0.11.10\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_stacker\",\"req\":\"^0.1.8\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{\"alloc\":[\"serde/alloc\"],\"arbitrary_precision\":[],\"default\":[\"std\"],\"float_roundtrip\":[],\"preserve_order\":[\"indexmap\",\"std\"],\"raw_value\":[],\"std\":[\"memchr/std\",\"serde/std\"],\"unbounded_depth\":[]}}", + "serde_json_1.0.149": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0.11\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.2.3\"},{\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2.0.2\"},{\"name\":\"itoa\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"kind\":\"dev\",\"name\":\"serde_bytes\",\"req\":\"^0.11.10\"},{\"default_features\":false,\"name\":\"serde_core\",\"req\":\"^1.0.220\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_stacker\",\"req\":\"^0.1.8\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"},{\"name\":\"zmij\",\"req\":\"^1.0\"}],\"features\":{\"alloc\":[\"serde_core/alloc\"],\"arbitrary_precision\":[],\"default\":[\"std\"],\"float_roundtrip\":[],\"preserve_order\":[\"indexmap\",\"std\"],\"raw_value\":[],\"std\":[\"memchr/std\",\"serde_core/std\"],\"unbounded_depth\":[]}}", + "shlex_1.3.0": "{\"dependencies\":[],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "signal-hook-registry_1.4.8": "{\"dependencies\":[{\"name\":\"errno\",\"req\":\">=0.2, <0.4\"},{\"name\":\"libc\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"signal-hook\",\"req\":\"~0.3\"}],\"features\":{}}", + "slab_0.4.12": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.95\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "smallvec_1.13.2": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.0.1\"},{\"kind\":\"dev\",\"name\":\"debugger_test\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"debugger_test_parser\",\"req\":\"^0.1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"const_generics\":[],\"const_new\":[\"const_generics\"],\"debugger_visualizer\":[],\"drain_filter\":[],\"drain_keep_rest\":[\"drain_filter\"],\"may_dangle\":[],\"specialization\":[],\"union\":[],\"write\":[]}}", + "smallvec_1.15.1": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"name\":\"bincode\",\"optional\":true,\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"bincode1\",\"package\":\"bincode\",\"req\":\"^1.0.1\"},{\"kind\":\"dev\",\"name\":\"debugger_test\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"debugger_test_parser\",\"req\":\"^0.1.0\"},{\"default_features\":false,\"name\":\"malloc_size_of\",\"optional\":true,\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"name\":\"unty\",\"optional\":true,\"req\":\"^0.0.4\"}],\"features\":{\"const_generics\":[],\"const_new\":[\"const_generics\"],\"debugger_visualizer\":[],\"drain_filter\":[],\"drain_keep_rest\":[\"drain_filter\"],\"impl_bincode\":[\"bincode\",\"unty\"],\"may_dangle\":[],\"specialization\":[],\"union\":[],\"write\":[]}}", + "socket2_0.5.10": "{\"dependencies\":[{\"name\":\"libc\",\"req\":\"^0.2.171\",\"target\":\"cfg(unix)\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\",\"Win32_System_IO\",\"Win32_System_Threading\",\"Win32_System_WindowsProgramming\"],\"name\":\"windows-sys\",\"req\":\"^0.52\",\"target\":\"cfg(windows)\"}],\"features\":{\"all\":[]}}", + "socket2_0.6.3": "{\"dependencies\":[{\"name\":\"libc\",\"req\":\"^0.2.172\",\"target\":\"cfg(any(unix, target_os = \\\"wasi\\\"))\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\",\"Win32_System_IO\",\"Win32_System_Threading\",\"Win32_System_WindowsProgramming\"],\"name\":\"windows-sys\",\"req\":\">=0.60, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"all\":[]}}", + "spm_precompiled_0.1.4": "{\"dependencies\":[{\"name\":\"base64\",\"req\":\"^0.13\"},{\"name\":\"nom\",\"req\":\"^7.1.1\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"unicode-segmentation\",\"req\":\"^1.9\"}],\"features\":{}}", + "static_assertions_1.1.0": "{\"dependencies\":[],\"features\":{\"nightly\":[]}}", + "strsim_0.11.1": "{\"dependencies\":[],\"features\":{}}", + "syn_2.0.117": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"insta\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.91\"},{\"default_features\":false,\"name\":\"quote\",\"optional\":true,\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1\"},{\"features\":[\"blocking\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.13\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"syn-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4.16\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"termcolor\",\"req\":\"^1\"},{\"name\":\"unicode-ident\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\",\"target\":\"cfg(not(miri))\"}],\"features\":{\"clone-impls\":[],\"default\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\"],\"derive\":[],\"extra-traits\":[],\"fold\":[],\"full\":[],\"parsing\":[],\"printing\":[\"dep:quote\"],\"proc-macro\":[\"proc-macro2/proc-macro\",\"quote?/proc-macro\"],\"test\":[\"syn-test-suite/all-features\"],\"visit\":[],\"visit-mut\":[]}}", + "syn_2.0.91": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"insta\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.91\"},{\"default_features\":false,\"name\":\"quote\",\"optional\":true,\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1\"},{\"features\":[\"blocking\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.12\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"syn-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4.16\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"termcolor\",\"req\":\"^1\"},{\"name\":\"unicode-ident\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\",\"target\":\"cfg(not(miri))\"}],\"features\":{\"clone-impls\":[],\"default\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\"],\"derive\":[],\"extra-traits\":[],\"fold\":[],\"full\":[],\"parsing\":[],\"printing\":[\"dep:quote\"],\"proc-macro\":[\"proc-macro2/proc-macro\",\"quote?/proc-macro\"],\"test\":[\"syn-test-suite/all-features\"],\"visit\":[],\"visit-mut\":[]}}", + "sync_wrapper_1.0.2": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"}],\"features\":{\"futures\":[\"futures-core\"]}}", + "tempfile_3.27.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"name\":\"fastrand\",\"req\":\"^2.1.1\"},{\"default_features\":false,\"name\":\"getrandom\",\"optional\":true,\"req\":\">=0.3.0, <0.5\",\"target\":\"cfg(any(unix, windows, target_os = \\\"wasi\\\"))\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"once_cell\",\"req\":\"^1.19.0\"},{\"features\":[\"fs\"],\"name\":\"rustix\",\"req\":\"^1.1.4\",\"target\":\"cfg(any(unix, target_os = \\\"wasi\\\"))\"},{\"features\":[\"Win32_Storage_FileSystem\",\"Win32_Foundation\"],\"name\":\"windows-sys\",\"req\":\">=0.52, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[\"getrandom\"],\"nightly\":[]}}", + "thiserror-impl_2.0.12": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"name\":\"syn\",\"req\":\"^2.0.87\"}],\"features\":{}}", + "thiserror_2.0.12": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.73\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"name\":\"thiserror-impl\",\"req\":\"=2.0.12\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "tokenizers_0.22.2": "{\"dependencies\":[{\"features\":[\"serde\"],\"name\":\"ahash\",\"req\":\"^0.8.11\"},{\"name\":\"aho-corasick\",\"req\":\"^1.1\"},{\"kind\":\"dev\",\"name\":\"assert_approx_eq\",\"req\":\"^1.1\"},{\"features\":[\"serde\"],\"name\":\"compact_str\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.6\"},{\"features\":[\"serde\"],\"name\":\"dary_heap\",\"req\":\"^0.3.6\"},{\"name\":\"derive_builder\",\"req\":\"^0.20\"},{\"default_features\":false,\"name\":\"esaxx-rs\",\"req\":\"^0.1.10\"},{\"name\":\"fancy-regex\",\"optional\":true,\"req\":\"^0.14\"},{\"name\":\"getrandom\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"ureq\"],\"name\":\"hf-hub\",\"optional\":true,\"req\":\"^0.4.1\"},{\"name\":\"indicatif\",\"optional\":true,\"req\":\"^0.18\"},{\"name\":\"itertools\",\"req\":\"^0.14\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"macro_rules_attribute\",\"req\":\"^0.2.0\"},{\"name\":\"monostate\",\"req\":\"^0.1.12\"},{\"default_features\":false,\"name\":\"onig\",\"optional\":true,\"req\":\"^6.5.1\"},{\"name\":\"paste\",\"req\":\"^1.0.14\"},{\"name\":\"rand\",\"req\":\"^0.9\"},{\"name\":\"rayon\",\"req\":\"^1.10\"},{\"name\":\"rayon-cond\",\"req\":\"^0.4\"},{\"name\":\"regex\",\"req\":\"^1.10\"},{\"name\":\"regex-syntax\",\"req\":\"^0.8\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"spm_precompiled\",\"req\":\"^0.1.3\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.10\"},{\"name\":\"thiserror\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3.18\"},{\"name\":\"unicode-normalization-alignments\",\"req\":\"^0.1\"},{\"name\":\"unicode-segmentation\",\"req\":\"^1.11\"},{\"name\":\"unicode_categories\",\"req\":\"^0.1\"}],\"features\":{\"default\":[\"progressbar\",\"onig\",\"esaxx_fast\"],\"esaxx_fast\":[\"esaxx-rs/cpp\"],\"http\":[\"hf-hub\"],\"progressbar\":[\"indicatif\"],\"rustls-tls\":[\"hf-hub?/rustls-tls\"],\"unstable_wasm\":[\"fancy-regex\",\"getrandom/wasm_js\"]}}", + "tokio-macros_2.4.0": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0.0\"}],\"features\":{}}", + "tokio-stream_0.1.15": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-stream\",\"req\":\"^0.3\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"name\":\"futures-core\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"parking_lot\",\"req\":\"^0.12.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.11\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"req\":\"^1.15.0\"},{\"features\":[\"full\",\"test-util\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.2.0\"},{\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7.0\"}],\"features\":{\"default\":[\"time\"],\"fs\":[\"tokio/fs\"],\"full\":[\"time\",\"net\",\"io-util\",\"fs\",\"sync\",\"signal\"],\"io-util\":[\"tokio/io-util\"],\"net\":[\"tokio/net\"],\"signal\":[\"tokio/signal\"],\"sync\":[\"tokio/sync\",\"tokio-util\"],\"time\":[\"tokio/time\"]}}", + "tokio-util_0.7.17": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-stream\",\"req\":\"^0.3.0\"},{\"name\":\"bytes\",\"req\":\"^1.5.0\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.0\"},{\"name\":\"futures-core\",\"req\":\"^0.3.0\"},{\"name\":\"futures-io\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"futures-sink\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"futures-test\",\"req\":\"^0.3.5\"},{\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.0\"},{\"default_features\":false,\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.15.0\"},{\"kind\":\"dev\",\"name\":\"parking_lot\",\"req\":\"^0.12.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.11\"},{\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4.4\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.1.0\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"req\":\"^1.28.0\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.29\"}],\"features\":{\"__docs_rs\":[\"futures-util\"],\"codec\":[],\"compat\":[\"futures-io\"],\"default\":[],\"full\":[\"codec\",\"compat\",\"io-util\",\"time\",\"net\",\"rt\",\"join-map\"],\"io\":[],\"io-util\":[\"io\",\"tokio/rt\",\"tokio/io-util\"],\"join-map\":[\"rt\",\"hashbrown\"],\"net\":[\"tokio/net\"],\"rt\":[\"tokio/rt\",\"tokio/sync\",\"futures-util\"],\"time\":[\"tokio/time\",\"slab\"]}}", + "tokio_1.39.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-stream\",\"req\":\"^0.3\"},{\"name\":\"backtrace\",\"req\":\"^0.3.58\",\"target\":\"cfg(tokio_taskdump)\"},{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1.0.0\"},{\"features\":[\"async-await\"],\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.0\"},{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.149\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.149\",\"target\":\"cfg(unix)\"},{\"features\":[\"futures\",\"checkpoint\"],\"kind\":\"dev\",\"name\":\"loom\",\"req\":\"^0.7\",\"target\":\"cfg(loom)\"},{\"default_features\":false,\"name\":\"mio\",\"optional\":true,\"req\":\"^1.0.1\"},{\"features\":[\"tokio\"],\"kind\":\"dev\",\"name\":\"mio-aio\",\"req\":\"^0.9.0\",\"target\":\"cfg(target_os = \\\"freebsd\\\")\"},{\"kind\":\"dev\",\"name\":\"mockall\",\"req\":\"^0.11.1\"},{\"default_features\":false,\"features\":[\"aio\",\"fs\",\"socket\"],\"kind\":\"dev\",\"name\":\"nix\",\"req\":\"^0.29.0\",\"target\":\"cfg(unix)\"},{\"name\":\"parking_lot\",\"optional\":true,\"req\":\"^0.12.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.11\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.0\",\"target\":\"cfg(not(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\")))\"},{\"name\":\"signal-hook-registry\",\"optional\":true,\"req\":\"^1.1.1\",\"target\":\"cfg(unix)\"},{\"features\":[\"all\"],\"name\":\"socket2\",\"optional\":true,\"req\":\"^0.5.5\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"kind\":\"dev\",\"name\":\"socket2\",\"req\":\"^0.5.5\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.1.0\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"name\":\"tokio-macros\",\"optional\":true,\"req\":\"~2.4.0\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.25\",\"target\":\"cfg(tokio_unstable)\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.0\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", not(target_os = \\\"wasi\\\")))\"},{\"name\":\"windows-sys\",\"optional\":true,\"req\":\"^0.52\",\"target\":\"cfg(windows)\"},{\"features\":[\"Win32_Foundation\",\"Win32_Security_Authorization\"],\"kind\":\"dev\",\"name\":\"windows-sys\",\"req\":\"^0.52\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[],\"fs\":[],\"full\":[\"fs\",\"io-util\",\"io-std\",\"macros\",\"net\",\"parking_lot\",\"process\",\"rt\",\"rt-multi-thread\",\"signal\",\"sync\",\"time\"],\"io-std\":[],\"io-util\":[\"bytes\"],\"macros\":[\"tokio-macros\"],\"net\":[\"libc\",\"mio/os-poll\",\"mio/os-ext\",\"mio/net\",\"socket2\",\"windows-sys/Win32_Foundation\",\"windows-sys/Win32_Security\",\"windows-sys/Win32_Storage_FileSystem\",\"windows-sys/Win32_System_Pipes\",\"windows-sys/Win32_System_SystemServices\"],\"process\":[\"bytes\",\"libc\",\"mio/os-poll\",\"mio/os-ext\",\"mio/net\",\"signal-hook-registry\",\"windows-sys/Win32_Foundation\",\"windows-sys/Win32_System_Threading\",\"windows-sys/Win32_System_WindowsProgramming\"],\"rt\":[],\"rt-multi-thread\":[\"rt\"],\"signal\":[\"libc\",\"mio/os-poll\",\"mio/net\",\"mio/os-ext\",\"signal-hook-registry\",\"windows-sys/Win32_Foundation\",\"windows-sys/Win32_System_Console\"],\"sync\":[],\"test-util\":[\"rt\",\"sync\",\"time\"],\"time\":[]}}", + "tonic-build_0.12.3": "{\"dependencies\":[{\"name\":\"prettyplease\",\"req\":\"^0.2\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"name\":\"prost-build\",\"optional\":true,\"req\":\"^0.13\"},{\"name\":\"prost-types\",\"optional\":true,\"req\":\"^0.13\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"name\":\"syn\",\"req\":\"^2.0\"}],\"features\":{\"cleanup-markdown\":[\"prost\",\"prost-build/cleanup-markdown\"],\"default\":[\"transport\",\"prost\"],\"prost\":[\"prost-build\",\"dep:prost-types\"],\"transport\":[]}}", + "tonic_0.12.1": "{\"dependencies\":[{\"name\":\"async-stream\",\"optional\":true,\"req\":\"^0.3\"},{\"name\":\"async-trait\",\"optional\":true,\"req\":\"^0.1.13\"},{\"default_features\":false,\"name\":\"axum\",\"optional\":true,\"req\":\"^0.7\"},{\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.5\"},{\"name\":\"bytes\",\"req\":\"^1.0\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"h2\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"http-body\",\"req\":\"^1\"},{\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"features\":[\"http1\",\"http2\"],\"name\":\"hyper\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"hyper-timeout\",\"optional\":true,\"req\":\"^0.5\"},{\"features\":[\"tokio\"],\"name\":\"hyper-util\",\"optional\":true,\"req\":\"^0.1.4\"},{\"name\":\"percent-encoding\",\"req\":\"^2.1\"},{\"name\":\"pin-project\",\"req\":\"^1.0.11\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"prost\",\"optional\":true,\"req\":\"^0.13\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"name\":\"rustls-native-certs\",\"optional\":true,\"req\":\"^0.7\"},{\"name\":\"rustls-pemfile\",\"optional\":true,\"req\":\"^2.0\"},{\"features\":[\"all\"],\"name\":\"socket2\",\"optional\":true,\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"rt\",\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"logging\",\"tls12\",\"ring\"],\"name\":\"tokio-rustls\",\"optional\":true,\"req\":\"^0.26\"},{\"default_features\":false,\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"tower\",\"optional\":true,\"req\":\"^0.4.7\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tower\",\"req\":\"^0.4.7\"},{\"name\":\"tower-layer\",\"req\":\"^0.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"name\":\"tracing\",\"req\":\"^0.1\"},{\"name\":\"webpki-roots\",\"optional\":true,\"req\":\"^0.26\"},{\"name\":\"zstd\",\"optional\":true,\"req\":\"^0.13.0\"}],\"features\":{\"channel\":[\"dep:hyper\",\"hyper?/client\",\"dep:hyper-util\",\"hyper-util?/client-legacy\",\"dep:tower\",\"tower?/balance\",\"tower?/buffer\",\"tower?/discover\",\"tower?/limit\",\"dep:tokio\",\"tokio?/time\",\"dep:hyper-timeout\"],\"codegen\":[\"dep:async-trait\"],\"default\":[\"transport\",\"codegen\",\"prost\"],\"gzip\":[\"dep:flate2\"],\"prost\":[\"dep:prost\"],\"router\":[\"dep:axum\",\"dep:tower\",\"tower?/util\"],\"server\":[\"router\",\"dep:async-stream\",\"dep:h2\",\"dep:hyper\",\"hyper?/server\",\"dep:hyper-util\",\"hyper-util?/service\",\"hyper-util?/server-auto\",\"dep:socket2\",\"dep:tokio\",\"tokio?/macros\",\"tokio?/net\",\"tokio?/time\",\"tokio-stream/net\",\"dep:tower\",\"tower?/util\",\"tower?/limit\"],\"tls\":[\"dep:rustls-pemfile\",\"dep:tokio-rustls\",\"dep:tokio\",\"tokio?/rt\",\"tokio?/macros\"],\"tls-roots\":[\"tls\",\"channel\",\"dep:rustls-native-certs\"],\"tls-webpki-roots\":[\"tls\",\"channel\",\"dep:webpki-roots\"],\"transport\":[\"server\",\"channel\"],\"zstd\":[\"dep:zstd\"]}}", + "tower-layer_0.3.3": "{\"dependencies\":[],\"features\":{}}", + "tower-service_0.3.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.22\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^0.2\"},{\"features\":[\"macros\",\"time\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.6.2\"},{\"kind\":\"dev\",\"name\":\"tower-layer\",\"req\":\"^0.3\"}],\"features\":{}}", + "tower_0.4.13": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"hdrhistogram\",\"optional\":true,\"req\":\"^7.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"hdrhistogram\",\"req\":\"^7.0\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^0.2\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^1.0.2\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4.0\"},{\"name\":\"pin-project\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"pin-project-lite\",\"optional\":true,\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"},{\"features\":[\"small_rng\"],\"name\":\"rand\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.6\"},{\"features\":[\"macros\",\"sync\",\"test-util\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.6.2\"},{\"name\":\"tokio-stream\",\"optional\":true,\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7.0\"},{\"name\":\"tower-layer\",\"req\":\"^0.3.1\"},{\"name\":\"tower-service\",\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"tower-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.2\"},{\"default_features\":false,\"features\":[\"fmt\",\"ansi\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"}],\"features\":{\"__common\":[\"futures-core\",\"pin-project-lite\"],\"balance\":[\"discover\",\"load\",\"ready-cache\",\"make\",\"rand\",\"slab\"],\"buffer\":[\"__common\",\"tokio/sync\",\"tokio/rt\",\"tokio-util\",\"tracing\"],\"default\":[\"log\"],\"discover\":[\"__common\"],\"filter\":[\"__common\",\"futures-util\"],\"full\":[\"balance\",\"buffer\",\"discover\",\"filter\",\"hedge\",\"limit\",\"load\",\"load-shed\",\"make\",\"ready-cache\",\"reconnect\",\"retry\",\"spawn-ready\",\"steer\",\"timeout\",\"util\"],\"hedge\":[\"util\",\"filter\",\"futures-util\",\"hdrhistogram\",\"tokio/time\",\"tracing\"],\"limit\":[\"__common\",\"tokio/time\",\"tokio/sync\",\"tokio-util\",\"tracing\"],\"load\":[\"__common\",\"tokio/time\",\"tracing\"],\"load-shed\":[\"__common\"],\"log\":[\"tracing/log\"],\"make\":[\"futures-util\",\"pin-project-lite\",\"tokio/io-std\"],\"ready-cache\":[\"futures-core\",\"futures-util\",\"indexmap\",\"tokio/sync\",\"tracing\",\"pin-project-lite\"],\"reconnect\":[\"make\",\"tokio/io-std\",\"tracing\"],\"retry\":[\"__common\",\"tokio/time\"],\"spawn-ready\":[\"__common\",\"futures-util\",\"tokio/sync\",\"tokio/rt\",\"util\",\"tracing\"],\"steer\":[],\"timeout\":[\"pin-project-lite\",\"tokio/time\"],\"util\":[\"__common\",\"futures-util\",\"pin-project\"]}}", + "tower_0.5.3": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.22\"},{\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3.22\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.22\"},{\"default_features\":false,\"features\":[\"async-await-macro\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3.22\"},{\"default_features\":false,\"name\":\"hdrhistogram\",\"optional\":true,\"req\":\"^7.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"hdrhistogram\",\"req\":\"^7.0\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.0.2\"},{\"name\":\"pin-project-lite\",\"optional\":true,\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9\"},{\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4.9\"},{\"name\":\"sync_wrapper\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.6.2\"},{\"features\":[\"macros\",\"sync\",\"test-util\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.6.2\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1.1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7.0\"},{\"name\":\"tower-layer\",\"req\":\"^0.3.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3.3\"},{\"kind\":\"dev\",\"name\":\"tower-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.2\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1.2\"},{\"default_features\":false,\"features\":[\"fmt\",\"ansi\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"}],\"features\":{\"balance\":[\"discover\",\"load\",\"ready-cache\",\"make\",\"slab\",\"util\"],\"buffer\":[\"tokio/sync\",\"tokio/rt\",\"tokio-util\",\"tracing\",\"pin-project-lite\"],\"discover\":[\"futures-core\",\"pin-project-lite\"],\"filter\":[\"futures-util\",\"pin-project-lite\"],\"full\":[\"balance\",\"buffer\",\"discover\",\"filter\",\"hedge\",\"limit\",\"load\",\"load-shed\",\"make\",\"ready-cache\",\"reconnect\",\"retry\",\"spawn-ready\",\"steer\",\"timeout\",\"util\"],\"hedge\":[\"util\",\"filter\",\"futures-util\",\"hdrhistogram\",\"tokio/time\",\"tracing\"],\"limit\":[\"tokio/time\",\"tokio/sync\",\"tokio-util\",\"tracing\",\"pin-project-lite\"],\"load\":[\"tokio/time\",\"tracing\",\"pin-project-lite\"],\"load-shed\":[\"pin-project-lite\"],\"log\":[\"tracing/log\"],\"make\":[\"pin-project-lite\",\"tokio\"],\"ready-cache\":[\"futures-core\",\"futures-util\",\"indexmap\",\"tokio/sync\",\"tracing\",\"pin-project-lite\"],\"reconnect\":[\"make\",\"tracing\"],\"retry\":[\"tokio/time\",\"util\"],\"spawn-ready\":[\"futures-util\",\"tokio/sync\",\"tokio/rt\",\"util\",\"tracing\"],\"steer\":[],\"timeout\":[\"pin-project-lite\",\"tokio/time\"],\"tokio-stream\":[],\"util\":[\"futures-core\",\"futures-util\",\"pin-project-lite\",\"sync_wrapper\"]}}", + "tracing-attributes_0.1.31": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-trait\",\"req\":\"^0.1.67\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1.0.20\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.9\"},{\"default_features\":false,\"features\":[\"full\",\"parsing\",\"printing\",\"visit-mut\",\"clone-impls\",\"extra-traits\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4.2\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1.35\"},{\"features\":[\"env-filter\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.64\"}],\"features\":{\"async-await\":[]}}", + "tracing-core_0.1.36": "{\"dependencies\":[{\"name\":\"once_cell\",\"optional\":true,\"req\":\"^1.13.0\"},{\"default_features\":false,\"name\":\"valuable\",\"optional\":true,\"req\":\"^0.1.0\",\"target\":\"cfg(tracing_unstable)\"}],\"features\":{\"default\":[\"std\",\"valuable?/std\"],\"std\":[\"once_cell\"]}}", + "tracing_0.1.44": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.6\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.21\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4.17\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.9\"},{\"name\":\"tracing-attributes\",\"optional\":true,\"req\":\"^0.1.31\"},{\"default_features\":false,\"name\":\"tracing-core\",\"req\":\"^0.1.36\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.38\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"}],\"features\":{\"async-await\":[],\"attributes\":[\"tracing-attributes\"],\"default\":[\"std\",\"attributes\"],\"log-always\":[\"log\"],\"max_level_debug\":[],\"max_level_error\":[],\"max_level_info\":[],\"max_level_off\":[],\"max_level_trace\":[],\"max_level_warn\":[],\"release_max_level_debug\":[],\"release_max_level_error\":[],\"release_max_level_info\":[],\"release_max_level_off\":[],\"release_max_level_trace\":[],\"release_max_level_warn\":[],\"std\":[\"tracing-core/std\"],\"valuable\":[\"tracing-core/valuable\"]}}", + "try-lock_0.2.5": "{\"dependencies\":[],\"features\":{}}", + "unicode-ident_1.0.14": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"fst\",\"req\":\"^0.4\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"roaring\",\"req\":\"^0.10\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"ucd-trie\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"unicode-xid\",\"req\":\"^0.2.6\"}],\"features\":{}}", + "unicode-ident_1.0.24": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"fst\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"roaring\",\"req\":\"^0.11\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"ucd-trie\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"unicode-xid\",\"req\":\"^0.2.6\"}],\"features\":{}}", + "unicode-normalization-alignments_0.1.12": "{\"dependencies\":[{\"name\":\"smallvec\",\"req\":\"^1.0\"}],\"features\":{}}", + "unicode-segmentation_1.12.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.7\"}],\"features\":{\"no_std\":[]}}", + "unicode-xid_0.2.6": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3\"}],\"features\":{\"bench\":[],\"default\":[],\"no_std\":[]}}", + "unicode_categories_0.1.1": "{\"dependencies\":[],\"features\":{}}", + "version_check_0.9.5": "{\"dependencies\":[],\"features\":{}}", + "want_0.3.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"tokio-executor\",\"req\":\"^0.2.0-alpha.2\"},{\"kind\":\"dev\",\"name\":\"tokio-sync\",\"req\":\"^0.2.0-alpha.2\"},{\"name\":\"try-lock\",\"req\":\"^0.2.4\"}],\"features\":{}}", + "wasi_0.11.1+wasi-snapshot-preview1": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"rustc-dep-of-std\":[\"core\",\"rustc-std-workspace-alloc\"],\"std\":[]}}", + "wasi_0.14.2+wasi-0.2.4": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"bitflags\"],\"name\":\"wit-bindgen-rt\",\"req\":\"^0.39.0\"}],\"features\":{\"default\":[\"std\"],\"rustc-dep-of-std\":[\"compiler_builtins\",\"core\",\"rustc-std-workspace-alloc\"],\"std\":[]}}", + "wasip2_1.0.2+wasi-0.2.9": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"wit-bindgen\",\"req\":\"^0.51.0\"}],\"features\":{\"bitflags\":[\"wit-bindgen/bitflags\"],\"default\":[\"std\",\"bitflags\"],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"wit-bindgen/rustc-dep-of-std\"],\"std\":[]}}", + "wasip3_0.4.0+wasi-0.3.0-rc-2026-01-06": "{\"dependencies\":[{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1.10.1\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.31\"},{\"name\":\"http\",\"optional\":true,\"req\":\"^1.3.1\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^1.3.1\"},{\"name\":\"http-body\",\"optional\":true,\"req\":\"^1.0.1\"},{\"name\":\"thiserror\",\"optional\":true,\"req\":\"^2.0.17\"},{\"default_features\":false,\"features\":[\"async\"],\"name\":\"wit-bindgen\",\"req\":\"^0.51.0\"},{\"default_features\":false,\"features\":[\"async-spawn\"],\"kind\":\"dev\",\"name\":\"wit-bindgen\",\"req\":\"^0.51.0\"}],\"features\":{\"http-compat\":[\"dep:bytes\",\"dep:http-body\",\"dep:http\",\"dep:thiserror\",\"wit-bindgen/async-spawn\"]}}", + "wasm-encoder_0.244.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"default_features\":false,\"name\":\"leb128fmt\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.2.0\"},{\"default_features\":false,\"features\":[\"simd\",\"simd\"],\"name\":\"wasmparser\",\"optional\":true,\"req\":\"^0.244.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"wasmprinter\",\"req\":\"^0.244.0\"}],\"features\":{\"component-model\":[\"wasmparser?/component-model\"],\"default\":[\"std\",\"component-model\"],\"std\":[\"wasmparser?/std\"]}}", + "wasm-metadata_0.244.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"name\":\"auditable-serde\",\"optional\":true,\"req\":\"^0.8.0\"},{\"features\":[\"derive\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4.0.0\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1.1.0\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"indexmap\",\"req\":\"^2.7.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"spdx\",\"optional\":true,\"req\":\"^0.10.1\"},{\"name\":\"url\",\"optional\":true,\"req\":\"^2.0.0\"},{\"default_features\":false,\"features\":[\"std\",\"component-model\"],\"name\":\"wasm-encoder\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"simd\",\"std\",\"component-model\",\"hash-collections\"],\"name\":\"wasmparser\",\"req\":\"^0.244.0\"}],\"features\":{\"default\":[\"oci\",\"serde\"],\"oci\":[\"dep:auditable-serde\",\"dep:flate2\",\"dep:url\",\"dep:spdx\",\"dep:serde_json\",\"serde\"],\"serde\":[\"dep:serde_derive\",\"dep:serde\"]}}", + "wasmparser_0.244.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"name\":\"bitflags\",\"req\":\"^2.4.1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5.1\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"default-hasher\"],\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.15.2\"},{\"default_features\":false,\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.7.0\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.13.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.3\"},{\"default_features\":false,\"name\":\"semver\",\"optional\":true,\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.166\"}],\"features\":{\"component-model\":[\"dep:semver\"],\"default\":[\"std\",\"validate\",\"serde\",\"features\",\"component-model\",\"hash-collections\",\"simd\"],\"features\":[],\"hash-collections\":[\"dep:hashbrown\",\"dep:indexmap\"],\"prefer-btree-collections\":[],\"serde\":[\"dep:serde\",\"indexmap?/serde\",\"hashbrown?/serde\"],\"simd\":[],\"std\":[\"indexmap?/std\"],\"validate\":[]}}", + "windows-link_0.2.1": "{\"dependencies\":[],\"features\":{}}", + "windows-sys_0.52.0": "{\"dependencies\":[{\"name\":\"windows-targets\",\"req\":\"^0.52.0\"}],\"features\":{\"Wdk\":[],\"Wdk_Foundation\":[\"Wdk\"],\"Wdk_Graphics\":[\"Wdk\"],\"Wdk_Graphics_Direct3D\":[\"Wdk_Graphics\"],\"Wdk_Storage\":[\"Wdk\"],\"Wdk_Storage_FileSystem\":[\"Wdk_Storage\"],\"Wdk_Storage_FileSystem_Minifilters\":[\"Wdk_Storage_FileSystem\"],\"Wdk_System\":[\"Wdk\"],\"Wdk_System_IO\":[\"Wdk_System\"],\"Wdk_System_OfflineRegistry\":[\"Wdk_System\"],\"Wdk_System_Registry\":[\"Wdk_System\"],\"Wdk_System_SystemInformation\":[\"Wdk_System\"],\"Wdk_System_SystemServices\":[\"Wdk_System\"],\"Wdk_System_Threading\":[\"Wdk_System\"],\"Win32\":[],\"Win32_Data\":[\"Win32\"],\"Win32_Data_HtmlHelp\":[\"Win32_Data\"],\"Win32_Data_RightsManagement\":[\"Win32_Data\"],\"Win32_Devices\":[\"Win32\"],\"Win32_Devices_AllJoyn\":[\"Win32_Devices\"],\"Win32_Devices_BiometricFramework\":[\"Win32_Devices\"],\"Win32_Devices_Bluetooth\":[\"Win32_Devices\"],\"Win32_Devices_Communication\":[\"Win32_Devices\"],\"Win32_Devices_DeviceAndDriverInstallation\":[\"Win32_Devices\"],\"Win32_Devices_DeviceQuery\":[\"Win32_Devices\"],\"Win32_Devices_Display\":[\"Win32_Devices\"],\"Win32_Devices_Enumeration\":[\"Win32_Devices\"],\"Win32_Devices_Enumeration_Pnp\":[\"Win32_Devices_Enumeration\"],\"Win32_Devices_Fax\":[\"Win32_Devices\"],\"Win32_Devices_HumanInterfaceDevice\":[\"Win32_Devices\"],\"Win32_Devices_PortableDevices\":[\"Win32_Devices\"],\"Win32_Devices_Properties\":[\"Win32_Devices\"],\"Win32_Devices_Pwm\":[\"Win32_Devices\"],\"Win32_Devices_Sensors\":[\"Win32_Devices\"],\"Win32_Devices_SerialCommunication\":[\"Win32_Devices\"],\"Win32_Devices_Tapi\":[\"Win32_Devices\"],\"Win32_Devices_Usb\":[\"Win32_Devices\"],\"Win32_Devices_WebServicesOnDevices\":[\"Win32_Devices\"],\"Win32_Foundation\":[\"Win32\"],\"Win32_Gaming\":[\"Win32\"],\"Win32_Globalization\":[\"Win32\"],\"Win32_Graphics\":[\"Win32\"],\"Win32_Graphics_Dwm\":[\"Win32_Graphics\"],\"Win32_Graphics_Gdi\":[\"Win32_Graphics\"],\"Win32_Graphics_GdiPlus\":[\"Win32_Graphics\"],\"Win32_Graphics_Hlsl\":[\"Win32_Graphics\"],\"Win32_Graphics_OpenGL\":[\"Win32_Graphics\"],\"Win32_Graphics_Printing\":[\"Win32_Graphics\"],\"Win32_Graphics_Printing_PrintTicket\":[\"Win32_Graphics_Printing\"],\"Win32_Management\":[\"Win32\"],\"Win32_Management_MobileDeviceManagementRegistration\":[\"Win32_Management\"],\"Win32_Media\":[\"Win32\"],\"Win32_Media_Audio\":[\"Win32_Media\"],\"Win32_Media_DxMediaObjects\":[\"Win32_Media\"],\"Win32_Media_KernelStreaming\":[\"Win32_Media\"],\"Win32_Media_Multimedia\":[\"Win32_Media\"],\"Win32_Media_Streaming\":[\"Win32_Media\"],\"Win32_Media_WindowsMediaFormat\":[\"Win32_Media\"],\"Win32_NetworkManagement\":[\"Win32\"],\"Win32_NetworkManagement_Dhcp\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Dns\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_InternetConnectionWizard\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_IpHelper\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Multicast\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Ndis\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetBios\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetManagement\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetShell\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetworkDiagnosticsFramework\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_P2P\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_QoS\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Rras\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Snmp\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WNet\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WebDav\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WiFi\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsConnectionManager\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsFilteringPlatform\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsFirewall\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsNetworkVirtualization\":[\"Win32_NetworkManagement\"],\"Win32_Networking\":[\"Win32\"],\"Win32_Networking_ActiveDirectory\":[\"Win32_Networking\"],\"Win32_Networking_Clustering\":[\"Win32_Networking\"],\"Win32_Networking_HttpServer\":[\"Win32_Networking\"],\"Win32_Networking_Ldap\":[\"Win32_Networking\"],\"Win32_Networking_WebSocket\":[\"Win32_Networking\"],\"Win32_Networking_WinHttp\":[\"Win32_Networking\"],\"Win32_Networking_WinInet\":[\"Win32_Networking\"],\"Win32_Networking_WinSock\":[\"Win32_Networking\"],\"Win32_Networking_WindowsWebServices\":[\"Win32_Networking\"],\"Win32_Security\":[\"Win32\"],\"Win32_Security_AppLocker\":[\"Win32_Security\"],\"Win32_Security_Authentication\":[\"Win32_Security\"],\"Win32_Security_Authentication_Identity\":[\"Win32_Security_Authentication\"],\"Win32_Security_Authorization\":[\"Win32_Security\"],\"Win32_Security_Credentials\":[\"Win32_Security\"],\"Win32_Security_Cryptography\":[\"Win32_Security\"],\"Win32_Security_Cryptography_Catalog\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_Certificates\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_Sip\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_UI\":[\"Win32_Security_Cryptography\"],\"Win32_Security_DiagnosticDataQuery\":[\"Win32_Security\"],\"Win32_Security_DirectoryServices\":[\"Win32_Security\"],\"Win32_Security_EnterpriseData\":[\"Win32_Security\"],\"Win32_Security_ExtensibleAuthenticationProtocol\":[\"Win32_Security\"],\"Win32_Security_Isolation\":[\"Win32_Security\"],\"Win32_Security_LicenseProtection\":[\"Win32_Security\"],\"Win32_Security_NetworkAccessProtection\":[\"Win32_Security\"],\"Win32_Security_WinTrust\":[\"Win32_Security\"],\"Win32_Security_WinWlx\":[\"Win32_Security\"],\"Win32_Storage\":[\"Win32\"],\"Win32_Storage_Cabinets\":[\"Win32_Storage\"],\"Win32_Storage_CloudFilters\":[\"Win32_Storage\"],\"Win32_Storage_Compression\":[\"Win32_Storage\"],\"Win32_Storage_DistributedFileSystem\":[\"Win32_Storage\"],\"Win32_Storage_FileHistory\":[\"Win32_Storage\"],\"Win32_Storage_FileSystem\":[\"Win32_Storage\"],\"Win32_Storage_Imapi\":[\"Win32_Storage\"],\"Win32_Storage_IndexServer\":[\"Win32_Storage\"],\"Win32_Storage_InstallableFileSystems\":[\"Win32_Storage\"],\"Win32_Storage_IscsiDisc\":[\"Win32_Storage\"],\"Win32_Storage_Jet\":[\"Win32_Storage\"],\"Win32_Storage_Nvme\":[\"Win32_Storage\"],\"Win32_Storage_OfflineFiles\":[\"Win32_Storage\"],\"Win32_Storage_OperationRecorder\":[\"Win32_Storage\"],\"Win32_Storage_Packaging\":[\"Win32_Storage\"],\"Win32_Storage_Packaging_Appx\":[\"Win32_Storage_Packaging\"],\"Win32_Storage_ProjectedFileSystem\":[\"Win32_Storage\"],\"Win32_Storage_StructuredStorage\":[\"Win32_Storage\"],\"Win32_Storage_Vhd\":[\"Win32_Storage\"],\"Win32_Storage_Xps\":[\"Win32_Storage\"],\"Win32_System\":[\"Win32\"],\"Win32_System_AddressBook\":[\"Win32_System\"],\"Win32_System_Antimalware\":[\"Win32_System\"],\"Win32_System_ApplicationInstallationAndServicing\":[\"Win32_System\"],\"Win32_System_ApplicationVerifier\":[\"Win32_System\"],\"Win32_System_ClrHosting\":[\"Win32_System\"],\"Win32_System_Com\":[\"Win32_System\"],\"Win32_System_Com_Marshal\":[\"Win32_System_Com\"],\"Win32_System_Com_StructuredStorage\":[\"Win32_System_Com\"],\"Win32_System_Com_Urlmon\":[\"Win32_System_Com\"],\"Win32_System_ComponentServices\":[\"Win32_System\"],\"Win32_System_Console\":[\"Win32_System\"],\"Win32_System_CorrelationVector\":[\"Win32_System\"],\"Win32_System_DataExchange\":[\"Win32_System\"],\"Win32_System_DeploymentServices\":[\"Win32_System\"],\"Win32_System_DeveloperLicensing\":[\"Win32_System\"],\"Win32_System_Diagnostics\":[\"Win32_System\"],\"Win32_System_Diagnostics_Ceip\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_Debug\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_Debug_Extensions\":[\"Win32_System_Diagnostics_Debug\"],\"Win32_System_Diagnostics_Etw\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_ProcessSnapshotting\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_ToolHelp\":[\"Win32_System_Diagnostics\"],\"Win32_System_DistributedTransactionCoordinator\":[\"Win32_System\"],\"Win32_System_Environment\":[\"Win32_System\"],\"Win32_System_ErrorReporting\":[\"Win32_System\"],\"Win32_System_EventCollector\":[\"Win32_System\"],\"Win32_System_EventLog\":[\"Win32_System\"],\"Win32_System_EventNotificationService\":[\"Win32_System\"],\"Win32_System_GroupPolicy\":[\"Win32_System\"],\"Win32_System_HostCompute\":[\"Win32_System\"],\"Win32_System_HostComputeNetwork\":[\"Win32_System\"],\"Win32_System_HostComputeSystem\":[\"Win32_System\"],\"Win32_System_Hypervisor\":[\"Win32_System\"],\"Win32_System_IO\":[\"Win32_System\"],\"Win32_System_Iis\":[\"Win32_System\"],\"Win32_System_Ioctl\":[\"Win32_System\"],\"Win32_System_JobObjects\":[\"Win32_System\"],\"Win32_System_Js\":[\"Win32_System\"],\"Win32_System_Kernel\":[\"Win32_System\"],\"Win32_System_LibraryLoader\":[\"Win32_System\"],\"Win32_System_Mailslots\":[\"Win32_System\"],\"Win32_System_Mapi\":[\"Win32_System\"],\"Win32_System_Memory\":[\"Win32_System\"],\"Win32_System_Memory_NonVolatile\":[\"Win32_System_Memory\"],\"Win32_System_MessageQueuing\":[\"Win32_System\"],\"Win32_System_MixedReality\":[\"Win32_System\"],\"Win32_System_Ole\":[\"Win32_System\"],\"Win32_System_PasswordManagement\":[\"Win32_System\"],\"Win32_System_Performance\":[\"Win32_System\"],\"Win32_System_Performance_HardwareCounterProfiling\":[\"Win32_System_Performance\"],\"Win32_System_Pipes\":[\"Win32_System\"],\"Win32_System_Power\":[\"Win32_System\"],\"Win32_System_ProcessStatus\":[\"Win32_System\"],\"Win32_System_Recovery\":[\"Win32_System\"],\"Win32_System_Registry\":[\"Win32_System\"],\"Win32_System_RemoteDesktop\":[\"Win32_System\"],\"Win32_System_RemoteManagement\":[\"Win32_System\"],\"Win32_System_RestartManager\":[\"Win32_System\"],\"Win32_System_Restore\":[\"Win32_System\"],\"Win32_System_Rpc\":[\"Win32_System\"],\"Win32_System_Search\":[\"Win32_System\"],\"Win32_System_Search_Common\":[\"Win32_System_Search\"],\"Win32_System_SecurityCenter\":[\"Win32_System\"],\"Win32_System_Services\":[\"Win32_System\"],\"Win32_System_SetupAndMigration\":[\"Win32_System\"],\"Win32_System_Shutdown\":[\"Win32_System\"],\"Win32_System_StationsAndDesktops\":[\"Win32_System\"],\"Win32_System_SubsystemForLinux\":[\"Win32_System\"],\"Win32_System_SystemInformation\":[\"Win32_System\"],\"Win32_System_SystemServices\":[\"Win32_System\"],\"Win32_System_Threading\":[\"Win32_System\"],\"Win32_System_Time\":[\"Win32_System\"],\"Win32_System_TpmBaseServices\":[\"Win32_System\"],\"Win32_System_UserAccessLogging\":[\"Win32_System\"],\"Win32_System_Variant\":[\"Win32_System\"],\"Win32_System_VirtualDosMachines\":[\"Win32_System\"],\"Win32_System_WindowsProgramming\":[\"Win32_System\"],\"Win32_System_Wmi\":[\"Win32_System\"],\"Win32_UI\":[\"Win32\"],\"Win32_UI_Accessibility\":[\"Win32_UI\"],\"Win32_UI_ColorSystem\":[\"Win32_UI\"],\"Win32_UI_Controls\":[\"Win32_UI\"],\"Win32_UI_Controls_Dialogs\":[\"Win32_UI_Controls\"],\"Win32_UI_HiDpi\":[\"Win32_UI\"],\"Win32_UI_Input\":[\"Win32_UI\"],\"Win32_UI_Input_Ime\":[\"Win32_UI_Input\"],\"Win32_UI_Input_KeyboardAndMouse\":[\"Win32_UI_Input\"],\"Win32_UI_Input_Pointer\":[\"Win32_UI_Input\"],\"Win32_UI_Input_Touch\":[\"Win32_UI_Input\"],\"Win32_UI_Input_XboxController\":[\"Win32_UI_Input\"],\"Win32_UI_InteractionContext\":[\"Win32_UI\"],\"Win32_UI_Magnification\":[\"Win32_UI\"],\"Win32_UI_Shell\":[\"Win32_UI\"],\"Win32_UI_Shell_PropertiesSystem\":[\"Win32_UI_Shell\"],\"Win32_UI_TabletPC\":[\"Win32_UI\"],\"Win32_UI_TextServices\":[\"Win32_UI\"],\"Win32_UI_WindowsAndMessaging\":[\"Win32_UI\"],\"Win32_Web\":[\"Win32\"],\"Win32_Web_InternetExplorer\":[\"Win32_Web\"],\"default\":[],\"docs\":[]}}", + "windows-sys_0.61.2": "{\"dependencies\":[{\"default_features\":false,\"name\":\"windows-link\",\"req\":\"^0.2.1\"}],\"features\":{\"Wdk\":[\"Win32_Foundation\"],\"Wdk_Devices\":[\"Wdk\"],\"Wdk_Devices_Bluetooth\":[\"Wdk_Devices\"],\"Wdk_Devices_HumanInterfaceDevice\":[\"Wdk_Devices\"],\"Wdk_Foundation\":[\"Wdk\"],\"Wdk_Graphics\":[\"Wdk\"],\"Wdk_Graphics_Direct3D\":[\"Wdk_Graphics\"],\"Wdk_NetworkManagement\":[\"Wdk\"],\"Wdk_NetworkManagement_Ndis\":[\"Wdk_NetworkManagement\"],\"Wdk_NetworkManagement_WindowsFilteringPlatform\":[\"Wdk_NetworkManagement\"],\"Wdk_Storage\":[\"Wdk\"],\"Wdk_Storage_FileSystem\":[\"Wdk_Storage\"],\"Wdk_Storage_FileSystem_Minifilters\":[\"Wdk_Storage_FileSystem\"],\"Wdk_System\":[\"Wdk\"],\"Wdk_System_IO\":[\"Wdk_System\"],\"Wdk_System_Memory\":[\"Wdk_System\"],\"Wdk_System_OfflineRegistry\":[\"Wdk_System\"],\"Wdk_System_Registry\":[\"Wdk_System\"],\"Wdk_System_SystemInformation\":[\"Wdk_System\"],\"Wdk_System_SystemServices\":[\"Wdk_System\"],\"Wdk_System_Threading\":[\"Wdk_System\"],\"Win32\":[\"Win32_Foundation\"],\"Win32_Data\":[\"Win32\"],\"Win32_Data_HtmlHelp\":[\"Win32_Data\"],\"Win32_Data_RightsManagement\":[\"Win32_Data\"],\"Win32_Devices\":[\"Win32\"],\"Win32_Devices_AllJoyn\":[\"Win32_Devices\"],\"Win32_Devices_Beep\":[\"Win32_Devices\"],\"Win32_Devices_BiometricFramework\":[\"Win32_Devices\"],\"Win32_Devices_Bluetooth\":[\"Win32_Devices\"],\"Win32_Devices_Cdrom\":[\"Win32_Devices\"],\"Win32_Devices_Communication\":[\"Win32_Devices\"],\"Win32_Devices_DeviceAndDriverInstallation\":[\"Win32_Devices\"],\"Win32_Devices_DeviceQuery\":[\"Win32_Devices\"],\"Win32_Devices_Display\":[\"Win32_Devices\"],\"Win32_Devices_Dvd\":[\"Win32_Devices\"],\"Win32_Devices_Enumeration\":[\"Win32_Devices\"],\"Win32_Devices_Enumeration_Pnp\":[\"Win32_Devices_Enumeration\"],\"Win32_Devices_Fax\":[\"Win32_Devices\"],\"Win32_Devices_HumanInterfaceDevice\":[\"Win32_Devices\"],\"Win32_Devices_Nfc\":[\"Win32_Devices\"],\"Win32_Devices_Nfp\":[\"Win32_Devices\"],\"Win32_Devices_PortableDevices\":[\"Win32_Devices\"],\"Win32_Devices_Properties\":[\"Win32_Devices\"],\"Win32_Devices_Pwm\":[\"Win32_Devices\"],\"Win32_Devices_Sensors\":[\"Win32_Devices\"],\"Win32_Devices_SerialCommunication\":[\"Win32_Devices\"],\"Win32_Devices_Tapi\":[\"Win32_Devices\"],\"Win32_Devices_Usb\":[\"Win32_Devices\"],\"Win32_Devices_WebServicesOnDevices\":[\"Win32_Devices\"],\"Win32_Foundation\":[\"Win32\"],\"Win32_Gaming\":[\"Win32\"],\"Win32_Globalization\":[\"Win32\"],\"Win32_Graphics\":[\"Win32\"],\"Win32_Graphics_Dwm\":[\"Win32_Graphics\"],\"Win32_Graphics_Gdi\":[\"Win32_Graphics\"],\"Win32_Graphics_GdiPlus\":[\"Win32_Graphics\"],\"Win32_Graphics_Hlsl\":[\"Win32_Graphics\"],\"Win32_Graphics_OpenGL\":[\"Win32_Graphics\"],\"Win32_Graphics_Printing\":[\"Win32_Graphics\"],\"Win32_Graphics_Printing_PrintTicket\":[\"Win32_Graphics_Printing\"],\"Win32_Management\":[\"Win32\"],\"Win32_Management_MobileDeviceManagementRegistration\":[\"Win32_Management\"],\"Win32_Media\":[\"Win32\"],\"Win32_Media_Audio\":[\"Win32_Media\"],\"Win32_Media_DxMediaObjects\":[\"Win32_Media\"],\"Win32_Media_KernelStreaming\":[\"Win32_Media\"],\"Win32_Media_Multimedia\":[\"Win32_Media\"],\"Win32_Media_Streaming\":[\"Win32_Media\"],\"Win32_Media_WindowsMediaFormat\":[\"Win32_Media\"],\"Win32_NetworkManagement\":[\"Win32\"],\"Win32_NetworkManagement_Dhcp\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Dns\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_InternetConnectionWizard\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_IpHelper\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Multicast\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Ndis\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetBios\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetManagement\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetShell\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetworkDiagnosticsFramework\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_P2P\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_QoS\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Rras\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Snmp\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WNet\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WebDav\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WiFi\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsConnectionManager\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsFilteringPlatform\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsFirewall\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsNetworkVirtualization\":[\"Win32_NetworkManagement\"],\"Win32_Networking\":[\"Win32\"],\"Win32_Networking_ActiveDirectory\":[\"Win32_Networking\"],\"Win32_Networking_Clustering\":[\"Win32_Networking\"],\"Win32_Networking_HttpServer\":[\"Win32_Networking\"],\"Win32_Networking_Ldap\":[\"Win32_Networking\"],\"Win32_Networking_WebSocket\":[\"Win32_Networking\"],\"Win32_Networking_WinHttp\":[\"Win32_Networking\"],\"Win32_Networking_WinInet\":[\"Win32_Networking\"],\"Win32_Networking_WinSock\":[\"Win32_Networking\"],\"Win32_Networking_WindowsWebServices\":[\"Win32_Networking\"],\"Win32_Security\":[\"Win32\"],\"Win32_Security_AppLocker\":[\"Win32_Security\"],\"Win32_Security_Authentication\":[\"Win32_Security\"],\"Win32_Security_Authentication_Identity\":[\"Win32_Security_Authentication\"],\"Win32_Security_Authorization\":[\"Win32_Security\"],\"Win32_Security_Credentials\":[\"Win32_Security\"],\"Win32_Security_Cryptography\":[\"Win32_Security\"],\"Win32_Security_Cryptography_Catalog\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_Certificates\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_Sip\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_UI\":[\"Win32_Security_Cryptography\"],\"Win32_Security_DiagnosticDataQuery\":[\"Win32_Security\"],\"Win32_Security_DirectoryServices\":[\"Win32_Security\"],\"Win32_Security_EnterpriseData\":[\"Win32_Security\"],\"Win32_Security_ExtensibleAuthenticationProtocol\":[\"Win32_Security\"],\"Win32_Security_Isolation\":[\"Win32_Security\"],\"Win32_Security_LicenseProtection\":[\"Win32_Security\"],\"Win32_Security_NetworkAccessProtection\":[\"Win32_Security\"],\"Win32_Security_WinTrust\":[\"Win32_Security\"],\"Win32_Security_WinWlx\":[\"Win32_Security\"],\"Win32_Storage\":[\"Win32\"],\"Win32_Storage_Cabinets\":[\"Win32_Storage\"],\"Win32_Storage_CloudFilters\":[\"Win32_Storage\"],\"Win32_Storage_Compression\":[\"Win32_Storage\"],\"Win32_Storage_DistributedFileSystem\":[\"Win32_Storage\"],\"Win32_Storage_FileHistory\":[\"Win32_Storage\"],\"Win32_Storage_FileSystem\":[\"Win32_Storage\"],\"Win32_Storage_Imapi\":[\"Win32_Storage\"],\"Win32_Storage_IndexServer\":[\"Win32_Storage\"],\"Win32_Storage_InstallableFileSystems\":[\"Win32_Storage\"],\"Win32_Storage_IscsiDisc\":[\"Win32_Storage\"],\"Win32_Storage_Jet\":[\"Win32_Storage\"],\"Win32_Storage_Nvme\":[\"Win32_Storage\"],\"Win32_Storage_OfflineFiles\":[\"Win32_Storage\"],\"Win32_Storage_OperationRecorder\":[\"Win32_Storage\"],\"Win32_Storage_Packaging\":[\"Win32_Storage\"],\"Win32_Storage_Packaging_Appx\":[\"Win32_Storage_Packaging\"],\"Win32_Storage_ProjectedFileSystem\":[\"Win32_Storage\"],\"Win32_Storage_StructuredStorage\":[\"Win32_Storage\"],\"Win32_Storage_Vhd\":[\"Win32_Storage\"],\"Win32_Storage_Xps\":[\"Win32_Storage\"],\"Win32_System\":[\"Win32\"],\"Win32_System_AddressBook\":[\"Win32_System\"],\"Win32_System_Antimalware\":[\"Win32_System\"],\"Win32_System_ApplicationInstallationAndServicing\":[\"Win32_System\"],\"Win32_System_ApplicationVerifier\":[\"Win32_System\"],\"Win32_System_ClrHosting\":[\"Win32_System\"],\"Win32_System_Com\":[\"Win32_System\"],\"Win32_System_Com_Marshal\":[\"Win32_System_Com\"],\"Win32_System_Com_StructuredStorage\":[\"Win32_System_Com\"],\"Win32_System_Com_Urlmon\":[\"Win32_System_Com\"],\"Win32_System_ComponentServices\":[\"Win32_System\"],\"Win32_System_Console\":[\"Win32_System\"],\"Win32_System_CorrelationVector\":[\"Win32_System\"],\"Win32_System_DataExchange\":[\"Win32_System\"],\"Win32_System_DeploymentServices\":[\"Win32_System\"],\"Win32_System_DeveloperLicensing\":[\"Win32_System\"],\"Win32_System_Diagnostics\":[\"Win32_System\"],\"Win32_System_Diagnostics_Ceip\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_Debug\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_Debug_Extensions\":[\"Win32_System_Diagnostics_Debug\"],\"Win32_System_Diagnostics_Etw\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_ProcessSnapshotting\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_ToolHelp\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_TraceLogging\":[\"Win32_System_Diagnostics\"],\"Win32_System_DistributedTransactionCoordinator\":[\"Win32_System\"],\"Win32_System_Environment\":[\"Win32_System\"],\"Win32_System_ErrorReporting\":[\"Win32_System\"],\"Win32_System_EventCollector\":[\"Win32_System\"],\"Win32_System_EventLog\":[\"Win32_System\"],\"Win32_System_EventNotificationService\":[\"Win32_System\"],\"Win32_System_GroupPolicy\":[\"Win32_System\"],\"Win32_System_HostCompute\":[\"Win32_System\"],\"Win32_System_HostComputeNetwork\":[\"Win32_System\"],\"Win32_System_HostComputeSystem\":[\"Win32_System\"],\"Win32_System_Hypervisor\":[\"Win32_System\"],\"Win32_System_IO\":[\"Win32_System\"],\"Win32_System_Iis\":[\"Win32_System\"],\"Win32_System_Ioctl\":[\"Win32_System\"],\"Win32_System_JobObjects\":[\"Win32_System\"],\"Win32_System_Js\":[\"Win32_System\"],\"Win32_System_Kernel\":[\"Win32_System\"],\"Win32_System_LibraryLoader\":[\"Win32_System\"],\"Win32_System_Mailslots\":[\"Win32_System\"],\"Win32_System_Mapi\":[\"Win32_System\"],\"Win32_System_Memory\":[\"Win32_System\"],\"Win32_System_Memory_NonVolatile\":[\"Win32_System_Memory\"],\"Win32_System_MessageQueuing\":[\"Win32_System\"],\"Win32_System_MixedReality\":[\"Win32_System\"],\"Win32_System_Ole\":[\"Win32_System\"],\"Win32_System_PasswordManagement\":[\"Win32_System\"],\"Win32_System_Performance\":[\"Win32_System\"],\"Win32_System_Performance_HardwareCounterProfiling\":[\"Win32_System_Performance\"],\"Win32_System_Pipes\":[\"Win32_System\"],\"Win32_System_Power\":[\"Win32_System\"],\"Win32_System_ProcessStatus\":[\"Win32_System\"],\"Win32_System_Recovery\":[\"Win32_System\"],\"Win32_System_Registry\":[\"Win32_System\"],\"Win32_System_RemoteDesktop\":[\"Win32_System\"],\"Win32_System_RemoteManagement\":[\"Win32_System\"],\"Win32_System_RestartManager\":[\"Win32_System\"],\"Win32_System_Restore\":[\"Win32_System\"],\"Win32_System_Rpc\":[\"Win32_System\"],\"Win32_System_Search\":[\"Win32_System\"],\"Win32_System_Search_Common\":[\"Win32_System_Search\"],\"Win32_System_SecurityCenter\":[\"Win32_System\"],\"Win32_System_Services\":[\"Win32_System\"],\"Win32_System_SetupAndMigration\":[\"Win32_System\"],\"Win32_System_Shutdown\":[\"Win32_System\"],\"Win32_System_StationsAndDesktops\":[\"Win32_System\"],\"Win32_System_SubsystemForLinux\":[\"Win32_System\"],\"Win32_System_SystemInformation\":[\"Win32_System\"],\"Win32_System_SystemServices\":[\"Win32_System\"],\"Win32_System_Threading\":[\"Win32_System\"],\"Win32_System_Time\":[\"Win32_System\"],\"Win32_System_TpmBaseServices\":[\"Win32_System\"],\"Win32_System_UserAccessLogging\":[\"Win32_System\"],\"Win32_System_Variant\":[\"Win32_System\"],\"Win32_System_VirtualDosMachines\":[\"Win32_System\"],\"Win32_System_WindowsProgramming\":[\"Win32_System\"],\"Win32_System_Wmi\":[\"Win32_System\"],\"Win32_UI\":[\"Win32\"],\"Win32_UI_Accessibility\":[\"Win32_UI\"],\"Win32_UI_ColorSystem\":[\"Win32_UI\"],\"Win32_UI_Controls\":[\"Win32_UI\"],\"Win32_UI_Controls_Dialogs\":[\"Win32_UI_Controls\"],\"Win32_UI_HiDpi\":[\"Win32_UI\"],\"Win32_UI_Input\":[\"Win32_UI\"],\"Win32_UI_Input_Ime\":[\"Win32_UI_Input\"],\"Win32_UI_Input_KeyboardAndMouse\":[\"Win32_UI_Input\"],\"Win32_UI_Input_Pointer\":[\"Win32_UI_Input\"],\"Win32_UI_Input_Touch\":[\"Win32_UI_Input\"],\"Win32_UI_Input_XboxController\":[\"Win32_UI_Input\"],\"Win32_UI_InteractionContext\":[\"Win32_UI\"],\"Win32_UI_Magnification\":[\"Win32_UI\"],\"Win32_UI_Shell\":[\"Win32_UI\"],\"Win32_UI_Shell_Common\":[\"Win32_UI_Shell\"],\"Win32_UI_Shell_PropertiesSystem\":[\"Win32_UI_Shell\"],\"Win32_UI_TabletPC\":[\"Win32_UI\"],\"Win32_UI_TextServices\":[\"Win32_UI\"],\"Win32_UI_WindowsAndMessaging\":[\"Win32_UI\"],\"Win32_Web\":[\"Win32\"],\"Win32_Web_InternetExplorer\":[\"Win32_Web\"],\"default\":[],\"docs\":[]}}", + "windows-targets_0.52.6": "{\"dependencies\":[{\"name\":\"windows_aarch64_gnullvm\",\"req\":\"^0.52.6\",\"target\":\"aarch64-pc-windows-gnullvm\"},{\"name\":\"windows_aarch64_msvc\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(target_arch = \\\"aarch64\\\", target_env = \\\"msvc\\\", not(windows_raw_dylib)))\"},{\"name\":\"windows_i686_gnu\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(target_arch = \\\"x86\\\", target_env = \\\"gnu\\\", not(target_abi = \\\"llvm\\\"), not(windows_raw_dylib)))\"},{\"name\":\"windows_i686_gnullvm\",\"req\":\"^0.52.6\",\"target\":\"i686-pc-windows-gnullvm\"},{\"name\":\"windows_i686_msvc\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(target_arch = \\\"x86\\\", target_env = \\\"msvc\\\", not(windows_raw_dylib)))\"},{\"name\":\"windows_x86_64_gnu\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(target_arch = \\\"x86_64\\\", target_env = \\\"gnu\\\", not(target_abi = \\\"llvm\\\"), not(windows_raw_dylib)))\"},{\"name\":\"windows_x86_64_gnullvm\",\"req\":\"^0.52.6\",\"target\":\"x86_64-pc-windows-gnullvm\"},{\"name\":\"windows_x86_64_msvc\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(any(target_arch = \\\"x86_64\\\", target_arch = \\\"arm64ec\\\"), target_env = \\\"msvc\\\", not(windows_raw_dylib)))\"}],\"features\":{}}", + "windows_aarch64_gnullvm_0.52.6": "{\"dependencies\":[],\"features\":{}}", + "windows_aarch64_msvc_0.52.6": "{\"dependencies\":[],\"features\":{}}", + "windows_i686_gnu_0.52.6": "{\"dependencies\":[],\"features\":{}}", + "windows_i686_gnullvm_0.52.6": "{\"dependencies\":[],\"features\":{}}", + "windows_i686_msvc_0.52.6": "{\"dependencies\":[],\"features\":{}}", + "windows_x86_64_gnu_0.52.6": "{\"dependencies\":[],\"features\":{}}", + "windows_x86_64_gnullvm_0.52.6": "{\"dependencies\":[],\"features\":{}}", + "windows_x86_64_msvc_0.52.6": "{\"dependencies\":[],\"features\":{}}", + "wit-bindgen-core_0.51.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.72\"},{\"features\":[\"derive\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4.3.19\"},{\"name\":\"heck\",\"req\":\"^0.5\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.218\"},{\"name\":\"wit-parser\",\"req\":\"^0.244.0\"}],\"features\":{\"clap\":[\"dep:clap\"],\"serde\":[\"dep:serde\"]}}", + "wit-bindgen-rt_0.39.0": "{\"dependencies\":[{\"name\":\"bitflags\",\"optional\":true,\"req\":\"^2.3.3\"},{\"name\":\"futures\",\"optional\":true,\"req\":\"^0.3.30\"},{\"name\":\"once_cell\",\"optional\":true,\"req\":\"^1.19.0\"}],\"features\":{\"async\":[\"dep:futures\",\"dep:once_cell\"]}}", + "wit-bindgen-rust-macro_0.51.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.72\"},{\"name\":\"prettyplease\",\"req\":\"^0.2.20\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"printing\"],\"name\":\"syn\",\"req\":\"^2.0.89\"},{\"name\":\"wit-bindgen-core\",\"req\":\"^0.51.0\"},{\"name\":\"wit-bindgen-rust\",\"req\":\"^0.51.0\"}],\"features\":{\"async\":[]}}", + "wit-bindgen-rust_0.51.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.72\"},{\"kind\":\"dev\",\"name\":\"bytes\",\"req\":\"^1\"},{\"features\":[\"derive\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4.3.19\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.31\"},{\"name\":\"heck\",\"req\":\"^0.5\"},{\"name\":\"indexmap\",\"req\":\"^2.0.0\"},{\"name\":\"prettyplease\",\"req\":\"^0.2.20\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.218\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"features\":[\"printing\"],\"name\":\"syn\",\"req\":\"^2.0.89\"},{\"default_features\":false,\"name\":\"wasm-metadata\",\"req\":\"^0.244.0\"},{\"name\":\"wit-bindgen-core\",\"req\":\"^0.51.0\"},{\"name\":\"wit-component\",\"req\":\"^0.244.0\"}],\"features\":{\"clap\":[\"dep:clap\",\"wit-bindgen-core/clap\"],\"serde\":[\"dep:serde\",\"wit-bindgen-core/serde\"]}}", + "wit-bindgen_0.51.0": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0\"},{\"name\":\"bitflags\",\"optional\":true,\"req\":\"^2.3.3\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"futures\",\"optional\":true,\"req\":\"^0.3.30\"},{\"name\":\"wit-bindgen-rust-macro\",\"optional\":true,\"req\":\"^0.51.0\"}],\"features\":{\"async\":[\"std\",\"wit-bindgen-rust-macro?/async\"],\"async-spawn\":[\"async\",\"dep:futures\"],\"bitflags\":[\"dep:bitflags\"],\"default\":[\"macros\",\"realloc\",\"async\",\"std\",\"bitflags\"],\"inter-task-wakeup\":[\"async\"],\"macros\":[\"dep:wit-bindgen-rust-macro\"],\"realloc\":[],\"rustc-dep-of-std\":[\"dep:core\",\"dep:alloc\"],\"std\":[]}}", + "wit-component_0.244.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"name\":\"bitflags\",\"req\":\"^2.3.3\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"kind\":\"dev\",\"name\":\"glob\",\"req\":\"^0.3.0\"},{\"default_features\":false,\"name\":\"indexmap\",\"req\":\"^2.7.0\"},{\"kind\":\"dev\",\"name\":\"libtest-mimic\",\"req\":\"^0.8.1\"},{\"name\":\"log\",\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.3.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"req\":\"^1.0.166\"},{\"name\":\"serde_derive\",\"req\":\"^1.0.166\"},{\"name\":\"serde_json\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"std\",\"wasmparser\"],\"name\":\"wasm-encoder\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"name\":\"wasm-metadata\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"oci\"],\"kind\":\"dev\",\"name\":\"wasm-metadata\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"simd\",\"std\",\"component-model\",\"simd\"],\"name\":\"wasmparser\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"simd\",\"std\",\"component-model\",\"features\"],\"kind\":\"dev\",\"name\":\"wasmparser\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"component-model\"],\"kind\":\"dev\",\"name\":\"wasmprinter\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"cranelift\",\"component-model\",\"runtime\",\"gc-drc\"],\"kind\":\"dev\",\"name\":\"wasmtime\",\"req\":\"^34.0.1\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"default_features\":false,\"name\":\"wast\",\"optional\":true,\"req\":\"^244.0.0\"},{\"default_features\":false,\"name\":\"wat\",\"optional\":true,\"req\":\"^1.244.0\"},{\"default_features\":false,\"features\":[\"component-model\"],\"kind\":\"dev\",\"name\":\"wat\",\"req\":\"^1.244.0\"},{\"features\":[\"decoding\",\"serde\"],\"name\":\"wit-parser\",\"req\":\"^0.244.0\"}],\"features\":{\"dummy-module\":[\"dep:wat\"],\"semver-check\":[\"dummy-module\"],\"wat\":[\"dep:wast\",\"dep:wat\"]}}", + "wit-parser_0.244.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"id-arena\",\"req\":\"^2\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"indexmap\",\"req\":\"^2.7.0\"},{\"kind\":\"dev\",\"name\":\"libtest-mimic\",\"req\":\"^0.8.1\"},{\"name\":\"log\",\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.3.0\"},{\"default_features\":false,\"name\":\"semver\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"name\":\"unicode-xid\",\"req\":\"^0.2.2\"},{\"default_features\":false,\"features\":[\"simd\",\"std\",\"validate\",\"component-model\",\"features\"],\"name\":\"wasmparser\",\"optional\":true,\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"component-model\"],\"name\":\"wat\",\"optional\":true,\"req\":\"^1.244.0\"}],\"features\":{\"decoding\":[\"dep:wasmparser\"],\"default\":[\"serde\",\"decoding\"],\"serde\":[\"dep:serde\",\"dep:serde_derive\",\"indexmap/serde\",\"serde_json\"],\"wat\":[\"decoding\",\"dep:wat\"]}}", + "zerocopy-derive_0.7.35": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.10\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"name\":\"syn\",\"req\":\"^2.0.31\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.85\"}],\"features\":{}}", + "zerocopy-derive_0.8.26": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"dissimilar\",\"req\":\"^1.0.9\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"=0.2.163\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"=1.9\"},{\"kind\":\"dev\",\"name\":\"prettyplease\",\"req\":\"=0.2.17\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.10\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.89\"}],\"features\":{}}", + "zerocopy-derive_0.8.42": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"dissimilar\",\"req\":\"^1.0.9\"},{\"kind\":\"dev\",\"name\":\"prettyplease\",\"req\":\"=0.2.17\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.40\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"features\":[\"visit\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.46\"}],\"features\":{}}", + "zerocopy_0.7.35": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"assert_matches\",\"req\":\"^1.5\"},{\"default_features\":false,\"name\":\"byteorder\",\"optional\":true,\"req\":\"^1.3\"},{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.85\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.7.35\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.7.35\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.7.35\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\"],\"alloc\":[],\"default\":[\"byteorder\"],\"derive\":[\"zerocopy-derive\"],\"simd\":[],\"simd-nightly\":[\"simd\"]}}", + "zerocopy_0.8.26": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"either\",\"req\":\"=1.13.0\"},{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.89\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.8.26\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\",\"std\"],\"alloc\":[],\"derive\":[\"zerocopy-derive\"],\"float-nightly\":[],\"simd\":[],\"simd-nightly\":[\"simd\"],\"std\":[\"alloc\"]}}", + "zerocopy_0.8.42": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.8.42\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.8.42\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.8.42\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\",\"std\"],\"alloc\":[],\"derive\":[\"zerocopy-derive\"],\"float-nightly\":[],\"simd\":[],\"simd-nightly\":[\"simd\"],\"std\":[\"alloc\"]}}", + "zmij_1.0.21": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\",\"target\":\"cfg(not(miri))\"},{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1.36\"},{\"kind\":\"dev\",\"name\":\"num-bigint\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"num-integer\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"num_cpus\",\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"opt-level\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"ryu\",\"req\":\"^1\"}],\"features\":{}}" + }, + "@@rules_rs+//rs/experimental/toolchains:module_extension.bzl%toolchains": { + "cargo-1.94.0-aarch64-apple-darwin.tar.xz": "7497ab873993cf38b806548c85bf818ffd71d1c3cf77b5bb4c3d3f1769588a08", + "cargo-1.94.0-aarch64-pc-windows-msvc.tar.xz": "0eacb6af29cacb850af4b390e698b2c8060f639a70e6b9f0575cdcc82b0822b2", + "cargo-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "d57ab7b23b52cba46e58c78327904d0c7db19892988677b0d59495c8af4bb790", + "cargo-1.94.0-x86_64-apple-darwin.tar.xz": "3beb7874010ebd757780284c0c9da47fc9f0b18091d7415d7b7231a8c19ec44f", + "cargo-1.94.0-x86_64-pc-windows-msvc.tar.xz": "6cf7113e1d5633721739a3bd6fee7148d6394bbab748fbede941210100374741", + "cargo-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "8e17624f3de39e079845bfb25ed15a042f4b50ceca78e37c56c4b9b15949b9f7", + "clippy-1.94.0-aarch64-apple-darwin.tar.xz": "42a683cbac0b81b96ad3b887cdade36a57ef3a131c203583ca7e3ce0be98f415", + "clippy-1.94.0-aarch64-pc-windows-msvc.tar.xz": "44468016e27457640c7ab62b7ceaddb44308e8ec77ee4344c1bd36113448c8eb", + "clippy-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "f7450c414e394450b53be602d37c374e2a2c5a2f51be2b54faf3d1e3272160b1", + "clippy-1.94.0-x86_64-apple-darwin.tar.xz": "1393cc688bc4d784a9d66619d2eec683a1ddb30b263f03ab615c0a2b18053126", + "clippy-1.94.0-x86_64-pc-windows-msvc.tar.xz": "e4dcc5a796857589e364064493abb3afb469e0e36e90155e7b0a2c5ce4bb7503", + "clippy-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "935529b0b9150084fa9cc1734c6598efe86b6e7d76f31ad3f8d4bb79ebf6031a", + "rust-std-1.94.0-aarch64-apple-darwin.tar.xz": "d3293668128b0a7838ef2544ae341eadd4437460ca22098af723349885a10ba2", + "rust-std-1.94.0-aarch64-apple-ios-macabi.tar.xz": "914f085065539ea1038d2832a5af1c22852cd5e9a063356b20e46698e72e770e", + "rust-std-1.94.0-aarch64-apple-ios-sim.tar.xz": "b4d16842b8f98ff2f431d6ab4aced298f4028ca00f0f5d588bc6efc94faa0680", + "rust-std-1.94.0-aarch64-apple-ios.tar.xz": "41ec1d18f4762f85231a19cf8aba3385da78fece2ff900a435c62881a3a3b44b", + "rust-std-1.94.0-aarch64-linux-android.tar.xz": "402c80df672a4b9fc948b6dc8bde21a63e1093eb1c8a2c2bd42221c309d9eba2", + "rust-std-1.94.0-aarch64-pc-windows-gnullvm.tar.xz": "8c89521f6826a1d2bf07b27c95405f4d6bf7c24f3de832c02758c41d4c1a511f", + "rust-std-1.94.0-aarch64-pc-windows-msvc.tar.xz": "d7964debd444dca92e771c953c3cd582fb927031b536c1439e1a831014d5f0d1", + "rust-std-1.94.0-aarch64-unknown-fuchsia.tar.xz": "87e11d97055540a0675bdc3dacf3d3d8cca5f806171ad0b51d0427f7cb9510c9", + "rust-std-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "c781b3ef4fefa5508fbe05820eddc95e46351d905a30921cc020febd9c596a2e", + "rust-std-1.94.0-aarch64-unknown-linux-musl.tar.xz": "3513032b2953d795bd3ac1489de4e8174e72500a31b4e7d1b349a23ab78a7a7b", + "rust-std-1.94.0-aarch64-unknown-none-softfloat.tar.xz": "375c496af31cf208ce67613c0aa1bde1894b5d674b00050cd46f7f954ed4a668", + "rust-std-1.94.0-aarch64-unknown-none.tar.xz": "854a9f8cea0ba889f553a7ea664c2a6479fc1fc49a4b56f84f99fc37001c4322", + "rust-std-1.94.0-aarch64-unknown-uefi.tar.xz": "f038d9e3df557d4aecdcf65989896179bcf0ecbc4d62bde71d33d7b914d3c253", + "rust-std-1.94.0-arm-linux-androideabi.tar.xz": "d2113ef50ed0b974f33b41102b961588c26abaf34f45a289a293b3d31ba01113", + "rust-std-1.94.0-arm-unknown-linux-gnueabi.tar.xz": "ad364b54b668e866315900ebff0d225616c556ab5228599792470aede05ba42c", + "rust-std-1.94.0-arm-unknown-linux-gnueabihf.tar.xz": "d5db943974fe05306ae43d100b0b7f3540108b93ef52de670675ce966af8a0fd", + "rust-std-1.94.0-arm-unknown-linux-musleabi.tar.xz": "c724b1a1b5dddeca563cda9b9695610bb56e681727b3af0a352174960087069a", + "rust-std-1.94.0-arm-unknown-linux-musleabihf.tar.xz": "9f8b707760a632b80864e91ea81a1b19c3f5452ff5e18b96da75792be1cf0a3c", + "rust-std-1.94.0-armv7-linux-androideabi.tar.xz": "e0b6883364f939300c281d96d6778ecf7fd97fe867b11936f0aa5c9a29f488ad", + "rust-std-1.94.0-armv7-unknown-linux-gnueabi.tar.xz": "9b1d8143881d510785d89413ea09b9fe910b64e3f300ab30e52e7fdd652ea66a", + "rust-std-1.94.0-armv7-unknown-linux-gnueabihf.tar.xz": "4b1744a8e48f58c2741afc1e076d0629a86051d785bd36e2c97f329eab6a5b49", + "rust-std-1.94.0-armv7-unknown-linux-musleabi.tar.xz": "fc2754f50ece0c7d3d32b2e0034b19c8b8bba9b471e0eb00f8ece7fa0bcb2837", + "rust-std-1.94.0-armv7-unknown-linux-musleabihf.tar.xz": "abfc0e7c1eaff1e3332b5c6262d12da309035a228261d07155add44464ed52f4", + "rust-std-1.94.0-i686-linux-android.tar.xz": "9093a894d87568a4b5eb4a144758bf49d4b62743677f747848afc3807c23f126", + "rust-std-1.94.0-i686-pc-windows-gnu.tar.xz": "188490bde6649163347050a70fd81c6d9e2e972ef12ba646f7f4fa106e21a956", + "rust-std-1.94.0-i686-pc-windows-gnullvm.tar.xz": "ba61dc038c3f4102a832a3381935192e88b0a1f79ff50b6d5590d476bdfa6879", + "rust-std-1.94.0-i686-pc-windows-msvc.tar.xz": "388cc2bb17ef9f12864c08e462cdbe3222d5ddafd84969dcf4ed944da27efcbd", + "rust-std-1.94.0-i686-unknown-freebsd.tar.xz": "3ce634092bcecb000c793c9d4bb0ccc6fd45c66fa8259fbb4f18c59844c4b13b", + "rust-std-1.94.0-i686-unknown-linux-gnu.tar.xz": "528c1dda491b25bb855426f48757773f4740c75d3c7271efffe7412c2fd5730d", + "rust-std-1.94.0-i686-unknown-linux-musl.tar.xz": "90ff16d879f774048c9b81d07188af4b63ed73a31848759cf560a9f51baa175f", + "rust-std-1.94.0-i686-unknown-uefi.tar.xz": "8c095069ec09d6e1d0b7e00f340fd7ea925a31acbad845d5956e1bc4a7d47733", + "rust-std-1.94.0-powerpc-unknown-linux-gnu.tar.xz": "23e32cab484d7787b7966a0508c49baf231bcdb4a79a421ee85ca69e81f2a06f", + "rust-std-1.94.0-powerpc64-unknown-linux-gnu.tar.xz": "7713b676a37fbd0a52a711c2e3c205a917f8b05614e17cd9f7f37874a88469a3", + "rust-std-1.94.0-powerpc64le-unknown-linux-gnu.tar.xz": "094f3955828d6630b70a9c7c71e3165c3ffc7a301929601dde08501def623347", + "rust-std-1.94.0-powerpc64le-unknown-linux-musl.tar.xz": "f3956b29463ec58d730f054ff2334755f929c602181ad67a25af9725c8500f68", + "rust-std-1.94.0-riscv32imc-unknown-none-elf.tar.xz": "5d10c7ab74cd5d10451480ec894ac5b98940a8527c333465eeec4a45b1125d43", + "rust-std-1.94.0-riscv64gc-unknown-linux-gnu.tar.xz": "32e9b7006f1269e48fb65af6298f6e8a222e1a8a2a20b112528bfb3dad2f65a8", + "rust-std-1.94.0-riscv64gc-unknown-linux-musl.tar.xz": "2c63d21143c7f1d44f3e3b1c9a82f3101576344c3fd1feb2d43ffc9d035e94f1", + "rust-std-1.94.0-riscv64gc-unknown-none-elf.tar.xz": "071a1ae54f7dead1b8147b9bae7d831a5554a37a1133f8cd52b265a765595a12", + "rust-std-1.94.0-s390x-unknown-linux-gnu.tar.xz": "ea69833addbc9521664464418c7468871f7a80b16fd2c385f6213a6c99cedfe9", + "rust-std-1.94.0-thumbv6m-none-eabi.tar.xz": "b022df94f6efa3bd2a6c4042472e114d2325bf058983b3ac7d586e24c8d77f25", + "rust-std-1.94.0-thumbv7em-none-eabi.tar.xz": "41894db05c4d8f421bf0dba7d0bed995685032393779f92b4ea1f3e6468badfb", + "rust-std-1.94.0-thumbv7em-none-eabihf.tar.xz": "be7e81535e7118fbad58a5dc31a1acfa7f2a36a2348aed23595169418559d67f", + "rust-std-1.94.0-thumbv7m-none-eabi.tar.xz": "4702e615be264df97b120fd075bed4e46710cc9a03c48115ce2ee3abfbf3822e", + "rust-std-1.94.0-thumbv8m.main-none-eabi.tar.xz": "5c1fb5ed8ad31580241b6e095583a2cac0ef90ea29dad7705c8a0ce0933a7797", + "rust-std-1.94.0-thumbv8m.main-none-eabihf.tar.xz": "573e9570bea039316ae3b55e323cbc741617ace1e0b87df8f3ecbbdf34b3de8a", + "rust-std-1.94.0-wasm32-unknown-emscripten.tar.xz": "0a607ba5d78afb47e9f5f134633ed5a954ece5558298e081084abe44f5265463", + "rust-std-1.94.0-wasm32-unknown-unknown.tar.xz": "bc65af3c0bfa2e5b20e299358710a26ce33a3674c762a2a010bc383c5b7fc202", + "rust-std-1.94.0-wasm32-wasip1-threads.tar.xz": "311b9fe1e94c90c16695899e670f0d76a366d493cdda29008191ce2d436cd22c", + "rust-std-1.94.0-wasm32-wasip1.tar.xz": "acbacacdca3e0ec0d93ff46d5f3579ae8cd5133b86e31a893da22cd94e28d0bf", + "rust-std-1.94.0-wasm32-wasip2.tar.xz": "e947e730da3fea212453b2835765802d9b0c80e19405cab188a6854834196e6c", + "rust-std-1.94.0-x86_64-apple-darwin.tar.xz": "cf716b5a5fab007398ce76536a85a3449df074b66e7c762c0f5de1d6c56a1342", + "rust-std-1.94.0-x86_64-apple-ios-macabi.tar.xz": "f39f4c1121dafcf8ea70a21c91be9552e30ea76debcad9bff28be945aaa79282", + "rust-std-1.94.0-x86_64-apple-ios.tar.xz": "60e70b80469dd3900dd8b1705831f5f02ac44fa97088b8a81fdda110ed421091", + "rust-std-1.94.0-x86_64-linux-android.tar.xz": "488c8a581612ac4779a05f05967b76a96e0ff0bdd1ab8d99c13a859c19056b01", + "rust-std-1.94.0-x86_64-pc-windows-gnu.tar.xz": "c9f98d62213d0559c8d2ee6d7eafbafbf5dfe04921c9e7cb15b66a770903bc33", + "rust-std-1.94.0-x86_64-pc-windows-gnullvm.tar.xz": "54c28d0030fab585457a431c555e5d23502132364d0483e5b3b2dfbfc3929cc6", + "rust-std-1.94.0-x86_64-pc-windows-msvc.tar.xz": "f8e2b43e52957e8224d97de68d6ec5da01613eb98bce4199de80071678b5fda0", + "rust-std-1.94.0-x86_64-unknown-freebsd.tar.xz": "081972f2f386ee859c15ed619695a9440896a80d1814d0e29d7620b7ef96207c", + "rust-std-1.94.0-x86_64-unknown-fuchsia.tar.xz": "6c18112553cf72cf991352208cce2bc96c712ca7b209acdd098abac894d674ef", + "rust-std-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "dd33653107c36e040082050d9e547e64dac5b456ba74069430d838c00c189a05", + "rust-std-1.94.0-x86_64-unknown-linux-musl.tar.xz": "b6786b5c490df2dcef76829039177799058f94e17725b42f912a057c0f6224f2", + "rust-std-1.94.0-x86_64-unknown-netbsd.tar.xz": "f763e56ca1ce3f819fb2136638d774777dd696f836cc52e6564431859a0f5159", + "rust-std-1.94.0-x86_64-unknown-none.tar.xz": "8509489165a8aa60a0590aeaa668f63423b3e7a33bb93e8e5719ff5404b6f6aa", + "rust-std-1.94.0-x86_64-unknown-uefi.tar.xz": "31b56e8178dd2319e51ccc474959d247db2767cda7149fe3704d636df9aee1c9", + "rustc-1.94.0-aarch64-apple-darwin.tar.xz": "6ba00a4f486af11826335cd7fe405592d959f50afb9489176fc1463cbe75dbf2", + "rustc-1.94.0-aarch64-pc-windows-msvc.tar.xz": "6d88d89c200a147357ded65e9bc426e14ebb6f5046d9772f3916b6a1ac60cdd5", + "rustc-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "7261b2fcdb88aef8b21f4b9f915d9a75c959674193f5b38e13fab4569684fc5b", + "rustc-1.94.0-x86_64-apple-darwin.tar.xz": "b1374b0fb63f5abcd8eb65607e43a499bfdc5e8160746d1f6f993aee9313f320", + "rustc-1.94.0-x86_64-pc-windows-msvc.tar.xz": "e0f59503778b563d3ee6d4edecd52b44056779ffe456181bad1fb4dc069802e6", + "rustc-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "31a0d3ac9383dfdeb4fce86eeed5ade3230131c635264c0eab7252dbf235f28e", + "rustfmt-1.94.0-aarch64-apple-darwin.tar.xz": "0eb716d201dc0d6d24ef8207d3ef27611d9d15d2212e3b27aa11e97286ed02ac", + "rustfmt-1.94.0-aarch64-pc-windows-msvc.tar.xz": "705ac3e797c4abed9e722e0864902f7a03872a8a201dcd7b70ff1b749eed2327", + "rustfmt-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "1ac17dbe7f8f6c98db5b62cb149fcd10c7565d624a59471115fc40055db9af28", + "rustfmt-1.94.0-x86_64-apple-darwin.tar.xz": "e58601a10ee85e956611ea2814bf26de88934a8224c1663bb11aa23f727663ec", + "rustfmt-1.94.0-x86_64-pc-windows-msvc.tar.xz": "7b32d8b0dd99f9bf6ddaa4866d39b7b96e8c2db46644d8527096f7f34b4fbe26", + "rustfmt-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "903e768f7626f78ed2d6cb82c1820b93bd339afec8575102e847fd3a379d33e5" + } + } } diff --git a/devenv.lock b/devenv.lock index ccdf5c5d..afbf1114 100644 --- a/devenv.lock +++ b/devenv.lock @@ -3,11 +3,11 @@ "devenv": { "locked": { "dir": "src/modules", - "lastModified": 1773672671, - "narHash": "sha256-TcU3EYJ6vVNRE6Kdhwh7wI3X6UTqyX+K84PnP03YK7Y=", + "lastModified": 1774261302, + "narHash": "sha256-P5noO+IXXDBL7l8dEVcig94OXfzLiN1K1KT0COjPcTU=", "owner": "cachix", "repo": "devenv", - "rev": "9dfa16743f60733f62634ae2826c8c8a4762e255", + "rev": "406f2ed97937995234c5d3de43cf77f31c42ca2b", "type": "github" }, "original": { @@ -22,11 +22,11 @@ "nixpkgs-src": "nixpkgs-src" }, "locked": { - "lastModified": 1772749504, - "narHash": "sha256-eqtQIz0alxkQPym+Zh/33gdDjkkch9o6eHnMPnXFXN0=", + "lastModified": 1773704619, + "narHash": "sha256-LKtmit8Sr81z8+N2vpIaN/fyiQJ8f7XJ6tMSKyDVQ9s=", "owner": "cachix", "repo": "devenv-nixpkgs", - "rev": "08543693199362c1fddb8f52126030d0d374ba2e", + "rev": "906534d75b0e2fe74a719559dfb1ad3563485f43", "type": "github" }, "original": { @@ -39,11 +39,11 @@ "nixpkgs-src": { "flake": false, "locked": { - "lastModified": 1772173633, - "narHash": "sha256-MOH58F4AIbCkh6qlQcwMycyk5SWvsqnS/TCfnqDlpj4=", + "lastModified": 1773597492, + "narHash": "sha256-hQ284SkIeNaeyud+LS0WVLX+WL2rxcVZLFEaK0e03zg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c0f3d81a7ddbc2b1332be0d8481a672b4f6004d6", + "rev": "a07d4ce6bee67d7c838a8a5796e75dff9caa21ef", "type": "github" }, "original": { diff --git a/devenv.nix b/devenv.nix index 6b77a707..cafd0f90 100644 --- a/devenv.nix +++ b/devenv.nix @@ -7,6 +7,7 @@ bazelisk python3 uv + zig ]; env.ZLS_CMD = "${config.devenv.root}/tools/zls.sh"; scripts.bazel.exec = ''bazelisk "$@"''; diff --git a/examples/lfm/BUILD.bazel b/examples/lfm/BUILD.bazel deleted file mode 100644 index a49cea2a..00000000 --- a/examples/lfm/BUILD.bazel +++ /dev/null @@ -1,10 +0,0 @@ -load("@bazel_skylib//rules:build_test.bzl", "build_test") -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_test") - -zig_binary( - name = "lfm", - srcs = ["model.zig", "config.zig", "inference.zig", "discussion.zig", "test_runner.zig"], - main = "main.zig", - visibility = ["//visibility:public"], - deps = ["//zml", "@linenoise"], -) diff --git a/examples/lfm/config.zig b/examples/lfm/config.zig deleted file mode 100644 index 82ea9762..00000000 --- a/examples/lfm/config.zig +++ /dev/null @@ -1,83 +0,0 @@ -/// Copied from https://github.com/huggingface/transformers/blob/0997c2f2ab08c32c8e2f90aaad06e29a7108535b/src/transformers/models/lfm2/configuration_lfm2.py -/// -/// Configuration parsed from LFM2-style `config.json`. -/// Notes: -/// - `layer_types` is expected to be fully specified (HF can derive it from `full_attn_idxs`). -/// - HF may use `block_ff_dim` as the effective MLP size; keep `intermediate_size` and `block_ff_dim` -/// in sync in the JSON if you rely on both. -/// - HF uses `tie_embedding` as an alias for `tie_word_embeddings`. -/// - `eos_token_id` can differ from the HF default (e.g. 7 in LFM2 configs). -pub const Config = struct { - /// Model class names in the source config. - architectures: []const []const u8, - /// Whether to auto-adjust the FFN dimension. - block_auto_adjust_ff_dim: bool, - /// Base block hidden dimension. - block_dim: u32, - /// Block FFN dimension (HF uses this as the effective `intermediate_size` when present). - block_ff_dim: u32, - /// Multiplier for FFN dimension. - block_ffn_dim_multiplier: f32, - /// MLP init scale in block config. - block_mlp_init_scale: f32, - /// Multiple for intermediate size rounding. - block_multiple_of: u32, - /// RMS norm epsilon within blocks. - block_norm_eps: f32, - /// Output init scale in block config. - block_out_init_scale: f32, - /// Whether to use SwiGLU in blocks. - block_use_swiglu: bool, - /// Whether to use Xavier init in blocks. - block_use_xavier_init: bool, - /// BOS token id. - bos_token_id: u32, - /// Convolution cache length. - conv_L_cache: u32, - /// Whether conv layers use bias. - conv_bias: bool, - /// Convolution dimension. - conv_dim: u32, - /// Whether to use Xavier init in conv layers. - conv_use_xavier_init: bool, - /// Serialized dtype (e.g. "bfloat16"). - dtype: []const u8, - /// EOS token id. - eos_token_id: u32, - /// Hidden size of the model. - hidden_size: u32, - /// Weight init stddev. - initializer_range: f32, - /// FFN hidden dimension (HF may override with `block_ff_dim`). - intermediate_size: u32, - /// Per-layer operator types (e.g. "conv" or "full_attention"). - layer_types: []const []const u8, - /// Max sequence length supported by RoPE. - max_position_embeddings: u32, - /// Model type tag, expected "lfm2". - model_type: []const u8, - /// RMS norm epsilon. - norm_eps: f32, - /// Number of attention heads. - num_attention_heads: u32, - /// Total number of heads (may duplicate `num_attention_heads`). - num_heads: u32, - /// Number of transformer layers. - num_hidden_layers: u32, - /// Number of key/value heads for GQA/MQA. - num_key_value_heads: u32, - /// PAD token id. - pad_token_id: u32, - /// RoPE theta. - rope_theta: f32, - /// Whether to tie embeddings (aka `tie_word_embeddings`). - tie_embedding: bool, - /// Transformers version string. - transformers_version: []const u8, - /// Whether to use KV cache. - use_cache: bool, - /// Whether to use positional encodings. - use_pos_enc: bool, - /// Vocabulary size. - vocab_size: u32, -}; diff --git a/examples/lfm/discussion.zig b/examples/lfm/discussion.zig deleted file mode 100644 index ae657805..00000000 --- a/examples/lfm/discussion.zig +++ /dev/null @@ -1,301 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const zml = @import("zml"); -const stdx = zml.stdx; -const attention = zml.attention.attention; - -const inference = @import("inference.zig"); -const model = @import("model.zig"); - -const log = std.log.scoped(.lfm); - -pub const DEFAULT_PROMPT: [:0]const u8 = "> "; - -pub const Context = struct { - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - model_buffers: *zml.Bufferized(model.Model), - exe: inference.Inference, - tokenizer: zml.tokenizer.Tokenizer, - config: model.Config, - seqlen: u32, - cache_buffers: zml.Bufferized(model.Cache), - attention_metadata_buffers: zml.Bufferized(attention.Metadata), - rng_buf: zml.Bufferized(zml.Tensor.Rng), - generated_token_slice: zml.Slice, - token_pos: u32, - all_tokens: std.ArrayList(u32), - sharding: zml.sharding.Sharding, - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - model_buffers: *zml.Bufferized(model.Model), - cache: model.Cache, - attention_metadata: attention.Metadata, - exe: inference.Inference, - tokenizer: zml.tokenizer.Tokenizer, - config: model.Config, - seqlen: u32, - sharding: zml.sharding.Sharding, - ) !Context { - const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); - return .{ - .allocator = allocator, - .io = io, - .platform = platform, - .model_buffers = model_buffers, - .exe = exe, - .tokenizer = tokenizer, - .config = config, - .seqlen = seqlen, - .cache_buffers = try cache.initBuffers(allocator, io, platform, sharding), - .attention_metadata_buffers = try attention_metadata.initBuffer(io, platform, sharding), - .rng_buf = try zml.Tensor.Rng.initBuffer(platform, seed, io, sharding), - .generated_token_slice = try .alloc(allocator, zml.Shape.init(.{ .batch = 1, .seq = 1 }, .u32)), - .token_pos = 0, - .all_tokens = try .initCapacity(allocator, seqlen), - .sharding = sharding, - }; - } - - pub fn deinit(self: *Context) void { - model.Cache.unloadBuffers(&self.cache_buffers); - attention.Metadata.deinitBuffer(&self.attention_metadata_buffers); - zml.Tensor.Rng.deinitBuffer(&self.rng_buf); - self.generated_token_slice.free(self.allocator); - self.all_tokens.deinit(self.allocator); - } - - pub fn runOnce(self: *Context, prompt: []const u8) !void { - const prompt_tokens: []const u32 = try tokenizePrompt( - self.allocator, - self.tokenizer, - self.config, - prompt, - ); - defer self.allocator.free(prompt_tokens); - - var stdout = std.Io.File.stdout().writer(self.io, &.{}); - - log.info("Running prefill...", .{}); - try self.runPrefill(prompt_tokens); - - log.info("Running decode...", .{}); - try self.runDecode(&stdout.interface); - } - - pub fn start(self: *Context, prompt: []const u8) !void { - var turn_tokens: []const u32 = try tokenizePrompt( - self.allocator, - self.tokenizer, - self.config, - prompt, - ); - defer self.allocator.free(turn_tokens); - - var stdout = std.Io.File.stdout().writer(self.io, &.{}); - - while (self.remainingTokens() > 0) { - try stdout.interface.writeAll("\x1b[2mprefill...\x1b[0m"); - try stdout.interface.flush(); - - const prefill_start: std.Io.Timestamp = .now(self.io, .awake); - const prefill_pos_before = self.token_pos; - try self.runPrefill(turn_tokens); - const prefill_duration = prefill_start.untilNow(self.io, .awake); - const prefill_tokens: u64 = self.token_pos - prefill_pos_before; - - try stdout.interface.writeAll("\r \r"); - try stdout.interface.flush(); - - const decode_start: std.Io.Timestamp = .now(self.io, .awake); - const decode_pos_before = self.token_pos; - try self.runDecode(&stdout.interface); - const decode_duration = decode_start.untilNow(self.io, .awake); - const decode_tokens: u64 = self.token_pos - decode_pos_before; - - try stdout.interface.writeAll("\n\n"); - - try stdout.interface.print("\x1b[36mprefill\x1b[0m \x1b[2m{f} \xc2\xb7 {:.1}tok/s\x1b[0m \x1b[2m\xe2\x94\x82\x1b[0m \x1b[36mdecode\x1b[0m \x1b[2m{f} \xc2\xb7 {:.1}tok/s\x1b[0m\n\n\n", .{ - prefill_duration, - tokensPerSecond(prefill_duration, prefill_tokens), - decode_duration, - tokensPerSecond(decode_duration, decode_tokens), - }); - try stdout.interface.flush(); - - while (true) { - const line = c.linenoise(DEFAULT_PROMPT.ptr) orelse { - try stdout.interface.print("\x1b[2m{}/{} tokens used\x1b[0m\n\n", .{ self.token_pos, self.seqlen }); - try stdout.interface.flush(); - return; - }; - defer c.linenoiseFree(line); - const input = std.mem.sliceTo(line, 0); - if (input.len == 0) continue; - - self.allocator.free(turn_tokens); - turn_tokens = try tokenizeTurn(self.allocator, self.tokenizer, input); - break; - } - - if (turn_tokens.len > self.remainingTokens()) { - log.warn("Not enough tokens remaining ({} available, {} needed). Consider using a higher seqlen, for example: `--seqlen={d}`.", .{ self.remainingTokens(), turn_tokens.len, std.math.ceilPowerOfTwo(u32, self.seqlen + 1) catch std.math.maxInt(u32) }); - break; - } - } - } - - fn remainingTokens(self: Context) u32 { - return self.seqlen -| self.token_pos; - } - - fn runPrefill(self: *Context, prompt_tokens: []const u32) !void { - try self.all_tokens.appendSlice(self.allocator, prompt_tokens); - - if (self.all_tokens.items.len > self.seqlen) { - return error.PromptTooLong; - } - - const tokens_slice: zml.Slice = try .alloc(self.allocator, .init(.{ .batch = 1, .seq = self.seqlen }, .u32)); - defer tokens_slice.free(self.allocator); - const tokens = tokens_slice.items(u32); - @memset(tokens, self.config.pad_token_id); - @memcpy(tokens[0..self.all_tokens.items.len], self.all_tokens.items); - - var tokens_buf: zml.Buffer = try .fromSlice(self.io, self.platform, tokens_slice, self.sharding); - defer tokens_buf.deinit(); - - // Always prefill from position 0 with all accumulated tokens. - const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{0})); - var tokens_pos_buf: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice, self.sharding); - defer tokens_pos_buf.deinit(); - - const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(self.all_tokens.items.len)})); - var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice, self.sharding); - defer actual_seq_len_buf.deinit(); - - try self.exe.prefill.run(.{ - .allocator = self.allocator, - .io = self.io, - .platform = self.platform, - .model_buffers = self.model_buffers, - .tokens_buf = &tokens_buf, - .tokens_pos_buf = &tokens_pos_buf, - .actual_seq_len_buf = &actual_seq_len_buf, - .rng_buf = &self.rng_buf, - .cache_buffers = &self.cache_buffers, - .attention_metadata_buffers = self.attention_metadata_buffers, - .sharding = self.sharding, - }); - - try tokens_buf.toSlice(self.io, tokens_slice); - self.generated_token_slice.items(u32)[0] = tokens_slice.items(u32)[self.all_tokens.items.len - 1]; - self.token_pos = @intCast(self.all_tokens.items.len); - } - - fn runDecode(self: *Context, writer: *std.Io.Writer) !void { - var tokenizer_decoder = try self.tokenizer.decoder(); - defer tokenizer_decoder.deinit(); - - var current_token_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, self.generated_token_slice, self.sharding); - defer current_token_buffer.deinit(); - - const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{0})); - var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice, self.sharding); - defer actual_seq_len_buf.deinit(); - - const think_start = self.tokenizer.tokenToId("") orelse unreachable; - const think_end = self.tokenizer.tokenToId("") orelse unreachable; - - generation: while (true) { - const generated_token = self.generated_token_slice.items(u32)[0]; - - if (generated_token == self.config.eos_token_id) break :generation; - if (try tokenizer_decoder.next(generated_token)) |chunk| { - if (generated_token == think_start) { - try writer.writeAll("\x1b[2m"); - } - try writer.writeAll(chunk); - if (generated_token == think_end) { - try writer.writeAll("\x1b[0m"); - } - try writer.flush(); - } - - // Accumulate the generated token. - try self.all_tokens.append(self.allocator, generated_token); - - if (self.remainingTokens() == 0) break :generation; - const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{self.token_pos})); - var token_pos_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice, self.sharding); - defer token_pos_buffer.deinit(); - - try self.exe.decode.run(.{ - .allocator = self.allocator, - .io = self.io, - .platform = self.platform, - .model_buffers = self.model_buffers, - .tokens_buf = ¤t_token_buffer, - .tokens_pos_buf = &token_pos_buffer, - .actual_seq_len_buf = &actual_seq_len_buf, - .rng_buf = &self.rng_buf, - .cache_buffers = &self.cache_buffers, - .attention_metadata_buffers = self.attention_metadata_buffers, - .sharding = self.sharding, - }); - - try current_token_buffer.toSlice(self.io, self.generated_token_slice); - self.token_pos += 1; - } - } -}; - -fn tokensPerSecond(duration: std.Io.Duration, tokens: u64) f64 { - if (tokens == 0) return 0; - const seconds = @as(f64, @floatFromInt(duration.toNanoseconds())) / 1e9; - if (seconds <= 0) return 0; - return @as(f64, @floatFromInt(tokens)) / seconds; -} - -fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: model.Config, prompt: []const u8) ![]u32 { - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - const im_start = tokenizer.tokenToId("<|im_start|>") orelse return error.NoSuchToken; - const im_end = tokenizer.tokenToId("<|im_end|>") orelse return error.NoSuchToken; - const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; - const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; - const newline = (try encoder.encode("\n"))[0]; - - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); - try tokens.appendSlice(allocator, &.{ config.bos_token_id, im_start, user, newline }); - try tokens.appendSlice(allocator, try encoder.encode(prompt)); - try tokens.appendSlice(allocator, &.{ im_end, newline }); - try tokens.appendSlice(allocator, &.{ im_start, assistant, newline }); - - return tokens.toOwnedSlice(allocator); -} - -fn tokenizeTurn(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]u32 { - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - const im_start = tokenizer.tokenToId("<|im_start|>") orelse return error.NoSuchToken; - const im_end = tokenizer.tokenToId("<|im_end|>") orelse return error.NoSuchToken; - const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; - const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; - const newline = (try encoder.encode("\n"))[0]; - - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); - try tokens.appendSlice(allocator, &.{ im_end, newline, im_start, user, newline }); - try tokens.appendSlice(allocator, try encoder.encode(prompt)); - try tokens.appendSlice(allocator, &.{ im_end, newline }); - try tokens.appendSlice(allocator, &.{ im_start, assistant, newline }); - - return tokens.toOwnedSlice(allocator); -} diff --git a/examples/lfm/main.zig b/examples/lfm/main.zig deleted file mode 100644 index c34144d6..00000000 --- a/examples/lfm/main.zig +++ /dev/null @@ -1,173 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const zml = @import("zml"); -const attention = zml.attention.attention; -const stdx = zml.stdx; - -const discussion = @import("discussion.zig"); -const inference = @import("inference.zig"); -const model = @import("model.zig"); -const test_runner = @import("test_runner.zig"); - -pub const std_options: std.Options = .{ - .log_level = .info, -}; - -const log = std.log.scoped(.lfm); -const Gpa = std.heap.GeneralPurposeAllocator(.{ .thread_safe = true }); - -const Args = struct { - model: []const u8, - prompt: ?[]const u8 = null, - seqlen: u32 = 2048, - backend: ?attention.Backend = null, - run_tests: bool = false, - single: bool = false, - - pub const help = - \\Use lfm --model= [options] - \\ - \\ Run LFM model inference. - \\ - \\ Options: - \\ --model= Path to the model repository (required) - \\ --prompt= Prompt to use for generation (default: none) - \\ --seqlen= Sequence length (default: 1024) - \\ --backend= Attention backend to use ([vanilla, cuda_fa2, cuda_fa3], default: auto-selection) - \\ --run-tests Run tests instead of inference - \\ --single Create a single kernel encompassing all the layers. - \\ - ; -}; - -pub fn main(init: std.process.Init) !void { - const allocator = init.gpa; - const io = init.io; - - const platform: *zml.Platform = try .auto(allocator, io, .{ .cuda = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.8 } } } }); - defer platform.deinit(allocator); - - log.info("\n{f}", .{platform.fmtVerbose()}); - - const args = stdx.flags.parse(init.minimal.args, Args); - - log.info("Resolving model repository..", .{}); - const repo = try zml.safetensors.resolveModelRepo(io, args.model); - - log.info("Loading configuration...", .{}); - const parsed_config = try loadConfig(model.Config, allocator, io, repo); - defer parsed_config.deinit(); - const config = parsed_config.value; - - log.info("Initializing model...", .{}); - var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); - defer registry.deinit(); - - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); - defer store.deinit(); - - var mdl: model.Model = .init(allocator, store.view(), config); - defer mdl.deinit(allocator); - - const attention_backend = args.backend orelse b: { - const selected = attention.Backend.auto(platform); - log.info("Selected backend: {}", .{selected}); - break :b selected; - }; - - var progress = std.Progress.start(io, .{ .root_name = args.model }); - errdefer progress.end(); - - var tokenizer_fut = try io.concurrent(loadTokenizer, .{ allocator, io, repo, &progress }); - defer if (tokenizer_fut.cancel(io)) |tokenizer| { - var tk = tokenizer; - tk.deinit(); - } else |_| {}; - - const params: inference.CompilationOptions = .init(mdl, config, args.seqlen, attention_backend, args.single); - - const replicated_sharding = try zml.sharding.replicatedSharding(platform); - var compilation_fut = try io.concurrent(inference.Inference.init, .{ allocator, io, platform, mdl, params, args.seqlen, &progress, replicated_sharding }); - defer if (compilation_fut.cancel(io)) |exe| { - exe.deinit(); - } else |_| {}; - - var load_model_fut = try io.concurrent(model.Model.loadBuffers, .{ - &mdl, - allocator, - io, - platform, - &store, - &progress, - }); - defer if (load_model_fut.cancel(io)) |model_buffers| { - var mb = model_buffers; - model.Model.unloadBuffers(&mb, allocator); - } else |_| {}; - - const exe: inference.Inference = try compilation_fut.await(io); - var model_buffers: zml.Bufferized(model.Model) = try load_model_fut.await(io); - const tokenizer = try tokenizer_fut.await(io); - progress.end(); - - if (args.run_tests) { - return test_runner.run( - allocator, - io, - platform, - config, - mdl, - &model_buffers, - &store, - params.attention_metadata, - params.attention_parameters, - ); - } - - const interactive = args.prompt == null; - const prompt = if (args.prompt) |p| p else blk: { - const line = c.linenoise(discussion.DEFAULT_PROMPT) orelse return; - defer c.linenoiseFree(line); - break :blk try allocator.dupe(u8, std.mem.sliceTo(line, 0)); - }; - defer allocator.free(prompt); - - var ctx = try discussion.Context.init(allocator, io, platform, &model_buffers, params.cache, params.attention_metadata, exe, tokenizer, config, args.seqlen, replicated_sharding); - defer ctx.deinit(); - - if (interactive) { - return ctx.start(prompt); - } else { - return ctx.runOnce(prompt); - } -} - -pub fn loadConfig(comptime T: type, allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !std.json.Parsed(T) { - const file = try dir.openFile(io, "config.json", .{}); - defer file.close(io); - - var buffer: [256]u8 = undefined; - var file_reader = file.reader(io, &buffer); - var reader: std.json.Reader = .init(allocator, &file_reader.interface); - defer reader.deinit(); - - return try std.json.parseFromTokenSource(model.Config, allocator, &reader, .{ .ignore_unknown_fields = true }); -} - -pub fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, progress: *std.Progress.Node) !zml.tokenizer.Tokenizer { - progress.increaseEstimatedTotalItems(1); - var node = progress.start("Loading tokenizer...", 1); - defer node.end(); - const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Loaded tokenizer [{f}]", .{now.untilNow(io, .awake)}); - const bytes = b: { - const file = try dir.openFile(io, "tokenizer.json", .{}); - defer file.close(io); - var reader = file.reader(io, &.{}); - break :b try reader.interface.readAlloc(allocator, try file.length(io)); - }; - defer allocator.free(bytes); - - return try .fromBytes(allocator, io, bytes); -} diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel deleted file mode 100644 index c2a6da49..00000000 --- a/examples/llama/BUILD.bazel +++ /dev/null @@ -1,76 +0,0 @@ -load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") -load("@bazel_skylib//rules:build_test.bzl", "build_test") -load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") -load("@rules_zig//zig:defs.bzl", "zig_binary") -load("@tar.bzl//tar:mtree.bzl", "mtree_spec") -load("@tar.bzl//tar:tar.bzl", "tar") - -zig_binary( - name = "llama", - srcs = ["llama.zig"], - main = "main.zig", - visibility = ["//visibility:public"], - deps = [ - "//bazel", - "//zml", - ], -) - -mtree_spec( - name = "mtree", - srcs = [":llama"], - tags = ["manual"] -) - -tar( - name = "archive", - srcs = [":llama"], - args = [ - "--options", - ",".join([ - "zstd:compression-level=9", - "zstd:threads=16", - ]), - ], - compress = "zstd", - mtree = ":mtree", - tags = ["manual"] -) - -oci_image( - name = "image_", - base = "@distroless_cc_debian12", - entrypoint = ["/{}/llama".format(package_name())], - target_compatible_with = [ - "@platforms//os:linux", - ], - tars = [":archive"], - tags = ["manual"] -) - -platform_transition_filegroup( - name = "image", - srcs = [":image_"], - tags = ["manual"], - target_platform = "//platforms:linux_amd64", -) - -oci_load( - name = "load", - image = ":image", - repo_tags = ["zmlai/llama:latest"], - tags = ["manual"], -) - -oci_push( - name = "push", - image = ":image", - remote_tags = ["latest"], - repository = "index.docker.io/zmlai/llama", - tags = ["manual"], -) - -build_test( - name = "test", - targets = [":llama"], -) diff --git a/examples/llama/main.zig b/examples/llama/main.zig deleted file mode 100644 index fdd49ef4..00000000 --- a/examples/llama/main.zig +++ /dev/null @@ -1,483 +0,0 @@ -const std = @import("std"); -const log = std.log; - -const zml = @import("zml"); -const Buffer = zml.Buffer; -const Tensor = zml.Tensor; -const ShapeOf = zml.ShapeOf; -const stdx = zml.stdx; - -const llama = @import("llama.zig"); -const LlamaLM = llama.LlamaLM; -const Llama = llama.Llama; -const KvCache = llama.KvCache; -const TransformerLayer = llama.TransformerLayer; -const SelfAttn = llama.SelfAttn; - -pub const std_options: std.Options = .{ - .log_level = .info, -}; - -const CliArgs = struct { - model: []const u8, - prompt: ?[]const u8 = null, - seqlen: u32 = 512, - reader_buffer_size: stdx.flags.ByteSize = .{ .value = 32, .unit = .mib }, - writer_buffer_size: stdx.flags.ByteSize = .{ .value = 32, .unit = .mib }, - async_limit: ?u32 = null, - backend: ?zml.attention.attention.Backend = null, - - pub const help = - \\Usage: llama --model= [options] - \\ - \\Run LLaMA inference on a model. - \\ - \\Options: - \\ --model= Path to model (local path or HuggingFace repo) - \\ --prompt= Input prompt (reads from stdin if not provided) - \\ --seqlen= Maximum sequence length (default: 512) - \\ --backend= Attention backend to use ([vanilla, cuda_fa2, cuda_fa3], default: auto-selection) - \\ --reader-buffer-size= Reader buffer size (default: 32MiB) - \\ --writer-buffer-size= Writer buffer size (default: 32MiB) - \\ --async-limit= Async I/O concurrency limit - \\ -h, --help Show this help message - \\ - \\Examples: - \\ llama --model=hf://meta-llama/Llama-3.1-8B-Instruct --prompt="Hello, world!" - \\ llama --model=$(realpath ../Llama-3.1-8B-Instruct) --seqlen=1024 - \\ echo "What is 2+2?" | llama --model=meta-llama/Llama-3.1-8B-Instruct - \\ - ; -}; - -pub fn main(init: std.process.Init) !void { - const allocator = init.gpa; - const arena = init.arena; - - log.info("LLama was compiled with {}", .{@import("builtin").mode}); - - const args = stdx.flags.parse(init.minimal.args, CliArgs); - - var http_client: std.http.Client = .{ .allocator = allocator, .io = init.io }; - - try http_client.initDefaultProxies(allocator, init.environ_map); - defer http_client.deinit(); - - var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); - defer vfs_file.deinit(); - - var vfs_https: zml.io.VFS.HTTP = try .init(allocator, init.io, &http_client, .https); - defer vfs_https.deinit(); - - var hf_vfs: zml.io.VFS.HF = try .auto(allocator, init.io, &http_client, init.environ_map); - defer hf_vfs.deinit(); - - var s3_vfs: zml.io.VFS.S3 = try .auto(allocator, init.io, &http_client, init.environ_map); - defer s3_vfs.deinit(); - - var vfs: zml.io.VFS = try .init(allocator, init.io); - defer vfs.deinit(); - - try vfs.register("file", vfs_file.io()); - try vfs.register("https", vfs_https.io()); - try vfs.register("hf", hf_vfs.io()); - try vfs.register("s3", s3_vfs.io()); - - const io = vfs.io(); - - log.info("Resolving model repo", .{}); - const repo = try zml.safetensors.resolveModelRepo(io, args.model); - - const parsed_config = try parseConfig(allocator, io, repo); - defer parsed_config.deinit(); - const config = parsed_config.value; - - var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); - defer registry.deinit(); - - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); - defer store.deinit(); - - // Write metadata from the config file into the LlamaLm struct. - const llama_options: llama.LlamaLM.Options = .{ - .max_seq_len = args.seqlen, - .sampling_strategy = .{ - .topk = 1, - .temperature = 1.0, - }, - }; - - var platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator); - log.info("\n{f}", .{platform.fmtVerbose()}); - - // Initialize the Llama struct and map the content of the .safetensors to the model tensors. - var llama_model: llama.LlamaLM = try .init(allocator, store.view(), config, llama_options); - defer llama_model.deinit(allocator); - - const backend = args.backend orelse b: { - const selected = zml.attention.attention.Backend.auto(platform); - log.info("Selected backend: {}", .{selected}); - break :b selected; - }; - - // Specify shapes of input arguments - const dtype = llama_model.model.embed_tokens.weight.dtype(); - const llama_parameters: LlamaParameters = .{ - .prefill_tokens = .init(.{ .s = llama_options.max_seq_len }, .u32), - .decode_tokens = .init(.{ .s = 1 }, .u32), - .token_index = .init(.{}, .u32), - .kv_cache = .init(.init(.{ - .layer = llama_model.model.layers.len, - .k = args.seqlen, - .h = config.num_key_value_heads, - .hd = config.head_dim orelse @divExact(config.hidden_size, config.num_attention_heads), - }, dtype)), - .rng = .init(), - .attention_metadata = .init(.fromBackend(backend, @intCast(args.seqlen), @intCast(config.num_attention_heads))), - .attention_parameters = .init(.fromBackend(backend)), - }; - - const tp_mesh: zml.sharding.LogicalMesh = try .init("tp_mesh", .{ .model = .high_bandwidth }); - const tp_strategy: zml.sharding.Strategy = try .suggest(tp_mesh, platform.physical_mesh); - const sharding_tp: zml.sharding.Sharding = try .initFromStrategy(platform, tp_mesh, tp_strategy); - const sharding_replicated = try zml.sharding.replicatedSharding(platform); - - var progress = std.Progress.start(io, .{ .root_name = args.model }); - - var tokenizer_future = try io.concurrent(loadTokenizer, .{ allocator, io, repo, &progress }); - errdefer blk: { - var v = tokenizer_future.cancel(io) catch break :blk; - v.deinit(); - } - - var compiled_model_result_future = try io.concurrent(compileModel, .{ allocator, io, platform, llama_model, llama_parameters, &.{sharding_tp}, &progress }); - defer if (compiled_model_result_future.cancel(io)) |v| { - v.prefill_exe.deinit(); - v.decode_exe.deinit(); - } else |_| {}; - - var load_group: stdx.Io.LimitedGroup = .init(16); - defer load_group.cancel(io); - - var llama_buffers_future = try io.concurrent(LlamaLM.load, .{ - &llama_model, - arena.allocator(), - io, - platform, - &store, - &.{sharding_tp}, - &progress, - }); - defer blk: { - var v = llama_buffers_future.cancel(io) catch break :blk; - LlamaLM.unloadBuffers(&v, arena.allocator()); - } - - const compiled_model_result = try compiled_model_result_future.await(io); - - var llama_buffers = try llama_buffers_future.await(io); - progress.end(); - - log.info("Creating KvCache", .{}); - var kv_cache_buffers = try llama_parameters.kv_cache.initBuffer(io, platform, sharding_tp); - defer llama.KvCache.deinitBuffer(&kv_cache_buffers); - - var attention_metadata_buffers: zml.Bufferized(zml.attention.attention.Metadata) = try llama_parameters.attention_metadata.initBuffer(io, platform, sharding_tp); - defer zml.attention.attention.Metadata.deinitBuffer(&attention_metadata_buffers); - - const tokenizer = try tokenizer_future.await(io); - - const prompt = if (args.prompt) |p| p else blk: { - var reader = std.Io.File.stdin().reader(io, &.{}); - break :blk try reader.interface.allocRemaining(allocator, .unlimited); - }; - - log.info("✅ Prompt: {s}", .{prompt}); - - // Unbuffered writing of the tokens to stdout. - var stdout = std.Io.File.stdout().writer(io, &.{}); - - try generateText( - allocator, - io, - &llama_buffers, - compiled_model_result.prefill_exe, - compiled_model_result.decode_exe, - &kv_cache_buffers, - &attention_metadata_buffers, - tokenizer, - config, - llama_options, - @intCast(std.Io.Clock.now(.awake, io).toNanoseconds()), - prompt[0..], - false, - &stdout.interface, - platform, - sharding_replicated, - ); -} - -fn parseConfig(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !std.json.Parsed(LlamaLM.Config) { - const now: std.Io.Timestamp = .now(io, .awake); - log.info("Loading model config", .{}); - defer log.info("Loaded model config [{f}]", .{now.untilNow(io, .awake)}); - - const parsed_config = blk: { - const config_json_file = try dir.openFile(io, "config.json", .{}); - defer config_json_file.close(io); - var config_json_buffer: [256]u8 = undefined; - var config_reader = config_json_file.reader(io, &config_json_buffer); - var reader: std.json.Reader = .init(allocator, &config_reader.interface); - defer reader.deinit(); - break :blk try std.json.parseFromTokenSource(llama.LlamaLM.Config, allocator, &reader, .{ .ignore_unknown_fields = true }); - }; - errdefer parsed_config.deinit(); - - return parsed_config; -} - -fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, progress: *std.Progress.Node) !zml.tokenizer.Tokenizer { - // log.info("Loading tokenizer", .{}); - progress.increaseEstimatedTotalItems(1); - var node = progress.start("Loading tokenizer...", 1); - defer node.end(); - const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Loaded tokenizer [{f}]", .{now.untilNow(io, .awake)}); - const bytes = b: { - const file = try dir.openFile(io, "tokenizer.json", .{}); - defer file.close(io); - var reader = file.reader(io, &.{}); - break :b try reader.interface.readAlloc(allocator, try file.length(io)); - }; - defer allocator.free(bytes); - - return try .fromBytes(allocator, io, bytes); -} - -const LlamaParameters = struct { - prefill_tokens: zml.Tensor, - decode_tokens: zml.Tensor, - token_index: zml.Tensor, - kv_cache: llama.KvCache, - rng: zml.Tensor.Rng, - attention_metadata: zml.attention.attention.Metadata, - attention_parameters: zml.attention.attention.Parameters, -}; - -const CompileModelResult = struct { - prefill_exe: zml.Exe, - decode_exe: zml.Exe, -}; - -fn compileModel( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - llama_model: llama.LlamaLM, - parameters: LlamaParameters, - shardings: []const zml.sharding.Sharding, - progress: *std.Progress.Node, -) !CompileModelResult { - const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Compiled model [{f}]", .{now.untilNow(io, .awake)}); - - // Compile the model twice, one for prefill, one for generation. - var prefill_future = try io.concurrent(struct { - fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, llama_model_: llama.LlamaLM, parameters_: LlamaParameters, shardings_: []const zml.sharding.Sharding, progress_: *std.Progress.Node) !zml.Exe { - progress_.increaseEstimatedTotalItems(1); - var node_ = progress_.start("Compiling prefill...", 1); - defer node_.end(); - const now_: std.Io.Timestamp = .now(io_, .awake); - defer log.info("Compiled prefill [{f}]", .{now_.untilNow(io_, .awake)}); - return platform_.compile(allocator_, io_, llama_model_, .forward, .{ - parameters_.prefill_tokens, - parameters_.token_index, - parameters_.kv_cache, - parameters_.rng, - parameters_.attention_metadata, - parameters_.attention_parameters, - }, .{ .shardings = shardings_ }); - } - }.call, .{ allocator, io, platform, llama_model, parameters, shardings, progress }); - errdefer if (prefill_future.cancel(io)) |v| { - v.deinit(); - } else |_| {}; - - var decode_future = try io.concurrent(struct { - fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, llama_model_: llama.LlamaLM, parameters_: LlamaParameters, shardings_: []const zml.sharding.Sharding, progress_: *std.Progress.Node) !zml.Exe { - progress_.increaseEstimatedTotalItems(1); - var node_ = progress_.start("Compiling decode...", 1); - defer node_.end(); - const now_: std.Io.Timestamp = .now(io_, .awake); - defer log.info("Compiled decode [{f}]", .{now_.untilNow(io_, .awake)}); - return platform_.compile(allocator_, io_, llama_model_, .forward, .{ - parameters_.decode_tokens, - parameters_.token_index, - parameters_.kv_cache, - parameters_.rng, - parameters_.attention_metadata, - parameters_.attention_parameters, - }, .{ .shardings = shardings_ }); - } - }.call, .{ allocator, io, platform, llama_model, parameters, shardings, progress }); - errdefer if (decode_future.cancel(io)) |v| { - v.deinit(); - } else |_| {}; - - const prefill_exe = try prefill_future.await(io); - const decode_exe = try decode_future.await(io); - - return .{ .prefill_exe = prefill_exe, .decode_exe = decode_exe }; -} - -pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: LlamaLM.Config, prompt: []const u8, skip_llama3_encoding: bool) ![]u32 { - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - if (skip_llama3_encoding) { - // Copy so the ownership is the same in both branches. - return try allocator.dupe(u32, try encoder.encode(prompt)); - } - - const start_header = tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; - const end_header = tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; - const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; - const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; - const eot = tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; - const newline = (try encoder.encode("\n"))[0]; - - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); - try tokens.appendSlice(allocator, &.{ config.bos_token_id, start_header, user, end_header, newline }); - - try tokens.appendSlice(allocator, try encoder.encode(prompt)); - try tokens.appendSlice(allocator, &.{ eot, newline }); - - try tokens.appendSlice(allocator, &.{ start_header, assistant, end_header, newline }); - - return tokens.toOwnedSlice(allocator); -} - -pub fn generateText( - allocator: std.mem.Allocator, - io: std.Io, - llama_buffers: *zml.Bufferized(LlamaLM), - prefill_exe: zml.exe.Exe, - decode_exe: zml.exe.Exe, - kv_cache_buffers: *zml.Bufferized(llama.KvCache), - attention_metadata_buffers: *zml.Bufferized(zml.attention.attention.Metadata), - tokenizer: zml.tokenizer.Tokenizer, - config: LlamaLM.Config, - options: LlamaLM.Options, - seed: u128, - prompt: []const u8, - skip_llama3_encoding: bool, - writer: *std.Io.Writer, - platform: *const zml.Platform, - replicated_sharding: zml.sharding.Sharding, -) !void { - const prompt_tok: []const u32 = try tokenizePrompt(allocator, tokenizer, config, prompt, skip_llama3_encoding); - defer allocator.free(prompt_tok); - - var tokenizer_decoder = try tokenizer.decoder(); - defer tokenizer_decoder.deinit(); - - const max_seq_len = options.max_seq_len; - - // init RNG and buffers - var rng_buffers = try zml.Tensor.Rng.initBuffer(platform, seed, io, replicated_sharding); - defer zml.Tensor.Rng.deinitBuffer(&rng_buffers); - var generated_token_slice: zml.Slice = try .alloc(allocator, zml.Shape.init(.{ .s = 1 }, .u32)); - defer generated_token_slice.free(allocator); - - { - var prefill_args = try prefill_exe.args(allocator); - defer prefill_args.deinit(allocator); - - var prefill_results = try prefill_exe.results(allocator); - defer prefill_results.deinit(allocator); - - // prepare device buffers for the prefill tokens and their positions - const prefill_tokens_slice: zml.Slice = try .alloc(allocator, .init(.{max_seq_len}, .u32)); - defer prefill_tokens_slice.free(allocator); - @memcpy(prefill_tokens_slice.items(u32)[0..prompt_tok.len], prompt_tok); - - var prefill_tokens_buffer: zml.Buffer = try .fromSlice(io, platform, prefill_tokens_slice, replicated_sharding); - defer prefill_tokens_buffer.deinit(); - var prefill_token_pos_buffer = try zml.Buffer.scalar(io, platform, 0, .u32, replicated_sharding); - defer prefill_token_pos_buffer.deinit(); - - prefill_args.set(.{ llama_buffers, prefill_tokens_buffer, prefill_token_pos_buffer, kv_cache_buffers, rng_buffers, attention_metadata_buffers }); - - prefill_exe.call(prefill_args, &prefill_results); - - prefill_results.fill(.{ &prefill_tokens_buffer, kv_cache_buffers, &rng_buffers }); - try prefill_tokens_buffer.toSlice(io, prefill_tokens_slice); - generated_token_slice.items(u32)[0] = prefill_tokens_slice.items(u32)[prompt_tok.len - 1]; - - std.log.info("\n\ngenerated tokens: {any}\n\n", .{prefill_tokens_slice.constItems(u32)[0..prompt_tok.len]}); - const out = try tokenizer_decoder.decode(prefill_tokens_slice.constItems(u32)[0..prompt_tok.len]); - std.log.info("\n\ngenerated: {s}\n\n", .{out}); - } - - // Prepare for token-by-token generation, - - var decode_args = try decode_exe.args(allocator); - defer decode_args.deinit(allocator); - - var decode_results = try decode_exe.results(allocator); - defer decode_results.deinit(allocator); - - // start with the token generated based on the full prompt. - var current_token_buffer: zml.Buffer = try .fromSlice(io, platform, generated_token_slice, replicated_sharding); - defer current_token_buffer.deinit(); - - const output_tokens_len = max_seq_len - prompt_tok.len - 1; - const now: std.Io.Timestamp = .now(io, .awake); - - // One token has alreadyh been generated by the prefill. - var num_tokens_generated: usize = 1; - - generation: for (0..output_tokens_len + 1) |i| { - // collect and print generated sequence - num_tokens_generated += 1; - const generated_token = generated_token_slice.items(u32)[0]; - if (try tokenizer_decoder.next(generated_token)) |chunk| { - try writer.writeAll(chunk); - try writer.flush(); - } - - // check for eos - if (i == output_tokens_len) break :generation; - switch (config.eos_token_id.value) { - .int => |eos| if (generated_token == @as(u32, @intCast(eos))) break :generation, - .ints => |eos_list| { - for (eos_list) |eos| { - if (generated_token == @as(u32, @intCast(eos))) break :generation; - } - }, - } - - // current token pos needs to go into a zml.Buffer - const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(prompt_tok.len + i)})); - var token_pos_buffer: zml.Buffer = try .fromSlice(io, platform, token_pos_slice, replicated_sharding); - defer token_pos_buffer.deinit(); - - // call to generate the next token - decode_args.set(.{ llama_buffers, current_token_buffer, token_pos_buffer, kv_cache_buffers, rng_buffers, attention_metadata_buffers }); - - decode_exe.call(decode_args, &decode_results); - - decode_results.fill(.{ ¤t_token_buffer, kv_cache_buffers, &rng_buffers }); - - // extract the generated token from the buffer - try current_token_buffer.toSlice(io, generated_token_slice); - } - const duration = now.untilNow(io, .awake); - std.debug.print("\n", .{}); - log.info("✅ Generated {} tokens in {f}: {:.3}tok/s", .{ - num_tokens_generated, - duration, - stdx.Io.Duration.hzFloat(stdx.Io.Duration.div(duration, num_tokens_generated)), - }); -} diff --git a/examples/llama/test.zig b/examples/llama/test.zig deleted file mode 100644 index 892c76f4..00000000 --- a/examples/llama/test.zig +++ /dev/null @@ -1,119 +0,0 @@ -const std = @import("std"); - -const async = @import("async"); -const zml = @import("zml"); -const Tensor = zml.Tensor; -const stdx = zml.stdx; -const flags = stdx.flags; - -const llama_mod = @import("./llama.zig"); -const LlamaLM = llama_mod.LlamaLM; - -pub fn main() !void { - try async.AsyncThread.main(std.heap.c_allocator, asyncMain); -} - -pub fn asyncMain() !void { - const CliArgs = struct { - pub const help = - \\ test-implementation --weights=llama3.8B.safetensors --config=config.json --reference=activation.safetensors - ; - weights: []const u8, - config: []const u8, - reference: []const u8, - num_heads: ?i64 = null, - num_kv_heads: ?i64 = null, - rope_freq_base: ?i64 = null, - }; - var gpa = std.heap.GeneralPurposeAllocator(.{ .thread_safe = true }){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - // Create ZML context - var context = try zml.Context.init(); - defer context.deinit(); - - // Select platform - const platform = context.autoPlatform(.{}); - - // Parse program args - var args = std.process.args(); - const cli_args = flags.parse(&args, CliArgs); - const model_file = cli_args.weights; - - // Memory arena dedicated to model shapes and weights - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const model_arena = arena_state.allocator(); - - std.log.info("Model file: {s}", .{model_file}); - - // Read model shapes. - var buffer_store = try zml.aio.detectFormatAndOpen(allocator, model_file); - defer buffer_store.deinit(); - - // Create the model and configure it. - var llama = try zml.aio.populateModel(LlamaLM, model_arena, buffer_store); - const num_heads: i64 = cli_args.num_heads orelse buffer_store.metadata("num_heads", .int) orelse @panic("--num_heads is required for this model"); - const num_kv_heads: i64 = cli_args.num_kv_heads orelse buffer_store.metadata("num_kv_heads", .int) orelse num_heads; - - const config = blk: { - var config_json_file = try async.File.open(cli_args.config, .{ .mode = .read_only }); - defer config_json_file.close() catch unreachable; - var reader = std.json.reader(allocator, config_json_file.reader()); - defer reader.deinit(); - const config_obj = try std.json.parseFromTokenSourceLeaky(LlamaLM.Config, allocator, &reader, .{ .ignore_unknown_fields = true }); - break :blk config_obj; - }; - std.log.info("Parsed llama config: {}", .{config}); - - const llama_config: LlamaLM.Config = .{ - .eos_token_id = config.eos_token_id, - .bos_token_id = config.bos_token_id, - .num_key_value_heads = @intCast(num_kv_heads), - .num_hidden_layers = @intCast(config.num_hidden_layers), - .num_attention_heads = @intCast(num_heads), - .max_position_embeddings = config.max_position_embeddings, - .rope_theta = config.rope_theta, - .rms_norm_eps = @floatCast(buffer_store.metadata("rms_norm_eps", .float) orelse 1e-5), - .hf_rope_impl = true, - }; - - const llama_options: LlamaLM.Options = .{ - .max_seq_len = 256, - .sampling_strategy = .{ - .topk = 1, - .temperature = 1.0, - }, - }; - - std.log.info("Parsed llama config: {}", .{llama_options}); - llama.init(llama_config, llama_options); - - // Load the weights. - var llama_weights = try zml.aio.loadBuffers(LlamaLM, .{ llama_config, llama_options }, buffer_store, model_arena, platform); - defer zml.aio.unloadBuffers(&llama_weights); - - // Load the activations. - var activation_buffer_store = try zml.aio.torch.open(allocator, cli_args.reference); - defer activation_buffer_store.deinit(); - - // Test implementation - try testImplementation(platform, llama, llama_weights, activation_buffer_store); -} - -fn testImplementation( - platform: *const zml.Platform, - llama: LlamaLM, - llama_weights: zml.Bufferized(LlamaLM), - buffer_store: zml.aio.BufferStore, -) !void { - try zml.testing.testLayer(platform, buffer_store, "embed_tokens", llama.model.embed_tokens, llama_weights.model.embed_tokens, 1e-3); - try zml.testing.testLayer(platform, buffer_store, "layers.0.self_attn.v_proj", llama.model.layers[0].self_attn.v_proj, llama_weights.model.layers[0].self_attn.v_proj, 1e-2); - try zml.testing.testLayer(platform, buffer_store, "layers.0.self_attn.q_proj", llama.model.layers[0].self_attn.q_proj, llama_weights.model.layers[0].self_attn.q_proj, 2e-2); - try zml.testing.testLayer(platform, buffer_store, "layers.0.self_attn.k_proj", llama.model.layers[0].self_attn.k_proj, llama_weights.model.layers[0].self_attn.k_proj, 2e-2); - try zml.testing.testLayer(platform, buffer_store, "layers.0.self_attn.o_proj", llama.model.layers[0].self_attn.o_proj, llama_weights.model.layers[0].self_attn.o_proj, 2e-2); - try zml.testing.testLayer(platform, buffer_store, "layers.0.mlp", llama.model.layers[0].mlp, llama_weights.model.layers[0].mlp, 1e-2); - try zml.testing.testLayer(platform, buffer_store, "layers.0.input_layernorm", llama.model.layers[0].input_layernorm, llama_weights.model.layers[0].input_layernorm, 1e-2); - try zml.testing.testLayer(platform, buffer_store, "layers.0.post_attention_layernorm", llama.model.layers[0].post_attention_layernorm, llama_weights.model.layers[0].post_attention_layernorm, 1e-2); -} diff --git a/examples/llm/BUILD.bazel b/examples/llm/BUILD.bazel new file mode 100644 index 00000000..7d1fa565 --- /dev/null +++ b/examples/llm/BUILD.bazel @@ -0,0 +1,31 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load("@rules_zig//zig:defs.bzl", "zig_binary") + +zig_binary( + name = "llm", + srcs = glob(["**/*.zig"]), + main = "main.zig", + visibility = ["//visibility:public"], + deps = ["//zml", "@linenoise"], +) + +zig_binary( + name = "lfm2_tests", + srcs = glob(["**/*.zig"]), + main = "models/lfm2_tests.zig", + visibility = ["//visibility:public"], + deps = ["//zml"], +) + +zig_binary( + name = "llama_tests", + srcs = glob(["**/*.zig"]), + main = "models/llama_tests.zig", + visibility = ["//visibility:public"], + deps = ["//zml"], +) + +build_test( + name = "test", + targets = [":llm"], +) diff --git a/examples/llm/README.md b/examples/llm/README.md new file mode 100644 index 00000000..1b581514 --- /dev/null +++ b/examples/llm/README.md @@ -0,0 +1,41 @@ +# LLM + +`//examples/llm` runs interactive or one-shot text generation from a model repository from HuggingFace. + +We support the following models, automatically detected from the `model_type` in the `config.json`: + +- Llama 3.1 +- Qwen 3.5 +- LFM 2.5 + +## Run + +To load a model from HuggingFace directly: + +```bash +# CPU +bazel run //examples/llm -- --model=hf://meta-llama/Llama-3.1-8B-Instruct +# CUDA +bazel run //examples/llm --@zml//platforms:cuda=true -- --model=hf://meta-llama/Llama-3.1-8B-Instruct +# ROCm +bazel run //examples/llm --@zml//platforms:rocm=true -- --model=hf://meta-llama/Llama-3.1-8B-Instruct +``` + +From a local directory: + +```bash +bazel run //examples/llm --@zml//platforms:cuda=true -- --model=/var/models/meta-llama/Llama-3.1-8B-Instruct/ +``` + +For a single non-interactive prompt: + +```bash +bazel run //examples/llm --@zml//platforms:cuda=true -- --model=hf://meta-llama/Llama-3.1-8B-Instruct --prompt="Write a haiku about Zig" +``` + +## Options + +- `--model=`: Required. Model repository to load. This can be a local path or a huggingface/S3 URI such as `hf://...` or `s3://...`. +- `--prompt=`: Optional. Runs a single prompt instead of opening the interactive chat loop. +- `--seqlen=`: Optional. Maximum sequence length. Defaults to `2048`. +- `--backend=`: Optional. Attention backend. If omitted, the program auto-selects one for the current platform. diff --git a/examples/llm/chat.zig b/examples/llm/chat.zig new file mode 100644 index 00000000..6bcaa8f4 --- /dev/null +++ b/examples/llm/chat.zig @@ -0,0 +1,143 @@ +const std = @import("std"); + +const c = @import("c"); +const zml = @import("zml"); + +const models = @import("models.zig"); + +const log = std.log.scoped(.llm); + +pub const prompt_prefix: [:0]const u8 = "> "; +pub const history_file_path: [:0]const u8 = ".llm_chat_history"; + +pub fn initHistory() void { + _ = c.linenoiseHistorySetMaxLen(1_000); + _ = c.linenoiseHistoryLoad(history_file_path.ptr); +} + +pub fn rememberPrompt(line: [*:0]const u8) void { + if (line[0] == 0) return; + if (c.linenoiseHistoryAdd(line) == 1) { + _ = c.linenoiseHistorySave(history_file_path.ptr); + } +} + +pub const Chat = struct { + allocator: std.mem.Allocator, + io: std.Io, + tokenizer: zml.tokenizer.Tokenizer, + repo: *models.Repository, + session: *models.Session, + tokens: std.ArrayList(u32), + + pub fn init(allocator: std.mem.Allocator, io: std.Io, tokenizer: zml.tokenizer.Tokenizer, repo: *models.Repository, session: *models.Session) !Chat { + return .{ + .allocator = allocator, + .io = io, + .tokenizer = tokenizer, + .repo = repo, + .session = session, + .tokens = try .initCapacity(allocator, session.maxTokens()), + }; + } + + pub fn deinit(self: *Chat) void { + self.tokens.deinit(self.allocator); + } + + pub fn runOnce(self: *Chat, prompt: []const u8) !void { + const prompt_tokens = try self.repo.tokenizePrompt(self.allocator, self.tokenizer, prompt); + defer self.allocator.free(prompt_tokens); + + var stdout = std.Io.File.stdout().writer(self.io, &.{}); + + log.info("Running prefill...", .{}); + try self.tokens.appendSlice(self.allocator, prompt_tokens); + try self.session.runPrefill(self.tokens.items); + + log.info("Running decode...", .{}); + try self.session.runDecode(&self.tokens, &stdout.interface); + + try stdout.interface.writeAll("\n\n"); + try stdout.interface.flush(); + } + + pub fn runInteractive(self: *Chat, initial_prompt: []const u8) !void { + var turn_tokens = try self.repo.tokenizePrompt(self.allocator, self.tokenizer, initial_prompt); + defer self.allocator.free(turn_tokens); + + var stdout = std.Io.File.stdout().writer(self.io, &.{}); + + while (self.tokens.items.len <= self.session.maxTokens()) { + try self.runPrefill(turn_tokens, &stdout.interface); + try self.runDecodeTurn(&stdout.interface); + + turn_tokens = if (self.tokens.items.len >= self.session.maxTokens()) b: { + self.allocator.free(turn_tokens); + break :b try self.allocator.alloc(u32, 0); + } else try self.readNextTurn(turn_tokens, &stdout.interface) orelse return; + if (self.tokens.items.len + turn_tokens.len >= self.session.maxTokens()) { + log.warn("Not enough tokens remaining ({} available, {} needed). Consider using a higher seqlen, for example: `--seqlen={d}`.", .{ + self.session.maxTokens() -| self.tokens.items.len, + turn_tokens.len, + std.math.ceilPowerOfTwo(u32, self.session.maxTokens() + 1) catch std.math.maxInt(u32), + }); + return; + } + } + } + + fn runPrefill(self: *Chat, prompt_tokens: []const u32, stdout: *std.Io.Writer) !void { + if (prompt_tokens.len + self.tokens.items.len > self.session.maxTokens()) { + return error.PromptTooLong; + } + try stdout.writeAll("\x1b[2mprefill...\x1b[0m"); + try stdout.flush(); + + try self.tokens.appendSlice(self.allocator, prompt_tokens); + try self.session.runPrefill(self.tokens.items); + + try stdout.writeAll("\r \r"); + try stdout.flush(); + } + + fn runDecodeTurn(self: *Chat, stdout: *std.Io.Writer) !void { + const decode_start: std.Io.Timestamp = .now(self.io, .awake); + const decode_pos_before = self.tokens.items.len; + try self.session.runDecode(&self.tokens, stdout); + const decode_duration = decode_start.untilNow(self.io, .awake); + const decode_tokens: u64 = self.tokens.items.len - decode_pos_before; + + try stdout.writeAll("\n\n"); + try stdout.print("\x1b[36mdecode\x1b[0m \x1b[2m{f} · {:.1}tok/s\x1b[0m\n\n\n", .{ + decode_duration, + tokensPerSecond(decode_duration, decode_tokens), + }); + try stdout.flush(); + } + + fn readNextTurn(self: *Chat, previous_turn_tokens: []const u32, stdout: *std.Io.Writer) !?[]const u32 { + while (true) { + const line = c.linenoise(prompt_prefix.ptr) orelse { + try stdout.print("\x1b[2m{}/{} tokens used\x1b[0m\n\n", .{ self.tokens.items.len, self.session.maxTokens() }); + try stdout.flush(); + return null; + }; + defer c.linenoiseFree(line); + + rememberPrompt(line); + const input = std.mem.sliceTo(line, 0); + if (input.len == 0) continue; + + self.allocator.free(previous_turn_tokens); + return try self.repo.tokenizeTurn(self.allocator, self.tokenizer, input); + } + } +}; + +fn tokensPerSecond(duration: std.Io.Duration, tokens: u64) f64 { + if (tokens == 0) return 0; + const seconds = @as(f64, @floatFromInt(duration.toNanoseconds())) / 1e9; + if (seconds <= 0) return 0; + return @as(f64, @floatFromInt(tokens)) / seconds; +} diff --git a/examples/llm/main.zig b/examples/llm/main.zig new file mode 100644 index 00000000..b4fa52eb --- /dev/null +++ b/examples/llm/main.zig @@ -0,0 +1,186 @@ +const std = @import("std"); + +const c = @import("c"); +const zml = @import("zml"); +const stdx = zml.stdx; + +const chat = @import("chat.zig"); +const models = @import("models.zig"); + +pub const std_options: std.Options = .{ + .log_level = .info, +}; + +const log = std.log.scoped(.llm); + +const Args = struct { + model: []const u8, + prompt: ?[]const u8 = null, + seqlen: u32 = 2048, + backend: ?zml.attention.attention.Backend = null, + single: bool = false, + + pub const help = + \\Use llm --model= [options] + \\ + \\ Run text generation with a model selected from `config.json.model_type`. + \\ + \\ Options: + \\ --model= Path to the model repository (required) + \\ --prompt= Prompt to use for generation (default: none) + \\ --seqlen= Sequence length (default: 2048) + \\ --backend= Attention backend to use ([vanilla, cuda_fa2, cuda_fa3], default: auto-selection) + \\ --single Create a single kernel encompassing all the layers when supported + \\ + ; +}; + +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const args = stdx.flags.parse(init.minimal.args, Args); + + // + // Virtual File Systems + // + var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); + defer vfs_file.deinit(); + + var http_client: std.http.Client = .{ .allocator = allocator, .io = init.io }; + defer http_client.deinit(); + + var hf_vfs: zml.io.VFS.HF = try .auto(allocator, init.io, &http_client, init.environ_map); + defer hf_vfs.deinit(); + + var s3_vfs: zml.io.VFS.S3 = try .auto(allocator, init.io, &http_client, init.environ_map); + defer s3_vfs.deinit(); + + var vfs: zml.io.VFS = try .init(allocator, init.io); + defer vfs.deinit(); + + try vfs.register("file", vfs_file.io()); + try vfs.register("hf", hf_vfs.io()); + try vfs.register("s3", s3_vfs.io()); + + const io = vfs.io(); + + const platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator); + + log.info("\n{f}", .{platform.fmtVerbose()}); + + log.info("Resolving model repository..", .{}); + const repo = try zml.safetensors.resolveModelRepo(io, args.model); + + const model_type = try models.detectModelType(allocator, io, repo); + log.info("Detected model type: {}", .{model_type}); + + log.info("Initializing model..", .{}); + var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); + defer registry.deinit(); + + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + defer store.deinit(); + + var model = try models.Repository.init(model_type, allocator, io, repo, store.view()); + defer model.deinit(allocator); + + const backend = args.backend orelse b: { + const selected = zml.attention.attention.Backend.auto(platform); + log.info("Selected backend: {}", .{selected}); + break :b selected; + }; + + var progress = std.Progress.start(io, .{ .root_name = args.model }); + errdefer progress.end(); + + const tokenizer = try loadTokenizer(allocator, io, repo, &progress); + defer { + var tk = tokenizer; + tk.deinit(); + } + + const tp_mesh: zml.sharding.LogicalMesh = try .init("tp_mesh", .{ .model = .high_bandwidth }); + const tp_strategy: zml.sharding.Strategy = try .suggest(tp_mesh, platform.physical_mesh); + const shardings: models.Shardings = .{ + .replicated = try zml.sharding.replicatedSharding(platform), + .model = try .initFromStrategy(platform, tp_mesh, tp_strategy), + }; + + var model_buffers = try model.loadBuffers(allocator, io, platform, &store, &progress, shardings); + defer model_buffers.deinit(allocator); + + var session = try model.initSession( + allocator, + io, + platform, + &model_buffers, + tokenizer, + .{ + .seqlen = args.seqlen, + .backend = backend, + .single = args.single, + }, + &progress, + shardings, + ); + defer session.deinit(); + + progress.end(); + + try printLogo(io); + + const interactive = args.prompt == null; + const prompt = if (args.prompt) |p| try allocator.dupe(u8, p) else blk: { + chat.initHistory(); + const line = c.linenoise(chat.prompt_prefix) orelse return; + defer c.linenoiseFree(line); + chat.rememberPrompt(line); + break :blk try allocator.dupe(u8, std.mem.sliceTo(line, 0)); + }; + defer allocator.free(prompt); + + var llm_chat = try chat.Chat.init(allocator, io, tokenizer, &model, &session); + defer llm_chat.deinit(); + + if (interactive) { + try llm_chat.runInteractive(prompt); + } else { + try llm_chat.runOnce(prompt); + } +} + +fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, progress: *std.Progress.Node) !zml.tokenizer.Tokenizer { + progress.increaseEstimatedTotalItems(1); + var node = progress.start("Loading tokenizer...", 1); + defer node.end(); + const now: std.Io.Timestamp = .now(io, .awake); + defer log.info("Loaded tokenizer [{f}]", .{now.untilNow(io, .awake)}); + const bytes = b: { + const file = try dir.openFile(io, "tokenizer.json", .{}); + defer file.close(io); + var reader = file.reader(io, &.{}); + break :b try reader.interface.readAlloc(allocator, try file.length(io)); + }; + defer allocator.free(bytes); + + return try .fromBytes(allocator, io, bytes); +} + +pub fn printLogo(io: std.Io) !void { + const LOGO = + \\ + \\ + \\ ███████╗███╗ ███╗██╗ + \\ ╚══███╔╝████╗ ████║██║ + \\ ███╔╝ ██╔████╔██║██║ + \\ ███╔╝ ██║╚██╔╝██║██║ .ai + \\ ███████╗██║ ╚═╝ ██║███████╗ + \\ ╚══════╝╚═╝ ╚═╝╚══════╝ + \\ + \\ + \\ + ; + var writer = std.Io.File.stdout().writer(io, &.{}); + try writer.interface.writeAll(LOGO); + try writer.interface.flush(); +} diff --git a/examples/llm/models.zig b/examples/llm/models.zig new file mode 100644 index 00000000..70da703b --- /dev/null +++ b/examples/llm/models.zig @@ -0,0 +1,165 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const common = @import("models/common.zig"); +const SessionOptions = common.SessionOptions; +pub const Shardings = common.Shardings; +pub const parseConfig = common.parseConfig; +pub const lfm2 = @import("models/lfm2.zig"); +pub const llama = @import("models/llama.zig"); +pub const qwen3_5 = @import("models/qwen3_5.zig"); + +pub const ModelType = enum { + lfm2, + llama, + qwen3_5, +}; + +const RawConfig = struct { + model_type: []const u8, +}; + +pub const Repository = union(ModelType) { + lfm2: lfm2.Repository, + llama: llama.Repository, + qwen3_5: qwen3_5.Repository, + + pub fn init(model_type: ModelType, allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir, store: zml.io.TensorStore.View) !Repository { + return switch (model_type) { + .lfm2 => .{ .lfm2 = try lfm2.Repository.init(allocator, io, repo, store) }, + .llama => .{ .llama = try llama.Repository.init(allocator, io, repo, store) }, + .qwen3_5 => .{ .qwen3_5 = try qwen3_5.Repository.init(allocator, io, repo, store) }, + }; + } + + pub fn deinit(self: *Repository, allocator: std.mem.Allocator) void { + switch (self.*) { + inline else => |*m| m.deinit(allocator), + } + } + + pub fn loadBuffers(self: *Repository, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, shardings: Shardings) !Buffers { + return switch (self.*) { + .lfm2 => |*m| .{ .lfm2 = try m.loadBuffers(allocator, io, platform, store, progress, shardings) }, + .llama => |*m| .{ .llama = try m.loadBuffers(allocator, io, platform, store, progress, shardings) }, + .qwen3_5 => |*m| .{ .qwen3_5 = try m.loadBuffers(allocator, io, platform, store, progress, shardings) }, + }; + } + + pub fn tokenizePrompt(self: *const Repository, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]const u32 { + return switch (self.*) { + inline else => |*m| m.tokenizePrompt(allocator, tokenizer, prompt), + }; + } + + pub fn tokenizeTurn(self: *const Repository, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]const u32 { + return switch (self.*) { + inline else => |*m| m.tokenizeTurn(allocator, tokenizer, prompt), + }; + } + + pub fn initSession( + self: *const Repository, + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + model_buffers: *Buffers, + tokenizer: zml.tokenizer.Tokenizer, + opts: SessionOptions, + progress: *std.Progress.Node, + shardings: Shardings, + ) !Session { + const inner: Session.Inner = switch (self.*) { + .lfm2 => |repo| .{ .lfm2 = try lfm2.Session.init( + allocator, + io, + platform, + &model_buffers.lfm2, + tokenizer, + repo, + opts, + progress, + shardings, + ) }, + .llama => |repo| .{ .llama = try llama.Session.init( + allocator, + io, + platform, + &model_buffers.llama, + tokenizer, + repo, + opts, + progress, + shardings, + ) }, + .qwen3_5 => |repo| .{ .qwen3_5 = try qwen3_5.Session.init( + allocator, + io, + platform, + &model_buffers.qwen3_5, + tokenizer, + repo, + opts, + progress, + shardings, + ) }, + }; + return .{ .inner = inner, .seqlen = opts.seqlen }; + } +}; + +pub const Buffers = union(ModelType) { + lfm2: lfm2.Buffers, + llama: llama.Buffers, + qwen3_5: qwen3_5.Buffers, + + pub fn deinit(self: *Buffers, allocator: std.mem.Allocator) void { + switch (self.*) { + .lfm2 => |*b| lfm2.Repository.unloadBuffers(b, allocator), + .llama => |*b| llama.Repository.unloadBuffers(b, allocator), + .qwen3_5 => |*b| qwen3_5.Repository.unloadBuffers(b, allocator), + } + } +}; + +pub const Session = struct { + const Inner = union(ModelType) { + lfm2: lfm2.Session, + llama: llama.Session, + qwen3_5: qwen3_5.Session, + }; + + inner: Inner, + seqlen: u32, + + pub fn deinit(self: *Session) void { + switch (self.inner) { + inline else => |*s| s.deinit(), + } + } + + pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { + try switch (self.inner) { + inline else => |*s| s.runPrefill(all_tokens), + }; + } + + pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), writer: *std.Io.Writer) !void { + try switch (self.inner) { + inline else => |*s| s.runDecode(all_tokens, writer), + }; + } + + pub fn maxTokens(self: *const Session) u32 { + return self.seqlen; + } +}; + +pub fn detectModelType(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir) !ModelType { + const parsed = try parseConfig(RawConfig, allocator, io, repo); + defer parsed.deinit(); + if (std.meta.stringToEnum(ModelType, parsed.value.model_type)) |model_type| return model_type; + if (std.mem.eql(u8, parsed.value.model_type, "qwen3_next")) return .qwen3_5; + return error.UnknownModelType; +} diff --git a/examples/llm/models/common.zig b/examples/llm/models/common.zig new file mode 100644 index 00000000..953061dd --- /dev/null +++ b/examples/llm/models/common.zig @@ -0,0 +1,30 @@ +const std = @import("std"); + +const zml = @import("zml"); + +pub const SessionOptions = struct { + seqlen: u32, + backend: zml.attention.attention.Backend, + single: bool, +}; + +pub const Shardings = struct { + replicated: zml.sharding.Sharding, + model: zml.sharding.Sharding, + + pub fn all(self: *const Shardings) [2]zml.sharding.Sharding { + return .{ self.replicated, self.model }; + } +}; + +pub fn parseConfig(comptime T: type, allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !std.json.Parsed(T) { + const file = try dir.openFile(io, "config.json", .{}); + defer file.close(io); + + var buffer: [256]u8 = undefined; + var file_reader = file.reader(io, &buffer); + var reader: std.json.Reader = .init(allocator, &file_reader.interface); + defer reader.deinit(); + + return try std.json.parseFromTokenSource(T, allocator, &reader, .{ .ignore_unknown_fields = true }); +} diff --git a/examples/llm/models/lfm2.zig b/examples/llm/models/lfm2.zig new file mode 100644 index 00000000..bf186ace --- /dev/null +++ b/examples/llm/models/lfm2.zig @@ -0,0 +1,10 @@ +pub const inference = @import("lfm2/inference.zig"); +pub const CompilationOptions = inference.CompilationOptions; +pub const model = @import("lfm2/model.zig"); +pub const Config = model.Config; +pub const Buffers = model.Buffers; +pub const Model = model.Model; +pub const repository = @import("lfm2/repository.zig"); +pub const Repository = repository.Repository; +pub const session = @import("lfm2/session.zig"); +pub const Session = session.Session; diff --git a/examples/lfm/inference.zig b/examples/llm/models/lfm2/inference.zig similarity index 78% rename from examples/lfm/inference.zig rename to examples/llm/models/lfm2/inference.zig index 917e80ef..0095b822 100644 --- a/examples/lfm/inference.zig +++ b/examples/llm/models/lfm2/inference.zig @@ -5,6 +5,7 @@ const stdx = zml.stdx; const attention = zml.attention.attention; const model = @import("model.zig"); +const Shardings = @import("../common.zig").Shardings; const log = std.log.scoped(.lfm); @@ -51,14 +52,13 @@ pub const Args = struct { allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, - model_buffers: *zml.Bufferized(model.Model), + model_buffers: *model.Buffers, tokens_buf: *zml.Buffer, tokens_pos_buf: *zml.Buffer, actual_seq_len_buf: *zml.Buffer, rng_buf: *zml.Bufferized(zml.Tensor.Rng), cache_buffers: *zml.Bufferized(model.Cache), attention_metadata_buffers: zml.Bufferized(attention.Metadata), - sharding: zml.sharding.Sharding, }; pub const Inference = struct { @@ -73,19 +73,20 @@ pub const Inference = struct { opts: CompilationOptions, seqlen: u32, progress: *std.Progress.Node, - sharding: zml.sharding.Sharding, + shardings: Shardings, ) !Inference { if (opts.single) { - const prefill_exe = try compileSingleKernelExe(allocator, io, platform, mdl, opts, seqlen, true, progress, sharding); - const decode_exe = try compileSingleKernelExe(allocator, io, platform, mdl, opts, 1, false, progress, sharding); + const prefill_exe = try compileSingleKernelExe(allocator, io, platform, mdl, opts, seqlen, true, progress, shardings); + const decode_exe = try compileSingleKernelExe(allocator, io, platform, mdl, opts, 1, false, progress, shardings); return .{ .prefill = .{ .single = prefill_exe }, .decode = .{ .single = decode_exe }, }; } else { - const prefill_exe = try compileComposedKernelExe(allocator, io, platform, mdl, opts, seqlen, true, progress, sharding); - const decode_exe = try compileComposedKernelExe(allocator, io, platform, mdl, opts, 1, false, progress, sharding); + const prefill_exe = try compileComposedKernelExe(allocator, io, platform, mdl, opts, seqlen, true, progress, shardings); + const decode_exe = try compileComposedKernelExe(allocator, io, platform, mdl, opts, 1, false, progress, shardings); + return .{ .prefill = .{ .composed = prefill_exe }, .decode = .{ .composed = decode_exe }, @@ -154,7 +155,17 @@ pub const SingleKernelExe = struct { } }; -fn compileSingleKernelExe(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, mdl: model.Model, opts: CompilationOptions, seqlen: u32, is_prefill: bool, progress: *std.Progress.Node, sharding: zml.sharding.Sharding) !SingleKernelExe { +fn compileSingleKernelExe( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + mdl: model.Model, + opts: CompilationOptions, + seqlen: u32, + is_prefill: bool, + progress: *std.Progress.Node, + shardings: Shardings, +) !SingleKernelExe { progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling single kernel...", 1); defer node.end(); @@ -165,13 +176,23 @@ fn compileSingleKernelExe(allocator: std.mem.Allocator, io: std.Io, platform: *z const tokens: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32); const token_position_offset: zml.Tensor = .init(.{ .batch = opts.batch_dim }, .u32); + const all_shardings = shardings.all(); const exe = try platform.compile( allocator, io, mdl, .forward, - .{ tokens, token_position_offset, actual_seq_len, opts.rng, opts.cache, opts.attention_metadata, opts.attention_parameters, model.ConvParameters{ .is_prefill = is_prefill } }, - .{ .shardings = &.{sharding} }, + .{ + tokens, + token_position_offset, + actual_seq_len, + opts.rng, + opts.cache, + opts.attention_metadata, + opts.attention_parameters, + model.ConvParameters{ .is_prefill = is_prefill }, + }, + .{ .shardings = &all_shardings }, ); return .{ .exe = exe }; @@ -190,13 +211,7 @@ pub const ComposedKernelExe = struct { self.lm_head.deinit(); } - /// Orchestrates the 4 sub-executables to replicate Model.forward behavior. - /// - /// Inputs and outputs mirror the monolithic Model.forward signature: - /// - Inputs: model_buffers, tokens, position_offset, actual_seq_len, rng, cache, attention_metadata - /// - Outputs: tokens (mutated), cache (mutated), rng (mutated) pub fn run(self: *const ComposedKernelExe, args: Args) !void { - // Step 1: embed_tokens — inputs: [weight, tokens] → output: [hidden] var hidden_buf: zml.Buffer = b: { var exe_args = try self.embed_tokens.args(args.allocator); defer exe_args.deinit(args.allocator); @@ -209,10 +224,10 @@ pub const ComposedKernelExe = struct { }; defer hidden_buf.deinit(); - // Step 2: iterate over layers, dispatching to conv_layer or attn_layer exe. - var conv_cache_index_buf: zml.Buffer = try .scalar(args.io, args.platform, @as(u32, 0), .u32, args.sharding); + const replicated_sharding = try zml.sharding.replicatedSharding(args.platform); + var conv_cache_index_buf: zml.Buffer = try .scalar(args.io, args.platform, @as(u32, 0), .u32, replicated_sharding); defer conv_cache_index_buf.deinit(); - var kv_cache_index_buf: zml.Buffer = try .scalar(args.io, args.platform, @as(u32, 0), .u32, args.sharding); + var kv_cache_index_buf: zml.Buffer = try .scalar(args.io, args.platform, @as(u32, 0), .u32, replicated_sharding); defer kv_cache_index_buf.deinit(); for (args.model_buffers.layers) |layer_bufs| { @@ -226,10 +241,18 @@ pub const ComposedKernelExe = struct { var results = try exe.results(args.allocator); defer results.deinit(args.allocator); - exe_args.set(.{ layer_bufs, &hidden_buf, args.tokens_pos_buf, args.actual_seq_len_buf, args.cache_buffers, &conv_cache_index_buf, &kv_cache_index_buf, args.attention_metadata_buffers }); + exe_args.set(.{ + layer_bufs, + &hidden_buf, + args.tokens_pos_buf, + args.actual_seq_len_buf, + args.cache_buffers, + &conv_cache_index_buf, + &kv_cache_index_buf, + args.attention_metadata_buffers, + }); exe.call(exe_args, &results); - // Outputs: { hidden, cache, conv_cache_index, kv_cache_index } var new_hidden, var new_cache, var new_conv_cache_index, var new_kv_cache_index = results.get(struct { zml.Buffer, zml.Bufferized(model.Cache), @@ -242,7 +265,6 @@ pub const ComposedKernelExe = struct { replaceBuffer(&kv_cache_index_buf, &new_kv_cache_index); } - // Step 3: lm_head — inputs: [lm_head buffers, hidden, embed_tokens.weight, tokens, rng] → outputs: [tokens, rng] { var exe_args = try self.lm_head.args(args.allocator); defer exe_args.deinit(args.allocator); @@ -251,7 +273,10 @@ pub const ComposedKernelExe = struct { exe_args.set(.{ args.model_buffers.lm_head, hidden_buf, args.model_buffers.embed_tokens, args.tokens_buf, args.rng_buf }); self.lm_head.call(exe_args, &results); - var new_tokens, var new_rng = results.get(struct { zml.Buffer, zml.Bufferized(zml.Tensor.Rng) }); + var new_tokens, var new_rng = results.get(struct { + zml.Buffer, + zml.Bufferized(zml.Tensor.Rng), + }); replaceBuffer(args.tokens_buf, &new_tokens); replaceBuffer(&args.rng_buf._state, &new_rng._state); } @@ -279,7 +304,7 @@ fn sameBufferHandle(a: zml.Buffer, b: zml.Buffer) bool { return true; } -pub fn compileComposedKernelExe( +fn compileComposedKernelExe( allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, @@ -288,13 +313,12 @@ pub fn compileComposedKernelExe( seqlen: u32, is_prefill: bool, progress: *std.Progress.Node, - sharding: zml.sharding.Sharding, + shardings: Shardings, ) !ComposedKernelExe { - const embed_tokens_exe = try compileEmbedTokens(allocator, io, platform, mdl.embed_tokens, opts, seqlen, progress, sharding); - const conv_layer_exe = try compileConvLayer(allocator, io, platform, mdl, opts, seqlen, is_prefill, progress, sharding); - const attn_layer_exe = try compileAttnLayer(allocator, io, platform, mdl, opts, seqlen, is_prefill, progress, sharding); - const lm_head_exe = try compileLmHead(allocator, io, platform, mdl, opts, seqlen, progress, sharding); - + const embed_tokens_exe = try compileEmbedTokens(allocator, io, platform, mdl.embed_tokens, opts, seqlen, progress, shardings); + const conv_layer_exe = try compileConvLayer(allocator, io, platform, mdl, opts, seqlen, is_prefill, progress, shardings); + const attn_layer_exe = try compileAttnLayer(allocator, io, platform, mdl, opts, seqlen, is_prefill, progress, shardings); + const lm_head_exe = try compileLmHead(allocator, io, platform, mdl, opts, seqlen, progress, shardings); return .{ .embed_tokens = embed_tokens_exe, .conv_layer = conv_layer_exe, @@ -303,7 +327,16 @@ pub fn compileComposedKernelExe( }; } -fn compileEmbedTokens(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, embed_tokens: model.TokenEmbedding, opts: CompilationOptions, seqlen: u32, progress: *std.Progress.Node, sharding: zml.sharding.Sharding) !zml.Exe { +fn compileEmbedTokens( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + embed_tokens: model.TokenEmbedding, + opts: CompilationOptions, + seqlen: u32, + progress: *std.Progress.Node, + shardings: Shardings, +) !zml.Exe { progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling embed_tokens...", 1); defer node.end(); @@ -311,17 +344,27 @@ fn compileEmbedTokens(allocator: std.mem.Allocator, io: std.Io, platform: *zml.P defer log.info("Compiled embed_tokens [{f}]", .{now.untilNow(io, .awake)}); const tokens: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32); - return platform.compile(allocator, io, embed_tokens, .forward, .{tokens}, .{ .shardings = &.{sharding} }); + const all_shardings = shardings.all(); + return platform.compile(allocator, io, embed_tokens, .forward, .{tokens}, .{ .shardings = &all_shardings }); } -fn compileConvLayer(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, mdl: model.Model, opts: CompilationOptions, seqlen: u32, is_prefill: bool, progress: *std.Progress.Node, sharding: zml.sharding.Sharding) !zml.Exe { +fn compileConvLayer( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + mdl: model.Model, + opts: CompilationOptions, + seqlen: u32, + is_prefill: bool, + progress: *std.Progress.Node, + shardings: Shardings, +) !zml.Exe { progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling conv layer...", 1); defer node.end(); const now: std.Io.Timestamp = .now(io, .awake); defer log.info("Compiled conv layer [{f}]", .{now.untilNow(io, .awake)}); - // Find the first conv layer as a representative for compilation. const conv_layer = for (mdl.layers) |layer| { if (layer.operator == .conv) break layer; } else unreachable; @@ -332,6 +375,7 @@ fn compileConvLayer(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Pla const conv_cache_index: zml.Tensor = .init(.{}, .u32); const kv_cache_index: zml.Tensor = .init(.{}, .u32); + const all_shardings = shardings.all(); return platform.compile(allocator, io, conv_layer, .forward, .{ hidden, token_position_offset, @@ -342,17 +386,26 @@ fn compileConvLayer(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Pla opts.attention_metadata, opts.attention_parameters, model.ConvParameters{ .is_prefill = is_prefill }, - }, .{ .shardings = &.{sharding} }); + }, .{ .shardings = &all_shardings }); } -fn compileAttnLayer(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, mdl: model.Model, opts: CompilationOptions, seqlen: u32, is_prefill: bool, progress: *std.Progress.Node, sharding: zml.sharding.Sharding) !zml.Exe { +fn compileAttnLayer( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + mdl: model.Model, + opts: CompilationOptions, + seqlen: u32, + is_prefill: bool, + progress: *std.Progress.Node, + shardings: Shardings, +) !zml.Exe { progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling attn layer...", 1); defer node.end(); const now: std.Io.Timestamp = .now(io, .awake); defer log.info("Compiled attn layer [{f}]", .{now.untilNow(io, .awake)}); - // Find the first attention layer as a representative for compilation. const attn_layer = for (mdl.layers) |layer| { if (layer.operator == .self_attn) break layer; } else unreachable; @@ -363,6 +416,7 @@ fn compileAttnLayer(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Pla const conv_cache_index: zml.Tensor = .init(.{}, .u32); const kv_cache_index: zml.Tensor = .init(.{}, .u32); + const all_shardings = shardings.all(); return platform.compile(allocator, io, attn_layer, .forward, .{ hidden, token_position_offset, @@ -373,10 +427,19 @@ fn compileAttnLayer(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Pla opts.attention_metadata, opts.attention_parameters, model.ConvParameters{ .is_prefill = is_prefill }, - }, .{ .shardings = &.{sharding} }); + }, .{ .shardings = &all_shardings }); } -fn compileLmHead(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, mdl: model.Model, opts: CompilationOptions, seqlen: u32, progress: *std.Progress.Node, sharding: zml.sharding.Sharding) !zml.Exe { +fn compileLmHead( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + mdl: model.Model, + opts: CompilationOptions, + seqlen: u32, + progress: *std.Progress.Node, + shardings: Shardings, +) !zml.Exe { progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling lm_head...", 1); defer node.end(); @@ -385,6 +448,6 @@ fn compileLmHead(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platfo const hidden: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()); const tokens: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32); - - return platform.compile(allocator, io, mdl.lm_head, .forward, .{ hidden, mdl.embed_tokens, tokens, opts.rng }, .{ .shardings = &.{sharding} }); + const all_shardings = shardings.all(); + return platform.compile(allocator, io, mdl.lm_head, .forward, .{ hidden, mdl.embed_tokens, tokens, opts.rng }, .{ .shardings = &all_shardings }); } diff --git a/examples/lfm/model.zig b/examples/llm/models/lfm2/model.zig similarity index 66% rename from examples/lfm/model.zig rename to examples/llm/models/lfm2/model.zig index 417fd62d..4f2b61b7 100644 --- a/examples/lfm/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -2,15 +2,53 @@ const std = @import("std"); const zml = @import("zml"); const stdx = zml.stdx; -const Tensor = zml.Tensor; -const Shape = zml.Shape; -const attention = zml.attention.attention; -const cfg = @import("config.zig"); -pub const Config = cfg.Config; +pub const parseConfig = @import("../common.zig").parseConfig; +pub const Shardings = @import("../common.zig").Shardings; const log = std.log.scoped(.lfm); +pub const Config = struct { + architectures: []const []const u8, + block_auto_adjust_ff_dim: bool, + block_dim: u32, + block_ff_dim: u32, + block_ffn_dim_multiplier: f32, + block_mlp_init_scale: f32, + block_multiple_of: u32, + block_norm_eps: f32, + block_out_init_scale: f32, + block_use_swiglu: bool, + block_use_xavier_init: bool, + bos_token_id: u32, + conv_L_cache: u32, + conv_bias: bool, + conv_dim: u32, + conv_use_xavier_init: bool, + dtype: []const u8, + eos_token_id: u32, + hidden_size: u32, + initializer_range: f32, + intermediate_size: u32, + layer_types: []const []const u8, + max_position_embeddings: u32, + model_type: []const u8, + norm_eps: f32, + num_attention_heads: u32, + num_heads: u32, + num_hidden_layers: u32, + num_key_value_heads: u32, + pad_token_id: u32, + rope_theta: f32, + tie_embedding: bool, + transformers_version: []const u8, + use_cache: bool, + use_pos_enc: bool, + vocab_size: u32, +}; + +pub const Buffers = zml.Bufferized(Model); + pub const Model = struct { embed_tokens: TokenEmbedding, lm_head: LmHead, @@ -51,12 +89,11 @@ pub const Model = struct { platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, + shardings: Shardings, ) !zml.Bufferized(Model) { progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); var total_bytes: usize = 0; - defer { const took = now.untilNow(io, .awake); const took_ns: usize = @max(1, @as(usize, @intCast(took.toNanoseconds()))); @@ -66,8 +103,7 @@ pub const Model = struct { total_bytes * std.time.ns_per_s / took_ns, }); } - - const sharding = try zml.sharding.replicatedSharding(platform); + const all_shardings = shardings.all(); return zml.io.load(Model, self, allocator, io, platform, .{ .dma_chunks = 8, .dma_chunk_size = 128 * zml.MiB, @@ -75,7 +111,7 @@ pub const Model = struct { .store = store, .parallelism = 16, .total_bytes = &total_bytes, - .shardings = &.{sharding}, + .shardings = &all_shardings, }); } @@ -89,29 +125,38 @@ pub const Model = struct { } pub fn forward( - // self: Model, - tokens: Tensor, - tokens_position_offset: Tensor, - actual_seq_len: Tensor, - rng: Tensor.Rng, + tokens: zml.Tensor, + tokens_position_offset: zml.Tensor, + actual_seq_len: zml.Tensor, + rng: zml.Tensor.Rng, cache_: Cache, - attention_metadata: attention.Metadata, - attention_parameters: attention.Parameters, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, conv_parameters: ConvParameters, - ) struct { Tensor, Cache, Tensor.Rng } { + ) struct { zml.Tensor, Cache, zml.Tensor.Rng } { stdx.debug.assert(tokens.shape().hasTags(.{ .batch, .seq }), "Tokens should have tags {{.batch, .seq}}, got {f}", .{tokens.shape()}); const embeds = self.embed_tokens.forward(tokens); var hidden = embeds; var cache = cache_; - var conv_cache_index = Tensor.scalar(@as(u32, 0), .u32); - var kv_cache_index = Tensor.scalar(@as(u32, 0), .u32); + // Incremented by the individual layers. + var conv_cache_index = zml.Tensor.scalar(@as(u32, 0), .u32); + var kv_cache_index = zml.Tensor.scalar(@as(u32, 0), .u32); for (self.layers) |layer| { - hidden, cache, conv_cache_index, kv_cache_index = layer.forward(hidden, tokens_position_offset, actual_seq_len, cache, conv_cache_index, kv_cache_index, attention_metadata, attention_parameters, conv_parameters); + hidden, cache, conv_cache_index, kv_cache_index = layer.forward( + hidden, + tokens_position_offset, + actual_seq_len, + cache, + conv_cache_index, + kv_cache_index, + attention_metadata, + attention_parameters, + conv_parameters, + ); } - const new_tokens, const new_rng = self.lm_head.forward(hidden, self.embed_tokens, tokens, rng); return .{ new_tokens, cache.reuseBuffer(cache_), new_rng }; } @@ -122,19 +167,19 @@ pub const Model = struct { }; pub const TokenEmbedding = struct { - weight: Tensor, + weight: zml.Tensor, pub fn init(store: zml.io.TensorStore.View) TokenEmbedding { return .{ .weight = store.createTensor("weight", .{ .voc, .d }, null) }; } - pub fn forward(self: TokenEmbedding, tokens: Tensor) Tensor { + pub fn forward(self: TokenEmbedding, tokens: zml.Tensor) zml.Tensor { stdx.debug.assert(tokens.dtype().isInteger(), "TokenEmbedding expects an integer input, received: {f}", .{tokens}); - stdx.debug.assert(self.weight.rank() == 2, "TokenEmbedding expects it's weight Tensor to be a 2D matrix, got {f}", .{self.weight}); + stdx.debug.assert(self.weight.rank() == 2, "TokenEmbedding expects it's weight zml.Tensor to be a 2D matrix, got {f}", .{self.weight}); return self.weight.gather(.{ .voc = tokens }, .{}); } - pub fn unembed(self: TokenEmbedding, embeds: Tensor) Tensor { + pub fn unembed(self: TokenEmbedding, embeds: zml.Tensor) zml.Tensor { stdx.debug.assert(embeds.shape().hasTags(.{.d}), "TokenEmbedding expects the input embeds to have a .d tag, got {f}", .{embeds.shape()}); return self.weight.dot(embeds, .d); } @@ -149,9 +194,11 @@ pub const LmHead = struct { }; } - pub fn forward(self: LmHead, hidden: Tensor, embed_tokens: TokenEmbedding, tokens: Tensor, rng: Tensor.Rng) struct { Tensor, Tensor.Rng } { + pub fn forward(self: LmHead, hidden: zml.Tensor, embed_tokens: TokenEmbedding, tokens: zml.Tensor, rng: zml.Tensor.Rng) struct { zml.Tensor, zml.Tensor.Rng } { const logits = embed_tokens.unembed(self.embedding_norm.forward(hidden)); - const new_tokens, const new_rng = zml.nn.sampleTokens(logits, .{}, rng); + const new_tokens, const new_rng = zml.nn.sampleTokens(logits, .{ + .topk = 4, + }, rng); return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), new_rng }; } @@ -161,10 +208,7 @@ pub const LmHead = struct { }; pub const OperatorKind = enum { conv, full_attention }; -const Operator = union(enum) { - conv: ShortConv, - self_attn: Attention, -}; +const Operator = union(enum) { conv: ShortConv, self_attn: Attention }; pub const DecoderLayer = struct { operator: Operator, @@ -198,16 +242,16 @@ pub const DecoderLayer = struct { pub fn forward( self: DecoderLayer, - input: Tensor, - tokens_position_offset: Tensor, - actual_seq_len: Tensor, + input: zml.Tensor, + tokens_position_offset: zml.Tensor, + actual_seq_len: zml.Tensor, cache_: Cache, - conv_cache_index_: Tensor, - kv_cache_index_: Tensor, - attention_metadata: attention.Metadata, - attention_parameters: attention.Parameters, + conv_cache_index_: zml.Tensor, + kv_cache_index_: zml.Tensor, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, conv_parameters: ConvParameters, - ) struct { Tensor, Cache, Tensor, Tensor } { + ) struct { zml.Tensor, Cache, zml.Tensor, zml.Tensor } { var cache = cache_; var conv_cache_index = conv_cache_index_; var kv_cache_index = kv_cache_index_; @@ -222,7 +266,7 @@ pub const DecoderLayer = struct { conv_parameters, ); cache.conv = updated_conv_cache; - conv_cache_index = conv_cache_index.add(Tensor.scalar(@as(u32, 1), .u32)); + conv_cache_index = conv_cache_index.add(zml.Tensor.scalar(@as(u32, 1), .u32)); break :b residual; }, .self_attn => |operator| b: { @@ -235,11 +279,13 @@ pub const DecoderLayer = struct { attention_parameters, ); cache.kv = updated_kv_cache; - kv_cache_index = kv_cache_index.add(Tensor.scalar(@as(u32, 1), .u32)); + kv_cache_index = kv_cache_index.add(zml.Tensor.scalar(@as(u32, 1), .u32)); break :b residual; }, }; + const x = input.add(residual); + return .{ x.add(self.feed_forward.forward(self.ffn_norm.forward(x))).reuseBuffer(input), cache.reuseBuffer(cache_), @@ -269,20 +315,16 @@ pub const ShortConv = struct { pub fn init(config: Config, store: zml.io.TensorStore.View) ShortConv { stdx.debug.assert(!config.conv_bias, "conv_bias is not supported.", .{}); - return ShortConv{ .in_proj = initLinear(store.withPrefix("in_proj"), .d), .out_proj = initLinear(store.withPrefix("out_proj"), .d), .kernel = store.createTensor("conv.weight", .{ .out, .in, .kernel_size }, null), .config = config }; + return .{ + .in_proj = initLinear(store.withPrefix("in_proj"), .d), + .out_proj = initLinear(store.withPrefix("out_proj"), .d), + .kernel = store.createTensor("conv.weight", .{ .out, .in, .kernel_size }, null), + .config = config, + }; } - pub fn forward( - self: ShortConv, - input: Tensor, - tokens_position_offset: Tensor, - actual_seq_len: Tensor, - cache_: ConvCache, - cache_index: Tensor, - parameters: ConvParameters, - ) struct { Tensor, ConvCache } { + pub fn forward(self: ShortConv, input: zml.Tensor, tokens_position_offset: zml.Tensor, actual_seq_len: zml.Tensor, cache_: ConvCache, cache_index: zml.Tensor, parameters: ConvParameters) struct { zml.Tensor, ConvCache } { var cache = cache_; - // Allows us to have a consistent API for the generic DecoderLayer const BCx = self.in_proj.forward(input); const B, const C, const x = BCx.chunkExact(.d, 3); @@ -290,25 +332,21 @@ pub const ShortConv = struct { const conv_out = if (parameters.is_prefill) b: { const actual_seq_len_i32 = actual_seq_len.convert(.i32); - const start = actual_seq_len_i32.sub(Tensor.scalar(@as(i32, @intCast(self.config.conv_L_cache)), .i32)).maximum(Tensor.scalar(@as(i32, 0), .i32)); + const start = actual_seq_len_i32.sub(zml.Tensor.scalar(@as(i32, @intCast(self.config.conv_L_cache)), .i32)).maximum(zml.Tensor.scalar(@as(i32, 0), .i32)); const cache_seq_indices = lkp: { const n = actual_seq_len_i32.sub(start); - const left_pad = Tensor.scalar(@as(i32, @intCast(self.config.conv_L_cache)), .i32).sub(n); + const left_pad = zml.Tensor.scalar(@as(i32, @intCast(self.config.conv_L_cache)), .i32).sub(n); const sh = tokens_position_offset.shape().insert(.last, .{ .seq = self.config.conv_L_cache }); - break :lkp Tensor.iota(sh, .seq).convert(.u32).add(left_pad.convert(.u32).broad(sh)).broad(sh); + break :lkp zml.Tensor.iota(sh, .seq).convert(.u32).add(left_pad.convert(.u32).broad(sh)).broad(sh); }; - const scatter_data = Bx.dynamicSlice(.{ .seq = @as(Tensor.DynSlice, .{ .start = start.convert(.u32), .len = @intCast(self.config.conv_L_cache) }) }); - cache.state = cache.state.scatterSlices( - .{ .layer = cache_index, .seq = cache_seq_indices }, - scatter_data, - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(cache.state); + const scatter_data = Bx.dynamicSlice(.{ .seq = @as(zml.Tensor.DynSlice, .{ .start = start.convert(.u32), .len = @intCast(self.config.conv_L_cache) }) }); + cache.state = cache.state.scatterSlices(.{ .layer = cache_index, .seq = cache_seq_indices }, scatter_data, .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(cache.state); const padding = self.config.conv_L_cache - 1; - const conv_out = Tensor.conv1d(Bx, self.kernel, .{ + const conv_out_ = zml.Tensor.conv1d(Bx, self.kernel, .{ .padding = &.{ padding, 0 }, .input_batch_dimension = Bx.axis(.batch), .input_feature_dimension = B.axis(.d), - .input_spatial_dimensions = B.axis(.seq), + .input_spatial_dimensions = Bx.axis(.seq), .kernel_output_feature_dimension = self.kernel.axis(.out), .kernel_input_feature_dimension = self.kernel.axis(.in), .kernel_spatial_dimensions = self.kernel.axis(.kernel_size), @@ -317,20 +355,12 @@ pub const ShortConv = struct { .output_spatial_dimensions = Bx.axis(.seq), .feature_group_count = Bx.dim(.d), }); - break :b conv_out.slice1d(.seq, .{ .end = Bx.dim(.seq) }); + break :b conv_out_.slice1d(.seq, .{ .end = Bx.dim(.seq) }); } else b: { - cache.state = cache.state.rollRight1d(.seq, -1) - .scatterSlices( - .{ .layer = cache_index, .seq = tokens_position_offset.convert(.u32).clamp(.scalar(@as(u32, @intCast(0)), .u32), .scalar(self.config.conv_L_cache - 1, .u32)) }, - Bx, - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(cache.state); + cache.state = cache.state.rollRight1d(.seq, -1).scatterSlices(.{ .layer = cache_index, .seq = tokens_position_offset.convert(.u32).clamp(.scalar(@as(u32, 0), .u32), .scalar(self.config.conv_L_cache - 1, .u32)) }, Bx, .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(cache.state); const kernel = self.kernel.squeeze(.in).rename(.{ .out = .d, .kernel_size = .seq }); const sh = kernel.shape().insert(.last, .{ .batch = tokens_position_offset.dim(.batch) }); - - break :b cache.state.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = cache_index, .len = 1 } }).squeeze(.layer) - .mul(kernel.broad(sh).transpose(.{ .batch, .seq, .d })) - .sum(.seq); + break :b cache.state.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = cache_index, .len = 1 } }).squeeze(.layer).mul(kernel.broad(sh).transpose(.{ .batch, .seq, .d })).sum(.seq); }; const y = C.mul(conv_out); @@ -353,7 +383,6 @@ pub const Attention = struct { out_proj: Linear, q_layernorm: RmsNorm, k_layernorm: RmsNorm, - head_dim: usize, num_key_value_groups: usize, rope_opts: zml.nn.RopeOpts, @@ -361,9 +390,7 @@ pub const Attention = struct { pub fn init(config: Config, store: zml.io.TensorStore.View) Attention { const head_dim = config.hidden_size / config.num_attention_heads; const num_key_value_groups = config.num_attention_heads / config.num_key_value_heads; - return .{ - // .q_proj = initLinear(store.withPrefix("q_proj"), .d), .k_proj = initLinear(store.withPrefix("k_proj"), .d), .v_proj = initLinear(store.withPrefix("v_proj"), .d), @@ -372,22 +399,19 @@ pub const Attention = struct { .k_layernorm = RmsNorm.init(store.withPrefix("k_layernorm"), config.norm_eps, .hd), .head_dim = head_dim, .num_key_value_groups = num_key_value_groups, - .rope_opts = .{ - .layout = .sequential, - .scaling = .{ .default = .{ .rope_theta = config.rope_theta } }, - }, + .rope_opts = .{ .layout = .sequential, .scaling = .{ .default = .{ .rope_theta = config.rope_theta } } }, }; } pub fn forward( self: Attention, - x: Tensor, - tokens_position_offset: Tensor, + x: zml.Tensor, + tokens_position_offset: zml.Tensor, kv_cache: KvCache, - cache_index: Tensor, - attention_metadata: attention.Metadata, - attention_parameters: attention.Parameters, - ) struct { Tensor, KvCache } { + cache_index: zml.Tensor, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, + ) struct { zml.Tensor, KvCache } { var q = self.q_proj.forward(x).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); var k = self.k_proj.forward(x).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); var v = self.v_proj.forward(x).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); @@ -397,7 +421,7 @@ pub const Attention = struct { const token_positions = b: { const sh = tokens_position_offset.shape().insert(.last, .{ .seq = x.dim(.seq) }); - break :b Tensor.iota(sh, .seq).convert(.u32).add(tokens_position_offset.broad(sh)); + break :b zml.Tensor.iota(sh, .seq).convert(.u32).add(tokens_position_offset.broad(sh)); }; q = zml.nn.rope(q, token_positions, self.rope_opts); @@ -413,23 +437,8 @@ pub const Attention = struct { stdx.debug.assert(q.dim(.batch) == 1, "LFM attention currently expects batch size 1 for flash attention backend, got {}", .{q.dim(.batch)}); const attn = switch (attention_parameters) { - .vanilla => attention.attention( - q, - k, - v, - tokens_position_offset, - attention_metadata, - attention_parameters, - ).merge(.{ .d = .{ .h, .hd } }), - // flash attention doesn't support batch dimension at all in practice. - else => attention.attention( - q.squeeze(.batch), - k.squeeze(.batch), - v.squeeze(.batch), - tokens_position_offset.squeeze(.batch), - attention_metadata, - attention_parameters, - ).merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .seq }).insertAxes(.seq, .{.batch}), + .vanilla => zml.attention.attention.attention(q, k, v, tokens_position_offset, attention_metadata, attention_parameters).merge(.{ .d = .{ .h, .hd } }), + else => zml.attention.attention.attention(q.squeeze(.batch), k.squeeze(.batch), v.squeeze(.batch), tokens_position_offset.squeeze(.batch), attention_metadata, attention_parameters).merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .seq }).insertAxes(.seq, .{.batch}), }; return .{ self.out_proj.forward(attn).reuseBuffer(x), new_kv_cache.reuseBuffer(kv_cache) }; @@ -450,10 +459,7 @@ pub const Cache = struct { kv: KvCache, pub fn initBuffers(self: Cache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !zml.Bufferized(Cache) { - return .{ - .conv = try self.conv.initBuffers(allocator, io, platform, sharding), - .kv = try self.kv.initBuffers(io, platform, sharding), - }; + return .{ .conv = try self.conv.initBuffers(allocator, io, platform, sharding), .kv = try self.kv.initBuffers(io, platform, sharding) }; } pub fn unloadBuffers(self: *zml.Bufferized(Cache)) void { @@ -462,20 +468,15 @@ pub const Cache = struct { } pub fn reuseBuffer(self: Cache, other: Cache) Cache { - return .{ - .conv = self.conv.reuseBuffer(other.conv), - .kv = self.kv.reuseBuffer(other.kv), - }; + return .{ .conv = self.conv.reuseBuffer(other.conv), .kv = self.kv.reuseBuffer(other.kv) }; } }; pub const ConvCache = struct { - state: Tensor, + state: zml.Tensor, pub fn init(shape: zml.Shape) ConvCache { - return .{ - .state = .fromShape(shape), - }; + return .{ .state = .fromShape(shape) }; } pub fn initBuffers(self: ConvCache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !zml.Bufferized(ConvCache) { @@ -483,7 +484,6 @@ pub const ConvCache = struct { const host = try allocator.alloc(u8, sh.byteSize()); defer allocator.free(host); @memset(host, 0); - return .{ .state = try zml.Buffer.fromBytes(io, platform, sh, sharding, host) }; } @@ -492,28 +492,20 @@ pub const ConvCache = struct { } pub fn reuseBuffer(self: ConvCache, other: ConvCache) ConvCache { - return .{ - .state = self.state.reuseBuffer(other.state), - }; + return .{ .state = self.state.reuseBuffer(other.state) }; } }; pub const KvCache = struct { - k: Tensor, - v: Tensor, + k: zml.Tensor, + v: zml.Tensor, pub fn init(kv_shape: zml.Shape) KvCache { - return .{ - .k = .fromShape(kv_shape), - .v = .fromShape(kv_shape), - }; + return .{ .k = .fromShape(kv_shape), .v = .fromShape(kv_shape) }; } pub fn initBuffers(self: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !zml.Bufferized(KvCache) { - return .{ - .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), sharding, .{}), - .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), sharding, .{}), - }; + return .{ .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), sharding, .{}), .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), sharding, .{}) }; } pub fn unloadBuffers(self: *zml.Bufferized(KvCache)) void { @@ -522,35 +514,23 @@ pub const KvCache = struct { } pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { - return .{ - .k = self.k.reuseBuffer(other.k), - .v = self.v.reuseBuffer(other.v), - }; + return .{ .k = self.k.reuseBuffer(other.k), .v = self.v.reuseBuffer(other.v) }; } - pub fn keys(self: KvCache, cache_index: Tensor) Tensor { - return self.k.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = cache_index, .len = 1 } }).squeeze(.layer); + pub fn keys(self: KvCache, cache_index: zml.Tensor) zml.Tensor { + return self.k.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = cache_index, .len = 1 } }).squeeze(.layer); } - pub fn values(self: KvCache, cache_index: Tensor) Tensor { - return self.v.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = cache_index, .len = 1 } }).squeeze(.layer); + pub fn values(self: KvCache, cache_index: zml.Tensor) zml.Tensor { + return self.v.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = cache_index, .len = 1 } }).squeeze(.layer); } - pub fn update(self: KvCache, new_k: Tensor, new_v: Tensor, token_position: Tensor, cache_index: Tensor) KvCache { + pub fn update(self: KvCache, new_k: zml.Tensor, new_v: zml.Tensor, token_position: zml.Tensor, cache_index: zml.Tensor) KvCache { const k_shape = self.k.shape().drop(.layer); const layer = cache_index.broad(token_position.shape()); - return .{ - .k = self.k.scatterSlices( - .{ .layer = layer, .k = token_position }, - new_k.transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.k), - .v = self.v.scatterSlices( - .{ .layer = layer, .k = token_position }, - new_v.transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.v), + .k = self.k.scatterSlices(.{ .layer = layer, .k = token_position }, new_k.transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(self.k), + .v = self.v.scatterSlices(.{ .layer = layer, .k = token_position }, new_v.transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(self.v), }; } }; @@ -560,22 +540,16 @@ fn initLinear(store: zml.io.TensorStore.View, tag: anytype) Linear { } pub const Linear = struct { - weight: Tensor, - bias: ?Tensor = null, - in_tag: Shape.Tag, - out_tag: Shape.Tag, + weight: zml.Tensor, + bias: ?zml.Tensor = null, + in_tag: zml.Shape.Tag, + out_tag: zml.Shape.Tag, - /// Assumes that the weights are tagged `.in` and `.out`. - pub fn init(weight: Tensor, bias: ?Tensor, tag: anytype) Linear { + pub fn init(weight: zml.Tensor, bias: ?zml.Tensor, tag: anytype) Linear { stdx.debug.guard(weight.shape().hasTag(tag) != null, @src()); const axis = weight.shape().axis(tag); const out_tag = weight.shape().tag(1 - axis); - return .{ - .weight = weight, - .bias = bias, - .in_tag = zml.Shape.toTag(tag), - .out_tag = out_tag, - }; + return .{ .weight = weight, .bias = bias, .in_tag = zml.Shape.toTag(tag), .out_tag = out_tag }; } pub fn unloadBuffers(linear: *zml.Bufferized(Linear)) void { @@ -583,9 +557,8 @@ pub const Linear = struct { if (linear.bias) |*bias| bias.deinit(); } - pub fn forward(self: Linear, x: Tensor) Tensor { + pub fn forward(self: Linear, x: zml.Tensor) zml.Tensor { var y = x.dot(self.weight, self.in_tag).renameTag(self.out_tag, self.in_tag); - return if (self.bias) |bias| y.add(bias.broad(y.shape())).reuseBuffer(x) else y; } }; @@ -596,11 +569,7 @@ const Mlp = struct { w3: Linear, pub fn init(store: zml.io.TensorStore.View) Mlp { - return .{ - .w1 = initLinear(store.withPrefix("w1"), .d), - .w2 = initLinear(store.withPrefix("w2"), .d), - .w3 = initLinear(store.withPrefix("w3"), .d), - }; + return .{ .w1 = initLinear(store.withPrefix("w1"), .d), .w2 = initLinear(store.withPrefix("w2"), .d), .w3 = initLinear(store.withPrefix("w3"), .d) }; } pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { @@ -609,15 +578,15 @@ const Mlp = struct { Linear.unloadBuffers(&self.w3); } - pub fn forward(self: Mlp, x: Tensor) Tensor { + pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { return self.w2.forward(self.w1.forward(x).silu().mul(self.w3.forward(x))).reuseBuffer(x); } }; const RmsNorm = struct { - weight: Tensor, + weight: zml.Tensor, eps: f32, - tag: Shape.Tag, + tag: zml.Shape.Tag, pub fn init(store: zml.io.TensorStore.View, eps: f32, tag: anytype) RmsNorm { return .{ .weight = store.createTensor("weight", .{tag}, null), .eps = eps, .tag = zml.Shape.toTag(tag) }; @@ -627,8 +596,7 @@ const RmsNorm = struct { self.weight.deinit(); } - /// L2 normalization of input tensor along `.d` axis. - pub fn forward(self: RmsNorm, input: Tensor) Tensor { + pub fn forward(self: RmsNorm, input: zml.Tensor) zml.Tensor { const normalized = zml.nn.rmsNorm(input, self.tag, self.eps); return normalized.mul(self.weight.broad(input.shape())).reuseBuffer(input); } diff --git a/examples/llm/models/lfm2/repository.zig b/examples/llm/models/lfm2/repository.zig new file mode 100644 index 00000000..e7ab7084 --- /dev/null +++ b/examples/llm/models/lfm2/repository.zig @@ -0,0 +1,76 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const model = @import("model.zig"); +const common = @import("../common.zig"); + +const parseConfig = common.parseConfig; +const Shardings = common.Shardings; + +pub const Repository = struct { + inner: model.Model, + parsed_config: std.json.Parsed(model.Config), + + pub fn init(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir, store: zml.io.TensorStore.View) !Repository { + const parsed_config = try parseConfig(model.Config, allocator, io, repo); + errdefer parsed_config.deinit(); + + return .{ + .inner = .init(allocator, store, parsed_config.value), + .parsed_config = parsed_config, + }; + } + + pub fn deinit(self: *Repository, allocator: std.mem.Allocator) void { + self.inner.deinit(allocator); + self.parsed_config.deinit(); + } + + pub fn loadBuffers(self: *Repository, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, shardings: Shardings) !model.Buffers { + return try self.inner.loadBuffers(allocator, io, platform, store, progress, shardings); + } + + pub fn unloadBuffers(self: *model.Buffers, allocator: std.mem.Allocator) void { + model.Model.unloadBuffers(self, allocator); + } + + pub fn tokenizePrompt(self: *const Repository, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]u32 { + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const im_start = tokenizer.tokenToId("<|im_start|>") orelse return error.NoSuchToken; + const im_end = tokenizer.tokenToId("<|im_end|>") orelse return error.NoSuchToken; + const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; + const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; + const newline = (try encoder.encode("\n"))[0]; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); + try tokens.appendSlice(allocator, &.{ self.parsed_config.value.bos_token_id, im_start, user, newline }); + try tokens.appendSlice(allocator, try encoder.encode(prompt)); + try tokens.appendSlice(allocator, &.{ im_end, newline }); + try tokens.appendSlice(allocator, &.{ im_start, assistant, newline }); + + return tokens.toOwnedSlice(allocator); + } + + pub fn tokenizeTurn(self: *const Repository, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]u32 { + _ = self; + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const im_start = tokenizer.tokenToId("<|im_start|>") orelse return error.NoSuchToken; + const im_end = tokenizer.tokenToId("<|im_end|>") orelse return error.NoSuchToken; + const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; + const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; + const newline = (try encoder.encode("\n"))[0]; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); + try tokens.appendSlice(allocator, &.{ im_end, newline, im_start, user, newline }); + try tokens.appendSlice(allocator, try encoder.encode(prompt)); + try tokens.appendSlice(allocator, &.{ im_end, newline }); + try tokens.appendSlice(allocator, &.{ im_start, assistant, newline }); + + return tokens.toOwnedSlice(allocator); + } +}; diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig new file mode 100644 index 00000000..b4e5b643 --- /dev/null +++ b/examples/llm/models/lfm2/session.zig @@ -0,0 +1,160 @@ +const std = @import("std"); + +const zml = @import("zml"); +const attention = zml.attention.attention; + +const common = @import("../common.zig"); +const Shardings = common.Shardings; +const SessionOptions = common.SessionOptions; +const inference = @import("inference.zig"); +const model = @import("model.zig"); +const repository = @import("repository.zig"); + +pub const Session = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + model_buffers: *model.Buffers, + exe: inference.Inference, + config: model.Config, + seqlen: u32, + cache_buffers: zml.Bufferized(model.Cache), + attention_metadata_buffers: zml.Bufferized(attention.Metadata), + rng_buf: zml.Bufferized(zml.Tensor.Rng), + generated_token_slice: zml.Slice, + think_start: ?u32, + think_end: ?u32, + tokenizer: zml.tokenizer.Tokenizer, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + model_buffers: *model.Buffers, + tokenizer: zml.tokenizer.Tokenizer, + repo: repository.Repository, + opts: SessionOptions, + progress: *std.Progress.Node, + shardings: Shardings, + ) !Session { + const params = inference.CompilationOptions.init(repo.inner, repo.parsed_config.value, opts.seqlen, opts.backend, opts.single); + const exe = try inference.Inference.init(allocator, io, platform, repo.inner, params, opts.seqlen, progress, shardings); + errdefer exe.deinit(); + + const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); + return .{ + .allocator = allocator, + .io = io, + .platform = platform, + .model_buffers = model_buffers, + .exe = exe, + .tokenizer = tokenizer, + .config = repo.parsed_config.value, + .seqlen = opts.seqlen, + .cache_buffers = try params.cache.initBuffers(allocator, io, platform, shardings.replicated), + .attention_metadata_buffers = try params.attention_metadata.initBuffer(io, platform, shardings.model), + .rng_buf = try zml.Tensor.Rng.initBuffer(platform, seed, io, shardings.replicated), + .generated_token_slice = try .alloc(allocator, zml.Shape.init(.{ .batch = 1, .seq = 1 }, .u32)), + .think_start = tokenizer.tokenToId("") orelse unreachable, + .think_end = tokenizer.tokenToId("") orelse unreachable, + }; + } + + pub fn deinit(self: *Session) void { + self.exe.deinit(); + model.Cache.unloadBuffers(&self.cache_buffers); + attention.Metadata.deinitBuffer(&self.attention_metadata_buffers); + zml.Tensor.Rng.deinitBuffer(&self.rng_buf); + self.generated_token_slice.free(self.allocator); + } + + pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { + const tokens_slice: zml.Slice = try .alloc(self.allocator, .init(.{ .batch = 1, .seq = self.seqlen }, .u32)); + defer tokens_slice.free(self.allocator); + const tokens = tokens_slice.items(u32); + @memset(tokens, self.config.pad_token_id); + @memcpy(tokens[0..all_tokens.len], all_tokens); + + const sharding = try zml.sharding.replicatedSharding(self.platform); + + var tokens_buf: zml.Buffer = try .fromSlice(self.io, self.platform, tokens_slice, sharding); + defer tokens_buf.deinit(); + + const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{0})); + var tokens_pos_buf: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice, sharding); + defer tokens_pos_buf.deinit(); + + const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(all_tokens.len)})); + var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice, sharding); + defer actual_seq_len_buf.deinit(); + + try self.exe.prefill.run(.{ + .allocator = self.allocator, + .io = self.io, + .platform = self.platform, + .model_buffers = self.model_buffers, + .tokens_buf = &tokens_buf, + .tokens_pos_buf = &tokens_pos_buf, + .actual_seq_len_buf = &actual_seq_len_buf, + .rng_buf = &self.rng_buf, + .cache_buffers = &self.cache_buffers, + .attention_metadata_buffers = self.attention_metadata_buffers, + }); + + try tokens_buf.toSlice(self.io, tokens_slice); + self.generated_token_slice.items(u32)[0] = tokens_slice.items(u32)[all_tokens.len - 1]; + } + + pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { + var decoder = try self.tokenizer.decoder(); + defer decoder.deinit(); + + const sharding = try zml.sharding.replicatedSharding(self.platform); + + var current_token_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, self.generated_token_slice, sharding); + defer current_token_buffer.deinit(); + + const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{0})); + var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice, sharding); + defer actual_seq_len_buf.deinit(); + + generation: while (true) { + const token_id = self.generated_token_slice.items(u32)[0]; + + if (token_id == self.config.eos_token_id) break :generation; + + if (try decoder.next(token_id)) |token| { + if (self.think_start) |think_start| if (token_id == think_start) { + try stdout.writeAll("\x1b[2m"); + }; + try stdout.writeAll(token); + if (self.think_end) |think_end| if (token_id == think_end) { + try stdout.writeAll("\x1b[0m"); + }; + try stdout.flush(); + } + + try all_tokens.append(self.allocator, token_id); + if (all_tokens.items.len >= self.seqlen) break :generation; + + const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(all_tokens.items.len)})); + var token_pos_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice, sharding); + defer token_pos_buffer.deinit(); + + try self.exe.decode.run(.{ + .allocator = self.allocator, + .io = self.io, + .platform = self.platform, + .model_buffers = self.model_buffers, + .tokens_buf = ¤t_token_buffer, + .tokens_pos_buf = &token_pos_buffer, + .actual_seq_len_buf = &actual_seq_len_buf, + .rng_buf = &self.rng_buf, + .cache_buffers = &self.cache_buffers, + .attention_metadata_buffers = self.attention_metadata_buffers, + }); + + try current_token_buffer.toSlice(self.io, self.generated_token_slice); + } + } +}; diff --git a/examples/lfm/test_runner.zig b/examples/llm/models/lfm2_tests.zig similarity index 69% rename from examples/lfm/test_runner.zig rename to examples/llm/models/lfm2_tests.zig index db508033..5fada1d3 100644 --- a/examples/lfm/test_runner.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -1,21 +1,87 @@ const std = @import("std"); const zml = @import("zml"); -const stdx = zml.stdx; const attention = zml.attention.attention; -const model = @import("model.zig"); +const lfm2 = @import("lfm2.zig"); +const model = @import("lfm2/model.zig"); +const Shardings = @import("common.zig").Shardings; -pub fn run(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, config: model.Config, mdl: model.Model, model_buffers: *zml.Bufferized(model.Model), store: *zml.io.TensorStore, attention_metadata: attention.Metadata, attention_parameters: attention.Parameters) !void { - _ = store; - var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, "/Users/brabier/github/huggingface/transformers/lfm_activations.safetensors"); +pub const std_options: std.Options = .{ + .log_level = .info, +}; + +const Args = struct { + model: []const u8, + activations: []const u8, + backend: ?attention.Backend = null, + + pub const help = + \\Use lfm2_tests --model= --activations= [options] + \\ + \\ Validate the LFM2 implementation against activation fixtures. + \\ + \\ Options: + \\ --model= Path to the model repository + \\ --activations= Path to activation safetensors + \\ --backend= Attention backend to use + \\ + ; +}; + +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const io = init.io; + const args = zml.stdx.flags.parse(init.minimal.args, Args); + + const platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator); + + const repo = try zml.safetensors.resolveModelRepo(io, args.model); + var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); + defer registry.deinit(); + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + defer store.deinit(); + + var repo_model = try lfm2.Repository.init(allocator, io, repo, store.view()); + defer repo_model.deinit(allocator); + + var progress = std.Progress.start(io, .{ .root_name = args.model }); + const tp_mesh: zml.sharding.LogicalMesh = try .init("tp_mesh", .{ .model = .high_bandwidth }); + const tp_strategy: zml.sharding.Strategy = try .suggest(tp_mesh, platform.physical_mesh); + const shardings: Shardings = .{ + .replicated = try zml.sharding.replicatedSharding(platform), + .model = try .initFromStrategy(platform, tp_mesh, tp_strategy), + }; + + var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress, shardings); + defer lfm2.Repository.unloadBuffers(&model_buffers, allocator); + + const backend = args.backend orelse attention.Backend.auto(platform); + const params = lfm2.CompilationOptions.init(repo_model.inner, repo_model.parsed_config.value, repo_model.parsed_config.value.max_position_embeddings, backend, false); + progress.end(); + + try run(allocator, io, platform, args.activations, repo_model.parsed_config.value, repo_model.inner, &model_buffers, params.attention_metadata, params.attention_parameters); +} + +pub fn run( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + activations_path: []const u8, + config: lfm2.Config, + mdl: lfm2.Model, + model_buffers: *lfm2.Buffers, + attention_metadata: attention.Metadata, + attention_parameters: attention.Parameters, +) !void { + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, activations_path); defer registry.deinit(); var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); defer activation_store.deinit(); const sharding = try zml.sharding.replicatedSharding(platform); - var ctx = TestContext{ .allocator = allocator, .io = io, @@ -31,10 +97,7 @@ pub fn run(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, co var num_attn_layers: usize = 0; var num_conv_layers: usize = 0; for (0..config.num_hidden_layers) |i| { - const kind = std.meta.stringToEnum(model.OperatorKind, config.layer_types[i]) orelse { - stdx.debug.assert(false, "Unsupported layer type {s}", .{config.layer_types[i]}); - unreachable; - }; + const kind = std.meta.stringToEnum(model.OperatorKind, config.layer_types[i]) orelse unreachable; const layer = mdl.layers[i]; const layer_buffers = model_buffers.layers[i]; switch (kind) { @@ -73,15 +136,7 @@ const TestContext = struct { attention_parameters: attention.Parameters, sharding: zml.sharding.Sharding, - fn testLayerPrint( - self: *TestContext, - comptime name_fmt: []const u8, - name_args: anytype, - tagz: anytype, - layer: anytype, - layer_buffers: anytype, - opts: zml.testing.CompareOpts, - ) !void { + fn testLayerPrint(self: *TestContext, comptime name_fmt: []const u8, name_args: anytype, tagz: anytype, layer: anytype, layer_buffers: anytype, opts: zml.testing.CompareOpts) !void { const name = try std.fmt.allocPrint(self.allocator, name_fmt, name_args); defer self.allocator.free(name); try self.testLayer(name, tagz, layer, layer_buffers, opts); @@ -92,12 +147,14 @@ const TestContext = struct { const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); defer self.allocator.free(in_key); - const in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key, self.sharding); + var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key, self.sharding); + defer in_buffer.deinit(); const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(tagz); const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); defer self.allocator.free(out_key); - const out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key, self.sharding); + var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key, self.sharding); + defer out_buffer_expected.deinit(); const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor }, .{ .shardings = &.{self.sharding} }); defer exe.deinit(); @@ -111,7 +168,8 @@ const TestContext = struct { exe.call(args, &res); - const out_result = res.get(zml.Buffer); + var out_result = res.get(zml.Buffer); + defer out_result.deinit(); try zml.testing.expectClose(self.io, out_result, out_buffer_expected, opts); std.log.info("Layer {s} passed!", .{name}); @@ -124,26 +182,29 @@ const TestContext = struct { const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); defer self.allocator.free(in_key); - const in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key, self.sharding); + var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key, self.sharding); + defer in_buffer.deinit(); const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(.{ .batch, .seq, .d }); const cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache", .{name}); defer self.allocator.free(cache_key); - const cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_key, self.sharding); + var cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_key, self.sharding); + defer cache_buffer.deinit(); const cache_tensor = zml.Tensor.fromShape(cache_buffer.shape()).withTags(.{ .layer, .batch, .seq, .d }); const cache_pos_key = try std.fmt.allocPrint(self.allocator, "{s}.cache_position", .{name}); defer self.allocator.free(cache_pos_key); - const cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key, self.sharding); + var cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key, self.sharding); + defer cache_pos_buffer.deinit(); const cache_pos_tensor = zml.Tensor.fromShape(cache_pos_buffer.shape()).withTags(.{.batch}); const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); defer self.allocator.free(out_key); - const out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key, self.sharding); + var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key, self.sharding); + defer out_buffer_expected.deinit(); const actual_seq_len_tensor: zml.Tensor = .init(.{}, .u32); const actual_seq_len: u32 = @intCast(in_tensor.dim(.seq)); - const cache_index_tensor: zml.Tensor = .init(.{}, .u32); const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, actual_seq_len_tensor, model.ConvCache{ .state = cache_tensor }, cache_index_tensor, model.ConvParameters{ .is_prefill = false } }, .{ .shardings = &.{self.sharding} }); @@ -152,20 +213,23 @@ const TestContext = struct { var args = try exe.args(self.allocator); defer args.deinit(self.allocator); const conv_cache: zml.Bufferized(model.ConvCache) = .{ .state = cache_buffer }; + const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{actual_seq_len})); var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice, self.sharding); defer actual_seq_len_buf.deinit(); + const cache_index_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(cache_ix)})); var cache_index_buf: zml.Buffer = try .fromSlice(self.io, self.platform, cache_index_slice, self.sharding); defer cache_index_buf.deinit(); + args.set(.{ layer_buffers, in_buffer, cache_pos_buffer, actual_seq_len_buf, conv_cache, cache_index_buf }); var res = try exe.results(self.allocator); defer res.deinit(self.allocator); - exe.call(args, &res); - const out_result = res.get(zml.Buffer); + var out_result = res.get(zml.Buffer); + defer out_result.deinit(); try zml.testing.expectClose(self.io, out_result, out_buffer_expected, opts); std.log.info("Layer {s} passed!", .{name}); @@ -178,27 +242,32 @@ const TestContext = struct { const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); defer self.allocator.free(in_key); - const in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key, self.sharding); + var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key, self.sharding); + defer in_buffer.deinit(); const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(.{ .batch, .seq, .d }); const key_cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache.key", .{name}); defer self.allocator.free(key_cache_key); - const key_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, key_cache_key, self.sharding); + var key_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, key_cache_key, self.sharding); + defer key_cache_buffer.deinit(); const key_cache_tensor = zml.Tensor.fromShape(key_cache_buffer.shape()).withTags(.{ .layer, .batch, .h, .k, .hd }); const value_cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache.value", .{name}); defer self.allocator.free(value_cache_key); - const value_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, value_cache_key, self.sharding); + var value_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, value_cache_key, self.sharding); + defer value_cache_buffer.deinit(); const value_cache_tensor = zml.Tensor.fromShape(value_cache_buffer.shape()).withTags(.{ .layer, .batch, .h, .k, .hd }); const cache_pos_key = try std.fmt.allocPrint(self.allocator, "{s}.cache_position", .{name}); defer self.allocator.free(cache_pos_key); - const cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key, self.sharding); + var cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key, self.sharding); + defer cache_pos_buffer.deinit(); const cache_pos_tensor = zml.Tensor.fromShape(cache_pos_buffer.shape()).withTags(.{.batch}); const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); defer self.allocator.free(out_key); - const out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key, self.sharding); + var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key, self.sharding); + defer out_buffer_expected.deinit(); const cache_index_tensor: zml.Tensor = .init(.{}, .u32); @@ -208,16 +277,18 @@ const TestContext = struct { var args = try exe.args(self.allocator); defer args.deinit(self.allocator); const kv_cache: zml.Bufferized(model.KvCache) = .{ .k = key_cache_buffer, .v = value_cache_buffer }; + var attention_metadata_buffers = try self.attention_metadata.initBuffer(self.io, self.platform, self.sharding); defer attention.Metadata.deinitBuffer(&attention_metadata_buffers); + const cache_index_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(cache_ix)})); var cache_index_buf: zml.Buffer = try .fromSlice(self.io, self.platform, cache_index_slice, self.sharding); defer cache_index_buf.deinit(); + args.set(.{ layer_buffers, in_buffer, cache_pos_buffer, kv_cache, cache_index_buf, attention_metadata_buffers }); var res = try exe.results(self.allocator); defer res.deinit(self.allocator); - exe.call(args, &res); var out_result, var updated_kv = res.get(struct { zml.Buffer, zml.Bufferized(model.KvCache) }); @@ -238,6 +309,7 @@ fn loadBufferFromStore(allocator: std.mem.Allocator, io: anytype, platform: *zml var io_buffer: [8 * 1024]u8 = undefined; var reader = try store.view().getReader(key, io, &io_buffer); defer reader.deinit(); + _ = try reader.interface.readSliceAll(host_bytes); return zml.Buffer.fromBytes(io, platform, shape, sharding, host_bytes); diff --git a/examples/llm/models/llama.zig b/examples/llm/models/llama.zig new file mode 100644 index 00000000..025c6853 --- /dev/null +++ b/examples/llm/models/llama.zig @@ -0,0 +1,10 @@ +pub const inference = @import("llama/inference.zig"); +pub const CompilationOptions = inference.CompilationOptions; +pub const model = @import("llama/model.zig"); +pub const Config = model.Config; +pub const Buffers = model.Buffers; +pub const Model = model.Model; +pub const repository = @import("llama/repository.zig"); +pub const Repository = repository.Repository; +pub const session = @import("llama/session.zig"); +pub const Session = session.Session; diff --git a/examples/llm/models/llama/inference.zig b/examples/llm/models/llama/inference.zig new file mode 100644 index 00000000..490cea47 --- /dev/null +++ b/examples/llm/models/llama/inference.zig @@ -0,0 +1,150 @@ +const std = @import("std"); + +const zml = @import("zml"); +const attention = zml.attention.attention; + +const Shardings = @import("../common.zig").Shardings; +const model = @import("model.zig"); + +const log = std.log.scoped(.llama); + +pub const CompilationOptions = struct { + prefill_tokens: zml.Tensor, + decode_tokens: zml.Tensor, + token_index: zml.Tensor, + kv_cache: model.KvCache, + rng: zml.Tensor.Rng, + attention_metadata: attention.Metadata, + attention_parameters: attention.Parameters, + + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: attention.Backend) CompilationOptions { + return .{ + .prefill_tokens = .init(.{ .s = seqlen }, .u32), + .decode_tokens = .init(.{ .s = 1 }, .u32), + .token_index = .init(.{}, .u32), + .kv_cache = .init(.init(.{ + .layer = mdl.model.layers.len, + .k = seqlen, + .h = config.num_key_value_heads, + .hd = config.head_dim orelse @divExact(config.hidden_size, config.num_attention_heads), + }, mdl.model.embed_tokens.weight.dtype())), + .rng = .init(), + .attention_metadata = .init(.fromBackend(backend, @intCast(seqlen), @intCast(config.num_attention_heads))), + .attention_parameters = .init(.fromBackend(backend)), + }; + } +}; + +pub const Inference = struct { + prefill_exe: zml.Exe, + decode_exe: zml.Exe, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + llama_model: model.Model, + parameters: CompilationOptions, + shardings: Shardings, + progress: *std.Progress.Node, + ) !Inference { + const compiled = try compileModel(allocator, io, platform, llama_model, parameters, shardings, progress); + return .{ .prefill_exe = compiled.prefill_exe, .decode_exe = compiled.decode_exe }; + } + + pub fn deinit(self: *Inference) void { + self.prefill_exe.deinit(); + self.decode_exe.deinit(); + } +}; + +const CompileModelResult = struct { + prefill_exe: zml.Exe, + decode_exe: zml.Exe, +}; + +fn compileModel( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + llama_model: model.Model, + parameters: CompilationOptions, + shardings: Shardings, + progress: *std.Progress.Node, +) !CompileModelResult { + const now: std.Io.Timestamp = .now(io, .awake); + defer log.info("Compiled model [{f}]", .{now.untilNow(io, .awake)}); + + const all_shardings = shardings.all(); + + var prefill_future = try io.concurrent(struct { + fn call( + allocator_: std.mem.Allocator, + io_: std.Io, + platform_: *const zml.Platform, + llama_model_: model.Model, + parameters_: CompilationOptions, + shardings_: [2]zml.sharding.Sharding, + progress_: *std.Progress.Node, + ) !zml.Exe { + progress_.increaseEstimatedTotalItems(1); + var node_ = progress_.start("Compiling prefill...", 1); + defer node_.end(); + + return platform_.compile( + allocator_, + io_, + llama_model_, + .forward, + .{ + parameters_.prefill_tokens, + parameters_.token_index, + parameters_.kv_cache, + parameters_.rng, + parameters_.attention_metadata, + parameters_.attention_parameters, + }, + .{ .shardings = &shardings_ }, + ); + } + }.call, .{ allocator, io, platform, llama_model, parameters, all_shardings, progress }); + errdefer if (prefill_future.cancel(io)) |v| v.deinit() else |_| {}; + + var decode_future = try io.concurrent(struct { + fn call( + allocator_: std.mem.Allocator, + io_: std.Io, + platform_: *const zml.Platform, + llama_model_: model.Model, + parameters_: CompilationOptions, + shardings_: [2]zml.sharding.Sharding, + progress_: *std.Progress.Node, + ) !zml.Exe { + progress_.increaseEstimatedTotalItems(1); + var node_ = progress_.start("Compiling decode...", 1); + defer node_.end(); + + return platform_.compile( + allocator_, + io_, + llama_model_, + .forward, + .{ + parameters_.decode_tokens, + parameters_.token_index, + parameters_.kv_cache, + parameters_.rng, + parameters_.attention_metadata, + parameters_.attention_parameters, + }, + .{ .shardings = &shardings_ }, + ); + } + }.call, .{ allocator, io, platform, llama_model, parameters, all_shardings, progress }); + errdefer if (decode_future.cancel(io)) |v| v.deinit() else |_| {}; + + return .{ + .prefill_exe = try prefill_future.await(io), + .decode_exe = try decode_future.await(io), + }; +} diff --git a/examples/llama/llama.zig b/examples/llm/models/llama/model.zig similarity index 68% rename from examples/llama/llama.zig rename to examples/llm/models/llama/model.zig index d4f0ce80..a84f5703 100644 --- a/examples/llama/llama.zig +++ b/examples/llm/models/llama/model.zig @@ -1,57 +1,56 @@ const std = @import("std"); -const testing = std.testing; const zml = @import("zml"); -const Buffer = zml.Buffer; -const Tensor = zml.Tensor; -const ShapeOf = zml.ShapeOf; const stdx = zml.stdx; -const log = std.log.scoped(.llama); - -pub const TransferCtx = struct { - allocator: std.mem.Allocator, - pool: *zml.mem.DynamicBufferPool, - transferred_bytes: *usize, - progress: *std.Progress.Node, +pub const Config = struct { + bos_token_id: u32, + eos_token_id: stdx.json.Union(union(enum) { + int: u32, + ints: []u32, + }), + head_dim: ?u32 = null, + hidden_size: u32, + num_hidden_layers: u32, + num_attention_heads: u32, + num_key_value_heads: u32, + rope_theta: f32, + max_position_embeddings: u32, + rms_norm_eps: f32, + hf_rope_impl: bool = true, + tie_word_embeddings: bool = false, + rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = .{} }, }; +pub const Options = struct { + sampling_strategy: ?zml.nn.SamplingStrategy, + max_seq_len: u32, +}; + +pub const Buffers = zml.Bufferized(Model); + /// Llama architecture, using huggingface transformers naming. /// Dimensions of activations: {.b, .s, .d} -pub const LlamaLM = struct { - pub const Config = struct { - bos_token_id: u32, - eos_token_id: stdx.json.Union(union(enum) { - int: u32, - ints: []u32, - }), - head_dim: ?u32 = null, - hidden_size: u32, - num_hidden_layers: u32, - num_attention_heads: u32, - num_key_value_heads: u32, - rope_theta: f32, - max_position_embeddings: u32, - rms_norm_eps: f32, - hf_rope_impl: bool = true, - tie_word_embeddings: bool = false, - rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = .{} }, - }; - - pub const Options = struct { - sampling_strategy: ?zml.nn.SamplingStrategy, - max_seq_len: u32, - }; - +pub const Model = struct { lm_head: ?zml.nn.Linear, model: Llama, - // Options controlling generation - gen_opts: zml.nn.SamplingStrategy = .{}, + gen_opts: zml.nn.SamplingStrategy = .{ + .topk = 4, + }, config: Config, - pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config, options: Options) !LlamaLM { - const lm_head: ?zml.nn.Linear = if (store.withPrefix("lm_head").maybeCreateTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated })) |weight| + pub fn init( + allocator: std.mem.Allocator, + store: zml.io.TensorStore.View, + config: Config, + options: Options, + ) !Model { + const lm_head: ?zml.nn.Linear = if (store.withPrefix("lm_head").maybeCreateTensor( + "weight", + .{ .dout, .d }, + .{ .dout = .model, .d = .replicated }, + )) |weight| .init(weight, null, .d) else null; @@ -64,28 +63,33 @@ pub const LlamaLM = struct { }; } - pub fn deinit(self: LlamaLM, allocator: std.mem.Allocator) void { + pub fn deinit(self: Model, allocator: std.mem.Allocator) void { self.model.deinit(allocator); } - pub fn load( - self: *const LlamaLM, + pub fn loadBuffers( + self: *const Model, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, shardings: []const zml.sharding.Sharding, progress: *std.Progress.Node, - ) !zml.Bufferized(LlamaLM) { + ) !zml.Bufferized(Model) { progress.increaseEstimatedTotalItems(store.view().count()); const now: std.Io.Timestamp = .now(io, .awake); var total_bytes: usize = 0; + defer { const took = now.untilNow(io, .awake); - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + const bytes_per_sec: u64 = @intFromFloat( + @as(f64, @floatFromInt(total_bytes)) / + (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s), + ); + std.log.scoped(.llama).info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); } - return zml.io.load(LlamaLM, self, allocator, io, platform, .{ + + return zml.io.load(Model, self, allocator, io, platform, .{ .dma_chunks = 32, .dma_chunk_size = 128 * zml.MiB, .progress = progress, @@ -96,7 +100,7 @@ pub const LlamaLM = struct { }); } - pub fn unloadBuffers(self: *zml.Bufferized(LlamaLM), allocator: std.mem.Allocator) void { + pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { if (self.lm_head) |*lm_head| lm_head.weight.deinit(); Llama.unloadBuffers(&self.model, allocator); } @@ -107,28 +111,34 @@ pub const LlamaLM = struct { /// - updated KV cache /// - a Rng state to allow for probabilistic generation pub fn forward( - self: LlamaLM, - tokens_: Tensor, - token_index: Tensor, + self: Model, + tokens_: zml.Tensor, + token_index: zml.Tensor, kv_cache: KvCache, - rng: Tensor.Rng, + rng: zml.Tensor.Rng, attention_metadata: zml.attention.attention.Metadata, attention_parameters: zml.attention.attention.Parameters, - ) struct { Tensor, KvCache, Tensor.Rng } { - stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Llama ! Expected >=1d tokens and 0d token_index, got: {f} and {f}", .{ tokens_, token_index }); + ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { const tokens = tokens_.withPartialTags(.{.s}); - const out, const updated_kv_cache = self.model.forward(tokens, token_index, kv_cache, attention_metadata, attention_parameters); + const out, const updated_kv_cache = self.model.forward( + tokens, + token_index, + kv_cache, + attention_metadata, + attention_parameters, + ); const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); + return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; } pub fn sampleTokens( - self: LlamaLM, + self: Model, lm_head_: ?zml.nn.Linear, - out_: Tensor, - rng: Tensor.Rng, + out_: zml.Tensor, + rng: zml.Tensor.Rng, opts: zml.nn.SamplingStrategy, - ) struct { Tensor, Tensor.Rng } { + ) struct { zml.Tensor, zml.Tensor.Rng } { const out = out_.withPartialTags(.{ .s, .d }); var logits = blk: { @@ -152,15 +162,7 @@ pub const Llama = struct { norm: RmsNorm, layers: []TransformerLayer, - max_seq_len: u32 = 0, - num_heads: u32 = 32, - num_kv_heads: u32 = 32, - rope_opts: zml.nn.RopeOpts = .{ - .layout = .interleaved, - .scaling = .{ .default = .{ .rope_theta = 10_000 } }, - }, - - pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: LlamaLM.Config) !Llama { + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config) !Llama { const layers = try allocator.alloc(TransformerLayer, config.num_hidden_layers); errdefer allocator.free(layers); @@ -169,8 +171,15 @@ pub const Llama = struct { } return .{ - .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight", .{ .voc, .d }, .{ .voc = .replicated, .d = .model }) }, - .norm = .{ .weight = store.withPrefix("norm").createTensor("weight", .{.d}, .{ .d = .replicated }), .eps = config.rms_norm_eps }, + .embed_tokens = .{ .weight = store.createTensor( + "embed_tokens.weight", + .{ .voc, .d }, + .{ .voc = .replicated, .d = .model }, + ) }, + .norm = .{ + .weight = store.withPrefix("norm").createTensor("weight", .{.d}, .{ .d = .replicated }), + .eps = config.rms_norm_eps, + }, .layers = layers, }; } @@ -192,36 +201,41 @@ pub const Llama = struct { /// Returns result and updated KV cache. pub fn forward( self: Llama, - tokens: Tensor, - token_index: Tensor, + tokens: zml.Tensor, + token_index: zml.Tensor, kv_cache: KvCache, attention_metadata: zml.attention.attention.Metadata, attention_parameters: zml.attention.attention.Parameters, - ) struct { Tensor, KvCache } { - const embeds = embed(self.embed_tokens, tokens); + ) struct { zml.Tensor, KvCache } { + const embeds = embed_tokensForward(self.embed_tokens, tokens); var hidden = embeds; - var updated_kv_cache = kv_cache; + for (self.layers, 0..) |layer, i| { - hidden, updated_kv_cache = layer.forward(hidden, token_index, updated_kv_cache.atLayer(i), attention_metadata, attention_parameters); + hidden, updated_kv_cache = layer.forward( + hidden, + token_index, + updated_kv_cache.atLayer(i), + attention_metadata, + attention_parameters, + ); } - const output = self.norm.forward(hidden); - return .{ output, updated_kv_cache.reuseBuffer(kv_cache) }; - } - - pub fn embed(embed_tokens_: zml.nn.TokenEmbedding, tokens_: Tensor) Tensor { - return embed_tokens_.forward(tokens_).withPartialTags(.{.d}); + return .{ self.norm.forward(hidden), updated_kv_cache.reuseBuffer(kv_cache) }; } }; +fn embed_tokensForward(embed_tokens_: zml.nn.TokenEmbedding, tokens_: zml.Tensor) zml.Tensor { + return embed_tokens_.forward(tokens_).withPartialTags(.{.d}); +} + pub const TransformerLayer = struct { input_layernorm: RmsNorm, self_attn: SelfAttn, post_attention_layernorm: RmsNorm, mlp: Mlp, - pub fn init(store: zml.io.TensorStore.View, config: LlamaLM.Config) !TransformerLayer { + pub fn init(store: zml.io.TensorStore.View, config: Config) !TransformerLayer { return .{ .input_layernorm = .init(store.withPrefix("input_layernorm"), config.rms_norm_eps), .self_attn = try .init(store.withPrefix("self_attn"), config), @@ -239,12 +253,12 @@ pub const TransformerLayer = struct { pub fn forward( self: TransformerLayer, - x0: Tensor, - token_index: Tensor, + x0: zml.Tensor, + token_index: zml.Tensor, kv_cache: KvCache, attention_metadata: zml.attention.attention.Metadata, attention_parameters: zml.attention.attention.Parameters, - ) struct { Tensor, KvCache } { + ) struct { zml.Tensor, KvCache } { // Self Attention //log.debug("TransformerLayer({f}) -> {f}", .{ x0, self.input_layernorm.forward(x0) }); stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {f}", .{x0}); @@ -252,10 +266,16 @@ pub const TransformerLayer = struct { // Keep the residual stream replicated to avoid repeated gathers before q/k/v. const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); const x0_normalized = self.input_layernorm.forward(x0_replicated); - const delta0, const updated_kv_cache = self.self_attn.forward(x0_normalized, token_index, kv_cache, attention_metadata, attention_parameters); - const x1 = x0_replicated.add(delta0).withPartitioning(.{ .d = .replicated }); + const delta0, const updated_kv_cache = self.self_attn.forward( + x0_normalized, + token_index, + kv_cache, + attention_metadata, + attention_parameters, + ); // Fully Connected + const x1 = x0_replicated.add(delta0).withPartitioning(.{ .d = .replicated }); const x1_normalized = self.post_attention_layernorm.forward(x1); const x2 = self.mlp.forward(x1_normalized) .rename(.{ .dout = .d }) @@ -268,11 +288,14 @@ pub const TransformerLayer = struct { }; const RmsNorm = struct { - weight: Tensor, - eps: f32 = 1e-5, + weight: zml.Tensor, + eps: f32, pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { - return .{ .weight = store.createTensor("weight", .{.d}, .{ .d = .replicated }), .eps = eps }; + return .{ + .weight = store.createTensor("weight", .{.d}, .{ .d = .replicated }), + .eps = eps, + }; } pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { @@ -280,7 +303,7 @@ const RmsNorm = struct { } /// L2 normalization of input tensor along `.d` axis. - pub fn forward(self: RmsNorm, input: Tensor) Tensor { + pub fn forward(self: RmsNorm, input: zml.Tensor) zml.Tensor { const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); const normalized = zml.nn.rmsNorm(x, .d, self.eps); return normalized.mul(self.weight.convert(x.dtype()).withTags(.{.d}).broad(x.shape())); @@ -309,7 +332,7 @@ const Mlp = struct { if (self.down_proj.bias) |*bias| bias.deinit(); } - pub fn forward(self: Mlp, x: Tensor) Tensor { + pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { const proj = self.up_proj.forward(x); var output = self.gate_proj.forward(x); output = output.silu().mul(proj).rename(.{ .dout = .d }); @@ -330,7 +353,7 @@ pub const SelfAttn = struct { num_kv_heads: i64 = 0, rope_opts: zml.nn.RopeOpts = undefined, - pub fn init(store: zml.io.TensorStore.View, config: LlamaLM.Config) !SelfAttn { + pub fn init(store: zml.io.TensorStore.View, config: Config) !SelfAttn { var rope_scaling = config.rope_scaling; rope_scaling.setRopeTheta(config.rope_theta); return .{ @@ -364,7 +387,7 @@ pub const SelfAttn = struct { if (self.k_norm) |*k_norm| RmsNorm.unloadBuffers(k_norm); } - /// Self Attention. + /// Self zml.attention.attention. /// - If token_index is set, x is assumed to be the representation of one new token, /// and kv_cache will be read for the previous tokens. /// - If token_index is not set, x is assumed to be the representation of all tokens @@ -373,13 +396,14 @@ pub const SelfAttn = struct { /// x: {.b, .s, .d } -> .{.b, .s, .d} pub fn forward( self: SelfAttn, - x: Tensor, - token_index: Tensor, + x: zml.Tensor, + token_index: zml.Tensor, kv_cache: KvCache, attention_metadata: zml.attention.attention.Metadata, attention_parameters: zml.attention.attention.Parameters, - ) struct { Tensor, KvCache } { + ) struct { zml.Tensor, KvCache } { const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; + // Make hidden state replicated once and reuse it across q/k/v projections. // This avoids paying gather-style collectives independently for each projection. const x_qkv = x.withPartitioning(.{ .d = .replicated }); @@ -393,7 +417,7 @@ pub const SelfAttn = struct { // In self-attention, .s axis is used both for keys and queries. const pos_index = b: { - const temp = Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()).withTags(.{.s}).broad(zml.Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); + const temp = zml.Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()).withTags(.{.s}).broad(zml.Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); break :b temp.add(token_index.broad(temp.shape())); }; @@ -415,8 +439,14 @@ pub const SelfAttn = struct { k = k.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); v = v.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); - const attn_output = zml.attention.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters) - .withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }); + const attn_output = zml.attention.attention.attention( + q, + k, + v, + token_index, + attention_metadata, + attention_parameters, + ).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }); const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); const delta = self.o_proj.forward(attn) @@ -427,9 +457,9 @@ pub const SelfAttn = struct { }; pub const KvCache = struct { - k: Tensor, - v: Tensor, - layer_index: Tensor, + k: zml.Tensor, + v: zml.Tensor, + layer_index: zml.Tensor, pub fn init(kv_shape: zml.Shape) KvCache { const sharded_shape = kv_shape.withPartitioning(.{ .h = .model }); @@ -441,14 +471,6 @@ pub const KvCache = struct { }; } - pub fn initShape(kv_shape: zml.Shape) ShapeOf(KvCache) { - return .{ - .k = kv_shape, - .v = kv_shape, - .layer_index = zml.Shape.init(.{}, .u32), - }; - } - pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !zml.Bufferized(KvCache) { return .{ .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), sharding, .{}), @@ -463,51 +485,38 @@ pub const KvCache = struct { self.layer_index.deinit(); } - pub fn keys(self: KvCache) Tensor { - return self.k.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + pub fn keys(self: KvCache) zml.Tensor { + return self.k.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); } - pub fn values(self: KvCache) Tensor { - return self.v.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + pub fn values(self: KvCache) zml.Tensor { + return self.v.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); } - pub fn update(self: KvCache, new_k: Tensor, new_v: Tensor, token_index: ?Tensor) KvCache { + pub fn update(self: KvCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: ?zml.Tensor) KvCache { const k_shape = self.k.shape().drop(.layer); var layer = self.layer_index; layer = if (token_index) |idx| layer.broad(idx.shape()) else layer; - return if (token_index) |idx| .{ - .k = self.k.scatterSlices( - .{ .layer = layer, .k = idx }, - new_k.convert(self.k.dtype()).transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.k), - .v = self.v.scatterSlices( - .{ .layer = layer, .k = idx }, - new_v.convert(self.v.dtype()).transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.v), - .layer_index = self.layer_index, - } else .{ - .k = self.k.scatterSlices( - .{ .layer = layer }, - new_k.convert(self.k.dtype()).transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.k), - .v = self.v.scatterSlices( - .{ .layer = layer }, - new_v.convert(self.v.dtype()).transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.v), - .layer_index = self.layer_index, - }; + return if (token_index) |idx| + .{ + .k = self.k.scatterSlices(.{ .layer = layer, .k = idx }, new_k.convert(self.k.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(self.k), + .v = self.v.scatterSlices(.{ .layer = layer, .k = idx }, new_v.convert(self.v.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(self.v), + .layer_index = self.layer_index, + } + else + .{ + .k = self.k.scatterSlices(.{ .layer = layer }, new_k.convert(self.k.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(self.k), + .v = self.v.scatterSlices(.{ .layer = layer }, new_v.convert(self.v.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(self.v), + .layer_index = self.layer_index, + }; } pub fn atLayer(self: KvCache, layer_index: usize) KvCache { return .{ .k = self.k, .v = self.v, - .layer_index = Tensor.scalar(layer_index, .u32), + .layer_index = zml.Tensor.scalar(layer_index, .u32), }; } diff --git a/examples/llm/models/llama/repository.zig b/examples/llm/models/llama/repository.zig new file mode 100644 index 00000000..77a1d9cc --- /dev/null +++ b/examples/llm/models/llama/repository.zig @@ -0,0 +1,81 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const common = @import("../common.zig"); +const parseConfig = common.parseConfig; +const Shardings = common.Shardings; +const model = @import("model.zig"); + +pub const Repository = struct { + inner: model.Model, + parsed_config: std.json.Parsed(model.Config), + + pub fn init(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir, store: zml.io.TensorStore.View) !Repository { + const parsed_config = try parseConfig(model.Config, allocator, io, repo); + errdefer parsed_config.deinit(); + + const options: model.Options = .{ + .sampling_strategy = .{}, + .max_seq_len = parsed_config.value.max_position_embeddings, + }; + + return .{ + .inner = try .init(allocator, store, parsed_config.value, options), + .parsed_config = parsed_config, + }; + } + + pub fn deinit(self: *Repository, allocator: std.mem.Allocator) void { + self.inner.deinit(allocator); + self.parsed_config.deinit(); + } + + pub fn loadBuffers(self: *Repository, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, shardings: Shardings) !model.Buffers { + const all_shardings = shardings.all(); + return try self.inner.loadBuffers(allocator, io, platform, store, &all_shardings, progress); + } + + pub fn unloadBuffers(self: *model.Buffers, allocator: std.mem.Allocator) void { + model.Model.unloadBuffers(self, allocator); + } + + pub fn tokenizePrompt(self: *const Repository, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]u32 { + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const start_header = tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; + const end_header = tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; + const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; + const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; + const eot = tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; + const newline = (try encoder.encode("\n"))[0]; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); + try tokens.appendSlice(allocator, &.{ self.parsed_config.value.bos_token_id, start_header, user, end_header, newline }); + try tokens.appendSlice(allocator, try encoder.encode(prompt)); + try tokens.appendSlice(allocator, &.{ eot, newline, start_header, assistant, end_header, newline }); + + return tokens.toOwnedSlice(allocator); + } + + pub fn tokenizeTurn(self: *const Repository, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]u32 { + _ = self; + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const start_header = tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; + const end_header = tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; + const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; + const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; + const eot = tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; + const newline = (try encoder.encode("\n"))[0]; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); + try tokens.appendSlice(allocator, &.{ eot, newline, start_header, user, end_header, newline }); + try tokens.appendSlice(allocator, try encoder.encode(prompt)); + try tokens.appendSlice(allocator, &.{ eot, newline, start_header, assistant, end_header, newline }); + + return tokens.toOwnedSlice(allocator); + } +}; diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig new file mode 100644 index 00000000..eb10dc17 --- /dev/null +++ b/examples/llm/models/llama/session.zig @@ -0,0 +1,167 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const common = @import("../common.zig"); +const Shardings = common.Shardings; +const SessionOptions = common.SessionOptions; +const inference = @import("inference.zig"); +const model = @import("model.zig"); +const repository = @import("repository.zig"); + +pub const Session = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + model_buffers: *model.Buffers, + exe: inference.Inference, + kv_cache_buffers: zml.Bufferized(model.KvCache), + attention_metadata_buffers: zml.Bufferized(zml.attention.attention.Metadata), + rng_buffers: zml.Bufferized(zml.Tensor.Rng), + generated_token_slice: zml.Slice, + tokenizer: zml.tokenizer.Tokenizer, + config: model.Config, + seqlen: u32, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + model_buffers: *model.Buffers, + tokenizer: zml.tokenizer.Tokenizer, + repo: repository.Repository, + opts: SessionOptions, + progress: *std.Progress.Node, + shardings: Shardings, + ) !Session { + if (opts.single) return error.SingleKernelNotSupported; + + const params = inference.CompilationOptions.init(repo.inner, repo.parsed_config.value, opts.seqlen, opts.backend); + var exe = try inference.Inference.init(allocator, io, platform, repo.inner, params, shardings, progress); + errdefer exe.deinit(); + + var kv_cache_buffers = try params.kv_cache.initBuffer(io, platform, shardings.model); + errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); + + var attention_metadata_buffers = try params.attention_metadata.initBuffer(io, platform, shardings.model); + errdefer zml.attention.attention.Metadata.deinitBuffer(&attention_metadata_buffers); + + const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); + var rng_buffers = try zml.Tensor.Rng.initBuffer(platform, seed, io, shardings.replicated); + errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); + + return .{ + .allocator = allocator, + .io = io, + .platform = platform, + .model_buffers = model_buffers, + .exe = exe, + .kv_cache_buffers = kv_cache_buffers, + .attention_metadata_buffers = attention_metadata_buffers, + .rng_buffers = rng_buffers, + .generated_token_slice = try .alloc(allocator, zml.Shape.init(.{ .s = 1 }, .u32)), + .tokenizer = tokenizer, + .config = repo.parsed_config.value, + .seqlen = opts.seqlen, + }; + } + + pub fn deinit(self: *Session) void { + self.exe.deinit(); + model.KvCache.deinitBuffer(&self.kv_cache_buffers); + zml.attention.attention.Metadata.deinitBuffer(&self.attention_metadata_buffers); + zml.Tensor.Rng.deinitBuffer(&self.rng_buffers); + self.generated_token_slice.free(self.allocator); + } + + pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { + var prefill_args = try self.exe.prefill_exe.args(self.allocator); + defer prefill_args.deinit(self.allocator); + + var prefill_results = try self.exe.prefill_exe.results(self.allocator); + defer prefill_results.deinit(self.allocator); + + const prefill_tokens_slice: zml.Slice = try .alloc(self.allocator, .init(.{self.seqlen}, .u32)); + defer prefill_tokens_slice.free(self.allocator); + @memcpy(prefill_tokens_slice.items(u32)[0..all_tokens.len], all_tokens); + + const replicated_sharding = try zml.sharding.replicatedSharding(self.platform); + + var prefill_tokens_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, prefill_tokens_slice, replicated_sharding); + defer prefill_tokens_buffer.deinit(); + + var prefill_token_pos_buffer = try zml.Buffer.scalar(self.io, self.platform, 0, .u32, replicated_sharding); + defer prefill_token_pos_buffer.deinit(); + + prefill_args.set(.{ + self.model_buffers, + prefill_tokens_buffer, + prefill_token_pos_buffer, + &self.kv_cache_buffers, + &self.rng_buffers, + &self.attention_metadata_buffers, + }); + self.exe.prefill_exe.call(prefill_args, &prefill_results); + + prefill_results.fill(.{ &prefill_tokens_buffer, &self.kv_cache_buffers, &self.rng_buffers }); + try prefill_tokens_buffer.toSlice(self.io, prefill_tokens_slice); + + self.generated_token_slice.items(u32)[0] = prefill_tokens_slice.items(u32)[all_tokens.len - 1]; + } + + pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { + var decoder = try self.tokenizer.decoder(); + defer decoder.deinit(); + + var decode_args = try self.exe.decode_exe.args(self.allocator); + defer decode_args.deinit(self.allocator); + + var decode_results = try self.exe.decode_exe.results(self.allocator); + defer decode_results.deinit(self.allocator); + + const replicated_sharding = try zml.sharding.replicatedSharding(self.platform); + + var current_token_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, self.generated_token_slice, replicated_sharding); + defer current_token_buffer.deinit(); + + generation: while (true) { + const token_id = self.generated_token_slice.items(u32)[0]; + + if (isEosToken(self.config, token_id)) break :generation; + + if (try decoder.next(token_id)) |token| { + try stdout.writeAll(token); + try stdout.flush(); + } + + try all_tokens.append(self.allocator, token_id); + if (all_tokens.items.len >= self.seqlen) break :generation; + + const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(all_tokens.items.len)})); + var token_pos_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice, replicated_sharding); + defer token_pos_buffer.deinit(); + + decode_args.set(.{ + self.model_buffers, + current_token_buffer, + token_pos_buffer, + &self.kv_cache_buffers, + &self.rng_buffers, + &self.attention_metadata_buffers, + }); + self.exe.decode_exe.call(decode_args, &decode_results); + + decode_results.fill(.{ ¤t_token_buffer, &self.kv_cache_buffers, &self.rng_buffers }); + try current_token_buffer.toSlice(self.io, self.generated_token_slice); + } + } +}; + +fn isEosToken(config: model.Config, token_id: u32) bool { + return switch (config.eos_token_id.value) { + .int => |eos| token_id == eos, + .ints => |eos_list| for (eos_list) |eos| { + if (token_id == eos) break true; + } else false, + }; +} diff --git a/examples/llm/models/llama_tests.zig b/examples/llm/models/llama_tests.zig new file mode 100644 index 00000000..2767cfab --- /dev/null +++ b/examples/llm/models/llama_tests.zig @@ -0,0 +1,146 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const llama = @import("llama.zig"); +const model = @import("llama/model.zig"); +const Shardings = @import("common.zig").Shardings; +const parseConfig = @import("common.zig").parseConfig; + +pub const std_options: std.Options = .{ + .log_level = .info, +}; + +const Args = struct { + model: []const u8, + activations: []const u8, + + pub const help = + \\Use llama_tests --model= --activations= + \\ + \\ Validate the LLaMA implementation against activation fixtures. + \\ + \\ Options: + \\ --model= Path to the model repository + \\ --activations= Path to activation safetensors + \\ + ; +}; + +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const io = init.io; + const args = zml.stdx.flags.parse(init.minimal.args, Args); + + const platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator); + + const repo = try zml.safetensors.resolveModelRepo(io, args.model); + var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); + defer registry.deinit(); + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + defer store.deinit(); + + var repo_model = try llama.Repository.init(allocator, io, repo, store.view()); + defer repo_model.deinit(allocator); + + var progress = std.Progress.start(io, .{ .root_name = args.model }); + const tp_mesh: zml.sharding.LogicalMesh = try .init("tp_mesh", .{ .model = .high_bandwidth }); + const tp_strategy: zml.sharding.Strategy = try .suggest(tp_mesh, platform.physical_mesh); + const shardings: Shardings = .{ + .replicated = try zml.sharding.replicatedSharding(platform), + .model = try .initFromStrategy(platform, tp_mesh, tp_strategy), + }; + + var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress, shardings); + defer llama.Repository.unloadBuffers(&model_buffers, allocator); + progress.end(); + + try run(allocator, io, platform, args.activations, repo_model.inner, &model_buffers, shardings.replicated); +} + +fn run( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + activations_path: []const u8, + mdl: model.Model, + model_buffers: *model.Buffers, + sharding: zml.sharding.Sharding, +) !void { + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, activations_path); + defer registry.deinit(); + + var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + defer activation_store.deinit(); + + try testLayer(allocator, io, platform, activation_store.view(), "embed_tokens", mdl.model.embed_tokens, model_buffers.model.embed_tokens, sharding, .{ .absolute_tolerance = 1e-3 }); + + if (mdl.model.layers.len == 0) return; + + const layer = mdl.model.layers[0]; + const layer_buffers = model_buffers.model.layers[0]; + + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.v_proj", layer.self_attn.v_proj, layer_buffers.self_attn.v_proj, sharding, .{ .absolute_tolerance = 1e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.q_proj", layer.self_attn.q_proj, layer_buffers.self_attn.q_proj, sharding, .{ .absolute_tolerance = 2e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.k_proj", layer.self_attn.k_proj, layer_buffers.self_attn.k_proj, sharding, .{ .absolute_tolerance = 2e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.o_proj", layer.self_attn.o_proj, layer_buffers.self_attn.o_proj, sharding, .{ .absolute_tolerance = 2e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.mlp", layer.mlp, layer_buffers.mlp, sharding, .{ .absolute_tolerance = 1e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.input_layernorm", layer.input_layernorm, layer_buffers.input_layernorm, sharding, .{ .absolute_tolerance = 1e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.post_attention_layernorm", layer.post_attention_layernorm, layer_buffers.post_attention_layernorm, sharding, .{ .absolute_tolerance = 1e-2 }); +} + +fn testLayer( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + activation_store: zml.io.TensorStore.View, + name: []const u8, + layer: anytype, + layer_weights: zml.Bufferized(@TypeOf(layer)), + sharding: zml.sharding.Sharding, + opts: zml.testing.CompareOpts, +) !void { + const in_key = try std.fmt.allocPrint(allocator, "{s}.in", .{name}); + defer allocator.free(in_key); + const in_shape = activation_store.getShape(in_key) orelse return error.NotFound; + var in_buffer = try loadBufferFromStore(allocator, io, platform, activation_store, in_key, sharding); + defer in_buffer.deinit(); + const in_tensor = zml.Tensor.fromShape(in_shape); + + const out_key = try std.fmt.allocPrint(allocator, "{s}.out", .{name}); + defer allocator.free(out_key); + var out_buffer_expected = try loadBufferFromStore(allocator, io, platform, activation_store, out_key, sharding); + defer out_buffer_expected.deinit(); + + const exe = try platform.compileFn(allocator, io, @TypeOf(layer).forward, .{ layer, in_tensor }, .{ .shardings = &.{sharding} }); + defer exe.deinit(); + + var args = try exe.args(allocator); + defer args.deinit(allocator); + args.set(.{ layer_weights, in_buffer }); + + var res = try exe.results(allocator); + defer res.deinit(allocator); + + exe.call(args, &res); + + var out_result = res.get(zml.Buffer); + defer out_result.deinit(); + try zml.testing.expectClose(io, out_result, out_buffer_expected, opts); +} + +fn loadBufferFromStore(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: zml.io.TensorStore.View, key: []const u8, sharding: zml.sharding.Sharding) !zml.Buffer { + const shape = store.getShape(key) orelse return error.NotFound; + + const host_bytes = try allocator.alloc(u8, shape.byteSize()); + defer allocator.free(host_bytes); + + var io_buffer: [8 * 1024]u8 = undefined; + var reader = try store.getReader(key, io, &io_buffer); + defer reader.deinit(); + + _ = try reader.interface.readSliceAll(host_bytes); + + return zml.Buffer.fromBytes(io, platform, shape, sharding, host_bytes); +} diff --git a/examples/llm/models/qwen3_5.zig b/examples/llm/models/qwen3_5.zig new file mode 100644 index 00000000..aded98ae --- /dev/null +++ b/examples/llm/models/qwen3_5.zig @@ -0,0 +1,10 @@ +pub const inference = @import("qwen3_5/inference.zig"); +pub const CompilationOptions = inference.CompilationOptions; +pub const model = @import("qwen3_5/model.zig"); +pub const Config = model.Config; +pub const Buffers = model.Buffers; +pub const Model = model.Model; +pub const repository = @import("qwen3_5/repository.zig"); +pub const Repository = repository.Repository; +pub const session = @import("qwen3_5/session.zig"); +pub const Session = session.Session; diff --git a/examples/llm/models/qwen3_5/inference.zig b/examples/llm/models/qwen3_5/inference.zig new file mode 100644 index 00000000..a630af76 --- /dev/null +++ b/examples/llm/models/qwen3_5/inference.zig @@ -0,0 +1,143 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const Shardings = @import("../common.zig").Shardings; +const model = @import("model.zig"); + +const log = std.log.scoped(.qwen3_5); + +const CompileModelResult = struct { + prefill_exe: ?zml.Exe = null, + decode_exe: ?zml.Exe = null, +}; + +pub const CompilationOptions = struct { + prefill_tokens: zml.Tensor, + decode_tokens: zml.Tensor, + token_index: zml.Tensor, + kv_cache: model.KvCache, + rng: zml.Tensor.Rng, + + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32) CompilationOptions { + const dtype = mdl.text_model.embed_tokens.weight.dtype(); + return .{ + .prefill_tokens = .init(.{ .b = 1, .s = seqlen }, .u32), + .decode_tokens = .init(.{ .b = 1, .s = 1 }, .u32), + .token_index = .init(.{}, .u32), + .kv_cache = .init(config, 1, seqlen, dtype, .f32), + .rng = .init(), + }; + } +}; + +pub const Inference = struct { + prefill_exe: zml.Exe, + decode_exe: zml.Exe, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + qwen_model: model.Model, + parameters: CompilationOptions, + shardings: Shardings, + progress: *std.Progress.Node, + ) !Inference { + const compile_result = try compileModel(allocator, io, platform, qwen_model, parameters, shardings, progress); + return .{ .prefill_exe = compile_result.prefill_exe.?, .decode_exe = compile_result.decode_exe.? }; + } + + pub fn deinit(self: *Inference) void { + self.prefill_exe.deinit(); + self.decode_exe.deinit(); + } +}; + +fn compileModel( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + qwen_model: model.Model, + parameters: CompilationOptions, + shardings: Shardings, + progress: *std.Progress.Node, +) !CompileModelResult { + const now: std.Io.Timestamp = .now(io, .awake); + defer log.info("Compiled model [{f}]", .{now.untilNow(io, .awake)}); + + const all_shardings = [_]zml.sharding.Sharding{shardings.replicated}; + + var prefill_future = try io.concurrent(struct { + fn call( + allocator_: std.mem.Allocator, + io_: std.Io, + platform_: *const zml.Platform, + qwen_model_: model.Model, + parameters_: CompilationOptions, + shardings_: [1]zml.sharding.Sharding, + progress_: *std.Progress.Node, + ) !zml.Exe { + progress_.increaseEstimatedTotalItems(1); + var node_ = progress_.start("Compiling prefill...", 1); + defer node_.end(); + + const now_: std.Io.Timestamp = .now(io_, .awake); + defer log.info("Compiled prefill [{f}]", .{now_.untilNow(io_, .awake)}); + + return platform_.compile( + allocator_, + io_, + qwen_model_, + .forward, + .{ + parameters_.prefill_tokens, + parameters_.token_index, + parameters_.kv_cache, + parameters_.rng, + }, + .{ .shardings = &shardings_ }, + ); + } + }.call, .{ allocator, io, platform, qwen_model, parameters, all_shardings, progress }); + errdefer if (prefill_future.cancel(io)) |v| v.deinit() else |_| {}; + + var decode_future = try io.concurrent(struct { + fn call( + allocator_: std.mem.Allocator, + io_: std.Io, + platform_: *const zml.Platform, + qwen_model_: model.Model, + parameters_: CompilationOptions, + shardings_: [1]zml.sharding.Sharding, + progress_: *std.Progress.Node, + ) !zml.Exe { + progress_.increaseEstimatedTotalItems(1); + var node_ = progress_.start("Compiling decode...", 1); + defer node_.end(); + + const now_: std.Io.Timestamp = .now(io_, .awake); + defer log.info("Compiled decode [{f}]", .{now_.untilNow(io_, .awake)}); + + return platform_.compile( + allocator_, + io_, + qwen_model_, + .forward, + .{ + parameters_.decode_tokens, + parameters_.token_index, + parameters_.kv_cache, + parameters_.rng, + }, + .{ .shardings = &shardings_ }, + ); + } + }.call, .{ allocator, io, platform, qwen_model, parameters, all_shardings, progress }); + errdefer if (decode_future.cancel(io)) |v| v.deinit() else |_| {}; + + return .{ + .prefill_exe = try prefill_future.await(io), + .decode_exe = try decode_future.await(io), + }; +} diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig new file mode 100644 index 00000000..76b4f9a4 --- /dev/null +++ b/examples/llm/models/qwen3_5/model.zig @@ -0,0 +1,984 @@ +const std = @import("std"); + +const zml = @import("zml"); +const stdx = zml.stdx; + +const log = std.log.scoped(.qwen3_5); + +pub const Config = struct { + text_config: TextConfig, +}; + +pub const TextConfig = struct { + // General + num_hidden_layers: i64, + layer_types: []const LayerType, + hidden_size: i64, + max_position_embeddings: i64, + rms_norm_eps: f32, + // Self attention + head_dim: i64, + num_attention_heads: i64, + num_key_value_heads: i64, + rope_parameters: RopeParameters, + // Linear attention + linear_conv_kernel_dim: i64, + linear_key_head_dim: i64, + linear_num_key_heads: i64, + linear_num_value_heads: i64, + linear_value_head_dim: i64, +}; + +// Each layer uses either: full attention (SelfAttn) or linear attention (GatedDeltaNet). +pub const LayerType = enum { + linear_attention, + full_attention, +}; + +pub const RopeParameters = struct { + mrope_section: [3]i64, + partial_rotary_factor: f32, + rope_theta: f32, +}; + +pub const Buffers = zml.Bufferized(Model); + +pub const Model = struct { + pub const GenOptions = struct { sampling_strategy: zml.nn.SamplingStrategy = .{ + .topk = 4, + }, max_seq_len: i64 }; + + pub const SpecialTokens = struct { + im_start_token_id: u32, + im_end_token_id: u32, + end_of_text_token_id: u32, + }; + + text_model: TextModel, + lm_head: zml.nn.Linear, + + config: Config, + gen_options: GenOptions, + special_tokens: SpecialTokens = .{ + .im_start_token_id = 248045, + .im_end_token_id = 248046, + .end_of_text_token_id = 248044, + }, + + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config, gen_options: GenOptions) !Model { + // For some Qwen3.5 versions, the output projection lm_head has a standalone weight tensor, while for others it's the same as the input embedding layer + const lm_head_prefix = if (store.hasKey("lm_head.weight")) "lm_head" else "model.language_model.embed_tokens"; + return .{ + .text_model = try .init(allocator, store.withPrefix("model.language_model"), config), + .lm_head = .init(store.withPrefix(lm_head_prefix).createTensor("weight", .{ .dout, .d }, null), null, .d), + .config = config, + .gen_options = gen_options, + }; + } + + pub fn deinit(self: Model, allocator: std.mem.Allocator) void { + self.text_model.deinit(allocator); + } + + pub fn load( + self: *const Model, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + shardings: []const zml.sharding.Sharding, + progress: *std.Progress.Node, + ) !zml.Bufferized(Model) { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + var total_bytes: usize = 0; + defer { + const took = now.untilNow(io, .awake); + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + } + return zml.io.load(Model, self, allocator, io, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 128 * zml.MiB, + .progress = progress, + .store = store, + .shardings = shardings, + .parallelism = 16, + .total_bytes = &total_bytes, + }); + } + + pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { + TextModel.unloadBuffers(&self.text_model, allocator); + self.lm_head.weight.deinit(); + } + + pub fn forward( + self: Model, + tokens_: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache, + rng: zml.Tensor.Rng, + ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { + const tokens = tokens_.withPartialTags(.{.s}); + const text_model_output, const updated_kv_cache = self.text_model.forward(tokens, token_index, kv_cache); + const new_tokens, const new_rng = self.sampleTokens(text_model_output, rng); + return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; + } + + pub fn sampleTokens( + self: Model, + out: zml.Tensor, + rng: zml.Tensor.Rng, + ) struct { zml.Tensor, zml.Tensor.Rng } { + const logits = self.lm_head.forward(out.withPartialTags(.{.d})).rename(.{ .dout = .voc }); + const next_tokens, const new_rng = zml.nn.sampleTokens(logits, self.gen_options.sampling_strategy, rng); + return .{ next_tokens, new_rng }; + } +}; + +//========================Text model======================== + +pub const TextModel = struct { + embed_tokens: zml.nn.TokenEmbedding, + layers: []TransformerLayer, + norm: RmsNorm, + + pub fn init( + allocator: std.mem.Allocator, + store: zml.io.TensorStore.View, + config: Config, + ) !TextModel { + const layers = try allocator.alloc(TransformerLayer, @intCast(config.text_config.num_hidden_layers)); + errdefer allocator.free(layers); + + for (layers, 0..) |*layer, i| { + layer.* = try .init(store.withPrefix("layers").withLayer(i), config, i); + } + + return .{ + .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight", .{ .voc, .d }, null) }, + .layers = layers, + .norm = RmsNorm.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), + }; + } + + pub fn deinit(self: TextModel, allocator: std.mem.Allocator) void { + allocator.free(self.layers); + } + + pub fn unloadBuffers(self: *zml.Bufferized(TextModel), allocator: std.mem.Allocator) void { + self.embed_tokens.weight.deinit(); + for (self.layers) |*layer| { + TransformerLayer.unloadBuffers(layer); + } + allocator.free(self.layers); + RmsNorm.unloadBuffers(&self.norm); + } + + pub fn forward( + self: TextModel, + tokens: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache, + ) struct { zml.Tensor, KvCache } { + var hidden_states = self.embed_tokens.weight.gather(.{ .voc = tokens }, .{}); + + var updated_kv_cache = kv_cache; + for (self.layers, 0..) |layer, i| { + hidden_states, updated_kv_cache = layer.forward(hidden_states, token_index, updated_kv_cache.atLayer(i)); + } + + hidden_states = self.norm.forward(hidden_states); + return .{ hidden_states, updated_kv_cache.reuseBuffer(kv_cache) }; + } +}; + +pub const TransformerLayer = struct { + const Attn = union(enum) { + self_attn: SelfAttn, + linear_attn: GatedDeltaNet, + }; + + input_layernorm: RmsNorm, + attn: Attn, + mlp: Mlp, + post_attention_layernorm: RmsNorm, + + pub fn init(store: zml.io.TensorStore.View, config: Config, layer_index: usize) !TransformerLayer { + const is_full_attention = config.text_config.layer_types[layer_index] == .full_attention; + return .{ + .input_layernorm = RmsNorm.init(store.withPrefix("input_layernorm"), config.text_config.rms_norm_eps), + .attn = if (is_full_attention) + .{ .self_attn = try .init(store.withPrefix("self_attn"), config) } + else + .{ .linear_attn = .init(store.withPrefix("linear_attn"), config) }, + .mlp = .init(store.withPrefix("mlp")), + .post_attention_layernorm = RmsNorm.init(store.withPrefix("post_attention_layernorm"), config.text_config.rms_norm_eps), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer)) void { + RmsNorm.unloadBuffers(&self.input_layernorm); + switch (self.attn) { + .self_attn => |*self_attn| SelfAttn.unloadBuffers(self_attn), + .linear_attn => |*linear_attn| GatedDeltaNet.unloadBuffers(linear_attn), + } + Mlp.unloadBuffers(&self.mlp); + RmsNorm.unloadBuffers(&self.post_attention_layernorm); + } + + pub fn forward( + self: TransformerLayer, + x0: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache.LayerView, + ) struct { zml.Tensor, KvCache } { + const residual0 = x0; + const normalized_x0 = self.input_layernorm.forward(x0); + + var attention_output: zml.Tensor = undefined; + var updated_kv_cache: KvCache = kv_cache.parent; + switch (self.attn) { + .self_attn => |*self_attn| { + const result = self_attn.forward(normalized_x0, token_index, kv_cache.cache.self_attn); + attention_output = result[0]; + updated_kv_cache.self_attn = result[1]; + }, + .linear_attn => |*linear_attn| { + const result = linear_attn.forward(normalized_x0, kv_cache.cache.linear_attn); + attention_output = result[0]; + updated_kv_cache.gated_delta_net = result[1]; + }, + } + + const x1 = attention_output.add(residual0); + const residual1 = x1; + const normalized_hidden = self.post_attention_layernorm.forward(x1); + + const mlp_output = self.mlp.forward(normalized_hidden); + + return .{ mlp_output.add(residual1), updated_kv_cache }; + } +}; + +pub const Mlp = struct { + up_proj: zml.nn.Linear, + gate_proj: zml.nn.Linear, + down_proj: zml.nn.Linear, + + pub fn init(store: zml.io.TensorStore.View) Mlp { + return .{ + .up_proj = .init(store.withPrefix("up_proj").createTensor("weight", .{ .dout, .d }, null), store.withPrefix("up_proj").maybeCreateTensor("bias", .{.dout}, null), .d), + .gate_proj = .init(store.withPrefix("gate_proj").createTensor("weight", .{ .dout, .d }, null), store.withPrefix("gate_proj").maybeCreateTensor("bias", .{.dout}, null), .d), + .down_proj = .init(store.withPrefix("down_proj").createTensor("weight", .{ .d, .dout }, null), store.withPrefix("down_proj").maybeCreateTensor("bias", .{.d}, null), .dout), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { + self.up_proj.weight.deinit(); + if (self.up_proj.bias) |*bias| bias.deinit(); + self.gate_proj.weight.deinit(); + if (self.gate_proj.bias) |*bias| bias.deinit(); + self.down_proj.weight.deinit(); + if (self.down_proj.bias) |*bias| bias.deinit(); + } + + pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { + const up_projed = self.up_proj.forward(x); + const gate = self.gate_proj.forward(x); + const hidden = gate.silu().mul(up_projed); + + const output = self.down_proj.forward(hidden); + return output; + } +}; + +pub const SelfAttn = struct { + q_proj: zml.nn.Linear, + k_proj: zml.nn.Linear, + v_proj: zml.nn.Linear, + + q_norm: RmsNorm, + k_norm: RmsNorm, + + num_heads: i64, + num_kv_heads: i64, + head_dim: i64, + rotary_dim: i64, + rotary_embed: TextRotaryEmbedding, + o_proj: zml.nn.Linear, + + fn initProj(store: zml.io.TensorStore.View) zml.nn.Linear { + return .init(store.createTensor("weight", .{ .dout, .d }, null), store.maybeCreateTensor("bias", .{.dout}, null), .d); + } + + pub fn init(store: zml.io.TensorStore.View, config: Config) !SelfAttn { + const rotary_dim: i64 = @intFromFloat(@as(f32, @floatFromInt(config.text_config.head_dim)) * + config.text_config.rope_parameters.partial_rotary_factor); + return .{ + .q_proj = initProj(store.withPrefix("q_proj")), + .k_proj = initProj(store.withPrefix("k_proj")), + .v_proj = initProj(store.withPrefix("v_proj")), + .o_proj = initProj(store.withPrefix("o_proj")), + .q_norm = RmsNorm.init(store.withPrefix("q_norm"), config.text_config.rms_norm_eps), + .k_norm = RmsNorm.init(store.withPrefix("k_norm"), config.text_config.rms_norm_eps), + .num_heads = config.text_config.num_attention_heads, + .num_kv_heads = config.text_config.num_key_value_heads, + .head_dim = config.text_config.head_dim, + .rotary_dim = rotary_dim, + .rotary_embed = .init(rotary_dim, config.text_config.rope_parameters.rope_theta, config.text_config.rope_parameters.mrope_section), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { + self.q_proj.weight.deinit(); + if (self.q_proj.bias) |*bias| bias.deinit(); + self.k_proj.weight.deinit(); + if (self.k_proj.bias) |*bias| bias.deinit(); + self.v_proj.weight.deinit(); + if (self.v_proj.bias) |*bias| bias.deinit(); + self.o_proj.weight.deinit(); + if (self.o_proj.bias) |*bias| bias.deinit(); + RmsNorm.unloadBuffers(&self.q_norm); + RmsNorm.unloadBuffers(&self.k_norm); + } + + fn projectQAndGate(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { + const q_proj = self.q_proj.forward(x).splitAxis(.dout, .{ .h = self.num_heads, .hd = 2 * self.head_dim }); + const q, var gate = q_proj.chunkExact(.hd, 2); + gate = gate.merge(.{ .d_out_proj = .{ .h, .hd } }); + return .{ q, gate }; + } + + fn projectKV(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { + const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; + const k = self.k_proj.forward(x).splitAxis(.dout, .{ .h = num_kv_heads, .hd = self.head_dim }); + const v = self.v_proj.forward(x).splitAxis(.dout, .{ .h = num_kv_heads, .hd = self.head_dim }); + return .{ k, v }; + } + + pub fn forward( + self: SelfAttn, + x: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache.SelfAttnCache, + ) struct { zml.Tensor, KvCache.SelfAttnCache } { + var q, const gate = self.projectQAndGate(x); + var k, var v = self.projectKV(x); + q = self.q_norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); + k = self.k_norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); + + const dtype = q.dtype(); + const position_ids = zml.Tensor.arange(.{ .end = x.dim(.s) }, .i64) + .withTags(.{.s}).insertAxes(.s, .{.b}).broad(zml.Shape.init(.{ .b = x.dim(.b), .s = x.dim(.s) }, .i64)) + .add(token_index.convert(.i64).broad(zml.Shape.init(.{ .b = x.dim(.b), .s = x.dim(.s) }, .i64))); + + const cos, const sin = self.rotary_embed.getCosAndSin(position_ids, dtype); + q = self.rotary_embed.applyRope(q, cos, sin); + k = self.rotary_embed.applyRope(k, cos, sin); + + const new_kv_cache = kv_cache.update(k, v, token_index.convert(.u32)); + k = new_kv_cache.keys().convert(dtype); + v = new_kv_cache.values().convert(dtype); + + const attn_output = zml.attention.attention.attention( + q.rename(.{ .s = .q }), + k.rename(.{ .s = .k }), + v.rename(.{ .s = .k }), + token_index, + zml.attention.attention.Metadata.init(.fromBackend(.vanilla, x.dim(.s), self.num_heads)), + zml.attention.attention.Parameters.init(.fromBackend(.vanilla)), + ).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); + + const gated_output = attn_output.mul(gate.sigmoid()); + const projected_output = self.o_proj.forward(gated_output.rename(.{ .d_out_proj = .d })).rename(.{ .dout = .d }); + + return .{ projected_output, new_kv_cache }; + } +}; + +pub const TextRotaryEmbedding = struct { + rope_opts: zml.nn.RopeOpts, + rotary_dim: i64, + mrope_section: [3]i64, + + pub fn init(rotary_dim: i64, theta: f32, mrope_section: [3]i64) TextRotaryEmbedding { + return .{ + .rope_opts = .{ + .layout = .sequential, + .scaling = .{ .default = .{ .rope_theta = theta } }, + }, + .rotary_dim = rotary_dim, + .mrope_section = mrope_section, + }; + } + + pub fn getCosAndSin(self: TextRotaryEmbedding, position_ids: zml.Tensor, dtype: zml.DataType) struct { zml.Tensor, zml.Tensor } { + const inv_freq = zml.nn.invFreq(self.rotary_dim, self.rope_opts).withTags(.{.hd}); + + const freqs_t = position_ids.convert(.f32).outer(inv_freq); + + const emb = zml.Tensor.concatenate(&.{ freqs_t, freqs_t }, -1); + const cos = emb.cos().convert(dtype); + const sin = emb.sin().convert(dtype); + + return .{ cos, sin }; + } + + pub fn getCosAndSinInterleaved(self: TextRotaryEmbedding, position_ids: zml.Tensor, dtype: zml.DataType) struct { zml.Tensor, zml.Tensor } { // To be used later in image extension + const stacked_position_ids = zml.Tensor.stack(&.{ position_ids, position_ids, position_ids }, 0, .g).convert(.f32); + const inv_freq = zml.nn.InvFreq(self.rotary_dim, self.rope_opts).withTags(.{.hd}); + + var freqs = stacked_position_ids.outer(inv_freq); + var freqs_t, var freqs_h, var freqs_w = freqs.chunkExact(.g, 3); + freqs_t = freqs_t.squeeze(.g); + freqs_h = freqs_h.squeeze(.g); + freqs_w = freqs_w.squeeze(.g); + + const h_indices = zml.Tensor.iota(zml.Shape.init(.{ .h = self.mrope_section[1] }, .i32), .h).scale(3).addConstant(1); + const w_indices = zml.Tensor.iota(zml.Shape.init(.{ .h = self.mrope_section[2] }, .i32), .h).scale(3).addConstant(2); + + const h_input = freqs_h.gather(.{ .dh = h_indices }, .{ .indices_are_sorted = true }); + const w_input = freqs_w.gather(.{ .dh = w_indices }, .{ .indices_are_sorted = true }); + freqs_t = freqs_t.scatterSlices(.{ .dh = h_indices }, h_input, .{ .update_fn = zml.Tensor.ScatterOpts.override }); + freqs = freqs_t.scatterSlices(.{ .dh = w_indices }, w_input, .{ .update_fn = zml.Tensor.ScatterOpts.override }); + + const emb = zml.Tensor.concatenate(&.{ freqs, freqs }, -1); + const cos = emb.cos().convert(dtype); + const sin = emb.sin().convert(dtype); + + return .{ cos, sin }; + } + + fn rotateHalf(x: zml.Tensor) zml.Tensor { + const half_dim = @divExact(x.dim(-1), 2); + const x1 = x.slice1d(-1, .{ .start = 0, .end = half_dim }); + const x2 = x.slice1d(-1, .{ .start = half_dim, .end = x.dim(-1) }); + return zml.Tensor.concatenate(&.{ x2.negate(), x1 }, -1); + } + + pub fn applyRope(self: TextRotaryEmbedding, x: zml.Tensor, cos: zml.Tensor, sin: zml.Tensor) zml.Tensor { + const x_rot = x.slice1d(-1, .{ .start = 0, .end = self.rotary_dim }); + const x_pass = x.slice1d(-1, .{ .start = self.rotary_dim, .end = x.dim(-1) }); + + const cos_x = cos.insertAxes(.hd, .{.h}).broad(x_rot.shape()); + const sin_x = sin.insertAxes(.hd, .{.h}).broad(x_rot.shape()); + + const rotated = x_rot.mul(cos_x).add(rotateHalf(x_rot).mul(sin_x)); + + return zml.Tensor.concatenate(&.{ rotated, x_pass }, -1); + } +}; + +pub const GatedDeltaNet = struct { + in_proj_qkv: zml.nn.Linear, + in_proj_z: zml.nn.Linear, + in_proj_b: zml.nn.Linear, + in_proj_a: zml.nn.Linear, + out_proj: zml.nn.Linear, + conv1d_weight: zml.Tensor, + dt_bias: zml.Tensor, + aLog: zml.Tensor, + norm: RmsNormGated, + + num_k_heads: i64, + num_v_heads: i64, + qk_head_repetition: i64, + head_k_dim: i64, + head_v_dim: i64, + conv_kernel_size: i64, + + fn initProj(store: zml.io.TensorStore.View) zml.nn.Linear { + return .init(store.createTensor("weight", .{ .dout, .d }, null), null, .d); + } + + pub fn init(store: zml.io.TensorStore.View, config: Config) GatedDeltaNet { + const qk_head_repetition = + @divExact(config.text_config.linear_num_value_heads, config.text_config.linear_num_key_heads); + return .{ + .in_proj_qkv = initProj(store.withPrefix("in_proj_qkv")), + .in_proj_z = initProj(store.withPrefix("in_proj_z")), + .in_proj_b = initProj(store.withPrefix("in_proj_b")), + .in_proj_a = initProj(store.withPrefix("in_proj_a")), + .out_proj = initProj(store.withPrefix("out_proj")), + .conv1d_weight = store.withPrefix("conv1d").createTensor("weight", .{ .out, .in, .kernel_size }, null), + .dt_bias = store.createTensor("dt_bias", .{.vh}, null), + .aLog = store.createTensor("A_log", .{.vh}, null), + .norm = RmsNormGated.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), + .num_k_heads = config.text_config.linear_num_key_heads, + .num_v_heads = config.text_config.linear_num_value_heads, + .qk_head_repetition = qk_head_repetition, + .head_k_dim = config.text_config.linear_key_head_dim, + .head_v_dim = config.text_config.linear_value_head_dim, + .conv_kernel_size = config.text_config.linear_conv_kernel_dim, + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(GatedDeltaNet)) void { + self.in_proj_qkv.weight.deinit(); + self.in_proj_z.weight.deinit(); + self.in_proj_b.weight.deinit(); + self.in_proj_a.weight.deinit(); + self.out_proj.weight.deinit(); + self.conv1d_weight.deinit(); + self.dt_bias.deinit(); + self.aLog.deinit(); + RmsNormGated.unloadBuffers(&self.norm); + } + + fn recurrent_gated_delta_rule(query: zml.Tensor, key: zml.Tensor, value: zml.Tensor, g: zml.Tensor, beta: zml.Tensor, initial_state: ?zml.Tensor) struct { zml.Tensor, zml.Tensor } { + const scale: f32 = 1.0 / @sqrt(@as(f32, @floatFromInt(query.dim(.khd)))); + const query_norm = zml.nn.normalizeL2(query.rename(.{ .kh = .vh }), 1e-6); + const key_norm = zml.nn.normalizeL2(key.rename(.{ .kh = .vh }), 1e-6); + + const query_f32, const key_f32, const value_f32, const alpha_f32, const beta_f32 = .{ + query_norm.convert(.f32).scale(scale).rename(.{ .vh = .h, .khd = .k }), + key_norm.convert(.f32).rename(.{ .vh = .h, .khd = .k }), + value.convert(.f32).rename(.{ .vh = .h, .vhd = .v }), + g.convert(.f32).exp().rename(.{ .vh = .h }), + beta.convert(.f32).rename(.{ .vh = .h }), + }; + + const initial_recurrent_state = if (initial_state) |state| + state.convert(.f32).transpose(.{ .b, .vh, .vhd, .khd }).rename(.{ .vh = .h, .vhd = .v, .khd = .k }) + else + zml.Tensor.constant(zml.DataType.zero(.f32)).broad(zml.Shape.init(.{ + .b = value.dim(.b), + .h = value.dim(.vh), + .v = value.dim(.vhd), + .k = query.dim(.khd), + }, .f32)); + + const result = zml.nn.GatedDeltaNet.forward( + query_f32, + key_f32, + value_f32, + alpha_f32, + beta_f32, + .{ .s = initial_recurrent_state }, + ); + + return .{ + result.outputs.rename(.{ .h = .vh, .v = .vhd }).convert(query.dtype()), + result.state.s.transpose(.{ .b, .h, .k, .v }).rename(.{ .h = .vh, .k = .khd, .v = .vhd }), + }; + } + + fn buildUpdatedConvState(input: zml.Tensor, left_pad: i64) zml.Tensor { + const copy_len = @min(input.dim(.s), left_pad); + const tail = input.slice1d(.s, .{ .start = input.dim(.s) - copy_len, .end = input.dim(.s) }); + if (copy_len == left_pad) return tail; + + const padding_shape = zml.Shape.init(.{ .b = input.dim(.b), .s = left_pad - copy_len, .mix = input.dim(.mix) }, input.dtype()); + const padding = zml.Tensor.constant(input.dtype().zero()).broad(padding_shape); + return zml.Tensor.concatenate(&.{ padding, tail }, .s); + } + + pub fn forward(self: GatedDeltaNet, x: zml.Tensor, cache: KvCache.GatedDeltaNetCache) struct { zml.Tensor, KvCache.GatedDeltaNetCache } { + const key_dim = self.num_k_heads * self.head_k_dim; + const value_dim = self.num_v_heads * self.head_v_dim; + const conv_dim = 2 * key_dim + value_dim; + const left_pad = self.conv_kernel_size - 1; + + const projected_qkv = self.in_proj_qkv.forward(x).rename(.{ .dout = .mix }); + const use_cached_state = x.dim(.s) == 1 and left_pad > 0; + const conv_input = if (use_cached_state) + zml.Tensor.concatenate(&.{ cache.convState(), projected_qkv }, .s) + else + projected_qkv; + + const kernel = self.conv1d_weight; + var mixed_qkv = zml.Tensor.conv1d( + conv_input, + kernel, + .{ + .padding = &.{ left_pad, 0 }, + .input_batch_dimension = 0, + .input_feature_dimension = 2, + .input_spatial_dimensions = 1, + .kernel_output_feature_dimension = 0, + .kernel_input_feature_dimension = 1, + .kernel_spatial_dimensions = 2, + .output_batch_dimension = 0, + .output_feature_dimension = 2, + .output_spatial_dimensions = 1, + .feature_group_count = conv_dim, + }, + ) + .silu(); + + if (use_cached_state) { + mixed_qkv = mixed_qkv.slice1d(.s, .{ .start = mixed_qkv.dim(.s) - 1, .end = mixed_qkv.dim(.s) }); + } + + const z = self.in_proj_z.forward(x).splitAxis(.dout, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }); + const b = self.in_proj_b.forward(x).rename(.{ .dout = .vh }); + const a = self.in_proj_a.forward(x).rename(.{ .dout = .vh }); + + const query = mixed_qkv + .slice1d(.mix, .{ .start = 0, .end = key_dim }) + .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }); + const key = mixed_qkv + .slice1d(.mix, .{ .start = key_dim, .end = 2 * key_dim }) + .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }); + const value = mixed_qkv + .slice1d(.mix, .{ .start = 2 * key_dim, .end = 2 * key_dim + value_dim }) + .splitAxis(.mix, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }); + + const beta = b.sigmoid(); + const aLog_type = self.aLog.dtype(); + const g = self.aLog.broad(a.shape()).exp().mul(softplus(a.convert(aLog_type).add(self.dt_bias.convert(aLog_type).broad(a.shape())))).negate(); + + const query_for_rule = if (self.qk_head_repetition == 1) query else query.stutter1d(@intCast(query.axis(.kh)), @intCast(self.qk_head_repetition)); + const key_for_rule = if (self.qk_head_repetition == 1) key else key.stutter1d(@intCast(key.axis(.kh)), @intCast(self.qk_head_repetition)); + + const core_attn_out, const last_recurrent_state = recurrent_gated_delta_rule( + query_for_rule, + key_for_rule, + value, + g, + beta, + if (use_cached_state) cache.recurrentState() else null, + ); + + const core_attn_out_normed = self.norm + .forward( + core_attn_out.rename(.{ .vhd = .d }), + z.rename(.{ .vhd = .d }), + ) + .rename(.{ .d = .vhd }); + + const output = self.out_proj.forward(core_attn_out_normed.merge(.{ .d = .{ .vh, .vhd } })).rename(.{ .dout = .d }); + const updated_cache = cache.update( + buildUpdatedConvState(conv_input, left_pad), + last_recurrent_state, + ); + return .{ output, updated_cache }; + } +}; + +pub const RmsNorm = struct { + weight: zml.Tensor, + eps: f32 = 1e-6, + + pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { + return .{ .weight = store.createTensor("weight", .{.d}, null), .eps = eps }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { + self.weight.deinit(); + } + + pub fn forward(self: RmsNorm, x: zml.Tensor) zml.Tensor { + const x_f32 = x.convert(.f32); + const weight_f32 = self.weight.convert(.f32); + + const normalized = zml.nn.rmsNorm(x_f32, .d, self.eps); + return normalized.mul(weight_f32.broad(x.shape())).add(normalized).convert(x.dtype()); + } +}; + +pub const RmsNormGated = struct { + weight: zml.Tensor, + eps: f32 = 1e-6, + + pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNormGated { + return .{ .weight = store.createTensor("weight", .{.d}, null), .eps = eps }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(RmsNormGated)) void { + self.weight.deinit(); + } + + pub fn forward(self: RmsNormGated, x: zml.Tensor, gate: zml.Tensor) zml.Tensor { + const x_f32 = x.convert(.f32); + const gate_f32 = gate.convert(.f32); + + const normalized = zml.nn.rmsNorm(x_f32, .d, self.eps); + const output = normalized.mul(self.weight.broad(x.shape())); + + const gated_output = output.mul(gate_f32.silu()); + return gated_output.convert(x.dtype()); + } +}; + +pub const KvCache = struct { + layer_types: []const LayerType, + self_attn: SelfAttnCache, + gated_delta_net: GatedDeltaNetCache, + + pub const SelfAttnCache = struct { + k: zml.Tensor, + v: zml.Tensor, + layer_index: zml.Tensor, + + pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType) SelfAttnCache { + const num_self_attn_layers = countLayers(config.text_config.layer_types, .full_attention); + const kv_shape = zml.Shape.init(.{ + .b = batch_dim, + .layer = num_self_attn_layers, + .s = max_seq_len, + .h = config.text_config.num_key_value_heads, + .hd = config.text_config.head_dim, + }, dtype); + return .{ + .k = .fromShape(kv_shape), + .v = .fromShape(kv_shape), + .layer_index = .init(.{}, .u32), + }; + } + + pub fn initBuffer(self: SelfAttnCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(SelfAttnCache) { + const sharding = try zml.sharding.replicatedSharding(platform); + return .{ + .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), sharding, .{}), + .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), sharding, .{}), + .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32, sharding), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(SelfAttnCache)) void { + self.k.deinit(); + self.v.deinit(); + self.layer_index.deinit(); + } + + pub fn keys(self: SelfAttnCache) zml.Tensor { + return self.k.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn values(self: SelfAttnCache) zml.Tensor { + return self.v.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn update(self: SelfAttnCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: ?zml.Tensor) SelfAttnCache { + const k_shape = self.k.shape().drop(.layer); + var layer = self.layer_index; + layer = if (token_index) |idx| layer.broad(idx.shape()) else layer; + + return if (token_index) |idx| .{ + .k = self.k.scatterSlices( + .{ .layer = layer, .s = idx }, + new_k.convert(self.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.k), + .v = self.v.scatterSlices( + .{ .layer = layer, .s = idx }, + new_v.convert(self.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.v), + .layer_index = self.layer_index, + } else .{ + .k = self.k.scatterSlices( + .{ .layer = layer }, + new_k.convert(self.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.k), + .v = self.v.scatterSlices( + .{ .layer = layer }, + new_v.convert(self.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.v), + .layer_index = self.layer_index, + }; + } + + pub fn atLayer(self: SelfAttnCache, layer_index: usize) SelfAttnCache { + return .{ + .k = self.k, + .v = self.v, + .layer_index = zml.Tensor.scalar(layer_index, .u32), + }; + } + + pub fn reuseBuffer(self: SelfAttnCache, other: SelfAttnCache) SelfAttnCache { + return .{ + .k = self.k.reuseBuffer(other.k), + .v = self.v.reuseBuffer(other.v), + .layer_index = self.layer_index.reuseBuffer(other.layer_index), + }; + } + }; + + pub const GatedDeltaNetCache = struct { + conv_state: zml.Tensor, + recurrent_state: zml.Tensor, + layer_index: zml.Tensor, + + pub fn init(config: Config, batch_dim: i64, conv_dtype: zml.DataType, recurrent_dtype: zml.DataType) GatedDeltaNetCache { + const num_linear_attn_layers = countLayers(config.text_config.layer_types, .linear_attention); + const conv_dim = 2 * config.text_config.linear_num_key_heads * config.text_config.linear_key_head_dim + config.text_config.linear_num_value_heads * config.text_config.linear_value_head_dim; + const conv_state_shape = zml.Shape.init(.{ + .b = batch_dim, + .layer = num_linear_attn_layers, + .s = config.text_config.linear_conv_kernel_dim - 1, + .mix = conv_dim, + }, conv_dtype); + const recurrent_state_shape = zml.Shape.init(.{ + .b = batch_dim, + .layer = num_linear_attn_layers, + .vh = config.text_config.linear_num_value_heads, + .khd = config.text_config.linear_key_head_dim, + .vhd = config.text_config.linear_value_head_dim, + }, recurrent_dtype); + return .{ + .conv_state = .fromShape(conv_state_shape), + .recurrent_state = .fromShape(recurrent_state_shape), + .layer_index = .init(.{}, .u32), + }; + } + + pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(GatedDeltaNetCache) { + const sharding = try zml.sharding.replicatedSharding(platform); + return .{ + .conv_state = try zml.Buffer.uninitialized(io, platform, self.conv_state.shape(), sharding, .{}), + .recurrent_state = try zml.Buffer.uninitialized(io, platform, self.recurrent_state.shape(), sharding, .{}), + .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32, sharding), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(GatedDeltaNetCache)) void { + self.conv_state.deinit(); + self.recurrent_state.deinit(); + self.layer_index.deinit(); + } + + pub fn convState(self: GatedDeltaNetCache) zml.Tensor { + return self.conv_state.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn recurrentState(self: GatedDeltaNetCache) zml.Tensor { + return self.recurrent_state.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn update(self: GatedDeltaNetCache, new_conv_state: ?zml.Tensor, new_recurrent_state: ?zml.Tensor) GatedDeltaNetCache { + const conv_state = if (new_conv_state) |state| + self.conv_state.scatterSlices( + .{ .layer = self.layer_index }, + state.convert(self.conv_state.dtype()).transpose(self.conv_state.shape().drop(.layer)), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.conv_state) + else + self.conv_state; + + const recurrent_state = if (new_recurrent_state) |state| + self.recurrent_state.scatterSlices( + .{ .layer = self.layer_index }, + state.convert(self.recurrent_state.dtype()).transpose(self.recurrent_state.shape().drop(.layer)), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.recurrent_state) + else + self.recurrent_state; + + return .{ + .conv_state = conv_state, + .recurrent_state = recurrent_state, + .layer_index = self.layer_index, + }; + } + + pub fn atLayer(self: GatedDeltaNetCache, layer_index: usize) GatedDeltaNetCache { + return .{ + .conv_state = self.conv_state, + .recurrent_state = self.recurrent_state, + .layer_index = zml.Tensor.scalar(layer_index, .u32), + }; + } + + pub fn reuseBuffer(self: GatedDeltaNetCache, other: GatedDeltaNetCache) GatedDeltaNetCache { + return .{ + .conv_state = self.conv_state.reuseBuffer(other.conv_state), + .recurrent_state = self.recurrent_state.reuseBuffer(other.recurrent_state), + .layer_index = self.layer_index.reuseBuffer(other.layer_index), + }; + } + }; + + pub fn init( + config: Config, + batch_dim: i64, + max_seq_len: i64, + cache_dtype: zml.DataType, + recurrent_dtype: zml.DataType, + ) KvCache { + return .{ + .layer_types = config.text_config.layer_types, + .self_attn = SelfAttnCache.init(config, batch_dim, max_seq_len, cache_dtype), + .gated_delta_net = GatedDeltaNetCache.init(config, batch_dim, cache_dtype, recurrent_dtype), + }; + } + + pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { + return .{ + .self_attn = try self.self_attn.initBuffer(io, platform), + .gated_delta_net = try self.gated_delta_net.initBuffer(io, platform), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(KvCache)) void { + SelfAttnCache.deinitBuffer(&self.self_attn); + GatedDeltaNetCache.deinitBuffer(&self.gated_delta_net); + } + + pub const LayerView = struct { + parent: KvCache, + cache: union(enum) { + self_attn: SelfAttnCache, + linear_attn: GatedDeltaNetCache, + }, + }; + + pub fn atLayer(self: KvCache, layer_index: usize) LayerView { + return switch (getDenseIndex(self.layer_types, layer_index)) { + .full_attention => |dense_index| .{ + .parent = self, + .cache = .{ .self_attn = self.self_attn.atLayer(dense_index.layer_dense_index) }, + }, + .linear_attention => |dense_index| .{ + .parent = self, + .cache = .{ .linear_attn = self.gated_delta_net.atLayer(dense_index.layer_dense_index) }, + }, + }; + } + + pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { + return .{ + .layer_types = self.layer_types, + .self_attn = self.self_attn.reuseBuffer(other.self_attn), + .gated_delta_net = self.gated_delta_net.reuseBuffer(other.gated_delta_net), + }; + } + + fn countLayers(layer_types: []const LayerType, layer_type: LayerType) i64 { + var count: i64 = 0; + for (layer_types) |registered_layer_type| { + if (registered_layer_type == layer_type) count += 1; + } + return count; + } + + fn getDenseIndex(layer_types: []const LayerType, layer_index: usize) union(enum) { + full_attention: struct { layer_dense_index: usize }, + linear_attention: struct { layer_dense_index: usize }, + } { + var self_attn_layer_index: usize = 0; + var linear_attn_layer_index: usize = 0; + for (layer_types[0..layer_index]) |layer_type| { + switch (layer_type) { + .full_attention => self_attn_layer_index += 1, + .linear_attention => linear_attn_layer_index += 1, + } + } + return switch (layer_types[layer_index]) { + .full_attention => .{ .full_attention = .{ .layer_dense_index = self_attn_layer_index } }, + .linear_attention => .{ .linear_attention = .{ .layer_dense_index = linear_attn_layer_index } }, + }; + } +}; + +//========================Utils======================== + +fn softplus(x: zml.Tensor) zml.Tensor { + return x.exp().addConstant(1).log(); +} diff --git a/examples/llm/models/qwen3_5/repository.zig b/examples/llm/models/qwen3_5/repository.zig new file mode 100644 index 00000000..fe745a16 --- /dev/null +++ b/examples/llm/models/qwen3_5/repository.zig @@ -0,0 +1,83 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const common = @import("../common.zig"); +const parseConfig = common.parseConfig; +const Shardings = common.Shardings; +const model = @import("model.zig"); + +pub const Repository = struct { + inner: model.Model, + parsed_config: std.json.Parsed(model.Config), + + pub fn init(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir, store: zml.io.TensorStore.View) !Repository { + const parsed_config = try parseConfig(model.Config, allocator, io, repo); + errdefer parsed_config.deinit(); + + const options: model.Model.GenOptions = .{ + .sampling_strategy = .{}, + .max_seq_len = parsed_config.value.text_config.max_position_embeddings, + }; + + return .{ + .inner = try .init(allocator, store, parsed_config.value, options), + .parsed_config = parsed_config, + }; + } + + pub fn deinit(self: *Repository, allocator: std.mem.Allocator) void { + self.inner.deinit(allocator); + self.parsed_config.deinit(); + } + + pub fn loadBuffers(self: *Repository, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, shardings: Shardings) !model.Buffers { + const all_shardings = [_]zml.sharding.Sharding{shardings.replicated}; + return try self.inner.load(allocator, io, platform, store, &all_shardings, progress); + } + + pub fn unloadBuffers(self: *model.Buffers, allocator: std.mem.Allocator) void { + model.Model.unloadBuffers(self, allocator); + } + + pub fn tokenizePrompt(self: *const Repository, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]u32 { + return tokenizeChatPrompt(allocator, tokenizer, prompt, self.inner.special_tokens, true); + } + + pub fn tokenizeTurn(self: *const Repository, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]u32 { + return tokenizeChatPrompt(allocator, tokenizer, prompt, self.inner.special_tokens, false); + } +}; + +fn tokenizeChatPrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8, special_tokens: model.Model.SpecialTokens, is_first_turn: bool) ![]u32 { + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const im_start = tokenizer.tokenToId("<|im_start|>") orelse special_tokens.im_start_token_id; + const im_end = tokenizer.tokenToId("<|im_end|>") orelse special_tokens.im_end_token_id; + const think = tokenizer.tokenToId("") orelse return error.NoSuchToken; + const newline = try encodeSingleToken(&encoder, "\n"); + const user_prefix = try encoder.encode("user\n"); + const assistant_prefix = try encoder.encode("assistant\n"); + const encoded_prompt = try encoder.encode(prompt); + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, encoded_prompt.len + user_prefix.len + assistant_prefix.len + 8); + if (!is_first_turn) { + try tokens.appendSlice(allocator, &.{ im_end, newline }); + } + + try tokens.append(allocator, im_start); + try tokens.appendSlice(allocator, user_prefix); + try tokens.appendSlice(allocator, encoded_prompt); + try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); + try tokens.appendSlice(allocator, assistant_prefix); + try tokens.appendSlice(allocator, &.{ think, newline }); + + return tokens.toOwnedSlice(allocator); +} + +fn encodeSingleToken(encoder: *zml.tokenizer.Tokenizer.Encoder, text: []const u8) !u32 { + const encoded = try encoder.encode(text); + if (encoded.len != 1) return error.InvalidTokenizerEncoding; + return encoded[0]; +} diff --git a/examples/llm/models/qwen3_5/session.zig b/examples/llm/models/qwen3_5/session.zig new file mode 100644 index 00000000..ca32c4a2 --- /dev/null +++ b/examples/llm/models/qwen3_5/session.zig @@ -0,0 +1,180 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const common = @import("../common.zig"); +const Shardings = common.Shardings; +const SessionOptions = common.SessionOptions; +const inference = @import("inference.zig"); +const model = @import("model.zig"); +const repository = @import("repository.zig"); + +pub const Session = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + replicated_sharding: zml.sharding.Sharding, + model_buffers: *model.Buffers, + compilation_options: inference.CompilationOptions, + exe: inference.Inference, + kv_cache_buffers: zml.Bufferized(model.KvCache), + rng_buffers: zml.Bufferized(zml.Tensor.Rng), + tokenizer: zml.tokenizer.Tokenizer, + step_token_slice: zml.Slice, + generated_token_slice: zml.Slice, + seqlen: u32, + eos_token_id: u32, + think_start: ?u32, + think_end: ?u32, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + model_buffers: *model.Buffers, + tokenizer: zml.tokenizer.Tokenizer, + repo: repository.Repository, + opts: SessionOptions, + progress: *std.Progress.Node, + shardings: Shardings, + ) !Session { + _ = opts.backend; + if (opts.single) return error.SingleKernelNotSupported; + + //======================= Model & cache init ======================== + + const params = inference.CompilationOptions.init(repo.inner, repo.parsed_config.value, opts.seqlen); + const replicated_sharding = shardings.replicated; + var exe = try inference.Inference.init(allocator, io, platform, repo.inner, params, shardings, progress); + errdefer exe.deinit(); + + var kv_cache_buffers = try params.kv_cache.initBuffer(io, platform); + errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); + + const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); + var rng_buffers = try zml.Tensor.Rng.initBuffer(platform, seed, io, replicated_sharding); + errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); + + //======================= Generation setup ======================== + + return .{ + .allocator = allocator, + .io = io, + .platform = platform, + .replicated_sharding = replicated_sharding, + .model_buffers = model_buffers, + .compilation_options = params, + .exe = exe, + .kv_cache_buffers = kv_cache_buffers, + .rng_buffers = rng_buffers, + .tokenizer = tokenizer, + .step_token_slice = try .alloc(allocator, zml.Shape.init(.{ .b = 1, .s = 1 }, .u32)), + .generated_token_slice = try .alloc(allocator, zml.Shape.init(.{ .b = 1, .s = 1 }, .u32)), + .seqlen = opts.seqlen, + .eos_token_id = repo.inner.special_tokens.end_of_text_token_id, + .think_start = tokenizer.tokenToId(""), + .think_end = tokenizer.tokenToId(""), + }; + } + + pub fn deinit(self: *Session) void { + self.exe.deinit(); + model.KvCache.deinitBuffer(&self.kv_cache_buffers); + zml.Tensor.Rng.deinitBuffer(&self.rng_buffers); + self.step_token_slice.free(self.allocator); + self.generated_token_slice.free(self.allocator); + } + + pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { + //======================= Running model for prompt ======================== + + const prefill_tokens_shape = zml.Shape.init(.{ .b = 1, .s = self.seqlen }, .u32); + const prefill_tokens_slice = try zml.Slice.alloc(self.allocator, prefill_tokens_shape); + defer prefill_tokens_slice.free(self.allocator); + @memset(prefill_tokens_slice.items(u32), 0); + @memcpy(prefill_tokens_slice.items(u32)[0..all_tokens.len], all_tokens); + + var prefill_tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, prefill_tokens_slice, self.replicated_sharding); + defer prefill_tokens_buffer.deinit(); + + var prefill_token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 0), .u32, self.replicated_sharding); + defer prefill_token_index_buffer.deinit(); + + var args = try self.exe.prefill_exe.args(self.allocator); + defer args.deinit(self.allocator); + + var results = try self.exe.prefill_exe.results(self.allocator); + defer results.deinit(self.allocator); + + args.set(.{ + self.model_buffers, + prefill_tokens_buffer, + prefill_token_index_buffer, + &self.kv_cache_buffers, + &self.rng_buffers, + }); + self.exe.prefill_exe.call(args, &results); + + results.fill(.{ &prefill_tokens_buffer, &self.kv_cache_buffers, &self.rng_buffers }); + try prefill_tokens_buffer.toSlice(self.io, prefill_tokens_slice); + + const generated_token = prefill_tokens_slice.items(u32)[all_tokens.len - 1]; + self.generated_token_slice.items(u32)[0] = generated_token; + } + + pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { + //======================= Generation setup ======================== + + var decoder = try self.tokenizer.decoder(); + defer decoder.deinit(); + + generation: while (true) { + const token_id = self.generated_token_slice.items(u32)[0]; + if (token_id == self.eos_token_id) break :generation; + + if (try decoder.next(token_id)) |token| { + if (self.think_start) |think_start| if (token_id == think_start) { + try stdout.writeAll("\x1b[2m"); + }; + try stdout.writeAll(token); + if (self.think_end) |think_end| if (token_id == think_end) { + try stdout.writeAll("\x1b[0m"); + }; + try stdout.flush(); + } + + try all_tokens.append(self.allocator, token_id); + if (all_tokens.items.len >= self.seqlen) break :generation; + + try self.runDecodeStep(token_id, @intCast(all_tokens.items.len)); + } + } + + fn runDecodeStep(self: *Session, token_id: u32, token_index: u32) !void { + self.step_token_slice.items(u32)[0] = token_id; + + var token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.step_token_slice, self.replicated_sharding); + defer token_buffer.deinit(); + + var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, token_index, .u32, self.replicated_sharding); + defer token_index_buffer.deinit(); + + var args = try self.exe.decode_exe.args(self.allocator); + defer args.deinit(self.allocator); + + var results = try self.exe.decode_exe.results(self.allocator); + defer results.deinit(self.allocator); + + args.set(.{ + self.model_buffers, + token_buffer, + token_index_buffer, + &self.kv_cache_buffers, + &self.rng_buffers, + }); + self.exe.decode_exe.call(args, &results); + + results.fill(.{ &token_buffer, &self.kv_cache_buffers, &self.rng_buffers }); + try token_buffer.toSlice(self.io, self.generated_token_slice); + } +}; diff --git a/zml/nn.zig b/zml/nn.zig index a5673402..7d01bc97 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -320,7 +320,7 @@ pub fn mergeRealImg(x_real: Tensor, x_imag: Tensor, layout: RopeOpts.Layout) Ten } /// {exp( - n * ln(10_000) / N ) | n in [0..N] } -fn invFreq(N: i64, opts: RopeOpts) Tensor { +pub fn invFreq(N: i64, opts: RopeOpts) Tensor { const allocator = zml.module.CompilationContext.current().allocator; const N_half: usize = @intCast(@divExact(N, 2)); const inv_freq = allocator.alloc(f32, N_half) catch @panic("OOM"); From 2eb8423e8aa31ad5117413b4f75e95124a14e88b Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 24 Mar 2026 17:00:10 +0100 Subject: [PATCH 383/881] Update doc for V2 (#428) --- MODULE.bazel | 2 +- README.md | 2 +- docs/README.md | 1 - docs/howtos/add_weights.md | 69 ---- docs/howtos/deploy_on_server.md | 48 +-- docs/howtos/dockerize_models.md | 59 ++-- docs/howtos/howto_torch2zml.md | 156 +++++---- docs/howtos/huggingface_access_token.md | 7 +- docs/learn/concepts.md | 56 ++-- docs/misc/style_guide.md | 2 +- docs/tutorials/getting_started.md | 32 +- docs/tutorials/write_first_model.md | 419 +++++++++--------------- examples/io/main.zig | 3 +- examples/llm/BUILD.bazel | 58 ++++ examples/llm/models/lfm2/model.zig | 3 +- examples/llm/models/llama/model.zig | 3 +- examples/llm/models/qwen3_5/model.zig | 3 +- examples/mnist/BUILD.bazel | 74 ++++- examples/mnist/mnist.zig | 3 +- zml/io.zig | 11 +- zml/testing.zig | 12 +- 21 files changed, 495 insertions(+), 528 deletions(-) delete mode 100644 docs/howtos/add_weights.md diff --git a/MODULE.bazel b/MODULE.bazel index fa3f5f7f..7bfc9b39 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -229,6 +229,6 @@ use_repo(oci, "distroless_cc_debian12", "distroless_cc_debian12_linux_amd64") git_override( module_name = "rules_zig", - commit = "3f4ab088be5f21a7409a03f7c19bf25a82cc7bde", + commit = "7b205b78fb21e11efe5adcc9ec1c37fa5852dc77", remote = "https://github.com/zml/rules_zig.git", ) diff --git a/README.md b/README.md index b511fc1d..4a2584d0 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ approval from Meta on Huggingface**, which can take a few hours to get granted. Once you've been granted access, you're ready to download a gated model like `Llama-3.1-8B-Instruct`! -First, you need to download the model using the [huggingface-cli](https://huggingface.co/docs/huggingface_hub/en/guides/cli). +First, you need to download the model using the [hf](https://huggingface.co/docs/huggingface_hub/en/guides/cli) CLI. Note you don't need to install it yourself, you can just use the packaged version `bazel run //tools/hf --`. diff --git a/docs/README.md b/docs/README.md index eedfdd9b..3bcfb3f3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -17,7 +17,6 @@ Ready to write some code? Try starting with [your first model in ZML](./tutorial ## How to ? - [HuggingFace Authentication](./howtos/huggingface_access_token.md) - [Port Pytorch models to ZML](./howtos/howto_torch2zml.md) -- [Add Weights Files to your projects](./howtos/add_weights.md) - [Cross Compile and Deploy Models on a Server](./howtos/deploy_on_server.md) - [Dockerize Models](./howtos/dockerize_models.md) diff --git a/docs/howtos/add_weights.md b/docs/howtos/add_weights.md deleted file mode 100644 index 2caf2fc9..00000000 --- a/docs/howtos/add_weights.md +++ /dev/null @@ -1,69 +0,0 @@ - -# Adding Weights Files - -Our [first model](../tutorials/write_first_model.md) did not need any weights files. -We just created weights and biases at runtime. - -But real-world models typically need weights files, and maybe some other -supporting files. - -We recommend, for easy deployments, you upload those files. In many instances, -you will use a site like [🤗 Hugging Face](https://huggingface.co). - -We also recommend to add a `weights.bzl` file to your project root directory, so -you don't "pollute" your build file with long URLs and SHAs: - -```python -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -def _weights_impl(mctx): - http_archive( - name = "mnist", - sha256 = "075905e433ea0cce13c3fc08832448ab86225d089b5d412be67f59c29388fb19", - url = "https://mirror.zml.ai/data/mnist.tar.zst", - build_file_content = """exports_files(glob(["**"]), visibility = ["//visibility:public"])""", - ) - - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -weights = module_extension( - implementation = _weights_impl, -) -``` - -The above `weights.bzl` shows how we load files for MNIST: - -- `mnist.pt` (model weights) -- `mnist.ylc` (dataset for picking sample images) - -Then, in your `BUILD.bazel`, you can refer to the files you defined above, in -the following way: - -```python -zig_binary( - name = "mnist", - args = [ - "$(location @mnist//:mnist.pt)", - "$(location @mnist//:t10k-images.idx3-ubyte)", - ], - data = [ - "@mnist//:mnist.pt", - "@mnist//:t10k-images.idx3-ubyte", - ], - main = "mnist.zig", - deps = [ - "//async", - "//zml", - ], -) -``` - -See how: - -- we use `data = [ ... ]` to reference the files in `weights.bzl` -- we use `args = [ ... ]` to pass the files as command-line arguments to the - MNIST executable at runtime, automatically. diff --git a/docs/howtos/deploy_on_server.md b/docs/howtos/deploy_on_server.md index 05a88889..a8a6452e 100644 --- a/docs/howtos/deploy_on_server.md +++ b/docs/howtos/deploy_on_server.md @@ -24,7 +24,11 @@ So, to run the Llama model from above **on your development machine** housing an NVIDIA GPU, run the following: ``` -bazel run --config=release //examples/llama --@zml//platforms:cuda=true -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct +bazel run --config=release //examples/llm \ + --@zml//platforms:cuda=true \ + -- \ + --model=hf://meta-llama/Llama-3.2-1B-Instruct \ + --prompt="What is the capital of France?" ``` @@ -40,7 +44,7 @@ architectures: As an example, here is how you build above Llama for CUDA on Linux X86_64: ``` -bazel build --config=release //examples/llama \ +bazel build --config=release //examples/llm \ --@zml//platforms:cuda=true \ --@zml//platforms:cpu=false \ --platforms=@zml//platforms:linux_amd64 @@ -57,7 +61,8 @@ If not present already, add an "archive" target to the model's `BUILD.bazel`, like this: ```python -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") +load("@tar.bzl//tar:mtree.bzl", "mtree_spec") +load("@tar.bzl//tar:tar.bzl", "tar") # Manifest, required for building the tar archive mtree_spec( @@ -81,15 +86,15 @@ tar( ... and then build the TAR archive: ``` -bazel build --config=release //mnist:archive \ +bazel build --config=release //examples/mnist:archive \ --@zml//platforms:cuda=true \ --@zml//platforms:cpu=false \ --platforms=@zml//platforms:linux_amd64 ``` -Note the `//mnist:archive` notation. +Note the `//examples/mnist:archive` notation. -The resulting tar file will be in `bazel-bin/mnist/archive.tar.zst`. +The resulting tar file will be in `bazel-bin/examples/mnist/archive.tar.zst`. ### Run it on the server @@ -98,36 +103,35 @@ and run it: ```bash # on your machine -scp bazel-bin/mnist/archive.tar.zst destination-server: +scp bazel-bin/examples/mnist/archive.tar.zst destination-server: ssh destination-server # to enter the server # ... on the server tar xvf archive.tar.zst -./mnist \ - 'mnist.runfiles/_main~_repo_rules~com_github_ggerganov_ggml_mnist/file/mnist.pt' \ - 'mnist.runfiles/_main~_repo_rules~com_github_ggerganov_ggml_mnist_data/file/mnist.ylc' +./examples/mnist/mnist \ + $PWD/examples/mnist/mnist.runfiles/+non_module_deps+mnist/mnist.safetensors \ + $PWD/examples/mnist/mnist.runfiles/+non_module_deps+mnist/t10k-images.idx3-ubyte ``` The easiest way to figure out the commandline arguments of an example model is to consult the model's `BUILD.bazel` and check out its `args` section. It will -reference e.g. weights files that are defined either in the same `BUILD.bazel` -file or in a `weights.bzl` file. +reference e.g. weights files that are defined in the same `BUILD.bazel`. You can also consult the console output when running your model locally: ```bash -bazel run //mnist +bazel run //examples/mnist -INFO: Analyzed target //mnist:mnist (0 packages loaded, 0 targets configured). +INFO: Analyzed target //examples/mnist:mnist (0 packages loaded, 0 targets configured). INFO: Found 1 target... -Target //mnist:mnist up-to-date: - bazel-bin/mnist/mnist -INFO: Elapsed time: 0.302s, Critical Path: 0.00s -INFO: 3 processes: 3 internal. -INFO: Build completed successfully, 3 total actions -INFO: Running command line: bazel-bin/mnist/mnist ../_main~_repo_rules~com_github_ggerganov_ggml_mnist/file/mnist.pt ../_main~_repo_rules~com_github_ggerganov_ggml_mnist_data/file/mnist.ylc -# ... +Target //examples/mnist:mnist up-to-date: + bazel-bin/examples/mnist/mnist +INFO: Elapsed time: 0.114s, Critical Path: 0.00s +INFO: 1 process: 1 internal. +INFO: Build completed successfully, 1 total action +INFO: Running command line: bazel-bin/examples/mnist/mnist ../+non_module_deps+mnist/mnist.safetensors ../+non_module_deps+mnist/t10k-images.idx3-ubyte. +# .. ``` You see the command line right up there. On the server, you just need to replace -`../` with the 'runfiles' directory of your TAR. +`../` with the absolute path to the file. diff --git a/docs/howtos/dockerize_models.md b/docs/howtos/dockerize_models.md index ac35990c..7414b985 100644 --- a/docs/howtos/dockerize_models.md +++ b/docs/howtos/dockerize_models.md @@ -38,16 +38,16 @@ We need to add a few "imports" at the beginning of our `BUILD.bazel` so we can use their rules to define our 5 additional targets: ```python -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") -load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") load("@rules_zig//zig:defs.bzl", "zig_binary") +load("@tar.bzl//tar:mtree.bzl", "mtree_spec") +load("@tar.bzl//tar:tar.bzl", "tar") zig_binary( name = "simple_layer", main = "main.zig", deps = [ - "@zml//async", "@zml//zml", ], ) @@ -135,11 +135,11 @@ And that's almost it! You can already build the image: ``` bazel build --config=release //examples/simple_layer:image -INFO: Analyzed target //simple_layer:image (1 packages loaded, 8 targets configured). +INFO: Analyzed target //examples/simple_layer:image (0 packages loaded, 0 targets configured). INFO: Found 1 target... -Target //simple_layer:image up-to-date: - bazel-out/k8-dbg-ST-f832ad0148ae/bin/simple_layer/image_ -INFO: Elapsed time: 0.279s, Critical Path: 0.00s +Target //examples/simple_layer:image up-to-date: + bazel-out/linux_amd64-opt/bin/examples/simple_layer/image_ +INFO: Elapsed time: 0.285s, Critical Path: 0.00s INFO: 1 process: 1 internal. INFO: Build completed successfully, 1 total action ``` @@ -168,7 +168,7 @@ oci_load( ... then we can load the image and run it with the following commands: ``` -bazel run --config=release //simple_layer:load +bazel run --config=release //examples/simple_layer:load docker run --rm distroless/simple_layer:latest ``` @@ -193,7 +193,7 @@ This will push the `simple_layer` image with the tag `latest` (you can add more) to the docker registry: ``` -bazel run --config=release //simple_layer:push +bazel run --config=release //examples/simple_layer:push ``` When dealing with maybe a public and a private container registry - or if you @@ -201,7 +201,7 @@ just want to try it out **right now**, you can always override the repository on the command line: ``` -bazel run --config=release //simple_layer:push -- --repository my.server.com/org/image +bazel run --config=release //examples/simple_layer:push -- --repository my.server.com/org/image ``` @@ -215,8 +215,8 @@ We'll use the [MNIST example](https://github.com/zml/zml/tree/master/examples/mnist) to illustrate how to build Docker images that also contain data files. -You can `bazel run --config=release //mnist:push -- --repository -index.docker.io/my_org/zml_mnist` in the `./examples` folder if you want to try +You can `bazel run --config=release //examples/mnist:push -- --repository +index.docker.io/my_org/zml_mnist` if you want to try it out. **Note: Please add one more of the following parameters to specify all the @@ -231,7 +231,7 @@ platforms your containerized model should support.** **Example:** ``` -bazel run //mnist:push --config=release --@zml//platforms:cuda=true -- --repository index.docker.io/my_org/zml_mnist +bazel run //examples/mnist:push --config=release --@zml//platforms:cuda=true -- --repository index.docker.io/my_org/zml_mnist ``` @@ -244,26 +244,27 @@ same. Let's start with creating the manifest and archive: ```python -load("@aspect_bazel_lib//lib:expand_template.bzl", "expand_template") -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") -load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") load("@rules_zig//zig:defs.bzl", "zig_binary") +load("@tar.bzl//tar:mtree.bzl", "mtree_spec") +load("@tar.bzl//tar:tar.bzl", "tar") # The executable zig_binary( name = "mnist", args = [ - "$(location @com_github_ggerganov_ggml_mnist//file)", - "$(location @com_github_ggerganov_ggml_mnist_data//file)", + "$(location @mnist//:mnist.safetensors)", + "$(location @mnist//:t10k-images.idx3-ubyte)", ], data = [ - "@com_github_ggerganov_ggml_mnist//file", - "@com_github_ggerganov_ggml_mnist_data//file", + "@mnist//:mnist.safetensors", + "@mnist//:t10k-images.idx3-ubyte", ], main = "mnist.zig", + visibility = ["//visibility:public"], deps = [ - "@zml//async", + "@zml//bazel", "@zml//zml", ], ) @@ -296,22 +297,24 @@ simple string interpolation will not be enough. For this reason, we use the `expand_template` rule, like this: ```python +load("@bazel_lib//lib:expand_template.bzl", "expand_template") + # A convenience template for creating the "command line" for the entrypoint expand_template( name = "entrypoint", data = [ ":mnist", - "@com_github_ggerganov_ggml_mnist//file", - "@com_github_ggerganov_ggml_mnist_data//file", + "@mnist//:mnist.safetensors", + "@mnist//:t10k-images.idx3-ubyte", ], substitutions = { - ":model": "$(rlocationpath @com_github_ggerganov_ggml_mnist//file)", - ":data": "$(rlocationpath @com_github_ggerganov_ggml_mnist_data//file)", + ":model": "$(rlocationpath @mnist//:mnist.safetensors)", + ":data": "$(rlocationpath @mnist//:t10k-images.idx3-ubyte)", }, template = [ "./{}/mnist".format(package_name()), - "./{}/mnist.runfiles/:model".format(package_name()), - "./{}/mnist.runfiles/:data".format(package_name()), + "/{}/mnist.runfiles/:model".format(package_name()), + "/{}/mnist.runfiles/:data".format(package_name()), ], ) ``` @@ -372,5 +375,5 @@ And that's it! With one simple bazel command, you can push a neatly packaged MNIST model, including weights and dataset, to the docker registry: ``` -bazel run //mnist:push --@zml//platforms:cuda=true -- --repository index.docker.io/my_org/zml_mnist +bazel run //examples/mnist:push --@zml//platforms:cuda=true -- --repository index.docker.io/my_org/zml_mnist ``` diff --git a/docs/howtos/howto_torch2zml.md b/docs/howtos/howto_torch2zml.md index 5c751c53..d4c1fffd 100644 --- a/docs/howtos/howto_torch2zml.md +++ b/docs/howtos/howto_torch2zml.md @@ -67,14 +67,15 @@ package, so the simplest way to use it, is to copy it next to your python script. Then wrap the model/pipeline in a `zml_utils.ActivationCollector`. The collector wraps the given model, and returns the original results AND the activations in a dict of `torch.Tensor` when it's being called. After that, you -can save those activations to a `.pt` file. +can save those activations to a `.safetensors` file. ```python import torch import transformers +from safetensors.torch import save_file import zml_utils -model_path = "meta-llama/Meta-Llama-3-8B" +model_path = "meta-llama/Llama-3.2-1B-Instruct" pipeline = transformers.pipeline( "text-generation", @@ -97,8 +98,8 @@ if output: print(output) # Save activations to a file. -filename = model_path.split("/")[-1] + ".activations.pt" -torch.save(activations, filename) +filename = model_path.split("/")[-1] + ".activations.safetensors" +save_file(activations, filename) print(f"Saved {len(activations)} activations to {filename}") ``` @@ -118,36 +119,27 @@ model. Put the following in `my_project/torch2zml.zig`. const std = @import("std"); const log = std.log; -const async = @import("async"); const zml = @import("zml"); -pub fn main() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - try async.AsyncThread.main(gpa.allocator(), asyncMain, .{}); -} - -pub fn asyncMain() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - const args = try std.process.argsAlloc(allocator); - defer std.process.argsFree(allocator, args); +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const io = init.io; + const args = try init.minimal.args.toSlice(allocator); + defer allocator.free(args); const model_path, const activations_path = args[1..3].*; - const activations = try zml.aio.torch.open(allocator, activations_path); - defer activations.deinit(); - log.info("Found {} activations in {s}", .{ activations.buffers.count(), activations_path }); + var activations_registry = try zml.safetensors.TensorRegistry.fromPath(allocator, io, activations_path); + defer activations_registry.deinit(); + log.info("Found {} activations in {s}", .{ activations_registry.tensors.count(), activations_path }); - const model_weights = try zml.aio.detectFormatAndOpen(allocator, model_path); - defer model_weights.deinit(); - log.info("Found {} model layers in {s}", .{ model_weights.buffers.count(), activations_path }); + var model_registry = try zml.safetensors.TensorRegistry.fromPath(allocator, io, model_path); + defer model_registry.deinit(); + log.info("Found {} activations in {s}", .{ model_registry.tensors.count(), model_path }); } ``` -And add a `zig_cc_binary` target in `my_project/BUILD.bazel`: +And add a `zig_binary` target in `my_project/BUILD.bazel`: ```python load("@rules_zig//zig:defs.bzl", "zig_binary") @@ -156,7 +148,6 @@ zig_binary( name = "torch2zml", main = "torch2zml.zig", deps = [ - "@zml//async", "@zml//zml", ], ) @@ -166,19 +157,16 @@ Now check that the weights can be loaded correctly using the bazel CLI. ```bash bazel build //my_project:torch2zml -./bazel-bin/my_project/torch2zml /path/to/my/model.safetensors.index.json ./my_project/Meta-Llama-3-8B.activations.pt +./bazel-bin/my_project/torch2zml /path/to/my/model.safetensors.index.json ./my_project/Llama-3.2-1B-Instruct.activations.safetensors -info: Found 1108 activations in /Users/guw/Documents/zml/models/torch2zml/Meta-Llama-3-8B.activations.pt -debug(zml_io): Loading shard: model-00004-of-00004.safetensors -debug(zml_io): Loading shard: model-00001-of-00004.safetensors -debug(zml_io): Loading shard: model-00002-of-00004.safetensors -debug(zml_io): Loading shard: model-00003-of-00004.safetensors -info: Found 291 model layers in /Users/guw/Documents/zml/models/torch2zml/Meta-Llama-3-8B.activations.pt +info: Found 564 activations in /Users/corendos/dev/zml/python/temp/Llama-3.2-1B-Instruct.activations.safetensors +debug(zml/safetensors): Added new metadata key=format with value=pt +info: Found 146 activations in /Users/corendos/models/Llama-3.2-1B-Instruct/model.safetensors ``` -## Loading an individual layer +## Initializing an individual layer -In the above Zig code, the `model_weights` struct is a wrapper around a flat +In the above Zig code, the `model_registry` struct is a wrapper around a flat dictionary, containing an entry for each tensor in the model (similar to a "state dict"). Manipulating a dictionary is generally not very convenient, so let's convert it to a Zig struct. @@ -196,26 +184,70 @@ const Mlp = struct { The `zml.nn.Linear` is the equivalent of `torch.nn.Linear` and is defined by its `weight` and optional `bias` tensors. -To create such a struct from our `model_weights` dictionary, we can use the -`zml.aio.populateModelWithPrefix` helper: +To create such a struct from our `model_registry` dictionary, we first need to +convert it to a `TensorStore`: ```zig -pub fn asyncMain() !void { +pub fn main(init: std.process.Init) !void { ... - const mlp_shape = try zml.aio.populateModelWithPrefix(Mlp, allocator, model_weights, "model.layers.0.mlp"); - log.info("layer.0.mlp: {}", .{mlp_shape}); + var model_store: zml.io.TensorStore = .fromRegistry(allocator, &model_registry); + defer model_store.deinit(); +} +``` + +Then, we can initialize our MLP: + +```zig +pub fn main(init: std.process.Init) !void { + ... + + const mlp_view = model_store.view().withPrefix("model.layers.0.mlp"); + + const mlp: Mlp = .{ + .up_proj = .init( + mlp_view.createTensor("up_proj.weight", .{ .dout, .d }, null), + mlp_view.maybeCreateTensor("up_proj.bias", .{.dout}, null), + .d, + ), + .gate_proj = .init( + mlp_view.createTensor("gate_proj.weight", .{ .dout, .d }, null), + mlp_view.maybeCreateTensor("gate_proj.bias", .{.dout}, null), + .d, + ), + .down_proj = .init( + mlp_view.createTensor("down_proj.weight", .{ .d, .dout }, null), + mlp_view.maybeCreateTensor("down_proj.bias", .{.d}, null), + .dout, + ), + }; } ``` Build and run, using previous commands. -Typical errors are of the form _"Layer not found: ..."_. This is typically due -to the naming of layers in Zig not matching the naming in the file. +Typical errors are due to the naming of layers in Zig not matching the naming +in the file. Double-check everything and don't hesitate to print more things, e.g. in the Python script. Alternatively, Huggingface's web-interface allows to peek into `.safetensor` files. +## Loading an individual layer + +Currently, the layer is initialized but the weights are still on the disk. +ZML provides a convenience to automatically load weights from an initialized model: + +```zig +pub fn main(init: std.process.Init) !void { + ... + + var platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator); + var mlp_weights = try zml.io.load(Mlp, &mlp, allocator, io, platform, &model_store, .auto); +} +``` + + ## Testing an individual layer Finally, we are going to write the actual math code for our `MLP` layer. @@ -226,33 +258,39 @@ const Mlp = struct { gate_proj: zml.nn.Linear, down_proj: zml.nn.Linear, - pub fn forward(self: Mlp, x: Tensor) Tensor { - const proj = zml.call(self.up_proj, .forward, .{x}); - var output = zml.call(self.gate_proj, .forward, .{x}); + pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { + const proj = self.up_proj.forward(x); + var output = self.gate_proj.forward(x); output = output.silu().mul(proj); - return zml.call(self.down_proj, .forward, .{output}); + return self.down_proj.forward(output); } }; ``` -Note that we use `zml.call` instead of directly calling -`self.up_proj.forward(x)`. Calling `forward` directly results in the same -computation happening at runtime; but going through `zml.call` allows ZML to -generate an MLIR representation that is closer to the Zig code and therefore -easier to read. - -We can test the MLP layer with the `zml.testing.testLayer` utility: +Our MLP layer is now almost ready to be tested with the `zml.testing.testLayer` utility. +As you may have noticed, ZML makes heavy-use of tagged tensors. In the MLP, we expect +to do a dot product on the `.d` dimension. But the activations we load from the +activations files are not tagged. One solution could be to tag the tensors before saving +them to file, but a much simpler solution is to intercept the inputs before providing +them to the layer we want to test, using a wrapper. ```zig -pub fn asyncMain() !void { +pub const Wrapper = struct { + mlp: Mlp, + + pub fn forward(wrapper: Wrapper, input: zml.Tensor) zml.Tensor { + const tagged_input = input.withTags(.{.b, .s, .d}); + return wrapper.mlp.forward(tagged_input); + } +}; + +pub fn main(init: std.process.Init) !void { ... - var ctx = try zml.Context.init(); - defer ctx.deinit(); - const platform = ctx.autoPlatform(.{}); - const mlp_weights = try zml.aio.loadModelBuffers(Mlp, mlp_shape, model_weights, allocator, platform); + var activations_store: zml.io.TensorStore = .fromRegistry(allocator, &activations_registry); + defer activations_store.deinit(); - zml.testing.testLayer(platform, activations, "model.layers.0.mlp", mlp_shape, mlp_weights, 1e-3); + try zml.testing.testLayer(allocator, io, platform, Wrapper{ .mlp = mlp }, .forward, activations_store.view(), "model.model.layers.0.mlp", .{ .mlp = mlp_weights }, &.{}, .{}); } ``` diff --git a/docs/howtos/huggingface_access_token.md b/docs/howtos/huggingface_access_token.md index 797c6dfb..cc11ffbb 100644 --- a/docs/howtos/huggingface_access_token.md +++ b/docs/howtos/huggingface_access_token.md @@ -3,7 +3,7 @@ Some models have restrictions and may require some sort of approval or agreement process, which, by consequence, **requires token-authentication with Huggingface**. -The easiest way might be to use the `huggingface-cli login` command. +The easiest way might be to use the `hf auth login` command. Alternatively, here is how you can generate a **"read-only public repositories"** access token to log into your account on Huggingface, directly from `bazel`, in order to download models. @@ -29,7 +29,6 @@ Now you're ready to download a gated model like `Meta-Llama-3.2-1b`! ``` # requires token in $HOME/.cache/huggingface/token, as created by the -# `huggingface-cli login` command, or the `HUGGINGFACE_TOKEN` environment variable. -bazel run @zml//tools:hf -- download meta-llama/Llama-3.2-1B-Instruct --local-dir $HOME/Llama-3.2-1B-Instruct --exclude='*.pth' -bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" +# `hf auth login` command, or the `HUGGINGFACE_TOKEN` environment variable. +bazel run --config=release //examples/llm -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" ``` diff --git a/docs/learn/concepts.md b/docs/learn/concepts.md index de14e15b..0b9a6f35 100644 --- a/docs/learn/concepts.md +++ b/docs/learn/concepts.md @@ -8,8 +8,8 @@ particulary Neural Networks (NN). The lifecycle of a model is implemented in the following steps: -1. Open the model file and read the shapes of the weights, but leave the - weights on the disk. +1. Open the model file and read the shapes of the weights, but don't load the weights + yet. 2. Using the loaded shapes and optional metadata, instantiate a model struct with `Tensor`s, representing the shape and layout of each layer of the NN. @@ -20,28 +20,21 @@ The lifecycle of a model is implemented in the following steps: 4. Load the model weights from disk, onto the accelerator memory. -5. Bind the model weights to the executable. +5. Load some user inputs, and copy them to the accelerator. -6. Load some user inputs, and copy them to the accelerator. +6. Call the executable using the weights and the user inputs. -7. Call the executable on the user inputs. - -8. Fetch the returned model output from accelerator into host memory, and +7. Fetch the returned model output from accelerator into host memory, and finally present it to the user. -9. When all user inputs have been processed, free the executable resources and - the associated weights. +8. When all user inputs have been processed, free the executable resources, weights and + inputs. **Some details:** Note that the compilation and weight loading steps are both bottlenecks to your -model startup time, but they can be done in parallel. **ZML provides -asynchronous primitives** to make that easy. - -The **compilation can be cached** across runs, and if you're always using the -same model architecture with the same shapes, it's possible to by-pass it -entirely. +model startup time, but they can be done in parallel thanks to Zig's `std.Io` interface. The accelerator is typically a GPU, but can be another chip, or even the CPU itself, churning vector instructions. @@ -51,7 +44,7 @@ itself, churning vector instructions. In ZML, we leverage Zig's static type system to differentiate between a few concepts, hence we not only have a `Tensor` to work with, like other ML -frameworks, but also `Buffer`, `HostBuffer`, and `Shape`. +frameworks, but also `Buffer`, `Slice`, and `Shape`. Let's explain all that. @@ -65,25 +58,19 @@ Let's explain all that. `Shape` struct can also represent a regular number, aka a scalar: `Shape.init(.{}, .i32)` represents a 32-bit signed integer. -* `HostBuffer`: _is_ a multi-dimensional array, whose memory is allocated **on - the CPU**. - - points to the slice of memory containing the array - - typically owns the underlying memory - but has a flag to remember when it - doesn't. +* `Slice`: _is_ the combination of a `Shape` and raw bytes (bytes that are **on the CPU**). + - can own the underlying memory - but can also accomodate non-owned memory. * `Buffer`: _is_ a multi-dimension array, whose memory is allocated **on an accelerator**. - contains a handle that the ZML runtime can use to convert it into a physical address, but there is no guarantee this address is visible from the CPU. - - can be created by loading weights from disk directly to the device via - `zml.aio.loadBuffers` - - can be created by calling `HostBuffer.toDevice(accelerator)`. + - can be created from a `Slice` by calling `Buffer.fromSlice(...)`. * `Tensor`: is a mathematical object representing an intermediary result of a - computation. - - is basically a `Shape` with an attached MLIR value representing the - mathematical operation that produced this `Tensor`. + computation or an input to an executable (including weights). + - is basically a `Shape` with an attached MLIR value when in the context of a compilation. ## The model struct @@ -137,25 +124,22 @@ struct of `Model` but with a `Buffer` replacing every `Tensor`. Let's look at the model life cycle again, but this time annotated with the corresponding types. -1. Open the model file and read the shapes of the weights -> `zml.HostBuffer` - (using memory mapping, no actual copies happen yet) +1. Open the model file and read the shapes of the weights -> `zml.Shape` 2. Instantiate a model struct -> `Model` struct (with `zml.Tensor` inside) 3. Compile the model struct and its `forward` function into an executable. `foward` is a `Tensor -> Tensor` function, executable is a - `zml.FnExe(Model.forward)` + `zml.Executable` 4. Load the model weights from disk, onto accelerator memory -> `zml.Bufferized(Model)` struct (with `zml.Buffer` inside) -5. Bind the model weights to the executable `zml.ModuleExe(Model.forward)` +5. Load some user inputs (custom struct), encode them into arrays of numbers + (`zml.Slice`), and copy them to the accelerator (`zml.Buffer`). -6. Load some user inputs (custom struct), encode them into arrays of numbers - (`zml.HostBuffer`), and copy them to the accelerator (`zml.Buffer`). - -7. Call the executable on the user inputs. `module.call` accepts `zml.Buffer` +6. Call the executable on the user inputs. `module.call` accepts `zml.Buffer` arguments and returns `zml.Buffer` -8. Return the model output (`zml.Buffer`) to the host (`zml.HostBuffer`), +7. Return the model output (`zml.Buffer`) to the host (`zml.Slice`), decode it (custom struct) and finally return to the user. diff --git a/docs/misc/style_guide.md b/docs/misc/style_guide.md index 165555ce..a112377c 100644 --- a/docs/misc/style_guide.md +++ b/docs/misc/style_guide.md @@ -1,7 +1,7 @@ # ZML Style Guide -We prefer to keep it simple and adhere to the [Zig Style Guide](https://ziglang.org/documentation/0.13.0/#Style-Guide). +We prefer to keep it simple and adhere to the [Zig Style Guide](https://ziglang.org/documentation/0.15.1/#Style-Guide). We use ZLS to auto-format code. diff --git a/docs/tutorials/getting_started.md b/docs/tutorials/getting_started.md index 31f8b4db..ababeb15 100644 --- a/docs/tutorials/getting_started.md +++ b/docs/tutorials/getting_started.md @@ -21,14 +21,14 @@ through `bazelisk`, a version manager for `bazel`. **macOs:** ``` - brew install bazelisk +brew install bazelisk ``` **Linux:** ``` - curl -L -o /usr/local/bin/bazel 'https://github.com/bazelbuild/bazelisk/releases/download/v1.25.0/bazelisk-linux-amd64' - chmod +x /usr/local/bin/bazel +curl -L -o /usr/local/bin/bazel 'https://github.com/bazelbuild/bazelisk/releases/download/v1.28.1/bazelisk-linux-amd64' +chmod +x /usr/local/bin/bazel ``` @@ -49,7 +49,7 @@ compile it, and classify a randomly picked example from the test dataset. On the command line: ``` -bazel run --config=release //mnist +bazel run --config=release //examples/mnist ``` ### Llama @@ -71,11 +71,10 @@ Once you've been granted access, you're ready to download a gated model like `Meta-Llama-3.1-8B-Instruct`! ``` -# requires token in $HOME/.cache/huggingface/token, as created by the -# `huggingface-cli login` command, or the `HUGGINGFACE_TOKEN` environment variable. -bazel run @zml//tools:hf -- download meta-llama/Llama-3.1-8B-Instruct --local-dir $HOME/Llama-3.1-8B-Instruct --exclude='*.pth' -bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct -bazel run --config=release //llama -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct --prompt="What is the capital of France?" +# Authenticate on Huggingface. +bazel run //tools/hf -- auth login +# Run the model, loading directly from HF. +bazel run --config=release //examples/llm -- --model=hf://meta-llama/Llama-3.1-8B-Instruct --prompt="What is the capital of France?" ``` You can also try `Llama-3.1-70B-Instruct` if you have enough memory. @@ -86,9 +85,8 @@ Like the 8B model above, this model also requires approval. See [here](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) for access requirements. ``` -bazel run @zml//tools:hf -- download meta-llama/Llama-3.2-1B-Instruct --local-dir $HOME/Llama-3.2-1B-Instruct --exclude='*.pth' -bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct -bazel run --config=release //llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" +bazel run //tools/hf -- auth login +bazel run --config=release //examples/llm -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" ``` For a larger 3.2 model, you can also try `Llama-3.2-3B-Instruct`. @@ -114,13 +112,15 @@ following arguments to the command line when compiling or running a model: The latter, avoiding compilation for CPU, cuts down compilation time. -So, to run the OpenLLama model from above on your host sporting an NVIDIA GPU, +So, to run the Llama model from above on your host sporting an NVIDIA GPU, run the following: ``` -bazel run --config=release //examples/llama:Llama-3.2-1B-Instruct \ - --@zml//platforms:cuda=true \ - -- --prompt="What is the capital of France?" +bazel run --config=release //examples/llm \ + --@zml//platforms:cuda=true \ + -- \ + --model=hf://meta-llama/Llama-3.1-8B-Instruct \ + --prompt="What is the capital of France?" ``` diff --git a/docs/tutorials/write_first_model.md b/docs/tutorials/write_first_model.md index ebaf6c2f..b88c495e 100644 --- a/docs/tutorials/write_first_model.md +++ b/docs/tutorials/write_first_model.md @@ -42,13 +42,6 @@ touch simple_layer/main.zig touch simple_layer/BUILD.bazel ``` -By the way, you can access the complete source code of this walkthrough here: - -- [main.zig](https://github.com/zml/zml/tree/master/examples/simple_layer/main.zig) -- [BUILD.bazel](https://github.com/zml/zml/tree/master/examples/simple_layer/BUILD.bazel) - - - ## The high-level Overview Before firing up our editor, let's quickly talk about a few basic ZML @@ -57,35 +50,31 @@ fundamentals. In ZML, we describe a _Module_, which represents our AI model, as a Zig `struct`. That struct can contain Tensor fields that are used for computation, e.g. weights and biases. In the _forward_ function of a Module, we describe the -computation by calling tensor operations like _mul_, _add_, _dotGeneral_, +computation by calling tensor operations like _mul_, _add_, _dot_, _conv2D_, etc., or even nested Modules. ZML creates an MLIR representation of the computation when we compile the -Module. For compilation, only the _Shapes_ of all tensors must be known. No -actual tensor data is needed at this step. This is important for large models: -we can compile them while the actual weight data is being fetched from disk. +_Module_. For compilation, only the _Tensors_ are required. No actual tensor data +is needed at this step. This is important for large models: we can compile them +while the actual weight data is being fetched from disk. -To accomplish this, ZML uses a _BufferStore_. The _BufferStore_ knows how to -only load shapes and when to load actual tensor data. In our example, we will -fake the _BufferStore_ a bit: we won't load from disk; we'll use float arrays -instead. +To accomplish this, ZML uses a _TensorStore_. The _TensorStore_ loads everything +that is required to make _Tensors_ and later materialize them to _Buffers_. In our +example though, we won't even use a _TensorStore_ as it's optional. We will directly +create tensors using their shape and data type. -After compilation is done (and the _BufferStore_ has finished loading weights), -we can send the weights from the _BufferStore_ to our computation device. That -produces an _executable_ module which we can call with different _inputs_. - -In our example, we then copy the result from the computation device to CPU -memory and print it. +After compilation is done, we get what we call an _Executable_. From this _Executable_ +we can create _Args_ and _Results_, two structs that will store respectively the inputs +and outputs of a computation done with an _Executable_. **So the steps for us are:** - describe the computation as ZML _Module_, using tensor operations -- create a _BufferStore_ that provides _Shapes_ and data of weights and bias - (ca. 5 lines of code). -- compile the _Module_ **asynchronously** -- make the compiled _Module_ send the weights (and bias) to the computation - device utilizing the _BufferStore_, producing an _executable_ module -- prepare input tensor and call the _executable_ module. +- initialize the _Module_ +- compile the _Module_ to produce an _Executable_ +- make a "bufferized" version of the _Module_, containing the actual data on the + computation device +- prepare the _Args_ and _Results_ of a computation and call the _Executable_ - get the result back to CPU memory and print it If you like to read more about the underlying concepts of the above, please see @@ -99,13 +88,9 @@ Let's start by writing some Zig code, importing ZML and often-used modules: ```zig const std = @import("std"); const zml = @import("zml"); -const async = @import("async"); ``` -You will use above lines probably in all ZML projects. Also, note that **ZML is -async** and comes with its own async runtime, thanks to -[zigcoro](https://github.com/rsepassi/zigcoro). - +You will use above lines probably in all ZML projects. ### Defining our Model @@ -137,205 +122,132 @@ There are more things to observe: - more advanced use-cases are passing in / returning structs or tuples, like `struct { Tensor, Tensor }` as an example for a tuple of two tensors. You can see such use-cases, for example in the - [Llama Model](https://github.com/zml/zml/tree/master/examples/llama) + [LLM CLI](https://github.com/zml/zml/tree/master/examples/llm) - in the model, tensors may be optional. As is the case with `bias`. ### Adding a main() function -ZML code is async. Hence, We need to provide an async main function. It works -like this: - ```zig -pub fn main() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - try async.AsyncThread.main(gpa.allocator(), asyncMain); -} +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const io = init.io; - -pub fn asyncMain() !void { - // ... -``` - -The above `main()` function only creates an allocator and an async main thread -that executes our `asyncMain()` function by calling it with no (`.{}`) -arguments. - -So, let's start with the async main function: - -```zig -pub fn asyncMain() !void { - // Short lived allocations - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - // Arena allocator for BufferStore etc. - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - // Create ZML context - var context = try zml.Context.init(); - defer context.deinit(); - - const platform = context.autoPlatform(.{}); + var platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator); + ... } ``` -This is boilerplate code that provides a general-purpose allocator and, for -convenience, an arena allocator that we will use later. The advantage of arena -allocators is that you don't need to deallocate individual allocations; you -simply call `.deinit()` to deinitialize the entire arena instead! +The above `main()` function takes advantage of Zig's "fat main" to avoid having to write +boilerplate code for allocations and io. -We also initialize the ZML context `context` and get our CPU `platform` -automatically. +We also get our ZML CPU platform `platform` automatically. +### Initializing the Module -### The BufferStore - -Next, we need to set up the concrete weight and bias tensors for our model. -Typically, we would load them from disk. But since our example works without -stored weights, we are going to create a BufferStore manually, containing -_HostBuffers_ (buffers on the CPU) for both the `weight` and the `bias` tensor. - -A BufferStore basically contains a dictionary with string keys that match the -name of the struct fields of our `Layer` struct. So, let's create this -dictionary: +Next, we need to initialize the module so that we can compile it. ```zig -// Our weights and bias to use -var weights = [3]f16{ 2.0, 2.0, 2.0 }; -var bias = [3]f16{ 1.0, 2.0, 3.0 }; -const input_shape = zml.Shape.init(.{3}, .f16); - -// We manually produce a BufferStore. You would not normally do that. -// A BufferStore is usually created by loading model data from a file. -var buffers: zml.aio.BufferStore.Buffers = .{}; -try buffers.put(arena, "weight", zml.HostBuffer.fromArray(&weights)); -try buffers.put(arena, "bias", zml.HostBuffer.fromArray(&bias)); - -// the actual BufferStore -const bs: zml.aio.BufferStore = .{ - .arena = arena_state, - .buffers = buffers, +const layer: Layer = .{ + .weight = zml.Tensor.init(.{3}, .f16), + .bias = zml.Tensor.init(.{3}, .f16), }; ``` -Our weights are `{2.0, 2.0, 2.0}`, and our bias is just `{1.0, 2.0, 3.0}`. The -shape of the weight and bias tensors is `{3}`, and because of that, the **shape -of the input tensor** is also going to be `{3}`! - -Note that `zml.Shape` always takes the data type associated with the tensor. In -our example, that is `f16`, expressed as the enum value `.f16`. - +As you can see, it's pretty simple. You can create _Tensors_ from their shape and data type. +Alternatively, you are free to add an arbitrary `.init()` function to your _Module_ if the +initialization code is complex. ### Compiling our Module for the accelerator We're only going to use the CPU for our simple model, but we need to compile the -`forward()` function nonetheless. This compilation is usually done -asynchronously. That means, we can continue doing other things while the module -is compiling: +`forward()` function nonetheless: ```zig -// A clone of our model, consisting of shapes. We only need shapes for compiling. -// We use the BufferStore to infer the shapes. -const model_shapes = try zml.aio.populateModel(Layer, allocator, bs); +// Our computation require an input tensor +const input: zml.Tensor = .init(.{3}, .f16); -// Start compiling. This uses the inferred shapes from the BufferStore. -// The shape of the input tensor, we have to pass in manually. -var compilation = try async.async( - zml.compileModel, - .{ allocator, Layer.forward, model_shapes, .{input_shape}, platform }, -); +const sharding = try zml.sharding.replicatedSharding(platform); -// Produce a bufferized weights struct from the fake BufferStore. -// This is like the inferred shapes, but with actual values. -// We will need to send those to the computation device later. -var model_weights = try zml.aio.loadBuffers(Layer, .{}, bs, arena, platform); -defer zml.aio.unloadBuffers(&model_weights); // for good practice - -// Wait for compilation to finish -const compiled = try compilation.await(); +var executable = try platform.compile(allocator, io, layer, .forward, .{input}, .{ .shardings = &.{sharding}}); +defer executable.deinit(); ``` -Compiling is happening in the background via the `async` function. We call -`async` with the `zml.compileModel` function and its arguments -separately. The arguments themselves are basically the shapes of the weights in -the BufferStore, the `.forward` function name in order to compile -`Layer.forward`, the shape of the input tensor(s), and the platform for which to -compile (we used auto platform). +You might wonder what this `sharding` variable is for ? ZML supports sharding tensors across multiple devices, +and the `compile()` function needs to know how the tensors are sharded in order to compile the module correctly. +For this simple example, we just replicate the tensors across all devices for simplicity. +### Creating the "bufferized" Model -### Creating the Executable Model - -Now that we have compiled the module utilizing the shapes, we turn it into an -executable. +Now that we have compiled the module, we can take care of loading the actual tensor data. +In our case, we'll create the _Buffers_ from the raw data directly (but keep in mind that ZML +provides ways to have extra-low loading times). ```zig -// pass the model weights to the compiled module to create an executable module -// all required memory has been allocated in `compile`. -var executable = compiled.prepare(model_weights); -defer executable.deinit(); + +const weight_slice: zml.Slice = .init(layer.weight.shape(), std.mem.sliceAsBytes(&[3]f16{1.0, 2.0, 3.0})); +const bias_slice: zml.Slice = .init(layer.bias.?.shape(), std.mem.sliceAsBytes(&[3]f16{1.0, 1.0, 1.0})); +var layer_buffers: zml.Bufferized(Layer) = .{ + .weight = try zml.Buffer.fromSlice(io, platform, weight_slice, sharding), + .bias = try zml.Buffer.fromSlice(io, platform, bias_slice, sharding), +}; +defer layer_buffers.weight.deinit(); +defer layer_buffers.bias.?.deinit(); ``` ### Calling / running the Model -The executable can now be invoked with an input of our choice. +Before being able to run our _Executable_ there is still two steps: +- create the input buffer +- create and fill the _Args_ and _Results_ structs -To create the `input`, we directly use `zml.Buffer` by calling -`zml.Buffer.fromArray()`. It's important to note that `Buffer`s reside in -_accelerator_ (or _device_) memory, which is precisely where the input needs to -be for the executable to process it on the device. - -For clarity, let's recap the distinction: `HostBuffer`s are located in standard -_host_ memory, which is accessible by the CPU. When we initialized the weights, -we used `HostBuffers` to set up the `BufferStore`. This is because the -`BufferStore` typically loads weights from disk into `HostBuffer`s, and then -converts them into `Buffer`s when we call `loadBuffers()`. - -However, for inputs, we bypass the `BufferStore` and create `Buffer`s directly -in device memory. +To create the `input` we already know how to do it, as we did it for the _Module_ +buffers. +To create the _Args_ and _Results_, the _Executable_ exposes two conveniences. ```zig -// prepare an input buffer -// Here, we use zml.HostBuffer.fromSlice to show how you would create a -// HostBuffer with a specific shape from an array. -// For situations where e.g. you have an [4]f16 array but need a .{2, 2} input -// shape. -var input = [3]f16{ 5.0, 5.0, 5.0 }; -var input_buffer = try zml.Buffer.from( - platform, - zml.HostBuffer.fromSlice(input_shape, &input), -); +// create the input buffer +const input_slice: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[3]f16{ 5.0, 5.0, 5.0 })); +var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice, sharding); defer input_buffer.deinit(); -// call our executable module -var result: zml.Buffer = executable.call(.{input_buffer}); +// create the Args and Results structs +var args = try executable.args(allocator); +defer args.deinit(allocator); + +var results = try executable.results(allocator); +defer results.deinit(allocator); + +// fill the Args +args.set(.{ layer_buffers, input_buffer }); + +// call our executable +executable.call(args, &results); + +// Retrieve the resulting buffer +var result = results.get(zml.Buffer); defer result.deinit(); // fetch the result buffer to CPU memory -const cpu_result = try result.toHostAlloc(arena); +const result_slice = try result.toSliceAlloc(allocator, io); +defer result_slice.free(allocator); + std.debug.print( "\n\nThe result of {d} * {d} + {d} = {d}\n", - .{ &weights, &input, &bias, cpu_result.items(f16) }, + .{ weight_slice, input_slice, bias_slice, result_slice }, ); ``` Note that the result of a computation is usually residing in the memory of the -computation device, so with `.toHostAlloc()` we bring it back to CPU memory in -the form of a `HostBuffer`. After that, we can print it. - -In order to print it, we need to tell the host buffer how to interpret the -memory. We do that by calling `.items(f16)`, making it cast the memory to `f16` -items. +computation device, so with `.toSliceAlloc()` we bring it back to CPU memory in +the form of a `Slice`. After that, we can print it. And that's it! Now, let's have a look at building and actually running this example! @@ -355,20 +267,18 @@ zig_binary( name = "simple_layer", main = "main.zig", deps = [ - "@zml//async", "@zml//zml", ], ) ``` -To produce an executable, we import `zig_cc_binary` from the zig rules, and -pass it a name and the zig file we just wrote. The dependencies in `deps` are -what's needed for a basic ZML executable and correlate with our imports at the +To produce an executable, we import `zig_binary` from the zig rules, and +pass it a name and the zig file we just wrote. The dependency in `deps` is +what's needed for a basic ZML executable and correlate with our import at the top of the Zig file: ```zig const zml = @import("zml"); -const async = @import("async"); ``` @@ -381,41 +291,44 @@ With everything in place now, running the model is easy: bazel run --config=release //examples/simple_layer # compile and run debug version -bazel run //simple_layer +bazel run //examples/simple_layer ``` And voila! Here's the output: ``` -bazel run --config=release //simple_layer -INFO: Analyzed target //simple_layer:simple_layer (0 packages loaded, 0 targets configured). +bazel run --config=release //examples/simple_layer +INFO: Analyzed target //examples/simple_layer:simple_layer (0 packages loaded, 0 targets configured). INFO: Found 1 target... -Target //simple_layer:simple_layer up-to-date: - bazel-bin/simple_layer/simple_layer +Target //examples/simple_layer:simple_layer up-to-date: + bazel-bin/examples/simple_layer/simple_layer INFO: Elapsed time: 0.120s, Critical Path: 0.00s INFO: 1 process: 1 internal. INFO: Build completed successfully, 1 total action -INFO: Running command line: bazel-bin/simple_layer/simple_layer +INFO: Running command line: bazel-bin/examples/simple_layer/simple_layer info(pjrt): Loaded library: libpjrt_cpu.dylib -info(zml_module): Compiling main.Layer.forward with { Shape({3}, dtype=.f16) } +debug(zml/module): +******** ZML generated MLIR ******** +module @zml attributes {mhlo.num_partitions = 4 : i32, mhlo.num_replicas = 1 : i32} { + sdy.mesh @replicated = <["bus"=4]> + func.func public @main(%arg0: tensor<3xf16> {sdy.sharding = #sdy.sharding<@replicated, [{}], replicated={"bus"}>}, %arg1: tensor<3xf16> {sdy.sharding = #sdy.sharding<@replicated, [{}], replicated={"bus"}>}, %arg2: tensor<3xf16> {sdy.sharding = #sdy.sharding<@replicated, [{}], replicated={"bus"}>}) -> (tensor<3xf16> {sdy.sharding = #sdy.sharding<@replicated, [{}], replicated={"bus"}>}) { + %0 = stablehlo.multiply %arg1, %arg2 : tensor<3xf16> + %1 = stablehlo.add %0, %arg0 : tensor<3xf16> + return %1 : tensor<3xf16> + } +} -The result of { 2, 2, 2 } * { 5, 5, 5 } + { 1, 2, 3 } = { 11, 12, 13 } + + +The result of {1,2,3} * {5,5,5} + {1,1,1} = {6,11,16} ``` ---- - -You can access the complete source code of this walkthrough here: - -- [main.zig](https://github.com/zml/zml/tree/master/examples/simple_layer/main.zig) -- [BUILD.bazel](https://github.com/zml/zml/tree/master/examples/simple_layer/BUILD.bazel) - ## The complete example ```zig const std = @import("std"); const zml = @import("zml"); -const async = @import("async"); /// Model definition const Layer = struct { @@ -431,96 +344,70 @@ const Layer = struct { } }; -pub fn main() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - try async.AsyncThread.main(gpa.allocator(), asyncMain); -} +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const io = init.io; -pub fn asyncMain() !void { - // Short lived allocations - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); + var platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator); - // Arena allocator for BufferStore etc. - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - // Create ZML context - var context = try zml.Context.init(); - defer context.deinit(); - - const platform = context.autoPlatform(.{}); - - // Our weights and bias to use - var weights = [3]f16{ 2.0, 2.0, 2.0 }; - var bias = [3]f16{ 1.0, 2.0, 3.0 }; - const input_shape = zml.Shape.init(.{3}, .f16); - - // We manually produce a BufferStore. You would not normally do that. - // A BufferStore is usually created by loading model data from a file. - var buffers: zml.aio.BufferStore.Buffers = .{}; - try buffers.put(arena, "weight", zml.HostBuffer.fromArray(&weights)); - try buffers.put(arena, "bias", zml.HostBuffer.fromArray(&bias)); - - // the actual BufferStore - const bs: zml.aio.BufferStore = .{ - .arena = arena_state, - .buffers = buffers, + const layer: Layer = .{ + .weight = zml.Tensor.init(.{3}, .f16), + .bias = zml.Tensor.init(.{3}, .f16), }; - // A clone of our model, consisting of shapes. We only need shapes for - // compiling. We use the BufferStore to infer the shapes. - const model_shapes = try zml.aio.populateModel(Layer, allocator, bs); + // Our computation require an input tensor + const input: zml.Tensor = .init(.{3}, .f16); - // Start compiling. This uses the inferred shapes from the BufferStore. - // The shape of the input tensor, we have to pass in manually. - var compilation = try async.async(zml.compileModel, .{ allocator, Layer.forward, model_shapes, .{input_shape}, platform }); + const sharding = try zml.sharding.replicatedSharding(platform); - // Produce a bufferized weights struct from the fake BufferStore. - // This is like the inferred shapes, but with actual values. - // We will need to send those to the computation device later. - var model_weights = try zml.aio.loadBuffers(Layer, .{}, bs, arena, platform); - defer zml.aio.unloadBuffers(&model_weights); // for good practice - - // Wait for compilation to finish - const compiled = try compilation.await(); - - // pass the model weights to the compiled module to create an executable - // module - var executable = compiled.prepare(model_weights); + var executable = try platform.compile(allocator, io, layer, .forward, .{input}, .{ .shardings = &.{sharding} }); defer executable.deinit(); - // prepare an input buffer - // Here, we use zml.HostBuffer.fromSlice to show how you would create a - // HostBuffer with a specific shape from an array. - // For situations where e.g. you have an [4]f16 array but need a .{2, 2} - // input shape. - var input = [3]f16{ 5.0, 5.0, 5.0 }; - var input_buffer = try zml.Buffer.from( - platform, - zml.HostBuffer.fromSlice(input_shape, &input), - ); + const weight_slice: zml.Slice = .init(layer.weight.shape(), std.mem.sliceAsBytes(&[3]f16{ 1.0, 2.0, 3.0 })); + const bias_slice: zml.Slice = .init(layer.bias.?.shape(), std.mem.sliceAsBytes(&[3]f16{ 1.0, 1.0, 1.0 })); + var layer_buffers: zml.Bufferized(Layer) = .{ + .weight = try zml.Buffer.fromSlice(io, platform, weight_slice, sharding), + .bias = try zml.Buffer.fromSlice(io, platform, bias_slice, sharding), + }; + defer layer_buffers.weight.deinit(); + defer layer_buffers.bias.?.deinit(); + + // create the input buffer + const input_slice: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[3]f16{ 5.0, 5.0, 5.0 })); + var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice, sharding); defer input_buffer.deinit(); - // call our executable module - var result: zml.Buffer = executable.call(.{input_buffer}); + // create the Args and Results structs + var args = try executable.args(allocator); + defer args.deinit(allocator); + + var results = try executable.results(allocator); + defer results.deinit(allocator); + + // fill the Args + args.set(.{ layer_buffers, input_buffer }); + + // call our executable + executable.call(args, &results); + + // Retrieve the resulting buffer + var result = results.get(zml.Buffer); defer result.deinit(); - // fetch the result to CPU memory - const cpu_result = try result.toHostAlloc(arena); + // fetch the result buffer to CPU memory + const result_slice = try result.toSliceAlloc(allocator, io); + defer result_slice.free(allocator); + std.debug.print( "\n\nThe result of {d} * {d} + {d} = {d}\n", - .{ &weights, &input, &bias, cpu_result.items(f16) }, + .{ weight_slice, input_slice, bias_slice, result_slice }, ); } ``` ## Where to go from here -- [Add some weights files to your model](../howtos/add_weights.md) - [Run the model on GPU](../tutorials/getting_started.md) - [Deploy the model on a server](../howtos/deploy_on_server.md) - [Dockerize this model](../howtos/dockerize_models.md) diff --git a/examples/io/main.zig b/examples/io/main.zig index 8b4d65bc..2d14db6b 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -206,8 +206,7 @@ pub fn main(init: std.process.Init) !void { log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); } - _ = try zml.io.load(AllTensorsModel, &model, init.arena.allocator(), io, platform, .{ - .store = &store, + _ = try zml.io.load(AllTensorsModel, &model, init.arena.allocator(), io, platform, &store, .{ .shardings = &.{ replicated_sharding, sharded_sharding }, .parallelism = 16, .dma_chunks = 16, diff --git a/examples/llm/BUILD.bazel b/examples/llm/BUILD.bazel index 7d1fa565..34283122 100644 --- a/examples/llm/BUILD.bazel +++ b/examples/llm/BUILD.bazel @@ -1,5 +1,9 @@ +load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") load("@bazel_skylib//rules:build_test.bzl", "build_test") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") load("@rules_zig//zig:defs.bzl", "zig_binary") +load("@tar.bzl//tar:mtree.bzl", "mtree_spec") +load("@tar.bzl//tar:tar.bzl", "tar") zig_binary( name = "llm", @@ -29,3 +33,57 @@ build_test( name = "test", targets = [":llm"], ) + +mtree_spec( + name = "mtree", + srcs = [":llm"], + tags = ["manual"] +) + +tar( + name = "archive", + srcs = [":llm"], + args = [ + "--options", + ",".join([ + "zstd:compression-level=9", + "zstd:threads=16", + ]), + ], + compress = "zstd", + mtree = ":mtree", + tags = ["manual"] +) + +oci_image( + name = "image_", + base = "@distroless_cc_debian12", + entrypoint = ["./{}/llm".format(package_name())], + target_compatible_with = [ + "@platforms//os:linux", + ], + tars = [":archive"], + tags = ["manual"] +) + +platform_transition_filegroup( + name = "image", + srcs = [":image_"], + tags = ["manual"], + target_platform = "//platforms:linux_amd64", +) + +oci_load( + name = "load", + image = ":image", + repo_tags = ["zmlai/llm:latest"], + tags = ["manual"], +) + +oci_push( + name = "push", + image = ":image", + remote_tags = ["latest"], + repository = "index.docker.io/zmlai/llm", + tags = ["manual"], +) diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index 4f2b61b7..9e5bdbea 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -104,11 +104,10 @@ pub const Model = struct { }); } const all_shardings = shardings.all(); - return zml.io.load(Model, self, allocator, io, platform, .{ + return zml.io.load(Model, self, allocator, io, platform, store, .{ .dma_chunks = 8, .dma_chunk_size = 128 * zml.MiB, .progress = progress, - .store = store, .parallelism = 16, .total_bytes = &total_bytes, .shardings = &all_shardings, diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index a84f5703..fee2ceaf 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -89,11 +89,10 @@ pub const Model = struct { std.log.scoped(.llama).info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); } - return zml.io.load(Model, self, allocator, io, platform, .{ + return zml.io.load(Model, self, allocator, io, platform, store, .{ .dma_chunks = 32, .dma_chunk_size = 128 * zml.MiB, .progress = progress, - .store = store, .shardings = shardings, .parallelism = 16, .total_bytes = &total_bytes, diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 76b4f9a4..86ccd690 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -97,11 +97,10 @@ pub const Model = struct { const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); } - return zml.io.load(Model, self, allocator, io, platform, .{ + return zml.io.load(Model, self, allocator, io, platform, store, .{ .dma_chunks = 32, .dma_chunk_size = 128 * zml.MiB, .progress = progress, - .store = store, .shardings = shardings, .parallelism = 16, .total_bytes = &total_bytes, diff --git a/examples/mnist/BUILD.bazel b/examples/mnist/BUILD.bazel index ad10f9f1..89fdbaab 100644 --- a/examples/mnist/BUILD.bazel +++ b/examples/mnist/BUILD.bazel @@ -1,5 +1,9 @@ -load("@bazel_skylib//rules:build_test.bzl", "build_test") +load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@bazel_lib//lib:expand_template.bzl", "expand_template") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") load("@rules_zig//zig:defs.bzl", "zig_binary") +load("@tar.bzl//tar:mtree.bzl", "mtree_spec") +load("@tar.bzl//tar:tar.bzl", "tar") zig_binary( name = "mnist", @@ -19,7 +23,69 @@ zig_binary( ], ) -build_test( - name = "test", - targets = [":mnist"], +mtree_spec( + name = "mtree", + srcs = [":mnist"], +) + +tar( + name = "archive", + srcs = [":mnist"], + args = [ + "--options", + "zstd:compression-level=9", + ], + compress = "zstd", + mtree = ":mtree", +) + +expand_template( + name = "entrypoint", + data = [ + ":mnist", + "@mnist//:mnist.safetensors", + "@mnist//:t10k-images.idx3-ubyte", + ], + substitutions = { + ":model": "$(rlocationpath @mnist//:mnist.safetensors)", + ":data": "$(rlocationpath @mnist//:t10k-images.idx3-ubyte)", + }, + template = [ + "./{}/mnist".format(package_name()), + "/{}/mnist.runfiles/:model".format(package_name()), + "/{}/mnist.runfiles/:data".format(package_name()), + ], +) + +# The actual docker image, with entrypoint, created from tar archive +oci_image( + name = "image_", + base = "@distroless_cc_debian12", + entrypoint = ":entrypoint", + tars = [":archive"], +) + +# We always want to create the image for Linux +platform_transition_filegroup( + name = "image", + srcs = [":image_"], + target_platform = "@zml//platforms:linux_amd64", +) + +# Load will immediately load the image (eg: docker load) +oci_load( + name = "load", + image = ":image", + repo_tags = [ + "distroless/mnist:latest", + ], +) + +# Bazel target for pushing the Linux image to our docker registry +oci_push( + name = "push", + image = ":image", + remote_tags = ["latest"], + # override with -- --repository foo.bar/org/image + repository = "index.docker.io/steeve/mnist", ) diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 16488b43..de77ac8a 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -44,8 +44,7 @@ const Mnist = struct { store: *const zml.io.TensorStore, shardings: []const zml.sharding.Sharding, ) !zml.Bufferized(Mnist) { - return zml.io.load(Mnist, self, allocator, io, platform, .{ - .store = store, + return zml.io.load(Mnist, self, allocator, io, platform, store, .{ .shardings = shardings, .parallelism = 1, .dma_chunks = 1, diff --git a/zml/io.zig b/zml/io.zig index c1f87de9..91d1b5d8 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -1058,8 +1058,14 @@ pub const DirectMemoryWriter = struct { }; pub const LoadOpts = struct { + pub const auto: LoadOpts = .{ + .parallelism = 1, + .shardings = &.{}, + .dma_chunks = 2, + .dma_chunk_size = 4096, + }; + parallelism: usize, - store: *const TensorStore, shardings: []const Sharding, progress: ?*std.Progress.Node = null, dma_chunks: usize, @@ -1073,6 +1079,7 @@ pub fn load( allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, + store: *const TensorStore, opts: LoadOpts, ) !Bufferized(ModelType) { var bufferized = try mem.bufferize(allocator, ModelType, model); @@ -1112,7 +1119,7 @@ pub fn load( var walk_ctx: Ctx = .{ .platform = platform, .buffers = try allocator.alloc(*Buffer, meta.count(Tensor, model)), - .store = opts.store, + .store = store, .allocator = allocator, .dma_allocators = dma_allocators, .pinned_buffer_pools = buffer_pools, diff --git a/zml/testing.zig b/zml/testing.zig index 265a19c6..508cd820 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -337,7 +337,9 @@ pub fn testLayer( } }.cb, &ctx, &args); - const exe = try platform.compile(allocator, io, layer, func, args, .{ .shardings = shardings }); + const replicated_sharding = try zml.sharding.replicatedSharding(platform); + + const exe = try platform.compile(allocator, io, layer, func, args, .{ .shardings = if (shardings.len != 0) shardings else &.{replicated_sharding} }); defer exe.deinit(); const output_name = try std.fmt.allocPrint(arena.allocator(), "{s}.out", .{name}); @@ -354,13 +356,7 @@ pub fn testLayer( var exe_results = try exe.results(allocator); defer exe_results.deinit(allocator); - var args_buffers = try zml.io.load(ArgsT, &args, allocator, io, platform, .{ - .dma_chunks = 1, - .dma_chunk_size = 4096, - .store = activation_store.store, - .parallelism = 1, - .shardings = shardings, - }); + var args_buffers = try zml.io.load(ArgsT, &args, allocator, io, platform, activation_store.store, .auto); defer zml.meta.visit(struct { fn cb(_: void, b: *zml.Buffer) void { b.deinit(); From 8de2c42fe619ff7564cbbb25de61f45909986f89 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 24 Mar 2026 18:16:25 +0100 Subject: [PATCH 384/881] examples/llm: Simplifications (#430) --- examples/llm/chat.zig | 32 +++- examples/llm/main.zig | 85 +++++----- examples/llm/models.zig | 181 ++++++++++++++------- examples/llm/models/common.zig | 9 + examples/llm/models/lfm2.zig | 7 +- examples/llm/models/lfm2/inference.zig | 51 +++--- examples/llm/models/lfm2/model.zig | 115 ++++++++++++- examples/llm/models/lfm2/repository.zig | 76 --------- examples/llm/models/lfm2/session.zig | 33 ++-- examples/llm/models/lfm2_tests.zig | 10 +- examples/llm/models/llama.zig | 5 +- examples/llm/models/llama/inference.zig | 52 +++--- examples/llm/models/llama/model.zig | 118 +++++++++++++- examples/llm/models/llama/repository.zig | 81 --------- examples/llm/models/llama/session.zig | 42 ++--- examples/llm/models/llama_tests.zig | 9 +- examples/llm/models/qwen3_5.zig | 7 +- examples/llm/models/qwen3_5/inference.zig | 46 ++++-- examples/llm/models/qwen3_5/model.zig | 123 +++++++++++++- examples/llm/models/qwen3_5/repository.zig | 83 ---------- examples/llm/models/qwen3_5/session.zig | 66 +++----- 21 files changed, 708 insertions(+), 523 deletions(-) delete mode 100644 examples/llm/models/lfm2/repository.zig delete mode 100644 examples/llm/models/llama/repository.zig delete mode 100644 examples/llm/models/qwen3_5/repository.zig diff --git a/examples/llm/chat.zig b/examples/llm/chat.zig index 6bcaa8f4..3310f39d 100644 --- a/examples/llm/chat.zig +++ b/examples/llm/chat.zig @@ -26,16 +26,33 @@ pub const Chat = struct { allocator: std.mem.Allocator, io: std.Io, tokenizer: zml.tokenizer.Tokenizer, - repo: *models.Repository, - session: *models.Session, + loaded_model: *models.LoadedModel, + session: models.Session, tokens: std.ArrayList(u32), - pub fn init(allocator: std.mem.Allocator, io: std.Io, tokenizer: zml.tokenizer.Tokenizer, repo: *models.Repository, session: *models.Session) !Chat { + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + tokenizer: zml.tokenizer.Tokenizer, + loaded_model: *models.LoadedModel, + compiled_model: *models.CompiledModel, + model_buffers: *models.Buffers, + ) !Chat { + var session = try compiled_model.newSession( + allocator, + io, + platform, + model_buffers, + tokenizer, + ); + errdefer session.deinit(); + return .{ .allocator = allocator, .io = io, .tokenizer = tokenizer, - .repo = repo, + .loaded_model = loaded_model, .session = session, .tokens = try .initCapacity(allocator, session.maxTokens()), }; @@ -43,10 +60,11 @@ pub const Chat = struct { pub fn deinit(self: *Chat) void { self.tokens.deinit(self.allocator); + self.session.deinit(); } pub fn runOnce(self: *Chat, prompt: []const u8) !void { - const prompt_tokens = try self.repo.tokenizePrompt(self.allocator, self.tokenizer, prompt); + const prompt_tokens = try self.loaded_model.tokenizePrompt(self.allocator, self.tokenizer, prompt); defer self.allocator.free(prompt_tokens); var stdout = std.Io.File.stdout().writer(self.io, &.{}); @@ -63,7 +81,7 @@ pub const Chat = struct { } pub fn runInteractive(self: *Chat, initial_prompt: []const u8) !void { - var turn_tokens = try self.repo.tokenizePrompt(self.allocator, self.tokenizer, initial_prompt); + var turn_tokens = try self.loaded_model.tokenizePrompt(self.allocator, self.tokenizer, initial_prompt); defer self.allocator.free(turn_tokens); var stdout = std.Io.File.stdout().writer(self.io, &.{}); @@ -130,7 +148,7 @@ pub const Chat = struct { if (input.len == 0) continue; self.allocator.free(previous_turn_tokens); - return try self.repo.tokenizeTurn(self.allocator, self.tokenizer, input); + return try self.loaded_model.tokenizeTurn(self.allocator, self.tokenizer, input); } } }; diff --git a/examples/llm/main.zig b/examples/llm/main.zig index b4fa52eb..f86be31c 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -63,17 +63,26 @@ pub fn main(init: std.process.Init) !void { const io = vfs.io(); + // + // Platform and Backend Selection + // const platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator); log.info("\n{f}", .{platform.fmtVerbose()}); + const backend = args.backend orelse b: { + const selected = zml.attention.attention.Backend.auto(platform); + log.info("Selected backend: {}", .{selected}); + break :b selected; + }; + + // + // Model initialization + // log.info("Resolving model repository..", .{}); const repo = try zml.safetensors.resolveModelRepo(io, args.model); - const model_type = try models.detectModelType(allocator, io, repo); - log.info("Detected model type: {}", .{model_type}); - log.info("Initializing model..", .{}); var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); defer registry.deinit(); @@ -81,65 +90,55 @@ pub fn main(init: std.process.Init) !void { var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); defer store.deinit(); - var model = try models.Repository.init(model_type, allocator, io, repo, store.view()); + var model = try models.LoadedModel.load(allocator, io, repo, store.view()); defer model.deinit(allocator); - const backend = args.backend orelse b: { - const selected = zml.attention.attention.Backend.auto(platform); - log.info("Selected backend: {}", .{selected}); - break :b selected; - }; + // Defines how the model's tensors are sharded across the available devices. + const shardings: models.Shardings = try .init(platform); + // + // Load the model and compile it + // var progress = std.Progress.start(io, .{ .root_name = args.model }); errdefer progress.end(); - const tokenizer = try loadTokenizer(allocator, io, repo, &progress); - defer { - var tk = tokenizer; - tk.deinit(); - } + var tokenizer = try loadTokenizer(allocator, io, repo, &progress); + defer tokenizer.deinit(); - const tp_mesh: zml.sharding.LogicalMesh = try .init("tp_mesh", .{ .model = .high_bandwidth }); - const tp_strategy: zml.sharding.Strategy = try .suggest(tp_mesh, platform.physical_mesh); - const shardings: models.Shardings = .{ - .replicated = try zml.sharding.replicatedSharding(platform), - .model = try .initFromStrategy(platform, tp_mesh, tp_strategy), - }; + var model_buffers = try models.LoadedModel.loadBuffers(&model, allocator, io, platform, &store, &progress, shardings); + defer model.unloadBuffers(&model_buffers, allocator); - var model_buffers = try model.loadBuffers(allocator, io, platform, &store, &progress, shardings); - defer model_buffers.deinit(allocator); - - var session = try model.initSession( - allocator, - io, - platform, - &model_buffers, - tokenizer, - .{ - .seqlen = args.seqlen, - .backend = backend, - .single = args.single, - }, - &progress, - shardings, - ); - defer session.deinit(); + var compiled_model = try models.LoadedModel.compile(&model, allocator, io, platform, backend, shardings, args.seqlen, &progress); + defer compiled_model.deinit(); progress.end(); - try printLogo(io); + // + // We're ready to go! Let's start a chat... + // + try printZmlLogo(io); const interactive = args.prompt == null; - const prompt = if (args.prompt) |p| try allocator.dupe(u8, p) else blk: { + const prompt = if (args.prompt) |prompt| b: { + break :b try allocator.dupe(u8, prompt); + } else b: { chat.initHistory(); const line = c.linenoise(chat.prompt_prefix) orelse return; defer c.linenoiseFree(line); chat.rememberPrompt(line); - break :blk try allocator.dupe(u8, std.mem.sliceTo(line, 0)); + break :b try allocator.dupe(u8, std.mem.sliceTo(line, 0)); }; defer allocator.free(prompt); - var llm_chat = try chat.Chat.init(allocator, io, tokenizer, &model, &session); + var llm_chat = try chat.Chat.init( + allocator, + io, + platform, + tokenizer, + &model, + &compiled_model, + &model_buffers, + ); defer llm_chat.deinit(); if (interactive) { @@ -166,7 +165,7 @@ fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, prog return try .fromBytes(allocator, io, bytes); } -pub fn printLogo(io: std.Io) !void { +pub fn printZmlLogo(io: std.Io) !void { const LOGO = \\ \\ diff --git a/examples/llm/models.zig b/examples/llm/models.zig index 70da703b..6318f4fc 100644 --- a/examples/llm/models.zig +++ b/examples/llm/models.zig @@ -3,13 +3,14 @@ const std = @import("std"); const zml = @import("zml"); const common = @import("models/common.zig"); -const SessionOptions = common.SessionOptions; pub const Shardings = common.Shardings; pub const parseConfig = common.parseConfig; pub const lfm2 = @import("models/lfm2.zig"); pub const llama = @import("models/llama.zig"); pub const qwen3_5 = @import("models/qwen3_5.zig"); +const log = std.log.scoped(.llm); + pub const ModelType = enum { lfm2, llama, @@ -20,26 +21,29 @@ const RawConfig = struct { model_type: []const u8, }; -pub const Repository = union(ModelType) { - lfm2: lfm2.Repository, - llama: llama.Repository, - qwen3_5: qwen3_5.Repository, +pub const LoadedModel = union(ModelType) { + lfm2: lfm2.LoadedModel, + llama: llama.LoadedModel, + qwen3_5: qwen3_5.LoadedModel, + + pub fn load(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir, store: zml.io.TensorStore.View) !LoadedModel { + const model_type = try detectModelType(allocator, io, repo); + log.info("Detected model type: {}", .{model_type}); - pub fn init(model_type: ModelType, allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir, store: zml.io.TensorStore.View) !Repository { return switch (model_type) { - .lfm2 => .{ .lfm2 = try lfm2.Repository.init(allocator, io, repo, store) }, - .llama => .{ .llama = try llama.Repository.init(allocator, io, repo, store) }, - .qwen3_5 => .{ .qwen3_5 = try qwen3_5.Repository.init(allocator, io, repo, store) }, + .lfm2 => .{ .lfm2 = try lfm2.LoadedModel.init(allocator, io, repo, store) }, + .llama => .{ .llama = try llama.LoadedModel.init(allocator, io, repo, store) }, + .qwen3_5 => .{ .qwen3_5 = try qwen3_5.LoadedModel.init(allocator, io, repo, store) }, }; } - pub fn deinit(self: *Repository, allocator: std.mem.Allocator) void { + pub fn deinit(self: *LoadedModel, allocator: std.mem.Allocator) void { switch (self.*) { inline else => |*m| m.deinit(allocator), } } - pub fn loadBuffers(self: *Repository, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, shardings: Shardings) !Buffers { + pub fn loadBuffers(self: *LoadedModel, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, shardings: Shardings) !Buffers { return switch (self.*) { .lfm2 => |*m| .{ .lfm2 = try m.loadBuffers(allocator, io, platform, store, progress, shardings) }, .llama => |*m| .{ .llama = try m.loadBuffers(allocator, io, platform, store, progress, shardings) }, @@ -47,65 +51,128 @@ pub const Repository = union(ModelType) { }; } - pub fn tokenizePrompt(self: *const Repository, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]const u32 { + pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { + switch (self.*) { + .lfm2 => |*loaded_model| switch (buffers.*) { + .lfm2 => |*loaded_buffers| loaded_model.unloadBuffers(loaded_buffers, allocator), + else => unreachable, + }, + .llama => |*loaded_model| switch (buffers.*) { + .llama => |*loaded_buffers| loaded_model.unloadBuffers(loaded_buffers, allocator), + else => unreachable, + }, + .qwen3_5 => |*loaded_model| switch (buffers.*) { + .qwen3_5 => |*loaded_buffers| loaded_model.unloadBuffers(loaded_buffers, allocator), + else => unreachable, + }, + } + } + + pub fn tokenizePrompt(self: *const LoadedModel, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]const u32 { return switch (self.*) { inline else => |*m| m.tokenizePrompt(allocator, tokenizer, prompt), }; } - pub fn tokenizeTurn(self: *const Repository, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]const u32 { + pub fn tokenizeTurn(self: *const LoadedModel, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]const u32 { return switch (self.*) { inline else => |*m| m.tokenizeTurn(allocator, tokenizer, prompt), }; } - pub fn initSession( - self: *const Repository, + pub fn compile( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + backend: zml.attention.attention.Backend, + shardings: Shardings, + seqlen: usize, + progress: *std.Progress.Node, + ) !CompiledModel { + return switch (self.*) { + .lfm2 => |*m| .{ .lfm2 = try m.compile( + allocator, + io, + platform, + backend, + shardings, + seqlen, + progress, + ) }, + .llama => |*m| .{ .llama = try m.compile( + allocator, + io, + platform, + backend, + shardings, + seqlen, + progress, + ) }, + .qwen3_5 => |*m| .{ .qwen3_5 = try m.compile(allocator, io, platform, backend, shardings, seqlen, progress) }, + }; + } +}; + +pub const CompiledModel = union(ModelType) { + lfm2: lfm2.inference.CompiledModel, + llama: llama.inference.CompiledModel, + qwen3_5: qwen3_5.inference.CompiledModel, + + pub fn deinit(self: *CompiledModel) void { + switch (self.*) { + .lfm2 => |*b| b.deinit(), + .llama => |*b| b.deinit(), + .qwen3_5 => |*b| b.deinit(), + } + } + + pub fn newSession( + self: *CompiledModel, allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, model_buffers: *Buffers, tokenizer: zml.tokenizer.Tokenizer, - opts: SessionOptions, - progress: *std.Progress.Node, - shardings: Shardings, ) !Session { - const inner: Session.Inner = switch (self.*) { - .lfm2 => |repo| .{ .lfm2 = try lfm2.Session.init( - allocator, - io, - platform, - &model_buffers.lfm2, - tokenizer, - repo, - opts, - progress, - shardings, - ) }, - .llama => |repo| .{ .llama = try llama.Session.init( - allocator, - io, - platform, - &model_buffers.llama, - tokenizer, - repo, - opts, - progress, - shardings, - ) }, - .qwen3_5 => |repo| .{ .qwen3_5 = try qwen3_5.Session.init( - allocator, - io, - platform, - &model_buffers.qwen3_5, - tokenizer, - repo, - opts, - progress, - shardings, - ) }, + return switch (self.*) { + .lfm2 => |*compiled| .{ + .inner = .{ .lfm2 = try lfm2.Session.init( + allocator, + io, + platform, + &model_buffers.lfm2, + tokenizer, + compiled.loaded_model.*, + compiled, + ) }, + .seqlen = compiled.params.seqlen, + }, + .llama => |*compiled| .{ + .inner = .{ .llama = try llama.Session.init( + allocator, + io, + platform, + &model_buffers.llama, + tokenizer, + compiled.loaded_model.parsed_config.value, + compiled, + ) }, + .seqlen = @intCast(compiled.params.seqlen), + }, + .qwen3_5 => |*compiled| .{ + .inner = .{ .qwen3_5 = try qwen3_5.Session.init( + allocator, + io, + platform, + &model_buffers.qwen3_5, + tokenizer, + compiled.loaded_model.inner, + compiled, + ) }, + .seqlen = compiled.params.seqlen, + }, }; - return .{ .inner = inner, .seqlen = opts.seqlen }; } }; @@ -113,14 +180,6 @@ pub const Buffers = union(ModelType) { lfm2: lfm2.Buffers, llama: llama.Buffers, qwen3_5: qwen3_5.Buffers, - - pub fn deinit(self: *Buffers, allocator: std.mem.Allocator) void { - switch (self.*) { - .lfm2 => |*b| lfm2.Repository.unloadBuffers(b, allocator), - .llama => |*b| llama.Repository.unloadBuffers(b, allocator), - .qwen3_5 => |*b| qwen3_5.Repository.unloadBuffers(b, allocator), - } - } }; pub const Session = struct { diff --git a/examples/llm/models/common.zig b/examples/llm/models/common.zig index 953061dd..971fbfb7 100644 --- a/examples/llm/models/common.zig +++ b/examples/llm/models/common.zig @@ -12,6 +12,15 @@ pub const Shardings = struct { replicated: zml.sharding.Sharding, model: zml.sharding.Sharding, + pub fn init(platform: *zml.Platform) !Shardings { + const model_mesh: zml.sharding.LogicalMesh = try .init("model", .{ .model = .high_bandwidth }); + const model_sharding_strategy: zml.sharding.Strategy = try .suggest(model_mesh, platform.physical_mesh); + return .{ + .replicated = try zml.sharding.replicatedSharding(platform), + .model = try .initFromStrategy(platform, model_mesh, model_sharding_strategy), + }; + } + pub fn all(self: *const Shardings) [2]zml.sharding.Sharding { return .{ self.replicated, self.model }; } diff --git a/examples/llm/models/lfm2.zig b/examples/llm/models/lfm2.zig index bf186ace..4b58c18a 100644 --- a/examples/llm/models/lfm2.zig +++ b/examples/llm/models/lfm2.zig @@ -1,10 +1,11 @@ pub const inference = @import("lfm2/inference.zig"); -pub const CompilationOptions = inference.CompilationOptions; +pub const CompilationParameters = inference.CompilationParameters; +pub const CompilationOptions = inference.CompilationParameters; +pub const CompiledModel = inference.CompiledModel; pub const model = @import("lfm2/model.zig"); pub const Config = model.Config; pub const Buffers = model.Buffers; pub const Model = model.Model; -pub const repository = @import("lfm2/repository.zig"); -pub const Repository = repository.Repository; +pub const LoadedModel = model.LoadedModel; pub const session = @import("lfm2/session.zig"); pub const Session = session.Session; diff --git a/examples/llm/models/lfm2/inference.zig b/examples/llm/models/lfm2/inference.zig index 0095b822..91908f47 100644 --- a/examples/llm/models/lfm2/inference.zig +++ b/examples/llm/models/lfm2/inference.zig @@ -4,12 +4,12 @@ const zml = @import("zml"); const stdx = zml.stdx; const attention = zml.attention.attention; +const common = @import("../common.zig"); const model = @import("model.zig"); -const Shardings = @import("../common.zig").Shardings; const log = std.log.scoped(.lfm); -pub const CompilationOptions = struct { +pub const CompilationParameters = struct { hidden_dim: usize, batch_dim: usize, single: bool, @@ -17,8 +17,10 @@ pub const CompilationOptions = struct { cache: model.Cache, attention_metadata: attention.Metadata, attention_parameters: attention.Parameters, + seqlen: u32, + shardings: common.Shardings, - pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: attention.Backend, single: bool) CompilationOptions { + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: attention.Backend, single: bool, shardings: common.Shardings) CompilationParameters { stdx.debug.assert(seqlen >= config.conv_L_cache, "seqlen ({}) must be at least conv_L_cache ({})", .{ seqlen, config.conv_L_cache }); const cache: model.Cache = .{ .kv = .init(.init(.{ @@ -44,10 +46,14 @@ pub const CompilationOptions = struct { .cache = cache, .attention_metadata = .init(.fromBackend(backend, seqlen, config.num_attention_heads)), .attention_parameters = .init(.fromBackend(backend)), + .seqlen = seqlen, + .shardings = shardings, }; } }; +pub const CompilationOptions = CompilationParameters; + pub const Args = struct { allocator: std.mem.Allocator, io: std.Io, @@ -61,45 +67,52 @@ pub const Args = struct { attention_metadata_buffers: zml.Bufferized(attention.Metadata), }; -pub const Inference = struct { +pub const CompiledModel = struct { + loaded_model: *const model.LoadedModel, prefill: KernelExe, decode: KernelExe, + params: CompilationParameters, pub fn init( allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, + loaded_model: *const model.LoadedModel, mdl: model.Model, - opts: CompilationOptions, - seqlen: u32, + opts: CompilationParameters, progress: *std.Progress.Node, - shardings: Shardings, - ) !Inference { + ) !CompiledModel { if (opts.single) { - const prefill_exe = try compileSingleKernelExe(allocator, io, platform, mdl, opts, seqlen, true, progress, shardings); - const decode_exe = try compileSingleKernelExe(allocator, io, platform, mdl, opts, 1, false, progress, shardings); + const prefill_exe = try compileSingleKernelExe(allocator, io, platform, mdl, opts, opts.seqlen, true, progress, opts.shardings); + const decode_exe = try compileSingleKernelExe(allocator, io, platform, mdl, opts, 1, false, progress, opts.shardings); return .{ + .loaded_model = loaded_model, .prefill = .{ .single = prefill_exe }, .decode = .{ .single = decode_exe }, + .params = opts, }; } else { - const prefill_exe = try compileComposedKernelExe(allocator, io, platform, mdl, opts, seqlen, true, progress, shardings); - const decode_exe = try compileComposedKernelExe(allocator, io, platform, mdl, opts, 1, false, progress, shardings); + const prefill_exe = try compileComposedKernelExe(allocator, io, platform, mdl, opts, opts.seqlen, true, progress, opts.shardings); + const decode_exe = try compileComposedKernelExe(allocator, io, platform, mdl, opts, 1, false, progress, opts.shardings); return .{ + .loaded_model = loaded_model, .prefill = .{ .composed = prefill_exe }, .decode = .{ .composed = decode_exe }, + .params = opts, }; } } - pub fn deinit(self: Inference) void { + pub fn deinit(self: *CompiledModel) void { self.prefill.deinit(); self.decode.deinit(); } }; +pub const Inference = CompiledModel; + pub const KernelExe = union(enum) { single: SingleKernelExe, composed: ComposedKernelExe, @@ -164,7 +177,7 @@ fn compileSingleKernelExe( seqlen: u32, is_prefill: bool, progress: *std.Progress.Node, - shardings: Shardings, + shardings: common.Shardings, ) !SingleKernelExe { progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling single kernel...", 1); @@ -313,7 +326,7 @@ fn compileComposedKernelExe( seqlen: u32, is_prefill: bool, progress: *std.Progress.Node, - shardings: Shardings, + shardings: common.Shardings, ) !ComposedKernelExe { const embed_tokens_exe = try compileEmbedTokens(allocator, io, platform, mdl.embed_tokens, opts, seqlen, progress, shardings); const conv_layer_exe = try compileConvLayer(allocator, io, platform, mdl, opts, seqlen, is_prefill, progress, shardings); @@ -335,7 +348,7 @@ fn compileEmbedTokens( opts: CompilationOptions, seqlen: u32, progress: *std.Progress.Node, - shardings: Shardings, + shardings: common.Shardings, ) !zml.Exe { progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling embed_tokens...", 1); @@ -357,7 +370,7 @@ fn compileConvLayer( seqlen: u32, is_prefill: bool, progress: *std.Progress.Node, - shardings: Shardings, + shardings: common.Shardings, ) !zml.Exe { progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling conv layer...", 1); @@ -398,7 +411,7 @@ fn compileAttnLayer( seqlen: u32, is_prefill: bool, progress: *std.Progress.Node, - shardings: Shardings, + shardings: common.Shardings, ) !zml.Exe { progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling attn layer...", 1); @@ -438,7 +451,7 @@ fn compileLmHead( opts: CompilationOptions, seqlen: u32, progress: *std.Progress.Node, - shardings: Shardings, + shardings: common.Shardings, ) !zml.Exe { progress.increaseEstimatedTotalItems(1); var node = progress.start("Compiling lm_head...", 1); diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index 9e5bdbea..ca3bf626 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -2,9 +2,10 @@ const std = @import("std"); const zml = @import("zml"); const stdx = zml.stdx; +const attention = zml.attention.attention; -pub const parseConfig = @import("../common.zig").parseConfig; -pub const Shardings = @import("../common.zig").Shardings; +const common = @import("../common.zig"); +const inference = @import("inference.zig"); const log = std.log.scoped(.lfm); @@ -47,6 +48,114 @@ pub const Config = struct { vocab_size: u32, }; +pub const LoadedModel = struct { + inner: Model, + parsed_config: std.json.Parsed(Config), + + pub fn init(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir, store: zml.io.TensorStore.View) !LoadedModel { + const parsed_config = try common.parseConfig(Config, allocator, io, repo); + errdefer parsed_config.deinit(); + + return .{ + .inner = .init(allocator, store, parsed_config.value), + .parsed_config = parsed_config, + }; + } + + pub fn deinit(self: *LoadedModel, allocator: std.mem.Allocator) void { + self.inner.deinit(allocator); + self.parsed_config.deinit(); + } + + pub fn tokenizePrompt(self: *const LoadedModel, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]u32 { + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const im_start = tokenizer.tokenToId("<|im_start|>") orelse return error.NoSuchToken; + const im_end = tokenizer.tokenToId("<|im_end|>") orelse return error.NoSuchToken; + const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; + const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; + const newline = (try encoder.encode("\n"))[0]; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); + try tokens.appendSlice(allocator, &.{ self.parsed_config.value.bos_token_id, im_start, user, newline }); + try tokens.appendSlice(allocator, try encoder.encode(prompt)); + try tokens.appendSlice(allocator, &.{ im_end, newline }); + try tokens.appendSlice(allocator, &.{ im_start, assistant, newline }); + return tokens.toOwnedSlice(allocator); + } + + pub fn tokenizeTurn(self: *const LoadedModel, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]u32 { + _ = self; + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const im_start = tokenizer.tokenToId("<|im_start|>") orelse return error.NoSuchToken; + const im_end = tokenizer.tokenToId("<|im_end|>") orelse return error.NoSuchToken; + const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; + const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; + const newline = (try encoder.encode("\n"))[0]; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); + try tokens.appendSlice(allocator, &.{ im_end, newline, im_start, user, newline }); + try tokens.appendSlice(allocator, try encoder.encode(prompt)); + try tokens.appendSlice(allocator, &.{ im_end, newline }); + try tokens.appendSlice(allocator, &.{ im_start, assistant, newline }); + return tokens.toOwnedSlice(allocator); + } + + pub fn loadBuffers( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + shardings: common.Shardings, + ) !Buffers { + const now: std.Io.Timestamp = .now(io, .awake); + var total_bytes: usize = 0; + + defer { + const took = now.untilNow(io, .awake); + const bytes_per_sec: u64 = @intFromFloat( + @as(f64, @floatFromInt(total_bytes)) / + (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s), + ); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + } + + const all_shardings = shardings.all(); + return zml.io.load(Model, &self.inner, allocator, io, platform, store, .{ + .dma_chunks = 32, + .dma_chunk_size = 128 * zml.MiB, + .progress = progress, + .shardings = &all_shardings, + .parallelism = 16, + .total_bytes = &total_bytes, + }); + } + + pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { + _ = self; + Model.unloadBuffers(buffers, allocator); + } + + pub fn compile( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + backend: zml.attention.attention.Backend, + shardings: common.Shardings, + seqlen: usize, + progress: *std.Progress.Node, + ) !inference.CompiledModel { + const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), backend, false, shardings); + return inference.CompiledModel.init(allocator, io, @constCast(platform), self, self.inner, params, progress); + } +}; + pub const Buffers = zml.Bufferized(Model); pub const Model = struct { @@ -89,7 +198,7 @@ pub const Model = struct { platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, - shardings: Shardings, + shardings: common.Shardings, ) !zml.Bufferized(Model) { progress.increaseEstimatedTotalItems(store.view().count()); const now: std.Io.Timestamp = .now(io, .awake); diff --git a/examples/llm/models/lfm2/repository.zig b/examples/llm/models/lfm2/repository.zig deleted file mode 100644 index e7ab7084..00000000 --- a/examples/llm/models/lfm2/repository.zig +++ /dev/null @@ -1,76 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); - -const model = @import("model.zig"); -const common = @import("../common.zig"); - -const parseConfig = common.parseConfig; -const Shardings = common.Shardings; - -pub const Repository = struct { - inner: model.Model, - parsed_config: std.json.Parsed(model.Config), - - pub fn init(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir, store: zml.io.TensorStore.View) !Repository { - const parsed_config = try parseConfig(model.Config, allocator, io, repo); - errdefer parsed_config.deinit(); - - return .{ - .inner = .init(allocator, store, parsed_config.value), - .parsed_config = parsed_config, - }; - } - - pub fn deinit(self: *Repository, allocator: std.mem.Allocator) void { - self.inner.deinit(allocator); - self.parsed_config.deinit(); - } - - pub fn loadBuffers(self: *Repository, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, shardings: Shardings) !model.Buffers { - return try self.inner.loadBuffers(allocator, io, platform, store, progress, shardings); - } - - pub fn unloadBuffers(self: *model.Buffers, allocator: std.mem.Allocator) void { - model.Model.unloadBuffers(self, allocator); - } - - pub fn tokenizePrompt(self: *const Repository, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]u32 { - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - const im_start = tokenizer.tokenToId("<|im_start|>") orelse return error.NoSuchToken; - const im_end = tokenizer.tokenToId("<|im_end|>") orelse return error.NoSuchToken; - const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; - const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; - const newline = (try encoder.encode("\n"))[0]; - - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); - try tokens.appendSlice(allocator, &.{ self.parsed_config.value.bos_token_id, im_start, user, newline }); - try tokens.appendSlice(allocator, try encoder.encode(prompt)); - try tokens.appendSlice(allocator, &.{ im_end, newline }); - try tokens.appendSlice(allocator, &.{ im_start, assistant, newline }); - - return tokens.toOwnedSlice(allocator); - } - - pub fn tokenizeTurn(self: *const Repository, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]u32 { - _ = self; - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - const im_start = tokenizer.tokenToId("<|im_start|>") orelse return error.NoSuchToken; - const im_end = tokenizer.tokenToId("<|im_end|>") orelse return error.NoSuchToken; - const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; - const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; - const newline = (try encoder.encode("\n"))[0]; - - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); - try tokens.appendSlice(allocator, &.{ im_end, newline, im_start, user, newline }); - try tokens.appendSlice(allocator, try encoder.encode(prompt)); - try tokens.appendSlice(allocator, &.{ im_end, newline }); - try tokens.appendSlice(allocator, &.{ im_start, assistant, newline }); - - return tokens.toOwnedSlice(allocator); - } -}; diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig index b4e5b643..a2506d8e 100644 --- a/examples/llm/models/lfm2/session.zig +++ b/examples/llm/models/lfm2/session.zig @@ -3,19 +3,15 @@ const std = @import("std"); const zml = @import("zml"); const attention = zml.attention.attention; -const common = @import("../common.zig"); -const Shardings = common.Shardings; -const SessionOptions = common.SessionOptions; const inference = @import("inference.zig"); const model = @import("model.zig"); -const repository = @import("repository.zig"); pub const Session = struct { allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, model_buffers: *model.Buffers, - exe: inference.Inference, + compiled_model: *inference.CompiledModel, config: model.Config, seqlen: u32, cache_buffers: zml.Bufferized(model.Cache), @@ -32,28 +28,22 @@ pub const Session = struct { platform: *zml.Platform, model_buffers: *model.Buffers, tokenizer: zml.tokenizer.Tokenizer, - repo: repository.Repository, - opts: SessionOptions, - progress: *std.Progress.Node, - shardings: Shardings, + loaded_model: model.LoadedModel, + compiled_model: *inference.CompiledModel, ) !Session { - const params = inference.CompilationOptions.init(repo.inner, repo.parsed_config.value, opts.seqlen, opts.backend, opts.single); - const exe = try inference.Inference.init(allocator, io, platform, repo.inner, params, opts.seqlen, progress, shardings); - errdefer exe.deinit(); - const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); return .{ .allocator = allocator, .io = io, .platform = platform, .model_buffers = model_buffers, - .exe = exe, + .compiled_model = compiled_model, .tokenizer = tokenizer, - .config = repo.parsed_config.value, - .seqlen = opts.seqlen, - .cache_buffers = try params.cache.initBuffers(allocator, io, platform, shardings.replicated), - .attention_metadata_buffers = try params.attention_metadata.initBuffer(io, platform, shardings.model), - .rng_buf = try zml.Tensor.Rng.initBuffer(platform, seed, io, shardings.replicated), + .config = loaded_model.parsed_config.value, + .seqlen = compiled_model.params.seqlen, + .cache_buffers = try compiled_model.params.cache.initBuffers(allocator, io, platform, compiled_model.params.shardings.replicated), + .attention_metadata_buffers = try compiled_model.params.attention_metadata.initBuffer(io, platform, compiled_model.params.shardings.model), + .rng_buf = try zml.Tensor.Rng.initBuffer(platform, seed, io, compiled_model.params.shardings.replicated), .generated_token_slice = try .alloc(allocator, zml.Shape.init(.{ .batch = 1, .seq = 1 }, .u32)), .think_start = tokenizer.tokenToId("") orelse unreachable, .think_end = tokenizer.tokenToId("") orelse unreachable, @@ -61,7 +51,6 @@ pub const Session = struct { } pub fn deinit(self: *Session) void { - self.exe.deinit(); model.Cache.unloadBuffers(&self.cache_buffers); attention.Metadata.deinitBuffer(&self.attention_metadata_buffers); zml.Tensor.Rng.deinitBuffer(&self.rng_buf); @@ -88,7 +77,7 @@ pub const Session = struct { var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice, sharding); defer actual_seq_len_buf.deinit(); - try self.exe.prefill.run(.{ + try self.compiled_model.prefill.run(.{ .allocator = self.allocator, .io = self.io, .platform = self.platform, @@ -141,7 +130,7 @@ pub const Session = struct { var token_pos_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice, sharding); defer token_pos_buffer.deinit(); - try self.exe.decode.run(.{ + try self.compiled_model.decode.run(.{ .allocator = self.allocator, .io = self.io, .platform = self.platform, diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig index 5fada1d3..a43d5d00 100644 --- a/examples/llm/models/lfm2_tests.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -3,9 +3,9 @@ const std = @import("std"); const zml = @import("zml"); const attention = zml.attention.attention; +const common = @import("common.zig"); const lfm2 = @import("lfm2.zig"); const model = @import("lfm2/model.zig"); -const Shardings = @import("common.zig").Shardings; pub const std_options: std.Options = .{ .log_level = .info, @@ -43,22 +43,22 @@ pub fn main(init: std.process.Init) !void { var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); defer store.deinit(); - var repo_model = try lfm2.Repository.init(allocator, io, repo, store.view()); + var repo_model = try lfm2.LoadedModel.init(allocator, io, repo, store.view()); defer repo_model.deinit(allocator); var progress = std.Progress.start(io, .{ .root_name = args.model }); const tp_mesh: zml.sharding.LogicalMesh = try .init("tp_mesh", .{ .model = .high_bandwidth }); const tp_strategy: zml.sharding.Strategy = try .suggest(tp_mesh, platform.physical_mesh); - const shardings: Shardings = .{ + const shardings: common.Shardings = .{ .replicated = try zml.sharding.replicatedSharding(platform), .model = try .initFromStrategy(platform, tp_mesh, tp_strategy), }; var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress, shardings); - defer lfm2.Repository.unloadBuffers(&model_buffers, allocator); + defer repo_model.unloadBuffers(&model_buffers, allocator); const backend = args.backend orelse attention.Backend.auto(platform); - const params = lfm2.CompilationOptions.init(repo_model.inner, repo_model.parsed_config.value, repo_model.parsed_config.value.max_position_embeddings, backend, false); + const params = lfm2.CompilationParameters.init(repo_model.inner, repo_model.parsed_config.value, repo_model.parsed_config.value.max_position_embeddings, backend, false, shardings); progress.end(); try run(allocator, io, platform, args.activations, repo_model.parsed_config.value, repo_model.inner, &model_buffers, params.attention_metadata, params.attention_parameters); diff --git a/examples/llm/models/llama.zig b/examples/llm/models/llama.zig index 025c6853..79f69e36 100644 --- a/examples/llm/models/llama.zig +++ b/examples/llm/models/llama.zig @@ -1,10 +1,9 @@ pub const inference = @import("llama/inference.zig"); -pub const CompilationOptions = inference.CompilationOptions; +pub const CompilationParameters = inference.CompilationParameters; pub const model = @import("llama/model.zig"); pub const Config = model.Config; pub const Buffers = model.Buffers; pub const Model = model.Model; -pub const repository = @import("llama/repository.zig"); -pub const Repository = repository.Repository; +pub const LoadedModel = model.LoadedModel; pub const session = @import("llama/session.zig"); pub const Session = session.Session; diff --git a/examples/llm/models/llama/inference.zig b/examples/llm/models/llama/inference.zig index 490cea47..7597ce4b 100644 --- a/examples/llm/models/llama/inference.zig +++ b/examples/llm/models/llama/inference.zig @@ -3,12 +3,12 @@ const std = @import("std"); const zml = @import("zml"); const attention = zml.attention.attention; -const Shardings = @import("../common.zig").Shardings; +const common = @import("../common.zig"); const model = @import("model.zig"); const log = std.log.scoped(.llama); -pub const CompilationOptions = struct { +pub const CompilationParameters = struct { prefill_tokens: zml.Tensor, decode_tokens: zml.Tensor, token_index: zml.Tensor, @@ -16,8 +16,10 @@ pub const CompilationOptions = struct { rng: zml.Tensor.Rng, attention_metadata: attention.Metadata, attention_parameters: attention.Parameters, + seqlen: usize, + shardings: common.Shardings, - pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: attention.Backend) CompilationOptions { + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: attention.Backend, shardings: common.Shardings) CompilationParameters { return .{ .prefill_tokens = .init(.{ .s = seqlen }, .u32), .decode_tokens = .init(.{ .s = 1 }, .u32), @@ -31,28 +33,32 @@ pub const CompilationOptions = struct { .rng = .init(), .attention_metadata = .init(.fromBackend(backend, @intCast(seqlen), @intCast(config.num_attention_heads))), .attention_parameters = .init(.fromBackend(backend)), + .seqlen = seqlen, + .shardings = shardings, }; } }; -pub const Inference = struct { +pub const CompiledModel = struct { + loaded_model: *const model.LoadedModel, prefill_exe: zml.Exe, decode_exe: zml.Exe, + params: CompilationParameters, pub fn init( allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, + loaded_model: *const model.LoadedModel, llama_model: model.Model, - parameters: CompilationOptions, - shardings: Shardings, + parameters: CompilationParameters, progress: *std.Progress.Node, - ) !Inference { - const compiled = try compileModel(allocator, io, platform, llama_model, parameters, shardings, progress); - return .{ .prefill_exe = compiled.prefill_exe, .decode_exe = compiled.decode_exe }; + ) !CompiledModel { + const compiled = try compileModel(allocator, io, platform, llama_model, parameters, progress); + return .{ .loaded_model = loaded_model, .prefill_exe = compiled.prefill_exe, .decode_exe = compiled.decode_exe, .params = parameters }; } - pub fn deinit(self: *Inference) void { + pub fn deinit(self: *CompiledModel) void { self.prefill_exe.deinit(); self.decode_exe.deinit(); } @@ -68,14 +74,13 @@ fn compileModel( io: std.Io, platform: *const zml.Platform, llama_model: model.Model, - parameters: CompilationOptions, - shardings: Shardings, + parameters: CompilationParameters, progress: *std.Progress.Node, ) !CompileModelResult { const now: std.Io.Timestamp = .now(io, .awake); defer log.info("Compiled model [{f}]", .{now.untilNow(io, .awake)}); - const all_shardings = shardings.all(); + const all_shardings = parameters.shardings.all(); var prefill_future = try io.concurrent(struct { fn call( @@ -83,7 +88,7 @@ fn compileModel( io_: std.Io, platform_: *const zml.Platform, llama_model_: model.Model, - parameters_: CompilationOptions, + parameters_: CompilationParameters, shardings_: [2]zml.sharding.Sharding, progress_: *std.Progress.Node, ) !zml.Exe { @@ -108,7 +113,8 @@ fn compileModel( ); } }.call, .{ allocator, io, platform, llama_model, parameters, all_shardings, progress }); - errdefer if (prefill_future.cancel(io)) |v| v.deinit() else |_| {}; + var prefill_future_awaited = false; + errdefer if (!prefill_future_awaited) if (prefill_future.cancel(io)) |v| v.deinit() else |_| {}; var decode_future = try io.concurrent(struct { fn call( @@ -116,7 +122,7 @@ fn compileModel( io_: std.Io, platform_: *const zml.Platform, llama_model_: model.Model, - parameters_: CompilationOptions, + parameters_: CompilationParameters, shardings_: [2]zml.sharding.Sharding, progress_: *std.Progress.Node, ) !zml.Exe { @@ -141,10 +147,18 @@ fn compileModel( ); } }.call, .{ allocator, io, platform, llama_model, parameters, all_shardings, progress }); - errdefer if (decode_future.cancel(io)) |v| v.deinit() else |_| {}; + var decode_future_awaited = false; + errdefer if (!decode_future_awaited) if (decode_future.cancel(io)) |v| v.deinit() else |_| {}; + + const prefill_exe = try prefill_future.await(io); + prefill_future_awaited = true; + errdefer prefill_exe.deinit(); + + const decode_exe = try decode_future.await(io); + decode_future_awaited = true; return .{ - .prefill_exe = try prefill_future.await(io), - .decode_exe = try decode_future.await(io), + .prefill_exe = prefill_exe, + .decode_exe = decode_exe, }; } diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index fee2ceaf..1d8011e5 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -3,6 +3,9 @@ const std = @import("std"); const zml = @import("zml"); const stdx = zml.stdx; +const common = @import("../common.zig"); +const inference = @import("inference.zig"); + pub const Config = struct { bos_token_id: u32, eos_token_id: stdx.json.Union(union(enum) { @@ -27,6 +30,120 @@ pub const Options = struct { max_seq_len: u32, }; +pub const LoadedModel = struct { + inner: Model, + parsed_config: std.json.Parsed(Config), + + pub fn init(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir, store: zml.io.TensorStore.View) !LoadedModel { + const parsed_config = try common.parseConfig(Config, allocator, io, repo); + errdefer parsed_config.deinit(); + + const options: Options = .{ + .sampling_strategy = .{}, + .max_seq_len = parsed_config.value.max_position_embeddings, + }; + + return .{ + .inner = try .init(allocator, store, parsed_config.value, options), + .parsed_config = parsed_config, + }; + } + + pub fn deinit(self: *LoadedModel, allocator: std.mem.Allocator) void { + self.inner.deinit(allocator); + self.parsed_config.deinit(); + } + + pub fn tokenizePrompt(self: *const LoadedModel, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]const u32 { + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const start_header = tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; + const end_header = tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; + const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; + const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; + const eot = tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; + const newline = (try encoder.encode("\n"))[0]; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); + try tokens.appendSlice(allocator, &.{ self.parsed_config.value.bos_token_id, start_header, user, end_header, newline }); + try tokens.appendSlice(allocator, try encoder.encode(prompt)); + try tokens.appendSlice(allocator, &.{ eot, newline, start_header, assistant, end_header, newline }); + return tokens.toOwnedSlice(allocator); + } + + pub fn tokenizeTurn(self: *const LoadedModel, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]const u32 { + _ = self; + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const start_header = tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; + const end_header = tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; + const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; + const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; + const eot = tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; + const newline = (try encoder.encode("\n"))[0]; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); + try tokens.appendSlice(allocator, &.{ eot, newline, start_header, user, end_header, newline }); + try tokens.appendSlice(allocator, try encoder.encode(prompt)); + try tokens.appendSlice(allocator, &.{ eot, newline, start_header, assistant, end_header, newline }); + return tokens.toOwnedSlice(allocator); + } + + pub fn loadBuffers( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + shardings: common.Shardings, + ) !Buffers { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + var total_bytes: usize = 0; + defer { + const took = now.untilNow(io, .awake); + const bytes_per_sec: u64 = @intFromFloat( + @as(f64, @floatFromInt(total_bytes)) / + (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s), + ); + std.log.scoped(.llama).info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + } + + const all_shardings = shardings.all(); + return zml.io.load(Model, &self.inner, allocator, io, platform, store, .{ + .dma_chunks = 32, + .dma_chunk_size = 128 * zml.MiB, + .progress = progress, + .shardings = &all_shardings, + .parallelism = 16, + .total_bytes = &total_bytes, + }); + } + + pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { + _ = self; + if (buffers.lm_head) |*lm_head| lm_head.weight.deinit(); + Llama.unloadBuffers(&buffers.model, allocator); + } + + pub fn compile( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + backend: zml.attention.attention.Backend, + shardings: common.Shardings, + seqlen: usize, + progress: *std.Progress.Node, + ) !inference.CompiledModel { + const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), backend, shardings); + return inference.CompiledModel.init(allocator, io, platform, self, self.inner, params, progress); + } +}; + pub const Buffers = zml.Bufferized(Model); /// Llama architecture, using huggingface transformers naming. @@ -103,7 +220,6 @@ pub const Model = struct { if (self.lm_head) |*lm_head| lm_head.weight.deinit(); Llama.unloadBuffers(&self.model, allocator); } - /// Predicts the token at `token_index` position. /// Returns: /// - updated `tokens`, diff --git a/examples/llm/models/llama/repository.zig b/examples/llm/models/llama/repository.zig deleted file mode 100644 index 77a1d9cc..00000000 --- a/examples/llm/models/llama/repository.zig +++ /dev/null @@ -1,81 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); - -const common = @import("../common.zig"); -const parseConfig = common.parseConfig; -const Shardings = common.Shardings; -const model = @import("model.zig"); - -pub const Repository = struct { - inner: model.Model, - parsed_config: std.json.Parsed(model.Config), - - pub fn init(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir, store: zml.io.TensorStore.View) !Repository { - const parsed_config = try parseConfig(model.Config, allocator, io, repo); - errdefer parsed_config.deinit(); - - const options: model.Options = .{ - .sampling_strategy = .{}, - .max_seq_len = parsed_config.value.max_position_embeddings, - }; - - return .{ - .inner = try .init(allocator, store, parsed_config.value, options), - .parsed_config = parsed_config, - }; - } - - pub fn deinit(self: *Repository, allocator: std.mem.Allocator) void { - self.inner.deinit(allocator); - self.parsed_config.deinit(); - } - - pub fn loadBuffers(self: *Repository, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, shardings: Shardings) !model.Buffers { - const all_shardings = shardings.all(); - return try self.inner.loadBuffers(allocator, io, platform, store, &all_shardings, progress); - } - - pub fn unloadBuffers(self: *model.Buffers, allocator: std.mem.Allocator) void { - model.Model.unloadBuffers(self, allocator); - } - - pub fn tokenizePrompt(self: *const Repository, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]u32 { - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - const start_header = tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; - const end_header = tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; - const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; - const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; - const eot = tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; - const newline = (try encoder.encode("\n"))[0]; - - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); - try tokens.appendSlice(allocator, &.{ self.parsed_config.value.bos_token_id, start_header, user, end_header, newline }); - try tokens.appendSlice(allocator, try encoder.encode(prompt)); - try tokens.appendSlice(allocator, &.{ eot, newline, start_header, assistant, end_header, newline }); - - return tokens.toOwnedSlice(allocator); - } - - pub fn tokenizeTurn(self: *const Repository, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]u32 { - _ = self; - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - const start_header = tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; - const end_header = tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; - const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; - const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; - const eot = tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; - const newline = (try encoder.encode("\n"))[0]; - - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); - try tokens.appendSlice(allocator, &.{ eot, newline, start_header, user, end_header, newline }); - try tokens.appendSlice(allocator, try encoder.encode(prompt)); - try tokens.appendSlice(allocator, &.{ eot, newline, start_header, assistant, end_header, newline }); - - return tokens.toOwnedSlice(allocator); - } -}; diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig index eb10dc17..2218cfbb 100644 --- a/examples/llm/models/llama/session.zig +++ b/examples/llm/models/llama/session.zig @@ -2,19 +2,15 @@ const std = @import("std"); const zml = @import("zml"); -const common = @import("../common.zig"); -const Shardings = common.Shardings; -const SessionOptions = common.SessionOptions; const inference = @import("inference.zig"); const model = @import("model.zig"); -const repository = @import("repository.zig"); pub const Session = struct { allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, model_buffers: *model.Buffers, - exe: inference.Inference, + compiled_model: *inference.CompiledModel, kv_cache_buffers: zml.Bufferized(model.KvCache), attention_metadata_buffers: zml.Bufferized(zml.attention.attention.Metadata), rng_buffers: zml.Bufferized(zml.Tensor.Rng), @@ -29,21 +25,14 @@ pub const Session = struct { platform: *zml.Platform, model_buffers: *model.Buffers, tokenizer: zml.tokenizer.Tokenizer, - repo: repository.Repository, - opts: SessionOptions, - progress: *std.Progress.Node, - shardings: Shardings, + config: model.Config, + compiled_model: *inference.CompiledModel, ) !Session { - if (opts.single) return error.SingleKernelNotSupported; - - const params = inference.CompilationOptions.init(repo.inner, repo.parsed_config.value, opts.seqlen, opts.backend); - var exe = try inference.Inference.init(allocator, io, platform, repo.inner, params, shardings, progress); - errdefer exe.deinit(); - - var kv_cache_buffers = try params.kv_cache.initBuffer(io, platform, shardings.model); + const shardings = compiled_model.params.shardings; + var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform, shardings.model); errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); - var attention_metadata_buffers = try params.attention_metadata.initBuffer(io, platform, shardings.model); + var attention_metadata_buffers = try compiled_model.params.attention_metadata.initBuffer(io, platform, shardings.model); errdefer zml.attention.attention.Metadata.deinitBuffer(&attention_metadata_buffers); const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); @@ -55,19 +44,18 @@ pub const Session = struct { .io = io, .platform = platform, .model_buffers = model_buffers, - .exe = exe, + .compiled_model = compiled_model, .kv_cache_buffers = kv_cache_buffers, .attention_metadata_buffers = attention_metadata_buffers, .rng_buffers = rng_buffers, .generated_token_slice = try .alloc(allocator, zml.Shape.init(.{ .s = 1 }, .u32)), .tokenizer = tokenizer, - .config = repo.parsed_config.value, - .seqlen = opts.seqlen, + .config = config, + .seqlen = @intCast(compiled_model.params.seqlen), }; } pub fn deinit(self: *Session) void { - self.exe.deinit(); model.KvCache.deinitBuffer(&self.kv_cache_buffers); zml.attention.attention.Metadata.deinitBuffer(&self.attention_metadata_buffers); zml.Tensor.Rng.deinitBuffer(&self.rng_buffers); @@ -75,10 +63,10 @@ pub const Session = struct { } pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { - var prefill_args = try self.exe.prefill_exe.args(self.allocator); + var prefill_args = try self.compiled_model.prefill_exe.args(self.allocator); defer prefill_args.deinit(self.allocator); - var prefill_results = try self.exe.prefill_exe.results(self.allocator); + var prefill_results = try self.compiled_model.prefill_exe.results(self.allocator); defer prefill_results.deinit(self.allocator); const prefill_tokens_slice: zml.Slice = try .alloc(self.allocator, .init(.{self.seqlen}, .u32)); @@ -101,7 +89,7 @@ pub const Session = struct { &self.rng_buffers, &self.attention_metadata_buffers, }); - self.exe.prefill_exe.call(prefill_args, &prefill_results); + self.compiled_model.prefill_exe.call(prefill_args, &prefill_results); prefill_results.fill(.{ &prefill_tokens_buffer, &self.kv_cache_buffers, &self.rng_buffers }); try prefill_tokens_buffer.toSlice(self.io, prefill_tokens_slice); @@ -113,10 +101,10 @@ pub const Session = struct { var decoder = try self.tokenizer.decoder(); defer decoder.deinit(); - var decode_args = try self.exe.decode_exe.args(self.allocator); + var decode_args = try self.compiled_model.decode_exe.args(self.allocator); defer decode_args.deinit(self.allocator); - var decode_results = try self.exe.decode_exe.results(self.allocator); + var decode_results = try self.compiled_model.decode_exe.results(self.allocator); defer decode_results.deinit(self.allocator); const replicated_sharding = try zml.sharding.replicatedSharding(self.platform); @@ -149,7 +137,7 @@ pub const Session = struct { &self.rng_buffers, &self.attention_metadata_buffers, }); - self.exe.decode_exe.call(decode_args, &decode_results); + self.compiled_model.decode_exe.call(decode_args, &decode_results); decode_results.fill(.{ ¤t_token_buffer, &self.kv_cache_buffers, &self.rng_buffers }); try current_token_buffer.toSlice(self.io, self.generated_token_slice); diff --git a/examples/llm/models/llama_tests.zig b/examples/llm/models/llama_tests.zig index 2767cfab..8ffeda7c 100644 --- a/examples/llm/models/llama_tests.zig +++ b/examples/llm/models/llama_tests.zig @@ -2,10 +2,9 @@ const std = @import("std"); const zml = @import("zml"); +const common = @import("common.zig"); const llama = @import("llama.zig"); const model = @import("llama/model.zig"); -const Shardings = @import("common.zig").Shardings; -const parseConfig = @import("common.zig").parseConfig; pub const std_options: std.Options = .{ .log_level = .info, @@ -41,19 +40,19 @@ pub fn main(init: std.process.Init) !void { var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); defer store.deinit(); - var repo_model = try llama.Repository.init(allocator, io, repo, store.view()); + var repo_model = try llama.LoadedModel.init(allocator, io, repo, store.view()); defer repo_model.deinit(allocator); var progress = std.Progress.start(io, .{ .root_name = args.model }); const tp_mesh: zml.sharding.LogicalMesh = try .init("tp_mesh", .{ .model = .high_bandwidth }); const tp_strategy: zml.sharding.Strategy = try .suggest(tp_mesh, platform.physical_mesh); - const shardings: Shardings = .{ + const shardings: common.Shardings = .{ .replicated = try zml.sharding.replicatedSharding(platform), .model = try .initFromStrategy(platform, tp_mesh, tp_strategy), }; var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress, shardings); - defer llama.Repository.unloadBuffers(&model_buffers, allocator); + defer repo_model.unloadBuffers(&model_buffers, allocator); progress.end(); try run(allocator, io, platform, args.activations, repo_model.inner, &model_buffers, shardings.replicated); diff --git a/examples/llm/models/qwen3_5.zig b/examples/llm/models/qwen3_5.zig index aded98ae..4147f695 100644 --- a/examples/llm/models/qwen3_5.zig +++ b/examples/llm/models/qwen3_5.zig @@ -1,10 +1,11 @@ pub const inference = @import("qwen3_5/inference.zig"); -pub const CompilationOptions = inference.CompilationOptions; +pub const CompilationParameters = inference.CompilationParameters; +pub const CompilationOptions = inference.CompilationParameters; +pub const CompiledModel = inference.CompiledModel; pub const model = @import("qwen3_5/model.zig"); pub const Config = model.Config; pub const Buffers = model.Buffers; pub const Model = model.Model; -pub const repository = @import("qwen3_5/repository.zig"); -pub const Repository = repository.Repository; +pub const LoadedModel = model.LoadedModel; pub const session = @import("qwen3_5/session.zig"); pub const Session = session.Session; diff --git a/examples/llm/models/qwen3_5/inference.zig b/examples/llm/models/qwen3_5/inference.zig index a630af76..e719acc1 100644 --- a/examples/llm/models/qwen3_5/inference.zig +++ b/examples/llm/models/qwen3_5/inference.zig @@ -2,7 +2,7 @@ const std = @import("std"); const zml = @import("zml"); -const Shardings = @import("../common.zig").Shardings; +const common = @import("../common.zig"); const model = @import("model.zig"); const log = std.log.scoped(.qwen3_5); @@ -12,14 +12,16 @@ const CompileModelResult = struct { decode_exe: ?zml.Exe = null, }; -pub const CompilationOptions = struct { +pub const CompilationParameters = struct { prefill_tokens: zml.Tensor, decode_tokens: zml.Tensor, token_index: zml.Tensor, kv_cache: model.KvCache, rng: zml.Tensor.Rng, + seqlen: u32, + shardings: common.Shardings, - pub fn init(mdl: model.Model, config: model.Config, seqlen: u32) CompilationOptions { + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, shardings: common.Shardings) CompilationParameters { const dtype = mdl.text_model.embed_tokens.weight.dtype(); return .{ .prefill_tokens = .init(.{ .b = 1, .s = seqlen }, .u32), @@ -27,46 +29,53 @@ pub const CompilationOptions = struct { .token_index = .init(.{}, .u32), .kv_cache = .init(config, 1, seqlen, dtype, .f32), .rng = .init(), + .seqlen = seqlen, + .shardings = shardings, }; } }; -pub const Inference = struct { +pub const CompilationOptions = CompilationParameters; + +pub const CompiledModel = struct { + loaded_model: *const model.LoadedModel, prefill_exe: zml.Exe, decode_exe: zml.Exe, + params: CompilationParameters, pub fn init( allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, + loaded_model: *const model.LoadedModel, qwen_model: model.Model, - parameters: CompilationOptions, - shardings: Shardings, + parameters: CompilationParameters, progress: *std.Progress.Node, - ) !Inference { - const compile_result = try compileModel(allocator, io, platform, qwen_model, parameters, shardings, progress); - return .{ .prefill_exe = compile_result.prefill_exe.?, .decode_exe = compile_result.decode_exe.? }; + ) !CompiledModel { + const compile_result = try compileModel(allocator, io, platform, qwen_model, parameters, progress); + return .{ .loaded_model = loaded_model, .prefill_exe = compile_result.prefill_exe.?, .decode_exe = compile_result.decode_exe.?, .params = parameters }; } - pub fn deinit(self: *Inference) void { + pub fn deinit(self: *CompiledModel) void { self.prefill_exe.deinit(); self.decode_exe.deinit(); } }; +pub const Inference = CompiledModel; + fn compileModel( allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, qwen_model: model.Model, - parameters: CompilationOptions, - shardings: Shardings, + parameters: CompilationParameters, progress: *std.Progress.Node, ) !CompileModelResult { const now: std.Io.Timestamp = .now(io, .awake); defer log.info("Compiled model [{f}]", .{now.untilNow(io, .awake)}); - const all_shardings = [_]zml.sharding.Sharding{shardings.replicated}; + const all_shardings = [_]zml.sharding.Sharding{parameters.shardings.replicated}; var prefill_future = try io.concurrent(struct { fn call( @@ -74,7 +83,7 @@ fn compileModel( io_: std.Io, platform_: *const zml.Platform, qwen_model_: model.Model, - parameters_: CompilationOptions, + parameters_: CompilationParameters, shardings_: [1]zml.sharding.Sharding, progress_: *std.Progress.Node, ) !zml.Exe { @@ -108,7 +117,7 @@ fn compileModel( io_: std.Io, platform_: *const zml.Platform, qwen_model_: model.Model, - parameters_: CompilationOptions, + parameters_: CompilationParameters, shardings_: [1]zml.sharding.Sharding, progress_: *std.Progress.Node, ) !zml.Exe { @@ -136,8 +145,11 @@ fn compileModel( }.call, .{ allocator, io, platform, qwen_model, parameters, all_shardings, progress }); errdefer if (decode_future.cancel(io)) |v| v.deinit() else |_| {}; + const prefill_exe = try prefill_future.await(io); + const decode_exe = try decode_future.await(io); + return .{ - .prefill_exe = try prefill_future.await(io), - .decode_exe = try decode_future.await(io), + .prefill_exe = prefill_exe, + .decode_exe = decode_exe, }; } diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 86ccd690..ddf7fb75 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -3,6 +3,9 @@ const std = @import("std"); const zml = @import("zml"); const stdx = zml.stdx; +const common = @import("../common.zig"); +const inference = @import("inference.zig"); + const log = std.log.scoped(.qwen3_5); pub const Config = struct { @@ -41,6 +44,92 @@ pub const RopeParameters = struct { rope_theta: f32, }; +pub const LoadedModel = struct { + inner: Model, + parsed_config: std.json.Parsed(Config), + + pub fn init(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir, store: zml.io.TensorStore.View) !LoadedModel { + const parsed_config = try common.parseConfig(Config, allocator, io, repo); + errdefer parsed_config.deinit(); + + const options: Model.GenOptions = .{ + .sampling_strategy = .{}, + .max_seq_len = parsed_config.value.text_config.max_position_embeddings, + }; + + return .{ + .inner = try .init(allocator, store, parsed_config.value, options), + .parsed_config = parsed_config, + }; + } + + pub fn deinit(self: *LoadedModel, allocator: std.mem.Allocator) void { + self.inner.deinit(allocator); + self.parsed_config.deinit(); + } + + pub fn tokenizePrompt(self: *const LoadedModel, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]const u32 { + return tokenizeChatPrompt(allocator, tokenizer, prompt, self.inner.special_tokens, true); + } + + pub fn tokenizeTurn(self: *const LoadedModel, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]const u32 { + return tokenizeChatPrompt(allocator, tokenizer, prompt, self.inner.special_tokens, false); + } + + pub fn loadBuffers( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + shardings: common.Shardings, + ) !Buffers { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + var total_bytes: usize = 0; + defer { + const took = now.untilNow(io, .awake); + const bytes_per_sec: u64 = @intFromFloat( + @as(f64, @floatFromInt(total_bytes)) / + (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s), + ); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + } + + const all_shardings = [_]zml.sharding.Sharding{shardings.replicated}; + return zml.io.load(Model, &self.inner, allocator, io, platform, store, .{ + .dma_chunks = 32, + .dma_chunk_size = 128 * zml.MiB, + .progress = progress, + .shardings = &all_shardings, + .parallelism = 16, + .total_bytes = &total_bytes, + }); + } + + pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { + _ = self; + TextModel.unloadBuffers(&buffers.text_model, allocator); + buffers.lm_head.weight.deinit(); + } + + pub fn compile( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + backend: zml.attention.attention.Backend, + shardings: common.Shardings, + seqlen: usize, + progress: *std.Progress.Node, + ) !inference.CompiledModel { + _ = backend; + const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), shardings); + return inference.CompiledModel.init(allocator, io, platform, self, self.inner, params, progress); + } +}; + pub const Buffers = zml.Bufferized(Model); pub const Model = struct { @@ -111,7 +200,6 @@ pub const Model = struct { TextModel.unloadBuffers(&self.text_model, allocator); self.lm_head.weight.deinit(); } - pub fn forward( self: Model, tokens_: zml.Tensor, @@ -261,6 +349,39 @@ pub const TransformerLayer = struct { } }; +fn tokenizeChatPrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8, special_tokens: Model.SpecialTokens, is_first_turn: bool) ![]u32 { + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const im_start = tokenizer.tokenToId("<|im_start|>") orelse special_tokens.im_start_token_id; + const im_end = tokenizer.tokenToId("<|im_end|>") orelse special_tokens.im_end_token_id; + const think = tokenizer.tokenToId("") orelse return error.NoSuchToken; + const newline = try encodeSingleToken(&encoder, "\n"); + const user_prefix = try encoder.encode("user\n"); + const assistant_prefix = try encoder.encode("assistant\n"); + const encoded_prompt = try encoder.encode(prompt); + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, encoded_prompt.len + user_prefix.len + assistant_prefix.len + 8); + if (!is_first_turn) { + try tokens.appendSlice(allocator, &.{ im_end, newline }); + } + + try tokens.append(allocator, im_start); + try tokens.appendSlice(allocator, user_prefix); + try tokens.appendSlice(allocator, encoded_prompt); + try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); + try tokens.appendSlice(allocator, assistant_prefix); + try tokens.appendSlice(allocator, &.{ think, newline }); + + return tokens.toOwnedSlice(allocator); +} + +fn encodeSingleToken(encoder: *zml.tokenizer.Tokenizer.Encoder, text: []const u8) !u32 { + const encoded = try encoder.encode(text); + if (encoded.len != 1) return error.InvalidTokenizerEncoding; + return encoded[0]; +} + pub const Mlp = struct { up_proj: zml.nn.Linear, gate_proj: zml.nn.Linear, diff --git a/examples/llm/models/qwen3_5/repository.zig b/examples/llm/models/qwen3_5/repository.zig deleted file mode 100644 index fe745a16..00000000 --- a/examples/llm/models/qwen3_5/repository.zig +++ /dev/null @@ -1,83 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); - -const common = @import("../common.zig"); -const parseConfig = common.parseConfig; -const Shardings = common.Shardings; -const model = @import("model.zig"); - -pub const Repository = struct { - inner: model.Model, - parsed_config: std.json.Parsed(model.Config), - - pub fn init(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir, store: zml.io.TensorStore.View) !Repository { - const parsed_config = try parseConfig(model.Config, allocator, io, repo); - errdefer parsed_config.deinit(); - - const options: model.Model.GenOptions = .{ - .sampling_strategy = .{}, - .max_seq_len = parsed_config.value.text_config.max_position_embeddings, - }; - - return .{ - .inner = try .init(allocator, store, parsed_config.value, options), - .parsed_config = parsed_config, - }; - } - - pub fn deinit(self: *Repository, allocator: std.mem.Allocator) void { - self.inner.deinit(allocator); - self.parsed_config.deinit(); - } - - pub fn loadBuffers(self: *Repository, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, shardings: Shardings) !model.Buffers { - const all_shardings = [_]zml.sharding.Sharding{shardings.replicated}; - return try self.inner.load(allocator, io, platform, store, &all_shardings, progress); - } - - pub fn unloadBuffers(self: *model.Buffers, allocator: std.mem.Allocator) void { - model.Model.unloadBuffers(self, allocator); - } - - pub fn tokenizePrompt(self: *const Repository, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]u32 { - return tokenizeChatPrompt(allocator, tokenizer, prompt, self.inner.special_tokens, true); - } - - pub fn tokenizeTurn(self: *const Repository, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]u32 { - return tokenizeChatPrompt(allocator, tokenizer, prompt, self.inner.special_tokens, false); - } -}; - -fn tokenizeChatPrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8, special_tokens: model.Model.SpecialTokens, is_first_turn: bool) ![]u32 { - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - const im_start = tokenizer.tokenToId("<|im_start|>") orelse special_tokens.im_start_token_id; - const im_end = tokenizer.tokenToId("<|im_end|>") orelse special_tokens.im_end_token_id; - const think = tokenizer.tokenToId("") orelse return error.NoSuchToken; - const newline = try encodeSingleToken(&encoder, "\n"); - const user_prefix = try encoder.encode("user\n"); - const assistant_prefix = try encoder.encode("assistant\n"); - const encoded_prompt = try encoder.encode(prompt); - - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, encoded_prompt.len + user_prefix.len + assistant_prefix.len + 8); - if (!is_first_turn) { - try tokens.appendSlice(allocator, &.{ im_end, newline }); - } - - try tokens.append(allocator, im_start); - try tokens.appendSlice(allocator, user_prefix); - try tokens.appendSlice(allocator, encoded_prompt); - try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); - try tokens.appendSlice(allocator, assistant_prefix); - try tokens.appendSlice(allocator, &.{ think, newline }); - - return tokens.toOwnedSlice(allocator); -} - -fn encodeSingleToken(encoder: *zml.tokenizer.Tokenizer.Encoder, text: []const u8) !u32 { - const encoded = try encoder.encode(text); - if (encoded.len != 1) return error.InvalidTokenizerEncoding; - return encoded[0]; -} diff --git a/examples/llm/models/qwen3_5/session.zig b/examples/llm/models/qwen3_5/session.zig index ca32c4a2..f7a35519 100644 --- a/examples/llm/models/qwen3_5/session.zig +++ b/examples/llm/models/qwen3_5/session.zig @@ -2,21 +2,15 @@ const std = @import("std"); const zml = @import("zml"); -const common = @import("../common.zig"); -const Shardings = common.Shardings; -const SessionOptions = common.SessionOptions; const inference = @import("inference.zig"); const model = @import("model.zig"); -const repository = @import("repository.zig"); pub const Session = struct { allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, - replicated_sharding: zml.sharding.Sharding, model_buffers: *model.Buffers, - compilation_options: inference.CompilationOptions, - exe: inference.Inference, + compiled_model: *inference.CompiledModel, kv_cache_buffers: zml.Bufferized(model.KvCache), rng_buffers: zml.Bufferized(zml.Tensor.Rng), tokenizer: zml.tokenizer.Tokenizer, @@ -33,52 +27,36 @@ pub const Session = struct { platform: *zml.Platform, model_buffers: *model.Buffers, tokenizer: zml.tokenizer.Tokenizer, - repo: repository.Repository, - opts: SessionOptions, - progress: *std.Progress.Node, - shardings: Shardings, + mdl: model.Model, + compiled_model: *inference.CompiledModel, ) !Session { - _ = opts.backend; - if (opts.single) return error.SingleKernelNotSupported; - - //======================= Model & cache init ======================== - - const params = inference.CompilationOptions.init(repo.inner, repo.parsed_config.value, opts.seqlen); - const replicated_sharding = shardings.replicated; - var exe = try inference.Inference.init(allocator, io, platform, repo.inner, params, shardings, progress); - errdefer exe.deinit(); - - var kv_cache_buffers = try params.kv_cache.initBuffer(io, platform); + const replicated_sharding = compiled_model.params.shardings.replicated; + var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform); errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); var rng_buffers = try zml.Tensor.Rng.initBuffer(platform, seed, io, replicated_sharding); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); - //======================= Generation setup ======================== - return .{ .allocator = allocator, .io = io, .platform = platform, - .replicated_sharding = replicated_sharding, .model_buffers = model_buffers, - .compilation_options = params, - .exe = exe, + .compiled_model = compiled_model, .kv_cache_buffers = kv_cache_buffers, .rng_buffers = rng_buffers, .tokenizer = tokenizer, .step_token_slice = try .alloc(allocator, zml.Shape.init(.{ .b = 1, .s = 1 }, .u32)), .generated_token_slice = try .alloc(allocator, zml.Shape.init(.{ .b = 1, .s = 1 }, .u32)), - .seqlen = opts.seqlen, - .eos_token_id = repo.inner.special_tokens.end_of_text_token_id, + .seqlen = compiled_model.params.seqlen, + .eos_token_id = mdl.special_tokens.end_of_text_token_id, .think_start = tokenizer.tokenToId(""), .think_end = tokenizer.tokenToId(""), }; } pub fn deinit(self: *Session) void { - self.exe.deinit(); model.KvCache.deinitBuffer(&self.kv_cache_buffers); zml.Tensor.Rng.deinitBuffer(&self.rng_buffers); self.step_token_slice.free(self.allocator); @@ -86,24 +64,24 @@ pub const Session = struct { } pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { - //======================= Running model for prompt ======================== - const prefill_tokens_shape = zml.Shape.init(.{ .b = 1, .s = self.seqlen }, .u32); const prefill_tokens_slice = try zml.Slice.alloc(self.allocator, prefill_tokens_shape); defer prefill_tokens_slice.free(self.allocator); @memset(prefill_tokens_slice.items(u32), 0); @memcpy(prefill_tokens_slice.items(u32)[0..all_tokens.len], all_tokens); - var prefill_tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, prefill_tokens_slice, self.replicated_sharding); + const replicated_sharding = self.compiled_model.params.shardings.replicated; + + var prefill_tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, prefill_tokens_slice, replicated_sharding); defer prefill_tokens_buffer.deinit(); - var prefill_token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 0), .u32, self.replicated_sharding); + var prefill_token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 0), .u32, replicated_sharding); defer prefill_token_index_buffer.deinit(); - var args = try self.exe.prefill_exe.args(self.allocator); + var args = try self.compiled_model.prefill_exe.args(self.allocator); defer args.deinit(self.allocator); - var results = try self.exe.prefill_exe.results(self.allocator); + var results = try self.compiled_model.prefill_exe.results(self.allocator); defer results.deinit(self.allocator); args.set(.{ @@ -113,7 +91,7 @@ pub const Session = struct { &self.kv_cache_buffers, &self.rng_buffers, }); - self.exe.prefill_exe.call(args, &results); + self.compiled_model.prefill_exe.call(args, &results); results.fill(.{ &prefill_tokens_buffer, &self.kv_cache_buffers, &self.rng_buffers }); try prefill_tokens_buffer.toSlice(self.io, prefill_tokens_slice); @@ -123,8 +101,6 @@ pub const Session = struct { } pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { - //======================= Generation setup ======================== - var decoder = try self.tokenizer.decoder(); defer decoder.deinit(); @@ -153,16 +129,18 @@ pub const Session = struct { fn runDecodeStep(self: *Session, token_id: u32, token_index: u32) !void { self.step_token_slice.items(u32)[0] = token_id; - var token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.step_token_slice, self.replicated_sharding); + const replicated_sharding = self.compiled_model.params.shardings.replicated; + + var token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.step_token_slice, replicated_sharding); defer token_buffer.deinit(); - var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, token_index, .u32, self.replicated_sharding); + var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, token_index, .u32, replicated_sharding); defer token_index_buffer.deinit(); - var args = try self.exe.decode_exe.args(self.allocator); + var args = try self.compiled_model.decode_exe.args(self.allocator); defer args.deinit(self.allocator); - var results = try self.exe.decode_exe.results(self.allocator); + var results = try self.compiled_model.decode_exe.results(self.allocator); defer results.deinit(self.allocator); args.set(.{ @@ -172,7 +150,7 @@ pub const Session = struct { &self.kv_cache_buffers, &self.rng_buffers, }); - self.exe.decode_exe.call(args, &results); + self.compiled_model.decode_exe.call(args, &results); results.fill(.{ &token_buffer, &self.kv_cache_buffers, &self.rng_buffers }); try token_buffer.toSlice(self.io, self.generated_token_slice); From f17291e7a4874d2b5b960978f3cab0eb125a3702 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Tue, 24 Mar 2026 18:18:33 +0100 Subject: [PATCH 385/881] workspace: update readme for zml/v2 (#431) --- README.md | 210 +++++++++++++++++++++--------------------------------- 1 file changed, 82 insertions(+), 128 deletions(-) diff --git a/README.md b/README.md index 4a2584d0..e688a90f 100644 --- a/README.md +++ b/README.md @@ -2,89 +2,49 @@ Website | Getting Started - | Documentation + | Documentation | Discord | Contributing [ZML]: https://zml.ai/ [Getting Started]: #getting-started -[Documentation]: https://docs.zml.ai +[Documentation]: ./docs/README.md [Contributing]: ./CONTRIBUTING.md [Discord]: https://discord.gg/6y72SN2E7H -# Bonjour 👋 +# About -At ZML, we are creating exciting AI products on top of our high-performance -AI inference stack. Our stack is built for production, using the amazing -[Zig](https://ziglang.org) language, [MLIR](https://mlir.llvm.org), and the -power of [Bazel](https://bazel.build). +ZML is a production inference stack, purpose-built to decouple AI workloads from proprietary hardware. -
-
Take me straight to getting started or give me a taste 🥐!
-
- ---- - -  - -# We're happy to share! -We're very happy to share our inference stack with the World and hope it allows -you, too, to build cool and exciting AI projects. - -To give you a glimpse of what you can do with ZML, here is an early demo: - -
- -It shows a prototype running a LLama3 model sharded on 1 NVIDIA RTX 4090, 1 AMD -6800XT, and 1 Google Cloud TPU v2. All accelerators were hosted in different -locations, with activations being passed over a VPN. - -All processes used the same model code, cross-compiled on a Mac, and copied onto -the servers. - -For more inspiration, see also the examples below or check out the -[examples](./examples) folder. +Any model, many hardwares, one codebase, peak performance. +Compiled directly to NVIDIA, AMD, TPU, Trainium for peak hardware performance on any accelerator. No rewriting. +It is built using the +[Zig](https://ziglang.org) language, [MLIR](https://mlir.llvm.org), and [Bazel](https://bazel.build). # Getting started - - ## Prerequisites We use `bazel` to build ZML and its dependencies. The only prerequisite is `bazel`, which we recommend to download through `bazelisk`, a version manager for `bazel`. -**Please note: If you do not wish to install `bazel`** system-wide, we provide -[bazel.sh](bazel.sh) which downloads it to your home folder -and runs it. - **Install Bazel** (recommended): -
- ### macOS - - ``` brew install bazelisk ``` -
- -
### Linux - ``` -curl -L -o /usr/local/bin/bazel 'https://github.com/bazelbuild/bazelisk/releases/download/v1.25.0/bazelisk-linux-amd64' +curl -L -o /usr/local/bin/bazel 'https://github.com/bazelbuild/bazelisk/releases/download/v1.28.0/bazelisk-linux-amd64' chmod +x /usr/local/bin/bazel ``` -
- ## Run a pre-packaged model @@ -103,53 +63,41 @@ compile it, and classify a randomly picked example from the test dataset. On the command line: ``` -bazel run --config=release //examples/mnist - -# or -./bazel.sh run --config=release //examples/mnist +bazel run //examples/mnist ``` +### Meta Llama 3.2 1B + +This model has restrictions, see +[here](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct). It **requires +approval from Meta on Hugging Face**, which can take a few hours to get granted. + +Ensure you are authenticated with the Hugging Face CLI: +``` +hf auth login +``` +Alternatively, set the `HF_TOKEN` environment variable. + +Now, you can run the model like so: +``` +bazel run //examples/llm -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" +``` + +For a larger 3.2 model, you can also try `Llama-3.2-3B-Instruct`. ### Meta Llama 3.1 8B -This model has restrictions, see -[here](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct). It **requires -approval from Meta on Huggingface**, which can take a few hours to get granted. +Like the 1B model above, this model also requires approval. See +[here](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) for access requirements. -Once you've been granted access, you're ready to download a gated model like -`Llama-3.1-8B-Instruct`! - -First, you need to download the model using the [hf](https://huggingface.co/docs/huggingface_hub/en/guides/cli) CLI. -Note you don't need to install it yourself, -you can just use the packaged version `bazel run //tools/hf --`. ``` -bazel run //tools/hf -- download meta-llama/Llama-3.1-8B-Instruct --local-dir $HOME/Llama-3.1-8B-Instruct --exclude='*.pth' -``` - -Then, you can run the model. - -``` -bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct -bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct --prompt="What is the capital of France?" +bazel run //examples/llm -- --model=hf://meta-llama/Llama-3.1-8B-Instruct --prompt="What is the capital of France?" ``` You can also try `Llama-3.1-70B-Instruct` if you have enough memory. -### Meta Llama 3.2 1B - -Like the 8B model above, this model also requires approval. See -[here](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) for access requirements. - -``` -bazel run //tools/hf -- download meta-llama/Llama-3.2-1B-Instruct --local-dir $HOME/Llama-3.2-1B-Instruct --exclude='*.pth' -bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct -bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct -- --prompt="What is the capital of France?" -``` - -For a larger 3.2 model, you can also try `Llama-3.2-3B-Instruct`. - ## Running Models on GPU / TPU You can compile models for accelerator runtimes by appending one or more of the @@ -163,16 +111,20 @@ following arguments to the command line when compiling / running a model: The latter, avoiding compilation for CPU, cuts down compilation time. -So, to run the Llama 3.1 8B model from above on your host sporting an NVIDIA GPU, +So, to run the Llama 3.1 8B model from above on your host supporting an NVIDIA GPU, run the following: ``` -bazel run --config=release //examples/llama \ - --@zml//platforms:cuda=true \ - -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct \ - --prompt="What is the capital of France?" +bazel run //examples/llm --@zml//platforms:cuda=true -- --model=hf://meta-llama/Llama-3.1-8B-Instruct ``` +And on your host supporting an AMD GPU: + +``` +bazel run //examples/llm --@zml//platforms:rocm=true -- --model=hf://meta-llama/Llama-3.1-8B-Instruct +``` + +Same goes for all supported platforms. ## Run Tests @@ -180,19 +132,12 @@ bazel run --config=release //examples/llama \ bazel test //zml:test ``` - # A taste of ZML - - ## MNIST ```zig -const std = @import("std"); -const zml = @import("zml"); - -/// Model definition const Mnist = struct { fc1: Layer, fc2: Layer, @@ -201,64 +146,73 @@ const Mnist = struct { weight: zml.Tensor, bias: zml.Tensor, + pub fn init(store: zml.io.TensorStore.View) Layer { + return .{ + .weight = store.createTensor("weight", .{ .d_out, .d }, null), + .bias = store.createTensor("bias", .{.d_out}, null), + }; + } + pub fn forward(self: Layer, input: zml.Tensor) zml.Tensor { - return self.weight.matmul(input).add(self.bias).relu(); + return self.weight.dot(input, .d).add(self.bias).relu().withTags(.{.d}); } }; + pub fn init(store: zml.io.TensorStore.View) Mnist { + return .{ + .fc1 = .init(store.withPrefix("fc1")), + .fc2 = .init(store.withPrefix("fc2")), + }; + } + + pub fn load( + self: *const Mnist, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *const zml.io.TensorStore, + shardings: []const zml.sharding.Sharding, + ) !zml.Bufferized(Mnist) { + return zml.io.load(Mnist, self, allocator, io, platform, store, .{ + .shardings = shardings, + .parallelism = 1, + .dma_chunks = 1, + .dma_chunk_size = 16 * 1024 * 1024, + }); + } + + pub fn unloadBuffers(self: *zml.Bufferized(Mnist)) void { + self.fc1.weight.deinit(); + self.fc1.bias.deinit(); + self.fc2.weight.deinit(); + self.fc2.bias.deinit(); + } + /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { - std.log.info("Compiling for target: {s}", .{@tagName(input.getContext().target())}); - var x = input.flattenAll().convert(.f32); + var x = input.flatten().convert(.f32).withTags(.{.d}); const layers: []const Layer = &.{ self.fc1, self.fc2 }; for (layers) |layer| { - x = zml.call(layer, .forward, .{x}); + x = layer.forward(x); } - return x.argMax(0, .u8).indices; + return x.argMax(0).indices.convert(.u8); } }; ``` - - -## Tagged Tensors - -```zig -const Sdpa = struct { - pub fn forward(_: Sdpa, ctx: *zml.Context, q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor) zml.Tensor { - const q = q_.withTags(.{ .b, .h, .q, .hd }); - const k = k_.withTags(.{ .b, .h, .k, .hd }); - const v = v_.withTags(.{ .b, .h, .k, .hd }); - const attn_mask = zml.nn.causalAttnMask(ctx, .{ .q = q.dim(.q), .k = k.dim(.k) }, q.dtype(), null); - return zml.nn.sdpa(ctx, q, k, v, .{ .attn_mask = attn_mask }); - } -}; -``` - - - - # Where to go next: You might want to check out more [examples](./examples), read through the -[documentation directly on GitHub](./docs/README.md), or, for the full rendering -experience, browse the -[online documentation with included API reference](https://docs.zml.ai). - - +[documentation directly on GitHub](./docs/README.md). # Contributing See [here][Contributing]. - - # License ZML is licensed under the [Apache 2.0 license](./LICENSE). - - # Thanks to our contributors From 0d0d5020cf13498d9b281fef65d74e24882b4de9 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 25 Mar 2026 10:16:26 +0100 Subject: [PATCH 386/881] example/llm: Further simplifications (#433) --- examples/llm/chat.zig | 15 ++--- examples/llm/main.zig | 1 - examples/llm/models.zig | 77 +++++++++++++++---------- examples/llm/models/lfm2/inference.zig | 2 +- examples/llm/models/lfm2/model.zig | 37 ------------ examples/llm/models/lfm2/session.zig | 51 +++++++++++++--- examples/llm/models/llama/model.zig | 37 ------------ examples/llm/models/llama/session.zig | 53 ++++++++++++++--- examples/llm/models/qwen3_5/model.zig | 41 ------------- examples/llm/models/qwen3_5/session.zig | 56 +++++++++++++++--- 10 files changed, 187 insertions(+), 183 deletions(-) diff --git a/examples/llm/chat.zig b/examples/llm/chat.zig index 3310f39d..0c12ee87 100644 --- a/examples/llm/chat.zig +++ b/examples/llm/chat.zig @@ -25,18 +25,15 @@ pub fn rememberPrompt(line: [*:0]const u8) void { pub const Chat = struct { allocator: std.mem.Allocator, io: std.Io, - tokenizer: zml.tokenizer.Tokenizer, - loaded_model: *models.LoadedModel, session: models.Session, tokens: std.ArrayList(u32), pub fn init( allocator: std.mem.Allocator, io: std.Io, - platform: *zml.Platform, + platform: *const zml.Platform, tokenizer: zml.tokenizer.Tokenizer, - loaded_model: *models.LoadedModel, - compiled_model: *models.CompiledModel, + compiled_model: *const models.CompiledModel, model_buffers: *models.Buffers, ) !Chat { var session = try compiled_model.newSession( @@ -51,8 +48,6 @@ pub const Chat = struct { return .{ .allocator = allocator, .io = io, - .tokenizer = tokenizer, - .loaded_model = loaded_model, .session = session, .tokens = try .initCapacity(allocator, session.maxTokens()), }; @@ -64,7 +59,7 @@ pub const Chat = struct { } pub fn runOnce(self: *Chat, prompt: []const u8) !void { - const prompt_tokens = try self.loaded_model.tokenizePrompt(self.allocator, self.tokenizer, prompt); + const prompt_tokens = try self.session.tokenizePrompt(self.allocator, prompt); defer self.allocator.free(prompt_tokens); var stdout = std.Io.File.stdout().writer(self.io, &.{}); @@ -81,7 +76,7 @@ pub const Chat = struct { } pub fn runInteractive(self: *Chat, initial_prompt: []const u8) !void { - var turn_tokens = try self.loaded_model.tokenizePrompt(self.allocator, self.tokenizer, initial_prompt); + var turn_tokens = try self.session.tokenizePrompt(self.allocator, initial_prompt); defer self.allocator.free(turn_tokens); var stdout = std.Io.File.stdout().writer(self.io, &.{}); @@ -148,7 +143,7 @@ pub const Chat = struct { if (input.len == 0) continue; self.allocator.free(previous_turn_tokens); - return try self.loaded_model.tokenizeTurn(self.allocator, self.tokenizer, input); + return try self.session.tokenizeTurn(self.allocator, input); } } }; diff --git a/examples/llm/main.zig b/examples/llm/main.zig index f86be31c..5134c66f 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -135,7 +135,6 @@ pub fn main(init: std.process.Init) !void { io, platform, tokenizer, - &model, &compiled_model, &model_buffers, ); diff --git a/examples/llm/models.zig b/examples/llm/models.zig index 6318f4fc..c5f3bfaa 100644 --- a/examples/llm/models.zig +++ b/examples/llm/models.zig @@ -68,18 +68,6 @@ pub const LoadedModel = union(ModelType) { } } - pub fn tokenizePrompt(self: *const LoadedModel, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]const u32 { - return switch (self.*) { - inline else => |*m| m.tokenizePrompt(allocator, tokenizer, prompt), - }; - } - - pub fn tokenizeTurn(self: *const LoadedModel, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]const u32 { - return switch (self.*) { - inline else => |*m| m.tokenizeTurn(allocator, tokenizer, prompt), - }; - } - pub fn compile( self: *const LoadedModel, allocator: std.mem.Allocator, @@ -90,7 +78,7 @@ pub const LoadedModel = union(ModelType) { seqlen: usize, progress: *std.Progress.Node, ) !CompiledModel { - return switch (self.*) { + const inner: CompiledModel.Inner = switch (self.*) { .lfm2 => |*m| .{ .lfm2 = try m.compile( allocator, io, @@ -109,18 +97,35 @@ pub const LoadedModel = union(ModelType) { seqlen, progress, ) }, - .qwen3_5 => |*m| .{ .qwen3_5 = try m.compile(allocator, io, platform, backend, shardings, seqlen, progress) }, + .qwen3_5 => |*m| .{ .qwen3_5 = try m.compile( + allocator, + io, + platform, + backend, + shardings, + seqlen, + progress, + ) }, + }; + return .{ + .inner = inner, + .seqlen = @intCast(seqlen), }; } }; -pub const CompiledModel = union(ModelType) { - lfm2: lfm2.inference.CompiledModel, - llama: llama.inference.CompiledModel, - qwen3_5: qwen3_5.inference.CompiledModel, +pub const CompiledModel = struct { + const Inner = union(ModelType) { + lfm2: lfm2.inference.CompiledModel, + llama: llama.inference.CompiledModel, + qwen3_5: qwen3_5.inference.CompiledModel, + }; + + inner: Inner, + seqlen: u32, pub fn deinit(self: *CompiledModel) void { - switch (self.*) { + switch (self.inner) { .lfm2 => |*b| b.deinit(), .llama => |*b| b.deinit(), .qwen3_5 => |*b| b.deinit(), @@ -128,49 +133,46 @@ pub const CompiledModel = union(ModelType) { } pub fn newSession( - self: *CompiledModel, + self: *const CompiledModel, allocator: std.mem.Allocator, io: std.Io, - platform: *zml.Platform, + platform: *const zml.Platform, model_buffers: *Buffers, tokenizer: zml.tokenizer.Tokenizer, ) !Session { - return switch (self.*) { + return switch (self.inner) { .lfm2 => |*compiled| .{ .inner = .{ .lfm2 = try lfm2.Session.init( allocator, io, platform, - &model_buffers.lfm2, tokenizer, - compiled.loaded_model.*, compiled, + &model_buffers.lfm2, ) }, - .seqlen = compiled.params.seqlen, + .seqlen = self.seqlen, }, .llama => |*compiled| .{ .inner = .{ .llama = try llama.Session.init( allocator, io, platform, - &model_buffers.llama, tokenizer, - compiled.loaded_model.parsed_config.value, compiled, + &model_buffers.llama, ) }, - .seqlen = @intCast(compiled.params.seqlen), + .seqlen = self.seqlen, }, .qwen3_5 => |*compiled| .{ .inner = .{ .qwen3_5 = try qwen3_5.Session.init( allocator, io, platform, - &model_buffers.qwen3_5, tokenizer, - compiled.loaded_model.inner, compiled, + &model_buffers.qwen3_5, ) }, - .seqlen = compiled.params.seqlen, + .seqlen = self.seqlen, }, }; } @@ -210,6 +212,18 @@ pub const Session = struct { }; } + pub fn tokenizePrompt(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + return switch (self.inner) { + inline else => |*s| s.tokenizePrompt(allocator, prompt), + }; + } + + pub fn tokenizeTurn(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + return switch (self.inner) { + inline else => |*s| s.tokenizeTurn(allocator, prompt), + }; + } + pub fn maxTokens(self: *const Session) u32 { return self.seqlen; } @@ -219,6 +233,5 @@ pub fn detectModelType(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Di const parsed = try parseConfig(RawConfig, allocator, io, repo); defer parsed.deinit(); if (std.meta.stringToEnum(ModelType, parsed.value.model_type)) |model_type| return model_type; - if (std.mem.eql(u8, parsed.value.model_type, "qwen3_next")) return .qwen3_5; return error.UnknownModelType; } diff --git a/examples/llm/models/lfm2/inference.zig b/examples/llm/models/lfm2/inference.zig index 91908f47..c75f22e4 100644 --- a/examples/llm/models/lfm2/inference.zig +++ b/examples/llm/models/lfm2/inference.zig @@ -57,7 +57,7 @@ pub const CompilationOptions = CompilationParameters; pub const Args = struct { allocator: std.mem.Allocator, io: std.Io, - platform: *zml.Platform, + platform: *const zml.Platform, model_buffers: *model.Buffers, tokens_buf: *zml.Buffer, tokens_pos_buf: *zml.Buffer, diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index ca3bf626..aff56bec 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -67,43 +67,6 @@ pub const LoadedModel = struct { self.parsed_config.deinit(); } - pub fn tokenizePrompt(self: *const LoadedModel, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]u32 { - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - const im_start = tokenizer.tokenToId("<|im_start|>") orelse return error.NoSuchToken; - const im_end = tokenizer.tokenToId("<|im_end|>") orelse return error.NoSuchToken; - const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; - const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; - const newline = (try encoder.encode("\n"))[0]; - - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); - try tokens.appendSlice(allocator, &.{ self.parsed_config.value.bos_token_id, im_start, user, newline }); - try tokens.appendSlice(allocator, try encoder.encode(prompt)); - try tokens.appendSlice(allocator, &.{ im_end, newline }); - try tokens.appendSlice(allocator, &.{ im_start, assistant, newline }); - return tokens.toOwnedSlice(allocator); - } - - pub fn tokenizeTurn(self: *const LoadedModel, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]u32 { - _ = self; - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - const im_start = tokenizer.tokenToId("<|im_start|>") orelse return error.NoSuchToken; - const im_end = tokenizer.tokenToId("<|im_end|>") orelse return error.NoSuchToken; - const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; - const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; - const newline = (try encoder.encode("\n"))[0]; - - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); - try tokens.appendSlice(allocator, &.{ im_end, newline, im_start, user, newline }); - try tokens.appendSlice(allocator, try encoder.encode(prompt)); - try tokens.appendSlice(allocator, &.{ im_end, newline }); - try tokens.appendSlice(allocator, &.{ im_start, assistant, newline }); - return tokens.toOwnedSlice(allocator); - } - pub fn loadBuffers( self: *const LoadedModel, allocator: std.mem.Allocator, diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig index a2506d8e..f0bd8b72 100644 --- a/examples/llm/models/lfm2/session.zig +++ b/examples/llm/models/lfm2/session.zig @@ -9,10 +9,10 @@ const model = @import("model.zig"); pub const Session = struct { allocator: std.mem.Allocator, io: std.Io, - platform: *zml.Platform, + platform: *const zml.Platform, model_buffers: *model.Buffers, - compiled_model: *inference.CompiledModel, - config: model.Config, + compiled_model: *const inference.CompiledModel, + config: *const model.Config, seqlen: u32, cache_buffers: zml.Bufferized(model.Cache), attention_metadata_buffers: zml.Bufferized(attention.Metadata), @@ -25,11 +25,10 @@ pub const Session = struct { pub fn init( allocator: std.mem.Allocator, io: std.Io, - platform: *zml.Platform, - model_buffers: *model.Buffers, + platform: *const zml.Platform, tokenizer: zml.tokenizer.Tokenizer, - loaded_model: model.LoadedModel, - compiled_model: *inference.CompiledModel, + compiled_model: *const inference.CompiledModel, + model_buffers: *model.Buffers, ) !Session { const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); return .{ @@ -39,7 +38,7 @@ pub const Session = struct { .model_buffers = model_buffers, .compiled_model = compiled_model, .tokenizer = tokenizer, - .config = loaded_model.parsed_config.value, + .config = &compiled_model.loaded_model.parsed_config.value, .seqlen = compiled_model.params.seqlen, .cache_buffers = try compiled_model.params.cache.initBuffers(allocator, io, platform, compiled_model.params.shardings.replicated), .attention_metadata_buffers = try compiled_model.params.attention_metadata.initBuffer(io, platform, compiled_model.params.shardings.model), @@ -57,6 +56,42 @@ pub const Session = struct { self.generated_token_slice.free(self.allocator); } + pub fn tokenizePrompt(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + var encoder = try self.tokenizer.encoder(); + defer encoder.deinit(); + + const im_start = self.tokenizer.tokenToId("<|im_start|>") orelse return error.NoSuchToken; + const im_end = self.tokenizer.tokenToId("<|im_end|>") orelse return error.NoSuchToken; + const user = self.tokenizer.tokenToId("user") orelse return error.NoSuchToken; + const assistant = self.tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; + const newline = (try encoder.encode("\n"))[0]; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); + try tokens.appendSlice(allocator, &.{ self.config.bos_token_id, im_start, user, newline }); + try tokens.appendSlice(allocator, try encoder.encode(prompt)); + try tokens.appendSlice(allocator, &.{ im_end, newline }); + try tokens.appendSlice(allocator, &.{ im_start, assistant, newline }); + return tokens.toOwnedSlice(allocator); + } + + pub fn tokenizeTurn(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + var encoder = try self.tokenizer.encoder(); + defer encoder.deinit(); + + const im_start = self.tokenizer.tokenToId("<|im_start|>") orelse return error.NoSuchToken; + const im_end = self.tokenizer.tokenToId("<|im_end|>") orelse return error.NoSuchToken; + const user = self.tokenizer.tokenToId("user") orelse return error.NoSuchToken; + const assistant = self.tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; + const newline = (try encoder.encode("\n"))[0]; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); + try tokens.appendSlice(allocator, &.{ im_end, newline, im_start, user, newline }); + try tokens.appendSlice(allocator, try encoder.encode(prompt)); + try tokens.appendSlice(allocator, &.{ im_end, newline }); + try tokens.appendSlice(allocator, &.{ im_start, assistant, newline }); + return tokens.toOwnedSlice(allocator); + } + pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { const tokens_slice: zml.Slice = try .alloc(self.allocator, .init(.{ .batch = 1, .seq = self.seqlen }, .u32)); defer tokens_slice.free(self.allocator); diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index 1d8011e5..c174fbb0 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -54,43 +54,6 @@ pub const LoadedModel = struct { self.parsed_config.deinit(); } - pub fn tokenizePrompt(self: *const LoadedModel, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]const u32 { - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - const start_header = tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; - const end_header = tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; - const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; - const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; - const eot = tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; - const newline = (try encoder.encode("\n"))[0]; - - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); - try tokens.appendSlice(allocator, &.{ self.parsed_config.value.bos_token_id, start_header, user, end_header, newline }); - try tokens.appendSlice(allocator, try encoder.encode(prompt)); - try tokens.appendSlice(allocator, &.{ eot, newline, start_header, assistant, end_header, newline }); - return tokens.toOwnedSlice(allocator); - } - - pub fn tokenizeTurn(self: *const LoadedModel, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]const u32 { - _ = self; - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - const start_header = tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; - const end_header = tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; - const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; - const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; - const eot = tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; - const newline = (try encoder.encode("\n"))[0]; - - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); - try tokens.appendSlice(allocator, &.{ eot, newline, start_header, user, end_header, newline }); - try tokens.appendSlice(allocator, try encoder.encode(prompt)); - try tokens.appendSlice(allocator, &.{ eot, newline, start_header, assistant, end_header, newline }); - return tokens.toOwnedSlice(allocator); - } - pub fn loadBuffers( self: *const LoadedModel, allocator: std.mem.Allocator, diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig index 2218cfbb..5bca14c3 100644 --- a/examples/llm/models/llama/session.zig +++ b/examples/llm/models/llama/session.zig @@ -8,25 +8,24 @@ const model = @import("model.zig"); pub const Session = struct { allocator: std.mem.Allocator, io: std.Io, - platform: *zml.Platform, + platform: *const zml.Platform, model_buffers: *model.Buffers, - compiled_model: *inference.CompiledModel, + compiled_model: *const inference.CompiledModel, kv_cache_buffers: zml.Bufferized(model.KvCache), attention_metadata_buffers: zml.Bufferized(zml.attention.attention.Metadata), rng_buffers: zml.Bufferized(zml.Tensor.Rng), generated_token_slice: zml.Slice, tokenizer: zml.tokenizer.Tokenizer, - config: model.Config, + config: *const model.Config, seqlen: u32, pub fn init( allocator: std.mem.Allocator, io: std.Io, - platform: *zml.Platform, - model_buffers: *model.Buffers, + platform: *const zml.Platform, tokenizer: zml.tokenizer.Tokenizer, - config: model.Config, - compiled_model: *inference.CompiledModel, + compiled_model: *const inference.CompiledModel, + model_buffers: *model.Buffers, ) !Session { const shardings = compiled_model.params.shardings; var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform, shardings.model); @@ -50,7 +49,7 @@ pub const Session = struct { .rng_buffers = rng_buffers, .generated_token_slice = try .alloc(allocator, zml.Shape.init(.{ .s = 1 }, .u32)), .tokenizer = tokenizer, - .config = config, + .config = &compiled_model.loaded_model.parsed_config.value, .seqlen = @intCast(compiled_model.params.seqlen), }; } @@ -62,6 +61,42 @@ pub const Session = struct { self.generated_token_slice.free(self.allocator); } + pub fn tokenizePrompt(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + var encoder = try self.tokenizer.encoder(); + defer encoder.deinit(); + + const start_header = self.tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; + const end_header = self.tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; + const user = self.tokenizer.tokenToId("user") orelse return error.NoSuchToken; + const assistant = self.tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; + const eot = self.tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; + const newline = (try encoder.encode("\n"))[0]; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); + try tokens.appendSlice(allocator, &.{ self.config.bos_token_id, start_header, user, end_header, newline }); + try tokens.appendSlice(allocator, try encoder.encode(prompt)); + try tokens.appendSlice(allocator, &.{ eot, newline, start_header, assistant, end_header, newline }); + return tokens.toOwnedSlice(allocator); + } + + pub fn tokenizeTurn(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + var encoder = try self.tokenizer.encoder(); + defer encoder.deinit(); + + const start_header = self.tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; + const end_header = self.tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; + const user = self.tokenizer.tokenToId("user") orelse return error.NoSuchToken; + const assistant = self.tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; + const eot = self.tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; + const newline = (try encoder.encode("\n"))[0]; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); + try tokens.appendSlice(allocator, &.{ eot, newline, start_header, user, end_header, newline }); + try tokens.appendSlice(allocator, try encoder.encode(prompt)); + try tokens.appendSlice(allocator, &.{ eot, newline, start_header, assistant, end_header, newline }); + return tokens.toOwnedSlice(allocator); + } + pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { var prefill_args = try self.compiled_model.prefill_exe.args(self.allocator); defer prefill_args.deinit(self.allocator); @@ -145,7 +180,7 @@ pub const Session = struct { } }; -fn isEosToken(config: model.Config, token_id: u32) bool { +fn isEosToken(config: *const model.Config, token_id: u32) bool { return switch (config.eos_token_id.value) { .int => |eos| token_id == eos, .ints => |eos_list| for (eos_list) |eos| { diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index ddf7fb75..0b467b60 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -68,14 +68,6 @@ pub const LoadedModel = struct { self.parsed_config.deinit(); } - pub fn tokenizePrompt(self: *const LoadedModel, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]const u32 { - return tokenizeChatPrompt(allocator, tokenizer, prompt, self.inner.special_tokens, true); - } - - pub fn tokenizeTurn(self: *const LoadedModel, allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]const u32 { - return tokenizeChatPrompt(allocator, tokenizer, prompt, self.inner.special_tokens, false); - } - pub fn loadBuffers( self: *const LoadedModel, allocator: std.mem.Allocator, @@ -349,39 +341,6 @@ pub const TransformerLayer = struct { } }; -fn tokenizeChatPrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8, special_tokens: Model.SpecialTokens, is_first_turn: bool) ![]u32 { - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - const im_start = tokenizer.tokenToId("<|im_start|>") orelse special_tokens.im_start_token_id; - const im_end = tokenizer.tokenToId("<|im_end|>") orelse special_tokens.im_end_token_id; - const think = tokenizer.tokenToId("") orelse return error.NoSuchToken; - const newline = try encodeSingleToken(&encoder, "\n"); - const user_prefix = try encoder.encode("user\n"); - const assistant_prefix = try encoder.encode("assistant\n"); - const encoded_prompt = try encoder.encode(prompt); - - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, encoded_prompt.len + user_prefix.len + assistant_prefix.len + 8); - if (!is_first_turn) { - try tokens.appendSlice(allocator, &.{ im_end, newline }); - } - - try tokens.append(allocator, im_start); - try tokens.appendSlice(allocator, user_prefix); - try tokens.appendSlice(allocator, encoded_prompt); - try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); - try tokens.appendSlice(allocator, assistant_prefix); - try tokens.appendSlice(allocator, &.{ think, newline }); - - return tokens.toOwnedSlice(allocator); -} - -fn encodeSingleToken(encoder: *zml.tokenizer.Tokenizer.Encoder, text: []const u8) !u32 { - const encoded = try encoder.encode(text); - if (encoded.len != 1) return error.InvalidTokenizerEncoding; - return encoded[0]; -} - pub const Mlp = struct { up_proj: zml.nn.Linear, gate_proj: zml.nn.Linear, diff --git a/examples/llm/models/qwen3_5/session.zig b/examples/llm/models/qwen3_5/session.zig index f7a35519..b9d8d8a9 100644 --- a/examples/llm/models/qwen3_5/session.zig +++ b/examples/llm/models/qwen3_5/session.zig @@ -8,9 +8,9 @@ const model = @import("model.zig"); pub const Session = struct { allocator: std.mem.Allocator, io: std.Io, - platform: *zml.Platform, + platform: *const zml.Platform, model_buffers: *model.Buffers, - compiled_model: *inference.CompiledModel, + compiled_model: *const inference.CompiledModel, kv_cache_buffers: zml.Bufferized(model.KvCache), rng_buffers: zml.Bufferized(zml.Tensor.Rng), tokenizer: zml.tokenizer.Tokenizer, @@ -18,17 +18,17 @@ pub const Session = struct { generated_token_slice: zml.Slice, seqlen: u32, eos_token_id: u32, + special_tokens: model.Model.SpecialTokens, think_start: ?u32, think_end: ?u32, pub fn init( allocator: std.mem.Allocator, io: std.Io, - platform: *zml.Platform, - model_buffers: *model.Buffers, + platform: *const zml.Platform, tokenizer: zml.tokenizer.Tokenizer, - mdl: model.Model, - compiled_model: *inference.CompiledModel, + compiled_model: *const inference.CompiledModel, + model_buffers: *model.Buffers, ) !Session { const replicated_sharding = compiled_model.params.shardings.replicated; var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform); @@ -50,7 +50,8 @@ pub const Session = struct { .step_token_slice = try .alloc(allocator, zml.Shape.init(.{ .b = 1, .s = 1 }, .u32)), .generated_token_slice = try .alloc(allocator, zml.Shape.init(.{ .b = 1, .s = 1 }, .u32)), .seqlen = compiled_model.params.seqlen, - .eos_token_id = mdl.special_tokens.end_of_text_token_id, + .eos_token_id = compiled_model.loaded_model.inner.special_tokens.end_of_text_token_id, + .special_tokens = compiled_model.loaded_model.inner.special_tokens, .think_start = tokenizer.tokenToId(""), .think_end = tokenizer.tokenToId(""), }; @@ -63,6 +64,14 @@ pub const Session = struct { self.generated_token_slice.free(self.allocator); } + pub fn tokenizePrompt(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + return tokenizeChatPrompt(allocator, self.tokenizer, prompt, self.special_tokens, true); + } + + pub fn tokenizeTurn(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + return tokenizeChatPrompt(allocator, self.tokenizer, prompt, self.special_tokens, false); + } + pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { const prefill_tokens_shape = zml.Shape.init(.{ .b = 1, .s = self.seqlen }, .u32); const prefill_tokens_slice = try zml.Slice.alloc(self.allocator, prefill_tokens_shape); @@ -156,3 +165,36 @@ pub const Session = struct { try token_buffer.toSlice(self.io, self.generated_token_slice); } }; + +fn tokenizeChatPrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8, special_tokens: model.Model.SpecialTokens, is_first_turn: bool) ![]const u32 { + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const im_start = tokenizer.tokenToId("<|im_start|>") orelse special_tokens.im_start_token_id; + const im_end = tokenizer.tokenToId("<|im_end|>") orelse special_tokens.im_end_token_id; + const think = tokenizer.tokenToId("") orelse return error.NoSuchToken; + const newline = try encodeSingleToken(&encoder, "\n"); + const user_prefix = try encoder.encode("user\n"); + const assistant_prefix = try encoder.encode("assistant\n"); + const encoded_prompt = try encoder.encode(prompt); + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, encoded_prompt.len + user_prefix.len + assistant_prefix.len + 8); + if (!is_first_turn) { + try tokens.appendSlice(allocator, &.{ im_end, newline }); + } + + try tokens.append(allocator, im_start); + try tokens.appendSlice(allocator, user_prefix); + try tokens.appendSlice(allocator, encoded_prompt); + try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); + try tokens.appendSlice(allocator, assistant_prefix); + try tokens.appendSlice(allocator, &.{ think, newline }); + + return tokens.toOwnedSlice(allocator); +} + +fn encodeSingleToken(encoder: *zml.tokenizer.Tokenizer.Encoder, text: []const u8) !u32 { + const encoded = try encoder.encode(text); + if (encoded.len != 1) return error.InvalidTokenizerEncoding; + return encoded[0]; +} From 22db5548eca8fb81b529eb5703e37162e86bf221 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 25 Mar 2026 13:55:06 +0100 Subject: [PATCH 387/881] zml/examples/llm: add sharding to Qwen 3.5 (#435) Using tensor parallelism strategy. From 86 toks/s to 119 tok/s for Qwen 3.5 on 2x5090 --- examples/llm/models/qwen3_5/inference.zig | 6 +- examples/llm/models/qwen3_5/model.zig | 152 ++++++++++++++-------- examples/llm/models/qwen3_5/session.zig | 2 +- 3 files changed, 99 insertions(+), 61 deletions(-) diff --git a/examples/llm/models/qwen3_5/inference.zig b/examples/llm/models/qwen3_5/inference.zig index e719acc1..37e6d84b 100644 --- a/examples/llm/models/qwen3_5/inference.zig +++ b/examples/llm/models/qwen3_5/inference.zig @@ -75,7 +75,7 @@ fn compileModel( const now: std.Io.Timestamp = .now(io, .awake); defer log.info("Compiled model [{f}]", .{now.untilNow(io, .awake)}); - const all_shardings = [_]zml.sharding.Sharding{parameters.shardings.replicated}; + const all_shardings = parameters.shardings.all(); var prefill_future = try io.concurrent(struct { fn call( @@ -84,7 +84,7 @@ fn compileModel( platform_: *const zml.Platform, qwen_model_: model.Model, parameters_: CompilationParameters, - shardings_: [1]zml.sharding.Sharding, + shardings_: [2]zml.sharding.Sharding, progress_: *std.Progress.Node, ) !zml.Exe { progress_.increaseEstimatedTotalItems(1); @@ -118,7 +118,7 @@ fn compileModel( platform_: *const zml.Platform, qwen_model_: model.Model, parameters_: CompilationParameters, - shardings_: [1]zml.sharding.Sharding, + shardings_: [2]zml.sharding.Sharding, progress_: *std.Progress.Node, ) !zml.Exe { progress_.increaseEstimatedTotalItems(1); diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 0b467b60..5d372c99 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -89,7 +89,7 @@ pub const LoadedModel = struct { log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); } - const all_shardings = [_]zml.sharding.Sharding{shardings.replicated}; + const all_shardings = shardings.all(); return zml.io.load(Model, &self.inner, allocator, io, platform, store, .{ .dma_chunks = 32, .dma_chunk_size = 128 * zml.MiB, @@ -151,7 +151,11 @@ pub const Model = struct { const lm_head_prefix = if (store.hasKey("lm_head.weight")) "lm_head" else "model.language_model.embed_tokens"; return .{ .text_model = try .init(allocator, store.withPrefix("model.language_model"), config), - .lm_head = .init(store.withPrefix(lm_head_prefix).createTensor("weight", .{ .dout, .d }, null), null, .d), + .lm_head = .init( + store.withPrefix(lm_head_prefix).createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), + null, + .d, + ), .config = config, .gen_options = gen_options, }; @@ -236,7 +240,7 @@ pub const TextModel = struct { } return .{ - .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight", .{ .voc, .d }, null) }, + .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight", .{ .voc, .d }, .{ .voc = .replicated, .d = .model }) }, .layers = layers, .norm = RmsNorm.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), }; @@ -313,8 +317,8 @@ pub const TransformerLayer = struct { token_index: zml.Tensor, kv_cache: KvCache.LayerView, ) struct { zml.Tensor, KvCache } { - const residual0 = x0; - const normalized_x0 = self.input_layernorm.forward(x0); + const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const normalized_x0 = self.input_layernorm.forward(x0_replicated); var attention_output: zml.Tensor = undefined; var updated_kv_cache: KvCache = kv_cache.parent; @@ -331,13 +335,12 @@ pub const TransformerLayer = struct { }, } - const x1 = attention_output.add(residual0); - const residual1 = x1; + const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); - const mlp_output = self.mlp.forward(normalized_hidden); + const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.{ .d = .replicated }); - return .{ mlp_output.add(residual1), updated_kv_cache }; + return .{ mlp_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), updated_kv_cache }; } }; @@ -348,9 +351,21 @@ pub const Mlp = struct { pub fn init(store: zml.io.TensorStore.View) Mlp { return .{ - .up_proj = .init(store.withPrefix("up_proj").createTensor("weight", .{ .dout, .d }, null), store.withPrefix("up_proj").maybeCreateTensor("bias", .{.dout}, null), .d), - .gate_proj = .init(store.withPrefix("gate_proj").createTensor("weight", .{ .dout, .d }, null), store.withPrefix("gate_proj").maybeCreateTensor("bias", .{.dout}, null), .d), - .down_proj = .init(store.withPrefix("down_proj").createTensor("weight", .{ .d, .dout }, null), store.withPrefix("down_proj").maybeCreateTensor("bias", .{.d}, null), .dout), + .up_proj = .init( + store.withPrefix("up_proj").createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), + store.withPrefix("up_proj").maybeCreateTensor("bias", .{.dout}, .{ .dout = .model }), + .d, + ), + .gate_proj = .init( + store.withPrefix("gate_proj").createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), + store.withPrefix("gate_proj").maybeCreateTensor("bias", .{.dout}, .{ .dout = .model }), + .d, + ), + .down_proj = .init( + store.withPrefix("down_proj").createTensor("weight", .{ .d, .dout }, .{ .d = .replicated, .dout = .model }), + store.withPrefix("down_proj").maybeCreateTensor("bias", .{.d}, .{ .d = .replicated }), + .dout, + ), }; } @@ -388,18 +403,22 @@ pub const SelfAttn = struct { rotary_embed: TextRotaryEmbedding, o_proj: zml.nn.Linear, - fn initProj(store: zml.io.TensorStore.View) zml.nn.Linear { - return .init(store.createTensor("weight", .{ .dout, .d }, null), store.maybeCreateTensor("bias", .{.dout}, null), .d); + fn initProj(store: zml.io.TensorStore.View, partitions: anytype, bias_partitions: anytype) zml.nn.Linear { + return .init( + store.createTensor("weight", .{ .dout, .d }, partitions), + store.maybeCreateTensor("bias", .{.dout}, bias_partitions), + .d, + ); } pub fn init(store: zml.io.TensorStore.View, config: Config) !SelfAttn { const rotary_dim: i64 = @intFromFloat(@as(f32, @floatFromInt(config.text_config.head_dim)) * config.text_config.rope_parameters.partial_rotary_factor); return .{ - .q_proj = initProj(store.withPrefix("q_proj")), - .k_proj = initProj(store.withPrefix("k_proj")), - .v_proj = initProj(store.withPrefix("v_proj")), - .o_proj = initProj(store.withPrefix("o_proj")), + .q_proj = initProj(store.withPrefix("q_proj"), .{ .dout = .model, .d = .replicated }, .{ .dout = .model }), + .k_proj = initProj(store.withPrefix("k_proj"), .{ .dout = .model, .d = .replicated }, .{ .dout = .model }), + .v_proj = initProj(store.withPrefix("v_proj"), .{ .dout = .model, .d = .replicated }, .{ .dout = .model }), + .o_proj = initProj(store.withPrefix("o_proj"), .{ .dout = .replicated, .d = .model }, .{ .dout = .replicated }), .q_norm = RmsNorm.init(store.withPrefix("q_norm"), config.text_config.rms_norm_eps), .k_norm = RmsNorm.init(store.withPrefix("k_norm"), config.text_config.rms_norm_eps), .num_heads = config.text_config.num_attention_heads, @@ -443,8 +462,14 @@ pub const SelfAttn = struct { token_index: zml.Tensor, kv_cache: KvCache.SelfAttnCache, ) struct { zml.Tensor, KvCache.SelfAttnCache } { - var q, const gate = self.projectQAndGate(x); - var k, var v = self.projectKV(x); + const x_qkv = x.withPartitioning(.{ .d = .replicated }); + + var q, var gate = self.projectQAndGate(x_qkv); + var k, var v = self.projectKV(x_qkv); + q = q.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); + gate = gate.withPartitioning(.{ .s = .replicated, .d_out_proj = .model }); + k = k.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); + v = v.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); q = self.q_norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); k = self.k_norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); @@ -456,22 +481,28 @@ pub const SelfAttn = struct { const cos, const sin = self.rotary_embed.getCosAndSin(position_ids, dtype); q = self.rotary_embed.applyRope(q, cos, sin); k = self.rotary_embed.applyRope(k, cos, sin); + q = q.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); + k = k.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); + v = v.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); const new_kv_cache = kv_cache.update(k, v, token_index.convert(.u32)); k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); + q = q.rename(.{ .s = .q }).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }); + k = k.rename(.{ .s = .k }).withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); + v = v.rename(.{ .s = .k }).withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); const attn_output = zml.attention.attention.attention( - q.rename(.{ .s = .q }), - k.rename(.{ .s = .k }), - v.rename(.{ .s = .k }), + q, + k, + v, token_index, zml.attention.attention.Metadata.init(.fromBackend(.vanilla, x.dim(.s), self.num_heads)), zml.attention.attention.Parameters.init(.fromBackend(.vanilla)), - ).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); + ).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); const gated_output = attn_output.mul(gate.sigmoid()); - const projected_output = self.o_proj.forward(gated_output.rename(.{ .d_out_proj = .d })).rename(.{ .dout = .d }); + const projected_output = self.o_proj.forward(gated_output.rename(.{ .d_out_proj = .d })).rename(.{ .dout = .d }).withPartitioning(.{ .d = .replicated }); return .{ projected_output, new_kv_cache }; } @@ -568,22 +599,22 @@ pub const GatedDeltaNet = struct { head_v_dim: i64, conv_kernel_size: i64, - fn initProj(store: zml.io.TensorStore.View) zml.nn.Linear { - return .init(store.createTensor("weight", .{ .dout, .d }, null), null, .d); + fn initProj(store: zml.io.TensorStore.View, partitions: anytype) zml.nn.Linear { + return .init(store.createTensor("weight", .{ .dout, .d }, partitions), null, .d); } pub fn init(store: zml.io.TensorStore.View, config: Config) GatedDeltaNet { const qk_head_repetition = @divExact(config.text_config.linear_num_value_heads, config.text_config.linear_num_key_heads); return .{ - .in_proj_qkv = initProj(store.withPrefix("in_proj_qkv")), - .in_proj_z = initProj(store.withPrefix("in_proj_z")), - .in_proj_b = initProj(store.withPrefix("in_proj_b")), - .in_proj_a = initProj(store.withPrefix("in_proj_a")), - .out_proj = initProj(store.withPrefix("out_proj")), - .conv1d_weight = store.withPrefix("conv1d").createTensor("weight", .{ .out, .in, .kernel_size }, null), - .dt_bias = store.createTensor("dt_bias", .{.vh}, null), - .aLog = store.createTensor("A_log", .{.vh}, null), + .in_proj_qkv = initProj(store.withPrefix("in_proj_qkv"), .{ .dout = .model, .d = .replicated }), + .in_proj_z = initProj(store.withPrefix("in_proj_z"), .{ .dout = .model, .d = .replicated }), + .in_proj_b = initProj(store.withPrefix("in_proj_b"), .{ .dout = .model, .d = .replicated }), + .in_proj_a = initProj(store.withPrefix("in_proj_a"), .{ .dout = .model, .d = .replicated }), + .out_proj = initProj(store.withPrefix("out_proj"), .{ .dout = .replicated, .d = .model }), + .conv1d_weight = store.withPrefix("conv1d").createTensor("weight", .{ .out, .in, .kernel_size }, .{ .out = .model, .in = .replicated, .kernel_size = .replicated }), + .dt_bias = store.createTensor("dt_bias", .{.vh}, .{ .vh = .model }), + .aLog = store.createTensor("A_log", .{.vh}, .{ .vh = .model }), .norm = RmsNormGated.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), .num_k_heads = config.text_config.linear_num_key_heads, .num_v_heads = config.text_config.linear_num_value_heads, @@ -660,7 +691,8 @@ pub const GatedDeltaNet = struct { const conv_dim = 2 * key_dim + value_dim; const left_pad = self.conv_kernel_size - 1; - const projected_qkv = self.in_proj_qkv.forward(x).rename(.{ .dout = .mix }); + const x_in = x.withPartitioning(.{ .d = .replicated }); + const projected_qkv = self.in_proj_qkv.forward(x_in).rename(.{ .dout = .mix }).withPartitioning(.{ .s = .replicated, .mix = .model }); const use_cached_state = x.dim(.s) == 1 and left_pad > 0; const conv_input = if (use_cached_state) zml.Tensor.concatenate(&.{ cache.convState(), projected_qkv }, .s) @@ -690,20 +722,24 @@ pub const GatedDeltaNet = struct { if (use_cached_state) { mixed_qkv = mixed_qkv.slice1d(.s, .{ .start = mixed_qkv.dim(.s) - 1, .end = mixed_qkv.dim(.s) }); } + mixed_qkv = mixed_qkv.withPartitioning(.{ .s = .replicated, .mix = .model }); - const z = self.in_proj_z.forward(x).splitAxis(.dout, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }); - const b = self.in_proj_b.forward(x).rename(.{ .dout = .vh }); - const a = self.in_proj_a.forward(x).rename(.{ .dout = .vh }); + const z = self.in_proj_z.forward(x_in).splitAxis(.dout, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }).withPartitioning(.{ .s = .replicated, .vh = .model, .vhd = .replicated }); + const b = self.in_proj_b.forward(x_in).rename(.{ .dout = .vh }).withPartitioning(.{ .s = .replicated, .vh = .model }); + const a = self.in_proj_a.forward(x_in).rename(.{ .dout = .vh }).withPartitioning(.{ .s = .replicated, .vh = .model }); const query = mixed_qkv .slice1d(.mix, .{ .start = 0, .end = key_dim }) - .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }); + .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }) + .withPartitioning(.{ .s = .replicated, .kh = .model, .khd = .replicated }); const key = mixed_qkv .slice1d(.mix, .{ .start = key_dim, .end = 2 * key_dim }) - .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }); + .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }) + .withPartitioning(.{ .s = .replicated, .kh = .model, .khd = .replicated }); const value = mixed_qkv .slice1d(.mix, .{ .start = 2 * key_dim, .end = 2 * key_dim + value_dim }) - .splitAxis(.mix, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }); + .splitAxis(.mix, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }) + .withPartitioning(.{ .s = .replicated, .vh = .model, .vhd = .replicated }); const beta = b.sigmoid(); const aLog_type = self.aLog.dtype(); @@ -726,9 +762,10 @@ pub const GatedDeltaNet = struct { core_attn_out.rename(.{ .vhd = .d }), z.rename(.{ .vhd = .d }), ) - .rename(.{ .d = .vhd }); + .rename(.{ .d = .vhd }) + .withPartitioning(.{ .s = .replicated, .vh = .model, .vhd = .replicated }); - const output = self.out_proj.forward(core_attn_out_normed.merge(.{ .d = .{ .vh, .vhd } })).rename(.{ .dout = .d }); + const output = self.out_proj.forward(core_attn_out_normed.merge(.{ .d = .{ .vh, .vhd } })).rename(.{ .dout = .d }).withPartitioning(.{ .d = .replicated }); const updated_cache = cache.update( buildUpdatedConvState(conv_input, left_pad), last_recurrent_state, @@ -742,7 +779,7 @@ pub const RmsNorm = struct { eps: f32 = 1e-6, pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { - return .{ .weight = store.createTensor("weight", .{.d}, null), .eps = eps }; + return .{ .weight = store.createTensor("weight", .{.d}, .{ .d = .replicated }), .eps = eps }; } pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { @@ -763,7 +800,7 @@ pub const RmsNormGated = struct { eps: f32 = 1e-6, pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNormGated { - return .{ .weight = store.createTensor("weight", .{.d}, null), .eps = eps }; + return .{ .weight = store.createTensor("weight", .{.d}, .{ .d = .replicated }), .eps = eps }; } pub fn unloadBuffers(self: *zml.Bufferized(RmsNormGated)) void { @@ -801,15 +838,15 @@ pub const KvCache = struct { .h = config.text_config.num_key_value_heads, .hd = config.text_config.head_dim, }, dtype); + const sharded_kv_shape = kv_shape.withPartitioning(.{ .h = .model }); return .{ - .k = .fromShape(kv_shape), - .v = .fromShape(kv_shape), + .k = .fromShape(sharded_kv_shape), + .v = .fromShape(sharded_kv_shape), .layer_index = .init(.{}, .u32), }; } - pub fn initBuffer(self: SelfAttnCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(SelfAttnCache) { - const sharding = try zml.sharding.replicatedSharding(platform); + pub fn initBuffer(self: SelfAttnCache, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !zml.Bufferized(SelfAttnCache) { return .{ .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), sharding, .{}), .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), sharding, .{}), @@ -901,15 +938,16 @@ pub const KvCache = struct { .khd = config.text_config.linear_key_head_dim, .vhd = config.text_config.linear_value_head_dim, }, recurrent_dtype); + const sharded_conv_state_shape = conv_state_shape.withPartitioning(.{ .mix = .model }); + const sharded_recurrent_state_shape = recurrent_state_shape.withPartitioning(.{ .vh = .model }); return .{ - .conv_state = .fromShape(conv_state_shape), - .recurrent_state = .fromShape(recurrent_state_shape), + .conv_state = .fromShape(sharded_conv_state_shape), + .recurrent_state = .fromShape(sharded_recurrent_state_shape), .layer_index = .init(.{}, .u32), }; } - pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(GatedDeltaNetCache) { - const sharding = try zml.sharding.replicatedSharding(platform); + pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !zml.Bufferized(GatedDeltaNetCache) { return .{ .conv_state = try zml.Buffer.uninitialized(io, platform, self.conv_state.shape(), sharding, .{}), .recurrent_state = try zml.Buffer.uninitialized(io, platform, self.recurrent_state.shape(), sharding, .{}), @@ -988,10 +1026,10 @@ pub const KvCache = struct { }; } - pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { + pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !zml.Bufferized(KvCache) { return .{ - .self_attn = try self.self_attn.initBuffer(io, platform), - .gated_delta_net = try self.gated_delta_net.initBuffer(io, platform), + .self_attn = try self.self_attn.initBuffer(io, platform, sharding), + .gated_delta_net = try self.gated_delta_net.initBuffer(io, platform, sharding), }; } diff --git a/examples/llm/models/qwen3_5/session.zig b/examples/llm/models/qwen3_5/session.zig index b9d8d8a9..acb8afd4 100644 --- a/examples/llm/models/qwen3_5/session.zig +++ b/examples/llm/models/qwen3_5/session.zig @@ -31,7 +31,7 @@ pub const Session = struct { model_buffers: *model.Buffers, ) !Session { const replicated_sharding = compiled_model.params.shardings.replicated; - var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform); + var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform, compiled_model.params.shardings.model); errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); From bba9230f0b8aef839b26de464150fe5497a26b3e Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 25 Mar 2026 14:01:03 +0100 Subject: [PATCH 388/881] ci: Increase CPU core count on runners (#434) ``` 4cpus: total: 2m56s ci-cpu: 2m14 ci-neuron: 2m07s ci-tpu: 1m58s 8cpus: total: 2m26s ci-cpu: 1m37s ci-neuron: 1m47s ci-tpu: 1m36s 16cpus (mi8-flex): total: 2m07s ci-cpu: 1m24s ci-neuron: 1m26s ci-tpu: 1m19s 16cpus (c8i-flex): total: 1m58s ci-cpu: 1m17s ci-neuron: 1m15s ci-tpu: 1m17s 32cpus: total: 1m48s ci-cpu: 1m05s ci-neuron: 1m06s ci-tpu: 1m02s ``` RunsOn tends to prefer m8i instance (memory-optimzed) rather than c8i ones for some reason, so forced the family with `c8+c7`, maybe we could just use `c8`. But not sure if it's worth being that selective. RunsOn selects the best there is. --- .github/workflows/ci.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c5238a56..b5b048f6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -24,7 +24,8 @@ jobs: run_tests: true runner_labels: - runs-on - - runner=4cpu-linux-x64 + - runner=16cpu-linux-x64 + - family=c8+c7 - image=ubuntu24-amd64 - run-id=${{ github.run_id }} - extras=s3-cache @@ -34,7 +35,8 @@ jobs: run_tests: false runner_labels: - runs-on - - runner=4cpu-linux-x64 + - runner=16cpu-linux-x64 + - family=c8+c7 - image=ubuntu24-amd64 - run-id=${{ github.run_id }} - extras=s3-cache @@ -44,7 +46,8 @@ jobs: run_tests: false runner_labels: - runs-on - - runner=4cpu-linux-x64 + - runner=16cpu-linux-x64 + - family=c8+c7 - image=ubuntu24-amd64 - run-id=${{ github.run_id }} - extras=s3-cache From 268ef6e0585fd7500e0bfa72bbd62722438b0115 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 25 Mar 2026 15:14:22 +0100 Subject: [PATCH 389/881] examples/llm: fix --help & relative/home paths (#437) --- examples/llm/main.zig | 16 +++++++++++++--- stdx/flags.zig | 10 +++++++++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/examples/llm/main.zig b/examples/llm/main.zig index 5134c66f..af12e540 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -21,22 +21,32 @@ const Args = struct { single: bool = false, pub const help = - \\Use llm --model= [options] + \\ Use llm --model= [options] \\ - \\ Run text generation with a model selected from `config.json.model_type`. + \\ Run text generation with a model selected from `model_type` in the `config.json`. \\ \\ Options: \\ --model= Path to the model repository (required) \\ --prompt= Prompt to use for generation (default: none) \\ --seqlen= Sequence length (default: 2048) \\ --backend= Attention backend to use ([vanilla, cuda_fa2, cuda_fa3], default: auto-selection) - \\ --single Create a single kernel encompassing all the layers when supported + \\ --single Create a single kernel encompassing all the layers when supported + \\ (only used by LFM2 which uses multiple kernels by default) \\ ; }; pub fn main(init: std.process.Init) !void { const allocator = init.gpa; + + // `bazel run` executes binaries from Bazel's runfiles tree by default. + // If available, switch back to the shell's original working directory. + if (init.environ_map.get("BUILD_WORKING_DIRECTORY")) |build_working_directory| { + var working_dir = try std.Io.Dir.openDirAbsolute(init.io, build_working_directory, .{}); + defer working_dir.close(init.io); + try std.process.setCurrentDir(init.io, working_dir); + } + const args = stdx.flags.parse(init.minimal.args, Args); // diff --git a/stdx/flags.zig b/stdx/flags.zig index 9a0f32ae..c23ae55d 100644 --- a/stdx/flags.zig +++ b/stdx/flags.zig @@ -104,7 +104,7 @@ fn parse_commands(args: *std.process.Args.Iterator, comptime Commands: type) Com // NB: help must be declared as *pub* const to be visible here. if (@hasDecl(Commands, "help")) { if (std.mem.eql(u8, first_arg, "-h") or std.mem.eql(u8, first_arg, "--help")) { - std.io.getStdOut().writeAll(Commands.help) catch std.process.exit(1); + std.debug.print(Commands.help, .{}); std.process.exit(0); } } @@ -212,6 +212,14 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { var parsed_positional = false; next_arg: while (args.next()) |arg| { + // NB: help must be declared as *pub* const to be visible here. + if (@hasDecl(Flags, "help")) { + if (std.mem.eql(u8, arg, "-h") or std.mem.eql(u8, arg, "--help")) { + std.debug.print(Flags.help, .{}); + std.process.exit(0); + } + } + comptime var field_len_prev = std.math.maxInt(usize); inline for (fields[0..field_count]) |field| { const flag = comptime flag_name(field); From 3828a571bc002292c1a2e7f2bf0db955128e5051 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Wed, 25 Mar 2026 18:05:15 +0000 Subject: [PATCH 390/881] platforms/neuron: minor neuron fixes --- platforms/neuron/libneuronxla.zig | 4 +--- platforms/neuron/neuronx-cc.zig | 7 ++++-- platforms/neuron/requirements.lock.txt | 30 +++++++++++++------------- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/platforms/neuron/libneuronxla.zig b/platforms/neuron/libneuronxla.zig index 58b70c1e..e11907cb 100644 --- a/platforms/neuron/libneuronxla.zig +++ b/platforms/neuron/libneuronxla.zig @@ -204,7 +204,7 @@ fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) "--framework=XLA", "--target", target, - "--verbose=user", + "--verbose=35", "--enable-internal-neff-wrapper", "--output", neff_file, @@ -223,8 +223,6 @@ fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) }); _ = try child.wait(io); - std.debug.print(">>>> {s}\n", .{tmp_dir}); - const neff_hlo_bytes = wrapNeffAsCustomCall(arena.allocator(), io, code, neff_file) catch |err| { log.err("Error wrapping NEFF as custom call: {}\n", .{err}); return err; diff --git a/platforms/neuron/neuronx-cc.zig b/platforms/neuron/neuronx-cc.zig index a0bbb526..d57094b7 100644 --- a/platforms/neuron/neuronx-cc.zig +++ b/platforms/neuron/neuronx-cc.zig @@ -90,8 +90,11 @@ pub fn main(init: std.process.Init) !void { defer c.Py_Finalize(); const neuronxcc_main = blk: { - const module = c.PyImport_ImportModule("neuronxcc.driver.CommandDriver"); - std.debug.print(">>> MODULE: {any}\n", .{module}); + const module = c.PyImport_ImportModule("neuronxcc.driver.CommandDriver") orelse { + std.log.err("Failed to import neuronxcc.driver.CommandDriver", .{}); + c.PyErr_Print(); + return error.FailedToImportModule; + }; defer c.Py_DecRef(module); break :blk c.PyObject_GetAttrString(module, "main"); }; diff --git a/platforms/neuron/requirements.lock.txt b/platforms/neuron/requirements.lock.txt index 4ff8e3aa..b84f682a 100644 --- a/platforms/neuron/requirements.lock.txt +++ b/platforms/neuron/requirements.lock.txt @@ -4,13 +4,13 @@ --extra-index-url https://pip.repos.neuron.amazonaws.com --find-links https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html -boto3==1.42.72 \ - --hash=sha256:2b5fdac4f202b2ccb9ed21f8b84229463b15573ea16941c2b1b8db1c69e08b63 \ - --hash=sha256:932f023ea3b54fd85df453dcfff7d8c5a0f8be144c0ad57568943505c72c1e6a +boto3==1.42.75 \ + --hash=sha256:16bc657d16403ee8e11c8b6920c245629e37a36ea60352b919da566f82b4cb4c \ + --hash=sha256:3c7fd95a50c69271bd7707b7eda07dcfddb30e961a392613010f7ee81d91acb3 # via libneuronxla -botocore==1.42.72 \ - --hash=sha256:038f34553da68df2ce70edc729f170cc198678daaad43f98649727c59f6404d4 \ - --hash=sha256:491b75eb41d46cf99cf8d6cab89f2e2c3602ce8e90d738a1da217a5fb6b2b7ef +botocore==1.42.75 \ + --hash=sha256:915e43b7ac8f50cf3dbc937ba713de5acb999ea48ad8fecd1589d92ad415f787 \ + --hash=sha256:95c8e716b6be903ee1601531caa4f50217400aa877c18fe9a2c3047d2945d477 # via # boto3 # libneuronxla @@ -350,15 +350,15 @@ pgzip==0.4.0 \ --hash=sha256:41e44c03a9f1bbbd390dd6b6e37a11484113f8c159a049fa81cd8a9ddde82a21 \ --hash=sha256:91f6526f0c6e1f6c2d3522707777e5843fad7c791401745593fd01bd869b7a05 # via neuronx-cc -protobuf==7.34.0 \ - --hash=sha256:3871a3df67c710aaf7bb8d214cc997342e63ceebd940c8c7fc65c9b3d697591a \ - --hash=sha256:4a72a8ec94e7a9f7ef7fe818ed26d073305f347f8b3b5ba31e22f81fd85fca02 \ - --hash=sha256:8e329966799f2c271d5e05e236459fe1cbfdb8755aaa3b0914fa60947ddea408 \ - --hash=sha256:964cf977e07f479c0697964e83deda72bcbc75c3badab506fb061b352d991b01 \ - --hash=sha256:9d7a5005fb96f3c1e64f397f91500b0eb371b28da81296ae73a6b08a5b76cdd6 \ - --hash=sha256:9f9079f1dde4e32342ecbd1c118d76367090d4aaa19da78230c38101c5b3dd40 \ - --hash=sha256:e3b914dd77fa33fa06ab2baa97937746ab25695f389869afdf03e81f34e45dc7 \ - --hash=sha256:f791ec509707a1d91bd02e07df157e75e4fb9fbdad12a81b7396201ec244e2e3 +protobuf==7.34.1 \ + --hash=sha256:34b84ce27680df7cca9f231043ada0daa55d0c44a2ddfaa58ec1d0d89d8bf60a \ + --hash=sha256:403b093a6e28a960372b44e5eb081775c9b056e816a8029c61231743d63f881a \ + --hash=sha256:5185e0e948d07abe94bb76ec9b8416b604cfe5da6f871d67aad30cbf24c3110b \ + --hash=sha256:8ff40ce8cd688f7265326b38d5a1bed9bfdf5e6723d49961432f83e21d5713e4 \ + --hash=sha256:9ce42245e704cc5027be797c1db1eb93184d44d1cdd71811fb2d9b25ad541280 \ + --hash=sha256:bb3812cd53aefea2b028ef42bd780f5b96407247f20c6ef7c679807e9d188f11 \ + --hash=sha256:d8b2cc79c4d8f62b293ad9b11ec3aebce9af481fa73e64556969f7345ebf9fc7 \ + --hash=sha256:e97b55646e6ce5cbb0954a8c28cd39a5869b59090dfaa7df4598a7fba869468c # via neuronx-cc psutil==7.2.2 \ --hash=sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372 \ From 127b195a5227691a38d6be4ab5e75b09b33e57cc Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Thu, 26 Mar 2026 15:47:47 +0100 Subject: [PATCH 391/881] workspace: minor README edits (#432) --- README.md | 128 +++++++++++++++++++++++++++--------------------------- 1 file changed, 64 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index e688a90f..528ce6b9 100644 --- a/README.md +++ b/README.md @@ -24,118 +24,108 @@ Compiled directly to NVIDIA, AMD, TPU, Trainium for peak hardware performance on It is built using the [Zig](https://ziglang.org) language, [MLIR](https://mlir.llvm.org), and [Bazel](https://bazel.build). -# Getting started +# Getting Started ## Prerequisites We use `bazel` to build ZML and its dependencies. The only prerequisite is -`bazel`, which we recommend to download through `bazelisk`, a version manager -for `bazel`. - -**Install Bazel** (recommended): +`bazel`, which we recommend installing through `bazelisk`. ### macOS -``` + +```bash brew install bazelisk ``` ### Linux -``` +```bash curl -L -o /usr/local/bin/bazel 'https://github.com/bazelbuild/bazelisk/releases/download/v1.28.0/bazelisk-linux-amd64' chmod +x /usr/local/bin/bazel ``` -## Run a pre-packaged model +## 30-Second Smoke Test -We have implemented a variety of example models in ZML. See our reference -implementations in the -[examples](https://github.com/zml/zml/tree/master/examples/) folder. +Run the MNIST example: -### MNIST - -The [classic](https://en.wikipedia.org/wiki/MNIST_database) handwritten digits -recognition task. The model is tasked to recognize a handwritten digit, which -has been converted to a 28x28 pixel monochrome image. `Bazel` will download a -pre-trained model, and the test dataset. The program will load the model, -compile it, and classify a randomly picked example from the test dataset. - -On the command line: - -``` +```bash bazel run //examples/mnist ``` -### Meta Llama 3.2 1B +This downloads a small pretrained MNIST model, compiles it, loads the weights, and +classifies a random handwritten digit. -This model has restrictions, see -[here](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct). It **requires -approval from Meta on Hugging Face**, which can take a few hours to get granted. +## LLM Quickstart -Ensure you are authenticated with the Hugging Face CLI: -``` -hf auth login +The main LLM example is [`//examples/llm`](./examples/llm). It currently supports: + +- Llama 3.1 / 3.2 +- Qwen 3.5 +- LFM 2.5 + +Authenticate with Hugging Face if you want to load gated repos such as Meta +Llama: + +```bash +bazel run //tools/hf -- auth login ``` + Alternatively, set the `HF_TOKEN` environment variable. -Now, you can run the model like so: -``` +Then run a prompt directly: + +```bash bazel run //examples/llm -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" ``` -For a larger 3.2 model, you can also try `Llama-3.2-3B-Instruct`. +Open the interactive chat loop by omitting `--prompt`: - -### Meta Llama 3.1 8B - -Like the 1B model above, this model also requires approval. See -[here](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) for access requirements. - - -``` -bazel run //examples/llm -- --model=hf://meta-llama/Llama-3.1-8B-Instruct --prompt="What is the capital of France?" +```bash +bazel run //examples/llm -- --model=hf://meta-llama/Llama-3.2-1B-Instruct ``` -You can also try `Llama-3.1-70B-Instruct` if you have enough memory. +You can also load from: + +- a local directory: `--model=/var/models/meta-llama/Llama-3.2-1B-Instruct` +- S3: `--model=s3://bucket/path/to/model` ## Running Models on GPU / TPU -You can compile models for accelerator runtimes by appending one or more of the -following arguments to the command line when compiling / running a model: +Append one or more platform flags when compiling or running: - NVIDIA CUDA: `--@zml//platforms:cuda=true` - AMD RoCM: `--@zml//platforms:rocm=true` - Google TPU: `--@zml//platforms:tpu=true` -- AWS Trainium/Inferentia 2: `--@zml//platforms:neuron=true` -- **AVOID CPU:** `--@zml//platforms:cpu=false` +- AWS Trainium / Inferentia 2: `--@zml//platforms:neuron=true` +- Disable CPU compilation: `--@zml//platforms:cpu=false` -The latter, avoiding compilation for CPU, cuts down compilation time. +Example on CUDA: -So, to run the Llama 3.1 8B model from above on your host supporting an NVIDIA GPU, -run the following: - -``` -bazel run //examples/llm --@zml//platforms:cuda=true -- --model=hf://meta-llama/Llama-3.1-8B-Instruct +```bash +bazel run //examples/llm --@zml//platforms:cuda=true -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="Write a haiku about Zig" ``` -And on your host supporting an AMD GPU: +Example on ROCm: +```bash +bazel run //examples/llm --@zml//platforms:rocm=true -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="Write a haiku about Zig" ``` -bazel run //examples/llm --@zml//platforms:rocm=true -- --model=hf://meta-llama/Llama-3.1-8B-Instruct -``` - -Same goes for all supported platforms. ## Run Tests -``` +```bash bazel test //zml:test ``` -# A taste of ZML +# Examples -## MNIST +- [`examples/llm`](./examples/llm): unified LLM CLI for Llama, Qwen, and LFM +- [`examples/mnist`](./examples/mnist): smallest end-to-end model run +- [`examples/sharding`](./examples/sharding): logical mesh, partitioners, shard-local execution, profiler output +- [`examples/io`](./examples/io): inspect and load local, `hf://`, `https://`, and `s3://` repositories through the VFS layer +- [`examples/benchmark`](./examples/benchmark): measure loading and execution performance +# A Taste Of ZML ```zig const Mnist = struct { @@ -200,10 +190,20 @@ const Mnist = struct { }; ``` -# Where to go next: +For a full walkthrough, see: -You might want to check out more [examples](./examples), read through the -[documentation directly on GitHub](./docs/README.md). +- [Getting Started](./docs/tutorials/getting_started.md) +- [Writing your first model](./docs/tutorials/write_first_model.md) +- [ZML Concepts](./docs/learn/concepts.md) +- [Deploying on a server](./docs/howtos/deploy_on_server.md) + +# Where To Go Next + +- Run more examples in [`./examples`](./examples) +- Read the example-specific notes in [`examples/llm/README.md`](./examples/llm/README.md) +- Learn tagged dimensions in [`working_with_tensors.md`](./docs/tutorials/working_with_tensors.md) +- Start building a model with [`write_first_model.md`](./docs/tutorials/write_first_model.md) +- Explore deployment in [`deploy_on_server.md`](./docs/howtos/deploy_on_server.md) # Contributing @@ -213,7 +213,7 @@ See [here][Contributing]. ZML is licensed under the [Apache 2.0 license](./LICENSE). -# Thanks to our contributors +# Thanks To Our Contributors From 9fa7768808b241e4e8714363d26cdacfe6f5df6d Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Thu, 26 Mar 2026 15:58:03 +0100 Subject: [PATCH 392/881] examples/llm: better single prompt UX + --topk (#438) - Same UX as the chat - Add `--topk` argument --- examples/llm/chat.zig | 8 ++------ examples/llm/main.zig | 10 +++++++++- examples/llm/models.zig | 15 +++++++++++---- examples/llm/models/common.zig | 4 ++++ examples/llm/models/lfm2/model.zig | 27 +++++++++++++++++++-------- examples/llm/models/lfm2_tests.zig | 2 +- examples/llm/models/llama/model.zig | 14 +++++++++----- examples/llm/models/llama_tests.zig | 2 +- examples/llm/models/qwen3_5/model.zig | 14 +++++++++----- 9 files changed, 65 insertions(+), 31 deletions(-) diff --git a/examples/llm/chat.zig b/examples/llm/chat.zig index 0c12ee87..6d1d311a 100644 --- a/examples/llm/chat.zig +++ b/examples/llm/chat.zig @@ -64,12 +64,8 @@ pub const Chat = struct { var stdout = std.Io.File.stdout().writer(self.io, &.{}); - log.info("Running prefill...", .{}); - try self.tokens.appendSlice(self.allocator, prompt_tokens); - try self.session.runPrefill(self.tokens.items); - - log.info("Running decode...", .{}); - try self.session.runDecode(&self.tokens, &stdout.interface); + try self.runPrefill(prompt_tokens, &stdout.interface); + try self.runDecodeTurn(&stdout.interface); try stdout.interface.writeAll("\n\n"); try stdout.interface.flush(); diff --git a/examples/llm/main.zig b/examples/llm/main.zig index af12e540..c058a3c6 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -17,6 +17,7 @@ const Args = struct { model: []const u8, prompt: ?[]const u8 = null, seqlen: u32 = 2048, + topk: u32 = 4, backend: ?zml.attention.attention.Backend = null, single: bool = false, @@ -29,6 +30,7 @@ const Args = struct { \\ --model= Path to the model repository (required) \\ --prompt= Prompt to use for generation (default: none) \\ --seqlen= Sequence length (default: 2048) + \\ --topk= Top-k sampling cutoff (default: 4) \\ --backend= Attention backend to use ([vanilla, cuda_fa2, cuda_fa3], default: auto-selection) \\ --single Create a single kernel encompassing all the layers when supported \\ (only used by LFM2 which uses multiple kernels by default) @@ -100,7 +102,13 @@ pub fn main(init: std.process.Init) !void { var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); defer store.deinit(); - var model = try models.LoadedModel.load(allocator, io, repo, store.view()); + const generation: models.GenerationOptions = .{ + .sampling_strategy = .{ + .topk = args.topk, + }, + }; + + var model = try models.LoadedModel.load(allocator, io, repo, store.view(), generation); defer model.deinit(allocator); // Defines how the model's tensors are sharded across the available devices. diff --git a/examples/llm/models.zig b/examples/llm/models.zig index c5f3bfaa..2c5c7e26 100644 --- a/examples/llm/models.zig +++ b/examples/llm/models.zig @@ -4,6 +4,7 @@ const zml = @import("zml"); const common = @import("models/common.zig"); pub const Shardings = common.Shardings; +pub const GenerationOptions = common.GenerationOptions; pub const parseConfig = common.parseConfig; pub const lfm2 = @import("models/lfm2.zig"); pub const llama = @import("models/llama.zig"); @@ -26,14 +27,20 @@ pub const LoadedModel = union(ModelType) { llama: llama.LoadedModel, qwen3_5: qwen3_5.LoadedModel, - pub fn load(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir, store: zml.io.TensorStore.View) !LoadedModel { + pub fn load( + allocator: std.mem.Allocator, + io: std.Io, + repo: std.Io.Dir, + store: zml.io.TensorStore.View, + generation: GenerationOptions, + ) !LoadedModel { const model_type = try detectModelType(allocator, io, repo); log.info("Detected model type: {}", .{model_type}); return switch (model_type) { - .lfm2 => .{ .lfm2 = try lfm2.LoadedModel.init(allocator, io, repo, store) }, - .llama => .{ .llama = try llama.LoadedModel.init(allocator, io, repo, store) }, - .qwen3_5 => .{ .qwen3_5 = try qwen3_5.LoadedModel.init(allocator, io, repo, store) }, + .lfm2 => .{ .lfm2 = try lfm2.LoadedModel.init(allocator, io, repo, store, generation) }, + .llama => .{ .llama = try llama.LoadedModel.init(allocator, io, repo, store, generation) }, + .qwen3_5 => .{ .qwen3_5 = try qwen3_5.LoadedModel.init(allocator, io, repo, store, generation) }, }; } diff --git a/examples/llm/models/common.zig b/examples/llm/models/common.zig index 971fbfb7..b5eb5c0b 100644 --- a/examples/llm/models/common.zig +++ b/examples/llm/models/common.zig @@ -8,6 +8,10 @@ pub const SessionOptions = struct { single: bool, }; +pub const GenerationOptions = struct { + sampling_strategy: zml.nn.SamplingStrategy = .{}, +}; + pub const Shardings = struct { replicated: zml.sharding.Sharding, model: zml.sharding.Sharding, diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index aff56bec..f7b009ed 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -52,12 +52,18 @@ pub const LoadedModel = struct { inner: Model, parsed_config: std.json.Parsed(Config), - pub fn init(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir, store: zml.io.TensorStore.View) !LoadedModel { + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + repo: std.Io.Dir, + store: zml.io.TensorStore.View, + generation: common.GenerationOptions, + ) !LoadedModel { const parsed_config = try common.parseConfig(Config, allocator, io, repo); errdefer parsed_config.deinit(); return .{ - .inner = .init(allocator, store, parsed_config.value), + .inner = .init(allocator, store, parsed_config.value, generation), .parsed_config = parsed_config, }; } @@ -128,7 +134,12 @@ pub const Model = struct { num_attention_layers: usize, num_conv_layers: usize, - pub fn init(allocator: std.mem.Allocator, root_store: zml.io.TensorStore.View, config: Config) Model { + pub fn init( + allocator: std.mem.Allocator, + root_store: zml.io.TensorStore.View, + config: Config, + generation: common.GenerationOptions, + ) Model { const store = root_store.withPrefix("model"); stdx.debug.assert(config.layer_types.len == config.num_hidden_layers, "Expected layer_types len {} to match num_hidden_layers {}", .{ config.layer_types.len, config.num_hidden_layers }); @@ -147,7 +158,7 @@ pub const Model = struct { return .{ .embed_tokens = .init(store.withPrefix("embed_tokens")), - .lm_head = LmHead.init(store, config), + .lm_head = LmHead.init(store, config, generation.sampling_strategy), .layers = layers, .num_attention_layers = num_attention_layers, .num_conv_layers = num_conv_layers, @@ -258,18 +269,18 @@ pub const TokenEmbedding = struct { pub const LmHead = struct { embedding_norm: RmsNorm, + sampling_strategy: zml.nn.SamplingStrategy, - pub fn init(store: zml.io.TensorStore.View, config: Config) LmHead { + pub fn init(store: zml.io.TensorStore.View, config: Config, sampling_strategy: zml.nn.SamplingStrategy) LmHead { return .{ .embedding_norm = RmsNorm.init(store.withPrefix("embedding_norm"), config.norm_eps, .d), + .sampling_strategy = sampling_strategy, }; } pub fn forward(self: LmHead, hidden: zml.Tensor, embed_tokens: TokenEmbedding, tokens: zml.Tensor, rng: zml.Tensor.Rng) struct { zml.Tensor, zml.Tensor.Rng } { const logits = embed_tokens.unembed(self.embedding_norm.forward(hidden)); - const new_tokens, const new_rng = zml.nn.sampleTokens(logits, .{ - .topk = 4, - }, rng); + const new_tokens, const new_rng = zml.nn.sampleTokens(logits, self.sampling_strategy, rng); return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), new_rng }; } diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig index a43d5d00..288fecd1 100644 --- a/examples/llm/models/lfm2_tests.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -43,7 +43,7 @@ pub fn main(init: std.process.Init) !void { var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); defer store.deinit(); - var repo_model = try lfm2.LoadedModel.init(allocator, io, repo, store.view()); + var repo_model = try lfm2.LoadedModel.init(allocator, io, repo, store.view(), .{}); defer repo_model.deinit(allocator); var progress = std.Progress.start(io, .{ .root_name = args.model }); diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index c174fbb0..1de9b509 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -34,12 +34,18 @@ pub const LoadedModel = struct { inner: Model, parsed_config: std.json.Parsed(Config), - pub fn init(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir, store: zml.io.TensorStore.View) !LoadedModel { + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + repo: std.Io.Dir, + store: zml.io.TensorStore.View, + generation: common.GenerationOptions, + ) !LoadedModel { const parsed_config = try common.parseConfig(Config, allocator, io, repo); errdefer parsed_config.deinit(); const options: Options = .{ - .sampling_strategy = .{}, + .sampling_strategy = generation.sampling_strategy, .max_seq_len = parsed_config.value.max_position_embeddings, }; @@ -115,9 +121,7 @@ pub const Model = struct { lm_head: ?zml.nn.Linear, model: Llama, - gen_opts: zml.nn.SamplingStrategy = .{ - .topk = 4, - }, + gen_opts: zml.nn.SamplingStrategy = .{}, config: Config, pub fn init( diff --git a/examples/llm/models/llama_tests.zig b/examples/llm/models/llama_tests.zig index 8ffeda7c..afec873d 100644 --- a/examples/llm/models/llama_tests.zig +++ b/examples/llm/models/llama_tests.zig @@ -40,7 +40,7 @@ pub fn main(init: std.process.Init) !void { var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); defer store.deinit(); - var repo_model = try llama.LoadedModel.init(allocator, io, repo, store.view()); + var repo_model = try llama.LoadedModel.init(allocator, io, repo, store.view(), .{}); defer repo_model.deinit(allocator); var progress = std.Progress.start(io, .{ .root_name = args.model }); diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 5d372c99..23d439df 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -48,12 +48,18 @@ pub const LoadedModel = struct { inner: Model, parsed_config: std.json.Parsed(Config), - pub fn init(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir, store: zml.io.TensorStore.View) !LoadedModel { + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + repo: std.Io.Dir, + store: zml.io.TensorStore.View, + generation: common.GenerationOptions, + ) !LoadedModel { const parsed_config = try common.parseConfig(Config, allocator, io, repo); errdefer parsed_config.deinit(); const options: Model.GenOptions = .{ - .sampling_strategy = .{}, + .sampling_strategy = generation.sampling_strategy, .max_seq_len = parsed_config.value.text_config.max_position_embeddings, }; @@ -125,9 +131,7 @@ pub const LoadedModel = struct { pub const Buffers = zml.Bufferized(Model); pub const Model = struct { - pub const GenOptions = struct { sampling_strategy: zml.nn.SamplingStrategy = .{ - .topk = 4, - }, max_seq_len: i64 }; + pub const GenOptions = struct { sampling_strategy: zml.nn.SamplingStrategy = .{}, max_seq_len: i64 }; pub const SpecialTokens = struct { im_start_token_id: u32, From 6e7cb90728c0040bf125a59f4046b16f6d7d04e6 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 26 Mar 2026 17:52:48 +0100 Subject: [PATCH 393/881] zml/platforms/rocm: update to ROCm 7.2.1 with full hermeticity (#439) Fix ROCm runtime hermeticity in the packaged LLM sandbox by patching transitive ROCm libraries to use zmlxrocm_dlopen and by adding missing libdw/liblzma/libbz2 runtime deps for distroless execution. Also update the ROCm packaging to 7.2.1 and refresh the pinned libpjrt_rocm --- MODULE.bazel | 16 +- MODULE.bazel.lock | 35 +- examples/llm/BUILD.bazel | 23 + platforms/rocm/libpjrt_rocm.BUILD.bazel | 17 + platforms/rocm/packages.lock.json | 1199 ++++++++++++++--------- platforms/rocm/packages.yaml | 8 +- platforms/rocm/rocm.bzl | 127 ++- 7 files changed, 929 insertions(+), 496 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 7bfc9b39..af0b018d 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -225,7 +225,21 @@ oci.pull( image = "gcr.io/distroless/cc-debian12", platforms = ["linux/amd64"], ) -use_repo(oci, "distroless_cc_debian12", "distroless_cc_debian12_linux_amd64") + +oci.pull( + name = "debian_12", + image = "docker.io/library/debian:12", + digest = "sha256:bc960ef50e6feed90686c593361df158517556ed1d2d98e5d1df3724024e0f49", + platforms = ["linux/amd64"], +) + +use_repo( + oci, + "distroless_cc_debian12", + "distroless_cc_debian12_linux_amd64", + "debian_12", + "debian_12_linux_amd64", +) git_override( module_name = "rules_zig", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index eeec762c..175ba7ea 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -426,7 +426,7 @@ "@@rules_oci+//oci:extensions.bzl%oci": { "general": { "bzlTransitiveDigest": "twSeB3VPTVkayY0uwqEzDzal5MbnZX45iC6jdheZQRo=", - "usagesDigest": "IIGPr69MT/mSUo6fNUffrIiU31EqGjlEYm5qFmNjmUc=", + "usagesDigest": "PFMCBQvV3Q+zC654PFY3m911buSFwM1rzl1zow5lOWE=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -460,6 +460,35 @@ "reproducible": true } }, + "debian_12_linux_amd64": { + "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_pull", + "attributes": { + "www_authenticate_challenges": {}, + "scheme": "https", + "registry": "index.docker.io", + "repository": "library/debian", + "identifier": "sha256:bc960ef50e6feed90686c593361df158517556ed1d2d98e5d1df3724024e0f49", + "platform": "linux/amd64", + "target_name": "debian_12_linux_amd64", + "bazel_tags": [] + } + }, + "debian_12": { + "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_alias", + "attributes": { + "target_name": "debian_12", + "www_authenticate_challenges": {}, + "scheme": "https", + "registry": "index.docker.io", + "repository": "library/debian", + "identifier": "sha256:bc960ef50e6feed90686c593361df158517556ed1d2d98e5d1df3724024e0f49", + "platforms": { + "@@platforms//cpu:x86_64": "@debian_12_linux_amd64" + }, + "bzlmod_repository": "debian_12", + "reproducible": true + } + }, "oci_crane_darwin_amd64": { "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", "attributes": { @@ -577,7 +606,9 @@ "moduleExtensionMetadata": { "explicitRootModuleDirectDeps": [ "distroless_cc_debian12", - "distroless_cc_debian12_linux_amd64" + "distroless_cc_debian12_linux_amd64", + "debian_12", + "debian_12_linux_amd64" ], "explicitRootModuleDirectDevDeps": [], "useAllRepos": "NO", diff --git a/examples/llm/BUILD.bazel b/examples/llm/BUILD.bazel index 34283122..d033d488 100644 --- a/examples/llm/BUILD.bazel +++ b/examples/llm/BUILD.bazel @@ -87,3 +87,26 @@ oci_push( repository = "index.docker.io/zmlai/llm", tags = ["manual"], ) + + +oci_image( + name = "image_debug_", + base = "@debian_12", + entrypoint = ["./examples/llm/llm"], + tars = [":archive"], + tags = ["manual"], +) + +platform_transition_filegroup( + name = "image_debug", + srcs = [":image_debug_"], + tags = ["manual"], + target_platform = "//platforms:linux_amd64", +) + +oci_load( + name = "load_debug", + image = ":image_debug", + repo_tags = ["zmlai/llm:debug"], + tags = ["manual"], +) diff --git a/platforms/rocm/libpjrt_rocm.BUILD.bazel b/platforms/rocm/libpjrt_rocm.BUILD.bazel index ee8200de..9e3c9c3c 100644 --- a/platforms/rocm/libpjrt_rocm.BUILD.bazel +++ b/platforms/rocm/libpjrt_rocm.BUILD.bazel @@ -41,7 +41,9 @@ copy_to_directory( "@libdrm-amdgpu-amdgpu1", "@libdrm-amdgpu-common//:amdgpu_ids", "@libdrm2-amdgpu", + "@libbz2-1.0", "@libelf1", + "@liblzma5", "@libnuma1", "@libtinfo6", "@libzstd1", @@ -62,13 +64,20 @@ copy_to_directory( "@roctracer", "@roctracer//:roctx", "@zlib1g", + "@libdw1", "@zml//platforms/rocm:zmlxrocm", ] + select({ ":_hipblaslt": ["@hipblaslt//:runfiles"], "//conditions:default": [], }), replace_prefixes = { + "MIOpen": "lib", "amdhip": "lib", + "amd_comgr": "lib", + "hiprtc_so": "lib", + "hipsolver": "lib", + "hsa-amd-aqlprofile": "lib", + "hsa-runtime": "lib", "hipblaslt": "lib", "lib/x86_64-linux-gnu": "lib", "libdrm-amdgpu-amdgpu1": "lib", @@ -76,8 +85,16 @@ copy_to_directory( "libpjrt_rocm_so": "lib", "opt/amdgpu/lib/x86_64-linux-gnu": "lib", "opt/amdgpu/share": "share", + "rccl": "lib", "rocblas": "lib", + "rocm-core": "lib", + "rocprofiler-register": "lib", + "rocprofiler-sdk_so": "lib", + "rocm_smi": "lib", + "rocsolver": "lib", "rocroller": "lib", + "roctracer": "lib", + "roctx": "lib", "platforms/rocm": "lib", "usr/lib/x86_64-linux-gnu": "lib", }, diff --git a/platforms/rocm/packages.lock.json b/platforms/rocm/packages.lock.json index a495ec2c..16763572 100755 --- a/platforms/rocm/packages.lock.json +++ b/platforms/rocm/packages.lock.json @@ -4,37 +4,219 @@ "arch": "amd64", "dependencies": [ { - "key": "amdgpu-core_1-7.2.70200-2278374.22.04_amd64", - "name": "amdgpu-core", - "version": "1:7.2.70200-2278374.22.04" + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" } ], - "key": "libdrm-amdgpu-common_1.0.0.70200-2278374.22.04_amd64", - "name": "libdrm-amdgpu-common", - "sha256": "baa11bc7e3f8e6931a69dea7f1485c2f982da184811ac7545342953ffb64abc4", + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "sha256": "3bfeaf4259eadbb7faa09feee86cd6cad172cd95907d7465afd0eb5aebb5433f", "urls": [ - "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/libd/libdrm-amdgpu-common/libdrm-amdgpu-common_1.0.0.70200-2278374.22.04_all.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/b/bzip2/libbz2-1.0_1.0.8-5build1_amd64.deb" ], - "version": "1.0.0.70200-2278374.22.04" + "version": "1.0.8-5build1" }, { "arch": "amd64", "dependencies": [], - "key": "amdgpu-core_1-7.2.70200-2278374.22.04_amd64", - "name": "amdgpu-core", - "sha256": "916fdee8e388c36a67abda89e0868e5789f4b77a3c351f945db4218fbcbe226c", + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "sha256": "d8e36ba6cc55719d728c38166bf753650710878e02e30af2aa79aefa244eb275", "urls": [ - "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/a/amdgpu-core/amdgpu-core_7.2.70200-2278374.22.04_all.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc6_2.35-0ubuntu3.10_amd64.deb" ], - "version": "1:7.2.70200-2278374.22.04" + "version": "2.35-0ubuntu3.10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "sha256": "3fa566e9f861a08736cbc5a97562d9d6e4f0c00450fbeafcb6d7583423b04a98", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libx/libxcrypt/libcrypt1_4.4.27-1_amd64.deb" + ], + "version": "1:4.4.27-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "sha256": "8ca7b573446d22ef46a97b4ec537c38acc8653f76391127253040acc7b74af5f", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libgcc-s1_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "sha256": "80a1f5df097af3978f7df2808006ca77197865e5eb7a923712aee6876e7a14cb", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/gcc-12-base_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libelf1_0.186-1ubuntu0.1_amd64", + "name": "libelf1", + "version": "0.186-1ubuntu0.1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + } + ], + "key": "libdw1_0.186-1ubuntu0.1_amd64", + "name": "libdw1", + "sha256": "487120e8e4468844a1bd024c3684e97dbe831fbaa04b975a337e8482370c4e17", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/e/elfutils/libdw1_0.186-1ubuntu0.1_amd64.deb" + ], + "version": "0.186-1ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "sha256": "9dc17e51a1be2d9ed63b7b84ef0e4e29c5abe6f1bc62cb03e7181483cce8a2f2", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/z/zlib/zlib1g_1.2.11.dfsg-2ubuntu9.2_amd64.deb" + ], + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + } + ], + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "sha256": "8f1c46e7d3f5102a5e4fdca7c949728a343ba71c2a7c124118df2c13d4c444f7", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/x/xz-utils/liblzma5_5.2.5-2ubuntu1_amd64.deb" + ], + "version": "5.2.5-2ubuntu1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libelf1_0.186-1ubuntu0.1_amd64", + "name": "libelf1", + "sha256": "2cc781614b045b450a1e073ee3b0902f393ee58ede1d2ea4526ad805fcccf538", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/e/elfutils/libelf1_0.186-1ubuntu0.1_amd64.deb" + ], + "version": "0.186-1ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "amdgpu-core_1-7.2.70201-2303469.22.04_amd64", + "name": "amdgpu-core", + "version": "1:7.2.70201-2303469.22.04" + } + ], + "key": "libdrm-amdgpu-common_1.0.0.70201-2303469.22.04_amd64", + "name": "libdrm-amdgpu-common", + "sha256": "288719d661cc1fbba776ec08d10053d745d53a35afe8cbfcf232828aa5a27091", + "urls": [ + "https://repo.radeon.com/graphics/7.2.1/ubuntu/pool/main/libd/libdrm-amdgpu-common/libdrm-amdgpu-common_1.0.0.70201-2303469.22.04_all.deb" + ], + "version": "1.0.0.70201-2303469.22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdgpu-core_1-7.2.70201-2303469.22.04_amd64", + "name": "amdgpu-core", + "sha256": "54af80759a376028bb442344cdcf39565063438d3708a40f125f02ccb2d8813e", + "urls": [ + "https://repo.radeon.com/graphics/7.2.1/ubuntu/pool/main/a/amdgpu-core/amdgpu-core_7.2.70201-2303469.22.04_all.deb" + ], + "version": "1:7.2.70201-2303469.22.04" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "rocprofiler-register_0.6.0.70201-81_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70200-43~22.04" + "version": "0.6.0.70201-81~22.04" }, { "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", @@ -62,9 +244,9 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "key": "rocm-core_7.2.1.70201-81_22.04_amd64", "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "version": "7.2.1.70201-81~22.04" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -272,33 +454,33 @@ "version": "2.0.14-3ubuntu2" }, { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70200-2278374.22.04_amd64", + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70201-2303469.22.04_amd64", "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.125.70200-2278374.22.04" + "version": "1:2.4.125.70201-2303469.22.04" }, { - "key": "libdrm-amdgpu-common_1.0.0.70200-2278374.22.04_amd64", + "key": "libdrm-amdgpu-common_1.0.0.70201-2303469.22.04_amd64", "name": "libdrm-amdgpu-common", - "version": "1.0.0.70200-2278374.22.04" + "version": "1.0.0.70201-2303469.22.04" }, { - "key": "amdgpu-core_1-7.2.70200-2278374.22.04_amd64", + "key": "amdgpu-core_1-7.2.70201-2303469.22.04_amd64", "name": "amdgpu-core", - "version": "1:7.2.70200-2278374.22.04" + "version": "1:7.2.70201-2303469.22.04" }, { - "key": "libdrm2-amdgpu_1-2.4.125.70200-2278374.22.04_amd64", + "key": "libdrm2-amdgpu_1-2.4.125.70201-2303469.22.04_amd64", "name": "libdrm2-amdgpu", - "version": "1:2.4.125.70200-2278374.22.04" + "version": "1:2.4.125.70201-2303469.22.04" } ], - "key": "hsa-rocr_1.18.0.70200-43_22.04_amd64", + "key": "hsa-rocr_1.18.0.70201-81_22.04_amd64", "name": "hsa-rocr", - "sha256": "c2ee2205b61a0c6b00478e70fa0cde4441fc9658cddcedb8bff07ab07569719c", + "sha256": "0cea908f2d05be0b6cb98556b429def4f114199ceb7903eca521ec310182ac41", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hsa-rocr/hsa-rocr_1.18.0.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/h/hsa-rocr/hsa-rocr_1.18.0.70201-81~22.04_amd64.deb" ], - "version": "1.18.0.70200-43~22.04" + "version": "1.18.0.70201-81~22.04" }, { "arch": "amd64", @@ -329,9 +511,9 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "key": "rocm-core_7.2.1.70201-81_22.04_amd64", "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "version": "7.2.1.70201-81~22.04" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -529,13 +711,13 @@ "version": "3.10.6-1~22.04.1" } ], - "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70201-81_22.04_amd64", "name": "rocprofiler-register", - "sha256": "144d516c55f368e82c231df469c1d3097ee4f44302ef3b71fe3eeded4199f5d3", + "sha256": "55bca0525ca02921240e5caf74ce3de0fc5791264e495bf026a49a4fedc0e1f3", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocprofiler-register/rocprofiler-register_0.6.0.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocprofiler-register/rocprofiler-register_0.6.0.70201-81~22.04_amd64.deb" ], - "version": "0.6.0.70200-43~22.04" + "version": "0.6.0.70201-81~22.04" }, { "arch": "amd64", @@ -551,57 +733,13 @@ { "arch": "amd64", "dependencies": [], - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "sha256": "8ca7b573446d22ef46a97b4ec537c38acc8653f76391127253040acc7b74af5f", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libgcc-s1_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "sha256": "d8e36ba6cc55719d728c38166bf753650710878e02e30af2aa79aefa244eb275", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc6_2.35-0ubuntu3.10_amd64.deb" - ], - "version": "2.35-0ubuntu3.10" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "sha256": "3fa566e9f861a08736cbc5a97562d9d6e4f0c00450fbeafcb6d7583423b04a98", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libx/libxcrypt/libcrypt1_4.4.27-1_amd64.deb" - ], - "version": "1:4.4.27-1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "sha256": "80a1f5df097af3978f7df2808006ca77197865e5eb7a923712aee6876e7a14cb", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/gcc-12-base_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "key": "rocm-core_7.2.1.70201-81_22.04_amd64", "name": "rocm-core", - "sha256": "9e8c61b3b260547f88b795f521d82249dfae50f9f82522c6ba9a4fad8fe10f0a", + "sha256": "b83fe314e67f6bd3cef2b5695fb6de7153110a14727add1992d0056d81a7e9a9", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocm-core/rocm-core_7.2.0.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocm-core/rocm-core_7.2.1.70201-81~22.04_amd64.deb" ], - "version": "7.2.0.70200-43~22.04" + "version": "7.2.1.70201-81~22.04" }, { "arch": "amd64", @@ -790,17 +928,6 @@ ], "version": "2.3.1-1" }, - { - "arch": "amd64", - "dependencies": [], - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "sha256": "9dc17e51a1be2d9ed63b7b84ef0e4e29c5abe6f1bc62cb03e7181483cce8a2f2", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/z/zlib/zlib1g_1.2.11.dfsg-2ubuntu9.2_amd64.deb" - ], - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, { "arch": "amd64", "dependencies": [], @@ -812,28 +939,6 @@ ], "version": "1.4.8+dfsg-3build1" }, - { - "arch": "amd64", - "dependencies": [], - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "sha256": "8f1c46e7d3f5102a5e4fdca7c949728a343ba71c2a7c124118df2c13d4c444f7", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/x/xz-utils/liblzma5_5.2.5-2ubuntu1_amd64.deb" - ], - "version": "5.2.5-2ubuntu1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "sha256": "3bfeaf4259eadbb7faa09feee86cd6cad172cd95907d7465afd0eb5aebb5433f", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/b/bzip2/libbz2-1.0_1.0.8-5build1_amd64.deb" - ], - "version": "1.0.8-5build1" - }, { "arch": "amd64", "dependencies": [], @@ -1032,17 +1137,6 @@ ], "version": "3.10.6-1~22.04.1" }, - { - "arch": "amd64", - "dependencies": [], - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "sha256": "2cc781614b045b450a1e073ee3b0902f393ee58ede1d2ea4526ad805fcccf538", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/e/elfutils/libelf1_0.186-1ubuntu0.1_amd64.deb" - ], - "version": "0.186-1ubuntu0.1" - }, { "arch": "amd64", "dependencies": [], @@ -1057,32 +1151,32 @@ { "arch": "amd64", "dependencies": [], - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70200-2278374.22.04_amd64", + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70201-2303469.22.04_amd64", "name": "libdrm-amdgpu-amdgpu1", - "sha256": "460f6dd499dbff0d4bc368d40b0c3f943dbf981df77f362ac05cce7befcf7779", + "sha256": "f8b455610e6603c2966ffed5e7530b2b2c1a6742fba99aab08f0930b4f638f58", "urls": [ - "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-amdgpu1_2.4.125.70200-2278374.22.04_amd64.deb" + "https://repo.radeon.com/graphics/7.2.1/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-amdgpu1_2.4.125.70201-2303469.22.04_amd64.deb" ], - "version": "1:2.4.125.70200-2278374.22.04" + "version": "1:2.4.125.70201-2303469.22.04" }, { "arch": "amd64", "dependencies": [], - "key": "libdrm2-amdgpu_1-2.4.125.70200-2278374.22.04_amd64", + "key": "libdrm2-amdgpu_1-2.4.125.70201-2303469.22.04_amd64", "name": "libdrm2-amdgpu", - "sha256": "a5a620541bcd852d6feb8299dda804c41cb979a3870cf82a783112b58ad9d7e4", + "sha256": "320439f513cada1bc308482a94eb07b7af04cd90a8163ad627983279c0be5ef3", "urls": [ - "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm2-amdgpu_2.4.125.70200-2278374.22.04_amd64.deb" + "https://repo.radeon.com/graphics/7.2.1/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm2-amdgpu_2.4.125.70201-2303469.22.04_amd64.deb" ], - "version": "1:2.4.125.70200-2278374.22.04" + "version": "1:2.4.125.70201-2303469.22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "key": "rocm-core_7.2.1.70201-81_22.04_amd64", "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "version": "7.2.1.70201-81~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -1305,34 +1399,53 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.30.13-2281980.22.04" + "key": "hsa-rocr_1.18.0.70201-81_22.04_amd64", + "name": "hsa-rocr", + "version": "1.18.0.70201-81~22.04" + }, + { + "key": "rocprofiler-register_0.6.0.70201-81_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70201-81~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", "name": "libelf1", "version": "0.186-1ubuntu0.1" + }, + { + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" + }, + { + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70201-2303469.22.04_amd64", + "name": "libdrm-amdgpu-amdgpu1", + "version": "1:2.4.125.70201-2303469.22.04" + }, + { + "key": "libdrm-amdgpu-common_1.0.0.70201-2303469.22.04_amd64", + "name": "libdrm-amdgpu-common", + "version": "1.0.0.70201-2303469.22.04" + }, + { + "key": "amdgpu-core_1-7.2.70201-2303469.22.04_amd64", + "name": "amdgpu-core", + "version": "1:7.2.70201-2303469.22.04" + }, + { + "key": "libdrm2-amdgpu_1-2.4.125.70201-2303469.22.04_amd64", + "name": "libdrm2-amdgpu", + "version": "1:2.4.125.70201-2303469.22.04" } ], - "key": "hsa-amd-aqlprofile_1.0.0.70200-43_22.04_amd64", + "key": "hsa-amd-aqlprofile_1.0.0.70201-81_22.04_amd64", "name": "hsa-amd-aqlprofile", - "sha256": "c4eabbbac5adb81d30442fe4172660591b4b11f70596035c0cd6b5dc80bd27be", + "sha256": "9a205f71abf149f59d8d8bac934f24a826a7eb0a7993d1159446aedc9e7aa364", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hsa-amd-aqlprofile/hsa-amd-aqlprofile_1.0.0.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/h/hsa-amd-aqlprofile/hsa-amd-aqlprofile_1.0.0.70201-81~22.04_amd64.deb" ], - "version": "1.0.0.70200-43~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "sha256": "a62cc2bbf9cfc46b263b17ff90bea66ae01267c03014b55a9764eab6895db4a7", - "urls": [ - "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/h/hsa-runtime-rocr4wsl-amdgpu/hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64.deb" - ], - "version": "25.30.13-2281980.22.04" + "version": "1.0.0.70201-81~22.04" }, { "arch": "amd64", @@ -1373,9 +1486,9 @@ "version": "1.4.8+dfsg-3build1" }, { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "key": "rocm-core_7.2.1.70201-81_22.04_amd64", "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "version": "7.2.1.70201-81~22.04" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -1618,13 +1731,13 @@ "version": "6.3-2ubuntu0.1" } ], - "key": "comgr_3.0.0.70200-43_22.04_amd64", + "key": "comgr_3.0.0.70201-81_22.04_amd64", "name": "comgr", - "sha256": "8f58b1958811430ba19426489608b3cbf685e5a78c1d6d9414c7f23cc8311a13", + "sha256": "096a51d616f01f431d8fd7716257ca602d811c393075300a4b947244333cdc50", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/c/comgr/comgr_3.0.0.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/c/comgr/comgr_3.0.0.70201-81~22.04_amd64.deb" ], - "version": "3.0.0.70200-43~22.04" + "version": "3.0.0.70201-81~22.04" }, { "arch": "amd64", @@ -1751,14 +1864,14 @@ "arch": "amd64", "dependencies": [ { - "key": "rocprofiler-sdk-rocpd_1.1.0-43_22.04_amd64", + "key": "rocprofiler-sdk-rocpd_1.1.0-81_22.04_amd64", "name": "rocprofiler-sdk-rocpd", - "version": "1.1.0-43~22.04" + "version": "1.1.0-81~22.04" }, { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "key": "rocm-core_7.2.1.70201-81_22.04_amd64", "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "version": "7.2.1.70201-81~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -1981,53 +2094,53 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "rocprofiler-sdk-roctx_1.1.0-43_22.04_amd64", + "key": "rocprofiler-sdk-roctx_1.1.0-81_22.04_amd64", "name": "rocprofiler-sdk-roctx", - "version": "1.1.0-43~22.04" + "version": "1.1.0-81~22.04" }, { - "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70201-81_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70200-43~22.04" + "version": "0.6.0.70201-81~22.04" } ], - "key": "rocprofiler-sdk_1.1.0-43_22.04_amd64", + "key": "rocprofiler-sdk_1.1.0-81_22.04_amd64", "name": "rocprofiler-sdk", - "sha256": "adaebb45a2b713fb7677cca10d50afbce33b025f6bbf5a84cf37707fc413127d", + "sha256": "03b11e5e48e2125b32676515e0191158109aa581fe4785c5edb4a3c0c30f3282", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocprofiler-sdk/rocprofiler-sdk_1.1.0-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocprofiler-sdk/rocprofiler-sdk_1.1.0-81~22.04_amd64.deb" ], - "version": "1.1.0-43~22.04" + "version": "1.1.0-81~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocprofiler-sdk-rocpd_1.1.0-43_22.04_amd64", + "key": "rocprofiler-sdk-rocpd_1.1.0-81_22.04_amd64", "name": "rocprofiler-sdk-rocpd", - "sha256": "2dbfad4c5c1e55bb65771bb3aa34460c52e6c7952767f26756d8aa7d98c47414", + "sha256": "1dfbc1887f4f7de7b78e3fd2959b53f54fa9f613763b8f4cf3c4c6f97b34b7c7", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocprofiler-sdk-rocpd/rocprofiler-sdk-rocpd_1.1.0-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocprofiler-sdk-rocpd/rocprofiler-sdk-rocpd_1.1.0-81~22.04_amd64.deb" ], - "version": "1.1.0-43~22.04" + "version": "1.1.0-81~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocprofiler-sdk-roctx_1.1.0-43_22.04_amd64", + "key": "rocprofiler-sdk-roctx_1.1.0-81_22.04_amd64", "name": "rocprofiler-sdk-roctx", - "sha256": "4c74438154cd91d81768c5ecbbed24859f787db638dcacce6b70445abf8577c7", + "sha256": "49ac3668842c82767a91128c0107af77a44b1c6c7e7c4c16248eac613f17ddcf", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocprofiler-sdk-roctx/rocprofiler-sdk-roctx_1.1.0-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocprofiler-sdk-roctx/rocprofiler-sdk-roctx_1.1.0-81~22.04_amd64.deb" ], - "version": "1.1.0-43~22.04" + "version": "1.1.0-81~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "key": "rocm-core_7.2.1.70201-81_22.04_amd64", "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "version": "7.2.1.70201-81~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -2250,29 +2363,29 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "rocrand_4.2.0.70200-43_22.04_amd64", + "key": "rocrand_4.2.0.70201-81_22.04_amd64", "name": "rocrand", - "version": "4.2.0.70200-43~22.04" + "version": "4.2.0.70201-81~22.04" }, { - "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", + "key": "hipblaslt_1.2.2.70201-81_22.04_amd64", "name": "hipblaslt", - "version": "1.2.1.70200-43~22.04" + "version": "1.2.2.70201-81~22.04" }, { - "key": "roctracer_4.1.70200.70200-43_22.04_amd64", + "key": "roctracer_4.1.70201.70201-81_22.04_amd64", "name": "roctracer", - "version": "4.1.70200.70200-43~22.04" + "version": "4.1.70201.70201-81~22.04" }, { - "key": "rocblas_5.2.0.70200-43_22.04_amd64", + "key": "rocblas_5.2.0.70201-81_22.04_amd64", "name": "rocblas", - "version": "5.2.0.70200-43~22.04" + "version": "5.2.0.70201-81~22.04" }, { - "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", + "key": "hip-runtime-amd_7.2.53211.70201-81_22.04_amd64", "name": "hip-runtime-amd", - "version": "7.2.26015.70200-43~22.04" + "version": "7.2.53211.70201-81~22.04" }, { "key": "libnuma1_2.0.14-3ubuntu2_amd64", @@ -2280,14 +2393,14 @@ "version": "2.0.14-3ubuntu2" }, { - "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70201-81_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70200-43~22.04" + "version": "0.6.0.70201-81~22.04" }, { - "key": "comgr_3.0.0.70200-43_22.04_amd64", + "key": "comgr_3.0.0.70201-81_22.04_amd64", "name": "comgr", - "version": "3.0.0.70200-43~22.04" + "version": "3.0.0.70201-81~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -2345,9 +2458,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70200-43_22.04_amd64", + "key": "rocminfo_1.0.0.70201-81_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.70200-43~22.04" + "version": "1.0.0.70201-81~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -2375,42 +2488,62 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.30.13-2281980.22.04" + "key": "hsa-rocr_1.18.0.70201-81_22.04_amd64", + "name": "hsa-rocr", + "version": "1.18.0.70201-81~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", "name": "libelf1", "version": "0.186-1ubuntu0.1" + }, + { + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70201-2303469.22.04_amd64", + "name": "libdrm-amdgpu-amdgpu1", + "version": "1:2.4.125.70201-2303469.22.04" + }, + { + "key": "libdrm-amdgpu-common_1.0.0.70201-2303469.22.04_amd64", + "name": "libdrm-amdgpu-common", + "version": "1.0.0.70201-2303469.22.04" + }, + { + "key": "amdgpu-core_1-7.2.70201-2303469.22.04_amd64", + "name": "amdgpu-core", + "version": "1:7.2.70201-2303469.22.04" + }, + { + "key": "libdrm2-amdgpu_1-2.4.125.70201-2303469.22.04_amd64", + "name": "libdrm2-amdgpu", + "version": "1:2.4.125.70201-2303469.22.04" } ], - "key": "miopen-hip_3.5.1.70200-43_22.04_amd64", + "key": "miopen-hip_3.5.1.70201-81_22.04_amd64", "name": "miopen-hip", - "sha256": "c9b6ea5b4f07aa6ee21f95f69ac3dd88ef87449e52c41d751b01b071ab121656", + "sha256": "6602376a310d0f8702acea8d0fa78138800060373f462bfd80994218ff87a674", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/m/miopen-hip/miopen-hip_3.5.1.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/m/miopen-hip/miopen-hip_3.5.1.70201-81~22.04_amd64.deb" ], - "version": "3.5.1.70200-43~22.04" + "version": "3.5.1.70201-81~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocrand_4.2.0.70200-43_22.04_amd64", + "key": "rocrand_4.2.0.70201-81_22.04_amd64", "name": "rocrand", - "sha256": "2b22a1cc2861eff622c6756b7cba160fb0b9211cbd9452b89b7d3ab3ce19e6c8", + "sha256": "6a772e0be70c346a62aa6be3420c7ae7e4e70ca7f472abdd9b4fd6b42eaf9297", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocrand/rocrand_4.2.0.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocrand/rocrand_4.2.0.70201-81~22.04_amd64.deb" ], - "version": "4.2.0.70200-43~22.04" + "version": "4.2.0.70201-81~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "key": "rocm-core_7.2.1.70201-81_22.04_amd64", "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "version": "7.2.1.70201-81~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -2633,26 +2766,26 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "roctracer_4.1.70200.70200-43_22.04_amd64", + "key": "roctracer_4.1.70201.70201-81_22.04_amd64", "name": "roctracer", - "version": "4.1.70200.70200-43~22.04" + "version": "4.1.70201.70201-81~22.04" } ], - "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", + "key": "hipblaslt_1.2.2.70201-81_22.04_amd64", "name": "hipblaslt", - "sha256": "a7baff52908f4a25b134d7e2613c1eb2b51b369509465d2a502b457f3ce3c1b2", + "sha256": "d44468114be65720eb0a82aa6009426de0022e345785a039caacc0cd63a0dc18", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hipblaslt/hipblaslt_1.2.1.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/h/hipblaslt/hipblaslt_1.2.2.70201-81~22.04_amd64.deb" ], - "version": "1.2.1.70200-43~22.04" + "version": "1.2.2.70201-81~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "key": "rocm-core_7.2.1.70201-81_22.04_amd64", "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "version": "7.2.1.70201-81~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -2875,21 +3008,21 @@ "version": "3.10.6-1~22.04.1" } ], - "key": "roctracer_4.1.70200.70200-43_22.04_amd64", + "key": "roctracer_4.1.70201.70201-81_22.04_amd64", "name": "roctracer", - "sha256": "4b40ae9c1981979f3fb69e9713c5b9fc1a17f868a83732f38f8fb5e53be3a612", + "sha256": "51c372d9a3e238e83b9725743f4fbc58a7dd93cb67b5b5adb72bf19148a12e26", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/roctracer/roctracer_4.1.70200.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/roctracer/roctracer_4.1.70201.70201-81~22.04_amd64.deb" ], - "version": "4.1.70200.70200-43~22.04" + "version": "4.1.70201.70201-81~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "key": "rocm-core_7.2.1.70201-81_22.04_amd64", "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "version": "7.2.1.70201-81~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -3112,9 +3245,9 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", + "key": "hip-runtime-amd_7.2.53211.70201-81_22.04_amd64", "name": "hip-runtime-amd", - "version": "7.2.26015.70200-43~22.04" + "version": "7.2.53211.70201-81~22.04" }, { "key": "libnuma1_2.0.14-3ubuntu2_amd64", @@ -3122,14 +3255,14 @@ "version": "2.0.14-3ubuntu2" }, { - "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70201-81_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70200-43~22.04" + "version": "0.6.0.70201-81~22.04" }, { - "key": "comgr_3.0.0.70200-43_22.04_amd64", + "key": "comgr_3.0.0.70201-81_22.04_amd64", "name": "comgr", - "version": "3.0.0.70200-43~22.04" + "version": "3.0.0.70201-81~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -3187,9 +3320,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70200-43_22.04_amd64", + "key": "rocminfo_1.0.0.70201-81_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.70200-43~22.04" + "version": "1.0.0.70201-81~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -3217,9 +3350,9 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.30.13-2281980.22.04" + "key": "hsa-rocr_1.18.0.70201-81_22.04_amd64", + "name": "hsa-rocr", + "version": "1.18.0.70201-81~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", @@ -3227,23 +3360,43 @@ "version": "0.186-1ubuntu0.1" }, { - "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", - "name": "hipblaslt", - "version": "1.2.1.70200-43~22.04" + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70201-2303469.22.04_amd64", + "name": "libdrm-amdgpu-amdgpu1", + "version": "1:2.4.125.70201-2303469.22.04" }, { - "key": "roctracer_4.1.70200.70200-43_22.04_amd64", + "key": "libdrm-amdgpu-common_1.0.0.70201-2303469.22.04_amd64", + "name": "libdrm-amdgpu-common", + "version": "1.0.0.70201-2303469.22.04" + }, + { + "key": "amdgpu-core_1-7.2.70201-2303469.22.04_amd64", + "name": "amdgpu-core", + "version": "1:7.2.70201-2303469.22.04" + }, + { + "key": "libdrm2-amdgpu_1-2.4.125.70201-2303469.22.04_amd64", + "name": "libdrm2-amdgpu", + "version": "1:2.4.125.70201-2303469.22.04" + }, + { + "key": "hipblaslt_1.2.2.70201-81_22.04_amd64", + "name": "hipblaslt", + "version": "1.2.2.70201-81~22.04" + }, + { + "key": "roctracer_4.1.70201.70201-81_22.04_amd64", "name": "roctracer", - "version": "4.1.70200.70200-43~22.04" + "version": "4.1.70201.70201-81~22.04" } ], - "key": "rocblas_5.2.0.70200-43_22.04_amd64", + "key": "rocblas_5.2.0.70201-81_22.04_amd64", "name": "rocblas", - "sha256": "0581ada80350b0d959e1863af4ff4be3ef31fe9838e941d09b8f0693dda7d23d", + "sha256": "835c40b7943ec98298fe889958a9e40ea70ed196a84cfb23c88e98967d30e057", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocblas/rocblas_5.2.0.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocblas/rocblas_5.2.0.70201-81~22.04_amd64.deb" ], - "version": "5.2.0.70200-43~22.04" + "version": "5.2.0.70201-81~22.04" }, { "arch": "amd64", @@ -3279,14 +3432,14 @@ "version": "2.0.14-3ubuntu2" }, { - "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70201-81_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70200-43~22.04" + "version": "0.6.0.70201-81~22.04" }, { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "key": "rocm-core_7.2.1.70201-81_22.04_amd64", "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "version": "7.2.1.70201-81~22.04" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -3484,9 +3637,9 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "comgr_3.0.0.70200-43_22.04_amd64", + "key": "comgr_3.0.0.70201-81_22.04_amd64", "name": "comgr", - "version": "3.0.0.70200-43~22.04" + "version": "3.0.0.70201-81~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -3544,9 +3697,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70200-43_22.04_amd64", + "key": "rocminfo_1.0.0.70201-81_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.70200-43~22.04" + "version": "1.0.0.70201-81~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -3574,34 +3727,54 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.30.13-2281980.22.04" + "key": "hsa-rocr_1.18.0.70201-81_22.04_amd64", + "name": "hsa-rocr", + "version": "1.18.0.70201-81~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", "name": "libelf1", "version": "0.186-1ubuntu0.1" + }, + { + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70201-2303469.22.04_amd64", + "name": "libdrm-amdgpu-amdgpu1", + "version": "1:2.4.125.70201-2303469.22.04" + }, + { + "key": "libdrm-amdgpu-common_1.0.0.70201-2303469.22.04_amd64", + "name": "libdrm-amdgpu-common", + "version": "1.0.0.70201-2303469.22.04" + }, + { + "key": "amdgpu-core_1-7.2.70201-2303469.22.04_amd64", + "name": "amdgpu-core", + "version": "1:7.2.70201-2303469.22.04" + }, + { + "key": "libdrm2-amdgpu_1-2.4.125.70201-2303469.22.04_amd64", + "name": "libdrm2-amdgpu", + "version": "1:2.4.125.70201-2303469.22.04" } ], - "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", + "key": "hip-runtime-amd_7.2.53211.70201-81_22.04_amd64", "name": "hip-runtime-amd", - "sha256": "97f477244025fb3bd8762d8d730d18545cafe080cb938f3f6520aeaff6e05c42", + "sha256": "2d265ee445042558e06318e445fe7aa0fbfd00e7795bd4d72ada4997b9a748c1", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hip-runtime-amd/hip-runtime-amd_7.2.26015.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/h/hip-runtime-amd/hip-runtime-amd_7.2.53211.70201-81~22.04_amd64.deb" ], - "version": "7.2.26015.70200-43~22.04" + "version": "7.2.53211.70201-81~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocminfo_1.0.0.70200-43_22.04_amd64", + "key": "rocminfo_1.0.0.70201-81_22.04_amd64", "name": "rocminfo", - "sha256": "6b8bcd3f5d0589a3b6447353c140b4ef676c0fbe319050a0f584fde4aa83f16f", + "sha256": "774cb4be2da5a7674e2056494fd3456e7c6ea74de50a81554ca38963c8dad317", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocminfo/rocminfo_1.0.0.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocminfo/rocminfo_1.0.0.70201-81~22.04_amd64.deb" ], - "version": "1.0.0.70200-43~22.04" + "version": "1.0.0.70201-81~22.04" }, { "arch": "amd64", @@ -3687,9 +3860,9 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "key": "rocm-core_7.2.1.70201-81_22.04_amd64", "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "version": "7.2.1.70201-81~22.04" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -3887,14 +4060,14 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "rocm-smi-lib_7.8.0.70200-43_22.04_amd64", + "key": "rocm-smi-lib_7.8.0.70201-81_22.04_amd64", "name": "rocm-smi-lib", - "version": "7.8.0.70200-43~22.04" + "version": "7.8.0.70201-81~22.04" }, { - "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", + "key": "hip-runtime-amd_7.2.53211.70201-81_22.04_amd64", "name": "hip-runtime-amd", - "version": "7.2.26015.70200-43~22.04" + "version": "7.2.53211.70201-81~22.04" }, { "key": "libnuma1_2.0.14-3ubuntu2_amd64", @@ -3902,14 +4075,14 @@ "version": "2.0.14-3ubuntu2" }, { - "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70201-81_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70200-43~22.04" + "version": "0.6.0.70201-81~22.04" }, { - "key": "comgr_3.0.0.70200-43_22.04_amd64", + "key": "comgr_3.0.0.70201-81_22.04_amd64", "name": "comgr", - "version": "3.0.0.70200-43~22.04" + "version": "3.0.0.70201-81~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -3967,9 +4140,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70200-43_22.04_amd64", + "key": "rocminfo_1.0.0.70201-81_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.70200-43~22.04" + "version": "1.0.0.70201-81~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -3997,42 +4170,62 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.30.13-2281980.22.04" + "key": "hsa-rocr_1.18.0.70201-81_22.04_amd64", + "name": "hsa-rocr", + "version": "1.18.0.70201-81~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", "name": "libelf1", "version": "0.186-1ubuntu0.1" + }, + { + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70201-2303469.22.04_amd64", + "name": "libdrm-amdgpu-amdgpu1", + "version": "1:2.4.125.70201-2303469.22.04" + }, + { + "key": "libdrm-amdgpu-common_1.0.0.70201-2303469.22.04_amd64", + "name": "libdrm-amdgpu-common", + "version": "1.0.0.70201-2303469.22.04" + }, + { + "key": "amdgpu-core_1-7.2.70201-2303469.22.04_amd64", + "name": "amdgpu-core", + "version": "1:7.2.70201-2303469.22.04" + }, + { + "key": "libdrm2-amdgpu_1-2.4.125.70201-2303469.22.04_amd64", + "name": "libdrm2-amdgpu", + "version": "1:2.4.125.70201-2303469.22.04" } ], - "key": "rccl_2.27.7.70200-43_22.04_amd64", + "key": "rccl_2.27.7.70201-81_22.04_amd64", "name": "rccl", - "sha256": "bb450a46ee36578a2b171b9af891f5e94ef04e9f743f75ec16e99246ddaff878", + "sha256": "cd84ed31a1bcce67d82ed4599d375b6e1e29b487313dba65297b856ce38ef8cf", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rccl/rccl_2.27.7.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rccl/rccl_2.27.7.70201-81~22.04_amd64.deb" ], - "version": "2.27.7.70200-43~22.04" + "version": "2.27.7.70201-81~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocm-smi-lib_7.8.0.70200-43_22.04_amd64", + "key": "rocm-smi-lib_7.8.0.70201-81_22.04_amd64", "name": "rocm-smi-lib", - "sha256": "bfd4ef1c3382eab7685c92c218ef14593ea1bd06836bb8cc903e4100e9125d79", + "sha256": "fc470d30b9e33ffc026ae829d7584e7fea04562a488e44c00a27b670d495829d", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocm-smi-lib/rocm-smi-lib_7.8.0.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocm-smi-lib/rocm-smi-lib_7.8.0.70201-81~22.04_amd64.deb" ], - "version": "7.8.0.70200-43~22.04" + "version": "7.8.0.70201-81~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "key": "rocm-core_7.2.1.70201-81_22.04_amd64", "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "version": "7.2.1.70201-81~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -4255,31 +4448,31 @@ "version": "3.10.6-1~22.04.1" } ], - "key": "rocm-device-libs_1.0.0.70200-43_22.04_amd64", + "key": "rocm-device-libs_1.0.0.70201-81_22.04_amd64", "name": "rocm-device-libs", - "sha256": "f1fe8f8fea614c3f734d002f6ae688929156bcfde0f27457412a7c3074d18d51", + "sha256": "86ecdcb81a4face9077f51b66b06f9cfc1ea15aa33043a6c0a3e771f8dec5916", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocm-device-libs/rocm-device-libs_1.0.0.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocm-device-libs/rocm-device-libs_1.0.0.70201-81~22.04_amd64.deb" ], - "version": "1.0.0.70200-43~22.04" + "version": "1.0.0.70201-81~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "hipcc_1.1.1.70200-43_22.04_amd64", + "key": "hipcc_1.1.1.70201-81_22.04_amd64", "name": "hipcc", - "version": "1.1.1.70200-43~22.04" + "version": "1.1.1.70201-81~22.04" }, { - "key": "rocm-llvm_22.0.0.26014.70200-43_22.04_amd64", + "key": "rocm-llvm_22.0.0.26084.70201-81_22.04_amd64", "name": "rocm-llvm", - "version": "22.0.0.26014.70200-43~22.04" + "version": "22.0.0.26084.70201-81~22.04" }, { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "key": "rocm-core_7.2.1.70201-81_22.04_amd64", "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "version": "7.2.1.70201-81~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -4592,69 +4785,69 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "hsa-rocr-dev_1.18.0.70200-43_22.04_amd64", + "key": "hsa-rocr-dev_1.18.0.70201-81_22.04_amd64", "name": "hsa-rocr-dev", - "version": "1.18.0.70200-43~22.04" + "version": "1.18.0.70201-81~22.04" }, { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.30.13-2281980.22.04" + "key": "hsa-rocr_1.18.0.70201-81_22.04_amd64", + "name": "hsa-rocr", + "version": "1.18.0.70201-81~22.04" + }, + { + "key": "rocprofiler-register_0.6.0.70201-81_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70201-81~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", "name": "libelf1", "version": "0.186-1ubuntu0.1" }, - { - "key": "libdrm-amdgpu-dev_1-2.4.125.70200-2278374.22.04_amd64", - "name": "libdrm-amdgpu-dev", - "version": "1:2.4.125.70200-2278374.22.04" - }, - { - "key": "libdrm-amdgpu-radeon1_1-2.4.125.70200-2278374.22.04_amd64", - "name": "libdrm-amdgpu-radeon1", - "version": "1:2.4.125.70200-2278374.22.04" - }, - { - "key": "libdrm2-amdgpu_1-2.4.125.70200-2278374.22.04_amd64", - "name": "libdrm2-amdgpu", - "version": "1:2.4.125.70200-2278374.22.04" - }, - { - "key": "amdgpu-core_1-7.2.70200-2278374.22.04_amd64", - "name": "amdgpu-core", - "version": "1:7.2.70200-2278374.22.04" - }, - { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70200-2278374.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.125.70200-2278374.22.04" - }, - { - "key": "libdrm-amdgpu-common_1.0.0.70200-2278374.22.04_amd64", - "name": "libdrm-amdgpu-common", - "version": "1.0.0.70200-2278374.22.04" - }, - { - "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", - "name": "hip-runtime-amd", - "version": "7.2.26015.70200-43~22.04" - }, { "key": "libnuma1_2.0.14-3ubuntu2_amd64", "name": "libnuma1", "version": "2.0.14-3ubuntu2" }, { - "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.6.0.70200-43~22.04" + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70201-2303469.22.04_amd64", + "name": "libdrm-amdgpu-amdgpu1", + "version": "1:2.4.125.70201-2303469.22.04" }, { - "key": "comgr_3.0.0.70200-43_22.04_amd64", + "key": "libdrm-amdgpu-common_1.0.0.70201-2303469.22.04_amd64", + "name": "libdrm-amdgpu-common", + "version": "1.0.0.70201-2303469.22.04" + }, + { + "key": "amdgpu-core_1-7.2.70201-2303469.22.04_amd64", + "name": "amdgpu-core", + "version": "1:7.2.70201-2303469.22.04" + }, + { + "key": "libdrm2-amdgpu_1-2.4.125.70201-2303469.22.04_amd64", + "name": "libdrm2-amdgpu", + "version": "1:2.4.125.70201-2303469.22.04" + }, + { + "key": "libdrm-amdgpu-dev_1-2.4.125.70201-2303469.22.04_amd64", + "name": "libdrm-amdgpu-dev", + "version": "1:2.4.125.70201-2303469.22.04" + }, + { + "key": "libdrm-amdgpu-radeon1_1-2.4.125.70201-2303469.22.04_amd64", + "name": "libdrm-amdgpu-radeon1", + "version": "1:2.4.125.70201-2303469.22.04" + }, + { + "key": "hip-runtime-amd_7.2.53211.70201-81_22.04_amd64", + "name": "hip-runtime-amd", + "version": "7.2.53211.70201-81~22.04" + }, + { + "key": "comgr_3.0.0.70201-81_22.04_amd64", "name": "comgr", - "version": "3.0.0.70200-43~22.04" + "version": "3.0.0.70201-81~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -4677,9 +4870,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70200-43_22.04_amd64", + "key": "rocminfo_1.0.0.70201-81_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.70200-43~22.04" + "version": "1.0.0.70201-81~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -4722,35 +4915,35 @@ "version": "1:5.41-3ubuntu0.1" } ], - "key": "hip-dev_7.2.26015.70200-43_22.04_amd64", + "key": "hip-dev_7.2.53211.70201-81_22.04_amd64", "name": "hip-dev", - "sha256": "19d62d6f520b4e521fe90246d9ae872b30528e490f80c6e265f83a583302f0ad", + "sha256": "5cc26232d8b7fd3ced173c07183be1d8c262beadce74fc44360e986d9f10dfc0", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hip-dev/hip-dev_7.2.26015.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/h/hip-dev/hip-dev_7.2.53211.70201-81~22.04_amd64.deb" ], - "version": "7.2.26015.70200-43~22.04" + "version": "7.2.53211.70201-81~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "hipcc_1.1.1.70200-43_22.04_amd64", + "key": "hipcc_1.1.1.70201-81_22.04_amd64", "name": "hipcc", - "sha256": "3c037af2114bd9058fb1420c00586c5d1c01a63d1d34500d8e61b39802ac570b", + "sha256": "fb047beabb6e93401d9179aadf26dbbb30df476f24525535c543d26e6c6f0327", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hipcc/hipcc_1.1.1.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/h/hipcc/hipcc_1.1.1.70201-81~22.04_amd64.deb" ], - "version": "1.1.1.70200-43~22.04" + "version": "1.1.1.70201-81~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocm-llvm_22.0.0.26014.70200-43_22.04_amd64", + "key": "rocm-llvm_22.0.0.26084.70201-81_22.04_amd64", "name": "rocm-llvm", - "sha256": "88d604fe0ab4a2502f0c7e0f8d3b7b49e829ae3057971de681a68b2fdb2505d2", + "sha256": "81dfdbac523318101e06e551c6a98aae55c595886d84de43e893dcf2ca41a3ba", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocm-llvm/rocm-llvm_22.0.0.26014.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocm-llvm/rocm-llvm_22.0.0.26084.70201-81~22.04_amd64.deb" ], - "version": "22.0.0.26014.70200-43~22.04" + "version": "22.0.0.26084.70201-81~22.04" }, { "arch": "amd64", @@ -4876,35 +5069,35 @@ { "arch": "amd64", "dependencies": [], - "key": "hsa-rocr-dev_1.18.0.70200-43_22.04_amd64", + "key": "hsa-rocr-dev_1.18.0.70201-81_22.04_amd64", "name": "hsa-rocr-dev", - "sha256": "71233ffaac8f28550c35bcf3e287a377a3c5c938a1e815e65d4155139cf4bdff", + "sha256": "95043500f730e6cb0e796274228f3f94deb0e36e7f80793584a2932e6ec707e5", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hsa-rocr-dev/hsa-rocr-dev_1.18.0.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/h/hsa-rocr-dev/hsa-rocr-dev_1.18.0.70201-81~22.04_amd64.deb" ], - "version": "1.18.0.70200-43~22.04" + "version": "1.18.0.70201-81~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "libdrm-amdgpu-dev_1-2.4.125.70200-2278374.22.04_amd64", + "key": "libdrm-amdgpu-dev_1-2.4.125.70201-2303469.22.04_amd64", "name": "libdrm-amdgpu-dev", - "sha256": "fec583602169ae32d3c9313dcb24c00732911234712495cbb6bb10f2194d2aee", + "sha256": "23b402f6a8915f401e26a3986429b2a64207477e5eec99293021e608ed812ea3", "urls": [ - "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-dev_2.4.125.70200-2278374.22.04_amd64.deb" + "https://repo.radeon.com/graphics/7.2.1/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-dev_2.4.125.70201-2303469.22.04_amd64.deb" ], - "version": "1:2.4.125.70200-2278374.22.04" + "version": "1:2.4.125.70201-2303469.22.04" }, { "arch": "amd64", "dependencies": [], - "key": "libdrm-amdgpu-radeon1_1-2.4.125.70200-2278374.22.04_amd64", + "key": "libdrm-amdgpu-radeon1_1-2.4.125.70201-2303469.22.04_amd64", "name": "libdrm-amdgpu-radeon1", - "sha256": "70049c919cff1b728107a03cd893b69106de8c00ac344e2d4019922d6bdbcd8b", + "sha256": "4b735739687e0796c3d967397ffead2d81499acab208f7e1709342b237657ef3", "urls": [ - "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-radeon1_2.4.125.70200-2278374.22.04_amd64.deb" + "https://repo.radeon.com/graphics/7.2.1/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-radeon1_2.4.125.70201-2303469.22.04_amd64.deb" ], - "version": "1:2.4.125.70200-2278374.22.04" + "version": "1:2.4.125.70201-2303469.22.04" }, { "arch": "amd64", @@ -4943,9 +5136,9 @@ "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "key": "rocm-core_7.2.1.70201-81_22.04_amd64", "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "version": "7.2.1.70201-81~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -5168,14 +5361,14 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "rocblas_5.2.0.70200-43_22.04_amd64", + "key": "rocblas_5.2.0.70201-81_22.04_amd64", "name": "rocblas", - "version": "5.2.0.70200-43~22.04" + "version": "5.2.0.70201-81~22.04" }, { - "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", + "key": "hip-runtime-amd_7.2.53211.70201-81_22.04_amd64", "name": "hip-runtime-amd", - "version": "7.2.26015.70200-43~22.04" + "version": "7.2.53211.70201-81~22.04" }, { "key": "libnuma1_2.0.14-3ubuntu2_amd64", @@ -5183,14 +5376,14 @@ "version": "2.0.14-3ubuntu2" }, { - "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70201-81_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70200-43~22.04" + "version": "0.6.0.70201-81~22.04" }, { - "key": "comgr_3.0.0.70200-43_22.04_amd64", + "key": "comgr_3.0.0.70201-81_22.04_amd64", "name": "comgr", - "version": "3.0.0.70200-43~22.04" + "version": "3.0.0.70201-81~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -5248,9 +5441,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70200-43_22.04_amd64", + "key": "rocminfo_1.0.0.70201-81_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.70200-43~22.04" + "version": "1.0.0.70201-81~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -5278,9 +5471,9 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.30.13-2281980.22.04" + "key": "hsa-rocr_1.18.0.70201-81_22.04_amd64", + "name": "hsa-rocr", + "version": "1.18.0.70201-81~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", @@ -5288,31 +5481,51 @@ "version": "0.186-1ubuntu0.1" }, { - "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", - "name": "hipblaslt", - "version": "1.2.1.70200-43~22.04" + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70201-2303469.22.04_amd64", + "name": "libdrm-amdgpu-amdgpu1", + "version": "1:2.4.125.70201-2303469.22.04" }, { - "key": "roctracer_4.1.70200.70200-43_22.04_amd64", + "key": "libdrm-amdgpu-common_1.0.0.70201-2303469.22.04_amd64", + "name": "libdrm-amdgpu-common", + "version": "1.0.0.70201-2303469.22.04" + }, + { + "key": "amdgpu-core_1-7.2.70201-2303469.22.04_amd64", + "name": "amdgpu-core", + "version": "1:7.2.70201-2303469.22.04" + }, + { + "key": "libdrm2-amdgpu_1-2.4.125.70201-2303469.22.04_amd64", + "name": "libdrm2-amdgpu", + "version": "1:2.4.125.70201-2303469.22.04" + }, + { + "key": "hipblaslt_1.2.2.70201-81_22.04_amd64", + "name": "hipblaslt", + "version": "1.2.2.70201-81~22.04" + }, + { + "key": "roctracer_4.1.70201.70201-81_22.04_amd64", "name": "roctracer", - "version": "4.1.70200.70200-43~22.04" + "version": "4.1.70201.70201-81~22.04" } ], - "key": "rocsolver_3.32.0.70200-43_22.04_amd64", + "key": "rocsolver_3.32.0.70201-81_22.04_amd64", "name": "rocsolver", - "sha256": "f18b57d3ebeba075b2b91f9e71d92da3efba73f34c5cff697cea1259b829547a", + "sha256": "6208660df71b8a5c55e2147c64bcaf65778288c1a9b24b25e1df9c1e9a071d24", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocsolver/rocsolver_3.32.0.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocsolver/rocsolver_3.32.0.70201-81~22.04_amd64.deb" ], - "version": "3.32.0.70200-43~22.04" + "version": "3.32.0.70201-81~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "key": "rocm-core_7.2.1.70201-81_22.04_amd64", "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "version": "7.2.1.70201-81~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -5535,19 +5748,19 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "rocsolver_3.32.0.70200-43_22.04_amd64", + "key": "rocsolver_3.32.0.70201-81_22.04_amd64", "name": "rocsolver", - "version": "3.32.0.70200-43~22.04" + "version": "3.32.0.70201-81~22.04" }, { - "key": "rocblas_5.2.0.70200-43_22.04_amd64", + "key": "rocblas_5.2.0.70201-81_22.04_amd64", "name": "rocblas", - "version": "5.2.0.70200-43~22.04" + "version": "5.2.0.70201-81~22.04" }, { - "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", + "key": "hip-runtime-amd_7.2.53211.70201-81_22.04_amd64", "name": "hip-runtime-amd", - "version": "7.2.26015.70200-43~22.04" + "version": "7.2.53211.70201-81~22.04" }, { "key": "libnuma1_2.0.14-3ubuntu2_amd64", @@ -5555,14 +5768,14 @@ "version": "2.0.14-3ubuntu2" }, { - "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70201-81_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70200-43~22.04" + "version": "0.6.0.70201-81~22.04" }, { - "key": "comgr_3.0.0.70200-43_22.04_amd64", + "key": "comgr_3.0.0.70201-81_22.04_amd64", "name": "comgr", - "version": "3.0.0.70200-43~22.04" + "version": "3.0.0.70201-81~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -5620,9 +5833,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70200-43_22.04_amd64", + "key": "rocminfo_1.0.0.70201-81_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.70200-43~22.04" + "version": "1.0.0.70201-81~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -5650,9 +5863,9 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.30.13-2281980.22.04" + "key": "hsa-rocr_1.18.0.70201-81_22.04_amd64", + "name": "hsa-rocr", + "version": "1.18.0.70201-81~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", @@ -5660,31 +5873,51 @@ "version": "0.186-1ubuntu0.1" }, { - "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", - "name": "hipblaslt", - "version": "1.2.1.70200-43~22.04" + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70201-2303469.22.04_amd64", + "name": "libdrm-amdgpu-amdgpu1", + "version": "1:2.4.125.70201-2303469.22.04" }, { - "key": "roctracer_4.1.70200.70200-43_22.04_amd64", + "key": "libdrm-amdgpu-common_1.0.0.70201-2303469.22.04_amd64", + "name": "libdrm-amdgpu-common", + "version": "1.0.0.70201-2303469.22.04" + }, + { + "key": "amdgpu-core_1-7.2.70201-2303469.22.04_amd64", + "name": "amdgpu-core", + "version": "1:7.2.70201-2303469.22.04" + }, + { + "key": "libdrm2-amdgpu_1-2.4.125.70201-2303469.22.04_amd64", + "name": "libdrm2-amdgpu", + "version": "1:2.4.125.70201-2303469.22.04" + }, + { + "key": "hipblaslt_1.2.2.70201-81_22.04_amd64", + "name": "hipblaslt", + "version": "1.2.2.70201-81~22.04" + }, + { + "key": "roctracer_4.1.70201.70201-81_22.04_amd64", "name": "roctracer", - "version": "4.1.70200.70200-43~22.04" + "version": "4.1.70201.70201-81~22.04" } ], - "key": "hipsolver_3.2.0.70200-43_22.04_amd64", + "key": "hipsolver_3.2.0.70201-81_22.04_amd64", "name": "hipsolver", - "sha256": "0fbff8007eada3d952db641c0653df26854854ee9e7058b4bd63202db66f2af8", + "sha256": "dc4f18451d0e2bd370e30e3ab75137429563a253deba56f44e955e5a903f0230", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hipsolver/hipsolver_3.2.0.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/h/hipsolver/hipsolver_3.2.0.70201-81~22.04_amd64.deb" ], - "version": "3.2.0.70200-43~22.04" + "version": "3.2.0.70201-81~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "key": "rocm-core_7.2.1.70201-81_22.04_amd64", "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "version": "7.2.1.70201-81~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -5907,42 +6140,42 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "rocfft_1.0.36.70200-43_22.04_amd64", + "key": "rocfft_1.0.36.70201-81_22.04_amd64", "name": "rocfft", - "version": "1.0.36.70200-43~22.04" + "version": "1.0.36.70201-81~22.04" } ], - "key": "hipfft_1.0.22.70200-43_22.04_amd64", + "key": "hipfft_1.0.22.70201-81_22.04_amd64", "name": "hipfft", - "sha256": "55a791b56dbeadcc3a1f1d0d770afc71a7fbb06aed4cd190b75ee27c19c18c61", + "sha256": "19cee2bfd969648b46bb4ee152be823631810ce1e099d008292067f7468aec32", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hipfft/hipfft_1.0.22.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/h/hipfft/hipfft_1.0.22.70201-81~22.04_amd64.deb" ], - "version": "1.0.22.70200-43~22.04" + "version": "1.0.22.70201-81~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocfft_1.0.36.70200-43_22.04_amd64", + "key": "rocfft_1.0.36.70201-81_22.04_amd64", "name": "rocfft", - "sha256": "8dbfef90c0055ef0d572451cb4a7a3380c3394485ed33cab4c116283d2fea7fe", + "sha256": "4866b772d074f7f01e6fb882bef28667baa5d634ef902e95341087864d5e53e4", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocfft/rocfft_1.0.36.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocfft/rocfft_1.0.36.70201-81~22.04_amd64.deb" ], - "version": "1.0.36.70200-43~22.04" + "version": "1.0.36.70201-81~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", + "key": "hipblaslt_1.2.2.70201-81_22.04_amd64", "name": "hipblaslt", - "version": "1.2.1.70200-43~22.04" + "version": "1.2.2.70201-81~22.04" }, { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "key": "rocm-core_7.2.1.70201-81_22.04_amd64", "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "version": "7.2.1.70201-81~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -6165,42 +6398,42 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "roctracer_4.1.70200.70200-43_22.04_amd64", + "key": "roctracer_4.1.70201.70201-81_22.04_amd64", "name": "roctracer", - "version": "4.1.70200.70200-43~22.04" + "version": "4.1.70201.70201-81~22.04" }, { - "key": "hipblas-common-dev_1.4.0.70200-43_22.04_amd64", + "key": "hipblas-common-dev_1.4.0.70201-81_22.04_amd64", "name": "hipblas-common-dev", - "version": "1.4.0.70200-43~22.04" + "version": "1.4.0.70201-81~22.04" } ], - "key": "hipblaslt-dev_1.2.1.70200-43_22.04_amd64", + "key": "hipblaslt-dev_1.2.2.70201-81_22.04_amd64", "name": "hipblaslt-dev", - "sha256": "7bb01f62e51be15495ea4a124db7b6c4e600d27d401ce636fb9b3daaf6b4b958", + "sha256": "b6b90b4b9576ad8cf615528ab468537351eadb53576abf0e06a7319d0cbab2d0", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hipblaslt-dev/hipblaslt-dev_1.2.1.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/h/hipblaslt-dev/hipblaslt-dev_1.2.2.70201-81~22.04_amd64.deb" ], - "version": "1.2.1.70200-43~22.04" + "version": "1.2.2.70201-81~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "hipblas-common-dev_1.4.0.70200-43_22.04_amd64", + "key": "hipblas-common-dev_1.4.0.70201-81_22.04_amd64", "name": "hipblas-common-dev", - "sha256": "1eed050f8367b58d697db452c3928780ae408ee055694f1676326d54f4caa739", + "sha256": "400298c925c13791860486014e2c9215681be640e525945fc2da7e9eafa6fbfa", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hipblas-common-dev/hipblas-common-dev_1.4.0.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/h/hipblas-common-dev/hipblas-common-dev_1.4.0.70201-81~22.04_amd64.deb" ], - "version": "1.4.0.70200-43~22.04" + "version": "1.4.0.70201-81~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "key": "rocm-core_7.2.1.70201-81_22.04_amd64", "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "version": "7.2.1.70201-81~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -6423,14 +6656,14 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "rocblas_5.2.0.70200-43_22.04_amd64", + "key": "rocblas_5.2.0.70201-81_22.04_amd64", "name": "rocblas", - "version": "5.2.0.70200-43~22.04" + "version": "5.2.0.70201-81~22.04" }, { - "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", + "key": "hip-runtime-amd_7.2.53211.70201-81_22.04_amd64", "name": "hip-runtime-amd", - "version": "7.2.26015.70200-43~22.04" + "version": "7.2.53211.70201-81~22.04" }, { "key": "libnuma1_2.0.14-3ubuntu2_amd64", @@ -6438,14 +6671,14 @@ "version": "2.0.14-3ubuntu2" }, { - "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70201-81_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70200-43~22.04" + "version": "0.6.0.70201-81~22.04" }, { - "key": "comgr_3.0.0.70200-43_22.04_amd64", + "key": "comgr_3.0.0.70201-81_22.04_amd64", "name": "comgr", - "version": "3.0.0.70200-43~22.04" + "version": "3.0.0.70201-81~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -6503,9 +6736,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70200-43_22.04_amd64", + "key": "rocminfo_1.0.0.70201-81_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.70200-43~22.04" + "version": "1.0.0.70201-81~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -6533,9 +6766,9 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.30.13-2281980.22.04" + "key": "hsa-rocr_1.18.0.70201-81_22.04_amd64", + "name": "hsa-rocr", + "version": "1.18.0.70201-81~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", @@ -6543,23 +6776,43 @@ "version": "0.186-1ubuntu0.1" }, { - "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", - "name": "hipblaslt", - "version": "1.2.1.70200-43~22.04" + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70201-2303469.22.04_amd64", + "name": "libdrm-amdgpu-amdgpu1", + "version": "1:2.4.125.70201-2303469.22.04" }, { - "key": "roctracer_4.1.70200.70200-43_22.04_amd64", + "key": "libdrm-amdgpu-common_1.0.0.70201-2303469.22.04_amd64", + "name": "libdrm-amdgpu-common", + "version": "1.0.0.70201-2303469.22.04" + }, + { + "key": "amdgpu-core_1-7.2.70201-2303469.22.04_amd64", + "name": "amdgpu-core", + "version": "1:7.2.70201-2303469.22.04" + }, + { + "key": "libdrm2-amdgpu_1-2.4.125.70201-2303469.22.04_amd64", + "name": "libdrm2-amdgpu", + "version": "1:2.4.125.70201-2303469.22.04" + }, + { + "key": "hipblaslt_1.2.2.70201-81_22.04_amd64", + "name": "hipblaslt", + "version": "1.2.2.70201-81~22.04" + }, + { + "key": "roctracer_4.1.70201.70201-81_22.04_amd64", "name": "roctracer", - "version": "4.1.70200.70200-43~22.04" + "version": "4.1.70201.70201-81~22.04" } ], - "key": "rocsparse_4.2.0.70200-43_22.04_amd64", + "key": "rocsparse_4.2.0.70201-81_22.04_amd64", "name": "rocsparse", - "sha256": "f72489fc3632558c7142810090ecf01565aca6b9c702c751bec4492450de9a8a", + "sha256": "f29106f3e1a6374cca59c0ee2ba78b1901934316f46951b2ecc136363b5a7bb9", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocsparse/rocsparse_4.2.0.70200-43~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocsparse/rocsparse_4.2.0.70201-81~22.04_amd64.deb" ], - "version": "4.2.0.70200-43~22.04" + "version": "4.2.0.70201-81~22.04" } ], "version": 1 diff --git a/platforms/rocm/packages.yaml b/platforms/rocm/packages.yaml index d98be620..3427388e 100644 --- a/platforms/rocm/packages.yaml +++ b/platforms/rocm/packages.yaml @@ -5,9 +5,9 @@ version: 1 sources: - channel: jammy main - url: https://repo.radeon.com/rocm/apt/7.2 + url: https://repo.radeon.com/rocm/apt/7.2.1 - channel: jammy main - url: https://repo.radeon.com/graphics/7.2/ubuntu + url: https://repo.radeon.com/graphics/7.2.1/ubuntu - channel: jammy main url: https://snapshot.ubuntu.com/ubuntu/20250711T030400Z - channel: jammy-security main @@ -19,7 +19,11 @@ archs: - "amd64" # readelf -d libpjrt_rocm.so | grep NEEDED +# find $(bazel info bazel-bin)/external/+rocm_packages+libpjrt_rocm/sandbox/ -type f -name '*.so*' -exec readelf -d {} + 2>/dev/null | grep NEEDED | sed -E 's/.*Shared library: \[(.*)\]/\1/' | sort -u packages: + - "libbz2-1.0" + - "libdw1" + - "liblzma5" # - "rocm-smi-lib" - "libdrm-amdgpu-common" # Do not use hsa-rocr from amdgpu repo because it is overriden by hsa-runtime-rocr4wsl-amdgpu diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 2b49a128..316b0f0a 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -6,9 +6,24 @@ _BUILD_FILE_DEFAULT_VISIBILITY = """\ package(default_visibility = ["//visibility:public"]) """ -_ROCM_STRIP_PREFIX = "./opt/rocm-7.2.0" +_ROCM_STRIP_PREFIX = "./opt/rocm-7.2.1" + +def _rocm_dlopen_patchelf(name, src): + return "\n".join([ + packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), + packages.patchelf( + name = name, + src = src, + add_needed = ["libzmlxrocm.so.0"], + set_rpath = "$ORIGIN", + rename_dynamic_symbols = { + "dlopen": "zmlxrocm_dlopen", + }, + ), + ]) _UBUNTU_PACKAGES = { + "libbz2-1.0": packages.filegroup(name = "libbz2-1.0", srcs = ["lib/x86_64-linux-gnu/libbz2.so.1.0"]), "libdrm2-amdgpu": packages.filegroup(name = "libdrm2-amdgpu", srcs = ["opt/amdgpu/lib/x86_64-linux-gnu/libdrm.so.2"]), "libelf1": "\n".join([ packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), @@ -20,6 +35,7 @@ _UBUNTU_PACKAGES = { ]), "libdrm-amdgpu-common": packages.filegroup(name = "amdgpu_ids", srcs = ["opt/amdgpu/share/libdrm/amdgpu.ids"]), "libnuma1": packages.filegroup(name = "libnuma1", srcs = ["usr/lib/x86_64-linux-gnu/libnuma.so.1"]), + "liblzma5": packages.filegroup(name = "liblzma5", srcs = ["lib/x86_64-linux-gnu/liblzma.so.5"]), "libzstd1": packages.filegroup(name = "libzstd1", srcs = ["usr/lib/x86_64-linux-gnu/libzstd.so.1"]), "libdrm-amdgpu-amdgpu1": "\n".join([ packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), @@ -35,28 +51,70 @@ _UBUNTU_PACKAGES = { ]), "libtinfo6": packages.filegroup(name = "libtinfo6", srcs = ["lib/x86_64-linux-gnu/libtinfo.so.6"]), "zlib1g": packages.filegroup(name = "zlib1g", srcs = ["lib/x86_64-linux-gnu/libz.so.1"]), + "libdw1": packages.filegroup(name = "libdw1", srcs = ["usr/lib/x86_64-linux-gnu/libdw.so.1"]), } _ROCM_PACKAGES = { - "rocm-core": packages.filegroup(name = "rocm-core", srcs = ["lib/librocm-core.so.1"]), - "rocm-smi-lib": packages.filegroup(name = "rocm_smi", srcs = ["lib/librocm_smi64.so.1"]), + "rocm-core": _rocm_dlopen_patchelf( + name = "rocm-core", + src = "lib/librocm-core.so.1", + ), + "rocm-smi-lib": _rocm_dlopen_patchelf( + name = "rocm_smi", + src = "lib/librocm_smi64.so.1", + ), "rocprofiler-sdk": "\n".join([ + packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), + packages.patchelf( + name = "rocprofiler-sdk_so", + src = "lib/librocprofiler-sdk.so.1", + add_needed = ["libzmlxrocm.so.0"], + set_rpath = "$ORIGIN", + rename_dynamic_symbols = { + "dlopen": "zmlxrocm_dlopen", + }, + ), + packages.filegroup(name = "rocprofiler-sdk-attach", srcs = ["lib/librocprofiler-sdk-attach.so.1"]), packages.filegroup(name = "rocprofiler-sdk", srcs = [ - "lib/librocprofiler-sdk.so.1", - "lib/librocprofiler-sdk-attach.so.1", + ":rocprofiler-sdk_so", + ":rocprofiler-sdk-attach", ]), ]), "rocprofiler-sdk-rocpd": packages.filegroup(name = "rocprofiler-sdk-rocpd", srcs = ["lib/librocprofiler-sdk-rocpd.so.1"]), "rocprofiler-sdk-roctx": packages.filegroup(name = "rocprofiler-sdk-roctx", srcs = ["lib/librocprofiler-sdk-roctx.so.1"]), - "hsa-rocr": packages.filegroup(name = "hsa-runtime", srcs = ["lib/libhsa-runtime64.so.1"]), - "hsa-amd-aqlprofile": packages.filegroup(name = "hsa-amd-aqlprofile", srcs = ["lib/libhsa-amd-aqlprofile64.so.1"]), - "comgr": packages.filegroup(name = "amd_comgr", srcs = ["lib/libamd_comgr.so.3"]), - "rocprofiler-register": packages.filegroup(name = "rocprofiler-register", srcs = ["lib/librocprofiler-register.so.0"]), + "hsa-rocr": _rocm_dlopen_patchelf( + name = "hsa-runtime", + src = "lib/libhsa-runtime64.so.1", + ), + "hsa-amd-aqlprofile": _rocm_dlopen_patchelf( + name = "hsa-amd-aqlprofile", + src = "lib/libhsa-amd-aqlprofile64.so.1", + ), + "comgr": _rocm_dlopen_patchelf( + name = "amd_comgr", + src = "lib/libamd_comgr.so.3", + ), + "rocprofiler-register": _rocm_dlopen_patchelf( + name = "rocprofiler-register", + src = "lib/librocprofiler-register.so.0", + ), "miopen-hip": "\n".join([ - packages.filegroup(name = "MIOpen", srcs = ["lib/libMIOpen.so.1"]), + packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), + packages.patchelf( + name = "MIOpen", + src = "lib/libMIOpen.so.1", + add_needed = ["libzmlxrocm.so.0"], + set_rpath = "$ORIGIN", + rename_dynamic_symbols = { + "dlopen": "zmlxrocm_dlopen", + }, + ), """filegroup(name = "runfiles", srcs = glob(["share/miopen/**"]))""", ]), - "rccl": packages.filegroup(name = "rccl", srcs = ["lib/librccl.so.1"]), + "rccl": _rocm_dlopen_patchelf( + name = "rccl", + src = "lib/librccl.so.1", + ), "rocm-device-libs": """filegroup(name = "runfiles", srcs = glob(["amdgcn/**"]))""", "hip-dev": """filegroup(name = "runfiles", srcs = glob(["share/**"]))""", "rocblas": "\n".join([ @@ -76,10 +134,30 @@ _ROCM_PACKAGES = { """ ]), "rocfft": packages.filegroup(name = "rocfft", srcs = ["lib/librocfft.so.0"]), - "rocsolver": packages.filegroup(name = "rocsolver", srcs = ["lib/librocsolver.so.0"]), + "rocsolver": _rocm_dlopen_patchelf( + name = "rocsolver", + src = "lib/librocsolver.so.0", + ), "roctracer": "\n".join([ - packages.filegroup(name = "roctracer", srcs = ["lib/libroctracer64.so.4"]), - packages.filegroup(name = "roctx", srcs = ["lib/libroctx64.so.4"]), + packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), + packages.patchelf( + name = "roctracer", + src = "lib/libroctracer64.so.4", + add_needed = ["libzmlxrocm.so.0"], + set_rpath = "$ORIGIN", + rename_dynamic_symbols = { + "dlopen": "zmlxrocm_dlopen", + }, + ), + packages.patchelf( + name = "roctx", + src = "lib/libroctx64.so.4", + add_needed = ["libzmlxrocm.so.0"], + set_rpath = "$ORIGIN", + rename_dynamic_symbols = { + "dlopen": "zmlxrocm_dlopen", + }, + ), ]), "hipblaslt": "\n".join([ packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), @@ -113,13 +191,26 @@ _ROCM_PACKAGES = { name = "amdhip", src = "lib/libamdhip64.so.7", add_needed = ["libzmlxrocm.so.0"], + set_rpath = "$ORIGIN", rename_dynamic_symbols = { "dlopen": "zmlxrocm_dlopen", }, ), - packages.filegroup(name = "hiprtc", srcs = ["lib/libhiprtc.so.7", "lib/libhiprtc-builtins.so.7"]), + packages.patchelf( + name = "hiprtc_so", + src = "lib/libhiprtc.so.7", + add_needed = ["libzmlxrocm.so.0"], + set_rpath = "$ORIGIN", + rename_dynamic_symbols = { + "dlopen": "zmlxrocm_dlopen", + }, + ), + packages.filegroup(name = "hiprtc", srcs = [":hiprtc_so", "lib/libhiprtc-builtins.so.7"]), ]), - "hipsolver": packages.filegroup(name = "hipsolver", srcs = ["lib/libhipsolver.so.1"]), + "hipsolver": _rocm_dlopen_patchelf( + name = "hipsolver", + src = "lib/libhipsolver.so.1", + ), "rocsparse": packages.filegroup(name = "rocsparse", srcs = ["lib/librocsparse.so.1"]), } @@ -150,8 +241,8 @@ def _rocm_impl(mctx): http_archive( name = "libpjrt_rocm", build_file = "libpjrt_rocm.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/nightly-2026-03-21/pjrt-rocm_linux-amd64.tar.gz", - sha256 = "fd324748258192fe9dd1b830f97b619365407362bf2bdb80b8d68bec7baba7ea", + url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-03-26T11-30-00Z/pjrt-rocm_linux-amd64.tar.gz", + sha256 = "6649e89831570926bf127f7e57f25dca4f526e22764f2df0d689818badc1d4fe", ) return mctx.extension_metadata( From 4ecdd92cb094b26bac96d6281ff91e433c0bc33c Mon Sep 17 00:00:00 2001 From: elogir <90246794+elogir@users.noreply.github.com> Date: Fri, 27 Mar 2026 14:45:35 +0100 Subject: [PATCH 394/881] platforms/rocm: merge rocm and mesa amdgpu.ids (#441) This PR introduces a script to merge `amdgpu.ids` from two sources: Mesa and ROCm. Both provide GPU ID-to-name mappings, but each may contain entries the other lacks. For example, Mesa includes `AMD Radeon 8060S Graphics` while ROCm does not, and ROCm lists newer GPUs absent from Mesa. The script merges the two sources together, with ROCm entries taking priority in case of conflicts. --- platforms/rocm/BUILD.bazel | 5 ++ platforms/rocm/libpjrt_rocm.BUILD.bazel | 25 ++++++++- platforms/rocm/merge_amdgpu_ids.zig | 73 +++++++++++++++++++++++++ platforms/rocm/rocm.bzl | 8 +++ 4 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 platforms/rocm/merge_amdgpu_ids.zig diff --git a/platforms/rocm/BUILD.bazel b/platforms/rocm/BUILD.bazel index 5e1e8c05..93027194 100644 --- a/platforms/rocm/BUILD.bazel +++ b/platforms/rocm/BUILD.bazel @@ -1,6 +1,11 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library", "zig_shared_library") +exports_files( + ["merge_amdgpu_ids.zig"], + visibility = ["@libpjrt_rocm//:__subpackages__"], +) + zig_shared_library( name = "zmlxrocm", main = "zmlxrocm.zig", diff --git a/platforms/rocm/libpjrt_rocm.BUILD.bazel b/platforms/rocm/libpjrt_rocm.BUILD.bazel index 9e3c9c3c..e6152d1d 100644 --- a/platforms/rocm/libpjrt_rocm.BUILD.bazel +++ b/platforms/rocm/libpjrt_rocm.BUILD.bazel @@ -1,5 +1,7 @@ load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "string_list_flag") +load("@bazel_skylib//rules:run_binary.bzl", "run_binary") +load("@rules_zig//zig:defs.bzl", "zig_binary") load("@zml//bazel:patchelf.bzl", "patchelf") bool_flag( @@ -25,6 +27,26 @@ patchelf( set_rpath = "$ORIGIN", ) +zig_binary( + name = "merge_amdgpu_ids", + main = "@zml//platforms/rocm:merge_amdgpu_ids.zig", +) + +run_binary( + name = "amdgpu_ids", + srcs = [ + "@libdrm-amdgpu-common//:amdgpu_ids", + "@libdrm_mesa_amdgpu_ids//file:amdgpu.ids", + ], + outs = ["amdgpu.ids"], + args = [ + "$(location @libdrm-amdgpu-common//:amdgpu_ids)", + "$(location @libdrm_mesa_amdgpu_ids//file:amdgpu.ids)", + "$(location amdgpu.ids)", + ], + tool = ":merge_amdgpu_ids", +) + copy_to_directory( name = "sandbox", srcs = [ @@ -39,7 +61,7 @@ copy_to_directory( "@hsa-amd-aqlprofile//:hsa-amd-aqlprofile", "@hsa-rocr//:hsa-runtime", "@libdrm-amdgpu-amdgpu1", - "@libdrm-amdgpu-common//:amdgpu_ids", + ":amdgpu_ids", "@libdrm2-amdgpu", "@libbz2-1.0", "@libelf1", @@ -85,6 +107,7 @@ copy_to_directory( "libpjrt_rocm_so": "lib", "opt/amdgpu/lib/x86_64-linux-gnu": "lib", "opt/amdgpu/share": "share", + "amdgpu.ids": "share/libdrm/amdgpu.ids", "rccl": "lib", "rocblas": "lib", "rocm-core": "lib", diff --git a/platforms/rocm/merge_amdgpu_ids.zig b/platforms/rocm/merge_amdgpu_ids.zig new file mode 100644 index 00000000..b939df82 --- /dev/null +++ b/platforms/rocm/merge_amdgpu_ids.zig @@ -0,0 +1,73 @@ +const std = @import("std"); + +pub fn main(init: std.process.Init) !void { + const io = init.io; + const arena = init.arena.allocator(); + + const args = try init.minimal.args.toSlice(arena); + + if (args.len != 4) { + try std.Io.File.stderr().writeStreamingAll(io, "Usage: merge_amdgpu_ids \n"); + std.process.exit(1); + } + + var entries: std.StringHashMapUnmanaged([]const u8) = .empty; + defer entries.deinit(arena); + + const cwd = std.Io.Dir.cwd(); + + try parseFile(io, cwd, arena, args[2], &entries); + try parseFile(io, cwd, arena, args[1], &entries); + + const out = try cwd.createFile(io, args[3], .{}); + defer out.close(io); + + var buf: [4096]u8 = undefined; + var writer = out.writer(io, &buf); + + try writer.interface.print("# List of AMDGPU IDs\n" ++ + "#\n" ++ + "# Syntax:\n" ++ + "# device_id,\trevision_id,\tproduct_name <-- single tab after comma\n" ++ + "\n" ++ + "1.0.0\n", .{}); + + var it = entries.valueIterator(); + while (it.next()) |v| { + try writer.interface.print("{s}\n", .{v.*}); + } + + try writer.flush(); +} + +fn parseFile( + io: std.Io, + cwd: std.Io.Dir, + arena: std.mem.Allocator, + path: []const u8, + entries: *std.StringHashMapUnmanaged([]const u8), +) !void { + const file = try cwd.openFile(io, path, .{ .mode = .read_only }); + defer file.close(io); + + var buf: [4096]u8 = undefined; + var reader = file.reader(io, &buf); + + while (true) { + const raw = try reader.interface.takeDelimiter('\n') orelse break; + + const line = std.mem.trimEnd(u8, raw, " \t\r"); + if (line.len == 0 or line[0] == '#') continue; + + var iter = std.mem.splitSequence(u8, line, ","); + + const device = std.mem.trim(u8, iter.next() orelse continue, " \t"); + const revision = std.mem.trim(u8, iter.next() orelse continue, " \t"); + const product = std.mem.trim(u8, iter.rest(), " \t"); + + const key = try std.fmt.allocPrint(arena, "{s},{s}", .{ device, revision }); + const value = try std.fmt.allocPrint(arena, "{s},\t{s},\t{s}", .{ device, revision, product }); + + try entries.put(arena, key, value); + } +} diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 316b0f0a..7c7b3973 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -1,3 +1,4 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") load("//platforms:packages.bzl", "packages") @@ -238,6 +239,13 @@ def _rocm_impl(mctx): build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, ) + http_file( + name = "libdrm_mesa_amdgpu_ids", + url = "https://cgit.freedesktop.org/mesa/drm/plain/data/amdgpu.ids?id=b9dea73dfa310bc945ae6f09004a08fd624952ec", + sha256 = "ffd2a8f1bfa755f4d90f537b4969fc4676f116e5af051ce2f18ef93a96d8beb6", + downloaded_file_path = "amdgpu.ids", + ) + http_archive( name = "libpjrt_rocm", build_file = "libpjrt_rocm.BUILD.bazel", From b675cfbbe3c2e7ba30e1ca7ed8892d5edaa63d89 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 31 Mar 2026 10:28:15 +0200 Subject: [PATCH 395/881] workspace: bump @llvm=0.7.0 For macOS SDK fix --- MODULE.bazel | 16 +++++++--------- MODULE.bazel.lock | 6 +++--- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index af0b018d..b7fb97e7 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -22,7 +22,7 @@ bazel_dep(name = "rules_rs", version = "0.0.44") bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "rules_zig", version = "0.12.2") bazel_dep(name = "tar.bzl", version = "0.9.0") -bazel_dep(name = "llvm", version = "0.6.8") +bazel_dep(name = "llvm", version = "0.7.0") bazel_dep(name = "jsoncpp", version = "1.9.6", repo_name = "jsoncpp_git") bazel_dep(name = "with_cfg.bzl", version = "0.14.4") @@ -83,12 +83,10 @@ python.toolchain(python_version = "3.12") use_repo(python, "python_versions") pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") -pip_neuron = use_extension("@rules_python//python/extensions:pip.bzl", "pip") +pip_neuron = use_extension("@rules_python//python/extensions:pip.bzl", "pip") pip_neuron.parse( download_only = True, - hub_name = "neuron_py_deps", - python_version = "3.12", extra_pip_args = [ "--abi=cp312", "--implementation=cp", @@ -99,6 +97,8 @@ pip_neuron.parse( "--platform=manylinux_2_28_x86_64", "--platform=manylinux_2_29_x86_64", ], + hub_name = "neuron_py_deps", + python_version = "3.12", requirements_lock = "//platforms/neuron:requirements.lock.txt", target_platforms = ["linux_x86_64"], ) @@ -225,20 +225,18 @@ oci.pull( image = "gcr.io/distroless/cc-debian12", platforms = ["linux/amd64"], ) - oci.pull( name = "debian_12", - image = "docker.io/library/debian:12", digest = "sha256:bc960ef50e6feed90686c593361df158517556ed1d2d98e5d1df3724024e0f49", + image = "docker.io/library/debian:12", platforms = ["linux/amd64"], ) - use_repo( oci, - "distroless_cc_debian12", - "distroless_cc_debian12_linux_amd64", "debian_12", "debian_12_linux_amd64", + "distroless_cc_debian12", + "distroless_cc_debian12_linux_amd64", ) git_override( diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 175ba7ea..e9a0eaed 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -93,8 +93,8 @@ "https://bcr.bazel.build/modules/jsoncpp/1.9.6/source.json": "a04756d367a2126c3541682864ecec52f92cdee80a35735a3cb249ce015ca000", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", "https://bcr.bazel.build/modules/llvm/0.6.7/MODULE.bazel": "d37a2e10571864dc6a5bb53c29216d90b9400bbcadb422337f49107fd2eaf0d2", - "https://bcr.bazel.build/modules/llvm/0.6.8/MODULE.bazel": "53468e4a4be409c2d34e5b7331d2e1fef982151b777655ca3c0047225b333629", - "https://bcr.bazel.build/modules/llvm/0.6.8/source.json": "b673af466f716e01d6243f59e47729e99f37dc5e17026d2bf18c98206f09b6c5", + "https://bcr.bazel.build/modules/llvm/0.7.0/MODULE.bazel": "3c07a4e5734b0ad41fe24dedaacbf3a35ce4377b7e1a21f24488a0c9ac4f1e6b", + "https://bcr.bazel.build/modules/llvm/0.7.0/source.json": "5f0a015da1e7db09ef61aa254ba5eb55b0c322fadd1e8eefeca368d7723a24a0", "https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92", "https://bcr.bazel.build/modules/package_metadata/0.0.5/MODULE.bazel": "ef4f9439e3270fdd6b9fd4dbc3d2f29d13888e44c529a1b243f7a31dfbc2e8e4", "https://bcr.bazel.build/modules/package_metadata/0.0.5/source.json": "2326db2f6592578177751c3e1f74786b79382cd6008834c9d01ec865b9126a85", @@ -426,7 +426,7 @@ "@@rules_oci+//oci:extensions.bzl%oci": { "general": { "bzlTransitiveDigest": "twSeB3VPTVkayY0uwqEzDzal5MbnZX45iC6jdheZQRo=", - "usagesDigest": "PFMCBQvV3Q+zC654PFY3m911buSFwM1rzl1zow5lOWE=", + "usagesDigest": "V0dQbNX+eB3aThhopTuYblOol9rtleO/RA16gC5Zn+Y=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, From 67b4fbdbb85f861543f585be9c1de854eeb27d62 Mon Sep 17 00:00:00 2001 From: elogir <90246794+elogir@users.noreply.github.com> Date: Tue, 31 Mar 2026 12:47:07 +0200 Subject: [PATCH 396/881] bin: Introduce zml-smi (#440) This PR introduces **zml-smi**, a tool to monitor metrics of any platform supported by ZML, featuring but not limited to: - real time `--top` mode - per process tracking - responsive layout --------- Co-authored-by: Steeve Morin --- .bazelrc | 1 + MODULE.bazel | 21 +- bin/zml-smi/BUILD.bazel | 182 + bin/zml-smi/bindings/amdsmi/amdsmi.zig | 279 ++ bin/zml-smi/bindings/amdsmi/metrics.zig | 145 + bin/zml-smi/bindings/amdsmi/process.zig | 148 + bin/zml-smi/bindings/dynlib.zig | 131 + bin/zml-smi/bindings/linux/metrics.zig | 98 + bin/zml-smi/bindings/linux/process.zig | 117 + bin/zml-smi/bindings/neuron/metrics.zig | 170 + bin/zml-smi/bindings/neuron/nrt.zig | 128 + bin/zml-smi/bindings/neuron/process.zig | 110 + bin/zml-smi/bindings/nvml/metrics.zig | 151 + bin/zml-smi/bindings/nvml/nvml.zig | 316 ++ bin/zml-smi/bindings/nvml/process.zig | 87 + bin/zml-smi/bindings/tpu/libtpu/tpuinfo.cc | 84 + bin/zml-smi/bindings/tpu/libtpu/tpuinfo.h | 20 + bin/zml-smi/bindings/tpu/metrics.zig | 175 + bin/zml-smi/bindings/tpu/process.zig | 84 + bin/zml-smi/bindings/tpu/tpuinfo.zig | 37 + bin/zml-smi/collector.zig | 57 + bin/zml-smi/info/device_info.zig | 101 + bin/zml-smi/info/host_info.zig | 16 + bin/zml-smi/info/process_info.zig | 11 + bin/zml-smi/main.zig | 93 + bin/zml-smi/platform.zig | 36 + bin/zml-smi/scripts/remote-install.sh | 167 + bin/zml-smi/tui/assets/amd.png | Bin 0 -> 421 bytes bin/zml-smi/tui/assets/logo.png | Bin 0 -> 108686 bytes bin/zml-smi/tui/assets/neuron.png | Bin 0 -> 1916 bytes bin/zml-smi/tui/assets/nvidia.png | Bin 0 -> 1779 bytes bin/zml-smi/tui/assets/tpu.png | Bin 0 -> 1878 bytes bin/zml-smi/tui/data.zig | 146 + bin/zml-smi/tui/image_cache.zig | 36 + bin/zml-smi/tui/lib/compose.zig | 46 + bin/zml-smi/tui/lib/image.zig | 78 + bin/zml-smi/tui/lib/ui.zig | 87 + bin/zml-smi/tui/lib/utils.zig | 30 + bin/zml-smi/tui/pages/detail.zig | 28 + .../tui/pages/detail/detail_common.zig | 55 + bin/zml-smi/tui/pages/detail/gpu_detail.zig | 144 + .../tui/pages/detail/neuron_detail.zig | 79 + bin/zml-smi/tui/pages/detail/tpu_detail.zig | 54 + bin/zml-smi/tui/pages/overview.zig | 121 + bin/zml-smi/tui/print.zig | 95 + bin/zml-smi/tui/theme.zig | 50 + bin/zml-smi/tui/tui.zig | 166 + bin/zml-smi/tui/widgets/braille_chart.zig | 76 + bin/zml-smi/tui/widgets/chart.zig | 141 + bin/zml-smi/tui/widgets/column_layout.zig | 64 + bin/zml-smi/tui/widgets/device_card.zig | 124 + bin/zml-smi/tui/widgets/gauge.zig | 103 + bin/zml-smi/tui/widgets/info_lines.zig | 111 + bin/zml-smi/tui/widgets/logo.zig | 51 + bin/zml-smi/tui/widgets/metric_card.zig | 163 + bin/zml-smi/tui/widgets/process_table.zig | 179 + bin/zml-smi/tui/widgets/status_line.zig | 77 + bin/zml-smi/tui/widgets/titled_border.zig | 112 + bin/zml-smi/utils/double_buffer.zig | 22 + bin/zml-smi/utils/sandbox.zig | 44 + bin/zml-smi/utils/sysfs.zig | 70 + bin/zml-smi/worker.zig | 62 + platforms/cuda/cuda.bzl | 10 +- platforms/neuron/BUILD.bazel | 3 +- platforms/neuron/nrt.h | 2 + platforms/rocm/BUILD.bazel | 7 +- platforms/rocm/libpjrt_rocm.BUILD.bazel | 1 + platforms/rocm/packages.lock.json | 1265 +++-- platforms/rocm/packages.yaml | 1 + platforms/rocm/rocm.bzl | 11 +- .../cloud_accelerator_diagnostics/BUILD.bazel | 0 .../cloud_accelerator_diagnostics.BUILD.bazel | 34 + .../cloud_accelerator_diagnostics/repo.bzl | 29 + third_party/libvaxis/BUILD.bazel | 1 + third_party/libvaxis/bump-zig.patch | 2019 ++++++++ third_party/libvaxis/fixes.patch | 231 + third_party/libvaxis/libvaxis.bazel | 17 + third_party/libvaxis/repo.bzl | 11 + third_party/non_module_deps.bzl | 8 + third_party/uucode/BUILD.bazel | 4 + third_party/uucode/build_config.zig | 18 + third_party/uucode/repo.bzl | 25 + third_party/uucode/uucode.bazel | 159 + third_party/zigimg/BUILD.bazel | 1 + third_party/zigimg/bump-zig.patch | 4338 +++++++++++++++++ third_party/zigimg/repo.bzl | 25 + third_party/zigimg/zigimg.bazel | 18 + zml/BUILD.bazel | 7 +- 88 files changed, 13391 insertions(+), 433 deletions(-) create mode 100644 bin/zml-smi/BUILD.bazel create mode 100644 bin/zml-smi/bindings/amdsmi/amdsmi.zig create mode 100644 bin/zml-smi/bindings/amdsmi/metrics.zig create mode 100644 bin/zml-smi/bindings/amdsmi/process.zig create mode 100644 bin/zml-smi/bindings/dynlib.zig create mode 100644 bin/zml-smi/bindings/linux/metrics.zig create mode 100644 bin/zml-smi/bindings/linux/process.zig create mode 100644 bin/zml-smi/bindings/neuron/metrics.zig create mode 100644 bin/zml-smi/bindings/neuron/nrt.zig create mode 100644 bin/zml-smi/bindings/neuron/process.zig create mode 100644 bin/zml-smi/bindings/nvml/metrics.zig create mode 100644 bin/zml-smi/bindings/nvml/nvml.zig create mode 100644 bin/zml-smi/bindings/nvml/process.zig create mode 100644 bin/zml-smi/bindings/tpu/libtpu/tpuinfo.cc create mode 100644 bin/zml-smi/bindings/tpu/libtpu/tpuinfo.h create mode 100644 bin/zml-smi/bindings/tpu/metrics.zig create mode 100644 bin/zml-smi/bindings/tpu/process.zig create mode 100644 bin/zml-smi/bindings/tpu/tpuinfo.zig create mode 100644 bin/zml-smi/collector.zig create mode 100644 bin/zml-smi/info/device_info.zig create mode 100644 bin/zml-smi/info/host_info.zig create mode 100644 bin/zml-smi/info/process_info.zig create mode 100644 bin/zml-smi/main.zig create mode 100644 bin/zml-smi/platform.zig create mode 100644 bin/zml-smi/scripts/remote-install.sh create mode 100644 bin/zml-smi/tui/assets/amd.png create mode 100644 bin/zml-smi/tui/assets/logo.png create mode 100644 bin/zml-smi/tui/assets/neuron.png create mode 100644 bin/zml-smi/tui/assets/nvidia.png create mode 100644 bin/zml-smi/tui/assets/tpu.png create mode 100644 bin/zml-smi/tui/data.zig create mode 100644 bin/zml-smi/tui/image_cache.zig create mode 100644 bin/zml-smi/tui/lib/compose.zig create mode 100644 bin/zml-smi/tui/lib/image.zig create mode 100644 bin/zml-smi/tui/lib/ui.zig create mode 100644 bin/zml-smi/tui/lib/utils.zig create mode 100644 bin/zml-smi/tui/pages/detail.zig create mode 100644 bin/zml-smi/tui/pages/detail/detail_common.zig create mode 100644 bin/zml-smi/tui/pages/detail/gpu_detail.zig create mode 100644 bin/zml-smi/tui/pages/detail/neuron_detail.zig create mode 100644 bin/zml-smi/tui/pages/detail/tpu_detail.zig create mode 100644 bin/zml-smi/tui/pages/overview.zig create mode 100644 bin/zml-smi/tui/print.zig create mode 100644 bin/zml-smi/tui/theme.zig create mode 100644 bin/zml-smi/tui/tui.zig create mode 100644 bin/zml-smi/tui/widgets/braille_chart.zig create mode 100644 bin/zml-smi/tui/widgets/chart.zig create mode 100644 bin/zml-smi/tui/widgets/column_layout.zig create mode 100644 bin/zml-smi/tui/widgets/device_card.zig create mode 100644 bin/zml-smi/tui/widgets/gauge.zig create mode 100644 bin/zml-smi/tui/widgets/info_lines.zig create mode 100644 bin/zml-smi/tui/widgets/logo.zig create mode 100644 bin/zml-smi/tui/widgets/metric_card.zig create mode 100644 bin/zml-smi/tui/widgets/process_table.zig create mode 100644 bin/zml-smi/tui/widgets/status_line.zig create mode 100644 bin/zml-smi/tui/widgets/titled_border.zig create mode 100644 bin/zml-smi/utils/double_buffer.zig create mode 100644 bin/zml-smi/utils/sandbox.zig create mode 100644 bin/zml-smi/utils/sysfs.zig create mode 100644 bin/zml-smi/worker.zig mode change 100755 => 100644 platforms/rocm/packages.lock.json create mode 100644 third_party/cloud_accelerator_diagnostics/BUILD.bazel create mode 100644 third_party/cloud_accelerator_diagnostics/cloud_accelerator_diagnostics.BUILD.bazel create mode 100644 third_party/cloud_accelerator_diagnostics/repo.bzl create mode 100644 third_party/libvaxis/BUILD.bazel create mode 100644 third_party/libvaxis/bump-zig.patch create mode 100644 third_party/libvaxis/fixes.patch create mode 100644 third_party/libvaxis/libvaxis.bazel create mode 100644 third_party/libvaxis/repo.bzl create mode 100644 third_party/uucode/BUILD.bazel create mode 100644 third_party/uucode/build_config.zig create mode 100644 third_party/uucode/repo.bzl create mode 100644 third_party/uucode/uucode.bazel create mode 100644 third_party/zigimg/BUILD.bazel create mode 100644 third_party/zigimg/bump-zig.patch create mode 100644 third_party/zigimg/repo.bzl create mode 100644 third_party/zigimg/zigimg.bazel diff --git a/.bazelrc b/.bazelrc index 8286df2d..52406f5b 100644 --- a/.bazelrc +++ b/.bazelrc @@ -113,6 +113,7 @@ common:alldebug --strategy=local common:release --compilation_mode=opt common:release --copt=-Ofast +common:release --linkopt=-Wl,--strip-all common:release --@rules_zig//zig/settings:mode=release_safe # --config=native builds everything optimized for the local CPU diff --git a/MODULE.bazel b/MODULE.bazel index b7fb97e7..1d839851 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -2,14 +2,14 @@ module( name = "zml", ) -bazel_dep(name = "abseil-cpp", version = "20260107.1", repo_name = "com_google_absl") +bazel_dep(name = "abseil-cpp", version = "20250814.1", repo_name = "com_google_absl") bazel_dep(name = "bazel_lib", version = "3.2.2") bazel_dep(name = "aspect_rules_py", version = "1.8.4") bazel_dep(name = "bazel_skylib", version = "1.9.0") bazel_dep(name = "patchelf", version = "0.18.0.bcr.1") bazel_dep(name = "pcre2", version = "10.45") bazel_dep(name = "platforms", version = "1.0.0") -bazel_dep(name = "protobuf", version = "34.0.bcr.1", repo_name = "com_google_protobuf") +bazel_dep(name = "protobuf", version = "33.6", repo_name = "com_google_protobuf") bazel_dep(name = "pybind11_bazel", version = "2.13.6") bazel_dep(name = "re2", version = "2025-11-05", repo_name = "com_googlesource_code_re2") bazel_dep(name = "rules_cc", version = "0.2.16") @@ -25,6 +25,7 @@ bazel_dep(name = "tar.bzl", version = "0.9.0") bazel_dep(name = "llvm", version = "0.7.0") bazel_dep(name = "jsoncpp", version = "1.9.6", repo_name = "jsoncpp_git") bazel_dep(name = "with_cfg.bzl", version = "0.14.4") +bazel_dep(name = "grpc", version = "1.74.0") bazel_dep(name = "buildifier_prebuilt", version = "8.5.1", dev_dependency = True) @@ -125,10 +126,10 @@ cpu = use_extension("//platforms/cpu:cpu.bzl", "cpu_pjrt_plugin") use_repo(cpu, "libpjrt_cpu_darwin_amd64", "libpjrt_cpu_darwin_arm64", "libpjrt_cpu_linux_amd64") cuda = use_extension("//platforms/cuda:cuda.bzl", "cuda_packages") -use_repo(cuda, "libpjrt_cuda") +use_repo(cuda, "cuda_nvml_dev", "libpjrt_cuda") rocm = use_extension("//platforms/rocm:rocm.bzl", "rocm_packages") -use_repo(rocm, "hipblaslt", "libpjrt_rocm", "rocblas") +use_repo(rocm, "amd-smi-lib", "hipblaslt", "libpjrt_rocm", "libdrm2-amdgpu", "libdrm-amdgpu-amdgpu1", "rocblas") tpu = use_extension("//platforms/tpu:tpu.bzl", "tpu_packages") use_repo(tpu, "libpjrt_tpu") @@ -142,7 +143,7 @@ use_repo(zls, "zls_toolchains") register_toolchains("@zls_toolchains//:all") non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") -use_repo(non_module_deps, "arocc", "com_google_sentencepiece", "flashattn", "linenoise", "mnist", "org_swig_swig", "stb", "translate-c", "xla") +use_repo(non_module_deps, "arocc", "cloud_accelerator_diagnostics", "com_google_sentencepiece", "flashattn", "libvaxis", "linenoise", "mnist", "org_swig_swig", "translate-c", "stb", "uucode", "xla", "zigimg") xla = use_extension("//third_party/xla:xla.bzl", "xla") use_repo( @@ -244,3 +245,13 @@ git_override( commit = "7b205b78fb21e11efe5adcc9ec1c37fa5852dc77", remote = "https://github.com/zml/rules_zig.git", ) + +single_version_override( + module_name = "protobuf", + version = "33.6", +) + +single_version_override( + module_name = "grpc-java", + version = "1.74.0", +) diff --git a/bin/zml-smi/BUILD.bazel b/bin/zml-smi/BUILD.bazel new file mode 100644 index 00000000..5e29d6af --- /dev/null +++ b/bin/zml-smi/BUILD.bazel @@ -0,0 +1,182 @@ +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@bazel_skylib//rules:run_binary.bzl", "run_binary") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load") +load("@rules_zig//zig:defs.bzl", "zig_binary") +load("@tar.bzl//tar:mtree.bzl", "mtree_spec") +load("@tar.bzl//tar:tar.bzl", "tar") + +# tpu grpc + +cc_library( + name = "tpuinfo", + srcs = ["bindings/tpu/libtpu/tpuinfo.cc"], + hdrs = ["bindings/tpu/libtpu/tpuinfo.h"], + strip_include_prefix = "bindings/tpu/libtpu", + deps = [ + "@cloud_accelerator_diagnostics//:tpu_metric_service_cc_grpc", + "@grpc//:grpc++_unsecure", + ], +) + +# smi + +run_binary( + name = "libnrt.stripped", + srcs = ["@aws-neuronx-runtime-lib//:libnrt.patchelf"], + outs = ["libnrt.so.1"], + args = [ + "--strip-debug", + "-o", + "$(location libnrt.so.1)", + "$(location @aws-neuronx-runtime-lib//:libnrt.patchelf)", + ], + tool = "@llvm//tools:llvm-strip", +) + +run_binary( + name = "libncfw.stripped", + srcs = ["@aws-neuronx-runtime-lib//:libncfw.patchelf"], + outs = ["libncfw.so.2"], + args = [ + "--strip-debug", + "-o", + "$(location libncfw.so.2)", + "$(location @aws-neuronx-runtime-lib//:libncfw.patchelf)", + ], + tool = "@llvm//tools:llvm-strip", +) + +copy_to_directory( + name = "sandbox", + srcs = select({ + "@platforms//os:macos": [], + "//conditions:default": [ + ":libncfw.stripped", + ":libnrt.stripped", + "//platforms/rocm:zmlxrocm", + "@amd-smi-lib//:libamd_smi", + "@libdrm-amdgpu-amdgpu1", + "@libdrm2-amdgpu//:libdrm2-amdgpu", + "@libpjrt_rocm//:amdgpu_ids", + ], + }), + include_external_repositories = ["**"], + replace_prefixes = { + "opt/amdgpu/lib/x86_64-linux-gnu": "lib", + "amdgpu.ids": "share/libdrm/amdgpu.ids", + "libdrm-amdgpu-amdgpu1": "lib", + "platforms/rocm": "lib", + "libnrt.so.1": "lib/libnrt.so.1", + "libncfw.so.2": "lib/libncfw.so.2", + }, +) + +zig_binary( + name = "zml-smi", + srcs = glob([ + "tui/**/*.zig", + ]) + [ + "bindings/amdsmi/amdsmi.zig", + "bindings/amdsmi/metrics.zig", + "bindings/amdsmi/process.zig", + "bindings/dynlib.zig", + "bindings/linux/metrics.zig", + "bindings/linux/process.zig", + "bindings/neuron/metrics.zig", + "bindings/neuron/nrt.zig", + "bindings/neuron/process.zig", + "bindings/nvml/metrics.zig", + "bindings/nvml/nvml.zig", + "bindings/nvml/process.zig", + "bindings/tpu/metrics.zig", + "bindings/tpu/process.zig", + "bindings/tpu/tpuinfo.zig", + "collector.zig", + "info/device_info.zig", + "info/host_info.zig", + "info/process_info.zig", + "platform.zig", + "utils/double_buffer.zig", + "utils/sandbox.zig", + "utils/sysfs.zig", + "worker.zig", + ], + data = [":sandbox"], + extra_srcs = glob([ + "tui/assets/*", + ]), + main = "main.zig", + deps = [ + "//stdx", + "@libvaxis//:vaxis", + "@rules_zig//zig/runfiles", + ] + select({ + "@platforms//os:macos": [], + "//conditions:default": [ + ":tpuinfo", + "//platforms/neuron:libnrt_headers", + "@amd-smi-lib//:amdsmi", + "@cuda_nvml_dev//:nvml", + ], + }), +) + +# docker image + +mtree_spec( + name = "mtree_spec", + srcs = [":zml-smi"], +) + +# Use this genrule until mtree_mutate is fixed +genrule( + name = "mtree", + srcs = [":mtree_spec"], + outs = ["mtree.txt"], + cmd = """sed -e '/type=dir/d' -e 's#^bin/##g' $< > $@""", +) + +# mtree_mutate( +# name = "mtree", +# mtree = ":mtree_spec", +# strip_prefix = "bin", +# ) + +tar( + name = "archive", + srcs = [":zml-smi"], + args = [ + "--options", + ",".join([ + "zstd:compression-level=22", + "zstd:threads=16", + ]), + ], + compress = "zstd", + mtree = ":mtree", +) + +oci_image( + name = "image_", + base = "@distroless_cc_debian12", + entrypoint = ["/zml-smi/zml-smi"], + tars = [ + ":archive", + ], +) + +platform_transition_filegroup( + name = "image", + srcs = [":image_"], + tags = ["manual"], + target_platform = "//platforms:linux_amd64", +) + +oci_load( + name = "load", + image = ":image", + repo_tags = ["zmlai/zml-smi:latest"], + tags = ["manual"], +) diff --git a/bin/zml-smi/bindings/amdsmi/amdsmi.zig b/bin/zml-smi/bindings/amdsmi/amdsmi.zig new file mode 100644 index 00000000..0320dc61 --- /dev/null +++ b/bin/zml-smi/bindings/amdsmi/amdsmi.zig @@ -0,0 +1,279 @@ +const std = @import("std"); +const c = @import("c"); +const stdx = @import("stdx"); +const DynLib = @import("../dynlib.zig"); +const sandbox = @import("../../utils/sandbox.zig"); + +const AmdSmi = @This(); + +pub const Error = ReturnError || error{AmdSmiUnavailable}; +pub const Handle = c.amdsmi_processor_handle; +pub const ProcInfo = c.amdsmi_proc_info_t; + +lib: Fns, +gpu_handles: []c.amdsmi_processor_handle, + +const Fns = struct { + amdsmi_init: DynLib.Fn("amdsmi_init"), + amdsmi_get_socket_handles: DynLib.Fn("amdsmi_get_socket_handles"), + amdsmi_get_processor_handles: DynLib.Fn("amdsmi_get_processor_handles"), + amdsmi_get_gpu_asic_info: DynLib.Fn("amdsmi_get_gpu_asic_info"), + amdsmi_get_gpu_metrics_info: DynLib.Fn("amdsmi_get_gpu_metrics_info"), + amdsmi_get_power_info: DynLib.Fn("amdsmi_get_power_info"), + amdsmi_get_temp_metric: DynLib.Fn("amdsmi_get_temp_metric"), + amdsmi_get_gpu_fan_speed: DynLib.Fn("amdsmi_get_gpu_fan_speed"), + amdsmi_get_gpu_activity: DynLib.Fn("amdsmi_get_gpu_activity"), + amdsmi_get_clock_info: DynLib.Fn("amdsmi_get_clock_info"), + amdsmi_get_gpu_memory_total: DynLib.Fn("amdsmi_get_gpu_memory_total"), + amdsmi_get_gpu_memory_usage: DynLib.Fn("amdsmi_get_gpu_memory_usage"), + amdsmi_get_pcie_info: DynLib.Fn("amdsmi_get_pcie_info"), + amdsmi_get_gpu_bdf_id: DynLib.Fn("amdsmi_get_gpu_bdf_id"), + amdsmi_get_gpu_process_list: DynLib.Fn("amdsmi_get_gpu_process_list"), +}; + +pub fn init(allocator: std.mem.Allocator) !AmdSmi { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = sandbox.path(&path_buf) orelse return error.AmdSmiUnavailable; + + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libamd_smi.so.26" }); + + const fns = DynLib.open(Fns, path) orelse return error.AmdSmiUnavailable; + + try check(fns.amdsmi_init(c.AMDSMI_INIT_AMD_GPUS)); + + var socket_count: u32 = 0; + try check(fns.amdsmi_get_socket_handles(&socket_count, null)); + + const sockets = try allocator.alloc(c.amdsmi_socket_handle, socket_count); + defer allocator.free(sockets); + try check(fns.amdsmi_get_socket_handles(&socket_count, @ptrCast(sockets.ptr))); + + var handle_list: std.ArrayList(c.amdsmi_processor_handle) = .{}; + defer handle_list.deinit(allocator); + + for (sockets[0..socket_count]) |socket| { + var proc_count: u32 = 0; + try check(fns.amdsmi_get_processor_handles(socket, &proc_count, null)); + + const procs = try allocator.alloc(c.amdsmi_processor_handle, proc_count); + defer allocator.free(procs); + + try check(fns.amdsmi_get_processor_handles(socket, &proc_count, @ptrCast(procs.ptr))); + + for (procs[0..proc_count]) |proc| { + try handle_list.append(allocator, proc); + } + } + + return .{ .lib = fns, .gpu_handles = try handle_list.toOwnedSlice(allocator) }; +} + +pub fn handleByIndex(self: AmdSmi, device_id: u32) Error!Handle { + if (device_id >= self.gpu_handles.len) { + return error.not_found; + } + return self.gpu_handles[device_id]; +} + +pub fn deviceCount(self: AmdSmi) u32 { + return @intCast(self.gpu_handles.len); +} + +pub const name_buf_len = c.AMDSMI_MAX_STRING_LENGTH; + +pub fn name(self: AmdSmi, handle: Handle, buf: *[c.AMDSMI_MAX_STRING_LENGTH]u8) Error![:0]const u8 { + var info: c.amdsmi_asic_info_t = undefined; + try check(self.lib.amdsmi_get_gpu_asic_info(handle, &info)); + @memcpy(buf, &info.market_name); + return std.mem.sliceTo(@as([*:0]const u8, @ptrCast(buf)), 0); +} + +pub fn powerUsage(self: AmdSmi, handle: Handle) Error!u32 { + var metrics: c.amdsmi_gpu_metrics_t = undefined; + try check(self.lib.amdsmi_get_gpu_metrics_info(handle, &metrics)); + const unsupported16 = std.math.maxInt(u16); + if (metrics.current_socket_power != unsupported16) { + return metrics.current_socket_power; + } + if (metrics.average_socket_power != unsupported16) { + return metrics.average_socket_power; + } + return error.not_supported; +} + +pub fn powerLimit(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_power_info_t = undefined; + try check(self.lib.amdsmi_get_power_info(handle, &info)); + return info.power_limit; +} + +pub fn temperature(self: AmdSmi, handle: Handle) Error!i64 { + var temp: i64 = 0; + try check(self.lib.amdsmi_get_temp_metric(handle, c.AMDSMI_TEMPERATURE_TYPE_EDGE, c.AMDSMI_TEMP_CURRENT, &temp)); + return temp; +} + +pub fn fanSpeed(self: AmdSmi, handle: Handle) Error!i64 { + var speed: i64 = 0; + try check(self.lib.amdsmi_get_gpu_fan_speed(handle, @as(u32, 0), &speed)); + return speed; +} + +pub fn gpuUtil(self: AmdSmi, handle: Handle) Error!u32 { + var usage: c.amdsmi_engine_usage_t = undefined; + try check(self.lib.amdsmi_get_gpu_activity(handle, &usage)); + return usage.gfx_activity; +} + +pub fn clockGraphics(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_clk_info_t = undefined; + try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_SYS, &info)); + return info.clk; +} + +pub fn maxClockGraphics(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_clk_info_t = undefined; + try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_SYS, &info)); + return info.max_clk; +} + +pub fn clockMem(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_clk_info_t = undefined; + try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_MEM, &info)); + return info.clk; +} + +pub fn maxClockMem(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_clk_info_t = undefined; + try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_MEM, &info)); + return info.max_clk; +} + +pub fn clockSoc(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_clk_info_t = undefined; + try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_SOC, &info)); + return info.clk; +} + +pub fn memTotal(self: AmdSmi, handle: Handle) Error!u64 { + var total: u64 = 0; + try check(self.lib.amdsmi_get_gpu_memory_total(handle, c.AMDSMI_MEM_TYPE_VRAM, &total)); + return total; +} + +pub fn memUsed(self: AmdSmi, handle: Handle) Error!u64 { + var used: u64 = 0; + try check(self.lib.amdsmi_get_gpu_memory_usage(handle, c.AMDSMI_MEM_TYPE_VRAM, &used)); + return used; +} + +pub fn pcieWidth(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_pcie_info_t = undefined; + try check(self.lib.amdsmi_get_pcie_info(handle, &info)); + return @intCast(info.pcie_metric.pcie_width); +} + +pub fn pcieBandwidth(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_pcie_info_t = undefined; + try check(self.lib.amdsmi_get_pcie_info(handle, &info)); + return info.pcie_metric.pcie_bandwidth; +} + +pub fn pcieLinkGen(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_pcie_info_t = undefined; + try check(self.lib.amdsmi_get_pcie_info(handle, &info)); + return info.pcie_static.pcie_interface_version; +} + +pub fn bdfId(self: AmdSmi, handle: Handle) Error!u64 { + var bdf_id: u64 = 0; + try check(self.lib.amdsmi_get_gpu_bdf_id(handle, &bdf_id)); + return bdf_id; +} + +pub fn processList(self: AmdSmi, allocator: std.mem.Allocator, handle: Handle) (Error || error{OutOfMemory})![]const ProcInfo { + var count: u32 = 0; + try check(self.lib.amdsmi_get_gpu_process_list(handle, &count, null)); + if (count == 0) { + return &.{}; + } + const procs = try allocator.alloc(ProcInfo, count); + errdefer allocator.free(procs); + try check(self.lib.amdsmi_get_gpu_process_list(handle, &count, @ptrCast(procs.ptr))); + return procs; +} + +// Error handling + +pub const ReturnError = error{ + inval, + not_supported, + not_yet_implemented, + fail_load_module, + fail_load_symbol, + drm_error, + api_failed, + timeout, + retry, + no_perm, + interrupt, + io_error, + address_fault, + file_error, + out_of_resources, + internal_exception, + input_out_of_bounds, + init_error, + refcount_overflow, + directory_not_found, + busy, + not_found, + not_init, + no_slot, + driver_not_loaded, + no_data, + insufficient_size, + unexpected_size, + unexpected_data, + unknown_error, +}; + +fn check(ret: c.amdsmi_status_t) Error!void { + if (ret == c.AMDSMI_STATUS_SUCCESS) { + return; + } + + return switch (ret) { + c.AMDSMI_STATUS_INVAL => error.inval, + c.AMDSMI_STATUS_NOT_SUPPORTED => error.not_supported, + c.AMDSMI_STATUS_NOT_YET_IMPLEMENTED => error.not_yet_implemented, + c.AMDSMI_STATUS_FAIL_LOAD_MODULE => error.fail_load_module, + c.AMDSMI_STATUS_FAIL_LOAD_SYMBOL => error.fail_load_symbol, + c.AMDSMI_STATUS_DRM_ERROR => error.drm_error, + c.AMDSMI_STATUS_API_FAILED => error.api_failed, + c.AMDSMI_STATUS_TIMEOUT => error.timeout, + c.AMDSMI_STATUS_RETRY => error.retry, + c.AMDSMI_STATUS_NO_PERM => error.no_perm, + c.AMDSMI_STATUS_INTERRUPT => error.interrupt, + c.AMDSMI_STATUS_IO => error.io_error, + c.AMDSMI_STATUS_ADDRESS_FAULT => error.address_fault, + c.AMDSMI_STATUS_FILE_ERROR => error.file_error, + c.AMDSMI_STATUS_OUT_OF_RESOURCES => error.out_of_resources, + c.AMDSMI_STATUS_INTERNAL_EXCEPTION => error.internal_exception, + c.AMDSMI_STATUS_INPUT_OUT_OF_BOUNDS => error.input_out_of_bounds, + c.AMDSMI_STATUS_INIT_ERROR => error.init_error, + c.AMDSMI_STATUS_REFCOUNT_OVERFLOW => error.refcount_overflow, + c.AMDSMI_STATUS_DIRECTORY_NOT_FOUND => error.directory_not_found, + c.AMDSMI_STATUS_BUSY => error.busy, + c.AMDSMI_STATUS_NOT_FOUND => error.not_found, + c.AMDSMI_STATUS_NOT_INIT => error.not_init, + c.AMDSMI_STATUS_NO_SLOT => error.no_slot, + c.AMDSMI_STATUS_DRIVER_NOT_LOADED => error.driver_not_loaded, + c.AMDSMI_STATUS_NO_DATA => error.no_data, + c.AMDSMI_STATUS_INSUFFICIENT_SIZE => error.insufficient_size, + c.AMDSMI_STATUS_UNEXPECTED_SIZE => error.unexpected_size, + c.AMDSMI_STATUS_UNEXPECTED_DATA => error.unexpected_data, + else => error.unknown_error, + }; +} diff --git a/bin/zml-smi/bindings/amdsmi/metrics.zig b/bin/zml-smi/bindings/amdsmi/metrics.zig new file mode 100644 index 00000000..8e724ba2 --- /dev/null +++ b/bin/zml-smi/bindings/amdsmi/metrics.zig @@ -0,0 +1,145 @@ +const std = @import("std"); +const AmdSmi = @import("amdsmi.zig"); +const device_info = @import("../../info/device_info.zig"); +const DeviceInfo = device_info.DeviceInfo; +const GpuInfo = device_info.GpuInfo; +const DoubleBuffer = @import("../../utils/double_buffer.zig").DoubleBuffer; +const Collector = @import("../../collector.zig").Collector; +const Worker = @import("../../worker.zig").Worker; +const process = @import("process.zig"); + +pub const target: device_info.Target = .rocm; + +pub fn start(collector: *Collector) !void { + const amdsmi = try collector.arena.create(AmdSmi); + amdsmi.* = try AmdSmi.init(collector.arena); + + const count = amdsmi.deviceCount(); + const dev_offset: u8 = @intCast(collector.device_infos.items.len); + + for (0..count) |i| { + const dev = Device.open(amdsmi, @intCast(i)) catch continue; + const initial: GpuInfo = .{ .name = dev.name(collector.arena) catch null }; + const info = try collector.addDevice(.{ .rocm = .{ .values = .{ initial, initial } } }); + + try collector.worker.spawn(collector.io, pollDevice, .{ collector.io, collector.worker, &info.rocm, dev }); + } + + const processes = try collector.createProcessList(); + try process.init(collector.worker, collector.io, collector.gpa, processes, amdsmi, dev_offset); +} + +const pollDevice = Worker.pollMetrics(*DoubleBuffer(GpuInfo), Device, metrics); + +const Device = struct { + amdsmi: *const AmdSmi, + handle: AmdSmi.Handle, + + pub fn open(amdsmi: *const AmdSmi, index: u32) !Device { + return .{ .amdsmi = amdsmi, .handle = try amdsmi.handleByIndex(index) }; + } + + fn name(self: Device, arena: std.mem.Allocator) ![]const u8 { + var buf: [AmdSmi.name_buf_len]u8 = .{0} ** AmdSmi.name_buf_len; + const slice = try self.amdsmi.name(self.handle, &buf); + + return try arena.dupe(u8, slice); + } + + // Power + pub fn powerUsage(self: Device) !u64 { + const pw = try self.amdsmi.powerUsage(self.handle); + return @as(u64, pw) * 1000; + } + + pub fn powerLimit(self: Device) !u64 { + const limit = try self.amdsmi.powerLimit(self.handle); + if (limit == std.math.maxInt(u32)) { + return error.not_supported; + } + + return @as(u64, limit) / 1000; + } + + // Thermal + pub fn temperature(self: Device) !u64 { + const temp = try self.amdsmi.temperature(self.handle); + return @intCast(temp); + } + + pub fn fanSpeed(self: Device) !u64 { + const speed = try self.amdsmi.fanSpeed(self.handle); + return @intCast(@divTrunc(speed * 100, 255)); + } + + // Utilization + pub fn gpuUtil(self: Device) !u64 { + return try self.amdsmi.gpuUtil(self.handle); + } + + // Clocks + pub fn clockGraphics(self: Device) !u64 { + return try self.amdsmi.clockGraphics(self.handle); + } + + pub fn clockSoc(self: Device) !u64 { + return try self.amdsmi.clockSoc(self.handle); + } + + pub fn clockMem(self: Device) !u64 { + return try self.amdsmi.clockMem(self.handle); + } + + pub fn maxClockGraphics(self: Device) !u64 { + return try self.amdsmi.maxClockGraphics(self.handle); + } + + pub fn maxClockMem(self: Device) !u64 { + return try self.amdsmi.maxClockMem(self.handle); + } + + // Memory + pub fn memUsed(self: Device) !u64 { + return self.amdsmi.memUsed(self.handle); + } + + pub fn memTotal(self: Device) !u64 { + return self.amdsmi.memTotal(self.handle); + } + + // PCIe + pub fn pcieBandwidth(self: Device) !u64 { + const bw = try self.amdsmi.pcieBandwidth(self.handle); + if (bw == std.math.maxInt(u32)) { + return error.not_supported; + } + + return bw; + } + + pub fn pcieLinkGen(self: Device) !u64 { + return try self.amdsmi.pcieLinkGen(self.handle); + } + + pub fn pcieLinkWidth(self: Device) !u64 { + return try self.amdsmi.pcieWidth(self.handle); + } +}; + +const metrics = .{ + .{ .field = "power_mw", .query = Device.powerUsage }, + .{ .field = "power_limit_mw", .query = Device.powerLimit }, + .{ .field = "temperature", .query = Device.temperature }, + .{ .field = "fan_speed_percent", .query = Device.fanSpeed }, + .{ .field = "util_percent", .query = Device.gpuUtil }, + .{ .field = "clock_graphics_mhz", .query = Device.clockGraphics }, + .{ .field = "clock_soc_mhz", .query = Device.clockSoc }, + .{ .field = "clock_mem_mhz", .query = Device.clockMem }, + .{ .field = "clock_graphics_max_mhz", .query = Device.maxClockGraphics }, + .{ .field = "clock_mem_max_mhz", .query = Device.maxClockMem }, + .{ .field = "mem_used_bytes", .query = Device.memUsed }, + .{ .field = "mem_total_bytes", .query = Device.memTotal }, + .{ .field = "pcie_bandwidth_mbps", .query = Device.pcieBandwidth }, + .{ .field = "pcie_link_gen", .query = Device.pcieLinkGen }, + .{ .field = "pcie_link_width", .query = Device.pcieLinkWidth }, +}; diff --git a/bin/zml-smi/bindings/amdsmi/process.zig b/bin/zml-smi/bindings/amdsmi/process.zig new file mode 100644 index 00000000..ecc4e379 --- /dev/null +++ b/bin/zml-smi/bindings/amdsmi/process.zig @@ -0,0 +1,148 @@ +const std = @import("std"); +const c = std.c; +const AmdSmi = @import("amdsmi.zig"); +const pi = @import("../../info/process_info.zig"); +const ProcessDoubleBuffer = @import("../../utils/double_buffer.zig").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const Worker = @import("../../worker.zig").Worker; + +const bdf_len = "0000:00:00.0".len; + +pub fn init(w: *Worker, io: std.Io, allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, amdsmi: *const AmdSmi, dev_offset: u8) !void { + try w.spawn(io, pollLoop, .{ io, w, allocator, list, amdsmi, dev_offset }); +} + +fn pollLoop(io: std.Io, w: *const Worker, allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, amdsmi: *const AmdSmi, dev_offset: u8) void { + const interval: std.Io.Duration = .fromMilliseconds(w.poll_interval_ms); + io.sleep(interval, .awake) catch {}; + + const device_count = amdsmi.deviceCount(); + const pci_slots = allocator.alloc(?[bdf_len]u8, device_count) catch return; + defer allocator.free(pci_slots); + + for (pci_slots, 0..) |*slot, i| { + const handle = amdsmi.handleByIndex(@intCast(i)) catch { + slot.* = null; + continue; + }; + + slot.* = formatBdf(amdsmi.bdfId(handle) catch { + slot.* = null; + continue; + }); + } + + while (w.isRunning()) { + const start: std.Io.Timestamp = .now(io, .awake); + const back = list.back(); + + back.clearRetainingCapacity(); + + for (0..device_count) |dev_idx| { + const handle = amdsmi.handleByIndex(@intCast(dev_idx)) catch continue; + + const procs = blk: { + const saved = c.dup(c.STDERR_FILENO); + const devnull = c.open("/dev/null", .{ .ACCMODE = .WRONLY }); + + if (devnull >= 0) { + _ = c.dup2(devnull, c.STDERR_FILENO); + _ = c.close(devnull); + } + + defer if (saved >= 0) { + _ = c.dup2(saved, c.STDERR_FILENO); + _ = c.close(saved); + }; + + break :blk amdsmi.processList(allocator, handle) catch continue; + }; + defer { + if (procs.len > 0) { + allocator.free(procs); + } + } + + const pci_slot = if (dev_idx < pci_slots.len) &(pci_slots[dev_idx] orelse continue) else continue; + + for (procs) |proc| { + back.append(allocator, .{ + .pid = proc.pid, + .device_idx = @intCast(dev_idx + dev_offset), + .dev_util_percent = @intCast(proc.engine_usage.gfx + proc.engine_usage.enc), + .dev_mem_kib = readProcessGpuVram(io, proc.pid, pci_slot), + }) catch break; + } + } + + list.swap(); + + const elapsed = start.untilNow(io, .awake); + if (elapsed.nanoseconds < interval.nanoseconds) { + io.sleep(.fromNanoseconds(interval.nanoseconds - elapsed.nanoseconds), .awake) catch {}; + } + } +} + +/// Reads per-GPU VRAM from /proc//fdinfo/, filtered by PCI slot. +/// amdsmi reports global VRAM across all GPUs; fdinfo gives per-GPU values. +fn readProcessGpuVram(io: std.Io, pid: u32, pci_slot: *const [bdf_len]u8) u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const dir_path = std.fmt.bufPrint(&path_buf, "/proc/{d}/fdinfo", .{pid}) catch return 0; + + var dir = std.Io.Dir.openDirAbsolute(io, dir_path, .{ .iterate = true }) catch return 0; + defer dir.close(io); + + var total_vram_kib: u64 = 0; + var it = dir.iterate(); + while (it.next(io) catch null) |entry| { + var file_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const file_path = std.fmt.bufPrint(&file_buf, "{s}/{s}", .{ dir_path, entry.name }) catch continue; + + total_vram_kib += parseFdinfoVram(io, file_path, pci_slot); + } + + return total_vram_kib; +} + +fn parseFdinfoVram(io: std.Io, path: []const u8, pci_slot: *const [bdf_len]u8) u64 { + var file = std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }) catch return 0; + defer file.close(io); + + var read_buf: [4096]u8 = undefined; + var reader = file.reader(io, &read_buf); + + var found_pci = false; + var vram_kib: ?u64 = null; + + while (true) { + const line = reader.interface.takeDelimiterExclusive('\n') catch break; + reader.interface.toss(1); + + if (std.mem.indexOf(u8, line, pci_slot) != null) found_pci = true; + + // Support both new and old fdinfo keys (see amdgpu_fdinfo.c) + inline for ([_][]const u8{ "drm-memory-vram:\t", "vram mem:\t" }) |key| { + if (std.mem.startsWith(u8, line, key)) { + const rest = line[key.len..]; + const end = std.mem.indexOfAny(u8, rest, " \t\n") orelse rest.len; + vram_kib = std.fmt.parseInt(u64, rest[0..end], 10) catch 0; + } + } + + if (found_pci and vram_kib != null) return vram_kib.?; + } + + return 0; +} + +fn formatBdf(bdf_id: u64) [bdf_len]u8 { + var buf: [bdf_len]u8 = undefined; + _ = std.fmt.bufPrint(&buf, "{x:0>4}:{x:0>2}:{x:0>2}.{x}", .{ + (bdf_id >> 16) & 0xFFFF, + (bdf_id >> 8) & 0xFF, + (bdf_id >> 3) & 0x1F, + bdf_id & 0x7, + }) catch unreachable; + + return buf; +} diff --git a/bin/zml-smi/bindings/dynlib.zig b/bin/zml-smi/bindings/dynlib.zig new file mode 100644 index 00000000..2cea9205 --- /dev/null +++ b/bin/zml-smi/bindings/dynlib.zig @@ -0,0 +1,131 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const elf = std.elf; +const c = @import("c"); + +pub fn Fn(comptime name: [:0]const u8) type { + return *const @TypeOf(@field(c, name)); +} + +pub fn open(comptime F: type, path: [:0]const u8) ?F { + const handle = openDl(path) orelse return null; + return resolveAll(F, handle, null, 0); +} + +pub fn openElf(comptime F: type, io: std.Io, path: [:0]const u8, comptime known_sym: [:0]const u8) !F { + const handle = openDl(path) orelse { + if (std.c.dlerror()) |err| std.log.err("dlopen: {s}", .{err}); + return error.DlOpenFailed; + }; + + const sym_addr = std.c.dlsym(handle, known_sym) orelse + return error.KnownSymbolNotFound; + + var info: DlInfo = undefined; + if (dladdr(sym_addr, &info) == 0) { + return error.DladdrFailed; + } + + const elf_base = @intFromPtr(info.dli_fbase); + const elf_path = std.mem.sliceTo(info.dli_fname, 0); + const file = try std.Io.Dir.openFileAbsolute(io, elf_path, .{}); + defer file.close(io); + + const stat = try file.stat(io); + const mmap = try std.posix.mmap( + null, + @intCast(stat.size), + .{ .READ = true }, + .{ .TYPE = .PRIVATE }, + file.handle, + 0, + ); + defer std.posix.munmap(mmap); + + return resolveAll(F, handle, mmap, elf_base) orelse error.SymbolResolutionFailed; +} + +fn resolveAll(comptime F: type, handle: *anyopaque, elf_mmap: ?[]align(std.heap.page_size_min) u8, elf_base: usize) ?F { + var result: F = undefined; + var unresolved: std.hash_map.StringHashMapUnmanaged(*usize) = .{}; + defer unresolved.deinit(std.heap.page_allocator); + + // Phase 1: try dlsym for each field; for unresolved ones, store a pointer to the result field + inline for (std.meta.fields(F)) |field| { + if (std.c.dlsym(handle, field.name)) |p| { + @field(result, field.name) = @as(field.type, @ptrFromInt(@intFromPtr(p))); + } else { + unresolved.put(std.heap.page_allocator, field.name, @ptrCast(&@field(result, field.name))) catch return null; + } + } + + if (unresolved.count() == 0) { // no need to parse ELF + return result; + } + + // Phase 2: scan ELF symbols, write resolved addresses directly into result fields + const buf = elf_mmap orelse return null; + const ehdr: *const elf.ElfN.Ehdr = @ptrCast(@alignCast(buf.ptr)); + const shdrs: [*]const elf.ElfN.Shdr = @ptrCast(@alignCast(buf.ptr + @as(usize, ehdr.shoff))); + + var remaining = unresolved.count(); + + outer: for (0..ehdr.shnum) |i| { + const sh = shdrs[i]; + if (sh.type != .SYMTAB) { + continue; + } + + const strtab_sh = shdrs[sh.link]; + const strtab: [*]const u8 = @ptrCast(buf.ptr + @as(usize, strtab_sh.offset)); + const syms: []const elf.ElfN.Sym = @as( + [*]const elf.ElfN.Sym, + @ptrCast(@alignCast(buf.ptr + @as(usize, sh.offset))), + )[0 .. @as(usize, sh.size) / @sizeOf(elf.ElfN.Sym)]; + + for (syms) |s| { + if (s.value == 0) { + continue; + } + + const sym_name = std.mem.sliceTo(@as([*:0]const u8, @ptrCast(strtab + @as(usize, s.name))), 0); + if (unresolved.get(sym_name)) |field_ptr| { + field_ptr.* = elf_base + @as(usize, s.value); + remaining -= 1; + + if (remaining == 0) { + break :outer; + } + } + } + } + + if (remaining > 0) { + return null; + } + + return result; +} + +fn openDl(path: [:0]const u8) ?*anyopaque { + return switch (builtin.os.tag) { + .linux, .macos => std.c.dlopen(path, switch (builtin.os.tag) { + .linux => .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }, + .macos => .{ .LAZY = true, .LOCAL = true }, + else => unreachable, + }), + else => blk: { + const dl = std.DynLib.open(path) catch break :blk null; + break :blk dl.inner.handle; + }, + }; +} + +const DlInfo = extern struct { + dli_fname: [*:0]const u8, + dli_fbase: *anyopaque, + dli_sname: [*c]const u8, + dli_saddr: ?*anyopaque, +}; + +extern fn dladdr(addr: *anyopaque, info: *DlInfo) c_int; diff --git a/bin/zml-smi/bindings/linux/metrics.zig b/bin/zml-smi/bindings/linux/metrics.zig new file mode 100644 index 00000000..376c75d7 --- /dev/null +++ b/bin/zml-smi/bindings/linux/metrics.zig @@ -0,0 +1,98 @@ +const std = @import("std"); +const sysfs = @import("../../utils/sysfs.zig"); +const host_info = @import("../../info/host_info.zig"); +const HostInfo = host_info.HostInfo; +const HostData = host_info.HostData; +const Worker = @import("../../worker.zig").Worker; + +const Collector = @import("../../collector.zig").Collector; + +pub fn init(w: *Worker, io: std.Io, collector: *Collector, info: *HostInfo) !void { + const poll_arena = try collector.createPollArena(); + const host: Host = .{ .io = io, .arena = poll_arena }; + + // read once metrics + var initial = info.front().*; + initial.hostname = host.hostname(collector.arena) catch null; + initial.kernel = host.kernel(collector.arena) catch null; + initial.cpu_name = host.cpuName(collector.arena) catch null; + initial.cpu_cores = host.cpuCores() catch null; + initial.mem_total_kib = host.memTotal() catch null; + info.back().* = initial; + info.swap(); + + try w.spawn(io, pollHost, .{ io, w, info, host }); +} + +const pollHost = Worker.pollMetrics(*HostInfo, Host, metrics); + +const Host = struct { + io: std.Io, + arena: *std.heap.ArenaAllocator, + + pub fn hostname(self: Host, allocator: std.mem.Allocator) ![]const u8 { + return sysfs.readString(allocator, self.io, "/proc/sys/kernel/hostname"); + } + + pub fn kernel(self: Host, allocator: std.mem.Allocator) ![]const u8 { + return sysfs.readString(allocator, self.io, "/proc/sys/kernel/osrelease"); + } + + pub fn cpuName(self: Host, allocator: std.mem.Allocator) ![]const u8 { + return sysfs.readFieldString(allocator, self.io, "/proc/cpuinfo", "model name"); + } + + pub fn cpuCores(self: Host) !u64 { + const data = try sysfs.readString(self.arena.allocator(), self.io, "/sys/devices/system/cpu/present"); + const dash = std.mem.indexOfScalar(u8, data, '-') orelse return 1; + + return (try std.fmt.parseInt(u64, data[dash + 1 ..], 10)) + 1; + } + + pub fn memTotal(self: Host) !u64 { + return sysfs.readFieldInt(self.arena.allocator(), self.io, "/proc/meminfo", "MemTotal"); + } + + pub fn memAvailable(self: Host) !u64 { + return sysfs.readFieldInt(self.arena.allocator(), self.io, "/proc/meminfo", "MemAvailable"); + } + + fn parseLoadAvg(self: Host, comptime n: usize) !f32 { + const data = try sysfs.readString(self.arena.allocator(), self.io, "/proc/loadavg"); + var iter = std.mem.splitScalar(u8, data, ' '); + + var i: usize = 0; + while (iter.next()) |tok| : (i += 1) { + if (i == n) return std.fmt.parseFloat(f32, tok); + } + + return error.NotFound; + } + + pub fn load1(self: Host) !f32 { + return self.parseLoadAvg(0); + } + + pub fn load5(self: Host) !f32 { + return self.parseLoadAvg(1); + } + + pub fn load15(self: Host) !f32 { + return self.parseLoadAvg(2); + } + + pub fn uptime(self: Host) !u64 { + const data = try sysfs.readString(self.arena.allocator(), self.io, "/proc/uptime"); + const dot = std.mem.indexOfScalar(u8, data, '.') orelse data.len; + + return std.fmt.parseInt(u64, data[0..dot], 10); + } +}; + +const metrics = .{ + .{ .field = "mem_available_kib", .query = Host.memAvailable }, + .{ .field = "load_1", .query = Host.load1 }, + .{ .field = "load_5", .query = Host.load5 }, + .{ .field = "load_15", .query = Host.load15 }, + .{ .field = "uptime_seconds", .query = Host.uptime }, +}; diff --git a/bin/zml-smi/bindings/linux/process.zig b/bin/zml-smi/bindings/linux/process.zig new file mode 100644 index 00000000..364a2f6b --- /dev/null +++ b/bin/zml-smi/bindings/linux/process.zig @@ -0,0 +1,117 @@ +const std = @import("std"); +const sysfs = @import("../../utils/sysfs.zig"); +const pi = @import("../../info/process_info.zig"); + +const ProcessInfo = pi.ProcessInfo; + +pub const ProcessEnricher = struct { + gpa: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + prev_ticks: std.AutoHashMapUnmanaged(u32, u64) = .{}, + prev_total_ticks: u64 = 0, + + pub fn init(gpa: std.mem.Allocator, io: std.Io) !ProcessEnricher { + var arena = std.heap.ArenaAllocator.init(gpa); + errdefer arena.deinit(); + + const prev_total_ticks = readTotalCpuTicks(arena.allocator(), io); + var enricher: ProcessEnricher = .{ + .gpa = gpa, + .arena = arena, + .prev_total_ticks = prev_total_ticks, + .prev_ticks = .{}, + }; + + try enricher.prev_ticks.ensureTotalCapacity(gpa, 64); + + return enricher; + } + + pub fn deinit(self: *ProcessEnricher) void { + self.prev_ticks.deinit(self.gpa); + self.arena.deinit(); + } + + pub fn enrich(self: *ProcessEnricher, io: std.Io, procs: []ProcessInfo) void { + _ = self.arena.reset(.retain_capacity); + const alloc = self.arena.allocator(); + + const curr_total = readTotalCpuTicks(alloc, io); + const delta_total = curr_total -| self.prev_total_ticks; + + var curr_ticks: std.AutoHashMapUnmanaged(u32, u64) = .{}; + curr_ticks.ensureTotalCapacity(self.gpa, @intCast(procs.len)) catch return; + + for (procs) |*info| { + // Read Uid, VmRSS from /proc/{pid}/status + const status_path = std.fmt.allocPrint(alloc, "/proc/{d}/status", .{info.pid}) catch continue; + const uid: u32 = @intCast(sysfs.readFieldInt(alloc, io, status_path, "Uid") catch 0); + info.uid = uid; + info.username = lookupUsername(alloc, uid) catch continue; + info.rss_kib = sysfs.readFieldInt(alloc, io, status_path, "VmRSS") catch 0; + + // Read cmdline + const cmdline_path = std.fmt.allocPrint(alloc, "/proc/{d}/cmdline", .{info.pid}) catch continue; + info.comm = readCmdline(alloc, io, cmdline_path) catch continue; + + // CPU% delta + const ticks = readProcTicks(alloc, io, info.pid) orelse continue; + if (delta_total > 0) { + if (self.prev_ticks.get(info.pid)) |prev| { + const delta_proc = ticks -| prev; + info.cpu_percent = @intCast(@min(delta_proc * 1000 / delta_total, std.math.maxInt(u16))); + } + } + + curr_ticks.getOrPutAssumeCapacity(info.pid).value_ptr.* = ticks; + } + + self.prev_ticks.deinit(self.gpa); + self.prev_ticks = curr_ticks; + self.prev_total_ticks = curr_total; + } +}; + +fn lookupUsername(allocator: std.mem.Allocator, uid: u32) ![]const u8 { + const pw = std.c.getpwuid(uid) orelse return error.NoUser; + const name = std.mem.span(pw.name orelse return error.NoName); + + return try allocator.dupe(u8, name); +} + +/// Extract utime + stime from /proc/{pid}/stat. +fn readProcTicks(allocator: std.mem.Allocator, io: std.Io, pid: u32) ?u64 { + const path = std.fmt.allocPrint(allocator, "/proc/{d}/stat", .{pid}) catch return null; + const data = sysfs.readFirstLine(allocator, io, path) catch return null; + + // Fields after ") ": state=0, ppid=1, ..., utime=11, stime=12 + const rest = data[(std.mem.lastIndexOfScalar(u8, data, ')') orelse return null) + 2 ..]; + + return sysfs.nthTokenInt(rest, 11) + sysfs.nthTokenInt(rest, 12); +} + +fn readCmdline(allocator: std.mem.Allocator, io: std.Io, path: []const u8) ![]const u8 { + const data = try sysfs.readFirstLine(allocator, io, path); + + for (data) |*c| { + if (c.* == 0) { + c.* = ' '; + } + } + + return std.mem.trimEnd(u8, data, &std.ascii.whitespace); +} + +fn readTotalCpuTicks(allocator: std.mem.Allocator, io: std.Io) u64 { + const data = sysfs.readFirstLine(allocator, io, "/proc/stat") catch return 0; + + var total: u64 = 0; + var iter = std.mem.tokenizeAny(u8, data, " \t"); + + _ = iter.next(); // skip "cpu" + while (iter.next()) |tok| { + total += std.fmt.parseInt(u64, tok, 10) catch 0; + } + + return total; +} diff --git a/bin/zml-smi/bindings/neuron/metrics.zig b/bin/zml-smi/bindings/neuron/metrics.zig new file mode 100644 index 00000000..b3d47c64 --- /dev/null +++ b/bin/zml-smi/bindings/neuron/metrics.zig @@ -0,0 +1,170 @@ +const std = @import("std"); +const sysfs = @import("../../utils/sysfs.zig"); +const device_info = @import("../../info/device_info.zig"); +const DeviceInfo = device_info.DeviceInfo; +const NeuronInfo = device_info.NeuronInfo; +const DoubleBuffer = @import("../../utils/double_buffer.zig").DoubleBuffer; +const Collector = @import("../../collector.zig").Collector; +const Worker = @import("../../worker.zig").Worker; +const Nrt = @import("nrt.zig"); +const process = @import("process.zig"); + +const base_path = "/sys/devices/virtual/neuron_device"; + +pub const target: device_info.Target = .neuron; + +pub fn start(collector: *Collector) !void { + const nrt = try collector.arena.create(Nrt); + nrt.* = try Nrt.init(collector.arena, collector.io); + + if (nrt.handles.len == 0) { + return; + } + + const total_nc = nrt.totalNcCount() catch return; + const nc_per_device = total_nc / @as(u32, @intCast(nrt.handles.len)); + + const GiB: u64 = 1024 * 1024 * 1024; + var neuron_infos: std.ArrayList(*DeviceInfo) = .{}; + const dev_offset: u8 = @intCast(collector.device_infos.items.len); + + for (nrt.handles, nrt.device_indexes) |dev_handle, device_idx| { + const device_type = Nrt.deviceType(dev_handle); + const mem_per_device: u64 = switch (device_type) { + .inf1 => 8 * GiB, + .inf2_trn1 => 32 * GiB, + .trn2 => 96 * GiB, + .trn3 => 144 * GiB, + else => 0, + }; + + for (0..nc_per_device) |ci| { + const poll_arena = try collector.createPollArena(); + + const dev: Device = .{ + .io = collector.io, + .arena = poll_arena, + .device_idx = @intCast(device_idx), + .core_idx = @intCast(ci), + .mem_per_device = mem_per_device, + .nc_per_device = nc_per_device, + }; + + const initial: NeuronInfo = .{ .name = dev.name(collector.arena) catch null }; + const info = try collector.addDevice(.{ .neuron = .{ .values = .{ initial, initial } } }); + try neuron_infos.append(collector.arena, info); + + try collector.worker.spawn(collector.io, pollDevice, .{ collector.io, collector.worker, &info.neuron, dev }); + } + } + + if (neuron_infos.items.len > 0) { + const processes = try collector.createProcessList(); + try process.init(collector.worker, collector.io, collector.gpa, processes, nrt, nc_per_device, neuron_infos.items, dev_offset); + } +} + +const pollDevice = Worker.pollMetrics(*DoubleBuffer(NeuronInfo), Device, metrics); + +const Device = struct { + io: std.Io, + arena: *std.heap.ArenaAllocator, + device_idx: u32, + core_idx: u32, + mem_per_device: u64 = 0, + nc_per_device: u32 = 1, + + fn devicePath(self: Device, sub_path: []const u8) ![]const u8 { + return std.fmt.allocPrint(self.arena.allocator(), base_path ++ "/neuron{d}/{s}", .{ self.device_idx, sub_path }); + } + + fn corePath(self: Device, sub_path: []const u8) ![]const u8 { + return std.fmt.allocPrint(self.arena.allocator(), base_path ++ "/neuron{d}/neuron_core{d}/{s}", .{ self.device_idx, self.core_idx, sub_path }); + } + + fn name(self: Device, allocator: std.mem.Allocator) ![]const u8 { + return sysfs.readString(allocator, self.io, try self.devicePath("info/architecture/device_name")); + } + + fn readCoreMem(self: Device, comptime subdir: []const u8) !u64 { + return sysfs.readInt(self.arena.allocator(), self.io, try self.corePath("stats/memory_usage/device_mem/" ++ subdir ++ "/total")); + } + + pub fn memTotal(self: Device) !u64 { + if (self.mem_per_device == 0) { + return error.NrtUnavailable; + } + return self.mem_per_device; + } + + // Sysfs-based metrics — RAM is shared across cores, so sum all cores on the device. + pub fn memUsed(self: Device) !u64 { + var total: u64 = 0; + for (0..self.nc_per_device) |ci| { + const path = try std.fmt.allocPrint(self.arena.allocator(), base_path ++ "/neuron{d}/neuron_core{d}/stats/memory_usage/device_mem/total", .{ self.device_idx, ci }); + total += try sysfs.readInt(self.arena.allocator(), self.io, path); + } + + return total; + } + + pub fn tensors(self: Device) !u64 { + return self.readCoreMem("tensors"); + } + + pub fn constants(self: Device) !u64 { + return self.readCoreMem("constants"); + } + + pub fn modelCode(self: Device) !u64 { + return self.readCoreMem("model_code"); + } + + pub fn sharedScratchpad(self: Device) !u64 { + return self.readCoreMem("model_shared_scratchpad"); + } + + pub fn nonsharedScratchpad(self: Device) !u64 { + return self.readCoreMem("nonshared_scratchpad"); + } + + pub fn runtime(self: Device) !u64 { + return self.readCoreMem("runtime_memory"); + } + + pub fn driver(self: Device) !u64 { + return self.readCoreMem("driver_memory"); + } + + pub fn dmaRings(self: Device) !u64 { + return self.readCoreMem("dma_rings"); + } + + pub fn collectives(self: Device) !u64 { + return self.readCoreMem("collectives"); + } + + pub fn notifications(self: Device) !u64 { + return self.readCoreMem("notifications"); + } + + pub fn uncategorized(self: Device) !u64 { + return self.readCoreMem("uncategorized"); + } +}; + +const metrics = .{ + .{ .field = "mem_total_bytes", .query = Device.memTotal }, + .{ .field = "mem_used_bytes", .query = Device.memUsed }, + .{ .field = "nc_tensors", .query = Device.tensors }, + .{ .field = "nc_constants", .query = Device.constants }, + .{ .field = "nc_model_code", .query = Device.modelCode }, + .{ .field = "nc_shared_scratchpad", .query = Device.sharedScratchpad }, + .{ .field = "nc_nonshared_scratchpad", .query = Device.nonsharedScratchpad }, + .{ .field = "nc_runtime", .query = Device.runtime }, + .{ .field = "nc_driver", .query = Device.driver }, + .{ .field = "nc_dma_rings", .query = Device.dmaRings }, + .{ .field = "nc_collectives", .query = Device.collectives }, + .{ .field = "nc_notifications", .query = Device.notifications }, + .{ .field = "nc_uncategorized", .query = Device.uncategorized }, +}; diff --git a/bin/zml-smi/bindings/neuron/nrt.zig b/bin/zml-smi/bindings/neuron/nrt.zig new file mode 100644 index 00000000..5e1e82cf --- /dev/null +++ b/bin/zml-smi/bindings/neuron/nrt.zig @@ -0,0 +1,128 @@ +const std = @import("std"); +const c = @import("c"); +const stdx = @import("stdx"); +const DynLib = @import("../dynlib.zig"); +const sandbox = @import("../../utils/sandbox.zig"); + +const Nrt = @This(); + +pub const Error = error{ nrt_error, NrtUnavailable }; +pub const AppInfo = c.struct_neuron_app_info; +pub const DeviceType = enum(c_int) { + inf1 = 1, + inf2_trn1 = 2, + trn2 = 3, + trn3 = 4, + _, +}; + +lib: Fns, +handles: []const *c.ndl_device_t, +device_indexes: []const c_int, + +const Fns = struct { + ndl_available_devices: DynLib.Fn("ndl_available_devices"), + ndl_open_device: DynLib.Fn("ndl_open_device"), + ndl_close_device: DynLib.Fn("ndl_close_device"), + ndl_get_all_apps_info: DynLib.Fn("ndl_get_all_apps_info"), + nds_open: DynLib.Fn("nds_open"), + nds_close: DynLib.Fn("nds_close"), + nds_get_nc_counter: DynLib.Fn("nds_get_nc_counter"), + nrt_get_total_nc_count: DynLib.Fn("nrt_get_total_nc_count"), +}; + +pub fn init(allocator: std.mem.Allocator, io: std.Io) Error!Nrt { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = sandbox.path(&path_buf) orelse { + std.log.err("neuron: sandbox path unavailable", .{}); + return error.NrtUnavailable; + }; + + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libnrt.so.1" }) catch { + std.log.err("neuron: failed to construct libnrt.so.1 path", .{}); + return error.NrtUnavailable; + }; + + const lib = DynLib.openElf(Fns, io, path, "nrt_init") catch |err| { + std.log.err("neuron: failed to open {s}: {s}", .{ path, @errorName(err) }); + return error.NrtUnavailable; + }; + + var dev_index_buf: [c.MAX_NEURON_DEVICE_COUNT]c_int = undefined; + const count: usize = @intCast(@max(0, lib.ndl_available_devices(&dev_index_buf, c.MAX_NEURON_DEVICE_COUNT))); + + var handle_list: std.ArrayList(*c.ndl_device_t) = .{}; + var index_list: std.ArrayList(c_int) = .{}; + + for (dev_index_buf[0..count]) |device_idx| { + var dev: ?*c.ndl_device_t = null; + var t: c.struct_ndl_device_init_param = .{ + .initialize_device = false, + .map_hbm = false, + .num_dram_regions = 0, + }; + + if (lib.ndl_open_device(device_idx, &t, &dev) == 0) { + if (dev) |d| { + handle_list.append(allocator, d) catch continue; + index_list.append(allocator, device_idx) catch continue; + } + } + } + + return .{ + .lib = lib, + .handles = handle_list.toOwnedSlice(allocator) catch return error.NrtUnavailable, + .device_indexes = index_list.toOwnedSlice(allocator) catch return error.NrtUnavailable, + }; +} + +pub fn deviceCount(self: Nrt) u32 { + return @intCast(self.handles.len); +} + +pub fn allAppsInfo(self: Nrt, dev: *c.ndl_device_t) Error!struct { ptr: ?[*]AppInfo, count: usize } { + var info: ?[*]AppInfo = null; + var count: usize = 0; + if (self.lib.ndl_get_all_apps_info(dev, &info, &count, c.APP_INFO_ALL) != 0) { + return error.nrt_error; + } + return .{ .ptr = info, .count = count }; +} + +pub fn ndsOpen(self: Nrt, dev: *c.ndl_device_t, pid: c.pid_t) Error!*c.nds_instance_t { + var inst: ?*c.nds_instance_t = null; + if (self.lib.nds_open(dev, pid, &inst) != 0) { + return error.nrt_error; + } + return inst orelse error.nrt_error; +} + +pub fn ndsClose(self: Nrt, inst: *c.nds_instance_t) void { + _ = self.lib.nds_close(inst); +} + +pub fn ncCounter(self: Nrt, inst: *c.nds_instance_t, pnc_index: c_int, counter_index: u32) Error!u64 { + var value: u64 = 0; + if (self.lib.nds_get_nc_counter(inst, pnc_index, counter_index, &value) != 0) { + return error.nrt_error; + } + return value; +} + +pub fn totalNcCount(self: Nrt) Error!u32 { + var count: u32 = 0; + if (self.lib.nrt_get_total_nc_count(&count) != 0) { + return error.nrt_error; + } + return count; +} + +pub fn hbmSize(dev: *c.ndl_device_t) usize { + return dev.hbm_size; +} + +pub fn deviceType(dev: *c.ndl_device_t) DeviceType { + return @enumFromInt(dev.device_type); +} diff --git a/bin/zml-smi/bindings/neuron/process.zig b/bin/zml-smi/bindings/neuron/process.zig new file mode 100644 index 00000000..bd895ec2 --- /dev/null +++ b/bin/zml-smi/bindings/neuron/process.zig @@ -0,0 +1,110 @@ +const std = @import("std"); +const c = @import("c"); +const Nrt = @import("nrt.zig"); +const pi = @import("../../info/process_info.zig"); +const DeviceInfo = @import("../../info/device_info.zig").DeviceInfo; +const ProcessDoubleBuffer = @import("../../utils/double_buffer.zig").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const Worker = @import("../../worker.zig").Worker; + +pub fn init(w: *Worker, io: std.Io, allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, nrt: *const Nrt, nc_per_device: u32, device_infos: []*DeviceInfo, dev_offset: u8) !void { + try w.spawn(io, pollLoop, .{ io, w, allocator, list, nrt, nc_per_device, device_infos, dev_offset }); +} + +fn pollLoop(io: std.Io, w: *const Worker, allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, nrt: *const Nrt, nc_per_device: u32, device_infos: []*DeviceInfo, dev_offset: u8) void { + const interval: std.Io.Duration = .fromMilliseconds(w.poll_interval_ms); + io.sleep(interval, .awake) catch {}; + + if (nrt.handles.len == 0) { + return; + } + + while (w.isRunning()) { + const start: std.Io.Timestamp = .now(io, .awake); + const back = list.back(); + + back.clearRetainingCapacity(); + + for (nrt.handles, 0..) |handle, dev_i| { + const apps_result = nrt.allAppsInfo(handle) catch continue; + defer { + if (apps_result.ptr) |ptr| { + std.c.free(@ptrCast(ptr)); + } + } + + const apps = if (apps_result.ptr) |ptr| ptr[0..apps_result.count] else continue; + + var total_us_per_core: [c.MAX_NC_PER_DEVICE]u64 = .{0} ** c.MAX_NC_PER_DEVICE; + + for (apps) |*app| { + if (app.pid <= 0) { + continue; + } + + const nds = nrt.ndsOpen(handle, app.pid) catch continue; + defer nrt.ndsClose(nds); + + for (0..nc_per_device) |ci| { + const time_in_use = nrt.ncCounter(nds, @intCast(ci), c.NDS_NC_COUNTER_TIME_IN_USE) catch continue; + total_us_per_core[ci] += time_in_use; + + if (time_in_use == 0) { + continue; + } + + const dev_idx: u32 = @intCast(dev_i * nc_per_device + ci); + + var info: pi.ProcessInfo = .{ + .pid = @bitCast(app.pid), + .device_idx = @intCast(dev_idx + dev_offset), + .dev_mem_kib = @intCast(app.device_mem_size / 1024), + }; + + if (dev_idx < device_infos.len) { + if (device_infos[dev_idx].neuron.front().util_percent) |util| { + info.dev_util_percent = @intCast(util); + } + } + + if (info.dev_mem_kib != null and info.dev_mem_kib.? > 0) + back.append(allocator, info) catch break; + } + } + + const now: std.Io.Timestamp = .now(io, .awake); + + for (0..nc_per_device) |ci| { + const dev_idx = dev_i * nc_per_device + ci; + if (dev_idx >= device_infos.len) { + break; + } + + const ni_back = device_infos[dev_idx].neuron.back(); + ni_back.* = device_infos[dev_idx].neuron.front().*; + const prev_us = ni_back.prev_total_us; + + ni_back.prev_total_us = total_us_per_core[ci]; + + if (ni_back.prev_timestamp) |prev| { + const elapsed = prev.untilNow(io, .awake); + const elapsed_us: u64 = @intCast(@divFloor(elapsed.nanoseconds, std.time.ns_per_us)); + + if (elapsed_us > 0) { + const delta_us = total_us_per_core[ci] -| prev_us; + ni_back.util_percent = @min(100, delta_us * 100 / elapsed_us); + } + } + + ni_back.prev_timestamp = now; + device_infos[dev_idx].neuron.swap(); + } + } + + list.swap(); + + const elapsed = start.untilNow(io, .awake); + if (elapsed.nanoseconds < interval.nanoseconds) { + io.sleep(.fromNanoseconds(interval.nanoseconds - elapsed.nanoseconds), .awake) catch {}; + } + } +} diff --git a/bin/zml-smi/bindings/nvml/metrics.zig b/bin/zml-smi/bindings/nvml/metrics.zig new file mode 100644 index 00000000..082d4761 --- /dev/null +++ b/bin/zml-smi/bindings/nvml/metrics.zig @@ -0,0 +1,151 @@ +const std = @import("std"); +const Nvml = @import("nvml.zig"); +const device_info = @import("../../info/device_info.zig"); +const DeviceInfo = device_info.DeviceInfo; +const GpuInfo = device_info.GpuInfo; +const DoubleBuffer = @import("../../utils/double_buffer.zig").DoubleBuffer; +const Collector = @import("../../collector.zig").Collector; +const Worker = @import("../../worker.zig").Worker; +const process = @import("process.zig"); + +pub const target: device_info.Target = .cuda; + +pub fn start(collector: *Collector) !void { + const nvml = try collector.arena.create(Nvml); + nvml.* = try Nvml.init(); + + const count = try nvml.deviceCount(); + const dev_offset: u8 = @intCast(collector.device_infos.items.len); + + for (0..count) |i| { + const dev = Device.open(nvml, @intCast(i)) catch continue; + const initial: GpuInfo = .{ .name = dev.name(collector.arena) catch null }; + const info = try collector.addDevice(.{ .cuda = .{ .values = .{ initial, initial } } }); + try collector.worker.spawn(collector.io, pollDevice, .{ collector.io, collector.worker, &info.cuda, dev }); + } + + const processes = try collector.createProcessList(); + try process.init(collector.worker, collector.io, collector.gpa, processes, nvml, dev_offset); +} + +const pollDevice = Worker.pollMetrics(*DoubleBuffer(GpuInfo), Device, metrics); + +const Device = struct { + nvml: *const Nvml, + handle: Nvml.Handle, + + pub fn open(nvml: *const Nvml, index: u32) !Device { + return .{ .nvml = nvml, .handle = try nvml.handleByIndex(index) }; + } + + fn name(self: Device, arena: std.mem.Allocator) ![]const u8 { + var buf: [256]u8 = .{0} ** 256; + const slice = try self.nvml.name(self.handle, &buf); + + return try arena.dupe(u8, slice); + } + + // Power + pub fn powerUsage(self: Device) !u64 { + return @intCast(try self.nvml.powerUsage(self.handle)); + } + + pub fn powerLimit(self: Device) !u64 { + return @intCast(try self.nvml.powerLimit(self.handle)); + } + + // Thermal + pub fn temperature(self: Device) !u64 { + return @intCast(try self.nvml.temperature(self.handle)); + } + + pub fn fanSpeed(self: Device) !u64 { + return @intCast(try self.nvml.fanSpeed(self.handle)); + } + + // Utilization + pub fn gpuUtil(self: Device) !u64 { + return @intCast(try self.nvml.utilizationGpu(self.handle)); + } + + pub fn encoderUtil(self: Device) !u64 { + return @intCast(try self.nvml.encoderUtil(self.handle)); + } + + pub fn decoderUtil(self: Device) !u64 { + return @intCast(try self.nvml.decoderUtil(self.handle)); + } + + // Clocks + pub fn clockGraphics(self: Device) !u64 { + return @intCast(try self.nvml.clockGraphics(self.handle)); + } + + pub fn clockSm(self: Device) !u64 { + return @intCast(try self.nvml.clockSm(self.handle)); + } + + pub fn clockMem(self: Device) !u64 { + return @intCast(try self.nvml.clockMem(self.handle)); + } + + pub fn maxClockGraphics(self: Device) !u64 { + return @intCast(try self.nvml.maxClockGraphics(self.handle)); + } + + pub fn maxClockMem(self: Device) !u64 { + return @intCast(try self.nvml.maxClockMem(self.handle)); + } + + // Memory + pub fn memUsed(self: Device) !u64 { + return self.nvml.memUsed(self.handle); + } + + pub fn memTotal(self: Device) !u64 { + return self.nvml.memTotal(self.handle); + } + + pub fn memBusWidth(self: Device) !u64 { + return @intCast(try self.nvml.memBusWidth(self.handle)); + } + + // PCIe + pub fn pcieTx(self: Device) !u64 { + return @intCast(try self.nvml.pcieTxKBps(self.handle)); + } + + pub fn pcieRx(self: Device) !u64 { + return @intCast(try self.nvml.pcieRxKBps(self.handle)); + } + + pub fn pcieLinkGen(self: Device) !u64 { + return @intCast(try self.nvml.pcieLinkGen(self.handle)); + } + + pub fn pcieLinkWidth(self: Device) !u64 { + return @intCast(try self.nvml.pcieLinkWidth(self.handle)); + } +}; + +const metrics = .{ + .{ .field = "power_mw", .query = Device.powerUsage }, + .{ .field = "power_limit_mw", .query = Device.powerLimit }, + .{ .field = "temperature", .query = Device.temperature }, + .{ .field = "fan_speed_percent", .query = Device.fanSpeed }, + .{ .field = "util_percent", .query = Device.gpuUtil }, + .{ .field = "encoder_util_percent", .query = Device.encoderUtil }, + .{ .field = "decoder_util_percent", .query = Device.decoderUtil }, + .{ .field = "clock_graphics_mhz", .query = Device.clockGraphics }, + .{ .field = "clock_sm_mhz", .query = Device.clockSm }, + .{ .field = "clock_mem_mhz", .query = Device.clockMem }, + .{ .field = "clock_graphics_max_mhz", .query = Device.maxClockGraphics }, + .{ .field = "clock_mem_max_mhz", .query = Device.maxClockMem }, + .{ .field = "mem_used_bytes", .query = Device.memUsed }, + .{ .field = "mem_total_bytes", .query = Device.memTotal }, + .{ .field = "mem_bus_width", .query = Device.memBusWidth }, + .{ .field = "pcie_tx_kbps", .query = Device.pcieTx }, + .{ .field = "pcie_rx_kbps", .query = Device.pcieRx }, + .{ .field = "pcie_link_gen", .query = Device.pcieLinkGen }, + .{ .field = "pcie_link_width", .query = Device.pcieLinkWidth }, +}; diff --git a/bin/zml-smi/bindings/nvml/nvml.zig b/bin/zml-smi/bindings/nvml/nvml.zig new file mode 100644 index 00000000..50d9d794 --- /dev/null +++ b/bin/zml-smi/bindings/nvml/nvml.zig @@ -0,0 +1,316 @@ +const std = @import("std"); +const c = @import("c"); +const DynLib = @import("../dynlib.zig"); + +const Nvml = @This(); + +pub const Error = ReturnError || error{NvmlUnavailable}; +pub const Handle = c.nvmlDevice_t; +pub const ProcessInfo_t = c.nvmlProcessInfo_t; + +lib: Fns, + +const Fns = struct { + nvmlInit_v2: DynLib.Fn("nvmlInit_v2"), + nvmlDeviceGetHandleByIndex_v2: DynLib.Fn("nvmlDeviceGetHandleByIndex_v2"), + nvmlDeviceGetCount_v2: DynLib.Fn("nvmlDeviceGetCount_v2"), + nvmlDeviceGetName: DynLib.Fn("nvmlDeviceGetName"), + nvmlDeviceGetPowerUsage: DynLib.Fn("nvmlDeviceGetPowerUsage"), + nvmlDeviceGetTemperature: DynLib.Fn("nvmlDeviceGetTemperature"), + nvmlDeviceGetUtilizationRates: DynLib.Fn("nvmlDeviceGetUtilizationRates"), + nvmlDeviceGetClockInfo: DynLib.Fn("nvmlDeviceGetClockInfo"), + nvmlDeviceGetMaxClockInfo: DynLib.Fn("nvmlDeviceGetMaxClockInfo"), + nvmlDeviceGetMemoryInfo: DynLib.Fn("nvmlDeviceGetMemoryInfo"), + nvmlDeviceGetFanSpeed: DynLib.Fn("nvmlDeviceGetFanSpeed"), + nvmlDeviceGetEnforcedPowerLimit: DynLib.Fn("nvmlDeviceGetEnforcedPowerLimit"), + nvmlDeviceGetPcieThroughput: DynLib.Fn("nvmlDeviceGetPcieThroughput"), + nvmlDeviceGetEncoderUtilization: DynLib.Fn("nvmlDeviceGetEncoderUtilization"), + nvmlDeviceGetDecoderUtilization: DynLib.Fn("nvmlDeviceGetDecoderUtilization"), + nvmlDeviceGetCurrPcieLinkGeneration: DynLib.Fn("nvmlDeviceGetCurrPcieLinkGeneration"), + nvmlDeviceGetCurrPcieLinkWidth: DynLib.Fn("nvmlDeviceGetCurrPcieLinkWidth"), + nvmlDeviceGetMemoryBusWidth: DynLib.Fn("nvmlDeviceGetMemoryBusWidth"), + nvmlDeviceGetComputeRunningProcesses_v3: DynLib.Fn("nvmlDeviceGetComputeRunningProcesses_v3"), + nvmlDeviceGetGraphicsRunningProcesses_v3: DynLib.Fn("nvmlDeviceGetGraphicsRunningProcesses_v3"), + nvmlDeviceGetProcessUtilization: DynLib.Fn("nvmlDeviceGetProcessUtilization"), +}; + +pub fn init() Error!Nvml { + const fns = DynLib.open(Fns, "libnvidia-ml.so.1") orelse + DynLib.open(Fns, "libnvidia-ml.so") orelse + return error.NvmlUnavailable; + try check(fns.nvmlInit_v2()); + return .{ .lib = fns }; +} + +pub fn handleByIndex(self: Nvml, device_id: u32) Error!c.nvmlDevice_t { + var handle: c.nvmlDevice_t = undefined; + try check(self.lib.nvmlDeviceGetHandleByIndex_v2(device_id, &handle)); + return handle; +} + +pub fn deviceCount(self: Nvml) Error!u32 { + var count: c_uint = 0; + try check(self.lib.nvmlDeviceGetCount_v2(&count)); + return @intCast(count); +} + +pub fn name(self: Nvml, handle: c.nvmlDevice_t, buf: *[256]u8) Error![:0]const u8 { + try check(self.lib.nvmlDeviceGetName(handle, buf, buf.len)); + return std.mem.sliceTo(@as([*:0]const u8, @ptrCast(buf)), 0); +} + +pub fn powerUsage(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var power_mw: c_uint = 0; + try check(self.lib.nvmlDeviceGetPowerUsage(handle, &power_mw)); + return power_mw; +} + +pub fn temperature(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var temp: c_uint = 0; + try check(self.lib.nvmlDeviceGetTemperature(handle, c.NVML_TEMPERATURE_GPU, &temp)); + return temp; +} + +pub fn utilizationGpu(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var util: c.nvmlUtilization_t = undefined; + try check(self.lib.nvmlDeviceGetUtilizationRates(handle, &util)); + return util.gpu; +} + +pub fn clockGraphics(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var clock: c_uint = 0; + try check(self.lib.nvmlDeviceGetClockInfo(handle, c.NVML_CLOCK_GRAPHICS, &clock)); + return clock; +} + +pub fn clockSm(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var clock: c_uint = 0; + try check(self.lib.nvmlDeviceGetClockInfo(handle, c.NVML_CLOCK_SM, &clock)); + return clock; +} + +pub fn clockMem(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var clock: c_uint = 0; + try check(self.lib.nvmlDeviceGetClockInfo(handle, c.NVML_CLOCK_MEM, &clock)); + return clock; +} + +pub fn maxClockGraphics(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var clock: c_uint = 0; + try check(self.lib.nvmlDeviceGetMaxClockInfo(handle, c.NVML_CLOCK_GRAPHICS, &clock)); + return clock; +} + +pub fn maxClockMem(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var clock: c_uint = 0; + try check(self.lib.nvmlDeviceGetMaxClockInfo(handle, c.NVML_CLOCK_MEM, &clock)); + return clock; +} + +pub fn memTotal(self: Nvml, handle: c.nvmlDevice_t) Error!u64 { + var mem: c.nvmlMemory_t = undefined; + try check(self.lib.nvmlDeviceGetMemoryInfo(handle, &mem)); + return @intCast(mem.total); +} + +pub fn memUsed(self: Nvml, handle: c.nvmlDevice_t) Error!u64 { + var mem: c.nvmlMemory_t = undefined; + try check(self.lib.nvmlDeviceGetMemoryInfo(handle, &mem)); + return @intCast(mem.used); +} + +pub fn fanSpeed(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var speed: c_uint = 0; + try check(self.lib.nvmlDeviceGetFanSpeed(handle, &speed)); + return speed; +} + +pub fn powerLimit(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var limit: c_uint = 0; + try check(self.lib.nvmlDeviceGetEnforcedPowerLimit(handle, &limit)); + return limit; +} + +pub fn pcieTxKBps(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var value: c_uint = 0; + try check(self.lib.nvmlDeviceGetPcieThroughput(handle, c.NVML_PCIE_UTIL_TX_BYTES, &value)); + return value; +} + +pub fn pcieRxKBps(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var value: c_uint = 0; + try check(self.lib.nvmlDeviceGetPcieThroughput(handle, c.NVML_PCIE_UTIL_RX_BYTES, &value)); + return value; +} + +pub fn encoderUtil(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var util: c_uint = 0; + var sampling: c_uint = 0; + try check(self.lib.nvmlDeviceGetEncoderUtilization(handle, &util, &sampling)); + return util; +} + +pub fn decoderUtil(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var util: c_uint = 0; + var sampling: c_uint = 0; + try check(self.lib.nvmlDeviceGetDecoderUtilization(handle, &util, &sampling)); + return util; +} + +pub fn pcieLinkGen(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var gen: c_uint = 0; + try check(self.lib.nvmlDeviceGetCurrPcieLinkGeneration(handle, &gen)); + return gen; +} + +pub fn pcieLinkWidth(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var width: c_uint = 0; + try check(self.lib.nvmlDeviceGetCurrPcieLinkWidth(handle, &width)); + return width; +} + +pub fn memBusWidth(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var width: c_uint = 0; + try check(self.lib.nvmlDeviceGetMemoryBusWidth(handle, &width)); + return width; +} + +pub fn computeRunningProcesses(self: Nvml, allocator: std.mem.Allocator, handle: c.nvmlDevice_t) (Error || error{OutOfMemory})![]c.nvmlProcessInfo_t { + var count: c_uint = 0; + const ret = self.lib.nvmlDeviceGetComputeRunningProcesses_v3(handle, &count, null); + if (ret != c.NVML_ERROR_INSUFFICIENT_SIZE) { + if (ret == c.NVML_SUCCESS) { + return &.{}; + } + try check(ret); + } + if (count == 0) { + return &.{}; + } + const infos = try allocator.alloc(c.nvmlProcessInfo_t, count); + errdefer allocator.free(infos); + try check(self.lib.nvmlDeviceGetComputeRunningProcesses_v3(handle, &count, @ptrCast(infos.ptr))); + return infos; +} + +pub fn graphicsRunningProcesses(self: Nvml, allocator: std.mem.Allocator, handle: c.nvmlDevice_t) (Error || error{OutOfMemory})![]const c.nvmlProcessInfo_t { + var count: c_uint = 0; + const ret = self.lib.nvmlDeviceGetGraphicsRunningProcesses_v3(handle, &count, null); + + if (ret != c.NVML_ERROR_INSUFFICIENT_SIZE) { + if (ret == c.NVML_SUCCESS) { + return &.{}; + } + try check(ret); + } + + if (count == 0) { + return &.{}; + } + + const infos = try allocator.alloc(c.nvmlProcessInfo_t, count); + errdefer allocator.free(infos); + + try check(self.lib.nvmlDeviceGetGraphicsRunningProcesses_v3(handle, &count, @ptrCast(infos.ptr))); + + return infos; +} + +pub fn processUtilization(self: Nvml, allocator: std.mem.Allocator, handle: c.nvmlDevice_t, last_seen: u64) (Error || error{OutOfMemory})![]const c.nvmlProcessUtilizationSample_t { + var count: c_uint = 0; + const ret = self.lib.nvmlDeviceGetProcessUtilization(handle, null, &count, last_seen); + + if (ret != c.NVML_ERROR_INSUFFICIENT_SIZE) { + if (ret == c.NVML_SUCCESS) { + return &.{}; + } + try check(ret); + } + + if (count == 0) { + return &.{}; + } + + const samples = try allocator.alloc(c.nvmlProcessUtilizationSample_t, count); + errdefer allocator.free(samples); + + try check(self.lib.nvmlDeviceGetProcessUtilization(handle, @ptrCast(samples.ptr), &count, last_seen)); + + return samples; +} + +// Error handling + +pub const ReturnError = error{ + error_uninitialized, + error_invalid_argument, + error_not_supported, + error_no_permission, + error_already_initialized, + error_not_found, + error_insufficient_size, + error_insufficient_power, + error_driver_not_loaded, + error_timeout, + error_irq_issue, + error_library_not_found, + error_function_not_found, + error_corrupted_inforom, + error_gpu_is_lost, + error_reset_required, + error_operating_system, + error_lib_rm_version_mismatch, + error_in_use, + error_memory, + error_no_data, + error_vgpu_ecc_not_supported, + error_insufficient_resources, + error_freq_not_supported, + error_argument_version_mismatch, + error_deprecated, + error_not_ready, + error_gpu_not_found, + error_invalid_state, + error_reset_type_not_supported, + error_unknown, +}; + +fn check(ret: c_uint) Error!void { + if (ret == c.NVML_SUCCESS) { + return; + } + return switch (ret) { + c.NVML_ERROR_UNINITIALIZED => error.error_uninitialized, + c.NVML_ERROR_INVALID_ARGUMENT => error.error_invalid_argument, + c.NVML_ERROR_NOT_SUPPORTED => error.error_not_supported, + c.NVML_ERROR_NO_PERMISSION => error.error_no_permission, + c.NVML_ERROR_ALREADY_INITIALIZED => error.error_already_initialized, + c.NVML_ERROR_NOT_FOUND => error.error_not_found, + c.NVML_ERROR_INSUFFICIENT_SIZE => error.error_insufficient_size, + c.NVML_ERROR_INSUFFICIENT_POWER => error.error_insufficient_power, + c.NVML_ERROR_DRIVER_NOT_LOADED => error.error_driver_not_loaded, + c.NVML_ERROR_TIMEOUT => error.error_timeout, + c.NVML_ERROR_IRQ_ISSUE => error.error_irq_issue, + c.NVML_ERROR_LIBRARY_NOT_FOUND => error.error_library_not_found, + c.NVML_ERROR_FUNCTION_NOT_FOUND => error.error_function_not_found, + c.NVML_ERROR_CORRUPTED_INFOROM => error.error_corrupted_inforom, + c.NVML_ERROR_GPU_IS_LOST => error.error_gpu_is_lost, + c.NVML_ERROR_RESET_REQUIRED => error.error_reset_required, + c.NVML_ERROR_OPERATING_SYSTEM => error.error_operating_system, + c.NVML_ERROR_LIB_RM_VERSION_MISMATCH => error.error_lib_rm_version_mismatch, + c.NVML_ERROR_IN_USE => error.error_in_use, + c.NVML_ERROR_MEMORY => error.error_memory, + c.NVML_ERROR_NO_DATA => error.error_no_data, + c.NVML_ERROR_VGPU_ECC_NOT_SUPPORTED => error.error_vgpu_ecc_not_supported, + c.NVML_ERROR_INSUFFICIENT_RESOURCES => error.error_insufficient_resources, + c.NVML_ERROR_FREQ_NOT_SUPPORTED => error.error_freq_not_supported, + c.NVML_ERROR_ARGUMENT_VERSION_MISMATCH => error.error_argument_version_mismatch, + c.NVML_ERROR_DEPRECATED => error.error_deprecated, + c.NVML_ERROR_NOT_READY => error.error_not_ready, + c.NVML_ERROR_GPU_NOT_FOUND => error.error_gpu_not_found, + c.NVML_ERROR_INVALID_STATE => error.error_invalid_state, + c.NVML_ERROR_RESET_TYPE_NOT_SUPPORTED => error.error_reset_type_not_supported, + c.NVML_ERROR_UNKNOWN => error.error_unknown, + else => error.error_unknown, + }; +} diff --git a/bin/zml-smi/bindings/nvml/process.zig b/bin/zml-smi/bindings/nvml/process.zig new file mode 100644 index 00000000..510d72fe --- /dev/null +++ b/bin/zml-smi/bindings/nvml/process.zig @@ -0,0 +1,87 @@ +const std = @import("std"); +const Nvml = @import("nvml.zig"); +const pi = @import("../../info/process_info.zig"); +const ProcessDoubleBuffer = @import("../../utils/double_buffer.zig").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const Worker = @import("../../worker.zig").Worker; + +pub fn init(w: *Worker, io: std.Io, allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, nvml: *const Nvml, dev_offset: u8) !void { + try w.spawn(io, pollLoop, .{ io, w, allocator, list, nvml, dev_offset }); +} + +fn pollLoop(io: std.Io, w: *const Worker, allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, nvml: *const Nvml, dev_offset: u8) void { + const interval: std.Io.Duration = .fromMilliseconds(w.poll_interval_ms); + io.sleep(interval, .awake) catch {}; + + const device_count = nvml.deviceCount() catch 0; + const last_seen_ts = allocator.alloc(u64, device_count) catch return; + defer allocator.free(last_seen_ts); + @memset(last_seen_ts, 0); + + while (w.isRunning()) { + const start: std.Io.Timestamp = .now(io, .awake); + const back = list.back(); + + back.clearRetainingCapacity(); + + for (0..device_count) |dev_idx| { + const handle = nvml.handleByIndex(@intCast(dev_idx)) catch continue; + const idx: u8 = @intCast(dev_idx + dev_offset); + + const compute = nvml.computeRunningProcesses(allocator, handle) catch &.{}; + defer { + if (compute.len > 0) { + allocator.free(compute); + } + } + collectFromQuery(allocator, back, idx, compute); + + const graphics = nvml.graphicsRunningProcesses(allocator, handle) catch &.{}; + defer { + if (graphics.len > 0) { + allocator.free(graphics); + } + } + collectFromQuery(allocator, back, idx, graphics); + + // Apply utilization samples + const last_ts = last_seen_ts[dev_idx]; + const utils = nvml.processUtilization(allocator, handle, last_ts) catch continue; + defer { + if (utils.len > 0) { + allocator.free(utils); + } + } + + for (utils) |sample| { + if (sample.smUtil > 100 or sample.timeStamp <= last_ts) { + continue; + } + + last_seen_ts[dev_idx] = @max(last_seen_ts[dev_idx], sample.timeStamp); + + for (back.items) |*entry| { + if (entry.pid == sample.pid and entry.device_idx == idx) { + entry.dev_util_percent = @intCast(sample.smUtil); + } + } + } + } + + list.swap(); + + const elapsed = start.untilNow(io, .awake); + if (elapsed.nanoseconds < interval.nanoseconds) { + io.sleep(.fromNanoseconds(interval.nanoseconds - elapsed.nanoseconds), .awake) catch {}; + } + } +} + +fn collectFromQuery(allocator: std.mem.Allocator, back: *std.ArrayList(pi.ProcessInfo), dev_idx: u8, procs: []const Nvml.ProcessInfo_t) void { + for (procs) |gp| { + back.append(allocator, .{ + .pid = gp.pid, + .device_idx = dev_idx, + .dev_mem_kib = @intCast(gp.usedGpuMemory / 1024), + }) catch return; + } +} diff --git a/bin/zml-smi/bindings/tpu/libtpu/tpuinfo.cc b/bin/zml-smi/bindings/tpu/libtpu/tpuinfo.cc new file mode 100644 index 00000000..a1e3fbda --- /dev/null +++ b/bin/zml-smi/bindings/tpu/libtpu/tpuinfo.cc @@ -0,0 +1,84 @@ +#include "tpuinfo.h" + +#include +#include +#include +#include + +#include + +#include "tpu_metric_service.grpc.pb.h" +#include "tpu_metric_service.pb.h" + +using tpu::monitoring::runtime::MetricRequest; +using tpu::monitoring::runtime::MetricResponse; +using tpu::monitoring::runtime::RuntimeMetricService; + +static MetricResponse FetchMetric(const char* address, const char* metric_name, + bool* ok) { + auto channel = + grpc::CreateChannel(address, grpc::InsecureChannelCredentials()); + auto stub = RuntimeMetricService::NewStub(channel); + + grpc::ClientContext ctx; + ctx.set_deadline(std::chrono::system_clock::now() + std::chrono::seconds(1)); + + MetricRequest req; + req.set_metric_name(metric_name); + + MetricResponse resp; + auto status = stub->GetRuntimeMetric(&ctx, req, &resp); + *ok = status.ok(); + + return resp; +} + +extern "C" int tpu_query_int(const char* address, const char* metric_name, + long long* device_ids, long long* values, + int max_n) { + bool ok; + auto resp = FetchMetric(address, metric_name, &ok); + if (!ok) return -1; + + std::map sorted; + for (const auto& m : resp.metric().metrics()) { + int id = static_cast(m.attribute().value().int_attr()); + sorted[id] = m.gauge().as_int(); + } + + int i = 0; + for (const auto& [id, val] : sorted) { + if (i >= max_n) break; + + device_ids[i] = id; + values[i] = val; + ++i; + } + + return i; +} + +extern "C" int tpu_query_double(const char* address, const char* metric_name, + long long* device_ids, double* values, + int max_n) { + bool ok; + auto resp = FetchMetric(address, metric_name, &ok); + if (!ok) return -1; + + std::map sorted; + for (const auto& m : resp.metric().metrics()) { + int id = static_cast(m.attribute().value().int_attr()); + sorted[id] = m.gauge().as_double(); + } + + int i = 0; + for (const auto& [id, val] : sorted) { + if (i >= max_n) break; + + device_ids[i] = id; + values[i] = val; + ++i; + } + + return i; +} diff --git a/bin/zml-smi/bindings/tpu/libtpu/tpuinfo.h b/bin/zml-smi/bindings/tpu/libtpu/tpuinfo.h new file mode 100644 index 00000000..f2dc7db6 --- /dev/null +++ b/bin/zml-smi/bindings/tpu/libtpu/tpuinfo.h @@ -0,0 +1,20 @@ +#ifndef TPUINFO_H_ +#define TPUINFO_H_ + +#define TPUINFO_MAX_DEVICES 64 + +#ifdef __cplusplus +extern "C" { +#endif + +int tpu_query_int(const char* address, const char* metric_name, + long long* device_ids, long long* values, int max_n); + +int tpu_query_double(const char* address, const char* metric_name, + long long* device_ids, double* values, int max_n); + +#ifdef __cplusplus +} +#endif + +#endif /* TPUINFO_H_ */ diff --git a/bin/zml-smi/bindings/tpu/metrics.zig b/bin/zml-smi/bindings/tpu/metrics.zig new file mode 100644 index 00000000..e7c2e531 --- /dev/null +++ b/bin/zml-smi/bindings/tpu/metrics.zig @@ -0,0 +1,175 @@ +const std = @import("std"); +const sysfs = @import("../../utils/sysfs.zig"); +const tpuinfo = @import("tpuinfo.zig"); +const device_info = @import("../../info/device_info.zig"); +const DeviceInfo = device_info.DeviceInfo; +const TpuInfo = device_info.TpuInfo; +const Collector = @import("../../collector.zig").Collector; +const Worker = @import("../../worker.zig").Worker; +const tpu_process = @import("process.zig"); + +const address = "localhost:8431"; + +pub const target: device_info.Target = .tpu; + +pub fn start(collector: *Collector) !void { + const chip = scanPciChips(collector.arena, collector.io) orelse return error.TpuUnavailable; + const device_count = chip.chip_count * chip.devices_per_chip; + var tpu_infos: std.ArrayList(*DeviceInfo) = .{}; + const dev_offset: u8 = @intCast(collector.device_infos.items.len); + + for (0..device_count) |_| { + const chip_name = try collector.arena.dupe(u8, chip.name); + const initial: TpuInfo = .{ .name = chip_name }; + const info = try collector.addDevice(.{ .tpu = .{ .values = .{ initial, initial } } }); + try tpu_infos.append(collector.arena, info); + } + + try collector.worker.spawn(collector.io, pollAllDevices, .{ collector.io, collector.worker, tpu_infos.items }); + + if (tpu_infos.items.len > 0) { + const processes = try collector.createProcessList(); + try tpu_process.init(collector.worker, collector.io, collector.gpa, chip.devices_per_chip, tpu_infos.items, processes, dev_offset); + } +} + +const MetricKind = enum { int, double }; + +fn pollAllDevices(io: std.Io, w: *const Worker, infos: []*DeviceInfo) void { + w.pollLoop(io, struct { + fn poll(devs: []*DeviceInfo) void { + var mem_used: [tpuinfo.max_devices]?u64 = undefined; + var mem_total: [tpuinfo.max_devices]?u64 = undefined; + var util: [tpuinfo.max_devices]?u64 = undefined; + + for (devs, 0..) |info, i| { + const front = info.tpu.front(); + mem_used[i] = front.mem_used_bytes; + mem_total[i] = front.mem_total_bytes; + util[i] = front.util_percent; + } + + queryRaw("tpu.runtime.hbm.memory.usage.bytes", .int, mem_used[0..devs.len]); + queryRaw("tpu.runtime.hbm.memory.total.bytes", .int, mem_total[0..devs.len]); + queryRaw("tpu.runtime.tensorcore.dutycycle.percent", .double, util[0..devs.len]); + + for (devs, 0..) |info, i| { + const back = info.tpu.back(); + back.* = info.tpu.front().*; + back.mem_used_bytes = mem_used[i]; + back.mem_total_bytes = mem_total[i]; + back.util_percent = util[i]; + info.tpu.swap(); + } + } + }.poll, .{infos}); +} + +fn queryRaw(metric_name: [:0]const u8, comptime kind: MetricKind, out: []?u64) void { + var ids: [tpuinfo.max_devices]c_longlong = undefined; + switch (kind) { + .int => { + var vals: [tpuinfo.max_devices]c_longlong = undefined; + const n = tpuinfo.queryInt(address, metric_name, &ids, &vals) catch { + @memset(out, null); + return; + }; + + for (ids[0..n], vals[0..n]) |id, val| { + if (id >= 0 and id < out.len) out[@intCast(id)] = @intCast(val); + } + }, + .double => { + var vals: [tpuinfo.max_devices]f64 = undefined; + const n = tpuinfo.queryDouble(address, metric_name, &ids, &vals) catch { + @memset(out, null); + return; + }; + + for (ids[0..n], vals[0..n]) |id, val| { + if (id >= 0 and id < out.len and std.math.isFinite(val)) + out[@intCast(id)] = @intFromFloat(@round(val)); + } + }, + } +} + +// -- Discover available chips +// We need to scan /sys/bus/pci/devices for Google TPU chips. +// From: https://github.com/AI-Hypercomputer/cloud-accelerator-diagnostics/blob/c94f463dd3bf99eaecf4f5af46f87cc2a5ff1b22/tpu_info/tpu_info/device.py#L43-L92 + +const ChipInfo = struct { + name: []const u8, + devices_per_chip: u32, + chip_count: u32, +}; + +const google_pci_vendor_id = "0x1ae0"; +const pci_base = "/sys/bus/pci/devices"; + +const ChipEntry = struct { + device_id: []const u8, + subsystem_id: ?[]const u8 = null, + name: []const u8, + devices_per_chip: u32, +}; + +const chip_table = [_]ChipEntry{ + .{ .device_id = "0x0027", .subsystem_id = "0x004e", .name = "TPU v2", .devices_per_chip = 2 }, + .{ .device_id = "0x0027", .subsystem_id = "0x004f", .name = "TPU v3", .devices_per_chip = 2 }, + .{ .device_id = "0x005e", .name = "TPU v4", .devices_per_chip = 1 }, + .{ .device_id = "0x0063", .name = "TPU v5e", .devices_per_chip = 1 }, + .{ .device_id = "0x0062", .name = "TPU v5p", .devices_per_chip = 1 }, + .{ .device_id = "0x006f", .name = "TPU v6e", .devices_per_chip = 1 }, + .{ .device_id = "0x0076", .name = "TPU7x", .devices_per_chip = 2 }, +}; + +fn scanPciChips(allocator: std.mem.Allocator, io: std.Io) ?ChipInfo { + var pci_dir = std.Io.Dir.openDir(.cwd(), io, pci_base, .{ .iterate = true }) catch return null; + defer pci_dir.close(io); + + var it = pci_dir.iterate(); + var chip_count: u32 = 0; + var result: ChipInfo = undefined; + + while (it.next(io) catch null) |entry| { + const vendor = readSysfs(allocator, io, entry.name, "vendor") catch continue; + if (!std.mem.eql(u8, vendor, google_pci_vendor_id)) { + continue; + } + + const device_raw = readSysfs(allocator, io, entry.name, "device") catch continue; + const subsystem_raw: ?[]const u8 = readSysfs(allocator, io, entry.name, "subsystem_device") catch null; + + for (chip_table) |chip| { + const dev_match = std.mem.eql(u8, device_raw, chip.device_id); + const sub_match = if (chip.subsystem_id) |expected_sub| + if (subsystem_raw) |actual_sub| std.mem.eql(u8, actual_sub, expected_sub) else false + else + true; + + if (dev_match and sub_match) { + result = .{ + .name = chip.name, + .devices_per_chip = chip.devices_per_chip, + .chip_count = undefined, + }; + + chip_count += 1; + break; + } + } + } + + if (chip_count > 0) { + result.chip_count = chip_count; + return result; + } + + return null; +} + +fn readSysfs(allocator: std.mem.Allocator, io: std.Io, slot: []const u8, file: []const u8) ![]const u8 { + const path = try std.fmt.allocPrint(allocator, pci_base ++ "/{s}/{s}", .{ slot, file }); + return sysfs.readString(allocator, io, path); +} diff --git a/bin/zml-smi/bindings/tpu/process.zig b/bin/zml-smi/bindings/tpu/process.zig new file mode 100644 index 00000000..21553449 --- /dev/null +++ b/bin/zml-smi/bindings/tpu/process.zig @@ -0,0 +1,84 @@ +const std = @import("std"); +const pi = @import("../../info/process_info.zig"); +const ProcessDoubleBuffer = @import("../../utils/double_buffer.zig").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const DeviceInfo = @import("../../info/device_info.zig").DeviceInfo; +const Worker = @import("../../worker.zig").Worker; + +pub fn init(w: *Worker, io: std.Io, allocator: std.mem.Allocator, devices_per_chip: u32, device_infos: []*DeviceInfo, list: *ProcessDoubleBuffer, dev_offset: u8) !void { + try w.spawn(io, scanLoop, .{ io, w, allocator, list, devices_per_chip, device_infos, dev_offset }); +} + +fn scanLoop(io: std.Io, w: *const Worker, allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, devices_per_chip: u32, device_infos: []*DeviceInfo, dev_offset: u8) void { + const interval: std.Io.Duration = .fromMilliseconds(w.poll_interval_ms); + + while (w.isRunning()) { + const start: std.Io.Timestamp = .now(io, .awake); + + scan(io, allocator, list.back(), devices_per_chip, device_infos, dev_offset); + + list.swap(); + + const elapsed = start.untilNow(io, .awake); + if (elapsed.nanoseconds < interval.nanoseconds) { + io.sleep(.fromNanoseconds(interval.nanoseconds - elapsed.nanoseconds), .awake) catch {}; + } + } +} + +fn scan(io: std.Io, allocator: std.mem.Allocator, back: *std.ArrayList(pi.ProcessInfo), devices_per_chip: u32, infos: []*DeviceInfo, dev_offset: u8) void { + back.clearRetainingCapacity(); + + var proc_dir = std.Io.Dir.openDirAbsolute(io, "/proc", .{ .iterate = true }) catch return; + defer proc_dir.close(io); + + var proc_it = proc_dir.iterate(); + while (proc_it.next(io) catch null) |proc_entry| { + const pid = std.fmt.parseInt(u32, proc_entry.name, 10) catch continue; + + var fd_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const fd_sub = std.fmt.bufPrint(&fd_path_buf, "{d}/fd", .{pid}) catch continue; + var fd_dir = proc_dir.openDir(io, fd_sub, .{ .iterate = true }) catch continue; + defer fd_dir.close(io); + + var fd_it = fd_dir.iterate(); + while (fd_it.next(io) catch null) |fd_entry| { + var link_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const len = fd_dir.readLink(io, fd_entry.name, &link_buf) catch continue; + const target = link_buf[0..len]; + + if (parseChipIndex(target)) |chip_idx| { + const base: u8 = @intCast(chip_idx * devices_per_chip); + for (0..devices_per_chip) |d| { + const local_idx: u8 = @intCast(base + d); + var info: pi.ProcessInfo = .{ .pid = pid, .device_idx = local_idx + dev_offset }; + + if (local_idx < infos.len) { + const tpu = infos[local_idx].tpu.front().*; + + if (tpu.mem_used_bytes) |mem| { + info.dev_mem_kib = @intCast(mem / 1024); + } + + if (tpu.util_percent) |util| { + info.dev_util_percent = @intCast(util); + } + } + + back.append(allocator, info) catch return; + } + } + } + } +} + +fn parseChipIndex(target: []const u8) ?u32 { + if (std.mem.startsWith(u8, target, "/dev/accel")) { + return std.fmt.parseInt(u32, target["/dev/accel".len..], 10) catch null; + } + + if (std.mem.startsWith(u8, target, "/dev/vfio/")) { + return std.fmt.parseInt(u32, target["/dev/vfio/".len..], 10) catch null; + } + + return null; +} diff --git a/bin/zml-smi/bindings/tpu/tpuinfo.zig b/bin/zml-smi/bindings/tpu/tpuinfo.zig new file mode 100644 index 00000000..1baf62df --- /dev/null +++ b/bin/zml-smi/bindings/tpu/tpuinfo.zig @@ -0,0 +1,37 @@ +const std = @import("std"); +const c = @import("c"); + +pub const max_devices: usize = c.TPUINFO_MAX_DEVICES; + +pub const Error = error{ + TpuUnavailable, + QueryFailed, +}; + +pub fn queryInt( + address: [:0]const u8, + metric_name: [:0]const u8, + device_ids: []c_longlong, + values: []c_longlong, +) Error!u32 { + const n = c.tpu_query_int(address.ptr, metric_name.ptr, device_ids.ptr, values.ptr, @intCast(device_ids.len)); + if (n < 0) { + return error.QueryFailed; + } + + return @intCast(n); +} + +pub fn queryDouble( + address: [:0]const u8, + metric_name: [:0]const u8, + device_ids: []c_longlong, + values: []f64, +) Error!u32 { + const n = c.tpu_query_double(address.ptr, metric_name.ptr, device_ids.ptr, values.ptr, @intCast(device_ids.len)); + if (n < 0) { + return error.QueryFailed; + } + + return @intCast(n); +} diff --git a/bin/zml-smi/collector.zig b/bin/zml-smi/collector.zig new file mode 100644 index 00000000..16ded554 --- /dev/null +++ b/bin/zml-smi/collector.zig @@ -0,0 +1,57 @@ +const std = @import("std"); +const DeviceInfo = @import("info/device_info.zig").DeviceInfo; +const pi = @import("info/process_info.zig"); +const ProcessDoubleBuffer = @import("utils/double_buffer.zig").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const Worker = @import("worker.zig").Worker; + +pub const Collector = struct { + device_infos: std.ArrayList(*DeviceInfo) = .empty, + process_lists: std.ArrayList(*ProcessDoubleBuffer) = .empty, + poll_arenas: std.ArrayList(*std.heap.ArenaAllocator) = .empty, + arena: std.mem.Allocator, + gpa: std.mem.Allocator, + worker: *Worker, + io: std.Io, + + pub fn addDevice(self: *Collector, initial: DeviceInfo) !*DeviceInfo { + const info = try self.arena.create(DeviceInfo); + info.* = initial; + try self.device_infos.append(self.arena, info); + + return info; + } + + pub fn createProcessList(self: *Collector) !*ProcessDoubleBuffer { + const list = try self.arena.create(ProcessDoubleBuffer); + list.* = .{ .values = .{ .empty, .empty } }; + try self.process_lists.append(self.arena, list); + + return list; + } + + pub fn createPollArena(self: *Collector) !*std.heap.ArenaAllocator { + const poll_arena = try self.arena.create(std.heap.ArenaAllocator); + poll_arena.* = std.heap.ArenaAllocator.init(self.gpa); + try self.poll_arenas.append(self.arena, poll_arena); + + return poll_arena; + } + + pub fn deinit(self: *Collector) void { + for (self.device_infos.items) |info| { + self.arena.destroy(info); + } + self.device_infos.deinit(self.arena); + + for (self.process_lists.items) |list| { + list.values[0].deinit(self.gpa); + list.values[1].deinit(self.gpa); + } + self.process_lists.deinit(self.arena); + + for (self.poll_arenas.items) |poll_arena| { + poll_arena.deinit(); + } + self.poll_arenas.deinit(self.arena); + } +}; diff --git a/bin/zml-smi/info/device_info.zig b/bin/zml-smi/info/device_info.zig new file mode 100644 index 00000000..5a52272e --- /dev/null +++ b/bin/zml-smi/info/device_info.zig @@ -0,0 +1,101 @@ +const std = @import("std"); +const DoubleBuffer = @import("../utils/double_buffer.zig").DoubleBuffer; + +pub const Target = enum { + cuda, + rocm, + neuron, + tpu, + + pub fn deviceLabel(self: Target) []const u8 { + return switch (self) { + .cuda, .rocm => "GPU", + .neuron => "NC", + .tpu => "TPU", + }; + } + + pub fn utilLabel(self: Target) []const u8 { + return switch (self) { + .cuda, .rocm => "GPU", + .neuron => "Core", + .tpu => "Duty", + }; + } +}; + +pub const DeviceInfo = union(Target) { + cuda: DoubleBuffer(GpuInfo), + rocm: DoubleBuffer(GpuInfo), + neuron: DoubleBuffer(NeuronInfo), + tpu: DoubleBuffer(TpuInfo), +}; + +pub const GpuInfo = struct { + name: ?[]const u8 = null, + + // Utilization + util_percent: ?u64 = null, + encoder_util_percent: ?u64 = null, + decoder_util_percent: ?u64 = null, + + // Power + power_mw: ?u64 = null, + power_limit_mw: ?u64 = null, + // Thermal + temperature: ?u64 = null, + fan_speed_percent: ?u64 = null, + + // Clocks + clock_graphics_mhz: ?u64 = null, + clock_sm_mhz: ?u64 = null, + clock_soc_mhz: ?u64 = null, + clock_mem_mhz: ?u64 = null, + clock_graphics_max_mhz: ?u64 = null, + clock_mem_max_mhz: ?u64 = null, + + // Memory + mem_used_bytes: ?u64 = null, + mem_total_bytes: ?u64 = null, + mem_bus_width: ?u64 = null, + + // PCIe + pcie_tx_kbps: ?u64 = null, + pcie_rx_kbps: ?u64 = null, + pcie_bandwidth_mbps: ?u64 = null, + pcie_link_gen: ?u64 = null, + pcie_link_width: ?u64 = null, +}; + +pub const NeuronInfo = struct { + name: ?[]const u8 = null, + + util_percent: ?u64 = null, + mem_used_bytes: ?u64 = null, + mem_total_bytes: ?u64 = null, + + // Utilization delta tracking (written by metrics worker) + prev_total_us: u64 = 0, + prev_timestamp: ?std.Io.Timestamp = null, + + // Neuron per-core HBM breakdown (bytes) + nc_tensors: ?u64 = null, + nc_constants: ?u64 = null, + nc_model_code: ?u64 = null, + nc_shared_scratchpad: ?u64 = null, + nc_nonshared_scratchpad: ?u64 = null, + nc_runtime: ?u64 = null, + nc_driver: ?u64 = null, + nc_dma_rings: ?u64 = null, + nc_collectives: ?u64 = null, + nc_notifications: ?u64 = null, + nc_uncategorized: ?u64 = null, +}; + +pub const TpuInfo = struct { + name: ?[]const u8 = null, + + util_percent: ?u64 = null, + mem_used_bytes: ?u64 = null, + mem_total_bytes: ?u64 = null, +}; diff --git a/bin/zml-smi/info/host_info.zig b/bin/zml-smi/info/host_info.zig new file mode 100644 index 00000000..7b824fef --- /dev/null +++ b/bin/zml-smi/info/host_info.zig @@ -0,0 +1,16 @@ +const DoubleBuffer = @import("../utils/double_buffer.zig").DoubleBuffer; + +pub const HostInfo = DoubleBuffer(HostData); + +pub const HostData = struct { + hostname: ?[]const u8 = null, + kernel: ?[]const u8 = null, + cpu_name: ?[]const u8 = null, + cpu_cores: ?u64 = null, + mem_total_kib: ?u64 = null, + mem_available_kib: ?u64 = null, + load_1: ?f32 = null, + load_5: ?f32 = null, + load_15: ?f32 = null, + uptime_seconds: ?u64 = null, +}; diff --git a/bin/zml-smi/info/process_info.zig b/bin/zml-smi/info/process_info.zig new file mode 100644 index 00000000..705dfe38 --- /dev/null +++ b/bin/zml-smi/info/process_info.zig @@ -0,0 +1,11 @@ +pub const ProcessInfo = struct { + pid: u32 = 0, + device_idx: u8 = 0, + dev_util_percent: ?u16 = null, + dev_mem_kib: ?u64 = null, + uid: u32 = 0, + username: []const u8 = "", + comm: []const u8 = "", + cpu_percent: u16 = 0, // *10, e.g. 125 = 12.5% + rss_kib: u64 = 0, +}; diff --git a/bin/zml-smi/main.zig b/bin/zml-smi/main.zig new file mode 100644 index 00000000..6328b21d --- /dev/null +++ b/bin/zml-smi/main.zig @@ -0,0 +1,93 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const stdx = @import("stdx"); + +pub const std_options: std.Options = .{ + .log_scope_levels = &.{ + .{ .scope = .vaxis, .level = .warn }, + }, +}; + +const Collector = @import("collector.zig").Collector; +const HostInfo = @import("info/host_info.zig").HostInfo; +const ProcessEnricher = @import("bindings/linux/process.zig").ProcessEnricher; +const platform = @import("platform.zig"); +const Worker = @import("worker.zig").Worker; +const host = @import("bindings/linux/metrics.zig"); +const tui = @import("tui/tui.zig"); +const static_print = @import("tui/print.zig"); +const data = @import("tui/data.zig"); + +const CliArgs = struct { + top: bool = false, + tui_refresh_rate: u16 = 100, + poll_interval: u16 = 500, + + pub const help = + \\ zml-smi [--top] [--tui-refresh-rate MS] [--poll-interval MS] + \\ + \\ --top Interactive TUI mode + \\ --tui-refresh-rate TUI refresh rate in ms (default: 100) + \\ --poll-interval Device polling interval in ms (default: 500) + \\ + ; +}; + +const device_backends = if (builtin.os.tag != .macos) .{ + @import("bindings/nvml/metrics.zig"), + @import("bindings/amdsmi/metrics.zig"), + @import("bindings/neuron/metrics.zig"), + @import("bindings/tpu/metrics.zig"), +} else .{}; + +pub fn main(init: std.process.Init) !void { + const gpa = init.gpa; + const arena = init.arena.allocator(); + const io = init.io; + const args = stdx.flags.parse(init.minimal.args, CliArgs); + + var w: Worker = .{ .poll_interval_ms = args.poll_interval }; + + const targets = platform.detect(io); + + var collector: Collector = .{ + .arena = arena, + .gpa = gpa, + .worker = &w, + .io = io, + }; + defer collector.deinit(); + defer w.shutdown(io); + + var host_info: HostInfo = .{ .values = .{ .{}, .{} } }; + try host.init(&w, io, &collector, &host_info); + + var enricher: ProcessEnricher = try .init(gpa, io); + defer enricher.deinit(); + + inline for (device_backends) |backend| { + if (targets.contains(backend.target)) { + backend.start(&collector) catch |err| { + std.log.err("{s} skipped: {s}", .{ @tagName(backend.target), @errorName(err) }); + }; + } + } + + var state = try data.SystemState.init(arena, .{ + .devices = collector.device_infos.items, + .host = &host_info, + .targets = targets, + .tui_refresh_rate = args.tui_refresh_rate, + .process_lists = collector.process_lists.items, + .enricher = &enricher, + .io = io, + }); + defer state.deinit(arena); + + if (args.top) { + try tui.run(gpa, io, &state); + } else { + try static_print.run(arena, io, &state); + } +} diff --git a/bin/zml-smi/platform.zig b/bin/zml-smi/platform.zig new file mode 100644 index 00000000..4a9cb7db --- /dev/null +++ b/bin/zml-smi/platform.zig @@ -0,0 +1,36 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const c = @import("c"); + +pub const Target = @import("info/device_info.zig").Target; + +fn hasDevice(io: std.Io, path: []const u8) bool { + std.Io.Dir.accessAbsolute(io, path, .{ .read = true }) catch return false; + return true; +} + +pub const Targets = std.EnumSet(Target); + +pub fn detect(io: std.Io) Targets { + var targets: Targets = .{}; + + if (comptime builtin.os.tag != .macos) { + if (hasDevice(io, "/dev/nvidiactl")) { + targets.insert(.cuda); + } + + if (hasDevice(io, "/dev/kfd")) { + targets.insert(.rocm); + } + + if (hasDevice(io, c.NEURON_DEVICE_PREFIX ++ "0")) { + targets.insert(.neuron); + } + + if (hasDevice(io, "/dev/accel0") or hasDevice(io, "/dev/vfio/0")) { + targets.insert(.tpu); + } + } + + return targets; +} diff --git a/bin/zml-smi/scripts/remote-install.sh b/bin/zml-smi/scripts/remote-install.sh new file mode 100644 index 00000000..77bba926 --- /dev/null +++ b/bin/zml-smi/scripts/remote-install.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE_URL="" +BINARY_NAME="zml-smi" + +resolve_dirs() { + if [ -n "${ZML_SMI_INSTALL_DIR:-}" ] || [ -n "${ZML_SMI_BIN_DIR:-}" ]; then + INSTALL_DIR="${ZML_SMI_INSTALL_DIR:-/usr/lib/zml-smi}" + BIN_DIR="${ZML_SMI_BIN_DIR:-/usr/bin}" + return + fi + + INSTALL_DIR="/usr/lib/zml-smi" + BIN_DIR="/usr/bin" + + if mkdir -p "$INSTALL_DIR" 2>/dev/null && [ -w "$INSTALL_DIR" ] && + mkdir -p "$BIN_DIR" 2>/dev/null && [ -w "$BIN_DIR" ]; then + return + fi + + warn "Cannot write to /usr/lib or /usr/bin — falling back to ~/.local" + INSTALL_DIR="${HOME}/.local/lib/zml-smi" + BIN_DIR="${HOME}/.local/bin" +} + +if [ -t 1 ]; then + DIM='\033[2m' + RESET='\033[0m' + BLUE='\033[1;38;2;140;180;255m' + GREEN='\033[1;32m' + RED='\033[1;31m' + YELLOW='\033[1;33m' + CYAN='\033[1;36m' + WHITE='\033[1;37m' +else + DIM='' RESET='' BLUE='' GREEN='' RED='' YELLOW='' CYAN='' WHITE='' +fi + +info() { printf "${BLUE} │${RESET} %b\n" "$*"; } +success() { printf "${GREEN} ✓${RESET} %b\n" "$*"; } +warn() { printf "${YELLOW} !${RESET} %b\n" "$*" >&2; } +fail() { printf "${RED} ✗ %b${RESET}\n" "$*" >&2; exit 1; } +step() { printf "\n${WHITE} ▸ %b${RESET}\n" "$*"; } + +print_logo() { + printf "\n" + printf "${BLUE} █████╗███╗ ███╗██╗ ${RESET}\n" + printf "${BLUE} ╚═██╔╝████╗ ████║██║ ${RESET}\n" + printf "${BLUE} ██╔╝ ██╔████╔██║██║ ${RESET}\n" + printf "${BLUE} █████╗██║ ╚═╝ ██║████╗${RESET}\n" + printf "${BLUE} ╚════╝╚═╝ ╚═╝╚═══╝${RESET}\n" +} + +detect_platform() { + step "Detecting platform" + + OS="$(uname -s)" + ARCH="$(uname -m)" + + case "$OS" in + Linux) OS_LABEL="linux" ;; + Darwin) OS_LABEL="macos" ;; + MINGW*|MSYS*|CYGWIN*) + fail "Windows is not supported. Use WSL2 instead." ;; + *) + fail "Unsupported operating system: ${OS}" ;; + esac + + case "$ARCH" in + x86_64|amd64) ARCH_LABEL="x86_64" ;; + aarch64|arm64) ARCH_LABEL="aarch64" ;; + *) + fail "Unsupported architecture: ${ARCH}" ;; + esac + + success "${OS_LABEL} ${ARCH_LABEL}" + + DOWNLOAD_URL="${BASE_URL}/${BINARY_NAME}-${OS_LABEL}-${ARCH_LABEL}.tar.zst" +} + +check_deps() { + step "Checking dependencies" + local missing=() + + if command -v curl &>/dev/null; then + DOWNLOADER="curl" + success "curl" + elif command -v wget &>/dev/null; then + DOWNLOADER="wget" + success "wget" + else + missing+=("curl or wget") + fi + + if command -v zstd &>/dev/null; then + success "zstd" + else + missing+=("zstd") + fi + + if command -v tar &>/dev/null; then + success "tar" + else + missing+=("tar") + fi + + if [ ${#missing[@]} -gt 0 ]; then + fail "Missing required tools: ${missing[*]}" + fi +} + +download_and_install() { + step "Downloading & extracting" + + rm -rf "$INSTALL_DIR" + mkdir -p "$INSTALL_DIR" + + if [ "$DOWNLOADER" = "curl" ]; then + if ! curl -fsSL "$DOWNLOAD_URL" | tar --zstd -xf - --strip-components=1 -C "$INSTALL_DIR"; then + fail "Download failed — is this OS/arch supported?" + fi + else + if ! wget -q -O- "$DOWNLOAD_URL" | tar --zstd -xf - --strip-components=1 -C "$INSTALL_DIR"; then + fail "Download failed — is this OS/arch supported?" + fi + fi + + chmod 755 "${INSTALL_DIR}/${BINARY_NAME}" + success "Installed to ${INSTALL_DIR}/" + + mkdir -p "$BIN_DIR" + ln -sf "${INSTALL_DIR}/${BINARY_NAME}" "${BIN_DIR}/${BINARY_NAME}" + success "Symlink at ${BIN_DIR}/${BINARY_NAME}" +} + +check_path() { + if ! echo "$PATH" | tr ':' '\n' | grep -qx "$BIN_DIR"; then + printf "\n" + warn "${BIN_DIR} is not in your PATH" + step "Add to your shell profile:" + info "${CYAN}export PATH=\"${BIN_DIR}:\$PATH\"" + printf "\n" + fi +} + +main() { + print_logo + resolve_dirs + detect_platform + check_deps + + info "Install to: ${CYAN}${INSTALL_DIR}${RESET}" + info "Bin dir: ${CYAN}${BIN_DIR}${RESET}" + + download_and_install + check_path + + printf "\n" + printf " ${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}\n" + printf " ${GREEN} Installation complete!${RESET}\n" + printf " ${DIM} Run '${BINARY_NAME}' to get started.${RESET}\n" + printf " ${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}\n" + printf "\n" +} + +main diff --git a/bin/zml-smi/tui/assets/amd.png b/bin/zml-smi/tui/assets/amd.png new file mode 100644 index 0000000000000000000000000000000000000000..1de65ece239445bc04270a5bde2cd2a4fc16e9cb GIT binary patch literal 421 zcmeAS@N?(olHy`uVBq!ia0vp^4j|0I3?%1nZ+ru!BuiW)N`mv#O3D+9QW?t2%k?tz zvWt@w3sUv+i_&MmvylQS;woyPE=fuW%RBr-jNVJ28)b_$pQRF)Q7YB~jIf?!FIUoeBjwetK=@8&WxO_$#E zPo}He|HYySK*`IVE{-7)hu>cH_;BxW2gs|KF|C=VjN6T)umR|~t3;TpVyi#58Qgp#v)&;NsI^5sg zxbQnu@_UwL_t;hTa(h(sPx#8G@cymC#o3LU%$Y+)8h^0Y3GI-xfAjI%?#RE7ufBZy zJNfle zX%ovMVWo$RMX`zWpc%P!f8_4)W06q-XM%vJs3@zDQ=w;%21Eo|g`Ww|oZ6@*y-6m* z<-l10pse}d4}5osxlj5Uz|(!t*6k7QD1d$g&@TY`6+{{Vv>HGg0JI)JrR~oE@)eML zm0l$x&C(kHw3)=OQ4oFs5CA|)kN-83r*aL2ZE}BmG^?6rM|(6m-<3OhG^*NF+JC9G z{8DX`w!1Z|rT=Y|-yLdSJJr5)syBA2H*}~qw<&UeD0fS*rr7bj)atv{Yq~XR2#<%m z)N6^E!_qS$Ub9_lb<*=bely+bwXPx4%kB@Zx-=p4Xn50^$t%wDhS$c)4~LiC9Z|SH zVir5Dd23W9ZB#B}L^pYCdCGtjG-#YOu;pCuHRpc6ppKGNpI0OfI9%$xy0h(;V>j(Q zciZWK{ek^Q13F*XHoaKYc74b9ki%UUy}Ir>em}G+rOe`M%BI{$#?L#~RGn76D+@Tt zg6mNf#iJ5C%H37utQ@lnQ^IwLW91Ur!Fv)CP<$n0S7f*zqw~wSu{g?`Ua zudz7LaznqacaeW|K!m>huHyfd@_A85`#){vXU3WH>c#Ub*Vd?(ExJ%@tu{5NR0-ts z{#gWmH(psTn>i!@ewb=lGPsjXXZ)KZjlHGrBgdOmY{Oeyd;m>-+j{{GZw6lbuUK ziTt5Ir8aE}p#uwF>RgCL*4yf3?tj+P^OnQ|Jsc8GG^L{6oSlF37qAho7%kv=SROTP^;=!(sHgG?cCiOvPCV<@Zoe@6Atq@2dWAx;CxV_3D!*RR zsUJ5p$Jx*X<3mR#?H8#5{8dT_bOAOerqRKO7>7?qWV! zI)!Dm#0_p#zq4Ky z9ipCozSq*_%L@y1b(t-cr0%M6_A<$6+H}Cxwg30|0{*%x_PL3I!>!N2eVCb5Hy>31 zV>3)c;bz5l3o4GXKpH6yWa8&b>&O1a9+auwW^;l$S{wjH4rWGr9*u1Ty)Ive~y4_2fg0Zu+_y`pP?N-!HRoaS;<%@sZ=|W$hp$($mvd~yYSL+Rx<9y_Sg%Xw(q>k zW+FusG~G`dd^z`kakxx_V%IfL>^oh`)sqn)Ew5h{yBqWK4c!u#lba!SXbZPPthjnn z`Bo++*_=*=Broa$gkqY7zkuQ=Rl1rA$M$Vrdf7Wr+()eObWTX}K$Yt`R2WB+cT zn#ATKg3{LMxW1-OigO3e_JsI{5v8->(%%+4=zdJ$f~adaKRi{^J3Ueg8-VHm6WmuB zj-_>pvUtiF@4o^Q#X}o8Zk&E#8>kQGzlws)C11X$CNLe@n}1&ZGM9JB4lrh|#QUmH zdN`(f2=cG5SGaS28Jzq|7C9}imeC77g-zFr_|KSeePg0kr`1ZWdxvbncKUrB&xUmi z6N9e}rJdL#UM2|}UeP*Dyi!PUUpJ(6To&{D2>ltHuOQ&(Fpu<6(K@4`3-tFe1G86w zgx_*;iGLFbn~VBh%wcffuAEq}I-MZtnXj)EAu`WKVeRT<(JAT~UhI z!14n7;r)`I+`d=@6LVW)_AtB@436DFjIgD8)A#6sI+tcj|HW8~<%S@QbDn#J(;*d4 z-@+zf4R}V5rz@KXWY=?;;mDcsYrM=s{YpE0g zpTi7AoOq@E9c?Y;4V}zi>gQvRVe-MPn9*%454n6-@U-=a*Ce7PwH#Wg1c;H3wkft~ zWEZXUC6<$5>99`d(i`AT{Fak=C?)3Ag0lcWzk1PVZ0FmSrZb~-wO9&qi+qwRONVep zA>W(+Sf>6nbGVcFNNgsQ5$FqnD-R+WnpSDE@G;O%##CYC8C=szNCZrgbK-~prl0qn z5Wfa&km1A0dpNCtzZrEGHra$o>H#+6u4 zo)_+1TsgZkZhA_u=#Z7^MpgDvQxquOx6& zo30h;>A4orU9lk=u##sS{U6{Bwj99oO0Qmhv5Cw0#Lqj0fJpw5@_Zi8;-Ox)hkxqp9!StHwf{{VfXwoNL2t*w6`ehaCyWwI}TCRSHLBF23N zV+v;s7lS4yFP%kS_U&u=IVM4@ZL~_XOjkzKyTXz1Mwtlmg?$C^e*A=Rma!AnK?eTx zoj@YSO10THH@ixdrgJeHY&Lo(f}2Pmt#l#+9jIBj+&y%lG^YYB63jLCu~40A>(}Ag z=(ZZ<{8DeFV@u{whdYkzEp4=Hj9DAFK(7L)2{;TEVPi?4Lk(J7G+#V{v|pvkK)(bEb=gf{uk4*MD67^gG4hndc7g}xq)u%nrN-QuuHnx-kE$TuBi4VVmKD!9 zDU@V%L`HOfW^y`kR>7P>2&5vTX6 z1@0@JN@3jqo`Gv+N)$YcWG<_=s$q_hJ2#1K(Uk)2yC^ITl;b?Z`5zhViSDvf4^sC| zuTxwVI|d%W_&mow3t{yV<#ef2*m0FsYg|0$7$j&Zg_I!&QD_?@o7N;z5~oF1^zCj` z-c=T`$NxC}Aer<7ZUSFpZtca-KgL!As|T6K(Odn@TE+=vtz>#54?F9|JNb{+dnIlB zAL4b=a|<>fbi`yDMLe|Uq+{nbJpY3)VWmAq3i>6*3`~V})0pJI;kcn@AP;jyQ;?AkH48$UnK_J5v4IH3FWFrDXKW9Z5T_-3 z&p{g3Sef863|EKyNkfA^MMi6RiawAAv|6Zmm;Fab%<4OaZrX|r{}iP*9C&2bzL9!> zbx8h(0%Z{#RedTx#fea%lQWJekW7Nr(eZBR43Cij;U#5r*u{s5#jhkmbK-m^?QI~h zOGj}~RsR9p0v?bnQ7O@QOxi-a5ZT1eU7Vgoj?7u-u(iZ`2hl<-awYScP@Fz?=cE4+ zXe%o)^8BL++fp-e?{lQi@ygP=2M_^?Sfr)#h|PLG8153hIDc|ng2)CUi(f@PDt^uHtWr%_5M9Vd28z86QDQHR?2qXp86GvE9*4Xg)ua zr|7Zxx`_GiO6-MjNy{E5Em==3mm)=u5p@_?xFj`V9ZnEj2&iCm)TI&jtP*{vaC-rC zYSEf!T_kG!MAH*C)iXJYmtxv@TT8NnhOrsM#0t<7w2`|f0$jno>zLN-ocg~FR(L^F ztUi2rfUSTIcA`lfcG+Z3X^h>1^P=j+%38L%;z_I##zt91sWA>n$G%qKZP3GAz_*8C z^5rP;r*zR8;bLkcB#VmOqGvzTP9jEGsmkvkXPZW31nqq?@#yVh7)a z-%~udgG(;-%H?Gd=5VF5)N7v?>pc?t^rNo{kcMR8m-Q>_x6!zmwjuVreYaKJ`;q~eOZdeU-0O#m2Ehk&;VCFx)yMr}VBNJ_$1>GmLZNA;!=3gW$JZ33AISIqZi z@P;R_JU4Fi*Mha4JK>eE5fBG+VK)nc+yihxKER%+ld6fS26_hAM@3QnP@X%oV|u$Z z8Ob{+SrIJZ%QOPVxuSY3l}wfKC&DLz$GBZ})+**5vG=9ImQ?hxXLud2*`@pfo}2b& zRguTxzoLhG*f&wH zTkJFmP36F;F5m zs6y8v;Ml%5QTFahMgwFcRu-h+g}TIV{xGVUwMa9^<7x5LDf8bl_Xw+GIaKZ$It{P$ zrinuW=LVpUi!>K>Xezaq-H3S4Uz=aSQ%}*$@vvnJua~Y}`c4GZ>GG{gF4w>^1=Axy zH&N|cT#aE>l@%*UfrJe^D~_5`>_%T2NM;R$3fa{Bk&AWd)TlMZ2Zgk9X;>WbN;t2dmBf-%8b9k}yg8 ziCpH=_KSzR^tPd(U!1#7nN*O>G4uP5CH4; zGq0mzB99_T);g}W9AY3b5t|5c3(!ivf_-{HT~v8oxV&tVlBW*60XP5#Kf)-i!bg(8 znT6Q1lc>0hd0{qv68V)0#mrl0LQllDQk?&ERJI*`4nPu~iMah;7|vLKWEk&)*-8=6 zF8jRcd4E44*7Zj1WMEpLXZxaQ{^a!^*>y&yaSF2=4i3tcJ^Xq2V$i3n%jvS0EH30F z+)3&1(@a`NkQ2z`G&Jt;?|h{7t>+)Q;l4`Q&+LJv_dVHpk;8CWwGO=#7{q)z$f4z^ z(AVJJuK^9QSI%Qt15hureXwu2XlFcIRZ=rJ9Rg{Kez#9w)f+_uOoiKHzx#TcM65ip z=-V`W&yW-X>w^!lNUayA6nrL2%EeB4)1Mi3Doc&U=8A~DZYlUXt!?ROzn(heBRa`r z2XORM1R8^iuNCawBJOc8fd^XWz%UhF7~KWTRcU=}0N;q^VExH}Ej&%UCCBTUL4EWQ z-=)0am`!{=4!Ul$u%Q-JOp0L$z3%tbRmw*bSs7T9p_Kuu6|!a0l%-MT*;%?0zHFEz zjl!zfyE{2KYm{wPafUgx0JLtFtt)2Kb3H;Koa9=h1ygyF!mIeUvoEVJi;;6B0?w{l zHl`YH$<+WYe%#;WL@`IbgD4zWHYK*uhkd*-ik zQ{87MOMeoJWETgABAnW_I?K5vJGtv?C^KLu;n{9D`W4W-EN~~FjNL}ixFQag6-}oi zuVzy>pc;rZEZkn(SG@FwORIgiSx=-P*k4n62vnsTkhBQ93V1dTu|}>`CoocBS}9}7FqFi)VLe#Eq{XWtU%ahUFi)BM@$)H9XXHWH6-Sc)@0{$#Z!Wj|;}^jynT zeGTfG(l;o#9$n`j2>azhLyOFG2z|!*F1dV^<}KJg0?D_9Hz8g&uVl|i)k>aSSvk4F zf9K-(rp3SU+-}gH(0Dr5o>+U59;FU!!^$Xg>fqa8X5?fhp$fZ%AwvZ}Q zMWEVeth~kPeViil^4Nde&^WB8E%iP|Lz%a+5lq71q-hEKv!_1I43W&3PTcwtu zw;#=)zqS=g8q*ed5U}hz+e0=SqY}4s#8#MIOAyq-X_z=~I1AhO5UxGwuTIdAVTFol zW=PK>b5tTKMD=XN=&}UV`j)VV7hZ?Tr^e*AsH-CSLr;R@Wwv5L8J$ie?poq;z19eh z-c9-ArrZuwyuUIa3p!j`nYI#lQ_hBp=v|*-8lYbMmPe2r6j}7mk^T41I%`U1_u3aMsE+gp&Y`YK4HI^akMS8 zZ({vL?UAUb@Mz=)CNt}|>UgeS)XxnL4w)^<&4<5SEnCwnCueWz&w!G6;1l^I6|4@K z2s=R1&}4}OsW|kW1buQJ-E>_PzH4dbH7$-kEWvkAD+uWQscU9sI4y3^ z7=3+xfBX*nRjL=xH5C-}^oYjC{izb0>xu!>mX#abgdU6p??A7EkB)w!p-6{ z6CN$*o&2kpK3e6t+f*5G^T0nu&I$NFPDj1M`LFK78NLk=YS5z!XxkXN+EdI=hU~>|T+~4pB1pvhr`O1cl7H-t zT~eAMOkDJ%-75`xva;#g0AnWS+@e9lwUkrMKacXxx0+pyMR&7Bw|*;MH|1F z#FFqjY$p?L@bt!jkum~eHj#d+j@@V1aC4bD=2iO78FIr5gljpK|r6`NrhIG5xUVzL?dK||eZe;$?RikI5_o=<%`xud6l`$h<%o`3m{nv!6c$Qs?YKwIp-!aM$7k>-@o3#%F5np8O5j{CJ^e^wT+(ljU7xHeBr56{Yb~+!}SHdy# z(U|`Z*q=7F4{+H6-u4DIX(XM9%>e!7NZ;gl_`~5PDj7-W;H=&*)MMr$?Vecg&8&3- zZ5!>ju*H6qQ5dYQQ@4^Ov*Vp(HZ4<@zOBrW_qwwyE3rQ-`7 z`QhIBo)sPCXB_|XIxcg;={U}0xoqcJpaZ)MZ}S{z!F}XffWNk0O-PWMh_C@Vu{0e% zIDsmkn5P{^R`NY+Xm`Y?dBNW3kx%8VHx8vQJzDzI{jNif^++;tvUz+m`GCZ^JlwOn zXmTHa^`jcIpc!}3(js`kBI1+}E|NNHf=e=@d(w|$kO?W@R-MRoF4i|)i8oorNkUJ~ zD7Hv$??cnX;WFqiaf8Qn&`mah{hHUCbS`sVS73g>D${%UVUBL*%K6yWxfYavX+8*l4N}p_9!G-4rVqDUXUwl? zgFbA2HGA=xYsl25u@&VpU$e5BExvT$D-~_nZufapsbZM!ws(4eCkMmUt=`tWe9 zJoZXqNHcz{PbL7@XvtN_-pZiD2jh@HR)+%V2wfmOy^#BSGcD%@4TH}Iv5VCBahfZZ~GRSbPKOBs7b8SCf|l+eW2 z(vgTnDCR~db8Lvvnr~hl+ z9VA;JiHp-?t$LuunYw}Ws2tN~Q{fuU(86Q;AkQ7(AzPb+GVuTKZ?PO&)kPD0VZM}< z@>+6Hmr3h*+N9((kA|+Ppwg%QUDiH(>*xDYGbW+8^r38wYUtdnvgqTd^wz!EIWzY8 zdS#eKKXDIzN4zfy*-XcvW-h=PI7GIcvEji#Om_^h2K=o-%^(kIatE*Ye)|8l05gq% z^AIYSpH?~#l@^eMS`g7 zB~B9J;dZy`Gu<7C<~+AECRyui+``A3a1TF{0er}p2$e2}TIDWCGsyP>C@l^hUW4$H z#-y<$nH&Be``HvfP~(|sXrezZ|*KVv)8xj z_l-|WIfbmaE72ofxb$>obFw z%Byaa$i9Bqm0q=32QAB@3m;{D!c26|pXs6j&LU0sU)`ExmXrum8$U_5`+qH5}Ibm;LOO#C; zGx0;CoXcwDk|dy!AhnIlKI#!wz$pQ`mWDhX)ytBU-a{d?A-yDI+WaSDq(8FyywhJ~} zFU!5PEHN4wCmI=WUv0%5*WkR~#H+?X2F0meyy7p=*(Ks-=)Y`;F0%H~UJ5&dt>5gp zIb`5$NpN=G(Rj~+Ubq6|Kay(7N?1M;qRf>RM#xVEEWJ@tvOk!ZMUM@aQSm1?8evql zdCA5k4IS)FzZD#NAaFhER)szX`}-ykc^>-!@ChwAYQw+l7Bl-ae7~PoB;!+Y{OJYU zUmaxA;@aK6huf@8VYNyFLXui`EVgdh8v4CZ#Bz$}9z$#dPOSk>k^j884n%4DCt_zD-zd@ShbM)g!hP|p#xYF053`YKwH0{QkxGng0Gmzw+#?ZL+A{E6+{ z;g6Mpz(zB1(*S5B)0!>E;iOuh%hl-efoRZwI4BFdHr-emk|@uJ5P{ zwK3*C=?;mSDPuzdZee)cD!Psf`Bm31FLv0yyiWX}B_Gt{KUAZ_cINB?uahnq+yK52 z@+EZtQ2IWCeV>&-Re%;;mO#fj?82q=c&N2gO53Z?f9{*T?b*(9YIuFBazbZLj1|Bg<>h~2q> zvl6Qt7G;wF4eB4wRwBVZ+_}n(0KfSQxhG43Ci1+Cv3sc_2z?a3lp&-x?PVMNV@&;^ z2{aD-EnE&e{eb~*VgdJbsLy5cS1e;yNSQ3^0DNq>5f}n^j}g|~^R2_wn!AJ5wgczM zlqh0`FxH6z*kS9resl$b##YS8I>fq*7k(XGnC(+SwA2&45-0Kv2zH;0va_dg~aSiV0>26CKJM7Jn z_e6dIxUC9j3c@`Ex&^{6{maNe-bA9_=@4tCn4?!J+4tmxLWAAPqItuX>CXj&CSZIK zQ=RIP_`9a2urMntCFOkiAuF}y9|KNovjefr67VS(E{E zm)27FO0@MN&0OqrXCbogqV4<~*XIl5nhg|9{5|vgM54yx#<7cG>bhzI#$zibbNUB( z|229ZP%B5>at22B76gD{Gox4h9 z;|m9X1#&C}#Cmjop)Vfo3lKNlh3aRm`I6yV`&L;NlkdkoSa>s{*NR@ihiV8Wv*Ugr znNpP$+w428Y!J}|{ISKCXjaNDBZisiaMQp4F*YJ>Ou!7TzPhRl*sXG0hrFk9<9rnCBN6Cy|FuY!Df7-gFl$Ip)sOvayAGO*HiL>xlYC=(-2SaD-_z3u>8$`)lo2c7>AxC42DM-Y0^2y^TIfk8t5fhB1T*S^}u$v?ZewqT-?_LAU4xk@Y(Ietv zbM)L0Qy$sr>KpT5TwwUeKxj@nYFn&>Qjd+>RMLe=lvWFU{p;%N*h?ssfn@W484`JiBaG|Q{K>U>cbb2RZY z`@f^*!n_1@$LCUWRLMt>Q;HMS{V)?AQ%kA?6wxSmkV=Gq;cUSXTm2D$eec2gAxrKL zmEGX6W&TqHjT|}_D=uWn*^moi6$hA@1nojllm0$))NY8m4pl7|wtThBT_WxNo7$## z;k~LC-NM}neP{s3lWue>MJBA}YMlcbfsZ8DB}K*&@lo5mv2S&xo9_;{jIS zq#m#upt(+RWC@L%z;->c!yvR)R6L5z+(l0a81NhRkQU;StvEE0-E%Zi4LT&kdei{M zQ^v=aPf?l~8}GIb+rxubujxOBPmTeNP78Wo^q0Vg@DFakpT}dxA0*$5*HLIYnog*& z!W3<805w~H0GMe=C_Cv745d+Zu`Zccb+==#-3EGRP<0EW3!y0^c7NHc5PgxJJX-b@ zu~|ahSD}03kgYy~hPpoHTxj6ydLzJ8u0>Q7k5%lF5+hjsAI$ta($9W|Nbd29w~p|~XZ#Cg#LElCb^3loS;dzFM2ai^qLmux>EATp zrSzX@QRfoVZQ;OSqi))51cw|1OVlnR0}(Z}G0NMS^O#ixdN)uepAiV)5P?(jL6D}4 z7wk~+jL0CLNyW4P*qiM*?)ATVH_)%6eTR2R#(klt=XFaeK4!m=1$gphY?WTT@8rOI zi8F&5d*pJS^J`&fFiXxnJqNl(bZ^{_Bc_Cwc_? zlVa^|R6(GA8U6Ae`ftI<_c1e1aV~**;;o?6UQDDDO14F!tdh<@8RFR?i-RJOh}q>d zy*Lxjc0Uz%kGK_UuSAl;Zy|H#ZKZXVNxYUO_>t`5sPhhN>d*7=Q*9~%Y&{fVF3+t8 zuB_(u+z!Fd=Ybi&(H`D(7#jPFy@v0bviZ`W-gtF4(;nStvBBH8vqo5WY%l2PgL&9Z6&(lGm-1S`nYT=>9aLTv!u z|I@3_zZoyDZ}#UuIF`(MrPsJUmQSn(`cl?l1MAVKA_4mj8TA)?{BIpt&`z8BIK3Zm zkES$3Hq(_9@iO7?zMSZ{u`d7vGO4<8fghyb3H|9AFASwJkL{&sQmrz=_-^qNk*4At zIhL~->&H^~7VxbZH@$ayHrtwywUp4cu^WLzjE9u>Yy-+B0Gocj4QS6z=#_-87%`Ws zgo3k-%?rUkjflm<8tHP^WZw<0wLOpv#7*sDV9>RMm~dUoiHDl7Eoc&VhISAeUh&ze z)!P6#B4B52ZP7HP-^7^fg;zP?@Y*)^gtEbUAV&ooC<73#qawwr8bRr2Xy{{$#|>fB zVdFz0$<+-GkjFj13qzS2w2}U<2Jm4$DU9e%D7*U*oK3~b;jx|aa1Q`0px>8h05S~=87b&MA?nZf z*fi@Sho%V``{~U0cm$+1cgTn(a4h2%fITRhb7Yqy4X(vI{IoLaH^5)PG!>rwY2eMg zlN|5CB5rK%k*}pV38F;Phn)3yQ>bgO8d&kOAHbv4ao)BZUM3Kv$XX9*;1FXp{092= z4Z1_3PK0g?hnJn=&|+!tkdFYxp30aE|N8=M2r1ezJ=gJhqaW){Jf5C@4vdg6jg|iP(vIvegyAVDdA1{Zv+^aV2}EZu zpMp`$RoZ4dS#Kcp+5v}{A9qF1rt+{5ejT4Tc6;FiUyF7p0g8p#e`?TO&+oX(21ep0 z3l)liW2XreHyHIN+jC9npHN$#`ki&LEy?i0;YLF5-L0@8w%2wQM^y^v>cMRRN4Xag z0ZstGiAS|HdBw@-npuV=YF9Zp$J{XV-_+4@ws|aDHTpSWy>CV4>29Jbvc{ERj1h;{ z2v=Vc?mm#A+Bi=qtBpvQ72}sr&TXh!r}7%Rf*HZvlhTPirc!sCHPAFZaY{ zw=p-a_U#9hP~ZJRMbDnT1QW3e|5RssO8#(;leO3>r?+Mg5ES?A*P{O<7>6lyRprQM zgImb0i0HE%>5Yxy!`LiQ{Tj}5(5hR5^9;PDt-iv(%?#zDhnisc?3lz+{+FWwll(&%S#4>plIbEvFXnl=uH1w&R=* zf=dkHk9fK|(zkj5eTzpaqP|{nUObc-u~?b;vu_X?I-(tV_aWgs<5j+FJASoWY&q7o-dSR%2Llyx7<4T3di{*-4=00me^RTtEOM zD;}-|bl(3(jd1AVEc)yNKXvou?p0`T=fUiCKd8|0GlJ4*!-F;o96OZWKkZUfb(#?u z)BIf6v~S+x@AYNk--YTe%;$%1{*-Yiq{;b{{bP*1fRy58_%I6j_(2ncr5pcNUfl|F z@qV^CN;YW(m)CIL%?wb%&m+Qv^l#mS!5Pk*US?`i-nt8=o zIsN`6%*zn^8Sjh)urmjnqNuP()8F78$aUp(mIGJM0ggNKqJrJT?#f>VK3&fk12P> zF}&&OJMQk{qbAN;*yJ1P_M-u?$0seI7Wj7_<~-q^egOQzaeBas&0ttDulR|mkdLVL zXf_@{D=3Y`^XzL;zxP9uSUQWdcUu{^IVXKLM%YVXgHC zDf@w)ciJ^F!W3-w$ErV;V`V9w_dW*yC1X2#z{CmCMwT!9n7~@VaiXq+Wh;b?EB}fQ ziWy&>9;od@Wo1udP-9&rVuoE2;3m9U(!vQho7+2anu%HBE2kFqYsU$!O83g1os`fU``<5p;T;(;^egchtu`(eux|3AyuXb~vOb;hg+3MX*4f*e}&dFVg7K+hiWcr?I{oCw~u z+M$WDXuLovK%WmD=)B6Iz58UVBAhgX>WV}s?P50i-87#qaN>LgvkKpNl8;N6;4oDd?`T~100Yx}$30UGq_R#@N$>ReBtODiXUS_!mP%zP(+ zK8P4$14GqjoK+=DrJSFTg#ubg6=Yszt4o3yPg|A=F1AONNv!WTve#P*HtCee<^p4w z9ZJ-6!;74z#h0P)M~){qBrNPY9$l653$VqOrEsF?zk#Esp!hPd(--~&YQRDT`blhS z$Sw3Nwx3aiG&IwWpWBo7Mnk#_WSd{T#noo^B{wY<)L+^yO-_<#&d#YQK5|nGbvg4K z*0VmmlgovRw<=OQ8i!{uJ|g=6+&6p3TdP&FZG&+jl^|#>1QmLcV>w3tkE8HQ9^f@2 z0G4hgx`^@b27o$LGe^>*fOro=A#2UOc-v1<%-W)Md!esncp7B1{P!*!;X)bXs_N3@ zC}M$Q-^H$lGx5u1_5v@ZA1%fz9n}kw5Bo9NM*KT>{Jn}@Ie{~5wpslLJh-fyKqw;} z1EOrxG-4gsax_-MJ^)>w?gskD83Vh-%=8xYd>vFMn#T2rt=FP^GmmHacWxi7-b48u z9vL*gM^c_j3siUm3W9=CNe4tqUFFK$4~0cR&9tl^s{O((W<< z6}7EUNozT~j54aM=ArY@!&{zvk)wR>GPf%=@fYyNo6eG#?u4Z50d+XQV}E(z5qLrP ztPYsxqG1!Laf{^Mcj&EoZo27g&H>Th6W-Gd9Ic90Hr~RL6zE>K&6Zm^Tk$~@-`Lm2 zL6Pm|;E#s|V&*fH=!v8+BJFOsZ${TB9qEVmssjFk>GRId*cFY8ndy-? z97vAC(9JhVtCp|6PG-raZNHikyWl=J1aD9V1b;pq2_wO_oQnX~9k&YDDAO5C;7x&K z8{khiE@I0Rpf(e}0sMLk^^h0P3QAG8;h4Wac{Z`pwQYK4;Yx^aK>=R{lu{S#@*UsIe(5=S0_vYVdDdq%b9#8~YEwg7{3rKh}> zd7kkG9@dW*ytFO(k}C61XC+o}lT*6gA3L!Qq$>f%pno0v2+%l#V*SNz|2kR<+Je*E zkahi;ZRX4L3vWXN>}%E_LZ9}CLfotL4*ff;vZRO;d=^a5puV<1y80k-^NnPn6F+ej zQ+EHT0X@t9^u8I>pT%cn(sVR+>Vcv}>VGQSQR1p_vIajowrV9D4#)%Q2uXe)XeaRE80Md?Oz(&0t?-76-+kr}HvI70I&JXg|IlRLYcg6-7~&xgzu}OC^-bl(Hp8l);$$ zo8RyC`uzhl^E&4|XP)OdpU?Y&4P7Mpsk%r}2ie0DQpi@#48KYc$fMgOft~0t31taO ztTY$>B7bGoozlva?A=tn=to4C1N*sNUO9w3>b(zOU&XqpUO;i;ablANBOX{VH+qY|hCdK^8VKd+NFd>#+HWH6;u1`fHQ+YsS~9U#>S zR2d35{h(2a8AtAtd%X;NtyTlW9i~#~)HD|rsQ8GkgdV5)QSgy~qqn;3R?@~3jtiew zt!u)g?}LEGS6peffa<(&PyT50M3v!a>*+8A*LKTL6_kH$$(x|p613&0r~)A1&OM-7 zBPF|QHMLXcY6o1a)J1r&F=stLrr(JlpE}lvSOJVuEb9gR%b@2g=0mfBeYUG#|!#%Rasj1lvrF9jjQF zRmkCT0BbDFTsYMowts$4%hQb1)7UQ}hqP;9QsN9{t7Em6g)ty4|}0n#$y4fE=o zL6;xAiKXG->+rqRu#+Lm4e=UH^50zG#=raGw$Gwt5-Q9%MFnwTHlbJ*|GkauwA`a#j z=C3%yutpBaO35x`7RlYQW>!d9t?N7++?5|8k?@=q25 z(m6bTi#`#I2|sp!D^Hy5MuQK2iJbi{At=X7>dOwsn$~{59C4a(!NdDbq52=|o%Fn? zXD8-$e&kc$%d743R$iC5Sq`Zkp?a$<)o|FL=(RD+ zw;((|X5CRoZPM-fu*v$=$=cbgsegYC=bOG#US{R}=j&wgz}%GqhfaB+bYJf=h<}H5 zkLfA4-% zceSvm2zUfDqw$>+E&}eB7c)9dTxDz2PPwCG&w;H1JKMSTXq~eH4YTP58-d*dEJ{u% zacu~Keqx|xL@XMDLHmC-%Y%^bj0<v!HzCJH`!(6pQLt zU)(Y7WAT;O#tR!&7XBv-W(j_KT%=^(Q2d(>*m7|8@8sOvnaR`|6<%rs*JwZb`pR0M z!YH_|QpBQlwM!P%5CV^|*FvlhpyCDNE*SZv<0$>fV4aU?Xk)p-UQus!@<9^o?LuJ{ z_xQrs4I1NyHq)~dq05%82TKZ`J6rJZGLEEg*iU|Rr`<7<*oMzqUl2>^GNX38qsy{n z`|_5u3Lc=L;<`bZwOsTU2ED-79320PA6)P`_KhLDF&8&7ZGIECU+7(@gnnI7&v+!5 z{l#yYAoqBs7M`2q{?ZYgTf%GHb4RDbjjv;ahFH{YID*d9jecoJ+`7&Vq7>G|*2XK^{<Tr=`zLiMK2=M(wMZZw6$RG$Pg5?sRR5V}|hwr)d-YvAK{5ZM<+(Trkvc|v!sW2Yd{WYa=x=h;~iPwx;b9wdOc4k=s zrnH%0ue$~WC`?Mwx(cPS@){tU05Wc=LdU5ftPSSC_yzsNxU;Loq>CO)-1zy}$a#A5 zTJ=cspH=f$;%w&Gb`&T%_8#s7hm8FTVh8@R)XFT05K$+NE@2_}5Zd5r60;+w&>rq1 zsyP1`D~$VjV$xb4?aOk%zO8YJoNICz8)!L#-BC$dF+_I!;HO4E2w@*!vqbO#MxR0~ zbwHoPS5m=)D5)!4&xa1lbJO3F)nz@;ipY{+7UIp|K`i{Q`_s;!q{~7{q4m0eB`5#y%)+0D`av``Uey!H0v_mUf_XRQ ztyw)V+5p`d$E%CJNF=8QjGpVfU7GATf5xZtEIjXH3}_I)kG!)u|Lk z3G)aT2|o^l7!WT|`OBDzCc5g3z!%<66%IFCxI3302{?Z(|4OiP=kqxS7TtN(uY^Lm zmcvP$lsba|$e$9p{+G-yI5rDEicr>KJ8J__7m}*f8H2F=vSwRjiw9#)lmt>^emM!Omen z$6x8Zesh`6xUwO9&N7SP9QV1I>&UN=wx9Ilvl9pzqux`|tZB7w*|THlUn|&q@YkC~ zX^V8nus^Fz2!cd=UoD8B#`D>y##_A-uK5Qm1QG%dqECqEWf1=|09e@qcU`h?b^u-j zGX*7A-BdznTmG;M5qz7(U!T6Y%sloWt%Q)pj?qC&iqH83OAqmuXTlN3bU9}YD{iYe z3FR}~<HLKHxcYHgvkIufr2pAAD?acG`D;3 zmBL)PI3NZAXa2(X_uR1F4bwqU;$5*Tp6$K*dCjoWtZLJP$n!z073xx80=3naq+1U3 zA6?~Glm|#YRxIk*3v1B~38p&{9dp2@V5TXcz?_GWsxpweE-a*jZ#u8>otYx;0Z;tE z$wzjX7ulXv(UmpXCB@uwceac38n}f>D|oqyl}cI%nSqpy8_2pORNxQv5P}@1)z2kW z1!%6qa$t@Vgzf=%_&QVUG9hDHXDPm!O2oEv=HJ|$FJI~FwW$1q0oaoHdAZucu>HK- z!(qWsJwX_$fWlnqvU7Ty?0z6qPSsG`+(AmX%DN)S`AxVZnQ1tagXx_hO7}}aeagT9 zrZb4mo}fn02M}ez^+k5wJUJhS1(xA+GNlarP6_Ezok#5CN0!%kCF@w?Xf(fvm~6gm zL)UOawLVLC4*{-b*w zbj#h?O=PY`|6FTSUdvQ*HR~i)(T{JVU^eMkja68_y+=G!C=AtS*kE0n$HdP&KSa7lpxVtNnG)_UBFHt1=+&3 zx8s;{=n;^K6dh(dz(A2#)a2iFwKmg{4tlfzSt8yD2savo{Q?;Z^Yv|s0sgktQ(@iT zbgJ=tAI7fGNz9>=?mZnB1MBrZ~$fniU#M{j)TAFF0O2-T6^fo zVB87(uZcVOYJZ2Vx7a@{>X8%FlDHw{nEDkmls}SmF9XE${g_7P7lSGqouPjG8Rw1? zrx2Q{$KKyQ=}7k_fRpGqC=h7U(80H|6?c*NPxZixhQ0w6BbB2TpYdgl+@rtQI|(9_ zJm^EUbmzj|{?xQn*uXM=p`Vlob4Wpt5Rtt_7E)HUTYXkvmP}U!=|N|a5KZVXQ^Ly% zui%8&j}m6s)}V@X!;XkPL!yR(q7+C_3R+wU*Y3b`3N2P;2u)TqLPTL@@uQ6YgkG`& z!`nLw?oZZf7zu`@X@b#%KZ85S501>~lnmR#svB+~DPNryqbfF#`f8W5ELWjgDqxr= z%?ts8dNXj8gmMGY96wnPA%YT%RTkDlKPoW6tB=gY6$&-7N$f=|8z+BM^gSryj#|-V zy$T%QDul#Ky*xyrD?|Tsm}I9e?RhAb1@ZXocVFQjOF^3gNWtx<@lpxFi=%iNwHhz` zpk4|J<}Q@tMvMLLM5~D6kKLJZm2fz5T!BY9YLB&!Gxiaz404FdP}eIn6J6Ak)b$sB zAl65CqxEWZ2Uq4U*ou%0?i0X9AUVy}f=bL&tD6{7Q2boyTl~Lw&TT@ny3j7J=Thvi z<&eW;+u|$ixQ}Z) zikN0R>o7a&Z|lhfnH;fzd7L)u`SG;l}LS|p?oMyvKpR-J)tHQ>?> zs0VrFULR;)9r{c`dyw$}VC0CtqoCuUKohNvD9&p}_wRdlbYNZ>CMA6wbl$HPslR6r z%4V{cLcDSUJ7b{EDB`3N7+K{-LOyL-Zby2y`yovZt6K7gjuah7uQmIj8OEsVKDcWi zM0Y~nI_=WtYupt{J=uA8)6C-q?)==(jN-Z58E5ATVbRH+?jI4RQREm`IXWHMPVgPj z(A!?_(d$d`Xmvao4D{~Uw$yPiD6)zdqU>?Hbu1h?-pUoJ; z+oKcFZ#o?d<)yf)IH8z#=S^w}bdK}FUhoVJLsY7!-Wd_y9CpFK2IRbnuA17upwHJSqn3FLIL(@76}_M$aJ2wUC$3Y&zD*C*Y1yK z%Rg;l-U?pH$|#;n5;8z*^zmWBZ@jW495x|co~$G-chF?MDI_u-%4^!!HdK}w_L z+YB`!Otq4Lj)XA5mdHWXuCF+F)9UF9Rf>) z60R6`v;nAU?Ih7P;ZJLmjCWJ4rrDfDl~sS6*epc!0E!gdb(I#KQ@Tos$^OkWgE{J~ zfLGA@3#DO3aEdJ{!sc?c9eP)#@0hN=gCeJuz}yOo-@gIPj^MLo#dbo0Di|qCH#|PL zcAJRZ!N)&6U9h}^d-C)DN(dABb7h2#|9B<7DBjd*S+0`tBFye%Sl8ah&(h86^ie{Z zo*c`i6P&-`br_1cvt(VRmflwhf+G}}8bX-Kh5NjrPjQgFzBx2f2aY;{7fMWxC8A{# zuXFqqrx`pEeuIM}9m0O6SXJRnK%Bo6m`gQrSWbc6A)2#0)i?=LCC%l5ak942uR&_WC~wN2c2eH z*n}~>)+JJ;Dp)+6#Xf`LD^kBiFc!|w$q6nd=4COzt#XG=deJ2MHJ}IcLNTrP?eC<^ z&_lMWQ56JJC+LfP*6c2el)m#IQ5<4aG!>%TgJ3z)YasHHTV(HTVe8>?^X0FE{n@XD zuZVb|VYS?MSHZHG+8k7;x4+@@@P*C=`t-C6;*pNw(K}wPriGNwi917j@Y&-z{X3To z?hGXKX;~-2iEA<9>{4c>8un1B?=rCZd_bZr0`^5KaQy-8jqnY6UYQQF-GOhFxP=!2D}uQ6Gz1ag0RbJr@R`n zr}Cxe6v>U;0vxKvKTmK~e)ZFSe=zn1UJ+96NJpsqKN#8qT00a%XN zLD33m$z^%43xq!JkxwXYeyMmnEncV~4W>%=*W#C)4Atk3HCnatB7fbQxkVF$@UpoF zOxOJAwE|uUx`SzQWI49X1fd{_LP>1$$|kMdgbt857~p|!53gt?vLrDob3MGl9}et< z5H0LI&ww5)6E7xagoE+Nko4S+xbtqp=bECI&zA6YH1U@s++FX3m4R5Ud4<4=PslLXzXqKYidVT!wJklj4T+E-uXcIhS_f%$;YRzvc4r zljX6m*@i?><|*MRdw+q16H^X$*5V;}$ARqelB(dl-Sav(Y=IeBIdGY52ON-k(YyD8 z?eWyxp<~;{D`v0icLuFEhn9WSp9!$CLa@4V&B4%Sa%sGb}?hl8O_#@~-0IxbvD37ch!hi2&IaFY= z%w`CuOY*{s;fcs1xZotyPhPC0CM=+G3($h2j_sH#dIzSbNJgPh;F|;@r}p@c*v(}= zR>Iitc;m4wJf77!S@3fc8vY%<93q+G2MI&+`@aNjBBXaPY6 z;4EAN9StCh;XzNb_?-msSrir}24XA|RGJ|625QXHmVPKL z6Y#|J>%z(ssX`5xDaPiJE2;uEqktD@#dlf_3NEaMm=;WaIT{xHOZPO}M*UyC2M9;2 zGu3%&2fwsO9GcDSCo_VJEW|I8YK0)&JzkN@@0jvJ?P2hl7?T7n$xi^2#Jrpz-x#P=&i&s~@kXwe8}Bn^BE;CGS4W?^v7B2s-D=gRj3t zxmzP3hnW|WfLNfu9DeI)^%QY&=Itz(N!d7E)0Hq}4z(IV-)F&HY2Mu^Y0hRMD=oh> z)?{c2b<3o`lhiI?8}RKP<5b6&Z~|_aO%<8Jj3e9}g61|G&S^*?gTJ83jSF+WdLl19 zfH%n9e{gq$S{0*)9)fCqRS(Qoj5zWPT=Wk9sC1)5I$|?T@{3f&7ujNOD#%dutpH*> z;DwXYyxaoB-pvucv?D3JIutLp?=JjaVggcxo$s6ZTy~0YH_8?R9R5NO{>yE}m88#2 zOemDpHb0IMiNZ?{R{cspCd{B#R8=LG?`#A$%3^b+MQ`J8aJ6=*TJWR#{o!@6&9tw) z?6^3uZ*+U(`l9|O7G4OZlA+OLs7#u-dBlZ&X+1WkO}_Iw?`9c!k=U>9qYV66)or&W?(WvHk=(Btu@g%cc ziglV$l*TM4s8aoC-huF8dC-%9bsSCryJ7ya8ggJF5cj}YqNn=&fhcS$EC%1r7K2D$ zPd1z+WIQdd;($ZyYtS(=omRwm7j>RFD)d=Qn(#Ue^GdavfsBK2ct4wJMCu_RnkPJv z7iO2B6Tz@5RQ$-4XwpgZUJYwz1K%NZ+p!;DKZ2gx<1(qhf(mTGMx4u1!jtE-7d281 zRp-L9!@dI-1d!w914m|J8g)#2;u>BSuC67)92)E3#lXzmt_Cp)=||hKehAf|CKDci zSYZgQXv&PBTHTAG+Ha+@=;#^(`Vs2Z2aG>H&`uK5KqeWq_uTVYBed*`!A<`mc*&bz z2^mVlSK}im5mD83=v}BO{XwHM9-oVHi&B0Jq`kPEGs7ZQ)YT0T?8rMvoFnMx%Fio8 zB#YbpjYOv1q>A-0XA|$}K_6dDs_bd7g94f~tt=tJ3FtHdeMk<}f*-%mee%O;Y{Q05 zX;vs$BXJ5}->^`wa~eO4={Wr9tZ?;09_z5)LCF>4mL#=|BZ9p++ev9wYybiexAO}V zoQt6S&6G8*+v@CULM_!!SwQTyPoX;9)d!sEzO*0mJa1Jh>mM{Y3^*XAcQt6!$Bbb5 z8aV!Ld*q7Y)JpL;j~38?RGDDlb6)V6#=W*l`kT(0wq9#7(X>>$#x+5-G(wQsKNs8E ztMQ?37{9+2vLZ3#9-t5;BbG9}{`ve%!81#wiv5+FB7Te>D(9Lg8m1Z$IpJ=#Cg-+Wq@D%Lh%R|VEsxrcl~YK{@O01MLn z|I}aDqbM4QT5vo3UpruYm`9m*1#WQhsw8Sb0Z^Typk+*|uQGT}Lu0^9^9|)rqF>Kp zf3Qv&xBC~n6lZ3ZY-hKe!)I(?fm{tZgnXk~kFOF;brZE4J~^>BJE>g41T>t2qoDzw zC-8vziKuzfw7{5SyYU;rcdO(tB=qfhnCuQ0S3iJ0*+C@l+N`AzZTjti$>W#d~mHu|W@2<0VIccT6}m?RIvT~gy7Ied7oeW)P+ zrAeCyS5xS;r#doDeHD0YPpv}KSQhSusGE$Y(pM$Pu1XJ>x0jQ5iJUX*2N&nNf@?%< zYm+MQf;cnco+qaqj2{*Fv^Yk-T*77^Z#M_%`|lFtZ#O-t7iHph*&1`_&L#Q0lCZrA z@7DigCnuP^_TiGr#MXR%-VebuQ~X2OgZ|1W2fKC}Rti}mOs8yTLw@T`v^LQ@D!_s7 zaoe+W#3$7KlO9lDqB0|epfrQZ`UAOopwmtIV08=}loS^{TGW!VO^C5HtDeW4>2%>C{W$jNsd&{|Gu(c5%8q8;0InXA@ z0iT~B%b4F1EX0|ioY7U2&B`T}f?LnqdHJCW*V34k5>}3~7EB!?+5%~Q7>ADsz=#K^S+KVC!E|b4{+5J$9zhwQ8*;RN3?>V;57;>JmyH+x^|_C` z8lg%rz)P_hRSg`H)0G`?5uE#Jr}(@xKGFD`lTw59QV;ZXLns;{W*Niln4s8?fDu_L z1wMdj4)S(vNZ6GbVvt+^K6^JhgKP&6mCdyAoyYKR2%}Y0bpx`!i=7Bo@UO?z9RAFn zN>ZM@E3x~I*5kX3iCxi3Q2N$LGAM})*6D~cS z7ck*pB0d-3JL5o9KE8y=)xZyutlFbk>hS~zIYL&6I;f_M(-wW^kBrUU_mP|5C(pVD z0!B3Er}W<$qiQn+FLcaz5WZhN`gz+euZ|Ix?cazpo_L-{}pWtpNstzO$or8QHb7m)}bBA7d)Y zJoi22Z6>H?X(>Li=SiKz&speYHb-zHBZbh)g1ueCOoh@MJ zaM)En)|+BN%@-z?!aTS}V7ytJflzrt9z{5deuz-QhEIp% zZU(Z#xUWXq;53j*pjA9JLoL6uNb43IT9qpy3U99dB`=>_>w#ShgN4mqQa?P=*Yio3 z-x58HoNhq~OoZH3z;kVM9n3mW1^%f0zW8Ln<2L2o;psQoV=FuWtPY&7&!)URfp_L8 z3Kjn@TRPEM!~T&>@TV2%vL#5|2(ZQntVri=fkl_MgkAglUm5UBqOsOG%OaZMSrHCb ziNf4IFn(*tq|og4O+B7=5kXZWt6rYfZ3b-!_C!;Dqpv8a-+5{DF5%94a7u7>JS{gw zb;{)Rz)?tS-Lg~G0P6*9&F7G(~>A?W*7Iy7$r zR+eJ!zlltyjBFhAz3eeFWbBWm{s&7rNrPHmPY3Tvg3_zZk_NIi@7V{0VuMzAz+{!jLk5#eg-sFx@YzkgNl8WC zg|VlsI2u1^;RJXeQ<0&7_HMWc+7i&qCqVbRUD>d>)OMBvzabb;iQ${>3!{jZXca6v z*{U6t5b4LS(p3`Yt*k+Zaovm<7T7$d;gqeKO=cn^%t z+KA%BBGSb~dLw&)B&xcH+&cFHl!CIQg5;%ZMfx)~g@#Jlwmrx3x+k67?oT=ApqfS> zRVGOmJ@H7vN|N13Hv3`zl4(NN(6}Bt@LN8+H-;*fu;w?L5$Ymu0s|!u9-OuEIVvTM ztpzy?OwbV~8-xbJvOQHJ)1GQn&~<-$a9Svo@!?NO(FE^}8D6>Hx%?~C(>ls4mbzt} z5ZfhG70{P)I>u7YPoAws5PRuCysw)wL-YsZlFTf&%RfY7E_V7I%LdC`__%2LM_!@f z^8o^T3AM|X(0Ut0`kaQ>jKlvv5m8O13=RE+91a2_{yj1KI_k!PZ(6P*F!#fK1_1wN zO3g4HFY`)St}QtDqqXyc` zLCr2v&TCZ>_S||i_ULSbD6(`+ovTLQecM7`A5F|W4INcOrG#f8RFfY5 z9wZurS@3idrf`XX1+hFsV!j8}ZoM9$VQoinayvZxSQjgV>ttao#mgYXMh>;meid>! z)M1@k2=mNlpp&8qLbdt$@-GVi_SwXzK5^$y_sNtj<8eP&>jC^dyYT_bf}*L8y*(|l zcvx)Az77SNktZMCRKy}cphWTDuU{6=+kzxE{d$9Fs_#Oc9fE8PiJ$zz;CaqfBW4WM zN*)r#$`E)fwGq!3?58OTX>*l#Khl-nTu>?Ybf!00JN^ek1L6AWQA`jyric~D1I z74g&xxYELfE&@+fzcCqN9v5ayh-Bxp?>IA02p9J4&wxZf9vvp2?fYDboLePBQ;?pR ziOLV`r~KFf)XGR4A|rk1-m{R2i#xY~La_j{;*_%bDZ5I6A!dEMLzDU14Klk7e{sN$ z`Jt5c;A;Z7A&IR989&D%o;gUPC)hBz>o`J!YG&M2NyI~cT^PF_ONbYpJ1TU}pa(vv z7BGJ=Htnq|4x=shov+<8s@V0iYrE4Xm`}Yuqay}fr;+13d4Luq7;c;X_Tq&r|M`>f zbI>hWsXmei$imI2PLKhJ`uM;~rt73DWD#VHeH8hETrq5b5D^92d=GP!Jj}y)5MbG8 z_-js=d$LxOGe>4#xvqkMnE=~RX|dVyHzaxXX|5&|c0c-P$0%Ssmp)}JSwbw8ZL8V*5|OE-c& z48BilA*34t7{>~rzFJgQ4jjfUn%E9qiw<2cO+Wy|TG&bEUeFDEaAdO}?Y)fX@ejj_ z4E4pvPUQz6=Qeus9f7Cu^+SGhdNWAejJBR$j#|f&3KT-+1WSY}+N((gWqX4TGjBkE z#`h^A99GskxRu=1+I#I79CFGH%1JZE_M~3}IqA%+1m1zyn&|dtB-NAPuqRlEP~7t< zW!Xi^Gadp=rQzE-mN~PuLymc}!}R9AvVZ}*vks0|9mKRZXm%!Cd_b4Gf++E39AWJF z6g_=okd?Hl2h!{Q!~7J=i@6=wF(PmoYLxy4tapdMk(xM=$T)p-8xCrRl_n<8B!4FsrNKR&l7loIgA9b*acc-g%2drJsCuTh$ugousH{h-utb~ z0AVTdoY%o;{yP-@Yf_pbDnhEh-v!r7SFfeTx3IUwd#hE5D+s4sR*+yx3*$2JvlFDc z9SQV2cHxv@U;!_bX$LM&Xfq{YJ?;+)1bCZ4hGsJt) zE#bh-)N=gsm6jOzESb&(Zpmj)HLP;WoT0 zAIm>9P7!+C7#KT|JFc7rT~2xuc6MH>hp^`R`D;Yq4dB$zt=c(?^c9HJ7KlbvpHn&8 z=2PIK^XzlS^?GIdAyt;_EX(!gp{Hf*JiiZ;Y8icJIApGD5z`fr!%erinEZ zyHZ2;XwBfMd579>#~ZCZMi zoPlg)*1dLQ1;zYo-RI2x@Pj#dwB32;!k(GpP*#}M{f#offtlge>_+0dpZk+n;`+}p*plL4ROaok)S%SV^r(szq2(c^$udY>pY>J{c4Cz6MNGjB6=%BrXWa6 zP~$x^M2-%E6PwZEHFIUR`h3Qng5|;Z;N8yI%H@Oe!BAiBmS-!=eNit98vp5j&#udE zG`?HDqS!rF)Pc+lknLB-oCo#KQ6(64uei{o@P35GU`b#KNAb12l4u46?9;{doRYlu zLGr@)Fv42{+%i7Ds5Cz!24+Llep=)5;)DfB^8$_M>vva;M5Ng+`bgjq@jLE^X5 zB7}Emv31l2B(z=l{oI@?P`Qu53yzn!^_nvhw+pvm=}Y8^DTeYyD}OkB1tH3qnZKUd z;=z){uURk^Ac_ttQ3P8;XTFsc0B^&@{N4A!ry#)2SKd1N5@qi{qy4n(0DPP*djPAd zIj4h$^~@$MX%8zE7Y|vYpvtS;m!cj_jm26E@c@tfdT_`hxgTBo(@@eJy$@q&M7l4b znP4PdT;KdCKf*`}jLtj?VIS!)`TsQ;Gk4tF)F>z)FP15hmc zsM$$+#C`D$VE=XpPrVX_`?cnRz93J*O6bA^V*I1Gl|WWQSXlUCPM@UBlo=gtI`v3F z78~e6b^3JvRUlDCFBr*CGxC{v6D*r|7@NC&ac}L%nAOxrvFb3#0Bk3Bu}SfrZD9Wf zQoPpJ-UPwim3r~E4t$L9+IDb+`1Ra>8;Q885o$r^N0Y`1B?U+vnYS6KvW2c0pmxjU zIqp(cwlslI@+JGO%@wPV_dhSy!l*pjVw|m`;TU*a+w|qO%&2PODs)+PkbD1x>%JbXB6DQmy&j-ql4;C@H zja$@dtIyni;XkNzm-cv>t!B8<%Y9L1E()z@>IvP4yGP^4pB;Y?)VzKlUHaenq2apW z)>Xq_r+SiF}2RKj2Diq+fGR96GHn?3~?#fc#@PmbO|Y58P0U zcfC00@Gs7r%5x7NKDq@cDWS`c<1wFAfb>sbwdlDL)ej!JEexx%2sD9i=oGAX?Z8J{ zjar-=qT82UI_VO#f)}SoEU?8q5F%walC%yg5?5DiCAw-P#bxxNE$*PNO953+0>RQs z=DlOphrnyUR#xiGus=9|vk-jV6Fo+Np@2gHI+ zsLiwjH!j+%S=7_<;h9Y+;$17L7`(6ylC>kCTyZg60BPn%05vwn=#LQ^HgL&E%sSLH zp;@&UW3tDfC)4{G6XZLJR!$#gc3t=g22A zzx`H2{2}J^gZ3vd20;(q2>kSYuR&jTJOD7!{>v({(@@C)5asYC$em;Scft`?KfBoLNkeZ#u-P`5C-Q$HPuDc1s zhREE>w(vz&;Y?hA9ek&cz-s486sdskJ3x_GZP2ETJ@Nyx#~SjX6geQ{s*e)Td~WC0 zcN>~tqZyjWkY+4aoxXx|2`)~w*Bgoarb2Y50mILVn6WlUROSRL6lg$!VnoOmaO%0+ zf|_9J4v+48%^G|$xp{m*IwCorAQ{*g*D+RkUoZ@_cOgt;-uF{|KdNH|y{iLPojfk!s6PoWC1n-I(ilCZi1?4dhDt$)IaBISv>=%!CFFlU;2efdxL ztlPdmMUuG$Y|NDJ1NM}*)huR>l;q0mPa2qPOBi>4B@77V#umtqb6G$XN=WNo8Q zmrO|?sClRW>hnw>CX8iT0&kz>qmXx>Du`ULA#?g>jUfslXF1eOj)|xw?zT8I`Dk8N z!;!|ShVGK=KUiz{s(_{C$YXglo`(4&XDQ6>Z&EAs;j_S6^m`9@1M~7vWBHsCR}~sW zSd=ArYAdbdIZ45gI$DTNUJlI*D|y;Ad8T_yYIyyG9PgPgt*Fo>J@C(fQ9|E^(FZGs zHv^@9P(}pOVx~$VMWi(iPVJ*u*`O!Jg$qx_lTAi1=yv+WwUhI4q32v%*R=nGs$z z*%<@#4(jode+kAE_$;x2j=cf$u)Ogj6S^!kAy^gAJ-@5{df3N7g&n@SvYwFX(qFTB z>}TWGbWX;GLUrZkAf)n|G8sK(4ZMND3`8Ha&T#l3?$j4fJp&{;zOg0o%&~Q$iA&|3 zgSc;6iXY1hoa{8LI8cZR;qY5<%YyC%#XknA-cFe#=b<(NDAoD$Yb3MD)uPMiUz^0= zm*v6cl?WvdeFD?f1qqaS2b3WmT69n>Vh)k?_r%EZ=M>*UN<7G(^d|mox$#VHR;Ajn z_847(S2PlH8gW%T!`=WG>KdU$E6VA<9UP)n2%t39z09(FH&B2a?2fQY!{Qd+9m5a^mZ!2e8yd; zvS#J&eBev#JFmx;nYX%S@rTCq>ZLJfRWZ0BMxH*nd2w?U*GQf{)nBcw6HnNPZnu=1 zx=Zm}#)OJvurP8g=4T5F^&U$_Ip`YQBc?39L5$i2+$3b>P?Ill^+Cfj_MHwO?NvA& zUwvTbmmWC2TaM2@Y3if8AaDztE~4F_NHML^E=ZHO#?Pv;nfZNV!o@7MYxr!k{G!nl z`E!qM{~}+%i9V*_Jd6Y$fUlDu0>HzUgTJl8mhq+v8vxDi_%xhTqDnaavb}x%{NVkU zpe%Dqi+Hi4Sw+vxcI3V<>)Tt63oms!>+C{W72`%LaDW6TO|K>*tfg= zRA2_rP*EcuZu6yNqVt;R?~SOf51+xPfX6>=Cw$TuK)aX zl&fR5z(HZFdRi$bxba>3ctf9p znJPOoQwrtoCNRy0jDRxV^f8&YGh@Ysw>M9gBLOa6U-vi9gQ>gZ&1$a$}Mj?3sgCX zG;c*%w~Cx_aS>hA3mJ`&J`1uL8T%N(4-MJi3#$qb)iS&vSk9!Rc-hOxI0$ApZ+Ck1 z`*DW%wMc=W^LZ?Kg@C~)fgU)#O@h8NL$z9EZ@%t~#yzoSrlQaA2*GkXkJmcrx_HLs z1pA%rc**IQ9b)kvL+1`5I)?euI#Zj-hp|^3QD_B?t`1QWp?iv0Dv1aP=!xf+ZHKG@ z1^oimJ^QFFSb{g1uD3Xf6cKL>6PH{0>|mavjci-t|f62!N`psu%Tdsx_N)v^i zy)xa^ki{`)z(RPtyJ;BQX!+7Y8l-MtiL!>sGo>n4=&|rJLK}t1xH`A*`+$EYqfo7C z9@W)-eHJ!qaW?~!=@UB;Qh#PDyn;8H6w4$MM^3;6GK8d!;NdT{!wQb~eQ;{*>R%&S z)bv5oP8BrrTO&@E6RuX}=!sX5>dowik1iI{E zj^0g7Hg#JUTz4%N3h{di=H(d?llD}QIw8rtIA<-G(y~UW3WyQStt3hy*Hccw>28d9 z)#Bx6VRvU`BP>2XV84{_gpKwtBVU*@F;Qo$&c z_Ibn5mQ#t0^T?dY_cBnd4BL79E8hWI-EDB2GT{X5`7#8YV-2Sb zM}rTu&!}bz&NrQAE`?QbittO-LT~TYKGCb>tq}gP$yE^2{Rt%Aj<=q2{o`&I1MY{s z3jlkK#iy?+E(B&!!BL=>wHY;(M|^jQh?nL*0+Q9~6*hqT!)~X@sSk?&L z6*_N>-hjR)yaQDl6<+GRztSt9;!`)k*SF)ukpoc2=25p_PnOJwK$WN!QC+Vsf|!VB(ch;pU`WqX#) zk$;ATSe=8Z#5>BY4YcVqqVbI=`{_+6R zDw_)P)|-KbNB7~-XWcnNcFhPXHY4meqM=Jw!xl2u)~C*<_Z~=h<4?+MeJg0z1iLhr zE*d_YW1P?6mF1>F`3ZQkrZEp>18jO4(AW{UO6_H zp$M7D1AWnB*CCb14B?LW58YCSmxAlSYS~Q(kGI7SB6hKI zczf;vO(ch>=Ck2N)4$LW_jd>@h)B`L9vJb&gN&94#N3pX{5}8pG1WJ!-FLoxnJ43H zO+x$TCBo)&PjCK*Of6=vGiJGN0$$C4qr}VxOqUi*!Qj$AtLQ48`U(goE3Cztga50S znOOXZkV(u&?r6ard6}~k|5^``qsUwP)bp0UcEl#-kuQ5lmyQRK`5V!v$wVt6m{mu` zx8n`av0Dj!O65sBe|hLZK*@}DLT25ogsb;gJOWm+gtRYXz%euIZ>!#IsK&lXRCld< zA+le2vC^mLzS!8P!xsBo@UMz|%arNf?nPz{o73>?hGIpP-Wstpu!6>GXb=xD@?eLY zn>%6_YV=#*Gf0NcT+kQd`PF5yCvBXX{Q70gADl5;cO+>3TG)LqXU1~IPO$ys_-cua zfYD)Mrdtlpa`h@GdjI6~W)}KA`|3lN6$3ykL|NlN`W-#_; z?8}UbWQnv8F_RP}Q9==>MJcj`$TD*zQixJXSw<-&5rsEpMjJ&)3&}FFmTl~VS$^m9 zeLQ}DpMS2$xwq@Suh;c_1qUZYfk@ThaD`@`T6d#wYxP>z+WxTeEY+P*>l{{x6?6!Z z?h#M-z=ZXR%PN3B1XkQ@0_eh`eqyp5rd7kB0n;xz5Hl~2oTQ1G*sflclWD&p!fZ@j z9y&C>xxr~p_&N=e3_G^c7BTXUPw5ab-`f~}tYeP671WElfo`$Co$1Eb%c_Xmzsg(W zy}aGCx*VtWe#1L%dJshiY3Dtl+WRni*6qH0Q^;<4X zwiILQh!evi8U&WpSHKDNQI>rU2oTp+tD^jL-%%s+D|k?Ro?j>Nh4{kFxsIF}8@r9G zDkvNo`3k@{I?Y~`KdA7Mm~;o;mVha*-6sXSA?!y`BF?H<%QNa}zGH$UD;xijLPbg#mt#kN=xW}N<;5QV} zOIY9tc0(24q00~{T)o^69+Sxv{i6qZVS*QHJ_cuc{6(V3FA_wuqBszrV1>au_{!|* zo4vw`Pl-~3gvCD7Pr^ABjEZ^HPysy^`*MIr8gF+Nq0Q0+^-lx4W-k1HEdZ!*`0wLJ z%Ch59;86)c@BObP-W2?vcDc_yTF_O;X-77{UodWWP8|O;pPVV^Ryst|=(#?s@jWQz4?}qJKQQ-yLSm zZ&l;H>(iqOA3u&0+8T@V)m=q-hg*%?9KW_zkKV?rQE&*&7G*c8OF46QmJukCD%2Es%!KE(zvub-^iH2yOcs_qLyvW2VI^e* zx34eC?2<+*Pg};Vt*dS3IT3Bx%9moYhUoVo$)5fTy-crAy-~PJrXT>gi68NLtj&J$ z6!guBf#ykn)|A!J4|`^ac%GLGm5Mk3gE_N1Bg8E%rn!Yhtqq$VXsH6i{savWDOQ7E z;<_1DLBu5VxkpZUuG4b%XDKLemkV@Rjin+XUkcGQEY(aLfqgz$xq|~r&?7~FA^%qs zan~o%1r8=tz$rW@!E8>(&RuxM($gWSS7;$xC;a<6)!m$X6p)mF3bJ19hUdk(zla{*NPv=^`G`J#h)k&+>)uU>0 zk!0Bi1looMvMQ{rkCdVDsjpHN&#;}If2dS8zA>5dA4J&)PA@5Ep@Keg5zBah?2$$V zfg|ow#Lt2q{N~|&V2hScN}m_6|5o#JejlS4qCMD&nP_vxj6qZ%WpQ>+u$)N-+?D?A zhrz3%K;3CI;oj%-hyIH-h!ZxtW3cSsC|vjCBb!^xM`_^`g6KgDQJx`aQBYuKNNv?q}kzan-+wckOk9}qhWUaOky z21%N@6Z#0ji}D<e1``A;)RsWz(H?2>J+gr-hyA<4Z5|Ut+d2Oo=v3 zKW*|#g3Q|924gk zH?--10NCaYozuN*CC*kB z0yl-Jj@F6O?n;{)JpRe-vKCKIM-IHw*V;G3aO0lz=nPunN!S6D2ieL!15KSvkZVe>^Ds>wwv+ZPyH*8%KYewf}o8KCiK=qxP5G9=M5JxOo2><=Z{R@^>asQquHIV=7{A!J@r3rED z{mD{T>@dhR+6yrTeU395@XR>=>1!fL&L^c-aIA0oA#)>!wOHi@BT>v}{OcZLr_zP% zxMb^Z#eMA+8G@p7+t_vGBAgt#5kG|RhH?pY9N3D)XKjOzic?)S04>Kh34!r$))zMf z^y~%O9_Y-Cvm+)28shxEjd@9~>!iUyC`s64ouQlp+V8QWD+nZ1Xc+^uOjeR#hBmT! zyAOUcxFfRng0WGq<1D!q*mPx%@1JbRx&j5Psv>uI&C-~pB-9OFg}W+Qg1upK<~q4( zhushTM5!b{q(*9IBrs3^iq#wPz)+Pabn*p~*aed86JRl0^}}G~={291A}Zau_P^r1 zH(i~X2>D?<@iYilb(R5j`2bOgyWrXWDTujj@f;=v&{j?_L?^~t%_!J0_s*cdT!!Grm$)cARb)(c3W&Glu3Z}`geJU(hLtuMKfE5~ITjaTevA`oX=vXK( zMJ7^KWlhlRtP3(&Ab%ue9q{yjB6aVDtX?a=)dfSSwUPnjaEBSJ`Q;&yj(vC(X^Pd% z-%8mb{I(tBMDnLGbgcxM7*FCT%gOC*;JLt!ETAX&O^j@upB5Rx5WahyH6DA{;jyyQICSSbwV7cC zBE^vG?xltc>f4R?w}{$Y1@;g{f!VZbIM@8X4NHdV`#!X(uvUgFe&WY|@YK)&(n8jQ zF;?1o{w@u{uFH_*KmxEVQ$-jCM9P6#g>l^w=^O#ElDaJq zO7N_m%$LB*i{-Rd6X0lmF4GVCUUFa60W_SKl57dUz)(}hAHTDFnk2<(G61J=)XcN| zhj+OSQGbHE+Y1-9j)T%qE@)<#&_h$_YY2~vFw9mzzU$8iLfZ|X7^`(=8>7MXLt*VU zE%PtV$fJ)t2Sk0%f}ftWiB-QWUfhb6Mm^K5r?*fF#m2VFuxm>}&u$|YIST2#Mgo-< z+~^m@)yiyTNs==T1!+F}IPZVbOqn+q2g9tMGCKu&B~~}~N7DmdZLq#J$Z(H^KpPKU z33Eh8#&R`n)dl@?-4aAuL$r9h$JH?nQb-gLlM&hVVcHV92KZRj7-<^0Or%uJC3!UV<&WRpjEs&PJ-QdlLCIAoN+ zQ-|wtc*1N_*5s-8DKGP__`N_2P^yU(;r!pff6u-JReUW<`*@{`FkMb663bb@5c8Hd z|B^-48JxG1HC2%`fe@pHm$I_O&&_|Lwp>aOqs-jM^V8>(F!kfmG}CGPwPFOZ~0#m3;^PbS_QLg7Vzg zCNhk`TE6snAzL}Q3XayHJR8b<=#wc>li)02=!;J+pt~gocMeu6qSh@8N0EuEi7>WR zv`GuBApjEztficoIYHwfxGLiXDI#)sKV=5`2#m-91s-uEXbIzGJw1x2wSpuGSwC#Y zY<+0!4U2+DUq61t_V0Rv;Gy708$4I{mH%1K8XfbweTP33d%U+9kgAEc!ufe$GA8oo zo)!PGGluM`k<+ugZ4ru2aC|qi)g4kP1ve8|K4>SQ6o@@~AF@(oaQI44nynHvsS8#U zD|C^o0~^_Qm3bHGK^qfjjOH`iL7X{YGQ9-u{`l|y#%Z&{i(CC(|Epi zi>k1QfBx~hiDHY}xn^e5eTs5%ubH;d2LyLbkSxnEJK2|~G^u~05;fzVOQ46q%LJB9 z;w82yddo=yEF(}Zf>cAzyB`2}D@zf}dt+X#4AWnshDs329fwE{ZEfq?yfi)4Fc&8x z;$6QehTLlWTm2$va(LAp7A`QUk_-_>|0mvwoa?thz6J9ZPoDQcI6BqsF8vb66~PjqJLhR8o!xz*m4ba=1JR_)%-%Fe#%(%8kUtgd}a+fyboWs{fdU)2YB zE#g%8zwBt+wpG(%kUUGG$&FEiP<1W_~ixN1k<}2(^8_JO8ezY*silacDJe&nIz8DuH*gFX&Hw zi~`K%D?XJ}QC$A8eWm+rW?d6_WQiIRo>4DidmnU*@koF2N^ywKA3H@R z{mHM;TMa~QkOLL$p{3F;dVjtJ4Qb=o`C5yjx;Pw$KGVqZKte)x+aJLkZnr?bg5p6? zfG;n5V8f;uDm=MpL&Y(tTh=lC*Vp zT)`I}nUwd5>)>6=kstahN!lT6A~@rL5#uDMMP=$DJyJboHVkdVPzj|_Xmx}F)vCCK zhoJj7E?D$(UXD9SU@RD=P8-o)tsWegP+)yc$J%cknH#q5={75_UYOlr{gfzH3J$mH z`~14T>PzIy3xE90@K>ME-i+L9GHyG8qQH&2?)#j~LnBaFP#9}g&=9+KuXoK>imMK& zkY@YHu+uQe0q_~C@~;oe5F`D7yE3=oyDD+P`ZiEZy0u8~QlG(@dAYjMyljVy-2XzrkX%5Cql;Viovmu>; zuZW>0^E+S#xJNCD?r)E1rh$4R^)f7u-BnfNPEfMtw0G@8Pkwk z82ET#Q$x_Uh!<(PeKOd=p1VpQvd1_O){@8#CswVT0Hagz+K;5t1lPV$2UnFE?^;X#D)4H=(Jvb)D#t$s5>RUHUuFiV>>%hIYyL@sBB?&w$8?hQ~A-uXQgNDW($#uBB6{ex}yC7S$!PS?mcRe_Da#2I7E90D#l3)0dBD;Jud zbJ8vgABo~=FD4!lp22Lh+0S3BM!&y`&Y*^5dh~_*J-)Jp`TGdVA|PaaZbl68UH#ac z)qk6swM;P#hVdHV8aw9p0ZF#%;G$1(2W@P&?i2Y7Ca9wn3|1-}!ZaD5bLAhV8|!Rf zxv1_(3KW|NFj49}?$aq)7Ms2oI)ZA1!QXMf-v}xtif+9ZK7I1_+#!sL1a;ASW5l}y zsAa*)(&7X`_!$OW2FVLrJ$LtQEa-a^qcPt-IqF~g<@14>SJUmE0~tiW?_yy8#I85Q zxAo@qcyy$)IL4BdPCZxz3Ila*&nGZ|_YV@HxO zs9y2PlTPf71t1uPvge zilic3c}#|?K>2u(Z5IPa2FSjVU&lXS@&Mf8_E)VT%R+a4u0P4d8vs9u_Dmj8-XkQk zj3tqgsTT=U5%yi~!orTl#azVzpV80zQpZI1VUeJP-B{Xawt4*~=B8-{7DNHb6&=&( zXgI$AS`AbwSoK$u{qWA>sMlAYq%Zc z6}}R)rw?WlK&V79mK6%Zm8-*6q!xcDYWArx|BV|Atj4(&BA`5jCN7|{J2%&-02M)+ ziz^aAm_PW%7-qu4rl#WClI&lXJx8h)5W_P^1q^;+D*ie2Q)WH;s_(OhU%{CWWs`P1 zRSp~#Yq5~uk^Tv$BqIpmI6(A!RTAvc8dxuU-Jj&$j`_3Oj|IF@e+gvtB+6U+EnGnA z=tBa)2*2lwl9hquUy%3|JlGp5Lu zl=ro*1)nHfzO)b4xO0|#f8NIdwHlybe}gXOqnm$bRM*B-HqmbIhp%8@Pe}LlCGGMU zl^Cyv3|W?2&_-&IF2xvIF0&pCL(+pvthBf1DPeNXeDRVMv*M>4N`uuonGYq(H*P>W zfnZ->pcJeBem9!eFu;)`_RZiW`>OKn^P0}k|2$W3P1Qf?mnkp+uMTV-S`5E`cp$w}+F=Y`Fz!0lB)r)zCkpL^zbahx&L>|HcA{{CV=YUs|H~77SI>M|_Ihdoa~5isKEoRy4?WDTHjBxoa{BW( zgZC3lYnaykNm%4FxWSTSsE--1Cuc77|HX#oZel4hiL8~5f7LCFex>PKLGY;v=xWp! zNDOZBgCtcYsF7Vv1-(8@kMLx*nBUi;gJ2pLgtcZFQ4)j}t4&^yjjO5b{$wxf7b3t0 zJSsWoi8clOT-b3nd;HR8Yp$yUIl{);P9Kb`@EqHNe*Ek;*g@%**}%p&3S0GWLXl&Q zYna*~7x2bG7Bp0PO<;5SYuR2_Fv!_gA<4eoBPAYx_X1p^vC5Dgy9b6e^IP2*JIC*` z^R+hOHJd7Av&(@AoUX-IXxPT^xwnnfj-y^jhSy=tk6pP&`R~=uhd^z|YFYBqe!fd7 zu%1)Cbe!GK|Iue{L(YIJ@r(^ay>`gJ6bQmh+b~vB zhPXIvdnsO47m1VrvfvM7t@=%*0PW;_36^m>d9GtqUhY0Is>nlnYdmUuos$zRlCHG8 zox7~C2yP73Rj)!rj)Jy+swUOVP-KqNmiHd(M|P}+(L3ZIb%WReP=l8}CTZTzC$VG? zVIvI#Alpd9?6@*w@aWW<_IfL&fAAl1N-;eoaLD_8S@qH0cL@RnAZSqP0%IEm1QjMK z&uu?W%ozxJOCzlXtZ=JufeB?t2eBr*_6SbS>Ko5Ah_Ri*L}V40m*2IS*+2o)*mV?b zm!SwRx)jg%i%4_61!9O*n!JRdp_QB|wp5nA2h!GH(_bl289qvsHZ8VG2aIGO&b!WP zx(@ATprcXTC3boO5o$Qn+>I#1^O?kU+dKj|Q_hlV#?&^)5 zjXSkvb<1YsPv+(A`+WhTtB%4Jx~z8RvO$31SAjKW;yzik-ot*1Kjcy=e+mQePXkg@ z6Gnk`={p4y2+C=6ujB2Rd~*zXM&SviOKp>lpH2NUS{;unHR3lMt-moPdI)B0w|lwp z`8GYLf$Z)tKNcag9`U(qip)1n%Yp%jU}{f$v}|k`d+EnalA2H77Dev?F^tv77)mor2!WQtpfTtsuWyd;?vnxQRy#J3m`+ zO4jh!Z(wzhP38wT z>}(c)`+Sq1PJ!EHvC70B8y#G4cki?zoqyj_ez^R|#Zz^6y|kaDrmnNQ>l;w2(-A%0 zlec*(_rJ}DPW7vnx_8N+?wYpjUq*etTU8Y)=A7|{G`C^9%XAkaOPT&Eh4*%$?Derv z=Q{5)z+5gnc}}@dJoQ*Q#0y%rs(Nxh8X}vth>*ko_3nBAip{mCS8(l*f8zb<70=h^pIMCIKNiam) zmO+nb*XhQ>NdBLOPfqDa04I)ePJ~(rZ0E^~TqbcX<*zI5vzrd7 zXEw#(yJYheP&SiVpP)w&%G~jM?S3V3mN+lcDkUG>&d#F?oqzHTGPkw*w3p3gCT;8bxR<3_q?&#o z+?NSSR$wWlO8E6l#%$S0)lU@VO4lfw?2zlJQJ_>Rb2EEjy^e1#w+nUO*d77RYcTh8 zmKNgezKSEVlu@quWdhId(xybPTOA$M`8iV`W^rrbukQ-yCg1hG2@^x|ALpIsCA{U0 z`NhvetT>-FsDsXwiXx)JEruf#2?!3_@Ar@Y``gp0={KNMDnO=nhpjjW5Lu^-bToec z4;kn`ziT7)k11v7FE|aGXXAi8;w#5+lsUjsLiG7YMu_MyRS_C0Zh*yHp&Lej3)#k1 ziLcG^NGilqMBJ9O%^ zy`%324(c+xKew%{5SZk9^BEyFP7xO`=JuuUW^MfpLgjQR2Yp1EE^U1C+Pn_3zWW&z z9*2`o!UZW>Y|j}hN?mAN{x%Zr;&lg0FtmHNnAiQQk}AiMOAs2;jNCij-L;OQA#RwR zj+W$!eb^+4#2R2Ihro+TYt2P|QrhU6>bFMxTVI^yQL$A8O`O4C684a5t?f8@C%YhEdMk=Yw`v z)czsXP}p8}w5=N#9R8@={E@qmcQFfX& z2c^JWXY$G9KkMf%MPsN99Uz^6jKt)Im`wR>OPy9J_Vp^S3@T1kKyf<%4zZ2TGNygp zO$4c0h!J<@3jw8v?H^^nDV&)8`QjkB2Wn)yMsyJq z4BsiF&$mSWl{#BtEIrr=%1+S*3D6<@Z^R}Rxu95X$%bftN3q-@o5j5Y4aMFY*FjC2 zCnBb8R4J#ouciWYL+xkEVarKjl8mJ9=f@7#{_@I{dRJ!~M|@+r zm0i`h=lr?b`iRGu;`q4!yoI!>OW80w7hQ+SIuuJmffkc7)rkm;>5(xU`p?RUT_=!u ze7S5=HKsgms)lPgI0>?Wcj9LtR|nN%wo@$5cZ#372)XRt!OHVr0d=f6MqXDAzGG1l z?Ke+*o<MJvI*)3GEWYAdmzmF`K>Y`8 z#`3#5uG~NMk>c3>lWDf$^Mk`bexi0ku-H{vV)Ss4%mmu`iI&M$v^fsg;E%STLyzd+ zTjo@DvA~cTC9RM3KFi(6u94x-Xhdf=|L0hSHhcbQ#ooLC-h#io>W~e+S(Kp0Iqu4-KV_<}x=7W}y-PCEh2bCSy2W|-`n**2dy zcV0N77~HE7GgSp6-j4ox9bd~Dt&2Hk=OXMBVJfTh13yj=ri!z4X9r#PDC;1Nr*X*7 zU6#Evd75z@z(azX|pzg!auq4A! zfsz-%M+pu>2@kNjdCmOl(!#IKM?lG(ySu}O%Uj(zkm+O||F?qPnH+l4rJh!&a8a zisP9$%{X@smGZ@g_($m-eeP3!6(;=Je~hY)RC~BUejXCpjD)cdm%rbQoIL<`|Gkvz zu;qFL9488EIDZcnkJLnw8+_@X{F5tNhp_Sp^?Rn9#{agRC%d)}Ls~Cili}3qB)`8w zKh)w0!W3W=$JD}->vQAKYQ6}4cRUS_6+V`X2H!+CmrqsW`SLYqH?ieUY+zy8rIf)x zoA-xy3v@k=PaeQY$Wq+9e_~B)QKZix>!BFwt9|ee>7p;AkB9euc6!6qSdGsX1D;^q zbiOR3Y!LNQ`#DPLB_mfC%N>r=GbLF#O|Fi}k(A>=3t@Vc< z1)JFeg3zLu?s@DT19sjQo5=Ff9kVkohY=Qi@Pqe9tyKZtWKifgz5k-b7AV*t-5UEEbKBlMUg5Wj6}{03vff0)93l$>?w2b<-!g9DF7$SWBgs4Y~k{ z9kbD;7NP=Ii+YCQ7cm*_a8Q z$&s8zd0Qj5A*efJYAod*2~nuRGO7Uq9;;f9ILo0$>>C3O!`IK(NKg|-1t&DqU=};V zWJR;gwf6c?l{!9oXm8(1qsFhgtI$}+TC0<);1;a1DZ9C_<=@8#v7Sl?nDR@$@_vf< zV(7-6`ewFuOc_to_` z-0TmTGZWQYQRNhirw@Z|fL-L|w7?IW`S!Y!&JmeHSu*n-#Byk``I__ZGEiT zqk1vc_9}=oLIRk@VZ{LmbjdvXB!&xapf2lNVRGLX{K}Mt_GgKaI{#qHRZo%`9wJn% z?8#k>Ks0Z;DO#k1P}EtXAtlI58Ce(P8d&L_7GMLhU4&0`locfyJ4Jq5KN#hJ*#wH6t%Uo@l zcY5AV#b`fR_|ruD`m6e|UyF>q2t$K{2XC!+3nz;=sH5s~+6~&v00I(Q-2X>)`o%@o zX4ZbhV2`r%cjy7$GHweN%>&4)P{AWSX^%Rk*&S`!hx)EGLRu|aX)vX>KZGywmzEft zLX*O$6l;`!4_@5$i>9)2>Vw(;1Q#i*I(@yPiWeiMTTl5rYJcv^-P0HoLM-wJnAKuGC?EHpX59W zztiW=6excXcGd>4=tkR9TdN5x*kzBgRfSmLUwusq{YCPD6p*<8TXaE6e{_9a-S5`Z z{=4EndFhn?bGKaS6_!e^h11uuLTx}WdvuUiE#(THfXF-jgk?N$J<7Yfe2BTvz9u<% z`KK_euC`C_`R0J)qKAhANr%=&9lZ06m9X zS6Zhgyh(9)FXb*>BB$x&Iot^2g&o<=Wgqk~0Yt}(JGzXwLmJ=ZSaAi|^cp#Ccsf|K zifnnu3LScn<#i}SQ>^VQXK^adYiY4uoWk;12&3P8C_EvD$$O2*Q@*mIHGe<-r^t*! zh^D{yW}}+{>F4r}^ZS8QccZvc_T-$|LPwfwr^jimiS|{nk2(FW-WEiJ$AWwhpY}P2 zC+;9k7;5&-hR%fjFnbDY*gcHbZ~xR}O8oILQ%vi-kEQwzCax$C!XdaJN+ezo1$aiOIy(D&FOkOT|V4uRx9 z9+H78bnHbq@M?0bb(?VrYiOM|5@ipaa>&|vfoA42LA#h@CPy#%hmGU~#U9YN%vIlJ z6mId`c`JtbI0!2y&ysPGXGPqSW%(D2ks3^b5g;8j>w9g?ee>b6q&2LS=WTOk?P*pA z!MZ`aFW>lL%UX-8St5EB#(g?}kw4L%w!3J@kM1h(R5sY*qJ!oivX`u7W|xwHtYG!y z7n}V2PxB>q%X{ofrbf6mRP;W!&lGCva?hPncnm0CjoZ`8q$KGHuzjkaqq_c4+;i(5 z{E!cEGmf+IrRwe5(A%~b*|->$LOGHa^g8zm$|M!tjS#S$yC-ylAIMP@^7tSUuVJ!^{Ku!hRkCE$Y4+3F2i`+3^&P2Yp?q%eijZ3~QV!KcF9a9)@)Ns_S#BHne{3lbH z#=YBMS!Y0Ou;P2aP7`DFT=gFQV$*Oan&#^x^!pY*JrsjA39;OTXXo#kbs6&+sH?vQ z-%B7M$NoML-%guO2y(CmR}?81{LinlnDxmGKJ>$7{Xg{(Yd=!@UC5^GMUWc%@4C)bCsLGt(RpdI@lG$SNw6GffxPb=V}?C;MgQ~7DWA3N zK6=-aJCFkOcczzpNP2xSqID|V$!bi<$0NIQ|n!UDK}@NOj*f9@WzsYxeaqB zUq4%2_E~DF(2!ch)U8@uNB<0L^2Z8pvk$gD6=TuOGn*E--h@fo6rW2i`UE$to_O|p zMM{KuW;0$|Mcu&n3hb;op=v$$NZJ5+&$&biBuphFZ^@EkrCG#z7mn~ITga7P3rb<6 z0*Sco3s6%T6m>kb4t;w&kqJdDGQ`@GXN(_%jxDg3_3L+O4k`z4J&7}e<10~S%TQVS z12-s)okIM|@39B$H;+s5UH~V2Ovgd9i7LYqwZ)a=6B$sSoSmeslbsQCSRV{8=2AU{ zV^@zsx0{W>T>hnzdUEc0-Ca-b))%s1FZkR-!!bTu^)DtXvYu6mb6i!HO-J~9o`QwL z;*7_zkQH*;Y>gt?ftMX9;x-HlOj%AR!eN{uavdOsj4H`=1carsEZ}w{?6-O7yeZG! z)s^!0{dcQk0R25_HZ@c?RF%|5LZ6M{GtC>aH+~SF3=~p|9sb6 zK#Hsj#USGtr1ymD`a3F5enJ87>}^1Jb_7eUo7{0ha0GG|S9z4YY2Rg_$5jq5j2}5g8zBZ@x)O3BsMH|PFatAAhRD;ZttQ!+wx+(*( zpTRzus~Tw=X#+=2-ceww%VA~zlPOA79v!UqEjk2hq!?1epkI)3GIt=4*84R9XgYpu z3)~{tWH9uK@i6X0I@~lKPx;&ybiP1DyKH)M-G&BwZN|ilMKP+D+MsW7{c<5;^)|K} z_w7cUFws-9b#!(MOv0MnoRI^L3&!UfoQ&?m45Twsurp>rT?bj|ERGkXkDie%ClmvQ zDJaIwigu-+CEW*%$J9+?Y|&3*l+cZSC=%&DCUWgxYX%AWF1z^Y_iTP{-^M|wLi}Uk zz2nGCdmC%IcN#TXlN9JnuNd^~WU9)#`s|Ei9^u-9JXR6r|d)HX#g{YAxJl zCAb-aqdapn`5giyA7ZAk4dgpu3xph&)MOs+bQBEGpse^xs6Vg7{FK|qVW;BQdjKyNlJIT)B1 zF>R*t!iE~P^5@I`eK^Lyu}uF}uD8Ni<73bq=Ke^GaUDi>xB>b{NdyO0g9fm&(p<$e zJU=nS01<0z+Zf=mz(uDpK&Nny(LrvWxIPke1xAaY5ANz{v767>T05Y2GJXbBEHr=7 zOJwX?HZG-0Ok*#Yv4~6CKQnjh8L~9G1tscL*8P4vEioUDWmz$I#|!r1w0)d`0_34Y z;gGAtH{Won+CLt2lf8sKHQlYguHyj2;C8%_a+B${_{*V^`lV3}92lBi_H`5G-WuJ0 z8EZ0_!H49F+SYs<1-k#Obk@;MjGcU>~yurQLEx zBX;f9e=`BmmLLC8KjFa9Gb(En2l-by~UR44hkxkLPts^^_{)gP5K4g8xP9!hj={ z^)QD`AeFZdRA0-Qzt3CBS@#aqd$Z%OM%5W&#?VWFVsKtA=u6q5Y-lbXeComUzUl#wwvRHE7RO)RMX zc5No_jw0pM86Qgu7_R#JbI*VhD@G6{z>a`hI2GdrJcS$kVIVhPfP4^CM&W{w%g^#q zr>giyGyA%Y&LX$t1xIrGeQ+qu`>=>gmHk@8UXYxetDa!X3QYpE8R0G<>a7Z;A7E>0 zDWuwyGmeR>R4}hxnT|49OgjwnPWJlXRURX_zaz>hzkbT&VGS?qtlJ|-o8`q2m~!L~ z79qgc#N`oJ|CLS~tn>5#oYhz%lLI>rg7xhQTagrM8Xyft*9pEF)x+&kw8aJa$xbU( z)%nIn-a~N;j)I`~A#fKm47L3%bjUJ_=UN0zz7%K6?gukfm%|;wo3mn+uOgh+BY0q& zRqvB5jM-J?9pGCTH2r*?D=59*q4eI$5Tn0tIMc7?rEk{HU*=uwb5_y*dnJ5;P9wc) zcs+PLIO@6j7Rzb6(5u@HANeG+7KeEMFQ<$x*pH{~5v7MTDSIzFp|ILmX-XF7!F)si z+BkJJT>71OR5udW#kQk|QQ0!#Wpb29sBlt_vLUnV=#7UDQ>R~!r5b(Q(w-I*9k&Zx zZXBywK3do74n-@9V(PO3l*Tv%+Z2G)RTFa5C;5wwdAp>K;9eKP!hUUBaOI)64W!%h z_KejJF3y|3B)V}#e=kfvdF`<75=3}MBerk{PVyec+{P>%E&Y7?ypDt z384B>v=chmijZC?@)gT*={HQj1tPK3sIfw`U8kJv+#2-RRWRQ{J#_QtrJot{$TMKB zo?WM8Haxndc>Y_M_4SX|=ezu$9jNqL^I;1f?1gM|+IBha1jCN*nIh!}L9pf~R(Lj! z!fi{-G~I`jy+%6#D2fz4dg|zsDSBbHSK<_yjyVl4XQ2m`^1Qen^2`Yu%Y;+4;UYC_ zP5yXv)4^N#Bsk8TXp2jtNjjmR;MLuqRZVf zk^u$(k=ibIq|9uq6OafQ0exhvnC|fDXb2g49ym8V3a+%5CG(v52j>awyKJ!S%^&iQ zYupzYdmx%#b0=MxcS~zPXo6uK)ku_p3e>YUAsFg05AmXYnI0{0*-DfbmB)8upv0E| z5neF>1DPjCpsd460a(`$$deC|YcQUJYQT1zc-)?t7i+a|X{A!z*q0A z+e#mY6Nnc1fb#2J+Rur1psQ&gXev^O3zl)-A#DguAQLH*9n+sbpY{b!E;JcfenbOg z@?9%eUSMhxG%2Z0E2V3gLx6p}2~x?$*nsyJE>D~c69=bXMlN~&x8(`MT9U*;^`#=L zk~S$|Hvxjfu0W(VHwO+;!fRi|eh@iyjpxkcFjIb;gPOX=hgr5pdHXl|O9?%n-?bdx zd+d}(C!y;f#}cJ(jJ5AP7uqkCE@t@SL5O*RM{TD?J$ZBPi~VVijunMrQ8Nw z{Wq`GlK~w;n#+HWP^3X5=oB9K3f%)HsmkzBfgLhdwzGo&<{0f^ICwIN6lt+`{vJiJ zVD*_6Tm*4EJf*3}u!zA*M&*3|peW+zpJGRQj|mQ&06(=ZYOO4blYBchhqvVnyXx`s z2Ctu>uZ9;siH9k~r4Oe3m*&DmOSvP54}g($980WbLc}y21iU403-sfRkAwN{GsYJn z7v#n2_ob`y%UKwE2A=XZ2g5xU3|1--buqB1B4@X51IYlXObrEP(VB=RfOQ}n2>h1R zgV>gnw=t0|z}m8cHcUl>mFvbU{@~b13s}E?R;_W`>#3(v zM(vuKD(6OyHZWNn9&}wz?^9>z<#6_5-NxH~4u+gVEhSrHUtpZC#_n9H>1O6(+Mg27 z=4!MUT|Kt+E-dXVx~kBol+5GW8^27ErZgbmJ?xfgL7;1&ZbneD&L!P& zt7_-MF9+R@5h9VRH*zp0;iGb&7g}c*ci5~k3Eez+0DC}b{L5htb1(wEZmvkem|V}R zT2)p38`KHe>iks@GFREt#4zpIP*JcTEV<$@F53c`|2kenFj0E+wosYSt^)3>fqR9( z&IAMyG4q#P!1Y4pDmo`80mTd>$TY&}W$dL-U7puI?JdUraMB#Ac|m$gPAxMcik73$ zJ#4@6GgPzfyMearn`YQi@8b=`*Hwn~Zg}8WJY_WkvJKs=pMBdcALn`rPt{KN?sN;d z6MP&skWHM*ueq2c@j2Q<{@5>3CcntxrcW5~lczR8V-TBhpR(!h_a~~QDi=YG%+JS9 zzN;6V0bPX{$^*rn@er#_1BkM8s*b_OPR1CuQ$N*6`yFeq51t7D`+3pk{W3Q6ZH9cy zJEtlF{HnIEEB)@{^dUzH`JNdO|DoP@jay1@hM5ix`heB^db5D^` z970&~#bPglz9yLuOo#P+d)|@NLK{VL1h^)xiG*G<&nq9zmKUDAg|L=2VhB5YiH=|_9{zlp75K)vR zg(9O8X`w8UWe%m)R*E)^N)(||lrY*Tl_`p{P4=u~9}M$5pYP-G`wM2d&wZcky6)HO z`MNm^F-5m64kY&^s4&$YBJszyly^?WWCL2^!j9U75Ye6>PoEK0?s8$8@Kr$B{N^3u z%m3E`49x!;lT|_cy9UK5>$PC5vH1zFq++^Q8Q$}hCmzVIRaZT)k_69i7xSW=5|8|7 zSlQb`+(pqn@+6{?^+HkuM--{R;htuA3D`Ot74*}v?^ zHoEtEsti@ZV+2SgiqYP2R4S6&VVA#TGP1$}<>V+~uwq4E{@ny|($XPB6>&TT4*jeg zAQ=R{wX>G6EPUd3p_&nR_#%nyJ^pt0 z%7S}E7>m9lFbxzSJxQXiiXjC?md?&zUM>*=TV-++YSY%F>cl7;F_ud)^>t0n3cKO# zihnvUQcvR|!DtYpnuXje-INe$g0|R|CWaqT#sKl`&Em^t_?dk~R&c#ywTU3dG6FOn z0{8Wm(O=ayf$9v+ZMc~w7#VG_GVMB zCo}8ro3tk$T>Pm3mdoyXp3Tc104l?pTFB6l#WiASGL|7g{Xw;79@+6j8Am(kw)|pL zuqaFnaa}atkZ-+h?)T3oHzy-O#@C%r(X7KDhT8gy%6AwLPk$O6fpH+-{BQv(mDn_d z*aT(1tM?Ymn_C}WQyKbtf-EeCj`YZb5XTFxx!)axK!HL=g-)+5tvmTLV;^LF2o;{%?l8a^l?r3iB!Pb!6(^{xkSfUH{|;GsXU_{U z0yj^A<;Gpg6?)pQ(z+Ylq`uQH0P{X!bc-ESG4%>&*UaoeRn(rUkVhh7J28Iu+$zK> zlzg?>_-g6BYU=y_qa##OdeEcZixVUbSr8etY$JxnToc5N_Lb)_@eH4sa#TX%La^WI zcfmhF_>0AdUlZe1kjC5MJ9w}IB={1+eWp2<}9W)(>>Us@;fWSkRaVD-&#)666?H#;*Dx+}z{xB$p)V*A77BXk1JHN0F z?H;j-)*n142;8>p4|}m+1RSw)!95~>BEK;kn?IAKp9SNczmCb*ZNmx*Of}f;Z+q4p zO}EB+jH(vM+?R+#(aQb%Zgu~NEUhA-M z3M%P^cuT?VN8e)fF=>f)eaZf!cF0_8p6|>U-bjPpB~+Ld>hc&g+5g+ZE5UyY5ES0Y z26;yZiq@;Xhj={zosUn{xgfr%ntQ+T@>7$GqF`agfG{rnOscvO_;{)n>hBt=nePLX zXEp`n&n~^QOQE}uVp|C z+loj5o_R3K?VRmZsw)WJbmOxd(|g^jM;OHUjURpfM0CYO1ZFEdDZBDS;MZLaU`({E z&mKQo5MpQrh7B&Q#Z=@6U+qAGr#CRyGNUcb(pjA^`9VXSyi+kc)}WHDq^Vb z(7S`lmkdZKt+wF{;8+XB>wxf793`VmaxIrhsN;m~`ENYq3eV?JV&ua(5%jCZHc(Ma zwbXD{OHj%U4Bj1T))xr@gH}WD@nZ!>QwB#)Zr<_?g!Aqclhb1R`vghgc$Zdp9cx3S z4pQ~r4GNd6a}`7C6)C4;E%W`Prh-2wB^9c~QG6yWfLJ9C$n**3&9Tm#J_&2-~;NO!szucDj=hW;j8;WwJM00VK$OI zzd$m5Lr*+Op!!9NRl;8#eEYjF7su9J10fOW zj12%lYf%?o5;n@+P-GT%d44jrNgPe%z2|G483(ter!=*GAERF<(y!~NoloEj$@3qp z6?JbV{nY??P^~ocSa`l)1@%s$N&|^uERv&0Uy%K)u5GyG4<;Ak;eq`3@3(N_a>7eI z#!&X*!O5OYmcRvYOYx)sTl-oZm#20b1h7KESEL+NO%OX*kbHkhZUI&A+U2xrp-y%VU=b*st7{0A!gC4&w|&a z!)m*9_clX`5pOM1pLU_WzbO~h_*6-`Rd^=Dnu#8Rc~L< zkJ?QS`^CA>@0$2B5e-;Jr{5)!;gN?Re;L9Ydd%2O*gMqe51tvQ3XWA_L6Q<7hFEn! z7+@fq-yrMRh5zCgUXc@#A%T`;7zAXYrAVQ{$6g#(TQQbN~LXwYl)-;`8xEliuN5xv}?t@kuOx zoA1We=jqdn?^SGvd~pAT`)#g-85GmiwTUybfQGM*7w|>BIP_%&Jk$SEQ;X$lX6@SI zz6zqmGXZz;>8ckmdYvLrqsI|o+*ni4@;-sNkRt$4FhmNy3X8D;-f)M#AOi-G;HAlO z=QnP~E|lyj>XIISX>x%ft~YY@j=k%ib* zk}#z!4*3bdapKHXi?n5pBos7V`9>2&-qUYeae|`oW-;lGB7iHv64>0VC1d+5F=AHb z2fyBT<@+rnVE=30toP^fJ|W;qbe+Qqhns%sz7r%yWA%2|FotKUzMc19nrr%%x^78)LXUCPKepm^2##PKw%j&S~ta-C3L=U z69V?Amth4sEpq=ZmORE;Szb{?%)n1QB;kGU)+*(^)s}9_2MMu~MwPgxII)<2mruLH z7gd>MPastFJ76b#gDTB|w$0ZoD##i~(hM9s@qSnWEhB=b*phVVrpi6^53+5q$Z+WW zl2Rgct3Va3Pg;#-z4Q+S>_=!z5+No;A?N|{J2=%4jbQ^M(NJ(o5c?nQt2zlg^Bh(h z#E_kh1R~(~ry3c(KakIYyhPFCP&1SOVxBBuUJJ%Qp(!wP>s|Pz_kn9)1kFLjSH&5( z&hhyjjnMm2zVCOKZNcL2-no3Jqv+3KN(n#QfCTWxS97xf8C- z-s-@wbIvL*TEHB55v|u^omX3xKljg6O5AfQ+o(k45lko?1-0TjZtn(syC^eC{@^F~^g6P^z-nwV7DfYrig?^`1&*OZEm;({W1a8C6}Lenx?I%< zefbFsBX>?{jh#-swi9>06uOu{vQrSG$y%V9pyTOoi0z6~8Zw=@ra?exqRjgV>F^yn zJQ~z}=^H3b#?wBSj9xism|5QgqnimF5ndfJ5^Oa{u%1g>43ys!!-(-tmyz+qyLVHljOg0&&EZ(P0IJCZ@Lrpk6%30`{98OTBKecQ!?3^1dN8LW~rA|5QK+fHYZCo zoWo%fP_u1;l+dUmwO8f=pE9H03SsF5^Qa7rq}|Ew*oZagd}*gVLR00 z0kego0TGh!=Mc7WF2&hy2MV)^)2%zEvde9fN8Qf;CI5$5VGfGDfe^)gNyFBKX@0iz zTK;9)0eb-4&WhO@{KQDm;qNn4QL{(UK*=d_cp}lfz5VM~&ZS1Wo2$}xK}!Al(?cj$ z*;Wo03;|X#24N6G+jR-dI6E`cMh&Al1J=B!P5-oc^=9rseSZOj2qse2j=ofVJ`qTv z3s2fU%`;dyqKPhn0e*@_#&Qs``ld~mb2#4TBEmlboR?>w7GK4gvn8p=W`#Y$4cx_t zUGz&2SCVhpI&zg9G3^5-}{hJ}|0x39_L89Mem1<5{No%43Z!y?*JP*|^ zluEd~t%9z?f!SLG(9=~Sn3EUJtT_O_U?O!aTdQ}Ml9Uq>gqM-Uc*dsyV9Zgwz3~zw zE=M;oLkprJ&6KyVvK7WQ%(!hh6QvLNTr&r@e{-f<$xm>cj1|EyF(y@*HZ%vJ0e{x3 zOimcxuXpALs-oKfg`}Ne5Atyo5EJYIX!~wCVy-ccys2>J7C^Dd3IYnAFhvkTK8QPJ z7%JeJkAwh}*Wjy{e5LM793hpjOlqWGH5=OMgiXHRJn*b7s(b@gM7oTFwuB|0vw6Zs zW|v{GaU#MD-F75`Ma*8Pu2j3rJ!_8tCii{MsX=p8T@mn{+r{fFh3_kjGX&vE2;Wm8@S?7ePYtY@r&AHpo>sCx4!)S((jC`s1d) zs`X1+Jc=h~qFVJ{jA=;UzZCF8)hf*CDvZBE{iDRpC@Zwt|KftDtrTC~oUhfj?Z*U= zsfEZr!Vnsfj_bgh#>tP!0pRsQ9ZA6cE4`6_-)4+Z{ZecX-Ep+basH~kR2om!|{n4kKV42AkoJ6WIWLvZR<&bwT`8 z{WY7<7SC@sLlp&Ndc*UJ+fm0pupB8tlO8#?r8qutZrA)-myHD<&VGqp223m9L05D0 zyZ9bDN2)agQ-1mW>65C1_amMK84Ef;F14H$L#}@Duityy`3%nH$6d(01xM-1V^~6* zk|w~3CIZv9g+K-D&AL|o2u8PkO#BSxtbNaMJQwhw{jm_|t_kv4ckhXNLbFO}!7_+vrQ9s_w@7a`8eV{=1Qfv=&T^Oh!O9STR+aWK&^?wgK zBsXC9-vFv`@OF;+3^7KbXMujX_fmc9CRF0}8p8#+fSS#_XCNKhM>)(%*OgylNvW z1D>JYy#)KDKCRug5MHe&P2#hblcCbuUm?RYn(7ZtmQpMK?~{K&QjRxpjvg#d!f_!B z^zuW!tcz2V>s2Viv?5UjBT$vuJ^=wR*Io&wCQbJ_Sf1^~UyRf=VT1dv@o?j|q05 z14HcX@BI&SO!$j+oC6gsyBZC2g)qDB%*k($8XwnRb?ljq=sSJEQ3X69?0I8Etb^KK zE7~tt+bp_`9pX5Nuyz5Tk>uqZ3)J`4@bH<4hUUika1c`vmSP5af8=q@5*I7P6Ey}C zpY!;wnPQY2{cYHw(r1;D-KN8ACZV}1V4x)Fwo*P?pQQLlQ)mC}x0^qxvNImm@4}%a z%ij-ChN9sIV#op2{TnsF8C-~2NC#PShMZ_On6bqL0$%|y|IN|3q0V|H?_@QrSX)(e#9=YzKOr5$su)=l2K z*CSl)Sk7nKfyT>OALaFYujQ6aDEyK%P(L|*>hQNO77-RNGjYCNUVSOilU!Lj1_Qym9*gOm{0$uZG1VP^OKw04ie(NdP)c}WiH_FNVge? z+@@MBIRt|1eQxut;R88N{Afe~WlK1gz?l&!?hSUQQ zYp4+N^4;)$84$KOwf?ufGi@q8e;ZWH@7m%FHv0w)?fZKOI*|WfN&`KC`4<;E?ZE5* z`{ls4&_$zeqx?+K-7&8`KhCC-l{6U(0_8E5e9p$x_Px-f)t&w!rc*`TB8ynrX1et$ zL;TJLTg>_4LM;l~rL9QY!O~VHH{_l|Q=7o- zZm-^bMO>F|;I-uiFrH<%zg-f2BgZ9bqOgs| z;0YaSqTP>=^LTKrcF6h9X^d_83BLSOYkt{sJj~)UJr|->^EPtsYbtff3OAEBob*G6)S`~II$H~+o9Bt6WUMo zGZY%*Z%@|zb*tzlpX(2PjAK0^B;zTO2Zqf87*Axaek43lo)DA@#51(<@vYN9oC>K7 z$|6_vZ=_Nc$WybGF8#lM7p_Vp*OJK8Yw3plyj0 zdZ^_S+@k)`?6D8=7ryP{1%yE3P~}{UI+GWO%$tdlblpsGl3=?eZB`j zI}EcIMsjbF;*~1?&eFnOz$}A_!0^*MCOXM++^NQrGoA8pU>0vL+4ooE9<>zV*dH^x z%u>Krg?{VNGBw$DMG&^F{XK}BO;ON?EXW<}e9_%oLC&4ub$YL#jc-#EkGujQyVvt> ztvLMyM^Ixm-nO}bt+6 z^F-@2K0@qZCD`2yRxQAy93;`7U&Ml2tFJaEx@1(j-at-N)mP;-S}#USJusTdDy2!K zPd2ARlriMad*Gh2Z1Hz>W0LLhr{QlI;QjICV9vC4pf+#|$`93NzIk&!=lOD%8r5ZK zSI9%UZK~ldZ*cq5zuQ4+&(rn%eB-zms&;b(HsbB$lDEbXIKQecfI}(Zm^b1IWVeFB zKjCN0t-d(kwtkl3ngX7k?6u;>F*6Ke_VtTm`LkK$k)rndRTi8gEDR=2A8>h3H&%fd z9-ycDOaY8oFtZjm0$&gQjg%-0=>3rYw#53Pn>i6nLZcH}FclQUH#<5(ylrm4-aC5P z)C=S|*HoM%vV9hx5`T+pjK)c^DsJ*ts2q2(_w z%l-7?oif-RL|}ClUyNHUHtfqkDvsHlPQmUIZ8m2!hK~f07(QRF)NRv;=4uv3pSC8F z77_&)7Tk_U@*+v>yU|kYz~38Z{cnx6{&vCh6PD^o9-#3e%+x_!EE#1xHNosl#v2%n z-+fTyc0tzo$;9Hp+Bd(46_>l&!ygk>w&DPr#Q0(9!Mne`tF`zFmntu}R*Pr+en{3u z579`7EA)UpbtH3C%?$Ed`QR6zJscqfq;?&T=9!WZs&MroU$fW%wJ!(81MJDJV9J;f z-c=8eUMX%%sq}IO=g8J&Y&6sf1O;wA+=1mm8Buo zB^~ZWepL((@rPsrqE&2Fmz_>oB*#S-bcmHl%;0I(_xeS8Zf_1J&^q2o^F<>L6WEW& z{yI>9f(IYP&$$3&dtl-wl7h{>4%NoeVi@91YTZBscnV#VMcS-|6*M6pQwtiek!sol z^k2@p#m69Ot&Z=kDVu)|r>^DzE7>`E2{I*p(0$xi( zDmF^$SD^d$8;qu)$vYsis{xOu3k(RcTe)ExSdzj1`m&oOh`X0;#C*Q4rn-Ig1a7Izi6g=|K%ToTI#hml*+9Vb<}uV|Qmcr$6#J zD%3*w<}KEg044{P6MRr>(ZK0#zlWEQltmMz7|(^!a=d2?jzprROx)P>I@ejFEUP7I z*z79z)K4Q(M#M_DMg8p1CUEGTI#U)Yj3TodpX~fe^#VrUQp`6&+dxAx2@cKAG z!e(u4#e*CK3M81P#ht;jD&#<@_)KJoeNHwI+SE^02_4WJ}FkqQRI zx@ceX;qj~{JI?(lg2p_pKOw*?Z^CppiRoSG5BVJnKf<5&quNogTu%k(e%RWrUU0{x zpin&exBLzv^{1Y$-E#TZYg=$NU-P&ux;cCU?frfcg&R6R3rD;Q>f{w5g)02X7Mxmy z43d(YTiG4J6&66khMqiKrGkbNRBmbH%aF0^Y%UEdVF3p2)c~x8bAvyh$(vi=& zf03`tQ(Lv8hu{TJd3*qiU(8w^i=$ix0r(%+o;eQma$atSb0sXby~I z^Ph0_Q`cU6tXJutszB`_&IcO<>zkEh`lCa?nrbmZ3IONr`|x6v#`$r0aSkZm89j&^Wt6H zEOUA8$n3ETgKdHuz)KKBpln%Yd$aI}Y5{$6=?rz2=ZW1~%3yD-#IPUavErB}BC=(w zB>U&}#etHuc%HsJ7MR@K{ZdfECKb;b{4L#NElPO^DD*6q8=Tg->4K6OBT0~4{cR#g z;6JeGYaDjvhrp{U)R|5@7BxFb{#a+j#$T;>cJAyhAh8ehFWtq|F}J2%EHZ~1GEQ9c z1>)S24?8ng?RnXfsAhdlm~WM=e{5zt|A{FuE(PnZ25ilQnOMCoyfFI?0S{xDRT?*t zsgoo|%HsB||555nxW5D{XVF>H#>LmG+vjRTK@@?tcc&Ncm88(zNe#)FM|cHc769p! z!Mj40zpe?(ngfvCko!ltu1u4Fj^FW&T$fQlm|vkxi~GHdHJo4K9!|(T_IWp z!H}!qfJv>H#tz`Sw1>42CwAw)}2#k+fcAIgkY zGs-?Byy8$5fTJG+U;It5ljEv7b@RRVHfh^ZdB!z_!vN{7rzBI?B8L|yC(En^H@IZS zO8|rR<7Z(Cl)}HT>upE=?Nfo|^?vuk{+(&PY9`>F25SGMN6~@hcE6JZ9QQ#EiLw@w zHfr+`dhQ{3&SUKMbM!wfc$ZKr1^ax^Dr8i5c}b$tY7(hJgSBh0B8XbmWNtVHxXFC^ z^#N@r4!q87h!Hkv9LZmC>3{C8lCf9-I`-z~nKbAE4LEx`KH&2670=0sUf@G|1kY5C z@eP;iyT|u?*3`QOvOoX+U_YJgu5}yncwL@kG27#Z{mlqKlSUHF><|xZQ*@M0{h%Qt|2+h zeeE5avLYqPWZys3D(du>e?ju4*OBO74}$2!(n!O}?9R@8mxwWjNot6eHY&D9 zfYW?Y0}+9+bQ#Pd^XY zKnfbT3X0(vtt3qv#}u3ujCn7_tW-umLH0|JPBT6J#_#}Q8W*fHtI{Q6Oci7)k!SpO0HCl2u~Xd zqqz^y((M}f=5ofnHn9ku}XN0@`5W&c%JG42~iaze$3h$F%jER|4wIVS`yw zh4u1BXO0wr0~y075U7hNjcRsy`y%lkJ4Zra&)1t?%bq3=LYA{z#lt@~(iPL~9HlcG z$Q<%jLj2$@38t|PWL+e|6qdv@q97fpGTMWurJD?lL^HGN-rrZ*Cmdp89SBqob+E#q zy#)dQ4oII{H4Au{)QJ;W8h`u_glC`4?)}{GdSpT;YSEv6sCfXw8spG!9`$he#c4#ye+fd5l?n8jLpox$LHK4nEcul$u$ChLs9s0)no`O!} zIXrbKBvn{j7QX1o+_MubJ<=P?ezvEroS*g@&3g~4b;K=#H6}EKMazJg^p=3Jh}%!u zIxzdnZTjwg*Ao;^$X)2(3xJx4pZEe%z~p z-WCXg=0NMH{Z_i4lH`4)ho|ZV5a=d;wG(V89t~T&J zlHGWa_eu2^zbzGuy5^mUj=oe|&7GOK(naK02oBtWV>-QoDU1lP&Y)&e%#1u6DCBaI zqdF3&tA!5OPTjTsp5VP!Z7#@sE)zgHw4b{{O5yw_M#QaG%@H|q{|#+HYc~HrQ=Y|B zSvHTrVGqB^n7j>bY3Ws6)6jk|X#8g(c+hWYPhvh@smV?7E(bh~d=mdc3wxwzYx63R ztM$9VM(0od(B9CQ*DnOC1=MJJ(+>XIn?2M_4TL)Wu|5zGn88xdN&M zM!)rN0*N9Lnm|Xu{DU8|Ur0j+v1slr+!2QVNj>;oL7r2pKC}qWdCYs5(oHSP;7!OP({KFgd6}@btMBUH#cp8%f zzY~*wa->g{L_ao^b+28~MO=cE)Xr6UsQvrtQyVJm6KM`qP=Ui4OCpSaVW)U(*1}W-0!BNM-`&qoK5ZGDZVB){O#_B*?6r|IiKpf>zaPwq2lg>-JOgBvYN7W7; zUyJ#dF)}r!s)f(&e7*p_DLVN;)$$vkN<0F*?iD3Cvz6yGP029-9k6jqns8YkOB;pe zWO)I?KEPui)cpK8J!g;Mza2YUz?>w11k$8t{>_L2SdDDW%-JB!DCh_AaL9lUJNvdj z;*8S>A+i#!KPiIc2kiHIk(uv>jj)MLR)#SY|AI<6iOLX}d3HwNam3HO&=x-)Vqm zX}i6l5s+K%=zsy`5=bYvM8F-xPKER|1KJtAi}bgQ4a%5Pmor#5cfWCH6GZHz?ow@2 z26u>G_0RSMcLZ~_ya7M>xIVRaGDDEMZ{^=f{ytY|eCru7+;?FsR6SW}Z|W5SuyKX| zXC2$GFJ_9K?Mmn<-`uPKU~n>e^Du}L2G&fLa%z<#vbJsm@Z((v*hYB|RJH7bz&bc& zFPwP*xZ*U5g|&C$PhJb!3AVHT&BWkrJTnIto}M#=j`V zrGlXRo^ZdX{{eV+V(a!YLnUslCt&I7p#INc^I$Us(w1Ff0Kdii_O3#x9W+2z>)sWA zk4gO$%6isOW~KWly65UK7i}K5Jvdc}{m1}~ytH(a-M`6qO|T@V#O8lzjkgZx zZ1b9V0PjO><454GS0BSMXz+^)mlIVU&Le%~87tG!=wZFPt z_oD+x>y9W7@NG|BRlW!cFtBXXderf#Far)kj(l304H!|pobcb)wKUmqh`d2}ZdmLd-*vqEod zL{%dQtcM#{KRlZ<S`3MHZ<0jIygmOI%?BAWEM^djokzQJ{{}K(tkl~fM#}F!p zym0!zSk2>5L6$-UR{y zVYWrdcJ>xcRLKwdg3}C9M{+PcgT&sIf!1%)Va@~pB0?-rIEC-uYTe~E7ceLQ((Uw5 zlie|!-1Ji8j{9`;xBpw_zTvTba<=&o7FVZM;&3BHc<>x^VD>x!O&yz+f&9qNN_NZ6 z%7(vL&fvHn%&^0*IJZjEKE5U~Z{>Hh3ZH&t!I-7mwGyZ;kr=b4*P*s8z|dA2X6=Sk zi0s{ILMVwc-Lr2(0w(_NG4%i3*m>l*n$b6c>vkcO`F_kT1h7XQ-L3*rg{ z6cm0g4-s2Vc-1@A9KUVG<>r*P+N*8>S?1D{pI>jFLKjKwuSR$dTxUGWYT$Iz;CTef7C=oM8MA5mOB1zjOz7 z;uQ~{fNZ?`*kM#|mk84j*&)LEv27?xjZ`RiKcX$zt1%#% zzx$Llru089PuUAq{)>yZSikVMU^8HRHUw+?6CCs3jL&|^D_>llxkf%Qm%F~1zDfd0 zryJ@`bV2*7b_l^T#U*leb1ZOD=B?m)GOLDv73U;yK!|sB7ZSaFJ02F~Nk3XkZc zkw!PtsuUflulxfX#50czxEf)`ImAL#gC*}Z4UYB*dznCO`-Nl@4A`Htaw{cJZ4|E! z?7&XDN|)awiR=DoPhYVSHokz#H`$PUCxEGJRC2VK$|KWpC9Rp~D zt9u5R3Np6g04{!h0D}tT5i+}O<_1q4+nbQl`fp5zXY?KBRKa(5a`c#&r0Wy65P>RG zY5G!rgzpH{rO_g&XX5${I*|VTs6-sP%w~0S*fSM#bS^> zw01>TD#8n6C+PyI?kp~EDw{q%8Ut`(&AQX3rcle!5tu-4e3?a_Q2c!G9sz|oi?~NG z=A{XIyVJ!{a)5qm%7oiW(+^+40H$&zn$tCh++Nen&8otb>}FI6EY8 z&qwBZu&7=$sxL4GD-y zbT_eRpp+Mm4_tiK6d9_LKG>G=Ig54iLY@bX5tJY~=< zsMsaRmB@S`DItma8dexXZwB2!=jjJ`|6`u$xMXUu0|fgm^9q_LHMTog=n0ODNL$d! z{KABo=-6~hqz5~o7aC9iO+yE@Y;S*@h@2!#2rw%ud2M_1g_jifl`NK|Du2YYKB`io zwO3)+?Q)gGX-z5fT{{R%u7s{GHhb#xJ9**Tz=)7}dX8#@U|*lLl@bVpwR&E{RN)?# ze1%1*XgJKRzm=>E;g|Z}>7kKZwyc)~4sfLeuS%l<*Y#WD3YZ!r(q=Opfj};z{sdve zlg1@)sEVlzg42Lhv17hu`3s{u%-o|E)%%kdG8pjZ<7EHZ2>Tj88IGkP1xd) zbgxeP_o=e^pf{t``<3&rPpw|`!zSMw&H)*Yyj;=Ye-{&@A_EvO*Y7MQP62@D@9)24 zAVjx=Gw~F0a2FZEt#x{UN?G=woGL$dMmFRwjG2V3HsIU`@2Ig!jNdx2#@MweHDQ`J zO_Ju1aVT*;z?9uBJ;4ui^;oi?5mYpFF&!=a;3ZCK^|2@2Za&|)k9gKqFn?GE`} zaj-RzU%t^rdzT!*s29E;paI>JV2N{casmHeQ595{u*~_j+mKtUh-~1jLymkI9!!Y! z5QrOWEQFoI#Sv}vu{Bb^lMOI6ZcE;rMh@=XMWWn#vDy?l$L z;rZs1{#uxtv=q1VggIxp=CuM(5f}ML1ZkmTN$|teX84ePrZ!q$wiwz#*2h&OqU#v8 zm!ZVLFwFh<7*H(B+IS^Rr)XGw>1s&R zA(pA?*Y=+FF_hfF_P7Z#EzzwX3dp74kp7N+V-fy6FS>_o{dKupWSPaJHR^%t`(NXE z!-sbziYgY6GQ;LQiNhNaR;LC0uIT630&xjki%+- z@DAwq=vyMQTC&0tJL@(wdI)Vd!M0mC4QDY#10Bad(_`8m+hM-KVZ}#jqERS z(IaU~jGJHIy6V(_{|l(_XI33w{8_tU_)rkLg&Vo7%`X4WnLV)_`52rbpeprY-vcv# zQq_9yYoi4%U;75m!}QJPsZx-_+P=GbOdZ01VniJEQrt&HboWC?q?UjZQQ7(*bU;AZ zydP@zYl!WLhla~EU^QO-bgCk+Hwx^*kQf-T~@+9#y$`c<5-6r9i!Z=#KLqTKqerV;9P>{v)@k7?7IwiD8OknH z^QAPSCqll%m$YY}G8S&k^>u*R2JC&eWed9S9Xp8>$7VqGlxE%{5j*7_UF9#^{x|v- z@li#4nx#2Y3<0Je<9{nGHy>R0`noPBG~H+Qe1X~D=$cvIRo3c)->4cc#D4h?W_1xV z+1|fS=DAygQfT8<8I+O4l|?L0>nm}g)xLv65K3rrMH3@9aV99^4vH)Mac)15c}n1* z_0tZ(^pr5r0m<2C_TXdOwt$m!(2^8lwx!{&aKjx@SKtoLL7s6^_K;VF5apLw?8ItF}*VS;125kE2FQ)v{>N?r+Xl zGn5kFROA@5=fQbnI=rN=e8w{#b25)z-_q9x9?i zU$39wPb4poUZ$qT>Z8zJ!ZBylUE-8avdAtPnY}EDJV^t#c+SgHWbIF5s1ObO!B5xz z!pAIpQQs+_m40p*sIC)o$4s$`2&Nzu8H{!tzeM z|2h`fypj-Luv0G%nYus!o|7*1L&sk&2uW08_AaJMt1=u3TcA2cChCDy>%&A>J~9l+$$(? z#c~J6rt1JirH-Vs(LX1#2%gzT0k54 z3088K+!A%Hr$pf?GRnTx{>q~ zQNH;RbK)aA10haT0;NVPVVR1+&Kv#F{}wc9e@YZMJ$pF$!~~!I)X*yaERixd zQY35@)#ziPuu?6hPz5&)J46aeR#g1T`f52M&H%Ejv4U$alImmbIu*qVWab%sm5&`Q zud8vWKBEtW|3eQy23X(#iQRqr5^(>RbV~}gBeRq-ELg21EvrDdI_n1XvgoD|^El!q z=$iAP=X#&R-!mThu*yDSGbC_}Konq}VvA&IHGw9Ei}i`|MsfbjroUa|x_Ue3rRG*7 z5ynFOifP4_uWX+DQ;Fdv@eSymyZ?`-GjWIN|KI=XoY}`V7!qR0P6*jjP9j@bl0sry zL`hV#GY5qhOImmrnWCr^tz>DAN>Pbu(Sm8Q6lFpT#?0@0uHW_j4`$A5doK6$af3S# z_jP{ybU$UY3=XL^NiES=c9^_@M5bF7T|{1j zlzW~hsgGEMz8X&3y49brsnopC{6u}>iI~p6)%iyA&cn)mNlH{5q(r(F`D$heG*sq~ z!)<$Cphk1;J1@_8sPpv?h;x2A6~wAq0CGw`5GX^-BAX7JGeF6%(3q$78YPsDa#v%; z$3cyY4vH*_nH-{ifU4l$r3T;~)boC4_~B2bW=GCF_MS7y-bSlf0*u>0!Y@ZcmH9Bt z%ivg@9uWK1=96c09|c@oa#aw1+kW3gSlz1kUG9E@>|!RY|J@QaNWO8)ED>v5eYbZp z_z0>L+21w?^RsQk`^o!oNl4DXd#P`A8$rq45+iOKQ~^hPqS5(F3Sk4!mNU| zYTgBq1vbzZ8+hPUDPY5YFghrLs_!g?hO{VM&%fj6w?Gdv?gm1yqKP}Qti52@zB+#X z$Dh$RjJix}-n5?m_;oui^*YFsQ5o#ZmvXGaLY^~(9RC>nN7Je&U zJ!CgdI^2?_H{tzlM@RQA#Rd~AwGFDweYejs1=sKG3(T&M*yv&N%hNo`=k7A)s~Nc+ zPV8lh&pYlMP-xxWczBYr*1)iN=8I&uIo_&pJYx5*Re9qxrQ<2XUseXdERrB>Q-7$OGp6+5+{KK3+@4fW*wS4UIe*F;;{qwm{8Rzsd~ zHB+m{8fPY)8aTQ!&J3(bwPB!(;8TMBB&u-ZhRf|+INN-I1@wjf0vw@$I}#`Olw+sa zBW3oo1~OOHgRqP$aVG3?mPaypKEUvyoS^H{et&4@uoKv=hoQURieoZ4RDN}xG)m>S z7$qNqe}JO1uGjt}1bBLunWRpKXGuP^EJ5PS9>%1FnJMv-XT<-0hkI{%yp9S*jHZRk zu>yz&TQmE96+>~+??ydDa&b_K8=ylriYG*gH9%OpMiI9gB<=!TgdDT55TcF?Zc4*e z6x3GWovSVqZkigq#N3?X^8%RCSZ0(vdzoUJ{k7`I6+heZUqG$^2!F)&sv_Oc)E}Gv zj@esdOQr_<=GV*=ZSJbfh#vW6*)(~;jM6Q| zUGc_f?zbo>fB4f;=j^srO%i9?z!o5*-_t`GPWG>+GNDOOJwRB3f?+%xf@*9g2JyB!A#q?7rlnXJRMNBvN?6!*I} zZhw+uMuTIN;2tXyzR|i}z*L{R4T}=vu-9WLrA-76g6(v^%&A^Z0%u%_=iME;X1k^~ z=O_&rk*sjDJRf5XqYrn2nlTUS7)vsNB^anaz1*)-8R#=dVT5MML%CdWEgE&b` z`Go_aZ9N?_C)3#}M~uNWnJQk7HsVdg?(6hcGKyEP+X${Ha)%1R*l$|Rt(9Urw#WG$ zx9Hrti)16T=07NAkDL-Ov^yUOecYBENqV$#8Iphe7g@&>cqp;TU-awDGSf4Oi_kV;olRh3pRb*HOMY833?-d|@v|STi-7c_$Yx3jzIR-GL1a^S zgFkK_(ccQ+ccU??@X5;R(2pzsw?JlKk`(I$A%R@D%S_$|U8*VFvav`BjdKOx(pTQb zI+^GusERTT$H|Ev#3}kjEMi~n+c*DMGgYITj&_rr9gxN1Y8fyCUCn}N+?q^nbR+4l z7U)Y|LxPS(OF3^qrvi_{%thv>6I&MZ{qOe*o^dL-+mz&}V@c75!BDX(XkQbW7WrY@ znW;r|;ymeOLhMdD*XbD81lma+XB9pGUjN5e0eu2+Sqied2zC7}G7t>^623Zm8tEG# zos<$6Wo4DkJU>Lwa;?E^wHvbD=!2?CfU=3;S^mY&PGM_P`za}*Kyca@`|#Fp5F6{IOg)B zlakteiN`3m!W)8n8n0I0T|)#Z&Sdt&^F`w86aySd|MWP3h>WQ$cJsEDU^}d(z#d!P zyPzTkbf9(K$-&tATt)0xUvgfq;h7TaVX*R!5^FL9k5vAwnlf{bOy>4xGeW`GT9Lk> zvWUKcylVyWs3zFcVPgI;d_Dm@l18wyRw1spJ93vhype)Y=%>uyupR-w$lAYWJq{nOn{%+UdvGXydN?VMk; z4!6HU%(%KZLfGF%U3s$gNo%&^JGzgQ9~d2&_W9emgonQ+9Ag85WqJ{8bVmU!N3qaR zg}HyLz@D9~pQE>=@bBfqS>^f8Q6k(7`OBCu_&d}|bjh4Ie|E*_1*{0f>7R*mEoD-h zG#0)Z;NE!eAs@K{Lv*165+d`o&luoizwTaBDI=vv4CM1S5u&fa(YG zHJ7u0_6nT^u5!#iL1tbpbn`5OB&d$x9(q{BU)5(cr6q%fdQ5rzMw0OBG-yoLancJ5 zYhDH=$fEiB3c|m)E(H=7)HRNf9r#<0A>E*7iL-de?TAH_rOOc%^wPGWpY}iaYtpKC z?`XJ*ID>=KWjDbFH*ppU>IShO^Q`ZV2IyNV?qUvoB6ZbxgJ&)%1J3ayQ%6avDhfnY z2Skz1a|Q7;aL`6spVXtZ{g>q)KyvS|)>UmrpQqFLyn!j{xq}zvn(jYWoSj`%D243s zxsDnGeP!Z@s|_;ZdVg?C^`eiUyhcSb()Q-GF|Z+-^AAn+v_+p3yTfTm799AzVjJ2_ zV!U#}j46S>2;AR&h_K<+xhK1g3JJ^HXk4<-ZfRB+cuG)T<-)}3eZW)LcDXYZ`6y$s zUmlA$EYQGFozSL50WylM-;#cvc#7p z4gd^N%n#wI(@tO{2?U$|!p?)h@G_u^PXXKHqzog}@;Z#FbJqt!*P|v(qC9R}4+c(E zmsk9qMel=}Mr!JoZi%bh)E*5?Fn$TB_^oUvIsVty__6wZ60%=B9K4T}fMR7aXg8T# zh^-%M8_@2p{K}EN0<52{J|<~R=%NTXF;M>GI+)wnWJc1r^RSa!WGiaj&;0`1uApnM zn9vbeMa4E<3Fpo}wFm1#YO4Eh%2ft7qo|GI?kypT4hd&;Aw|W<^I zNffC*q4_xf-EK!`f7piLfOhQPE?H~$b6}l2hi^cXN?!#z67Q@u5*`jpucxsRaqLl- zJ!p0d1eJK@_49$H5|M+fIOZ!1LMIOSklHTHf!WcHrP+ zRK{STMx}1ilBt_YD=P6<{QHkfHF^t=8XQj&R7k;}mkLfVUvfjvEiv4n9KHxsM_zvA zEfsXW?#P%flc+vlxkZF0pZ5dMUcG_Vrzw-U~ze zD#m>lYto)eFEF1vTQT4|<5~dPrRbq?H(Sk~#etx4>QGgUu|47j@^QrmCV3gaFLT+s zU-`K>P+oGB$kIbX2;y0`p!~prc4u55yFzkl{hM=sI+ed2v<2CJozmp27lF!W&#H(j zS~%AQ>pJyaKfiG==$FQ7<$L`}Q@Y^$ZuVVu*3rKlW-xYaiWmo7BNpHJaubA|Cw-|8 z=>u^t`fD6T5VP^RE_8Ag>Y2q~n&K#w^;-lJC+%_x0CvW zWols;;WlONeM<+CpUMB2Vsd_K@mrGV4i4Z^1{6_4Hn)a;#IbzIW6aVC{FFv)pz=v&^!_Vw#Ns&!r* zPAnTv92lQ}wgx_FDaBVMee;CVNi1@S?$f(+bFdZT9W7<>2GF~cAtntrdENh5G!m4& zGgq8*;qV)_t9jU(0#Wd02UHjpG^-@|+y+?C^M!|@wZF^1=eRH7plC0c@dX};p@ zK02=S%T3G^Cq_|mCiA*_07w8vUBJR0%a(5n*ePim>&CU3;6V|9&>YOCs&Jb1w z^Y)*;;Gg#H7`aNUNP25UKs$h}<5Izse}URi?QbfonqO%v{HQtps@+owxG;d%g61V^ z%o!K7n#QMsHsEv~^{hBs6_Mw%!Sxgg$D}2du1JzSeww zS}F2lri9GOz#A%_JvuMcwOHss)Ua~!&Mp1xk_LvsyboGT6wSQukjgsC3;fz!cVh!( zenU3=OW%9 zbwpH>rRG|YnG|wP6-9AP7$%D@O*=j^w%PbFh z;9N~AR}vdNL+gWL!_GnL7-%A0rPij(2^c`HmFp>NCZC2|;xfw7qTHI&^F|G2pOLy56C%0wb;hXzGKOTTII>uc1xk2*j#A||M z$QDI!aHj7P5*SDHLwS!)pGurgA3~SeaznO3Q#EaCxS!~1#2uxm=S3=eJU;(w<&F@= zV?@lJ3jPKgbyiI#`{D2$YL}7yNTnW%N@_93JSgq0w;#9;kJ!86HuB2m)pFSG^n)re zK0nJu^@;4|tS?w(?n~>&`%QBv8owMb1ZA^FP{R$pHFxtf2Y$Itz0|_$wUCb=+aD&l@P>x>>^`k|+XB9E9Y1vP{=J%Y48I6c|&r7=Tz6yO%K0?qGqR0mB~ z!0rgwU7tL(HFrtr?yVE~zE2Qu3l>lk?QY=wZoQk?+d7V}17}j#?2si`2rp#vOj|+& zF`ImFy^x^ZYjaH|=O4$VjC5=1^e-!EH(|c?_`$|8#O!-;Z; zHI$W&cRdVIvEpSK;|JLI8n;G5oejCq39*efWCB^JG)}-(P1stH-@5x(7j08*&9=L~ zax-;5smS8V%wG#dxf+cAK0>{keniYn*wW%2wbtsYcOkRTIc z{gFt>S-RMHno1+ib5*d80mc_rE#nNFbb7o>6U%!-zS z$|IRyvNbOnl{kqkNvJBwb5Hwr3?f0VI6c%you^h3Qls%I`;NH{7g7~*8HcI3^;`El zV((0NrWN1JF30Sv6z|TB!0pQ1Zf6=_1=Uj{-s;w2r4QVe#BB&XigMcHMhMqJ1-O7CBZ|3@lc9^0!b znAbr=k9gKyp|s!;AbkW1dXINM5@{SOTs@PPDR6Z61Go#(U=Z$uWgg5J(M9*)mn$e| z041#WJi(sn6DvmuI+*v%i|}mOr#~0e%}8$R0Dk5_!{QwWX08vwWsSEh5F^3&D_=Ep zkucwJf>oe&js-ZMzTrIi)?Gc6E@?i)pYa5mYIHE1WV?t71=UO@a=~#xJ_Ti^Lshhx zEIag1g)B?x!~H>wvF-nh+;&3ur!raDS|ATLqysDLG^WDP#q{CHsH0M!mTyFbM8WXe ztCHW4WN!Mm86kf5gaBM?$$u9bj=C%(?b2gI_CU#+8u2|DSDp@~BEy;Kj!WiJ6|nZ4 zl{ql?x2c=(<9-SqJ$wq76U;q!f+sBKHJb>G)AJ%_`2pvp1+|?XEDoG7uoeU!1?%A- zwQ@j%i;=(;G><4)PuS{`cvV%J@Q^<+G!Sm7g-WlNoTwv--v1H*Yl}6jBJokj#2eAS zPF0Jp1D43F!4FtdiDs*%*q*XgHc_9Hz-|!q)0l%vvz=jlZg@y;$JR_vF-SO-9qGIg zpYqm#N`C$vr;uW_cN5tq5FKud4aIU8=te{t&dbfm2YzK2xduH_KsRB(2A`3?1pRsV zc3`^a9+8_N8DDpm^P8PF>KW6Qw{32w#x&VgxZ?maYpf{nUMdX#G7=w%7S;j}D)Jq( zqHelkhv1wgz}<{{rPCXKloZ0Tvh{kT3-y=Mo?*kq`xLO^u|yzo)rR zUY3PB1}%0ZoYV$AzTEQXkDX9;8GV+Q|G z)?Z&%rlid|M9(I@fGUPN3q%x>) z%2o+e1rl2i;DFrgv2=5+Hvjm*npjc!*?Y2vB$gK=poxYpMf?9KvOu9Ndc+f2O2w8z zc{)_+@jQ4;5Nzijx~w>|hiI>KV##%D8yjy!P#NqI`m{)w0Q)>oIe9)ca^H)}rB`QX zii&2QcsRPGgx}vimWf3avbcNwo^@Q#x{OdG&`L5f>cRFgtFGOoRe!pf z33zg1p20dAnjL&(V((L(J&g|%mzl#R7dwX!lgzEGt)#c)eL_=L;Yyp|?;Fu=|2cys zvWgqH<^&cK$O-5d%4?p&JlM&EPtr;JZSnyotB1plI~U3i%tnU%i@LBCsBsG2(JlLX zqA$$fd=^asurhxOO3S7F4Thkz>w_&RWOs)?hC^pj!?70j_%xecoVY zGINzYf6Vc}w*XrWW}^Pu9YwmfGhZ^X%+^Oq!<1 zRR^t=XBe-2Fk#ibg@KcMH6VdijS;#AlXTTi0sVR0(%}|yy(9StZdgTw95h%IBBG4Y zpcn>LDIcNK>L3WdaUe1Ck9h&;|NTI?UK`qfb%ywGF=ZKD@|nwMQ6yKg(* z8IDsjN{a-VyA}hijEO_0aiJejM%1Ya99GcGLdQd}?qH*K@X#rNHf)?-{ zm+V2u#ml+cP)XyLt2XY?YtnR5N@GgT+o|}mGx_(I<$dW5Ejp!H+WtK5*N1n!=krB9 zR~Mv3o2Kn(#0%(gdzpFzCG4^SjGf2!X7S4=D7-cB2o;qGDr<8)FGU@a?@Fup_@v3v z*t%L4sPO%E0asW$on(*_-1bJ@hvUtud9(=k^gizb+NrF=RK73h3({mg|I69@05Tv6 zlEBLb?FXCnY1kjD#$T~Bk<)uq?umpviTHQl`H4VYm3n>r_vxDpTR-rgmM6RjYX>Pc zA^%w3@w1Y>!$X>{jCml<-F%Jv4^&-T=dqX3Hp&ZUP;ohw5ziOS*$SPgtX-#BLfbnn zyN9CyZUskfJAeOD#7GG-op% zn;`QzYwPuDzgk@%&P*LU3C3V_B^qmgw*bPka=^rxzF2d=mr!ecQ72b(>1?jYDw#VB-mPlm$*&z_@|;=W5(`CM)qU!58#4Go4Uh_1yX3?N$|UL z8Ba+-kz#2h$`C_%!kYWkRt`8RL75tqAusU832@{Jn9KPtj{%QJu6NmhX&;89ZfPJM z=)3+;SGAj&5Lu$}>?|bxJaBb2#QhXuy9C?##0bI{C@wWKIsM!>z}p$HC3T%#=S>~V zRpx7GxwOyf%(IR=q>#$l>tnIFnesXaTY_z!-y|+R=YkgnPC@2`;`3MNtg{{;ZQvY4oW87obOOB0=WUpE1(<5At?94s5;#Zb$gR)&R_d|JGW8U3IX1p1cBr%pcm?c4 zcb&odPA{)+Ywi(TKXzn*%CLe3OVLhm4cOo)sbSp9v_hut|qcO&;yyF=N6t8*3h6>1+dQ8OTb&%pq6yx zCQu20t?YhmhgT~*u%|cgAy#>UGpYz4%GxPIwRkB4s@Pa77ZH30eE$vG4m*Ki(G0;w z7GlZ?NGBjIQc2ihbSnd9kZKWKq;DxHz=hV889HEcLM5wD$}&c_USfz!0x$VQFm=RI zGDQs34!i(F5hrik^yibj4N!d?TzFRU5i04wU)O5x;2VgM5j+se=! z@(!018z_PDzLcdyTOayw-mE=wbKdzFSORmG5)u1M9Dk2&_63RoL^r*{ZYa1O7ku>| zT8f?nDc%+l3DIGqX_K&_nZB_;563F=BjiGsi!OXzYk`sS~1E!er!uwS;?0|e13G+4jC zLBA|W3AZ;}z1+X0K;6d|G`$R0p<=yu4!4-}mqwzaM8QU|K;}Io{s=Q>iT^s75iyYN zcpSpEp;tF8{MfiVbmrJMpboRL1>rC&(*;eu3if+>Pnx2ex=TTc3&tYyzkzCEjOj=_ z_tYq;R*T7a4+YSxjDQ|q{*Xt!PJs)N+tmJSo!C7_ehtV&CS^#g_{sOPVHv3@pPA?u zYFE+5GFK9-W>#*aNKRRJSSV>8IAR+Sw_)X2tLSsTzckYB`{oaBP?AYzl^dak84_b{ z7*$0cKY>c}^$ZRNw5`Y=|J{Ejqr|HrFv=H)xgs*avG>S%l}!#!pG2e zSy&L8JeHy#-8G<-`mO2wMVIY?Bp0L+8T{Z{wiYM?-(vY^6ks<}ZJQDjG$6ZAC(`YkL{`O&S8H2MBU(^(Zh2$=;ER%I<}c|a@%xGvgiurU@bmgLllL|83^w~ zXjwva{Dpx-Pbakux$~P$B@c$vPgt{K0Fb;Y(FV7|wBRNSyGsVBIgHq%G4g0GfV03G z2LxWsa<>9e6lFTgQx}wjE?mBR?uSL@#$7VTPFg*KUc36L|_)|5Y@!P5PO-ldB!$-3#Fjg&S+nHIC-t*d$t ztIIB*8!i1F?w>9;I?8(ions&>=-r|PBZ~BDe8n9nI(AxHf6*`fn5r%C&N?LC|K=Y@ zZz&V4g4k-Hj|#Q{Uv=aN6_rQcGFTN9l?`nnT$(bW+X8ui9Ut)il|DEZLAeInzgTrbDC)&>%p?4mNNWRZN5`wnS00bbX9%z-5*0;s`)bKoQ~sP)^- zADf-=1}&1N9R%KG=!?_;K4;%8Se!yt5TtV=-LM)NkaD3V{|Xe7*Bs*R3-CFS5tP7M zYPr)F1SY9aA3zx}HymQTay|sMYlD@L5MRty`1b6QI+MX*Z$1OrlDaCdWWd6zd?;|O zd0N~hrtnxXASm#zrwzblT8tw5=1iyK9lBP>uQ*7BSQkFVCW1L zsAE?7z%(6De}EJ!|Ih#x@djyIGoF;tFT0_q4yY0)R<&3VQ5E(@V{j3UmHx1i^*OS+ z^&oc12EuoXk$=+1ujA_>+<1sD`F?DRxGBuCX1sg+Dj{esNgP3DeTdrM*jX3XSYZHi zLGK!&6omcd_{T!ER5SqO0N43YB9quv=EQwM=)#Au{;SZ!d)7V5|Npd%LvJ^-uWd2u zH@DN>s?ehPFzAW=Onlx4W2IH1zKz)9gVKDttMiU6XUk5Om&b=5edc%rycLAKENd^j z|HmjwQZCQ-S_El}drII<5b#Idli2xSE`S)TfH343HyIIji>VDzm@)oF(IGay_*Ns{ zr23@V0Y#pVbX$)MgO*TV74A1LXrD7lQkTCgGH+5nziOv&vv}orh+|5^=XA+<%4b)( zYY;p7^N}0PUui7ajvpQPIF`UwR8A=7y=I~#Fau`E-z8Ajf;D-hgHn*gD#Yn0xrC*` zF7!mJ%^?#K>&XVV>8$s*eM%=j#~4qm4E?l8_`tt!$v-yWapf|fx5FRoWf%|<-OIz_ zd|q_wj_lt*_DiH=nmNXjQNN&Bny_6s*C@c<(u~0h*@WfqVjBMYeGzs1uXx5vIL@Dw zdnq}2UoK}I*kTT&gce`umgHAwg;|kp z(7@KjfQ7veer3|rtQO*g$wz>{ta!wjchEMw`)*nOqY26InqWRJHObgf%}X>s-+#|~ zIatxr?d{m$?P&qOrGf9?*U#yOf#aOV;DfyQ!x>XtsMJMbTqQHUK)p-x0oAmln96ym zUGm1D2!E=3rL5NM#I9X1BGqmN7Qs6(^-ieWtodL64tUIq&t`0peEn;QNIYDgKy9!W zCFb(k(A8zGsk^uI&l`AmZhuAw8l~D|S zFS%6nntiDZ<)ns%UAglOb>#E*w(z)U8^7}Tc)-k{?dx&rGvw+Sim0v=7VSz=x&&$` z_m*$F@h(`v`!Lx1oOAfo;S+XvvXi5VIsEUXx=ckY(+`jYlaK6y4UCKV3Su(L$40WA#2g8ystrYlfP-)j&HkfKG<@oX{G>rHOJCT>)f)LA$EgQX+DQ5z?&^; zS!u(!QU=eg?tnrn_2(tY*8C6O1{{G^Qilz5A@<<0YwK`#JKeCRcO4(C{6IJ1)izfW zkY#(=Nk_a=fba_@EegdyhK!tf#{ZtIDIe7R7gWnI&_z^fe4G8tSwGNV!kF@jzViWp zAIhMM@-o-Bi}E&6<*U(YnP zW5H0~g>BZ?V*R2TE5AuD?th{2I>!_XHJ0bsP{lPHd@1JbUIfoPqitZh9atoar1F2L zWxU4soCyhI3M6COHn;@`s>(bue_~?Mmlw3=&Z}Do40Dbj_my9k>GC$@m7e)OR^IQ& z`;#oCNhGIb^y9BBXzjbXkFjzt^?uobQ-%D?xxPVC%%HV<7bj)8pZjkW8Q+f~CWhf@Zc<#fOja!Kep9qii&4bqWVcrx+8?>T}B8HpO2sGCu2 zd$?eyllD&o#&NQ}pIxE$7H_luGtq;7dpRwf#PF|1o6;IWZpLbW$5hrIgF9D(58qgI zD011B6<|%{g9F7%CoC|cBKV#GDyS?)Q_yWoeQO9+3_=x=kO84fCFaK;>Ji|HfIBdC zoTxx!nTAyAA(z9%cl|NmcxsYBQ8?o!!rW+ zoBcT<2Lgy7_w=Hb!M|V9xh`RPmca3d8W=8CLywjyKzU~EtI*}4b8Uw=^sB(LA#JEzy@h19vd09M3AtMh>& z1qA>u4h%Y9a%2rVO(ABz0@xP*o?xzkG-&V}E`x@r4lx|ZB(&v%z~QO<8%mchc0dKP z(n#Ed~F9LOZlg z16>cUC7@N7E!(Q)b7tUNnw+^KT^e&jT@v=*DFAuPDHYO!GsR%bj_}fo&t=HWhtVfL z{A%7`162y7uE{P?lti)*(0sGH0O)9(NZYr4H3Gtl?NY zW6lWRy5NKQC>qroqD;mf)43vvuqXbiNA zn%<9&Do_}76=g#%9+Q-<9h6wF7A~HfH8v;u2x?-XTLFV|D*-#5Lj`&4@cY5h$7F^Y zYWxX|l+$6KBf+jj*gSU?7y?;MnaFlx!DWJ*7jSm~!O}zu0am?f$tAp zNHv6xYyjDf{D|p+ABR%nu9m(2LYUj8v~Zj#=n9Q$#x;-m&=HAQRyGKlui<*Z!HWgY zZ%J}b56KQVulNcP<<;?jp9w{%XTZf*V3dJ9cE;0g&+jSph>!!|KiO=sV<2^PDy62v@9F#JEQbuY{Fj(?J&(lH+wMXB;*=P6RZeF1x(g};1(OD934CVsJdrUOQ>ODKt;}XN;ts@y|}me-_U#j z-wYCZx@=oGK{a=vylp^_W~d9VQ1Qur(ZfthqlEZWqAZLr=^5JGS9>LHpERJN=W@Vp ziuoE9_TB?owy68S8g3p5nX*F%vZrx7&4%ApR_QQ-zkE2PUc;MY4ou`PspZL-f=JgKZ(-jdJDY9=TlYcz-kSyRvt zYvh3BMLOs3;lsKi3y%A(H>~_s^>9fPKmZqaA;SCcL{QdQSdn-9&(}M zasQ44RJMj(kqXWL8)ton2_9k~&T7hMD7}k(On+FUKSuH)vtSXlYhM=VJ^+-7Y#au73M8xu2)&wbfQr+o(0Fp4s9d*hoTr+rf`@*mmiUoh_kLAq^D4 zZt&O;xd-eB!GVHoymhQ5M+39n_9(@x3Lg_Of6l~+biDO@8TbsLJ&Oqs_Q31pvGKwL zRPZ?+A9OgK@fE>(Z*CD)b`hfAUC zxNqSpKWy%7wP@f5KR;h0njhNyuD0lUN2(M7z-;TSFJHgzI5pwjh$`AKfoCv42`cvv zfFWw_q#<|zI`CDRF$5{(5>D*F)Vkv+1sLghj zn20G;2dzDO{tE<`z-1kD=>>FnaDw*zcf<50&V!Xk3p-x!OTa;gM076HvwXssj!Fs( z6=1Rzv_NNV5Y#LjlsD$wnd5xf_h{YJ{IqyA)R^{RZ4>@u)~%wUuJ9ozcvr3h7I-!o`+P8QON!@lQ8=e} zIeLg2Rm$;{<`;pZ;hN*RRid+nt|r^{SZ+|l6XeAy{w8AyiOk8E=+;qnk+5b@`{i)RN?7D1jULqDe|S;mwGiDD z_2<0xsW5;j;!a7gmibU(lHpEhmWZK4;MVgK;CD3P{!(Bp$IpkB`hi~%o2B9k;baq6 z$OmR?qA{pD{3DE<&=i=qUGyjT#%``+8!!r{85jz8Zc5AFFya_MV*(4(nYUD*C<04Z zwh%Pi)54p4Fk5~=E^1XB4WD#)n?Q6+26ns9YuCD#V-XaFEm7c7qJiCyjMHG<^aTqv z=oKNyvSkE%CNEJTyr5$S(3uea_Qe^{jf<1Z{Nbv4K|IjMH1eavFqbb_R8ZgX<;6Id3 zAGt1$)Nnc_J8PY*D^7;;L-$R;esBy-+<0My4ITSCe>akBFH;I#Oe#oqHkJgb3wGLE z(q4|VzJy}wjCVYyt<3sB{;zb&u>T3A50WQ%t}RZy=>_okULCH!Egh5h5073dGqb>3 zDz+LvnXFc&3&mQ~yWUu4_`th-;FJ~Ev-DL`wNLRp6b*dR49?TgwNOnD8*O?G99jD{ z@lDk4kS;AjnOGIhm&AZLx|<+b=+qEtR%4nQB{v_<#2t!!t^ywQK6vaX5AJ?cyqNs_K(4$z^vr(0erQR((^<2@BdSQ zfkaKh7s==6#Wx=vf8cRm%D^QI=t>TtXIeOcMjQk2mYX8U@+AO2W7(S_^t5BCuAru) zgHFLeTru?8Z6~%ymOuA#B`|~CenGG+RIP)UB>^L7Xbr(!%KSBzRTr@g+C=Wnr^7cX z8L{MmL?uO$noxFEdaX3F9i(2762}+OMcajS7v}mcY`~4JF+{emJ&2j+LOvICiEMqm zs)yt44k3TlkxC6#sS1M$Jb)PyD4I7od>beH5ZIrk(zs7Sr;Sa=8t!kTz>F3=8(GO8 zz=~ug8+~M}jslm3^^C3gw^W6Wu|%=XoZ~@&;8%KHsKjjNyMd-7~7PS z^465#a6vg!^LD2pR=SXEj9&I$f_`b0w9>_g#;2nnP{a|!Ek}j%DyW!@##Q(bx_sf{ z+5q(2B>46Mh!p zk=@IR!KTrdb^#&EXW(-c`ovOtE|3yJ^HYIS@z9r2-hn%k< zD+x{V=U{Df^c}S}V_+>*JK{J#4U~hAp}F*LL`h~mZ?bJ%j`MIjv=}hI0HKiNNO91& z0Qc6(T;^q{vx9)&Tlk(m`0ykU)e~=!65NT=EMP*yF$y(`ko8&7tuoVTG3cbKs;gHh zfM8VqJiPwnK5VyMJu{*X((!;AtO!jn64jvSK7P0YmPyE9qH9Sh`}hMVrNue_TfYGd z4fhU!N+qF8-J2H#l`zsp>g~4%fl_!EP>e&Db!h+>6%s6Hs5Q$2mkmBkR47=H3|Y8p zsUVk`te4>Y&<^zD{DlyH9hY|&7KE_UWdfS2_+@s<0@r(z6>rGg=bw!m?~u6<$`Qc_ zC1puZc`)ep0C$Lj#OIbM)eZ}rzsj#&zyc^^*-ostP?bzVc#a3U^;`(o#1cB6v60(H z4Q3X>6)Vu`yQ4AxkEb(#gzEj@|9#G^7>s?*Om-ziSz63diBf10rOXtCQnZo899vY1 zBD66|C`23WbEHKt6-7}hgNhQyKA4&FIqx67-@o9@ea|`f@_b&`X;Bgwmmry2lOWQC!p0r;A@Y%%CBk^5YPc^`>uZ=3;Aq0I{feQcn z@2XO}D~ayU12SsJ-ZKtSt|QJ4?TbvZ?0^bsOxECl)bAwc6wYif)JvS$1X$X3AGm$) z@(060ma&|6EBp1Uq6P9#CX{)|K9Et8?X1M0V9wo~R}6tNBtD^pUwH$W&Pj~7gl-KR z5;Lc=c^h~O*%S5N^YC67@~cyz`=GdV?E2C+4SsHJ(0_&6BG>JIKO6%TXV(HFlp^J9 zt!}{gjXps+#`18vFh?=KBJMpbO32dMwj2_a-k_lifiWm~b$Kn)mK>U>Kt?*WMLOIg z>ImjiH~a-1e1q^v*m2_Sll(-Bz29Pgocdp&p>2SCBx*HTcLx;(cfVEoOz(P=yyRDt z(uZ9K>%x&ATWE(4!|!Gupi@SY#V~*KjH(njdt`sPKgBl3Z^jC(la!hvR}6ru+~NX> zeG7_y1Lv5dHKs`V8^W`}5c{Yq=u308VD*@lbP`I|?Gg8?v+rI@wW0u0UtQi4YSfVO zf4>T541k+Hx|fzelqd|A`2Xs`*NYWhUctOo|AAKhc2H4zUef`z>>_B0AnQKhXsv-y z@SsQAP-brmv5;I*M~&2)o9O?WR%NrWxmd_tg@%g^_86o``nhEQ6wscuy-WVFXJj*C zVsVA~wxb`lQLe6@xt+)_A&D|~<8FB8oIu*YFI$WwitZ_*|1AGvS`WP9&~#YDz{Iw} zP{QZ)HV6x*I~Xn~*~KOxH`Ch}+2sMJ2g=T|GoPJT!6mPJ;rvu^TQ#ghf2(Uykn1g4 zP?4_S-r}NrxF?I}+n8Z7jk69v@z&8f4V*O8?U0CNV2)`7AYQ@O0r-D$lWh zHo*pK1Dw;hseSI??3_gZYo2JrBEzgm#Trj^)g{&TkKnW~!<_d1}r-P(9&glcLji z9*h~|4_D#)K{ao01doKAmsa- z5wi_c7UiI;*{dM)eW#1a0)6&reWZ`{)eCA?srD~an< zrO(8L*)RDdF;_{7kibg*9I!f(%ANp8aI+O9W0|V&$3#9JkNE#2W1qa~Ko8F9(9qok zZ|!&TNd6E6YG(67LPl!*;D89!WH#wuH05Xe57EU*cM*3+=a@nrFbYQoU-KX^IrzO+ zM@d@x0mGw9u8CAJ;h{aRYi_3!n#4t0V>kcqX7MRKd|Dc~ZYd2^A$Brde~o5tl6m4* z#X*Q}4u;ic@BIlJ6%dc)I+vmnvaYc@(aXy2Sp$d6rvZup;qXi@Qv;Ic7cH#Q>TXBI z^LT2$3u@sg3D=o0$X(`sVP#AGW6qqOS9i7DLFJv4#W`5FzZWQ>i@ZQJrR*0FSylsx zd0#2u_Yy)wo&Jq+XrVyea*1;iJ+^2>3Lwz*e z{nxKPX@_oxJ&)3XkCNR<67x1ycfTX=kb)gz2Fl4nLbrJ2s}iQi-ZPD!T;YJq9C$~LsnQysZ1!)7Z%QbBX1wFOx~7BC3a&U#>Y z_onAUQkf&QVHT0cF?n+6a(o^4JmrzfPd!wM6MyTM-{hSC=J=mla}1Tw%y3*zU5$kOX&3==o9)ab2tfKZad=D!wr(}5$c!6 zY4u-<;lA_3_+AiwxU;N>!Az2GET{I_K#aqIVCy+8pmqQ)Qx|y!(L>_k3uJT`dlMbC z2I9i^Y*MS)IS}f_-i~TrU(YVINL^hw@k`UJR)NvqN*_K4z#p}{LmC(4N-r-L+6ur>Q2AiWG(F?yDrkJB?mOTR9S`7t7rzX6#9XtN|tp0rEhVT9EatBO!OcEKP zTmCNTu!6Yo$@^|K|YG_%hgK^%VJ9)&e}<1eoR-PlA1iGCPPPSraY#SsCqS; zL+Ra|Rlrl~K?73Z_6RuaJV;f+LeDJ)S8hVlL2&vpW^6&t(RD*cw-FXY{xQ&IaS3On(HI(#An(_gp50s}tR!CY$Jmdh{4Oc$ z*z72&&>}`Ja<<&38DL5gSl7A=NTLABDW1cf^C9MQc*!0G@EtV@Oe!PivkB@l%D7z1 z{-W}8$-;Z&`@oF*viAKC=WXz3AJ>iZu|1n~A9|NI1-|F&t4T|%-Yd?`+*M`$?vBFm zag~E@xs_WwYx-riYPiASSzA!ezd%6B_cpgB$c#71(S)pe>5DH)S*R&C2#XED`=C58 zc*!wBg!d`wH7#%XGxeLgg4k2&g+D<*GfhS~rfqk(j&-x{;-iE!;>yr>>E~hzzjOLR z3t4IHQuZ`1v?>9|Z;*KVK|C2lYAHwunR#2?a{n6QDsgKWlJt-`CvVbW&m#tN2=;1M z6>J<%&8BVb9}B%AJ1aFNtKHNUnZO4opXnswpQbXU1n%}rnrdT^t_HF zu4DPr|PB4y32`*;1DPxr85wF)btW%P^jAOK+4hpc``B*r6b4((k&2~ z^t{KC>`M{lf?7RN#_4rZ4Jun~u^HH*8`XK8+K4W1XKdB($BKTv=T&!7rWZj?gN6Hl zX42&`rJ;sX#80r1UXz6;{pkJfq=uhj$i&u4hktv|Qbx4#4fXfP`8jGp*6RgQvcd2p zVwC(CZCL&DGE6s;My8g9YMvu62+2Sk&9c@Wy1F#+{b62^v}x-5N`^6SqV%1T!BmrD zuT#}l>n`r6pep?iqoFSoL&r3x`LQ(Y>PBf>>&095x;clS%rj@qmqtTd0VHxVY|R5- z^MHTS0k8r7ww)2R2R6ml7LbbdC?P?6&M`2HM-b(SslMp+ulrU0OaF-2IclQAhv-W^ z4?WVEsz-66dF-{vV6%3~>@}0ci@(1yPLiu`UU1dlXy?ia)79sXb|p@c{@j9}C+RBJS1 zV)f-r_K(A#2%gr=`-i1-nhJ_Uw*K<_+e=#}cgFQz@z;}Ev<&r4S7k!@;%Fs-ug%-H z;QCy!tQ1}Dn0PGdx$v7;(#IU&G6#AV{g0;If-p%VhEcJriU|ywDZ1$62Ytbujo7-G zyovCDDct`%IDEvs6dJ32w)^le4>u?>A>ZQ{pTBpm&_G)Bm+yCV(&HZF&jKl6Rm|xTnM}CSs1ac9^TgT zxUwXJTFt?O8N8HZmXxiY_6VZ;U}P+E^rmzu((_kllJGLeU|{`BWJ>37!pxy9_#amK zbmzAnQ-w@|2jH+A<&P|;h%CT9vo&Bn6YTHQ&}Boe1n|zN-3rYDKI!BmD@?(5;07Hs zMEpK2pQl1Hd?vph-|Yve_^l2Gaa~Dzjy#80b+#0_uf&V6<;s1AZcW~5gYcy-72X`b zgOuTjSzA7z#?^IY0IyJ=-*f6ZF_m3 zJ}N%8|9Nh)a`U}fH=h9y@wGaZ3jfVW=Mw6S?O@sDDJi>iW+3?PUpiQmfwwhTyZ7;v zaC;x{jmoM2k|=P zYo(3919BnwIri34B$e^^X-n$6xM!1vu3t$roVd{MlUKIR49rFoZbHl{&Hzx9anZ<| zK$;27D4>u3D1jTqEp{7o@H{xNC6Na`o7hMD*@I>p*(Q+#u8O(4)PW^1ATg=!`TLHv zmtA2}!}P|>eB#?%*N;GdHXu91RY5LdB3gwVYnv|Z42;HjjZjJ2u}qf z2gHm6iD@(2KqplXWe^-*DC9iMHbpy~1?5_j08f;l2A~RIykKw>y4jNaqm-D*w-4Ne zwy_3+@bU}mvBOlnkvWz&F1z04lZHQIr_W|H+Jcq=4z%Pz2J&%hqRsm?cE9R|or~NLBzYqN~Y}E=5oZ9jg2jGzu8ar(STZ zX6bx%ME2}*hn%#f>ppQe$@+EnO|ekEE*`({Q5Ez%d{Nk%#b~T2P>LX9G1R`Io}I`? zSk(Ap9!e(L#VBNro(B=ysFBj~z5O*blqo2>8VsHTgZJ(}wsg|vlWE4{&GR$n5d&K2 zzrGX=5#g`>sQeYHYxXi(Qcb0Ec?<{;P30dcvZ zCx}%PwTr)iQuym}v&9!p=KcAcS<6Hx!nJQ~U3GEhcy%?+G+pR6= zG)MA)#CRu^6#$!tlfMaV*MrEp%#uU|woW9AB#|i1(y4X6BenUG!L6ODcxtq$WIM;C zpfO=<#N()20{bC3JsdvJy{4H|XvkcHnoyXAJU0fa0>8PxXq&{o>mAxc7vJUnvPcehT9g6L zajG7vGOjCBkdA*yDD6;sc&dA7cj?UT8o5c~)#YAlRP+jDS-lWkzn})VP}I;#C3J;4 zGU2|xs2(I#BD>*qvZZ$y;W}KZN*+?FseN|msjBS}P2H0TNB6!`hdydXou11vlAi6$ z^ph1yhiY~`eKwPO$G5c2W$Hvn!ii5<5jz{PuTf+q~Z8J@H2#DC@pbgNlM72g58) zX(YkPbVw2=RCf*xw#jB)?$7-^xu|qu=!8c?glKoVqq;Im*#$JW;HL<)kUJy}cZAt{ z6`18(&NDSUb4Wg&?nMaq{&SSi8woEveb$hae{5iG6gJuxVbh+T3tg(;wM$c6gmUy{ zBL_LrQ@!tmVKbbNG7D==m?ddbwVs35y2-O`AIsxDqYpHYHfD7;kc#nGbtuafT2Tzn zd_wk@Bo0c7feTB4y$Cw>jcje3cdFYLHPa%0Ooq(Ifw(2ag5>LluH&4sfEUcW48Jc- z&=qC=of`dH^*RcVJ2S#njoLqEJi(uZ_y`}$Sv-R`fxdVx|Mxgc9l0K0Nu38PAJuFB zegqadSV7E7z&;z$HL^y3qGtwp6&i?ta!&54#x)R1Woxpv$bZ+yjcX{uZ-YIUzLU`z zSMYC(TpLp0Ihmul=lP!sYx@T~z6Q4lv-M}(Ts0QR@0;AhYh6f8$k$p@ z0Lu?H(-DTWCle}}4Q$+C@&yz%jM^W2W5otEDH+!HxJV|28FZ5^rTK=3+gBiAe|RR) zxZK&ClyZhzXFgoo*~t?JH?uV4(-*i?lV+q{cvpS1tsj1Wew|$92CnoTV0UTa6Bf`5 z!f!{(zYj1k@FiRjqK+9+i4EfbX3V0&RNr}CdovE}Q3LLqp{rsEqcO#Y{NQEXt_U@O z@}~9O=pJf<=Jed;e1*p54RYDyhplqf2JyjvZ3ta(zeleT4~GH6x8d$`;4L-T1*#qa zdM-#*N~$kS>}Dy5^{6}sFc*-kwr{W`S>7`_V!j-F#U7I1hd8*E9$u3k(mJEFUynUU zm8^2)U_F+(oWGMf#4mIGF|L5isuD`$UhLqM$htzc$_8ign((qKpEl2?#5_p0)4dAGnyb~6&2&Rj*m^B|3BZ_#@ zJS>(DE|Vn-3>2a|IkZEUfFpsnN8HlEn@u2s&KvJ^RhrIDQMUU$*)xMye@{FU!&nR_ zIuU#0{KxXKt6iOs5_9k?MU}D((^XwPoRMnjh9a4;u+~aSFu$A`XpFM2#0m>cC)*vP zS}umrJ!D>B;+?c4_#iU5LVQk_>Kn!Yw&5?H&G+4Eu!DKoCiUir1onGsg>r<$+8!bcTUN(2!|OJA3HG zUU&@=kxQvdP-hFY`rbglbVWsc3&BR~MFeL8O1o%CVzj%`=WBk24baw{l}|gpM5G0> z0T&gM^NoP-5GVHE)V+sa8S5tMT>jtvO|=KNl&@+#8paJHfc?f2SGL@w!i4B) zCK}M>rY{Ra@wjf}^1}$4BRLJ+GeC$1xMPOW6+~C4 z=Nh4*BL)y%15_eLBwvbvDV=-TVHGuO)7rL&cDqTB+CBDE-sXHhR|aIY2N&@6F^M!` zet|4`n!Eh=f;_xtwA32;#|Z)%`FLkgfJhmYjxl`iK}XmNNrd#8YD8MI`hWBG#S?I; zVg~}pw^|dYoa7%_`}k(r9fsiP!2%@{omoo%^xDfi4Z`2t8Xf)bJUcNqk+bTbPo8x{ z8Gm`|7>n0*OW_*0P#qTXVXSa1IJP_#>?iDMbW4kHaL$P2oq<~_u-g>Z*fGR=(-j0Q z!2PIYLkaM@U;GMcKM6|7`Qm-RUa8oXy-pLm2;1(w*?XVLH~!<(DY<*9NnSM7xpW8C z`~_ZoJwaG+P>}PEF?uOeh7J{0kJxP%6Z{{sD#7mRLk?Zv+34iL#DH! z!t*p~O)}^`Y@JQHRk?RiRl~K_eiQq!cDkxOPb+*(Rb8H$^tsdKbCKJ0?d0X6Law+G z`On0WEf2K-VIV8!2|dK>0GU1)>`SW!oY4+a&c`{T3RAEX?XWUbRCI*akbHG8edy%X zExAib3(5z_?S7YMN0%PFb7lfM{q$6dCRoJt)A!D!%JW9rX=ATl{`Md3YI0qN$B*s( zbNJEg4FgRuaG6piAxykwm}LZhD*XVwXfpjaahb1x^2>pz9rIrv>D6EI68c}Vvo88A z<*F`iN8awe3&))u(JN{fs*`u%p+md3e0y-GEPJ~>=>pThT&}g5zW2D=#i9!r#&Yhw z$t+7sp;o5qkRT-!=5{+cS@Y-z_MJbvD=MDG&EC?e?zh*}>7KSpdf(GO18*NsKJ_18 zJzg~FxARLu+u9h~4vX)V2L~Va7R>D4zsgeZ>1-{Wbb3DN!H>TyX#VGuxJuK8vX$b= z&r0Wi3zv@C${aS0n^5egYrmfAQK-N(X8p$9N$9S#knclMm;)(fp>nZ4^?~gyntDwV zZ+7+t5Jox|vy_S{-ws&#MTZQKW3)U#`mMAbo6JK`{ zIlHB3?m_wLE6|fqR)N}GO2K!heqBFF?|Ta?X(Arm%Cm;PP_^fFz#9tU8OX{SJL)LPS0YT^E zN9N+mz>ngz5uY^vo4f1d)?&&?Z~fTiG2wX2*PQ>&DB5mrTx5 zhc%@ISB4BLIo!JR)IQdbJbR>bgDE=x;=%%^T)A{*BPWf23-WzQ7A&G+$H~47lBFYL zOcGgFPu*`OJKNSQ(#Mvn*__74=vvUn(a?Y71O?gS8}5okkNnMID52@A)6-4IIi|QQ zW2DL&--dC<@vx~Xmths(IS^YP@V~O4oHKv`p0PH+5*CxocBnI*ApV;$SWb;)J2HM6 zgMzN&YfzNF*gQ6dpsD}Sa`U9sF3z<=mFcRR6z%Gp?oP_~&7ROGOXmkjF9lDrOgnvY zW_67hyS3u^b)8oBz}_6)vKSmy;oX1*C@3C5cdf;APM8ZYD$&oGOaeI9V=Z>-;8JQ- zWKUr5mk#B0kgBN|g;|e9-r73T+gyyDc$&)%kf>_W_;xcU{BE^W;kv1`X--|EFl@Y( zip;Wstj>VEFF;x}O+_jdB&YXj@Lp4SHrN?*^G;ODC$nH=?YUE61-3px9~jyN9*X=t zp@*(c%%0So36subO`}~WhYvZEa`rK3ElSh1pIbDMsb;KkHTBC=iQZbU2l$B4)Uj zu+kCPZ3X<3FCEfsUhoNQBv&t0UrB1xz8Mdqu+5FCgR7Gi=jCagO4Lw^Z<^cFQTLhu z%r}XXA@AkMho>Xg%ON~?6RtPi|1bs+F<^Hzum@0eH^_AZ2Xsh3UnNGHp<6Vna> z+ki91lN)4hR=6Ghx*s2IJ@8kU0f~_hz%B+f&IYOFZi*Mt7pBNA?IZjcTf5g(Pu;}V z9v*AIDY6;9z3K>;fUrX0T9z-h;BP08ce^9Rb#mw4DY2#Uy)r!zcFw zF?6Oq$oKn+riXVw>;y6AzgilMM;S3EzOJSjS!f^9F{u4m{}6mZ)t@W4$GrN?mid>o z%sCnV_h#Xq!rgeZ$H41Hm#3eee?2P zu-`&gq2$^yZ3EwZQz}7aI%_!7b?MBMswxot;ifh6SAx_$beBHZWC6~pAe+h6{%}|Y z6}68lRz3t7$Eg) z%2#HJh6&RL_g?@j;z6Hu8(^P(Fdv*3Tx(EQYo4$eEF3l zk6Jvw<}V+XkGBF*`5KblSLm2-Cu|xFJQslTToUh~Rl{lxvHPo*=j&JU4!IBtI#&jH zsZ(OwBjJOVP!M@h8@d*REC6RK|2wR-Y|r+|z4u|-;Tm7*%-<^<(+bQ!;-x!hdbiz{ zaCxqi@;94K4Uf>sYNfFC2JDYO<2l$6 zPZe(pr|LePNQ9pkEu!$2RU2$vpyeDeHd8z?b835_o?KUY!M0GDswN-C{Ez-F?0s}; zRgO$UCOp~&>=r|8HK^u0`1Ux)Q(Yi6XY)0YOGLSncmj}iNkg84mJk-=;T^VZt}5A* zaxZjytwBPWA-Kio-!hYR>`mbxsZ*3AIiU2BFnCfW8h^ETy7LVkc1*}?N2M^H z4sd#MrD|zYg{m&Sf7jS381Bre6GoG&RdJd-_-`#Xbo~<8OtrLwv+J>ICX19;un3F6N6HhsU1)=% z$l6XyI%7ya}5)ll*Pt#7EWuCSSwid$Mio}*RA{uD~2oyWuQwi8kx zTcD{7Mu$LFA&^g04_B=LQ`umxfvC&X#5V*mDB0$~X8(RWbK^?gd)9Kvh{8 z+&G1GQlW1hI`o_AoI1bHV!zB|tnKgpZ*ndbSDnxWg>$-v0bt`w5I02I z_+6oTH~DuMaUWf~h>lIp{{XW6OBmn-=5|1q@kG~+A_^(5Y=T0YmKSDW?`D?6{fiNs zw=<#BeUX#ezOpKZa^b2`)l58m0eL3#9)1<77RpPNkyX;b-QZ@u@GQAHoD^aTA52SJ zpw=*gC|{%>F!w1rPG&w#e6Ft_YoQ6wd&l1}tAm%@ksoz^kDfL{S&!uj)rQeur#U&> z^KZ(BT^3I7d=#^6CQ}?~I*lg*4q&~BVwd{{m5?@JPQtebdC@YfTPXq8hNz$s&HvB0Gdn)nK z10`zMUom%qf+RqwV7G>Bo`IQ?nxCTPEkH#J@(#|>tR8>cv^szwfHf7kDt9gH80zaTV$ccn* zTm=8#!jm1KMwk5=e504Gb7HP{E2#&LOkGCzUQNjwy6AN&hvo_G+6AttM(@fYOJCM` zB`$m*OOJ;WS|YF1%H<e8A$B;l=d@FlrD^1 zRcLr*;=EAlZ6-Oi1K3|fjg=9S+gU2M0(-H4utZxI(=vdL3@tGWmC9n%HF-Z2njMdv z7(kpJxcF)ZMBe)2#5nq$DmgJhvmXvTaSpeh=IxS&wi9Uv$4Ftnse)s$T^q2x2&97o zaij-qXMh~G5+&5AJ!k~@nxzhQ`qUll^bH`K{I(J~wzYHb4y#lmSY$S8!I4(?ee~wF z*VI8*w(m(?4>pUU7A0BTm{FTZb}tXzF{u>HiI9bowoG*`50r$d@FR~}G~k?N68{=` zsTj8-;WG^MvVv&46VR?rIuwCQ5#L2B>@aXq@p#-U@zKfS^_{P@E0F0eTV5^>FQ!O_ zt<zfccBw{dGI#QlONK|r{&>$O&YH{==BxsJ9_kTQA1V3vh>6vrI?s} zZ>Nc&tQvT^_W9FD-EI@6c7WO)t#CX0i$B^`;J8q-aaeDi`T=Fd#3R>I#VqmBAMxDr zzcc6mp5GeXGA|GRhpi0NE<3L?uMzscFo&wz!B_me5AcB^C8$2PtsR3wTE zDCw&F+cR8~51gJtJgysOxeaqGp(x=}%xjj;-%h4(JvCsoK;z#HXvrREff75-wSG*; zY!myt=?WKgF1k0s?4m;1S?Hd{nY*AFi!Xx?&R!;520TW%oEh&L@AL_A_l$rH()0dbi7vrYHLoX-|A2)Cv9jB-H2V+y`5dOge zchJ%CIQ#Z$J)RcbcO?=oSaOAJ^uUFyJi=))rkOhyPX*55+o(&++veSzxwK{?_)li8 zx};YdmYxO8p3{Z<2XH*pSBl4JO3UYi^=#03kMw0P+Q1OMt)0bwL1WKyqfhHakWq%^ zh26U7Q79dJU3wN&tU+!tcsqCppSOa(#HPZ*-%h*W9V}Ak$ti)v;*w&$>pN+`yWV%@L`7=kJ*C(L{N8W!x)%AW7gLuUaMQTu5H?5qRc-^b6#McA$P8G{w6;bH!{U zV#L&W)DVY|X@h^N|K{4m0qdG?0P)w^_Qd#Zykewu?(07k{q7gomGq|f@#CrPR~bc$ zYczOx_<^PK{M~>0SlnIr($bQ#M%403{*cS0zU+u709it}uNc{_mvI`n{lCh*&Ml-h)ZZV zvg4}2i&gNF2-NvH8pgnCsoPnA>blEPx_uAg?Oj@Sf;fCy4X*g_Lt*>uN!My zn1N@6$eLLAZDow(VGjn^fp2$}MV%__KctXV`hw6e%&--Z`-WMhpZgN}AJTa`x4vKL z*>m$lb~lLIhpuqjbMHh0xLtQ>iP6EGq6aC?oPwkfMBq_ov8>Q=-^FiAm2XR4n9j1V z&K6`E%@+5F@v6+5PrlBYE#0HHwpL`DgNJnEs`H0&o(5XB3dvVywyWt@&O(-vFaeTA zM<(NUjw1)27d#xe4w}DOX{nZEkhY|1*{1aDyS_gB{>p0asnkv7sjlpl*kE_t8_U%3 z;fjwxPgy3Qc;Hp?zVizAE(JHvgg)jQkKsqU@bDKf_K}M0R!7V3V*x99AwKoNuziu|3m-?xI<#Eh1s{11v!xEZiv&d0Jz$+y%r+J3&`b@PG{ad`FQy8 zKryHQHPerx#lNec+}X;$uC$B&@kL=;LWgoA@Vg=0Aiw5i@=kcPOs;wocQrwt4i5zb z=_eM>E!M(q;DC?REIZibElpt(3YZV@cXc!Y3``O3Hht{x+B|Qm16=m$%x+<*iXfIh zrEexToJyK5wGj%@V%wNoMyahO30D-=n)u9)YB1N;US3d;-yogYRGlZ zz%jXoUurshbaxB*X8f-b?7r{h5%wv-W9a9-na7ZD{>2Qpsk|zmv}K~u`GrfZo;>!O zY7>nVw)6D|k!M-$B z)|`aKehpo)?PRj>{&*qA+QP4f8+Ikv;zUZHmjSRLHJd<^=PW#&4HT#ndo70OqSik< z*iJf6u{l%czZPIjI;IJTfFVQc*R!r&dR|euW?^}5Rp|GUL>uiPEMD|(N%JKvs&3#nq+dt~C@zyoG81lgk0WJ1qD(ASc&NjhBr&a2gJ#;w^C*$3}h1C;6yoZx$%6?!H zMbXEeagHRrKzg6eKDKU(6zGu83t3` zg}eVWedoM8hd;`_{akoij(b1HRbTUu&;*B+cw6Y~eam^p>yc#)qyn=xqF3H^n;rJ< z;2S%{wYL3FoE1pn>#i#f|LPWsH-Bwk94j#Ob-Av=aGOl7oKJ30o!6}7bE!V*;~CbF zxaw?o;ZC_r7t2g;o67H;ha^(5>@h8%SxavG0`}{nwPcsmXoVRZ;04lw!)&#z#=@qV z`xlo(S%!cv5(!#d7Tx?rR?qidsXB)hws+ZeRx&H-(RTWIz+!^AJ%Sp`#>pj)8w9H?co5FSq`J?>D8^A5F zsM+*TYxuvmirK82p^;{$({t?}<*zZuT&dQLH-(8-njZKL1q=`31&=8|+7?$Sxx9gBJ-= zL|?1wf`MXJEeqM(_2m?X&Ga9o%0qALAF(H6f3oi6u=*1lJ4TBWUjA0;7$a#4Hop@V zd>4jH7xqgzP1E>>LIrHU*!DXt;nCT|d-&Ltv}9W@ba^HEz7aY$o~;2n>)7=!-bVpe zNHP;M1tgD>kON-?crko9B zY)|S7J;<3ira{84$c0fQ_{0dGjB^&rujvCTQ{r==--mvlN}R8Ybt)cnvmoJQhZz}>7LEX zK4Y=)uc9JImCt@7cT@S6u*7Gqc9G6%93-rbspI}9j~T+bFdjs~b~4b7-P~PV0pTVn zfIeTf1GVt?S|q5gr@g=*L+h=T2nDPuYhkBu0D(0}`MmJ*{l zPiE+nG-=n@-ws%@K;%ohtRS}QT znf|oW$od6Ic?J2$8j-t79jionDPp{kES+BU`1|qYstt?oS`o~-Y8s*i?nBw_Jv|-M z*A({R%O-{}VVcaM1b-rBxybRgD!A7na9s!gyBew+0PA$o_X-RfHW(>}R|58dy08r{T&7y)Xi6}a=#qE=mc$?U^)Ti<&;W8qBbwnehqp2C;- z{bq?Uf`lz!2xRBbou@HTG)iZPmt2OPw17>@4Ou5{8`CPxRhhrj^L^QxjVM7>RTPA6 z0P()yY0Yt)6`&eCxU#Yo+UQ$8%g9K6a?_OmP#A6$4_wcfBUyDA6M7;KP9%z*7-jiaO-NJP4tSiqDW& zq%kqIBCrx3>J;+F|97bu<7xiY&2g3z1^FIB+*BT@2)a5I__x5y7bHww;Pptl ziG~|qnBI9<-gnVs!Zuqza}Vdtu;Tn%^k1KMm({&Et8ro9<38?(Q&$xjHkz9=mX4@e z$-weWn=S8oWHh`Aq9-$U+A+)xluKrRy%4c$@vcoz2iFo>_rxF{zo$pXf5Z>{m|Vw_ zZ8lr1cHpve(!oKO-h#=$b}#WF3%NTa*aRdYJqm*LCe?maUH#9A2bkmD*Ap;oH zX-$K07p4#XM8XoFw>=>@dR7l-yE=O(TCVCpbb$K6o1+VPGP2Y${HH^)x8URMN* z(79*uh+9IrWQz#zZ_5D*=ZGZNt?&~WT@&0TfZ%LRWDa;HeS2o&=g$&Ahq?tXlM zG*N2%8sEUi)BI$@#q!!8AVQx+EI5q4jf5<@%oy$B5EPev{mn{`1EnRdwFmKbXb4!Uv?R%x9b$}>p>LMB2DW9lAh}9Urh=_affKMMF%{9B z06*%F;p49}aZD5IEnH2*jEUIXtrxVAlQOPtK)X)WuMXIs-zTy7$fwP4G{u~Xtg{@}EW)P36H5&lu0Hzc1g;x|`S;A7{I=Ltz%H=)9ck0_YaZAD{D<7B z;xnnBY9~!#2@?6i)T~G-fg^m94GKY2z=}7P+dkG;TnI*Lin!9eH%EG`!)HOWS9CA;dIozQdHO+h}O0Z{SaoFI|-j&C@m{oc5$=+$252R z-voIxX#XzUA~*Xgd`-{*UgGufuY?cP`kpatp`ATCAM!o8S{i?#f@^>EmPt;!_43Qs!`G1OJdbSrYkD z!XF9gpku*W)$6Ic%?xxeNfbwVvCZw+Q_~5^z#E#sZl0-8TIaLb;=F8*tL$tX-#crh zsq@1yuGx-5t@!z2JTJapIQfCs9m1Q-@O??W+|Z9!KX7ywy%by~J8q?jo(SAHKMFX~ zZu4}(cp8nrLgEvRzmU9E!Rv-QjpVF<%`i5XD%nM02wIcUPAVU{8A+n6u{$-_-y$Vl zHGFViv%1US2ZQf2wP8^DJ5Qb|{?55A(-8EE&0#for88<=2YHt?C1 zMBnF5kZ-ktX7yolW^t>JDxUMZaQjr~5BYE{z95k=C5!fTcb+31b-=FeX7F}X^dnF= zvZoHyv%^VGR$yTpEm6+{ z@RY@9cv{Qn-5iZK6H1bnUb#18kkuidwYW}5l@ARm$>Y5#3tj_nH;nn6wqvv4n}V}n z8#J{Nokxk)x-LR@dO-8;%ds&ZMf2ttc$p?{p({NxR&`Mtse#AT>f;7)!9!|R1UO5% zX^iZl9`AltJwn|wvGx6;Cyo`e(nh)p|NexnaD=px-~5YZj<+S_klIOr9;ygi z8VF9I(mU*dkIk1!Cr=|?Zt_HxUiH%~%}H1Q=k>42GUKw{&2h1PrWgYTw2%;cnb z(-wF6jS`m8rLV%nM3EOfAxWGxP@z)UYXIXRR`j*7*p69*8q=9xiT`>^4#ErX;kpRt zY!c_lue5pz|F@(3>1Y0rQ9SWA9Hm3YBH)|$SmJpoLX#j-eO=bfhKolb@pD1CPRh&j zKX(jBkFFr+LoY_k#yvV_NRr?#)t%!8~O zk60QcRx_ck)D<6&e0T#oyM*M2tUhVI0jIUE5aleCUy#J4;}sHqYpb+#wlwfPrSdFP zd@3pDf3yWn7P=1IO&+<9jvc-DuWH-kvj4=HrsKBWQl2>NH>*l6=wbbk2F{nq)Z>$J z>3E5r=u$W2&Tp4}u$6x^ zln$vt3;x0Sp&*-X=L?G975WYP7|d9nH@R_ht!A|+{>5G~JuL-4>n`i*;ck5fj|F5I#4r?O&+B1_zLX{#_5)cFt6+{#PNl;O+ z5mA5G)?iz(f}o;HnF(OS0tzmmVxpj+=z^jGB4iL1l)ADC2yQ}=j)WeP$;>zV<-f`E zzQLS~v%0LyVgwYgX z-adfOG;vD=4zqcD#Ld=l{Sr?1Fz@8KO;h7<3!%0B?86>#q@t-+GK;WQ56UGE$w^^E zFZf^lsFR)&;A^@$=(1+773b1#$%9KPYECTRW#dlcO@HfN$M!WzSXir5-z1qaJB~Hc z&{B`f#AmHAi8aM@>UV%UD@Y8KnQfbhUz6$DKu!B?U9uC-?8H7?mh#U_t|-&qaZT{T z>DU8PQY-u@m6)D_2w$zB1+*wsgNBf7#?Qh6M9Wsx*t`{vloY zTJWoh{6p|1W!gvX+i90Jd@^;+zf92(j!WanApw&0oHET^L)tfE@R2>b5nRHcepvyg zYYZ7PHX5gG>&bC|PY(y*#b+zBnqwz*fBb~=6!42SMScY^-4=`pOG%nG#F$F6$P^pu zjh*3ZNDrI-baE5c%-p(fQxr#D;k_A5Iw4LXB?^FV}(X)emJ2j`$%({3!g?71quC2wx17)AWx&MH~M6=UYwDCVH_7aMY$c@Jv8GI-t{NFTP4ib#dBa z$vjp%=h7^>-M!D!xY%%|L967{uja#8)0D&+lm3{CEpGr$#{(V|u@~w`yK8MomR9fh zMR%QTzo^2ZA-Ju?>T2-#p@ZM2?%(I$S33_TZHA9w8|oyT-$jXIQf)#SshFM%h*G@k zwPgD!Iz|xShkVog@&Nda)^{%G20h47&$raOkN!Sqd0P+Tk*0~lR_8REi8)OL2gW#0 zm_OpO;DgxsK#~N5ecRwCcyAoBaxUOc7wgjdmZ2zMX&odEK%ZZRG>mu=B*rAmagEX^ zz0*NB8ol65LQ%g=l;&u_T}?$U%_ z?hg_efx8U(X2M|lT!fES`-1y8zu7+Frd*|U*12L@s>_;FU;mOkz)tt(i`iro2=?I? zVU6WE5FS|QEDC=adXK8LaSM=1a|=goC}LB+%^coK-ptq0@aOwV(BqG0=2AMZeU{u+ zvoQa^yYnsZiHVB?f+P53ZI_frSdNkb$n>h@OmKh!F0vM{^8t66g*ZbtwA7R6EHh@} zMyDlteV@Y1nM=d+l46<^f%p9zVs&6)>Eu5LTzNPhi5S zb&rS$#w0iOg#_Q%a1I-6lTeQSH<`4x_^ujZ2B6qDJ4g72B~Jw}{0I^&!GX(ey0s^| zw%1|M%rF1kyCaOgzzTmcalUdQ@Bv(+NE*hgM~RlNc%(vlik`~@VKHBCkSkie zx4=@0*m1}HAAK)E!>!X_S5S+Zb~;Qg4uuH2rSsv1QxRr3fct;_#(gnUgi5_hA%ZGx zt9x2ujan49HQ?s8Bp%qhIpSqh{(M>T>rM4(^Nq6iIF&uK9pCy-PJuwF6TNO|ZXxmh zv|{kPXcOH3nH!05&)gu6%ZPeQ?m{PIt6rl$4c&_{7(9Iw;W#vR6ODfX$O508a`8Lu zy`C|isyDkZRX-rWm%Y>GeXH>cLHfh9{n>b%GoY^2k$?Ql__aTqy8ja_`Ndz zTYmV!x|kpN7h?aw+PWqBM1~yBuY|8?A$ttJUDD274|4VQp_;}l#2t91IX7xmuifME zB)f8A@3CKj0M{I+4=0_4)u-8NYs}-R6)~quqEf*gSd<~ep)B1;d2$7~35gB(?(g1c ziG6{2P;~|)~&I*$rdQJ9IyA+p#%afp~E#T=t(Gk5k1< z5P=?FN}o_|;v;4n0q!!Y0d!;oFBj;yzps13;HoZqMWK&Z`mavd|I7TK#8a7++6pH+ zz=ss(FS&Q|#F>0gTs8D+CAI4n7_uc=~vF2mXewopYomkIgWVUDEBo! zhy|LVWvuAm_5*3qvYoVzqB*`N=p?|)mY`@!V`Pa&bUk^WGsm%AQzKDhGbppw=vyFf zcm*fovtR&TgcbI4KM;pQRMU%g5FSL0l%bIi-E!s>YQ9_s2FKF&tfKAE(Tf5^^jxZq z_FYlDf0M&osVyA`X!S0GLjMnk*b~u;(U=oc`9J$avNQ&qLPi+u z8SYAtXZb`XF=@&v*C2z7L&nLE!CElHE_c0b*GptOy}C?zR})7>f8BYDf6lKQsf}HZ zd-Q+L_r%t9Nj>mLJn#V0XOirS6w&y!OPUpXbbTuk64duV<4fSSc4|{4w|Bf^P5_{s zZWu<5KUEFoeB);Mx2%(!Pi)BIVo!-va;^^XhbsRPfhi^9(y6ul``}Aw@qOrZB6QUZ zjpFTvTA=2qnjr^3WE#U7R>Yt6^cCOO{E5h3Fz=l2K6Wj(bmrv5f>!@Z(Z5Qo{?K;S zWR)ZfzcVO_#U=k-6LF^M7q?T-8W;cRd+Ig^i>7yr2K$ZCo~H*kx`8!iyBi`lA%VOUX} zw6&`<{}Sx29>MZSU`J>13+h3`piSgXO5)IXaQtLx&CqS;Bl8cUh7DKcrdErC%ddL{ z!E3p;#PVfRyIUKkNeAIVmXb=Z5}1Pc_X(lyk_=Ee=CW%?klW5^K634)X7W7Hoh9Fi z-bD9MQ-#P8nTy0d+X2LU!vTrI&i6-DcVnoMZ>sapLeV)YAg#UIh*Jkez6TFmG&=Iy&~}Z9RkAG!_oi2jSt*laPX1heUG1qlu8dTC zlTKZ=z)ITSlUUR@xQwVW;_pM^oH<^MFxoL1t>8SzoR%E{T7%1UIGTLz_9%oboMzlR z!m-NVvN$=ZkulWto!gXjo7)f@*3(ca2qBi>A4a9OiCC=p*DB<>Mq{L~LO*#65^hgc zIC%tZLyJjf$oOn#-)xSV;zhM_7P}jfQL+~GWBzx7anE!$P{EzS%RiG!ii0)W2%&q_@wL*45 z-K)JDp$RkbdfBoA!q7x|F5{Sy#(u?-LY;J2@kU}llN@8BlG>~0sVSJF>gy+Nf+R=n zF$fEYXKjG3ZO|54ZWeF`HM)s*Bhr_wKC|RCP9?$r2sS}tA1(iqDZM!r`koAb;1myXb{CQ07S4z>Jz)c{ z3_8UiG3CvL-9RPe3+bA1tW7osy+Nm=jSF&)dg-Z1&T^S3EjDe}_{E6>n16)ABMm-7 zLXwb5?)XK06C3J8f|&}1 z=;Nrs9rqv@vv|+IJ+_=jNZrTBKqdH?xu;{EQy>@u=7Ni$Hrn%E^Z0vlNoL#zb$>SO z{_D<70U}L;%Q1$0xel?a3pPd*we%K(uNDT)-K&>+1n@FXy@E!ss(u1|1Ev6TZA6DY zH+p<@(UPPp>@p^b3>YeuiCRSUN@ukTt`QrL2Vi-fqAuw;5o8Rd&KEzT9_$4~kf$s1 zlzLd#MCy!Orw14$zX4tYO-KvV!Sd=#8vB8$I=F({6pAihM?6r}j7kx50T3;8jWi_k z7OQbiL6Ze95eS7oL90x~)@OlnJNQDM4%i@0Loar(o@zq5N!a?)wBA@vqO+AP8@WQf zQA%e7{{W{C@H#AHd}ILmv8)U610xLdY~&QDwM!A7u!hIhly+KMkl$HuSL}I_z)DM!|JT#$ zG4GalUq!8p+-Sl|HP*;|LBjM|;nzAP`NaPY1FD7ZE^b4P+E&|yc~8ahphSB04o(T6 zkQF6Omv$^~X)zeHQn@IRm;hCwB8xa1JOGb|V{ISce4@&QjzlsNddU0IJuA^vpa5y4 zCDW5NPGm5DZRb>T#@fRN=1=S9$7eOP`v2X~A>G*uf46{7&on>9Bcq$QK;P>~6m<;` zq=E8o4dBWfM9Ub(OtHbzg<4OEWw9f=mNS>5nK4ddyXCuECG)G`=ZX=G9S`m;3$Fv& zv$ybslpVA5=+F{8 zHUR0A*;~<%|3hRz2zXzUqpde9caiTZlWgFt6aGCx^^AlkvY&QP7EI)H`v=Dy9>`HN zl}R&_Nay_PC%aTN(j(Cfnpuhbj>zVrw<&j7_GbW0&v}XV&?yF527Cp^(GaMHF%>a+ zKslW7M}dr8owlEdd%$hfeH8mz^XstBiUlsN`A_=ds9L?8C;2N5xdbXVCr>-Z$_m6Lm;F6Cc$HVtn1~{b7e^cvg78 znpp8(-IzEpAsoC8zJNcJ7j;Qvl+81O8j?+in(wv%t39%AxM$1g3?ZH zx)gTf&(rn6zRC?#llIfPi`Q5Q-1-*?rRRw%7UqXV;z==D8fid;QR8N?1-wAT^K_|= z&`riRgW6TTdyCVZC2yTX9Y@W&3wR-Gup%U0?bBxpH|&jlA-gg%n9|Z`02?&qE^5r}y zKD28AXAnrEW}iczG5px+x>_fuzPWsyp(&nA2hj~JBX|2kFY5ZXAG4h*l$XAfo+q0D zd+*`oi{DVi=CVyMsV!~14In#MaHCRMdIpP-iJRdbJZ(=Nfxs;639b zZg{{{=;pRYb_G|DaKP8&k>Jyhz^_B%S1gSpB+g9M%KBVI`4Ss8g;>Y0)PI+I-gNBp z#LBG98;c)E44!;mdr|65WOYhsxSbx}mXy8--VBGI4Fnq$EJZhkx#R+$gCe(14ZXk3 z<#Xfn+lN#BkR1Ivl8F^6@+#p+YSsVy&(XDL)_id-SbhyDH*Va-@v~Cv?J7(3=#8~o zWG@q3!sC%by265IE4I+tG3foJGVe{1VKJuc1Q{hE4yJ4qT^L`hdU8WT$zx(7m;KQCb5<=Cx&;@dP|Wb!X~}6d)wG zUy+;f-{#`AV27D360s59rN6HG4VAt~tFU~)Mg(GRJRzM_>>if*6IE|Cqitx8oxlg^ z-m-#)^I1ych@hnEqyhmC-hf46+&f)Ji$#z$VFug;}iyk;Jc>M(L%H^wUG>ky;gze7|ZFvV*+C5wr` zMaWJ4f&%Eao9tx?v_IlUtGasB$u+@CfUe)xA`SOnf#C__3yPdJ!Yn! zR5bsm#L-a5A_92~crp#$g61Wm5`)w@L8S=DsgcJUJN;=A&(hpi7+kP49Hsg08 z%Vy{5t?J_9*;eZP3YEO5ji@0hIJ(v}M%wiKk8GyfqvMuNwd|iC$6YOnBgo8`V7m?x zGfiykK`0L5FJRSiVagK4cIDh3@H--Z3FbjGJfglQ16kSeNAwn*0-Lb0)RxdSaFvAX zY8O|3oV-+G-*x45kq~ZG4?tCvqAyF3hudITikdv4Tm@e*}+Goodw802#e8eQ{bh2P*7kVn}YKcQKKU@MDJE;z$p17+Ezqj zWZ5Pcp)60?*=u$)Bik;D{s)IDmVc0p69*%O-yM(zoL!Vt3~D0M03}&L(i->C=!++n z>L1*zu#`N~u0YRO>hX~`+HyD47R;gPbb=!rPCgGcg4amwFmoBv>C-F8hs@>!*Pus~ zX#KF=;xer`_(f6-03&30qS*%L^dC+`{$%lwA~K|y`e3$wz^V`I4s6jeMX*w8ov})( zdmip3GD0a7mR=EP#W-ur&k;VnmvmfKHBtIRQZREg9iKyxhzSu!vTaBcSh7>hhmMZ& zx@oHw;}eq8+u^yG2d+N+7@5|grMioEKuLCSeR;JNf;E!Z{v5xV5saI+g5#l4Y$5+n zb|>h&r7}qfP!BHVGqFw%Hw6=mNcVf(FlYs5D>a$=*|Fd%Gf}YKGIrlk@5J(V#Lt>? zY_JMs@OjrL1=PrD3Kg8!sosF>lKmLH&yE*}CG)T-<#bgc>+Ewew2jedO*RVr z;VQ4M@j|#8C8juDC%OjrNw(4=DogvCM8oHSc1z~77fVyyULppgF7 zUAFZUun6>_9M~?-?Axeh5soMD&B_|N9dQhtPeWZ%M~JDp!#cM>sP5q6rwVtg$~pz5 z*kx{04zi0(y8(`}MDeml9?38}rv|phH|_zxL4%_81SzuUxq-a*;A?8PgWf3bvd)8?S51lB08N;welJ?_l%{Ycvj=%p3(W*qCoh%~ zQfXi;H3v5s4W`?R^Xc|Fa$Bdx8@8CY7?WpESF+E!cJ9Gx0H9L+)_8BT!#94~6f}?v z-llZw%NK^FGG0g2e*A|?uID^?Z@zj-*+QU{#UWQQ2Xx)A;zATVMh_2EvQ3p;fpDik z&4iVk2UZ^v-{R$g;PO_=&qaLy(T{E7{1^En_^cX{!sQypYwbV44-#km<&s8ZS!sXO z505{bo6CKOtUIxL)fp>|g1#ZQD4l6w5$owWL7a7;M$ZEim24oiZ)69ld!_?t3+2rf zgim{IwqyJkc9JIEwtBY-B3T_@1)IoRaUW+e^EPhVJA^A99PanW-w7e&18jn~`7} zbdK{V4jQl;`%N+2ne^X}58R^^vBDNn?om0K>1MgfLe*$v>O$2rtg_DzuWigj4NoHr zz*;Dsd4!_d^IVE@k=V`#iyWXCG-saL5md}vm2#{lvZJAi6qd=RTyPgRL+6+IW~$_^ zQdGB*Ex_L@T~rHzlO()*vQ6?oz>)Kbx))ofm&5-Fx?0Y}bi#oEnvHhqS+V<_QaxFM$ z(g5yR3OQ*ZQ$PinNZqD6^b=HL<}B`diBR%^OGAe+3zMg$iI-~bur%RNdYw*J2Q#pe zpZ>~>_#?amhQ0o!ov5+*9;UA6#~uFZF|%+-(ljMEDTd5Tvz=V^^t`=?{Gg$A9lc}0 z_3B$lKax&8sOe^AVn>nfI2Uv+#uOiVC6cO~gSVhz46$Y3Ein3mS5I^EqCAmgyz=gV zLy$Yi9&Dl%Yh38g;0anW^#nQQjVn2Gg-_{cW(!wb7M6c3XL5bHqrvsPz;4J?)6LSv zow!rMwn56pd%nCFTaUiMPW*Ka&64fax z9(gI@5>-Zx2I`7D1W-`58_4MYSIBKw&j)(+lR$*JQIlt8G6_%46X8?hIBfHacxt;+|f(|FDnl1533 zB>BS0&c5kj@?oSG;6nwBGknlf1IxM*+cV;&k>H3N6$danR_E-RIyW5rZyEWGUNgEm z!+J*J9ch3(ci>O7%CN@vQ&lUW%k2-gM^fB5UM{q-2N>meeZrjV<86ij`SJ7FyymY} HL2>^F+ocNw literal 0 HcmV?d00001 diff --git a/bin/zml-smi/tui/assets/neuron.png b/bin/zml-smi/tui/assets/neuron.png new file mode 100644 index 0000000000000000000000000000000000000000..ea57a753c935c00be5bd00ac2b0595144fa64853 GIT binary patch literal 1916 zcmV-?2ZQ*DP)F@J0g}dkO^J}Kf&*1Apj=<&Z@_DY&J&V6rn#bMf@H>jWIf=gT^Y(kM(znv&W1-5v z*5*!?#9Exl&EM@n84fG=I@`x-J-_ch_}|b)8w_$)-?Gr-)8g%Dq|Bbg+={r?xzpvT%HYo5>yEnEu+HN2 z_4jnE&_9g8FN3?U&EmV%<+{}6fU?t)z1cR0y-$_IgS6B`kHK!I&gSm(=+!A3;keS|ox|MR=g| zi@DcboydQ&(?gEHnZeuR?DAfp$<*WRW~0l6wbe+H!eOAw$lU2nl*7W<=WVCW#@pzY zE@bWi00k^bL_t(o!?joEUlU6d-SfoV&88Pph0qeJl!QR&T{=h$O?s7P#riCWfL&44 z_ugOo?rcC3O0vP9`LLO}v-j-GZF4Ti{n_8WuIZ1zp>7^(2AUT#FQ~2=F97UzpkQ2c z5&ggq;L5Grw{NurKhhFbS6vrje0!4(2VNQ>PN))6Pwp^5=*vHv=JyFObtEf4eC}g9 z_c2GWz+TmVl7idxnMVZEgY8J>gz zG$A8`0WO>8v$;SJ##T*Y0D09wH3h8UWs>d{G;zyfks*z7r$8Yv}>2(a18CR*sKs6$5# zfGv6T0K6(97m|=G#}$u$`5PrM0OsJQX9HYvR(sG)|AozHkn^|*%&^tC5d)x2z^78r zXZ$A@rNRg3%TMb=mJFx=fft_AdI-K40J@s6sl^>LGvef7DzJz!kBZ@0O;s#6s(a{4 zC|M&vk`uP;cT1iyIS65T`RUyF?n8SY(YON3A5CC$LD*Rc!B( z)g4sVuI~l$QeK!)R1b1t*Kva)#yFju2Y9!RTLxdsX8ApGRG zs;6i@l7#S-Qm|AIlo@dlMn-OHU_Ca<94m@E?wk)$cYIl!P54G}yLtS^2UJody3u~j zno|m*03HJ6&m+9R&sr;yRd^(z(o|`kkCy6(JAuk#wW5x}(&{?dCZ$mW<+^22 z+_UsAQpZ&Dt4&h9H4-;yN~CGh=a&QRt>*BCsDUca*C*OF)@~EK$xTt`63?lt)m@-* zw6MOlY@YS5<_q?&d6Z+Y-N^fm?zwSxw&D->_x}Sw=Ef6{mP1hh0000zwev(ectDJzW1Lq87|HwaZv?P0Dw5z!PX7# zmhjQS3c*!>yy8CGlO-v(T z)Pk4>KyUt*iRs-u*8my^Bs3sjVXuIbS_c>$KyLzZ4M_E;`v1Umz9<8N>k$c9WqftA z=?7XuNE!hhDmL*Cd|lWp;E59?8-eGhLr4Ov2>E6&B&YHDw`O26p5Kdj+j18q3AF?U z_$%y1l3Oub{0Q)fLxHHE#8(s~A)CM?Twz4FgOe?YAH)nlVp_pTz7U*%AYdah5SWCb z96)W~yw2tjAmD!i?R||cCCC8h21Nzl1S+E)Yol7RoP1IP_T&hFsT&d*ODtIBPl(8Ak(#(elVC?!pJ z`Og@xI9!}4&%^#n|4#X+u8GL|T%VI_N-^%%uj*FEt-9C~tcOdaZiWPNV1_JY7pd3$ zc8Gho)ySdX{+UaU^f{Kh*58e0QqPwTFRLVJ&P`W6*&p9q)Nu1w|BD|A#eS)5vCq?S zl-w>G9bl-qH}Tjh^FVJWV-1Y?5R>E9hb% zfxMwnZBxC@>z;jj^c|ja3MYK~rH!UqZfd%Sw>5$KIgPt8ttzK8tJCzR!X!)Dp=ZSB zo;)j$@^O5bvcTJ5WJ!kRD_9zg%d;{r^pAMq#K~%J$0hPUg4@i`oo! z3eQ#dd(?2^a+0}pDy6ltInhM(S$~GmGuZcK=7jr13Ao9(sfdXBb zcG@3X=3l#*3&i|#5>o$&ldfqU4V_cHnYiOD6R*5FJxVU_($_RBFHV-VNKWV>l;~-r z%PIHL={T95%OPxnDay}!=Ye2hVz?}7PNIE2w)9CU*EFcgB-z`5v94J1{ky2K+t{f* zyzSeZHSGpl$?m#vwil6UCM)K8M=D=7Uy4zd?LLJv%X7P1(p*Vm4_jlVQn3;$WBWp+ z#^RMfT{^CvyEfXYQbh~F3SZN)aH*Q}>>#g&7k7!UtVN9enxm!&<)J*hhm}I)OO49# z0p_)o_}-P_l@WrUe^&Rk;?{Emei zHV=u=IwWRAZOeL=u`+Y$k?!=QR$*ZcaR>K_Bcmc{d-l8(ruRniwb10Vc1fs+Bzug- zyvz;rs2}SMNo4uNM?k6-?rKC{7L8Gn)Nhzd)MSUsSjyAA$50J5F4ZKbt;%D*=r B0Y?A; literal 0 HcmV?d00001 diff --git a/bin/zml-smi/tui/assets/tpu.png b/bin/zml-smi/tui/assets/tpu.png new file mode 100644 index 0000000000000000000000000000000000000000..97a6ae3c04c778ca665b6f2897770b74a9e19a95 GIT binary patch literal 1878 zcmYjR4K!5g8^74uCatym*VdtxZCbUJm5tg+%ooi_wzOV64j$G*uTWvoL9S2~lf;2UB9Y_11pcAe7y-xeU=pun z!rcZWEflcA4nfeOv&3ye4($wvAk)g7zQJDG1ioCAo~Ob&Wf-><=a%Y$<$+BZ@PJpG zr$TvU7{?fb3|FAJ1Y&~l`cc+h@O^-GBYPYAw!1J zq!^u8C|Fyv#(uL3DMLWA;38&2s984feruVGc)xx1PwWBV92!dQMyzIa;UI@ zeGH@#?SP*MAsErxh+woJ4WlCG^_{*en976tJ|vrO6ntuahgmA$|H;Bi4ztO zq#-({kv_<0O6F(7=?G3s*U~aD01n15qUqvFu<#;0vJga#)czF^;Lwarfdb5QFzV}h zg&BY;0x09*3Y?mu1Kj~EGXq3120%uzNF|_yexsZR)B)}h0Nf}SDD+C7qNh*N05t^l z$|97ERN~-tXoS1~iqIS=59SAJzKRXM1T3fsMw1Zj-lZYBtw1+`BgJ^e10^UJyMS|i zw~?YdU<1GbhYmd9g;-D?aZ} z_Dszrd4cHk$iZ)$w3$QKeyt5XS-AKO7Az4XFE+2uKkwzSDz2^iV$Dgq z!BfrenzmkT#k28BeM7Z0G3AFZ`2m|&JM+-T#mbqJ&&&EJUY==>d~eOFWQDv4 zuOp%rmyqENce|Zzn>~jgH~%%;Z{H~D8`|ug;ucV;ERQLah3)62B-IVas{@_3cR1MB zF78UIBE4l$XBR7c$~iQXwQKFT(@yK)us!E}r|P<1J&h4im-}SvwiNi2d-->keOeG@ zDo9*B5O@eG>ke*iO*Pvc^<8q>Y=h6wgx-JcYUnN4AAL{Ul9Tzvx?& zsQhwxHT9f3)s@72&VqJ)EV}Ix-8v9<_jC}1yEu0&U(#rHxIN6QA}r@^@vLpg{t$H9 zYP;nN_Sbb|hgTwAZ))5xo|30RQ9+kklk1NM_BDK2$-TD6#PxWtu(mJj7ug5q&(81F zew0?_Gp5nT&{>pHc9Ae5Su1l`q-{>?27N~Gt*YMRu~|^cRGDx^u>r4>jBAV zM^l`(@g`|}L6uFUJOsYJD9Hh`u~&F%sya7)cSi7t%yN&@+82IO6)m50@Q*FGO0i`` zR9An0|7vlQOYZmQE@*CE{s{_y_+oI`K;6J@eP*^i$*txhxp+lG zZNNBLT}~cd-OwI;6(&!^ITST(5})%vy2?!nk37g}(gaN6>RomzaQ4^YAqUTI+O1Dm zGJp8Zw9D@7ivH=n>i8%9a_R$bi)-o$rF+Y|{zr_EURI3=Zal@4-0hX@aI6mZzAIkH z`l3dYmC=z^CEatPwJ@(aABcHw* z-74r?6;0uVTY3zfdG9mp#0&9PV>TW;y7}#!n*o1k>gHG?(f$oBQx6QDZ3_f^N6O&?;LicEz5JL}^k~UHS@+&q literal 0 HcmV?d00001 diff --git a/bin/zml-smi/tui/data.zig b/bin/zml-smi/tui/data.zig new file mode 100644 index 00000000..efb9f028 --- /dev/null +++ b/bin/zml-smi/tui/data.zig @@ -0,0 +1,146 @@ +const std = @import("std"); +const di = @import("../info/device_info.zig"); +pub const DeviceInfo = di.DeviceInfo; +pub const GpuInfo = di.GpuInfo; +pub const NeuronInfo = di.NeuronInfo; +pub const TpuInfo = di.TpuInfo; +pub const Target = di.Target; +pub const Targets = @import("../platform.zig").Targets; +const hi = @import("../info/host_info.zig"); +pub const HostInfo = hi.HostInfo; +pub const HostData = hi.HostData; +pub const pi = @import("../info/process_info.zig"); +pub const ProcessDoubleBuffer = @import("../utils/double_buffer.zig").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +pub const ProcessEnricher = @import("../bindings/linux/process.zig").ProcessEnricher; + +pub const history_len: usize = 500; + +pub fn RingBuffer(comptime T: type, comptime capacity: usize) type { + return struct { + const Self = @This(); + + buf: [capacity]T = [_]T{0} ** capacity, + head: usize = 0, + len: usize = 0, + + pub fn push(self: *Self, value: T) void { + self.buf[self.head] = value; + self.head = (self.head + 1) % capacity; + if (self.len < capacity) { + self.len += 1; + } + } + + pub fn sliceLast(self: *const Self, arena: std.mem.Allocator, n: usize) std.mem.Allocator.Error![]const T { + const count = @min(n, self.len); + if (count == 0) { + return &.{}; + } + const out = try arena.alloc(T, count); + const start = (self.head + capacity - count) % capacity; + for (0..count) |i| { + out[i] = self.buf[(start + i) % capacity]; + } + return out; + } + }; +} + +pub const HistoryBuffers = struct { + util: []RingBuffer(u64, history_len) = &.{}, + mem_util: []RingBuffer(u64, history_len) = &.{}, + temp: []RingBuffer(u64, history_len) = &.{}, + power: []RingBuffer(u64, history_len) = &.{}, + + pub fn init(allocator: std.mem.Allocator, count: usize) !HistoryBuffers { + var h: HistoryBuffers = .{ + .util = try allocator.alloc(RingBuffer(u64, history_len), count), + .mem_util = try allocator.alloc(RingBuffer(u64, history_len), count), + .temp = try allocator.alloc(RingBuffer(u64, history_len), count), + .power = try allocator.alloc(RingBuffer(u64, history_len), count), + }; + + @memset(h.util, .{}); + @memset(h.mem_util, .{}); + @memset(h.temp, .{}); + @memset(h.power, .{}); + + return h; + } + + pub fn deinit(self: *HistoryBuffers, allocator: std.mem.Allocator) void { + allocator.free(self.util); + allocator.free(self.mem_util); + allocator.free(self.temp); + allocator.free(self.power); + } +}; + +pub const SystemState = struct { + devices: []*DeviceInfo, + host: *HostInfo, + history: HistoryBuffers = .{}, + targets: Targets, + process_lists: []*ProcessDoubleBuffer, + enricher: *ProcessEnricher, + allocator: std.mem.Allocator, + io: std.Io, + tui_refresh_rate: u16, + + pub const Config = struct { + devices: []*DeviceInfo, + host: *HostInfo, + targets: Targets, + tui_refresh_rate: u16, + process_lists: []*ProcessDoubleBuffer, + enricher: *ProcessEnricher, + io: std.Io, + }; + + pub fn init(allocator: std.mem.Allocator, cfg: Config) !SystemState { + return .{ + .devices = cfg.devices, + .host = cfg.host, + .history = try HistoryBuffers.init(allocator, cfg.devices.len), + .targets = cfg.targets, + .tui_refresh_rate = cfg.tui_refresh_rate, + .process_lists = cfg.process_lists, + .enricher = cfg.enricher, + .allocator = allocator, + .io = cfg.io, + }; + } + + pub fn deinit(self: *SystemState, allocator: std.mem.Allocator) void { + self.history.deinit(allocator); + } + + pub fn deviceCount(self: *const SystemState) usize { + return self.devices.len; + } + + pub fn recordHistory(self: *SystemState) void { + for (0..self.deviceCount()) |i| { + switch (self.devices[i].*) { + .cuda, .rocm => |*sv| { + const gpu = sv.front().*; + self.history.util[i].push(gpu.util_percent orelse 0); + const used = gpu.mem_used_bytes orelse 0; + const total = gpu.mem_total_bytes orelse 0; + self.history.mem_util[i].push(if (total > 0) used * 100 / total else 0); + self.history.temp[i].push(gpu.temperature orelse 0); + self.history.power[i].push(gpu.power_mw orelse 0); + }, + inline else => |*sv| { + const info = sv.front().*; + self.history.util[i].push(info.util_percent orelse 0); + const used = info.mem_used_bytes orelse 0; + const total = info.mem_total_bytes orelse 0; + self.history.mem_util[i].push(if (total > 0) used * 100 / total else 0); + self.history.temp[i].push(0); + self.history.power[i].push(0); + }, + } + } + } +}; diff --git a/bin/zml-smi/tui/image_cache.zig b/bin/zml-smi/tui/image_cache.zig new file mode 100644 index 00000000..a2340ba5 --- /dev/null +++ b/bin/zml-smi/tui/image_cache.zig @@ -0,0 +1,36 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); + +const ImageCache = @This(); + +pub var global: ImageCache = .{}; + +map: std.StringHashMapUnmanaged(vaxis.Image) = .empty, + +pub fn deinit(self: *ImageCache, allocator: std.mem.Allocator) void { + self.map.deinit(allocator); +} + +pub fn load( + self: *ImageCache, + vx: *vaxis.Vaxis, + allocator: std.mem.Allocator, + writer: *std.Io.Writer, + key: []const u8, + data: []const u8, +) void { + const image = vx.loadImage(allocator, undefined, writer, .{ .mem = data }) catch return; + self.map.put(allocator, key, image) catch return; +} + +pub fn loadAll(self: *ImageCache, vx: *vaxis.Vaxis, allocator: std.mem.Allocator, writer: *std.Io.Writer) void { + self.load(vx, allocator, writer, "logo", @embedFile("assets/logo.png")); + self.load(vx, allocator, writer, "gpu_cuda", @embedFile("assets/nvidia.png")); + self.load(vx, allocator, writer, "gpu_rocm", @embedFile("assets/amd.png")); + self.load(vx, allocator, writer, "gpu_neuron", @embedFile("assets/neuron.png")); + self.load(vx, allocator, writer, "gpu_tpu", @embedFile("assets/tpu.png")); +} + +pub fn get(self: *const ImageCache, key: []const u8) ?vaxis.Image { + return self.map.get(key); +} diff --git a/bin/zml-smi/tui/lib/compose.zig b/bin/zml-smi/tui/lib/compose.zig new file mode 100644 index 00000000..e3438762 --- /dev/null +++ b/bin/zml-smi/tui/lib/compose.zig @@ -0,0 +1,46 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; + +// ── SurfaceBuilder ───────────────────────────────────────────────── + +pub const SurfaceBuilder = struct { + arena: std.mem.Allocator, + children: std.ArrayList(vxfw.SubSurface), + + pub fn add(self: *SurfaceBuilder, row: i17, col: i17, surface: vxfw.Surface) !void { + try self.addZ(row, col, surface, 0); + } + + pub fn addZ(self: *SurfaceBuilder, row: i17, col: i17, surface: vxfw.Surface, z_index: u8) !void { + try self.children.append(self.arena, .{ .origin = .{ .row = row, .col = col }, .surface = surface, .z_index = z_index }); + } + + pub fn finish(self: *SurfaceBuilder, size: vxfw.Size, wgt: vxfw.Widget) vxfw.Surface { + return .{ .size = size, .widget = wgt, .buffer = &.{}, .children = self.children.items }; + } +}; + +pub fn surfaceBuilder(arena: std.mem.Allocator) SurfaceBuilder { + return .{ .arena = arena, .children = .empty }; +} + +// ── Fluent widget wrappers ───────────────────────────────────────── + +pub fn center(arena: std.mem.Allocator, child: vxfw.Widget) std.mem.Allocator.Error!vxfw.Widget { + const c = try arena.create(vxfw.Center); + c.* = .{ .child = child }; + return c.widget(); +} + +pub fn sized(arena: std.mem.Allocator, child: vxfw.Widget, size: vxfw.Size) std.mem.Allocator.Error!vxfw.Widget { + const s = try arena.create(vxfw.SizedBox); + s.* = .{ .child = child, .size = size }; + return s.widget(); +} + +pub fn pad(arena: std.mem.Allocator, child: vxfw.Widget, padding: vxfw.Padding.PadValues) std.mem.Allocator.Error!vxfw.Widget { + const p = try arena.create(vxfw.Padding); + p.* = .{ .child = child, .padding = padding }; + return p.widget(); +} diff --git a/bin/zml-smi/tui/lib/image.zig b/bin/zml-smi/tui/lib/image.zig new file mode 100644 index 00000000..4fce069d --- /dev/null +++ b/bin/zml-smi/tui/lib/image.zig @@ -0,0 +1,78 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; + +const ui = @import("ui.zig"); +const Allocator = std.mem.Allocator; + +const Image = @This(); + +/// The loaded image to display. +image: vaxis.Image, +/// Desired height in cell rows. Width is computed to maintain aspect ratio. +/// If null, uses max constraint height (or 1 if unbounded). +rows: ?u16 = null, + +pub fn draw(self: *const Image, ctx: vxfw.DrawContext) Allocator.Error!vxfw.Surface { + const target_rows = self.rows orelse ctx.max.height orelse 1; + if (target_rows == 0 or self.image.width == 0 or self.image.height == 0) { + return .{ + .size = .{ .width = 0, .height = 0 }, + .widget = ui.widget(self), + .buffer = &.{}, + .children = &.{}, + }; + } + + // Compute cell columns from image aspect ratio and terminal cell pixel size + const cell_h: u32 = if (ctx.cell_size.height > 0) ctx.cell_size.height else 20; + const cell_w: u32 = if (ctx.cell_size.width > 0) ctx.cell_size.width else 10; + const height_px = @as(u32, target_rows) * cell_h; + const scale_f = @as(f64, @floatFromInt(height_px)) / @as(f64, @floatFromInt(self.image.height)); + const width_px: u32 = @intFromFloat(@as(f64, @floatFromInt(self.image.width)) * scale_f); + const cols: u16 = @intCast((width_px + cell_w - 1) / cell_w); + + const size: vxfw.Size = .{ .width = cols, .height = target_rows }; + var surf = try vxfw.Surface.init(ctx.arena, ui.widget(self), size); + surf.writeCell(0, 0, .{ + .image = .{ + .img_id = self.image.id, + .options = .{ + .size = .{ .rows = target_rows }, + }, + }, + }); + return surf; +} + +test Image { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + vxfw.DrawContext.init(.unicode); + + const img: Image = .{ + .image = .{ .id = 1, .width = 100, .height = 50 }, + .rows = 5, + }; + const w = ui.widget(&img); + + const draw_ctx: vxfw.DrawContext = .{ + .arena = arena.allocator(), + .min = .{}, + .max = .{ .width = 40, .height = 10 }, + .cell_size = .{ .width = 10, .height = 20 }, + }; + + const surface = try w.draw(draw_ctx); + try std.testing.expectEqual(@as(u16, 5), surface.size.height); + // 5 rows * 20px = 100px height. scale = 100/50 = 2.0. width = 100*2 = 200px. cols = 200/10 = 20 + try std.testing.expectEqual(@as(u16, 20), surface.size.width); + // Image cell should be at (0, 0) + const cell = surface.buffer[0]; + try std.testing.expect(cell.image != null); + try std.testing.expectEqual(@as(u32, 1), cell.image.?.img_id); +} + +test "refAllDecls" { + std.testing.refAllDecls(@This()); +} diff --git a/bin/zml-smi/tui/lib/ui.zig b/bin/zml-smi/tui/lib/ui.zig new file mode 100644 index 00000000..6008d1e0 --- /dev/null +++ b/bin/zml-smi/tui/lib/ui.zig @@ -0,0 +1,87 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; + +/// Creates a vxfw.Widget from any pointer whose type has a `draw` method +/// (and optionally a `handleEvent` method). +pub fn widget(ptr: anytype) vxfw.Widget { + const Ptr = @TypeOf(ptr); + const T = @typeInfo(Ptr).pointer.child; + return .{ + .userdata = @ptrCast(@constCast(ptr)), + .drawFn = struct { + fn f(p: *anyopaque, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const self: *T = @ptrCast(@alignCast(p)); + return self.draw(ctx); + } + }.f, + .eventHandler = if (@hasDecl(T, "handleEvent")) struct { + fn f(p: *anyopaque, ctx: *vxfw.EventContext, event: vxfw.Event) anyerror!void { + const self: *T = @ptrCast(@alignCast(p)); + return self.handleEvent(ctx, event); + } + }.f else null, + }; +} + +/// Creates a vxfw.Widget that calls a specific draw function (for non-standard widget patterns +/// like MetricCard.drawContent or Chart.drawContent). +pub fn drawWidget(ptr: anytype, comptime drawFn: anytype) vxfw.Widget { + const Ptr = @TypeOf(ptr); + const T = @typeInfo(Ptr).pointer.child; + return .{ + .userdata = @ptrCast(@constCast(ptr)), + .drawFn = struct { + fn f(p: *anyopaque, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const self: *T = @ptrCast(@alignCast(p)); + return @call(.auto, drawFn, .{ self, ctx }); + } + }.f, + }; +} + +// ── DrawContext constraint helpers ────────────────────────────────── + +/// min = max = {w, h} — exact size. +pub fn fixedSize(ctx: vxfw.DrawContext, w: u16, h: u16) vxfw.DrawContext { + return ctx.withConstraints(.{ .width = w, .height = h }, .{ .width = w, .height = h }); +} + +/// Width fixed (min = max = w), height unbounded. +pub fn fixedWidth(ctx: vxfw.DrawContext, w: u16) vxfw.DrawContext { + return ctx.withConstraints(.{ .width = w }, .{ .width = w, .height = null }); +} + +/// max = {w, h}, min = default — bounded but unconstrained from below. +pub fn maxSize(ctx: vxfw.DrawContext, w: u16, h: u16) vxfw.DrawContext { + return ctx.withConstraints(.{}, .{ .width = w, .height = h }); +} + +// ── Drawing helpers ──────────────────────────────────────────────── + +/// Draw a single-line RichText with softwrap=false, overflow=clip, max width. +pub fn drawRichLine(ctx: vxfw.DrawContext, segments: []const vaxis.Cell.Segment, max_w: u16) std.mem.Allocator.Error!vxfw.Surface { + const rich: vxfw.RichText = .{ + .text = segments, + .softwrap = false, + .overflow = .clip, + }; + return rich.draw(ctx.withConstraints(.{}, .{ .width = max_w, .height = 1 })); +} + +pub const ImageCellSize = struct { + cols: u16, + rows: u16, +}; + +/// Compute the cell dimensions for an image scaled to a target row count, +/// maintaining aspect ratio based on the terminal's cell pixel size. +pub fn imageCellSize(img: vaxis.Image, target_rows: u16, cell_size: vxfw.Size) ImageCellSize { + const cell_h: u32 = if (cell_size.height > 0) cell_size.height else 20; + const cell_w: u32 = if (cell_size.width > 0) cell_size.width else 10; + const height_px = @as(u32, target_rows) * cell_h; + const scale_f = @as(f64, @floatFromInt(height_px)) / @as(f64, @floatFromInt(img.height)); + const width_px: u32 = @intFromFloat(@as(f64, @floatFromInt(img.width)) * scale_f); + const img_cols: u16 = @intCast((width_px + cell_w - 1) / cell_w); + return .{ .cols = img_cols, .rows = target_rows }; +} diff --git a/bin/zml-smi/tui/lib/utils.zig b/bin/zml-smi/tui/lib/utils.zig new file mode 100644 index 00000000..993703c9 --- /dev/null +++ b/bin/zml-smi/tui/lib/utils.zig @@ -0,0 +1,30 @@ +const std = @import("std"); + +pub fn bytesToMb(val: ?u64) u64 { + return if (val) |v| v / (1024 * 1024) else 0; +} + +pub fn repeatStr(arena: std.mem.Allocator, char: []const u8, count: u16) std.mem.Allocator.Error![]const u8 { + const buf = try arena.alloc(u8, char.len * count); + for (0..count) |i| { + @memcpy(buf[i * char.len ..][0..char.len], char); + } + return buf; +} + +/// Maps values from [min_val, max_val] to [0, 100], clamped. +pub fn normalizeRange(arena: std.mem.Allocator, raw: []const u64, min_val: u64, max_val: u64) std.mem.Allocator.Error![]u8 { + const result = try arena.alloc(u8, raw.len); + const range = max_val -| min_val; + for (raw, 0..) |v, i| { + result[i] = if (v <= min_val) + 0 + else if (v >= max_val) + 100 + else if (range > 0) + @intCast((v - min_val) * 100 / range) + else + 0; + } + return result; +} diff --git a/bin/zml-smi/tui/pages/detail.zig b/bin/zml-smi/tui/pages/detail.zig new file mode 100644 index 00000000..a17dc17b --- /dev/null +++ b/bin/zml-smi/tui/pages/detail.zig @@ -0,0 +1,28 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("../data.zig"); +const ui = @import("../lib/ui.zig"); +const ProcessTable = @import("../widgets/process_table.zig"); +const gpu_detail = @import("detail/gpu_detail.zig"); +const neuron_detail = @import("detail/neuron_detail.zig"); +const tpu_detail = @import("detail/tpu_detail.zig"); + +const Detail = @This(); + +state: *const data.SystemState, +device_id: u8, +process_table: *ProcessTable = undefined, + +pub fn draw(self: *const Detail, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const w = ctx.max.width orelse 80; + const id = self.device_id; + const dev = self.state.devices[id]; + const wgt = ui.widget(self); + + return switch (dev.*) { + .cuda, .rocm => |*sv| try gpu_detail.draw(self.state, self.process_table, ctx, w, id, sv.front().*, wgt), + .neuron => |*sv| try neuron_detail.draw(self.state, self.process_table, ctx, w, id, sv.front().*, wgt), + .tpu => |*sv| try tpu_detail.draw(self.state, self.process_table, ctx, w, id, sv.front().*, wgt), + }; +} diff --git a/bin/zml-smi/tui/pages/detail/detail_common.zig b/bin/zml-smi/tui/pages/detail/detail_common.zig new file mode 100644 index 00000000..e499fcae --- /dev/null +++ b/bin/zml-smi/tui/pages/detail/detail_common.zig @@ -0,0 +1,55 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("../../data.zig"); +const theme = @import("../../theme.zig"); +const utils = @import("../../lib/utils.zig"); +const ui = @import("../../lib/ui.zig"); +const compose = @import("../../lib/compose.zig"); +const Chart = @import("../../widgets/chart.zig"); +const ColumnLayout = @import("../../widgets/column_layout.zig"); + +pub fn headerText(arena: std.mem.Allocator, id: u8, name: []const u8) std.mem.Allocator.Error!vxfw.Text { + return .{ + .text = try std.fmt.allocPrint(arena, "Device {d}: {s}", .{ id, name }), + .style = theme.header_style, + .softwrap = false, + .overflow = .clip, + }; +} + +/// Creates a Chart pre-filled with history data and standard defaults. +pub fn historyChart( + ctx: vxfw.DrawContext, + state: *const data.SystemState, + id: u8, + history: anytype, + opts: struct { + title: []const u8, + value_label: []const u8, + info_line: ?[]const u8 = null, + y_min: u32 = 0, + y_max: u32 = 100, + y_unit: []const u8 = "%", + }, +) std.mem.Allocator.Error!Chart { + return .{ + .title = opts.title, + .data = try utils.normalizeRange(ctx.arena, try history[id].sliceLast(ctx.arena, data.history_len), opts.y_min, opts.y_max), + .value_label = opts.value_label, + .info_line = opts.info_line, + .y_min = opts.y_min, + .y_max = opts.y_max, + .y_unit = opts.y_unit, + .chart_height = 8, + .tui_refresh_rate = state.tui_refresh_rate, + }; +} + +/// Draws a page-level layout with standard margin (top=1, left/right=2) inside a full-width surface. +pub fn pageFrame(ctx: vxfw.DrawContext, layout: ColumnLayout, w: u16, parent_widget: vxfw.Widget) std.mem.Allocator.Error!vxfw.Surface { + const padded = try compose.pad(ctx.arena, ui.widget(&layout), .{ .top = 1, .left = 2, .right = 2 }); + var surf = try padded.draw(ui.fixedWidth(ctx, w)); + surf.widget = parent_widget; + return surf; +} diff --git a/bin/zml-smi/tui/pages/detail/gpu_detail.zig b/bin/zml-smi/tui/pages/detail/gpu_detail.zig new file mode 100644 index 00000000..6d440b4a --- /dev/null +++ b/bin/zml-smi/tui/pages/detail/gpu_detail.zig @@ -0,0 +1,144 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("../../data.zig"); +const utils = @import("../../lib/utils.zig"); +const ui = @import("../../lib/ui.zig"); +const Chart = @import("../../widgets/chart.zig"); +const ColumnLayout = @import("../../widgets/column_layout.zig"); +const MetricCard = @import("../../widgets/metric_card.zig"); +const ProcessTable = @import("../../widgets/process_table.zig"); +const common = @import("detail_common.zig"); + +pub fn draw( + state: *const data.SystemState, + process_table: *ProcessTable, + ctx: vxfw.DrawContext, + w: u16, + id: u8, + gpu: data.GpuInfo, + parent_widget: vxfw.Widget, +) std.mem.Allocator.Error!vxfw.Surface { + // ── Header ────────────────────────────────────────────── + const header = try common.headerText(ctx.arena, id, gpu.name orelse "Unknown"); + + // ── GPU Utilization + Memory charts ───────────────────── + const util_chart = try common.historyChart(ctx, state, id, state.history.util, .{ + .title = "GPU Utilization", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{gpu.util_percent orelse 0}), + .info_line = if (gpu.encoder_util_percent != null or gpu.decoder_util_percent != null) + try std.fmt.allocPrint(ctx.arena, "Enc {d}% Dec {d}%", .{ + gpu.encoder_util_percent orelse 0, gpu.decoder_util_percent orelse 0, + }) + else + null, + }); + const mem_chart = try common.historyChart(ctx, state, id, state.history.mem_util, .{ + .title = "Memory", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{if (gpu.mem_total_bytes) |total| if (total > 0) (gpu.mem_used_bytes orelse 0) * 100 / total else 0 else 0}), + .info_line = if (gpu.mem_bus_width) |bw| + try std.fmt.allocPrint(ctx.arena, "VRAM {d} / {d} MB ({d}-bit)", .{ + utils.bytesToMb(gpu.mem_used_bytes), utils.bytesToMb(gpu.mem_total_bytes), bw, + }) + else + try std.fmt.allocPrint(ctx.arena, "VRAM {d} / {d} MB", .{ + utils.bytesToMb(gpu.mem_used_bytes), utils.bytesToMb(gpu.mem_total_bytes), + }), + }); + + // ── Temperature + Power charts ────────────────────────── + const temp_chart = try common.historyChart(ctx, state, id, state.history.temp, .{ + .title = "Temperature", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}\u{00b0}C", .{gpu.temperature orelse 0}), + .info_line = try std.fmt.allocPrint(ctx.arena, "Fan {d}%", .{gpu.fan_speed_percent orelse 0}), + .y_min = 20, + .y_max = 100, + .y_unit = "\u{00b0}C", + }); + + const power_limit_mw = gpu.power_limit_mw orelse 0; + const power_limit_w = power_limit_mw / 1000; + const power_raw = try state.history.power[id].sliceLast(ctx.arena, data.history_len); + const power_watts = try ctx.arena.alloc(u64, power_raw.len); + for (power_raw, 0..) |p, j| { + power_watts[j] = p / 1000; + } + const y_max = if (power_limit_w > 0) power_limit_w else 1; + const power_chart: Chart = .{ + .title = "Power", + .data = try utils.normalizeRange(ctx.arena, power_watts, 0, y_max), + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}W", .{(gpu.power_mw orelse 0) / 1000}), + .info_line = if (power_limit_w > 0) + try std.fmt.allocPrint(ctx.arena, "Limit {d}W", .{power_limit_w}) + else + "N/A", + .y_min = 0, + .y_max = @intCast(@min(y_max, std.math.maxInt(u32))), + .y_unit = "W", + .chart_height = 8, + .tui_refresh_rate = state.tui_refresh_rate, + }; + + const charts_flow: ColumnLayout = .{ + .children = &.{ ui.widget(&util_chart), ui.widget(&mem_chart), ui.widget(&temp_chart), ui.widget(&power_chart) }, + .min_child_width = 60, + .gap = 1, + }; + + // ── Clocks + PCIe ─────────────────────────────────────── + var clock_lines: std.ArrayList(MetricCard.LineEntry) = .empty; + try clock_lines.append(ctx.arena, .{ .label = "Graphics ", .value = try std.fmt.allocPrint(ctx.arena, " {d} / {d} MHz", .{ + gpu.clock_graphics_mhz orelse 0, gpu.clock_graphics_max_mhz orelse 0, + }) }); + if (gpu.clock_sm_mhz) |sm| { + try clock_lines.append(ctx.arena, .{ .label = "SM ", .value = try std.fmt.allocPrint(ctx.arena, " {d} MHz", .{sm}) }); + } + if (gpu.clock_soc_mhz) |soc| { + try clock_lines.append(ctx.arena, .{ .label = "SOC ", .value = try std.fmt.allocPrint(ctx.arena, " {d} MHz", .{soc}) }); + } + try clock_lines.append(ctx.arena, .{ .label = "Memory ", .value = try std.fmt.allocPrint(ctx.arena, " {d} / {d} MHz", .{ + gpu.clock_mem_mhz orelse 0, gpu.clock_mem_max_mhz orelse 0, + }) }); + const clocks_card: MetricCard = .{ .title = "Clocks", .lines = clock_lines.items }; + + var pcie_lines: std.ArrayList(MetricCard.LineEntry) = .empty; + try pcie_lines.append(ctx.arena, .{ .label = "Link ", .value = try std.fmt.allocPrint(ctx.arena, " Gen{d} x{d}", .{ + gpu.pcie_link_gen orelse 0, gpu.pcie_link_width orelse 0, + }) }); + if (gpu.pcie_tx_kbps != null or gpu.pcie_rx_kbps != null) { + try pcie_lines.append(ctx.arena, .{ .label = "TX ", .value = try formatBandwidth(ctx.arena, gpu.pcie_tx_kbps orelse 0) }); + try pcie_lines.append(ctx.arena, .{ .label = "RX ", .value = try formatBandwidth(ctx.arena, gpu.pcie_rx_kbps orelse 0) }); + } + if (gpu.pcie_bandwidth_mbps) |bw| { + try pcie_lines.append(ctx.arena, .{ .label = "BW ", .value = try std.fmt.allocPrint(ctx.arena, " {d} Mb/s", .{bw}) }); + } + const pcie_card: MetricCard = .{ .title = "PCIe", .lines = pcie_lines.items }; + + const cards_flow: ColumnLayout = .{ + .children = &.{ ui.widget(&clocks_card), ui.widget(&pcie_card) }, + .min_child_width = 30, + }; + + // ── Compose vertical layout ───────────────────────────── + return common.pageFrame(ctx, .{ + .children = &.{ + header.widget(), + ui.widget(&charts_flow), + ui.widget(&cards_flow), + ui.widget(process_table), + }, + .gap = 1, + }, w, parent_widget); +} + +fn formatBandwidth(arena: std.mem.Allocator, kbps: u64) std.mem.Allocator.Error![]const u8 { + if (kbps >= 1_000_000) { + const gbps = @as(f64, @floatFromInt(kbps)) / 1_000_000.0; + return std.fmt.allocPrint(arena, " {d:.1} GB/s", .{gbps}); + } else if (kbps >= 1_000) { + const mbps = @as(f64, @floatFromInt(kbps)) / 1_000.0; + return std.fmt.allocPrint(arena, " {d:.1} MB/s", .{mbps}); + } else { + return std.fmt.allocPrint(arena, " {d} KB/s", .{kbps}); + } +} diff --git a/bin/zml-smi/tui/pages/detail/neuron_detail.zig b/bin/zml-smi/tui/pages/detail/neuron_detail.zig new file mode 100644 index 00000000..2629406f --- /dev/null +++ b/bin/zml-smi/tui/pages/detail/neuron_detail.zig @@ -0,0 +1,79 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("../../data.zig"); +const utils = @import("../../lib/utils.zig"); +const ui = @import("../../lib/ui.zig"); +const ColumnLayout = @import("../../widgets/column_layout.zig"); +const MetricCard = @import("../../widgets/metric_card.zig"); +const ProcessTable = @import("../../widgets/process_table.zig"); +const common = @import("detail_common.zig"); + +pub fn draw( + state: *const data.SystemState, + process_table: *ProcessTable, + ctx: vxfw.DrawContext, + w: u16, + id: u8, + nc: data.NeuronInfo, + parent_widget: vxfw.Widget, +) std.mem.Allocator.Error!vxfw.Surface { + // ── Header ────────────────────────────────────────────── + const header = try common.headerText(ctx.arena, id, nc.name orelse "Unknown"); + + // ── Utilization + Memory charts ───────────────────────── + const util_chart = try common.historyChart(ctx, state, id, state.history.util, .{ + .title = "Core Utilization", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{nc.util_percent orelse 0}), + }); + + const mem_used = nc.mem_used_bytes orelse 0; + const mem_total = nc.mem_total_bytes orelse 0; + const mem_chart = try common.historyChart(ctx, state, id, state.history.mem_util, .{ + .title = "Memory", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{if (mem_total > 0) mem_used * 100 / mem_total else 0}), + .info_line = try std.fmt.allocPrint(ctx.arena, "HBM {d} / {d} MB", .{ + utils.bytesToMb(nc.mem_used_bytes), utils.bytesToMb(nc.mem_total_bytes), + }), + }); + + const charts_flow: ColumnLayout = .{ + .children = &.{ ui.widget(&util_chart), ui.widget(&mem_chart) }, + .min_child_width = 40, + .gap = 1, + }; + + // ── HBM Breakdown ─────────────────────────────────────── + const fmt_mb = struct { + fn f(arena: std.mem.Allocator, val: ?u64) std.mem.Allocator.Error![]const u8 { + return std.fmt.allocPrint(arena, " {d} MB", .{utils.bytesToMb(val)}); + } + }.f; + const hbm_card: MetricCard = .{ + .title = "HBM Breakdown", + .lines = &.{ + .{ .label = "Tensors ", .value = try fmt_mb(ctx.arena, nc.nc_tensors) }, + .{ .label = "Constants ", .value = try fmt_mb(ctx.arena, nc.nc_constants) }, + .{ .label = "Model Code ", .value = try fmt_mb(ctx.arena, nc.nc_model_code) }, + .{ .label = "Shared Scratch", .value = try fmt_mb(ctx.arena, nc.nc_shared_scratchpad) }, + .{ .label = "Nonshared ", .value = try fmt_mb(ctx.arena, nc.nc_nonshared_scratchpad) }, + .{ .label = "Runtime ", .value = try fmt_mb(ctx.arena, nc.nc_runtime) }, + .{ .label = "Driver ", .value = try fmt_mb(ctx.arena, nc.nc_driver) }, + .{ .label = "DMA Rings ", .value = try fmt_mb(ctx.arena, nc.nc_dma_rings) }, + .{ .label = "Collectives ", .value = try fmt_mb(ctx.arena, nc.nc_collectives) }, + .{ .label = "Notifications ", .value = try fmt_mb(ctx.arena, nc.nc_notifications) }, + .{ .label = "Uncategorized ", .value = try fmt_mb(ctx.arena, nc.nc_uncategorized) }, + }, + }; + + // ── Compose vertical layout ───────────────────────────── + return common.pageFrame(ctx, .{ + .children = &.{ + header.widget(), + ui.widget(&charts_flow), + ui.widget(&hbm_card), + ui.widget(process_table), + }, + .gap = 1, + }, w, parent_widget); +} diff --git a/bin/zml-smi/tui/pages/detail/tpu_detail.zig b/bin/zml-smi/tui/pages/detail/tpu_detail.zig new file mode 100644 index 00000000..7d0fac12 --- /dev/null +++ b/bin/zml-smi/tui/pages/detail/tpu_detail.zig @@ -0,0 +1,54 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("../../data.zig"); +const utils = @import("../../lib/utils.zig"); +const ui = @import("../../lib/ui.zig"); +const ColumnLayout = @import("../../widgets/column_layout.zig"); +const ProcessTable = @import("../../widgets/process_table.zig"); +const common = @import("detail_common.zig"); + +pub fn draw( + state: *const data.SystemState, + process_table: *ProcessTable, + ctx: vxfw.DrawContext, + w: u16, + id: u8, + tp: data.TpuInfo, + parent_widget: vxfw.Widget, +) std.mem.Allocator.Error!vxfw.Surface { + // ── Header ────────────────────────────────────────────── + const header = try common.headerText(ctx.arena, id, tp.name orelse "Unknown"); + + // ── Utilization + Memory charts ───────────────────────── + const util_chart = try common.historyChart(ctx, state, id, state.history.util, .{ + .title = "Duty Cycle", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{tp.util_percent orelse 0}), + }); + + const mem_used = tp.mem_used_bytes orelse 0; + const mem_total = tp.mem_total_bytes orelse 0; + const mem_chart = try common.historyChart(ctx, state, id, state.history.mem_util, .{ + .title = "Memory", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{if (mem_total > 0) mem_used * 100 / mem_total else 0}), + .info_line = try std.fmt.allocPrint(ctx.arena, "HBM {d} / {d} MB", .{ + utils.bytesToMb(tp.mem_used_bytes), utils.bytesToMb(tp.mem_total_bytes), + }), + }); + + const charts_flow: ColumnLayout = .{ + .children = &.{ ui.widget(&util_chart), ui.widget(&mem_chart) }, + .min_child_width = 40, + .gap = 1, + }; + + // ── Compose vertical layout ───────────────────────────── + return common.pageFrame(ctx, .{ + .children = &.{ + header.widget(), + ui.widget(&charts_flow), + ui.widget(process_table), + }, + .gap = 1, + }, w, parent_widget); +} diff --git a/bin/zml-smi/tui/pages/overview.zig b/bin/zml-smi/tui/pages/overview.zig new file mode 100644 index 00000000..d1ed28a0 --- /dev/null +++ b/bin/zml-smi/tui/pages/overview.zig @@ -0,0 +1,121 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("../data.zig"); +const ui = @import("../lib/ui.zig"); +const compose = @import("../lib/compose.zig"); +const image_cache = @import("../image_cache.zig"); +const ColumnLayout = @import("../widgets/column_layout.zig"); +const common = @import("detail/detail_common.zig"); +const DeviceCard = @import("../widgets/device_card.zig"); +const Logo = @import("../widgets/logo.zig"); +const InfoLines = @import("../widgets/info_lines.zig"); +const ProcessTable = @import("../widgets/process_table.zig"); + +const Overview = @This(); + +const two_col_breakpoint: u16 = 120; +const narrow_breakpoint: u16 = 80; +const max_info_width: u16 = 50; +/// Total width of the wide banner: logo box (incl. left margin) + info lines + right margin. +pub const host_line_width: u16 = (Logo.logo_width + 6) + max_info_width + 2; + +state: *const data.SystemState, +device_cards: []DeviceCard = &.{}, +process_table: ?*ProcessTable = null, // maybe null if we decide not to print the process table in print mode +viewing_device: *?u8 = undefined, +use_braille: bool = false, + +pub fn init(allocator: std.mem.Allocator, state: *const data.SystemState, process_table: ?*ProcessTable, viewing_device: *?u8) !Overview { + const count = state.deviceCount(); + const cards = try allocator.alloc(DeviceCard, count); + for (cards, 0..) |*card, i| { + card.* = .{ + .device_id = @intCast(i), + .state = state, + }; + } + return .{ + .state = state, + .device_cards = cards, + .process_table = process_table, + .viewing_device = viewing_device, + }; +} + +pub fn deinit(self: *Overview, allocator: std.mem.Allocator) void { + allocator.free(self.device_cards); +} + +fn drawNarrowBanner(self: *const Overview, ctx: vxfw.DrawContext, content_w: u16) !vxfw.Surface { + const logo: Logo = .{ .image = image_cache.global.get("logo"), .compact = true }; + const info_lines: InfoLines = .{ .state = self.state }; + + const logo_h: u16 = if (logo.image != null) Logo.compact_image_height else Logo.compact_height; + const children = [2]vxfw.Widget{ + try compose.sized(ctx.arena, try compose.center(ctx.arena, ui.widget(&logo)), .{ .width = content_w, .height = logo_h }), + ui.widget(&info_lines), + }; + const layout: ColumnLayout = .{ .children = &children, .gap = 1 }; + return ui.widget(&layout).draw(ui.fixedWidth(ctx, content_w)); +} + +fn drawWideBanner(self: *const Overview, ctx: vxfw.DrawContext, content_w: u16) !vxfw.Surface { + const logo: Logo = .{ .image = image_cache.global.get("logo"), .compact = false }; + const info_lines: InfoLines = .{ .state = self.state }; + + const logo_h: u16 = if (logo.image != null) Logo.image_height else Logo.logo_height; + const logo_box_w = Logo.logo_width + 6; // +4 centering + 2 absorbed page margin + const info_max_w = @min(content_w -| logo_box_w, max_info_width); + const banner_h = @max(logo_h, InfoLines.entry_count); + + const flex_items = [2]vxfw.FlexItem{ + .{ .widget = try compose.sized(ctx.arena, try compose.center(ctx.arena, ui.widget(&logo)), .{ .width = logo_box_w, .height = banner_h }), .flex = 0 }, + .{ .widget = try compose.sized(ctx.arena, try compose.center(ctx.arena, ui.widget(&info_lines)), .{ .width = info_max_w, .height = banner_h }), .flex = 0 }, + }; + const sized_banner = try compose.sized(ctx.arena, (vxfw.FlexRow{ .children = &flex_items }).widget(), .{ .width = content_w, .height = banner_h }); + return sized_banner.draw(ui.fixedWidth(ctx, content_w)); +} + +pub fn draw(self: *Overview, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const w = ctx.max.width orelse 80; + const narrow = w < narrow_breakpoint; + const content_w: u16 = w -| 4; + const content_ctx = ui.fixedWidth(ctx, content_w); + + var sb = compose.surfaceBuilder(ctx.arena); + var row: i17 = 1; // top margin + + // ── Banner (Logo + Info Lines) ───────────────────────── + const banner_surf = try if (narrow) self.drawNarrowBanner(ctx, content_w) else self.drawWideBanner(ctx, content_w + 2); + try sb.add(row, if (narrow) @as(i17, 2) else 0, banner_surf); + row += @intCast(banner_surf.size.height + 1); + + // ── Device Grid ───────────────────────────────────────── + for (self.device_cards) |*card| { + card.use_braille = self.use_braille; + card.viewing_device = self.viewing_device; + } + const card_widgets = try ctx.arena.alloc(vxfw.Widget, self.device_cards.len); + for (self.device_cards, card_widgets) |*card, *cw| { + cw.* = ui.widget(card); + } + const device_grid: ColumnLayout = .{ + .children = card_widgets, + .min_child_width = two_col_breakpoint / 2, + .col_gap = 2, + .gap = 1, + }; + const grid_surf = try ui.widget(&device_grid).draw(content_ctx); + try sb.add(row, 2, grid_surf); + row += @intCast(grid_surf.size.height + 1); + + // ── Process Table ──────────────────────────────────────── + if (self.process_table) |pt| { + const pt_surf = try ui.widget(pt).draw(content_ctx); + try sb.add(row, 2, pt_surf); + row += @intCast(pt_surf.size.height); + } + + return sb.finish(.{ .width = w, .height = @intCast(row) }, ui.widget(self)); +} diff --git a/bin/zml-smi/tui/print.zig b/bin/zml-smi/tui/print.zig new file mode 100644 index 00000000..c2e5c562 --- /dev/null +++ b/bin/zml-smi/tui/print.zig @@ -0,0 +1,95 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("data.zig"); +const ui = @import("lib/ui.zig"); +const image_cache = @import("image_cache.zig"); +const Overview = @import("pages/overview.zig"); + +fn silentResize(vx: *vaxis.Vaxis, allocator: std.mem.Allocator, winsize: vaxis.Winsize) !void { + vx.screen.deinit(allocator); + vx.screen = try vaxis.Screen.init(allocator, winsize); + vx.screen.width_method = vx.caps.unicode; + vx.screen_last.deinit(allocator); + vx.screen_last = try vaxis.AllocatingScreen.init(allocator, winsize.cols, winsize.rows); +} + +pub fn run(allocator: std.mem.Allocator, io: std.Io, state: *data.SystemState) !void { + var tty = try vaxis.Tty.init(io); + defer tty.deinit(); + + var vx = try vaxis.init(allocator, .{}); + defer { + // Reset terminal modes without clearing screen content. + vx.resetModes(tty.writer()) catch {}; + var buf: [256]u8 = undefined; + var discarding: std.Io.Writer.Discarding = .init(&buf); + vx.deinit(allocator, &discarding.writer); + } + + const ws = try vaxis.Tty.getWinsize(tty.fd); + try silentResize(&vx, allocator, ws); + + { + const EventLoop = vaxis.Loop(vxfw.Event); + var loop: EventLoop = .{ .tty = &tty, .vaxis = &vx, .io = io, .queue = .{ .io = io } }; + try loop.start(); + defer loop.stop(); + + // queryTerminal sends cursor home (\x1b[H) for capability detection. + // Save/restore cursor so we stay at the current position. + const writer = tty.writer(); + try writer.writeAll("\x1b7"); // DECSC: save cursor + try writer.flush(); + try vx.queryTerminal(writer, io, 1 * std.time.ns_per_s); + try writer.writeAll("\x1b8"); // DECRC: restore cursor + try writer.flush(); + } + + image_cache.global.loadAll(&vx, allocator, tty.writer()); + defer image_cache.global.deinit(allocator); + + vxfw.DrawContext.init(vx.screen.width_method); + + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + const content_w: u16 = @min(@as(u16, @intCast(vx.screen.width)), Overview.host_line_width + 2); + + // Neuron util% is delta-based; wait for 2 poll cycles to get a real sample. + if (state.targets.contains(.neuron)) { + io.sleep(.fromSeconds(2), .awake) catch {}; + } + + var viewing_device: ?u8 = null; + var overview = try Overview.init(allocator, state, null, &viewing_device); + defer overview.deinit(allocator); + + const draw_ctx: vxfw.DrawContext = .{ + .arena = arena.allocator(), + .min = .{ .width = content_w }, + .max = .{ .width = content_w, .height = null }, + .cell_size = .{ + .width = if (vx.screen.width > 0) vx.screen.width_pix / vx.screen.width else 0, + .height = if (vx.screen.height > 0) vx.screen.height_pix / vx.screen.height else 0, + }, + }; + + const surface = try overview.draw(draw_ctx); + + try silentResize(&vx, allocator, .{ + .rows = surface.size.height, + .cols = content_w, + .x_pixel = ws.x_pixel, + .y_pixel = ws.y_pixel, + }); + + const win = vx.window(); + const root_win = win.child(.{ + .width = surface.size.width, + .height = surface.size.height, + }); + surface.render(root_win, ui.widget(&overview)); + + try vx.prettyPrint(tty.writer()); +} diff --git a/bin/zml-smi/tui/theme.zig b/bin/zml-smi/tui/theme.zig new file mode 100644 index 00000000..e037f67f --- /dev/null +++ b/bin/zml-smi/tui/theme.zig @@ -0,0 +1,50 @@ +const vaxis = @import("vaxis"); + +const Color = vaxis.Cell.Color; +const Style = vaxis.Cell.Style; + +/// Returns a color on a green → yellow → red gradient based on percentage (0-100). +pub fn colorForPercent(pct: u8) Color { + if (pct <= 50) { + // Green (#00cc66) → Yellow (#ffcc00) + const t: u16 = @as(u16, pct) * 255 / 50; + return .{ .rgb = .{ @intCast(t), 204, @intCast(102 -| (t * 102 / 255)) } }; + } else if (pct <= 80) { + // Yellow (#ffcc00) → Orange (#ff6600) + const t: u16 = (@as(u16, pct) - 50) * 255 / 30; + return .{ .rgb = .{ 255, @intCast(204 -| (t * 102 / 255)), 0 } }; + } else { + // Orange (#ff6600) → Red (#ff2200) + const t: u16 = (@as(u16, pct) - 80) * 255 / 20; + return .{ .rgb = .{ 255, @intCast(102 -| (t * 68 / 255)), 0 } }; + } +} + +/// Returns a color for temperature (20-100°C range). +pub fn colorForTemp(temp_c: u16) Color { + if (temp_c <= 20) { + return colorForPercent(0); + } + if (temp_c >= 100) { + return colorForPercent(100); + } + const pct: u8 = @intCast((@as(u32, temp_c) - 20) * 100 / 80); + return colorForPercent(pct); +} + +// ── Style Constants ────────────────────────────────────────────── + +pub const accent: Color = .{ .rgb = .{ 120, 180, 255 } }; +pub const accent_secondary: Color = .{ .rgb = .{ 200, 160, 255 } }; +pub const dim: Color = .{ .rgb = .{ 100, 100, 120 } }; +pub const text_primary: Color = .{ .rgb = .{ 230, 230, 240 } }; +pub const text_secondary: Color = .{ .rgb = .{ 160, 160, 175 } }; +pub const surface_border: Color = .{ .rgb = .{ 70, 70, 90 } }; +pub const gauge_empty: Color = .{ .rgb = .{ 50, 50, 65 } }; + +pub const header_style: Style = .{ .bold = true, .fg = accent }; +pub const title_style: Style = .{ .bold = true, .fg = accent_secondary }; +pub const label_style: Style = .{ .fg = text_secondary }; +pub const value_style: Style = .{ .bold = true, .fg = text_primary }; +pub const border_style: Style = .{ .fg = surface_border }; +pub const dim_style: Style = .{ .fg = dim }; diff --git a/bin/zml-smi/tui/tui.zig b/bin/zml-smi/tui/tui.zig new file mode 100644 index 00000000..6e00abf5 --- /dev/null +++ b/bin/zml-smi/tui/tui.zig @@ -0,0 +1,166 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; + +const data = @import("data.zig"); +const ui = @import("lib/ui.zig"); +const compose = @import("lib/compose.zig"); +const image_cache = @import("image_cache.zig"); +const Logo = @import("widgets/logo.zig"); +const ProcessTable = @import("widgets/process_table.zig"); +const Overview = @import("pages/overview.zig"); +const Detail = @import("pages/detail.zig"); +const StatusLine = @import("widgets/status_line.zig"); + +const min_width: u16 = 45; +const min_height: u16 = 24; + +const ScrollState = struct { + row: i17 = 0, + col: i17 = 0, + + pub fn reset(self: *ScrollState) void { + self.row = 0; + self.col = 0; + } + + pub fn scrollBy(self: *ScrollState, dr: i17, dc: i17) void { + self.row = @max(0, self.row + dr); + self.col = @max(0, self.col + dc); + } + + pub fn clamp(self: *ScrollState, content_h: u16, viewport_h: u16, content_w: u16, viewport_w: u16) void { + const max_row: i17 = @max(0, @as(i17, content_h) - @as(i17, viewport_h)); + const max_col: i17 = @max(0, @as(i17, content_w) - @as(i17, viewport_w)); + self.row = std.math.clamp(self.row, 0, max_row); + self.col = std.math.clamp(self.col, 0, max_col); + } +}; + +const Model = struct { + allocator: std.mem.Allocator, + state: *data.SystemState, + viewing_device: ?u8 = null, + prev_viewing_device: ?u8 = null, + scroll: ScrollState = .{}, + vx: *vaxis.Vaxis, + tty: *vaxis.Tty, + process_table: ProcessTable = .{}, + overview: Overview = undefined, + + fn deinit(self: *Model) void { + self.overview.deinit(self.allocator); + image_cache.global.deinit(self.allocator); + } + + pub fn handleEvent(self: *Model, ctx: *vxfw.EventContext, event: vxfw.Event) anyerror!void { + switch (event) { + .init => { + self.overview = try Overview.init(self.allocator, self.state, &self.process_table, &self.viewing_device); + self.overview.use_braille = true; + image_cache.global.loadAll(self.vx, self.allocator, self.tty.writer()); + try ctx.tick(1000, ui.widget(self)); + }, + .tick => { + self.state.recordHistory(); + try ctx.tick(self.state.tui_refresh_rate, ui.widget(self)); + ctx.redraw = true; + }, + .key_press => |key| { + if (key.matches('q', .{}) or key.matches('c', .{ .ctrl = true })) { + ctx.quit = true; + return; + } + if (self.viewing_device != null) { + if (key.matches(vaxis.Key.escape, .{}) or key.matches(vaxis.Key.backspace, .{})) { + self.viewing_device = null; + return ctx.consumeAndRedraw(); + } + } else { + if (key.matches('v', .{})) { + self.overview.use_braille = !self.overview.use_braille; + return ctx.consumeAndRedraw(); + } + } + if (key.matches(vaxis.Key.down, .{})) { + self.scroll.scrollBy(1, 0); + } else if (key.matches(vaxis.Key.up, .{})) { + self.scroll.scrollBy(-1, 0); + } else if (key.matches(vaxis.Key.right, .{})) { + self.scroll.scrollBy(0, 1); + } else if (key.matches(vaxis.Key.left, .{})) { + self.scroll.scrollBy(0, -1); + } else return; + return ctx.consumeAndRedraw(); + }, + .mouse => |mouse| { + switch (mouse.button) { + .wheel_up => self.scroll.scrollBy(-3, 0), + .wheel_down => self.scroll.scrollBy(3, 0), + .wheel_left => self.scroll.scrollBy(0, 3), + .wheel_right => self.scroll.scrollBy(0, -3), + else => return, + } + return ctx.consumeAndRedraw(); + }, + else => {}, + } + } + + pub fn draw(self: *Model, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const screen = ctx.max.size(); + const content_w = @max(screen.width, min_width); + + if (self.viewing_device != self.prev_viewing_device) { + self.scroll.reset(); + self.process_table.resetScroll(); + self.prev_viewing_device = self.viewing_device; + } + + self.process_table.prepare(self.state, self.viewing_device); + + var content_surf: vxfw.Surface = undefined; + + if (self.viewing_device) |dev_id| { + const detail: Detail = .{ + .state = self.state, + .device_id = dev_id, + .process_table = &self.process_table, + }; + content_surf = try detail.draw(ui.fixedWidth(ctx, content_w)); + } else { + content_surf = try self.overview.draw(ui.fixedWidth(ctx, content_w)); + } + + content_surf.size.height = @max(content_surf.size.height, min_height); + + const content_h = screen.height -| 1; + self.scroll.clamp(content_surf.size.height, content_h, content_surf.size.width, screen.width); + + const status_line: StatusLine = .{ + .viewing_device = self.viewing_device, + .use_braille = self.overview.use_braille, + }; + const status_surf = try status_line.draw(ui.fixedSize(ctx, screen.width, 1)); + + var sb = compose.surfaceBuilder(ctx.arena); + try sb.add(-self.scroll.row, -self.scroll.col, content_surf); + try sb.addZ(@intCast(content_h), 0, status_surf, 1); + return sb.finish(screen, ui.widget(self)); + } +}; + +pub fn run(allocator: std.mem.Allocator, io: std.Io, state: *data.SystemState) !void { + var app = try vxfw.App.init(allocator, io); + defer app.deinit(); + + var model: Model = .{ + .allocator = allocator, + .state = state, + .vx = &app.vx, + .tty = &app.tty, + }; + defer model.deinit(); + + try app.run(ui.widget(&model), .{}); +} diff --git a/bin/zml-smi/tui/widgets/braille_chart.zig b/bin/zml-smi/tui/widgets/braille_chart.zig new file mode 100644 index 00000000..498739d1 --- /dev/null +++ b/bin/zml-smi/tui/widgets/braille_chart.zig @@ -0,0 +1,76 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const theme = @import("../theme.zig"); + +const ui = @import("../lib/ui.zig"); + +const BrailleChart = @This(); + +values: []const u8, // 0-100 +height: u16 = 2, + +pub const chart_bg: vaxis.Cell.Color = .{ .rgb = .{ 30, 30, 40 } }; + +/// Braille dot bit positions: [row_in_cell (0-3)][col_in_cell (0-1)] +/// Each braille character is a 2x4 dot grid encoded as U+2800 + pattern byte. +pub const dot_bits = [4][2]u8{ + .{ 0x01, 0x08 }, // row 0 + .{ 0x02, 0x10 }, // row 1 + .{ 0x04, 0x20 }, // row 2 + .{ 0x40, 0x80 }, // row 3 +}; + +pub fn draw(self: *const BrailleChart, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const w = ctx.max.width orelse 40; + var surface = try vxfw.Surface.init(ctx.arena, ui.widget(self), .{ .width = w, .height = self.height }); + try self.renderTo(ctx.arena, &surface, 0, 0, w); + return surface; +} + +/// Render braille data into an existing surface at the given offset. +pub fn renderTo(self: *const BrailleChart, arena: std.mem.Allocator, surface: *vxfw.Surface, col_offset: u16, row_offset: u16, w: u16) std.mem.Allocator.Error!void { + const h = self.height; + const dot_rows: u32 = @as(u32, h) * 4; + const dot_cols: u32 = @as(u32, w) * 2; + const data_len: u32 = @intCast(self.values.len); + + for (0..w) |cell_col| { + var max_val: u8 = 0; + for (0..2) |dx| { + const dot_x: u32 = @as(u32, @intCast(cell_col)) * 2 + @as(u32, @intCast(dx)); + if (dot_x + data_len >= dot_cols) { + max_val = @max(max_val, self.values[@intCast(dot_x + data_len - dot_cols)]); + } + } + const color = theme.colorForPercent(max_val); + + for (0..h) |cell_row| { + var pattern: u8 = 0; + for (0..2) |dx| { + const dot_x: u32 = @as(u32, @intCast(cell_col)) * 2 + @as(u32, @intCast(dx)); + if (dot_x + data_len >= dot_cols) { + const val = self.values[@intCast(dot_x + data_len - dot_cols)]; + const fill: u32 = @as(u32, val) * dot_rows / 100; + const threshold: u32 = dot_rows - fill; + for (0..4) |dr| { + const abs_row: u32 = @as(u32, @intCast(cell_row)) * 4 + @as(u32, @intCast(dr)); + if (abs_row >= threshold) { + pattern |= dot_bits[dr][dx]; + } + } + } + } + + const buf = try arena.alloc(u8, 3); + buf[0] = 0xE2; + buf[1] = 0xA0 + (pattern >> 6); + buf[2] = 0x80 + (pattern & 0x3F); + + surface.writeCell(col_offset + @as(u16, @intCast(cell_col)), row_offset + @as(u16, @intCast(cell_row)), .{ + .char = .{ .grapheme = buf, .width = 1 }, + .style = .{ .fg = color, .bg = chart_bg }, + }); + } + } +} diff --git a/bin/zml-smi/tui/widgets/chart.zig b/bin/zml-smi/tui/widgets/chart.zig new file mode 100644 index 00000000..5421cd1f --- /dev/null +++ b/bin/zml-smi/tui/widgets/chart.zig @@ -0,0 +1,141 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const theme = @import("../theme.zig"); +const ui = @import("../lib/ui.zig"); +const TitledBorder = @import("titled_border.zig"); +const BrailleChart = @import("braille_chart.zig"); + +const Chart = @This(); + +// Layout constants for rows below the chart data area +const x_axis_h: u16 = 1; // x-axis line +const time_labels_h: u16 = 1; // time labels below x-axis +const info_line_h: u16 = 1; // info text below time labels + +title: []const u8, +data: []const u8, // 0-100 normalized +value_label: ?[]const u8 = null, +info_line: ?[]const u8 = null, +y_min: u32 = 0, +y_max: u32 = 100, +y_unit: []const u8 = "%", +chart_height: u16 = 6, +tui_refresh_rate: u16, + +pub fn draw(self: *const Chart, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const tb: TitledBorder = .{ + .child = ui.drawWidget(self, drawContent), + .title = self.title, + .value_label = self.value_label, + }; + return tb.draw(ctx); +} + +fn drawContent(self: *const Chart, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const max_w = ctx.max.width orelse 40; + const content_h: u16 = self.chart_height + x_axis_h + time_labels_h + info_line_h; + + // ── Y-axis label widths ────────────────────────────────── + const y_labels = [3]struct { row: u16, val: u32 }{ + .{ .row = 0, .val = self.y_max }, + .{ .row = self.chart_height / 2, .val = (self.y_min + self.y_max) / 2 }, + .{ .row = self.chart_height, .val = self.y_min }, + }; + var y_strs: [3][]const u8 = undefined; + var y_str_widths: [3]u16 = undefined; + var y_label_w: u16 = 0; + for (&y_strs, &y_str_widths, y_labels) |*s, *sw, lbl| { + s.* = try std.fmt.allocPrint(ctx.arena, "{d}{s}", .{ lbl.val, self.y_unit }); + sw.* = @intCast(ctx.stringWidth(s.*)); + y_label_w = @max(y_label_w, sw.*); + } + + // ── Column layout ──────────────────────────────────────── + const chart_start: u16 = y_label_w + 1; + const chart_w: u16 = if (max_w > chart_start) max_w - chart_start else 0; + + var surface = try vxfw.Surface.init(ctx.arena, ui.drawWidget(self, drawContent), .{ .width = max_w, .height = content_h }); + if (chart_w == 0) { + return surface; + } + + // ── Y-axis labels + axis line ──────────────────────────── + for (y_strs, y_str_widths, y_labels) |text, sw, lbl| { + writeStr(&surface, y_label_w - sw, lbl.row, text, theme.dim_style); + } + for (0..self.chart_height + 1) |r| { + const row: u16 = @intCast(r); + const ch: []const u8 = if (row == self.chart_height) + "\u{2514}" + else + "\u{2502}"; + surface.writeCell(y_label_w, row, .{ + .char = .{ .grapheme = ch, .width = 1 }, + .style = theme.dim_style, + }); + } + + // ── Braille data area ──────────────────────────────────── + const braille: BrailleChart = .{ .values = self.data, .height = self.chart_height }; + try braille.renderTo(ctx.arena, &surface, chart_start, 0, chart_w); + + // ── X-axis line ────────────────────────────────────────── + const tick_mid: u16 = chart_w / 2; + const tick_right: u16 = chart_w -| 1; + for (0..chart_w) |c| { + const col: u16 = @intCast(c); + const is_tick = (col == 0 or col == tick_mid or col == tick_right); + surface.writeCell(chart_start + col, self.chart_height, .{ + .char = .{ .grapheme = if (is_tick) "\u{252C}" else "\u{2500}", .width = 1 }, + .style = theme.dim_style, + }); + } + + // ── X-axis time labels ─────────────────────────────────── + const total_ms: u32 = @as(u32, chart_w) * 2 * @as(u32, self.tui_refresh_rate); + const total_s: u16 = @intCast(std.math.divCeil(u32, total_ms, 1000) catch unreachable); + const mid_s: u16 = @intCast(std.math.divCeil(u32, total_ms / 2, 1000) catch unreachable); + const label_row = self.chart_height + x_axis_h; + const left_str = try fmtTime(ctx.arena, total_s); + const mid_str = try fmtTime(ctx.arena, mid_s); + + writeStr(&surface, chart_start, label_row, left_str, theme.dim_style); + + const mid_w: u16 = @intCast(ctx.stringWidth(mid_str)); + writeStr(&surface, chart_start + tick_mid -| (mid_w / 2), label_row, mid_str, theme.dim_style); + + const right_str: []const u8 = "now"; + const right_w: u16 = @intCast(ctx.stringWidth(right_str)); + writeStr(&surface, chart_start + tick_right + 1 -| right_w, label_row, right_str, theme.dim_style); + + // ── Info line ──────────────────────────────────────────── + if (self.info_line) |info| { + writeStr(&surface, chart_start, self.chart_height + x_axis_h + time_labels_h, info, theme.dim_style); + } + + return surface; +} + +fn writeStr(surface: *vxfw.Surface, col: u16, row: u16, text: []const u8, style: vaxis.Cell.Style) void { + var c = col; + var i: usize = 0; + while (i < text.len) { + const len = std.unicode.utf8ByteSequenceLength(text[i]) catch 1; + const end = @min(i + len, text.len); + surface.writeCell(c, row, .{ + .char = .{ .grapheme = text[i..end], .width = 1 }, + .style = style, + }); + c += 1; + i = end; + } +} + +fn fmtTime(arena: std.mem.Allocator, seconds: u16) std.mem.Allocator.Error![]const u8 { + if (seconds >= 120) { + return std.fmt.allocPrint(arena, "-{d}m", .{seconds / 60}); + } else { + return std.fmt.allocPrint(arena, "-{d}s", .{seconds}); + } +} diff --git a/bin/zml-smi/tui/widgets/column_layout.zig b/bin/zml-smi/tui/widgets/column_layout.zig new file mode 100644 index 00000000..f307d40c --- /dev/null +++ b/bin/zml-smi/tui/widgets/column_layout.zig @@ -0,0 +1,64 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const ui = @import("../lib/ui.zig"); +const compose = @import("../lib/compose.zig"); + +const ColumnLayout = @This(); + +children: []const vxfw.Widget, +gap: u16 = 0, +col_gap: u16 = 0, +/// When set, children wrap into a grid with this minimum column width. +/// When null (default), behaves as a simple vertical stack. +min_child_width: ?u16 = null, + +pub fn draw(self: *const ColumnLayout, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const n = self.children.len; + if (n == 0) { + return vxfw.Surface.init(ctx.arena, ui.widget(self), ctx.min); + } + + const available_w = ctx.max.width orelse 80; + const cols: u16 = if (self.min_child_width) |mcw| + @intCast(@max(1, @min(n, available_w / @max(mcw, 1)))) + else + 1; + + var sb = compose.surfaceBuilder(ctx.arena); + var row_y: u16 = 0; + var max_width: u16 = 0; + var i: usize = 0; + + while (i < n) { + if (i > 0) { + row_y += self.gap; + } + + const row_end = @min(i + cols, n); + var row_h: u16 = 0; + + for (i..row_end) |j| { + const col_idx: u16 = @intCast(j - i); + + const col_x, const w = if (cols > 1) blk: { + const total_col_gaps = @as(u16, cols - 1) * self.col_gap; + const usable_w = available_w -| total_col_gaps; + const child_w = usable_w / cols; + const x = col_idx * (child_w + self.col_gap); + break :blk .{ x, if (col_idx == cols - 1) available_w - x else child_w }; + } else .{ 0, available_w }; + + const surf = try self.children[j].draw(ui.fixedWidth(ctx, w)); + try sb.add(row_y, col_x, surf); + max_width = @max(max_width, col_x + surf.size.width); + row_h = @max(row_h, surf.size.height); + } + + row_y += row_h; + i = row_end; + } + + const result_w = if (ctx.max.width) |mw| @min(@max(max_width, ctx.min.width), mw) else @max(max_width, ctx.min.width); + return sb.finish(.{ .width = result_w, .height = row_y }, ui.widget(self)); +} diff --git a/bin/zml-smi/tui/widgets/device_card.zig b/bin/zml-smi/tui/widgets/device_card.zig new file mode 100644 index 00000000..f646840b --- /dev/null +++ b/bin/zml-smi/tui/widgets/device_card.zig @@ -0,0 +1,124 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("../data.zig"); +const theme = @import("../theme.zig"); +const utils = @import("../lib/utils.zig"); +const ui = @import("../lib/ui.zig"); +const image_cache = @import("../image_cache.zig"); +const MetricCard = @import("metric_card.zig"); + +const DeviceCard = @This(); + +device_id: u8 = 0, +highlighted: bool = false, + +state: *const data.SystemState = undefined, +use_braille: bool = false, +viewing_device: *?u8 = undefined, + +pub fn handleEvent(self: *DeviceCard, ctx: *vxfw.EventContext, event: vxfw.Event) anyerror!void { + switch (event) { + .mouse => |mouse| { + if (mouse.type == .press and mouse.button == .left) { + self.viewing_device.* = self.device_id; + return ctx.consumeAndRedraw(); + } + }, + .mouse_enter => { + self.highlighted = true; + try ctx.setMouseShape(.pointer); + return ctx.consumeAndRedraw(); + }, + .mouse_leave => { + self.highlighted = false; + try ctx.setMouseShape(.default); + ctx.redraw = true; + }, + else => {}, + } +} + +pub fn draw(self: *DeviceCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const i = self.device_id; + const dev = self.state.devices[i]; + const target: data.Target = std.meta.activeTag(dev.*); + const cf = switch (dev.*) { + inline else => |*sv| blk: { + const f = sv.front().*; + break :blk .{ + .name = f.name orelse "Unknown", + .util_percent = @as(u8, @intCast(@min(f.util_percent orelse 0, 100))), + .mem_used = f.mem_used_bytes orelse 0, + .mem_total = f.mem_total_bytes orelse 0, + }; + }, + }; + const mem_pct: u8 = if (cf.mem_total > 0) @intCast(@min(cf.mem_used * 100 / cf.mem_total, 100)) else 0; + + const card_title = try std.fmt.allocPrint(ctx.arena, "{s} {d}: {s}", .{ target.deviceLabel(), i, cf.name }); + const util_label = target.utilLabel(); + + const dev_suffix = switch (dev.*) { + .cuda, .rocm => |*sv| blk: { + const gpu = sv.front().*; + const power = (gpu.power_mw orelse 0) / 1000; + const power_limit = (gpu.power_limit_mw orelse 0) / 1000; + + const power_limit_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{power_limit}); + const power_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{power}); + + const pad = try utils.repeatStr(ctx.arena, " ", @intCast(power_limit_str.len -| power_str.len)); + break :blk try std.fmt.allocPrint(ctx.arena, "{d}\u{00b0}C | {s}{s} W", .{ + gpu.temperature orelse 0, pad, power_str, + }); + }, + else => "", + }; + + const mem_used_mb = utils.bytesToMb(cf.mem_used); + const mem_total_mb = utils.bytesToMb(cf.mem_total); + const mem_total_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{mem_total_mb}); + const mem_used_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{mem_used_mb}); + + const mem_pad = try utils.repeatStr(ctx.arena, " ", @intCast(mem_total_str.len -| mem_used_str.len)); + const mem_suffix = try std.fmt.allocPrint(ctx.arena, "{s}{s} / {s} MB", .{ + mem_pad, mem_used_str, mem_total_str, + }); + + var card: MetricCard = .{ + .title = card_title, + .title_image = image_cache.global.get(switch (dev.*) { + .cuda => "gpu_cuda", + .rocm => "gpu_rocm", + .neuron => "gpu_neuron", + .tpu => "gpu_tpu", + }), + .cell_size = ctx.cell_size, + .highlighted = self.highlighted, + }; + + const util_hist = try self.state.history.util[i].sliceLast(ctx.arena, data.history_len); + const mem_hist = try self.state.history.mem_util[i].sliceLast(ctx.arena, data.history_len); + const util_chart_data = try utils.normalizeRange(ctx.arena, util_hist, 0, 100); + const mem_chart_data = try utils.normalizeRange(ctx.arena, mem_hist, 0, 100); + + const chart_entries = [2]MetricCard.ChartEntry{ + .{ .label = util_label, .value = cf.util_percent, .suffix = dev_suffix, .data = util_chart_data }, + .{ .label = "MEM", .value = mem_pct, .suffix = mem_suffix, .data = mem_chart_data }, + }; + const gauge_entries = [2]MetricCard.GaugeEntry{ + .{ .label = util_label, .value = cf.util_percent, .suffix = dev_suffix }, + .{ .label = "MEM", .value = mem_pct, .suffix = mem_suffix }, + }; + + if (self.use_braille) { + card.charts = &chart_entries; + } else { + card.gauges = &gauge_entries; + } + + var card_surf = try card.draw(ctx); + card_surf.widget = ui.widget(self); + return card_surf; +} diff --git a/bin/zml-smi/tui/widgets/gauge.zig b/bin/zml-smi/tui/widgets/gauge.zig new file mode 100644 index 00000000..865a45f5 --- /dev/null +++ b/bin/zml-smi/tui/widgets/gauge.zig @@ -0,0 +1,103 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const theme = @import("../theme.zig"); +const utils = @import("../lib/utils.zig"); +const ui = @import("../lib/ui.zig"); + +const RichText = vxfw.RichText; + +const Gauge = @This(); + +value: u8, // 0-100 +label: ?[]const u8 = null, +suffix: ?[]const u8 = null, +/// Minimum label space to reserve (for aligning gauges with different label widths). +label_reserve: u16 = 0, +/// Minimum suffix space to reserve (for aligning gauges with different suffix widths). +suffix_reserve: u16 = 0, + +pub const sep_w: u16 = 3; // " " before suffix + +pub fn draw(self: *const Gauge, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const max_w = ctx.max.width orelse 30; + const width: u16 = @max(ctx.min.width, max_w); + + // Label width (use reserve for alignment if set) + var label_w: u16 = 0; + if (self.label) |label| { + label_w = @intCast(ctx.stringWidth(label)); + label_w += 1; // trailing space + } + const effective_label_w = @max(label_w, self.label_reserve); + + // Percentage text: " XX%" + const pct_str = try std.fmt.allocPrint(ctx.arena, "{d:>3}%", .{self.value}); + const pct_width: u16 = @intCast(ctx.stringWidth(pct_str)); + + // Suffix area (text + separator); use reserve for bar calculation, actual for rendering + const actual_suffix_area: u16 = if (self.suffix) |s| @as(u16, @intCast(ctx.stringWidth(s))) + sep_w else 0; + const effective_suffix_area = @max(actual_suffix_area, self.suffix_reserve); + + // Reserved: label + "[" + "]" + pct + suffix_area + const reserved = effective_label_w + pct_width + 2 + effective_suffix_area; // 2 for [ ] + const bar_width = width -| reserved; + const filled: u16 = @intCast(@as(u32, bar_width) * @as(u32, @min(self.value, 100)) / 100); + const empty_count = bar_width - filled; + + var spans: std.ArrayList(vaxis.Cell.Segment) = .empty; + + // Label (padded to effective_label_w for alignment) + if (self.label) |label| { + try spans.append(ctx.arena, .{ .text = label, .style = theme.label_style }); + const pad = effective_label_w -| label_w; + const pad_str = try utils.repeatStr(ctx.arena, " ", pad + 1); // +1 for trailing space + try spans.append(ctx.arena, .{ .text = pad_str, .style = theme.label_style }); + } + + // Opening bracket + try spans.append(ctx.arena, .{ .text = "[", .style = theme.dim_style }); + + // Filled blocks — per-block gradient color + for (0..filled) |i| { + const pos_pct: u8 = if (bar_width > 1) + @intCast(@as(u32, @intCast(i)) * 100 / (@as(u32, bar_width) - 1)) + else + 0; + try spans.append(ctx.arena, .{ + .text = "\u{2588}", + .style = .{ .fg = theme.colorForPercent(pos_pct) }, + }); + } + + // Empty blocks + if (empty_count > 0) { + const empty_str = try utils.repeatStr(ctx.arena, "\u{2591}", empty_count); + try spans.append(ctx.arena, .{ .text = empty_str, .style = .{ .fg = theme.gauge_empty } }); + } + + // Closing bracket (no space — pct% collides with bar) + try spans.append(ctx.arena, .{ .text = "]", .style = theme.dim_style }); + + // Percentage text + try spans.append(ctx.arena, .{ .text = pct_str, .style = .{ .bold = true, .fg = theme.colorForPercent(self.value) } }); + + // Suffix (right-aligned) + if (self.suffix) |suffix| { + const suffix_text_w: u16 = @intCast(ctx.stringWidth(suffix)); + const pad = effective_suffix_area -| suffix_text_w; + if (pad > 0) { + const pad_str = try utils.repeatStr(ctx.arena, " ", pad); + try spans.append(ctx.arena, .{ .text = pad_str, .style = theme.dim_style }); + } + try spans.append(ctx.arena, .{ .text = suffix, .style = theme.dim_style }); + } + + const rich: RichText = .{ + .text = spans.items, + .softwrap = false, + .overflow = .clip, + .width_basis = .parent, + }; + return rich.draw(ui.fixedSize(ctx, width, 1)); +} diff --git a/bin/zml-smi/tui/widgets/info_lines.zig b/bin/zml-smi/tui/widgets/info_lines.zig new file mode 100644 index 00000000..3e7ac880 --- /dev/null +++ b/bin/zml-smi/tui/widgets/info_lines.zig @@ -0,0 +1,111 @@ +const std = @import("std"); + +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; + +const data = @import("../data.zig"); +const compose = @import("../lib/compose.zig"); +const ui = @import("../lib/ui.zig"); +const theme = @import("../theme.zig"); + +const InfoLines = @This(); + +state: *const data.SystemState, + +const labels = [_][]const u8{ + "zml-smi ", + "Hostname ", + "Kernel ", + "CPU ", + "Cores ", + "Memory ", + "Uptime ", + "Load ", + "Devices ", +}; + +pub const entry_count: u16 = labels.len; + +const Entry = struct { + label: []const u8, + value: []const u8, +}; + +pub fn draw(self: *const InfoLines, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const w = ctx.max.width orelse 80; + const host = self.state.host.front().*; + + const hostname_str = try std.fmt.allocPrint(ctx.arena, "{s}", .{host.hostname orelse "Unkown"}); + const kernel_str = try std.fmt.allocPrint(ctx.arena, "{s}", .{host.kernel orelse "Unknown"}); + const cpu_str = try std.fmt.allocPrint(ctx.arena, "{s}", .{host.cpu_name orelse "Unknown"}); + const uptime_str = try formatUptime(ctx.arena, host.uptime_seconds orelse 0); + const cores_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{host.cpu_cores orelse 0}); + const load_str = try std.fmt.allocPrint(ctx.arena, "{d:.2} / {d:.2} / {d:.2}", .{ + host.load_1 orelse 0, host.load_5 orelse 0, host.load_15 orelse 0, + }); + const total_kib = host.mem_total_kib orelse 0; + const avail_kib = host.mem_available_kib orelse 0; + const used_gb = (total_kib -| avail_kib) / (1024 * 1024); + const total_gb = total_kib / (1024 * 1024); + const mem_str = try std.fmt.allocPrint(ctx.arena, "{d} / {d} GB", .{ used_gb, total_gb }); + const devices_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{self.state.deviceCount()}); + const values = [entry_count][]const u8{ + "v0.2", + hostname_str, + kernel_str, + cpu_str, + cores_str, + mem_str, + uptime_str, + load_str, + devices_str, + }; + var entries: [entry_count]Entry = undefined; + for (&entries, labels, values) |*e, label, value| { + e.* = .{ .label = label, .value = value }; + } + + var sb = compose.surfaceBuilder(ctx.arena); + + for (entries, 0..) |entry, i| { + if (w == 0) { + continue; + } + + const total_len = entry.label.len + entry.value.len; + const segments = if (total_len < w) blk: { + const pad_len = w - total_len; + const pad = try ctx.arena.alloc(u8, pad_len); + @memset(pad, '-'); + pad[pad_len - 1] = ' '; + break :blk try ctx.arena.dupe(vaxis.Cell.Segment, &.{ + .{ .text = entry.label, .style = theme.header_style }, + .{ .text = pad, .style = theme.dim_style }, + .{ .text = entry.value, .style = theme.value_style }, + }); + } else try ctx.arena.dupe(vaxis.Cell.Segment, &.{ + .{ .text = entry.label, .style = theme.header_style }, + .{ .text = entry.value, .style = theme.value_style }, + }); + + const text_surf = try ui.drawRichLine(ctx, segments, w); + + try sb.add(@intCast(i), 0, text_surf); + } + + const total_h: u16 = @intCast(entries.len); + return sb.finish(.{ .width = w, .height = total_h }, ui.widget(self)); +} + +fn formatUptime(arena: std.mem.Allocator, seconds: u64) std.mem.Allocator.Error![]const u8 { + const days = seconds / 86400; + const hours = (seconds % 86400) / 3600; + const mins = (seconds % 3600) / 60; + if (days > 0) { + return std.fmt.allocPrint(arena, "{d}d {d}h {d}m", .{ days, hours, mins }); + } else if (hours > 0) { + return std.fmt.allocPrint(arena, "{d}h {d}m", .{ hours, mins }); + } else { + return std.fmt.allocPrint(arena, "{d}m", .{mins}); + } +} diff --git a/bin/zml-smi/tui/widgets/logo.zig b/bin/zml-smi/tui/widgets/logo.zig new file mode 100644 index 00000000..b530c890 --- /dev/null +++ b/bin/zml-smi/tui/widgets/logo.zig @@ -0,0 +1,51 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const theme = @import("../theme.zig"); +const ui = @import("../lib/ui.zig"); +const compose = @import("../lib/compose.zig"); +const Image = @import("../lib/image.zig"); + +const Logo = @This(); + +style: vaxis.Cell.Style = theme.header_style, +image: ?vaxis.Image = null, +compact: bool = false, + +const ascii_lines = [_][]const u8{ + " ", + "█████╗███╗ ███╗██╗ ", + "╚═██╔╝████╗ ████║██║ ", + " ██╔╝ ██╔████╔██║██║ ", + "█████╗██║ ╚═╝ ██║████╗", + "╚════╝╚═╝ ╚═╝╚═══╝", +}; + +pub const logo_width: u16 = 22; +pub const logo_height: u16 = ascii_lines.len; +pub const compact_height: u16 = ascii_lines.len - 1; +pub const image_height: u16 = 11; +pub const compact_image_height: u16 = 7; + +pub fn draw(self: *const Logo, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + if (self.image) |img| { + const h = if (self.compact) compact_image_height else image_height; + const img_w: Image = .{ .image = img, .rows = h }; + return img_w.draw(ctx); + } + + const lines: []const []const u8 = if (self.compact) ascii_lines[1..] else &ascii_lines; + const h: u16 = if (self.compact) compact_height else logo_height; + const logo_color: vaxis.Cell.Color = .{ .rgb = .{ 140, 180, 255 } }; + + var sb = compose.surfaceBuilder(ctx.arena); + for (lines, 0..) |line, row| { + const text: vxfw.Text = .{ + .text = line, + .style = .{ .bold = self.style.bold, .fg = logo_color }, + .softwrap = false, + }; + try sb.add(@intCast(row), 0, try text.draw(ui.maxSize(ctx, logo_width, 1))); + } + return sb.finish(.{ .width = logo_width, .height = h }, ui.widget(self)); +} diff --git a/bin/zml-smi/tui/widgets/metric_card.zig b/bin/zml-smi/tui/widgets/metric_card.zig new file mode 100644 index 00000000..d14378e7 --- /dev/null +++ b/bin/zml-smi/tui/widgets/metric_card.zig @@ -0,0 +1,163 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const theme = @import("../theme.zig"); +const ui = @import("../lib/ui.zig"); +const compose = @import("../lib/compose.zig"); +const Gauge = @import("gauge.zig"); +const BrailleChart = @import("braille_chart.zig"); +const TitledBorder = @import("titled_border.zig"); + +const MetricCard = @This(); + +title: []const u8, +title_image: ?vaxis.Image = null, +cell_size: vxfw.Size = .{}, +/// Lines of "label: value" pairs to display. +lines: []const LineEntry = &.{}, +/// Optional gauge bars. +gauges: []const GaugeEntry = &.{}, +/// Optional braille mini-charts. +charts: []const ChartEntry = &.{}, +/// Whether this card is highlighted (e.g. hovered). +highlighted: bool = false, + +pub const LineEntry = struct { + label: []const u8, + value: []const u8, + style: vaxis.Cell.Style = theme.value_style, +}; + +pub const GaugeEntry = struct { + label: []const u8, + value: u8, + suffix: ?[]const u8 = null, +}; + +pub const ChartEntry = struct { + label: []const u8, + value: u8, + suffix: ?[]const u8 = null, + data: []const u8, // history values 0-100 + chart_height: u16 = 3, +}; + +pub fn draw(self: *const MetricCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const bstyle: vaxis.Cell.Style = if (self.highlighted) + .{ .fg = theme.accent, .bold = true } + else + theme.border_style; + const tb: TitledBorder = .{ + .child = ui.drawWidget(self, drawContent), + .title = self.title, + .title_image = self.title_image, + .cell_size = self.cell_size, + .border_style = bstyle, + }; + return tb.draw(ctx); +} + +fn maxFieldWidth(ctx: vxfw.DrawContext, entries: anytype, comptime field: []const u8) u16 { + var max_w: u16 = 0; + for (entries) |entry| { + const raw = @field(entry, field); + const value: []const u8 = if (@typeInfo(@TypeOf(raw)) == .optional) raw orelse continue else raw; + max_w = @max(max_w, @as(u16, @intCast(ctx.stringWidth(value)))); + } + return max_w; +} + +fn drawContent(self: *const MetricCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const inner_w = ctx.max.width orelse 40; + + var sb = compose.surfaceBuilder(ctx.arena); + var row: i17 = 0; + + // Charts (gauge-like layout: label [braille] pct% suffix, braille spans multiple rows) + { + const max_chart_label_w = maxFieldWidth(ctx, self.charts, "label") + 1; // +1 trailing space + const max_chart_suffix_w = maxFieldWidth(ctx, self.charts, "suffix") + Gauge.sep_w; + + for (self.charts, 0..) |chart, ci| { + if (ci > 0) { + row += 1; + } + + const ch_h = chart.chart_height; + + const pct_str = try std.fmt.allocPrint(ctx.arena, "{d:>3}%", .{chart.value}); + const pct_w: u16 = @intCast(ctx.stringWidth(pct_str)); + + const reserved = max_chart_label_w + pct_w + 1 + max_chart_suffix_w; + const braille_w: u16 = if (reserved < inner_w) inner_w - reserved else 0; + const text_row: i17 = row + @as(i17, @intCast(ch_h / 2)); + + // Label (vertically centered, col 0) + const label_surf = try ui.drawRichLine(ctx, &.{ + .{ .text = chart.label, .style = theme.label_style }, + }, max_chart_label_w); + try sb.add(text_row, 0, label_surf); + + // Braille chart (all rows, after label) + if (braille_w > 0) { + const braille: BrailleChart = .{ .values = chart.data, .height = ch_h }; + const braille_surf = try braille.draw(ui.fixedSize(ctx, braille_w, ch_h)); + try sb.add(row, @intCast(max_chart_label_w), braille_surf); + } + + // Pct + suffix (vertically centered, after braille) + const right_col: i17 = @intCast(max_chart_label_w + braille_w); + const pct_surf = try ui.drawRichLine(ctx, &.{ + .{ .text = " ", .style = .{} }, + .{ .text = pct_str, .style = .{ .bold = true, .fg = theme.colorForPercent(chart.value) } }, + }, pct_w + 1); + try sb.add(text_row, right_col, pct_surf); + + if (chart.suffix) |suffix| { + const suffix_text_w: u16 = @intCast(ctx.stringWidth(suffix)); + const suffix_col: i17 = @intCast(inner_w -| suffix_text_w); + const suffix_surf = try ui.drawRichLine(ctx, &.{ + .{ .text = suffix, .style = theme.dim_style }, + }, suffix_text_w); + try sb.add(text_row, suffix_col, suffix_surf); + } + + row += @intCast(ch_h); + } + } + + // Gap between charts and gauges + if (self.charts.len > 0 and self.gauges.len > 0) { + row += 1; + } + + // Gauges (with 1-row gap between consecutive gauges) + const max_label_w = maxFieldWidth(ctx, self.gauges, "label") + 1; // +1 trailing space + const max_suffix_w = maxFieldWidth(ctx, self.gauges, "suffix") + Gauge.sep_w; + for (self.gauges, 0..) |ge, gi| { + const gauge: Gauge = .{ + .value = ge.value, + .label = ge.label, + .suffix = ge.suffix, + .label_reserve = max_label_w, + .suffix_reserve = max_suffix_w, + }; + const gauge_surf = try gauge.draw(ui.fixedSize(ctx, inner_w, 1)); + try sb.add(row, 0, gauge_surf); + row += 1; + if (gi + 1 < self.gauges.len) row += 1; // gap between gauges + } + + // Text lines + for (self.lines) |line| { + const text_surf = try ui.drawRichLine(ctx, &.{ + .{ .text = line.label, .style = theme.label_style }, + .{ .text = line.value, .style = line.style }, + }, inner_w); + try sb.add(row, 0, text_surf); + row += 1; + } + + const content_h: u16 = @intCast(@max(row, 1)); + return sb.finish(.{ .width = inner_w, .height = content_h }, ui.drawWidget(self, drawContent)); +} diff --git a/bin/zml-smi/tui/widgets/process_table.zig b/bin/zml-smi/tui/widgets/process_table.zig new file mode 100644 index 00000000..e4b120c8 --- /dev/null +++ b/bin/zml-smi/tui/widgets/process_table.zig @@ -0,0 +1,179 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("../data.zig"); +const theme = @import("../theme.zig"); +const ui = @import("../lib/ui.zig"); +const compose = @import("../lib/compose.zig"); +const TitledBorder = @import("titled_border.zig"); +const ColumnLayout = @import("column_layout.zig"); +const pi = @import("../../info/process_info.zig"); +const ProcessInfo = pi.ProcessInfo; + +const RichText = vxfw.RichText; +const Segment = vaxis.Cell.Segment; + +const ProcessTable = @This(); + +const max_visible_rows: u16 = 20; + +scroll_bars: vxfw.ScrollBars = .{ + .scroll_view = .{ + .children = .{ .builder = .{ .userdata = undefined, .buildFn = buildRow } }, + .draw_cursor = false, + }, + .draw_horizontal_scrollbar = false, +}, +merged: std.ArrayList(ProcessInfo) = .{}, + +pub fn prepare(self: *ProcessTable, state: *data.SystemState, device_id: ?u8) void { + self.scroll_bars.scroll_view.children.builder.userdata = self; + self.merged.clearRetainingCapacity(); + + for (state.process_lists) |pl| { + const pl_items = pl.front().items; + if (device_id) |did| { + for (pl_items) |entry| { + if (entry.device_idx != did) { + continue; + } + self.merged.append(state.allocator, entry) catch break; + } + } else { + self.merged.appendSlice(state.allocator, pl_items) catch break; + } + } + + // Enrich with host data (uid, username, cpu%, rss, cmdline) + state.enricher.enrich(state.io, self.merged.items); + + // Sort by dev_mem descending + std.sort.insertion(ProcessInfo, self.merged.items, {}, struct { + fn cmp(_: void, a: ProcessInfo, b: ProcessInfo) bool { + return (a.dev_mem_kib orelse 0) > (b.dev_mem_kib orelse 0); + } + }.cmp); + + const count: u32 = @intCast(self.merged.items.len); + self.scroll_bars.estimated_content_height = count; + self.scroll_bars.scroll_view.item_count = count; +} + +pub fn resetScroll(self: *ProcessTable) void { + self.scroll_bars.scroll_view.scroll = .{}; + self.scroll_bars.scroll_view.cursor = 0; +} + +pub fn draw(self: *ProcessTable, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const w = ctx.max.width orelse 80; + const row_count = self.merged.items.len; + + var header_rt: RichText = .{ .text = &header_segs, .softwrap = false, .overflow = .clip }; + const empty_rt: RichText = .{ + .text = &.{.{ .text = " No process data available", .style = theme.dim_style }}, + .softwrap = false, + .overflow = .clip, + }; + + var children: std.ArrayList(vxfw.Widget) = .empty; + try children.append(ctx.arena, header_rt.widget()); + + if (row_count == 0) { + try children.append(ctx.arena, try compose.sized( + ctx.arena, + try compose.pad(ctx.arena, empty_rt.widget(), .{ .top = 1 }), + .{ .width = w, .height = 2 }, + )); + } else { + const scroll_h: u16 = @min(@as(u16, @intCast(row_count)), max_visible_rows); + try children.append(ctx.arena, try compose.sized(ctx.arena, self.scroll_bars.widget(), .{ .width = w, .height = scroll_h })); + } + + const content: ColumnLayout = .{ .children = children.items }; + const tb: TitledBorder = .{ + .child = ui.widget(&content), + .title = "Processes", + }; + return tb.draw(ui.fixedWidth(ctx, w)); +} + +/// Builds only data rows (no header) for the scroll view. +fn buildRow(ptr: *const anyopaque, idx: usize, _: usize) ?vxfw.Widget { + const self: *ProcessTable = @ptrCast(@alignCast(@constCast(ptr))); + if (idx >= self.merged.items.len) { + return null; + } + return .{ .userdata = @ptrCast(&self.merged.items[idx]), .drawFn = drawProcessRow }; +} + +// Column format strings — shared between header (comptime) and row (runtime). +const col_fmt = struct { + const pid = " {s: >7} "; + const user = "{s: <10} "; + const dev = "{s: <5} "; + const util = "{s: >6} "; + const mem = "{s: >8} "; + const cpu = "{s: >6} "; + const host_mem = "{s: >8} "; +}; + +const header_segs = blk: { + const s = theme.label_style; + break :blk [_]Segment{ + .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.pid, .{"PID"}) }, + .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.user, .{"USER"}) }, + .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.dev, .{"DEV"}) }, + .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.util, .{"UTIL"}) }, + .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.mem, .{"DEV MEM"}) }, + .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.cpu, .{"CPU"}) }, + .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.host_mem, .{"HOST MEM"}) }, + .{ .style = s, .text = "Command" }, + }; +}; + +/// Format a value into a column: col(arena, col_fmt.pid, "{d}", .{pid}) +fn col(arena: std.mem.Allocator, comptime column: []const u8, comptime fmt: []const u8, args: anytype) std.mem.Allocator.Error![]const u8 { + return std.fmt.allocPrint(arena, column, .{try std.fmt.allocPrint(arena, fmt, args)}); +} + +fn drawProcessRow(ptr: *anyopaque, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const info: *const ProcessInfo = @ptrCast(@alignCast(ptr)); + const val: vaxis.Style = .{ .fg = theme.text_primary }; + const dim: vaxis.Style = .{ .fg = theme.dim }; + + const segs = try ctx.arena.alloc(Segment, 8); + segs[0] = .{ .style = val, .text = try col(ctx.arena, col_fmt.pid, "{d}", .{info.pid}) }; + segs[1] = .{ .style = val, .text = try col(ctx.arena, col_fmt.user, "{s}", .{info.username[0..@min(info.username.len, 10)]}) }; + segs[2] = .{ .style = val, .text = try col(ctx.arena, col_fmt.dev, "{d}", .{info.device_idx}) }; + segs[3] = if (info.dev_util_percent) |util| + .{ .style = val, .text = try col(ctx.arena, col_fmt.util, "{d}%", .{util}) } + else + .{ .style = dim, .text = try col(ctx.arena, col_fmt.util, "-", .{}) }; + segs[4] = if (info.dev_mem_kib) |mem| + .{ .style = val, .text = try col(ctx.arena, col_fmt.mem, "{s}", .{try fmtMem(ctx.arena, mem)}) } + else + .{ .style = dim, .text = try col(ctx.arena, col_fmt.mem, "-", .{}) }; + segs[5] = .{ .style = val, .text = try col(ctx.arena, col_fmt.cpu, "{d}.{d}%", .{ info.cpu_percent / 10, info.cpu_percent % 10 }) }; + segs[6] = .{ .style = val, .text = try col(ctx.arena, col_fmt.host_mem, "{s}", .{try fmtMem(ctx.arena, info.rss_kib)}) }; + segs[7] = .{ .style = val, .text = info.comm }; + + const rich: RichText = .{ .text = segs, .softwrap = false, .overflow = .clip }; + return rich.draw(ctx.withConstraints( + .{ .width = ctx.min.width, .height = 1 }, + .{ .width = ctx.max.width, .height = 1 }, + )); +} + +fn fmtMem(arena: std.mem.Allocator, kib: u64) std.mem.Allocator.Error![]const u8 { + if (kib >= 1024 * 1024) { + const whole = kib / (1024 * 1024); + const frac = (kib % (1024 * 1024)) * 10 / (1024 * 1024); + return std.fmt.allocPrint(arena, "{d}.{d}G", .{ whole, frac }); + } else if (kib >= 1024) { + const whole = kib / 1024; + const frac = (kib % 1024) * 10 / 1024; + return std.fmt.allocPrint(arena, "{d}.{d}M", .{ whole, frac }); + } else { + return std.fmt.allocPrint(arena, "{d}K", .{kib}); + } +} diff --git a/bin/zml-smi/tui/widgets/status_line.zig b/bin/zml-smi/tui/widgets/status_line.zig new file mode 100644 index 00000000..86a6d1ef --- /dev/null +++ b/bin/zml-smi/tui/widgets/status_line.zig @@ -0,0 +1,77 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const theme = @import("../theme.zig"); +const ui = @import("../lib/ui.zig"); + +const StatusLine = @This(); + +viewing_device: ?u8, +use_braille: bool, + +const Binding = struct { + key: []const u8, + desc: []const u8, +}; + +pub fn draw(self: *const StatusLine, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const w = ctx.max.width orelse 80; + + const key_style: vaxis.Style = .{ .bold = true, .fg = .{ .rgb = .{ 30, 30, 40 } }, .bg = theme.accent }; + const desc_style: vaxis.Style = .{ .fg = theme.text_secondary }; + const sep_style: vaxis.Style = .{ .fg = theme.dim }; + const bg_style: vaxis.Style = .{ .bg = .{ .rgb = .{ 30, 30, 45 } } }; + + const chart_label = if (self.use_braille) "gauges" else "charts"; + + var bindings_buf: [8]Binding = undefined; + var n: usize = 0; + + if (self.viewing_device != null) { + bindings_buf[n] = .{ .key = "Esc", .desc = "back" }; + n += 1; + } else { + bindings_buf[n] = .{ .key = "v", .desc = chart_label }; + n += 1; + bindings_buf[n] = .{ .key = "click", .desc = "device details" }; + n += 1; + } + bindings_buf[n] = .{ .key = "q", .desc = "quit" }; + n += 1; + + const bindings = bindings_buf[0..n]; + + // Build segments: " key desc | key desc | ... " + var segments: std.ArrayList(vaxis.Cell.Segment) = .empty; + + // Leading space + try segments.append(ctx.arena, .{ .text = " ", .style = bg_style }); + + for (bindings, 0..) |b, i| { + try segments.append(ctx.arena, .{ .text = " ", .style = bg_style }); + try segments.append(ctx.arena, .{ .text = b.key, .style = key_style }); + try segments.append(ctx.arena, .{ .text = " ", .style = bg_style }); + try segments.append(ctx.arena, .{ .text = b.desc, .style = desc_style }); + if (i + 1 < bindings.len) { + try segments.append(ctx.arena, .{ .text = " \u{2502}", .style = sep_style }); + } + } + + const rich: vxfw.RichText = .{ + .text = segments.items, + .softwrap = false, + .overflow = .clip, + }; + var surf = try rich.draw(ui.fixedSize(ctx, w, 1)); + + // Fill background across full width + if (surf.buffer.len > 0) { + for (surf.buffer) |*cell| { + if (cell.style.bg == .default) { + cell.style.bg = bg_style.bg; + } + } + } + + return surf; +} diff --git a/bin/zml-smi/tui/widgets/titled_border.zig b/bin/zml-smi/tui/widgets/titled_border.zig new file mode 100644 index 00000000..d9c16171 --- /dev/null +++ b/bin/zml-smi/tui/widgets/titled_border.zig @@ -0,0 +1,112 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const theme = @import("../theme.zig"); +const ui = @import("../lib/ui.zig"); +const Image = @import("../lib/image.zig"); + +const Text = vxfw.Text; + +const TitledBorder = @This(); + +child: vxfw.Widget, +title: []const u8 = "", +title_style: vaxis.Cell.Style = theme.title_style, +title_image: ?vaxis.Image = null, +cell_size: vxfw.Size = .{}, +value_label: ?[]const u8 = null, +value_label_style: vaxis.Cell.Style = theme.value_style, +border_style: vaxis.Cell.Style = theme.border_style, +padding: vxfw.Padding.PadValues = .{ .left = 1, .right = 1, .top = 1, .bottom = 1 }, + +pub fn draw(self: *const TitledBorder, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const pad: vxfw.Padding = .{ + .child = self.child, + .padding = self.padding, + }; + const border: vxfw.Border = .{ + .child = pad.widget(), + .style = self.border_style, + }; + var surf = try border.draw(ctx); + + // Count overlay children needed + var overlay_count: usize = 0; + var title_col: u16 = 2; + var has_image = false; + + if (self.title_image) |img| { + const icon = ui.imageCellSize(img, 1, self.cell_size); + has_image = true; + title_col += icon.cols; + overlay_count += 1; + } + if (self.title.len > 0) { + overlay_count += 1; + } + if (self.value_label != null) { + overlay_count += 1; + } + + if (overlay_count == 0) { + return surf; + } + + const new_children = try ctx.arena.alloc(vxfw.SubSurface, surf.children.len + overlay_count); + @memcpy(new_children[0..surf.children.len], surf.children); + var idx = surf.children.len; + + // Image icon overlay + if (has_image) { + const img = self.title_image.?; + const img_widget: Image = .{ .image = img, .rows = 1 }; + const img_surf = try img_widget.draw(ctx); + new_children[idx] = .{ + .origin = .{ .row = 0, .col = 2 }, + .surface = img_surf, + .z_index = 1, + }; + idx += 1; + } + + // Title text overlay (top-left) + if (self.title.len > 0) { + const title_text = try std.fmt.allocPrint(ctx.arena, " {s} ", .{self.title}); + const title_widget: Text = .{ + .text = title_text, + .style = self.title_style, + .softwrap = false, + .overflow = .clip, + }; + const max_title_w = surf.size.width -| (title_col + 1); + const title_surf = try title_widget.draw(ui.maxSize(ctx, max_title_w, 1)); + new_children[idx] = .{ + .origin = .{ .row = 0, .col = @intCast(title_col) }, + .surface = title_surf, + .z_index = 1, + }; + idx += 1; + } + + // Value label overlay (top-right) + if (self.value_label) |vlabel| { + const label_text = try std.fmt.allocPrint(ctx.arena, " {s} ", .{vlabel}); + const label_widget: Text = .{ + .text = label_text, + .style = self.value_label_style, + .softwrap = false, + .overflow = .clip, + }; + const right = surf.size.width -| 1; + const lw: u16 = @intCast(@min(ctx.stringWidth(label_text), right -| 2)); + const label_surf = try label_widget.draw(ui.maxSize(ctx, lw, 1)); + new_children[idx] = .{ + .origin = .{ .row = 0, .col = @intCast(right -| lw) }, + .surface = label_surf, + .z_index = 1, + }; + } + + surf.children = new_children; + return surf; +} diff --git a/bin/zml-smi/utils/double_buffer.zig b/bin/zml-smi/utils/double_buffer.zig new file mode 100644 index 00000000..32ed7a1d --- /dev/null +++ b/bin/zml-smi/utils/double_buffer.zig @@ -0,0 +1,22 @@ +const std = @import("std"); + +pub fn DoubleBuffer(comptime T: type) type { + return struct { + const Self = @This(); + + values: [2]T, + current: std.atomic.Value(u8) = std.atomic.Value(u8).init(0), + + pub fn front(self: *const Self) *const T { + return &self.values[self.current.load(.acquire)]; + } + + pub fn back(self: *Self) *T { + return &self.values[1 - self.current.load(.acquire)]; + } + + pub fn swap(self: *Self) void { + self.current.store(1 - self.current.load(.acquire), .release); + } + }; +} diff --git a/bin/zml-smi/utils/sandbox.zig b/bin/zml-smi/utils/sandbox.zig new file mode 100644 index 00000000..1b53c9ea --- /dev/null +++ b/bin/zml-smi/utils/sandbox.zig @@ -0,0 +1,44 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const bazel_runfiles = @import("runfiles"); +const bazel_builtin = @import("bazel_builtin"); + +var runfiles_global: bazel_runfiles.Runfiles = undefined; + +const init_array_section = switch (builtin.object_format) { + .macho => "__DATA,__mod_init_func", + .elf => ".init_array", + else => "", +}; + +export const _ linksection(init_array_section) = &struct { + fn call(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { + _ = argc; + _ = argv; + _ = envp; + var threaded: std.Io.Threaded = .init_single_threaded; + const io = threaded.io(); + + var exe_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const exe_size = std.process.executablePath(io, &exe_path_buf) catch + std.debug.panic("Unable to get executable path", .{}); + + runfiles_global = bazel_runfiles.Runfiles.create(.{ + .allocator = std.heap.c_allocator, + .io = io, + .argv0 = exe_path_buf[0..exe_size], + .directory = if (std.c.getenv("RUNFILES_DIR")) |directory| std.mem.span(directory) else null, + .manifest = if (std.c.getenv("RUNFILES_MANIFEST_FILE")) |manifest| std.mem.span(manifest) else null, + }) catch { + std.debug.panic("Unable to find runfiles", .{}); + } orelse { + std.debug.panic("Unable to initialize runfiles", .{}); + }; + } +}.call; + +pub fn path(buf: *[std.Io.Dir.max_path_bytes]u8) ?[]const u8 { + const with_repo = runfiles_global.withSourceRepo(bazel_builtin.current_repository); + + return with_repo.rlocation("zml/bin/zml-smi/sandbox", buf) catch null; +} diff --git a/bin/zml-smi/utils/sysfs.zig b/bin/zml-smi/utils/sysfs.zig new file mode 100644 index 00000000..636883b2 --- /dev/null +++ b/bin/zml-smi/utils/sysfs.zig @@ -0,0 +1,70 @@ +const std = @import("std"); + +pub fn readInt(allocator: std.mem.Allocator, io: std.Io, path: []const u8) !u64 { + const data = try readFirstLine(allocator, io, path); + return std.fmt.parseInt(u64, data, 10); +} + +pub fn readString(allocator: std.mem.Allocator, io: std.Io, path: []const u8) ![]const u8 { + return readFirstLine(allocator, io, path); +} + +pub fn readFieldInt(allocator: std.mem.Allocator, io: std.Io, path: []const u8, comptime key: []const u8) !u64 { + return nthTokenInt(try findField(allocator, io, path, key), 0); +} + +pub fn nthTokenInt(data: []const u8, n: usize) u64 { + var i: usize = 0; + + var iter = std.mem.tokenizeAny(u8, data, " \t"); + while (iter.next()) |tok| : (i += 1) { + if (i == n) { + return std.fmt.parseInt(u64, tok, 10) catch 0; + } + } + + return 0; +} + +pub fn readFieldString(allocator: std.mem.Allocator, io: std.Io, path: []const u8, comptime key: []const u8) ![]const u8 { + const value = try findField(allocator, io, path, key); + return std.mem.trimEnd(u8, value, &std.ascii.whitespace); +} + +fn findField(allocator: std.mem.Allocator, io: std.Io, path: []const u8, comptime key: []const u8) ![]const u8 { + var file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); + defer file.close(io); + + var read_buf: [4096]u8 = undefined; + var reader = file.reader(io, &read_buf); + + var line_writer: std.Io.Writer.Allocating = .init(allocator); + errdefer line_writer.deinit(); + + while (true) { + line_writer.clearRetainingCapacity(); + _ = reader.interface.streamDelimiter(&line_writer.writer, '\n') catch return error.NotFound; + reader.interface.toss(1); + + const line = line_writer.written(); + if (std.mem.startsWith(u8, line, key)) { + const owned = line_writer.toOwnedSlice() catch return error.NotFound; + return std.mem.trimStart(u8, owned[key.len..], " :\t"); + } + } +} + +pub fn readFirstLine(allocator: std.mem.Allocator, io: std.Io, path: []const u8) ![]u8 { + var file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); + defer file.close(io); + + var read_buf: [4096]u8 = undefined; + var reader = file.reader(io, &read_buf); + var writer: std.Io.Writer.Allocating = .init(allocator); + errdefer writer.deinit(); + _ = reader.interface.streamDelimiter(&writer.writer, '\n') catch |err| switch (err) { + error.EndOfStream => 0, + else => |e| return e, + }; + return writer.toOwnedSlice(); +} diff --git a/bin/zml-smi/worker.zig b/bin/zml-smi/worker.zig new file mode 100644 index 00000000..9bb4b748 --- /dev/null +++ b/bin/zml-smi/worker.zig @@ -0,0 +1,62 @@ +const std = @import("std"); + +pub const Worker = struct { + poll_interval_ms: u16, + should_stop: std.atomic.Value(bool) = .init(false), + group: std.Io.Group = .init, + + pub fn shutdown(self: *Worker, io: std.Io) void { + self.should_stop.store(true, .release); + self.group.await(io) catch {}; + } + + pub fn isRunning(self: *const Worker) bool { + return !self.should_stop.load(.acquire); + } + + pub fn spawn( + self: *Worker, + io: std.Io, + comptime runFn: anytype, + args: std.meta.ArgsTuple(@TypeOf(runFn)), + ) !void { + try self.group.concurrent(io, runFn, args); + } + + pub fn pollMetrics(comptime DB: type, comptime Dev: type, comptime table: anytype) fn (std.Io, *const Worker, DB, Dev) void { + return struct { + fn f(io: std.Io, w: *const Worker, db: DB, dev: Dev) void { + w.pollLoop(io, struct { + fn poll(s: DB, d: Dev) void { + if (@hasField(Dev, "arena")) { + _ = d.arena.reset(.retain_capacity); + } + + const back = s.back(); + back.* = s.front().*; + + inline for (table) |m| { + @field(back, m.field) = m.query(d) catch null; + } + + s.swap(); + } + }.poll, .{ db, dev }); + } + }.f; + } + + pub fn pollLoop(self: *const Worker, io: std.Io, comptime func: anytype, args: std.meta.ArgsTuple(@TypeOf(func))) void { + const interval: std.Io.Duration = .fromMilliseconds(self.poll_interval_ms); + while (!self.should_stop.load(.acquire)) { + const start: std.Io.Timestamp = .now(io, .awake); + + @call(.auto, func, args); + + const elapsed = start.untilNow(io, .awake); + if (elapsed.nanoseconds < interval.nanoseconds) { + io.sleep(.fromNanoseconds(interval.nanoseconds - elapsed.nanoseconds), .awake) catch {}; + } + } + } +}; diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index dc27fb96..a1524824 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -39,6 +39,14 @@ _UBUNTU_PACKAGES = { } CUDA_PACKAGES = { + "cuda_nvml_dev": "\n".join([ + packages.cc_library( + name = "nvml", + hdrs = ["include/nvml.h"], + includes = ["include"], + visibility = ["//visibility:public"], + ), + ]), "cuda_cudart": "\n".join([ # Driver API only packages.cc_library( @@ -275,7 +283,7 @@ def _cuda_impl(mctx): return mctx.extension_metadata( reproducible = True, - root_module_direct_deps = ["libpjrt_cuda"], + root_module_direct_deps = ["cuda_nvml_dev", "libpjrt_cuda"], root_module_direct_dev_deps = [], ) diff --git a/platforms/neuron/BUILD.bazel b/platforms/neuron/BUILD.bazel index 09ca892e..4927764a 100644 --- a/platforms/neuron/BUILD.bazel +++ b/platforms/neuron/BUILD.bazel @@ -1,4 +1,4 @@ -load("@com_google_protobuf//upb/bazel:upb_c_proto_library.bzl", "upb_c_proto_library") +load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_c_proto_library") load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_python//python/uv:lock.bzl", uv_lock = "lock") load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_shared_library") @@ -66,6 +66,7 @@ cc_library( name = "libnrt_headers", tags = ["manual"], hdrs = ["nrt.h"], + visibility = ["//bin/zml-smi:__pkg__"], deps = ["@libpjrt_neuron//:libnrt_headers"], ) diff --git a/platforms/neuron/nrt.h b/platforms/neuron/nrt.h index d391b8eb..0da39452 100644 --- a/platforms/neuron/nrt.h +++ b/platforms/neuron/nrt.h @@ -1,2 +1,4 @@ #include #include +#include +#include diff --git a/platforms/rocm/BUILD.bazel b/platforms/rocm/BUILD.bazel index 93027194..ce2a28c8 100644 --- a/platforms/rocm/BUILD.bazel +++ b/platforms/rocm/BUILD.bazel @@ -10,12 +10,15 @@ zig_shared_library( name = "zmlxrocm", main = "zmlxrocm.zig", shared_lib_name = "libzmlxrocm.so.0", - visibility = ["@libpjrt_rocm//:__subpackages__"], + visibility = [ + "//bin/zml-smi:__pkg__", + "@libpjrt_rocm//:__subpackages__", + ], # Use Clang's compiler-rt, but disable stack checking # to avoid requiring on the _zig_probe_stack symbol. zigopts = ["-fno-stack-check"], deps = ["//stdx"], - target_compatible_with = ["@platforms//os:linux"] + target_compatible_with = ["@platforms//os:linux"], ) alias( diff --git a/platforms/rocm/libpjrt_rocm.BUILD.bazel b/platforms/rocm/libpjrt_rocm.BUILD.bazel index e6152d1d..21967a6c 100644 --- a/platforms/rocm/libpjrt_rocm.BUILD.bazel +++ b/platforms/rocm/libpjrt_rocm.BUILD.bazel @@ -45,6 +45,7 @@ run_binary( "$(location amdgpu.ids)", ], tool = ":merge_amdgpu_ids", + visibility = ["@zml//bin/zml-smi:__pkg__"], ) copy_to_directory( diff --git a/platforms/rocm/packages.lock.json b/platforms/rocm/packages.lock.json old mode 100755 new mode 100644 index 16763572..07234598 --- a/platforms/rocm/packages.lock.json +++ b/platforms/rocm/packages.lock.json @@ -182,6 +182,853 @@ ], "version": "0.186-1ubuntu0.1" }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "rocm-core_7.2.1.70201-81_22.04_amd64", + "name": "rocm-core", + "version": "7.2.1.70201-81~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "python3-setuptools_59.6.0-1.2ubuntu0.22.04.3_amd64", + "name": "python3-setuptools", + "version": "59.6.0-1.2ubuntu0.22.04.3" + }, + { + "key": "python3-distutils_3.10.8-1_22.04_amd64", + "name": "python3-distutils", + "version": "3.10.8-1~22.04" + }, + { + "key": "python3-lib2to3_3.10.8-1_22.04_amd64", + "name": "python3-lib2to3", + "version": "3.10.8-1~22.04" + }, + { + "key": "python3-pkg-resources_59.6.0-1.2ubuntu0.22.04.3_amd64", + "name": "python3-pkg-resources", + "version": "59.6.0-1.2ubuntu0.22.04.3" + }, + { + "key": "sudo_1.9.9-1ubuntu2.5_amd64", + "name": "sudo", + "version": "1.9.9-1ubuntu2.5" + }, + { + "key": "lsb-base_11.1.0ubuntu4_amd64", + "name": "lsb-base", + "version": "11.1.0ubuntu4" + }, + { + "key": "libpam-modules_1.4.0-11ubuntu2.6_amd64", + "name": "libpam-modules", + "version": "1.4.0-11ubuntu2.6" + }, + { + "key": "libpam-modules-bin_1.4.0-11ubuntu2.6_amd64", + "name": "libpam-modules-bin", + "version": "1.4.0-11ubuntu2.6" + }, + { + "key": "libpam0g_1.4.0-11ubuntu2.6_amd64", + "name": "libpam0g", + "version": "1.4.0-11ubuntu2.6" + }, + { + "key": "libaudit1_1-3.0.7-1build1_amd64", + "name": "libaudit1", + "version": "1:3.0.7-1build1" + }, + { + "key": "libcap-ng0_0.7.9-2.2build3_amd64", + "name": "libcap-ng0", + "version": "0.7.9-2.2build3" + }, + { + "key": "libaudit-common_1-3.0.7-1build1_amd64", + "name": "libaudit-common", + "version": "1:3.0.7-1build1" + } + ], + "key": "amd-smi-lib_26.2.2.70201-81_22.04_amd64", + "name": "amd-smi-lib", + "sha256": "43566b517bdbd90fe7b5bc400c30f14dd062de6a304642808079cef394f606f6", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/a/amd-smi-lib/amd-smi-lib_26.2.2.70201-81~22.04_amd64.deb" + ], + "version": "26.2.2.70201-81~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "rocm-core_7.2.1.70201-81_22.04_amd64", + "name": "rocm-core", + "sha256": "b83fe314e67f6bd3cef2b5695fb6de7153110a14727add1992d0056d81a7e9a9", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocm-core/rocm-core_7.2.1.70201-81~22.04_amd64.deb" + ], + "version": "7.2.1.70201-81~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "sha256": "43dbcc12d790ed9360be4a1dc690ac0a7464a5e7c3170785886df84811a3f5a5", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3-defaults/python3_3.10.6-1~22.04.1_amd64.deb" + ], + "version": "3.10.6-1~22.04.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "sha256": "41b8f285f0a1661ece778a63592dcb556710cce3c559d83114fad245bfdf56ea", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3-defaults/libpython3-stdlib_3.10.6-1~22.04.1_amd64.deb" + ], + "version": "3.10.6-1~22.04.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "sha256": "123b338b5857a35e145c2bd7ce890434dbed2e7f6cd03505e86377f9e0aac40b", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3.10/libpython3.10-stdlib_3.10.12-1~22.04.10_amd64.deb" + ], + "version": "3.10.12-1~22.04.10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "sha256": "3a88f3c8c58a32c623b79f1caace557f63cb997c03fa18eaa31ca0285f635919", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/u/util-linux/libuuid1_2.37.2-4ubuntu3.4_amd64.deb" + ], + "version": "2.37.2-4ubuntu3.4" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "sha256": "f2f07d1ab8ae8a8a6e893975df0bca8683caec516e7a30e8dadbb87d33aa52c4", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libt/libtirpc/libtirpc3_1.3.2-2ubuntu0.1_amd64.deb" + ], + "version": "1.3.2-2ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "sha256": "cd7b220447999402e87d75f7cedcfad159a2e5d4236927a414d77f2932219bee", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libt/libtirpc/libtirpc-common_1.3.2-2ubuntu0.1_all.deb" + ], + "version": "1.3.2-2ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "sha256": "2bc13d97c9cd1045271bf600c53ca3ae7516c61d9102f1b9c9d2fa722c35183b", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/k/krb5/libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64.deb" + ], + "version": "1.19.2-2ubuntu0.7" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "sha256": "fd2f8c23945590b38f576f7a58dcba7eb63bb6dc97d37271a15b9d8b2c29210f", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/k/krb5/libkrb5support0_1.19.2-2ubuntu0.7_amd64.deb" + ], + "version": "1.19.2-2ubuntu0.7" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "sha256": "75fc4ab539697f7df8afa2e226b39515b6425111a80aecce1ea41ce392896224", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/k/krb5/libkrb5-3_1.19.2-2ubuntu0.7_amd64.deb" + ], + "version": "1.19.2-2ubuntu0.7" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "sha256": "3a4b984a91fb0fda4d287f1aceeea0f8f1d8a4ed6c9fe7dac114208be92e0f1f", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/o/openssl/libssl3_3.0.2-0ubuntu1.19_amd64.deb" + ], + "version": "3.0.2-0ubuntu1.19" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "sha256": "395a65b3a9539304391456d4d0e5531f4249f297b6e67eb0fcaf2ff77ddc3dc3", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/d/debconf/debconf_1.5.79ubuntu1_all.deb" + ], + "version": "1.5.79ubuntu1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "sha256": "465f2523d9bbbd64c1dc9ffdb894885214723d6e8e6942b4bae6b4b8811fdaa5", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/perl/perl-base_5.34.0-3ubuntu1.4_amd64.deb" + ], + "version": "5.34.0-3ubuntu1.4" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "sha256": "92c21b47281c1bf24d788fe02d717c058fd9701b9c224b0b5c2c1179fb5d152d", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/d/dpkg/dpkg_1.21.1ubuntu2.3_amd64.deb" + ], + "version": "1.21.1ubuntu2.3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "sha256": "151e3a61137aeb04cd714656852877a00d47d918246806f9e5b08d1cef32b816", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/t/tar/tar_1.34+dfsg-1ubuntu0.1.22.04.2_amd64.deb" + ], + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "sha256": "b96c6b40ee2388bd51341cb11c0f1d5bcca29b9180b6e3a77a06b881f2913f7e", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libs/libselinux/libselinux1_3.3-1build2_amd64.deb" + ], + "version": "3.3-1build2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "sha256": "01a5a74f3d305cad32fffffdc200c2a92fd8e5a01bff97de76ac598fb444d53e", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pcre2/libpcre2-8-0_10.39-3ubuntu0.1_amd64.deb" + ], + "version": "10.39-3ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "sha256": "4db2c64ec74f673ed022e92cce7b83d0cbe0b779e02ca60a56ba59ae07754e05", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/a/acl/libacl1_2.3.1-1_amd64.deb" + ], + "version": "2.3.1-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "sha256": "ae7db00ce8b093e50c994518b90203544e063b4bc574836a048bb142b950b2c9", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libz/libzstd/libzstd1_1.4.8+dfsg-3build1_amd64.deb" + ], + "version": "1.4.8+dfsg-3build1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "sha256": "940daf78ee0229549b4eabc92ffd79dd038ed96a44e2e912305a23540cf22a0a", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/k/keyutils/libkeyutils1_1.6.1-2ubuntu3_amd64.deb" + ], + "version": "1.6.1-2ubuntu3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "sha256": "76174ecc0948b587a0e92a08b023c9f8655f581aaff476c52f50299e80e92056", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/k/krb5/libk5crypto3_1.19.2-2ubuntu0.7_amd64.deb" + ], + "version": "1.19.2-2ubuntu0.7" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "sha256": "552bc8169451cfdcf174a8da845b13ced0a55777149af5cf47491193161ed791", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/e/e2fsprogs/libcom-err2_1.46.5-2ubuntu1.2_amd64.deb" + ], + "version": "1.46.5-2ubuntu1.2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "sha256": "1ec03fe34a7dfd8640a0564a3cad385a10b4f68ebf2190a4c08ccadb395133d3", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libtinfo6_6.3-2ubuntu0.1_amd64.deb" + ], + "version": "6.3-2ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "sha256": "5bdd1506004599dca5f659e43b09fa419565168d2b4a5ea4ba11c9e830eae53f", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/s/sqlite3/libsqlite3-0_3.37.2-2ubuntu0.4_amd64.deb" + ], + "version": "3.37.2-2ubuntu0.4" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "sha256": "51fd16f608efa710c987e4b071ffba0bec63a9ac8e55eddd5770f5e5252edc09", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/r/readline/libreadline8_8.1.2-1_amd64.deb" + ], + "version": "8.1.2-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "sha256": "a78d02935f6107a12d3d349c25ce4f92fcdcbf6b65e37f1f5a13a9054b5b9e94", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/r/readline/readline-common_8.1.2-1_all.deb" + ], + "version": "8.1.2-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "sha256": "cfeef478f96ace59617f4f93c2497776b98a33c99bf3602af46844ccf9cba9d3", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libn/libnsl/libnsl2_1.3.0-2build2_amd64.deb" + ], + "version": "1.3.0-2build2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "sha256": "88e5178ccdd4317e263dbe03db1a54a6e35ae1f1fd057585655c2b06ab487f6e", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libncursesw6_6.3-2ubuntu0.1_amd64.deb" + ], + "version": "6.3-2ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "sha256": "941ca0b2e73d26522f75a801b1bf529afa5ceb2ac9b00cbf324b59f474cae813", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/m/mpdecimal/libmpdec3_2.5.1-2build2_amd64.deb" + ], + "version": "2.5.1-2build2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "sha256": "83873058e692936a09649ede2f8e70b87dde1f3f5488db53da8081b81c79a5d9", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libstdc++6_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "sha256": "b4f88c91fa6f4c942097be6abfc365fb133c5e147640168cbb7704fd855eac10", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libf/libffi/libffi8_3.4.2-4_amd64.deb" + ], + "version": "3.4.2-4" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "sha256": "ce60e446c9cb0e94bb4b4d0fe4f969e772edec250deb186bacf955e4d196548b", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/d/db5.3/libdb5.3_5.3.28+dfsg1-0.8ubuntu3_amd64.deb" + ], + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "sha256": "a993af16a8ccdcefaf75eef360f295f8a3a91a78fd5e5ea1e3fd4f2376df6615", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/m/media-types/media-types_7.0.0_all.deb" + ], + "version": "7.0.0" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "sha256": "9ccaf10c03370cdaaad5182f33a10ae02d65a1aa292486d431efbccf1a207078", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3.10/libpython3.10-minimal_3.10.12-1~22.04.10_amd64.deb" + ], + "version": "3.10.12-1~22.04.10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "sha256": "72189c001d158fd4088b83389b0b0aead43ddde72176c17ead0dd72c8fcb5722", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3.10/python3.10_3.10.12-1~22.04.10_amd64.deb" + ], + "version": "3.10.12-1~22.04.10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "sha256": "2b78a48f72f3cbef21afdfd21a92423dcce4a4596413b85a8ae56cfb6bcc3e5d", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3.10/python3.10-minimal_3.10.12-1~22.04.10_amd64.deb" + ], + "version": "3.10.12-1~22.04.10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "sha256": "b3f93e3fc3f15ca134fd02ba6c59df80a8673f41b04dec9f66c63ce7183fc046", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/e/expat/libexpat1_2.4.7-1ubuntu0.6_amd64.deb" + ], + "version": "2.4.7-1ubuntu0.6" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "sha256": "c2f9ebfa720344485d192f9b493065bee4a29d5021b093f9300c8101483de9dd", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3-defaults/python3-minimal_3.10.6-1~22.04.1_amd64.deb" + ], + "version": "3.10.6-1~22.04.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "python3-setuptools_59.6.0-1.2ubuntu0.22.04.3_amd64", + "name": "python3-setuptools", + "sha256": "553fd6b076e22c35123f29b4b5ff08e762164b39c72569f05af7265571a85c13", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/s/setuptools/python3-setuptools_59.6.0-1.2ubuntu0.22.04.3_all.deb" + ], + "version": "59.6.0-1.2ubuntu0.22.04.3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "python3-distutils_3.10.8-1_22.04_amd64", + "name": "python3-distutils", + "sha256": "b2bada10bef81fa7dc4bde61a1a462f37ca5027805e7c96e58685e9034c6094f", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3-stdlib-extensions/python3-distutils_3.10.8-1~22.04_all.deb" + ], + "version": "3.10.8-1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "python3-lib2to3_3.10.8-1_22.04_amd64", + "name": "python3-lib2to3", + "sha256": "7bd212e61e8bb7fabf4768629e5dc97e6f8a7f98cc9335b5a3ce8c11cb633248", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3-stdlib-extensions/python3-lib2to3_3.10.8-1~22.04_all.deb" + ], + "version": "3.10.8-1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "python3-pkg-resources_59.6.0-1.2ubuntu0.22.04.3_amd64", + "name": "python3-pkg-resources", + "sha256": "e838b1d16912f11c8c955d99ba5b7f378025ebb29b3ea72096cc7e9ec4d5e8f8", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/s/setuptools/python3-pkg-resources_59.6.0-1.2ubuntu0.22.04.3_all.deb" + ], + "version": "59.6.0-1.2ubuntu0.22.04.3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "sudo_1.9.9-1ubuntu2.5_amd64", + "name": "sudo", + "sha256": "479c1e3247887315cc69dcec0b2a58504c77be3f45448de3a0b0e44b0f4ca775", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/s/sudo/sudo_1.9.9-1ubuntu2.5_amd64.deb" + ], + "version": "1.9.9-1ubuntu2.5" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "lsb-base_11.1.0ubuntu4_amd64", + "name": "lsb-base", + "sha256": "23ac3e99b0c2cd0179a7f40fa5a24248345e16c515f222164dad2a638b4f04cf", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/l/lsb/lsb-base_11.1.0ubuntu4_all.deb" + ], + "version": "11.1.0ubuntu4" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libpam-modules_1.4.0-11ubuntu2.6_amd64", + "name": "libpam-modules", + "sha256": "6f03b50258a365d69a037fdd9c29a6b976fa3953c84bafacfc19385cd8fe4b68", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pam/libpam-modules_1.4.0-11ubuntu2.6_amd64.deb" + ], + "version": "1.4.0-11ubuntu2.6" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libpam-modules-bin_1.4.0-11ubuntu2.6_amd64", + "name": "libpam-modules-bin", + "sha256": "3b85776823736451ec59708181ebdcdde0429ef9bdea76ffeba8f2428146331c", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pam/libpam-modules-bin_1.4.0-11ubuntu2.6_amd64.deb" + ], + "version": "1.4.0-11ubuntu2.6" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libpam0g_1.4.0-11ubuntu2.6_amd64", + "name": "libpam0g", + "sha256": "9c0e509cdf68364212910805ed673f481b7762109eb6499b68d7f9bf599b0c4a", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pam/libpam0g_1.4.0-11ubuntu2.6_amd64.deb" + ], + "version": "1.4.0-11ubuntu2.6" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libaudit1_1-3.0.7-1build1_amd64", + "name": "libaudit1", + "sha256": "36710fbc49150a13f14f1f9c2c8288ee840cf9425f8a609c22fefa6b64eb51c1", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/a/audit/libaudit1_3.0.7-1build1_amd64.deb" + ], + "version": "1:3.0.7-1build1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libcap-ng0_0.7.9-2.2build3_amd64", + "name": "libcap-ng0", + "sha256": "124d0c8748a841f279e996298ef8aac69a249e294792f4b5e16dc00496f1d3ac", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libc/libcap-ng/libcap-ng0_0.7.9-2.2build3_amd64.deb" + ], + "version": "0.7.9-2.2build3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libaudit-common_1-3.0.7-1build1_amd64", + "name": "libaudit-common", + "sha256": "86697036d05c956b5b42339bf61ecf3743f7c6fd4d59a67ab16add9a2cd7802a", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/a/audit/libaudit-common_3.0.7-1build1_all.deb" + ], + "version": "1:3.0.7-1build1" + }, { "arch": "amd64", "dependencies": [ @@ -719,424 +1566,6 @@ ], "version": "0.6.0.70201-81~22.04" }, - { - "arch": "amd64", - "dependencies": [], - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "sha256": "83873058e692936a09649ede2f8e70b87dde1f3f5488db53da8081b81c79a5d9", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libstdc++6_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "rocm-core_7.2.1.70201-81_22.04_amd64", - "name": "rocm-core", - "sha256": "b83fe314e67f6bd3cef2b5695fb6de7153110a14727add1992d0056d81a7e9a9", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocm-core/rocm-core_7.2.1.70201-81~22.04_amd64.deb" - ], - "version": "7.2.1.70201-81~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "sha256": "43dbcc12d790ed9360be4a1dc690ac0a7464a5e7c3170785886df84811a3f5a5", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3-defaults/python3_3.10.6-1~22.04.1_amd64.deb" - ], - "version": "3.10.6-1~22.04.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "sha256": "41b8f285f0a1661ece778a63592dcb556710cce3c559d83114fad245bfdf56ea", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3-defaults/libpython3-stdlib_3.10.6-1~22.04.1_amd64.deb" - ], - "version": "3.10.6-1~22.04.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "sha256": "123b338b5857a35e145c2bd7ce890434dbed2e7f6cd03505e86377f9e0aac40b", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3.10/libpython3.10-stdlib_3.10.12-1~22.04.10_amd64.deb" - ], - "version": "3.10.12-1~22.04.10" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "sha256": "3a88f3c8c58a32c623b79f1caace557f63cb997c03fa18eaa31ca0285f635919", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/u/util-linux/libuuid1_2.37.2-4ubuntu3.4_amd64.deb" - ], - "version": "2.37.2-4ubuntu3.4" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "sha256": "f2f07d1ab8ae8a8a6e893975df0bca8683caec516e7a30e8dadbb87d33aa52c4", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libt/libtirpc/libtirpc3_1.3.2-2ubuntu0.1_amd64.deb" - ], - "version": "1.3.2-2ubuntu0.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "sha256": "cd7b220447999402e87d75f7cedcfad159a2e5d4236927a414d77f2932219bee", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libt/libtirpc/libtirpc-common_1.3.2-2ubuntu0.1_all.deb" - ], - "version": "1.3.2-2ubuntu0.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "sha256": "2bc13d97c9cd1045271bf600c53ca3ae7516c61d9102f1b9c9d2fa722c35183b", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/k/krb5/libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64.deb" - ], - "version": "1.19.2-2ubuntu0.7" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "sha256": "fd2f8c23945590b38f576f7a58dcba7eb63bb6dc97d37271a15b9d8b2c29210f", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/k/krb5/libkrb5support0_1.19.2-2ubuntu0.7_amd64.deb" - ], - "version": "1.19.2-2ubuntu0.7" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "sha256": "75fc4ab539697f7df8afa2e226b39515b6425111a80aecce1ea41ce392896224", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/k/krb5/libkrb5-3_1.19.2-2ubuntu0.7_amd64.deb" - ], - "version": "1.19.2-2ubuntu0.7" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "sha256": "3a4b984a91fb0fda4d287f1aceeea0f8f1d8a4ed6c9fe7dac114208be92e0f1f", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/o/openssl/libssl3_3.0.2-0ubuntu1.19_amd64.deb" - ], - "version": "3.0.2-0ubuntu1.19" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "sha256": "395a65b3a9539304391456d4d0e5531f4249f297b6e67eb0fcaf2ff77ddc3dc3", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/d/debconf/debconf_1.5.79ubuntu1_all.deb" - ], - "version": "1.5.79ubuntu1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "sha256": "465f2523d9bbbd64c1dc9ffdb894885214723d6e8e6942b4bae6b4b8811fdaa5", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/perl/perl-base_5.34.0-3ubuntu1.4_amd64.deb" - ], - "version": "5.34.0-3ubuntu1.4" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "sha256": "92c21b47281c1bf24d788fe02d717c058fd9701b9c224b0b5c2c1179fb5d152d", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/d/dpkg/dpkg_1.21.1ubuntu2.3_amd64.deb" - ], - "version": "1.21.1ubuntu2.3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "sha256": "151e3a61137aeb04cd714656852877a00d47d918246806f9e5b08d1cef32b816", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/t/tar/tar_1.34+dfsg-1ubuntu0.1.22.04.2_amd64.deb" - ], - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "sha256": "b96c6b40ee2388bd51341cb11c0f1d5bcca29b9180b6e3a77a06b881f2913f7e", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libs/libselinux/libselinux1_3.3-1build2_amd64.deb" - ], - "version": "3.3-1build2" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "sha256": "01a5a74f3d305cad32fffffdc200c2a92fd8e5a01bff97de76ac598fb444d53e", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pcre2/libpcre2-8-0_10.39-3ubuntu0.1_amd64.deb" - ], - "version": "10.39-3ubuntu0.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "sha256": "4db2c64ec74f673ed022e92cce7b83d0cbe0b779e02ca60a56ba59ae07754e05", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/a/acl/libacl1_2.3.1-1_amd64.deb" - ], - "version": "2.3.1-1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "sha256": "ae7db00ce8b093e50c994518b90203544e063b4bc574836a048bb142b950b2c9", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libz/libzstd/libzstd1_1.4.8+dfsg-3build1_amd64.deb" - ], - "version": "1.4.8+dfsg-3build1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "sha256": "940daf78ee0229549b4eabc92ffd79dd038ed96a44e2e912305a23540cf22a0a", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/k/keyutils/libkeyutils1_1.6.1-2ubuntu3_amd64.deb" - ], - "version": "1.6.1-2ubuntu3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "sha256": "76174ecc0948b587a0e92a08b023c9f8655f581aaff476c52f50299e80e92056", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/k/krb5/libk5crypto3_1.19.2-2ubuntu0.7_amd64.deb" - ], - "version": "1.19.2-2ubuntu0.7" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "sha256": "552bc8169451cfdcf174a8da845b13ced0a55777149af5cf47491193161ed791", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/e/e2fsprogs/libcom-err2_1.46.5-2ubuntu1.2_amd64.deb" - ], - "version": "1.46.5-2ubuntu1.2" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "sha256": "1ec03fe34a7dfd8640a0564a3cad385a10b4f68ebf2190a4c08ccadb395133d3", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libtinfo6_6.3-2ubuntu0.1_amd64.deb" - ], - "version": "6.3-2ubuntu0.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "sha256": "5bdd1506004599dca5f659e43b09fa419565168d2b4a5ea4ba11c9e830eae53f", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/s/sqlite3/libsqlite3-0_3.37.2-2ubuntu0.4_amd64.deb" - ], - "version": "3.37.2-2ubuntu0.4" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "sha256": "51fd16f608efa710c987e4b071ffba0bec63a9ac8e55eddd5770f5e5252edc09", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/r/readline/libreadline8_8.1.2-1_amd64.deb" - ], - "version": "8.1.2-1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "sha256": "a78d02935f6107a12d3d349c25ce4f92fcdcbf6b65e37f1f5a13a9054b5b9e94", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/r/readline/readline-common_8.1.2-1_all.deb" - ], - "version": "8.1.2-1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "sha256": "cfeef478f96ace59617f4f93c2497776b98a33c99bf3602af46844ccf9cba9d3", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libn/libnsl/libnsl2_1.3.0-2build2_amd64.deb" - ], - "version": "1.3.0-2build2" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "sha256": "88e5178ccdd4317e263dbe03db1a54a6e35ae1f1fd057585655c2b06ab487f6e", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libncursesw6_6.3-2ubuntu0.1_amd64.deb" - ], - "version": "6.3-2ubuntu0.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "sha256": "941ca0b2e73d26522f75a801b1bf529afa5ceb2ac9b00cbf324b59f474cae813", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/m/mpdecimal/libmpdec3_2.5.1-2build2_amd64.deb" - ], - "version": "2.5.1-2build2" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "sha256": "b4f88c91fa6f4c942097be6abfc365fb133c5e147640168cbb7704fd855eac10", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libf/libffi/libffi8_3.4.2-4_amd64.deb" - ], - "version": "3.4.2-4" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "sha256": "ce60e446c9cb0e94bb4b4d0fe4f969e772edec250deb186bacf955e4d196548b", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/d/db5.3/libdb5.3_5.3.28+dfsg1-0.8ubuntu3_amd64.deb" - ], - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "sha256": "a993af16a8ccdcefaf75eef360f295f8a3a91a78fd5e5ea1e3fd4f2376df6615", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/m/media-types/media-types_7.0.0_all.deb" - ], - "version": "7.0.0" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "sha256": "9ccaf10c03370cdaaad5182f33a10ae02d65a1aa292486d431efbccf1a207078", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3.10/libpython3.10-minimal_3.10.12-1~22.04.10_amd64.deb" - ], - "version": "3.10.12-1~22.04.10" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "sha256": "72189c001d158fd4088b83389b0b0aead43ddde72176c17ead0dd72c8fcb5722", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3.10/python3.10_3.10.12-1~22.04.10_amd64.deb" - ], - "version": "3.10.12-1~22.04.10" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "sha256": "2b78a48f72f3cbef21afdfd21a92423dcce4a4596413b85a8ae56cfb6bcc3e5d", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3.10/python3.10-minimal_3.10.12-1~22.04.10_amd64.deb" - ], - "version": "3.10.12-1~22.04.10" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "sha256": "b3f93e3fc3f15ca134fd02ba6c59df80a8673f41b04dec9f66c63ce7183fc046", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/e/expat/libexpat1_2.4.7-1ubuntu0.6_amd64.deb" - ], - "version": "2.4.7-1ubuntu0.6" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "sha256": "c2f9ebfa720344485d192f9b493065bee4a29d5021b093f9300c8101483de9dd", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3-defaults/python3-minimal_3.10.6-1~22.04.1_amd64.deb" - ], - "version": "3.10.6-1~22.04.1" - }, { "arch": "amd64", "dependencies": [], diff --git a/platforms/rocm/packages.yaml b/platforms/rocm/packages.yaml index 3427388e..f7af47e1 100644 --- a/platforms/rocm/packages.yaml +++ b/platforms/rocm/packages.yaml @@ -25,6 +25,7 @@ packages: - "libdw1" - "liblzma5" # - "rocm-smi-lib" + - "amd-smi-lib" - "libdrm-amdgpu-common" # Do not use hsa-rocr from amdgpu repo because it is overriden by hsa-runtime-rocr4wsl-amdgpu - "hsa-rocr (<< 20)" diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 7c7b3973..98b6291d 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -49,6 +49,7 @@ _UBUNTU_PACKAGES = { "fopen64": "zmlxrocm_fopen64", }, ), + packages.filegroup(name = "libdrm_amdgpu", srcs = ["opt/amdgpu/lib/x86_64-linux-gnu/libdrm_amdgpu.so.1"]), ]), "libtinfo6": packages.filegroup(name = "libtinfo6", srcs = ["lib/x86_64-linux-gnu/libtinfo.so.6"]), "zlib1g": packages.filegroup(name = "zlib1g", srcs = ["lib/x86_64-linux-gnu/libz.so.1"]), @@ -64,6 +65,10 @@ _ROCM_PACKAGES = { name = "rocm_smi", src = "lib/librocm_smi64.so.1", ), + "amd-smi-lib": "\n".join([ + packages.cc_library(name = "amdsmi", hdrs = ["include/amd_smi/amdsmi.h"], includes = ["include/amd_smi"]), + packages.filegroup(name = "libamd_smi", srcs = ["lib/libamd_smi.so.26"]), + ]), "rocprofiler-sdk": "\n".join([ packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), packages.patchelf( @@ -132,7 +137,7 @@ _ROCM_PACKAGES = { name = "runfiles", srcs = glob(["lib/rocblas/library/**"]), ) - """ + """, ]), "rocfft": packages.filegroup(name = "rocfft", srcs = ["lib/librocfft.so.0"]), "rocsolver": _rocm_dlopen_patchelf( @@ -183,7 +188,7 @@ _ROCM_PACKAGES = { name = "runfiles", srcs = glob(["lib/hipblaslt/library/**"]), ) - """ + """, ]), "hipfft": packages.filegroup(name = "hipfft", srcs = ["lib/libhipfft.so.0"]), "hip-runtime-amd": "\n".join([ @@ -255,7 +260,7 @@ def _rocm_impl(mctx): return mctx.extension_metadata( reproducible = True, - root_module_direct_deps = ["libpjrt_rocm", "hipblaslt", "rocblas"], + root_module_direct_deps = ["amd-smi-lib", "libpjrt_rocm", "libdrm2-amdgpu", "libdrm-amdgpu-amdgpu1", "hipblaslt", "rocblas"], root_module_direct_dev_deps = [], ) diff --git a/third_party/cloud_accelerator_diagnostics/BUILD.bazel b/third_party/cloud_accelerator_diagnostics/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/third_party/cloud_accelerator_diagnostics/cloud_accelerator_diagnostics.BUILD.bazel b/third_party/cloud_accelerator_diagnostics/cloud_accelerator_diagnostics.BUILD.bazel new file mode 100644 index 00000000..563999d2 --- /dev/null +++ b/third_party/cloud_accelerator_diagnostics/cloud_accelerator_diagnostics.BUILD.bazel @@ -0,0 +1,34 @@ +load("@com_google_protobuf//bazel:cc_proto_library.bzl", "cc_proto_library") +load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") +load("@grpc//bazel:cc_grpc_library.bzl", "cc_grpc_library") + +proto_library( + name = "tpu_telemetry_proto", + srcs = ["tpu_telemetry.proto"], + import_prefix = "tpu_info/proto", + visibility = ["//visibility:public"], +) + +proto_library( + name = "tpu_metric_service_proto", + srcs = ["tpu_metric_service.proto"], + deps = [ + ":tpu_telemetry_proto", + "@com_google_protobuf//:timestamp_proto", + ], + visibility = ["//visibility:public"], +) + +cc_proto_library( + name = "tpu_metric_service_cc_proto", + deps = [":tpu_metric_service_proto"], + visibility = ["//visibility:public"], +) + +cc_grpc_library( + name = "tpu_metric_service_cc_grpc", + srcs = [":tpu_metric_service_proto"], + grpc_only = True, + deps = [":tpu_metric_service_cc_proto"], + visibility = ["//visibility:public"], +) diff --git a/third_party/cloud_accelerator_diagnostics/repo.bzl b/third_party/cloud_accelerator_diagnostics/repo.bzl new file mode 100644 index 00000000..661e45ec --- /dev/null +++ b/third_party/cloud_accelerator_diagnostics/repo.bzl @@ -0,0 +1,29 @@ +_COMMIT = "6eccaf47364904d1c55209dd8795e3bf9cac21e1" +_URL_PREFIX = "https://raw.githubusercontent.com/AI-Hypercomputer/cloud-accelerator-diagnostics/" + _COMMIT + "/tpu_info/tpu_info/proto/" + +_PROTOS = { + "tpu_metric_service.proto": "a89d80938c75177ff45fb4c6130f5d71bc5164df473a77948e31b7dedd96a1bb", + "tpu_telemetry.proto": "4ad6096e40de5ba00b6c7badea1218c801069b82be829ec56dc0ce693b5a2505", +} + +def _impl(ctx): + for output, sha256 in _PROTOS.items(): + ctx.download( + url = _URL_PREFIX + output, + sha256 = sha256, + output = output, + ) + ctx.template("BUILD.bazel", ctx.attr.build_file) + +_cloud_accelerator_diagnostics = repository_rule( + implementation = _impl, + attrs = { + "build_file": attr.label(mandatory = True), + }, +) + +def repo(): + _cloud_accelerator_diagnostics( + name = "cloud_accelerator_diagnostics", + build_file = "//third_party/cloud_accelerator_diagnostics:cloud_accelerator_diagnostics.BUILD.bazel", + ) diff --git a/third_party/libvaxis/BUILD.bazel b/third_party/libvaxis/BUILD.bazel new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/third_party/libvaxis/BUILD.bazel @@ -0,0 +1 @@ + diff --git a/third_party/libvaxis/bump-zig.patch b/third_party/libvaxis/bump-zig.patch new file mode 100644 index 00000000..45e221b7 --- /dev/null +++ b/third_party/libvaxis/bump-zig.patch @@ -0,0 +1,2019 @@ +diff --git a/build.zig.zon b/build.zig.zon +index 6192c1a..4ca04f8 100644 +--- a/build.zig.zon ++++ b/build.zig.zon +@@ -2,15 +2,15 @@ + .name = .vaxis, + .fingerprint = 0x14fbbb94fc556305, + .version = "0.5.1", +- .minimum_zig_version = "0.15.1", ++ .minimum_zig_version = "0.16.0-dev.2261+d6b3dd25a", + .dependencies = .{ + .zigimg = .{ +- .url = "git+https://github.com/zigimg/zigimg#eab2522c023b9259db8b13f2f90d609b7437e5f6", +- .hash = "zigimg-0.1.0-8_eo2vUZFgAAtN1c6dAO5DdqL0d4cEWHtn6iR5ucZJti", ++ .url = "https://github.com/unexge/zigimg/archive/refs/heads/master.zip", ++ .hash = "zigimg-0.1.0-8_eo2tGrFwD-VzWa_E8HuIuNenM2US7PRXBj5EJdGfrw", + }, + .uucode = .{ +- .url = "git+https://github.com/jacobsandlund/uucode#5f05f8f83a75caea201f12cc8ea32a2d82ea9732", +- .hash = "uucode-0.1.0-ZZjBPj96QADXyt5sqwBJUnhaDYs_qBeeKijZvlRa0eqM", ++ .url = "https://github.com/jacobsandlund/uucode/archive/refs/heads/zig-0.16.zip", ++ .hash = "uucode-0.2.0-ZZjBPkJiVABQGPdyw9BOtU2sflHv3vovBzmYtPRj2hvm", + }, + }, + .paths = .{ +diff --git a/examples/cli.zig b/examples/cli.zig +index 3b44676..2b9a288 100644 +--- a/examples/cli.zig ++++ b/examples/cli.zig +@@ -4,30 +4,23 @@ const Cell = vaxis.Cell; + const TextInput = vaxis.widgets.TextInput; + + const log = std.log.scoped(.main); +-pub fn main() !void { +- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +- defer { +- const deinit_status = gpa.deinit(); +- if (deinit_status == .leak) { +- log.err("memory leak", .{}); +- } +- } +- const alloc = gpa.allocator(); ++pub fn main(init: std.process.Init) !void { ++ const alloc = init.gpa; + + var buffer: [1024]u8 = undefined; +- var tty = try vaxis.Tty.init(&buffer); ++ var tty = try vaxis.Tty.init(init.io, &buffer); + defer tty.deinit(); + + var vx = try vaxis.init(alloc, .{}); + defer vx.deinit(alloc, tty.writer()); + +- var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx }; ++ var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx, .io = init.io, .queue = .{ .io = init.io } }; + try loop.init(); + + try loop.start(); + defer loop.stop(); + +- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_s); ++ try vx.queryTerminal(tty.writer(), init.io, 1 * std.time.ns_per_s); + + var text_input = TextInput.init(alloc); + defer text_input.deinit(); +diff --git a/examples/counter.zig b/examples/counter.zig +index d19af42..e68420b 100644 +--- a/examples/counter.zig ++++ b/examples/counter.zig +@@ -111,13 +111,10 @@ const Model = struct { + } + }; + +-pub fn main() !void { +- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +- defer _ = gpa.deinit(); ++pub fn main(init: std.process.Init) !void { ++ const allocator = init.gpa; + +- const allocator = gpa.allocator(); +- +- var app = try vxfw.App.init(allocator); ++ var app = try vxfw.App.init(allocator, init.io); + defer app.deinit(); + + // We heap allocate our model because we will require a stable pointer to it in our Button +diff --git a/examples/fuzzy.zig b/examples/fuzzy.zig +index 6ffbe6a..7fef980 100644 +--- a/examples/fuzzy.zig ++++ b/examples/fuzzy.zig +@@ -11,7 +11,6 @@ const Model = struct { + + /// Used for filtered RichText Spans and result + arena: std.heap.ArenaAllocator, +- filtered: std.ArrayList(vxfw.RichText), + result: []const u8, + + pub fn init(gpa: std.mem.Allocator) !*Model { +@@ -201,31 +200,24 @@ fn toLower(arena: std.mem.Allocator, src: []const u8) std.mem.Allocator.Error![] + return lower; + } + +-pub fn main() !void { +- var debug_allocator = std.heap.GeneralPurposeAllocator(.{}){}; +- defer _ = debug_allocator.deinit(); +- +- const gpa = debug_allocator.allocator(); ++pub fn main(init: std.process.Init) !void { ++ const gpa = init.gpa; + +- var app = try vxfw.App.init(gpa); ++ var app = try vxfw.App.init(gpa, init.io); + errdefer app.deinit(); + + const model = try Model.init(gpa); + defer model.deinit(gpa); + + // Run the command +- var fd = std.process.Child.init(&.{"fd"}, gpa); +- fd.stdout_behavior = .Pipe; +- fd.stderr_behavior = .Pipe; +- var stdout = std.ArrayList(u8).empty; +- var stderr = std.ArrayList(u8).empty; +- defer stdout.deinit(gpa); +- defer stderr.deinit(gpa); +- try fd.spawn(); +- try fd.collectOutput(gpa, &stdout, &stderr, 10_000_000); +- _ = try fd.wait(); +- +- var iter = std.mem.splitScalar(u8, stdout.items, '\n'); ++ const result = try std.process.run(gpa, init.io, .{ ++ .argv = &.{"fd"}, ++ .stdout_limit = .limited(10_000_000), ++ }); ++ defer gpa.free(result.stdout); ++ defer gpa.free(result.stderr); ++ ++ var iter = std.mem.splitScalar(u8, result.stdout, '\n'); + while (iter.next()) |line| { + if (line.len == 0) continue; + try model.list.append(gpa, .{ .text = line }); +@@ -235,8 +227,8 @@ pub fn main() !void { + app.deinit(); + + if (model.result.len > 0) { +- _ = try std.posix.write(std.posix.STDOUT_FILENO, model.result); +- _ = try std.posix.write(std.posix.STDOUT_FILENO, "\n"); ++ try std.Io.File.stdout().writeStreamingAll(init.io, model.result); ++ try std.Io.File.stdout().writeStreamingAll(init.io, "\n"); + } else { + std.process.exit(130); + } +diff --git a/examples/image.zig b/examples/image.zig +index c14bbdb..af62862 100644 +--- a/examples/image.zig ++++ b/examples/image.zig +@@ -8,41 +8,34 @@ const Event = union(enum) { + winsize: vaxis.Winsize, + }; + +-pub fn main() !void { +- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +- defer { +- const deinit_status = gpa.deinit(); +- if (deinit_status == .leak) { +- log.err("memory leak", .{}); +- } +- } +- const alloc = gpa.allocator(); ++pub fn main(init: std.process.Init) !void { ++ const alloc = init.gpa; + + var buffer: [1024]u8 = undefined; +- var tty = try vaxis.Tty.init(&buffer); ++ var tty = try vaxis.Tty.init(init.io, &buffer); + defer tty.deinit(); + + var vx = try vaxis.init(alloc, .{}); + defer vx.deinit(alloc, tty.writer()); + +- var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx }; ++ var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx, .io = init.io, .queue = .{ .io = init.io } }; + try loop.init(); + + try loop.start(); + defer loop.stop(); + + try vx.enterAltScreen(tty.writer()); +- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_s); ++ try vx.queryTerminal(tty.writer(), init.io, 1 * std.time.ns_per_s); + + var read_buffer: [1024 * 1024]u8 = undefined; // 1MB buffer +- var img1 = try vaxis.zigimg.Image.fromFilePath(alloc, "examples/zig.png", &read_buffer); ++ var img1 = try vaxis.zigimg.Image.fromFilePath(alloc, init.io, "examples/zig.png", &read_buffer); + defer img1.deinit(alloc); + + const imgs = [_]vaxis.Image{ + try vx.transmitImage(alloc, tty.writer(), &img1, .rgba), + // var img1 = try vaxis.zigimg.Image.fromFilePath(alloc, "examples/zig.png"); +- // try vx.loadImage(alloc, tty.writer(), .{ .path = "examples/zig.png" }), +- try vx.loadImage(alloc, tty.writer(), .{ .path = "examples/vaxis.png" }), ++ // try vx.loadImage(alloc, init.io, tty.writer(), .{ .path = "examples/zig.png" }), ++ try vx.loadImage(alloc, init.io, tty.writer(), .{ .path = "examples/vaxis.png" }), + }; + defer vx.freeImage(tty.writer(), imgs[0].id); + defer vx.freeImage(tty.writer(), imgs[1].id); +diff --git a/examples/list_view.zig b/examples/list_view.zig +index 24cbf8d..5b961e9 100644 +--- a/examples/list_view.zig ++++ b/examples/list_view.zig +@@ -52,13 +52,10 @@ const Model = struct { + } + }; + +-pub fn main() !void { +- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +- defer _ = gpa.deinit(); ++pub fn main(init: std.process.Init) !void { ++ const allocator = init.gpa; + +- const allocator = gpa.allocator(); +- +- var app = try vxfw.App.init(allocator); ++ var app = try vxfw.App.init(allocator, init.io); + defer app.deinit(); + + const model = try allocator.create(Model); +diff --git a/examples/main.zig b/examples/main.zig +index 2a4b96b..3caa7d9 100644 +--- a/examples/main.zig ++++ b/examples/main.zig +@@ -3,25 +3,17 @@ const vaxis = @import("vaxis"); + const Cell = vaxis.Cell; + + const log = std.log.scoped(.main); +-pub fn main() !void { +- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +- defer { +- const deinit_status = gpa.deinit(); +- //fail test; can't try in defer as defer is executed after we return +- if (deinit_status == .leak) { +- log.err("memory leak", .{}); +- } +- } +- const alloc = gpa.allocator(); ++pub fn main(init: std.process.Init) !void { ++ const alloc = init.gpa; + + var buffer: [1024]u8 = undefined; +- var tty = try vaxis.Tty.init(&buffer); ++ var tty = try vaxis.Tty.init(init.io, &buffer); + defer tty.deinit(); + + var vx = try vaxis.init(alloc, .{}); + defer vx.deinit(alloc, tty.writer()); + +- var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx }; ++ var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx, .io = init.io, .queue = .{ .io = init.io } }; + try loop.init(); + + try loop.start(); +@@ -29,7 +21,7 @@ pub fn main() !void { + + // Optionally enter the alternate screen + try vx.enterAltScreen(tty.writer()); +- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_s); ++ try vx.queryTerminal(tty.writer(), init.io, 1 * std.time.ns_per_s); + + // We'll adjust the color index every keypress + var color_idx: u8 = 0; +diff --git a/examples/scroll.zig b/examples/scroll.zig +index 0f92573..adf1f44 100644 +--- a/examples/scroll.zig ++++ b/examples/scroll.zig +@@ -151,13 +151,10 @@ const Model = struct { + } + }; + +-pub fn main() !void { +- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +- defer _ = gpa.deinit(); ++pub fn main(init: std.process.Init) !void { ++ const allocator = init.gpa; + +- const allocator = gpa.allocator(); +- +- var app = try vxfw.App.init(allocator); ++ var app = try vxfw.App.init(allocator, init.io); + errdefer app.deinit(); + + var arena = std.heap.ArenaAllocator.init(allocator); +diff --git a/examples/split_view.zig b/examples/split_view.zig +index 78d4e6f..5c48432 100644 +--- a/examples/split_view.zig ++++ b/examples/split_view.zig +@@ -49,13 +49,10 @@ const Model = struct { + } + }; + +-pub fn main() !void { +- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +- defer _ = gpa.deinit(); ++pub fn main(init: std.process.Init) !void { ++ const allocator = init.gpa; + +- const allocator = gpa.allocator(); +- +- var app = try vxfw.App.init(allocator); ++ var app = try vxfw.App.init(allocator, init.io); + defer app.deinit(); + + const model = try allocator.create(Model); +diff --git a/examples/table.zig b/examples/table.zig +index 430c3d5..c489828 100644 +--- a/examples/table.zig ++++ b/examples/table.zig +@@ -14,16 +14,14 @@ const ActiveSection = enum { + btm, + }; + +-pub fn main() !void { +- var gpa = heap.GeneralPurposeAllocator(.{}){}; +- defer if (gpa.detectLeaks()) log.err("Memory leak detected!", .{}); +- const alloc = gpa.allocator(); ++pub fn main(init: std.process.Init) !void { ++ const alloc = init.gpa; + + // Users set up below the main function + const users_buf = try alloc.dupe(User, users[0..]); + + var buffer: [1024]u8 = undefined; +- var tty = try vaxis.Tty.init(&buffer); ++ var tty = try vaxis.Tty.init(init.io, &buffer); + defer tty.deinit(); + const tty_writer = tty.writer(); + var vx = try vaxis.init(alloc, .{ +@@ -35,12 +33,12 @@ pub fn main() !void { + key_press: vaxis.Key, + winsize: vaxis.Winsize, + table_upd, +- }) = .{ .tty = &tty, .vaxis = &vx }; ++ }) = .{ .tty = &tty, .vaxis = &vx, .io = init.io, .queue = .{ .io = init.io } }; + try loop.init(); + try loop.start(); + defer loop.stop(); + try vx.enterAltScreen(tty.writer()); +- try vx.queryTerminal(tty.writer(), 250 * std.time.ns_per_ms); ++ try vx.queryTerminal(tty.writer(), init.io, 250 * std.time.ns_per_ms); + + const logo = + \\░█░█░█▀█░█░█░▀█▀░█▀▀░░░▀█▀░█▀█░█▀▄░█░░░█▀▀░ +diff --git a/examples/text_input.zig b/examples/text_input.zig +index cea0466..5c3c6c6 100644 +--- a/examples/text_input.zig ++++ b/examples/text_input.zig +@@ -18,20 +18,12 @@ const Event = union(enum) { + foo: u8, + }; + +-pub fn main() !void { +- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +- defer { +- const deinit_status = gpa.deinit(); +- //fail test; can't try in defer as defer is executed after we return +- if (deinit_status == .leak) { +- log.err("memory leak", .{}); +- } +- } +- const alloc = gpa.allocator(); ++pub fn main(init: std.process.Init) !void { ++ const alloc = init.gpa; + + // Initalize a tty + var buffer: [1024]u8 = undefined; +- var tty = try vaxis.Tty.init(&buffer); ++ var tty = try vaxis.Tty.init(init.io, &buffer); + defer tty.deinit(); + + // Use a buffered writer for better performance. There are a lot of writes +@@ -47,6 +39,8 @@ pub fn main() !void { + var loop: vaxis.Loop(Event) = .{ + .vaxis = &vx, + .tty = &tty, ++ .io = init.io, ++ .queue = .{ .io = init.io }, + }; + try loop.init(); + +@@ -71,7 +65,7 @@ pub fn main() !void { + try writer.flush(); + // Sends queries to terminal to detect certain features. This should + // _always_ be called, but is left to the application to decide when +- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_s); ++ try vx.queryTerminal(tty.writer(), init.io, 1 * std.time.ns_per_s); + + // The main event loop. Vaxis provides a thread safe, blocking, buffered + // queue which can serve as the primary event queue for an application +@@ -94,8 +88,10 @@ pub fn main() !void { + } else if (key.matches('n', .{ .ctrl = true })) { + try vx.notify(tty.writer(), "vaxis", "hello from vaxis"); + loop.stop(); +- var child = std.process.Child.init(&.{"nvim"}, alloc); +- _ = try child.spawnAndWait(); ++ var child = try std.process.spawn(init.io, .{ ++ .argv = &.{"nvim"}, ++ }); ++ _ = try child.wait(init.io); + try loop.start(); + try vx.enterAltScreen(tty.writer()); + vx.queueRefresh(); +diff --git a/examples/text_view.zig b/examples/text_view.zig +index 73e8e54..d49cb5b 100644 +--- a/examples/text_view.zig ++++ b/examples/text_view.zig +@@ -9,31 +9,24 @@ const Event = union(enum) { + winsize: vaxis.Winsize, + }; + +-pub fn main() !void { +- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +- +- defer { +- const deinit_status = gpa.deinit(); +- if (deinit_status == .leak) { +- log.err("memory leak", .{}); +- } +- } +- +- const alloc = gpa.allocator(); ++pub fn main(init: std.process.Init) !void { ++ const alloc = init.gpa; + var buffer: [1024]u8 = undefined; +- var tty = try vaxis.Tty.init(&buffer); ++ var tty = try vaxis.Tty.init(init.io, &buffer); + defer tty.deinit(); + var vx = try vaxis.init(alloc, .{}); + defer vx.deinit(alloc, tty.writer()); + var loop: vaxis.Loop(Event) = .{ + .vaxis = &vx, + .tty = &tty, ++ .io = init.io, ++ .queue = .{ .io = init.io }, + }; + try loop.init(); + try loop.start(); + defer loop.stop(); + try vx.enterAltScreen(tty.writer()); +- try vx.queryTerminal(tty.writer(), 20 * std.time.ns_per_s); ++ try vx.queryTerminal(tty.writer(), init.io, 20 * std.time.ns_per_s); + var text_view = TextView{}; + var text_view_buffer = TextView.Buffer{}; + defer text_view_buffer.deinit(alloc); +@@ -61,6 +54,6 @@ pub fn main() !void { + win.clear(); + text_view.draw(win, text_view_buffer); + try vx.render(tty.writer()); +- try tty.writer.flush(); ++ try tty.writer().flush(); + } + } +diff --git a/examples/vaxis.zig b/examples/vaxis.zig +index 5ca566c..8a12a66 100644 +--- a/examples/vaxis.zig ++++ b/examples/vaxis.zig +@@ -9,32 +9,24 @@ const Event = union(enum) { + + pub const panic = vaxis.panic_handler; + +-pub fn main() !void { +- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +- defer { +- const deinit_status = gpa.deinit(); +- //fail test; can't try in defer as defer is executed after we return +- if (deinit_status == .leak) { +- std.log.err("memory leak", .{}); +- } +- } +- const alloc = gpa.allocator(); ++pub fn main(init: std.process.Init) !void { ++ const alloc = init.gpa; + + var buffer: [1024]u8 = undefined; +- var tty = try vaxis.Tty.init(&buffer); ++ var tty = try vaxis.Tty.init(init.io, &buffer); + defer tty.deinit(); + + var vx = try vaxis.init(alloc, .{}); + defer vx.deinit(alloc, tty.writer()); + +- var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx }; ++ var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx, .io = init.io, .queue = .{ .io = init.io } }; + try loop.init(); + + try loop.start(); + defer loop.stop(); + + try vx.enterAltScreen(tty.writer()); +- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_s); ++ try vx.queryTerminal(tty.writer(), init.io, 1 * std.time.ns_per_s); + + try vx.queryColor(tty.writer(), .fg); + try vx.queryColor(tty.writer(), .bg); +@@ -84,7 +76,7 @@ pub fn main() !void { + // try vx.render(bw.writer().any()); + // try bw.flush(); + try vx.render(tty.writer()); +- std.Thread.sleep(16 * std.time.ns_per_ms); ++ try std.Io.sleep(init.io, std.Io.Duration.fromMilliseconds(16), .real); + switch (dir) { + .up => { + pct += 1; +diff --git a/examples/view.zig b/examples/view.zig +index 1b01dfd..3f5108a 100644 +--- a/examples/view.zig ++++ b/examples/view.zig +@@ -13,15 +13,8 @@ const Event = union(enum) { + winsize: vaxis.Winsize, + }; + +-pub fn main() !void { +- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +- defer { +- const deinit_status = gpa.deinit(); +- if (deinit_status == .leak) { +- log.err("memory leak", .{}); +- } +- } +- const alloc = gpa.allocator(); ++pub fn main(init: std.process.Init) !void { ++ const alloc = init.gpa; + + var world_map: []const u8 = lg_world_map; + var map_width = lg_map_width; +@@ -45,7 +38,7 @@ pub fn main() !void { + }); + + var buffer: [1024]u8 = undefined; +- var tty = try vaxis.Tty.init(&buffer); ++ var tty = try vaxis.Tty.init(init.io, &buffer); + defer tty.deinit(); + + const writer = tty.writer(); +@@ -58,13 +51,15 @@ pub fn main() !void { + var loop: vaxis.Loop(Event) = .{ + .vaxis = &vx, + .tty = &tty, ++ .io = init.io, ++ .queue = .{ .io = init.io }, + }; + try loop.init(); + try loop.start(); + defer loop.stop(); + try vx.enterAltScreen(writer); + try writer.flush(); +- try vx.queryTerminal(tty.writer(), 20 * std.time.ns_per_s); ++ try vx.queryTerminal(tty.writer(), init.io, 20 * std.time.ns_per_s); + + // Initialize Views + // - Large Map +diff --git a/examples/vt.zig b/examples/vt.zig +index fc13149..6d7f647 100644 +--- a/examples/vt.zig ++++ b/examples/vt.zig +@@ -9,33 +9,24 @@ const Event = union(enum) { + + pub const panic = vaxis.panic_handler; + +-pub fn main() !void { +- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +- defer { +- const deinit_status = gpa.deinit(); +- //fail test; can't try in defer as defer is executed after we return +- if (deinit_status == .leak) { +- std.log.err("memory leak", .{}); +- } +- } +- const alloc = gpa.allocator(); ++pub fn main(init: std.process.Init) !void { ++ const alloc = init.gpa; + + var buffer: [1024]u8 = undefined; +- var tty = try vaxis.Tty.init(&buffer); ++ var tty = try vaxis.Tty.init(init.io, &buffer); + const writer = tty.writer(); + var vx = try vaxis.init(alloc, .{}); + defer vx.deinit(alloc, writer); + +- var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx }; ++ var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx, .io = init.io, .queue = .{ .io = init.io } }; + try loop.init(); + + try loop.start(); + defer loop.stop(); + + try vx.enterAltScreen(writer); +- try vx.queryTerminal(writer, 1 * std.time.ns_per_s); +- var env = try std.process.getEnvMap(alloc); +- defer env.deinit(); ++ try vx.queryTerminal(writer, init.io, 1 * std.time.ns_per_s); ++ const env = init.environ_map; + + const vt_opts: vaxis.widgets.Terminal.Options = .{ + .winsize = .{ +@@ -52,8 +43,9 @@ pub fn main() !void { + var write_buf: [4096]u8 = undefined; + var vt = try vaxis.widgets.Terminal.init( + alloc, ++ init.io, + &argv, +- &env, ++ env, + vt_opts, + &write_buf, + ); +@@ -62,7 +54,7 @@ pub fn main() !void { + + var redraw: bool = false; + while (true) { +- std.Thread.sleep(8 * std.time.ns_per_ms); ++ std.Io.sleep(init.io, .fromMilliseconds(8), .awake) catch {}; + // try vt events first + while (vt.tryEvent()) |event| { + redraw = true; +diff --git a/src/Loop.zig b/src/Loop.zig +index a11e2f7..cd7e9d9 100644 +--- a/src/Loop.zig ++++ b/src/Loop.zig +@@ -18,8 +18,9 @@ pub fn Loop(comptime T: type) type { + + tty: *Tty, + vaxis: *Vaxis, ++ io: std.Io, + +- queue: Queue(T, 512) = .{}, ++ queue: Queue(T, 512), + thread: ?std.Thread = null, + should_quit: bool = false, + +@@ -214,7 +215,7 @@ pub fn handleEventGeneric(self: anytype, vx: *Vaxis, cache: *GraphemeCache, Even + } + }, + .cap_da1 => { +- std.Thread.Futex.wake(&vx.query_futex, 10); ++ std.Io.futexWake(self.io, u32, &vx.query_futex.raw, 10); + vx.queries_done.store(true, .unordered); + }, + .mouse => |mouse| { +@@ -359,7 +360,7 @@ pub fn handleEventGeneric(self: anytype, vx: *Vaxis, cache: *GraphemeCache, Even + vx.caps.multi_cursor = true; + }, + .cap_da1 => { +- std.Thread.Futex.wake(&vx.query_futex, 10); ++ std.Io.futexWake(self.io, u32, &vx.query_futex.raw, 10); + vx.queries_done.store(true, .unordered); + }, + .winsize => |winsize| { +@@ -386,13 +387,13 @@ test Loop { + foo: u8, + }; + +- var tty = try vaxis.Tty.init(&.{}); ++ var tty = try vaxis.Tty.init(std.testing.io); + defer tty.deinit(); + + var vx = try vaxis.init(std.testing.allocator, .{}); + defer vx.deinit(std.testing.allocator, tty.writer()); + +- var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx }; ++ var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx, .io = std.testing.io, .queue = .{ .io = std.testing.io } }; + try loop.init(); + + try loop.start(); +@@ -400,5 +401,5 @@ test Loop { + + // Optionally enter the alternate screen + try vx.enterAltScreen(tty.writer()); +- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_ms); ++ try vx.queryTerminal(tty.writer(), std.testing.io, 1 * std.time.ns_per_ms); + } +diff --git a/src/Parser.zig b/src/Parser.zig +index 6358269..1b125dd 100644 +--- a/src/Parser.zig ++++ b/src/Parser.zig +@@ -118,7 +118,7 @@ inline fn parseGround(input: []const u8) !Result { + var grapheme_len: usize = 0; + var cp_count: usize = 0; + +- while (grapheme_iter.next()) |result| { ++ while (grapheme_iter.nextCodePoint()) |result| { + cp_count += 1; + if (result.is_break) { + // Found the first grapheme boundary +diff --git a/src/Vaxis.zig b/src/Vaxis.zig +index a6a3830..2e24e9b 100644 +--- a/src/Vaxis.zig ++++ b/src/Vaxis.zig +@@ -3,7 +3,7 @@ const builtin = @import("builtin"); + const atomic = std.atomic; + const base64Encoder = std.base64.standard.Encoder; + const zigimg = @import("zigimg"); +-const IoWriter = std.io.Writer; ++const IoWriter = std.Io.Writer; + + const Cell = @import("Cell.zig"); + const Image = @import("Image.zig"); +@@ -250,10 +250,10 @@ pub fn exitAltScreen(self: *Vaxis, tty: *IoWriter) !void { + /// + /// This call will block until Vaxis.query_futex is woken up, or the timeout. + /// Event loops can wake up this futex when cap_da1 is received +-pub fn queryTerminal(self: *Vaxis, tty: *IoWriter, timeout_ns: u64) !void { ++pub fn queryTerminal(self: *Vaxis, tty: *IoWriter, io: std.Io, timeout_ns: u64) !void { + try self.queryTerminalSend(tty); + // 1 second timeout +- std.Thread.Futex.timedWait(&self.query_futex, 0, timeout_ns) catch {}; ++ std.Io.futexWaitTimeout(io, u32, &self.query_futex.raw, 0, .{ .duration = .{ .raw = .{ .nanoseconds = @intCast(timeout_ns) }, .clock = .awake } }) catch {}; + self.queries_done.store(true, .unordered); + try self.enableDetectedFeatures(tty); + } +@@ -265,7 +265,7 @@ pub fn queryTerminalSend(vx: *Vaxis, tty: *IoWriter) !void { + vx.queries_done.store(false, .unordered); + + // TODO: re-enable this +- // const colorterm = std.posix.getenv("COLORTERM") orelse ""; ++ // const colorterm = std.c.getenv("COLORTERM") orelse ""; + // if (std.mem.eql(u8, colorterm, "truecolor") or + // std.mem.eql(u8, colorterm, "24bit")) + // { +@@ -320,22 +320,22 @@ pub fn enableDetectedFeatures(self: *Vaxis, tty: *IoWriter) !void { + }, + else => { + // Apply any environment variables +- if (std.posix.getenv("TERMUX_VERSION")) |_| ++ if (std.c.getenv("TERMUX_VERSION")) |_| + self.sgr = .legacy; +- if (std.posix.getenv("VHS_RECORD")) |_| { ++ if (std.c.getenv("VHS_RECORD")) |_| { + self.caps.unicode = .wcwidth; + self.caps.kitty_keyboard = false; + self.sgr = .legacy; + } +- if (std.posix.getenv("TERM_PROGRAM")) |prg| { +- if (std.mem.eql(u8, prg, "vscode")) ++ if (std.c.getenv("TERM_PROGRAM")) |prg| { ++ if (std.mem.eql(u8, std.mem.span(prg), "vscode")) + self.sgr = .legacy; + } +- if (std.posix.getenv("VAXIS_FORCE_LEGACY_SGR")) |_| ++ if (std.c.getenv("VAXIS_FORCE_LEGACY_SGR")) |_| + self.sgr = .legacy; +- if (std.posix.getenv("VAXIS_FORCE_WCWIDTH")) |_| ++ if (std.c.getenv("VAXIS_FORCE_WCWIDTH")) |_| + self.caps.unicode = .wcwidth; +- if (std.posix.getenv("VAXIS_FORCE_UNICODE")) |_| ++ if (std.c.getenv("VAXIS_FORCE_UNICODE")) |_| + self.caps.unicode = .unicode; + + // enable detected features +@@ -1067,6 +1067,7 @@ pub fn transmitImage( + pub fn loadImage( + self: *Vaxis, + alloc: std.mem.Allocator, ++ io: std.Io, + tty: *IoWriter, + src: Image.Source, + ) !Image { +@@ -1074,7 +1075,7 @@ pub fn loadImage( + + var read_buffer: [1024 * 1024]u8 = undefined; // 1MB buffer + var img = switch (src) { +- .path => |path| try zigimg.Image.fromFilePath(alloc, path, &read_buffer), ++ .path => |path| try zigimg.Image.fromFilePath(alloc, io, path, &read_buffer), + .mem => |bytes| try zigimg.Image.fromMemory(alloc, bytes), + }; + defer img.deinit(alloc); +@@ -1481,7 +1482,7 @@ pub fn setTerminalWorkingDirectory(_: *Vaxis, tty: *IoWriter, path: []const u8) + return error.InvalidAbsolutePath; + const hostname = switch (builtin.os.tag) { + .windows => null, +- else => std.posix.getenv("HOSTNAME"), ++ else => std.c.getenv("HOSTNAME"), + } orelse "localhost"; + + const uri: std.Uri = .{ +@@ -1495,11 +1496,11 @@ pub fn setTerminalWorkingDirectory(_: *Vaxis, tty: *IoWriter, path: []const u8) + + test "render: no output when no changes" { + var vx = try Vaxis.init(std.testing.allocator, .{}); +- var deinit_writer = std.io.Writer.Allocating.init(std.testing.allocator); ++ var deinit_writer: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer deinit_writer.deinit(); + defer vx.deinit(std.testing.allocator, &deinit_writer.writer); + +- var render_writer = std.io.Writer.Allocating.init(std.testing.allocator); ++ var render_writer: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer render_writer.deinit(); + try vx.render(&render_writer.writer); + const output = try render_writer.toOwnedSlice(); +diff --git a/src/gwidth.zig b/src/gwidth.zig +index 194139f..ecffaf7 100644 +--- a/src/gwidth.zig ++++ b/src/gwidth.zig +@@ -55,10 +55,10 @@ pub fn gwidth(str: []const u8, method: Method) u16 { + var grapheme_start: usize = 0; + var prev_break: bool = true; + +- while (grapheme_iter.next()) |result| { ++ while (grapheme_iter.nextCodePoint()) |result| { + if (prev_break and !result.is_break) { + // Start of a new grapheme +- const cp_len: usize = std.unicode.utf8CodepointSequenceLength(result.cp) catch 1; ++ const cp_len: usize = std.unicode.utf8CodepointSequenceLength(result.code_point) catch 1; + grapheme_start = grapheme_iter.i - cp_len; + } + +diff --git a/src/queue.zig b/src/queue.zig +index 6d847ea..908cf70 100644 +--- a/src/queue.zig ++++ b/src/queue.zig +@@ -1,7 +1,7 @@ + const std = @import("std"); + const assert = std.debug.assert; + const atomic = std.atomic; +-const Condition = std.Thread.Condition; ++const Condition = std.Io.Condition; + + /// Thread safe. Fixed size. Blocking push and pop. + pub fn Queue( +@@ -11,23 +11,25 @@ pub fn Queue( + return struct { + buf: [size]T = undefined, + ++ io: std.Io, ++ + read_index: usize = 0, + write_index: usize = 0, + +- mutex: std.Thread.Mutex = .{}, ++ mutex: std.Io.Mutex = .init, + // blocks when the buffer is full +- not_full: Condition = .{}, ++ not_full: Condition = .init, + // ...or empty +- not_empty: Condition = .{}, ++ not_empty: Condition = .init, + + const Self = @This(); + + /// Pop an item from the queue. Blocks until an item is available. + pub fn pop(self: *Self) T { +- self.mutex.lock(); +- defer self.mutex.unlock(); ++ self.mutex.lockUncancelable(self.io); ++ defer self.mutex.unlock(self.io); + while (self.isEmptyLH()) { +- self.not_empty.wait(&self.mutex); ++ self.not_empty.waitUncancelable(self.io, &self.mutex); + } + std.debug.assert(!self.isEmptyLH()); + return self.popAndSignalLH(); +@@ -36,10 +38,10 @@ pub fn Queue( + /// Push an item into the queue. Blocks until an item has been + /// put in the queue. + pub fn push(self: *Self, item: T) void { +- self.mutex.lock(); +- defer self.mutex.unlock(); ++ self.mutex.lockUncancelable(self.io); ++ defer self.mutex.unlock(self.io); + while (self.isFullLH()) { +- self.not_full.wait(&self.mutex); ++ self.not_full.waitUncancelable(self.io, &self.mutex); + } + std.debug.assert(!self.isFullLH()); + self.pushAndSignalLH(item); +@@ -49,8 +51,8 @@ pub fn Queue( + /// was successfully placed in the queue, false if the queue + /// was full. + pub fn tryPush(self: *Self, item: T) bool { +- self.mutex.lock(); +- defer self.mutex.unlock(); ++ self.mutex.lockUncancelable(self.io); ++ defer self.mutex.unlock(self.io); + if (self.isFullLH()) return false; + self.pushAndSignalLH(item); + return true; +@@ -59,28 +61,28 @@ pub fn Queue( + /// Pop an item from the queue. Returns null when no item is + /// available. + pub fn tryPop(self: *Self) ?T { +- self.mutex.lock(); +- defer self.mutex.unlock(); ++ self.mutex.lockUncancelable(self.io); ++ defer self.mutex.unlock(self.io); + if (self.isEmptyLH()) return null; + return self.popAndSignalLH(); + } + + /// Poll the queue. This call blocks until events are in the queue + pub fn poll(self: *Self) void { +- self.mutex.lock(); +- defer self.mutex.unlock(); ++ self.mutex.lockUncancelable(self.io); ++ defer self.mutex.unlock(self.io); + while (self.isEmptyLH()) { +- self.not_empty.wait(&self.mutex); ++ self.not_empty.waitUncancelable(self.io, &self.mutex); + } + std.debug.assert(!self.isEmptyLH()); + } + + pub fn lock(self: *Self) void { +- self.mutex.lock(); ++ self.mutex.lockUncancelable(self.io); + } + + pub fn unlock(self: *Self) void { +- self.mutex.unlock(); ++ self.mutex.unlock(self.io); + } + + /// Used to efficiently drain the queue while the lock is externally held +@@ -92,7 +94,7 @@ pub fn Queue( + const was_full = self.isFullLH(); + const item = self.popLH(); + if (was_full) { +- self.not_full.signal(); ++ self.not_full.signal(self.io); + } + return item; + } +@@ -108,15 +110,15 @@ pub fn Queue( + + /// Returns `true` if the queue is empty and `false` otherwise. + pub fn isEmpty(self: *Self) bool { +- self.mutex.lock(); +- defer self.mutex.unlock(); ++ self.mutex.lockUncancelable(self.io); ++ defer self.mutex.unlock(self.io); + return self.isEmptyLH(); + } + + /// Returns `true` if the queue is full and `false` otherwise. + pub fn isFull(self: *Self) bool { +- self.mutex.lock(); +- defer self.mutex.unlock(); ++ self.mutex.lockUncancelable(self.io); ++ defer self.mutex.unlock(self.io); + return self.isFullLH(); + } + +@@ -143,7 +145,7 @@ pub fn Queue( + self.buf[self.mask(self.write_index)] = item; + self.write_index = self.mask2(self.write_index + 1); + if (was_empty) { +- self.not_empty.signal(); ++ self.not_empty.signal(self.io); + } + } + +@@ -151,7 +153,7 @@ pub fn Queue( + const was_full = self.isFullLH(); + const result = self.popLH(); + if (was_full) { +- self.not_full.signal(); ++ self.not_full.signal(self.io); + } + return result; + } +@@ -167,7 +169,7 @@ pub fn Queue( + const testing = std.testing; + const cfg = Thread.SpawnConfig{ .allocator = testing.allocator }; + test "Queue: simple push / pop" { +- var queue: Queue(u8, 16) = .{}; ++ var queue: Queue(u8, 16) = .{ .io = std.testing.io }; + queue.push(1); + queue.push(2); + const pop = queue.pop(); +@@ -182,7 +184,7 @@ fn testPushPop(q: *Queue(u8, 2)) !void { + } + + test "Fill, wait to push, pop once in another thread" { +- var queue: Queue(u8, 2) = .{}; ++ var queue: Queue(u8, 2) = .{ .io = std.testing.io }; + queue.push(1); + queue.push(2); + const t = try Thread.spawn(cfg, testPushPop, .{&queue}); +@@ -202,7 +204,7 @@ fn testPush(q: *Queue(u8, 2)) void { + } + + test "Try to pop, fill from another thread" { +- var queue: Queue(u8, 2) = .{}; ++ var queue: Queue(u8, 2) = .{ .io = std.testing.io }; + const thread = try Thread.spawn(cfg, testPush, .{&queue}); + for (0..5) |idx| { + try testing.expectEqual(@as(u8, @intCast(idx)), queue.pop()); +@@ -217,8 +219,8 @@ fn sleepyPop(q: *Queue(u8, 2), state: *atomic.Value(u8)) !void { + + // Then we spuriously wake it up, because that's a thing that can + // happen. +- q.not_full.signal(); +- q.not_empty.signal(); ++ q.not_full.signal(q.io); ++ q.not_empty.signal(q.io); + + // Then give the other thread a good chance of waking up. It's not + // clear that yield guarantees the other thread will be scheduled, +@@ -226,7 +228,7 @@ fn sleepyPop(q: *Queue(u8, 2), state: *atomic.Value(u8)) !void { + // still full and the push in the other thread is still blocked + // waiting for space. + try Thread.yield(); +- std.Thread.sleep(10 * std.time.ns_per_ms); ++ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; + // Finally, let that other thread go. + try std.testing.expectEqual(1, q.pop()); + +@@ -235,15 +237,15 @@ fn sleepyPop(q: *Queue(u8, 2), state: *atomic.Value(u8)) !void { + try Thread.yield(); + // But we want to ensure that there's a second push waiting, so + // here's another sleep. +- std.Thread.sleep(10 * std.time.ns_per_ms); ++ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; + + // Another spurious wake... +- q.not_full.signal(); +- q.not_empty.signal(); ++ q.not_full.signal(q.io); ++ q.not_empty.signal(q.io); + // And another chance for the other thread to see that it's + // spurious and go back to sleep. + try Thread.yield(); +- std.Thread.sleep(10 * std.time.ns_per_ms); ++ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; + + // Pop that thing and we're done. + try std.testing.expectEqual(2, q.pop()); +@@ -256,19 +258,20 @@ test "Fill, block, fill, block" { + // that too (after some time) then drain the queue. This test + // fails if the while loop in `push` is turned into an `if`. + +- var queue: Queue(u8, 2) = .{}; ++ var queue: Queue(u8, 2) = .{ .io = std.testing.io }; + var state = atomic.Value(u8).init(0); + const thread = try Thread.spawn(cfg, sleepyPop, .{ &queue, &state }); + queue.push(1); + queue.push(2); + state.store(1, .release); +- const now = std.time.milliTimestamp(); ++ const start = std.Io.Clock.Timestamp.now(std.testing.io, .awake); + queue.push(3); // This one should block. +- const then = std.time.milliTimestamp(); ++ const end = std.Io.Clock.Timestamp.now(std.testing.io, .awake); + + // Just to make sure the sleeps are yielding to this thread, make + // sure it took at least 5ms to do the push. +- try std.testing.expect(then - now > 5); ++ const elapsed = start.durationTo(end); ++ try std.testing.expect(elapsed.raw.toMilliseconds() > 5); + + state.store(2, .release); + // This should block again, waiting for the other thread. +@@ -283,14 +286,14 @@ test "Fill, block, fill, block" { + fn sleepyPush(q: *Queue(u8, 1), state: *atomic.Value(u8)) !void { + // Try to ensure the other thread has already started trying to pop. + try Thread.yield(); +- std.Thread.sleep(10 * std.time.ns_per_ms); ++ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; + + // Spurious wake +- q.not_full.signal(); +- q.not_empty.signal(); ++ q.not_full.signal(q.io); ++ q.not_empty.signal(q.io); + + try Thread.yield(); +- std.Thread.sleep(10 * std.time.ns_per_ms); ++ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; + + // Stick something in the queue so it can be popped. + q.push(1); +@@ -299,11 +302,11 @@ fn sleepyPush(q: *Queue(u8, 1), state: *atomic.Value(u8)) !void { + try Thread.yield(); + // Give the other thread time to block again. + try Thread.yield(); +- std.Thread.sleep(10 * std.time.ns_per_ms); ++ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; + + // Spurious wake +- q.not_full.signal(); +- q.not_empty.signal(); ++ q.not_full.signal(q.io); ++ q.not_empty.signal(q.io); + + q.push(2); + } +@@ -313,7 +316,7 @@ test "Drain, block, drain, block" { + // test should fail if the `while` loop in `pop` is turned into an + // `if`. + +- var queue: Queue(u8, 1) = .{}; ++ var queue: Queue(u8, 1) = .{ .io = std.testing.io }; + var state = atomic.Value(u8).init(0); + const thread = try Thread.spawn(cfg, sleepyPush, .{ &queue, &state }); + try std.testing.expectEqual(1, queue.pop()); +@@ -328,11 +331,11 @@ fn readerThread(q: *Queue(u8, 1)) !void { + + test "2 readers" { + // 2 threads read, one thread writes +- var queue: Queue(u8, 1) = .{}; ++ var queue: Queue(u8, 1) = .{ .io = std.testing.io }; + const t1 = try Thread.spawn(cfg, readerThread, .{&queue}); + const t2 = try Thread.spawn(cfg, readerThread, .{&queue}); + try Thread.yield(); +- std.Thread.sleep(10 * std.time.ns_per_ms); ++ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; + queue.push(1); + queue.push(1); + t1.join(); +@@ -344,7 +347,7 @@ fn writerThread(q: *Queue(u8, 1)) !void { + } + + test "2 writers" { +- var queue: Queue(u8, 1) = .{}; ++ var queue: Queue(u8, 1) = .{ .io = std.testing.io }; + const t1 = try Thread.spawn(cfg, writerThread, .{&queue}); + const t2 = try Thread.spawn(cfg, writerThread, .{&queue}); + +diff --git a/src/tty.zig b/src/tty.zig +index 4a3c142..c1da795 100644 +--- a/src/tty.zig ++++ b/src/tty.zig +@@ -33,8 +33,11 @@ pub const PosixTty = struct { + /// The file descriptor of the tty + fd: posix.fd_t, + ++ io: std.Io, ++ + /// File.Writer for efficient buffered writing +- tty_writer: std.fs.File.Writer, ++ tty_buffer: [1024]u8 = undefined, ++ tty_writer: ?std.Io.File.Writer = null, + + pub const SignalHandler = struct { + context: *anyopaque, +@@ -43,7 +46,8 @@ pub const PosixTty = struct { + + /// global signal handlers + var handlers: [8]SignalHandler = undefined; +- var handler_mutex: std.Thread.Mutex = .{}; ++ var handler_mutex: std.Io.Mutex = .init; ++ var handler_io: std.Io = undefined; + var handler_idx: usize = 0; + + var handler_installed: bool = false; +@@ -51,9 +55,9 @@ pub const PosixTty = struct { + /// initializes a Tty instance by opening /dev/tty and "making it raw". A + /// signal handler is installed for SIGWINCH. No callbacks are installed, be + /// sure to register a callback when initializing the event loop +- pub fn init(buffer: []u8) !PosixTty { ++ pub fn init(io: std.Io) !PosixTty { + // Open our tty +- const fd = try posix.open("/dev/tty", .{ .ACCMODE = .RDWR }, 0); ++ const fd = try posix.openat(posix.AT.FDCWD, "/dev/tty", .{ .ACCMODE = .RDWR }, 0); + + // Set the termios of the tty + const termios = try makeRaw(fd); +@@ -69,13 +73,8 @@ pub const PosixTty = struct { + posix.sigaction(posix.SIG.WINCH, &act, null); + handler_installed = true; + +- const file = std.fs.File{ .handle = fd }; +- +- const self: PosixTty = .{ +- .fd = fd, +- .termios = termios, +- .tty_writer = .initStreaming(file, buffer), +- }; ++ handler_io = io; ++ const self: PosixTty = .{ .fd = fd, .io = io, .termios = termios }; + + global_tty = self; + +@@ -88,7 +87,7 @@ pub const PosixTty = struct { + std.log.err("couldn't restore terminal: {}", .{err}); + }; + if (builtin.os.tag != .macos) // closing /dev/tty may block indefinitely on macos +- posix.close(self.fd); ++ (std.Io.File{ .handle = self.fd, .flags = .{ .nonblocking = false } }).close(handler_io); + } + + /// Resets the signal handler to it's default +@@ -107,7 +106,12 @@ pub const PosixTty = struct { + } + + pub fn writer(self: *PosixTty) *std.Io.Writer { +- return &self.tty_writer.interface; ++ if (self.tty_writer == null) { ++ const file: std.Io.File = .{ .handle = self.fd, .flags = .{ .nonblocking = false } }; ++ self.tty_writer = .initStreaming(file, self.io, &self.tty_buffer); ++ } ++ ++ return &self.tty_writer.?.interface; + } + + pub fn read(self: *const PosixTty, buf: []u8) !usize { +@@ -117,16 +121,16 @@ pub const PosixTty = struct { + /// Install a signal handler for winsize. A maximum of 8 handlers may be + /// installed + pub fn notifyWinsize(handler: SignalHandler) !void { +- handler_mutex.lock(); +- defer handler_mutex.unlock(); ++ handler_mutex.lockUncancelable(handler_io); ++ defer handler_mutex.unlock(handler_io); + if (handler_idx == handlers.len) return error.OutOfMemory; + handlers[handler_idx] = handler; + handler_idx += 1; + } + +- fn handleWinch(_: c_int) callconv(.c) void { +- handler_mutex.lock(); +- defer handler_mutex.unlock(); ++ fn handleWinch(_: posix.SIG) callconv(.c) void { ++ if (!handler_mutex.tryLock()) return; ++ defer handler_mutex.unlock(handler_io); + var i: usize = 0; + while (i < handler_idx) : (i += 1) { + const handler = handlers[i]; +@@ -189,6 +193,7 @@ pub const PosixTty = struct { + pub const WindowsTty = struct { + stdin: windows.HANDLE, + stdout: windows.HANDLE, ++ io: std.Io, + + initial_codepage: c_uint, + initial_input_mode: CONSOLE_MODE_INPUT, +@@ -198,7 +203,8 @@ pub const WindowsTty = struct { + buf: [4]u8 = undefined, + + /// File.Writer for efficient buffered writing +- tty_writer: std.fs.File.Writer, ++ tty_buffer: [1024]u8 = undefined, ++ tty_writer: ?std.Io.File.Writer = null, + + /// The last mouse button that was pressed. We store the previous state of button presses on each + /// mouse event so we can detect which button was released +@@ -221,7 +227,7 @@ pub const WindowsTty = struct { + .ENABLE_LVB_GRID_WORLDWIDE = 1, // enables reverse video and underline + }; + +- pub fn init(buffer: []u8) !Tty { ++ pub fn init(io: std.Io) !Tty { + const stdin: std.fs.File = .stdin(); + const stdout: std.fs.File = .stdout(); + +@@ -237,12 +243,12 @@ pub const WindowsTty = struct { + return windows.unexpectedError(windows.kernel32.GetLastError()); + + const self: Tty = .{ ++ .io = io, + .stdin = stdin.handle, + .stdout = stdout.handle, + .initial_codepage = initial_output_codepage, + .initial_input_mode = initial_input_mode, + .initial_output_mode = initial_output_mode, +- .tty_writer = .initStreaming(stdout, buffer), + }; + + // save a copy of this tty as the global_tty for panic handling +@@ -298,7 +304,12 @@ pub const WindowsTty = struct { + } + + pub fn writer(self: *Tty) *std.Io.Writer { +- return &self.tty_writer.interface; ++ if (self.tty_writer == null) { ++ const stdout: std.fs.File = .stdout(); ++ self.tty_writer = .initStreaming(stdout, &self.tty_buffer); ++ } ++ ++ return &self.tty_writer.?.interface; + } + + pub fn read(self: *const Tty, buf: []u8) !usize { +@@ -694,20 +705,20 @@ pub const WindowsTty = struct { + pub const TestTty = struct { + /// Used for API compat + fd: posix.fd_t, ++ io: std.Io, + pipe_read: posix.fd_t, + pipe_write: posix.fd_t, + tty_writer: *std.Io.Writer.Allocating, + + /// Initializes a TestTty. +- pub fn init(buffer: []u8) !TestTty { +- _ = buffer; +- ++ pub fn init(io: std.Io) !TestTty { + if (builtin.os.tag == .windows) return error.SkipZigTest; + const list = try std.testing.allocator.create(std.Io.Writer.Allocating); + list.* = .init(std.testing.allocator); +- const r, const w = try posix.pipe(); ++ const r, const w = try std.Io.Threaded.pipe2(.{}); + return .{ + .fd = r, ++ .io = io, + .pipe_read = r, + .pipe_write = w, + .tty_writer = list, +@@ -715,8 +726,9 @@ pub const TestTty = struct { + } + + pub fn deinit(self: TestTty) void { +- std.posix.close(self.pipe_read); +- std.posix.close(self.pipe_write); ++ const io = std.testing.io; ++ (std.Io.File{ .handle = self.pipe_read, .flags = .{ .nonblocking = false } }).close(io); ++ (std.Io.File{ .handle = self.pipe_write, .flags = .{ .nonblocking = false } }).close(io); + self.tty_writer.deinit(); + std.testing.allocator.destroy(self.tty_writer); + } +diff --git a/src/unicode.zig b/src/unicode.zig +index a5253ac..4ff2e4c 100644 +--- a/src/unicode.zig ++++ b/src/unicode.zig +@@ -26,10 +26,10 @@ pub const GraphemeIterator = struct { + } + + pub fn next(self: *GraphemeIterator) ?Grapheme { +- while (self.inner.next()) |res| { ++ while (self.inner.nextCodePoint()) |res| { + // When leaving a break and entering a non-break, set the start of a cluster + if (self.prev_break and !res.is_break) { +- const cp_len: usize = std.unicode.utf8CodepointSequenceLength(res.cp) catch 1; ++ const cp_len: usize = std.unicode.utf8CodepointSequenceLength(res.code_point) catch 1; + self.start = self.inner.i - cp_len; + } + +diff --git a/src/vxfw/App.zig b/src/vxfw/App.zig +index 6d394a9..fc3962f 100644 +--- a/src/vxfw/App.zig ++++ b/src/vxfw/App.zig +@@ -16,7 +16,7 @@ tty: vaxis.Tty, + vx: vaxis.Vaxis, + timers: std.ArrayList(vxfw.Tick), + wants_focus: ?vxfw.Widget, +-buffer: [1024]u8, ++io: std.Io, + + /// Runtime options + pub const Options = struct { +@@ -27,10 +27,10 @@ pub const Options = struct { + /// Create an application. We require stable pointers to do the set up, so this will create an App + /// object on the heap. Call destroy when the app is complete to reset terminal state and release + /// resources +-pub fn init(allocator: Allocator) !App { +- var app: App = .{ ++pub fn init(allocator: Allocator, io: std.Io) !App { ++ return .{ + .allocator = allocator, +- .tty = undefined, ++ .tty = try vaxis.Tty.init(io), + .vx = try vaxis.init(allocator, .{ + .system_clipboard_allocator = allocator, + .kitty_keyboard_flags = .{ +@@ -39,10 +39,8 @@ pub fn init(allocator: Allocator) !App { + }), + .timers = std.ArrayList(vxfw.Tick){}, + .wants_focus = null, +- .buffer = undefined, ++ .io = io, + }; +- app.tty = try vaxis.Tty.init(&app.buffer); +- return app; + } + + pub fn deinit(self: *App) void { +@@ -55,7 +53,8 @@ pub fn run(self: *App, widget: vxfw.Widget, opts: Options) anyerror!void { + const tty = &self.tty; + const vx = &self.vx; + +- var loop: EventLoop = .{ .tty = tty, .vaxis = vx }; ++ const io = self.io; ++ var loop: EventLoop = .{ .tty = tty, .vaxis = vx, .io = io, .queue = .{ .io = io } }; + try loop.start(); + defer loop.stop(); + +@@ -65,7 +64,7 @@ pub fn run(self: *App, widget: vxfw.Widget, opts: Options) anyerror!void { + loop.postEvent(.focus_in); + + try vx.enterAltScreen(tty.writer()); +- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_s); ++ try vx.queryTerminal(tty.writer(), io, 1 * std.time.ns_per_s); + try vx.setBracketedPaste(tty.writer(), true); + try vx.subscribeToColorSchemeUpdates(tty.writer()); + +@@ -97,13 +96,14 @@ pub fn run(self: *App, widget: vxfw.Widget, opts: Options) anyerror!void { + defer focus_handler.deinit(self.allocator); + + // Timestamp of our next frame +- var next_frame_ms: u64 = @intCast(std.time.milliTimestamp()); ++ var next_frame_ms: u64 = nowMs(io); + + // Create our event context + var ctx: vxfw.EventContext = .{ + .alloc = self.allocator, + .phase = .capturing, + .cmds = vxfw.CommandList{}, ++ .io = io, + .consume_event = false, + .redraw = false, + .quit = false, +@@ -111,13 +111,13 @@ pub fn run(self: *App, widget: vxfw.Widget, opts: Options) anyerror!void { + defer ctx.cmds.deinit(self.allocator); + + while (true) { +- const now_ms: u64 = @intCast(std.time.milliTimestamp()); ++ const now_ms: u64 = nowMs(io); + if (now_ms >= next_frame_ms) { + // Deadline exceeded. Schedule the next frame + next_frame_ms = now_ms + tick_ms; + } else { + // Sleep until the deadline +- std.Thread.sleep((next_frame_ms - now_ms) * std.time.ns_per_ms); ++ try std.Io.sleep(io, .fromMilliseconds(@intCast(next_frame_ms - now_ms)), .real); + next_frame_ms += tick_ms; + } + +@@ -296,7 +296,7 @@ fn handleCommand(self: *App, cmds: *vxfw.CommandList) Allocator.Error!void { + } + + fn checkTimers(self: *App, ctx: *vxfw.EventContext) anyerror!void { +- const now_ms = std.time.milliTimestamp(); ++ const now_ms = nowMs(ctx.io); + + // timers are always sorted descending + while (self.timers.pop()) |tick| { +@@ -349,12 +349,16 @@ const MouseHandler = struct { + .surface = last_frame, + .z_index = 0, + }; +- const mouse_point: vxfw.Point = .{ +- .row = @intCast(mouse.row), +- .col = @intCast(mouse.col), +- }; +- if (sub.containsPoint(mouse_point)) { +- try last_frame.hitTest(app.allocator, &hits, mouse_point); ++ ++ if (mouse.row >= 0 and mouse.col >= 0) { ++ const mouse_point: vxfw.Point = .{ ++ .row = @intCast(mouse.row), ++ .col = @intCast(mouse.col), ++ }; ++ ++ if (sub.containsPoint(mouse_point)) { ++ try last_frame.hitTest(app.allocator, &hits, mouse_point); ++ } + } + + // We store the hit list from the last mouse event to determine mouse_enter and mouse_leave +@@ -408,12 +412,16 @@ const MouseHandler = struct { + .surface = last_frame, + .z_index = 0, + }; +- const mouse_point: vxfw.Point = .{ +- .row = @intCast(mouse.row), +- .col = @intCast(mouse.col), +- }; +- if (sub.containsPoint(mouse_point)) { +- try last_frame.hitTest(app.allocator, &hits, mouse_point); ++ ++ if (mouse.row >= 0 and mouse.col >= 0) { ++ const mouse_point: vxfw.Point = .{ ++ .row = @intCast(mouse.row), ++ .col = @intCast(mouse.col), ++ }; ++ ++ if (sub.containsPoint(mouse_point)) { ++ try last_frame.hitTest(app.allocator, &hits, mouse_point); ++ } + } + + // Handle mouse_enter and mouse_leave events +@@ -602,3 +610,8 @@ const FocusHandler = struct { + } + } + }; ++ ++fn nowMs(io: std.Io) u64 { ++ const now = std.Io.Clock.now(.real, io); ++ return @intCast(now.toMilliseconds()); ++} +diff --git a/src/vxfw/Button.zig b/src/vxfw/Button.zig +index d4b2a9c..075627e 100644 +--- a/src/vxfw/Button.zig ++++ b/src/vxfw/Button.zig +@@ -142,6 +142,7 @@ test Button { + var ctx: vxfw.EventContext = .{ + .alloc = std.testing.allocator, + .cmds = .empty, ++ .io = std.testing.io, + }; + defer ctx.cmds.deinit(ctx.alloc); + +diff --git a/src/vxfw/ListView.zig b/src/vxfw/ListView.zig +index 8ff9551..de93d5d 100644 +--- a/src/vxfw/ListView.zig ++++ b/src/vxfw/ListView.zig +@@ -564,6 +564,7 @@ test ListView { + var ctx: vxfw.EventContext = .{ + .alloc = std.testing.allocator, + .cmds = .empty, ++ .io = std.testing.io, + }; + defer ctx.cmds.deinit(ctx.alloc); + +@@ -731,6 +732,7 @@ test "ListView: uneven scroll" { + var ctx: vxfw.EventContext = .{ + .alloc = std.testing.allocator, + .cmds = .empty, ++ .io = std.testing.io, + }; + defer ctx.cmds.deinit(ctx.alloc); + +diff --git a/src/vxfw/ScrollView.zig b/src/vxfw/ScrollView.zig +index 36319a4..38859ea 100644 +--- a/src/vxfw/ScrollView.zig ++++ b/src/vxfw/ScrollView.zig +@@ -646,6 +646,7 @@ test ScrollView { + var ctx: vxfw.EventContext = .{ + .alloc = std.testing.allocator, + .cmds = .empty, ++ .io = std.testing.io, + }; + defer ctx.cmds.deinit(ctx.alloc); + +@@ -1044,6 +1045,7 @@ test "ScrollView: uneven scroll" { + var ctx: vxfw.EventContext = .{ + .alloc = std.testing.allocator, + .cmds = .empty, ++ .io = std.testing.io, + }; + defer ctx.cmds.deinit(ctx.alloc); + +diff --git a/src/vxfw/Spinner.zig b/src/vxfw/Spinner.zig +index 6619522..eb3d8ba 100644 +--- a/src/vxfw/Spinner.zig ++++ b/src/vxfw/Spinner.zig +@@ -20,11 +20,11 @@ frame: u4 = 0, + was_spinning: std.atomic.Value(bool) = .{ .raw = false }, + + /// Start, or add one, to the spinner counter. Thread safe. +-pub fn start(self: *Spinner) ?vxfw.Command { ++pub fn start(self: *Spinner, io: std.Io) ?vxfw.Command { + self.was_spinning.store(true, .unordered); + const count = self.count.fetchAdd(1, .monotonic); + if (count == 0) { +- return vxfw.Tick.in(time_lapse, self.widget()); ++ return vxfw.Tick.in(io, time_lapse, self.widget()); + } + return null; + } +@@ -111,13 +111,13 @@ test Spinner { + // Start the spinner. This (maybe) returns a Tick command to schedule the next frame. If the + // spinner is already running, no command is returned. Calling start is thread safe. The + // returned command can be added to an EventContext to schedule the frame +- const maybe_cmd = spinner.start(); ++ const maybe_cmd = spinner.start(std.testing.io); + try std.testing.expect(maybe_cmd != null); + try std.testing.expect(maybe_cmd.? == .tick); + try std.testing.expectEqual(1, spinner.count.load(.unordered)); + + // If we call start again, we won't get another command but our counter will go up +- const maybe_cmd2 = spinner.start(); ++ const maybe_cmd2 = spinner.start(std.testing.io); + try std.testing.expect(maybe_cmd2 == null); + try std.testing.expectEqual(2, spinner.count.load(.unordered)); + +@@ -126,6 +126,7 @@ test Spinner { + var ctx: vxfw.EventContext = .{ + .alloc = arena.allocator(), + .cmds = .empty, ++ .io = std.testing.io, + }; + + // The event loop handles the tick event and calls us back with a .tick event. If we should keep +diff --git a/src/vxfw/SplitView.zig b/src/vxfw/SplitView.zig +index 9a355a0..591e5e9 100644 +--- a/src/vxfw/SplitView.zig ++++ b/src/vxfw/SplitView.zig +@@ -229,6 +229,7 @@ test SplitView { + var ctx: vxfw.EventContext = .{ + .alloc = arena.allocator(), + .cmds = .empty, ++ .io = std.testing.io, + }; + try split_widget.handleEvent(&ctx, .{ .mouse = mouse }); + // We should get a command to change the mouse shape +diff --git a/src/vxfw/Text.zig b/src/vxfw/Text.zig +index 1f76cba..a7b71bb 100644 +--- a/src/vxfw/Text.zig ++++ b/src/vxfw/Text.zig +@@ -223,7 +223,7 @@ pub const SoftwrapIterator = struct { + // Advance the hard iterator + if (self.index == self.line.len) { + self.line = self.hard_iter.next() orelse return null; +- self.line = std.mem.trimRight(u8, self.line, " \t"); ++ self.line = std.mem.trimEnd(u8, self.line, " \t"); + self.index = 0; + } + +@@ -237,7 +237,7 @@ pub const SoftwrapIterator = struct { + if (self.ctx.max.width) |max| { + if (cur_width + next_width > max) { + // Trim the word to see if it can fit on a line by itself +- const trimmed = std.mem.trimLeft(u8, word, " \t"); ++ const trimmed = std.mem.trimStart(u8, word, " \t"); + const trimmed_bytes = word.len - trimmed.len; + // The number of bytes we trimmed is equal to the reduction in length + const trimmed_width = next_width - trimmed_bytes; +diff --git a/src/vxfw/TextField.zig b/src/vxfw/TextField.zig +index 6ad108b..e5a4d5b 100644 +--- a/src/vxfw/TextField.zig ++++ b/src/vxfw/TextField.zig +@@ -346,7 +346,7 @@ pub fn deleteAfterCursor(self: *TextField) void { + /// Moves the cursor backward by words. If the character before the cursor is a space, the cursor is + /// positioned just after the next previous space + pub fn moveBackwardWordwise(self: *TextField) void { +- const trimmed = std.mem.trimRight(u8, self.buf.firstHalf(), " "); ++ const trimmed = std.mem.trimEnd(u8, self.buf.firstHalf(), " "); + const idx = if (std.mem.lastIndexOfScalar(u8, trimmed, ' ')) |last| + last + 1 + else +@@ -562,6 +562,7 @@ test TextField { + var ctx: vxfw.EventContext = .{ + .alloc = arena.allocator(), + .cmds = .empty, ++ .io = std.testing.io, + }; + + // Enough boiler plate...Create the text field +diff --git a/src/vxfw/vxfw.zig b/src/vxfw/vxfw.zig +index 1510e71..f768d56 100644 +--- a/src/vxfw/vxfw.zig ++++ b/src/vxfw/vxfw.zig +@@ -61,8 +61,8 @@ pub const Tick = struct { + return lhs.deadline_ms > rhs.deadline_ms; + } + +- pub fn in(ms: u32, widget: Widget) Command { +- const now = std.time.milliTimestamp(); ++ pub fn in(io: std.Io, ms: u32, widget: Widget) Command { ++ const now = std.Io.Clock.now(.real, io).toMilliseconds(); + return .{ .tick = .{ + .deadline_ms = now + ms, + .widget = widget, +@@ -101,6 +101,7 @@ pub const EventContext = struct { + phase: Phase = .at_target, + alloc: Allocator, + cmds: CommandList, ++ io: std.Io, + + /// The event was handled, do not pass it on + consume_event: bool = false, +@@ -120,7 +121,7 @@ pub const EventContext = struct { + } + + pub fn tick(self: *EventContext, ms: u32, widget: Widget) Allocator.Error!void { +- try self.addCmd(Tick.in(ms, widget)); ++ try self.addCmd(Tick.in(self.io, ms, widget)); + } + + pub fn consumeAndRedraw(self: *EventContext) void { +@@ -541,10 +542,10 @@ test "All widgets have a doctest and refAllDecls test" { + // it easy to fail CI early, or spot bad tests vs non-existant tests + const excludes = &[_][]const u8{ "vxfw.zig", "App.zig" }; + +- var cwd = try std.fs.cwd().openDir("./src/vxfw", .{ .iterate = true }); ++ var cwd = try std.Io.Dir.cwd().openDir(std.testing.io, "./src/vxfw", .{ .iterate = true }); + var iter = cwd.iterate(); +- defer cwd.close(); +- outer: while (try iter.next()) |file| { ++ defer cwd.close(std.testing.io); ++ outer: while (try iter.next(std.testing.io)) |file| { + if (file.kind != .file) continue; + for (excludes) |ex| if (std.mem.eql(u8, ex, file.name)) continue :outer; + +@@ -552,7 +553,7 @@ test "All widgets have a doctest and refAllDecls test" { + file.name[0..idx] + else + continue; +- const data = try cwd.readFileAllocOptions(std.testing.allocator, file.name, 10_000_000, null, .of(u8), 0x00); ++ const data = try cwd.readFileAllocOptions(std.testing.io, file.name, std.testing.allocator, std.Io.Limit.limited(10_000_000), .of(u8), 0x00); + defer std.testing.allocator.free(data); + var ast = try std.zig.Ast.parse(std.testing.allocator, data, .zig); + defer ast.deinit(std.testing.allocator); +diff --git a/src/widgets/TextInput.zig b/src/widgets/TextInput.zig +index a6236a4..6b8871b 100644 +--- a/src/widgets/TextInput.zig ++++ b/src/widgets/TextInput.zig +@@ -266,7 +266,7 @@ pub fn deleteAfterCursor(self: *TextInput) void { + /// Moves the cursor backward by words. If the character before the cursor is a space, the cursor is + /// positioned just after the next previous space + pub fn moveBackwardWordwise(self: *TextInput) void { +- const trimmed = std.mem.trimRight(u8, self.buf.firstHalf(), " "); ++ const trimmed = std.mem.trimEnd(u8, self.buf.firstHalf(), " "); + const idx = if (std.mem.lastIndexOfScalar(u8, trimmed, ' ')) |last| + last + 1 + else +diff --git a/src/widgets/TextView.zig b/src/widgets/TextView.zig +index 45f9e44..ee326cf 100644 +--- a/src/widgets/TextView.zig ++++ b/src/widgets/TextView.zig +@@ -79,52 +79,22 @@ pub const Buffer = struct { + var cols: usize = self.last_cols; + var iter = uucode.grapheme.Iterator(uucode.utf8.Iterator).init(.init(content.bytes)); + +- var grapheme_start: usize = 0; +- var prev_break: bool = true; +- +- while (iter.next()) |result| { +- if (prev_break and !result.is_break) { +- // Start of a new grapheme +- const cp_len: usize = std.unicode.utf8CodepointSequenceLength(result.cp) catch 1; +- grapheme_start = iter.i - cp_len; +- } +- +- if (result.is_break) { +- // End of a grapheme +- const grapheme_end = iter.i; +- const grapheme_len = grapheme_end - grapheme_start; +- +- try self.grapheme.append(allocator, .{ +- .len = @intCast(grapheme_len), +- .offset = @intCast(self.content.items.len + grapheme_start), +- }); +- +- const cluster = content.bytes[grapheme_start..grapheme_end]; +- if (std.mem.eql(u8, cluster, "\n")) { +- self.cols = @max(self.cols, cols); +- cols = 0; +- } else { +- // Calculate width using gwidth +- const w = vaxis.gwidth.gwidth(cluster, .unicode); +- cols +|= w; +- } +- +- grapheme_start = grapheme_end; +- } +- prev_break = result.is_break; +- } +- +- // Flush the last grapheme if we ended mid-cluster +- if (!prev_break and grapheme_start < content.bytes.len) { +- const grapheme_len = content.bytes.len - grapheme_start; ++ while (iter.nextGrapheme()) |grapheme| { ++ const grapheme_start = grapheme.start; ++ const grapheme_end = grapheme.end; ++ const grapheme_len = grapheme_end - grapheme_start; + + try self.grapheme.append(allocator, .{ + .len = @intCast(grapheme_len), + .offset = @intCast(self.content.items.len + grapheme_start), + }); + +- const cluster = content.bytes[grapheme_start..]; +- if (!std.mem.eql(u8, cluster, "\n")) { ++ const cluster = content.bytes[grapheme_start..grapheme_end]; ++ if (std.mem.eql(u8, cluster, "\n")) { ++ self.cols = @max(self.cols, cols); ++ cols = 0; ++ } else { ++ // Calculate width using gwidth + const w = vaxis.gwidth.gwidth(cluster, .unicode); + cols +|= w; + } +diff --git a/src/widgets/terminal/Command.zig b/src/widgets/terminal/Command.zig +index 0cd24f7..ed41553 100644 +--- a/src/widgets/terminal/Command.zig ++++ b/src/widgets/terminal/Command.zig +@@ -14,7 +14,7 @@ working_directory: ?[]const u8, + // Set after spawn() + pid: ?std.posix.pid_t = null, + +-env_map: *const std.process.EnvMap, ++env_map: *const std.process.Environ.Map, + + pty: Pty, + +@@ -43,8 +43,8 @@ pub fn spawn(self: *Command, allocator: std.mem.Allocator) !void { + try posix.dup2(self.pty.tty.handle, std.posix.STDOUT_FILENO); + try posix.dup2(self.pty.tty.handle, std.posix.STDERR_FILENO); + +- self.pty.tty.close(); +- if (self.pty.pty.handle > 2) self.pty.pty.close(); ++ _ = std.os.linux.close(self.pty.tty.handle); ++ if (self.pty.pty.handle > 2) _ = std.os.linux.close(self.pty.pty.handle); + + if (self.working_directory) |wd| { + try std.posix.chdir(wd); +@@ -78,8 +78,8 @@ pub fn spawn(self: *Command, allocator: std.mem.Allocator) !void { + fn handleSigChild(_: c_int) callconv(.c) void { + const result = std.posix.waitpid(-1, 0); + +- Terminal.global_vt_mutex.lock(); +- defer Terminal.global_vt_mutex.unlock(); ++ if (!Terminal.global_vt_mutex.tryLock()) return; ++ defer Terminal.global_vt_mutex.unlock(Terminal.global_vt_io); + if (Terminal.global_vts) |vts| { + var vt = vts.get(result.pid) orelse return; + vt.event_queue.push(.exited); +@@ -97,7 +97,7 @@ pub fn kill(self: *Command) void { + /// hash map plus options. + fn createEnvironFromMap( + arena: std.mem.Allocator, +- map: *const std.process.EnvMap, ++ map: *const std.process.Environ.Map, + ) ![:null]?[*:0]u8 { + const envp_count: usize = map.count(); + +diff --git a/src/widgets/terminal/Pty.zig b/src/widgets/terminal/Pty.zig +index d9f4138..4c67328 100644 +--- a/src/widgets/terminal/Pty.zig ++++ b/src/widgets/terminal/Pty.zig +@@ -7,8 +7,8 @@ const Winsize = @import("../../main.zig").Winsize; + + const posix = std.posix; + +-pty: std.fs.File, +-tty: std.fs.File, ++pty: std.Io.File, ++tty: std.Io.File, + + /// opens a new tty/pty pair + pub fn init() !Pty { +@@ -19,9 +19,9 @@ pub fn init() !Pty { + } + + /// closes the tty and pty +-pub fn deinit(self: Pty) void { +- self.pty.close(); +- self.tty.close(); ++pub fn deinit(self: Pty, io: std.Io) void { ++ self.pty.close(io); ++ self.tty.close(io); + } + + /// sets the size of the pty +@@ -37,7 +37,7 @@ pub fn setSize(self: Pty, ws: Winsize) !void { + } + + fn openPtyLinux() !Pty { +- const p = try posix.open("/dev/ptmx", .{ .ACCMODE = .RDWR, .NOCTTY = true }, 0); ++ const p = try posix.openat(posix.AT.FDCWD, "/dev/ptmx", .{ .ACCMODE = .RDWR, .NOCTTY = true }, 0); + errdefer posix.close(p); + + // unlockpt +@@ -50,7 +50,7 @@ fn openPtyLinux() !Pty { + const sname = try std.fmt.bufPrint(&buf, "/dev/pts/{d}", .{n}); + std.log.debug("pts: {s}", .{sname}); + +- const t = try posix.open(sname, .{ .ACCMODE = .RDWR, .NOCTTY = true }, 0); ++ const t = try posix.openat(posix.AT.FDCWD, sname, .{ .ACCMODE = .RDWR, .NOCTTY = true }, 0); + + return .{ + .pty = .{ .handle = p }, +diff --git a/src/widgets/terminal/Terminal.zig b/src/widgets/terminal/Terminal.zig +index 36d61d7..786ca85 100644 +--- a/src/widgets/terminal/Terminal.zig ++++ b/src/widgets/terminal/Terminal.zig +@@ -43,21 +43,23 @@ pub const InputEvent = union(enum) { + key_press: vaxis.Key, + }; + +-pub var global_vt_mutex: std.Thread.Mutex = .{}; ++pub var global_vt_mutex: std.Io.Mutex = .init; ++var global_vt_io: std.Io = undefined; + pub var global_vts: ?std.AutoHashMap(i32, *Terminal) = null; + pub var global_sigchild_installed: bool = false; + + allocator: std.mem.Allocator, ++io: std.Io, + scrollback_size: u16, + + pty: Pty, +-pty_writer: std.fs.File.Writer, ++pty_writer: std.Io.File.Writer, + cmd: Command, + thread: ?std.Thread = null, + + /// the screen we draw from + front_screen: Screen, +-front_mutex: std.Thread.Mutex = .{}, ++front_mutex: std.Io.Mutex = .init, + + /// the back screens + back_screen: *Screen = undefined, +@@ -65,7 +67,7 @@ back_screen_pri: Screen, + back_screen_alt: Screen, + // only applies to primary screen + scroll_offset: usize = 0, +-back_mutex: std.Thread.Mutex = .{}, ++back_mutex: std.Io.Mutex = .init, + // dirty is protected by back_mutex. Only access this field when you hold that mutex + dirty: bool = false, + +@@ -79,14 +81,15 @@ working_directory: std.ArrayList(u8) = .empty, + + last_printed: []const u8 = "", + +-event_queue: Queue = .{}, ++event_queue: Queue, + + /// initialize a Terminal. This sets the size of the underlying pty and allocates the sizes of the + /// screen + pub fn init( + allocator: std.mem.Allocator, ++ io: std.Io, + argv: []const []const u8, +- env: *const std.process.EnvMap, ++ env: *const std.process.Environ.Map, + opts: Options, + write_buf: []u8, + ) !Terminal { +@@ -107,8 +110,10 @@ pub fn init( + while (col < opts.winsize.cols) : (col += 8) { + try tabs.append(allocator, col); + } ++ global_vt_io = io; + return .{ + .allocator = allocator, ++ .io = io, + .pty = pty, + .pty_writer = pty.pty.writerStreaming(write_buf), + .cmd = cmd, +@@ -117,6 +122,7 @@ pub fn init( + .back_screen_pri = try Screen.init(allocator, opts.winsize.cols, opts.winsize.rows + opts.scrollback_size), + .back_screen_alt = try Screen.init(allocator, opts.winsize.cols, opts.winsize.rows), + .tab_stops = tabs, ++ .event_queue = .{ .io = io }, + }; + } + +@@ -125,8 +131,8 @@ pub fn deinit(self: *Terminal) void { + self.should_quit = true; + + pid: { +- global_vt_mutex.lock(); +- defer global_vt_mutex.unlock(); ++ global_vt_mutex.lockUncancelable(global_vt_io); ++ defer global_vt_mutex.unlock(global_vt_io); + var vts = global_vts orelse break :pid; + if (self.cmd.pid) |pid| + _ = vts.remove(pid); +@@ -143,7 +149,7 @@ pub fn deinit(self: *Terminal) void { + thread.join(); + self.thread = null; + } +- self.pty.deinit(); ++ self.pty.deinit(self.io); + self.front_screen.deinit(self.allocator); + self.back_screen_pri.deinit(self.allocator); + self.back_screen_alt.deinit(self.allocator); +@@ -162,16 +168,15 @@ pub fn spawn(self: *Terminal) !void { + if (self.cmd.working_directory) |pwd| { + try self.working_directory.appendSlice(self.allocator, pwd); + } else { +- const pwd = std.fs.cwd(); + var buffer: [std.fs.max_path_bytes]u8 = undefined; +- const out_path = try std.os.getFdPath(pwd.fd, &buffer); ++ const out_path = try std.posix.readlinkZ("/proc/self/cwd", &buffer); + try self.working_directory.appendSlice(self.allocator, out_path); + } + + { + // add to our global list +- global_vt_mutex.lock(); +- defer global_vt_mutex.unlock(); ++ global_vt_mutex.lockUncancelable(global_vt_io); ++ defer global_vt_mutex.unlock(global_vt_io); + if (global_vts == null) + global_vts = std.AutoHashMap(i32, *Terminal).init(self.allocator); + if (self.cmd.pid) |pid| +@@ -190,8 +195,8 @@ pub fn resize(self: *Terminal, ws: Winsize) !void { + ws.rows == self.front_screen.height) + return; + +- self.back_mutex.lock(); +- defer self.back_mutex.unlock(); ++ self.back_mutex.lockUncancelable(self.io); ++ defer self.back_mutex.unlock(self.io); + + self.front_screen.deinit(self.allocator); + self.front_screen = try Screen.init(self.allocator, ws.cols, ws.rows); +@@ -206,7 +211,7 @@ pub fn resize(self: *Terminal, ws: Winsize) !void { + + pub fn draw(self: *Terminal, allocator: std.mem.Allocator, win: vaxis.Window) !void { + if (self.back_mutex.tryLock()) { +- defer self.back_mutex.unlock(); ++ defer self.back_mutex.unlock(self.io); + // We keep this as a separate condition so we don't deadlock by obtaining the lock but not + // having sync + if (!self.mode.sync) { +@@ -249,7 +254,7 @@ pub fn get_pty_writer(self: *Terminal) *std.Io.Writer { + return &self.pty_writer.interface; + } + +-fn reader(self: *const Terminal, buf: []u8) std.fs.File.Reader { ++fn reader(self: *const Terminal, buf: []u8) std.Io.File.Reader { + return self.pty.pty.readerStreaming(buf); + } + +@@ -265,8 +270,8 @@ fn run(self: *Terminal) !void { + + while (!self.should_quit) { + const event = try parser.parseReader(&reader_.interface); +- self.back_mutex.lock(); +- defer self.back_mutex.unlock(); ++ self.back_mutex.lockUncancelable(self.io); ++ defer self.back_mutex.unlock(self.io); + + if (!self.dirty and self.event_queue.tryPush(.redraw)) + self.dirty = true; diff --git a/third_party/libvaxis/fixes.patch b/third_party/libvaxis/fixes.patch new file mode 100644 index 00000000..c31bcf68 --- /dev/null +++ b/third_party/libvaxis/fixes.patch @@ -0,0 +1,231 @@ +diff --git a/examples/table.zig b/examples/table.zig +index c489828..16aefdc 100644 +--- a/examples/table.zig ++++ b/examples/table.zig +@@ -20,8 +20,8 @@ pub fn main(init: std.process.Init) !void { + // Users set up below the main function + const users_buf = try alloc.dupe(User, users[0..]); + +- var buffer: [1024]u8 = undefined; +- var tty = try vaxis.Tty.init(init.io, &buffer); ++ // var buffer: [1024]u8 = undefined; ++ var tty = try vaxis.Tty.init(init.io); + defer tty.deinit(); + const tty_writer = tty.writer(); + var vx = try vaxis.init(alloc, .{ +diff --git a/src/Vaxis.zig b/src/Vaxis.zig +index 2e24e9b..2b2ee49 100644 +--- a/src/Vaxis.zig ++++ b/src/Vaxis.zig +@@ -127,14 +127,13 @@ pub fn deinit(self: *Vaxis, alloc: ?std.mem.Allocator, tty: *IoWriter) void { + } + } + +-/// resets enabled features, sends cursor to home and clears below cursor +-pub fn resetState(self: *Vaxis, tty: *IoWriter) !void { +- // always show the cursor on state reset ++/// Resets enabled terminal features without clearing screen content. ++/// Use this for non-interactive / single-shot rendering where the ++/// output should remain visible after exit. ++pub fn resetModes(self: *Vaxis, tty: *IoWriter) !void { + tty.writeAll(ctlseqs.show_cursor) catch {}; + tty.writeAll(ctlseqs.sgr_reset) catch {}; + if (self.screen.cursor_shape != .default) { +- // In many terminals, `.default` will set to the configured cursor shape. Others, it will +- // change to a blinking block. + tty.print(ctlseqs.cursor_shape, .{@intFromEnum(Cell.CursorShape.default)}) catch {}; + } + if (self.state.kitty_keyboard) { +@@ -147,18 +146,6 @@ pub fn resetState(self: *Vaxis, tty: *IoWriter) !void { + if (self.state.bracketed_paste) { + try self.setBracketedPaste(tty, false); + } +- if (self.state.alt_screen) { +- try tty.writeAll(ctlseqs.home); +- try tty.writeAll(ctlseqs.erase_below_cursor); +- try self.exitAltScreen(tty); +- } else { +- try tty.writeByte('\r'); +- var i: u16 = 0; +- while (i < self.state.cursor.row) : (i += 1) { +- try tty.writeAll(ctlseqs.ri); +- } +- try tty.writeAll(ctlseqs.erase_below_cursor); +- } + if (self.state.color_scheme_updates) { + try tty.writeAll(ctlseqs.color_scheme_reset); + self.state.color_scheme_updates = false; +@@ -179,7 +166,24 @@ pub fn resetState(self: *Vaxis, tty: *IoWriter) !void { + try tty.writeAll(ctlseqs.osc12_reset); + self.state.changed_cursor_color = false; + } ++ try tty.flush(); ++} + ++/// resets enabled features, sends cursor to home and clears below cursor ++pub fn resetState(self: *Vaxis, tty: *IoWriter) !void { ++ self.resetModes(tty) catch {}; ++ if (self.state.alt_screen) { ++ try tty.writeAll(ctlseqs.home); ++ try tty.writeAll(ctlseqs.erase_below_cursor); ++ try self.exitAltScreen(tty); ++ } else { ++ try tty.writeByte('\r'); ++ var i: u16 = 0; ++ while (i < self.state.cursor.row) : (i += 1) { ++ try tty.writeAll(ctlseqs.ri); ++ } ++ try tty.writeAll(ctlseqs.erase_below_cursor); ++ } + try tty.flush(); + } + +@@ -943,19 +947,19 @@ pub fn transmitLocalImagePath( + switch (format) { + .rgb => { + try tty.print( +- "\x1b_Gf=24,s={d},v={d},i={d},t={c};{s}\x1b\\", ++ "\x1b_Gf=24,s={d},v={d},i={d},t={c},q=2;{s}\x1b\\", + .{ width, height, id, medium_char, encoded }, + ); + }, + .rgba => { + try tty.print( +- "\x1b_Gf=32,s={d},v={d},i={d},t={c};{s}\x1b\\", ++ "\x1b_Gf=32,s={d},v={d},i={d},t={c},q=2;{s}\x1b\\", + .{ width, height, id, medium_char, encoded }, + ); + }, + .png => { + try tty.print( +- "\x1b_Gf=100,i={d},t={c};{s}\x1b\\", ++ "\x1b_Gf=100,i={d},t={c},q=2;{s}\x1b\\", + .{ id, medium_char, encoded }, + ); + }, +@@ -991,7 +995,7 @@ pub fn transmitPreEncodedImage( + + if (bytes.len < 4096) { + try tty.print( +- "\x1b_Gf={d},s={d},v={d},i={d};{s}\x1b\\", ++ "\x1b_Gf={d},s={d},v={d},i={d},q=2;{s}\x1b\\", + .{ + fmt, + width, +@@ -1004,14 +1008,14 @@ pub fn transmitPreEncodedImage( + var n: usize = 4096; + + try tty.print( +- "\x1b_Gf={d},s={d},v={d},i={d},m=1;{s}\x1b\\", ++ "\x1b_Gf={d},s={d},v={d},i={d},m=1,q=2;{s}\x1b\\", + .{ fmt, width, height, id, bytes[0..n] }, + ); + while (n < bytes.len) : (n += 4096) { + const end: usize = @min(n + 4096, bytes.len); + const m: u2 = if (end == bytes.len) 0 else 1; + try tty.print( +- "\x1b_Gm={d};{s}\x1b\\", ++ "\x1b_Gm={d},q=2;{s}\x1b\\", + .{ + m, + bytes[n..end], +diff --git a/src/ctlseqs.zig b/src/ctlseqs.zig +index ab415c6..4baa496 100644 +--- a/src/ctlseqs.zig ++++ b/src/ctlseqs.zig +@@ -138,8 +138,8 @@ pub const osc52_clipboard_copy = "\x1b]52;c;{s}\x1b\\"; + pub const osc52_clipboard_request = "\x1b]52;c;?\x1b\\"; + + // Kitty graphics +-pub const kitty_graphics_clear = "\x1b_Ga=d\x1b\\"; +-pub const kitty_graphics_preamble = "\x1b_Ga=p,i={d}"; ++pub const kitty_graphics_clear = "\x1b_Ga=d,q=2\x1b\\"; ++pub const kitty_graphics_preamble = "\x1b_Ga=p,q=2,i={d}"; + pub const kitty_graphics_closing = ",C=1\x1b\\"; + + // Color control sequences +diff --git a/src/tty.zig b/src/tty.zig +index c1da795..7f6a303 100644 +--- a/src/tty.zig ++++ b/src/tty.zig +@@ -52,6 +52,10 @@ pub const PosixTty = struct { + + var handler_installed: bool = false; + ++ /// Atomic flag set by the SIGWINCH signal handler. Checked and cleared ++ /// by pollWinch() from a non-signal context to avoid deadlocks. ++ var winch_pending: std.atomic.Value(bool) = .{ .raw = false }; ++ + /// initializes a Tty instance by opening /dev/tty and "making it raw". A + /// signal handler is installed for SIGWINCH. No callbacks are installed, be + /// sure to register a callback when initializing the event loop +@@ -129,7 +133,20 @@ pub const PosixTty = struct { + } + + fn handleWinch(_: posix.SIG) callconv(.c) void { +- if (!handler_mutex.tryLock()) return; ++ // Only set an atomic flag. Calling mutex.lock() or queue.push() ++ // from a signal handler is not async-signal-safe and causes ++ // deadlocks when the signal is delivered to a thread that already ++ // holds one of those locks. Registered handlers are dispatched ++ // from pollWinch() instead. ++ winch_pending.store(true, .release); ++ } ++ ++ /// Check if a SIGWINCH was received and dispatch registered handlers. ++ /// Must be called from a non-signal context (e.g., the main event loop) ++ /// to avoid the async-signal-safety issues with the old approach. ++ pub fn pollWinch() void { ++ if (!winch_pending.swap(false, .acquire)) return; ++ handler_mutex.lockUncancelable(handler_io); + defer handler_mutex.unlock(handler_io); + var i: usize = 0; + while (i < handler_idx) : (i += 1) { +diff --git a/src/vxfw/App.zig b/src/vxfw/App.zig +index fc3962f..2661cf4 100644 +--- a/src/vxfw/App.zig ++++ b/src/vxfw/App.zig +@@ -1,4 +1,5 @@ + const std = @import("std"); ++const builtin = @import("builtin"); + const vaxis = @import("../main.zig"); + const vxfw = @import("vxfw.zig"); + +@@ -123,6 +124,13 @@ pub fn run(self: *App, widget: vxfw.Widget, opts: Options) anyerror!void { + + try self.checkTimers(&ctx); + ++ // Poll for pending SIGWINCH outside of queue lock. ++ // The signal handler only sets an atomic flag; we dispatch ++ // the registered callbacks here where mutex operations are safe. ++ if (builtin.os.tag != .windows and !builtin.is_test) { ++ if (!vx.state.in_band_resize) vaxis.Tty.pollWinch(); ++ } ++ + { + loop.queue.lock(); + defer loop.queue.unlock(); +diff --git a/src/vxfw/Padding.zig b/src/vxfw/Padding.zig +index e60162f..6e5f40b 100644 +--- a/src/vxfw/Padding.zig ++++ b/src/vxfw/Padding.zig +@@ -6,7 +6,7 @@ const Allocator = std.mem.Allocator; + const vxfw = @import("vxfw.zig"); + + const Padding = @This(); +-const PadValues = struct { ++pub const PadValues = struct { + left: u16 = 0, + right: u16 = 0, + top: u16 = 0, +@@ -54,10 +54,6 @@ fn typeErasedDrawFn(ptr: *anyopaque, ctx: vxfw.DrawContext) Allocator.Error!vxfw + + pub fn draw(self: *const Padding, ctx: vxfw.DrawContext) Allocator.Error!vxfw.Surface { + const pad = self.padding; +- if (pad.left > 0 or pad.right > 0) +- std.debug.assert(ctx.max.width != null); +- if (pad.top > 0 or pad.bottom > 0) +- std.debug.assert(ctx.max.height != null); + const inner_min: vxfw.Size = .{ + .width = ctx.min.width -| (pad.right + pad.left), + .height = ctx.min.height -| (pad.top + pad.bottom), diff --git a/third_party/libvaxis/libvaxis.bazel b/third_party/libvaxis/libvaxis.bazel new file mode 100644 index 00000000..122e25a1 --- /dev/null +++ b/third_party/libvaxis/libvaxis.bazel @@ -0,0 +1,17 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "vaxis", + srcs = glob([ + "src/*.zig", + "src/vxfw/*.zig", + "src/widgets/*.zig", + "src/widgets/terminal/*.zig", + ]), + main = "src/main.zig", + deps = [ + "@zigimg", + "@uucode", + ], + visibility = ["//visibility:public"], +) diff --git a/third_party/libvaxis/repo.bzl b/third_party/libvaxis/repo.bzl new file mode 100644 index 00000000..9396332b --- /dev/null +++ b/third_party/libvaxis/repo.bzl @@ -0,0 +1,11 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") + +def repo(): + new_git_repository( + name = "libvaxis", + remote = "https://github.com/rockorager/libvaxis.git", + commit = "a3ae1d53feeeeaeb6218de3d38837559811acae4", + build_file = "//third_party/libvaxis:libvaxis.bazel", + patches = ["//third_party/libvaxis:bump-zig.patch", "//third_party/libvaxis:fixes.patch"], + patch_args = ["-p1"], + ) diff --git a/third_party/non_module_deps.bzl b/third_party/non_module_deps.bzl index af773459..4a1434df 100644 --- a/third_party/non_module_deps.bzl +++ b/third_party/non_module_deps.bzl @@ -1,4 +1,5 @@ load("//third_party/arocc:repo.bzl", arocc = "repo") +load("//third_party/cloud_accelerator_diagnostics:repo.bzl", cloud_accelerator_diagnostics = "repo") load("//third_party/com_google_sentencepiece:repo.bzl", com_google_sentencepiece = "repo") load("//third_party/mnist:repo.bzl", mnist = "repo") load("//third_party/org_swig_swig:repo.bzl", org_swig_swig = "repo") @@ -7,8 +8,12 @@ load("//third_party/xla:repo.bzl", xla = "repo") load("//third_party/flashattn:repo.bzl", flashattn = "repo") load("//third_party/linenoise:repo.bzl", linenoise = "repo") load("//third_party/stb:repo.bzl", stb = "repo") +load("//third_party/zigimg:repo.bzl", zigimg = "repo") +load("//third_party/uucode:repo.bzl", uucode = "repo") +load("//third_party/libvaxis:repo.bzl", libvaxis = "repo") def _non_module_deps_impl(mctx): + cloud_accelerator_diagnostics() com_google_sentencepiece() org_swig_swig() mnist() @@ -18,6 +23,9 @@ def _non_module_deps_impl(mctx): flashattn() linenoise() stb() + zigimg() + uucode() + libvaxis() return mctx.extension_metadata( reproducible = True, diff --git a/third_party/uucode/BUILD.bazel b/third_party/uucode/BUILD.bazel new file mode 100644 index 00000000..49e3a399 --- /dev/null +++ b/third_party/uucode/BUILD.bazel @@ -0,0 +1,4 @@ +exports_files([ + "build_config.zig", + "uucode.bazel", +]) diff --git a/third_party/uucode/build_config.zig b/third_party/uucode/build_config.zig new file mode 100644 index 00000000..2a656ae9 --- /dev/null +++ b/third_party/uucode/build_config.zig @@ -0,0 +1,18 @@ +const config = @import("config.zig"); +const config_x = @import("config.x.zig"); + +pub const tables = [_]config.Table{ + .{ + .extensions = &.{}, + .fields = &config._resolveFields( + config_x, + &.{ + "east_asian_width", + "grapheme_break", + "general_category", + "is_emoji_presentation", + }, + &.{}, + ), + }, +}; diff --git a/third_party/uucode/repo.bzl b/third_party/uucode/repo.bzl new file mode 100644 index 00000000..034fa2c5 --- /dev/null +++ b/third_party/uucode/repo.bzl @@ -0,0 +1,25 @@ +def _uucode_repo_impl(ctx): + ctx.download_and_extract( + url = "https://github.com/jacobsandlund/uucode/archive/{}.tar.gz".format(ctx.attr.commit), + stripPrefix = "uucode-{}".format(ctx.attr.commit), + ) + + ctx.symlink(ctx.attr.build_config, "build_config.zig") + ctx.symlink(ctx.attr.build_file, "BUILD.bazel") + +_uucode_repo = repository_rule( + implementation = _uucode_repo_impl, + attrs = { + "commit": attr.string(mandatory = True), + "build_config": attr.label(mandatory = True, allow_single_file = True), + "build_file": attr.label(mandatory = True, allow_single_file = True), + }, +) + +def repo(): + _uucode_repo( + name = "uucode", + commit = "faab8894d753207878283d523add01817e23b805", + build_config = "//third_party/uucode:build_config.zig", + build_file = "//third_party/uucode:uucode.bazel", + ) diff --git a/third_party/uucode/uucode.bazel b/third_party/uucode/uucode.bazel new file mode 100644 index 00000000..8b152287 --- /dev/null +++ b/third_party/uucode/uucode.bazel @@ -0,0 +1,159 @@ +load("@bazel_skylib//rules:run_binary.bzl", "run_binary") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") + +# Generator + +zig_library( + name = "_config_gen", + import_name = "config.zig", + main = "src/config.zig", + srcs = ["src/quirks.zig"], + zigopts = ["--dep", "types.zig=_types_gen"], # circular: config.zig <-> types.zig +) + +zig_library( + name = "_types_gen", + import_name = "types.zig", + main = "src/types.zig", + srcs = ["src/get.zig"], + deps = [":_config_gen"], +) + +zig_library( + name = "_config_x_gen", + import_name = "config.x.zig", + main = "src/x/config.x.zig", + srcs = [ + "src/x/config_x/grapheme_break.zig", + "src/x/config_x/wcwidth.zig", + ], + deps = [ + ":_config_gen", + ":_types_gen", + ":types_x", + ], +) + +zig_library( + name = "_build_config_gen", + import_name = "build_config", + main = "build_config.zig", + deps = [ + ":_config_gen", + ":_config_x_gen", + ], +) + +zig_binary( + name = "gen_tables", + main = "src/build/tables.zig", + srcs = ["src/build/Ucd.zig"], + deps = [ + ":_build_config_gen", + ":_config_gen", + ":_types_gen", + ], +) + +genrule( + name = "tables", + srcs = glob(["ucd/**"]), + outs = ["tables.zig"], + cmd = "OUT=$$PWD/$@ && TOOL=$$PWD/$(execpath :gen_tables) && cd $$(dirname $(location ucd/UnicodeData.txt))/.. && $$TOOL $$OUT", + tools = [":gen_tables"], +) + +# Library + +zig_library( + name = "config", + import_name = "config.zig", + main = "src/config.zig", + srcs = ["src/quirks.zig"], + zigopts = ["--dep", "types.zig=types"], # circular: config.zig <-> types.zig +) + +zig_library( + name = "types", + import_name = "types.zig", + main = "src/types.zig", + deps = [":config"], + zigopts = ["--dep", "get.zig=get"], # circular: types.zig -> get.zig -> tables -> types.zig +) + +zig_library( + name = "types_x", + import_name = "types.x.zig", + main = "src/x/types.x.zig", + srcs = ["src/x/types_x/grapheme.zig"], +) + +zig_library( + name = "config_x", + import_name = "config.x.zig", + main = "src/x/config.x.zig", + srcs = [ + "src/x/config_x/grapheme_break.zig", + "src/x/config_x/wcwidth.zig", + ], + deps = [ + ":config", + ":types", + ":types_x", + ], +) + +zig_library( + name = "build_config", + import_name = "build_config", + main = "build_config.zig", + deps = [ + ":config", + ":config_x", + ], +) + +zig_library( + name = "tables_lib", + import_name = "tables", + main = ":tables", + deps = [ + ":build_config", + ":config", + ":types", + ":types_x", + ], +) + +zig_library( + name = "get", + import_name = "get.zig", + main = "src/get.zig", + deps = [ + ":tables_lib", + ":types", + ], +) + +zig_library( + name = "uucode", + import_name = "uucode", + main = "src/root.zig", + srcs = [ + "src/ascii.zig", + "src/build/Ucd.zig", + "src/code_point.zig", + "src/grapheme.zig", + "src/utf8.zig", + "src/x/grapheme.zig", + "src/x/root.zig", + ], + deps = [ + ":config", + ":get", + ":tables_lib", + ":types", + ":types_x", + ], + visibility = ["//visibility:public"], +) diff --git a/third_party/zigimg/BUILD.bazel b/third_party/zigimg/BUILD.bazel new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/third_party/zigimg/BUILD.bazel @@ -0,0 +1 @@ + diff --git a/third_party/zigimg/bump-zig.patch b/third_party/zigimg/bump-zig.patch new file mode 100644 index 00000000..e6367c9c --- /dev/null +++ b/third_party/zigimg/bump-zig.patch @@ -0,0 +1,4338 @@ +diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml +index 400aee8..72ad6f9 100644 +--- a/.github/workflows/main.yml ++++ b/.github/workflows/main.yml +@@ -25,7 +25,7 @@ jobs: + - name: Setup Zig + uses: mlugg/setup-zig@v2 + with: +- version: 0.15.2 ++ version: 0.16.0-dev.2261+d6b3dd25a + - name: AST Verify + run: zig fmt --check --ast-check src/ + working-directory: zigimg +diff --git a/build.zig.zon b/build.zig.zon +index 3ff4816..536d37b 100644 +--- a/build.zig.zon ++++ b/build.zig.zon +@@ -1,7 +1,7 @@ + .{ + .name = .zigimg, + .fingerprint = 0x6d2b443cdaa8f7f3, +- .minimum_zig_version = "0.15.2", ++ .minimum_zig_version = "0.16.0-dev.2261+d6b3dd25a", + .version = "0.1.0", + .paths = .{ + "build.zig", +diff --git a/src/Image.zig b/src/Image.zig +index d67cd61..6472129 100644 +--- a/src/Image.zig ++++ b/src/Image.zig +@@ -71,7 +71,7 @@ pub const WriteError = Error || + std.mem.Allocator.Error || + io.ReadStream.Error || + io.WriteStream.Error || +- std.fs.File.OpenError || ++ std.Io.File.OpenError || + error{ EndOfStream, InvalidData, UnfinishedBits }; + + pub const ConvertError = Error || +@@ -150,16 +150,16 @@ pub fn deinit(self: *Image, allocator: std.mem.Allocator) void { + } + + /// Detect which image format is used by the file path +-pub fn detectFormatFromFilePath(file_path: []const u8, read_buffer: []u8) !Format { +- var file = try std.fs.cwd().openFile(file_path, .{}); +- defer file.close(); ++pub fn detectFormatFromFilePath(std_io: std.Io, file_path: []const u8, read_buffer: []u8) !Format { ++ var file = try std.Io.Dir.openFile(std.Io.Dir.cwd(), std_io, file_path, .{}); ++ defer file.close(std_io); + +- return detectFormatFromFile(file, read_buffer); ++ return detectFormatFromFile(std_io, file, read_buffer); + } + + /// Detect which image format is used by the file +-pub fn detectFormatFromFile(file: std.fs.File, read_buffer: []u8) !Format { +- var read_stream = io.ReadStream.initFile(file, read_buffer); ++pub fn detectFormatFromFile(std_io: std.Io, file: std.Io.File, read_buffer: []u8) !Format { ++ var read_stream = io.ReadStream.initFile(std_io, file, read_buffer); + return internalDetectFormat(&read_stream); + } + +@@ -170,16 +170,16 @@ pub fn detectFormatFromMemory(buffer: []const u8) !Format { + } + + /// Load an image from a file path +-pub fn fromFilePath(allocator: std.mem.Allocator, file_path: []const u8, read_buffer: []u8) !Image { +- var file = try std.fs.cwd().openFile(file_path, .{}); +- defer file.close(); ++pub fn fromFilePath(allocator: std.mem.Allocator, std_io: std.Io, file_path: []const u8, read_buffer: []u8) !Image { ++ var file = try std.Io.Dir.openFile(std.Io.Dir.cwd(), std_io, file_path, .{}); ++ defer file.close(std_io); + +- return fromFile(allocator, file, read_buffer); ++ return fromFile(allocator, std_io, file, read_buffer); + } + +-/// Load an image from a standard library std.fs.File +-pub fn fromFile(allocator: std.mem.Allocator, file: std.fs.File, read_buffer: []u8) !Image { +- var read_stream = io.ReadStream.initFile(file, read_buffer); ++/// Load an image from a standard library std.Io.File ++pub fn fromFile(allocator: std.mem.Allocator, std_io: std.Io, file: std.Io.File, read_buffer: []u8) !Image { ++ var read_stream = io.ReadStream.initFile(std_io, file, read_buffer); + return internalRead(allocator, &read_stream); + } + +@@ -260,16 +260,16 @@ pub fn toManaged(self: Image, allocator: std.mem.Allocator) Managed { + } + + /// Write the image to an image format to the specified path +-pub fn writeToFilePath(self: Image, allocator: std.mem.Allocator, file_path: []const u8, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { +- var file = try std.fs.cwd().createFile(file_path, .{}); +- defer file.close(); ++pub fn writeToFilePath(self: Image, allocator: std.mem.Allocator, std_io: std.Io, file_path: []const u8, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { ++ var file = try std.Io.Dir.createFile(std.Io.Dir.cwd(), std_io, file_path, .{}); ++ defer file.close(std_io); + +- try self.writeToFile(allocator, file, write_buffer, encoder_options); ++ try self.writeToFile(allocator, std_io, file, write_buffer, encoder_options); + } + +-/// Write the image to an image format to the specified std.fs.File +-pub fn writeToFile(self: Image, allocator: std.mem.Allocator, file: std.fs.File, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { +- var write_stream = io.WriteStream.initFile(file, write_buffer); ++/// Write the image to an image format to the specified std.Io.File ++pub fn writeToFile(self: Image, allocator: std.mem.Allocator, std_io: std.Io, file: std.Io.File, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { ++ var write_stream = io.WriteStream.initFile(std_io, file, write_buffer); + + try self.internalWrite(allocator, &write_stream, encoder_options); + } +diff --git a/src/Image/Managed.zig b/src/Image/Managed.zig +index 608b1d6..1517e7c 100644 +--- a/src/Image/Managed.zig ++++ b/src/Image/Managed.zig +@@ -20,14 +20,16 @@ width: usize = 0, + height: usize = 0, + pixels: color.PixelStorage = .{ .invalid = void{} }, + animation: Animation = .{}, ++io: std.Io = undefined, + allocator: std.mem.Allocator = undefined, // Allocator needs to be last in order to be able to ptrCast to Image + + const Managed = @This(); + + /// Init an empty image with no pixel data +-pub fn init(allocator: std.mem.Allocator) Managed { ++pub fn init(allocator: std.mem.Allocator, io: std.Io) Managed { + return Managed{ + .allocator = allocator, ++ .io = io, + }; + } + +@@ -37,13 +39,13 @@ pub fn deinit(self: *Managed) void { + } + + /// Detect which image format is used by the file path +-pub fn detectFormatFromFilePath(file_path: []const u8, read_buffer: []u8) !Format { +- return Image.detectFormatFromFilePath(file_path, read_buffer); ++pub fn detectFormatFromFilePath(io: std.Io, file_path: []const u8, read_buffer: []u8) !Format { ++ return Image.detectFormatFromFilePath(io, file_path, read_buffer); + } + + /// Detect which image format is used by the file +-pub fn detectFormatFromFile(file: std.fs.File, read_buffer: []u8) !Format { +- return Image.detectFormatFromFile(file, read_buffer); ++pub fn detectFormatFromFile(io: std.Io, file: std.Io.File, read_buffer: []u8) !Format { ++ return Image.detectFormatFromFile(io, file, read_buffer); + } + + /// Detect which image format is used by the memory buffer +@@ -52,13 +54,13 @@ pub fn detectFormatFromMemory(buffer: []const u8) !Format { + } + + /// Load an image from a file path +-pub fn fromFilePath(allocator: std.mem.Allocator, file_path: []const u8, read_buffer: []u8) !Managed { +- return (try Image.fromFilePath(allocator, file_path, read_buffer)).toManaged(allocator); ++pub fn fromFilePath(allocator: std.mem.Allocator, io: std.Io, file_path: []const u8, read_buffer: []u8) !Managed { ++ return (try Image.fromFilePath(allocator, io, file_path, read_buffer)).toManaged(allocator); + } + +-/// Load an image from a standard library std.fs.File +-pub fn fromFile(allocator: std.mem.Allocator, file: std.fs.File, read_buffer: []u8) !Managed { +- return (try Image.fromFile(allocator, file, read_buffer)).toManaged(allocator); ++/// Load an image from a standard library std.Io.File ++pub fn fromFile(allocator: std.mem.Allocator, io: std.Io, file: std.Io.File, read_buffer: []u8) !Managed { ++ return (try Image.fromFile(allocator, io, file, read_buffer)).toManaged(allocator); + } + + /// Load an image from a memory buffer +@@ -116,12 +118,12 @@ pub fn isAnimation(self: Managed) bool { + + /// Write the image to an image format to the specified path + pub fn writeToFilePath(self: Managed, file_path: []const u8, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { +- return Image.writeToFilePath(self.toUnmanaged(), self.allocator, file_path, write_buffer, encoder_options); ++ return Image.writeToFilePath(self.toUnmanaged(), self.allocator, self.io, file_path, write_buffer, encoder_options); + } + +-/// Write the image to an image format to the specified std.fs.File +-pub fn writeToFile(self: Managed, file: std.fs.File, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { +- return Image.writeToFile(self.toUnmanaged(), self.allocator, file, write_buffer, encoder_options); ++/// Write the image to an image format to the specified std.Io.File ++pub fn writeToFile(self: Managed, file: std.Io.File, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { ++ return Image.writeToFile(self.toUnmanaged(), self.allocator, file, self.io, write_buffer, encoder_options); + } + + /// Write the image to an image format in a memory buffer. The memory buffer is not grown +diff --git a/src/color.zig b/src/color.zig +index 4591bce..8a1a6ac 100644 +--- a/src/color.zig ++++ b/src/color.zig +@@ -55,10 +55,7 @@ pub fn ScaleValue(comptime T: type) fn (anytype) T { + // In order to fit `value * out_max` we need an + // int of size `in_bits + out_bits` in the worst + // case scenario. +- const FitInt = @Type(.{ .int = .{ +- .bits = in_bits + out_bits, +- .signedness = .unsigned, +- } }); ++ const FitInt = @Int(.unsigned, in_bits + out_bits); + + return @truncate((@as(FitInt, value) * out_max + in_max / 2) / in_max); + } else return value; +@@ -395,7 +392,7 @@ fn ToMethods( + return packed struct(u0) { + const To = @This(); + +- fn getSelf(to: *const To) *align(1) const T { ++ fn getSelf(to: anytype) *align(1) const T { + // @fieldParentPtr is broken for packed structs. + // See: https://github.com/ziglang/zig/issues/20458 + if (@typeInfo(T).@"struct".layout == .@"packed") { +@@ -408,47 +405,47 @@ fn ToMethods( + } + + /// Assumes the target color type has `FromMethods` for it. +- pub fn color(to: *const To, ColorT: type) ColorT { +- return ColorT.from.color(to.getSelf().*); ++ pub fn color(to: anytype, ColorT: type) ColorT { ++ return ColorT.from.color(getSelf(to).*); + } + +- pub fn u32Rgba(to: *const To) u32 { +- const self = to.getSelf(); ++ pub fn u32Rgba(to: anytype) u32 { ++ const self = getSelf(to); + return @as(u32, toU8(self.r)) << 24 | + @as(u32, toU8(self.g)) << 16 | + @as(u32, toU8(self.b)) << 8 | + if (has_alpha) toU8(self.a) else 0xff; + } + +- pub fn u32Rgb(to: *const To) u32 { +- const self = to.getSelf(); ++ pub fn u32Rgb(to: anytype) u32 { ++ const self = getSelf(to); + return @as(u32, toU8(self.r)) << 16 | + @as(u32, toU8(self.g)) << 8 | + toU8(self.b); + } + +- pub fn u64Rgba(to: *const To) u64 { +- const self = to.getSelf(); ++ pub fn u64Rgba(to: anytype) u64 { ++ const self = getSelf(to); + return @as(u64, toU16(self.r)) << 48 | + @as(u64, toU16(self.g)) << 32 | + @as(u64, toU16(self.b)) << 16 | + if (has_alpha) toU16(self.a) else 0xffff; + } + +- pub fn u64Rgb(to: *const To) u64 { +- const self = to.getSelf(); ++ pub fn u64Rgb(to: anytype) u64 { ++ const self = getSelf(to); + return @as(u64, toU16(self.r)) << 32 | + @as(u64, toU16(self.g)) << 16 | + toU16(self.b); + } + + /// Only valid for color types where all channels are the same type. +- pub fn array(to: *const To) [4]RedT { ++ pub fn array(to: anytype) [4]RedT { + if (comptime multiple_channel_types) { + @compileError("Color.to.array may only be used when all channels in the color are the same type."); + } + +- const self = to.getSelf(); ++ const self = getSelf(to); + + return .{ + self.r, +@@ -461,8 +458,8 @@ fn ToMethods( + }; + } + +- pub fn float4(to: *const To) math.float4 { +- const self = to.getSelf(); ++ pub fn float4(to: anytype) math.float4 { ++ const self = getSelf(to); + + return .{ + toF32(self.r), +@@ -477,8 +474,8 @@ fn ToMethods( + + /// For int channels, premultiplication + /// is done with a round-trip to f32. +- pub fn premultipliedAlpha(to: *const To) T { +- const self = to.getSelf(); ++ pub fn premultipliedAlpha(to: anytype) T { ++ const self = getSelf(to); + var res = self.*; + if (!has_alpha) return res; + +diff --git a/src/formats/pam.zig b/src/formats/pam.zig +index f366d95..7ef9833 100644 +--- a/src/formats/pam.zig ++++ b/src/formats/pam.zig +@@ -77,7 +77,7 @@ const Header = struct { + } + + { +- var line_buffer_stream = try std.io.Writer.Allocating.initCapacity(allocator, 32); ++ var line_buffer_stream = try std.Io.Writer.Allocating.initCapacity(allocator, 32); + defer line_buffer_stream.deinit(); + + while (true) { +diff --git a/src/formats/pcx.zig b/src/formats/pcx.zig +index b694762..52c2bb0 100644 +--- a/src/formats/pcx.zig ++++ b/src/formats/pcx.zig +@@ -218,7 +218,7 @@ test "PCX RLE Fast encoder" { + const uncompressed_data = [_]u8{ 1, 1, 1, 1, 1, 1, 1, 1, 1, 64, 64, 2, 2, 2, 2, 2, 215, 215, 215, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 200, 200, 200, 200, 210, 210 }; + const compressed_data = [_]u8{ 0xC9, 0x01, 0xC2, 0x40, 0xC5, 0x02, 0xC3, 0xD7, 0xCA, 0x03, 0xC4, 0xC8, 0xC2, 0xD2 }; + +- var writer_alloc = std.io.Writer.Allocating.init(std.testing.allocator); ++ var writer_alloc = std.Io.Writer.Allocating.init(std.testing.allocator); + defer writer_alloc.deinit(); + + try RLEFastEncoder.encode(uncompressed_data[0..], &writer_alloc.writer); +@@ -233,7 +233,7 @@ test "PCX RLE Fast encoder should encore more than 63 bytes similar" { + + const compressed_data = [_]u8{ 0xFF, 0x45, 0x45, 0x45, 0xC4, 0x1 }; + +- var writer_alloc = std.io.Writer.Allocating.init(std.testing.allocator); ++ var writer_alloc = std.Io.Writer.Allocating.init(std.testing.allocator); + defer writer_alloc.deinit(); + + try RLEFastEncoder.encode(uncompressed_data[0..], &writer_alloc.writer); +diff --git a/src/formats/tiff/types.zig b/src/formats/tiff/types.zig +index 8bcce86..067480a 100644 +--- a/src/formats/tiff/types.zig ++++ b/src/formats/tiff/types.zig +@@ -335,7 +335,7 @@ pub const IFD = struct { + // Skip unknown tags instead of panicking on invalid enum values. + // Many TIFF files contain vendor/proprietary tags (Make, Model, ICC profile, etc.) + // that we don't need to handle for image decoding. +- const maybe_tag_id = std.meta.intToEnum(TagId, tag_id_raw) catch continue; ++ const maybe_tag_id = std.enums.fromInt(TagId, tag_id_raw) orelse continue; + + try self.tags_map.put(maybe_tag_id, TagField{ + .data_type = std.mem.toNative(u16, tag.data_type, endianess), +diff --git a/src/formats/xbm.zig b/src/formats/xbm.zig +index d72e4ec..19ac9c0 100644 +--- a/src/formats/xbm.zig ++++ b/src/formats/xbm.zig +@@ -100,7 +100,7 @@ pub const XBM = struct { + + var line_full = try reader.takeDelimiterInclusive('\n'); + while (line_full.len > 0) { +- const line = std.mem.trimRight(u8, line_full, " \r\n"); ++ const line = std.mem.trimEnd(u8, line_full, " \r\n"); + if (isDefineLine(line)) { + if (parseDefineValue(line)) |value| { + if (width == null) { +diff --git a/src/io.zig b/src/io.zig +index 2c1363d..8d463a1 100644 +--- a/src/io.zig ++++ b/src/io.zig +@@ -5,11 +5,11 @@ pub const DEFAULT_BUFFER_SIZE = 4096; + + pub const ReadStream = union(enum) { + memory: std.Io.Reader, +- file: std.fs.File.Reader, ++ file: std.Io.File.Reader, + + pub const Error = SeekError || EndPosError || std.Io.Reader.Error || std.Io.Reader.StreamError; +- pub const SeekError = std.fs.File.Reader.SeekError; +- pub const EndPosError = std.fs.File.Reader.SizeError; ++ pub const SeekError = std.Io.File.Reader.SeekError; ++ pub const EndPosError = std.Io.File.Reader.SizeError; + + pub fn initMemory(buffer: []const u8) ReadStream { + return .{ +@@ -17,9 +17,9 @@ pub const ReadStream = union(enum) { + }; + } + +- pub fn initFile(file: std.fs.File, buffer: []u8) ReadStream { ++ pub fn initFile(io: std.Io, file: std.Io.File, buffer: []u8) ReadStream { + return .{ +- .file = file.reader(buffer), ++ .file = file.reader(io, buffer), + }; + } + +@@ -58,7 +58,7 @@ pub const ReadStream = union(enum) { + .memory => |*memory| { + const new_pos: i64 = @as(i64, @intCast(memory.seek)) + offset; + if (new_pos < 0 or new_pos >= memory.end) { +- return std.fs.File.SeekError.Unseekable; ++ return std.Io.File.SeekError.Unseekable; + } + + memory.seek = @intCast(new_pos); +@@ -96,10 +96,10 @@ pub const ReadStream = union(enum) { + + pub const WriteStream = union(enum) { + memory: std.Io.Writer, +- file: std.fs.File.Writer, ++ file: std.Io.File.Writer, + + pub const Error = SeekError || std.Io.Writer.Error; +- pub const SeekError = std.fs.File.SeekError; ++ pub const SeekError = std.Io.File.SeekError || std.Io.Writer.Error; + + pub fn initMemory(buffer: []u8) WriteStream { + return .{ +@@ -107,9 +107,9 @@ pub const WriteStream = union(enum) { + }; + } + +- pub fn initFile(file: std.fs.File, buffer: []u8) WriteStream { ++ pub fn initFile(io: std.Io, file: std.Io.File, buffer: []u8) WriteStream { + return .{ +- .file = file.writer(buffer), ++ .file = file.writer(io, buffer), + }; + } + +diff --git a/src/math.zig b/src/math.zig +index c4e5910..bad9687 100644 +--- a/src/math.zig ++++ b/src/math.zig +@@ -24,11 +24,11 @@ pub fn Matrix(comptime T: type) type { + var temp = self; + + // To find the determinant, use a Gaussian elimination to transform the temp matrix into the row echelon form +- for (0..(ComponentSize - 1)) |row| { +- for ((row + 1)..ComponentSize) |next_row| { ++ inline for (0..(ComponentSize - 1)) |row| { ++ inline for ((row + 1)..ComponentSize) |next_row| { + const factor = temp.matrix[next_row][row] / temp.matrix[row][row]; + +- for (0..ComponentSize) |column| { ++ inline for (0..ComponentSize) |column| { + temp.matrix[next_row][column] = temp.matrix[next_row][column] - factor * temp.matrix[row][column]; + } + } +@@ -37,7 +37,7 @@ pub fn Matrix(comptime T: type) type { + // Once the temp matrix is in row echelon form, multiply the diagonal to find the determinant + var result: f32 = temp.matrix[0][0]; + +- for (1..ComponentSize) |diagonal| { ++ inline for (1..ComponentSize) |diagonal| { + result *= temp.matrix[diagonal][diagonal]; + } + +@@ -48,9 +48,9 @@ pub fn Matrix(comptime T: type) type { + pub fn fromArray(array: [ComponentSize * ComponentSize]f32) Self { + var result: Self = undefined; + +- for (0..ComponentSize) |row| { ++ inline for (0..ComponentSize) |row| { + const stride = row * ComponentSize; +- for (0..ComponentSize) |column| { ++ inline for (0..ComponentSize) |column| { + result.matrix[row][column] = array[stride + column]; + } + } +@@ -72,20 +72,20 @@ pub fn Matrix(comptime T: type) type { + var temp = self; + var result: Self = identity(); + +- for (0..ComponentSize) |row| { ++ inline for (0..ComponentSize) |row| { + // Transform the pivot to 1 by multiplying the row by its inverse + const inverse_pivot = 1.0 / temp.matrix[row][row]; + +- for (0..ComponentSize) |column| { ++ inline for (0..ComponentSize) |column| { + temp.matrix[row][column] *= inverse_pivot; + result.matrix[row][column] *= inverse_pivot; + } + + // Then do Gaussian elimination from current row to bottom +- for ((row + 1)..ComponentSize) |next_row| { ++ inline for ((row + 1)..ComponentSize) |next_row| { + const factor = temp.matrix[next_row][row] / temp.matrix[row][row]; + +- for (0..ComponentSize) |column| { ++ inline for (0..ComponentSize) |column| { + temp.matrix[next_row][column] = temp.matrix[next_row][column] - factor * temp.matrix[row][column]; + result.matrix[next_row][column] = result.matrix[next_row][column] - factor * result.matrix[row][column]; + } +@@ -94,14 +94,12 @@ pub fn Matrix(comptime T: type) type { + + // Then do a Gaussian elimination bottom-up to transform the temp matrix into an identity matrix. + // The result will be the inverse of the matrix +- var row: usize = ComponentSize -% 1; +- while (row >= 1 and row < ComponentSize) : (row -%= 1) { +- var previous_row = row -% 1; +- +- while (previous_row >= 0 and previous_row < ComponentSize) : (previous_row -%= 1) { ++ inline for (1..ComponentSize) |i| { ++ const row = ComponentSize - i; ++ inline for (0..row) |previous_row| { + const factor = temp.matrix[previous_row][row] / temp.matrix[row][row]; + +- for (0..ComponentSize) |column| { ++ inline for (0..ComponentSize) |column| { + temp.matrix[previous_row][column] = temp.matrix[previous_row][column] - factor * temp.matrix[row][column]; + result.matrix[previous_row][column] = result.matrix[previous_row][column] - factor * result.matrix[row][column]; + } +@@ -126,8 +124,8 @@ pub fn Matrix(comptime T: type) type { + + const transposed_right = right.transpose(); + +- for (0..ComponentSize) |row| { +- for (0..ComponentSize) |column| { ++ inline for (0..ComponentSize) |row| { ++ inline for (0..ComponentSize) |column| { + result.matrix[row][column] = @reduce(.Add, self.matrix[row] * transposed_right.matrix[column]); + } + } +@@ -138,8 +136,8 @@ pub fn Matrix(comptime T: type) type { + pub fn transpose(self: Self) Self { + var result = std.mem.zeroes(Self); + +- for (0..ComponentSize) |row| { +- for (0..ComponentSize) |column| { ++ inline for (0..ComponentSize) |row| { ++ inline for (0..ComponentSize) |column| { + result.matrix[row][column] = self.matrix[column][row]; + } + } +diff --git a/src/simd.zig b/src/simd.zig +index 20c5dcb..cb115c9 100644 +--- a/src/simd.zig ++++ b/src/simd.zig +@@ -78,8 +78,10 @@ pub fn vpermd(v: @Vector(8, i32), mask: @Vector(8, i32)) @Vector(8, i32) { + [v] "x" (v), + ); + } else { +- var res: @Vector(8, i32) = undefined; +- inline for (0..8) |i| res[i] = v[@as(u32, @bitCast(mask[i]))]; ++ const v_arr: [8]i32 = v; ++ const mask_arr: [8]i32 = mask; ++ var res: [8]i32 = undefined; ++ for (0..8) |i| res[i] = v_arr[@intCast(mask_arr[i] & 7)]; + return res; + } + } +diff --git a/src/utils.zig b/src/utils.zig +index f55c12c..a12733e 100644 +--- a/src/utils.zig ++++ b/src/utils.zig +@@ -4,7 +4,7 @@ const std = @import("std"); + const native_endian = builtin.target.cpu.arch.endian(); + + pub const StructReadError = error{InvalidData} || std.Io.Reader.Error; +-pub const StructWriteError = std.io.StreamSource.WriteError; ++pub const StructWriteError = std.Io.Writer.Error; + + pub fn FixedStorage(comptime T: type, comptime storage_size: usize) type { + return struct { +diff --git a/tests/formats/bmp_test.zig b/tests/formats/bmp_test.zig +index cc80b74..635dc04 100644 +--- a/tests/formats/bmp_test.zig ++++ b/tests/formats/bmp_test.zig +@@ -87,12 +87,12 @@ fn verifyBitmapRGBAV5(the_bitmap: bmp.BMP, pixels: zigimg.color.PixelStorage) !v + + test "Read simple version 4 24-bit RGB bitmap" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_bitmap = bmp.BMP{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_bitmap.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -145,10 +145,10 @@ test "Read simple version 4 24-bit RGB bitmap" { + + test "Read a valid version 5 RGBA bitmap from file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/windows_rgba_v5.bmp"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var the_bitmap = bmp.BMP{}; + +@@ -174,10 +174,10 @@ test "Read a valid version 5 RGBA bitmap from memory" { + + test "Should error when reading an invalid file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/notbmp.png"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var the_bitmap = bmp.BMP{}; + +@@ -215,17 +215,17 @@ test "Write a v4 bitmap when writing an image with bgr24 pixel format" { + pixels.bgr24[7] = zigimg.color.Bgr24.from.rgb(255, 255, 0); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .bmp = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .bmp = .{} }); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var read_bmp = bmp.BMP{}; + +@@ -277,17 +277,17 @@ test "Write a v5 bitmap when writing an image with bgra32 pixel format" { + pixels.bgra32[8] = zigimg.color.Bgra32.from.rgba(0, 0, 0, 0); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .bmp = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .bmp = .{} }); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var read_bmp = bmp.BMP{}; + +diff --git a/tests/formats/farbfeld_test.zig b/tests/formats/farbfeld_test.zig +index a48a7fb..00a73a0 100644 +--- a/tests/formats/farbfeld_test.zig ++++ b/tests/formats/farbfeld_test.zig +@@ -5,10 +5,10 @@ const farbfeld = zigimg.formats.farbfeld; + + test "Farbfeld: Read yellow file" { + const yellow_file = try helpers.testOpenFile(helpers.fixtures_path ++ "farbfeld/yellow-1x1-semitransparent.png.ff"); +- defer yellow_file.close(); ++ defer yellow_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var yellow_read_stream = zigimg.io.ReadStream.initFile(yellow_file, read_buffer[0..]); ++ var yellow_read_stream = zigimg.io.ReadStream.initFile(std.testing.io, yellow_file, read_buffer[0..]); + + var yellow_image = farbfeld.Farbfeld{}; + const yellow_pixels = try yellow_image.read(helpers.zigimg_test_allocator, &yellow_read_stream); +@@ -21,10 +21,10 @@ test "Farbfeld: Read yellow file" { + + test "Farbfeld: read dragon file" { + const dragon_file = try helpers.testOpenFile(helpers.fixtures_path ++ "farbfeld/dragon.ff"); +- defer dragon_file.close(); ++ defer dragon_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var dragon_read_stream = zigimg.io.ReadStream.initFile(dragon_file, read_buffer[0..]); ++ var dragon_read_stream = zigimg.io.ReadStream.initFile(std.testing.io, dragon_file, read_buffer[0..]); + + var dragon_image = farbfeld.Farbfeld{}; + const dragon_pixels = try dragon_image.read(helpers.zigimg_test_allocator, &dragon_read_stream); +@@ -37,10 +37,10 @@ test "Farbfeld: read dragon file" { + + test "Farbfeld: invalid file format" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "farbfeld/dragon.png"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var farbfeld_image = farbfeld.Farbfeld{}; + const image_error = farbfeld_image.read(helpers.zigimg_test_allocator, &read_stream); +@@ -50,10 +50,10 @@ test "Farbfeld: invalid file format" { + + test "Farbfeld: read writeImage output" { + const source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "farbfeld/yellow-1x1-semitransparent.png.ff"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var source_read_stream = zigimg.io.ReadStream.initFile(source_file, read_buffer[0..]); ++ var source_read_stream = zigimg.io.ReadStream.initFile(std.testing.io, source_file, read_buffer[0..]); + + var source_image: farbfeld.Farbfeld = .{}; + const source_pixels = try source_image.read(helpers.zigimg_test_allocator, &source_read_stream); +diff --git a/tests/formats/gif_test.zig b/tests/formats/gif_test.zig +index 496e14f..bdd84f6 100644 +--- a/tests/formats/gif_test.zig ++++ b/tests/formats/gif_test.zig +@@ -325,10 +325,10 @@ test "GIF writer roundtrip read existing GIF, write, read again" { + + // Read an existing GIF file using low-level reader + const gif_input_file = try helpers.testOpenFile(helpers.fixtures_path ++ "gif/rotating_earth.gif"); +- defer gif_input_file.close(); ++ defer gif_input_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(gif_input_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, gif_input_file, read_buffer[0..]); + + var original_gif = gif.GIF.init(allocator); + defer original_gif.deinit(); +@@ -489,10 +489,10 @@ test "GIF writer rejects non-indexed data without auto convert" { + + test "Should error on non GIF images" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var gif_file = gif.GIF.init(helpers.zigimg_test_allocator); + defer gif_file.deinit(); +@@ -563,7 +563,7 @@ const SINGLE_GIF_FILE_TEST = false; + fn loadRotatingEarthImage(allocator: std.mem.Allocator) !zigimg.Image { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_path = helpers.fixtures_path ++ "gif/rotating_earth.gif"; +- return zigimg.Image.fromFilePath(allocator, image_path, read_buffer[0..]); ++ return zigimg.Image.fromFilePath(allocator, std.testing.io, image_path, read_buffer[0..]); + } + + test "GIF test suite" { +@@ -575,10 +575,10 @@ test "GIF test suite" { + defer test_list.deinit(helpers.zigimg_test_allocator); + + const test_list_file = try helpers.testOpenFile(helpers.fixtures_path ++ "gif/TESTS"); +- defer test_list_file.close(); ++ defer test_list_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(test_list_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, test_list_file, read_buffer[0..]); + + var reader = read_stream.reader(); + +@@ -598,11 +598,11 @@ test "GIF test suite" { + for (test_list.items) |entry| { + doGifTest(entry) catch |err| { + has_failed_file = true; +- std.debug.print("Error: {}\n", .{err}); ++ std.log.info("Error: {}\n", .{err}); + continue; + }; + +- std.debug.print("OK\n", .{}); ++ std.log.info("OK\n", .{}); + } + + if (has_failed_file) { +@@ -612,10 +612,10 @@ test "GIF test suite" { + + test "Rotating Earth GIF" { + const gif_input_file = try helpers.testOpenFile(helpers.fixtures_path ++ "gif/rotating_earth.gif"); +- defer gif_input_file.close(); ++ defer gif_input_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(gif_input_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, gif_input_file, read_buffer[0..]); + + var gif_file = gif.GIF.init(helpers.zigimg_test_allocator); + defer gif_file.deinit(); +@@ -710,8 +710,8 @@ const IniFile = struct { + const equals_sign_position_opt = std.mem.indexOf(u8, read_line[0..], "="); + + if (equals_sign_position_opt) |equals_sign_position| { +- const key_name = std.mem.trimRight(u8, read_line[0..(equals_sign_position - 1)], " "); +- const string_value = std.mem.trimLeft(u8, read_line[(equals_sign_position + 1)..], " "); ++ const key_name = std.mem.trimEnd(u8, read_line[0..(equals_sign_position - 1)], " "); ++ const string_value = std.mem.trimStart(u8, read_line[(equals_sign_position + 1)..], " "); + + if (self.sections.getPtr(current_section)) |section_entry| { + const value = blk: { +@@ -760,7 +760,7 @@ const IniFile = struct { + }; + + fn doGifTest(entry_name: []const u8) !void { +- std.debug.print("GIF test {s}... ", .{entry_name}); ++ std.log.info("GIF test {s}... ", .{entry_name}); + + var area_alloc = std.heap.ArenaAllocator.init(helpers.zigimg_test_allocator); + const area_allocator = area_alloc.allocator(); +@@ -770,13 +770,13 @@ fn doGifTest(entry_name: []const u8) !void { + const config_filepath = try std.fs.path.resolve(area_allocator, &[_][]const u8{ helpers.fixtures_path, "gif", config_filename }); + + const config_file = try helpers.testOpenFile(config_filepath); +- defer config_file.close(); ++ defer config_file.close(std.testing.io); + + var config_ini = IniFile.init(helpers.zigimg_test_allocator); + defer config_ini.deinit(); + + var ini_read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var ini_read_stream = zigimg.io.ReadStream.initFile(config_file, ini_read_buffer[0..]); ++ var ini_read_stream = zigimg.io.ReadStream.initFile(std.testing.io, config_file, ini_read_buffer[0..]); + + try config_ini.parse(ini_read_stream.reader()); + +@@ -804,10 +804,10 @@ fn doGifTest(entry_name: []const u8) !void { + + const gif_input_filepath = try std.fs.path.resolve(area_allocator, &[_][]const u8{ helpers.fixtures_path, "gif", input_filename.string }); + const gif_input_file = try helpers.testOpenFile(gif_input_filepath); +- defer gif_input_file.close(); ++ defer gif_input_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(gif_input_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, gif_input_file, read_buffer[0..]); + + var gif_file = gif.GIF.init(helpers.zigimg_test_allocator); + defer gif_file.deinit(); +@@ -856,10 +856,10 @@ fn doGifTest(entry_name: []const u8) !void { + const pixels_filename = frame_section.getValue("pixels") orelse return error.InvalidGifConfigFile; + const pixels_filepath = try std.fs.path.resolve(area_allocator, &[_][]const u8{ helpers.fixtures_path, "gif", pixels_filename.string }); + const pixels_file = try helpers.testOpenFile(pixels_filepath); +- defer pixels_file.close(); ++ defer pixels_file.close(std.testing.io); + + var pixels_read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var pixels_read_stream = zigimg.io.ReadStream.initFile(pixels_file, pixels_read_buffer[0..]); ++ var pixels_read_stream = zigimg.io.ReadStream.initFile(std.testing.io, pixels_file, pixels_read_buffer[0..]); + + var pixels_reader = pixels_read_stream.reader(); + +@@ -894,7 +894,7 @@ fn doGifTest(entry_name: []const u8) !void { + try helpers.expectEq(actual_color.to.color(zigimg.color.Rgba32), gif_background_color); + } else { + helpers.expectEq(actual_color.to.color(zigimg.color.Rgba32), expected_color) catch |err| { +- std.debug.print("Pixel #{} does not match: expected={}, actual={}\n", .{ index, expected_color, actual_color.to.color(zigimg.color.Rgba32) }); ++ std.log.info("Pixel #{} does not match: expected={}, actual={}\n", .{ index, expected_color, actual_color.to.color(zigimg.color.Rgba32) }); + return err; + }; + } +@@ -929,27 +929,27 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { + + // Build path to GIF file + const gif_filename = std.fmt.allocPrint(area_allocator, "{s}.gif", .{entry_name}) catch { +- std.debug.print("FAIL (allocPrint failed)\n", .{}); ++ std.log.info("FAIL (allocPrint failed)\n", .{}); + return .failed; + }; + const gif_filepath = std.fs.path.resolve(area_allocator, &[_][]const u8{ helpers.fixtures_path, "gif", gif_filename }) catch { +- std.debug.print("FAIL (path resolve failed)\n", .{}); ++ std.log.info("FAIL (path resolve failed)\n", .{}); + return .failed; + }; + + // Read original GIF +- const gif_input_file = try std.fs.cwd().openFile(gif_filepath, .{}); +- defer gif_input_file.close(); ++ const gif_input_file = try std.Io.Dir.openFile(std.Io.Dir.cwd(), std.testing.io, gif_filepath, .{}); ++ defer gif_input_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(gif_input_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, gif_input_file, read_buffer[0..]); + + var original_gif = gif.GIF.init(helpers.zigimg_test_allocator); + defer original_gif.deinit(); + + var original_frames = original_gif.read(&read_stream) catch { + // Reader correctly rejected bad data +- std.debug.print("OK (reader correctly rejected bad data)\n", .{}); ++ std.log.info("OK (reader correctly rejected bad data)\n", .{}); + return .passed; + }; + defer { +@@ -961,7 +961,7 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { + + if (original_frames.items.len == 0) { + // No frames means nothing to write +- std.debug.print("OK (no frames to write)\n", .{}); ++ std.log.info("OK (no frames to write)\n", .{}); + return .passed; + } + +@@ -974,13 +974,13 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { + + // Non-indexed formats can't be written without auto_convert + if (!image.pixelFormat().isIndexed()) { +- std.debug.print("OK (non-indexed format correctly not written: {s})\n", .{@tagName(image.pixels)}); ++ std.log.info("OK (non-indexed format correctly not written: {s})\n", .{@tagName(image.pixels)}); + return .passed; + } + + // Zero dimensions can't be written + if (image.width == 0 or image.height == 0) { +- std.debug.print("OK (zero dimensions correctly not written)\n", .{}); ++ std.log.info("OK (zero dimensions correctly not written)\n", .{}); + return .passed; + } + +@@ -991,16 +991,16 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { + gif.GIF.writeImage(helpers.zigimg_test_allocator, &write_stream, image, .{ .gif = .{} }) catch |err| { + // For very large images, running out of buffer space is expected + if (err == error.WriteFailed and (image.width > 1000 or image.height > 1000)) { +- std.debug.print("OK (large image exceeded buffer)\n", .{}); ++ std.log.info("OK (large image exceeded buffer)\n", .{}); + return .passed; + } +- std.debug.print("FAIL (write error: {})\n", .{err}); ++ std.log.info("FAIL (write error: {})\n", .{err}); + return .failed; + }; + + const written_len = write_stream.writer().end; + if (written_len == 0) { +- std.debug.print("FAIL (wrote 0 bytes)\n", .{}); ++ std.log.info("FAIL (wrote 0 bytes)\n", .{}); + return .failed; + } + +@@ -1011,7 +1011,7 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { + defer decoded_gif.deinit(); + + var decoded_frames = decoded_gif.read(&read_back_stream) catch |err| { +- std.debug.print("FAIL (read-back error: {})\n", .{err}); ++ std.log.info("FAIL (read-back error: {})\n", .{err}); + return .failed; + }; + defer { +@@ -1025,7 +1025,7 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { + if (original_gif.header.width != decoded_gif.header.width or + original_gif.header.height != decoded_gif.header.height) + { +- std.debug.print("FAIL (dimension mismatch: {}x{} vs {}x{})\n", .{ ++ std.log.info("FAIL (dimension mismatch: {}x{} vs {}x{})\n", .{ + original_gif.header.width, + original_gif.header.height, + decoded_gif.header.width, +@@ -1036,7 +1036,7 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { + + // Verify we got at least one frame + if (decoded_frames.items.len == 0) { +- std.debug.print("FAIL (no frames in output)\n", .{}); ++ std.log.info("FAIL (no frames in output)\n", .{}); + return .failed; + } + +@@ -1048,14 +1048,14 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { + while (original_iter.next()) |original_color| { + const decoded_color_opt = decoded_iter.next(); + if (decoded_color_opt == null) { +- std.debug.print("FAIL (missing pixel at index {})\n", .{pixel_index}); ++ std.log.info("FAIL (missing pixel at index {})\n", .{pixel_index}); + return .failed; + } + const original = original_color.to.color(zigimg.color.Rgba32); + const decoded = decoded_color_opt.?.to.color(zigimg.color.Rgba32); + + if (original.r != decoded.r or original.g != decoded.g or original.b != decoded.b) { +- std.debug.print("FAIL (pixel {} mismatch: ({},{},{}) vs ({},{},{}))\n", .{ ++ std.log.info("FAIL (pixel {} mismatch: ({},{},{}) vs ({},{},{}))\n", .{ + pixel_index, original.r, original.g, original.b, decoded.r, decoded.g, decoded.b, + }); + return .failed; +@@ -1071,10 +1071,10 @@ test "GIF writer roundtrip test suite" { + defer test_list.deinit(helpers.zigimg_test_allocator); + + const test_list_file = try helpers.testOpenFile(helpers.fixtures_path ++ "gif/TESTS"); +- defer test_list_file.close(); ++ defer test_list_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(test_list_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, test_list_file, read_buffer[0..]); + + var reader = read_stream.reader(); + +diff --git a/tests/formats/iff_test.zig b/tests/formats/iff_test.zig +index ab2dd6b..6a3594d 100644 +--- a/tests/formats/iff_test.zig ++++ b/tests/formats/iff_test.zig +@@ -5,12 +5,12 @@ const zigimg = @import("zigimg"); + + test "IFF-PBM indexed8 (chunky Deluxe Paint DOS file)" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-pbm.iff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_iff = iff.IFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -37,12 +37,12 @@ test "IFF-PBM indexed8 (chunky Deluxe Paint DOS file)" { + + test "IFF-ILBM indexed8 8 bitplanes" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-ilbm-8bit-compressed.iff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_iff = iff.IFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -63,12 +63,12 @@ test "IFF-ILBM indexed8 8 bitplanes" { + + test "IFF-ILBM indexed8 8 bitplanes uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-ilbm-8bit-uncompressed.iff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_iff = iff.IFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -89,12 +89,12 @@ test "IFF-ILBM indexed8 8 bitplanes uncompressed" { + + test "IFF-ILBM indexed8 6 bitplanes EHB" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-ehb.iff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_iff = iff.IFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -121,12 +121,12 @@ test "IFF-ILBM indexed8 6 bitplanes EHB" { + + test "IFF-ILBM indexed8 4 bitplanes HAM" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-ham.iff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_iff = iff.IFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -150,12 +150,12 @@ test "IFF-ILBM indexed8 4 bitplanes HAM" { + + test "IFF-ILBM indexed8 6 bitplanes HAM8" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-ham8.iff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_iff = iff.IFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -179,12 +179,12 @@ test "IFF-ILBM indexed8 6 bitplanes HAM8" { + + test "IFF-ILBM 24bit" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-24bit.iff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_iff = iff.IFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -208,12 +208,12 @@ test "IFF-ILBM 24bit" { + + test "IFF-ILBM indexed8 4 bitplanes Atari ST" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-ilbm-4bit-compressed-atari.iff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_iff = iff.IFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -240,12 +240,12 @@ test "IFF-ILBM indexed8 4 bitplanes Atari ST" { + + test "IFF-ACBM indexed8 3 bitplanes uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-8bit.acbm"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_iff = iff.IFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); + defer pixels.deinit(helpers.zigimg_test_allocator); +diff --git a/tests/formats/jpeg_test.zig b/tests/formats/jpeg_test.zig +index 8288b62..0f1254f 100644 +--- a/tests/formats/jpeg_test.zig ++++ b/tests/formats/jpeg_test.zig +@@ -7,10 +7,10 @@ const color = zigimg.color; + + test "Should error on non JPEG images" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); + defer jpeg_file.deinit(); +@@ -28,10 +28,10 @@ test "Should error on non JPEG images" { + + test "Read JFIF header properly and decode simple Huffman stream" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "jpeg/huff_simple0.jpg"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); + defer jpeg_file.deinit(); +@@ -59,10 +59,10 @@ test "Read JFIF header properly and decode simple Huffman stream" { + + test "Read the tuba properly" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "jpeg/tuba.jpg"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); + defer jpeg_file.deinit(); +@@ -95,10 +95,10 @@ test "Read the tuba properly" { + + test "Read grayscale images" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "jpeg/grayscale_sample0.jpg"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); + defer jpeg_file.deinit(); +@@ -130,19 +130,19 @@ test "Read grayscale images" { + } + + test "Read subsampling images" { +- var testdir = std.fs.cwd().openDir(helpers.fixtures_path ++ "jpeg/", .{ .access_sub_paths = false, .no_follow = true, .iterate = true }) catch null; ++ var testdir = std.Io.Dir.openDir(std.Io.Dir.cwd(), std.testing.io, helpers.fixtures_path ++ "jpeg/", .{ .access_sub_paths = false, .iterate = true }) catch null; + if (testdir) |*idir| { +- defer idir.close(); ++ defer idir.close(std.testing.io); + + var it = idir.iterate(); +- while (try it.next()) |entry| { ++ while (try it.next(std.testing.io)) |entry| { + if (entry.kind != .file or !std.mem.endsWith(u8, entry.name, ".jpg") or !std.mem.startsWith(u8, entry.name, "subsampling_")) continue; + +- var test_file = try idir.openFile(entry.name, .{ .mode = .read_only }); +- defer test_file.close(); ++ var test_file = try idir.openFile(std.testing.io, entry.name, .{ .mode = .read_only }); ++ defer test_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(test_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, test_file, read_buffer[0..]); + + var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); + defer jpeg_file.deinit(); +@@ -182,10 +182,10 @@ test "Read subsampling images" { + + test "Read progressive jpeg with restart intervals" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "jpeg/tuba_restart_prog.jpg"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); + defer jpeg_file.deinit(); +@@ -403,10 +403,10 @@ test "JPEG writer round-trip with all test fixtures" { + for (test_fixtures) |fixture_path| { + // Skip if file doesn't exist + const file = helpers.testOpenFile(fixture_path) catch continue; +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [4096]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); + defer jpeg_file.deinit(); + +diff --git a/tests/formats/netpbm_test.zig b/tests/formats/netpbm_test.zig +index ee06575..e0ffe05 100644 +--- a/tests/formats/netpbm_test.zig ++++ b/tests/formats/netpbm_test.zig +@@ -5,10 +5,10 @@ const zigimg = @import("zigimg"); + + test "Load ASCII PBM image" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/pbm_ascii.pbm"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var pbm_file = netpbm.PBM{}; + +@@ -27,10 +27,10 @@ test "Load ASCII PBM image" { + + test "Load binary PBM image" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/pbm_binary.pbm"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var pbm_file = netpbm.PBM{}; + +@@ -49,10 +49,10 @@ test "Load binary PBM image" { + + test "Load ASCII PGM 8-bit grayscale image" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/pgm_ascii_grayscale8.pgm"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var pgm_file = netpbm.PGM{}; + +@@ -71,10 +71,10 @@ test "Load ASCII PGM 8-bit grayscale image" { + + test "Load Binary PGM 8-bit grayscale image" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/pgm_binary_grayscale8.pgm"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var pgm_file = netpbm.PGM{}; + +@@ -93,10 +93,10 @@ test "Load Binary PGM 8-bit grayscale image" { + + test "Load ASCII PGM 16-bit grayscale image" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/pgm_ascii_grayscale16.pgm"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var pgm_file = netpbm.PGM{}; + +@@ -115,10 +115,10 @@ test "Load ASCII PGM 16-bit grayscale image" { + + test "Load Binary PGM 16-bit grayscale image" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/pgm_binary_grayscale16.pgm"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var pgm_file = netpbm.PGM{}; + +@@ -137,10 +137,10 @@ test "Load Binary PGM 16-bit grayscale image" { + + test "Load ASCII PPM image" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/ppm_ascii_rgb24.ppm"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var ppm_file = netpbm.PPM{}; + +@@ -176,10 +176,10 @@ test "Load ASCII PPM image" { + + test "Load binary PPM image" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/ppm_binary_rgb24.ppm"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var ppm_file = netpbm.PPM{}; + +@@ -234,16 +234,16 @@ test "Write bitmap(grayscale1) ASCII PBM file" { + } + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .pbm = .{ .binary = false }, + }); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch unreachable; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; + } + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, image_file_name, read_buffer[0..]); ++ var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, read_buffer[0..]); + defer read_image.deinit(helpers.zigimg_test_allocator); + + try helpers.expectEq(read_image.width, width); +@@ -281,16 +281,16 @@ test "Write bitmap(Grayscale1) binary PBM file" { + } + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .pbm = .{ .binary = true }, + }); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch unreachable; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; + } + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, image_file_name, read_buffer[0..]); ++ var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, read_buffer[0..]); + defer read_image.deinit(helpers.zigimg_test_allocator); + + try helpers.expectEq(read_image.width, width); +@@ -324,16 +324,16 @@ test "Write grayscale8 ASCII PGM file" { + } + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .pgm = .{ .binary = false }, + }); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch unreachable; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; + } + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, image_file_name, read_buffer[0..]); ++ var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, read_buffer[0..]); + defer read_image.deinit(helpers.zigimg_test_allocator); + + try helpers.expectEq(read_image.width, width); +@@ -367,16 +367,16 @@ test "Write grayscale8 binary PGM file" { + } + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .pgm = .{ .binary = true }, + }); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch unreachable; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; + } + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, image_file_name, read_buffer[0..]); ++ var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, read_buffer[0..]); + defer read_image.deinit(helpers.zigimg_test_allocator); + + try helpers.expectEq(read_image.width, width); +@@ -420,16 +420,16 @@ test "Writing Rgb24 ASCII PPM format" { + pixels.rgb24[7] = zigimg.color.Rgb24.from.rgb(255, 255, 0); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .ppm = .{ .binary = false }, + }); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch unreachable; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; + } + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, image_file_name, read_buffer[0..]); ++ var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, read_buffer[0..]); + defer read_image.deinit(helpers.zigimg_test_allocator); + + try helpers.expectEq(read_image.width, width); +@@ -474,16 +474,16 @@ test "Writing Rgb24 binary PPM format" { + pixels.rgb24[7] = zigimg.color.Rgb24.from.rgb(255, 255, 0); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .ppm = .{ .binary = true }, + }); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch unreachable; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; + } + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, image_file_name, read_buffer[0..]); ++ var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, read_buffer[0..]); + defer read_image.deinit(helpers.zigimg_test_allocator); + + try helpers.expectEq(read_image.width, width); +@@ -505,7 +505,7 @@ test "Trying to write a bitmap or grayscale Netpbm with an true color pixel form + defer source_image.deinit(helpers.zigimg_test_allocator); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch unreachable; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; + } + + const pixels = source_image.pixels; +@@ -527,12 +527,12 @@ test "Trying to write a bitmap or grayscale Netpbm with an true color pixel form + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + + { +- const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .pbm = .{} }); ++ const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .pbm = .{} }); + try std.testing.expectError(zigimg.Image.WriteError.Unsupported, write_error); + } + + { +- const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .pgm = .{} }); ++ const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .pgm = .{} }); + try std.testing.expectError(zigimg.Image.WriteError.Unsupported, write_error); + } + } +@@ -551,7 +551,7 @@ test "Trying to write a bitmap or true color Netpbm with a 8-bit grayscale pixel + defer source_image.deinit(helpers.zigimg_test_allocator); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch unreachable; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; + } + + const source = source_image.pixels; +@@ -562,12 +562,12 @@ test "Trying to write a bitmap or true color Netpbm with a 8-bit grayscale pixel + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + + { +- const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .pbm = .{} }); ++ const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .pbm = .{} }); + try std.testing.expectError(zigimg.Image.WriteError.Unsupported, write_error); + } + + { +- const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .ppm = .{} }); ++ const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .ppm = .{} }); + try std.testing.expectError(zigimg.Image.WriteError.Unsupported, write_error); + } + } +@@ -589,7 +589,7 @@ test "Trying to write a grayscale or true color Netbpm with a 1-bit grayscale pi + defer source_image.deinit(helpers.zigimg_test_allocator); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch unreachable; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; + } + + const source = source_image.pixels; +@@ -601,12 +601,12 @@ test "Trying to write a grayscale or true color Netbpm with a 1-bit grayscale pi + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + + { +- const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .pgm = .{} }); ++ const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .pgm = .{} }); + try std.testing.expectError(zigimg.Image.WriteError.Unsupported, write_error); + } + + { +- const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .ppm = .{} }); ++ const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .ppm = .{} }); + try std.testing.expectError(zigimg.Image.WriteError.Unsupported, write_error); + } + } +diff --git a/tests/formats/pam_test.zig b/tests/formats/pam_test.zig +index 2d58d45..5742f77 100644 +--- a/tests/formats/pam_test.zig ++++ b/tests/formats/pam_test.zig +@@ -5,10 +5,10 @@ const zigimg = @import("zigimg"); + + test "rejects non-PAM images" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + +@@ -17,10 +17,10 @@ test "rejects non-PAM images" { + + test "rejects PAM images with unsupported depth" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/unsupported_depth.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + +@@ -29,10 +29,10 @@ test "rejects PAM images with unsupported depth" { + + test "rejects PAM images with invalid maxval" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/invalid_maxval.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + +@@ -41,10 +41,10 @@ test "rejects PAM images with invalid maxval" { + + test "rejects PAM images with component values greater than maxval" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/value_greater_than_maxval.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + try helpers.expectError(invalid, zigimg.Image.ReadError.InvalidData); +@@ -52,10 +52,10 @@ test "rejects PAM images with component values greater than maxval" { + + test "rejects PAM images with unknown tuple type" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/unknown_tupletype.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + +@@ -64,10 +64,10 @@ test "rejects PAM images with unknown tuple type" { + + test "rejects PAM images with invalid first token" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/invalid_first_token.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + +@@ -76,10 +76,10 @@ test "rejects PAM images with invalid first token" { + + test "rejects PAM images with tuple type not matching other parameters" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/non_matching_tuple_type.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + +@@ -88,10 +88,10 @@ test "rejects PAM images with tuple type not matching other parameters" { + + test "accepts comments" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_blackandwhite_comments.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + defer image.deinit(helpers.zigimg_test_allocator); +@@ -110,10 +110,10 @@ test "accepts comments" { + + test "reads blackandwhite pam" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_blackandwhite.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + defer image.deinit(helpers.zigimg_test_allocator); +@@ -132,10 +132,10 @@ test "reads blackandwhite pam" { + + test "reads blackandwhite_alpha pam" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_blackandwhite_alpha.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + defer image.deinit(helpers.zigimg_test_allocator); +@@ -154,10 +154,10 @@ test "reads blackandwhite_alpha pam" { + + test "reads grayscale pam with maxval 255" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_grayscale_maxval_255.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + defer image.deinit(helpers.zigimg_test_allocator); +@@ -176,10 +176,10 @@ test "reads grayscale pam with maxval 255" { + + test "reads grayscale alpha pam with maxval 255" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_grayscale_alpha_maxval_255.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + defer image.deinit(helpers.zigimg_test_allocator); +@@ -198,10 +198,10 @@ test "reads grayscale alpha pam with maxval 255" { + + test "read of rgb pam with maxval 255" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/horse.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + defer image.deinit(helpers.zigimg_test_allocator); +@@ -211,10 +211,10 @@ test "read of rgb pam with maxval 255" { + + test "basic read-write-read produces same result" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_grayscale_alpha_maxval_255.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + defer image.deinit(helpers.zigimg_test_allocator); +@@ -239,7 +239,7 @@ test "basic read-write-read produces same result" { + + var decoded_image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_back_stream); + defer decoded_image.deinit(helpers.zigimg_test_allocator); +- ++ + try std.testing.expectEqual(@as(usize, 4), decoded_image.height); + try std.testing.expectEqual(@as(usize, 4), decoded_image.width); + try helpers.expectEqSlice(zigimg.color.Grayscale8Alpha, decoded_image.pixels.grayscale8Alpha, &[16]zigimg.color.Grayscale8Alpha{ +@@ -254,10 +254,10 @@ test "basic read-write-read produces same result" { + + test "reads rgba pam with maxval 255" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_rgba_maxval_255.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + defer image.deinit(helpers.zigimg_test_allocator); +@@ -273,10 +273,10 @@ test "reads rgba pam with maxval 255" { + + test "reads rgba pam with maxval 65535" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_rgba_maxval_65535.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + defer image.deinit(helpers.zigimg_test_allocator); +diff --git a/tests/formats/pcx_test.zig b/tests/formats/pcx_test.zig +index db5d556..ee51851 100644 +--- a/tests/formats/pcx_test.zig ++++ b/tests/formats/pcx_test.zig +@@ -7,10 +7,10 @@ const toU8 = zigimg.color.ScaleValue(u8); + + test "PCX indexed1 (linear)" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp1.pcx"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var pcx_file = pcx.PCX{}; + +@@ -45,10 +45,10 @@ test "PCX indexed1 (linear)" { + + test "PCX indexed4 (linear)" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp4.pcx"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var pcx_file = pcx.PCX{}; + +@@ -84,10 +84,10 @@ test "PCX indexed4 (linear)" { + + test "PCX indexed8 (linear)" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp8.pcx"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var pcx_file = pcx.PCX{}; + +@@ -125,10 +125,10 @@ test "PCX indexed8 (linear)" { + + test "PCX indexed24 (planar)" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp24.pcx"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var pcx_file = pcx.PCX{}; + +@@ -169,25 +169,25 @@ test "Write PCX indexed1 (odd width)" { + const image_file_name = "zigimg_pcx_indexed1_odd.pcx"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp1.pcx"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .pcx = .{}, + }); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var pcx_file = pcx.PCX{}; + +@@ -242,18 +242,18 @@ test "Write PCX indexed 1 (even width)" { + const image_file_name = "zigimg_pcx_indexed1_even.pcx"; + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try image_pattern.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try image_pattern.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .pcx = .{}, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var pcx_file = pcx.PCX{}; + +@@ -285,25 +285,25 @@ test "Write PCX indexed4 (odd width)" { + const image_file_name = "zigimg_pcx_indexed4_odd.pcx"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp4.pcx"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .pcx = .{}, + }); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var pcx_file = pcx.PCX{}; + +@@ -359,18 +359,18 @@ test "Write PCX indexed 4 (even width)" { + const image_file_name = "zigimg_pcx_indexed4_even.pcx"; + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try rainbow_test.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try rainbow_test.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .pcx = .{}, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var pcx_file = pcx.PCX{}; + +@@ -400,25 +400,25 @@ test "Write PCX indexed8 (odd width)" { + const image_file_name = "zigimg_pcx_indexed8_odd.pcx"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp8.pcx"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .pcx = .{}, + }); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var pcx_file = pcx.PCX{}; + +@@ -472,18 +472,18 @@ test "Write PCX indexed 8 (even width)" { + const image_file_name = "zigimg_pcx_indexed8_even.pcx"; + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try rainbow_test.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try rainbow_test.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .pcx = .{}, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var pcx_file = pcx.PCX{}; + +@@ -513,25 +513,25 @@ test "Write PCX rgb24 (odd width)" { + const image_file_name = "zigimg_pcx_bpp24_odd.pcx"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp24.pcx"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .pcx = .{}, + }); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var pcx_file = pcx.PCX{}; + +@@ -594,18 +594,18 @@ test "Write PCX rgb24 (even width)" { + const image_file_name = "zigimg_pcx_rgb24_even.pcx"; + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try rainbow_test.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try rainbow_test.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .pcx = .{}, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var pcx_file = pcx.PCX{}; + +diff --git a/tests/formats/png_test.zig b/tests/formats/png_test.zig +index ec39855..def25c7 100644 +--- a/tests/formats/png_test.zig ++++ b/tests/formats/png_test.zig +@@ -8,10 +8,10 @@ const valid_header_data = png.magic_header ++ "\x00\x00\x00\x0d" ++ png.Chunks.I + + test "png: Should error on non PNG images" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const invalidFile = png.PNG.readImage(helpers.zigimg_test_allocator, &read_stream); + +@@ -111,10 +111,10 @@ test "png: Indexed PNG with transparency (Aseprite output)" { + // mlarouche: While the full test suite already test this image, I like having a smaller test that I can verify + // some specific info myself + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "png/aseprite_indexed_transparent.png"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var png_image = try png.PNG.readImage(helpers.zigimg_test_allocator, &read_stream); + defer png_image.deinit(helpers.zigimg_test_allocator); +@@ -180,18 +180,18 @@ test "png: Don't write tRNS chunk in indexed format when there is no alpha" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_indexed_no_trns.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .png = .{}, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var check_trns_processor: CheckTrnsPresentProcessor = .{}; + +@@ -220,18 +220,18 @@ test "png: Write tRNS chunk in indexed format only when alpha is present" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_indexed_trns.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .png = .{}, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var check_trns_processor: CheckTrnsPresentProcessor = .{}; + +@@ -268,16 +268,16 @@ test "png: Write indexed1 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_indexed1.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -322,16 +322,16 @@ test "png: Write indexed2 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_indexed2.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -376,16 +376,16 @@ test "png: Write indexed4 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_indexed4.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -430,16 +430,16 @@ test "png: Write indexed8 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_indexed8.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -475,16 +475,16 @@ test "png: Write grayscale1 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_grayscale1.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -513,16 +513,16 @@ test "png: Write grayscale2 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_grayscale2.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -551,16 +551,16 @@ test "png: Write grayscale4 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_grayscale4.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -589,16 +589,16 @@ test "png: Write grayscale8 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_grayscale8.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -628,16 +628,16 @@ test "png: Write grayscale8Alpha format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_grayscale8Alpha.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -667,16 +667,16 @@ test "png: Write grayscale16 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_grayscale16.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -706,16 +706,16 @@ test "png: Write grayscale16Alpha format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_grayscale16Alpha.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -747,16 +747,16 @@ test "png: Write rgb24 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_rgb24.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -790,16 +790,16 @@ test "png: Write rgba32 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_rgba32.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -833,16 +833,16 @@ test "png: Write rgb48 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_rgb48.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -876,16 +876,16 @@ test "png: Write rgba64 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_rgba64.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -910,32 +910,32 @@ test "png: Official Test Suite" { + + // Useful to quickly test everything on full dir of images + pub fn testWithDir(directory: []const u8, test_md5_signature: bool) !void { +- var test_dir_opt = std.fs.cwd().openDir(directory, .{ .access_sub_paths = false, .no_follow = true, .iterate = true }) catch null; ++ var test_dir_opt = std.Io.Dir.openDir(std.Io.Dir.cwd(), std.testing.io, directory, .{ .access_sub_paths = false, .follow_symlinks = false, .iterate = true }) catch null; + if (test_dir_opt) |*test_dir| { +- defer test_dir.close(); ++ defer test_dir.close(std.testing.io); + var it = test_dir.iterate(); + if (test_md5_signature) { +- std.debug.print("\n", .{}); ++ std.log.info("\n", .{}); + } +- while (try it.next()) |entry| { ++ while (try it.next(std.testing.io)) |entry| { + if (entry.kind != .file or !std.mem.eql(u8, std.fs.path.extension(entry.name), ".png")) { + continue; + } + + if (test_md5_signature) { +- std.debug.print("Testing file {s} ... ", .{entry.name}); ++ std.log.info("Testing file {s} ... ", .{entry.name}); + } + +- var test_file = try test_dir.openFile(entry.name, .{ .mode = .read_only }); +- defer test_file.close(); ++ var test_file = try test_dir.openFile(std.testing.io, entry.name, .{ .mode = .read_only }); ++ defer test_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(test_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, test_file, read_buffer[0..]); + + if (entry.name[0] == 'x' and entry.name[2] != 't' and entry.name[2] != 's') { + try std.testing.expectError(zigimg.Image.ReadError.InvalidData, png.loadHeader(&read_stream)); + if (test_md5_signature) { +- std.debug.print("OK\n", .{}); ++ std.log.info("OK\n", .{}); + } + continue; + } +@@ -946,7 +946,7 @@ pub fn testWithDir(directory: []const u8, test_md5_signature: bool) !void { + const error_result = png.loadWithHeader(&read_stream, &header, std.testing.allocator, default_options.get()); + try std.testing.expectError(zigimg.Image.ReadError.InvalidData, error_result); + if (test_md5_signature) { +- std.debug.print("OK\n", .{}); ++ std.log.info("OK\n", .{}); + } + continue; + } +@@ -968,11 +968,11 @@ pub fn testWithDir(directory: []const u8, test_md5_signature: bool) !void { + @memcpy(test_data_name[len - 3 .. len], "tsd"); + + // Read test data and check with it +- if (test_dir.openFile(test_data_name[0..len], .{ .mode = .read_only })) |test_data_file| { +- defer test_data_file.close(); ++ if (test_dir.openFile(std.testing.io, test_data_name[0..len], .{ .mode = .read_only })) |test_data_file| { ++ defer test_data_file.close(std.testing.io); + + var test_read_buffer: [512]u8 = undefined; +- var test_file_reader = test_data_file.reader(test_read_buffer[0..]); ++ var test_file_reader = test_data_file.reader(std.testing.io, test_read_buffer[0..]); + var test_reader = &test_file_reader.interface; + + var expected_md5: [16]u8 = undefined; +@@ -983,31 +983,31 @@ pub fn testWithDir(directory: []const u8, test_md5_signature: bool) !void { + const str_md5 = try test_reader.takeDelimiterExclusive('\n'); + _ = try std.fmt.hexToBytes(expected_md5[0..], str_md5); + try std.testing.expectEqual(expected_pixel_format, std.meta.activeTag(result)); +- try std.testing.expectEqualSlices(u8, expected_md5[0..], md5_val[0..]); // catch std.debug.print("MD5 Expected: {s} Got {s}\n", .{std.fmt.fmtSliceHexUpper(expected_md5[0..]), std.fmt.fmtSliceHexUpper(md5_val[0..])}); ++ try std.testing.expectEqualSlices(u8, expected_md5[0..], md5_val[0..]); // catch std.log.info("MD5 Expected: {s} Got {s}\n", .{std.fmt.fmtSliceHexUpper(expected_md5[0..]), std.fmt.fmtSliceHexUpper(md5_val[0..])}); + } else |_| { + // If there is no test data assume test is correct and write it out + try writeTestData(test_dir, test_data_name[0..len], &result, md5_val[0..]); + } + + if (test_md5_signature) { +- std.debug.print("OK\n", .{}); ++ std.log.info("OK\n", .{}); + } + + // Write Raw bytes + // std.mem.copyForwards(u8, tst_data_name[len - 3 .. len + 1], "data"); + // var rawoutput = try idir.createFile(tst_data_name[0 .. len + 1], .{}); +- // defer rawoutput.close(); ++ // defer rawoutput.close(std.testing.io); + // try rawoutput.writeAll(result_bytes); + } + } + } + +-fn writeTestData(dir: *std.fs.Dir, test_data_name: []const u8, result: *zigimg.color.PixelStorage, md5_value: []const u8) !void { +- var test_output = try dir.createFile(test_data_name, .{}); +- defer test_output.close(); ++fn writeTestData(dir: *std.Io.Dir, test_data_name: []const u8, result: *zigimg.color.PixelStorage, md5_value: []const u8) !void { ++ var test_output = try dir.createFile(std.testing.io, test_data_name, .{}); ++ defer test_output.close(std.testing.io); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var file_writer = test_output.writer(write_buffer[0..]); ++ var file_writer = test_output.writer(std.testing.io, write_buffer[0..]); + var writer = &file_writer.interface; + + try writer.print("{s}\n{X}", .{ @tagName(result.*), md5_value }); +@@ -1018,14 +1018,14 @@ fn writeTestData(dir: *std.fs.Dir, test_data_name: []const u8, result: *zigimg.c + test "png: InfoProcessor on Png Test suite" { + const directory = helpers.fixtures_path ++ "png/"; + +- var test_dir_opt = std.fs.cwd().openDir(directory, .{ .access_sub_paths = false, .no_follow = true, .iterate = true }) catch null; ++ var test_dir_opt = std.Io.Dir.openDir(std.Io.Dir.cwd(), std.testing.io, directory, .{ .access_sub_paths = false, .iterate = true }) catch null; + if (test_dir_opt) |*test_dir| { +- defer test_dir.close(); ++ defer test_dir.close(std.testing.io); + var it = test_dir.iterate(); + + var info_buffer: [16384]u8 = undefined; + +- while (try it.next()) |entry| { ++ while (try it.next(std.testing.io)) |entry| { + if (entry.kind != .file or !std.mem.eql(u8, std.fs.path.extension(entry.name), ".png")) { + continue; + } +@@ -1033,11 +1033,11 @@ test "png: InfoProcessor on Png Test suite" { + var info_stream = zigimg.io.WriteStream.initMemory(info_buffer[0..]); + var options = png.InfoProcessor.PngInfoOptions.init(png.InfoProcessor.init(info_stream.writer())); + +- var test_file = try test_dir.openFile(entry.name, .{ .mode = .read_only }); +- defer test_file.close(); ++ var test_file = try test_dir.openFile(std.testing.io, entry.name, .{ .mode = .read_only }); ++ defer test_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(test_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, test_file, read_buffer[0..]); + if (entry.name[0] == 'x') { + continue; + } +@@ -1051,13 +1051,13 @@ test "png: InfoProcessor on Png Test suite" { + @memcpy(test_data_name[len - 4 .. len], "info"); + + // Read test data and check with it +- if (test_dir.openFile(test_data_name[0..len], .{ .mode = .read_only })) |test_data_file| { +- defer test_data_file.close(); ++ if (test_dir.openFile(std.testing.io, test_data_name[0..len], .{ .mode = .read_only })) |test_data_file| { ++ defer test_data_file.close(std.testing.io); + + var expected_data_buffer: [16384]u8 = undefined; + + var test_read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var file_reader = test_data_file.reader(test_read_buffer[0..]); ++ var file_reader = test_data_file.reader(std.testing.io, test_read_buffer[0..]); + var reader = &file_reader.interface; + + const loaded = try reader.readSliceShort(expected_data_buffer[0..]); +@@ -1065,11 +1065,11 @@ test "png: InfoProcessor on Png Test suite" { + try std.testing.expectEqualSlices(u8, expected_data_buffer[0..loaded], info_buffer[0..loaded]); + } else |_| { + // If there is no test data assume test is correct and write it out +- var test_output_file = try test_dir.createFile(test_data_name[0..len], .{}); +- defer test_output_file.close(); ++ var test_output_file = try test_dir.createFile(std.testing.io, test_data_name[0..len], .{}); ++ defer test_output_file.close(std.testing.io); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var file_writer = test_output_file.writer(write_buffer[0..]); ++ var file_writer = test_output_file.writer(std.testing.io, write_buffer[0..]); + var writer = &file_writer.interface; + + try writer.writeAll(info_buffer[0..@intCast(info_stream.getPos())]); +diff --git a/tests/formats/qoi_test.zig b/tests/formats/qoi_test.zig +index 96256e3..5a9f985 100644 +--- a/tests/formats/qoi_test.zig ++++ b/tests/formats/qoi_test.zig +@@ -8,10 +8,10 @@ const zero_qoi_file = helpers.fixtures_path ++ "qoi/zero.qoi"; + + test "Should error on non QOI images" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var qoi_file = qoi.QOI{}; + +@@ -22,10 +22,10 @@ test "Should error on non QOI images" { + + test "Read zero.qoi file" { + const file = try helpers.testOpenFile(zero_qoi_file); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var qoi_file = qoi.QOI{}; + +diff --git a/tests/formats/ras_test.zig b/tests/formats/ras_test.zig +index d7240a1..5452098 100644 +--- a/tests/formats/ras_test.zig ++++ b/tests/formats/ras_test.zig +@@ -5,10 +5,10 @@ const helpers = @import("../helpers.zig"); + + test "Should error on non RAS images" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var ras_file = ras.RAS{}; + +@@ -18,12 +18,12 @@ test "Should error on non RAS images" { + + test "Sun-Raster 24-bit RGB24 uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-rgb24.ras"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var ras_file = ras.RAS{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -46,12 +46,12 @@ test "Sun-Raster 24-bit RGB24 uncompressed" { + + test "Sun-Raster 24-bit BGR24 uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-bgr24.ras"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var ras_file = ras.RAS{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -74,12 +74,12 @@ test "Sun-Raster 24-bit BGR24 uncompressed" { + + test "Sun-Raster 32-bit xRGB uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-xrgb32.ras"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var ras_file = ras.RAS{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -98,12 +98,12 @@ test "Sun-Raster 32-bit xRGB uncompressed" { + + test "Sun-Raster 8-bit with palette uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-8bit.ras"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var ras_file = ras.RAS{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -124,12 +124,12 @@ test "Sun-Raster 8-bit with palette uncompressed" { + + test "Sun-Raster 8-bit grayscale uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-8bit-grayscale.ras"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var ras_file = ras.RAS{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -144,12 +144,12 @@ test "Sun-Raster 8-bit grayscale uncompressed" { + + test "Sun-Raster 1-bit black & white uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-blackwhite.ras"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var ras_file = ras.RAS{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -164,12 +164,12 @@ test "Sun-Raster 1-bit black & white uncompressed" { + + test "Sun-Raster bgr24 rle compressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-24bit-bgr-rle.ras"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var ras_file = ras.RAS{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +diff --git a/tests/formats/sgi_test.zig b/tests/formats/sgi_test.zig +index d098ef7..ed23769 100644 +--- a/tests/formats/sgi_test.zig ++++ b/tests/formats/sgi_test.zig +@@ -5,10 +5,10 @@ const zigimg = @import("zigimg"); + + test "Should error on non SGI images" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var sgi_file = sgi.SGI{}; + +@@ -18,12 +18,12 @@ test "Should error on non SGI images" { + + test "SGI 24-bit uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-rgb24.sgi"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var sgi_file = sgi.SGI{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -46,12 +46,12 @@ test "SGI 24-bit uncompressed" { + + test "SGI grayscale uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-blackwhite.sgi"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var sgi_file = sgi.SGI{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -66,12 +66,12 @@ test "SGI grayscale uncompressed" { + + test "SGI 32-bit RGBA uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-rgba.sgi"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var sgi_file = sgi.SGI{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -94,12 +94,12 @@ test "SGI 32-bit RGBA uncompressed" { + + test "SGI RGB48be uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-rgb48be.sgi"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var sgi_file = sgi.SGI{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -122,12 +122,12 @@ test "SGI RGB48be uncompressed" { + + test "SGI grayscale rle compressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-gray-rle.sgi"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var sgi_file = sgi.SGI{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -142,12 +142,12 @@ test "SGI grayscale rle compressed" { + + test "SGI 24-bit rle compressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-24bit-rle.sgi"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var sgi_file = sgi.SGI{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -170,12 +170,12 @@ test "SGI 24-bit rle compressed" { + + test "SGI RGB48be rle uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-rgb48be-rle.sgi"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var sgi_file = sgi.SGI{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -198,12 +198,12 @@ test "SGI RGB48be rle uncompressed" { + + test "SGI 32-bit RGBA rle compressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-rgba-rle.sgi"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var sgi_file = sgi.SGI{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +diff --git a/tests/formats/tga_test.zig b/tests/formats/tga_test.zig +index 83f2d76..c3182d4 100644 +--- a/tests/formats/tga_test.zig ++++ b/tests/formats/tga_test.zig +@@ -5,10 +5,10 @@ const zigimg = @import("zigimg"); + + test "Should error on non TGA images" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -19,10 +19,10 @@ test "Should error on non TGA images" { + + test "Read ubw8 TGA file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ubw8.tga"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -56,10 +56,10 @@ test "Read ubw8 TGA file" { + + test "Read ucm8 TGA file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ucm8.tga"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -101,10 +101,10 @@ test "Read ucm8 TGA file" { + + test "Read utc16 TGA file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc16.tga"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -140,10 +140,10 @@ test "Read utc16 TGA file" { + + test "Read utc24 TGA file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc24.tga"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -179,10 +179,10 @@ test "Read utc24 TGA file" { + + test "Read utc32 TGA file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc32.tga"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -218,10 +218,10 @@ test "Read utc32 TGA file" { + + test "Read cbw8 TGA file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/cbw8.tga"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -255,10 +255,10 @@ test "Read cbw8 TGA file" { + + test "Read ccm8 TGA file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ccm8.tga"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -300,10 +300,10 @@ test "Read ccm8 TGA file" { + + test "Read ctc24 TGA file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ctc24.tga"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -339,10 +339,10 @@ test "Read ctc24 TGA file" { + + test "Read matte-01 TGA file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/matte-01.tga"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -384,10 +384,10 @@ test "Read matte-01 TGA file" { + + test "Read font TGA file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/font.tga"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -411,10 +411,10 @@ test "Read font TGA file" { + + test "Read stopsignsmall TGA v1 file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/stopsignsmall.tga"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -436,10 +436,10 @@ test "Read stopsignsmall TGA v1 file" { + + test "Read stopsignsmallcompressed TGA v1 file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/stopsignsmallcompressed.tga"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -463,11 +463,11 @@ test "Write TGA uncompressed grayscale8" { + const image_file_name = "zigimg_tga_uncompressed_grayscale8.tga"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ubw8.tga"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + const encoder_options = zigimg.Image.EncoderOptions{ +@@ -500,15 +500,15 @@ test "Write TGA uncompressed grayscale8" { + }; + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], encoder_options); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], encoder_options); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -558,15 +558,15 @@ test "Write TGA compressed grayscale8" { + const image_file_name = "zigimg_tga_compressed_grayscale8.tga"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/cbw8.tga"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = true, + .color_map_depth = 16, +@@ -575,13 +575,13 @@ test "Write TGA compressed grayscale8" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -617,15 +617,15 @@ test "Write uncompressed indexed8 (color map 16-bit)" { + const image_file_name = "zigimg_tga_uncompressed_indexed8.tga"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ucm8.tga"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = false, + .color_map_depth = 16, +@@ -634,13 +634,13 @@ test "Write uncompressed indexed8 (color map 16-bit)" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -684,15 +684,15 @@ test "Write uncompressed indexed8 (color map 24-bit)" { + const image_file_name = "zigimg_tga_uncompressed_indexed8.tga"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ucm8.tga"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = false, + .color_map_depth = 24, +@@ -701,13 +701,13 @@ test "Write uncompressed indexed8 (color map 24-bit)" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -751,15 +751,15 @@ test "Write compressed indexed8 (color map 16-bit)" { + const image_file_name = "zigimg_tga_compressed_indexed8.tga"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ccm8.tga"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = true, + .color_map_depth = 16, +@@ -768,13 +768,13 @@ test "Write compressed indexed8 (color map 16-bit)" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -818,15 +818,15 @@ test "Write compressed indexed8 (color map 24-bit)" { + const image_file_name = "zigimg_tga_compressed_indexed8.tga"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ccm8.tga"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = true, + .color_map_depth = 24, +@@ -835,13 +835,13 @@ test "Write compressed indexed8 (color map 24-bit)" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -885,15 +885,15 @@ test "Write uncompressed 16-bit true color TGA" { + const image_file_name = "zigimg_tga_uncompressed_true_color_16.tga"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc16.tga"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = false, + .color_map_depth = 16, +@@ -902,13 +902,13 @@ test "Write uncompressed 16-bit true color TGA" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -946,15 +946,15 @@ test "Write compressed 16-bit true color TGA" { + const image_file_name = "zigimg_tga_compressed_true_color_16.tga"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc16.tga"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = true, + .color_map_depth = 16, +@@ -963,13 +963,13 @@ test "Write compressed 16-bit true color TGA" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -1007,15 +1007,15 @@ test "Write uncompressed 24-bit true color TGA" { + const image_file_name = "zigimg_tga_uncompressed_true_color_24.tga"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc24.tga"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = false, + .color_map_depth = 24, +@@ -1024,13 +1024,13 @@ test "Write uncompressed 24-bit true color TGA" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -1068,15 +1068,15 @@ test "Write compressed 24-bit true color TGA" { + const image_file_name = "zigimg_tga_compressed_true_color_24.tga"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ctc24.tga"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = true, + .color_map_depth = 24, +@@ -1085,13 +1085,13 @@ test "Write compressed 24-bit true color TGA" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -1129,15 +1129,15 @@ test "Write uncompressed 32-bit true color TGA" { + const image_file_name = "zigimg_tga_uncompressed_true_color_32.tga"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc32.tga"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = false, + .color_map_depth = 24, +@@ -1146,13 +1146,13 @@ test "Write uncompressed 32-bit true color TGA" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -1190,15 +1190,15 @@ test "Write compressed 32-bit true color TGA" { + const image_file_name = "zigimg_tga_compressed_true_color_32.tga"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ctc32.tga"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = true, + .color_map_depth = 24, +@@ -1207,13 +1207,13 @@ test "Write compressed 32-bit true color TGA" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -1270,7 +1270,7 @@ test "Write uncompressed Rgb24 to TGA" { + @memcpy(source_image.pixels.rgb24[0..], uncompressed_source[0..]); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = false, + .color_map_depth = 24, +@@ -1279,14 +1279,14 @@ test "Write uncompressed Rgb24 to TGA" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -1329,7 +1329,7 @@ test "Write compressed Rgb24 to TGA" { + @memcpy(source_image.pixels.rgb24[0..], uncompressed_source[0..]); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = true, + .color_map_depth = 24, +@@ -1338,14 +1338,14 @@ test "Write compressed Rgb24 to TGA" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -1403,7 +1403,7 @@ test "Write uncompressed Rgba32 to TGA" { + @memcpy(source_image.pixels.rgba32[0..], uncompressed_source[0..]); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = false, + .color_map_depth = 24, +@@ -1412,14 +1412,14 @@ test "Write uncompressed Rgba32 to TGA" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -1478,7 +1478,7 @@ test "Write compressed Rgba32 to TGA" { + @memcpy(source_image.pixels.rgba32[0..], uncompressed_source[0..]); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = true, + .color_map_depth = 24, +@@ -1487,14 +1487,14 @@ test "Write compressed Rgba32 to TGA" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +diff --git a/tests/formats/tiff_test.zig b/tests/formats/tiff_test.zig +index cce4cfa..2a8a042 100644 +--- a/tests/formats/tiff_test.zig ++++ b/tests/formats/tiff_test.zig +@@ -5,10 +5,10 @@ const zigimg = @import("zigimg"); + + test "Should error on non TIFF images" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var sgi_file = tiff.TIFF{}; + +@@ -18,12 +18,12 @@ test "Should error on non TIFF images" { + + test "TIFF/LE monochrome black uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-monob-raw.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -39,12 +39,12 @@ test "TIFF/LE monochrome black uncompressed" { + + test "TIFF/LE grayscale8 uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-grayscale8-raw.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -61,12 +61,12 @@ test "TIFF/LE grayscale8 uncompressed" { + + test "TIFF/LE 8-bit with colormap uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-pal8-raw.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -87,12 +87,12 @@ test "TIFF/LE 8-bit with colormap uncompressed" { + + test "TIFF/LE 24-bit uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgb24-raw.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -115,12 +115,12 @@ test "TIFF/LE 24-bit uncompressed" { + + test "TIFF/BE rgb24 gray single strip uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/big-endian/sample-rgb24-single-strip.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -143,12 +143,12 @@ test "TIFF/BE rgb24 gray single strip uncompressed" { + + test "TIFF/BE rgb24 color single strip uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/big-endian/sample-pal8-raw.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -171,12 +171,12 @@ test "TIFF/BE rgb24 color single strip uncompressed" { + + test "TIFF/BE 24-bit uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/big-endian/sample-rgb24-raw.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -199,12 +199,12 @@ test "TIFF/BE 24-bit uncompressed" { + + test "TIFF/LE RGBA uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgba-raw.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -223,12 +223,12 @@ test "TIFF/LE RGBA uncompressed" { + + test "TIFF/LE monochrome black packbits" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-monob-packbits.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -244,12 +244,12 @@ test "TIFF/LE monochrome black packbits" { + + test "TIFF/LE grayscale8 packbits" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-grayscale8-packbits.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -266,12 +266,12 @@ test "TIFF/LE grayscale8 packbits" { + + test "TIFF/LE 8-bit with colormap packbits" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-pal8-packbits.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -292,12 +292,12 @@ test "TIFF/LE 8-bit with colormap packbits" { + + test "TIFF/LE 24-bit packbits" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgb24-packbits.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -320,12 +320,12 @@ test "TIFF/LE 24-bit packbits" { + + test "TIFF/LE RGBA packbits" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgba-packbits.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -344,12 +344,12 @@ test "TIFF/LE RGBA packbits" { + + test "TIFF/LE monochrome black CCITT" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/ccitt_rle.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -364,12 +364,12 @@ test "TIFF/LE monochrome black CCITT" { + + test "TIFF/LE monochrome black LZW" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-monob-lzw.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -385,12 +385,12 @@ test "TIFF/LE monochrome black LZW" { + + test "TIFF/LE grayscale8 LZW" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-grayscale8-lzw.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -407,12 +407,12 @@ test "TIFF/LE grayscale8 LZW" { + + test "TIFF/LE 8-bit with colormap LZW" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-pal8-lzw.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -433,12 +433,12 @@ test "TIFF/LE 8-bit with colormap LZW" { + + test "TIFF/LE 24-bit LZW" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgb24-lzw.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -461,12 +461,12 @@ test "TIFF/LE 24-bit LZW" { + + test "TIFF/LE RGBA LZW" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgba-lzw.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -485,12 +485,12 @@ test "TIFF/LE RGBA LZW" { + + test "TIFF/LE monochrome black Deflate" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-monob-deflate.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_bitmap = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_bitmap.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -506,12 +506,12 @@ test "TIFF/LE monochrome black Deflate" { + + test "TIFF/LE grayscale8 Deflate" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-grayscale8-deflate.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -528,12 +528,12 @@ test "TIFF/LE grayscale8 Deflate" { + + test "TIFF/LE 8-bit with colormap Deflate" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-pal8-deflate.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -554,12 +554,12 @@ test "TIFF/LE 8-bit with colormap Deflate" { + + test "TIFF/LE 24-bit Deflate" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgb24-deflate.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -582,12 +582,12 @@ test "TIFF/LE 24-bit Deflate" { + + test "TIFF/LE RGBA Deflate" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgba-deflate.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +diff --git a/tests/formats/xbm_test.zig b/tests/formats/xbm_test.zig +index 52eb29c..053e41e 100644 +--- a/tests/formats/xbm_test.zig ++++ b/tests/formats/xbm_test.zig +@@ -30,12 +30,12 @@ const simple_4x4_xbm = + test "XBM: invalid file format" { + { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "xbm/bad_missing_dim.xbm"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_xbm = xbm.XBM{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const actual_error = the_xbm.read(helpers.zigimg_test_allocator, &read_stream); + try helpers.expectError(actual_error, zigimg.Image.ReadError.InvalidData); +@@ -43,12 +43,12 @@ test "XBM: invalid file format" { + + { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "xbm/bad_missing_pixels.xbm"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_xbm = xbm.XBM{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const actual_error = the_xbm.read(helpers.zigimg_test_allocator, &read_stream); + try helpers.expectError(actual_error, zigimg.Image.ReadError.InvalidData); +diff --git a/tests/helpers.zig b/tests/helpers.zig +index 9a4ef29..b9f06b1 100644 +--- a/tests/helpers.zig ++++ b/tests/helpers.zig +@@ -30,17 +30,17 @@ pub inline fn expectApproxEqRel(actual: anytype, expected: anytype, tolerance: a + return try std.testing.expectApproxEqRel(expected, actual, tolerance); + } + +-pub fn testOpenFile(file_path: []const u8) !std.fs.File { +- return std.fs.cwd().openFile(file_path, .{}) catch |err| ++pub fn testOpenFile(file_path: []const u8) !std.Io.File { ++ return std.Io.Dir.openFile(std.Io.Dir.cwd(), std.testing.io, file_path, .{}) catch |err| + if (err == error.FileNotFound) return error.SkipZigTest else return err; + } + + pub fn testImageFromFile(image_path: []const u8, buffer: []u8) !zigimg.Image { +- return zigimg.Image.fromFilePath(zigimg_test_allocator, image_path, buffer) catch |err| ++ return zigimg.Image.fromFilePath(zigimg_test_allocator, std.testing.io, image_path, buffer) catch |err| + if (err == error.FileNotFound) return error.SkipZigTest else return err; + } + + pub fn testReadFile(file_path: []const u8, buffer: []u8) ![]u8 { +- return std.fs.cwd().readFile(file_path, buffer) catch |err| ++ return std.Io.Dir.readFile(std.Io.Dir.cwd(), std.testing.io, file_path, buffer) catch |err| + if (err == error.FileNotFound) return error.SkipZigTest else return err; + } +diff --git a/tests/image_test.zig b/tests/image_test.zig +index ccd2d11..529be41 100644 +--- a/tests/image_test.zig ++++ b/tests/image_test.zig +@@ -187,7 +187,7 @@ test "Should detect BMP properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .bmp); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -213,7 +213,7 @@ test "Should detect GIF properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .gif); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -231,7 +231,7 @@ test "Should detect PCX properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .pcx); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -247,7 +247,7 @@ test "Should detect PBM properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .pbm); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -265,7 +265,7 @@ test "Should detect PGM properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .pgm); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -281,7 +281,7 @@ test "Should detect PPM properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .ppm); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -297,7 +297,7 @@ test "Should detect PNG properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .png); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -321,7 +321,7 @@ test "Should detect TGA properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .tga); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -334,7 +334,7 @@ test "Should detect QOI properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .qoi); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -350,7 +350,7 @@ test "Should detect JPEG properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .jpeg); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -366,7 +366,7 @@ test "Should detect Farbfeld properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .farbfeld); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -381,7 +381,7 @@ test "Should detect IFF/PBM properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .iff); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -396,7 +396,7 @@ test "Should detect RAS properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .ras); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -411,7 +411,7 @@ test "Should detect SGI properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .sgi); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -426,7 +426,7 @@ test "Should detect TIFF properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .tiff); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -449,7 +449,7 @@ test "Should detect XBM properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .xbm); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +diff --git a/tests/io_test.zig b/tests/io_test.zig +index 2dc6f7a..1ec5d60 100644 +--- a/tests/io_test.zig ++++ b/tests/io_test.zig +@@ -55,12 +55,12 @@ test "io.ReadStream: should read and seek properly within a unbuffered file" { + var temp_folder = std.testing.tmpDir(.{}); + defer temp_folder.cleanup(); + +- try temp_folder.dir.writeFile(.{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); ++ try temp_folder.dir.writeFile(std.testing.io, .{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); + +- const read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); +- defer read_file.close(); ++ const read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, &.{}); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, &.{}); + + var reader = read_stream.reader(); + +@@ -99,12 +99,12 @@ test "io.ReadStream: should error on invalid seek within unbuffered file" { + var temp_folder = std.testing.tmpDir(.{}); + defer temp_folder.cleanup(); + +- try temp_folder.dir.writeFile(.{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); ++ try temp_folder.dir.writeFile(std.testing.io, .{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); + +- const read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); +- defer read_file.close(); ++ const read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, &.{}); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, &.{}); + + const seek_to_error = read_stream.seekTo(123456); + try helpers.expectError(seek_to_error, zigimg.io.ReadStream.SeekError.Unseekable); +@@ -122,13 +122,13 @@ test "io.ReadStream: should read and seek properly within a file with a small bu + var temp_folder = std.testing.tmpDir(.{}); + defer temp_folder.cleanup(); + +- try temp_folder.dir.writeFile(.{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); ++ try temp_folder.dir.writeFile(std.testing.io, .{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); + +- const read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); +- defer read_file.close(); ++ const read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); ++ defer read_file.close(std.testing.io); + + var small_buffer: [30]u8 = @splat(0); +- var read_stream = zigimg.io.ReadStream.initFile(read_file, small_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, small_buffer[0..]); + + var reader = read_stream.reader(); + +@@ -164,13 +164,13 @@ test "io.ReadStream: should read and seek properly within a file with a large bu + var temp_folder = std.testing.tmpDir(.{}); + defer temp_folder.cleanup(); + +- try temp_folder.dir.writeFile(.{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); ++ try temp_folder.dir.writeFile(std.testing.io, .{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); + +- const read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); +- defer read_file.close(); ++ const read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var reader = read_stream.reader(); + +@@ -206,13 +206,13 @@ test "io.ReadStream: should error on invalid seek within a file with a large buf + var temp_folder = std.testing.tmpDir(.{}); + defer temp_folder.cleanup(); + +- try temp_folder.dir.writeFile(.{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); ++ try temp_folder.dir.writeFile(std.testing.io, .{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); + +- const read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); +- defer read_file.close(); ++ const read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + const seek_to_error = read_stream.seekTo(123456); + try helpers.expectError(seek_to_error, zigimg.io.ReadStream.SeekError.Unseekable); +@@ -261,10 +261,10 @@ test "io.WriteStream: should write property within a file with a small buffer" { + defer temp_folder.cleanup(); + + { +- const write_file = try temp_folder.dir.createFile(TEST_FILENAME, .{}); +- defer write_file.close(); ++ const write_file = try temp_folder.dir.createFile(std.testing.io, TEST_FILENAME, .{}); ++ defer write_file.close(std.testing.io); + +- var write_stream = zigimg.io.WriteStream.initFile(write_file, buffer[0..]); ++ var write_stream = zigimg.io.WriteStream.initFile(std.testing.io, write_file, buffer[0..]); + + var writer = write_stream.writer(); + try writer.writeAll(TEST_FILE_CONTENTS[0..3]); +@@ -283,11 +283,11 @@ test "io.WriteStream: should write property within a file with a small buffer" { + try writer.flush(); + } + +- var read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); +- defer read_file.close(); ++ var read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); ++ defer read_file.close(std.testing.io); + + var read_buffer: [1024]u8 = @splat(0); +- var file_reader = read_file.reader(read_buffer[0..]); ++ var file_reader = read_file.reader(std.testing.io, read_buffer[0..]); + + var reader = &file_reader.interface; + +@@ -305,11 +305,11 @@ test "io.WriteStream: should write property within a file with a large buffer" { + defer temp_folder.cleanup(); + + { +- const write_file = try temp_folder.dir.createFile(TEST_FILENAME, .{}); +- defer write_file.close(); ++ const write_file = try temp_folder.dir.createFile(std.testing.io, TEST_FILENAME, .{}); ++ defer write_file.close(std.testing.io); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var write_stream = zigimg.io.WriteStream.initFile(write_file, write_buffer[0..]); ++ var write_stream = zigimg.io.WriteStream.initFile(std.testing.io, write_file, write_buffer[0..]); + + var writer = write_stream.writer(); + try writer.writeAll(TEST_FILE_CONTENTS[0..3]); +@@ -328,11 +328,11 @@ test "io.WriteStream: should write property within a file with a large buffer" { + try writer.flush(); + } + +- var read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); +- defer read_file.close(); ++ var read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); ++ defer read_file.close(std.testing.io); + + var read_buffer: [1024]u8 = @splat(0); +- var file_reader = read_file.reader(read_buffer[0..]); ++ var file_reader = read_file.reader(std.testing.io, read_buffer[0..]); + + var reader = &file_reader.interface; + +diff --git a/zigimg.zig b/zigimg.zig +index 9f6b1a8..a53b23c 100644 +--- a/zigimg.zig ++++ b/zigimg.zig +@@ -38,5 +38,5 @@ test { + @import("tests/octree_quantizer_test.zig"), + @import("tests/pixel_format_converter_test.zig"), + @import("tests/pixel_format_test.zig"), +- }) |source_file| std.testing.refAllDeclsRecursive(source_file); ++ }) |source_file| std.testing.refAllDecls(source_file); + } diff --git a/third_party/zigimg/repo.bzl b/third_party/zigimg/repo.bzl new file mode 100644 index 00000000..aa375cfc --- /dev/null +++ b/third_party/zigimg/repo.bzl @@ -0,0 +1,25 @@ +def _zigimg_repo_impl(ctx): + ctx.download_and_extract( + url = "https://github.com/zigimg/zigimg/archive/{}.tar.gz".format(ctx.attr.commit), + stripPrefix = "zigimg-{}".format(ctx.attr.commit), + ) + + ctx.patch(ctx.attr.patch, strip = 1) + ctx.symlink(ctx.attr.build_file, "BUILD.bazel") + +_zigimg_repo = repository_rule( + implementation = _zigimg_repo_impl, + attrs = { + "commit": attr.string(mandatory = True), + "patch": attr.label(mandatory = True, allow_single_file = True), + "build_file": attr.label(mandatory = True, allow_single_file = True), + }, +) + +def repo(): + _zigimg_repo( + name = "zigimg", + commit = "7b98e82621fe302a9edc147df1191f4d1b7ff7a5", + patch = "//third_party/zigimg:bump-zig.patch", + build_file = "//third_party/zigimg:zigimg.bazel", + ) diff --git a/third_party/zigimg/zigimg.bazel b/third_party/zigimg/zigimg.bazel new file mode 100644 index 00000000..f02d9380 --- /dev/null +++ b/third_party/zigimg/zigimg.bazel @@ -0,0 +1,18 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "zigimg", + srcs = glob([ + "src/*.zig", + "src/compressions/*.zig", + "src/compressions/deflate/*.zig", + "src/formats/*.zig", + "src/formats/jpeg/*.zig", + "src/formats/png/*.zig", + "src/formats/tiff/*.zig", + "src/Image/*.zig", + "tests/**/*.zig", + ]), + main = "zigimg.zig", + visibility = ["//visibility:public"], +) diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 40be0245..8a2bfdc3 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -1,4 +1,4 @@ -load("@com_google_protobuf//upb/bazel:upb_c_proto_library.bzl", "upb_c_proto_library") +load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_c_proto_library") load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") @@ -45,13 +45,13 @@ zig_library( "mlirx.zig", "module.zig", "nn.zig", + "sharding.zig", "ops.zig", "pjrtx.zig", "platform.zig", "profiler.zig", "safetensors.zig", "shape.zig", - "sharding.zig", "slice.zig", "tensor.zig", "testing.zig", @@ -66,11 +66,11 @@ zig_library( ], deps = [ ":posix", - ":profiler_options", ":xla_compile_options_upb", ":xla_data_upb", "//bazel", "//ffi:zig", + ":profiler_options", "//mlir", "//mlir/dialects", "//pjrt", @@ -89,6 +89,7 @@ zig_library( }), ) + zig_test( name = "test", deps = ["//zml"], From 65e9b4487e3fc118ecb9d1222c5cdd59b3b33f3e Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 31 Mar 2026 18:07:02 +0200 Subject: [PATCH 397/881] examples/llm: replicate Qwen 3.5 KV heads when TP exceeds KV head count (#444) Handle Qwen3.5 full-attention K/V heads like vLLM when num_key_value_heads < tp: replicate K/V heads across the model mesh instead of sharding the KV head axis. **Motivation** Qwen3.5 9B has 4 KV heads. On 8-way TP, sharding the KV head axis over `.model` is invalid because the dimension is not divisible by the tensor-parallel degree. This is a placement issue, not a model-shape issue. The model still has 4 logical KV heads, so the right fix is to replicate those heads across ranks rather than materialize fake extra heads. This also matches cost modeling better: replication represents duplicated placement of the same logical data, while repeating would incorrectly make the tensor look larger than the model actually defines. **Changes** - Add a policy helper to detect when KV heads should be replicated instead of model-sharded. - Apply that policy to projected K/V tensors in Qwen3.5 self-attention. - Apply the same policy to KV cache shape partitioning. - Thread model partition count through the Qwen3.5 inference compilation path so cache placement follows the same rule. **Why replicate instead of repeat** Replicating: - preserves the logical model shape - matches grouped-query attention semantics - keeps cost modeling honest by representing placement, not model expansion Repeating: - would change the graph from 4 KV heads to 8 KV heads - would increase KV cache size and bandwidth for no new information - would distort cost modeling by turning a placement choice into fake tensor growth **Result** Qwen3.5 now supports TP setups such as 8 GPUs where KV head count is smaller than the tensor-parallel degree. --- examples/llm/models/qwen3_5/inference.zig | 3 +- examples/llm/models/qwen3_5/model.zig | 46 ++++++++++++++++++----- zml/platform.zig | 16 ++++++++ 3 files changed, 55 insertions(+), 10 deletions(-) diff --git a/examples/llm/models/qwen3_5/inference.zig b/examples/llm/models/qwen3_5/inference.zig index 37e6d84b..650b57cb 100644 --- a/examples/llm/models/qwen3_5/inference.zig +++ b/examples/llm/models/qwen3_5/inference.zig @@ -23,11 +23,12 @@ pub const CompilationParameters = struct { pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, shardings: common.Shardings) CompilationParameters { const dtype = mdl.text_model.embed_tokens.weight.dtype(); + const model_partitions = shardings.model.numPartitionsForLogicalAxis(.model); return .{ .prefill_tokens = .init(.{ .b = 1, .s = seqlen }, .u32), .decode_tokens = .init(.{ .b = 1, .s = 1 }, .u32), .token_index = .init(.{}, .u32), - .kv_cache = .init(config, 1, seqlen, dtype, .f32), + .kv_cache = .init(config, 1, seqlen, dtype, .f32, model_partitions), .rng = .init(), .seqlen = seqlen, .shardings = shardings, diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 23d439df..551e972b 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -44,6 +44,17 @@ pub const RopeParameters = struct { rope_theta: f32, }; +fn kvHeadsAreReplicated(axis_size: i64, model_partitions: i64) bool { + return axis_size > 0 and axis_size < model_partitions and @rem(model_partitions, axis_size) == 0; +} + +fn partitionKvCacheShape(kv_shape: zml.Shape, kv_heads: i64, model_partitions: i64) zml.Shape { + return if (kvHeadsAreReplicated(kv_heads, model_partitions)) + kv_shape.withPartitioning(.{ .h = .replicated }) + else + kv_shape.withPartitioning(.{ .h = .model }); +} + pub const LoadedModel = struct { inner: Model, parsed_config: std.json.Parsed(Config), @@ -460,20 +471,36 @@ pub const SelfAttn = struct { return .{ k, v }; } + fn partitionProjectedKv(tensor: zml.Tensor, replicate_kv_heads: bool) zml.Tensor { + return if (replicate_kv_heads) + tensor.withPartitioning(.{ .s = .replicated, .h = .replicated, .hd = .replicated }) + else + tensor.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); + } + + fn partitionCachedKv(tensor: zml.Tensor, replicate_kv_heads: bool) zml.Tensor { + return if (replicate_kv_heads) + tensor.rename(.{ .s = .k }).withPartitioning(.{ .k = .replicated, .h = .replicated, .hd = .replicated }) + else + tensor.rename(.{ .s = .k }).withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); + } + pub fn forward( self: SelfAttn, x: zml.Tensor, token_index: zml.Tensor, kv_cache: KvCache.SelfAttnCache, ) struct { zml.Tensor, KvCache.SelfAttnCache } { + const model_partitions = zml.module.CompilationContext.current().partitioning.numPartitionsForLogicalAxis(self.q_proj.weight.shape(), .model) catch unreachable; + const replicate_kv_heads = kvHeadsAreReplicated(self.num_kv_heads, model_partitions); const x_qkv = x.withPartitioning(.{ .d = .replicated }); var q, var gate = self.projectQAndGate(x_qkv); var k, var v = self.projectKV(x_qkv); q = q.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); gate = gate.withPartitioning(.{ .s = .replicated, .d_out_proj = .model }); - k = k.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); - v = v.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); + k = partitionProjectedKv(k, replicate_kv_heads); + v = partitionProjectedKv(v, replicate_kv_heads); q = self.q_norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); k = self.k_norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); @@ -486,15 +513,15 @@ pub const SelfAttn = struct { q = self.rotary_embed.applyRope(q, cos, sin); k = self.rotary_embed.applyRope(k, cos, sin); q = q.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); - k = k.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); - v = v.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); + k = partitionProjectedKv(k, replicate_kv_heads); + v = partitionProjectedKv(v, replicate_kv_heads); const new_kv_cache = kv_cache.update(k, v, token_index.convert(.u32)); k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); q = q.rename(.{ .s = .q }).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }); - k = k.rename(.{ .s = .k }).withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); - v = v.rename(.{ .s = .k }).withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); + k = partitionCachedKv(k, replicate_kv_heads); + v = partitionCachedKv(v, replicate_kv_heads); const attn_output = zml.attention.attention.attention( q, @@ -833,7 +860,7 @@ pub const KvCache = struct { v: zml.Tensor, layer_index: zml.Tensor, - pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType) SelfAttnCache { + pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, model_partitions: i64) SelfAttnCache { const num_self_attn_layers = countLayers(config.text_config.layer_types, .full_attention); const kv_shape = zml.Shape.init(.{ .b = batch_dim, @@ -842,7 +869,7 @@ pub const KvCache = struct { .h = config.text_config.num_key_value_heads, .hd = config.text_config.head_dim, }, dtype); - const sharded_kv_shape = kv_shape.withPartitioning(.{ .h = .model }); + const sharded_kv_shape = partitionKvCacheShape(kv_shape, config.text_config.num_key_value_heads, model_partitions); return .{ .k = .fromShape(sharded_kv_shape), .v = .fromShape(sharded_kv_shape), @@ -1022,10 +1049,11 @@ pub const KvCache = struct { max_seq_len: i64, cache_dtype: zml.DataType, recurrent_dtype: zml.DataType, + model_partitions: i64, ) KvCache { return .{ .layer_types = config.text_config.layer_types, - .self_attn = SelfAttnCache.init(config, batch_dim, max_seq_len, cache_dtype), + .self_attn = SelfAttnCache.init(config, batch_dim, max_seq_len, cache_dtype, model_partitions), .gated_delta_net = GatedDeltaNetCache.init(config, batch_dim, cache_dtype, recurrent_dtype), }; } diff --git a/zml/platform.zig b/zml/platform.zig index 42e3d648..ad3bf7f6 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -41,6 +41,21 @@ fn disableXlaLogs() void { ); } +fn validateDeviceCount(target: Target, num_devices: usize) !void { + switch (target) { + .cpu, .cuda, .rocm, .tpu, .neuron => { + if (num_devices == 0) { + log.err("Platform {} requires at least 1 device, got {}", .{ target, num_devices }); + return error.ZeroVisibleDevices; + } + if (!std.math.isPowerOfTwo(num_devices)) { + log.err("Platform {} requires a power-of-two device count, got {}", .{ target, num_devices }); + return error.InvalidDeviceCount; + } + }, + } +} + fn loadOrGetApi(allocator: std.mem.Allocator, io: std.Io, target: Target) !*const pjrt.Api { return switch (target) { inline else => |tag| @field(api_map, @tagName(tag)) orelse b: { @@ -194,6 +209,7 @@ pub const Platform = struct { var named_values_buf: [16]pjrt.NamedValue = undefined; const pjrt_client = try pjrt.Client.init(api, options.toNamedValues(target, &named_values_buf)); const pjrt_devices = pjrt_client.addressableDevices(api); + try validateDeviceCount(target, pjrt_devices.len); if (pjrt_devices.len > MAX_NUM_DEVICES) { log.warn("platform {} got {} devices, but ZML only support up to {} devices. Some devices won't be used.", .{ target, pjrt_devices.len, MAX_NUM_DEVICES }); } From c817fe94f826e8d42175535ea7dcbe0cf24b90f2 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 1 Apr 2026 10:13:12 +0200 Subject: [PATCH 398/881] Revert "flashattn: update flash-attention to v2026.3.23" (#446) This reverts commit 7f71fa58c2ead570be1d57678d1c0bb693cb80a4. --- platforms/cuda/flashattn/flashattn.zig | 2 +- third_party/flashattn/flashattention.BUILD.bazel | 4 ++-- third_party/flashattn/repo.bzl | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/platforms/cuda/flashattn/flashattn.zig b/platforms/cuda/flashattn/flashattn.zig index eb4a8c72..368d0ffb 100644 --- a/platforms/cuda/flashattn/flashattn.zig +++ b/platforms/cuda/flashattn/flashattn.zig @@ -100,7 +100,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { const r = try bazel.runfiles(bazel_builtin.current_repository); var buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; - const library = (try r.rlocation("flashattn/flashattn/libflashattn.so", &buffer)) orelse return error.NotFound; + const library = (try r.rlocation("flashattn/flashattn/lib/libflashattn.so", &buffer)) orelse return error.NotFound; var lib = std.DynLib.open(library) catch |err| { std.log.err("Failed to open libflashattn.so: {any}", .{err}); return err; diff --git a/third_party/flashattn/flashattention.BUILD.bazel b/third_party/flashattn/flashattention.BUILD.bazel index 90f6aff5..3f32c61c 100644 --- a/third_party/flashattn/flashattention.BUILD.bazel +++ b/third_party/flashattn/flashattention.BUILD.bazel @@ -3,11 +3,11 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") filegroup( name = "libflashattn", visibility = ["//visibility:public"], - srcs = ["flashattn/libflashattn.so"], + srcs = ["flashattn/lib/libflashattn.so"], ) cc_library( name = "c", - hdrs = ["flashattn/flashattn.h"], + hdrs = ["flashattn/include/capi.h"], visibility = ["//visibility:public"], ) diff --git a/third_party/flashattn/repo.bzl b/third_party/flashattn/repo.bzl index 146e5cc6..ced98a5b 100644 --- a/third_party/flashattn/repo.bzl +++ b/third_party/flashattn/repo.bzl @@ -4,9 +4,10 @@ load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") def repo(): http_archive( name = "flashattn", - url = "https://github.com/zml/flash-attention/releases/download/v2026.3.23/flashattn-linux-amd64.tar.zst", - sha256 = "79496eaa5a373170a201a31abe822c1774138fce8ffcd2aa39c6f0fc46c79e2d", + url = "https://github.com/zml/flash-attention/releases/download/v0.0.6-rc7/flashattn.tar.gz", + sha256 = "6b8195a5ab285404fcb084122c1c32f342ff695040b8652c9144afd2289b71b3", build_file = "//:third_party/flashattn/flashattention.BUILD.bazel", ) #new_local_repository(name = "flashattn", build_file="//:third_party/flashattn/BUILD.bazel", path="/home/corendos/flashattn/") + From c3c0a696897bb415b4508eb3b7210fa61dabf061 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 1 Apr 2026 10:48:10 +0200 Subject: [PATCH 399/881] ci: update actions (#447) - Update setup-bazel & checkout to latest - Add a bunch of `tags = ["manual"]` for docker image related targets --- .github/workflows/ci.yaml | 4 +- MODULE.bazel.lock | 658 +++++++++++++++++++++++++++++++++++-- bin/zml-smi/BUILD.bazel | 4 + examples/mnist/BUILD.bazel | 7 + 4 files changed, 641 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b5b048f6..115d6388 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -72,7 +72,7 @@ jobs: run: mkdir -p /tmp/zig-cache && mkdir -p /tmp/zml && sudo chmod -R 777 /tmp/zig-cache && sudo chmod -R 777 /tmp/zml - name: Setup Bazel - uses: bazel-contrib/setup-bazel@0.18.0 + uses: bazel-contrib/setup-bazel@0.19.0 with: bazelisk-version: 1.28.1 bazelisk-cache: ${{ !matrix.self_hosted }} @@ -107,7 +107,7 @@ jobs: steps: - uses: runs-on/action@v1 - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Setup Zig uses: mlugg/setup-zig@v2 diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index e9a0eaed..173129ab 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -3,18 +3,24 @@ "registryFileHashes": { "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2", - "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/MODULE.bazel": "70390338f7a5106231d20620712f7cccb659cd0e9d073d1991c038eb9fc57589", + "https://bcr.bazel.build/modules/abseil-cpp/20220623.1/MODULE.bazel": "73ae41b6818d423a11fd79d95aedef1258f304448193d4db4ff90e5e7a0f076c", "https://bcr.bazel.build/modules/abseil-cpp/20230125.1/MODULE.bazel": "89047429cb0207707b2dface14ba7f8df85273d484c2572755be4bab7ce9c3a0", "https://bcr.bazel.build/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "1c8cec495288dccd14fdae6e3f95f772c1c91857047a098fad772034264cc8cb", "https://bcr.bazel.build/modules/abseil-cpp/20230802.0/MODULE.bazel": "d253ae36a8bd9ee3c5955384096ccb6baf16a1b1e93e858370da0a3b94f77c16", "https://bcr.bazel.build/modules/abseil-cpp/20230802.1/MODULE.bazel": "fa92e2eb41a04df73cdabeec37107316f7e5272650f81d6cc096418fe647b915", + "https://bcr.bazel.build/modules/abseil-cpp/20240116.0/MODULE.bazel": "98dc378d64c12a4e4741ad3362f87fb737ee6a0886b2d90c3cdbb4d93ea3e0bf", "https://bcr.bazel.build/modules/abseil-cpp/20240116.1/MODULE.bazel": "37bcdb4440fbb61df6a1c296ae01b327f19e9bb521f9b8e26ec854b6f97309ed", "https://bcr.bazel.build/modules/abseil-cpp/20240116.2/MODULE.bazel": "73939767a4686cd9a520d16af5ab440071ed75cec1a876bf2fcfaf1f71987a16", + "https://bcr.bazel.build/modules/abseil-cpp/20240722.0/MODULE.bazel": "88668a07647adbdc14cb3a7cd116fb23c9dda37a90a1681590b6c9d8339a5b84", + "https://bcr.bazel.build/modules/abseil-cpp/20250127.0/MODULE.bazel": "d1086e248cda6576862b4b3fe9ad76a214e08c189af5b42557a6e1888812c5d5", "https://bcr.bazel.build/modules/abseil-cpp/20250127.1/MODULE.bazel": "c4a89e7ceb9bf1e25cf84a9f830ff6b817b72874088bf5141b314726e46a57c1", + "https://bcr.bazel.build/modules/abseil-cpp/20250512.0/MODULE.bazel": "c4d02dd22cd87458516655a45512060246ee2a4732f1fbe948a5bd9eb614e626", "https://bcr.bazel.build/modules/abseil-cpp/20250512.1/MODULE.bazel": "d209fdb6f36ffaf61c509fcc81b19e81b411a999a934a032e10cd009a0226215", - "https://bcr.bazel.build/modules/abseil-cpp/20260107.1/MODULE.bazel": "e33b3801443f5fd64465262084534115db76363df13d2168a42bbfacc747be81", - "https://bcr.bazel.build/modules/abseil-cpp/20260107.1/source.json": "7a9a88969b1e79268cf613728ca8ff8fa4bc4b1a9abee9ec1fb5f113ca751971", + "https://bcr.bazel.build/modules/abseil-cpp/20250814.1/MODULE.bazel": "51f2312901470cdab0dbdf3b88c40cd21c62a7ed58a3de45b365ddc5b11bcab2", + "https://bcr.bazel.build/modules/abseil-cpp/20250814.1/source.json": "cea3901d7e299da7320700abbaafe57a65d039f10d0d7ea601c4a66938ea4b0c", + "https://bcr.bazel.build/modules/apple_support/1.11.1/MODULE.bazel": "1843d7cd8a58369a444fc6000e7304425fba600ff641592161d9f15b179fb896", "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", + "https://bcr.bazel.build/modules/apple_support/1.17.1/MODULE.bazel": "655c922ab1209978a94ef6ca7d9d43e940cd97d9c172fb55f94d91ac53f8610b", "https://bcr.bazel.build/modules/apple_support/1.23.1/MODULE.bazel": "53763fed456a968cf919b3240427cf3a9d5481ec5466abc9d5dc51bc70087442", "https://bcr.bazel.build/modules/apple_support/1.24.1/MODULE.bazel": "f46e8ddad60aef170ee92b2f3d00ef66c147ceafea68b6877cb45bd91737f5f8", "https://bcr.bazel.build/modules/apple_support/2.2.0/MODULE.bazel": "5e32bc42e413a4544df7afee7fa4c7aff73e670a44e56e8e45ce1954332034ad", @@ -30,6 +36,7 @@ "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.2/MODULE.bazel": "598e7fe3b54f5fa64fdbeead1027653963a359cc23561d43680006f3b463d5a4", "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.3/MODULE.bazel": "37c764292861c2f70314efa9846bb6dbb44fc0308903b3285da6528305450183", "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.3/source.json": "605086bbc197743a0d360f7ddc550a1d4dfa0441bc807236e17170f636153348", + "https://bcr.bazel.build/modules/bazel_features/1.1.0/MODULE.bazel": "cfd42ff3b815a5f39554d97182657f8c4b9719568eb7fded2b9135f084bf760b", "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", "https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101", "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", @@ -42,6 +49,7 @@ "https://bcr.bazel.build/modules/bazel_features/1.24.0/MODULE.bazel": "4796b4c25b47053e9bbffa792b3792d07e228ff66cd0405faef56a978708acd4", "https://bcr.bazel.build/modules/bazel_features/1.27.0/MODULE.bazel": "621eeee06c4458a9121d1f104efb80f39d34deff4984e778359c60eaf1a8cb65", "https://bcr.bazel.build/modules/bazel_features/1.28.0/MODULE.bazel": "4b4200e6cbf8fa335b2c3f43e1d6ef3e240319c33d43d60cc0fbd4b87ece299d", + "https://bcr.bazel.build/modules/bazel_features/1.3.0/MODULE.bazel": "cdcafe83ec318cda34e02948e81d790aab8df7a929cec6f6969f13a489ccecd9", "https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87", "https://bcr.bazel.build/modules/bazel_features/1.32.0/MODULE.bazel": "095d67022a58cb20f7e20e1aefecfa65257a222c18a938e2914fd257b5f1ccdc", "https://bcr.bazel.build/modules/bazel_features/1.33.0/MODULE.bazel": "8b8dc9d2a4c88609409c3191165bccec0e4cb044cd7a72ccbe826583303459f6", @@ -66,35 +74,108 @@ "https://bcr.bazel.build/modules/bazel_skylib/1.4.2/MODULE.bazel": "3bd40978e7a1fac911d5989e6b09d8f64921865a45822d8b09e815eaa726a651", "https://bcr.bazel.build/modules/bazel_skylib/1.5.0/MODULE.bazel": "32880f5e2945ce6a03d1fbd588e9198c0a959bb42297b2cfaf1685b7bc32e138", "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/MODULE.bazel": "8fdee2dbaace6c252131c00e1de4b165dc65af02ea278476187765e1a617b917", - "https://bcr.bazel.build/modules/bazel_skylib/1.7.0/MODULE.bazel": "0db596f4563de7938de764cc8deeabec291f55e8ec15299718b93c4423e9796d", "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/MODULE.bazel": "3120d80c5861aa616222ec015332e5f8d3171e062e3e804a2a0253e1be26e59b", "https://bcr.bazel.build/modules/bazel_skylib/1.8.1/MODULE.bazel": "88ade7293becda963e0e3ea33e7d54d3425127e0a326e0d17da085a5f1f03ff6", "https://bcr.bazel.build/modules/bazel_skylib/1.8.2/MODULE.bazel": "69ad6927098316848b34a9142bcc975e018ba27f08c4ff403f50c1b6e646ca67", "https://bcr.bazel.build/modules/bazel_skylib/1.9.0/MODULE.bazel": "72997b29dfd95c3fa0d0c48322d05590418edef451f8db8db5509c57875fb4b7", "https://bcr.bazel.build/modules/bazel_skylib/1.9.0/source.json": "7ad77c1e8c1b84222d9b3f3cae016a76639435744c19330b0b37c0a3c9da7dc0", + "https://bcr.bazel.build/modules/boringssl/0.0.0-20211025-d4f1ab9/MODULE.bazel": "6ee6353f8b1a701fe2178e1d925034294971350b6d3ac37e67e5a7d463267834", + "https://bcr.bazel.build/modules/boringssl/0.0.0-20230215-5c22014/MODULE.bazel": "4b03dc0d04375fa0271174badcd202ed249870c8e895b26664fd7298abea7282", + "https://bcr.bazel.build/modules/boringssl/0.0.0-20240530-2db0eb3/MODULE.bazel": "d0405b762c5e87cd445b7015f2b8da5400ef9a8dbca0bfefa6c1cea79d528a97", + "https://bcr.bazel.build/modules/boringssl/0.20240913.0/MODULE.bazel": "fcaa7503a5213290831a91ed1eb538551cf11ac0bc3a6ad92d0fef92c5bd25fb", + "https://bcr.bazel.build/modules/boringssl/0.20241024.0/MODULE.bazel": "b540cff73d948cb79cb0bc108d7cef391d2098a25adabfda5043e4ef548dbc87", + "https://bcr.bazel.build/modules/boringssl/0.20241024.0/source.json": "d843092e682b84188c043ac742965d7f96e04c846c7e338187e03238674909a9", "https://bcr.bazel.build/modules/buildifier_prebuilt/8.5.1/MODULE.bazel": "77f2a1958d1d07376dd3ce3ae16540f2c1b01921c1fd21930827271260e75a66", "https://bcr.bazel.build/modules/buildifier_prebuilt/8.5.1/source.json": "ae9f3d9dc7bec033976cf47165a78788bebad2b5c272241063ae31bad8664fd4", "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", + "https://bcr.bazel.build/modules/c-ares/1.15.0/MODULE.bazel": "ba0a78360fdc83f02f437a9e7df0532ad1fbaa59b722f6e715c11effebaa0166", + "https://bcr.bazel.build/modules/c-ares/1.19.1/MODULE.bazel": "73bca21720772370ff91cc8e88bbbaf14897720c6473e87c1ddc0f848284c313", + "https://bcr.bazel.build/modules/c-ares/1.19.1/source.json": "56bfa95b01e4e0012e90eaa9f1bab823c48c3af4454286b889b9cc74f5098da1", + "https://bcr.bazel.build/modules/cel-spec/0.15.0/MODULE.bazel": "e1eed53d233acbdcf024b4b0bc1528116d92c29713251b5154078ab1348cb600", + "https://bcr.bazel.build/modules/cel-spec/0.15.0/source.json": "ab7dccdf21ea2261c0f809b5a5221a4d7f8b580309f285fdf1444baaca75d44a", + "https://bcr.bazel.build/modules/civetweb/1.16/MODULE.bazel": "46a38f9daeb57392e3827fce7d40926be0c802bd23cdd6bfd3a96c804de42fae", + "https://bcr.bazel.build/modules/civetweb/1.16/source.json": "ba8b9585adb8355cb51b999d57172fd05e7a762c56b8d4bac6db42c99de3beb7", + "https://bcr.bazel.build/modules/curl/8.4.0/MODULE.bazel": "0bc250aa1cb69590049383df7a9537c809591fcf876c620f5f097c58fdc9bc10", + "https://bcr.bazel.build/modules/curl/8.7.1/MODULE.bazel": "088221c35a2939c555e6e47cb31a81c15f8b59f4daa8009b1e9271a502d33485", + "https://bcr.bazel.build/modules/curl/8.8.0/MODULE.bazel": "7da3b3e79b0b4ee8f8c95d640bc6ad7b430ce66ef6e9c9d2bc29b3b5ef85f6fe", + "https://bcr.bazel.build/modules/curl/8.8.0/source.json": "d7d138b6878cf38891692fee0649ace35357fd549b425614d571786f054374d4", + "https://bcr.bazel.build/modules/cython/3.0.11-1/MODULE.bazel": "868b3f5c956c3657420d2302004c6bb92606bfa47e314bab7f2ba0630c7c966c", + "https://bcr.bazel.build/modules/cython/3.0.11-1/source.json": "da318be900b8ca9c3d1018839d3bebc5a8e1645620d0848fa2c696d4ecf7c296", + "https://bcr.bazel.build/modules/envoy_api/0.0.0-20241214-918efc9/MODULE.bazel": "24e05f6f52f37be63a795192848555a2c8c855e7814dbc1ed419fb04a7005464", + "https://bcr.bazel.build/modules/envoy_api/0.0.0-20250128-4de3c74/MODULE.bazel": "1fe72489212c530086e3ffb0e018b2bfef4663200ca03571570f9f006bef1d75", + "https://bcr.bazel.build/modules/envoy_api/0.0.0-20250128-4de3c74/source.json": "028519164a2e24563f4b43d810fdedc702daed90e71e7042d45ba82ad807b46f", "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.1/MODULE.bazel": "cdf8cbe5ee750db04b78878c9633cc76e80dcf4416cbe982ac3a9222f80713c8", "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.3/MODULE.bazel": "f1b7bb2dd53e8f2ef984b39485ec8a44e9076dda5c4b8efd2fb4c6a6e856a31d", "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.3/source.json": "ebe931bfe362e4b41e59ee00a528db6074157ff2ced92eb9e970acab2e1089c9", + "https://bcr.bazel.build/modules/gazelle/0.27.0/MODULE.bazel": "3446abd608295de6d90b4a8a118ed64a9ce11dcb3dda2dc3290a22056bd20996", + "https://bcr.bazel.build/modules/gazelle/0.30.0/MODULE.bazel": "f888a1effe338491f35f0e0e85003b47bb9d8295ccba73c37e07702d8d31c65b", + "https://bcr.bazel.build/modules/gazelle/0.32.0/MODULE.bazel": "b499f58a5d0d3537f3cf5b76d8ada18242f64ec474d8391247438bf04f58c7b8", + "https://bcr.bazel.build/modules/gazelle/0.33.0/MODULE.bazel": "a13a0f279b462b784fb8dd52a4074526c4a2afe70e114c7d09066097a46b3350", + "https://bcr.bazel.build/modules/gazelle/0.34.0/MODULE.bazel": "abdd8ce4d70978933209db92e436deb3a8b737859e9354fb5fd11fb5c2004c8a", + "https://bcr.bazel.build/modules/gazelle/0.36.0/MODULE.bazel": "e375d5d6e9a6ca59b0cb38b0540bc9a05b6aa926d322f2de268ad267a2ee74c0", + "https://bcr.bazel.build/modules/gazelle/0.37.0/MODULE.bazel": "d1327ba0907d0275ed5103bfbbb13518f6c04955b402213319d0d6c0ce9839d4", + "https://bcr.bazel.build/modules/gazelle/0.37.0/source.json": "b3adc10e2394e7f63ea88fb1d622d4894bfe9ec6961c493ae9a887723ab16831", "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb", - "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", + "https://bcr.bazel.build/modules/google_benchmark/1.8.4/MODULE.bazel": "c6d54a11dcf64ee63545f42561eda3fd94c1b5f5ebe1357011de63ae33739d5e", + "https://bcr.bazel.build/modules/google_benchmark/1.8.5/MODULE.bazel": "9ba9b31b984022828a950e3300410977eda2e35df35584c6b0b2d0c2e52766b7", + "https://bcr.bazel.build/modules/google_benchmark/1.8.5/source.json": "2c9c685f9b496f125b9e3a9c696c549d1ed2f33b75830a2fb6ac94fab23c0398", + "https://bcr.bazel.build/modules/googleapis/0.0.0-20240326-1c8d509c5/MODULE.bazel": "a4b7e46393c1cdcc5a00e6f85524467c48c565256b22b5fae20f84ab4a999a68", + "https://bcr.bazel.build/modules/googleapis/0.0.0-20240819-fe8ba054a/MODULE.bazel": "117b7c7be7327ed5d6c482274533f2dbd78631313f607094d4625c28203cacdf", + "https://bcr.bazel.build/modules/googleapis/0.0.0-20240819-fe8ba054a/source.json": "b31fc7eb283a83f71d2e5bfc3d1c562d2994198fa1278409fbe8caec3afc1d3e", "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6", "https://bcr.bazel.build/modules/googletest/1.14.0/MODULE.bazel": "cfbcbf3e6eac06ef9d85900f64424708cc08687d1b527f0ef65aa7517af8118f", "https://bcr.bazel.build/modules/googletest/1.15.2/MODULE.bazel": "6de1edc1d26cafb0ea1a6ab3f4d4192d91a312fd2d360b63adaa213cd00b2108", + "https://bcr.bazel.build/modules/googletest/1.16.0/MODULE.bazel": "a175623c69e94fca4ca7acbc12031e637b0c489318cd4805606981d4d7adb34a", "https://bcr.bazel.build/modules/googletest/1.17.0/MODULE.bazel": "dbec758171594a705933a29fcf69293d2468c49ec1f2ebca65c36f504d72df46", "https://bcr.bazel.build/modules/googletest/1.17.0/source.json": "38e4454b25fc30f15439c0378e57909ab1fd0a443158aa35aec685da727cd713", + "https://bcr.bazel.build/modules/grpc-java/1.74.0/MODULE.bazel": "6e6b8214d1b77244263e13cf62b71467db8284369ad60567cf6ffcc0650dc621", + "https://bcr.bazel.build/modules/grpc-java/1.74.0/source.json": "340f0cec843668dcdb6b7d1460c8f93937dd915e866c26d20ef1f30ffb4c3125", + "https://bcr.bazel.build/modules/grpc-proto/0.0.0-20240627-ec30f58/MODULE.bazel": "88de79051e668a04726e9ea94a481ec6f1692086735fd6f488ab908b3b909238", + "https://bcr.bazel.build/modules/grpc-proto/0.0.0-20240627-ec30f58/source.json": "5035d379c61042930244ab59e750106d893ec440add92ec0df6a0098ca7f131d", + "https://bcr.bazel.build/modules/grpc/1.41.0/MODULE.bazel": "5bcbfc2b274dabea628f0649dc50c90cf36543b1cfc31624832538644ad1aae8", + "https://bcr.bazel.build/modules/grpc/1.56.3.bcr.1/MODULE.bazel": "cd5b1eb276b806ec5ab85032921f24acc51735a69ace781be586880af20ab33f", + "https://bcr.bazel.build/modules/grpc/1.62.1/MODULE.bazel": "2998211594b8a79a6b459c4e797cfa19f0fb8b3be3149760ec7b8c99abfd426f", + "https://bcr.bazel.build/modules/grpc/1.63.1.bcr.1/MODULE.bazel": "d7b9fef03bd175e6825237b521b18a3c29f1ac15f8aa52c8a1a0f3bd8f33d54b", + "https://bcr.bazel.build/modules/grpc/1.66.0.bcr.2/MODULE.bazel": "0fa2b0fd028ce354febf0fe90f1ed8fecfbfc33118cddd95ac0418cc283333a0", + "https://bcr.bazel.build/modules/grpc/1.66.0.bcr.3/MODULE.bazel": "f6047e89faf488f5e3e65cb2594c6f5e86992abec7487163ff6b623526e543b0", + "https://bcr.bazel.build/modules/grpc/1.69.0/MODULE.bazel": "4e26e05c9e1ef291ccbc96aad8e457b1b8abedbc141623831629da2f8168eef6", + "https://bcr.bazel.build/modules/grpc/1.71.0/MODULE.bazel": "7fcab2c05530373f1a442c362b17740dd0c75b6a2a975eec8f5bf4c70a37928a", + "https://bcr.bazel.build/modules/grpc/1.74.0/MODULE.bazel": "83bca3739c1a3eba91f63f38b352e8681c9e58576f2def545e49a1e461b98a1f", + "https://bcr.bazel.build/modules/grpc/1.74.0/source.json": "87c009e2f3226fd43170ce4896a3f17c819bd4c7388ff1336158b12b636a0ed9", "https://bcr.bazel.build/modules/jq.bzl/0.1.0/MODULE.bazel": "2ce69b1af49952cd4121a9c3055faa679e748ce774c7f1fda9657f936cae902f", "https://bcr.bazel.build/modules/jq.bzl/0.1.0/source.json": "746bf13cac0860f091df5e4911d0c593971cd8796b5ad4e809b2f8e133eee3d5", - "https://bcr.bazel.build/modules/jsoncpp/1.9.5/MODULE.bazel": "31271aedc59e815656f5736f282bb7509a97c7ecb43e927ac1a37966e0578075", "https://bcr.bazel.build/modules/jsoncpp/1.9.6/MODULE.bazel": "2f8d20d3b7d54143213c4dfc3d98225c42de7d666011528dc8fe91591e2e17b0", "https://bcr.bazel.build/modules/jsoncpp/1.9.6/source.json": "a04756d367a2126c3541682864ecec52f92cdee80a35735a3cb249ce015ca000", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", + "https://bcr.bazel.build/modules/libpfm/4.11.0/source.json": "caaffb3ac2b59b8aac456917a4ecf3167d40478ee79f15ab7a877ec9273937c9", "https://bcr.bazel.build/modules/llvm/0.6.7/MODULE.bazel": "d37a2e10571864dc6a5bb53c29216d90b9400bbcadb422337f49107fd2eaf0d2", "https://bcr.bazel.build/modules/llvm/0.7.0/MODULE.bazel": "3c07a4e5734b0ad41fe24dedaacbf3a35ce4377b7e1a21f24488a0c9ac4f1e6b", "https://bcr.bazel.build/modules/llvm/0.7.0/source.json": "5f0a015da1e7db09ef61aa254ba5eb55b0c322fadd1e8eefeca368d7723a24a0", + "https://bcr.bazel.build/modules/mbedtls/3.6.0/MODULE.bazel": "8e380e4698107c5f8766264d4df92e36766248447858db28187151d884995a09", + "https://bcr.bazel.build/modules/mbedtls/3.6.0/source.json": "1dbe7eb5258050afcc3806b9d43050f71c6f539ce0175535c670df606790b30c", + "https://bcr.bazel.build/modules/nlohmann_json/3.11.3/MODULE.bazel": "87023db2f55fc3a9949c7b08dc711fae4d4be339a80a99d04453c4bb3998eefc", + "https://bcr.bazel.build/modules/nlohmann_json/3.11.3/source.json": "296c63a90c6813e53b3812d24245711981fc7e563d98fe15625f55181494488a", + "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", + "https://bcr.bazel.build/modules/opencensus-cpp/0.0.0-20230502-50eb5de.bcr.2/MODULE.bazel": "cc18734138dd18c912c6ce2a59186db28f85d8058c99c9f21b46ca3e0aba0ebe", + "https://bcr.bazel.build/modules/opencensus-cpp/0.0.0-20230502-50eb5de.bcr.2/source.json": "7c135f9d42bb3b045669c3c6ab3bb3c208e00b46aca4422eea64c29811a5b240", + "https://bcr.bazel.build/modules/opencensus-cpp/0.0.0-20230502-50eb5de/MODULE.bazel": "02201d2921dadb4ec90c4980eca4b2a02904eddcf6fa02f3da7594fb7b0d821c", + "https://bcr.bazel.build/modules/opencensus-proto/0.4.1.bcr.2/MODULE.bazel": "789706a714855f92c5c8cfcf1ef32bbb64dcd3b7c9066756ad7986ec59709d29", + "https://bcr.bazel.build/modules/opencensus-proto/0.4.1.bcr.2/source.json": "aadf3f53e08b72376506b7c4ea3d167010c9efb160d7d6e1e304ed646bac1b36", + "https://bcr.bazel.build/modules/opencensus-proto/0.4.1/MODULE.bazel": "4a2e8b4d0b544002502474d611a5a183aa282251e14f6a01afe841c0c1b10372", + "https://bcr.bazel.build/modules/openssl/3.3.1.bcr.1/MODULE.bazel": "49c0c07e8fb87b480bccb842cfee1b32617f11dac590f732573c69058699a3d1", + "https://bcr.bazel.build/modules/openssl/3.3.1.bcr.1/source.json": "0c0872e048bbea052a9c541fb47019481a19201ba5555a71d762ad591bf94e1f", + "https://bcr.bazel.build/modules/opentelemetry-cpp/1.14.2/MODULE.bazel": "089a5613c2a159c7dfde098dabfc61e966889c7d6a81a98422a84c51535ed17d", + "https://bcr.bazel.build/modules/opentelemetry-cpp/1.16.0/MODULE.bazel": "b7379a140f538cea3f749179a2d481ed81942cc6f7b05a6113723eb34ac3b3e7", + "https://bcr.bazel.build/modules/opentelemetry-cpp/1.19.0/MODULE.bazel": "3455326c08b28415648a3d60d8e3c811847ebdbe64474f75b25878f25585aea1", + "https://bcr.bazel.build/modules/opentelemetry-cpp/1.19.0/source.json": "4e48137e4c3ecb99401ff99876df8fa330598d7da051869bec643446e8a8ff95", + "https://bcr.bazel.build/modules/opentelemetry-proto/1.1.0/MODULE.bazel": "a49f406e99bf05ab43ed4f5b3322fbd33adfd484b6546948929d1316299b68bf", + "https://bcr.bazel.build/modules/opentelemetry-proto/1.3.1/MODULE.bazel": "0141a50e989576ee064c11ce8dd5ec89993525bd9f9a09c5618e4dacc8df9352", + "https://bcr.bazel.build/modules/opentelemetry-proto/1.4.0.bcr.1/MODULE.bazel": "5ceaf25e11170d22eded4c8032728b4a3f273765fccda32f9e94f463755c4167", + "https://bcr.bazel.build/modules/opentelemetry-proto/1.5.0/MODULE.bazel": "7543d91a53b98e7b5b37c5a0865b93bff12c1ee022b1e322cd236b968894b030", + "https://bcr.bazel.build/modules/opentelemetry-proto/1.5.0/source.json": "046b721ce203e88cdaad44d7dd17a86b7200eab9388b663b234e72e13ff7b143", + "https://bcr.bazel.build/modules/opentracing-cpp/1.6.0/MODULE.bazel": "b3925269f63561b8b880ae7cf62ccf81f6ece55b62cd791eda9925147ae116ec", + "https://bcr.bazel.build/modules/opentracing-cpp/1.6.0/source.json": "da1cb1add160f5e5074b7272e9db6fd8f1b3336c15032cd0a653af9d2f484aed", "https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92", "https://bcr.bazel.build/modules/package_metadata/0.0.5/MODULE.bazel": "ef4f9439e3270fdd6b9fd4dbc3d2f29d13888e44c529a1b243f7a31dfbc2e8e4", "https://bcr.bazel.build/modules/package_metadata/0.0.5/source.json": "2326db2f6592578177751c3e1f74786b79382cd6008834c9d01ec865b9126a85", @@ -112,29 +193,35 @@ "https://bcr.bazel.build/modules/platforms/0.0.9/MODULE.bazel": "4a87a60c927b56ddd67db50c89acaa62f4ce2a1d2149ccb63ffd871d5ce29ebc", "https://bcr.bazel.build/modules/platforms/1.0.0/MODULE.bazel": "f05feb42b48f1b3c225e4ccf351f367be0371411a803198ec34a389fb22aa580", "https://bcr.bazel.build/modules/platforms/1.0.0/source.json": "f4ff1fd412e0246fd38c82328eb209130ead81d62dcd5a9e40910f867f733d96", - "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7", - "https://bcr.bazel.build/modules/protobuf/23.1/MODULE.bazel": "88b393b3eb4101d18129e5db51847cd40a5517a53e81216144a8c32dfeeca52a", - "https://bcr.bazel.build/modules/protobuf/24.4/MODULE.bazel": "7bc7ce5f2abf36b3b7b7c8218d3acdebb9426aeb35c2257c96445756f970eb12", - "https://bcr.bazel.build/modules/protobuf/27.0/MODULE.bazel": "7873b60be88844a0a1d8f80b9d5d20cfbd8495a689b8763e76c6372998d3f64c", - "https://bcr.bazel.build/modules/protobuf/27.1/MODULE.bazel": "703a7b614728bb06647f965264967a8ef1c39e09e8f167b3ca0bb1fd80449c0d", - "https://bcr.bazel.build/modules/protobuf/29.0-rc2/MODULE.bazel": "6241d35983510143049943fc0d57937937122baf1b287862f9dc8590fc4c37df", - "https://bcr.bazel.build/modules/protobuf/29.0-rc3/MODULE.bazel": "33c2dfa286578573afc55a7acaea3cada4122b9631007c594bf0729f41c8de92", - "https://bcr.bazel.build/modules/protobuf/29.0/MODULE.bazel": "319dc8bf4c679ff87e71b1ccfb5a6e90a6dbc4693501d471f48662ac46d04e4e", - "https://bcr.bazel.build/modules/protobuf/29.1/MODULE.bazel": "557c3457560ff49e122ed76c0bc3397a64af9574691cb8201b4e46d4ab2ecb95", - "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0", - "https://bcr.bazel.build/modules/protobuf/34.0.bcr.1/MODULE.bazel": "74e541b0ba877813da786a11707d4e394433c157841d5111a36be0d44b907931", - "https://bcr.bazel.build/modules/protobuf/34.0.bcr.1/source.json": "fc174b3d6215aa14197d1bd779f98bb72d9fd666ee5ec0d6bba6ae986baa4535", + "https://bcr.bazel.build/modules/prometheus-cpp/1.2.4/MODULE.bazel": "0fbe5dcff66311947a3f6b86ebc6a6d9328e31a28413ca864debc4a043f371e5", + "https://bcr.bazel.build/modules/prometheus-cpp/1.3.0.bcr.1/MODULE.bazel": "116ad46e97c1d2aeb020fe2899a342a7e703574ce7c0faf7e4810f938c974a9a", + "https://bcr.bazel.build/modules/prometheus-cpp/1.3.0.bcr.1/source.json": "e813cce2d450708cfcb26e309c5172583a7440776edf354e83e6788c768e5cca", + "https://bcr.bazel.build/modules/prometheus-cpp/1.3.0/MODULE.bazel": "ce82e086bbc0b60267e970f6a54b2ca6d0f22d3eb6633e00e2cc2899c700f3d8", + "https://bcr.bazel.build/modules/protobuf/33.6/MODULE.bazel": "af76890dbc467c56f592796b66f8f507013b34190f4b569a1b69318ea9a26c8b", + "https://bcr.bazel.build/modules/protobuf/33.6/source.json": "8921cf8549b5648ff3aaaee6c89160a23e4794528f45886b6743381127731f46", + "https://bcr.bazel.build/modules/protoc-gen-validate/1.0.4.bcr.2/MODULE.bazel": "c4bd2c850211ff5b7dadf9d2d0496c1c922fdedc303c775b01dfd3b3efc907ed", + "https://bcr.bazel.build/modules/protoc-gen-validate/1.0.4/MODULE.bazel": "b8913c154b16177990f6126d2d2477d187f9ddc568e95ee3e2d50fc65d2c494a", + "https://bcr.bazel.build/modules/protoc-gen-validate/1.2.1.bcr.1/MODULE.bazel": "4bf09676b62fa587ae07e073420a76ec8766dcce7545e5f8c68cfa8e484b5120", + "https://bcr.bazel.build/modules/protoc-gen-validate/1.2.1.bcr.1/source.json": "c19071ebc4b53b5f1cfab9c66eefaf6e4179eb8a998970d07b1077687e777f29", + "https://bcr.bazel.build/modules/protoc-gen-validate/1.2.1/MODULE.bazel": "52b51f50533ec4fbd5d613cd093773f979ac2e035d954e02ca11de383f502505", "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e", "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/MODULE.bazel": "e6f4c20442eaa7c90d7190d8dc539d0ab422f95c65a57cc59562170c58ae3d34", "https://bcr.bazel.build/modules/pybind11_bazel/2.13.6/MODULE.bazel": "2d746fda559464b253b2b2e6073cb51643a2ac79009ca02100ebbc44b4548656", "https://bcr.bazel.build/modules/pybind11_bazel/2.13.6/source.json": "6aa0703de8efb20cc897bbdbeb928582ee7beaf278bcd001ac253e1605bddfae", + "https://bcr.bazel.build/modules/rapidjson/1.1.0.bcr.20241007/MODULE.bazel": "82fbcb2e42f9e0040e76ccc74c06c3e46dfd33c64ca359293f8b84df0e6dff4c", + "https://bcr.bazel.build/modules/rapidjson/1.1.0.bcr.20241007/source.json": "5c42389ad0e21fc06b95ad7c0b730008271624a2fa3292e0eab5f30e15adeee3", + "https://bcr.bazel.build/modules/re2/2021-09-01/MODULE.bazel": "bcb6b96f3b071e6fe2d8bed9cc8ada137a105f9d2c5912e91d27528b3d123833", "https://bcr.bazel.build/modules/re2/2023-09-01/MODULE.bazel": "cb3d511531b16cfc78a225a9e2136007a48cf8a677e4264baeab57fe78a80206", + "https://bcr.bazel.build/modules/re2/2024-05-01/MODULE.bazel": "55a3f059538f381107824e7d00df5df6d061ba1fb80e874e4909c0f0549e8f3e", "https://bcr.bazel.build/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "b4963dda9b31080be1905ef085ecd7dd6cd47c05c79b9cdf83ade83ab2ab271a", "https://bcr.bazel.build/modules/re2/2024-07-02/MODULE.bazel": "0eadc4395959969297cbcf31a249ff457f2f1d456228c67719480205aa306daa", "https://bcr.bazel.build/modules/re2/2025-11-05/MODULE.bazel": "d97e240364f132017500a3af816493af7efd4be40aab2bb403ed449cc96f70a3", "https://bcr.bazel.build/modules/re2/2025-11-05/source.json": "7df1394aabda1c9bc188a302f5d54b1c657924edd04ebc57d2be29dbd7efd141", "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", + "https://bcr.bazel.build/modules/rules_apple/3.16.0/MODULE.bazel": "0d1caf0b8375942ce98ea944be754a18874041e4e0459401d925577624d3a54a", + "https://bcr.bazel.build/modules/rules_apple/3.16.0/source.json": "d8b5fe461272018cc07cfafce11fe369c7525330804c37eec5a82f84cd475366", + "https://bcr.bazel.build/modules/rules_apple/3.5.1/MODULE.bazel": "3d1bbf65ad3692003d36d8a29eff54d4e5c1c5f4bfb60f79e28646a924d9101c", "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", "https://bcr.bazel.build/modules/rules_cc/0.0.10/MODULE.bazel": "ec1705118f7eaedd6e118508d3d26deba2a4e76476ada7e0e3965211be012002", "https://bcr.bazel.build/modules/rules_cc/0.0.13/MODULE.bazel": "0e8529ed7b323dad0775ff924d2ae5af7640b23553dfcd4d34344c7e7a867191", @@ -143,45 +230,56 @@ "https://bcr.bazel.build/modules/rules_cc/0.0.16/MODULE.bazel": "7661303b8fc1b4d7f532e54e9d6565771fea666fbdf839e0a86affcd02defe87", "https://bcr.bazel.build/modules/rules_cc/0.0.17/MODULE.bazel": "2ae1d8f4238ec67d7185d8861cb0a2cdf4bc608697c331b95bf990e69b62e64a", "https://bcr.bazel.build/modules/rules_cc/0.0.2/MODULE.bazel": "6915987c90970493ab97393024c156ea8fb9f3bea953b2f3ec05c34f19b5695c", + "https://bcr.bazel.build/modules/rules_cc/0.0.5/MODULE.bazel": "be41f87587998fe8890cd82ea4e848ed8eb799e053c224f78f3ff7fe1a1d9b74", "https://bcr.bazel.build/modules/rules_cc/0.0.6/MODULE.bazel": "abf360251023dfe3efcef65ab9d56beefa8394d4176dd29529750e1c57eaa33f", "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", "https://bcr.bazel.build/modules/rules_cc/0.1.1/MODULE.bazel": "2f0222a6f229f0bf44cd711dc13c858dad98c62d52bd51d8fc3a764a83125513", "https://bcr.bazel.build/modules/rules_cc/0.1.4/MODULE.bazel": "bb03a452a7527ac25a7518fb86a946ef63df860b9657d8323a0c50f8504fb0b9", "https://bcr.bazel.build/modules/rules_cc/0.1.5/MODULE.bazel": "88dfc9361e8b5ae1008ac38f7cdfd45ad738e4fa676a3ad67d19204f045a1fd8", + "https://bcr.bazel.build/modules/rules_cc/0.2.0/MODULE.bazel": "b5c17f90458caae90d2ccd114c81970062946f49f355610ed89bebf954f5783c", "https://bcr.bazel.build/modules/rules_cc/0.2.15/MODULE.bazel": "6a0a4a75a57aa6dc888300d848053a58c6b12a29f89d4304e1c41448514ec6e8", "https://bcr.bazel.build/modules/rules_cc/0.2.16/MODULE.bazel": "9242fa89f950c6ef7702801ab53922e99c69b02310c39fb6e62b2bd30df2a1d4", "https://bcr.bazel.build/modules/rules_cc/0.2.16/source.json": "d03d5cde49376d87e14ec14b666c56075e5e3926930327fd5d0484a1ff2ac1cc", "https://bcr.bazel.build/modules/rules_cc/0.2.4/MODULE.bazel": "1ff1223dfd24f3ecf8f028446d4a27608aa43c3f41e346d22838a4223980b8cc", "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", - "https://bcr.bazel.build/modules/rules_cc/0.2.9/MODULE.bazel": "34263f1dca62ea664265438cef714d7db124c03e1ed55ebb4f1dc860164308d1", "https://bcr.bazel.build/modules/rules_distroless/0.6.2/MODULE.bazel": "eafe8b473c0c78038d9a6e1dbd9278b7bf83457ddc78207297483bfbe1012d56", "https://bcr.bazel.build/modules/rules_distroless/0.6.2/source.json": "830d7a25ce382e571e58791fbed3b47e6da3237805624bab8997ae59d763bdc2", + "https://bcr.bazel.build/modules/rules_foreign_cc/0.10.1/MODULE.bazel": "b9527010e5fef060af92b6724edb3691970a5b1f76f74b21d39f7d433641be60", + "https://bcr.bazel.build/modules/rules_foreign_cc/0.10.1/source.json": "9300e71df0cdde0952f10afff1401fa664e9fc5d9ae6204660ba1b158d90d6a6", "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", - "https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/MODULE.bazel": "40c97d1144356f52905566c55811f13b299453a14ac7769dfba2ac38192337a8", + "https://bcr.bazel.build/modules/rules_go/0.33.0/MODULE.bazel": "a2b11b64cd24bf94f57454f53288a5dacfe6cb86453eee7761b7637728c1910c", + "https://bcr.bazel.build/modules/rules_go/0.38.1/MODULE.bazel": "fb8e73dd3b6fc4ff9d260ceacd830114891d49904f5bda1c16bc147bcc254f71", + "https://bcr.bazel.build/modules/rules_go/0.39.1/MODULE.bazel": "d34fb2a249403a5f4339c754f1e63dc9e5ad70b47c5e97faee1441fc6636cd61", + "https://bcr.bazel.build/modules/rules_go/0.41.0/MODULE.bazel": "55861d8e8bb0e62cbd2896f60ff303f62ffcb0eddb74ecb0e5c0cbe36fc292c8", + "https://bcr.bazel.build/modules/rules_go/0.42.0/MODULE.bazel": "8cfa875b9aa8c6fce2b2e5925e73c1388173ea3c32a0db4d2b4804b453c14270", + "https://bcr.bazel.build/modules/rules_go/0.45.1/MODULE.bazel": "6d7884f0edf890024eba8ab31a621faa98714df0ec9d512389519f0edff0281a", + "https://bcr.bazel.build/modules/rules_go/0.46.0/MODULE.bazel": "3477df8bdcc49e698b9d25f734c4f3a9f5931ff34ee48a2c662be168f5f2d3fd", + "https://bcr.bazel.build/modules/rules_go/0.48.0/MODULE.bazel": "d00ebcae0908ee3f5e6d53f68677a303d6d59a77beef879598700049c3980a03", + "https://bcr.bazel.build/modules/rules_go/0.50.1/MODULE.bazel": "b91a308dc5782bb0a8021ad4330c81fea5bda77f96b9e4c117b9b9c8f6665ee0", + "https://bcr.bazel.build/modules/rules_go/0.50.1/source.json": "205765fd30216c70321f84c9a967267684bdc74350af3f3c46c857d9f80a4fa2", "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", + "https://bcr.bazel.build/modules/rules_java/5.1.0/MODULE.bazel": "324b6478b0343a3ce7a9add8586ad75d24076d6d43d2f622990b9c1cfd8a1b15", "https://bcr.bazel.build/modules/rules_java/5.3.5/MODULE.bazel": "a4ec4f2db570171e3e5eb753276ee4b389bae16b96207e9d3230895c99644b86", + "https://bcr.bazel.build/modules/rules_java/5.5.0/MODULE.bazel": "486ad1aa15cdc881af632b4b1448b0136c76025a1fe1ad1b65c5899376b83a50", "https://bcr.bazel.build/modules/rules_java/6.0.0/MODULE.bazel": "8a43b7df601a7ec1af61d79345c17b31ea1fedc6711fd4abfd013ea612978e39", "https://bcr.bazel.build/modules/rules_java/6.3.0/MODULE.bazel": "a97c7678c19f236a956ad260d59c86e10a463badb7eb2eda787490f4c969b963", "https://bcr.bazel.build/modules/rules_java/6.4.0/MODULE.bazel": "e986a9fe25aeaa84ac17ca093ef13a4637f6107375f64667a15999f77db6c8f6", - "https://bcr.bazel.build/modules/rules_java/6.5.2/MODULE.bazel": "1d440d262d0e08453fa0c4d8f699ba81609ed0e9a9a0f02cd10b3e7942e61e31", - "https://bcr.bazel.build/modules/rules_java/7.1.0/MODULE.bazel": "30d9135a2b6561c761bd67bd4990da591e6bdc128790ce3e7afd6a3558b2fb64", "https://bcr.bazel.build/modules/rules_java/7.10.0/MODULE.bazel": "530c3beb3067e870561739f1144329a21c851ff771cd752a49e06e3dc9c2e71a", "https://bcr.bazel.build/modules/rules_java/7.12.2/MODULE.bazel": "579c505165ee757a4280ef83cda0150eea193eed3bef50b1004ba88b99da6de6", "https://bcr.bazel.build/modules/rules_java/7.2.0/MODULE.bazel": "06c0334c9be61e6cef2c8c84a7800cef502063269a5af25ceb100b192453d4ab", "https://bcr.bazel.build/modules/rules_java/7.3.2/MODULE.bazel": "50dece891cfdf1741ea230d001aa9c14398062f2b7c066470accace78e412bc2", + "https://bcr.bazel.build/modules/rules_java/7.4.0/MODULE.bazel": "a592852f8a3dd539e82ee6542013bf2cadfc4c6946be8941e189d224500a8934", "https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe", "https://bcr.bazel.build/modules/rules_java/8.14.0/MODULE.bazel": "717717ed40cc69994596a45aec6ea78135ea434b8402fb91b009b9151dd65615", "https://bcr.bazel.build/modules/rules_java/8.14.0/source.json": "8a88c4ca9e8759da53cddc88123880565c520503321e2566b4e33d0287a3d4bc", - "https://bcr.bazel.build/modules/rules_java/8.3.2/MODULE.bazel": "7336d5511ad5af0b8615fdc7477535a2e4e723a357b6713af439fe8cf0195017", "https://bcr.bazel.build/modules/rules_java/8.5.1/MODULE.bazel": "d8a9e38cc5228881f7055a6079f6f7821a073df3744d441978e7a43e20226939", "https://bcr.bazel.build/modules/rules_java/8.6.0/MODULE.bazel": "9c064c434606d75a086f15ade5edb514308cccd1544c2b2a89bbac4310e41c71", "https://bcr.bazel.build/modules/rules_java/8.6.1/MODULE.bazel": "f4808e2ab5b0197f094cabce9f4b006a27766beb6a9975931da07099560ca9c2", "https://bcr.bazel.build/modules/rules_java/8.8.0/MODULE.bazel": "de589d0880911ac007abd521b9f0ddcd8b0dbd05c8553e6f8124a050b83acf7d", - "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7", - "https://bcr.bazel.build/modules/rules_jvm_external/5.1/MODULE.bazel": "33f6f999e03183f7d088c9be518a63467dfd0be94a11d0055fe2d210f89aa909", "https://bcr.bazel.build/modules/rules_jvm_external/5.2/MODULE.bazel": "d9351ba35217ad0de03816ef3ed63f89d411349353077348a45348b096615036", "https://bcr.bazel.build/modules/rules_jvm_external/5.3/MODULE.bazel": "bf93870767689637164657731849fb887ad086739bd5d360d90007a581d5527d", + "https://bcr.bazel.build/modules/rules_jvm_external/6.0/MODULE.bazel": "37c93a5a78d32e895d52f86a8d0416176e915daabd029ccb5594db422e87c495", "https://bcr.bazel.build/modules/rules_jvm_external/6.1/MODULE.bazel": "75b5fec090dbd46cf9b7d8ea08cf84a0472d92ba3585b476f44c326eda8059c4", "https://bcr.bazel.build/modules/rules_jvm_external/6.3/MODULE.bazel": "c998e060b85f71e00de5ec552019347c8bca255062c990ac02d051bb80a38df0", "https://bcr.bazel.build/modules/rules_jvm_external/6.7/MODULE.bazel": "e717beabc4d091ecb2c803c2d341b88590e9116b8bf7947915eeb33aab4f96dd", @@ -195,23 +293,30 @@ "https://bcr.bazel.build/modules/rules_license/1.0.0/source.json": "a52c89e54cc311196e478f8382df91c15f7a2bfdf4c6cd0e2675cc2ff0b56efb", "https://bcr.bazel.build/modules/rules_oci/2.2.7/MODULE.bazel": "f6150e4b224d459f7f6523ef65967464ca4efdd266c7fbf2f5a2a51011957e0c", "https://bcr.bazel.build/modules/rules_oci/2.2.7/source.json": "b099f02af330f47f19dc67fc9300ef6e1937a8c86882690db0e7a2fcea8c7f6b", + "https://bcr.bazel.build/modules/rules_perl/0.2.4/MODULE.bazel": "5f5af7be4bf5fb88d91af7469518f0fd2161718aefc606188f7cd51f436ca938", + "https://bcr.bazel.build/modules/rules_perl/0.2.4/source.json": "574317d6b3c7e4843fe611b76f15e62a1889949f5570702e1ee4ad335ea3c339", "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc", "https://bcr.bazel.build/modules/rules_pkg/1.0.1/MODULE.bazel": "5b1df97dbc29623bccdf2b0dcd0f5cb08e2f2c9050aab1092fd39a41e82686ff", "https://bcr.bazel.build/modules/rules_pkg/1.0.1/source.json": "bd82e5d7b9ce2d31e380dd9f50c111d678c3bdaca190cb76b0e1c71b05e1ba8a", "https://bcr.bazel.build/modules/rules_proto/4.0.0/MODULE.bazel": "a7a7b6ce9bee418c1a760b3d84f83a299ad6952f9903c67f19e4edd964894e06", "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "e8dff86b0971688790ae75528fe1813f71809b5afd57facb44dad9e8eca631b7", "https://bcr.bazel.build/modules/rules_proto/6.0.0-rc1/MODULE.bazel": "1e5b502e2e1a9e825eef74476a5a1ee524a92297085015a052510b09a1a09483", + "https://bcr.bazel.build/modules/rules_proto/6.0.0/MODULE.bazel": "b531d7f09f58dce456cd61b4579ce8c86b38544da75184eadaf0a7cb7966453f", "https://bcr.bazel.build/modules/rules_proto/6.0.2/MODULE.bazel": "ce916b775a62b90b61888052a416ccdda405212b6aaeb39522f7dc53431a5e73", "https://bcr.bazel.build/modules/rules_proto/7.0.2/MODULE.bazel": "bf81793bd6d2ad89a37a40693e56c61b0ee30f7a7fdbaf3eabbf5f39de47dea2", "https://bcr.bazel.build/modules/rules_proto/7.1.0/MODULE.bazel": "002d62d9108f75bb807cd56245d45648f38275cb3a99dcd45dfb864c5d74cb96", "https://bcr.bazel.build/modules/rules_proto/7.1.0/source.json": "39f89066c12c24097854e8f57ab8558929f9c8d474d34b2c00ac04630ad8940e", - "https://bcr.bazel.build/modules/rules_python/0.10.2/MODULE.bazel": "cc82bc96f2997baa545ab3ce73f196d040ffb8756fd2d66125a530031cd90e5f", + "https://bcr.bazel.build/modules/rules_python/0.20.0/MODULE.bazel": "bfe14d17f20e3fe900b9588f526f52c967a6f281e47a1d6b988679bd15082286", + "https://bcr.bazel.build/modules/rules_python/0.22.0/MODULE.bazel": "b8057bafa11a9e0f4b08fc3b7cd7bee0dcbccea209ac6fc9a3ff051cd03e19e9", + "https://bcr.bazel.build/modules/rules_python/0.22.1/MODULE.bazel": "26114f0c0b5e93018c0c066d6673f1a2c3737c7e90af95eff30cfee38d0bbac7", "https://bcr.bazel.build/modules/rules_python/0.23.1/MODULE.bazel": "49ffccf0511cb8414de28321f5fcf2a31312b47c40cc21577144b7447f2bf300", "https://bcr.bazel.build/modules/rules_python/0.25.0/MODULE.bazel": "72f1506841c920a1afec76975b35312410eea3aa7b63267436bfb1dd91d2d382", - "https://bcr.bazel.build/modules/rules_python/0.28.0/MODULE.bazel": "cba2573d870babc976664a912539b320cbaa7114cd3e8f053c720171cde331ed", + "https://bcr.bazel.build/modules/rules_python/0.29.0/MODULE.bazel": "2ac8cd70524b4b9ec49a0b8284c79e4cd86199296f82f6e0d5da3f783d660c82", "https://bcr.bazel.build/modules/rules_python/0.31.0/MODULE.bazel": "93a43dc47ee570e6ec9f5779b2e64c1476a6ce921c48cc9a1678a91dd5f8fd58", + "https://bcr.bazel.build/modules/rules_python/0.32.2/MODULE.bazel": "01052470fc30b49de91fb8483d26bea6f664500cfad0b078d4605b03e3a83ed4", "https://bcr.bazel.build/modules/rules_python/0.33.2/MODULE.bazel": "3e036c4ad8d804a4dad897d333d8dce200d943df4827cb849840055be8d2e937", "https://bcr.bazel.build/modules/rules_python/0.34.0/MODULE.bazel": "1d623d026e075b78c9fde483a889cda7996f5da4f36dffb24c246ab30f06513a", + "https://bcr.bazel.build/modules/rules_python/0.37.1/MODULE.bazel": "3faeb2d9fa0a81f8980643ee33f212308f4d93eea4b9ce6f36d0b742e71e9500", "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", "https://bcr.bazel.build/modules/rules_python/0.40.0/MODULE.bazel": "9d1a3cd88ed7d8e39583d9ffe56ae8a244f67783ae89b60caafc9f5cf318ada7", "https://bcr.bazel.build/modules/rules_python/1.0.0/MODULE.bazel": "898a3d999c22caa585eb062b600f88654bf92efb204fa346fb55f6f8edffca43", @@ -227,7 +332,10 @@ "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", "https://bcr.bazel.build/modules/rules_shell/0.6.1/MODULE.bazel": "72e76b0eea4e81611ef5452aa82b3da34caca0c8b7b5c0c9584338aa93bae26b", "https://bcr.bazel.build/modules/rules_shell/0.6.1/source.json": "20ec05cd5e592055e214b2da8ccb283c7f2a421ea0dc2acbf1aa792e11c03d0c", - "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", + "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", + "https://bcr.bazel.build/modules/rules_swift/1.18.0/MODULE.bazel": "a6aba73625d0dc64c7b4a1e831549b6e375fbddb9d2dde9d80c9de6ec45b24c9", + "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", + "https://bcr.bazel.build/modules/rules_swift/2.1.1/source.json": "40fc69dfaac64deddbb75bd99cdac55f4427d9ca0afbe408576a65428427a186", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", "https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4", "https://bcr.bazel.build/modules/stardoc/0.5.6/MODULE.bazel": "c43dabc564990eeab55e25ed61c07a1aadafe9ece96a4efabb3f8bf9063b71ef", @@ -236,25 +344,34 @@ "https://bcr.bazel.build/modules/stardoc/0.7.1/MODULE.bazel": "3548faea4ee5dda5580f9af150e79d0f6aea934fc60c1cc50f4efdd9420759e7", "https://bcr.bazel.build/modules/stardoc/0.7.2/MODULE.bazel": "fc152419aa2ea0f51c29583fab1e8c99ddefd5b3778421845606ee628629e0e5", "https://bcr.bazel.build/modules/stardoc/0.7.2/source.json": "58b029e5e901d6802967754adf0a9056747e8176f017cfe3607c0851f4d42216", + "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "5e463fbfba7b1701d957555ed45097d7f984211330106ccd1352c6e0af0dcf91", + "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/source.json": "32bd87e5f4d7acc57c5b2ff7c325ae3061d5e242c0c4c214ae87e0f1c13e54cb", "https://bcr.bazel.build/modules/tar.bzl/0.2.1/MODULE.bazel": "52d1c00a80a8cc67acbd01649e83d8dd6a9dc426a6c0b754a04fe8c219c76468", "https://bcr.bazel.build/modules/tar.bzl/0.5.1/MODULE.bazel": "7c2eb3dcfc53b0f3d6f9acdfd911ca803eaf92aadf54f8ca6e4c1f3aee288351", "https://bcr.bazel.build/modules/tar.bzl/0.5.5/MODULE.bazel": "4bfab9bbc7a1966c2c5f7371f5848f5e2d27c465951b4435adc9aaf00ed681da", "https://bcr.bazel.build/modules/tar.bzl/0.6.0/MODULE.bazel": "a3584b4edcfafcabd9b0ef9819808f05b372957bbdff41601429d5fd0aac2e7c", "https://bcr.bazel.build/modules/tar.bzl/0.9.0/MODULE.bazel": "452a22d7f02b1c9d7a22ab25edf20f46f3e1101f0f67dc4bfbf9a474ddf02445", "https://bcr.bazel.build/modules/tar.bzl/0.9.0/source.json": "c732760a374831a2cf5b08839e4be75017196b4d796a5aa55235272ee17cd839", - "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", + "https://bcr.bazel.build/modules/upb/0.0.0-20211020-160625a/MODULE.bazel": "6cced416be2dc5b9c05efd5b997049ba795e5e4e6fafbe1624f4587767638928", "https://bcr.bazel.build/modules/upb/0.0.0-20230516-61a97ef/MODULE.bazel": "c0df5e35ad55e264160417fd0875932ee3c9dda63d9fccace35ac62f45e1b6f9", + "https://bcr.bazel.build/modules/upb/0.0.0-20230907-e7430e6/MODULE.bazel": "3a7dedadf70346e678dc059dbe44d05cbf3ab17f1ce43a1c7a42edc7cbf93fd9", "https://bcr.bazel.build/modules/with_cfg.bzl/0.12.0/MODULE.bazel": "b573395fe63aef4299ba095173e2f62ccfee5ad9bbf7acaa95dba73af9fc2b38", "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.1/MODULE.bazel": "aa0ef3f6c67dd35db7ac38c76785dc02f48b53f34e16b20487e70f51b5324d0a", "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.4/MODULE.bazel": "be2276f83878525d8f228b9ec777daee5eb7baba6100dad45081510d0cec35e3", "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.4/source.json": "63d661a0c144ca74fd8b0aa61e458c7627554c7d4a8e0a4d1ce00975437c776a", + "https://bcr.bazel.build/modules/xds/0.0.0-20240423-555b57e/MODULE.bazel": "cea509976a77e34131411684ef05a1d6ad194dd71a8d5816643bc5b0af16dc0f", + "https://bcr.bazel.build/modules/xds/0.0.0-20240423-555b57e/source.json": "7227e1fcad55f3f3cab1a08691ecd753cb29cc6380a47bc650851be9f9ad6d20", "https://bcr.bazel.build/modules/yq.bzl/0.1.1/MODULE.bazel": "9039681f9bcb8958ee2c87ffc74bdafba9f4369096a2b5634b88abc0eaefa072", "https://bcr.bazel.build/modules/yq.bzl/0.3.1/MODULE.bazel": "9bcb7151b3cd4681b89d350530eaf7b45e32a44dda94843b8932b0cb1cd4594a", "https://bcr.bazel.build/modules/yq.bzl/0.3.1/source.json": "f0b0f204a2a6b0e34b4c9541efe8c04f2ef1af65948daa784eccea738b21dbd2", "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", + "https://bcr.bazel.build/modules/zlib/1.2.13/MODULE.bazel": "aa6deb1b83c18ffecd940c4119aff9567cd0a671d7bba756741cb2ef043a29d5", + "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.1/MODULE.bazel": "6a9fe6e3fc865715a7be9823ce694ceb01e364c35f7a846bf0d2b34762bc066b", + "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "af322bc08976524477c79d1e45e241b6efbeb918c497e8840b8ab116802dda79", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "eec517b5bbe5492629466e11dae908d043364302283de25581e3eb944326c4ca", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/source.json": "22bc55c47af97246cfc093d0acf683a7869377de362b5d1c552c2c2e16b7a806", - "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198" + "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198", + "https://bcr.bazel.build/modules/zlib/1.3/MODULE.bazel": "6a9c02f19a24dcedb05572b2381446e27c272cd383aed11d41d99da9e3167a72" }, "selectedYankedVersions": {}, "moduleExtensions": { @@ -359,6 +476,487 @@ ] } }, + "@@googleapis+//:extensions.bzl%switched_rules": { + "general": { + "bzlTransitiveDigest": "JLpojYeGHLdgnwSUuQWOrtkL9UMygBUUEFJZh7ASlmA=", + "usagesDigest": "66rLsrgz6YqDpU0pD+02mTGNxYxhb/of6Fh0ulHzjdE=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "com_google_googleapis_imports": { + "repoRuleId": "@@googleapis+//:repository_rules.bzl%switched_rules", + "attributes": { + "rules": { + "proto_library_with_info": [ + "", + "" + ], + "moved_proto_library": [ + "", + "" + ], + "java_proto_library": [ + "", + "" + ], + "java_grpc_library": [ + "", + "" + ], + "java_gapic_library": [ + "", + "" + ], + "java_gapic_test": [ + "", + "" + ], + "java_gapic_assembly_gradle_pkg": [ + "", + "" + ], + "py_proto_library": [ + "", + "" + ], + "py_grpc_library": [ + "", + "" + ], + "py_gapic_library": [ + "", + "" + ], + "py_test": [ + "", + "" + ], + "py_gapic_assembly_pkg": [ + "", + "" + ], + "py_import": [ + "", + "" + ], + "go_proto_library": [ + "", + "" + ], + "go_grpc_library": [ + "", + "" + ], + "go_library": [ + "", + "" + ], + "go_test": [ + "", + "" + ], + "go_gapic_library": [ + "", + "" + ], + "go_gapic_assembly_pkg": [ + "", + "" + ], + "cc_proto_library": [ + "", + "" + ], + "cc_grpc_library": [ + "", + "" + ], + "cc_gapic_library": [ + "", + "" + ], + "php_proto_library": [ + "", + "php_proto_library" + ], + "php_grpc_library": [ + "", + "php_grpc_library" + ], + "php_gapic_library": [ + "", + "php_gapic_library" + ], + "php_gapic_assembly_pkg": [ + "", + "php_gapic_assembly_pkg" + ], + "nodejs_gapic_library": [ + "", + "typescript_gapic_library" + ], + "nodejs_gapic_assembly_pkg": [ + "", + "typescript_gapic_assembly_pkg" + ], + "ruby_proto_library": [ + "", + "" + ], + "ruby_grpc_library": [ + "", + "" + ], + "ruby_ads_gapic_library": [ + "", + "" + ], + "ruby_cloud_gapic_library": [ + "", + "" + ], + "ruby_gapic_assembly_pkg": [ + "", + "" + ], + "csharp_proto_library": [ + "", + "" + ], + "csharp_grpc_library": [ + "", + "" + ], + "csharp_gapic_library": [ + "", + "" + ], + "csharp_gapic_assembly_pkg": [ + "", + "" + ] + } + } + } + }, + "recordedRepoMappingEntries": [] + } + }, + "@@rules_foreign_cc+//foreign_cc:extensions.bzl%tools": { + "general": { + "bzlTransitiveDigest": "b41JwymKkuHbovxSrZD6eawffiN8+iwKcSQZrLCkvbw=", + "usagesDigest": "9LXdVp01HkdYQT8gYPjYLO6VLVJHo9uFfxWaU1ymiRE=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "rules_foreign_cc_framework_toolchain_linux": { + "repoRuleId": "@@rules_foreign_cc+//foreign_cc/private/framework:toolchain.bzl%framework_toolchain_repository", + "attributes": { + "commands_src": "@rules_foreign_cc//foreign_cc/private/framework/toolchains:linux_commands.bzl", + "exec_compatible_with": [ + "@platforms//os:linux" + ] + } + }, + "rules_foreign_cc_framework_toolchain_freebsd": { + "repoRuleId": "@@rules_foreign_cc+//foreign_cc/private/framework:toolchain.bzl%framework_toolchain_repository", + "attributes": { + "commands_src": "@rules_foreign_cc//foreign_cc/private/framework/toolchains:freebsd_commands.bzl", + "exec_compatible_with": [ + "@platforms//os:freebsd" + ] + } + }, + "rules_foreign_cc_framework_toolchain_windows": { + "repoRuleId": "@@rules_foreign_cc+//foreign_cc/private/framework:toolchain.bzl%framework_toolchain_repository", + "attributes": { + "commands_src": "@rules_foreign_cc//foreign_cc/private/framework/toolchains:windows_commands.bzl", + "exec_compatible_with": [ + "@platforms//os:windows" + ] + } + }, + "rules_foreign_cc_framework_toolchain_macos": { + "repoRuleId": "@@rules_foreign_cc+//foreign_cc/private/framework:toolchain.bzl%framework_toolchain_repository", + "attributes": { + "commands_src": "@rules_foreign_cc//foreign_cc/private/framework/toolchains:macos_commands.bzl", + "exec_compatible_with": [ + "@platforms//os:macos" + ] + } + }, + "rules_foreign_cc_framework_toolchains": { + "repoRuleId": "@@rules_foreign_cc+//foreign_cc/private/framework:toolchain.bzl%framework_toolchain_repository_hub", + "attributes": {} + }, + "cmake_src": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file_content": "filegroup(\n name = \"all_srcs\",\n srcs = glob([\"**\"]),\n visibility = [\"//visibility:public\"],\n)\n", + "sha256": "f316b40053466f9a416adf981efda41b160ca859e97f6a484b447ea299ff26aa", + "strip_prefix": "cmake-3.23.2", + "urls": [ + "https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2.tar.gz" + ] + } + }, + "gnumake_src": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file_content": "filegroup(\n name = \"all_srcs\",\n srcs = glob([\"**\"]),\n visibility = [\"//visibility:public\"],\n)\n", + "sha256": "581f4d4e872da74b3941c874215898a7d35802f03732bdccee1d4a7979105d18", + "strip_prefix": "make-4.4", + "urls": [ + "https://mirror.bazel.build/ftpmirror.gnu.org/gnu/make/make-4.4.tar.gz", + "http://ftpmirror.gnu.org/gnu/make/make-4.4.tar.gz" + ] + } + }, + "ninja_build_src": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file_content": "filegroup(\n name = \"all_srcs\",\n srcs = glob([\"**\"]),\n visibility = [\"//visibility:public\"],\n)\n", + "sha256": "31747ae633213f1eda3842686f83c2aa1412e0f5691d1c14dbbcc67fe7400cea", + "strip_prefix": "ninja-1.11.1", + "urls": [ + "https://github.com/ninja-build/ninja/archive/v1.11.1.tar.gz" + ] + } + }, + "meson_src": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file_content": "exports_files([\"meson.py\"])\n\nfilegroup(\n name = \"runtime\",\n srcs = glob([\"mesonbuild/**\"]),\n visibility = [\"//visibility:public\"],\n)\n", + "strip_prefix": "meson-1.1.1", + "url": "https://github.com/mesonbuild/meson/releases/download/1.1.1/meson-1.1.1.tar.gz" + } + }, + "glib_dev": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file_content": "\nload(\"@rules_cc//cc:defs.bzl\", \"cc_library\")\n\ncc_import(\n name = \"glib_dev\",\n hdrs = glob([\"include/**\"]),\n shared_library = \"@glib_runtime//:bin/libglib-2.0-0.dll\",\n visibility = [\"//visibility:public\"],\n)\n ", + "sha256": "bdf18506df304d38be98a4b3f18055b8b8cca81beabecad0eece6ce95319c369", + "urls": [ + "https://download.gnome.org/binaries/win64/glib/2.26/glib-dev_2.26.1-1_win64.zip" + ] + } + }, + "glib_src": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file_content": "\ncc_import(\n name = \"msvc_hdr\",\n hdrs = [\"msvc_recommended_pragmas.h\"],\n visibility = [\"//visibility:public\"],\n)\n ", + "sha256": "bc96f63112823b7d6c9f06572d2ad626ddac7eb452c04d762592197f6e07898e", + "strip_prefix": "glib-2.26.1", + "urls": [ + "https://download.gnome.org/sources/glib/2.26/glib-2.26.1.tar.gz" + ] + } + }, + "glib_runtime": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file_content": "\nexports_files(\n [\n \"bin/libgio-2.0-0.dll\",\n \"bin/libglib-2.0-0.dll\",\n \"bin/libgmodule-2.0-0.dll\",\n \"bin/libgobject-2.0-0.dll\",\n \"bin/libgthread-2.0-0.dll\",\n ],\n visibility = [\"//visibility:public\"],\n)\n ", + "sha256": "88d857087e86f16a9be651ee7021880b3f7ba050d34a1ed9f06113b8799cb973", + "urls": [ + "https://download.gnome.org/binaries/win64/glib/2.26/glib_2.26.1-1_win64.zip" + ] + } + }, + "gettext_runtime": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file_content": "\ncc_import(\n name = \"gettext_runtime\",\n shared_library = \"bin/libintl-8.dll\",\n visibility = [\"//visibility:public\"],\n)\n ", + "sha256": "1f4269c0e021076d60a54e98da6f978a3195013f6de21674ba0edbc339c5b079", + "urls": [ + "https://download.gnome.org/binaries/win64/dependencies/gettext-runtime_0.18.1.1-2_win64.zip" + ] + } + }, + "pkgconfig_src": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file_content": "filegroup(\n name = \"all_srcs\",\n srcs = glob([\"**\"]),\n visibility = [\"//visibility:public\"],\n)\n", + "sha256": "6fc69c01688c9458a57eb9a1664c9aba372ccda420a02bf4429fe610e7e7d591", + "strip_prefix": "pkg-config-0.29.2", + "patches": [ + "@@rules_foreign_cc+//toolchains:pkgconfig-detectenv.patch", + "@@rules_foreign_cc+//toolchains:pkgconfig-makefile-vc.patch" + ], + "urls": [ + "https://pkgconfig.freedesktop.org/releases/pkg-config-0.29.2.tar.gz" + ] + } + }, + "bazel_skylib": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.2.1/bazel-skylib-1.2.1.tar.gz", + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.2.1/bazel-skylib-1.2.1.tar.gz" + ], + "sha256": "f7be3474d42aae265405a592bb7da8e171919d74c16f082a5457840f06054728" + } + }, + "rules_python": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "sha256": "84aec9e21cc56fbc7f1335035a71c850d1b9b5cc6ff497306f84cced9a769841", + "strip_prefix": "rules_python-0.23.1", + "url": "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.23.1.tar.gz" + } + }, + "cmake-3.23.2-linux-aarch64": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-linux-aarch64.tar.gz" + ], + "sha256": "f2654bf780b53f170bbbec44d8ac67d401d24788e590faa53036a89476efa91e", + "strip_prefix": "cmake-3.23.2-linux-aarch64", + "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"cmake_data\",\n srcs = glob(\n [\n \"**\",\n ],\n exclude = [\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n \"BUILD\",\n \"BUILD.bazel\",\n ],\n ),\n)\n\nnative_tool_toolchain(\n name = \"cmake_tool\",\n path = \"bin/cmake\",\n target = \":cmake_data\",\n)\n" + } + }, + "cmake-3.23.2-linux-x86_64": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-linux-x86_64.tar.gz" + ], + "sha256": "aaced6f745b86ce853661a595bdac6c5314a60f8181b6912a0a4920acfa32708", + "strip_prefix": "cmake-3.23.2-linux-x86_64", + "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"cmake_data\",\n srcs = glob(\n [\n \"**\",\n ],\n exclude = [\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n \"BUILD\",\n \"BUILD.bazel\",\n ],\n ),\n)\n\nnative_tool_toolchain(\n name = \"cmake_tool\",\n path = \"bin/cmake\",\n target = \":cmake_data\",\n)\n" + } + }, + "cmake-3.23.2-macos-universal": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-macos-universal.tar.gz" + ], + "sha256": "853a0f9af148c5ef47282ffffee06c4c9f257be2635936755f39ca13c3286c88", + "strip_prefix": "cmake-3.23.2-macos-universal/CMake.app/Contents", + "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"cmake_data\",\n srcs = glob(\n [\n \"**\",\n ],\n exclude = [\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n \"BUILD\",\n \"BUILD.bazel\",\n ],\n ),\n)\n\nnative_tool_toolchain(\n name = \"cmake_tool\",\n path = \"bin/cmake\",\n target = \":cmake_data\",\n)\n" + } + }, + "cmake-3.23.2-windows-i386": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-windows-i386.zip" + ], + "sha256": "6a4fcd6a2315b93cb23c93507efccacc30c449c2bf98f14d6032bb226c582e07", + "strip_prefix": "cmake-3.23.2-windows-i386", + "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"cmake_data\",\n srcs = glob(\n [\n \"**\",\n ],\n exclude = [\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n \"BUILD\",\n \"BUILD.bazel\",\n ],\n ),\n)\n\nnative_tool_toolchain(\n name = \"cmake_tool\",\n path = \"bin/cmake.exe\",\n target = \":cmake_data\",\n)\n" + } + }, + "cmake-3.23.2-windows-x86_64": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-windows-x86_64.zip" + ], + "sha256": "2329387f3166b84c25091c86389fb891193967740c9bcf01e7f6d3306f7ffda0", + "strip_prefix": "cmake-3.23.2-windows-x86_64", + "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"cmake_data\",\n srcs = glob(\n [\n \"**\",\n ],\n exclude = [\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n \"BUILD\",\n \"BUILD.bazel\",\n ],\n ),\n)\n\nnative_tool_toolchain(\n name = \"cmake_tool\",\n path = \"bin/cmake.exe\",\n target = \":cmake_data\",\n)\n" + } + }, + "cmake_3.23.2_toolchains": { + "repoRuleId": "@@rules_foreign_cc+//toolchains:prebuilt_toolchains_repository.bzl%prebuilt_toolchains_repository", + "attributes": { + "repos": { + "cmake-3.23.2-linux-aarch64": [ + "@platforms//cpu:aarch64", + "@platforms//os:linux" + ], + "cmake-3.23.2-linux-x86_64": [ + "@platforms//cpu:x86_64", + "@platforms//os:linux" + ], + "cmake-3.23.2-macos-universal": [ + "@platforms//os:macos" + ], + "cmake-3.23.2-windows-i386": [ + "@platforms//cpu:x86_32", + "@platforms//os:windows" + ], + "cmake-3.23.2-windows-x86_64": [ + "@platforms//cpu:x86_64", + "@platforms//os:windows" + ] + }, + "tool": "cmake" + } + }, + "ninja_1.11.1_linux": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-linux.zip" + ], + "sha256": "b901ba96e486dce377f9a070ed4ef3f79deb45f4ffe2938f8e7ddc69cfb3df77", + "strip_prefix": "", + "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"ninja_bin\",\n srcs = [\"ninja\"],\n)\n\nnative_tool_toolchain(\n name = \"ninja_tool\",\n env = {\"NINJA\": \"$(execpath :ninja_bin)\"},\n path = \"$(execpath :ninja_bin)\",\n target = \":ninja_bin\",\n)\n" + } + }, + "ninja_1.11.1_mac": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-mac.zip" + ], + "sha256": "482ecb23c59ae3d4f158029112de172dd96bb0e97549c4b1ca32d8fad11f873e", + "strip_prefix": "", + "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"ninja_bin\",\n srcs = [\"ninja\"],\n)\n\nnative_tool_toolchain(\n name = \"ninja_tool\",\n env = {\"NINJA\": \"$(execpath :ninja_bin)\"},\n path = \"$(execpath :ninja_bin)\",\n target = \":ninja_bin\",\n)\n" + } + }, + "ninja_1.11.1_win": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-win.zip" + ], + "sha256": "524b344a1a9a55005eaf868d991e090ab8ce07fa109f1820d40e74642e289abc", + "strip_prefix": "", + "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"ninja_bin\",\n srcs = [\"ninja.exe\"],\n)\n\nnative_tool_toolchain(\n name = \"ninja_tool\",\n env = {\"NINJA\": \"$(execpath :ninja_bin)\"},\n path = \"$(execpath :ninja_bin)\",\n target = \":ninja_bin\",\n)\n" + } + }, + "ninja_1.11.1_toolchains": { + "repoRuleId": "@@rules_foreign_cc+//toolchains:prebuilt_toolchains_repository.bzl%prebuilt_toolchains_repository", + "attributes": { + "repos": { + "ninja_1.11.1_linux": [ + "@platforms//cpu:x86_64", + "@platforms//os:linux" + ], + "ninja_1.11.1_mac": [ + "@platforms//cpu:x86_64", + "@platforms//os:macos" + ], + "ninja_1.11.1_win": [ + "@platforms//cpu:x86_64", + "@platforms//os:windows" + ] + }, + "tool": "ninja" + } + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_foreign_cc+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_foreign_cc+", + "rules_foreign_cc", + "rules_foreign_cc+" + ] + ] + } + }, "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { "general": { "bzlTransitiveDigest": "AEsQvKwdstptzt98AO1bJYfoWpPUn2upx53ra6o/Rc8=", diff --git a/bin/zml-smi/BUILD.bazel b/bin/zml-smi/BUILD.bazel index 5e29d6af..84651d34 100644 --- a/bin/zml-smi/BUILD.bazel +++ b/bin/zml-smi/BUILD.bazel @@ -128,6 +128,7 @@ zig_binary( mtree_spec( name = "mtree_spec", srcs = [":zml-smi"], + tags = ["manual"], ) # Use this genrule until mtree_mutate is fixed @@ -136,6 +137,7 @@ genrule( srcs = [":mtree_spec"], outs = ["mtree.txt"], cmd = """sed -e '/type=dir/d' -e 's#^bin/##g' $< > $@""", + tags = ["manual"], ) # mtree_mutate( @@ -156,6 +158,7 @@ tar( ], compress = "zstd", mtree = ":mtree", + tags = ["manual"], ) oci_image( @@ -165,6 +168,7 @@ oci_image( tars = [ ":archive", ], + tags = ["manual"], ) platform_transition_filegroup( diff --git a/examples/mnist/BUILD.bazel b/examples/mnist/BUILD.bazel index 89fdbaab..0b996ea7 100644 --- a/examples/mnist/BUILD.bazel +++ b/examples/mnist/BUILD.bazel @@ -26,6 +26,7 @@ zig_binary( mtree_spec( name = "mtree", srcs = [":mnist"], + tags = ["manual"], ) tar( @@ -37,6 +38,7 @@ tar( ], compress = "zstd", mtree = ":mtree", + tags = ["manual"], ) expand_template( @@ -55,6 +57,7 @@ expand_template( "/{}/mnist.runfiles/:model".format(package_name()), "/{}/mnist.runfiles/:data".format(package_name()), ], + tags = ["manual"], ) # The actual docker image, with entrypoint, created from tar archive @@ -63,6 +66,7 @@ oci_image( base = "@distroless_cc_debian12", entrypoint = ":entrypoint", tars = [":archive"], + tags = ["manual"], ) # We always want to create the image for Linux @@ -70,6 +74,7 @@ platform_transition_filegroup( name = "image", srcs = [":image_"], target_platform = "@zml//platforms:linux_amd64", + tags = ["manual"], ) # Load will immediately load the image (eg: docker load) @@ -79,6 +84,7 @@ oci_load( repo_tags = [ "distroless/mnist:latest", ], + tags = ["manual"], ) # Bazel target for pushing the Linux image to our docker registry @@ -88,4 +94,5 @@ oci_push( remote_tags = ["latest"], # override with -- --repository foo.bar/org/image repository = "index.docker.io/steeve/mnist", + tags = ["manual"], ) From c6931ba2be0c974e67e643e3d38ee67d7c697315 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 3 Apr 2026 09:18:40 +0000 Subject: [PATCH 400/881] bazel/patchelf: use --force-rpath in rpath ops This allows setting DT_RPATH instead of DT_RUNPATH. DT_RPATH takes precendence to LD_LIBRARY_PATH. Sometimes LD_LIBRARY_PATH is useful, sometimes it's a pain. Using this enables us at least to precede it. --- bazel/patchelf.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bazel/patchelf.bzl b/bazel/patchelf.bzl index ac79033c..50691b82 100644 --- a/bazel/patchelf.bzl +++ b/bazel/patchelf.bzl @@ -23,10 +23,10 @@ def _patchelf_impl(ctx): commands.append(""" "$1" --replace-needed '{}' '{}' "$3" """.format(k, v)) if ctx.attr.set_rpath: - commands.append(""" "$1" --set-rpath '{}' "$3" """.format(ctx.attr.set_rpath)) + commands.append(""" "$1" --set-rpath '{}' --force-rpath "$3" """.format(ctx.attr.set_rpath)) if ctx.attr.add_rpath: for path in ctx.attr.add_rpath: - commands.append(""" "$1" --add-rpath '{}' "$3" """.format(path)) + commands.append(""" "$1" --add-rpath '{}' --force-rpath "$3" """.format(path)) if ctx.attr.remove_rpath: for path in ctx.attr.remove_rpath: commands.append(""" "$1" --remove-rpath '{}' "$3" """.format(path)) From 333ec93c4774cfca7ed00f6172880cfd6908d464 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20Gauti=C3=A9?= <51361266+tgautie@users.noreply.github.com> Date: Fri, 3 Apr 2026 13:11:45 +0200 Subject: [PATCH 401/881] Fix slice lifetime issue (#451) We fix a small slice lifetime issue in the prompt template of the Qwen3.5 model in LLM CLI --- examples/llm/models/qwen3_5/session.zig | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/examples/llm/models/qwen3_5/session.zig b/examples/llm/models/qwen3_5/session.zig index acb8afd4..3a1bc8b6 100644 --- a/examples/llm/models/qwen3_5/session.zig +++ b/examples/llm/models/qwen3_5/session.zig @@ -172,23 +172,18 @@ fn tokenizeChatPrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tok const im_start = tokenizer.tokenToId("<|im_start|>") orelse special_tokens.im_start_token_id; const im_end = tokenizer.tokenToId("<|im_end|>") orelse special_tokens.im_end_token_id; - const think = tokenizer.tokenToId("") orelse return error.NoSuchToken; const newline = try encodeSingleToken(&encoder, "\n"); - const user_prefix = try encoder.encode("user\n"); - const assistant_prefix = try encoder.encode("assistant\n"); - const encoded_prompt = try encoder.encode(prompt); - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, encoded_prompt.len + user_prefix.len + assistant_prefix.len + 8); + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, 32); if (!is_first_turn) { try tokens.appendSlice(allocator, &.{ im_end, newline }); } try tokens.append(allocator, im_start); - try tokens.appendSlice(allocator, user_prefix); - try tokens.appendSlice(allocator, encoded_prompt); + try tokens.appendSlice(allocator, try encoder.encode("user\n")); + try tokens.appendSlice(allocator, try encoder.encode(prompt)); try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); - try tokens.appendSlice(allocator, assistant_prefix); - try tokens.appendSlice(allocator, &.{ think, newline }); + try tokens.appendSlice(allocator, try encoder.encode("assistant\n")); return tokens.toOwnedSlice(allocator); } From e69508f3643b39ac26c80cfc6d1d89ed38686708 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 3 Apr 2026 09:20:25 +0000 Subject: [PATCH 402/881] zml/io/vfs/s3: don't treat 404 as Unexpected It's completely normal to stat a missing file. --- zml/io/vfs/s3.zig | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/zml/io/vfs/s3.zig b/zml/io/vfs/s3.zig index 193009e0..37953ef4 100644 --- a/zml/io/vfs/s3.zig +++ b/zml/io/vfs/s3.zig @@ -396,7 +396,11 @@ pub const S3 = struct { fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const size = self.fetchSize(dir, sub_path) catch return std.Io.Dir.StatFileError.Unexpected; + const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { + error.FileNotFound => return std.Io.File.OpenError.FileNotFound, + error.BadPathName => return std.Io.File.OpenError.BadPathName, + else => return std.Io.File.OpenError.Unexpected, + }; return .{ .inode = 0, @@ -415,7 +419,11 @@ pub const S3 = struct { fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const size = self.fetchSize(dir, sub_path) catch return std.Io.File.OpenError.Unexpected; + const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { + error.FileNotFound => return std.Io.File.OpenError.FileNotFound, + error.BadPathName => return std.Io.File.OpenError.BadPathName, + else => return std.Io.File.OpenError.Unexpected, + }; var path_buffer: [8 * 1024]u8 = undefined; const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; @@ -744,13 +752,16 @@ pub const S3 = struct { var redirect_buffer: [8 * 1024]u8 = undefined; const res = try req.receiveHead(&redirect_buffer); - switch (res.head.status.class()) { - .success => return res.head.content_length.?, - else => { - log.err("Failed to fetch size for {s}: {s}", .{ url, res.head.bytes }); - return error.ServerError; + return switch (res.head.status.class()) { + .success => res.head.content_length.?, + else => switch (res.head.status) { + .not_found => return error.FileNotFound, + else => blk: { + log.err("Failed to fetch size for {s}: {s}", .{ url, res.head.bytes }); + break :blk error.ServerError; + }, }, - } + }; } fn performRead(self: *S3, handle: *Handle, data: []const []u8, offset: u64) !usize { From 96414d29c4df764e548f92e0cc98c24770e34b10 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 3 Apr 2026 12:55:13 +0000 Subject: [PATCH 403/881] workspace: add stamping and helpers --- .bazelrc | 4 ++++ bazel/stamp/BUILD.bazel | 26 ++++++++++++++++++++++++++ bazel/stamp/stamp.zig | 5 +++++ tools/workspace_status.sh | 6 ++++++ 4 files changed, 41 insertions(+) create mode 100644 bazel/stamp/BUILD.bazel create mode 100644 bazel/stamp/stamp.zig create mode 100755 tools/workspace_status.sh diff --git a/.bazelrc b/.bazelrc index 52406f5b..cda51e23 100644 --- a/.bazelrc +++ b/.bazelrc @@ -81,6 +81,10 @@ common --@llvm//config:experimental_stub_libgcc_s=true common --macos_minimum_os=13.0 +# Enable stamping +common --stamp=true +common --workspace_status_command=./tools/workspace_status.sh + common --@rules_cc//cc/toolchains/args/archiver_flags:use_libtool_on_macos=false common --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 common --repo_env=BAZEL_NO_APPLE_CPP_TOOLCHAIN=1 diff --git a/bazel/stamp/BUILD.bazel b/bazel/stamp/BUILD.bazel new file mode 100644 index 00000000..80827efc --- /dev/null +++ b/bazel/stamp/BUILD.bazel @@ -0,0 +1,26 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library") + +genrule( + name = "asm_src", + outs = ["stamp.S"], + cmd = """\ +echo ".section .rodata" > $(@) +cat bazel-out/stable-status.txt bazel-out/volatile-status.txt | sed -Ee's/^(\\w+) (.*)/.globl \\1\\n\\1:\\n .asciz "\\2"/g' >> $(@) +""", + stamp = -1, +) + +cc_library( + name = "asm", + srcs = [":asm_src"], + visibility = ["//visibility:public"], +) + +zig_library( + name = "zig", + import_name = "bazel/stamp", + main = "stamp.zig", + deps = [":asm"], + visibility = ["//visibility:public"], +) diff --git a/bazel/stamp/stamp.zig b/bazel/stamp/stamp.zig new file mode 100644 index 00000000..0e60e2c7 --- /dev/null +++ b/bazel/stamp/stamp.zig @@ -0,0 +1,5 @@ +const std = @import("std"); + +pub fn stamp(comptime name: []const u8) []const u8 { + return std.mem.span(@extern([*c]const u8, .{ .name = name })); +} diff --git a/tools/workspace_status.sh b/tools/workspace_status.sh new file mode 100755 index 00000000..3af7cd3b --- /dev/null +++ b/tools/workspace_status.sh @@ -0,0 +1,6 @@ +#!/bin/bash +if git diff --quiet; then + echo BUILD_SCM_REVISION "$(git rev-parse --short HEAD)" +else + echo BUILD_SCM_REVISION "dev" +fi From 1a9e66c1c3f010f822a7b4243dcef7f2790690c5 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 3 Apr 2026 15:53:54 +0200 Subject: [PATCH 404/881] bazel/stamp: generate .c file and replace genrule by run_binary+tool This makes it remote build compatible and cross platform compatible. --- bazel/stamp/BUILD.bazel | 33 +++++++---- bazel/stamp/genstamp.c | 121 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 11 deletions(-) create mode 100644 bazel/stamp/genstamp.c diff --git a/bazel/stamp/BUILD.bazel b/bazel/stamp/BUILD.bazel index 80827efc..387f189c 100644 --- a/bazel/stamp/BUILD.bazel +++ b/bazel/stamp/BUILD.bazel @@ -1,19 +1,30 @@ +load("@bazel_lib//lib:run_binary.bzl", "run_binary") +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library") -genrule( - name = "asm_src", - outs = ["stamp.S"], - cmd = """\ -echo ".section .rodata" > $(@) -cat bazel-out/stable-status.txt bazel-out/volatile-status.txt | sed -Ee's/^(\\w+) (.*)/.globl \\1\\n\\1:\\n .asciz "\\2"/g' >> $(@) -""", - stamp = -1, +run_binary( + name = "stamp_cc_src", + outs = ["stamp.c"], + args = [ + "$(location stamp.c)", + "bazel-out/stable-status.txt", + "bazel-out/volatile-status.txt", + ], + stamp = 1, + tool = ":genstamp", + visibility = ["//visibility:private"], +) + +cc_binary( + name = "genstamp", + srcs = ["genstamp.c"], + visibility = ["//visibility:private"], ) cc_library( - name = "asm", - srcs = [":asm_src"], + name = "cc", + srcs = [":stamp_cc_src"], visibility = ["//visibility:public"], ) @@ -21,6 +32,6 @@ zig_library( name = "zig", import_name = "bazel/stamp", main = "stamp.zig", - deps = [":asm"], visibility = ["//visibility:public"], + deps = [":cc"], ) diff --git a/bazel/stamp/genstamp.c b/bazel/stamp/genstamp.c new file mode 100644 index 00000000..4fba8e6f --- /dev/null +++ b/bazel/stamp/genstamp.c @@ -0,0 +1,121 @@ +#include +#include +#include + +static void write_escaped(FILE *out, const char *value) { + for (const unsigned char *p = (const unsigned char *)value; *p != '\0'; ++p) { + switch (*p) { + case '\\': + fputs("\\\\", out); + break; + case '"': + fputs("\\\"", out); + break; + case '\n': + fputs("\\n", out); + break; + case '\r': + fputs("\\r", out); + break; + case '\t': + fputs("\\t", out); + break; + default: + fputc(*p, out); + break; + } + } +} + +static int process_stream(FILE *out, FILE *in, const char *path) { + char *line = NULL; + size_t cap = 0; + ssize_t len; + + while ((len = getline(&line, &cap, in)) != -1) { + while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r')) { + line[--len] = '\0'; + } + + if (len == 0) { + continue; + } + + char *space = strchr(line, ' '); + if (space == NULL) { + continue; + } + + *space = '\0'; + const char *name = line; + const char *value = space + 1; + + if (*name == '\0') { + continue; + } + + fputs("const char ", out); + fputs(name, out); + fputs("[] = \"", out); + write_escaped(out, value); + fputs("\";\n", out); + } + + if (ferror(in)) { + fprintf(stderr, "genstamp: failed while reading %s\n", path); + free(line); + return 1; + } + + free(line); + return 0; +} + +int main(int argc, char **argv) { + if (argc < 3) { + fprintf(stderr, "usage: %s [ ...]\n", argv[0]); + return 1; + } + + const char *output_path = argv[1]; + FILE *out = fopen(output_path, "w"); + if (out == NULL) { + fprintf(stderr, "genstamp: failed to open %s for writing\n", output_path); + return 1; + } + + for (int i = 2; i < argc; ++i) { + const char *path = argv[i]; + FILE *in = fopen(path, "r"); + if (in == NULL) { + fprintf(stderr, "genstamp: failed to open %s\n", path); + fclose(out); + return 1; + } + + int rc = process_stream(out, in, path); + if (fclose(in) != 0) { + fprintf(stderr, "genstamp: failed to close %s\n", path); + fclose(out); + return 1; + } + + if (rc != 0) { + fclose(out); + return rc; + } + } + + if (fflush(out) != 0) { + fprintf(stderr, "genstamp: failed to flush %s\n", output_path); + fclose(out); + return 1; + } + + if (fclose(out) != 0) { + fprintf(stderr, "genstamp: failed to close %s\n", output_path); + return 1; + } + + return 0; +} \ No newline at end of file From 70c5d1c5883813e13481cf35fab841f8a18152aa Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 7 Apr 2026 15:17:33 +0200 Subject: [PATCH 405/881] zml/ops: Stronger typing of custom calls and more code re-use beyond flashattn (#454) Today, creating a custom call is a bit tedious and error prone. The primary goal of this PR is to only accept structs as input/output/attributes and rely more strongly on the names everywhere when creating as custom call. I've migrated the `fa2` custom to provide an example. Can migrate the rest. - Custom call relies on the position of the input/outputs/attributes to be in the correct order to work. The `output_operand_aliases` also maps the outputs to the input field position. - If the custom call is "sharding aware", as in it knows it works on a single partition on multi devices setup, one must use `zml.ops.manualComputation` which is a bit tedious to set up. - `flashattn.zig` has a bunch of logic to register said custom call but also to map inputs into something more usable which can be shared. - both `zml.ops.customCall` and `zml.ops.manualComputation` tend to accept to many different types IMHO which makes it a bit hard to follow. But well, to be fair not sure how much simpler my code is. I do constrain myself to only accepting simple structs for the generic types. I've re-used the existing `manualComputationInternal` and `customCallInteral` for this new API without touching them. The only exception is the `output_operand_alises` where I needed a bit more flexibility. --- mlir/dialects/stablehlo/stablehlo.zig | 13 +- zml/attention/flashattn.zig | 162 ++++++----- zml/ops.zig | 392 +++++++++++++++++++++++++- zml/pjrtx.zig | 25 ++ 4 files changed, 509 insertions(+), 83 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 51edfddf..4cbc9187 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -733,13 +733,18 @@ pub const CustomCallOpts = struct { typed_ffi: *const mlir.Attribute, }; + pub const OutputOperandAlias = struct { + output_index: i64, + operand_index: i64, + }; + call_target_name: []const u8, has_side_effect: ?bool = null, api_version: ?ApiVersion = null, backend_config: ?BackendConfig = null, operand_layouts: ?[]const []const usize = null, result_layouts: ?[]const []const usize = null, - output_operand_aliases: ?[]const i64 = null, + output_operand_aliases: ?[]const OutputOperandAlias = null, additional_attributes: []const mlir.NamedAttribute = &.{}, }; @@ -783,10 +788,10 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result if (opts.output_operand_aliases) |output_operand_aliases| { var buffer: stdx.BoundedArray(*const mlir.Attribute, MAX_RESULTS) = .{}; - for (output_operand_aliases, 0..) |alias, output_index| { - const output_tuple_indices = if (result_types.len > 1) &[1]i64{@intCast(output_index)} else &.{}; + for (output_operand_aliases) |alias| { + const output_tuple_indices = if (result_types.len > 1) &[1]i64{alias.output_index} else &.{}; buffer.appendAssumeCapacity( - outputOperandAliasAttribute(ctx, .{ .operand_index = alias, .output_tuple_indices = output_tuple_indices }), + outputOperandAliasAttribute(ctx, .{ .operand_index = alias.operand_index, .output_tuple_indices = output_tuple_indices }), ); } attrs.appendAssumeCapacity( diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 74aa5dc0..8bfc800d 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -19,8 +19,8 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { pub fn register(platform: *const zml.Platform) !void { if (comptime platforms.isEnabled(.cuda)) { - try fa2.register(platform); try fa3.register(platform); + try fa2.fa2_mha_varlen_fwd.register(platform); try paged_fa2.Decode.register(platform); try paged_fa2.Prefill.register(platform); try paged_fa3.Decode.register(platform); @@ -39,6 +39,15 @@ fn flashattnDataTypeFromZmlDataType(dtype: zml.DataType) flashattn.DataType { }; } +fn toFlashattnTensor(buffer: zml.pjrtx.CustomCallBuffer) flashattn.Tensor { + return .init( + buffer.ptr, + buffer.shape.dims(), + buffer.shape.withDtype(.u8).computeByteStrides().constSlice(), + flashattnDataTypeFromZmlDataType(buffer.shape.dtype()), + ); +} + const Buffer = struct { shape: zml.Shape, ptr: *anyopaque, @@ -145,63 +154,63 @@ pub fn Wrapper(comptime T: type, run_func: std.meta.DeclEnum(T)) type { } pub const fa2 = struct { - const custom_call_name = "fa2_mha_varlen_fwd"; - const Wrapped = Wrapper(@This(), .runInner); + const Input = struct { + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + cu_seqlens_q: zml.Tensor, + cu_seqlens_k: zml.Tensor, + softmax_lse: zml.Tensor, + softmax_lse_accum: zml.Tensor, + out_accum: zml.Tensor, + }; - const register = Wrapped.register; - const run = Wrapped.run; + const Output = struct { + o: zml.Shape, + }; - pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { - if (call_frame.registeringHook()) return null; - - const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); - const k = bufferFromFfiBuffer(call_frame.args.buffers()[1]); - const v = bufferFromFfiBuffer(call_frame.args.buffers()[2]); - const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); - const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); - const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[5]); - const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[6]); - const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); - const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); - - const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { - const head_dim = q.shape.dim(2); - break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); - }; - const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; - const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; - const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; - const max_seqlen_q: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_q").?; - const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; - const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; - - const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); - const stream = call_frame.api.stream(ctx); + const Attributes = struct { + softmax_scale: f32, + is_causal: bool, + window_size_left: i32, + window_size_right: i32, + max_seqlen_q: i32, + max_seqlen_k: i32, + num_heads: i32, + }; + fn ffiCall( + call_frame: *zml.pjrt.ffi.CallFrame, + input: zml.ops.TensorToCustomCallBuffer(Input), + output: zml.ops.ShapeToCustomCallBuffer(Output), + attributes: Attributes, + ) !?*zml.pjrt.ffi.Error { const params: flashattn.FA2MhaVarlenFwdParams = .{ - .max_seqlen_q = max_seqlen_q, - .max_seqlen_k = max_seqlen_k, - .is_causal = is_causal, - .softmax_scale = softmax_scale, - .window_size_left = window_size_left, - .window_size_right = window_size_right, + .max_seqlen_q = attributes.max_seqlen_q, + .max_seqlen_k = attributes.max_seqlen_k, + .is_causal = attributes.is_causal, + .softmax_scale = attributes.softmax_scale, + .window_size_left = attributes.window_size_left, + .window_size_right = attributes.window_size_right, .num_splits = 0, - .num_heads = num_heads, + .num_heads = attributes.num_heads, }; + const stream = call_frame.api.stream(call_frame.ctx); + flashattn.fa2_mha_varlen_fwd( - &q.toFlashattnTensor(), - &k.toFlashattnTensor(), - &v.toFlashattnTensor(), - &o.toFlashattnTensor(), - &cu_seqlens_q.toFlashattnTensor(), - &cu_seqlens_k.toFlashattnTensor(), + &toFlashattnTensor(input.q), + &toFlashattnTensor(input.k), + &toFlashattnTensor(input.v), + &toFlashattnTensor(output.o), + &toFlashattnTensor(input.cu_seqlens_q), + &toFlashattnTensor(input.cu_seqlens_k), null, null, - &softmax_lse.toFlashattnTensor(), + &toFlashattnTensor(input.softmax_lse), null, - &softmax_lse_accum.toFlashattnTensor(), - &out_accum.toFlashattnTensor(), + &toFlashattnTensor(input.softmax_lse_accum), + &toFlashattnTensor(input.out_accum), ¶ms, stream, ); @@ -209,6 +218,13 @@ pub const fa2 = struct { return null; } + const fa2_mha_varlen_fwd = zml.ops.CustomCall(Input, Output, Attributes, ffiCall, .{ + .name = "fa2_mha_varlen_fwd", + .sharding_aware = false, + .has_side_effect = false, + .output_operand_aliases = .{ .o = .q }, + }); + pub const Parameters = struct { pub const InitOptions = struct {}; @@ -294,40 +310,34 @@ pub const fa2 = struct { const q_sharded = q.withPartitioning(.{ .h = .model }); const model_partitions = ctx.partitioning.numPartitionsForLogicalAxis(q_sharded.shape(), .model) catch unreachable; - const output_shape = q_sharded.shape(); - var o = zml.ops.manualComputation( + const output = fa2_mha_varlen_fwd.call( .{ - q_sharded, - k, - v, - cu_seqlens_q, - cu_seqlens_k.withTags(.{.i}).withPartitioning(.{ .i = .replicated }), - metadata.softmax_lse.withPartitioning(.{ .h = .model }), - metadata.softmax_lse_accum.withPartitioning(.{ .h = .model }), - metadata.out_accum.withPartitioning(.{ .h = .model }), + .q = q_sharded, + .k = k, + .v = v, + .cu_seqlens_q = cu_seqlens_q, + .cu_seqlens_k = cu_seqlens_k.withTags(.{.i}).withPartitioning(.{ .i = .replicated }), + .softmax_lse = metadata.softmax_lse.withPartitioning(.{ .h = .model }), + .softmax_lse_accum = metadata.softmax_lse_accum.withPartitioning(.{ .h = .model }), + .out_accum = metadata.out_accum.withPartitioning(.{ .h = .model }), }, - output_shape, .{ - .metadata = .{ - //.softmax_scale = @as(f32, 1.0), - .is_causal = true, - .window_size_left = @as(i32, -1), - .window_size_right = @as(i32, -1), - .max_seqlen_q = max_seqlen_q, - .max_seqlen_k = max_seqlen_k, - .num_heads = @as(i32, @intCast(@divExact(num_heads, model_partitions))), + .o = q_sharded.shape(), + }, + .{ + .softmax_scale = b: { + const head_dim = q.shape().dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); }, - .opts = zml.ops.CustomCallOptions{ - .output_operand_aliases = &.{0}, - .has_side_effect = false, - }, + .is_causal = true, + .window_size_left = @as(i32, -1), + .window_size_right = @as(i32, -1), + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + .num_heads = @as(i32, @intCast(@divExact(num_heads, model_partitions))), }, - (struct { - fn body(ctx_: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output: zml.Shape) zml.Tensor { - return zml.ops.customCall(custom_call_name, sharded_inputs, output, ctx_.metadata, ctx_.opts); - } - }).body, ); + var o = output.o; if (seqlenq_ngroups_swapped) { o = o.splitAxis(.tot, .{ .tot = original_tot, .ngroups = ngroups }).transpose(.{ .tot, .h, .ngroups, .hd }).merge(.{ .h = .{ .h, .ngroups } }); @@ -501,7 +511,7 @@ pub const fa3 = struct { .max_seqlen_k = max_seqlen_k, }, .{ - .output_operand_aliases = &.{0}, + .output_operand_aliases = &.{.{ .output_index = 0, .operand_index = 0 }}, .has_side_effect = false, }, ); diff --git a/zml/ops.zig b/zml/ops.zig index d1ae194d..5c344193 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -2,12 +2,18 @@ const std = @import("std"); const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); +const pjrt = @import("pjrt"); const stdx = @import("stdx"); +const Buffer = @import("buffer.zig").Buffer; +const Bufferized = @import("mem.zig").Bufferized; const CompilationContext = @import("module.zig").CompilationContext; const constants = @import("constants.zig"); +const CustomCallBuffer = @import("pjrtx.zig").CustomCallBuffer; +const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); +const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const Tensor = @import("tensor.zig").Tensor; @@ -445,7 +451,7 @@ pub const TritonOps = struct { grid: [3]i32, num_stages: i32, num_warps: i32, - output_operand_aliases: []const i64 = &.{}, + output_operand_aliases: []const dialects.stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, }; /// Generate an MLIR call to the given member function with the given tensors. @@ -1046,9 +1052,40 @@ fn CustomCallResultTypeFromOutputSpec(comptime OutputSpecT: type) type { pub const CustomCallOptions = struct { has_side_effect: bool, - output_operand_aliases: ?[]const i64 = null, + output_operand_aliases: ?[]const dialects.stablehlo.CustomCallOpts.OutputOperandAlias = null, }; +fn customCallOutputOperandAliases( + comptime I: type, + comptime O: type, + comptime aliases: ?CustomCallOutputOperandAliases(I, O), +) ?[]const dialects.stablehlo.CustomCallOpts.OutputOperandAlias { + const aliases_ = aliases orelse return null; + + const output_fields = @typeInfo(O).@"struct".fields; + comptime var num_aliases = 0; + inline for (output_fields) |field| { + if (@field(aliases_, field.name) != null) num_aliases += 1; + } + + const output_operand_aliases = comptime blk: { + var aliases_buffer: [num_aliases]dialects.stablehlo.CustomCallOpts.OutputOperandAlias = undefined; + var i = 0; + for (output_fields, 0..) |field, output_index| { + if (@field(aliases_, field.name)) |operand| { + aliases_buffer[i] = .{ + .output_index = @intCast(output_index), + .operand_index = @intCast(@intFromEnum(operand)), + }; + i += 1; + } + } + break :blk aliases_buffer; + }; + + return &output_operand_aliases; +} + pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, metadata: anytype, opts: CustomCallOptions) CustomCallResultTypeFromOutputSpec(@TypeOf(outputs)) { // Transform generic inputs to flat slice. const inputs_: []const Tensor = switch (@typeInfo(@TypeOf(inputs))) { @@ -1563,6 +1600,328 @@ fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs return outputs_; } +pub fn CustomCall( + // A, I and O are assumed to be simple struct without any nesting like: { a: zml.Tensor } + I: type, + O: type, + A: type, + // Receives the custom call buffers for the input and outputs a struct with the same fields as I and O + // and the provided attributes. So if I is `{a: zml.Tensor}`, the first argument is `{a: CustomCallBuffer}` + // (TensorToCustomCallBuffer(i), ShapeToCustomCallBuffer(O), A) -> !?*pjrt.ffi.Error + comptime func: anytype, + comptime params: struct { + /// Name of the custom call, must be unique. + name: [:0]const u8, + /// Custom call will see the data of a single device. So for a sharded buffer, it will only see a single partition. This applies + /// for both the input and output buffers. By default, the partitioning algorithm will assume nothing on the custom call input/output, + /// so everything will be forced as replicated. If do you know / ensure that the custom call is consistent with the provided input/output + /// sharding, this setting will avoid the shuffling of the buffer's data. + sharding_aware: bool, + /// Whether the function has any side-effect, meaning that XLA cannot re-order it/optimize it out. A typical example is print. + has_side_effect: bool, + /// Similar to reuseBuffer on tensors, tells XLA that the custom call re-uses an input buffer for its output. + /// Use the input/output field names to express the mapping: `{ . = . }` + output_operand_aliases: ?CustomCallOutputOperandAliases(I, O) = null, + }, +) type { + return struct { + pub const InputTensors = I; + pub const OutputShapes = O; + pub const Attributes = A; + + /// Register the custom call so you can use it. + pub fn register(platform: *const Platform) !void { + try platform.registerFfi(.{ + .name = params.name, + .platform_name = null, + .handler = @This().handler, + .traits = .{ + .command_buffer_compatible = true, + }, + }); + } + + pub fn call(input_tensors: I, output_shapes: O, attributes: A) ShapeToTensor(O) { + const args: CustomCallArgs(I, O, A) = .{ + .input_tensors = input_tensors, + .output_shapes = output_shapes, + .attributes = attributes, + }; + const opts: CustomCallOptions = .{ + .has_side_effect = params.has_side_effect, + .output_operand_aliases = comptime customCallOutputOperandAliases(I, O, params.output_operand_aliases), + }; + if (params.sharding_aware) { + return shardingAwareTypedCustomCall( + I, + O, + A, + params.name, + opts, + args, + ); + } else { + return typedCustomCall( + I, + O, + A, + params.name, + opts, + args, + ); + } + } + + fn handler(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { + if (call_frame.registeringHook()) return null; + + const input_buffers: TensorToCustomCallBuffer(I), const output_buffers: ShapeToCustomCallBuffer(O), const attributes: A = customCallArgsFromPjrtCallFrame(I, O, A, call_frame); + return func(call_frame, input_buffers, output_buffers, attributes) catch |err| + pjrt.ffi.Error.create(call_frame.api, .unknown, @errorName(err)); + } + }; +} + +fn CustomCallArgs(I: type, O: type, A: type) type { + return struct { + input_tensors: I, + output_shapes: O, + attributes: A, + }; +} + +pub fn TensorToCustomCallBuffer(T: type) type { + return meta.MapRestrict(Tensor, CustomCallBuffer).map(T); +} + +pub fn ShapeToCustomCallBuffer(T: type) type { + return meta.MapRestrict(Shape, CustomCallBuffer).map(T); +} + +pub fn ShapeToTensor(T: type) type { + return meta.MapRestrict(Shape, Tensor).map(T); +} + +pub fn CustomCallOutputOperandAliases(I: type, O: type) type { + return meta.MapRestrict(Shape, ?std.meta.FieldEnum(I)).map(O); +} + +pub fn shardingAwareTypedCustomCall( + I: type, + O: type, + A: type, + comptime target_name: [:0]const u8, + comptime opts: CustomCallOptions, + args: CustomCallArgs(I, O, A), +) ShapeToTensor(O) { + // Convert to slices so that manualComputationInternal can inspect the input/output. + var input_tensors: [@typeInfo(@TypeOf(args.input_tensors)).@"struct".fields.len]Tensor = undefined; + inline for (@typeInfo(@TypeOf(args.input_tensors)).@"struct".fields, 0..) |field, i| { + input_tensors[i] = @field(args.input_tensors, field.name); + } + + var output_shapes: [@typeInfo(@TypeOf(args.output_shapes)).@"struct".fields.len]Shape = undefined; + inline for (@typeInfo(@TypeOf(args.output_shapes)).@"struct".fields, 0..) |field, i| { + output_shapes[i] = @field(args.output_shapes, field.name); + } + + const output_tensors = manualComputationInternal(&input_tensors, &output_shapes, args.attributes, (struct { + fn body(attributes: A, _: std.mem.Allocator, sharded_input_tensors: []const Tensor, sharded_output_shapes: []const Shape) []const Tensor { + return typedCustomCall( + []const Tensor, + []const Shape, + A, + target_name, + opts, + .{ + .input_tensors = sharded_input_tensors, + .output_shapes = sharded_output_shapes, + .attributes = attributes, + }, + ); + } + }).body); + + var out: ShapeToTensor(O) = undefined; + inline for (@typeInfo(@TypeOf(args.output_shapes)).@"struct".fields, 0..) |field, i| { + @field(out, field.name) = output_tensors[i]; + } + return out; +} + +pub fn typedCustomCall( + I: type, + O: type, + A: type, + comptime target_name: [:0]const u8, + comptime opts: CustomCallOptions, + args: CustomCallArgs(I, O, A), +) ShapeToTensor(O) { + const ctx = CompilationContext.current(); + const allocator = ctx.arena.allocator(); + + stdx.debug.assert(!opts.has_side_effect or ctx.manual_computation_depth > 0, "side-effect customCall '{s}' must be emitted inside manualComputation", .{target_name}); + + const input_tensors: []const Tensor = switch (@typeInfo(@TypeOf(args.input_tensors))) { + .@"struct" => |struct_info| b: { + var input_tensors: [struct_info.fields.len]Tensor = undefined; + inline for (struct_info.fields, 0..) |field, i| { + input_tensors[i] = @field(args.input_tensors, field.name); + } + break :b &input_tensors; + }, + // Extra case to support []const Tensor from shardingAwareTypedCustomCall + .pointer => |pointer_info| b: { + if (pointer_info.size != .slice) @compileError("Expected input slice"); + break :b args.input_tensors; + }, + else => @compileError("Unsupported input type: " ++ @typeName(@TypeOf(args.input_tensor))), + }; + + const output_shapes: []const Shape = switch (@typeInfo(@TypeOf(args.output_shapes))) { + .@"struct" => |struct_info| b: { + var output_shapes: [struct_info.fields.len]Shape = undefined; + inline for (struct_info.fields, 0..) |field, i| { + output_shapes[i] = @field(args.output_shapes, field.name); + } + break :b &output_shapes; + }, + // Extra case to support []const Shape from shardingAwareTypedCustomCall + .pointer => |pointer_info| b: { + if (pointer_info.size != .slice) @compileError("Expected input slice"); + break :b args.output_shapes; + }, + else => @compileError("Unsupported input type: " ++ @typeName(@TypeOf(args.input_tensor))), + }; + + const metadata_attributes = + switch (@typeInfo(@TypeOf(args.attributes))) { + .@"struct" => |struct_info| b: { + var mlir_attributes_array: [struct_info.fields.len]mlir.NamedAttribute = undefined; + inline for (struct_info.fields, 0..) |field, i| { + const attribute: *const mlir.Attribute = switch (@typeInfo(field.type)) { + .comptime_int => mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @field(args.attributes, field.name))), + .int => |int_field| switch (int_field.signedness) { + .signed => switch (int_field.bits) { + 8 => mlir.integerAttribute(ctx.mlir_ctx, .i8, @field(args.attributes, field.name)), + 16 => mlir.integerAttribute(ctx.mlir_ctx, .i16, @field(args.attributes, field.name)), + 32 => mlir.integerAttribute(ctx.mlir_ctx, .i32, @field(args.attributes, field.name)), + 64 => mlir.integerAttribute(ctx.mlir_ctx, .i64, @field(args.attributes, field.name)), + else => @panic("Unsupported DataType"), + }, + .unsigned => switch (int_field.bits) { + 8 => mlir.integerAttribute(ctx.mlir_ctx, .u8, @field(args.attributes, field.name)), + 16 => mlir.integerAttribute(ctx.mlir_ctx, .u16, @field(args.attributes, field.name)), + 32 => mlir.integerAttribute(ctx.mlir_ctx, .u32, @field(args.attributes, field.name)), + 64 => mlir.integerAttribute(ctx.mlir_ctx, .u64, @field(args.attributes, field.name)), + else => @panic("Unsupported DataType"), + }, + }, + .float => |float_field| switch (float_field.bits) { + 16 => mlir.floatAttribute(ctx.mlir_ctx, .f16, @as(f64, @field(args.attributes, field.name))), + 32 => mlir.floatAttribute(ctx.mlir_ctx, .f32, @as(f64, @field(args.attributes, field.name))), + 64 => mlir.floatAttribute(ctx.mlir_ctx, .f64, @as(f64, @field(args.attributes, field.name))), + else => @panic("Unsupported DataType"), + }, + .bool => mlir.boolAttribute(ctx.mlir_ctx, @field(args.attributes, field.name)), + .pointer => |pointer_info| switch (pointer_info.size) { + .slice => if (pointer_info.child == u8) + mlir.stringAttribute(ctx.mlir_ctx, @field(args.attributes, field.name)) + else + @panic("Unsupported pointer type in metadata"), + else => @panic("Unsupported pointer type in metadata"), + }, + else => @compileError("Unsupported metadata type: " ++ @typeName(field.type)), + }; + mlir_attributes_array[i] = mlir.NamedAttribute.named(ctx.mlir_ctx, field.name, attribute); + } + + break :b mlir_attributes_array; + }, + else => @compileError("Unsupported type: " ++ @typeName(@TypeOf(args.attributes))), + }; + + const values = allocator.alloc(*const mlir.Value, input_tensors.len) catch unreachable; + const input_shapes = allocator.alloc(Shape, input_tensors.len) catch unreachable; + for (input_tensors, 0..) |input, i| { + values[i] = input.value(); + input_shapes[i] = input.shape(); + } + + const result_types = allocator.alloc(*const mlir.Type, output_shapes.len) catch unreachable; + for (output_shapes, 0..) |shape, i| { + result_types[i] = mlir.rankedTensorType(shape.dims(), mlirx.Type.fromDType(ctx.mlir_ctx, shape.dtype())); + } + + const backend_config = mlir.dictionaryAttribute(ctx.mlir_ctx, &(metadata_attributes ++ [_]mlir.NamedAttribute{ + .named(ctx.mlir_ctx, "pjrt_api", mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_api))))), + .named(ctx.mlir_ctx, "pjrt_client", mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_client))))), + })); + + const operand_layouts = allocator.alloc([]const usize, input_shapes.len) catch unreachable; + for (input_shapes, 0..) |shape, i| { + operand_layouts[i] = allocator.dupe(usize, toUsize(constants.minorToMajor(shape.rank())).constSlice()) catch unreachable; + } + + const result_layouts = allocator.alloc([]const usize, output_shapes.len) catch unreachable; + for (output_shapes, 0..) |shape, i| { + result_layouts[i] = allocator.dupe(usize, toUsize(constants.minorToMajor(shape.rank())).constSlice()) catch unreachable; + } + + const op = dialects.stablehlo.custom_call( + ctx.mlir_ctx, + values, + result_types, + .{ + .call_target_name = target_name, + .backend_config = .{ .typed_ffi = backend_config }, + .has_side_effect = opts.has_side_effect, + .operand_layouts = operand_layouts, + .result_layouts = result_layouts, + .output_operand_aliases = opts.output_operand_aliases orelse &.{}, + .additional_attributes = &.{}, + }, + .unknown(ctx.mlir_ctx), + ).appendTo(ctx.currentScope().block); + + if (ctx.manual_computation_depth > 0 and ctx.partitioning.partitioner == .gspmd) { + op.setAttributeByName("mhlo.sharding", mlir.stringAttribute(ctx.mlir_ctx, "{manual}")); + } + + var out: ShapeToTensor(O) = undefined; + inline for (output_shapes, @typeInfo(@TypeOf(args.output_shapes)).@"struct".fields, 0..) |output_shape, field, i| { + @field(out, field.name) = Tensor._result(output_shape, op.result(i)); + } + return out; +} + +fn customCallArgsFromPjrtCallFrame(I: type, O: type, A: type, call_frame: *pjrt.ffi.CallFrame) struct { + TensorToCustomCallBuffer(I), + ShapeToCustomCallBuffer(O), + A, +} { + var input: TensorToCustomCallBuffer(I) = undefined; + inline for (@typeInfo(I).@"struct".fields, 0..) |field, i| { + const buf = call_frame.args.buffers()[i]; + @field(input, field.name) = .fromPjrt(buf); + } + + var output: ShapeToCustomCallBuffer(O) = undefined; + inline for (@typeInfo(O).@"struct".fields, 0..) |field, i| { + const shape_buf = call_frame.results.buffers()[i]; + @field(output, field.name) = .fromPjrt(shape_buf); + } + + var attributes: A = undefined; + inline for (@typeInfo(A).@"struct".fields) |field| { + const attribute = call_frame.attrs.getByName(.scalar, field.name) orelse + @panic("Attribute not found: " ++ field.name); + @field(attributes, field.name) = attribute.get(field.type); + } + + return .{ input, output, attributes }; +} + test customCall { const zml = @import("zml.zig"); const platform = zml.testing.env(); @@ -1580,12 +1939,39 @@ test customCall { const input = Tensor.constant(zml.DataType.bf16.constant(0)).broad(shape); const output = customCall("my_custom_call", .{input}, .{zml.Shape.init(.{128}, .bf16)}, .{}, .{ .has_side_effect = false, - .output_operand_aliases = &.{0}, + .output_operand_aliases = &.{.{ .output_index = 0, .operand_index = 0 }}, }); try zml.testing.expectEqualShapes(input.shape(), output.shape()); } +test "CustomCallOutputOperandAliases maps named fields to indices" { + const Input = struct { + q: Tensor, + k: Tensor, + v: Tensor, + }; + const Output = struct { + attn: Shape, + scratch: Shape, + }; + + const aliases = comptime customCallOutputOperandAliases(Input, Output, .{ + .attn = .q, + .scratch = .v, + }).?; + + try std.testing.expectEqual(@as(usize, 2), aliases.len); + try std.testing.expectEqualDeep( + dialects.stablehlo.CustomCallOpts.OutputOperandAlias{ .output_index = 0, .operand_index = 0 }, + aliases[0], + ); + try std.testing.expectEqualDeep( + dialects.stablehlo.CustomCallOpts.OutputOperandAlias{ .output_index = 1, .operand_index = 2 }, + aliases[1], + ); +} + fn toUsize(values: anytype) stdx.BoundedArray(usize, constants.MAX_RANK) { var res: stdx.BoundedArray(usize, constants.MAX_RANK) = .{}; for (values) |val| res.appendAssumeCapacity(@intCast(val)); diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 36a4ef30..e328c5de 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -5,6 +5,7 @@ const mlir = @import("mlir"); const pjrt = @import("pjrt"); pub const DataType = @import("dtype.zig").DataType; +const Shape = @import("shape.zig").Shape; const log = std.log.scoped(.@"zml/pjrtx"); @@ -42,6 +43,30 @@ pub const Client = opaque { } }; +pub const CustomCallBuffer = struct { + shape: Shape, + ptr: *anyopaque, + + pub fn format( + self: @This(), + writer: *std.Io.Writer, + ) std.Io.Writer.Error!void { + try writer.print("{f}@{*}", .{ self.shape, self.ptr }); + } + + pub fn fromPjrt(buffer: *const pjrt.ffi.Buffer) @This() { + const dt = switch (buffer.dtype) { + .invalid => @panic("Found an invalid pjrt buffer"), + .token => @panic("Found an token pjrt buffer"), + inline else => |tag| @field(DataType, @tagName(tag)), + }; + return .{ + .ptr = buffer.data, + .shape = .init(buffer.dims(), dt), + }; + } +}; + pub fn bufferTypeFromDtype(dt: DataType) pjrt.BufferType { return switch (dt) { inline else => |tag| @field(pjrt.BufferType, @tagName(tag)), From 448084908ac7ee68039e459e1ae128337710c5a1 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 7 Apr 2026 16:42:55 +0200 Subject: [PATCH 406/881] zml/tokenizer: add IREE tokenizer (#416) Adding the [IREE tokenizer](https://github.com/iree-org/iree/blob/main/runtime/src/iree/tokenizer/README.md).Being made from the ground up with streaming (de-)tokenization in mind, I've extended the APIs to expose a `std.Io.Writer` for the tokenization. I've quickly benchmarked IREE's tokenizer vs HuggingFace's one with a request I retrieved from a coding agent (opencode) in the middle of a session. The request body was 360KB encoded in 95K tokens. I also did another test with a similarly sized request with the same results as shown hereafter. I've tested two cases: - oneshot: the template is rendered all at once and only then we encode the whole string. - stream: the template rendering and encoding are pipelined to avoid the intermediate allocation. There is a slight difference in the generated tokens between HuggingFace and IREE. Some tokens like `introdu` were encoded as `i`, `n`, `t`, etc. Decoding them returns the same text. It's very close in terms of total token count: - hf: 95779 - iree/stream: 95705 - iree/oneshot: 95587 | Model | IREE avg (oneshot / stream) ms | HuggingFace avg (ms) | Uplift vs HF (oneshot / stream) | |---|---:|---:|---| | liquidai/LFM2.5-1.2B-Instruct | 8.5 / 7.3 | 40.5 | 4.8x / 5.5x | | Qwen/Qwen3.5-9B | 6.0 / 8.2 | 59.4 | 9.9x / 7.2x | | mistralai/Ministral-3-3B-Reasoning-2512 | 7.4 / 6.9 | 59.2 | 8x / 8.6x | It's unclear why the Qwen's chat template is somehow slower in the stream case, maybe because there are more writes. Adding non-empty buffer in the interface helps a bit, but tends to slow down mistral case. So well --------- Co-authored-by: Corentin Kerisit --- BUILD.bazel | 1 + MODULE.bazel | 5 +- MODULE.bazel.lock | 2 + examples/llm/main.zig | 2 +- examples/llm/models/lfm2/session.zig | 60 +-- examples/llm/models/llama/session.zig | 50 ++- examples/llm/models/qwen3_5/session.zig | 44 +- third_party/iree/BUILD.bazel | 0 third_party/iree/repo.bzl | 21 + third_party/iree/tokenizer-only.patch | 48 +++ third_party/non_module_deps.bzl | 2 + third_party/xla/llvm_raw_only.bzl | 13 + zml/tokenizer/BUILD.bazel | 6 +- zml/tokenizer/hftokenizers/.gitignore | 1 + zml/tokenizer/hftokenizers/hftokenizers.zig | 2 +- zml/tokenizer/homemade.zig | 2 +- zml/tokenizer/iree/BUILD.bazel | 20 + zml/tokenizer/iree/tokenizer.zig | 405 ++++++++++++++++++ zml/tokenizer/main.zig | 14 +- zml/tokenizer/sentencepiece/sentencepiece.zig | 2 +- zml/tokenizer/tokenizer.zig | 287 +++++++++++-- 21 files changed, 855 insertions(+), 132 deletions(-) create mode 100644 third_party/iree/BUILD.bazel create mode 100644 third_party/iree/repo.bzl create mode 100644 third_party/iree/tokenizer-only.patch create mode 100644 third_party/xla/llvm_raw_only.bzl create mode 100644 zml/tokenizer/hftokenizers/.gitignore create mode 100644 zml/tokenizer/iree/BUILD.bazel create mode 100644 zml/tokenizer/iree/tokenizer.zig diff --git a/BUILD.bazel b/BUILD.bazel index c133b6a0..94dc6e48 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -9,6 +9,7 @@ zls_completion( "//examples/sharding", "//stdx", "//zml", + "//zml/tokenizer/iree", ], ) diff --git a/MODULE.bazel b/MODULE.bazel index 1d839851..24d2c98a 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -6,6 +6,8 @@ bazel_dep(name = "abseil-cpp", version = "20250814.1", repo_name = "com_google_a bazel_dep(name = "bazel_lib", version = "3.2.2") bazel_dep(name = "aspect_rules_py", version = "1.8.4") bazel_dep(name = "bazel_skylib", version = "1.9.0") +## Needed by iree +bazel_dep(name = "libbacktrace", version = "1.0.0-20250926-7939218") bazel_dep(name = "patchelf", version = "0.18.0.bcr.1") bazel_dep(name = "pcre2", version = "10.45") bazel_dep(name = "platforms", version = "1.0.0") @@ -143,7 +145,7 @@ use_repo(zls, "zls_toolchains") register_toolchains("@zls_toolchains//:all") non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") -use_repo(non_module_deps, "arocc", "cloud_accelerator_diagnostics", "com_google_sentencepiece", "flashattn", "libvaxis", "linenoise", "mnist", "org_swig_swig", "translate-c", "stb", "uucode", "xla", "zigimg") +use_repo(non_module_deps, "arocc", "cloud_accelerator_diagnostics", "com_google_sentencepiece", "flashattn", "iree", "libvaxis", "linenoise", "mnist", "org_swig_swig", "translate-c", "stb", "uucode", "xla", "zigimg") xla = use_extension("//third_party/xla:xla.bzl", "xla") use_repo( @@ -240,6 +242,7 @@ use_repo( "distroless_cc_debian12_linux_amd64", ) + git_override( module_name = "rules_zig", commit = "7b205b78fb21e11efe5adcc9ec1c37fa5852dc77", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 173129ab..ee860aa2 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -147,6 +147,8 @@ "https://bcr.bazel.build/modules/jq.bzl/0.1.0/source.json": "746bf13cac0860f091df5e4911d0c593971cd8796b5ad4e809b2f8e133eee3d5", "https://bcr.bazel.build/modules/jsoncpp/1.9.6/MODULE.bazel": "2f8d20d3b7d54143213c4dfc3d98225c42de7d666011528dc8fe91591e2e17b0", "https://bcr.bazel.build/modules/jsoncpp/1.9.6/source.json": "a04756d367a2126c3541682864ecec52f92cdee80a35735a3cb249ce015ca000", + "https://bcr.bazel.build/modules/libbacktrace/1.0.0-20250926-7939218/MODULE.bazel": "4cd5b9355f1c2533c30f01904b71ac1b1a9bf61c43c81ba9972a5bf8f216476b", + "https://bcr.bazel.build/modules/libbacktrace/1.0.0-20250926-7939218/source.json": "83239989d09748f63dee4bdd430d261e217a012670e066e2238925a303a65d89", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", "https://bcr.bazel.build/modules/libpfm/4.11.0/source.json": "caaffb3ac2b59b8aac456917a4ecf3167d40478ee79f15ab7a877ec9273937c9", "https://bcr.bazel.build/modules/llvm/0.6.7/MODULE.bazel": "d37a2e10571864dc6a5bb53c29216d90b9400bbcadb422337f49107fd2eaf0d2", diff --git a/examples/llm/main.zig b/examples/llm/main.zig index c058a3c6..cc3c3342 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -179,7 +179,7 @@ fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, prog }; defer allocator.free(bytes); - return try .fromBytes(allocator, io, bytes); + return try .fromBytes(allocator, bytes); } pub fn printZmlLogo(io: std.Io) !void { diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig index f0bd8b72..0fe2cc75 100644 --- a/examples/llm/models/lfm2/session.zig +++ b/examples/llm/models/lfm2/session.zig @@ -44,8 +44,8 @@ pub const Session = struct { .attention_metadata_buffers = try compiled_model.params.attention_metadata.initBuffer(io, platform, compiled_model.params.shardings.model), .rng_buf = try zml.Tensor.Rng.initBuffer(platform, seed, io, compiled_model.params.shardings.replicated), .generated_token_slice = try .alloc(allocator, zml.Shape.init(.{ .batch = 1, .seq = 1 }, .u32)), - .think_start = tokenizer.tokenToId("") orelse unreachable, - .think_end = tokenizer.tokenToId("") orelse unreachable, + .think_start = tokenizer.tokenId("") orelse unreachable, + .think_end = tokenizer.tokenId("") orelse unreachable, }; } @@ -60,17 +60,16 @@ pub const Session = struct { var encoder = try self.tokenizer.encoder(); defer encoder.deinit(); - const im_start = self.tokenizer.tokenToId("<|im_start|>") orelse return error.NoSuchToken; - const im_end = self.tokenizer.tokenToId("<|im_end|>") orelse return error.NoSuchToken; - const user = self.tokenizer.tokenToId("user") orelse return error.NoSuchToken; - const assistant = self.tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; - const newline = (try encoder.encode("\n"))[0]; + const im_start = self.tokenizer.tokenId("<|im_start|>") orelse return error.NoSuchToken; + const im_end = self.tokenizer.tokenId("<|im_end|>") orelse return error.NoSuchToken; + const newline = self.tokenizer.tokenId("\n") orelse return error.NoSuchToken; var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); - try tokens.appendSlice(allocator, &.{ self.config.bos_token_id, im_start, user, newline }); - try tokens.appendSlice(allocator, try encoder.encode(prompt)); - try tokens.appendSlice(allocator, &.{ im_end, newline }); - try tokens.appendSlice(allocator, &.{ im_start, assistant, newline }); + try tokens.appendSlice(allocator, &.{ self.config.bos_token_id, im_start }); + try encoder.encodeAppend(allocator, &tokens, "user\n"); + try encoder.encodeAppend(allocator, &tokens, prompt); + try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); + try encoder.encodeAppend(allocator, &tokens, "assistant\n"); return tokens.toOwnedSlice(allocator); } @@ -78,17 +77,16 @@ pub const Session = struct { var encoder = try self.tokenizer.encoder(); defer encoder.deinit(); - const im_start = self.tokenizer.tokenToId("<|im_start|>") orelse return error.NoSuchToken; - const im_end = self.tokenizer.tokenToId("<|im_end|>") orelse return error.NoSuchToken; - const user = self.tokenizer.tokenToId("user") orelse return error.NoSuchToken; - const assistant = self.tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; - const newline = (try encoder.encode("\n"))[0]; + const im_start = self.tokenizer.tokenId("<|im_start|>") orelse return error.NoSuchToken; + const im_end = self.tokenizer.tokenId("<|im_end|>") orelse return error.NoSuchToken; + const newline = self.tokenizer.tokenId("\n") orelse return error.NoSuchToken; var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); - try tokens.appendSlice(allocator, &.{ im_end, newline, im_start, user, newline }); - try tokens.appendSlice(allocator, try encoder.encode(prompt)); - try tokens.appendSlice(allocator, &.{ im_end, newline }); - try tokens.appendSlice(allocator, &.{ im_start, assistant, newline }); + try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); + try encoder.encodeAppend(allocator, &tokens, "user\n"); + try encoder.encodeAppend(allocator, &tokens, prompt); + try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); + try encoder.encodeAppend(allocator, &tokens, "assistant\n"); return tokens.toOwnedSlice(allocator); } @@ -147,16 +145,15 @@ pub const Session = struct { if (token_id == self.config.eos_token_id) break :generation; - if (try decoder.next(token_id)) |token| { - if (self.think_start) |think_start| if (token_id == think_start) { - try stdout.writeAll("\x1b[2m"); - }; - try stdout.writeAll(token); - if (self.think_end) |think_end| if (token_id == think_end) { - try stdout.writeAll("\x1b[0m"); - }; - try stdout.flush(); - } + const token = try decoder.feed_one(token_id); + if (self.think_start) |think_start| if (token_id == think_start) { + try stdout.writeAll("\x1b[2m"); + }; + try stdout.writeAll(token); + if (self.think_end) |think_end| if (token_id == think_end) { + try stdout.writeAll("\x1b[0m"); + }; + try stdout.flush(); try all_tokens.append(self.allocator, token_id); if (all_tokens.items.len >= self.seqlen) break :generation; @@ -180,5 +177,8 @@ pub const Session = struct { try current_token_buffer.toSlice(self.io, self.generated_token_slice); } + + try stdout.writeAll(try decoder.finalize()); + try stdout.flush(); } }; diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig index 5bca14c3..29124859 100644 --- a/examples/llm/models/llama/session.zig +++ b/examples/llm/models/llama/session.zig @@ -65,17 +65,19 @@ pub const Session = struct { var encoder = try self.tokenizer.encoder(); defer encoder.deinit(); - const start_header = self.tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; - const end_header = self.tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; - const user = self.tokenizer.tokenToId("user") orelse return error.NoSuchToken; - const assistant = self.tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; - const eot = self.tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; - const newline = (try encoder.encode("\n"))[0]; + const start_header = self.tokenizer.tokenId("<|start_header_id|>") orelse return error.NoSuchToken; + const end_header = self.tokenizer.tokenId("<|end_header_id|>") orelse return error.NoSuchToken; + const eot = self.tokenizer.tokenId("<|eot_id|>") orelse return error.NoSuchToken; + const newline = self.tokenizer.tokenId("\\n") orelse return error.NoSuchToken; var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); - try tokens.appendSlice(allocator, &.{ self.config.bos_token_id, start_header, user, end_header, newline }); - try tokens.appendSlice(allocator, try encoder.encode(prompt)); - try tokens.appendSlice(allocator, &.{ eot, newline, start_header, assistant, end_header, newline }); + try tokens.appendSlice(allocator, &.{ self.config.bos_token_id, start_header }); + try encoder.encodeAppend(allocator, &tokens, "user"); + try tokens.appendSlice(allocator, &.{ end_header, newline }); + try encoder.encodeAppend(allocator, &tokens, prompt); + try tokens.appendSlice(allocator, &.{ eot, newline, start_header }); + try encoder.encodeAppend(allocator, &tokens, "assistant"); + try tokens.appendSlice(allocator, &.{ end_header, newline }); return tokens.toOwnedSlice(allocator); } @@ -83,17 +85,19 @@ pub const Session = struct { var encoder = try self.tokenizer.encoder(); defer encoder.deinit(); - const start_header = self.tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; - const end_header = self.tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; - const user = self.tokenizer.tokenToId("user") orelse return error.NoSuchToken; - const assistant = self.tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; - const eot = self.tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; - const newline = (try encoder.encode("\n"))[0]; + const start_header = self.tokenizer.tokenId("<|start_header_id|>") orelse return error.NoSuchToken; + const end_header = self.tokenizer.tokenId("<|end_header_id|>") orelse return error.NoSuchToken; + const eot = self.tokenizer.tokenId("<|eot_id|>") orelse return error.NoSuchToken; + const newline = self.tokenizer.tokenId("\\n") orelse return error.NoSuchToken; var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); - try tokens.appendSlice(allocator, &.{ eot, newline, start_header, user, end_header, newline }); - try tokens.appendSlice(allocator, try encoder.encode(prompt)); - try tokens.appendSlice(allocator, &.{ eot, newline, start_header, assistant, end_header, newline }); + try tokens.appendSlice(allocator, &.{ eot, newline, start_header }); + try encoder.encodeAppend(allocator, &tokens, "user"); + try tokens.appendSlice(allocator, &.{ end_header, newline }); + try encoder.encodeAppend(allocator, &tokens, prompt); + try tokens.appendSlice(allocator, &.{ eot, newline, start_header }); + try encoder.encodeAppend(allocator, &tokens, "assistant"); + try tokens.appendSlice(allocator, &.{ end_header, newline }); return tokens.toOwnedSlice(allocator); } @@ -152,10 +156,9 @@ pub const Session = struct { if (isEosToken(self.config, token_id)) break :generation; - if (try decoder.next(token_id)) |token| { - try stdout.writeAll(token); - try stdout.flush(); - } + const token = try decoder.feed_one(token_id); + try stdout.writeAll(token); + try stdout.flush(); try all_tokens.append(self.allocator, token_id); if (all_tokens.items.len >= self.seqlen) break :generation; @@ -177,6 +180,9 @@ pub const Session = struct { decode_results.fill(.{ ¤t_token_buffer, &self.kv_cache_buffers, &self.rng_buffers }); try current_token_buffer.toSlice(self.io, self.generated_token_slice); } + + try stdout.writeAll(try decoder.finalize()); + try stdout.flush(); } }; diff --git a/examples/llm/models/qwen3_5/session.zig b/examples/llm/models/qwen3_5/session.zig index 3a1bc8b6..e4eefce5 100644 --- a/examples/llm/models/qwen3_5/session.zig +++ b/examples/llm/models/qwen3_5/session.zig @@ -52,8 +52,8 @@ pub const Session = struct { .seqlen = compiled_model.params.seqlen, .eos_token_id = compiled_model.loaded_model.inner.special_tokens.end_of_text_token_id, .special_tokens = compiled_model.loaded_model.inner.special_tokens, - .think_start = tokenizer.tokenToId(""), - .think_end = tokenizer.tokenToId(""), + .think_start = tokenizer.tokenId(""), + .think_end = tokenizer.tokenId(""), }; } @@ -117,22 +117,24 @@ pub const Session = struct { const token_id = self.generated_token_slice.items(u32)[0]; if (token_id == self.eos_token_id) break :generation; - if (try decoder.next(token_id)) |token| { - if (self.think_start) |think_start| if (token_id == think_start) { - try stdout.writeAll("\x1b[2m"); - }; - try stdout.writeAll(token); - if (self.think_end) |think_end| if (token_id == think_end) { - try stdout.writeAll("\x1b[0m"); - }; - try stdout.flush(); - } + const token = try decoder.feed_one(token_id); + if (self.think_start) |think_start| if (token_id == think_start) { + try stdout.writeAll("\x1b[2m"); + }; + try stdout.writeAll(token); + if (self.think_end) |think_end| if (token_id == think_end) { + try stdout.writeAll("\x1b[0m"); + }; + try stdout.flush(); try all_tokens.append(self.allocator, token_id); if (all_tokens.items.len >= self.seqlen) break :generation; try self.runDecodeStep(token_id, @intCast(all_tokens.items.len)); } + + try stdout.writeAll(try decoder.finalize()); + try stdout.flush(); } fn runDecodeStep(self: *Session, token_id: u32, token_index: u32) !void { @@ -170,9 +172,9 @@ fn tokenizeChatPrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tok var encoder = try tokenizer.encoder(); defer encoder.deinit(); - const im_start = tokenizer.tokenToId("<|im_start|>") orelse special_tokens.im_start_token_id; - const im_end = tokenizer.tokenToId("<|im_end|>") orelse special_tokens.im_end_token_id; - const newline = try encodeSingleToken(&encoder, "\n"); + const im_start = tokenizer.tokenId("<|im_start|>") orelse special_tokens.im_start_token_id; + const im_end = tokenizer.tokenId("<|im_end|>") orelse special_tokens.im_end_token_id; + const newline = tokenizer.tokenId("\n") orelse return error.NoSuchToken; var tokens: std.ArrayList(u32) = try .initCapacity(allocator, 32); if (!is_first_turn) { @@ -180,16 +182,10 @@ fn tokenizeChatPrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tok } try tokens.append(allocator, im_start); - try tokens.appendSlice(allocator, try encoder.encode("user\n")); - try tokens.appendSlice(allocator, try encoder.encode(prompt)); + try encoder.encodeAppend(allocator, &tokens, "user\n"); + try encoder.encodeAppend(allocator, &tokens, prompt); try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); - try tokens.appendSlice(allocator, try encoder.encode("assistant\n")); + try encoder.encodeAppend(allocator, &tokens, "assistant\n"); return tokens.toOwnedSlice(allocator); } - -fn encodeSingleToken(encoder: *zml.tokenizer.Tokenizer.Encoder, text: []const u8) !u32 { - const encoded = try encoder.encode(text); - if (encoded.len != 1) return error.InvalidTokenizerEncoding; - return encoded[0]; -} diff --git a/third_party/iree/BUILD.bazel b/third_party/iree/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/third_party/iree/repo.bzl b/third_party/iree/repo.bzl new file mode 100644 index 00000000..35740408 --- /dev/null +++ b/third_party/iree/repo.bzl @@ -0,0 +1,21 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + git_repository( + name = "iree", + remote = "https://github.com/iree-org/iree.git", + commit = "71af3a5e41a8e265330bc693194c708cf6df4724", + sparse_checkout_patterns = [ + "runtime/src/**", + "build_tools/bazel/**", + "build_tools/third_party/libbacktrace/**", + "BUILD.bazel", + ], + patches = [ + "//third_party/iree:tokenizer-only.patch", + ], + patch_args = ["-p1"], + ) + + #new_local_repository(name = "flashattn", build_file="//:third_party/flashattn/BUILD.bazel", path="/home/corendos/flashattn/") + diff --git a/third_party/iree/tokenizer-only.patch b/third_party/iree/tokenizer-only.patch new file mode 100644 index 00000000..26b5408b --- /dev/null +++ b/third_party/iree/tokenizer-only.patch @@ -0,0 +1,48 @@ +diff --git a/build_tools/bazel/build_defs.oss.bzl b/build_tools/bazel/build_defs.oss.bzl +index 090c87f860..ed1ab94fd0 100644 +--- a/build_tools/bazel/build_defs.oss.bzl ++++ b/build_tools/bazel/build_defs.oss.bzl +@@ -4,23 +4,33 @@ + # See https://llvm.org/LICENSE.txt for license information. + # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +-load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "gentbl_filegroup", "td_library") ++# load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "gentbl_filegroup", "td_library") + + # All load statements must come first in Starlark. + load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") +-load( +- "//build_tools/bazel:iree_cc_fuzz.bzl", +- _iree_cc_fuzz = "iree_cc_fuzz", +- _iree_compiler_cc_fuzz = "iree_compiler_cc_fuzz", +- _iree_runtime_cc_fuzz = "iree_runtime_cc_fuzz", +-) ++# load( ++# "//build_tools/bazel:iree_cc_fuzz.bzl", ++# _iree_cc_fuzz = "iree_cc_fuzz", ++# _iree_compiler_cc_fuzz = "iree_compiler_cc_fuzz", ++# _iree_runtime_cc_fuzz = "iree_runtime_cc_fuzz", ++# ) + + """Common Bazel definitions for IREE.""" + + # Re-export fuzz rules for external use. +-iree_cc_fuzz = _iree_cc_fuzz +-iree_compiler_cc_fuzz = _iree_compiler_cc_fuzz +-iree_runtime_cc_fuzz = _iree_runtime_cc_fuzz ++# iree_cc_fuzz = _iree_cc_fuzz ++# iree_compiler_cc_fuzz = _iree_compiler_cc_fuzz ++# iree_runtime_cc_fuzz = _iree_runtime_cc_fuzz ++ ++def generic_pass(**kwargs): ++ pass ++ ++gentbl_cc_library = generic_pass ++gentbl_filegroup = generic_pass ++td_library = generic_pass ++iree_runtime_cc_fuzz = generic_pass ++iree_compiler_cc_fuzz = generic_pass ++iree_cc_fuzz = generic_pass + + def defaulting_select(selector): + """Pass through to select() with special semantics when converting to CMake. diff --git a/third_party/non_module_deps.bzl b/third_party/non_module_deps.bzl index 4a1434df..4c513371 100644 --- a/third_party/non_module_deps.bzl +++ b/third_party/non_module_deps.bzl @@ -1,6 +1,7 @@ load("//third_party/arocc:repo.bzl", arocc = "repo") load("//third_party/cloud_accelerator_diagnostics:repo.bzl", cloud_accelerator_diagnostics = "repo") load("//third_party/com_google_sentencepiece:repo.bzl", com_google_sentencepiece = "repo") +load("//third_party/iree:repo.bzl", iree = "repo") load("//third_party/mnist:repo.bzl", mnist = "repo") load("//third_party/org_swig_swig:repo.bzl", org_swig_swig = "repo") load("//third_party/translate-c:repo.bzl", translate_c = "repo") @@ -26,6 +27,7 @@ def _non_module_deps_impl(mctx): zigimg() uucode() libvaxis() + iree() return mctx.extension_metadata( reproducible = True, diff --git a/third_party/xla/llvm_raw_only.bzl b/third_party/xla/llvm_raw_only.bzl new file mode 100644 index 00000000..f2758994 --- /dev/null +++ b/third_party/xla/llvm_raw_only.bzl @@ -0,0 +1,13 @@ +load("@xla//third_party/llvm:workspace.bzl", llvm = "repo") + +def _llvm_raw_only_impl(mctx): + llvm("llvm-raw") + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = ["llvm-raw"], + root_module_direct_dev_deps = [], + ) + +llvm_raw_only = module_extension( + implementation = _llvm_raw_only_impl, +) diff --git a/zml/tokenizer/BUILD.bazel b/zml/tokenizer/BUILD.bazel index b00dd2ef..92082ff5 100644 --- a/zml/tokenizer/BUILD.bazel +++ b/zml/tokenizer/BUILD.bazel @@ -12,6 +12,7 @@ zig_library( "//ffi:zig", "//stdx", "//zml/tokenizer/hftokenizers", + "//zml/tokenizer/iree", "//zml/tokenizer/sentencepiece", ], ) @@ -28,11 +29,8 @@ zig_binary( zig_test( name = "test", - main = "homemade.zig", - visibility = ["//visibility:public"], deps = [ - "//ffi:zig", - "//stdx", + ":tokenizer" ], ) diff --git a/zml/tokenizer/hftokenizers/.gitignore b/zml/tokenizer/hftokenizers/.gitignore new file mode 100644 index 00000000..2f7896d1 --- /dev/null +++ b/zml/tokenizer/hftokenizers/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/zml/tokenizer/hftokenizers/hftokenizers.zig b/zml/tokenizer/hftokenizers/hftokenizers.zig index 0871d6f9..f5829ce6 100644 --- a/zml/tokenizer/hftokenizers/hftokenizers.zig +++ b/zml/tokenizer/hftokenizers/hftokenizers.zig @@ -113,7 +113,7 @@ pub const HFTokenizer = opaque { return Decoder.init(self); } - pub fn tokenToId(self: *HFTokenizer, token: []const u8) ?u32 { + pub fn tokenId(self: *HFTokenizer, token: []const u8) ?u32 { const id = c.hftokenizers_token_to_id(@ptrCast(self), ffi.ZigSlice.from(token)); return if (id == std.math.maxInt(u32)) null else id; } diff --git a/zml/tokenizer/homemade.zig b/zml/tokenizer/homemade.zig index bd16d183..c14feb1a 100644 --- a/zml/tokenizer/homemade.zig +++ b/zml/tokenizer/homemade.zig @@ -125,7 +125,7 @@ pub const Tokenizer = struct { return self.token_lookup.get(str); } - pub fn tokenToId(self: *const Tokenizer, token: []const u8) ?u32 { + pub fn tokenId(self: *const Tokenizer, token: []const u8) ?u32 { return self.token_lookup.get(token); } diff --git a/zml/tokenizer/iree/BUILD.bazel b/zml/tokenizer/iree/BUILD.bazel new file mode 100644 index 00000000..ebed7df4 --- /dev/null +++ b/zml/tokenizer/iree/BUILD.bazel @@ -0,0 +1,20 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "iree", + srcs = ["tokenizer.zig"], + main = "tokenizer.zig", + visibility = ["//visibility:public"], + deps = [ + "@iree//runtime/src/iree/tokenizer", + "@iree//runtime/src/iree/tokenizer/format/huggingface:tokenizer_json", + "@iree//runtime/src/iree/tokenizer/vocab:vocab", + ], +) + +zig_test( + name = "test", + deps = [ + ":iree", + ], +) diff --git a/zml/tokenizer/iree/tokenizer.zig b/zml/tokenizer/iree/tokenizer.zig new file mode 100644 index 00000000..09e72f73 --- /dev/null +++ b/zml/tokenizer/iree/tokenizer.zig @@ -0,0 +1,405 @@ +const std = @import("std"); + +const c = @import("c"); + +pub const Error = error{ + Cancelled, + Unknown, + InvalidArgument, + DeadlineExceeded, + NotFound, + AlreadyExists, + PermissionDenied, + ResourceExhausted, + FailedPrecondition, + Aborted, + OutOfRange, + Unimplemented, + Internal, + Unavailable, + DataLoss, + Unauthenticated, +}; + +const StatusCodeMask = 0x1F; + +inline fn stringView(s: []const u8) c.iree_string_view_t { + return .{ .data = s.ptr, .size = s.len }; +} + +inline fn mutableStringView(s: []u8) c.iree_mutable_string_view_t { + return .{ .data = @ptrCast(s.ptr), .size = s.len }; +} + +inline fn byteSpan(bytes: []u8) c.iree_byte_span_t { + return .{ .data = bytes.ptr, .data_length = bytes.len }; +} + +inline fn tokenOutput(ids: []c.iree_tokenizer_token_id_t) c.iree_tokenizer_token_output_t { + return .{ + .capacity = ids.len, + .token_ids = ids.ptr, + .token_offsets = null, + .type_ids = null, + }; +} + +inline fn tokenIdList(ids: []const c.iree_tokenizer_token_id_t) c.iree_tokenizer_token_id_list_t { + return .{ .count = ids.len, .values = ids.ptr }; +} + +inline fn statusCode(status: c.iree_status_t) u32 { + return @intCast(@intFromPtr(status.?) & StatusCodeMask); +} + +fn assertOk(status: c.iree_status_t) !void { + if (status == null) return; + const code = statusCode(status); + _ = c.iree_status_ignore(status); + return switch (code) { + c.IREE_STATUS_CANCELLED => Error.Cancelled, + c.IREE_STATUS_UNKNOWN => Error.Unknown, + c.IREE_STATUS_INVALID_ARGUMENT => Error.InvalidArgument, + c.IREE_STATUS_DEADLINE_EXCEEDED => Error.DeadlineExceeded, + c.IREE_STATUS_NOT_FOUND => Error.NotFound, + c.IREE_STATUS_ALREADY_EXISTS => Error.AlreadyExists, + c.IREE_STATUS_PERMISSION_DENIED => Error.PermissionDenied, + c.IREE_STATUS_RESOURCE_EXHAUSTED => Error.ResourceExhausted, + c.IREE_STATUS_FAILED_PRECONDITION => Error.FailedPrecondition, + c.IREE_STATUS_ABORTED => Error.Aborted, + c.IREE_STATUS_OUT_OF_RANGE => Error.OutOfRange, + c.IREE_STATUS_UNIMPLEMENTED => Error.Unimplemented, + c.IREE_STATUS_INTERNAL => Error.Internal, + c.IREE_STATUS_UNAVAILABLE => Error.Unavailable, + c.IREE_STATUS_DATA_LOSS => Error.DataLoss, + c.IREE_STATUS_UNAUTHENTICATED => Error.Unauthenticated, + else => Error.Internal, + }; +} + +pub const Tokenizer = struct { + const EncodeBatchSize = 256; + const DecodeOutputBatchSize = c.IREE_TOKENIZER_DECODE_OUTPUT_RECOMMENDED_SIZE; + const DecodeTokenBatchSize = 256; + + allocator: std.mem.Allocator, + inner: *c.iree_tokenizer_t, + + pub fn fromHuggingFaceJson(allocator: std.mem.Allocator, json: []const u8) !Tokenizer { + var raw: ?*c.iree_tokenizer_t = null; + try assertOk(c.iree_tokenizer_from_huggingface_json(stringView(json), c.iree_allocator_system(), &raw)); + return .{ + .allocator = allocator, + .inner = raw.?, + }; + } + + pub fn fromBytes(allocator: std.mem.Allocator, bytes: []const u8) !Tokenizer { + return fromHuggingFaceJson(allocator, bytes); + } + + pub fn fromFile(allocator: std.mem.Allocator, io: std.Io, model: []const u8) !Tokenizer { + const json = try std.Io.Dir.cwd().readFileAlloc(io, model, allocator, .unlimited); + defer allocator.free(json); + return try fromHuggingFaceJson(allocator, json); + } + + pub fn deinit(self: *Tokenizer) void { + c.iree_tokenizer_free(self.inner); + } + + pub fn encoder(self: *const Tokenizer) !Encoder { + return Encoder.init(self); + } + + pub fn decoder(self: *const Tokenizer) !Decoder { + return Decoder.init(self); + } + + pub fn tokenId(self: *const Tokenizer, token: []const u8) ?u32 { + const vocab = c.iree_tokenizer_vocab(self.inner); + if (vocab == null) return null; + + // Looks up a string in the vocabulary. + // Returns the token ID if found, or -1 if not found. + const id = c.iree_tokenizer_vocab_lookup(vocab, stringView(token)); + return if (id < 0) null else @intCast(id); + } + + pub const Encoder = struct { + tokenizer: *const Tokenizer, + state_storage: []u8, + transform_buffer: []u8, + state: *c.iree_tokenizer_encode_state_t, + output: [EncodeBatchSize]c.iree_tokenizer_token_id_t, + output_token_ids: [EncodeBatchSize]u32, + + fn init(tokenizer: *const Tokenizer) !Encoder { + var state_size: usize = undefined; + try assertOk(c.iree_tokenizer_encode_state_calculate_size(tokenizer.inner, &state_size)); + const state_storage = try tokenizer.allocator.alloc(u8, state_size); + errdefer tokenizer.allocator.free(state_storage); + + const transform_size: usize = c.IREE_TOKENIZER_TRANSFORM_BUFFER_MAX_SIZE; + const transform_buffer = try tokenizer.allocator.alloc(u8, transform_size); + errdefer tokenizer.allocator.free(transform_buffer); + + var state: ?*c.iree_tokenizer_encode_state_t = null; + try assertOk(c.iree_tokenizer_encode_state_initialize( + tokenizer.inner, + byteSpan(state_storage), + byteSpan(transform_buffer), + c.iree_tokenizer_offset_run_list_empty(), + c.IREE_TOKENIZER_ENCODE_FLAG_AT_INPUT_START, + &state, + )); + + return .{ + .tokenizer = tokenizer, + .state_storage = state_storage, + .transform_buffer = transform_buffer, + .state = state.?, + .output = undefined, + .output_token_ids = undefined, + }; + } + + pub fn deinit(self: *Encoder) void { + c.iree_tokenizer_encode_state_deinitialize(self.state); + self.tokenizer.allocator.free(self.state_storage); + self.tokenizer.allocator.free(self.transform_buffer); + } + + pub fn reset(self: *Encoder) void { + c.iree_tokenizer_encode_state_reset(self.state, c.IREE_TOKENIZER_ENCODE_FLAG_AT_INPUT_START); + } + + pub fn feed(self: *Encoder, chunk: []const u8) !struct { + usize, + []const u32, + } { + var consumed: usize = 0; + var produced: usize = 0; + + // Feeds an input |chunk| to the encoder. Tokens that can be definitively + // produced are written to |output|. Additional tokens may be produced by + // subsequent feed calls or by finalize once all input has been provided. + // + // Pull-based model: Output buffer capacity drives processing. The encoder + // pulls data through the pipeline (normalizer → segmenter → model) to + // fill the output buffer. Processing stops when output is full or input + // is exhausted. + // + // Returns: + // - |out_bytes_consumed|: Bytes consumed from |chunk|. May be less than + // chunk.size if internal buffers (transform_buffer) fill up. Caller should + // retry with remaining bytes: chunk.data + *out_bytes_consumed. + // - |out_token_count|: Tokens written to |output|. + // + // The function always makes progress if possible: it will consume input + // and/or produce tokens. Returns iree_ok_status() even if not all input + // was consumed (check out_bytes_consumed). + try assertOk(c.iree_tokenizer_encode_state_feed( + self.state, + stringView(chunk), + tokenOutput(&self.output), + &consumed, + &produced, + )); + + return .{ + consumed, + @as([*]const u32, @ptrCast(&self.output))[0..produced], + }; + } + + pub fn finalize(self: *Encoder) ![]const u32 { + var produced: usize = 0; + + // Finalizes encoding by flushing any buffered data through the pipeline. Must + // be called after all input chunks have been fed. Any remaining tokens are + // written to |output|. + // + // Returns the number of tokens written in |out_token_count|. Returns + // IREE_STATUS_RESOURCE_EXHAUSTED if the output would exceed output.capacity. + try assertOk(c.iree_tokenizer_encode_state_finalize( + self.state, + tokenOutput(&self.output), + &produced, + )); + + return @as([*]const u32, @ptrCast(&self.output))[0..produced]; + } + }; + + pub const Decoder = struct { + tokenizer: *const Tokenizer, + state_storage: []u8, + state: *c.iree_tokenizer_decode_state_t, + output: [DecodeOutputBatchSize]u8, + input_ids: [DecodeTokenBatchSize]c.iree_tokenizer_token_id_t, + + fn init(tokenizer: *const Tokenizer) (Error || std.mem.Allocator.Error)!Decoder { + var state_size: usize = undefined; + try assertOk(c.iree_tokenizer_decode_state_calculate_size(tokenizer.inner, &state_size)); + const state_storage = try tokenizer.allocator.alloc(u8, state_size); + errdefer tokenizer.allocator.free(state_storage); + + var state: ?*c.iree_tokenizer_decode_state_t = null; + try assertOk(c.iree_tokenizer_decode_state_initialize( + tokenizer.inner, + c.IREE_TOKENIZER_DECODE_FLAG_NONE, + byteSpan(state_storage), + &state, + )); + + return .{ + .tokenizer = tokenizer, + .state_storage = state_storage, + .state = state.?, + .output = undefined, + .input_ids = undefined, + }; + } + + pub fn deinit(self: *Decoder) void { + c.iree_tokenizer_decode_state_deinitialize(self.state); + self.tokenizer.allocator.free(self.state_storage); + } + + pub fn reset(self: *Decoder) !void { + c.iree_tokenizer_decode_state_deinitialize(self.state); + var state: ?*c.iree_tokenizer_decode_state_t = null; + try assertOk(c.iree_tokenizer_decode_state_initialize( + self.tokenizer.inner, + c.IREE_TOKENIZER_DECODE_FLAG_NONE, + byteSpan(self.state_storage), + &state, + )); + self.state = state.?; + } + + pub fn feed(self: *Decoder, token_ids_: []const u32) !struct { + usize, + []const u8, + } { + const token_ids: []const c.iree_tokenizer_token_id_t = + @as([*]const i32, @ptrCast(token_ids_.ptr))[0..token_ids_.len]; + + var consumed: usize = 0; + var written: usize = 0; + + // Feeds |tokens| to the decoder. Text that can be definitively produced is + // written to |text_output| as raw UTF-8 bytes (not NUL-terminated). Additional + // text may be produced by subsequent feed calls or by finalize once all tokens + // have been provided. + // + // Pull-based model: Output buffer capacity drives processing. The decoder + // pulls tokens through the pipeline to fill the text output. Processing + // stops when output is full or tokens are exhausted. + // + // Returns iree_ok_status() always (errors are only from internal pipeline + // failures). Progress is indicated by the output parameters: + // - |out_tokens_consumed|: Tokens consumed from |tokens|. May be less than + // tokens.count if text output fills up. Caller should retry with remaining + // tokens: tokens.values + *out_tokens_consumed. + // - |out_text_length|: Bytes written to |text_output|. + // + // Zero-progress case: If both out_tokens_consumed and out_text_length are 0, + // the output buffer is genuinely exhausted (the next token's text does not fit + // in |text_output|). Callers must check for this to avoid infinite loops. + // Provide a larger output buffer or consume the already-written text. + try assertOk(c.iree_tokenizer_decode_state_feed( + self.state, + tokenIdList(token_ids), + mutableStringView(&self.output), + &consumed, + &written, + )); + + return .{ + consumed, + self.output[0..written], + }; + } + + pub fn finalize(self: *Decoder) ![]const u8 { + var written: usize = 0; + + // Finalizes decoding by flushing any buffered data through the pipeline. Must + // be called after all tokens have been fed. Any remaining text is written to + // |text_output| as raw UTF-8 bytes (not NUL-terminated). + // + // Returns the number of bytes written in |out_text_length|. Returns + // IREE_STATUS_RESOURCE_EXHAUSTED if the output would exceed text_output.size. + try assertOk(c.iree_tokenizer_decode_state_finalize( + self.state, + mutableStringView(&self.output), + &written, + )); + return self.output[0..written]; + } + }; +}; + +test "huggingface json streaming encode/decode" { + const allocator = std.testing.allocator; + const json = + \\{ + \\ "model": { + \\ "type": "WordPiece", + \\ "unk_token": "[UNK]", + \\ "continuing_subword_prefix": "##", + \\ "max_input_chars_per_word": 100, + \\ "vocab": { + \\ "[UNK]": 0, + \\ "hello": 1, + \\ "world": 2 + \\ } + \\ }, + \\ "pre_tokenizer": {"type": "Whitespace"}, + \\ "decoder": {"type": "WordPiece", "prefix": "##", "cleanup": false} + \\} + ; + + var tokenizer = try Tokenizer.fromHuggingFaceJson(allocator, json); + defer tokenizer.deinit(); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const text = "hello world"; + + var token_ids: std.ArrayList(u32) = .empty; + defer token_ids.deinit(allocator); + var remaining_txt: []const u8 = text; + while (remaining_txt.len > 0) { + const consumed, const ids = try encoder.feed(remaining_txt); + try token_ids.appendSlice(allocator, ids); + remaining_txt = remaining_txt[consumed..]; + } + try token_ids.appendSlice(allocator, try encoder.finalize()); + + try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, token_ids.items); + + const token_hello = tokenizer.tokenId("hello") orelse unreachable; + try std.testing.expectEqual(@as(u32, 1), token_hello); + + const unknown = tokenizer.tokenId("missing"); + try std.testing.expect(unknown == null); + + var decoder = try tokenizer.decoder(); + defer decoder.deinit(); + + var tokens: std.ArrayList(u8) = .empty; + defer tokens.deinit(allocator); + var remaining_ids: []const u32 = token_ids.items; + while (remaining_ids.len > 0) { + const consumed, const chunk = try decoder.feed(remaining_ids); + try tokens.appendSlice(allocator, chunk); + remaining_ids = remaining_ids[consumed..]; + } + try tokens.appendSlice(allocator, try decoder.finalize()); + + try std.testing.expectEqualStrings("hello world", tokens.items); +} diff --git a/zml/tokenizer/main.zig b/zml/tokenizer/main.zig index 76039b89..8388a11b 100644 --- a/zml/tokenizer/main.zig +++ b/zml/tokenizer/main.zig @@ -34,15 +34,17 @@ pub fn main(init: std.process.Init) !void { var decoder = try tokenizer.decoder(); defer decoder.deinit(); - const prompt_tok = try encoder.encode(args.prompt); + var prompt_tok = try encoder.encodeAlloc(allocator, args.prompt); + defer prompt_tok.deinit(allocator); - log.info("Input: {s}\nOutput: {any}", .{ args.prompt, prompt_tok }); + log.info("Input: {s}\nOutput: {any}", .{ args.prompt, prompt_tok.items }); var errors: u8 = 0; { - const reconstructed = try decoder.decode(prompt_tok); - if (!std.mem.eql(u8, args.prompt, reconstructed)) { - log.err("Reconstructed string from tokens doesn't match source: {s}", .{reconstructed}); + var reconstructed = try decoder.decodeAlloc(allocator, prompt_tok.items); + defer reconstructed.deinit(allocator); + if (!std.mem.eql(u8, args.prompt, reconstructed.items)) { + log.err("Reconstructed string from tokens doesn't match source: {s}", .{reconstructed.items}); errors += 1; } } @@ -54,7 +56,7 @@ pub fn main(init: std.process.Init) !void { const tok = try std.fmt.parseInt(u32, int_token, 10); try expected.append(allocator, tok); } - if (!std.mem.eql(u32, expected.items, prompt_tok)) { + if (!std.mem.eql(u32, expected.items, prompt_tok.items)) { log.err("Doesn't match expected: {any}", .{expected.items}); errors += 1; } diff --git a/zml/tokenizer/sentencepiece/sentencepiece.zig b/zml/tokenizer/sentencepiece/sentencepiece.zig index c98b0740..b3749583 100644 --- a/zml/tokenizer/sentencepiece/sentencepiece.zig +++ b/zml/tokenizer/sentencepiece/sentencepiece.zig @@ -192,7 +192,7 @@ pub const SentencePieceProcessor = opaque { return try Decoder.init(self); } - pub fn tokenToId(self: *SentencePieceProcessor, token: []const u8) u32 { + pub fn tokenId(self: *SentencePieceProcessor, token: []const u8) u32 { return @intCast(c.SentencePieceProcessor_PieceToId(@ptrCast(self), ffi.ZigSlice.from(token))); } }; diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig index 9c563daa..e63a3560 100644 --- a/zml/tokenizer/tokenizer.zig +++ b/zml/tokenizer/tokenizer.zig @@ -1,16 +1,16 @@ const std = @import("std"); pub const hftokenizers = @import("hftokenizers"); +pub const iree = @import("iree"); pub const sentencepiece = @import("sentencepiece"); pub const homemade = @import("homemade.zig"); -test { - std.testing.refAllDecls(@This()); -} +const log = std.log.scoped(.@"zml/tokenizer"); const Tokenizers = enum { hftokenizers, + iree, sentencepiece, homemade, }; @@ -18,6 +18,7 @@ const Tokenizers = enum { pub const Tokenizer = union(Tokenizers) { pub const Encoder = union(Tokenizers) { hftokenizers: hftokenizers.Encoder, + iree: iree.Tokenizer.Encoder, sentencepiece: sentencepiece.Encoder, homemade: homemade.Encoder, @@ -33,21 +34,148 @@ pub const Tokenizer = union(Tokenizers) { } } - pub fn encode(self: *Encoder, input: []const u8) ![]const u32 { + // + // Convenience methods built on top of the stream API + // + + pub fn encodeAlloc(self: *Encoder, allocator: std.mem.Allocator, tokens: []const u8) !std.ArrayList(u32) { + var token_ids: std.ArrayList(u32) = try .initCapacity(allocator, tokens.len / 4); + try self.encodeAppend(allocator, &token_ids, tokens); + return token_ids; + } + + pub fn encodeAppend(self: *Encoder, allocator: std.mem.Allocator, out: *std.ArrayList(u32), tokens: []const u8) !void { + var remaining = tokens; + while (remaining.len > 0) { + const fout = try self.feed(remaining); + try out.appendSlice(allocator, fout.token_ids); + remaining = remaining[fout.consumed..]; + } + try out.appendSlice(allocator, try self.finalize()); + } + + // + // Stream API + // + + pub const FeedOutput = struct { + consumed: usize, + /// Only valid until the next feed/finalize call and while the encoder is alive. + token_ids: []const u32, + }; + + pub fn feed(self: *Encoder, tokens: []const u8) !FeedOutput { return switch (self.*) { - inline else => |*v| v.encode(input), + .iree => |*v| { + const consumed, const token_ids = try v.feed(tokens); + return .{ .consumed = consumed, .token_ids = token_ids }; + }, + inline else => |*v| { + const token_ids = try v.encode(tokens); + return .{ .consumed = tokens.len, .token_ids = token_ids }; + }, }; } - pub fn ids(self: Encoder) []const u32 { - return switch (self) { - inline else => |v| v.ids(), + /// Output is only valid until the next feed/finalize call and while the encoder is alive. + pub fn finalize(self: *Encoder) ![]const u32 { + return switch (self.*) { + .iree => |*v| v.finalize(), + inline else => &.{}, }; } + + /// Writes the token ids as u8 into the out Writer. + /// Remember to call the finalize() method! + pub fn writer(self: *Encoder, allocator: std.mem.Allocator, buffer: []u8, out: *std.Io.Writer) !Writer { + return try Writer.init(allocator, self, buffer, out); + } + + pub const Writer = struct { + _allocator: std.mem.Allocator, + _encoder: *Tokenizer.Encoder, + _out: *std.Io.Writer, + interface: std.Io.Writer, + + pub fn init(allocator: std.mem.Allocator, encoder_: *Encoder, buffer: []u8, out: *std.Io.Writer) !Writer { + return .{ + ._allocator = allocator, + ._encoder = encoder_, + ._out = out, + .interface = .{ + .buffer = buffer, + .vtable = &.{ + .drain = drain, + .flush = flush, + .rebase = std.Io.Writer.failingRebase, + }, + }, + }; + } + + pub fn finalize(self: *Writer) std.Io.Writer.Error!void { + try self.interface.flush(); + const token_ids = self._encoder.finalize() catch |err| { + log.err("writer drain finalize failed: {s}", .{@errorName(err)}); + return error.WriteFailed; + }; + self._out.writeAll(std.mem.sliceAsBytes(token_ids)) catch |err| { + log.err("writer drain append failed: {s}", .{@errorName(err)}); + return err; + }; + } + + fn appendChunk(self: *Writer, chunk: []const u8) std.Io.Writer.Error!void { + var remaining = chunk; + while (remaining.len > 0) { + const fout = self._encoder.feed(remaining) catch |err| { + log.err("writer drain feed failed: {s}", .{@errorName(err)}); + return error.WriteFailed; + }; + self._out.writeAll(std.mem.sliceAsBytes(fout.token_ids)) catch |err| { + log.err("writer drain append failed: {s}", .{@errorName(err)}); + return err; + }; + remaining = remaining[fout.consumed..]; + } + } + + fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { + if (w.end == 0) return; + const self: *Writer = @alignCast(@fieldParentPtr("interface", w)); + try self.appendChunk(w.buffer[0..w.end]); + w.end = 0; + } + + fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + const self: *Writer = @alignCast(@fieldParentPtr("interface", w)); + + if (w.end != 0) { + try self.appendChunk(w.buffer[0..w.end]); + w.end = 0; + } + + var total: usize = 0; + // Prefix chunks: written once each. + for (data[0 .. data.len - 1]) |chunk| { + try self.appendChunk(chunk); + total += chunk.len; + } + // Last chunk is the splat pattern. + const pattern = data[data.len - 1]; + if (pattern.len == 0) return total; + for (0..splat) |_| { + try self.appendChunk(pattern); + total += pattern.len; + } + return total; + } + }; }; pub const Decoder = union(Tokenizers) { hftokenizers: hftokenizers.Decoder, + iree: iree.Tokenizer.Decoder, sentencepiece: sentencepiece.Decoder, homemade: homemade.Decoder, @@ -63,81 +191,158 @@ pub const Tokenizer = union(Tokenizers) { } } - pub fn decode(self: *Decoder, ids_: []const u32) ![]const u8 { + // + // Convenience methods built on top of the stream API + // + + pub fn decodeAlloc(self: *Decoder, allocator: std.mem.Allocator, token_ids: []const u32) !std.ArrayList(u8) { + var tokens: std.ArrayList(u8) = try .initCapacity(allocator, token_ids.len * 4); + var remaining = token_ids; + while (remaining.len > 0) { + const fout = try self.feed(remaining); + try tokens.appendSlice(allocator, fout.tokens); + remaining = remaining[fout.consumed..]; + } + try tokens.appendSlice(allocator, try self.finalize()); + return tokens; + } + + // + // Stream API + // + + pub const FeedOutput = struct { + consumed: usize, + /// Only valid until the next feed/finalize call and while the decoder is alive. + tokens: []const u8, + }; + + pub fn feed(self: *Decoder, token_ids: []const u32) !FeedOutput { return switch (self.*) { - inline else => |*v| v.decode(ids_), + .iree => |*v| { + const consumed, const tokens = try v.feed(token_ids); + return .{ .consumed = consumed, .tokens = tokens }; + }, + inline else => |*v| { + const tokens = try v.decode(token_ids); + return .{ .consumed = token_ids.len, .tokens = tokens }; + }, }; } - pub fn string(self: *Decoder) []const u8 { - return switch (self.*) { - inline else => |v| v.string(), - }; + // Only valid until the next decode and while the decoder is alive. + pub fn feed_one(self: *Decoder, token_id: u32) ![]const u8 { + const fout = try self.feed(&.{token_id}); + return fout.tokens; } - //pub fn ids(self: *Decoder) []u32 { - // return switch (self.*) { - // inline else => |v| v.ids(), - // }; - //} - - pub fn next(self: *Decoder, token_id: u32) !?[]const u8 { + /// Output is only valid until the next feed/finalize call and while the decoder is alive. + pub fn finalize(self: *Decoder) ![]const u8 { return switch (self.*) { - inline else => |*v| v.next(token_id), + .iree => |*v| v.finalize(), + inline else => &.{}, }; } }; hftokenizers: *hftokenizers.HFTokenizer, + iree: iree.Tokenizer, sentencepiece: *sentencepiece.SentencePieceProcessor, homemade: *homemade.Tokenizer, pub fn fromFile(allocator: std.mem.Allocator, io: std.Io, model: []const u8) !Tokenizer { - _ = allocator; - _ = io; if (std.mem.endsWith(u8, model, ".pb")) { return .{ .sentencepiece = try .fromFile(model) }; } if (std.mem.endsWith(u8, model, ".json")) { - return .{ .hftokenizers = try .fromFile(model) }; + return .{ .iree = try .fromFile(allocator, io, model) }; } return error.InvalidArgument; } - pub fn fromBytes(allocator: std.mem.Allocator, io: std.Io, bytes: []const u8) !Tokenizer { - _ = allocator; - _ = io; + pub fn fromBytes(allocator: std.mem.Allocator, bytes: []const u8) !Tokenizer { if (bytes[0] == '{') { - return .{ .hftokenizers = try .fromBytes(bytes) }; + return .{ .iree = try .fromBytes(allocator, bytes) }; } return .{ .sentencepiece = try .fromBytes(bytes) }; } pub fn deinit(self: *Tokenizer) void { switch (self.*) { - inline else => |t| t.deinit(), + inline else => |*t| t.*.deinit(), } } - pub fn encoder(self: Tokenizer) !Encoder { - return switch (self) { - inline else => |v, tag| @unionInit(Encoder, @tagName(tag), try v.encoder()), + pub fn encoder(self: *const Tokenizer) !Encoder { + return switch (self.*) { + inline else => |*v, tag| @unionInit(Encoder, @tagName(tag), try v.*.encoder()), }; } - pub fn decoder(self: Tokenizer) !Decoder { - return switch (self) { - inline else => |v, tag| @unionInit(Decoder, @tagName(tag), try v.decoder()), + pub fn decoder(self: *const Tokenizer) !Decoder { + return switch (self.*) { + inline else => |*v, tag| @unionInit(Decoder, @tagName(tag), try v.*.decoder()), }; } - pub fn tokenToId(self: Tokenizer, token: []const u8) ?u32 { - return switch (self) { - inline else => |v| v.tokenToId(token), + pub fn tokenId(self: *const Tokenizer, token: []const u8) ?u32 { + return switch (self.*) { + inline else => |v| v.tokenId(token), }; } }; -test {} -test {} +test { + std.testing.refAllDecls(@This()); +} + +test "iree tokenizer smoke: alloc and writer" { + const allocator = std.testing.allocator; + const json = + \\{ + \\ "model": { + \\ "type": "WordPiece", + \\ "unk_token": "[UNK]", + \\ "continuing_subword_prefix": "##", + \\ "max_input_chars_per_word": 100, + \\ "vocab": { + \\ "[UNK]": 0, + \\ "hello": 1, + \\ "world": 2 + \\ } + \\ }, + \\ "pre_tokenizer": {"type": "Whitespace"}, + \\ "decoder": {"type": "WordPiece", "prefix": "##", "cleanup": false} + \\} + ; + + var tokenizer = try Tokenizer.fromBytes(allocator, json); + defer tokenizer.deinit(); + + const text = "hello world"; + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + var token_ids_alloc = try encoder.encodeAlloc(allocator, text); + defer token_ids_alloc.deinit(allocator); + try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, token_ids_alloc.items); + + encoder.reset(); + var token_ids_writer: std.Io.Writer.Allocating = .initAligned(allocator, std.mem.Alignment.of(u32)); + defer token_ids_writer.deinit(); + var writer = try encoder.writer(allocator, &.{}, &token_ids_writer.writer); + try writer.interface.writeAll(text); + try writer.finalize(); + + const token_ids: []u32 = @alignCast(std.mem.bytesAsSlice(u32, token_ids_writer.written())); + try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, token_ids); + + var decoder = try tokenizer.decoder(); + defer decoder.deinit(); + + var tokens = try decoder.decodeAlloc(allocator, token_ids); + defer tokens.deinit(allocator); + try std.testing.expectEqualStrings("hello world", tokens.items); +} From 57546912f125eb0947c504ab1a228da8cab64591 Mon Sep 17 00:00:00 2001 From: elogir <90246794+elogir@users.noreply.github.com> Date: Tue, 7 Apr 2026 16:44:52 +0200 Subject: [PATCH 407/881] bin/zml-smi: Add CSV export and driver version reporting (#452) - Restructured `bin/zml-smi/` - Added CSV export - Added driver version reporting - Minor fixes --- BUILD.bazel | 1 + bin/zml-smi/BUILD.bazel | 60 +---- bin/zml-smi/bindings/dynlib.zig | 131 ---------- bin/zml-smi/bindings/neuron/nrt.zig | 128 ---------- bin/zml-smi/bindings/neuron/process.zig | 110 --------- bin/zml-smi/bindings/nvml/process.zig | 87 ------- bin/zml-smi/collector/BUILD.bazel | 13 + bin/zml-smi/{ => collector}/collector.zig | 44 +++- bin/zml-smi/collector/worker.zig | 25 ++ bin/zml-smi/csv/BUILD.bazel | 11 + bin/zml-smi/csv/csv.zig | 53 ++++ bin/zml-smi/info/BUILD.bazel | 15 ++ bin/zml-smi/info/device_info.zig | 25 +- bin/zml-smi/info/host_info.zig | 2 +- bin/zml-smi/info/info.zig | 9 + bin/zml-smi/info/poll_metrics.zig | 18 ++ bin/zml-smi/main.zig | 95 +++++--- bin/zml-smi/platform.zig | 36 --- bin/zml-smi/platforms/amdsmi/BUILD.bazel | 21 ++ .../{bindings => platforms}/amdsmi/amdsmi.zig | 54 +++-- .../amdsmi/metrics.zig | 30 ++- .../amdsmi/process.zig | 85 +++---- bin/zml-smi/platforms/linux/BUILD.bazel | 17 ++ bin/zml-smi/platforms/linux/linux.zig | 2 + .../{bindings => platforms}/linux/metrics.zig | 17 +- .../{bindings => platforms}/linux/process.zig | 6 +- bin/zml-smi/platforms/neuron/BUILD.bazel | 22 ++ .../neuron/metrics.zig | 53 +++- bin/zml-smi/platforms/neuron/nrt.zig | 229 ++++++++++++++++++ bin/zml-smi/platforms/neuron/process.zig | 108 +++++++++ bin/zml-smi/platforms/nvml/BUILD.bazel | 19 ++ .../{bindings => platforms}/nvml/metrics.zig | 44 +++- .../{bindings => platforms}/nvml/nvml.zig | 90 ++++--- bin/zml-smi/platforms/nvml/process.zig | 75 ++++++ bin/zml-smi/platforms/tpu/BUILD.bazel | 30 +++ .../{bindings => platforms}/tpu/metrics.zig | 59 ++--- .../{bindings => platforms}/tpu/process.zig | 31 +-- .../tpu/libtpu => platforms/tpu}/tpuinfo.cc | 0 .../tpu/libtpu => platforms/tpu}/tpuinfo.h | 0 .../{bindings => platforms}/tpu/tpuinfo.zig | 0 bin/zml-smi/tui/BUILD.bazel | 50 ++++ bin/zml-smi/tui/data.zig | 13 +- bin/zml-smi/tui/pages/detail/gpu_detail.zig | 14 +- .../tui/pages/detail/neuron_detail.zig | 6 +- bin/zml-smi/tui/print.zig | 5 - bin/zml-smi/tui/top.zig | 166 +++++++++++++ bin/zml-smi/tui/tui.zig | 169 +------------ bin/zml-smi/tui/widgets/device_card.zig | 26 +- bin/zml-smi/tui/widgets/info_lines.zig | 5 +- bin/zml-smi/tui/widgets/process_table.zig | 2 +- bin/zml-smi/utils/BUILD.bazel | 30 +++ bin/zml-smi/utils/dynlib.zig | 10 + bin/zml-smi/worker.zig | 62 ----- platforms/neuron/BUILD.bazel | 2 +- platforms/neuron/nrt.h | 1 + tools/workspace_status.sh | 2 + 56 files changed, 1383 insertions(+), 1035 deletions(-) delete mode 100644 bin/zml-smi/bindings/dynlib.zig delete mode 100644 bin/zml-smi/bindings/neuron/nrt.zig delete mode 100644 bin/zml-smi/bindings/neuron/process.zig delete mode 100644 bin/zml-smi/bindings/nvml/process.zig create mode 100644 bin/zml-smi/collector/BUILD.bazel rename bin/zml-smi/{ => collector}/collector.zig (50%) create mode 100644 bin/zml-smi/collector/worker.zig create mode 100644 bin/zml-smi/csv/BUILD.bazel create mode 100644 bin/zml-smi/csv/csv.zig create mode 100644 bin/zml-smi/info/BUILD.bazel create mode 100644 bin/zml-smi/info/info.zig create mode 100644 bin/zml-smi/info/poll_metrics.zig delete mode 100644 bin/zml-smi/platform.zig create mode 100644 bin/zml-smi/platforms/amdsmi/BUILD.bazel rename bin/zml-smi/{bindings => platforms}/amdsmi/amdsmi.zig (80%) rename bin/zml-smi/{bindings => platforms}/amdsmi/metrics.zig (81%) rename bin/zml-smi/{bindings => platforms}/amdsmi/process.zig (54%) create mode 100644 bin/zml-smi/platforms/linux/BUILD.bazel create mode 100644 bin/zml-smi/platforms/linux/linux.zig rename bin/zml-smi/{bindings => platforms}/linux/metrics.zig (86%) rename bin/zml-smi/{bindings => platforms}/linux/process.zig (96%) create mode 100644 bin/zml-smi/platforms/neuron/BUILD.bazel rename bin/zml-smi/{bindings => platforms}/neuron/metrics.zig (75%) create mode 100644 bin/zml-smi/platforms/neuron/nrt.zig create mode 100644 bin/zml-smi/platforms/neuron/process.zig create mode 100644 bin/zml-smi/platforms/nvml/BUILD.bazel rename bin/zml-smi/{bindings => platforms}/nvml/metrics.zig (75%) rename bin/zml-smi/{bindings => platforms}/nvml/nvml.zig (73%) create mode 100644 bin/zml-smi/platforms/nvml/process.zig create mode 100644 bin/zml-smi/platforms/tpu/BUILD.bazel rename bin/zml-smi/{bindings => platforms}/tpu/metrics.zig (74%) rename bin/zml-smi/{bindings => platforms}/tpu/process.zig (65%) rename bin/zml-smi/{bindings/tpu/libtpu => platforms/tpu}/tpuinfo.cc (100%) rename bin/zml-smi/{bindings/tpu/libtpu => platforms/tpu}/tpuinfo.h (100%) rename bin/zml-smi/{bindings => platforms}/tpu/tpuinfo.zig (100%) create mode 100644 bin/zml-smi/tui/BUILD.bazel create mode 100644 bin/zml-smi/tui/top.zig create mode 100644 bin/zml-smi/utils/BUILD.bazel create mode 100644 bin/zml-smi/utils/dynlib.zig delete mode 100644 bin/zml-smi/worker.zig diff --git a/BUILD.bazel b/BUILD.bazel index 94dc6e48..be48a7a8 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -4,6 +4,7 @@ zls_completion( name = "completion", visibility = ["//visibility:public"], deps = [ + "//bin/zml-smi", "//examples/llm", "//examples/mnist", "//examples/sharding", diff --git a/bin/zml-smi/BUILD.bazel b/bin/zml-smi/BUILD.bazel index 84651d34..fc803eb2 100644 --- a/bin/zml-smi/BUILD.bazel +++ b/bin/zml-smi/BUILD.bazel @@ -1,24 +1,12 @@ load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") load("@bazel_skylib//rules:run_binary.bzl", "run_binary") -load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load") -load("@rules_zig//zig:defs.bzl", "zig_binary") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") load("@tar.bzl//tar:mtree.bzl", "mtree_spec") load("@tar.bzl//tar:tar.bzl", "tar") -# tpu grpc - -cc_library( - name = "tpuinfo", - srcs = ["bindings/tpu/libtpu/tpuinfo.cc"], - hdrs = ["bindings/tpu/libtpu/tpuinfo.h"], - strip_include_prefix = "bindings/tpu/libtpu", - deps = [ - "@cloud_accelerator_diagnostics//:tpu_metric_service_cc_grpc", - "@grpc//:grpc++_unsecure", - ], -) +# zig libraries # smi @@ -75,50 +63,26 @@ copy_to_directory( zig_binary( name = "zml-smi", - srcs = glob([ - "tui/**/*.zig", - ]) + [ - "bindings/amdsmi/amdsmi.zig", - "bindings/amdsmi/metrics.zig", - "bindings/amdsmi/process.zig", - "bindings/dynlib.zig", - "bindings/linux/metrics.zig", - "bindings/linux/process.zig", - "bindings/neuron/metrics.zig", - "bindings/neuron/nrt.zig", - "bindings/neuron/process.zig", - "bindings/nvml/metrics.zig", - "bindings/nvml/nvml.zig", - "bindings/nvml/process.zig", - "bindings/tpu/metrics.zig", - "bindings/tpu/process.zig", - "bindings/tpu/tpuinfo.zig", - "collector.zig", - "info/device_info.zig", - "info/host_info.zig", - "info/process_info.zig", - "platform.zig", - "utils/double_buffer.zig", - "utils/sandbox.zig", - "utils/sysfs.zig", - "worker.zig", - ], + visibility = ["//visibility:public"], data = [":sandbox"], - extra_srcs = glob([ - "tui/assets/*", - ]), main = "main.zig", deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/csv", + "//bin/zml-smi/platforms/linux", + "//bin/zml-smi/info", + "//bin/zml-smi/tui", "//stdx", "@libvaxis//:vaxis", "@rules_zig//zig/runfiles", ] + select({ "@platforms//os:macos": [], "//conditions:default": [ - ":tpuinfo", + "//bin/zml-smi/platforms/amdsmi", + "//bin/zml-smi/platforms/neuron", + "//bin/zml-smi/platforms/nvml", + "//bin/zml-smi/platforms/tpu", "//platforms/neuron:libnrt_headers", - "@amd-smi-lib//:amdsmi", - "@cuda_nvml_dev//:nvml", ], }), ) diff --git a/bin/zml-smi/bindings/dynlib.zig b/bin/zml-smi/bindings/dynlib.zig deleted file mode 100644 index 2cea9205..00000000 --- a/bin/zml-smi/bindings/dynlib.zig +++ /dev/null @@ -1,131 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); -const elf = std.elf; -const c = @import("c"); - -pub fn Fn(comptime name: [:0]const u8) type { - return *const @TypeOf(@field(c, name)); -} - -pub fn open(comptime F: type, path: [:0]const u8) ?F { - const handle = openDl(path) orelse return null; - return resolveAll(F, handle, null, 0); -} - -pub fn openElf(comptime F: type, io: std.Io, path: [:0]const u8, comptime known_sym: [:0]const u8) !F { - const handle = openDl(path) orelse { - if (std.c.dlerror()) |err| std.log.err("dlopen: {s}", .{err}); - return error.DlOpenFailed; - }; - - const sym_addr = std.c.dlsym(handle, known_sym) orelse - return error.KnownSymbolNotFound; - - var info: DlInfo = undefined; - if (dladdr(sym_addr, &info) == 0) { - return error.DladdrFailed; - } - - const elf_base = @intFromPtr(info.dli_fbase); - const elf_path = std.mem.sliceTo(info.dli_fname, 0); - const file = try std.Io.Dir.openFileAbsolute(io, elf_path, .{}); - defer file.close(io); - - const stat = try file.stat(io); - const mmap = try std.posix.mmap( - null, - @intCast(stat.size), - .{ .READ = true }, - .{ .TYPE = .PRIVATE }, - file.handle, - 0, - ); - defer std.posix.munmap(mmap); - - return resolveAll(F, handle, mmap, elf_base) orelse error.SymbolResolutionFailed; -} - -fn resolveAll(comptime F: type, handle: *anyopaque, elf_mmap: ?[]align(std.heap.page_size_min) u8, elf_base: usize) ?F { - var result: F = undefined; - var unresolved: std.hash_map.StringHashMapUnmanaged(*usize) = .{}; - defer unresolved.deinit(std.heap.page_allocator); - - // Phase 1: try dlsym for each field; for unresolved ones, store a pointer to the result field - inline for (std.meta.fields(F)) |field| { - if (std.c.dlsym(handle, field.name)) |p| { - @field(result, field.name) = @as(field.type, @ptrFromInt(@intFromPtr(p))); - } else { - unresolved.put(std.heap.page_allocator, field.name, @ptrCast(&@field(result, field.name))) catch return null; - } - } - - if (unresolved.count() == 0) { // no need to parse ELF - return result; - } - - // Phase 2: scan ELF symbols, write resolved addresses directly into result fields - const buf = elf_mmap orelse return null; - const ehdr: *const elf.ElfN.Ehdr = @ptrCast(@alignCast(buf.ptr)); - const shdrs: [*]const elf.ElfN.Shdr = @ptrCast(@alignCast(buf.ptr + @as(usize, ehdr.shoff))); - - var remaining = unresolved.count(); - - outer: for (0..ehdr.shnum) |i| { - const sh = shdrs[i]; - if (sh.type != .SYMTAB) { - continue; - } - - const strtab_sh = shdrs[sh.link]; - const strtab: [*]const u8 = @ptrCast(buf.ptr + @as(usize, strtab_sh.offset)); - const syms: []const elf.ElfN.Sym = @as( - [*]const elf.ElfN.Sym, - @ptrCast(@alignCast(buf.ptr + @as(usize, sh.offset))), - )[0 .. @as(usize, sh.size) / @sizeOf(elf.ElfN.Sym)]; - - for (syms) |s| { - if (s.value == 0) { - continue; - } - - const sym_name = std.mem.sliceTo(@as([*:0]const u8, @ptrCast(strtab + @as(usize, s.name))), 0); - if (unresolved.get(sym_name)) |field_ptr| { - field_ptr.* = elf_base + @as(usize, s.value); - remaining -= 1; - - if (remaining == 0) { - break :outer; - } - } - } - } - - if (remaining > 0) { - return null; - } - - return result; -} - -fn openDl(path: [:0]const u8) ?*anyopaque { - return switch (builtin.os.tag) { - .linux, .macos => std.c.dlopen(path, switch (builtin.os.tag) { - .linux => .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }, - .macos => .{ .LAZY = true, .LOCAL = true }, - else => unreachable, - }), - else => blk: { - const dl = std.DynLib.open(path) catch break :blk null; - break :blk dl.inner.handle; - }, - }; -} - -const DlInfo = extern struct { - dli_fname: [*:0]const u8, - dli_fbase: *anyopaque, - dli_sname: [*c]const u8, - dli_saddr: ?*anyopaque, -}; - -extern fn dladdr(addr: *anyopaque, info: *DlInfo) c_int; diff --git a/bin/zml-smi/bindings/neuron/nrt.zig b/bin/zml-smi/bindings/neuron/nrt.zig deleted file mode 100644 index 5e1e82cf..00000000 --- a/bin/zml-smi/bindings/neuron/nrt.zig +++ /dev/null @@ -1,128 +0,0 @@ -const std = @import("std"); -const c = @import("c"); -const stdx = @import("stdx"); -const DynLib = @import("../dynlib.zig"); -const sandbox = @import("../../utils/sandbox.zig"); - -const Nrt = @This(); - -pub const Error = error{ nrt_error, NrtUnavailable }; -pub const AppInfo = c.struct_neuron_app_info; -pub const DeviceType = enum(c_int) { - inf1 = 1, - inf2_trn1 = 2, - trn2 = 3, - trn3 = 4, - _, -}; - -lib: Fns, -handles: []const *c.ndl_device_t, -device_indexes: []const c_int, - -const Fns = struct { - ndl_available_devices: DynLib.Fn("ndl_available_devices"), - ndl_open_device: DynLib.Fn("ndl_open_device"), - ndl_close_device: DynLib.Fn("ndl_close_device"), - ndl_get_all_apps_info: DynLib.Fn("ndl_get_all_apps_info"), - nds_open: DynLib.Fn("nds_open"), - nds_close: DynLib.Fn("nds_close"), - nds_get_nc_counter: DynLib.Fn("nds_get_nc_counter"), - nrt_get_total_nc_count: DynLib.Fn("nrt_get_total_nc_count"), -}; - -pub fn init(allocator: std.mem.Allocator, io: std.Io) Error!Nrt { - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = sandbox.path(&path_buf) orelse { - std.log.err("neuron: sandbox path unavailable", .{}); - return error.NrtUnavailable; - }; - - var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libnrt.so.1" }) catch { - std.log.err("neuron: failed to construct libnrt.so.1 path", .{}); - return error.NrtUnavailable; - }; - - const lib = DynLib.openElf(Fns, io, path, "nrt_init") catch |err| { - std.log.err("neuron: failed to open {s}: {s}", .{ path, @errorName(err) }); - return error.NrtUnavailable; - }; - - var dev_index_buf: [c.MAX_NEURON_DEVICE_COUNT]c_int = undefined; - const count: usize = @intCast(@max(0, lib.ndl_available_devices(&dev_index_buf, c.MAX_NEURON_DEVICE_COUNT))); - - var handle_list: std.ArrayList(*c.ndl_device_t) = .{}; - var index_list: std.ArrayList(c_int) = .{}; - - for (dev_index_buf[0..count]) |device_idx| { - var dev: ?*c.ndl_device_t = null; - var t: c.struct_ndl_device_init_param = .{ - .initialize_device = false, - .map_hbm = false, - .num_dram_regions = 0, - }; - - if (lib.ndl_open_device(device_idx, &t, &dev) == 0) { - if (dev) |d| { - handle_list.append(allocator, d) catch continue; - index_list.append(allocator, device_idx) catch continue; - } - } - } - - return .{ - .lib = lib, - .handles = handle_list.toOwnedSlice(allocator) catch return error.NrtUnavailable, - .device_indexes = index_list.toOwnedSlice(allocator) catch return error.NrtUnavailable, - }; -} - -pub fn deviceCount(self: Nrt) u32 { - return @intCast(self.handles.len); -} - -pub fn allAppsInfo(self: Nrt, dev: *c.ndl_device_t) Error!struct { ptr: ?[*]AppInfo, count: usize } { - var info: ?[*]AppInfo = null; - var count: usize = 0; - if (self.lib.ndl_get_all_apps_info(dev, &info, &count, c.APP_INFO_ALL) != 0) { - return error.nrt_error; - } - return .{ .ptr = info, .count = count }; -} - -pub fn ndsOpen(self: Nrt, dev: *c.ndl_device_t, pid: c.pid_t) Error!*c.nds_instance_t { - var inst: ?*c.nds_instance_t = null; - if (self.lib.nds_open(dev, pid, &inst) != 0) { - return error.nrt_error; - } - return inst orelse error.nrt_error; -} - -pub fn ndsClose(self: Nrt, inst: *c.nds_instance_t) void { - _ = self.lib.nds_close(inst); -} - -pub fn ncCounter(self: Nrt, inst: *c.nds_instance_t, pnc_index: c_int, counter_index: u32) Error!u64 { - var value: u64 = 0; - if (self.lib.nds_get_nc_counter(inst, pnc_index, counter_index, &value) != 0) { - return error.nrt_error; - } - return value; -} - -pub fn totalNcCount(self: Nrt) Error!u32 { - var count: u32 = 0; - if (self.lib.nrt_get_total_nc_count(&count) != 0) { - return error.nrt_error; - } - return count; -} - -pub fn hbmSize(dev: *c.ndl_device_t) usize { - return dev.hbm_size; -} - -pub fn deviceType(dev: *c.ndl_device_t) DeviceType { - return @enumFromInt(dev.device_type); -} diff --git a/bin/zml-smi/bindings/neuron/process.zig b/bin/zml-smi/bindings/neuron/process.zig deleted file mode 100644 index bd895ec2..00000000 --- a/bin/zml-smi/bindings/neuron/process.zig +++ /dev/null @@ -1,110 +0,0 @@ -const std = @import("std"); -const c = @import("c"); -const Nrt = @import("nrt.zig"); -const pi = @import("../../info/process_info.zig"); -const DeviceInfo = @import("../../info/device_info.zig").DeviceInfo; -const ProcessDoubleBuffer = @import("../../utils/double_buffer.zig").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); -const Worker = @import("../../worker.zig").Worker; - -pub fn init(w: *Worker, io: std.Io, allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, nrt: *const Nrt, nc_per_device: u32, device_infos: []*DeviceInfo, dev_offset: u8) !void { - try w.spawn(io, pollLoop, .{ io, w, allocator, list, nrt, nc_per_device, device_infos, dev_offset }); -} - -fn pollLoop(io: std.Io, w: *const Worker, allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, nrt: *const Nrt, nc_per_device: u32, device_infos: []*DeviceInfo, dev_offset: u8) void { - const interval: std.Io.Duration = .fromMilliseconds(w.poll_interval_ms); - io.sleep(interval, .awake) catch {}; - - if (nrt.handles.len == 0) { - return; - } - - while (w.isRunning()) { - const start: std.Io.Timestamp = .now(io, .awake); - const back = list.back(); - - back.clearRetainingCapacity(); - - for (nrt.handles, 0..) |handle, dev_i| { - const apps_result = nrt.allAppsInfo(handle) catch continue; - defer { - if (apps_result.ptr) |ptr| { - std.c.free(@ptrCast(ptr)); - } - } - - const apps = if (apps_result.ptr) |ptr| ptr[0..apps_result.count] else continue; - - var total_us_per_core: [c.MAX_NC_PER_DEVICE]u64 = .{0} ** c.MAX_NC_PER_DEVICE; - - for (apps) |*app| { - if (app.pid <= 0) { - continue; - } - - const nds = nrt.ndsOpen(handle, app.pid) catch continue; - defer nrt.ndsClose(nds); - - for (0..nc_per_device) |ci| { - const time_in_use = nrt.ncCounter(nds, @intCast(ci), c.NDS_NC_COUNTER_TIME_IN_USE) catch continue; - total_us_per_core[ci] += time_in_use; - - if (time_in_use == 0) { - continue; - } - - const dev_idx: u32 = @intCast(dev_i * nc_per_device + ci); - - var info: pi.ProcessInfo = .{ - .pid = @bitCast(app.pid), - .device_idx = @intCast(dev_idx + dev_offset), - .dev_mem_kib = @intCast(app.device_mem_size / 1024), - }; - - if (dev_idx < device_infos.len) { - if (device_infos[dev_idx].neuron.front().util_percent) |util| { - info.dev_util_percent = @intCast(util); - } - } - - if (info.dev_mem_kib != null and info.dev_mem_kib.? > 0) - back.append(allocator, info) catch break; - } - } - - const now: std.Io.Timestamp = .now(io, .awake); - - for (0..nc_per_device) |ci| { - const dev_idx = dev_i * nc_per_device + ci; - if (dev_idx >= device_infos.len) { - break; - } - - const ni_back = device_infos[dev_idx].neuron.back(); - ni_back.* = device_infos[dev_idx].neuron.front().*; - const prev_us = ni_back.prev_total_us; - - ni_back.prev_total_us = total_us_per_core[ci]; - - if (ni_back.prev_timestamp) |prev| { - const elapsed = prev.untilNow(io, .awake); - const elapsed_us: u64 = @intCast(@divFloor(elapsed.nanoseconds, std.time.ns_per_us)); - - if (elapsed_us > 0) { - const delta_us = total_us_per_core[ci] -| prev_us; - ni_back.util_percent = @min(100, delta_us * 100 / elapsed_us); - } - } - - ni_back.prev_timestamp = now; - device_infos[dev_idx].neuron.swap(); - } - } - - list.swap(); - - const elapsed = start.untilNow(io, .awake); - if (elapsed.nanoseconds < interval.nanoseconds) { - io.sleep(.fromNanoseconds(interval.nanoseconds - elapsed.nanoseconds), .awake) catch {}; - } - } -} diff --git a/bin/zml-smi/bindings/nvml/process.zig b/bin/zml-smi/bindings/nvml/process.zig deleted file mode 100644 index 510d72fe..00000000 --- a/bin/zml-smi/bindings/nvml/process.zig +++ /dev/null @@ -1,87 +0,0 @@ -const std = @import("std"); -const Nvml = @import("nvml.zig"); -const pi = @import("../../info/process_info.zig"); -const ProcessDoubleBuffer = @import("../../utils/double_buffer.zig").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); -const Worker = @import("../../worker.zig").Worker; - -pub fn init(w: *Worker, io: std.Io, allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, nvml: *const Nvml, dev_offset: u8) !void { - try w.spawn(io, pollLoop, .{ io, w, allocator, list, nvml, dev_offset }); -} - -fn pollLoop(io: std.Io, w: *const Worker, allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, nvml: *const Nvml, dev_offset: u8) void { - const interval: std.Io.Duration = .fromMilliseconds(w.poll_interval_ms); - io.sleep(interval, .awake) catch {}; - - const device_count = nvml.deviceCount() catch 0; - const last_seen_ts = allocator.alloc(u64, device_count) catch return; - defer allocator.free(last_seen_ts); - @memset(last_seen_ts, 0); - - while (w.isRunning()) { - const start: std.Io.Timestamp = .now(io, .awake); - const back = list.back(); - - back.clearRetainingCapacity(); - - for (0..device_count) |dev_idx| { - const handle = nvml.handleByIndex(@intCast(dev_idx)) catch continue; - const idx: u8 = @intCast(dev_idx + dev_offset); - - const compute = nvml.computeRunningProcesses(allocator, handle) catch &.{}; - defer { - if (compute.len > 0) { - allocator.free(compute); - } - } - collectFromQuery(allocator, back, idx, compute); - - const graphics = nvml.graphicsRunningProcesses(allocator, handle) catch &.{}; - defer { - if (graphics.len > 0) { - allocator.free(graphics); - } - } - collectFromQuery(allocator, back, idx, graphics); - - // Apply utilization samples - const last_ts = last_seen_ts[dev_idx]; - const utils = nvml.processUtilization(allocator, handle, last_ts) catch continue; - defer { - if (utils.len > 0) { - allocator.free(utils); - } - } - - for (utils) |sample| { - if (sample.smUtil > 100 or sample.timeStamp <= last_ts) { - continue; - } - - last_seen_ts[dev_idx] = @max(last_seen_ts[dev_idx], sample.timeStamp); - - for (back.items) |*entry| { - if (entry.pid == sample.pid and entry.device_idx == idx) { - entry.dev_util_percent = @intCast(sample.smUtil); - } - } - } - } - - list.swap(); - - const elapsed = start.untilNow(io, .awake); - if (elapsed.nanoseconds < interval.nanoseconds) { - io.sleep(.fromNanoseconds(interval.nanoseconds - elapsed.nanoseconds), .awake) catch {}; - } - } -} - -fn collectFromQuery(allocator: std.mem.Allocator, back: *std.ArrayList(pi.ProcessInfo), dev_idx: u8, procs: []const Nvml.ProcessInfo_t) void { - for (procs) |gp| { - back.append(allocator, .{ - .pid = gp.pid, - .device_idx = dev_idx, - .dev_mem_kib = @intCast(gp.usedGpuMemory / 1024), - }) catch return; - } -} diff --git a/bin/zml-smi/collector/BUILD.bazel b/bin/zml-smi/collector/BUILD.bazel new file mode 100644 index 00000000..0bf5fa8e --- /dev/null +++ b/bin/zml-smi/collector/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "collector", + import_name = "zml-smi/collector", + main = "collector.zig", + srcs = ["worker.zig"], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + ], +) diff --git a/bin/zml-smi/collector.zig b/bin/zml-smi/collector/collector.zig similarity index 50% rename from bin/zml-smi/collector.zig rename to bin/zml-smi/collector/collector.zig index 16ded554..d98b204d 100644 --- a/bin/zml-smi/collector.zig +++ b/bin/zml-smi/collector/collector.zig @@ -1,7 +1,8 @@ const std = @import("std"); -const DeviceInfo = @import("info/device_info.zig").DeviceInfo; -const pi = @import("info/process_info.zig"); -const ProcessDoubleBuffer = @import("utils/double_buffer.zig").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const pi = smi_info.process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); const Worker = @import("worker.zig").Worker; pub const Collector = struct { @@ -10,8 +11,19 @@ pub const Collector = struct { poll_arenas: std.ArrayList(*std.heap.ArenaAllocator) = .empty, arena: std.mem.Allocator, gpa: std.mem.Allocator, - worker: *Worker, + worker: Worker, io: std.Io, + poll_only: bool, + + pub fn init(arena: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, options: struct { poll_interval_ms: u16, poll_only: bool = false }) Collector { + return .{ + .arena = arena, + .gpa = gpa, + .io = io, + .worker = .{ .poll_interval_ms = options.poll_interval_ms }, + .poll_only = options.poll_only, + }; + } pub fn addDevice(self: *Collector, initial: DeviceInfo) !*DeviceInfo { const info = try self.arena.create(DeviceInfo); @@ -37,7 +49,31 @@ pub const Collector = struct { return poll_arena; } + pub fn spawnPoll(self: *Collector, comptime pollOnce: anytype, args: std.meta.ArgsTuple(@TypeOf(pollOnce))) !void { + @call(.auto, pollOnce, args); + if (self.poll_only) return; + + const Args = std.meta.ArgsTuple(@TypeOf(pollOnce)); + try self.worker.spawn(self.io, struct { + fn f(io: std.Io, w: *const Worker, a: Args) void { + const interval: std.Io.Duration = .fromMilliseconds(w.poll_interval_ms); + while (w.isRunning()) { + const start: std.Io.Timestamp = .now(io, .awake); + + @call(.auto, pollOnce, a); + + const elapsed = start.untilNow(io, .awake); + if (elapsed.nanoseconds < interval.nanoseconds) { + io.sleep(.fromNanoseconds(interval.nanoseconds - elapsed.nanoseconds), .awake) catch {}; + } + } + } + }.f, .{ self.io, &self.worker, args }); + } + pub fn deinit(self: *Collector) void { + self.worker.shutdown(self.io); + for (self.device_infos.items) |info| { self.arena.destroy(info); } diff --git a/bin/zml-smi/collector/worker.zig b/bin/zml-smi/collector/worker.zig new file mode 100644 index 00000000..125440c5 --- /dev/null +++ b/bin/zml-smi/collector/worker.zig @@ -0,0 +1,25 @@ +const std = @import("std"); + +pub const Worker = struct { + poll_interval_ms: u16, + should_stop: std.atomic.Value(bool) = .init(false), + group: std.Io.Group = .init, + + pub fn shutdown(self: *Worker, io: std.Io) void { + self.should_stop.store(true, .release); + self.group.await(io) catch {}; + } + + pub fn isRunning(self: *const Worker) bool { + return !self.should_stop.load(.acquire); + } + + pub fn spawn( + self: *Worker, + io: std.Io, + comptime runFn: anytype, + args: std.meta.ArgsTuple(@TypeOf(runFn)), + ) !void { + try self.group.concurrent(io, runFn, args); + } +}; diff --git a/bin/zml-smi/csv/BUILD.bazel b/bin/zml-smi/csv/BUILD.bazel new file mode 100644 index 00000000..a0cb3422 --- /dev/null +++ b/bin/zml-smi/csv/BUILD.bazel @@ -0,0 +1,11 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "csv", + import_name = "zml-smi/csv", + main = "csv.zig", + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/info", + ], +) diff --git a/bin/zml-smi/csv/csv.zig b/bin/zml-smi/csv/csv.zig new file mode 100644 index 00000000..4e11b398 --- /dev/null +++ b/bin/zml-smi/csv/csv.zig @@ -0,0 +1,53 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; + +pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo) !void { + inline for (@typeInfo(DeviceInfo).@"union".fields) |tp| { + const tag = @field(smi_info.Target, tp.name); + const Inner = resolveType(tp.type); + var header_printed = false; + + for (devices, 0..) |dev, i| { + switch (dev.*) { + tag => |*db| { + if (!header_printed) { + try writer.writeAll("index,type"); + inline for (@typeInfo(Inner).@"struct".fields) |f| { + try writer.writeAll("," ++ f.name); + } + try writer.writeAll("\n"); + + header_printed = true; + } + + const val = db.front().*; + + try writer.print("{d},{s}", .{ i, tp.name }); + inline for (@typeInfo(Inner).@"struct".fields) |f| { + try writer.writeAll(","); + try writeValue(writer, @field(val, f.name)); + } + try writer.writeAll("\n"); + }, + else => {}, + } + } + } +} + +fn writeValue(writer: *std.Io.Writer, field: anytype) !void { + if (@typeInfo(@TypeOf(field)) == .optional) { + if (field) |v| return writeValue(writer, v); + } else switch (@typeInfo(@TypeOf(field))) { + .pointer => return writer.writeAll(field), + else => return writer.print("{any}", .{field}), + } +} + +fn resolveType(comptime DB: type) type { + for (@typeInfo(DB).@"struct".fields) |df| { + if (std.mem.eql(u8, df.name, "values")) return @typeInfo(df.type).array.child; + } + unreachable; +} diff --git a/bin/zml-smi/info/BUILD.bazel b/bin/zml-smi/info/BUILD.bazel new file mode 100644 index 00000000..d035b161 --- /dev/null +++ b/bin/zml-smi/info/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "info", + import_name = "zml-smi/info", + main = "info.zig", + srcs = [ + "device_info.zig", + "host_info.zig", + "poll_metrics.zig", + "process_info.zig", + ], + visibility = ["//visibility:public"], + deps = ["//bin/zml-smi/utils:double_buffer"], +) diff --git a/bin/zml-smi/info/device_info.zig b/bin/zml-smi/info/device_info.zig index 5a52272e..e84fb88e 100644 --- a/bin/zml-smi/info/device_info.zig +++ b/bin/zml-smi/info/device_info.zig @@ -1,27 +1,11 @@ const std = @import("std"); -const DoubleBuffer = @import("../utils/double_buffer.zig").DoubleBuffer; +const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; pub const Target = enum { cuda, rocm, neuron, tpu, - - pub fn deviceLabel(self: Target) []const u8 { - return switch (self) { - .cuda, .rocm => "GPU", - .neuron => "NC", - .tpu => "TPU", - }; - } - - pub fn utilLabel(self: Target) []const u8 { - return switch (self) { - .cuda, .rocm => "GPU", - .neuron => "Core", - .tpu => "Duty", - }; - } }; pub const DeviceInfo = union(Target) { @@ -33,6 +17,8 @@ pub const DeviceInfo = union(Target) { pub const GpuInfo = struct { name: ?[]const u8 = null, + driver_version: ?[]const u8 = null, + cuda_driver_version: ?[]const u8 = null, // Utilization util_percent: ?u64 = null, @@ -69,15 +55,12 @@ pub const GpuInfo = struct { pub const NeuronInfo = struct { name: ?[]const u8 = null, + driver_version: ?[]const u8 = null, util_percent: ?u64 = null, mem_used_bytes: ?u64 = null, mem_total_bytes: ?u64 = null, - // Utilization delta tracking (written by metrics worker) - prev_total_us: u64 = 0, - prev_timestamp: ?std.Io.Timestamp = null, - // Neuron per-core HBM breakdown (bytes) nc_tensors: ?u64 = null, nc_constants: ?u64 = null, diff --git a/bin/zml-smi/info/host_info.zig b/bin/zml-smi/info/host_info.zig index 7b824fef..f33f6fd8 100644 --- a/bin/zml-smi/info/host_info.zig +++ b/bin/zml-smi/info/host_info.zig @@ -1,4 +1,4 @@ -const DoubleBuffer = @import("../utils/double_buffer.zig").DoubleBuffer; +const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; pub const HostInfo = DoubleBuffer(HostData); diff --git a/bin/zml-smi/info/info.zig b/bin/zml-smi/info/info.zig new file mode 100644 index 00000000..42e1b1b3 --- /dev/null +++ b/bin/zml-smi/info/info.zig @@ -0,0 +1,9 @@ +const std = @import("std"); + +pub const device_info = @import("device_info.zig"); +pub const host_info = @import("host_info.zig"); +pub const poll_metrics = @import("poll_metrics.zig"); +pub const process_info = @import("process_info.zig"); + +pub const Target = device_info.Target; +pub const Targets = std.EnumSet(Target); diff --git a/bin/zml-smi/info/poll_metrics.zig b/bin/zml-smi/info/poll_metrics.zig new file mode 100644 index 00000000..f0d916ff --- /dev/null +++ b/bin/zml-smi/info/poll_metrics.zig @@ -0,0 +1,18 @@ +const std = @import("std"); + +pub fn poll(comptime DB: type, comptime Dev: type, comptime table: anytype) fn (?*std.heap.ArenaAllocator, DB, Dev) void { + return struct { + fn f(arena: ?*std.heap.ArenaAllocator, db: DB, dev: Dev) void { + if (arena) |a| _ = a.reset(.retain_capacity); + + const back = db.back(); + back.* = db.front().*; + + inline for (table) |m| { + @field(back, m.field) = m.query(dev) catch null; + } + + db.swap(); + } + }.f; +} diff --git a/bin/zml-smi/main.zig b/bin/zml-smi/main.zig index 6328b21d..f7284587 100644 --- a/bin/zml-smi/main.zig +++ b/bin/zml-smi/main.zig @@ -9,25 +9,30 @@ pub const std_options: std.Options = .{ }, }; -const Collector = @import("collector.zig").Collector; -const HostInfo = @import("info/host_info.zig").HostInfo; -const ProcessEnricher = @import("bindings/linux/process.zig").ProcessEnricher; -const platform = @import("platform.zig"); -const Worker = @import("worker.zig").Worker; -const host = @import("bindings/linux/metrics.zig"); -const tui = @import("tui/tui.zig"); -const static_print = @import("tui/print.zig"); -const data = @import("tui/data.zig"); +const Collector = @import("zml-smi/collector").Collector; +const smi_info = @import("zml-smi/info"); +const HostInfo = smi_info.host_info.HostInfo; +const linux = @import("zml-smi/platforms/linux"); +const ProcessEnricher = linux.process.ProcessEnricher; +const host = linux.metrics; +const c = @import("c"); +const csv = @import("zml-smi/csv"); +const smi_tui = @import("zml-smi/tui"); +const tui = smi_tui.top; +const static_print = smi_tui.print; +const data = smi_tui.data; const CliArgs = struct { top: bool = false, + csv: bool = false, tui_refresh_rate: u16 = 100, poll_interval: u16 = 500, pub const help = - \\ zml-smi [--top] [--tui-refresh-rate MS] [--poll-interval MS] + \\ zml-smi [--top] [--csv] [--tui-refresh-rate MS] [--poll-interval MS] \\ \\ --top Interactive TUI mode + \\ --csv Output device metrics as CSV \\ --tui-refresh-rate TUI refresh rate in ms (default: 100) \\ --poll-interval Device polling interval in ms (default: 500) \\ @@ -35,10 +40,10 @@ const CliArgs = struct { }; const device_backends = if (builtin.os.tag != .macos) .{ - @import("bindings/nvml/metrics.zig"), - @import("bindings/amdsmi/metrics.zig"), - @import("bindings/neuron/metrics.zig"), - @import("bindings/tpu/metrics.zig"), + .{ .cuda, @import("zml-smi/platforms/nvml") }, + .{ .rocm, @import("zml-smi/platforms/amdsmi") }, + .{ .neuron, @import("zml-smi/platforms/neuron") }, + .{ .tpu, @import("zml-smi/platforms/tpu") }, } else .{}; pub fn main(init: std.process.Init) !void { @@ -47,29 +52,25 @@ pub fn main(init: std.process.Init) !void { const io = init.io; const args = stdx.flags.parse(init.minimal.args, CliArgs); - var w: Worker = .{ .poll_interval_ms = args.poll_interval }; + const targets = detect(io); - const targets = platform.detect(io); - - var collector: Collector = .{ - .arena = arena, - .gpa = gpa, - .worker = &w, - .io = io, - }; + var collector: Collector = .init(arena, gpa, io, .{ + .poll_interval_ms = args.poll_interval, + .poll_only = !args.top, + }); defer collector.deinit(); - defer w.shutdown(io); var host_info: HostInfo = .{ .values = .{ .{}, .{} } }; - try host.init(&w, io, &collector, &host_info); + try host.init(&collector, &host_info); var enricher: ProcessEnricher = try .init(gpa, io); defer enricher.deinit(); - inline for (device_backends) |backend| { - if (targets.contains(backend.target)) { + inline for (device_backends) |entry| { + const target, const backend = entry; + if (targets.contains(target)) { backend.start(&collector) catch |err| { - std.log.err("{s} skipped: {s}", .{ @tagName(backend.target), @errorName(err) }); + std.log.err("{s} skipped: {s}", .{ @tagName(target), @errorName(err) }); }; } } @@ -85,9 +86,45 @@ pub fn main(init: std.process.Init) !void { }); defer state.deinit(arena); - if (args.top) { + if (args.csv) { + var csv_buf: [4096]u8 = undefined; + var csv_writer = std.Io.File.stdout().writer(io, &csv_buf); + + try csv.write(&csv_writer.interface, collector.device_infos.items); + try csv_writer.flush(); + } else if (args.top) { try tui.run(gpa, io, &state); } else { try static_print.run(arena, io, &state); } } + +fn detect(io: std.Io) smi_info.Targets { + if (comptime builtin.os.tag == .macos) { + return .{}; + } + + var targets: smi_info.Targets = .{}; + if (hasDevice(io, "/dev/nvidiactl")) { + targets.insert(.cuda); + } + + if (hasDevice(io, "/dev/kfd")) { + targets.insert(.rocm); + } + + if (hasDevice(io, c.NEURON_DEVICE_PREFIX ++ "0")) { + targets.insert(.neuron); + } + + if (hasDevice(io, "/dev/accel0") or hasDevice(io, "/dev/vfio/0")) { + targets.insert(.tpu); + } + + return targets; +} + +fn hasDevice(io: std.Io, path: []const u8) bool { + std.Io.Dir.accessAbsolute(io, path, .{ .read = true }) catch return false; + return true; +} diff --git a/bin/zml-smi/platform.zig b/bin/zml-smi/platform.zig deleted file mode 100644 index 4a9cb7db..00000000 --- a/bin/zml-smi/platform.zig +++ /dev/null @@ -1,36 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); -const c = @import("c"); - -pub const Target = @import("info/device_info.zig").Target; - -fn hasDevice(io: std.Io, path: []const u8) bool { - std.Io.Dir.accessAbsolute(io, path, .{ .read = true }) catch return false; - return true; -} - -pub const Targets = std.EnumSet(Target); - -pub fn detect(io: std.Io) Targets { - var targets: Targets = .{}; - - if (comptime builtin.os.tag != .macos) { - if (hasDevice(io, "/dev/nvidiactl")) { - targets.insert(.cuda); - } - - if (hasDevice(io, "/dev/kfd")) { - targets.insert(.rocm); - } - - if (hasDevice(io, c.NEURON_DEVICE_PREFIX ++ "0")) { - targets.insert(.neuron); - } - - if (hasDevice(io, "/dev/accel0") or hasDevice(io, "/dev/vfio/0")) { - targets.insert(.tpu); - } - } - - return targets; -} diff --git a/bin/zml-smi/platforms/amdsmi/BUILD.bazel b/bin/zml-smi/platforms/amdsmi/BUILD.bazel new file mode 100644 index 00000000..1543e391 --- /dev/null +++ b/bin/zml-smi/platforms/amdsmi/BUILD.bazel @@ -0,0 +1,21 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "amdsmi", + import_name = "zml-smi/platforms/amdsmi", + main = "metrics.zig", + srcs = [ + "amdsmi.zig", + "process.zig", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + "//bin/zml-smi/utils:dynlib", + "//bin/zml-smi/utils:sandbox", + "//stdx", + "@amd-smi-lib//:amdsmi", + ], +) diff --git a/bin/zml-smi/bindings/amdsmi/amdsmi.zig b/bin/zml-smi/platforms/amdsmi/amdsmi.zig similarity index 80% rename from bin/zml-smi/bindings/amdsmi/amdsmi.zig rename to bin/zml-smi/platforms/amdsmi/amdsmi.zig index 0320dc61..06656501 100644 --- a/bin/zml-smi/bindings/amdsmi/amdsmi.zig +++ b/bin/zml-smi/platforms/amdsmi/amdsmi.zig @@ -1,8 +1,8 @@ const std = @import("std"); const c = @import("c"); const stdx = @import("stdx"); -const DynLib = @import("../dynlib.zig"); -const sandbox = @import("../../utils/sandbox.zig"); +const DynLib = @import("zml-smi/dynlib"); +const sandbox = @import("zml-smi/sandbox"); const AmdSmi = @This(); @@ -14,21 +14,22 @@ lib: Fns, gpu_handles: []c.amdsmi_processor_handle, const Fns = struct { - amdsmi_init: DynLib.Fn("amdsmi_init"), - amdsmi_get_socket_handles: DynLib.Fn("amdsmi_get_socket_handles"), - amdsmi_get_processor_handles: DynLib.Fn("amdsmi_get_processor_handles"), - amdsmi_get_gpu_asic_info: DynLib.Fn("amdsmi_get_gpu_asic_info"), - amdsmi_get_gpu_metrics_info: DynLib.Fn("amdsmi_get_gpu_metrics_info"), - amdsmi_get_power_info: DynLib.Fn("amdsmi_get_power_info"), - amdsmi_get_temp_metric: DynLib.Fn("amdsmi_get_temp_metric"), - amdsmi_get_gpu_fan_speed: DynLib.Fn("amdsmi_get_gpu_fan_speed"), - amdsmi_get_gpu_activity: DynLib.Fn("amdsmi_get_gpu_activity"), - amdsmi_get_clock_info: DynLib.Fn("amdsmi_get_clock_info"), - amdsmi_get_gpu_memory_total: DynLib.Fn("amdsmi_get_gpu_memory_total"), - amdsmi_get_gpu_memory_usage: DynLib.Fn("amdsmi_get_gpu_memory_usage"), - amdsmi_get_pcie_info: DynLib.Fn("amdsmi_get_pcie_info"), - amdsmi_get_gpu_bdf_id: DynLib.Fn("amdsmi_get_gpu_bdf_id"), - amdsmi_get_gpu_process_list: DynLib.Fn("amdsmi_get_gpu_process_list"), + amdsmi_init: *const @TypeOf(c.amdsmi_init), + amdsmi_get_socket_handles: *const @TypeOf(c.amdsmi_get_socket_handles), + amdsmi_get_processor_handles: *const @TypeOf(c.amdsmi_get_processor_handles), + amdsmi_get_gpu_asic_info: *const @TypeOf(c.amdsmi_get_gpu_asic_info), + amdsmi_get_gpu_metrics_info: *const @TypeOf(c.amdsmi_get_gpu_metrics_info), + amdsmi_get_power_info: *const @TypeOf(c.amdsmi_get_power_info), + amdsmi_get_temp_metric: *const @TypeOf(c.amdsmi_get_temp_metric), + amdsmi_get_gpu_fan_speed: *const @TypeOf(c.amdsmi_get_gpu_fan_speed), + amdsmi_get_gpu_activity: *const @TypeOf(c.amdsmi_get_gpu_activity), + amdsmi_get_clock_info: *const @TypeOf(c.amdsmi_get_clock_info), + amdsmi_get_gpu_memory_total: *const @TypeOf(c.amdsmi_get_gpu_memory_total), + amdsmi_get_gpu_memory_usage: *const @TypeOf(c.amdsmi_get_gpu_memory_usage), + amdsmi_get_pcie_info: *const @TypeOf(c.amdsmi_get_pcie_info), + amdsmi_get_gpu_bdf_id: *const @TypeOf(c.amdsmi_get_gpu_bdf_id), + amdsmi_get_gpu_process_list: *const @TypeOf(c.amdsmi_get_gpu_process_list), + amdsmi_get_gpu_driver_info: *const @TypeOf(c.amdsmi_get_gpu_driver_info), }; pub fn init(allocator: std.mem.Allocator) !AmdSmi { @@ -38,7 +39,13 @@ pub fn init(allocator: std.mem.Allocator) !AmdSmi { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libamd_smi.so.26" }); - const fns = DynLib.open(Fns, path) orelse return error.AmdSmiUnavailable; + var dynlib: std.DynLib = .{ .inner = .{ + .handle = std.c.dlopen(path, .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { + if (std.c.dlerror()) |err| std.log.err("amdsmi: dlopen: {s}", .{err}); + return error.AmdSmiUnavailable; + }, + } }; + const fns = DynLib.lookupStruct(&dynlib, Fns) catch return error.AmdSmiUnavailable; try check(fns.amdsmi_init(c.AMDSMI_INIT_AMD_GPUS)); @@ -86,7 +93,7 @@ pub fn name(self: AmdSmi, handle: Handle, buf: *[c.AMDSMI_MAX_STRING_LENGTH]u8) var info: c.amdsmi_asic_info_t = undefined; try check(self.lib.amdsmi_get_gpu_asic_info(handle, &info)); @memcpy(buf, &info.market_name); - return std.mem.sliceTo(@as([*:0]const u8, @ptrCast(buf)), 0); + return std.mem.span(@as([*c]const u8, @ptrCast(buf))); } pub fn powerUsage(self: AmdSmi, handle: Handle) Error!u32 { @@ -192,6 +199,15 @@ pub fn bdfId(self: AmdSmi, handle: Handle) Error!u64 { return bdf_id; } +pub const driver_version_buf_len = @typeInfo(@TypeOf(@as(c.amdsmi_driver_info_t, undefined).driver_version)).array.len; + +pub fn driverVersion(self: AmdSmi, handle: Handle, buf: *[driver_version_buf_len]u8) Error![:0]const u8 { + var info: c.amdsmi_driver_info_t = std.mem.zeroes(c.amdsmi_driver_info_t); + try check(self.lib.amdsmi_get_gpu_driver_info(handle, &info)); + @memcpy(buf, &info.driver_version); + return std.mem.span(@as([*c]const u8, @ptrCast(buf))); +} + pub fn processList(self: AmdSmi, allocator: std.mem.Allocator, handle: Handle) (Error || error{OutOfMemory})![]const ProcInfo { var count: u32 = 0; try check(self.lib.amdsmi_get_gpu_process_list(handle, &count, null)); diff --git a/bin/zml-smi/bindings/amdsmi/metrics.zig b/bin/zml-smi/platforms/amdsmi/metrics.zig similarity index 81% rename from bin/zml-smi/bindings/amdsmi/metrics.zig rename to bin/zml-smi/platforms/amdsmi/metrics.zig index 8e724ba2..a0d4cca2 100644 --- a/bin/zml-smi/bindings/amdsmi/metrics.zig +++ b/bin/zml-smi/platforms/amdsmi/metrics.zig @@ -1,15 +1,13 @@ const std = @import("std"); const AmdSmi = @import("amdsmi.zig"); -const device_info = @import("../../info/device_info.zig"); +const device_info = @import("zml-smi/info").device_info; const DeviceInfo = device_info.DeviceInfo; const GpuInfo = device_info.GpuInfo; -const DoubleBuffer = @import("../../utils/double_buffer.zig").DoubleBuffer; -const Collector = @import("../../collector.zig").Collector; -const Worker = @import("../../worker.zig").Worker; +const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; +const Collector = @import("zml-smi/collector").Collector; +const poll_metrics = @import("zml-smi/info").poll_metrics; const process = @import("process.zig"); -pub const target: device_info.Target = .rocm; - pub fn start(collector: *Collector) !void { const amdsmi = try collector.arena.create(AmdSmi); amdsmi.* = try AmdSmi.init(collector.arena); @@ -19,17 +17,19 @@ pub fn start(collector: *Collector) !void { for (0..count) |i| { const dev = Device.open(amdsmi, @intCast(i)) catch continue; - const initial: GpuInfo = .{ .name = dev.name(collector.arena) catch null }; + const initial: GpuInfo = .{ + .name = dev.name(collector.arena) catch null, + .driver_version = dev.driverVersion(collector.arena) catch null, + }; const info = try collector.addDevice(.{ .rocm = .{ .values = .{ initial, initial } } }); - - try collector.worker.spawn(collector.io, pollDevice, .{ collector.io, collector.worker, &info.rocm, dev }); + try collector.spawnPoll(pollOnce, .{ null, &info.rocm, dev }); } const processes = try collector.createProcessList(); - try process.init(collector.worker, collector.io, collector.gpa, processes, amdsmi, dev_offset); + try process.init(collector, processes, amdsmi, dev_offset); } -const pollDevice = Worker.pollMetrics(*DoubleBuffer(GpuInfo), Device, metrics); +const pollOnce = poll_metrics.poll(*DoubleBuffer(GpuInfo), Device, metrics); const Device = struct { amdsmi: *const AmdSmi, @@ -40,12 +40,18 @@ const Device = struct { } fn name(self: Device, arena: std.mem.Allocator) ![]const u8 { - var buf: [AmdSmi.name_buf_len]u8 = .{0} ** AmdSmi.name_buf_len; + var buf: [AmdSmi.name_buf_len]u8 = undefined; const slice = try self.amdsmi.name(self.handle, &buf); return try arena.dupe(u8, slice); } + fn driverVersion(self: Device, arena: std.mem.Allocator) ![]const u8 { + var buf: [AmdSmi.driver_version_buf_len]u8 = undefined; + const slice = try self.amdsmi.driverVersion(self.handle, &buf); + return try arena.dupe(u8, slice); + } + // Power pub fn powerUsage(self: Device) !u64 { const pw = try self.amdsmi.powerUsage(self.handle); diff --git a/bin/zml-smi/bindings/amdsmi/process.zig b/bin/zml-smi/platforms/amdsmi/process.zig similarity index 54% rename from bin/zml-smi/bindings/amdsmi/process.zig rename to bin/zml-smi/platforms/amdsmi/process.zig index ecc4e379..b1952a0b 100644 --- a/bin/zml-smi/bindings/amdsmi/process.zig +++ b/bin/zml-smi/platforms/amdsmi/process.zig @@ -1,23 +1,15 @@ const std = @import("std"); const c = std.c; const AmdSmi = @import("amdsmi.zig"); -const pi = @import("../../info/process_info.zig"); -const ProcessDoubleBuffer = @import("../../utils/double_buffer.zig").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); -const Worker = @import("../../worker.zig").Worker; +const pi = @import("zml-smi/info").process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const Collector = @import("zml-smi/collector").Collector; const bdf_len = "0000:00:00.0".len; -pub fn init(w: *Worker, io: std.Io, allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, amdsmi: *const AmdSmi, dev_offset: u8) !void { - try w.spawn(io, pollLoop, .{ io, w, allocator, list, amdsmi, dev_offset }); -} - -fn pollLoop(io: std.Io, w: *const Worker, allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, amdsmi: *const AmdSmi, dev_offset: u8) void { - const interval: std.Io.Duration = .fromMilliseconds(w.poll_interval_ms); - io.sleep(interval, .awake) catch {}; - +pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, amdsmi: *const AmdSmi, dev_offset: u8) !void { const device_count = amdsmi.deviceCount(); - const pci_slots = allocator.alloc(?[bdf_len]u8, device_count) catch return; - defer allocator.free(pci_slots); + const pci_slots = try collector.arena.alloc(?[bdf_len]u8, device_count); for (pci_slots, 0..) |*slot, i| { const handle = amdsmi.handleByIndex(@intCast(i)) catch { @@ -31,56 +23,55 @@ fn pollLoop(io: std.Io, w: *const Worker, allocator: std.mem.Allocator, list: *P }); } - while (w.isRunning()) { - const start: std.Io.Timestamp = .now(io, .awake); - const back = list.back(); + try collector.spawnPoll(pollOnce, .{ collector.io, collector.gpa, list, amdsmi, dev_offset, device_count, pci_slots }); +} - back.clearRetainingCapacity(); +fn pollOnce(io: std.Io, allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, amdsmi: *const AmdSmi, dev_offset: u8, device_count: u32, pci_slots: []?[bdf_len]u8) void { + const back = list.back(); - for (0..device_count) |dev_idx| { - const handle = amdsmi.handleByIndex(@intCast(dev_idx)) catch continue; + back.clearRetainingCapacity(); - const procs = blk: { - const saved = c.dup(c.STDERR_FILENO); - const devnull = c.open("/dev/null", .{ .ACCMODE = .WRONLY }); + for (0..device_count) |dev_idx| { + const handle = amdsmi.handleByIndex(@intCast(dev_idx)) catch continue; - if (devnull >= 0) { + const procs = blk: { + const saved = c.dup(c.STDERR_FILENO); + const devnull = c.open("/dev/null", .{ .ACCMODE = .WRONLY }); + + if (devnull >= 0) { + if (saved >= 0) { _ = c.dup2(devnull, c.STDERR_FILENO); - _ = c.close(devnull); } - defer if (saved >= 0) { - _ = c.dup2(saved, c.STDERR_FILENO); - _ = c.close(saved); - }; - - break :blk amdsmi.processList(allocator, handle) catch continue; - }; - defer { - if (procs.len > 0) { - allocator.free(procs); - } + _ = c.close(devnull); } - const pci_slot = if (dev_idx < pci_slots.len) &(pci_slots[dev_idx] orelse continue) else continue; + defer if (saved >= 0) { + _ = c.dup2(saved, c.STDERR_FILENO); + _ = c.close(saved); + }; - for (procs) |proc| { - back.append(allocator, .{ - .pid = proc.pid, - .device_idx = @intCast(dev_idx + dev_offset), - .dev_util_percent = @intCast(proc.engine_usage.gfx + proc.engine_usage.enc), - .dev_mem_kib = readProcessGpuVram(io, proc.pid, pci_slot), - }) catch break; + break :blk amdsmi.processList(allocator, handle) catch continue; + }; + defer { + if (procs.len > 0) { + allocator.free(procs); } } - list.swap(); + const pci_slot = if (dev_idx < pci_slots.len) &(pci_slots[dev_idx] orelse continue) else continue; - const elapsed = start.untilNow(io, .awake); - if (elapsed.nanoseconds < interval.nanoseconds) { - io.sleep(.fromNanoseconds(interval.nanoseconds - elapsed.nanoseconds), .awake) catch {}; + for (procs) |proc| { + back.append(allocator, .{ + .pid = proc.pid, + .device_idx = @intCast(dev_idx + dev_offset), + .dev_util_percent = @intCast(proc.engine_usage.gfx + proc.engine_usage.enc), + .dev_mem_kib = readProcessGpuVram(io, proc.pid, pci_slot), + }) catch break; } } + + list.swap(); } /// Reads per-GPU VRAM from /proc//fdinfo/, filtered by PCI slot. diff --git a/bin/zml-smi/platforms/linux/BUILD.bazel b/bin/zml-smi/platforms/linux/BUILD.bazel new file mode 100644 index 00000000..57bb96d7 --- /dev/null +++ b/bin/zml-smi/platforms/linux/BUILD.bazel @@ -0,0 +1,17 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "linux", + import_name = "zml-smi/platforms/linux", + main = "linux.zig", + srcs = [ + "metrics.zig", + "process.zig", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:sysfs", + ], +) diff --git a/bin/zml-smi/platforms/linux/linux.zig b/bin/zml-smi/platforms/linux/linux.zig new file mode 100644 index 00000000..ffab3ec4 --- /dev/null +++ b/bin/zml-smi/platforms/linux/linux.zig @@ -0,0 +1,2 @@ +pub const metrics = @import("metrics.zig"); +pub const process = @import("process.zig"); diff --git a/bin/zml-smi/bindings/linux/metrics.zig b/bin/zml-smi/platforms/linux/metrics.zig similarity index 86% rename from bin/zml-smi/bindings/linux/metrics.zig rename to bin/zml-smi/platforms/linux/metrics.zig index 376c75d7..5a69663a 100644 --- a/bin/zml-smi/bindings/linux/metrics.zig +++ b/bin/zml-smi/platforms/linux/metrics.zig @@ -1,15 +1,14 @@ const std = @import("std"); -const sysfs = @import("../../utils/sysfs.zig"); -const host_info = @import("../../info/host_info.zig"); +const sysfs = @import("zml-smi/sysfs"); +const host_info = @import("zml-smi/info").host_info; const HostInfo = host_info.HostInfo; const HostData = host_info.HostData; -const Worker = @import("../../worker.zig").Worker; +const poll_metrics = @import("zml-smi/info").poll_metrics; +const Collector = @import("zml-smi/collector").Collector; -const Collector = @import("../../collector.zig").Collector; - -pub fn init(w: *Worker, io: std.Io, collector: *Collector, info: *HostInfo) !void { +pub fn init(collector: *Collector, info: *HostInfo) !void { const poll_arena = try collector.createPollArena(); - const host: Host = .{ .io = io, .arena = poll_arena }; + const host: Host = .{ .io = collector.io, .arena = poll_arena }; // read once metrics var initial = info.front().*; @@ -21,10 +20,10 @@ pub fn init(w: *Worker, io: std.Io, collector: *Collector, info: *HostInfo) !voi info.back().* = initial; info.swap(); - try w.spawn(io, pollHost, .{ io, w, info, host }); + try collector.spawnPoll(pollOnce, .{ poll_arena, info, host }); } -const pollHost = Worker.pollMetrics(*HostInfo, Host, metrics); +const pollOnce = poll_metrics.poll(*HostInfo, Host, metrics); const Host = struct { io: std.Io, diff --git a/bin/zml-smi/bindings/linux/process.zig b/bin/zml-smi/platforms/linux/process.zig similarity index 96% rename from bin/zml-smi/bindings/linux/process.zig rename to bin/zml-smi/platforms/linux/process.zig index 364a2f6b..3aa5ff01 100644 --- a/bin/zml-smi/bindings/linux/process.zig +++ b/bin/zml-smi/platforms/linux/process.zig @@ -1,6 +1,6 @@ const std = @import("std"); -const sysfs = @import("../../utils/sysfs.zig"); -const pi = @import("../../info/process_info.zig"); +const sysfs = @import("zml-smi/sysfs"); +const pi = @import("zml-smi/info").process_info; const ProcessInfo = pi.ProcessInfo; @@ -59,7 +59,7 @@ pub const ProcessEnricher = struct { if (delta_total > 0) { if (self.prev_ticks.get(info.pid)) |prev| { const delta_proc = ticks -| prev; - info.cpu_percent = @intCast(@min(delta_proc * 1000 / delta_total, std.math.maxInt(u16))); + info.cpu_percent = @intCast(@min(delta_proc *| 1000 / delta_total, std.math.maxInt(u16))); } } diff --git a/bin/zml-smi/platforms/neuron/BUILD.bazel b/bin/zml-smi/platforms/neuron/BUILD.bazel new file mode 100644 index 00000000..c7129ac7 --- /dev/null +++ b/bin/zml-smi/platforms/neuron/BUILD.bazel @@ -0,0 +1,22 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "neuron", + import_name = "zml-smi/platforms/neuron", + main = "metrics.zig", + srcs = [ + "nrt.zig", + "process.zig", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + "//bin/zml-smi/utils:dynlib", + "//bin/zml-smi/utils:sandbox", + "//bin/zml-smi/utils:sysfs", + "//platforms/neuron:libnrt_headers", + "//stdx", + ], +) diff --git a/bin/zml-smi/bindings/neuron/metrics.zig b/bin/zml-smi/platforms/neuron/metrics.zig similarity index 75% rename from bin/zml-smi/bindings/neuron/metrics.zig rename to bin/zml-smi/platforms/neuron/metrics.zig index b3d47c64..ee667c0a 100644 --- a/bin/zml-smi/bindings/neuron/metrics.zig +++ b/bin/zml-smi/platforms/neuron/metrics.zig @@ -1,17 +1,17 @@ const std = @import("std"); -const sysfs = @import("../../utils/sysfs.zig"); -const device_info = @import("../../info/device_info.zig"); +const sysfs = @import("zml-smi/sysfs"); +const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; +const device_info = @import("zml-smi/info").device_info; const DeviceInfo = device_info.DeviceInfo; const NeuronInfo = device_info.NeuronInfo; -const DoubleBuffer = @import("../../utils/double_buffer.zig").DoubleBuffer; -const Collector = @import("../../collector.zig").Collector; -const Worker = @import("../../worker.zig").Worker; +const Collector = @import("zml-smi/collector").Collector; +const poll_metrics = @import("zml-smi/info").poll_metrics; const Nrt = @import("nrt.zig"); const process = @import("process.zig"); const base_path = "/sys/devices/virtual/neuron_device"; -pub const target: device_info.Target = .neuron; +pub const UtilAtomic = std.atomic.Value(u64); pub fn start(collector: *Collector) !void { const nrt = try collector.arena.create(Nrt); @@ -24,10 +24,21 @@ pub fn start(collector: *Collector) !void { const total_nc = nrt.totalNcCount() catch return; const nc_per_device = total_nc / @as(u32, @intCast(nrt.handles.len)); + const total_cores = @as(usize, nrt.handles.len) * nc_per_device; + const util_buf = try collector.arena.alloc(UtilAtomic, total_cores); + for (util_buf) |*u| u.* = .init(0); + const GiB: u64 = 1024 * 1024 * 1024; var neuron_infos: std.ArrayList(*DeviceInfo) = .{}; const dev_offset: u8 = @intCast(collector.device_infos.items.len); + var ver_buf: [Nrt.version_buf_len]u8 = undefined; + const nrt_ver: ?[]const u8 = if (nrt.version(&ver_buf)) |v| + collector.arena.dupe(u8, v) catch null + else |_| + null; + + var flat_core: usize = 0; for (nrt.handles, nrt.device_indexes) |dev_handle, device_idx| { const device_type = Nrt.deviceType(dev_handle); const mem_per_device: u64 = switch (device_type) { @@ -48,23 +59,37 @@ pub fn start(collector: *Collector) !void { .core_idx = @intCast(ci), .mem_per_device = mem_per_device, .nc_per_device = nc_per_device, + .util_src = &util_buf[flat_core], }; - const initial: NeuronInfo = .{ .name = dev.name(collector.arena) catch null }; + const initial: NeuronInfo = .{ + .name = dev.name(collector.arena) catch null, + .driver_version = nrt_ver, + }; const info = try collector.addDevice(.{ .neuron = .{ .values = .{ initial, initial } } }); try neuron_infos.append(collector.arena, info); - - try collector.worker.spawn(collector.io, pollDevice, .{ collector.io, collector.worker, &info.neuron, dev }); + try collector.spawnPoll(pollOnce, .{ poll_arena, &info.neuron, dev }); + flat_core += 1; } } if (neuron_infos.items.len > 0) { const processes = try collector.createProcessList(); - try process.init(collector.worker, collector.io, collector.gpa, processes, nrt, nc_per_device, neuron_infos.items, dev_offset); + const warmup = collector.poll_only; + try process.init(collector, processes, nrt, nc_per_device, util_buf, dev_offset, warmup); + + if (warmup) { + for (neuron_infos.items, util_buf) |info, *u| { + const back = info.neuron.back(); + back.* = info.neuron.front().*; + back.util_percent = u.load(.acquire); + info.neuron.swap(); + } + } } } -const pollDevice = Worker.pollMetrics(*DoubleBuffer(NeuronInfo), Device, metrics); +const pollOnce = poll_metrics.poll(*DoubleBuffer(NeuronInfo), Device, metrics); const Device = struct { io: std.Io, @@ -73,6 +98,11 @@ const Device = struct { core_idx: u32, mem_per_device: u64 = 0, nc_per_device: u32 = 1, + util_src: *const UtilAtomic, + + pub fn utilPercent(self: Device) !u64 { + return self.util_src.load(.acquire); + } fn devicePath(self: Device, sub_path: []const u8) ![]const u8 { return std.fmt.allocPrint(self.arena.allocator(), base_path ++ "/neuron{d}/{s}", .{ self.device_idx, sub_path }); @@ -154,6 +184,7 @@ const Device = struct { }; const metrics = .{ + .{ .field = "util_percent", .query = Device.utilPercent }, .{ .field = "mem_total_bytes", .query = Device.memTotal }, .{ .field = "mem_used_bytes", .query = Device.memUsed }, .{ .field = "nc_tensors", .query = Device.tensors }, diff --git a/bin/zml-smi/platforms/neuron/nrt.zig b/bin/zml-smi/platforms/neuron/nrt.zig new file mode 100644 index 00000000..e5236cff --- /dev/null +++ b/bin/zml-smi/platforms/neuron/nrt.zig @@ -0,0 +1,229 @@ +const std = @import("std"); +const c = @import("c"); +const elf = std.elf; +const stdx = @import("stdx"); +const DynLib = @import("zml-smi/dynlib"); +const sandbox = @import("zml-smi/sandbox"); + +const Nrt = @This(); + +pub const Error = error{ nrt_error, NrtUnavailable }; +pub const AppInfo = c.struct_neuron_app_info; +pub const DeviceType = enum(c_int) { + inf1 = 1, + inf2_trn1 = 2, + trn2 = 3, + trn3 = 4, + _, +}; + +lib: Fns, +private_lib: PrivateFns, +handles: []const *c.ndl_device_t, +device_indexes: []const c_int, + +const Fns = struct { + nrt_get_total_nc_count: *const @TypeOf(c.nrt_get_total_nc_count), + nrt_get_version: *const @TypeOf(c.nrt_get_version), +}; + +const PrivateFns = struct { + nds_open: *const @TypeOf(c.nds_open), + ndl_available_devices: *const @TypeOf(c.ndl_available_devices), + ndl_open_device: *const @TypeOf(c.ndl_open_device), + ndl_close_device: *const @TypeOf(c.ndl_close_device), + ndl_get_all_apps_info: *const @TypeOf(c.ndl_get_all_apps_info), + nds_close: *const @TypeOf(c.nds_close), + nds_get_nc_counter: *const @TypeOf(c.nds_get_nc_counter), +}; + +pub fn init(allocator: std.mem.Allocator, io: std.Io) !Nrt { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = sandbox.path(&path_buf) orelse { + std.log.err("neuron: sandbox path unavailable", .{}); + return error.NrtUnavailable; + }; + + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libnrt.so.1" }) catch { + std.log.err("neuron: failed to construct libnrt.so.1 path", .{}); + return error.NrtUnavailable; + }; + + const dl_handle = std.c.dlopen(path, .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { + if (std.c.dlerror()) |err| std.log.err("neuron: dlopen: {s}", .{err}); + return error.NrtUnavailable; + }; + var dynlib: std.DynLib = .{ .inner = .{ .handle = dl_handle } }; + + const fns = DynLib.lookupStruct(&dynlib, Fns) catch return error.NrtUnavailable; + const private_fns = resolveFromElf(dl_handle, io, "nrt_init") catch |err| { + std.log.err("neuron: ELF resolution failed: {s}", .{@errorName(err)}); + return error.NrtUnavailable; + }; + + var dev_index_buf: [c.MAX_NEURON_DEVICE_COUNT]c_int = undefined; + const count: usize = @intCast(@max(0, private_fns.ndl_available_devices(&dev_index_buf, c.MAX_NEURON_DEVICE_COUNT))); + + var handle_list: std.ArrayList(*c.ndl_device_t) = .{}; + var index_list: std.ArrayList(c_int) = .{}; + + for (dev_index_buf[0..count]) |device_idx| { + var dev: ?*c.ndl_device_t = null; + var t: c.struct_ndl_device_init_param = .{ + .initialize_device = false, + .map_hbm = false, + .num_dram_regions = 0, + }; + + if (private_fns.ndl_open_device(device_idx, &t, &dev) == 0) { + if (dev) |d| { + errdefer _ = private_fns.ndl_close_device(d); + try handle_list.append(allocator, d); + try index_list.append(allocator, device_idx); + } + } + } + + return .{ + .lib = fns, + .private_lib = private_fns, + .handles = try handle_list.toOwnedSlice(allocator), + .device_indexes = try index_list.toOwnedSlice(allocator), + }; +} + +pub fn deviceCount(self: Nrt) u32 { + return @intCast(self.handles.len); +} + +pub fn allAppsInfo(self: Nrt, dev: *c.ndl_device_t) Error!struct { ptr: ?[*]AppInfo, count: usize } { + var info: ?[*]AppInfo = null; + var count: usize = 0; + if (self.private_lib.ndl_get_all_apps_info(dev, &info, &count, c.APP_INFO_ALL) != 0) { + return error.nrt_error; + } + return .{ .ptr = info, .count = count }; +} + +pub fn ndsOpen(self: Nrt, dev: *c.ndl_device_t, pid: c.pid_t) Error!*c.nds_instance_t { + var inst: ?*c.nds_instance_t = null; + if (self.private_lib.nds_open(dev, pid, &inst) != 0) { + return error.nrt_error; + } + return inst orelse error.nrt_error; +} + +pub fn ndsClose(self: Nrt, inst: *c.nds_instance_t) void { + _ = self.private_lib.nds_close(inst); +} + +pub fn ncCounter(self: Nrt, inst: *c.nds_instance_t, pnc_index: c_int, counter_index: u32) Error!u64 { + var value: u64 = 0; + if (self.private_lib.nds_get_nc_counter(inst, pnc_index, counter_index, &value) != 0) { + return error.nrt_error; + } + return value; +} + +pub const version_buf_len = c.RT_VERSION_DETAIL_LEN; + +pub fn version(self: Nrt, buf: *[version_buf_len]u8) Error![:0]const u8 { + var ver: c.nrt_version_t = std.mem.zeroes(c.nrt_version_t); + if (self.lib.nrt_get_version(&ver, @sizeOf(c.nrt_version_t)) != 0) { + return error.nrt_error; + } + @memcpy(buf, &ver.rt_detail); + return std.mem.span(@as([*c]const u8, @ptrCast(buf))); +} + +pub fn totalNcCount(self: Nrt) Error!u32 { + var count: u32 = 0; + if (self.lib.nrt_get_total_nc_count(&count) != 0) { + return error.nrt_error; + } + return count; +} + +pub fn hbmSize(dev: *c.ndl_device_t) usize { + return dev.hbm_size; +} + +pub fn deviceType(dev: *c.ndl_device_t) DeviceType { + return @enumFromInt(dev.device_type); +} + +fn resolveFromElf(handle: *anyopaque, io: std.Io, comptime known_sym: [:0]const u8) !PrivateFns { + const fields = std.meta.fields(PrivateFns); + + const sym_addr = std.c.dlsym(handle, known_sym) orelse + return error.SymbolNotFound; + + var dl_info: DlInfo = undefined; + if (dladdr(sym_addr, &dl_info) == 0) return error.DladdrFailed; + + const elf_base = @intFromPtr(dl_info.dli_fbase); + const elf_path = std.mem.span(dl_info.dli_fname); + + const file = try std.Io.Dir.openFileAbsolute(io, elf_path, .{}); + defer file.close(io); + + const stat = try file.stat(io); + const mmap = try std.posix.mmap( + null, + @intCast(stat.size), + .{ .READ = true }, + .{ .TYPE = .PRIVATE }, + file.handle, + 0, + ); + defer std.posix.munmap(mmap); + + var result: PrivateFns = undefined; + var remaining: usize = fields.len; + + const ehdr: *const elf.ElfN.Ehdr = @ptrCast(@alignCast(mmap)); + const shdrs: []const elf.ElfN.Shdr = @as([*]const elf.ElfN.Shdr, @ptrCast(@alignCast(mmap[ehdr.shoff..])))[0..ehdr.shnum]; + + for (shdrs) |sh| { + if (sh.type != .SYMTAB) { + continue; + } + + const strtab_sh = shdrs[sh.link]; + const strtab: []const u8 = mmap[strtab_sh.offset..][0..strtab_sh.size]; + + for (std.mem.bytesAsSlice(elf.ElfN.Sym, mmap[sh.offset..][0..sh.size])) |s| { + if (s.value == 0) { + continue; + } + + const sym_name = std.mem.span(@as([*c]const u8, @ptrCast(strtab[@as(usize, s.name)..]))); + + inline for (fields) |field| { + if (std.mem.eql(u8, sym_name, field.name)) { + @field(result, field.name) = @ptrFromInt(elf_base + @as(usize, s.value)); + remaining -= 1; + } + } + + if (remaining == 0) { + return result; + } + } + } + + if (remaining > 0) { + return error.SymbolResolutionFailed; + } + return result; +} + +const DlInfo = extern struct { + dli_fname: [*:0]const u8, + dli_fbase: *anyopaque, + dli_sname: [*c]const u8, + dli_saddr: ?*anyopaque, +}; + +extern fn dladdr(addr: *anyopaque, info: *DlInfo) c_int; diff --git a/bin/zml-smi/platforms/neuron/process.zig b/bin/zml-smi/platforms/neuron/process.zig new file mode 100644 index 00000000..40ec167e --- /dev/null +++ b/bin/zml-smi/platforms/neuron/process.zig @@ -0,0 +1,108 @@ +const std = @import("std"); +const c = @import("c"); +const Nrt = @import("nrt.zig"); +const smi_info = @import("zml-smi/info"); +const pi = smi_info.process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const Collector = @import("zml-smi/collector").Collector; +const UtilAtomic = @import("metrics.zig").UtilAtomic; + +const DeltaState = struct { + prev_total_us: u64 = 0, + prev_timestamp: ?std.Io.Timestamp = null, +}; + +pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, nrt: *const Nrt, nc_per_device: u32, util_buf: []UtilAtomic, dev_offset: u8, warmup: bool) !void { + if (nrt.handles.len == 0) return; + const delta_states = try collector.arena.alloc(DeltaState, util_buf.len); + @memset(delta_states, .{}); + const args: std.meta.ArgsTuple(@TypeOf(pollOnce)) = .{ collector.io, collector.gpa, list, nrt, nc_per_device, util_buf, dev_offset, delta_states }; + if (warmup) { + @call(.auto, pollOnce, args); + collector.io.sleep(.fromMilliseconds(500), .awake) catch {}; + @call(.auto, pollOnce, args); + collector.io.sleep(.fromMilliseconds(500), .awake) catch {}; + } + + try collector.spawnPoll(pollOnce, args); +} + +fn pollOnce(io: std.Io, allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, nrt: *const Nrt, nc_per_device: u32, util_buf: []UtilAtomic, dev_offset: u8, delta_states: []DeltaState) void { + const back = list.back(); + + back.clearRetainingCapacity(); + + for (nrt.handles, 0..) |handle, dev_i| { + const apps_result = nrt.allAppsInfo(handle) catch continue; + defer { + if (apps_result.ptr) |ptr| { + std.c.free(@ptrCast(ptr)); + } + } + + const apps = if (apps_result.ptr) |ptr| ptr[0..apps_result.count] else continue; + + var total_us_per_core: [c.MAX_NC_PER_DEVICE]u64 = .{0} ** c.MAX_NC_PER_DEVICE; + + for (apps) |*app| { + if (app.pid <= 0) { + continue; + } + + const nds = nrt.ndsOpen(handle, app.pid) catch continue; + defer nrt.ndsClose(nds); + + for (0..nc_per_device) |ci| { + const time_in_use = nrt.ncCounter(nds, @intCast(ci), c.NDS_NC_COUNTER_TIME_IN_USE) catch continue; + total_us_per_core[ci] += time_in_use; + + if (time_in_use == 0) { + continue; + } + + const dev_idx: u32 = @intCast(dev_i * nc_per_device + ci); + + const pid = std.math.cast(u32, app.pid) orelse continue; + var info: pi.ProcessInfo = .{ + .pid = pid, + .device_idx = @intCast(dev_idx + dev_offset), + .dev_mem_kib = @intCast(app.device_mem_size / 1024), + }; + + if (dev_idx < util_buf.len) { + info.dev_util_percent = @intCast(util_buf[dev_idx].load(.acquire)); + } + + if (info.dev_mem_kib != null and info.dev_mem_kib.? > 0) + back.append(allocator, info) catch break; + } + } + + const now: std.Io.Timestamp = .now(io, .awake); + + for (0..nc_per_device) |ci| { + const dev_idx = dev_i * nc_per_device + ci; + if (dev_idx >= delta_states.len) { + break; + } + + const ds = &delta_states[dev_idx]; + const prev_us = ds.prev_total_us; + ds.prev_total_us = total_us_per_core[ci]; + + if (ds.prev_timestamp) |prev| { + const elapsed = prev.untilNow(io, .awake); + const elapsed_us: u64 = @intCast(@divFloor(elapsed.nanoseconds, std.time.ns_per_us)); + + if (elapsed_us > 0) { + const delta_us = total_us_per_core[ci] -| prev_us; + util_buf[dev_idx].store(@min(100, delta_us * 100 / elapsed_us), .release); + } + } + + ds.prev_timestamp = now; + } + } + + list.swap(); +} diff --git a/bin/zml-smi/platforms/nvml/BUILD.bazel b/bin/zml-smi/platforms/nvml/BUILD.bazel new file mode 100644 index 00000000..5bcb7a62 --- /dev/null +++ b/bin/zml-smi/platforms/nvml/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "nvml", + import_name = "zml-smi/platforms/nvml", + main = "metrics.zig", + srcs = [ + "nvml.zig", + "process.zig", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + "//bin/zml-smi/utils:dynlib", + "@cuda_nvml_dev//:nvml", + ], +) diff --git a/bin/zml-smi/bindings/nvml/metrics.zig b/bin/zml-smi/platforms/nvml/metrics.zig similarity index 75% rename from bin/zml-smi/bindings/nvml/metrics.zig rename to bin/zml-smi/platforms/nvml/metrics.zig index 082d4761..f134e4c2 100644 --- a/bin/zml-smi/bindings/nvml/metrics.zig +++ b/bin/zml-smi/platforms/nvml/metrics.zig @@ -1,15 +1,13 @@ const std = @import("std"); const Nvml = @import("nvml.zig"); -const device_info = @import("../../info/device_info.zig"); +const device_info = @import("zml-smi/info").device_info; const DeviceInfo = device_info.DeviceInfo; const GpuInfo = device_info.GpuInfo; -const DoubleBuffer = @import("../../utils/double_buffer.zig").DoubleBuffer; -const Collector = @import("../../collector.zig").Collector; -const Worker = @import("../../worker.zig").Worker; +const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; +const Collector = @import("zml-smi/collector").Collector; +const poll_metrics = @import("zml-smi/info").poll_metrics; const process = @import("process.zig"); -pub const target: device_info.Target = .cuda; - pub fn start(collector: *Collector) !void { const nvml = try collector.arena.create(Nvml); nvml.* = try Nvml.init(); @@ -19,16 +17,20 @@ pub fn start(collector: *Collector) !void { for (0..count) |i| { const dev = Device.open(nvml, @intCast(i)) catch continue; - const initial: GpuInfo = .{ .name = dev.name(collector.arena) catch null }; + const initial: GpuInfo = .{ + .name = dev.name(collector.arena) catch null, + .driver_version = dev.driverVersion(collector.arena) catch null, + .cuda_driver_version = dev.cudaDriverVersion(collector.arena) catch null, + }; const info = try collector.addDevice(.{ .cuda = .{ .values = .{ initial, initial } } }); - try collector.worker.spawn(collector.io, pollDevice, .{ collector.io, collector.worker, &info.cuda, dev }); + try collector.spawnPoll(pollOnce, .{ null, &info.cuda, dev }); } const processes = try collector.createProcessList(); - try process.init(collector.worker, collector.io, collector.gpa, processes, nvml, dev_offset); + try process.init(collector, processes, nvml, dev_offset); } -const pollDevice = Worker.pollMetrics(*DoubleBuffer(GpuInfo), Device, metrics); +const pollOnce = poll_metrics.poll(*DoubleBuffer(GpuInfo), Device, metrics); const Device = struct { nvml: *const Nvml, @@ -39,12 +41,27 @@ const Device = struct { } fn name(self: Device, arena: std.mem.Allocator) ![]const u8 { - var buf: [256]u8 = .{0} ** 256; + var buf: [Nvml.name_buf_len]u8 = undefined; const slice = try self.nvml.name(self.handle, &buf); return try arena.dupe(u8, slice); } + fn driverVersion(self: Device, arena: std.mem.Allocator) ![]const u8 { + var buf: [Nvml.driver_version_buf_size]u8 = undefined; + const slice = try self.nvml.driverVersion(&buf); + + return try arena.dupe(u8, slice); + } + + fn cudaDriverVersion(self: Device, arena: std.mem.Allocator) ![]const u8 { + const v = try self.nvml.cudaDriverVersion(); + return try std.fmt.allocPrint(arena, "{}.{}", .{ + Nvml.cudaDriverVersionMajor(v), + Nvml.cudaDriverVersionMinor(v), + }); + } + // Power pub fn powerUsage(self: Device) !u64 { return @intCast(try self.nvml.powerUsage(self.handle)); @@ -119,6 +136,10 @@ const Device = struct { return @intCast(try self.nvml.pcieRxKBps(self.handle)); } + pub fn pcieBandwidth(self: Device) !u64 { + return @intCast(try self.nvml.pcieSpeed(self.handle)); + } + pub fn pcieLinkGen(self: Device) !u64 { return @intCast(try self.nvml.pcieLinkGen(self.handle)); } @@ -146,6 +167,7 @@ const metrics = .{ .{ .field = "mem_bus_width", .query = Device.memBusWidth }, .{ .field = "pcie_tx_kbps", .query = Device.pcieTx }, .{ .field = "pcie_rx_kbps", .query = Device.pcieRx }, + .{ .field = "pcie_bandwidth_mbps", .query = Device.pcieBandwidth }, .{ .field = "pcie_link_gen", .query = Device.pcieLinkGen }, .{ .field = "pcie_link_width", .query = Device.pcieLinkWidth }, }; diff --git a/bin/zml-smi/bindings/nvml/nvml.zig b/bin/zml-smi/platforms/nvml/nvml.zig similarity index 73% rename from bin/zml-smi/bindings/nvml/nvml.zig rename to bin/zml-smi/platforms/nvml/nvml.zig index 50d9d794..87d064ce 100644 --- a/bin/zml-smi/bindings/nvml/nvml.zig +++ b/bin/zml-smi/platforms/nvml/nvml.zig @@ -1,9 +1,10 @@ const std = @import("std"); const c = @import("c"); -const DynLib = @import("../dynlib.zig"); +const DynLib = @import("zml-smi/dynlib"); const Nvml = @This(); +pub const driver_version_buf_size = c.NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE; pub const Error = ReturnError || error{NvmlUnavailable}; pub const Handle = c.nvmlDevice_t; pub const ProcessInfo_t = c.nvmlProcessInfo_t; @@ -11,33 +12,41 @@ pub const ProcessInfo_t = c.nvmlProcessInfo_t; lib: Fns, const Fns = struct { - nvmlInit_v2: DynLib.Fn("nvmlInit_v2"), - nvmlDeviceGetHandleByIndex_v2: DynLib.Fn("nvmlDeviceGetHandleByIndex_v2"), - nvmlDeviceGetCount_v2: DynLib.Fn("nvmlDeviceGetCount_v2"), - nvmlDeviceGetName: DynLib.Fn("nvmlDeviceGetName"), - nvmlDeviceGetPowerUsage: DynLib.Fn("nvmlDeviceGetPowerUsage"), - nvmlDeviceGetTemperature: DynLib.Fn("nvmlDeviceGetTemperature"), - nvmlDeviceGetUtilizationRates: DynLib.Fn("nvmlDeviceGetUtilizationRates"), - nvmlDeviceGetClockInfo: DynLib.Fn("nvmlDeviceGetClockInfo"), - nvmlDeviceGetMaxClockInfo: DynLib.Fn("nvmlDeviceGetMaxClockInfo"), - nvmlDeviceGetMemoryInfo: DynLib.Fn("nvmlDeviceGetMemoryInfo"), - nvmlDeviceGetFanSpeed: DynLib.Fn("nvmlDeviceGetFanSpeed"), - nvmlDeviceGetEnforcedPowerLimit: DynLib.Fn("nvmlDeviceGetEnforcedPowerLimit"), - nvmlDeviceGetPcieThroughput: DynLib.Fn("nvmlDeviceGetPcieThroughput"), - nvmlDeviceGetEncoderUtilization: DynLib.Fn("nvmlDeviceGetEncoderUtilization"), - nvmlDeviceGetDecoderUtilization: DynLib.Fn("nvmlDeviceGetDecoderUtilization"), - nvmlDeviceGetCurrPcieLinkGeneration: DynLib.Fn("nvmlDeviceGetCurrPcieLinkGeneration"), - nvmlDeviceGetCurrPcieLinkWidth: DynLib.Fn("nvmlDeviceGetCurrPcieLinkWidth"), - nvmlDeviceGetMemoryBusWidth: DynLib.Fn("nvmlDeviceGetMemoryBusWidth"), - nvmlDeviceGetComputeRunningProcesses_v3: DynLib.Fn("nvmlDeviceGetComputeRunningProcesses_v3"), - nvmlDeviceGetGraphicsRunningProcesses_v3: DynLib.Fn("nvmlDeviceGetGraphicsRunningProcesses_v3"), - nvmlDeviceGetProcessUtilization: DynLib.Fn("nvmlDeviceGetProcessUtilization"), + nvmlInit_v2: *const @TypeOf(c.nvmlInit_v2), + nvmlDeviceGetHandleByIndex_v2: *const @TypeOf(c.nvmlDeviceGetHandleByIndex_v2), + nvmlDeviceGetCount_v2: *const @TypeOf(c.nvmlDeviceGetCount_v2), + nvmlDeviceGetName: *const @TypeOf(c.nvmlDeviceGetName), + nvmlDeviceGetPowerUsage: *const @TypeOf(c.nvmlDeviceGetPowerUsage), + nvmlDeviceGetTemperature: *const @TypeOf(c.nvmlDeviceGetTemperature), + nvmlDeviceGetUtilizationRates: *const @TypeOf(c.nvmlDeviceGetUtilizationRates), + nvmlDeviceGetClockInfo: *const @TypeOf(c.nvmlDeviceGetClockInfo), + nvmlDeviceGetMaxClockInfo: *const @TypeOf(c.nvmlDeviceGetMaxClockInfo), + nvmlDeviceGetMemoryInfo: *const @TypeOf(c.nvmlDeviceGetMemoryInfo), + nvmlDeviceGetFanSpeed: *const @TypeOf(c.nvmlDeviceGetFanSpeed), + nvmlDeviceGetEnforcedPowerLimit: *const @TypeOf(c.nvmlDeviceGetEnforcedPowerLimit), + nvmlDeviceGetPcieThroughput: *const @TypeOf(c.nvmlDeviceGetPcieThroughput), + nvmlDeviceGetEncoderUtilization: *const @TypeOf(c.nvmlDeviceGetEncoderUtilization), + nvmlDeviceGetDecoderUtilization: *const @TypeOf(c.nvmlDeviceGetDecoderUtilization), + nvmlDeviceGetCurrPcieLinkGeneration: *const @TypeOf(c.nvmlDeviceGetCurrPcieLinkGeneration), + nvmlDeviceGetCurrPcieLinkWidth: *const @TypeOf(c.nvmlDeviceGetCurrPcieLinkWidth), + nvmlDeviceGetMemoryBusWidth: *const @TypeOf(c.nvmlDeviceGetMemoryBusWidth), + nvmlDeviceGetPcieSpeed: *const @TypeOf(c.nvmlDeviceGetPcieSpeed), + nvmlDeviceGetComputeRunningProcesses_v3: *const @TypeOf(c.nvmlDeviceGetComputeRunningProcesses_v3), + nvmlDeviceGetGraphicsRunningProcesses_v3: *const @TypeOf(c.nvmlDeviceGetGraphicsRunningProcesses_v3), + nvmlDeviceGetProcessUtilization: *const @TypeOf(c.nvmlDeviceGetProcessUtilization), + nvmlSystemGetCudaDriverVersion_v2: *const @TypeOf(c.nvmlSystemGetCudaDriverVersion_v2), + nvmlSystemGetDriverVersion: *const @TypeOf(c.nvmlSystemGetDriverVersion), }; pub fn init() Error!Nvml { - const fns = DynLib.open(Fns, "libnvidia-ml.so.1") orelse - DynLib.open(Fns, "libnvidia-ml.so") orelse - return error.NvmlUnavailable; + var dynlib: std.DynLib = .{ .inner = .{ + .handle = std.c.dlopen("libnvidia-ml.so.1", .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse + std.c.dlopen("libnvidia-ml.so", .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { + if (std.c.dlerror()) |err| std.log.err("nvml: dlopen: {s}", .{err}); + return error.NvmlUnavailable; + }, + } }; + const fns = DynLib.lookupStruct(&dynlib, Fns) catch return error.NvmlUnavailable; try check(fns.nvmlInit_v2()); return .{ .lib = fns }; } @@ -54,9 +63,11 @@ pub fn deviceCount(self: Nvml) Error!u32 { return @intCast(count); } -pub fn name(self: Nvml, handle: c.nvmlDevice_t, buf: *[256]u8) Error![:0]const u8 { +pub const name_buf_len = c.NVML_DEVICE_NAME_V2_BUFFER_SIZE; + +pub fn name(self: Nvml, handle: c.nvmlDevice_t, buf: *[c.NVML_DEVICE_NAME_V2_BUFFER_SIZE]u8) Error![:0]const u8 { try check(self.lib.nvmlDeviceGetName(handle, buf, buf.len)); - return std.mem.sliceTo(@as([*:0]const u8, @ptrCast(buf)), 0); + return std.mem.span(@as([*c]const u8, @ptrCast(buf))); } pub fn powerUsage(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { @@ -169,6 +180,31 @@ pub fn pcieLinkWidth(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { return width; } +pub fn pcieSpeed(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var speed: c_uint = 0; + try check(self.lib.nvmlDeviceGetPcieSpeed(handle, &speed)); + return speed; +} + +pub fn driverVersion(self: Nvml, buf: *[c.NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE]u8) Error![:0]const u8 { + try check(self.lib.nvmlSystemGetDriverVersion(buf, buf.len)); + return std.mem.span(@as([*c]const u8, @ptrCast(buf))); +} + +pub fn cudaDriverVersionMajor(v: c_int) c_int { + return @divTrunc(v, 1000); +} + +pub fn cudaDriverVersionMinor(v: c_int) c_int { + return @divTrunc(@mod(v, 1000), 10); +} + +pub fn cudaDriverVersion(self: Nvml) Error!c_int { + var version: c_int = 0; + try check(self.lib.nvmlSystemGetCudaDriverVersion_v2(&version)); + return version; +} + pub fn memBusWidth(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { var width: c_uint = 0; try check(self.lib.nvmlDeviceGetMemoryBusWidth(handle, &width)); diff --git a/bin/zml-smi/platforms/nvml/process.zig b/bin/zml-smi/platforms/nvml/process.zig new file mode 100644 index 00000000..c1dd39d2 --- /dev/null +++ b/bin/zml-smi/platforms/nvml/process.zig @@ -0,0 +1,75 @@ +const std = @import("std"); +const Nvml = @import("nvml.zig"); +const pi = @import("zml-smi/info").process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const Collector = @import("zml-smi/collector").Collector; + +pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, nvml: *const Nvml, dev_offset: u8) !void { + const device_count: u32 = nvml.deviceCount() catch 0; + const last_seen_ts = try collector.arena.alloc(u64, device_count); + @memset(last_seen_ts, 0); + + try collector.spawnPoll(pollOnce, .{ collector.gpa, list, nvml, dev_offset, device_count, last_seen_ts }); +} + +fn pollOnce(allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, nvml: *const Nvml, dev_offset: u8, device_count: u32, last_seen_ts: []u64) void { + const back = list.back(); + + back.clearRetainingCapacity(); + + for (0..device_count) |dev_idx| { + const handle = nvml.handleByIndex(@intCast(dev_idx)) catch continue; + const idx: u8 = @intCast(dev_idx + dev_offset); + + const compute = nvml.computeRunningProcesses(allocator, handle) catch &.{}; + defer { + if (compute.len > 0) { + allocator.free(compute); + } + } + collectFromQuery(allocator, back, idx, compute); + + const graphics = nvml.graphicsRunningProcesses(allocator, handle) catch &.{}; + defer { + if (graphics.len > 0) { + allocator.free(graphics); + } + } + collectFromQuery(allocator, back, idx, graphics); + + // Apply utilization samples + const last_ts = last_seen_ts[dev_idx]; + const utils = nvml.processUtilization(allocator, handle, last_ts) catch continue; + defer { + if (utils.len > 0) { + allocator.free(utils); + } + } + + for (utils) |sample| { + if (sample.smUtil > 100 or sample.timeStamp <= last_ts) { + continue; + } + + last_seen_ts[dev_idx] = @max(last_seen_ts[dev_idx], sample.timeStamp); + + for (back.items) |*entry| { + if (entry.pid == sample.pid and entry.device_idx == idx) { + entry.dev_util_percent = @intCast(sample.smUtil); + } + } + } + } + + list.swap(); +} + +fn collectFromQuery(allocator: std.mem.Allocator, back: *std.ArrayList(pi.ProcessInfo), dev_idx: u8, procs: []const Nvml.ProcessInfo_t) void { + for (procs) |gp| { + back.append(allocator, .{ + .pid = gp.pid, + .device_idx = dev_idx, + .dev_mem_kib = @intCast(gp.usedGpuMemory / 1024), + }) catch return; + } +} diff --git a/bin/zml-smi/platforms/tpu/BUILD.bazel b/bin/zml-smi/platforms/tpu/BUILD.bazel new file mode 100644 index 00000000..a65377b3 --- /dev/null +++ b/bin/zml-smi/platforms/tpu/BUILD.bazel @@ -0,0 +1,30 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library") + +cc_library( + name = "tpuinfo", + srcs = ["tpuinfo.cc"], + hdrs = ["tpuinfo.h"], + deps = [ + "@cloud_accelerator_diagnostics//:tpu_metric_service_cc_grpc", + "@grpc//:grpc++_unsecure", + ], +) + +zig_library( + name = "tpu", + import_name = "zml-smi/platforms/tpu", + main = "metrics.zig", + srcs = [ + "process.zig", + "tpuinfo.zig", + ], + visibility = ["//visibility:public"], + deps = [ + ":tpuinfo", + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + "//bin/zml-smi/utils:sysfs", + ], +) diff --git a/bin/zml-smi/bindings/tpu/metrics.zig b/bin/zml-smi/platforms/tpu/metrics.zig similarity index 74% rename from bin/zml-smi/bindings/tpu/metrics.zig rename to bin/zml-smi/platforms/tpu/metrics.zig index e7c2e531..8ef832d8 100644 --- a/bin/zml-smi/bindings/tpu/metrics.zig +++ b/bin/zml-smi/platforms/tpu/metrics.zig @@ -1,17 +1,14 @@ const std = @import("std"); -const sysfs = @import("../../utils/sysfs.zig"); +const sysfs = @import("zml-smi/sysfs"); const tpuinfo = @import("tpuinfo.zig"); -const device_info = @import("../../info/device_info.zig"); +const device_info = @import("zml-smi/info").device_info; const DeviceInfo = device_info.DeviceInfo; const TpuInfo = device_info.TpuInfo; -const Collector = @import("../../collector.zig").Collector; -const Worker = @import("../../worker.zig").Worker; +const Collector = @import("zml-smi/collector").Collector; const tpu_process = @import("process.zig"); const address = "localhost:8431"; -pub const target: device_info.Target = .tpu; - pub fn start(collector: *Collector) !void { const chip = scanPciChips(collector.arena, collector.io) orelse return error.TpuUnavailable; const device_count = chip.chip_count * chip.devices_per_chip; @@ -25,44 +22,40 @@ pub fn start(collector: *Collector) !void { try tpu_infos.append(collector.arena, info); } - try collector.worker.spawn(collector.io, pollAllDevices, .{ collector.io, collector.worker, tpu_infos.items }); + try collector.spawnPoll(pollAllDevices, .{tpu_infos.items}); if (tpu_infos.items.len > 0) { const processes = try collector.createProcessList(); - try tpu_process.init(collector.worker, collector.io, collector.gpa, chip.devices_per_chip, tpu_infos.items, processes, dev_offset); + try tpu_process.init(collector, chip.devices_per_chip, tpu_infos.items, processes, dev_offset); } } const MetricKind = enum { int, double }; -fn pollAllDevices(io: std.Io, w: *const Worker, infos: []*DeviceInfo) void { - w.pollLoop(io, struct { - fn poll(devs: []*DeviceInfo) void { - var mem_used: [tpuinfo.max_devices]?u64 = undefined; - var mem_total: [tpuinfo.max_devices]?u64 = undefined; - var util: [tpuinfo.max_devices]?u64 = undefined; +fn pollAllDevices(devs: []*DeviceInfo) void { + var mem_used: [tpuinfo.max_devices]?u64 = undefined; + var mem_total: [tpuinfo.max_devices]?u64 = undefined; + var util: [tpuinfo.max_devices]?u64 = undefined; - for (devs, 0..) |info, i| { - const front = info.tpu.front(); - mem_used[i] = front.mem_used_bytes; - mem_total[i] = front.mem_total_bytes; - util[i] = front.util_percent; - } + for (devs, 0..) |info, i| { + const front = info.tpu.front(); + mem_used[i] = front.mem_used_bytes; + mem_total[i] = front.mem_total_bytes; + util[i] = front.util_percent; + } - queryRaw("tpu.runtime.hbm.memory.usage.bytes", .int, mem_used[0..devs.len]); - queryRaw("tpu.runtime.hbm.memory.total.bytes", .int, mem_total[0..devs.len]); - queryRaw("tpu.runtime.tensorcore.dutycycle.percent", .double, util[0..devs.len]); + queryRaw("tpu.runtime.hbm.memory.usage.bytes", .int, mem_used[0..devs.len]); + queryRaw("tpu.runtime.hbm.memory.total.bytes", .int, mem_total[0..devs.len]); + queryRaw("tpu.runtime.tensorcore.dutycycle.percent", .double, util[0..devs.len]); - for (devs, 0..) |info, i| { - const back = info.tpu.back(); - back.* = info.tpu.front().*; - back.mem_used_bytes = mem_used[i]; - back.mem_total_bytes = mem_total[i]; - back.util_percent = util[i]; - info.tpu.swap(); - } - } - }.poll, .{infos}); + for (devs, 0..) |info, i| { + const back = info.tpu.back(); + back.* = info.tpu.front().*; + back.mem_used_bytes = mem_used[i]; + back.mem_total_bytes = mem_total[i]; + back.util_percent = util[i]; + info.tpu.swap(); + } } fn queryRaw(metric_name: [:0]const u8, comptime kind: MetricKind, out: []?u64) void { diff --git a/bin/zml-smi/bindings/tpu/process.zig b/bin/zml-smi/platforms/tpu/process.zig similarity index 65% rename from bin/zml-smi/bindings/tpu/process.zig rename to bin/zml-smi/platforms/tpu/process.zig index 21553449..62a73757 100644 --- a/bin/zml-smi/bindings/tpu/process.zig +++ b/bin/zml-smi/platforms/tpu/process.zig @@ -1,28 +1,17 @@ const std = @import("std"); -const pi = @import("../../info/process_info.zig"); -const ProcessDoubleBuffer = @import("../../utils/double_buffer.zig").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); -const DeviceInfo = @import("../../info/device_info.zig").DeviceInfo; -const Worker = @import("../../worker.zig").Worker; +const smi_info = @import("zml-smi/info"); +const pi = smi_info.process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const Collector = @import("zml-smi/collector").Collector; -pub fn init(w: *Worker, io: std.Io, allocator: std.mem.Allocator, devices_per_chip: u32, device_infos: []*DeviceInfo, list: *ProcessDoubleBuffer, dev_offset: u8) !void { - try w.spawn(io, scanLoop, .{ io, w, allocator, list, devices_per_chip, device_infos, dev_offset }); +pub fn init(collector: *Collector, devices_per_chip: u32, device_infos: []*DeviceInfo, list: *ProcessDoubleBuffer, dev_offset: u8) !void { + try collector.spawnPoll(pollOnce, .{ collector.io, collector.gpa, list, devices_per_chip, device_infos, dev_offset }); } -fn scanLoop(io: std.Io, w: *const Worker, allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, devices_per_chip: u32, device_infos: []*DeviceInfo, dev_offset: u8) void { - const interval: std.Io.Duration = .fromMilliseconds(w.poll_interval_ms); - - while (w.isRunning()) { - const start: std.Io.Timestamp = .now(io, .awake); - - scan(io, allocator, list.back(), devices_per_chip, device_infos, dev_offset); - - list.swap(); - - const elapsed = start.untilNow(io, .awake); - if (elapsed.nanoseconds < interval.nanoseconds) { - io.sleep(.fromNanoseconds(interval.nanoseconds - elapsed.nanoseconds), .awake) catch {}; - } - } +fn pollOnce(io: std.Io, allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, devices_per_chip: u32, device_infos: []*DeviceInfo, dev_offset: u8) void { + scan(io, allocator, list.back(), devices_per_chip, device_infos, dev_offset); + list.swap(); } fn scan(io: std.Io, allocator: std.mem.Allocator, back: *std.ArrayList(pi.ProcessInfo), devices_per_chip: u32, infos: []*DeviceInfo, dev_offset: u8) void { diff --git a/bin/zml-smi/bindings/tpu/libtpu/tpuinfo.cc b/bin/zml-smi/platforms/tpu/tpuinfo.cc similarity index 100% rename from bin/zml-smi/bindings/tpu/libtpu/tpuinfo.cc rename to bin/zml-smi/platforms/tpu/tpuinfo.cc diff --git a/bin/zml-smi/bindings/tpu/libtpu/tpuinfo.h b/bin/zml-smi/platforms/tpu/tpuinfo.h similarity index 100% rename from bin/zml-smi/bindings/tpu/libtpu/tpuinfo.h rename to bin/zml-smi/platforms/tpu/tpuinfo.h diff --git a/bin/zml-smi/bindings/tpu/tpuinfo.zig b/bin/zml-smi/platforms/tpu/tpuinfo.zig similarity index 100% rename from bin/zml-smi/bindings/tpu/tpuinfo.zig rename to bin/zml-smi/platforms/tpu/tpuinfo.zig diff --git a/bin/zml-smi/tui/BUILD.bazel b/bin/zml-smi/tui/BUILD.bazel new file mode 100644 index 00000000..93ca38f8 --- /dev/null +++ b/bin/zml-smi/tui/BUILD.bazel @@ -0,0 +1,50 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "tui", + import_name = "zml-smi/tui", + main = "tui.zig", + srcs = [ + "data.zig", + "image_cache.zig", + "lib/compose.zig", + "lib/image.zig", + "lib/ui.zig", + "lib/utils.zig", + "pages/detail.zig", + "pages/detail/detail_common.zig", + "pages/detail/gpu_detail.zig", + "pages/detail/neuron_detail.zig", + "pages/detail/tpu_detail.zig", + "pages/overview.zig", + "print.zig", + "theme.zig", + "top.zig", + "widgets/braille_chart.zig", + "widgets/chart.zig", + "widgets/column_layout.zig", + "widgets/device_card.zig", + "widgets/gauge.zig", + "widgets/info_lines.zig", + "widgets/logo.zig", + "widgets/metric_card.zig", + "widgets/process_table.zig", + "widgets/status_line.zig", + "widgets/titled_border.zig", + ], + extra_srcs = [ + "assets/amd.png", + "assets/logo.png", + "assets/neuron.png", + "assets/nvidia.png", + "assets/tpu.png", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/platforms/linux", + "//bazel/stamp:zig", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + "@libvaxis//:vaxis", + ], +) diff --git a/bin/zml-smi/tui/data.zig b/bin/zml-smi/tui/data.zig index efb9f028..49809210 100644 --- a/bin/zml-smi/tui/data.zig +++ b/bin/zml-smi/tui/data.zig @@ -1,17 +1,18 @@ const std = @import("std"); -const di = @import("../info/device_info.zig"); +const smi_info = @import("zml-smi/info"); +const di = smi_info.device_info; pub const DeviceInfo = di.DeviceInfo; pub const GpuInfo = di.GpuInfo; pub const NeuronInfo = di.NeuronInfo; pub const TpuInfo = di.TpuInfo; pub const Target = di.Target; -pub const Targets = @import("../platform.zig").Targets; -const hi = @import("../info/host_info.zig"); +pub const Targets = smi_info.Targets; +const hi = smi_info.host_info; pub const HostInfo = hi.HostInfo; pub const HostData = hi.HostData; -pub const pi = @import("../info/process_info.zig"); -pub const ProcessDoubleBuffer = @import("../utils/double_buffer.zig").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); -pub const ProcessEnricher = @import("../bindings/linux/process.zig").ProcessEnricher; +pub const pi = smi_info.process_info; +pub const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +pub const ProcessEnricher = @import("zml-smi/platforms/linux").process.ProcessEnricher; pub const history_len: usize = 500; diff --git a/bin/zml-smi/tui/pages/detail/gpu_detail.zig b/bin/zml-smi/tui/pages/detail/gpu_detail.zig index 6d440b4a..6f1c5d42 100644 --- a/bin/zml-smi/tui/pages/detail/gpu_detail.zig +++ b/bin/zml-smi/tui/pages/detail/gpu_detail.zig @@ -20,7 +20,19 @@ pub fn draw( parent_widget: vxfw.Widget, ) std.mem.Allocator.Error!vxfw.Surface { // ── Header ────────────────────────────────────────────── - const header = try common.headerText(ctx.arena, id, gpu.name orelse "Unknown"); + const name_suffix: []const u8 = blk: { + const drv = gpu.driver_version orelse ""; + const cuda = gpu.cuda_driver_version orelse ""; + break :blk if (drv.len > 0 and cuda.len > 0) + try std.fmt.allocPrint(ctx.arena, " ({s}, CUDA {s})", .{ drv, cuda }) + else if (drv.len > 0) + try std.fmt.allocPrint(ctx.arena, " ({s})", .{drv}) + else if (cuda.len > 0) + try std.fmt.allocPrint(ctx.arena, " (CUDA {s})", .{cuda}) + else + @as([]const u8, ""); + }; + const header = try common.headerText(ctx.arena, id, try std.fmt.allocPrint(ctx.arena, "{s}{s}", .{ gpu.name orelse "Unknown", name_suffix })); // ── GPU Utilization + Memory charts ───────────────────── const util_chart = try common.historyChart(ctx, state, id, state.history.util, .{ diff --git a/bin/zml-smi/tui/pages/detail/neuron_detail.zig b/bin/zml-smi/tui/pages/detail/neuron_detail.zig index 2629406f..209d5360 100644 --- a/bin/zml-smi/tui/pages/detail/neuron_detail.zig +++ b/bin/zml-smi/tui/pages/detail/neuron_detail.zig @@ -19,7 +19,11 @@ pub fn draw( parent_widget: vxfw.Widget, ) std.mem.Allocator.Error!vxfw.Surface { // ── Header ────────────────────────────────────────────── - const header = try common.headerText(ctx.arena, id, nc.name orelse "Unknown"); + const name_suffix: []const u8 = if (nc.driver_version) |v| + try std.fmt.allocPrint(ctx.arena, " ({s})", .{v}) + else + ""; + const header = try common.headerText(ctx.arena, id, try std.fmt.allocPrint(ctx.arena, "{s}{s}", .{ nc.name orelse "Unknown", name_suffix })); // ── Utilization + Memory charts ───────────────────────── const util_chart = try common.historyChart(ctx, state, id, state.history.util, .{ diff --git a/bin/zml-smi/tui/print.zig b/bin/zml-smi/tui/print.zig index c2e5c562..96c3737a 100644 --- a/bin/zml-smi/tui/print.zig +++ b/bin/zml-smi/tui/print.zig @@ -56,11 +56,6 @@ pub fn run(allocator: std.mem.Allocator, io: std.Io, state: *data.SystemState) ! const content_w: u16 = @min(@as(u16, @intCast(vx.screen.width)), Overview.host_line_width + 2); - // Neuron util% is delta-based; wait for 2 poll cycles to get a real sample. - if (state.targets.contains(.neuron)) { - io.sleep(.fromSeconds(2), .awake) catch {}; - } - var viewing_device: ?u8 = null; var overview = try Overview.init(allocator, state, null, &viewing_device); defer overview.deinit(allocator); diff --git a/bin/zml-smi/tui/top.zig b/bin/zml-smi/tui/top.zig new file mode 100644 index 00000000..6e00abf5 --- /dev/null +++ b/bin/zml-smi/tui/top.zig @@ -0,0 +1,166 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; + +const data = @import("data.zig"); +const ui = @import("lib/ui.zig"); +const compose = @import("lib/compose.zig"); +const image_cache = @import("image_cache.zig"); +const Logo = @import("widgets/logo.zig"); +const ProcessTable = @import("widgets/process_table.zig"); +const Overview = @import("pages/overview.zig"); +const Detail = @import("pages/detail.zig"); +const StatusLine = @import("widgets/status_line.zig"); + +const min_width: u16 = 45; +const min_height: u16 = 24; + +const ScrollState = struct { + row: i17 = 0, + col: i17 = 0, + + pub fn reset(self: *ScrollState) void { + self.row = 0; + self.col = 0; + } + + pub fn scrollBy(self: *ScrollState, dr: i17, dc: i17) void { + self.row = @max(0, self.row + dr); + self.col = @max(0, self.col + dc); + } + + pub fn clamp(self: *ScrollState, content_h: u16, viewport_h: u16, content_w: u16, viewport_w: u16) void { + const max_row: i17 = @max(0, @as(i17, content_h) - @as(i17, viewport_h)); + const max_col: i17 = @max(0, @as(i17, content_w) - @as(i17, viewport_w)); + self.row = std.math.clamp(self.row, 0, max_row); + self.col = std.math.clamp(self.col, 0, max_col); + } +}; + +const Model = struct { + allocator: std.mem.Allocator, + state: *data.SystemState, + viewing_device: ?u8 = null, + prev_viewing_device: ?u8 = null, + scroll: ScrollState = .{}, + vx: *vaxis.Vaxis, + tty: *vaxis.Tty, + process_table: ProcessTable = .{}, + overview: Overview = undefined, + + fn deinit(self: *Model) void { + self.overview.deinit(self.allocator); + image_cache.global.deinit(self.allocator); + } + + pub fn handleEvent(self: *Model, ctx: *vxfw.EventContext, event: vxfw.Event) anyerror!void { + switch (event) { + .init => { + self.overview = try Overview.init(self.allocator, self.state, &self.process_table, &self.viewing_device); + self.overview.use_braille = true; + image_cache.global.loadAll(self.vx, self.allocator, self.tty.writer()); + try ctx.tick(1000, ui.widget(self)); + }, + .tick => { + self.state.recordHistory(); + try ctx.tick(self.state.tui_refresh_rate, ui.widget(self)); + ctx.redraw = true; + }, + .key_press => |key| { + if (key.matches('q', .{}) or key.matches('c', .{ .ctrl = true })) { + ctx.quit = true; + return; + } + if (self.viewing_device != null) { + if (key.matches(vaxis.Key.escape, .{}) or key.matches(vaxis.Key.backspace, .{})) { + self.viewing_device = null; + return ctx.consumeAndRedraw(); + } + } else { + if (key.matches('v', .{})) { + self.overview.use_braille = !self.overview.use_braille; + return ctx.consumeAndRedraw(); + } + } + if (key.matches(vaxis.Key.down, .{})) { + self.scroll.scrollBy(1, 0); + } else if (key.matches(vaxis.Key.up, .{})) { + self.scroll.scrollBy(-1, 0); + } else if (key.matches(vaxis.Key.right, .{})) { + self.scroll.scrollBy(0, 1); + } else if (key.matches(vaxis.Key.left, .{})) { + self.scroll.scrollBy(0, -1); + } else return; + return ctx.consumeAndRedraw(); + }, + .mouse => |mouse| { + switch (mouse.button) { + .wheel_up => self.scroll.scrollBy(-3, 0), + .wheel_down => self.scroll.scrollBy(3, 0), + .wheel_left => self.scroll.scrollBy(0, 3), + .wheel_right => self.scroll.scrollBy(0, -3), + else => return, + } + return ctx.consumeAndRedraw(); + }, + else => {}, + } + } + + pub fn draw(self: *Model, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const screen = ctx.max.size(); + const content_w = @max(screen.width, min_width); + + if (self.viewing_device != self.prev_viewing_device) { + self.scroll.reset(); + self.process_table.resetScroll(); + self.prev_viewing_device = self.viewing_device; + } + + self.process_table.prepare(self.state, self.viewing_device); + + var content_surf: vxfw.Surface = undefined; + + if (self.viewing_device) |dev_id| { + const detail: Detail = .{ + .state = self.state, + .device_id = dev_id, + .process_table = &self.process_table, + }; + content_surf = try detail.draw(ui.fixedWidth(ctx, content_w)); + } else { + content_surf = try self.overview.draw(ui.fixedWidth(ctx, content_w)); + } + + content_surf.size.height = @max(content_surf.size.height, min_height); + + const content_h = screen.height -| 1; + self.scroll.clamp(content_surf.size.height, content_h, content_surf.size.width, screen.width); + + const status_line: StatusLine = .{ + .viewing_device = self.viewing_device, + .use_braille = self.overview.use_braille, + }; + const status_surf = try status_line.draw(ui.fixedSize(ctx, screen.width, 1)); + + var sb = compose.surfaceBuilder(ctx.arena); + try sb.add(-self.scroll.row, -self.scroll.col, content_surf); + try sb.addZ(@intCast(content_h), 0, status_surf, 1); + return sb.finish(screen, ui.widget(self)); + } +}; + +pub fn run(allocator: std.mem.Allocator, io: std.Io, state: *data.SystemState) !void { + var app = try vxfw.App.init(allocator, io); + defer app.deinit(); + + var model: Model = .{ + .allocator = allocator, + .state = state, + .vx = &app.vx, + .tty = &app.tty, + }; + defer model.deinit(); + + try app.run(ui.widget(&model), .{}); +} diff --git a/bin/zml-smi/tui/tui.zig b/bin/zml-smi/tui/tui.zig index 6e00abf5..015e3ce2 100644 --- a/bin/zml-smi/tui/tui.zig +++ b/bin/zml-smi/tui/tui.zig @@ -1,166 +1,3 @@ -const std = @import("std"); -const vaxis = @import("vaxis"); -const vxfw = vaxis.vxfw; - -const data = @import("data.zig"); -const ui = @import("lib/ui.zig"); -const compose = @import("lib/compose.zig"); -const image_cache = @import("image_cache.zig"); -const Logo = @import("widgets/logo.zig"); -const ProcessTable = @import("widgets/process_table.zig"); -const Overview = @import("pages/overview.zig"); -const Detail = @import("pages/detail.zig"); -const StatusLine = @import("widgets/status_line.zig"); - -const min_width: u16 = 45; -const min_height: u16 = 24; - -const ScrollState = struct { - row: i17 = 0, - col: i17 = 0, - - pub fn reset(self: *ScrollState) void { - self.row = 0; - self.col = 0; - } - - pub fn scrollBy(self: *ScrollState, dr: i17, dc: i17) void { - self.row = @max(0, self.row + dr); - self.col = @max(0, self.col + dc); - } - - pub fn clamp(self: *ScrollState, content_h: u16, viewport_h: u16, content_w: u16, viewport_w: u16) void { - const max_row: i17 = @max(0, @as(i17, content_h) - @as(i17, viewport_h)); - const max_col: i17 = @max(0, @as(i17, content_w) - @as(i17, viewport_w)); - self.row = std.math.clamp(self.row, 0, max_row); - self.col = std.math.clamp(self.col, 0, max_col); - } -}; - -const Model = struct { - allocator: std.mem.Allocator, - state: *data.SystemState, - viewing_device: ?u8 = null, - prev_viewing_device: ?u8 = null, - scroll: ScrollState = .{}, - vx: *vaxis.Vaxis, - tty: *vaxis.Tty, - process_table: ProcessTable = .{}, - overview: Overview = undefined, - - fn deinit(self: *Model) void { - self.overview.deinit(self.allocator); - image_cache.global.deinit(self.allocator); - } - - pub fn handleEvent(self: *Model, ctx: *vxfw.EventContext, event: vxfw.Event) anyerror!void { - switch (event) { - .init => { - self.overview = try Overview.init(self.allocator, self.state, &self.process_table, &self.viewing_device); - self.overview.use_braille = true; - image_cache.global.loadAll(self.vx, self.allocator, self.tty.writer()); - try ctx.tick(1000, ui.widget(self)); - }, - .tick => { - self.state.recordHistory(); - try ctx.tick(self.state.tui_refresh_rate, ui.widget(self)); - ctx.redraw = true; - }, - .key_press => |key| { - if (key.matches('q', .{}) or key.matches('c', .{ .ctrl = true })) { - ctx.quit = true; - return; - } - if (self.viewing_device != null) { - if (key.matches(vaxis.Key.escape, .{}) or key.matches(vaxis.Key.backspace, .{})) { - self.viewing_device = null; - return ctx.consumeAndRedraw(); - } - } else { - if (key.matches('v', .{})) { - self.overview.use_braille = !self.overview.use_braille; - return ctx.consumeAndRedraw(); - } - } - if (key.matches(vaxis.Key.down, .{})) { - self.scroll.scrollBy(1, 0); - } else if (key.matches(vaxis.Key.up, .{})) { - self.scroll.scrollBy(-1, 0); - } else if (key.matches(vaxis.Key.right, .{})) { - self.scroll.scrollBy(0, 1); - } else if (key.matches(vaxis.Key.left, .{})) { - self.scroll.scrollBy(0, -1); - } else return; - return ctx.consumeAndRedraw(); - }, - .mouse => |mouse| { - switch (mouse.button) { - .wheel_up => self.scroll.scrollBy(-3, 0), - .wheel_down => self.scroll.scrollBy(3, 0), - .wheel_left => self.scroll.scrollBy(0, 3), - .wheel_right => self.scroll.scrollBy(0, -3), - else => return, - } - return ctx.consumeAndRedraw(); - }, - else => {}, - } - } - - pub fn draw(self: *Model, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { - const screen = ctx.max.size(); - const content_w = @max(screen.width, min_width); - - if (self.viewing_device != self.prev_viewing_device) { - self.scroll.reset(); - self.process_table.resetScroll(); - self.prev_viewing_device = self.viewing_device; - } - - self.process_table.prepare(self.state, self.viewing_device); - - var content_surf: vxfw.Surface = undefined; - - if (self.viewing_device) |dev_id| { - const detail: Detail = .{ - .state = self.state, - .device_id = dev_id, - .process_table = &self.process_table, - }; - content_surf = try detail.draw(ui.fixedWidth(ctx, content_w)); - } else { - content_surf = try self.overview.draw(ui.fixedWidth(ctx, content_w)); - } - - content_surf.size.height = @max(content_surf.size.height, min_height); - - const content_h = screen.height -| 1; - self.scroll.clamp(content_surf.size.height, content_h, content_surf.size.width, screen.width); - - const status_line: StatusLine = .{ - .viewing_device = self.viewing_device, - .use_braille = self.overview.use_braille, - }; - const status_surf = try status_line.draw(ui.fixedSize(ctx, screen.width, 1)); - - var sb = compose.surfaceBuilder(ctx.arena); - try sb.add(-self.scroll.row, -self.scroll.col, content_surf); - try sb.addZ(@intCast(content_h), 0, status_surf, 1); - return sb.finish(screen, ui.widget(self)); - } -}; - -pub fn run(allocator: std.mem.Allocator, io: std.Io, state: *data.SystemState) !void { - var app = try vxfw.App.init(allocator, io); - defer app.deinit(); - - var model: Model = .{ - .allocator = allocator, - .state = state, - .vx = &app.vx, - .tty = &app.tty, - }; - defer model.deinit(); - - try app.run(ui.widget(&model), .{}); -} +pub const data = @import("data.zig"); +pub const print = @import("print.zig"); +pub const top = @import("top.zig"); diff --git a/bin/zml-smi/tui/widgets/device_card.zig b/bin/zml-smi/tui/widgets/device_card.zig index f646840b..ba025753 100644 --- a/bin/zml-smi/tui/widgets/device_card.zig +++ b/bin/zml-smi/tui/widgets/device_card.zig @@ -56,8 +56,30 @@ pub fn draw(self: *DeviceCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vx }; const mem_pct: u8 = if (cf.mem_total > 0) @intCast(@min(cf.mem_used * 100 / cf.mem_total, 100)) else 0; - const card_title = try std.fmt.allocPrint(ctx.arena, "{s} {d}: {s}", .{ target.deviceLabel(), i, cf.name }); - const util_label = target.utilLabel(); + const device_label, const util_label = switch (target) { + .cuda, .rocm => .{ "GPU", "GPU" }, + .neuron => .{ "NC", "Core" }, + .tpu => .{ "TPU", "Duty" }, + }; + const driver_ver: []const u8 = switch (dev.*) { + .cuda => |*sv| blk: { + const gpu = sv.front(); + const drv = gpu.driver_version orelse ""; + const cuda = gpu.cuda_driver_version orelse ""; + break :blk if (drv.len > 0 and cuda.len > 0) + try std.fmt.allocPrint(ctx.arena, " ({s}, CUDA {s})", .{ drv, cuda }) + else if (drv.len > 0) + try std.fmt.allocPrint(ctx.arena, " ({s})", .{drv}) + else if (cuda.len > 0) + try std.fmt.allocPrint(ctx.arena, " (CUDA {s})", .{cuda}) + else + @as([]const u8, ""); + }, + .rocm => |*sv| if (sv.front().driver_version) |v| try std.fmt.allocPrint(ctx.arena, " ({s})", .{v}) else "", + .neuron => |*sv| if (sv.front().driver_version) |v| try std.fmt.allocPrint(ctx.arena, " ({s})", .{v}) else "", + else => "", + }; + const card_title = try std.fmt.allocPrint(ctx.arena, "{s} {d}: {s}{s}", .{ device_label, i, cf.name, driver_ver }); const dev_suffix = switch (dev.*) { .cuda, .rocm => |*sv| blk: { diff --git a/bin/zml-smi/tui/widgets/info_lines.zig b/bin/zml-smi/tui/widgets/info_lines.zig index 3e7ac880..fcf1ae0d 100644 --- a/bin/zml-smi/tui/widgets/info_lines.zig +++ b/bin/zml-smi/tui/widgets/info_lines.zig @@ -7,6 +7,7 @@ const data = @import("../data.zig"); const compose = @import("../lib/compose.zig"); const ui = @import("../lib/ui.zig"); const theme = @import("../theme.zig"); +const stamp = @import("bazel/stamp"); const InfoLines = @This(); @@ -35,7 +36,7 @@ pub fn draw(self: *const InfoLines, ctx: vxfw.DrawContext) std.mem.Allocator.Err const w = ctx.max.width orelse 80; const host = self.state.host.front().*; - const hostname_str = try std.fmt.allocPrint(ctx.arena, "{s}", .{host.hostname orelse "Unkown"}); + const hostname_str = try std.fmt.allocPrint(ctx.arena, "{s}", .{host.hostname orelse "Unknown"}); const kernel_str = try std.fmt.allocPrint(ctx.arena, "{s}", .{host.kernel orelse "Unknown"}); const cpu_str = try std.fmt.allocPrint(ctx.arena, "{s}", .{host.cpu_name orelse "Unknown"}); const uptime_str = try formatUptime(ctx.arena, host.uptime_seconds orelse 0); @@ -50,7 +51,7 @@ pub fn draw(self: *const InfoLines, ctx: vxfw.DrawContext) std.mem.Allocator.Err const mem_str = try std.fmt.allocPrint(ctx.arena, "{d} / {d} GB", .{ used_gb, total_gb }); const devices_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{self.state.deviceCount()}); const values = [entry_count][]const u8{ - "v0.2", + stamp.stamp("STABLE_ZML_SMI_VERSION"), hostname_str, kernel_str, cpu_str, diff --git a/bin/zml-smi/tui/widgets/process_table.zig b/bin/zml-smi/tui/widgets/process_table.zig index e4b120c8..87956143 100644 --- a/bin/zml-smi/tui/widgets/process_table.zig +++ b/bin/zml-smi/tui/widgets/process_table.zig @@ -7,7 +7,7 @@ const ui = @import("../lib/ui.zig"); const compose = @import("../lib/compose.zig"); const TitledBorder = @import("titled_border.zig"); const ColumnLayout = @import("column_layout.zig"); -const pi = @import("../../info/process_info.zig"); +const pi = @import("zml-smi/info").process_info; const ProcessInfo = pi.ProcessInfo; const RichText = vxfw.RichText; diff --git a/bin/zml-smi/utils/BUILD.bazel b/bin/zml-smi/utils/BUILD.bazel new file mode 100644 index 00000000..a154556c --- /dev/null +++ b/bin/zml-smi/utils/BUILD.bazel @@ -0,0 +1,30 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "double_buffer", + import_name = "zml-smi/double_buffer", + main = "double_buffer.zig", + visibility = ["//visibility:public"], +) + +zig_library( + name = "dynlib", + import_name = "zml-smi/dynlib", + main = "dynlib.zig", + visibility = ["//visibility:public"], +) + +zig_library( + name = "sandbox", + import_name = "zml-smi/sandbox", + main = "sandbox.zig", + visibility = ["//visibility:public"], + deps = ["@rules_zig//zig/runfiles"], +) + +zig_library( + name = "sysfs", + import_name = "zml-smi/sysfs", + main = "sysfs.zig", + visibility = ["//visibility:public"], +) diff --git a/bin/zml-smi/utils/dynlib.zig b/bin/zml-smi/utils/dynlib.zig new file mode 100644 index 00000000..730b495d --- /dev/null +++ b/bin/zml-smi/utils/dynlib.zig @@ -0,0 +1,10 @@ +const std = @import("std"); + +pub fn lookupStruct(dynlib: *std.DynLib, comptime VTable: type) !VTable { + var result: VTable = undefined; + inline for (std.meta.fields(VTable)) |field| { + @field(result, field.name) = dynlib.lookup(field.type, field.name) orelse return error.SymbolResolutionFailed; + } + + return result; +} diff --git a/bin/zml-smi/worker.zig b/bin/zml-smi/worker.zig deleted file mode 100644 index 9bb4b748..00000000 --- a/bin/zml-smi/worker.zig +++ /dev/null @@ -1,62 +0,0 @@ -const std = @import("std"); - -pub const Worker = struct { - poll_interval_ms: u16, - should_stop: std.atomic.Value(bool) = .init(false), - group: std.Io.Group = .init, - - pub fn shutdown(self: *Worker, io: std.Io) void { - self.should_stop.store(true, .release); - self.group.await(io) catch {}; - } - - pub fn isRunning(self: *const Worker) bool { - return !self.should_stop.load(.acquire); - } - - pub fn spawn( - self: *Worker, - io: std.Io, - comptime runFn: anytype, - args: std.meta.ArgsTuple(@TypeOf(runFn)), - ) !void { - try self.group.concurrent(io, runFn, args); - } - - pub fn pollMetrics(comptime DB: type, comptime Dev: type, comptime table: anytype) fn (std.Io, *const Worker, DB, Dev) void { - return struct { - fn f(io: std.Io, w: *const Worker, db: DB, dev: Dev) void { - w.pollLoop(io, struct { - fn poll(s: DB, d: Dev) void { - if (@hasField(Dev, "arena")) { - _ = d.arena.reset(.retain_capacity); - } - - const back = s.back(); - back.* = s.front().*; - - inline for (table) |m| { - @field(back, m.field) = m.query(d) catch null; - } - - s.swap(); - } - }.poll, .{ db, dev }); - } - }.f; - } - - pub fn pollLoop(self: *const Worker, io: std.Io, comptime func: anytype, args: std.meta.ArgsTuple(@TypeOf(func))) void { - const interval: std.Io.Duration = .fromMilliseconds(self.poll_interval_ms); - while (!self.should_stop.load(.acquire)) { - const start: std.Io.Timestamp = .now(io, .awake); - - @call(.auto, func, args); - - const elapsed = start.untilNow(io, .awake); - if (elapsed.nanoseconds < interval.nanoseconds) { - io.sleep(.fromNanoseconds(interval.nanoseconds - elapsed.nanoseconds), .awake) catch {}; - } - } - } -}; diff --git a/platforms/neuron/BUILD.bazel b/platforms/neuron/BUILD.bazel index 4927764a..91cd6cc9 100644 --- a/platforms/neuron/BUILD.bazel +++ b/platforms/neuron/BUILD.bazel @@ -66,7 +66,7 @@ cc_library( name = "libnrt_headers", tags = ["manual"], hdrs = ["nrt.h"], - visibility = ["//bin/zml-smi:__pkg__"], + visibility = ["//bin/zml-smi:__subpackages__"], deps = ["@libpjrt_neuron//:libnrt_headers"], ) diff --git a/platforms/neuron/nrt.h b/platforms/neuron/nrt.h index 0da39452..9970d0bd 100644 --- a/platforms/neuron/nrt.h +++ b/platforms/neuron/nrt.h @@ -1,4 +1,5 @@ #include #include +#include #include #include diff --git a/tools/workspace_status.sh b/tools/workspace_status.sh index 3af7cd3b..05f02bed 100755 --- a/tools/workspace_status.sh +++ b/tools/workspace_status.sh @@ -4,3 +4,5 @@ if git diff --quiet; then else echo BUILD_SCM_REVISION "dev" fi + +echo STABLE_ZML_SMI_VERSION "$(git describe --tags --dirty --match 'zml-smi-v*' 2>/dev/null || echo "dev")" From adee932efa875d181cfc3ec8cbc4eb822db76035 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 7 Apr 2026 19:05:43 +0200 Subject: [PATCH 408/881] zml/io/vfs: add GCS VFS (#453) This pull request adds support for Google Cloud Storage (GCS) to the virtual file system (VFS) module, alongside some build configuration updates. The main changes introduce the new `GCS` backend, make it available in the VFS API, and register it in the example application. Authentication is handled like other GCS libraries via the `GOOGLE_APPLICATION_CREDENTIALS` env or application default account files. **IMPORTANT NOTE**: Only Authorized User and Metadata Server auth are implemented. Service Account auth isn't implemented yet. --- examples/io/main.zig | 4 + zml/io/vfs.zig | 1 + zml/io/vfs/BUILD.bazel | 9 +- zml/io/vfs/gcs.zig | 974 +++++++++++++++++++++++++++++++++++++++++ zml/io/vfs/index.zig | 1 + 5 files changed, 985 insertions(+), 4 deletions(-) create mode 100644 zml/io/vfs/gcs.zig diff --git a/examples/io/main.zig b/examples/io/main.zig index 2d14db6b..658bf2bb 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -40,6 +40,9 @@ pub fn main(init: std.process.Init) !void { var s3_vfs: zml.io.VFS.S3 = try .auto(allocator, init.io, &http_client, init.environ_map); defer s3_vfs.deinit(); + var gcs_vfs: zml.io.VFS.GCS = try .auto(allocator, init.io, &http_client, init.environ_map); + defer gcs_vfs.deinit(); + var vfs: zml.io.VFS = try .init(allocator, init.io); defer vfs.deinit(); @@ -47,6 +50,7 @@ pub fn main(init: std.process.Init) !void { try vfs.register("https", vfs_https.io()); try vfs.register("hf", hf_vfs.io()); try vfs.register("s3", s3_vfs.io()); + try vfs.register("gs", gcs_vfs.io()); const io = vfs.io(); diff --git a/zml/io/vfs.zig b/zml/io/vfs.zig index a282371c..e1cec6f2 100644 --- a/zml/io/vfs.zig +++ b/zml/io/vfs.zig @@ -9,6 +9,7 @@ pub const VFS = struct { pub const HTTP = vfs.HTTP; pub const HF = vfs.HF; pub const S3 = vfs.S3; + pub const GCS = vfs.GCS; const VFSBase = vfs.VFSBase; const Handles = std.ArrayList(Handle); diff --git a/zml/io/vfs/BUILD.bazel b/zml/io/vfs/BUILD.bazel index 86221e9b..6d53b5d2 100644 --- a/zml/io/vfs/BUILD.bazel +++ b/zml/io/vfs/BUILD.bazel @@ -1,18 +1,19 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_binary") +load("@rules_zig//zig:defs.bzl", "zig_library") zig_library( name = "vfs", - tags = ["manual"], srcs = [ "base.zig", "file.zig", + "gcs.zig", "hf.zig", "http.zig", "s3.zig", ], + main = "index.zig", + tags = ["manual"], + visibility = ["//visibility:public"], deps = [ "//stdx", ], - main = "index.zig", - visibility = ["//visibility:public"], ) diff --git a/zml/io/vfs/gcs.zig b/zml/io/vfs/gcs.zig new file mode 100644 index 00000000..98d1f9bf --- /dev/null +++ b/zml/io/vfs/gcs.zig @@ -0,0 +1,974 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const stdx = @import("stdx"); + +const VFSBase = @import("base.zig").VFSBase; + +const log = std.log.scoped(.@"zml/io/vfs/gcs"); + +const EndpointUrl = "https://storage.googleapis.com"; +const MetadataUrl = "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"; + +/// Check if running on Google Compute Engine / Cloud Run host infrastructure, +/// because metadata server probing should only happen on GCP. +/// Mirrors the TPU platform detection approach. +fn isOnGCP(io: std.Io) !bool { + if (builtin.os.tag != .linux) return false; + + const GoogleComputeEngine = "Google Compute Engine"; + var buffer: [GoogleComputeEngine.len]u8 = undefined; + return std.mem.eql( + u8, + GoogleComputeEngine, + try std.Io.Dir.readFile( + .cwd(), + io, + "/sys/devices/virtual/dmi/id/product_name", + &buffer, + ), + ); +} + +const OAuthToken = struct { + // https://developers.google.com/identity/protocols/oauth2#size + const Size = 2048; + + access_token: []const u8, + expires_in: u64, +}; + +const Credentials = union(enum) { + const AuthorizedUser = struct { + type: []const u8, + client_id: []const u8, + client_secret: []const u8, + refresh_token: []const u8, + quota_project_id: ?[]const u8 = null, + account: []const u8, + universe_domain: []const u8, + }; + + const ServiceAccount = struct { + pub const JwtAssertion = struct { + iss: []const u8, + scope: []const u8 = "https://www.googleapis.com/auth/devstorage.read_only", + aud: []const u8 = "https://oauth2.googleapis.com/token", + exp: i64, + iat: i64, + sub: ?[]const u8 = null, + }; + + type: []const u8, + client_email: []const u8, + private_key: []const u8, + token_uri: []const u8, + quota_project_id: ?[]const u8 = null, + + pub fn makeAssertion(self: *const ServiceAccount, io_: std.Io) JwtAssertion { + const now: std.Io.Timestamp = .now(io_, .real); + return .{ + .iss = self.client_email, + .iat = now.toSeconds(), + .exp = now.addDuration(.fromSeconds(3600)).toSeconds(), + }; + } + }; + + authorized_user: AuthorizedUser, + service_account: ServiceAccount, + metadata_server: void, + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!Credentials { + return jsonParseFromValue( + allocator, + try std.json.innerParse(std.json.Value, allocator, source, options), + options, + ); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) std.json.ParseFromValueError!Credentials { + return switch (source) { + .object => |obj| blk: { + const obj_type = obj.get("type") orelse return std.json.ParseFromValueError.UnexpectedToken; + if (std.ascii.eqlIgnoreCase(obj_type.string, "authorized_user")) { + break :blk .{ + .authorized_user = try std.json.parseFromValueLeaky( + AuthorizedUser, + allocator, + source, + options, + ), + }; + } else if (std.ascii.eqlIgnoreCase(obj_type.string, "service_account")) { + break :blk .{ + .service_account = try std.json.parseFromValueLeaky( + ServiceAccount, + allocator, + source, + options, + ), + }; + } + unreachable; + }, + else => std.json.ParseFromValueError.UnexpectedToken, + }; + } +}; + +const ReadState = struct { index: usize, objects: [][]const u8 }; + +pub const GCS = struct { + const Token = struct { + header: []u8, + expires_at: std.Io.Timestamp, + + fn expired(self: *const Token, io_: std.Io) bool { + const now: std.Io.Timestamp = .now(io_, .real); + return now.toSeconds() >= self.expires_at.toSeconds(); + } + }; + + pub const Config = struct { + credentials: ?Credentials = null, + endpoint_uri: std.Uri, + region: []const u8 = "auto", + }; + + const Handle = struct { + pub const Type = enum { file, directory }; + + type: Type, + uri: []const u8, + pos: u64, + size: u64, + + pub fn init(allocator: std.mem.Allocator, type_: Type, path: []const u8, size: u64) !Handle { + return .{ + .type = type_, + .uri = try allocator.dupe(u8, path), + .pos = 0, + .size = size, + }; + } + + pub fn deinit(self: *Handle, allocator: std.mem.Allocator) void { + allocator.free(self.uri); + } + }; + + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + mutex: std.Io.Mutex = .init, + client: *std.http.Client, + config: Config, + token: Token, + handles: stdx.SegmentedList(Handle, 0) = .{}, + closed_handles: std.ArrayList(u32) = .{}, + dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, + base: VFSBase, + + pub const InitArgs = struct { + credentials: ?union(enum) { + json: *std.Io.Reader, + metadata_server: void, + } = null, + endpoint_url: []const u8 = "https://storage.googleapis.com", + region: []const u8 = "auto", + }; + + pub const InitError = error{ + InvalidCredentialJson, + RequestFailed, + Unexpected, + } || std.mem.Allocator.Error; + + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, args: InitArgs) InitError!GCS { + var arena: std.heap.ArenaAllocator = .init(allocator); + errdefer arena.deinit(); + + const config: Config = .{ + .credentials = if (args.credentials) |creds| switch (creds) { + .json => |reader| blk: { + var json_reader: std.json.Reader = .init(allocator, reader); + defer json_reader.deinit(); + + break :blk std.json.parseFromTokenSourceLeaky( + Credentials, + arena.allocator(), + &json_reader, + .{ .allocate = .alloc_always }, + ) catch return InitError.InvalidCredentialJson; + }, + .metadata_server => .{ .metadata_server = {} }, + } else null, + .endpoint_uri = std.Uri.parse(try arena.allocator().dupe(u8, args.endpoint_url)) catch return InitError.Unexpected, + .region = try arena.allocator().dupe(u8, args.region), + }; + + const token: Token = .{ + .header = try arena.allocator().alloc(u8, "Bearer ".len + OAuthToken.Size), + .expires_at = .zero, + }; + + return .{ + .allocator = allocator, + .arena = arena, + .base = .init(inner), + .client = http_client, + .config = config, + .token = token, + }; + } + + pub fn auto(allocator: std.mem.Allocator, inner_io: std.Io, http_client: *std.http.Client, environ_map: *std.process.Environ.Map) !GCS { + var jsonBuffer: [1024]u8 = undefined; + + if (environ_map.get("GOOGLE_APPLICATION_CREDENTIALS")) |json_path| { + var f = try std.Io.Dir.openFile(.cwd(), inner_io, json_path, .{}); + defer f.close(inner_io); + var reader = f.reader(inner_io, &jsonBuffer); + return try .init(allocator, inner_io, http_client, .{ .credentials = .{ .json = &reader.interface } }); + } + + if (applicationDefaultCredentials(inner_io, environ_map)) |f| { + defer f.close(inner_io); + var reader = f.reader(inner_io, &jsonBuffer); + return .init(allocator, inner_io, http_client, .{ .credentials = .{ .json = &reader.interface } }); + } + + if (isOnGCP(inner_io) catch false) { + return .init(allocator, inner_io, http_client, .{ .credentials = .{ .metadata_server = {} } }); + } + + return .init(allocator, inner_io, http_client, .{}); + } + + fn applicationDefaultCredentials(io_: std.Io, environ_map: *std.process.Environ.Map) ?std.Io.File { + var buffer: [1024]u8 = undefined; + const path = stdx.Io.Dir.path.bufJoin(&buffer, switch (builtin.os.tag) { + .windows => &.{ environ_map.get("APPDATA") orelse return null, "gcloud", "application_default_credentials.json" }, + else => &.{ environ_map.get("HOME") orelse return null, ".config", "gcloud", "application_default_credentials.json" }, + }) catch unreachable; + return std.Io.Dir.openFile(.cwd(), io_, path, .{}) catch null; + } + + fn refreshMetadataServerToken(client: *std.http.Client, buffer: []u8) !?[]const u8 { + var response_writer: std.Io.Writer = .fixed(buffer); + const result = try client.fetch(.{ + .location = .{ .url = MetadataUrl }, + .method = .GET, + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + }, + .extra_headers = &.{ + .{ .name = "Metadata-Flavor", .value = "Google" }, + }, + .response_writer = &response_writer, + }); + + if (result.status != .ok) { + return null; + } + + return response_writer.buffered(); + } + + fn refreshAuthorizedUserToken(client: *std.http.Client, authorized_user: Credentials.AuthorizedUser, buffer: []u8) ![]const u8 { + var response_writer: std.Io.Writer = .fixed(buffer); + const result = try client.fetch(.{ + .location = .{ .url = "https://oauth2.googleapis.com/token" }, + .method = .POST, + .payload = try std.fmt.bufPrint(buffer, "grant_type=refresh_token&client_id={f}&client_secret={f}&refresh_token={f}", .{ + std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_id }, .formatQuery), + std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_secret }, .formatQuery), + std.fmt.alt(std.Uri.Component{ .raw = authorized_user.refresh_token }, .formatQuery), + }), + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .content_type = .{ .override = "application/x-www-form-urlencoded" }, + }, + .response_writer = &response_writer, + }); + if (result.status != .ok) { + log.err("Failed to refresh ADC token: {s}", .{response_writer.buffered()}); + return error.RequestFailed; + } + return response_writer.buffered(); + } + + fn refreshServiceAccountToken(io_: std.Io, client: *std.http.Client, service_account: Credentials.ServiceAccount, buffer: []u8) ![]const u8 { + _ = io_; + _ = client; + _ = service_account; + _ = buffer; + return error.Unimplemented; + + // const assertion = service_account.makeAssertion(io_); + + // var tmp_buffer: [8 * 1024]u8 = undefined; + + // var buffer_writer: std.Io.Writer = .fixed(buffer); + // var signing_input_buffer: [2 * 1024]u8 = undefined; + // var signing_input_writer: std.Io.Writer = .fixed(signing_input_buffer); + + // const encoded_rs256_header = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"; + // try signing_input_writer.print("{s}.", .{encoded_rs256_header}); + // std.base64.url_safe_no_pad.Encoder.encodeWriter( + // signing_input_writer, + // try std.fmt.bufPrint(tmp_buffer, "{f}", .{ + // std.json.fmt(assertion, .{}), + // }), + // ); + // const signing_input = signing_input_writer.buffered(); + // _ = signing_input; + + // const result = try client.fetch(.{ + // .location = .{ .url = "https://oauth2.googleapis.com/token" }, + // .method = .POST, + // .payload = try std.fmt.bufPrint(buffer, "grant_type=refresh_token&client_id={f}&client_secret={f}&refresh_token={f}", .{ + // std.fmt.alt(std.Uri.Component{ .raw = service_account.client_id }, .formatQuery), + // std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_secret }, .formatQuery), + // std.fmt.alt(std.Uri.Component{ .raw = authorized_user.refresh_token }, .formatQuery), + // }), + // .headers = .{ + // .accept_encoding = .{ .override = "identity" }, + // .content_type = .{ .override = "application/x-www-form-urlencoded" }, + // }, + // .response_writer = &response_writer, + // }); + // if (result.status != .ok) { + // log.err("Failed to refresh ADC token: {s}", .{response_writer.buffered()}); + // return error.RequestFailed; + // } + } + + fn refreshToken(self: *GCS) !void { + var buffer: [4 * 1024]u8 = undefined; + const payload = switch (self.config.credentials.?) { + .authorized_user => |authorized_user| try refreshAuthorizedUserToken(self.client, authorized_user, &buffer), + .service_account => |service_account| try refreshServiceAccountToken(self.base.inner, self.client, service_account, &buffer), + .metadata_server => try refreshMetadataServerToken(self.client, &buffer) orelse return error.RequestFailed, + }; + const oauth_token = try std.json.parseFromSlice(OAuthToken, self.allocator, payload, .{ + .allocate = .alloc_if_needed, + .ignore_unknown_fields = true, + }); + defer oauth_token.deinit(); + self.token = .{ + .header = try std.fmt.bufPrint(self.token.header, "Bearer {s}", .{oauth_token.value.access_token}), + .expires_at = std.Io.Clock.now(.real, self.base.inner).addDuration(.fromSeconds(@intCast(oauth_token.value.expires_in))), + }; + } + + fn getOrRefreshToken(self: *GCS) !std.http.Client.Request.Headers.Value { + if (self.config.credentials == null) { + return .omit; + } + if (self.token.expired(self.base.inner)) { + try self.refreshToken(); + } + return .{ .override = self.token.header }; + } + + pub fn deinit(self: *GCS) void { + var idx: usize = 0; + while (idx < self.handles.len) : (idx += 1) { + const is_closed = for (self.closed_handles.items) |closed_idx| { + if (closed_idx == idx) break true; + } else false; + + if (!is_closed) { + self.handles.at(idx).deinit(self.allocator); + } + } + self.handles.deinit(self.allocator); + self.closed_handles.deinit(self.allocator); + + var it = self.dir_read_states.iterator(); + while (it.next()) |entry| { + for (entry.value_ptr.objects) |obj| { + self.allocator.free(obj); + } + self.allocator.free(entry.value_ptr.objects); + } + self.dir_read_states.deinit(self.allocator); + self.arena.deinit(); + } + + pub fn io(self: *GCS) std.Io { + return .{ + .userdata = &self.base, + .vtable = &comptime VFSBase.vtable(.{ + .operate = operate, + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRead = dirRead, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileRealPath = fileRealPath, + }), + }; + } + + fn openHandle(self: *GCS) !struct { u32, *Handle } { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + if (self.closed_handles.pop()) |idx| return .{ idx, self.handles.at(idx) }; + + return .{ @intCast(self.handles.len), try self.handles.addOne(self.allocator) }; + } + + fn closeHandle(self: *GCS, idx: u32) !void { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + self.handles.at(idx).deinit(self.allocator); + try self.closed_handles.append(self.allocator, idx); + } + + fn getFileHandle(self: *GCS, file: std.Io.File) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + return self.handles.at(@intCast(file.handle)); + } + + fn getDirHandle(self: *GCS, dir: std.Io.Dir) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + return self.handles.at(@intCast(dir.handle)); + } + + fn resolvePath(self: *GCS, dir: std.Io.Dir, sub_path: []const u8, out_buffer: []u8) ![]u8 { + if (std.meta.eql(dir, std.Io.Dir.cwd())) { + return try std.fmt.bufPrint(out_buffer, "{s}", .{sub_path}); + } + + const handle = self.getDirHandle(dir); + + const trimmed_uri = std.mem.trimEnd(u8, handle.uri, "/"); + const trimmed_sub_path = std.mem.trimStart(u8, sub_path, "/"); + + if (trimmed_uri.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_sub_path}); + if (trimmed_sub_path.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_uri}); + + return try std.fmt.bufPrint(out_buffer, "{s}/{s}", .{ trimmed_uri, trimmed_sub_path }); + } + + fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + switch (operation) { + .file_read_streaming => |o| { + const handle = self.getFileHandle(o.file); + const total = self.performRead(handle, o.data, handle.pos) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); + return .{ .file_read_streaming = std.Io.File.ReadStreamingError.EndOfStream }; + }; + + if (total == 0) { + return .{ .file_read_streaming = std.Io.File.ReadStreamingError.EndOfStream }; + } + + handle.pos += @intCast(total); + return .{ .file_read_streaming = total }; + }, + .file_write_streaming, .device_io_control => { + return self.base.inner.vtable.operate(self.base.inner.userdata, operation); + }, + } + } + + fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.OpenError.SystemResources; + + const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .directory, path, 0) catch return std.Io.Dir.OpenError.Unexpected; + + return .{ .handle = @intCast(idx) }; + } + + fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const handle = self.getDirHandle(dir); + + return .{ + .inode = @intCast(@intFromPtr(handle)), + .nlink = 0, + .size = handle.size, + .permissions = .fromMode(0o444), + .kind = .directory, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + .block_size = 1, + }; + } + + fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { + error.FileNotFound => return std.Io.File.OpenError.FileNotFound, + error.PermissionDenied => return std.Io.File.OpenError.PermissionDenied, + else => return std.Io.File.OpenError.Unexpected, + }; + + return .{ + .inode = 0, + .nlink = 0, + .size = size, + .permissions = .fromMode(0o444), + .kind = .file, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + .block_size = 1, + }; + } + + fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} + + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { + error.FileNotFound => return std.Io.File.OpenError.FileNotFound, + error.PermissionDenied => return std.Io.File.OpenError.PermissionDenied, + else => return std.Io.File.OpenError.Unexpected, + }; + + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; + + return .{ .handle = @intCast(idx), .flags = .{ .nonblocking = false } }; + } + + fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + for (dirs) |dir| { + self.closeHandle(@intCast(dir.handle)) catch unreachable; + } + } + + fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + _ = reader; + _ = entries; + _ = self; + + return 0; + + // if (reader.state == .finished) return 0; + + // if (reader.state == .reset) { + // if (self.dir_read_states.fetchRemove(reader)) |kv| { + // for (kv.value.objects) |obj| self.allocator.free(obj); + // self.allocator.free(kv.value.objects); + // } + + // const handle = self.getDirHandle(reader.dir); + // const objects = self.listObjects(handle.uri) catch return std.Io.Dir.Reader.Error.Unexpected; + + // self.dir_read_states.put(self.allocator, reader, .{ + // .index = 0, + // .objects = objects, + // }) catch return std.Io.Dir.Reader.Error.Unexpected; + + // reader.state = if (objects.len > 0) .reading else .finished; + + // if (objects.len == 0) return 0; + // } + + // const state = self.dir_read_states.getPtr(reader) orelse return std.Io.Dir.Reader.Error.Unexpected; + + // var count: usize = 0; + // while (count < entries.len and state.index < state.objects.len) { + // const obj_key = state.objects[state.index]; + // const kind: std.Io.File.Kind = if (std.mem.endsWith(u8, obj_key, "/")) .directory else .file; + + // const name = if (std.mem.lastIndexOfScalar(u8, std.mem.trimEnd(u8, obj_key, "/"), '/')) |idx| + // obj_key[idx + 1 ..] + // else + // obj_key; + + // entries[count] = .{ + // .name = std.mem.trimEnd(u8, name, "/"), + // .kind = kind, + // .inode = state.index, + // }; + // count += 1; + // state.index += 1; + // } + + // if (state.index >= state.objects.len) { + // reader.state = .finished; + // } + + // return count; + } + + fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const handle = self.getDirHandle(dir); + const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.Dir.RealPathError.SystemResources; + return path.len; + } + + fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const real_path = self.resolvePath(dir, path_name, out_buffer) catch return std.Io.Dir.RealPathFileError.NameTooLong; + return real_path.len; + } + + fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const handle = self.getFileHandle(file); + + return .{ + .inode = @intCast(file.handle), + .nlink = 0, + .size = handle.size, + .permissions = .fromMode(0o444), + .kind = .file, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + .block_size = 1, + }; + } + + fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + return self.getFileHandle(file).size; + } + + fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + for (files) |file| { + self.closeHandle(@intCast(file.handle)) catch unreachable; + } + } + + fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const handle = self.getFileHandle(file); + return self.performRead(handle, data, offset) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); + return std.Io.File.Reader.Error.Unexpected; + }; + } + + fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const handle = self.getFileHandle(file); + + if (relative_offset >= 0) { + handle.pos = handle.pos +| @as(u64, @intCast(relative_offset)); + } else { + const abs_offset: u64 = @intCast(-relative_offset); + if (abs_offset > handle.pos) { + handle.pos = 0; + } else { + handle.pos -= abs_offset; + } + } + } + + fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const handle = self.getFileHandle(file); + handle.pos = absolute_offset; + } + + fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const handle = self.getFileHandle(file); + const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.File.RealPathError.SystemResources; + return path.len; + } + + fn getTimestamp(self: *GCS, buf: []u8) ![]const u8 { + const now_ts = std.Io.Clock.now(.real, self.base.inner); + + const now: u64 = @intCast(@divFloor(now_ts.nanoseconds, std.time.ns_per_s)); + const epoch_secs: std.time.epoch.EpochSeconds = .{ .secs = now }; + const year_day = epoch_secs.getEpochDay().calculateYearDay(); + const month_day = year_day.calculateMonthDay(); + const day_seconds = epoch_secs.getDaySeconds(); + + return try std.fmt.bufPrint(buf, "{d:0>4}{d:0>2}{d:0>2}T{d:0>2}{d:0>2}{d:0>2}Z", .{ + year_day.year, + @intFromEnum(month_day.month), + month_day.day_index + 1, + day_seconds.getHoursIntoDay(), + day_seconds.getMinutesIntoHour(), + day_seconds.getSecondsIntoMinute(), + }); + } + + // fn pathComponents(self: *GCS, path_: []const u8) struct { []const u8, []const u8, []const u8 } { + // const endpoint = std.mem.trimEnd(u8, self.config.endpoint_url, "/"); + // const path = std.mem.trim(u8, path_, "/"); + + // if (std.mem.findScalar(u8, path, '/')) |idx| { + // return .{ endpoint, path[0..idx], if (idx + 1 < path.len) path[idx + 1 ..] else "" }; + // } else { + // return .{ endpoint, path, "" }; + // } + // } + + fn gcsUri(self: *GCS, path: []const u8) std.Uri { + var uri = self.config.endpoint_uri; + uri.path = .{ .raw = path }; + uri.query = .{ .percent_encoded = "alt=media" }; + return uri; + } + + // fn authHeaders(self: *GCS, _: std.http.Method, _: std.Uri, _: []const u8, authorization_buffer: []u8) !struct { + // authorization: ?[]const u8, + // extra_headers: [3]?std.http.Header, + // extra_len: usize, + // } { + // if (self.config.oauth_access_token == null) { + // if (self.config.googleCredentialsJsonStr) |json| { + // const oauth = try refreshAuthorizedUserTokenFromJson(self.allocator, self.base.inner, json); + // if (self.config.oauth_access_token) |old_token| self.allocator.free(old_token); + // if (self.config.quota_project_id) |old_project| self.allocator.free(old_project); + // self.config.oauth_access_token = oauth.access_token; + // self.config.quota_project_id = oauth.quota_project_id; + // } + // } + + // const token = self.config.oauth_access_token orelse return error.MissingCredentials; + // const authorization = try std.fmt.bufPrint(authorization_buffer, "Bearer {s}", .{token}); + // var extra_headers: [3]?std.http.Header = .{ null, null, null }; + // var extra_len: usize = 0; + // if (self.config.quota_project_id) |project| { + // extra_headers[0] = .{ .name = "x-goog-user-project", .value = project }; + // extra_len = 1; + // } + // return .{ + // .authorization = authorization, + // .extra_headers = extra_headers, + // .extra_len = extra_len, + // }; + // } + + // fn listObjects(self: *GCS, prefix: []const u8) ![][]const u8 { + // const endpoint, const bucket, const key_prefix = self.pathComponents(prefix); + + // var query_buf: [4096]u8 = undefined; + // var query_writer = std.Io.Writer.fixed(&query_buf); + + // try query_writer.writeAll("delimiter="); + // try std.Uri.Component.percentEncode(&query_writer, "/", gcsEncodeIsValid); + + // if (key_prefix.len > 0) { + // try query_writer.writeAll("&prefix="); + // try std.Uri.Component.percentEncode(&query_writer, key_prefix, gcsEncodeIsValid); + // try std.Uri.Component.percentEncode(&query_writer, "/", gcsEncodeIsValid); + // } + + // const endpoint_uri = try std.Uri.parse(endpoint); + + // var path_buf: [256]u8 = undefined; + + // const uri: std.Uri = .{ + // .scheme = endpoint_uri.scheme, + // .host = endpoint_uri.host, + // .port = endpoint_uri.port, + // .path = .{ .percent_encoded = try std.fmt.bufPrint(&path_buf, "/{s}", .{bucket}) }, + // .query = .{ .percent_encoded = query_writer.buffered() }, + // .fragment = null, + // }; + + // var timestamp_buf: [16]u8 = undefined; + // const timestamp = try self.getTimestamp(×tamp_buf); + + // var authorization_buffer: [1024]u8 = undefined; + // const auth = try self.authHeaders(.GET, uri, timestamp, &authorization_buffer); + + // var extra_headers_buf: [3]std.http.Header = undefined; + // for (0..auth.extra_len) |i| { + // extra_headers_buf[i] = auth.extra_headers[i].?; + // } + + // var req = try self.client.request(.GET, uri, .{ + // .redirect_behavior = .not_allowed, + // .headers = .{ + // .accept_encoding = .{ .override = "identity" }, + // .authorization = if (self.getOrRefreshToken()) |hdr| .{ .override = hdr } else .omit, + // }, + // .extra_headers = if (self.config.quota_project_id) |project| + // &.{.{ .name = "x-goog-user-project", .value = project }} + // else + // &.{}, + // }); + // defer req.deinit(); + + // try req.sendBodiless(); + + // var redirect_buffer: [2 * 1024]u8 = undefined; + // var res = try req.receiveHead(&redirect_buffer); + + // if (res.head.status != .ok) { + // log.err("Failed to list object {f}", .{uri}); + // log.err("{s}", .{res.head.bytes}); + // return error.RequestFailed; + // } + + // const body = try res.reader(&.{}).readAlloc(self.allocator, res.head.content_length orelse 1024 * 1024); + // defer self.allocator.free(body); + + // return try self.parseListObjectsResponse(body, key_prefix); + // } + + // fn parseListObjectsResponse(self: *GCS, xml: []const u8, prefix: []const u8) ![][]const u8 { + // var results = std.ArrayListUnmanaged([]const u8){}; + // errdefer { + // for (results.items) |item| self.allocator.free(item); + // results.deinit(self.allocator); + // } + + // inline for (.{ "Key", "Prefix" }) |tag| { + // var pos: usize = 0; + // while (std.mem.indexOfPos(u8, xml, pos, "<" ++ tag ++ ">")) |start| { + // const v_start = start + tag.len + 2; + // const end = std.mem.indexOfPos(u8, xml, v_start, "") orelse break; + // const val = xml[v_start..end]; + + // const is_self = std.mem.eql(u8, std.mem.trimEnd(u8, val, "/"), std.mem.trimEnd(u8, prefix, "/")); + // if (val.len > 0 and !is_self) { + // try results.append(self.allocator, try self.allocator.dupe(u8, val)); + // } + // pos = end + tag.len + 3; + // } + // } + + // return try results.toOwnedSlice(self.allocator); + // } + + fn fetchSize(self: *GCS, dir: std.Io.Dir, sub_path: []const u8) !u64 { + var path_buffer: [8 * 1024]u8 = undefined; + const path = try self.resolvePath(dir, sub_path, &path_buffer); + const uri = self.gcsUri(path); + var req = try self.client.request(.HEAD, uri, .{ + .redirect_behavior = .not_allowed, + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = try self.getOrRefreshToken(), + }, + .extra_headers = &.{}, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var redirect_buffer: [8 * 1024]u8 = undefined; + const res = try req.receiveHead(&redirect_buffer); + + const size = switch (res.head.status.class()) { + .success => res.head.content_length.?, + else => switch (res.head.status) { + .not_found => return error.FileNotFound, + .unauthorized, .forbidden => return error.PermissionDenied, + else => blk: { + log.err("Failed to fetch size for {f}: {s}", .{ uri, res.head.bytes }); + break :blk error.ServerError; + }, + }, + }; + return size; + } + + fn performRead(self: *GCS, handle: *Handle, data: []const []u8, offset: u64) !usize { + if (offset >= handle.size) return 0; + + var range_buf: [64]u8 = undefined; + const range_header = blk: { + var total_bytes: u64 = 0; + for (data) |buf| { + total_bytes += @as(u64, buf.len); + } + const remaining = handle.size - offset; + const take = @min(remaining, total_bytes); + const end = offset + take - 1; + break :blk std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ offset, end }) catch unreachable; + }; + + const uri = self.gcsUri(handle.uri); + var req = try self.client.request( + .GET, + uri, + .{ + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = try self.getOrRefreshToken(), + }, + .extra_headers = &.{.{ .name = "Range", .value = range_header }}, + }, + ); + defer req.deinit(); + + try req.sendBodiless(); + + var redirect_buffer: [8 * 1024]u8 = undefined; + var res = try req.receiveHead(&redirect_buffer); + + if (res.head.status != .partial_content and res.head.status != .ok) { + log.err("Failed to read {s}: {s}", .{ handle.uri, res.head.bytes }); + return error.RequestFailed; + } + + const content_range = blk: { + var it = res.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { + break :blk parseContentRange(header.value); + } + } + break :blk null; + }; + + const reader = res.reader(&.{}); + + if (content_range) |cr| { + if (cr.start < offset) { + try reader.discardAll(offset - cr.start); + } + } + + return try reader.readSliceShort(data[0]); + } + + const ContentRange = struct { + start: u64, + end: u64, + total: u64, + }; + + fn parseContentRange(value: []const u8) ?ContentRange { + const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; + const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; + const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; + + return .{ + .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, + .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, + .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, + }; + } +}; diff --git a/zml/io/vfs/index.zig b/zml/io/vfs/index.zig index 605fc8c6..9e6ca3c1 100644 --- a/zml/io/vfs/index.zig +++ b/zml/io/vfs/index.zig @@ -3,6 +3,7 @@ const log = std.log.scoped(.@"zml/io/vfs/vfs"); const std = @import("std"); pub const File = @import("file.zig").File; +pub const GCS = @import("gcs.zig").GCS; pub const HF = @import("hf.zig").HF; pub const HTTP = @import("http.zig").HTTP; pub const S3 = @import("s3.zig").S3; From ef66f310ec835478998f0d1065217e29dcb6621d Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 9 Apr 2026 10:23:11 +0200 Subject: [PATCH 409/881] workspace: bump rules_zig to segregate exec platform only options --- .bazelrc | 1 + MODULE.bazel | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index cda51e23..ec5c8408 100644 --- a/.bazelrc +++ b/.bazelrc @@ -16,6 +16,7 @@ common --enable_platform_specific_config # Ensure host tools are built optimized common --host_compilation_mode=opt +common --@rules_zig//zig/settings:mode=release_safe # Suppress all warnings, LLVM and friends are too noisy common --copt='-w' diff --git a/MODULE.bazel b/MODULE.bazel index 24d2c98a..5ed79f39 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -245,7 +245,7 @@ use_repo( git_override( module_name = "rules_zig", - commit = "7b205b78fb21e11efe5adcc9ec1c37fa5852dc77", + commit = "56911842f4527df036e8d52e78dd0f54a22d4215", remote = "https://github.com/zml/rules_zig.git", ) From 8701a54882a35bb0b309ad1b3584ca5dd92f9dc4 Mon Sep 17 00:00:00 2001 From: elogir <90246794+elogir@users.noreply.github.com> Date: Fri, 10 Apr 2026 17:26:09 +0200 Subject: [PATCH 410/881] bin/zml-smi: add prometheus endpoint, json and macos host infos (#455) This PR introduces: - Prometheus metrics endpoint through `--prometheus` and optionally `--prometheus-port` - JSON output mode with `--json` - macOS arm64 host information - An install and release script --- bin/zml-smi/BUILD.bazel | 76 +++- bin/zml-smi/api/BUILD.bazel | 21 + bin/zml-smi/api/api.zig | 2 + bin/zml-smi/api/client.zig | 138 ++++++ bin/zml-smi/api/server.zig | 105 +++++ bin/zml-smi/csv/csv.zig | 12 +- bin/zml-smi/info/device_info.zig | 9 + bin/zml-smi/info/process_info.zig | 3 +- bin/zml-smi/json/BUILD.bazel | 11 + bin/zml-smi/json/json.zig | 15 + bin/zml-smi/main.zig | 72 +++- bin/zml-smi/platforms/amdsmi/metrics.zig | 2 +- bin/zml-smi/platforms/amdsmi/process.zig | 6 +- bin/zml-smi/platforms/linux/process.zig | 1 + bin/zml-smi/platforms/macos/BUILD.bazel | 16 + bin/zml-smi/platforms/macos/macos.zig | 2 + bin/zml-smi/platforms/macos/metrics.zig | 161 +++++++ bin/zml-smi/platforms/macos/process.zig | 12 + bin/zml-smi/platforms/neuron/metrics.zig | 2 +- bin/zml-smi/platforms/neuron/process.zig | 6 +- bin/zml-smi/platforms/nvml/metrics.zig | 2 +- bin/zml-smi/platforms/nvml/process.zig | 26 +- bin/zml-smi/platforms/tpu/metrics.zig | 2 +- bin/zml-smi/platforms/tpu/process.zig | 14 +- bin/zml-smi/prometheus/BUILD.bazel | 16 + bin/zml-smi/prometheus/exposition.zig | 159 +++++++ bin/zml-smi/prometheus/prometheus.zig | 2 + bin/zml-smi/prometheus/server.zig | 87 ++++ bin/zml-smi/scripts/release.sh | 394 ++++++++++++++++++ bin/zml-smi/scripts/remote-install.sh | 23 +- bin/zml-smi/tui/BUILD.bazel | 6 +- bin/zml-smi/tui/data.zig | 18 +- bin/zml-smi/tui/pages/detail.zig | 2 +- .../tui/pages/detail/detail_common.zig | 4 +- bin/zml-smi/tui/pages/detail/gpu_detail.zig | 2 +- .../tui/pages/detail/neuron_detail.zig | 2 +- bin/zml-smi/tui/pages/detail/tpu_detail.zig | 2 +- bin/zml-smi/tui/pages/overview.zig | 6 +- bin/zml-smi/tui/print.zig | 2 +- bin/zml-smi/tui/top.zig | 4 +- bin/zml-smi/tui/widgets/device_card.zig | 5 +- bin/zml-smi/tui/widgets/metric_card.zig | 2 + bin/zml-smi/tui/widgets/process_table.zig | 6 +- bin/zml-smi/tui/widgets/status_line.zig | 2 +- bin/zml-smi/utils/double_buffer.zig | 10 + 45 files changed, 1374 insertions(+), 96 deletions(-) create mode 100644 bin/zml-smi/api/BUILD.bazel create mode 100644 bin/zml-smi/api/api.zig create mode 100644 bin/zml-smi/api/client.zig create mode 100644 bin/zml-smi/api/server.zig create mode 100644 bin/zml-smi/json/BUILD.bazel create mode 100644 bin/zml-smi/json/json.zig create mode 100644 bin/zml-smi/platforms/macos/BUILD.bazel create mode 100644 bin/zml-smi/platforms/macos/macos.zig create mode 100644 bin/zml-smi/platforms/macos/metrics.zig create mode 100644 bin/zml-smi/platforms/macos/process.zig create mode 100644 bin/zml-smi/prometheus/BUILD.bazel create mode 100644 bin/zml-smi/prometheus/exposition.zig create mode 100644 bin/zml-smi/prometheus/prometheus.zig create mode 100644 bin/zml-smi/prometheus/server.zig create mode 100755 bin/zml-smi/scripts/release.sh diff --git a/bin/zml-smi/BUILD.bazel b/bin/zml-smi/BUILD.bazel index fc803eb2..32ec3103 100644 --- a/bin/zml-smi/BUILD.bazel +++ b/bin/zml-smi/BUILD.bazel @@ -1,5 +1,6 @@ load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@bazel_lib//tools/release:hashes.bzl", "hashes") load("@bazel_skylib//rules:run_binary.bzl", "run_binary") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load") load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") @@ -61,6 +62,19 @@ copy_to_directory( }, ) +run_binary( + name = "zml-smi.stripped", + srcs = [":zml-smi"], + outs = ["zml-smi.stripped"], + args = [ + "--strip-all", + "-o", + "$(location zml-smi.stripped)", + "$(location :zml-smi)", + ], + tool = "@llvm//tools:llvm-strip", +) + zig_binary( name = "zml-smi", visibility = ["//visibility:public"], @@ -69,15 +83,18 @@ zig_binary( deps = [ "//bin/zml-smi/collector", "//bin/zml-smi/csv", - "//bin/zml-smi/platforms/linux", + "//bin/zml-smi/json", + "//bin/zml-smi/api", + "//bin/zml-smi/prometheus", "//bin/zml-smi/info", "//bin/zml-smi/tui", "//stdx", "@libvaxis//:vaxis", "@rules_zig//zig/runfiles", ] + select({ - "@platforms//os:macos": [], + "@platforms//os:macos": ["//bin/zml-smi/platforms/macos"], "//conditions:default": [ + "//bin/zml-smi/platforms/linux", "//bin/zml-smi/platforms/amdsmi", "//bin/zml-smi/platforms/neuron", "//bin/zml-smi/platforms/nvml", @@ -91,7 +108,10 @@ zig_binary( mtree_spec( name = "mtree_spec", - srcs = [":zml-smi"], + srcs = [ + ":zml-smi", + ":zml-smi.stripped", + ], tags = ["manual"], ) @@ -100,7 +120,7 @@ genrule( name = "mtree", srcs = [":mtree_spec"], outs = ["mtree.txt"], - cmd = """sed -e '/type=dir/d' -e 's#^bin/##g' $< > $@""", + cmd = """sed -e '/type=dir/d' -e '/zml-smi\\/zml-smi /d' -e 's#^bin/##g' -e 's#zml-smi\\.stripped#zml-smi#' -e '/stamp.c/d' $< > $@""", tags = ["manual"], ) @@ -110,9 +130,23 @@ genrule( # strip_prefix = "bin", # ) -tar( - name = "archive", - srcs = [":zml-smi"], +# Release archives, one per platform, correctly named for distribution +# Build one: bazel build --config=release //bin/zml-smi:zml-smi-linux-amd64 +# Build all: bazel build --config=release //bin/zml-smi:archives + +_RELEASE_PLATFORMS = [ + ("linux", "amd64"), + ("linux", "arm64"), + ("macos", "arm64"), +] + +[tar( + name = "archive_" + os + "_" + arch, + srcs = [ + ":zml-smi", + ":zml-smi.stripped", + ], + out = "zml-smi-" + os + "-" + arch + ".tar.zst", args = [ "--options", ",".join([ @@ -123,6 +157,32 @@ tar( compress = "zstd", mtree = ":mtree", tags = ["manual"], +) for os, arch in _RELEASE_PLATFORMS] + +[platform_transition_filegroup( + name = "zml-smi-" + os + "-" + arch, + srcs = [":archive_" + os + "_" + arch], + target_platform = "//platforms:" + os + "_" + arch, + tags = ["manual"], +) for os, arch in _RELEASE_PLATFORMS] + +[hashes( + name = "zml-smi-" + os + "-" + arch + ".sha256", + src = ":zml-smi-" + os + "-" + arch, + tags = ["manual"], +) for os, arch in _RELEASE_PLATFORMS] + +sh_binary( + name = "release", + srcs = ["scripts/release.sh"], + tags = ["manual"], +) + +filegroup( + name = "archives", + srcs = [":zml-smi-" + os + "-" + arch for os, arch in _RELEASE_PLATFORMS] + + [":zml-smi-" + os + "-" + arch + ".sha256" for os, arch in _RELEASE_PLATFORMS], + tags = ["manual"], ) oci_image( @@ -130,7 +190,7 @@ oci_image( base = "@distroless_cc_debian12", entrypoint = ["/zml-smi/zml-smi"], tars = [ - ":archive", + ":archive_linux_amd64", ], tags = ["manual"], ) diff --git a/bin/zml-smi/api/BUILD.bazel b/bin/zml-smi/api/BUILD.bazel new file mode 100644 index 00000000..1a09f597 --- /dev/null +++ b/bin/zml-smi/api/BUILD.bazel @@ -0,0 +1,21 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "api", + import_name = "zml-smi/api", + main = "api.zig", + srcs = [ + "client.zig", + "server.zig", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/json", + "//bin/zml-smi/utils:double_buffer", + ] + select({ + "@platforms//os:macos": ["//bin/zml-smi/platforms/macos"], + "//conditions:default": ["//bin/zml-smi/platforms/linux"], + }), +) diff --git a/bin/zml-smi/api/api.zig b/bin/zml-smi/api/api.zig new file mode 100644 index 00000000..fb39ddd8 --- /dev/null +++ b/bin/zml-smi/api/api.zig @@ -0,0 +1,2 @@ +pub const Server = @import("server.zig").Server; +pub const addRemotes = @import("client.zig").addRemotes; diff --git a/bin/zml-smi/api/client.zig b/bin/zml-smi/api/client.zig new file mode 100644 index 00000000..17e49917 --- /dev/null +++ b/bin/zml-smi/api/client.zig @@ -0,0 +1,138 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const pi = smi_info.process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const Collector = @import("zml-smi/collector").Collector; + +fn setRemote(device: *DeviceInfo) void { + inline for (@typeInfo(DeviceInfo).@"union".fields) |field| { + if (device.* == @field(smi_info.device_info.Target, field.name)) { + @field(device, field.name).values[0].remote = true; + @field(device, field.name).values[1].remote = true; + } + } +} + +pub fn addRemotes(collector: *Collector, hosts: []const u8) !void { + var it = std.mem.splitScalar(u8, hosts, ','); + while (it.next()) |host| { + addHost(collector, host) catch |err| { + std.log.err("{s}: {s}", .{ host, @errorName(err) }); + }; + } +} + +fn addHost(collector: *Collector, host: []const u8) !void { + const body = try httpGet(collector.gpa, collector.io, host); + defer collector.gpa.free(body); + + const parsed = try std.json.parseFromSlice(std.json.Value, collector.gpa, body, .{}); + defer parsed.deinit(); + + const root = parsed.value.object; + const json_devices = if (root.get("devices")) |v| v.array.items else &.{}; + + const dev_offset: u16 = @intCast(collector.device_infos.items.len); + + var devices: std.ArrayList(*DeviceInfo) = .empty; + for (json_devices) |item| { + var device = std.json.parseFromValueLeaky(DeviceInfo, collector.arena, item, .{ .ignore_unknown_fields = true }) catch continue; + setRemote(&device); + const info = try collector.addDevice(device); + try devices.append(collector.arena, info); + } + + if (devices.items.len == 0) { + return; + } + + const processes = try collector.createProcessList(); + const str_arenas: [2]*std.heap.ArenaAllocator = .{ try collector.createPollArena(), try collector.createPollArena() }; + + const json_processes = if (root.get("processes")) |v| v.array.items else &.{}; + const back_idx: usize = 1 - processes.current.load(.acquire); + const back = processes.back(); + for (json_processes) |item| { + var proc = std.json.parseFromValueLeaky(pi.ProcessInfo, str_arenas[back_idx].allocator(), item, .{ .ignore_unknown_fields = true }) catch continue; + proc.device_idx += dev_offset; + proc.remote = true; + back.append(collector.gpa, proc) catch continue; + } + processes.swap(); + + const url = try collector.arena.dupe(u8, host); + const poll_arena = try collector.createPollArena(); + try collector.spawnPoll(pollOnce, .{ poll_arena, collector.gpa, collector.io, url, devices.items, processes, dev_offset, str_arenas }); +} + +fn pollOnce(poll_arena: *std.heap.ArenaAllocator, gpa: std.mem.Allocator, io: std.Io, host: []const u8, devices: []const *DeviceInfo, processes: *ProcessDoubleBuffer, dev_offset: u16, str_arenas: [2]*std.heap.ArenaAllocator) void { + _ = poll_arena.reset(.retain_capacity); + + const body = httpGet(gpa, io, host) catch return; + defer gpa.free(body); + + const parsed = std.json.parseFromSlice(std.json.Value, gpa, body, .{}) catch return; + defer parsed.deinit(); + + const root = parsed.value.object; + const json_devices = if (root.get("devices")) |v| v.array.items else &.{}; + + for (json_devices, 0..) |item, i| { + if (i >= devices.len) { + std.log.warn("{s}: remote has more devices than expected ({d} > {d})", .{ host, json_devices.len, devices.len }); + break; + } + var device = std.json.parseFromValueLeaky(DeviceInfo, poll_arena.allocator(), item, .{ .ignore_unknown_fields = true }) catch continue; + setRemote(&device); + + inline for (@typeInfo(DeviceInfo).@"union".fields) |field| { + if (devices[i].* == @field(smi_info.device_info.Target, field.name)) { + @field(devices[i], field.name).back().* = @field(device, field.name).front().*; + @field(devices[i], field.name).swap(); + } + } + } + + const json_processes = if (root.get("processes")) |v| v.array.items else &.{}; + const back_idx: usize = 1 - processes.current.load(.acquire); + _ = str_arenas[back_idx].reset(.retain_capacity); + const alloc = str_arenas[back_idx].allocator(); + const back = processes.back(); + back.clearRetainingCapacity(); + for (json_processes) |item| { + var proc = std.json.parseFromValueLeaky(pi.ProcessInfo, alloc, item, .{ .ignore_unknown_fields = true }) catch continue; + proc.device_idx += dev_offset; + proc.remote = true; + back.append(gpa, proc) catch continue; + } + processes.swap(); +} + +fn httpGet(allocator: std.mem.Allocator, io: std.Io, host: []const u8) ![]const u8 { + if (!std.mem.startsWith(u8, host, "http://") and !std.mem.startsWith(u8, host, "https://")) { + std.log.err("remote host must start with http:// or https://: {s}", .{host}); + return error.InvalidUrl; + } + + const url = try std.fmt.allocPrint(allocator, "{s}/metrics", .{host}); + defer allocator.free(url); + + var client: std.http.Client = .{ .allocator = allocator, .io = io }; + defer client.deinit(); + + var aw: std.Io.Writer.Allocating = .init(allocator); + errdefer aw.deinit(); + + const result = try client.fetch(.{ + .location = .{ .url = url }, + .response_writer = &aw.writer, + }); + + if (result.status != .ok) { + aw.deinit(); + return error.HttpError; + } + + return try aw.toOwnedSlice(); +} diff --git a/bin/zml-smi/api/server.zig b/bin/zml-smi/api/server.zig new file mode 100644 index 00000000..e05aa7fe --- /dev/null +++ b/bin/zml-smi/api/server.zig @@ -0,0 +1,105 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const ProcessInfo = smi_info.process_info.ProcessInfo; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(ProcessInfo)); +const json = @import("zml-smi/json"); +const ProcessEnricher = if (builtin.os.tag == .macos) + @import("zml-smi/platforms/macos").process.ProcessEnricher +else + @import("zml-smi/platforms/linux").process.ProcessEnricher; + +pub const Server = struct { + allocator: std.mem.Allocator, + io: std.Io, + devices: []const *DeviceInfo, + process_lists: []const *const ProcessDoubleBuffer, + enricher: *ProcessEnricher, + tcp: std.Io.net.Server, + connection_group: std.Io.Group = .init, + + pub fn init(allocator: std.mem.Allocator, io: std.Io, port: u16, devices: []const *DeviceInfo, process_lists: []const *const ProcessDoubleBuffer, enricher: *ProcessEnricher) !Server { + const address: std.Io.net.IpAddress = .{ .ip4 = .{ + .bytes = .{ 0, 0, 0, 0 }, + .port = port, + } }; + var tcp = try address.listen(io, .{ .reuse_address = true }); + errdefer tcp.deinit(io); + + return .{ + .allocator = allocator, + .io = io, + .devices = devices, + .process_lists = process_lists, + .enricher = enricher, + .tcp = tcp, + }; + } + + pub fn run(allocator: std.mem.Allocator, io: std.Io, port: u16, devices: []const *DeviceInfo, process_lists: []const *const ProcessDoubleBuffer, enricher: *ProcessEnricher) void { + var self = init(allocator, io, port, devices, process_lists, enricher) catch |err| { + std.log.err("api server failed to start: {s}", .{@errorName(err)}); + return; + }; + defer self.deinit(); + self.serve(); + } + + pub fn deinit(self: *Server) void { + self.connection_group.await(self.io) catch {}; + self.tcp.deinit(self.io); + } + + pub fn serve(self: *Server) void { + while (true) { + const stream = self.tcp.accept(self.io) catch break; + self.connection_group.concurrent(self.io, Server.onConnection, .{ self, stream }) catch break; + } + } + + fn onConnection(self: *Server, stream: std.Io.net.Stream) std.Io.Cancelable!void { + defer stream.close(self.io); + self.handleConn(stream) catch {}; + } + + fn handleConn(self: *Server, stream: std.Io.net.Stream) !void { + var read_buf: [8192]u8 = undefined; + var reader = stream.reader(self.io, &read_buf); + + var write_buf: [4096]u8 = undefined; + var writer = stream.writer(self.io, &write_buf); + + var http_server: std.http.Server = .init(&reader.interface, &writer.interface); + + var procs: std.ArrayList(ProcessInfo) = .empty; + defer procs.deinit(self.allocator); + + while (true) { + var request = http_server.receiveHead() catch |err| switch (err) { + error.HttpConnectionClosing => return, + else => return err, + }; + + procs.clearRetainingCapacity(); + for (self.process_lists) |pl| { + procs.appendSlice(self.allocator, pl.front().items) catch {}; + } + self.enricher.enrich(self.io, procs.items); + + var body_buf: [4096]u8 = undefined; + var body_writer = try request.respondStreaming(&body_buf, .{ + .respond_options = .{ + .status = .ok, + .extra_headers = &.{ + .{ .name = "content-type", .value = "application/json" }, + .{ .name = "access-control-allow-origin", .value = "*" }, + }, + }, + }); + + try json.write(&body_writer.writer, self.devices, procs.items); + try body_writer.end(); + } + } +}; diff --git a/bin/zml-smi/csv/csv.zig b/bin/zml-smi/csv/csv.zig index 4e11b398..a05c5a4f 100644 --- a/bin/zml-smi/csv/csv.zig +++ b/bin/zml-smi/csv/csv.zig @@ -5,7 +5,6 @@ const DeviceInfo = smi_info.device_info.DeviceInfo; pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo) !void { inline for (@typeInfo(DeviceInfo).@"union".fields) |tp| { const tag = @field(smi_info.Target, tp.name); - const Inner = resolveType(tp.type); var header_printed = false; for (devices, 0..) |dev, i| { @@ -13,7 +12,7 @@ pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo) !void { tag => |*db| { if (!header_printed) { try writer.writeAll("index,type"); - inline for (@typeInfo(Inner).@"struct".fields) |f| { + inline for (@typeInfo(tp.type.Value).@"struct".fields) |f| { try writer.writeAll("," ++ f.name); } try writer.writeAll("\n"); @@ -24,7 +23,7 @@ pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo) !void { const val = db.front().*; try writer.print("{d},{s}", .{ i, tp.name }); - inline for (@typeInfo(Inner).@"struct".fields) |f| { + inline for (@typeInfo(tp.type.Value).@"struct".fields) |f| { try writer.writeAll(","); try writeValue(writer, @field(val, f.name)); } @@ -44,10 +43,3 @@ fn writeValue(writer: *std.Io.Writer, field: anytype) !void { else => return writer.print("{any}", .{field}), } } - -fn resolveType(comptime DB: type) type { - for (@typeInfo(DB).@"struct".fields) |df| { - if (std.mem.eql(u8, df.name, "values")) return @typeInfo(df.type).array.child; - } - unreachable; -} diff --git a/bin/zml-smi/info/device_info.zig b/bin/zml-smi/info/device_info.zig index e84fb88e..faab1c11 100644 --- a/bin/zml-smi/info/device_info.zig +++ b/bin/zml-smi/info/device_info.zig @@ -13,9 +13,16 @@ pub const DeviceInfo = union(Target) { rocm: DoubleBuffer(GpuInfo), neuron: DoubleBuffer(NeuronInfo), tpu: DoubleBuffer(TpuInfo), + + pub fn isRemote(self: *const DeviceInfo) bool { + switch (self.*) { + inline else => |*db| return db.front().remote, + } + } }; pub const GpuInfo = struct { + remote: bool = false, name: ?[]const u8 = null, driver_version: ?[]const u8 = null, cuda_driver_version: ?[]const u8 = null, @@ -54,6 +61,7 @@ pub const GpuInfo = struct { }; pub const NeuronInfo = struct { + remote: bool = false, name: ?[]const u8 = null, driver_version: ?[]const u8 = null, @@ -76,6 +84,7 @@ pub const NeuronInfo = struct { }; pub const TpuInfo = struct { + remote: bool = false, name: ?[]const u8 = null, util_percent: ?u64 = null, diff --git a/bin/zml-smi/info/process_info.zig b/bin/zml-smi/info/process_info.zig index 705dfe38..73504656 100644 --- a/bin/zml-smi/info/process_info.zig +++ b/bin/zml-smi/info/process_info.zig @@ -1,6 +1,6 @@ pub const ProcessInfo = struct { pid: u32 = 0, - device_idx: u8 = 0, + device_idx: u16 = 0, dev_util_percent: ?u16 = null, dev_mem_kib: ?u64 = null, uid: u32 = 0, @@ -8,4 +8,5 @@ pub const ProcessInfo = struct { comm: []const u8 = "", cpu_percent: u16 = 0, // *10, e.g. 125 = 12.5% rss_kib: u64 = 0, + remote: bool = false, }; diff --git a/bin/zml-smi/json/BUILD.bazel b/bin/zml-smi/json/BUILD.bazel new file mode 100644 index 00000000..decc8ca1 --- /dev/null +++ b/bin/zml-smi/json/BUILD.bazel @@ -0,0 +1,11 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "json", + import_name = "zml-smi/json", + main = "json.zig", + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/info", + ], +) diff --git a/bin/zml-smi/json/json.zig b/bin/zml-smi/json/json.zig new file mode 100644 index 00000000..c32a16cf --- /dev/null +++ b/bin/zml-smi/json/json.zig @@ -0,0 +1,15 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const ProcessInfo = smi_info.process_info.ProcessInfo; + +pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo, processes: []const ProcessInfo) !void { + var jw: std.json.Stringify = .{ .writer = writer }; + try jw.write(Response{ .devices = devices, .processes = processes }); + try writer.writeAll("\n"); +} + +const Response = struct { + devices: []const *DeviceInfo, + processes: []const ProcessInfo, +}; diff --git a/bin/zml-smi/main.zig b/bin/zml-smi/main.zig index f7284587..ccb55786 100644 --- a/bin/zml-smi/main.zig +++ b/bin/zml-smi/main.zig @@ -12,11 +12,17 @@ pub const std_options: std.Options = .{ const Collector = @import("zml-smi/collector").Collector; const smi_info = @import("zml-smi/info"); const HostInfo = smi_info.host_info.HostInfo; -const linux = @import("zml-smi/platforms/linux"); -const ProcessEnricher = linux.process.ProcessEnricher; -const host = linux.metrics; +const platform = if (builtin.os.tag == .macos) + @import("zml-smi/platforms/macos") +else + @import("zml-smi/platforms/linux"); +const ProcessEnricher = platform.process.ProcessEnricher; +const host = platform.metrics; const c = @import("c"); const csv = @import("zml-smi/csv"); +const json = @import("zml-smi/json"); +const api = @import("zml-smi/api"); +const prometheus = @import("zml-smi/prometheus"); const smi_tui = @import("zml-smi/tui"); const tui = smi_tui.top; const static_print = smi_tui.print; @@ -25,18 +31,31 @@ const data = smi_tui.data; const CliArgs = struct { top: bool = false, csv: bool = false, + json: bool = false, + api: bool = false, + api_port: u16 = 9090, + remotes: ?[]const u8 = null, + prometheus_listen: ?[]const u8 = null, tui_refresh_rate: u16 = 100, poll_interval: u16 = 500, pub const help = - \\ zml-smi [--top] [--csv] [--tui-refresh-rate MS] [--poll-interval MS] + \\ zml-smi [--top] [--csv] [--json] [--prometheus-listen HOST:PORT] + \\ [--tui-refresh-rate MS] [--poll-interval MS] \\ \\ --top Interactive TUI mode \\ --csv Output device metrics as CSV + \\ --json Output device metrics as JSON + \\ --prometheus-listen Expose metrics as Prometheus endpoint on HOST:PORT \\ --tui-refresh-rate TUI refresh rate in ms (default: 100) \\ --poll-interval Device polling interval in ms (default: 500) \\ ; + + // [--api] [--port PORT] [--remotes HOST,...] + // \\ --api Expose device metrics as HTTP JSON endpoint + // \\ --api-port API server port (default: 9090) + // \\ --remotes Add devices from remote hosts (comma-separated URLs) }; const device_backends = if (builtin.os.tag != .macos) .{ @@ -56,7 +75,7 @@ pub fn main(init: std.process.Init) !void { var collector: Collector = .init(arena, gpa, io, .{ .poll_interval_ms = args.poll_interval, - .poll_only = !args.top, + .poll_only = !args.top and args.prometheus_listen == null and !args.api, }); defer collector.deinit(); @@ -66,6 +85,14 @@ pub fn main(init: std.process.Init) !void { var enricher: ProcessEnricher = try .init(gpa, io); defer enricher.deinit(); + if (args.api) { + std.log.info("serving api metrics on port {d}", .{args.api_port}); + } + + if (args.prometheus_listen) |listen| { + std.log.info("serving prometheus metrics on {s}", .{listen}); + } + inline for (device_backends) |entry| { const target, const backend = entry; if (targets.contains(target)) { @@ -75,23 +102,54 @@ pub fn main(init: std.process.Init) !void { } } - var state = try data.SystemState.init(arena, .{ + var api_group: std.Io.Group = .init; + + if (args.remotes) |hosts| { + try api.addRemotes(&collector, hosts); + } + + var state = try data.SystemState.init(.{ .devices = collector.device_infos.items, .host = &host_info, .targets = targets, .tui_refresh_rate = args.tui_refresh_rate, .process_lists = collector.process_lists.items, .enricher = &enricher, + .gpa = gpa, + .arena = arena, .io = io, }); defer state.deinit(arena); - if (args.csv) { + if (args.prometheus_listen) |listen| { + try api_group.concurrent(io, prometheus.Server.run, .{ io, listen, collector.device_infos.items, &host_info }); + } + + if (args.api) { + try api_group.concurrent(io, api.Server.run, .{ gpa, io, args.api_port, collector.device_infos.items, collector.process_lists.items, &enricher }); + } + + if (args.prometheus_listen != null or args.api) { + try api_group.await(io); + } else if (args.csv) { var csv_buf: [4096]u8 = undefined; var csv_writer = std.Io.File.stdout().writer(io, &csv_buf); try csv.write(&csv_writer.interface, collector.device_infos.items); try csv_writer.flush(); + } else if (args.json) { + var procs: std.ArrayList(smi_info.process_info.ProcessInfo) = .empty; + defer procs.deinit(gpa); + for (collector.process_lists.items) |pl| { + try procs.appendSlice(gpa, pl.front().items); + } + enricher.enrich(io, procs.items); + + var json_buf: [4096]u8 = undefined; + var json_writer = std.Io.File.stdout().writer(io, &json_buf); + + try json.write(&json_writer.interface, collector.device_infos.items, procs.items); + try json_writer.flush(); } else if (args.top) { try tui.run(gpa, io, &state); } else { diff --git a/bin/zml-smi/platforms/amdsmi/metrics.zig b/bin/zml-smi/platforms/amdsmi/metrics.zig index a0d4cca2..8aa0487c 100644 --- a/bin/zml-smi/platforms/amdsmi/metrics.zig +++ b/bin/zml-smi/platforms/amdsmi/metrics.zig @@ -13,7 +13,7 @@ pub fn start(collector: *Collector) !void { amdsmi.* = try AmdSmi.init(collector.arena); const count = amdsmi.deviceCount(); - const dev_offset: u8 = @intCast(collector.device_infos.items.len); + const dev_offset: u16 = @intCast(collector.device_infos.items.len); for (0..count) |i| { const dev = Device.open(amdsmi, @intCast(i)) catch continue; diff --git a/bin/zml-smi/platforms/amdsmi/process.zig b/bin/zml-smi/platforms/amdsmi/process.zig index b1952a0b..8d1fdd01 100644 --- a/bin/zml-smi/platforms/amdsmi/process.zig +++ b/bin/zml-smi/platforms/amdsmi/process.zig @@ -7,7 +7,7 @@ const Collector = @import("zml-smi/collector").Collector; const bdf_len = "0000:00:00.0".len; -pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, amdsmi: *const AmdSmi, dev_offset: u8) !void { +pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, amdsmi: *const AmdSmi, dev_offset: u16) !void { const device_count = amdsmi.deviceCount(); const pci_slots = try collector.arena.alloc(?[bdf_len]u8, device_count); @@ -23,10 +23,10 @@ pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, amdsmi: *const Am }); } - try collector.spawnPoll(pollOnce, .{ collector.io, collector.gpa, list, amdsmi, dev_offset, device_count, pci_slots }); + try collector.spawnPoll(pollOnce, .{ collector.gpa, collector.io, list, amdsmi, dev_offset, device_count, pci_slots }); } -fn pollOnce(io: std.Io, allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, amdsmi: *const AmdSmi, dev_offset: u8, device_count: u32, pci_slots: []?[bdf_len]u8) void { +fn pollOnce(allocator: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, amdsmi: *const AmdSmi, dev_offset: u16, device_count: u32, pci_slots: []?[bdf_len]u8) void { const back = list.back(); back.clearRetainingCapacity(); diff --git a/bin/zml-smi/platforms/linux/process.zig b/bin/zml-smi/platforms/linux/process.zig index 3aa5ff01..c57d1725 100644 --- a/bin/zml-smi/platforms/linux/process.zig +++ b/bin/zml-smi/platforms/linux/process.zig @@ -43,6 +43,7 @@ pub const ProcessEnricher = struct { curr_ticks.ensureTotalCapacity(self.gpa, @intCast(procs.len)) catch return; for (procs) |*info| { + if (info.remote) continue; // Read Uid, VmRSS from /proc/{pid}/status const status_path = std.fmt.allocPrint(alloc, "/proc/{d}/status", .{info.pid}) catch continue; const uid: u32 = @intCast(sysfs.readFieldInt(alloc, io, status_path, "Uid") catch 0); diff --git a/bin/zml-smi/platforms/macos/BUILD.bazel b/bin/zml-smi/platforms/macos/BUILD.bazel new file mode 100644 index 00000000..a92e298e --- /dev/null +++ b/bin/zml-smi/platforms/macos/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "macos", + import_name = "zml-smi/platforms/macos", + main = "macos.zig", + srcs = [ + "metrics.zig", + "process.zig", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + ], +) diff --git a/bin/zml-smi/platforms/macos/macos.zig b/bin/zml-smi/platforms/macos/macos.zig new file mode 100644 index 00000000..ffab3ec4 --- /dev/null +++ b/bin/zml-smi/platforms/macos/macos.zig @@ -0,0 +1,2 @@ +pub const metrics = @import("metrics.zig"); +pub const process = @import("process.zig"); diff --git a/bin/zml-smi/platforms/macos/metrics.zig b/bin/zml-smi/platforms/macos/metrics.zig new file mode 100644 index 00000000..616168cd --- /dev/null +++ b/bin/zml-smi/platforms/macos/metrics.zig @@ -0,0 +1,161 @@ +const std = @import("std"); +const host_info = @import("zml-smi/info").host_info; +const HostInfo = host_info.HostInfo; +const poll_metrics = @import("zml-smi/info").poll_metrics; +const Collector = @import("zml-smi/collector").Collector; + +const c = std.c; + +const vm_statistics64 = extern struct { + free_count: c_uint, + active_count: c_uint, + inactive_count: c_uint, + wire_count: c_uint, + zero_fill_count: u64, + reactivations: u64, + pageins: u64, + pageouts: u64, + faults: u64, + cow_faults: u64, + lookups: u64, + hits: u64, + purges: u64, + purgeable_count: c_uint, + speculative_count: c_uint, + decompressions: u64, + compressions: u64, + swapins: u64, + swapouts: u64, + compressor_page_count: c_uint, + throttled_count: c_uint, + external_page_count: c_uint, + internal_page_count: c_uint, + total_uncompressed_pages_in_compressor: u64, +}; + +const HOST_VM_INFO64: c_int = 4; +const HOST_VM_INFO64_COUNT: c_uint = @sizeOf(vm_statistics64) / @sizeOf(c_int); +const KERN_SUCCESS: c_int = 0; + +extern "c" fn host_statistics64(host: c.mach_port_t, flavor: c_int, info: *anyopaque, count: *c_uint) c.kern_return_t; +extern "c" fn getloadavg(loadavg: *[3]f64, nelem: c_int) c_int; + +pub fn init(collector: *Collector, info: *HostInfo) !void { + const poll_arena = try collector.createPollArena(); + const host: Host = .{}; + + var initial = info.front().*; + initial.hostname = sysctlString(collector.arena, "kern.hostname") catch null; + initial.kernel = sysctlString(collector.arena, "kern.osrelease") catch null; + initial.cpu_name = sysctlString(collector.arena, "machdep.cpu.brand_string") catch null; + initial.cpu_cores = host.cpuCores() catch null; + initial.mem_total_kib = host.memTotal() catch null; + info.back().* = initial; + info.swap(); + + try collector.spawnPoll(pollOnce, .{ poll_arena, info, host }); +} + +const pollOnce = poll_metrics.poll(*HostInfo, Host, metrics); + +const Host = struct { + pub fn cpuCores(_: Host) !u64 { + return @intCast(try sysctlU32("hw.logicalcpu")); + } + + pub fn memTotal(_: Host) !u64 { + return (try sysctlU64("hw.memsize")) / 1024; + } + + pub fn memAvailable(_: Host) !u64 { + var vm_stat: vm_statistics64 = undefined; + var count: c_uint = HOST_VM_INFO64_COUNT; + + if (host_statistics64(c.mach_host_self(), HOST_VM_INFO64, &vm_stat, &count) != KERN_SUCCESS) { + return error.HostStatsFailed; + } + + var page_size: u64 = 0; + if (c._host_page_size(c.mach_host_self(), &page_size) != KERN_SUCCESS) { + return error.PageSizeFailed; + } + + const available_pages: u64 = @as(u64, vm_stat.free_count) + + @as(u64, vm_stat.inactive_count) + + @as(u64, vm_stat.purgeable_count); + + return (available_pages * page_size) / 1024; + } + + fn loadAvg() ![3]f64 { + var avg: [3]f64 = undefined; + if (getloadavg(&avg, 3) != 3) { + return error.LoadAvgFailed; + } + + return avg; + } + + pub fn load1(_: Host) !f32 { + return @floatCast((try loadAvg())[0]); + } + + pub fn load5(_: Host) !f32 { + return @floatCast((try loadAvg())[1]); + } + + pub fn load15(_: Host) !f32 { + return @floatCast((try loadAvg())[2]); + } + + pub fn uptime(_: Host) !u64 { + var boottime: c.timeval = undefined; + var len: usize = @sizeOf(c.timeval); + if (c.sysctlbyname("kern.boottime", std.mem.asBytes(&boottime), &len, null, 0) != 0) { + return error.SysctlFailed; + } + + var now: c.timeval = undefined; + _ = c.gettimeofday(&now, null); + + return @intCast(now.sec - boottime.sec); + } +}; + +fn sysctlString(allocator: std.mem.Allocator, name: [*:0]const u8) ![]const u8 { + var buf: [256]u8 = undefined; + var len: usize = buf.len; + if (c.sysctlbyname(name, &buf, &len, null, 0) != 0) { + return error.SysctlFailed; + } + + return allocator.dupe(u8, std.mem.span(@as([*:0]const u8, @ptrCast(buf[0..len])))); +} + +fn sysctlU64(name: [*:0]const u8) !u64 { + var value: u64 = 0; + var len: usize = @sizeOf(u64); + if (c.sysctlbyname(name, std.mem.asBytes(&value), &len, null, 0) != 0) { + return error.SysctlFailed; + } + + return value; +} + +fn sysctlU32(name: [*:0]const u8) !u32 { + var value: u32 = 0; + var len: usize = @sizeOf(u32); + if (c.sysctlbyname(name, std.mem.asBytes(&value), &len, null, 0) != 0) { + return error.SysctlFailed; + } + + return value; +} + +const metrics = .{ + .{ .field = "mem_available_kib", .query = Host.memAvailable }, + .{ .field = "load_1", .query = Host.load1 }, + .{ .field = "load_5", .query = Host.load5 }, + .{ .field = "load_15", .query = Host.load15 }, + .{ .field = "uptime_seconds", .query = Host.uptime }, +}; diff --git a/bin/zml-smi/platforms/macos/process.zig b/bin/zml-smi/platforms/macos/process.zig new file mode 100644 index 00000000..a48e9831 --- /dev/null +++ b/bin/zml-smi/platforms/macos/process.zig @@ -0,0 +1,12 @@ +const std = @import("std"); +const pi = @import("zml-smi/info").process_info; + +pub const ProcessEnricher = struct { + pub fn init(_: std.mem.Allocator, _: std.Io) !ProcessEnricher { + return .{}; + } + + pub fn deinit(_: *ProcessEnricher) void {} + + pub fn enrich(_: *ProcessEnricher, _: std.Io, _: []pi.ProcessInfo) void {} +}; diff --git a/bin/zml-smi/platforms/neuron/metrics.zig b/bin/zml-smi/platforms/neuron/metrics.zig index ee667c0a..b0bf5ac2 100644 --- a/bin/zml-smi/platforms/neuron/metrics.zig +++ b/bin/zml-smi/platforms/neuron/metrics.zig @@ -30,7 +30,7 @@ pub fn start(collector: *Collector) !void { const GiB: u64 = 1024 * 1024 * 1024; var neuron_infos: std.ArrayList(*DeviceInfo) = .{}; - const dev_offset: u8 = @intCast(collector.device_infos.items.len); + const dev_offset: u16 = @intCast(collector.device_infos.items.len); var ver_buf: [Nrt.version_buf_len]u8 = undefined; const nrt_ver: ?[]const u8 = if (nrt.version(&ver_buf)) |v| diff --git a/bin/zml-smi/platforms/neuron/process.zig b/bin/zml-smi/platforms/neuron/process.zig index 40ec167e..70bab67b 100644 --- a/bin/zml-smi/platforms/neuron/process.zig +++ b/bin/zml-smi/platforms/neuron/process.zig @@ -12,11 +12,11 @@ const DeltaState = struct { prev_timestamp: ?std.Io.Timestamp = null, }; -pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, nrt: *const Nrt, nc_per_device: u32, util_buf: []UtilAtomic, dev_offset: u8, warmup: bool) !void { +pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, nrt: *const Nrt, nc_per_device: u32, util_buf: []UtilAtomic, dev_offset: u16, warmup: bool) !void { if (nrt.handles.len == 0) return; const delta_states = try collector.arena.alloc(DeltaState, util_buf.len); @memset(delta_states, .{}); - const args: std.meta.ArgsTuple(@TypeOf(pollOnce)) = .{ collector.io, collector.gpa, list, nrt, nc_per_device, util_buf, dev_offset, delta_states }; + const args: std.meta.ArgsTuple(@TypeOf(pollOnce)) = .{ collector.gpa, collector.io, list, nrt, nc_per_device, util_buf, dev_offset, delta_states }; if (warmup) { @call(.auto, pollOnce, args); collector.io.sleep(.fromMilliseconds(500), .awake) catch {}; @@ -27,7 +27,7 @@ pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, nrt: *const Nrt, try collector.spawnPoll(pollOnce, args); } -fn pollOnce(io: std.Io, allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, nrt: *const Nrt, nc_per_device: u32, util_buf: []UtilAtomic, dev_offset: u8, delta_states: []DeltaState) void { +fn pollOnce(allocator: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, nrt: *const Nrt, nc_per_device: u32, util_buf: []UtilAtomic, dev_offset: u16, delta_states: []DeltaState) void { const back = list.back(); back.clearRetainingCapacity(); diff --git a/bin/zml-smi/platforms/nvml/metrics.zig b/bin/zml-smi/platforms/nvml/metrics.zig index f134e4c2..4e0ee2bd 100644 --- a/bin/zml-smi/platforms/nvml/metrics.zig +++ b/bin/zml-smi/platforms/nvml/metrics.zig @@ -13,7 +13,7 @@ pub fn start(collector: *Collector) !void { nvml.* = try Nvml.init(); const count = try nvml.deviceCount(); - const dev_offset: u8 = @intCast(collector.device_infos.items.len); + const dev_offset: u16 = @intCast(collector.device_infos.items.len); for (0..count) |i| { const dev = Device.open(nvml, @intCast(i)) catch continue; diff --git a/bin/zml-smi/platforms/nvml/process.zig b/bin/zml-smi/platforms/nvml/process.zig index c1dd39d2..3250a047 100644 --- a/bin/zml-smi/platforms/nvml/process.zig +++ b/bin/zml-smi/platforms/nvml/process.zig @@ -4,7 +4,7 @@ const pi = @import("zml-smi/info").process_info; const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); const Collector = @import("zml-smi/collector").Collector; -pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, nvml: *const Nvml, dev_offset: u8) !void { +pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, nvml: *const Nvml, dev_offset: u16) !void { const device_count: u32 = nvml.deviceCount() catch 0; const last_seen_ts = try collector.arena.alloc(u64, device_count); @memset(last_seen_ts, 0); @@ -12,39 +12,27 @@ pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, nvml: *const Nvml try collector.spawnPoll(pollOnce, .{ collector.gpa, list, nvml, dev_offset, device_count, last_seen_ts }); } -fn pollOnce(allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, nvml: *const Nvml, dev_offset: u8, device_count: u32, last_seen_ts: []u64) void { +fn pollOnce(allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, nvml: *const Nvml, dev_offset: u16, device_count: u32, last_seen_ts: []u64) void { const back = list.back(); back.clearRetainingCapacity(); for (0..device_count) |dev_idx| { const handle = nvml.handleByIndex(@intCast(dev_idx)) catch continue; - const idx: u8 = @intCast(dev_idx + dev_offset); + const idx: u16 = @intCast(dev_idx + dev_offset); const compute = nvml.computeRunningProcesses(allocator, handle) catch &.{}; - defer { - if (compute.len > 0) { - allocator.free(compute); - } - } + defer if (compute.len > 0) allocator.free(compute); collectFromQuery(allocator, back, idx, compute); const graphics = nvml.graphicsRunningProcesses(allocator, handle) catch &.{}; - defer { - if (graphics.len > 0) { - allocator.free(graphics); - } - } + defer if (graphics.len > 0) allocator.free(graphics); collectFromQuery(allocator, back, idx, graphics); // Apply utilization samples const last_ts = last_seen_ts[dev_idx]; const utils = nvml.processUtilization(allocator, handle, last_ts) catch continue; - defer { - if (utils.len > 0) { - allocator.free(utils); - } - } + defer if (utils.len > 0) allocator.free(utils); for (utils) |sample| { if (sample.smUtil > 100 or sample.timeStamp <= last_ts) { @@ -64,7 +52,7 @@ fn pollOnce(allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, nvml: *con list.swap(); } -fn collectFromQuery(allocator: std.mem.Allocator, back: *std.ArrayList(pi.ProcessInfo), dev_idx: u8, procs: []const Nvml.ProcessInfo_t) void { +fn collectFromQuery(allocator: std.mem.Allocator, back: *std.ArrayList(pi.ProcessInfo), dev_idx: u16, procs: []const Nvml.ProcessInfo_t) void { for (procs) |gp| { back.append(allocator, .{ .pid = gp.pid, diff --git a/bin/zml-smi/platforms/tpu/metrics.zig b/bin/zml-smi/platforms/tpu/metrics.zig index 8ef832d8..d4248f95 100644 --- a/bin/zml-smi/platforms/tpu/metrics.zig +++ b/bin/zml-smi/platforms/tpu/metrics.zig @@ -13,7 +13,7 @@ pub fn start(collector: *Collector) !void { const chip = scanPciChips(collector.arena, collector.io) orelse return error.TpuUnavailable; const device_count = chip.chip_count * chip.devices_per_chip; var tpu_infos: std.ArrayList(*DeviceInfo) = .{}; - const dev_offset: u8 = @intCast(collector.device_infos.items.len); + const dev_offset: u16 = @intCast(collector.device_infos.items.len); for (0..device_count) |_| { const chip_name = try collector.arena.dupe(u8, chip.name); diff --git a/bin/zml-smi/platforms/tpu/process.zig b/bin/zml-smi/platforms/tpu/process.zig index 62a73757..cca8f6a1 100644 --- a/bin/zml-smi/platforms/tpu/process.zig +++ b/bin/zml-smi/platforms/tpu/process.zig @@ -5,16 +5,16 @@ const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.Ar const DeviceInfo = smi_info.device_info.DeviceInfo; const Collector = @import("zml-smi/collector").Collector; -pub fn init(collector: *Collector, devices_per_chip: u32, device_infos: []*DeviceInfo, list: *ProcessDoubleBuffer, dev_offset: u8) !void { - try collector.spawnPoll(pollOnce, .{ collector.io, collector.gpa, list, devices_per_chip, device_infos, dev_offset }); +pub fn init(collector: *Collector, devices_per_chip: u32, device_infos: []*DeviceInfo, list: *ProcessDoubleBuffer, dev_offset: u16) !void { + try collector.spawnPoll(pollOnce, .{ collector.gpa, collector.io, list, devices_per_chip, device_infos, dev_offset }); } -fn pollOnce(io: std.Io, allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, devices_per_chip: u32, device_infos: []*DeviceInfo, dev_offset: u8) void { - scan(io, allocator, list.back(), devices_per_chip, device_infos, dev_offset); +fn pollOnce(allocator: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, devices_per_chip: u32, device_infos: []*DeviceInfo, dev_offset: u16) void { + scan(allocator, io, list.back(), devices_per_chip, device_infos, dev_offset); list.swap(); } -fn scan(io: std.Io, allocator: std.mem.Allocator, back: *std.ArrayList(pi.ProcessInfo), devices_per_chip: u32, infos: []*DeviceInfo, dev_offset: u8) void { +fn scan(allocator: std.mem.Allocator, io: std.Io, back: *std.ArrayList(pi.ProcessInfo), devices_per_chip: u32, infos: []*DeviceInfo, dev_offset: u16) void { back.clearRetainingCapacity(); var proc_dir = std.Io.Dir.openDirAbsolute(io, "/proc", .{ .iterate = true }) catch return; @@ -36,9 +36,9 @@ fn scan(io: std.Io, allocator: std.mem.Allocator, back: *std.ArrayList(pi.Proces const target = link_buf[0..len]; if (parseChipIndex(target)) |chip_idx| { - const base: u8 = @intCast(chip_idx * devices_per_chip); + const base: u16 = @intCast(chip_idx * devices_per_chip); for (0..devices_per_chip) |d| { - const local_idx: u8 = @intCast(base + d); + const local_idx: u16 = @intCast(base + d); var info: pi.ProcessInfo = .{ .pid = pid, .device_idx = local_idx + dev_offset }; if (local_idx < infos.len) { diff --git a/bin/zml-smi/prometheus/BUILD.bazel b/bin/zml-smi/prometheus/BUILD.bazel new file mode 100644 index 00000000..c00a26b5 --- /dev/null +++ b/bin/zml-smi/prometheus/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "prometheus", + import_name = "zml-smi/prometheus", + main = "prometheus.zig", + srcs = [ + "server.zig", + "exposition.zig", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + ], +) diff --git a/bin/zml-smi/prometheus/exposition.zig b/bin/zml-smi/prometheus/exposition.zig new file mode 100644 index 00000000..ac04d70d --- /dev/null +++ b/bin/zml-smi/prometheus/exposition.zig @@ -0,0 +1,159 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const HostInfo = smi_info.host_info.HostInfo; + +const device_metrics = .{ + // Shared across all device types + .{ .field = "util_percent", .metric = "zml_device_utilization_percent", .help = "Device utilization percentage" }, + .{ .field = "mem_used_bytes", .metric = "zml_device_memory_used_bytes", .help = "Device memory used in bytes" }, + .{ .field = "mem_total_bytes", .metric = "zml_device_memory_total_bytes", .help = "Device memory total in bytes" }, + // GPU (CUDA/ROCm) + .{ .field = "encoder_util_percent", .metric = "zml_device_encoder_utilization_percent", .help = "Encoder utilization percentage" }, + .{ .field = "decoder_util_percent", .metric = "zml_device_decoder_utilization_percent", .help = "Decoder utilization percentage" }, + .{ .field = "power_mw", .metric = "zml_device_power_watts", .help = "Power draw in watts", .fmt = fmtMilliwattsAsWatts }, + .{ .field = "power_limit_mw", .metric = "zml_device_power_limit_watts", .help = "Power limit in watts", .fmt = fmtMilliwattsAsWatts }, + .{ .field = "temperature", .metric = "zml_device_temperature_celsius", .help = "Device temperature in celsius" }, + .{ .field = "fan_speed_percent", .metric = "zml_device_fan_speed_percent", .help = "Fan speed percentage" }, + .{ .field = "clock_graphics_mhz", .metric = "zml_device_clock_graphics_mhz", .help = "Graphics clock frequency in MHz" }, + .{ .field = "clock_sm_mhz", .metric = "zml_device_clock_sm_mhz", .help = "SM clock frequency in MHz" }, + .{ .field = "clock_mem_mhz", .metric = "zml_device_clock_memory_mhz", .help = "Memory clock frequency in MHz" }, + .{ .field = "pcie_tx_kbps", .metric = "zml_device_pcie_tx_bytes_per_second", .help = "PCIe TX throughput in bytes per second", .fmt = fmtScaled(128) }, + .{ .field = "pcie_rx_kbps", .metric = "zml_device_pcie_rx_bytes_per_second", .help = "PCIe RX throughput in bytes per second", .fmt = fmtScaled(128) }, + // Neuron + .{ .field = "nc_tensors", .metric = "zml_device_neuron_tensors_bytes", .help = "Neuron core tensor memory in bytes" }, + .{ .field = "nc_constants", .metric = "zml_device_neuron_constants_bytes", .help = "Neuron core constants memory in bytes" }, + .{ .field = "nc_model_code", .metric = "zml_device_neuron_model_code_bytes", .help = "Neuron core model code memory in bytes" }, + .{ .field = "nc_shared_scratchpad", .metric = "zml_device_neuron_shared_scratchpad_bytes", .help = "Neuron core shared scratchpad memory in bytes" }, + .{ .field = "nc_nonshared_scratchpad", .metric = "zml_device_neuron_nonshared_scratchpad_bytes", .help = "Neuron core non-shared scratchpad memory in bytes" }, + .{ .field = "nc_runtime", .metric = "zml_device_neuron_runtime_bytes", .help = "Neuron core runtime memory in bytes" }, + .{ .field = "nc_driver", .metric = "zml_device_neuron_driver_bytes", .help = "Neuron core driver memory in bytes" }, + .{ .field = "nc_dma_rings", .metric = "zml_device_neuron_dma_rings_bytes", .help = "Neuron core DMA rings memory in bytes" }, + .{ .field = "nc_collectives", .metric = "zml_device_neuron_collectives_bytes", .help = "Neuron core collectives memory in bytes" }, + .{ .field = "nc_notifications", .metric = "zml_device_neuron_notifications_bytes", .help = "Neuron core notifications memory in bytes" }, + .{ .field = "nc_uncategorized", .metric = "zml_device_neuron_uncategorized_bytes", .help = "Neuron core uncategorized memory in bytes" }, +}; + +const host_metrics = .{ + .{ .field = "cpu_cores", .metric = "zml_host_cpu_cores", .help = "Number of CPU cores" }, + .{ .field = "mem_total_kib", .metric = "zml_host_memory_total_bytes", .help = "Total host memory in bytes", .fmt = fmtScaled(1024) }, + .{ .field = "mem_available_kib", .metric = "zml_host_memory_available_bytes", .help = "Available host memory in bytes", .fmt = fmtScaled(1024) }, + .{ .field = "load_1", .metric = "zml_host_load_1", .help = "1-minute load average" }, + .{ .field = "load_5", .metric = "zml_host_load_5", .help = "5-minute load average" }, + .{ .field = "load_15", .metric = "zml_host_load_15", .help = "15-minute load average" }, + .{ .field = "uptime_seconds", .metric = "zml_host_uptime_seconds", .help = "System uptime in seconds" }, +}; + +pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo, host: *const HostInfo) !void { + try writer.writeAll("# HELP zml_device_info Device metadata\n"); + try writer.writeAll("# TYPE zml_device_info gauge\n"); + for (devices, 0..) |dev, i| { + inline for (@typeInfo(DeviceInfo).@"union".fields) |tp| { + const tag = @field(smi_info.device_info.Target, tp.name); + + switch (dev.*) { + tag => |*db| { + const val = db.front().*; + + try writer.print("zml_device_info{{device=\"{d}\",type=\"{s}\"", .{ i, tp.name }); + + inline for (.{ + .{ "name", "name" }, + .{ "driver_version", "driver" }, + .{ "cuda_driver_version", "cuda_version" }, + }) |label| { + if (@hasField(@TypeOf(val), label[0])) { + if (@field(val, label[0])) |v| { + try writer.print(",{s}=\"", .{label[1]}); + try std.zig.stringEscape(v, writer); + try writer.writeAll("\""); + } + } + } + + try writer.writeAll("} 1\n"); + }, + else => {}, + } + } + } + try writer.writeAll("\n"); + + try writeAllDeviceMetrics(writer, device_metrics, devices); + try writeHostMetrics(writer, host.front().*); +} + +fn writeAllDeviceMetrics( + writer: *std.Io.Writer, + comptime metrics: anytype, + devices: []const *DeviceInfo, +) !void { + inline for (metrics) |m| { + var header_written = false; + + for (devices, 0..) |dev, i| { + switch (dev.*) { + inline else => |*db| { + const val = db.front().*; + if (@hasField(@TypeOf(val), m.field)) { + if (@field(val, m.field)) |raw| { + if (!header_written) { + try writer.print("# HELP {s} {s}\n# TYPE {s} gauge\n", .{ m.metric, m.help, m.metric }); + header_written = true; + } + + const device_type = @tagName(std.meta.activeTag(dev.*)); + try writer.print("{s}{{device=\"{d}\",type=\"{s}\",name=\"", .{ m.metric, i, device_type }); + try std.zig.stringEscape(val.name orelse "", writer); + try writer.writeAll("\"} "); + try fmtValue(m, writer, raw); + try writer.writeAll("\n"); + } + } + }, + } + } + + if (header_written) { + try writer.writeAll("\n"); + } + } +} + +fn writeHostMetrics(writer: *std.Io.Writer, val: anytype) !void { + inline for (host_metrics) |m| { + if (@field(val, m.field)) |raw| { + try writer.print("# HELP {s} {s}\n# TYPE {s} gauge\n{s} ", .{ m.metric, m.help, m.metric, m.metric }); + try fmtValue(m, writer, raw); + try writer.writeAll("\n\n"); + } + } +} + +fn fmtValue(comptime m: anytype, writer: *std.Io.Writer, raw: anytype) !void { + if (@hasField(@TypeOf(m), "fmt")) { + try m.fmt(writer, raw); + } else { + try writeValue(writer, raw); + } +} + +fn writeValue(writer: *std.Io.Writer, val: anytype) std.Io.Writer.Error!void { + switch (@typeInfo(@TypeOf(val))) { + .float, .comptime_float => try writer.print("{d:.2}", .{val}), + .pointer => try writer.writeAll(val), + else => try writer.print("{d}", .{val}), + } +} + +fn fmtMilliwattsAsWatts(writer: *std.Io.Writer, val: u64) std.Io.Writer.Error!void { + try writer.print("{d}.{d:0>3}", .{ val / 1000, val % 1000 }); +} + +fn fmtScaled(comptime scale: u64) fn (*std.Io.Writer, u64) std.Io.Writer.Error!void { + return struct { + fn f(writer: *std.Io.Writer, val: u64) std.Io.Writer.Error!void { + try writer.print("{d}", .{val * scale}); + } + }.f; +} diff --git a/bin/zml-smi/prometheus/prometheus.zig b/bin/zml-smi/prometheus/prometheus.zig new file mode 100644 index 00000000..cfa2fc28 --- /dev/null +++ b/bin/zml-smi/prometheus/prometheus.zig @@ -0,0 +1,2 @@ +pub const Server = @import("server.zig").Server; +pub const exposition = @import("exposition.zig"); diff --git a/bin/zml-smi/prometheus/server.zig b/bin/zml-smi/prometheus/server.zig new file mode 100644 index 00000000..9517ecac --- /dev/null +++ b/bin/zml-smi/prometheus/server.zig @@ -0,0 +1,87 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const HostInfo = smi_info.host_info.HostInfo; +const exposition = @import("exposition.zig"); + +pub const Server = struct { + io: std.Io, + devices: []const *DeviceInfo, + host: *const HostInfo, + tcp: std.Io.net.Server, + connection_group: std.Io.Group = .init, + + pub fn init(io: std.Io, listen: []const u8, devices: []const *DeviceInfo, host: *const HostInfo) !Server { + const address = try std.Io.net.IpAddress.parseLiteral(listen); + var tcp = try address.listen(io, .{ .reuse_address = true }); + errdefer tcp.deinit(io); + + return .{ + .io = io, + .devices = devices, + .host = host, + .tcp = tcp, + }; + } + + pub fn run(io: std.Io, listen: []const u8, devices: []const *DeviceInfo, host_info: *const HostInfo) void { + var self = init(io, listen, devices, host_info) catch |err| { + std.log.err("prometheus server failed to start: {s}", .{@errorName(err)}); + return; + }; + defer self.deinit(); + self.serve(); + } + + pub fn deinit(self: *Server) void { + self.connection_group.await(self.io) catch {}; + self.tcp.deinit(self.io); + } + + pub fn serve(self: *Server) void { + while (true) { + const stream = self.tcp.accept(self.io) catch break; + self.connection_group.concurrent(self.io, Server.onConnection, .{ self, stream }) catch break; + } + } + + fn onConnection(self: *Server, stream: std.Io.net.Stream) std.Io.Cancelable!void { + defer stream.close(self.io); + self.handleConn(stream) catch {}; + } + + fn handleConn(self: *Server, stream: std.Io.net.Stream) !void { + var read_buf: [8192]u8 = undefined; + var reader = stream.reader(self.io, &read_buf); + + var write_buf: [4096]u8 = undefined; + var writer = stream.writer(self.io, &write_buf); + + var http_server: std.http.Server = .init(&reader.interface, &writer.interface); + + while (true) { + var request = http_server.receiveHead() catch |err| switch (err) { + error.HttpConnectionClosing => return, + else => return err, + }; + + if (request.head.method != .GET) { + try request.respond("Method Not Allowed\n", .{ .status = .method_not_allowed }); + continue; + } + + var body_buf: [4096]u8 = undefined; + var body_writer = try request.respondStreaming(&body_buf, .{ + .respond_options = .{ + .status = .ok, + .extra_headers = &.{ + .{ .name = "content-type", .value = "text/plain" }, + }, + }, + }); + + try exposition.write(&body_writer.writer, self.devices, self.host); + try body_writer.end(); + } + } +}; diff --git a/bin/zml-smi/scripts/release.sh b/bin/zml-smi/scripts/release.sh new file mode 100755 index 00000000..9cd0e50a --- /dev/null +++ b/bin/zml-smi/scripts/release.sh @@ -0,0 +1,394 @@ +#!/usr/bin/env bash +set -euo pipefail + +# --- UI --- + +if [ -t 1 ]; then + DIM='\033[2m' + RESET='\033[0m' + BLUE='\033[1;38;2;140;180;255m' + GREEN='\033[1;32m' + RED='\033[1;31m' + YELLOW='\033[1;33m' + CYAN='\033[1;36m' + WHITE='\033[1;37m' +else + DIM='' RESET='' BLUE='' GREEN='' RED='' YELLOW='' CYAN='' WHITE='' +fi + +info() { printf "${BLUE} │${RESET} %b\n" "$*"; } +success() { printf "${GREEN} ✓${RESET} %b\n" "$*"; } +warn() { printf "${YELLOW} !${RESET} %b\n" "$*"; } +fail() { printf "${RED} ✗ %b${RESET}\n" "$*" >&2; exit 1; } +step() { printf "\n${WHITE} ▸ %b${RESET}\n" "$*"; } + +confirm() { + if [ "$YES" = true ]; then + return 0 + fi + printf "${WHITE} ? %b ${DIM}[y/N]${RESET} " "$*" + local answer + read -r answer + case "$answer" in + y|Y|yes|Yes) return 0 ;; + *) return 1 ;; + esac +} + +usage() { + printf >&2 " + ${WHITE}Usage${RESET} + bazel run //bin/zml-smi:release -- [flags] + + ${WHITE}Arguments${RESET} + ${CYAN}version${RESET} Semantic version, e.g. ${DIM}1.0.0${RESET} (tagged as ${DIM}zml-smi-v1.0.0${RESET}) + + ${WHITE}Flags${RESET} + ${CYAN}--dry-run${RESET} Build only — skip tagging, uploading, and publishing + ${CYAN}--verify${RESET} Verify uploaded files by comparing sha256 against the CDN + ${CYAN}--publish${RESET} Push git tag and create a GitHub release + ${CYAN}-y, --yes${RESET} Skip all confirmation prompts + ${CYAN}-h, --help${RESET} Show this help + + ${WHITE}Environment${RESET} + ${CYAN}BUCKET${RESET} S3-compatible bucket name + ${CYAN}BASE_URL${RESET} Public base URL serving the bucket + ${CYAN}RCLONE_REMOTE${RESET} Rclone remote name ${DIM}(default: r2)${RESET} + + ${DIM}If rclone is not configured, you can set the remote via env vars:${RESET} + ${DIM} RCLONE_CONFIG_R2_TYPE=s3${RESET} + ${DIM} RCLONE_CONFIG_R2_PROVIDER=Cloudflare${RESET} + ${DIM} RCLONE_CONFIG_R2_ACCESS_KEY_ID=your_access_key${RESET} + ${DIM} RCLONE_CONFIG_R2_SECRET_ACCESS_KEY=your_secret_key${RESET} + ${DIM} RCLONE_CONFIG_R2_ENDPOINT=https://.r2.cloudflarestorage.com${RESET} + + ${WHITE}Examples${RESET} + ${DIM}# Interactive, local only${RESET} + bazel run //bin/zml-smi:release -- 0.4.0 + + ${DIM}# Full release, non-interactive${RESET} + bazel run //bin/zml-smi:release -- --publish --verify --yes 0.4.0 + +" + exit 0 +} + +# --- Checks --- + +check_env() { + [ -z "${BUILD_WORKSPACE_DIRECTORY:-}" ] && fail "Must be run via 'bazel run //bin/zml-smi:release -- '" + local missing=() + [ -z "${BUCKET:-}" ] && missing+=("BUCKET") + [ -z "${BASE_URL:-}" ] && missing+=("BASE_URL") + if [ ${#missing[@]} -gt 0 ]; then + fail "Missing required environment variables: ${CYAN}${missing[*]}" + fi +} + +# --- Steps --- + +changelog() { + local prev_tag + prev_tag="$(git describe --tags --abbrev=0 --match 'zml-smi-v*' 2>/dev/null || true)" + local range="${prev_tag:+${prev_tag}..HEAD}" + + git log ${range:-HEAD} --pretty=format:"- %s" -- bin/zml-smi/ | grep -vi WIP || true +} + +tag() { + step "Tagging ${CYAN}${TAG}${WHITE}" + + # Cache changelog before tagging, otherwise the new tag makes the range empty + CACHED_CHANGELOG="$(changelog)" + + if [ -z "$CACHED_CHANGELOG" ]; then + warn "No changes since last release" + fi + + if git rev-parse "$TAG" &>/dev/null; then + warn "Tag ${TAG} already exists, skipping" + return + fi + + if [ "$DRY_RUN" = true ]; then + warn "Dry run — skipping tag creation" + elif confirm "Create tag ${CYAN}${TAG}${RESET}${WHITE}?"; then + git tag -a "$TAG" -m "$(printf '%s\n\n%s' "$TAG" "$CACHED_CHANGELOG")" + success "Created tag ${CYAN}${TAG}" + else + warn "Skipped tagging" + return + fi + printf "\n" + + printf "${DIM}" + echo "$CACHED_CHANGELOG" + printf "${RESET}" +} + +build() { + step "Building release archives" + + if ! confirm "Build all platforms?"; then + warn "Skipped build" + return 1 + fi + + printf "${DIM}%s${RESET}\n" "────────────────────────────────────────" + + bazel build --config=release //bin/zml-smi:archives + + printf "${DIM}%s${RESET}\n" "────────────────────────────────────────" + mapfile -t FILES < <(bazel cquery --config=release --output=files //bin/zml-smi:archives 2>/dev/null) + success "Built ${CYAN}${#FILES[@]}${RESET}${GREEN} artifacts" +} + +upload() { + local rclone_remote="${RCLONE_REMOTE:-r2}" + local install_url="${BASE_URL}/zml-smi/${VERSION}" + + step "Patching install script" + + INSTALL_SCRIPT="$(mktemp)" + sed -e "s|^BASE_URL=.*|BASE_URL=\"${install_url}\"|" \ + -e "s|^VERSION=.*|VERSION=\"${VERSION}\"|" \ + "${BUILD_WORKSPACE_DIRECTORY}/bin/zml-smi/scripts/remote-install.sh" > "$INSTALL_SCRIPT" + success "BASE_URL=${CYAN}${install_url}" + + if [ "$DRY_RUN" = true ]; then + step "Uploading to ${CYAN}${rclone_remote}:${BUCKET}" + warn "Dry run — skipping upload" + return + fi + + step "Uploading to ${CYAN}${rclone_remote}:${BUCKET}" + + if ! confirm "Upload artifacts?"; then + warn "Skipped upload" + return + fi + + for f in "${FILES[@]}"; do + if [ ! -f "$f" ]; then + warn "SKIP $(basename "$f") ${DIM}(not built for this host)${RESET}" + continue + fi + local name remote_name + name="$(basename "$f")" + remote_name="${name/zml-smi-/zml-smi-v${VERSION}-}" + rclone copyto "$f" "${rclone_remote}:${BUCKET}/zml-smi/${VERSION}/${remote_name}" + success "${remote_name}" + done + + rclone copyto "$INSTALL_SCRIPT" "${rclone_remote}:${BUCKET}/zml-smi/${VERSION}/install.sh" + success "zml-smi/${VERSION}/install.sh" + + if confirm "Set ${CYAN}${VERSION}${RESET}${WHITE} as the default install version?"; then + rclone copyto "$INSTALL_SCRIPT" "${rclone_remote}:${BUCKET}/zml-smi/install.sh" + success "zml-smi/install.sh" + DEFAULT_INSTALL=true + else + warn "Skipped default install script" + DEFAULT_INSTALL=false + fi +} + +verify() { + local install_url="${BASE_URL}/zml-smi/${VERSION}" + + step "Verifying uploads" + + for f in "${FILES[@]}"; do + if [ ! -f "$f" ]; then + continue + fi + local name remote_name + name="$(basename "$f")" + remote_name="${name/zml-smi-/zml-smi-v${VERSION}-}" + local local_sha remote_sha + local_sha="$(shasum -a 256 "$f" | awk '{print $1}')" + remote_sha="$(curl -fsSL "${install_url}/${remote_name}" | shasum -a 256 | awk '{print $1}')" + if [ "$local_sha" = "$remote_sha" ]; then + success "${remote_name}" + else + fail "${remote_name} — sha256 mismatch (local: ${local_sha}, remote: ${remote_sha})" + fi + done + + local local_sha remote_sha + local_sha="$(shasum -a 256 "$INSTALL_SCRIPT" | awk '{print $1}')" + remote_sha="$(curl -fsSL "${install_url}/install.sh" | shasum -a 256 | awk '{print $1}')" + if [ "$local_sha" = "$remote_sha" ]; then + success "install.sh" + else + fail "install.sh — sha256 mismatch (local: ${local_sha}, remote: ${remote_sha})" + fi +} + +release_notes() { + local install_url="${BASE_URL}/zml-smi/${VERSION}" + local log="$CACHED_CHANGELOG" + + local downloads="" + for f in "${FILES[@]}"; do + [ -f "$f" ] || continue + local name remote_name + name="$(basename "$f")" + [[ "$name" == *.sha256 ]] && continue + local sha_file="" + for sf in "${FILES[@]}"; do + if [ "$(basename "$sf")" = "${name}.sha256" ]; then + sha_file="$sf" + break + fi + done + if [ -z "$sha_file" ] || [ ! -f "$sha_file" ]; then + fail "Missing sha256 sidecar for ${name}" + fi + local sha + sha="$(awk '{print $1}' "$sha_file")" + local actual + actual="$(shasum -a 256 "$f" | awk '{print $1}')" + if [ "$sha" != "$actual" ]; then + fail "${name} sha256 mismatch — sidecar: ${sha}, actual: ${actual}" + fi + remote_name="${name/zml-smi-/zml-smi-v${VERSION}-}" + downloads="${downloads}| [${remote_name}](${install_url}/${remote_name}) | \`${sha}\` |"$'\n' + done + + cat </dev/null && [ -w "$INSTALL_DIR" ] && mkdir -p "$BIN_DIR" 2>/dev/null && [ -w "$BIN_DIR" ]; then return fi - warn "Cannot write to /usr/lib or /usr/bin — falling back to ~/.local" - INSTALL_DIR="${HOME}/.local/lib/zml-smi" + warn "Cannot write to /opt or /usr/local/bin — falling back to ~/.local" + INSTALL_DIR="${HOME}/.local/zml-smi" BIN_DIR="${HOME}/.local/bin" } @@ -68,15 +69,19 @@ detect_platform() { esac case "$ARCH" in - x86_64|amd64) ARCH_LABEL="x86_64" ;; - aarch64|arm64) ARCH_LABEL="aarch64" ;; + x86_64|amd64) ARCH_LABEL="amd64" ;; + aarch64|arm64) ARCH_LABEL="arm64" ;; *) fail "Unsupported architecture: ${ARCH}" ;; esac + if [ "$OS_LABEL" = "macos" ] && [ "$ARCH_LABEL" = "amd64" ]; then + fail "macOS x86_64 is not supported. Only Apple Silicon (arm64) is supported." + fi + success "${OS_LABEL} ${ARCH_LABEL}" - DOWNLOAD_URL="${BASE_URL}/${BINARY_NAME}-${OS_LABEL}-${ARCH_LABEL}.tar.zst" + DOWNLOAD_URL="${BASE_URL}/${BINARY_NAME}-v${VERSION}-${OS_LABEL}-${ARCH_LABEL}.tar.zst" } check_deps() { diff --git a/bin/zml-smi/tui/BUILD.bazel b/bin/zml-smi/tui/BUILD.bazel index 93ca38f8..8b75b4c0 100644 --- a/bin/zml-smi/tui/BUILD.bazel +++ b/bin/zml-smi/tui/BUILD.bazel @@ -41,10 +41,12 @@ zig_library( ], visibility = ["//visibility:public"], deps = [ - "//bin/zml-smi/platforms/linux", "//bazel/stamp:zig", "//bin/zml-smi/info", "//bin/zml-smi/utils:double_buffer", "@libvaxis//:vaxis", - ], + ] + select({ + "@platforms//os:macos": ["//bin/zml-smi/platforms/macos"], + "//conditions:default": ["//bin/zml-smi/platforms/linux"], + }), ) diff --git a/bin/zml-smi/tui/data.zig b/bin/zml-smi/tui/data.zig index 49809210..ddd0f0eb 100644 --- a/bin/zml-smi/tui/data.zig +++ b/bin/zml-smi/tui/data.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const builtin = @import("builtin"); const smi_info = @import("zml-smi/info"); const di = smi_info.device_info; pub const DeviceInfo = di.DeviceInfo; @@ -12,7 +13,10 @@ pub const HostInfo = hi.HostInfo; pub const HostData = hi.HostData; pub const pi = smi_info.process_info; pub const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); -pub const ProcessEnricher = @import("zml-smi/platforms/linux").process.ProcessEnricher; +pub const ProcessEnricher = if (builtin.os.tag == .macos) + @import("zml-smi/platforms/macos").process.ProcessEnricher +else + @import("zml-smi/platforms/linux").process.ProcessEnricher; pub const history_len: usize = 500; @@ -84,7 +88,8 @@ pub const SystemState = struct { targets: Targets, process_lists: []*ProcessDoubleBuffer, enricher: *ProcessEnricher, - allocator: std.mem.Allocator, + gpa: std.mem.Allocator, + arena: std.mem.Allocator, io: std.Io, tui_refresh_rate: u16, @@ -95,19 +100,22 @@ pub const SystemState = struct { tui_refresh_rate: u16, process_lists: []*ProcessDoubleBuffer, enricher: *ProcessEnricher, + gpa: std.mem.Allocator, + arena: std.mem.Allocator, io: std.Io, }; - pub fn init(allocator: std.mem.Allocator, cfg: Config) !SystemState { + pub fn init(cfg: Config) !SystemState { return .{ .devices = cfg.devices, .host = cfg.host, - .history = try HistoryBuffers.init(allocator, cfg.devices.len), + .history = try HistoryBuffers.init(cfg.arena, cfg.devices.len), .targets = cfg.targets, .tui_refresh_rate = cfg.tui_refresh_rate, .process_lists = cfg.process_lists, .enricher = cfg.enricher, - .allocator = allocator, + .gpa = cfg.gpa, + .arena = cfg.arena, .io = cfg.io, }; } diff --git a/bin/zml-smi/tui/pages/detail.zig b/bin/zml-smi/tui/pages/detail.zig index a17dc17b..d307bf5e 100644 --- a/bin/zml-smi/tui/pages/detail.zig +++ b/bin/zml-smi/tui/pages/detail.zig @@ -11,7 +11,7 @@ const tpu_detail = @import("detail/tpu_detail.zig"); const Detail = @This(); state: *const data.SystemState, -device_id: u8, +device_id: u16, process_table: *ProcessTable = undefined, pub fn draw(self: *const Detail, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { diff --git a/bin/zml-smi/tui/pages/detail/detail_common.zig b/bin/zml-smi/tui/pages/detail/detail_common.zig index e499fcae..93e8addb 100644 --- a/bin/zml-smi/tui/pages/detail/detail_common.zig +++ b/bin/zml-smi/tui/pages/detail/detail_common.zig @@ -9,7 +9,7 @@ const compose = @import("../../lib/compose.zig"); const Chart = @import("../../widgets/chart.zig"); const ColumnLayout = @import("../../widgets/column_layout.zig"); -pub fn headerText(arena: std.mem.Allocator, id: u8, name: []const u8) std.mem.Allocator.Error!vxfw.Text { +pub fn headerText(arena: std.mem.Allocator, id: u16, name: []const u8) std.mem.Allocator.Error!vxfw.Text { return .{ .text = try std.fmt.allocPrint(arena, "Device {d}: {s}", .{ id, name }), .style = theme.header_style, @@ -22,7 +22,7 @@ pub fn headerText(arena: std.mem.Allocator, id: u8, name: []const u8) std.mem.Al pub fn historyChart( ctx: vxfw.DrawContext, state: *const data.SystemState, - id: u8, + id: u16, history: anytype, opts: struct { title: []const u8, diff --git a/bin/zml-smi/tui/pages/detail/gpu_detail.zig b/bin/zml-smi/tui/pages/detail/gpu_detail.zig index 6f1c5d42..5943cfd2 100644 --- a/bin/zml-smi/tui/pages/detail/gpu_detail.zig +++ b/bin/zml-smi/tui/pages/detail/gpu_detail.zig @@ -15,7 +15,7 @@ pub fn draw( process_table: *ProcessTable, ctx: vxfw.DrawContext, w: u16, - id: u8, + id: u16, gpu: data.GpuInfo, parent_widget: vxfw.Widget, ) std.mem.Allocator.Error!vxfw.Surface { diff --git a/bin/zml-smi/tui/pages/detail/neuron_detail.zig b/bin/zml-smi/tui/pages/detail/neuron_detail.zig index 209d5360..e0dc3bcd 100644 --- a/bin/zml-smi/tui/pages/detail/neuron_detail.zig +++ b/bin/zml-smi/tui/pages/detail/neuron_detail.zig @@ -14,7 +14,7 @@ pub fn draw( process_table: *ProcessTable, ctx: vxfw.DrawContext, w: u16, - id: u8, + id: u16, nc: data.NeuronInfo, parent_widget: vxfw.Widget, ) std.mem.Allocator.Error!vxfw.Surface { diff --git a/bin/zml-smi/tui/pages/detail/tpu_detail.zig b/bin/zml-smi/tui/pages/detail/tpu_detail.zig index 7d0fac12..ca6d3b66 100644 --- a/bin/zml-smi/tui/pages/detail/tpu_detail.zig +++ b/bin/zml-smi/tui/pages/detail/tpu_detail.zig @@ -13,7 +13,7 @@ pub fn draw( process_table: *ProcessTable, ctx: vxfw.DrawContext, w: u16, - id: u8, + id: u16, tp: data.TpuInfo, parent_widget: vxfw.Widget, ) std.mem.Allocator.Error!vxfw.Surface { diff --git a/bin/zml-smi/tui/pages/overview.zig b/bin/zml-smi/tui/pages/overview.zig index d1ed28a0..6605d2ff 100644 --- a/bin/zml-smi/tui/pages/overview.zig +++ b/bin/zml-smi/tui/pages/overview.zig @@ -14,7 +14,7 @@ const ProcessTable = @import("../widgets/process_table.zig"); const Overview = @This(); -const two_col_breakpoint: u16 = 120; +const two_col_breakpoint: u16 = 140; const narrow_breakpoint: u16 = 80; const max_info_width: u16 = 50; /// Total width of the wide banner: logo box (incl. left margin) + info lines + right margin. @@ -23,10 +23,10 @@ pub const host_line_width: u16 = (Logo.logo_width + 6) + max_info_width + 2; state: *const data.SystemState, device_cards: []DeviceCard = &.{}, process_table: ?*ProcessTable = null, // maybe null if we decide not to print the process table in print mode -viewing_device: *?u8 = undefined, +viewing_device: *?u16 = undefined, use_braille: bool = false, -pub fn init(allocator: std.mem.Allocator, state: *const data.SystemState, process_table: ?*ProcessTable, viewing_device: *?u8) !Overview { +pub fn init(allocator: std.mem.Allocator, state: *const data.SystemState, process_table: ?*ProcessTable, viewing_device: *?u16) !Overview { const count = state.deviceCount(); const cards = try allocator.alloc(DeviceCard, count); for (cards, 0..) |*card, i| { diff --git a/bin/zml-smi/tui/print.zig b/bin/zml-smi/tui/print.zig index 96c3737a..7338d20e 100644 --- a/bin/zml-smi/tui/print.zig +++ b/bin/zml-smi/tui/print.zig @@ -56,7 +56,7 @@ pub fn run(allocator: std.mem.Allocator, io: std.Io, state: *data.SystemState) ! const content_w: u16 = @min(@as(u16, @intCast(vx.screen.width)), Overview.host_line_width + 2); - var viewing_device: ?u8 = null; + var viewing_device: ?u16 = null; var overview = try Overview.init(allocator, state, null, &viewing_device); defer overview.deinit(allocator); diff --git a/bin/zml-smi/tui/top.zig b/bin/zml-smi/tui/top.zig index 6e00abf5..0ad41424 100644 --- a/bin/zml-smi/tui/top.zig +++ b/bin/zml-smi/tui/top.zig @@ -40,8 +40,8 @@ const ScrollState = struct { const Model = struct { allocator: std.mem.Allocator, state: *data.SystemState, - viewing_device: ?u8 = null, - prev_viewing_device: ?u8 = null, + viewing_device: ?u16 = null, + prev_viewing_device: ?u16 = null, scroll: ScrollState = .{}, vx: *vaxis.Vaxis, tty: *vaxis.Tty, diff --git a/bin/zml-smi/tui/widgets/device_card.zig b/bin/zml-smi/tui/widgets/device_card.zig index ba025753..9d56f1f2 100644 --- a/bin/zml-smi/tui/widgets/device_card.zig +++ b/bin/zml-smi/tui/widgets/device_card.zig @@ -10,12 +10,12 @@ const MetricCard = @import("metric_card.zig"); const DeviceCard = @This(); -device_id: u8 = 0, +device_id: u16 = 0, highlighted: bool = false, state: *const data.SystemState = undefined, use_braille: bool = false, -viewing_device: *?u8 = undefined, +viewing_device: *?u16 = undefined, pub fn handleEvent(self: *DeviceCard, ctx: *vxfw.EventContext, event: vxfw.Event) anyerror!void { switch (event) { @@ -116,6 +116,7 @@ pub fn draw(self: *DeviceCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vx .neuron => "gpu_neuron", .tpu => "gpu_tpu", }), + .title_right = if (self.state.devices[i].isRemote()) "remote" else null, .cell_size = ctx.cell_size, .highlighted = self.highlighted, }; diff --git a/bin/zml-smi/tui/widgets/metric_card.zig b/bin/zml-smi/tui/widgets/metric_card.zig index d14378e7..6ceb9069 100644 --- a/bin/zml-smi/tui/widgets/metric_card.zig +++ b/bin/zml-smi/tui/widgets/metric_card.zig @@ -12,6 +12,7 @@ const MetricCard = @This(); title: []const u8, title_image: ?vaxis.Image = null, +title_right: ?[]const u8 = null, cell_size: vxfw.Size = .{}, /// Lines of "label: value" pairs to display. lines: []const LineEntry = &.{}, @@ -51,6 +52,7 @@ pub fn draw(self: *const MetricCard, ctx: vxfw.DrawContext) std.mem.Allocator.Er .child = ui.drawWidget(self, drawContent), .title = self.title, .title_image = self.title_image, + .value_label = self.title_right, .cell_size = self.cell_size, .border_style = bstyle, }; diff --git a/bin/zml-smi/tui/widgets/process_table.zig b/bin/zml-smi/tui/widgets/process_table.zig index 87956143..5cc08ef6 100644 --- a/bin/zml-smi/tui/widgets/process_table.zig +++ b/bin/zml-smi/tui/widgets/process_table.zig @@ -26,7 +26,7 @@ scroll_bars: vxfw.ScrollBars = .{ }, merged: std.ArrayList(ProcessInfo) = .{}, -pub fn prepare(self: *ProcessTable, state: *data.SystemState, device_id: ?u8) void { +pub fn prepare(self: *ProcessTable, state: *data.SystemState, device_id: ?u16) void { self.scroll_bars.scroll_view.children.builder.userdata = self; self.merged.clearRetainingCapacity(); @@ -37,10 +37,10 @@ pub fn prepare(self: *ProcessTable, state: *data.SystemState, device_id: ?u8) vo if (entry.device_idx != did) { continue; } - self.merged.append(state.allocator, entry) catch break; + self.merged.append(state.gpa, entry) catch break; } } else { - self.merged.appendSlice(state.allocator, pl_items) catch break; + self.merged.appendSlice(state.gpa, pl_items) catch break; } } diff --git a/bin/zml-smi/tui/widgets/status_line.zig b/bin/zml-smi/tui/widgets/status_line.zig index 86a6d1ef..59290ef6 100644 --- a/bin/zml-smi/tui/widgets/status_line.zig +++ b/bin/zml-smi/tui/widgets/status_line.zig @@ -6,7 +6,7 @@ const ui = @import("../lib/ui.zig"); const StatusLine = @This(); -viewing_device: ?u8, +viewing_device: ?u16, use_braille: bool, const Binding = struct { diff --git a/bin/zml-smi/utils/double_buffer.zig b/bin/zml-smi/utils/double_buffer.zig index 32ed7a1d..add3037b 100644 --- a/bin/zml-smi/utils/double_buffer.zig +++ b/bin/zml-smi/utils/double_buffer.zig @@ -3,6 +3,7 @@ const std = @import("std"); pub fn DoubleBuffer(comptime T: type) type { return struct { const Self = @This(); + pub const Value = T; values: [2]T, current: std.atomic.Value(u8) = std.atomic.Value(u8).init(0), @@ -18,5 +19,14 @@ pub fn DoubleBuffer(comptime T: type) type { pub fn swap(self: *Self) void { self.current.store(1 - self.current.load(.acquire), .release); } + + pub fn jsonStringify(self: *const Self, jw: *std.json.Stringify) !void { + try jw.write(self.front().*); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !Self { + const val = try std.json.innerParseFromValue(T, allocator, source, options); + return .{ .values = .{ val, val } }; + } }; } From e71c5236050aab0104f6bc90e0fc5b6b3ccfb98d Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 10 Apr 2026 17:30:09 +0200 Subject: [PATCH 411/881] workspace: swap strip linkopt to bazel flag because --strip-all doesn't exist on macos --- .bazelrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index ec5c8408..42f50189 100644 --- a/.bazelrc +++ b/.bazelrc @@ -118,7 +118,7 @@ common:alldebug --strategy=local common:release --compilation_mode=opt common:release --copt=-Ofast -common:release --linkopt=-Wl,--strip-all +common:release --strip=always common:release --@rules_zig//zig/settings:mode=release_safe # --config=native builds everything optimized for the local CPU From 572ba04a0cc36b3a425004a8b1919ce5e93c611b Mon Sep 17 00:00:00 2001 From: elogir <90246794+elogir@users.noreply.github.com> Date: Fri, 10 Apr 2026 17:55:27 +0200 Subject: [PATCH 412/881] bin/zml-smi: fix mapfile (#458) Replace mapfile with manual IFS read --- bin/zml-smi/scripts/release.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/zml-smi/scripts/release.sh b/bin/zml-smi/scripts/release.sh index 9cd0e50a..4fe95d0a 100755 --- a/bin/zml-smi/scripts/release.sh +++ b/bin/zml-smi/scripts/release.sh @@ -139,7 +139,10 @@ build() { bazel build --config=release //bin/zml-smi:archives printf "${DIM}%s${RESET}\n" "────────────────────────────────────────" - mapfile -t FILES < <(bazel cquery --config=release --output=files //bin/zml-smi:archives 2>/dev/null) + FILES=() + while IFS= read -r line; do + FILES+=("$line") + done < <(bazel cquery --config=release --output=files //bin/zml-smi:archives 2>/dev/null) success "Built ${CYAN}${#FILES[@]}${RESET}${GREEN} artifacts" } From 177fddabebb4675340fe4e919c489cfe813c64eb Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Sat, 11 Apr 2026 10:18:44 +0200 Subject: [PATCH 413/881] zml/attention & zml/ops: mark fa2 as sharding aware and fix implementation (#461) Sharding aware case wasn't properly handled and I marked fa2 as sharding unaware even though it is. --- zml/attention/flashattn.zig | 2 +- zml/ops.zig | 23 +++++++++++++++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 8bfc800d..9f95a5fa 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -220,7 +220,7 @@ pub const fa2 = struct { const fa2_mha_varlen_fwd = zml.ops.CustomCall(Input, Output, Attributes, ffiCall, .{ .name = "fa2_mha_varlen_fwd", - .sharding_aware = false, + .sharding_aware = true, .has_side_effect = false, .output_operand_aliases = .{ .o = .q }, }); diff --git a/zml/ops.zig b/zml/ops.zig index 5c344193..4e5558d4 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1742,6 +1742,7 @@ pub fn shardingAwareTypedCustomCall( } }).body); + // Convert the slice back to a struct var out: ShapeToTensor(O) = undefined; inline for (@typeInfo(@TypeOf(args.output_shapes)).@"struct".fields, 0..) |field, i| { @field(out, field.name) = output_tensors[i]; @@ -1888,11 +1889,25 @@ pub fn typedCustomCall( op.setAttributeByName("mhlo.sharding", mlir.stringAttribute(ctx.mlir_ctx, "{manual}")); } - var out: ShapeToTensor(O) = undefined; - inline for (output_shapes, @typeInfo(@TypeOf(args.output_shapes)).@"struct".fields, 0..) |output_shape, field, i| { - @field(out, field.name) = Tensor._result(output_shape, op.result(i)); + switch (@typeInfo(@TypeOf(args.output_shapes))) { + .@"struct" => |struct_info| { + var out: ShapeToTensor(O) = undefined; + inline for (output_shapes, struct_info.fields, 0..) |output_shape, field, i| { + @field(out, field.name) = Tensor._result(output_shape, op.result(i)); + } + return out; + }, + // Extra case to support []const Shape from shardingAwareTypedCustomCall + .pointer => |pointer_info| { + if (pointer_info.size != .slice) @compileError("Expected input slice"); + var out: []Tensor = allocator.alloc(Tensor, args.output_shapes.len) catch unreachable; + for (output_shapes, 0..) |output_shape, i| { + out[i] = Tensor._result(output_shape, op.result(i)); + } + return out; + }, + else => @compileError("Unsupported input type: " ++ @typeName(@TypeOf(args.input_tensor))), } - return out; } fn customCallArgsFromPjrtCallFrame(I: type, O: type, A: type, call_frame: *pjrt.ffi.CallFrame) struct { From 1362a2cc45fb4321d8d346900fd1f54a7257aa4b Mon Sep 17 00:00:00 2001 From: elogir <90246794+elogir@users.noreply.github.com> Date: Mon, 13 Apr 2026 11:09:43 +0200 Subject: [PATCH 414/881] bin/zml-smi: fix release script (#460) - bucket name in rclone config - version name will be tag --- bin/zml-smi/scripts/release.sh | 34 +++++++++++++-------------- bin/zml-smi/scripts/remote-install.sh | 2 +- 2 files changed, 17 insertions(+), 19 deletions(-) mode change 100644 => 100755 bin/zml-smi/scripts/remote-install.sh diff --git a/bin/zml-smi/scripts/release.sh b/bin/zml-smi/scripts/release.sh index 4fe95d0a..d1e89902 100755 --- a/bin/zml-smi/scripts/release.sh +++ b/bin/zml-smi/scripts/release.sh @@ -41,7 +41,7 @@ usage() { bazel run //bin/zml-smi:release -- [flags] ${WHITE}Arguments${RESET} - ${CYAN}version${RESET} Semantic version, e.g. ${DIM}1.0.0${RESET} (tagged as ${DIM}zml-smi-v1.0.0${RESET}) + ${CYAN}version${RESET} Version string, e.g. ${DIM}v1.0.0${RESET} (tagged as ${DIM}zml-smi-v1.0.0${RESET}) ${WHITE}Flags${RESET} ${CYAN}--dry-run${RESET} Build only — skip tagging, uploading, and publishing @@ -51,7 +51,6 @@ usage() { ${CYAN}-h, --help${RESET} Show this help ${WHITE}Environment${RESET} - ${CYAN}BUCKET${RESET} S3-compatible bucket name ${CYAN}BASE_URL${RESET} Public base URL serving the bucket ${CYAN}RCLONE_REMOTE${RESET} Rclone remote name ${DIM}(default: r2)${RESET} @@ -76,12 +75,11 @@ usage() { # --- Checks --- check_env() { - [ -z "${BUILD_WORKSPACE_DIRECTORY:-}" ] && fail "Must be run via 'bazel run //bin/zml-smi:release -- '" - local missing=() - [ -z "${BUCKET:-}" ] && missing+=("BUCKET") - [ -z "${BASE_URL:-}" ] && missing+=("BASE_URL") - if [ ${#missing[@]} -gt 0 ]; then - fail "Missing required environment variables: ${CYAN}${missing[*]}" + if [ -z "${BUILD_WORKSPACE_DIRECTORY:-}" ]; then + fail "Must be run via 'bazel run //bin/zml-smi:release -- '" + fi + if [ -z "${BASE_URL:-}" ]; then + fail "Missing required environment variable: ${CYAN}BASE_URL" fi } @@ -89,7 +87,7 @@ check_env() { changelog() { local prev_tag - prev_tag="$(git describe --tags --abbrev=0 --match 'zml-smi-v*' 2>/dev/null || true)" + prev_tag="$(git describe --tags --abbrev=0 --match 'zml-smi-*' 2>/dev/null || true)" local range="${prev_tag:+${prev_tag}..HEAD}" git log ${range:-HEAD} --pretty=format:"- %s" -- bin/zml-smi/ | grep -vi WIP || true @@ -159,12 +157,12 @@ upload() { success "BASE_URL=${CYAN}${install_url}" if [ "$DRY_RUN" = true ]; then - step "Uploading to ${CYAN}${rclone_remote}:${BUCKET}" + step "Uploading to ${CYAN}${rclone_remote}" warn "Dry run — skipping upload" return fi - step "Uploading to ${CYAN}${rclone_remote}:${BUCKET}" + step "Uploading to ${CYAN}${rclone_remote}" if ! confirm "Upload artifacts?"; then warn "Skipped upload" @@ -178,16 +176,16 @@ upload() { fi local name remote_name name="$(basename "$f")" - remote_name="${name/zml-smi-/zml-smi-v${VERSION}-}" - rclone copyto "$f" "${rclone_remote}:${BUCKET}/zml-smi/${VERSION}/${remote_name}" + remote_name="${name/zml-smi-/zml-smi-${VERSION}-}" + rclone copyto "$f" "${rclone_remote}:zml-smi/${VERSION}/${remote_name}" success "${remote_name}" done - rclone copyto "$INSTALL_SCRIPT" "${rclone_remote}:${BUCKET}/zml-smi/${VERSION}/install.sh" + rclone copyto "$INSTALL_SCRIPT" "${rclone_remote}:zml-smi/${VERSION}/install.sh" success "zml-smi/${VERSION}/install.sh" if confirm "Set ${CYAN}${VERSION}${RESET}${WHITE} as the default install version?"; then - rclone copyto "$INSTALL_SCRIPT" "${rclone_remote}:${BUCKET}/zml-smi/install.sh" + rclone copyto "$INSTALL_SCRIPT" "${rclone_remote}:zml-smi/install.sh" success "zml-smi/install.sh" DEFAULT_INSTALL=true else @@ -207,7 +205,7 @@ verify() { fi local name remote_name name="$(basename "$f")" - remote_name="${name/zml-smi-/zml-smi-v${VERSION}-}" + remote_name="${name/zml-smi-/zml-smi-${VERSION}-}" local local_sha remote_sha local_sha="$(shasum -a 256 "$f" | awk '{print $1}')" remote_sha="$(curl -fsSL "${install_url}/${remote_name}" | shasum -a 256 | awk '{print $1}')" @@ -255,7 +253,7 @@ release_notes() { if [ "$sha" != "$actual" ]; then fail "${name} sha256 mismatch — sidecar: ${sha}, actual: ${actual}" fi - remote_name="${name/zml-smi-/zml-smi-v${VERSION}-}" + remote_name="${name/zml-smi-/zml-smi-${VERSION}-}" downloads="${downloads}| [${remote_name}](${install_url}/${remote_name}) | \`${sha}\` |"$'\n' done @@ -349,7 +347,7 @@ main() { check_env - TAG="zml-smi-v${VERSION}" + TAG="zml-smi-${VERSION}" local title="zml-smi release ${TAG}" if [ "$DRY_RUN" = true ]; then diff --git a/bin/zml-smi/scripts/remote-install.sh b/bin/zml-smi/scripts/remote-install.sh old mode 100644 new mode 100755 index 66d24a0b..a909f53e --- a/bin/zml-smi/scripts/remote-install.sh +++ b/bin/zml-smi/scripts/remote-install.sh @@ -81,7 +81,7 @@ detect_platform() { success "${OS_LABEL} ${ARCH_LABEL}" - DOWNLOAD_URL="${BASE_URL}/${BINARY_NAME}-v${VERSION}-${OS_LABEL}-${ARCH_LABEL}.tar.zst" + DOWNLOAD_URL="${BASE_URL}/${BINARY_NAME}-${VERSION}-${OS_LABEL}-${ARCH_LABEL}.tar.zst" } check_deps() { From ac9bc32d89eeca04573d6d62c4d1b8ed34c69032 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 13 Apr 2026 11:39:08 +0200 Subject: [PATCH 415/881] zml/attention: add Triton Unified Attention backend (#417) This PR introduces a Triton paged attention backend, unlocking ROCm support. It relies on a Python sandbox to generate the TTIR. The major change is that the KVCache now needs to be per-layer for paged attention. The `layer_index` is now superfluous and will probably be removed in the future. --- .nvim.lua | 72 +- MODULE.bazel | 13 +- MODULE.bazel.lock | 410 +------ examples/benchmark/main.zig | 2 +- examples/io/main.zig | 2 +- examples/llm/main.zig | 2 +- examples/llm/models/lfm2_tests.zig | 2 +- examples/llm/models/llama_tests.zig | 2 +- examples/mnist/mnist.zig | 2 +- examples/sharding/main.zig | 2 +- zml/BUILD.bazel | 5 + zml/attention.zig | 1 + zml/attention/flashattn.zig | 50 +- zml/attention/paged_attention.zig | 25 +- zml/attention/triton.zig | 664 ++++++++++ zml/attention/triton/BUILD.bazel | 110 ++ zml/attention/triton/fake_plugin/__init__.py | 0 zml/attention/triton/fake_plugin/compiler.py | 97 ++ zml/attention/triton/fake_plugin/driver.py | 55 + zml/attention/triton/generate.py | 365 ++++++ zml/attention/triton/generate.zig | 111 ++ zml/attention/triton/libpython.h | 3 + zml/attention/triton/requirements.in | 1 + zml/attention/triton/requirements.lock.txt | 20 + .../triton/triton_kernels/__init__.py | 0 .../triton_kernels/unified_attention.py | 1073 +++++++++++++++++ zml/io.zig | 2 +- zml/module.zig | 17 +- zml/platform.zig | 18 +- 29 files changed, 2643 insertions(+), 483 deletions(-) create mode 100644 zml/attention/triton.zig create mode 100644 zml/attention/triton/BUILD.bazel create mode 100755 zml/attention/triton/fake_plugin/__init__.py create mode 100755 zml/attention/triton/fake_plugin/compiler.py create mode 100755 zml/attention/triton/fake_plugin/driver.py create mode 100755 zml/attention/triton/generate.py create mode 100644 zml/attention/triton/generate.zig create mode 100644 zml/attention/triton/libpython.h create mode 100644 zml/attention/triton/requirements.in create mode 100644 zml/attention/triton/requirements.lock.txt create mode 100755 zml/attention/triton/triton_kernels/__init__.py create mode 100755 zml/attention/triton/triton_kernels/unified_attention.py diff --git a/.nvim.lua b/.nvim.lua index a1f9ad95..56c43b69 100644 --- a/.nvim.lua +++ b/.nvim.lua @@ -6,31 +6,61 @@ local f, err = loadfile(".nvim.local.lua") if not err then f() else - local capabilities = vim.lsp.protocol.make_client_capabilities() - - -- Prepend CWD to relative paths - local zls_cmd = cwd .. "/../zml/tools/zls.sh" - vim.lsp.config["zls"] = { - capabilities = capabilities, - cmd = { zls_cmd }, - root_marker = { "build.zig" }, - filetypes = {"zig"}, + cmd = { cwd .. "/tools/zls.sh" }, } - vim.lsp.enable('zls') + vim.lsp.enable("zls") + + function runFixAll() + local params = vim.lsp.util.make_range_params() + params.context = { + only = { "source.fixAll" }, + diagnostics = {}, + } + + -- Send synchronous request (timeout 2 seconds) + local responses = vim.lsp.buf_request_sync(0, "textDocument/codeAction", params, 1000) + if not responses then return end + + for _, resp in pairs(responses) do + for _, action in ipairs(resp.result or {}) do + -- If action has edits, apply them + if action.edit then + vim.lsp.util.apply_workspace_edit(action.edit, "utf-16") + end + -- If the action has a command, execute it + if action.command then + vim.lsp.buf.execute_command(action.command) + end + end + end + end vim.api.nvim_create_autocmd('BufWritePre',{ pattern = {"*.zig", "*.zon"}, - callback = function(ev) - vim.lsp.buf.code_action({ - context = { only = { "source.organizeImports" } }, - apply = true, - }) - vim.lsp.buf.code_action({ - context = { only = { "source.fixAll" } }, - apply = true, - }) - vim.loop.sleep(100) - end + callback = runFixAll, }) + + local ok, mod = pcall(require, "conform") + if ok then + mod.setup({ + formatters_by_ft = { + bzl = { "buildifier" }, + }, + formatters = { + buildifier = { + command = "tools/buildifier.sh", + } + }, + }) + + vim.api.nvim_create_autocmd("FileType", { + pattern = "bzl", + callback = function() + vim.keymap.set('n', 'f', function() + mod.format() + end, { remap = false, silent = true }) + end + }) + end end diff --git a/MODULE.bazel b/MODULE.bazel index 5ed79f39..79c99a03 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -19,7 +19,7 @@ bazel_dep(name = "rules_distroless", version = "0.6.2") bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep(name = "rules_oci", version = "2.2.7") bazel_dep(name = "rules_proto", version = "7.1.0") -bazel_dep(name = "rules_python", version = "1.8.4") +bazel_dep(name = "rules_python", version = "1.9.0") bazel_dep(name = "rules_rs", version = "0.0.44") bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "rules_zig", version = "0.12.2") @@ -83,7 +83,7 @@ uv.configure(version = "0.9.30") python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain(python_version = "3.12") -use_repo(python, "python_versions") +use_repo(python, "python_versions", "python_3_12") pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") @@ -124,6 +124,15 @@ pip.parse( ) use_repo(pip, "xprof_nightly") +# For Triton generation +pip.parse( + download_only = True, + hub_name = "triton_py_deps", + python_version = "3.12", + requirements_lock = "//zml/attention/triton:requirements.lock.txt", +) +use_repo(pip, "triton_py_deps") + cpu = use_extension("//platforms/cpu:cpu.bzl", "cpu_pjrt_plugin") use_repo(cpu, "libpjrt_cpu_darwin_amd64", "libpjrt_cpu_darwin_arm64", "libpjrt_cpu_linux_amd64") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index ee860aa2..bb22c713 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -325,8 +325,8 @@ "https://bcr.bazel.build/modules/rules_python/1.5.1/MODULE.bazel": "acfe65880942d44a69129d4c5c3122d57baaf3edf58ae5a6bd4edea114906bf5", "https://bcr.bazel.build/modules/rules_python/1.6.0/MODULE.bazel": "7e04ad8f8d5bea40451cf80b1bd8262552aa73f841415d20db96b7241bd027d8", "https://bcr.bazel.build/modules/rules_python/1.7.0/MODULE.bazel": "d01f995ecd137abf30238ad9ce97f8fc3ac57289c8b24bd0bf53324d937a14f8", - "https://bcr.bazel.build/modules/rules_python/1.8.4/MODULE.bazel": "33e3971e66161a3e955f7a0d411a8d1f291c4ce4c561851512466f3c77ff8ece", - "https://bcr.bazel.build/modules/rules_python/1.8.4/source.json": "9fbc0e57bae52cddcc3831d668bce87a47e0c655104a85098d4459dd9a3b0a10", + "https://bcr.bazel.build/modules/rules_python/1.9.0/MODULE.bazel": "afc3a05f29f09f2d3ee95ad99a145250dab41a2b2d8d6f82cc91936b3213282c", + "https://bcr.bazel.build/modules/rules_python/1.9.0/source.json": "3921ea0b65298d51aead5b9e4a82203d7be9b5918619b58b53f1c259f4e63169", "https://bcr.bazel.build/modules/rules_rs/0.0.44/MODULE.bazel": "aa151d9a96e5d9363732df8943c7fb5327738cc160732fa74c02e5de7822007b", "https://bcr.bazel.build/modules/rules_rs/0.0.44/source.json": "1358c9418e1325fdb5d0caf53ef32f98d525fe2030668279cae2e97c34cd29eb", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", @@ -478,173 +478,6 @@ ] } }, - "@@googleapis+//:extensions.bzl%switched_rules": { - "general": { - "bzlTransitiveDigest": "JLpojYeGHLdgnwSUuQWOrtkL9UMygBUUEFJZh7ASlmA=", - "usagesDigest": "66rLsrgz6YqDpU0pD+02mTGNxYxhb/of6Fh0ulHzjdE=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "com_google_googleapis_imports": { - "repoRuleId": "@@googleapis+//:repository_rules.bzl%switched_rules", - "attributes": { - "rules": { - "proto_library_with_info": [ - "", - "" - ], - "moved_proto_library": [ - "", - "" - ], - "java_proto_library": [ - "", - "" - ], - "java_grpc_library": [ - "", - "" - ], - "java_gapic_library": [ - "", - "" - ], - "java_gapic_test": [ - "", - "" - ], - "java_gapic_assembly_gradle_pkg": [ - "", - "" - ], - "py_proto_library": [ - "", - "" - ], - "py_grpc_library": [ - "", - "" - ], - "py_gapic_library": [ - "", - "" - ], - "py_test": [ - "", - "" - ], - "py_gapic_assembly_pkg": [ - "", - "" - ], - "py_import": [ - "", - "" - ], - "go_proto_library": [ - "", - "" - ], - "go_grpc_library": [ - "", - "" - ], - "go_library": [ - "", - "" - ], - "go_test": [ - "", - "" - ], - "go_gapic_library": [ - "", - "" - ], - "go_gapic_assembly_pkg": [ - "", - "" - ], - "cc_proto_library": [ - "", - "" - ], - "cc_grpc_library": [ - "", - "" - ], - "cc_gapic_library": [ - "", - "" - ], - "php_proto_library": [ - "", - "php_proto_library" - ], - "php_grpc_library": [ - "", - "php_grpc_library" - ], - "php_gapic_library": [ - "", - "php_gapic_library" - ], - "php_gapic_assembly_pkg": [ - "", - "php_gapic_assembly_pkg" - ], - "nodejs_gapic_library": [ - "", - "typescript_gapic_library" - ], - "nodejs_gapic_assembly_pkg": [ - "", - "typescript_gapic_assembly_pkg" - ], - "ruby_proto_library": [ - "", - "" - ], - "ruby_grpc_library": [ - "", - "" - ], - "ruby_ads_gapic_library": [ - "", - "" - ], - "ruby_cloud_gapic_library": [ - "", - "" - ], - "ruby_gapic_assembly_pkg": [ - "", - "" - ], - "csharp_proto_library": [ - "", - "" - ], - "csharp_grpc_library": [ - "", - "" - ], - "csharp_gapic_library": [ - "", - "" - ], - "csharp_gapic_assembly_pkg": [ - "", - "" - ] - } - } - } - }, - "recordedRepoMappingEntries": [] - } - }, "@@rules_foreign_cc+//foreign_cc:extensions.bzl%tools": { "general": { "bzlTransitiveDigest": "b41JwymKkuHbovxSrZD6eawffiN8+iwKcSQZrLCkvbw=", @@ -1243,245 +1076,10 @@ ] } }, - "@@rules_python+//python/extensions:config.bzl%config": { - "general": { - "bzlTransitiveDigest": "X49Lbeli5+XwNYRk+isOvNKFMIF8g/rnxL3Fdlwrffc=", - "usagesDigest": "p2al+dDKI5UlCyNvheMVynbWSGbdiji/jMz53fMNfJA=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "rules_python_internal": { - "repoRuleId": "@@rules_python+//python/private:internal_config_repo.bzl%internal_config_repo", - "attributes": { - "transition_setting_generators": {}, - "transition_settings": [] - } - }, - "pypi__build": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/e2/03/f3c8ba0a6b6e30d7d18c40faab90807c9bb5e9a1e3b2fe2008af624a9c97/build-1.2.1-py3-none-any.whl", - "sha256": "75e10f767a433d9a86e50d83f418e83efc18ede923ee5ff7df93b6cb0306c5d4", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__click": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl", - "sha256": "ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__colorama": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", - "sha256": "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__importlib_metadata": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/2d/0a/679461c511447ffaf176567d5c496d1de27cbe34a87df6677d7171b2fbd4/importlib_metadata-7.1.0-py3-none-any.whl", - "sha256": "30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__installer": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl", - "sha256": "05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__more_itertools": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/50/e2/8e10e465ee3987bb7c9ab69efb91d867d93959095f4807db102d07995d94/more_itertools-10.2.0-py3-none-any.whl", - "sha256": "686b06abe565edfab151cb8fd385a05651e1fdf8f0a14191e4439283421f8684", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__packaging": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/49/df/1fceb2f8900f8639e278b056416d49134fb8d84c5942ffaa01ad34782422/packaging-24.0-py3-none-any.whl", - "sha256": "2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__pep517": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/25/6e/ca4a5434eb0e502210f591b97537d322546e4833dcb4d470a48c375c5540/pep517-0.13.1-py3-none-any.whl", - "sha256": "31b206f67165b3536dd577c5c3f1518e8fbaf38cbc57efff8369a392feff1721", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__pip": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/8a/6a/19e9fe04fca059ccf770861c7d5721ab4c2aebc539889e97c7977528a53b/pip-24.0-py3-none-any.whl", - "sha256": "ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__pip_tools": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/0d/dc/38f4ce065e92c66f058ea7a368a9c5de4e702272b479c0992059f7693941/pip_tools-7.4.1-py3-none-any.whl", - "sha256": "4c690e5fbae2f21e87843e89c26191f0d9454f362d8acdbd695716493ec8b3a9", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__pyproject_hooks": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/ae/f3/431b9d5fe7d14af7a32340792ef43b8a714e7726f1d7b69cc4e8e7a3f1d7/pyproject_hooks-1.1.0-py3-none-any.whl", - "sha256": "7ceeefe9aec63a1064c18d939bdc3adf2d8aa1988a510afec15151578b232aa2", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__setuptools": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/90/99/158ad0609729111163fc1f674a5a42f2605371a4cf036d0441070e2f7455/setuptools-78.1.1-py3-none-any.whl", - "sha256": "c3a9c4211ff4c309edb8b8c4f1cbfa7ae324c4ba9f91ff254e3d305b9fd54561", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__tomli": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", - "sha256": "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__wheel": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/7d/cd/d7460c9a869b16c3dd4e1e403cce337df165368c71d6af229a74699622ce/wheel-0.43.0-py3-none-any.whl", - "sha256": "55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__zipp": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/da/55/a03fd7240714916507e1fcf7ae355bd9d9ed2e6db492595f1a67f61681be/zipp-3.18.2-py3-none-any.whl", - "sha256": "dce197b859eb796242b0622af1b8beb0a722d52aa2f57133ead08edd5bf5374e", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_python+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_python+", - "pypi__build", - "rules_python++config+pypi__build" - ], - [ - "rules_python+", - "pypi__click", - "rules_python++config+pypi__click" - ], - [ - "rules_python+", - "pypi__colorama", - "rules_python++config+pypi__colorama" - ], - [ - "rules_python+", - "pypi__importlib_metadata", - "rules_python++config+pypi__importlib_metadata" - ], - [ - "rules_python+", - "pypi__installer", - "rules_python++config+pypi__installer" - ], - [ - "rules_python+", - "pypi__more_itertools", - "rules_python++config+pypi__more_itertools" - ], - [ - "rules_python+", - "pypi__packaging", - "rules_python++config+pypi__packaging" - ], - [ - "rules_python+", - "pypi__pep517", - "rules_python++config+pypi__pep517" - ], - [ - "rules_python+", - "pypi__pip", - "rules_python++config+pypi__pip" - ], - [ - "rules_python+", - "pypi__pip_tools", - "rules_python++config+pypi__pip_tools" - ], - [ - "rules_python+", - "pypi__pyproject_hooks", - "rules_python++config+pypi__pyproject_hooks" - ], - [ - "rules_python+", - "pypi__setuptools", - "rules_python++config+pypi__setuptools" - ], - [ - "rules_python+", - "pypi__tomli", - "rules_python++config+pypi__tomli" - ], - [ - "rules_python+", - "pypi__wheel", - "rules_python++config+pypi__wheel" - ], - [ - "rules_python+", - "pypi__zipp", - "rules_python++config+pypi__zipp" - ] - ] - } - }, "@@rules_python+//python/uv:uv.bzl%uv": { "general": { - "bzlTransitiveDigest": "CJBWuaTerWIqLpLWl77vjWFD1RRpXfzC2UGMwEk6eVg=", - "usagesDigest": "zIGk4Tqt5ZS3xJ783QnGoVqaEhMIIzUMmK0DHMRGugg=", + "bzlTransitiveDigest": "LM8+g04lV+afYP67YcE7wenTJMrNqvF5u/ZnOctwctY=", + "usagesDigest": "Ucfum/vsVnlkhtglTERDLTj+bBeG7dcpbV9h0LP9HOQ=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index 304cea91..b7be859d 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -22,7 +22,7 @@ pub fn main(init: std.process.Init) !void { // Auto-select platform const platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator); + defer platform.deinit(allocator, io); const logical_mesh: zml.sharding.LogicalMesh = try .init("benchmark_mesh", .{ .m = .low_bandwidth, diff --git a/examples/io/main.zig b/examples/io/main.zig index 658bf2bb..53280396 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -159,7 +159,7 @@ pub fn main(init: std.process.Init) !void { const sharding_type: ShardingType = std.meta.stringToEnum(ShardingType, it.next() orelse "sharded") orelse return error.InvalidShardingKind; const platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator); + defer platform.deinit(allocator, io); var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); defer registry.deinit(); diff --git a/examples/llm/main.zig b/examples/llm/main.zig index cc3c3342..abd3a86a 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -79,7 +79,7 @@ pub fn main(init: std.process.Init) !void { // Platform and Backend Selection // const platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator); + defer platform.deinit(allocator, io); log.info("\n{f}", .{platform.fmtVerbose()}); diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig index 288fecd1..73a539a9 100644 --- a/examples/llm/models/lfm2_tests.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -35,7 +35,7 @@ pub fn main(init: std.process.Init) !void { const args = zml.stdx.flags.parse(init.minimal.args, Args); const platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator); + defer platform.deinit(allocator, io); const repo = try zml.safetensors.resolveModelRepo(io, args.model); var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); diff --git a/examples/llm/models/llama_tests.zig b/examples/llm/models/llama_tests.zig index afec873d..777517ec 100644 --- a/examples/llm/models/llama_tests.zig +++ b/examples/llm/models/llama_tests.zig @@ -32,7 +32,7 @@ pub fn main(init: std.process.Init) !void { const args = zml.stdx.flags.parse(init.minimal.args, Args); const platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator); + defer platform.deinit(allocator, io); const repo = try zml.safetensors.resolveModelRepo(io, args.model); var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index de77ac8a..c3f7c01a 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -91,7 +91,7 @@ pub fn main(init: std.process.Init) !void { // Auto-select platform const platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator); + defer platform.deinit(allocator, io); const replicated_sharding = try zml.sharding.replicatedSharding(platform); diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index cfa67892..e6631ade 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -167,7 +167,7 @@ pub fn main(init: std.process.Init) !void { }; var platform: *zml.Platform = try .auto(allocator, io, create_options); - defer platform.deinit(allocator); + defer platform.deinit(allocator, io); log.info("\n{f}", .{platform.fmtVerbose()}); diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 8a2bfdc3..a8cee145 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -34,6 +34,7 @@ zig_library( "attention/attention.zig", "attention/flashattn.zig", "attention/paged_attention.zig", + "attention/triton.zig", "buffer.zig", "constants.zig", "dtype.zig", @@ -58,6 +59,10 @@ zig_library( "tracer.zig", "zml.zig", ], + data = select({ + "@platforms//os:macos": [], + "//conditions:default": ["//zml/attention/triton:sandbox"], + }), main = "zml.zig", visibility = ["//visibility:public"], zigopts = [ diff --git a/zml/attention.zig b/zml/attention.zig index 1aa8001b..909bbd14 100644 --- a/zml/attention.zig +++ b/zml/attention.zig @@ -1,3 +1,4 @@ pub const attention = @import("attention/attention.zig"); pub const paged_attention = @import("attention/paged_attention.zig"); pub const flashattn = @import("attention/flashattn.zig"); +pub const triton = @import("attention/triton.zig"); diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 9f95a5fa..80ea0ee8 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -123,17 +123,6 @@ fn getScalarAttributeAs(comptime T: type, call_frame: *ffi.CallFrame, attribute_ return attribute.get(T); } -fn fixupKvCacheBuffer(buffer: Buffer, layer_index: i64) Buffer { - var shape = buffer.shape; - const layer_stride = shape.computeByteStrides().get(0); - shape = shape.remove(0); - const ptr = @as([*]u8, @ptrCast(buffer.ptr)); - return .{ - .shape = shape, - .ptr = ptr + @as(usize, @intCast(layer_stride * layer_index)), - }; -} - pub fn Wrapper(comptime T: type, run_func: std.meta.DeclEnum(T)) type { return struct { pub fn register(platform: *const zml.Platform) !void { @@ -765,12 +754,9 @@ pub const paged_fa2 = struct { pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { if (call_frame.registeringHook()) return null; - const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[11]); - const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; - const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); - const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); - const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const paged_k = bufferFromFfiBuffer(call_frame.args.buffers()[1]); + const paged_v = bufferFromFfiBuffer(call_frame.args.buffers()[2]); const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[5]); @@ -837,12 +823,9 @@ pub const paged_fa2 = struct { pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { if (call_frame.registeringHook()) return null; - const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[10]); - const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; - const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); - const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); - const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const paged_k = bufferFromFfiBuffer(call_frame.args.buffers()[1]); + const paged_v = bufferFromFfiBuffer(call_frame.args.buffers()[2]); const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[5]); @@ -897,7 +880,7 @@ pub const paged_fa2 = struct { } }; - pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index_: zml.Tensor, opts: AttentionOptions) zml.Tensor { + pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { stdx.debug.assert(q.shape().hasTags(.{ .b, .hg, .hkv, .hd }), "Expected q to have tags .b, .h, .hd", .{}); stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); @@ -907,7 +890,6 @@ pub const paged_fa2 = struct { const num_kv_heads = q.dim(.hkv); const head_dim = q.dim(.hd); const num_heads = num_head_groups * num_kv_heads; - const layer_index = layer_index_.reshape(.{1}).withPartitioning(.{ ._0 = .replicated }); // FIXME: remove unreachable and propagate error correctly. const num_heads_per_shard = @divExact(num_heads, ctx.partitioning.numPartitionsForLogicalAxis(q.shape(), .model) catch unreachable); @@ -960,7 +942,6 @@ pub const paged_fa2 = struct { softmax_lse, softmax_lse_accum, out_accum, - layer_index, }, output_shape, .{ @@ -1037,7 +1018,6 @@ pub const paged_fa2 = struct { softmax_lse_accum_prefill, out_accum_prefill, host_metadata, - layer_index, }, output_shape, .{ @@ -1101,7 +1081,6 @@ pub const paged_fa2 = struct { softmax_lse_decode, softmax_lse_accum_decode, out_accum_decode, - layer_index, }, output_shape_decode, .{ @@ -1379,12 +1358,9 @@ pub const paged_fa3 = struct { pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { if (call_frame.registeringHook()) return null; - const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[11]); - const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; - const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); - const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); - const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const paged_k = bufferFromFfiBuffer(call_frame.args.buffers()[1]); + const paged_v = bufferFromFfiBuffer(call_frame.args.buffers()[2]); const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[5]); @@ -1460,12 +1436,9 @@ pub const paged_fa3 = struct { pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { if (call_frame.registeringHook()) return null; - const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[10]); - const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; - const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); - const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); - const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const paged_k = bufferFromFfiBuffer(call_frame.args.buffers()[1]); + const paged_v = bufferFromFfiBuffer(call_frame.args.buffers()[2]); const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[5]); @@ -1529,7 +1502,7 @@ pub const paged_fa3 = struct { } }; - pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { + pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { stdx.debug.assert(q.shape().hasTags(.{ .b, .hg, .hkv, .hd }), "Expected q to have tags .b, .h, .hd", .{}); stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); @@ -1573,7 +1546,6 @@ pub const paged_fa3 = struct { softmax_lse_accum, out_accum, scheduler_metadata, - layer_index, }, output_shape, .{ @@ -1638,7 +1610,6 @@ pub const paged_fa3 = struct { out_accum, scheduler_metadata_prefill, host_metadata, - layer_index, }, output_shape, .{ @@ -1688,7 +1659,6 @@ pub const paged_fa3 = struct { softmax_lse_accum_decode, out_accum, scheduler_metadata_decode, - layer_index, }, decode_output_shape, .{ diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 10a86939..aa6f0864 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -4,12 +4,14 @@ const stdx = @import("stdx"); const zml = @import("../zml.zig"); const flashattn = @import("flashattn.zig"); +const triton = @import("triton.zig"); const PagedAttention = @This(); pub const Backend = enum { cuda_fa2, cuda_fa3, + triton, pub fn auto(platform: *const zml.Platform) Backend { return switch (platform.target) { @@ -24,6 +26,7 @@ pub const Backend = enum { break :b .cuda_fa2; }, + .rocm => .triton, else => stdx.debug.panic("Paged attention is not supported on {s} yet", .{@tagName(platform.target)}), }; } @@ -32,6 +35,7 @@ pub const Backend = enum { pub const Options = union(Backend) { cuda_fa2: flashattn.paged_fa2.Options, cuda_fa3: flashattn.paged_fa3.Options, + triton: triton.paged.Options, const Args = struct { backend: Backend, @@ -43,6 +47,7 @@ pub const Options = union(Backend) { num_heads: u32, num_kv_heads: u32, head_dim: u32, + max_seqlen_q: u32, }; pub fn fromBackend(args: Args) Options { @@ -100,6 +105,14 @@ pub const Options = union(Backend) { }, }, }, + .triton => .{ + .triton = .{ + .batch_size = args.batch_size, + .max_num_pages = max_num_pages, + .max_seqlen_q = args.max_seqlen_q, + .is_prefill = args.is_prefill, + }, + }, }; } @@ -119,11 +132,13 @@ pub const Options = union(Backend) { pub const Parameters = union(Backend) { cuda_fa2: flashattn.paged_fa2.Parameters, cuda_fa3: flashattn.paged_fa3.Parameters, + triton: triton.paged.Parameters, pub fn init(options_: Options) Parameters { return switch (options_) { .cuda_fa2 => |cuda_fa2_options| .{ .cuda_fa2 = flashattn.paged_fa2.Parameters.init(cuda_fa2_options) }, .cuda_fa3 => |cuda_fa3_options| .{ .cuda_fa3 = flashattn.paged_fa3.Parameters.init(cuda_fa3_options) }, + .triton => |triton_options| .{ .triton = triton.paged.Parameters.init(triton_options) }, }; } @@ -131,6 +146,7 @@ pub const Parameters = union(Backend) { return switch (self) { .cuda_fa2 => |v| .{ .cuda_fa2 = v.options() }, .cuda_fa3 => |v| .{ .cuda_fa3 = v.options() }, + .triton => |v| .{ .triton = v.options() }, }; } @@ -145,11 +161,13 @@ pub const Parameters = union(Backend) { pub const Context = union(Backend) { cuda_fa2: flashattn.paged_fa2.Context, cuda_fa3: flashattn.paged_fa3.Context, + triton: triton.paged.Context, pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { return switch (parameters) { .cuda_fa2 => |cuda_fa2_parameters| .{ .cuda_fa2 = flashattn.paged_fa2.Context.init(cuda_fa2_parameters, num_heads, num_kv_heads, head_dim, page_size) }, .cuda_fa3 => |cuda_fa3_parameters| .{ .cuda_fa3 = flashattn.paged_fa3.Context.init(cuda_fa3_parameters, num_heads, num_kv_heads, head_dim, page_size) }, + .triton => |triton_parameters| .{ .triton = triton.paged.Context.init(triton_parameters, num_heads, num_kv_heads, head_dim, page_size) }, }; } }; @@ -159,11 +177,12 @@ pub const AttentionOptions = struct { sliding_window: i32 = -1, }; -pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { +pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { _ = k; _ = v; return switch (parameters) { - .cuda_fa2 => |cuda_fa2_parameters| flashattn.paged_fa2.pagedAttention(cuda_fa2_parameters, context.cuda_fa2, q, k_cache, v_cache, layer_index, opts), - .cuda_fa3 => |cuda_fa3_parameters| flashattn.paged_fa3.pagedAttention(cuda_fa3_parameters, context.cuda_fa3, q, k_cache, v_cache, layer_index, opts), + .cuda_fa2 => |cuda_fa2_parameters| flashattn.paged_fa2.pagedAttention(cuda_fa2_parameters, context.cuda_fa2, q, k_cache, v_cache, opts), + .cuda_fa3 => |cuda_fa3_parameters| flashattn.paged_fa3.pagedAttention(cuda_fa3_parameters, context.cuda_fa3, q, k_cache, v_cache, opts), + .triton => |triton_parameters| triton.paged.pagedAttention(triton_parameters, context.triton, q, k_cache, v_cache, opts), }; } diff --git a/zml/attention/triton.zig b/zml/attention/triton.zig new file mode 100644 index 00000000..4e8be6fa --- /dev/null +++ b/zml/attention/triton.zig @@ -0,0 +1,664 @@ +const std = @import("std"); + +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); +const stdx = @import("stdx"); + +const zml = @import("../zml.zig"); +const AttentionOptions = @import("paged_attention.zig").AttentionOptions; + +const log = std.log.scoped(.@"zml/attention/triton"); + +fn use2dKernel(head_size: usize, sliding_window: usize, all_decode: bool, max_seqlen_q: usize, max_seqlen_k: usize, target_num_prgms: usize, num_2d_prgms: usize) bool { + _ = head_size; + _ = all_decode; + _ = max_seqlen_q; + return sliding_window > 0 or max_seqlen_k <= 512 or num_2d_prgms > target_num_prgms; +} + +pub const Config2D = struct { + block_m: usize, + block_q: usize, + tile_size: usize, + num_warps: usize, + num_stages: usize, + total_q_blocks: usize, +}; + +fn select2dConfig(options: paged.PagedAttentionOptions) Config2D { + const max_num_stages_2d: usize = if (options.head_dim <= 128) 4 else 2; + + var num_stages_2d: usize, var num_warps: usize, var tile_size: usize = if (!options.all_decode) .{ 1, 2, 64 } else .{ 3, 2, options.block_size }; + + var block_m = options.block_m; + var block_q = options.block_q; + if (options.max_seqlen_q >= 256) { + if (options.head_dim >= 256) { + block_m = 64; + tile_size = 16; + } else { + block_m = 128; + } + num_stages_2d = 1; + num_warps = 4; + } + block_q = block_m / options.numQueriesPerKv(); + const total_q_blocks = options.num_tokens / block_q + options.batch_size; + + return .{ + .block_m = block_m, + .block_q = block_q, + .tile_size = tile_size, + .num_warps = num_warps, + .num_stages = @min(max_num_stages_2d, num_stages_2d), + .total_q_blocks = total_q_blocks, + }; +} + +pub const Config3D = struct { + const AttentionConfig = struct { + tile_size: usize, + num_segments_per_seq: usize, + num_warps: usize, + num_stages: usize, + block_q: usize, + block_m: usize, + total_q_blocks: usize, + }; + const ReduceConfig = struct { + tile_size: usize, + num_segments_per_seq: usize, + num_warps: usize, + num_stages: usize, + block_q: usize, + block_m: usize, + }; + attention: AttentionConfig, + reduce: ReduceConfig, +}; + +fn select3dConfig(options: paged.PagedAttentionOptions) Config3D { + var reduce_num_warps: usize = 2; + const attn_warps: usize = 2; + const tile_size = options.block_size; + + //const MAX_SEGMENTS: usize = @min(128, std.math.divCeil(usize, max_seqlen_k, tile_size)); + var num_segments = std.math.divCeil(usize, options.target_num_prgms, options.num_2d_prgms) catch unreachable; + num_segments = std.math.ceilPowerOfTwoAssert(usize, num_segments); + num_segments = @min(num_segments, 128); + const min_segments: usize = if (tile_size <= 16) 16 else 8; + num_segments = @max(num_segments, min_segments); + if (num_segments == min_segments) { + reduce_num_warps = 1; + } + + return .{ + .attention = .{ + .tile_size = tile_size, + .num_segments_per_seq = num_segments, + .num_warps = attn_warps, + .num_stages = 1, + .block_m = options.block_m, + .block_q = options.block_q, + .total_q_blocks = options.total_q_blocks, + }, + .reduce = .{ + .tile_size = tile_size, + .num_segments_per_seq = num_segments, + .num_warps = reduce_num_warps, + .num_stages = 1, + .block_m = options.block_m, + .block_q = options.block_q, + }, + }; +} + +pub const KernelKind = enum { + kernel_unified_attention_2d_ptr, + kernel_unified_attention_3d_ptr, + reduce_segments_ptr, +}; + +pub const GenerationConfig2D = struct { + pub const Dimensions = struct { + num_tokens: usize, + num_heads: usize, + num_kv_heads: usize, + head_dim: usize, + num_blocks: usize, + block_size: usize, + batch_size: usize, + num_blocks_per_seq: usize, + num_qq_tokens: ?usize = null, + max_seqlen_q: usize, + }; + + pub const FeatureFlags = struct { + use_alibi_slopes: bool, + use_softcap: bool, + use_sinks: bool, + sliding_window: usize, + use_fp8: bool, + all_decode: bool, + }; + + dimensions: Dimensions, + config: Config2D, + feature_flags: FeatureFlags, +}; + +pub const GenerationConfig3D = struct { + pub const Dimensions = struct { + num_tokens: usize, + num_heads: usize, + num_kv_heads: usize, + head_dim: usize, + num_blocks: usize, + block_size: usize, + batch_size: usize, + num_blocks_per_seq: usize, + num_qq_tokens: ?usize = null, + cu_count: usize, + }; + + pub const FeatureFlags = struct { + use_alibi_slopes: bool, + use_softcap: bool, + use_sinks: bool, + sliding_window: usize, + all_decode: bool, + }; + + dimensions: Dimensions, + config: Config3D.AttentionConfig, + feature_flags: FeatureFlags, +}; + +pub const GenerationConfigReduce = struct { + pub const Dimensions = struct { + num_tokens: usize, + num_heads: usize, + num_kv_heads: usize, + head_dim: usize, + block_size: usize, + batch_size: usize, + num_blocks_per_seq: usize, + cu_count: usize, + }; + + pub const FeatureFlags = struct { + use_fp8: bool, + }; + + dimensions: Dimensions, + config: Config3D.ReduceConfig, + feature_flags: FeatureFlags, +}; + +pub const GenerationConfig = union(KernelKind) { + kernel_unified_attention_2d_ptr: GenerationConfig2D, + kernel_unified_attention_3d_ptr: GenerationConfig3D, + reduce_segments_ptr: GenerationConfigReduce, +}; + +fn getGenerateBinPath(allocator: std.mem.Allocator) ![]const u8 { + const runfiles = bazel.runfiles(bazel_builtin.current_repository) catch unreachable; + + var sandbox_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = runfiles.rlocation("zml/zml/attention/triton/sandbox", &sandbox_path_buf) catch unreachable; + + return std.fs.path.join(allocator, &.{ sandbox_path.?, "bin", "generate" }); +} + +fn generateTtir(allocator: std.mem.Allocator, io: std.Io, config: GenerationConfig) ![:0]const u8 { + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + const writer_buffer = try arena.allocator().alloc(u8, 4096); + const reader_buffer = try arena.allocator().alloc(u8, 4096); + + const compilation_context = zml.module.CompilationContext.current(); + const platform = compilation_context.platform; + const triton_runtime = platform.triton_runtime.?; + + var writer = triton_runtime.process.stdin.?.writer(io, writer_buffer); + try writer.interface.print("{f}\n", .{std.json.fmt(config, .{ .emit_null_optional_fields = false })}); + try writer.interface.flush(); + + var reader = triton_runtime.process.stdout.?.reader(io, reader_buffer); + var allocating: std.Io.Writer.Allocating = .init(arena.allocator()); + _ = try reader.interface.streamDelimiter(&allocating.writer, '\n'); + _ = try reader.interface.discardShort(1); + + const response: std.json.Value = try std.json.parseFromSliceLeaky(std.json.Value, arena.allocator(), allocating.written(), .{}); + return if (response.object.get("ok").?.bool) + try allocator.dupeZ(u8, response.object.get("result").?.string) + else + error.GenerationFailed; +} + +fn getCuCount() usize { + const platform = zml.module.CompilationContext.current().platform; + + return @intCast(platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count").?.int64); +} + +pub const paged = struct { + pub const Options = struct { + batch_size: usize, + max_num_pages: usize, + max_seqlen_q: usize, + is_prefill: bool, + + pub fn isPrefill(self: Options) bool { + return self.is_prefill; + } + + pub fn maxNumPages(self: Options) usize { + return self.max_num_pages; + } + }; + + pub const Parameters = struct { + block_table: zml.Tensor, + seq_lens: zml.Tensor, + query_start_len: zml.Tensor, + options_: Options, + + pub fn init(options_: Options) Parameters { + return .{ + .block_table = .init(.{ .b = options_.batch_size, .num_pages_per_seq = options_.max_num_pages }, .i32), + .seq_lens = .init(.{ .b = options_.batch_size }, .i32), + .query_start_len = .init(.{ .b = options_.batch_size + 1 }, .i32), + .options_ = options_, + }; + } + + pub fn allocationSize(self: Parameters) usize { + var allocation_size: usize = 0; + + allocation_size += self.block_table.byteSize(); + allocation_size += self.seq_lens.byteSize(); + allocation_size += self.query_start_len.byteSize(); + + return allocation_size; + } + + pub fn options(self: Parameters) Options { + return self.options_; + } + }; + + pub const Context = struct { + is_prefill: bool, + max_seqlen_q: usize, + + pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { + _ = num_heads; + _ = num_kv_heads; + _ = head_dim; + _ = page_size; + return .{ .is_prefill = parameters.options_.is_prefill, .max_seqlen_q = parameters.options_.max_seqlen_q }; + } + }; + + pub const PagedAttentionOptions = struct { + cu_count: usize, + all_decode: bool, + num_tokens: usize, + num_heads: usize, + num_kv_heads: usize, + head_dim: usize, + batch_size: usize, + block_size: usize, + num_blocks: usize, + max_num_block_per_seq: usize, + sliding_window: usize, + block_m: usize, + block_q: usize, + total_q_blocks: usize, + target_num_prgms: usize, + num_2d_prgms: usize, + max_seqlen_q: usize, + + pub fn numQueriesPerKv(self: PagedAttentionOptions) usize { + return self.num_heads / self.num_kv_heads; + } + + pub fn maxSeqLenK(self: PagedAttentionOptions) usize { + return self.max_num_block_per_seq * self.block_size; + } + }; + + pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { + const output = zml.ops.manualComputation( + .{ + q, + k_cache, + v_cache, + parameters.block_table, + parameters.seq_lens, + parameters.query_start_len, + }, + q.shape(), + .{ + .opts = opts, + .options = parameters.options_, + .context = context, + }, + (struct { + fn body(ctx_: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, _: zml.Shape) zml.Tensor { + const q_ = sharded_inputs[0]; + const k_cache_ = sharded_inputs[1]; + const v_cache_ = sharded_inputs[2]; + const parameters_: Parameters = .{ .block_table = sharded_inputs[3], .seq_lens = sharded_inputs[4], .query_start_len = sharded_inputs[5], .options_ = ctx_.options }; + + const cu_count = getCuCount(); + const num_heads: usize = @intCast(q_.dim(.hkv) * q_.dim(.hg)); + const num_kv_heads: usize = @intCast(k_cache_.dim(.hkv)); + const num_queries_per_kv: usize = num_heads / num_kv_heads; + const block_m: usize = if (num_queries_per_kv <= 16) 16 else std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv); + const block_q: usize = block_m / num_queries_per_kv; + const num_tokens: usize = @intCast(q_.dim(.b)); + const num_seqs: usize = @intCast(parameters_.block_table.dim(.b)); + const total_q_blocks: usize = num_tokens / block_q + num_seqs; + const target_num_prgms: usize = cu_count * 4; + const num_2d_prgms: usize = total_q_blocks * num_kv_heads; + + const paged_attention_opts: PagedAttentionOptions = .{ + .cu_count = getCuCount(), + .all_decode = !ctx_.options.is_prefill, + .num_tokens = num_tokens, + .num_heads = num_heads, + .num_kv_heads = num_kv_heads, + .head_dim = @intCast(q_.dim(.hd)), + .batch_size = @intCast(parameters_.block_table.dim(.b)), + .block_size = @intCast(k_cache_.dim(.k_chunk)), + .num_blocks = @intCast(k_cache_.dim(.page)), + .max_num_block_per_seq = @intCast(parameters_.block_table.dim(.num_pages_per_seq)), + .sliding_window = if (ctx_.opts.sliding_window < 0) 0 else @intCast(ctx_.opts.sliding_window), + .block_m = block_m, + .block_q = block_q, + .total_q_blocks = total_q_blocks, + .target_num_prgms = target_num_prgms, + .num_2d_prgms = num_2d_prgms, + .max_seqlen_q = ctx_.options.max_seqlen_q, + }; + + const use_2d_kernel = use2dKernel( + paged_attention_opts.head_dim, + paged_attention_opts.sliding_window, + paged_attention_opts.all_decode, + paged_attention_opts.max_seqlen_q, + paged_attention_opts.maxSeqLenK(), + paged_attention_opts.target_num_prgms, + paged_attention_opts.num_2d_prgms, + ); + const output = if (use_2d_kernel) + pagedAttention2d(parameters_, ctx_.context, q_, k_cache_, v_cache_, ctx_.opts, paged_attention_opts) + else + pagedAttention3d(parameters_, ctx_.context, q_, k_cache_, v_cache_, ctx_.opts, paged_attention_opts); + + return output; + } + }).body, + ); + + return output; + } + + pub fn pagedAttention2d(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, paged_attention_opts: PagedAttentionOptions) zml.Tensor { + _ = context; + _ = opts; + const config = select2dConfig(paged_attention_opts); + const generation_config: GenerationConfig = .{ + .kernel_unified_attention_2d_ptr = .{ + .dimensions = .{ + .batch_size = paged_attention_opts.batch_size, + .block_size = paged_attention_opts.block_size, + .head_dim = paged_attention_opts.head_dim, + .num_blocks = paged_attention_opts.num_blocks, + .num_blocks_per_seq = paged_attention_opts.max_num_block_per_seq, + .num_heads = paged_attention_opts.num_heads, + .num_kv_heads = paged_attention_opts.num_kv_heads, + .num_tokens = paged_attention_opts.num_tokens, + .max_seqlen_q = parameters.options_.max_seqlen_q, + }, + .config = config, + .feature_flags = .{ + .all_decode = paged_attention_opts.all_decode, + .sliding_window = paged_attention_opts.sliding_window, + .use_alibi_slopes = false, + .use_fp8 = false, + .use_sinks = false, + .use_softcap = false, + }, + }, + }; + + log.debug("pagedAttention2d config: {any}", .{generation_config}); + + const io = zml.module.CompilationContext.current().io; + const allocator = zml.module.CompilationContext.current().allocator; + const ttir = generateTtir(allocator, io, generation_config) catch unreachable; + defer allocator.free(ttir); + + const dummy = zml.Tensor.constant(zml.DataType.i8.zero()); + const block_table_strides = parameters.block_table.shape().computeElementStrides().constSlice(); + const block_table_strides_ptr = zml.Tensor.constant(zml.DataType.i64.constant(block_table_strides[0])); + const q_shape = q.shape().mergeAxes(.{ .h = .{ .hkv, .hg } }); + const q_strides = q_shape.computeElementStrides().constSlice(); + const q_strides_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(q_strides[0])); + const q_strides_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(q_strides[1])); + const k_strides = k_cache.shape().computeElementStrides().constSlice(); + const k_strides_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(k_strides[0])); + const k_strides_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(k_strides[1])); + const k_strides_2_ptr = zml.Tensor.constant(zml.DataType.i64.constant(k_strides[2])); + const v_strides = v_cache.shape().computeElementStrides().constSlice(); + const v_strides_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(v_strides[0])); + const v_strides_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(v_strides[1])); + const v_strides_2_ptr = zml.Tensor.constant(zml.DataType.i64.constant(v_strides[2])); + const num_seqs_ptr = zml.Tensor.constant(zml.DataType.i32.constant(parameters.block_table.dim(0))); + const scale: f32 = @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))); + const scale_ptr = zml.Tensor.constant(zml.DataType.f32.constant(scale)); + + const output = zml.ops.triton(.{ + q, + k_cache, + v_cache, + dummy, // sink_ptr + parameters.block_table, + parameters.seq_lens, + dummy, // alibi_slopes_ptr + dummy, // qq_bias_ptr + scale_ptr, + dummy, // k_scale_ptr + dummy, // v_scale_ptr + dummy, // out_scale_ptr + dummy, // softcap_ptr + block_table_strides_ptr, + q_strides_0_ptr, + q_strides_1_ptr, + q_strides_0_ptr, + q_strides_1_ptr, + dummy, // qq_bias_stride_0_ptr + k_strides_0_ptr, + k_strides_1_ptr, + k_strides_2_ptr, + v_strides_0_ptr, + v_strides_1_ptr, + v_strides_2_ptr, + parameters.query_start_len, + num_seqs_ptr, + }, .{q.shape()}, .{ + .debug = false, + .name = "kernel_unified_attention_2d_ptr", + .ir = ttir, + .grid = .{ @intCast(paged_attention_opts.num_kv_heads), @intCast(config.total_q_blocks), 1 }, + .num_stages = @intCast(config.num_stages), + .num_warps = @intCast(config.num_warps), + }); + return output[0]; + } + + pub fn pagedAttention3d(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, paged_attention_opts: PagedAttentionOptions) zml.Tensor { + _ = context; + _ = opts; + + const config = select3dConfig(paged_attention_opts); + + const attn_generation_config: GenerationConfig = .{ + .kernel_unified_attention_3d_ptr = .{ + .dimensions = .{ + .batch_size = paged_attention_opts.batch_size, + .block_size = paged_attention_opts.block_size, + .head_dim = paged_attention_opts.head_dim, + .num_blocks = paged_attention_opts.num_blocks, + .num_blocks_per_seq = paged_attention_opts.max_num_block_per_seq, + .num_heads = paged_attention_opts.num_heads, + .num_kv_heads = paged_attention_opts.num_kv_heads, + .num_tokens = paged_attention_opts.num_tokens, + .cu_count = paged_attention_opts.cu_count, + }, + .config = config.attention, + .feature_flags = .{ + .all_decode = paged_attention_opts.all_decode, + .sliding_window = paged_attention_opts.sliding_window, + .use_alibi_slopes = false, + .use_sinks = false, + .use_softcap = false, + }, + }, + }; + + log.debug("pagedAttention3d attention config: {any}", .{attn_generation_config}); + + const io = zml.module.CompilationContext.current().io; + const allocator = zml.module.CompilationContext.current().allocator; + const attn_ttir = generateTtir(allocator, io, attn_generation_config) catch unreachable; + defer allocator.free(attn_ttir); + + const reduce_generation_config: GenerationConfig = .{ + .reduce_segments_ptr = .{ + .dimensions = .{ + .num_tokens = paged_attention_opts.num_tokens, + .num_heads = paged_attention_opts.num_heads, + .num_kv_heads = paged_attention_opts.num_kv_heads, + .head_dim = paged_attention_opts.head_dim, + .block_size = paged_attention_opts.block_size, + .batch_size = paged_attention_opts.batch_size, + .num_blocks_per_seq = paged_attention_opts.max_num_block_per_seq, + .cu_count = paged_attention_opts.cu_count, + }, + .config = config.reduce, + .feature_flags = .{ + .use_fp8 = false, + }, + }, + }; + log.debug("pagedAttention3d reduce config: {any}", .{reduce_generation_config}); + const reduce_ttir = generateTtir(std.heap.c_allocator, io, reduce_generation_config) catch unreachable; + defer std.heap.c_allocator.free(reduce_ttir); + + const dummy = zml.Tensor.constant(zml.DataType.i8.zero()); + const block_table_strides = parameters.block_table.shape().computeElementStrides().constSlice(); + const block_table_strides_ptr = zml.Tensor.constant(zml.DataType.i64.constant(block_table_strides[0])); + const q_shape = q.shape().mergeAxes(.{ .h = .{ .hkv, .hg } }); + const q_strides = q_shape.computeElementStrides().constSlice(); + const q_strides_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(q_strides[0])); + const q_strides_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(q_strides[1])); + const k_strides = k_cache.shape().computeElementStrides().constSlice(); + const k_strides_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(k_strides[0])); + const k_strides_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(k_strides[1])); + const k_strides_2_ptr = zml.Tensor.constant(zml.DataType.i64.constant(k_strides[2])); + const v_strides = v_cache.shape().computeElementStrides().constSlice(); + const v_strides_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(v_strides[0])); + const v_strides_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(v_strides[1])); + const v_strides_2_ptr = zml.Tensor.constant(zml.DataType.i64.constant(v_strides[2])); + const num_seqs_ptr = zml.Tensor.constant(zml.DataType.i32.constant(parameters.block_table.dim(0))); + const scale: f32 = @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))); + const scale_ptr = zml.Tensor.constant(zml.DataType.f32.constant(scale)); + + const attn_grid: [3]i32 = .{ @intCast(config.attention.total_q_blocks), @intCast(paged_attention_opts.num_kv_heads), @intCast(config.attention.num_segments_per_seq) }; + const attn_output = zml.ops.triton(.{ + q, + k_cache, + v_cache, + dummy, // sink_ptr + parameters.block_table, + parameters.seq_lens, + dummy, // alibi_slopes_ptr + dummy, // qq_bias_ptr + scale_ptr, + dummy, // k_scale_ptr + dummy, // v_scale_ptr + dummy, // softcap_ptr + block_table_strides_ptr, + q_strides_0_ptr, + q_strides_1_ptr, + dummy, // qq_bias_stride_0_ptr + k_strides_0_ptr, + k_strides_1_ptr, + k_strides_2_ptr, + v_strides_0_ptr, + v_strides_1_ptr, + v_strides_2_ptr, + parameters.query_start_len, + num_seqs_ptr, + }, .{ + zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq, std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim) }, .f32), + zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq }, .f32), + zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq }, .f32), + }, .{ + .debug = false, + .name = "kernel_unified_attention_3d_ptr", + .ir = attn_ttir, + .grid = attn_grid, + .num_stages = @intCast(config.attention.num_stages), + .num_warps = @intCast(config.attention.num_warps), + }); + + const output = zml.ops.triton(.{ + attn_output[0], + attn_output[1], + attn_output[2], + parameters.seq_lens, + num_seqs_ptr, + dummy, // out_scale_inv_ptr + q_strides_0_ptr, + q_strides_1_ptr, + block_table_strides_ptr, + parameters.query_start_len, + }, .{ + q.shape(), + }, .{ + .debug = false, + .name = "reduce_segments_ptr", + .ir = reduce_ttir, + .grid = .{ @intCast(paged_attention_opts.num_tokens), @intCast(paged_attention_opts.num_heads), 1 }, + .num_stages = @intCast(config.reduce.num_stages), + .num_warps = @intCast(config.reduce.num_warps), + }); + + return output[0]; + } +}; + +pub const Runtime = struct { + process: std.process.Child, + + pub fn init(allocator: std.mem.Allocator, io: std.Io) !Runtime { + const path = try getGenerateBinPath(allocator); + defer allocator.free(path); + + const process = try std.process.spawn(io, .{ .argv = &.{path}, .stdin = .pipe, .stdout = .pipe }); + errdefer process.kill(io); + + return .{ .process = process }; + } + + pub fn deinit(self: *Runtime, io: std.Io) void { + // NOTE(Corendos): This is not portable, but I couldn't find a better way with the current std.Io. + _ = std.c.kill(self.process.id.?, .INT); + _ = self.process.wait(io) catch unreachable; + } +}; diff --git a/zml/attention/triton/BUILD.bazel b/zml/attention/triton/BUILD.bazel new file mode 100644 index 00000000..a8f6f59a --- /dev/null +++ b/zml/attention/triton/BUILD.bazel @@ -0,0 +1,110 @@ +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_python//python:py_library.bzl", "py_library") +load("@rules_python//python/uv:lock.bzl", uv_lock = "lock") +load("@rules_zig//zig:defs.bzl", "zig_binary") +load("@zml//bazel:runfiles.bzl", "runfiles_to_default") + +uv_lock( + name = "requirements", + srcs = ["requirements.in"], + out = "requirements.lock.txt", + args = [ + "--emit-index-url", + "--emit-find-links", + "--index-strategy=unsafe-best-match", + "--upgrade", + "--python-platform=x86_64-manylinux_2_37", + ], + python_version = "3.12", + tags = ["manual"], +) + +py_library( + name = "fake_plugin", + srcs = [ + "fake_plugin/__init__.py", + "fake_plugin/compiler.py", + "fake_plugin/driver.py", + ], + deps = ["@triton_py_deps//triton"], +) + +py_library( + name = "triton_kernels", + srcs = [ + "triton_kernels/__init__.py", + "triton_kernels/unified_attention.py", + ], + deps = ["@triton_py_deps//triton"], +) + +py_binary( + name = "generate_py", + srcs = ["generate.py"], + main = "generate.py", + python_version = "3.12", + deps = [ + ":fake_plugin", + ":triton_kernels", + "@triton_py_deps//triton", + ], +) + +cc_library( + name = "libpython", + hdrs = ["libpython.h"], + deps = [ + "@python_3_12//:libpython", + "@python_3_12//:python_headers", + ], +) + +zig_binary( + name = "generate", + data = [":generate_py"], + linkopts = ["-Wl,-rpath,$$ORIGIN/../lib"], + main = "generate.zig", + tags = ["manual"], + deps = [":libpython"], +) + +runfiles_to_default( + name = "generate_files", + visibility = ["//visibility:public"], + deps = [ + ":generate", + ":generate_py", + ], +) + +copy_to_directory( + name = "sandbox", + srcs = [ + ":generate_files", + ], + add_directory_to_runfiles = False, + allow_overwrites = True, + exclude_srcs_patterns = [ + "**/include/**", + "**/_solib_*/**", + ], + hardlink = "auto", + include_external_repositories = ["**"], + include_srcs_patterns = [ + "generate", + "generate.py", + "fake_plugin/*", + "triton_kernels/*", + "**/lib**", + "lib/python*/**", + "site-packages/**", + ], + replace_prefixes = { + "fake_plugin": "site-packages/fake_plugin", + "triton_kernels": "site-packages/triton_kernels", + "generate.py": "site-packages/generate.py", + "generate": "bin/generate", + }, + visibility = ["//visibility:public"], +) diff --git a/zml/attention/triton/fake_plugin/__init__.py b/zml/attention/triton/fake_plugin/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/zml/attention/triton/fake_plugin/compiler.py b/zml/attention/triton/fake_plugin/compiler.py new file mode 100755 index 00000000..2e0a6c36 --- /dev/null +++ b/zml/attention/triton/fake_plugin/compiler.py @@ -0,0 +1,97 @@ +from dataclasses import dataclass +from types import ModuleType +from typing import Any, Dict + +from triton.backends.compiler import BaseBackend, GPUTarget, Language + + +@dataclass(frozen=True) +class FakeOptions: + num_warps: int = 4 + num_ctas: int = 1 + num_stages: int = 2 + enable_fp_fusion: bool = True + launch_cooperative_grid: bool = False + extern_libs: tuple = () + sanitize_overflow: bool = True + debug: bool = False + instrumentation_mode: str = "" + default_dot_input_precision: str = "tf32" + allowed_dot_input_precisions: tuple = ("tf32", "tf32x3", "ieee", "bf16x3", "bf16x6") + max_num_imprecise_acc_default: int = 0 + backend_name: str = "cpu" + + def hash(self) -> str: + return "fake-options-v1" + + +class Backend(BaseBackend): + def __init__(self, target: GPUTarget) -> None: + super().__init__(target) + self.binary_ext = "fakebin" + + @staticmethod + def supports_target(target: GPUTarget): + return target.backend == "cpu" + + def hash(self) -> str: + return "fake-backend-v1" + + def parse_options(self, options: dict) -> object: + args = { + k: options[k] for k in FakeOptions.__dataclass_fields__.keys() if k in options and options[k] is not None + } + return FakeOptions(**args) + + def add_stages(self, stages: dict, options: object, language: Language = Language.TRITON) -> None: + del options, language + stages["ttir"] = lambda src, metadata: self._keep_ttir(src, metadata) + stages[self.binary_ext] = lambda src, metadata: self._emit_fake_binary(src, metadata) + + def _keep_ttir(self, src, metadata): + metadata.setdefault("shared", 0) + metadata.setdefault("name", "fake_kernel") + return src + + def _emit_fake_binary(self, src, metadata) -> bytes: + del src + metadata.setdefault("shared", 0) + metadata.setdefault("name", "fake_kernel") + return b"fake-binary" + + def pack_metadata(self, metadata): + return ( + getattr(metadata, "num_warps", 1), + getattr(metadata, "num_ctas", 1), + getattr(metadata, "shared", 0), + ) + + def get_codegen_implementation(self, options: Any): + del options + return {"min_dot_size": lambda lhs_type, rhs_type: (1, 1, 1)} + + def load_dialects(self, context): + del context + + def get_module_map(self) -> Dict[str, ModuleType]: + return {} + + @staticmethod + def parse_attr(desc): + assert isinstance(desc, str) + ret = [] + if "D" in desc: + ret += [["tt.divisibility", 16]] + return ret + + @staticmethod + def get_int_specialization(arg, **kwargs): + if arg % 16 == 0 and kwargs.get("align", False): + return "D" + return "" + + @staticmethod + def get_tensor_specialization(arg, **kwargs): + if arg.data_ptr() % 16 == 0 and kwargs.get("align", False): + return "D" + return "" diff --git a/zml/attention/triton/fake_plugin/driver.py b/zml/attention/triton/fake_plugin/driver.py new file mode 100755 index 00000000..44d339dd --- /dev/null +++ b/zml/attention/triton/fake_plugin/driver.py @@ -0,0 +1,55 @@ +from triton.backends.compiler import GPUTarget +from triton.backends.driver import Benchmarker, DriverBase + + +_TYPE_MAP = { + "i1": "int8_t", + "i8": "int8_t", + "i16": "int16_t", + "i32": "int32_t", + "i64": "int64_t", + "u1": "uint8_t", + "u8": "uint8_t", + "u16": "uint16_t", + "u32": "uint32_t", + "u64": "uint64_t", + "fp16": "double", + "bf16": "double", + "fp32": "double", + "f32": "double", + "fp64": "double", +} + + +class Driver(DriverBase): + @classmethod + def is_active(cls): + return True + + def map_python_to_cpp_type(self, ty: str) -> str: + if ty.startswith("*"): + return "void*" + return _TYPE_MAP[ty] + + def get_current_target(self): + return GPUTarget("cpu", "generic", 32) + + def get_current_device(self): + return "cpu" + + def get_current_stream(self, device): + del device + return None + + def get_active_torch_device(self): + return "cpu" + + def get_benchmarker(self) -> Benchmarker: + def _benchmarker(kernel_call, *, quantiles, **kwargs): + del kernel_call, kwargs + return [0.0 for _ in quantiles] + + return _benchmarker + + def __init__(self) -> None: + pass diff --git a/zml/attention/triton/generate.py b/zml/attention/triton/generate.py new file mode 100755 index 00000000..06b9cc55 --- /dev/null +++ b/zml/attention/triton/generate.py @@ -0,0 +1,365 @@ +import signal +import sys +import json +import math +import triton + +import triton.backends as triton_backends +from triton.backends import Backend as BackendRegistration +from triton.runtime.driver import driver as runtime_driver + +from fake_plugin.compiler import Backend as FakeCompilerBackend +from fake_plugin.driver import Driver as FakeDriver +from triton_kernels.unified_attention import ( + kernel_unified_attention_2d_ptr, + kernel_unified_attention_3d_ptr, + reduce_segments_ptr, +) + + +class FakeTensor: + def __init__(self, dtype: str, shape, strides=None): + self.dtype = dtype + self.shape = tuple(shape) + self._strides = tuple(strides) if strides is not None else contiguous_strides(self.shape) + + def stride(self, dim: int) -> int: + return self._strides[dim] + + @staticmethod + def data_ptr() -> int: + return 0 + + +def contiguous_strides(shape: tuple[int, ...]) -> tuple[int, ...]: + stride = 1 + out = [] + for size in reversed(shape): + out.append(stride) + stride *= int(size) + return tuple(reversed(out)) + + +def next_power_of_2(x: int) -> int: + return 1 if x <= 1 else 1 << (x - 1).bit_length() + + +def select_3d_config( + block_size: int, + max_seqlen_k: int, + target_num_prgms: int, + num_2d_prgms: int, +) -> tuple[dict, dict]: + reduce_num_warps = 2 + attn_warps = 2 + tile_size = block_size + + num_segments = math.ceil(target_num_prgms / max(1, num_2d_prgms)) + num_segments = next_power_of_2(max(1, num_segments)) + num_segments = min(num_segments, 128) + min_segments = 16 if tile_size <= 16 else 8 + num_segments = max(num_segments, min_segments) + + if num_segments == min_segments: + reduce_num_warps = 1 + + attn_config = { + "TILE_SIZE": tile_size, + "NUM_SEGMENTS_PER_SEQ": num_segments, + "num_warps": attn_warps, + "num_stages": 1, + } + reduce_config = { + "TILE_SIZE": tile_size, + "NUM_SEGMENTS_PER_SEQ": num_segments, + "num_warps": reduce_num_warps, + "num_stages": 1, + } + return attn_config, reduce_config + + +def register_fake_backend() -> None: + triton_backends.backends["mybackend_runtime"] = BackendRegistration( + compiler=FakeCompilerBackend, + driver=FakeDriver, + ) + runtime_driver.set_active(FakeDriver()) + + +def compile_2d_ptr(cfg: dict) -> str: + dims = cfg["dimensions"] + + num_tokens = dims["num_tokens"] + num_heads = dims["num_heads"] + num_kv_heads = dims["num_kv_heads"] + head_dim = dims["head_dim"] + padded_head_dim = next_power_of_2(head_dim) + num_blocks = dims["num_blocks"] + block_size = dims["block_size"] + batch_size = dims["batch_size"] + num_blocks_per_seq = dims["num_blocks_per_seq"] + num_qq_tokens = dims.get("num_qq_tokens", 1) + max_seqlen_q = dims["max_seqlen_q"] + + num_queries_per_kv = num_heads // num_kv_heads + + q_shape = (num_tokens, num_heads, head_dim) + kv_shape = (num_blocks, block_size, num_kv_heads, head_dim) + + flags = cfg["feature_flags"] + use_alibi_slopes = flags["use_alibi_slopes"] + use_softcap = flags["use_softcap"] + use_sinks = flags["use_sinks"] + use_fp8 = flags["use_fp8"] + use_qq_bias = "num_qq_tokens" in dims + all_decode = flags["all_decode"] + sliding_window = flags["sliding_window"] + + cfg_2d = cfg["config"] + block_q = cfg_2d["block_q"] + block_m = cfg_2d["block_m"] + tile_size = cfg_2d["tile_size"] + num_warps = cfg_2d["num_warps"] + num_stages = cfg_2d["num_stages"] + total_q_blocks = cfg_2d["total_q_blocks"] + + kwargs = { + "output_ptr": FakeTensor("bf16", q_shape), + "query_ptr": FakeTensor("bf16", q_shape), + "key_cache_ptr": FakeTensor("bf16", kv_shape), + "value_cache_ptr": FakeTensor("bf16", kv_shape), + "sink_ptr": FakeTensor("fp32", (num_heads,)), + "block_tables_ptr": FakeTensor("i32", (batch_size, num_blocks_per_seq)), + "seq_lens_ptr": FakeTensor("i32", (batch_size,)), + "alibi_slopes_ptr": FakeTensor("fp32", (num_heads,)), + "qq_bias_ptr": FakeTensor("fp32", (num_qq_tokens, num_qq_tokens)), + "scale_ptr": FakeTensor("fp32", (1,)), + "k_scale_ptr": FakeTensor("fp32", (1,)), + "v_scale_ptr": FakeTensor("fp32", (1,)), + "out_scale_ptr": FakeTensor("fp32", (1,)), + "softcap_ptr": FakeTensor("fp32", (1,)), + "block_table_stride_ptr": FakeTensor("i64", (1,)), + "query_stride_0_ptr": FakeTensor("i64", (1,)), + "query_stride_1_ptr": FakeTensor("i64", (1,)), + "output_stride_0_ptr": FakeTensor("i64", (1,)), + "output_stride_1_ptr": FakeTensor("i64", (1,)), + "qq_bias_stride_0_ptr": FakeTensor("i64", (1,)), + "stride_k_cache_0_ptr": FakeTensor("i64", (1,)), + "stride_k_cache_1_ptr": FakeTensor("i64", (1,)), + "stride_k_cache_2_ptr": FakeTensor("i64", (1,)), + "stride_v_cache_0_ptr": FakeTensor("i64", (1,)), + "stride_v_cache_1_ptr": FakeTensor("i64", (1,)), + "stride_v_cache_2_ptr": FakeTensor("i64", (1,)), + "query_start_len_ptr": FakeTensor("i32", (batch_size + 1,)), + "num_seqs_ptr": FakeTensor("i32", (1,)), + "num_query_heads": num_heads, + "num_queries_per_kv": num_queries_per_kv, + "BLOCK_SIZE": block_size, + "TILE_SIZE": tile_size, + "HEAD_SIZE": head_dim, + "HEAD_SIZE_PADDED": padded_head_dim, + "USE_ALIBI_SLOPES": use_alibi_slopes, + "USE_QQ_BIAS": use_qq_bias, + "USE_SOFTCAP": use_softcap, + "USE_SINKS": use_sinks, + "SLIDING_WINDOW": sliding_window, + "stride_k_cache_3": 1, + "stride_v_cache_3": 1, + "BLOCK_Q": block_q, + "BLOCK_M": block_m, + "USE_FP8": use_fp8, + "ALL_DECODE": all_decode, + # runtime options + "num_warps": num_warps, + "num_stages": num_stages, + } + + kernel = kernel_unified_attention_2d_ptr.warmup(grid=(num_kv_heads, total_q_blocks), **kwargs) + return kernel.asm["ttir"] + + +def compile_3d_ptr(cfg: dict) -> str: + dims = cfg["dimensions"] + + num_tokens = dims["num_tokens"] + num_heads = dims["num_heads"] + num_kv_heads = dims["num_kv_heads"] + head_dim = dims["head_dim"] + padded_head_dim = next_power_of_2(head_dim) + num_blocks = dims["num_blocks"] + block_size = dims["block_size"] + batch_size = dims["batch_size"] + num_blocks_per_seq = dims["num_blocks_per_seq"] + num_qq_tokens = dims.get("num_qq_tokens", 1) + cu_count = dims["cu_count"] + num_queries_per_kv = num_heads // num_kv_heads + + # unused + max_seqlen_k = 0 + attn_cfg = cfg["config"] + + block_m = attn_cfg["block_m"] + block_q = attn_cfg["block_q"] + tile_size = attn_cfg["tile_size"] + num_warps = attn_cfg["num_warps"] + num_stages = attn_cfg["num_stages"] + num_segments_per_seq = attn_cfg["num_segments_per_seq"] + total_q_blocks = attn_cfg["total_q_blocks"] + + q_shape = (num_tokens, num_heads, head_dim) + kv_shape = (num_blocks, block_size, num_kv_heads, head_dim) + segm_out_shape = (num_tokens, num_heads, num_segments_per_seq, triton.next_power_of_2(head_dim)) + segm_stat_shape = (num_tokens, num_heads, num_segments_per_seq) + + flags = cfg["feature_flags"] + use_alibi_slopes = flags["use_alibi_slopes"] + use_softcap = flags["use_softcap"] + use_sinks = flags["use_sinks"] + all_decode = flags["all_decode"] + sliding_window = flags["sliding_window"] + use_qq_bias = "num_qq_tokens" in dims + + kwargs = { + "query_ptr": FakeTensor("bf16", q_shape), + "key_cache_ptr": FakeTensor("bf16", kv_shape), + "value_cache_ptr": FakeTensor("bf16", kv_shape), + "sink_ptr": FakeTensor("fp32", (num_heads,)), + "block_tables_ptr": FakeTensor("i32", (batch_size, num_blocks_per_seq)), + "seq_lens_ptr": FakeTensor("i32", (batch_size,)), + "alibi_slopes_ptr": FakeTensor("fp32", (num_heads,)), + "qq_bias_ptr": FakeTensor("fp32", (num_qq_tokens, num_qq_tokens)), + "scale_ptr": FakeTensor("fp32", (1,)), + "k_scale_ptr": FakeTensor("fp32", (1,)), + "v_scale_ptr": FakeTensor("fp32", (1,)), + "softcap_ptr": FakeTensor("fp32", (1,)), + "num_query_heads": num_heads, + "num_queries_per_kv": num_queries_per_kv, + "block_table_stride_ptr": FakeTensor("i64", (1,)), + "query_stride_0_ptr": FakeTensor("i64", (1,)), + "query_stride_1_ptr": FakeTensor("i64", (1,)), + "qq_bias_stride_0_ptr": FakeTensor("i64", (1,)), + "BLOCK_SIZE": block_size, + "TILE_SIZE": tile_size, + "HEAD_SIZE": head_dim, + "HEAD_SIZE_PADDED": padded_head_dim, + "USE_ALIBI_SLOPES": use_alibi_slopes, + "USE_QQ_BIAS": use_qq_bias, + "USE_SOFTCAP": use_softcap, + "USE_SINKS": use_sinks, + "SLIDING_WINDOW": sliding_window, + "stride_k_cache_0_ptr": FakeTensor("i64", (1,)), + "stride_k_cache_1_ptr": FakeTensor("i64", (1,)), + "stride_k_cache_2_ptr": FakeTensor("i64", (1,)), + "stride_k_cache_3": 1, + "stride_v_cache_0_ptr": FakeTensor("i64", (1,)), + "stride_v_cache_1_ptr": FakeTensor("i64", (1,)), + "stride_v_cache_2_ptr": FakeTensor("i64", (1,)), + "stride_v_cache_3": 1, + "query_start_len_ptr": FakeTensor("i32", (batch_size + 1,)), + "BLOCK_Q": block_q, + "num_seqs_ptr": FakeTensor("i32", (1,)), + "BLOCK_M": block_m, + "NUM_SEGMENTS_PER_SEQ": num_segments_per_seq, + "ALL_DECODE": all_decode, + "segm_output_ptr": FakeTensor("fp32", segm_out_shape), + "segm_max_ptr": FakeTensor("fp32", segm_stat_shape), + "segm_expsum_ptr": FakeTensor("fp32", segm_stat_shape), + "num_warps": num_warps, + "num_stages": num_stages, + } + + kernel = kernel_unified_attention_3d_ptr.warmup( + grid=(total_q_blocks, num_kv_heads, num_segments_per_seq), + **kwargs, + ) + return kernel.asm["ttir"] + + +def compile_reduce_ptr(cfg: dict) -> str: + dims = cfg["dimensions"] + + num_tokens = dims["num_tokens"] + num_heads = dims["num_heads"] + head_dim = dims["head_dim"] + padded_head_dim = next_power_of_2(head_dim) + batch_size = dims["batch_size"] + block_size = dims["block_size"] + num_kv_heads = dims.get("num_kv_heads", 1) + cu_count = dims["cu_count"] + num_queries_per_kv = max(1, num_heads // num_kv_heads) + + # unused + max_seqlen_k = 0 + + + reduce_cfg = cfg["config"] + num_segments_per_seq = reduce_cfg["num_segments_per_seq"] + tile_size = reduce_cfg["tile_size"] + num_warps = reduce_cfg["num_warps"] + num_stages = reduce_cfg["num_stages"] + block_q = reduce_cfg["block_q"] + + out_shape = (num_tokens, num_heads, head_dim) + segm_out_shape = (num_tokens, num_heads, num_segments_per_seq, head_dim) + segm_stat_shape = (num_tokens, num_heads, num_segments_per_seq) + out_strides = contiguous_strides(out_shape) + + use_fp8 = cfg["feature_flags"]["use_fp8"] + + kwargs = { + "segm_output_ptr": FakeTensor("fp32", segm_out_shape), + "segm_max_ptr": FakeTensor("fp32", segm_stat_shape), + "segm_expsum_ptr": FakeTensor("fp32", segm_stat_shape), + "seq_lens_ptr": FakeTensor("i32", (batch_size,)), + "num_seqs_ptr": FakeTensor("i32", (1,)), + "num_query_heads": num_heads, + "out_scale_inv_ptr": FakeTensor("fp32", (1,)), + "output_stride_0_ptr": FakeTensor("i64", (1,)), + "output_stride_1_ptr": FakeTensor("i64", (1,)), + "block_table_stride_ptr": FakeTensor("i64", (1,)), + "TILE_SIZE": tile_size, + "HEAD_SIZE": head_dim, + "HEAD_SIZE_PADDED": padded_head_dim, + "query_start_len_ptr": FakeTensor("i32", (batch_size + 1,)), + "BLOCK_Q": block_q, + "NUM_SEGMENTS_PER_SEQ": num_segments_per_seq, + "USE_FP8": use_fp8, + "output_ptr": FakeTensor("bf16", out_shape), + "num_warps": num_warps, + "num_stages": num_stages, + } + + kernel = reduce_segments_ptr.warmup(grid=(num_tokens, num_heads), **kwargs) + return kernel.asm["ttir"] + + +def handle_sigint(signum, frame): + sys.exit(0) + + +def main() -> None: + register_fake_backend() + signal.signal(signal.SIGINT, handle_sigint) + + for raw_line in sys.stdin: + line = raw_line.strip() + if not line: + continue + + try: + cfg = json.loads(line) + if not isinstance(cfg, dict): + raise ValueError("request must be a JSON object") + + if "kernel_unified_attention_2d_ptr" in cfg: + ttir = compile_2d_ptr(cfg["kernel_unified_attention_2d_ptr"]) + elif "kernel_unified_attention_3d_ptr" in cfg: + ttir = compile_3d_ptr(cfg["kernel_unified_attention_3d_ptr"]) + elif "reduce_segments_ptr" in cfg: + ttir = compile_reduce_ptr(cfg["reduce_segments_ptr"]) + + response = {"ok": True, "result": ttir} + except Exception as exc: + response = {"ok": False, "error": str(exc)} + + print(json.dumps(response), flush=True) diff --git a/zml/attention/triton/generate.zig b/zml/attention/triton/generate.zig new file mode 100644 index 00000000..872c7026 --- /dev/null +++ b/zml/attention/triton/generate.zig @@ -0,0 +1,111 @@ +const std = @import("std"); + +const c = @import("c"); +const runfiles = @import("runfiles"); + +fn pyStatusCheck(status: c.PyStatus) void { + if (c.PyStatus_Exception(status) != 0) { + if (c.PyStatus_IsExit(status) != 0) { + std.process.exit(@intCast(status.exitcode)); + } + c.Py_ExitStatusException(status); + } +} + +pub fn toPosixPathW(file_path: []const u8) error{ InvalidUtf8, NameTooLong, CodepointTooLarge }![std.posix.PATH_MAX - 1:0]c.wchar_t { + var view = std.unicode.Utf8View.init(file_path) catch return error.InvalidUtf8; + var it = view.iterator(); + + var path_with_null: [std.posix.PATH_MAX - 1:0]c.wchar_t = undefined; + var len: usize = 0; + while (it.nextCodepoint()) |cp| { + if (len + 1 >= path_with_null.len) return error.NameTooLong; + path_with_null[len] = std.math.cast(c.wchar_t, cp) orelse return error.CodepointTooLarge; + len += 1; + } + path_with_null[len] = 0; + return path_with_null; +} + +pub fn main(init: std.process.Init.Minimal) !void { + const allocator = std.heap.c_allocator; + + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + + const io = threaded.io(); + + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + { + var preconfig: c.PyPreConfig = undefined; + c.PyPreConfig_InitIsolatedConfig(&preconfig); + preconfig.utf8_mode = 1; + pyStatusCheck(c.Py_PreInitialize(&preconfig)); + } + + var config: c.PyConfig = undefined; + c.PyConfig_InitIsolatedConfig(&config); + defer c.PyConfig_Clear(&config); + + config.module_search_paths_set = 1; + config.optimization_level = 2; + config.write_bytecode = 0; + + _ = c.PyConfig_SetBytesArgv(&config, @intCast(init.args.vector.len), @ptrCast(init.args.vector.ptr)); + + var self_exe_dir_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const self_exe_dir = blk: { + const n = try std.process.executableDirPath(io, &self_exe_dir_buf); + break :blk self_exe_dir_buf[0..n]; + }; + + { + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const home = try std.fmt.bufPrintZ(&buf, "{f}{d}.{d}", .{ + std.Io.Dir.path.fmtJoin(&.{ + self_exe_dir, + "..", + "lib", + "python", + }), + c.PY_MAJOR_VERSION, + c.PY_MINOR_VERSION, + }); + pyStatusCheck(c.PyConfig_SetBytesString(&config, &config.home, home)); + pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(home))); + } + + { + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const site_packages = try std.fmt.bufPrint(&buf, "{f}", .{ + std.Io.Dir.path.fmtJoin(&.{ + self_exe_dir, + "..", + "site-packages", + }), + }); + pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(site_packages))); + } + + pyStatusCheck(c.Py_InitializeFromConfig(&config)); + defer c.Py_Finalize(); + + const module = c.PyImport_ImportModule("generate"); + if (module == null) { + c.PyErr_Print(); + return; + } + defer c.Py_DecRef(module); + + const func = c.PyObject_GetAttrString(module, "main"); + defer c.Py_DecRef(func); + + const result = c.PyObject_CallNoArgs(func); + if (result == null) { + c.PyErr_Print(); + return; + } + defer c.Py_DecRef(result); +} diff --git a/zml/attention/triton/libpython.h b/zml/attention/triton/libpython.h new file mode 100644 index 00000000..a5b5c4a1 --- /dev/null +++ b/zml/attention/triton/libpython.h @@ -0,0 +1,3 @@ +#define PY_SSIZE_T_CLEAN +#include + diff --git a/zml/attention/triton/requirements.in b/zml/attention/triton/requirements.in new file mode 100644 index 00000000..e0b3eb03 --- /dev/null +++ b/zml/attention/triton/requirements.in @@ -0,0 +1 @@ +triton==3.6.0 diff --git a/zml/attention/triton/requirements.lock.txt b/zml/attention/triton/requirements.lock.txt new file mode 100644 index 00000000..2e3244c2 --- /dev/null +++ b/zml/attention/triton/requirements.lock.txt @@ -0,0 +1,20 @@ +# This file was autogenerated by uv via the following command: +# bazel run //zml/attention/triton:requirements.update +--index-url https://pypi.org/simple + +triton==3.6.0 \ + --hash=sha256:0b3a97e8ed304dfa9bd23bb41ca04cdf6b2e617d5e782a8653d616037a5d537d \ + --hash=sha256:10c7f76c6e72d2ef08df639e3d0d30729112f47a56b0c81672edc05ee5116ac9 \ + --hash=sha256:1722e172d34e32abc3eb7711d0025bb69d7959ebea84e3b7f7a341cd7ed694d6 \ + --hash=sha256:374f52c11a711fd062b4bfbb201fd9ac0a5febd28a96fb41b4a0f51dde3157f4 \ + --hash=sha256:448e02fe6dc898e9e5aa89cf0ee5c371e99df5aa5e8ad976a80b93334f3494fd \ + --hash=sha256:46bd1c1af4b6704e554cad2eeb3b0a6513a980d470ccfa63189737340c7746a7 \ + --hash=sha256:49df5ef37379c0c2b5c0012286f80174fcf0e073e5ade1ca9a86c36814553651 \ + --hash=sha256:6c723cfb12f6842a0ae94ac307dba7e7a44741d720a40cf0e270ed4a4e3be781 \ + --hash=sha256:74caf5e34b66d9f3a429af689c1c7128daba1d8208df60e81106b115c00d6fca \ + --hash=sha256:a17a5d5985f0ac494ed8a8e54568f092f7057ef60e1b0fa09d3fd1512064e803 \ + --hash=sha256:a6550fae429e0667e397e5de64b332d1e5695b73650ee75a6146e2e902770bea \ + --hash=sha256:d002e07d7180fd65e622134fbd980c9a3d4211fb85224b56a0a0efbd422ab72f \ + --hash=sha256:e8e323d608e3a9bfcc2d9efcc90ceefb764a82b99dea12a86d643c72539ad5d3 \ + --hash=sha256:ef5523241e7d1abca00f1d240949eebdd7c673b005edbbce0aca95b8191f1d43 + # via -r zml/attention/triton/requirements.in diff --git a/zml/attention/triton/triton_kernels/__init__.py b/zml/attention/triton/triton_kernels/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/zml/attention/triton/triton_kernels/unified_attention.py b/zml/attention/triton/triton_kernels/unified_attention.py new file mode 100755 index 00000000..ad6242e7 --- /dev/null +++ b/zml/attention/triton/triton_kernels/unified_attention.py @@ -0,0 +1,1073 @@ +# The kernels in this file are adapted from vLLM: +# https://github.com/vllm-project/vllm/blob/main/vllm/attention/ops/triton_unified_attention.py +# +# NOTE: +# `scale` in `kernel_unified_attention_2d` is intentionally a runtime scalar +# (not `tl.constexpr`) so `kernel_unified_attention_2d_ptr` can load it from +# pointer arguments and forward it. If you see a performance regression, first +# thing to try is restoring `scale` to `tl.constexpr` and bypassing the ptr +# wrapper for that argument. +import triton +import triton.language as tl + +FP8_E4M3_MIN = -448.0 +FP8_E4M3_MAX = 448.0 + + +@triton.jit +def fast_exp(x): + RCP_LN2: tl.constexpr = 1.4426950408889634 + return tl.math.exp2(x * RCP_LN2) + + +@triton.jit +def cdiv_fn(x, y): + return (x + y - 1) // y + + +@triton.jit +def apply_softcap(S, x): + Sdiv = S / x + p1 = tl.math.exp2(Sdiv) + p2 = tl.math.exp2(-Sdiv) + return x * (p1 - p2) / (p1 + p2) + + +@triton.jit +def find_seq_idx( + query_start_len_ptr, + target_idx, + num_seqs, + BLOCK_Q: tl.constexpr, + use_q_block_mode: tl.constexpr, +): + left: tl.int32 = 0 + right = num_seqs + while left < right: + mid = (left + right) // 2 + val = tl.load(query_start_len_ptr + mid) + mid_val = val // BLOCK_Q + mid if use_q_block_mode else val + + if mid_val <= target_idx: + left = mid + 1 + else: + right = mid + + return left - 1 + + +@triton.jit +def kernel_unified_attention_2d( + output_ptr, # [num_tokens, num_query_heads, head_size] + query_ptr, # [num_tokens, num_query_heads, head_size] + key_cache_ptr, # [num_blks, blk_size, num_kv_heads, head_size] + value_cache_ptr, # [num_blks, blk_size, num_kv_heads, head_size] + sink_ptr, # [num_query_heads] + block_tables_ptr, # [num_seqs, max_num_blocks_per_seq] + seq_lens_ptr, # [num_seqs] + alibi_slopes_ptr, # [num_query_heads] + qq_bias_ptr, # [num_query_tokens, num_query_tokens] + scale, # float32 + k_scale, # float32 + v_scale, # float32 + out_scale, # float32 + softcap, # float32 + num_query_heads: tl.constexpr, # int + num_queries_per_kv: tl.constexpr, # int + block_table_stride: tl.int64, # int + query_stride_0: tl.int64, # int + query_stride_1: tl.int64, # int, should be equal to head_size + output_stride_0: tl.int64, # int + output_stride_1: tl.int64, # int, should be equal to head_size + qq_bias_stride_0: tl.int64, # int + BLOCK_SIZE: tl.constexpr, # int + TILE_SIZE: tl.constexpr, # int must be power of 2 + HEAD_SIZE: tl.constexpr, # int + HEAD_SIZE_PADDED: tl.constexpr, # int, must be power of 2 + USE_ALIBI_SLOPES: tl.constexpr, # bool + USE_QQ_BIAS: tl.constexpr, # bool + USE_SOFTCAP: tl.constexpr, # bool + USE_SINKS: tl.constexpr, # bool + SLIDING_WINDOW: tl.constexpr, # int + stride_k_cache_0: tl.int64, # int + stride_k_cache_1: tl.int64, # int + stride_k_cache_2: tl.int64, # int + stride_k_cache_3: tl.constexpr, # int + stride_v_cache_0: tl.int64, # int + stride_v_cache_1: tl.int64, # int + stride_v_cache_2: tl.int64, # int + stride_v_cache_3: tl.constexpr, # int + query_start_len_ptr, # [num_seqs+1] + BLOCK_Q: tl.constexpr, # int + num_seqs: tl.int32, + BLOCK_M: tl.constexpr, # int + USE_FP8: tl.constexpr, # bool + FP8_MIN: tl.constexpr = FP8_E4M3_MIN, + FP8_MAX: tl.constexpr = FP8_E4M3_MAX, + ALL_DECODE: tl.constexpr = False, # bool +): + kv_head_idx = tl.program_id(0) + q_block_global_idx = tl.program_id(1) + + # needed to use exp2 (exp2 -> exp conversion) + RCP_LN2 = 1.4426950408889634 + qk_scale = scale * RCP_LN2 + + seq_idx = find_seq_idx( + query_start_len_ptr, q_block_global_idx, num_seqs, BLOCK_Q, True + ) + + q_block_start_idx = tl.load(query_start_len_ptr + seq_idx) // BLOCK_Q + seq_idx + + q_block_local_idx = q_block_global_idx - q_block_start_idx + + cur_batch_in_all_start_index = tl.load(query_start_len_ptr + seq_idx) + cur_batch_in_all_stop_index = tl.load(query_start_len_ptr + seq_idx + 1) + + cur_batch_query_len = cur_batch_in_all_stop_index - cur_batch_in_all_start_index + + if q_block_local_idx * BLOCK_Q >= cur_batch_query_len: + return + + offs_m = tl.arange(0, BLOCK_M) + offs_d = tl.arange(0, HEAD_SIZE_PADDED) + offs_t = tl.arange(0, TILE_SIZE) + query_pos = q_block_local_idx * BLOCK_Q + offs_m // num_queries_per_kv + + query_offset_0 = cur_batch_in_all_start_index + query_pos + query_offset_1 = kv_head_idx * num_queries_per_kv + offs_m % num_queries_per_kv + query_offset = ( + query_offset_0[:, None] * query_stride_0 + + query_offset_1[:, None] * query_stride_1 + + offs_d[None, :] + ) + + if HEAD_SIZE_PADDED != HEAD_SIZE: + dim_mask = offs_d < HEAD_SIZE + else: + dim_mask = tl.full((1,), 1, dtype=tl.int1) + query_mask_0 = query_pos < cur_batch_query_len + query_mask_1 = query_offset_1 < num_query_heads + + if ALL_DECODE or BLOCK_M >= num_query_heads: + Q_cache_modifier: tl.constexpr = ".cg" + else: + Q_cache_modifier: tl.constexpr = "" + # Q : (BLOCK_M, HEAD_SIZE_PADDED) + Q = tl.load( + query_ptr + query_offset, + mask=dim_mask[None, :] & query_mask_0[:, None] & query_mask_1[:, None], + other=0.0, + cache_modifier=Q_cache_modifier, + ) + + block_table_offset = seq_idx * block_table_stride + + if not USE_SINKS: + M = tl.full([BLOCK_M], float("-inf"), dtype=tl.float32) + else: + # Prescale with RCP_LN2, needed for exp2 + M = ( + tl.load( + sink_ptr + query_offset_1, + mask=query_mask_1, + other=float("-inf"), + ).to(dtype=tl.float32) + * RCP_LN2 + ) + + L = tl.full([BLOCK_M], 1.0, dtype=tl.float32) + acc = tl.zeros([BLOCK_M, HEAD_SIZE_PADDED], dtype=tl.float32) + + # sequence len for this particular sequence + seq_len = tl.load(seq_lens_ptr + seq_idx) + + # context length for this particular sequences + context_len = seq_len - cur_batch_query_len + + # alibi slope for this head + if USE_ALIBI_SLOPES: + alibi_slope = tl.load( + alibi_slopes_ptr + query_offset_1, mask=query_mask_1, other=0.0 + ) + + # query-query attention bias + if USE_QQ_BIAS: + qq_bias_row_ptrs = ( + qq_bias_ptr + query_pos[:, None] * qq_bias_stride_0 + ) # shape: [BLOCK_M] + + # compute the length of the longest sequence prefix spanned by any + # query token in the current q_block (q_block_local_idx) + max_seq_prefix_len = ( + context_len + + q_block_local_idx * BLOCK_Q + + (BLOCK_M - 1) // num_queries_per_kv + + 1 + ) + + # adjust for potential padding in the last q_block by considering the + # actual sequence length + max_seq_prefix_len = tl.minimum(max_seq_prefix_len, seq_len) + + # calculate the number of tiles that need to be processed to + # cover the longest sequence prefix (due to causal masking, tiles beyond + # this prefix can be skipped) + num_tiles = cdiv_fn(max_seq_prefix_len, TILE_SIZE) + + # ---- Sliding-window tile pruning -------------------- + # Default: keep previous global behavior + tile_start = 0 + tile_end = num_tiles + if SLIDING_WINDOW > 0: + # Query rows covered by this Q-block + qpos_lo = q_block_local_idx * BLOCK_Q + qpos_hi = tl.minimum( + qpos_lo + (BLOCK_M - 1) // num_queries_per_kv, + cur_batch_query_len - 1, + ) + # For sliding window, each query position q can only attend to + # keys in the range [q_abs - SLIDING_WINDOW + 1, q_abs] + # where q_abs = context_len + q + # The union of allowed key positions for this Q-block is: + # [context_len + qpos_lo - SLIDING_WINDOW + 1, context_len + qpos_hi] + first_allowed_key = context_len + qpos_lo - SLIDING_WINDOW + 1 + last_allowed_key = context_len + qpos_hi + # Convert to tile indices and clamp + tile_start = tl.maximum(0, first_allowed_key // TILE_SIZE) + tile_end = tl.minimum((last_allowed_key // TILE_SIZE) + 1, num_tiles) + + KV_cache_modifier: tl.constexpr = ".cg" if ALL_DECODE else "" + # iterate through tiles (now limited to the sliding window range) + for j in range(tile_start, tile_end): + seq_offset = j * TILE_SIZE + offs_t + # to reduce the masking effect when not needed + if TILE_SIZE == BLOCK_SIZE: + tile_mask = tl.full((1,), 1, dtype=tl.int1) + else: + tile_mask = seq_offset < max_seq_prefix_len + + physical_block_idx = tl.load( + block_tables_ptr + block_table_offset + seq_offset // BLOCK_SIZE + ).to(tl.int64) + + v_offset = ( + physical_block_idx[:, None] * stride_v_cache_0 + + kv_head_idx * stride_v_cache_2 + + offs_d[None, :] * stride_v_cache_3 + + (seq_offset % BLOCK_SIZE)[:, None] * stride_v_cache_1 + ) + + k_offset = ( + physical_block_idx[None, :] * stride_k_cache_0 + + kv_head_idx * stride_k_cache_2 + + offs_d[:, None] * stride_k_cache_3 + + (seq_offset % BLOCK_SIZE)[None, :] * stride_k_cache_1 + ) + + # K : (HEAD_SIZE, TILE_SIZE) + K_load = tl.load( + key_cache_ptr + k_offset, + mask=dim_mask[:, None] & tile_mask[None, :], + other=0.0, + cache_modifier=KV_cache_modifier, + ) + + if K_load.dtype.is_fp8(): + if Q.dtype.is_fp8(): + K = K_load + else: + K = (K_load.to(tl.float32) * k_scale).to(Q.dtype) + else: + K = K_load + + # V : (TILE_SIZE, HEAD_SIZE) + V_load = tl.load( + value_cache_ptr + v_offset, + mask=dim_mask[None, :] & tile_mask[:, None], + other=0.0, + cache_modifier=KV_cache_modifier, + ) + + if V_load.dtype.is_fp8(): + if Q.dtype.is_fp8(): + V = V_load + else: + V = (V_load.to(tl.float32) * v_scale).to(Q.dtype) + else: + V = V_load + + # S : (BLOCK_M, TILE_SIZE) + # qk_scale = scale * RCP_LN2 (log_2 e) so that we can use exp2 later + S = qk_scale * tl.dot(Q, K) + + if USE_SOFTCAP: + # softcap here uses exp2 and consumes RCP_LN2 conversion. + # multiply by RCP_LN2 again to be used in later exp2 + S = apply_softcap(S, softcap) * RCP_LN2 + seq_mask = seq_offset[None, :] < context_len + query_pos[:, None] + 1 + + S = tl.where( + query_mask_1[:, None] & query_mask_0[:, None] & seq_mask, S, float("-inf") + ) + + if SLIDING_WINDOW > 0: + S = tl.where( + (context_len + query_pos[:, None] - seq_offset) < SLIDING_WINDOW, + S, + float("-inf"), + ) + + if USE_ALIBI_SLOPES: + # prescale w. RCP_LN2 for later exp2 + S += alibi_slope[:, None] * (seq_offset - context_len) * RCP_LN2 + + if USE_QQ_BIAS: + # compute key positions relative to query section + key_rel_pos = seq_offset - context_len # shape: [BLOCK_SIZE] + # load bias only for keys that correspond to queries + is_query_key = key_rel_pos >= 0 and key_rel_pos < qq_bias_stride_0 + qq_bias = tl.load( + qq_bias_row_ptrs + key_rel_pos[None, :], + mask=is_query_key[None, :], # avoid OOB for context keys + other=0.0, + ) + # prescale w. RCP_LN2 for later exp2 + S += qq_bias * RCP_LN2 + + # compute running maximum + # m_j : (BLOCK_M,) + m_j = tl.maximum(M, tl.max(S, axis=1)) + + # For sliding window there's a chance the max is -inf due to masking of + # the entire row. In this case we need to set m_j 0 to avoid NaN + m_j = tl.where(m_j > float("-inf"), m_j, 0.0) + + # P : (BLOCK_M, TILE_SIZE) + P = tl.math.exp2(S - m_j[:, None]) + + # l_j : (BLOCK_M,) + l_j = tl.sum(P, axis=1) + + # alpha : (BLOCK_M, ) + alpha = tl.math.exp2(M - m_j) + + # acc : (BLOCK_M, HEAD_SIZE_PADDED) + acc = acc * alpha[:, None] + + # update constants + L = L * alpha + l_j + M = m_j + + # acc : (BLOCK_M, HEAD_SIZE_PADDED) + acc += tl.dot(P.to(V.dtype), V) + + # epilogue + # This helps the compiler do Newton Raphson on l_i vs on acc which is much larger. + one_over_L = 1.0 / L[:, None] + acc = acc * one_over_L + if USE_FP8: + acc = acc * out_scale + acc = tl.clamp(acc, FP8_MIN, FP8_MAX) + + output_offset = ( + query_offset_0[:, None] * output_stride_0 + + query_offset_1[:, None] * output_stride_1 + + offs_d[None, :] + ) + + tl.store( + output_ptr + output_offset, + acc, + mask=dim_mask[None, :] & query_mask_0[:, None] & query_mask_1[:, None], + ) + + +@triton.jit +def kernel_unified_attention_3d( + segm_output_ptr, + # [num_tokens, num_query_heads, num_segments, head_size] + segm_max_ptr, # [num_tokens, num_query_heads, num_segments] + segm_expsum_ptr, # [num_tokens, num_query_heads, num_segments] + query_ptr, # [num_tokens, num_query_heads, head_size] + key_cache_ptr, # [num_blks, blk_size, num_kv_heads, head_size] + value_cache_ptr, # [num_blks, blk_size, num_kv_heads, head_size] + sink_ptr, # [num_query_heads] + block_tables_ptr, # [num_seqs, max_num_blocks_per_seq] + seq_lens_ptr, # [num_seqs] + alibi_slopes_ptr, # [num_query_heads] + qq_bias_ptr, # [num_query_tokens, num_query_tokens] + scale, # float32 + k_scale, # float32 + v_scale, # float32 + softcap, # float32 + num_query_heads: tl.constexpr, # int + num_queries_per_kv: tl.constexpr, # int + block_table_stride: tl.int64, # int + query_stride_0: tl.int64, # int + query_stride_1: tl.int64, # int, should be equal to head_size + qq_bias_stride_0: tl.int64, # int + BLOCK_SIZE: tl.constexpr, # int + TILE_SIZE: tl.constexpr, # int, must be power of 2 + HEAD_SIZE: tl.constexpr, # int + HEAD_SIZE_PADDED: tl.constexpr, # int, must be power of 2 + USE_ALIBI_SLOPES: tl.constexpr, # bool + USE_QQ_BIAS: tl.constexpr, # bool + USE_SOFTCAP: tl.constexpr, # bool + USE_SINKS: tl.constexpr, # bool + SLIDING_WINDOW: tl.constexpr, # int + stride_k_cache_0: tl.int64, # int + stride_k_cache_1: tl.int64, # int + stride_k_cache_2: tl.int64, # int + stride_k_cache_3: tl.constexpr, # int + stride_v_cache_0: tl.int64, # int + stride_v_cache_1: tl.int64, # int + stride_v_cache_2: tl.int64, # int + stride_v_cache_3: tl.constexpr, # int + query_start_len_ptr, # [num_seqs+1] + BLOCK_Q: tl.constexpr, # int + num_seqs: tl.int32, + BLOCK_M: tl.constexpr, # int + NUM_SEGMENTS_PER_SEQ: tl.constexpr, # int + ALL_DECODE: tl.constexpr = False, # bool +): + q_block_global_idx = tl.program_id(0) + kv_head_idx = tl.program_id(1) + segm_idx = tl.program_id(2) + + # needed to use exp2 (exp2 -> exp conversion) + RCP_LN2 = 1.4426950408889634 + qk_scale = scale * RCP_LN2 + + seq_idx = find_seq_idx( + query_start_len_ptr, q_block_global_idx, num_seqs, BLOCK_Q, True + ) + + q_block_start_idx = tl.load(query_start_len_ptr + seq_idx) // BLOCK_Q + seq_idx + + q_block_local_idx = q_block_global_idx - q_block_start_idx + + cur_batch_in_all_start_index = tl.load(query_start_len_ptr + seq_idx) + cur_batch_in_all_stop_index = tl.load(query_start_len_ptr + seq_idx + 1) + + cur_batch_query_len = cur_batch_in_all_stop_index - cur_batch_in_all_start_index + + if q_block_local_idx * BLOCK_Q >= cur_batch_query_len: + return + + # sequence len for this particular sequence + seq_len = tl.load(seq_lens_ptr + seq_idx) + + # number of segments for this particular sequence + num_segments = NUM_SEGMENTS_PER_SEQ + tiles_per_segment = cdiv_fn(seq_len, num_segments * TILE_SIZE) + + if segm_idx * tiles_per_segment * TILE_SIZE >= seq_len: + return + + offs_m = tl.arange(0, BLOCK_M) + offs_d = tl.arange(0, HEAD_SIZE_PADDED) + offs_t = tl.arange(0, TILE_SIZE) + query_pos = q_block_local_idx * BLOCK_Q + offs_m // num_queries_per_kv + + query_offset_0 = cur_batch_in_all_start_index + query_pos + query_offset_1 = kv_head_idx * num_queries_per_kv + offs_m % num_queries_per_kv + query_offset = ( + query_offset_0[:, None] * query_stride_0 + + query_offset_1[:, None] * query_stride_1 + + offs_d[None, :] + ) + + if HEAD_SIZE_PADDED != HEAD_SIZE: + dim_mask = offs_d < HEAD_SIZE + else: + dim_mask = tl.full((1,), 1, dtype=tl.int1) + query_mask_0 = query_pos < cur_batch_query_len + query_mask_1 = query_offset_1 < num_query_heads + + # Q : (BLOCK_M, HEAD_SIZE_PADDED) + Q = tl.load( + query_ptr + query_offset, + mask=dim_mask[None, :] & query_mask_0[:, None] & query_mask_1[:, None], + other=0.0, + ) + + block_table_offset = seq_idx * block_table_stride + + if USE_SINKS: + if segm_idx == 0: + # Prescale with RCP_LN2, needed for exp2 + M = ( + tl.load( + sink_ptr + query_offset_1, + mask=query_mask_1, + other=float("-inf"), + ).to(dtype=tl.float32) + * RCP_LN2 + ) + else: + M = tl.full([BLOCK_M], float("-inf"), dtype=tl.float32) + else: + M = tl.full([BLOCK_M], float("-inf"), dtype=tl.float32) + + L = tl.full([BLOCK_M], 1.0, dtype=tl.float32) + acc = tl.zeros([BLOCK_M, HEAD_SIZE_PADDED], dtype=tl.float32) + + # context length for this particular sequences + context_len = seq_len - cur_batch_query_len + + # alibi slope for this head + if USE_ALIBI_SLOPES: + alibi_slope = tl.load( + alibi_slopes_ptr + query_offset_1, mask=query_mask_1, other=0.0 + ) + + # query-query attention bias + if USE_QQ_BIAS: + qq_bias_row_ptrs = ( + qq_bias_ptr + query_pos[:, None] * qq_bias_stride_0 + ) # shape: [BLOCK_M] + + # compute the length of the longest sequence prefix spanned by any + # query token in the current q_block (q_block_local_idx) + max_seq_prefix_len = ( + context_len + + q_block_local_idx * BLOCK_Q + + (BLOCK_M - 1) // num_queries_per_kv + + 1 + ) + + # adjust for potential padding in the last q_block by considering the + # actual sequence length + max_seq_prefix_len = tl.minimum(max_seq_prefix_len, seq_len) + + # calculate the number of tiles that need to be processed to + # cover the longest sequence prefix (due to causal masking, tiles beyond + # this prefix can be skipped) + num_tiles = cdiv_fn(max_seq_prefix_len, TILE_SIZE) + + KV_cache_modifier: tl.constexpr = ".cg" if ALL_DECODE else "" + # iterate through tiles within current segment + for j in range( + segm_idx * tiles_per_segment, + min((segm_idx + 1) * tiles_per_segment, num_tiles), + ): + seq_offset = j * TILE_SIZE + offs_t + if TILE_SIZE == BLOCK_SIZE: + tile_mask = tl.full((1,), 1, dtype=tl.int1) + else: + tile_mask = seq_offset < max_seq_prefix_len + + physical_block_idx = tl.load( + block_tables_ptr + block_table_offset + seq_offset // BLOCK_SIZE + ).to(tl.int64) + + v_offset = ( + physical_block_idx[:, None] * stride_v_cache_0 + + kv_head_idx * stride_v_cache_2 + + offs_d[None, :] * stride_v_cache_3 + + (seq_offset % BLOCK_SIZE)[:, None] * stride_v_cache_1 + ) + + k_offset = ( + physical_block_idx[None, :] * stride_k_cache_0 + + kv_head_idx * stride_k_cache_2 + + offs_d[:, None] * stride_k_cache_3 + + (seq_offset % BLOCK_SIZE)[None, :] * stride_k_cache_1 + ) + + # K : (HEAD_SIZE, TILE_SIZE) + K_load = tl.load( + key_cache_ptr + k_offset, + mask=dim_mask[:, None] & tile_mask[None, :], + other=0.0, + cache_modifier=KV_cache_modifier, + ) + + if K_load.dtype.is_fp8(): + if Q.dtype.is_fp8(): + K = K_load + else: + K = (K_load.to(tl.float32) * k_scale).to(Q.dtype) + else: + K = K_load + + # V : (TILE_SIZE, HEAD_SIZE) + V_load = tl.load( + value_cache_ptr + v_offset, + mask=dim_mask[None, :] & tile_mask[:, None], + other=0.0, + cache_modifier=KV_cache_modifier, + ) + + if V_load.dtype.is_fp8(): + if Q.dtype.is_fp8(): + V = V_load + else: + V = (V_load.to(tl.float32) * v_scale).to(Q.dtype) + else: + V = V_load + + seq_mask = seq_offset[None, :] < context_len + query_pos[:, None] + 1 + + # S : (BLOCK_M, TILE_SIZE) + # qk_scale = scale * RCP_LN2 (log_2 e) so that we can use exp2 later + S = qk_scale * tl.dot(Q, K) + + if USE_SOFTCAP: + # softcap here uses exp2 and consumes RCP_LN2 conversion. + # multiply by RCP_LN2 again to be used in later exp2 + S = apply_softcap(S, softcap) * RCP_LN2 + + S = tl.where( + query_mask_1[:, None] & query_mask_0[:, None] & seq_mask, S, float("-inf") + ) + + if SLIDING_WINDOW > 0: + S = tl.where( + (context_len + query_pos[:, None] - seq_offset) < SLIDING_WINDOW, + S, + float("-inf"), + ) + + if USE_ALIBI_SLOPES: + # prescale w. RCP_LN2 for later exp2 + S += alibi_slope[:, None] * (seq_offset - context_len) * RCP_LN2 + + if USE_QQ_BIAS: + # compute key positions relative to query section + key_rel_pos = seq_offset - context_len # shape: [BLOCK_SIZE] + # load bias only for keys that correspond to queries + is_query_key = key_rel_pos >= 0 and key_rel_pos < qq_bias_stride_0 + qq_bias = tl.load( + qq_bias_row_ptrs + key_rel_pos[None, :], + mask=is_query_key[None, :], # avoid OOB for context keys + other=0.0, + ) + # prescale w. RCP_LN2 for later exp2 + S += qq_bias * RCP_LN2 + + # compute running maximum + # m_j : (BLOCK_M,) + m_j = tl.maximum(M, tl.max(S, axis=1)) + + # For sliding window there's a chance the max is -inf due to masking of + # the entire row. In this case we need to set m_j 0 to avoid NaN + m_j = tl.where(m_j > float("-inf"), m_j, 0.0) + + # P : (BLOCK_M, TILE_SIZE,) + P = tl.math.exp2(S - m_j[:, None]) + + # l_j : (BLOCK_M,) + l_j = tl.sum(P, axis=1) + + # alpha : (BLOCK_M, ) + alpha = tl.math.exp2(M - m_j) + + # acc : (BLOCK_M, HEAD_SIZE_PADDED) + acc = acc * alpha[:, None] + + # update constants + L = L * alpha + l_j + M = m_j + + # acc : (BLOCK_M, HEAD_SIZE_PADDED) + acc += tl.dot(P.to(V.dtype), V) + + segm_output_offset = ( + query_offset_0[:, None].to(tl.int64) + * (num_query_heads * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) + + query_offset_1[:, None] * (NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) + + segm_idx * HEAD_SIZE_PADDED + + tl.arange(0, HEAD_SIZE_PADDED)[None, :] + ) + tl.store( + segm_output_ptr + segm_output_offset, + acc, + mask=dim_mask[None, :] & query_mask_0[:, None] & query_mask_1[:, None], + ) + segm_offset = ( + query_offset_0.to(tl.int64) * (num_query_heads * NUM_SEGMENTS_PER_SEQ) + + query_offset_1 * NUM_SEGMENTS_PER_SEQ + + segm_idx + ) + tl.store(segm_max_ptr + segm_offset, M, mask=query_mask_0 & query_mask_1) + tl.store(segm_expsum_ptr + segm_offset, L, mask=query_mask_0 & query_mask_1) + + +@triton.jit +def reduce_segments( + output_ptr, # [num_tokens, num_query_heads, head_size] + segm_output_ptr, + # [num_tokens, num_query_heads, max_num_segments, head_size] + segm_max_ptr, # [num_tokens, num_query_heads, max_num_segments] + segm_expsum_ptr, # [num_tokens, num_query_heads, max_num_segments] + seq_lens_ptr, # [num_seqs] + num_seqs, # int + num_query_heads: tl.constexpr, # int + out_scale_inv, # float32 + output_stride_0: tl.int64, # int + output_stride_1: tl.int64, # int, should be equal to head_size + block_table_stride: tl.int64, # int + TILE_SIZE: tl.constexpr, # int + HEAD_SIZE: tl.constexpr, # int, must be power of 2 + HEAD_SIZE_PADDED: tl.constexpr, # int, must be power of 2 + query_start_len_ptr, # [num_seqs+1] + BLOCK_Q: tl.constexpr, # int + NUM_SEGMENTS_PER_SEQ: tl.constexpr, # int + USE_FP8: tl.constexpr, # bool + FP8_MIN: tl.constexpr = FP8_E4M3_MIN, + FP8_MAX: tl.constexpr = FP8_E4M3_MAX, +): + query_token_idx = tl.program_id(0) + query_head_idx = tl.program_id(1) + + seq_idx = find_seq_idx( + query_start_len_ptr, query_token_idx, num_seqs, BLOCK_Q, False + ) + + # sequence len for this particular sequence + seq_len = tl.load(seq_lens_ptr + seq_idx) + + # number of segments for this particular sequence + num_segments = NUM_SEGMENTS_PER_SEQ + tiles_per_segment = cdiv_fn(seq_len, num_segments * TILE_SIZE) + + # create masks for subsequent loads + act_num_segments = cdiv_fn(seq_len, tiles_per_segment * TILE_SIZE) + segm_mask = tl.arange(0, NUM_SEGMENTS_PER_SEQ) < tl.full( + [NUM_SEGMENTS_PER_SEQ], act_num_segments, dtype=tl.int32 + ) + + if HEAD_SIZE_PADDED != HEAD_SIZE: + dim_mask = offs_d < HEAD_SIZE + else: + dim_mask = tl.full((1,), 1, dtype=tl.int1) + + # load segment maxima + segm_offset = ( + query_token_idx.to(tl.int64) * (num_query_heads * NUM_SEGMENTS_PER_SEQ) + + query_head_idx * NUM_SEGMENTS_PER_SEQ + + tl.arange(0, NUM_SEGMENTS_PER_SEQ) + ) + segm_max = tl.load(segm_max_ptr + segm_offset, mask=segm_mask, other=float("-inf")) + overall_max = tl.max(segm_max) + + # load and rescale segment exp sums + segm_expsum = tl.load(segm_expsum_ptr + segm_offset, mask=segm_mask, other=0.0) + segm_expsum = segm_expsum * tl.math.exp2(segm_max - overall_max) + overall_expsum = tl.sum(segm_expsum) + + # load, rescale, and add segment attention outputs + segm_output_offset = ( + query_token_idx.to(tl.int64) + * (num_query_heads * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) + + query_head_idx * (NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) + + tl.arange(0, NUM_SEGMENTS_PER_SEQ)[:, None] * HEAD_SIZE_PADDED + + tl.arange(0, HEAD_SIZE_PADDED)[None, :] + ) + segm_output = tl.load( + segm_output_ptr + segm_output_offset, + mask=segm_mask[:, None] & dim_mask[None, :], + other=0.0, + ) + segm_output *= tl.math.exp2(segm_max - overall_max)[:, None] + acc_sum = tl.sum(segm_output, axis=0) + # safely divide by overall_expsum, returning 0.0 if overall_expsum is 0 + acc = tl.where(overall_expsum == 0.0, 0.0, acc_sum / overall_expsum) + + if USE_FP8: + acc = acc * out_scale_inv + acc = tl.clamp(acc, FP8_MIN, FP8_MAX) + + # write result + output_offset = ( + query_token_idx * output_stride_0 + + query_head_idx * output_stride_1 + + tl.arange(0, HEAD_SIZE_PADDED) + ) + tl.store(output_ptr + output_offset, acc, mask=dim_mask) + + +@triton.jit +def kernel_unified_attention_2d_ptr( + query_ptr, + key_cache_ptr, + value_cache_ptr, + sink_ptr, + block_tables_ptr, + seq_lens_ptr, + alibi_slopes_ptr, + qq_bias_ptr, + scale_ptr, + k_scale_ptr, + v_scale_ptr, + out_scale_ptr, + softcap_ptr, + num_query_heads: tl.constexpr, + num_queries_per_kv: tl.constexpr, + block_table_stride_ptr, + query_stride_0_ptr, + query_stride_1_ptr, + output_stride_0_ptr, + output_stride_1_ptr, + qq_bias_stride_0_ptr, + BLOCK_SIZE: tl.constexpr, + TILE_SIZE: tl.constexpr, + HEAD_SIZE: tl.constexpr, + HEAD_SIZE_PADDED: tl.constexpr, + USE_ALIBI_SLOPES: tl.constexpr, + USE_QQ_BIAS: tl.constexpr, + USE_SOFTCAP: tl.constexpr, + USE_SINKS: tl.constexpr, + SLIDING_WINDOW: tl.constexpr, + stride_k_cache_0_ptr, + stride_k_cache_1_ptr, + stride_k_cache_2_ptr, + stride_k_cache_3: tl.constexpr, + stride_v_cache_0_ptr, + stride_v_cache_1_ptr, + stride_v_cache_2_ptr, + stride_v_cache_3: tl.constexpr, + query_start_len_ptr, + BLOCK_Q: tl.constexpr, + num_seqs_ptr, + BLOCK_M: tl.constexpr, + USE_FP8: tl.constexpr, + FP8_MIN: tl.constexpr = FP8_E4M3_MIN, + FP8_MAX: tl.constexpr = FP8_E4M3_MAX, + ALL_DECODE: tl.constexpr = False, + output_ptr=None, +): + scale = tl.load(scale_ptr) + k_scale = tl.load(k_scale_ptr) + v_scale = tl.load(v_scale_ptr) + out_scale = tl.load(out_scale_ptr) + softcap = tl.load(softcap_ptr) + block_table_stride = tl.load(block_table_stride_ptr) + query_stride_0 = tl.load(query_stride_0_ptr) + query_stride_1 = tl.load(query_stride_1_ptr) + output_stride_0 = tl.load(output_stride_0_ptr) + output_stride_1 = tl.load(output_stride_1_ptr) + qq_bias_stride_0 = tl.load(qq_bias_stride_0_ptr) + stride_k_cache_0 = tl.load(stride_k_cache_0_ptr) + stride_k_cache_1 = tl.load(stride_k_cache_1_ptr) + stride_k_cache_2 = tl.load(stride_k_cache_2_ptr) + stride_v_cache_0 = tl.load(stride_v_cache_0_ptr) + stride_v_cache_1 = tl.load(stride_v_cache_1_ptr) + stride_v_cache_2 = tl.load(stride_v_cache_2_ptr) + num_seqs = tl.load(num_seqs_ptr) + + kernel_unified_attention_2d( + output_ptr, + query_ptr, + key_cache_ptr, + value_cache_ptr, + sink_ptr, + block_tables_ptr, + seq_lens_ptr, + alibi_slopes_ptr, + qq_bias_ptr, + scale, + k_scale, + v_scale, + out_scale, + softcap, + num_query_heads=num_query_heads, + num_queries_per_kv=num_queries_per_kv, + block_table_stride=block_table_stride, + query_stride_0=query_stride_0, + query_stride_1=query_stride_1, + output_stride_0=output_stride_0, + output_stride_1=output_stride_1, + qq_bias_stride_0=qq_bias_stride_0, + BLOCK_SIZE=BLOCK_SIZE, + TILE_SIZE=TILE_SIZE, + HEAD_SIZE=HEAD_SIZE, + HEAD_SIZE_PADDED=HEAD_SIZE_PADDED, + USE_ALIBI_SLOPES=USE_ALIBI_SLOPES, + USE_QQ_BIAS=USE_QQ_BIAS, + USE_SOFTCAP=USE_SOFTCAP, + USE_SINKS=USE_SINKS, + SLIDING_WINDOW=SLIDING_WINDOW, + stride_k_cache_0=stride_k_cache_0, + stride_k_cache_1=stride_k_cache_1, + stride_k_cache_2=stride_k_cache_2, + stride_k_cache_3=stride_k_cache_3, + stride_v_cache_0=stride_v_cache_0, + stride_v_cache_1=stride_v_cache_1, + stride_v_cache_2=stride_v_cache_2, + stride_v_cache_3=stride_v_cache_3, + query_start_len_ptr=query_start_len_ptr, + BLOCK_Q=BLOCK_Q, + num_seqs=num_seqs, + BLOCK_M=BLOCK_M, + USE_FP8=USE_FP8, + FP8_MIN=FP8_MIN, + FP8_MAX=FP8_MAX, + ALL_DECODE=ALL_DECODE, + ) + + +@triton.jit +def kernel_unified_attention_3d_ptr( + query_ptr, + key_cache_ptr, + value_cache_ptr, + sink_ptr, + block_tables_ptr, + seq_lens_ptr, + alibi_slopes_ptr, + qq_bias_ptr, + scale_ptr, + k_scale_ptr, + v_scale_ptr, + softcap_ptr, + num_query_heads: tl.constexpr, + num_queries_per_kv: tl.constexpr, + block_table_stride_ptr, + query_stride_0_ptr, + query_stride_1_ptr, + qq_bias_stride_0_ptr, + BLOCK_SIZE: tl.constexpr, + TILE_SIZE: tl.constexpr, + HEAD_SIZE: tl.constexpr, + HEAD_SIZE_PADDED: tl.constexpr, + USE_ALIBI_SLOPES: tl.constexpr, + USE_QQ_BIAS: tl.constexpr, + USE_SOFTCAP: tl.constexpr, + USE_SINKS: tl.constexpr, + SLIDING_WINDOW: tl.constexpr, + stride_k_cache_0_ptr, + stride_k_cache_1_ptr, + stride_k_cache_2_ptr, + stride_k_cache_3: tl.constexpr, + stride_v_cache_0_ptr, + stride_v_cache_1_ptr, + stride_v_cache_2_ptr, + stride_v_cache_3: tl.constexpr, + query_start_len_ptr, + BLOCK_Q: tl.constexpr, + num_seqs_ptr, + BLOCK_M: tl.constexpr, + NUM_SEGMENTS_PER_SEQ: tl.constexpr, + ALL_DECODE: tl.constexpr = False, + segm_output_ptr=None, + segm_max_ptr=None, + segm_expsum_ptr=None, +): + scale = tl.load(scale_ptr) + k_scale = tl.load(k_scale_ptr) + v_scale = tl.load(v_scale_ptr) + softcap = tl.load(softcap_ptr) + block_table_stride = tl.load(block_table_stride_ptr) + query_stride_0 = tl.load(query_stride_0_ptr) + query_stride_1 = tl.load(query_stride_1_ptr) + qq_bias_stride_0 = tl.load(qq_bias_stride_0_ptr) + stride_k_cache_0 = tl.load(stride_k_cache_0_ptr) + stride_k_cache_1 = tl.load(stride_k_cache_1_ptr) + stride_k_cache_2 = tl.load(stride_k_cache_2_ptr) + stride_v_cache_0 = tl.load(stride_v_cache_0_ptr) + stride_v_cache_1 = tl.load(stride_v_cache_1_ptr) + stride_v_cache_2 = tl.load(stride_v_cache_2_ptr) + num_seqs = tl.load(num_seqs_ptr) + + kernel_unified_attention_3d( + segm_output_ptr, + segm_max_ptr, + segm_expsum_ptr, + query_ptr, + key_cache_ptr, + value_cache_ptr, + sink_ptr, + block_tables_ptr, + seq_lens_ptr, + alibi_slopes_ptr, + qq_bias_ptr, + scale, + k_scale, + v_scale, + softcap, + num_query_heads=num_query_heads, + num_queries_per_kv=num_queries_per_kv, + block_table_stride=block_table_stride, + query_stride_0=query_stride_0, + query_stride_1=query_stride_1, + qq_bias_stride_0=qq_bias_stride_0, + BLOCK_SIZE=BLOCK_SIZE, + TILE_SIZE=TILE_SIZE, + HEAD_SIZE=HEAD_SIZE, + HEAD_SIZE_PADDED=HEAD_SIZE_PADDED, + USE_ALIBI_SLOPES=USE_ALIBI_SLOPES, + USE_QQ_BIAS=USE_QQ_BIAS, + USE_SOFTCAP=USE_SOFTCAP, + USE_SINKS=USE_SINKS, + SLIDING_WINDOW=SLIDING_WINDOW, + stride_k_cache_0=stride_k_cache_0, + stride_k_cache_1=stride_k_cache_1, + stride_k_cache_2=stride_k_cache_2, + stride_k_cache_3=stride_k_cache_3, + stride_v_cache_0=stride_v_cache_0, + stride_v_cache_1=stride_v_cache_1, + stride_v_cache_2=stride_v_cache_2, + stride_v_cache_3=stride_v_cache_3, + query_start_len_ptr=query_start_len_ptr, + BLOCK_Q=BLOCK_Q, + num_seqs=num_seqs, + BLOCK_M=BLOCK_M, + NUM_SEGMENTS_PER_SEQ=NUM_SEGMENTS_PER_SEQ, + ALL_DECODE=ALL_DECODE, + ) + + +@triton.jit +def reduce_segments_ptr( + segm_output_ptr, + segm_max_ptr, + segm_expsum_ptr, + seq_lens_ptr, + num_seqs_ptr, + num_query_heads: tl.constexpr, + out_scale_inv_ptr, + output_stride_0_ptr, + output_stride_1_ptr, + block_table_stride_ptr, + TILE_SIZE: tl.constexpr, + HEAD_SIZE: tl.constexpr, + HEAD_SIZE_PADDED: tl.constexpr, + query_start_len_ptr, + BLOCK_Q: tl.constexpr, + NUM_SEGMENTS_PER_SEQ: tl.constexpr, + USE_FP8: tl.constexpr, + FP8_MIN: tl.constexpr = FP8_E4M3_MIN, + FP8_MAX: tl.constexpr = FP8_E4M3_MAX, + output_ptr=None, +): + num_seqs = tl.load(num_seqs_ptr) + out_scale_inv = tl.load(out_scale_inv_ptr) + output_stride_0 = tl.load(output_stride_0_ptr) + output_stride_1 = tl.load(output_stride_1_ptr) + block_table_stride = tl.load(block_table_stride_ptr) + + reduce_segments( + output_ptr, + segm_output_ptr, + segm_max_ptr, + segm_expsum_ptr, + seq_lens_ptr, + num_seqs, + num_query_heads=num_query_heads, + out_scale_inv=out_scale_inv, + output_stride_0=output_stride_0, + output_stride_1=output_stride_1, + block_table_stride=block_table_stride, + TILE_SIZE=TILE_SIZE, + HEAD_SIZE=HEAD_SIZE, + HEAD_SIZE_PADDED=HEAD_SIZE_PADDED, + query_start_len_ptr=query_start_len_ptr, + BLOCK_Q=BLOCK_Q, + NUM_SEGMENTS_PER_SEQ=NUM_SEGMENTS_PER_SEQ, + USE_FP8=USE_FP8, + FP8_MIN=FP8_MIN, + FP8_MAX=FP8_MAX, + ) diff --git a/zml/io.zig b/zml/io.zig index 91d1b5d8..97d0fbe1 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -1269,7 +1269,7 @@ const DirectMemoryWriterDeviceTest = struct { _ = scenario.name; var platform = Platform.auto(self.allocator, self.io, scenario.create_options) catch return error.SkipZigTest; - defer platform.deinit(self.allocator); + defer platform.deinit(self.allocator, self.io); const sharding: Sharding = try .initFromStrategy(platform, scenario.logical_mesh, scenario.strategy); try self.runDirectMemoryWriter( diff --git a/zml/module.zig b/zml/module.zig index 6afc2254..6b141ad8 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -81,6 +81,7 @@ pub const CompilationContext = struct { }; allocator: std.mem.Allocator, + io: std.Io, arena: std.heap.ArenaAllocator, mlir_registry: *mlir.DialectRegistry, @@ -120,6 +121,7 @@ pub const CompilationContext = struct { return .{ .allocator = allocator, + .io = io, .arena = std.heap.ArenaAllocator.init(allocator), .mlir_registry = mlir_registry, .mlir_ctx = mlir_ctx, @@ -176,7 +178,18 @@ pub fn compile( platform: *const Platform, opts: CompilationOptions, ) !Exe { - var compilation_context: CompilationContext = .init(allocator, io, platform, opts); + // TODO: Here we have somewhat of a requirement + // Emitting MLIR requires to have the compilation context available at all times using `CompilationContext.current()`. + // If in the future, we inject an Io that is not thread-based, we might have some surprises. + // + // I think the correct implementation would be to dispatch `emitMlir` to a thread pool, then wait for the result + // asynchronously using the provided Io. For now, we'll simply make that blocking as it's not a big deal but keep + // in mind we might want to revisit that later. + _ = io; + var st_io: std.Io.Threaded = .init_single_threaded; + defer st_io.deinit(); + + var compilation_context: CompilationContext = .init(allocator, st_io.io(), platform, opts); defer compilation_context.deinit(); const result = emitMlir(&compilation_context, func, args) catch unreachable; @@ -210,7 +223,7 @@ pub fn compile( var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); - const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), io, platform, compilation_context.module, compilation_context.partitioning, opts) catch unreachable; + const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), st_io.io(), platform, compilation_context.module, compilation_context.partitioning, opts) catch unreachable; log.debug("\n******** ZML generated MLIR ********\n{f}", .{compilation_context.module.operation()}); const exe = try Exe.init( diff --git a/zml/platform.zig b/zml/platform.zig index ad3bf7f6..2bcecba7 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -11,6 +11,8 @@ const pjrtx = @import("pjrtx.zig"); const profiler_ = @import("profiler.zig"); const zml = @import("zml.zig"); +const attention = @import("attention.zig"); + const log = std.log.scoped(.zml); fn StaticPlatformMap(comptime E: type, comptime T: type) type { @@ -201,6 +203,19 @@ pub const Platform = struct { memories: []const Memory, physical_mesh: zml.sharding.PhysicalMesh, + triton_runtime: ?attention.triton.Runtime = null, + + pub fn initBackend(self: *Platform, allocator: std.mem.Allocator, io: std.Io, backend: attention.paged_attention.Backend) !void { + switch (backend) { + .triton => { + if (self.triton_runtime == null) { + self.triton_runtime = try zml.attention.triton.Runtime.init(allocator, io); + } + }, + else => {}, + } + } + pub const MAX_NUM_DEVICES: u16 = if (platforms.isEnabled(.tpu)) 64 else 32; pub fn init(allocator: std.mem.Allocator, io: std.Io, target: Target, options: CreateOptions) !*Platform { @@ -401,8 +416,9 @@ pub const Platform = struct { return .{ .data = self }; } - pub fn deinit(self: *Platform, allocator: std.mem.Allocator) void { + pub fn deinit(self: *Platform, allocator: std.mem.Allocator, io: std.Io) void { self.pjrt_client.deinit(self.pjrt_api); + if (self.triton_runtime) |*rt| rt.deinit(io); self.arena_state.promote(allocator).deinit(); } From 8edf02d0c38efdc6ae0c677da8c1a3646e7a55ed Mon Sep 17 00:00:00 2001 From: KB Date: Mon, 13 Apr 2026 11:56:32 +0200 Subject: [PATCH 416/881] zml/chore: bump Zig to 0.16.0-dev.3133 and adapt codebase (#462) Update Zig compiler to 0.16.0-dev.3133+5ec8e45f3 and adjust code for breaking standard library changes: - Replace ArrayListUnmanaged/ArrayList init with .empty - Switch to unmanaged AutoArrayHashMap/StringArrayHashMap - Use debug.GeneralPurposeAllocator (renamed upstream) - Replace std.Io.File.ReadStreamingError with error.EndOfStream - Refactor meta.visit to use structured visitor pattern - Fix alignment allocation in mem.bufferizeInner Also update third-party dependencies: flashattn 0.0.6-rc8, arocc, uucode, libvaxis, and zigimg patches. Co-authored-by: Kevin Barre --- MODULE.bazel | 2 +- MODULE.bazel.lock | 255 ++++++++++++++++++---- bazel/zig_index.json | 173 ++++++++------- bin/zml-smi/platforms/amdsmi/amdsmi.zig | 2 +- bin/zml-smi/platforms/neuron/metrics.zig | 4 +- bin/zml-smi/platforms/neuron/nrt.zig | 4 +- bin/zml-smi/platforms/tpu/metrics.zig | 4 +- bin/zml-smi/tui/data.zig | 2 +- bin/zml-smi/tui/widgets/process_table.zig | 2 +- bin/zml-smi/tui/widgets/status_line.zig | 2 +- examples/io/main.zig | 10 +- platforms/neuron/libpjrt_neuron.zig | 3 +- third_party/arocc/repo.bzl | 2 +- third_party/flashattn/repo.bzl | 4 +- third_party/libvaxis/fixes.patch | 46 ++++ third_party/uucode/build_config.zig | 24 +- third_party/uucode/repo.bzl | 2 +- third_party/uucode/uucode.bazel | 128 +++-------- third_party/zigimg/bump-zig.patch | 153 +++++++++++++ zml/io.zig | 7 +- zml/io/vfs.zig | 12 +- zml/io/vfs/base.zig | 20 +- zml/io/vfs/file.zig | 4 +- zml/io/vfs/gcs.zig | 10 +- zml/io/vfs/hf.zig | 18 +- zml/io/vfs/http.zig | 10 +- zml/io/vfs/s3.zig | 12 +- zml/mem.zig | 2 +- zml/meta.zig | 126 ++++++----- zml/sharding.zig | 8 +- zml/tensor.zig | 2 +- zml/tokenizer/homemade.zig | 9 +- zml/tokenizer/main.zig | 3 +- 33 files changed, 694 insertions(+), 371 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 79c99a03..b1ce05e7 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -49,7 +49,7 @@ zig = use_extension("@rules_zig//zig:extensions.bzl", "zig") zig.index(file = "//bazel:zig_index.json") zig.toolchain( translate_c = "@translate-c", - zig_version = "0.16.0-dev.2722+f16eb18ce", + zig_version = "0.16.0-dev.3153+d6f43caad", ) zig.mirrors(urls = [ "https://pkg.machengine.org/zig", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index bb22c713..66de42db 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -478,6 +478,173 @@ ] } }, + "@@googleapis+//:extensions.bzl%switched_rules": { + "general": { + "bzlTransitiveDigest": "JLpojYeGHLdgnwSUuQWOrtkL9UMygBUUEFJZh7ASlmA=", + "usagesDigest": "66rLsrgz6YqDpU0pD+02mTGNxYxhb/of6Fh0ulHzjdE=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "com_google_googleapis_imports": { + "repoRuleId": "@@googleapis+//:repository_rules.bzl%switched_rules", + "attributes": { + "rules": { + "proto_library_with_info": [ + "", + "" + ], + "moved_proto_library": [ + "", + "" + ], + "java_proto_library": [ + "", + "" + ], + "java_grpc_library": [ + "", + "" + ], + "java_gapic_library": [ + "", + "" + ], + "java_gapic_test": [ + "", + "" + ], + "java_gapic_assembly_gradle_pkg": [ + "", + "" + ], + "py_proto_library": [ + "", + "" + ], + "py_grpc_library": [ + "", + "" + ], + "py_gapic_library": [ + "", + "" + ], + "py_test": [ + "", + "" + ], + "py_gapic_assembly_pkg": [ + "", + "" + ], + "py_import": [ + "", + "" + ], + "go_proto_library": [ + "", + "" + ], + "go_grpc_library": [ + "", + "" + ], + "go_library": [ + "", + "" + ], + "go_test": [ + "", + "" + ], + "go_gapic_library": [ + "", + "" + ], + "go_gapic_assembly_pkg": [ + "", + "" + ], + "cc_proto_library": [ + "", + "" + ], + "cc_grpc_library": [ + "", + "" + ], + "cc_gapic_library": [ + "", + "" + ], + "php_proto_library": [ + "", + "php_proto_library" + ], + "php_grpc_library": [ + "", + "php_grpc_library" + ], + "php_gapic_library": [ + "", + "php_gapic_library" + ], + "php_gapic_assembly_pkg": [ + "", + "php_gapic_assembly_pkg" + ], + "nodejs_gapic_library": [ + "", + "typescript_gapic_library" + ], + "nodejs_gapic_assembly_pkg": [ + "", + "typescript_gapic_assembly_pkg" + ], + "ruby_proto_library": [ + "", + "" + ], + "ruby_grpc_library": [ + "", + "" + ], + "ruby_ads_gapic_library": [ + "", + "" + ], + "ruby_cloud_gapic_library": [ + "", + "" + ], + "ruby_gapic_assembly_pkg": [ + "", + "" + ], + "csharp_proto_library": [ + "", + "" + ], + "csharp_grpc_library": [ + "", + "" + ], + "csharp_gapic_library": [ + "", + "" + ], + "csharp_gapic_assembly_pkg": [ + "", + "" + ] + } + } + } + }, + "recordedRepoMappingEntries": [] + } + }, "@@rules_foreign_cc+//foreign_cc:extensions.bzl%tools": { "general": { "bzlTransitiveDigest": "b41JwymKkuHbovxSrZD6eawffiN8+iwKcSQZrLCkvbw=", @@ -1251,18 +1418,18 @@ "@@rules_zig+//zig:extensions.bzl%zig": { "general": { "bzlTransitiveDigest": "r7tCqbjlVKc7XEXV8Hiak2ElFfkjvDdMf8Ym+7zxukU=", - "usagesDigest": "lqvJAPUaAfUjthqXXAa4LwhcuZ2d3CNerCqKcLkkNPo=", + "usagesDigest": "DObSrV9vF3T1Diz4Mp2HSPgtKmxRPL/y5KPhcX6zjvU=", "recordedFileInputs": { - "@@//bazel/zig_index.json": "809b545c21f13c11c4587dccbfe81de2b767a292325c25ffb785b03c601bbc2e", + "@@//bazel/zig_index.json": "4d85811c879dbd2c11d1d7a40decbe25070abe49d4a1ba704b532999b246ad24", "@@rules_zig+//zig/private/versions.json": "6eb85ebaee72c4e6fbc82d8ad73f69e0899d839b70a7e19caa792ab6a5fa0c36" }, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { - "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-linux": { + "zig_0.16.0-dev.3153_Pd6f43caad_aarch64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.2722+f16eb18ce.tar.xz", + "url": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.3153+d6f43caad.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1279,16 +1446,16 @@ "https://fs.liujiacai.net/zigbuilds", "https://ziglang.org/builds/" ], - "sha256": "03725d297a2e2a8331d774eb601163af42ec61979b82753d41b2c460eefab1be", - "zig_version": "0.16.0-dev.2722+f16eb18ce", + "sha256": "e46037f7747492b405a42d40bb9bf3ca3f28c71cf9c1ccdf165a4de74cb484bb", + "zig_version": "0.16.0-dev.3153+d6f43caad", "platform": "aarch64-linux", "translate_c": "@@+non_module_deps+translate-c//:translate-c" } }, - "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-macos": { + "zig_0.16.0-dev.3153_Pd6f43caad_aarch64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.2722+f16eb18ce.tar.xz", + "url": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.3153+d6f43caad.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1305,16 +1472,16 @@ "https://fs.liujiacai.net/zigbuilds", "https://ziglang.org/builds/" ], - "sha256": "268aa8342e8ef539693e88e1d0982361b1d5dc4a080949264f2d5a14e5eb83e0", - "zig_version": "0.16.0-dev.2722+f16eb18ce", + "sha256": "9121e3f03515a3434ef9f362f8c541540968316d33ed6485b2d8a31dc7332fe6", + "zig_version": "0.16.0-dev.3153+d6f43caad", "platform": "aarch64-macos", "translate_c": "@@+non_module_deps+translate-c//:translate-c" } }, - "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-windows": { + "zig_0.16.0-dev.3153_Pd6f43caad_aarch64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.2722+f16eb18ce.zip", + "url": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.3153+d6f43caad.zip", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1331,16 +1498,16 @@ "https://fs.liujiacai.net/zigbuilds", "https://ziglang.org/builds/" ], - "sha256": "43daaad311bc3b06a733a8a87d2c3180214d62db2067df1591b921826de453ea", - "zig_version": "0.16.0-dev.2722+f16eb18ce", + "sha256": "b25c7046c111f5a4e614fae1a1b33c8db73fbc3de4c135acabb5740455a3b6fc", + "zig_version": "0.16.0-dev.3153+d6f43caad", "platform": "aarch64-windows", "translate_c": "@@+non_module_deps+translate-c//:translate-c" } }, - "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-linux": { + "zig_0.16.0-dev.3153_Pd6f43caad_x86_64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.2722+f16eb18ce.tar.xz", + "url": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.3153+d6f43caad.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1357,16 +1524,16 @@ "https://fs.liujiacai.net/zigbuilds", "https://ziglang.org/builds/" ], - "sha256": "b0fe8fd996b4974cd80fc0967460b13e3a6e3105c4bec19ea0566bd47d73ad81", - "zig_version": "0.16.0-dev.2722+f16eb18ce", + "sha256": "afb3016a94a8b5608d364e58ec7b01111af775a020d5f93f69eaebde2053fe77", + "zig_version": "0.16.0-dev.3153+d6f43caad", "platform": "x86_64-linux", "translate_c": "@@+non_module_deps+translate-c//:translate-c" } }, - "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-macos": { + "zig_0.16.0-dev.3153_Pd6f43caad_x86_64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.2722+f16eb18ce.tar.xz", + "url": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.3153+d6f43caad.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1383,16 +1550,16 @@ "https://fs.liujiacai.net/zigbuilds", "https://ziglang.org/builds/" ], - "sha256": "8e09fb9e90c0566629dfa52bce433e886f4cf7e2cd9985a358d7b8c473c4e9ce", - "zig_version": "0.16.0-dev.2722+f16eb18ce", + "sha256": "0251a8414f98513a33a7a7bd1f6cc13fb59f391168a742eee47b8c685e9827a0", + "zig_version": "0.16.0-dev.3153+d6f43caad", "platform": "x86_64-macos", "translate_c": "@@+non_module_deps+translate-c//:translate-c" } }, - "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-windows": { + "zig_0.16.0-dev.3153_Pd6f43caad_x86_64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.2722+f16eb18ce.zip", + "url": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.3153+d6f43caad.zip", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1409,8 +1576,8 @@ "https://fs.liujiacai.net/zigbuilds", "https://ziglang.org/builds/" ], - "sha256": "0f92e7234ed1d9608444487ca8fb928bd38da79dea744acf6c7a8fa0c440da6a", - "zig_version": "0.16.0-dev.2722+f16eb18ce", + "sha256": "ab31039adca06ae87c95c11b3ff8ed82808335866d58bc6e508cd3174d2d7e3d", + "zig_version": "0.16.0-dev.3153+d6f43caad", "platform": "x86_64-windows", "translate_c": "@@+non_module_deps+translate-c//:translate-c" } @@ -1419,28 +1586,28 @@ "repoRuleId": "@@rules_zig+//zig/private/repo:toolchains_repo.bzl%toolchains_repo", "attributes": { "names": [ - "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-linux", - "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-macos", - "zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-windows", - "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-linux", - "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-macos", - "zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-windows" + "zig_0.16.0-dev.3153_Pd6f43caad_aarch64-linux", + "zig_0.16.0-dev.3153_Pd6f43caad_aarch64-macos", + "zig_0.16.0-dev.3153_Pd6f43caad_aarch64-windows", + "zig_0.16.0-dev.3153_Pd6f43caad_x86_64-linux", + "zig_0.16.0-dev.3153_Pd6f43caad_x86_64-macos", + "zig_0.16.0-dev.3153_Pd6f43caad_x86_64-windows" ], "labels": [ - "@zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-linux//:zig_toolchain", - "@zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-macos//:zig_toolchain", - "@zig_0.16.0-dev.2722_Pf16eb18ce_aarch64-windows//:zig_toolchain", - "@zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-linux//:zig_toolchain", - "@zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-macos//:zig_toolchain", - "@zig_0.16.0-dev.2722_Pf16eb18ce_x86_64-windows//:zig_toolchain" + "@zig_0.16.0-dev.3153_Pd6f43caad_aarch64-linux//:zig_toolchain", + "@zig_0.16.0-dev.3153_Pd6f43caad_aarch64-macos//:zig_toolchain", + "@zig_0.16.0-dev.3153_Pd6f43caad_aarch64-windows//:zig_toolchain", + "@zig_0.16.0-dev.3153_Pd6f43caad_x86_64-linux//:zig_toolchain", + "@zig_0.16.0-dev.3153_Pd6f43caad_x86_64-macos//:zig_toolchain", + "@zig_0.16.0-dev.3153_Pd6f43caad_x86_64-windows//:zig_toolchain" ], "zig_versions": [ - "0.16.0-dev.2722+f16eb18ce", - "0.16.0-dev.2722+f16eb18ce", - "0.16.0-dev.2722+f16eb18ce", - "0.16.0-dev.2722+f16eb18ce", - "0.16.0-dev.2722+f16eb18ce", - "0.16.0-dev.2722+f16eb18ce" + "0.16.0-dev.3153+d6f43caad", + "0.16.0-dev.3153+d6f43caad", + "0.16.0-dev.3153+d6f43caad", + "0.16.0-dev.3153+d6f43caad", + "0.16.0-dev.3153+d6f43caad", + "0.16.0-dev.3153+d6f43caad" ], "exec_lengths": [ 2, diff --git a/bazel/zig_index.json b/bazel/zig_index.json index ff679151..11251b13 100644 --- a/bazel/zig_index.json +++ b/bazel/zig_index.json @@ -1,133 +1,148 @@ { "master": { - "version": "0.16.0-dev.2722+f16eb18ce", - "date": "2026-03-08", + "version": "0.16.0-dev.3153+d6f43caad", + "date": "2026-04-12", "docs": "https://ziglang.org/documentation/master/", "stdDocs": "https://ziglang.org/documentation/master/std/", "src": { - "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.2722+f16eb18ce.tar.xz", - "shasum": "1880ddb5502206d8f7322ca1e0138b67b3928fb88dbab3defff80d1dd8a311bb", - "size": "22460744" + "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "cb598f754d00a71af7586d437f490056b084eb6e54587fc2bb9f55148757fd49", + "size": "22477252" }, "bootstrap": { - "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.2722+f16eb18ce.tar.xz", - "shasum": "4e3f14dfdf73f9448fcb3eb7bf61c4e0b26f4e0949e572824fbd47ea62adf97e", - "size": "55225064" + "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "a2ccc95aaed569b3c25d8693f01187144f09e08a2a9b828bad3e5f2c19a4c2bc", + "size": "55240588" }, "x86_64-macos": { - "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.2722+f16eb18ce.tar.xz", - "shasum": "8e09fb9e90c0566629dfa52bce433e886f4cf7e2cd9985a358d7b8c473c4e9ce", - "size": "57648992" + "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "0251a8414f98513a33a7a7bd1f6cc13fb59f391168a742eee47b8c685e9827a0", + "size": "57381556" }, "aarch64-macos": { - "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.2722+f16eb18ce.tar.xz", - "shasum": "268aa8342e8ef539693e88e1d0982361b1d5dc4a080949264f2d5a14e5eb83e0", - "size": "52489044" + "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "9121e3f03515a3434ef9f362f8c541540968316d33ed6485b2d8a31dc7332fe6", + "size": "52217412" }, "x86_64-linux": { - "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.2722+f16eb18ce.tar.xz", - "shasum": "b0fe8fd996b4974cd80fc0967460b13e3a6e3105c4bec19ea0566bd47d73ad81", - "size": "55512692" + "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "afb3016a94a8b5608d364e58ec7b01111af775a020d5f93f69eaebde2053fe77", + "size": "55445192" }, "aarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.2722+f16eb18ce.tar.xz", - "shasum": "03725d297a2e2a8331d774eb601163af42ec61979b82753d41b2c460eefab1be", - "size": "51286740" + "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "e46037f7747492b405a42d40bb9bf3ca3f28c71cf9c1ccdf165a4de74cb484bb", + "size": "51193908" }, "arm-linux": { - "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.2722+f16eb18ce.tar.xz", - "shasum": "b4407543ef223d459d5925b2f1b8c700e61da3193a79fae3b6104c5116e092e7", - "size": "52149112" + "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "aec322b170a88c998c1c7b59c731e808543310008960b1654caab569567e7c95", + "size": "52093456" }, "riscv64-linux": { - "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.2722+f16eb18ce.tar.xz", - "shasum": "473e13775ef2eefe715b727b933462d2ee84cbdebb606cc8988aa70cb6ae72f1", - "size": "55352592" + "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "ad83132048c075edd1031a1e2fc9b29921e342f6b1d58142923e60e203a5b53a", + "size": "55303468" }, "powerpc64le-linux": { - "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.2722+f16eb18ce.tar.xz", - "shasum": "2382d40afa44218a961dddd935cb5ce1f6800d5c5dd9df39a27a25026e0b09de", - "size": "55387192" + "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "1866a073fdcd0773397d345b21f98221a20126f7e1cec298c16ff0f4ca3fb78f", + "size": "55241328" }, "x86-linux": { - "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.2722+f16eb18ce.tar.xz", - "shasum": "6cb242bd3e0a709a70c1b0e45a81afe1253b6d8367d411e7e25e464344f815b1", - "size": "58228456" + "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "6572c8768de14c0a0bbaf1a0401e19849681840594744a875ade93e4dc040edd", + "size": "58171580" }, "loongarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.2722+f16eb18ce.tar.xz", - "shasum": "5564d7f8323386df38327e72d0b515834487f72d2e7a34673d5195019b552a4d", - "size": "52590788" + "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "9bdf0f91187ede6b418b898d178521c4ce2065171b8bcfaa68cae0adf1d0a8f0", + "size": "52511048" }, "s390x-linux": { - "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.2722+f16eb18ce.tar.xz", - "shasum": "c7d74bcbbe8b6edb54e7498281b8efe76e7ddbdde9f7bef7699f39d020045bba", - "size": "55040152" + "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "c893299d0e486337390262ef480615d4f573da158f73a647eb104e10a2811f0d", + "size": "54926720" }, "x86_64-windows": { - "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.2722+f16eb18ce.zip", - "shasum": "0f92e7234ed1d9608444487ca8fb928bd38da79dea744acf6c7a8fa0c440da6a", - "size": "98134892" + "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.3153+d6f43caad.zip", + "shasum": "ab31039adca06ae87c95c11b3ff8ed82808335866d58bc6e508cd3174d2d7e3d", + "size": "97965399" }, "aarch64-windows": { - "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.2722+f16eb18ce.zip", - "shasum": "43daaad311bc3b06a733a8a87d2c3180214d62db2067df1591b921826de453ea", - "size": "94131162" + "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.3153+d6f43caad.zip", + "shasum": "b25c7046c111f5a4e614fae1a1b33c8db73fbc3de4c135acabb5740455a3b6fc", + "size": "93857928" }, "x86-windows": { - "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.2722+f16eb18ce.zip", - "shasum": "3e3c3224040329bdc7120a1d16a68117f82572a26985234cb6e68bd800eed11b", - "size": "99937580" + "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.3153+d6f43caad.zip", + "shasum": "51d579299617bd5fae4d0333d6f2b06a8078c1a6f2f6db2dd08289236fad70d4", + "size": "99767726" }, "aarch64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.2722+f16eb18ce.tar.xz", - "shasum": "fd84224ef223721ce7ee45d31296ec520c0de34ae97c035a4927370d67e955a1", - "size": "51224920" + "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "fe2a97ed29641cc214d58fa2c054fb3ad3a7120b7b43241e6b833c0978187bb6", + "size": "51125424" }, "arm-freebsd": { - "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.2722+f16eb18ce.tar.xz", - "shasum": "01446e45de800ef17f1e8076a3765a51be91c8aa3630166bf8722104e7bf9db2", - "size": "52823280" - }, - "powerpc64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-powerpc64-freebsd-0.16.0-dev.2722+f16eb18ce.tar.xz", - "shasum": "7b68f44fb289651b09318b45e754413f2473ae5956736307f5b55a6d89745ebf", - "size": "53947616" + "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "0aa627f73f0bc9c100394d512a375188afb8d33168409050d07b0c8b87adbc0b", + "size": "52702132" }, "powerpc64le-freebsd": { - "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.2722+f16eb18ce.tar.xz", - "shasum": "a77432752e3f9d2746a99b973563a6439bf1908b512a6ad6b74ed8b0bd02c999", - "size": "55402492" + "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "18500e3b2e742c07541b2d35377e3dc770758ea41e9a3bf09535eacad5266583", + "size": "55254720" }, "riscv64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.2722+f16eb18ce.tar.xz", - "shasum": "d75e1c261075c64a295ae92c28a4871c6ea13541f2604e9399effb31a1473e3a", - "size": "55496140" + "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "593c19b8feb76a2e66c6d49f3a96390eed22fc4e2c780a96429d202bf4912afc", + "size": "55393696" }, "x86_64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.2722+f16eb18ce.tar.xz", - "shasum": "edd6ba3241cc0475f177ac2e4bcba692193259c30d0b5da288a4d76f427e0737", - "size": "55622060" + "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "b043fa6731e205f208d2e50fa84d1d79c7060e04688a4f5e3bff6dace337d5f8", + "size": "55587512" }, "aarch64-netbsd": { - "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.2722+f16eb18ce.tar.xz", - "shasum": "0499caf08e5bd91bb4201b4cad68faa71be016edab9dbbd25454721549d61bfb", - "size": "51189128" + "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "c1625244d6fe4008feb27d8bc7bf49739da98d04bc961295124a4e8732cf3b18", + "size": "51092376" }, "arm-netbsd": { - "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.2722+f16eb18ce.tar.xz", - "shasum": "46a90dc70bc59384adf9e5f2d70da6119dd4414c8a7e696309313d3b8f9e3022", - "size": "53863744" + "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "582116aeee90ea3d38ede6901f93138e543a07c6dc32d2645bf0b4e6fee813c6", + "size": "53732592" }, "x86-netbsd": { - "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.2722+f16eb18ce.tar.xz", - "shasum": "92c420f1291b9f8cd4bb423000cca24403ef6dea03ec9f468e81fc20554d0db3", - "size": "58827296" + "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "eba2eb8e8efe0748820fcedb4066233a96406f97c3fa7b326e220f7c3a9e07f3", + "size": "58765376" }, "x86_64-netbsd": { - "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.2722+f16eb18ce.tar.xz", - "shasum": "5b0940ac45f96718ef697de9f5edcb66bd7985928aad2de91004d807a8bca07a", - "size": "55576856" + "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "281d172d159326167a75f2bc87fbe18afe4e37a26b5a1440cd65c8eef952626a", + "size": "55515836" + }, + "aarch64-openbsd": { + "tarball": "https://ziglang.org/builds/zig-aarch64-openbsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "1d750bb3ed57ff4a83b1411a07df7e38f86a0a611123fcbede1ed84a48937585", + "size": "51551144" + }, + "arm-openbsd": { + "tarball": "https://ziglang.org/builds/zig-arm-openbsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "2bd936eff1c2fadd5aa842e30f35f8509ee07f980bcb42a407940b1ae8fe5f29", + "size": "52270724" + }, + "riscv64-openbsd": { + "tarball": "https://ziglang.org/builds/zig-riscv64-openbsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "c2067369e07e991f2e1cfe8f657c9185d726487a4ab4cbc1fde9768c8fae363d", + "size": "55713660" + }, + "x86_64-openbsd": { + "tarball": "https://ziglang.org/builds/zig-x86_64-openbsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "01122836baaa88627de2e82addf7cdcdcfb3ba104ae9d051d92e36aaad4f4e27", + "size": "56927908" } }, "0.15.2": { diff --git a/bin/zml-smi/platforms/amdsmi/amdsmi.zig b/bin/zml-smi/platforms/amdsmi/amdsmi.zig index 06656501..538660e8 100644 --- a/bin/zml-smi/platforms/amdsmi/amdsmi.zig +++ b/bin/zml-smi/platforms/amdsmi/amdsmi.zig @@ -56,7 +56,7 @@ pub fn init(allocator: std.mem.Allocator) !AmdSmi { defer allocator.free(sockets); try check(fns.amdsmi_get_socket_handles(&socket_count, @ptrCast(sockets.ptr))); - var handle_list: std.ArrayList(c.amdsmi_processor_handle) = .{}; + var handle_list: std.ArrayList(c.amdsmi_processor_handle) = .empty; defer handle_list.deinit(allocator); for (sockets[0..socket_count]) |socket| { diff --git a/bin/zml-smi/platforms/neuron/metrics.zig b/bin/zml-smi/platforms/neuron/metrics.zig index b0bf5ac2..0b5a5ddd 100644 --- a/bin/zml-smi/platforms/neuron/metrics.zig +++ b/bin/zml-smi/platforms/neuron/metrics.zig @@ -29,8 +29,8 @@ pub fn start(collector: *Collector) !void { for (util_buf) |*u| u.* = .init(0); const GiB: u64 = 1024 * 1024 * 1024; - var neuron_infos: std.ArrayList(*DeviceInfo) = .{}; - const dev_offset: u16 = @intCast(collector.device_infos.items.len); + var neuron_infos: std.ArrayList(*DeviceInfo) = .empty; + const dev_offset: u8 = @intCast(collector.device_infos.items.len); var ver_buf: [Nrt.version_buf_len]u8 = undefined; const nrt_ver: ?[]const u8 = if (nrt.version(&ver_buf)) |v| diff --git a/bin/zml-smi/platforms/neuron/nrt.zig b/bin/zml-smi/platforms/neuron/nrt.zig index e5236cff..259300f0 100644 --- a/bin/zml-smi/platforms/neuron/nrt.zig +++ b/bin/zml-smi/platforms/neuron/nrt.zig @@ -65,8 +65,8 @@ pub fn init(allocator: std.mem.Allocator, io: std.Io) !Nrt { var dev_index_buf: [c.MAX_NEURON_DEVICE_COUNT]c_int = undefined; const count: usize = @intCast(@max(0, private_fns.ndl_available_devices(&dev_index_buf, c.MAX_NEURON_DEVICE_COUNT))); - var handle_list: std.ArrayList(*c.ndl_device_t) = .{}; - var index_list: std.ArrayList(c_int) = .{}; + var handle_list: std.ArrayList(*c.ndl_device_t) = .empty; + var index_list: std.ArrayList(c_int) = .empty; for (dev_index_buf[0..count]) |device_idx| { var dev: ?*c.ndl_device_t = null; diff --git a/bin/zml-smi/platforms/tpu/metrics.zig b/bin/zml-smi/platforms/tpu/metrics.zig index d4248f95..9410b3f8 100644 --- a/bin/zml-smi/platforms/tpu/metrics.zig +++ b/bin/zml-smi/platforms/tpu/metrics.zig @@ -12,8 +12,8 @@ const address = "localhost:8431"; pub fn start(collector: *Collector) !void { const chip = scanPciChips(collector.arena, collector.io) orelse return error.TpuUnavailable; const device_count = chip.chip_count * chip.devices_per_chip; - var tpu_infos: std.ArrayList(*DeviceInfo) = .{}; - const dev_offset: u16 = @intCast(collector.device_infos.items.len); + var tpu_infos: std.ArrayList(*DeviceInfo) = .empty; + const dev_offset: u8 = @intCast(collector.device_infos.items.len); for (0..device_count) |_| { const chip_name = try collector.arena.dupe(u8, chip.name); diff --git a/bin/zml-smi/tui/data.zig b/bin/zml-smi/tui/data.zig index ddd0f0eb..fb925f68 100644 --- a/bin/zml-smi/tui/data.zig +++ b/bin/zml-smi/tui/data.zig @@ -58,7 +58,7 @@ pub const HistoryBuffers = struct { power: []RingBuffer(u64, history_len) = &.{}, pub fn init(allocator: std.mem.Allocator, count: usize) !HistoryBuffers { - var h: HistoryBuffers = .{ + const h: HistoryBuffers = .{ .util = try allocator.alloc(RingBuffer(u64, history_len), count), .mem_util = try allocator.alloc(RingBuffer(u64, history_len), count), .temp = try allocator.alloc(RingBuffer(u64, history_len), count), diff --git a/bin/zml-smi/tui/widgets/process_table.zig b/bin/zml-smi/tui/widgets/process_table.zig index 5cc08ef6..6a2d57aa 100644 --- a/bin/zml-smi/tui/widgets/process_table.zig +++ b/bin/zml-smi/tui/widgets/process_table.zig @@ -24,7 +24,7 @@ scroll_bars: vxfw.ScrollBars = .{ }, .draw_horizontal_scrollbar = false, }, -merged: std.ArrayList(ProcessInfo) = .{}, +merged: std.ArrayList(ProcessInfo) = .empty, pub fn prepare(self: *ProcessTable, state: *data.SystemState, device_id: ?u16) void { self.scroll_bars.scroll_view.children.builder.userdata = self; diff --git a/bin/zml-smi/tui/widgets/status_line.zig b/bin/zml-smi/tui/widgets/status_line.zig index 59290ef6..fb32541e 100644 --- a/bin/zml-smi/tui/widgets/status_line.zig +++ b/bin/zml-smi/tui/widgets/status_line.zig @@ -62,7 +62,7 @@ pub fn draw(self: *const StatusLine, ctx: vxfw.DrawContext) std.mem.Allocator.Er .softwrap = false, .overflow = .clip, }; - var surf = try rich.draw(ui.fixedSize(ctx, w, 1)); + const surf = try rich.draw(ui.fixedSize(ctx, w, 1)); // Fill background across full width if (surf.buffer.len > 0) { diff --git a/examples/io/main.zig b/examples/io/main.zig index 53280396..3bb823a0 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -139,7 +139,7 @@ pub fn main(init: std.process.Init) !void { var current = root; var parts = std.mem.tokenizeScalar(u8, name, '.'); while (parts.next()) |part| { - const gop = try current.children.getOrPut(part); + const gop = try current.children.getOrPut(current.allocator, part); if (!gop.found_existing) { gop.value_ptr.* = try TensorNode.init(init.arena.allocator(), part); } @@ -274,15 +274,15 @@ fn printTree(io: std.Io, writer: *std.Io.Writer, dir: std.Io.Dir, prefix: []cons const TensorNode = struct { name: []const u8, - children: std.StringArrayHashMap(*TensorNode), + children: std.StringArrayHashMapUnmanaged(*TensorNode) = .empty, + allocator: std.mem.Allocator, tensor: ?zml.safetensors.Tensor = null, fn init(allocator: std.mem.Allocator, name: []const u8) !*TensorNode { const node = try allocator.create(TensorNode); node.* = .{ .name = name, - .children = std.StringArrayHashMap(*TensorNode).init(allocator), - .tensor = null, + .allocator = allocator, }; return node; } @@ -314,7 +314,7 @@ fn printTensorTree( is_last: bool, is_root: bool, ) !void { - const allocator = node.children.allocator; + const allocator = node.allocator; if (is_root) { const children_nodes = try getSortedChildren(allocator, node); diff --git a/platforms/neuron/libpjrt_neuron.zig b/platforms/neuron/libpjrt_neuron.zig index b132de2f..1cfaafc9 100644 --- a/platforms/neuron/libpjrt_neuron.zig +++ b/platforms/neuron/libpjrt_neuron.zig @@ -8,8 +8,9 @@ const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/platforms/neuron"); fn findFreeTcpPort(io: std.Io) !u16 { + const addr: std.Io.net.IpAddress = .{ .ip4 = .{ .bytes = .{ 127, 0, 0, 1 }, .port = 0 } }; var server = try std.Io.net.IpAddress.listen( - .{ .ip4 = .{ .bytes = .{ 127, 0, 0, 1 }, .port = 0 } }, + &addr, io, .{}, ); diff --git a/third_party/arocc/repo.bzl b/third_party/arocc/repo.bzl index 627dca69..10ff796b 100644 --- a/third_party/arocc/repo.bzl +++ b/third_party/arocc/repo.bzl @@ -4,6 +4,6 @@ def repo(): new_git_repository( name = "arocc", remote = "https://github.com/Vexu/arocc.git", - commit = "a08e2ca4b0e83059c3892b8b3e026754467e0ed1", + commit = "5f5a050569a95ecc40a426f0c3666ae7ef987ede", build_file = "//third_party/arocc:arocc.bazel", ) diff --git a/third_party/flashattn/repo.bzl b/third_party/flashattn/repo.bzl index ced98a5b..4fabc298 100644 --- a/third_party/flashattn/repo.bzl +++ b/third_party/flashattn/repo.bzl @@ -4,8 +4,8 @@ load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") def repo(): http_archive( name = "flashattn", - url = "https://github.com/zml/flash-attention/releases/download/v0.0.6-rc7/flashattn.tar.gz", - sha256 = "6b8195a5ab285404fcb084122c1c32f342ff695040b8652c9144afd2289b71b3", + url = "https://github.com/zml/flash-attention/releases/download/v0.0.6-rc8/flashattn.tar.gz", + sha256 = "360caace18b84f03779fe7af8aa1cdff64e99b6c17cbe05774be100aaf23b579", build_file = "//:third_party/flashattn/flashattention.BUILD.bazel", ) diff --git a/third_party/libvaxis/fixes.patch b/third_party/libvaxis/fixes.patch index c31bcf68..aca9c136 100644 --- a/third_party/libvaxis/fixes.patch +++ b/third_party/libvaxis/fixes.patch @@ -191,6 +191,25 @@ index fc3962f..2661cf4 100644 const vaxis = @import("../main.zig"); const vxfw = @import("vxfw.zig"); +@@ -36,8 +37,8 @@ fn init(allocator: Allocator, io: std.Io) !App { + .kitty_keyboard_flags = .{ + .report_events = true, + }, + }), +- .timers = std.ArrayList(vxfw.Tick){}, ++ .timers = .empty, + .wants_focus = null, + .io = io, + }; +@@ -102,7 +103,7 @@ + var ctx: vxfw.EventContext = .{ + .alloc = self.allocator, + .phase = .capturing, +- .cmds = vxfw.CommandList{}, ++ .cmds = .empty, + .io = io, + .consume_event = false, + .redraw = false, @@ -123,6 +124,13 @@ pub fn run(self: *App, widget: vxfw.Widget, opts: Options) anyerror!void { try self.checkTimers(&ctx); @@ -205,6 +224,33 @@ index fc3962f..2661cf4 100644 { loop.queue.lock(); defer loop.queue.unlock(); +@@ -342,7 +350,7 @@ + // For mouse events we store the last frame and use that for hit testing + const last_frame = surface; + +- var hits = std.ArrayList(vxfw.HitResult){}; ++ var hits: std.ArrayList(vxfw.HitResult) = .empty; + defer hits.deinit(app.allocator); + const sub: vxfw.SubSurface = .{ + .origin = .{ .row = 0, .col = 0 }, +@@ -405,7 +413,7 @@ + const last_frame = self.last_frame; + self.mouse = mouse; + +- var hits = std.ArrayList(vxfw.HitResult){}; ++ var hits: std.ArrayList(vxfw.HitResult) = .empty; + defer hits.deinit(app.allocator); + const sub: vxfw.SubSurface = .{ + .origin = .{ .row = 0, .col = 0 }, +@@ -524,7 +532,7 @@ + return .{ + .root = root, + .focused_widget = root, +- .path_to_focused = std.ArrayList(Widget){}, ++ .path_to_focused = .empty, + }; + } + diff --git a/src/vxfw/Padding.zig b/src/vxfw/Padding.zig index e60162f..6e5f40b 100644 --- a/src/vxfw/Padding.zig diff --git a/third_party/uucode/build_config.zig b/third_party/uucode/build_config.zig index 2a656ae9..a4e76bff 100644 --- a/third_party/uucode/build_config.zig +++ b/third_party/uucode/build_config.zig @@ -1,18 +1,16 @@ const config = @import("config.zig"); -const config_x = @import("config.x.zig"); -pub const tables = [_]config.Table{ +pub const fields = config.fields; +pub const build_components = config.build_components; +pub const get_components = config.get_components; + +pub const tables: []const config.Table = &.{ .{ - .extensions = &.{}, - .fields = &config._resolveFields( - config_x, - &.{ - "east_asian_width", - "grapheme_break", - "general_category", - "is_emoji_presentation", - }, - &.{}, - ), + .fields = &.{ + "east_asian_width", + "grapheme_break", + "general_category", + "is_emoji_presentation", + }, }, }; diff --git a/third_party/uucode/repo.bzl b/third_party/uucode/repo.bzl index 034fa2c5..6b913bbc 100644 --- a/third_party/uucode/repo.bzl +++ b/third_party/uucode/repo.bzl @@ -19,7 +19,7 @@ _uucode_repo = repository_rule( def repo(): _uucode_repo( name = "uucode", - commit = "faab8894d753207878283d523add01817e23b805", + commit = "f1c687f09174f9f11eda14d3e3ce497d68ab09a6", build_config = "//third_party/uucode:build_config.zig", build_file = "//third_party/uucode:uucode.bazel", ) diff --git a/third_party/uucode/uucode.bazel b/third_party/uucode/uucode.bazel index 8b152287..67b7eb39 100644 --- a/third_party/uucode/uucode.bazel +++ b/third_party/uucode/uucode.bazel @@ -1,4 +1,3 @@ -load("@bazel_skylib//rules:run_binary.bzl", "run_binary") load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") # Generator @@ -7,52 +6,34 @@ zig_library( name = "_config_gen", import_name = "config.zig", main = "src/config.zig", - srcs = ["src/quirks.zig"], - zigopts = ["--dep", "types.zig=_types_gen"], # circular: config.zig <-> types.zig -) - -zig_library( - name = "_types_gen", - import_name = "types.zig", - main = "src/types.zig", - srcs = ["src/get.zig"], - deps = [":_config_gen"], -) - -zig_library( - name = "_config_x_gen", - import_name = "config.x.zig", - main = "src/x/config.x.zig", srcs = [ - "src/x/config_x/grapheme_break.zig", - "src/x/config_x/wcwidth.zig", - ], - deps = [ - ":_config_gen", - ":_types_gen", - ":types_x", + "src/components.zig", + "src/fields.zig", + "src/multi_slice.zig", + "src/quirks.zig", + "src/types.zig", ], + zigopts = ["--dep", "storage.zig=_storage_gen"], # circular: config.zig <-> storage.zig +) + +zig_library( + name = "_storage_gen", + import_name = "storage.zig", + main = "src/storage.zig", + deps = [":_config_gen"], ) zig_library( name = "_build_config_gen", import_name = "build_config", main = "build_config.zig", - deps = [ - ":_config_gen", - ":_config_x_gen", - ], + deps = [":_config_gen", ":_storage_gen"], ) zig_binary( name = "gen_tables", - main = "src/build/tables.zig", - srcs = ["src/build/Ucd.zig"], - deps = [ - ":_build_config_gen", - ":_config_gen", - ":_types_gen", - ], + main = "src/generate.zig", + deps = [":_build_config_gen", ":_config_gen", ":_storage_gen"], ) genrule( @@ -65,74 +46,45 @@ genrule( # Library -zig_library( - name = "config", - import_name = "config.zig", - main = "src/config.zig", - srcs = ["src/quirks.zig"], - zigopts = ["--dep", "types.zig=types"], # circular: config.zig <-> types.zig -) - zig_library( name = "types", import_name = "types.zig", main = "src/types.zig", - deps = [":config"], - zigopts = ["--dep", "get.zig=get"], # circular: types.zig -> get.zig -> tables -> types.zig ) zig_library( - name = "types_x", - import_name = "types.x.zig", - main = "src/x/types.x.zig", - srcs = ["src/x/types_x/grapheme.zig"], -) - -zig_library( - name = "config_x", - import_name = "config.x.zig", - main = "src/x/config.x.zig", + name = "config", + import_name = "config.zig", + main = "src/config.zig", srcs = [ - "src/x/config_x/grapheme_break.zig", - "src/x/config_x/wcwidth.zig", - ], - deps = [ - ":config", - ":types", - ":types_x", + "src/components.zig", + "src/fields.zig", + "src/multi_slice.zig", + "src/quirks.zig", ], + deps = [":types"], + zigopts = ["--dep", "storage.zig=storage"], # circular: config.zig <-> storage.zig +) + +zig_library( + name = "storage", + import_name = "storage.zig", + main = "src/storage.zig", + deps = [":config"], ) zig_library( name = "build_config", import_name = "build_config", main = "build_config.zig", - deps = [ - ":config", - ":config_x", - ], + deps = [":config", ":storage"], ) zig_library( name = "tables_lib", import_name = "tables", main = ":tables", - deps = [ - ":build_config", - ":config", - ":types", - ":types_x", - ], -) - -zig_library( - name = "get", - import_name = "get.zig", - main = "src/get.zig", - deps = [ - ":tables_lib", - ":types", - ], + deps = [":build_config", ":config", ":storage"], ) zig_library( @@ -141,19 +93,11 @@ zig_library( main = "src/root.zig", srcs = [ "src/ascii.zig", - "src/build/Ucd.zig", "src/code_point.zig", + "src/get.zig", "src/grapheme.zig", "src/utf8.zig", - "src/x/grapheme.zig", - "src/x/root.zig", - ], - deps = [ - ":config", - ":get", - ":tables_lib", - ":types", - ":types_x", ], + deps = [":config", ":tables_lib", ":types"], visibility = ["//visibility:public"], ) diff --git a/third_party/zigimg/bump-zig.patch b/third_party/zigimg/bump-zig.patch index e6367c9c..b668dadc 100644 --- a/third_party/zigimg/bump-zig.patch +++ b/third_party/zigimg/bump-zig.patch @@ -37,6 +37,15 @@ index d67cd61..6472129 100644 error{ EndOfStream, InvalidData, UnfinishedBits }; pub const ConvertError = Error || +@@ -101,7 +101,7 @@ + }; + + pub const Animation = struct { +- frames: FrameList = .{}, ++ frames: FrameList = .empty, + loop_count: i32 = AnimationLoopInfinite, + + pub const FrameList = std.ArrayList(AnimationFrame); @@ -150,16 +150,16 @@ pub fn deinit(self: *Image, allocator: std.mem.Allocator) void { } @@ -295,6 +304,14 @@ diff --git a/src/formats/pam.zig b/src/formats/pam.zig index f366d95..7ef9833 100644 --- a/src/formats/pam.zig +++ b/src/formats/pam.zig +@@ -70,7 +70,7 @@ + var maybe_maxval: ?u16 = null; + var maybe_tuple_type: ?TupleType = null; +- var comments = std.ArrayListUnmanaged([]const u8){}; ++ var comments: std.ArrayListUnmanaged([]const u8) = .empty; + defer { + for (comments.items) |comment| allocator.free(comment); + comments.deinit(allocator); @@ -77,7 +77,7 @@ const Header = struct { } @@ -4336,3 +4353,139 @@ index 9f6b1a8..a53b23c 100644 - }) |source_file| std.testing.refAllDeclsRecursive(source_file); + }) |source_file| std.testing.refAllDecls(source_file); } +diff --git a/src/formats/gif.zig b/src/formats/gif.zig +--- a/src/formats/gif.zig ++++ b/src/formats/gif.zig +@@ -8,7 +8,7 @@ + const std = @import("std"); + const utils = @import("../utils.zig"); + +-pub const HeaderFlags = packed struct { ++pub const HeaderFlags = packed struct(u8) { + global_color_table_size: u3 = 0, + sorted: bool = false, + color_resolution: u3 = 0, +@@ -129,9 +129,9 @@ + pub const GIF = struct { + header: Header = .{}, + global_color_table: utils.FixedStorage(color.Rgb24, 256) = .{}, +- frames: std.ArrayList(FrameData) = .{}, +- comments: std.ArrayList(CommentExtension) = .{}, +- application_infos: std.ArrayList(ApplicationExtension) = .{}, ++ frames: std.ArrayList(FrameData) = .empty, ++ comments: std.ArrayList(CommentExtension) = .empty, ++ application_infos: std.ArrayList(ApplicationExtension) = .empty, + arena_allocator: std.heap.ArenaAllocator = undefined, + + pub const SubImage = struct { +@@ -147,6 +147,6 @@ + pub const FrameData = struct { + graphics_control: ?GraphicControlExtension = null, +- sub_images: std.ArrayList(*SubImage) = .{}, ++ sub_images: std.ArrayList(*SubImage) = .empty, + + pub fn deinit(self: *FrameData, allocator: std.mem.Allocator) void { + for (self.sub_images.items) |sub_image| { +@@ -683,7 +683,7 @@ + const final_pixel_format = self.findBestPixelFormat(); + + const frame_list_allocator = self.arena_allocator.child_allocator; + +- var frame_list = Image.Animation.FrameList{}; ++ var frame_list: Image.Animation.FrameList = .empty; + + if (self.frames.items.len == 0) { +diff --git a/src/compressions/lzw.zig b/src/compressions/lzw.zig +--- a/src/compressions/lzw.zig ++++ b/src/compressions/lzw.zig +@@ -11,9 +11,10 @@ + next_code: u13 = 0, + previous_code: ?u13 = null, +- dictionary: std.AutoArrayHashMap(u13, []const u8), ++ dictionary: std.AutoArrayHashMapUnmanaged(u13, []const u8) = .empty, ++ dict_allocator: std.mem.Allocator, + + remaining_data: ?u13 = null, + remaining_bits: u4 = 0, +@@ -27,7 +28,7 @@ + var result = Self{ + .area_allocator = std.heap.ArenaAllocator.init(allocator), + .code_size = initial_code_size, +- .dictionary = std.AutoArrayHashMap(u13, []const u8).init(allocator), ++ .dict_allocator = allocator, + .initial_code_size = initial_code_size, + .clear_code = @as(u13, 1) << @intCast(initial_code_size), + .end_information_code = (@as(u13, 1) << @intCast(initial_code_size)) + 1, +@@ -43,7 +44,7 @@ + + pub fn deinit(self: *Self) void { + self.area_allocator.deinit(); +- self.dictionary.deinit(); ++ self.dictionary.deinit(self.dict_allocator); + } + + pub fn decode(self: *Self, reader: *std.Io.Reader, writer: *std.Io.Writer) !void { +@@ -85,7 +86,7 @@ + var new_value = try allocator.alloc(u8, previous_value.len + 1); + std.mem.copyForwards(u8, new_value, previous_value); + new_value[previous_value.len] = value[0]; +- try self.dictionary.put(self.next_code, new_value); ++ try self.dictionary.put(self.dict_allocator, self.next_code, new_value); + + self.next_code += 1; + +@@ -109,7 +110,7 @@ + var new_value = try allocator.alloc(u8, previous_value.len + 1); + std.mem.copyForwards(u8, new_value, previous_value); + new_value[previous_value.len] = previous_value[0]; +- try self.dictionary.put(self.next_code, new_value); ++ try self.dictionary.put(self.dict_allocator, self.next_code, new_value); + + _ = try writer.write(new_value); + +@@ -154,7 +155,7 @@ + var data = try allocator.alloc(u8, 1); + data[0] = @as(u8, @truncate(index)); + +- try self.dictionary.put(index, data); ++ try self.dictionary.put(self.dict_allocator, index, data); + } + } + }; +diff --git a/src/formats/jpeg/huffman.zig b/src/formats/jpeg/huffman.zig +--- a/src/formats/jpeg/huffman.zig ++++ b/src/formats/jpeg/huffman.zig +@@ -9,7 +9,7 @@ + + const HuffmanCode = struct { length_minus_one: u4, code: u16 }; +-const HuffmanCodeMap = std.AutoArrayHashMap(HuffmanCode, u8); ++const HuffmanCodeMap = std.AutoArrayHashMapUnmanaged(HuffmanCode, u8); + + const JPEG_DEBUG = false; + const JPEG_VERY_DEBUG = false; +@@ -40,8 +40,8 @@ + for (code_counts) |count| total_huffman_codes += count; + +- var huffman_code_map = HuffmanCodeMap.init(allocator); +- errdefer huffman_code_map.deinit(); ++ var huffman_code_map: HuffmanCodeMap = .empty; ++ errdefer huffman_code_map.deinit(allocator); + + var fast_table: [1 << fast_bits]u8 = @splat(255); + var fast_size: [1 << fast_bits]u5 = @splat(0); +@@ -65,7 +65,7 @@ + + const byte = try reader.takeByte(); +- try huffman_code_map.put(.{ .length_minus_one = @as(u4, @intCast(i)), .code = code }, byte); ++ try huffman_code_map.put(allocator, .{ .length_minus_one = @as(u4, @intCast(i)), .code = code }, byte); + + // construct accelaration structure see stb_image + if (i + 1 <= fast_bits) { +@@ -96,5 +96,5 @@ + + pub fn deinit(self: *Table) void { +- self.code_map.deinit(); ++ self.code_map.deinit(self.allocator); + } + }; + diff --git a/zml/io.zig b/zml/io.zig index 97d0fbe1..5cc755e4 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -591,9 +591,9 @@ pub const DirectMemoryWriter = struct { allocator: std.mem.Allocator, shape: Shape, placement: Placement, - raw_segments: std.ArrayListUnmanaged(RawSegment) = .{}, - segments: std.ArrayListUnmanaged(StreamSegment) = .{}, - mirrors: std.ArrayListUnmanaged(usize) = .{}, + raw_segments: std.ArrayListUnmanaged(RawSegment) = .empty, + segments: std.ArrayListUnmanaged(StreamSegment) = .empty, + mirrors: std.ArrayListUnmanaged(usize) = .empty, fn init(allocator: std.mem.Allocator, shape: Shape, placement: Placement) StreamPlanner { return .{ @@ -617,6 +617,7 @@ pub const DirectMemoryWriter = struct { const owned_segments = try self.segments.toOwnedSlice(self.allocator); errdefer self.allocator.free(owned_segments); const owned_mirrors = try self.mirrors.toOwnedSlice(self.allocator); + errdefer self.allocator.free(owned_mirrors); return .{ .segments = owned_segments, diff --git a/zml/io/vfs.zig b/zml/io/vfs.zig index e1cec6f2..45ff4e72 100644 --- a/zml/io/vfs.zig +++ b/zml/io/vfs.zig @@ -23,15 +23,15 @@ pub const VFS = struct { mutex: std.Io.Mutex = .init, backends: std.StringArrayHashMapUnmanaged(std.Io) = .empty, - handles: Handles = .{}, - closed_handles: ClosedHandles = .{}, + handles: Handles = .empty, + closed_handles: ClosedHandles = .empty, base: VFSBase, pub fn init(allocator: std.mem.Allocator, base_io: std.Io) !VFS { const base = VFSBase.init(base_io); - var handles: Handles = .{}; + var handles: Handles = .empty; try handles.append(allocator, .{ .handle = CWD_HANDLE, .backend_idx = null }); try handles.append(allocator, .{ .handle = std.posix.STDIN_FILENO, .backend_idx = null }); try handles.append(allocator, .{ .handle = std.posix.STDOUT_FILENO, .backend_idx = null }); @@ -176,7 +176,7 @@ pub const VFS = struct { .data = o.data, } }); }, - .device_io_control, .file_write_streaming => { + .device_io_control, .file_write_streaming, .net_receive => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -228,7 +228,7 @@ pub const VFS = struct { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { log.err("Failed to lookup backend for dir create file '{s}' : {any}", .{ sub_path, err }); - return std.Io.Dir.AccessError.Unexpected; + return std.Io.File.OpenError.Unexpected; }; const file = try backend.vtable.dirCreateFile(backend.userdata, dir_, stripScheme(sub_path), flags); @@ -271,7 +271,7 @@ pub const VFS = struct { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); _, const dir_, const backend = self.lookupDir(reader.dir, null) catch |err| { log.err("Failed to lookup backend for dir real path : {any}", .{err}); - return std.Io.Dir.RealPathError.Unexpected; + return std.Io.Dir.Reader.Error.Unexpected; }; const original_dir = reader.dir; diff --git a/zml/io/vfs/base.zig b/zml/io/vfs/base.zig index 7042c883..33d79509 100644 --- a/zml/io/vfs/base.zig +++ b/zml/io/vfs/base.zig @@ -91,6 +91,7 @@ pub const VFSBase = struct { .unlockStderr = unlockStderr, .processCurrentPath = processCurrentPath, .processSetCurrentDir = processSetCurrentDir, + .processSetCurrentPath = processSetCurrentPath, .processReplace = processReplace, .processReplacePath = processReplacePath, .processSpawn = processSpawn, @@ -111,7 +112,6 @@ pub const VFSBase = struct { .netConnectUnix = netConnectUnix, .netSocketCreatePair = netSocketCreatePair, .netSend = netSend, - .netReceive = netReceive, .netRead = netRead, .netWrite = netWrite, .netWriteFile = netWriteFile, @@ -531,6 +531,11 @@ pub const VFSBase = struct { return self.inner.vtable.processSetCurrentDir(self.inner.userdata, dir); } + pub fn processSetCurrentPath(userdata: ?*anyopaque, path: []const u8) std.process.SetCurrentPathError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.processSetCurrentPath(self.inner.userdata, path); + } + pub fn processReplace(userdata: ?*anyopaque, options: std.process.ReplaceOptions) std.process.ReplaceError { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.processReplace(self.inner.userdata, options); @@ -591,14 +596,14 @@ pub const VFSBase = struct { return self.inner.vtable.randomSecure(self.inner.userdata, buffer); } - pub fn netListenIp(userdata: ?*anyopaque, address: std.Io.net.IpAddress, options: std.Io.net.IpAddress.ListenOptions) std.Io.net.IpAddress.ListenError!std.Io.net.Server { + pub fn netListenIp(userdata: ?*anyopaque, address: *const std.Io.net.IpAddress, options: std.Io.net.IpAddress.ListenOptions) std.Io.net.IpAddress.ListenError!std.Io.net.Socket { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netListenIp(self.inner.userdata, address, options); } - pub fn netAccept(userdata: ?*anyopaque, server: std.Io.net.Socket.Handle) std.Io.net.Server.AcceptError!std.Io.net.Stream { + pub fn netAccept(userdata: ?*anyopaque, server: std.Io.net.Socket.Handle, options: std.Io.net.Server.AcceptOptions) std.Io.net.Server.AcceptError!std.Io.net.Socket { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.netAccept(self.inner.userdata, server); + return self.inner.vtable.netAccept(self.inner.userdata, server, options); } pub fn netBindIp(userdata: ?*anyopaque, address: *const std.Io.net.IpAddress, options: std.Io.net.IpAddress.BindOptions) std.Io.net.IpAddress.BindError!std.Io.net.Socket { @@ -606,7 +611,7 @@ pub const VFSBase = struct { return self.inner.vtable.netBindIp(self.inner.userdata, address, options); } - pub fn netConnectIp(userdata: ?*anyopaque, address: *const std.Io.net.IpAddress, options: std.Io.net.IpAddress.ConnectOptions) std.Io.net.IpAddress.ConnectError!std.Io.net.Stream { + pub fn netConnectIp(userdata: ?*anyopaque, address: *const std.Io.net.IpAddress, options: std.Io.net.IpAddress.ConnectOptions) std.Io.net.IpAddress.ConnectError!std.Io.net.Socket { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netConnectIp(self.inner.userdata, address, options); } @@ -631,11 +636,6 @@ pub const VFSBase = struct { return self.inner.vtable.netSend(self.inner.userdata, handle, msgs, flags); } - pub fn netReceive(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, message_buffer: []std.Io.net.IncomingMessage, data_buffer: []u8, flags: std.Io.net.ReceiveFlags, timeout: std.Io.Timeout) struct { ?std.Io.net.Socket.ReceiveTimeoutError, usize } { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.netReceive(self.inner.userdata, handle, message_buffer, data_buffer, flags, timeout); - } - pub fn netRead(userdata: ?*anyopaque, src: std.Io.net.Socket.Handle, data: [][]u8) std.Io.net.Stream.Reader.Error!usize { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netRead(self.inner.userdata, src, data); diff --git a/zml/io/vfs/file.zig b/zml/io/vfs/file.zig index ff8df678..497c6b83 100644 --- a/zml/io/vfs/file.zig +++ b/zml/io/vfs/file.zig @@ -85,7 +85,7 @@ pub const File = struct { allocator: std.mem.Allocator, mutex: std.Io.Mutex = .init, handles: stdx.SegmentedList(Handle, 0) = .{}, - closed_handles: std.ArrayList(u32) = .{}, + closed_handles: std.ArrayList(u32) = .empty, config: Config, base: VFSBase, @@ -201,7 +201,7 @@ pub const File = struct { }, }); }, - .file_write_streaming, .device_io_control => { + .file_write_streaming, .device_io_control, .net_receive => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } diff --git a/zml/io/vfs/gcs.zig b/zml/io/vfs/gcs.zig index 98d1f9bf..4bf7f873 100644 --- a/zml/io/vfs/gcs.zig +++ b/zml/io/vfs/gcs.zig @@ -165,7 +165,7 @@ pub const GCS = struct { config: Config, token: Token, handles: stdx.SegmentedList(Handle, 0) = .{}, - closed_handles: std.ArrayList(u32) = .{}, + closed_handles: std.ArrayList(u32) = .empty, dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, base: VFSBase, @@ -473,17 +473,17 @@ pub const GCS = struct { const handle = self.getFileHandle(o.file); const total = self.performRead(handle, o.data, handle.pos) catch |err| { log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = std.Io.File.ReadStreamingError.EndOfStream }; + return .{ .file_read_streaming = error.EndOfStream }; }; if (total == 0) { - return .{ .file_read_streaming = std.Io.File.ReadStreamingError.EndOfStream }; + return .{ .file_read_streaming = error.EndOfStream }; } handle.pos += @intCast(total); return .{ .file_read_streaming = total }; }, - .file_write_streaming, .device_io_control => { + .file_write_streaming, .device_io_control, .net_receive => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -667,7 +667,7 @@ pub const GCS = struct { const handle = self.getFileHandle(file); return self.performRead(handle, data, offset) catch |err| { log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); - return std.Io.File.Reader.Error.Unexpected; + return std.Io.File.ReadPositionalError.Unexpected; }; } diff --git a/zml/io/vfs/hf.zig b/zml/io/vfs/hf.zig index 3fc38e01..614803a3 100644 --- a/zml/io/vfs/hf.zig +++ b/zml/io/vfs/hf.zig @@ -103,7 +103,7 @@ pub const HF = struct { client: *std.http.Client, authorization: std.http.Client.Request.Headers.Value, handles: stdx.SegmentedList(Handle, 0) = .{}, - closed_handles: std.ArrayList(u32) = .{}, + closed_handles: std.ArrayList(u32) = .empty, base: VFSBase, trees: std.StringHashMapUnmanaged(std.ArrayList(TreeNode)) = .{}, dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, @@ -319,7 +319,7 @@ pub const HF = struct { ); defer parsed.deinit(); - var tree_root: std.ArrayList(TreeNode) = .{}; + var tree_root: std.ArrayList(TreeNode) = .empty; for (parsed.value) |item| { try insertTreeNode(self.allocator, &tree_root, item); } @@ -407,7 +407,7 @@ pub const HF = struct { var parts = std.mem.tokenizeScalar(u8, item.path, '/'); var current_list = root; - var parents: std.ArrayList(*TreeNode) = .{}; + var parents: std.ArrayList(*TreeNode) = .empty; defer parents.deinit(allocator); while (true) { @@ -426,7 +426,7 @@ pub const HF = struct { if (!is_last) { try parents.append(allocator, f); if (f.children == null) { - f.children = .{}; + f.children = .empty; } current_list = &f.children.?; } else if (std.mem.eql(u8, item.type, "file")) { @@ -438,7 +438,7 @@ pub const HF = struct { .name = try allocator.dupe(u8, part), .kind = if (is_file) .file else .directory, .size = if (is_last) item.size else 0, - .children = if (is_file) null else .{}, + .children = if (is_file) null else .empty, }; try current_list.append(allocator, new_node); @@ -462,17 +462,17 @@ pub const HF = struct { const handle = self.getFileHandle(o.file); const total = self.performRead(handle, o.data, handle.pos) catch |err| { log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = std.Io.File.ReadStreamingError.EndOfStream }; + return .{ .file_read_streaming = error.EndOfStream }; }; if (total == 0) { - return .{ .file_read_streaming = std.Io.File.ReadStreamingError.EndOfStream }; + return .{ .file_read_streaming = error.EndOfStream }; } handle.pos += @intCast(total); return .{ .file_read_streaming = total }; }, - .file_write_streaming, .device_io_control => { + .file_write_streaming, .device_io_control, .net_receive => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -667,7 +667,7 @@ pub const HF = struct { const handle = self.getFileHandle(file); return self.performRead(handle, data, offset) catch |err| { log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); - return std.Io.File.Reader.Error.Unexpected; + return std.Io.File.ReadPositionalError.Unexpected; }; } diff --git a/zml/io/vfs/http.zig b/zml/io/vfs/http.zig index c2bfa332..2661a51b 100644 --- a/zml/io/vfs/http.zig +++ b/zml/io/vfs/http.zig @@ -42,7 +42,7 @@ pub const HTTP = struct { client: *std.http.Client, protocol: Protocol, handles: stdx.SegmentedList(Handle, 0) = .{}, - closed_handles: std.ArrayList(u32) = .{}, + closed_handles: std.ArrayList(u32) = .empty, base: VFSBase, pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, protocol: Protocol) !HTTP { @@ -146,17 +146,17 @@ pub const HTTP = struct { const handle = self.getFileHandle(o.file); const total = self.performRead(handle, o.data, handle.pos) catch |err| { log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = std.Io.File.ReadStreamingError.EndOfStream }; + return .{ .file_read_streaming = error.EndOfStream }; }; if (total == 0) { - return .{ .file_read_streaming = std.Io.File.ReadStreamingError.EndOfStream }; + return .{ .file_read_streaming = error.EndOfStream }; } handle.pos += @intCast(total); return .{ .file_read_streaming = total }; }, - .file_write_streaming, .device_io_control => { + .file_write_streaming, .device_io_control, .net_receive => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -284,7 +284,7 @@ pub const HTTP = struct { const handle = self.getFileHandle(file); return self.performRead(handle, data, offset) catch |err| { log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); - return std.Io.File.Reader.Error.Unexpected; + return std.Io.File.ReadPositionalError.Unexpected; }; } diff --git a/zml/io/vfs/s3.zig b/zml/io/vfs/s3.zig index 37953ef4..b6bd2a52 100644 --- a/zml/io/vfs/s3.zig +++ b/zml/io/vfs/s3.zig @@ -175,7 +175,7 @@ pub const S3 = struct { client: *std.http.Client, config: Config, handles: stdx.SegmentedList(Handle, 0) = .{}, - closed_handles: std.ArrayList(u32) = .{}, + closed_handles: std.ArrayList(u32) = .empty, dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, base: VFSBase, @@ -350,17 +350,17 @@ pub const S3 = struct { const handle = self.getFileHandle(o.file); const total = self.performRead(handle, o.data, handle.pos) catch |err| { log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = std.Io.File.ReadStreamingError.EndOfStream }; + return .{ .file_read_streaming = error.EndOfStream }; }; if (total == 0) { - return .{ .file_read_streaming = std.Io.File.ReadStreamingError.EndOfStream }; + return .{ .file_read_streaming = error.EndOfStream }; } handle.pos += @intCast(total); return .{ .file_read_streaming = total }; }, - .file_write_streaming, .device_io_control => { + .file_write_streaming, .device_io_control, .net_receive => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -539,7 +539,7 @@ pub const S3 = struct { const handle = self.getFileHandle(file); return self.performRead(handle, data, offset) catch |err| { log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); - return std.Io.File.Reader.Error.Unexpected; + return std.Io.File.ReadPositionalError.Unexpected; }; } @@ -687,7 +687,7 @@ pub const S3 = struct { } fn parseListObjectsResponse(self: *S3, xml: []const u8, prefix: []const u8) ![][]const u8 { - var results = std.ArrayListUnmanaged([]const u8){}; + var results: std.ArrayListUnmanaged([]const u8) = .empty; errdefer { for (results.items) |item| self.allocator.free(item); results.deinit(self.allocator); diff --git a/zml/mem.zig b/zml/mem.zig index c12a2f61..4e1610e5 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -360,7 +360,7 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu .pointer => |p| { switch (p.size) { .slice => { - bufferized_.* = try allocator.alignedAlloc(p.child, .fromByteUnits(p.alignment), model.len); + bufferized_.* = try allocator.alignedAlloc(p.child, .fromByteUnits(p.alignment orelse @alignOf(p.child)), model.len); for (model, bufferized_.*) |src, *dst| { try bufferizeInner(allocator, src, dst); } diff --git a/zml/meta.zig b/zml/meta.zig index 8fcd07ef..7c8a7c62 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -444,83 +444,81 @@ pub fn VisitReturn(comptime cb: anytype) type { void; } -pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) VisitReturn(cb) { - const Callback = @TypeOf(cb); - const Ptr = @TypeOf(v); - const type_info_v = @typeInfo(Ptr); - if (type_info_v != .pointer) { - stdx.debug.compileError("zml.meta.visit({}) is expecting a pointer/slice input, but received: {}", .{ Callback, Ptr }); - } - const ptr_info = type_info_v.pointer; - const Child = ptr_info.child; - const can_error = stdx.meta.FnReturnErrorSet(cb) != null; - - const K, const mutating_cb = switch (@typeInfo(FnParam(cb, 1))) { - .pointer => |info| .{ info.child, !info.is_const }, - else => stdx.debug.compileError("zml.meta.visit is expecting a callback with a pointer as second argument but found {}", .{FnParam(cb, 1)}), +const Visitor = struct { + pub const VisitorType = enum { + callback, + recurse, }; - // Abort if v doesnt' contain any K. - if (comptime !Contains(Ptr, K)) return; - // Handle simple cases. - switch (Ptr) { - *const K, *K => return cb(ctx, v), - *const ?K, *?K => return if (v.*) |*val| cb(ctx, val) else {}, - []const K, []K => { - for (v) |*v_elem| if (can_error) try cb(ctx, v_elem) else cb(ctx, v_elem); - return; - }, - else => {}, + pub fn determineAction(comptime callback: anytype, comptime PtrTypeOfV: type) VisitorType { + const ptr_info = switch (@typeInfo(PtrTypeOfV)) { + .pointer => |info| info, + else => stdx.debug.compileError("zml.meta.visit({any}) is expecting a pointer/slice input, but received: {any}", .{ @TypeOf(callback), PtrTypeOfV }), + }; + + const TargetType = switch (@typeInfo(FnParam(callback, 1))) { + .pointer => |info| info.child, + else => stdx.debug.compileError("zml.meta.visit({any}) is expecting a callback with a pointer as second argument but found {any}", .{ @TypeOf(callback), FnParam(callback, 1) }), + }; + + if (ptr_info.size == .one and ptr_info.child == TargetType) { + return .callback; + } + + return .recurse; } +}; - // Handle stdx.BoundedArray that contains uninitalized data. - if (@typeInfo(Child) == .@"struct" and @hasDecl(Child, "constSlice") and @hasDecl(Child, "slice")) { - return visit(cb, ctx, if (mutating_cb) v.slice() else v.constSlice()); - } +pub fn visit(comptime callback: anytype, ctx: FnParam(callback, 0), v: anytype) VisitReturn(callback) { + const action = comptime Visitor.determineAction(callback, @TypeOf(v)); - // Recursively visit fields of v. - switch (ptr_info.size) { - .one => switch (@typeInfo(Child)) { - .@"struct" => |s| inline for (s.fields) |field| { - if (field.is_comptime or comptime !Contains(field.type, K)) continue; - const field_type_info = @typeInfo(field.type); - // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. - switch (field_type_info) { - .pointer => if (can_error) try visit(cb, ctx, @field(v, field.name)) else visit(cb, ctx, @field(v, field.name)), - .array, .optional, .@"union", .@"struct" => if (can_error) try visit(cb, ctx, &@field(v, field.name)) else visit(cb, ctx, &@field(v, field.name)), - else => {}, - } - }, - .array => for (v) |*elem| if (can_error) try visit(cb, ctx, elem) else visit(cb, ctx, elem), - .optional => if (v.* != null) if (can_error) try visit(cb, ctx, &v.*.?) else visit(cb, ctx, &v.*.?), - .@"union" => switch (v.*) { - inline else => |*v_field| if (can_error) try visit(cb, ctx, v_field) else visit(cb, ctx, v_field), - }, - else => stdx.debug.compileError("zml.meta.visit({}) doesn't support fields of type: {}", .{ Callback, Child }), + const PtrTypeOfV = @TypeOf(v); + const ptr_info_v = @typeInfo(PtrTypeOfV).pointer; + const ChildTypeV = ptr_info_v.child; + + const can_error = stdx.meta.FnReturnErrorSet(callback) != null; + + switch (action) { + .callback => { + return if (can_error) try callback(ctx, v) else callback(ctx, v); }, - .slice => { - for (v) |*v_elem| { - switch (@typeInfo(Child)) { + .recurse => { + const TargetType, const mutating_cb = switch (@typeInfo(FnParam(callback, 1))) { + .pointer => |info| .{ info.child, !info.is_const }, + else => stdx.debug.compileError("zml.meta.visit({any}) is expecting a callback with a pointer as second argument but found {any}", .{ @TypeOf(callback), FnParam(callback, 1) }), + }; + + if (comptime !Contains(PtrTypeOfV, TargetType)) return; + + if (@typeInfo(ChildTypeV) == .@"struct" and @hasDecl(ChildTypeV, "constSlice") and @hasDecl(ChildTypeV, "slice")) { + return visit(callback, ctx, if (mutating_cb) v.slice() else v.constSlice()); + } + + switch (ptr_info_v.size) { + .one => switch (@typeInfo(ChildTypeV)) { .@"struct" => |s| inline for (s.fields) |field| { - if (field.is_comptime or comptime !Contains(field.type, K)) continue; - const field_type_info = @typeInfo(field.type); - // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. - if (field_type_info == .pointer) { - if (can_error) try visit(cb, ctx, @field(v_elem, field.name)) else visit(cb, ctx, @field(v_elem, field.name)); + if (field.is_comptime or comptime !Contains(field.type, TargetType)) continue; + if (@typeInfo(field.type) == .pointer) { + if (can_error) try visit(callback, ctx, @field(v, field.name)) else visit(callback, ctx, @field(v, field.name)); } else { - if (can_error) try visit(cb, ctx, &@field(v_elem, field.name)) else visit(cb, ctx, &@field(v_elem, field.name)); + if (can_error) try visit(callback, ctx, &@field(v, field.name)) else visit(callback, ctx, &@field(v, field.name)); } }, - .array => for (v) |*elem| if (can_error) try visit(cb, ctx, elem) else visit(cb, ctx, elem), - .optional => if (v.* != null) if (can_error) try visit(cb, ctx, &v.*.?) else visit(cb, ctx, &v.*.?), - .@"union" => switch (v_elem.*) { - inline else => |*v_field| if (can_error) try visit(cb, ctx, v_field) else visit(cb, ctx, v_field), + .array => inline for (v) |*elem| if (can_error) try visit(callback, ctx, elem) else visit(callback, ctx, elem), + .optional => if (v.* != null) if (can_error) try visit(callback, ctx, &v.*.?) else visit(callback, ctx, &v.*.?), + .@"union" => switch (v.*) { + inline else => |*v_field| if (can_error) try visit(callback, ctx, v_field) else visit(callback, ctx, v_field), }, - else => stdx.debug.compileError("zml.meta.visit({}) doesn't support fields of type: {}", .{ Callback, Child }), - } + else => stdx.debug.compileError("zml.meta.visit({any}) doesn't support visiting pointer type {any}", .{ @TypeOf(callback), ChildTypeV }), + }, + .slice => { + for (v) |*v_elem| { + if (can_error) try visit(callback, ctx, v_elem) else visit(callback, ctx, v_elem); + } + }, + .many, .c => stdx.debug.compileError("zml.meta.visit({any}) doesn't support [*] style pointers got {any}", .{ @TypeOf(callback), ChildTypeV }), } }, - .many, .c => stdx.debug.compileError("zml.meta.visit({}) doesn't support [*] style pointers, got: {}", .{ Callback, Ptr }), } } diff --git a/zml/sharding.zig b/zml/sharding.zig index 42894f90..1c6a777c 100644 --- a/zml/sharding.zig +++ b/zml/sharding.zig @@ -1109,7 +1109,7 @@ pub const Sharding = struct { } var folds: Folds = try .init(0); - var folds_consumed = std.EnumSet(PhysicalAxisTag).initEmpty(); + var folds_consumed: std.EnumSet(PhysicalAxisTag) = .empty; for (strategy.folding.constSlice()) |entry| { for (entry.sources.constSlice()) |src| { if (!physical.hasAxis(src)) return error.InvalidPhysicalAxis; @@ -1188,7 +1188,7 @@ pub const Sharding = struct { const logical_tag = Shape.toTag(logical_axis); const bound_axes = self.binding(logical_tag) orelse return 1; - var physical_axes = std.EnumSet(PhysicalAxisTag).initEmpty(); + var physical_axes: std.EnumSet(PhysicalAxisTag) = .empty; for (bound_axes) |bound_axis| { if (self.foldSources(bound_axis)) |sources| { for (sources) |source| physical_axes.insert(source); @@ -1242,7 +1242,7 @@ pub const Sharding = struct { const view = self.physicalView(); var axes_per_dim = stdx.BoundedArray(stdx.BoundedArray(usize, Shape.MAX_RANK), Shape.MAX_RANK).init(0) catch unreachable; var used_mask = [_]bool{false} ** Shape.MAX_RANK; - var globally_used = std.EnumSet(PhysicalAxisTag).initEmpty(); + var globally_used: std.EnumSet(PhysicalAxisTag) = .empty; for (0..shape.rank()) |ax| { var dim_axes = stdx.BoundedArray(usize, Shape.MAX_RANK).init(0) catch unreachable; @@ -1715,7 +1715,7 @@ pub const Placement = struct { }; for (self.shards.slice()) |*s| { - var used_axes = std.EnumSet(PhysicalAxisTag).initEmpty(); + var used_axes: std.EnumSet(PhysicalAxisTag) = .empty; for (0..shape.rank()) |ax| { const axis_index: u8 = @intCast(ax); diff --git a/zml/tensor.zig b/zml/tensor.zig index edbadf89..4d037aea 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -209,7 +209,7 @@ pub const Tensor = struct { .unknown(ctx.mlir_ctx), ).appendTo(currentBlock()); - var res = _result(self._shape, op.result(0)); + const res = _result(self._shape, op.result(0)); ctx.currentScope().id_to_output_memory_kind.put(ctx.currentScope().arena.allocator(), res.id, kind) catch unreachable; return res; } diff --git a/zml/tokenizer/homemade.zig b/zml/tokenizer/homemade.zig index c14feb1a..602c6956 100644 --- a/zml/tokenizer/homemade.zig +++ b/zml/tokenizer/homemade.zig @@ -963,13 +963,14 @@ pub const Gpt2TextDecoder = struct { const Code = stdx.BoundedArray(u8, 2); // TODO: benchmark this is more efficient than doing the conversion at runtime. - code_to_byte: std.AutoArrayHashMap(Code, u8), + code_to_byte: std.AutoArrayHashMapUnmanaged(Code, u8) = .empty, + allocator: std.mem.Allocator, pub fn init(allocator: std.mem.Allocator) !Gpt2TextDecoder { var self = Gpt2TextDecoder{ - .code_to_byte = std.AutoArrayHashMap(Code, u8).init(allocator), + .allocator = allocator, }; - try self.code_to_byte.ensureTotalCapacity(256); + try self.code_to_byte.ensureTotalCapacity(allocator, 256); errdefer unreachable; var n: usize = 0; @@ -995,7 +996,7 @@ pub const Gpt2TextDecoder = struct { } pub fn deinit(self: *Gpt2TextDecoder) void { - self.code_to_byte.deinit(); + self.code_to_byte.deinit(self.allocator); } /// Transform bytes representing text under the gpt2 encoding, diff --git a/zml/tokenizer/main.zig b/zml/tokenizer/main.zig index 8388a11b..e275e1d4 100644 --- a/zml/tokenizer/main.zig +++ b/zml/tokenizer/main.zig @@ -13,8 +13,7 @@ const Flags = struct { }; pub fn main(init: std.process.Init) !void { - var gpa: std.heap.GeneralPurposeAllocator(.{}) = .{}; - const allocator = gpa.allocator(); + const allocator = init.gpa; var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); From 8ec67ee5c32498902eb4984ee89f66d02d275096 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 13 Apr 2026 12:42:44 +0200 Subject: [PATCH 417/881] zml: remove HuggingFace tokenizer (#463) Following #416, the HuggingFace tokenizer is redundant. This also removes the Rust code from the repo, as well as rules_rust and rules_rs. --- .bazelrc | 1 - .vscode/settings.json | 7 +- MODULE.bazel | 33 +- MODULE.bazel.lock | 334 +-------- zml/tokenizer/BUILD.bazel | 6 +- zml/tokenizer/hftokenizers/.gitignore | 1 - zml/tokenizer/hftokenizers/BUILD.bazel | 35 - zml/tokenizer/hftokenizers/Cargo.lock | 713 -------------------- zml/tokenizer/hftokenizers/Cargo.toml | 15 - zml/tokenizer/hftokenizers/hftokenizers.h | 17 - zml/tokenizer/hftokenizers/hftokenizers.rs | 109 --- zml/tokenizer/hftokenizers/hftokenizers.zig | 120 ---- zml/tokenizer/tokenizer.zig | 5 - 13 files changed, 11 insertions(+), 1385 deletions(-) delete mode 100644 zml/tokenizer/hftokenizers/.gitignore delete mode 100644 zml/tokenizer/hftokenizers/BUILD.bazel delete mode 100644 zml/tokenizer/hftokenizers/Cargo.lock delete mode 100644 zml/tokenizer/hftokenizers/Cargo.toml delete mode 100644 zml/tokenizer/hftokenizers/hftokenizers.h delete mode 100644 zml/tokenizer/hftokenizers/hftokenizers.rs delete mode 100644 zml/tokenizer/hftokenizers/hftokenizers.zig diff --git a/.bazelrc b/.bazelrc index 42f50189..92b9888b 100644 --- a/.bazelrc +++ b/.bazelrc @@ -124,7 +124,6 @@ common:release --@rules_zig//zig/settings:mode=release_safe # --config=native builds everything optimized for the local CPU common:native --copt=-march=native common:native --@rules_zig//zig/settings:zigopt=-mcpu=native -common:native --@@rules_rs++rules_rust+rules_rust//rust/settings:extra_rustc_flag=-Ctarget-cpu=native common:linux --host_platform=//platforms:host_glibc diff --git a/.vscode/settings.json b/.vscode/settings.json index 82050af6..bc25445d 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -6,7 +6,7 @@ "**/.hg": true, "**/CVS": true, "**/.DS_Store": true, - "**/bazel-*": true + "**/bazel-*": true, }, "files.watcherExclude": { "**/bazel-*": true, @@ -29,9 +29,6 @@ "editor.stickyScroll.defaultModel": "foldingProviderModel", "editor.codeActionsOnSave": { "source.fixAll": "explicit", - } + }, }, - "rust-analyzer.linkedProjects": [ - "${workspaceFolder}/zml/tokenizer/hftokenizers/Cargo.toml" - ] } diff --git a/MODULE.bazel b/MODULE.bazel index b1ce05e7..2dac4cc2 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -6,6 +6,7 @@ bazel_dep(name = "abseil-cpp", version = "20250814.1", repo_name = "com_google_a bazel_dep(name = "bazel_lib", version = "3.2.2") bazel_dep(name = "aspect_rules_py", version = "1.8.4") bazel_dep(name = "bazel_skylib", version = "1.9.0") + ## Needed by iree bazel_dep(name = "libbacktrace", version = "1.0.0-20250926-7939218") bazel_dep(name = "patchelf", version = "0.18.0.bcr.1") @@ -20,7 +21,6 @@ bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep(name = "rules_oci", version = "2.2.7") bazel_dep(name = "rules_proto", version = "7.1.0") bazel_dep(name = "rules_python", version = "1.9.0") -bazel_dep(name = "rules_rs", version = "0.0.44") bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "rules_zig", version = "0.12.2") bazel_dep(name = "tar.bzl", version = "0.9.0") @@ -83,7 +83,7 @@ uv.configure(version = "0.9.30") python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain(python_version = "3.12") -use_repo(python, "python_versions", "python_3_12") +use_repo(python, "python_3_12", "python_versions") pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") @@ -140,7 +140,7 @@ cuda = use_extension("//platforms/cuda:cuda.bzl", "cuda_packages") use_repo(cuda, "cuda_nvml_dev", "libpjrt_cuda") rocm = use_extension("//platforms/rocm:rocm.bzl", "rocm_packages") -use_repo(rocm, "amd-smi-lib", "hipblaslt", "libpjrt_rocm", "libdrm2-amdgpu", "libdrm-amdgpu-amdgpu1", "rocblas") +use_repo(rocm, "amd-smi-lib", "hipblaslt", "libdrm-amdgpu-amdgpu1", "libdrm2-amdgpu", "libpjrt_rocm", "rocblas") tpu = use_extension("//platforms/tpu:tpu.bzl", "tpu_packages") use_repo(tpu, "libpjrt_tpu") @@ -154,7 +154,7 @@ use_repo(zls, "zls_toolchains") register_toolchains("@zls_toolchains//:all") non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") -use_repo(non_module_deps, "arocc", "cloud_accelerator_diagnostics", "com_google_sentencepiece", "flashattn", "iree", "libvaxis", "linenoise", "mnist", "org_swig_swig", "translate-c", "stb", "uucode", "xla", "zigimg") +use_repo(non_module_deps, "arocc", "cloud_accelerator_diagnostics", "com_google_sentencepiece", "flashattn", "iree", "libvaxis", "linenoise", "mnist", "org_swig_swig", "stb", "translate-c", "uucode", "xla", "zigimg") xla = use_extension("//third_party/xla:xla.bzl", "xla") use_repo( @@ -184,30 +184,6 @@ llvm = use_extension("//third_party/xla:llvm.bzl", "llvm") llvm.configure() use_repo(llvm, "llvm-project") -rs_toolchains = use_extension("@rules_rs//rs/experimental/toolchains:module_extension.bzl", "toolchains") -rs_toolchains.toolchain( - edition = "2024", - version = "1.94.0", -) -use_repo(rs_toolchains, "default_rust_toolchains") - -register_toolchains("@default_rust_toolchains//:all") - -crate = use_extension("@rules_rs//rs:extensions.bzl", "crate") -crate.from_cargo( - name = "crates", - cargo_lock = "//zml/tokenizer/hftokenizers:Cargo.lock", - cargo_toml = "//zml/tokenizer/hftokenizers:Cargo.toml", - platform_triples = [ - "aarch64-apple-darwin", - "x86_64-apple-darwin", - "aarch64-unknown-linux-gnu", - "x86_64-unknown-linux-gnu", - ], - use_experimental_platforms = True, -) -use_repo(crate, "crates") - apt = use_extension("@rules_distroless//apt:extensions.bzl", "apt") apt.install( name = "apt_cuda", @@ -251,7 +227,6 @@ use_repo( "distroless_cc_debian12_linux_amd64", ) - git_override( module_name = "rules_zig", commit = "56911842f4527df036e8d52e78dd0f54a22d4215", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 66de42db..8b6d62ba 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -22,7 +22,6 @@ "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", "https://bcr.bazel.build/modules/apple_support/1.17.1/MODULE.bazel": "655c922ab1209978a94ef6ca7d9d43e940cd97d9c172fb55f94d91ac53f8610b", "https://bcr.bazel.build/modules/apple_support/1.23.1/MODULE.bazel": "53763fed456a968cf919b3240427cf3a9d5481ec5466abc9d5dc51bc70087442", - "https://bcr.bazel.build/modules/apple_support/1.24.1/MODULE.bazel": "f46e8ddad60aef170ee92b2f3d00ef66c147ceafea68b6877cb45bd91737f5f8", "https://bcr.bazel.build/modules/apple_support/2.2.0/MODULE.bazel": "5e32bc42e413a4544df7afee7fa4c7aff73e670a44e56e8e45ce1954332034ad", "https://bcr.bazel.build/modules/apple_support/2.2.0/source.json": "368bf1de5dace2a411e792e6df2ea5384eb8db3899680ed51b37eb48b88fd8ac", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", @@ -33,7 +32,6 @@ "https://bcr.bazel.build/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "812d2dd42f65dca362152101fbec418029cc8fd34cbad1a2fde905383d705838", "https://bcr.bazel.build/modules/aspect_rules_py/1.8.4/MODULE.bazel": "4ab4a78e9d65cf15e755807f7ae504655e1771cefd7c3343ad34dbbb625b9564", "https://bcr.bazel.build/modules/aspect_rules_py/1.8.4/source.json": "7cd80ad811f0c2b3aad3c878506ae602fce2fe53b78487ecfbcc8bcde07d220e", - "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.2/MODULE.bazel": "598e7fe3b54f5fa64fdbeead1027653963a359cc23561d43680006f3b463d5a4", "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.3/MODULE.bazel": "37c764292861c2f70314efa9846bb6dbb44fc0308903b3285da6528305450183", "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.3/source.json": "605086bbc197743a0d360f7ddc550a1d4dfa0441bc807236e17170f636153348", "https://bcr.bazel.build/modules/bazel_features/1.1.0/MODULE.bazel": "cfd42ff3b815a5f39554d97182657f8c4b9719568eb7fded2b9135f084bf760b", @@ -51,7 +49,6 @@ "https://bcr.bazel.build/modules/bazel_features/1.28.0/MODULE.bazel": "4b4200e6cbf8fa335b2c3f43e1d6ef3e240319c33d43d60cc0fbd4b87ece299d", "https://bcr.bazel.build/modules/bazel_features/1.3.0/MODULE.bazel": "cdcafe83ec318cda34e02948e81d790aab8df7a929cec6f6969f13a489ccecd9", "https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87", - "https://bcr.bazel.build/modules/bazel_features/1.32.0/MODULE.bazel": "095d67022a58cb20f7e20e1aefecfa65257a222c18a938e2914fd257b5f1ccdc", "https://bcr.bazel.build/modules/bazel_features/1.33.0/MODULE.bazel": "8b8dc9d2a4c88609409c3191165bccec0e4cb044cd7a72ccbe826583303459f6", "https://bcr.bazel.build/modules/bazel_features/1.34.0/MODULE.bazel": "e8475ad7c8965542e0c7aac8af68eb48c4af904be3d614b6aa6274c092c2ea1e", "https://bcr.bazel.build/modules/bazel_features/1.38.0/MODULE.bazel": "f9b8a9c890ebd216b4049fd12a31d3c2602e3403c7af636b04fbbd7453edc9c9", @@ -151,7 +148,6 @@ "https://bcr.bazel.build/modules/libbacktrace/1.0.0-20250926-7939218/source.json": "83239989d09748f63dee4bdd430d261e217a012670e066e2238925a303a65d89", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", "https://bcr.bazel.build/modules/libpfm/4.11.0/source.json": "caaffb3ac2b59b8aac456917a4ecf3167d40478ee79f15ab7a877ec9273937c9", - "https://bcr.bazel.build/modules/llvm/0.6.7/MODULE.bazel": "d37a2e10571864dc6a5bb53c29216d90b9400bbcadb422337f49107fd2eaf0d2", "https://bcr.bazel.build/modules/llvm/0.7.0/MODULE.bazel": "3c07a4e5734b0ad41fe24dedaacbf3a35ce4377b7e1a21f24488a0c9ac4f1e6b", "https://bcr.bazel.build/modules/llvm/0.7.0/source.json": "5f0a015da1e7db09ef61aa254ba5eb55b0c322fadd1e8eefeca368d7723a24a0", "https://bcr.bazel.build/modules/mbedtls/3.6.0/MODULE.bazel": "8e380e4698107c5f8766264d4df92e36766248447858db28187151d884995a09", @@ -179,8 +175,7 @@ "https://bcr.bazel.build/modules/opentracing-cpp/1.6.0/MODULE.bazel": "b3925269f63561b8b880ae7cf62ccf81f6ece55b62cd791eda9925147ae116ec", "https://bcr.bazel.build/modules/opentracing-cpp/1.6.0/source.json": "da1cb1add160f5e5074b7272e9db6fd8f1b3336c15032cd0a653af9d2f484aed", "https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92", - "https://bcr.bazel.build/modules/package_metadata/0.0.5/MODULE.bazel": "ef4f9439e3270fdd6b9fd4dbc3d2f29d13888e44c529a1b243f7a31dfbc2e8e4", - "https://bcr.bazel.build/modules/package_metadata/0.0.5/source.json": "2326db2f6592578177751c3e1f74786b79382cd6008834c9d01ec865b9126a85", + "https://bcr.bazel.build/modules/package_metadata/0.0.2/source.json": "e53a759a72488d2c0576f57491ef2da0cf4aab05ac0997314012495935531b73", "https://bcr.bazel.build/modules/patchelf/0.18.0.bcr.1/MODULE.bazel": "35320668481400503dea7c75313473b9ffd6dea4e7ba583b96a896199b0088d5", "https://bcr.bazel.build/modules/patchelf/0.18.0.bcr.1/source.json": "d877d271ef5d531407885322054c99c7f0617a9a85ff99c0a70f4ced7f13e13a", "https://bcr.bazel.build/modules/pcre2/10.45/MODULE.bazel": "9bdc838aa2e21be030d7181385e45a5c87b14575bb7ff35d10b5be3d56913bed", @@ -327,8 +322,6 @@ "https://bcr.bazel.build/modules/rules_python/1.7.0/MODULE.bazel": "d01f995ecd137abf30238ad9ce97f8fc3ac57289c8b24bd0bf53324d937a14f8", "https://bcr.bazel.build/modules/rules_python/1.9.0/MODULE.bazel": "afc3a05f29f09f2d3ee95ad99a145250dab41a2b2d8d6f82cc91936b3213282c", "https://bcr.bazel.build/modules/rules_python/1.9.0/source.json": "3921ea0b65298d51aead5b9e4a82203d7be9b5918619b58b53f1c259f4e63169", - "https://bcr.bazel.build/modules/rules_rs/0.0.44/MODULE.bazel": "aa151d9a96e5d9363732df8943c7fb5327738cc160732fa74c02e5de7822007b", - "https://bcr.bazel.build/modules/rules_rs/0.0.44/source.json": "1358c9418e1325fdb5d0caf53ef32f98d525fe2030668279cae2e97c34cd29eb", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", @@ -448,7 +441,7 @@ "@@aspect_tools_telemetry+//:extension.bzl%telemetry": { "general": { "bzlTransitiveDigest": "MePriaXmQNSqUfE+YQvEtzBc2bU1mjITINIffFiZYgo=", - "usagesDigest": "TU1quwRKpvAEOEF35/IVh4NeoIS2hayv4fGp8f5RXoY=", + "usagesDigest": "pyph4lSDn5CHIn4s2mtKJTkRaLV/o2zIFgnrpufr/8o=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -458,7 +451,6 @@ "attributes": { "deps": { "aspect_rules_py": "1.8.4", - "rules_rs": "0.0.44", "aspect_tools_telemetry": "0.3.3" } } @@ -1728,325 +1720,5 @@ } } }, - "facts": { - "@@rules_rs+//rs:extensions.bzl%crate": { - "addr2line_0.25.1": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"backtrace\",\"req\":\"^0.3.13\"},{\"features\":[\"wrap_help\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4.3.21\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"cpp_demangle\",\"optional\":true,\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.7.0\"},{\"default_features\":false,\"name\":\"fallible-iterator\",\"optional\":true,\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"findshlibs\",\"req\":\"^0.10\"},{\"default_features\":false,\"features\":[\"read\"],\"name\":\"gimli\",\"req\":\"^0.32.0\"},{\"kind\":\"dev\",\"name\":\"libtest-mimic\",\"req\":\"^0.8.1\"},{\"name\":\"memmap2\",\"optional\":true,\"req\":\"^0.9.4\"},{\"default_features\":false,\"features\":[\"read\",\"compression\"],\"name\":\"object\",\"optional\":true,\"req\":\"^0.37.0\"},{\"name\":\"rustc-demangle\",\"optional\":true,\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"typed-arena\",\"optional\":true,\"req\":\"^2\"}],\"features\":{\"all\":[\"bin\",\"wasm\"],\"bin\":[\"loader\",\"rustc-demangle\",\"cpp_demangle\",\"fallible-iterator\",\"smallvec\",\"dep:clap\"],\"cargo-all\":[],\"default\":[\"rustc-demangle\",\"cpp_demangle\",\"loader\",\"fallible-iterator\",\"smallvec\"],\"loader\":[\"std\",\"dep:object\",\"dep:memmap2\",\"dep:typed-arena\"],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"gimli/rustc-dep-of-std\"],\"std\":[\"gimli/std\"],\"wasm\":[\"object/wasm\"]}}", - "adler2_2.0.1": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"std\"],\"rustc-dep-of-std\":[\"core\"],\"std\":[]}}", - "ahash_0.8.12": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1.0\"},{\"name\":\"const-random\",\"optional\":true,\"req\":\"^0.1.17\"},{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.2\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.5\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2.1\"},{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"hashbrown\",\"req\":\"^0.14.3\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.2\"},{\"kind\":\"dev\",\"name\":\"no-panic\",\"req\":\"^0.1.10\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"once_cell\",\"req\":\"^1.18.0\",\"target\":\"cfg(not(all(target_arch = \\\"arm\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"pcg-mwc\",\"req\":\"^0.2.1\"},{\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"seahash\",\"req\":\"^4.0\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.117\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.59\"},{\"kind\":\"dev\",\"name\":\"smallvec\",\"req\":\"^1.13.1\"},{\"kind\":\"build\",\"name\":\"version_check\",\"req\":\"^0.9.4\"},{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"zerocopy\",\"req\":\"^0.8.24\"}],\"features\":{\"atomic-polyfill\":[\"dep:portable-atomic\",\"once_cell/critical-section\"],\"compile-time-rng\":[\"const-random\"],\"default\":[\"std\",\"runtime-rng\"],\"nightly-arm-aes\":[],\"no-rng\":[],\"runtime-rng\":[\"getrandom\"],\"std\":[]}}", - "aho-corasick_1.1.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.17\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.4.0\"}],\"features\":{\"default\":[\"std\",\"perf-literal\"],\"logging\":[\"dep:log\"],\"perf-literal\":[\"dep:memchr\"],\"std\":[\"memchr?/std\"]}}", - "aho-corasick_1.1.4": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.17\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.4.0\"}],\"features\":{\"default\":[\"std\",\"perf-literal\"],\"logging\":[\"dep:log\"],\"perf-literal\":[\"dep:memchr\"],\"std\":[\"memchr?/std\"]}}", - "anyhow_1.0.102": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.6\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0\"},{\"kind\":\"dev\",\"name\":\"thiserror\",\"req\":\"^2\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"}],\"features\":{\"backtrace\":[],\"default\":[\"std\"],\"std\":[]}}", - "async-stream-impl_0.3.6": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"full\",\"visit-mut\"],\"name\":\"syn\",\"req\":\"^2.0.2\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"}],\"features\":{}}", - "async-stream_0.3.6": "{\"dependencies\":[{\"name\":\"async-stream-impl\",\"req\":\"=0.3.6\"},{\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1\"}],\"features\":{}}", - "async-trait_0.1.89": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.30\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"full\",\"parsing\",\"printing\",\"proc-macro\",\"visit-mut\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1.40\"},{\"kind\":\"dev\",\"name\":\"tracing-attributes\",\"req\":\"^0.1.27\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{}}", - "atomic-waker_1.1.2": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"cargo_bench_support\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.5\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.7.0\"}],\"features\":{}}", - "autocfg_1.5.0": "{\"dependencies\":[],\"features\":{}}", - "axum-core_0.4.5": "{\"dependencies\":[{\"name\":\"async-trait\",\"req\":\"^0.1.67\"},{\"kind\":\"dev\",\"name\":\"axum\",\"req\":\"^0.7.2\"},{\"name\":\"bytes\",\"req\":\"^1.2\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"http\",\"req\":\"^1.0.0\"},{\"name\":\"http-body\",\"req\":\"^1.0.0\"},{\"name\":\"http-body-util\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^1.0.0\"},{\"name\":\"mime\",\"req\":\"^0.3.16\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"},{\"name\":\"rustversion\",\"req\":\"^1.0.9\"},{\"name\":\"sync_wrapper\",\"req\":\"^1.0.0\"},{\"features\":[\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.25.0\"},{\"features\":[\"limit\"],\"name\":\"tower-http\",\"optional\":true,\"req\":\"^0.6.0\"},{\"features\":[\"limit\"],\"kind\":\"dev\",\"name\":\"tower-http\",\"req\":\"^0.6.0\"},{\"name\":\"tower-layer\",\"req\":\"^0.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.37\"}],\"features\":{\"__private_docs\":[\"dep:tower-http\"],\"tracing\":[\"dep:tracing\"]}}", - "axum_0.7.9": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0\"},{\"name\":\"async-trait\",\"req\":\"^0.1.67\"},{\"name\":\"axum-core\",\"req\":\"^0.4.5\"},{\"name\":\"axum-macros\",\"optional\":true,\"req\":\"^0.4.2\"},{\"features\":[\"__private\"],\"kind\":\"dev\",\"name\":\"axum-macros\",\"req\":\"^0.4.1\"},{\"name\":\"base64\",\"optional\":true,\"req\":\"^0.22.1\"},{\"name\":\"bytes\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"http\",\"req\":\"^1.0.0\"},{\"name\":\"http-body\",\"req\":\"^1.0.0\"},{\"name\":\"http-body-util\",\"req\":\"^0.1.0\"},{\"name\":\"hyper\",\"optional\":true,\"req\":\"^1.1.0\"},{\"features\":[\"tokio\",\"server\",\"service\"],\"name\":\"hyper-util\",\"optional\":true,\"req\":\"^0.1.3\"},{\"name\":\"itoa\",\"req\":\"^1.0.5\"},{\"name\":\"matchit\",\"req\":\"^0.7\"},{\"name\":\"memchr\",\"req\":\"^2.4.1\"},{\"name\":\"mime\",\"req\":\"^0.3.16\"},{\"name\":\"multer\",\"optional\":true,\"req\":\"^3.0.0\"},{\"name\":\"percent-encoding\",\"req\":\"^2.1\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"json\",\"stream\",\"multipart\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.12\"},{\"name\":\"rustversion\",\"req\":\"^1.0.9\"},{\"name\":\"serde\",\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"features\":[\"raw_value\"],\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"raw_value\"],\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"serde_path_to_error\",\"optional\":true,\"req\":\"^0.1.8\"},{\"name\":\"serde_urlencoded\",\"optional\":true,\"req\":\"^0.7\"},{\"name\":\"sha1\",\"optional\":true,\"req\":\"^0.10\"},{\"name\":\"sync_wrapper\",\"req\":\"^1.0.0\"},{\"features\":[\"serde-human-readable\"],\"kind\":\"dev\",\"name\":\"time\",\"req\":\"^0.3\"},{\"features\":[\"time\"],\"name\":\"tokio\",\"optional\":true,\"package\":\"tokio\",\"req\":\"^1.25.0\"},{\"features\":[\"macros\",\"rt\",\"rt-multi-thread\",\"net\",\"test-util\"],\"kind\":\"dev\",\"name\":\"tokio\",\"package\":\"tokio\",\"req\":\"^1.25.0\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"name\":\"tokio-tungstenite\",\"optional\":true,\"req\":\"^0.24.0\"},{\"kind\":\"dev\",\"name\":\"tokio-tungstenite\",\"req\":\"^0.24.0\"},{\"default_features\":false,\"features\":[\"util\"],\"name\":\"tower\",\"req\":\"^0.5.1\"},{\"features\":[\"util\",\"timeout\",\"limit\",\"load-shed\",\"steer\",\"filter\"],\"kind\":\"dev\",\"name\":\"tower\",\"package\":\"tower\",\"req\":\"^0.5.1\"},{\"features\":[\"add-extension\",\"auth\",\"catch-panic\",\"compression-br\",\"compression-deflate\",\"compression-gzip\",\"cors\",\"decompression-br\",\"decompression-deflate\",\"decompression-gzip\",\"follow-redirect\",\"fs\",\"limit\",\"map-request-body\",\"map-response-body\",\"metrics\",\"normalize-path\",\"propagate-header\",\"redirect\",\"request-id\",\"sensitive-headers\",\"set-header\",\"set-status\",\"timeout\",\"trace\",\"util\",\"validate-request\"],\"name\":\"tower-http\",\"optional\":true,\"req\":\"^0.6.0\"},{\"features\":[\"add-extension\",\"auth\",\"catch-panic\",\"compression-br\",\"compression-deflate\",\"compression-gzip\",\"cors\",\"decompression-br\",\"decompression-deflate\",\"decompression-gzip\",\"follow-redirect\",\"fs\",\"limit\",\"map-request-body\",\"map-response-body\",\"metrics\",\"normalize-path\",\"propagate-header\",\"redirect\",\"request-id\",\"sensitive-headers\",\"set-header\",\"set-status\",\"timeout\",\"trace\",\"util\",\"validate-request\"],\"kind\":\"dev\",\"name\":\"tower-http\",\"req\":\"^0.6.0\"},{\"name\":\"tower-layer\",\"req\":\"^0.3.2\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1\"},{\"features\":[\"json\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"},{\"features\":[\"serde\",\"v4\"],\"kind\":\"dev\",\"name\":\"uuid\",\"req\":\"^1.0\"}],\"features\":{\"__private_docs\":[\"axum-core/__private_docs\",\"tower/full\",\"dep:tower-http\"],\"default\":[\"form\",\"http1\",\"json\",\"matched-path\",\"original-uri\",\"query\",\"tokio\",\"tower-log\",\"tracing\"],\"form\":[\"dep:serde_urlencoded\"],\"http1\":[\"dep:hyper\",\"hyper?/http1\",\"hyper-util?/http1\"],\"http2\":[\"dep:hyper\",\"hyper?/http2\",\"hyper-util?/http2\"],\"json\":[\"dep:serde_json\",\"dep:serde_path_to_error\"],\"macros\":[\"dep:axum-macros\"],\"matched-path\":[],\"multipart\":[\"dep:multer\"],\"original-uri\":[],\"query\":[\"dep:serde_urlencoded\"],\"tokio\":[\"dep:hyper-util\",\"dep:tokio\",\"tokio/net\",\"tokio/rt\",\"tower/make\",\"tokio/macros\"],\"tower-log\":[\"tower/log\"],\"tracing\":[\"dep:tracing\",\"axum-core/tracing\"],\"ws\":[\"dep:hyper\",\"tokio\",\"dep:tokio-tungstenite\",\"dep:sha1\",\"dep:base64\"]}}", - "backtrace_0.3.76": "{\"dependencies\":[{\"default_features\":false,\"name\":\"addr2line\",\"req\":\"^0.25.0\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"name\":\"cfg-if\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"cpp_demangle\",\"optional\":true,\"req\":\"^0.5.0\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.156\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"kind\":\"dev\",\"name\":\"libloading\",\"req\":\"^0.8\"},{\"default_features\":false,\"name\":\"miniz_oxide\",\"req\":\"^0.8\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"default_features\":false,\"features\":[\"read_core\",\"elf\",\"macho\",\"pe\",\"xcoff\",\"unaligned\",\"archive\"],\"name\":\"object\",\"req\":\"^0.37.0\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"name\":\"rustc-demangle\",\"req\":\"^0.1.24\"},{\"default_features\":false,\"name\":\"ruzstd\",\"optional\":true,\"req\":\"^0.8.1\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"windows-link\",\"req\":\"^0.2\",\"target\":\"cfg(any(windows, target_os = \\\"cygwin\\\"))\"}],\"features\":{\"coresymbolication\":[],\"dbghelp\":[],\"default\":[\"std\"],\"dl_iterate_phdr\":[],\"dladdr\":[],\"kernel32\":[],\"libunwind\":[],\"ruzstd\":[\"dep:ruzstd\"],\"serialize-serde\":[\"serde\"],\"std\":[],\"unix-backtrace\":[]}}", - "base64_0.13.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"=0.3.2\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.6.1\"},{\"kind\":\"dev\",\"name\":\"structopt\",\"req\":\"^0.3\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[]}}", - "base64_0.22.1": "{\"dependencies\":[{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"clap\",\"req\":\"^3.2.25\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rstest\",\"req\":\"^0.13.0\"},{\"kind\":\"dev\",\"name\":\"rstest_reuse\",\"req\":\"^0.6.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"strum\",\"req\":\"^0.25\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", - "bitflags_2.11.0": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"arbitrary\",\"req\":\"^1.0\"},{\"name\":\"bytemuck\",\"optional\":true,\"req\":\"^1.12\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"bytemuck\",\"req\":\"^1.12.2\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.228\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde_lib\",\"package\":\"serde\",\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.19\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.18\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"zerocopy\",\"req\":\"^0.8\"}],\"features\":{\"example_generated\":[],\"serde\":[\"serde_core\"],\"std\":[]}}", - "bitflags_2.9.1": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"arbitrary\",\"req\":\"^1.0\"},{\"name\":\"bytemuck\",\"optional\":true,\"req\":\"^1.12\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"bytemuck\",\"req\":\"^1.12.2\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.19\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.18\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"zerocopy\",\"req\":\"^0.8\"}],\"features\":{\"example_generated\":[],\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"],\"std\":[]}}", - "byteorder_1.5.0": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9.2\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"std\"],\"i128\":[],\"std\":[]}}", - "bytes_1.11.1": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"require-cas\"],\"name\":\"extra-platforms\",\"optional\":true,\"package\":\"portable-atomic\",\"req\":\"^1.3\"},{\"kind\":\"dev\",\"name\":\"loom\",\"req\":\"^0.7\",\"target\":\"cfg(loom)\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.60\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "castaway_0.2.4": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1\"},{\"name\":\"rustversion\",\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", - "cc_1.2.33": "{\"dependencies\":[{\"default_features\":false,\"name\":\"jobserver\",\"optional\":true,\"req\":\"^0.1.30\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.62\",\"target\":\"cfg(unix)\"},{\"name\":\"shlex\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{\"jobserver\":[],\"parallel\":[\"dep:libc\",\"dep:jobserver\"]}}", - "cfg-if_1.0.0": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"]}}", - "cfg-if_1.0.4": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"rustc-dep-of-std\":[\"core\"]}}", - "compact_str_0.9.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"borsh\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"castaway\",\"req\":\"^0.2.3\"},{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"cfg-if\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"diesel\",\"optional\":true,\"req\":\"^2\"},{\"name\":\"itoa\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"markup\",\"optional\":true,\"req\":\"^0.15\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"proptest\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"rkyv\",\"optional\":true,\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rkyv\",\"req\":\"^0.8.8\"},{\"name\":\"rustversion\",\"req\":\"^1\"},{\"name\":\"ryu\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"derive\",\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"features\":[\"union\"],\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"name\":\"sqlx\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"static_assertions\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"test-case\",\"req\":\"^3\"},{\"kind\":\"dev\",\"name\":\"test-strategy\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"borsh\":[\"dep:borsh\"],\"bytes\":[\"dep:bytes\"],\"default\":[\"std\"],\"diesel\":[\"dep:diesel\"],\"markup\":[\"dep:markup\"],\"proptest\":[\"dep:proptest\"],\"quickcheck\":[\"dep:quickcheck\"],\"rkyv\":[\"dep:rkyv\"],\"serde\":[\"dep:serde\"],\"smallvec\":[\"dep:smallvec\"],\"sqlx\":[\"dep:sqlx\",\"std\"],\"sqlx-mysql\":[\"sqlx\",\"sqlx/mysql\"],\"sqlx-postgres\":[\"sqlx\",\"sqlx/postgres\"],\"sqlx-sqlite\":[\"sqlx\",\"sqlx/sqlite\"],\"std\":[],\"zeroize\":[\"dep:zeroize\"]}}", - "crossbeam-deque_0.8.6": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-epoch\",\"req\":\"^0.9.17\"},{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"crossbeam-epoch/std\",\"crossbeam-utils/std\"]}}", - "crossbeam-epoch_0.9.18": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"name\":\"loom-crate\",\"optional\":true,\"package\":\"loom\",\"req\":\"^0.7.1\",\"target\":\"cfg(crossbeam_loom)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"loom\":[\"loom-crate\",\"crossbeam-utils/loom\"],\"nightly\":[\"crossbeam-utils/nightly\"],\"std\":[\"alloc\",\"crossbeam-utils/std\"]}}", - "crossbeam-utils_0.8.21": "{\"dependencies\":[{\"name\":\"loom\",\"optional\":true,\"req\":\"^0.7.1\",\"target\":\"cfg(crossbeam_loom)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"std\"],\"nightly\":[],\"std\":[]}}", - "darling_0.20.10": "{\"dependencies\":[{\"name\":\"darling_core\",\"req\":\"=0.20.10\"},{\"name\":\"darling_macro\",\"req\":\"=0.20.10\"},{\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.37\"},{\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.9\",\"target\":\"cfg(compiletests)\"},{\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.15\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.38\",\"target\":\"cfg(compiletests)\"}],\"features\":{\"default\":[\"suggestions\"],\"diagnostics\":[\"darling_core/diagnostics\"],\"suggestions\":[\"darling_core/suggestions\"]}}", - "darling_core_0.20.10": "{\"dependencies\":[{\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"name\":\"ident_case\",\"req\":\"^1.0.1\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.37\"},{\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"name\":\"strsim\",\"optional\":true,\"req\":\"^0.11.1\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"diagnostics\":[],\"suggestions\":[\"strsim\"]}}", - "darling_macro_0.20.10": "{\"dependencies\":[{\"name\":\"darling_core\",\"req\":\"=0.20.10\"},{\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{}}", - "dary_heap_0.3.7": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1\"}],\"features\":{\"extra\":[],\"unstable\":[],\"unstable_nightly\":[]}}", - "derive_builder_0.20.2": "{\"dependencies\":[{\"name\":\"derive_builder_macro\",\"req\":\"=0.20.2\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.4\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.38\",\"target\":\"cfg(compiletests)\"}],\"features\":{\"alloc\":[\"derive_builder_macro/alloc\"],\"clippy\":[\"derive_builder_macro/clippy\"],\"default\":[\"std\"],\"std\":[\"derive_builder_macro/lib_has_std\"]}}", - "derive_builder_core_0.20.2": "{\"dependencies\":[{\"name\":\"darling\",\"req\":\"^0.20.10\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.37\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"alloc\":[],\"clippy\":[],\"lib_has_std\":[]}}", - "derive_builder_macro_0.20.2": "{\"dependencies\":[{\"name\":\"derive_builder_core\",\"req\":\"=0.20.2\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"alloc\":[\"derive_builder_core/alloc\"],\"clippy\":[\"derive_builder_core/clippy\"],\"lib_has_std\":[\"derive_builder_core/lib_has_std\"]}}", - "either_1.13.0": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"use_std\"],\"use_std\":[]}}", - "either_1.15.0": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"alloc\",\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.95\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[],\"use_std\":[\"std\"]}}", - "equivalent_1.0.2": "{\"dependencies\":[],\"features\":{}}", - "errno_0.3.14": "{\"dependencies\":[{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(target_os=\\\"hermit\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(target_os=\\\"wasi\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(unix)\"},{\"features\":[\"Win32_Foundation\",\"Win32_System_Diagnostics_Debug\"],\"name\":\"windows-sys\",\"req\":\">=0.52, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"libc/std\"]}}", - "esaxx-rs_0.1.10": "{\"dependencies\":[{\"kind\":\"build\",\"name\":\"cc\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"}],\"features\":{\"cpp\":[\"cc\"],\"default\":[\"cpp\"]}}", - "fastrand_2.3.0": "{\"dependencies\":[{\"features\":[\"js\"],\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.2\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"getrandom\",\"req\":\"^0.2\"},{\"features\":[\"js\"],\"kind\":\"dev\",\"name\":\"getrandom\",\"req\":\"^0.2\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"wyhash\",\"req\":\"^0.5\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"js\":[\"std\",\"getrandom\"],\"std\":[\"alloc\"]}}", - "fixedbitset_0.4.2": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "fnv_1.0.7": "{\"dependencies\":[],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "foldhash_0.1.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"ahash\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"chrono\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"hashbrown\",\"req\":\"^0.14\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"uuid\",\"req\":\"^1.8\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "futures-channel_0.3.32": "{\"dependencies\":[{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-sink\",\"optional\":true,\"req\":\"^0.3.32\"}],\"features\":{\"alloc\":[\"futures-core/alloc\"],\"cfg-target-has-atomic\":[],\"default\":[\"std\"],\"sink\":[\"futures-sink\"],\"std\":[\"alloc\",\"futures-core/std\"],\"unstable\":[]}}", - "futures-core_0.3.32": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"require-cas\"],\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.3\"}],\"features\":{\"alloc\":[],\"cfg-target-has-atomic\":[],\"default\":[\"std\"],\"std\":[\"alloc\"],\"unstable\":[]}}", - "futures-sink_0.3.32": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", - "futures-task_0.3.32": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"cfg-target-has-atomic\":[],\"default\":[\"std\"],\"std\":[\"alloc\"],\"unstable\":[]}}", - "futures-util_0.3.32": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"std\"],\"name\":\"futures-channel\",\"optional\":true,\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3.32\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"futures-io\",\"optional\":true,\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-macro\",\"optional\":true,\"req\":\"=0.3.32\"},{\"default_features\":false,\"name\":\"futures-sink\",\"optional\":true,\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-task\",\"req\":\"^0.3.32\"},{\"name\":\"futures_01\",\"optional\":true,\"package\":\"futures\",\"req\":\"^0.1.25\"},{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.26\"},{\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.2\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.6\"},{\"default_features\":false,\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4.7\"},{\"name\":\"spin\",\"optional\":true,\"req\":\"^0.10.0\"},{\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^0.1.11\"},{\"name\":\"tokio-io\",\"optional\":true,\"req\":\"^0.1.9\"}],\"features\":{\"alloc\":[\"futures-core/alloc\",\"futures-task/alloc\",\"slab\"],\"async-await\":[],\"async-await-macro\":[\"async-await\",\"futures-macro\"],\"bilock\":[],\"cfg-target-has-atomic\":[],\"channel\":[\"std\",\"futures-channel\"],\"compat\":[\"std\",\"futures_01\",\"libc\"],\"default\":[\"std\",\"async-await\",\"async-await-macro\"],\"io\":[\"std\",\"futures-io\",\"memchr\"],\"io-compat\":[\"io\",\"compat\",\"tokio-io\",\"libc\"],\"portable-atomic\":[\"futures-core/portable-atomic\"],\"sink\":[\"futures-sink\"],\"std\":[\"alloc\",\"futures-core/std\",\"futures-task/std\",\"slab/std\"],\"unstable\":[\"futures-core/unstable\",\"futures-task/unstable\"],\"write-all-vectored\":[\"io\"]}}", - "getrandom_0.2.17": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(unix)\"},{\"default_features\":false,\"name\":\"wasi\",\"req\":\"^0.11\",\"target\":\"cfg(target_os = \\\"wasi\\\")\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.62\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.18\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"}],\"features\":{\"custom\":[],\"js\":[\"wasm-bindgen\",\"js-sys\"],\"linux_disable_fallback\":[],\"rdrand\":[],\"rustc-dep-of-std\":[\"compiler_builtins\",\"core\",\"libc/rustc-dep-of-std\",\"wasi/rustc-dep-of-std\"],\"std\":[],\"test-in-browser\":[]}}", - "getrandom_0.3.3": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3.77\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"), target_feature = \\\"atomics\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"netbsd\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"solaris\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"vxworks\\\")\"},{\"default_features\":false,\"name\":\"r-efi\",\"req\":\"^5.1\",\"target\":\"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\"},{\"default_features\":false,\"name\":\"wasi\",\"req\":\"^0.14\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.98\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"}],\"features\":{\"rustc-dep-of-std\":[\"dep:compiler_builtins\",\"dep:core\"],\"std\":[],\"wasm_js\":[\"dep:wasm-bindgen\",\"dep:js-sys\"]}}", - "getrandom_0.4.2": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3.77\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"), target_feature = \\\"atomics\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"netbsd\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"solaris\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"vxworks\\\")\"},{\"default_features\":false,\"name\":\"r-efi\",\"req\":\"^6\",\"target\":\"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\"},{\"name\":\"rand_core\",\"optional\":true,\"req\":\"^0.10.0\"},{\"default_features\":false,\"name\":\"wasip2\",\"req\":\"^1\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\"},{\"name\":\"wasip3\",\"req\":\"^0.4\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p3\\\"))\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.98\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"}],\"features\":{\"std\":[],\"sys_rng\":[\"dep:rand_core\"],\"wasm_js\":[\"dep:wasm-bindgen\",\"dep:js-sys\"]}}", - "gimli_0.32.3": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"fallible-iterator\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.0.0\"},{\"default_features\":false,\"name\":\"stable_deref_trait\",\"optional\":true,\"req\":\"^1.1.0\"},{\"kind\":\"dev\",\"name\":\"test-assembler\",\"req\":\"^0.1.3\"}],\"features\":{\"default\":[\"read-all\",\"write\"],\"endian-reader\":[\"read\",\"dep:stable_deref_trait\"],\"fallible-iterator\":[\"dep:fallible-iterator\"],\"read\":[\"read-core\"],\"read-all\":[\"read\",\"std\",\"fallible-iterator\",\"endian-reader\"],\"read-core\":[],\"rustc-dep-of-std\":[\"dep:core\",\"dep:alloc\"],\"std\":[\"fallible-iterator?/std\",\"stable_deref_trait?/std\"],\"write\":[\"dep:indexmap\"]}}", - "h2_0.4.6": "{\"dependencies\":[{\"name\":\"atomic-waker\",\"req\":\"^1.0.0\"},{\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.10\"},{\"name\":\"fnv\",\"req\":\"^1.0.5\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-sink\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.3\"},{\"name\":\"http\",\"req\":\"^1\"},{\"features\":[\"std\"],\"name\":\"indexmap\",\"req\":\"^2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.4\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.0\"},{\"name\":\"slab\",\"req\":\"^0.4.2\"},{\"features\":[\"io-util\"],\"name\":\"tokio\",\"req\":\"^1\"},{\"features\":[\"rt-multi-thread\",\"macros\",\"sync\",\"net\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tokio-rustls\",\"req\":\"^0.26\"},{\"features\":[\"codec\",\"io\"],\"name\":\"tokio-util\",\"req\":\"^0.7.1\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"req\":\"^0.1.35\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\"},{\"kind\":\"dev\",\"name\":\"webpki-roots\",\"req\":\"^0.26\"}],\"features\":{\"stream\":[],\"unstable\":[]}}", - "hashbrown_0.12.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"ahash\",\"optional\":true,\"req\":\"^0.7.0\"},{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"bumpalo\",\"optional\":true,\"req\":\"^3.5.0\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.3\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.25\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"ahash-compile-time-rng\":[\"ahash/compile-time-rng\"],\"default\":[\"ahash\",\"inline-more\"],\"inline-more\":[],\"nightly\":[],\"raw\":[],\"rustc-dep-of-std\":[\"nightly\",\"core\",\"compiler_builtins\",\"alloc\",\"rustc-internal-api\"],\"rustc-internal-api\":[]}}", - "hashbrown_0.15.5": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"allocator-api2\",\"optional\":true,\"req\":\"^0.2.9\"},{\"features\":[\"allocator-api2\"],\"kind\":\"dev\",\"name\":\"bumpalo\",\"req\":\"^3.13.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.1\"},{\"default_features\":false,\"name\":\"equivalent\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"default_features\":false,\"name\":\"foldhash\",\"optional\":true,\"req\":\"^0.1.2\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9.0\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.2\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.2\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.25\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"default-hasher\",\"inline-more\",\"allocator-api2\",\"equivalent\",\"raw-entry\"],\"default-hasher\":[\"dep:foldhash\"],\"inline-more\":[],\"nightly\":[\"bumpalo/allocator_api\"],\"raw-entry\":[],\"rustc-dep-of-std\":[\"nightly\",\"core\",\"alloc\",\"rustc-internal-api\"],\"rustc-internal-api\":[]}}", - "hashbrown_0.16.1": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"allocator-api2\",\"optional\":true,\"req\":\"^0.2.9\"},{\"features\":[\"allocator-api2\"],\"kind\":\"dev\",\"name\":\"bumpalo\",\"req\":\"^3.13.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"equivalent\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"default_features\":false,\"name\":\"foldhash\",\"optional\":true,\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.155\",\"target\":\"cfg(unix)\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9.0\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.9.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.2\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.221\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"default-hasher\",\"inline-more\",\"allocator-api2\",\"equivalent\",\"raw-entry\"],\"default-hasher\":[\"dep:foldhash\"],\"inline-more\":[],\"nightly\":[\"foldhash?/nightly\",\"bumpalo/allocator_api\"],\"raw-entry\":[],\"rustc-dep-of-std\":[\"nightly\",\"core\",\"alloc\",\"rustc-internal-api\"],\"rustc-internal-api\":[],\"serde\":[\"dep:serde_core\",\"dep:serde\"]}}", - "heck_0.5.0": "{\"dependencies\":[],\"features\":{}}", - "http-body-util_0.1.3": "{\"dependencies\":[{\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"http-body\",\"req\":\"^1\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"macros\",\"rt\",\"sync\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"}],\"features\":{\"channel\":[\"dep:tokio\"],\"default\":[],\"full\":[\"channel\"]}}", - "http-body_1.0.1": "{\"dependencies\":[{\"name\":\"bytes\",\"req\":\"^1\"},{\"name\":\"http\",\"req\":\"^1\"}],\"features\":{}}", - "http_1.4.0": "{\"dependencies\":[{\"name\":\"bytes\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"name\":\"itoa\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.0\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "httparse_1.10.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.5\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "httpdate_1.0.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"}],\"features\":{}}", - "hyper-timeout_0.5.2": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"name\":\"hyper\",\"req\":\"^1.1\"},{\"features\":[\"http1\"],\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^1.1\"},{\"kind\":\"dev\",\"name\":\"hyper-tls\",\"req\":\"^0.6\"},{\"features\":[\"client-legacy\",\"http1\"],\"name\":\"hyper-util\",\"req\":\"^0.1.10\"},{\"features\":[\"client-legacy\",\"http1\",\"server\",\"server-graceful\"],\"kind\":\"dev\",\"name\":\"hyper-util\",\"req\":\"^0.1.10\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"name\":\"tokio\",\"req\":\"^1.35\"},{\"features\":[\"io-std\",\"io-util\",\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.35\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"}],\"features\":{}}", - "hyper-util_0.1.20": "{\"dependencies\":[{\"name\":\"base64\",\"optional\":true,\"req\":\"^0.22\"},{\"name\":\"bytes\",\"req\":\"^1.7.1\"},{\"kind\":\"dev\",\"name\":\"bytes\",\"req\":\"^1\"},{\"name\":\"futures-channel\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.16\"},{\"default_features\":false,\"features\":[\"alloc\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3.16\"},{\"name\":\"http\",\"req\":\"^1.0\"},{\"name\":\"http-body\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1.0\"},{\"name\":\"hyper\",\"req\":\"^1.8.0\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^1.4.0\"},{\"name\":\"ipnet\",\"optional\":true,\"req\":\"^2.9\"},{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2\"},{\"name\":\"percent-encoding\",\"optional\":true,\"req\":\"^2.3\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.4\"},{\"kind\":\"dev\",\"name\":\"pnet_datalink\",\"req\":\"^0.35.0\",\"target\":\"cfg(any(target_os = \\\"linux\\\", target_os = \\\"macos\\\"))\"},{\"kind\":\"dev\",\"name\":\"pretty_env_logger\",\"req\":\"^0.5\"},{\"features\":[\"all\"],\"name\":\"socket2\",\"optional\":true,\"req\":\">=0.5.9, <0.7\"},{\"name\":\"system-configuration\",\"optional\":true,\"req\":\"^0.7\",\"target\":\"cfg(target_os = \\\"macos\\\")\"},{\"default_features\":false,\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"macros\",\"test-util\",\"signal\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"name\":\"tower-layer\",\"optional\":true,\"req\":\"^0.3\"},{\"name\":\"tower-service\",\"optional\":true,\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"tower-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"windows-registry\",\"optional\":true,\"req\":\">=0.3, <0.7\",\"target\":\"cfg(windows)\"}],\"features\":{\"__internal_happy_eyeballs_tests\":[],\"client\":[\"hyper/client\",\"tokio/net\",\"dep:tracing\",\"dep:futures-channel\",\"dep:tower-service\"],\"client-legacy\":[\"client\",\"dep:socket2\",\"tokio/sync\",\"dep:libc\",\"dep:futures-util\"],\"client-pool\":[\"client\",\"dep:futures-util\",\"dep:tower-layer\"],\"client-proxy\":[\"client\",\"dep:base64\",\"dep:ipnet\",\"dep:percent-encoding\"],\"client-proxy-system\":[\"dep:system-configuration\",\"dep:windows-registry\"],\"default\":[],\"full\":[\"client\",\"client-legacy\",\"client-pool\",\"client-proxy\",\"client-proxy-system\",\"server\",\"server-auto\",\"server-graceful\",\"service\",\"http1\",\"http2\",\"tokio\",\"tracing\"],\"http1\":[\"hyper/http1\"],\"http2\":[\"hyper/http2\"],\"server\":[\"hyper/server\"],\"server-auto\":[\"server\",\"http1\",\"http2\"],\"server-graceful\":[\"server\",\"tokio/sync\"],\"service\":[\"dep:tower-service\"],\"tokio\":[\"dep:tokio\",\"tokio/rt\",\"tokio/time\"],\"tracing\":[\"dep:tracing\"]}}", - "hyper_1.8.1": "{\"dependencies\":[{\"name\":\"atomic-waker\",\"optional\":true,\"req\":\"^1.1.2\"},{\"name\":\"bytes\",\"req\":\"^1.2\"},{\"kind\":\"dev\",\"name\":\"form_urlencoded\",\"req\":\"^1\"},{\"name\":\"futures-channel\",\"optional\":true,\"req\":\"^0.3\"},{\"features\":[\"sink\"],\"kind\":\"dev\",\"name\":\"futures-channel\",\"req\":\"^0.3\"},{\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3.31\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\",\"sink\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"h2\",\"optional\":true,\"req\":\"^0.4.2\"},{\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"http-body\",\"req\":\"^1\"},{\"name\":\"http-body-util\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"name\":\"httparse\",\"optional\":true,\"req\":\"^1.9\"},{\"name\":\"httpdate\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"itoa\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"pin-project-lite\",\"optional\":true,\"req\":\"^0.2.4\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.4\"},{\"name\":\"pin-utils\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"pretty_env_logger\",\"req\":\"^0.5\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"features\":[\"const_generics\",\"const_new\"],\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1.12\"},{\"kind\":\"dev\",\"name\":\"spmc\",\"req\":\"^0.3\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"req\":\"^1\"},{\"features\":[\"fs\",\"macros\",\"net\",\"io-std\",\"io-util\",\"rt\",\"rt-multi-thread\",\"sync\",\"time\",\"test-util\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"tokio-util\",\"req\":\"^0.7.10\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"want\",\"optional\":true,\"req\":\"^0.3\"}],\"features\":{\"capi\":[],\"client\":[\"dep:want\",\"dep:pin-project-lite\",\"dep:smallvec\"],\"default\":[],\"ffi\":[\"dep:http-body-util\",\"dep:futures-util\"],\"full\":[\"client\",\"http1\",\"http2\",\"server\"],\"http1\":[\"dep:atomic-waker\",\"dep:futures-channel\",\"dep:futures-core\",\"dep:httparse\",\"dep:itoa\",\"dep:pin-utils\"],\"http2\":[\"dep:futures-channel\",\"dep:futures-core\",\"dep:h2\"],\"nightly\":[],\"server\":[\"dep:httpdate\",\"dep:pin-project-lite\",\"dep:smallvec\"],\"tracing\":[\"dep:tracing\"]}}", - "id-arena_2.3.0": "{\"dependencies\":[{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.0.3\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "ident_case_1.0.1": "{\"dependencies\":[],\"features\":{}}", - "indexmap_1.9.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"build\",\"name\":\"autocfg\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2.1\"},{\"default_features\":false,\"features\":[\"raw\"],\"name\":\"hashbrown\",\"req\":\"^0.12\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.3\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.4.1\"},{\"name\":\"rustc-rayon\",\"optional\":true,\"package\":\"rustc-rayon\",\"req\":\"^0.5\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0\"}],\"features\":{\"serde-1\":[\"serde\"],\"std\":[],\"test_debug\":[],\"test_low_transition_point\":[]}}", - "indexmap_2.13.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"borsh\",\"optional\":true,\"req\":\"^1.2\"},{\"default_features\":false,\"name\":\"equivalent\",\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"fastrand\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"hashbrown\",\"req\":\"^0.16.1\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.14\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.9\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"},{\"default_features\":false,\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.220\"},{\"default_features\":false,\"name\":\"sval\",\"optional\":true,\"req\":\"^2\"}],\"features\":{\"default\":[\"std\"],\"serde\":[\"dep:serde_core\",\"dep:serde\"],\"std\":[],\"test_debug\":[]}}", - "itertools_0.13.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"permutohedron\",\"req\":\"^0.2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"use_std\"],\"use_alloc\":[],\"use_std\":[\"use_alloc\",\"either/use_std\"]}}", - "itertools_0.14.0": "{\"dependencies\":[{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"permutohedron\",\"req\":\"^0.2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"use_std\"],\"use_alloc\":[],\"use_std\":[\"use_alloc\",\"either/use_std\"]}}", - "itoa_1.0.14": "{\"dependencies\":[{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"}],\"features\":{}}", - "itoa_1.0.17": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\",\"target\":\"cfg(not(miri))\"},{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"}],\"features\":{}}", - "leb128fmt_0.1.0": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[]}}", - "libc_0.2.169": "{\"dependencies\":[{\"name\":\"rustc-std-workspace-core\",\"optional\":true,\"req\":\"^1.0.0\"}],\"features\":{\"align\":[],\"const-extern-fn\":[],\"default\":[\"std\"],\"extra_traits\":[],\"rustc-dep-of-std\":[\"align\",\"rustc-std-workspace-core\"],\"std\":[],\"use_std\":[\"std\"]}}", - "libc_0.2.183": "{\"dependencies\":[{\"name\":\"rustc-std-workspace-core\",\"optional\":true,\"req\":\"^1.0.1\"}],\"features\":{\"align\":[],\"const-extern-fn\":[],\"default\":[\"std\"],\"extra_traits\":[],\"rustc-dep-of-std\":[\"align\",\"rustc-std-workspace-core\"],\"std\":[],\"use_std\":[\"std\"]}}", - "linux-raw-sys_0.12.1": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.100\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1.0\"}],\"features\":{\"auxvec\":[],\"bootparam\":[],\"btrfs\":[],\"default\":[\"std\",\"general\",\"errno\"],\"elf\":[],\"elf_uapi\":[],\"errno\":[],\"general\":[],\"if_arp\":[],\"if_ether\":[],\"if_packet\":[],\"if_tun\":[],\"image\":[],\"io_uring\":[],\"ioctl\":[],\"landlock\":[],\"loop_device\":[],\"mempolicy\":[],\"net\":[],\"netlink\":[],\"no_std\":[],\"prctl\":[],\"ptrace\":[],\"rustc-dep-of-std\":[\"core\",\"no_std\"],\"std\":[],\"system\":[],\"vm_sockets\":[],\"xdp\":[]}}", - "lock_api_0.4.14": "{\"dependencies\":[{\"name\":\"owning_ref\",\"optional\":true,\"req\":\"^0.4.1\"},{\"default_features\":false,\"name\":\"scopeguard\",\"req\":\"^1.1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.126\"}],\"features\":{\"arc_lock\":[],\"atomic_usize\":[],\"default\":[\"atomic_usize\"],\"nightly\":[]}}", - "log_0.4.22": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.63\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"sval\",\"optional\":true,\"req\":\"^2.1\"},{\"kind\":\"dev\",\"name\":\"sval\",\"req\":\"^2.1\"},{\"kind\":\"dev\",\"name\":\"sval_derive\",\"req\":\"^2.1\"},{\"default_features\":false,\"name\":\"sval_ref\",\"optional\":true,\"req\":\"^2.1\"},{\"default_features\":false,\"features\":[\"inline-i128\"],\"name\":\"value-bag\",\"optional\":true,\"req\":\"^1.7\"},{\"features\":[\"test\"],\"kind\":\"dev\",\"name\":\"value-bag\",\"req\":\"^1.7\"}],\"features\":{\"kv\":[],\"kv_serde\":[\"kv_std\",\"value-bag/serde\",\"serde\"],\"kv_std\":[\"std\",\"kv\",\"value-bag/error\"],\"kv_sval\":[\"kv\",\"value-bag/sval\",\"sval\",\"sval_ref\"],\"kv_unstable\":[\"kv\",\"value-bag\"],\"kv_unstable_serde\":[\"kv_serde\",\"kv_unstable_std\"],\"kv_unstable_std\":[\"kv_std\",\"kv_unstable\"],\"kv_unstable_sval\":[\"kv_sval\",\"kv_unstable\"],\"max_level_debug\":[],\"max_level_error\":[],\"max_level_info\":[],\"max_level_off\":[],\"max_level_trace\":[],\"max_level_warn\":[],\"release_max_level_debug\":[],\"release_max_level_error\":[],\"release_max_level_info\":[],\"release_max_level_off\":[],\"release_max_level_trace\":[],\"release_max_level_warn\":[],\"std\":[]}}", - "log_0.4.29": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.63\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"sval\",\"optional\":true,\"req\":\"^2.16\"},{\"kind\":\"dev\",\"name\":\"sval\",\"req\":\"^2.16\"},{\"kind\":\"dev\",\"name\":\"sval_derive\",\"req\":\"^2.16\"},{\"default_features\":false,\"name\":\"sval_ref\",\"optional\":true,\"req\":\"^2.16\"},{\"default_features\":false,\"features\":[\"inline-i128\"],\"name\":\"value-bag\",\"optional\":true,\"req\":\"^1.12\"},{\"features\":[\"test\"],\"kind\":\"dev\",\"name\":\"value-bag\",\"req\":\"^1.12\"}],\"features\":{\"kv\":[],\"kv_serde\":[\"kv_std\",\"value-bag/serde\",\"serde\"],\"kv_std\":[\"std\",\"kv\",\"value-bag/error\"],\"kv_sval\":[\"kv\",\"value-bag/sval\",\"sval\",\"sval_ref\"],\"kv_unstable\":[\"kv\",\"value-bag\"],\"kv_unstable_serde\":[\"kv_serde\",\"kv_unstable_std\"],\"kv_unstable_std\":[\"kv_std\",\"kv_unstable\"],\"kv_unstable_sval\":[\"kv_sval\",\"kv_unstable\"],\"max_level_debug\":[],\"max_level_error\":[],\"max_level_info\":[],\"max_level_off\":[],\"max_level_trace\":[],\"max_level_warn\":[],\"release_max_level_debug\":[],\"release_max_level_error\":[],\"release_max_level_info\":[],\"release_max_level_off\":[],\"release_max_level_trace\":[],\"release_max_level_warn\":[],\"serde\":[\"serde_core\"],\"std\":[]}}", - "macro_rules_attribute-proc_macro_0.2.0": "{\"dependencies\":[],\"features\":{\"verbose-expansions\":[]}}", - "macro_rules_attribute_0.2.0": "{\"dependencies\":[{\"name\":\"macro_rules_attribute-proc_macro\",\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.10.0\"},{\"name\":\"paste\",\"req\":\"^1.0.7\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.8\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.136\"}],\"features\":{\"better-docs\":[],\"default\":[],\"verbose-expansions\":[\"macro_rules_attribute-proc_macro/verbose-expansions\"]}}", - "matchit_0.7.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"actix-router\",\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.4\"},{\"kind\":\"dev\",\"name\":\"gonzales\",\"req\":\"^0.0.3-beta\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^0.14\"},{\"kind\":\"dev\",\"name\":\"path-tree\",\"req\":\"^0.2.2\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.5.4\"},{\"kind\":\"dev\",\"name\":\"route-recognizer\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"routefinder\",\"req\":\"^0.5.2\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"features\":[\"make\",\"util\"],\"kind\":\"dev\",\"name\":\"tower\",\"req\":\"^0.4\"}],\"features\":{\"__test_helpers\":[],\"default\":[]}}", - "memchr_2.7.4": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.20\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"libc\":[],\"logging\":[\"dep:log\"],\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"],\"std\":[\"alloc\"],\"use_std\":[\"std\"]}}", - "memchr_2.8.0": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.20\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"libc\":[],\"logging\":[\"dep:log\"],\"rustc-dep-of-std\":[\"core\"],\"std\":[\"alloc\"],\"use_std\":[\"std\"]}}", - "mime_0.3.17": "{\"dependencies\":[],\"features\":{}}", - "minimal-lexical_0.2.1": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"compact\":[],\"default\":[\"std\"],\"lint\":[],\"nightly\":[],\"std\":[]}}", - "miniz_oxide_0.8.9": "{\"dependencies\":[{\"default_features\":false,\"name\":\"adler2\",\"req\":\"^2.0\"},{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"simd-adler32\",\"optional\":true,\"req\":\"^0.3.3\"}],\"features\":{\"block-boundary\":[],\"default\":[\"with-alloc\"],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"adler2/rustc-dep-of-std\"],\"simd\":[\"simd-adler32\"],\"std\":[],\"with-alloc\":[]}}", - "mio_1.1.1": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"libc\",\"req\":\"^0.2.178\",\"target\":\"cfg(target_os = \\\"hermit\\\")\"},{\"name\":\"libc\",\"req\":\"^0.2.178\",\"target\":\"cfg(target_os = \\\"wasi\\\")\"},{\"name\":\"libc\",\"req\":\"^0.2.178\",\"target\":\"cfg(unix)\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.8\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9\"},{\"name\":\"wasi\",\"req\":\"^0.11.0\",\"target\":\"cfg(target_os = \\\"wasi\\\")\"},{\"features\":[\"Wdk_Foundation\",\"Wdk_Storage_FileSystem\",\"Wdk_System_IO\",\"Win32_Foundation\",\"Win32_Networking_WinSock\",\"Win32_Storage_FileSystem\",\"Win32_Security\",\"Win32_System_IO\",\"Win32_System_WindowsProgramming\"],\"name\":\"windows-sys\",\"req\":\"^0.61\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[\"log\"],\"net\":[],\"os-ext\":[\"os-poll\",\"windows-sys/Win32_System_Pipes\",\"windows-sys/Win32_Security\"],\"os-poll\":[]}}", - "monostate-impl_0.1.13": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"default_features\":false,\"features\":[\"parsing\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0.59\"}],\"features\":{}}", - "monostate_0.1.13": "{\"dependencies\":[{\"name\":\"monostate-impl\",\"req\":\"=0.1.13\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.166\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.99\"}],\"features\":{}}", - "multimap_0.10.1": "{\"dependencies\":[{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"serde_impl\"],\"serde_impl\":[\"serde\"]}}", - "nom_7.1.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2.3\"},{\"default_features\":false,\"name\":\"minimal-lexical\",\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.0.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"docsrs\":[],\"std\":[\"alloc\",\"memchr/std\",\"minimal-lexical/std\"]}}", - "object_0.37.3": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"crc32fast\",\"optional\":true,\"req\":\"^1.2\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"default-hasher\"],\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.15.0\"},{\"default_features\":false,\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.0\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2.4.1\"},{\"name\":\"ruzstd\",\"optional\":true,\"req\":\"^0.8.1\"},{\"default_features\":false,\"name\":\"wasmparser\",\"optional\":true,\"req\":\"^0.236.0\"}],\"features\":{\"all\":[\"read\",\"write\",\"build\",\"std\",\"compression\",\"wasm\"],\"archive\":[],\"build\":[\"build_core\",\"write_std\",\"elf\"],\"build_core\":[\"read_core\",\"write_core\"],\"cargo-all\":[],\"coff\":[],\"compression\":[\"dep:flate2\",\"dep:ruzstd\",\"std\"],\"default\":[\"read\",\"compression\"],\"doc\":[\"read_core\",\"write_std\",\"build_core\",\"std\",\"compression\",\"archive\",\"coff\",\"elf\",\"macho\",\"pe\",\"wasm\",\"xcoff\"],\"elf\":[],\"macho\":[],\"pe\":[\"coff\"],\"read\":[\"read_core\",\"archive\",\"coff\",\"elf\",\"macho\",\"pe\",\"xcoff\",\"unaligned\"],\"read_core\":[],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"memchr/rustc-dep-of-std\"],\"std\":[\"memchr/std\"],\"unaligned\":[],\"unstable\":[],\"unstable-all\":[\"all\",\"unstable\"],\"wasm\":[\"dep:wasmparser\"],\"write\":[\"write_std\",\"coff\",\"elf\",\"macho\",\"pe\",\"xcoff\"],\"write_core\":[\"dep:crc32fast\",\"dep:indexmap\",\"dep:hashbrown\"],\"write_std\":[\"write_core\",\"std\",\"indexmap?/std\",\"crc32fast?/std\"],\"xcoff\":[]}}", - "once_cell_1.20.2": "{\"dependencies\":[{\"name\":\"critical-section\",\"optional\":true,\"req\":\"^1.1.3\"},{\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"critical-section\",\"req\":\"^1.1.3\"},{\"default_features\":false,\"name\":\"parking_lot_core\",\"optional\":true,\"req\":\"^0.9.10\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.10.6\"}],\"features\":{\"alloc\":[\"race\"],\"atomic-polyfill\":[\"critical-section\"],\"critical-section\":[\"dep:critical-section\",\"portable-atomic\"],\"default\":[\"std\"],\"parking_lot\":[\"dep:parking_lot_core\"],\"portable-atomic\":[\"dep:portable-atomic\"],\"race\":[],\"std\":[\"alloc\"],\"unstable\":[]}}", - "once_cell_1.21.4": "{\"dependencies\":[{\"name\":\"critical-section\",\"optional\":true,\"req\":\"^1.1.3\"},{\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"critical-section\",\"req\":\"^1.1.3\"},{\"default_features\":false,\"name\":\"parking_lot_core\",\"optional\":true,\"req\":\"^0.9.10\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.10.6\"}],\"features\":{\"alloc\":[\"race\"],\"atomic-polyfill\":[\"critical-section\"],\"critical-section\":[\"dep:critical-section\",\"portable-atomic\"],\"default\":[\"std\"],\"parking_lot\":[\"dep:parking_lot_core\"],\"portable-atomic\":[\"dep:portable-atomic\"],\"race\":[],\"std\":[\"alloc\"],\"unstable\":[]}}", - "onig_6.5.1": "{\"dependencies\":[{\"name\":\"bitflags\",\"req\":\"^2.4.0\"},{\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(windows)\"},{\"name\":\"once_cell\",\"req\":\"^1.12\"},{\"default_features\":false,\"name\":\"onig_sys\",\"req\":\"^69.9.1\"}],\"features\":{\"default\":[\"generate\"],\"generate\":[\"onig_sys/generate\"],\"posix-api\":[\"onig_sys/posix-api\"],\"print-debug\":[\"onig_sys/print-debug\"],\"std-pattern\":[]}}", - "onig_sys_69.9.1": "{\"dependencies\":[{\"features\":[\"runtime\"],\"kind\":\"build\",\"name\":\"bindgen\",\"optional\":true,\"req\":\"^0.71\"},{\"kind\":\"build\",\"name\":\"cc\",\"req\":\"^1.0\"},{\"kind\":\"build\",\"name\":\"pkg-config\",\"req\":\"^0.3.16\"}],\"features\":{\"default\":[\"generate\"],\"generate\":[\"bindgen\"],\"posix-api\":[],\"print-debug\":[]}}", - "parking_lot_0.12.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.3.3\"},{\"name\":\"lock_api\",\"req\":\"^0.4.14\"},{\"name\":\"parking_lot_core\",\"req\":\"^0.9.12\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.3\"}],\"features\":{\"arc_lock\":[\"lock_api/arc_lock\"],\"deadlock_detection\":[\"parking_lot_core/deadlock_detection\"],\"default\":[],\"hardware-lock-elision\":[],\"nightly\":[\"parking_lot_core/nightly\",\"lock_api/nightly\"],\"owning_ref\":[\"lock_api/owning_ref\"],\"send_guard\":[],\"serde\":[\"lock_api/serde\"]}}", - "parking_lot_core_0.9.12": "{\"dependencies\":[{\"name\":\"backtrace\",\"optional\":true,\"req\":\"^0.3.60\"},{\"name\":\"cfg-if\",\"req\":\"^1.0.0\"},{\"name\":\"libc\",\"req\":\"^0.2.95\",\"target\":\"cfg(unix)\"},{\"name\":\"petgraph\",\"optional\":true,\"req\":\"^0.6.0\"},{\"name\":\"redox_syscall\",\"req\":\"^0.5\",\"target\":\"cfg(target_os = \\\"redox\\\")\"},{\"name\":\"smallvec\",\"req\":\"^1.6.1\"},{\"name\":\"windows-link\",\"req\":\"^0.2.0\",\"target\":\"cfg(windows)\"}],\"features\":{\"deadlock_detection\":[\"petgraph\",\"backtrace\"],\"nightly\":[]}}", - "paste_1.0.15": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"paste-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", - "percent-encoding_2.3.2": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", - "petgraph_0.6.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"ahash\",\"req\":\"^0.7.2\"},{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.3.3\"},{\"kind\":\"dev\",\"name\":\"defmac\",\"req\":\"^0.2.1\"},{\"default_features\":false,\"name\":\"fixedbitset\",\"req\":\"^0.4.0\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2.1\"},{\"name\":\"indexmap\",\"req\":\"^2.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.12.1\"},{\"kind\":\"dev\",\"name\":\"odds\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.5.5\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.5.3\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1.0\"}],\"features\":{\"all\":[\"unstable\",\"quickcheck\",\"matrix_graph\",\"stable_graph\",\"graphmap\",\"rayon\"],\"default\":[\"graphmap\",\"stable_graph\",\"matrix_graph\"],\"generate\":[],\"graphmap\":[],\"matrix_graph\":[],\"rayon\":[\"dep:rayon\",\"indexmap/rayon\"],\"serde-1\":[\"serde\",\"serde_derive\"],\"stable_graph\":[],\"unstable\":[\"generate\"]}}", - "pin-project-internal_1.1.11": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1.0.25\"},{\"default_features\":false,\"features\":[\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\",\"full\",\"visit-mut\"],\"name\":\"syn\",\"req\":\"^2.0.1\"}],\"features\":{}}", - "pin-project-lite_0.2.17": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1\"}],\"features\":{}}", - "pin-project_1.1.11": "{\"dependencies\":[{\"name\":\"pin-project-internal\",\"req\":\"=1.1.11\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1\"}],\"features\":{}}", - "pin-utils_0.1.0": "{\"dependencies\":[],\"features\":{}}", - "pkg-config_0.3.32": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1\"}],\"features\":{}}", - "ppv-lite86_0.2.20": "{\"dependencies\":[{\"features\":[\"simd\",\"derive\"],\"name\":\"zerocopy\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"std\"],\"no_simd\":[],\"simd\":[],\"std\":[]}}", - "ppv-lite86_0.2.21": "{\"dependencies\":[{\"features\":[\"simd\"],\"name\":\"zerocopy\",\"req\":\"^0.8.23\"}],\"features\":{\"default\":[\"std\"],\"no_simd\":[],\"simd\":[],\"std\":[]}}", - "prettyplease_0.2.37": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"default_features\":false,\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.105\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"extra-traits\",\"parsing\",\"printing\",\"visit-mut\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.105\"}],\"features\":{\"verbatim\":[\"syn/parsing\"]}}", - "proc-macro2_1.0.106": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4\"},{\"name\":\"unicode-ident\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"proc-macro\"],\"nightly\":[],\"proc-macro\":[],\"span-locations\":[]}}", - "proc-macro2_1.0.92": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4\"},{\"name\":\"unicode-ident\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"proc-macro\"],\"nightly\":[],\"proc-macro\":[],\"span-locations\":[]}}", - "prost-build_0.13.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"heck\",\"req\":\">=0.4, <=0.5\"},{\"default_features\":false,\"features\":[\"use_alloc\"],\"name\":\"itertools\",\"req\":\">=0.10, <=0.13\"},{\"name\":\"log\",\"req\":\"^0.4.4\"},{\"default_features\":false,\"name\":\"multimap\",\"req\":\">=0.8, <=0.10\"},{\"name\":\"once_cell\",\"req\":\"^1.17.1\"},{\"default_features\":false,\"name\":\"petgraph\",\"req\":\"^0.6\"},{\"name\":\"prettyplease\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"name\":\"prost\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-types\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"pulldown-cmark\",\"optional\":true,\"req\":\"^0.9.1\"},{\"name\":\"pulldown-cmark-to-cmark\",\"optional\":true,\"req\":\"^10.0.1\"},{\"default_features\":false,\"features\":[\"std\",\"unicode-bool\"],\"name\":\"regex\",\"req\":\"^1.8.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"optional\":true,\"req\":\"^2\"},{\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{\"cleanup-markdown\":[\"dep:pulldown-cmark\",\"dep:pulldown-cmark-to-cmark\"],\"default\":[\"format\"],\"format\":[\"dep:prettyplease\",\"dep:syn\"]}}", - "prost-derive_0.13.1": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.1\"},{\"name\":\"itertools\",\"req\":\">=0.10.1, <=0.13\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2\"}],\"features\":{}}", - "prost-types_0.13.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"prost-derive\"],\"name\":\"prost\",\"req\":\"^0.13.1\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"prost/std\"]}}", - "prost_0.13.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"name\":\"prost-derive\",\"optional\":true,\"req\":\"^0.13.1\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"derive\",\"std\"],\"derive\":[\"dep:prost-derive\"],\"no-recursion-limit\":[],\"prost-derive\":[\"derive\"],\"std\":[]}}", - "protoc-gen-prost_0.4.0": "{\"dependencies\":[{\"name\":\"once_cell\",\"req\":\"^1.10.0\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"prost\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-build\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-types\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"regex\",\"req\":\"^1.5.5\"}],\"features\":{}}", - "protoc-gen-tonic_0.4.1": "{\"dependencies\":[{\"name\":\"heck\",\"req\":\"^0.5.0\"},{\"name\":\"prettyplease\",\"req\":\"^0.2.9\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"prost\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-build\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-types\",\"req\":\"^0.13.1\"},{\"name\":\"protoc-gen-prost\",\"req\":\"^0.4.0\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"regex\",\"req\":\"^1.5.5\"},{\"features\":[\"parsing\",\"full\"],\"name\":\"syn\",\"req\":\"^2.0.22\"},{\"name\":\"tonic-build\",\"req\":\"^0.12.0\"}],\"features\":{}}", - "quote_1.0.37": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.66\"}],\"features\":{\"default\":[\"proc-macro\"],\"proc-macro\":[\"proc-macro2/proc-macro\"]}}", - "quote_1.0.45": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"}],\"features\":{\"default\":[\"proc-macro\"],\"proc-macro\":[\"proc-macro2/proc-macro\"]}}", - "r-efi_5.3.0": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"efiapi\":[],\"examples\":[\"native\"],\"native\":[],\"rustc-dep-of-std\":[\"core\"]}}", - "r-efi_6.0.0": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"native\":[],\"rustc-dep-of-std\":[\"core\"]}}", - "rand_0.8.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.2.1\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.22\",\"target\":\"cfg(unix)\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.4\"},{\"features\":[\"into_bits\"],\"name\":\"packed_simd\",\"optional\":true,\"package\":\"packed_simd_2\",\"req\":\"^0.3.7\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"rand_core\",\"req\":\"^0.6.0\"},{\"kind\":\"dev\",\"name\":\"rand_pcg\",\"req\":\"^0.3.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"}],\"features\":{\"alloc\":[\"rand_core/alloc\"],\"default\":[\"std\",\"std_rng\"],\"getrandom\":[\"rand_core/getrandom\"],\"min_const_gen\":[],\"nightly\":[],\"serde1\":[\"serde\",\"rand_core/serde1\"],\"simd_support\":[\"packed_simd\"],\"small_rng\":[],\"std\":[\"rand_core/std\",\"rand_chacha/std\",\"alloc\",\"getrandom\",\"libc\"],\"std_rng\":[\"rand_chacha\"]}}", - "rand_0.9.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.2.1\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.4\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"optional\":true,\"req\":\"^0.9.0\"},{\"default_features\":false,\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rand_pcg\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.7\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"std_rng\",\"os_rng\",\"small_rng\",\"thread_rng\"],\"log\":[\"dep:log\"],\"nightly\":[],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\",\"rand_core/serde\"],\"simd_support\":[],\"small_rng\":[],\"std\":[\"rand_core/std\",\"rand_chacha?/std\",\"alloc\"],\"std_rng\":[\"dep:rand_chacha\"],\"thread_rng\":[\"std\",\"std_rng\",\"os_rng\"],\"unbiased\":[]}}", - "rand_chacha_0.3.1": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"ppv-lite86\",\"req\":\"^0.2.8\"},{\"name\":\"rand_core\",\"req\":\"^0.6.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"serde1\":[\"serde\"],\"simd\":[],\"std\":[\"ppv-lite86/std\"]}}", - "rand_chacha_0.9.0": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"ppv-lite86\",\"req\":\"^0.2.14\"},{\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"features\":[\"os_rng\"],\"kind\":\"dev\",\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\"],\"std\":[\"ppv-lite86/std\",\"rand_core/std\"]}}", - "rand_core_0.6.4": "{\"dependencies\":[{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.2\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"serde1\":[\"serde\"],\"std\":[\"alloc\",\"getrandom\",\"getrandom/std\"]}}", - "rand_core_0.9.3": "{\"dependencies\":[{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.3.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"os_rng\":[\"dep:getrandom\"],\"serde\":[\"dep:serde\"],\"std\":[\"getrandom?/std\"]}}", - "rayon-cond_0.4.0": "{\"dependencies\":[{\"name\":\"either\",\"req\":\"^1.6\"},{\"name\":\"itertools\",\"req\":\"^0.14\"},{\"name\":\"rayon\",\"req\":\"^1.7\"}],\"features\":{}}", - "rayon-core_1.12.1": "{\"dependencies\":[{\"name\":\"crossbeam-deque\",\"req\":\"^0.8.1\"},{\"name\":\"crossbeam-utils\",\"req\":\"^0.8.0\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"scoped-tls\",\"req\":\"^1.0\"},{\"name\":\"wasm_sync\",\"optional\":true,\"req\":\"^0.1.0\"}],\"features\":{\"web_spin_lock\":[\"dep:wasm_sync\"]}}", - "rayon_1.10.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"name\":\"rayon-core\",\"req\":\"^1.12.1\"},{\"name\":\"wasm_sync\",\"optional\":true,\"req\":\"^0.1.0\"}],\"features\":{\"web_spin_lock\":[\"dep:wasm_sync\",\"rayon-core/web_spin_lock\"]}}", - "redox_syscall_0.5.18": "{\"dependencies\":[{\"name\":\"bitflags\",\"req\":\"^2.4\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"loom\",\"req\":\"^0.7\",\"target\":\"cfg(loom)\"}],\"features\":{\"default\":[\"userspace\"],\"rustc-dep-of-std\":[\"core\",\"bitflags/rustc-dep-of-std\"],\"std\":[],\"userspace\":[]}}", - "regex-automata_0.4.14": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"bstr\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.14\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"optional\":true,\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"syntax\",\"perf\",\"unicode\",\"meta\",\"nfa\",\"dfa\",\"hybrid\"],\"dfa\":[\"dfa-build\",\"dfa-search\",\"dfa-onepass\"],\"dfa-build\":[\"nfa-thompson\",\"dfa-search\"],\"dfa-onepass\":[\"nfa-thompson\"],\"dfa-search\":[],\"hybrid\":[\"alloc\",\"nfa-thompson\"],\"internal-instrument\":[\"internal-instrument-pikevm\"],\"internal-instrument-pikevm\":[\"logging\",\"std\"],\"logging\":[\"dep:log\",\"aho-corasick?/logging\",\"memchr?/logging\"],\"meta\":[\"syntax\",\"nfa-pikevm\"],\"nfa\":[\"nfa-thompson\",\"nfa-pikevm\",\"nfa-backtrack\"],\"nfa-backtrack\":[\"nfa-thompson\"],\"nfa-pikevm\":[\"nfa-thompson\"],\"nfa-thompson\":[\"alloc\"],\"perf\":[\"perf-inline\",\"perf-literal\"],\"perf-inline\":[],\"perf-literal\":[\"perf-literal-substring\",\"perf-literal-multisubstring\"],\"perf-literal-multisubstring\":[\"dep:aho-corasick\"],\"perf-literal-substring\":[\"aho-corasick?/perf-literal\",\"dep:memchr\"],\"std\":[\"regex-syntax?/std\",\"memchr?/std\",\"aho-corasick?/std\",\"alloc\"],\"syntax\":[\"dep:regex-syntax\",\"alloc\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"unicode-word-boundary\",\"regex-syntax?/unicode\"],\"unicode-age\":[\"regex-syntax?/unicode-age\"],\"unicode-bool\":[\"regex-syntax?/unicode-bool\"],\"unicode-case\":[\"regex-syntax?/unicode-case\"],\"unicode-gencat\":[\"regex-syntax?/unicode-gencat\"],\"unicode-perl\":[\"regex-syntax?/unicode-perl\"],\"unicode-script\":[\"regex-syntax?/unicode-script\"],\"unicode-segment\":[\"regex-syntax?/unicode-segment\"],\"unicode-word-boundary\":[]}}", - "regex-automata_0.4.9": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"bstr\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.14\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"optional\":true,\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"syntax\",\"perf\",\"unicode\",\"meta\",\"nfa\",\"dfa\",\"hybrid\"],\"dfa\":[\"dfa-build\",\"dfa-search\",\"dfa-onepass\"],\"dfa-build\":[\"nfa-thompson\",\"dfa-search\"],\"dfa-onepass\":[\"nfa-thompson\"],\"dfa-search\":[],\"hybrid\":[\"alloc\",\"nfa-thompson\"],\"internal-instrument\":[\"internal-instrument-pikevm\"],\"internal-instrument-pikevm\":[\"logging\",\"std\"],\"logging\":[\"dep:log\",\"aho-corasick?/logging\",\"memchr?/logging\"],\"meta\":[\"syntax\",\"nfa-pikevm\"],\"nfa\":[\"nfa-thompson\",\"nfa-pikevm\",\"nfa-backtrack\"],\"nfa-backtrack\":[\"nfa-thompson\"],\"nfa-pikevm\":[\"nfa-thompson\"],\"nfa-thompson\":[\"alloc\"],\"perf\":[\"perf-inline\",\"perf-literal\"],\"perf-inline\":[],\"perf-literal\":[\"perf-literal-substring\",\"perf-literal-multisubstring\"],\"perf-literal-multisubstring\":[\"std\",\"dep:aho-corasick\"],\"perf-literal-substring\":[\"aho-corasick?/perf-literal\",\"dep:memchr\"],\"std\":[\"regex-syntax?/std\",\"memchr?/std\",\"aho-corasick?/std\",\"alloc\"],\"syntax\":[\"dep:regex-syntax\",\"alloc\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"unicode-word-boundary\",\"regex-syntax?/unicode\"],\"unicode-age\":[\"regex-syntax?/unicode-age\"],\"unicode-bool\":[\"regex-syntax?/unicode-bool\"],\"unicode-case\":[\"regex-syntax?/unicode-case\"],\"unicode-gencat\":[\"regex-syntax?/unicode-gencat\"],\"unicode-perl\":[\"regex-syntax?/unicode-perl\"],\"unicode-script\":[\"regex-syntax?/unicode-script\"],\"unicode-segment\":[\"regex-syntax?/unicode-segment\"],\"unicode-word-boundary\":[]}}", - "regex-syntax_0.8.10": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.3.0\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"default\":[\"std\",\"unicode\"],\"std\":[],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\"],\"unicode-age\":[],\"unicode-bool\":[],\"unicode-case\":[],\"unicode-gencat\":[],\"unicode-perl\":[],\"unicode-script\":[],\"unicode-segment\":[]}}", - "regex-syntax_0.8.5": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.3.0\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"default\":[\"std\",\"unicode\"],\"std\":[],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\"],\"unicode-age\":[],\"unicode-bool\":[],\"unicode-case\":[],\"unicode-gencat\":[],\"unicode-perl\":[],\"unicode-script\":[],\"unicode-segment\":[]}}", - "regex_1.11.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.17.1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"features\":[\"alloc\",\"syntax\",\"meta\",\"nfa-pikevm\"],\"name\":\"regex-automata\",\"req\":\"^0.4.8\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"default\":[\"std\",\"perf\",\"unicode\",\"regex-syntax/default\"],\"logging\":[\"aho-corasick?/logging\",\"memchr?/logging\",\"regex-automata/logging\"],\"pattern\":[],\"perf\":[\"perf-cache\",\"perf-dfa\",\"perf-onepass\",\"perf-backtrack\",\"perf-inline\",\"perf-literal\"],\"perf-backtrack\":[\"regex-automata/nfa-backtrack\"],\"perf-cache\":[],\"perf-dfa\":[\"regex-automata/hybrid\"],\"perf-dfa-full\":[\"regex-automata/dfa-build\",\"regex-automata/dfa-search\"],\"perf-inline\":[\"regex-automata/perf-inline\"],\"perf-literal\":[\"dep:aho-corasick\",\"dep:memchr\",\"regex-automata/perf-literal\"],\"perf-onepass\":[\"regex-automata/dfa-onepass\"],\"std\":[\"aho-corasick?/std\",\"memchr?/std\",\"regex-automata/std\",\"regex-syntax/std\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"regex-automata/unicode\",\"regex-syntax/unicode\"],\"unicode-age\":[\"regex-automata/unicode-age\",\"regex-syntax/unicode-age\"],\"unicode-bool\":[\"regex-automata/unicode-bool\",\"regex-syntax/unicode-bool\"],\"unicode-case\":[\"regex-automata/unicode-case\",\"regex-syntax/unicode-case\"],\"unicode-gencat\":[\"regex-automata/unicode-gencat\",\"regex-syntax/unicode-gencat\"],\"unicode-perl\":[\"regex-automata/unicode-perl\",\"regex-automata/unicode-word-boundary\",\"regex-syntax/unicode-perl\"],\"unicode-script\":[\"regex-automata/unicode-script\",\"regex-syntax/unicode-script\"],\"unicode-segment\":[\"regex-automata/unicode-segment\",\"regex-syntax/unicode-segment\"],\"unstable\":[\"pattern\"],\"use_std\":[\"std\"]}}", - "regex_1.12.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"features\":[\"alloc\",\"syntax\",\"meta\",\"nfa-pikevm\"],\"name\":\"regex-automata\",\"req\":\"^0.4.12\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"default\":[\"std\",\"perf\",\"unicode\",\"regex-syntax/default\"],\"logging\":[\"aho-corasick?/logging\",\"memchr?/logging\",\"regex-automata/logging\"],\"pattern\":[],\"perf\":[\"perf-cache\",\"perf-dfa\",\"perf-onepass\",\"perf-backtrack\",\"perf-inline\",\"perf-literal\"],\"perf-backtrack\":[\"regex-automata/nfa-backtrack\"],\"perf-cache\":[],\"perf-dfa\":[\"regex-automata/hybrid\"],\"perf-dfa-full\":[\"regex-automata/dfa-build\",\"regex-automata/dfa-search\"],\"perf-inline\":[\"regex-automata/perf-inline\"],\"perf-literal\":[\"dep:aho-corasick\",\"dep:memchr\",\"regex-automata/perf-literal\"],\"perf-onepass\":[\"regex-automata/dfa-onepass\"],\"std\":[\"aho-corasick?/std\",\"memchr?/std\",\"regex-automata/std\",\"regex-syntax/std\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"regex-automata/unicode\",\"regex-syntax/unicode\"],\"unicode-age\":[\"regex-automata/unicode-age\",\"regex-syntax/unicode-age\"],\"unicode-bool\":[\"regex-automata/unicode-bool\",\"regex-syntax/unicode-bool\"],\"unicode-case\":[\"regex-automata/unicode-case\",\"regex-syntax/unicode-case\"],\"unicode-gencat\":[\"regex-automata/unicode-gencat\",\"regex-syntax/unicode-gencat\"],\"unicode-perl\":[\"regex-automata/unicode-perl\",\"regex-automata/unicode-word-boundary\",\"regex-syntax/unicode-perl\"],\"unicode-script\":[\"regex-automata/unicode-script\",\"regex-syntax/unicode-script\"],\"unicode-segment\":[\"regex-automata/unicode-segment\",\"regex-syntax/unicode-segment\"],\"unstable\":[\"pattern\"],\"use_std\":[\"std\"]}}", - "rustc-demangle_0.1.27": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"compiler_builtins\":[],\"rustc-dep-of-std\":[\"core\"],\"std\":[]}}", - "rustix_1.1.4": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bitflags\",\"req\":\"^2.4.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4\",\"target\":\"cfg(all(criterion, not(any(target_os = \\\"emscripten\\\", target_os = \\\"wasi\\\"))))\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.182\",\"target\":\"cfg(all(not(windows), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.182\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.171\"},{\"default_features\":false,\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(all(not(windows), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(windows)\"},{\"default_features\":false,\"name\":\"libc_errno\",\"optional\":true,\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\"},{\"default_features\":false,\"features\":[\"general\",\"ioctl\",\"no_std\"],\"name\":\"linux-raw-sys\",\"req\":\"^0.12\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"features\":[\"auxvec\",\"general\",\"errno\",\"ioctl\",\"no_std\",\"elf\"],\"name\":\"linux-raw-sys\",\"req\":\"^0.12\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"kind\":\"dev\",\"name\":\"memoffset\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.20.3\",\"target\":\"cfg(windows)\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"serial_test\",\"req\":\"^2.0.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.5.0\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\"],\"name\":\"windows-sys\",\"req\":\">=0.52, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"all-apis\":[\"event\",\"fs\",\"io_uring\",\"mm\",\"mount\",\"net\",\"param\",\"pipe\",\"process\",\"pty\",\"rand\",\"runtime\",\"shm\",\"stdio\",\"system\",\"termios\",\"thread\",\"time\"],\"alloc\":[],\"default\":[\"std\"],\"event\":[],\"fs\":[],\"io_uring\":[\"event\",\"fs\",\"net\",\"thread\",\"linux-raw-sys/io_uring\"],\"linux_4_11\":[],\"linux_5_1\":[\"linux_4_11\"],\"linux_5_11\":[\"linux_5_1\"],\"linux_latest\":[\"linux_5_11\"],\"mm\":[],\"mount\":[],\"net\":[\"linux-raw-sys/net\",\"linux-raw-sys/netlink\",\"linux-raw-sys/if_ether\",\"linux-raw-sys/xdp\"],\"param\":[],\"pipe\":[],\"process\":[\"linux-raw-sys/prctl\"],\"pty\":[\"fs\"],\"rand\":[],\"runtime\":[\"linux-raw-sys/prctl\"],\"rustc-dep-of-std\":[\"core\",\"rustc-std-workspace-alloc\",\"linux-raw-sys/rustc-dep-of-std\",\"bitflags/rustc-dep-of-std\"],\"shm\":[\"fs\"],\"std\":[\"bitflags/std\",\"alloc\",\"libc?/std\",\"libc_errno?/std\"],\"stdio\":[],\"system\":[\"linux-raw-sys/system\"],\"termios\":[],\"thread\":[\"linux-raw-sys/prctl\"],\"time\":[],\"try_close\":[],\"use-explicitly-provided-auxv\":[],\"use-libc\":[\"libc_errno\",\"libc\"],\"use-libc-auxv\":[]}}", - "rustversion_1.0.21": "{\"dependencies\":[{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", - "rustversion_1.0.22": "{\"dependencies\":[{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", - "ryu_1.0.18": "{\"dependencies\":[{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"num_cpus\",\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"}],\"features\":{\"small\":[]}}", - "scopeguard_1.2.0": "{\"dependencies\":[],\"features\":{\"default\":[\"use_std\"],\"use_std\":[]}}", - "semver_1.0.27": "{\"dependencies\":[{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"package\":\"serde_core\",\"req\":\"^1.0.220\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"}],\"features\":{\"default\":[\"std\"],\"serde\":[\"dep:serde\"],\"std\":[]}}", - "serde_1.0.216": "{\"dependencies\":[{\"name\":\"serde_derive\",\"req\":\"=1.0.216\",\"target\":\"cfg(any())\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"derive\":[\"serde_derive\"],\"rc\":[],\"std\":[],\"unstable\":[]}}", - "serde_1.0.228": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"result\"],\"name\":\"serde_core\",\"req\":\"=1.0.228\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[\"serde_core/alloc\"],\"default\":[\"std\"],\"derive\":[\"serde_derive\"],\"rc\":[\"serde_core/rc\"],\"std\":[\"serde_core/std\"],\"unstable\":[\"serde_core/unstable\"]}}", - "serde_core_1.0.228": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"name\":\"serde_derive\",\"req\":\"=1.0.228\",\"target\":\"cfg(any())\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"result\"],\"rc\":[],\"result\":[],\"std\":[],\"unstable\":[]}}", - "serde_derive_1.0.216": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"derive\",\"parsing\",\"printing\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0.81\"}],\"features\":{\"default\":[],\"deserialize_in_place\":[]}}", - "serde_derive_1.0.228": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"derive\",\"parsing\",\"printing\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0.81\"}],\"features\":{\"default\":[],\"deserialize_in_place\":[]}}", - "serde_json_1.0.134": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0.11\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.2.3\"},{\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2.0.2\"},{\"name\":\"itoa\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"name\":\"ryu\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"kind\":\"dev\",\"name\":\"serde_bytes\",\"req\":\"^0.11.10\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_stacker\",\"req\":\"^0.1.8\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{\"alloc\":[\"serde/alloc\"],\"arbitrary_precision\":[],\"default\":[\"std\"],\"float_roundtrip\":[],\"preserve_order\":[\"indexmap\",\"std\"],\"raw_value\":[],\"std\":[\"memchr/std\",\"serde/std\"],\"unbounded_depth\":[]}}", - "serde_json_1.0.149": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0.11\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.2.3\"},{\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2.0.2\"},{\"name\":\"itoa\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"kind\":\"dev\",\"name\":\"serde_bytes\",\"req\":\"^0.11.10\"},{\"default_features\":false,\"name\":\"serde_core\",\"req\":\"^1.0.220\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_stacker\",\"req\":\"^0.1.8\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"},{\"name\":\"zmij\",\"req\":\"^1.0\"}],\"features\":{\"alloc\":[\"serde_core/alloc\"],\"arbitrary_precision\":[],\"default\":[\"std\"],\"float_roundtrip\":[],\"preserve_order\":[\"indexmap\",\"std\"],\"raw_value\":[],\"std\":[\"memchr/std\",\"serde_core/std\"],\"unbounded_depth\":[]}}", - "shlex_1.3.0": "{\"dependencies\":[],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "signal-hook-registry_1.4.8": "{\"dependencies\":[{\"name\":\"errno\",\"req\":\">=0.2, <0.4\"},{\"name\":\"libc\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"signal-hook\",\"req\":\"~0.3\"}],\"features\":{}}", - "slab_0.4.12": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.95\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "smallvec_1.13.2": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.0.1\"},{\"kind\":\"dev\",\"name\":\"debugger_test\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"debugger_test_parser\",\"req\":\"^0.1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"const_generics\":[],\"const_new\":[\"const_generics\"],\"debugger_visualizer\":[],\"drain_filter\":[],\"drain_keep_rest\":[\"drain_filter\"],\"may_dangle\":[],\"specialization\":[],\"union\":[],\"write\":[]}}", - "smallvec_1.15.1": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"name\":\"bincode\",\"optional\":true,\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"bincode1\",\"package\":\"bincode\",\"req\":\"^1.0.1\"},{\"kind\":\"dev\",\"name\":\"debugger_test\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"debugger_test_parser\",\"req\":\"^0.1.0\"},{\"default_features\":false,\"name\":\"malloc_size_of\",\"optional\":true,\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"name\":\"unty\",\"optional\":true,\"req\":\"^0.0.4\"}],\"features\":{\"const_generics\":[],\"const_new\":[\"const_generics\"],\"debugger_visualizer\":[],\"drain_filter\":[],\"drain_keep_rest\":[\"drain_filter\"],\"impl_bincode\":[\"bincode\",\"unty\"],\"may_dangle\":[],\"specialization\":[],\"union\":[],\"write\":[]}}", - "socket2_0.5.10": "{\"dependencies\":[{\"name\":\"libc\",\"req\":\"^0.2.171\",\"target\":\"cfg(unix)\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\",\"Win32_System_IO\",\"Win32_System_Threading\",\"Win32_System_WindowsProgramming\"],\"name\":\"windows-sys\",\"req\":\"^0.52\",\"target\":\"cfg(windows)\"}],\"features\":{\"all\":[]}}", - "socket2_0.6.3": "{\"dependencies\":[{\"name\":\"libc\",\"req\":\"^0.2.172\",\"target\":\"cfg(any(unix, target_os = \\\"wasi\\\"))\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\",\"Win32_System_IO\",\"Win32_System_Threading\",\"Win32_System_WindowsProgramming\"],\"name\":\"windows-sys\",\"req\":\">=0.60, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"all\":[]}}", - "spm_precompiled_0.1.4": "{\"dependencies\":[{\"name\":\"base64\",\"req\":\"^0.13\"},{\"name\":\"nom\",\"req\":\"^7.1.1\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"unicode-segmentation\",\"req\":\"^1.9\"}],\"features\":{}}", - "static_assertions_1.1.0": "{\"dependencies\":[],\"features\":{\"nightly\":[]}}", - "strsim_0.11.1": "{\"dependencies\":[],\"features\":{}}", - "syn_2.0.117": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"insta\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.91\"},{\"default_features\":false,\"name\":\"quote\",\"optional\":true,\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1\"},{\"features\":[\"blocking\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.13\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"syn-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4.16\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"termcolor\",\"req\":\"^1\"},{\"name\":\"unicode-ident\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\",\"target\":\"cfg(not(miri))\"}],\"features\":{\"clone-impls\":[],\"default\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\"],\"derive\":[],\"extra-traits\":[],\"fold\":[],\"full\":[],\"parsing\":[],\"printing\":[\"dep:quote\"],\"proc-macro\":[\"proc-macro2/proc-macro\",\"quote?/proc-macro\"],\"test\":[\"syn-test-suite/all-features\"],\"visit\":[],\"visit-mut\":[]}}", - "syn_2.0.91": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"insta\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.91\"},{\"default_features\":false,\"name\":\"quote\",\"optional\":true,\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1\"},{\"features\":[\"blocking\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.12\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"syn-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4.16\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"termcolor\",\"req\":\"^1\"},{\"name\":\"unicode-ident\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\",\"target\":\"cfg(not(miri))\"}],\"features\":{\"clone-impls\":[],\"default\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\"],\"derive\":[],\"extra-traits\":[],\"fold\":[],\"full\":[],\"parsing\":[],\"printing\":[\"dep:quote\"],\"proc-macro\":[\"proc-macro2/proc-macro\",\"quote?/proc-macro\"],\"test\":[\"syn-test-suite/all-features\"],\"visit\":[],\"visit-mut\":[]}}", - "sync_wrapper_1.0.2": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"}],\"features\":{\"futures\":[\"futures-core\"]}}", - "tempfile_3.27.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"name\":\"fastrand\",\"req\":\"^2.1.1\"},{\"default_features\":false,\"name\":\"getrandom\",\"optional\":true,\"req\":\">=0.3.0, <0.5\",\"target\":\"cfg(any(unix, windows, target_os = \\\"wasi\\\"))\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"once_cell\",\"req\":\"^1.19.0\"},{\"features\":[\"fs\"],\"name\":\"rustix\",\"req\":\"^1.1.4\",\"target\":\"cfg(any(unix, target_os = \\\"wasi\\\"))\"},{\"features\":[\"Win32_Storage_FileSystem\",\"Win32_Foundation\"],\"name\":\"windows-sys\",\"req\":\">=0.52, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[\"getrandom\"],\"nightly\":[]}}", - "thiserror-impl_2.0.12": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"name\":\"syn\",\"req\":\"^2.0.87\"}],\"features\":{}}", - "thiserror_2.0.12": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.73\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"name\":\"thiserror-impl\",\"req\":\"=2.0.12\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "tokenizers_0.22.2": "{\"dependencies\":[{\"features\":[\"serde\"],\"name\":\"ahash\",\"req\":\"^0.8.11\"},{\"name\":\"aho-corasick\",\"req\":\"^1.1\"},{\"kind\":\"dev\",\"name\":\"assert_approx_eq\",\"req\":\"^1.1\"},{\"features\":[\"serde\"],\"name\":\"compact_str\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.6\"},{\"features\":[\"serde\"],\"name\":\"dary_heap\",\"req\":\"^0.3.6\"},{\"name\":\"derive_builder\",\"req\":\"^0.20\"},{\"default_features\":false,\"name\":\"esaxx-rs\",\"req\":\"^0.1.10\"},{\"name\":\"fancy-regex\",\"optional\":true,\"req\":\"^0.14\"},{\"name\":\"getrandom\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"ureq\"],\"name\":\"hf-hub\",\"optional\":true,\"req\":\"^0.4.1\"},{\"name\":\"indicatif\",\"optional\":true,\"req\":\"^0.18\"},{\"name\":\"itertools\",\"req\":\"^0.14\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"macro_rules_attribute\",\"req\":\"^0.2.0\"},{\"name\":\"monostate\",\"req\":\"^0.1.12\"},{\"default_features\":false,\"name\":\"onig\",\"optional\":true,\"req\":\"^6.5.1\"},{\"name\":\"paste\",\"req\":\"^1.0.14\"},{\"name\":\"rand\",\"req\":\"^0.9\"},{\"name\":\"rayon\",\"req\":\"^1.10\"},{\"name\":\"rayon-cond\",\"req\":\"^0.4\"},{\"name\":\"regex\",\"req\":\"^1.10\"},{\"name\":\"regex-syntax\",\"req\":\"^0.8\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"spm_precompiled\",\"req\":\"^0.1.3\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.10\"},{\"name\":\"thiserror\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3.18\"},{\"name\":\"unicode-normalization-alignments\",\"req\":\"^0.1\"},{\"name\":\"unicode-segmentation\",\"req\":\"^1.11\"},{\"name\":\"unicode_categories\",\"req\":\"^0.1\"}],\"features\":{\"default\":[\"progressbar\",\"onig\",\"esaxx_fast\"],\"esaxx_fast\":[\"esaxx-rs/cpp\"],\"http\":[\"hf-hub\"],\"progressbar\":[\"indicatif\"],\"rustls-tls\":[\"hf-hub?/rustls-tls\"],\"unstable_wasm\":[\"fancy-regex\",\"getrandom/wasm_js\"]}}", - "tokio-macros_2.4.0": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0.0\"}],\"features\":{}}", - "tokio-stream_0.1.15": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-stream\",\"req\":\"^0.3\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"name\":\"futures-core\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"parking_lot\",\"req\":\"^0.12.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.11\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"req\":\"^1.15.0\"},{\"features\":[\"full\",\"test-util\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.2.0\"},{\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7.0\"}],\"features\":{\"default\":[\"time\"],\"fs\":[\"tokio/fs\"],\"full\":[\"time\",\"net\",\"io-util\",\"fs\",\"sync\",\"signal\"],\"io-util\":[\"tokio/io-util\"],\"net\":[\"tokio/net\"],\"signal\":[\"tokio/signal\"],\"sync\":[\"tokio/sync\",\"tokio-util\"],\"time\":[\"tokio/time\"]}}", - "tokio-util_0.7.17": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-stream\",\"req\":\"^0.3.0\"},{\"name\":\"bytes\",\"req\":\"^1.5.0\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.0\"},{\"name\":\"futures-core\",\"req\":\"^0.3.0\"},{\"name\":\"futures-io\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"futures-sink\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"futures-test\",\"req\":\"^0.3.5\"},{\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.0\"},{\"default_features\":false,\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.15.0\"},{\"kind\":\"dev\",\"name\":\"parking_lot\",\"req\":\"^0.12.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.11\"},{\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4.4\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.1.0\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"req\":\"^1.28.0\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.29\"}],\"features\":{\"__docs_rs\":[\"futures-util\"],\"codec\":[],\"compat\":[\"futures-io\"],\"default\":[],\"full\":[\"codec\",\"compat\",\"io-util\",\"time\",\"net\",\"rt\",\"join-map\"],\"io\":[],\"io-util\":[\"io\",\"tokio/rt\",\"tokio/io-util\"],\"join-map\":[\"rt\",\"hashbrown\"],\"net\":[\"tokio/net\"],\"rt\":[\"tokio/rt\",\"tokio/sync\",\"futures-util\"],\"time\":[\"tokio/time\",\"slab\"]}}", - "tokio_1.39.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-stream\",\"req\":\"^0.3\"},{\"name\":\"backtrace\",\"req\":\"^0.3.58\",\"target\":\"cfg(tokio_taskdump)\"},{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1.0.0\"},{\"features\":[\"async-await\"],\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.0\"},{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.149\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.149\",\"target\":\"cfg(unix)\"},{\"features\":[\"futures\",\"checkpoint\"],\"kind\":\"dev\",\"name\":\"loom\",\"req\":\"^0.7\",\"target\":\"cfg(loom)\"},{\"default_features\":false,\"name\":\"mio\",\"optional\":true,\"req\":\"^1.0.1\"},{\"features\":[\"tokio\"],\"kind\":\"dev\",\"name\":\"mio-aio\",\"req\":\"^0.9.0\",\"target\":\"cfg(target_os = \\\"freebsd\\\")\"},{\"kind\":\"dev\",\"name\":\"mockall\",\"req\":\"^0.11.1\"},{\"default_features\":false,\"features\":[\"aio\",\"fs\",\"socket\"],\"kind\":\"dev\",\"name\":\"nix\",\"req\":\"^0.29.0\",\"target\":\"cfg(unix)\"},{\"name\":\"parking_lot\",\"optional\":true,\"req\":\"^0.12.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.11\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.0\",\"target\":\"cfg(not(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\")))\"},{\"name\":\"signal-hook-registry\",\"optional\":true,\"req\":\"^1.1.1\",\"target\":\"cfg(unix)\"},{\"features\":[\"all\"],\"name\":\"socket2\",\"optional\":true,\"req\":\"^0.5.5\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"kind\":\"dev\",\"name\":\"socket2\",\"req\":\"^0.5.5\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.1.0\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"name\":\"tokio-macros\",\"optional\":true,\"req\":\"~2.4.0\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.25\",\"target\":\"cfg(tokio_unstable)\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.0\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", not(target_os = \\\"wasi\\\")))\"},{\"name\":\"windows-sys\",\"optional\":true,\"req\":\"^0.52\",\"target\":\"cfg(windows)\"},{\"features\":[\"Win32_Foundation\",\"Win32_Security_Authorization\"],\"kind\":\"dev\",\"name\":\"windows-sys\",\"req\":\"^0.52\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[],\"fs\":[],\"full\":[\"fs\",\"io-util\",\"io-std\",\"macros\",\"net\",\"parking_lot\",\"process\",\"rt\",\"rt-multi-thread\",\"signal\",\"sync\",\"time\"],\"io-std\":[],\"io-util\":[\"bytes\"],\"macros\":[\"tokio-macros\"],\"net\":[\"libc\",\"mio/os-poll\",\"mio/os-ext\",\"mio/net\",\"socket2\",\"windows-sys/Win32_Foundation\",\"windows-sys/Win32_Security\",\"windows-sys/Win32_Storage_FileSystem\",\"windows-sys/Win32_System_Pipes\",\"windows-sys/Win32_System_SystemServices\"],\"process\":[\"bytes\",\"libc\",\"mio/os-poll\",\"mio/os-ext\",\"mio/net\",\"signal-hook-registry\",\"windows-sys/Win32_Foundation\",\"windows-sys/Win32_System_Threading\",\"windows-sys/Win32_System_WindowsProgramming\"],\"rt\":[],\"rt-multi-thread\":[\"rt\"],\"signal\":[\"libc\",\"mio/os-poll\",\"mio/net\",\"mio/os-ext\",\"signal-hook-registry\",\"windows-sys/Win32_Foundation\",\"windows-sys/Win32_System_Console\"],\"sync\":[],\"test-util\":[\"rt\",\"sync\",\"time\"],\"time\":[]}}", - "tonic-build_0.12.3": "{\"dependencies\":[{\"name\":\"prettyplease\",\"req\":\"^0.2\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"name\":\"prost-build\",\"optional\":true,\"req\":\"^0.13\"},{\"name\":\"prost-types\",\"optional\":true,\"req\":\"^0.13\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"name\":\"syn\",\"req\":\"^2.0\"}],\"features\":{\"cleanup-markdown\":[\"prost\",\"prost-build/cleanup-markdown\"],\"default\":[\"transport\",\"prost\"],\"prost\":[\"prost-build\",\"dep:prost-types\"],\"transport\":[]}}", - "tonic_0.12.1": "{\"dependencies\":[{\"name\":\"async-stream\",\"optional\":true,\"req\":\"^0.3\"},{\"name\":\"async-trait\",\"optional\":true,\"req\":\"^0.1.13\"},{\"default_features\":false,\"name\":\"axum\",\"optional\":true,\"req\":\"^0.7\"},{\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.5\"},{\"name\":\"bytes\",\"req\":\"^1.0\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"h2\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"http-body\",\"req\":\"^1\"},{\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"features\":[\"http1\",\"http2\"],\"name\":\"hyper\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"hyper-timeout\",\"optional\":true,\"req\":\"^0.5\"},{\"features\":[\"tokio\"],\"name\":\"hyper-util\",\"optional\":true,\"req\":\"^0.1.4\"},{\"name\":\"percent-encoding\",\"req\":\"^2.1\"},{\"name\":\"pin-project\",\"req\":\"^1.0.11\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"prost\",\"optional\":true,\"req\":\"^0.13\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"name\":\"rustls-native-certs\",\"optional\":true,\"req\":\"^0.7\"},{\"name\":\"rustls-pemfile\",\"optional\":true,\"req\":\"^2.0\"},{\"features\":[\"all\"],\"name\":\"socket2\",\"optional\":true,\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"rt\",\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"logging\",\"tls12\",\"ring\"],\"name\":\"tokio-rustls\",\"optional\":true,\"req\":\"^0.26\"},{\"default_features\":false,\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"tower\",\"optional\":true,\"req\":\"^0.4.7\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tower\",\"req\":\"^0.4.7\"},{\"name\":\"tower-layer\",\"req\":\"^0.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"name\":\"tracing\",\"req\":\"^0.1\"},{\"name\":\"webpki-roots\",\"optional\":true,\"req\":\"^0.26\"},{\"name\":\"zstd\",\"optional\":true,\"req\":\"^0.13.0\"}],\"features\":{\"channel\":[\"dep:hyper\",\"hyper?/client\",\"dep:hyper-util\",\"hyper-util?/client-legacy\",\"dep:tower\",\"tower?/balance\",\"tower?/buffer\",\"tower?/discover\",\"tower?/limit\",\"dep:tokio\",\"tokio?/time\",\"dep:hyper-timeout\"],\"codegen\":[\"dep:async-trait\"],\"default\":[\"transport\",\"codegen\",\"prost\"],\"gzip\":[\"dep:flate2\"],\"prost\":[\"dep:prost\"],\"router\":[\"dep:axum\",\"dep:tower\",\"tower?/util\"],\"server\":[\"router\",\"dep:async-stream\",\"dep:h2\",\"dep:hyper\",\"hyper?/server\",\"dep:hyper-util\",\"hyper-util?/service\",\"hyper-util?/server-auto\",\"dep:socket2\",\"dep:tokio\",\"tokio?/macros\",\"tokio?/net\",\"tokio?/time\",\"tokio-stream/net\",\"dep:tower\",\"tower?/util\",\"tower?/limit\"],\"tls\":[\"dep:rustls-pemfile\",\"dep:tokio-rustls\",\"dep:tokio\",\"tokio?/rt\",\"tokio?/macros\"],\"tls-roots\":[\"tls\",\"channel\",\"dep:rustls-native-certs\"],\"tls-webpki-roots\":[\"tls\",\"channel\",\"dep:webpki-roots\"],\"transport\":[\"server\",\"channel\"],\"zstd\":[\"dep:zstd\"]}}", - "tower-layer_0.3.3": "{\"dependencies\":[],\"features\":{}}", - "tower-service_0.3.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.22\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^0.2\"},{\"features\":[\"macros\",\"time\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.6.2\"},{\"kind\":\"dev\",\"name\":\"tower-layer\",\"req\":\"^0.3\"}],\"features\":{}}", - "tower_0.4.13": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"hdrhistogram\",\"optional\":true,\"req\":\"^7.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"hdrhistogram\",\"req\":\"^7.0\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^0.2\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^1.0.2\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4.0\"},{\"name\":\"pin-project\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"pin-project-lite\",\"optional\":true,\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"},{\"features\":[\"small_rng\"],\"name\":\"rand\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.6\"},{\"features\":[\"macros\",\"sync\",\"test-util\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.6.2\"},{\"name\":\"tokio-stream\",\"optional\":true,\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7.0\"},{\"name\":\"tower-layer\",\"req\":\"^0.3.1\"},{\"name\":\"tower-service\",\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"tower-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.2\"},{\"default_features\":false,\"features\":[\"fmt\",\"ansi\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"}],\"features\":{\"__common\":[\"futures-core\",\"pin-project-lite\"],\"balance\":[\"discover\",\"load\",\"ready-cache\",\"make\",\"rand\",\"slab\"],\"buffer\":[\"__common\",\"tokio/sync\",\"tokio/rt\",\"tokio-util\",\"tracing\"],\"default\":[\"log\"],\"discover\":[\"__common\"],\"filter\":[\"__common\",\"futures-util\"],\"full\":[\"balance\",\"buffer\",\"discover\",\"filter\",\"hedge\",\"limit\",\"load\",\"load-shed\",\"make\",\"ready-cache\",\"reconnect\",\"retry\",\"spawn-ready\",\"steer\",\"timeout\",\"util\"],\"hedge\":[\"util\",\"filter\",\"futures-util\",\"hdrhistogram\",\"tokio/time\",\"tracing\"],\"limit\":[\"__common\",\"tokio/time\",\"tokio/sync\",\"tokio-util\",\"tracing\"],\"load\":[\"__common\",\"tokio/time\",\"tracing\"],\"load-shed\":[\"__common\"],\"log\":[\"tracing/log\"],\"make\":[\"futures-util\",\"pin-project-lite\",\"tokio/io-std\"],\"ready-cache\":[\"futures-core\",\"futures-util\",\"indexmap\",\"tokio/sync\",\"tracing\",\"pin-project-lite\"],\"reconnect\":[\"make\",\"tokio/io-std\",\"tracing\"],\"retry\":[\"__common\",\"tokio/time\"],\"spawn-ready\":[\"__common\",\"futures-util\",\"tokio/sync\",\"tokio/rt\",\"util\",\"tracing\"],\"steer\":[],\"timeout\":[\"pin-project-lite\",\"tokio/time\"],\"util\":[\"__common\",\"futures-util\",\"pin-project\"]}}", - "tower_0.5.3": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.22\"},{\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3.22\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.22\"},{\"default_features\":false,\"features\":[\"async-await-macro\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3.22\"},{\"default_features\":false,\"name\":\"hdrhistogram\",\"optional\":true,\"req\":\"^7.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"hdrhistogram\",\"req\":\"^7.0\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.0.2\"},{\"name\":\"pin-project-lite\",\"optional\":true,\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9\"},{\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4.9\"},{\"name\":\"sync_wrapper\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.6.2\"},{\"features\":[\"macros\",\"sync\",\"test-util\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.6.2\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1.1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7.0\"},{\"name\":\"tower-layer\",\"req\":\"^0.3.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3.3\"},{\"kind\":\"dev\",\"name\":\"tower-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.2\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1.2\"},{\"default_features\":false,\"features\":[\"fmt\",\"ansi\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"}],\"features\":{\"balance\":[\"discover\",\"load\",\"ready-cache\",\"make\",\"slab\",\"util\"],\"buffer\":[\"tokio/sync\",\"tokio/rt\",\"tokio-util\",\"tracing\",\"pin-project-lite\"],\"discover\":[\"futures-core\",\"pin-project-lite\"],\"filter\":[\"futures-util\",\"pin-project-lite\"],\"full\":[\"balance\",\"buffer\",\"discover\",\"filter\",\"hedge\",\"limit\",\"load\",\"load-shed\",\"make\",\"ready-cache\",\"reconnect\",\"retry\",\"spawn-ready\",\"steer\",\"timeout\",\"util\"],\"hedge\":[\"util\",\"filter\",\"futures-util\",\"hdrhistogram\",\"tokio/time\",\"tracing\"],\"limit\":[\"tokio/time\",\"tokio/sync\",\"tokio-util\",\"tracing\",\"pin-project-lite\"],\"load\":[\"tokio/time\",\"tracing\",\"pin-project-lite\"],\"load-shed\":[\"pin-project-lite\"],\"log\":[\"tracing/log\"],\"make\":[\"pin-project-lite\",\"tokio\"],\"ready-cache\":[\"futures-core\",\"futures-util\",\"indexmap\",\"tokio/sync\",\"tracing\",\"pin-project-lite\"],\"reconnect\":[\"make\",\"tracing\"],\"retry\":[\"tokio/time\",\"util\"],\"spawn-ready\":[\"futures-util\",\"tokio/sync\",\"tokio/rt\",\"util\",\"tracing\"],\"steer\":[],\"timeout\":[\"pin-project-lite\",\"tokio/time\"],\"tokio-stream\":[],\"util\":[\"futures-core\",\"futures-util\",\"pin-project-lite\",\"sync_wrapper\"]}}", - "tracing-attributes_0.1.31": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-trait\",\"req\":\"^0.1.67\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1.0.20\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.9\"},{\"default_features\":false,\"features\":[\"full\",\"parsing\",\"printing\",\"visit-mut\",\"clone-impls\",\"extra-traits\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4.2\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1.35\"},{\"features\":[\"env-filter\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.64\"}],\"features\":{\"async-await\":[]}}", - "tracing-core_0.1.36": "{\"dependencies\":[{\"name\":\"once_cell\",\"optional\":true,\"req\":\"^1.13.0\"},{\"default_features\":false,\"name\":\"valuable\",\"optional\":true,\"req\":\"^0.1.0\",\"target\":\"cfg(tracing_unstable)\"}],\"features\":{\"default\":[\"std\",\"valuable?/std\"],\"std\":[\"once_cell\"]}}", - "tracing_0.1.44": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.6\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.21\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4.17\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.9\"},{\"name\":\"tracing-attributes\",\"optional\":true,\"req\":\"^0.1.31\"},{\"default_features\":false,\"name\":\"tracing-core\",\"req\":\"^0.1.36\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.38\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"}],\"features\":{\"async-await\":[],\"attributes\":[\"tracing-attributes\"],\"default\":[\"std\",\"attributes\"],\"log-always\":[\"log\"],\"max_level_debug\":[],\"max_level_error\":[],\"max_level_info\":[],\"max_level_off\":[],\"max_level_trace\":[],\"max_level_warn\":[],\"release_max_level_debug\":[],\"release_max_level_error\":[],\"release_max_level_info\":[],\"release_max_level_off\":[],\"release_max_level_trace\":[],\"release_max_level_warn\":[],\"std\":[\"tracing-core/std\"],\"valuable\":[\"tracing-core/valuable\"]}}", - "try-lock_0.2.5": "{\"dependencies\":[],\"features\":{}}", - "unicode-ident_1.0.14": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"fst\",\"req\":\"^0.4\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"roaring\",\"req\":\"^0.10\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"ucd-trie\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"unicode-xid\",\"req\":\"^0.2.6\"}],\"features\":{}}", - "unicode-ident_1.0.24": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"fst\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"roaring\",\"req\":\"^0.11\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"ucd-trie\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"unicode-xid\",\"req\":\"^0.2.6\"}],\"features\":{}}", - "unicode-normalization-alignments_0.1.12": "{\"dependencies\":[{\"name\":\"smallvec\",\"req\":\"^1.0\"}],\"features\":{}}", - "unicode-segmentation_1.12.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.7\"}],\"features\":{\"no_std\":[]}}", - "unicode-xid_0.2.6": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3\"}],\"features\":{\"bench\":[],\"default\":[],\"no_std\":[]}}", - "unicode_categories_0.1.1": "{\"dependencies\":[],\"features\":{}}", - "version_check_0.9.5": "{\"dependencies\":[],\"features\":{}}", - "want_0.3.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"tokio-executor\",\"req\":\"^0.2.0-alpha.2\"},{\"kind\":\"dev\",\"name\":\"tokio-sync\",\"req\":\"^0.2.0-alpha.2\"},{\"name\":\"try-lock\",\"req\":\"^0.2.4\"}],\"features\":{}}", - "wasi_0.11.1+wasi-snapshot-preview1": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"rustc-dep-of-std\":[\"core\",\"rustc-std-workspace-alloc\"],\"std\":[]}}", - "wasi_0.14.2+wasi-0.2.4": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"bitflags\"],\"name\":\"wit-bindgen-rt\",\"req\":\"^0.39.0\"}],\"features\":{\"default\":[\"std\"],\"rustc-dep-of-std\":[\"compiler_builtins\",\"core\",\"rustc-std-workspace-alloc\"],\"std\":[]}}", - "wasip2_1.0.2+wasi-0.2.9": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"wit-bindgen\",\"req\":\"^0.51.0\"}],\"features\":{\"bitflags\":[\"wit-bindgen/bitflags\"],\"default\":[\"std\",\"bitflags\"],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"wit-bindgen/rustc-dep-of-std\"],\"std\":[]}}", - "wasip3_0.4.0+wasi-0.3.0-rc-2026-01-06": "{\"dependencies\":[{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1.10.1\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.31\"},{\"name\":\"http\",\"optional\":true,\"req\":\"^1.3.1\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^1.3.1\"},{\"name\":\"http-body\",\"optional\":true,\"req\":\"^1.0.1\"},{\"name\":\"thiserror\",\"optional\":true,\"req\":\"^2.0.17\"},{\"default_features\":false,\"features\":[\"async\"],\"name\":\"wit-bindgen\",\"req\":\"^0.51.0\"},{\"default_features\":false,\"features\":[\"async-spawn\"],\"kind\":\"dev\",\"name\":\"wit-bindgen\",\"req\":\"^0.51.0\"}],\"features\":{\"http-compat\":[\"dep:bytes\",\"dep:http-body\",\"dep:http\",\"dep:thiserror\",\"wit-bindgen/async-spawn\"]}}", - "wasm-encoder_0.244.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"default_features\":false,\"name\":\"leb128fmt\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.2.0\"},{\"default_features\":false,\"features\":[\"simd\",\"simd\"],\"name\":\"wasmparser\",\"optional\":true,\"req\":\"^0.244.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"wasmprinter\",\"req\":\"^0.244.0\"}],\"features\":{\"component-model\":[\"wasmparser?/component-model\"],\"default\":[\"std\",\"component-model\"],\"std\":[\"wasmparser?/std\"]}}", - "wasm-metadata_0.244.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"name\":\"auditable-serde\",\"optional\":true,\"req\":\"^0.8.0\"},{\"features\":[\"derive\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4.0.0\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1.1.0\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"indexmap\",\"req\":\"^2.7.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"spdx\",\"optional\":true,\"req\":\"^0.10.1\"},{\"name\":\"url\",\"optional\":true,\"req\":\"^2.0.0\"},{\"default_features\":false,\"features\":[\"std\",\"component-model\"],\"name\":\"wasm-encoder\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"simd\",\"std\",\"component-model\",\"hash-collections\"],\"name\":\"wasmparser\",\"req\":\"^0.244.0\"}],\"features\":{\"default\":[\"oci\",\"serde\"],\"oci\":[\"dep:auditable-serde\",\"dep:flate2\",\"dep:url\",\"dep:spdx\",\"dep:serde_json\",\"serde\"],\"serde\":[\"dep:serde_derive\",\"dep:serde\"]}}", - "wasmparser_0.244.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"name\":\"bitflags\",\"req\":\"^2.4.1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5.1\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"default-hasher\"],\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.15.2\"},{\"default_features\":false,\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.7.0\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.13.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.3\"},{\"default_features\":false,\"name\":\"semver\",\"optional\":true,\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.166\"}],\"features\":{\"component-model\":[\"dep:semver\"],\"default\":[\"std\",\"validate\",\"serde\",\"features\",\"component-model\",\"hash-collections\",\"simd\"],\"features\":[],\"hash-collections\":[\"dep:hashbrown\",\"dep:indexmap\"],\"prefer-btree-collections\":[],\"serde\":[\"dep:serde\",\"indexmap?/serde\",\"hashbrown?/serde\"],\"simd\":[],\"std\":[\"indexmap?/std\"],\"validate\":[]}}", - "windows-link_0.2.1": "{\"dependencies\":[],\"features\":{}}", - "windows-sys_0.52.0": "{\"dependencies\":[{\"name\":\"windows-targets\",\"req\":\"^0.52.0\"}],\"features\":{\"Wdk\":[],\"Wdk_Foundation\":[\"Wdk\"],\"Wdk_Graphics\":[\"Wdk\"],\"Wdk_Graphics_Direct3D\":[\"Wdk_Graphics\"],\"Wdk_Storage\":[\"Wdk\"],\"Wdk_Storage_FileSystem\":[\"Wdk_Storage\"],\"Wdk_Storage_FileSystem_Minifilters\":[\"Wdk_Storage_FileSystem\"],\"Wdk_System\":[\"Wdk\"],\"Wdk_System_IO\":[\"Wdk_System\"],\"Wdk_System_OfflineRegistry\":[\"Wdk_System\"],\"Wdk_System_Registry\":[\"Wdk_System\"],\"Wdk_System_SystemInformation\":[\"Wdk_System\"],\"Wdk_System_SystemServices\":[\"Wdk_System\"],\"Wdk_System_Threading\":[\"Wdk_System\"],\"Win32\":[],\"Win32_Data\":[\"Win32\"],\"Win32_Data_HtmlHelp\":[\"Win32_Data\"],\"Win32_Data_RightsManagement\":[\"Win32_Data\"],\"Win32_Devices\":[\"Win32\"],\"Win32_Devices_AllJoyn\":[\"Win32_Devices\"],\"Win32_Devices_BiometricFramework\":[\"Win32_Devices\"],\"Win32_Devices_Bluetooth\":[\"Win32_Devices\"],\"Win32_Devices_Communication\":[\"Win32_Devices\"],\"Win32_Devices_DeviceAndDriverInstallation\":[\"Win32_Devices\"],\"Win32_Devices_DeviceQuery\":[\"Win32_Devices\"],\"Win32_Devices_Display\":[\"Win32_Devices\"],\"Win32_Devices_Enumeration\":[\"Win32_Devices\"],\"Win32_Devices_Enumeration_Pnp\":[\"Win32_Devices_Enumeration\"],\"Win32_Devices_Fax\":[\"Win32_Devices\"],\"Win32_Devices_HumanInterfaceDevice\":[\"Win32_Devices\"],\"Win32_Devices_PortableDevices\":[\"Win32_Devices\"],\"Win32_Devices_Properties\":[\"Win32_Devices\"],\"Win32_Devices_Pwm\":[\"Win32_Devices\"],\"Win32_Devices_Sensors\":[\"Win32_Devices\"],\"Win32_Devices_SerialCommunication\":[\"Win32_Devices\"],\"Win32_Devices_Tapi\":[\"Win32_Devices\"],\"Win32_Devices_Usb\":[\"Win32_Devices\"],\"Win32_Devices_WebServicesOnDevices\":[\"Win32_Devices\"],\"Win32_Foundation\":[\"Win32\"],\"Win32_Gaming\":[\"Win32\"],\"Win32_Globalization\":[\"Win32\"],\"Win32_Graphics\":[\"Win32\"],\"Win32_Graphics_Dwm\":[\"Win32_Graphics\"],\"Win32_Graphics_Gdi\":[\"Win32_Graphics\"],\"Win32_Graphics_GdiPlus\":[\"Win32_Graphics\"],\"Win32_Graphics_Hlsl\":[\"Win32_Graphics\"],\"Win32_Graphics_OpenGL\":[\"Win32_Graphics\"],\"Win32_Graphics_Printing\":[\"Win32_Graphics\"],\"Win32_Graphics_Printing_PrintTicket\":[\"Win32_Graphics_Printing\"],\"Win32_Management\":[\"Win32\"],\"Win32_Management_MobileDeviceManagementRegistration\":[\"Win32_Management\"],\"Win32_Media\":[\"Win32\"],\"Win32_Media_Audio\":[\"Win32_Media\"],\"Win32_Media_DxMediaObjects\":[\"Win32_Media\"],\"Win32_Media_KernelStreaming\":[\"Win32_Media\"],\"Win32_Media_Multimedia\":[\"Win32_Media\"],\"Win32_Media_Streaming\":[\"Win32_Media\"],\"Win32_Media_WindowsMediaFormat\":[\"Win32_Media\"],\"Win32_NetworkManagement\":[\"Win32\"],\"Win32_NetworkManagement_Dhcp\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Dns\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_InternetConnectionWizard\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_IpHelper\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Multicast\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Ndis\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetBios\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetManagement\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetShell\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetworkDiagnosticsFramework\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_P2P\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_QoS\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Rras\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Snmp\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WNet\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WebDav\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WiFi\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsConnectionManager\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsFilteringPlatform\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsFirewall\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsNetworkVirtualization\":[\"Win32_NetworkManagement\"],\"Win32_Networking\":[\"Win32\"],\"Win32_Networking_ActiveDirectory\":[\"Win32_Networking\"],\"Win32_Networking_Clustering\":[\"Win32_Networking\"],\"Win32_Networking_HttpServer\":[\"Win32_Networking\"],\"Win32_Networking_Ldap\":[\"Win32_Networking\"],\"Win32_Networking_WebSocket\":[\"Win32_Networking\"],\"Win32_Networking_WinHttp\":[\"Win32_Networking\"],\"Win32_Networking_WinInet\":[\"Win32_Networking\"],\"Win32_Networking_WinSock\":[\"Win32_Networking\"],\"Win32_Networking_WindowsWebServices\":[\"Win32_Networking\"],\"Win32_Security\":[\"Win32\"],\"Win32_Security_AppLocker\":[\"Win32_Security\"],\"Win32_Security_Authentication\":[\"Win32_Security\"],\"Win32_Security_Authentication_Identity\":[\"Win32_Security_Authentication\"],\"Win32_Security_Authorization\":[\"Win32_Security\"],\"Win32_Security_Credentials\":[\"Win32_Security\"],\"Win32_Security_Cryptography\":[\"Win32_Security\"],\"Win32_Security_Cryptography_Catalog\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_Certificates\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_Sip\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_UI\":[\"Win32_Security_Cryptography\"],\"Win32_Security_DiagnosticDataQuery\":[\"Win32_Security\"],\"Win32_Security_DirectoryServices\":[\"Win32_Security\"],\"Win32_Security_EnterpriseData\":[\"Win32_Security\"],\"Win32_Security_ExtensibleAuthenticationProtocol\":[\"Win32_Security\"],\"Win32_Security_Isolation\":[\"Win32_Security\"],\"Win32_Security_LicenseProtection\":[\"Win32_Security\"],\"Win32_Security_NetworkAccessProtection\":[\"Win32_Security\"],\"Win32_Security_WinTrust\":[\"Win32_Security\"],\"Win32_Security_WinWlx\":[\"Win32_Security\"],\"Win32_Storage\":[\"Win32\"],\"Win32_Storage_Cabinets\":[\"Win32_Storage\"],\"Win32_Storage_CloudFilters\":[\"Win32_Storage\"],\"Win32_Storage_Compression\":[\"Win32_Storage\"],\"Win32_Storage_DistributedFileSystem\":[\"Win32_Storage\"],\"Win32_Storage_FileHistory\":[\"Win32_Storage\"],\"Win32_Storage_FileSystem\":[\"Win32_Storage\"],\"Win32_Storage_Imapi\":[\"Win32_Storage\"],\"Win32_Storage_IndexServer\":[\"Win32_Storage\"],\"Win32_Storage_InstallableFileSystems\":[\"Win32_Storage\"],\"Win32_Storage_IscsiDisc\":[\"Win32_Storage\"],\"Win32_Storage_Jet\":[\"Win32_Storage\"],\"Win32_Storage_Nvme\":[\"Win32_Storage\"],\"Win32_Storage_OfflineFiles\":[\"Win32_Storage\"],\"Win32_Storage_OperationRecorder\":[\"Win32_Storage\"],\"Win32_Storage_Packaging\":[\"Win32_Storage\"],\"Win32_Storage_Packaging_Appx\":[\"Win32_Storage_Packaging\"],\"Win32_Storage_ProjectedFileSystem\":[\"Win32_Storage\"],\"Win32_Storage_StructuredStorage\":[\"Win32_Storage\"],\"Win32_Storage_Vhd\":[\"Win32_Storage\"],\"Win32_Storage_Xps\":[\"Win32_Storage\"],\"Win32_System\":[\"Win32\"],\"Win32_System_AddressBook\":[\"Win32_System\"],\"Win32_System_Antimalware\":[\"Win32_System\"],\"Win32_System_ApplicationInstallationAndServicing\":[\"Win32_System\"],\"Win32_System_ApplicationVerifier\":[\"Win32_System\"],\"Win32_System_ClrHosting\":[\"Win32_System\"],\"Win32_System_Com\":[\"Win32_System\"],\"Win32_System_Com_Marshal\":[\"Win32_System_Com\"],\"Win32_System_Com_StructuredStorage\":[\"Win32_System_Com\"],\"Win32_System_Com_Urlmon\":[\"Win32_System_Com\"],\"Win32_System_ComponentServices\":[\"Win32_System\"],\"Win32_System_Console\":[\"Win32_System\"],\"Win32_System_CorrelationVector\":[\"Win32_System\"],\"Win32_System_DataExchange\":[\"Win32_System\"],\"Win32_System_DeploymentServices\":[\"Win32_System\"],\"Win32_System_DeveloperLicensing\":[\"Win32_System\"],\"Win32_System_Diagnostics\":[\"Win32_System\"],\"Win32_System_Diagnostics_Ceip\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_Debug\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_Debug_Extensions\":[\"Win32_System_Diagnostics_Debug\"],\"Win32_System_Diagnostics_Etw\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_ProcessSnapshotting\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_ToolHelp\":[\"Win32_System_Diagnostics\"],\"Win32_System_DistributedTransactionCoordinator\":[\"Win32_System\"],\"Win32_System_Environment\":[\"Win32_System\"],\"Win32_System_ErrorReporting\":[\"Win32_System\"],\"Win32_System_EventCollector\":[\"Win32_System\"],\"Win32_System_EventLog\":[\"Win32_System\"],\"Win32_System_EventNotificationService\":[\"Win32_System\"],\"Win32_System_GroupPolicy\":[\"Win32_System\"],\"Win32_System_HostCompute\":[\"Win32_System\"],\"Win32_System_HostComputeNetwork\":[\"Win32_System\"],\"Win32_System_HostComputeSystem\":[\"Win32_System\"],\"Win32_System_Hypervisor\":[\"Win32_System\"],\"Win32_System_IO\":[\"Win32_System\"],\"Win32_System_Iis\":[\"Win32_System\"],\"Win32_System_Ioctl\":[\"Win32_System\"],\"Win32_System_JobObjects\":[\"Win32_System\"],\"Win32_System_Js\":[\"Win32_System\"],\"Win32_System_Kernel\":[\"Win32_System\"],\"Win32_System_LibraryLoader\":[\"Win32_System\"],\"Win32_System_Mailslots\":[\"Win32_System\"],\"Win32_System_Mapi\":[\"Win32_System\"],\"Win32_System_Memory\":[\"Win32_System\"],\"Win32_System_Memory_NonVolatile\":[\"Win32_System_Memory\"],\"Win32_System_MessageQueuing\":[\"Win32_System\"],\"Win32_System_MixedReality\":[\"Win32_System\"],\"Win32_System_Ole\":[\"Win32_System\"],\"Win32_System_PasswordManagement\":[\"Win32_System\"],\"Win32_System_Performance\":[\"Win32_System\"],\"Win32_System_Performance_HardwareCounterProfiling\":[\"Win32_System_Performance\"],\"Win32_System_Pipes\":[\"Win32_System\"],\"Win32_System_Power\":[\"Win32_System\"],\"Win32_System_ProcessStatus\":[\"Win32_System\"],\"Win32_System_Recovery\":[\"Win32_System\"],\"Win32_System_Registry\":[\"Win32_System\"],\"Win32_System_RemoteDesktop\":[\"Win32_System\"],\"Win32_System_RemoteManagement\":[\"Win32_System\"],\"Win32_System_RestartManager\":[\"Win32_System\"],\"Win32_System_Restore\":[\"Win32_System\"],\"Win32_System_Rpc\":[\"Win32_System\"],\"Win32_System_Search\":[\"Win32_System\"],\"Win32_System_Search_Common\":[\"Win32_System_Search\"],\"Win32_System_SecurityCenter\":[\"Win32_System\"],\"Win32_System_Services\":[\"Win32_System\"],\"Win32_System_SetupAndMigration\":[\"Win32_System\"],\"Win32_System_Shutdown\":[\"Win32_System\"],\"Win32_System_StationsAndDesktops\":[\"Win32_System\"],\"Win32_System_SubsystemForLinux\":[\"Win32_System\"],\"Win32_System_SystemInformation\":[\"Win32_System\"],\"Win32_System_SystemServices\":[\"Win32_System\"],\"Win32_System_Threading\":[\"Win32_System\"],\"Win32_System_Time\":[\"Win32_System\"],\"Win32_System_TpmBaseServices\":[\"Win32_System\"],\"Win32_System_UserAccessLogging\":[\"Win32_System\"],\"Win32_System_Variant\":[\"Win32_System\"],\"Win32_System_VirtualDosMachines\":[\"Win32_System\"],\"Win32_System_WindowsProgramming\":[\"Win32_System\"],\"Win32_System_Wmi\":[\"Win32_System\"],\"Win32_UI\":[\"Win32\"],\"Win32_UI_Accessibility\":[\"Win32_UI\"],\"Win32_UI_ColorSystem\":[\"Win32_UI\"],\"Win32_UI_Controls\":[\"Win32_UI\"],\"Win32_UI_Controls_Dialogs\":[\"Win32_UI_Controls\"],\"Win32_UI_HiDpi\":[\"Win32_UI\"],\"Win32_UI_Input\":[\"Win32_UI\"],\"Win32_UI_Input_Ime\":[\"Win32_UI_Input\"],\"Win32_UI_Input_KeyboardAndMouse\":[\"Win32_UI_Input\"],\"Win32_UI_Input_Pointer\":[\"Win32_UI_Input\"],\"Win32_UI_Input_Touch\":[\"Win32_UI_Input\"],\"Win32_UI_Input_XboxController\":[\"Win32_UI_Input\"],\"Win32_UI_InteractionContext\":[\"Win32_UI\"],\"Win32_UI_Magnification\":[\"Win32_UI\"],\"Win32_UI_Shell\":[\"Win32_UI\"],\"Win32_UI_Shell_PropertiesSystem\":[\"Win32_UI_Shell\"],\"Win32_UI_TabletPC\":[\"Win32_UI\"],\"Win32_UI_TextServices\":[\"Win32_UI\"],\"Win32_UI_WindowsAndMessaging\":[\"Win32_UI\"],\"Win32_Web\":[\"Win32\"],\"Win32_Web_InternetExplorer\":[\"Win32_Web\"],\"default\":[],\"docs\":[]}}", - "windows-sys_0.61.2": "{\"dependencies\":[{\"default_features\":false,\"name\":\"windows-link\",\"req\":\"^0.2.1\"}],\"features\":{\"Wdk\":[\"Win32_Foundation\"],\"Wdk_Devices\":[\"Wdk\"],\"Wdk_Devices_Bluetooth\":[\"Wdk_Devices\"],\"Wdk_Devices_HumanInterfaceDevice\":[\"Wdk_Devices\"],\"Wdk_Foundation\":[\"Wdk\"],\"Wdk_Graphics\":[\"Wdk\"],\"Wdk_Graphics_Direct3D\":[\"Wdk_Graphics\"],\"Wdk_NetworkManagement\":[\"Wdk\"],\"Wdk_NetworkManagement_Ndis\":[\"Wdk_NetworkManagement\"],\"Wdk_NetworkManagement_WindowsFilteringPlatform\":[\"Wdk_NetworkManagement\"],\"Wdk_Storage\":[\"Wdk\"],\"Wdk_Storage_FileSystem\":[\"Wdk_Storage\"],\"Wdk_Storage_FileSystem_Minifilters\":[\"Wdk_Storage_FileSystem\"],\"Wdk_System\":[\"Wdk\"],\"Wdk_System_IO\":[\"Wdk_System\"],\"Wdk_System_Memory\":[\"Wdk_System\"],\"Wdk_System_OfflineRegistry\":[\"Wdk_System\"],\"Wdk_System_Registry\":[\"Wdk_System\"],\"Wdk_System_SystemInformation\":[\"Wdk_System\"],\"Wdk_System_SystemServices\":[\"Wdk_System\"],\"Wdk_System_Threading\":[\"Wdk_System\"],\"Win32\":[\"Win32_Foundation\"],\"Win32_Data\":[\"Win32\"],\"Win32_Data_HtmlHelp\":[\"Win32_Data\"],\"Win32_Data_RightsManagement\":[\"Win32_Data\"],\"Win32_Devices\":[\"Win32\"],\"Win32_Devices_AllJoyn\":[\"Win32_Devices\"],\"Win32_Devices_Beep\":[\"Win32_Devices\"],\"Win32_Devices_BiometricFramework\":[\"Win32_Devices\"],\"Win32_Devices_Bluetooth\":[\"Win32_Devices\"],\"Win32_Devices_Cdrom\":[\"Win32_Devices\"],\"Win32_Devices_Communication\":[\"Win32_Devices\"],\"Win32_Devices_DeviceAndDriverInstallation\":[\"Win32_Devices\"],\"Win32_Devices_DeviceQuery\":[\"Win32_Devices\"],\"Win32_Devices_Display\":[\"Win32_Devices\"],\"Win32_Devices_Dvd\":[\"Win32_Devices\"],\"Win32_Devices_Enumeration\":[\"Win32_Devices\"],\"Win32_Devices_Enumeration_Pnp\":[\"Win32_Devices_Enumeration\"],\"Win32_Devices_Fax\":[\"Win32_Devices\"],\"Win32_Devices_HumanInterfaceDevice\":[\"Win32_Devices\"],\"Win32_Devices_Nfc\":[\"Win32_Devices\"],\"Win32_Devices_Nfp\":[\"Win32_Devices\"],\"Win32_Devices_PortableDevices\":[\"Win32_Devices\"],\"Win32_Devices_Properties\":[\"Win32_Devices\"],\"Win32_Devices_Pwm\":[\"Win32_Devices\"],\"Win32_Devices_Sensors\":[\"Win32_Devices\"],\"Win32_Devices_SerialCommunication\":[\"Win32_Devices\"],\"Win32_Devices_Tapi\":[\"Win32_Devices\"],\"Win32_Devices_Usb\":[\"Win32_Devices\"],\"Win32_Devices_WebServicesOnDevices\":[\"Win32_Devices\"],\"Win32_Foundation\":[\"Win32\"],\"Win32_Gaming\":[\"Win32\"],\"Win32_Globalization\":[\"Win32\"],\"Win32_Graphics\":[\"Win32\"],\"Win32_Graphics_Dwm\":[\"Win32_Graphics\"],\"Win32_Graphics_Gdi\":[\"Win32_Graphics\"],\"Win32_Graphics_GdiPlus\":[\"Win32_Graphics\"],\"Win32_Graphics_Hlsl\":[\"Win32_Graphics\"],\"Win32_Graphics_OpenGL\":[\"Win32_Graphics\"],\"Win32_Graphics_Printing\":[\"Win32_Graphics\"],\"Win32_Graphics_Printing_PrintTicket\":[\"Win32_Graphics_Printing\"],\"Win32_Management\":[\"Win32\"],\"Win32_Management_MobileDeviceManagementRegistration\":[\"Win32_Management\"],\"Win32_Media\":[\"Win32\"],\"Win32_Media_Audio\":[\"Win32_Media\"],\"Win32_Media_DxMediaObjects\":[\"Win32_Media\"],\"Win32_Media_KernelStreaming\":[\"Win32_Media\"],\"Win32_Media_Multimedia\":[\"Win32_Media\"],\"Win32_Media_Streaming\":[\"Win32_Media\"],\"Win32_Media_WindowsMediaFormat\":[\"Win32_Media\"],\"Win32_NetworkManagement\":[\"Win32\"],\"Win32_NetworkManagement_Dhcp\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Dns\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_InternetConnectionWizard\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_IpHelper\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Multicast\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Ndis\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetBios\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetManagement\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetShell\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetworkDiagnosticsFramework\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_P2P\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_QoS\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Rras\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Snmp\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WNet\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WebDav\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WiFi\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsConnectionManager\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsFilteringPlatform\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsFirewall\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsNetworkVirtualization\":[\"Win32_NetworkManagement\"],\"Win32_Networking\":[\"Win32\"],\"Win32_Networking_ActiveDirectory\":[\"Win32_Networking\"],\"Win32_Networking_Clustering\":[\"Win32_Networking\"],\"Win32_Networking_HttpServer\":[\"Win32_Networking\"],\"Win32_Networking_Ldap\":[\"Win32_Networking\"],\"Win32_Networking_WebSocket\":[\"Win32_Networking\"],\"Win32_Networking_WinHttp\":[\"Win32_Networking\"],\"Win32_Networking_WinInet\":[\"Win32_Networking\"],\"Win32_Networking_WinSock\":[\"Win32_Networking\"],\"Win32_Networking_WindowsWebServices\":[\"Win32_Networking\"],\"Win32_Security\":[\"Win32\"],\"Win32_Security_AppLocker\":[\"Win32_Security\"],\"Win32_Security_Authentication\":[\"Win32_Security\"],\"Win32_Security_Authentication_Identity\":[\"Win32_Security_Authentication\"],\"Win32_Security_Authorization\":[\"Win32_Security\"],\"Win32_Security_Credentials\":[\"Win32_Security\"],\"Win32_Security_Cryptography\":[\"Win32_Security\"],\"Win32_Security_Cryptography_Catalog\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_Certificates\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_Sip\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_UI\":[\"Win32_Security_Cryptography\"],\"Win32_Security_DiagnosticDataQuery\":[\"Win32_Security\"],\"Win32_Security_DirectoryServices\":[\"Win32_Security\"],\"Win32_Security_EnterpriseData\":[\"Win32_Security\"],\"Win32_Security_ExtensibleAuthenticationProtocol\":[\"Win32_Security\"],\"Win32_Security_Isolation\":[\"Win32_Security\"],\"Win32_Security_LicenseProtection\":[\"Win32_Security\"],\"Win32_Security_NetworkAccessProtection\":[\"Win32_Security\"],\"Win32_Security_WinTrust\":[\"Win32_Security\"],\"Win32_Security_WinWlx\":[\"Win32_Security\"],\"Win32_Storage\":[\"Win32\"],\"Win32_Storage_Cabinets\":[\"Win32_Storage\"],\"Win32_Storage_CloudFilters\":[\"Win32_Storage\"],\"Win32_Storage_Compression\":[\"Win32_Storage\"],\"Win32_Storage_DistributedFileSystem\":[\"Win32_Storage\"],\"Win32_Storage_FileHistory\":[\"Win32_Storage\"],\"Win32_Storage_FileSystem\":[\"Win32_Storage\"],\"Win32_Storage_Imapi\":[\"Win32_Storage\"],\"Win32_Storage_IndexServer\":[\"Win32_Storage\"],\"Win32_Storage_InstallableFileSystems\":[\"Win32_Storage\"],\"Win32_Storage_IscsiDisc\":[\"Win32_Storage\"],\"Win32_Storage_Jet\":[\"Win32_Storage\"],\"Win32_Storage_Nvme\":[\"Win32_Storage\"],\"Win32_Storage_OfflineFiles\":[\"Win32_Storage\"],\"Win32_Storage_OperationRecorder\":[\"Win32_Storage\"],\"Win32_Storage_Packaging\":[\"Win32_Storage\"],\"Win32_Storage_Packaging_Appx\":[\"Win32_Storage_Packaging\"],\"Win32_Storage_ProjectedFileSystem\":[\"Win32_Storage\"],\"Win32_Storage_StructuredStorage\":[\"Win32_Storage\"],\"Win32_Storage_Vhd\":[\"Win32_Storage\"],\"Win32_Storage_Xps\":[\"Win32_Storage\"],\"Win32_System\":[\"Win32\"],\"Win32_System_AddressBook\":[\"Win32_System\"],\"Win32_System_Antimalware\":[\"Win32_System\"],\"Win32_System_ApplicationInstallationAndServicing\":[\"Win32_System\"],\"Win32_System_ApplicationVerifier\":[\"Win32_System\"],\"Win32_System_ClrHosting\":[\"Win32_System\"],\"Win32_System_Com\":[\"Win32_System\"],\"Win32_System_Com_Marshal\":[\"Win32_System_Com\"],\"Win32_System_Com_StructuredStorage\":[\"Win32_System_Com\"],\"Win32_System_Com_Urlmon\":[\"Win32_System_Com\"],\"Win32_System_ComponentServices\":[\"Win32_System\"],\"Win32_System_Console\":[\"Win32_System\"],\"Win32_System_CorrelationVector\":[\"Win32_System\"],\"Win32_System_DataExchange\":[\"Win32_System\"],\"Win32_System_DeploymentServices\":[\"Win32_System\"],\"Win32_System_DeveloperLicensing\":[\"Win32_System\"],\"Win32_System_Diagnostics\":[\"Win32_System\"],\"Win32_System_Diagnostics_Ceip\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_Debug\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_Debug_Extensions\":[\"Win32_System_Diagnostics_Debug\"],\"Win32_System_Diagnostics_Etw\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_ProcessSnapshotting\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_ToolHelp\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_TraceLogging\":[\"Win32_System_Diagnostics\"],\"Win32_System_DistributedTransactionCoordinator\":[\"Win32_System\"],\"Win32_System_Environment\":[\"Win32_System\"],\"Win32_System_ErrorReporting\":[\"Win32_System\"],\"Win32_System_EventCollector\":[\"Win32_System\"],\"Win32_System_EventLog\":[\"Win32_System\"],\"Win32_System_EventNotificationService\":[\"Win32_System\"],\"Win32_System_GroupPolicy\":[\"Win32_System\"],\"Win32_System_HostCompute\":[\"Win32_System\"],\"Win32_System_HostComputeNetwork\":[\"Win32_System\"],\"Win32_System_HostComputeSystem\":[\"Win32_System\"],\"Win32_System_Hypervisor\":[\"Win32_System\"],\"Win32_System_IO\":[\"Win32_System\"],\"Win32_System_Iis\":[\"Win32_System\"],\"Win32_System_Ioctl\":[\"Win32_System\"],\"Win32_System_JobObjects\":[\"Win32_System\"],\"Win32_System_Js\":[\"Win32_System\"],\"Win32_System_Kernel\":[\"Win32_System\"],\"Win32_System_LibraryLoader\":[\"Win32_System\"],\"Win32_System_Mailslots\":[\"Win32_System\"],\"Win32_System_Mapi\":[\"Win32_System\"],\"Win32_System_Memory\":[\"Win32_System\"],\"Win32_System_Memory_NonVolatile\":[\"Win32_System_Memory\"],\"Win32_System_MessageQueuing\":[\"Win32_System\"],\"Win32_System_MixedReality\":[\"Win32_System\"],\"Win32_System_Ole\":[\"Win32_System\"],\"Win32_System_PasswordManagement\":[\"Win32_System\"],\"Win32_System_Performance\":[\"Win32_System\"],\"Win32_System_Performance_HardwareCounterProfiling\":[\"Win32_System_Performance\"],\"Win32_System_Pipes\":[\"Win32_System\"],\"Win32_System_Power\":[\"Win32_System\"],\"Win32_System_ProcessStatus\":[\"Win32_System\"],\"Win32_System_Recovery\":[\"Win32_System\"],\"Win32_System_Registry\":[\"Win32_System\"],\"Win32_System_RemoteDesktop\":[\"Win32_System\"],\"Win32_System_RemoteManagement\":[\"Win32_System\"],\"Win32_System_RestartManager\":[\"Win32_System\"],\"Win32_System_Restore\":[\"Win32_System\"],\"Win32_System_Rpc\":[\"Win32_System\"],\"Win32_System_Search\":[\"Win32_System\"],\"Win32_System_Search_Common\":[\"Win32_System_Search\"],\"Win32_System_SecurityCenter\":[\"Win32_System\"],\"Win32_System_Services\":[\"Win32_System\"],\"Win32_System_SetupAndMigration\":[\"Win32_System\"],\"Win32_System_Shutdown\":[\"Win32_System\"],\"Win32_System_StationsAndDesktops\":[\"Win32_System\"],\"Win32_System_SubsystemForLinux\":[\"Win32_System\"],\"Win32_System_SystemInformation\":[\"Win32_System\"],\"Win32_System_SystemServices\":[\"Win32_System\"],\"Win32_System_Threading\":[\"Win32_System\"],\"Win32_System_Time\":[\"Win32_System\"],\"Win32_System_TpmBaseServices\":[\"Win32_System\"],\"Win32_System_UserAccessLogging\":[\"Win32_System\"],\"Win32_System_Variant\":[\"Win32_System\"],\"Win32_System_VirtualDosMachines\":[\"Win32_System\"],\"Win32_System_WindowsProgramming\":[\"Win32_System\"],\"Win32_System_Wmi\":[\"Win32_System\"],\"Win32_UI\":[\"Win32\"],\"Win32_UI_Accessibility\":[\"Win32_UI\"],\"Win32_UI_ColorSystem\":[\"Win32_UI\"],\"Win32_UI_Controls\":[\"Win32_UI\"],\"Win32_UI_Controls_Dialogs\":[\"Win32_UI_Controls\"],\"Win32_UI_HiDpi\":[\"Win32_UI\"],\"Win32_UI_Input\":[\"Win32_UI\"],\"Win32_UI_Input_Ime\":[\"Win32_UI_Input\"],\"Win32_UI_Input_KeyboardAndMouse\":[\"Win32_UI_Input\"],\"Win32_UI_Input_Pointer\":[\"Win32_UI_Input\"],\"Win32_UI_Input_Touch\":[\"Win32_UI_Input\"],\"Win32_UI_Input_XboxController\":[\"Win32_UI_Input\"],\"Win32_UI_InteractionContext\":[\"Win32_UI\"],\"Win32_UI_Magnification\":[\"Win32_UI\"],\"Win32_UI_Shell\":[\"Win32_UI\"],\"Win32_UI_Shell_Common\":[\"Win32_UI_Shell\"],\"Win32_UI_Shell_PropertiesSystem\":[\"Win32_UI_Shell\"],\"Win32_UI_TabletPC\":[\"Win32_UI\"],\"Win32_UI_TextServices\":[\"Win32_UI\"],\"Win32_UI_WindowsAndMessaging\":[\"Win32_UI\"],\"Win32_Web\":[\"Win32\"],\"Win32_Web_InternetExplorer\":[\"Win32_Web\"],\"default\":[],\"docs\":[]}}", - "windows-targets_0.52.6": "{\"dependencies\":[{\"name\":\"windows_aarch64_gnullvm\",\"req\":\"^0.52.6\",\"target\":\"aarch64-pc-windows-gnullvm\"},{\"name\":\"windows_aarch64_msvc\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(target_arch = \\\"aarch64\\\", target_env = \\\"msvc\\\", not(windows_raw_dylib)))\"},{\"name\":\"windows_i686_gnu\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(target_arch = \\\"x86\\\", target_env = \\\"gnu\\\", not(target_abi = \\\"llvm\\\"), not(windows_raw_dylib)))\"},{\"name\":\"windows_i686_gnullvm\",\"req\":\"^0.52.6\",\"target\":\"i686-pc-windows-gnullvm\"},{\"name\":\"windows_i686_msvc\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(target_arch = \\\"x86\\\", target_env = \\\"msvc\\\", not(windows_raw_dylib)))\"},{\"name\":\"windows_x86_64_gnu\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(target_arch = \\\"x86_64\\\", target_env = \\\"gnu\\\", not(target_abi = \\\"llvm\\\"), not(windows_raw_dylib)))\"},{\"name\":\"windows_x86_64_gnullvm\",\"req\":\"^0.52.6\",\"target\":\"x86_64-pc-windows-gnullvm\"},{\"name\":\"windows_x86_64_msvc\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(any(target_arch = \\\"x86_64\\\", target_arch = \\\"arm64ec\\\"), target_env = \\\"msvc\\\", not(windows_raw_dylib)))\"}],\"features\":{}}", - "windows_aarch64_gnullvm_0.52.6": "{\"dependencies\":[],\"features\":{}}", - "windows_aarch64_msvc_0.52.6": "{\"dependencies\":[],\"features\":{}}", - "windows_i686_gnu_0.52.6": "{\"dependencies\":[],\"features\":{}}", - "windows_i686_gnullvm_0.52.6": "{\"dependencies\":[],\"features\":{}}", - "windows_i686_msvc_0.52.6": "{\"dependencies\":[],\"features\":{}}", - "windows_x86_64_gnu_0.52.6": "{\"dependencies\":[],\"features\":{}}", - "windows_x86_64_gnullvm_0.52.6": "{\"dependencies\":[],\"features\":{}}", - "windows_x86_64_msvc_0.52.6": "{\"dependencies\":[],\"features\":{}}", - "wit-bindgen-core_0.51.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.72\"},{\"features\":[\"derive\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4.3.19\"},{\"name\":\"heck\",\"req\":\"^0.5\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.218\"},{\"name\":\"wit-parser\",\"req\":\"^0.244.0\"}],\"features\":{\"clap\":[\"dep:clap\"],\"serde\":[\"dep:serde\"]}}", - "wit-bindgen-rt_0.39.0": "{\"dependencies\":[{\"name\":\"bitflags\",\"optional\":true,\"req\":\"^2.3.3\"},{\"name\":\"futures\",\"optional\":true,\"req\":\"^0.3.30\"},{\"name\":\"once_cell\",\"optional\":true,\"req\":\"^1.19.0\"}],\"features\":{\"async\":[\"dep:futures\",\"dep:once_cell\"]}}", - "wit-bindgen-rust-macro_0.51.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.72\"},{\"name\":\"prettyplease\",\"req\":\"^0.2.20\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"printing\"],\"name\":\"syn\",\"req\":\"^2.0.89\"},{\"name\":\"wit-bindgen-core\",\"req\":\"^0.51.0\"},{\"name\":\"wit-bindgen-rust\",\"req\":\"^0.51.0\"}],\"features\":{\"async\":[]}}", - "wit-bindgen-rust_0.51.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.72\"},{\"kind\":\"dev\",\"name\":\"bytes\",\"req\":\"^1\"},{\"features\":[\"derive\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4.3.19\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.31\"},{\"name\":\"heck\",\"req\":\"^0.5\"},{\"name\":\"indexmap\",\"req\":\"^2.0.0\"},{\"name\":\"prettyplease\",\"req\":\"^0.2.20\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.218\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"features\":[\"printing\"],\"name\":\"syn\",\"req\":\"^2.0.89\"},{\"default_features\":false,\"name\":\"wasm-metadata\",\"req\":\"^0.244.0\"},{\"name\":\"wit-bindgen-core\",\"req\":\"^0.51.0\"},{\"name\":\"wit-component\",\"req\":\"^0.244.0\"}],\"features\":{\"clap\":[\"dep:clap\",\"wit-bindgen-core/clap\"],\"serde\":[\"dep:serde\",\"wit-bindgen-core/serde\"]}}", - "wit-bindgen_0.51.0": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0\"},{\"name\":\"bitflags\",\"optional\":true,\"req\":\"^2.3.3\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"futures\",\"optional\":true,\"req\":\"^0.3.30\"},{\"name\":\"wit-bindgen-rust-macro\",\"optional\":true,\"req\":\"^0.51.0\"}],\"features\":{\"async\":[\"std\",\"wit-bindgen-rust-macro?/async\"],\"async-spawn\":[\"async\",\"dep:futures\"],\"bitflags\":[\"dep:bitflags\"],\"default\":[\"macros\",\"realloc\",\"async\",\"std\",\"bitflags\"],\"inter-task-wakeup\":[\"async\"],\"macros\":[\"dep:wit-bindgen-rust-macro\"],\"realloc\":[],\"rustc-dep-of-std\":[\"dep:core\",\"dep:alloc\"],\"std\":[]}}", - "wit-component_0.244.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"name\":\"bitflags\",\"req\":\"^2.3.3\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"kind\":\"dev\",\"name\":\"glob\",\"req\":\"^0.3.0\"},{\"default_features\":false,\"name\":\"indexmap\",\"req\":\"^2.7.0\"},{\"kind\":\"dev\",\"name\":\"libtest-mimic\",\"req\":\"^0.8.1\"},{\"name\":\"log\",\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.3.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"req\":\"^1.0.166\"},{\"name\":\"serde_derive\",\"req\":\"^1.0.166\"},{\"name\":\"serde_json\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"std\",\"wasmparser\"],\"name\":\"wasm-encoder\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"name\":\"wasm-metadata\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"oci\"],\"kind\":\"dev\",\"name\":\"wasm-metadata\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"simd\",\"std\",\"component-model\",\"simd\"],\"name\":\"wasmparser\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"simd\",\"std\",\"component-model\",\"features\"],\"kind\":\"dev\",\"name\":\"wasmparser\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"component-model\"],\"kind\":\"dev\",\"name\":\"wasmprinter\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"cranelift\",\"component-model\",\"runtime\",\"gc-drc\"],\"kind\":\"dev\",\"name\":\"wasmtime\",\"req\":\"^34.0.1\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"default_features\":false,\"name\":\"wast\",\"optional\":true,\"req\":\"^244.0.0\"},{\"default_features\":false,\"name\":\"wat\",\"optional\":true,\"req\":\"^1.244.0\"},{\"default_features\":false,\"features\":[\"component-model\"],\"kind\":\"dev\",\"name\":\"wat\",\"req\":\"^1.244.0\"},{\"features\":[\"decoding\",\"serde\"],\"name\":\"wit-parser\",\"req\":\"^0.244.0\"}],\"features\":{\"dummy-module\":[\"dep:wat\"],\"semver-check\":[\"dummy-module\"],\"wat\":[\"dep:wast\",\"dep:wat\"]}}", - "wit-parser_0.244.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"id-arena\",\"req\":\"^2\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"indexmap\",\"req\":\"^2.7.0\"},{\"kind\":\"dev\",\"name\":\"libtest-mimic\",\"req\":\"^0.8.1\"},{\"name\":\"log\",\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.3.0\"},{\"default_features\":false,\"name\":\"semver\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"name\":\"unicode-xid\",\"req\":\"^0.2.2\"},{\"default_features\":false,\"features\":[\"simd\",\"std\",\"validate\",\"component-model\",\"features\"],\"name\":\"wasmparser\",\"optional\":true,\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"component-model\"],\"name\":\"wat\",\"optional\":true,\"req\":\"^1.244.0\"}],\"features\":{\"decoding\":[\"dep:wasmparser\"],\"default\":[\"serde\",\"decoding\"],\"serde\":[\"dep:serde\",\"dep:serde_derive\",\"indexmap/serde\",\"serde_json\"],\"wat\":[\"decoding\",\"dep:wat\"]}}", - "zerocopy-derive_0.7.35": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.10\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"name\":\"syn\",\"req\":\"^2.0.31\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.85\"}],\"features\":{}}", - "zerocopy-derive_0.8.26": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"dissimilar\",\"req\":\"^1.0.9\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"=0.2.163\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"=1.9\"},{\"kind\":\"dev\",\"name\":\"prettyplease\",\"req\":\"=0.2.17\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.10\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.89\"}],\"features\":{}}", - "zerocopy-derive_0.8.42": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"dissimilar\",\"req\":\"^1.0.9\"},{\"kind\":\"dev\",\"name\":\"prettyplease\",\"req\":\"=0.2.17\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.40\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"features\":[\"visit\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.46\"}],\"features\":{}}", - "zerocopy_0.7.35": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"assert_matches\",\"req\":\"^1.5\"},{\"default_features\":false,\"name\":\"byteorder\",\"optional\":true,\"req\":\"^1.3\"},{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.85\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.7.35\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.7.35\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.7.35\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\"],\"alloc\":[],\"default\":[\"byteorder\"],\"derive\":[\"zerocopy-derive\"],\"simd\":[],\"simd-nightly\":[\"simd\"]}}", - "zerocopy_0.8.26": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"either\",\"req\":\"=1.13.0\"},{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.89\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.8.26\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\",\"std\"],\"alloc\":[],\"derive\":[\"zerocopy-derive\"],\"float-nightly\":[],\"simd\":[],\"simd-nightly\":[\"simd\"],\"std\":[\"alloc\"]}}", - "zerocopy_0.8.42": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.8.42\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.8.42\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.8.42\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\",\"std\"],\"alloc\":[],\"derive\":[\"zerocopy-derive\"],\"float-nightly\":[],\"simd\":[],\"simd-nightly\":[\"simd\"],\"std\":[\"alloc\"]}}", - "zmij_1.0.21": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\",\"target\":\"cfg(not(miri))\"},{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1.36\"},{\"kind\":\"dev\",\"name\":\"num-bigint\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"num-integer\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"num_cpus\",\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"opt-level\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"ryu\",\"req\":\"^1\"}],\"features\":{}}" - }, - "@@rules_rs+//rs/experimental/toolchains:module_extension.bzl%toolchains": { - "cargo-1.94.0-aarch64-apple-darwin.tar.xz": "7497ab873993cf38b806548c85bf818ffd71d1c3cf77b5bb4c3d3f1769588a08", - "cargo-1.94.0-aarch64-pc-windows-msvc.tar.xz": "0eacb6af29cacb850af4b390e698b2c8060f639a70e6b9f0575cdcc82b0822b2", - "cargo-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "d57ab7b23b52cba46e58c78327904d0c7db19892988677b0d59495c8af4bb790", - "cargo-1.94.0-x86_64-apple-darwin.tar.xz": "3beb7874010ebd757780284c0c9da47fc9f0b18091d7415d7b7231a8c19ec44f", - "cargo-1.94.0-x86_64-pc-windows-msvc.tar.xz": "6cf7113e1d5633721739a3bd6fee7148d6394bbab748fbede941210100374741", - "cargo-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "8e17624f3de39e079845bfb25ed15a042f4b50ceca78e37c56c4b9b15949b9f7", - "clippy-1.94.0-aarch64-apple-darwin.tar.xz": "42a683cbac0b81b96ad3b887cdade36a57ef3a131c203583ca7e3ce0be98f415", - "clippy-1.94.0-aarch64-pc-windows-msvc.tar.xz": "44468016e27457640c7ab62b7ceaddb44308e8ec77ee4344c1bd36113448c8eb", - "clippy-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "f7450c414e394450b53be602d37c374e2a2c5a2f51be2b54faf3d1e3272160b1", - "clippy-1.94.0-x86_64-apple-darwin.tar.xz": "1393cc688bc4d784a9d66619d2eec683a1ddb30b263f03ab615c0a2b18053126", - "clippy-1.94.0-x86_64-pc-windows-msvc.tar.xz": "e4dcc5a796857589e364064493abb3afb469e0e36e90155e7b0a2c5ce4bb7503", - "clippy-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "935529b0b9150084fa9cc1734c6598efe86b6e7d76f31ad3f8d4bb79ebf6031a", - "rust-std-1.94.0-aarch64-apple-darwin.tar.xz": "d3293668128b0a7838ef2544ae341eadd4437460ca22098af723349885a10ba2", - "rust-std-1.94.0-aarch64-apple-ios-macabi.tar.xz": "914f085065539ea1038d2832a5af1c22852cd5e9a063356b20e46698e72e770e", - "rust-std-1.94.0-aarch64-apple-ios-sim.tar.xz": "b4d16842b8f98ff2f431d6ab4aced298f4028ca00f0f5d588bc6efc94faa0680", - "rust-std-1.94.0-aarch64-apple-ios.tar.xz": "41ec1d18f4762f85231a19cf8aba3385da78fece2ff900a435c62881a3a3b44b", - "rust-std-1.94.0-aarch64-linux-android.tar.xz": "402c80df672a4b9fc948b6dc8bde21a63e1093eb1c8a2c2bd42221c309d9eba2", - "rust-std-1.94.0-aarch64-pc-windows-gnullvm.tar.xz": "8c89521f6826a1d2bf07b27c95405f4d6bf7c24f3de832c02758c41d4c1a511f", - "rust-std-1.94.0-aarch64-pc-windows-msvc.tar.xz": "d7964debd444dca92e771c953c3cd582fb927031b536c1439e1a831014d5f0d1", - "rust-std-1.94.0-aarch64-unknown-fuchsia.tar.xz": "87e11d97055540a0675bdc3dacf3d3d8cca5f806171ad0b51d0427f7cb9510c9", - "rust-std-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "c781b3ef4fefa5508fbe05820eddc95e46351d905a30921cc020febd9c596a2e", - "rust-std-1.94.0-aarch64-unknown-linux-musl.tar.xz": "3513032b2953d795bd3ac1489de4e8174e72500a31b4e7d1b349a23ab78a7a7b", - "rust-std-1.94.0-aarch64-unknown-none-softfloat.tar.xz": "375c496af31cf208ce67613c0aa1bde1894b5d674b00050cd46f7f954ed4a668", - "rust-std-1.94.0-aarch64-unknown-none.tar.xz": "854a9f8cea0ba889f553a7ea664c2a6479fc1fc49a4b56f84f99fc37001c4322", - "rust-std-1.94.0-aarch64-unknown-uefi.tar.xz": "f038d9e3df557d4aecdcf65989896179bcf0ecbc4d62bde71d33d7b914d3c253", - "rust-std-1.94.0-arm-linux-androideabi.tar.xz": "d2113ef50ed0b974f33b41102b961588c26abaf34f45a289a293b3d31ba01113", - "rust-std-1.94.0-arm-unknown-linux-gnueabi.tar.xz": "ad364b54b668e866315900ebff0d225616c556ab5228599792470aede05ba42c", - "rust-std-1.94.0-arm-unknown-linux-gnueabihf.tar.xz": "d5db943974fe05306ae43d100b0b7f3540108b93ef52de670675ce966af8a0fd", - "rust-std-1.94.0-arm-unknown-linux-musleabi.tar.xz": "c724b1a1b5dddeca563cda9b9695610bb56e681727b3af0a352174960087069a", - "rust-std-1.94.0-arm-unknown-linux-musleabihf.tar.xz": "9f8b707760a632b80864e91ea81a1b19c3f5452ff5e18b96da75792be1cf0a3c", - "rust-std-1.94.0-armv7-linux-androideabi.tar.xz": "e0b6883364f939300c281d96d6778ecf7fd97fe867b11936f0aa5c9a29f488ad", - "rust-std-1.94.0-armv7-unknown-linux-gnueabi.tar.xz": "9b1d8143881d510785d89413ea09b9fe910b64e3f300ab30e52e7fdd652ea66a", - "rust-std-1.94.0-armv7-unknown-linux-gnueabihf.tar.xz": "4b1744a8e48f58c2741afc1e076d0629a86051d785bd36e2c97f329eab6a5b49", - "rust-std-1.94.0-armv7-unknown-linux-musleabi.tar.xz": "fc2754f50ece0c7d3d32b2e0034b19c8b8bba9b471e0eb00f8ece7fa0bcb2837", - "rust-std-1.94.0-armv7-unknown-linux-musleabihf.tar.xz": "abfc0e7c1eaff1e3332b5c6262d12da309035a228261d07155add44464ed52f4", - "rust-std-1.94.0-i686-linux-android.tar.xz": "9093a894d87568a4b5eb4a144758bf49d4b62743677f747848afc3807c23f126", - "rust-std-1.94.0-i686-pc-windows-gnu.tar.xz": "188490bde6649163347050a70fd81c6d9e2e972ef12ba646f7f4fa106e21a956", - "rust-std-1.94.0-i686-pc-windows-gnullvm.tar.xz": "ba61dc038c3f4102a832a3381935192e88b0a1f79ff50b6d5590d476bdfa6879", - "rust-std-1.94.0-i686-pc-windows-msvc.tar.xz": "388cc2bb17ef9f12864c08e462cdbe3222d5ddafd84969dcf4ed944da27efcbd", - "rust-std-1.94.0-i686-unknown-freebsd.tar.xz": "3ce634092bcecb000c793c9d4bb0ccc6fd45c66fa8259fbb4f18c59844c4b13b", - "rust-std-1.94.0-i686-unknown-linux-gnu.tar.xz": "528c1dda491b25bb855426f48757773f4740c75d3c7271efffe7412c2fd5730d", - "rust-std-1.94.0-i686-unknown-linux-musl.tar.xz": "90ff16d879f774048c9b81d07188af4b63ed73a31848759cf560a9f51baa175f", - "rust-std-1.94.0-i686-unknown-uefi.tar.xz": "8c095069ec09d6e1d0b7e00f340fd7ea925a31acbad845d5956e1bc4a7d47733", - "rust-std-1.94.0-powerpc-unknown-linux-gnu.tar.xz": "23e32cab484d7787b7966a0508c49baf231bcdb4a79a421ee85ca69e81f2a06f", - "rust-std-1.94.0-powerpc64-unknown-linux-gnu.tar.xz": "7713b676a37fbd0a52a711c2e3c205a917f8b05614e17cd9f7f37874a88469a3", - "rust-std-1.94.0-powerpc64le-unknown-linux-gnu.tar.xz": "094f3955828d6630b70a9c7c71e3165c3ffc7a301929601dde08501def623347", - "rust-std-1.94.0-powerpc64le-unknown-linux-musl.tar.xz": "f3956b29463ec58d730f054ff2334755f929c602181ad67a25af9725c8500f68", - "rust-std-1.94.0-riscv32imc-unknown-none-elf.tar.xz": "5d10c7ab74cd5d10451480ec894ac5b98940a8527c333465eeec4a45b1125d43", - "rust-std-1.94.0-riscv64gc-unknown-linux-gnu.tar.xz": "32e9b7006f1269e48fb65af6298f6e8a222e1a8a2a20b112528bfb3dad2f65a8", - "rust-std-1.94.0-riscv64gc-unknown-linux-musl.tar.xz": "2c63d21143c7f1d44f3e3b1c9a82f3101576344c3fd1feb2d43ffc9d035e94f1", - "rust-std-1.94.0-riscv64gc-unknown-none-elf.tar.xz": "071a1ae54f7dead1b8147b9bae7d831a5554a37a1133f8cd52b265a765595a12", - "rust-std-1.94.0-s390x-unknown-linux-gnu.tar.xz": "ea69833addbc9521664464418c7468871f7a80b16fd2c385f6213a6c99cedfe9", - "rust-std-1.94.0-thumbv6m-none-eabi.tar.xz": "b022df94f6efa3bd2a6c4042472e114d2325bf058983b3ac7d586e24c8d77f25", - "rust-std-1.94.0-thumbv7em-none-eabi.tar.xz": "41894db05c4d8f421bf0dba7d0bed995685032393779f92b4ea1f3e6468badfb", - "rust-std-1.94.0-thumbv7em-none-eabihf.tar.xz": "be7e81535e7118fbad58a5dc31a1acfa7f2a36a2348aed23595169418559d67f", - "rust-std-1.94.0-thumbv7m-none-eabi.tar.xz": "4702e615be264df97b120fd075bed4e46710cc9a03c48115ce2ee3abfbf3822e", - "rust-std-1.94.0-thumbv8m.main-none-eabi.tar.xz": "5c1fb5ed8ad31580241b6e095583a2cac0ef90ea29dad7705c8a0ce0933a7797", - "rust-std-1.94.0-thumbv8m.main-none-eabihf.tar.xz": "573e9570bea039316ae3b55e323cbc741617ace1e0b87df8f3ecbbdf34b3de8a", - "rust-std-1.94.0-wasm32-unknown-emscripten.tar.xz": "0a607ba5d78afb47e9f5f134633ed5a954ece5558298e081084abe44f5265463", - "rust-std-1.94.0-wasm32-unknown-unknown.tar.xz": "bc65af3c0bfa2e5b20e299358710a26ce33a3674c762a2a010bc383c5b7fc202", - "rust-std-1.94.0-wasm32-wasip1-threads.tar.xz": "311b9fe1e94c90c16695899e670f0d76a366d493cdda29008191ce2d436cd22c", - "rust-std-1.94.0-wasm32-wasip1.tar.xz": "acbacacdca3e0ec0d93ff46d5f3579ae8cd5133b86e31a893da22cd94e28d0bf", - "rust-std-1.94.0-wasm32-wasip2.tar.xz": "e947e730da3fea212453b2835765802d9b0c80e19405cab188a6854834196e6c", - "rust-std-1.94.0-x86_64-apple-darwin.tar.xz": "cf716b5a5fab007398ce76536a85a3449df074b66e7c762c0f5de1d6c56a1342", - "rust-std-1.94.0-x86_64-apple-ios-macabi.tar.xz": "f39f4c1121dafcf8ea70a21c91be9552e30ea76debcad9bff28be945aaa79282", - "rust-std-1.94.0-x86_64-apple-ios.tar.xz": "60e70b80469dd3900dd8b1705831f5f02ac44fa97088b8a81fdda110ed421091", - "rust-std-1.94.0-x86_64-linux-android.tar.xz": "488c8a581612ac4779a05f05967b76a96e0ff0bdd1ab8d99c13a859c19056b01", - "rust-std-1.94.0-x86_64-pc-windows-gnu.tar.xz": "c9f98d62213d0559c8d2ee6d7eafbafbf5dfe04921c9e7cb15b66a770903bc33", - "rust-std-1.94.0-x86_64-pc-windows-gnullvm.tar.xz": "54c28d0030fab585457a431c555e5d23502132364d0483e5b3b2dfbfc3929cc6", - "rust-std-1.94.0-x86_64-pc-windows-msvc.tar.xz": "f8e2b43e52957e8224d97de68d6ec5da01613eb98bce4199de80071678b5fda0", - "rust-std-1.94.0-x86_64-unknown-freebsd.tar.xz": "081972f2f386ee859c15ed619695a9440896a80d1814d0e29d7620b7ef96207c", - "rust-std-1.94.0-x86_64-unknown-fuchsia.tar.xz": "6c18112553cf72cf991352208cce2bc96c712ca7b209acdd098abac894d674ef", - "rust-std-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "dd33653107c36e040082050d9e547e64dac5b456ba74069430d838c00c189a05", - "rust-std-1.94.0-x86_64-unknown-linux-musl.tar.xz": "b6786b5c490df2dcef76829039177799058f94e17725b42f912a057c0f6224f2", - "rust-std-1.94.0-x86_64-unknown-netbsd.tar.xz": "f763e56ca1ce3f819fb2136638d774777dd696f836cc52e6564431859a0f5159", - "rust-std-1.94.0-x86_64-unknown-none.tar.xz": "8509489165a8aa60a0590aeaa668f63423b3e7a33bb93e8e5719ff5404b6f6aa", - "rust-std-1.94.0-x86_64-unknown-uefi.tar.xz": "31b56e8178dd2319e51ccc474959d247db2767cda7149fe3704d636df9aee1c9", - "rustc-1.94.0-aarch64-apple-darwin.tar.xz": "6ba00a4f486af11826335cd7fe405592d959f50afb9489176fc1463cbe75dbf2", - "rustc-1.94.0-aarch64-pc-windows-msvc.tar.xz": "6d88d89c200a147357ded65e9bc426e14ebb6f5046d9772f3916b6a1ac60cdd5", - "rustc-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "7261b2fcdb88aef8b21f4b9f915d9a75c959674193f5b38e13fab4569684fc5b", - "rustc-1.94.0-x86_64-apple-darwin.tar.xz": "b1374b0fb63f5abcd8eb65607e43a499bfdc5e8160746d1f6f993aee9313f320", - "rustc-1.94.0-x86_64-pc-windows-msvc.tar.xz": "e0f59503778b563d3ee6d4edecd52b44056779ffe456181bad1fb4dc069802e6", - "rustc-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "31a0d3ac9383dfdeb4fce86eeed5ade3230131c635264c0eab7252dbf235f28e", - "rustfmt-1.94.0-aarch64-apple-darwin.tar.xz": "0eb716d201dc0d6d24ef8207d3ef27611d9d15d2212e3b27aa11e97286ed02ac", - "rustfmt-1.94.0-aarch64-pc-windows-msvc.tar.xz": "705ac3e797c4abed9e722e0864902f7a03872a8a201dcd7b70ff1b749eed2327", - "rustfmt-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "1ac17dbe7f8f6c98db5b62cb149fcd10c7565d624a59471115fc40055db9af28", - "rustfmt-1.94.0-x86_64-apple-darwin.tar.xz": "e58601a10ee85e956611ea2814bf26de88934a8224c1663bb11aa23f727663ec", - "rustfmt-1.94.0-x86_64-pc-windows-msvc.tar.xz": "7b32d8b0dd99f9bf6ddaa4866d39b7b96e8c2db46644d8527096f7f34b4fbe26", - "rustfmt-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "903e768f7626f78ed2d6cb82c1820b93bd339afec8575102e847fd3a379d33e5" - } - } + "facts": {} } diff --git a/zml/tokenizer/BUILD.bazel b/zml/tokenizer/BUILD.bazel index 92082ff5..ce2761e3 100644 --- a/zml/tokenizer/BUILD.bazel +++ b/zml/tokenizer/BUILD.bazel @@ -1,17 +1,15 @@ load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_test") -load("@zml//bazel:zig_srcs.bzl", "zig_srcs") zig_library( name = "tokenizer", - tags = ["manual"], srcs = ["homemade.zig"], import_name = "zml/tokenizer", main = "tokenizer.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ "//ffi:zig", "//stdx", - "//zml/tokenizer/hftokenizers", "//zml/tokenizer/iree", "//zml/tokenizer/sentencepiece", ], @@ -30,7 +28,7 @@ zig_binary( zig_test( name = "test", deps = [ - ":tokenizer" + ":tokenizer", ], ) diff --git a/zml/tokenizer/hftokenizers/.gitignore b/zml/tokenizer/hftokenizers/.gitignore deleted file mode 100644 index 2f7896d1..00000000 --- a/zml/tokenizer/hftokenizers/.gitignore +++ /dev/null @@ -1 +0,0 @@ -target/ diff --git a/zml/tokenizer/hftokenizers/BUILD.bazel b/zml/tokenizer/hftokenizers/BUILD.bazel deleted file mode 100644 index 43ee1fbe..00000000 --- a/zml/tokenizer/hftokenizers/BUILD.bazel +++ /dev/null @@ -1,35 +0,0 @@ -load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_rs//rs:rust_static_library.bzl", "rust_static_library") -load("@rules_zig//zig:defs.bzl", "zig_library") - -rust_static_library( - name = "hftokenizers_rs", - srcs = ["hftokenizers.rs"], - crate_name = "zml_tokenizer_hftokenizers", - edition = "2021", - proc_macro_deps = ["@crates//:tokenizers"], - deps = ["@crates//:tokenizers"], -) - -cc_library( - name = "hftokenizers_cc", - tags = ["manual"], - hdrs = ["hftokenizers.h"], - visibility = ["//visibility:public"], - deps = [ - ":hftokenizers_rs", - "//ffi:cc", - ], -) - -zig_library( - name = "hftokenizers", - tags = ["manual"], - main = "hftokenizers.zig", - visibility = ["//visibility:public"], - deps = [ - ":hftokenizers_cc", - "//ffi:zig", - "//stdx", - ], -) diff --git a/zml/tokenizer/hftokenizers/Cargo.lock b/zml/tokenizer/hftokenizers/Cargo.lock deleted file mode 100644 index 7164e77b..00000000 --- a/zml/tokenizer/hftokenizers/Cargo.lock +++ /dev/null @@ -1,713 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "ahash" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" -dependencies = [ - "cfg-if", - "getrandom", - "once_cell", - "serde", - "version_check", - "zerocopy 0.8.26", -] - -[[package]] -name = "aho-corasick" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" -dependencies = [ - "memchr", -] - -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - -[[package]] -name = "bitflags" -version = "2.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "castaway" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" -dependencies = [ - "rustversion", -] - -[[package]] -name = "cc" -version = "1.2.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f" -dependencies = [ - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "compact_str" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" -dependencies = [ - "castaway", - "cfg-if", - "itoa", - "rustversion", - "ryu", - "serde", - "static_assertions", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" - -[[package]] -name = "darling" -version = "0.20.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" -dependencies = [ - "darling_core", - "darling_macro", -] - -[[package]] -name = "darling_core" -version = "0.20.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn", -] - -[[package]] -name = "darling_macro" -version = "0.20.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" -dependencies = [ - "darling_core", - "quote", - "syn", -] - -[[package]] -name = "dary_heap" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728" -dependencies = [ - "serde", -] - -[[package]] -name = "derive_builder" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "derive_builder_macro" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" -dependencies = [ - "derive_builder_core", - "syn", -] - -[[package]] -name = "either" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" - -[[package]] -name = "esaxx-rs" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" -dependencies = [ - "cc", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "getrandom" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" -dependencies = [ - "cfg-if", - "libc", - "r-efi", - "wasi", -] - -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - -[[package]] -name = "itertools" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" - -[[package]] -name = "libc" -version = "0.2.169" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" - -[[package]] -name = "log" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" - -[[package]] -name = "macro_rules_attribute" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13" -dependencies = [ - "macro_rules_attribute-proc_macro", - "paste", -] - -[[package]] -name = "macro_rules_attribute-proc_macro" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568" - -[[package]] -name = "memchr" -version = "2.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" - -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - -[[package]] -name = "monostate" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e" -dependencies = [ - "monostate-impl", - "serde", -] - -[[package]] -name = "monostate-impl" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - -[[package]] -name = "once_cell" -version = "1.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" - -[[package]] -name = "onig" -version = "6.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0" -dependencies = [ - "bitflags", - "libc", - "once_cell", - "onig_sys", -] - -[[package]] -name = "onig_sys" -version = "69.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc" -dependencies = [ - "cc", - "pkg-config", -] - -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - -[[package]] -name = "pkg-config" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" - -[[package]] -name = "ppv-lite86" -version = "0.2.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" -dependencies = [ - "zerocopy 0.7.35", -] - -[[package]] -name = "proc-macro2" -version = "1.0.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "rand" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" -dependencies = [ - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" -dependencies = [ - "getrandom", -] - -[[package]] -name = "rayon" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-cond" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" -dependencies = [ - "either", - "itertools", - "rayon", -] - -[[package]] -name = "rayon-core" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - -[[package]] -name = "regex" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" - -[[package]] -name = "rustversion" -version = "1.0.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" - -[[package]] -name = "ryu" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" - -[[package]] -name = "serde" -version = "1.0.216" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.216" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.134" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" -dependencies = [ - "itoa", - "memchr", - "ryu", - "serde", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "smallvec" -version = "1.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" - -[[package]] -name = "spm_precompiled" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" -dependencies = [ - "base64", - "nom", - "serde", - "unicode-segmentation", -] - -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "syn" -version = "2.0.91" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "thiserror" -version = "2.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tokenizers" -version = "0.22.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223" -dependencies = [ - "ahash", - "aho-corasick", - "compact_str", - "dary_heap", - "derive_builder", - "esaxx-rs", - "getrandom", - "itertools", - "log", - "macro_rules_attribute", - "monostate", - "onig", - "paste", - "rand", - "rayon", - "rayon-cond", - "regex", - "regex-syntax", - "serde", - "serde_json", - "spm_precompiled", - "thiserror", - "unicode-normalization-alignments", - "unicode-segmentation", - "unicode_categories", -] - -[[package]] -name = "unicode-ident" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" - -[[package]] -name = "unicode-normalization-alignments" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" -dependencies = [ - "smallvec", -] - -[[package]] -name = "unicode-segmentation" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" - -[[package]] -name = "unicode_categories" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "wasi" -version = "0.14.2+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" -dependencies = [ - "wit-bindgen-rt", -] - -[[package]] -name = "wit-bindgen-rt" -version = "0.39.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" -dependencies = [ - "bitflags", -] - -[[package]] -name = "zerocopy" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" -dependencies = [ - "byteorder", - "zerocopy-derive 0.7.35", -] - -[[package]] -name = "zerocopy" -version = "0.8.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" -dependencies = [ - "zerocopy-derive 0.8.26", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zml_tokenizer_hftokenizers" -version = "0.1.0" -dependencies = [ - "tokenizers", -] diff --git a/zml/tokenizer/hftokenizers/Cargo.toml b/zml/tokenizer/hftokenizers/Cargo.toml deleted file mode 100644 index f905c004..00000000 --- a/zml/tokenizer/hftokenizers/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "zml_tokenizer_hftokenizers" -version = "0.1.0" -edition = "2021" - -[dependencies] -tokenizers = { version = "0.22.2", default-features = false, features = [ - # We don't need/want the progressbar feature - "onig", - "esaxx_fast", -] } - -[lib] -name = "zml_tokenizer_hftokenizers" -path = "hftokenizers.rs" diff --git a/zml/tokenizer/hftokenizers/hftokenizers.h b/zml/tokenizer/hftokenizers/hftokenizers.h deleted file mode 100644 index f779b575..00000000 --- a/zml/tokenizer/hftokenizers/hftokenizers.h +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -#include -#include - -#include "ffi/zig_slice.h" - -typedef struct hftokenizers hftokenizers; - -hftokenizers *hftokenizers_new(zig_slice); -hftokenizers *hftokenizers_new_from_bytes(zig_slice); -void hftokenizers_drop(hftokenizers *tokenizer); -zig_slice hftokenizers_encode(hftokenizers *tokenizer, zig_slice text); -void hftokenizers_tokens_drop(zig_slice tokens); -zig_slice hftokenizers_decode(hftokenizers *tokenizer, zig_slice tokens); -void hftokenizers_str_drop(zig_slice text); -uint32_t hftokenizers_token_to_id(hftokenizers *tokenizer, zig_slice token); diff --git a/zml/tokenizer/hftokenizers/hftokenizers.rs b/zml/tokenizer/hftokenizers/hftokenizers.rs deleted file mode 100644 index 048283cc..00000000 --- a/zml/tokenizer/hftokenizers/hftokenizers.rs +++ /dev/null @@ -1,109 +0,0 @@ -#[repr(C)] -struct ZigSlice { - ptr: *mut T, - len: usize, -} - -impl ZigSlice { - fn as_slice(&self) -> &[T] { - unsafe { std::slice::from_raw_parts(self.ptr, self.len) } - } - - fn as_slice_mut(&self) -> &mut [T] { - unsafe { std::slice::from_raw_parts_mut(self.ptr, self.len) } - } -} - -#[no_mangle] -extern "C" fn hftokenizers_new(path: ZigSlice) -> *mut tokenizers::Tokenizer { - return Box::into_raw(Box::new( - tokenizers::Tokenizer::from_file(std::path::Path::new( - std::str::from_utf8(path.as_slice()).unwrap(), - )) - .unwrap(), - )); -} - -#[no_mangle] -extern "C" fn hftokenizers_new_from_bytes(bytes: ZigSlice) -> *mut tokenizers::Tokenizer { - return Box::into_raw(Box::new( - tokenizers::Tokenizer::from_bytes(bytes.as_slice()) - .unwrap(), - )); -} - -#[no_mangle] -extern "C" fn hftokenizers_drop(t: *mut tokenizers::Tokenizer) { - drop(unsafe { Box::from_raw(t) }); -} - -#[no_mangle] -extern "C" fn hftokenizers_encode( - t: *mut tokenizers::Tokenizer, - string: ZigSlice, -) -> ZigSlice { - let input_str = std::str::from_utf8(string.as_slice()).unwrap(); - let encoded = unsafe { t.as_ref() } - .unwrap() - .encode_fast(input_str, false) - .unwrap(); - - // Convert the result to a boxed slice - let mut ids: Box<[u32]> = encoded.get_ids().to_owned().into_boxed_slice(); - - // Retrieve the zig slice associated to the boxed slice. - let slice = ZigSlice { - ptr: ids.as_mut_ptr(), - len: ids.len(), - }; - - // Leak the box so that it's not deallocated. - Box::leak(ids); - - return slice; -} - -#[no_mangle] -extern "C" fn hftokenizers_tokens_drop(tokens: ZigSlice) { - // Reconstruct the Box from the zig slice so that it's dropped. - drop(unsafe { Box::from_raw(tokens.as_slice_mut()) }); -} - -#[no_mangle] -extern "C" fn hftokenizers_decode( - t: *mut tokenizers::Tokenizer, - ids: ZigSlice, -) -> ZigSlice { - let decoded = unsafe { t.as_ref() } - .unwrap() - .decode(ids.as_slice(), false) - .unwrap(); - - // Convert the result to a boxed slice - let mut string: Box<[u8]> = decoded.into_bytes().into_boxed_slice(); - - // Retrieve the zig slice associated to the boxed slice. - let slice = ZigSlice { - ptr: string.as_mut_ptr(), - len: string.len(), - }; - - // Leak the box so that it's not deallocated. - Box::leak(string); - - return slice; -} - -#[no_mangle] -extern "C" fn hftokenizers_str_drop(tokens: ZigSlice) { - drop(unsafe { Box::from_raw(tokens.as_slice_mut()) }); -} - -#[no_mangle] -extern "C" fn hftokenizers_token_to_id(t: *mut tokenizers::Tokenizer, token: ZigSlice) -> u32 { - let id = unsafe { t.as_ref() } - .unwrap() - .token_to_id(std::str::from_utf8(token.as_slice()).unwrap()) - .unwrap_or(u32::MAX); - return id; -} diff --git a/zml/tokenizer/hftokenizers/hftokenizers.zig b/zml/tokenizer/hftokenizers/hftokenizers.zig deleted file mode 100644 index f5829ce6..00000000 --- a/zml/tokenizer/hftokenizers/hftokenizers.zig +++ /dev/null @@ -1,120 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const ffi = @import("ffi"); -const stdx = @import("stdx"); - -pub const Encoder = struct { - inner: *HFTokenizer, - current_ids: ?[]const u32 = null, - - fn init(inner: *HFTokenizer) Encoder { - return .{ .inner = inner }; - } - - pub fn reset(self: *Encoder) void { - if (self.current_ids) |current_ids_| { - c.hftokenizers_tokens_drop(ffi.ZigSlice.from(current_ids_)); - self.current_ids = null; - } - } - - pub fn deinit(self: *Encoder) void { - self.reset(); - } - - pub fn encode(self: *Encoder, input: []const u8) ![]const u32 { - self.reset(); - self.current_ids = ffi.ZigSlice.to(u32, c.hftokenizers_encode(@ptrCast(self.inner), ffi.ZigSlice.from(input))); - return self.ids(); - } - - pub fn ids(self: *const Encoder) []const u32 { - return self.current_ids orelse &.{}; - } -}; - -pub const Decoder = struct { - const StringBuffer = stdx.BoundedArray(u8, 128); - const TokensIdsBuffer = stdx.BoundedArray(u32, 4); - - inner: *HFTokenizer, - current_string: ?[]const u8 = null, - last_string: StringBuffer = .{ .len = 0 }, - last_token_ids: TokensIdsBuffer = .{ .len = 0 }, - - fn init(inner: *HFTokenizer) Decoder { - return .{ .inner = inner }; - } - - pub fn deinit(self: *Decoder) void { - self.reset(); - } - - pub fn reset(self: *Decoder) void { - if (self.current_string) |current_string_| { - c.hftokenizers_str_drop(ffi.ZigSlice.from(current_string_)); - self.current_string = null; - } - } - - pub fn decode(self: *Decoder, ids: []const u32) ![]const u8 { - self.reset(); - self.current_string = ffi.ZigSlice.to(u8, c.hftokenizers_decode(@ptrCast(self.inner), ffi.ZigSlice.from(ids))); - return self.string(); - } - - pub fn string(self: *const Decoder) []const u8 { - return self.current_string orelse &.{}; - } - - pub fn next(self: *Decoder, token_id: u32) !?[]const u8 { - if (self.last_token_ids.len >= self.last_token_ids.capacity()) { - _ = self.last_token_ids.orderedRemove(0); - } - self.last_token_ids.appendAssumeCapacity(token_id); - const new_string = try self.decode(self.last_token_ids.constSlice()); - if (self.last_string.len == 0) { - self.last_string = try StringBuffer.fromSlice(new_string); - return new_string; - } - var view = try std.unicode.Utf8View.init(self.last_string.constSlice()); - var it = view.iterator(); - while (it.nextCodepointSlice()) |cp| { - const start = it.i - cp.len; - if (std.mem.startsWith(u8, new_string, self.last_string.constSlice()[start..])) { - const chunk = new_string[self.last_string.len - start ..]; - self.last_string = try StringBuffer.fromSlice(new_string); - return chunk; - } - } - return null; - } -}; - -pub const HFTokenizer = opaque { - pub fn fromFile(model: []const u8) !*HFTokenizer { - return @ptrCast(c.hftokenizers_new(ffi.ZigSlice.from(model))); - } - - pub fn fromBytes(bytes: []const u8) !*HFTokenizer { - return @ptrCast(c.hftokenizers_new_from_bytes(ffi.ZigSlice.from(bytes))); - } - - pub fn deinit(self: *HFTokenizer) void { - return c.hftokenizers_drop(@ptrCast(self)); - } - - pub fn encoder(self: *HFTokenizer) !Encoder { - return Encoder.init(self); - } - - pub fn decoder(self: *HFTokenizer) !Decoder { - return Decoder.init(self); - } - - pub fn tokenId(self: *HFTokenizer, token: []const u8) ?u32 { - const id = c.hftokenizers_token_to_id(@ptrCast(self), ffi.ZigSlice.from(token)); - return if (id == std.math.maxInt(u32)) null else id; - } -}; diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig index e63a3560..8e8fea28 100644 --- a/zml/tokenizer/tokenizer.zig +++ b/zml/tokenizer/tokenizer.zig @@ -1,6 +1,5 @@ const std = @import("std"); -pub const hftokenizers = @import("hftokenizers"); pub const iree = @import("iree"); pub const sentencepiece = @import("sentencepiece"); @@ -9,7 +8,6 @@ pub const homemade = @import("homemade.zig"); const log = std.log.scoped(.@"zml/tokenizer"); const Tokenizers = enum { - hftokenizers, iree, sentencepiece, homemade, @@ -17,7 +15,6 @@ const Tokenizers = enum { pub const Tokenizer = union(Tokenizers) { pub const Encoder = union(Tokenizers) { - hftokenizers: hftokenizers.Encoder, iree: iree.Tokenizer.Encoder, sentencepiece: sentencepiece.Encoder, homemade: homemade.Encoder, @@ -174,7 +171,6 @@ pub const Tokenizer = union(Tokenizers) { }; pub const Decoder = union(Tokenizers) { - hftokenizers: hftokenizers.Decoder, iree: iree.Tokenizer.Decoder, sentencepiece: sentencepiece.Decoder, homemade: homemade.Decoder, @@ -245,7 +241,6 @@ pub const Tokenizer = union(Tokenizers) { } }; - hftokenizers: *hftokenizers.HFTokenizer, iree: iree.Tokenizer, sentencepiece: *sentencepiece.SentencePieceProcessor, homemade: *homemade.Tokenizer, From bd113697d407f91f6671079e0da22fae231de48f Mon Sep 17 00:00:00 2001 From: KB Date: Mon, 13 Apr 2026 14:56:55 +0200 Subject: [PATCH 418/881] zml/attention: add Linux compatibility only for Triton dependencies (#464) Co-authored-by: Kevin Barre --- zml/attention/triton/BUILD.bazel | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/zml/attention/triton/BUILD.bazel b/zml/attention/triton/BUILD.bazel index a8f6f59a..83f65f8f 100644 --- a/zml/attention/triton/BUILD.bazel +++ b/zml/attention/triton/BUILD.bazel @@ -27,6 +27,7 @@ py_library( "fake_plugin/compiler.py", "fake_plugin/driver.py", ], + target_compatible_with = ["@platforms//os:linux"], deps = ["@triton_py_deps//triton"], ) @@ -36,6 +37,7 @@ py_library( "triton_kernels/__init__.py", "triton_kernels/unified_attention.py", ], + target_compatible_with = ["@platforms//os:linux"], deps = ["@triton_py_deps//triton"], ) @@ -44,6 +46,7 @@ py_binary( srcs = ["generate.py"], main = "generate.py", python_version = "3.12", + target_compatible_with = ["@platforms//os:linux"], deps = [ ":fake_plugin", ":triton_kernels", @@ -66,11 +69,13 @@ zig_binary( linkopts = ["-Wl,-rpath,$$ORIGIN/../lib"], main = "generate.zig", tags = ["manual"], + target_compatible_with = ["@platforms//os:linux"], deps = [":libpython"], ) runfiles_to_default( name = "generate_files", + target_compatible_with = ["@platforms//os:linux"], visibility = ["//visibility:public"], deps = [ ":generate", @@ -83,6 +88,7 @@ copy_to_directory( srcs = [ ":generate_files", ], + target_compatible_with = ["@platforms//os:linux"], add_directory_to_runfiles = False, allow_overwrites = True, exclude_srcs_patterns = [ From 16fb132dec09b1f5278667a7708615f6397b2aca Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Mon, 13 Apr 2026 15:16:18 +0200 Subject: [PATCH 419/881] zml/attention/triton: rename .num_pages_per_seq to .p (#466) Using the same axis name as FlashAttention for consistency. --- zml/attention/triton.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zml/attention/triton.zig b/zml/attention/triton.zig index 4e8be6fa..3440948e 100644 --- a/zml/attention/triton.zig +++ b/zml/attention/triton.zig @@ -266,7 +266,7 @@ pub const paged = struct { pub fn init(options_: Options) Parameters { return .{ - .block_table = .init(.{ .b = options_.batch_size, .num_pages_per_seq = options_.max_num_pages }, .i32), + .block_table = .init(.{ .b = options_.batch_size, .p = options_.max_num_pages }, .i32), .seq_lens = .init(.{ .b = options_.batch_size }, .i32), .query_start_len = .init(.{ .b = options_.batch_size + 1 }, .i32), .options_ = options_, @@ -374,7 +374,7 @@ pub const paged = struct { .batch_size = @intCast(parameters_.block_table.dim(.b)), .block_size = @intCast(k_cache_.dim(.k_chunk)), .num_blocks = @intCast(k_cache_.dim(.page)), - .max_num_block_per_seq = @intCast(parameters_.block_table.dim(.num_pages_per_seq)), + .max_num_block_per_seq = @intCast(parameters_.block_table.dim(.p)), .sliding_window = if (ctx_.opts.sliding_window < 0) 0 else @intCast(ctx_.opts.sliding_window), .block_m = block_m, .block_q = block_q, From d343b84fd88d2d225cae84e96638c39628527d9b Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 14 Apr 2026 09:42:02 +0200 Subject: [PATCH 420/881] zml/tokenizer: Re-introduce HF tokenizer (#469) --- .bazelrc | 1 + .vscode/settings.json | 7 +- MODULE.bazel | 33 +- MODULE.bazel.lock | 334 ++++++++- zml/tokenizer/BUILD.bazel | 6 +- zml/tokenizer/hftokenizers/.gitignore | 1 + zml/tokenizer/hftokenizers/BUILD.bazel | 35 + zml/tokenizer/hftokenizers/Cargo.lock | 713 ++++++++++++++++++++ zml/tokenizer/hftokenizers/Cargo.toml | 15 + zml/tokenizer/hftokenizers/hftokenizers.h | 17 + zml/tokenizer/hftokenizers/hftokenizers.rs | 109 +++ zml/tokenizer/hftokenizers/hftokenizers.zig | 120 ++++ zml/tokenizer/tokenizer.zig | 12 +- 13 files changed, 1390 insertions(+), 13 deletions(-) create mode 100644 zml/tokenizer/hftokenizers/.gitignore create mode 100644 zml/tokenizer/hftokenizers/BUILD.bazel create mode 100644 zml/tokenizer/hftokenizers/Cargo.lock create mode 100644 zml/tokenizer/hftokenizers/Cargo.toml create mode 100644 zml/tokenizer/hftokenizers/hftokenizers.h create mode 100644 zml/tokenizer/hftokenizers/hftokenizers.rs create mode 100644 zml/tokenizer/hftokenizers/hftokenizers.zig diff --git a/.bazelrc b/.bazelrc index 92b9888b..42f50189 100644 --- a/.bazelrc +++ b/.bazelrc @@ -124,6 +124,7 @@ common:release --@rules_zig//zig/settings:mode=release_safe # --config=native builds everything optimized for the local CPU common:native --copt=-march=native common:native --@rules_zig//zig/settings:zigopt=-mcpu=native +common:native --@@rules_rs++rules_rust+rules_rust//rust/settings:extra_rustc_flag=-Ctarget-cpu=native common:linux --host_platform=//platforms:host_glibc diff --git a/.vscode/settings.json b/.vscode/settings.json index bc25445d..82050af6 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -6,7 +6,7 @@ "**/.hg": true, "**/CVS": true, "**/.DS_Store": true, - "**/bazel-*": true, + "**/bazel-*": true }, "files.watcherExclude": { "**/bazel-*": true, @@ -29,6 +29,9 @@ "editor.stickyScroll.defaultModel": "foldingProviderModel", "editor.codeActionsOnSave": { "source.fixAll": "explicit", - }, + } }, + "rust-analyzer.linkedProjects": [ + "${workspaceFolder}/zml/tokenizer/hftokenizers/Cargo.toml" + ] } diff --git a/MODULE.bazel b/MODULE.bazel index 2dac4cc2..b1ce05e7 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -6,7 +6,6 @@ bazel_dep(name = "abseil-cpp", version = "20250814.1", repo_name = "com_google_a bazel_dep(name = "bazel_lib", version = "3.2.2") bazel_dep(name = "aspect_rules_py", version = "1.8.4") bazel_dep(name = "bazel_skylib", version = "1.9.0") - ## Needed by iree bazel_dep(name = "libbacktrace", version = "1.0.0-20250926-7939218") bazel_dep(name = "patchelf", version = "0.18.0.bcr.1") @@ -21,6 +20,7 @@ bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep(name = "rules_oci", version = "2.2.7") bazel_dep(name = "rules_proto", version = "7.1.0") bazel_dep(name = "rules_python", version = "1.9.0") +bazel_dep(name = "rules_rs", version = "0.0.44") bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "rules_zig", version = "0.12.2") bazel_dep(name = "tar.bzl", version = "0.9.0") @@ -83,7 +83,7 @@ uv.configure(version = "0.9.30") python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain(python_version = "3.12") -use_repo(python, "python_3_12", "python_versions") +use_repo(python, "python_versions", "python_3_12") pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") @@ -140,7 +140,7 @@ cuda = use_extension("//platforms/cuda:cuda.bzl", "cuda_packages") use_repo(cuda, "cuda_nvml_dev", "libpjrt_cuda") rocm = use_extension("//platforms/rocm:rocm.bzl", "rocm_packages") -use_repo(rocm, "amd-smi-lib", "hipblaslt", "libdrm-amdgpu-amdgpu1", "libdrm2-amdgpu", "libpjrt_rocm", "rocblas") +use_repo(rocm, "amd-smi-lib", "hipblaslt", "libpjrt_rocm", "libdrm2-amdgpu", "libdrm-amdgpu-amdgpu1", "rocblas") tpu = use_extension("//platforms/tpu:tpu.bzl", "tpu_packages") use_repo(tpu, "libpjrt_tpu") @@ -154,7 +154,7 @@ use_repo(zls, "zls_toolchains") register_toolchains("@zls_toolchains//:all") non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") -use_repo(non_module_deps, "arocc", "cloud_accelerator_diagnostics", "com_google_sentencepiece", "flashattn", "iree", "libvaxis", "linenoise", "mnist", "org_swig_swig", "stb", "translate-c", "uucode", "xla", "zigimg") +use_repo(non_module_deps, "arocc", "cloud_accelerator_diagnostics", "com_google_sentencepiece", "flashattn", "iree", "libvaxis", "linenoise", "mnist", "org_swig_swig", "translate-c", "stb", "uucode", "xla", "zigimg") xla = use_extension("//third_party/xla:xla.bzl", "xla") use_repo( @@ -184,6 +184,30 @@ llvm = use_extension("//third_party/xla:llvm.bzl", "llvm") llvm.configure() use_repo(llvm, "llvm-project") +rs_toolchains = use_extension("@rules_rs//rs/experimental/toolchains:module_extension.bzl", "toolchains") +rs_toolchains.toolchain( + edition = "2024", + version = "1.94.0", +) +use_repo(rs_toolchains, "default_rust_toolchains") + +register_toolchains("@default_rust_toolchains//:all") + +crate = use_extension("@rules_rs//rs:extensions.bzl", "crate") +crate.from_cargo( + name = "crates", + cargo_lock = "//zml/tokenizer/hftokenizers:Cargo.lock", + cargo_toml = "//zml/tokenizer/hftokenizers:Cargo.toml", + platform_triples = [ + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-gnu", + ], + use_experimental_platforms = True, +) +use_repo(crate, "crates") + apt = use_extension("@rules_distroless//apt:extensions.bzl", "apt") apt.install( name = "apt_cuda", @@ -227,6 +251,7 @@ use_repo( "distroless_cc_debian12_linux_amd64", ) + git_override( module_name = "rules_zig", commit = "56911842f4527df036e8d52e78dd0f54a22d4215", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 8b6d62ba..66de42db 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -22,6 +22,7 @@ "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", "https://bcr.bazel.build/modules/apple_support/1.17.1/MODULE.bazel": "655c922ab1209978a94ef6ca7d9d43e940cd97d9c172fb55f94d91ac53f8610b", "https://bcr.bazel.build/modules/apple_support/1.23.1/MODULE.bazel": "53763fed456a968cf919b3240427cf3a9d5481ec5466abc9d5dc51bc70087442", + "https://bcr.bazel.build/modules/apple_support/1.24.1/MODULE.bazel": "f46e8ddad60aef170ee92b2f3d00ef66c147ceafea68b6877cb45bd91737f5f8", "https://bcr.bazel.build/modules/apple_support/2.2.0/MODULE.bazel": "5e32bc42e413a4544df7afee7fa4c7aff73e670a44e56e8e45ce1954332034ad", "https://bcr.bazel.build/modules/apple_support/2.2.0/source.json": "368bf1de5dace2a411e792e6df2ea5384eb8db3899680ed51b37eb48b88fd8ac", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", @@ -32,6 +33,7 @@ "https://bcr.bazel.build/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "812d2dd42f65dca362152101fbec418029cc8fd34cbad1a2fde905383d705838", "https://bcr.bazel.build/modules/aspect_rules_py/1.8.4/MODULE.bazel": "4ab4a78e9d65cf15e755807f7ae504655e1771cefd7c3343ad34dbbb625b9564", "https://bcr.bazel.build/modules/aspect_rules_py/1.8.4/source.json": "7cd80ad811f0c2b3aad3c878506ae602fce2fe53b78487ecfbcc8bcde07d220e", + "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.2/MODULE.bazel": "598e7fe3b54f5fa64fdbeead1027653963a359cc23561d43680006f3b463d5a4", "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.3/MODULE.bazel": "37c764292861c2f70314efa9846bb6dbb44fc0308903b3285da6528305450183", "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.3/source.json": "605086bbc197743a0d360f7ddc550a1d4dfa0441bc807236e17170f636153348", "https://bcr.bazel.build/modules/bazel_features/1.1.0/MODULE.bazel": "cfd42ff3b815a5f39554d97182657f8c4b9719568eb7fded2b9135f084bf760b", @@ -49,6 +51,7 @@ "https://bcr.bazel.build/modules/bazel_features/1.28.0/MODULE.bazel": "4b4200e6cbf8fa335b2c3f43e1d6ef3e240319c33d43d60cc0fbd4b87ece299d", "https://bcr.bazel.build/modules/bazel_features/1.3.0/MODULE.bazel": "cdcafe83ec318cda34e02948e81d790aab8df7a929cec6f6969f13a489ccecd9", "https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87", + "https://bcr.bazel.build/modules/bazel_features/1.32.0/MODULE.bazel": "095d67022a58cb20f7e20e1aefecfa65257a222c18a938e2914fd257b5f1ccdc", "https://bcr.bazel.build/modules/bazel_features/1.33.0/MODULE.bazel": "8b8dc9d2a4c88609409c3191165bccec0e4cb044cd7a72ccbe826583303459f6", "https://bcr.bazel.build/modules/bazel_features/1.34.0/MODULE.bazel": "e8475ad7c8965542e0c7aac8af68eb48c4af904be3d614b6aa6274c092c2ea1e", "https://bcr.bazel.build/modules/bazel_features/1.38.0/MODULE.bazel": "f9b8a9c890ebd216b4049fd12a31d3c2602e3403c7af636b04fbbd7453edc9c9", @@ -148,6 +151,7 @@ "https://bcr.bazel.build/modules/libbacktrace/1.0.0-20250926-7939218/source.json": "83239989d09748f63dee4bdd430d261e217a012670e066e2238925a303a65d89", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", "https://bcr.bazel.build/modules/libpfm/4.11.0/source.json": "caaffb3ac2b59b8aac456917a4ecf3167d40478ee79f15ab7a877ec9273937c9", + "https://bcr.bazel.build/modules/llvm/0.6.7/MODULE.bazel": "d37a2e10571864dc6a5bb53c29216d90b9400bbcadb422337f49107fd2eaf0d2", "https://bcr.bazel.build/modules/llvm/0.7.0/MODULE.bazel": "3c07a4e5734b0ad41fe24dedaacbf3a35ce4377b7e1a21f24488a0c9ac4f1e6b", "https://bcr.bazel.build/modules/llvm/0.7.0/source.json": "5f0a015da1e7db09ef61aa254ba5eb55b0c322fadd1e8eefeca368d7723a24a0", "https://bcr.bazel.build/modules/mbedtls/3.6.0/MODULE.bazel": "8e380e4698107c5f8766264d4df92e36766248447858db28187151d884995a09", @@ -175,7 +179,8 @@ "https://bcr.bazel.build/modules/opentracing-cpp/1.6.0/MODULE.bazel": "b3925269f63561b8b880ae7cf62ccf81f6ece55b62cd791eda9925147ae116ec", "https://bcr.bazel.build/modules/opentracing-cpp/1.6.0/source.json": "da1cb1add160f5e5074b7272e9db6fd8f1b3336c15032cd0a653af9d2f484aed", "https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92", - "https://bcr.bazel.build/modules/package_metadata/0.0.2/source.json": "e53a759a72488d2c0576f57491ef2da0cf4aab05ac0997314012495935531b73", + "https://bcr.bazel.build/modules/package_metadata/0.0.5/MODULE.bazel": "ef4f9439e3270fdd6b9fd4dbc3d2f29d13888e44c529a1b243f7a31dfbc2e8e4", + "https://bcr.bazel.build/modules/package_metadata/0.0.5/source.json": "2326db2f6592578177751c3e1f74786b79382cd6008834c9d01ec865b9126a85", "https://bcr.bazel.build/modules/patchelf/0.18.0.bcr.1/MODULE.bazel": "35320668481400503dea7c75313473b9ffd6dea4e7ba583b96a896199b0088d5", "https://bcr.bazel.build/modules/patchelf/0.18.0.bcr.1/source.json": "d877d271ef5d531407885322054c99c7f0617a9a85ff99c0a70f4ced7f13e13a", "https://bcr.bazel.build/modules/pcre2/10.45/MODULE.bazel": "9bdc838aa2e21be030d7181385e45a5c87b14575bb7ff35d10b5be3d56913bed", @@ -322,6 +327,8 @@ "https://bcr.bazel.build/modules/rules_python/1.7.0/MODULE.bazel": "d01f995ecd137abf30238ad9ce97f8fc3ac57289c8b24bd0bf53324d937a14f8", "https://bcr.bazel.build/modules/rules_python/1.9.0/MODULE.bazel": "afc3a05f29f09f2d3ee95ad99a145250dab41a2b2d8d6f82cc91936b3213282c", "https://bcr.bazel.build/modules/rules_python/1.9.0/source.json": "3921ea0b65298d51aead5b9e4a82203d7be9b5918619b58b53f1c259f4e63169", + "https://bcr.bazel.build/modules/rules_rs/0.0.44/MODULE.bazel": "aa151d9a96e5d9363732df8943c7fb5327738cc160732fa74c02e5de7822007b", + "https://bcr.bazel.build/modules/rules_rs/0.0.44/source.json": "1358c9418e1325fdb5d0caf53ef32f98d525fe2030668279cae2e97c34cd29eb", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", @@ -441,7 +448,7 @@ "@@aspect_tools_telemetry+//:extension.bzl%telemetry": { "general": { "bzlTransitiveDigest": "MePriaXmQNSqUfE+YQvEtzBc2bU1mjITINIffFiZYgo=", - "usagesDigest": "pyph4lSDn5CHIn4s2mtKJTkRaLV/o2zIFgnrpufr/8o=", + "usagesDigest": "TU1quwRKpvAEOEF35/IVh4NeoIS2hayv4fGp8f5RXoY=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -451,6 +458,7 @@ "attributes": { "deps": { "aspect_rules_py": "1.8.4", + "rules_rs": "0.0.44", "aspect_tools_telemetry": "0.3.3" } } @@ -1720,5 +1728,325 @@ } } }, - "facts": {} + "facts": { + "@@rules_rs+//rs:extensions.bzl%crate": { + "addr2line_0.25.1": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"backtrace\",\"req\":\"^0.3.13\"},{\"features\":[\"wrap_help\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4.3.21\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"cpp_demangle\",\"optional\":true,\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.7.0\"},{\"default_features\":false,\"name\":\"fallible-iterator\",\"optional\":true,\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"findshlibs\",\"req\":\"^0.10\"},{\"default_features\":false,\"features\":[\"read\"],\"name\":\"gimli\",\"req\":\"^0.32.0\"},{\"kind\":\"dev\",\"name\":\"libtest-mimic\",\"req\":\"^0.8.1\"},{\"name\":\"memmap2\",\"optional\":true,\"req\":\"^0.9.4\"},{\"default_features\":false,\"features\":[\"read\",\"compression\"],\"name\":\"object\",\"optional\":true,\"req\":\"^0.37.0\"},{\"name\":\"rustc-demangle\",\"optional\":true,\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"typed-arena\",\"optional\":true,\"req\":\"^2\"}],\"features\":{\"all\":[\"bin\",\"wasm\"],\"bin\":[\"loader\",\"rustc-demangle\",\"cpp_demangle\",\"fallible-iterator\",\"smallvec\",\"dep:clap\"],\"cargo-all\":[],\"default\":[\"rustc-demangle\",\"cpp_demangle\",\"loader\",\"fallible-iterator\",\"smallvec\"],\"loader\":[\"std\",\"dep:object\",\"dep:memmap2\",\"dep:typed-arena\"],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"gimli/rustc-dep-of-std\"],\"std\":[\"gimli/std\"],\"wasm\":[\"object/wasm\"]}}", + "adler2_2.0.1": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"std\"],\"rustc-dep-of-std\":[\"core\"],\"std\":[]}}", + "ahash_0.8.12": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1.0\"},{\"name\":\"const-random\",\"optional\":true,\"req\":\"^0.1.17\"},{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.2\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.5\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2.1\"},{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"hashbrown\",\"req\":\"^0.14.3\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.2\"},{\"kind\":\"dev\",\"name\":\"no-panic\",\"req\":\"^0.1.10\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"once_cell\",\"req\":\"^1.18.0\",\"target\":\"cfg(not(all(target_arch = \\\"arm\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"pcg-mwc\",\"req\":\"^0.2.1\"},{\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"seahash\",\"req\":\"^4.0\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.117\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.59\"},{\"kind\":\"dev\",\"name\":\"smallvec\",\"req\":\"^1.13.1\"},{\"kind\":\"build\",\"name\":\"version_check\",\"req\":\"^0.9.4\"},{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"zerocopy\",\"req\":\"^0.8.24\"}],\"features\":{\"atomic-polyfill\":[\"dep:portable-atomic\",\"once_cell/critical-section\"],\"compile-time-rng\":[\"const-random\"],\"default\":[\"std\",\"runtime-rng\"],\"nightly-arm-aes\":[],\"no-rng\":[],\"runtime-rng\":[\"getrandom\"],\"std\":[]}}", + "aho-corasick_1.1.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.17\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.4.0\"}],\"features\":{\"default\":[\"std\",\"perf-literal\"],\"logging\":[\"dep:log\"],\"perf-literal\":[\"dep:memchr\"],\"std\":[\"memchr?/std\"]}}", + "aho-corasick_1.1.4": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.17\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.4.0\"}],\"features\":{\"default\":[\"std\",\"perf-literal\"],\"logging\":[\"dep:log\"],\"perf-literal\":[\"dep:memchr\"],\"std\":[\"memchr?/std\"]}}", + "anyhow_1.0.102": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.6\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0\"},{\"kind\":\"dev\",\"name\":\"thiserror\",\"req\":\"^2\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"}],\"features\":{\"backtrace\":[],\"default\":[\"std\"],\"std\":[]}}", + "async-stream-impl_0.3.6": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"full\",\"visit-mut\"],\"name\":\"syn\",\"req\":\"^2.0.2\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"}],\"features\":{}}", + "async-stream_0.3.6": "{\"dependencies\":[{\"name\":\"async-stream-impl\",\"req\":\"=0.3.6\"},{\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1\"}],\"features\":{}}", + "async-trait_0.1.89": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.30\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"full\",\"parsing\",\"printing\",\"proc-macro\",\"visit-mut\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1.40\"},{\"kind\":\"dev\",\"name\":\"tracing-attributes\",\"req\":\"^0.1.27\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{}}", + "atomic-waker_1.1.2": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"cargo_bench_support\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.5\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.7.0\"}],\"features\":{}}", + "autocfg_1.5.0": "{\"dependencies\":[],\"features\":{}}", + "axum-core_0.4.5": "{\"dependencies\":[{\"name\":\"async-trait\",\"req\":\"^0.1.67\"},{\"kind\":\"dev\",\"name\":\"axum\",\"req\":\"^0.7.2\"},{\"name\":\"bytes\",\"req\":\"^1.2\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"http\",\"req\":\"^1.0.0\"},{\"name\":\"http-body\",\"req\":\"^1.0.0\"},{\"name\":\"http-body-util\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^1.0.0\"},{\"name\":\"mime\",\"req\":\"^0.3.16\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"},{\"name\":\"rustversion\",\"req\":\"^1.0.9\"},{\"name\":\"sync_wrapper\",\"req\":\"^1.0.0\"},{\"features\":[\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.25.0\"},{\"features\":[\"limit\"],\"name\":\"tower-http\",\"optional\":true,\"req\":\"^0.6.0\"},{\"features\":[\"limit\"],\"kind\":\"dev\",\"name\":\"tower-http\",\"req\":\"^0.6.0\"},{\"name\":\"tower-layer\",\"req\":\"^0.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.37\"}],\"features\":{\"__private_docs\":[\"dep:tower-http\"],\"tracing\":[\"dep:tracing\"]}}", + "axum_0.7.9": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0\"},{\"name\":\"async-trait\",\"req\":\"^0.1.67\"},{\"name\":\"axum-core\",\"req\":\"^0.4.5\"},{\"name\":\"axum-macros\",\"optional\":true,\"req\":\"^0.4.2\"},{\"features\":[\"__private\"],\"kind\":\"dev\",\"name\":\"axum-macros\",\"req\":\"^0.4.1\"},{\"name\":\"base64\",\"optional\":true,\"req\":\"^0.22.1\"},{\"name\":\"bytes\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"http\",\"req\":\"^1.0.0\"},{\"name\":\"http-body\",\"req\":\"^1.0.0\"},{\"name\":\"http-body-util\",\"req\":\"^0.1.0\"},{\"name\":\"hyper\",\"optional\":true,\"req\":\"^1.1.0\"},{\"features\":[\"tokio\",\"server\",\"service\"],\"name\":\"hyper-util\",\"optional\":true,\"req\":\"^0.1.3\"},{\"name\":\"itoa\",\"req\":\"^1.0.5\"},{\"name\":\"matchit\",\"req\":\"^0.7\"},{\"name\":\"memchr\",\"req\":\"^2.4.1\"},{\"name\":\"mime\",\"req\":\"^0.3.16\"},{\"name\":\"multer\",\"optional\":true,\"req\":\"^3.0.0\"},{\"name\":\"percent-encoding\",\"req\":\"^2.1\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"json\",\"stream\",\"multipart\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.12\"},{\"name\":\"rustversion\",\"req\":\"^1.0.9\"},{\"name\":\"serde\",\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"features\":[\"raw_value\"],\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"raw_value\"],\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"serde_path_to_error\",\"optional\":true,\"req\":\"^0.1.8\"},{\"name\":\"serde_urlencoded\",\"optional\":true,\"req\":\"^0.7\"},{\"name\":\"sha1\",\"optional\":true,\"req\":\"^0.10\"},{\"name\":\"sync_wrapper\",\"req\":\"^1.0.0\"},{\"features\":[\"serde-human-readable\"],\"kind\":\"dev\",\"name\":\"time\",\"req\":\"^0.3\"},{\"features\":[\"time\"],\"name\":\"tokio\",\"optional\":true,\"package\":\"tokio\",\"req\":\"^1.25.0\"},{\"features\":[\"macros\",\"rt\",\"rt-multi-thread\",\"net\",\"test-util\"],\"kind\":\"dev\",\"name\":\"tokio\",\"package\":\"tokio\",\"req\":\"^1.25.0\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"name\":\"tokio-tungstenite\",\"optional\":true,\"req\":\"^0.24.0\"},{\"kind\":\"dev\",\"name\":\"tokio-tungstenite\",\"req\":\"^0.24.0\"},{\"default_features\":false,\"features\":[\"util\"],\"name\":\"tower\",\"req\":\"^0.5.1\"},{\"features\":[\"util\",\"timeout\",\"limit\",\"load-shed\",\"steer\",\"filter\"],\"kind\":\"dev\",\"name\":\"tower\",\"package\":\"tower\",\"req\":\"^0.5.1\"},{\"features\":[\"add-extension\",\"auth\",\"catch-panic\",\"compression-br\",\"compression-deflate\",\"compression-gzip\",\"cors\",\"decompression-br\",\"decompression-deflate\",\"decompression-gzip\",\"follow-redirect\",\"fs\",\"limit\",\"map-request-body\",\"map-response-body\",\"metrics\",\"normalize-path\",\"propagate-header\",\"redirect\",\"request-id\",\"sensitive-headers\",\"set-header\",\"set-status\",\"timeout\",\"trace\",\"util\",\"validate-request\"],\"name\":\"tower-http\",\"optional\":true,\"req\":\"^0.6.0\"},{\"features\":[\"add-extension\",\"auth\",\"catch-panic\",\"compression-br\",\"compression-deflate\",\"compression-gzip\",\"cors\",\"decompression-br\",\"decompression-deflate\",\"decompression-gzip\",\"follow-redirect\",\"fs\",\"limit\",\"map-request-body\",\"map-response-body\",\"metrics\",\"normalize-path\",\"propagate-header\",\"redirect\",\"request-id\",\"sensitive-headers\",\"set-header\",\"set-status\",\"timeout\",\"trace\",\"util\",\"validate-request\"],\"kind\":\"dev\",\"name\":\"tower-http\",\"req\":\"^0.6.0\"},{\"name\":\"tower-layer\",\"req\":\"^0.3.2\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1\"},{\"features\":[\"json\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"},{\"features\":[\"serde\",\"v4\"],\"kind\":\"dev\",\"name\":\"uuid\",\"req\":\"^1.0\"}],\"features\":{\"__private_docs\":[\"axum-core/__private_docs\",\"tower/full\",\"dep:tower-http\"],\"default\":[\"form\",\"http1\",\"json\",\"matched-path\",\"original-uri\",\"query\",\"tokio\",\"tower-log\",\"tracing\"],\"form\":[\"dep:serde_urlencoded\"],\"http1\":[\"dep:hyper\",\"hyper?/http1\",\"hyper-util?/http1\"],\"http2\":[\"dep:hyper\",\"hyper?/http2\",\"hyper-util?/http2\"],\"json\":[\"dep:serde_json\",\"dep:serde_path_to_error\"],\"macros\":[\"dep:axum-macros\"],\"matched-path\":[],\"multipart\":[\"dep:multer\"],\"original-uri\":[],\"query\":[\"dep:serde_urlencoded\"],\"tokio\":[\"dep:hyper-util\",\"dep:tokio\",\"tokio/net\",\"tokio/rt\",\"tower/make\",\"tokio/macros\"],\"tower-log\":[\"tower/log\"],\"tracing\":[\"dep:tracing\",\"axum-core/tracing\"],\"ws\":[\"dep:hyper\",\"tokio\",\"dep:tokio-tungstenite\",\"dep:sha1\",\"dep:base64\"]}}", + "backtrace_0.3.76": "{\"dependencies\":[{\"default_features\":false,\"name\":\"addr2line\",\"req\":\"^0.25.0\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"name\":\"cfg-if\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"cpp_demangle\",\"optional\":true,\"req\":\"^0.5.0\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.156\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"kind\":\"dev\",\"name\":\"libloading\",\"req\":\"^0.8\"},{\"default_features\":false,\"name\":\"miniz_oxide\",\"req\":\"^0.8\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"default_features\":false,\"features\":[\"read_core\",\"elf\",\"macho\",\"pe\",\"xcoff\",\"unaligned\",\"archive\"],\"name\":\"object\",\"req\":\"^0.37.0\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"name\":\"rustc-demangle\",\"req\":\"^0.1.24\"},{\"default_features\":false,\"name\":\"ruzstd\",\"optional\":true,\"req\":\"^0.8.1\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"windows-link\",\"req\":\"^0.2\",\"target\":\"cfg(any(windows, target_os = \\\"cygwin\\\"))\"}],\"features\":{\"coresymbolication\":[],\"dbghelp\":[],\"default\":[\"std\"],\"dl_iterate_phdr\":[],\"dladdr\":[],\"kernel32\":[],\"libunwind\":[],\"ruzstd\":[\"dep:ruzstd\"],\"serialize-serde\":[\"serde\"],\"std\":[],\"unix-backtrace\":[]}}", + "base64_0.13.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"=0.3.2\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.6.1\"},{\"kind\":\"dev\",\"name\":\"structopt\",\"req\":\"^0.3\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[]}}", + "base64_0.22.1": "{\"dependencies\":[{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"clap\",\"req\":\"^3.2.25\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rstest\",\"req\":\"^0.13.0\"},{\"kind\":\"dev\",\"name\":\"rstest_reuse\",\"req\":\"^0.6.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"strum\",\"req\":\"^0.25\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", + "bitflags_2.11.0": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"arbitrary\",\"req\":\"^1.0\"},{\"name\":\"bytemuck\",\"optional\":true,\"req\":\"^1.12\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"bytemuck\",\"req\":\"^1.12.2\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.228\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde_lib\",\"package\":\"serde\",\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.19\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.18\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"zerocopy\",\"req\":\"^0.8\"}],\"features\":{\"example_generated\":[],\"serde\":[\"serde_core\"],\"std\":[]}}", + "bitflags_2.9.1": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"arbitrary\",\"req\":\"^1.0\"},{\"name\":\"bytemuck\",\"optional\":true,\"req\":\"^1.12\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"bytemuck\",\"req\":\"^1.12.2\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.19\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.18\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"zerocopy\",\"req\":\"^0.8\"}],\"features\":{\"example_generated\":[],\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"],\"std\":[]}}", + "byteorder_1.5.0": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9.2\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"std\"],\"i128\":[],\"std\":[]}}", + "bytes_1.11.1": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"require-cas\"],\"name\":\"extra-platforms\",\"optional\":true,\"package\":\"portable-atomic\",\"req\":\"^1.3\"},{\"kind\":\"dev\",\"name\":\"loom\",\"req\":\"^0.7\",\"target\":\"cfg(loom)\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.60\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "castaway_0.2.4": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1\"},{\"name\":\"rustversion\",\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", + "cc_1.2.33": "{\"dependencies\":[{\"default_features\":false,\"name\":\"jobserver\",\"optional\":true,\"req\":\"^0.1.30\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.62\",\"target\":\"cfg(unix)\"},{\"name\":\"shlex\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{\"jobserver\":[],\"parallel\":[\"dep:libc\",\"dep:jobserver\"]}}", + "cfg-if_1.0.0": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"]}}", + "cfg-if_1.0.4": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"rustc-dep-of-std\":[\"core\"]}}", + "compact_str_0.9.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"borsh\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"castaway\",\"req\":\"^0.2.3\"},{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"cfg-if\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"diesel\",\"optional\":true,\"req\":\"^2\"},{\"name\":\"itoa\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"markup\",\"optional\":true,\"req\":\"^0.15\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"proptest\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"rkyv\",\"optional\":true,\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rkyv\",\"req\":\"^0.8.8\"},{\"name\":\"rustversion\",\"req\":\"^1\"},{\"name\":\"ryu\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"derive\",\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"features\":[\"union\"],\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"name\":\"sqlx\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"static_assertions\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"test-case\",\"req\":\"^3\"},{\"kind\":\"dev\",\"name\":\"test-strategy\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"borsh\":[\"dep:borsh\"],\"bytes\":[\"dep:bytes\"],\"default\":[\"std\"],\"diesel\":[\"dep:diesel\"],\"markup\":[\"dep:markup\"],\"proptest\":[\"dep:proptest\"],\"quickcheck\":[\"dep:quickcheck\"],\"rkyv\":[\"dep:rkyv\"],\"serde\":[\"dep:serde\"],\"smallvec\":[\"dep:smallvec\"],\"sqlx\":[\"dep:sqlx\",\"std\"],\"sqlx-mysql\":[\"sqlx\",\"sqlx/mysql\"],\"sqlx-postgres\":[\"sqlx\",\"sqlx/postgres\"],\"sqlx-sqlite\":[\"sqlx\",\"sqlx/sqlite\"],\"std\":[],\"zeroize\":[\"dep:zeroize\"]}}", + "crossbeam-deque_0.8.6": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-epoch\",\"req\":\"^0.9.17\"},{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"crossbeam-epoch/std\",\"crossbeam-utils/std\"]}}", + "crossbeam-epoch_0.9.18": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"name\":\"loom-crate\",\"optional\":true,\"package\":\"loom\",\"req\":\"^0.7.1\",\"target\":\"cfg(crossbeam_loom)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"loom\":[\"loom-crate\",\"crossbeam-utils/loom\"],\"nightly\":[\"crossbeam-utils/nightly\"],\"std\":[\"alloc\",\"crossbeam-utils/std\"]}}", + "crossbeam-utils_0.8.21": "{\"dependencies\":[{\"name\":\"loom\",\"optional\":true,\"req\":\"^0.7.1\",\"target\":\"cfg(crossbeam_loom)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"std\"],\"nightly\":[],\"std\":[]}}", + "darling_0.20.10": "{\"dependencies\":[{\"name\":\"darling_core\",\"req\":\"=0.20.10\"},{\"name\":\"darling_macro\",\"req\":\"=0.20.10\"},{\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.37\"},{\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.9\",\"target\":\"cfg(compiletests)\"},{\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.15\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.38\",\"target\":\"cfg(compiletests)\"}],\"features\":{\"default\":[\"suggestions\"],\"diagnostics\":[\"darling_core/diagnostics\"],\"suggestions\":[\"darling_core/suggestions\"]}}", + "darling_core_0.20.10": "{\"dependencies\":[{\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"name\":\"ident_case\",\"req\":\"^1.0.1\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.37\"},{\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"name\":\"strsim\",\"optional\":true,\"req\":\"^0.11.1\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"diagnostics\":[],\"suggestions\":[\"strsim\"]}}", + "darling_macro_0.20.10": "{\"dependencies\":[{\"name\":\"darling_core\",\"req\":\"=0.20.10\"},{\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{}}", + "dary_heap_0.3.7": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1\"}],\"features\":{\"extra\":[],\"unstable\":[],\"unstable_nightly\":[]}}", + "derive_builder_0.20.2": "{\"dependencies\":[{\"name\":\"derive_builder_macro\",\"req\":\"=0.20.2\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.4\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.38\",\"target\":\"cfg(compiletests)\"}],\"features\":{\"alloc\":[\"derive_builder_macro/alloc\"],\"clippy\":[\"derive_builder_macro/clippy\"],\"default\":[\"std\"],\"std\":[\"derive_builder_macro/lib_has_std\"]}}", + "derive_builder_core_0.20.2": "{\"dependencies\":[{\"name\":\"darling\",\"req\":\"^0.20.10\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.37\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"alloc\":[],\"clippy\":[],\"lib_has_std\":[]}}", + "derive_builder_macro_0.20.2": "{\"dependencies\":[{\"name\":\"derive_builder_core\",\"req\":\"=0.20.2\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"alloc\":[\"derive_builder_core/alloc\"],\"clippy\":[\"derive_builder_core/clippy\"],\"lib_has_std\":[\"derive_builder_core/lib_has_std\"]}}", + "either_1.13.0": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"use_std\"],\"use_std\":[]}}", + "either_1.15.0": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"alloc\",\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.95\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[],\"use_std\":[\"std\"]}}", + "equivalent_1.0.2": "{\"dependencies\":[],\"features\":{}}", + "errno_0.3.14": "{\"dependencies\":[{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(target_os=\\\"hermit\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(target_os=\\\"wasi\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(unix)\"},{\"features\":[\"Win32_Foundation\",\"Win32_System_Diagnostics_Debug\"],\"name\":\"windows-sys\",\"req\":\">=0.52, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"libc/std\"]}}", + "esaxx-rs_0.1.10": "{\"dependencies\":[{\"kind\":\"build\",\"name\":\"cc\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"}],\"features\":{\"cpp\":[\"cc\"],\"default\":[\"cpp\"]}}", + "fastrand_2.3.0": "{\"dependencies\":[{\"features\":[\"js\"],\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.2\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"getrandom\",\"req\":\"^0.2\"},{\"features\":[\"js\"],\"kind\":\"dev\",\"name\":\"getrandom\",\"req\":\"^0.2\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"wyhash\",\"req\":\"^0.5\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"js\":[\"std\",\"getrandom\"],\"std\":[\"alloc\"]}}", + "fixedbitset_0.4.2": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "fnv_1.0.7": "{\"dependencies\":[],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "foldhash_0.1.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"ahash\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"chrono\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"hashbrown\",\"req\":\"^0.14\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"uuid\",\"req\":\"^1.8\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "futures-channel_0.3.32": "{\"dependencies\":[{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-sink\",\"optional\":true,\"req\":\"^0.3.32\"}],\"features\":{\"alloc\":[\"futures-core/alloc\"],\"cfg-target-has-atomic\":[],\"default\":[\"std\"],\"sink\":[\"futures-sink\"],\"std\":[\"alloc\",\"futures-core/std\"],\"unstable\":[]}}", + "futures-core_0.3.32": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"require-cas\"],\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.3\"}],\"features\":{\"alloc\":[],\"cfg-target-has-atomic\":[],\"default\":[\"std\"],\"std\":[\"alloc\"],\"unstable\":[]}}", + "futures-sink_0.3.32": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", + "futures-task_0.3.32": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"cfg-target-has-atomic\":[],\"default\":[\"std\"],\"std\":[\"alloc\"],\"unstable\":[]}}", + "futures-util_0.3.32": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"std\"],\"name\":\"futures-channel\",\"optional\":true,\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3.32\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"futures-io\",\"optional\":true,\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-macro\",\"optional\":true,\"req\":\"=0.3.32\"},{\"default_features\":false,\"name\":\"futures-sink\",\"optional\":true,\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-task\",\"req\":\"^0.3.32\"},{\"name\":\"futures_01\",\"optional\":true,\"package\":\"futures\",\"req\":\"^0.1.25\"},{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.26\"},{\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.2\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.6\"},{\"default_features\":false,\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4.7\"},{\"name\":\"spin\",\"optional\":true,\"req\":\"^0.10.0\"},{\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^0.1.11\"},{\"name\":\"tokio-io\",\"optional\":true,\"req\":\"^0.1.9\"}],\"features\":{\"alloc\":[\"futures-core/alloc\",\"futures-task/alloc\",\"slab\"],\"async-await\":[],\"async-await-macro\":[\"async-await\",\"futures-macro\"],\"bilock\":[],\"cfg-target-has-atomic\":[],\"channel\":[\"std\",\"futures-channel\"],\"compat\":[\"std\",\"futures_01\",\"libc\"],\"default\":[\"std\",\"async-await\",\"async-await-macro\"],\"io\":[\"std\",\"futures-io\",\"memchr\"],\"io-compat\":[\"io\",\"compat\",\"tokio-io\",\"libc\"],\"portable-atomic\":[\"futures-core/portable-atomic\"],\"sink\":[\"futures-sink\"],\"std\":[\"alloc\",\"futures-core/std\",\"futures-task/std\",\"slab/std\"],\"unstable\":[\"futures-core/unstable\",\"futures-task/unstable\"],\"write-all-vectored\":[\"io\"]}}", + "getrandom_0.2.17": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(unix)\"},{\"default_features\":false,\"name\":\"wasi\",\"req\":\"^0.11\",\"target\":\"cfg(target_os = \\\"wasi\\\")\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.62\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.18\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"}],\"features\":{\"custom\":[],\"js\":[\"wasm-bindgen\",\"js-sys\"],\"linux_disable_fallback\":[],\"rdrand\":[],\"rustc-dep-of-std\":[\"compiler_builtins\",\"core\",\"libc/rustc-dep-of-std\",\"wasi/rustc-dep-of-std\"],\"std\":[],\"test-in-browser\":[]}}", + "getrandom_0.3.3": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3.77\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"), target_feature = \\\"atomics\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"netbsd\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"solaris\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"vxworks\\\")\"},{\"default_features\":false,\"name\":\"r-efi\",\"req\":\"^5.1\",\"target\":\"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\"},{\"default_features\":false,\"name\":\"wasi\",\"req\":\"^0.14\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.98\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"}],\"features\":{\"rustc-dep-of-std\":[\"dep:compiler_builtins\",\"dep:core\"],\"std\":[],\"wasm_js\":[\"dep:wasm-bindgen\",\"dep:js-sys\"]}}", + "getrandom_0.4.2": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3.77\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"), target_feature = \\\"atomics\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"netbsd\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"solaris\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"vxworks\\\")\"},{\"default_features\":false,\"name\":\"r-efi\",\"req\":\"^6\",\"target\":\"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\"},{\"name\":\"rand_core\",\"optional\":true,\"req\":\"^0.10.0\"},{\"default_features\":false,\"name\":\"wasip2\",\"req\":\"^1\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\"},{\"name\":\"wasip3\",\"req\":\"^0.4\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p3\\\"))\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.98\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"}],\"features\":{\"std\":[],\"sys_rng\":[\"dep:rand_core\"],\"wasm_js\":[\"dep:wasm-bindgen\",\"dep:js-sys\"]}}", + "gimli_0.32.3": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"fallible-iterator\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.0.0\"},{\"default_features\":false,\"name\":\"stable_deref_trait\",\"optional\":true,\"req\":\"^1.1.0\"},{\"kind\":\"dev\",\"name\":\"test-assembler\",\"req\":\"^0.1.3\"}],\"features\":{\"default\":[\"read-all\",\"write\"],\"endian-reader\":[\"read\",\"dep:stable_deref_trait\"],\"fallible-iterator\":[\"dep:fallible-iterator\"],\"read\":[\"read-core\"],\"read-all\":[\"read\",\"std\",\"fallible-iterator\",\"endian-reader\"],\"read-core\":[],\"rustc-dep-of-std\":[\"dep:core\",\"dep:alloc\"],\"std\":[\"fallible-iterator?/std\",\"stable_deref_trait?/std\"],\"write\":[\"dep:indexmap\"]}}", + "h2_0.4.6": "{\"dependencies\":[{\"name\":\"atomic-waker\",\"req\":\"^1.0.0\"},{\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.10\"},{\"name\":\"fnv\",\"req\":\"^1.0.5\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-sink\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.3\"},{\"name\":\"http\",\"req\":\"^1\"},{\"features\":[\"std\"],\"name\":\"indexmap\",\"req\":\"^2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.4\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.0\"},{\"name\":\"slab\",\"req\":\"^0.4.2\"},{\"features\":[\"io-util\"],\"name\":\"tokio\",\"req\":\"^1\"},{\"features\":[\"rt-multi-thread\",\"macros\",\"sync\",\"net\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tokio-rustls\",\"req\":\"^0.26\"},{\"features\":[\"codec\",\"io\"],\"name\":\"tokio-util\",\"req\":\"^0.7.1\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"req\":\"^0.1.35\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\"},{\"kind\":\"dev\",\"name\":\"webpki-roots\",\"req\":\"^0.26\"}],\"features\":{\"stream\":[],\"unstable\":[]}}", + "hashbrown_0.12.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"ahash\",\"optional\":true,\"req\":\"^0.7.0\"},{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"bumpalo\",\"optional\":true,\"req\":\"^3.5.0\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.3\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.25\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"ahash-compile-time-rng\":[\"ahash/compile-time-rng\"],\"default\":[\"ahash\",\"inline-more\"],\"inline-more\":[],\"nightly\":[],\"raw\":[],\"rustc-dep-of-std\":[\"nightly\",\"core\",\"compiler_builtins\",\"alloc\",\"rustc-internal-api\"],\"rustc-internal-api\":[]}}", + "hashbrown_0.15.5": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"allocator-api2\",\"optional\":true,\"req\":\"^0.2.9\"},{\"features\":[\"allocator-api2\"],\"kind\":\"dev\",\"name\":\"bumpalo\",\"req\":\"^3.13.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.1\"},{\"default_features\":false,\"name\":\"equivalent\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"default_features\":false,\"name\":\"foldhash\",\"optional\":true,\"req\":\"^0.1.2\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9.0\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.2\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.2\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.25\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"default-hasher\",\"inline-more\",\"allocator-api2\",\"equivalent\",\"raw-entry\"],\"default-hasher\":[\"dep:foldhash\"],\"inline-more\":[],\"nightly\":[\"bumpalo/allocator_api\"],\"raw-entry\":[],\"rustc-dep-of-std\":[\"nightly\",\"core\",\"alloc\",\"rustc-internal-api\"],\"rustc-internal-api\":[]}}", + "hashbrown_0.16.1": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"allocator-api2\",\"optional\":true,\"req\":\"^0.2.9\"},{\"features\":[\"allocator-api2\"],\"kind\":\"dev\",\"name\":\"bumpalo\",\"req\":\"^3.13.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"equivalent\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"default_features\":false,\"name\":\"foldhash\",\"optional\":true,\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.155\",\"target\":\"cfg(unix)\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9.0\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.9.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.2\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.221\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"default-hasher\",\"inline-more\",\"allocator-api2\",\"equivalent\",\"raw-entry\"],\"default-hasher\":[\"dep:foldhash\"],\"inline-more\":[],\"nightly\":[\"foldhash?/nightly\",\"bumpalo/allocator_api\"],\"raw-entry\":[],\"rustc-dep-of-std\":[\"nightly\",\"core\",\"alloc\",\"rustc-internal-api\"],\"rustc-internal-api\":[],\"serde\":[\"dep:serde_core\",\"dep:serde\"]}}", + "heck_0.5.0": "{\"dependencies\":[],\"features\":{}}", + "http-body-util_0.1.3": "{\"dependencies\":[{\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"http-body\",\"req\":\"^1\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"macros\",\"rt\",\"sync\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"}],\"features\":{\"channel\":[\"dep:tokio\"],\"default\":[],\"full\":[\"channel\"]}}", + "http-body_1.0.1": "{\"dependencies\":[{\"name\":\"bytes\",\"req\":\"^1\"},{\"name\":\"http\",\"req\":\"^1\"}],\"features\":{}}", + "http_1.4.0": "{\"dependencies\":[{\"name\":\"bytes\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"name\":\"itoa\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.0\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "httparse_1.10.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.5\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "httpdate_1.0.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"}],\"features\":{}}", + "hyper-timeout_0.5.2": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"name\":\"hyper\",\"req\":\"^1.1\"},{\"features\":[\"http1\"],\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^1.1\"},{\"kind\":\"dev\",\"name\":\"hyper-tls\",\"req\":\"^0.6\"},{\"features\":[\"client-legacy\",\"http1\"],\"name\":\"hyper-util\",\"req\":\"^0.1.10\"},{\"features\":[\"client-legacy\",\"http1\",\"server\",\"server-graceful\"],\"kind\":\"dev\",\"name\":\"hyper-util\",\"req\":\"^0.1.10\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"name\":\"tokio\",\"req\":\"^1.35\"},{\"features\":[\"io-std\",\"io-util\",\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.35\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"}],\"features\":{}}", + "hyper-util_0.1.20": "{\"dependencies\":[{\"name\":\"base64\",\"optional\":true,\"req\":\"^0.22\"},{\"name\":\"bytes\",\"req\":\"^1.7.1\"},{\"kind\":\"dev\",\"name\":\"bytes\",\"req\":\"^1\"},{\"name\":\"futures-channel\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.16\"},{\"default_features\":false,\"features\":[\"alloc\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3.16\"},{\"name\":\"http\",\"req\":\"^1.0\"},{\"name\":\"http-body\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1.0\"},{\"name\":\"hyper\",\"req\":\"^1.8.0\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^1.4.0\"},{\"name\":\"ipnet\",\"optional\":true,\"req\":\"^2.9\"},{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2\"},{\"name\":\"percent-encoding\",\"optional\":true,\"req\":\"^2.3\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.4\"},{\"kind\":\"dev\",\"name\":\"pnet_datalink\",\"req\":\"^0.35.0\",\"target\":\"cfg(any(target_os = \\\"linux\\\", target_os = \\\"macos\\\"))\"},{\"kind\":\"dev\",\"name\":\"pretty_env_logger\",\"req\":\"^0.5\"},{\"features\":[\"all\"],\"name\":\"socket2\",\"optional\":true,\"req\":\">=0.5.9, <0.7\"},{\"name\":\"system-configuration\",\"optional\":true,\"req\":\"^0.7\",\"target\":\"cfg(target_os = \\\"macos\\\")\"},{\"default_features\":false,\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"macros\",\"test-util\",\"signal\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"name\":\"tower-layer\",\"optional\":true,\"req\":\"^0.3\"},{\"name\":\"tower-service\",\"optional\":true,\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"tower-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"windows-registry\",\"optional\":true,\"req\":\">=0.3, <0.7\",\"target\":\"cfg(windows)\"}],\"features\":{\"__internal_happy_eyeballs_tests\":[],\"client\":[\"hyper/client\",\"tokio/net\",\"dep:tracing\",\"dep:futures-channel\",\"dep:tower-service\"],\"client-legacy\":[\"client\",\"dep:socket2\",\"tokio/sync\",\"dep:libc\",\"dep:futures-util\"],\"client-pool\":[\"client\",\"dep:futures-util\",\"dep:tower-layer\"],\"client-proxy\":[\"client\",\"dep:base64\",\"dep:ipnet\",\"dep:percent-encoding\"],\"client-proxy-system\":[\"dep:system-configuration\",\"dep:windows-registry\"],\"default\":[],\"full\":[\"client\",\"client-legacy\",\"client-pool\",\"client-proxy\",\"client-proxy-system\",\"server\",\"server-auto\",\"server-graceful\",\"service\",\"http1\",\"http2\",\"tokio\",\"tracing\"],\"http1\":[\"hyper/http1\"],\"http2\":[\"hyper/http2\"],\"server\":[\"hyper/server\"],\"server-auto\":[\"server\",\"http1\",\"http2\"],\"server-graceful\":[\"server\",\"tokio/sync\"],\"service\":[\"dep:tower-service\"],\"tokio\":[\"dep:tokio\",\"tokio/rt\",\"tokio/time\"],\"tracing\":[\"dep:tracing\"]}}", + "hyper_1.8.1": "{\"dependencies\":[{\"name\":\"atomic-waker\",\"optional\":true,\"req\":\"^1.1.2\"},{\"name\":\"bytes\",\"req\":\"^1.2\"},{\"kind\":\"dev\",\"name\":\"form_urlencoded\",\"req\":\"^1\"},{\"name\":\"futures-channel\",\"optional\":true,\"req\":\"^0.3\"},{\"features\":[\"sink\"],\"kind\":\"dev\",\"name\":\"futures-channel\",\"req\":\"^0.3\"},{\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3.31\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\",\"sink\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"h2\",\"optional\":true,\"req\":\"^0.4.2\"},{\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"http-body\",\"req\":\"^1\"},{\"name\":\"http-body-util\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"name\":\"httparse\",\"optional\":true,\"req\":\"^1.9\"},{\"name\":\"httpdate\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"itoa\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"pin-project-lite\",\"optional\":true,\"req\":\"^0.2.4\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.4\"},{\"name\":\"pin-utils\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"pretty_env_logger\",\"req\":\"^0.5\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"features\":[\"const_generics\",\"const_new\"],\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1.12\"},{\"kind\":\"dev\",\"name\":\"spmc\",\"req\":\"^0.3\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"req\":\"^1\"},{\"features\":[\"fs\",\"macros\",\"net\",\"io-std\",\"io-util\",\"rt\",\"rt-multi-thread\",\"sync\",\"time\",\"test-util\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"tokio-util\",\"req\":\"^0.7.10\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"want\",\"optional\":true,\"req\":\"^0.3\"}],\"features\":{\"capi\":[],\"client\":[\"dep:want\",\"dep:pin-project-lite\",\"dep:smallvec\"],\"default\":[],\"ffi\":[\"dep:http-body-util\",\"dep:futures-util\"],\"full\":[\"client\",\"http1\",\"http2\",\"server\"],\"http1\":[\"dep:atomic-waker\",\"dep:futures-channel\",\"dep:futures-core\",\"dep:httparse\",\"dep:itoa\",\"dep:pin-utils\"],\"http2\":[\"dep:futures-channel\",\"dep:futures-core\",\"dep:h2\"],\"nightly\":[],\"server\":[\"dep:httpdate\",\"dep:pin-project-lite\",\"dep:smallvec\"],\"tracing\":[\"dep:tracing\"]}}", + "id-arena_2.3.0": "{\"dependencies\":[{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.0.3\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "ident_case_1.0.1": "{\"dependencies\":[],\"features\":{}}", + "indexmap_1.9.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"build\",\"name\":\"autocfg\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2.1\"},{\"default_features\":false,\"features\":[\"raw\"],\"name\":\"hashbrown\",\"req\":\"^0.12\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.3\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.4.1\"},{\"name\":\"rustc-rayon\",\"optional\":true,\"package\":\"rustc-rayon\",\"req\":\"^0.5\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0\"}],\"features\":{\"serde-1\":[\"serde\"],\"std\":[],\"test_debug\":[],\"test_low_transition_point\":[]}}", + "indexmap_2.13.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"borsh\",\"optional\":true,\"req\":\"^1.2\"},{\"default_features\":false,\"name\":\"equivalent\",\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"fastrand\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"hashbrown\",\"req\":\"^0.16.1\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.14\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.9\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"},{\"default_features\":false,\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.220\"},{\"default_features\":false,\"name\":\"sval\",\"optional\":true,\"req\":\"^2\"}],\"features\":{\"default\":[\"std\"],\"serde\":[\"dep:serde_core\",\"dep:serde\"],\"std\":[],\"test_debug\":[]}}", + "itertools_0.13.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"permutohedron\",\"req\":\"^0.2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"use_std\"],\"use_alloc\":[],\"use_std\":[\"use_alloc\",\"either/use_std\"]}}", + "itertools_0.14.0": "{\"dependencies\":[{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"permutohedron\",\"req\":\"^0.2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"use_std\"],\"use_alloc\":[],\"use_std\":[\"use_alloc\",\"either/use_std\"]}}", + "itoa_1.0.14": "{\"dependencies\":[{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"}],\"features\":{}}", + "itoa_1.0.17": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\",\"target\":\"cfg(not(miri))\"},{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"}],\"features\":{}}", + "leb128fmt_0.1.0": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[]}}", + "libc_0.2.169": "{\"dependencies\":[{\"name\":\"rustc-std-workspace-core\",\"optional\":true,\"req\":\"^1.0.0\"}],\"features\":{\"align\":[],\"const-extern-fn\":[],\"default\":[\"std\"],\"extra_traits\":[],\"rustc-dep-of-std\":[\"align\",\"rustc-std-workspace-core\"],\"std\":[],\"use_std\":[\"std\"]}}", + "libc_0.2.183": "{\"dependencies\":[{\"name\":\"rustc-std-workspace-core\",\"optional\":true,\"req\":\"^1.0.1\"}],\"features\":{\"align\":[],\"const-extern-fn\":[],\"default\":[\"std\"],\"extra_traits\":[],\"rustc-dep-of-std\":[\"align\",\"rustc-std-workspace-core\"],\"std\":[],\"use_std\":[\"std\"]}}", + "linux-raw-sys_0.12.1": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.100\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1.0\"}],\"features\":{\"auxvec\":[],\"bootparam\":[],\"btrfs\":[],\"default\":[\"std\",\"general\",\"errno\"],\"elf\":[],\"elf_uapi\":[],\"errno\":[],\"general\":[],\"if_arp\":[],\"if_ether\":[],\"if_packet\":[],\"if_tun\":[],\"image\":[],\"io_uring\":[],\"ioctl\":[],\"landlock\":[],\"loop_device\":[],\"mempolicy\":[],\"net\":[],\"netlink\":[],\"no_std\":[],\"prctl\":[],\"ptrace\":[],\"rustc-dep-of-std\":[\"core\",\"no_std\"],\"std\":[],\"system\":[],\"vm_sockets\":[],\"xdp\":[]}}", + "lock_api_0.4.14": "{\"dependencies\":[{\"name\":\"owning_ref\",\"optional\":true,\"req\":\"^0.4.1\"},{\"default_features\":false,\"name\":\"scopeguard\",\"req\":\"^1.1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.126\"}],\"features\":{\"arc_lock\":[],\"atomic_usize\":[],\"default\":[\"atomic_usize\"],\"nightly\":[]}}", + "log_0.4.22": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.63\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"sval\",\"optional\":true,\"req\":\"^2.1\"},{\"kind\":\"dev\",\"name\":\"sval\",\"req\":\"^2.1\"},{\"kind\":\"dev\",\"name\":\"sval_derive\",\"req\":\"^2.1\"},{\"default_features\":false,\"name\":\"sval_ref\",\"optional\":true,\"req\":\"^2.1\"},{\"default_features\":false,\"features\":[\"inline-i128\"],\"name\":\"value-bag\",\"optional\":true,\"req\":\"^1.7\"},{\"features\":[\"test\"],\"kind\":\"dev\",\"name\":\"value-bag\",\"req\":\"^1.7\"}],\"features\":{\"kv\":[],\"kv_serde\":[\"kv_std\",\"value-bag/serde\",\"serde\"],\"kv_std\":[\"std\",\"kv\",\"value-bag/error\"],\"kv_sval\":[\"kv\",\"value-bag/sval\",\"sval\",\"sval_ref\"],\"kv_unstable\":[\"kv\",\"value-bag\"],\"kv_unstable_serde\":[\"kv_serde\",\"kv_unstable_std\"],\"kv_unstable_std\":[\"kv_std\",\"kv_unstable\"],\"kv_unstable_sval\":[\"kv_sval\",\"kv_unstable\"],\"max_level_debug\":[],\"max_level_error\":[],\"max_level_info\":[],\"max_level_off\":[],\"max_level_trace\":[],\"max_level_warn\":[],\"release_max_level_debug\":[],\"release_max_level_error\":[],\"release_max_level_info\":[],\"release_max_level_off\":[],\"release_max_level_trace\":[],\"release_max_level_warn\":[],\"std\":[]}}", + "log_0.4.29": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.63\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"sval\",\"optional\":true,\"req\":\"^2.16\"},{\"kind\":\"dev\",\"name\":\"sval\",\"req\":\"^2.16\"},{\"kind\":\"dev\",\"name\":\"sval_derive\",\"req\":\"^2.16\"},{\"default_features\":false,\"name\":\"sval_ref\",\"optional\":true,\"req\":\"^2.16\"},{\"default_features\":false,\"features\":[\"inline-i128\"],\"name\":\"value-bag\",\"optional\":true,\"req\":\"^1.12\"},{\"features\":[\"test\"],\"kind\":\"dev\",\"name\":\"value-bag\",\"req\":\"^1.12\"}],\"features\":{\"kv\":[],\"kv_serde\":[\"kv_std\",\"value-bag/serde\",\"serde\"],\"kv_std\":[\"std\",\"kv\",\"value-bag/error\"],\"kv_sval\":[\"kv\",\"value-bag/sval\",\"sval\",\"sval_ref\"],\"kv_unstable\":[\"kv\",\"value-bag\"],\"kv_unstable_serde\":[\"kv_serde\",\"kv_unstable_std\"],\"kv_unstable_std\":[\"kv_std\",\"kv_unstable\"],\"kv_unstable_sval\":[\"kv_sval\",\"kv_unstable\"],\"max_level_debug\":[],\"max_level_error\":[],\"max_level_info\":[],\"max_level_off\":[],\"max_level_trace\":[],\"max_level_warn\":[],\"release_max_level_debug\":[],\"release_max_level_error\":[],\"release_max_level_info\":[],\"release_max_level_off\":[],\"release_max_level_trace\":[],\"release_max_level_warn\":[],\"serde\":[\"serde_core\"],\"std\":[]}}", + "macro_rules_attribute-proc_macro_0.2.0": "{\"dependencies\":[],\"features\":{\"verbose-expansions\":[]}}", + "macro_rules_attribute_0.2.0": "{\"dependencies\":[{\"name\":\"macro_rules_attribute-proc_macro\",\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.10.0\"},{\"name\":\"paste\",\"req\":\"^1.0.7\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.8\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.136\"}],\"features\":{\"better-docs\":[],\"default\":[],\"verbose-expansions\":[\"macro_rules_attribute-proc_macro/verbose-expansions\"]}}", + "matchit_0.7.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"actix-router\",\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.4\"},{\"kind\":\"dev\",\"name\":\"gonzales\",\"req\":\"^0.0.3-beta\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^0.14\"},{\"kind\":\"dev\",\"name\":\"path-tree\",\"req\":\"^0.2.2\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.5.4\"},{\"kind\":\"dev\",\"name\":\"route-recognizer\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"routefinder\",\"req\":\"^0.5.2\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"features\":[\"make\",\"util\"],\"kind\":\"dev\",\"name\":\"tower\",\"req\":\"^0.4\"}],\"features\":{\"__test_helpers\":[],\"default\":[]}}", + "memchr_2.7.4": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.20\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"libc\":[],\"logging\":[\"dep:log\"],\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"],\"std\":[\"alloc\"],\"use_std\":[\"std\"]}}", + "memchr_2.8.0": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.20\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"libc\":[],\"logging\":[\"dep:log\"],\"rustc-dep-of-std\":[\"core\"],\"std\":[\"alloc\"],\"use_std\":[\"std\"]}}", + "mime_0.3.17": "{\"dependencies\":[],\"features\":{}}", + "minimal-lexical_0.2.1": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"compact\":[],\"default\":[\"std\"],\"lint\":[],\"nightly\":[],\"std\":[]}}", + "miniz_oxide_0.8.9": "{\"dependencies\":[{\"default_features\":false,\"name\":\"adler2\",\"req\":\"^2.0\"},{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"simd-adler32\",\"optional\":true,\"req\":\"^0.3.3\"}],\"features\":{\"block-boundary\":[],\"default\":[\"with-alloc\"],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"adler2/rustc-dep-of-std\"],\"simd\":[\"simd-adler32\"],\"std\":[],\"with-alloc\":[]}}", + "mio_1.1.1": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"libc\",\"req\":\"^0.2.178\",\"target\":\"cfg(target_os = \\\"hermit\\\")\"},{\"name\":\"libc\",\"req\":\"^0.2.178\",\"target\":\"cfg(target_os = \\\"wasi\\\")\"},{\"name\":\"libc\",\"req\":\"^0.2.178\",\"target\":\"cfg(unix)\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.8\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9\"},{\"name\":\"wasi\",\"req\":\"^0.11.0\",\"target\":\"cfg(target_os = \\\"wasi\\\")\"},{\"features\":[\"Wdk_Foundation\",\"Wdk_Storage_FileSystem\",\"Wdk_System_IO\",\"Win32_Foundation\",\"Win32_Networking_WinSock\",\"Win32_Storage_FileSystem\",\"Win32_Security\",\"Win32_System_IO\",\"Win32_System_WindowsProgramming\"],\"name\":\"windows-sys\",\"req\":\"^0.61\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[\"log\"],\"net\":[],\"os-ext\":[\"os-poll\",\"windows-sys/Win32_System_Pipes\",\"windows-sys/Win32_Security\"],\"os-poll\":[]}}", + "monostate-impl_0.1.13": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"default_features\":false,\"features\":[\"parsing\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0.59\"}],\"features\":{}}", + "monostate_0.1.13": "{\"dependencies\":[{\"name\":\"monostate-impl\",\"req\":\"=0.1.13\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.166\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.99\"}],\"features\":{}}", + "multimap_0.10.1": "{\"dependencies\":[{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"serde_impl\"],\"serde_impl\":[\"serde\"]}}", + "nom_7.1.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2.3\"},{\"default_features\":false,\"name\":\"minimal-lexical\",\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.0.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"docsrs\":[],\"std\":[\"alloc\",\"memchr/std\",\"minimal-lexical/std\"]}}", + "object_0.37.3": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"crc32fast\",\"optional\":true,\"req\":\"^1.2\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"default-hasher\"],\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.15.0\"},{\"default_features\":false,\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.0\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2.4.1\"},{\"name\":\"ruzstd\",\"optional\":true,\"req\":\"^0.8.1\"},{\"default_features\":false,\"name\":\"wasmparser\",\"optional\":true,\"req\":\"^0.236.0\"}],\"features\":{\"all\":[\"read\",\"write\",\"build\",\"std\",\"compression\",\"wasm\"],\"archive\":[],\"build\":[\"build_core\",\"write_std\",\"elf\"],\"build_core\":[\"read_core\",\"write_core\"],\"cargo-all\":[],\"coff\":[],\"compression\":[\"dep:flate2\",\"dep:ruzstd\",\"std\"],\"default\":[\"read\",\"compression\"],\"doc\":[\"read_core\",\"write_std\",\"build_core\",\"std\",\"compression\",\"archive\",\"coff\",\"elf\",\"macho\",\"pe\",\"wasm\",\"xcoff\"],\"elf\":[],\"macho\":[],\"pe\":[\"coff\"],\"read\":[\"read_core\",\"archive\",\"coff\",\"elf\",\"macho\",\"pe\",\"xcoff\",\"unaligned\"],\"read_core\":[],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"memchr/rustc-dep-of-std\"],\"std\":[\"memchr/std\"],\"unaligned\":[],\"unstable\":[],\"unstable-all\":[\"all\",\"unstable\"],\"wasm\":[\"dep:wasmparser\"],\"write\":[\"write_std\",\"coff\",\"elf\",\"macho\",\"pe\",\"xcoff\"],\"write_core\":[\"dep:crc32fast\",\"dep:indexmap\",\"dep:hashbrown\"],\"write_std\":[\"write_core\",\"std\",\"indexmap?/std\",\"crc32fast?/std\"],\"xcoff\":[]}}", + "once_cell_1.20.2": "{\"dependencies\":[{\"name\":\"critical-section\",\"optional\":true,\"req\":\"^1.1.3\"},{\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"critical-section\",\"req\":\"^1.1.3\"},{\"default_features\":false,\"name\":\"parking_lot_core\",\"optional\":true,\"req\":\"^0.9.10\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.10.6\"}],\"features\":{\"alloc\":[\"race\"],\"atomic-polyfill\":[\"critical-section\"],\"critical-section\":[\"dep:critical-section\",\"portable-atomic\"],\"default\":[\"std\"],\"parking_lot\":[\"dep:parking_lot_core\"],\"portable-atomic\":[\"dep:portable-atomic\"],\"race\":[],\"std\":[\"alloc\"],\"unstable\":[]}}", + "once_cell_1.21.4": "{\"dependencies\":[{\"name\":\"critical-section\",\"optional\":true,\"req\":\"^1.1.3\"},{\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"critical-section\",\"req\":\"^1.1.3\"},{\"default_features\":false,\"name\":\"parking_lot_core\",\"optional\":true,\"req\":\"^0.9.10\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.10.6\"}],\"features\":{\"alloc\":[\"race\"],\"atomic-polyfill\":[\"critical-section\"],\"critical-section\":[\"dep:critical-section\",\"portable-atomic\"],\"default\":[\"std\"],\"parking_lot\":[\"dep:parking_lot_core\"],\"portable-atomic\":[\"dep:portable-atomic\"],\"race\":[],\"std\":[\"alloc\"],\"unstable\":[]}}", + "onig_6.5.1": "{\"dependencies\":[{\"name\":\"bitflags\",\"req\":\"^2.4.0\"},{\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(windows)\"},{\"name\":\"once_cell\",\"req\":\"^1.12\"},{\"default_features\":false,\"name\":\"onig_sys\",\"req\":\"^69.9.1\"}],\"features\":{\"default\":[\"generate\"],\"generate\":[\"onig_sys/generate\"],\"posix-api\":[\"onig_sys/posix-api\"],\"print-debug\":[\"onig_sys/print-debug\"],\"std-pattern\":[]}}", + "onig_sys_69.9.1": "{\"dependencies\":[{\"features\":[\"runtime\"],\"kind\":\"build\",\"name\":\"bindgen\",\"optional\":true,\"req\":\"^0.71\"},{\"kind\":\"build\",\"name\":\"cc\",\"req\":\"^1.0\"},{\"kind\":\"build\",\"name\":\"pkg-config\",\"req\":\"^0.3.16\"}],\"features\":{\"default\":[\"generate\"],\"generate\":[\"bindgen\"],\"posix-api\":[],\"print-debug\":[]}}", + "parking_lot_0.12.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.3.3\"},{\"name\":\"lock_api\",\"req\":\"^0.4.14\"},{\"name\":\"parking_lot_core\",\"req\":\"^0.9.12\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.3\"}],\"features\":{\"arc_lock\":[\"lock_api/arc_lock\"],\"deadlock_detection\":[\"parking_lot_core/deadlock_detection\"],\"default\":[],\"hardware-lock-elision\":[],\"nightly\":[\"parking_lot_core/nightly\",\"lock_api/nightly\"],\"owning_ref\":[\"lock_api/owning_ref\"],\"send_guard\":[],\"serde\":[\"lock_api/serde\"]}}", + "parking_lot_core_0.9.12": "{\"dependencies\":[{\"name\":\"backtrace\",\"optional\":true,\"req\":\"^0.3.60\"},{\"name\":\"cfg-if\",\"req\":\"^1.0.0\"},{\"name\":\"libc\",\"req\":\"^0.2.95\",\"target\":\"cfg(unix)\"},{\"name\":\"petgraph\",\"optional\":true,\"req\":\"^0.6.0\"},{\"name\":\"redox_syscall\",\"req\":\"^0.5\",\"target\":\"cfg(target_os = \\\"redox\\\")\"},{\"name\":\"smallvec\",\"req\":\"^1.6.1\"},{\"name\":\"windows-link\",\"req\":\"^0.2.0\",\"target\":\"cfg(windows)\"}],\"features\":{\"deadlock_detection\":[\"petgraph\",\"backtrace\"],\"nightly\":[]}}", + "paste_1.0.15": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"paste-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", + "percent-encoding_2.3.2": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", + "petgraph_0.6.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"ahash\",\"req\":\"^0.7.2\"},{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.3.3\"},{\"kind\":\"dev\",\"name\":\"defmac\",\"req\":\"^0.2.1\"},{\"default_features\":false,\"name\":\"fixedbitset\",\"req\":\"^0.4.0\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2.1\"},{\"name\":\"indexmap\",\"req\":\"^2.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.12.1\"},{\"kind\":\"dev\",\"name\":\"odds\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.5.5\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.5.3\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1.0\"}],\"features\":{\"all\":[\"unstable\",\"quickcheck\",\"matrix_graph\",\"stable_graph\",\"graphmap\",\"rayon\"],\"default\":[\"graphmap\",\"stable_graph\",\"matrix_graph\"],\"generate\":[],\"graphmap\":[],\"matrix_graph\":[],\"rayon\":[\"dep:rayon\",\"indexmap/rayon\"],\"serde-1\":[\"serde\",\"serde_derive\"],\"stable_graph\":[],\"unstable\":[\"generate\"]}}", + "pin-project-internal_1.1.11": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1.0.25\"},{\"default_features\":false,\"features\":[\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\",\"full\",\"visit-mut\"],\"name\":\"syn\",\"req\":\"^2.0.1\"}],\"features\":{}}", + "pin-project-lite_0.2.17": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1\"}],\"features\":{}}", + "pin-project_1.1.11": "{\"dependencies\":[{\"name\":\"pin-project-internal\",\"req\":\"=1.1.11\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1\"}],\"features\":{}}", + "pin-utils_0.1.0": "{\"dependencies\":[],\"features\":{}}", + "pkg-config_0.3.32": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1\"}],\"features\":{}}", + "ppv-lite86_0.2.20": "{\"dependencies\":[{\"features\":[\"simd\",\"derive\"],\"name\":\"zerocopy\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"std\"],\"no_simd\":[],\"simd\":[],\"std\":[]}}", + "ppv-lite86_0.2.21": "{\"dependencies\":[{\"features\":[\"simd\"],\"name\":\"zerocopy\",\"req\":\"^0.8.23\"}],\"features\":{\"default\":[\"std\"],\"no_simd\":[],\"simd\":[],\"std\":[]}}", + "prettyplease_0.2.37": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"default_features\":false,\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.105\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"extra-traits\",\"parsing\",\"printing\",\"visit-mut\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.105\"}],\"features\":{\"verbatim\":[\"syn/parsing\"]}}", + "proc-macro2_1.0.106": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4\"},{\"name\":\"unicode-ident\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"proc-macro\"],\"nightly\":[],\"proc-macro\":[],\"span-locations\":[]}}", + "proc-macro2_1.0.92": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4\"},{\"name\":\"unicode-ident\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"proc-macro\"],\"nightly\":[],\"proc-macro\":[],\"span-locations\":[]}}", + "prost-build_0.13.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"heck\",\"req\":\">=0.4, <=0.5\"},{\"default_features\":false,\"features\":[\"use_alloc\"],\"name\":\"itertools\",\"req\":\">=0.10, <=0.13\"},{\"name\":\"log\",\"req\":\"^0.4.4\"},{\"default_features\":false,\"name\":\"multimap\",\"req\":\">=0.8, <=0.10\"},{\"name\":\"once_cell\",\"req\":\"^1.17.1\"},{\"default_features\":false,\"name\":\"petgraph\",\"req\":\"^0.6\"},{\"name\":\"prettyplease\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"name\":\"prost\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-types\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"pulldown-cmark\",\"optional\":true,\"req\":\"^0.9.1\"},{\"name\":\"pulldown-cmark-to-cmark\",\"optional\":true,\"req\":\"^10.0.1\"},{\"default_features\":false,\"features\":[\"std\",\"unicode-bool\"],\"name\":\"regex\",\"req\":\"^1.8.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"optional\":true,\"req\":\"^2\"},{\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{\"cleanup-markdown\":[\"dep:pulldown-cmark\",\"dep:pulldown-cmark-to-cmark\"],\"default\":[\"format\"],\"format\":[\"dep:prettyplease\",\"dep:syn\"]}}", + "prost-derive_0.13.1": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.1\"},{\"name\":\"itertools\",\"req\":\">=0.10.1, <=0.13\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2\"}],\"features\":{}}", + "prost-types_0.13.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"prost-derive\"],\"name\":\"prost\",\"req\":\"^0.13.1\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"prost/std\"]}}", + "prost_0.13.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"name\":\"prost-derive\",\"optional\":true,\"req\":\"^0.13.1\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"derive\",\"std\"],\"derive\":[\"dep:prost-derive\"],\"no-recursion-limit\":[],\"prost-derive\":[\"derive\"],\"std\":[]}}", + "protoc-gen-prost_0.4.0": "{\"dependencies\":[{\"name\":\"once_cell\",\"req\":\"^1.10.0\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"prost\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-build\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-types\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"regex\",\"req\":\"^1.5.5\"}],\"features\":{}}", + "protoc-gen-tonic_0.4.1": "{\"dependencies\":[{\"name\":\"heck\",\"req\":\"^0.5.0\"},{\"name\":\"prettyplease\",\"req\":\"^0.2.9\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"prost\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-build\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-types\",\"req\":\"^0.13.1\"},{\"name\":\"protoc-gen-prost\",\"req\":\"^0.4.0\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"regex\",\"req\":\"^1.5.5\"},{\"features\":[\"parsing\",\"full\"],\"name\":\"syn\",\"req\":\"^2.0.22\"},{\"name\":\"tonic-build\",\"req\":\"^0.12.0\"}],\"features\":{}}", + "quote_1.0.37": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.66\"}],\"features\":{\"default\":[\"proc-macro\"],\"proc-macro\":[\"proc-macro2/proc-macro\"]}}", + "quote_1.0.45": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"}],\"features\":{\"default\":[\"proc-macro\"],\"proc-macro\":[\"proc-macro2/proc-macro\"]}}", + "r-efi_5.3.0": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"efiapi\":[],\"examples\":[\"native\"],\"native\":[],\"rustc-dep-of-std\":[\"core\"]}}", + "r-efi_6.0.0": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"native\":[],\"rustc-dep-of-std\":[\"core\"]}}", + "rand_0.8.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.2.1\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.22\",\"target\":\"cfg(unix)\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.4\"},{\"features\":[\"into_bits\"],\"name\":\"packed_simd\",\"optional\":true,\"package\":\"packed_simd_2\",\"req\":\"^0.3.7\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"rand_core\",\"req\":\"^0.6.0\"},{\"kind\":\"dev\",\"name\":\"rand_pcg\",\"req\":\"^0.3.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"}],\"features\":{\"alloc\":[\"rand_core/alloc\"],\"default\":[\"std\",\"std_rng\"],\"getrandom\":[\"rand_core/getrandom\"],\"min_const_gen\":[],\"nightly\":[],\"serde1\":[\"serde\",\"rand_core/serde1\"],\"simd_support\":[\"packed_simd\"],\"small_rng\":[],\"std\":[\"rand_core/std\",\"rand_chacha/std\",\"alloc\",\"getrandom\",\"libc\"],\"std_rng\":[\"rand_chacha\"]}}", + "rand_0.9.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.2.1\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.4\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"optional\":true,\"req\":\"^0.9.0\"},{\"default_features\":false,\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rand_pcg\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.7\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"std_rng\",\"os_rng\",\"small_rng\",\"thread_rng\"],\"log\":[\"dep:log\"],\"nightly\":[],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\",\"rand_core/serde\"],\"simd_support\":[],\"small_rng\":[],\"std\":[\"rand_core/std\",\"rand_chacha?/std\",\"alloc\"],\"std_rng\":[\"dep:rand_chacha\"],\"thread_rng\":[\"std\",\"std_rng\",\"os_rng\"],\"unbiased\":[]}}", + "rand_chacha_0.3.1": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"ppv-lite86\",\"req\":\"^0.2.8\"},{\"name\":\"rand_core\",\"req\":\"^0.6.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"serde1\":[\"serde\"],\"simd\":[],\"std\":[\"ppv-lite86/std\"]}}", + "rand_chacha_0.9.0": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"ppv-lite86\",\"req\":\"^0.2.14\"},{\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"features\":[\"os_rng\"],\"kind\":\"dev\",\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\"],\"std\":[\"ppv-lite86/std\",\"rand_core/std\"]}}", + "rand_core_0.6.4": "{\"dependencies\":[{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.2\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"serde1\":[\"serde\"],\"std\":[\"alloc\",\"getrandom\",\"getrandom/std\"]}}", + "rand_core_0.9.3": "{\"dependencies\":[{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.3.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"os_rng\":[\"dep:getrandom\"],\"serde\":[\"dep:serde\"],\"std\":[\"getrandom?/std\"]}}", + "rayon-cond_0.4.0": "{\"dependencies\":[{\"name\":\"either\",\"req\":\"^1.6\"},{\"name\":\"itertools\",\"req\":\"^0.14\"},{\"name\":\"rayon\",\"req\":\"^1.7\"}],\"features\":{}}", + "rayon-core_1.12.1": "{\"dependencies\":[{\"name\":\"crossbeam-deque\",\"req\":\"^0.8.1\"},{\"name\":\"crossbeam-utils\",\"req\":\"^0.8.0\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"scoped-tls\",\"req\":\"^1.0\"},{\"name\":\"wasm_sync\",\"optional\":true,\"req\":\"^0.1.0\"}],\"features\":{\"web_spin_lock\":[\"dep:wasm_sync\"]}}", + "rayon_1.10.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"name\":\"rayon-core\",\"req\":\"^1.12.1\"},{\"name\":\"wasm_sync\",\"optional\":true,\"req\":\"^0.1.0\"}],\"features\":{\"web_spin_lock\":[\"dep:wasm_sync\",\"rayon-core/web_spin_lock\"]}}", + "redox_syscall_0.5.18": "{\"dependencies\":[{\"name\":\"bitflags\",\"req\":\"^2.4\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"loom\",\"req\":\"^0.7\",\"target\":\"cfg(loom)\"}],\"features\":{\"default\":[\"userspace\"],\"rustc-dep-of-std\":[\"core\",\"bitflags/rustc-dep-of-std\"],\"std\":[],\"userspace\":[]}}", + "regex-automata_0.4.14": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"bstr\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.14\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"optional\":true,\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"syntax\",\"perf\",\"unicode\",\"meta\",\"nfa\",\"dfa\",\"hybrid\"],\"dfa\":[\"dfa-build\",\"dfa-search\",\"dfa-onepass\"],\"dfa-build\":[\"nfa-thompson\",\"dfa-search\"],\"dfa-onepass\":[\"nfa-thompson\"],\"dfa-search\":[],\"hybrid\":[\"alloc\",\"nfa-thompson\"],\"internal-instrument\":[\"internal-instrument-pikevm\"],\"internal-instrument-pikevm\":[\"logging\",\"std\"],\"logging\":[\"dep:log\",\"aho-corasick?/logging\",\"memchr?/logging\"],\"meta\":[\"syntax\",\"nfa-pikevm\"],\"nfa\":[\"nfa-thompson\",\"nfa-pikevm\",\"nfa-backtrack\"],\"nfa-backtrack\":[\"nfa-thompson\"],\"nfa-pikevm\":[\"nfa-thompson\"],\"nfa-thompson\":[\"alloc\"],\"perf\":[\"perf-inline\",\"perf-literal\"],\"perf-inline\":[],\"perf-literal\":[\"perf-literal-substring\",\"perf-literal-multisubstring\"],\"perf-literal-multisubstring\":[\"dep:aho-corasick\"],\"perf-literal-substring\":[\"aho-corasick?/perf-literal\",\"dep:memchr\"],\"std\":[\"regex-syntax?/std\",\"memchr?/std\",\"aho-corasick?/std\",\"alloc\"],\"syntax\":[\"dep:regex-syntax\",\"alloc\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"unicode-word-boundary\",\"regex-syntax?/unicode\"],\"unicode-age\":[\"regex-syntax?/unicode-age\"],\"unicode-bool\":[\"regex-syntax?/unicode-bool\"],\"unicode-case\":[\"regex-syntax?/unicode-case\"],\"unicode-gencat\":[\"regex-syntax?/unicode-gencat\"],\"unicode-perl\":[\"regex-syntax?/unicode-perl\"],\"unicode-script\":[\"regex-syntax?/unicode-script\"],\"unicode-segment\":[\"regex-syntax?/unicode-segment\"],\"unicode-word-boundary\":[]}}", + "regex-automata_0.4.9": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"bstr\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.14\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"optional\":true,\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"syntax\",\"perf\",\"unicode\",\"meta\",\"nfa\",\"dfa\",\"hybrid\"],\"dfa\":[\"dfa-build\",\"dfa-search\",\"dfa-onepass\"],\"dfa-build\":[\"nfa-thompson\",\"dfa-search\"],\"dfa-onepass\":[\"nfa-thompson\"],\"dfa-search\":[],\"hybrid\":[\"alloc\",\"nfa-thompson\"],\"internal-instrument\":[\"internal-instrument-pikevm\"],\"internal-instrument-pikevm\":[\"logging\",\"std\"],\"logging\":[\"dep:log\",\"aho-corasick?/logging\",\"memchr?/logging\"],\"meta\":[\"syntax\",\"nfa-pikevm\"],\"nfa\":[\"nfa-thompson\",\"nfa-pikevm\",\"nfa-backtrack\"],\"nfa-backtrack\":[\"nfa-thompson\"],\"nfa-pikevm\":[\"nfa-thompson\"],\"nfa-thompson\":[\"alloc\"],\"perf\":[\"perf-inline\",\"perf-literal\"],\"perf-inline\":[],\"perf-literal\":[\"perf-literal-substring\",\"perf-literal-multisubstring\"],\"perf-literal-multisubstring\":[\"std\",\"dep:aho-corasick\"],\"perf-literal-substring\":[\"aho-corasick?/perf-literal\",\"dep:memchr\"],\"std\":[\"regex-syntax?/std\",\"memchr?/std\",\"aho-corasick?/std\",\"alloc\"],\"syntax\":[\"dep:regex-syntax\",\"alloc\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"unicode-word-boundary\",\"regex-syntax?/unicode\"],\"unicode-age\":[\"regex-syntax?/unicode-age\"],\"unicode-bool\":[\"regex-syntax?/unicode-bool\"],\"unicode-case\":[\"regex-syntax?/unicode-case\"],\"unicode-gencat\":[\"regex-syntax?/unicode-gencat\"],\"unicode-perl\":[\"regex-syntax?/unicode-perl\"],\"unicode-script\":[\"regex-syntax?/unicode-script\"],\"unicode-segment\":[\"regex-syntax?/unicode-segment\"],\"unicode-word-boundary\":[]}}", + "regex-syntax_0.8.10": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.3.0\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"default\":[\"std\",\"unicode\"],\"std\":[],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\"],\"unicode-age\":[],\"unicode-bool\":[],\"unicode-case\":[],\"unicode-gencat\":[],\"unicode-perl\":[],\"unicode-script\":[],\"unicode-segment\":[]}}", + "regex-syntax_0.8.5": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.3.0\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"default\":[\"std\",\"unicode\"],\"std\":[],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\"],\"unicode-age\":[],\"unicode-bool\":[],\"unicode-case\":[],\"unicode-gencat\":[],\"unicode-perl\":[],\"unicode-script\":[],\"unicode-segment\":[]}}", + "regex_1.11.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.17.1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"features\":[\"alloc\",\"syntax\",\"meta\",\"nfa-pikevm\"],\"name\":\"regex-automata\",\"req\":\"^0.4.8\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"default\":[\"std\",\"perf\",\"unicode\",\"regex-syntax/default\"],\"logging\":[\"aho-corasick?/logging\",\"memchr?/logging\",\"regex-automata/logging\"],\"pattern\":[],\"perf\":[\"perf-cache\",\"perf-dfa\",\"perf-onepass\",\"perf-backtrack\",\"perf-inline\",\"perf-literal\"],\"perf-backtrack\":[\"regex-automata/nfa-backtrack\"],\"perf-cache\":[],\"perf-dfa\":[\"regex-automata/hybrid\"],\"perf-dfa-full\":[\"regex-automata/dfa-build\",\"regex-automata/dfa-search\"],\"perf-inline\":[\"regex-automata/perf-inline\"],\"perf-literal\":[\"dep:aho-corasick\",\"dep:memchr\",\"regex-automata/perf-literal\"],\"perf-onepass\":[\"regex-automata/dfa-onepass\"],\"std\":[\"aho-corasick?/std\",\"memchr?/std\",\"regex-automata/std\",\"regex-syntax/std\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"regex-automata/unicode\",\"regex-syntax/unicode\"],\"unicode-age\":[\"regex-automata/unicode-age\",\"regex-syntax/unicode-age\"],\"unicode-bool\":[\"regex-automata/unicode-bool\",\"regex-syntax/unicode-bool\"],\"unicode-case\":[\"regex-automata/unicode-case\",\"regex-syntax/unicode-case\"],\"unicode-gencat\":[\"regex-automata/unicode-gencat\",\"regex-syntax/unicode-gencat\"],\"unicode-perl\":[\"regex-automata/unicode-perl\",\"regex-automata/unicode-word-boundary\",\"regex-syntax/unicode-perl\"],\"unicode-script\":[\"regex-automata/unicode-script\",\"regex-syntax/unicode-script\"],\"unicode-segment\":[\"regex-automata/unicode-segment\",\"regex-syntax/unicode-segment\"],\"unstable\":[\"pattern\"],\"use_std\":[\"std\"]}}", + "regex_1.12.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"features\":[\"alloc\",\"syntax\",\"meta\",\"nfa-pikevm\"],\"name\":\"regex-automata\",\"req\":\"^0.4.12\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"default\":[\"std\",\"perf\",\"unicode\",\"regex-syntax/default\"],\"logging\":[\"aho-corasick?/logging\",\"memchr?/logging\",\"regex-automata/logging\"],\"pattern\":[],\"perf\":[\"perf-cache\",\"perf-dfa\",\"perf-onepass\",\"perf-backtrack\",\"perf-inline\",\"perf-literal\"],\"perf-backtrack\":[\"regex-automata/nfa-backtrack\"],\"perf-cache\":[],\"perf-dfa\":[\"regex-automata/hybrid\"],\"perf-dfa-full\":[\"regex-automata/dfa-build\",\"regex-automata/dfa-search\"],\"perf-inline\":[\"regex-automata/perf-inline\"],\"perf-literal\":[\"dep:aho-corasick\",\"dep:memchr\",\"regex-automata/perf-literal\"],\"perf-onepass\":[\"regex-automata/dfa-onepass\"],\"std\":[\"aho-corasick?/std\",\"memchr?/std\",\"regex-automata/std\",\"regex-syntax/std\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"regex-automata/unicode\",\"regex-syntax/unicode\"],\"unicode-age\":[\"regex-automata/unicode-age\",\"regex-syntax/unicode-age\"],\"unicode-bool\":[\"regex-automata/unicode-bool\",\"regex-syntax/unicode-bool\"],\"unicode-case\":[\"regex-automata/unicode-case\",\"regex-syntax/unicode-case\"],\"unicode-gencat\":[\"regex-automata/unicode-gencat\",\"regex-syntax/unicode-gencat\"],\"unicode-perl\":[\"regex-automata/unicode-perl\",\"regex-automata/unicode-word-boundary\",\"regex-syntax/unicode-perl\"],\"unicode-script\":[\"regex-automata/unicode-script\",\"regex-syntax/unicode-script\"],\"unicode-segment\":[\"regex-automata/unicode-segment\",\"regex-syntax/unicode-segment\"],\"unstable\":[\"pattern\"],\"use_std\":[\"std\"]}}", + "rustc-demangle_0.1.27": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"compiler_builtins\":[],\"rustc-dep-of-std\":[\"core\"],\"std\":[]}}", + "rustix_1.1.4": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bitflags\",\"req\":\"^2.4.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4\",\"target\":\"cfg(all(criterion, not(any(target_os = \\\"emscripten\\\", target_os = \\\"wasi\\\"))))\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.182\",\"target\":\"cfg(all(not(windows), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.182\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.171\"},{\"default_features\":false,\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(all(not(windows), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(windows)\"},{\"default_features\":false,\"name\":\"libc_errno\",\"optional\":true,\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\"},{\"default_features\":false,\"features\":[\"general\",\"ioctl\",\"no_std\"],\"name\":\"linux-raw-sys\",\"req\":\"^0.12\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"features\":[\"auxvec\",\"general\",\"errno\",\"ioctl\",\"no_std\",\"elf\"],\"name\":\"linux-raw-sys\",\"req\":\"^0.12\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"kind\":\"dev\",\"name\":\"memoffset\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.20.3\",\"target\":\"cfg(windows)\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"serial_test\",\"req\":\"^2.0.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.5.0\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\"],\"name\":\"windows-sys\",\"req\":\">=0.52, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"all-apis\":[\"event\",\"fs\",\"io_uring\",\"mm\",\"mount\",\"net\",\"param\",\"pipe\",\"process\",\"pty\",\"rand\",\"runtime\",\"shm\",\"stdio\",\"system\",\"termios\",\"thread\",\"time\"],\"alloc\":[],\"default\":[\"std\"],\"event\":[],\"fs\":[],\"io_uring\":[\"event\",\"fs\",\"net\",\"thread\",\"linux-raw-sys/io_uring\"],\"linux_4_11\":[],\"linux_5_1\":[\"linux_4_11\"],\"linux_5_11\":[\"linux_5_1\"],\"linux_latest\":[\"linux_5_11\"],\"mm\":[],\"mount\":[],\"net\":[\"linux-raw-sys/net\",\"linux-raw-sys/netlink\",\"linux-raw-sys/if_ether\",\"linux-raw-sys/xdp\"],\"param\":[],\"pipe\":[],\"process\":[\"linux-raw-sys/prctl\"],\"pty\":[\"fs\"],\"rand\":[],\"runtime\":[\"linux-raw-sys/prctl\"],\"rustc-dep-of-std\":[\"core\",\"rustc-std-workspace-alloc\",\"linux-raw-sys/rustc-dep-of-std\",\"bitflags/rustc-dep-of-std\"],\"shm\":[\"fs\"],\"std\":[\"bitflags/std\",\"alloc\",\"libc?/std\",\"libc_errno?/std\"],\"stdio\":[],\"system\":[\"linux-raw-sys/system\"],\"termios\":[],\"thread\":[\"linux-raw-sys/prctl\"],\"time\":[],\"try_close\":[],\"use-explicitly-provided-auxv\":[],\"use-libc\":[\"libc_errno\",\"libc\"],\"use-libc-auxv\":[]}}", + "rustversion_1.0.21": "{\"dependencies\":[{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", + "rustversion_1.0.22": "{\"dependencies\":[{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", + "ryu_1.0.18": "{\"dependencies\":[{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"num_cpus\",\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"}],\"features\":{\"small\":[]}}", + "scopeguard_1.2.0": "{\"dependencies\":[],\"features\":{\"default\":[\"use_std\"],\"use_std\":[]}}", + "semver_1.0.27": "{\"dependencies\":[{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"package\":\"serde_core\",\"req\":\"^1.0.220\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"}],\"features\":{\"default\":[\"std\"],\"serde\":[\"dep:serde\"],\"std\":[]}}", + "serde_1.0.216": "{\"dependencies\":[{\"name\":\"serde_derive\",\"req\":\"=1.0.216\",\"target\":\"cfg(any())\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"derive\":[\"serde_derive\"],\"rc\":[],\"std\":[],\"unstable\":[]}}", + "serde_1.0.228": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"result\"],\"name\":\"serde_core\",\"req\":\"=1.0.228\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[\"serde_core/alloc\"],\"default\":[\"std\"],\"derive\":[\"serde_derive\"],\"rc\":[\"serde_core/rc\"],\"std\":[\"serde_core/std\"],\"unstable\":[\"serde_core/unstable\"]}}", + "serde_core_1.0.228": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"name\":\"serde_derive\",\"req\":\"=1.0.228\",\"target\":\"cfg(any())\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"result\"],\"rc\":[],\"result\":[],\"std\":[],\"unstable\":[]}}", + "serde_derive_1.0.216": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"derive\",\"parsing\",\"printing\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0.81\"}],\"features\":{\"default\":[],\"deserialize_in_place\":[]}}", + "serde_derive_1.0.228": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"derive\",\"parsing\",\"printing\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0.81\"}],\"features\":{\"default\":[],\"deserialize_in_place\":[]}}", + "serde_json_1.0.134": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0.11\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.2.3\"},{\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2.0.2\"},{\"name\":\"itoa\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"name\":\"ryu\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"kind\":\"dev\",\"name\":\"serde_bytes\",\"req\":\"^0.11.10\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_stacker\",\"req\":\"^0.1.8\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{\"alloc\":[\"serde/alloc\"],\"arbitrary_precision\":[],\"default\":[\"std\"],\"float_roundtrip\":[],\"preserve_order\":[\"indexmap\",\"std\"],\"raw_value\":[],\"std\":[\"memchr/std\",\"serde/std\"],\"unbounded_depth\":[]}}", + "serde_json_1.0.149": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0.11\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.2.3\"},{\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2.0.2\"},{\"name\":\"itoa\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"kind\":\"dev\",\"name\":\"serde_bytes\",\"req\":\"^0.11.10\"},{\"default_features\":false,\"name\":\"serde_core\",\"req\":\"^1.0.220\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_stacker\",\"req\":\"^0.1.8\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"},{\"name\":\"zmij\",\"req\":\"^1.0\"}],\"features\":{\"alloc\":[\"serde_core/alloc\"],\"arbitrary_precision\":[],\"default\":[\"std\"],\"float_roundtrip\":[],\"preserve_order\":[\"indexmap\",\"std\"],\"raw_value\":[],\"std\":[\"memchr/std\",\"serde_core/std\"],\"unbounded_depth\":[]}}", + "shlex_1.3.0": "{\"dependencies\":[],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "signal-hook-registry_1.4.8": "{\"dependencies\":[{\"name\":\"errno\",\"req\":\">=0.2, <0.4\"},{\"name\":\"libc\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"signal-hook\",\"req\":\"~0.3\"}],\"features\":{}}", + "slab_0.4.12": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.95\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "smallvec_1.13.2": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.0.1\"},{\"kind\":\"dev\",\"name\":\"debugger_test\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"debugger_test_parser\",\"req\":\"^0.1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"const_generics\":[],\"const_new\":[\"const_generics\"],\"debugger_visualizer\":[],\"drain_filter\":[],\"drain_keep_rest\":[\"drain_filter\"],\"may_dangle\":[],\"specialization\":[],\"union\":[],\"write\":[]}}", + "smallvec_1.15.1": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"name\":\"bincode\",\"optional\":true,\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"bincode1\",\"package\":\"bincode\",\"req\":\"^1.0.1\"},{\"kind\":\"dev\",\"name\":\"debugger_test\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"debugger_test_parser\",\"req\":\"^0.1.0\"},{\"default_features\":false,\"name\":\"malloc_size_of\",\"optional\":true,\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"name\":\"unty\",\"optional\":true,\"req\":\"^0.0.4\"}],\"features\":{\"const_generics\":[],\"const_new\":[\"const_generics\"],\"debugger_visualizer\":[],\"drain_filter\":[],\"drain_keep_rest\":[\"drain_filter\"],\"impl_bincode\":[\"bincode\",\"unty\"],\"may_dangle\":[],\"specialization\":[],\"union\":[],\"write\":[]}}", + "socket2_0.5.10": "{\"dependencies\":[{\"name\":\"libc\",\"req\":\"^0.2.171\",\"target\":\"cfg(unix)\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\",\"Win32_System_IO\",\"Win32_System_Threading\",\"Win32_System_WindowsProgramming\"],\"name\":\"windows-sys\",\"req\":\"^0.52\",\"target\":\"cfg(windows)\"}],\"features\":{\"all\":[]}}", + "socket2_0.6.3": "{\"dependencies\":[{\"name\":\"libc\",\"req\":\"^0.2.172\",\"target\":\"cfg(any(unix, target_os = \\\"wasi\\\"))\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\",\"Win32_System_IO\",\"Win32_System_Threading\",\"Win32_System_WindowsProgramming\"],\"name\":\"windows-sys\",\"req\":\">=0.60, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"all\":[]}}", + "spm_precompiled_0.1.4": "{\"dependencies\":[{\"name\":\"base64\",\"req\":\"^0.13\"},{\"name\":\"nom\",\"req\":\"^7.1.1\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"unicode-segmentation\",\"req\":\"^1.9\"}],\"features\":{}}", + "static_assertions_1.1.0": "{\"dependencies\":[],\"features\":{\"nightly\":[]}}", + "strsim_0.11.1": "{\"dependencies\":[],\"features\":{}}", + "syn_2.0.117": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"insta\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.91\"},{\"default_features\":false,\"name\":\"quote\",\"optional\":true,\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1\"},{\"features\":[\"blocking\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.13\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"syn-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4.16\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"termcolor\",\"req\":\"^1\"},{\"name\":\"unicode-ident\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\",\"target\":\"cfg(not(miri))\"}],\"features\":{\"clone-impls\":[],\"default\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\"],\"derive\":[],\"extra-traits\":[],\"fold\":[],\"full\":[],\"parsing\":[],\"printing\":[\"dep:quote\"],\"proc-macro\":[\"proc-macro2/proc-macro\",\"quote?/proc-macro\"],\"test\":[\"syn-test-suite/all-features\"],\"visit\":[],\"visit-mut\":[]}}", + "syn_2.0.91": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"insta\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.91\"},{\"default_features\":false,\"name\":\"quote\",\"optional\":true,\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1\"},{\"features\":[\"blocking\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.12\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"syn-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4.16\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"termcolor\",\"req\":\"^1\"},{\"name\":\"unicode-ident\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\",\"target\":\"cfg(not(miri))\"}],\"features\":{\"clone-impls\":[],\"default\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\"],\"derive\":[],\"extra-traits\":[],\"fold\":[],\"full\":[],\"parsing\":[],\"printing\":[\"dep:quote\"],\"proc-macro\":[\"proc-macro2/proc-macro\",\"quote?/proc-macro\"],\"test\":[\"syn-test-suite/all-features\"],\"visit\":[],\"visit-mut\":[]}}", + "sync_wrapper_1.0.2": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"}],\"features\":{\"futures\":[\"futures-core\"]}}", + "tempfile_3.27.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"name\":\"fastrand\",\"req\":\"^2.1.1\"},{\"default_features\":false,\"name\":\"getrandom\",\"optional\":true,\"req\":\">=0.3.0, <0.5\",\"target\":\"cfg(any(unix, windows, target_os = \\\"wasi\\\"))\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"once_cell\",\"req\":\"^1.19.0\"},{\"features\":[\"fs\"],\"name\":\"rustix\",\"req\":\"^1.1.4\",\"target\":\"cfg(any(unix, target_os = \\\"wasi\\\"))\"},{\"features\":[\"Win32_Storage_FileSystem\",\"Win32_Foundation\"],\"name\":\"windows-sys\",\"req\":\">=0.52, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[\"getrandom\"],\"nightly\":[]}}", + "thiserror-impl_2.0.12": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"name\":\"syn\",\"req\":\"^2.0.87\"}],\"features\":{}}", + "thiserror_2.0.12": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.73\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"name\":\"thiserror-impl\",\"req\":\"=2.0.12\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "tokenizers_0.22.2": "{\"dependencies\":[{\"features\":[\"serde\"],\"name\":\"ahash\",\"req\":\"^0.8.11\"},{\"name\":\"aho-corasick\",\"req\":\"^1.1\"},{\"kind\":\"dev\",\"name\":\"assert_approx_eq\",\"req\":\"^1.1\"},{\"features\":[\"serde\"],\"name\":\"compact_str\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.6\"},{\"features\":[\"serde\"],\"name\":\"dary_heap\",\"req\":\"^0.3.6\"},{\"name\":\"derive_builder\",\"req\":\"^0.20\"},{\"default_features\":false,\"name\":\"esaxx-rs\",\"req\":\"^0.1.10\"},{\"name\":\"fancy-regex\",\"optional\":true,\"req\":\"^0.14\"},{\"name\":\"getrandom\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"ureq\"],\"name\":\"hf-hub\",\"optional\":true,\"req\":\"^0.4.1\"},{\"name\":\"indicatif\",\"optional\":true,\"req\":\"^0.18\"},{\"name\":\"itertools\",\"req\":\"^0.14\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"macro_rules_attribute\",\"req\":\"^0.2.0\"},{\"name\":\"monostate\",\"req\":\"^0.1.12\"},{\"default_features\":false,\"name\":\"onig\",\"optional\":true,\"req\":\"^6.5.1\"},{\"name\":\"paste\",\"req\":\"^1.0.14\"},{\"name\":\"rand\",\"req\":\"^0.9\"},{\"name\":\"rayon\",\"req\":\"^1.10\"},{\"name\":\"rayon-cond\",\"req\":\"^0.4\"},{\"name\":\"regex\",\"req\":\"^1.10\"},{\"name\":\"regex-syntax\",\"req\":\"^0.8\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"spm_precompiled\",\"req\":\"^0.1.3\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.10\"},{\"name\":\"thiserror\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3.18\"},{\"name\":\"unicode-normalization-alignments\",\"req\":\"^0.1\"},{\"name\":\"unicode-segmentation\",\"req\":\"^1.11\"},{\"name\":\"unicode_categories\",\"req\":\"^0.1\"}],\"features\":{\"default\":[\"progressbar\",\"onig\",\"esaxx_fast\"],\"esaxx_fast\":[\"esaxx-rs/cpp\"],\"http\":[\"hf-hub\"],\"progressbar\":[\"indicatif\"],\"rustls-tls\":[\"hf-hub?/rustls-tls\"],\"unstable_wasm\":[\"fancy-regex\",\"getrandom/wasm_js\"]}}", + "tokio-macros_2.4.0": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0.0\"}],\"features\":{}}", + "tokio-stream_0.1.15": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-stream\",\"req\":\"^0.3\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"name\":\"futures-core\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"parking_lot\",\"req\":\"^0.12.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.11\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"req\":\"^1.15.0\"},{\"features\":[\"full\",\"test-util\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.2.0\"},{\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7.0\"}],\"features\":{\"default\":[\"time\"],\"fs\":[\"tokio/fs\"],\"full\":[\"time\",\"net\",\"io-util\",\"fs\",\"sync\",\"signal\"],\"io-util\":[\"tokio/io-util\"],\"net\":[\"tokio/net\"],\"signal\":[\"tokio/signal\"],\"sync\":[\"tokio/sync\",\"tokio-util\"],\"time\":[\"tokio/time\"]}}", + "tokio-util_0.7.17": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-stream\",\"req\":\"^0.3.0\"},{\"name\":\"bytes\",\"req\":\"^1.5.0\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.0\"},{\"name\":\"futures-core\",\"req\":\"^0.3.0\"},{\"name\":\"futures-io\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"futures-sink\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"futures-test\",\"req\":\"^0.3.5\"},{\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.0\"},{\"default_features\":false,\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.15.0\"},{\"kind\":\"dev\",\"name\":\"parking_lot\",\"req\":\"^0.12.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.11\"},{\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4.4\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.1.0\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"req\":\"^1.28.0\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.29\"}],\"features\":{\"__docs_rs\":[\"futures-util\"],\"codec\":[],\"compat\":[\"futures-io\"],\"default\":[],\"full\":[\"codec\",\"compat\",\"io-util\",\"time\",\"net\",\"rt\",\"join-map\"],\"io\":[],\"io-util\":[\"io\",\"tokio/rt\",\"tokio/io-util\"],\"join-map\":[\"rt\",\"hashbrown\"],\"net\":[\"tokio/net\"],\"rt\":[\"tokio/rt\",\"tokio/sync\",\"futures-util\"],\"time\":[\"tokio/time\",\"slab\"]}}", + "tokio_1.39.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-stream\",\"req\":\"^0.3\"},{\"name\":\"backtrace\",\"req\":\"^0.3.58\",\"target\":\"cfg(tokio_taskdump)\"},{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1.0.0\"},{\"features\":[\"async-await\"],\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.0\"},{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.149\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.149\",\"target\":\"cfg(unix)\"},{\"features\":[\"futures\",\"checkpoint\"],\"kind\":\"dev\",\"name\":\"loom\",\"req\":\"^0.7\",\"target\":\"cfg(loom)\"},{\"default_features\":false,\"name\":\"mio\",\"optional\":true,\"req\":\"^1.0.1\"},{\"features\":[\"tokio\"],\"kind\":\"dev\",\"name\":\"mio-aio\",\"req\":\"^0.9.0\",\"target\":\"cfg(target_os = \\\"freebsd\\\")\"},{\"kind\":\"dev\",\"name\":\"mockall\",\"req\":\"^0.11.1\"},{\"default_features\":false,\"features\":[\"aio\",\"fs\",\"socket\"],\"kind\":\"dev\",\"name\":\"nix\",\"req\":\"^0.29.0\",\"target\":\"cfg(unix)\"},{\"name\":\"parking_lot\",\"optional\":true,\"req\":\"^0.12.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.11\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.0\",\"target\":\"cfg(not(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\")))\"},{\"name\":\"signal-hook-registry\",\"optional\":true,\"req\":\"^1.1.1\",\"target\":\"cfg(unix)\"},{\"features\":[\"all\"],\"name\":\"socket2\",\"optional\":true,\"req\":\"^0.5.5\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"kind\":\"dev\",\"name\":\"socket2\",\"req\":\"^0.5.5\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.1.0\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"name\":\"tokio-macros\",\"optional\":true,\"req\":\"~2.4.0\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.25\",\"target\":\"cfg(tokio_unstable)\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.0\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", not(target_os = \\\"wasi\\\")))\"},{\"name\":\"windows-sys\",\"optional\":true,\"req\":\"^0.52\",\"target\":\"cfg(windows)\"},{\"features\":[\"Win32_Foundation\",\"Win32_Security_Authorization\"],\"kind\":\"dev\",\"name\":\"windows-sys\",\"req\":\"^0.52\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[],\"fs\":[],\"full\":[\"fs\",\"io-util\",\"io-std\",\"macros\",\"net\",\"parking_lot\",\"process\",\"rt\",\"rt-multi-thread\",\"signal\",\"sync\",\"time\"],\"io-std\":[],\"io-util\":[\"bytes\"],\"macros\":[\"tokio-macros\"],\"net\":[\"libc\",\"mio/os-poll\",\"mio/os-ext\",\"mio/net\",\"socket2\",\"windows-sys/Win32_Foundation\",\"windows-sys/Win32_Security\",\"windows-sys/Win32_Storage_FileSystem\",\"windows-sys/Win32_System_Pipes\",\"windows-sys/Win32_System_SystemServices\"],\"process\":[\"bytes\",\"libc\",\"mio/os-poll\",\"mio/os-ext\",\"mio/net\",\"signal-hook-registry\",\"windows-sys/Win32_Foundation\",\"windows-sys/Win32_System_Threading\",\"windows-sys/Win32_System_WindowsProgramming\"],\"rt\":[],\"rt-multi-thread\":[\"rt\"],\"signal\":[\"libc\",\"mio/os-poll\",\"mio/net\",\"mio/os-ext\",\"signal-hook-registry\",\"windows-sys/Win32_Foundation\",\"windows-sys/Win32_System_Console\"],\"sync\":[],\"test-util\":[\"rt\",\"sync\",\"time\"],\"time\":[]}}", + "tonic-build_0.12.3": "{\"dependencies\":[{\"name\":\"prettyplease\",\"req\":\"^0.2\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"name\":\"prost-build\",\"optional\":true,\"req\":\"^0.13\"},{\"name\":\"prost-types\",\"optional\":true,\"req\":\"^0.13\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"name\":\"syn\",\"req\":\"^2.0\"}],\"features\":{\"cleanup-markdown\":[\"prost\",\"prost-build/cleanup-markdown\"],\"default\":[\"transport\",\"prost\"],\"prost\":[\"prost-build\",\"dep:prost-types\"],\"transport\":[]}}", + "tonic_0.12.1": "{\"dependencies\":[{\"name\":\"async-stream\",\"optional\":true,\"req\":\"^0.3\"},{\"name\":\"async-trait\",\"optional\":true,\"req\":\"^0.1.13\"},{\"default_features\":false,\"name\":\"axum\",\"optional\":true,\"req\":\"^0.7\"},{\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.5\"},{\"name\":\"bytes\",\"req\":\"^1.0\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"h2\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"http-body\",\"req\":\"^1\"},{\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"features\":[\"http1\",\"http2\"],\"name\":\"hyper\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"hyper-timeout\",\"optional\":true,\"req\":\"^0.5\"},{\"features\":[\"tokio\"],\"name\":\"hyper-util\",\"optional\":true,\"req\":\"^0.1.4\"},{\"name\":\"percent-encoding\",\"req\":\"^2.1\"},{\"name\":\"pin-project\",\"req\":\"^1.0.11\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"prost\",\"optional\":true,\"req\":\"^0.13\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"name\":\"rustls-native-certs\",\"optional\":true,\"req\":\"^0.7\"},{\"name\":\"rustls-pemfile\",\"optional\":true,\"req\":\"^2.0\"},{\"features\":[\"all\"],\"name\":\"socket2\",\"optional\":true,\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"rt\",\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"logging\",\"tls12\",\"ring\"],\"name\":\"tokio-rustls\",\"optional\":true,\"req\":\"^0.26\"},{\"default_features\":false,\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"tower\",\"optional\":true,\"req\":\"^0.4.7\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tower\",\"req\":\"^0.4.7\"},{\"name\":\"tower-layer\",\"req\":\"^0.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"name\":\"tracing\",\"req\":\"^0.1\"},{\"name\":\"webpki-roots\",\"optional\":true,\"req\":\"^0.26\"},{\"name\":\"zstd\",\"optional\":true,\"req\":\"^0.13.0\"}],\"features\":{\"channel\":[\"dep:hyper\",\"hyper?/client\",\"dep:hyper-util\",\"hyper-util?/client-legacy\",\"dep:tower\",\"tower?/balance\",\"tower?/buffer\",\"tower?/discover\",\"tower?/limit\",\"dep:tokio\",\"tokio?/time\",\"dep:hyper-timeout\"],\"codegen\":[\"dep:async-trait\"],\"default\":[\"transport\",\"codegen\",\"prost\"],\"gzip\":[\"dep:flate2\"],\"prost\":[\"dep:prost\"],\"router\":[\"dep:axum\",\"dep:tower\",\"tower?/util\"],\"server\":[\"router\",\"dep:async-stream\",\"dep:h2\",\"dep:hyper\",\"hyper?/server\",\"dep:hyper-util\",\"hyper-util?/service\",\"hyper-util?/server-auto\",\"dep:socket2\",\"dep:tokio\",\"tokio?/macros\",\"tokio?/net\",\"tokio?/time\",\"tokio-stream/net\",\"dep:tower\",\"tower?/util\",\"tower?/limit\"],\"tls\":[\"dep:rustls-pemfile\",\"dep:tokio-rustls\",\"dep:tokio\",\"tokio?/rt\",\"tokio?/macros\"],\"tls-roots\":[\"tls\",\"channel\",\"dep:rustls-native-certs\"],\"tls-webpki-roots\":[\"tls\",\"channel\",\"dep:webpki-roots\"],\"transport\":[\"server\",\"channel\"],\"zstd\":[\"dep:zstd\"]}}", + "tower-layer_0.3.3": "{\"dependencies\":[],\"features\":{}}", + "tower-service_0.3.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.22\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^0.2\"},{\"features\":[\"macros\",\"time\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.6.2\"},{\"kind\":\"dev\",\"name\":\"tower-layer\",\"req\":\"^0.3\"}],\"features\":{}}", + "tower_0.4.13": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"hdrhistogram\",\"optional\":true,\"req\":\"^7.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"hdrhistogram\",\"req\":\"^7.0\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^0.2\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^1.0.2\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4.0\"},{\"name\":\"pin-project\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"pin-project-lite\",\"optional\":true,\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"},{\"features\":[\"small_rng\"],\"name\":\"rand\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.6\"},{\"features\":[\"macros\",\"sync\",\"test-util\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.6.2\"},{\"name\":\"tokio-stream\",\"optional\":true,\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7.0\"},{\"name\":\"tower-layer\",\"req\":\"^0.3.1\"},{\"name\":\"tower-service\",\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"tower-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.2\"},{\"default_features\":false,\"features\":[\"fmt\",\"ansi\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"}],\"features\":{\"__common\":[\"futures-core\",\"pin-project-lite\"],\"balance\":[\"discover\",\"load\",\"ready-cache\",\"make\",\"rand\",\"slab\"],\"buffer\":[\"__common\",\"tokio/sync\",\"tokio/rt\",\"tokio-util\",\"tracing\"],\"default\":[\"log\"],\"discover\":[\"__common\"],\"filter\":[\"__common\",\"futures-util\"],\"full\":[\"balance\",\"buffer\",\"discover\",\"filter\",\"hedge\",\"limit\",\"load\",\"load-shed\",\"make\",\"ready-cache\",\"reconnect\",\"retry\",\"spawn-ready\",\"steer\",\"timeout\",\"util\"],\"hedge\":[\"util\",\"filter\",\"futures-util\",\"hdrhistogram\",\"tokio/time\",\"tracing\"],\"limit\":[\"__common\",\"tokio/time\",\"tokio/sync\",\"tokio-util\",\"tracing\"],\"load\":[\"__common\",\"tokio/time\",\"tracing\"],\"load-shed\":[\"__common\"],\"log\":[\"tracing/log\"],\"make\":[\"futures-util\",\"pin-project-lite\",\"tokio/io-std\"],\"ready-cache\":[\"futures-core\",\"futures-util\",\"indexmap\",\"tokio/sync\",\"tracing\",\"pin-project-lite\"],\"reconnect\":[\"make\",\"tokio/io-std\",\"tracing\"],\"retry\":[\"__common\",\"tokio/time\"],\"spawn-ready\":[\"__common\",\"futures-util\",\"tokio/sync\",\"tokio/rt\",\"util\",\"tracing\"],\"steer\":[],\"timeout\":[\"pin-project-lite\",\"tokio/time\"],\"util\":[\"__common\",\"futures-util\",\"pin-project\"]}}", + "tower_0.5.3": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.22\"},{\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3.22\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.22\"},{\"default_features\":false,\"features\":[\"async-await-macro\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3.22\"},{\"default_features\":false,\"name\":\"hdrhistogram\",\"optional\":true,\"req\":\"^7.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"hdrhistogram\",\"req\":\"^7.0\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.0.2\"},{\"name\":\"pin-project-lite\",\"optional\":true,\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9\"},{\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4.9\"},{\"name\":\"sync_wrapper\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.6.2\"},{\"features\":[\"macros\",\"sync\",\"test-util\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.6.2\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1.1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7.0\"},{\"name\":\"tower-layer\",\"req\":\"^0.3.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3.3\"},{\"kind\":\"dev\",\"name\":\"tower-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.2\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1.2\"},{\"default_features\":false,\"features\":[\"fmt\",\"ansi\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"}],\"features\":{\"balance\":[\"discover\",\"load\",\"ready-cache\",\"make\",\"slab\",\"util\"],\"buffer\":[\"tokio/sync\",\"tokio/rt\",\"tokio-util\",\"tracing\",\"pin-project-lite\"],\"discover\":[\"futures-core\",\"pin-project-lite\"],\"filter\":[\"futures-util\",\"pin-project-lite\"],\"full\":[\"balance\",\"buffer\",\"discover\",\"filter\",\"hedge\",\"limit\",\"load\",\"load-shed\",\"make\",\"ready-cache\",\"reconnect\",\"retry\",\"spawn-ready\",\"steer\",\"timeout\",\"util\"],\"hedge\":[\"util\",\"filter\",\"futures-util\",\"hdrhistogram\",\"tokio/time\",\"tracing\"],\"limit\":[\"tokio/time\",\"tokio/sync\",\"tokio-util\",\"tracing\",\"pin-project-lite\"],\"load\":[\"tokio/time\",\"tracing\",\"pin-project-lite\"],\"load-shed\":[\"pin-project-lite\"],\"log\":[\"tracing/log\"],\"make\":[\"pin-project-lite\",\"tokio\"],\"ready-cache\":[\"futures-core\",\"futures-util\",\"indexmap\",\"tokio/sync\",\"tracing\",\"pin-project-lite\"],\"reconnect\":[\"make\",\"tracing\"],\"retry\":[\"tokio/time\",\"util\"],\"spawn-ready\":[\"futures-util\",\"tokio/sync\",\"tokio/rt\",\"util\",\"tracing\"],\"steer\":[],\"timeout\":[\"pin-project-lite\",\"tokio/time\"],\"tokio-stream\":[],\"util\":[\"futures-core\",\"futures-util\",\"pin-project-lite\",\"sync_wrapper\"]}}", + "tracing-attributes_0.1.31": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-trait\",\"req\":\"^0.1.67\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1.0.20\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.9\"},{\"default_features\":false,\"features\":[\"full\",\"parsing\",\"printing\",\"visit-mut\",\"clone-impls\",\"extra-traits\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4.2\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1.35\"},{\"features\":[\"env-filter\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.64\"}],\"features\":{\"async-await\":[]}}", + "tracing-core_0.1.36": "{\"dependencies\":[{\"name\":\"once_cell\",\"optional\":true,\"req\":\"^1.13.0\"},{\"default_features\":false,\"name\":\"valuable\",\"optional\":true,\"req\":\"^0.1.0\",\"target\":\"cfg(tracing_unstable)\"}],\"features\":{\"default\":[\"std\",\"valuable?/std\"],\"std\":[\"once_cell\"]}}", + "tracing_0.1.44": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.6\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.21\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4.17\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.9\"},{\"name\":\"tracing-attributes\",\"optional\":true,\"req\":\"^0.1.31\"},{\"default_features\":false,\"name\":\"tracing-core\",\"req\":\"^0.1.36\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.38\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"}],\"features\":{\"async-await\":[],\"attributes\":[\"tracing-attributes\"],\"default\":[\"std\",\"attributes\"],\"log-always\":[\"log\"],\"max_level_debug\":[],\"max_level_error\":[],\"max_level_info\":[],\"max_level_off\":[],\"max_level_trace\":[],\"max_level_warn\":[],\"release_max_level_debug\":[],\"release_max_level_error\":[],\"release_max_level_info\":[],\"release_max_level_off\":[],\"release_max_level_trace\":[],\"release_max_level_warn\":[],\"std\":[\"tracing-core/std\"],\"valuable\":[\"tracing-core/valuable\"]}}", + "try-lock_0.2.5": "{\"dependencies\":[],\"features\":{}}", + "unicode-ident_1.0.14": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"fst\",\"req\":\"^0.4\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"roaring\",\"req\":\"^0.10\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"ucd-trie\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"unicode-xid\",\"req\":\"^0.2.6\"}],\"features\":{}}", + "unicode-ident_1.0.24": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"fst\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"roaring\",\"req\":\"^0.11\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"ucd-trie\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"unicode-xid\",\"req\":\"^0.2.6\"}],\"features\":{}}", + "unicode-normalization-alignments_0.1.12": "{\"dependencies\":[{\"name\":\"smallvec\",\"req\":\"^1.0\"}],\"features\":{}}", + "unicode-segmentation_1.12.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.7\"}],\"features\":{\"no_std\":[]}}", + "unicode-xid_0.2.6": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3\"}],\"features\":{\"bench\":[],\"default\":[],\"no_std\":[]}}", + "unicode_categories_0.1.1": "{\"dependencies\":[],\"features\":{}}", + "version_check_0.9.5": "{\"dependencies\":[],\"features\":{}}", + "want_0.3.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"tokio-executor\",\"req\":\"^0.2.0-alpha.2\"},{\"kind\":\"dev\",\"name\":\"tokio-sync\",\"req\":\"^0.2.0-alpha.2\"},{\"name\":\"try-lock\",\"req\":\"^0.2.4\"}],\"features\":{}}", + "wasi_0.11.1+wasi-snapshot-preview1": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"rustc-dep-of-std\":[\"core\",\"rustc-std-workspace-alloc\"],\"std\":[]}}", + "wasi_0.14.2+wasi-0.2.4": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"bitflags\"],\"name\":\"wit-bindgen-rt\",\"req\":\"^0.39.0\"}],\"features\":{\"default\":[\"std\"],\"rustc-dep-of-std\":[\"compiler_builtins\",\"core\",\"rustc-std-workspace-alloc\"],\"std\":[]}}", + "wasip2_1.0.2+wasi-0.2.9": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"wit-bindgen\",\"req\":\"^0.51.0\"}],\"features\":{\"bitflags\":[\"wit-bindgen/bitflags\"],\"default\":[\"std\",\"bitflags\"],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"wit-bindgen/rustc-dep-of-std\"],\"std\":[]}}", + "wasip3_0.4.0+wasi-0.3.0-rc-2026-01-06": "{\"dependencies\":[{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1.10.1\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.31\"},{\"name\":\"http\",\"optional\":true,\"req\":\"^1.3.1\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^1.3.1\"},{\"name\":\"http-body\",\"optional\":true,\"req\":\"^1.0.1\"},{\"name\":\"thiserror\",\"optional\":true,\"req\":\"^2.0.17\"},{\"default_features\":false,\"features\":[\"async\"],\"name\":\"wit-bindgen\",\"req\":\"^0.51.0\"},{\"default_features\":false,\"features\":[\"async-spawn\"],\"kind\":\"dev\",\"name\":\"wit-bindgen\",\"req\":\"^0.51.0\"}],\"features\":{\"http-compat\":[\"dep:bytes\",\"dep:http-body\",\"dep:http\",\"dep:thiserror\",\"wit-bindgen/async-spawn\"]}}", + "wasm-encoder_0.244.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"default_features\":false,\"name\":\"leb128fmt\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.2.0\"},{\"default_features\":false,\"features\":[\"simd\",\"simd\"],\"name\":\"wasmparser\",\"optional\":true,\"req\":\"^0.244.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"wasmprinter\",\"req\":\"^0.244.0\"}],\"features\":{\"component-model\":[\"wasmparser?/component-model\"],\"default\":[\"std\",\"component-model\"],\"std\":[\"wasmparser?/std\"]}}", + "wasm-metadata_0.244.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"name\":\"auditable-serde\",\"optional\":true,\"req\":\"^0.8.0\"},{\"features\":[\"derive\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4.0.0\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1.1.0\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"indexmap\",\"req\":\"^2.7.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"spdx\",\"optional\":true,\"req\":\"^0.10.1\"},{\"name\":\"url\",\"optional\":true,\"req\":\"^2.0.0\"},{\"default_features\":false,\"features\":[\"std\",\"component-model\"],\"name\":\"wasm-encoder\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"simd\",\"std\",\"component-model\",\"hash-collections\"],\"name\":\"wasmparser\",\"req\":\"^0.244.0\"}],\"features\":{\"default\":[\"oci\",\"serde\"],\"oci\":[\"dep:auditable-serde\",\"dep:flate2\",\"dep:url\",\"dep:spdx\",\"dep:serde_json\",\"serde\"],\"serde\":[\"dep:serde_derive\",\"dep:serde\"]}}", + "wasmparser_0.244.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"name\":\"bitflags\",\"req\":\"^2.4.1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5.1\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"default-hasher\"],\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.15.2\"},{\"default_features\":false,\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.7.0\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.13.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.3\"},{\"default_features\":false,\"name\":\"semver\",\"optional\":true,\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.166\"}],\"features\":{\"component-model\":[\"dep:semver\"],\"default\":[\"std\",\"validate\",\"serde\",\"features\",\"component-model\",\"hash-collections\",\"simd\"],\"features\":[],\"hash-collections\":[\"dep:hashbrown\",\"dep:indexmap\"],\"prefer-btree-collections\":[],\"serde\":[\"dep:serde\",\"indexmap?/serde\",\"hashbrown?/serde\"],\"simd\":[],\"std\":[\"indexmap?/std\"],\"validate\":[]}}", + "windows-link_0.2.1": "{\"dependencies\":[],\"features\":{}}", + "windows-sys_0.52.0": "{\"dependencies\":[{\"name\":\"windows-targets\",\"req\":\"^0.52.0\"}],\"features\":{\"Wdk\":[],\"Wdk_Foundation\":[\"Wdk\"],\"Wdk_Graphics\":[\"Wdk\"],\"Wdk_Graphics_Direct3D\":[\"Wdk_Graphics\"],\"Wdk_Storage\":[\"Wdk\"],\"Wdk_Storage_FileSystem\":[\"Wdk_Storage\"],\"Wdk_Storage_FileSystem_Minifilters\":[\"Wdk_Storage_FileSystem\"],\"Wdk_System\":[\"Wdk\"],\"Wdk_System_IO\":[\"Wdk_System\"],\"Wdk_System_OfflineRegistry\":[\"Wdk_System\"],\"Wdk_System_Registry\":[\"Wdk_System\"],\"Wdk_System_SystemInformation\":[\"Wdk_System\"],\"Wdk_System_SystemServices\":[\"Wdk_System\"],\"Wdk_System_Threading\":[\"Wdk_System\"],\"Win32\":[],\"Win32_Data\":[\"Win32\"],\"Win32_Data_HtmlHelp\":[\"Win32_Data\"],\"Win32_Data_RightsManagement\":[\"Win32_Data\"],\"Win32_Devices\":[\"Win32\"],\"Win32_Devices_AllJoyn\":[\"Win32_Devices\"],\"Win32_Devices_BiometricFramework\":[\"Win32_Devices\"],\"Win32_Devices_Bluetooth\":[\"Win32_Devices\"],\"Win32_Devices_Communication\":[\"Win32_Devices\"],\"Win32_Devices_DeviceAndDriverInstallation\":[\"Win32_Devices\"],\"Win32_Devices_DeviceQuery\":[\"Win32_Devices\"],\"Win32_Devices_Display\":[\"Win32_Devices\"],\"Win32_Devices_Enumeration\":[\"Win32_Devices\"],\"Win32_Devices_Enumeration_Pnp\":[\"Win32_Devices_Enumeration\"],\"Win32_Devices_Fax\":[\"Win32_Devices\"],\"Win32_Devices_HumanInterfaceDevice\":[\"Win32_Devices\"],\"Win32_Devices_PortableDevices\":[\"Win32_Devices\"],\"Win32_Devices_Properties\":[\"Win32_Devices\"],\"Win32_Devices_Pwm\":[\"Win32_Devices\"],\"Win32_Devices_Sensors\":[\"Win32_Devices\"],\"Win32_Devices_SerialCommunication\":[\"Win32_Devices\"],\"Win32_Devices_Tapi\":[\"Win32_Devices\"],\"Win32_Devices_Usb\":[\"Win32_Devices\"],\"Win32_Devices_WebServicesOnDevices\":[\"Win32_Devices\"],\"Win32_Foundation\":[\"Win32\"],\"Win32_Gaming\":[\"Win32\"],\"Win32_Globalization\":[\"Win32\"],\"Win32_Graphics\":[\"Win32\"],\"Win32_Graphics_Dwm\":[\"Win32_Graphics\"],\"Win32_Graphics_Gdi\":[\"Win32_Graphics\"],\"Win32_Graphics_GdiPlus\":[\"Win32_Graphics\"],\"Win32_Graphics_Hlsl\":[\"Win32_Graphics\"],\"Win32_Graphics_OpenGL\":[\"Win32_Graphics\"],\"Win32_Graphics_Printing\":[\"Win32_Graphics\"],\"Win32_Graphics_Printing_PrintTicket\":[\"Win32_Graphics_Printing\"],\"Win32_Management\":[\"Win32\"],\"Win32_Management_MobileDeviceManagementRegistration\":[\"Win32_Management\"],\"Win32_Media\":[\"Win32\"],\"Win32_Media_Audio\":[\"Win32_Media\"],\"Win32_Media_DxMediaObjects\":[\"Win32_Media\"],\"Win32_Media_KernelStreaming\":[\"Win32_Media\"],\"Win32_Media_Multimedia\":[\"Win32_Media\"],\"Win32_Media_Streaming\":[\"Win32_Media\"],\"Win32_Media_WindowsMediaFormat\":[\"Win32_Media\"],\"Win32_NetworkManagement\":[\"Win32\"],\"Win32_NetworkManagement_Dhcp\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Dns\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_InternetConnectionWizard\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_IpHelper\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Multicast\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Ndis\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetBios\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetManagement\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetShell\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetworkDiagnosticsFramework\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_P2P\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_QoS\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Rras\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Snmp\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WNet\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WebDav\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WiFi\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsConnectionManager\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsFilteringPlatform\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsFirewall\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsNetworkVirtualization\":[\"Win32_NetworkManagement\"],\"Win32_Networking\":[\"Win32\"],\"Win32_Networking_ActiveDirectory\":[\"Win32_Networking\"],\"Win32_Networking_Clustering\":[\"Win32_Networking\"],\"Win32_Networking_HttpServer\":[\"Win32_Networking\"],\"Win32_Networking_Ldap\":[\"Win32_Networking\"],\"Win32_Networking_WebSocket\":[\"Win32_Networking\"],\"Win32_Networking_WinHttp\":[\"Win32_Networking\"],\"Win32_Networking_WinInet\":[\"Win32_Networking\"],\"Win32_Networking_WinSock\":[\"Win32_Networking\"],\"Win32_Networking_WindowsWebServices\":[\"Win32_Networking\"],\"Win32_Security\":[\"Win32\"],\"Win32_Security_AppLocker\":[\"Win32_Security\"],\"Win32_Security_Authentication\":[\"Win32_Security\"],\"Win32_Security_Authentication_Identity\":[\"Win32_Security_Authentication\"],\"Win32_Security_Authorization\":[\"Win32_Security\"],\"Win32_Security_Credentials\":[\"Win32_Security\"],\"Win32_Security_Cryptography\":[\"Win32_Security\"],\"Win32_Security_Cryptography_Catalog\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_Certificates\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_Sip\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_UI\":[\"Win32_Security_Cryptography\"],\"Win32_Security_DiagnosticDataQuery\":[\"Win32_Security\"],\"Win32_Security_DirectoryServices\":[\"Win32_Security\"],\"Win32_Security_EnterpriseData\":[\"Win32_Security\"],\"Win32_Security_ExtensibleAuthenticationProtocol\":[\"Win32_Security\"],\"Win32_Security_Isolation\":[\"Win32_Security\"],\"Win32_Security_LicenseProtection\":[\"Win32_Security\"],\"Win32_Security_NetworkAccessProtection\":[\"Win32_Security\"],\"Win32_Security_WinTrust\":[\"Win32_Security\"],\"Win32_Security_WinWlx\":[\"Win32_Security\"],\"Win32_Storage\":[\"Win32\"],\"Win32_Storage_Cabinets\":[\"Win32_Storage\"],\"Win32_Storage_CloudFilters\":[\"Win32_Storage\"],\"Win32_Storage_Compression\":[\"Win32_Storage\"],\"Win32_Storage_DistributedFileSystem\":[\"Win32_Storage\"],\"Win32_Storage_FileHistory\":[\"Win32_Storage\"],\"Win32_Storage_FileSystem\":[\"Win32_Storage\"],\"Win32_Storage_Imapi\":[\"Win32_Storage\"],\"Win32_Storage_IndexServer\":[\"Win32_Storage\"],\"Win32_Storage_InstallableFileSystems\":[\"Win32_Storage\"],\"Win32_Storage_IscsiDisc\":[\"Win32_Storage\"],\"Win32_Storage_Jet\":[\"Win32_Storage\"],\"Win32_Storage_Nvme\":[\"Win32_Storage\"],\"Win32_Storage_OfflineFiles\":[\"Win32_Storage\"],\"Win32_Storage_OperationRecorder\":[\"Win32_Storage\"],\"Win32_Storage_Packaging\":[\"Win32_Storage\"],\"Win32_Storage_Packaging_Appx\":[\"Win32_Storage_Packaging\"],\"Win32_Storage_ProjectedFileSystem\":[\"Win32_Storage\"],\"Win32_Storage_StructuredStorage\":[\"Win32_Storage\"],\"Win32_Storage_Vhd\":[\"Win32_Storage\"],\"Win32_Storage_Xps\":[\"Win32_Storage\"],\"Win32_System\":[\"Win32\"],\"Win32_System_AddressBook\":[\"Win32_System\"],\"Win32_System_Antimalware\":[\"Win32_System\"],\"Win32_System_ApplicationInstallationAndServicing\":[\"Win32_System\"],\"Win32_System_ApplicationVerifier\":[\"Win32_System\"],\"Win32_System_ClrHosting\":[\"Win32_System\"],\"Win32_System_Com\":[\"Win32_System\"],\"Win32_System_Com_Marshal\":[\"Win32_System_Com\"],\"Win32_System_Com_StructuredStorage\":[\"Win32_System_Com\"],\"Win32_System_Com_Urlmon\":[\"Win32_System_Com\"],\"Win32_System_ComponentServices\":[\"Win32_System\"],\"Win32_System_Console\":[\"Win32_System\"],\"Win32_System_CorrelationVector\":[\"Win32_System\"],\"Win32_System_DataExchange\":[\"Win32_System\"],\"Win32_System_DeploymentServices\":[\"Win32_System\"],\"Win32_System_DeveloperLicensing\":[\"Win32_System\"],\"Win32_System_Diagnostics\":[\"Win32_System\"],\"Win32_System_Diagnostics_Ceip\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_Debug\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_Debug_Extensions\":[\"Win32_System_Diagnostics_Debug\"],\"Win32_System_Diagnostics_Etw\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_ProcessSnapshotting\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_ToolHelp\":[\"Win32_System_Diagnostics\"],\"Win32_System_DistributedTransactionCoordinator\":[\"Win32_System\"],\"Win32_System_Environment\":[\"Win32_System\"],\"Win32_System_ErrorReporting\":[\"Win32_System\"],\"Win32_System_EventCollector\":[\"Win32_System\"],\"Win32_System_EventLog\":[\"Win32_System\"],\"Win32_System_EventNotificationService\":[\"Win32_System\"],\"Win32_System_GroupPolicy\":[\"Win32_System\"],\"Win32_System_HostCompute\":[\"Win32_System\"],\"Win32_System_HostComputeNetwork\":[\"Win32_System\"],\"Win32_System_HostComputeSystem\":[\"Win32_System\"],\"Win32_System_Hypervisor\":[\"Win32_System\"],\"Win32_System_IO\":[\"Win32_System\"],\"Win32_System_Iis\":[\"Win32_System\"],\"Win32_System_Ioctl\":[\"Win32_System\"],\"Win32_System_JobObjects\":[\"Win32_System\"],\"Win32_System_Js\":[\"Win32_System\"],\"Win32_System_Kernel\":[\"Win32_System\"],\"Win32_System_LibraryLoader\":[\"Win32_System\"],\"Win32_System_Mailslots\":[\"Win32_System\"],\"Win32_System_Mapi\":[\"Win32_System\"],\"Win32_System_Memory\":[\"Win32_System\"],\"Win32_System_Memory_NonVolatile\":[\"Win32_System_Memory\"],\"Win32_System_MessageQueuing\":[\"Win32_System\"],\"Win32_System_MixedReality\":[\"Win32_System\"],\"Win32_System_Ole\":[\"Win32_System\"],\"Win32_System_PasswordManagement\":[\"Win32_System\"],\"Win32_System_Performance\":[\"Win32_System\"],\"Win32_System_Performance_HardwareCounterProfiling\":[\"Win32_System_Performance\"],\"Win32_System_Pipes\":[\"Win32_System\"],\"Win32_System_Power\":[\"Win32_System\"],\"Win32_System_ProcessStatus\":[\"Win32_System\"],\"Win32_System_Recovery\":[\"Win32_System\"],\"Win32_System_Registry\":[\"Win32_System\"],\"Win32_System_RemoteDesktop\":[\"Win32_System\"],\"Win32_System_RemoteManagement\":[\"Win32_System\"],\"Win32_System_RestartManager\":[\"Win32_System\"],\"Win32_System_Restore\":[\"Win32_System\"],\"Win32_System_Rpc\":[\"Win32_System\"],\"Win32_System_Search\":[\"Win32_System\"],\"Win32_System_Search_Common\":[\"Win32_System_Search\"],\"Win32_System_SecurityCenter\":[\"Win32_System\"],\"Win32_System_Services\":[\"Win32_System\"],\"Win32_System_SetupAndMigration\":[\"Win32_System\"],\"Win32_System_Shutdown\":[\"Win32_System\"],\"Win32_System_StationsAndDesktops\":[\"Win32_System\"],\"Win32_System_SubsystemForLinux\":[\"Win32_System\"],\"Win32_System_SystemInformation\":[\"Win32_System\"],\"Win32_System_SystemServices\":[\"Win32_System\"],\"Win32_System_Threading\":[\"Win32_System\"],\"Win32_System_Time\":[\"Win32_System\"],\"Win32_System_TpmBaseServices\":[\"Win32_System\"],\"Win32_System_UserAccessLogging\":[\"Win32_System\"],\"Win32_System_Variant\":[\"Win32_System\"],\"Win32_System_VirtualDosMachines\":[\"Win32_System\"],\"Win32_System_WindowsProgramming\":[\"Win32_System\"],\"Win32_System_Wmi\":[\"Win32_System\"],\"Win32_UI\":[\"Win32\"],\"Win32_UI_Accessibility\":[\"Win32_UI\"],\"Win32_UI_ColorSystem\":[\"Win32_UI\"],\"Win32_UI_Controls\":[\"Win32_UI\"],\"Win32_UI_Controls_Dialogs\":[\"Win32_UI_Controls\"],\"Win32_UI_HiDpi\":[\"Win32_UI\"],\"Win32_UI_Input\":[\"Win32_UI\"],\"Win32_UI_Input_Ime\":[\"Win32_UI_Input\"],\"Win32_UI_Input_KeyboardAndMouse\":[\"Win32_UI_Input\"],\"Win32_UI_Input_Pointer\":[\"Win32_UI_Input\"],\"Win32_UI_Input_Touch\":[\"Win32_UI_Input\"],\"Win32_UI_Input_XboxController\":[\"Win32_UI_Input\"],\"Win32_UI_InteractionContext\":[\"Win32_UI\"],\"Win32_UI_Magnification\":[\"Win32_UI\"],\"Win32_UI_Shell\":[\"Win32_UI\"],\"Win32_UI_Shell_PropertiesSystem\":[\"Win32_UI_Shell\"],\"Win32_UI_TabletPC\":[\"Win32_UI\"],\"Win32_UI_TextServices\":[\"Win32_UI\"],\"Win32_UI_WindowsAndMessaging\":[\"Win32_UI\"],\"Win32_Web\":[\"Win32\"],\"Win32_Web_InternetExplorer\":[\"Win32_Web\"],\"default\":[],\"docs\":[]}}", + "windows-sys_0.61.2": "{\"dependencies\":[{\"default_features\":false,\"name\":\"windows-link\",\"req\":\"^0.2.1\"}],\"features\":{\"Wdk\":[\"Win32_Foundation\"],\"Wdk_Devices\":[\"Wdk\"],\"Wdk_Devices_Bluetooth\":[\"Wdk_Devices\"],\"Wdk_Devices_HumanInterfaceDevice\":[\"Wdk_Devices\"],\"Wdk_Foundation\":[\"Wdk\"],\"Wdk_Graphics\":[\"Wdk\"],\"Wdk_Graphics_Direct3D\":[\"Wdk_Graphics\"],\"Wdk_NetworkManagement\":[\"Wdk\"],\"Wdk_NetworkManagement_Ndis\":[\"Wdk_NetworkManagement\"],\"Wdk_NetworkManagement_WindowsFilteringPlatform\":[\"Wdk_NetworkManagement\"],\"Wdk_Storage\":[\"Wdk\"],\"Wdk_Storage_FileSystem\":[\"Wdk_Storage\"],\"Wdk_Storage_FileSystem_Minifilters\":[\"Wdk_Storage_FileSystem\"],\"Wdk_System\":[\"Wdk\"],\"Wdk_System_IO\":[\"Wdk_System\"],\"Wdk_System_Memory\":[\"Wdk_System\"],\"Wdk_System_OfflineRegistry\":[\"Wdk_System\"],\"Wdk_System_Registry\":[\"Wdk_System\"],\"Wdk_System_SystemInformation\":[\"Wdk_System\"],\"Wdk_System_SystemServices\":[\"Wdk_System\"],\"Wdk_System_Threading\":[\"Wdk_System\"],\"Win32\":[\"Win32_Foundation\"],\"Win32_Data\":[\"Win32\"],\"Win32_Data_HtmlHelp\":[\"Win32_Data\"],\"Win32_Data_RightsManagement\":[\"Win32_Data\"],\"Win32_Devices\":[\"Win32\"],\"Win32_Devices_AllJoyn\":[\"Win32_Devices\"],\"Win32_Devices_Beep\":[\"Win32_Devices\"],\"Win32_Devices_BiometricFramework\":[\"Win32_Devices\"],\"Win32_Devices_Bluetooth\":[\"Win32_Devices\"],\"Win32_Devices_Cdrom\":[\"Win32_Devices\"],\"Win32_Devices_Communication\":[\"Win32_Devices\"],\"Win32_Devices_DeviceAndDriverInstallation\":[\"Win32_Devices\"],\"Win32_Devices_DeviceQuery\":[\"Win32_Devices\"],\"Win32_Devices_Display\":[\"Win32_Devices\"],\"Win32_Devices_Dvd\":[\"Win32_Devices\"],\"Win32_Devices_Enumeration\":[\"Win32_Devices\"],\"Win32_Devices_Enumeration_Pnp\":[\"Win32_Devices_Enumeration\"],\"Win32_Devices_Fax\":[\"Win32_Devices\"],\"Win32_Devices_HumanInterfaceDevice\":[\"Win32_Devices\"],\"Win32_Devices_Nfc\":[\"Win32_Devices\"],\"Win32_Devices_Nfp\":[\"Win32_Devices\"],\"Win32_Devices_PortableDevices\":[\"Win32_Devices\"],\"Win32_Devices_Properties\":[\"Win32_Devices\"],\"Win32_Devices_Pwm\":[\"Win32_Devices\"],\"Win32_Devices_Sensors\":[\"Win32_Devices\"],\"Win32_Devices_SerialCommunication\":[\"Win32_Devices\"],\"Win32_Devices_Tapi\":[\"Win32_Devices\"],\"Win32_Devices_Usb\":[\"Win32_Devices\"],\"Win32_Devices_WebServicesOnDevices\":[\"Win32_Devices\"],\"Win32_Foundation\":[\"Win32\"],\"Win32_Gaming\":[\"Win32\"],\"Win32_Globalization\":[\"Win32\"],\"Win32_Graphics\":[\"Win32\"],\"Win32_Graphics_Dwm\":[\"Win32_Graphics\"],\"Win32_Graphics_Gdi\":[\"Win32_Graphics\"],\"Win32_Graphics_GdiPlus\":[\"Win32_Graphics\"],\"Win32_Graphics_Hlsl\":[\"Win32_Graphics\"],\"Win32_Graphics_OpenGL\":[\"Win32_Graphics\"],\"Win32_Graphics_Printing\":[\"Win32_Graphics\"],\"Win32_Graphics_Printing_PrintTicket\":[\"Win32_Graphics_Printing\"],\"Win32_Management\":[\"Win32\"],\"Win32_Management_MobileDeviceManagementRegistration\":[\"Win32_Management\"],\"Win32_Media\":[\"Win32\"],\"Win32_Media_Audio\":[\"Win32_Media\"],\"Win32_Media_DxMediaObjects\":[\"Win32_Media\"],\"Win32_Media_KernelStreaming\":[\"Win32_Media\"],\"Win32_Media_Multimedia\":[\"Win32_Media\"],\"Win32_Media_Streaming\":[\"Win32_Media\"],\"Win32_Media_WindowsMediaFormat\":[\"Win32_Media\"],\"Win32_NetworkManagement\":[\"Win32\"],\"Win32_NetworkManagement_Dhcp\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Dns\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_InternetConnectionWizard\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_IpHelper\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Multicast\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Ndis\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetBios\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetManagement\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetShell\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetworkDiagnosticsFramework\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_P2P\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_QoS\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Rras\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Snmp\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WNet\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WebDav\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WiFi\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsConnectionManager\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsFilteringPlatform\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsFirewall\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsNetworkVirtualization\":[\"Win32_NetworkManagement\"],\"Win32_Networking\":[\"Win32\"],\"Win32_Networking_ActiveDirectory\":[\"Win32_Networking\"],\"Win32_Networking_Clustering\":[\"Win32_Networking\"],\"Win32_Networking_HttpServer\":[\"Win32_Networking\"],\"Win32_Networking_Ldap\":[\"Win32_Networking\"],\"Win32_Networking_WebSocket\":[\"Win32_Networking\"],\"Win32_Networking_WinHttp\":[\"Win32_Networking\"],\"Win32_Networking_WinInet\":[\"Win32_Networking\"],\"Win32_Networking_WinSock\":[\"Win32_Networking\"],\"Win32_Networking_WindowsWebServices\":[\"Win32_Networking\"],\"Win32_Security\":[\"Win32\"],\"Win32_Security_AppLocker\":[\"Win32_Security\"],\"Win32_Security_Authentication\":[\"Win32_Security\"],\"Win32_Security_Authentication_Identity\":[\"Win32_Security_Authentication\"],\"Win32_Security_Authorization\":[\"Win32_Security\"],\"Win32_Security_Credentials\":[\"Win32_Security\"],\"Win32_Security_Cryptography\":[\"Win32_Security\"],\"Win32_Security_Cryptography_Catalog\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_Certificates\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_Sip\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_UI\":[\"Win32_Security_Cryptography\"],\"Win32_Security_DiagnosticDataQuery\":[\"Win32_Security\"],\"Win32_Security_DirectoryServices\":[\"Win32_Security\"],\"Win32_Security_EnterpriseData\":[\"Win32_Security\"],\"Win32_Security_ExtensibleAuthenticationProtocol\":[\"Win32_Security\"],\"Win32_Security_Isolation\":[\"Win32_Security\"],\"Win32_Security_LicenseProtection\":[\"Win32_Security\"],\"Win32_Security_NetworkAccessProtection\":[\"Win32_Security\"],\"Win32_Security_WinTrust\":[\"Win32_Security\"],\"Win32_Security_WinWlx\":[\"Win32_Security\"],\"Win32_Storage\":[\"Win32\"],\"Win32_Storage_Cabinets\":[\"Win32_Storage\"],\"Win32_Storage_CloudFilters\":[\"Win32_Storage\"],\"Win32_Storage_Compression\":[\"Win32_Storage\"],\"Win32_Storage_DistributedFileSystem\":[\"Win32_Storage\"],\"Win32_Storage_FileHistory\":[\"Win32_Storage\"],\"Win32_Storage_FileSystem\":[\"Win32_Storage\"],\"Win32_Storage_Imapi\":[\"Win32_Storage\"],\"Win32_Storage_IndexServer\":[\"Win32_Storage\"],\"Win32_Storage_InstallableFileSystems\":[\"Win32_Storage\"],\"Win32_Storage_IscsiDisc\":[\"Win32_Storage\"],\"Win32_Storage_Jet\":[\"Win32_Storage\"],\"Win32_Storage_Nvme\":[\"Win32_Storage\"],\"Win32_Storage_OfflineFiles\":[\"Win32_Storage\"],\"Win32_Storage_OperationRecorder\":[\"Win32_Storage\"],\"Win32_Storage_Packaging\":[\"Win32_Storage\"],\"Win32_Storage_Packaging_Appx\":[\"Win32_Storage_Packaging\"],\"Win32_Storage_ProjectedFileSystem\":[\"Win32_Storage\"],\"Win32_Storage_StructuredStorage\":[\"Win32_Storage\"],\"Win32_Storage_Vhd\":[\"Win32_Storage\"],\"Win32_Storage_Xps\":[\"Win32_Storage\"],\"Win32_System\":[\"Win32\"],\"Win32_System_AddressBook\":[\"Win32_System\"],\"Win32_System_Antimalware\":[\"Win32_System\"],\"Win32_System_ApplicationInstallationAndServicing\":[\"Win32_System\"],\"Win32_System_ApplicationVerifier\":[\"Win32_System\"],\"Win32_System_ClrHosting\":[\"Win32_System\"],\"Win32_System_Com\":[\"Win32_System\"],\"Win32_System_Com_Marshal\":[\"Win32_System_Com\"],\"Win32_System_Com_StructuredStorage\":[\"Win32_System_Com\"],\"Win32_System_Com_Urlmon\":[\"Win32_System_Com\"],\"Win32_System_ComponentServices\":[\"Win32_System\"],\"Win32_System_Console\":[\"Win32_System\"],\"Win32_System_CorrelationVector\":[\"Win32_System\"],\"Win32_System_DataExchange\":[\"Win32_System\"],\"Win32_System_DeploymentServices\":[\"Win32_System\"],\"Win32_System_DeveloperLicensing\":[\"Win32_System\"],\"Win32_System_Diagnostics\":[\"Win32_System\"],\"Win32_System_Diagnostics_Ceip\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_Debug\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_Debug_Extensions\":[\"Win32_System_Diagnostics_Debug\"],\"Win32_System_Diagnostics_Etw\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_ProcessSnapshotting\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_ToolHelp\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_TraceLogging\":[\"Win32_System_Diagnostics\"],\"Win32_System_DistributedTransactionCoordinator\":[\"Win32_System\"],\"Win32_System_Environment\":[\"Win32_System\"],\"Win32_System_ErrorReporting\":[\"Win32_System\"],\"Win32_System_EventCollector\":[\"Win32_System\"],\"Win32_System_EventLog\":[\"Win32_System\"],\"Win32_System_EventNotificationService\":[\"Win32_System\"],\"Win32_System_GroupPolicy\":[\"Win32_System\"],\"Win32_System_HostCompute\":[\"Win32_System\"],\"Win32_System_HostComputeNetwork\":[\"Win32_System\"],\"Win32_System_HostComputeSystem\":[\"Win32_System\"],\"Win32_System_Hypervisor\":[\"Win32_System\"],\"Win32_System_IO\":[\"Win32_System\"],\"Win32_System_Iis\":[\"Win32_System\"],\"Win32_System_Ioctl\":[\"Win32_System\"],\"Win32_System_JobObjects\":[\"Win32_System\"],\"Win32_System_Js\":[\"Win32_System\"],\"Win32_System_Kernel\":[\"Win32_System\"],\"Win32_System_LibraryLoader\":[\"Win32_System\"],\"Win32_System_Mailslots\":[\"Win32_System\"],\"Win32_System_Mapi\":[\"Win32_System\"],\"Win32_System_Memory\":[\"Win32_System\"],\"Win32_System_Memory_NonVolatile\":[\"Win32_System_Memory\"],\"Win32_System_MessageQueuing\":[\"Win32_System\"],\"Win32_System_MixedReality\":[\"Win32_System\"],\"Win32_System_Ole\":[\"Win32_System\"],\"Win32_System_PasswordManagement\":[\"Win32_System\"],\"Win32_System_Performance\":[\"Win32_System\"],\"Win32_System_Performance_HardwareCounterProfiling\":[\"Win32_System_Performance\"],\"Win32_System_Pipes\":[\"Win32_System\"],\"Win32_System_Power\":[\"Win32_System\"],\"Win32_System_ProcessStatus\":[\"Win32_System\"],\"Win32_System_Recovery\":[\"Win32_System\"],\"Win32_System_Registry\":[\"Win32_System\"],\"Win32_System_RemoteDesktop\":[\"Win32_System\"],\"Win32_System_RemoteManagement\":[\"Win32_System\"],\"Win32_System_RestartManager\":[\"Win32_System\"],\"Win32_System_Restore\":[\"Win32_System\"],\"Win32_System_Rpc\":[\"Win32_System\"],\"Win32_System_Search\":[\"Win32_System\"],\"Win32_System_Search_Common\":[\"Win32_System_Search\"],\"Win32_System_SecurityCenter\":[\"Win32_System\"],\"Win32_System_Services\":[\"Win32_System\"],\"Win32_System_SetupAndMigration\":[\"Win32_System\"],\"Win32_System_Shutdown\":[\"Win32_System\"],\"Win32_System_StationsAndDesktops\":[\"Win32_System\"],\"Win32_System_SubsystemForLinux\":[\"Win32_System\"],\"Win32_System_SystemInformation\":[\"Win32_System\"],\"Win32_System_SystemServices\":[\"Win32_System\"],\"Win32_System_Threading\":[\"Win32_System\"],\"Win32_System_Time\":[\"Win32_System\"],\"Win32_System_TpmBaseServices\":[\"Win32_System\"],\"Win32_System_UserAccessLogging\":[\"Win32_System\"],\"Win32_System_Variant\":[\"Win32_System\"],\"Win32_System_VirtualDosMachines\":[\"Win32_System\"],\"Win32_System_WindowsProgramming\":[\"Win32_System\"],\"Win32_System_Wmi\":[\"Win32_System\"],\"Win32_UI\":[\"Win32\"],\"Win32_UI_Accessibility\":[\"Win32_UI\"],\"Win32_UI_ColorSystem\":[\"Win32_UI\"],\"Win32_UI_Controls\":[\"Win32_UI\"],\"Win32_UI_Controls_Dialogs\":[\"Win32_UI_Controls\"],\"Win32_UI_HiDpi\":[\"Win32_UI\"],\"Win32_UI_Input\":[\"Win32_UI\"],\"Win32_UI_Input_Ime\":[\"Win32_UI_Input\"],\"Win32_UI_Input_KeyboardAndMouse\":[\"Win32_UI_Input\"],\"Win32_UI_Input_Pointer\":[\"Win32_UI_Input\"],\"Win32_UI_Input_Touch\":[\"Win32_UI_Input\"],\"Win32_UI_Input_XboxController\":[\"Win32_UI_Input\"],\"Win32_UI_InteractionContext\":[\"Win32_UI\"],\"Win32_UI_Magnification\":[\"Win32_UI\"],\"Win32_UI_Shell\":[\"Win32_UI\"],\"Win32_UI_Shell_Common\":[\"Win32_UI_Shell\"],\"Win32_UI_Shell_PropertiesSystem\":[\"Win32_UI_Shell\"],\"Win32_UI_TabletPC\":[\"Win32_UI\"],\"Win32_UI_TextServices\":[\"Win32_UI\"],\"Win32_UI_WindowsAndMessaging\":[\"Win32_UI\"],\"Win32_Web\":[\"Win32\"],\"Win32_Web_InternetExplorer\":[\"Win32_Web\"],\"default\":[],\"docs\":[]}}", + "windows-targets_0.52.6": "{\"dependencies\":[{\"name\":\"windows_aarch64_gnullvm\",\"req\":\"^0.52.6\",\"target\":\"aarch64-pc-windows-gnullvm\"},{\"name\":\"windows_aarch64_msvc\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(target_arch = \\\"aarch64\\\", target_env = \\\"msvc\\\", not(windows_raw_dylib)))\"},{\"name\":\"windows_i686_gnu\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(target_arch = \\\"x86\\\", target_env = \\\"gnu\\\", not(target_abi = \\\"llvm\\\"), not(windows_raw_dylib)))\"},{\"name\":\"windows_i686_gnullvm\",\"req\":\"^0.52.6\",\"target\":\"i686-pc-windows-gnullvm\"},{\"name\":\"windows_i686_msvc\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(target_arch = \\\"x86\\\", target_env = \\\"msvc\\\", not(windows_raw_dylib)))\"},{\"name\":\"windows_x86_64_gnu\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(target_arch = \\\"x86_64\\\", target_env = \\\"gnu\\\", not(target_abi = \\\"llvm\\\"), not(windows_raw_dylib)))\"},{\"name\":\"windows_x86_64_gnullvm\",\"req\":\"^0.52.6\",\"target\":\"x86_64-pc-windows-gnullvm\"},{\"name\":\"windows_x86_64_msvc\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(any(target_arch = \\\"x86_64\\\", target_arch = \\\"arm64ec\\\"), target_env = \\\"msvc\\\", not(windows_raw_dylib)))\"}],\"features\":{}}", + "windows_aarch64_gnullvm_0.52.6": "{\"dependencies\":[],\"features\":{}}", + "windows_aarch64_msvc_0.52.6": "{\"dependencies\":[],\"features\":{}}", + "windows_i686_gnu_0.52.6": "{\"dependencies\":[],\"features\":{}}", + "windows_i686_gnullvm_0.52.6": "{\"dependencies\":[],\"features\":{}}", + "windows_i686_msvc_0.52.6": "{\"dependencies\":[],\"features\":{}}", + "windows_x86_64_gnu_0.52.6": "{\"dependencies\":[],\"features\":{}}", + "windows_x86_64_gnullvm_0.52.6": "{\"dependencies\":[],\"features\":{}}", + "windows_x86_64_msvc_0.52.6": "{\"dependencies\":[],\"features\":{}}", + "wit-bindgen-core_0.51.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.72\"},{\"features\":[\"derive\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4.3.19\"},{\"name\":\"heck\",\"req\":\"^0.5\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.218\"},{\"name\":\"wit-parser\",\"req\":\"^0.244.0\"}],\"features\":{\"clap\":[\"dep:clap\"],\"serde\":[\"dep:serde\"]}}", + "wit-bindgen-rt_0.39.0": "{\"dependencies\":[{\"name\":\"bitflags\",\"optional\":true,\"req\":\"^2.3.3\"},{\"name\":\"futures\",\"optional\":true,\"req\":\"^0.3.30\"},{\"name\":\"once_cell\",\"optional\":true,\"req\":\"^1.19.0\"}],\"features\":{\"async\":[\"dep:futures\",\"dep:once_cell\"]}}", + "wit-bindgen-rust-macro_0.51.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.72\"},{\"name\":\"prettyplease\",\"req\":\"^0.2.20\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"printing\"],\"name\":\"syn\",\"req\":\"^2.0.89\"},{\"name\":\"wit-bindgen-core\",\"req\":\"^0.51.0\"},{\"name\":\"wit-bindgen-rust\",\"req\":\"^0.51.0\"}],\"features\":{\"async\":[]}}", + "wit-bindgen-rust_0.51.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.72\"},{\"kind\":\"dev\",\"name\":\"bytes\",\"req\":\"^1\"},{\"features\":[\"derive\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4.3.19\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.31\"},{\"name\":\"heck\",\"req\":\"^0.5\"},{\"name\":\"indexmap\",\"req\":\"^2.0.0\"},{\"name\":\"prettyplease\",\"req\":\"^0.2.20\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.218\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"features\":[\"printing\"],\"name\":\"syn\",\"req\":\"^2.0.89\"},{\"default_features\":false,\"name\":\"wasm-metadata\",\"req\":\"^0.244.0\"},{\"name\":\"wit-bindgen-core\",\"req\":\"^0.51.0\"},{\"name\":\"wit-component\",\"req\":\"^0.244.0\"}],\"features\":{\"clap\":[\"dep:clap\",\"wit-bindgen-core/clap\"],\"serde\":[\"dep:serde\",\"wit-bindgen-core/serde\"]}}", + "wit-bindgen_0.51.0": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0\"},{\"name\":\"bitflags\",\"optional\":true,\"req\":\"^2.3.3\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"futures\",\"optional\":true,\"req\":\"^0.3.30\"},{\"name\":\"wit-bindgen-rust-macro\",\"optional\":true,\"req\":\"^0.51.0\"}],\"features\":{\"async\":[\"std\",\"wit-bindgen-rust-macro?/async\"],\"async-spawn\":[\"async\",\"dep:futures\"],\"bitflags\":[\"dep:bitflags\"],\"default\":[\"macros\",\"realloc\",\"async\",\"std\",\"bitflags\"],\"inter-task-wakeup\":[\"async\"],\"macros\":[\"dep:wit-bindgen-rust-macro\"],\"realloc\":[],\"rustc-dep-of-std\":[\"dep:core\",\"dep:alloc\"],\"std\":[]}}", + "wit-component_0.244.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"name\":\"bitflags\",\"req\":\"^2.3.3\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"kind\":\"dev\",\"name\":\"glob\",\"req\":\"^0.3.0\"},{\"default_features\":false,\"name\":\"indexmap\",\"req\":\"^2.7.0\"},{\"kind\":\"dev\",\"name\":\"libtest-mimic\",\"req\":\"^0.8.1\"},{\"name\":\"log\",\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.3.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"req\":\"^1.0.166\"},{\"name\":\"serde_derive\",\"req\":\"^1.0.166\"},{\"name\":\"serde_json\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"std\",\"wasmparser\"],\"name\":\"wasm-encoder\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"name\":\"wasm-metadata\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"oci\"],\"kind\":\"dev\",\"name\":\"wasm-metadata\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"simd\",\"std\",\"component-model\",\"simd\"],\"name\":\"wasmparser\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"simd\",\"std\",\"component-model\",\"features\"],\"kind\":\"dev\",\"name\":\"wasmparser\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"component-model\"],\"kind\":\"dev\",\"name\":\"wasmprinter\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"cranelift\",\"component-model\",\"runtime\",\"gc-drc\"],\"kind\":\"dev\",\"name\":\"wasmtime\",\"req\":\"^34.0.1\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"default_features\":false,\"name\":\"wast\",\"optional\":true,\"req\":\"^244.0.0\"},{\"default_features\":false,\"name\":\"wat\",\"optional\":true,\"req\":\"^1.244.0\"},{\"default_features\":false,\"features\":[\"component-model\"],\"kind\":\"dev\",\"name\":\"wat\",\"req\":\"^1.244.0\"},{\"features\":[\"decoding\",\"serde\"],\"name\":\"wit-parser\",\"req\":\"^0.244.0\"}],\"features\":{\"dummy-module\":[\"dep:wat\"],\"semver-check\":[\"dummy-module\"],\"wat\":[\"dep:wast\",\"dep:wat\"]}}", + "wit-parser_0.244.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"id-arena\",\"req\":\"^2\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"indexmap\",\"req\":\"^2.7.0\"},{\"kind\":\"dev\",\"name\":\"libtest-mimic\",\"req\":\"^0.8.1\"},{\"name\":\"log\",\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.3.0\"},{\"default_features\":false,\"name\":\"semver\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"name\":\"unicode-xid\",\"req\":\"^0.2.2\"},{\"default_features\":false,\"features\":[\"simd\",\"std\",\"validate\",\"component-model\",\"features\"],\"name\":\"wasmparser\",\"optional\":true,\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"component-model\"],\"name\":\"wat\",\"optional\":true,\"req\":\"^1.244.0\"}],\"features\":{\"decoding\":[\"dep:wasmparser\"],\"default\":[\"serde\",\"decoding\"],\"serde\":[\"dep:serde\",\"dep:serde_derive\",\"indexmap/serde\",\"serde_json\"],\"wat\":[\"decoding\",\"dep:wat\"]}}", + "zerocopy-derive_0.7.35": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.10\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"name\":\"syn\",\"req\":\"^2.0.31\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.85\"}],\"features\":{}}", + "zerocopy-derive_0.8.26": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"dissimilar\",\"req\":\"^1.0.9\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"=0.2.163\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"=1.9\"},{\"kind\":\"dev\",\"name\":\"prettyplease\",\"req\":\"=0.2.17\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.10\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.89\"}],\"features\":{}}", + "zerocopy-derive_0.8.42": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"dissimilar\",\"req\":\"^1.0.9\"},{\"kind\":\"dev\",\"name\":\"prettyplease\",\"req\":\"=0.2.17\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.40\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"features\":[\"visit\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.46\"}],\"features\":{}}", + "zerocopy_0.7.35": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"assert_matches\",\"req\":\"^1.5\"},{\"default_features\":false,\"name\":\"byteorder\",\"optional\":true,\"req\":\"^1.3\"},{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.85\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.7.35\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.7.35\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.7.35\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\"],\"alloc\":[],\"default\":[\"byteorder\"],\"derive\":[\"zerocopy-derive\"],\"simd\":[],\"simd-nightly\":[\"simd\"]}}", + "zerocopy_0.8.26": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"either\",\"req\":\"=1.13.0\"},{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.89\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.8.26\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\",\"std\"],\"alloc\":[],\"derive\":[\"zerocopy-derive\"],\"float-nightly\":[],\"simd\":[],\"simd-nightly\":[\"simd\"],\"std\":[\"alloc\"]}}", + "zerocopy_0.8.42": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.8.42\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.8.42\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.8.42\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\",\"std\"],\"alloc\":[],\"derive\":[\"zerocopy-derive\"],\"float-nightly\":[],\"simd\":[],\"simd-nightly\":[\"simd\"],\"std\":[\"alloc\"]}}", + "zmij_1.0.21": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\",\"target\":\"cfg(not(miri))\"},{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1.36\"},{\"kind\":\"dev\",\"name\":\"num-bigint\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"num-integer\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"num_cpus\",\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"opt-level\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"ryu\",\"req\":\"^1\"}],\"features\":{}}" + }, + "@@rules_rs+//rs/experimental/toolchains:module_extension.bzl%toolchains": { + "cargo-1.94.0-aarch64-apple-darwin.tar.xz": "7497ab873993cf38b806548c85bf818ffd71d1c3cf77b5bb4c3d3f1769588a08", + "cargo-1.94.0-aarch64-pc-windows-msvc.tar.xz": "0eacb6af29cacb850af4b390e698b2c8060f639a70e6b9f0575cdcc82b0822b2", + "cargo-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "d57ab7b23b52cba46e58c78327904d0c7db19892988677b0d59495c8af4bb790", + "cargo-1.94.0-x86_64-apple-darwin.tar.xz": "3beb7874010ebd757780284c0c9da47fc9f0b18091d7415d7b7231a8c19ec44f", + "cargo-1.94.0-x86_64-pc-windows-msvc.tar.xz": "6cf7113e1d5633721739a3bd6fee7148d6394bbab748fbede941210100374741", + "cargo-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "8e17624f3de39e079845bfb25ed15a042f4b50ceca78e37c56c4b9b15949b9f7", + "clippy-1.94.0-aarch64-apple-darwin.tar.xz": "42a683cbac0b81b96ad3b887cdade36a57ef3a131c203583ca7e3ce0be98f415", + "clippy-1.94.0-aarch64-pc-windows-msvc.tar.xz": "44468016e27457640c7ab62b7ceaddb44308e8ec77ee4344c1bd36113448c8eb", + "clippy-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "f7450c414e394450b53be602d37c374e2a2c5a2f51be2b54faf3d1e3272160b1", + "clippy-1.94.0-x86_64-apple-darwin.tar.xz": "1393cc688bc4d784a9d66619d2eec683a1ddb30b263f03ab615c0a2b18053126", + "clippy-1.94.0-x86_64-pc-windows-msvc.tar.xz": "e4dcc5a796857589e364064493abb3afb469e0e36e90155e7b0a2c5ce4bb7503", + "clippy-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "935529b0b9150084fa9cc1734c6598efe86b6e7d76f31ad3f8d4bb79ebf6031a", + "rust-std-1.94.0-aarch64-apple-darwin.tar.xz": "d3293668128b0a7838ef2544ae341eadd4437460ca22098af723349885a10ba2", + "rust-std-1.94.0-aarch64-apple-ios-macabi.tar.xz": "914f085065539ea1038d2832a5af1c22852cd5e9a063356b20e46698e72e770e", + "rust-std-1.94.0-aarch64-apple-ios-sim.tar.xz": "b4d16842b8f98ff2f431d6ab4aced298f4028ca00f0f5d588bc6efc94faa0680", + "rust-std-1.94.0-aarch64-apple-ios.tar.xz": "41ec1d18f4762f85231a19cf8aba3385da78fece2ff900a435c62881a3a3b44b", + "rust-std-1.94.0-aarch64-linux-android.tar.xz": "402c80df672a4b9fc948b6dc8bde21a63e1093eb1c8a2c2bd42221c309d9eba2", + "rust-std-1.94.0-aarch64-pc-windows-gnullvm.tar.xz": "8c89521f6826a1d2bf07b27c95405f4d6bf7c24f3de832c02758c41d4c1a511f", + "rust-std-1.94.0-aarch64-pc-windows-msvc.tar.xz": "d7964debd444dca92e771c953c3cd582fb927031b536c1439e1a831014d5f0d1", + "rust-std-1.94.0-aarch64-unknown-fuchsia.tar.xz": "87e11d97055540a0675bdc3dacf3d3d8cca5f806171ad0b51d0427f7cb9510c9", + "rust-std-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "c781b3ef4fefa5508fbe05820eddc95e46351d905a30921cc020febd9c596a2e", + "rust-std-1.94.0-aarch64-unknown-linux-musl.tar.xz": "3513032b2953d795bd3ac1489de4e8174e72500a31b4e7d1b349a23ab78a7a7b", + "rust-std-1.94.0-aarch64-unknown-none-softfloat.tar.xz": "375c496af31cf208ce67613c0aa1bde1894b5d674b00050cd46f7f954ed4a668", + "rust-std-1.94.0-aarch64-unknown-none.tar.xz": "854a9f8cea0ba889f553a7ea664c2a6479fc1fc49a4b56f84f99fc37001c4322", + "rust-std-1.94.0-aarch64-unknown-uefi.tar.xz": "f038d9e3df557d4aecdcf65989896179bcf0ecbc4d62bde71d33d7b914d3c253", + "rust-std-1.94.0-arm-linux-androideabi.tar.xz": "d2113ef50ed0b974f33b41102b961588c26abaf34f45a289a293b3d31ba01113", + "rust-std-1.94.0-arm-unknown-linux-gnueabi.tar.xz": "ad364b54b668e866315900ebff0d225616c556ab5228599792470aede05ba42c", + "rust-std-1.94.0-arm-unknown-linux-gnueabihf.tar.xz": "d5db943974fe05306ae43d100b0b7f3540108b93ef52de670675ce966af8a0fd", + "rust-std-1.94.0-arm-unknown-linux-musleabi.tar.xz": "c724b1a1b5dddeca563cda9b9695610bb56e681727b3af0a352174960087069a", + "rust-std-1.94.0-arm-unknown-linux-musleabihf.tar.xz": "9f8b707760a632b80864e91ea81a1b19c3f5452ff5e18b96da75792be1cf0a3c", + "rust-std-1.94.0-armv7-linux-androideabi.tar.xz": "e0b6883364f939300c281d96d6778ecf7fd97fe867b11936f0aa5c9a29f488ad", + "rust-std-1.94.0-armv7-unknown-linux-gnueabi.tar.xz": "9b1d8143881d510785d89413ea09b9fe910b64e3f300ab30e52e7fdd652ea66a", + "rust-std-1.94.0-armv7-unknown-linux-gnueabihf.tar.xz": "4b1744a8e48f58c2741afc1e076d0629a86051d785bd36e2c97f329eab6a5b49", + "rust-std-1.94.0-armv7-unknown-linux-musleabi.tar.xz": "fc2754f50ece0c7d3d32b2e0034b19c8b8bba9b471e0eb00f8ece7fa0bcb2837", + "rust-std-1.94.0-armv7-unknown-linux-musleabihf.tar.xz": "abfc0e7c1eaff1e3332b5c6262d12da309035a228261d07155add44464ed52f4", + "rust-std-1.94.0-i686-linux-android.tar.xz": "9093a894d87568a4b5eb4a144758bf49d4b62743677f747848afc3807c23f126", + "rust-std-1.94.0-i686-pc-windows-gnu.tar.xz": "188490bde6649163347050a70fd81c6d9e2e972ef12ba646f7f4fa106e21a956", + "rust-std-1.94.0-i686-pc-windows-gnullvm.tar.xz": "ba61dc038c3f4102a832a3381935192e88b0a1f79ff50b6d5590d476bdfa6879", + "rust-std-1.94.0-i686-pc-windows-msvc.tar.xz": "388cc2bb17ef9f12864c08e462cdbe3222d5ddafd84969dcf4ed944da27efcbd", + "rust-std-1.94.0-i686-unknown-freebsd.tar.xz": "3ce634092bcecb000c793c9d4bb0ccc6fd45c66fa8259fbb4f18c59844c4b13b", + "rust-std-1.94.0-i686-unknown-linux-gnu.tar.xz": "528c1dda491b25bb855426f48757773f4740c75d3c7271efffe7412c2fd5730d", + "rust-std-1.94.0-i686-unknown-linux-musl.tar.xz": "90ff16d879f774048c9b81d07188af4b63ed73a31848759cf560a9f51baa175f", + "rust-std-1.94.0-i686-unknown-uefi.tar.xz": "8c095069ec09d6e1d0b7e00f340fd7ea925a31acbad845d5956e1bc4a7d47733", + "rust-std-1.94.0-powerpc-unknown-linux-gnu.tar.xz": "23e32cab484d7787b7966a0508c49baf231bcdb4a79a421ee85ca69e81f2a06f", + "rust-std-1.94.0-powerpc64-unknown-linux-gnu.tar.xz": "7713b676a37fbd0a52a711c2e3c205a917f8b05614e17cd9f7f37874a88469a3", + "rust-std-1.94.0-powerpc64le-unknown-linux-gnu.tar.xz": "094f3955828d6630b70a9c7c71e3165c3ffc7a301929601dde08501def623347", + "rust-std-1.94.0-powerpc64le-unknown-linux-musl.tar.xz": "f3956b29463ec58d730f054ff2334755f929c602181ad67a25af9725c8500f68", + "rust-std-1.94.0-riscv32imc-unknown-none-elf.tar.xz": "5d10c7ab74cd5d10451480ec894ac5b98940a8527c333465eeec4a45b1125d43", + "rust-std-1.94.0-riscv64gc-unknown-linux-gnu.tar.xz": "32e9b7006f1269e48fb65af6298f6e8a222e1a8a2a20b112528bfb3dad2f65a8", + "rust-std-1.94.0-riscv64gc-unknown-linux-musl.tar.xz": "2c63d21143c7f1d44f3e3b1c9a82f3101576344c3fd1feb2d43ffc9d035e94f1", + "rust-std-1.94.0-riscv64gc-unknown-none-elf.tar.xz": "071a1ae54f7dead1b8147b9bae7d831a5554a37a1133f8cd52b265a765595a12", + "rust-std-1.94.0-s390x-unknown-linux-gnu.tar.xz": "ea69833addbc9521664464418c7468871f7a80b16fd2c385f6213a6c99cedfe9", + "rust-std-1.94.0-thumbv6m-none-eabi.tar.xz": "b022df94f6efa3bd2a6c4042472e114d2325bf058983b3ac7d586e24c8d77f25", + "rust-std-1.94.0-thumbv7em-none-eabi.tar.xz": "41894db05c4d8f421bf0dba7d0bed995685032393779f92b4ea1f3e6468badfb", + "rust-std-1.94.0-thumbv7em-none-eabihf.tar.xz": "be7e81535e7118fbad58a5dc31a1acfa7f2a36a2348aed23595169418559d67f", + "rust-std-1.94.0-thumbv7m-none-eabi.tar.xz": "4702e615be264df97b120fd075bed4e46710cc9a03c48115ce2ee3abfbf3822e", + "rust-std-1.94.0-thumbv8m.main-none-eabi.tar.xz": "5c1fb5ed8ad31580241b6e095583a2cac0ef90ea29dad7705c8a0ce0933a7797", + "rust-std-1.94.0-thumbv8m.main-none-eabihf.tar.xz": "573e9570bea039316ae3b55e323cbc741617ace1e0b87df8f3ecbbdf34b3de8a", + "rust-std-1.94.0-wasm32-unknown-emscripten.tar.xz": "0a607ba5d78afb47e9f5f134633ed5a954ece5558298e081084abe44f5265463", + "rust-std-1.94.0-wasm32-unknown-unknown.tar.xz": "bc65af3c0bfa2e5b20e299358710a26ce33a3674c762a2a010bc383c5b7fc202", + "rust-std-1.94.0-wasm32-wasip1-threads.tar.xz": "311b9fe1e94c90c16695899e670f0d76a366d493cdda29008191ce2d436cd22c", + "rust-std-1.94.0-wasm32-wasip1.tar.xz": "acbacacdca3e0ec0d93ff46d5f3579ae8cd5133b86e31a893da22cd94e28d0bf", + "rust-std-1.94.0-wasm32-wasip2.tar.xz": "e947e730da3fea212453b2835765802d9b0c80e19405cab188a6854834196e6c", + "rust-std-1.94.0-x86_64-apple-darwin.tar.xz": "cf716b5a5fab007398ce76536a85a3449df074b66e7c762c0f5de1d6c56a1342", + "rust-std-1.94.0-x86_64-apple-ios-macabi.tar.xz": "f39f4c1121dafcf8ea70a21c91be9552e30ea76debcad9bff28be945aaa79282", + "rust-std-1.94.0-x86_64-apple-ios.tar.xz": "60e70b80469dd3900dd8b1705831f5f02ac44fa97088b8a81fdda110ed421091", + "rust-std-1.94.0-x86_64-linux-android.tar.xz": "488c8a581612ac4779a05f05967b76a96e0ff0bdd1ab8d99c13a859c19056b01", + "rust-std-1.94.0-x86_64-pc-windows-gnu.tar.xz": "c9f98d62213d0559c8d2ee6d7eafbafbf5dfe04921c9e7cb15b66a770903bc33", + "rust-std-1.94.0-x86_64-pc-windows-gnullvm.tar.xz": "54c28d0030fab585457a431c555e5d23502132364d0483e5b3b2dfbfc3929cc6", + "rust-std-1.94.0-x86_64-pc-windows-msvc.tar.xz": "f8e2b43e52957e8224d97de68d6ec5da01613eb98bce4199de80071678b5fda0", + "rust-std-1.94.0-x86_64-unknown-freebsd.tar.xz": "081972f2f386ee859c15ed619695a9440896a80d1814d0e29d7620b7ef96207c", + "rust-std-1.94.0-x86_64-unknown-fuchsia.tar.xz": "6c18112553cf72cf991352208cce2bc96c712ca7b209acdd098abac894d674ef", + "rust-std-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "dd33653107c36e040082050d9e547e64dac5b456ba74069430d838c00c189a05", + "rust-std-1.94.0-x86_64-unknown-linux-musl.tar.xz": "b6786b5c490df2dcef76829039177799058f94e17725b42f912a057c0f6224f2", + "rust-std-1.94.0-x86_64-unknown-netbsd.tar.xz": "f763e56ca1ce3f819fb2136638d774777dd696f836cc52e6564431859a0f5159", + "rust-std-1.94.0-x86_64-unknown-none.tar.xz": "8509489165a8aa60a0590aeaa668f63423b3e7a33bb93e8e5719ff5404b6f6aa", + "rust-std-1.94.0-x86_64-unknown-uefi.tar.xz": "31b56e8178dd2319e51ccc474959d247db2767cda7149fe3704d636df9aee1c9", + "rustc-1.94.0-aarch64-apple-darwin.tar.xz": "6ba00a4f486af11826335cd7fe405592d959f50afb9489176fc1463cbe75dbf2", + "rustc-1.94.0-aarch64-pc-windows-msvc.tar.xz": "6d88d89c200a147357ded65e9bc426e14ebb6f5046d9772f3916b6a1ac60cdd5", + "rustc-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "7261b2fcdb88aef8b21f4b9f915d9a75c959674193f5b38e13fab4569684fc5b", + "rustc-1.94.0-x86_64-apple-darwin.tar.xz": "b1374b0fb63f5abcd8eb65607e43a499bfdc5e8160746d1f6f993aee9313f320", + "rustc-1.94.0-x86_64-pc-windows-msvc.tar.xz": "e0f59503778b563d3ee6d4edecd52b44056779ffe456181bad1fb4dc069802e6", + "rustc-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "31a0d3ac9383dfdeb4fce86eeed5ade3230131c635264c0eab7252dbf235f28e", + "rustfmt-1.94.0-aarch64-apple-darwin.tar.xz": "0eb716d201dc0d6d24ef8207d3ef27611d9d15d2212e3b27aa11e97286ed02ac", + "rustfmt-1.94.0-aarch64-pc-windows-msvc.tar.xz": "705ac3e797c4abed9e722e0864902f7a03872a8a201dcd7b70ff1b749eed2327", + "rustfmt-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "1ac17dbe7f8f6c98db5b62cb149fcd10c7565d624a59471115fc40055db9af28", + "rustfmt-1.94.0-x86_64-apple-darwin.tar.xz": "e58601a10ee85e956611ea2814bf26de88934a8224c1663bb11aa23f727663ec", + "rustfmt-1.94.0-x86_64-pc-windows-msvc.tar.xz": "7b32d8b0dd99f9bf6ddaa4866d39b7b96e8c2db46644d8527096f7f34b4fbe26", + "rustfmt-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "903e768f7626f78ed2d6cb82c1820b93bd339afec8575102e847fd3a379d33e5" + } + } } diff --git a/zml/tokenizer/BUILD.bazel b/zml/tokenizer/BUILD.bazel index ce2761e3..92082ff5 100644 --- a/zml/tokenizer/BUILD.bazel +++ b/zml/tokenizer/BUILD.bazel @@ -1,15 +1,17 @@ load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_test") +load("@zml//bazel:zig_srcs.bzl", "zig_srcs") zig_library( name = "tokenizer", + tags = ["manual"], srcs = ["homemade.zig"], import_name = "zml/tokenizer", main = "tokenizer.zig", - tags = ["manual"], visibility = ["//visibility:public"], deps = [ "//ffi:zig", "//stdx", + "//zml/tokenizer/hftokenizers", "//zml/tokenizer/iree", "//zml/tokenizer/sentencepiece", ], @@ -28,7 +30,7 @@ zig_binary( zig_test( name = "test", deps = [ - ":tokenizer", + ":tokenizer" ], ) diff --git a/zml/tokenizer/hftokenizers/.gitignore b/zml/tokenizer/hftokenizers/.gitignore new file mode 100644 index 00000000..2f7896d1 --- /dev/null +++ b/zml/tokenizer/hftokenizers/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/zml/tokenizer/hftokenizers/BUILD.bazel b/zml/tokenizer/hftokenizers/BUILD.bazel new file mode 100644 index 00000000..43ee1fbe --- /dev/null +++ b/zml/tokenizer/hftokenizers/BUILD.bazel @@ -0,0 +1,35 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_rs//rs:rust_static_library.bzl", "rust_static_library") +load("@rules_zig//zig:defs.bzl", "zig_library") + +rust_static_library( + name = "hftokenizers_rs", + srcs = ["hftokenizers.rs"], + crate_name = "zml_tokenizer_hftokenizers", + edition = "2021", + proc_macro_deps = ["@crates//:tokenizers"], + deps = ["@crates//:tokenizers"], +) + +cc_library( + name = "hftokenizers_cc", + tags = ["manual"], + hdrs = ["hftokenizers.h"], + visibility = ["//visibility:public"], + deps = [ + ":hftokenizers_rs", + "//ffi:cc", + ], +) + +zig_library( + name = "hftokenizers", + tags = ["manual"], + main = "hftokenizers.zig", + visibility = ["//visibility:public"], + deps = [ + ":hftokenizers_cc", + "//ffi:zig", + "//stdx", + ], +) diff --git a/zml/tokenizer/hftokenizers/Cargo.lock b/zml/tokenizer/hftokenizers/Cargo.lock new file mode 100644 index 00000000..7164e77b --- /dev/null +++ b/zml/tokenizer/hftokenizers/Cargo.lock @@ -0,0 +1,713 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "serde", + "version_check", + "zerocopy 0.8.26", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "bitflags" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.2.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "compact_str" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "darling" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dary_heap" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728" +dependencies = [ + "serde", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn", +] + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" +dependencies = [ + "cc", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" + +[[package]] +name = "libc" +version = "0.2.169" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "macro_rules_attribute" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "monostate" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e" +dependencies = [ + "monostate-impl", + "serde", +] + +[[package]] +name = "monostate-impl" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "onig" +version = "6.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0" +dependencies = [ + "bitflags", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy 0.7.35", +] + +[[package]] +name = "proc-macro2" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools", + "rayon", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rustversion" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "serde" +version = "1.0.216" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.216" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.134" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64", + "nom", + "serde", + "unicode-segmentation", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokenizers" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223" +dependencies = [ + "ahash", + "aho-corasick", + "compact_str", + "dary_heap", + "derive_builder", + "esaxx-rs", + "getrandom", + "itertools", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + +[[package]] +name = "unicode-ident" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" + +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive 0.7.35", +] + +[[package]] +name = "zerocopy" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" +dependencies = [ + "zerocopy-derive 0.8.26", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zml_tokenizer_hftokenizers" +version = "0.1.0" +dependencies = [ + "tokenizers", +] diff --git a/zml/tokenizer/hftokenizers/Cargo.toml b/zml/tokenizer/hftokenizers/Cargo.toml new file mode 100644 index 00000000..f905c004 --- /dev/null +++ b/zml/tokenizer/hftokenizers/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "zml_tokenizer_hftokenizers" +version = "0.1.0" +edition = "2021" + +[dependencies] +tokenizers = { version = "0.22.2", default-features = false, features = [ + # We don't need/want the progressbar feature + "onig", + "esaxx_fast", +] } + +[lib] +name = "zml_tokenizer_hftokenizers" +path = "hftokenizers.rs" diff --git a/zml/tokenizer/hftokenizers/hftokenizers.h b/zml/tokenizer/hftokenizers/hftokenizers.h new file mode 100644 index 00000000..f779b575 --- /dev/null +++ b/zml/tokenizer/hftokenizers/hftokenizers.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include + +#include "ffi/zig_slice.h" + +typedef struct hftokenizers hftokenizers; + +hftokenizers *hftokenizers_new(zig_slice); +hftokenizers *hftokenizers_new_from_bytes(zig_slice); +void hftokenizers_drop(hftokenizers *tokenizer); +zig_slice hftokenizers_encode(hftokenizers *tokenizer, zig_slice text); +void hftokenizers_tokens_drop(zig_slice tokens); +zig_slice hftokenizers_decode(hftokenizers *tokenizer, zig_slice tokens); +void hftokenizers_str_drop(zig_slice text); +uint32_t hftokenizers_token_to_id(hftokenizers *tokenizer, zig_slice token); diff --git a/zml/tokenizer/hftokenizers/hftokenizers.rs b/zml/tokenizer/hftokenizers/hftokenizers.rs new file mode 100644 index 00000000..048283cc --- /dev/null +++ b/zml/tokenizer/hftokenizers/hftokenizers.rs @@ -0,0 +1,109 @@ +#[repr(C)] +struct ZigSlice { + ptr: *mut T, + len: usize, +} + +impl ZigSlice { + fn as_slice(&self) -> &[T] { + unsafe { std::slice::from_raw_parts(self.ptr, self.len) } + } + + fn as_slice_mut(&self) -> &mut [T] { + unsafe { std::slice::from_raw_parts_mut(self.ptr, self.len) } + } +} + +#[no_mangle] +extern "C" fn hftokenizers_new(path: ZigSlice) -> *mut tokenizers::Tokenizer { + return Box::into_raw(Box::new( + tokenizers::Tokenizer::from_file(std::path::Path::new( + std::str::from_utf8(path.as_slice()).unwrap(), + )) + .unwrap(), + )); +} + +#[no_mangle] +extern "C" fn hftokenizers_new_from_bytes(bytes: ZigSlice) -> *mut tokenizers::Tokenizer { + return Box::into_raw(Box::new( + tokenizers::Tokenizer::from_bytes(bytes.as_slice()) + .unwrap(), + )); +} + +#[no_mangle] +extern "C" fn hftokenizers_drop(t: *mut tokenizers::Tokenizer) { + drop(unsafe { Box::from_raw(t) }); +} + +#[no_mangle] +extern "C" fn hftokenizers_encode( + t: *mut tokenizers::Tokenizer, + string: ZigSlice, +) -> ZigSlice { + let input_str = std::str::from_utf8(string.as_slice()).unwrap(); + let encoded = unsafe { t.as_ref() } + .unwrap() + .encode_fast(input_str, false) + .unwrap(); + + // Convert the result to a boxed slice + let mut ids: Box<[u32]> = encoded.get_ids().to_owned().into_boxed_slice(); + + // Retrieve the zig slice associated to the boxed slice. + let slice = ZigSlice { + ptr: ids.as_mut_ptr(), + len: ids.len(), + }; + + // Leak the box so that it's not deallocated. + Box::leak(ids); + + return slice; +} + +#[no_mangle] +extern "C" fn hftokenizers_tokens_drop(tokens: ZigSlice) { + // Reconstruct the Box from the zig slice so that it's dropped. + drop(unsafe { Box::from_raw(tokens.as_slice_mut()) }); +} + +#[no_mangle] +extern "C" fn hftokenizers_decode( + t: *mut tokenizers::Tokenizer, + ids: ZigSlice, +) -> ZigSlice { + let decoded = unsafe { t.as_ref() } + .unwrap() + .decode(ids.as_slice(), false) + .unwrap(); + + // Convert the result to a boxed slice + let mut string: Box<[u8]> = decoded.into_bytes().into_boxed_slice(); + + // Retrieve the zig slice associated to the boxed slice. + let slice = ZigSlice { + ptr: string.as_mut_ptr(), + len: string.len(), + }; + + // Leak the box so that it's not deallocated. + Box::leak(string); + + return slice; +} + +#[no_mangle] +extern "C" fn hftokenizers_str_drop(tokens: ZigSlice) { + drop(unsafe { Box::from_raw(tokens.as_slice_mut()) }); +} + +#[no_mangle] +extern "C" fn hftokenizers_token_to_id(t: *mut tokenizers::Tokenizer, token: ZigSlice) -> u32 { + let id = unsafe { t.as_ref() } + .unwrap() + .token_to_id(std::str::from_utf8(token.as_slice()).unwrap()) + .unwrap_or(u32::MAX); + return id; +} diff --git a/zml/tokenizer/hftokenizers/hftokenizers.zig b/zml/tokenizer/hftokenizers/hftokenizers.zig new file mode 100644 index 00000000..f5829ce6 --- /dev/null +++ b/zml/tokenizer/hftokenizers/hftokenizers.zig @@ -0,0 +1,120 @@ +const std = @import("std"); + +const c = @import("c"); +const ffi = @import("ffi"); +const stdx = @import("stdx"); + +pub const Encoder = struct { + inner: *HFTokenizer, + current_ids: ?[]const u32 = null, + + fn init(inner: *HFTokenizer) Encoder { + return .{ .inner = inner }; + } + + pub fn reset(self: *Encoder) void { + if (self.current_ids) |current_ids_| { + c.hftokenizers_tokens_drop(ffi.ZigSlice.from(current_ids_)); + self.current_ids = null; + } + } + + pub fn deinit(self: *Encoder) void { + self.reset(); + } + + pub fn encode(self: *Encoder, input: []const u8) ![]const u32 { + self.reset(); + self.current_ids = ffi.ZigSlice.to(u32, c.hftokenizers_encode(@ptrCast(self.inner), ffi.ZigSlice.from(input))); + return self.ids(); + } + + pub fn ids(self: *const Encoder) []const u32 { + return self.current_ids orelse &.{}; + } +}; + +pub const Decoder = struct { + const StringBuffer = stdx.BoundedArray(u8, 128); + const TokensIdsBuffer = stdx.BoundedArray(u32, 4); + + inner: *HFTokenizer, + current_string: ?[]const u8 = null, + last_string: StringBuffer = .{ .len = 0 }, + last_token_ids: TokensIdsBuffer = .{ .len = 0 }, + + fn init(inner: *HFTokenizer) Decoder { + return .{ .inner = inner }; + } + + pub fn deinit(self: *Decoder) void { + self.reset(); + } + + pub fn reset(self: *Decoder) void { + if (self.current_string) |current_string_| { + c.hftokenizers_str_drop(ffi.ZigSlice.from(current_string_)); + self.current_string = null; + } + } + + pub fn decode(self: *Decoder, ids: []const u32) ![]const u8 { + self.reset(); + self.current_string = ffi.ZigSlice.to(u8, c.hftokenizers_decode(@ptrCast(self.inner), ffi.ZigSlice.from(ids))); + return self.string(); + } + + pub fn string(self: *const Decoder) []const u8 { + return self.current_string orelse &.{}; + } + + pub fn next(self: *Decoder, token_id: u32) !?[]const u8 { + if (self.last_token_ids.len >= self.last_token_ids.capacity()) { + _ = self.last_token_ids.orderedRemove(0); + } + self.last_token_ids.appendAssumeCapacity(token_id); + const new_string = try self.decode(self.last_token_ids.constSlice()); + if (self.last_string.len == 0) { + self.last_string = try StringBuffer.fromSlice(new_string); + return new_string; + } + var view = try std.unicode.Utf8View.init(self.last_string.constSlice()); + var it = view.iterator(); + while (it.nextCodepointSlice()) |cp| { + const start = it.i - cp.len; + if (std.mem.startsWith(u8, new_string, self.last_string.constSlice()[start..])) { + const chunk = new_string[self.last_string.len - start ..]; + self.last_string = try StringBuffer.fromSlice(new_string); + return chunk; + } + } + return null; + } +}; + +pub const HFTokenizer = opaque { + pub fn fromFile(model: []const u8) !*HFTokenizer { + return @ptrCast(c.hftokenizers_new(ffi.ZigSlice.from(model))); + } + + pub fn fromBytes(bytes: []const u8) !*HFTokenizer { + return @ptrCast(c.hftokenizers_new_from_bytes(ffi.ZigSlice.from(bytes))); + } + + pub fn deinit(self: *HFTokenizer) void { + return c.hftokenizers_drop(@ptrCast(self)); + } + + pub fn encoder(self: *HFTokenizer) !Encoder { + return Encoder.init(self); + } + + pub fn decoder(self: *HFTokenizer) !Decoder { + return Decoder.init(self); + } + + pub fn tokenId(self: *HFTokenizer, token: []const u8) ?u32 { + const id = c.hftokenizers_token_to_id(@ptrCast(self), ffi.ZigSlice.from(token)); + return if (id == std.math.maxInt(u32)) null else id; + } +}; diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig index 8e8fea28..33f7d2e6 100644 --- a/zml/tokenizer/tokenizer.zig +++ b/zml/tokenizer/tokenizer.zig @@ -1,5 +1,6 @@ const std = @import("std"); +pub const hftokenizers = @import("hftokenizers"); pub const iree = @import("iree"); pub const sentencepiece = @import("sentencepiece"); @@ -8,6 +9,7 @@ pub const homemade = @import("homemade.zig"); const log = std.log.scoped(.@"zml/tokenizer"); const Tokenizers = enum { + hftokenizers, iree, sentencepiece, homemade, @@ -15,6 +17,7 @@ const Tokenizers = enum { pub const Tokenizer = union(Tokenizers) { pub const Encoder = union(Tokenizers) { + hftokenizers: hftokenizers.Encoder, iree: iree.Tokenizer.Encoder, sentencepiece: sentencepiece.Encoder, homemade: homemade.Encoder, @@ -171,6 +174,7 @@ pub const Tokenizer = union(Tokenizers) { }; pub const Decoder = union(Tokenizers) { + hftokenizers: hftokenizers.Decoder, iree: iree.Tokenizer.Decoder, sentencepiece: sentencepiece.Decoder, homemade: homemade.Decoder, @@ -241,24 +245,28 @@ pub const Tokenizer = union(Tokenizers) { } }; + hftokenizers: *hftokenizers.HFTokenizer, iree: iree.Tokenizer, sentencepiece: *sentencepiece.SentencePieceProcessor, homemade: *homemade.Tokenizer, pub fn fromFile(allocator: std.mem.Allocator, io: std.Io, model: []const u8) !Tokenizer { + _ = io; + _ = allocator; if (std.mem.endsWith(u8, model, ".pb")) { return .{ .sentencepiece = try .fromFile(model) }; } if (std.mem.endsWith(u8, model, ".json")) { - return .{ .iree = try .fromFile(allocator, io, model) }; + return .{ .hftokenizers = try .fromFile(model) }; } return error.InvalidArgument; } pub fn fromBytes(allocator: std.mem.Allocator, bytes: []const u8) !Tokenizer { + _ = allocator; if (bytes[0] == '{') { - return .{ .iree = try .fromBytes(allocator, bytes) }; + return .{ .hftokenizers = try .fromBytes(bytes) }; } return .{ .sentencepiece = try .fromBytes(bytes) }; } From e47b7ea390efe8a5822f122d3ebf67a5ccb4c3f0 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 14 Apr 2026 09:44:05 +0200 Subject: [PATCH 421/881] zml/nn: add proportional scaling option to RopeOpts and related tests (#471) Co-authored-by: Kevin Barre --- zml/nn.zig | 117 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/zml/nn.zig b/zml/nn.zig index 7d01bc97..7eafc705 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -156,6 +156,7 @@ pub const RopeOpts = struct { llama3: Llama3, yarn: Yarn, linear: Scaling.Linear, + proportional: Scaling.Proportional, pub const Default = struct { rope_theta: f32 = 10000, @@ -188,6 +189,11 @@ pub const RopeOpts = struct { rope_theta: f32 = 10000, }; + pub const Proportional = struct { + partial_rotary_factor: f32, + rope_theta: f32 = 10000, + }; + /// Read a Rope scaling config from HF config.json format. pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !Scaling { return jsonParseFromValue( @@ -210,10 +216,14 @@ pub const RopeOpts = struct { if (std.mem.eql(u8, impl.string, "llama3")) { // Note: leaky is fine here cause Llama3 struct don't need to allocate memory. return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; + } else if (std.mem.eql(u8, impl.string, "default")) { + return .{ .default = try std.json.parseFromValueLeaky(Default, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; } else if (std.mem.eql(u8, impl.string, "yarn")) { return .{ .yarn = try std.json.parseFromValueLeaky(Yarn, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; } else if (std.mem.eql(u8, impl.string, "linear")) { return .{ .linear = try std.json.parseFromValueLeaky(Scaling.Linear, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; + } else if (std.mem.eql(u8, impl.string, "proportional")) { + return .{ .proportional = try std.json.parseFromValueLeaky(Scaling.Proportional, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; } else { log.warn("Unsupported Rope implementation: {s}, will use the default one which will produce altered results", .{impl.string}); return .{ .default = .{} }; @@ -342,6 +352,13 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { .linear => |l| { for (inv_freq) |*f| f.* /= l.factor; }, + .proportional => |p| { + const rotary_fraction = std.math.clamp(p.partial_rotary_factor, 0.0, 1.0); + const rotary_frequencies: usize = @intFromFloat(@floor(@as(f32, @floatFromInt(N)) * rotary_fraction)); + for (inv_freq[rotary_frequencies..]) |*f| { + f.* = 0; + } + }, .llama3 => |s| { // https://arxiv.org/pdf/2309.16039 // After Llama2 they observed that the rope frequencies where too sharp and hurting long distance attention. @@ -439,6 +456,106 @@ test "invFreq Yarn" { try std.testing.expectApproxEqRel(1.3465735902799727, yarn_conf.scaling.attentionScaling(), 1e-5); } +test "RopeOpts.Scaling parses proportional" { + const json = + \\{ + \\ "rope_type": "proportional", + \\ "partial_rotary_factor": 0.25, + \\ "rope_theta": 1000000 + \\} + ; + var value = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, json, .{}); + defer value.deinit(); + + const scaling = try RopeOpts.Scaling.jsonParseFromValue(std.testing.allocator, value.value, .{}); + switch (scaling) { + .proportional => |p| { + try std.testing.expectApproxEqRel(0.25, p.partial_rotary_factor, 1e-6); + try std.testing.expectApproxEqRel(1_000_000, p.rope_theta, 1e-6); + }, + else => try std.testing.expect(false), + } +} + +test "RopeOpts.Scaling parses default rope type" { + const json = + \\{ + \\ "rope_type": "default", + \\ "rope_theta": 10000 + \\} + ; + var value = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, json, .{}); + defer value.deinit(); + + const scaling = try RopeOpts.Scaling.jsonParseFromValue(std.testing.allocator, value.value, .{}); + switch (scaling) { + .default => |d| { + try std.testing.expectApproxEqRel(10_000, d.rope_theta, 1e-6); + }, + else => try std.testing.expect(false), + } +} + +test "invFreq Proportional" { + const proportional_conf: RopeOpts = .{ + .scaling = .{ .proportional = .{ + .partial_rotary_factor = 0.25, + .rope_theta = 1_000_000, + } }, + }; + const default_conf: RopeOpts = .{ + .scaling = .{ .default = .{ + .rope_theta = 1_000_000, + } }, + }; + + var inv_freq: [16]f32 = undefined; + var default_inv_freq: [16]f32 = undefined; + _invFreq(proportional_conf, &inv_freq); + _invFreq(default_conf, &default_inv_freq); + + for (0..4) |i| { + try std.testing.expectApproxEqRel(default_inv_freq[i], inv_freq[i], 1e-6); + } + for (4..inv_freq.len) |i| { + try std.testing.expectEqual(@as(f32, 0.0), inv_freq[i]); + } +} + +test "invFreq Proportional clamps partial rotary factor" { + const default_conf: RopeOpts = .{ + .scaling = .{ .default = .{ + .rope_theta = 10_000, + } }, + }; + const negative_conf: RopeOpts = .{ + .scaling = .{ .proportional = .{ + .partial_rotary_factor = -0.5, + .rope_theta = 10_000, + } }, + }; + const overfull_conf: RopeOpts = .{ + .scaling = .{ .proportional = .{ + .partial_rotary_factor = 1.5, + .rope_theta = 10_000, + } }, + }; + + var default_inv_freq: [8]f32 = undefined; + var negative_inv_freq: [8]f32 = undefined; + var overfull_inv_freq: [8]f32 = undefined; + _invFreq(default_conf, &default_inv_freq); + _invFreq(negative_conf, &negative_inv_freq); + _invFreq(overfull_conf, &overfull_inv_freq); + + for (negative_inv_freq) |f| { + try std.testing.expectEqual(@as(f32, 0.0), f); + } + for (default_inv_freq, overfull_inv_freq) |expected, actual| { + try std.testing.expectApproxEqRel(expected, actual, 1e-6); + } +} + test "real/img" { const platform = zml.testing.env(); From 6ad3e267907a9bb67e00e96f0f643ae9df8a1256 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 14 Apr 2026 09:48:32 +0200 Subject: [PATCH 422/881] zml/attention: add option to set the softmax scaling (#470) I added a `scale` field in the `AttentionOptions` struct that is passed through the attention backend. I had to add support for optional in the custom call metadata. For now it only supports optional floats, we'll add support for other types later. --- zml/attention/flashattn.zig | 3 +++ zml/attention/paged_attention.zig | 1 + zml/attention/triton.zig | 6 ++++-- zml/ops.zig | 26 ++++++++++++++++++++------ 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 80ea0ee8..eae20e04 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -950,6 +950,7 @@ pub const paged_fa2 = struct { .max_seqlen_k = context.max_seqlen_k, .num_heads = num_heads_per_shard, .window_size_left = opts.sliding_window, + .softmax_scale = opts.scale, }, .opts = zml.ops.CustomCallOptions{ .has_side_effect = false, @@ -1026,6 +1027,7 @@ pub const paged_fa2 = struct { .max_seqlen_k = context.max_seqlen_k, .num_heads = num_heads_per_shard, .window_size_left = opts.sliding_window, + .softmax_scale = opts.scale, }, .opts = zml.ops.CustomCallOptions{ .has_side_effect = false, @@ -1089,6 +1091,7 @@ pub const paged_fa2 = struct { .max_seqlen_k = context.max_seqlen_k, .num_heads = num_heads_per_shard, .window_size_left = opts.sliding_window, + .softmax_scale = opts.scale, }, .opts = zml.ops.CustomCallOptions{ .has_side_effect = false, diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index aa6f0864..0191b823 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -175,6 +175,7 @@ pub const Context = union(Backend) { pub const AttentionOptions = struct { is_causal: bool = true, sliding_window: i32 = -1, + scale: ?f32 = null, }; pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { diff --git a/zml/attention/triton.zig b/zml/attention/triton.zig index 3440948e..74533c27 100644 --- a/zml/attention/triton.zig +++ b/zml/attention/triton.zig @@ -319,6 +319,7 @@ pub const paged = struct { target_num_prgms: usize, num_2d_prgms: usize, max_seqlen_q: usize, + scale: ?f32, pub fn numQueriesPerKv(self: PagedAttentionOptions) usize { return self.num_heads / self.num_kv_heads; @@ -382,6 +383,7 @@ pub const paged = struct { .target_num_prgms = target_num_prgms, .num_2d_prgms = num_2d_prgms, .max_seqlen_q = ctx_.options.max_seqlen_q, + .scale = ctx_.opts.scale, }; const use_2d_kernel = use2dKernel( @@ -458,7 +460,7 @@ pub const paged = struct { const v_strides_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(v_strides[1])); const v_strides_2_ptr = zml.Tensor.constant(zml.DataType.i64.constant(v_strides[2])); const num_seqs_ptr = zml.Tensor.constant(zml.DataType.i32.constant(parameters.block_table.dim(0))); - const scale: f32 = @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))); + const scale: f32 = paged_attention_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))); const scale_ptr = zml.Tensor.constant(zml.DataType.f32.constant(scale)); const output = zml.ops.triton(.{ @@ -575,7 +577,7 @@ pub const paged = struct { const v_strides_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(v_strides[1])); const v_strides_2_ptr = zml.Tensor.constant(zml.DataType.i64.constant(v_strides[2])); const num_seqs_ptr = zml.Tensor.constant(zml.DataType.i32.constant(parameters.block_table.dim(0))); - const scale: f32 = @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))); + const scale: f32 = paged_attention_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))); const scale_ptr = zml.Tensor.constant(zml.DataType.f32.constant(scale)); const attn_grid: [3]i32 = .{ @intCast(config.attention.total_q_blocks), @intCast(paged_attention_opts.num_kv_heads), @intCast(config.attention.num_segments_per_seq) }; diff --git a/zml/ops.zig b/zml/ops.zig index 4e5558d4..154d581f 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1518,9 +1518,9 @@ fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs } const metadata_type_info = @typeInfo(@TypeOf(metadata)); - var metadata_attributes: [metadata_type_info.@"struct".fields.len]mlir.NamedAttribute = undefined; - inline for (metadata_type_info.@"struct".fields, 0..) |field, i| { - const attribute: *const mlir.Attribute = switch (@typeInfo(field.type)) { + var metadata_attribute_list = std.ArrayList(mlir.NamedAttribute).initCapacity(allocator, metadata_type_info.@"struct".fields.len + 2) catch unreachable; + inline for (metadata_type_info.@"struct".fields) |field| { + const maybe_attribute: ?*const mlir.Attribute = switch (@typeInfo(field.type)) { .comptime_int => mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @field(metadata, field.name))), .int => |int_field| switch (int_field.signedness) { .signed => switch (int_field.bits) { @@ -1552,15 +1552,29 @@ fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs @panic("Unsupported pointer type in metadata"), else => @panic("Unsupported pointer type in metadata"), }, + .optional => |optional_info| if (@field(metadata, field.name) != null) switch (@typeInfo(optional_info.child)) { + .float => |float_field| switch (float_field.bits) { + 16 => mlir.floatAttribute(ctx.mlir_ctx, .f16, @as(f64, @field(metadata, field.name))), + 32 => mlir.floatAttribute(ctx.mlir_ctx, .f32, @as(f64, @field(metadata, field.name))), + 64 => mlir.floatAttribute(ctx.mlir_ctx, .f64, @as(f64, @field(metadata, field.name))), + else => @panic("Unsupported DataType"), + }, + + else => @panic("Unsupported optional child type in metadata"), + } else null, else => @compileError("Unsupported metadata type: " ++ @typeName(field.type)), }; - metadata_attributes[i] = mlir.NamedAttribute.named(ctx.mlir_ctx, field.name, attribute); + if (maybe_attribute) |attribute| { + metadata_attribute_list.appendAssumeCapacity(mlir.NamedAttribute.named(ctx.mlir_ctx, field.name, attribute)); + } } - const backend_config = mlir.dictionaryAttribute(ctx.mlir_ctx, &(metadata_attributes ++ [_]mlir.NamedAttribute{ + metadata_attribute_list.appendSliceAssumeCapacity(&[_]mlir.NamedAttribute{ .named(ctx.mlir_ctx, "pjrt_api", mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_api))))), .named(ctx.mlir_ctx, "pjrt_client", mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_client))))), - })); + }); + + const backend_config = mlir.dictionaryAttribute(ctx.mlir_ctx, metadata_attribute_list.items); const operand_layouts = allocator.alloc([]const usize, input_shapes.len) catch unreachable; for (input_shapes, 0..) |shape, i| { From 35e8ae908212545e306f052735eb501dceea2f9d Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 14 Apr 2026 10:07:03 +0200 Subject: [PATCH 423/881] zml/ops: fix comptime for optional float value in custom call metadata (#472) --- zml/ops.zig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zml/ops.zig b/zml/ops.zig index 154d581f..0dc645c5 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1552,11 +1552,11 @@ fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs @panic("Unsupported pointer type in metadata"), else => @panic("Unsupported pointer type in metadata"), }, - .optional => |optional_info| if (@field(metadata, field.name) != null) switch (@typeInfo(optional_info.child)) { + .optional => |optional_info| if (@field(metadata, field.name)) |value| switch (@typeInfo(optional_info.child)) { .float => |float_field| switch (float_field.bits) { - 16 => mlir.floatAttribute(ctx.mlir_ctx, .f16, @as(f64, @field(metadata, field.name))), - 32 => mlir.floatAttribute(ctx.mlir_ctx, .f32, @as(f64, @field(metadata, field.name))), - 64 => mlir.floatAttribute(ctx.mlir_ctx, .f64, @as(f64, @field(metadata, field.name))), + 16 => mlir.floatAttribute(ctx.mlir_ctx, .f16, @as(f64, value)), + 32 => mlir.floatAttribute(ctx.mlir_ctx, .f32, @as(f64, value)), + 64 => mlir.floatAttribute(ctx.mlir_ctx, .f64, @as(f64, value)), else => @panic("Unsupported DataType"), }, From e58c6af965da2121ff1b4fa207613abfec1da3a5 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 14 Apr 2026 13:07:54 +0200 Subject: [PATCH 424/881] workspace: pin hermeticbuild/hermetic-llvm to git this is until the release is done, this fixes duplicate symbols between clang and zig compiler_rt --- MODULE.bazel | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/MODULE.bazel b/MODULE.bazel index b1ce05e7..0fa473fc 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -25,6 +25,12 @@ bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "rules_zig", version = "0.12.2") bazel_dep(name = "tar.bzl", version = "0.9.0") bazel_dep(name = "llvm", version = "0.7.0") +git_override( + module_name = "llvm", + commit = "9086d034fa7b91b61db0ad66fc78453e576299cd", + remote = "https://github.com/hermeticbuild/hermetic-llvm.git", +) + bazel_dep(name = "jsoncpp", version = "1.9.6", repo_name = "jsoncpp_git") bazel_dep(name = "with_cfg.bzl", version = "0.14.4") bazel_dep(name = "grpc", version = "1.74.0") From 46862fa85a151da56006316adcac377ad6dbad46 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 14 Apr 2026 13:09:50 +0200 Subject: [PATCH 425/881] workspace: put overrides next to the dep it's clearer, also, do a buildifier pass --- MODULE.bazel | 30 +++++++++++++++--------------- MODULE.bazel.lock | 3 --- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 0fa473fc..611ea322 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -6,12 +6,18 @@ bazel_dep(name = "abseil-cpp", version = "20250814.1", repo_name = "com_google_a bazel_dep(name = "bazel_lib", version = "3.2.2") bazel_dep(name = "aspect_rules_py", version = "1.8.4") bazel_dep(name = "bazel_skylib", version = "1.9.0") + ## Needed by iree bazel_dep(name = "libbacktrace", version = "1.0.0-20250926-7939218") bazel_dep(name = "patchelf", version = "0.18.0.bcr.1") bazel_dep(name = "pcre2", version = "10.45") bazel_dep(name = "platforms", version = "1.0.0") bazel_dep(name = "protobuf", version = "33.6", repo_name = "com_google_protobuf") +single_version_override( + module_name = "protobuf", + version = "33.6", # important, until grpc-cpp is fixed +) + bazel_dep(name = "pybind11_bazel", version = "2.13.6") bazel_dep(name = "re2", version = "2025-11-05", repo_name = "com_googlesource_code_re2") bazel_dep(name = "rules_cc", version = "0.2.16") @@ -23,6 +29,12 @@ bazel_dep(name = "rules_python", version = "1.9.0") bazel_dep(name = "rules_rs", version = "0.0.44") bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "rules_zig", version = "0.12.2") +git_override( + module_name = "rules_zig", + commit = "56911842f4527df036e8d52e78dd0f54a22d4215", + remote = "https://github.com/zml/rules_zig.git", +) + bazel_dep(name = "tar.bzl", version = "0.9.0") bazel_dep(name = "llvm", version = "0.7.0") git_override( @@ -89,7 +101,7 @@ uv.configure(version = "0.9.30") python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain(python_version = "3.12") -use_repo(python, "python_versions", "python_3_12") +use_repo(python, "python_3_12", "python_versions") pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") @@ -146,7 +158,7 @@ cuda = use_extension("//platforms/cuda:cuda.bzl", "cuda_packages") use_repo(cuda, "cuda_nvml_dev", "libpjrt_cuda") rocm = use_extension("//platforms/rocm:rocm.bzl", "rocm_packages") -use_repo(rocm, "amd-smi-lib", "hipblaslt", "libpjrt_rocm", "libdrm2-amdgpu", "libdrm-amdgpu-amdgpu1", "rocblas") +use_repo(rocm, "amd-smi-lib", "hipblaslt", "libdrm-amdgpu-amdgpu1", "libdrm2-amdgpu", "libpjrt_rocm", "rocblas") tpu = use_extension("//platforms/tpu:tpu.bzl", "tpu_packages") use_repo(tpu, "libpjrt_tpu") @@ -160,7 +172,7 @@ use_repo(zls, "zls_toolchains") register_toolchains("@zls_toolchains//:all") non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") -use_repo(non_module_deps, "arocc", "cloud_accelerator_diagnostics", "com_google_sentencepiece", "flashattn", "iree", "libvaxis", "linenoise", "mnist", "org_swig_swig", "translate-c", "stb", "uucode", "xla", "zigimg") +use_repo(non_module_deps, "arocc", "cloud_accelerator_diagnostics", "com_google_sentencepiece", "flashattn", "iree", "libvaxis", "linenoise", "mnist", "org_swig_swig", "stb", "translate-c", "uucode", "xla", "zigimg") xla = use_extension("//third_party/xla:xla.bzl", "xla") use_repo( @@ -257,18 +269,6 @@ use_repo( "distroless_cc_debian12_linux_amd64", ) - -git_override( - module_name = "rules_zig", - commit = "56911842f4527df036e8d52e78dd0f54a22d4215", - remote = "https://github.com/zml/rules_zig.git", -) - -single_version_override( - module_name = "protobuf", - version = "33.6", -) - single_version_override( module_name = "grpc-java", version = "1.74.0", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 66de42db..5c1d157d 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -151,9 +151,6 @@ "https://bcr.bazel.build/modules/libbacktrace/1.0.0-20250926-7939218/source.json": "83239989d09748f63dee4bdd430d261e217a012670e066e2238925a303a65d89", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", "https://bcr.bazel.build/modules/libpfm/4.11.0/source.json": "caaffb3ac2b59b8aac456917a4ecf3167d40478ee79f15ab7a877ec9273937c9", - "https://bcr.bazel.build/modules/llvm/0.6.7/MODULE.bazel": "d37a2e10571864dc6a5bb53c29216d90b9400bbcadb422337f49107fd2eaf0d2", - "https://bcr.bazel.build/modules/llvm/0.7.0/MODULE.bazel": "3c07a4e5734b0ad41fe24dedaacbf3a35ce4377b7e1a21f24488a0c9ac4f1e6b", - "https://bcr.bazel.build/modules/llvm/0.7.0/source.json": "5f0a015da1e7db09ef61aa254ba5eb55b0c322fadd1e8eefeca368d7723a24a0", "https://bcr.bazel.build/modules/mbedtls/3.6.0/MODULE.bazel": "8e380e4698107c5f8766264d4df92e36766248447858db28187151d884995a09", "https://bcr.bazel.build/modules/mbedtls/3.6.0/source.json": "1dbe7eb5258050afcc3806b9d43050f71c6f539ce0175535c670df606790b30c", "https://bcr.bazel.build/modules/nlohmann_json/3.11.3/MODULE.bazel": "87023db2f55fc3a9949c7b08dc711fae4d4be339a80a99d04453c4bb3998eefc", From 2f11092deb0265fd20a23c08e1253a2656317f9f Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 14 Apr 2026 13:10:33 +0200 Subject: [PATCH 426/881] workspace: correctly set zig mode on host tools this was a typo --- .bazelrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index 42f50189..e56e95be 100644 --- a/.bazelrc +++ b/.bazelrc @@ -16,7 +16,7 @@ common --enable_platform_specific_config # Ensure host tools are built optimized common --host_compilation_mode=opt -common --@rules_zig//zig/settings:mode=release_safe +common --@rules_zig//zig/settings:host_mode=release_safe # Suppress all warnings, LLVM and friends are too noisy common --copt='-w' From 91f58097cc92123ff078d0ec146e8c3a4b03bed1 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 14 Apr 2026 17:13:26 +0200 Subject: [PATCH 427/881] zml/tokenizer: use correct API for decoding with HF (#478) The current API only worked with IREE. Reworked it to work with both HF and IREE --- examples/llm/models/lfm2/session.zig | 2 +- examples/llm/models/llama/session.zig | 2 +- examples/llm/models/qwen3_5/session.zig | 2 +- zml/tokenizer/tokenizer.zig | 63 ++++++++++--------------- 4 files changed, 28 insertions(+), 41 deletions(-) diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig index 0fe2cc75..37700acf 100644 --- a/examples/llm/models/lfm2/session.zig +++ b/examples/llm/models/lfm2/session.zig @@ -145,7 +145,7 @@ pub const Session = struct { if (token_id == self.config.eos_token_id) break :generation; - const token = try decoder.feed_one(token_id); + const token = try decoder.feedOne(token_id); if (self.think_start) |think_start| if (token_id == think_start) { try stdout.writeAll("\x1b[2m"); }; diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig index 29124859..8fde6361 100644 --- a/examples/llm/models/llama/session.zig +++ b/examples/llm/models/llama/session.zig @@ -156,7 +156,7 @@ pub const Session = struct { if (isEosToken(self.config, token_id)) break :generation; - const token = try decoder.feed_one(token_id); + const token = try decoder.feedOne(token_id); try stdout.writeAll(token); try stdout.flush(); diff --git a/examples/llm/models/qwen3_5/session.zig b/examples/llm/models/qwen3_5/session.zig index e4eefce5..b055f9b9 100644 --- a/examples/llm/models/qwen3_5/session.zig +++ b/examples/llm/models/qwen3_5/session.zig @@ -117,7 +117,7 @@ pub const Session = struct { const token_id = self.generated_token_slice.items(u32)[0]; if (token_id == self.eos_token_id) break :generation; - const token = try decoder.feed_one(token_id); + const token = try decoder.feedOne(token_id); if (self.think_start) |think_start| if (token_id == think_start) { try stdout.writeAll("\x1b[2m"); }; diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig index 33f7d2e6..e95bc7c9 100644 --- a/zml/tokenizer/tokenizer.zig +++ b/zml/tokenizer/tokenizer.zig @@ -191,49 +191,36 @@ pub const Tokenizer = union(Tokenizers) { } } - // - // Convenience methods built on top of the stream API - // - pub fn decodeAlloc(self: *Decoder, allocator: std.mem.Allocator, token_ids: []const u32) !std.ArrayList(u8) { - var tokens: std.ArrayList(u8) = try .initCapacity(allocator, token_ids.len * 4); - var remaining = token_ids; - while (remaining.len > 0) { - const fout = try self.feed(remaining); - try tokens.appendSlice(allocator, fout.tokens); - remaining = remaining[fout.consumed..]; + switch (self.*) { + .iree => |*d| { + var tokens: std.ArrayList(u8) = try .initCapacity(allocator, token_ids.len * 4); + var remaining = token_ids; + while (remaining.len > 0) { + const consumed, const decoded_tokens = try d.feed(remaining); + try tokens.appendSlice(allocator, decoded_tokens); + remaining = remaining[consumed..]; + } + try tokens.appendSlice(allocator, try self.finalize()); + return tokens; + }, + inline else => |*d| { + return .fromOwnedSlice(try allocator.dupe(u8, try d.decode(token_ids))); + }, } - try tokens.appendSlice(allocator, try self.finalize()); - return tokens; - } - - // - // Stream API - // - - pub const FeedOutput = struct { - consumed: usize, - /// Only valid until the next feed/finalize call and while the decoder is alive. - tokens: []const u8, - }; - - pub fn feed(self: *Decoder, token_ids: []const u32) !FeedOutput { - return switch (self.*) { - .iree => |*v| { - const consumed, const tokens = try v.feed(token_ids); - return .{ .consumed = consumed, .tokens = tokens }; - }, - inline else => |*v| { - const tokens = try v.decode(token_ids); - return .{ .consumed = token_ids.len, .tokens = tokens }; - }, - }; } // Only valid until the next decode and while the decoder is alive. - pub fn feed_one(self: *Decoder, token_id: u32) ![]const u8 { - const fout = try self.feed(&.{token_id}); - return fout.tokens; + pub fn feedOne(self: *Decoder, token_id: u32) ![]const u8 { + switch (self.*) { + .iree => |*d| { + _, const tokens = try d.feed(&.{token_id}); + return tokens; + }, + inline else => |*d| { + return try d.next(token_id) orelse &.{}; + }, + } } /// Output is only valid until the next feed/finalize call and while the decoder is alive. From b912b08c40d3cfd4e0694bf2cab76cd0c9fb425f Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Wed, 15 Apr 2026 21:32:09 +0200 Subject: [PATCH 428/881] zml/attention/triton: sandbox zlib1g when running the triton codegen else python fails in a container --- zml/attention/triton.zig | 26 ++++++++++++++------------ zml/attention/triton/BUILD.bazel | 2 ++ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/zml/attention/triton.zig b/zml/attention/triton.zig index 74533c27..3f731608 100644 --- a/zml/attention/triton.zig +++ b/zml/attention/triton.zig @@ -201,15 +201,6 @@ pub const GenerationConfig = union(KernelKind) { reduce_segments_ptr: GenerationConfigReduce, }; -fn getGenerateBinPath(allocator: std.mem.Allocator) ![]const u8 { - const runfiles = bazel.runfiles(bazel_builtin.current_repository) catch unreachable; - - var sandbox_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = runfiles.rlocation("zml/zml/attention/triton/sandbox", &sandbox_path_buf) catch unreachable; - - return std.fs.path.join(allocator, &.{ sandbox_path.?, "bin", "generate" }); -} - fn generateTtir(allocator: std.mem.Allocator, io: std.Io, config: GenerationConfig) ![:0]const u8 { var arena: std.heap.ArenaAllocator = .init(allocator); defer arena.deinit(); @@ -649,10 +640,21 @@ pub const Runtime = struct { process: std.process.Child, pub fn init(allocator: std.mem.Allocator, io: std.Io) !Runtime { - const path = try getGenerateBinPath(allocator); - defer allocator.free(path); + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); - const process = try std.process.spawn(io, .{ .argv = &.{path}, .stdin = .pipe, .stdout = .pipe }); + const runfiles = bazel.runfiles(bazel_builtin.current_repository) catch unreachable; + const sandbox_path = runfiles.rlocationAlloc(arena.allocator(), "zml/zml/attention/triton/sandbox") catch unreachable; + + var map: std.process.Environ.Map = .init(arena.allocator()); + try map.put("LD_LIBRARY_PATH", std.Io.Dir.path.join(arena.allocator(), &.{ sandbox_path.?, "lib" }) catch unreachable); + + const process = try std.process.spawn(io, .{ + .argv = &.{std.Io.Dir.path.join(arena.allocator(), &.{ sandbox_path.?, "bin", "generate" }) catch unreachable}, + .stdin = .pipe, + .stdout = .pipe, + .environ_map = &map, + }); errdefer process.kill(io); return .{ .process = process }; diff --git a/zml/attention/triton/BUILD.bazel b/zml/attention/triton/BUILD.bazel index 83f65f8f..a1e13506 100644 --- a/zml/attention/triton/BUILD.bazel +++ b/zml/attention/triton/BUILD.bazel @@ -87,6 +87,7 @@ copy_to_directory( name = "sandbox", srcs = [ ":generate_files", + "@zlib1g", ], target_compatible_with = ["@platforms//os:linux"], add_directory_to_runfiles = False, @@ -111,6 +112,7 @@ copy_to_directory( "triton_kernels": "site-packages/triton_kernels", "generate.py": "site-packages/generate.py", "generate": "bin/generate", + "lib/x86_64-linux-gnu": "lib", }, visibility = ["//visibility:public"], ) From 185e75f88474a89bc53414fdff5925a4fdfb8c31 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Wed, 15 Apr 2026 21:42:47 +0200 Subject: [PATCH 429/881] workspace: bump zig=0.16.0 also bump translate-c and remove a genrule in arocc --- MODULE.bazel | 2 +- MODULE.bazel.lock | 88 +++++++++--------- bazel/zig_index.json | 147 +++++++++++++++++++++++++++++++ third_party/arocc/arocc.bazel | 10 ++- third_party/translate-c/repo.bzl | 2 +- 5 files changed, 200 insertions(+), 49 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 611ea322..3f49e32f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -67,7 +67,7 @@ zig = use_extension("@rules_zig//zig:extensions.bzl", "zig") zig.index(file = "//bazel:zig_index.json") zig.toolchain( translate_c = "@translate-c", - zig_version = "0.16.0-dev.3153+d6f43caad", + zig_version = "0.16.0", ) zig.mirrors(urls = [ "https://pkg.machengine.org/zig", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 5c1d157d..3c7caf19 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1415,18 +1415,18 @@ "@@rules_zig+//zig:extensions.bzl%zig": { "general": { "bzlTransitiveDigest": "r7tCqbjlVKc7XEXV8Hiak2ElFfkjvDdMf8Ym+7zxukU=", - "usagesDigest": "DObSrV9vF3T1Diz4Mp2HSPgtKmxRPL/y5KPhcX6zjvU=", + "usagesDigest": "3YLtfwSfkXudL3+Coim7hrTFktQHln4CiUbszoYuRzE=", "recordedFileInputs": { - "@@//bazel/zig_index.json": "4d85811c879dbd2c11d1d7a40decbe25070abe49d4a1ba704b532999b246ad24", + "@@//bazel/zig_index.json": "2e3b4b83f1add5d3ca12fd05797c04c38af21d86c7fc7c1a9b356560141a4797", "@@rules_zig+//zig/private/versions.json": "6eb85ebaee72c4e6fbc82d8ad73f69e0899d839b70a7e19caa792ab6a5fa0c36" }, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { - "zig_0.16.0-dev.3153_Pd6f43caad_aarch64-linux": { + "zig_0.16.0_aarch64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "url": "https://ziglang.org/download/0.16.0/zig-aarch64-linux-0.16.0.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1443,16 +1443,16 @@ "https://fs.liujiacai.net/zigbuilds", "https://ziglang.org/builds/" ], - "sha256": "e46037f7747492b405a42d40bb9bf3ca3f28c71cf9c1ccdf165a4de74cb484bb", - "zig_version": "0.16.0-dev.3153+d6f43caad", + "sha256": "ea4b09bfb22ec6f6c6ceac57ab63efb6b46e17ab08d21f69f3a48b38e1534f17", + "zig_version": "0.16.0", "platform": "aarch64-linux", "translate_c": "@@+non_module_deps+translate-c//:translate-c" } }, - "zig_0.16.0-dev.3153_Pd6f43caad_aarch64-macos": { + "zig_0.16.0_aarch64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.3153+d6f43caad.tar.xz", + "url": "https://ziglang.org/download/0.16.0/zig-aarch64-macos-0.16.0.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1469,16 +1469,16 @@ "https://fs.liujiacai.net/zigbuilds", "https://ziglang.org/builds/" ], - "sha256": "9121e3f03515a3434ef9f362f8c541540968316d33ed6485b2d8a31dc7332fe6", - "zig_version": "0.16.0-dev.3153+d6f43caad", + "sha256": "b23d70deaa879b5c2d486ed3316f7eaa53e84acf6fc9cc747de152450d401489", + "zig_version": "0.16.0", "platform": "aarch64-macos", "translate_c": "@@+non_module_deps+translate-c//:translate-c" } }, - "zig_0.16.0-dev.3153_Pd6f43caad_aarch64-windows": { + "zig_0.16.0_aarch64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.3153+d6f43caad.zip", + "url": "https://ziglang.org/download/0.16.0/zig-aarch64-windows-0.16.0.zip", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1495,16 +1495,16 @@ "https://fs.liujiacai.net/zigbuilds", "https://ziglang.org/builds/" ], - "sha256": "b25c7046c111f5a4e614fae1a1b33c8db73fbc3de4c135acabb5740455a3b6fc", - "zig_version": "0.16.0-dev.3153+d6f43caad", + "sha256": "aee38316ee4111717900f45dd3130145c39289e105541d737eb8c5ed653c78ef", + "zig_version": "0.16.0", "platform": "aarch64-windows", "translate_c": "@@+non_module_deps+translate-c//:translate-c" } }, - "zig_0.16.0-dev.3153_Pd6f43caad_x86_64-linux": { + "zig_0.16.0_x86_64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "url": "https://ziglang.org/download/0.16.0/zig-x86_64-linux-0.16.0.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1521,16 +1521,16 @@ "https://fs.liujiacai.net/zigbuilds", "https://ziglang.org/builds/" ], - "sha256": "afb3016a94a8b5608d364e58ec7b01111af775a020d5f93f69eaebde2053fe77", - "zig_version": "0.16.0-dev.3153+d6f43caad", + "sha256": "70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00", + "zig_version": "0.16.0", "platform": "x86_64-linux", "translate_c": "@@+non_module_deps+translate-c//:translate-c" } }, - "zig_0.16.0-dev.3153_Pd6f43caad_x86_64-macos": { + "zig_0.16.0_x86_64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.3153+d6f43caad.tar.xz", + "url": "https://ziglang.org/download/0.16.0/zig-x86_64-macos-0.16.0.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1547,16 +1547,16 @@ "https://fs.liujiacai.net/zigbuilds", "https://ziglang.org/builds/" ], - "sha256": "0251a8414f98513a33a7a7bd1f6cc13fb59f391168a742eee47b8c685e9827a0", - "zig_version": "0.16.0-dev.3153+d6f43caad", + "sha256": "0387557ed1877bc6a2e1802c8391953baddba76081876301c522f52977b52ba7", + "zig_version": "0.16.0", "platform": "x86_64-macos", "translate_c": "@@+non_module_deps+translate-c//:translate-c" } }, - "zig_0.16.0-dev.3153_Pd6f43caad_x86_64-windows": { + "zig_0.16.0_x86_64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.3153+d6f43caad.zip", + "url": "https://ziglang.org/download/0.16.0/zig-x86_64-windows-0.16.0.zip", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1573,8 +1573,8 @@ "https://fs.liujiacai.net/zigbuilds", "https://ziglang.org/builds/" ], - "sha256": "ab31039adca06ae87c95c11b3ff8ed82808335866d58bc6e508cd3174d2d7e3d", - "zig_version": "0.16.0-dev.3153+d6f43caad", + "sha256": "68659eb5f1e4eb1437a722f1dd889c5a322c9954607f5edcf337bc3684a75a7e", + "zig_version": "0.16.0", "platform": "x86_64-windows", "translate_c": "@@+non_module_deps+translate-c//:translate-c" } @@ -1583,28 +1583,28 @@ "repoRuleId": "@@rules_zig+//zig/private/repo:toolchains_repo.bzl%toolchains_repo", "attributes": { "names": [ - "zig_0.16.0-dev.3153_Pd6f43caad_aarch64-linux", - "zig_0.16.0-dev.3153_Pd6f43caad_aarch64-macos", - "zig_0.16.0-dev.3153_Pd6f43caad_aarch64-windows", - "zig_0.16.0-dev.3153_Pd6f43caad_x86_64-linux", - "zig_0.16.0-dev.3153_Pd6f43caad_x86_64-macos", - "zig_0.16.0-dev.3153_Pd6f43caad_x86_64-windows" + "zig_0.16.0_aarch64-linux", + "zig_0.16.0_aarch64-macos", + "zig_0.16.0_aarch64-windows", + "zig_0.16.0_x86_64-linux", + "zig_0.16.0_x86_64-macos", + "zig_0.16.0_x86_64-windows" ], "labels": [ - "@zig_0.16.0-dev.3153_Pd6f43caad_aarch64-linux//:zig_toolchain", - "@zig_0.16.0-dev.3153_Pd6f43caad_aarch64-macos//:zig_toolchain", - "@zig_0.16.0-dev.3153_Pd6f43caad_aarch64-windows//:zig_toolchain", - "@zig_0.16.0-dev.3153_Pd6f43caad_x86_64-linux//:zig_toolchain", - "@zig_0.16.0-dev.3153_Pd6f43caad_x86_64-macos//:zig_toolchain", - "@zig_0.16.0-dev.3153_Pd6f43caad_x86_64-windows//:zig_toolchain" + "@zig_0.16.0_aarch64-linux//:zig_toolchain", + "@zig_0.16.0_aarch64-macos//:zig_toolchain", + "@zig_0.16.0_aarch64-windows//:zig_toolchain", + "@zig_0.16.0_x86_64-linux//:zig_toolchain", + "@zig_0.16.0_x86_64-macos//:zig_toolchain", + "@zig_0.16.0_x86_64-windows//:zig_toolchain" ], "zig_versions": [ - "0.16.0-dev.3153+d6f43caad", - "0.16.0-dev.3153+d6f43caad", - "0.16.0-dev.3153+d6f43caad", - "0.16.0-dev.3153+d6f43caad", - "0.16.0-dev.3153+d6f43caad", - "0.16.0-dev.3153+d6f43caad" + "0.16.0", + "0.16.0", + "0.16.0", + "0.16.0", + "0.16.0", + "0.16.0" ], "exec_lengths": [ 2, diff --git a/bazel/zig_index.json b/bazel/zig_index.json index 11251b13..8b39b699 100644 --- a/bazel/zig_index.json +++ b/bazel/zig_index.json @@ -145,6 +145,153 @@ "size": "56927908" } }, + "0.16.0": { + "version": "0.16.0", + "date": "2026-04-13", + "docs": "https://ziglang.org/documentation/0.16.0/", + "stdDocs": "https://ziglang.org/documentation/0.16.0/std/", + "notes": "https://ziglang.org/download/0.16.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.16.0/zig-0.16.0.tar.xz", + "shasum": "43186959edc87d5c7a1be7b7d2a25efffd22ce5807c7af99067f86f99641bfdf", + "size": "22503260" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.16.0/zig-bootstrap-0.16.0.tar.xz", + "shasum": "2a8266a4205772ef40838c8cbdf14875855a515ff3adf89b49c2d2ae93613d10", + "size": "55245980" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86_64-macos-0.16.0.tar.xz", + "shasum": "0387557ed1877bc6a2e1802c8391953baddba76081876301c522f52977b52ba7", + "size": "57396836" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.16.0/zig-aarch64-macos-0.16.0.tar.xz", + "shasum": "b23d70deaa879b5c2d486ed3316f7eaa53e84acf6fc9cc747de152450d401489", + "size": "52238004" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86_64-linux-0.16.0.tar.xz", + "shasum": "70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00", + "size": "55478392" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.16.0/zig-aarch64-linux-0.16.0.tar.xz", + "shasum": "ea4b09bfb22ec6f6c6ceac57ab63efb6b46e17ab08d21f69f3a48b38e1534f17", + "size": "51211944" + }, + "arm-linux": { + "tarball": "https://ziglang.org/download/0.16.0/zig-arm-linux-0.16.0.tar.xz", + "shasum": "f85116bf2f9189bb6ae280c7f92f03b89c2551a88e17881c0c2df86bf4e42c50", + "size": "52122000" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.16.0/zig-riscv64-linux-0.16.0.tar.xz", + "shasum": "bc069b0f2f568f54bafbdfc1d65b12fd386ed6a652044a37aee6a4f72f14076e", + "size": "55313616" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.16.0/zig-powerpc64le-linux-0.16.0.tar.xz", + "shasum": "18800b45c08bf40b335ca5ab79aea70aca287ca969036e938155772becaeebeb", + "size": "55242728" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86-linux-0.16.0.tar.xz", + "shasum": "4e34e279a9f856358de420490b531974c3d37f8f3707eef9f0342e92c14c301f", + "size": "58196012" + }, + "loongarch64-linux": { + "tarball": "https://ziglang.org/download/0.16.0/zig-loongarch64-linux-0.16.0.tar.xz", + "shasum": "2503be8ecc5965f1f7962471d267d9f83fcb3cc2f7ff78ac34093b9722bbea93", + "size": "52515080" + }, + "s390x-linux": { + "tarball": "https://ziglang.org/download/0.16.0/zig-s390x-linux-0.16.0.tar.xz", + "shasum": "14db4884b31414bd637879036c1e6e335f3ba44ccac59df104163728a0cab30e", + "size": "54995144" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86_64-windows-0.16.0.zip", + "shasum": "68659eb5f1e4eb1437a722f1dd889c5a322c9954607f5edcf337bc3684a75a7e", + "size": "97217739" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.16.0/zig-aarch64-windows-0.16.0.zip", + "shasum": "aee38316ee4111717900f45dd3130145c39289e105541d737eb8c5ed653c78ef", + "size": "93109828" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86-windows-0.16.0.zip", + "shasum": "8aee7e8a8deb998ba96cb95d89aa5fcdf32933fdc67de51d280d9e4d7396f1a0", + "size": "99015159" + }, + "aarch64-freebsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-aarch64-freebsd-0.16.0.tar.xz", + "shasum": "0a441f50696a34cfb9f9a0b3c510fd49a56b18b364e03c26525cb7999959c969", + "size": "51137568" + }, + "arm-freebsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-arm-freebsd-0.16.0.tar.xz", + "shasum": "78b2a7bfa74f504075dd4b5c00418347a21b893f0eae580cf4e56c553cf181f6", + "size": "52741336" + }, + "powerpc64le-freebsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-powerpc64le-freebsd-0.16.0.tar.xz", + "shasum": "46b3fa0239be42669d86ce8394f8de2d608a4f51e3e65cb8222bfd543b95e310", + "size": "55232224" + }, + "riscv64-freebsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-riscv64-freebsd-0.16.0.tar.xz", + "shasum": "aefbaea44f32bde07118f5522b7bb21de9967a6d09183e3e37b1f5be2c78f948", + "size": "55416052" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86_64-freebsd-0.16.0.tar.xz", + "shasum": "390379bfaf6b89b001c6bb4035b8fcbc5c12193fd9fd5048c3a0fe39e5d1cf72", + "size": "55591888" + }, + "aarch64-netbsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-aarch64-netbsd-0.16.0.tar.xz", + "shasum": "4767425d72c779275ebbc5a6319f5a3c5d109664d3d8c880125d550e87d9d46a", + "size": "51104512" + }, + "arm-netbsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-arm-netbsd-0.16.0.tar.xz", + "shasum": "07aaa403c18e9fa33687a3eb3109bb484a96aece95a4f6792a911d2eefac07c5", + "size": "53772868" + }, + "x86-netbsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86-netbsd-0.16.0.tar.xz", + "shasum": "aa36c02caad52d92e6354baec16dd866642e64b7c30e4d515806c8296d65586f", + "size": "58792260" + }, + "x86_64-netbsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86_64-netbsd-0.16.0.tar.xz", + "shasum": "e235ca96f63034a009aca831c69394a1c14498a8e06ff5f358fc3fb4bed73dfb", + "size": "55533332" + }, + "aarch64-openbsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-aarch64-openbsd-0.16.0.tar.xz", + "shasum": "f3c868ccc1bc30e2d2b68128186b2153d86c3efc41190f4170a43024a3cd7809", + "size": "51564888" + }, + "arm-openbsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-arm-openbsd-0.16.0.tar.xz", + "shasum": "33786d6487aaeca9f300e5a932d5ee5365d9254be0add44b3af8f6db8441ca2f", + "size": "52301840" + }, + "riscv64-openbsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-riscv64-openbsd-0.16.0.tar.xz", + "shasum": "ffb74aefa8184a4b886f8c0a9105db00ac9562d502330f4e764a7a49171e4f8e", + "size": "55734756" + }, + "x86_64-openbsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86_64-openbsd-0.16.0.tar.xz", + "shasum": "25b72946e8b2dde5c93bc03cbebeb17e4d90eea8040acce8bb78cefdeee97337", + "size": "56962688" + } + }, "0.15.2": { "version": "0.15.2", "date": "2025-10-11", diff --git a/third_party/arocc/arocc.bazel b/third_party/arocc/arocc.bazel index 3e3c62bd..71c1fd6e 100644 --- a/third_party/arocc/arocc.bazel +++ b/third_party/arocc/arocc.bazel @@ -1,3 +1,4 @@ +load("@bazel_skylib//rules:run_binary.bzl", "run_binary") load("@bazel_skylib//rules:write_file.bzl", "write_file") load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") @@ -92,12 +93,15 @@ zig_binary( ) [ - genrule( + run_binary( name = "def_{}".format(name), srcs = [name], outs = ["{}.zig".format(name)], - cmd = """$(location :generate-def) $< $@""", - tools = [":generate-def"], + args = [ + "$(location {})".format(name), + "$(location {}.zig)".format(name), + ], + tool = ":generate-def", ) for name in glob(["src/aro/**/*.def"]) ] diff --git a/third_party/translate-c/repo.bzl b/third_party/translate-c/repo.bzl index f601c6d3..d3a4bd19 100644 --- a/third_party/translate-c/repo.bzl +++ b/third_party/translate-c/repo.bzl @@ -4,6 +4,6 @@ def repo(): new_git_repository( name = "translate-c", remote = "https://codeberg.org/ziglang/translate-c", - commit = "5ac39f77661a216b75b195fe74ce7d0a04b33b7d", + commit = "46b5609b5ac4c0a896217d1d984f3ae50e4810b5", build_file = "//third_party/translate-c:translate-c.bazel", ) From 169f7a021bf9359255a9f8425195063b2e1db6c0 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Thu, 16 Apr 2026 10:20:08 +0200 Subject: [PATCH 430/881] workspace: bump llvm=0.7.1 (#479) This fixes #467 --- MODULE.bazel | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 3f49e32f..70b578b9 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -36,13 +36,7 @@ git_override( ) bazel_dep(name = "tar.bzl", version = "0.9.0") -bazel_dep(name = "llvm", version = "0.7.0") -git_override( - module_name = "llvm", - commit = "9086d034fa7b91b61db0ad66fc78453e576299cd", - remote = "https://github.com/hermeticbuild/hermetic-llvm.git", -) - +bazel_dep(name = "llvm", version = "0.7.1") bazel_dep(name = "jsoncpp", version = "1.9.6", repo_name = "jsoncpp_git") bazel_dep(name = "with_cfg.bzl", version = "0.14.4") bazel_dep(name = "grpc", version = "1.74.0") From 1e79469136a276ed30a0ba394737d0195097e9fd Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Thu, 16 Apr 2026 11:09:47 +0200 Subject: [PATCH 431/881] examples/llm: fix prompt tokenization (#480) Tested with IREE and HF on all three models. --- examples/llm/models/lfm2/session.zig | 4 ++-- examples/llm/models/qwen3_5/session.zig | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig index 37700acf..69d67d04 100644 --- a/examples/llm/models/lfm2/session.zig +++ b/examples/llm/models/lfm2/session.zig @@ -62,7 +62,7 @@ pub const Session = struct { const im_start = self.tokenizer.tokenId("<|im_start|>") orelse return error.NoSuchToken; const im_end = self.tokenizer.tokenId("<|im_end|>") orelse return error.NoSuchToken; - const newline = self.tokenizer.tokenId("\n") orelse return error.NoSuchToken; + const newline = self.tokenizer.tokenId("\\n") orelse return error.NoSuchToken; var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); try tokens.appendSlice(allocator, &.{ self.config.bos_token_id, im_start }); @@ -79,7 +79,7 @@ pub const Session = struct { const im_start = self.tokenizer.tokenId("<|im_start|>") orelse return error.NoSuchToken; const im_end = self.tokenizer.tokenId("<|im_end|>") orelse return error.NoSuchToken; - const newline = self.tokenizer.tokenId("\n") orelse return error.NoSuchToken; + const newline = self.tokenizer.tokenId("\\n") orelse return error.NoSuchToken; var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); diff --git a/examples/llm/models/qwen3_5/session.zig b/examples/llm/models/qwen3_5/session.zig index b055f9b9..834733d0 100644 --- a/examples/llm/models/qwen3_5/session.zig +++ b/examples/llm/models/qwen3_5/session.zig @@ -174,7 +174,7 @@ fn tokenizeChatPrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tok const im_start = tokenizer.tokenId("<|im_start|>") orelse special_tokens.im_start_token_id; const im_end = tokenizer.tokenId("<|im_end|>") orelse special_tokens.im_end_token_id; - const newline = tokenizer.tokenId("\n") orelse return error.NoSuchToken; + const newline = tokenizer.tokenId("\\n") orelse return error.NoSuchToken; var tokens: std.ArrayList(u32) = try .initCapacity(allocator, 32); if (!is_first_turn) { From f23a8a14d34661fc77669c961e11cd89b85d9993 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 17 Apr 2026 11:35:58 +0200 Subject: [PATCH 432/881] zml/ops: improve metadata conversion (#474) In #470 I introduced support for optional float in custom call metadata but it was incomplete. This PR cleans the code a bit and introduce support for all basic optional types. --- zml/ops.zig | 98 +++++++++++++++++++++++++++++------------------------ 1 file changed, 53 insertions(+), 45 deletions(-) diff --git a/zml/ops.zig b/zml/ops.zig index 0dc645c5..6de285ff 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1499,6 +1499,58 @@ fn manualComputationSliceToReturn(comptime ReturnT: type, outputs: []Tensor) Ret return outputs; } +fn metadataIntegerFieldToMlirAttribute(mlir_ctx: *mlir.Context, int_field: std.builtin.Type.Int, value: anytype) *const mlir.Attribute { + return switch (int_field.signedness) { + .signed => switch (int_field.bits) { + 8 => mlir.integerAttribute(mlir_ctx, .i8, value), + 16 => mlir.integerAttribute(mlir_ctx, .i16, value), + 32 => mlir.integerAttribute(mlir_ctx, .i32, value), + 64 => mlir.integerAttribute(mlir_ctx, .i64, value), + else => @panic("Unsupported DataType"), + }, + .unsigned => switch (int_field.bits) { + 8 => mlir.integerAttribute(mlir_ctx, .u8, value), + 16 => mlir.integerAttribute(mlir_ctx, .u16, value), + 32 => mlir.integerAttribute(mlir_ctx, .u32, value), + 64 => mlir.integerAttribute(mlir_ctx, .u64, value), + else => @panic("Unsupported DataType"), + }, + }; +} + +fn metadataFloatFieldToMlirAttribute(mlir_ctx: *mlir.Context, float_field: std.builtin.Type.Float, value: anytype) *const mlir.Attribute { + return switch (float_field.bits) { + 16 => mlir.floatAttribute(mlir_ctx, .f16, @as(f64, value)), + 32 => mlir.floatAttribute(mlir_ctx, .f32, @as(f64, value)), + 64 => mlir.floatAttribute(mlir_ctx, .f64, @as(f64, value)), + else => @panic("Unsupported DataType"), + }; +} + +fn metadataFieldToMlirAttribute(mlir_ctx: *mlir.Context, comptime T: type, value: anytype) ?*const mlir.Attribute { + const type_info = @typeInfo(T); + return switch (type_info) { + .comptime_int => mlir.integerAttribute(mlir_ctx, .u64, @as(u64, value)), + .int => |int_field| metadataIntegerFieldToMlirAttribute(mlir_ctx, int_field, value), + .float => |float_field| metadataFloatFieldToMlirAttribute(mlir_ctx, float_field, value), + .bool => mlir.boolAttribute(mlir_ctx, value), + .pointer => |pointer_info| switch (pointer_info.size) { + .slice => if (pointer_info.child == u8) + mlir.stringAttribute(mlir_ctx, value) + else + @panic("Unsupported pointer type in metadata"), + else => @panic("Unsupported pointer type in metadata"), + }, + .optional => |optional_info| if (value) |wrapped_value| switch (@typeInfo(optional_info.child)) { + .int => |int_field| metadataIntegerFieldToMlirAttribute(mlir_ctx, int_field, wrapped_value), + .float => |float_field| metadataFloatFieldToMlirAttribute(mlir_ctx, float_field, wrapped_value), + .bool => mlir.boolAttribute(mlir_ctx, wrapped_value), + else => @panic("Unsupported optional child type in metadata"), + } else null, + else => @compileError("Unsupported metadata type: " ++ @typeName(T)), + }; +} + fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs: []const Shape, metadata: anytype, opts: CustomCallOptions) []Tensor { const ctx = CompilationContext.current(); const allocator = ctx.arena.allocator(); @@ -1520,51 +1572,7 @@ fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs const metadata_type_info = @typeInfo(@TypeOf(metadata)); var metadata_attribute_list = std.ArrayList(mlir.NamedAttribute).initCapacity(allocator, metadata_type_info.@"struct".fields.len + 2) catch unreachable; inline for (metadata_type_info.@"struct".fields) |field| { - const maybe_attribute: ?*const mlir.Attribute = switch (@typeInfo(field.type)) { - .comptime_int => mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @field(metadata, field.name))), - .int => |int_field| switch (int_field.signedness) { - .signed => switch (int_field.bits) { - 8 => mlir.integerAttribute(ctx.mlir_ctx, .i8, @field(metadata, field.name)), - 16 => mlir.integerAttribute(ctx.mlir_ctx, .i16, @field(metadata, field.name)), - 32 => mlir.integerAttribute(ctx.mlir_ctx, .i32, @field(metadata, field.name)), - 64 => mlir.integerAttribute(ctx.mlir_ctx, .i64, @field(metadata, field.name)), - else => @panic("Unsupported DataType"), - }, - .unsigned => switch (int_field.bits) { - 8 => mlir.integerAttribute(ctx.mlir_ctx, .u8, @field(metadata, field.name)), - 16 => mlir.integerAttribute(ctx.mlir_ctx, .u16, @field(metadata, field.name)), - 32 => mlir.integerAttribute(ctx.mlir_ctx, .u32, @field(metadata, field.name)), - 64 => mlir.integerAttribute(ctx.mlir_ctx, .u64, @field(metadata, field.name)), - else => @panic("Unsupported DataType"), - }, - }, - .float => |float_field| switch (float_field.bits) { - 16 => mlir.floatAttribute(ctx.mlir_ctx, .f16, @as(f64, @field(metadata, field.name))), - 32 => mlir.floatAttribute(ctx.mlir_ctx, .f32, @as(f64, @field(metadata, field.name))), - 64 => mlir.floatAttribute(ctx.mlir_ctx, .f64, @as(f64, @field(metadata, field.name))), - else => @panic("Unsupported DataType"), - }, - .bool => mlir.boolAttribute(ctx.mlir_ctx, @field(metadata, field.name)), - .pointer => |pointer_info| switch (pointer_info.size) { - .slice => if (pointer_info.child == u8) - mlir.stringAttribute(ctx.mlir_ctx, @field(metadata, field.name)) - else - @panic("Unsupported pointer type in metadata"), - else => @panic("Unsupported pointer type in metadata"), - }, - .optional => |optional_info| if (@field(metadata, field.name)) |value| switch (@typeInfo(optional_info.child)) { - .float => |float_field| switch (float_field.bits) { - 16 => mlir.floatAttribute(ctx.mlir_ctx, .f16, @as(f64, value)), - 32 => mlir.floatAttribute(ctx.mlir_ctx, .f32, @as(f64, value)), - 64 => mlir.floatAttribute(ctx.mlir_ctx, .f64, @as(f64, value)), - else => @panic("Unsupported DataType"), - }, - - else => @panic("Unsupported optional child type in metadata"), - } else null, - else => @compileError("Unsupported metadata type: " ++ @typeName(field.type)), - }; - if (maybe_attribute) |attribute| { + if (metadataFieldToMlirAttribute(ctx.mlir_ctx, field.type, @field(metadata, field.name))) |attribute| { metadata_attribute_list.appendAssumeCapacity(mlir.NamedAttribute.named(ctx.mlir_ctx, field.name, attribute)); } } From 0c67102033a98c935f0d2ecb2bd780a11dc5a1d0 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 17 Apr 2026 11:41:09 +0200 Subject: [PATCH 433/881] zml/module: provide an Io to the CompilationContext (#457) This might come in handy in the future if we need to do some IO in the compilation pipeline From 8b12d2ef1e50644e609886bf5b99c5df5feee772 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Fri, 17 Apr 2026 11:48:09 +0200 Subject: [PATCH 434/881] bazel: make repo-wide cquery work across host and linux target analysis (#465) This PR fixes several Bazel analysis regressions so both of these commands work reliably: - `bazel cquery //...` - `bazel cquery --platforms=@zml//platforms:linux_amd64 //...` After recent changes and the rebase, repo-wide cquery was failing for three separate reasons: - ZLS toolchains were over-constrained by target platform: updated ZLS toolchain registration to constrain only the exec platform. This avoids cross-target analysis failures when host-side completion tooling is analyzed under a foreign target platform. - OCI packaging targets were analyzed on host platforms where their base images do not apply: marked Linux OCI image targets as Linux x86_64 compatible only. Also switched the Debian base image pull from Docker Hub to the GCR mirror to avoid CI fetch failures when resolving packaging images manifests due to rate limiting - Triton Python deps were fetched for the host Python platform instead of the Linux wheel platform the lockfile was generated for: restored Linux x86_64 wheel resolution for `triton_py_deps` in `MODULE.bazel`. This prevents macOS host analysis from attempting to fetch a `triton==3.6.0` wheel for an unsupported host platform. Also added a step in ci to prevent regression --- .github/workflows/ci.yaml | 3 ++- MODULE.bazel | 14 +++++++++++++- bin/zml-smi/BUILD.bazel | 4 ++++ examples/llm/BUILD.bazel | 5 +++++ examples/mnist/BUILD.bazel | 4 ++++ third_party/zls/zls_toolchain.bzl | 1 - 6 files changed, 28 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 115d6388..e20b0f30 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -90,7 +90,8 @@ jobs: ${{ matrix.self_hosted && 'common --disk_cache=/home/actions/.cache/bazel-disk' || '' }} - run: bazel mod graph - - run: bazel query ${{ matrix.bazel_opts }} //... + - run: bazel query --output=graph ${{ matrix.bazel_opts }} //... + - run: bazel cquery ${{ matrix.bazel_opts }} //... - run: bazel build ${{ matrix.bazel_opts }} //... - if: ${{ matrix.run_tests }} run: bazel test ${{ matrix.bazel_opts }} //... diff --git a/MODULE.bazel b/MODULE.bazel index 70b578b9..86812f7b 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -140,8 +140,20 @@ use_repo(pip, "xprof_nightly") pip.parse( download_only = True, hub_name = "triton_py_deps", + extra_pip_args = [ + "--abi=cp312", + "--implementation=cp", + "--python-version=312", + "--platform=linux_x86_64", + "--platform=manylinux2014_x86_64", + "--platform=manylinux_2_27_x86_64", + "--platform=manylinux_2_28_x86_64", + "--platform=manylinux_2_29_x86_64", + "--platform=manylinux_2_37_x86_64", + ], python_version = "3.12", requirements_lock = "//zml/attention/triton:requirements.lock.txt", + target_platforms = ["linux_x86_64"], ) use_repo(pip, "triton_py_deps") @@ -252,7 +264,7 @@ oci.pull( oci.pull( name = "debian_12", digest = "sha256:bc960ef50e6feed90686c593361df158517556ed1d2d98e5d1df3724024e0f49", - image = "docker.io/library/debian:12", + image = "mirror.gcr.io/library/debian:12", platforms = ["linux/amd64"], ) use_repo( diff --git a/bin/zml-smi/BUILD.bazel b/bin/zml-smi/BUILD.bazel index 32ec3103..2cdff3b5 100644 --- a/bin/zml-smi/BUILD.bazel +++ b/bin/zml-smi/BUILD.bazel @@ -189,6 +189,10 @@ oci_image( name = "image_", base = "@distroless_cc_debian12", entrypoint = ["/zml-smi/zml-smi"], + target_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], tars = [ ":archive_linux_amd64", ], diff --git a/examples/llm/BUILD.bazel b/examples/llm/BUILD.bazel index d033d488..6a501aea 100644 --- a/examples/llm/BUILD.bazel +++ b/examples/llm/BUILD.bazel @@ -61,6 +61,7 @@ oci_image( entrypoint = ["./{}/llm".format(package_name())], target_compatible_with = [ "@platforms//os:linux", + "@platforms//cpu:x86_64", ], tars = [":archive"], tags = ["manual"] @@ -93,6 +94,10 @@ oci_image( name = "image_debug_", base = "@debian_12", entrypoint = ["./examples/llm/llm"], + target_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], tars = [":archive"], tags = ["manual"], ) diff --git a/examples/mnist/BUILD.bazel b/examples/mnist/BUILD.bazel index 0b996ea7..0f9664c1 100644 --- a/examples/mnist/BUILD.bazel +++ b/examples/mnist/BUILD.bazel @@ -65,6 +65,10 @@ oci_image( name = "image_", base = "@distroless_cc_debian12", entrypoint = ":entrypoint", + target_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], tars = [":archive"], tags = ["manual"], ) diff --git a/third_party/zls/zls_toolchain.bzl b/third_party/zls/zls_toolchain.bzl index f19bd0d1..39b324cb 100644 --- a/third_party/zls/zls_toolchain.bzl +++ b/third_party/zls/zls_toolchain.bzl @@ -105,7 +105,6 @@ def _zls_toolchains_repo_impl(rctx): toolchain( name = "{platform}_toolchain", exec_compatible_with = {compatible_with}, - target_compatible_with = {compatible_with}, toolchain = "@zls_{platform}//:toolchain", toolchain_type = "@zml//third_party/zls:toolchain_type", ) From bd66cfa1f3ca7045759eecdcb03ddf1d85d734e3 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Fri, 17 Apr 2026 13:42:37 +0200 Subject: [PATCH 435/881] Update ZLS to 0.16.0 and rules_zig new providers (#483) Bump rules_zig to latest zig_toolchain changes (for bootstrap) Bump ZLS to 0.16.0 Amend ZLS completion rules and runners to match rules_zig new providers --- MODULE.bazel | 2 +- third_party/zls/zls_runner.zig | 23 ++++++-------------- third_party/zls/zls_toolchain.bzl | 8 +++---- third_party/zls/zls_write_runner_zig_src.bzl | 4 ++-- 4 files changed, 14 insertions(+), 23 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 86812f7b..d2980671 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -31,7 +31,7 @@ bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "rules_zig", version = "0.12.2") git_override( module_name = "rules_zig", - commit = "56911842f4527df036e8d52e78dd0f54a22d4215", + commit = "b65a2bb460b9dbcbcc3231ba648c3c1a266c4eb5", remote = "https://github.com/zml/rules_zig.git", ) diff --git a/third_party/zls/zls_runner.zig b/third_party/zls/zls_runner.zig index c0808d9e..aac52646 100644 --- a/third_party/zls/zls_runner.zig +++ b/third_party/zls/zls_runner.zig @@ -4,6 +4,7 @@ /// /// This file is used as a template by `zls_write_runner_zig_src.bzl`. const std = @import("std"); +const fs = std.fs; const bazel_builtin = @import("bazel_builtin"); const runfiles = @import("runfiles"); @@ -42,23 +43,13 @@ pub fn main(init: std.process.Init) !void { const zig_exe_rpath = "@@__ZIG_EXE_RPATH__@@"; var zig_exe_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const zig_exe_path = blk2: { - if (zig_exe_rpath[0] == '/') { - break :blk2 zig_exe_rpath; - } else { - break :blk2 try r.rlocation(zig_exe_rpath, &zig_exe_path_buf) orelse - return error.RLocationNotFound; - } - }; + const zig_exe_path = try r.rlocation(zig_exe_rpath, &zig_exe_path_buf) orelse + return error.RLocationNotFound; - const zig_lib_path = "@@__ZIG_LIB_PATH__@@"; - const zig_lib_computed_path = blk: { - if (zig_lib_path[0] == '/') { - break :blk zig_lib_path; - } else { - break :blk null; - } - }; + const zig_lib_path = "@@__ZIG_LIB_RPATH__@@"; + var zig_lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const zig_lib_computed_path = try r.rlocation(zig_lib_path, &zig_lib_path_buf) orelse + return error.RLocationNotFound; const zls_build_runner_rpath = "@@__ZLS_BUILD_RUNNER_RPATH__@@"; var zls_build_runner_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; diff --git a/third_party/zls/zls_toolchain.bzl b/third_party/zls/zls_toolchain.bzl index 39b324cb..4611ba95 100644 --- a/third_party/zls/zls_toolchain.bzl +++ b/third_party/zls/zls_toolchain.bzl @@ -2,7 +2,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -_VERSION = "0.16.0-dev.254+695a1d5c" +_VERSION = "0.16.0" _ZLS_PLATFORMS = { "x86_64-linux": struct( @@ -26,9 +26,9 @@ _ZLS_PLATFORMS = { } _ZLS_VERSIONS = { - "x86_64-linux": "b02f9623a0cec2475c26268f12f49f48b81e9c41f2a8a6628806452685cd1013", - "x86_64-macos": "4cab46f56a9685822a5ad3665af3a9b11d2556313defa634b5e8b85cac8151cb", - "aarch64-macos": "8e4ef0603042a2081ae02672ad72f271d8c840746bbfa0ed9934c6b72ea10d7b", + "x86_64-linux": "ded6d562a0b86ee878b1ddf70ffab2797ce3cdca3b02d6077548f9d56dff96b6", + "x86_64-macos": "49f716ea96c1aadaecaa5d9c0a50874cbcf443dc42b825f1e7ee35499ad3eb96", + "aarch64-macos": "b93ec549f8558a7e85984a840e9276d274f1059b54ade4254296ef4982958359", } ZlsToolchainInfo = provider( diff --git a/third_party/zls/zls_write_runner_zig_src.bzl b/third_party/zls/zls_write_runner_zig_src.bzl index a4af5770..224e732d 100644 --- a/third_party/zls/zls_write_runner_zig_src.bzl +++ b/third_party/zls/zls_write_runner_zig_src.bzl @@ -11,8 +11,8 @@ def _zls_write_runner_zig_src_impl(ctx): output = zls_runner, template = ctx.file._runner_tpl, substitutions = { - "@@__ZIG_EXE_RPATH__@@": zigtoolchaininfo.zig_exe_rpath, - "@@__ZIG_LIB_PATH__@@": zigtoolchaininfo.zig_lib_rpath, + "@@__ZIG_EXE_RPATH__@@": to_rlocation_path(ctx, zigtoolchaininfo.zig_exe), + "@@__ZIG_LIB_RPATH__@@": to_rlocation_path(ctx, zigtoolchaininfo.zig_lib), "@@__ZLS_BIN_RPATH__@@": to_rlocation_path(ctx, zlstoolchaininfo.bin), "@@__ZLS_BUILD_RUNNER_RPATH__@@": to_rlocation_path(ctx, ctx.file.build_runner), "@@__GLOBAL_CACHE_PATH__@@": zigtoolchaininfo.zig_cache, From 45f65c7b978957a2b70042eae5144bdc3ab37329 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Fri, 17 Apr 2026 14:54:14 +0200 Subject: [PATCH 436/881] workspace: expose translate-c so that downstream can use it (#484) --- BUILD.bazel | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/BUILD.bazel b/BUILD.bazel index be48a7a8..f049fbe8 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,5 +1,11 @@ load("//third_party/zls:zls_completion.bzl", "zls_completion") +alias( + name = "translate-c", + actual = "@translate-c//:translate-c", + visibility = ["//visibility:public"], +) + zls_completion( name = "completion", visibility = ["//visibility:public"], From 77bdfadc1ae1fe4392121c8c5177fad3c42e1f8f Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Fri, 17 Apr 2026 14:55:27 +0200 Subject: [PATCH 437/881] platforms/tpu: add ragged paged attention (#481) --- MODULE.bazel | 10 + MODULE.bazel.lock | 3 + platforms/tpu/BUILD.bazel | 81 +++- platforms/tpu/gen_ir.py | 636 ++++++++++++++++++++++++++ platforms/tpu/gen_ir.zig | 117 +++++ platforms/tpu/libpjrt_tpu.BUILD.bazel | 21 + platforms/tpu/libpython.h | 2 + platforms/tpu/requirements.in | 3 + platforms/tpu/requirements.lock.txt | 226 +++++++++ platforms/tpu/tpu.bzl | 4 +- platforms/tpu/tpu.zig | 1 - zml/BUILD.bazel | 1 + zml/attention.zig | 1 + zml/attention/paged_attention.zig | 69 ++- zml/attention/tpu_attention.zig | 412 +++++++++++++++++ zml/attention/triton/generate.zig | 2 +- zml/platform.zig | 10 +- 17 files changed, 1587 insertions(+), 12 deletions(-) create mode 100644 platforms/tpu/gen_ir.py create mode 100644 platforms/tpu/gen_ir.zig create mode 100644 platforms/tpu/libpython.h create mode 100644 platforms/tpu/requirements.in create mode 100644 platforms/tpu/requirements.lock.txt create mode 100644 zml/attention/tpu_attention.zig diff --git a/MODULE.bazel b/MODULE.bazel index d2980671..8e013f6e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -157,6 +157,16 @@ pip.parse( ) use_repo(pip, "triton_py_deps") +# For TPU IR generation +pip.parse( + download_only = False, + hub_name = "tpu_py_deps", + python_version = "3.12", + requirements_lock = "//platforms/tpu:requirements.lock.txt", + target_platforms = ["linux_x86_64"], +) +use_repo(pip, "tpu_py_deps") + cpu = use_extension("//platforms/cpu:cpu.bzl", "cpu_pjrt_plugin") use_repo(cpu, "libpjrt_cpu_darwin_amd64", "libpjrt_cpu_darwin_arm64", "libpjrt_cpu_linux_amd64") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 3c7caf19..c556bde6 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -151,6 +151,9 @@ "https://bcr.bazel.build/modules/libbacktrace/1.0.0-20250926-7939218/source.json": "83239989d09748f63dee4bdd430d261e217a012670e066e2238925a303a65d89", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", "https://bcr.bazel.build/modules/libpfm/4.11.0/source.json": "caaffb3ac2b59b8aac456917a4ecf3167d40478ee79f15ab7a877ec9273937c9", + "https://bcr.bazel.build/modules/llvm/0.6.7/MODULE.bazel": "d37a2e10571864dc6a5bb53c29216d90b9400bbcadb422337f49107fd2eaf0d2", + "https://bcr.bazel.build/modules/llvm/0.7.1/MODULE.bazel": "74ac75efc6385b8a95d83bfa36ad399500f747c3d0f50287f9b6f9e854ec4814", + "https://bcr.bazel.build/modules/llvm/0.7.1/source.json": "0cac59d04dafa0ca1f70ea21cf1569e034ef8736c2c7510e834a9e8141d7e631", "https://bcr.bazel.build/modules/mbedtls/3.6.0/MODULE.bazel": "8e380e4698107c5f8766264d4df92e36766248447858db28187151d884995a09", "https://bcr.bazel.build/modules/mbedtls/3.6.0/source.json": "1dbe7eb5258050afcc3806b9d43050f71c6f539ce0175535c670df606790b30c", "https://bcr.bazel.build/modules/nlohmann_json/3.11.3/MODULE.bazel": "87023db2f55fc3a9949c7b08dc711fae4d4be339a80a99d04453c4bb3998eefc", diff --git a/platforms/tpu/BUILD.bazel b/platforms/tpu/BUILD.bazel index 7f9ae972..7cae20c0 100644 --- a/platforms/tpu/BUILD.bazel +++ b/platforms/tpu/BUILD.bazel @@ -1,5 +1,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library") +load("@rules_python//python:py_binary.bzl", "py_binary") +load("@rules_python//python/uv:lock.bzl", uv_lock = "lock") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") +load("@zml//bazel:runfiles.bzl", "runfiles_to_default") cc_library( name = "empty", @@ -20,15 +23,85 @@ zig_library( import_name = "platforms/tpu", main = "tpu.zig", visibility = ["//visibility:public"], + data = select({ + "//platforms:tpu.enabled": [":gen_ir_files"], + "//conditions:default": [], + }), deps = [ + "//bazel", "//pjrt", + "//stdx", ] + select({ "//platforms:tpu.enabled": [ ":libpjrt_tpu", - "//bazel", - "//stdx", - "@rules_zig//zig/runfiles", ], "//conditions:default": [":empty"], }), ) + +# bazel run //platforms/tpu:requirements.update +uv_lock( + name = "requirements", + srcs = ["requirements.in"], + out = "requirements.lock.txt", + args = [ + "--emit-index-url", + "--emit-find-links", + "--index-strategy=unsafe-best-match", + "--upgrade", + "--universal", + "--prerelease=allow", + ], + python_version = "3.12", + tags = ["manual"], +) + +py_binary( + name = "gen_ir", + srcs = ["gen_ir.py"], + main = "gen_ir.py", + python_version = "3.12", + tags = ["manual"], + target_compatible_with = ["@platforms//os:linux"], + deps = [ + "@tpu_py_deps//jax", + "@tpu_py_deps//absl_py", + # "@tpu_py_deps//tpu_inference", + ], +) + +runfiles_to_default( + name = "gen_ir_files", + target_compatible_with = ["@platforms//os:linux"], + visibility = ["//visibility:public"], + deps = [ + ":gen_ir", + ":gen_ir_zig", + ], +) + +zig_binary( + name = "gen_ir_zig", + data = [":gen_ir"], + linkopts = select({ + "@platforms//os:macos": ["-Wl,-rpath,@executable_path/../lib"], + "//conditions:default": ["-Wl,-rpath,$$ORIGIN/../lib"], + }), + main = "gen_ir.zig", + tags = ["manual"], + target_compatible_with = ["@platforms//os:linux"], + deps = [ + ":libpython", + "//bazel", + "@rules_zig//zig/runfiles", + ], +) + +cc_library( + name = "libpython", + hdrs = ["libpython.h"], + deps = [ + "@rules_python//python/cc:current_py_cc_headers", + "@rules_python//python/cc:current_py_cc_libs", + ], +) diff --git a/platforms/tpu/gen_ir.py b/platforms/tpu/gen_ir.py new file mode 100644 index 00000000..014caf59 --- /dev/null +++ b/platforms/tpu/gen_ir.py @@ -0,0 +1,636 @@ +import contextlib +import json +import math +import os +import signal +import sys + +# This IR generator only needs CPU lowering. Forcing CPU here avoids JAX +# probing local TPU runtime state and emitting a fallback warning. +os.environ.setdefault("JAX_PLATFORMS", "cpu") + +import jax +import jax.numpy as jnp +from jax._src.interpreters import mlir +from jax._src.pallas import pallas_call +from jax.experimental import pallas as pallas_mod +from jax.experimental.pallas.ops.tpu.flash_attention import BlockSizes, flash_attention +from jax.experimental.pallas.ops.tpu.paged_attention import paged_attention +from jax.experimental.pallas.ops.tpu.ragged_paged_attention import ragged_paged_attention + +RAGGED_KERNEL_ENV = "ZML_TPU_RAGGED_KERNEL" +_TPU_INFERENCE_RAGGED_PATCHED = False + + +def _cpu_lowering_tpu_wrapper(ctx, *in_nodes, **params): + # Reuse TPU lowering on CPU-only envs used by this IR generator. + params.pop("backend", None) + params.pop("which_linear", None) + if "out_shapes" in params: + params["out_avals"] = params.pop("out_shapes") + from jax._src.pallas.mosaic.pallas_call_registration import ( + pallas_call_tpu_lowering_rule, + ) + + return pallas_call_tpu_lowering_rule(ctx, *in_nodes, **params) + + +mlir.register_lowering( + pallas_call.pallas_call_p, + _cpu_lowering_tpu_wrapper, + platform="cpu", +) + + +def _abstract_tpu_device_kind() -> str: + return ( + os.environ.get("ZML_TPU_IR_DEVICE_KIND") + or os.environ.get("TPU_DEVICE_KIND") + or "TPU v5 lite" + ) + + +def _abstract_tpu_num_cores() -> int: + raw = os.environ.get("ZML_TPU_IR_NUM_CORES", "1") + try: + cores = int(raw) + except ValueError: + cores = 1 + return max(1, cores) + + +def _tpu_version_from_kind(device_kind: str) -> int: + kind = device_kind.strip() + if "TPU" not in kind: + return -1 + if kind == "TPU7x": + return 7 + if kind.endswith(" lite"): + kind = kind[: -len(" lite")] + if not kind.startswith("TPU v"): + return -1 + version = [] + for ch in kind[len("TPU v"):]: + if not ch.isdigit(): + break + version.append(ch) + if not version: + return -1 + return int("".join(version)) + + +def _tpu_device_name_from_kind(device_kind: str, num_devices: int | None = None) -> str: + parts = device_kind.split() + name = " ".join(parts[:2]) if len(parts) >= 2 else device_kind + if num_devices is not None: + name += f"-{num_devices}" + return name + + +def _tpu_inference_device_name(num_devices: int | None = None) -> str: + kind = _abstract_tpu_device_kind() + if "TPU" not in kind: + return kind + suffix = "" + if kind.endswith(" lite"): + kind = kind[: -len(" lite")] + suffix = "e" + elif kind.endswith("e"): + kind = kind[:-1] + suffix = "e" + elif kind.endswith("p"): + kind = kind[:-1] + suffix = "p" + elif kind == "TPU7x": + kind = "TPU v7" + if not kind.startswith("TPU v"): + return kind + kind += suffix + if num_devices is not None: + kind += f"-{num_devices}" + return kind + + +def _ensure_vllm_logger_shim() -> None: + if "vllm.logger" in sys.modules: + return + import logging + import types + + vllm_mod = sys.modules.get("vllm") + if vllm_mod is None: + vllm_mod = types.ModuleType("vllm") + vllm_mod.__path__ = [] + sys.modules["vllm"] = vllm_mod + + logger_mod = types.ModuleType("vllm.logger") + + class _VllmLogger(logging.Logger): + def info_once(self, msg, *args, **kwargs): + self.info(msg, *args, **kwargs) + + def warning_once(self, msg, *args, **kwargs): + self.warning(msg, *args, **kwargs) + + def warn_once(self, msg, *args, **kwargs): + self.warning(msg, *args, **kwargs) + + def error_once(self, msg, *args, **kwargs): + self.error(msg, *args, **kwargs) + + def debug_once(self, msg, *args, **kwargs): + self.debug(msg, *args, **kwargs) + + def init_logger(name: str) -> _VllmLogger: + logger = logging.getLogger(name) + if not isinstance(logger, _VllmLogger): + logger = _VllmLogger(name) + logging.Logger.manager.loggerDict[name] = logger + return logger + + logger_mod._VllmLogger = _VllmLogger + logger_mod.init_logger = init_logger + sys.modules["vllm.logger"] = logger_mod + setattr(vllm_mod, "logger", logger_mod) + + +def _patch_tpu_inference_ragged_v3_device_probe() -> None: + global _TPU_INFERENCE_RAGGED_PATCHED + if _TPU_INFERENCE_RAGGED_PATCHED: + return + + _ensure_vllm_logger_shim() + + try: + from tpu_inference.kernels.ragged_paged_attention.v3 import kernel as v3_kernel + from tpu_inference.kernels.ragged_paged_attention.v3 import util as v3_util + except Exception: + return + try: + from tpu_inference.kernels.ragged_paged_attention.v3 import tuned_block_sizes as v3_tuned + except Exception: + v3_tuned = None + try: + from tpu_inference.kernels.ragged_paged_attention.v3 import tuned_block_sizes_hd64 as v3_tuned_hd64 + except Exception: + v3_tuned_hd64 = None + try: + from tpu_inference import utils as tpu_utils + except Exception: + tpu_utils = None + + tpu_version_fn = lambda: _tpu_version_from_kind(_abstract_tpu_device_kind()) + device_name_fn = lambda num_devices=None: _tpu_inference_device_name(num_devices) + + v3_util.get_tpu_version = tpu_version_fn + v3_kernel.get_tpu_version = tpu_version_fn + if v3_tuned is not None: + v3_tuned.get_tpu_version = tpu_version_fn + if hasattr(v3_tuned, "get_device_name"): + v3_tuned.get_device_name = device_name_fn + if v3_tuned_hd64 is not None: + v3_tuned_hd64.get_tpu_version = tpu_version_fn + if hasattr(v3_tuned_hd64, "get_device_name"): + v3_tuned_hd64.get_device_name = device_name_fn + if tpu_utils is not None: + tpu_utils.get_device_name = device_name_fn + + _TPU_INFERENCE_RAGGED_PATCHED = True + + +def _effective_backend_kind(requested_backend_kind: str) -> str: + if requested_backend_kind != "ragged_paged": + return requested_backend_kind + + raw_selector = os.environ.get(RAGGED_KERNEL_ENV) + selector = (raw_selector or "").strip().lower() + + if selector in ("", "mosaic", "jax"): + return "ragged_paged" + if selector in ("v3", "tpu_inference", "vllm", "v2"): + return "ragged_paged_v3" + + raise ValueError( + f"Invalid {RAGGED_KERNEL_ENV}={raw_selector!r}; expected one of: mosaic, v3" + ) + + +def _patch_ragged_tpu_tuning_device_probe() -> None: + # The tuned block-size helper probes jax.devices()[0] directly, which is + # CPU in this process by design. Override it with our abstract TPU kind. + from jax.experimental.pallas.ops.tpu.ragged_paged_attention import tuned_block_sizes + + tuned_block_sizes.get_tpu_version = lambda: _tpu_version_from_kind(_abstract_tpu_device_kind()) + tuned_block_sizes.get_device_name = ( + lambda num_devices=None: _tpu_device_name_from_kind(_abstract_tpu_device_kind(), num_devices) + ) + + +@contextlib.contextmanager +def _tpu_abstract_mesh_context(): + # Force TPU chip metadata lookup to use an abstract TPU device while we + # still lower on CPU-only JAX backends. + mesh = jax.sharding.AbstractMesh( + axis_sizes=(1,), + axis_names=("tpu_core",), + abstract_device=jax.sharding.AbstractDevice( + device_kind=_abstract_tpu_device_kind(), + num_cores=_abstract_tpu_num_cores(), + ), + ) + with jax.sharding.use_abstract_mesh(mesh): + yield + + +_patch_ragged_tpu_tuning_device_probe() + + +def _extract_backend_configs(op) -> list[str]: + configs: list[str] = [] + backend = op.attributes.get("backend_config") + if backend is not None: + configs.append(backend.value if hasattr(backend, "value") else str(backend)) + for region in op.regions: + for block in region.blocks: + for sub_op in block.operations: + configs.extend(_extract_backend_configs(sub_op)) + return configs + + +def _extract_backend_configs_with_kernel_name(op) -> list[tuple[str | None, str]]: + configs: list[tuple[str | None, str]] = [] + backend = op.attributes.get("backend_config") + if backend is not None: + kernel_attr = op.attributes.get("kernel_name") + kernel_name: str | None = None + if kernel_attr is not None: + kernel_name = kernel_attr.value if hasattr(kernel_attr, "value") else str(kernel_attr) + configs.append((kernel_name, backend.value if hasattr(backend, "value") else str(backend))) + for region in op.regions: + for block in region.blocks: + for sub_op in block.operations: + configs.extend(_extract_backend_configs_with_kernel_name(sub_op)) + return configs + + +def _extract_backend_config(op) -> str | None: + configs = _extract_backend_configs(op) + return configs[0] if configs else None + + +def flash_attention_on_tpu(kernel_params: dict) -> str | None: + batch_size = kernel_params["batch_size"] + num_heads = kernel_params["num_heads"] + q_seq_len = kernel_params["q_seq_len"] + kv_seq_len = kernel_params["kv_seq_len"] + d_model = kernel_params["d_model"] + + q = jax.ShapeDtypeStruct((batch_size, num_heads, q_seq_len, d_model), jnp.bfloat16) + k = jax.ShapeDtypeStruct((batch_size, num_heads, kv_seq_len, d_model), jnp.bfloat16) + v = jax.ShapeDtypeStruct((batch_size, num_heads, kv_seq_len, d_model), jnp.bfloat16) + + block = BlockSizes( + block_q=min(128, q_seq_len), + block_k_major=min(128, kv_seq_len), + block_k=min(128, kv_seq_len), + block_b=1, + ) + sm_scale = kernel_params.get("sm_scale", 1.0 / math.sqrt(d_model)) + + with _tpu_abstract_mesh_context(): + lowered = jax.jit( + flash_attention, + backend="cpu", + static_argnames=["block_sizes", "causal", "sm_scale"], + ).lower(q, k, v, block_sizes=block, causal=True, sm_scale=sm_scale) + configs_with_name = _extract_backend_configs_with_kernel_name(lowered.compiler_ir().operation) + for kernel_name, cfg in configs_with_name: + if kernel_name and "RPAm" in kernel_name: + return cfg + configs = [cfg for _, cfg in configs_with_name] + for cfg in configs: + if "RPAm" in cfg: + return cfg + return configs[-1] if configs else None + + +def _fix_paged_attention_acc_specs(fn, **kwargs): + grid_spec = kwargs.get("grid_spec") + out_shape = kwargs.get("out_shape") + out_specs = getattr(grid_spec, "out_specs", None) + + if ( + isinstance(out_specs, tuple) + and isinstance(out_shape, list) + and len(out_specs) == 3 + and len(out_shape) == 3 + and out_shape[1].shape[-1] == 1 + ): + q_block_shape = out_specs[0].block_shape + if q_block_shape is not None and q_block_shape[-1] not in (None, 1): + fixed = pallas_mod.BlockSpec((*q_block_shape[:-1], 1), out_specs[0].index_map) + grid_spec.out_specs = (out_specs[0], fixed, fixed) + + return _ORIG_PALLAS_CALL(fn, **kwargs) + + +def paged_attention_on_tpu(kernel_params: dict) -> str | None: + batch_size = kernel_params["batch_size"] + pages_per_sequence = kernel_params["pages_per_sequence"] + dtype = jnp.float32 + + q = jax.ShapeDtypeStruct((batch_size, kernel_params["num_q_heads"], kernel_params["head_dim"]), dtype) + k = jax.ShapeDtypeStruct( + (kernel_params["num_kv_heads"], kernel_params["total_num_pages"], kernel_params["page_size"], kernel_params["head_dim"]), + jnp.bfloat16, + ) + v = jax.ShapeDtypeStruct( + (kernel_params["num_kv_heads"], kernel_params["total_num_pages"], kernel_params["page_size"], kernel_params["head_dim"]), + jnp.bfloat16, + ) + lengths = jax.ShapeDtypeStruct((batch_size,), jnp.int32) + page_indices_flat = jax.ShapeDtypeStruct((batch_size * pages_per_sequence,), jnp.int32) + + def _paged_attention_with_flat_indices(q, k, v, lengths, page_indices_flat, *, pages_per_compute_block): + indices = page_indices_flat.reshape(batch_size, pages_per_sequence) + return paged_attention(q, k, v, lengths, indices, pages_per_compute_block=pages_per_compute_block) + + comp_target = jax.jit(_paged_attention_with_flat_indices, static_argnames=["pages_per_compute_block"]) + + pallas_mod.pallas_call = _fix_paged_attention_acc_specs + try: + with _tpu_abstract_mesh_context(): + lowered = comp_target.lower( + q, + k, + v, + lengths, + page_indices_flat, + pages_per_compute_block=kernel_params["pages_per_compute_block"], + ) + finally: + pallas_mod.pallas_call = _ORIG_PALLAS_CALL + + return _extract_backend_config(lowered.compiler_ir().operation) + + +def _parse_jnp_dtype(dtype_name: str): + allowed_dtypes = { + "jnp.bfloat16": jnp.bfloat16, + "jnp.float32": jnp.float32, + } + try: + return allowed_dtypes[dtype_name] + except KeyError as exc: + raise ValueError(f"Unsupported dtype: {dtype_name}") from exc + + +def ragged_paged_attention_on_tpu(kernel_params: dict) -> str | None: + q_dtype = _parse_jnp_dtype(kernel_params["q_dtype"]) + kv_dtype = _parse_jnp_dtype(kernel_params["kv_dtype"]) + q = jax.ShapeDtypeStruct( + ( + kernel_params["num_q_tokens"], + kernel_params["num_q_heads"], + kernel_params["head_dim"], + ), + q_dtype, + ) + kv_pages = jax.ShapeDtypeStruct( + ( + kernel_params["total_num_pages"], + kernel_params["page_size"], + kernel_params["num_kv_heads"] * 2, + kernel_params["head_dim"], + ), + kv_dtype, + ) + kv_lens = jax.ShapeDtypeStruct((kernel_params["max_num_seqs"],), jnp.int32) + page_indices = jax.ShapeDtypeStruct( + (kernel_params["max_num_seqs"], kernel_params["pages_per_seq"]), + jnp.int32, + ) + cu_q_lens = jax.ShapeDtypeStruct((kernel_params["max_num_seqs"] + 1,), jnp.int32) + num_seqs = jax.ShapeDtypeStruct((1,), jnp.int32) + + with _tpu_abstract_mesh_context(): + lowered = jax.jit( + ragged_paged_attention, + backend="cpu", + static_argnames=[ + "sm_scale", + "sliding_window", + "soft_cap", + "mask_value", + "k_scale", + "v_scale", + "num_kv_pages_per_block", + "num_queries_per_block", + "vmem_limit_bytes", + ], + ).lower( + q, + kv_pages, + kv_lens, + page_indices, + cu_q_lens, + num_seqs, + sm_scale=kernel_params.get("sm_scale", 1.0), + sliding_window=kernel_params.get("sliding_window"), + soft_cap=kernel_params.get("soft_cap"), + mask_value=kernel_params.get("mask_value"), + k_scale=kernel_params.get("k_scale"), + v_scale=kernel_params.get("v_scale"), + num_kv_pages_per_block=kernel_params.get("num_kv_pages_per_block"), + num_queries_per_block=kernel_params.get("num_queries_per_block"), + vmem_limit_bytes=kernel_params.get("vmem_limit_bytes"), + ) + return _extract_backend_config(lowered.compiler_ir().operation) + + +def ragged_paged_attention_v3_on_tpu(kernel_params: dict) -> str | None: + _ensure_vllm_logger_shim() + try: + from tpu_inference.kernels.ragged_paged_attention.v3 import kernel as v3_kernel + except Exception as exc: + raise ValueError(f"Failed to import tpu-inference v3 ragged kernel: {exc}") + + _patch_tpu_inference_ragged_v3_device_probe() + + q_dtype = _parse_jnp_dtype(kernel_params["q_dtype"]) + kv_dtype = _parse_jnp_dtype(kernel_params["kv_dtype"]) + + num_q_tokens = kernel_params["num_q_tokens"] + num_q_heads = kernel_params["num_q_heads"] + num_kv_heads = kernel_params["num_kv_heads"] + head_dim = kernel_params["head_dim"] + total_num_pages = kernel_params["total_num_pages"] + page_size = kernel_params["page_size"] + max_num_seqs = kernel_params["max_num_seqs"] + pages_per_seq = kernel_params["pages_per_seq"] + + kv_cache_shape = v3_kernel.get_kv_cache_shape( + total_num_pages, + page_size, + num_kv_heads, + head_dim, + kv_dtype, + ) + + q = jax.ShapeDtypeStruct((num_q_tokens, num_q_heads, head_dim), q_dtype) + k = jax.ShapeDtypeStruct((num_q_tokens, num_kv_heads, head_dim), kv_dtype) + v = jax.ShapeDtypeStruct((num_q_tokens, num_kv_heads, head_dim), kv_dtype) + kv_cache = jax.ShapeDtypeStruct(kv_cache_shape, kv_dtype) + kv_lens = jax.ShapeDtypeStruct((max_num_seqs,), jnp.int32) + page_indices = jax.ShapeDtypeStruct((max_num_seqs * pages_per_seq,), jnp.int32) + cu_q_lens = jax.ShapeDtypeStruct((max_num_seqs + 1,), jnp.int32) + distribution = jax.ShapeDtypeStruct((3,), jnp.int32) + + def _block_sizes(name: str): + value = kernel_params.get(name) + if value is None: + return None + if isinstance(value, (list, tuple)) and len(value) == 4: + return tuple(int(v) for v in value) + raise ValueError(f"Invalid {name}: expected 4-element list or tuple") + + out_dtype = kernel_params.get("out_dtype") + if isinstance(out_dtype, str): + out_dtype = _parse_jnp_dtype(out_dtype) + + with _tpu_abstract_mesh_context(): + lowered = jax.jit( + v3_kernel.ragged_paged_attention, + backend="cpu", + static_argnames=[ + "use_causal_mask", + "skip_kv_mask", + "sm_scale", + "sliding_window", + "soft_cap", + "out_dtype", + "mask_value", + "q_scale", + "k_scale", + "v_scale", + "chunk_prefill_size", + "d_block_sizes", + "p_block_sizes", + "m_block_sizes", + "vmem_limit_bytes", + "debug_mode", + "disable_bounds_checks", + "disable_semaphore_checks", + ], + ).lower( + q, + k, + v, + kv_cache, + kv_lens, + page_indices, + cu_q_lens, + distribution, + use_causal_mask=kernel_params.get("use_causal_mask", True), + skip_kv_mask=kernel_params.get("skip_kv_mask", False), + sm_scale=kernel_params.get("sm_scale", 1.0), + sliding_window=kernel_params.get("sliding_window"), + soft_cap=kernel_params.get("soft_cap"), + out_dtype=out_dtype, + mask_value=kernel_params.get("mask_value"), + q_scale=kernel_params.get("q_scale"), + k_scale=kernel_params.get("k_scale"), + v_scale=kernel_params.get("v_scale"), + chunk_prefill_size=kernel_params.get("chunk_prefill_size"), + d_block_sizes=_block_sizes("d_block_sizes"), + p_block_sizes=_block_sizes("p_block_sizes"), + m_block_sizes=_block_sizes("m_block_sizes"), + vmem_limit_bytes=kernel_params.get("vmem_limit_bytes"), + debug_mode=kernel_params.get("debug_mode", False), + disable_bounds_checks=kernel_params.get("disable_bounds_checks", True), + disable_semaphore_checks=kernel_params.get("disable_semaphore_checks", True), + ) + + configs_with_names = _extract_backend_configs_with_kernel_name(lowered.compiler_ir().operation) + if os.environ.get("ZML_TPU_IR_DEBUG"): + print(f"debug: v3 custom call configs={len(configs_with_names)}", file=sys.stderr) + for idx, (name, cfg) in enumerate(configs_with_names): + snippet = cfg[:160].replace("\n", " ") + print( + f"debug: cfg[{idx}] kernel_name={name!r} len={len(cfg)} snippet={snippet!r}", + file=sys.stderr, + ) + + if configs_with_names: + for name, cfg in configs_with_names: + if name and "RPAm" in name: + return cfg + # If kernel_name isn't present or doesn't mention RPAm, use the last + # custom call (mixed is emitted last in v3). + return configs_with_names[-1][1] + + configs = _extract_backend_configs(lowered.compiler_ir().operation) + if not configs: + return None + return configs[-1] + + +def handle_sigint(signum, frame): + sys.exit(0) + + +def main(): + signal.signal(signal.SIGINT, handle_sigint) + + for raw_line in sys.stdin: + line = raw_line.strip() + if not line: + continue + + try: + request = json.loads(line) + if not isinstance(request, dict): + raise ValueError("request must be a JSON object") + + backend_kind = _effective_backend_kind(request["backend_config"]) + params = request["params"] + + if backend_kind not in {"paged", "flash", "ragged_paged", "ragged_paged_v3"}: + raise ValueError(f"Unsupported backend_config: {backend_kind}") + if not isinstance(params, dict): + raise ValueError("Invalid request: 'params' must be a JSON object") + + tpu_code = None + + if backend_kind == "ragged_paged": + tpu_code = ragged_paged_attention_on_tpu(params) + if tpu_code is None: + raise ValueError("Failed to extract TPU backend config from ragged paged attention IR") + if backend_kind == "ragged_paged_v3": + tpu_code = ragged_paged_attention_v3_on_tpu(params) + if tpu_code is None: + raise ValueError("Failed to extract TPU backend config from v3 ragged paged attention IR") + if backend_kind == "paged": + # tpu_code = paged_attention_on_tpu(params) + # if tpu_code is None: + # raise ValueError("Failed to extract TPU backend config from paged attention IR") + raise ValueError(f"Unsupported backend_config: {backend_kind} for now") + if backend_kind == "flash": + # tpu_code = flash_attention_on_tpu(params) + # if tpu_code is None: + # raise ValueError("Failed to extract TPU backend config from flash attention IR") + raise ValueError(f"Unsupported backend_config: {backend_kind} for now") + + if tpu_code is None: + raise ValueError(f"Unsupported backend_config: {backend_kind}") + response = {"ok": True, "result": tpu_code} + except Exception as exc: + response = {"ok": False, "error": str(exc)} + + print(json.dumps(response), flush=True) + + +_ORIG_PALLAS_CALL = pallas_mod.pallas_call + +if __name__ == "__main__": + main() diff --git a/platforms/tpu/gen_ir.zig b/platforms/tpu/gen_ir.zig new file mode 100644 index 00000000..199ed67c --- /dev/null +++ b/platforms/tpu/gen_ir.zig @@ -0,0 +1,117 @@ +const std = @import("std"); + +const c = @import("c"); + +fn pyStatusCheck(status: c.PyStatus) void { + if (c.PyStatus_Exception(status) != 0) { + if (c.PyStatus_IsExit(status) != 0) { + std.process.exit(@intCast(status.exitcode)); + } + c.Py_ExitStatusException(status); + } +} + +pub fn toPosixPathW(file_path: []const u8) error{ InvalidUtf8, NameTooLong, CodepointTooLarge }![std.posix.PATH_MAX - 1:0]c.wchar_t { + var view = std.unicode.Utf8View.init(file_path) catch return error.InvalidUtf8; + var it = view.iterator(); + + var path_with_null: [std.posix.PATH_MAX - 1:0]c.wchar_t = undefined; + var len: usize = 0; + while (it.nextCodepoint()) |cp| { + if (len + 1 >= path_with_null.len) return error.NameTooLong; + path_with_null[len] = std.math.cast(c.wchar_t, cp) orelse return error.CodepointTooLarge; + len += 1; + } + path_with_null[len] = 0; + return path_with_null; +} + +pub fn main(init: std.process.Init.Minimal) !void { + const allocator = std.heap.c_allocator; + + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + + const io = threaded.io(); + + { + var preconfig: c.PyPreConfig = undefined; + c.PyPreConfig_InitIsolatedConfig(&preconfig); + preconfig.utf8_mode = 1; + pyStatusCheck(c.Py_PreInitialize(&preconfig)); + } + + var config: c.PyConfig = undefined; + c.PyConfig_InitIsolatedConfig(&config); + defer c.PyConfig_Clear(&config); + + config.module_search_paths_set = 1; + config.optimization_level = 2; + config.write_bytecode = 0; + + pyStatusCheck(c.PyConfig_SetBytesArgv(&config, @intCast(init.args.vector.len), @ptrCast(init.args.vector.ptr))); + + var self_exe_dir_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const self_exe_dir = blk: { + const n = try std.process.executableDirPath(io, &self_exe_dir_buf); + break :blk self_exe_dir_buf[0..n]; + }; + + { + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const home = try std.fmt.bufPrintZ(&buf, "{f}{d}.{d}", .{ + std.Io.Dir.path.fmtJoin(&.{ + self_exe_dir, + "..", + "lib", + "python", + }), + c.PY_MAJOR_VERSION, + c.PY_MINOR_VERSION, + }); + pyStatusCheck(c.PyConfig_SetBytesString(&config, &config.home, home)); + pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(home))); + } + + { + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const site_packages = try std.fmt.bufPrint(&buf, "{f}", .{ + std.Io.Dir.path.fmtJoin(&.{ + self_exe_dir, + "..", + "site-packages", + }), + }); + pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(site_packages))); + } + + pyStatusCheck(c.Py_InitializeFromConfig(&config)); + defer c.Py_Finalize(); + + const module = c.PyImport_ImportModule("gen_ir"); + if (module == null) { + c.PyErr_Print(); + return error.PythonModuleImportFailed; + } + defer c.Py_DecRef(module); + + const func = c.PyObject_GetAttrString(module, "main"); + if (func == null) { + std.log.err("Python module 'gen_ir' is missing required attribute 'main'", .{}); + c.PyErr_Print(); + return error.PythonAttributeLookupFailed; + } + defer c.Py_DecRef(func); + + if (c.PyCallable_Check(func) == 0) { + std.log.err("Python attribute 'gen_ir.main' is not callable", .{}); + return error.PythonAttributeNotCallable; + } + + const result = c.PyObject_CallNoArgs(func); + if (result == null) { + c.PyErr_Print(); + return error.PythonExecutionFailed; + } + defer c.Py_DecRef(result); +} diff --git a/platforms/tpu/libpjrt_tpu.BUILD.bazel b/platforms/tpu/libpjrt_tpu.BUILD.bazel index d85e167b..26ab93ac 100644 --- a/platforms/tpu/libpjrt_tpu.BUILD.bazel +++ b/platforms/tpu/libpjrt_tpu.BUILD.bazel @@ -11,7 +11,28 @@ copy_to_directory( name = "sandbox", srcs = [ "lib/libpjrt_tpu.so", + "@zml//platforms/tpu:gen_ir_files", ], + add_directory_to_runfiles = False, + allow_overwrites = True, + exclude_srcs_patterns = [ + "**/include/**", + "**/_solib_*/**", + ], + hardlink = "auto", + include_external_repositories = ["**"], + include_srcs_patterns = [ + "platforms/tpu/gen_ir.py", + "platforms/tpu/gen_ir_zig", + "**/lib**", + "lib/python*/**", + "site-packages/**", + ], + replace_prefixes = { + "platforms/tpu/gen_ir.py": "site-packages/gen_ir.py", + "platforms/tpu/gen_ir_zig": "bin/gen_ir_zig", + "lib/x86_64-linux-gnu": "lib", + }, ) cc_library( diff --git a/platforms/tpu/libpython.h b/platforms/tpu/libpython.h new file mode 100644 index 00000000..605f9fc5 --- /dev/null +++ b/platforms/tpu/libpython.h @@ -0,0 +1,2 @@ +#define PY_SSIZE_T_CLEAN +#include diff --git a/platforms/tpu/requirements.in b/platforms/tpu/requirements.in new file mode 100644 index 00000000..59bf4118 --- /dev/null +++ b/platforms/tpu/requirements.in @@ -0,0 +1,3 @@ +absl-py # required for jax +jax==0.9.2 +# tpu-inference @ https://github.com/vllm-project/tpu-inference/archive/ac940e32079763405e9af1e67fedb890011ad84f.tar.gz diff --git a/platforms/tpu/requirements.lock.txt b/platforms/tpu/requirements.lock.txt new file mode 100644 index 00000000..762a625d --- /dev/null +++ b/platforms/tpu/requirements.lock.txt @@ -0,0 +1,226 @@ +# This file was autogenerated by uv via the following command: +# bazel run //platforms/tpu:requirements.update +--index-url https://pypi.org/simple + +absl-py==2.4.0 \ + --hash=sha256:88476fd881ca8aab94ffa78b7b6c632a782ab3ba1cd19c9bd423abc4fb4cd28d \ + --hash=sha256:8c6af82722b35cf71e0f4d1d47dcaebfff286e27110a99fc359349b247dfb5d4 + # via -r platforms/tpu/requirements.in +jax==0.9.2 \ + --hash=sha256:42b28017b3e6b57a44b0274cc15f5153239c4873959030399ac1afc009c22365 \ + --hash=sha256:822a8ae155ab42e7bc59f2ae7a28705bcfccb01a7e76abfc8ae996190cdc5598 + # via -r platforms/tpu/requirements.in +jaxlib==0.9.2 \ + --hash=sha256:2d445dab57debd8c26b416c8bc91a4704ba6d7169788a961e4b15419bc3f4254 \ + --hash=sha256:306de54a1de7386c806c723e356ce332d923ef748f8a72d674fefb748121d4dc \ + --hash=sha256:35b473df72dbc2cfda0cb1b3de7521a2150a0aa5ef57ed7583eeceb012dc17c0 \ + --hash=sha256:3d7151140a4936f3218b2d1b1343dd237bd2865cf51442884b6d82fe884a3de7 \ + --hash=sha256:501df74472437ffc11aa3bd8f7fc8b1da274f80bd176d33012cf0d604093667d \ + --hash=sha256:52a0032508f8cf5791c7a7bee142531ee706c3c05518117fb0b6ee8d5e17fde7 \ + --hash=sha256:6949d7ecd869c117e7ea8361866e60cf229c3cd9d6afdc37425a43cf83fc89e9 \ + --hash=sha256:6ffb22eccf07bfc8c9760bfbcdaa268df9b3745739e8397bfce5daee5d79cb51 \ + --hash=sha256:785f177c3eb78cb7dc797c55ed5c4b6312141845c9a686957e484bacbfce5e88 \ + --hash=sha256:87bd42c9f18c9cc9a45371d02ecdbdb574ea1e2277149601a92e14a24c4bbc86 \ + --hash=sha256:88b276a71f4f2071b1fd2e922abfd67c87c6977a551a1036febcea78d5ef7e22 \ + --hash=sha256:97c2fbe58cbee4a27d94ca735d709d231b299ab6ed8b3b1075f52d864dfd32c1 \ + --hash=sha256:9ac995b4ba1aaeedae0d69f319987d515dcaecd4505b642b6312f9e15439351f \ + --hash=sha256:b6d5003e3add5c346a34ae9edc47058cbc2db60c8ed5c50096522176daf01c9f \ + --hash=sha256:b8998f9fa6e67bf956044c310023f6a7bbfaa0d8955f11d928404c8f6eb02fcf \ + --hash=sha256:bab168d25555464461bd077323484f690c471e69ce8b0c39a39fb81b3e3a8bf0 \ + --hash=sha256:bbe59bdef668ff5fd998c6d88e8df9a32ab95bec0dea3d2b5f7a11b86a9a6788 \ + --hash=sha256:be4627c42d44add7fe17d284ef579ff8d159e3cb6947f6437758f34177e878e6 \ + --hash=sha256:bef61eef36ed38cec1069ea973f88af9e03335e884f6501ec3fe7f6222a1555b \ + --hash=sha256:c2f0837cc0788746301e68ae9eda468e6a8a7734dc4d529f26a2cb60fb56c657 \ + --hash=sha256:e8e8165f0f647933f0ff9e1e4d9937d541841d3672a20db73f5ccb5e842b0edc \ + --hash=sha256:fef02d846863b726e72452993883a8596eac325f22a2ec7ea921da0fbc5509b4 + # via jax +ml-dtypes==0.5.4 \ + --hash=sha256:0d2ffd05a2575b1519dc928c0b93c06339eb67173ff53acb00724502cda231cf \ + --hash=sha256:11942cbf2cf92157db91e5022633c0d9474d4dfd813a909383bd23ce828a4b7d \ + --hash=sha256:14a4fd3228af936461db66faccef6e4f41c1d82fcc30e9f8d58a08916b1d811f \ + --hash=sha256:19b9a53598f21e453ea2fbda8aa783c20faff8e1eeb0d7ab899309a0053f1483 \ + --hash=sha256:2314892cdc3fcf05e373d76d72aaa15fda9fb98625effa73c1d646f331fcecb7 \ + --hash=sha256:2b857d3af6ac0d39db1de7c706e69c7f9791627209c3d6dedbfca8c7e5faec22 \ + --hash=sha256:304ad47faa395415b9ccbcc06a0350800bc50eda70f0e45326796e27c62f18b6 \ + --hash=sha256:35f29491a3e478407f7047b8a4834e4640a77d2737e0b294d049746507af5175 \ + --hash=sha256:388d399a2152dd79a3f0456a952284a99ee5c93d3e2f8dfe25977511e0515270 \ + --hash=sha256:3bbbe120b915090d9dd1375e4684dd17a20a2491ef25d640a908281da85e73f1 \ + --hash=sha256:3d277bf3637f2a62176f4575512e9ff9ef51d00e39626d9fe4a161992f355af2 \ + --hash=sha256:4381fe2f2452a2d7589689693d3162e876b3ddb0a832cde7a414f8e1adf7eab1 \ + --hash=sha256:4ff7f3e7ca2972e7de850e7b8fcbb355304271e2933dd90814c1cb847414d6e2 \ + --hash=sha256:531eff30e4d368cb6255bc2328d070e35836aa4f282a0fb5f3a0cd7260257298 \ + --hash=sha256:533ce891ba774eabf607172254f2e7260ba5f57bdd64030c9a4fcfbd99815d0d \ + --hash=sha256:557a31a390b7e9439056644cb80ed0735a6e3e3bb09d67fd5687e4b04238d1de \ + --hash=sha256:5a0f68ca8fd8d16583dfa7793973feb86f2fbb56ce3966daf9c9f748f52a2049 \ + --hash=sha256:6a0df4223b514d799b8a1629c65ddc351b3efa833ccf7f8ea0cf654a61d1e35d \ + --hash=sha256:6c7ecb74c4bd71db68a6bea1edf8da8c34f3d9fe218f038814fd1d310ac76c90 \ + --hash=sha256:7c23c54a00ae43edf48d44066a7ec31e05fdc2eee0be2b8b50dd1903a1db94bb \ + --hash=sha256:805cef3a38f4eafae3a5bf9ebdcdb741d0bcfd9e1bd90eb54abd24f928cd2465 \ + --hash=sha256:88c982aac7cb1cbe8cbb4e7f253072b1df872701fcaf48d84ffbb433b6568f24 \ + --hash=sha256:8ab06a50fb9bf9666dd0fe5dfb4676fa2b0ac0f31ecff72a6c3af8e22c063453 \ + --hash=sha256:8c6a2dcebd6f3903e05d51960a8058d6e131fe69f952a5397e5dbabc841b6d56 \ + --hash=sha256:8c760d85a2f82e2bed75867079188c9d18dae2ee77c25a54d60e9cc79be1bc48 \ + --hash=sha256:9ad459e99793fa6e13bd5b7e6792c8f9190b4e5a1b45c63aba14a4d0a7f1d5ff \ + --hash=sha256:9bad06436568442575beb2d03389aa7456c690a5b05892c471215bfd8cf39460 \ + --hash=sha256:a174837a64f5b16cab6f368171a1a03a27936b31699d167684073ff1c4237dac \ + --hash=sha256:a7f7c643e8b1320fd958bf098aa7ecf70623a42ec5154e3be3be673f4c34d900 \ + --hash=sha256:a9b61c19040397970d18d7737375cffd83b1f36a11dd4ad19f83a016f736c3ef \ + --hash=sha256:b4b801ebe0b477be666696bda493a9be8356f1f0057a57f1e35cd26928823e5a \ + --hash=sha256:b95e97e470fe60ed493fd9ae3911d8da4ebac16bd21f87ffa2b7c588bf22ea2c \ + --hash=sha256:bc11d7e8c44a65115d05e2ab9989d1e045125d7be8e05a071a48bc76eb6d6040 \ + --hash=sha256:bfc534409c5d4b0bf945af29e5d0ab075eae9eecbb549ff8a29280db822f34f9 \ + --hash=sha256:c1a953995cccb9e25a4ae19e34316671e4e2edaebe4cf538229b1fc7109087b7 \ + --hash=sha256:cb73dccfc991691c444acc8c0012bee8f2470da826a92e3a20bb333b1a7894e6 \ + --hash=sha256:ce756d3a10d0c4067172804c9cc276ba9cc0ff47af9078ad439b075d1abdc29b \ + --hash=sha256:d81fdb088defa30eb37bf390bb7dde35d3a83ec112ac8e33d75ab28cc29dd8b0 \ + --hash=sha256:f21c9219ef48ca5ee78402d5cc831bd58ea27ce89beda894428bc67a52da5328 + # via + # jax + # jaxlib +numpy==2.4.4 \ + --hash=sha256:07077278157d02f65c43b1b26a3886bce886f95d20aabd11f87932750dfb14ed \ + --hash=sha256:08f2e31ed5e6f04b118e49821397f12767934cfdd12a1ce86a058f91e004ee50 \ + --hash=sha256:0aec54fd785890ecca25a6003fd9a5aed47ad607bbac5cd64f836ad8666f4959 \ + --hash=sha256:0d35aea54ad1d420c812bfa0385c71cd7cc5bcf7c65fed95fc2cd02fe8c79827 \ + --hash=sha256:0d4e437e295f18ec29bc79daf55e8a47a9113df44d66f702f02a293d93a2d6dd \ + --hash=sha256:0dfd3f9d3adbe2920b68b5cd3d51444e13a10792ec7154cd0a2f6e74d4ab3233 \ + --hash=sha256:1378871da56ca8943c2ba674530924bb8ca40cd228358a3b5f302ad60cf875fc \ + --hash=sha256:15716cfef24d3a9762e3acdf87e27f58dc823d1348f765bbea6bef8c639bfa1b \ + --hash=sha256:19710a9ca9992d7174e9c52f643d4272dcd1558c5f7af7f6f8190f633bd651a7 \ + --hash=sha256:23cbfd4c17357c81021f21540da84ee282b9c8fba38a03b7b9d09ba6b951421e \ + --hash=sha256:2483e4584a1cb3092da4470b38866634bafb223cbcd551ee047633fd2584599a \ + --hash=sha256:27a8d92cd10f1382a67d7cf4db7ce18341b66438bdd9f691d7b0e48d104c2a9d \ + --hash=sha256:28a650663f7314afc3e6ec620f44f333c386aad9f6fc472030865dc0ebb26ee3 \ + --hash=sha256:2aa0613a5177c264ff5921051a5719d20095ea586ca88cc802c5c218d1c67d3e \ + --hash=sha256:2c194dd721e54ecad9ad387c1d35e63dce5c4450c6dc7dd5611283dda239aabb \ + --hash=sha256:2d19e6e2095506d1736b7d80595e0f252d76b89f5e715c35e06e937679ea7d7a \ + --hash=sha256:2d390634c5182175533585cc89f3608a4682ccb173cc9bb940b2881c8d6f8fa0 \ + --hash=sha256:30caa73029a225b2d40d9fae193e008e24b2026b7ee1a867b7ee8d96ca1a448e \ + --hash=sha256:42c16925aa5a02362f986765f9ebabf20de75cdefdca827d14315c568dcab113 \ + --hash=sha256:45dbed2ab436a9e826e302fcdcbe9133f9b0006e5af7168afb8963a6520da103 \ + --hash=sha256:4636de7fd195197b7535f231b5de9e4b36d2c440b6e566d2e4e4746e6af0ca93 \ + --hash=sha256:4a19d9dba1a76618dd86b164d608566f393f8ec6ac7c44f0cc879011c45e65af \ + --hash=sha256:4bbc7f303d125971f60ec0aaad5e12c62d0d2c925f0ab1273debd0e4ba37aba5 \ + --hash=sha256:4d6d57903571f86180eb98f8f0c839fa9ebbfb031356d87f1361be91e433f5b7 \ + --hash=sha256:4e874c976154687c1f71715b034739b45c7711bec81db01914770373d125e392 \ + --hash=sha256:51fc224f7ca4d92656d5a5eb315f12eb5fe2c97a66249aa7b5f562528a3be38c \ + --hash=sha256:58c8b5929fcb8287cbd6f0a3fae19c6e03a5c48402ae792962ac465224a629a4 \ + --hash=sha256:5a285b3b96f951841799528cd1f4f01cd70e7e0204b4abebac9463eecfcf2a40 \ + --hash=sha256:5c70f1cc1c4efbe316a572e2d8b9b9cc44e89b95f79ca3331553fbb63716e2bf \ + --hash=sha256:62d6b0f03b694173f9fcb1fb317f7222fd0b0b103e784c6549f5e53a27718c44 \ + --hash=sha256:6a246d5914aa1c820c9443ddcee9c02bec3e203b0c080349533fae17727dfd1b \ + --hash=sha256:6aa3236c78803afbcb255045fbef97a9e25a1f6c9888357d205ddc42f4d6eba5 \ + --hash=sha256:6bbe4eb67390b0a0265a2c25458f6b90a409d5d069f1041e6aff1e27e3d9a79e \ + --hash=sha256:715d1c092715954784bc79e1174fc2a90093dc4dc84ea15eb14dad8abdcdeb74 \ + --hash=sha256:72944b19f2324114e9dc86a159787333b77874143efcf89a5167ef83cfee8af0 \ + --hash=sha256:81f4a14bee47aec54f883e0cad2d73986640c1590eb9bfaaba7ad17394481e6e \ + --hash=sha256:846300f379b5b12cc769334464656bc882e0735d27d9726568bc932fdc49d5ec \ + --hash=sha256:86b6f55f5a352b48d7fbfd2dbc3d5b780b2d79f4d3c121f33eb6efb22e9a2015 \ + --hash=sha256:874f200b2a981c647340f841730fc3a2b54c9d940566a3c4149099591e2c4c3d \ + --hash=sha256:8a87ec22c87be071b6bdbd27920b129b94f2fc964358ce38f3822635a3e2e03d \ + --hash=sha256:8b3b60bb7cba2c8c81837661c488637eee696f59a877788a396d33150c35d842 \ + --hash=sha256:8e3ed142f2728df44263aaf5fb1f5b0b99f4070c553a0d7f033be65338329150 \ + --hash=sha256:93e15038125dc1e5345d9b5b68aa7f996ec33b98118d18c6ca0d0b7d6198b7e8 \ + --hash=sha256:989824e9faf85f96ec9c7761cd8d29c531ad857bfa1daa930cba85baaecf1a9a \ + --hash=sha256:99d838547ace2c4aace6c4f76e879ddfe02bb58a80c1549928477862b7a6d6ed \ + --hash=sha256:9b2aec6af35c113b05695ebb5749a787acd63cafc83086a05771d1e1cd1e555f \ + --hash=sha256:9c585a1790d5436a5374bac930dad6ed244c046ed91b2b2a3634eb2971d21008 \ + --hash=sha256:a7164afb23be6e37ad90b2f10426149fd75aee07ca55653d2aa41e66c4ef697e \ + --hash=sha256:ac6b31e35612a26483e20750126d30d0941f949426974cace8e6b5c58a3657b0 \ + --hash=sha256:ad2e2ef14e0b04e544ea2fa0a36463f847f113d314aa02e5b402fdf910ef309e \ + --hash=sha256:b268594bccac7d7cf5844c7732e3f20c50921d94e36d7ec9b79e9857694b1b2f \ + --hash=sha256:b5f0362dc928a6ecd9db58868fca5e48485205e3855957bdedea308f8672ea4a \ + --hash=sha256:ba1f4fc670ed79f876f70082eff4f9583c15fb9a4b89d6188412de4d18ae2f40 \ + --hash=sha256:ba203255017337d39f89bdd58417f03c4426f12beed0440cfd933cb15f8669c7 \ + --hash=sha256:c901b15172510173f5cb310eae652908340f8dede90fff9e3bf6c0d8dfd92f83 \ + --hash=sha256:c9b39d38a9bd2ae1becd7eac1303d031c5c110ad31f2b319c6e7d98b135c934d \ + --hash=sha256:d2a8490669bfe99a233298348acc2d824d496dee0e66e31b66a6022c2ad74a5c \ + --hash=sha256:dddbbd259598d7240b18c9d87c56a9d2fb3b02fe266f49a7c101532e78c1d871 \ + --hash=sha256:df3775294accfdd75f32c74ae39fcba920c9a378a2fc18a12b6820aa8c1fb502 \ + --hash=sha256:e44319a2953c738205bf3354537979eaa3998ed673395b964c1176083dd46252 \ + --hash=sha256:e4a010c27ff6f210ff4c6ef34394cd61470d01014439b192ec22552ee867f2a8 \ + --hash=sha256:e823b8b6edc81e747526f70f71a9c0a07ac4e7ad13020aa736bb7c9d67196115 \ + --hash=sha256:e892aff75639bbef0d2a2cfd55535510df26ff92f63c92cd84ef8d4ba5a5557f \ + --hash=sha256:eea7ac5d2dce4189771cedb559c738a71512768210dc4e4753b107a2048b3d0e \ + --hash=sha256:ef4059d6e5152fa1a39f888e344c73fdc926e1b2dd58c771d67b0acfbf2aa67d \ + --hash=sha256:f169b9a863d34f5d11b8698ead99febeaa17a13ca044961aa8e2662a6c7766a0 \ + --hash=sha256:f2cf083b324a467e1ab358c105f6cad5ea950f50524668a80c486ff1db24e119 \ + --hash=sha256:f8474c4241bc18b750be2abea9d7a9ec84f46ef861dbacf86a4f6e043401f79e \ + --hash=sha256:f983334aea213c99992053ede6168500e5f086ce74fbc4acc3f2b00f5762e9db \ + --hash=sha256:f9e75681b59ddaa5e659898085ae0eaea229d054f2ac0c7e563a62205a700121 \ + --hash=sha256:fbc356aae7adf9e6336d336b9c8111d390a05df88f1805573ebb0807bd06fd1d \ + --hash=sha256:fcfe2045fd2e8f3cb0ce9d4ba6dba6333b8fa05bb8a4939c908cd43322d14c7e + # via + # jax + # jaxlib + # ml-dtypes + # scipy +opt-einsum==3.4.0 \ + --hash=sha256:69bb92469f86a1565195ece4ac0323943e83477171b91d24c35afe028a90d7cd \ + --hash=sha256:96ca72f1b886d148241348783498194c577fa30a8faac108586b14f1ba4473ac + # via jax +scipy==1.17.1 \ + --hash=sha256:010f4333c96c9bb1a4516269e33cb5917b08ef2166d5556ca2fd9f082a9e6ea0 \ + --hash=sha256:02ae3b274fde71c5e92ac4d54bc06c42d80e399fec704383dcd99b301df37458 \ + --hash=sha256:08b900519463543aa604a06bec02461558a6e1cef8fdbb8098f77a48a83c8118 \ + --hash=sha256:131f5aaea57602008f9822e2115029b55d4b5f7c070287699fe45c661d051e39 \ + --hash=sha256:158dd96d2207e21c966063e1635b1063cd7787b627b6f07305315dd73d9c679e \ + --hash=sha256:1cc682cea2ae55524432f3cdff9e9a3be743d52a7443d0cba9017c23c87ae2f6 \ + --hash=sha256:1f95b894f13729334fb990162e911c9e5dc1ab390c58aa6cbecb389c5b5e28ec \ + --hash=sha256:200e1050faffacc162be6a486a984a0497866ec54149a01270adc8a59b7c7d21 \ + --hash=sha256:2040ad4d1795a0ae89bfc7e8429677f365d45aa9fd5e4587cf1ea737f927b4a1 \ + --hash=sha256:2b64ca7d4aee0102a97f3ba22124052b4bd2152522355073580bf4845e2550b6 \ + --hash=sha256:2ceb2d3e01c5f1d83c4189737a42d9cb2fc38a6eeed225e7515eef71ad301dce \ + --hash=sha256:35c3a56d2ef83efc372eaec584314bd0ef2e2f0d2adb21c55e6ad5b344c0dcb8 \ + --hash=sha256:37425bc9175607b0268f493d79a292c39f9d001a357bebb6b88fdfaff13f6448 \ + --hash=sha256:3877ac408e14da24a6196de0ddcace62092bfc12a83823e92e49e40747e52c19 \ + --hash=sha256:3fd1fcdab3ea951b610dc4cef356d416d5802991e7e32b5254828d342f7b7e0b \ + --hash=sha256:41b71f4a3a4cab9d366cd9065b288efc4d4f3c0b37a91a8e0947fb5bd7f31d87 \ + --hash=sha256:43af8d1f3bea642559019edfe64e9b11192a8978efbd1539d7bc2aaa23d92de4 \ + --hash=sha256:45abad819184f07240d8a696117a7aacd39787af9e0b719d00285549ed19a1e9 \ + --hash=sha256:4b400bdc6f79fa02a4d86640310dde87a21fba0c979efff5248908c6f15fad1b \ + --hash=sha256:4eb6c25dd62ee8d5edf68a8e1c171dd71c292fdae95d8aeb3dd7d7de4c364082 \ + --hash=sha256:581b2264fc0aa555f3f435a5944da7504ea3a065d7029ad60e7c3d1ae09c5464 \ + --hash=sha256:5cf36e801231b6a2059bf354720274b7558746f3b1a4efb43fcf557ccd484a87 \ + --hash=sha256:5e3c5c011904115f88a39308379c17f91546f77c1667cea98739fe0fccea804c \ + --hash=sha256:6609bc224e9568f65064cfa72edc0f24ee6655b47575954ec6339534b2798369 \ + --hash=sha256:6e3dcd57ab780c741fde8dc68619de988b966db759a3c3152e8e9142c26295ad \ + --hash=sha256:6fac755ca3d2c3edcb22f479fceaa241704111414831ddd3bc6056e18516892f \ + --hash=sha256:744b2bf3640d907b79f3fd7874efe432d1cf171ee721243e350f55234b4cec4c \ + --hash=sha256:74cbb80d93260fe2ffa334efa24cb8f2f0f622a9b9febf8b483c0b865bfb3475 \ + --hash=sha256:766e0dc5a616d026a3a1cffa379af959671729083882f50307e18175797b3dfd \ + --hash=sha256:7bdf2da170b67fdf10bca777614b1c7d96ae3ca5794fd9587dce41eb2966e866 \ + --hash=sha256:7ff200bf9d24f2e4d5dc6ee8c3ac64d739d3a89e2326ba68aaf6c4a2b838fd7d \ + --hash=sha256:844e165636711ef41f80b4103ed234181646b98a53c8f05da12ca5ca289134f6 \ + --hash=sha256:8a604bae87c6195d8b1045eddece0514d041604b14f2727bbc2b3020172045eb \ + --hash=sha256:94055a11dfebe37c656e70317e1996dc197e1a15bbcc351bcdd4610e128fe1ca \ + --hash=sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0 \ + --hash=sha256:9cdc1a2fcfd5c52cfb3045feb399f7b3ce822abdde3a193a6b9a60b3cb5854ca \ + --hash=sha256:9ecb4efb1cd6e8c4afea0daa91a87fbddbce1b99d2895d151596716c0b2e859d \ + --hash=sha256:a3472cfbca0a54177d0faa68f697d8ba4c80bbdc19908c3465556d9f7efce9ee \ + --hash=sha256:a4328d245944d09fd639771de275701ccadf5f781ba0ff092ad141e017eccda4 \ + --hash=sha256:a48a72c77a310327f6a3a920092fa2b8fd03d7deaa60f093038f22d98e096717 \ + --hash=sha256:a720477885a9d2411f94a93d16f9d89bad0f28ca23c3f8daa521e2dcc3f44d49 \ + --hash=sha256:a77cbd07b940d326d39a1d1b37817e2ee4d79cb30e7338f3d0cddffae70fcaa2 \ + --hash=sha256:a9956e4d4f4a301ebf6cde39850333a6b6110799d470dbbb1e25326ac447f52a \ + --hash=sha256:adb2642e060a6549c343603a3851ba76ef0b74cc8c079a9a58121c7ec9fe2350 \ + --hash=sha256:beeda3d4ae615106d7094f7e7cef6218392e4465cc95d25f900bebabfded0950 \ + --hash=sha256:c80be5ede8f3f8eded4eff73cc99a25c388ce98e555b17d31da05287015ffa5b \ + --hash=sha256:cc90d2e9c7e5c7f1a482c9875007c095c3194b1cfedca3c2f3291cdc2bc7c086 \ + --hash=sha256:cd96a1898c0a47be4520327e01f874acfd61fb48a9420f8aa9f6483412ffa444 \ + --hash=sha256:d2650c1fb97e184d12d8ba010493ee7b322864f7d3d00d3f9bb97d9c21de4068 \ + --hash=sha256:d30e57c72013c2a4fe441c2fcb8e77b14e152ad48b5464858e07e2ad9fbfceff \ + --hash=sha256:d59c30000a16d8edc7e64152e30220bfbd724c9bbb08368c054e24c651314f0a \ + --hash=sha256:dbc12c9f3d185f5c737d801da555fb74b3dcfa1a50b66a1a93e09190f41fab50 \ + --hash=sha256:e18f12c6b0bc5a592ed23d3f7b891f68fd7f8241d69b7883769eb5d5dfb52696 \ + --hash=sha256:e19ebea31758fac5893a2ac360fedd00116cbb7628e650842a6691ba7ca28a21 \ + --hash=sha256:e30bdeaa5deed6bc27b4cc490823cd0347d7dae09119b8803ae576ea0ce52e4c \ + --hash=sha256:eb092099205ef62cd1782b006658db09e2fed75bffcae7cc0d44052d8aa0f484 \ + --hash=sha256:eee2cfda04c00a857206a4330f0c5e3e56535494e30ca445eb19ec624ae75118 \ + --hash=sha256:f4115102802df98b2b0db3cce5cb9b92572633a1197c77b7553e5203f284a5b3 \ + --hash=sha256:f590cd684941912d10becc07325a3eeb77886fe981415660d9265c4c418d0bea \ + --hash=sha256:f8885db0bc2bffa59d5c1b72fad7a6a92d3e80e7257f967dd81abb553a90d293 \ + --hash=sha256:fcb310ddb270a06114bb64bbe53c94926b943f5b7f0842194d585c65eb4edd76 + # via + # jax + # jaxlib diff --git a/platforms/tpu/tpu.bzl b/platforms/tpu/tpu.bzl index 89d86e64..2e7538dd 100644 --- a/platforms/tpu/tpu.bzl +++ b/platforms/tpu/tpu.bzl @@ -4,9 +4,9 @@ def _tpu_impl(mctx): # https://storage.googleapis.com/jax-releases/libtpu_releases.html http_archive( name = "libpjrt_tpu", - url = "https://files.pythonhosted.org/packages/5e/ea/6271a8906d5509349dde55da1f516ac232e73b3cd8e1fba095f4132142bf/libtpu-0.0.36-cp314-cp314t-manylinux_2_31_x86_64.whl", + url = "https://files.pythonhosted.org/packages/42/7f/cc3ad9e9b866c701e90e6d8d4e332557fefb1f3ad1bebd9914ff09778691/libtpu-0.0.40-cp314-cp314t-manylinux_2_31_x86_64.whl", type = "zip", - sha256 = "6d0e6a2ec26b851f5a00c74933738b2b185af47c4eacbd161e23954b1d911ae4", + sha256 = "a1d01214bffe5a0910057014eaa17ed838eebc780e1538b4bd118684908ea120", build_file = "libpjrt_tpu.BUILD.bazel", ) return mctx.extension_metadata( diff --git a/platforms/tpu/tpu.zig b/platforms/tpu/tpu.zig index e19a7606..95c0d449 100644 --- a/platforms/tpu/tpu.zig +++ b/platforms/tpu/tpu.zig @@ -5,7 +5,6 @@ const bazel = @import("bazel"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); -const runfiles = @import("runfiles"); const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/platforms/tpu"); diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index a8cee145..8c41c76c 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -34,6 +34,7 @@ zig_library( "attention/attention.zig", "attention/flashattn.zig", "attention/paged_attention.zig", + "attention/tpu_attention.zig", "attention/triton.zig", "buffer.zig", "constants.zig", diff --git a/zml/attention.zig b/zml/attention.zig index 909bbd14..67869b3f 100644 --- a/zml/attention.zig +++ b/zml/attention.zig @@ -1,4 +1,5 @@ pub const attention = @import("attention/attention.zig"); pub const paged_attention = @import("attention/paged_attention.zig"); pub const flashattn = @import("attention/flashattn.zig"); +pub const tpu = @import("attention/tpu_attention.zig"); pub const triton = @import("attention/triton.zig"); diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 0191b823..9293276c 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -4,6 +4,7 @@ const stdx = @import("stdx"); const zml = @import("../zml.zig"); const flashattn = @import("flashattn.zig"); +const tpu = @import("tpu_attention.zig"); const triton = @import("triton.zig"); const PagedAttention = @This(); @@ -12,6 +13,7 @@ pub const Backend = enum { cuda_fa2, cuda_fa3, triton, + mosaic_tpu, pub fn auto(platform: *const zml.Platform) Backend { return switch (platform.target) { @@ -27,6 +29,7 @@ pub const Backend = enum { break :b .cuda_fa2; }, .rocm => .triton, + .tpu => .mosaic_tpu, else => stdx.debug.panic("Paged attention is not supported on {s} yet", .{@tagName(platform.target)}), }; } @@ -36,6 +39,7 @@ pub const Options = union(Backend) { cuda_fa2: flashattn.paged_fa2.Options, cuda_fa3: flashattn.paged_fa3.Options, triton: triton.paged.Options, + mosaic_tpu: tpu.mosaic_tpu.Options, const Args = struct { backend: Backend, @@ -113,6 +117,18 @@ pub const Options = union(Backend) { .is_prefill = args.is_prefill, }, }, + .mosaic_tpu => .{ + .mosaic_tpu = .{ + .is_prefill = args.is_prefill, + .batch_size = args.batch_size, + .max_num_pages = max_num_pages, + .max_seqlen_k = args.seq_len, + .max_token_count = args.max_token_count, + .num_heads = args.num_heads, + .num_kv_heads = args.num_kv_heads, + .head_dim = args.head_dim, + }, + }, }; } @@ -133,12 +149,14 @@ pub const Parameters = union(Backend) { cuda_fa2: flashattn.paged_fa2.Parameters, cuda_fa3: flashattn.paged_fa3.Parameters, triton: triton.paged.Parameters, + mosaic_tpu: tpu.mosaic_tpu.Parameters, pub fn init(options_: Options) Parameters { return switch (options_) { .cuda_fa2 => |cuda_fa2_options| .{ .cuda_fa2 = flashattn.paged_fa2.Parameters.init(cuda_fa2_options) }, .cuda_fa3 => |cuda_fa3_options| .{ .cuda_fa3 = flashattn.paged_fa3.Parameters.init(cuda_fa3_options) }, .triton => |triton_options| .{ .triton = triton.paged.Parameters.init(triton_options) }, + .mosaic_tpu => |mosaic_tpu_options| .{ .mosaic_tpu = tpu.mosaic_tpu.Parameters.init(mosaic_tpu_options) }, }; } @@ -147,6 +165,7 @@ pub const Parameters = union(Backend) { .cuda_fa2 => |v| .{ .cuda_fa2 = v.options() }, .cuda_fa3 => |v| .{ .cuda_fa3 = v.options() }, .triton => |v| .{ .triton = v.options() }, + .mosaic_tpu => |v| .{ .mosaic_tpu = v.options() }, }; } @@ -162,12 +181,14 @@ pub const Context = union(Backend) { cuda_fa2: flashattn.paged_fa2.Context, cuda_fa3: flashattn.paged_fa3.Context, triton: triton.paged.Context, + mosaic_tpu: tpu.mosaic_tpu.Context, pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { return switch (parameters) { .cuda_fa2 => |cuda_fa2_parameters| .{ .cuda_fa2 = flashattn.paged_fa2.Context.init(cuda_fa2_parameters, num_heads, num_kv_heads, head_dim, page_size) }, .cuda_fa3 => |cuda_fa3_parameters| .{ .cuda_fa3 = flashattn.paged_fa3.Context.init(cuda_fa3_parameters, num_heads, num_kv_heads, head_dim, page_size) }, .triton => |triton_parameters| .{ .triton = triton.paged.Context.init(triton_parameters, num_heads, num_kv_heads, head_dim, page_size) }, + .mosaic_tpu => |mosaic_tpu_parameters| .{ .mosaic_tpu = tpu.mosaic_tpu.Context.init(mosaic_tpu_parameters, num_heads, num_kv_heads, head_dim, page_size) }, }; } }; @@ -179,11 +200,55 @@ pub const AttentionOptions = struct { }; pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { - _ = k; - _ = v; return switch (parameters) { .cuda_fa2 => |cuda_fa2_parameters| flashattn.paged_fa2.pagedAttention(cuda_fa2_parameters, context.cuda_fa2, q, k_cache, v_cache, opts), .cuda_fa3 => |cuda_fa3_parameters| flashattn.paged_fa3.pagedAttention(cuda_fa3_parameters, context.cuda_fa3, q, k_cache, v_cache, opts), .triton => |triton_parameters| triton.paged.pagedAttention(triton_parameters, context.triton, q, k_cache, v_cache, opts), + .mosaic_tpu => |mosaic_tpu_parameters| tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, context.mosaic_tpu, q, k, v, k_cache, v_cache, opts), }; } + +test "Options.fromBackend configures mosaic_tpu ragged backend" { + const options = Options.fromBackend(.{ + .backend = .mosaic_tpu, + .is_prefill = false, + .batch_size = 4, + .seq_len = 512, + .page_chunk_size = 16, + .max_token_count = 4, + .num_heads = 16, + .num_kv_heads = 4, + .head_dim = 128, + .max_seqlen_q = 1, + }); + + switch (options) { + .mosaic_tpu => |tpu_option| { + try std.testing.expect(!tpu_option.is_prefill); + try std.testing.expectEqual(@as(usize, 4), tpu_option.batch_size); + try std.testing.expectEqual(@as(usize, 32), tpu_option.max_num_pages); + try std.testing.expectEqual(@as(usize, 512), tpu_option.max_seqlen_k); + try std.testing.expectEqual(@as(usize, 4), tpu_option.max_token_count); + try std.testing.expectEqual(@as(usize, 16), tpu_option.num_heads); + try std.testing.expectEqual(@as(usize, 4), tpu_option.num_kv_heads); + try std.testing.expectEqual(@as(usize, 128), tpu_option.head_dim); + }, + else => return error.UnexpectedBackendVariant, + } +} + +test "Backend.auto selects mosaic_tpu on TPU" { + const platform: zml.Platform = .{ + .arena_state = .{}, + .target = .tpu, + .pjrt_api = undefined, + .pjrt_client = undefined, + .devices = &[_]zml.platform.Device{}, + .memories = &[_]zml.platform.Memory{}, + .physical_mesh = undefined, + .triton_runtime = null, + .tpu_ir_runtime = null, + }; + + try std.testing.expectEqual(Backend.mosaic_tpu, Backend.auto(&platform)); +} diff --git a/zml/attention/tpu_attention.zig b/zml/attention/tpu_attention.zig new file mode 100644 index 00000000..1a052397 --- /dev/null +++ b/zml/attention/tpu_attention.zig @@ -0,0 +1,412 @@ +const std = @import("std"); + +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); +const dialects = @import("mlir/dialects"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +const CompilationContext = @import("../module.zig").CompilationContext; +const zml = @import("../zml.zig"); +const AttentionOptions = @import("paged_attention.zig").AttentionOptions; + +const log = std.log.scoped(.tpu_attention); + +pub const Runtime = struct { + process: std.process.Child, + request_mutex: std.Io.Mutex = .init, + + fn getGenerateBinPath(allocator: std.mem.Allocator) ![]const u8 { + const runfiles = try bazel.runfiles(bazel_builtin.current_repository); + + var sandbox_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = try runfiles.rlocation("libpjrt_tpu/sandbox", &sandbox_path_buf) orelse return error.FileNotFound; + + return std.fs.path.join(allocator, &.{ sandbox_path, "bin", "gen_ir_zig" }); + } + + pub fn init(allocator: std.mem.Allocator, io: std.Io) !Runtime { + const path = try getGenerateBinPath(allocator); + defer allocator.free(path); + + var process = try std.process.spawn(io, .{ + .argv = &.{path}, + .stdin = .pipe, + .stdout = .pipe, + }); + errdefer process.kill(io); + + return .{ .process = process }; + } + + pub fn request(self: *Runtime, allocator: std.mem.Allocator, io: std.Io, request_json: []const u8) ![]u8 { + self.request_mutex.lockUncancelable(io); + defer self.request_mutex.unlock(io); + + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + const writer_buffer = try arena.allocator().alloc(u8, 4096); + var writer = self.process.stdin.?.writer(io, writer_buffer); + try writer.interface.print("{s}\n", .{request_json}); + try writer.interface.flush(); + + const reader_buffer = try arena.allocator().alloc(u8, 4096); + var reader = self.process.stdout.?.reader(io, reader_buffer); + var allocating: std.Io.Writer.Allocating = .init(arena.allocator()); + _ = try reader.interface.streamDelimiter(&allocating.writer, '\n'); + _ = try reader.interface.discardShort(1); + + const response: std.json.Value = try std.json.parseFromSliceLeaky(std.json.Value, arena.allocator(), allocating.written(), .{}); + const response_object = switch (response) { + .object => |object| object, + else => { + log.err("TPU backend config response is not a JSON object: {s}", .{allocating.written()}); + return error.InvalidResponse; + }, + }; + const ok = response_object.get("ok") orelse { + log.err("TPU backend config response is missing `ok`: {s}", .{allocating.written()}); + return error.InvalidResponse; + }; + const ok_bool = switch (ok) { + .bool => |value| value, + else => { + log.err("TPU backend config response has non-bool `ok`: {s}", .{allocating.written()}); + return error.InvalidResponse; + }, + }; + + if (ok_bool) { + const result = response_object.get("result") orelse { + log.err("TPU backend config response is missing `result`: {s}", .{allocating.written()}); + return error.InvalidResponse; + }; + return switch (result) { + .string => |value| try allocator.dupe(u8, value), + else => { + log.err("TPU backend config response has non-string `result`: {s}", .{allocating.written()}); + return error.InvalidResponse; + }, + }; + } + + if (response_object.get("error")) |value| { + switch (value) { + .string => |message| log.err("TPU backend config generation failed: {s}", .{message}), + else => log.err("TPU backend config generation failed with invalid error payload: {s}", .{allocating.written()}), + } + } else { + log.err("TPU backend config generation failed without an error payload: {s}", .{allocating.written()}); + } + return error.GenerationFailed; + } + + pub fn deinit(self: *Runtime, io: std.Io) void { + _ = std.c.kill(self.process.id.?, .INT); + _ = self.process.wait(io) catch unreachable; + } +}; + +pub const mosaic_tpu = struct { + const RaggedPagedRequest = struct { + num_q_tokens: i64, + num_q_heads: i64, + num_kv_heads: i64, + head_dim: i64, + q_dtype: []const u8, + kv_dtype: []const u8, + max_num_seqs: i64, + pages_per_seq: i64, + total_num_pages: i64, + page_size: i64, + sm_scale: f32, + sliding_window: ?i64 = null, + num_kv_pages_per_block: u32, + num_queries_per_block: u32, + }; + + const PreparedInputs = struct { + q: zml.Tensor, + kv_pages: zml.Tensor, + kv_lens: zml.Tensor, + page_indices: zml.Tensor, + cu_seqlens_q: zml.Tensor, + num_seqs: zml.Tensor, + }; + + pub const Options = struct { + is_prefill: bool, + batch_size: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_token_count: usize, + num_heads: usize, + num_kv_heads: usize, + head_dim: usize, + pages_per_compute_block: usize = 1, + + pub fn isPrefill(self: Options) bool { + return self.is_prefill; + } + + pub fn maxNumPages(self: Options) usize { + return self.max_num_pages; + } + }; + + pub const Parameters = struct { + opts: Options, + cu_seqlens_q: zml.Tensor, + page_indices: zml.Tensor, + kv_lens: zml.Tensor, + + pub fn init(opts: Options) Parameters { + return .{ + .opts = opts, + .cu_seqlens_q = zml.Tensor.init(.{ .n = opts.batch_size + 1 }, .i32), + .page_indices = zml.Tensor.init(.{ opts.batch_size, opts.max_num_pages }, .i32), + .kv_lens = zml.Tensor.init(.{opts.batch_size}, .i32), + }; + } + + pub fn options(self: Parameters) Options { + return self.opts; + } + + pub fn allocationSize(self: Parameters) usize { + var allocation_size: usize = 0; + allocation_size += self.cu_seqlens_q.byteSize(); + allocation_size += self.page_indices.byteSize(); + allocation_size += self.kv_lens.byteSize(); + return allocation_size; + } + }; + + fn kernelFrontendAttrs(comp_ctx: *CompilationContext) *const mlir.Attribute { + return mlir.dictionaryAttribute(comp_ctx.mlir_ctx, &.{ + mlir.NamedAttribute.named(comp_ctx.mlir_ctx, "kernel_metadata", mlir.stringAttribute(comp_ctx.mlir_ctx, "{}")), + }); + } + + fn raggedPagedKernelCall( + q: zml.Tensor, + kv_pages: zml.Tensor, + kv_lens: zml.Tensor, + page_indices: zml.Tensor, + cu_q_lens: zml.Tensor, + num_seqs: zml.Tensor, + backend_config: []const u8, + ) zml.Tensor { + const seq_buf_idx = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{2}, .i32)); + + const comp_ctx = CompilationContext.current(); + const op = dialects.stablehlo.custom_call( + comp_ctx.mlir_ctx, + &.{ + kv_lens.value(), + page_indices.value(), + cu_q_lens.value(), + seq_buf_idx.value(), + num_seqs.value(), + q.value(), + kv_pages.value(), + }, + &.{q.value().type_()}, + .{ + .call_target_name = "tpu_custom_call", + .backend_config = .{ .original = backend_config }, + .has_side_effect = false, + .additional_attributes = &.{ + mlir.NamedAttribute.named(comp_ctx.mlir_ctx, "kernel_name", mlir.stringAttribute(comp_ctx.mlir_ctx, "ragged_paged_attention_kernel")), + mlir.NamedAttribute.named(comp_ctx.mlir_ctx, "mhlo.frontend_attributes", kernelFrontendAttrs(comp_ctx)), + }, + }, + .unknown(comp_ctx.mlir_ctx), + ).appendTo(comp_ctx.currentScope().block); + return zml.Tensor._result(q.shape(), op.result(0)); + } + + fn activeSequenceCount(cu_seqlens_q: zml.Tensor) zml.Tensor { + const start = cu_seqlens_q.slice1d(0, .{ .end = cu_seqlens_q.dim(0) - 1 }); + const end = cu_seqlens_q.slice1d(0, .{ .start = 1 }); + const query_lens = end.sub(start); + return query_lens + .cmp(.GT, zml.Tensor.constant(query_lens.dtype().zero()).broad(query_lens.shape())) + .convert(.i32) + .sum(0) + .reshape(zml.Shape.init(.{1}, .i32)); + } + + fn jnpDTypeExpr(dtype: zml.DataType) []const u8 { + return switch (dtype) { + .bf16 => "jnp.bfloat16", + .f32 => "jnp.float32", + else => stdx.debug.panic("mosaic_tpu ragged paged attention expects q/kv dtype bf16 or f32, got {}", .{dtype}), + }; + } + + fn renderBackendConfig(allocator: std.mem.Allocator, io: std.Io, request: RaggedPagedRequest) []u8 { + const compilation_context = CompilationContext.current(); + + const json_string = std.fmt.allocPrint( + allocator, + "{{\"backend_config\":\"ragged_paged\",\"params\":{f}}}", + .{std.json.fmt(request, .{})}, + ) catch |err| stdx.debug.panic("Failed to allocate ragged paged attention TPU params: {}", .{err}); + defer allocator.free(json_string); + + const platform = @constCast(compilation_context.platform); + const runtime = &(platform.tpu_ir_runtime orelse stdx.debug.panic("TPU backend config runtime is not initialized", .{})); + const backend_config = runtime.request(allocator, io, json_string) catch |err| { + stdx.debug.panic("Failed to generate TPU backend config through persistent runtime: {}", .{err}); + }; + defer allocator.free(backend_config); + + return allocator.dupe(u8, backend_config) catch |err| { + stdx.debug.panic("Failed to persist TPU backend config in compilation arena: {}", .{err}); + }; + } + + inline fn restoreQueryHeads(q_template: zml.Tensor, q_out: zml.Tensor) zml.Tensor { + return q_out.splitAxis(.h, .{ .hkv = q_template.dim(.hkv), .hg = q_template.dim(.hg) }); + } + + inline fn fuseKvPages(k_cache: zml.Tensor, v_cache: zml.Tensor) zml.Tensor { + const k_pages = k_cache.insertAxes(.hd, .{.kv}); + const v_pages = v_cache.insertAxes(.hd, .{.kv}); + return zml.Tensor.concatenate(&.{ k_pages, v_pages }, .kv) + .merge(.{ .hkv = .{ .hkv, .kv } }); + } + + inline fn prepareInputs(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor) PreparedInputs { + stdx.debug.assert(k_cache.shape().eqlWithTags(v_cache.shape()), "Expected paged K/V cache shapes to match, got {f} and {f}", .{ k_cache.shape(), v_cache.shape() }); + + const q_ragged = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + // Note : + // In a future PR, we'll have to change the pagedAttention API a bit so that we don't have to fuse keys and values + const kv_pages = fuseKvPages(k_cache, v_cache).withPartitioning(.{ .hkv = .model }); + const cu_seqlens_q = parameters.cu_seqlens_q.withPartitioning(.{ .n = .replicated }); + + // Note + // activeSequenceCount could probably be computed when preparing the inputs and injected through Parameters, but we'll keep that for later. + return .{ + .q = q_ragged, + .kv_pages = kv_pages, + .kv_lens = parameters.kv_lens.withPartitioning(.{ ._0 = .replicated }), + .page_indices = parameters.page_indices.withPartitioning(.{ ._0 = .replicated, ._1 = .replicated }), + .cu_seqlens_q = cu_seqlens_q, + .num_seqs = activeSequenceCount(cu_seqlens_q).withPartitioning(.{ ._0 = .replicated }), + }; + } + + fn checkedU32(value: usize, comptime field_name: []const u8) u32 { + stdx.debug.assert(value <= std.math.maxInt(u32), "mosaic_tpu ragged paged attention field '{s}' exceeds u32 range: {}", .{ field_name, value }); + return @intCast(value); + } + + fn buildRequest(prepared_inputs: PreparedInputs, parameters: Parameters, opts: AttentionOptions) RaggedPagedRequest { + const q_shape = prepared_inputs.q.shape(); + const kv_pages_shape = prepared_inputs.kv_pages.shape(); + const page_indices_shape = prepared_inputs.page_indices.shape(); + + const q_token_count = if (q_shape.hasTag(.q) != null) q_shape.dim(.q) else q_shape.dim(.b); + const max_num_seqs = page_indices_shape.dim(0); + const num_q_heads = q_shape.dim(.h); + stdx.debug.assert(@mod(kv_pages_shape.dim(.hkv), 2) == 0, "Expected fused KV pages .hkv dimension to be even, got {}", .{kv_pages_shape.dim(.hkv)}); + const num_kv_heads = std.math.divExact(i64, kv_pages_shape.dim(.hkv), 2) catch |err| switch (err) { + error.UnexpectedRemainder => stdx.debug.panic("Expected fused KV pages .hkv dimension to be even, got {}", .{kv_pages_shape.dim(.hkv)}), + error.DivisionByZero => stdx.debug.panic("KV pages .hkv dimension must be non-zero, got {}", .{kv_pages_shape.dim(.hkv)}), + error.Overflow => stdx.debug.panic("KV pages .hkv dimension is too large to fit in i64: {}", .{kv_pages_shape.dim(.hkv)}), + }; + const num_queries_per_block = @max(@as(i64, 1), @min(q_token_count, max_num_seqs)); + const sm_scale: f32 = opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q_shape.dim(.hd))))); + + return .{ + .num_q_tokens = q_token_count, + .num_q_heads = num_q_heads, + .num_kv_heads = num_kv_heads, + .head_dim = q_shape.dim(.hd), + .q_dtype = jnpDTypeExpr(q_shape.dtype()), + .kv_dtype = jnpDTypeExpr(kv_pages_shape.dtype()), + .max_num_seqs = max_num_seqs, + .pages_per_seq = @intCast(parameters.opts.max_num_pages), + .total_num_pages = kv_pages_shape.dim(.page), + .page_size = kv_pages_shape.dim(.k_chunk), + .sm_scale = sm_scale, + .sliding_window = if (opts.sliding_window < 0) null else @intCast(opts.sliding_window), + .num_kv_pages_per_block = checkedU32(parameters.opts.pages_per_compute_block, "num_kv_pages_per_block"), + .num_queries_per_block = checkedU32(@intCast(num_queries_per_block), "num_queries_per_block"), + }; + } + + pub const Context = struct { + pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { + _ = parameters; + _ = num_heads; + _ = num_kv_heads; + _ = head_dim; + _ = page_size; + return .{}; + } + }; + + pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { + _ = k; + _ = v; + stdx.debug.assert(opts.is_causal, "mosaic_tpu ragged paged attention currently only supports causal attention", .{}); + + const prepared = prepareInputs(parameters, q, k_cache, v_cache); + + // Keep sharding intent on the `manualComputation` boundary only. The + // TPU smoke compile regressed when the body restated + // `withPartitioning` constraints on intermediate tensors. + const q_out = zml.ops.manualComputation( + .{ + prepared.q, + prepared.kv_pages, + prepared.kv_lens, + prepared.page_indices, + prepared.cu_seqlens_q, + prepared.num_seqs, + }, + prepared.q.shape(), + .{ + .opts = opts, + .parameters = parameters, + .context = context, + }, + (struct { + fn body(body_context: anytype, allocator: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output: zml.Shape) zml.Tensor { + stdx.debug.assert(sharded_inputs.len == 6, "mosaic_tpu ragged paged manualComputation expects 6 inputs, got {}", .{sharded_inputs.len}); + + const prepared_inputs: PreparedInputs = .{ + .q = sharded_inputs[0], + .kv_pages = sharded_inputs[1], + .kv_lens = sharded_inputs[2], + .page_indices = sharded_inputs[3], + .cu_seqlens_q = sharded_inputs[4], + .num_seqs = sharded_inputs[5], + }; + + const backend_config = renderBackendConfig(allocator, CompilationContext.current().io, buildRequest(prepared_inputs, body_context.parameters, body_context.opts)); + const q_out = raggedPagedKernelCall( + prepared_inputs.q, + prepared_inputs.kv_pages, + prepared_inputs.kv_lens, + prepared_inputs.page_indices, + prepared_inputs.cu_seqlens_q, + prepared_inputs.num_seqs, + backend_config, + ); + stdx.debug.assert(q_out.shape().eql(output), "mosaic_tpu ragged paged attention output shape mismatch, got {f}, expected {f}", .{ q_out.shape(), output }); + return q_out; + } + }).body, + ); + + const restored = restoreQueryHeads(q, q_out); + stdx.debug.assert(restored.shape().eql(q.shape()), "mosaic_tpu ragged paged attention output shape mismatch, got {f}, expected {f}", .{ restored.shape(), q.shape() }); + return restored; + } +}; diff --git a/zml/attention/triton/generate.zig b/zml/attention/triton/generate.zig index 872c7026..8e38044e 100644 --- a/zml/attention/triton/generate.zig +++ b/zml/attention/triton/generate.zig @@ -53,7 +53,7 @@ pub fn main(init: std.process.Init.Minimal) !void { config.optimization_level = 2; config.write_bytecode = 0; - _ = c.PyConfig_SetBytesArgv(&config, @intCast(init.args.vector.len), @ptrCast(init.args.vector.ptr)); + pyStatusCheck(c.PyConfig_SetBytesArgv(&config, @intCast(init.args.vector.len), @ptrCast(init.args.vector.ptr))); var self_exe_dir_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const self_exe_dir = blk: { diff --git a/zml/platform.zig b/zml/platform.zig index 2bcecba7..dd43f787 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -6,13 +6,12 @@ const platforms = @import("platforms"); pub const Target = platforms.Platform; const stdx = @import("stdx"); +const attention = @import("attention.zig"); const Exe = @import("exe.zig").Exe; const pjrtx = @import("pjrtx.zig"); const profiler_ = @import("profiler.zig"); const zml = @import("zml.zig"); -const attention = @import("attention.zig"); - const log = std.log.scoped(.zml); fn StaticPlatformMap(comptime E: type, comptime T: type) type { @@ -204,6 +203,7 @@ pub const Platform = struct { physical_mesh: zml.sharding.PhysicalMesh, triton_runtime: ?attention.triton.Runtime = null, + tpu_ir_runtime: ?attention.tpu.Runtime = null, pub fn initBackend(self: *Platform, allocator: std.mem.Allocator, io: std.Io, backend: attention.paged_attention.Backend) !void { switch (backend) { @@ -212,6 +212,11 @@ pub const Platform = struct { self.triton_runtime = try zml.attention.triton.Runtime.init(allocator, io); } }, + .mosaic_tpu => { + if (self.tpu_ir_runtime == null) { + self.tpu_ir_runtime = try zml.attention.tpu.Runtime.init(allocator, io); + } + }, else => {}, } } @@ -418,6 +423,7 @@ pub const Platform = struct { pub fn deinit(self: *Platform, allocator: std.mem.Allocator, io: std.Io) void { self.pjrt_client.deinit(self.pjrt_api); + if (self.tpu_ir_runtime) |*rt| rt.deinit(io); if (self.triton_runtime) |*rt| rt.deinit(io); self.arena_state.promote(allocator).deinit(); } From da99fe93b584f61c29a7b1a2043524eaf35042c0 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 17 Apr 2026 16:48:08 +0000 Subject: [PATCH 438/881] platforms/rocm: bump ROCm=7.2.2 --- platforms/rocm/packages.lock.json | 596 +++++++++++++++--------------- platforms/rocm/packages.yaml | 2 +- platforms/rocm/rocm.bzl | 6 +- 3 files changed, 302 insertions(+), 302 deletions(-) diff --git a/platforms/rocm/packages.lock.json b/platforms/rocm/packages.lock.json index 07234598..e69e33f2 100644 --- a/platforms/rocm/packages.lock.json +++ b/platforms/rocm/packages.lock.json @@ -186,9 +186,9 @@ "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.1.70201-81_22.04_amd64", + "key": "rocm-core_7.2.2.70202-86_22.04_amd64", "name": "rocm-core", - "version": "7.2.1.70201-81~22.04" + "version": "7.2.2.70202-86~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -471,24 +471,24 @@ "version": "1:3.0.7-1build1" } ], - "key": "amd-smi-lib_26.2.2.70201-81_22.04_amd64", + "key": "amd-smi-lib_26.2.2.70202-86_22.04_amd64", "name": "amd-smi-lib", - "sha256": "43566b517bdbd90fe7b5bc400c30f14dd062de6a304642808079cef394f606f6", + "sha256": "406bcedc70b24b30e71037d5f625125cb15c4033f073fabcb1718d8261395e85", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/a/amd-smi-lib/amd-smi-lib_26.2.2.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/a/amd-smi-lib/amd-smi-lib_26.2.2.70202-86~22.04_amd64.deb" ], - "version": "26.2.2.70201-81~22.04" + "version": "26.2.2.70202-86~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocm-core_7.2.1.70201-81_22.04_amd64", + "key": "rocm-core_7.2.2.70202-86_22.04_amd64", "name": "rocm-core", - "sha256": "b83fe314e67f6bd3cef2b5695fb6de7153110a14727add1992d0056d81a7e9a9", + "sha256": "03fce430bfb5769fda2c9dcf0f23b6a20c1cf6fd7bf091b28a9a1b978716042e", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocm-core/rocm-core_7.2.1.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocm-core/rocm-core_7.2.2.70202-86~22.04_amd64.deb" ], - "version": "7.2.1.70201-81~22.04" + "version": "7.2.2.70202-86~22.04" }, { "arch": "amd64", @@ -1061,9 +1061,9 @@ "arch": "amd64", "dependencies": [ { - "key": "rocprofiler-register_0.6.0.70201-81_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70202-86_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70201-81~22.04" + "version": "0.6.0.70202-86~22.04" }, { "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", @@ -1091,9 +1091,9 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "rocm-core_7.2.1.70201-81_22.04_amd64", + "key": "rocm-core_7.2.2.70202-86_22.04_amd64", "name": "rocm-core", - "version": "7.2.1.70201-81~22.04" + "version": "7.2.2.70202-86~22.04" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -1321,13 +1321,13 @@ "version": "1:2.4.125.70201-2303469.22.04" } ], - "key": "hsa-rocr_1.18.0.70201-81_22.04_amd64", + "key": "hsa-rocr_1.18.0.70202-86_22.04_amd64", "name": "hsa-rocr", - "sha256": "0cea908f2d05be0b6cb98556b429def4f114199ceb7903eca521ec310182ac41", + "sha256": "a9e6ca15a56e91f38bc20325afdd5cbc271815f20b6f1c51088e914917338a04", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/h/hsa-rocr/hsa-rocr_1.18.0.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/h/hsa-rocr/hsa-rocr_1.18.0.70202-86~22.04_amd64.deb" ], - "version": "1.18.0.70201-81~22.04" + "version": "1.18.0.70202-86~22.04" }, { "arch": "amd64", @@ -1358,9 +1358,9 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "rocm-core_7.2.1.70201-81_22.04_amd64", + "key": "rocm-core_7.2.2.70202-86_22.04_amd64", "name": "rocm-core", - "version": "7.2.1.70201-81~22.04" + "version": "7.2.2.70202-86~22.04" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -1558,13 +1558,13 @@ "version": "3.10.6-1~22.04.1" } ], - "key": "rocprofiler-register_0.6.0.70201-81_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70202-86_22.04_amd64", "name": "rocprofiler-register", - "sha256": "55bca0525ca02921240e5caf74ce3de0fc5791264e495bf026a49a4fedc0e1f3", + "sha256": "1d6cefddc97dbda37e46aedff955ca9c27352ad26cf50934f82a8130a5b0b897", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocprofiler-register/rocprofiler-register_0.6.0.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocprofiler-register/rocprofiler-register_0.6.0.70202-86~22.04_amd64.deb" ], - "version": "0.6.0.70201-81~22.04" + "version": "0.6.0.70202-86~22.04" }, { "arch": "amd64", @@ -1603,9 +1603,9 @@ "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.1.70201-81_22.04_amd64", + "key": "rocm-core_7.2.2.70202-86_22.04_amd64", "name": "rocm-core", - "version": "7.2.1.70201-81~22.04" + "version": "7.2.2.70202-86~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -1828,14 +1828,14 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "hsa-rocr_1.18.0.70201-81_22.04_amd64", + "key": "hsa-rocr_1.18.0.70202-86_22.04_amd64", "name": "hsa-rocr", - "version": "1.18.0.70201-81~22.04" + "version": "1.18.0.70202-86~22.04" }, { - "key": "rocprofiler-register_0.6.0.70201-81_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70202-86_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70201-81~22.04" + "version": "0.6.0.70202-86~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", @@ -1868,13 +1868,13 @@ "version": "1:2.4.125.70201-2303469.22.04" } ], - "key": "hsa-amd-aqlprofile_1.0.0.70201-81_22.04_amd64", + "key": "hsa-amd-aqlprofile_1.0.0.70202-86_22.04_amd64", "name": "hsa-amd-aqlprofile", - "sha256": "9a205f71abf149f59d8d8bac934f24a826a7eb0a7993d1159446aedc9e7aa364", + "sha256": "55b92d285fab53a312d7d2ff132fd1e6216e2d46f2f3b47758299fcb0505a586", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/h/hsa-amd-aqlprofile/hsa-amd-aqlprofile_1.0.0.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/h/hsa-amd-aqlprofile/hsa-amd-aqlprofile_1.0.0.70202-86~22.04_amd64.deb" ], - "version": "1.0.0.70201-81~22.04" + "version": "1.0.0.70202-86~22.04" }, { "arch": "amd64", @@ -1915,9 +1915,9 @@ "version": "1.4.8+dfsg-3build1" }, { - "key": "rocm-core_7.2.1.70201-81_22.04_amd64", + "key": "rocm-core_7.2.2.70202-86_22.04_amd64", "name": "rocm-core", - "version": "7.2.1.70201-81~22.04" + "version": "7.2.2.70202-86~22.04" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -2160,13 +2160,13 @@ "version": "6.3-2ubuntu0.1" } ], - "key": "comgr_3.0.0.70201-81_22.04_amd64", + "key": "comgr_3.0.0.70202-86_22.04_amd64", "name": "comgr", - "sha256": "096a51d616f01f431d8fd7716257ca602d811c393075300a4b947244333cdc50", + "sha256": "e55ca10c183f8debdd595d8b3e4a8fd5df8e648ac8bc0edb1281b28795534eea", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/c/comgr/comgr_3.0.0.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/c/comgr/comgr_3.0.0.70202-86~22.04_amd64.deb" ], - "version": "3.0.0.70201-81~22.04" + "version": "3.0.0.70202-86~22.04" }, { "arch": "amd64", @@ -2293,14 +2293,14 @@ "arch": "amd64", "dependencies": [ { - "key": "rocprofiler-sdk-rocpd_1.1.0-81_22.04_amd64", + "key": "rocprofiler-sdk-rocpd_1.1.0-86_22.04_amd64", "name": "rocprofiler-sdk-rocpd", - "version": "1.1.0-81~22.04" + "version": "1.1.0-86~22.04" }, { - "key": "rocm-core_7.2.1.70201-81_22.04_amd64", + "key": "rocm-core_7.2.2.70202-86_22.04_amd64", "name": "rocm-core", - "version": "7.2.1.70201-81~22.04" + "version": "7.2.2.70202-86~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -2523,53 +2523,53 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "rocprofiler-sdk-roctx_1.1.0-81_22.04_amd64", + "key": "rocprofiler-sdk-roctx_1.1.0-86_22.04_amd64", "name": "rocprofiler-sdk-roctx", - "version": "1.1.0-81~22.04" + "version": "1.1.0-86~22.04" }, { - "key": "rocprofiler-register_0.6.0.70201-81_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70202-86_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70201-81~22.04" + "version": "0.6.0.70202-86~22.04" } ], - "key": "rocprofiler-sdk_1.1.0-81_22.04_amd64", + "key": "rocprofiler-sdk_1.1.0-86_22.04_amd64", "name": "rocprofiler-sdk", - "sha256": "03b11e5e48e2125b32676515e0191158109aa581fe4785c5edb4a3c0c30f3282", + "sha256": "42c2b64ab72e027fd8e9b5c0892413e1cc476f3593a7eaeb414df985896ce032", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocprofiler-sdk/rocprofiler-sdk_1.1.0-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocprofiler-sdk/rocprofiler-sdk_1.1.0-86~22.04_amd64.deb" ], - "version": "1.1.0-81~22.04" + "version": "1.1.0-86~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocprofiler-sdk-rocpd_1.1.0-81_22.04_amd64", + "key": "rocprofiler-sdk-rocpd_1.1.0-86_22.04_amd64", "name": "rocprofiler-sdk-rocpd", - "sha256": "1dfbc1887f4f7de7b78e3fd2959b53f54fa9f613763b8f4cf3c4c6f97b34b7c7", + "sha256": "09573c79fc2cc11e590c9dfae2143d0a9021ef3e44cf50220beb09130ca82061", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocprofiler-sdk-rocpd/rocprofiler-sdk-rocpd_1.1.0-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocprofiler-sdk-rocpd/rocprofiler-sdk-rocpd_1.1.0-86~22.04_amd64.deb" ], - "version": "1.1.0-81~22.04" + "version": "1.1.0-86~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocprofiler-sdk-roctx_1.1.0-81_22.04_amd64", + "key": "rocprofiler-sdk-roctx_1.1.0-86_22.04_amd64", "name": "rocprofiler-sdk-roctx", - "sha256": "49ac3668842c82767a91128c0107af77a44b1c6c7e7c4c16248eac613f17ddcf", + "sha256": "0a23a509af5ee30a5a05ff8832236be48271e2228fdbd00c362062a8baef2891", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocprofiler-sdk-roctx/rocprofiler-sdk-roctx_1.1.0-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocprofiler-sdk-roctx/rocprofiler-sdk-roctx_1.1.0-86~22.04_amd64.deb" ], - "version": "1.1.0-81~22.04" + "version": "1.1.0-86~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.1.70201-81_22.04_amd64", + "key": "rocm-core_7.2.2.70202-86_22.04_amd64", "name": "rocm-core", - "version": "7.2.1.70201-81~22.04" + "version": "7.2.2.70202-86~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -2792,29 +2792,29 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "rocrand_4.2.0.70201-81_22.04_amd64", + "key": "rocrand_4.2.0.70202-86_22.04_amd64", "name": "rocrand", - "version": "4.2.0.70201-81~22.04" + "version": "4.2.0.70202-86~22.04" }, { - "key": "hipblaslt_1.2.2.70201-81_22.04_amd64", + "key": "hipblaslt_1.2.2.70202-86_22.04_amd64", "name": "hipblaslt", - "version": "1.2.2.70201-81~22.04" + "version": "1.2.2.70202-86~22.04" }, { - "key": "roctracer_4.1.70201.70201-81_22.04_amd64", + "key": "roctracer_4.1.70202.70202-86_22.04_amd64", "name": "roctracer", - "version": "4.1.70201.70201-81~22.04" + "version": "4.1.70202.70202-86~22.04" }, { - "key": "rocblas_5.2.0.70201-81_22.04_amd64", + "key": "rocblas_5.2.0.70202-86_22.04_amd64", "name": "rocblas", - "version": "5.2.0.70201-81~22.04" + "version": "5.2.0.70202-86~22.04" }, { - "key": "hip-runtime-amd_7.2.53211.70201-81_22.04_amd64", + "key": "hip-runtime-amd_7.2.53211.70202-86_22.04_amd64", "name": "hip-runtime-amd", - "version": "7.2.53211.70201-81~22.04" + "version": "7.2.53211.70202-86~22.04" }, { "key": "libnuma1_2.0.14-3ubuntu2_amd64", @@ -2822,14 +2822,14 @@ "version": "2.0.14-3ubuntu2" }, { - "key": "rocprofiler-register_0.6.0.70201-81_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70202-86_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70201-81~22.04" + "version": "0.6.0.70202-86~22.04" }, { - "key": "comgr_3.0.0.70201-81_22.04_amd64", + "key": "comgr_3.0.0.70202-86_22.04_amd64", "name": "comgr", - "version": "3.0.0.70201-81~22.04" + "version": "3.0.0.70202-86~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -2887,9 +2887,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70201-81_22.04_amd64", + "key": "rocminfo_1.0.0.70202-86_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.70201-81~22.04" + "version": "1.0.0.70202-86~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -2917,9 +2917,9 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-rocr_1.18.0.70201-81_22.04_amd64", + "key": "hsa-rocr_1.18.0.70202-86_22.04_amd64", "name": "hsa-rocr", - "version": "1.18.0.70201-81~22.04" + "version": "1.18.0.70202-86~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", @@ -2947,32 +2947,32 @@ "version": "1:2.4.125.70201-2303469.22.04" } ], - "key": "miopen-hip_3.5.1.70201-81_22.04_amd64", + "key": "miopen-hip_3.5.1.70202-86_22.04_amd64", "name": "miopen-hip", - "sha256": "6602376a310d0f8702acea8d0fa78138800060373f462bfd80994218ff87a674", + "sha256": "d8b43108d7328d5681b9e5ad847ed9f3ac2b744312e46f7b11246c0cc887407b", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/m/miopen-hip/miopen-hip_3.5.1.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/m/miopen-hip/miopen-hip_3.5.1.70202-86~22.04_amd64.deb" ], - "version": "3.5.1.70201-81~22.04" + "version": "3.5.1.70202-86~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocrand_4.2.0.70201-81_22.04_amd64", + "key": "rocrand_4.2.0.70202-86_22.04_amd64", "name": "rocrand", - "sha256": "6a772e0be70c346a62aa6be3420c7ae7e4e70ca7f472abdd9b4fd6b42eaf9297", + "sha256": "2add5082b1322fb59e3d383bdd086676fd91f4437864b93b1ece7cc3c1380e56", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocrand/rocrand_4.2.0.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocrand/rocrand_4.2.0.70202-86~22.04_amd64.deb" ], - "version": "4.2.0.70201-81~22.04" + "version": "4.2.0.70202-86~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.1.70201-81_22.04_amd64", + "key": "rocm-core_7.2.2.70202-86_22.04_amd64", "name": "rocm-core", - "version": "7.2.1.70201-81~22.04" + "version": "7.2.2.70202-86~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -3195,26 +3195,26 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "roctracer_4.1.70201.70201-81_22.04_amd64", + "key": "roctracer_4.1.70202.70202-86_22.04_amd64", "name": "roctracer", - "version": "4.1.70201.70201-81~22.04" + "version": "4.1.70202.70202-86~22.04" } ], - "key": "hipblaslt_1.2.2.70201-81_22.04_amd64", + "key": "hipblaslt_1.2.2.70202-86_22.04_amd64", "name": "hipblaslt", - "sha256": "d44468114be65720eb0a82aa6009426de0022e345785a039caacc0cd63a0dc18", + "sha256": "c3f5086d42d040b4dc10cfa45743047e9d89c7abbe169955caf978c82d85a930", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/h/hipblaslt/hipblaslt_1.2.2.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/h/hipblaslt/hipblaslt_1.2.2.70202-86~22.04_amd64.deb" ], - "version": "1.2.2.70201-81~22.04" + "version": "1.2.2.70202-86~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.1.70201-81_22.04_amd64", + "key": "rocm-core_7.2.2.70202-86_22.04_amd64", "name": "rocm-core", - "version": "7.2.1.70201-81~22.04" + "version": "7.2.2.70202-86~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -3437,21 +3437,21 @@ "version": "3.10.6-1~22.04.1" } ], - "key": "roctracer_4.1.70201.70201-81_22.04_amd64", + "key": "roctracer_4.1.70202.70202-86_22.04_amd64", "name": "roctracer", - "sha256": "51c372d9a3e238e83b9725743f4fbc58a7dd93cb67b5b5adb72bf19148a12e26", + "sha256": "b5f436accb35050035e5b59e669d991b3808389781278be33ca110611aff4d7e", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/roctracer/roctracer_4.1.70201.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/roctracer/roctracer_4.1.70202.70202-86~22.04_amd64.deb" ], - "version": "4.1.70201.70201-81~22.04" + "version": "4.1.70202.70202-86~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.1.70201-81_22.04_amd64", + "key": "rocm-core_7.2.2.70202-86_22.04_amd64", "name": "rocm-core", - "version": "7.2.1.70201-81~22.04" + "version": "7.2.2.70202-86~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -3674,9 +3674,9 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "hip-runtime-amd_7.2.53211.70201-81_22.04_amd64", + "key": "hip-runtime-amd_7.2.53211.70202-86_22.04_amd64", "name": "hip-runtime-amd", - "version": "7.2.53211.70201-81~22.04" + "version": "7.2.53211.70202-86~22.04" }, { "key": "libnuma1_2.0.14-3ubuntu2_amd64", @@ -3684,14 +3684,14 @@ "version": "2.0.14-3ubuntu2" }, { - "key": "rocprofiler-register_0.6.0.70201-81_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70202-86_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70201-81~22.04" + "version": "0.6.0.70202-86~22.04" }, { - "key": "comgr_3.0.0.70201-81_22.04_amd64", + "key": "comgr_3.0.0.70202-86_22.04_amd64", "name": "comgr", - "version": "3.0.0.70201-81~22.04" + "version": "3.0.0.70202-86~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -3749,9 +3749,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70201-81_22.04_amd64", + "key": "rocminfo_1.0.0.70202-86_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.70201-81~22.04" + "version": "1.0.0.70202-86~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -3779,9 +3779,9 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-rocr_1.18.0.70201-81_22.04_amd64", + "key": "hsa-rocr_1.18.0.70202-86_22.04_amd64", "name": "hsa-rocr", - "version": "1.18.0.70201-81~22.04" + "version": "1.18.0.70202-86~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", @@ -3809,23 +3809,23 @@ "version": "1:2.4.125.70201-2303469.22.04" }, { - "key": "hipblaslt_1.2.2.70201-81_22.04_amd64", + "key": "hipblaslt_1.2.2.70202-86_22.04_amd64", "name": "hipblaslt", - "version": "1.2.2.70201-81~22.04" + "version": "1.2.2.70202-86~22.04" }, { - "key": "roctracer_4.1.70201.70201-81_22.04_amd64", + "key": "roctracer_4.1.70202.70202-86_22.04_amd64", "name": "roctracer", - "version": "4.1.70201.70201-81~22.04" + "version": "4.1.70202.70202-86~22.04" } ], - "key": "rocblas_5.2.0.70201-81_22.04_amd64", + "key": "rocblas_5.2.0.70202-86_22.04_amd64", "name": "rocblas", - "sha256": "835c40b7943ec98298fe889958a9e40ea70ed196a84cfb23c88e98967d30e057", + "sha256": "8c390b22297321d06453db51d8aa6de8329b47d952d96e91da934c8ffc413a30", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocblas/rocblas_5.2.0.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocblas/rocblas_5.2.0.70202-86~22.04_amd64.deb" ], - "version": "5.2.0.70201-81~22.04" + "version": "5.2.0.70202-86~22.04" }, { "arch": "amd64", @@ -3861,14 +3861,14 @@ "version": "2.0.14-3ubuntu2" }, { - "key": "rocprofiler-register_0.6.0.70201-81_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70202-86_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70201-81~22.04" + "version": "0.6.0.70202-86~22.04" }, { - "key": "rocm-core_7.2.1.70201-81_22.04_amd64", + "key": "rocm-core_7.2.2.70202-86_22.04_amd64", "name": "rocm-core", - "version": "7.2.1.70201-81~22.04" + "version": "7.2.2.70202-86~22.04" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -4066,9 +4066,9 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "comgr_3.0.0.70201-81_22.04_amd64", + "key": "comgr_3.0.0.70202-86_22.04_amd64", "name": "comgr", - "version": "3.0.0.70201-81~22.04" + "version": "3.0.0.70202-86~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -4126,9 +4126,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70201-81_22.04_amd64", + "key": "rocminfo_1.0.0.70202-86_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.70201-81~22.04" + "version": "1.0.0.70202-86~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -4156,9 +4156,9 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-rocr_1.18.0.70201-81_22.04_amd64", + "key": "hsa-rocr_1.18.0.70202-86_22.04_amd64", "name": "hsa-rocr", - "version": "1.18.0.70201-81~22.04" + "version": "1.18.0.70202-86~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", @@ -4186,24 +4186,24 @@ "version": "1:2.4.125.70201-2303469.22.04" } ], - "key": "hip-runtime-amd_7.2.53211.70201-81_22.04_amd64", + "key": "hip-runtime-amd_7.2.53211.70202-86_22.04_amd64", "name": "hip-runtime-amd", - "sha256": "2d265ee445042558e06318e445fe7aa0fbfd00e7795bd4d72ada4997b9a748c1", + "sha256": "4c70edc23440b7e7c99b5a9985538adc725d59ce7ffc97b2191db1d598c0708b", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/h/hip-runtime-amd/hip-runtime-amd_7.2.53211.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/h/hip-runtime-amd/hip-runtime-amd_7.2.53211.70202-86~22.04_amd64.deb" ], - "version": "7.2.53211.70201-81~22.04" + "version": "7.2.53211.70202-86~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocminfo_1.0.0.70201-81_22.04_amd64", + "key": "rocminfo_1.0.0.70202-86_22.04_amd64", "name": "rocminfo", - "sha256": "774cb4be2da5a7674e2056494fd3456e7c6ea74de50a81554ca38963c8dad317", + "sha256": "2373c0f0891e115914abfd4fa66358bfcae44236c1bae8c613ccd38c7da40c20", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocminfo/rocminfo_1.0.0.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocminfo/rocminfo_1.0.0.70202-86~22.04_amd64.deb" ], - "version": "1.0.0.70201-81~22.04" + "version": "1.0.0.70202-86~22.04" }, { "arch": "amd64", @@ -4289,9 +4289,9 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "rocm-core_7.2.1.70201-81_22.04_amd64", + "key": "rocm-core_7.2.2.70202-86_22.04_amd64", "name": "rocm-core", - "version": "7.2.1.70201-81~22.04" + "version": "7.2.2.70202-86~22.04" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -4489,14 +4489,14 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "rocm-smi-lib_7.8.0.70201-81_22.04_amd64", + "key": "rocm-smi-lib_7.8.0.70202-86_22.04_amd64", "name": "rocm-smi-lib", - "version": "7.8.0.70201-81~22.04" + "version": "7.8.0.70202-86~22.04" }, { - "key": "hip-runtime-amd_7.2.53211.70201-81_22.04_amd64", + "key": "hip-runtime-amd_7.2.53211.70202-86_22.04_amd64", "name": "hip-runtime-amd", - "version": "7.2.53211.70201-81~22.04" + "version": "7.2.53211.70202-86~22.04" }, { "key": "libnuma1_2.0.14-3ubuntu2_amd64", @@ -4504,14 +4504,14 @@ "version": "2.0.14-3ubuntu2" }, { - "key": "rocprofiler-register_0.6.0.70201-81_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70202-86_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70201-81~22.04" + "version": "0.6.0.70202-86~22.04" }, { - "key": "comgr_3.0.0.70201-81_22.04_amd64", + "key": "comgr_3.0.0.70202-86_22.04_amd64", "name": "comgr", - "version": "3.0.0.70201-81~22.04" + "version": "3.0.0.70202-86~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -4569,9 +4569,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70201-81_22.04_amd64", + "key": "rocminfo_1.0.0.70202-86_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.70201-81~22.04" + "version": "1.0.0.70202-86~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -4599,9 +4599,9 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-rocr_1.18.0.70201-81_22.04_amd64", + "key": "hsa-rocr_1.18.0.70202-86_22.04_amd64", "name": "hsa-rocr", - "version": "1.18.0.70201-81~22.04" + "version": "1.18.0.70202-86~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", @@ -4629,32 +4629,32 @@ "version": "1:2.4.125.70201-2303469.22.04" } ], - "key": "rccl_2.27.7.70201-81_22.04_amd64", + "key": "rccl_2.27.7.70202-86_22.04_amd64", "name": "rccl", - "sha256": "cd84ed31a1bcce67d82ed4599d375b6e1e29b487313dba65297b856ce38ef8cf", + "sha256": "942e98104888cc56bbe136d70b771b8dceba241ebfb1f460aebe5d01dd425ea9", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rccl/rccl_2.27.7.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rccl/rccl_2.27.7.70202-86~22.04_amd64.deb" ], - "version": "2.27.7.70201-81~22.04" + "version": "2.27.7.70202-86~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocm-smi-lib_7.8.0.70201-81_22.04_amd64", + "key": "rocm-smi-lib_7.8.0.70202-86_22.04_amd64", "name": "rocm-smi-lib", - "sha256": "fc470d30b9e33ffc026ae829d7584e7fea04562a488e44c00a27b670d495829d", + "sha256": "ee8c1d211b1db5d8340abfbe0d4cae99866a365f3f04f84445b97ba5394b00e2", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocm-smi-lib/rocm-smi-lib_7.8.0.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocm-smi-lib/rocm-smi-lib_7.8.0.70202-86~22.04_amd64.deb" ], - "version": "7.8.0.70201-81~22.04" + "version": "7.8.0.70202-86~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.1.70201-81_22.04_amd64", + "key": "rocm-core_7.2.2.70202-86_22.04_amd64", "name": "rocm-core", - "version": "7.2.1.70201-81~22.04" + "version": "7.2.2.70202-86~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -4877,31 +4877,31 @@ "version": "3.10.6-1~22.04.1" } ], - "key": "rocm-device-libs_1.0.0.70201-81_22.04_amd64", + "key": "rocm-device-libs_1.0.0.70202-86_22.04_amd64", "name": "rocm-device-libs", - "sha256": "86ecdcb81a4face9077f51b66b06f9cfc1ea15aa33043a6c0a3e771f8dec5916", + "sha256": "ceecef256d804a47a2d57184920cd9a1cbd6190937332998f2200d8b18a5fb27", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocm-device-libs/rocm-device-libs_1.0.0.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocm-device-libs/rocm-device-libs_1.0.0.70202-86~22.04_amd64.deb" ], - "version": "1.0.0.70201-81~22.04" + "version": "1.0.0.70202-86~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "hipcc_1.1.1.70201-81_22.04_amd64", + "key": "hipcc_1.1.1.70202-86_22.04_amd64", "name": "hipcc", - "version": "1.1.1.70201-81~22.04" + "version": "1.1.1.70202-86~22.04" }, { - "key": "rocm-llvm_22.0.0.26084.70201-81_22.04_amd64", + "key": "rocm-llvm_22.0.0.26084.70202-86_22.04_amd64", "name": "rocm-llvm", - "version": "22.0.0.26084.70201-81~22.04" + "version": "22.0.0.26084.70202-86~22.04" }, { - "key": "rocm-core_7.2.1.70201-81_22.04_amd64", + "key": "rocm-core_7.2.2.70202-86_22.04_amd64", "name": "rocm-core", - "version": "7.2.1.70201-81~22.04" + "version": "7.2.2.70202-86~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -5214,19 +5214,19 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "hsa-rocr-dev_1.18.0.70201-81_22.04_amd64", + "key": "hsa-rocr-dev_1.18.0.70202-86_22.04_amd64", "name": "hsa-rocr-dev", - "version": "1.18.0.70201-81~22.04" + "version": "1.18.0.70202-86~22.04" }, { - "key": "hsa-rocr_1.18.0.70201-81_22.04_amd64", + "key": "hsa-rocr_1.18.0.70202-86_22.04_amd64", "name": "hsa-rocr", - "version": "1.18.0.70201-81~22.04" + "version": "1.18.0.70202-86~22.04" }, { - "key": "rocprofiler-register_0.6.0.70201-81_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70202-86_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70201-81~22.04" + "version": "0.6.0.70202-86~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", @@ -5269,14 +5269,14 @@ "version": "1:2.4.125.70201-2303469.22.04" }, { - "key": "hip-runtime-amd_7.2.53211.70201-81_22.04_amd64", + "key": "hip-runtime-amd_7.2.53211.70202-86_22.04_amd64", "name": "hip-runtime-amd", - "version": "7.2.53211.70201-81~22.04" + "version": "7.2.53211.70202-86~22.04" }, { - "key": "comgr_3.0.0.70201-81_22.04_amd64", + "key": "comgr_3.0.0.70202-86_22.04_amd64", "name": "comgr", - "version": "3.0.0.70201-81~22.04" + "version": "3.0.0.70202-86~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -5299,9 +5299,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70201-81_22.04_amd64", + "key": "rocminfo_1.0.0.70202-86_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.70201-81~22.04" + "version": "1.0.0.70202-86~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -5344,35 +5344,35 @@ "version": "1:5.41-3ubuntu0.1" } ], - "key": "hip-dev_7.2.53211.70201-81_22.04_amd64", + "key": "hip-dev_7.2.53211.70202-86_22.04_amd64", "name": "hip-dev", - "sha256": "5cc26232d8b7fd3ced173c07183be1d8c262beadce74fc44360e986d9f10dfc0", + "sha256": "97fbde73590f02e5253d8287006269cbb8885d3b0d1366071bc96d3f51f598e5", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/h/hip-dev/hip-dev_7.2.53211.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/h/hip-dev/hip-dev_7.2.53211.70202-86~22.04_amd64.deb" ], - "version": "7.2.53211.70201-81~22.04" + "version": "7.2.53211.70202-86~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "hipcc_1.1.1.70201-81_22.04_amd64", + "key": "hipcc_1.1.1.70202-86_22.04_amd64", "name": "hipcc", - "sha256": "fb047beabb6e93401d9179aadf26dbbb30df476f24525535c543d26e6c6f0327", + "sha256": "2b2a0e83fa1f4d8929e12c532bef83914b56c68c651c34bb96c77deff8a9133b", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/h/hipcc/hipcc_1.1.1.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/h/hipcc/hipcc_1.1.1.70202-86~22.04_amd64.deb" ], - "version": "1.1.1.70201-81~22.04" + "version": "1.1.1.70202-86~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocm-llvm_22.0.0.26084.70201-81_22.04_amd64", + "key": "rocm-llvm_22.0.0.26084.70202-86_22.04_amd64", "name": "rocm-llvm", - "sha256": "81dfdbac523318101e06e551c6a98aae55c595886d84de43e893dcf2ca41a3ba", + "sha256": "506d8b818edea66827fc2d36f1f2d883245df6ae72afdbfd51a6ef4e3d50b067", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocm-llvm/rocm-llvm_22.0.0.26084.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocm-llvm/rocm-llvm_22.0.0.26084.70202-86~22.04_amd64.deb" ], - "version": "22.0.0.26084.70201-81~22.04" + "version": "22.0.0.26084.70202-86~22.04" }, { "arch": "amd64", @@ -5498,13 +5498,13 @@ { "arch": "amd64", "dependencies": [], - "key": "hsa-rocr-dev_1.18.0.70201-81_22.04_amd64", + "key": "hsa-rocr-dev_1.18.0.70202-86_22.04_amd64", "name": "hsa-rocr-dev", - "sha256": "95043500f730e6cb0e796274228f3f94deb0e36e7f80793584a2932e6ec707e5", + "sha256": "50cffa604bad1b459d2c06252c1aca80134cab6b31df4d595e078c51c77de079", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/h/hsa-rocr-dev/hsa-rocr-dev_1.18.0.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/h/hsa-rocr-dev/hsa-rocr-dev_1.18.0.70202-86~22.04_amd64.deb" ], - "version": "1.18.0.70201-81~22.04" + "version": "1.18.0.70202-86~22.04" }, { "arch": "amd64", @@ -5565,9 +5565,9 @@ "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.1.70201-81_22.04_amd64", + "key": "rocm-core_7.2.2.70202-86_22.04_amd64", "name": "rocm-core", - "version": "7.2.1.70201-81~22.04" + "version": "7.2.2.70202-86~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -5790,14 +5790,14 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "rocblas_5.2.0.70201-81_22.04_amd64", + "key": "rocblas_5.2.0.70202-86_22.04_amd64", "name": "rocblas", - "version": "5.2.0.70201-81~22.04" + "version": "5.2.0.70202-86~22.04" }, { - "key": "hip-runtime-amd_7.2.53211.70201-81_22.04_amd64", + "key": "hip-runtime-amd_7.2.53211.70202-86_22.04_amd64", "name": "hip-runtime-amd", - "version": "7.2.53211.70201-81~22.04" + "version": "7.2.53211.70202-86~22.04" }, { "key": "libnuma1_2.0.14-3ubuntu2_amd64", @@ -5805,14 +5805,14 @@ "version": "2.0.14-3ubuntu2" }, { - "key": "rocprofiler-register_0.6.0.70201-81_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70202-86_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70201-81~22.04" + "version": "0.6.0.70202-86~22.04" }, { - "key": "comgr_3.0.0.70201-81_22.04_amd64", + "key": "comgr_3.0.0.70202-86_22.04_amd64", "name": "comgr", - "version": "3.0.0.70201-81~22.04" + "version": "3.0.0.70202-86~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -5870,9 +5870,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70201-81_22.04_amd64", + "key": "rocminfo_1.0.0.70202-86_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.70201-81~22.04" + "version": "1.0.0.70202-86~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -5900,9 +5900,9 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-rocr_1.18.0.70201-81_22.04_amd64", + "key": "hsa-rocr_1.18.0.70202-86_22.04_amd64", "name": "hsa-rocr", - "version": "1.18.0.70201-81~22.04" + "version": "1.18.0.70202-86~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", @@ -5930,31 +5930,31 @@ "version": "1:2.4.125.70201-2303469.22.04" }, { - "key": "hipblaslt_1.2.2.70201-81_22.04_amd64", + "key": "hipblaslt_1.2.2.70202-86_22.04_amd64", "name": "hipblaslt", - "version": "1.2.2.70201-81~22.04" + "version": "1.2.2.70202-86~22.04" }, { - "key": "roctracer_4.1.70201.70201-81_22.04_amd64", + "key": "roctracer_4.1.70202.70202-86_22.04_amd64", "name": "roctracer", - "version": "4.1.70201.70201-81~22.04" + "version": "4.1.70202.70202-86~22.04" } ], - "key": "rocsolver_3.32.0.70201-81_22.04_amd64", + "key": "rocsolver_3.32.0.70202-86_22.04_amd64", "name": "rocsolver", - "sha256": "6208660df71b8a5c55e2147c64bcaf65778288c1a9b24b25e1df9c1e9a071d24", + "sha256": "71d201126423ea88e9d3274fdde5754334ed81189db0f653069ac47aa66236ec", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocsolver/rocsolver_3.32.0.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocsolver/rocsolver_3.32.0.70202-86~22.04_amd64.deb" ], - "version": "3.32.0.70201-81~22.04" + "version": "3.32.0.70202-86~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.1.70201-81_22.04_amd64", + "key": "rocm-core_7.2.2.70202-86_22.04_amd64", "name": "rocm-core", - "version": "7.2.1.70201-81~22.04" + "version": "7.2.2.70202-86~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -6177,19 +6177,19 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "rocsolver_3.32.0.70201-81_22.04_amd64", + "key": "rocsolver_3.32.0.70202-86_22.04_amd64", "name": "rocsolver", - "version": "3.32.0.70201-81~22.04" + "version": "3.32.0.70202-86~22.04" }, { - "key": "rocblas_5.2.0.70201-81_22.04_amd64", + "key": "rocblas_5.2.0.70202-86_22.04_amd64", "name": "rocblas", - "version": "5.2.0.70201-81~22.04" + "version": "5.2.0.70202-86~22.04" }, { - "key": "hip-runtime-amd_7.2.53211.70201-81_22.04_amd64", + "key": "hip-runtime-amd_7.2.53211.70202-86_22.04_amd64", "name": "hip-runtime-amd", - "version": "7.2.53211.70201-81~22.04" + "version": "7.2.53211.70202-86~22.04" }, { "key": "libnuma1_2.0.14-3ubuntu2_amd64", @@ -6197,14 +6197,14 @@ "version": "2.0.14-3ubuntu2" }, { - "key": "rocprofiler-register_0.6.0.70201-81_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70202-86_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70201-81~22.04" + "version": "0.6.0.70202-86~22.04" }, { - "key": "comgr_3.0.0.70201-81_22.04_amd64", + "key": "comgr_3.0.0.70202-86_22.04_amd64", "name": "comgr", - "version": "3.0.0.70201-81~22.04" + "version": "3.0.0.70202-86~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -6262,9 +6262,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70201-81_22.04_amd64", + "key": "rocminfo_1.0.0.70202-86_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.70201-81~22.04" + "version": "1.0.0.70202-86~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -6292,9 +6292,9 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-rocr_1.18.0.70201-81_22.04_amd64", + "key": "hsa-rocr_1.18.0.70202-86_22.04_amd64", "name": "hsa-rocr", - "version": "1.18.0.70201-81~22.04" + "version": "1.18.0.70202-86~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", @@ -6322,31 +6322,31 @@ "version": "1:2.4.125.70201-2303469.22.04" }, { - "key": "hipblaslt_1.2.2.70201-81_22.04_amd64", + "key": "hipblaslt_1.2.2.70202-86_22.04_amd64", "name": "hipblaslt", - "version": "1.2.2.70201-81~22.04" + "version": "1.2.2.70202-86~22.04" }, { - "key": "roctracer_4.1.70201.70201-81_22.04_amd64", + "key": "roctracer_4.1.70202.70202-86_22.04_amd64", "name": "roctracer", - "version": "4.1.70201.70201-81~22.04" + "version": "4.1.70202.70202-86~22.04" } ], - "key": "hipsolver_3.2.0.70201-81_22.04_amd64", + "key": "hipsolver_3.2.0.70202-86_22.04_amd64", "name": "hipsolver", - "sha256": "dc4f18451d0e2bd370e30e3ab75137429563a253deba56f44e955e5a903f0230", + "sha256": "ba2e5679edda0c87c698edd5d9749d9c9fca10be5e6c8f2b9c65320aeb2fea4b", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/h/hipsolver/hipsolver_3.2.0.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/h/hipsolver/hipsolver_3.2.0.70202-86~22.04_amd64.deb" ], - "version": "3.2.0.70201-81~22.04" + "version": "3.2.0.70202-86~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.1.70201-81_22.04_amd64", + "key": "rocm-core_7.2.2.70202-86_22.04_amd64", "name": "rocm-core", - "version": "7.2.1.70201-81~22.04" + "version": "7.2.2.70202-86~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -6569,42 +6569,42 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "rocfft_1.0.36.70201-81_22.04_amd64", + "key": "rocfft_1.0.36.70202-86_22.04_amd64", "name": "rocfft", - "version": "1.0.36.70201-81~22.04" + "version": "1.0.36.70202-86~22.04" } ], - "key": "hipfft_1.0.22.70201-81_22.04_amd64", + "key": "hipfft_1.0.22.70202-86_22.04_amd64", "name": "hipfft", - "sha256": "19cee2bfd969648b46bb4ee152be823631810ce1e099d008292067f7468aec32", + "sha256": "ac124bc6158c7df85b6facc4debdcc03a164f140d057d80845536c9712ad05d8", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/h/hipfft/hipfft_1.0.22.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/h/hipfft/hipfft_1.0.22.70202-86~22.04_amd64.deb" ], - "version": "1.0.22.70201-81~22.04" + "version": "1.0.22.70202-86~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rocfft_1.0.36.70201-81_22.04_amd64", + "key": "rocfft_1.0.36.70202-86_22.04_amd64", "name": "rocfft", - "sha256": "4866b772d074f7f01e6fb882bef28667baa5d634ef902e95341087864d5e53e4", + "sha256": "185e2b237d9e688f25d23309a25909823be1e1021163589c2c975c48c5dc7fe8", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocfft/rocfft_1.0.36.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocfft/rocfft_1.0.36.70202-86~22.04_amd64.deb" ], - "version": "1.0.36.70201-81~22.04" + "version": "1.0.36.70202-86~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "hipblaslt_1.2.2.70201-81_22.04_amd64", + "key": "hipblaslt_1.2.2.70202-86_22.04_amd64", "name": "hipblaslt", - "version": "1.2.2.70201-81~22.04" + "version": "1.2.2.70202-86~22.04" }, { - "key": "rocm-core_7.2.1.70201-81_22.04_amd64", + "key": "rocm-core_7.2.2.70202-86_22.04_amd64", "name": "rocm-core", - "version": "7.2.1.70201-81~22.04" + "version": "7.2.2.70202-86~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -6827,42 +6827,42 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "roctracer_4.1.70201.70201-81_22.04_amd64", + "key": "roctracer_4.1.70202.70202-86_22.04_amd64", "name": "roctracer", - "version": "4.1.70201.70201-81~22.04" + "version": "4.1.70202.70202-86~22.04" }, { - "key": "hipblas-common-dev_1.4.0.70201-81_22.04_amd64", + "key": "hipblas-common-dev_1.4.0.70202-86_22.04_amd64", "name": "hipblas-common-dev", - "version": "1.4.0.70201-81~22.04" + "version": "1.4.0.70202-86~22.04" } ], - "key": "hipblaslt-dev_1.2.2.70201-81_22.04_amd64", + "key": "hipblaslt-dev_1.2.2.70202-86_22.04_amd64", "name": "hipblaslt-dev", - "sha256": "b6b90b4b9576ad8cf615528ab468537351eadb53576abf0e06a7319d0cbab2d0", + "sha256": "6d9e236ccbbddca308a5939196bd61547002e67de93318b1add927e5680efc2e", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/h/hipblaslt-dev/hipblaslt-dev_1.2.2.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/h/hipblaslt-dev/hipblaslt-dev_1.2.2.70202-86~22.04_amd64.deb" ], - "version": "1.2.2.70201-81~22.04" + "version": "1.2.2.70202-86~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "hipblas-common-dev_1.4.0.70201-81_22.04_amd64", + "key": "hipblas-common-dev_1.4.0.70202-86_22.04_amd64", "name": "hipblas-common-dev", - "sha256": "400298c925c13791860486014e2c9215681be640e525945fc2da7e9eafa6fbfa", + "sha256": "92b5ab03085a869a9e19945fc9b0fed5686824c082ecf0c22058e6441823e3e3", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/h/hipblas-common-dev/hipblas-common-dev_1.4.0.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/h/hipblas-common-dev/hipblas-common-dev_1.4.0.70202-86~22.04_amd64.deb" ], - "version": "1.4.0.70201-81~22.04" + "version": "1.4.0.70202-86~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.1.70201-81_22.04_amd64", + "key": "rocm-core_7.2.2.70202-86_22.04_amd64", "name": "rocm-core", - "version": "7.2.1.70201-81~22.04" + "version": "7.2.2.70202-86~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -7085,14 +7085,14 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "rocblas_5.2.0.70201-81_22.04_amd64", + "key": "rocblas_5.2.0.70202-86_22.04_amd64", "name": "rocblas", - "version": "5.2.0.70201-81~22.04" + "version": "5.2.0.70202-86~22.04" }, { - "key": "hip-runtime-amd_7.2.53211.70201-81_22.04_amd64", + "key": "hip-runtime-amd_7.2.53211.70202-86_22.04_amd64", "name": "hip-runtime-amd", - "version": "7.2.53211.70201-81~22.04" + "version": "7.2.53211.70202-86~22.04" }, { "key": "libnuma1_2.0.14-3ubuntu2_amd64", @@ -7100,14 +7100,14 @@ "version": "2.0.14-3ubuntu2" }, { - "key": "rocprofiler-register_0.6.0.70201-81_22.04_amd64", + "key": "rocprofiler-register_0.6.0.70202-86_22.04_amd64", "name": "rocprofiler-register", - "version": "0.6.0.70201-81~22.04" + "version": "0.6.0.70202-86~22.04" }, { - "key": "comgr_3.0.0.70201-81_22.04_amd64", + "key": "comgr_3.0.0.70202-86_22.04_amd64", "name": "comgr", - "version": "3.0.0.70201-81~22.04" + "version": "3.0.0.70202-86~22.04" }, { "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", @@ -7165,9 +7165,9 @@ "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70201-81_22.04_amd64", + "key": "rocminfo_1.0.0.70202-86_22.04_amd64", "name": "rocminfo", - "version": "1.0.0.70201-81~22.04" + "version": "1.0.0.70202-86~22.04" }, { "key": "pciutils_1-3.7.0-6_amd64", @@ -7195,9 +7195,9 @@ "version": "29-1ubuntu1" }, { - "key": "hsa-rocr_1.18.0.70201-81_22.04_amd64", + "key": "hsa-rocr_1.18.0.70202-86_22.04_amd64", "name": "hsa-rocr", - "version": "1.18.0.70201-81~22.04" + "version": "1.18.0.70202-86~22.04" }, { "key": "libelf1_0.186-1ubuntu0.1_amd64", @@ -7225,23 +7225,23 @@ "version": "1:2.4.125.70201-2303469.22.04" }, { - "key": "hipblaslt_1.2.2.70201-81_22.04_amd64", + "key": "hipblaslt_1.2.2.70202-86_22.04_amd64", "name": "hipblaslt", - "version": "1.2.2.70201-81~22.04" + "version": "1.2.2.70202-86~22.04" }, { - "key": "roctracer_4.1.70201.70201-81_22.04_amd64", + "key": "roctracer_4.1.70202.70202-86_22.04_amd64", "name": "roctracer", - "version": "4.1.70201.70201-81~22.04" + "version": "4.1.70202.70202-86~22.04" } ], - "key": "rocsparse_4.2.0.70201-81_22.04_amd64", + "key": "rocsparse_4.2.0.70202-86_22.04_amd64", "name": "rocsparse", - "sha256": "f29106f3e1a6374cca59c0ee2ba78b1901934316f46951b2ecc136363b5a7bb9", + "sha256": "02ef09cbbf0b4dc801ae097617c3743a725f10dfbef5bd17b6a40589c533f3fc", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocsparse/rocsparse_4.2.0.70201-81~22.04_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocsparse/rocsparse_4.2.0.70202-86~22.04_amd64.deb" ], - "version": "4.2.0.70201-81~22.04" + "version": "4.2.0.70202-86~22.04" } ], "version": 1 diff --git a/platforms/rocm/packages.yaml b/platforms/rocm/packages.yaml index f7af47e1..686963e3 100644 --- a/platforms/rocm/packages.yaml +++ b/platforms/rocm/packages.yaml @@ -5,7 +5,7 @@ version: 1 sources: - channel: jammy main - url: https://repo.radeon.com/rocm/apt/7.2.1 + url: https://repo.radeon.com/rocm/apt/7.2.2 - channel: jammy main url: https://repo.radeon.com/graphics/7.2.1/ubuntu - channel: jammy main diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 98b6291d..f8e56e68 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -7,7 +7,7 @@ _BUILD_FILE_DEFAULT_VISIBILITY = """\ package(default_visibility = ["//visibility:public"]) """ -_ROCM_STRIP_PREFIX = "./opt/rocm-7.2.1" +_ROCM_STRIP_PREFIX = "./opt/rocm-7.2.2" def _rocm_dlopen_patchelf(name, src): return "\n".join([ @@ -254,8 +254,8 @@ def _rocm_impl(mctx): http_archive( name = "libpjrt_rocm", build_file = "libpjrt_rocm.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-03-26T11-30-00Z/pjrt-rocm_linux-amd64.tar.gz", - sha256 = "6649e89831570926bf127f7e57f25dca4f526e22764f2df0d689818badc1d4fe", + url = "https://github.com/zml/pjrt-artifacts/releases/download/nightly-2026-04-17/pjrt-rocm_linux-amd64.tar.gz", + sha256 = "b19db71d3d9aa43bda961a22e81b6a2f48ba2c633a18010c07a66b4ca48ec4d2", ) return mctx.extension_metadata( From 3e3f2f7297129df4c4c22d71fe9a589fecbfd63a Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 20 Apr 2026 10:18:45 +0200 Subject: [PATCH 439/881] zml/tensor: for argxmax, mirror more closely emitted stablehlo for pattern matching For instance, the tenstorrent plugin won't match it --- zml/tensor.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zml/tensor.zig b/zml/tensor.zig index 4d037aea..ab1099e5 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -2896,14 +2896,14 @@ pub const Tensor = struct { const left_gt_right = values.left.cmp(.GT, values.right); const is_nan = values.left.cmp(.NE, values.left); const left_gt_or_nan = left_gt_right.logical(.OR, is_nan); - // we are bubbling up Nan. - const max_val = left_gt_or_nan.select(values.left, values.right); // If values.left == values.right: keep the smallest idx. const is_same = values.left.cmp(.EQ, values.right); const is_first = indices.left.cmp(.LT, indices.right); const is_same_but_first = is_same.logical(.AND, is_first); const keep_left_idx = left_gt_or_nan.logical(.OR, is_same_but_first); + // we are bubbling up Nan. + const max_val = left_gt_or_nan.select(values.left, values.right); const max_idx = keep_left_idx.select(indices.left, indices.right); return .{ max_val, max_idx }; From df86cdfd965d50852c8cad2281d8bb3c1d912859 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Mon, 20 Apr 2026 11:15:37 +0200 Subject: [PATCH 440/881] zml/tokenizer/iree: Improve IREE tokenizer with upstream fix (#482) I've reworked the Tokenizer API to actually work with all tokenizers and not just IREE and moved the `Writer` part into the IREE tokenizer. I'm now avoiding intermediate copies as much as possible now. I'm also more defensive now to avoid the same infinite loop and put more effort into providing appropriate output buffers to IREE. It fails with empty output buffers and may return a `ResourceExhausted` if it's too small. Also added a bunch of tests. IREE tokenizer itself had a bug which I fixed here https://github.com/iree-org/iree/pull/24150. In the meantime, I'm pointing bazel to our fork. --- examples/llm/models/lfm2/session.zig | 30 +- examples/llm/models/llama/session.zig | 30 +- examples/llm/models/qwen3_5/session.zig | 18 +- third_party/iree/fix-gemma4-encode.patch | 149 +++++ third_party/iree/repo.bzl | 1 + zml/tokenizer/iree/tokenizer.zig | 721 ++++++++++++++++++----- zml/tokenizer/main.zig | 10 +- zml/tokenizer/tokenizer.zig | 250 +------- 8 files changed, 824 insertions(+), 385 deletions(-) create mode 100644 third_party/iree/fix-gemma4-encode.patch diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig index 69d67d04..95c79ce1 100644 --- a/examples/llm/models/lfm2/session.zig +++ b/examples/llm/models/lfm2/session.zig @@ -66,10 +66,16 @@ pub const Session = struct { var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); try tokens.appendSlice(allocator, &.{ self.config.bos_token_id, im_start }); - try encoder.encodeAppend(allocator, &tokens, "user\n"); - try encoder.encodeAppend(allocator, &tokens, prompt); + const user_tokens = try encoder.encodeAlloc(allocator, "user\n"); + defer allocator.free(user_tokens); + try tokens.appendSlice(allocator, user_tokens); + const prompt_tokens = try encoder.encodeAlloc(allocator, prompt); + defer allocator.free(prompt_tokens); + try tokens.appendSlice(allocator, prompt_tokens); try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); - try encoder.encodeAppend(allocator, &tokens, "assistant\n"); + const assistant_tokens = try encoder.encodeAlloc(allocator, "assistant\n"); + defer allocator.free(assistant_tokens); + try tokens.appendSlice(allocator, assistant_tokens); return tokens.toOwnedSlice(allocator); } @@ -83,10 +89,16 @@ pub const Session = struct { var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); - try encoder.encodeAppend(allocator, &tokens, "user\n"); - try encoder.encodeAppend(allocator, &tokens, prompt); + const user_tokens = try encoder.encodeAlloc(allocator, "user\n"); + defer allocator.free(user_tokens); + try tokens.appendSlice(allocator, user_tokens); + const prompt_tokens = try encoder.encodeAlloc(allocator, prompt); + defer allocator.free(prompt_tokens); + try tokens.appendSlice(allocator, prompt_tokens); try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); - try encoder.encodeAppend(allocator, &tokens, "assistant\n"); + const assistant_tokens = try encoder.encodeAlloc(allocator, "assistant\n"); + defer allocator.free(assistant_tokens); + try tokens.appendSlice(allocator, assistant_tokens); return tokens.toOwnedSlice(allocator); } @@ -140,12 +152,14 @@ pub const Session = struct { var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice, sharding); defer actual_seq_len_buf.deinit(); + const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); + defer self.allocator.free(out_tokens_buffer); generation: while (true) { const token_id = self.generated_token_slice.items(u32)[0]; if (token_id == self.config.eos_token_id) break :generation; - const token = try decoder.feedOne(token_id); + const token = try decoder.feedOne(token_id, out_tokens_buffer); if (self.think_start) |think_start| if (token_id == think_start) { try stdout.writeAll("\x1b[2m"); }; @@ -178,7 +192,7 @@ pub const Session = struct { try current_token_buffer.toSlice(self.io, self.generated_token_slice); } - try stdout.writeAll(try decoder.finalize()); + try stdout.writeAll(try decoder.finalize(out_tokens_buffer)); try stdout.flush(); } }; diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig index 8fde6361..c429c27b 100644 --- a/examples/llm/models/llama/session.zig +++ b/examples/llm/models/llama/session.zig @@ -72,11 +72,17 @@ pub const Session = struct { var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); try tokens.appendSlice(allocator, &.{ self.config.bos_token_id, start_header }); - try encoder.encodeAppend(allocator, &tokens, "user"); + const user_tokens = try encoder.encodeAlloc(allocator, "user"); + defer allocator.free(user_tokens); + try tokens.appendSlice(allocator, user_tokens); try tokens.appendSlice(allocator, &.{ end_header, newline }); - try encoder.encodeAppend(allocator, &tokens, prompt); + const prompt_tokens = try encoder.encodeAlloc(allocator, prompt); + defer allocator.free(prompt_tokens); + try tokens.appendSlice(allocator, prompt_tokens); try tokens.appendSlice(allocator, &.{ eot, newline, start_header }); - try encoder.encodeAppend(allocator, &tokens, "assistant"); + const assistant_tokens = try encoder.encodeAlloc(allocator, "assistant"); + defer allocator.free(assistant_tokens); + try tokens.appendSlice(allocator, assistant_tokens); try tokens.appendSlice(allocator, &.{ end_header, newline }); return tokens.toOwnedSlice(allocator); } @@ -92,11 +98,17 @@ pub const Session = struct { var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); try tokens.appendSlice(allocator, &.{ eot, newline, start_header }); - try encoder.encodeAppend(allocator, &tokens, "user"); + const user_tokens = try encoder.encodeAlloc(allocator, "user"); + defer allocator.free(user_tokens); + try tokens.appendSlice(allocator, user_tokens); try tokens.appendSlice(allocator, &.{ end_header, newline }); - try encoder.encodeAppend(allocator, &tokens, prompt); + const prompt_tokens = try encoder.encodeAlloc(allocator, prompt); + defer allocator.free(prompt_tokens); + try tokens.appendSlice(allocator, prompt_tokens); try tokens.appendSlice(allocator, &.{ eot, newline, start_header }); - try encoder.encodeAppend(allocator, &tokens, "assistant"); + const assistant_tokens = try encoder.encodeAlloc(allocator, "assistant"); + defer allocator.free(assistant_tokens); + try tokens.appendSlice(allocator, assistant_tokens); try tokens.appendSlice(allocator, &.{ end_header, newline }); return tokens.toOwnedSlice(allocator); } @@ -151,12 +163,14 @@ pub const Session = struct { var current_token_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, self.generated_token_slice, replicated_sharding); defer current_token_buffer.deinit(); + const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); + defer self.allocator.free(out_tokens_buffer); generation: while (true) { const token_id = self.generated_token_slice.items(u32)[0]; if (isEosToken(self.config, token_id)) break :generation; - const token = try decoder.feedOne(token_id); + const token = try decoder.feedOne(token_id, out_tokens_buffer); try stdout.writeAll(token); try stdout.flush(); @@ -181,7 +195,7 @@ pub const Session = struct { try current_token_buffer.toSlice(self.io, self.generated_token_slice); } - try stdout.writeAll(try decoder.finalize()); + try stdout.writeAll(try decoder.finalize(out_tokens_buffer)); try stdout.flush(); } }; diff --git a/examples/llm/models/qwen3_5/session.zig b/examples/llm/models/qwen3_5/session.zig index 834733d0..83feeb72 100644 --- a/examples/llm/models/qwen3_5/session.zig +++ b/examples/llm/models/qwen3_5/session.zig @@ -113,11 +113,13 @@ pub const Session = struct { var decoder = try self.tokenizer.decoder(); defer decoder.deinit(); + const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); + defer self.allocator.free(out_tokens_buffer); generation: while (true) { const token_id = self.generated_token_slice.items(u32)[0]; if (token_id == self.eos_token_id) break :generation; - const token = try decoder.feedOne(token_id); + const token = try decoder.feedOne(token_id, out_tokens_buffer); if (self.think_start) |think_start| if (token_id == think_start) { try stdout.writeAll("\x1b[2m"); }; @@ -133,7 +135,7 @@ pub const Session = struct { try self.runDecodeStep(token_id, @intCast(all_tokens.items.len)); } - try stdout.writeAll(try decoder.finalize()); + try stdout.writeAll(try decoder.finalize(out_tokens_buffer)); try stdout.flush(); } @@ -182,10 +184,16 @@ fn tokenizeChatPrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tok } try tokens.append(allocator, im_start); - try encoder.encodeAppend(allocator, &tokens, "user\n"); - try encoder.encodeAppend(allocator, &tokens, prompt); + const user_tokens = try encoder.encodeAlloc(allocator, "user\n"); + defer allocator.free(user_tokens); + try tokens.appendSlice(allocator, user_tokens); + const prompt_tokens = try encoder.encodeAlloc(allocator, prompt); + defer allocator.free(prompt_tokens); + try tokens.appendSlice(allocator, prompt_tokens); try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); - try encoder.encodeAppend(allocator, &tokens, "assistant\n"); + const assistant_tokens = try encoder.encodeAlloc(allocator, "assistant\n"); + defer allocator.free(assistant_tokens); + try tokens.appendSlice(allocator, assistant_tokens); return tokens.toOwnedSlice(allocator); } diff --git a/third_party/iree/fix-gemma4-encode.patch b/third_party/iree/fix-gemma4-encode.patch new file mode 100644 index 00000000..18f493e6 --- /dev/null +++ b/third_party/iree/fix-gemma4-encode.patch @@ -0,0 +1,149 @@ +From 96151c5c077dc56c00e2b608c4f39b5553089515 Mon Sep 17 00:00:00 2001 +From: Benjamin Rabier +Date: Thu, 16 Apr 2026 09:07:02 +0000 +Subject: [PATCH] Fix tokenization + +Signed-off-by: Benjamin Rabier +--- + runtime/src/iree/tokenizer/tokenizer.c | 13 ++- + .../iree/tokenizer/tokenizer_encode_test.cc | 100 ++++++++++++++++++ + 2 files changed, 109 insertions(+), 4 deletions(-) + +diff --git a/runtime/src/iree/tokenizer/tokenizer.c b/runtime/src/iree/tokenizer/tokenizer.c +index ff9fe6242e..37df477c9d 100644 +--- a/runtime/src/iree/tokenizer/tokenizer.c ++++ b/runtime/src/iree/tokenizer/tokenizer.c +@@ -2606,10 +2606,15 @@ static iree_status_t iree_tokenizer_encode_state_pump( + state->segmenter_view_start = state->read_position; + } + } +- // When a forced-complete drains the ring, exit partial mode so fresh +- // text after the special token goes through the normal segmenter path. +- if (force_complete && segment_complete && +- state->read_position >= state->write_position) { ++ // When a forced-complete segment finishes, retire the entire ring region ++ // that was just fed directly to the model. Unlike the partial reclaim path ++ // above, a non-partial completion resets the model to SEGMENT_START, so ++ // state_reclaim() returns 0 even though all bytes were fully processed. ++ // If we leave the ring positions unchanged, the same bytes are re-encoded ++ // on the next pump iteration and a pending special token can never flush. ++ if (force_complete && segment_complete) { ++ state->read_position = state->write_position; ++ state->segmenter_view_start = state->write_position; + state->has_partial_segment = false; + } + if (reclaimed > 0 || tokens_written > 0) { +diff --git a/runtime/src/iree/tokenizer/tokenizer_encode_test.cc b/runtime/src/iree/tokenizer/tokenizer_encode_test.cc +index a28811fe9b..0eb3f68dc8 100644 +--- a/runtime/src/iree/tokenizer/tokenizer_encode_test.cc ++++ b/runtime/src/iree/tokenizer/tokenizer_encode_test.cc +@@ -3152,6 +3152,106 @@ TEST_F(TokenizerSpecialTokenStreamingTest, FinalizeDoesNotSilentlyDropPartial) { + iree_tokenizer_free(tokenizer); + } + ++// Regression test for a deferred pre-norm special token that causes an infinite ++// loop when the pump enters partial-segment mode. ++// ++// The bug: after a force-complete of ring content (triggered by a pending ++// special token), state_reclaim() returns 0 because the model resets to ++// SEGMENT_START on non-partial completion. The old code required ++// read_position >= write_position to clear partial mode and advance positions, ++// but since reclaim never advanced read_position, this condition was always ++// false — creating an infinite loop re-encoding the same bytes. ++// ++// This test reproduces the real-world trigger: a Replace normalizer that ++// expands " " → "▁" (1 byte → 3 bytes). With a Split segmenter on space ++// (MergedWithPrevious), each word+space is a small segment, but the normalizer ++// expansion means N bytes of input produce up to 3N bytes of normalized output. ++// When enough text with spaces precedes a special token, the normalized output ++// overflows the ring buffer's logical capacity, forcing partial-segment mode. ++// The pending special token then triggers force_complete, exposing the bug. ++// ++// With a 128-byte ring (64-byte logical capacity), ~40 bytes of space-heavy ++// input normalizes to >64 bytes, reliably triggering the overflow. ++TEST_F(TokenizerSpecialTokenStreamingTest, ++ DeferredSpecialTokenDoesNotReplayForcedCompleteSegment) { ++ // ▁ (U+2581) = 0xE2 0x96 0x81 (3 bytes in UTF-8). ++ static const char kMetaspace[] = "\xE2\x96\x81"; ++ ++ ScopedVocabBuilder vocab_builder(257); ++ for (int i = 0; i < 256; ++i) { ++ char byte_token[2] = {static_cast(i), '\0'}; ++ vocab_builder.AddToken(i, byte_token); ++ } ++ // Add ▁ as a token so the BPE model can encode normalized spaces. ++ vocab_builder.AddToken(256, kMetaspace); ++ vocab_builder.AddToken(50256, "<|tool>"); ++ ScopedVocab vocab = vocab_builder.Build(); ++ ++ iree_tokenizer_special_tokens_builder_t st_builder; ++ iree_tokenizer_special_tokens_builder_initialize(iree_allocator_system(), ++ &st_builder); ++ IREE_ASSERT_OK(iree_tokenizer_special_tokens_builder_add( ++ &st_builder, IREE_SV("<|tool>"), 50256, ++ IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_NONE)); ++ iree_tokenizer_special_tokens_t special_tokens; ++ IREE_ASSERT_OK(iree_tokenizer_special_tokens_builder_build( ++ &st_builder, iree_allocator_system(), &special_tokens)); ++ iree_tokenizer_special_tokens_builder_deinitialize(&st_builder); ++ ++ // Replace normalizer: " " → "▁" (1→3 byte expansion, matching Gemma4). ++ iree_tokenizer_normalizer_t* replace_normalizer = nullptr; ++ IREE_ASSERT_OK(iree_tokenizer_normalizer_replace_allocate( ++ IREE_SV(" "), iree_make_string_view(kMetaspace, 3), ++ iree_allocator_system(), &replace_normalizer)); ++ ++ // Split on literal space with MergedWithPrevious (matching Gemma4 config). ++ iree_tokenizer_segmenter_t* segmenter = CreateSplitSegmenterWithBehavior( ++ " ", IREE_TOKENIZER_UTIL_REGEX_SPLIT_MERGED_WITH_PREVIOUS); ++ ASSERT_NE(segmenter, nullptr); ++ ++ ScopedBuilder builder; ++ iree_tokenizer_builder_set_normalizer(builder.get(), replace_normalizer); ++ iree_tokenizer_builder_set_segmenter(builder.get(), segmenter); ++ iree_tokenizer_builder_set_model(builder.get(), CreateBPEModel(vocab.get())); ++ iree_tokenizer_builder_set_special_tokens(builder.get(), &special_tokens); ++ iree_tokenizer_builder_set_vocab(builder.get(), vocab.release()); ++ ++ iree_tokenizer_t* tokenizer = BuildTokenizer(builder.get()); ++ ASSERT_NE(tokenizer, nullptr); ++ ++ // Normal text with many spaces. Each space expands from 1→3 bytes after ++ // normalization. With a 128-byte ring (64-byte logical capacity), even this ++ // short text overflows the ring due to normalizer expansion, forcing ++ // partial-segment mode. The special token then triggers force_complete. ++ std::string input = ++ "A knight of powder-horn and shot " ++ "Once fill'd his bag as I would not " ++ "Unless the feelings of my breast " ++ "By poverty were sorely press'd " ++ "With birds and squirrels for the spits " ++ "Of certain gormandizing cits " ++ "With merry heart the fellow went "; ++ input += "<|tool>d"; ++ ++ IREE_ASSERT_OK_AND_ASSIGN(auto large_tokens, ++ EncodeWithBufferSize(tokenizer, input, 1024)); ++ IREE_ASSERT_OK_AND_ASSIGN(auto small_tokens, ++ EncodeWithBufferSize(tokenizer, input, 256)); ++ ++ EXPECT_EQ(small_tokens, large_tokens) ++ << "Small-buffer streaming must match large-buffer streaming when a " ++ "deferred special token follows text with normalizer expansion"; ++ ++ // Verify the special token and trailing byte are present at the end. ++ ASSERT_GE(small_tokens.size(), 2u); ++ EXPECT_EQ(small_tokens[small_tokens.size() - 2], 50256u); ++ EXPECT_EQ(small_tokens[small_tokens.size() - 1], ++ static_cast('d')); ++ ++ iree_tokenizer_special_tokens_deinitialize(&special_tokens); ++ iree_tokenizer_free(tokenizer); ++} ++ + //===----------------------------------------------------------------------===// + // Strip Normalizer + Special Token Integration Tests + //===----------------------------------------------------------------------===// +-- +2.53.0 + diff --git a/third_party/iree/repo.bzl b/third_party/iree/repo.bzl index 35740408..fc84dee9 100644 --- a/third_party/iree/repo.bzl +++ b/third_party/iree/repo.bzl @@ -13,6 +13,7 @@ def repo(): ], patches = [ "//third_party/iree:tokenizer-only.patch", + "//third_party/iree:fix-gemma4-encode.patch", ], patch_args = ["-p1"], ) diff --git a/zml/tokenizer/iree/tokenizer.zig b/zml/tokenizer/iree/tokenizer.zig index 09e72f73..c38b214c 100644 --- a/zml/tokenizer/iree/tokenizer.zig +++ b/zml/tokenizer/iree/tokenizer.zig @@ -2,6 +2,8 @@ const std = @import("std"); const c = @import("c"); +const log = std.log.scoped(.@"zml/tokenizer"); + pub const Error = error{ Cancelled, Unknown, @@ -35,17 +37,27 @@ inline fn byteSpan(bytes: []u8) c.iree_byte_span_t { return .{ .data = bytes.ptr, .data_length = bytes.len }; } -inline fn tokenOutput(ids: []c.iree_tokenizer_token_id_t) c.iree_tokenizer_token_output_t { +inline fn tokenOutput(ids: []u32) c.iree_tokenizer_token_output_t { + comptime { + if (@sizeOf(c.iree_tokenizer_token_id_t) != @sizeOf(u32) or @alignOf(c.iree_tokenizer_token_id_t) != @alignOf(u32)) { + @compileError("iree_tokenizer_token_id_t must match u32 layout"); + } + } return .{ .capacity = ids.len, - .token_ids = ids.ptr, + .token_ids = @ptrCast(ids.ptr), .token_offsets = null, .type_ids = null, }; } -inline fn tokenIdList(ids: []const c.iree_tokenizer_token_id_t) c.iree_tokenizer_token_id_list_t { - return .{ .count = ids.len, .values = ids.ptr }; +inline fn tokenIdList(ids: []const u32) c.iree_tokenizer_token_id_list_t { + comptime { + if (@sizeOf(c.iree_tokenizer_token_id_t) != @sizeOf(u32) or @alignOf(c.iree_tokenizer_token_id_t) != @alignOf(u32)) { + @compileError("iree_tokenizer_token_id_t must match u32 layout"); + } + } + return .{ .count = ids.len, .values = @ptrCast(ids.ptr) }; } inline fn statusCode(status: c.iree_status_t) u32 { @@ -78,10 +90,6 @@ fn assertOk(status: c.iree_status_t) !void { } pub const Tokenizer = struct { - const EncodeBatchSize = 256; - const DecodeOutputBatchSize = c.IREE_TOKENIZER_DECODE_OUTPUT_RECOMMENDED_SIZE; - const DecodeTokenBatchSize = 256; - allocator: std.mem.Allocator, inner: *c.iree_tokenizer_t, @@ -131,8 +139,6 @@ pub const Tokenizer = struct { state_storage: []u8, transform_buffer: []u8, state: *c.iree_tokenizer_encode_state_t, - output: [EncodeBatchSize]c.iree_tokenizer_token_id_t, - output_token_ids: [EncodeBatchSize]u32, fn init(tokenizer: *const Tokenizer) !Encoder { var state_size: usize = undefined; @@ -159,8 +165,6 @@ pub const Tokenizer = struct { .state_storage = state_storage, .transform_buffer = transform_buffer, .state = state.?, - .output = undefined, - .output_token_ids = undefined, }; } @@ -174,61 +178,160 @@ pub const Tokenizer = struct { c.iree_tokenizer_encode_state_reset(self.state, c.IREE_TOKENIZER_ENCODE_FLAG_AT_INPUT_START); } - pub fn feed(self: *Encoder, chunk: []const u8) !struct { - usize, - []const u32, - } { - var consumed: usize = 0; - var produced: usize = 0; + pub fn encodeAlloc(self: *Encoder, allocator: std.mem.Allocator, text: []const u8) ![]u32 { + var token_ids = std.Io.Writer.Allocating.initAligned(allocator, .of(u32)); + errdefer token_ids.deinit(); - // Feeds an input |chunk| to the encoder. Tokens that can be definitively - // produced are written to |output|. Additional tokens may be produced by - // subsequent feed calls or by finalize once all input has been provided. - // - // Pull-based model: Output buffer capacity drives processing. The encoder - // pulls data through the pipeline (normalizer → segmenter → model) to - // fill the output buffer. Processing stops when output is full or input - // is exhausted. - // - // Returns: - // - |out_bytes_consumed|: Bytes consumed from |chunk|. May be less than - // chunk.size if internal buffers (transform_buffer) fill up. Caller should - // retry with remaining bytes: chunk.data + *out_bytes_consumed. - // - |out_token_count|: Tokens written to |output|. - // - // The function always makes progress if possible: it will consume input - // and/or produce tokens. Returns iree_ok_status() even if not all input - // was consumed (check out_bytes_consumed). - try assertOk(c.iree_tokenizer_encode_state_feed( - self.state, - stringView(chunk), - tokenOutput(&self.output), - &consumed, - &produced, - )); + try self.feed(text, &token_ids.writer); + try self.finalize(&token_ids.writer); - return .{ - consumed, - @as([*]const u32, @ptrCast(&self.output))[0..produced], - }; + const bytes = try token_ids.toOwnedSlice(); + const aligned_bytes: []align(4) u8 = @alignCast(bytes[0..bytes.len]); + return std.mem.bytesAsSlice(u32, aligned_bytes); } - pub fn finalize(self: *Encoder) ![]const u32 { - var produced: usize = 0; + /// /!\ Assumes the writer provides buffers with proper alignment. + /// /!\ Remember to call the finalize() method + pub fn writer(self: *Encoder, buffer: []u8, out: *std.Io.Writer) Writer { + return Writer.init(self, buffer, out); + } - // Finalizes encoding by flushing any buffered data through the pipeline. Must - // be called after all input chunks have been fed. Any remaining tokens are - // written to |output|. - // - // Returns the number of tokens written in |out_token_count|. Returns - // IREE_STATUS_RESOURCE_EXHAUSTED if the output would exceed output.capacity. - try assertOk(c.iree_tokenizer_encode_state_finalize( - self.state, - tokenOutput(&self.output), - &produced, - )); + pub const Writer = struct { + encoder: *Encoder, + out: *std.Io.Writer, + interface: std.Io.Writer, - return @as([*]const u32, @ptrCast(&self.output))[0..produced]; + fn init(encoder_: *Encoder, buffer: []u8, out: *std.Io.Writer) Writer { + return .{ + .encoder = encoder_, + .out = out, + .interface = .{ + .buffer = buffer, + .vtable = &.{ + .drain = drain, + .flush = flush, + .rebase = std.Io.Writer.failingRebase, + }, + }, + }; + } + + pub fn finalize(self: *Writer) std.Io.Writer.Error!void { + try self.interface.flush(); + self.encoder.finalize(self.out) catch |e| { + log.err("Finalize failed with {s}", .{@errorName(e)}); + return error.WriteFailed; + }; + } + + fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { + if (w.end == 0) return; + const self: *Writer = @alignCast(@fieldParentPtr("interface", w)); + try self.feed(w.buffer[0..w.end]); + w.end = 0; + } + + fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + const self: *Writer = @alignCast(@fieldParentPtr("interface", w)); + + if (w.end != 0) { + try self.feed(w.buffer[0..w.end]); + w.end = 0; + } + + var total: usize = 0; + // Prefix chunks: written once each. + for (data[0 .. data.len - 1]) |chunk| { + try self.feed(chunk); + total += chunk.len; + } + // Last chunk is the splat pattern. + const pattern = data[data.len - 1]; + if (pattern.len == 0) return total; + for (0..splat) |_| { + try self.feed(pattern); + total += pattern.len; + } + return total; + } + + fn feed(self: *Writer, chunk: []const u8) std.Io.Writer.Error!void { + self.encoder.feed(chunk, self.out) catch |e| { + log.err("Feed failed with {s}", .{@errorName(e)}); + return error.WriteFailed; + }; + } + }; + + /// /!\ Assumes the writer provides buffers with proper alignment. + pub fn feed(self: *Encoder, text: []const u8, out: *std.Io.Writer) !void { + var min_tokens: usize = @max(text.len / 4, 32); + var remaining = text; + while (remaining.len > 0) { + var consumed: usize = 0; + var produced: usize = 0; + + assertOk(c.iree_tokenizer_encode_state_feed( + self.state, + stringView(remaining), + tokenOutput(try writableTokenSliceGreedy(out, min_tokens)), + &consumed, + &produced, + )) catch |e| switch (e) { + error.ResourceExhausted => { + out.advance(produced * @sizeOf(u32)); + min_tokens -= produced; + remaining = remaining[consumed..]; + produced = 0; + consumed = 0; + }, + else => return e, + }; + + if (produced == 0 and consumed == 0) { + min_tokens *= 2; + std.debug.assert(min_tokens < 1 << 20); + continue; + } + + out.advance(produced * @sizeOf(u32)); + min_tokens -= produced; + remaining = remaining[consumed..]; + } + } + + /// /!\ Assumes the writer provides buffers with proper alignment. + pub fn finalize(self: *Encoder, out: *std.Io.Writer) !void { + var min_tokens: usize = 4; + while (true) { + var produced: usize = 0; + + assertOk(c.iree_tokenizer_encode_state_finalize( + self.state, + tokenOutput(try writableTokenSliceGreedy(out, min_tokens)), + &produced, + )) catch |e| switch (e) { + error.ResourceExhausted => { + out.advance(produced * @sizeOf(u32)); + min_tokens *= 2; + std.debug.assert(min_tokens < 1 << 10); + continue; + }, + else => return e, + }; + + out.advance(produced * @sizeOf(u32)); + break; + } + } + + fn writableTokenSliceGreedy(out: *std.Io.Writer, min_tokens: usize) std.Io.Writer.Error![]u32 { + std.debug.assert(@mod(out.end, @sizeOf(u32)) == 0); + const bytes = try out.writableSliceGreedy(min_tokens * @sizeOf(u32)); + std.debug.assert(@mod(@intFromPtr(bytes.ptr), 4) == 0); + const aligned_len = bytes.len - @mod(bytes.len, @sizeOf(u32)); + const aligned_bytes: []align(4) u8 = @alignCast(bytes[0..aligned_len]); + return std.mem.bytesAsSlice(u32, aligned_bytes); } }; @@ -236,10 +339,8 @@ pub const Tokenizer = struct { tokenizer: *const Tokenizer, state_storage: []u8, state: *c.iree_tokenizer_decode_state_t, - output: [DecodeOutputBatchSize]u8, - input_ids: [DecodeTokenBatchSize]c.iree_tokenizer_token_id_t, - fn init(tokenizer: *const Tokenizer) (Error || std.mem.Allocator.Error)!Decoder { + fn init(tokenizer: *const Tokenizer) !Decoder { var state_size: usize = undefined; try assertOk(c.iree_tokenizer_decode_state_calculate_size(tokenizer.inner, &state_size)); const state_storage = try tokenizer.allocator.alloc(u8, state_size); @@ -257,8 +358,6 @@ pub const Tokenizer = struct { .tokenizer = tokenizer, .state_storage = state_storage, .state = state.?, - .output = undefined, - .input_ids = undefined, }; } @@ -279,70 +378,108 @@ pub const Tokenizer = struct { self.state = state.?; } - pub fn feed(self: *Decoder, token_ids_: []const u32) !struct { - usize, - []const u8, - } { - const token_ids: []const c.iree_tokenizer_token_id_t = - @as([*]const i32, @ptrCast(token_ids_.ptr))[0..token_ids_.len]; - + pub fn feedOne(self: *Decoder, token_id: u32, out: []u8) ![]u8 { + std.debug.assert(out.len != 0); var consumed: usize = 0; - var written: usize = 0; - - // Feeds |tokens| to the decoder. Text that can be definitively produced is - // written to |text_output| as raw UTF-8 bytes (not NUL-terminated). Additional - // text may be produced by subsequent feed calls or by finalize once all tokens - // have been provided. - // - // Pull-based model: Output buffer capacity drives processing. The decoder - // pulls tokens through the pipeline to fill the text output. Processing - // stops when output is full or tokens are exhausted. - // - // Returns iree_ok_status() always (errors are only from internal pipeline - // failures). Progress is indicated by the output parameters: - // - |out_tokens_consumed|: Tokens consumed from |tokens|. May be less than - // tokens.count if text output fills up. Caller should retry with remaining - // tokens: tokens.values + *out_tokens_consumed. - // - |out_text_length|: Bytes written to |text_output|. - // - // Zero-progress case: If both out_tokens_consumed and out_text_length are 0, - // the output buffer is genuinely exhausted (the next token's text does not fit - // in |text_output|). Callers must check for this to avoid infinite loops. - // Provide a larger output buffer or consume the already-written text. + var produced: usize = 0; try assertOk(c.iree_tokenizer_decode_state_feed( self.state, - tokenIdList(token_ids), - mutableStringView(&self.output), + tokenIdList(&.{token_id}), + mutableStringView(out), &consumed, - &written, + &produced, )); - - return .{ - consumed, - self.output[0..written], - }; + std.debug.assert(consumed == 1); + return out[0..produced]; } - pub fn finalize(self: *Decoder) ![]const u8 { - var written: usize = 0; - - // Finalizes decoding by flushing any buffered data through the pipeline. Must - // be called after all tokens have been fed. Any remaining text is written to - // |text_output| as raw UTF-8 bytes (not NUL-terminated). - // - // Returns the number of bytes written in |out_text_length|. Returns - // IREE_STATUS_RESOURCE_EXHAUSTED if the output would exceed text_output.size. + pub fn finalize(self: *Decoder, out: []u8) ![]u8 { + std.debug.assert(out.len != 0); + var produced: usize = 0; try assertOk(c.iree_tokenizer_decode_state_finalize( self.state, - mutableStringView(&self.output), - &written, + mutableStringView(out), + &produced, )); - return self.output[0..written]; + return out[0..produced]; + } + + pub fn decode(self: *Decoder, token_ids: []const u32, out: *std.Io.Writer) !void { + var min_output: usize = @max(token_ids.len * 4, 32); + var remaining = token_ids; + while (remaining.len > 0) { + var consumed: usize = 0; + var produced: usize = 0; + + assertOk(c.iree_tokenizer_decode_state_feed( + self.state, + tokenIdList(remaining), + mutableStringView(try out.writableSliceGreedy(min_output)), + &consumed, + &produced, + )) catch |e| switch (e) { + error.ResourceExhausted => { + out.advance(produced); + min_output -= produced; + remaining = remaining[consumed..]; + produced = 0; + consumed = 0; + }, + else => return e, + }; + + if (produced == 0 and consumed == 0) { + @branchHint(.unlikely); + min_output *= 2; + std.debug.assert(min_output < 1 << 20); + continue; + } + + out.advance(produced); + min_output -= produced; + remaining = remaining[consumed..]; + } + + min_output = 32; + while (true) { + var produced: usize = 0; + + assertOk(c.iree_tokenizer_decode_state_finalize( + self.state, + mutableStringView(try out.writableSliceGreedy(min_output)), + &produced, + )) catch |e| switch (e) { + error.ResourceExhausted => { + out.advance(produced); + min_output *= 2; + std.debug.assert(min_output < 1 << 10); + continue; + }, + else => return e, + }; + + out.advance(produced); + break; + } } }; }; -test "huggingface json streaming encode/decode" { +fn tokenOutputWriter(allocator: std.mem.Allocator) std.Io.Writer.Allocating { + return std.Io.Writer.Allocating.initAligned(allocator, .of(u32)); +} + +fn writtenTokenIds(out: *std.Io.Writer.Allocating) []const u32 { + const bytes = out.written(); + const aligned_bytes: []align(4) u8 = @alignCast(bytes[0..bytes.len]); + return std.mem.bytesAsSlice(u32, aligned_bytes); +} + +fn decodedTextWriter(allocator: std.mem.Allocator) std.Io.Writer.Allocating { + return std.Io.Writer.Allocating.init(allocator); +} + +test "huggingface json encode/decode" { const allocator = std.testing.allocator; const json = \\{ @@ -370,17 +507,9 @@ test "huggingface json streaming encode/decode" { const text = "hello world"; - var token_ids: std.ArrayList(u32) = .empty; - defer token_ids.deinit(allocator); - var remaining_txt: []const u8 = text; - while (remaining_txt.len > 0) { - const consumed, const ids = try encoder.feed(remaining_txt); - try token_ids.appendSlice(allocator, ids); - remaining_txt = remaining_txt[consumed..]; - } - try token_ids.appendSlice(allocator, try encoder.finalize()); - - try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, token_ids.items); + const token_ids = try encoder.encodeAlloc(allocator, text); + defer allocator.free(token_ids); + try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, token_ids); const token_hello = tokenizer.tokenId("hello") orelse unreachable; try std.testing.expectEqual(@as(u32, 1), token_hello); @@ -391,15 +520,335 @@ test "huggingface json streaming encode/decode" { var decoder = try tokenizer.decoder(); defer decoder.deinit(); - var tokens: std.ArrayList(u8) = .empty; - defer tokens.deinit(allocator); - var remaining_ids: []const u32 = token_ids.items; - while (remaining_ids.len > 0) { - const consumed, const chunk = try decoder.feed(remaining_ids); - try tokens.appendSlice(allocator, chunk); - remaining_ids = remaining_ids[consumed..]; + var decoded = decodedTextWriter(allocator); + defer decoded.deinit(); + try decoder.decode(token_ids, &decoded.writer); + try std.testing.expectEqualStrings("hello world", decoded.written()); +} + +test "writer" { + const allocator = std.testing.allocator; + const json = + \\{ + \\ "model": { + \\ "type": "WordPiece", + \\ "unk_token": "[UNK]", + \\ "continuing_subword_prefix": "##", + \\ "max_input_chars_per_word": 100, + \\ "vocab": { + \\ "[UNK]": 0, + \\ "hello": 1, + \\ "world": 2 + \\ } + \\ }, + \\ "pre_tokenizer": {"type": "Whitespace"}, + \\ "decoder": {"type": "WordPiece", "prefix": "##", "cleanup": false} + \\} + ; + + var tokenizer = try Tokenizer.fromBytes(allocator, json); + defer tokenizer.deinit(); + + const text = "hello world"; + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const token_ids_alloc = try encoder.encodeAlloc(allocator, text); + defer allocator.free(token_ids_alloc); + try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, token_ids_alloc); + + encoder.reset(); + var buf: [1024]u8 = undefined; + var out = tokenOutputWriter(allocator); + defer out.deinit(); + var writer = encoder.writer(&buf, &out.writer); + try writer.interface.writeAll(text); + try writer.finalize(); + const token_ids = writtenTokenIds(&out); + + try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, token_ids); + + var decoder = try tokenizer.decoder(); + defer decoder.deinit(); + + var decoded = decodedTextWriter(allocator); + defer decoded.deinit(); + try decoder.decode(token_ids, &decoded.writer); + try std.testing.expectEqualStrings("hello world", decoded.written()); +} + +/// Shared tokenizer JSON config used by multiple tests. +const test_tokenizer_json = + \\{ + \\ "model": { + \\ "type": "WordPiece", + \\ "unk_token": "[UNK]", + \\ "continuing_subword_prefix": "##", + \\ "max_input_chars_per_word": 100, + \\ "vocab": { + \\ "[UNK]": 0, + \\ "hello": 1, + \\ "world": 2, + \\ "foo": 3, + \\ "bar": 4, + \\ "a": 5, + \\ "b": 6, + \\ "c": 7 + \\ } + \\ }, + \\ "pre_tokenizer": {"type": "Whitespace"}, + \\ "decoder": {"type": "WordPiece", "prefix": "##", "cleanup": false} + \\} +; + +test "writer with tiny buffer forces drain" { + // A buffer smaller than the input forces the drain path (not just flush). + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + // Buffer of 4 bytes; "hello world" is 11 bytes, so drain must be invoked. + var buf: [4]u8 = undefined; + var out = tokenOutputWriter(allocator); + defer out.deinit(); + var writer = encoder.writer(&buf, &out.writer); + try writer.interface.writeAll("hello world"); + try writer.finalize(); + + try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, writtenTokenIds(&out)); +} + +test "writer byte-by-byte" { + // Feed one byte at a time through the writer to stress buffer/drain boundaries. + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + var buf: [1]u8 = undefined; + var out = tokenOutputWriter(allocator); + defer out.deinit(); + var writer = encoder.writer(&buf, &out.writer); + for ("hello world") |byte| { + try writer.interface.writeByte(byte); } - try tokens.appendSlice(allocator, try decoder.finalize()); + try writer.finalize(); + + try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, writtenTokenIds(&out)); +} + +test "encoder reset between multiple encodes" { + // Ensure reset properly clears state so successive encodes are independent. + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + { + const ids = try encoder.encodeAlloc(allocator, "foo bar"); + defer allocator.free(ids); + try std.testing.expectEqualSlices(u32, &.{ 3, 4 }, ids); + } + + encoder.reset(); + + { + const ids = try encoder.encodeAlloc(allocator, "hello world"); + defer allocator.free(ids); + try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, ids); + } + + encoder.reset(); + + // Third encode to verify repeated resets work. + { + const ids = try encoder.encodeAlloc(allocator, "foo"); + defer allocator.free(ids); + try std.testing.expectEqualSlices(u32, &.{3}, ids); + } +} + +test "writer and encodeAlloc produce same results" { + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + const inputs = [_][]const u8{ + "hello", + "hello world", + "foo bar", + "a b c", + "foo bar hello world a b c", + }; + + for (inputs) |text| { + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const expected = try encoder.encodeAlloc(allocator, text); + defer allocator.free(expected); + + encoder.reset(); + + var buf: [8]u8 = undefined; + var out = tokenOutputWriter(allocator); + defer out.deinit(); + var writer = encoder.writer(&buf, &out.writer); + try writer.interface.writeAll(text); + try writer.finalize(); + + try std.testing.expectEqualSlices(u32, expected, writtenTokenIds(&out)); + } +} + +test "encode empty string" { + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + // encodeAlloc with empty string should produce no tokens. + { + const ids = try encoder.encodeAlloc(allocator, ""); + defer allocator.free(ids); + try std.testing.expectEqual(@as(usize, 0), ids.len); + } + + encoder.reset(); + + // Writer with empty string should also produce no tokens. + { + var buf: [16]u8 = undefined; + var out = tokenOutputWriter(allocator); + defer out.deinit(); + var writer = encoder.writer(&buf, &out.writer); + try writer.interface.writeAll(""); + try writer.finalize(); + try std.testing.expectEqual(@as(usize, 0), writtenTokenIds(&out).len); + } +} + +test "unknown tokens" { + // Words not in the vocabulary should map to the [UNK] token (id 0). + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const ids = try encoder.encodeAlloc(allocator, "unknown"); + defer allocator.free(ids); + try std.testing.expectEqualSlices(u32, &.{0}, ids); +} + +test "decode round-trip" { + // Encode then decode several strings and verify round-trip fidelity. + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + const inputs = [_][]const u8{ + "hello", + "hello world", + "foo bar", + "a b c", + }; + + for (inputs) |text| { + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const ids = try encoder.encodeAlloc(allocator, text); + defer allocator.free(ids); + + var decoder = try tokenizer.decoder(); + defer decoder.deinit(); + + var decoded = decodedTextWriter(allocator); + defer decoded.deinit(); + try decoder.decode(ids, &decoded.writer); + + try std.testing.expectEqualStrings(text, decoded.written()); + } +} + +test "decoder reset between decodes" { + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + var decoder = try tokenizer.decoder(); + defer decoder.deinit(); + + { + var text = decodedTextWriter(allocator); + defer text.deinit(); + try decoder.decode(&.{ 1, 2 }, &text.writer); + try std.testing.expectEqualStrings("hello world", text.written()); + } + + try decoder.reset(); + + { + var text = decodedTextWriter(allocator); + defer text.deinit(); + try decoder.decode(&.{ 3, 4 }, &text.writer); + try std.testing.expectEqualStrings("foo bar", text.written()); + } +} + +test "tokenId lookup" { + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + try std.testing.expectEqual(@as(u32, 0), tokenizer.tokenId("[UNK]").?); + try std.testing.expectEqual(@as(u32, 1), tokenizer.tokenId("hello").?); + try std.testing.expectEqual(@as(u32, 2), tokenizer.tokenId("world").?); + try std.testing.expectEqual(@as(u32, 3), tokenizer.tokenId("foo").?); + try std.testing.expectEqual(@as(u32, 4), tokenizer.tokenId("bar").?); + try std.testing.expect(tokenizer.tokenId("nonexistent") == null); + try std.testing.expect(tokenizer.tokenId("") == null); +} + +test "writer with pre-allocated output writer" { + // Pass an already-allocated aligned output writer to the encoder stream. + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + var out = tokenOutputWriter(allocator); + defer out.deinit(); + try out.ensureUnusedCapacity(64 * @sizeOf(u32)); + + var buf: [32]u8 = undefined; + var writer = encoder.writer(&buf, &out.writer); + try writer.interface.writeAll("hello world"); + try writer.finalize(); - try std.testing.expectEqualStrings("hello world", tokens.items); + try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, writtenTokenIds(&out)); } diff --git a/zml/tokenizer/main.zig b/zml/tokenizer/main.zig index e275e1d4..14eb1a87 100644 --- a/zml/tokenizer/main.zig +++ b/zml/tokenizer/main.zig @@ -33,14 +33,14 @@ pub fn main(init: std.process.Init) !void { var decoder = try tokenizer.decoder(); defer decoder.deinit(); - var prompt_tok = try encoder.encodeAlloc(allocator, args.prompt); - defer prompt_tok.deinit(allocator); + const prompt_tok = try encoder.encodeAlloc(allocator, args.prompt); + defer allocator.free(prompt_tok); - log.info("Input: {s}\nOutput: {any}", .{ args.prompt, prompt_tok.items }); + log.info("Input: {s}\nOutput: {any}", .{ args.prompt, prompt_tok }); var errors: u8 = 0; { - var reconstructed = try decoder.decodeAlloc(allocator, prompt_tok.items); + var reconstructed = try decoder.decodeAlloc(allocator, prompt_tok); defer reconstructed.deinit(allocator); if (!std.mem.eql(u8, args.prompt, reconstructed.items)) { log.err("Reconstructed string from tokens doesn't match source: {s}", .{reconstructed.items}); @@ -55,7 +55,7 @@ pub fn main(init: std.process.Init) !void { const tok = try std.fmt.parseInt(u32, int_token, 10); try expected.append(allocator, tok); } - if (!std.mem.eql(u32, expected.items, prompt_tok.items)) { + if (!std.mem.eql(u32, expected.items, prompt_tok)) { log.err("Doesn't match expected: {any}", .{expected.items}); errors += 1; } diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig index e95bc7c9..39e359b2 100644 --- a/zml/tokenizer/tokenizer.zig +++ b/zml/tokenizer/tokenizer.zig @@ -34,143 +34,12 @@ pub const Tokenizer = union(Tokenizers) { } } - // - // Convenience methods built on top of the stream API - // - - pub fn encodeAlloc(self: *Encoder, allocator: std.mem.Allocator, tokens: []const u8) !std.ArrayList(u32) { - var token_ids: std.ArrayList(u32) = try .initCapacity(allocator, tokens.len / 4); - try self.encodeAppend(allocator, &token_ids, tokens); - return token_ids; - } - - pub fn encodeAppend(self: *Encoder, allocator: std.mem.Allocator, out: *std.ArrayList(u32), tokens: []const u8) !void { - var remaining = tokens; - while (remaining.len > 0) { - const fout = try self.feed(remaining); - try out.appendSlice(allocator, fout.token_ids); - remaining = remaining[fout.consumed..]; - } - try out.appendSlice(allocator, try self.finalize()); - } - - // - // Stream API - // - - pub const FeedOutput = struct { - consumed: usize, - /// Only valid until the next feed/finalize call and while the encoder is alive. - token_ids: []const u32, - }; - - pub fn feed(self: *Encoder, tokens: []const u8) !FeedOutput { + pub fn encodeAlloc(self: *Encoder, allocator: std.mem.Allocator, text: []const u8) ![]u32 { return switch (self.*) { - .iree => |*v| { - const consumed, const token_ids = try v.feed(tokens); - return .{ .consumed = consumed, .token_ids = token_ids }; - }, - inline else => |*v| { - const token_ids = try v.encode(tokens); - return .{ .consumed = tokens.len, .token_ids = token_ids }; - }, + .iree => |*encoder_| encoder_.encodeAlloc(allocator, text), + inline else => |*encoder_| try allocator.dupe(u32, try encoder_.encode(text)), }; } - - /// Output is only valid until the next feed/finalize call and while the encoder is alive. - pub fn finalize(self: *Encoder) ![]const u32 { - return switch (self.*) { - .iree => |*v| v.finalize(), - inline else => &.{}, - }; - } - - /// Writes the token ids as u8 into the out Writer. - /// Remember to call the finalize() method! - pub fn writer(self: *Encoder, allocator: std.mem.Allocator, buffer: []u8, out: *std.Io.Writer) !Writer { - return try Writer.init(allocator, self, buffer, out); - } - - pub const Writer = struct { - _allocator: std.mem.Allocator, - _encoder: *Tokenizer.Encoder, - _out: *std.Io.Writer, - interface: std.Io.Writer, - - pub fn init(allocator: std.mem.Allocator, encoder_: *Encoder, buffer: []u8, out: *std.Io.Writer) !Writer { - return .{ - ._allocator = allocator, - ._encoder = encoder_, - ._out = out, - .interface = .{ - .buffer = buffer, - .vtable = &.{ - .drain = drain, - .flush = flush, - .rebase = std.Io.Writer.failingRebase, - }, - }, - }; - } - - pub fn finalize(self: *Writer) std.Io.Writer.Error!void { - try self.interface.flush(); - const token_ids = self._encoder.finalize() catch |err| { - log.err("writer drain finalize failed: {s}", .{@errorName(err)}); - return error.WriteFailed; - }; - self._out.writeAll(std.mem.sliceAsBytes(token_ids)) catch |err| { - log.err("writer drain append failed: {s}", .{@errorName(err)}); - return err; - }; - } - - fn appendChunk(self: *Writer, chunk: []const u8) std.Io.Writer.Error!void { - var remaining = chunk; - while (remaining.len > 0) { - const fout = self._encoder.feed(remaining) catch |err| { - log.err("writer drain feed failed: {s}", .{@errorName(err)}); - return error.WriteFailed; - }; - self._out.writeAll(std.mem.sliceAsBytes(fout.token_ids)) catch |err| { - log.err("writer drain append failed: {s}", .{@errorName(err)}); - return err; - }; - remaining = remaining[fout.consumed..]; - } - } - - fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { - if (w.end == 0) return; - const self: *Writer = @alignCast(@fieldParentPtr("interface", w)); - try self.appendChunk(w.buffer[0..w.end]); - w.end = 0; - } - - fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { - const self: *Writer = @alignCast(@fieldParentPtr("interface", w)); - - if (w.end != 0) { - try self.appendChunk(w.buffer[0..w.end]); - w.end = 0; - } - - var total: usize = 0; - // Prefix chunks: written once each. - for (data[0 .. data.len - 1]) |chunk| { - try self.appendChunk(chunk); - total += chunk.len; - } - // Last chunk is the splat pattern. - const pattern = data[data.len - 1]; - if (pattern.len == 0) return total; - for (0..splat) |_| { - try self.appendChunk(pattern); - total += pattern.len; - } - return total; - } - }; }; pub const Decoder = union(Tokenizers) { @@ -192,41 +61,29 @@ pub const Tokenizer = union(Tokenizers) { } pub fn decodeAlloc(self: *Decoder, allocator: std.mem.Allocator, token_ids: []const u32) !std.ArrayList(u8) { - switch (self.*) { - .iree => |*d| { - var tokens: std.ArrayList(u8) = try .initCapacity(allocator, token_ids.len * 4); - var remaining = token_ids; - while (remaining.len > 0) { - const consumed, const decoded_tokens = try d.feed(remaining); - try tokens.appendSlice(allocator, decoded_tokens); - remaining = remaining[consumed..]; - } - try tokens.appendSlice(allocator, try self.finalize()); - return tokens; - }, - inline else => |*d| { - return .fromOwnedSlice(try allocator.dupe(u8, try d.decode(token_ids))); - }, - } - } - - // Only valid until the next decode and while the decoder is alive. - pub fn feedOne(self: *Decoder, token_id: u32) ![]const u8 { - switch (self.*) { - .iree => |*d| { - _, const tokens = try d.feed(&.{token_id}); - return tokens; - }, - inline else => |*d| { - return try d.next(token_id) orelse &.{}; - }, - } - } - - /// Output is only valid until the next feed/finalize call and while the decoder is alive. - pub fn finalize(self: *Decoder) ![]const u8 { return switch (self.*) { - .iree => |*v| v.finalize(), + .iree => |*d| { + var writer: std.Io.Writer.Allocating = .init(allocator); + errdefer writer.deinit(); + try d.decode(token_ids, &writer.writer); + return writer.toArrayList(); + }, + inline else => |*d| .fromOwnedSlice(try allocator.dupe(u8, try d.decode(token_ids))), + }; + } + + // Only valid until the next decode and while the decoder AND buffer is alive. + pub fn feedOne(self: *Decoder, token_id: u32, buffer: []u8) ![]const u8 { + return switch (self.*) { + .iree => |*d| d.feedOne(token_id, buffer), + inline else => |*d| try d.next(token_id) orelse &.{}, + }; + } + + /// Output is only valid until the next feed/finalize call and while the decoder AND buffer is alive. + pub fn finalize(self: *Decoder, buffer: []u8) ![]const u8 { + return switch (self.*) { + .iree => |*v| v.finalize(buffer), inline else => &.{}, }; } @@ -238,22 +95,19 @@ pub const Tokenizer = union(Tokenizers) { homemade: *homemade.Tokenizer, pub fn fromFile(allocator: std.mem.Allocator, io: std.Io, model: []const u8) !Tokenizer { - _ = io; - _ = allocator; if (std.mem.endsWith(u8, model, ".pb")) { return .{ .sentencepiece = try .fromFile(model) }; } if (std.mem.endsWith(u8, model, ".json")) { - return .{ .hftokenizers = try .fromFile(model) }; + return .{ .iree = try .fromFile(allocator, io, model) }; } return error.InvalidArgument; } pub fn fromBytes(allocator: std.mem.Allocator, bytes: []const u8) !Tokenizer { - _ = allocator; if (bytes[0] == '{') { - return .{ .hftokenizers = try .fromBytes(bytes) }; + return .{ .iree = try .fromBytes(allocator, bytes) }; } return .{ .sentencepiece = try .fromBytes(bytes) }; } @@ -286,53 +140,3 @@ pub const Tokenizer = union(Tokenizers) { test { std.testing.refAllDecls(@This()); } - -test "iree tokenizer smoke: alloc and writer" { - const allocator = std.testing.allocator; - const json = - \\{ - \\ "model": { - \\ "type": "WordPiece", - \\ "unk_token": "[UNK]", - \\ "continuing_subword_prefix": "##", - \\ "max_input_chars_per_word": 100, - \\ "vocab": { - \\ "[UNK]": 0, - \\ "hello": 1, - \\ "world": 2 - \\ } - \\ }, - \\ "pre_tokenizer": {"type": "Whitespace"}, - \\ "decoder": {"type": "WordPiece", "prefix": "##", "cleanup": false} - \\} - ; - - var tokenizer = try Tokenizer.fromBytes(allocator, json); - defer tokenizer.deinit(); - - const text = "hello world"; - - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - var token_ids_alloc = try encoder.encodeAlloc(allocator, text); - defer token_ids_alloc.deinit(allocator); - try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, token_ids_alloc.items); - - encoder.reset(); - var token_ids_writer: std.Io.Writer.Allocating = .initAligned(allocator, std.mem.Alignment.of(u32)); - defer token_ids_writer.deinit(); - var writer = try encoder.writer(allocator, &.{}, &token_ids_writer.writer); - try writer.interface.writeAll(text); - try writer.finalize(); - - const token_ids: []u32 = @alignCast(std.mem.bytesAsSlice(u32, token_ids_writer.written())); - try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, token_ids); - - var decoder = try tokenizer.decoder(); - defer decoder.deinit(); - - var tokens = try decoder.decodeAlloc(allocator, token_ids); - defer tokens.deinit(allocator); - try std.testing.expectEqualStrings("hello world", tokens.items); -} From c1a902b1317268d7b91ebdb2d91afbcb516690f8 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Mon, 20 Apr 2026 17:16:45 +0200 Subject: [PATCH 441/881] workspace/xla: update to nightly-2026-04-20 (#485) --- MODULE.bazel.lock | 8 +++---- platforms/cpu/cpu.bzl | 12 +++++----- platforms/cuda/cuda.bzl | 4 ++-- platforms/rocm/rocm.bzl | 4 ++-- third_party/xla/repo.bzl | 2 +- third_party/xla/xla.bzl | 52 +++++++++++++++++++++++++++++++++++++--- 6 files changed, 64 insertions(+), 18 deletions(-) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index c556bde6..a390d19b 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1026,7 +1026,7 @@ "@@rules_oci+//oci:extensions.bzl%oci": { "general": { "bzlTransitiveDigest": "twSeB3VPTVkayY0uwqEzDzal5MbnZX45iC6jdheZQRo=", - "usagesDigest": "V0dQbNX+eB3aThhopTuYblOol9rtleO/RA16gC5Zn+Y=", + "usagesDigest": "JLbamplho20rviuebatMGqnQnaoj32ewCzLuRVusGoU=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -1065,7 +1065,7 @@ "attributes": { "www_authenticate_challenges": {}, "scheme": "https", - "registry": "index.docker.io", + "registry": "mirror.gcr.io", "repository": "library/debian", "identifier": "sha256:bc960ef50e6feed90686c593361df158517556ed1d2d98e5d1df3724024e0f49", "platform": "linux/amd64", @@ -1079,7 +1079,7 @@ "target_name": "debian_12", "www_authenticate_challenges": {}, "scheme": "https", - "registry": "index.docker.io", + "registry": "mirror.gcr.io", "repository": "library/debian", "identifier": "sha256:bc960ef50e6feed90686c593361df158517556ed1d2d98e5d1df3724024e0f49", "platforms": { @@ -1417,7 +1417,7 @@ }, "@@rules_zig+//zig:extensions.bzl%zig": { "general": { - "bzlTransitiveDigest": "r7tCqbjlVKc7XEXV8Hiak2ElFfkjvDdMf8Ym+7zxukU=", + "bzlTransitiveDigest": "xx2lb1ba6RcnnXdfUyaxI5j2wFLM03GTQfTv2UUie3Q=", "usagesDigest": "3YLtfwSfkXudL3+Coim7hrTFktQHln4CiUbszoYuRzE=", "recordedFileInputs": { "@@//bazel/zig_index.json": "2e3b4b83f1add5d3ca12fd05797c04c38af21d86c7fc7c1a9b356560141a4797", diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 35d0746e..7360887c 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -23,22 +23,22 @@ def _cpu_pjrt_plugin_impl(mctx): http_archive( name = "libpjrt_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "880a9575be4c950cd49b6ca20c2782869cea7ec445bac843ac55b21053b648ef", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v15.0.2/pjrt-cpu_linux-amd64.tar.gz", + sha256 = "867ea7129a61eccb2170fb928cf008577431b1ed5b1ef317766e14deb9660513", + url = "https://github.com/zml/pjrt-artifacts/releases/download/nightly-2026-04-20/pjrt-cpu_linux-amd64.tar.gz", ) http_archive( name = "libpjrt_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "e1bd183a862fa1c504ade5aca1e000b273628ab2bd2a9eb2971067584934d882", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v15.0.2/pjrt-cpu_darwin-amd64.tar.gz", + sha256 = "sha256:12515794094962e37b15c2527e4580b8802f83839d13801937191c1ab4b49a2e", + url = "https://github.com/zml/pjrt-artifacts/releases/download/nightly-2026-04-20/pjrt-cpu_darwin-amd64.tar.gz", ) http_archive( name = "libpjrt_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "fb733aeec1284bfe4573acbe78ca38292ab521e8f207a3547d56d1a6d7173d47", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v15.0.2/pjrt-cpu_darwin-arm64.tar.gz", + sha256 = "e1a648ce12d25ce04aa107b4cfa3aa4c5f675968488779fcbf17b3a98c207da3", + url = "https://github.com/zml/pjrt-artifacts/releases/download/nightly-2026-04-20/pjrt-cpu_darwin-arm64.tar.gz", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index a1524824..77547631 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -277,8 +277,8 @@ def _cuda_impl(mctx): http_archive( name = "libpjrt_cuda", build_file = "libpjrt_cuda.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/nightly-2026-03-24/pjrt-cuda_linux-amd64.tar.gz", - sha256 = "9ef9caf168974e71c8932361204739b739e4c61150637ab3c170ec4765ce9ac1", + url = "https://github.com/zml/pjrt-artifacts/releases/download/nightly-2026-04-20/pjrt-cuda_linux-amd64.tar.gz", + sha256 = "90c67e30414c2644d2b9b25bc7418b24ee077b3933ed782ba9ca7318dcc724c9", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index f8e56e68..d1a79ee6 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -254,8 +254,8 @@ def _rocm_impl(mctx): http_archive( name = "libpjrt_rocm", build_file = "libpjrt_rocm.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/nightly-2026-04-17/pjrt-rocm_linux-amd64.tar.gz", - sha256 = "b19db71d3d9aa43bda961a22e81b6a2f48ba2c633a18010c07a66b4ca48ec4d2", + url = "https://github.com/zml/pjrt-artifacts/releases/download/nightly-2026-04-20/pjrt-rocm_linux-amd64.tar.gz", + sha256 = "5f85f35c90e36aa9557a51ff210392ffd27f8853226e05020f81309c4ca4e5e7", ) return mctx.extension_metadata( diff --git a/third_party/xla/repo.bzl b/third_party/xla/repo.bzl index 7f3f0c1c..9250c314 100644 --- a/third_party/xla/repo.bzl +++ b/third_party/xla/repo.bzl @@ -4,7 +4,7 @@ def repo(): git_repository( name = "xla", remote = "https://github.com/openxla/xla.git", - commit = "01da52b9afe3a2e694bd926323a649a6e63a3785", + commit = "47f005bb8150a13cb0217c2d7daf108bcdca34cc", patch_args = ["-p0"], patches = ["//third_party/xla:xspace_to_perfetto.patch"], ) diff --git a/third_party/xla/xla.bzl b/third_party/xla/xla.bzl index 701f2da0..e0b20989 100644 --- a/third_party/xla/xla.bzl +++ b/third_party/xla/xla.bzl @@ -112,6 +112,7 @@ filegroup( "sycl/build_defs.bzl": _BZL_HELPERS + """\ if_sycl = always_if_false if_sycl_is_configured = always_if_false +sycl_library = native.cc_library """, }) simple_files(name = "local_config_remote_execution", files = { @@ -134,6 +135,51 @@ filegroup( "py_version.bzl": """USE_PYWRAP_RULES = False""", }) simple_files(name = "rules_ml_toolchain", files = { + "py/rules_pywrap/BUILD.bazel": """\ +package(default_visibility = ["//visibility:public"]) + +exports_files([ + "pywrap.default.bzl", + "pywrap.impl.bzl", +]) + +filegroup( + name = "pywrap_bzl", + srcs = [ + "pywrap.default.bzl", + "pywrap.impl.bzl", + ], +) +""", + "py/rules_pywrap/pywrap.default.bzl": """\ +def use_pywrap_rules(): + return False + +def _unsupported(*args, **kwargs): + fail("rules_ml_toolchain pywrap rules are not available in this workspace") + +pybind_extension = _unsupported +pywrap_aware_cc_import = _unsupported +pywrap_aware_filegroup = _unsupported +pywrap_aware_genrule = _unsupported +pywrap_binaries = _unsupported +pywrap_common_library = _unsupported +pywrap_library = _unsupported +stripped_cc_info = _unsupported +""", + "py/rules_pywrap/pywrap.impl.bzl": """\ +def _unsupported(*args, **kwargs): + fail("rules_ml_toolchain pywrap rules are not available in this workspace") + +collected_pywrap_infos = _unsupported +generated_common_win_def_file = _unsupported +pybind_extension = _unsupported +python_extension = _unsupported +pywrap_binaries = _unsupported +pywrap_common_library = _unsupported +pywrap_library = _unsupported +stripped_cc_info = _unsupported +""", "third_party/gpus/BUILD.bazel": "", "third_party/gpus/nvidia_common_rules.bzl": """cuda_rpath_flags = lambda *args, **kwargs: []""", "third_party/extensions/sycl_configure.bzl": "", @@ -152,10 +198,10 @@ def _xla_impl(mctx): tf_http_archive( name = "com_github_grpc_grpc", - sha256 = "dd6a2fa311ba8441bbefd2764c55b99136ff10f7ea42954be96006a2723d33fc", - strip_prefix = "grpc-1.74.0", + sha256 = "e2ace790a5f2d0f83259d1390a816a33b013ea34df2e86084d927e58daa4c5d9", + strip_prefix = "grpc-1.78.0", patch_file = ["//third_party/grpc:grpc.patch"], - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/refs/tags/v1.74.0.tar.gz"), + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/refs/tags/v1.78.0.tar.gz"), ) tf_vendored(name = "tsl", path = "third_party/tsl") From c7f26eeeac8e6741c0e832d99d3ade325e3f9fc0 Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Mon, 20 Apr 2026 18:14:09 +0200 Subject: [PATCH 442/881] zml/attention: standardization of the mosaic ragged page attention structure (#486) Aligns Mosaic TPU ragged paged-attention input/parameter naming with the terminology used by other backends (e.g., block table + sequence lengths) Changes: - Rename Mosaic TPU ragged attention inputs/parameters from kv_lens/page_indices/cu_seqlens_q to seq_lens/block_table/query_start_len. --- zml/attention/tpu_attention.zig | 74 ++++++++++++++++----------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/zml/attention/tpu_attention.zig b/zml/attention/tpu_attention.zig index 1a052397..266f89f7 100644 --- a/zml/attention/tpu_attention.zig +++ b/zml/attention/tpu_attention.zig @@ -129,9 +129,9 @@ pub const mosaic_tpu = struct { const PreparedInputs = struct { q: zml.Tensor, kv_pages: zml.Tensor, - kv_lens: zml.Tensor, - page_indices: zml.Tensor, - cu_seqlens_q: zml.Tensor, + seq_lens: zml.Tensor, + block_table: zml.Tensor, + query_start_len: zml.Tensor, num_seqs: zml.Tensor, }; @@ -157,16 +157,16 @@ pub const mosaic_tpu = struct { pub const Parameters = struct { opts: Options, - cu_seqlens_q: zml.Tensor, - page_indices: zml.Tensor, - kv_lens: zml.Tensor, + block_table: zml.Tensor, + seq_lens: zml.Tensor, + query_start_len: zml.Tensor, pub fn init(opts: Options) Parameters { return .{ .opts = opts, - .cu_seqlens_q = zml.Tensor.init(.{ .n = opts.batch_size + 1 }, .i32), - .page_indices = zml.Tensor.init(.{ opts.batch_size, opts.max_num_pages }, .i32), - .kv_lens = zml.Tensor.init(.{opts.batch_size}, .i32), + .block_table = zml.Tensor.init(.{ opts.batch_size, opts.max_num_pages }, .i32), + .seq_lens = zml.Tensor.init(.{opts.batch_size}, .i32), + .query_start_len = zml.Tensor.init(.{ .n = opts.batch_size + 1 }, .i32), }; } @@ -176,9 +176,9 @@ pub const mosaic_tpu = struct { pub fn allocationSize(self: Parameters) usize { var allocation_size: usize = 0; - allocation_size += self.cu_seqlens_q.byteSize(); - allocation_size += self.page_indices.byteSize(); - allocation_size += self.kv_lens.byteSize(); + allocation_size += self.block_table.byteSize(); + allocation_size += self.seq_lens.byteSize(); + allocation_size += self.query_start_len.byteSize(); return allocation_size; } }; @@ -192,9 +192,9 @@ pub const mosaic_tpu = struct { fn raggedPagedKernelCall( q: zml.Tensor, kv_pages: zml.Tensor, - kv_lens: zml.Tensor, - page_indices: zml.Tensor, - cu_q_lens: zml.Tensor, + seq_lens: zml.Tensor, + block_table: zml.Tensor, + query_start_len: zml.Tensor, num_seqs: zml.Tensor, backend_config: []const u8, ) zml.Tensor { @@ -204,9 +204,9 @@ pub const mosaic_tpu = struct { const op = dialects.stablehlo.custom_call( comp_ctx.mlir_ctx, &.{ - kv_lens.value(), - page_indices.value(), - cu_q_lens.value(), + seq_lens.value(), + block_table.value(), + query_start_len.value(), seq_buf_idx.value(), num_seqs.value(), q.value(), @@ -227,9 +227,9 @@ pub const mosaic_tpu = struct { return zml.Tensor._result(q.shape(), op.result(0)); } - fn activeSequenceCount(cu_seqlens_q: zml.Tensor) zml.Tensor { - const start = cu_seqlens_q.slice1d(0, .{ .end = cu_seqlens_q.dim(0) - 1 }); - const end = cu_seqlens_q.slice1d(0, .{ .start = 1 }); + fn activeSequenceCount(query_start_len: zml.Tensor) zml.Tensor { + const start = query_start_len.slice1d(0, .{ .end = query_start_len.dim(0) - 1 }); + const end = query_start_len.slice1d(0, .{ .start = 1 }); const query_lens = end.sub(start); return query_lens .cmp(.GT, zml.Tensor.constant(query_lens.dtype().zero()).broad(query_lens.shape())) @@ -286,17 +286,17 @@ pub const mosaic_tpu = struct { // Note : // In a future PR, we'll have to change the pagedAttention API a bit so that we don't have to fuse keys and values const kv_pages = fuseKvPages(k_cache, v_cache).withPartitioning(.{ .hkv = .model }); - const cu_seqlens_q = parameters.cu_seqlens_q.withPartitioning(.{ .n = .replicated }); + const query_start_len = parameters.query_start_len.withPartitioning(.{ .n = .replicated }); // Note // activeSequenceCount could probably be computed when preparing the inputs and injected through Parameters, but we'll keep that for later. return .{ .q = q_ragged, .kv_pages = kv_pages, - .kv_lens = parameters.kv_lens.withPartitioning(.{ ._0 = .replicated }), - .page_indices = parameters.page_indices.withPartitioning(.{ ._0 = .replicated, ._1 = .replicated }), - .cu_seqlens_q = cu_seqlens_q, - .num_seqs = activeSequenceCount(cu_seqlens_q).withPartitioning(.{ ._0 = .replicated }), + .seq_lens = parameters.seq_lens.withPartitioning(.{ ._0 = .replicated }), + .block_table = parameters.block_table.withPartitioning(.{ ._0 = .replicated, ._1 = .replicated }), + .query_start_len = query_start_len, + .num_seqs = activeSequenceCount(query_start_len).withPartitioning(.{ ._0 = .replicated }), }; } @@ -308,10 +308,10 @@ pub const mosaic_tpu = struct { fn buildRequest(prepared_inputs: PreparedInputs, parameters: Parameters, opts: AttentionOptions) RaggedPagedRequest { const q_shape = prepared_inputs.q.shape(); const kv_pages_shape = prepared_inputs.kv_pages.shape(); - const page_indices_shape = prepared_inputs.page_indices.shape(); + const block_table_shape = prepared_inputs.block_table.shape(); const q_token_count = if (q_shape.hasTag(.q) != null) q_shape.dim(.q) else q_shape.dim(.b); - const max_num_seqs = page_indices_shape.dim(0); + const max_num_seqs = block_table_shape.dim(0); const num_q_heads = q_shape.dim(.h); stdx.debug.assert(@mod(kv_pages_shape.dim(.hkv), 2) == 0, "Expected fused KV pages .hkv dimension to be even, got {}", .{kv_pages_shape.dim(.hkv)}); const num_kv_heads = std.math.divExact(i64, kv_pages_shape.dim(.hkv), 2) catch |err| switch (err) { @@ -365,9 +365,9 @@ pub const mosaic_tpu = struct { .{ prepared.q, prepared.kv_pages, - prepared.kv_lens, - prepared.page_indices, - prepared.cu_seqlens_q, + prepared.seq_lens, + prepared.block_table, + prepared.query_start_len, prepared.num_seqs, }, prepared.q.shape(), @@ -383,9 +383,9 @@ pub const mosaic_tpu = struct { const prepared_inputs: PreparedInputs = .{ .q = sharded_inputs[0], .kv_pages = sharded_inputs[1], - .kv_lens = sharded_inputs[2], - .page_indices = sharded_inputs[3], - .cu_seqlens_q = sharded_inputs[4], + .seq_lens = sharded_inputs[2], + .block_table = sharded_inputs[3], + .query_start_len = sharded_inputs[4], .num_seqs = sharded_inputs[5], }; @@ -393,9 +393,9 @@ pub const mosaic_tpu = struct { const q_out = raggedPagedKernelCall( prepared_inputs.q, prepared_inputs.kv_pages, - prepared_inputs.kv_lens, - prepared_inputs.page_indices, - prepared_inputs.cu_seqlens_q, + prepared_inputs.seq_lens, + prepared_inputs.block_table, + prepared_inputs.query_start_len, prepared_inputs.num_seqs, backend_config, ); From f258280ab80ce619edc902712d092438ee3143a7 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Mon, 20 Apr 2026 18:24:01 +0200 Subject: [PATCH 443/881] platforms/cuda+rocm: update pjrt artifact (#487) Update with the XLA patches for improved parallel compilation. `//examples/llm` goes from 27s to 8s for llama3.1 8B compilation --- examples/llm/main.zig | 7 ++++--- platforms/cuda/cuda.bzl | 4 ++-- platforms/rocm/rocm.bzl | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/examples/llm/main.zig b/examples/llm/main.zig index abd3a86a..9388eaa0 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -123,12 +123,13 @@ pub fn main(init: std.process.Init) !void { var tokenizer = try loadTokenizer(allocator, io, repo, &progress); defer tokenizer.deinit(); - var model_buffers = try models.LoadedModel.loadBuffers(&model, allocator, io, platform, &store, &progress, shardings); - defer model.unloadBuffers(&model_buffers, allocator); - var compiled_model = try models.LoadedModel.compile(&model, allocator, io, platform, backend, shardings, args.seqlen, &progress); defer compiled_model.deinit(); + // Load buffers after the model compilation to be sure to give enough room to the autotune. + var model_buffers = try models.LoadedModel.loadBuffers(&model, allocator, io, platform, &store, &progress, shardings); + defer model.unloadBuffers(&model_buffers, allocator); + progress.end(); // diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 77547631..10d4aa65 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -277,8 +277,8 @@ def _cuda_impl(mctx): http_archive( name = "libpjrt_cuda", build_file = "libpjrt_cuda.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/nightly-2026-04-20/pjrt-cuda_linux-amd64.tar.gz", - sha256 = "90c67e30414c2644d2b9b25bc7418b24ee077b3933ed782ba9ca7318dcc724c9", + url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-04-20T16-00-00Z-2/pjrt-cuda_linux-amd64.tar.gz", + sha256 = "abad33460249b8104c9acf6eb5486569fda9f3a14750f5fcaffff2d6a24ca74f", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index d1a79ee6..185ed519 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -254,8 +254,8 @@ def _rocm_impl(mctx): http_archive( name = "libpjrt_rocm", build_file = "libpjrt_rocm.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/nightly-2026-04-20/pjrt-rocm_linux-amd64.tar.gz", - sha256 = "5f85f35c90e36aa9557a51ff210392ffd27f8853226e05020f81309c4ca4e5e7", + url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-04-20T16-00-00Z-2/pjrt-rocm_linux-amd64.tar.gz", + sha256 = "50200ad4037d84149b87a5a541fa2a86b87ae159d43a799e92b905d863407449", ) return mctx.extension_metadata( From 49c805f7e34516565c2f5712a8b23f2aedafcf5a Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Mon, 20 Apr 2026 19:28:49 +0200 Subject: [PATCH 444/881] zml/attention/triton: thread-safe generateTtir (#488) generateTtir isn't thread-safe as we keep a single process alive. Using different processes didn't seem to be better from a quick try. So just locking it with a mutex. --- zml/attention/triton.zig | 33 +++++++++++++++++++++++---------- zml/platform.zig | 2 +- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/zml/attention/triton.zig b/zml/attention/triton.zig index 3f731608..cb8fb65d 100644 --- a/zml/attention/triton.zig +++ b/zml/attention/triton.zig @@ -211,14 +211,19 @@ fn generateTtir(allocator: std.mem.Allocator, io: std.Io, config: GenerationConf const platform = compilation_context.platform; const triton_runtime = platform.triton_runtime.?; - var writer = triton_runtime.process.stdin.?.writer(io, writer_buffer); - try writer.interface.print("{f}\n", .{std.json.fmt(config, .{ .emit_null_optional_fields = false })}); - try writer.interface.flush(); - - var reader = triton_runtime.process.stdout.?.reader(io, reader_buffer); var allocating: std.Io.Writer.Allocating = .init(arena.allocator()); - _ = try reader.interface.streamDelimiter(&allocating.writer, '\n'); - _ = try reader.interface.discardShort(1); + { + try triton_runtime.process_mutex.lock(compilation_context.io); + defer triton_runtime.process_mutex.unlock(compilation_context.io); + + var writer = triton_runtime.process.stdin.?.writer(io, writer_buffer); + try writer.interface.print("{f}\n", .{std.json.fmt(config, .{ .emit_null_optional_fields = false })}); + try writer.interface.flush(); + + var reader = triton_runtime.process.stdout.?.reader(io, reader_buffer); + _ = try reader.interface.streamDelimiter(&allocating.writer, '\n'); + _ = try reader.interface.discardShort(1); + } const response: std.json.Value = try std.json.parseFromSliceLeaky(std.json.Value, arena.allocator(), allocating.written(), .{}); return if (response.object.get("ok").?.bool) @@ -638,6 +643,7 @@ pub const paged = struct { pub const Runtime = struct { process: std.process.Child, + process_mutex: *std.Io.Mutex, pub fn init(allocator: std.mem.Allocator, io: std.Io) !Runtime { var arena: std.heap.ArenaAllocator = .init(allocator); @@ -649,18 +655,25 @@ pub const Runtime = struct { var map: std.process.Environ.Map = .init(arena.allocator()); try map.put("LD_LIBRARY_PATH", std.Io.Dir.path.join(arena.allocator(), &.{ sandbox_path.?, "lib" }) catch unreachable); + const process_mutex = try allocator.create(std.Io.Mutex); + errdefer allocator.destroy(process_mutex); + process_mutex.* = .init; + const process = try std.process.spawn(io, .{ .argv = &.{std.Io.Dir.path.join(arena.allocator(), &.{ sandbox_path.?, "bin", "generate" }) catch unreachable}, .stdin = .pipe, .stdout = .pipe, .environ_map = &map, }); - errdefer process.kill(io); - return .{ .process = process }; + return .{ + .process = process, + .process_mutex = process_mutex, + }; } - pub fn deinit(self: *Runtime, io: std.Io) void { + pub fn deinit(self: *Runtime, allocator: std.mem.Allocator, io: std.Io) void { + allocator.destroy(self.process_mutex); // NOTE(Corendos): This is not portable, but I couldn't find a better way with the current std.Io. _ = std.c.kill(self.process.id.?, .INT); _ = self.process.wait(io) catch unreachable; diff --git a/zml/platform.zig b/zml/platform.zig index dd43f787..01361ca1 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -424,7 +424,7 @@ pub const Platform = struct { pub fn deinit(self: *Platform, allocator: std.mem.Allocator, io: std.Io) void { self.pjrt_client.deinit(self.pjrt_api); if (self.tpu_ir_runtime) |*rt| rt.deinit(io); - if (self.triton_runtime) |*rt| rt.deinit(io); + if (self.triton_runtime) |*rt| rt.deinit(allocator, io); self.arena_state.promote(allocator).deinit(); } From b25349b384e55148f03ba938f835a0eb7185ebf6 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 21 Apr 2026 14:27:00 +0200 Subject: [PATCH 445/881] ci: bump ci actions (Node.js 20 actions are deprecated) (#490) Use newer versions of several GitHub Actions. Also bump used Zig version. `mlugg/setup-zig` is still using Node.js v20 so warning still here but only for this action. Check https://codeberg.org/mlugg/setup-zig/pulls/55 in a near future in order to update the action. --- .github/workflows/ci.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e20b0f30..faa25340 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -63,9 +63,9 @@ jobs: runner_labels: ["self-hosted", "platform=rocm"] runs-on: ${{ matrix.runner_labels }} steps: - - uses: runs-on/action@v1 + - uses: runs-on/action@v2.1.0 - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Ensure self hosted cache permissions if: ${{ matrix.self_hosted }} @@ -106,14 +106,14 @@ jobs: "extras=s3-cache", ] steps: - - uses: runs-on/action@v1 + - uses: runs-on/action@v2.1.0 - name: Checkout uses: actions/checkout@v6 - name: Setup Zig - uses: mlugg/setup-zig@v2 + uses: mlugg/setup-zig@v2.2.1 with: - version: 0.15.2 + version: 0.16.0 mirror: https://mirror.zml.ai/zig - name: Zig Format From 154a2cb641577f7623f94f5fe5a9c918a45991e2 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 21 Apr 2026 15:06:39 +0200 Subject: [PATCH 446/881] workspace/xla: Use rolledback version of pjrt (#491) Rolling back XLA to versions that work with triton 3.6 --- platforms/cpu/cpu.bzl | 12 ++++++------ platforms/cuda/cuda.bzl | 4 ++-- platforms/rocm/rocm.bzl | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 7360887c..0c22fbab 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -23,22 +23,22 @@ def _cpu_pjrt_plugin_impl(mctx): http_archive( name = "libpjrt_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "867ea7129a61eccb2170fb928cf008577431b1ed5b1ef317766e14deb9660513", - url = "https://github.com/zml/pjrt-artifacts/releases/download/nightly-2026-04-20/pjrt-cpu_linux-amd64.tar.gz", + sha256 = "05da8ee4d19edf327c0bcdc8a66811877bcd5eab75e44f1bbacc6b6b1e9947ca", + url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-04-21T10-00-00Z/pjrt-cpu_linux-amd64.tar.gz", ) http_archive( name = "libpjrt_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "sha256:12515794094962e37b15c2527e4580b8802f83839d13801937191c1ab4b49a2e", - url = "https://github.com/zml/pjrt-artifacts/releases/download/nightly-2026-04-20/pjrt-cpu_darwin-amd64.tar.gz", + sha256 = "sha256:7da35d4b81359f42b56e3b820cacb10c4569caeb4926f2e3b11da913cd8f7c11", + url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-04-21T10-00-00Z/pjrt-cpu_darwin-amd64.tar.gz", ) http_archive( name = "libpjrt_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "e1a648ce12d25ce04aa107b4cfa3aa4c5f675968488779fcbf17b3a98c207da3", - url = "https://github.com/zml/pjrt-artifacts/releases/download/nightly-2026-04-20/pjrt-cpu_darwin-arm64.tar.gz", + sha256 = "sha256:51513b318bd180d33ef88363b5c96fc20ee7ca9db325830ebf694888c6fbc4ec", + url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-04-21T10-00-00Z/pjrt-cpu_darwin-arm64.tar.gz", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 10d4aa65..4bda4e0b 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -277,8 +277,8 @@ def _cuda_impl(mctx): http_archive( name = "libpjrt_cuda", build_file = "libpjrt_cuda.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-04-20T16-00-00Z-2/pjrt-cuda_linux-amd64.tar.gz", - sha256 = "abad33460249b8104c9acf6eb5486569fda9f3a14750f5fcaffff2d6a24ca74f", + url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-04-21T10-00-00Z/pjrt-cuda_linux-amd64.tar.gz", + sha256 = "8e34f4ead657b697e1c670cb35acb562bee9f5ff31948411d1b8ad11416df417", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 185ed519..f8e56e68 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -254,8 +254,8 @@ def _rocm_impl(mctx): http_archive( name = "libpjrt_rocm", build_file = "libpjrt_rocm.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-04-20T16-00-00Z-2/pjrt-rocm_linux-amd64.tar.gz", - sha256 = "50200ad4037d84149b87a5a541fa2a86b87ae159d43a799e92b905d863407449", + url = "https://github.com/zml/pjrt-artifacts/releases/download/nightly-2026-04-17/pjrt-rocm_linux-amd64.tar.gz", + sha256 = "b19db71d3d9aa43bda961a22e81b6a2f48ba2c633a18010c07a66b4ca48ec4d2", ) return mctx.extension_metadata( From fa3811fcd01424a3403a4b36737067d00506544b Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 21 Apr 2026 15:09:21 +0200 Subject: [PATCH 447/881] tokenizer/iree: Fix integer underflow (#492) --- zml/tokenizer/iree/tokenizer.zig | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zml/tokenizer/iree/tokenizer.zig b/zml/tokenizer/iree/tokenizer.zig index c38b214c..52fa5628 100644 --- a/zml/tokenizer/iree/tokenizer.zig +++ b/zml/tokenizer/iree/tokenizer.zig @@ -280,7 +280,7 @@ pub const Tokenizer = struct { )) catch |e| switch (e) { error.ResourceExhausted => { out.advance(produced * @sizeOf(u32)); - min_tokens -= produced; + min_tokens = @max(32, min_tokens -| produced); remaining = remaining[consumed..]; produced = 0; consumed = 0; @@ -295,7 +295,7 @@ pub const Tokenizer = struct { } out.advance(produced * @sizeOf(u32)); - min_tokens -= produced; + min_tokens = @max(32, min_tokens -| produced); remaining = remaining[consumed..]; } } @@ -405,7 +405,7 @@ pub const Tokenizer = struct { } pub fn decode(self: *Decoder, token_ids: []const u32, out: *std.Io.Writer) !void { - var min_output: usize = @max(token_ids.len * 4, 32); + var min_output: usize = @max(token_ids.len * 4, 128); var remaining = token_ids; while (remaining.len > 0) { var consumed: usize = 0; @@ -420,7 +420,7 @@ pub const Tokenizer = struct { )) catch |e| switch (e) { error.ResourceExhausted => { out.advance(produced); - min_output -= produced; + min_output = @max(128, min_output -| produced); remaining = remaining[consumed..]; produced = 0; consumed = 0; @@ -436,7 +436,7 @@ pub const Tokenizer = struct { } out.advance(produced); - min_output -= produced; + min_output = @max(128, min_output -| produced); remaining = remaining[consumed..]; } From 241aaf7d3b6816602feed3b63043d1111688b546 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Wed, 22 Apr 2026 12:37:48 +0200 Subject: [PATCH 448/881] platforms/cuda: add arm64 support (#493) Co-authored-by: Steeve Morin --- MODULE.bazel | 22 +- bin/zml-smi/BUILD.bazel | 12 +- bin/zml-smi/main.zig | 61 +++- bin/zml-smi/platforms/nvml/BUILD.bazel | 6 +- platforms/BUILD.bazel | 12 +- platforms/cuda/BUILD.bazel | 20 +- platforms/cuda/compat_probe.zig | 13 +- platforms/cuda/cuda.bzl | 323 +++++++++++------- platforms/cuda/cuda.zig | 20 +- platforms/cuda/flashattn/BUILD.bazel | 10 +- platforms/cuda/flashattn/flashattn.zig | 15 +- platforms/cuda/libpjrt_cuda.BUILD.bazel | 67 ++-- platforms/cuda/packages.lock.json | 60 ++++ platforms/cuda/packages.yaml | 5 +- platforms/neuron/neuron.bzl | 11 +- platforms/packages.bzl | 6 +- platforms/rocm/rocm.bzl | 4 +- .../flashattn/flashattention.BUILD.bazel | 4 +- third_party/flashattn/repo.bzl | 27 +- zml/attention/triton/BUILD.bazel | 12 +- 20 files changed, 487 insertions(+), 223 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 8e013f6e..26cf0ad1 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -140,20 +140,12 @@ use_repo(pip, "xprof_nightly") pip.parse( download_only = True, hub_name = "triton_py_deps", - extra_pip_args = [ - "--abi=cp312", - "--implementation=cp", - "--python-version=312", - "--platform=linux_x86_64", - "--platform=manylinux2014_x86_64", - "--platform=manylinux_2_27_x86_64", - "--platform=manylinux_2_28_x86_64", - "--platform=manylinux_2_29_x86_64", - "--platform=manylinux_2_37_x86_64", - ], python_version = "3.12", requirements_lock = "//zml/attention/triton:requirements.lock.txt", - target_platforms = ["linux_x86_64"], + target_platforms = [ + "linux_x86_64", + "linux_arm64", + ], ) use_repo(pip, "triton_py_deps") @@ -171,7 +163,7 @@ cpu = use_extension("//platforms/cpu:cpu.bzl", "cpu_pjrt_plugin") use_repo(cpu, "libpjrt_cpu_darwin_amd64", "libpjrt_cpu_darwin_arm64", "libpjrt_cpu_linux_amd64") cuda = use_extension("//platforms/cuda:cuda.bzl", "cuda_packages") -use_repo(cuda, "cuda_nvml_dev", "libpjrt_cuda") +use_repo(cuda, "libpjrt_cuda_linux_amd64", "libpjrt_cuda_linux_arm64", "cuda_nvml_dev_linux_x86_64", "cuda_nvml_dev_linux_sbsa", "zlib1g_linux_arm64", "zlib1g_linux_amd64") rocm = use_extension("//platforms/rocm:rocm.bzl", "rocm_packages") use_repo(rocm, "amd-smi-lib", "hipblaslt", "libdrm-amdgpu-amdgpu1", "libdrm2-amdgpu", "libpjrt_rocm", "rocblas") @@ -180,7 +172,7 @@ tpu = use_extension("//platforms/tpu:tpu.bzl", "tpu_packages") use_repo(tpu, "libpjrt_tpu") neuron = use_extension("//platforms/neuron:neuron.bzl", "neuron_packages") -use_repo(neuron, "aws-neuronx-collectives", "aws-neuronx-runtime-lib", "libgomp1", "libpjrt_neuron", "zlib1g") +use_repo(neuron, "aws-neuronx-collectives", "aws-neuronx-runtime-lib", "libgomp1", "libpjrt_neuron") zls = use_extension("//third_party/zls:zls_toolchain.bzl", "zls_toolchains") use_repo(zls, "zls_toolchains") @@ -188,7 +180,7 @@ use_repo(zls, "zls_toolchains") register_toolchains("@zls_toolchains//:all") non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") -use_repo(non_module_deps, "arocc", "cloud_accelerator_diagnostics", "com_google_sentencepiece", "flashattn", "iree", "libvaxis", "linenoise", "mnist", "org_swig_swig", "stb", "translate-c", "uucode", "xla", "zigimg") +use_repo(non_module_deps, "arocc", "cloud_accelerator_diagnostics", "com_google_sentencepiece", "flashattn_linux_amd64", "flashattn_linux_arm64", "iree", "libvaxis", "linenoise", "mnist", "org_swig_swig", "stb", "translate-c", "uucode", "xla", "zigimg") xla = use_extension("//third_party/xla:xla.bzl", "xla") use_repo( diff --git a/bin/zml-smi/BUILD.bazel b/bin/zml-smi/BUILD.bazel index 2cdff3b5..a1757150 100644 --- a/bin/zml-smi/BUILD.bazel +++ b/bin/zml-smi/BUILD.bazel @@ -41,7 +41,7 @@ copy_to_directory( name = "sandbox", srcs = select({ "@platforms//os:macos": [], - "//conditions:default": [ + "@llvm//platforms/config:linux_x86_64": [ ":libncfw.stripped", ":libnrt.stripped", "//platforms/rocm:zmlxrocm", @@ -50,6 +50,7 @@ copy_to_directory( "@libdrm2-amdgpu//:libdrm2-amdgpu", "@libpjrt_rocm//:amdgpu_ids", ], + "@llvm//platforms/config:linux_aarch64": [], }), include_external_repositories = ["**"], replace_prefixes = { @@ -93,14 +94,19 @@ zig_binary( "@rules_zig//zig/runfiles", ] + select({ "@platforms//os:macos": ["//bin/zml-smi/platforms/macos"], - "//conditions:default": [ - "//bin/zml-smi/platforms/linux", + "@platforms//os:linux": ["//bin/zml-smi/platforms/linux"], + }) + select({ + "@llvm//platforms/config:linux_x86_64": [ "//bin/zml-smi/platforms/amdsmi", "//bin/zml-smi/platforms/neuron", "//bin/zml-smi/platforms/nvml", "//bin/zml-smi/platforms/tpu", "//platforms/neuron:libnrt_headers", ], + "@llvm//platforms/config:linux_aarch64": [ + "//bin/zml-smi/platforms/nvml", + ], + "//conditions:default": [], }), ) diff --git a/bin/zml-smi/main.zig b/bin/zml-smi/main.zig index ccb55786..e7ffcf67 100644 --- a/bin/zml-smi/main.zig +++ b/bin/zml-smi/main.zig @@ -58,12 +58,22 @@ const CliArgs = struct { // \\ --remotes Add devices from remote hosts (comma-separated URLs) }; -const device_backends = if (builtin.os.tag != .macos) .{ - .{ .cuda, @import("zml-smi/platforms/nvml") }, - .{ .rocm, @import("zml-smi/platforms/amdsmi") }, - .{ .neuron, @import("zml-smi/platforms/neuron") }, - .{ .tpu, @import("zml-smi/platforms/tpu") }, -} else .{}; +const device_backends = switch (builtin.os.tag) { + .macos => .{}, + .linux => switch (builtin.cpu.arch) { + .x86_64 => .{ + .{ .cuda, @import("zml-smi/platforms/nvml") }, + .{ .rocm, @import("zml-smi/platforms/amdsmi") }, + .{ .neuron, @import("zml-smi/platforms/neuron") }, + .{ .tpu, @import("zml-smi/platforms/tpu") }, + }, + .aarch64 => .{ + .{ .cuda, @import("zml-smi/platforms/nvml") }, + }, + else => unreachable, + }, + else => unreachable, +}; pub fn main(init: std.process.Init) !void { const gpa = init.gpa; @@ -163,20 +173,37 @@ fn detect(io: std.Io) smi_info.Targets { } var targets: smi_info.Targets = .{}; - if (hasDevice(io, "/dev/nvidiactl")) { - targets.insert(.cuda); - } - if (hasDevice(io, "/dev/kfd")) { - targets.insert(.rocm); - } + switch (builtin.os.tag) { + .macos => {}, + .linux => { + switch (builtin.cpu.arch) { + .x86_64 => { + if (hasDevice(io, "/dev/nvidiactl")) { + targets.insert(.cuda); + } - if (hasDevice(io, c.NEURON_DEVICE_PREFIX ++ "0")) { - targets.insert(.neuron); - } + if (hasDevice(io, "/dev/kfd")) { + targets.insert(.rocm); + } - if (hasDevice(io, "/dev/accel0") or hasDevice(io, "/dev/vfio/0")) { - targets.insert(.tpu); + if (hasDevice(io, c.NEURON_DEVICE_PREFIX ++ "0")) { + targets.insert(.neuron); + } + + if (hasDevice(io, "/dev/accel0") or hasDevice(io, "/dev/vfio/0")) { + targets.insert(.tpu); + } + }, + .aarch64 => { + if (hasDevice(io, "/dev/nvidiactl")) { + targets.insert(.cuda); + } + }, + else => unreachable, + } + }, + else => unreachable, } return targets; diff --git a/bin/zml-smi/platforms/nvml/BUILD.bazel b/bin/zml-smi/platforms/nvml/BUILD.bazel index 5bcb7a62..3a9e54ed 100644 --- a/bin/zml-smi/platforms/nvml/BUILD.bazel +++ b/bin/zml-smi/platforms/nvml/BUILD.bazel @@ -14,6 +14,8 @@ zig_library( "//bin/zml-smi/info", "//bin/zml-smi/utils:double_buffer", "//bin/zml-smi/utils:dynlib", - "@cuda_nvml_dev//:nvml", - ], + ] + select({ + "@llvm//platforms/config:linux_x86_64": ["@cuda_nvml_dev_linux_x86_64//:nvml"], + "@llvm//platforms/config:linux_aarch64": ["@cuda_nvml_dev_linux_sbsa//:nvml"], + }), ) diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index 873a6643..d4c9f450 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -17,13 +17,23 @@ PLATFORMS = { for runtime, default in PLATFORMS.items() ] +config_setting( + name = "rocm.enabled", + flag_values = {":rocm": "True"}, + constraint_values = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], + visibility = ["//visibility:public"], +) + [ config_setting( name = "{}.enabled".format(runtime), flag_values = {":{}".format(runtime): "True"}, visibility = ["//visibility:public"], ) - for runtime in PLATFORMS.keys() + for runtime in ["cpu", "cuda", "tpu", "neuron"] ] zig_library( diff --git a/platforms/cuda/BUILD.bazel b/platforms/cuda/BUILD.bazel index b0fead7e..7db26a1f 100644 --- a/platforms/cuda/BUILD.bazel +++ b/platforms/cuda/BUILD.bazel @@ -4,14 +4,14 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_shared_library", "zig_binar zig_binary( name = "compat_probe", main = "compat_probe.zig", - visibility = ["@libpjrt_cuda//:__subpackages__"], + visibility = ["//visibility:public"], ) zig_shared_library( name = "zmlxcuda", main = "zmlxcuda.zig", shared_lib_name = "libzmlxcuda.so.0", - visibility = ["@libpjrt_cuda//:__subpackages__"], + visibility = ["//visibility:public"], # Use Clang's compiler-rt, but disable stack checking # to avoid requiring on the _zig_probe_stack symbol. zigopts = ["-fno-stack-check"], @@ -24,23 +24,25 @@ cc_library( defines = ["ZML_RUNTIME_CUDA_DISABLED"], ) +cc_library( + name = "libpjrt_cuda_repo", + deps = select({ + "@llvm//platforms/config:linux_x86_64": ["@libpjrt_cuda_linux_amd64//:libpjrt_cuda"], + "@llvm//platforms/config:linux_aarch64": ["@libpjrt_cuda_linux_arm64//:libpjrt_cuda"], + }), +) + cc_library( name = "libpjrt_cuda", tags = ["manual"], hdrs = ["libpjrt_cuda.h"], defines = ["ZML_RUNTIME_CUDA"], - deps = ["@libpjrt_cuda"], + deps = [":libpjrt_cuda_repo"], ) zig_library( name = "cuda", tags = ["manual"], - data = select({ - "//platforms:cuda.enabled": [ - "@flashattn//:libflashattn", - ], - "//conditions:default": [":empty"], - }), srcs = ["compat_probe.zig"], import_name = "platforms/cuda", main = "cuda.zig", diff --git a/platforms/cuda/compat_probe.zig b/platforms/cuda/compat_probe.zig index 06e7e82c..9e962577 100644 --- a/platforms/cuda/compat_probe.zig +++ b/platforms/cuda/compat_probe.zig @@ -3,10 +3,12 @@ const std = @import("std"); pub const ExitCode = enum(u8) { Success = 0, CompatNotSupportedOnDevice = 1, - UnexpectedError = 2, + SystemDriverMismatch = 2, + UnexpectedError = 3, }; const CUDA_SUCCESS = 0; +const CUDA_ERROR_SYSTEM_DRIVER_MISMATCH = 803; const CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE = 804; pub fn main(init: std.process.Init) !u8 { @@ -18,12 +20,19 @@ pub fn main(init: std.process.Init) !u8 { var libcuda = try std.DynLib.open(libcuda_path); const cuInit = libcuda.lookup(*const fn (c_uint) callconv(.c) c_int, "cuInit").?; + const cuGetErrorString = libcuda.lookup(*const fn (c_int, *[*c]const u8) callconv(.c) c_int, "cuGetErrorString").?; return @intFromEnum(switch (cuInit(0)) { CUDA_SUCCESS => ExitCode.Success, + CUDA_ERROR_SYSTEM_DRIVER_MISMATCH => ExitCode.SystemDriverMismatch, CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE => ExitCode.CompatNotSupportedOnDevice, else => |v| blk: { - std.log.err("cuInit returned unexpected error code: {d}", .{v}); + var err_str_ptr: [*c]const u8 = undefined; + if (cuGetErrorString(v, &err_str_ptr) == CUDA_SUCCESS) { + std.log.err("cuInit returned unexpected error code: {d}: {s}", .{ v, err_str_ptr }); + } else { + std.log.err("cuInit returned unexpected error code: {d}", .{v}); + } break :blk ExitCode.UnexpectedError; }, }); diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 4bda4e0b..b13d0878 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -3,15 +3,7 @@ load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") load("//platforms:packages.bzl", "packages") -_BUILD_FILE_DEFAULT_VISIBILITY = """\ -package(default_visibility = ["//visibility:public"]) - -load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_cc//cc:cc_import.bzl", "cc_import") -load("@zml//bazel:patchelf.bzl", "patchelf") -""" - -ARCH = "linux-x86_64" +ARCHS = ["linux-x86_64", "linux-sbsa"] CUDA_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cuda/redist/" CUDA_VERSION = "13.1.1" @@ -26,6 +18,14 @@ NVSHMEM_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/nvshmem/r NVSHMEM_VERSION = "3.5.19" NVSHMEM_REDIST_JSON_SHA256 = "6dced4193eb728542504b346cfb768da6e3de2abca0cded95fda3a69729994d2" +_BUILD_FILE_DEFAULT_VISIBILITY = """\ +package(default_visibility = ["//visibility:public"]) + +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_cc//cc:cc_import.bzl", "cc_import") +load("@zml//bazel:patchelf.bzl", "patchelf") +""" + CUDA_COMPAT_FILES = [ "libcuda.so.1", "libcudadebugger.so.1", @@ -34,20 +34,16 @@ CUDA_COMPAT_FILES = [ "libnvidia-ptxjitcompiler.so.1", ] -_UBUNTU_PACKAGES = { - "zlib1g": packages.filegroup(name = "zlib1g", srcs = ["lib/x86_64-linux-gnu/libz.so.1"]), -} - CUDA_PACKAGES = { - "cuda_nvml_dev": "\n".join([ + "cuda_nvml_dev": [ packages.cc_library( name = "nvml", hdrs = ["include/nvml.h"], includes = ["include"], visibility = ["//visibility:public"], ), - ]), - "cuda_cudart": "\n".join([ + ], + "cuda_cudart": [ # Driver API only packages.cc_library( name = "cuda", @@ -59,18 +55,20 @@ CUDA_PACKAGES = { name = "cuda_cudart", srcs = ["lib/libcudart.so.13"], ), - ]), - "cuda_cupti": packages.filegroup( - name = "cuda_cupti", - srcs = ["lib/libcupti.so.13"], - ), - "cuda_nvtx": "\n".join([ + ], + "cuda_cupti": [ + packages.filegroup( + name = "cuda_cupti", + srcs = ["lib/libcupti.so.13"], + ), + ], + "cuda_nvtx": [ packages.filegroup( name = "cuda_nvtx", srcs = ["lib/libnvtx3interop.so"], ), - ]), - "cuda_compat": "\n".join([ + ], + "cuda_compat": [ packages.patchelf( name = "{}.patchelf".format(file), set_rpath = "$ORIGIN", @@ -79,35 +77,45 @@ CUDA_PACKAGES = { ) for file in CUDA_COMPAT_FILES ] + [ - packages.filegroup( + """filegroup( name = "cuda_compat", srcs = [ "compat/libnvidia-gpucomp.so.590.48.01", - "compat/libnvidia-pkcs11-openssl3.so.590.48.01", "compat/libnvidia-tileiras.so.590.48.01", - ] + [ - ":{}.patchelf".format(file) - for file in CUDA_COMPAT_FILES - ], + ] + {} + select({{ + "@llvm//platforms/config:linux_x86_64": ["compat/libnvidia-pkcs11-openssl3.so.590.48.01"], + "@llvm//platforms/config:linux_aarch64": [], + }}), + )""".format(repr([ + ":{}.patchelf".format(file) + for file in CUDA_COMPAT_FILES + ])), + ], + "libcufft": [ + packages.filegroup( + name = "libcufft", + srcs = ["lib/libcufft.so.12"], ), - ]), - "libcufft": packages.filegroup( - name = "libcufft", - srcs = ["lib/libcufft.so.12"], - ), - "libcusolver": packages.filegroup( - name = "libcusolver", - srcs = ["lib/libcusolver.so.12"], - ), - "libcusparse": packages.filegroup( - name = "libcusparse", - srcs = ["lib/libcusparse.so.12"], - ), - "libnvjitlink": packages.filegroup( - name = "libnvjitlink", - srcs = ["lib/libnvJitLink.so.13"], - ), - "cuda_nvcc": "\n".join([ + ], + "libcusolver": [ + packages.filegroup( + name = "libcusolver", + srcs = ["lib/libcusolver.so.12"], + ), + ], + "libcusparse": [ + packages.filegroup( + name = "libcusparse", + srcs = ["lib/libcusparse.so.12"], + ), + ], + "libnvjitlink": [ + packages.filegroup( + name = "libnvjitlink", + srcs = ["lib/libnvJitLink.so.13"], + ), + ], + "cuda_nvcc": [ packages.filegroup( name = "cuda_nvcc", srcs = [ @@ -119,8 +127,8 @@ CUDA_PACKAGES = { name = "nvptxcompiler", static_library = "lib/libnvptxcompiler_static.a", ), - ]), - "libnvvm": "\n".join([ + ], + "libnvvm": [ packages.filegroup( name = "libnvvm", srcs = [ @@ -128,8 +136,8 @@ CUDA_PACKAGES = { "nvvm/libdevice/libdevice.10.bc", ], ), - ]), - "cuda_nvrtc": "\n".join([ + ], + "cuda_nvrtc": [ packages.filegroup( name = "cuda_nvrtc", srcs = [ @@ -137,8 +145,8 @@ CUDA_PACKAGES = { "lib/libnvrtc-builtins.so.13.1", ], ), - ]), - "libcublas": "\n".join([ + ], + "libcublas": [ packages.filegroup( name = "libcublas", srcs = [ @@ -146,11 +154,11 @@ CUDA_PACKAGES = { "lib/libcublas.so.13", ], ), - ]), + ], } CUDNN_PACKAGES = { - "cudnn": "\n".join([ + "cudnn": [ packages.filegroup( name = "cudnn", srcs = [ @@ -164,20 +172,64 @@ CUDNN_PACKAGES = { "lib/libcudnn_heuristic.so.9", ], ), - ]), + ], } NVSHMEM_PACKAGES = { - "libnvshmem": packages.filegroup( - name = "libnvshmem", - srcs = [ - "lib/libnvshmem_host.so.3", - "lib/nvshmem_bootstrap_uid.so.3", - "lib/nvshmem_transport_ibrc.so.4", - ], - ), + "libnvshmem": [ + packages.filegroup( + name = "libnvshmem", + srcs = [ + "lib/libnvshmem_host.so.3", + "lib/nvshmem_bootstrap_uid.so.3", + "lib/nvshmem_transport_ibrc.so.4", + ], + ), + ], } +_UBUNTU_PACKAGES = { + "zlib1g": [ + """filegroup( + name = "zlib1g", + srcs = select({ + "@llvm//platforms/config:linux_x86_64": ["lib/x86_64-linux-gnu/libz.so.1"], + "@llvm//platforms/config:linux_aarch64": ["lib/aarch64-linux-gnu/libz.so.1"], + }), + )""", + ], +} + +PJRT_CUDA_RELEASE = "manual-2026-04-21T10-00-00Z" + +_PJRT_CUDA_ASSETS = { + "amd64": { + "sha256": "8e34f4ead657b697e1c670cb35acb562bee9f5ff31948411d1b8ad11416df417", + "url": "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-cuda_linux-amd64.tar.gz", + }, + "arm64": { + "sha256": "54da63a8342f848e42fb26ea1f854db5211b55f9abc753e93ad06ae2964bcb44", + "url": "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-cuda_linux-arm64.tar.gz", + }, +} + +_NCCL_ASSETS = { + "amd64": { + "url": "https://pypi.nvidia.com/nvidia-nccl-cu13/nvidia_nccl_cu13-2.29.3-py3-none-manylinux_2_18_x86_64.whl", + "sha256": "2a321629f49490e4e0122ecb578a4b4a6f89e72740dd988e04dfa4758fab7fc3", + }, + "arm64": { + "url": "https://pypi.nvidia.com/nvidia-nccl-cu13/nvidia_nccl_cu13-2.29.3-py3-none-manylinux_2_18_aarch64.whl", + "sha256": "eab9f5c565ab3326906f1d1b5be5773a174c2a1b47002faed76f9e957392f713", + }, +} + +def _repo_suffix(arch): + return "linux_{}".format(arch) + +def _repo_name(name, arch): + return "{}_{}".format(name, _repo_suffix(arch)) + def _read_redist_json(mctx, url, sha256): fname = ".{}.json".format(sha256) mctx.download( @@ -187,10 +239,12 @@ def _read_redist_json(mctx, url, sha256): ) return json.decode(mctx.read(fname)) + def _cuda_impl(mctx): loaded_packages = packages.read(mctx, [ "@zml//platforms/cuda:packages.lock.json", ]) + CUDA_REDIST = _read_redist_json( mctx, url = CUDA_REDIST_PREFIX + "redistrib_{}.json".format(CUDA_VERSION), @@ -209,81 +263,98 @@ def _cuda_impl(mctx): sha256 = CUDNN_REDIST_JSON_SHA256, ) + #TODO(cerisier): for each architecture for pkg_name, build_file_content in _UBUNTU_PACKAGES.items(): - pkg = loaded_packages[pkg_name] - http_deb_archive( - name = pkg_name, - urls = pkg["urls"], - sha256 = pkg["sha256"], - build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, - ) + pkgs = loaded_packages[pkg_name] + for arch, pkg in pkgs.items(): + http_deb_archive( + name = _repo_name(pkg_name, arch), + urls = pkg["urls"], + sha256 = pkg["sha256"], + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + "\n".join(build_file_content), + ) for pkg, build_file_content in CUDA_PACKAGES.items(): pkg_data = CUDA_REDIST[pkg] - arch_data = pkg_data.get(ARCH) - if not arch_data: - continue - arch_data = arch_data.get(CUDA_VARIANT, None) or arch_data - http_archive( - name = pkg, - build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, - url = CUDA_REDIST_PREFIX + arch_data["relative_path"], - sha256 = arch_data["sha256"], - strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), - ) + for arch in ARCHS: + pkg_repo_name = pkg + "_" + arch.replace("-", "_") + arch_data = pkg_data.get(arch) + if not arch_data: + fail("CUDA redist package {} does not have data for architecture {}".format(pkg, arch)) + arch_data = arch_data.get(CUDA_VARIANT, None) or arch_data + http_archive( + name = pkg_repo_name, + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + "\n".join(build_file_content), + url = CUDA_REDIST_PREFIX + arch_data["relative_path"], + sha256 = arch_data["sha256"], + strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), + ) for pkg, build_file_content in CUDNN_PACKAGES.items(): - pkg_data = CUDNN_REDIST[pkg] - arch_data = pkg_data.get(ARCH) - if not arch_data: - continue - arch_data = arch_data.get("cuda13", arch_data) - http_archive( - name = pkg, - build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, - url = CUDNN_REDIST_PREFIX + arch_data["relative_path"], - sha256 = arch_data["sha256"], - strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), - ) + for arch in ARCHS: + pkg_repo_name = pkg + "_" + arch.replace("-", "_") + pkg_data = CUDNN_REDIST[pkg] + arch_data = pkg_data.get(arch) + if not arch_data: + fail("CUDA redist package {} does not have data for architecture {}".format(pkg, arch)) + arch_data = arch_data.get("cuda13", arch_data) + http_archive( + name = pkg_repo_name, + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + "\n".join(build_file_content), + url = CUDNN_REDIST_PREFIX + arch_data["relative_path"], + sha256 = arch_data["sha256"], + strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), + ) for pkg, build_file_content in NVSHMEM_PACKAGES.items(): - pkg_data = NVSHMEM_REDIST[pkg] - arch_data = pkg_data.get(ARCH) - if not arch_data: - continue - arch_data = arch_data.get("cuda13", arch_data) + for arch in ARCHS: + pkg_repo_name = pkg + "_" + arch.replace("-", "_") + pkg_data = NVSHMEM_REDIST[pkg] + arch_data = pkg_data.get(arch) + if not arch_data: + fail("CUDA redist package {} does not have data for architecture {}".format(pkg, arch)) + arch_data = arch_data.get("cuda13", arch_data) + http_archive( + name = pkg_repo_name, + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + "\n".join(build_file_content), + url = NVSHMEM_REDIST_PREFIX + arch_data["relative_path"], + sha256 = arch_data["sha256"], + strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), + ) + + for arch, arch_config in _NCCL_ASSETS.items(): http_archive( - name = pkg, - build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, - url = NVSHMEM_REDIST_PREFIX + arch_data["relative_path"], - sha256 = arch_data["sha256"], - strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), + name = _repo_name("nccl", arch), + urls = [arch_config["url"]], + type = "zip", + sha256 = arch_config["sha256"], + build_file_content = "\n".join([ + _BUILD_FILE_DEFAULT_VISIBILITY, + packages.filegroup( + name = "nccl", + srcs = ["nvidia/nccl/lib/libnccl.so.2"], + ), + ]), ) - http_archive( - name = "nccl", - urls = ["https://pypi.nvidia.com/nvidia-nccl-cu13/nvidia_nccl_cu13-2.29.3-py3-none-manylinux_2_18_x86_64.whl"], - type = "zip", - sha256 = "2a321629f49490e4e0122ecb578a4b4a6f89e72740dd988e04dfa4758fab7fc3", - build_file_content = "\n".join([ - _BUILD_FILE_DEFAULT_VISIBILITY, - packages.filegroup( - name = "nccl", - srcs = ["nvidia/nccl/lib/libnccl.so.2"], - ), - ]), - ) - - http_archive( - name = "libpjrt_cuda", - build_file = "libpjrt_cuda.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-04-21T10-00-00Z/pjrt-cuda_linux-amd64.tar.gz", - sha256 = "8e34f4ead657b697e1c670cb35acb562bee9f5ff31948411d1b8ad11416df417", - ) + for arch, arch_config in _PJRT_CUDA_ASSETS.items(): + http_archive( + name = _repo_name("libpjrt_cuda", arch), + build_file = "libpjrt_cuda.BUILD.bazel", + url = arch_config["url"].format(release = PJRT_CUDA_RELEASE), + sha256 = arch_config["sha256"], + ) return mctx.extension_metadata( reproducible = True, - root_module_direct_deps = ["cuda_nvml_dev", "libpjrt_cuda"], + root_module_direct_deps = [ + "libpjrt_cuda_linux_amd64", + "libpjrt_cuda_linux_arm64", + "cuda_nvml_dev_linux_x86_64", + "cuda_nvml_dev_linux_sbsa", + "zlib1g_linux_arm64", + "zlib1g_linux_amd64", + ], root_module_direct_dev_deps = [], ) diff --git a/platforms/cuda/cuda.zig b/platforms/cuda/cuda.zig index 88e5f1be..2a711a64 100644 --- a/platforms/cuda/cuda.zig +++ b/platforms/cuda/cuda.zig @@ -13,6 +13,18 @@ const nvidiaLibsPath = "/cuda/"; const log = std.log.scoped(.@"zml/platforms/cuda"); +fn findCudaSandbox( + r: anytype, + buffer: *[std.Io.Dir.max_path_bytes]u8, +) !?[]const u8 { + const candidate = switch (builtin.cpu.arch) { + .aarch64 => "libpjrt_cuda_linux_arm64/sandbox", + .x86_64 => "libpjrt_cuda_linux_amd64/sandbox", + else => return null, + }; + return try r.rlocation(candidate, buffer); +} + pub fn isEnabled() bool { return @hasDecl(c, "ZML_RUNTIME_CUDA"); } @@ -25,8 +37,8 @@ pub fn needsCudaCompat(io: std.Io, sandbox_path: []const u8) !bool { .argv = &[_][]const u8{nvidia_compat_path}, .cwd = .{ .path = sandbox_path }, .stdin = .ignore, - .stdout = .ignore, - .stderr = .ignore, + .stdout = .inherit, + .stderr = .inherit, }); defer child.kill(io); @@ -38,7 +50,7 @@ pub fn needsCudaCompat(io: std.Io, sandbox_path: []const u8) !bool { return switch (result) { .Success => true, - .CompatNotSupportedOnDevice => false, + .SystemDriverMismatch, .CompatNotSupportedOnDevice => false, .UnexpectedError => blk: { log.err("CUDA compatibility probe returned unexpected error code", .{}); break :blk false; @@ -85,7 +97,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { const r = try bazel.runfiles(bazel_builtin.current_repository); var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = try r.rlocation("libpjrt_cuda/sandbox", &path_buf) orelse { + const sandbox_path = try findCudaSandbox(r, &path_buf) orelse { log.err("Failed to find sandbox path for CUDA runtime", .{}); return error.FileNotFound; }; diff --git a/platforms/cuda/flashattn/BUILD.bazel b/platforms/cuda/flashattn/BUILD.bazel index 31fab169..34bf0b14 100644 --- a/platforms/cuda/flashattn/BUILD.bazel +++ b/platforms/cuda/flashattn/BUILD.bazel @@ -8,7 +8,13 @@ zig_library( visibility = ["//visibility:public"], deps = [ "//bazel", - "@flashattn//:c", "@rules_zig//zig/runfiles", - ], + ] + select({ + "@llvm//platforms/config:linux_x86_64": ["@flashattn_linux_amd64//:c"], + "@llvm//platforms/config:linux_aarch64": ["@flashattn_linux_arm64//:c"], + }), + data = select({ + "@llvm//platforms/config:linux_x86_64": ["@flashattn_linux_amd64//:libflashattn"], + "@llvm//platforms/config:linux_aarch64": ["@flashattn_linux_arm64//:libflashattn"], + }), ) diff --git a/platforms/cuda/flashattn/flashattn.zig b/platforms/cuda/flashattn/flashattn.zig index 368d0ffb..b4014c25 100644 --- a/platforms/cuda/flashattn/flashattn.zig +++ b/platforms/cuda/flashattn/flashattn.zig @@ -2,6 +2,7 @@ const std = @import("std"); const bazel = @import("bazel"); const bazel_builtin = @import("bazel_builtin"); +const builtin = @import("builtin"); const c = @import("c"); pub const FA2MhaFwdParams = c.FA2MhaFwdParams; pub const FA2MhaVarlenFwdParams = c.FA2MhaVarlenFwdParams; @@ -93,6 +94,18 @@ pub var fa2_mha_fwd: Fa2MhaFwdFunc = undefined; pub var fa2_mha_varlen_fwd: Fa2MhaVarlenFwdFunc = undefined; pub var fa3_mha_fwd: Fa3MhaFwdFunc = undefined; +fn findFlashattnLibrary( + r: anytype, + buffer: *[std.Io.Dir.max_path_bytes]u8, +) !?[]const u8 { + const candidate = switch (builtin.cpu.arch) { + .aarch64 => "flashattn_linux_arm64/lib/libflashattn.so", + .x86_64 => "flashattn_linux_amd64/lib/libflashattn.so", + else => return null, + }; + return try r.rlocation(candidate, buffer); +} + pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { _ = io; _ = allocator; @@ -100,7 +113,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { const r = try bazel.runfiles(bazel_builtin.current_repository); var buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; - const library = (try r.rlocation("flashattn/flashattn/lib/libflashattn.so", &buffer)) orelse return error.NotFound; + const library = (try findFlashattnLibrary(r, &buffer)) orelse return error.NotFound; var lib = std.DynLib.open(library) catch |err| { std.log.err("Failed to open libflashattn.so: {any}", .{err}); return err; diff --git a/platforms/cuda/libpjrt_cuda.BUILD.bazel b/platforms/cuda/libpjrt_cuda.BUILD.bazel index 0cd7f7b6..bf4e545a 100644 --- a/platforms/cuda/libpjrt_cuda.BUILD.bazel +++ b/platforms/cuda/libpjrt_cuda.BUILD.bazel @@ -23,31 +23,53 @@ copy_to_directory( name = "sandbox", srcs = [ ":libpjrt_cuda_so", - "@cuda_cudart", - "@cuda_cupti", - "@cuda_compat", - "@cuda_nvcc", - "@libnvvm", - "@cuda_nvrtc", - "@cuda_nvtx", - "@cudnn", - "@libnvshmem", - "@libcublas", - "@libcufft", - "@libcusolver", - "@libcusparse", - "@libnvjitlink", - "@nccl", - "@zlib1g", "@zml//platforms/cuda:compat_probe", "@zml//platforms/cuda:zmlxcuda", - ], + ] + select({ + "@llvm//platforms/config:linux_x86_64": [ + "@cuda_cudart_linux_x86_64//:cuda_cudart", + "@cuda_cupti_linux_x86_64//:cuda_cupti", + "@cuda_compat_linux_x86_64//:cuda_compat", + "@cuda_nvcc_linux_x86_64//:cuda_nvcc", + "@libnvvm_linux_x86_64//:libnvvm", + "@cuda_nvrtc_linux_x86_64//:cuda_nvrtc", + "@cuda_nvtx_linux_x86_64//:cuda_nvtx", + "@cudnn_linux_x86_64//:cudnn", + "@libnvshmem_linux_x86_64//:libnvshmem", + "@libcublas_linux_x86_64//:libcublas", + "@libcufft_linux_x86_64//:libcufft", + "@libcusolver_linux_x86_64//:libcusolver", + "@libcusparse_linux_x86_64//:libcusparse", + "@libnvjitlink_linux_x86_64//:libnvjitlink", + "@nccl_linux_amd64//:nccl", + "@zlib1g_linux_amd64//:zlib1g", + ], + "@llvm//platforms/config:linux_aarch64": [ + "@cuda_cudart_linux_sbsa//:cuda_cudart", + "@cuda_cupti_linux_sbsa//:cuda_cupti", + "@cuda_compat_linux_sbsa//:cuda_compat", + "@cuda_nvcc_linux_sbsa//:cuda_nvcc", + "@libnvvm_linux_sbsa//:libnvvm", + "@cuda_nvrtc_linux_sbsa//:cuda_nvrtc", + "@cuda_nvtx_linux_sbsa//:cuda_nvtx", + "@cudnn_linux_sbsa//:cudnn", + "@libnvshmem_linux_sbsa//:libnvshmem", + "@libcublas_linux_sbsa//:libcublas", + "@libcufft_linux_sbsa//:libcufft", + "@libcusolver_linux_sbsa//:libcusolver", + "@libcusparse_linux_sbsa//:libcusparse", + "@libnvjitlink_linux_sbsa//:libnvjitlink", + "@nccl_linux_arm64//:nccl", + "@zlib1g_linux_arm64//:zlib1g", + ], + }), replace_prefixes = { "compat": "lib/compat", "nvidia/nccl/lib": "lib", "nvvm/lib64": "lib", "libpjrt_cuda_so": "lib", "lib/x86_64-linux-gnu": "lib", + "lib/aarch64-linux-gnu": "lib", "platforms/cuda/compat_probe": "bin/compat_probe", "platforms/cuda/libzmlxcuda": "lib/libzmlxcuda", } | { @@ -61,9 +83,14 @@ copy_to_directory( cc_library( name = "libpjrt_cuda", data = [":sandbox"], - deps = [ - "@cuda_cudart//:cuda", - ], + deps = select({ + "@llvm//platforms/config:linux_x86_64": [ + "@cuda_cudart_linux_x86_64//:cuda", + ], + "@llvm//platforms/config:linux_aarch64": [ + "@cuda_cudart_linux_sbsa//:cuda", + ], + }), linkopts = [ # Defer function call resolution until the function is called # (lazy loading) rather than at load time. diff --git a/platforms/cuda/packages.lock.json b/platforms/cuda/packages.lock.json index b80a2c36..1c39a387 100755 --- a/platforms/cuda/packages.lock.json +++ b/platforms/cuda/packages.lock.json @@ -59,6 +59,66 @@ "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/gcc-12-base_12.2.0-14+deb12u1_amd64.deb" ], "version": "12.2.0-14+deb12u1" + }, + { + "arch": "arm64", + "dependencies": [ + { + "key": "libc6_2.36-9-p-deb12u10_arm64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_arm64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_arm64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + } + ], + "key": "zlib1g_1-1.2.13.dfsg-1_arm64", + "name": "zlib1g", + "sha256": "52b8b8a145bbe1956bba82034f77022cbef0c3d0885c9e32d9817a7932fe1913", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_arm64.deb" + ], + "version": "1:1.2.13.dfsg-1" + }, + { + "arch": "arm64", + "dependencies": [], + "key": "libc6_2.36-9-p-deb12u10_arm64", + "name": "libc6", + "sha256": "9b25c7c2aaa9328b2cfd92b47cc48b24631bfdff0c61afe05cba937911fcf239", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/glibc/libc6_2.36-9+deb12u10_arm64.deb" + ], + "version": "2.36-9+deb12u10" + }, + { + "arch": "arm64", + "dependencies": [], + "key": "libgcc-s1_12.2.0-14-p-deb12u1_arm64", + "name": "libgcc-s1", + "sha256": "576926b283613db80168ddf76380a3bd877602778cf0d226caa7bfbfa71eacf3", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/libgcc-s1_12.2.0-14+deb12u1_arm64.deb" + ], + "version": "12.2.0-14+deb12u1" + }, + { + "arch": "arm64", + "dependencies": [], + "key": "gcc-12-base_12.2.0-14-p-deb12u1_arm64", + "name": "gcc-12-base", + "sha256": "674cf6cba6d432bd200c45fe866c1652c7a53523cc2e7a613e05bc4abf7b5440", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/gcc-12-base_12.2.0-14+deb12u1_arm64.deb" + ], + "version": "12.2.0-14+deb12u1" } ], "version": 1 diff --git a/platforms/cuda/packages.yaml b/platforms/cuda/packages.yaml index 4b3b04c0..aaa3c42f 100644 --- a/platforms/cuda/packages.yaml +++ b/platforms/cuda/packages.yaml @@ -8,7 +8,8 @@ sources: url: https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z archs: - - "amd64" + - "amd64" + - "arm64" packages: - - "zlib1g" + - "zlib1g" diff --git a/platforms/neuron/neuron.bzl b/platforms/neuron/neuron.bzl index 1c3f1f91..8814c246 100644 --- a/platforms/neuron/neuron.bzl +++ b/platforms/neuron/neuron.bzl @@ -70,7 +70,7 @@ def _neuron_impl(mctx): ) for pkg_name, build_file_content in _UBUNTU_PACKAGES.items(): - pkg = loaded_packages[pkg_name] + pkg = loaded_packages[pkg_name]["amd64"] http_deb_archive( name = pkg_name, urls = pkg["urls"], @@ -79,7 +79,7 @@ def _neuron_impl(mctx): ) for pkg_name, build_file_content in _NEURON_PACKAGES.items(): - pkg = loaded_packages[pkg_name] + pkg = loaded_packages[pkg_name]["amd64"] http_deb_archive( name = pkg_name, urls = pkg["urls"], @@ -90,7 +90,12 @@ def _neuron_impl(mctx): return mctx.extension_metadata( reproducible = True, - root_module_direct_deps = "all", + root_module_direct_deps = [ + "aws-neuronx-collectives", + "aws-neuronx-runtime-lib", + "libgomp1", + "libpjrt_neuron", + ], root_module_direct_dev_deps = [], ) diff --git a/platforms/packages.bzl b/platforms/packages.bzl index 633f98b1..6d3117cd 100644 --- a/platforms/packages.bzl +++ b/platforms/packages.bzl @@ -45,10 +45,8 @@ def _read(mctx, labels): ret = {} for label in labels: data = json.decode(mctx.read(Label(label))) - ret.update({ - pkg["name"]: pkg - for pkg in data["packages"] - }) + for pkg in data["packages"]: + ret.setdefault(pkg["name"], {})[pkg["arch"]] = pkg return ret packages = struct( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index f8e56e68..8ab97767 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -226,7 +226,7 @@ def _rocm_impl(mctx): ]) for pkg_name, build_file_content in _UBUNTU_PACKAGES.items(): - pkg = loaded_packages[pkg_name] + pkg = loaded_packages[pkg_name]["amd64"] http_deb_archive( name = pkg_name, urls = pkg["urls"], @@ -235,7 +235,7 @@ def _rocm_impl(mctx): ) for pkg_name, build_file_content in _ROCM_PACKAGES.items(): - pkg = loaded_packages[pkg_name] + pkg = loaded_packages[pkg_name]["amd64"] http_deb_archive( name = pkg_name, urls = pkg["urls"], diff --git a/third_party/flashattn/flashattention.BUILD.bazel b/third_party/flashattn/flashattention.BUILD.bazel index 3f32c61c..030e3d3f 100644 --- a/third_party/flashattn/flashattention.BUILD.bazel +++ b/third_party/flashattn/flashattention.BUILD.bazel @@ -3,11 +3,11 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") filegroup( name = "libflashattn", visibility = ["//visibility:public"], - srcs = ["flashattn/lib/libflashattn.so"], + srcs = ["lib/libflashattn.so"], ) cc_library( name = "c", - hdrs = ["flashattn/include/capi.h"], + hdrs = ["include/capi.h"], visibility = ["//visibility:public"], ) diff --git a/third_party/flashattn/repo.bzl b/third_party/flashattn/repo.bzl index 4fabc298..bcad73fe 100644 --- a/third_party/flashattn/repo.bzl +++ b/third_party/flashattn/repo.bzl @@ -1,13 +1,30 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") -def repo(): +_FLASHATTN_RELEASE = "v0.0.6-rc9" + +_FLASHATTN_ASSETS = { + "amd64": { + "sha256": "cb341c14ee0fc5f02a262497cbc80b552c23bce594d087a2aa278f14cfc82ef2", + "url": "https://github.com/zml/flash-attention/releases/download/{release}/flash-attention_linux-amd64.tar.gz", + }, + "arm64": { + "sha256": "55feeccb4b3728dec08da2aa051109d0b9fbd8c0d0786c878976c1b713e3b407", + "url": "https://github.com/zml/flash-attention/releases/download/{release}/flash-attention_linux-arm64.tar.gz", + }, +} + +def flasshattn_archive(name, arch): + asset = _FLASHATTN_ASSETS[arch] http_archive( - name = "flashattn", - url = "https://github.com/zml/flash-attention/releases/download/v0.0.6-rc8/flashattn.tar.gz", - sha256 = "360caace18b84f03779fe7af8aa1cdff64e99b6c17cbe05774be100aaf23b579", + name = name, + url = asset["url"].format(release = _FLASHATTN_RELEASE), + sha256 = asset["sha256"], build_file = "//:third_party/flashattn/flashattention.BUILD.bazel", ) - #new_local_repository(name = "flashattn", build_file="//:third_party/flashattn/BUILD.bazel", path="/home/corendos/flashattn/") +def repo(): + flasshattn_archive("flashattn_linux_amd64", "amd64") + flasshattn_archive("flashattn_linux_arm64", "arm64") + #new_local_repository(name = "flashattn", build_file="//:third_party/flashattn/BUILD.bazel", path="/home/corendos/flashattn/") diff --git a/zml/attention/triton/BUILD.bazel b/zml/attention/triton/BUILD.bazel index a1e13506..10e0894e 100644 --- a/zml/attention/triton/BUILD.bazel +++ b/zml/attention/triton/BUILD.bazel @@ -14,7 +14,6 @@ uv_lock( "--emit-find-links", "--index-strategy=unsafe-best-match", "--upgrade", - "--python-platform=x86_64-manylinux_2_37", ], python_version = "3.12", tags = ["manual"], @@ -87,8 +86,14 @@ copy_to_directory( name = "sandbox", srcs = [ ":generate_files", - "@zlib1g", - ], + ] + select({ + "@llvm//platforms/config:linux_x86_64": [ + "@zlib1g_linux_amd64//:zlib1g", + ], + "@llvm//platforms/config:linux_aarch64": [ + "@zlib1g_linux_arm64//:zlib1g", + ], + }), target_compatible_with = ["@platforms//os:linux"], add_directory_to_runfiles = False, allow_overwrites = True, @@ -113,6 +118,7 @@ copy_to_directory( "generate.py": "site-packages/generate.py", "generate": "bin/generate", "lib/x86_64-linux-gnu": "lib", + "lib/aarch64-linux-gnu": "lib", }, visibility = ["//visibility:public"], ) From 83ebe37e93ff4246595607781723a7dcbd53308d Mon Sep 17 00:00:00 2001 From: Imran Alam Date: Wed, 22 Apr 2026 20:39:14 +1000 Subject: [PATCH 449/881] bin/zml-smi: Fix zml logo rendering in KDE Konsole (#494) Specifying both `rows` and `cols` makes the zml logo in `zml-smi` render correctly in KDE Konsole. Verified that the fix didn't cause any regression in a terminal emulator where logo rendering was already functional (`Ghostty`). `BUILD.bazel` is also updated to run the `zml-smi` tests. **Before** image **With Fix** image **Test Discovery Fix** ``` master $ bazel query 'kind(zig_test, //bin/zml-smi/...)' INFO: Empty results master $ git checkout - Switched to branch 'fix/zml-smi-logo-size' fix/zml-smi-logo-size $ bazel query 'kind(zig_test, //bin/zml-smi/...)' //bin/zml-smi/tui:test ``` --- bin/zml-smi/tui/BUILD.bazel | 7 ++++++- bin/zml-smi/tui/lib/image.zig | 8 +++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/bin/zml-smi/tui/BUILD.bazel b/bin/zml-smi/tui/BUILD.bazel index 8b75b4c0..7602cde4 100644 --- a/bin/zml-smi/tui/BUILD.bazel +++ b/bin/zml-smi/tui/BUILD.bazel @@ -1,4 +1,4 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "tui", @@ -50,3 +50,8 @@ zig_library( "//conditions:default": ["//bin/zml-smi/platforms/linux"], }), ) + +zig_test( + name = "test", + deps = [":tui"], +) diff --git a/bin/zml-smi/tui/lib/image.zig b/bin/zml-smi/tui/lib/image.zig index 4fce069d..a365ac4d 100644 --- a/bin/zml-smi/tui/lib/image.zig +++ b/bin/zml-smi/tui/lib/image.zig @@ -38,7 +38,9 @@ pub fn draw(self: *const Image, ctx: vxfw.DrawContext) Allocator.Error!vxfw.Surf .image = .{ .img_id = self.image.id, .options = .{ - .size = .{ .rows = target_rows }, + // Emit both rows and cols so that target bounds are unambiguous. + // Rows alone lets some terminals fall back to native pixel size. + .size = .{ .rows = target_rows, .cols = cols }, }, }, }); @@ -71,6 +73,10 @@ test Image { const cell = surface.buffer[0]; try std.testing.expect(cell.image != null); try std.testing.expectEqual(@as(u32, 1), cell.image.?.img_id); + // Both rows and cols must be emitted so terminals don't fall back to native pixel size. + const size = cell.image.?.options.size.?; + try std.testing.expectEqual(@as(?u16, 5), size.rows); + try std.testing.expectEqual(@as(?u16, 20), size.cols); } test "refAllDecls" { From f80f925409990ee4fc30b55a5bb17e9152e85557 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 22 Apr 2026 16:23:36 +0200 Subject: [PATCH 450/881] examples/llm: add profiling option (#497) This allow use to easily trace what going on --- examples/llm/chat.zig | 33 +++++++++++++++++++++++++++++++-- examples/llm/main.zig | 8 ++++++-- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/examples/llm/chat.zig b/examples/llm/chat.zig index 6d1d311a..a319cf7a 100644 --- a/examples/llm/chat.zig +++ b/examples/llm/chat.zig @@ -25,6 +25,7 @@ pub fn rememberPrompt(line: [*:0]const u8) void { pub const Chat = struct { allocator: std.mem.Allocator, io: std.Io, + platform: *const zml.Platform, session: models.Session, tokens: std.ArrayList(u32), @@ -48,6 +49,7 @@ pub const Chat = struct { return .{ .allocator = allocator, .io = io, + .platform = platform, .session = session, .tokens = try .initCapacity(allocator, session.maxTokens()), }; @@ -58,16 +60,42 @@ pub const Chat = struct { self.session.deinit(); } - pub fn runOnce(self: *Chat, prompt: []const u8) !void { + pub const RunOnceOptions = struct { + profile: bool = false, + }; + + pub fn runOnce(self: *Chat, prompt: []const u8, opts: RunOnceOptions) !void { const prompt_tokens = try self.session.tokenizePrompt(self.allocator, prompt); defer self.allocator.free(prompt_tokens); var stdout = std.Io.File.stdout().writer(self.io, &.{}); + var profiler: ?zml.Platform.Profiler = null; + defer if (profiler) |*p| p.deinit(); + + if (opts.profile) { + var session_id_buf: [64]u8 = undefined; + const session_id = try std.fmt.bufPrint( + &session_id_buf, + "llm-runonce-{d}", + .{@as(u64, @intCast(std.Io.Clock.now(.real, self.io).toNanoseconds()))}, + ); + profiler = try self.platform.profiler(self.allocator, self.io, .{ + .session_id = session_id, + }); + try profiler.?.start(); + } try self.runPrefill(prompt_tokens, &stdout.interface); try self.runDecodeTurn(&stdout.interface); + const profile = if (profiler) |*p| try p.stop() else null; - try stdout.interface.writeAll("\n\n"); + if (opts.profile) { + if (profile) |trace| { + try stdout.interface.print("\x1b[2mprofile: perfetto: {s} protobuf: {s})\x1b[0m\n\n", .{ trace.perfetto_path, trace.protobuf_path }); + } else { + try stdout.interface.writeAll("\x1b[2mprofile: unavailable on this PJRT plugin\x1b[0m\n\n"); + } + } try stdout.interface.flush(); } @@ -122,6 +150,7 @@ pub const Chat = struct { decode_duration, tokensPerSecond(decode_duration, decode_tokens), }); + try stdout.flush(); } diff --git a/examples/llm/main.zig b/examples/llm/main.zig index 9388eaa0..2dd57480 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -20,6 +20,7 @@ const Args = struct { topk: u32 = 4, backend: ?zml.attention.attention.Backend = null, single: bool = false, + profile: bool = false, pub const help = \\ Use llm --model= [options] @@ -32,8 +33,9 @@ const Args = struct { \\ --seqlen= Sequence length (default: 2048) \\ --topk= Top-k sampling cutoff (default: 4) \\ --backend= Attention backend to use ([vanilla, cuda_fa2, cuda_fa3], default: auto-selection) - \\ --single Create a single kernel encompassing all the layers when supported + \\ --single Create a single kernel encompassing all the layers when supported \\ (only used by LFM2 which uses multiple kernels by default) + \\ --profile Capture a PJRT profile for non-interactive runs and write a Perfetto trace \\ ; }; @@ -162,7 +164,9 @@ pub fn main(init: std.process.Init) !void { if (interactive) { try llm_chat.runInteractive(prompt); } else { - try llm_chat.runOnce(prompt); + try llm_chat.runOnce(prompt, .{ + .profile = args.profile, + }); } } From 94d1b1582f26ad6453677753a8e0e8ba8818bf44 Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Wed, 22 Apr 2026 16:48:30 +0200 Subject: [PATCH 451/881] zml/tpu: implement pagedAttentionDense for fused K V cache (#489) refactor: introduce KVCache union and wire dense/split layout through pagedAttention Adds a `KVCache` union type to `paged_attention.zig` with two variants: - `.split { k, v }`: separate K and V page tensors (used by cuda_fa2, cuda_fa3, triton) - `.dense`: a single pre-fused K/V page tensor (used by mosaic_tpu) Changes: - `pagedAttention()` in `paged_attention.zig` now accepts a `kv_cache: KVCache` parameter alongside `k`/`v` (unused); dispatches `.split` to CUDA/Triton backends and the full `KVCache` union to `mosaic_tpu`. - `mosaic_tpu.pagedAttention()` in `tpu_attention.zig` now takes `layout: paged_attention.KVCache` instead of separate k/v tensors, and passes `layout.dense` to the existing `prepareInputs()` helper. - `KvCache.Layer` in `llmd/models.zig` is now a tagged union of `.dense` (fused tensor) and `.split { k, v }`. `attentionLayout()` converts it to `attention.KVCache`. The `.dense` update path in `Layer.update()` fuses K and V by inserting a `.kv` axis, concatenating, and merging `.hkv` before scattering into the page cache. --- zml/attention/paged_attention.zig | 29 ++++++++++++++++++++++++----- zml/attention/tpu_attention.zig | 24 ++++++------------------ 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 9293276c..10cc88c1 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -199,12 +199,31 @@ pub const AttentionOptions = struct { scale: ?f32 = null, }; -pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { +pub const KvCache = union(enum) { + split: struct { + k: zml.Tensor, + v: zml.Tensor, + }, + dense: zml.Tensor, +}; + +pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache, opts: AttentionOptions) zml.Tensor { + _ = k; + _ = v; return switch (parameters) { - .cuda_fa2 => |cuda_fa2_parameters| flashattn.paged_fa2.pagedAttention(cuda_fa2_parameters, context.cuda_fa2, q, k_cache, v_cache, opts), - .cuda_fa3 => |cuda_fa3_parameters| flashattn.paged_fa3.pagedAttention(cuda_fa3_parameters, context.cuda_fa3, q, k_cache, v_cache, opts), - .triton => |triton_parameters| triton.paged.pagedAttention(triton_parameters, context.triton, q, k_cache, v_cache, opts), - .mosaic_tpu => |mosaic_tpu_parameters| tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, context.mosaic_tpu, q, k, v, k_cache, v_cache, opts), + .cuda_fa2 => |cuda_fa2_parameters| switch (kv_cache) { + .split => |split| flashattn.paged_fa2.pagedAttention(cuda_fa2_parameters, context.cuda_fa2, q, split.k, split.v, opts), + .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + }, + .cuda_fa3 => |cuda_fa3_parameters| switch (kv_cache) { + .split => |split| flashattn.paged_fa3.pagedAttention(cuda_fa3_parameters, context.cuda_fa3, q, split.k, split.v, opts), + .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + }, + .triton => |triton_parameters| switch (kv_cache) { + .split => |split| triton.paged.pagedAttention(triton_parameters, context.triton, q, split.k, split.v, opts), + .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + }, + .mosaic_tpu => |mosaic_tpu_parameters| tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, context.mosaic_tpu, q, kv_cache.dense, opts), }; } diff --git a/zml/attention/tpu_attention.zig b/zml/attention/tpu_attention.zig index 266f89f7..54d39f0f 100644 --- a/zml/attention/tpu_attention.zig +++ b/zml/attention/tpu_attention.zig @@ -9,6 +9,7 @@ const stdx = @import("stdx"); const CompilationContext = @import("../module.zig").CompilationContext; const zml = @import("../zml.zig"); const AttentionOptions = @import("paged_attention.zig").AttentionOptions; +const paged_attention = @import("paged_attention.zig"); const log = std.log.scoped(.tpu_attention); @@ -272,27 +273,16 @@ pub const mosaic_tpu = struct { return q_out.splitAxis(.h, .{ .hkv = q_template.dim(.hkv), .hg = q_template.dim(.hg) }); } - inline fn fuseKvPages(k_cache: zml.Tensor, v_cache: zml.Tensor) zml.Tensor { - const k_pages = k_cache.insertAxes(.hd, .{.kv}); - const v_pages = v_cache.insertAxes(.hd, .{.kv}); - return zml.Tensor.concatenate(&.{ k_pages, v_pages }, .kv) - .merge(.{ .hkv = .{ .hkv, .kv } }); - } - - inline fn prepareInputs(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor) PreparedInputs { - stdx.debug.assert(k_cache.shape().eqlWithTags(v_cache.shape()), "Expected paged K/V cache shapes to match, got {f} and {f}", .{ k_cache.shape(), v_cache.shape() }); - + inline fn prepareInputs(parameters: Parameters, q: zml.Tensor, kv_pages: zml.Tensor) PreparedInputs { const q_ragged = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); - // Note : - // In a future PR, we'll have to change the pagedAttention API a bit so that we don't have to fuse keys and values - const kv_pages = fuseKvPages(k_cache, v_cache).withPartitioning(.{ .hkv = .model }); + const kv_pages_partitioned = kv_pages.withPartitioning(.{ .hkv = .model }); const query_start_len = parameters.query_start_len.withPartitioning(.{ .n = .replicated }); // Note // activeSequenceCount could probably be computed when preparing the inputs and injected through Parameters, but we'll keep that for later. return .{ .q = q_ragged, - .kv_pages = kv_pages, + .kv_pages = kv_pages_partitioned, .seq_lens = parameters.seq_lens.withPartitioning(.{ ._0 = .replicated }), .block_table = parameters.block_table.withPartitioning(.{ ._0 = .replicated, ._1 = .replicated }), .query_start_len = query_start_len, @@ -351,12 +341,10 @@ pub const mosaic_tpu = struct { } }; - pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { - _ = k; - _ = v; + pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, kv_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { stdx.debug.assert(opts.is_causal, "mosaic_tpu ragged paged attention currently only supports causal attention", .{}); - const prepared = prepareInputs(parameters, q, k_cache, v_cache); + const prepared = prepareInputs(parameters, q, kv_cache); // Keep sharding intent on the `manualComputation` boundary only. The // TPU smoke compile regressed when the body restated From e79311ccf4aa2bbc5b6b1bffca72b5a02b83ab2b Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Thu, 23 Apr 2026 11:17:40 +0200 Subject: [PATCH 452/881] platforms/cpu: remove `sha256:` prefix from the pjrt artifact's sh256 (#499) --- platforms/cpu/cpu.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 0c22fbab..5e206fb2 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -30,14 +30,14 @@ def _cpu_pjrt_plugin_impl(mctx): http_archive( name = "libpjrt_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "sha256:7da35d4b81359f42b56e3b820cacb10c4569caeb4926f2e3b11da913cd8f7c11", + sha256 = "7da35d4b81359f42b56e3b820cacb10c4569caeb4926f2e3b11da913cd8f7c11", url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-04-21T10-00-00Z/pjrt-cpu_darwin-amd64.tar.gz", ) http_archive( name = "libpjrt_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "sha256:51513b318bd180d33ef88363b5c96fc20ee7ca9db325830ebf694888c6fbc4ec", + sha256 = "51513b318bd180d33ef88363b5c96fc20ee7ca9db325830ebf694888c6fbc4ec", url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-04-21T10-00-00Z/pjrt-cpu_darwin-arm64.tar.gz", ) From bfb792342b429eb219fd7937e7a6d9dfdd29ddcd Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Thu, 23 Apr 2026 17:43:37 +0200 Subject: [PATCH 453/881] zml/io: do not fail .auto when application default can't be loaded (#502) Co-authored-by: Copilot --- zml/io/vfs/gcs.zig | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/zml/io/vfs/gcs.zig b/zml/io/vfs/gcs.zig index 4bf7f873..808a47c8 100644 --- a/zml/io/vfs/gcs.zig +++ b/zml/io/vfs/gcs.zig @@ -235,7 +235,13 @@ pub const GCS = struct { if (applicationDefaultCredentials(inner_io, environ_map)) |f| { defer f.close(inner_io); var reader = f.reader(inner_io, &jsonBuffer); - return .init(allocator, inner_io, http_client, .{ .credentials = .{ .json = &reader.interface } }); + return GCS.init(allocator, inner_io, http_client, .{ .credentials = .{ .json = &reader.interface } }) catch |err| switch (err) { + InitError.InvalidCredentialJson => { + log.warn("Invalid GCS credential JSON", .{}); + return .init(allocator, inner_io, http_client, .{}); + }, + else => return err, + }; } if (isOnGCP(inner_io) catch false) { From 6ee021faf99c680990705156cb08d4dad865d21a Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Fri, 24 Apr 2026 13:05:24 +0200 Subject: [PATCH 454/881] zml/tpu: vendor ragged paged attention kernel for strided KV loads (#505) Improve TPU ragged paged attention strided KV loading - add the TPU ragged paged attention kernel to the repo and package it with gen_ir - switch gen_ir to import the vendored kernel instead of JAX's default one - update tpu_attention tensor axis tags to match the kernel's expected shapes --- platforms/tpu/BUILD.bazel | 2 +- platforms/tpu/gen_ir.py | 2 +- platforms/tpu/kernel.py | 910 ++++++++++++++++++++++++++ platforms/tpu/libpjrt_tpu.BUILD.bazel | 2 + zml/attention/tpu_attention.zig | 16 +- 5 files changed, 922 insertions(+), 10 deletions(-) create mode 100644 platforms/tpu/kernel.py diff --git a/platforms/tpu/BUILD.bazel b/platforms/tpu/BUILD.bazel index 7cae20c0..c5159842 100644 --- a/platforms/tpu/BUILD.bazel +++ b/platforms/tpu/BUILD.bazel @@ -58,7 +58,7 @@ uv_lock( py_binary( name = "gen_ir", - srcs = ["gen_ir.py"], + srcs = ["gen_ir.py", "kernel.py"], main = "gen_ir.py", python_version = "3.12", tags = ["manual"], diff --git a/platforms/tpu/gen_ir.py b/platforms/tpu/gen_ir.py index 014caf59..8e94f88f 100644 --- a/platforms/tpu/gen_ir.py +++ b/platforms/tpu/gen_ir.py @@ -16,7 +16,7 @@ from jax._src.pallas import pallas_call from jax.experimental import pallas as pallas_mod from jax.experimental.pallas.ops.tpu.flash_attention import BlockSizes, flash_attention from jax.experimental.pallas.ops.tpu.paged_attention import paged_attention -from jax.experimental.pallas.ops.tpu.ragged_paged_attention import ragged_paged_attention +from kernel import ragged_paged_attention RAGGED_KERNEL_ENV = "ZML_TPU_RAGGED_KERNEL" _TPU_INFERENCE_RAGGED_PATCHED = False diff --git a/platforms/tpu/kernel.py b/platforms/tpu/kernel.py new file mode 100644 index 00000000..97cbfe16 --- /dev/null +++ b/platforms/tpu/kernel.py @@ -0,0 +1,910 @@ +# Copyright 2025 The JAX Authors. +# Copyright 2026 ZML +# +# Vendored from JAX and modified by ZML. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""TPU-Friendly Ragged Paged Attention kernel. + +This kernel offers a highly optimized implementation of ragged paged attention, +specifically designed for TPU and compatible with a wide range of model +specifications. It supports mixed prefill and decoding, enhancing throughput +during inference. +""" +import functools +import jax +from jax import lax +from jax._src import dtypes +from jax.experimental import pallas as pl +from jax.experimental.pallas import tpu as pltpu +from jax.experimental.pallas.ops.tpu.ragged_paged_attention.tuned_block_sizes import get_tuned_block_sizes +import jax.numpy as jnp + + +DEFAULT_MASK_VALUE = -0.7 * float(jnp.finfo(jnp.dtype("float32")).max) + + +class MultiPageAsyncCopyDescriptor: + """Descriptor for async copy of multiple K/V pages from HBM.""" + + def __init__( + self, + pages_hbm_ref, # [total_num_pages, page_size, num_combined_kv_heads_per_blk, head_dim] + vmem_buf, # [num_kv_pages_per_blk, page_size, num_combined_kv_heads_per_blk, head_dim] + sem, + page_indices_ref, # i32[max_num_seqs, pages_per_seq] + metadata, # [seq_idx, start_page_idx, end_page_idx] + ): + self._vmem_buf = vmem_buf + seq_id, start_page_idx, end_page_idx = metadata + self._async_copies = [] + # TODO(jevinjiang): Only fetch dynamic shape in need! This will insert + # a bunch of if-ops. Check the performance when we have benchmarking setup. + for i in range(vmem_buf.shape[0]): + page_idx = start_page_idx + i + page_idx = jax.lax.select(page_idx < end_page_idx, page_idx, 0) + self._async_copies.append( + pltpu.make_async_copy( + pages_hbm_ref.at[page_indices_ref[seq_id, page_idx]], + vmem_buf.at[i], + sem, + ) + ) + + def start(self): + """Starts the async copies.""" + for async_copy in self._async_copies: + async_copy.start() + + def wait(self): + for async_copy in self._async_copies: + async_copy.wait() + return self._vmem_buf + + +def ref_ragged_paged_attention( + queries: jax.Array, # [max_num_batched_tokens, num_q_heads, head_dim] + kv_pages: jax.Array, # [total_num_pages, page_size, num_combined_kv_heads, head_dim] + kv_lens: jax.Array, # i32[max_num_seqs] + page_indices: jax.Array, # i32[max_num_seqs, pages_per_seq] + cu_q_lens: jax.Array, # i32[max_num_seqs + 1] + num_seqs: jax.Array, # i32[1], + *, + sm_scale: float = 1.0, + sliding_window: int | None = None, + soft_cap: float | None = None, + mask_value: float | None = DEFAULT_MASK_VALUE, + k_scale: float | None = None, + v_scale: float | None = None, +): + static_validate_inputs( + queries, + kv_pages, + kv_lens, + page_indices, + cu_q_lens, + num_seqs, + sm_scale=sm_scale, + k_scale=k_scale, + v_scale=v_scale, + sliding_window=sliding_window, + soft_cap=soft_cap, + mask_value=mask_value, + ) + if mask_value is None: + mask_value = DEFAULT_MASK_VALUE + _, _, num_combined_kv_heads, head_dim = kv_pages.shape + assert num_combined_kv_heads % 2 == 0 + num_kv_heads = num_combined_kv_heads // 2 + num_q_heads = queries.shape[1] + assert num_q_heads % num_kv_heads == 0 + num_query_per_kv = num_q_heads // num_kv_heads + outputs = [] + for i in range(num_seqs[0]): + q_start = cu_q_lens[i] + q_end = cu_q_lens[i + 1] + q_len = q_end - q_start + kv_len = kv_lens[i] + indices = page_indices[i] + q = queries[q_start:q_end] + k = kv_pages[indices, :, 0::2, :].reshape(-1, num_kv_heads, head_dim)[ + :kv_len + ] + v = kv_pages[indices, :, 1::2, :].reshape(-1, num_kv_heads, head_dim)[ + :kv_len + ] + if k_scale is not None: + k = k.astype(jnp.float32) * k_scale + k = k.astype(q.dtype) + if v_scale is not None: + v = v.astype(jnp.float32) * v_scale + v = v.astype(q.dtype) + k = jnp.repeat(k, num_query_per_kv, axis=1) + v = jnp.repeat(v, num_query_per_kv, axis=1) + attn = jnp.einsum("qhd,khd->hqk", q, k, preferred_element_type=jnp.float32) + attn *= sm_scale + q_span = (kv_len - q_len) + jax.lax.broadcasted_iota( + jnp.int32, attn.shape, 1 + ) + kv_span = jax.lax.broadcasted_iota(jnp.int32, attn.shape, 2) + mask = q_span < kv_span + if sliding_window is not None: + mask = jnp.logical_or(mask, q_span - sliding_window >= kv_span) + if soft_cap is not None: + attn = soft_cap * jnp.tanh(attn / soft_cap) + attn += jnp.where(mask, mask_value, 0.0) + attn = jax.nn.softmax(attn, axis=-1).astype(v.dtype) + out = jnp.einsum("hqk,khd->qhd", attn, v).astype(queries.dtype) + outputs.append(out) + + return jnp.concatenate(outputs, axis=0) + + +# Expect to run these checks during runtime. +def dynamic_validate_inputs( + q: jax.Array, # [max_num_batched_tokens, num_q_heads, head_dim] + kv_pages: jax.Array, # [total_num_pages, page_size, num_combined_kv_heads, head_dim] + kv_lens: jax.Array, # i32[max_num_seqs] + page_indices: jax.Array, # i32[max_num_seqs, pages_per_seq] + cu_q_lens: jax.Array, # i32[max_num_seqs + 1] + num_seqs: jax.Array, # i32[1] + *, + # These inputs are optional. If not specified, we will not validate them. + sm_scale: float | None = None, + sliding_window: int | None = None, + soft_cap: float | None = None, + mask_value: float | None = None, + k_scale: float | None = None, + v_scale: float | None = None, + # Kernel tuning params. + num_kv_pages_per_block: int | None = None, + num_queries_per_block: int | None = None, + vmem_limit_bytes: int | None = None, +): + static_validate_inputs( + q, + kv_pages, + kv_lens, + page_indices, + cu_q_lens, + num_seqs, + sm_scale=sm_scale, + sliding_window=sliding_window, + soft_cap=soft_cap, + mask_value=mask_value, + k_scale=k_scale, + v_scale=v_scale, + num_kv_pages_per_block=num_kv_pages_per_block, + num_queries_per_block=num_queries_per_block, + vmem_limit_bytes=vmem_limit_bytes, + ) + max_num_batched_tokens = q.shape[0] + page_size = kv_pages.shape[1] + max_num_seqs, pages_per_seq = page_indices.shape + if num_seqs[0] > max_num_seqs: + raise ValueError(f"{num_seqs[0]=} must be less or equal to {max_num_seqs=}") + max_kv_len = jnp.max(kv_lens) + min_pages_per_seq = pl.cdiv(max_kv_len, page_size) + if pages_per_seq < min_pages_per_seq: + raise ValueError( + f"{pages_per_seq=} must be greater or equal to" + f" {min_pages_per_seq=} given {max_kv_len=} and {page_size=}." + ) + if cu_q_lens[num_seqs[0]] > max_num_batched_tokens: + raise ValueError( + f"Total q tokens {cu_q_lens[num_seqs[0]]} must be less or equal to" + f" {max_num_batched_tokens=}." + ) + for i in range(num_seqs[0]): + q_len = cu_q_lens[i + 1] - cu_q_lens[i] + kv_len = kv_lens[i] + if q_len > kv_len: + raise ValueError( + f"{q_len=} must be less or equal to {kv_len=} at sequence {i}." + ) + + +# Expect to run these checks during compile time. +def static_validate_inputs( + q: jax.Array, # [max_num_batched_tokens, num_q_heads, head_dim] + kv_pages: jax.Array, # [total_num_pages, page_size, num_combined_kv_heads, head_dim] + kv_lens: jax.Array, # i32[max_num_seqs] + page_indices: jax.Array, # i32[max_num_seqs, pages_per_seq] + cu_q_lens: jax.Array, # i32[max_num_seqs + 1] + num_seqs: jax.Array, # i32[1] + *, + # These inputs are optional. If not specified, we will not validate them. + sm_scale: float | None = None, + sliding_window: int | None = None, + soft_cap: float | None = None, + mask_value: float | None = None, + k_scale: float | None = None, + v_scale: float | None = None, + # Kernel tuning params. + num_kv_pages_per_block: int | None = None, + num_queries_per_block: int | None = None, + vmem_limit_bytes: int | None = None, +): + _, num_q_heads, head_dim = q.shape + _, _, num_combined_kv_heads, head_dim_k = kv_pages.shape + assert num_combined_kv_heads % 2 == 0 + assert isinstance(k_scale, float) or k_scale is None + assert isinstance(v_scale, float) or v_scale is None + num_kv_heads = num_combined_kv_heads // 2 + max_num_seqs, pages_per_seq = page_indices.shape + if num_seqs.shape != (1,): + raise ValueError(f"{num_seqs.shape=} must be (1,)") + if head_dim_k != head_dim: + raise ValueError( + f"Q head_dim {head_dim} must be the same as that of K/V {head_dim_k}." + ) + if kv_lens.shape != (max_num_seqs,): + raise ValueError( + f"Expected {kv_lens.shape=} to be ({max_num_seqs},) where" + " `max_num_seqs` is `page_indices.shape[0]`." + ) + if cu_q_lens.shape != (max_num_seqs + 1,): + raise ValueError( + f"Expected {cu_q_lens.shape=} to be ({max_num_seqs + 1},) where" + " `max_num_seqs` is `page_indices.shape[0]`." + ) + if ( + kv_lens.dtype != jnp.int32 + or page_indices.dtype != jnp.int32 + or cu_q_lens.dtype != jnp.int32 + ): + raise ValueError( + "The dtype of `kv_lens`, `page_indices`, and `cu_q_lens` must be" + f" int32. Got {kv_lens.dtype=}, {page_indices.dtype=}," + f" {cu_q_lens.dtype=}." + ) + if num_q_heads % num_kv_heads != 0: + raise ValueError(f"{num_q_heads=} must be divisible by {num_kv_heads=}") + if sliding_window is not None and sliding_window <= 0: + raise ValueError(f"{sliding_window=} must be positive.") + if soft_cap is not None and soft_cap == 0.0: + raise ValueError(f"{soft_cap=} must not be 0.0.") + if ( + num_kv_pages_per_block is not None + and not 0 < num_kv_pages_per_block <= pages_per_seq + ): + raise ValueError( + f"{num_kv_pages_per_block=} must be in range (0, {pages_per_seq}]." + ) + if num_queries_per_block is not None and num_queries_per_block <= 0: + raise ValueError(f"{num_queries_per_block=} must be positive.") + if vmem_limit_bytes is not None and vmem_limit_bytes <= 0: + raise ValueError(f"{vmem_limit_bytes=} must be positive.") + del sm_scale # No constraints on sm_scale. + del mask_value # No consstraints on mask_value. + + +def ragged_paged_attention_kernel( + # Prefetch + kv_lens_ref, # [max_num_seqs] + page_indices_ref, # [max_num_seqs, pages_per_seq] + cu_q_lens_ref, # [max_num_seqs + 1] + seq_buf_idx_ref, + # TODO(jevinjiang): if OOM in SMEM, consider pack to other scalar refs. + num_seqs_ref, + # Input + q_ref, # [num_q_per_blk, num_q_heads_per_blk, head_dim] + kv_pages_hbm_ref, # [total_num_pages, page_size, num_combined_kv_heads, head_dim] + # Output + o_ref, # [num_q_per_blk, num_q_heads_per_blk, head_dim] + # Scratch + kv_bufs, # [2, num_kv_pages_per_blk, page_size, num_combined_kv_heads_per_blk, head_dim] + sems, # [2, 2] + l_ref, # [num_kv_heads_per_blk, num_q_per_blk * num_q_heads_per_kv_head, 128] + m_ref, # [num_kv_heads_per_blk, num_q_per_blk * num_q_heads_per_kv_head, 128] + acc_ref, # [num_q_per_blk, num_q_heads_per_blk, head_dim] + *, + sm_scale: float, + sliding_window: int | None = None, + soft_cap: float | None = None, + mask_value: float | None = DEFAULT_MASK_VALUE, + k_scale: float | None = None, + v_scale: float | None = None, +): + if mask_value is None: + mask_value = DEFAULT_MASK_VALUE + num_q_per_blk, num_q_heads_per_blk, head_dim = q_ref.shape + pages_per_seq = page_indices_ref.shape[-1] + num_seqs = num_seqs_ref[0] + _, num_kv_pages_per_blk, page_size, num_combined_kv_heads_per_blk, _ = ( + kv_bufs.shape + ) + num_kv_heads_per_blk = num_combined_kv_heads_per_blk // 2 + num_kv_per_blk = num_kv_pages_per_blk * page_size + num_q_heads_per_kv_head = num_q_heads_per_blk // num_kv_heads_per_blk + heads_blk_idx, q_blk_idx = ( + pl.program_id(0), + pl.program_id(1), + ) + num_heads_blks = pl.num_programs(0) + init_seq_idx = seq_buf_idx_ref[0] + init_buf_idx = seq_buf_idx_ref[1] + q_len_start = q_blk_idx * num_q_per_blk + q_len_end = q_len_start + num_q_per_blk + + def create_kv_async_copy_descriptors( + heads_blk_idx, seq_idx, kv_blk_idx, buf_idx + ): + start_kv_page_idx = kv_blk_idx * num_kv_pages_per_blk + end_kv_page_idx = jnp.minimum( + pages_per_seq, pl.cdiv(kv_lens_ref[seq_idx], page_size) + ) + metadata = (seq_idx, start_kv_page_idx, end_kv_page_idx) + heads_start = heads_blk_idx * num_combined_kv_heads_per_blk + async_copy_kv = MultiPageAsyncCopyDescriptor( + kv_pages_hbm_ref.at[ + :, :, pl.ds(heads_start, num_combined_kv_heads_per_blk), : + ], + kv_bufs.at[buf_idx], + sems.at[buf_idx], + page_indices_ref, + metadata, + ) + return async_copy_kv + + def strided_load_kv(ref, start, step): + packing = get_dtype_packing(ref.dtype) + if packing == 1: + if ref.shape[-1] <= 128: + return [ref[start::step, :]], [ref[start + 1 :: step, :]] + num_rows = ref.shape[0] // step + k = jnp.stack([ref[start + i * step, :] for i in range(num_rows)], axis=0) + v = jnp.stack( + [ref[start + 1 + i * step, :] for i in range(num_rows)], axis=0 + ) + return [k], [v] + assert packing in (2, 4, 8) + assert step % packing == 0 + k_list, v_list = [], [] + b_start = start // packing + b_step = step // packing + b_ref = ref.bitcast(jnp.uint32) + if b_ref.shape[-1] <= 128: + b = b_ref[b_start::b_step, :] + else: + num_rows = ref.shape[0] // step + b = jnp.stack( + [b_ref[b_start + i * b_step, :] for i in range(num_rows)], axis=0 + ) + + if ref.dtype == jnp.bfloat16: + bk = b << 16 + bv = b & jnp.uint32(0xFFFF0000) + k = pltpu.bitcast(bk, jnp.float32).astype(jnp.bfloat16) + v = pltpu.bitcast(bv, jnp.float32).astype(jnp.bfloat16) + k_list.append(k) + v_list.append(v) + else: + bitwidth = 32 // packing + bitcast_dst_dtype = jnp.dtype(f"uint{bitwidth}") + for i in range(0, packing, 2): + bk = b >> (i * bitwidth) + k = pltpu.bitcast(bk.astype(bitcast_dst_dtype), ref.dtype) + k_list.append(k) + bv = b >> ((i + 1) * bitwidth) + v = pltpu.bitcast(bv.astype(bitcast_dst_dtype), ref.dtype) + v_list.append(v) + + return k_list, v_list + + def fold_on_2nd_minor(vec): + assert vec.dtype == jnp.bfloat16 or vec.dtype == jnp.float32 + assert len(vec.shape) >= 2 + last_dim = vec.shape[-1] + packing = get_dtype_packing(vec.dtype) + if vec.shape[-2] % packing != 0: + vec = vec.astype(jnp.float32) + return vec.reshape(-1, last_dim) + + @pl.when(heads_blk_idx + q_blk_idx == 0) + def prefetch_first_kv_blk(): + async_copy_kv = create_kv_async_copy_descriptors( + heads_blk_idx, init_seq_idx, 0, init_buf_idx + ) + async_copy_kv.start() + + def is_cur_q_blk_needed(q_states): + done, cur_seq_idx, _ = q_states + should_run = jnp.logical_and(q_len_start < cu_q_lens_ref[num_seqs], + cur_seq_idx < num_seqs) + return jnp.logical_and(done == 0, should_run) + + def compute_with_cur_q_blk(q_states): + done, cur_seq_idx, cur_buf_idx = q_states + q_start = cu_q_lens_ref[cur_seq_idx] + q_end = cu_q_lens_ref[cur_seq_idx + 1] + q_len = q_end - q_start + kv_len = kv_lens_ref[cur_seq_idx] + + def get_next_prefetch_ids( + heads_blk_idx, cur_seq_idx, kv_blk_idx, cur_buf_idx + ): + next_kv_blk_idx = kv_blk_idx + 1 + is_last_kv_blk = next_kv_blk_idx * num_kv_per_blk >= kv_len + next_kv_blk_idx = lax.select( + is_last_kv_blk, + 0, + next_kv_blk_idx, + ) + is_cur_seq_end_in_cur_q_blk = q_end <= q_len_end + next_seq_idx = lax.select( + is_last_kv_blk, + lax.select(is_cur_seq_end_in_cur_q_blk, cur_seq_idx + 1, cur_seq_idx), + cur_seq_idx, + ) + is_last_seq = next_seq_idx == num_seqs + next_seq_idx = lax.select( + is_last_seq, + 0, + next_seq_idx, + ) + next_heads_blk_idx = lax.select( + is_last_seq, + heads_blk_idx + 1, + heads_blk_idx, + ) + next_buf_idx = lax.select(cur_buf_idx == 0, 1, 0) + return next_heads_blk_idx, next_seq_idx, next_kv_blk_idx, next_buf_idx + + def flash_attention( + q, # [num_q_per_blk * num_q_heads_per_kv_head, head_dim] + k, # [num_kv_per_blk, head_dim] + v, # [num_kv_per_blk, head_dim] + head_l_ref, # [num_q_per_blk * num_q_heads_per_kv_head, 128] + head_m_ref, # [num_q_per_blk * num_q_heads_per_kv_head, 128] + head_acc_ref, # [num_q_per_blk, num_q_heads_per_kv_head, head_dim] + *, + kv_blk_idx, + ): + assert q.shape == ( + num_q_per_blk * num_q_heads_per_kv_head, + head_dim, + ) + assert ( + k.shape + == v.shape + == ( + num_kv_per_blk, + head_dim, + ) + ) + assert k.dtype == v.dtype + assert ( + head_m_ref.shape + == head_l_ref.shape + == ( + num_q_per_blk * num_q_heads_per_kv_head, + 128, + ) + ) + assert head_acc_ref.shape == ( + num_q_per_blk, + num_q_heads_per_kv_head, + head_dim, + ) + kv_len_start = kv_blk_idx * num_kv_per_blk + + def masked_store(ref, val, start, end, group=1): + iota = lax.broadcasted_iota(jnp.int32, ref.shape, 0) // group + pltpu.store(ref, val, mask=jnp.logical_and(iota >= start, iota < end)) + + def load_with_init(ref, init_val): + return jnp.where( + kv_blk_idx == 0, jnp.full_like(ref, init_val), ref[...] + ) + + # kv lens will be contracting dim, we should mask out the NaNs. + kv_mask = ( + lax.broadcasted_iota(jnp.int32, k.shape, 0) < kv_len - kv_len_start + ) + k = jnp.where(kv_mask, k.astype(jnp.float32), 0).astype(k.dtype) + v = jnp.where(kv_mask, v.astype(jnp.float32), 0).astype(v.dtype) + + qk = ( + jnp.einsum("nd,md->nm", q, k, preferred_element_type=jnp.float32) + * sm_scale + ) + store_start = jnp.maximum(q_start - q_len_start, 0) + store_end = jnp.minimum(q_end - q_len_start, num_q_per_blk) + + row_ids = ( + (kv_len - q_len) + + q_len_start + - q_start + + jax.lax.broadcasted_iota( + jnp.int32, + (num_q_per_blk * num_q_heads_per_kv_head, num_kv_per_blk), + 0, + ) + // num_q_heads_per_kv_head + ) + col_ids = kv_len_start + jax.lax.broadcasted_iota( + jnp.int32, + (num_q_per_blk * num_q_heads_per_kv_head, num_kv_per_blk), + 1, + ) + causal_mask = row_ids < col_ids + if sliding_window is not None: + causal_mask = jnp.logical_or(causal_mask, + row_ids - sliding_window >= col_ids) + if soft_cap is not None: + qk = soft_cap * jnp.tanh(qk / soft_cap) + qk += jnp.where(causal_mask, mask_value, 0.0) + m_curr = jnp.max(qk, axis=1, keepdims=True) + s_curr = jnp.exp(qk - m_curr) + qkv = jnp.dot(s_curr, v, preferred_element_type=jnp.float32) + lm_store_shape = head_m_ref.shape + m_curr = jnp.broadcast_to(m_curr, lm_store_shape) + l_curr = jnp.broadcast_to( + s_curr.sum(axis=1, keepdims=True), lm_store_shape + ) + m_prev = load_with_init(head_m_ref, -jnp.inf) + l_prev = load_with_init(head_l_ref, 0.0) + m_next = jnp.maximum(m_prev, m_curr) + masked_store( + head_m_ref, m_next, store_start, store_end, num_q_heads_per_kv_head + ) + alpha = jnp.exp(m_prev - m_next) + beta = jnp.exp(m_curr - m_next) + l_alpha = alpha * l_prev + l_next = l_alpha + beta * l_curr + l_next_safe = jnp.where(l_next == 0.0, 1.0, l_next) + masked_store( + head_l_ref, + l_next_safe, + store_start, + store_end, + num_q_heads_per_kv_head, + ) + + def broadcast_to_shape(arr, shape): + if arr.shape == shape: + return arr + assert len(arr.shape) == len(shape) + assert arr.shape[0] == shape[0] + assert shape[1] % arr.shape[1] == 0 + # no-op concatenation. + return jnp.concatenate( + [arr for _ in range(shape[1] // arr.shape[1])], axis=1 + ) + + o_curr = load_with_init(head_acc_ref, 0.0).reshape(-1, head_dim) + l_alpha = broadcast_to_shape(l_alpha, qkv.shape) + beta = broadcast_to_shape(beta, qkv.shape) + l_next_safe = broadcast_to_shape(l_next_safe, qkv.shape) + out = lax.div( + l_alpha * o_curr + beta * qkv, + l_next_safe, + ) + masked_store( + head_acc_ref, + out.reshape(head_acc_ref.shape), + store_start, + store_end, + ) + + def is_valid_kv_blk_in_cur_seq(kv_states): + kv_blk_idx, _ = kv_states + return kv_blk_idx * num_kv_per_blk < kv_len + + def compute_with_kv_blk_in_cur_seq(kv_states): + kv_blk_idx, cur_buf_idx = kv_states + next_heads_blk_idx, next_seq_idx, next_kv_blk_idx, next_buf_idx = ( + get_next_prefetch_ids( + heads_blk_idx, cur_seq_idx, kv_blk_idx, cur_buf_idx + ) + ) + + @pl.when(next_heads_blk_idx < num_heads_blks) + def prefetch_next_kv_blk(): + # TODO(jevinjiang): reuse the same buffer if it is already prefetched! + # TODO(jevinjiang): only fetch effective dynamic size to hold kv_len and + # DMA to fixed size buffer! + next_async_copy_kv = create_kv_async_copy_descriptors( + next_heads_blk_idx, next_seq_idx, next_kv_blk_idx, next_buf_idx + ) + next_async_copy_kv.start() + + cur_async_copy_kv = create_kv_async_copy_descriptors( + heads_blk_idx, cur_seq_idx, kv_blk_idx, cur_buf_idx + ) + kv_ref = cur_async_copy_kv.wait().reshape( + num_kv_pages_per_blk * page_size * num_combined_kv_heads_per_blk, + head_dim, + ) + kv_packing = get_dtype_packing(kv_ref.dtype) + # NOTE: kv_packing is divided by 2 because k and v are packed together. + kv_load_step = max(1, kv_packing // 2) + for kv_head_chunk_idx in range(0, num_kv_heads_per_blk, kv_load_step): + k_list, v_list = strided_load_kv( + kv_ref, kv_head_chunk_idx * 2, num_combined_kv_heads_per_blk + ) + for step_idx in range(kv_load_step): + k = k_list[step_idx] + v = v_list[step_idx] + if k_scale is not None: + # NOTE: Conversion between arbitrary data types is not supported. + # That's why it is converted to float32 first. + k = k.astype(jnp.float32) * k_scale + k = k.astype(q_ref.dtype) + if v_scale is not None: + v = v.astype(jnp.float32) * v_scale + v = v.astype(q_ref.dtype) + kv_head_idx = kv_head_chunk_idx + step_idx + q_head_idx = kv_head_idx * num_q_heads_per_kv_head + # TODO(jevinjiang): extra handling for packed type that can start at + # unaligned position! + q = fold_on_2nd_minor( + q_ref[:, q_head_idx : q_head_idx + num_q_heads_per_kv_head, :] + ) + flash_attention( + q, + k, + v, + l_ref.at[kv_head_idx], + m_ref.at[kv_head_idx], + acc_ref.at[ + :, q_head_idx : q_head_idx + num_q_heads_per_kv_head, : + ], + kv_blk_idx=kv_blk_idx, + ) + return kv_blk_idx + 1, next_buf_idx + + _, next_buf_idx = lax.while_loop( + is_valid_kv_blk_in_cur_seq, + compute_with_kv_blk_in_cur_seq, + (0, cur_buf_idx), # (kv_blk_idx, buf_idx) + ) + next_seq_idx = lax.select(q_end <= q_len_end, cur_seq_idx + 1, cur_seq_idx) + done = lax.select(q_end < q_len_end, done, 1) + return done, next_seq_idx, next_buf_idx + + _, seq_idx, buf_idx = lax.while_loop( + is_cur_q_blk_needed, + compute_with_cur_q_blk, + (0, init_seq_idx, init_buf_idx), # (done, seq_idx, buf_idx) + ) + # Reset seq_idx for next kv_heads_blk if run out of seqs! + seq_buf_idx_ref[0] = lax.select(seq_idx < num_seqs, seq_idx, 0) + seq_buf_idx_ref[1] = buf_idx + o_ref[...] = acc_ref[...].astype(q_ref.dtype) + + +def get_dtype_packing(dtype): + bits = dtypes.itemsize_bits(dtype) + return 32 // bits + + +def get_min_heads_per_blk( + num_q_heads, num_combined_kv_heads, q_dtype, kv_dtype +): + q_packing = get_dtype_packing(q_dtype) + kv_packing = get_dtype_packing(kv_dtype) + + def can_be_xla_fully_tiled(x, packing): + if x % packing != 0: + return False + x //= packing + return x in (1, 2, 4, 8) or x % 8 == 0 + + # TODO(jevinjiang): support unaligned number of heads! + if not can_be_xla_fully_tiled(num_combined_kv_heads, kv_packing): + raise ValueError( + f"Not implemented: {num_combined_kv_heads=} can not be XLA fully tiled." + ) + assert num_combined_kv_heads % 2 == 0 + num_kv_heads = num_combined_kv_heads // 2 + assert num_q_heads % num_kv_heads == 0 + ratio = num_q_heads // num_kv_heads + # TODO(jevinjiang): we can choose smaller tiling for packed type if large + # second minor tiling is not on. + max_combined_kv_tiling = 8 * kv_packing + min_combined_kv_heads = ( + max_combined_kv_tiling + if num_combined_kv_heads % max_combined_kv_tiling == 0 + else num_combined_kv_heads + ) + min_q_heads = min_combined_kv_heads // 2 * ratio + if can_be_xla_fully_tiled(min_q_heads, q_packing): + return min_q_heads, min_combined_kv_heads + return num_q_heads, num_combined_kv_heads + + +@functools.partial( + jax.jit, + static_argnames=[ + "sm_scale", + "mask_value", + "num_kv_pages_per_block", + "num_queries_per_block", + "vmem_limit_bytes", + "sliding_window", + "soft_cap", + "k_scale", + "v_scale", + ], +) +def ragged_paged_attention( + q: jax.Array, # [max_num_batched_tokens, num_q_heads, head_dim] + # TODO(jevinjiang): create a write_to_kv_cache kernel! + kv_pages: jax.Array, # [total_num_pages, page_size, num_combined_kv_heads, head_dim] + kv_lens: jax.Array, # i32[max_num_seqs] + page_indices: jax.Array, # i32[max_num_seqs, pages_per_seq] + cu_q_lens: jax.Array, # i32[max_num_seqs + 1] + num_seqs: jax.Array, # i32[1] + *, + sm_scale: float = 1.0, + sliding_window: int | None = None, + soft_cap: float | None = None, + mask_value: float | None = DEFAULT_MASK_VALUE, + k_scale: float | None = None, + v_scale: float | None = None, + num_kv_pages_per_block: int | None = None, + num_queries_per_block: int | None = None, + vmem_limit_bytes: int | None = None, +): + """Ragged paged attention that supports mixed prefill and decode. + + Args: + q: concatenated all sequences' queries. + kv_pages: paged KV cache. Normally in HBM. + kv_lens: padded kv lengths. Only the first num_seqs values are valid. + page_indices: the first index indicates which page to use in the kv cache + for each sequence. Only the first num_seqs values are valid. + cu_q_lens: the cumulative sum of the effective query lengths. Similar to + kv_lens, only the first num_seqs+1 values are valid. + num_seqs: the dynamic number of sequences. + sm_scale: the softmax scale which will be applied to the Q@K^T. + sliding_window: the sliding window size for the attention. + soft_cap: the logit soft cap for the attention. + mask_value: mask value for causal mask. + k_scale: the scale for the key cache. + v_scale: the scale for the value cache. + num_kv_pages_per_block: number of kv pages to be processed in one flash + attention block in the pallas kernel. + num_queries_per_block: number of kv pages to be processed in one flash + attention block in the pallas kernel. + vmem_limit_bytes: the vmem limit for the pallas kernel. + + Returns: + The output of the attention. + """ + static_validate_inputs( + q, + kv_pages, + kv_lens, + page_indices, + cu_q_lens, + num_seqs, + sm_scale=sm_scale, + sliding_window=sliding_window, + soft_cap=soft_cap, + mask_value=mask_value, + k_scale=k_scale, + v_scale=v_scale, + num_kv_pages_per_block=num_kv_pages_per_block, + num_queries_per_block=num_queries_per_block, + vmem_limit_bytes=vmem_limit_bytes, + ) + if mask_value is None: + mask_value = DEFAULT_MASK_VALUE + num_q_tokens, num_q_heads, head_dim = q.shape + _, page_size, num_combined_kv_heads, _ = kv_pages.shape + assert num_combined_kv_heads % 2 == 0 + num_kv_heads = num_combined_kv_heads // 2 + _, pages_per_seq = page_indices.shape + num_q_heads_per_blk, num_combined_kv_heads_per_blk = get_min_heads_per_blk( + num_q_heads, num_combined_kv_heads, q.dtype, kv_pages.dtype + ) + num_q_per_blk = num_queries_per_block + num_kv_pages_per_blk = num_kv_pages_per_block + if num_q_per_blk is None or num_kv_pages_per_blk is None: + num_kv_pages_per_blk, num_q_per_blk = get_tuned_block_sizes( + q.dtype, + kv_pages.dtype, + num_q_heads_per_blk, + num_combined_kv_heads_per_blk // 2, + head_dim, + page_size, + num_q_tokens, + pages_per_seq, + ) + num_q_heads_per_kv_head = num_q_heads // num_kv_heads + num_q_blks = pl.cdiv(num_q_tokens, num_q_per_blk) + assert num_combined_kv_heads_per_blk % 2 == 0 + num_kv_heads_per_blk = num_combined_kv_heads_per_blk // 2 + assert num_q_heads_per_blk % num_q_heads_per_kv_head == 0 + num_heads_blks = num_q_heads // num_q_heads_per_blk + grid = (num_heads_blks, num_q_blks) + + def q_index_map(heads_blk_idx, q_blk_idx, *_): + return (q_blk_idx, heads_blk_idx, 0) + + q_block_spec = pl.BlockSpec( + (num_q_per_blk, num_q_heads_per_blk, head_dim), + q_index_map, + ) + in_specs = [ + q_block_spec, + pl.BlockSpec(memory_space=pl.ANY), + ] + out_specs = q_block_spec + lm_scratch = pltpu.VMEM( + # TODO(jevinjiang): use 128 instead of 1 is due to Mosaic does not support + # unaligned slicing! + (num_kv_heads_per_blk, num_q_per_blk * num_q_heads_per_kv_head, 128), + jnp.float32, + ) + acc_scratch = pltpu.VMEM( + (num_q_per_blk, num_q_heads_per_blk, head_dim), + jnp.float32, + ) + double_buf_scratch = pltpu.VMEM( + ( + 2, # For double buffering during DMA copies. + num_kv_pages_per_blk, + page_size, + num_combined_kv_heads_per_blk, + head_dim, + ), + kv_pages.dtype, + ) + scratch_shapes = [ + double_buf_scratch, # kv_bufs + pltpu.SemaphoreType.DMA((2,)), # Semaphores for double buffers. + lm_scratch, # l_ref + lm_scratch, # m_ref + acc_scratch, + ] + scalar_prefetches = ( + kv_lens, + page_indices, + cu_q_lens, + jnp.array((0, 0), jnp.int32), # seq_idx, buf_idx + num_seqs, + ) + kernel = pl.pallas_call( + functools.partial( + ragged_paged_attention_kernel, + sm_scale=sm_scale, + sliding_window=sliding_window, + soft_cap=soft_cap, + mask_value=mask_value, + k_scale=k_scale, + v_scale=v_scale, + ), + grid_spec=pltpu.PrefetchScalarGridSpec( + num_scalar_prefetch=len(scalar_prefetches), + in_specs=in_specs, + out_specs=out_specs, + grid=grid, + scratch_shapes=scratch_shapes, + ), + compiler_params=pltpu.CompilerParams( + dimension_semantics=( + "arbitrary", + "arbitrary", + ), + vmem_limit_bytes=vmem_limit_bytes, + ), + out_shape=jax.ShapeDtypeStruct(shape=q.shape, dtype=q.dtype), + name="ragged_paged_attention_kernel", + ) + + return kernel(*scalar_prefetches, q, kv_pages) diff --git a/platforms/tpu/libpjrt_tpu.BUILD.bazel b/platforms/tpu/libpjrt_tpu.BUILD.bazel index 26ab93ac..910daf39 100644 --- a/platforms/tpu/libpjrt_tpu.BUILD.bazel +++ b/platforms/tpu/libpjrt_tpu.BUILD.bazel @@ -23,6 +23,7 @@ copy_to_directory( include_external_repositories = ["**"], include_srcs_patterns = [ "platforms/tpu/gen_ir.py", + "platforms/tpu/kernel.py", "platforms/tpu/gen_ir_zig", "**/lib**", "lib/python*/**", @@ -30,6 +31,7 @@ copy_to_directory( ], replace_prefixes = { "platforms/tpu/gen_ir.py": "site-packages/gen_ir.py", + "platforms/tpu/kernel.py": "site-packages/kernel.py", "platforms/tpu/gen_ir_zig": "bin/gen_ir_zig", "lib/x86_64-linux-gnu": "lib", }, diff --git a/zml/attention/tpu_attention.zig b/zml/attention/tpu_attention.zig index 54d39f0f..bad318e3 100644 --- a/zml/attention/tpu_attention.zig +++ b/zml/attention/tpu_attention.zig @@ -165,9 +165,9 @@ pub const mosaic_tpu = struct { pub fn init(opts: Options) Parameters { return .{ .opts = opts, - .block_table = zml.Tensor.init(.{ opts.batch_size, opts.max_num_pages }, .i32), - .seq_lens = zml.Tensor.init(.{opts.batch_size}, .i32), - .query_start_len = zml.Tensor.init(.{ .n = opts.batch_size + 1 }, .i32), + .block_table = zml.Tensor.init(.{ .b = opts.batch_size, .p = opts.max_num_pages }, .i32), + .seq_lens = zml.Tensor.init(.{ .b = opts.batch_size }, .i32), + .query_start_len = zml.Tensor.init(.{ .b = opts.batch_size + 1 }, .i32), }; } @@ -229,13 +229,13 @@ pub const mosaic_tpu = struct { } fn activeSequenceCount(query_start_len: zml.Tensor) zml.Tensor { - const start = query_start_len.slice1d(0, .{ .end = query_start_len.dim(0) - 1 }); - const end = query_start_len.slice1d(0, .{ .start = 1 }); + const start = query_start_len.slice1d(.b, .{ .end = query_start_len.dim(.b) - 1 }); + const end = query_start_len.slice1d(.b, .{ .start = 1 }); const query_lens = end.sub(start); return query_lens .cmp(.GT, zml.Tensor.constant(query_lens.dtype().zero()).broad(query_lens.shape())) .convert(.i32) - .sum(0) + .sum(.b) .reshape(zml.Shape.init(.{1}, .i32)); } @@ -276,7 +276,7 @@ pub const mosaic_tpu = struct { inline fn prepareInputs(parameters: Parameters, q: zml.Tensor, kv_pages: zml.Tensor) PreparedInputs { const q_ragged = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); const kv_pages_partitioned = kv_pages.withPartitioning(.{ .hkv = .model }); - const query_start_len = parameters.query_start_len.withPartitioning(.{ .n = .replicated }); + const query_start_len = parameters.query_start_len.withPartitioning(.{ .b = .replicated }); // Note // activeSequenceCount could probably be computed when preparing the inputs and injected through Parameters, but we'll keep that for later. @@ -301,7 +301,7 @@ pub const mosaic_tpu = struct { const block_table_shape = prepared_inputs.block_table.shape(); const q_token_count = if (q_shape.hasTag(.q) != null) q_shape.dim(.q) else q_shape.dim(.b); - const max_num_seqs = block_table_shape.dim(0); + const max_num_seqs = block_table_shape.dim(.b); const num_q_heads = q_shape.dim(.h); stdx.debug.assert(@mod(kv_pages_shape.dim(.hkv), 2) == 0, "Expected fused KV pages .hkv dimension to be even, got {}", .{kv_pages_shape.dim(.hkv)}); const num_kv_heads = std.math.divExact(i64, kv_pages_shape.dim(.hkv), 2) catch |err| switch (err) { From 9211a1770e292055d0ec945ae36a3a67ee575693 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 27 Apr 2026 10:31:52 +0200 Subject: [PATCH 455/881] Remove BoundedArray.init (#506) When looking at the new Sharding code I was triggered by the high number of "try". I then realized most of the try where traced back to `axes = try .init(0)` which can never fail. So I removed BoundedArray.init (aka Andrew mode) It's generally unsafe because it makes `slice()` return undefined.; I replaced`try init(0)` by `.empty` I replaced `try init(n)` by `.fromSlice` or with `.{ .buffer = undefined, .len = n }` to explicitly show the unsafeness This leads to widespread changes because it means that most of the sharding operations can't fail anymore. --- examples/benchmark/main.zig | 4 +- examples/io/main.zig | 6 +- examples/llm/models/common.zig | 4 +- examples/llm/models/lfm2_tests.zig | 4 +- examples/llm/models/llama_tests.zig | 4 +- examples/sharding/main.zig | 4 +- mlir/dialects/func.zig | 6 +- mlir/dialects/stablehlo/stablehlo.zig | 6 +- mlir/mlir.zig | 8 +- stdx/bounded_array.zig | 39 ++-- zml/buffer.zig | 4 +- zml/exe.zig | 4 +- zml/io.zig | 32 +-- zml/module.zig | 6 +- zml/ops.zig | 38 ++-- zml/safetensors.zig | 2 +- zml/shape.zig | 34 +-- zml/sharding.zig | 208 ++++++++---------- zml/tensor.zig | 32 +-- zml/tokenizer/hftokenizers/hftokenizers.zig | 4 +- zml/tokenizer/homemade.zig | 12 +- zml/tokenizer/sentencepiece/sentencepiece.zig | 2 +- 22 files changed, 222 insertions(+), 241 deletions(-) diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index b7be859d..953218d9 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -24,11 +24,11 @@ pub fn main(init: std.process.Init) !void { const platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator, io); - const logical_mesh: zml.sharding.LogicalMesh = try .init("benchmark_mesh", .{ + const logical_mesh: zml.sharding.LogicalMesh = .init("benchmark_mesh", .{ .m = .low_bandwidth, .n = .high_bandwidth, }); - const strategy: zml.sharding.Strategy = try .suggest(logical_mesh, platform.physical_mesh); + const strategy: zml.sharding.Strategy = .suggest(logical_mesh, platform.physical_mesh); const benchmark_sharding: zml.sharding.Sharding = try .initFromStrategy(platform, logical_mesh, strategy); const cli_args: CliArgs = stdx.flags.parse(init.minimal.args, CliArgs); diff --git a/examples/io/main.zig b/examples/io/main.zig index 3bb823a0..92f54aba 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -193,8 +193,8 @@ pub fn main(init: std.process.Init) !void { const replicated_sharding = try zml.sharding.replicatedSharding(platform); const sharded_sharding: zml.sharding.Sharding = blk: { - const model_logical_mesh: zml.sharding.LogicalMesh = try .init("playground_model", .{ .model = .high_bandwidth }); - const model_strategy: zml.sharding.Strategy = try .suggest(model_logical_mesh, platform.physical_mesh); + const model_logical_mesh: zml.sharding.LogicalMesh = .init("playground_model", .{ .model = .high_bandwidth }); + const model_strategy: zml.sharding.Strategy = .suggest(model_logical_mesh, platform.physical_mesh); break :blk try .initFromStrategy(platform, model_logical_mesh, model_strategy); }; @@ -230,7 +230,7 @@ const TreeCounts = struct { fn printTree(io: std.Io, writer: *std.Io.Writer, dir: std.Io.Dir, prefix: []const u8, max_depth: usize, counts: *TreeCounts) !void { if (max_depth == 0) return; - var entries: stdx.BoundedArray(std.Io.Dir.Entry, 1024) = .{}; + var entries: stdx.BoundedArray(std.Io.Dir.Entry, 1024) = .empty; var it = dir.iterate(); while (try it.next(io)) |entry| { entries.appendAssumeCapacity(entry); diff --git a/examples/llm/models/common.zig b/examples/llm/models/common.zig index b5eb5c0b..79112dcc 100644 --- a/examples/llm/models/common.zig +++ b/examples/llm/models/common.zig @@ -17,8 +17,8 @@ pub const Shardings = struct { model: zml.sharding.Sharding, pub fn init(platform: *zml.Platform) !Shardings { - const model_mesh: zml.sharding.LogicalMesh = try .init("model", .{ .model = .high_bandwidth }); - const model_sharding_strategy: zml.sharding.Strategy = try .suggest(model_mesh, platform.physical_mesh); + const model_mesh: zml.sharding.LogicalMesh = .init("model", .{ .model = .high_bandwidth }); + const model_sharding_strategy: zml.sharding.Strategy = .suggest(model_mesh, platform.physical_mesh); return .{ .replicated = try zml.sharding.replicatedSharding(platform), .model = try .initFromStrategy(platform, model_mesh, model_sharding_strategy), diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig index 73a539a9..93ce6ec1 100644 --- a/examples/llm/models/lfm2_tests.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -47,8 +47,8 @@ pub fn main(init: std.process.Init) !void { defer repo_model.deinit(allocator); var progress = std.Progress.start(io, .{ .root_name = args.model }); - const tp_mesh: zml.sharding.LogicalMesh = try .init("tp_mesh", .{ .model = .high_bandwidth }); - const tp_strategy: zml.sharding.Strategy = try .suggest(tp_mesh, platform.physical_mesh); + const tp_mesh: zml.sharding.LogicalMesh = .init("tp_mesh", .{ .model = .high_bandwidth }); + const tp_strategy: zml.sharding.Strategy = .suggest(tp_mesh, platform.physical_mesh); const shardings: common.Shardings = .{ .replicated = try zml.sharding.replicatedSharding(platform), .model = try .initFromStrategy(platform, tp_mesh, tp_strategy), diff --git a/examples/llm/models/llama_tests.zig b/examples/llm/models/llama_tests.zig index 777517ec..14faba7f 100644 --- a/examples/llm/models/llama_tests.zig +++ b/examples/llm/models/llama_tests.zig @@ -44,8 +44,8 @@ pub fn main(init: std.process.Init) !void { defer repo_model.deinit(allocator); var progress = std.Progress.start(io, .{ .root_name = args.model }); - const tp_mesh: zml.sharding.LogicalMesh = try .init("tp_mesh", .{ .model = .high_bandwidth }); - const tp_strategy: zml.sharding.Strategy = try .suggest(tp_mesh, platform.physical_mesh); + const tp_mesh: zml.sharding.LogicalMesh = .init("tp_mesh", .{ .model = .high_bandwidth }); + const tp_strategy: zml.sharding.Strategy = .suggest(tp_mesh, platform.physical_mesh); const shardings: common.Shardings = .{ .replicated = try zml.sharding.replicatedSharding(platform), .model = try .initFromStrategy(platform, tp_mesh, tp_strategy), diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index e6631ade..d31eb91e 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -189,11 +189,11 @@ pub fn main(init: std.process.Init) !void { log.info("{f}", .{platform.physical_mesh}); const physical_mesh = platform.physical_mesh; - const mesh: zml.sharding.LogicalMesh = try .init("demo_mesh", .{ + const mesh: zml.sharding.LogicalMesh = .init("demo_mesh", .{ .data = .low_bandwidth, .model = .high_bandwidth, }); - const strategy: zml.sharding.Strategy = try .suggest(mesh, physical_mesh); + const strategy: zml.sharding.Strategy = .suggest(mesh, physical_mesh); const sharding: zml.sharding.Sharding = try .initFromStrategy(platform, mesh, strategy); log.info("{f}", .{mesh}); diff --git a/mlir/dialects/func.zig b/mlir/dialects/func.zig index 3a63eb01..e9246ec3 100644 --- a/mlir/dialects/func.zig +++ b/mlir/dialects/func.zig @@ -22,10 +22,10 @@ const FuncOpArgs = struct { }; pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { - var args_buffer: stdx.BoundedArray(*const mlir.Type, 1024) = .{}; - var results_buffer: stdx.BoundedArray(*const mlir.Type, 32) = .{}; + var args_buffer: stdx.BoundedArray(*const mlir.Type, 1024) = .empty; + var results_buffer: stdx.BoundedArray(*const mlir.Type, 32) = .empty; - var attr_tuples_buffer: stdx.BoundedArray(mlir.NamedAttribute, 16) = .{}; + var attr_tuples_buffer: stdx.BoundedArray(mlir.NamedAttribute, 16) = .empty; attr_tuples_buffer.appendSliceAssumeCapacity(&.{ .named(ctx, "sym_name", mlir.stringAttribute(ctx, args.name)), .named(ctx, "sym_visibility", mlir.stringAttribute(ctx, @tagName(args.visibility))), diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 4cbc9187..180cc2b0 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -759,7 +759,7 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result break :blk ret; }; - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 32) = .{}; + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 32) = .empty; attrs.appendSliceAssumeCapacity(&.{ .named(ctx, "call_target_name", mlir.stringAttribute(ctx, opts.call_target_name)), }); @@ -787,7 +787,7 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result } if (opts.output_operand_aliases) |output_operand_aliases| { - var buffer: stdx.BoundedArray(*const mlir.Attribute, MAX_RESULTS) = .{}; + var buffer: stdx.BoundedArray(*const mlir.Attribute, MAX_RESULTS) = .empty; for (output_operand_aliases) |alias| { const output_tuple_indices = if (result_types.len > 1) &[1]i64{alias.output_index} else &.{}; buffer.appendAssumeCapacity( @@ -799,7 +799,7 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result ); } - var layouts_buffer: stdx.BoundedArray(*const mlir.Attribute, MAX_OPERANDS) = .{}; + var layouts_buffer: stdx.BoundedArray(*const mlir.Attribute, MAX_OPERANDS) = .empty; if (opts.operand_layouts) |layouts| { for (layouts) |ol| { layouts_buffer.appendAssumeCapacity( diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 14376974..2cd086b4 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -1169,7 +1169,7 @@ pub const Operation = opaque { if (args.operands) |operands| switch (operands) { .flat => |v| state.addOperands(v), .variadic => |segments| { - var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; + var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .empty; for (segments) |segment_operands| { state.addOperands(segment_operands); sizes.appendAssumeCapacity(@intCast(segment_operands.len)); @@ -1182,7 +1182,7 @@ pub const Operation = opaque { if (args.results) |results| switch (results) { .flat => |v| state.addResults(v), .variadic => |segments| { - var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; + var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .empty; for (segments) |segment_results| { state.addResults(segment_results); sizes.appendAssumeCapacity(@intCast(segment_results.len)); @@ -1393,7 +1393,7 @@ pub const RankedTensorType = opaque { } pub fn fromShaped(other: *const ShapedType) *const RankedTensorType { - var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .{}; + var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .empty; for (0..other.rank()) |i| { dims.appendAssumeCapacity(other.dimension(i)); } @@ -1618,7 +1618,7 @@ pub const MemRefType = opaque { } pub fn fromShaped(other: *const ShapedType) *const MemRefType { - var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .{}; + var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .empty; for (0..other.rank()) |i| { dims.appendAssumeCapacity(other.dimension(i)); } diff --git a/stdx/bounded_array.zig b/stdx/bounded_array.zig index 81871615..3f00f89e 100644 --- a/stdx/bounded_array.zig +++ b/stdx/bounded_array.zig @@ -27,12 +27,16 @@ pub fn BoundedArray(comptime T: type, comptime buffer_capacity: usize) type { /// only known at runtime, but whose maximum size is known at comptime, without /// requiring an `Allocator`. /// ```zig -// var a = try BoundedArrayAligned(u8, 16, 2).init(0); -// try a.append(255); -// try a.append(255); -// const b = @ptrCast(*const [1]u16, a.constSlice().ptr); -// try testing.expectEqual(@as(u16, 65535), b[0]); +/// var a: BoundedArrayAligned(u8, 16, 2) = .empty; +/// try a.append(255); +/// try a.append(255); +/// const b: *const [1]u16 = @ptrCast(a.constSlice().ptr); +/// try testing.expectEqual(@as(u16, 65535), b[0]); /// ``` +/// +/// Note: there is no init anymore. It's generally unsafe because it makes `slice()` return undefined.; +/// If you used `try init(0)` use `.empty` +/// If you used `try init(n)` use `.{ .buffer = undefined, .len = n }` and fill `buffer` yourself. pub fn BoundedArrayAligned( comptime T: type, comptime alignment: Alignment, @@ -40,15 +44,10 @@ pub fn BoundedArrayAligned( ) type { return struct { const Self = @This(); - buffer: [buffer_capacity]T align(alignment.toByteUnits()) = undefined, - len: usize = 0, + buffer: [buffer_capacity]T align(alignment.toByteUnits()), + len: usize, - /// Set the actual length of the slice. - /// Returns error.Overflow if it exceeds the length of the backing array. - pub fn init(len: usize) error{Overflow}!Self { - if (len > buffer_capacity) return error.Overflow; - return Self{ .len = len }; - } + pub const empty: Self = .{ .buffer = undefined, .len = 0 }; /// View the internal array as a slice whose size was previously set. pub fn slice(self: anytype) switch (@TypeOf(&self.buffer)) { @@ -78,8 +77,10 @@ pub fn BoundedArrayAligned( /// Copy the content of an existing slice. pub fn fromSlice(m: []const T) error{Overflow}!Self { - var list = try init(m.len); - @memcpy(list.slice(), m); + if (m.len > buffer_capacity) return error.Overflow; + + var list: Self = .{ .buffer = undefined, .len = m.len }; + @memcpy(list.buffer[0..m.len], m); return list; } @@ -294,7 +295,7 @@ pub fn BoundedArrayAligned( } test BoundedArray { - var a = try BoundedArray(u8, 64).init(32); + var a: BoundedArray(u8, 64) = .{ .buffer = undefined, .len = 32 }; try testing.expectEqual(a.capacity(), 64); try testing.expectEqual(a.slice().len, 32); @@ -303,8 +304,8 @@ test BoundedArray { try a.resize(48); try testing.expectEqual(a.len, 48); - const x = [_]u8{1} ** 10; - a = try BoundedArray(u8, 64).fromSlice(&x); + const x: [10]u8 = @splat(1); + a = try .fromSlice(&x); try testing.expectEqualSlices(u8, &x, a.constSlice()); var a2 = a; @@ -399,7 +400,7 @@ test BoundedArray { } test "BoundedArrayAligned" { - var a = try BoundedArrayAligned(u8, .@"16", 4).init(0); + var a: BoundedArrayAligned(u8, .@"16", 4) = .empty; try a.append(0); try a.append(0); try a.append(255); diff --git a/zml/buffer.zig b/zml/buffer.zig index dd108ad0..f97f7d7e 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -101,7 +101,7 @@ pub const Buffer = struct { ._platform = platform, ._shape = sh, ._sharding = sharding, - ._shards = .{}, + ._shards = .empty, }; const buffer_type = pjrtx.bufferTypeFromDtype(sh.dtype()); @@ -193,7 +193,7 @@ pub const Buffer = struct { ._platform = platform, ._shape = sh, ._sharding = sharding, - ._shards = .{}, + ._shards = .empty, }; errdefer for (res._shards.slice()) |shard| { shard.deinit(platform.pjrt_api); diff --git a/zml/exe.zig b/zml/exe.zig index 9d93f66d..254921f0 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -224,7 +224,7 @@ pub const Exe = struct { var context: LocalContext = .{ .self = self, .current_index = 0 }; meta.visit(struct { fn cb(context_: *LocalContext, buffer: *Buffer) void { - var shards: Buffer.Shards = .{}; + var shards: Buffer.Shards = .empty; for (0..context_.self.flat_buffers.num_devices) |device_index| { shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); } @@ -244,7 +244,7 @@ pub const Exe = struct { meta.visit(struct { fn cb(context_: *LocalContext, buffer: *Buffer) void { //stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected result {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); - var shards: Buffer.Shards = .{}; + var shards: Buffer.Shards = .empty; for (0..context_.self.flat_buffers.num_devices) |device_index| { shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); } diff --git a/zml/io.zig b/zml/io.zig index 5cc755e4..1d43b2d4 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -795,7 +795,7 @@ pub const DirectMemoryWriter = struct { writer.deinit(); }; - var pjrt_buffers: Buffer.Shards = .{}; + var pjrt_buffers: Buffer.Shards = .empty; for (placement.shards.constSlice(), 0..) |shard, i| { defer initialized += 1; @@ -1374,10 +1374,10 @@ test "DirectMemoryWriter: replicated with auto topology" { }, .shape = Shape.init(.{ .rows = 8, .cols = 128 }, .f32) .withPartitioning(.{ .rows = .replicated, .cols = .replicated }), - .logical_mesh = try .init("replicated_cpu", .{ .x = .high_bandwidth }), + .logical_mesh = .init("replicated_cpu", .{ .x = .high_bandwidth }), .strategy = blk: { var strategy: sharding_.Strategy = .init; - try strategy.addBinding(.x, .link_x); + strategy.addBinding(.x, .link_x); break :blk strategy; }, @@ -1398,10 +1398,10 @@ test "DirectMemoryWriter: 1D model split with 2x2 physical mesh" { }, .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32) .withPartitioning(.{ .rows = .replicated, .cols = .model }), - .logical_mesh = try .init("model_cpu", .{ .model = .high_bandwidth }), + .logical_mesh = .init("model_cpu", .{ .model = .high_bandwidth }), .strategy = blk: { var strategy: sharding_.Strategy = .init; - try strategy.addBinding(.model, .link_x); + strategy.addBinding(.model, .link_x); break :blk strategy; }, @@ -1422,14 +1422,14 @@ test "DirectMemoryWriter: 2D batch/model split with 2x2 physical mesh" { }, .shape = Shape.init(.{ .batch = 8, .model = 1024 }, .f32) .withPartitioning(.{ .batch = .batch, .model = .model }), - .logical_mesh = try .init("batch_model_cpu", .{ + .logical_mesh = .init("batch_model_cpu", .{ .batch = .low_bandwidth, .model = .high_bandwidth, }), .strategy = blk: { var strategy: sharding_.Strategy = .init; - try strategy.addBinding(.batch, .link_x); - try strategy.addBinding(.model, .link_y); + strategy.addBinding(.batch, .link_x); + strategy.addBinding(.model, .link_y); break :blk strategy; }, @@ -1449,11 +1449,11 @@ test "DirectMemoryWriter: folded model sharding with 2x2 physical mesh" { .cpu = .{ .device_count = 4 }, }, .shape = Shape.init(.{ .model = 4096 }, .f32).withPartitioning(.{ .model = .model }), - .logical_mesh = try .init("model_folded_cpu", .{ .model = .high_bandwidth }), + .logical_mesh = .init("model_folded_cpu", .{ .model = .high_bandwidth }), .strategy = blk: { var strategy: sharding_.Strategy = .init; - try strategy.addBinding(.model, .link_x); - try strategy.addFold(.link_x, &.{ .link_x, .link_y }); + strategy.addBinding(.model, .link_x); + strategy.addFold(.link_x, &.{ .link_x, .link_y }); break :blk strategy; }, @@ -1474,10 +1474,10 @@ test "DirectMemoryWriter: writableSliceGreedy with mirrored shards" { }, .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32) .withPartitioning(.{ .rows = .replicated, .cols = .model }), - .logical_mesh = try .init("model_cpu_greedy", .{ .model = .high_bandwidth }), + .logical_mesh = .init("model_cpu_greedy", .{ .model = .high_bandwidth }), .strategy = blk: { var strategy: sharding_.Strategy = .init; - try strategy.addBinding(.model, .link_x); + strategy.addBinding(.model, .link_x); break :blk strategy; }, @@ -1501,14 +1501,14 @@ test "DirectMemoryWriter: 3D topology folded model + replicated batch" { }, .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32) .withPartitioning(.{ .batch = .replicated, .model = .model }), - .logical_mesh = try .init("topology_3d_folded_model_mesh", .{ + .logical_mesh = .init("topology_3d_folded_model_mesh", .{ .batch = .low_bandwidth, .model = .high_bandwidth, }), .strategy = blk: { var strategy: sharding_.Strategy = .init; - try strategy.addBinding(.model, .link_x); - try strategy.addFold(.link_x, &.{ .link_x, .link_z }); + strategy.addBinding(.model, .link_x); + strategy.addFold(.link_x, &.{ .link_x, .link_z }); break :blk strategy; }, diff --git a/zml/module.zig b/zml/module.zig index 6b141ad8..61f32bae 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -91,7 +91,7 @@ pub const CompilationContext = struct { platform: *const Platform, partitioning: Partitioning, - scopes: stdx.BoundedArray(Scope, 16) = .{}, + scopes: stdx.BoundedArray(Scope, 16) = .empty, manual_computation_depth: usize = 0, threadlocal var _current: ?*CompilationContext = null; @@ -429,7 +429,7 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar errdefer input_info.deinit(compilation_context.allocator); const input_attributes = try arena.allocator().alloc(AttributeList, input_info.shapes.len); - @memset(input_attributes, .{}); + @memset(input_attributes, .empty); const output_info = b: { compilation_context.activate(); @@ -445,7 +445,7 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar errdefer output_info.deinit(compilation_context.allocator); const output_attributes = try arena.allocator().alloc(AttributeList, output_info.shapes.len); - @memset(output_attributes, .{}); + @memset(output_attributes, .empty); for (output_info.donations, 0..) |donation, index| if (donation) |argument_index| { input_attributes[argument_index].appendAssumeCapacity(.named(compilation_context.mlir_ctx, "tf.aliasing_output", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, index))); diff --git a/zml/ops.zig b/zml/ops.zig index 6de285ff..10cc47ea 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -88,7 +88,7 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func // To that order, we initialize `result` to `inputs`, then we use stdx.meta.visit, // to find the correct mlir.Value, but we first broadcast before creating the final // Tensor struct. - var broadcasting_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; + var broadcasting_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; for (0..constants.MAX_RANK) |i| { if (std.mem.indexOfScalar(i64, axes_, @intCast(i)) == null) { broadcasting_axes.append(@intCast(i)) catch unreachable; @@ -713,11 +713,11 @@ pub fn scatter( } const ScatterConfig = struct { - op_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .{}, - up_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .{}, - indices_batch_axes: Shape.DimsArray = .{}, - scatter_to_operand_axes: Shape.DimsArray = .{}, - updates_transpose: Shape.AxesArray = .{}, + op_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .empty, + up_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .empty, + indices_batch_axes: Shape.DimsArray = .empty, + scatter_to_operand_axes: Shape.DimsArray = .empty, + updates_transpose: Shape.AxesArray = .empty, }; const ScatterAxisKind = enum { batching, update_window, inserted_window, window_id }; @@ -728,11 +728,11 @@ fn scatterConfig( indices_per_axis: stdx.BoundedArray(Tensor, constants.MAX_RANK), indices_axes: Shape.TagsArray, ) ScatterConfig { - var op_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .{}; - var up_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .{}; - var indices_batch_axes: Shape.DimsArray = .{}; - var scatter_to_operand_axes: Shape.DimsArray = .{}; - var updates_transpose: Shape.AxesArray = .{}; + var op_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .empty; + var up_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .empty; + var indices_batch_axes: Shape.DimsArray = .empty; + var scatter_to_operand_axes: Shape.DimsArray = .empty; + var updates_transpose: Shape.AxesArray = .empty; const tagged_api = indices_axes.len > 0; const indices = indices_per_axis.get(0).shape(); @@ -878,12 +878,12 @@ fn scatterPrepareIndices( cfg.up_kind.buffer[update.axis(batch_tag)] = .window_id; old_scatter_to_op_axes.appendAssumeCapacity(batch_ax); } - cfg.indices_batch_axes = .{}; + cfg.indices_batch_axes = .empty; // Reorder the axes so that in indices_per_axis is ordered like in op if possible. // TODO: transpose updates if needed - var indices: stdx.BoundedArray(Tensor, constants.MAX_RANK) = .{}; - var scatter_to_op_axes: Shape.DimsArray = .{}; + var indices: stdx.BoundedArray(Tensor, constants.MAX_RANK) = .empty; + var scatter_to_op_axes: Shape.DimsArray = .empty; while (old_scatter_to_op_axes.len > 0) { const scatter_ax = std.sort.argMin(i64, old_scatter_to_op_axes.constSlice(), {}, std.sort.asc(i64)).?; @@ -927,7 +927,7 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "gather indices can't be broadcasted together {any}", .{idx_per_axis}); } - var idx_batch_axes: Shape.DimsArray = .{}; + var idx_batch_axes: Shape.DimsArray = .empty; var self_kind: stdx.BoundedArray(GatherAxisKind, constants.MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; @@ -950,7 +950,7 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, // compute res shape var res_shape = Shape.init(.{}, self.dtype()); - var res_kind: stdx.BoundedArray(GatherAxisKind, constants.MAX_RANK) = .{}; + var res_kind: stdx.BoundedArray(GatherAxisKind, constants.MAX_RANK) = .empty; for (self_kind.slice(), 0..) |kind, ax_usize| { const ax: u3 = @intCast(ax_usize); if (ax == idx_axes[0]) { @@ -980,7 +980,7 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, return self.dynamicSlice1d(idx_axes[0], .{ .start = idx_per_axis[0].asScalar(), .len = 1 }).reshape(res_shape); } - var slice_dims: Shape.DimsArray = .{}; + var slice_dims: Shape.DimsArray = .empty; for (self_kind.slice(), self.dims()) |k, d| { slice_dims.appendAssumeCapacity(switch (k) { .batching, .collapsed => 1, @@ -1015,7 +1015,7 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, } pub fn _collectAxes(T: type, bounded_array: stdx.BoundedArray(T, constants.MAX_RANK), value: T) stdx.BoundedArray(i64, constants.MAX_RANK) { - var res: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; + var res: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; for (bounded_array.constSlice(), 0..) |v, ax| { if (v == value) { res.appendAssumeCapacity(@intCast(ax)); @@ -2010,7 +2010,7 @@ test "CustomCallOutputOperandAliases maps named fields to indices" { } fn toUsize(values: anytype) stdx.BoundedArray(usize, constants.MAX_RANK) { - var res: stdx.BoundedArray(usize, constants.MAX_RANK) = .{}; + var res: stdx.BoundedArray(usize, constants.MAX_RANK) = .empty; for (values) |val| res.appendAssumeCapacity(@intCast(val)); return res; } diff --git a/zml/safetensors.zig b/zml/safetensors.zig index 272a864c..bbf3508a 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -434,7 +434,7 @@ pub fn parseSafetensors( const start: u64 = @intCast(offset_field.array.items[0].integer); const dtype = try stringToDtype(value.object.get("dtype").?.string); - var dims: Dims = .{}; + var dims: Dims = .empty; for (shape_field.items) |d| { dims.appendAssumeCapacity(d.integer); } diff --git a/zml/shape.zig b/zml/shape.zig index 56c724e0..01a2098b 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -156,9 +156,9 @@ pub const Shape = struct { const UnknownTags: TagsArray = .{ .len = 0, .buffer = [_]Tag{TagUnknown} ** constants.MAX_RANK }; _dtype: DataType, - _dims: DimsArray = .{}, + _dims: DimsArray = .empty, _tags: TagsArray = UnknownTags, - _partitioning: PartitionArray = .{}, + _partitioning: PartitionArray = .empty, pub fn parseDimensions(v: anytype) struct { DimsArray, TagsArray } { const T = @TypeOf(v); @@ -168,8 +168,8 @@ pub const Shape = struct { } if (comptime stdx.meta.isSliceOfAny(T, stdx.meta.isInteger)) { - var dims_ = DimsArray.init(0) catch unreachable; - var tags_ = TagsArray.init(0) catch unreachable; + var dims_: DimsArray = .empty; + var tags_: TagsArray = .empty; for (v) |d| { dims_.appendAssumeCapacity(@intCast(d)); tags_.appendAssumeCapacity(TagUnknown); @@ -178,8 +178,8 @@ pub const Shape = struct { } if (comptime stdx.meta.isStruct(T)) { - var dims_: DimsArray = .{}; - var tags_: TagsArray = .{}; + var dims_: DimsArray = .empty; + var tags_: TagsArray = .empty; inline for (std.meta.fields(T)) |field| { const fv = @field(v, field.name); if (comptime stdx.meta.isInteger(field.type)) { @@ -217,8 +217,8 @@ pub const Shape = struct { return self.parseAxes(self.tags()); } - var axes_ = AxesArray.init(0) catch unreachable; - var tags_ = TagsArray.init(0) catch unreachable; + var axes_: AxesArray = .empty; + var tags_: TagsArray = .empty; if (comptime stdx.meta.isSliceOfAny(T, isAxisConvertible)) { for (v) |d| { @@ -242,7 +242,7 @@ pub const Shape = struct { pub fn parseTags(v: anytype) TagsArray { const T = @TypeOf(v); stdx.debug.assertComptime(stdx.meta.isTupleOf(T, @EnumLiteral()), "Wrong type, got {}. Expected .{{ .a, .b }}", .{T}); - var tags_ = TagsArray.init(0) catch unreachable; + var tags_: TagsArray = .empty; inline for (v) |field| { tags_.appendAssumeCapacity(toTag(field)); } @@ -407,7 +407,7 @@ pub const Shape = struct { return self.axes(axes_.tags()); } - var res = AxesArray.init(0) catch unreachable; + var res: AxesArray = .empty; if (comptime stdx.meta.isSliceOfAny(T, stdx.meta.isInteger) or stdx.meta.isSliceOf(T, Tag)) { for (axes_) |ax| { @@ -917,7 +917,7 @@ pub const Shape = struct { } // Check that no mesh axis is used to partition multiple tensor dimensions. - var used_mesh_axes: stdx.BoundedArray(Tag, constants.MAX_RANK) = .{ .len = 0 }; + var used_mesh_axes: stdx.BoundedArray(Tag, constants.MAX_RANK) = .empty; for (res._partitioning.constSlice()) |spec| { const axis_tag = spec.toTag(); @@ -1249,7 +1249,7 @@ pub const Shape = struct { fn computeStrides(self: Shape, element_size: i64) stdx.BoundedArray(i64, constants.MAX_RANK) { const rk = self.rank(); - var strides: stdx.BoundedArray(i64, constants.MAX_RANK) = .{ .len = rk }; + var strides: stdx.BoundedArray(i64, constants.MAX_RANK) = .{ .buffer = undefined, .len = rk }; if (rk == 0) return strides; const V = @Vector(constants.MAX_RANK, i64); @@ -1269,7 +1269,7 @@ pub const Shape = struct { /// so that the given axes are contiguous. pub fn contiguousPerm(self: Shape, axes_: anytype) AxesArray { const axes__, _ = self.parseAxes(axes_); - var perms = AxesArray.init(0) catch unreachable; + var perms: AxesArray = .empty; for (0..self.rank()) |i| { if (std.mem.indexOfScalar(u3, axes__.constSlice(), @intCast(i))) |_| { continue; @@ -1493,7 +1493,7 @@ pub const Shape = struct { } fn intersectTags(a: []const Tag, b: []const Tag) TagsArray { - var res = TagsArray.init(0) catch unreachable; + var res: TagsArray = .empty; for (a) |tag_| { if (std.mem.indexOfScalar(Tag, b, tag_)) { res.appendAssumeCapacity(tag_); @@ -1505,8 +1505,8 @@ pub const Shape = struct { pub fn parseStruct(T: type, v: anytype) struct { stdx.BoundedArray(T, constants.MAX_RANK), TagsArray } { const V = @TypeOf(v); - var vals_: stdx.BoundedArray(T, constants.MAX_RANK) = .{}; - var tags_: TagsArray = .{}; + var vals_: stdx.BoundedArray(T, constants.MAX_RANK) = .empty; + var tags_: TagsArray = .empty; if (comptime stdx.meta.isSliceOf(V, T)) { for (v) |d| { @@ -1543,7 +1543,7 @@ pub const Shape = struct { pub fn parseAxesOptions(self: Shape, T: type, options: anytype, default: T) stdx.BoundedArray(T, constants.MAX_RANK) { const V = @TypeOf(options); - var res: stdx.BoundedArray(T, constants.MAX_RANK) = .{}; + var res: stdx.BoundedArray(T, constants.MAX_RANK) = .empty; if (comptime stdx.meta.isSliceOf(V, T)) { stdx.debug.assert(options.len == self.rank(), "expects exactly {} options in slice, for {} got {}", .{ self.rank(), self, options.len }); for (options) |d| { diff --git a/zml/sharding.zig b/zml/sharding.zig index 1c6a777c..3d7f8a10 100644 --- a/zml/sharding.zig +++ b/zml/sharding.zig @@ -194,7 +194,7 @@ pub const Device = struct { id: usize, /// Coordinates in the physical mesh - coords: stdx.BoundedArray(usize, Shape.MAX_RANK) = .{}, + coords: stdx.BoundedArray(usize, Shape.MAX_RANK) = .empty, /// Placeholder pjrt_device: ?*const pjrt.Device = null, @@ -279,7 +279,7 @@ pub const PhysicalNode = union(enum) { return .{ .leaf = .{ .id = @intCast(device_.id()), - .coords = .{}, + .coords = .empty, .pjrt_device = device_.pjrt_device, }, }; @@ -324,8 +324,8 @@ pub const PhysicalMesh = struct { depth_by_tag: DepthByTag, /// Build axis order and depth mapping from a topology tree. - pub fn init(root: PhysicalNode) !AxisTraversal { - var order: Order = try .init(0); + pub fn init(root: PhysicalNode) AxisTraversal { + var order: Order = .empty; axisOrderNode(root, &order); var depth_by_tag: DepthByTag = .initFill(null); @@ -415,17 +415,16 @@ pub const PhysicalMesh = struct { } fn fromOwnedTree(target: Target, root: PhysicalNode) !PhysicalMesh { + try validateGeometry(root); + var owned_root = root; - - try validateGeometry(owned_root); - - var path = [_]usize{0} ** Shape.MAX_RANK; - try assignCoords(&owned_root, &path, 0); + var path: [Shape.MAX_RANK]usize = @splat(0); + assignCoords(&owned_root, &path, 0); return .{ .target = target, .root = owned_root, - .axis_traversal = try .init(owned_root), + .axis_traversal = .init(owned_root), }; } @@ -513,20 +512,18 @@ pub const PhysicalMesh = struct { /// coords: /// (0,0) (0,1) /// (1,0) (1,1) - fn assignCoords(node: *PhysicalNode, path: *[Shape.MAX_RANK]usize, depth: usize) !void { + fn assignCoords(node: *PhysicalNode, path: *[Shape.MAX_RANK]usize, depth: usize) void { switch (node.*) { .leaf => |*d| { // Coords can already been set by caller if (d.coords.len > 0) return; - var coords = stdx.BoundedArray(usize, Shape.MAX_RANK).init(0) catch unreachable; - for (path[0..depth]) |c| coords.appendAssumeCapacity(c); - d.coords = coords; + d.coords = stdx.BoundedArray(usize, Shape.MAX_RANK).fromSlice(path[0..depth]) catch unreachable; }, .branch => |*b| { for (b.children, 0..) |*child, i| { path[depth] = i; - try assignCoords(child, path, depth + 1); + assignCoords(child, path, depth + 1); } }, } @@ -536,25 +533,18 @@ pub const PhysicalMesh = struct { return self.root.countDevices(); } - pub fn devices(self: PhysicalMesh, allocator: std.mem.Allocator) ![]Device { - var list: Devices = .init(0) catch unreachable; - try self.devicesInto(&list); - - const out = try allocator.alloc(Device, list.len); - @memcpy(out, list.constSlice()); - return out; + pub fn devices(self: PhysicalMesh) Devices { + var list: Devices = .empty; + devicesNodeInto(self.root, &list); + return list; } - pub fn devicesInto(self: PhysicalMesh, out: *Devices) !void { - try devicesNodeInto(self.root, out); - } - - fn devicesNodeInto(node: PhysicalNode, out: *Devices) !void { + fn devicesNodeInto(node: PhysicalNode, out: *Devices) void { switch (node) { .leaf => |d| out.appendAssumeCapacity(d), .branch => |b| { for (b.children) |child| { - try devicesNodeInto(child, out); + devicesNodeInto(child, out); } }, } @@ -692,8 +682,7 @@ pub const PhysicalMesh = struct { fn parseDevice(device: PlatformDevice) !CoordPlacement { const api = device.platform.pjrt_api; const coords_attr = device.pjrt_desc.attribute(api, "coords") orelse return error.MissingDeviceCoords; - var coords: stdx.BoundedArray(usize, Shape.MAX_RANK) = try .init(0); - + var coords: stdx.BoundedArray(usize, Shape.MAX_RANK) = .empty; switch (coords_attr) { .int64list => |values| { if (values.len == 0 or values.len > Shape.MAX_RANK) return error.InvalidDeviceCoords; @@ -782,20 +771,17 @@ pub const PhysicalMesh = struct { } pub fn leaf(placement: *const CoordPlacement, coords_slice: []const usize) !PhysicalNode { - var coords: stdx.BoundedArray(usize, Shape.MAX_RANK) = try .init(0); - for (coords_slice) |coord| coords.appendAssumeCapacity(coord); - return .{ .leaf = .{ .id = placement.device_id, - .coords = coords, + .coords = try .fromSlice(coords_slice), .pjrt_device = placement.pjrt_device, }, }; } pub fn axisStrides(axis_sizes: []const usize) ![Shape.MAX_RANK]usize { - var strides = [_]usize{0} ** Shape.MAX_RANK; + var strides: [Shape.MAX_RANK]usize = @splat(0); var stride: usize = 1; var i = axis_sizes.len; @@ -973,11 +959,11 @@ pub const LogicalMesh = struct { axes: Axes, intents: Intents, - pub fn init(name: []const u8, axes_: anytype) !LogicalMesh { + pub fn init(name: []const u8, axes_: anytype) LogicalMesh { const T = @TypeOf(axes_); - var axes: Axes = try .init(0); - var intents: Intents = try .init(0); + var axes: Axes = .empty; + var intents: Intents = .empty; inline for (std.meta.fields(T)) |field| { const value = @field(axes_, field.name); @@ -1099,16 +1085,12 @@ pub const Sharding = struct { const axis_order = physical.axisOrder().constSlice(); if (axis_order.len == 0) return error.InvalidPhysicalMesh; - var bindings: Bindings = try .init(0); + var bindings: Bindings = .empty; for (strategy.bindings.constSlice()) |bind| { - var list: AxisList = try .init(0); - for (bind.physical.constSlice()) |p_tag| { - list.appendAssumeCapacity(p_tag); - } - bindings.appendAssumeCapacity(.{ .logical = bind.logical, .physical = list }); + bindings.appendAssumeCapacity(.{ .logical = bind.logical, .physical = bind.physical }); } - var folds: Folds = try .init(0); + var folds: Folds = .empty; var folds_consumed: std.EnumSet(PhysicalAxisTag) = .empty; for (strategy.folding.constSlice()) |entry| { for (entry.sources.constSlice()) |src| { @@ -1137,7 +1119,7 @@ pub const Sharding = struct { pub fn physicalView(self: Sharding) PhysicalView { const physical = self.platform.physical_mesh; var view: PhysicalView = .{ - .axes = stdx.BoundedArray(Axis, Shape.MAX_RANK).init(0) catch unreachable, + .axes = .empty, .total_devices = 1, }; @@ -1147,7 +1129,7 @@ pub const Sharding = struct { if (!physical.hasAxis(tag)) continue; if (self.folds_consumed.contains(tag) and self.foldSources(tag) == null) continue; - var folded = stdx.BoundedArray(PhysicalAxisTag, Shape.MAX_RANK).init(0) catch unreachable; + var folded: stdx.BoundedArray(PhysicalAxisTag, Shape.MAX_RANK) = .empty; var size: i64 = 1; if (self.foldSources(tag)) |sources| { @@ -1240,12 +1222,12 @@ pub const Sharding = struct { /// Common logic to map tensor dimensions to physical mesh indices. fn getDimMapping(self: *const Sharding, shape: Shape) DimMapping { const view = self.physicalView(); - var axes_per_dim = stdx.BoundedArray(stdx.BoundedArray(usize, Shape.MAX_RANK), Shape.MAX_RANK).init(0) catch unreachable; - var used_mask = [_]bool{false} ** Shape.MAX_RANK; + var axes_per_dim: stdx.BoundedArray(stdx.BoundedArray(usize, Shape.MAX_RANK), Shape.MAX_RANK) = .empty; + var used_mask: [Shape.MAX_RANK]bool = @splat(false); var globally_used: std.EnumSet(PhysicalAxisTag) = .empty; for (0..shape.rank()) |ax| { - var dim_axes = stdx.BoundedArray(usize, Shape.MAX_RANK).init(0) catch unreachable; + var dim_axes: stdx.BoundedArray(usize, Shape.MAX_RANK) = .empty; const spec = shape.partition(ax); if (spec == .axis) { @@ -1265,7 +1247,7 @@ pub const Sharding = struct { axes_per_dim.appendAssumeCapacity(dim_axes); } - var replicated_axes = stdx.BoundedArray(usize, Shape.MAX_RANK).init(0) catch unreachable; + var replicated_axes: stdx.BoundedArray(usize, Shape.MAX_RANK) = .empty; for (0..view.axes.len) |i| { if (!used_mask[i]) replicated_axes.appendAssumeCapacity(i); } @@ -1349,7 +1331,7 @@ pub const Sharding = struct { if (!has_sharding) return try allocator.dupe(u8, "{replicated}"); const mapping = self.getDimMapping(shape); - var tile_shape = stdx.BoundedArray(i64, Shape.MAX_RANK + 1).init(0) catch unreachable; + var tile_shape: stdx.BoundedArray(i64, Shape.MAX_RANK + 1) = .empty; // Calculate tile sizes per tensor dimension for (mapping.axes_per_dim.constSlice()) |dim_axes| { @@ -1409,7 +1391,7 @@ pub const Sharding = struct { var ids = try allocator.alloc(usize, count); @memset(ids, std.math.maxInt(usize)); - const ordered_devices = try self.devicesInCanonicalOrder(); + const ordered_devices = self.devicesInCanonicalOrder(); for (ordered_devices.constSlice()) |d| { const coords = d.coordsSlice() orelse return error.MissingDeviceCoords; const idx = self.platform.physical_mesh.linearIndexFromCoords(coords); @@ -1423,10 +1405,9 @@ pub const Sharding = struct { return ids; } - fn devicesInCanonicalOrder(self: Sharding) !Devices { + fn devicesInCanonicalOrder(self: Sharding) Devices { const physical = self.platform.physical_mesh; - var devices: Devices = try .init(0); - try physical.devicesInto(&devices); + var devices: Devices = physical.devices(); const Order = struct { mesh: PhysicalMesh, @@ -1498,8 +1479,8 @@ pub const Sharding = struct { pub fn replicatedSharding(platform: *const Platform) !Sharding { const physical_mesh = platform.physical_mesh; - const logical_mesh: LogicalMesh = try .init("replicated", .{ .x = .high_bandwidth }); - const strategy: Strategy = try .suggest(logical_mesh, physical_mesh); + const logical_mesh: LogicalMesh = .init("replicated", .{ .x = .high_bandwidth }); + const strategy: Strategy = .suggest(logical_mesh, physical_mesh); return try .initFromStrategy(platform, logical_mesh, strategy); } @@ -1520,11 +1501,11 @@ pub const Strategy = struct { folding: Folding, pub const init: Strategy = .{ - .bindings = .{}, - .folding = .{}, + .bindings = .empty, + .folding = .empty, }; - pub fn addBinding(self: *Strategy, logical: anytype, physical: PhysicalAxisTag) !void { + pub fn addBinding(self: *Strategy, logical: anytype, physical: PhysicalAxisTag) void { const raw_tag = Shape.toTag(logical); const logical_tag: []const u8 = std.mem.span(raw_tag); @@ -1535,15 +1516,15 @@ pub const Strategy = struct { } } - var list: PhysicalList = try .init(0); + var list: PhysicalList = .empty; list.appendAssumeCapacity(physical); self.bindings.appendAssumeCapacity(.{ .logical = raw_tag, .physical = list }); } /// Explicitly fold axes: `target` is the kept axis, `sources` define order. /// If `target` is missing from `sources`, it is prepended. - pub fn addFold(self: *Strategy, target: PhysicalAxisTag, sources: []const PhysicalAxisTag) !void { - var list: PhysicalList = try .init(0); + pub fn addFold(self: *Strategy, target: PhysicalAxisTag, sources: []const PhysicalAxisTag) void { + var list: PhysicalList = .empty; var has_target = false; for (sources) |s| { @@ -1567,11 +1548,11 @@ pub const Strategy = struct { } /// suggest builds a Strategy from logical intents. - pub fn suggest(logical: LogicalMesh, physical: PhysicalMesh) !Strategy { + pub fn suggest(logical: LogicalMesh, physical: PhysicalMesh) Strategy { var strategy: Strategy = .init; const base_order = physical.shardableAxes(); - var available = stdx.BoundedArray(PhysicalAxisTag, Shape.MAX_RANK).init(0) catch unreachable; + var available: stdx.BoundedArray(PhysicalAxisTag, Shape.MAX_RANK) = .empty; for (base_order) |tag| { if (physical.hasAxis(tag)) available.appendAssumeCapacity(tag); } @@ -1594,7 +1575,7 @@ pub const Strategy = struct { .low_bandwidth => avail[avail.len - 1 - (i % avail.len)], }; - try strategy.addBinding(l_tag, p_tag); + strategy.addBinding(l_tag, p_tag); counters.set(target_intent, i + 1); } } @@ -1667,12 +1648,16 @@ pub const Placement = struct { counts: stdx.BoundedArray(i64, Shape.MAX_RANK), indices: stdx.BoundedArray(i64, Shape.MAX_RANK), - pub fn init() !AxisSplit { - return .{ - .product = 1, - .counts = try .init(0), - .indices = try .init(0), - }; + pub const empty: AxisSplit = .{ + .product = 1, + .counts = .empty, + .indices = .empty, + }; + + pub fn add(split: *AxisSplit, size: i64, coord: usize) void { + split.counts.appendAssumeCapacity(size); + split.indices.appendAssumeCapacity(@intCast(coord)); + split.product *= size; } pub fn linearIndex(self: AxisSplit) i64 { @@ -1692,9 +1677,8 @@ pub const Placement = struct { sharding: Sharding, shape: Shape, ) !Placement { - const ordered_devices = try sharding.devicesInCanonicalOrder(); - - var shards: Shards = try .init(0); + const ordered_devices = sharding.devicesInCanonicalOrder(); + var shards: Shards = .empty; for (ordered_devices.constSlice()) |d| { if (d.coords.len == 0) return error.MissingDeviceCoords; @@ -1704,7 +1688,7 @@ pub const Placement = struct { .device_coords = d.coords, .shape = undefined, // Calculated below .global_shape = shape, - .slices = try .init(0), + .slices = .empty, }); } @@ -1771,20 +1755,20 @@ pub const Placement = struct { device_coords: []const usize, used_axes: *std.EnumSet(PhysicalAxisTag), ) !AxisSplit { - var plan: AxisSplit = try .init(); + var plan: AxisSplit = .empty; for (binding) |p_tag| { // Expand the physical tag: check if it's a target for folded source axes if (self.sharding.foldSources(p_tag)) |sources| { for (sources) |src| { if (used_axes.contains(src)) continue; - try self.addPhysicalToSplit(&plan, src, device_coords); + self.addPhysicalToSplit(&plan, src, device_coords); used_axes.insert(src); } } else { // Standard case: directly bound, not part of an explicit fold if (used_axes.contains(p_tag)) continue; - try self.addPhysicalToSplit(&plan, p_tag, device_coords); + self.addPhysicalToSplit(&plan, p_tag, device_coords); used_axes.insert(p_tag); } } @@ -1797,15 +1781,13 @@ pub const Placement = struct { } /// Extract coordinate data from the physical mesh for a specific axis. - fn addPhysicalToSplit(self: *Placement, plan: *AxisSplit, tag: PhysicalAxisTag, device_coords: []const usize) !void { + fn addPhysicalToSplit(self: *Placement, plan: *AxisSplit, tag: PhysicalAxisTag, device_coords: []const usize) void { const physical = self.sharding.platform.physical_mesh; const info = physical.axisInfo(tag) orelse return; const depth = physical.axis_traversal.depth(tag) orelse return; const coord = device_coords[depth]; - plan.counts.appendAssumeCapacity(@intCast(info.size)); - plan.indices.appendAssumeCapacity(@intCast(coord)); - plan.product *= info.size; + plan.add(info.size, coord); } pub fn format(self: Placement, writer: *std.Io.Writer) !void { @@ -1872,17 +1854,15 @@ const ShardingTest = struct { /// Creates a 1D-3D PhysicalMesh from simple dimensions. pub fn physical(self: ShardingTest, dims: anytype, geometry: AxisGeometry) !PhysicalMesh { const info = @typeInfo(@TypeOf(dims)).@"struct"; - var tags = try stdx.BoundedArray(PhysicalAxisTag, 3).init(0); - var sizes = try stdx.BoundedArray(usize, 3).init(0); - - const tag_names = [_]PhysicalAxisTag{ .link_x, .link_y, .link_z }; - inline for (info.fields, 0..) |field, i| { - tags.appendAssumeCapacity(tag_names[i]); - sizes.appendAssumeCapacity(@intCast(@field(dims, field.name))); + const N = info.fields.len; + var sizes: [N]usize = undefined; + inline for (info.fields, sizes[0..]) |field, *s| { + s.* = @intCast(@field(dims, field.name)); } + const tags: [3]PhysicalAxisTag = .{ .link_x, .link_y, .link_z }; var next_id: usize = 0; - const root = try self.buildNode(tags.constSlice(), sizes.constSlice(), 0, &next_id, geometry); + const root = try self.buildNode(tags[0..N], sizes[0..N], 0, &next_id, geometry); return try .fromTree(self.allocator, .tpu, root); } @@ -1952,10 +1932,10 @@ test "sharding: unknown partitioning implies replication" { const runner: ShardingTest = .init(arena.allocator()); const physical: PhysicalMesh = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); - const logical: LogicalMesh = try .init("folded_mesh", .{ .batch = .low_bandwidth }); + const logical: LogicalMesh = .init("folded_mesh", .{ .batch = .low_bandwidth }); var strategy: Strategy = .init; - try strategy.addBinding(.batch, .link_x); + strategy.addBinding(.batch, .link_x); const scenario: ShardingTest.Scenario = .{ .platform = runner.platformForMesh(physical), @@ -1980,12 +1960,12 @@ test "sharding: suggest strategy realization" { const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); - const logical: LogicalMesh = try .init("suggested_mesh", .{ + const logical: LogicalMesh = .init("suggested_mesh", .{ .batch = .low_bandwidth, .model = .high_bandwidth, }); - const strategy: Strategy = try .suggest(logical, physical); + const strategy: Strategy = .suggest(logical, physical); const scenario: ShardingTest.Scenario = .{ .platform = runner.platformForMesh(physical), @@ -2015,12 +1995,12 @@ test "sharding: suggest folds logical axes with same intent" { const physical = try runner.physical(.{4}, .point_to_point); - const logical: LogicalMesh = try .init("fold_mesh", .{ + const logical: LogicalMesh = .init("fold_mesh", .{ .model = .high_bandwidth, .experts = .high_bandwidth, }); - const strategy: Strategy = try .suggest(logical, physical); + const strategy: Strategy = .suggest(logical, physical); const scenario: ShardingTest.Scenario = .{ .platform = runner.platformForMesh(physical), @@ -2045,11 +2025,11 @@ test "sharding: multiple physical axes on one logical dimension (folding)" { const runner: ShardingTest = .init(arena.allocator()); const physical: PhysicalMesh = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); - const logical: LogicalMesh = try .init("folded_mesh", .{ .model = .high_bandwidth }); + const logical: LogicalMesh = .init("folded_mesh", .{ .model = .high_bandwidth }); var strategy: Strategy = .init; - try strategy.addBinding(.model, .link_x); - try strategy.addBinding(.model, .link_y); + strategy.addBinding(.model, .link_x); + strategy.addBinding(.model, .link_y); const scenario: ShardingTest.Scenario = .{ .platform = runner.platformForMesh(physical), @@ -2073,11 +2053,11 @@ test "sharding: explicit strategy folding" { const runner: ShardingTest = .init(arena.allocator()); const physical: PhysicalMesh = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); - const logical: LogicalMesh = try .init("strategy_fold", .{ .model = .high_bandwidth }); + const logical: LogicalMesh = .init("strategy_fold", .{ .model = .high_bandwidth }); var strategy: Strategy = .init; - try strategy.addBinding(.model, .link_x); - try strategy.addFold(.link_x, &.{ .link_x, .link_z }); + strategy.addBinding(.model, .link_x); + strategy.addFold(.link_x, &.{ .link_x, .link_z }); const scenario: ShardingTest.Scenario = .{ .platform = runner.platformForMesh(physical), @@ -2105,11 +2085,11 @@ test "sharding: open and replicated dimension mix" { const runner: ShardingTest = .init(arena.allocator()); const physical: PhysicalMesh = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); - const logical: LogicalMesh = try .init("mix_mesh", .{ .batch = .low_bandwidth, .model = .high_bandwidth }); + const logical: LogicalMesh = .init("mix_mesh", .{ .batch = .low_bandwidth, .model = .high_bandwidth }); var strategy: Strategy = .init; - try strategy.addBinding(.batch, .link_x); - try strategy.addBinding(.model, .link_y); + strategy.addBinding(.batch, .link_x); + strategy.addBinding(.model, .link_y); const scenario: ShardingTest.Scenario = .{ .platform = runner.platformForMesh(physical), @@ -2133,12 +2113,12 @@ test "sharding: full 3D cluster sharding" { const runner: ShardingTest = .init(arena.allocator()); const physical: PhysicalMesh = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); - const logical: LogicalMesh = try .init("3d_mesh", .{ .batch = .low_bandwidth, .model = .high_bandwidth, .context = .balanced }); + const logical: LogicalMesh = .init("3d_mesh", .{ .batch = .low_bandwidth, .model = .high_bandwidth, .context = .balanced }); var strategy: Strategy = .init; - try strategy.addBinding(.batch, .link_x); - try strategy.addBinding(.model, .link_y); - try strategy.addBinding(.context, .link_z); + strategy.addBinding(.batch, .link_x); + strategy.addBinding(.model, .link_y); + strategy.addBinding(.context, .link_z); const scenario: ShardingTest.Scenario = .{ .platform = runner.platformForMesh(physical), @@ -2167,14 +2147,14 @@ test "sharding: num partitions for logical axis" { const runner: ShardingTest = .init(arena.allocator()); const physical: PhysicalMesh = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); - const logical: LogicalMesh = try .init("axis_parts_mesh", .{ + const logical: LogicalMesh = .init("axis_parts_mesh", .{ .model = .high_bandwidth, .batch = .low_bandwidth, }); var strategy: Strategy = .init; - try strategy.addBinding(.model, .link_x); - try strategy.addFold(.link_x, &.{ .link_x, .link_z }); + strategy.addBinding(.model, .link_x); + strategy.addFold(.link_x, &.{ .link_x, .link_z }); var platform = runner.platformForMesh(physical); const sharding: Sharding = try .initFromStrategy(&platform, logical, strategy); diff --git a/zml/tensor.zig b/zml/tensor.zig index ab1099e5..f86bb9b6 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1143,7 +1143,7 @@ pub const Tensor = struct { const lhs_contracting_dim: i8 = @intCast(lhs.shape().hasTag(args).?); const rhs_contracting_dim: i8 = @intCast(rhs.shape().hasTag(args).?); - var batching_axes: stdx.BoundedArray([2]i8, constants.MAX_RANK) = .{}; + var batching_axes: stdx.BoundedArray([2]i8, constants.MAX_RANK) = .empty; for (0..lhs.rank()) |lhs_tag_index| { const lhs_tag = lhs.shape().tag(lhs_tag_index); if (lhs_tag == Shape.toTag(args)) continue; @@ -1221,8 +1221,8 @@ pub const Tensor = struct { var res_shape: Shape = .{ ._dtype = lhs.dtype() }; // Validate batching axes - var lhs_batching_axes: Axes = .{}; - var rhs_batching_axes: Axes = .{}; + var lhs_batching_axes: Axes = .empty; + var rhs_batching_axes: Axes = .empty; for (batching_axes) |b_axes| { const l, const r = b_axes; stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects batching dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); @@ -1234,8 +1234,8 @@ pub const Tensor = struct { } // Validate contracting axes - var lhs_contracting_axes: Axes = .{}; - var rhs_contracting_axes: Axes = .{}; + var lhs_contracting_axes: Axes = .empty; + var rhs_contracting_axes: Axes = .empty; for (contracting_axes) |c_axes| { const l, const r = c_axes; stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects contracting dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); @@ -1531,7 +1531,7 @@ pub const Tensor = struct { pub fn swapAxes(self: Tensor, a: anytype, b: anytype) Tensor { if (self.axis(a) == self.axis(b)) return self; - var perm: Shape.AxesArray = .{}; + var perm: Shape.AxesArray = .empty; for (0..self.rank()) |i| { perm.appendAssumeCapacity(@intCast(i)); } @@ -1671,7 +1671,7 @@ pub const Tensor = struct { ).appendTo(currentBlock()); var res = _result(res_shape, slice_op.result(0)); - var to_remove: Shape.AxesArray = .{}; + var to_remove: Shape.AxesArray = .empty; for (slices, 0..) |s, a| { if (s.singleton) to_remove.appendAssumeCapacity(@intCast(a)); } @@ -2188,7 +2188,7 @@ pub const Tensor = struct { } // check that each axis of self maps to an axis of other - var axes_: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; + var axes_: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; for (self._shape.tags()) |t| { axes_.appendAssumeCapacity(@intCast(other.axis(t))); } @@ -2324,7 +2324,7 @@ pub const Tensor = struct { /// so that gather can pub fn gather(self: Tensor, _indices: anytype, opts: GatherOpts) Tensor { const idx_per_axis, const idx_tags = Shape.parseStruct(Tensor, _indices); - var idx_axes: Shape.AxesArray = .{}; + var idx_axes: Shape.AxesArray = .empty; for (idx_tags.slice()) |t| { idx_axes.appendAssumeCapacity(self.axis(t)); } @@ -2439,10 +2439,10 @@ pub const Tensor = struct { // Compute result shape var res_shape = indices._shape.remove(index_coord_axis).withDtype(self.dtype()); var slice_dims = self._shape._dims; - var self_batch_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; - var indices_batch_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; - var start_index_map: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; - var self_offset_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; + var self_batch_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; + var indices_batch_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; + var start_index_map: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; + var self_offset_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; for (self._shape.tags(), 0..self.rank()) |t, self_ax| { const maybe_slice_ax: ?u3 = if (tagged_api) slice_shape.hasTag(t) else @intCast(self_ax); @@ -4038,7 +4038,7 @@ pub const Tensor = struct { /// /// - res[a, b, c, d] == (A[a], B[b], C[c], D[d]) pub fn cartesianProductStacked(vectors: []const Tensor) Tensor { - var out = stdx.BoundedArray(Tensor, constants.MAX_RANK).init(vectors.len) catch unreachable; + var out: stdx.BoundedArray(Tensor, constants.MAX_RANK) = .{ .buffer = undefined, .len = vectors.len }; _cartesianProduct(vectors, out.slice()); return Tensor.stack(out.constSlice(), .last, .coord); @@ -4287,13 +4287,13 @@ fn _parseGatherCoord(self: Tensor, axes_: anytype) struct { bool, stdx.BoundedAr } fn toI64(values: anytype) stdx.BoundedArray(i64, constants.MAX_RANK) { - var res: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; + var res: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; for (values) |val| res.appendAssumeCapacity(@intCast(val)); return res; } fn transposeIsJustAReshape(x: Shape, permutation: []const i64) bool { - var perm: stdx.BoundedArray(struct { u8, bool }, constants.MAX_RANK) = .{}; + var perm: stdx.BoundedArray(struct { u8, bool }, constants.MAX_RANK) = .empty; // Don't rewrite on invalid inputs. if (permutation.len > x.rank()) return false; for (permutation) |ax| { diff --git a/zml/tokenizer/hftokenizers/hftokenizers.zig b/zml/tokenizer/hftokenizers/hftokenizers.zig index f5829ce6..4b845788 100644 --- a/zml/tokenizer/hftokenizers/hftokenizers.zig +++ b/zml/tokenizer/hftokenizers/hftokenizers.zig @@ -40,8 +40,8 @@ pub const Decoder = struct { inner: *HFTokenizer, current_string: ?[]const u8 = null, - last_string: StringBuffer = .{ .len = 0 }, - last_token_ids: TokensIdsBuffer = .{ .len = 0 }, + last_string: StringBuffer = .empty, + last_token_ids: TokensIdsBuffer = .empty, fn init(inner: *HFTokenizer) Decoder { return .{ .inner = inner }; diff --git a/zml/tokenizer/homemade.zig b/zml/tokenizer/homemade.zig index 602c6956..079529b3 100644 --- a/zml/tokenizer/homemade.zig +++ b/zml/tokenizer/homemade.zig @@ -46,7 +46,7 @@ pub const Tokenizer = struct { errdefer arena_state.deinit(); const arena = arena_state.allocator(); - var token_lookup: std.StringHashMapUnmanaged(u32) = .{}; + var token_lookup: std.StringHashMapUnmanaged(u32) = .empty; errdefer token_lookup.deinit(arena); try token_lookup.ensureTotalCapacity(arena, @intCast(vocab_size)); @@ -442,8 +442,8 @@ pub const Decoder = struct { arena: std.heap.ArenaAllocator, current_string: ?[]const u8 = null, - last_string: StringBuffer = .{ .len = 0 }, - last_token_ids: TokensIdsBuffer = .{ .len = 0 }, + last_string: StringBuffer = .empty, + last_token_ids: TokensIdsBuffer = .empty, fn init(inner: *Tokenizer) !Decoder { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); @@ -566,7 +566,7 @@ test CharTokenIterator { { tokenizer.byte_fallback = false; var it: CharTokenIterator = .{ .input = "ζℳL" }; - var res: stdx.BoundedArray(u32, 8) = .{}; + var res: stdx.BoundedArray(u32, 8) = .empty; while (try it.nextCodepointToken(&tokenizer)) |token| { res.appendAssumeCapacity(token); } @@ -577,7 +577,7 @@ test CharTokenIterator { { tokenizer.byte_fallback = true; var it: CharTokenIterator = .{ .input = "ζℳL" }; - var res: stdx.BoundedArray(u32, 8) = .{}; + var res: stdx.BoundedArray(u32, 8) = .empty; while (try it.nextCodepointToken(&tokenizer)) |token| { res.appendAssumeCapacity(token); } @@ -591,7 +591,7 @@ pub const Normalizer = struct { /// Space token used by sentencepiece derived tokenizer. pub const sentencepiece_space = "▁"; // \xe2\x96\x81 - _whitespace: stdx.BoundedArray(u8, 8) = .{}, + _whitespace: stdx.BoundedArray(u8, 8) = .empty, flags: packed struct { remove_extra_whitespaces: bool, diff --git a/zml/tokenizer/sentencepiece/sentencepiece.zig b/zml/tokenizer/sentencepiece/sentencepiece.zig index b3749583..9ba19001 100644 --- a/zml/tokenizer/sentencepiece/sentencepiece.zig +++ b/zml/tokenizer/sentencepiece/sentencepiece.zig @@ -89,7 +89,7 @@ pub const Decoder = struct { inner: *SentencePieceProcessor, vec: *c.std_vector_int, str: *c.std_string, - last_string: StringBuffer = .{ .len = 0 }, + last_string: StringBuffer = .empty, fn init(inner: *SentencePieceProcessor) !Decoder { const vec = try (c.std_vector_int_new() orelse std.mem.Allocator.Error.OutOfMemory); From 29ad4987724d4306440b9cfaa34df6480794aa20 Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Mon, 27 Apr 2026 10:56:57 +0200 Subject: [PATCH 456/881] zml/tpu: support padded KV-cache head dims in ragged paged attention (#507) fix(mosaic_tpu): support padded KV-cache head dims in ragged paged attention Pass both logical and cache head dimensions through the Mosaic TPU paged attention path so KV pages can remain 128-byte aligned without padding Q or trimming the final output. - add backend helpers and tests for Mosaic TPU kernel head-dim alignment - keep query/output tensors and softmax scaling on the logical head dim - allow cache head dims greater than or equal to query head dims in TPU IR and kernel validation - slice padded K/V vectors back to the logical head dim before attention math - fix the tensor pad call signature used by tensor padding --- zml/attention/tpu_attention.zig | 81 +++++++++++++++++++++++++++++++-- zml/tensor.zig | 2 +- 2 files changed, 78 insertions(+), 5 deletions(-) diff --git a/zml/attention/tpu_attention.zig b/zml/attention/tpu_attention.zig index bad318e3..264a554f 100644 --- a/zml/attention/tpu_attention.zig +++ b/zml/attention/tpu_attention.zig @@ -136,6 +136,12 @@ pub const mosaic_tpu = struct { num_seqs: zml.Tensor, }; + pub fn alignedKernelHeadDim(head_dim: usize) usize { + stdx.debug.assert(head_dim > 0, "mosaic_tpu ragged paged attention head_dim must be non-zero", .{}); + const alignment: usize = 128; + return (std.math.divCeil(usize, head_dim, alignment) catch unreachable) * alignment; + } + pub const Options = struct { is_prefill: bool, batch_size: usize, @@ -270,12 +276,38 @@ pub const mosaic_tpu = struct { } inline fn restoreQueryHeads(q_template: zml.Tensor, q_out: zml.Tensor) zml.Tensor { - return q_out.splitAxis(.h, .{ .hkv = q_template.dim(.hkv), .hg = q_template.dim(.hg) }); + const restored = q_out.splitAxis(.h, .{ .hkv = q_template.dim(.hkv), .hg = q_template.dim(.hg) }); + if (restored.dim(.hd) == q_template.dim(.hd)) return restored; + stdx.debug.assert( + restored.dim(.hd) > q_template.dim(.hd), + "mosaic_tpu ragged paged attention cannot restore output from head_dim {} to {}", + .{ restored.dim(.hd), q_template.dim(.hd) }, + ); + return restored.slice1d(.hd, .{ .end = q_template.dim(.hd) }); + } + + inline fn alignQueryHeadDimForKernel(q: zml.Tensor, target_head_dim: i64) zml.Tensor { + if (q.dim(.hd) == target_head_dim) return q; + stdx.debug.assert( + q.dim(.hd) < target_head_dim, + "mosaic_tpu ragged paged attention q head_dim {} exceeds kernel/cache head_dim {}", + .{ q.dim(.hd), target_head_dim }, + ); + return q.pad(0, .{ .hd = zml.Tensor.Pad{ .high = target_head_dim - q.dim(.hd) } }) + .withPartitioning(.{ .h = .model }); } inline fn prepareInputs(parameters: Parameters, q: zml.Tensor, kv_pages: zml.Tensor) PreparedInputs { - const q_ragged = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); const kv_pages_partitioned = kv_pages.withPartitioning(.{ .hkv = .model }); + const q_merged = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + const logical_head_dim: usize = @intCast(q_merged.dim(.hd)); + const cache_head_dim: usize = @intCast(kv_pages_partitioned.dim(.hd)); + stdx.debug.assert( + cache_head_dim == alignedKernelHeadDim(logical_head_dim), + "mosaic_tpu ragged paged attention expects KV cache head_dim {} to match aligned logical head_dim {}", + .{ cache_head_dim, logical_head_dim }, + ); + const q_ragged = alignQueryHeadDimForKernel(q_merged, kv_pages_partitioned.dim(.hd)); const query_start_len = parameters.query_start_len.withPartitioning(.{ .b = .replicated }); // Note @@ -303,6 +335,11 @@ pub const mosaic_tpu = struct { const q_token_count = if (q_shape.hasTag(.q) != null) q_shape.dim(.q) else q_shape.dim(.b); const max_num_seqs = block_table_shape.dim(.b); const num_q_heads = q_shape.dim(.h); + stdx.debug.assert( + q_shape.dim(.hd) == kv_pages_shape.dim(.hd), + "mosaic_tpu ragged paged attention expects q and KV cache head_dim to match, got {} and {}", + .{ q_shape.dim(.hd), kv_pages_shape.dim(.hd) }, + ); stdx.debug.assert(@mod(kv_pages_shape.dim(.hkv), 2) == 0, "Expected fused KV pages .hkv dimension to be even, got {}", .{kv_pages_shape.dim(.hkv)}); const num_kv_heads = std.math.divExact(i64, kv_pages_shape.dim(.hkv), 2) catch |err| switch (err) { error.UnexpectedRemainder => stdx.debug.panic("Expected fused KV pages .hkv dimension to be even, got {}", .{kv_pages_shape.dim(.hkv)}), @@ -310,7 +347,7 @@ pub const mosaic_tpu = struct { error.Overflow => stdx.debug.panic("KV pages .hkv dimension is too large to fit in i64: {}", .{kv_pages_shape.dim(.hkv)}), }; const num_queries_per_block = @max(@as(i64, 1), @min(q_token_count, max_num_seqs)); - const sm_scale: f32 = opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q_shape.dim(.hd))))); + const logical_head_dim: f64 = @floatFromInt(parameters.opts.head_dim); return .{ .num_q_tokens = q_token_count, @@ -323,7 +360,7 @@ pub const mosaic_tpu = struct { .pages_per_seq = @intCast(parameters.opts.max_num_pages), .total_num_pages = kv_pages_shape.dim(.page), .page_size = kv_pages_shape.dim(.k_chunk), - .sm_scale = sm_scale, + .sm_scale = opts.scale orelse @floatCast(1.0 / @sqrt(logical_head_dim)), .sliding_window = if (opts.sliding_window < 0) null else @intCast(opts.sliding_window), .num_kv_pages_per_block = checkedU32(parameters.opts.pages_per_compute_block, "num_kv_pages_per_block"), .num_queries_per_block = checkedU32(@intCast(num_queries_per_block), "num_queries_per_block"), @@ -398,3 +435,39 @@ pub const mosaic_tpu = struct { return restored; } }; + +test "mosaic_tpu aligns kernel head dimensions to 128" { + try std.testing.expectEqual(@as(usize, 128), mosaic_tpu.alignedKernelHeadDim(64)); + try std.testing.expectEqual(@as(usize, 128), mosaic_tpu.alignedKernelHeadDim(128)); + try std.testing.expectEqual(@as(usize, 256), mosaic_tpu.alignedKernelHeadDim(180)); + try std.testing.expectEqual(@as(usize, 256), mosaic_tpu.alignedKernelHeadDim(256)); +} + +test "mosaic_tpu request uses kernel head dim and logical scale" { + const parameters = mosaic_tpu.Parameters.init(.{ + .is_prefill = false, + .batch_size = 1, + .max_num_pages = 2, + .max_seqlen_k = 32, + .max_token_count = 1, + .num_heads = 8, + .num_kv_heads = 2, + .head_dim = 64, + }); + const prepared_inputs: mosaic_tpu.PreparedInputs = .{ + .q = zml.Tensor.init(.{ .b = 1, .h = 8, .hd = 128 }, .bf16), + .kv_pages = zml.Tensor.init(.{ .page = 2, .k_chunk = 16, .hkv = 4, .hd = 128 }, .bf16), + .seq_lens = parameters.seq_lens, + .block_table = parameters.block_table, + .query_start_len = parameters.query_start_len, + .num_seqs = zml.Tensor.init(.{1}, .i32), + }; + + const request = mosaic_tpu.buildRequest(prepared_inputs, parameters, .{}); + try std.testing.expectEqual(@as(i64, 128), request.head_dim); + try std.testing.expectApproxEqAbs( + @as(f32, 1.0 / @sqrt(@as(f32, 64.0))), + request.sm_scale, + 0.000001, + ); +} diff --git a/zml/tensor.zig b/zml/tensor.zig index f86bb9b6..14a9fee8 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -2241,8 +2241,8 @@ pub const Tensor = struct { mlirCtx(), self.value(), Tensor.scalar(padding_value, self.dtype()).value(), - .{ .low = low[0..rk], .high = high[0..rk], .interior = interior[0..rk] }, .unknown(mlirCtx()), + .{ .low = low[0..rk], .high = high[0..rk], .interior = interior[0..rk] }, ).appendTo(currentBlock()); return _result(res_shape, pad_op.result(0)); From 53009052288704908f93a560c5846e444f618812 Mon Sep 17 00:00:00 2001 From: elogir <90246794+elogir@users.noreply.github.com> Date: Mon, 27 Apr 2026 19:59:31 +0200 Subject: [PATCH 457/881] platforms/rocm: update amdgpuids URL and bump llvm version (#510) This PR fixes two issues: - Update `amdgpu.ids` URL to Gitlab, now that the old link is dead - Bump LLVM version to `0.7.3` --- MODULE.bazel | 2 +- platforms/rocm/rocm.bzl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 26cf0ad1..ccefd924 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -36,7 +36,7 @@ git_override( ) bazel_dep(name = "tar.bzl", version = "0.9.0") -bazel_dep(name = "llvm", version = "0.7.1") +bazel_dep(name = "llvm", version = "0.7.3") bazel_dep(name = "jsoncpp", version = "1.9.6", repo_name = "jsoncpp_git") bazel_dep(name = "with_cfg.bzl", version = "0.14.4") bazel_dep(name = "grpc", version = "1.74.0") diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 8ab97767..32946833 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -246,7 +246,7 @@ def _rocm_impl(mctx): http_file( name = "libdrm_mesa_amdgpu_ids", - url = "https://cgit.freedesktop.org/mesa/drm/plain/data/amdgpu.ids?id=b9dea73dfa310bc945ae6f09004a08fd624952ec", + url = "https://gitlab.freedesktop.org/mesa/libdrm/-/raw/979f607906ad64f629967ac1f3ba3590e756442c/data/amdgpu.ids?inline=false", sha256 = "ffd2a8f1bfa755f4d90f537b4969fc4676f116e5af051ce2f18ef93a96d8beb6", downloaded_file_path = "amdgpu.ids", ) From 0fd551e4d5ab5b2be1e8bddb717647e7aef5cc6b Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Tue, 28 Apr 2026 09:32:40 +0200 Subject: [PATCH 458/881] zml/zls: Add compatibility for aarch64-linux platform in zls_toolchain (#508) --- third_party/zls/zls_toolchain.bzl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/third_party/zls/zls_toolchain.bzl b/third_party/zls/zls_toolchain.bzl index 4611ba95..337be425 100644 --- a/third_party/zls/zls_toolchain.bzl +++ b/third_party/zls/zls_toolchain.bzl @@ -11,6 +11,12 @@ _ZLS_PLATFORMS = { "@platforms//cpu:x86_64", ], ), + "aarch64-linux": struct( + compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:aarch64", + ], + ), "x86_64-macos": struct( compatible_with = [ "@platforms//os:macos", @@ -27,6 +33,7 @@ _ZLS_PLATFORMS = { _ZLS_VERSIONS = { "x86_64-linux": "ded6d562a0b86ee878b1ddf70ffab2797ce3cdca3b02d6077548f9d56dff96b6", + "aarch64-linux": "430cd293d201eb70ae2519dbc96c854bf8791b8df7fc9392e8d2dc9680a2bed7", "x86_64-macos": "49f716ea96c1aadaecaa5d9c0a50874cbcf443dc42b825f1e7ee35499ad3eb96", "aarch64-macos": "b93ec549f8558a7e85984a840e9276d274f1059b54ade4254296ef4982958359", } From 039a638aeb11317c7821b64b2f5dad9ce8abb211 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 29 Apr 2026 10:38:19 +0200 Subject: [PATCH 459/881] examples/llm: cleanups (#512) I'm doing a pass on the example code we have. * llama/model.zig, qwen/model.zig: change layer_index from being a tensor to being an integer. It's clearer this way what is happening. Note that lfm doesn't have the issue because the layer index is passed everywhere. * tokenizePrompt: the code was doing unnecessary copies of data, I added some helper to tokenizer.encoder to streamline the code by removing the extra copies and allocations. * `.stdout().writer() -> .stdout().writerStreaming()` this one is weird but apparently using stdout by position isn't always allowed, in particular when running the exe from within SublimeText. * runDecode -> show case bake --- .bazelignore | 1 + MODULE.bazel.lock | 13 +- examples/llm/chat.zig | 4 +- examples/llm/main.zig | 2 +- examples/llm/models/lfm2/session.zig | 2 +- examples/llm/models/lfm2_tests.zig | 3 +- examples/llm/models/llama/model.zig | 66 ++++---- examples/llm/models/llama/session.zig | 93 ++++++------ examples/llm/models/qwen3_5/model.zig | 193 ++++++++++-------------- examples/llm/models/qwen3_5/session.zig | 2 +- zml/buffer.zig | 2 +- zml/io/vfs/hf.zig | 1 + zml/mem.zig | 8 +- zml/nn.zig | 2 +- zml/tensor.zig | 17 ++- zml/tokenizer/iree/tokenizer.zig | 38 ++--- zml/tokenizer/tokenizer.zig | 17 +++ 17 files changed, 218 insertions(+), 246 deletions(-) diff --git a/.bazelignore b/.bazelignore index 875340d8..084d45c8 100644 --- a/.bazelignore +++ b/.bazelignore @@ -2,3 +2,4 @@ third_party/modules .zig-cache .devenv +venv/ diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index a390d19b..ed055b4c 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -57,7 +57,8 @@ "https://bcr.bazel.build/modules/bazel_features/1.38.0/MODULE.bazel": "f9b8a9c890ebd216b4049fd12a31d3c2602e3403c7af636b04fbbd7453edc9c9", "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", "https://bcr.bazel.build/modules/bazel_features/1.42.0/MODULE.bazel": "e8ca15cb2639c5f12183db6dcb678735555d0cdd739b32a0418b6532b5e565f8", - "https://bcr.bazel.build/modules/bazel_features/1.42.0/source.json": "f2ea90e5dd0322481147114c7d5e4608c4b3fae2eeccae655e4d76a382389f6f", + "https://bcr.bazel.build/modules/bazel_features/1.43.0/MODULE.bazel": "defa2226f06ba20550d6548c3a2ea2a7929634437a52973869c20c225450eb91", + "https://bcr.bazel.build/modules/bazel_features/1.43.0/source.json": "1c4207dc858d6de0eecef30026793616bbf420c74aac27b6bad212534a730437", "https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b", "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", "https://bcr.bazel.build/modules/bazel_lib/3.0.0-beta.1/MODULE.bazel": "407729e232f611c3270005b016b437005daa7b1505826798ea584169a476e878", @@ -75,6 +76,7 @@ "https://bcr.bazel.build/modules/bazel_skylib/1.5.0/MODULE.bazel": "32880f5e2945ce6a03d1fbd588e9198c0a959bb42297b2cfaf1685b7bc32e138", "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/MODULE.bazel": "8fdee2dbaace6c252131c00e1de4b165dc65af02ea278476187765e1a617b917", "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/MODULE.bazel": "3120d80c5861aa616222ec015332e5f8d3171e062e3e804a2a0253e1be26e59b", + "https://bcr.bazel.build/modules/bazel_skylib/1.8.0/MODULE.bazel": "2fb3fb53675f6adfc1ca5bfbd5cfb655ae350fba4706d924a8ec7e3ba945671c", "https://bcr.bazel.build/modules/bazel_skylib/1.8.1/MODULE.bazel": "88ade7293becda963e0e3ea33e7d54d3425127e0a326e0d17da085a5f1f03ff6", "https://bcr.bazel.build/modules/bazel_skylib/1.8.2/MODULE.bazel": "69ad6927098316848b34a9142bcc975e018ba27f08c4ff403f50c1b6e646ca67", "https://bcr.bazel.build/modules/bazel_skylib/1.9.0/MODULE.bazel": "72997b29dfd95c3fa0d0c48322d05590418edef451f8db8db5509c57875fb4b7", @@ -152,8 +154,8 @@ "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", "https://bcr.bazel.build/modules/libpfm/4.11.0/source.json": "caaffb3ac2b59b8aac456917a4ecf3167d40478ee79f15ab7a877ec9273937c9", "https://bcr.bazel.build/modules/llvm/0.6.7/MODULE.bazel": "d37a2e10571864dc6a5bb53c29216d90b9400bbcadb422337f49107fd2eaf0d2", - "https://bcr.bazel.build/modules/llvm/0.7.1/MODULE.bazel": "74ac75efc6385b8a95d83bfa36ad399500f747c3d0f50287f9b6f9e854ec4814", - "https://bcr.bazel.build/modules/llvm/0.7.1/source.json": "0cac59d04dafa0ca1f70ea21cf1569e034ef8736c2c7510e834a9e8141d7e631", + "https://bcr.bazel.build/modules/llvm/0.7.3/MODULE.bazel": "b8ffe1ec9d18c0deb6d6dc1a6f78a64bcfcec8648c97b92ed6cd98a7fe49c64f", + "https://bcr.bazel.build/modules/llvm/0.7.3/source.json": "44d694e1559683ea4cf96052c56c5426d2604016636133798fcd57cc78542984", "https://bcr.bazel.build/modules/mbedtls/3.6.0/MODULE.bazel": "8e380e4698107c5f8766264d4df92e36766248447858db28187151d884995a09", "https://bcr.bazel.build/modules/mbedtls/3.6.0/source.json": "1dbe7eb5258050afcc3806b9d43050f71c6f539ce0175535c670df606790b30c", "https://bcr.bazel.build/modules/nlohmann_json/3.11.3/MODULE.bazel": "87023db2f55fc3a9949c7b08dc711fae4d4be339a80a99d04453c4bb3998eefc", @@ -242,7 +244,8 @@ "https://bcr.bazel.build/modules/rules_cc/0.2.0/MODULE.bazel": "b5c17f90458caae90d2ccd114c81970062946f49f355610ed89bebf954f5783c", "https://bcr.bazel.build/modules/rules_cc/0.2.15/MODULE.bazel": "6a0a4a75a57aa6dc888300d848053a58c6b12a29f89d4304e1c41448514ec6e8", "https://bcr.bazel.build/modules/rules_cc/0.2.16/MODULE.bazel": "9242fa89f950c6ef7702801ab53922e99c69b02310c39fb6e62b2bd30df2a1d4", - "https://bcr.bazel.build/modules/rules_cc/0.2.16/source.json": "d03d5cde49376d87e14ec14b666c56075e5e3926930327fd5d0484a1ff2ac1cc", + "https://bcr.bazel.build/modules/rules_cc/0.2.18/MODULE.bazel": "4460ec36adc8f722a6a2a4ac9374cb91f2acebadaa93fc37966129afb3dece87", + "https://bcr.bazel.build/modules/rules_cc/0.2.18/source.json": "abad668ff2fd63ada1ac49bf386d37e27048b89a3465a6fd968bb832b00a09d3", "https://bcr.bazel.build/modules/rules_cc/0.2.4/MODULE.bazel": "1ff1223dfd24f3ecf8f028446d4a27608aa43c3f41e346d22838a4223980b8cc", "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", "https://bcr.bazel.build/modules/rules_distroless/0.6.2/MODULE.bazel": "eafe8b473c0c78038d9a6e1dbd9278b7bf83457ddc78207297483bfbe1012d56", @@ -1025,7 +1028,7 @@ }, "@@rules_oci+//oci:extensions.bzl%oci": { "general": { - "bzlTransitiveDigest": "twSeB3VPTVkayY0uwqEzDzal5MbnZX45iC6jdheZQRo=", + "bzlTransitiveDigest": "rM01g4kd1EG6Z+4iqFqd+ZkaNCU9oVCGJf+Kg9jgr5k=", "usagesDigest": "JLbamplho20rviuebatMGqnQnaoj32ewCzLuRVusGoU=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, diff --git a/examples/llm/chat.zig b/examples/llm/chat.zig index a319cf7a..a755bd90 100644 --- a/examples/llm/chat.zig +++ b/examples/llm/chat.zig @@ -68,7 +68,7 @@ pub const Chat = struct { const prompt_tokens = try self.session.tokenizePrompt(self.allocator, prompt); defer self.allocator.free(prompt_tokens); - var stdout = std.Io.File.stdout().writer(self.io, &.{}); + var stdout = std.Io.File.stdout().writerStreaming(self.io, &.{}); var profiler: ?zml.Platform.Profiler = null; defer if (profiler) |*p| p.deinit(); @@ -103,7 +103,7 @@ pub const Chat = struct { var turn_tokens = try self.session.tokenizePrompt(self.allocator, initial_prompt); defer self.allocator.free(turn_tokens); - var stdout = std.Io.File.stdout().writer(self.io, &.{}); + var stdout = std.Io.File.stdout().writerStreaming(self.io, &.{}); while (self.tokens.items.len <= self.session.maxTokens()) { try self.runPrefill(turn_tokens, &stdout.interface); diff --git a/examples/llm/main.zig b/examples/llm/main.zig index 2dd57480..747d569a 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -201,7 +201,7 @@ pub fn printZmlLogo(io: std.Io) !void { \\ \\ ; - var writer = std.Io.File.stdout().writer(io, &.{}); + var writer = std.Io.File.stdout().writerStreaming(io, &.{}); try writer.interface.writeAll(LOGO); try writer.interface.flush(); } diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig index 95c79ce1..1a756115 100644 --- a/examples/llm/models/lfm2/session.zig +++ b/examples/llm/models/lfm2/session.zig @@ -42,7 +42,7 @@ pub const Session = struct { .seqlen = compiled_model.params.seqlen, .cache_buffers = try compiled_model.params.cache.initBuffers(allocator, io, platform, compiled_model.params.shardings.replicated), .attention_metadata_buffers = try compiled_model.params.attention_metadata.initBuffer(io, platform, compiled_model.params.shardings.model), - .rng_buf = try zml.Tensor.Rng.initBuffer(platform, seed, io, compiled_model.params.shardings.replicated), + .rng_buf = try zml.Tensor.Rng.initBuffer(io, platform, compiled_model.params.shardings.replicated, seed), .generated_token_slice = try .alloc(allocator, zml.Shape.init(.{ .batch = 1, .seq = 1 }, .u32)), .think_start = tokenizer.tokenId("") orelse unreachable, .think_end = tokenizer.tokenId("") orelse unreachable, diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig index 93ce6ec1..20e6893e 100644 --- a/examples/llm/models/lfm2_tests.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -281,8 +281,7 @@ const TestContext = struct { var attention_metadata_buffers = try self.attention_metadata.initBuffer(self.io, self.platform, self.sharding); defer attention.Metadata.deinitBuffer(&attention_metadata_buffers); - const cache_index_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(cache_ix)})); - var cache_index_buf: zml.Buffer = try .fromSlice(self.io, self.platform, cache_index_slice, self.sharding); + var cache_index_buf: zml.Buffer = try .scalar(self.io, self.platform, cache_ix, .u32, self.sharding); defer cache_index_buf.deinit(); args.set(.{ layer_buffers, in_buffer, cache_pos_buffer, kv_cache, cache_index_buf, attention_metadata_buffers }); diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index 1de9b509..a6fb2e1c 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -510,9 +510,6 @@ pub const SelfAttn = struct { q = q.rename(.{ .s = .q }); k = k.rename(.{ .s = .k }); v = v.rename(.{ .s = .k }); - q = q.withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }); - k = k.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); - v = v.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); const dtype = q.dtype(); const new_kv_cache = kv_cache.update(k, v, token_index); @@ -541,7 +538,9 @@ pub const SelfAttn = struct { pub const KvCache = struct { k: zml.Tensor, v: zml.Tensor, - layer_index: zml.Tensor, + layer_index: ?u32, + + pub const Buffer = zml.Bufferized(KvCache); pub fn init(kv_shape: zml.Shape) KvCache { const sharded_shape = kv_shape.withPartitioning(.{ .h = .model }); @@ -549,64 +548,61 @@ pub const KvCache = struct { return .{ .k = .fromShape(sharded_shape), .v = .fromShape(sharded_shape), - .layer_index = .init(.{}, .u32), + .layer_index = null, }; } - pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !zml.Bufferized(KvCache) { + pub fn initBuffer(kv: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !Buffer { return .{ - .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), sharding, .{}), - .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), sharding, .{}), - .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32, sharding), + .k = try zml.Buffer.uninitialized(io, platform, kv.k.shape(), sharding, .{}), + .v = try zml.Buffer.uninitialized(io, platform, kv.v.shape(), sharding, .{}), }; } - pub fn deinitBuffer(self: *zml.Bufferized(KvCache)) void { - self.k.deinit(); - self.v.deinit(); - self.layer_index.deinit(); + pub fn deinitBuffer(kv: *Buffer) void { + kv.k.deinit(); + kv.v.deinit(); } - pub fn keys(self: KvCache) zml.Tensor { - return self.k.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + pub fn keys(kv: KvCache) zml.Tensor { + return kv.k.slice1d(.layer, .single(kv.layer_index orelse @panic("forgot to call atLayer"))); } - pub fn values(self: KvCache) zml.Tensor { - return self.v.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + pub fn values(kv: KvCache) zml.Tensor { + return kv.v.slice1d(.layer, .single(kv.layer_index orelse @panic("forgot to call atLayer"))); } - pub fn update(self: KvCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: ?zml.Tensor) KvCache { - const k_shape = self.k.shape().drop(.layer); - var layer = self.layer_index; - layer = if (token_index) |idx| layer.broad(idx.shape()) else layer; + pub fn update(kv: KvCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: ?zml.Tensor) KvCache { + const k_shape = kv.k.shape().drop(.layer); + const layer: zml.Tensor = .scalar(kv.layer_index orelse @panic("forgot to call atLayer"), .u32); return if (token_index) |idx| .{ - .k = self.k.scatterSlices(.{ .layer = layer, .k = idx }, new_k.convert(self.k.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(self.k), - .v = self.v.scatterSlices(.{ .layer = layer, .k = idx }, new_v.convert(self.v.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(self.v), - .layer_index = self.layer_index, + .k = kv.k.scatterSlices(.{ .layer = layer, .k = idx }, new_k.convert(kv.k.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(kv.k), + .v = kv.v.scatterSlices(.{ .layer = layer, .k = idx }, new_v.convert(kv.v.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(kv.v), + .layer_index = kv.layer_index, } else .{ - .k = self.k.scatterSlices(.{ .layer = layer }, new_k.convert(self.k.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(self.k), - .v = self.v.scatterSlices(.{ .layer = layer }, new_v.convert(self.v.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(self.v), - .layer_index = self.layer_index, + .k = kv.k.scatterSlices(.{ .layer = layer }, new_k.convert(kv.k.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(kv.k), + .v = kv.v.scatterSlices(.{ .layer = layer }, new_v.convert(kv.v.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(kv.v), + .layer_index = kv.layer_index, }; } - pub fn atLayer(self: KvCache, layer_index: usize) KvCache { + pub fn atLayer(kv: KvCache, layer_index: usize) KvCache { return .{ - .k = self.k, - .v = self.v, - .layer_index = zml.Tensor.scalar(layer_index, .u32), + .k = kv.k, + .v = kv.v, + .layer_index = @intCast(layer_index), }; } - pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { + pub fn reuseBuffer(kv: KvCache, other: KvCache) KvCache { return .{ - .k = self.k.reuseBuffer(other.k), - .v = self.v.reuseBuffer(other.v), - .layer_index = self.layer_index.reuseBuffer(other.layer_index), + .k = kv.k.reuseBuffer(other.k), + .v = kv.v.reuseBuffer(other.v), + .layer_index = null, }; } }; diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig index c429c27b..7c23f619 100644 --- a/examples/llm/models/llama/session.zig +++ b/examples/llm/models/llama/session.zig @@ -14,10 +14,10 @@ pub const Session = struct { kv_cache_buffers: zml.Bufferized(model.KvCache), attention_metadata_buffers: zml.Bufferized(zml.attention.attention.Metadata), rng_buffers: zml.Bufferized(zml.Tensor.Rng), - generated_token_slice: zml.Slice, tokenizer: zml.tokenizer.Tokenizer, config: *const model.Config, seqlen: u32, + last_generated_token: u32 = 0, pub fn init( allocator: std.mem.Allocator, @@ -35,7 +35,7 @@ pub const Session = struct { errdefer zml.attention.attention.Metadata.deinitBuffer(&attention_metadata_buffers); const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); - var rng_buffers = try zml.Tensor.Rng.initBuffer(platform, seed, io, shardings.replicated); + var rng_buffers = try zml.Tensor.Rng.initBuffer(io, platform, shardings.replicated, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); return .{ @@ -47,7 +47,6 @@ pub const Session = struct { .kv_cache_buffers = kv_cache_buffers, .attention_metadata_buffers = attention_metadata_buffers, .rng_buffers = rng_buffers, - .generated_token_slice = try .alloc(allocator, zml.Shape.init(.{ .s = 1 }, .u32)), .tokenizer = tokenizer, .config = &compiled_model.loaded_model.parsed_config.value, .seqlen = @intCast(compiled_model.params.seqlen), @@ -58,7 +57,6 @@ pub const Session = struct { model.KvCache.deinitBuffer(&self.kv_cache_buffers); zml.attention.attention.Metadata.deinitBuffer(&self.attention_metadata_buffers); zml.Tensor.Rng.deinitBuffer(&self.rng_buffers); - self.generated_token_slice.free(self.allocator); } pub fn tokenizePrompt(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { @@ -70,21 +68,19 @@ pub const Session = struct { const eot = self.tokenizer.tokenId("<|eot_id|>") orelse return error.NoSuchToken; const newline = self.tokenizer.tokenId("\\n") orelse return error.NoSuchToken; - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); - try tokens.appendSlice(allocator, &.{ self.config.bos_token_id, start_header }); - const user_tokens = try encoder.encodeAlloc(allocator, "user"); - defer allocator.free(user_tokens); - try tokens.appendSlice(allocator, user_tokens); - try tokens.appendSlice(allocator, &.{ end_header, newline }); - const prompt_tokens = try encoder.encodeAlloc(allocator, prompt); - defer allocator.free(prompt_tokens); - try tokens.appendSlice(allocator, prompt_tokens); - try tokens.appendSlice(allocator, &.{ eot, newline, start_header }); - const assistant_tokens = try encoder.encodeAlloc(allocator, "assistant"); - defer allocator.free(assistant_tokens); - try tokens.appendSlice(allocator, assistant_tokens); - try tokens.appendSlice(allocator, &.{ end_header, newline }); - return tokens.toOwnedSlice(allocator); + var tokens = std.Io.Writer.Allocating.initAligned(allocator, .of(u32)); + try tokens.ensureUnusedCapacity(prompt.len); + + const w: *std.Io.Writer = &tokens.writer; + try encoder.appendTokens(w, &.{ self.config.bos_token_id, start_header }); + try encoder.encode(w, "user"); + try encoder.appendTokens(w, &.{ end_header, newline }); + try encoder.encode(w, prompt); + try encoder.appendTokens(w, &.{ eot, newline, start_header }); + try encoder.encode(w, "assistant"); + try encoder.appendTokens(w, &.{ end_header, newline }); + + return @ptrCast(@alignCast(try tokens.toOwnedSlice())); } pub fn tokenizeTurn(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { @@ -96,21 +92,19 @@ pub const Session = struct { const eot = self.tokenizer.tokenId("<|eot_id|>") orelse return error.NoSuchToken; const newline = self.tokenizer.tokenId("\\n") orelse return error.NoSuchToken; - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); - try tokens.appendSlice(allocator, &.{ eot, newline, start_header }); - const user_tokens = try encoder.encodeAlloc(allocator, "user"); - defer allocator.free(user_tokens); - try tokens.appendSlice(allocator, user_tokens); - try tokens.appendSlice(allocator, &.{ end_header, newline }); - const prompt_tokens = try encoder.encodeAlloc(allocator, prompt); - defer allocator.free(prompt_tokens); - try tokens.appendSlice(allocator, prompt_tokens); - try tokens.appendSlice(allocator, &.{ eot, newline, start_header }); - const assistant_tokens = try encoder.encodeAlloc(allocator, "assistant"); - defer allocator.free(assistant_tokens); - try tokens.appendSlice(allocator, assistant_tokens); - try tokens.appendSlice(allocator, &.{ end_header, newline }); - return tokens.toOwnedSlice(allocator); + var tokens = std.Io.Writer.Allocating.initAligned(allocator, .of(u32)); + try tokens.ensureUnusedCapacity(prompt.len); + + const w: *std.Io.Writer = &tokens.writer; + try encoder.appendTokens(w, &.{ eot, newline, start_header }); + try encoder.encode(w, "user"); + try encoder.appendTokens(w, &.{ end_header, newline }); + try encoder.encode(w, prompt); + try encoder.appendTokens(w, &.{ eot, newline, start_header }); + try encoder.encode(w, "assistant"); + try encoder.appendTokens(w, &.{ end_header, newline }); + + return @ptrCast(@alignCast(try tokens.toOwnedSlice())); } pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { @@ -145,44 +139,43 @@ pub const Session = struct { prefill_results.fill(.{ &prefill_tokens_buffer, &self.kv_cache_buffers, &self.rng_buffers }); try prefill_tokens_buffer.toSlice(self.io, prefill_tokens_slice); - self.generated_token_slice.items(u32)[0] = prefill_tokens_slice.items(u32)[all_tokens.len - 1]; + self.last_generated_token = prefill_tokens_slice.items(u32)[all_tokens.len - 1]; } pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { - var decoder = try self.tokenizer.decoder(); + var decoder: zml.tokenizer.Tokenizer.Decoder = try self.tokenizer.decoder(); defer decoder.deinit(); + const decoder_out_buffer: []u8 = try self.allocator.alloc(u8, 256); + defer self.allocator.free(decoder_out_buffer); + var decode_args = try self.compiled_model.decode_exe.args(self.allocator); defer decode_args.deinit(self.allocator); + // Fix model buffers, since they are stable for the full gen. + decode_args.bake(self.model_buffers.*); var decode_results = try self.compiled_model.decode_exe.results(self.allocator); defer decode_results.deinit(self.allocator); const replicated_sharding = try zml.sharding.replicatedSharding(self.platform); - var current_token_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, self.generated_token_slice, replicated_sharding); + var last_token_id: u32 = self.last_generated_token; + var current_token_buffer: zml.Buffer = try .fromBytes(self.io, self.platform, .init(.{ .s = 1 }, .u32), replicated_sharding, @ptrCast(&last_token_id)); defer current_token_buffer.deinit(); - const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); - defer self.allocator.free(out_tokens_buffer); generation: while (true) { - const token_id = self.generated_token_slice.items(u32)[0]; + if (isEosToken(self.config, last_token_id)) break :generation; - if (isEosToken(self.config, token_id)) break :generation; - - const token = try decoder.feedOne(token_id, out_tokens_buffer); - try stdout.writeAll(token); + try stdout.writeAll(try decoder.feedOne(last_token_id, decoder_out_buffer)); try stdout.flush(); - try all_tokens.append(self.allocator, token_id); + try all_tokens.append(self.allocator, last_token_id); if (all_tokens.items.len >= self.seqlen) break :generation; - const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(all_tokens.items.len)})); - var token_pos_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice, replicated_sharding); + var token_pos_buffer: zml.Buffer = try .scalar(self.io, self.platform, all_tokens.items.len, .u32, replicated_sharding); defer token_pos_buffer.deinit(); decode_args.set(.{ - self.model_buffers, current_token_buffer, token_pos_buffer, &self.kv_cache_buffers, @@ -192,10 +185,10 @@ pub const Session = struct { self.compiled_model.decode_exe.call(decode_args, &decode_results); decode_results.fill(.{ ¤t_token_buffer, &self.kv_cache_buffers, &self.rng_buffers }); - try current_token_buffer.toSlice(self.io, self.generated_token_slice); + last_token_id = try current_token_buffer.getValue(u32, self.io); } - try stdout.writeAll(try decoder.finalize(out_tokens_buffer)); + try stdout.writeAll(try decoder.finalize(decoder_out_buffer)); try stdout.flush(); } }; diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 551e972b..22f031e6 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -284,7 +284,7 @@ pub const TextModel = struct { var updated_kv_cache = kv_cache; for (self.layers, 0..) |layer, i| { - hidden_states, updated_kv_cache = layer.forward(hidden_states, token_index, updated_kv_cache.atLayer(i)); + hidden_states, updated_kv_cache = layer.forward(hidden_states, token_index, updated_kv_cache, @intCast(i)); } hidden_states = self.norm.forward(hidden_states); @@ -293,9 +293,9 @@ pub const TextModel = struct { }; pub const TransformerLayer = struct { - const Attn = union(enum) { - self_attn: SelfAttn, - linear_attn: GatedDeltaNet, + const Attn = union(LayerType) { + linear_attention: GatedDeltaNet, + full_attention: SelfAttn, }; input_layernorm: RmsNorm, @@ -308,9 +308,9 @@ pub const TransformerLayer = struct { return .{ .input_layernorm = RmsNorm.init(store.withPrefix("input_layernorm"), config.text_config.rms_norm_eps), .attn = if (is_full_attention) - .{ .self_attn = try .init(store.withPrefix("self_attn"), config) } + .{ .full_attention = try .init(store.withPrefix("self_attn"), config) } else - .{ .linear_attn = .init(store.withPrefix("linear_attn"), config) }, + .{ .linear_attention = .init(store.withPrefix("linear_attn"), config) }, .mlp = .init(store.withPrefix("mlp")), .post_attention_layernorm = RmsNorm.init(store.withPrefix("post_attention_layernorm"), config.text_config.rms_norm_eps), }; @@ -319,8 +319,8 @@ pub const TransformerLayer = struct { pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer)) void { RmsNorm.unloadBuffers(&self.input_layernorm); switch (self.attn) { - .self_attn => |*self_attn| SelfAttn.unloadBuffers(self_attn), - .linear_attn => |*linear_attn| GatedDeltaNet.unloadBuffers(linear_attn), + .full_attention => |*self_attn| SelfAttn.unloadBuffers(self_attn), + .linear_attention => |*linear_attn| GatedDeltaNet.unloadBuffers(linear_attn), } Mlp.unloadBuffers(&self.mlp); RmsNorm.unloadBuffers(&self.post_attention_layernorm); @@ -330,21 +330,24 @@ pub const TransformerLayer = struct { self: TransformerLayer, x0: zml.Tensor, token_index: zml.Tensor, - kv_cache: KvCache.LayerView, + kv_cache: KvCache, + layer_id: u32, ) struct { zml.Tensor, KvCache } { const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); var attention_output: zml.Tensor = undefined; - var updated_kv_cache: KvCache = kv_cache.parent; + var updated_kv_cache: KvCache = kv_cache; + const layer_type, const dense_id = kv_cache.getDenseIndex(layer_id); + std.debug.assert(self.attn == layer_type); switch (self.attn) { - .self_attn => |*self_attn| { - const result = self_attn.forward(normalized_x0, token_index, kv_cache.cache.self_attn); + .full_attention => |*self_attn| { + const result = self_attn.forward(normalized_x0, token_index, kv_cache.self_attn.atLayer(dense_id)); attention_output = result[0]; updated_kv_cache.self_attn = result[1]; }, - .linear_attn => |*linear_attn| { - const result = linear_attn.forward(normalized_x0, kv_cache.cache.linear_attn); + .linear_attention => |*linear_attn| { + const result = linear_attn.forward(normalized_x0, kv_cache.gated_delta_net.atLayer(dense_id)); attention_output = result[0]; updated_kv_cache.gated_delta_net = result[1]; }, @@ -858,7 +861,9 @@ pub const KvCache = struct { pub const SelfAttnCache = struct { k: zml.Tensor, v: zml.Tensor, - layer_index: zml.Tensor, + layer_index: ?u32, + + pub const Buffers = zml.Bufferized(SelfAttnCache); pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, model_partitions: i64) SelfAttnCache { const num_self_attn_layers = countLayers(config.text_config.layer_types, .full_attention); @@ -873,77 +878,60 @@ pub const KvCache = struct { return .{ .k = .fromShape(sharded_kv_shape), .v = .fromShape(sharded_kv_shape), - .layer_index = .init(.{}, .u32), + .layer_index = null, }; } - pub fn initBuffer(self: SelfAttnCache, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !zml.Bufferized(SelfAttnCache) { + pub fn initBuffer(kv: SelfAttnCache, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !SelfAttnCache.Buffers { return .{ - .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), sharding, .{}), - .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), sharding, .{}), - .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32, sharding), + .k = try zml.Buffer.uninitialized(io, platform, kv.k.shape(), sharding, .{}), + .v = try zml.Buffer.uninitialized(io, platform, kv.v.shape(), sharding, .{}), }; } - pub fn deinitBuffer(self: *zml.Bufferized(SelfAttnCache)) void { - self.k.deinit(); - self.v.deinit(); - self.layer_index.deinit(); + pub fn deinitBuffer(kv: *SelfAttnCache.Buffers) void { + kv.k.deinit(); + kv.v.deinit(); } - pub fn keys(self: SelfAttnCache) zml.Tensor { - return self.k.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + pub fn keys(kv: SelfAttnCache) zml.Tensor { + return kv.k.slice1d(.layer, .single(kv.layer_index orelse @panic("forgot to call atLayer"))); } - pub fn values(self: SelfAttnCache) zml.Tensor { - return self.v.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + pub fn values(kv: SelfAttnCache) zml.Tensor { + return kv.v.slice1d(.layer, .single(kv.layer_index orelse @panic("forgot to call atLayer"))); } - pub fn update(self: SelfAttnCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: ?zml.Tensor) SelfAttnCache { - const k_shape = self.k.shape().drop(.layer); - var layer = self.layer_index; - layer = if (token_index) |idx| layer.broad(idx.shape()) else layer; + pub fn update(kv: SelfAttnCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: ?zml.Tensor) SelfAttnCache { + const k_shape = kv.k.shape().drop(.layer); + const layer: zml.Tensor = .scalar(kv.layer_index orelse @panic("forgot to call atLayer"), .u32); - return if (token_index) |idx| .{ - .k = self.k.scatterSlices( - .{ .layer = layer, .s = idx }, - new_k.convert(self.k.dtype()).transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.k), - .v = self.v.scatterSlices( - .{ .layer = layer, .s = idx }, - new_v.convert(self.v.dtype()).transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.v), - .layer_index = self.layer_index, - } else .{ - .k = self.k.scatterSlices( - .{ .layer = layer }, - new_k.convert(self.k.dtype()).transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.k), - .v = self.v.scatterSlices( - .{ .layer = layer }, - new_v.convert(self.v.dtype()).transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.v), - .layer_index = self.layer_index, - }; + return if (token_index) |idx| + .{ + .k = kv.k.scatterSlices(.{ .layer = layer, .k = idx }, new_k.convert(kv.k.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(kv.k), + .v = kv.v.scatterSlices(.{ .layer = layer, .k = idx }, new_v.convert(kv.v.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(kv.v), + .layer_index = kv.layer_index, + } + else + .{ + .k = kv.k.scatterSlices(.{ .layer = layer }, new_k.convert(kv.k.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(kv.k), + .v = kv.v.scatterSlices(.{ .layer = layer }, new_v.convert(kv.v.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(kv.v), + .layer_index = kv.layer_index, + }; } - - pub fn atLayer(self: SelfAttnCache, layer_index: usize) SelfAttnCache { + pub fn atLayer(kv: SelfAttnCache, layer_index: usize) SelfAttnCache { return .{ - .k = self.k, - .v = self.v, - .layer_index = zml.Tensor.scalar(layer_index, .u32), + .k = kv.k, + .v = kv.v, + .layer_index = @intCast(layer_index), }; } - pub fn reuseBuffer(self: SelfAttnCache, other: SelfAttnCache) SelfAttnCache { + pub fn reuseBuffer(kv: SelfAttnCache, other: SelfAttnCache) SelfAttnCache { return .{ - .k = self.k.reuseBuffer(other.k), - .v = self.v.reuseBuffer(other.v), - .layer_index = self.layer_index.reuseBuffer(other.layer_index), + .k = kv.k.reuseBuffer(other.k), + .v = kv.v.reuseBuffer(other.v), + .layer_index = kv.layer_index, }; } }; @@ -951,7 +939,9 @@ pub const KvCache = struct { pub const GatedDeltaNetCache = struct { conv_state: zml.Tensor, recurrent_state: zml.Tensor, - layer_index: zml.Tensor, + layer_index: ?u32, + + pub const Buffers = zml.Bufferized(GatedDeltaNetCache); pub fn init(config: Config, batch_dim: i64, conv_dtype: zml.DataType, recurrent_dtype: zml.DataType) GatedDeltaNetCache { const num_linear_attn_layers = countLayers(config.text_config.layer_types, .linear_attention); @@ -974,36 +964,35 @@ pub const KvCache = struct { return .{ .conv_state = .fromShape(sharded_conv_state_shape), .recurrent_state = .fromShape(sharded_recurrent_state_shape), - .layer_index = .init(.{}, .u32), + .layer_index = null, }; } - pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !zml.Bufferized(GatedDeltaNetCache) { + pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !GatedDeltaNetCache.Buffers { return .{ .conv_state = try zml.Buffer.uninitialized(io, platform, self.conv_state.shape(), sharding, .{}), .recurrent_state = try zml.Buffer.uninitialized(io, platform, self.recurrent_state.shape(), sharding, .{}), - .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32, sharding), }; } - pub fn deinitBuffer(self: *zml.Bufferized(GatedDeltaNetCache)) void { + pub fn deinitBuffer(self: *GatedDeltaNetCache.Buffers) void { self.conv_state.deinit(); self.recurrent_state.deinit(); - self.layer_index.deinit(); } pub fn convState(self: GatedDeltaNetCache) zml.Tensor { - return self.conv_state.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + return self.conv_state.slice1d(.layer, .single(self.layer_index orelse @panic("forgot to call atLayer"))); } pub fn recurrentState(self: GatedDeltaNetCache) zml.Tensor { - return self.recurrent_state.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + return self.recurrent_state.slice1d(.layer, .single(self.layer_index orelse @panic("forgot to call atLayer"))); } pub fn update(self: GatedDeltaNetCache, new_conv_state: ?zml.Tensor, new_recurrent_state: ?zml.Tensor) GatedDeltaNetCache { + const layer: zml.Tensor = .scalar(self.layer_index, .u32); const conv_state = if (new_conv_state) |state| self.conv_state.scatterSlices( - .{ .layer = self.layer_index }, + .{ .layer = layer }, state.convert(self.conv_state.dtype()).transpose(self.conv_state.shape().drop(.layer)), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, ).reuseBuffer(self.conv_state) @@ -1012,7 +1001,7 @@ pub const KvCache = struct { const recurrent_state = if (new_recurrent_state) |state| self.recurrent_state.scatterSlices( - .{ .layer = self.layer_index }, + .{ .layer = layer }, state.convert(self.recurrent_state.dtype()).transpose(self.recurrent_state.shape().drop(.layer)), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, ).reuseBuffer(self.recurrent_state) @@ -1030,7 +1019,7 @@ pub const KvCache = struct { return .{ .conv_state = self.conv_state, .recurrent_state = self.recurrent_state, - .layer_index = zml.Tensor.scalar(layer_index, .u32), + .layer_index = @intCast(layer_index), }; } @@ -1038,7 +1027,7 @@ pub const KvCache = struct { return .{ .conv_state = self.conv_state.reuseBuffer(other.conv_state), .recurrent_state = self.recurrent_state.reuseBuffer(other.recurrent_state), - .layer_index = self.layer_index.reuseBuffer(other.layer_index), + .layer_index = null, }; } }; @@ -1053,8 +1042,8 @@ pub const KvCache = struct { ) KvCache { return .{ .layer_types = config.text_config.layer_types, - .self_attn = SelfAttnCache.init(config, batch_dim, max_seq_len, cache_dtype, model_partitions), - .gated_delta_net = GatedDeltaNetCache.init(config, batch_dim, cache_dtype, recurrent_dtype), + .self_attn = .init(config, batch_dim, max_seq_len, cache_dtype, model_partitions), + .gated_delta_net = .init(config, batch_dim, cache_dtype, recurrent_dtype), }; } @@ -1070,27 +1059,6 @@ pub const KvCache = struct { GatedDeltaNetCache.deinitBuffer(&self.gated_delta_net); } - pub const LayerView = struct { - parent: KvCache, - cache: union(enum) { - self_attn: SelfAttnCache, - linear_attn: GatedDeltaNetCache, - }, - }; - - pub fn atLayer(self: KvCache, layer_index: usize) LayerView { - return switch (getDenseIndex(self.layer_types, layer_index)) { - .full_attention => |dense_index| .{ - .parent = self, - .cache = .{ .self_attn = self.self_attn.atLayer(dense_index.layer_dense_index) }, - }, - .linear_attention => |dense_index| .{ - .parent = self, - .cache = .{ .linear_attn = self.gated_delta_net.atLayer(dense_index.layer_dense_index) }, - }, - }; - } - pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { return .{ .layer_types = self.layer_types, @@ -1100,28 +1068,21 @@ pub const KvCache = struct { } fn countLayers(layer_types: []const LayerType, layer_type: LayerType) i64 { - var count: i64 = 0; - for (layer_types) |registered_layer_type| { - if (registered_layer_type == layer_type) count += 1; - } - return count; + return @intCast(std.mem.countScalar(LayerType, layer_types, layer_type)); } - fn getDenseIndex(layer_types: []const LayerType, layer_index: usize) union(enum) { - full_attention: struct { layer_dense_index: usize }, - linear_attention: struct { layer_dense_index: usize }, - } { - var self_attn_layer_index: usize = 0; - var linear_attn_layer_index: usize = 0; - for (layer_types[0..layer_index]) |layer_type| { + pub fn getDenseIndex(cache: KvCache, layer_index: usize) struct { LayerType, u32 } { + var self_attn_layer_index: u32 = 0; + var linear_attn_layer_index: u32 = 0; + for (cache.layer_types[0..layer_index]) |layer_type| { switch (layer_type) { .full_attention => self_attn_layer_index += 1, .linear_attention => linear_attn_layer_index += 1, } } - return switch (layer_types[layer_index]) { - .full_attention => .{ .full_attention = .{ .layer_dense_index = self_attn_layer_index } }, - .linear_attention => .{ .linear_attention = .{ .layer_dense_index = linear_attn_layer_index } }, + return switch (cache.layer_types[layer_index]) { + .full_attention => .{ .full_attention, self_attn_layer_index }, + .linear_attention => .{ .linear_attention, linear_attn_layer_index }, }; } }; diff --git a/examples/llm/models/qwen3_5/session.zig b/examples/llm/models/qwen3_5/session.zig index 83feeb72..578a290f 100644 --- a/examples/llm/models/qwen3_5/session.zig +++ b/examples/llm/models/qwen3_5/session.zig @@ -35,7 +35,7 @@ pub const Session = struct { errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); - var rng_buffers = try zml.Tensor.Rng.initBuffer(platform, seed, io, replicated_sharding); + var rng_buffers = try zml.Tensor.Rng.initBuffer(io, platform, replicated_sharding, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); return .{ diff --git a/zml/buffer.zig b/zml/buffer.zig index f97f7d7e..23e25785 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -169,7 +169,7 @@ pub const Buffer = struct { /// Creates a Buffer with a single element. pub fn scalar(io: std.Io, platform: *const Platform, val: anytype, dtype_: DataType, sharding: Sharding) !Buffer { const x = dtype_.constant(val); - return fromBytes(io, platform, Shape.init(.{}, dtype_), sharding, x.asBytes()); + return fromBytes(io, platform, .scalar(dtype_), sharding, x.asBytes()); } pub fn await(self: Buffer, io: std.Io) !void { diff --git a/zml/io/vfs/hf.zig b/zml/io/vfs/hf.zig index 614803a3..0f38c98f 100644 --- a/zml/io/vfs/hf.zig +++ b/zml/io/vfs/hf.zig @@ -290,6 +290,7 @@ pub const HF = struct { .path = "", }); const uri = try std.Uri.parse(url); + log.info("Fetching from HF: {s}", .{url}); var req = try self.client.request(.GET, uri, .{ .headers = .{ diff --git a/zml/mem.zig b/zml/mem.zig index 4e1610e5..2d4fa6a8 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -225,9 +225,7 @@ pub const FixedBufferPool = struct { pub const DynamicBufferPool = struct { const Node = struct { next: ?*Node }; - const alignment: std.mem.Alignment = .max(.of(u8), .of(Node)); - const ItemPtr = [*]align(alignment.toByteUnits()) u8; - const Slice = []align(alignment.toByteUnits()) u8; + const alignment: std.mem.Alignment = .of(Node); block_size: usize, max_blocks: usize, @@ -311,8 +309,8 @@ pub const DynamicBufferPool = struct { } } - fn nodeToSlice(self: *DynamicBufferPool, node: *Node) Slice { - const ptr: ItemPtr = @ptrCast(@alignCast(node)); + fn nodeToSlice(self: DynamicBufferPool, node: *Node) []u8 { + const ptr: [*]u8 = @ptrCast(node); return ptr[0..self.block_size]; } }; diff --git a/zml/nn.zig b/zml/nn.zig index 7eafc705..f7c9dd49 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -1574,7 +1574,7 @@ test sampleTokens { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, sampleTokens, .{ activations, .{ .topk = 4, .temperature = 2.0 }, rng }, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); - var rng_buffer = try zml.Tensor.Rng.initBuffer(platform, 0xdeadbeef, std.testing.io, replicated_sharding); + var rng_buffer = try zml.Tensor.Rng.initBuffer(std.testing.io, platform, replicated_sharding, 0xdeadbeef); defer rng_buffer._state.deinit(); const inf = std.math.inf(f32); diff --git a/zml/tensor.zig b/zml/tensor.zig index 14a9fee8..b660767d 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -5,11 +5,10 @@ const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); const stdx = @import("stdx"); -const Buffer = @import("buffer.zig").Buffer; -const Bufferized = @import("zml.zig").Bufferized; const CompilationContext = @import("module.zig").CompilationContext; const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; +const mem = @import("mem.zig"); const Memory = @import("platform.zig").Memory; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); @@ -521,17 +520,19 @@ pub const Tensor = struct { _state: Tensor, algorithm: dialects.stablehlo.RngAlgorithm.Type = .DEFAULT, + pub const Buffer = mem.Bufferized(Rng); + pub fn init() Rng { return .{ ._state = .init(.{2}, .u64) }; } - pub fn initBuffer(platform: *const Platform, seed: u128, io: std.Io, sharding: Sharding) !Bufferized(Rng) { + pub fn initBuffer(io: std.Io, platform: *const Platform, sharding: Sharding, seed: u128) !Buffer { return .{ ._state = try .fromBytes(io, platform, Shape.init(.{2}, .u64), sharding, std.mem.asBytes(&seed)), }; } - pub fn deinitBuffer(self: *Bufferized(Rng)) void { + pub fn deinitBuffer(self: *Buffer) void { self._state.deinit(); } @@ -631,7 +632,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.uniformStats, .{ Rng.init(), Shape.init(.{1024}, .f32), .{ .min = -2, .max = 10 } }, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); - var rng_buffer = try Rng.initBuffer(platform, 1234, std.testing.io, replicated_sharding); + var rng_buffer = try Rng.initBuffer(std.testing.io, platform, replicated_sharding, 1234); defer rng_buffer._state.deinit(); var rand, var stats = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Stats.uniformStats, .{rng_buffer}); @@ -740,9 +741,9 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.gumbelStats, .{ Rng.init(), tgt_dist }, platform, .{ .shardings = &.{replicated_sharding} }); defer exe.deinit(); - var rng_buffer = try Rng.initBuffer(platform, 1234, std.testing.io, replicated_sharding); + var rng_buffer = try Rng.initBuffer(std.testing.io, platform, replicated_sharding, 1234); defer rng_buffer._state.deinit(); - var tgt_dist_buffer: Buffer = try .fromBytes(std.testing.io, platform, tgt_dist.shape(), replicated_sharding, std.mem.sliceAsBytes(&tgt_dist_data)); + var tgt_dist_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, tgt_dist.shape(), replicated_sharding, @ptrCast(&tgt_dist_data)); defer tgt_dist_buffer.deinit(); var rand, var stats = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Stats.gumbelStats, .{ rng_buffer, tgt_dist_buffer }); @@ -2775,7 +2776,7 @@ pub const Tensor = struct { var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), replicated_sharding, std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); defer a_buffer.deinit(); - var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2]i32{ 0, 2 })); + var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), replicated_sharding, @ptrCast(&[2]i32{ 0, 2 })); defer scatter_indices_buffer.deinit(); var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2][3]i32{ .{ 10, 20, 30 }, .{ 70, 80, 90 } })); defer updates_buffer.deinit(); diff --git a/zml/tokenizer/iree/tokenizer.zig b/zml/tokenizer/iree/tokenizer.zig index 52fa5628..3526f189 100644 --- a/zml/tokenizer/iree/tokenizer.zig +++ b/zml/tokenizer/iree/tokenizer.zig @@ -30,7 +30,7 @@ inline fn stringView(s: []const u8) c.iree_string_view_t { } inline fn mutableStringView(s: []u8) c.iree_mutable_string_view_t { - return .{ .data = @ptrCast(s.ptr), .size = s.len }; + return .{ .data = s.ptr, .size = s.len }; } inline fn byteSpan(bytes: []u8) c.iree_byte_span_t { @@ -64,7 +64,7 @@ inline fn statusCode(status: c.iree_status_t) u32 { return @intCast(@intFromPtr(status.?) & StatusCodeMask); } -fn assertOk(status: c.iree_status_t) !void { +fn checkOk(status: c.iree_status_t) !void { if (status == null) return; const code = statusCode(status); _ = c.iree_status_ignore(status); @@ -90,12 +90,14 @@ fn assertOk(status: c.iree_status_t) !void { } pub const Tokenizer = struct { + // TODO: use the ArrayList pattern. Stop copying allocator everywhere allocator: std.mem.Allocator, inner: *c.iree_tokenizer_t, pub fn fromHuggingFaceJson(allocator: std.mem.Allocator, json: []const u8) !Tokenizer { + // TODO: wrap allocator into an iree allocator var raw: ?*c.iree_tokenizer_t = null; - try assertOk(c.iree_tokenizer_from_huggingface_json(stringView(json), c.iree_allocator_system(), &raw)); + try checkOk(c.iree_tokenizer_from_huggingface_json(stringView(json), c.iree_allocator_system(), &raw)); return .{ .allocator = allocator, .inner = raw.?, @@ -121,7 +123,7 @@ pub const Tokenizer = struct { } pub fn decoder(self: *const Tokenizer) !Decoder { - return Decoder.init(self); + return Decoder.init(self, self.allocator); } pub fn tokenId(self: *const Tokenizer, token: []const u8) ?u32 { @@ -142,7 +144,7 @@ pub const Tokenizer = struct { fn init(tokenizer: *const Tokenizer) !Encoder { var state_size: usize = undefined; - try assertOk(c.iree_tokenizer_encode_state_calculate_size(tokenizer.inner, &state_size)); + try checkOk(c.iree_tokenizer_encode_state_calculate_size(tokenizer.inner, &state_size)); const state_storage = try tokenizer.allocator.alloc(u8, state_size); errdefer tokenizer.allocator.free(state_storage); @@ -151,7 +153,7 @@ pub const Tokenizer = struct { errdefer tokenizer.allocator.free(transform_buffer); var state: ?*c.iree_tokenizer_encode_state_t = null; - try assertOk(c.iree_tokenizer_encode_state_initialize( + try checkOk(c.iree_tokenizer_encode_state_initialize( tokenizer.inner, byteSpan(state_storage), byteSpan(transform_buffer), @@ -271,7 +273,7 @@ pub const Tokenizer = struct { var consumed: usize = 0; var produced: usize = 0; - assertOk(c.iree_tokenizer_encode_state_feed( + checkOk(c.iree_tokenizer_encode_state_feed( self.state, stringView(remaining), tokenOutput(try writableTokenSliceGreedy(out, min_tokens)), @@ -306,7 +308,7 @@ pub const Tokenizer = struct { while (true) { var produced: usize = 0; - assertOk(c.iree_tokenizer_encode_state_finalize( + checkOk(c.iree_tokenizer_encode_state_finalize( self.state, tokenOutput(try writableTokenSliceGreedy(out, min_tokens)), &produced, @@ -340,14 +342,14 @@ pub const Tokenizer = struct { state_storage: []u8, state: *c.iree_tokenizer_decode_state_t, - fn init(tokenizer: *const Tokenizer) !Decoder { + fn init(tokenizer: *const Tokenizer, allocator: std.mem.Allocator) !Decoder { var state_size: usize = undefined; - try assertOk(c.iree_tokenizer_decode_state_calculate_size(tokenizer.inner, &state_size)); - const state_storage = try tokenizer.allocator.alloc(u8, state_size); - errdefer tokenizer.allocator.free(state_storage); + try checkOk(c.iree_tokenizer_decode_state_calculate_size(tokenizer.inner, &state_size)); + const state_storage = try allocator.alloc(u8, state_size); + errdefer allocator.free(state_storage); var state: ?*c.iree_tokenizer_decode_state_t = null; - try assertOk(c.iree_tokenizer_decode_state_initialize( + try checkOk(c.iree_tokenizer_decode_state_initialize( tokenizer.inner, c.IREE_TOKENIZER_DECODE_FLAG_NONE, byteSpan(state_storage), @@ -369,7 +371,7 @@ pub const Tokenizer = struct { pub fn reset(self: *Decoder) !void { c.iree_tokenizer_decode_state_deinitialize(self.state); var state: ?*c.iree_tokenizer_decode_state_t = null; - try assertOk(c.iree_tokenizer_decode_state_initialize( + try checkOk(c.iree_tokenizer_decode_state_initialize( self.tokenizer.inner, c.IREE_TOKENIZER_DECODE_FLAG_NONE, byteSpan(self.state_storage), @@ -382,7 +384,7 @@ pub const Tokenizer = struct { std.debug.assert(out.len != 0); var consumed: usize = 0; var produced: usize = 0; - try assertOk(c.iree_tokenizer_decode_state_feed( + try checkOk(c.iree_tokenizer_decode_state_feed( self.state, tokenIdList(&.{token_id}), mutableStringView(out), @@ -396,7 +398,7 @@ pub const Tokenizer = struct { pub fn finalize(self: *Decoder, out: []u8) ![]u8 { std.debug.assert(out.len != 0); var produced: usize = 0; - try assertOk(c.iree_tokenizer_decode_state_finalize( + try checkOk(c.iree_tokenizer_decode_state_finalize( self.state, mutableStringView(out), &produced, @@ -411,7 +413,7 @@ pub const Tokenizer = struct { var consumed: usize = 0; var produced: usize = 0; - assertOk(c.iree_tokenizer_decode_state_feed( + checkOk(c.iree_tokenizer_decode_state_feed( self.state, tokenIdList(remaining), mutableStringView(try out.writableSliceGreedy(min_output)), @@ -444,7 +446,7 @@ pub const Tokenizer = struct { while (true) { var produced: usize = 0; - assertOk(c.iree_tokenizer_decode_state_finalize( + checkOk(c.iree_tokenizer_decode_state_finalize( self.state, mutableStringView(try out.writableSliceGreedy(min_output)), &produced, diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig index 39e359b2..ce863e3a 100644 --- a/zml/tokenizer/tokenizer.zig +++ b/zml/tokenizer/tokenizer.zig @@ -40,6 +40,23 @@ pub const Tokenizer = union(Tokenizers) { inline else => |*encoder_| try allocator.dupe(u32, try encoder_.encode(text)), }; } + + pub fn encode(self: *Encoder, writer: *std.Io.Writer, text: []const u8) !void { + return switch (self.*) { + .iree => |*e| { + try e.feed(text, writer); + try e.finalize(writer); + }, + inline else => |*e| { + const encoded: []const u32 = try e.encode(text); + try writer.writeAll(std.mem.sliceAsBytes(encoded)); + }, + }; + } + + pub fn appendTokens(_: *Encoder, writer: *std.Io.Writer, tokens: []const u32) !void { + try writer.writeAll(@ptrCast(tokens)); + } }; pub const Decoder = union(Tokenizers) { From b0b7e02692a018c6b2ccae07004282f4eea10802 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Wed, 29 Apr 2026 18:15:26 +0200 Subject: [PATCH 460/881] zml: remove hftokenizers (#518) This is the second take of #463, now that #482 is merged. --- .bazelrc | 1 - .vscode/settings.json | 13 +- MODULE.bazel | 25 - zml/tokenizer/BUILD.bazel | 1 - zml/tokenizer/hftokenizers/.gitignore | 1 - zml/tokenizer/hftokenizers/BUILD.bazel | 35 - zml/tokenizer/hftokenizers/Cargo.lock | 713 -------------------- zml/tokenizer/hftokenizers/Cargo.toml | 15 - zml/tokenizer/hftokenizers/hftokenizers.h | 17 - zml/tokenizer/hftokenizers/hftokenizers.rs | 109 --- zml/tokenizer/hftokenizers/hftokenizers.zig | 120 ---- zml/tokenizer/tokenizer.zig | 5 - 12 files changed, 4 insertions(+), 1051 deletions(-) delete mode 100644 zml/tokenizer/hftokenizers/.gitignore delete mode 100644 zml/tokenizer/hftokenizers/BUILD.bazel delete mode 100644 zml/tokenizer/hftokenizers/Cargo.lock delete mode 100644 zml/tokenizer/hftokenizers/Cargo.toml delete mode 100644 zml/tokenizer/hftokenizers/hftokenizers.h delete mode 100644 zml/tokenizer/hftokenizers/hftokenizers.rs delete mode 100644 zml/tokenizer/hftokenizers/hftokenizers.zig diff --git a/.bazelrc b/.bazelrc index e56e95be..76e445c7 100644 --- a/.bazelrc +++ b/.bazelrc @@ -124,7 +124,6 @@ common:release --@rules_zig//zig/settings:mode=release_safe # --config=native builds everything optimized for the local CPU common:native --copt=-march=native common:native --@rules_zig//zig/settings:zigopt=-mcpu=native -common:native --@@rules_rs++rules_rust+rules_rust//rust/settings:extra_rustc_flag=-Ctarget-cpu=native common:linux --host_platform=//platforms:host_glibc diff --git a/.vscode/settings.json b/.vscode/settings.json index 82050af6..772705fb 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -9,15 +9,13 @@ "**/bazel-*": true }, "files.watcherExclude": { - "**/bazel-*": true, + "**/bazel-*": true }, // "files.trimTrailingWhitespace": true, "files.insertFinalNewline": true, "bazel.buildifierFixOnFormat": true, "bazel.queriesShareServer": true, - "bazel.pathsToIgnore": [ - "." - ], + "bazel.pathsToIgnore": ["."], "bazel.buildifierExecutable": "@buildifier_prebuilt//:buildifier", "C_Cpp.formatting": "clangFormat", "C_Cpp.clang_format_style": "Microsoft", @@ -28,10 +26,7 @@ "editor.suggest.insertMode": "replace", "editor.stickyScroll.defaultModel": "foldingProviderModel", "editor.codeActionsOnSave": { - "source.fixAll": "explicit", + "source.fixAll": "explicit" } - }, - "rust-analyzer.linkedProjects": [ - "${workspaceFolder}/zml/tokenizer/hftokenizers/Cargo.toml" - ] + } } diff --git a/MODULE.bazel b/MODULE.bazel index ccefd924..f615687b 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -26,7 +26,6 @@ bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep(name = "rules_oci", version = "2.2.7") bazel_dep(name = "rules_proto", version = "7.1.0") bazel_dep(name = "rules_python", version = "1.9.0") -bazel_dep(name = "rules_rs", version = "0.0.44") bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "rules_zig", version = "0.12.2") git_override( @@ -210,30 +209,6 @@ llvm = use_extension("//third_party/xla:llvm.bzl", "llvm") llvm.configure() use_repo(llvm, "llvm-project") -rs_toolchains = use_extension("@rules_rs//rs/experimental/toolchains:module_extension.bzl", "toolchains") -rs_toolchains.toolchain( - edition = "2024", - version = "1.94.0", -) -use_repo(rs_toolchains, "default_rust_toolchains") - -register_toolchains("@default_rust_toolchains//:all") - -crate = use_extension("@rules_rs//rs:extensions.bzl", "crate") -crate.from_cargo( - name = "crates", - cargo_lock = "//zml/tokenizer/hftokenizers:Cargo.lock", - cargo_toml = "//zml/tokenizer/hftokenizers:Cargo.toml", - platform_triples = [ - "aarch64-apple-darwin", - "x86_64-apple-darwin", - "aarch64-unknown-linux-gnu", - "x86_64-unknown-linux-gnu", - ], - use_experimental_platforms = True, -) -use_repo(crate, "crates") - apt = use_extension("@rules_distroless//apt:extensions.bzl", "apt") apt.install( name = "apt_cuda", diff --git a/zml/tokenizer/BUILD.bazel b/zml/tokenizer/BUILD.bazel index 92082ff5..da1c8bda 100644 --- a/zml/tokenizer/BUILD.bazel +++ b/zml/tokenizer/BUILD.bazel @@ -11,7 +11,6 @@ zig_library( deps = [ "//ffi:zig", "//stdx", - "//zml/tokenizer/hftokenizers", "//zml/tokenizer/iree", "//zml/tokenizer/sentencepiece", ], diff --git a/zml/tokenizer/hftokenizers/.gitignore b/zml/tokenizer/hftokenizers/.gitignore deleted file mode 100644 index 2f7896d1..00000000 --- a/zml/tokenizer/hftokenizers/.gitignore +++ /dev/null @@ -1 +0,0 @@ -target/ diff --git a/zml/tokenizer/hftokenizers/BUILD.bazel b/zml/tokenizer/hftokenizers/BUILD.bazel deleted file mode 100644 index 43ee1fbe..00000000 --- a/zml/tokenizer/hftokenizers/BUILD.bazel +++ /dev/null @@ -1,35 +0,0 @@ -load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_rs//rs:rust_static_library.bzl", "rust_static_library") -load("@rules_zig//zig:defs.bzl", "zig_library") - -rust_static_library( - name = "hftokenizers_rs", - srcs = ["hftokenizers.rs"], - crate_name = "zml_tokenizer_hftokenizers", - edition = "2021", - proc_macro_deps = ["@crates//:tokenizers"], - deps = ["@crates//:tokenizers"], -) - -cc_library( - name = "hftokenizers_cc", - tags = ["manual"], - hdrs = ["hftokenizers.h"], - visibility = ["//visibility:public"], - deps = [ - ":hftokenizers_rs", - "//ffi:cc", - ], -) - -zig_library( - name = "hftokenizers", - tags = ["manual"], - main = "hftokenizers.zig", - visibility = ["//visibility:public"], - deps = [ - ":hftokenizers_cc", - "//ffi:zig", - "//stdx", - ], -) diff --git a/zml/tokenizer/hftokenizers/Cargo.lock b/zml/tokenizer/hftokenizers/Cargo.lock deleted file mode 100644 index 7164e77b..00000000 --- a/zml/tokenizer/hftokenizers/Cargo.lock +++ /dev/null @@ -1,713 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "ahash" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" -dependencies = [ - "cfg-if", - "getrandom", - "once_cell", - "serde", - "version_check", - "zerocopy 0.8.26", -] - -[[package]] -name = "aho-corasick" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" -dependencies = [ - "memchr", -] - -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - -[[package]] -name = "bitflags" -version = "2.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "castaway" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" -dependencies = [ - "rustversion", -] - -[[package]] -name = "cc" -version = "1.2.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f" -dependencies = [ - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "compact_str" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" -dependencies = [ - "castaway", - "cfg-if", - "itoa", - "rustversion", - "ryu", - "serde", - "static_assertions", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" - -[[package]] -name = "darling" -version = "0.20.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" -dependencies = [ - "darling_core", - "darling_macro", -] - -[[package]] -name = "darling_core" -version = "0.20.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn", -] - -[[package]] -name = "darling_macro" -version = "0.20.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" -dependencies = [ - "darling_core", - "quote", - "syn", -] - -[[package]] -name = "dary_heap" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728" -dependencies = [ - "serde", -] - -[[package]] -name = "derive_builder" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "derive_builder_macro" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" -dependencies = [ - "derive_builder_core", - "syn", -] - -[[package]] -name = "either" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" - -[[package]] -name = "esaxx-rs" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" -dependencies = [ - "cc", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "getrandom" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" -dependencies = [ - "cfg-if", - "libc", - "r-efi", - "wasi", -] - -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - -[[package]] -name = "itertools" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" - -[[package]] -name = "libc" -version = "0.2.169" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" - -[[package]] -name = "log" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" - -[[package]] -name = "macro_rules_attribute" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13" -dependencies = [ - "macro_rules_attribute-proc_macro", - "paste", -] - -[[package]] -name = "macro_rules_attribute-proc_macro" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568" - -[[package]] -name = "memchr" -version = "2.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" - -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - -[[package]] -name = "monostate" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e" -dependencies = [ - "monostate-impl", - "serde", -] - -[[package]] -name = "monostate-impl" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - -[[package]] -name = "once_cell" -version = "1.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" - -[[package]] -name = "onig" -version = "6.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0" -dependencies = [ - "bitflags", - "libc", - "once_cell", - "onig_sys", -] - -[[package]] -name = "onig_sys" -version = "69.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc" -dependencies = [ - "cc", - "pkg-config", -] - -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - -[[package]] -name = "pkg-config" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" - -[[package]] -name = "ppv-lite86" -version = "0.2.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" -dependencies = [ - "zerocopy 0.7.35", -] - -[[package]] -name = "proc-macro2" -version = "1.0.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "rand" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" -dependencies = [ - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" -dependencies = [ - "getrandom", -] - -[[package]] -name = "rayon" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-cond" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" -dependencies = [ - "either", - "itertools", - "rayon", -] - -[[package]] -name = "rayon-core" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - -[[package]] -name = "regex" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" - -[[package]] -name = "rustversion" -version = "1.0.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" - -[[package]] -name = "ryu" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" - -[[package]] -name = "serde" -version = "1.0.216" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.216" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.134" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" -dependencies = [ - "itoa", - "memchr", - "ryu", - "serde", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "smallvec" -version = "1.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" - -[[package]] -name = "spm_precompiled" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" -dependencies = [ - "base64", - "nom", - "serde", - "unicode-segmentation", -] - -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "syn" -version = "2.0.91" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "thiserror" -version = "2.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tokenizers" -version = "0.22.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223" -dependencies = [ - "ahash", - "aho-corasick", - "compact_str", - "dary_heap", - "derive_builder", - "esaxx-rs", - "getrandom", - "itertools", - "log", - "macro_rules_attribute", - "monostate", - "onig", - "paste", - "rand", - "rayon", - "rayon-cond", - "regex", - "regex-syntax", - "serde", - "serde_json", - "spm_precompiled", - "thiserror", - "unicode-normalization-alignments", - "unicode-segmentation", - "unicode_categories", -] - -[[package]] -name = "unicode-ident" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" - -[[package]] -name = "unicode-normalization-alignments" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" -dependencies = [ - "smallvec", -] - -[[package]] -name = "unicode-segmentation" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" - -[[package]] -name = "unicode_categories" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "wasi" -version = "0.14.2+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" -dependencies = [ - "wit-bindgen-rt", -] - -[[package]] -name = "wit-bindgen-rt" -version = "0.39.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" -dependencies = [ - "bitflags", -] - -[[package]] -name = "zerocopy" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" -dependencies = [ - "byteorder", - "zerocopy-derive 0.7.35", -] - -[[package]] -name = "zerocopy" -version = "0.8.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" -dependencies = [ - "zerocopy-derive 0.8.26", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zml_tokenizer_hftokenizers" -version = "0.1.0" -dependencies = [ - "tokenizers", -] diff --git a/zml/tokenizer/hftokenizers/Cargo.toml b/zml/tokenizer/hftokenizers/Cargo.toml deleted file mode 100644 index f905c004..00000000 --- a/zml/tokenizer/hftokenizers/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "zml_tokenizer_hftokenizers" -version = "0.1.0" -edition = "2021" - -[dependencies] -tokenizers = { version = "0.22.2", default-features = false, features = [ - # We don't need/want the progressbar feature - "onig", - "esaxx_fast", -] } - -[lib] -name = "zml_tokenizer_hftokenizers" -path = "hftokenizers.rs" diff --git a/zml/tokenizer/hftokenizers/hftokenizers.h b/zml/tokenizer/hftokenizers/hftokenizers.h deleted file mode 100644 index f779b575..00000000 --- a/zml/tokenizer/hftokenizers/hftokenizers.h +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -#include -#include - -#include "ffi/zig_slice.h" - -typedef struct hftokenizers hftokenizers; - -hftokenizers *hftokenizers_new(zig_slice); -hftokenizers *hftokenizers_new_from_bytes(zig_slice); -void hftokenizers_drop(hftokenizers *tokenizer); -zig_slice hftokenizers_encode(hftokenizers *tokenizer, zig_slice text); -void hftokenizers_tokens_drop(zig_slice tokens); -zig_slice hftokenizers_decode(hftokenizers *tokenizer, zig_slice tokens); -void hftokenizers_str_drop(zig_slice text); -uint32_t hftokenizers_token_to_id(hftokenizers *tokenizer, zig_slice token); diff --git a/zml/tokenizer/hftokenizers/hftokenizers.rs b/zml/tokenizer/hftokenizers/hftokenizers.rs deleted file mode 100644 index 048283cc..00000000 --- a/zml/tokenizer/hftokenizers/hftokenizers.rs +++ /dev/null @@ -1,109 +0,0 @@ -#[repr(C)] -struct ZigSlice { - ptr: *mut T, - len: usize, -} - -impl ZigSlice { - fn as_slice(&self) -> &[T] { - unsafe { std::slice::from_raw_parts(self.ptr, self.len) } - } - - fn as_slice_mut(&self) -> &mut [T] { - unsafe { std::slice::from_raw_parts_mut(self.ptr, self.len) } - } -} - -#[no_mangle] -extern "C" fn hftokenizers_new(path: ZigSlice) -> *mut tokenizers::Tokenizer { - return Box::into_raw(Box::new( - tokenizers::Tokenizer::from_file(std::path::Path::new( - std::str::from_utf8(path.as_slice()).unwrap(), - )) - .unwrap(), - )); -} - -#[no_mangle] -extern "C" fn hftokenizers_new_from_bytes(bytes: ZigSlice) -> *mut tokenizers::Tokenizer { - return Box::into_raw(Box::new( - tokenizers::Tokenizer::from_bytes(bytes.as_slice()) - .unwrap(), - )); -} - -#[no_mangle] -extern "C" fn hftokenizers_drop(t: *mut tokenizers::Tokenizer) { - drop(unsafe { Box::from_raw(t) }); -} - -#[no_mangle] -extern "C" fn hftokenizers_encode( - t: *mut tokenizers::Tokenizer, - string: ZigSlice, -) -> ZigSlice { - let input_str = std::str::from_utf8(string.as_slice()).unwrap(); - let encoded = unsafe { t.as_ref() } - .unwrap() - .encode_fast(input_str, false) - .unwrap(); - - // Convert the result to a boxed slice - let mut ids: Box<[u32]> = encoded.get_ids().to_owned().into_boxed_slice(); - - // Retrieve the zig slice associated to the boxed slice. - let slice = ZigSlice { - ptr: ids.as_mut_ptr(), - len: ids.len(), - }; - - // Leak the box so that it's not deallocated. - Box::leak(ids); - - return slice; -} - -#[no_mangle] -extern "C" fn hftokenizers_tokens_drop(tokens: ZigSlice) { - // Reconstruct the Box from the zig slice so that it's dropped. - drop(unsafe { Box::from_raw(tokens.as_slice_mut()) }); -} - -#[no_mangle] -extern "C" fn hftokenizers_decode( - t: *mut tokenizers::Tokenizer, - ids: ZigSlice, -) -> ZigSlice { - let decoded = unsafe { t.as_ref() } - .unwrap() - .decode(ids.as_slice(), false) - .unwrap(); - - // Convert the result to a boxed slice - let mut string: Box<[u8]> = decoded.into_bytes().into_boxed_slice(); - - // Retrieve the zig slice associated to the boxed slice. - let slice = ZigSlice { - ptr: string.as_mut_ptr(), - len: string.len(), - }; - - // Leak the box so that it's not deallocated. - Box::leak(string); - - return slice; -} - -#[no_mangle] -extern "C" fn hftokenizers_str_drop(tokens: ZigSlice) { - drop(unsafe { Box::from_raw(tokens.as_slice_mut()) }); -} - -#[no_mangle] -extern "C" fn hftokenizers_token_to_id(t: *mut tokenizers::Tokenizer, token: ZigSlice) -> u32 { - let id = unsafe { t.as_ref() } - .unwrap() - .token_to_id(std::str::from_utf8(token.as_slice()).unwrap()) - .unwrap_or(u32::MAX); - return id; -} diff --git a/zml/tokenizer/hftokenizers/hftokenizers.zig b/zml/tokenizer/hftokenizers/hftokenizers.zig deleted file mode 100644 index 4b845788..00000000 --- a/zml/tokenizer/hftokenizers/hftokenizers.zig +++ /dev/null @@ -1,120 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const ffi = @import("ffi"); -const stdx = @import("stdx"); - -pub const Encoder = struct { - inner: *HFTokenizer, - current_ids: ?[]const u32 = null, - - fn init(inner: *HFTokenizer) Encoder { - return .{ .inner = inner }; - } - - pub fn reset(self: *Encoder) void { - if (self.current_ids) |current_ids_| { - c.hftokenizers_tokens_drop(ffi.ZigSlice.from(current_ids_)); - self.current_ids = null; - } - } - - pub fn deinit(self: *Encoder) void { - self.reset(); - } - - pub fn encode(self: *Encoder, input: []const u8) ![]const u32 { - self.reset(); - self.current_ids = ffi.ZigSlice.to(u32, c.hftokenizers_encode(@ptrCast(self.inner), ffi.ZigSlice.from(input))); - return self.ids(); - } - - pub fn ids(self: *const Encoder) []const u32 { - return self.current_ids orelse &.{}; - } -}; - -pub const Decoder = struct { - const StringBuffer = stdx.BoundedArray(u8, 128); - const TokensIdsBuffer = stdx.BoundedArray(u32, 4); - - inner: *HFTokenizer, - current_string: ?[]const u8 = null, - last_string: StringBuffer = .empty, - last_token_ids: TokensIdsBuffer = .empty, - - fn init(inner: *HFTokenizer) Decoder { - return .{ .inner = inner }; - } - - pub fn deinit(self: *Decoder) void { - self.reset(); - } - - pub fn reset(self: *Decoder) void { - if (self.current_string) |current_string_| { - c.hftokenizers_str_drop(ffi.ZigSlice.from(current_string_)); - self.current_string = null; - } - } - - pub fn decode(self: *Decoder, ids: []const u32) ![]const u8 { - self.reset(); - self.current_string = ffi.ZigSlice.to(u8, c.hftokenizers_decode(@ptrCast(self.inner), ffi.ZigSlice.from(ids))); - return self.string(); - } - - pub fn string(self: *const Decoder) []const u8 { - return self.current_string orelse &.{}; - } - - pub fn next(self: *Decoder, token_id: u32) !?[]const u8 { - if (self.last_token_ids.len >= self.last_token_ids.capacity()) { - _ = self.last_token_ids.orderedRemove(0); - } - self.last_token_ids.appendAssumeCapacity(token_id); - const new_string = try self.decode(self.last_token_ids.constSlice()); - if (self.last_string.len == 0) { - self.last_string = try StringBuffer.fromSlice(new_string); - return new_string; - } - var view = try std.unicode.Utf8View.init(self.last_string.constSlice()); - var it = view.iterator(); - while (it.nextCodepointSlice()) |cp| { - const start = it.i - cp.len; - if (std.mem.startsWith(u8, new_string, self.last_string.constSlice()[start..])) { - const chunk = new_string[self.last_string.len - start ..]; - self.last_string = try StringBuffer.fromSlice(new_string); - return chunk; - } - } - return null; - } -}; - -pub const HFTokenizer = opaque { - pub fn fromFile(model: []const u8) !*HFTokenizer { - return @ptrCast(c.hftokenizers_new(ffi.ZigSlice.from(model))); - } - - pub fn fromBytes(bytes: []const u8) !*HFTokenizer { - return @ptrCast(c.hftokenizers_new_from_bytes(ffi.ZigSlice.from(bytes))); - } - - pub fn deinit(self: *HFTokenizer) void { - return c.hftokenizers_drop(@ptrCast(self)); - } - - pub fn encoder(self: *HFTokenizer) !Encoder { - return Encoder.init(self); - } - - pub fn decoder(self: *HFTokenizer) !Decoder { - return Decoder.init(self); - } - - pub fn tokenId(self: *HFTokenizer, token: []const u8) ?u32 { - const id = c.hftokenizers_token_to_id(@ptrCast(self), ffi.ZigSlice.from(token)); - return if (id == std.math.maxInt(u32)) null else id; - } -}; diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig index ce863e3a..b0aae1c6 100644 --- a/zml/tokenizer/tokenizer.zig +++ b/zml/tokenizer/tokenizer.zig @@ -1,6 +1,5 @@ const std = @import("std"); -pub const hftokenizers = @import("hftokenizers"); pub const iree = @import("iree"); pub const sentencepiece = @import("sentencepiece"); @@ -9,7 +8,6 @@ pub const homemade = @import("homemade.zig"); const log = std.log.scoped(.@"zml/tokenizer"); const Tokenizers = enum { - hftokenizers, iree, sentencepiece, homemade, @@ -17,7 +15,6 @@ const Tokenizers = enum { pub const Tokenizer = union(Tokenizers) { pub const Encoder = union(Tokenizers) { - hftokenizers: hftokenizers.Encoder, iree: iree.Tokenizer.Encoder, sentencepiece: sentencepiece.Encoder, homemade: homemade.Encoder, @@ -60,7 +57,6 @@ pub const Tokenizer = union(Tokenizers) { }; pub const Decoder = union(Tokenizers) { - hftokenizers: hftokenizers.Decoder, iree: iree.Tokenizer.Decoder, sentencepiece: sentencepiece.Decoder, homemade: homemade.Decoder, @@ -106,7 +102,6 @@ pub const Tokenizer = union(Tokenizers) { } }; - hftokenizers: *hftokenizers.HFTokenizer, iree: iree.Tokenizer, sentencepiece: *sentencepiece.SentencePieceProcessor, homemade: *homemade.Tokenizer, From df4bb4c3dd1ff199239e176e7ee2510f2db139a8 Mon Sep 17 00:00:00 2001 From: Olivier Boulant Date: Wed, 29 Apr 2026 18:24:43 +0200 Subject: [PATCH 461/881] bazel: suppress SWIG c-target warning due to intended behavior (#504) This suppresses SWIG Warning 524 in `bazel/swig.bzl`. The custom SWIG rule intentionally parses C++ interfaces while generating a C ABI wrapper for Zig consumption (in [sentencepiece.zig](zml/tokenizer/sentencepiece/sentencepiece.zig)), so this warning is expected noise from an intentional -c backend selection rather than a build problem. **Reproduce** ``` $ bazel build //examples/llm [...] SWIG:1: Warning 524: Experimental target language. Target language C specified by -c is an experimental language. See the 'Target Languages' section in the Introduction chapter of the SWIG documentation. [...] ``` **Validation** * Built `//examples/llm:llm` after a clean * Confirmed Warning 524 no longer appears * Build still succeeds without changing generated binding behavior --- bazel/swig.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/bazel/swig.bzl b/bazel/swig.bzl index d81310fb..d6274860 100644 --- a/bazel/swig.bzl +++ b/bazel/swig.bzl @@ -60,6 +60,7 @@ def _swig_cc_library_impl(ctx): ] args.add("-o", output_cpp) args.add("-w-305") + args.add("-w-524") args.add(ctx.file.interface) inputs = depset(ctx.attr.srcs, transitive = [ From 1c3b505b9e68d87474e7f927b84556cb31688341 Mon Sep 17 00:00:00 2001 From: Olivier Boulant Date: Wed, 29 Apr 2026 18:26:58 +0200 Subject: [PATCH 462/881] swig(tokenizer): remove unsupported SWIG optimal hint (#501) This PR removes `optimal="1"` from the `[const std::string&]` out typemap in `zml/tokenizer/sentencepiece/sentencepiece.i`. SWIG cannot apply the requested optimal code path for the `%extend std::string { data() }` wrapper and emitted `Warning 474: ` during SwigC. Removing the unsupported hint preserves the generated behavior while eliminating the warning. **Reproduce** ``` $ bazel build //examples/llm [...] zml/tokenizer/sentencepiece/sentencepiece.i:41: Warning 474: Method std::string::data() const usage of the optimal attribute ignored zml/tokenizer/sentencepiece/sentencepiece.i:25: Warning 474: in the out typemap as the following cannot be used to generate optimal code: cppresult = (std::string *) &std_string_data__SWIG((std::string const *)arg1); [...] ``` **Validation** * Rebuilt `//examples/llm:llm` and a clean * Confirmed `Warning 474: ` no longer appears * Confirmed build still succeeds * Warning 524 remains and is not addressed in this PR --- zml/tokenizer/sentencepiece/sentencepiece.i | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/tokenizer/sentencepiece/sentencepiece.i b/zml/tokenizer/sentencepiece/sentencepiece.i index 35ccc057..01476b59 100644 --- a/zml/tokenizer/sentencepiece/sentencepiece.i +++ b/zml/tokenizer/sentencepiece/sentencepiece.i @@ -19,7 +19,7 @@ $1 = absl::string_view((char *)$input.ptr, $input.len); } -%typemap(out, optimal="1") const std::string& %{ +%typemap(out) const std::string& %{ $result.ptr = (void *)($1->data()); $result.len = (size_t)($1->length()); %} From 8e4c333007de3cb16d8f48953ae8560415aec0b8 Mon Sep 17 00:00:00 2001 From: elogir <90246794+elogir@users.noreply.github.com> Date: Thu, 30 Apr 2026 16:13:48 +0200 Subject: [PATCH 463/881] zml: replace Python Triton/Mosaic IR generation with native Zig DSLs (#517) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drops the Python build-time dependency for Triton and Mosaic TPU kernels. ZML now emits TTIR and Mosaic TPU IR directly from Zig, against the same MLIR context as the rest of the compiler. So no more JAX, Triton at build time and a lighter sandbox. Split into two commits: - **`c72f44884` — Add Triton and Mosaic Zig DSL.** Additive: IR builders (`kernels/triton/`, `kernels/mosaic_tpu/`), upstream dialect bindings (`ttir`, `mosaic_tpu`, plus `arith`/`scf`/`math`/`memref`/`vector`/ `affine`/`cf`), and the high-level `zml.kernel.{triton,mosaic_tpu}.Kernel(Config, spec)` declarative form. Comes with author guides in each `README.md`. - **`cb61e529f` — Remove Python and replace with Zig kernels.** Migrates the two production kernels (`unified_attention` for Triton, `ragged_paged` for Mosaic TPU) and deletes everything Python related in the sandbox and what was generating the IR. --------- Co-authored-by: Steeve Morin --- MODULE.bazel | 28 +- MODULE.bazel.lock | 334 +-- kernels/common/BUILD.bazel | 22 + kernels/common/args.zig | 6 + kernels/common/control_flow.zig | 188 ++ kernels/common/dsl.zig | 17 + kernels/mosaic_tpu/BUILD.bazel | 24 + kernels/mosaic_tpu/README.md | 827 ++++++ kernels/mosaic_tpu/builder.zig | 2200 ++++++++++++++++ kernels/mosaic_tpu/dtype.zig | 72 + kernels/triton/BUILD.bazel | 24 + kernels/triton/PORTING.md | 748 ++++++ kernels/triton/README.md | 1091 ++++++++ kernels/triton/builder.zig | 2220 +++++++++++++++++ kernels/triton/dtype.zig | 72 + mlir/BUILD.bazel | 4 + mlir/dialects/BUILD.bazel | 9 + mlir/dialects/affine.zig | 414 +++ mlir/dialects/arith.zig | 408 +++ mlir/dialects/cf.zig | 166 ++ mlir/dialects/dialects.zig | 9 + mlir/dialects/func.zig | 12 +- mlir/dialects/math.zig | 212 ++ mlir/dialects/memref.zig | 665 +++++ mlir/dialects/mosaic_tpu/BUILD.bazel | 19 + mlir/dialects/mosaic_tpu/mosaic_tpu.zig | 1178 +++++++++ mlir/dialects/scf.zig | 355 +++ mlir/dialects/ttir/BUILD.bazel | 37 + mlir/dialects/ttir/ttir.zig | 1293 ++++++++++ mlir/dialects/ttir/ttir_capi.cc | 113 + mlir/dialects/ttir/ttir_capi.h | 47 + mlir/dialects/vector.zig | 776 ++++++ mlir/mlir.h | 8 + mlir/mlir.zig | 309 ++- mlir/register.cc | 7 + platforms/tpu/BUILD.bazel | 76 +- platforms/tpu/gen_ir.py | 636 ----- platforms/tpu/gen_ir.zig | 117 - platforms/tpu/kernel.py | 910 ------- platforms/tpu/libpjrt_tpu.BUILD.bazel | 9 - platforms/tpu/libpython.h | 2 - platforms/tpu/ragged_paged.zig | 662 +++++ platforms/tpu/requirements.in | 3 - platforms/tpu/requirements.lock.txt | 226 -- third_party/mosaic_tpu/BUILD.bazel | 1 + third_party/mosaic_tpu/mosaic_tpu.bazel | 275 ++ third_party/mosaic_tpu/repo.bzl | 11 + third_party/non_module_deps.bzl | 20 +- third_party/xla/xla.bzl | 16 +- zml/BUILD.bazel | 13 +- zml/attention.zig | 3 +- .../mosaic_tpu_kernels/ragged_attention.zig | 9 + zml/attention/paged_attention.zig | 4 +- zml/attention/tpu_attention.zig | 243 +- zml/attention/triton/BUILD.bazel | 124 - zml/attention/triton/fake_plugin/__init__.py | 0 zml/attention/triton/fake_plugin/compiler.py | 97 - zml/attention/triton/fake_plugin/driver.py | 55 - zml/attention/triton/generate.py | 365 --- zml/attention/triton/generate.zig | 111 - zml/attention/triton/libpython.h | 3 - zml/attention/triton/requirements.in | 1 - zml/attention/triton/requirements.lock.txt | 20 - .../triton/triton_kernels/__init__.py | 0 .../triton_kernels/unified_attention.py | 1073 -------- .../{triton.zig => triton_attention.zig} | 491 ++-- .../triton_kernels/unified_attention.zig | 1136 +++++++++ zml/kernel.zig | 309 +++ zml/module.zig | 6 +- zml/platform.zig | 22 +- zml/zml.zig | 1 + 71 files changed, 16207 insertions(+), 4757 deletions(-) create mode 100644 kernels/common/BUILD.bazel create mode 100644 kernels/common/args.zig create mode 100644 kernels/common/control_flow.zig create mode 100644 kernels/common/dsl.zig create mode 100644 kernels/mosaic_tpu/BUILD.bazel create mode 100644 kernels/mosaic_tpu/README.md create mode 100644 kernels/mosaic_tpu/builder.zig create mode 100644 kernels/mosaic_tpu/dtype.zig create mode 100644 kernels/triton/BUILD.bazel create mode 100644 kernels/triton/PORTING.md create mode 100644 kernels/triton/README.md create mode 100644 kernels/triton/builder.zig create mode 100644 kernels/triton/dtype.zig create mode 100644 mlir/dialects/affine.zig create mode 100644 mlir/dialects/arith.zig create mode 100644 mlir/dialects/cf.zig create mode 100644 mlir/dialects/math.zig create mode 100644 mlir/dialects/memref.zig create mode 100644 mlir/dialects/mosaic_tpu/BUILD.bazel create mode 100644 mlir/dialects/mosaic_tpu/mosaic_tpu.zig create mode 100644 mlir/dialects/scf.zig create mode 100644 mlir/dialects/ttir/BUILD.bazel create mode 100644 mlir/dialects/ttir/ttir.zig create mode 100644 mlir/dialects/ttir/ttir_capi.cc create mode 100644 mlir/dialects/ttir/ttir_capi.h create mode 100644 mlir/dialects/vector.zig delete mode 100644 platforms/tpu/gen_ir.py delete mode 100644 platforms/tpu/gen_ir.zig delete mode 100644 platforms/tpu/kernel.py delete mode 100644 platforms/tpu/libpython.h create mode 100644 platforms/tpu/ragged_paged.zig delete mode 100644 platforms/tpu/requirements.in delete mode 100644 platforms/tpu/requirements.lock.txt create mode 100644 third_party/mosaic_tpu/BUILD.bazel create mode 100644 third_party/mosaic_tpu/mosaic_tpu.bazel create mode 100644 third_party/mosaic_tpu/repo.bzl create mode 100644 zml/attention/mosaic_tpu_kernels/ragged_attention.zig delete mode 100644 zml/attention/triton/BUILD.bazel delete mode 100755 zml/attention/triton/fake_plugin/__init__.py delete mode 100755 zml/attention/triton/fake_plugin/compiler.py delete mode 100755 zml/attention/triton/fake_plugin/driver.py delete mode 100755 zml/attention/triton/generate.py delete mode 100644 zml/attention/triton/generate.zig delete mode 100644 zml/attention/triton/libpython.h delete mode 100644 zml/attention/triton/requirements.in delete mode 100644 zml/attention/triton/requirements.lock.txt delete mode 100755 zml/attention/triton/triton_kernels/__init__.py delete mode 100755 zml/attention/triton/triton_kernels/unified_attention.py rename zml/attention/{triton.zig => triton_attention.zig} (53%) create mode 100644 zml/attention/triton_kernels/unified_attention.zig create mode 100644 zml/kernel.zig diff --git a/MODULE.bazel b/MODULE.bazel index f615687b..a351a940 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -135,29 +135,6 @@ pip.parse( ) use_repo(pip, "xprof_nightly") -# For Triton generation -pip.parse( - download_only = True, - hub_name = "triton_py_deps", - python_version = "3.12", - requirements_lock = "//zml/attention/triton:requirements.lock.txt", - target_platforms = [ - "linux_x86_64", - "linux_arm64", - ], -) -use_repo(pip, "triton_py_deps") - -# For TPU IR generation -pip.parse( - download_only = False, - hub_name = "tpu_py_deps", - python_version = "3.12", - requirements_lock = "//platforms/tpu:requirements.lock.txt", - target_platforms = ["linux_x86_64"], -) -use_repo(pip, "tpu_py_deps") - cpu = use_extension("//platforms/cpu:cpu.bzl", "cpu_pjrt_plugin") use_repo(cpu, "libpjrt_cpu_darwin_amd64", "libpjrt_cpu_darwin_arm64", "libpjrt_cpu_linux_amd64") @@ -179,7 +156,7 @@ use_repo(zls, "zls_toolchains") register_toolchains("@zls_toolchains//:all") non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") -use_repo(non_module_deps, "arocc", "cloud_accelerator_diagnostics", "com_google_sentencepiece", "flashattn_linux_amd64", "flashattn_linux_arm64", "iree", "libvaxis", "linenoise", "mnist", "org_swig_swig", "stb", "translate-c", "uucode", "xla", "zigimg") +use_repo(non_module_deps, "arocc", "cloud_accelerator_diagnostics", "com_google_sentencepiece", "flashattn_linux_amd64", "flashattn_linux_arm64", "iree", "libvaxis", "linenoise", "mnist", "org_swig_swig", "stb", "translate-c", "uucode", "xla", "zigimg", "mosaic_tpu") xla = use_extension("//third_party/xla:xla.bzl", "xla") use_repo( @@ -203,6 +180,9 @@ use_repo( "sycl_configure_ext", "triton", "tsl", + "highwayhash", + "snappy", + "hwloc", ) llvm = use_extension("//third_party/xla:llvm.bzl", "llvm") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index ed055b4c..3f2e5294 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -22,7 +22,6 @@ "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", "https://bcr.bazel.build/modules/apple_support/1.17.1/MODULE.bazel": "655c922ab1209978a94ef6ca7d9d43e940cd97d9c172fb55f94d91ac53f8610b", "https://bcr.bazel.build/modules/apple_support/1.23.1/MODULE.bazel": "53763fed456a968cf919b3240427cf3a9d5481ec5466abc9d5dc51bc70087442", - "https://bcr.bazel.build/modules/apple_support/1.24.1/MODULE.bazel": "f46e8ddad60aef170ee92b2f3d00ef66c147ceafea68b6877cb45bd91737f5f8", "https://bcr.bazel.build/modules/apple_support/2.2.0/MODULE.bazel": "5e32bc42e413a4544df7afee7fa4c7aff73e670a44e56e8e45ce1954332034ad", "https://bcr.bazel.build/modules/apple_support/2.2.0/source.json": "368bf1de5dace2a411e792e6df2ea5384eb8db3899680ed51b37eb48b88fd8ac", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", @@ -33,7 +32,6 @@ "https://bcr.bazel.build/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "812d2dd42f65dca362152101fbec418029cc8fd34cbad1a2fde905383d705838", "https://bcr.bazel.build/modules/aspect_rules_py/1.8.4/MODULE.bazel": "4ab4a78e9d65cf15e755807f7ae504655e1771cefd7c3343ad34dbbb625b9564", "https://bcr.bazel.build/modules/aspect_rules_py/1.8.4/source.json": "7cd80ad811f0c2b3aad3c878506ae602fce2fe53b78487ecfbcc8bcde07d220e", - "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.2/MODULE.bazel": "598e7fe3b54f5fa64fdbeead1027653963a359cc23561d43680006f3b463d5a4", "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.3/MODULE.bazel": "37c764292861c2f70314efa9846bb6dbb44fc0308903b3285da6528305450183", "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.3/source.json": "605086bbc197743a0d360f7ddc550a1d4dfa0441bc807236e17170f636153348", "https://bcr.bazel.build/modules/bazel_features/1.1.0/MODULE.bazel": "cfd42ff3b815a5f39554d97182657f8c4b9719568eb7fded2b9135f084bf760b", @@ -51,7 +49,6 @@ "https://bcr.bazel.build/modules/bazel_features/1.28.0/MODULE.bazel": "4b4200e6cbf8fa335b2c3f43e1d6ef3e240319c33d43d60cc0fbd4b87ece299d", "https://bcr.bazel.build/modules/bazel_features/1.3.0/MODULE.bazel": "cdcafe83ec318cda34e02948e81d790aab8df7a929cec6f6969f13a489ccecd9", "https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87", - "https://bcr.bazel.build/modules/bazel_features/1.32.0/MODULE.bazel": "095d67022a58cb20f7e20e1aefecfa65257a222c18a938e2914fd257b5f1ccdc", "https://bcr.bazel.build/modules/bazel_features/1.33.0/MODULE.bazel": "8b8dc9d2a4c88609409c3191165bccec0e4cb044cd7a72ccbe826583303459f6", "https://bcr.bazel.build/modules/bazel_features/1.34.0/MODULE.bazel": "e8475ad7c8965542e0c7aac8af68eb48c4af904be3d614b6aa6274c092c2ea1e", "https://bcr.bazel.build/modules/bazel_features/1.38.0/MODULE.bazel": "f9b8a9c890ebd216b4049fd12a31d3c2602e3403c7af636b04fbbd7453edc9c9", @@ -153,7 +150,6 @@ "https://bcr.bazel.build/modules/libbacktrace/1.0.0-20250926-7939218/source.json": "83239989d09748f63dee4bdd430d261e217a012670e066e2238925a303a65d89", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", "https://bcr.bazel.build/modules/libpfm/4.11.0/source.json": "caaffb3ac2b59b8aac456917a4ecf3167d40478ee79f15ab7a877ec9273937c9", - "https://bcr.bazel.build/modules/llvm/0.6.7/MODULE.bazel": "d37a2e10571864dc6a5bb53c29216d90b9400bbcadb422337f49107fd2eaf0d2", "https://bcr.bazel.build/modules/llvm/0.7.3/MODULE.bazel": "b8ffe1ec9d18c0deb6d6dc1a6f78a64bcfcec8648c97b92ed6cd98a7fe49c64f", "https://bcr.bazel.build/modules/llvm/0.7.3/source.json": "44d694e1559683ea4cf96052c56c5426d2604016636133798fcd57cc78542984", "https://bcr.bazel.build/modules/mbedtls/3.6.0/MODULE.bazel": "8e380e4698107c5f8766264d4df92e36766248447858db28187151d884995a09", @@ -181,8 +177,7 @@ "https://bcr.bazel.build/modules/opentracing-cpp/1.6.0/MODULE.bazel": "b3925269f63561b8b880ae7cf62ccf81f6ece55b62cd791eda9925147ae116ec", "https://bcr.bazel.build/modules/opentracing-cpp/1.6.0/source.json": "da1cb1add160f5e5074b7272e9db6fd8f1b3336c15032cd0a653af9d2f484aed", "https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92", - "https://bcr.bazel.build/modules/package_metadata/0.0.5/MODULE.bazel": "ef4f9439e3270fdd6b9fd4dbc3d2f29d13888e44c529a1b243f7a31dfbc2e8e4", - "https://bcr.bazel.build/modules/package_metadata/0.0.5/source.json": "2326db2f6592578177751c3e1f74786b79382cd6008834c9d01ec865b9126a85", + "https://bcr.bazel.build/modules/package_metadata/0.0.2/source.json": "e53a759a72488d2c0576f57491ef2da0cf4aab05ac0997314012495935531b73", "https://bcr.bazel.build/modules/patchelf/0.18.0.bcr.1/MODULE.bazel": "35320668481400503dea7c75313473b9ffd6dea4e7ba583b96a896199b0088d5", "https://bcr.bazel.build/modules/patchelf/0.18.0.bcr.1/source.json": "d877d271ef5d531407885322054c99c7f0617a9a85ff99c0a70f4ced7f13e13a", "https://bcr.bazel.build/modules/pcre2/10.45/MODULE.bazel": "9bdc838aa2e21be030d7181385e45a5c87b14575bb7ff35d10b5be3d56913bed", @@ -330,8 +325,6 @@ "https://bcr.bazel.build/modules/rules_python/1.7.0/MODULE.bazel": "d01f995ecd137abf30238ad9ce97f8fc3ac57289c8b24bd0bf53324d937a14f8", "https://bcr.bazel.build/modules/rules_python/1.9.0/MODULE.bazel": "afc3a05f29f09f2d3ee95ad99a145250dab41a2b2d8d6f82cc91936b3213282c", "https://bcr.bazel.build/modules/rules_python/1.9.0/source.json": "3921ea0b65298d51aead5b9e4a82203d7be9b5918619b58b53f1c259f4e63169", - "https://bcr.bazel.build/modules/rules_rs/0.0.44/MODULE.bazel": "aa151d9a96e5d9363732df8943c7fb5327738cc160732fa74c02e5de7822007b", - "https://bcr.bazel.build/modules/rules_rs/0.0.44/source.json": "1358c9418e1325fdb5d0caf53ef32f98d525fe2030668279cae2e97c34cd29eb", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", @@ -451,7 +444,7 @@ "@@aspect_tools_telemetry+//:extension.bzl%telemetry": { "general": { "bzlTransitiveDigest": "MePriaXmQNSqUfE+YQvEtzBc2bU1mjITINIffFiZYgo=", - "usagesDigest": "TU1quwRKpvAEOEF35/IVh4NeoIS2hayv4fGp8f5RXoY=", + "usagesDigest": "pyph4lSDn5CHIn4s2mtKJTkRaLV/o2zIFgnrpufr/8o=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -461,7 +454,6 @@ "attributes": { "deps": { "aspect_rules_py": "1.8.4", - "rules_rs": "0.0.44", "aspect_tools_telemetry": "0.3.3" } } @@ -1731,325 +1723,5 @@ } } }, - "facts": { - "@@rules_rs+//rs:extensions.bzl%crate": { - "addr2line_0.25.1": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"backtrace\",\"req\":\"^0.3.13\"},{\"features\":[\"wrap_help\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4.3.21\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"cpp_demangle\",\"optional\":true,\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.7.0\"},{\"default_features\":false,\"name\":\"fallible-iterator\",\"optional\":true,\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"findshlibs\",\"req\":\"^0.10\"},{\"default_features\":false,\"features\":[\"read\"],\"name\":\"gimli\",\"req\":\"^0.32.0\"},{\"kind\":\"dev\",\"name\":\"libtest-mimic\",\"req\":\"^0.8.1\"},{\"name\":\"memmap2\",\"optional\":true,\"req\":\"^0.9.4\"},{\"default_features\":false,\"features\":[\"read\",\"compression\"],\"name\":\"object\",\"optional\":true,\"req\":\"^0.37.0\"},{\"name\":\"rustc-demangle\",\"optional\":true,\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"typed-arena\",\"optional\":true,\"req\":\"^2\"}],\"features\":{\"all\":[\"bin\",\"wasm\"],\"bin\":[\"loader\",\"rustc-demangle\",\"cpp_demangle\",\"fallible-iterator\",\"smallvec\",\"dep:clap\"],\"cargo-all\":[],\"default\":[\"rustc-demangle\",\"cpp_demangle\",\"loader\",\"fallible-iterator\",\"smallvec\"],\"loader\":[\"std\",\"dep:object\",\"dep:memmap2\",\"dep:typed-arena\"],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"gimli/rustc-dep-of-std\"],\"std\":[\"gimli/std\"],\"wasm\":[\"object/wasm\"]}}", - "adler2_2.0.1": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"std\"],\"rustc-dep-of-std\":[\"core\"],\"std\":[]}}", - "ahash_0.8.12": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1.0\"},{\"name\":\"const-random\",\"optional\":true,\"req\":\"^0.1.17\"},{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.2\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.5\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2.1\"},{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"hashbrown\",\"req\":\"^0.14.3\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.2\"},{\"kind\":\"dev\",\"name\":\"no-panic\",\"req\":\"^0.1.10\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"once_cell\",\"req\":\"^1.18.0\",\"target\":\"cfg(not(all(target_arch = \\\"arm\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"pcg-mwc\",\"req\":\"^0.2.1\"},{\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"seahash\",\"req\":\"^4.0\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.117\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.59\"},{\"kind\":\"dev\",\"name\":\"smallvec\",\"req\":\"^1.13.1\"},{\"kind\":\"build\",\"name\":\"version_check\",\"req\":\"^0.9.4\"},{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"zerocopy\",\"req\":\"^0.8.24\"}],\"features\":{\"atomic-polyfill\":[\"dep:portable-atomic\",\"once_cell/critical-section\"],\"compile-time-rng\":[\"const-random\"],\"default\":[\"std\",\"runtime-rng\"],\"nightly-arm-aes\":[],\"no-rng\":[],\"runtime-rng\":[\"getrandom\"],\"std\":[]}}", - "aho-corasick_1.1.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.17\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.4.0\"}],\"features\":{\"default\":[\"std\",\"perf-literal\"],\"logging\":[\"dep:log\"],\"perf-literal\":[\"dep:memchr\"],\"std\":[\"memchr?/std\"]}}", - "aho-corasick_1.1.4": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.17\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.4.0\"}],\"features\":{\"default\":[\"std\",\"perf-literal\"],\"logging\":[\"dep:log\"],\"perf-literal\":[\"dep:memchr\"],\"std\":[\"memchr?/std\"]}}", - "anyhow_1.0.102": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.6\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0\"},{\"kind\":\"dev\",\"name\":\"thiserror\",\"req\":\"^2\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"}],\"features\":{\"backtrace\":[],\"default\":[\"std\"],\"std\":[]}}", - "async-stream-impl_0.3.6": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"full\",\"visit-mut\"],\"name\":\"syn\",\"req\":\"^2.0.2\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"}],\"features\":{}}", - "async-stream_0.3.6": "{\"dependencies\":[{\"name\":\"async-stream-impl\",\"req\":\"=0.3.6\"},{\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1\"}],\"features\":{}}", - "async-trait_0.1.89": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.30\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"full\",\"parsing\",\"printing\",\"proc-macro\",\"visit-mut\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1.40\"},{\"kind\":\"dev\",\"name\":\"tracing-attributes\",\"req\":\"^0.1.27\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{}}", - "atomic-waker_1.1.2": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"cargo_bench_support\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.5\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.7.0\"}],\"features\":{}}", - "autocfg_1.5.0": "{\"dependencies\":[],\"features\":{}}", - "axum-core_0.4.5": "{\"dependencies\":[{\"name\":\"async-trait\",\"req\":\"^0.1.67\"},{\"kind\":\"dev\",\"name\":\"axum\",\"req\":\"^0.7.2\"},{\"name\":\"bytes\",\"req\":\"^1.2\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"http\",\"req\":\"^1.0.0\"},{\"name\":\"http-body\",\"req\":\"^1.0.0\"},{\"name\":\"http-body-util\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^1.0.0\"},{\"name\":\"mime\",\"req\":\"^0.3.16\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"},{\"name\":\"rustversion\",\"req\":\"^1.0.9\"},{\"name\":\"sync_wrapper\",\"req\":\"^1.0.0\"},{\"features\":[\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.25.0\"},{\"features\":[\"limit\"],\"name\":\"tower-http\",\"optional\":true,\"req\":\"^0.6.0\"},{\"features\":[\"limit\"],\"kind\":\"dev\",\"name\":\"tower-http\",\"req\":\"^0.6.0\"},{\"name\":\"tower-layer\",\"req\":\"^0.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.37\"}],\"features\":{\"__private_docs\":[\"dep:tower-http\"],\"tracing\":[\"dep:tracing\"]}}", - "axum_0.7.9": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0\"},{\"name\":\"async-trait\",\"req\":\"^0.1.67\"},{\"name\":\"axum-core\",\"req\":\"^0.4.5\"},{\"name\":\"axum-macros\",\"optional\":true,\"req\":\"^0.4.2\"},{\"features\":[\"__private\"],\"kind\":\"dev\",\"name\":\"axum-macros\",\"req\":\"^0.4.1\"},{\"name\":\"base64\",\"optional\":true,\"req\":\"^0.22.1\"},{\"name\":\"bytes\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"http\",\"req\":\"^1.0.0\"},{\"name\":\"http-body\",\"req\":\"^1.0.0\"},{\"name\":\"http-body-util\",\"req\":\"^0.1.0\"},{\"name\":\"hyper\",\"optional\":true,\"req\":\"^1.1.0\"},{\"features\":[\"tokio\",\"server\",\"service\"],\"name\":\"hyper-util\",\"optional\":true,\"req\":\"^0.1.3\"},{\"name\":\"itoa\",\"req\":\"^1.0.5\"},{\"name\":\"matchit\",\"req\":\"^0.7\"},{\"name\":\"memchr\",\"req\":\"^2.4.1\"},{\"name\":\"mime\",\"req\":\"^0.3.16\"},{\"name\":\"multer\",\"optional\":true,\"req\":\"^3.0.0\"},{\"name\":\"percent-encoding\",\"req\":\"^2.1\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"json\",\"stream\",\"multipart\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.12\"},{\"name\":\"rustversion\",\"req\":\"^1.0.9\"},{\"name\":\"serde\",\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"features\":[\"raw_value\"],\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"raw_value\"],\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"serde_path_to_error\",\"optional\":true,\"req\":\"^0.1.8\"},{\"name\":\"serde_urlencoded\",\"optional\":true,\"req\":\"^0.7\"},{\"name\":\"sha1\",\"optional\":true,\"req\":\"^0.10\"},{\"name\":\"sync_wrapper\",\"req\":\"^1.0.0\"},{\"features\":[\"serde-human-readable\"],\"kind\":\"dev\",\"name\":\"time\",\"req\":\"^0.3\"},{\"features\":[\"time\"],\"name\":\"tokio\",\"optional\":true,\"package\":\"tokio\",\"req\":\"^1.25.0\"},{\"features\":[\"macros\",\"rt\",\"rt-multi-thread\",\"net\",\"test-util\"],\"kind\":\"dev\",\"name\":\"tokio\",\"package\":\"tokio\",\"req\":\"^1.25.0\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"name\":\"tokio-tungstenite\",\"optional\":true,\"req\":\"^0.24.0\"},{\"kind\":\"dev\",\"name\":\"tokio-tungstenite\",\"req\":\"^0.24.0\"},{\"default_features\":false,\"features\":[\"util\"],\"name\":\"tower\",\"req\":\"^0.5.1\"},{\"features\":[\"util\",\"timeout\",\"limit\",\"load-shed\",\"steer\",\"filter\"],\"kind\":\"dev\",\"name\":\"tower\",\"package\":\"tower\",\"req\":\"^0.5.1\"},{\"features\":[\"add-extension\",\"auth\",\"catch-panic\",\"compression-br\",\"compression-deflate\",\"compression-gzip\",\"cors\",\"decompression-br\",\"decompression-deflate\",\"decompression-gzip\",\"follow-redirect\",\"fs\",\"limit\",\"map-request-body\",\"map-response-body\",\"metrics\",\"normalize-path\",\"propagate-header\",\"redirect\",\"request-id\",\"sensitive-headers\",\"set-header\",\"set-status\",\"timeout\",\"trace\",\"util\",\"validate-request\"],\"name\":\"tower-http\",\"optional\":true,\"req\":\"^0.6.0\"},{\"features\":[\"add-extension\",\"auth\",\"catch-panic\",\"compression-br\",\"compression-deflate\",\"compression-gzip\",\"cors\",\"decompression-br\",\"decompression-deflate\",\"decompression-gzip\",\"follow-redirect\",\"fs\",\"limit\",\"map-request-body\",\"map-response-body\",\"metrics\",\"normalize-path\",\"propagate-header\",\"redirect\",\"request-id\",\"sensitive-headers\",\"set-header\",\"set-status\",\"timeout\",\"trace\",\"util\",\"validate-request\"],\"kind\":\"dev\",\"name\":\"tower-http\",\"req\":\"^0.6.0\"},{\"name\":\"tower-layer\",\"req\":\"^0.3.2\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1\"},{\"features\":[\"json\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"},{\"features\":[\"serde\",\"v4\"],\"kind\":\"dev\",\"name\":\"uuid\",\"req\":\"^1.0\"}],\"features\":{\"__private_docs\":[\"axum-core/__private_docs\",\"tower/full\",\"dep:tower-http\"],\"default\":[\"form\",\"http1\",\"json\",\"matched-path\",\"original-uri\",\"query\",\"tokio\",\"tower-log\",\"tracing\"],\"form\":[\"dep:serde_urlencoded\"],\"http1\":[\"dep:hyper\",\"hyper?/http1\",\"hyper-util?/http1\"],\"http2\":[\"dep:hyper\",\"hyper?/http2\",\"hyper-util?/http2\"],\"json\":[\"dep:serde_json\",\"dep:serde_path_to_error\"],\"macros\":[\"dep:axum-macros\"],\"matched-path\":[],\"multipart\":[\"dep:multer\"],\"original-uri\":[],\"query\":[\"dep:serde_urlencoded\"],\"tokio\":[\"dep:hyper-util\",\"dep:tokio\",\"tokio/net\",\"tokio/rt\",\"tower/make\",\"tokio/macros\"],\"tower-log\":[\"tower/log\"],\"tracing\":[\"dep:tracing\",\"axum-core/tracing\"],\"ws\":[\"dep:hyper\",\"tokio\",\"dep:tokio-tungstenite\",\"dep:sha1\",\"dep:base64\"]}}", - "backtrace_0.3.76": "{\"dependencies\":[{\"default_features\":false,\"name\":\"addr2line\",\"req\":\"^0.25.0\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"name\":\"cfg-if\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"cpp_demangle\",\"optional\":true,\"req\":\"^0.5.0\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.156\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"kind\":\"dev\",\"name\":\"libloading\",\"req\":\"^0.8\"},{\"default_features\":false,\"name\":\"miniz_oxide\",\"req\":\"^0.8\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"default_features\":false,\"features\":[\"read_core\",\"elf\",\"macho\",\"pe\",\"xcoff\",\"unaligned\",\"archive\"],\"name\":\"object\",\"req\":\"^0.37.0\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"name\":\"rustc-demangle\",\"req\":\"^0.1.24\"},{\"default_features\":false,\"name\":\"ruzstd\",\"optional\":true,\"req\":\"^0.8.1\",\"target\":\"cfg(not(all(windows, target_env = \\\"msvc\\\", not(target_vendor = \\\"uwp\\\"))))\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"windows-link\",\"req\":\"^0.2\",\"target\":\"cfg(any(windows, target_os = \\\"cygwin\\\"))\"}],\"features\":{\"coresymbolication\":[],\"dbghelp\":[],\"default\":[\"std\"],\"dl_iterate_phdr\":[],\"dladdr\":[],\"kernel32\":[],\"libunwind\":[],\"ruzstd\":[\"dep:ruzstd\"],\"serialize-serde\":[\"serde\"],\"std\":[],\"unix-backtrace\":[]}}", - "base64_0.13.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"=0.3.2\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.6.1\"},{\"kind\":\"dev\",\"name\":\"structopt\",\"req\":\"^0.3\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[]}}", - "base64_0.22.1": "{\"dependencies\":[{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"clap\",\"req\":\"^3.2.25\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rstest\",\"req\":\"^0.13.0\"},{\"kind\":\"dev\",\"name\":\"rstest_reuse\",\"req\":\"^0.6.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"strum\",\"req\":\"^0.25\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", - "bitflags_2.11.0": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"arbitrary\",\"req\":\"^1.0\"},{\"name\":\"bytemuck\",\"optional\":true,\"req\":\"^1.12\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"bytemuck\",\"req\":\"^1.12.2\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.228\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde_lib\",\"package\":\"serde\",\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.19\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.18\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"zerocopy\",\"req\":\"^0.8\"}],\"features\":{\"example_generated\":[],\"serde\":[\"serde_core\"],\"std\":[]}}", - "bitflags_2.9.1": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"arbitrary\",\"req\":\"^1.0\"},{\"name\":\"bytemuck\",\"optional\":true,\"req\":\"^1.12\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"bytemuck\",\"req\":\"^1.12.2\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.19\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.18\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"zerocopy\",\"req\":\"^0.8\"}],\"features\":{\"example_generated\":[],\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"],\"std\":[]}}", - "byteorder_1.5.0": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9.2\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"std\"],\"i128\":[],\"std\":[]}}", - "bytes_1.11.1": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"require-cas\"],\"name\":\"extra-platforms\",\"optional\":true,\"package\":\"portable-atomic\",\"req\":\"^1.3\"},{\"kind\":\"dev\",\"name\":\"loom\",\"req\":\"^0.7\",\"target\":\"cfg(loom)\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.60\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "castaway_0.2.4": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1\"},{\"name\":\"rustversion\",\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", - "cc_1.2.33": "{\"dependencies\":[{\"default_features\":false,\"name\":\"jobserver\",\"optional\":true,\"req\":\"^0.1.30\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.62\",\"target\":\"cfg(unix)\"},{\"name\":\"shlex\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{\"jobserver\":[],\"parallel\":[\"dep:libc\",\"dep:jobserver\"]}}", - "cfg-if_1.0.0": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"]}}", - "cfg-if_1.0.4": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"rustc-dep-of-std\":[\"core\"]}}", - "compact_str_0.9.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"borsh\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"castaway\",\"req\":\"^0.2.3\"},{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"cfg-if\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"diesel\",\"optional\":true,\"req\":\"^2\"},{\"name\":\"itoa\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"markup\",\"optional\":true,\"req\":\"^0.15\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"proptest\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"rkyv\",\"optional\":true,\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rkyv\",\"req\":\"^0.8.8\"},{\"name\":\"rustversion\",\"req\":\"^1\"},{\"name\":\"ryu\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"derive\",\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"features\":[\"union\"],\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"name\":\"sqlx\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"static_assertions\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"test-case\",\"req\":\"^3\"},{\"kind\":\"dev\",\"name\":\"test-strategy\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"borsh\":[\"dep:borsh\"],\"bytes\":[\"dep:bytes\"],\"default\":[\"std\"],\"diesel\":[\"dep:diesel\"],\"markup\":[\"dep:markup\"],\"proptest\":[\"dep:proptest\"],\"quickcheck\":[\"dep:quickcheck\"],\"rkyv\":[\"dep:rkyv\"],\"serde\":[\"dep:serde\"],\"smallvec\":[\"dep:smallvec\"],\"sqlx\":[\"dep:sqlx\",\"std\"],\"sqlx-mysql\":[\"sqlx\",\"sqlx/mysql\"],\"sqlx-postgres\":[\"sqlx\",\"sqlx/postgres\"],\"sqlx-sqlite\":[\"sqlx\",\"sqlx/sqlite\"],\"std\":[],\"zeroize\":[\"dep:zeroize\"]}}", - "crossbeam-deque_0.8.6": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-epoch\",\"req\":\"^0.9.17\"},{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"crossbeam-epoch/std\",\"crossbeam-utils/std\"]}}", - "crossbeam-epoch_0.9.18": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"name\":\"loom-crate\",\"optional\":true,\"package\":\"loom\",\"req\":\"^0.7.1\",\"target\":\"cfg(crossbeam_loom)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"loom\":[\"loom-crate\",\"crossbeam-utils/loom\"],\"nightly\":[\"crossbeam-utils/nightly\"],\"std\":[\"alloc\",\"crossbeam-utils/std\"]}}", - "crossbeam-utils_0.8.21": "{\"dependencies\":[{\"name\":\"loom\",\"optional\":true,\"req\":\"^0.7.1\",\"target\":\"cfg(crossbeam_loom)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"std\"],\"nightly\":[],\"std\":[]}}", - "darling_0.20.10": "{\"dependencies\":[{\"name\":\"darling_core\",\"req\":\"=0.20.10\"},{\"name\":\"darling_macro\",\"req\":\"=0.20.10\"},{\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.37\"},{\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.9\",\"target\":\"cfg(compiletests)\"},{\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.15\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.38\",\"target\":\"cfg(compiletests)\"}],\"features\":{\"default\":[\"suggestions\"],\"diagnostics\":[\"darling_core/diagnostics\"],\"suggestions\":[\"darling_core/suggestions\"]}}", - "darling_core_0.20.10": "{\"dependencies\":[{\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"name\":\"ident_case\",\"req\":\"^1.0.1\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.37\"},{\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"name\":\"strsim\",\"optional\":true,\"req\":\"^0.11.1\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"diagnostics\":[],\"suggestions\":[\"strsim\"]}}", - "darling_macro_0.20.10": "{\"dependencies\":[{\"name\":\"darling_core\",\"req\":\"=0.20.10\"},{\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{}}", - "dary_heap_0.3.7": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1\"}],\"features\":{\"extra\":[],\"unstable\":[],\"unstable_nightly\":[]}}", - "derive_builder_0.20.2": "{\"dependencies\":[{\"name\":\"derive_builder_macro\",\"req\":\"=0.20.2\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.4\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.38\",\"target\":\"cfg(compiletests)\"}],\"features\":{\"alloc\":[\"derive_builder_macro/alloc\"],\"clippy\":[\"derive_builder_macro/clippy\"],\"default\":[\"std\"],\"std\":[\"derive_builder_macro/lib_has_std\"]}}", - "derive_builder_core_0.20.2": "{\"dependencies\":[{\"name\":\"darling\",\"req\":\"^0.20.10\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.37\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"alloc\":[],\"clippy\":[],\"lib_has_std\":[]}}", - "derive_builder_macro_0.20.2": "{\"dependencies\":[{\"name\":\"derive_builder_core\",\"req\":\"=0.20.2\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"alloc\":[\"derive_builder_core/alloc\"],\"clippy\":[\"derive_builder_core/clippy\"],\"lib_has_std\":[\"derive_builder_core/lib_has_std\"]}}", - "either_1.13.0": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"use_std\"],\"use_std\":[]}}", - "either_1.15.0": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"alloc\",\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.95\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[],\"use_std\":[\"std\"]}}", - "equivalent_1.0.2": "{\"dependencies\":[],\"features\":{}}", - "errno_0.3.14": "{\"dependencies\":[{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(target_os=\\\"hermit\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(target_os=\\\"wasi\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(unix)\"},{\"features\":[\"Win32_Foundation\",\"Win32_System_Diagnostics_Debug\"],\"name\":\"windows-sys\",\"req\":\">=0.52, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"libc/std\"]}}", - "esaxx-rs_0.1.10": "{\"dependencies\":[{\"kind\":\"build\",\"name\":\"cc\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"}],\"features\":{\"cpp\":[\"cc\"],\"default\":[\"cpp\"]}}", - "fastrand_2.3.0": "{\"dependencies\":[{\"features\":[\"js\"],\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.2\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"getrandom\",\"req\":\"^0.2\"},{\"features\":[\"js\"],\"kind\":\"dev\",\"name\":\"getrandom\",\"req\":\"^0.2\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"wyhash\",\"req\":\"^0.5\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"js\":[\"std\",\"getrandom\"],\"std\":[\"alloc\"]}}", - "fixedbitset_0.4.2": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "fnv_1.0.7": "{\"dependencies\":[],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "foldhash_0.1.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"ahash\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"chrono\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"hashbrown\",\"req\":\"^0.14\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"uuid\",\"req\":\"^1.8\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "futures-channel_0.3.32": "{\"dependencies\":[{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-sink\",\"optional\":true,\"req\":\"^0.3.32\"}],\"features\":{\"alloc\":[\"futures-core/alloc\"],\"cfg-target-has-atomic\":[],\"default\":[\"std\"],\"sink\":[\"futures-sink\"],\"std\":[\"alloc\",\"futures-core/std\"],\"unstable\":[]}}", - "futures-core_0.3.32": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"require-cas\"],\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.3\"}],\"features\":{\"alloc\":[],\"cfg-target-has-atomic\":[],\"default\":[\"std\"],\"std\":[\"alloc\"],\"unstable\":[]}}", - "futures-sink_0.3.32": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", - "futures-task_0.3.32": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"cfg-target-has-atomic\":[],\"default\":[\"std\"],\"std\":[\"alloc\"],\"unstable\":[]}}", - "futures-util_0.3.32": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"std\"],\"name\":\"futures-channel\",\"optional\":true,\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3.32\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"futures-io\",\"optional\":true,\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-macro\",\"optional\":true,\"req\":\"=0.3.32\"},{\"default_features\":false,\"name\":\"futures-sink\",\"optional\":true,\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-task\",\"req\":\"^0.3.32\"},{\"name\":\"futures_01\",\"optional\":true,\"package\":\"futures\",\"req\":\"^0.1.25\"},{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.26\"},{\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.2\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.6\"},{\"default_features\":false,\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4.7\"},{\"name\":\"spin\",\"optional\":true,\"req\":\"^0.10.0\"},{\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^0.1.11\"},{\"name\":\"tokio-io\",\"optional\":true,\"req\":\"^0.1.9\"}],\"features\":{\"alloc\":[\"futures-core/alloc\",\"futures-task/alloc\",\"slab\"],\"async-await\":[],\"async-await-macro\":[\"async-await\",\"futures-macro\"],\"bilock\":[],\"cfg-target-has-atomic\":[],\"channel\":[\"std\",\"futures-channel\"],\"compat\":[\"std\",\"futures_01\",\"libc\"],\"default\":[\"std\",\"async-await\",\"async-await-macro\"],\"io\":[\"std\",\"futures-io\",\"memchr\"],\"io-compat\":[\"io\",\"compat\",\"tokio-io\",\"libc\"],\"portable-atomic\":[\"futures-core/portable-atomic\"],\"sink\":[\"futures-sink\"],\"std\":[\"alloc\",\"futures-core/std\",\"futures-task/std\",\"slab/std\"],\"unstable\":[\"futures-core/unstable\",\"futures-task/unstable\"],\"write-all-vectored\":[\"io\"]}}", - "getrandom_0.2.17": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(unix)\"},{\"default_features\":false,\"name\":\"wasi\",\"req\":\"^0.11\",\"target\":\"cfg(target_os = \\\"wasi\\\")\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.62\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.18\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"}],\"features\":{\"custom\":[],\"js\":[\"wasm-bindgen\",\"js-sys\"],\"linux_disable_fallback\":[],\"rdrand\":[],\"rustc-dep-of-std\":[\"compiler_builtins\",\"core\",\"libc/rustc-dep-of-std\",\"wasi/rustc-dep-of-std\"],\"std\":[],\"test-in-browser\":[]}}", - "getrandom_0.3.3": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3.77\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"), target_feature = \\\"atomics\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"netbsd\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"solaris\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"vxworks\\\")\"},{\"default_features\":false,\"name\":\"r-efi\",\"req\":\"^5.1\",\"target\":\"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\"},{\"default_features\":false,\"name\":\"wasi\",\"req\":\"^0.14\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.98\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"}],\"features\":{\"rustc-dep-of-std\":[\"dep:compiler_builtins\",\"dep:core\"],\"std\":[],\"wasm_js\":[\"dep:wasm-bindgen\",\"dep:js-sys\"]}}", - "getrandom_0.4.2": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3.77\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"), target_feature = \\\"atomics\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"netbsd\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"solaris\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"vxworks\\\")\"},{\"default_features\":false,\"name\":\"r-efi\",\"req\":\"^6\",\"target\":\"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\"},{\"name\":\"rand_core\",\"optional\":true,\"req\":\"^0.10.0\"},{\"default_features\":false,\"name\":\"wasip2\",\"req\":\"^1\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\"},{\"name\":\"wasip3\",\"req\":\"^0.4\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p3\\\"))\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.98\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"}],\"features\":{\"std\":[],\"sys_rng\":[\"dep:rand_core\"],\"wasm_js\":[\"dep:wasm-bindgen\",\"dep:js-sys\"]}}", - "gimli_0.32.3": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"fallible-iterator\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.0.0\"},{\"default_features\":false,\"name\":\"stable_deref_trait\",\"optional\":true,\"req\":\"^1.1.0\"},{\"kind\":\"dev\",\"name\":\"test-assembler\",\"req\":\"^0.1.3\"}],\"features\":{\"default\":[\"read-all\",\"write\"],\"endian-reader\":[\"read\",\"dep:stable_deref_trait\"],\"fallible-iterator\":[\"dep:fallible-iterator\"],\"read\":[\"read-core\"],\"read-all\":[\"read\",\"std\",\"fallible-iterator\",\"endian-reader\"],\"read-core\":[],\"rustc-dep-of-std\":[\"dep:core\",\"dep:alloc\"],\"std\":[\"fallible-iterator?/std\",\"stable_deref_trait?/std\"],\"write\":[\"dep:indexmap\"]}}", - "h2_0.4.6": "{\"dependencies\":[{\"name\":\"atomic-waker\",\"req\":\"^1.0.0\"},{\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.10\"},{\"name\":\"fnv\",\"req\":\"^1.0.5\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-sink\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.3\"},{\"name\":\"http\",\"req\":\"^1\"},{\"features\":[\"std\"],\"name\":\"indexmap\",\"req\":\"^2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.4\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.0\"},{\"name\":\"slab\",\"req\":\"^0.4.2\"},{\"features\":[\"io-util\"],\"name\":\"tokio\",\"req\":\"^1\"},{\"features\":[\"rt-multi-thread\",\"macros\",\"sync\",\"net\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tokio-rustls\",\"req\":\"^0.26\"},{\"features\":[\"codec\",\"io\"],\"name\":\"tokio-util\",\"req\":\"^0.7.1\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"req\":\"^0.1.35\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\"},{\"kind\":\"dev\",\"name\":\"webpki-roots\",\"req\":\"^0.26\"}],\"features\":{\"stream\":[],\"unstable\":[]}}", - "hashbrown_0.12.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"ahash\",\"optional\":true,\"req\":\"^0.7.0\"},{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"bumpalo\",\"optional\":true,\"req\":\"^3.5.0\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.3\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.25\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"ahash-compile-time-rng\":[\"ahash/compile-time-rng\"],\"default\":[\"ahash\",\"inline-more\"],\"inline-more\":[],\"nightly\":[],\"raw\":[],\"rustc-dep-of-std\":[\"nightly\",\"core\",\"compiler_builtins\",\"alloc\",\"rustc-internal-api\"],\"rustc-internal-api\":[]}}", - "hashbrown_0.15.5": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"allocator-api2\",\"optional\":true,\"req\":\"^0.2.9\"},{\"features\":[\"allocator-api2\"],\"kind\":\"dev\",\"name\":\"bumpalo\",\"req\":\"^3.13.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.1\"},{\"default_features\":false,\"name\":\"equivalent\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"default_features\":false,\"name\":\"foldhash\",\"optional\":true,\"req\":\"^0.1.2\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9.0\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.2\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.2\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.25\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"default-hasher\",\"inline-more\",\"allocator-api2\",\"equivalent\",\"raw-entry\"],\"default-hasher\":[\"dep:foldhash\"],\"inline-more\":[],\"nightly\":[\"bumpalo/allocator_api\"],\"raw-entry\":[],\"rustc-dep-of-std\":[\"nightly\",\"core\",\"alloc\",\"rustc-internal-api\"],\"rustc-internal-api\":[]}}", - "hashbrown_0.16.1": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"allocator-api2\",\"optional\":true,\"req\":\"^0.2.9\"},{\"features\":[\"allocator-api2\"],\"kind\":\"dev\",\"name\":\"bumpalo\",\"req\":\"^3.13.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"equivalent\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"default_features\":false,\"name\":\"foldhash\",\"optional\":true,\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.155\",\"target\":\"cfg(unix)\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9.0\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.9.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.2\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.221\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"default-hasher\",\"inline-more\",\"allocator-api2\",\"equivalent\",\"raw-entry\"],\"default-hasher\":[\"dep:foldhash\"],\"inline-more\":[],\"nightly\":[\"foldhash?/nightly\",\"bumpalo/allocator_api\"],\"raw-entry\":[],\"rustc-dep-of-std\":[\"nightly\",\"core\",\"alloc\",\"rustc-internal-api\"],\"rustc-internal-api\":[],\"serde\":[\"dep:serde_core\",\"dep:serde\"]}}", - "heck_0.5.0": "{\"dependencies\":[],\"features\":{}}", - "http-body-util_0.1.3": "{\"dependencies\":[{\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"http-body\",\"req\":\"^1\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"macros\",\"rt\",\"sync\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"}],\"features\":{\"channel\":[\"dep:tokio\"],\"default\":[],\"full\":[\"channel\"]}}", - "http-body_1.0.1": "{\"dependencies\":[{\"name\":\"bytes\",\"req\":\"^1\"},{\"name\":\"http\",\"req\":\"^1\"}],\"features\":{}}", - "http_1.4.0": "{\"dependencies\":[{\"name\":\"bytes\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"name\":\"itoa\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.0\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "httparse_1.10.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.5\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "httpdate_1.0.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"}],\"features\":{}}", - "hyper-timeout_0.5.2": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"name\":\"hyper\",\"req\":\"^1.1\"},{\"features\":[\"http1\"],\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^1.1\"},{\"kind\":\"dev\",\"name\":\"hyper-tls\",\"req\":\"^0.6\"},{\"features\":[\"client-legacy\",\"http1\"],\"name\":\"hyper-util\",\"req\":\"^0.1.10\"},{\"features\":[\"client-legacy\",\"http1\",\"server\",\"server-graceful\"],\"kind\":\"dev\",\"name\":\"hyper-util\",\"req\":\"^0.1.10\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2\"},{\"name\":\"tokio\",\"req\":\"^1.35\"},{\"features\":[\"io-std\",\"io-util\",\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.35\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"}],\"features\":{}}", - "hyper-util_0.1.20": "{\"dependencies\":[{\"name\":\"base64\",\"optional\":true,\"req\":\"^0.22\"},{\"name\":\"bytes\",\"req\":\"^1.7.1\"},{\"kind\":\"dev\",\"name\":\"bytes\",\"req\":\"^1\"},{\"name\":\"futures-channel\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.16\"},{\"default_features\":false,\"features\":[\"alloc\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3.16\"},{\"name\":\"http\",\"req\":\"^1.0\"},{\"name\":\"http-body\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1.0\"},{\"name\":\"hyper\",\"req\":\"^1.8.0\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^1.4.0\"},{\"name\":\"ipnet\",\"optional\":true,\"req\":\"^2.9\"},{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2\"},{\"name\":\"percent-encoding\",\"optional\":true,\"req\":\"^2.3\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.4\"},{\"kind\":\"dev\",\"name\":\"pnet_datalink\",\"req\":\"^0.35.0\",\"target\":\"cfg(any(target_os = \\\"linux\\\", target_os = \\\"macos\\\"))\"},{\"kind\":\"dev\",\"name\":\"pretty_env_logger\",\"req\":\"^0.5\"},{\"features\":[\"all\"],\"name\":\"socket2\",\"optional\":true,\"req\":\">=0.5.9, <0.7\"},{\"name\":\"system-configuration\",\"optional\":true,\"req\":\"^0.7\",\"target\":\"cfg(target_os = \\\"macos\\\")\"},{\"default_features\":false,\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"macros\",\"test-util\",\"signal\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"name\":\"tower-layer\",\"optional\":true,\"req\":\"^0.3\"},{\"name\":\"tower-service\",\"optional\":true,\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"tower-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"windows-registry\",\"optional\":true,\"req\":\">=0.3, <0.7\",\"target\":\"cfg(windows)\"}],\"features\":{\"__internal_happy_eyeballs_tests\":[],\"client\":[\"hyper/client\",\"tokio/net\",\"dep:tracing\",\"dep:futures-channel\",\"dep:tower-service\"],\"client-legacy\":[\"client\",\"dep:socket2\",\"tokio/sync\",\"dep:libc\",\"dep:futures-util\"],\"client-pool\":[\"client\",\"dep:futures-util\",\"dep:tower-layer\"],\"client-proxy\":[\"client\",\"dep:base64\",\"dep:ipnet\",\"dep:percent-encoding\"],\"client-proxy-system\":[\"dep:system-configuration\",\"dep:windows-registry\"],\"default\":[],\"full\":[\"client\",\"client-legacy\",\"client-pool\",\"client-proxy\",\"client-proxy-system\",\"server\",\"server-auto\",\"server-graceful\",\"service\",\"http1\",\"http2\",\"tokio\",\"tracing\"],\"http1\":[\"hyper/http1\"],\"http2\":[\"hyper/http2\"],\"server\":[\"hyper/server\"],\"server-auto\":[\"server\",\"http1\",\"http2\"],\"server-graceful\":[\"server\",\"tokio/sync\"],\"service\":[\"dep:tower-service\"],\"tokio\":[\"dep:tokio\",\"tokio/rt\",\"tokio/time\"],\"tracing\":[\"dep:tracing\"]}}", - "hyper_1.8.1": "{\"dependencies\":[{\"name\":\"atomic-waker\",\"optional\":true,\"req\":\"^1.1.2\"},{\"name\":\"bytes\",\"req\":\"^1.2\"},{\"kind\":\"dev\",\"name\":\"form_urlencoded\",\"req\":\"^1\"},{\"name\":\"futures-channel\",\"optional\":true,\"req\":\"^0.3\"},{\"features\":[\"sink\"],\"kind\":\"dev\",\"name\":\"futures-channel\",\"req\":\"^0.3\"},{\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3.31\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\",\"sink\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"h2\",\"optional\":true,\"req\":\"^0.4.2\"},{\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"http-body\",\"req\":\"^1\"},{\"name\":\"http-body-util\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"name\":\"httparse\",\"optional\":true,\"req\":\"^1.9\"},{\"name\":\"httpdate\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"itoa\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"pin-project-lite\",\"optional\":true,\"req\":\"^0.2.4\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.4\"},{\"name\":\"pin-utils\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"pretty_env_logger\",\"req\":\"^0.5\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"features\":[\"const_generics\",\"const_new\"],\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1.12\"},{\"kind\":\"dev\",\"name\":\"spmc\",\"req\":\"^0.3\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"req\":\"^1\"},{\"features\":[\"fs\",\"macros\",\"net\",\"io-std\",\"io-util\",\"rt\",\"rt-multi-thread\",\"sync\",\"time\",\"test-util\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"tokio-util\",\"req\":\"^0.7.10\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"want\",\"optional\":true,\"req\":\"^0.3\"}],\"features\":{\"capi\":[],\"client\":[\"dep:want\",\"dep:pin-project-lite\",\"dep:smallvec\"],\"default\":[],\"ffi\":[\"dep:http-body-util\",\"dep:futures-util\"],\"full\":[\"client\",\"http1\",\"http2\",\"server\"],\"http1\":[\"dep:atomic-waker\",\"dep:futures-channel\",\"dep:futures-core\",\"dep:httparse\",\"dep:itoa\",\"dep:pin-utils\"],\"http2\":[\"dep:futures-channel\",\"dep:futures-core\",\"dep:h2\"],\"nightly\":[],\"server\":[\"dep:httpdate\",\"dep:pin-project-lite\",\"dep:smallvec\"],\"tracing\":[\"dep:tracing\"]}}", - "id-arena_2.3.0": "{\"dependencies\":[{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.0.3\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "ident_case_1.0.1": "{\"dependencies\":[],\"features\":{}}", - "indexmap_1.9.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"build\",\"name\":\"autocfg\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2.1\"},{\"default_features\":false,\"features\":[\"raw\"],\"name\":\"hashbrown\",\"req\":\"^0.12\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.3\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.4.1\"},{\"name\":\"rustc-rayon\",\"optional\":true,\"package\":\"rustc-rayon\",\"req\":\"^0.5\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0\"}],\"features\":{\"serde-1\":[\"serde\"],\"std\":[],\"test_debug\":[],\"test_low_transition_point\":[]}}", - "indexmap_2.13.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"borsh\",\"optional\":true,\"req\":\"^1.2\"},{\"default_features\":false,\"name\":\"equivalent\",\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"fastrand\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"hashbrown\",\"req\":\"^0.16.1\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.14\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.9\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"},{\"default_features\":false,\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.220\"},{\"default_features\":false,\"name\":\"sval\",\"optional\":true,\"req\":\"^2\"}],\"features\":{\"default\":[\"std\"],\"serde\":[\"dep:serde_core\",\"dep:serde\"],\"std\":[],\"test_debug\":[]}}", - "itertools_0.13.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"permutohedron\",\"req\":\"^0.2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"use_std\"],\"use_alloc\":[],\"use_std\":[\"use_alloc\",\"either/use_std\"]}}", - "itertools_0.14.0": "{\"dependencies\":[{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"permutohedron\",\"req\":\"^0.2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"use_std\"],\"use_alloc\":[],\"use_std\":[\"use_alloc\",\"either/use_std\"]}}", - "itoa_1.0.14": "{\"dependencies\":[{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"}],\"features\":{}}", - "itoa_1.0.17": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\",\"target\":\"cfg(not(miri))\"},{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"}],\"features\":{}}", - "leb128fmt_0.1.0": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[]}}", - "libc_0.2.169": "{\"dependencies\":[{\"name\":\"rustc-std-workspace-core\",\"optional\":true,\"req\":\"^1.0.0\"}],\"features\":{\"align\":[],\"const-extern-fn\":[],\"default\":[\"std\"],\"extra_traits\":[],\"rustc-dep-of-std\":[\"align\",\"rustc-std-workspace-core\"],\"std\":[],\"use_std\":[\"std\"]}}", - "libc_0.2.183": "{\"dependencies\":[{\"name\":\"rustc-std-workspace-core\",\"optional\":true,\"req\":\"^1.0.1\"}],\"features\":{\"align\":[],\"const-extern-fn\":[],\"default\":[\"std\"],\"extra_traits\":[],\"rustc-dep-of-std\":[\"align\",\"rustc-std-workspace-core\"],\"std\":[],\"use_std\":[\"std\"]}}", - "linux-raw-sys_0.12.1": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.100\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1.0\"}],\"features\":{\"auxvec\":[],\"bootparam\":[],\"btrfs\":[],\"default\":[\"std\",\"general\",\"errno\"],\"elf\":[],\"elf_uapi\":[],\"errno\":[],\"general\":[],\"if_arp\":[],\"if_ether\":[],\"if_packet\":[],\"if_tun\":[],\"image\":[],\"io_uring\":[],\"ioctl\":[],\"landlock\":[],\"loop_device\":[],\"mempolicy\":[],\"net\":[],\"netlink\":[],\"no_std\":[],\"prctl\":[],\"ptrace\":[],\"rustc-dep-of-std\":[\"core\",\"no_std\"],\"std\":[],\"system\":[],\"vm_sockets\":[],\"xdp\":[]}}", - "lock_api_0.4.14": "{\"dependencies\":[{\"name\":\"owning_ref\",\"optional\":true,\"req\":\"^0.4.1\"},{\"default_features\":false,\"name\":\"scopeguard\",\"req\":\"^1.1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.126\"}],\"features\":{\"arc_lock\":[],\"atomic_usize\":[],\"default\":[\"atomic_usize\"],\"nightly\":[]}}", - "log_0.4.22": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.63\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"sval\",\"optional\":true,\"req\":\"^2.1\"},{\"kind\":\"dev\",\"name\":\"sval\",\"req\":\"^2.1\"},{\"kind\":\"dev\",\"name\":\"sval_derive\",\"req\":\"^2.1\"},{\"default_features\":false,\"name\":\"sval_ref\",\"optional\":true,\"req\":\"^2.1\"},{\"default_features\":false,\"features\":[\"inline-i128\"],\"name\":\"value-bag\",\"optional\":true,\"req\":\"^1.7\"},{\"features\":[\"test\"],\"kind\":\"dev\",\"name\":\"value-bag\",\"req\":\"^1.7\"}],\"features\":{\"kv\":[],\"kv_serde\":[\"kv_std\",\"value-bag/serde\",\"serde\"],\"kv_std\":[\"std\",\"kv\",\"value-bag/error\"],\"kv_sval\":[\"kv\",\"value-bag/sval\",\"sval\",\"sval_ref\"],\"kv_unstable\":[\"kv\",\"value-bag\"],\"kv_unstable_serde\":[\"kv_serde\",\"kv_unstable_std\"],\"kv_unstable_std\":[\"kv_std\",\"kv_unstable\"],\"kv_unstable_sval\":[\"kv_sval\",\"kv_unstable\"],\"max_level_debug\":[],\"max_level_error\":[],\"max_level_info\":[],\"max_level_off\":[],\"max_level_trace\":[],\"max_level_warn\":[],\"release_max_level_debug\":[],\"release_max_level_error\":[],\"release_max_level_info\":[],\"release_max_level_off\":[],\"release_max_level_trace\":[],\"release_max_level_warn\":[],\"std\":[]}}", - "log_0.4.29": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.63\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"sval\",\"optional\":true,\"req\":\"^2.16\"},{\"kind\":\"dev\",\"name\":\"sval\",\"req\":\"^2.16\"},{\"kind\":\"dev\",\"name\":\"sval_derive\",\"req\":\"^2.16\"},{\"default_features\":false,\"name\":\"sval_ref\",\"optional\":true,\"req\":\"^2.16\"},{\"default_features\":false,\"features\":[\"inline-i128\"],\"name\":\"value-bag\",\"optional\":true,\"req\":\"^1.12\"},{\"features\":[\"test\"],\"kind\":\"dev\",\"name\":\"value-bag\",\"req\":\"^1.12\"}],\"features\":{\"kv\":[],\"kv_serde\":[\"kv_std\",\"value-bag/serde\",\"serde\"],\"kv_std\":[\"std\",\"kv\",\"value-bag/error\"],\"kv_sval\":[\"kv\",\"value-bag/sval\",\"sval\",\"sval_ref\"],\"kv_unstable\":[\"kv\",\"value-bag\"],\"kv_unstable_serde\":[\"kv_serde\",\"kv_unstable_std\"],\"kv_unstable_std\":[\"kv_std\",\"kv_unstable\"],\"kv_unstable_sval\":[\"kv_sval\",\"kv_unstable\"],\"max_level_debug\":[],\"max_level_error\":[],\"max_level_info\":[],\"max_level_off\":[],\"max_level_trace\":[],\"max_level_warn\":[],\"release_max_level_debug\":[],\"release_max_level_error\":[],\"release_max_level_info\":[],\"release_max_level_off\":[],\"release_max_level_trace\":[],\"release_max_level_warn\":[],\"serde\":[\"serde_core\"],\"std\":[]}}", - "macro_rules_attribute-proc_macro_0.2.0": "{\"dependencies\":[],\"features\":{\"verbose-expansions\":[]}}", - "macro_rules_attribute_0.2.0": "{\"dependencies\":[{\"name\":\"macro_rules_attribute-proc_macro\",\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.10.0\"},{\"name\":\"paste\",\"req\":\"^1.0.7\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.8\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.136\"}],\"features\":{\"better-docs\":[],\"default\":[],\"verbose-expansions\":[\"macro_rules_attribute-proc_macro/verbose-expansions\"]}}", - "matchit_0.7.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"actix-router\",\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.4\"},{\"kind\":\"dev\",\"name\":\"gonzales\",\"req\":\"^0.0.3-beta\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^0.14\"},{\"kind\":\"dev\",\"name\":\"path-tree\",\"req\":\"^0.2.2\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.5.4\"},{\"kind\":\"dev\",\"name\":\"route-recognizer\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"routefinder\",\"req\":\"^0.5.2\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"features\":[\"make\",\"util\"],\"kind\":\"dev\",\"name\":\"tower\",\"req\":\"^0.4\"}],\"features\":{\"__test_helpers\":[],\"default\":[]}}", - "memchr_2.7.4": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.20\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"libc\":[],\"logging\":[\"dep:log\"],\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"],\"std\":[\"alloc\"],\"use_std\":[\"std\"]}}", - "memchr_2.8.0": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.20\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"libc\":[],\"logging\":[\"dep:log\"],\"rustc-dep-of-std\":[\"core\"],\"std\":[\"alloc\"],\"use_std\":[\"std\"]}}", - "mime_0.3.17": "{\"dependencies\":[],\"features\":{}}", - "minimal-lexical_0.2.1": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"compact\":[],\"default\":[\"std\"],\"lint\":[],\"nightly\":[],\"std\":[]}}", - "miniz_oxide_0.8.9": "{\"dependencies\":[{\"default_features\":false,\"name\":\"adler2\",\"req\":\"^2.0\"},{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"simd-adler32\",\"optional\":true,\"req\":\"^0.3.3\"}],\"features\":{\"block-boundary\":[],\"default\":[\"with-alloc\"],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"adler2/rustc-dep-of-std\"],\"simd\":[\"simd-adler32\"],\"std\":[],\"with-alloc\":[]}}", - "mio_1.1.1": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"libc\",\"req\":\"^0.2.178\",\"target\":\"cfg(target_os = \\\"hermit\\\")\"},{\"name\":\"libc\",\"req\":\"^0.2.178\",\"target\":\"cfg(target_os = \\\"wasi\\\")\"},{\"name\":\"libc\",\"req\":\"^0.2.178\",\"target\":\"cfg(unix)\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.8\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9\"},{\"name\":\"wasi\",\"req\":\"^0.11.0\",\"target\":\"cfg(target_os = \\\"wasi\\\")\"},{\"features\":[\"Wdk_Foundation\",\"Wdk_Storage_FileSystem\",\"Wdk_System_IO\",\"Win32_Foundation\",\"Win32_Networking_WinSock\",\"Win32_Storage_FileSystem\",\"Win32_Security\",\"Win32_System_IO\",\"Win32_System_WindowsProgramming\"],\"name\":\"windows-sys\",\"req\":\"^0.61\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[\"log\"],\"net\":[],\"os-ext\":[\"os-poll\",\"windows-sys/Win32_System_Pipes\",\"windows-sys/Win32_Security\"],\"os-poll\":[]}}", - "monostate-impl_0.1.13": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"default_features\":false,\"features\":[\"parsing\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0.59\"}],\"features\":{}}", - "monostate_0.1.13": "{\"dependencies\":[{\"name\":\"monostate-impl\",\"req\":\"=0.1.13\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.166\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.99\"}],\"features\":{}}", - "multimap_0.10.1": "{\"dependencies\":[{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"serde_impl\"],\"serde_impl\":[\"serde\"]}}", - "nom_7.1.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2.3\"},{\"default_features\":false,\"name\":\"minimal-lexical\",\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.0.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"docsrs\":[],\"std\":[\"alloc\",\"memchr/std\",\"minimal-lexical/std\"]}}", - "object_0.37.3": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"crc32fast\",\"optional\":true,\"req\":\"^1.2\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"default-hasher\"],\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.15.0\"},{\"default_features\":false,\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.0\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2.4.1\"},{\"name\":\"ruzstd\",\"optional\":true,\"req\":\"^0.8.1\"},{\"default_features\":false,\"name\":\"wasmparser\",\"optional\":true,\"req\":\"^0.236.0\"}],\"features\":{\"all\":[\"read\",\"write\",\"build\",\"std\",\"compression\",\"wasm\"],\"archive\":[],\"build\":[\"build_core\",\"write_std\",\"elf\"],\"build_core\":[\"read_core\",\"write_core\"],\"cargo-all\":[],\"coff\":[],\"compression\":[\"dep:flate2\",\"dep:ruzstd\",\"std\"],\"default\":[\"read\",\"compression\"],\"doc\":[\"read_core\",\"write_std\",\"build_core\",\"std\",\"compression\",\"archive\",\"coff\",\"elf\",\"macho\",\"pe\",\"wasm\",\"xcoff\"],\"elf\":[],\"macho\":[],\"pe\":[\"coff\"],\"read\":[\"read_core\",\"archive\",\"coff\",\"elf\",\"macho\",\"pe\",\"xcoff\",\"unaligned\"],\"read_core\":[],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"memchr/rustc-dep-of-std\"],\"std\":[\"memchr/std\"],\"unaligned\":[],\"unstable\":[],\"unstable-all\":[\"all\",\"unstable\"],\"wasm\":[\"dep:wasmparser\"],\"write\":[\"write_std\",\"coff\",\"elf\",\"macho\",\"pe\",\"xcoff\"],\"write_core\":[\"dep:crc32fast\",\"dep:indexmap\",\"dep:hashbrown\"],\"write_std\":[\"write_core\",\"std\",\"indexmap?/std\",\"crc32fast?/std\"],\"xcoff\":[]}}", - "once_cell_1.20.2": "{\"dependencies\":[{\"name\":\"critical-section\",\"optional\":true,\"req\":\"^1.1.3\"},{\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"critical-section\",\"req\":\"^1.1.3\"},{\"default_features\":false,\"name\":\"parking_lot_core\",\"optional\":true,\"req\":\"^0.9.10\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.10.6\"}],\"features\":{\"alloc\":[\"race\"],\"atomic-polyfill\":[\"critical-section\"],\"critical-section\":[\"dep:critical-section\",\"portable-atomic\"],\"default\":[\"std\"],\"parking_lot\":[\"dep:parking_lot_core\"],\"portable-atomic\":[\"dep:portable-atomic\"],\"race\":[],\"std\":[\"alloc\"],\"unstable\":[]}}", - "once_cell_1.21.4": "{\"dependencies\":[{\"name\":\"critical-section\",\"optional\":true,\"req\":\"^1.1.3\"},{\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"critical-section\",\"req\":\"^1.1.3\"},{\"default_features\":false,\"name\":\"parking_lot_core\",\"optional\":true,\"req\":\"^0.9.10\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.10.6\"}],\"features\":{\"alloc\":[\"race\"],\"atomic-polyfill\":[\"critical-section\"],\"critical-section\":[\"dep:critical-section\",\"portable-atomic\"],\"default\":[\"std\"],\"parking_lot\":[\"dep:parking_lot_core\"],\"portable-atomic\":[\"dep:portable-atomic\"],\"race\":[],\"std\":[\"alloc\"],\"unstable\":[]}}", - "onig_6.5.1": "{\"dependencies\":[{\"name\":\"bitflags\",\"req\":\"^2.4.0\"},{\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(windows)\"},{\"name\":\"once_cell\",\"req\":\"^1.12\"},{\"default_features\":false,\"name\":\"onig_sys\",\"req\":\"^69.9.1\"}],\"features\":{\"default\":[\"generate\"],\"generate\":[\"onig_sys/generate\"],\"posix-api\":[\"onig_sys/posix-api\"],\"print-debug\":[\"onig_sys/print-debug\"],\"std-pattern\":[]}}", - "onig_sys_69.9.1": "{\"dependencies\":[{\"features\":[\"runtime\"],\"kind\":\"build\",\"name\":\"bindgen\",\"optional\":true,\"req\":\"^0.71\"},{\"kind\":\"build\",\"name\":\"cc\",\"req\":\"^1.0\"},{\"kind\":\"build\",\"name\":\"pkg-config\",\"req\":\"^0.3.16\"}],\"features\":{\"default\":[\"generate\"],\"generate\":[\"bindgen\"],\"posix-api\":[],\"print-debug\":[]}}", - "parking_lot_0.12.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.3.3\"},{\"name\":\"lock_api\",\"req\":\"^0.4.14\"},{\"name\":\"parking_lot_core\",\"req\":\"^0.9.12\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.3\"}],\"features\":{\"arc_lock\":[\"lock_api/arc_lock\"],\"deadlock_detection\":[\"parking_lot_core/deadlock_detection\"],\"default\":[],\"hardware-lock-elision\":[],\"nightly\":[\"parking_lot_core/nightly\",\"lock_api/nightly\"],\"owning_ref\":[\"lock_api/owning_ref\"],\"send_guard\":[],\"serde\":[\"lock_api/serde\"]}}", - "parking_lot_core_0.9.12": "{\"dependencies\":[{\"name\":\"backtrace\",\"optional\":true,\"req\":\"^0.3.60\"},{\"name\":\"cfg-if\",\"req\":\"^1.0.0\"},{\"name\":\"libc\",\"req\":\"^0.2.95\",\"target\":\"cfg(unix)\"},{\"name\":\"petgraph\",\"optional\":true,\"req\":\"^0.6.0\"},{\"name\":\"redox_syscall\",\"req\":\"^0.5\",\"target\":\"cfg(target_os = \\\"redox\\\")\"},{\"name\":\"smallvec\",\"req\":\"^1.6.1\"},{\"name\":\"windows-link\",\"req\":\"^0.2.0\",\"target\":\"cfg(windows)\"}],\"features\":{\"deadlock_detection\":[\"petgraph\",\"backtrace\"],\"nightly\":[]}}", - "paste_1.0.15": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"paste-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", - "percent-encoding_2.3.2": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", - "petgraph_0.6.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"ahash\",\"req\":\"^0.7.2\"},{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.3.3\"},{\"kind\":\"dev\",\"name\":\"defmac\",\"req\":\"^0.2.1\"},{\"default_features\":false,\"name\":\"fixedbitset\",\"req\":\"^0.4.0\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2.1\"},{\"name\":\"indexmap\",\"req\":\"^2.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.12.1\"},{\"kind\":\"dev\",\"name\":\"odds\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.5.5\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.5.3\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1.0\"}],\"features\":{\"all\":[\"unstable\",\"quickcheck\",\"matrix_graph\",\"stable_graph\",\"graphmap\",\"rayon\"],\"default\":[\"graphmap\",\"stable_graph\",\"matrix_graph\"],\"generate\":[],\"graphmap\":[],\"matrix_graph\":[],\"rayon\":[\"dep:rayon\",\"indexmap/rayon\"],\"serde-1\":[\"serde\",\"serde_derive\"],\"stable_graph\":[],\"unstable\":[\"generate\"]}}", - "pin-project-internal_1.1.11": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1.0.25\"},{\"default_features\":false,\"features\":[\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\",\"full\",\"visit-mut\"],\"name\":\"syn\",\"req\":\"^2.0.1\"}],\"features\":{}}", - "pin-project-lite_0.2.17": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1\"}],\"features\":{}}", - "pin-project_1.1.11": "{\"dependencies\":[{\"name\":\"pin-project-internal\",\"req\":\"=1.1.11\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1\"}],\"features\":{}}", - "pin-utils_0.1.0": "{\"dependencies\":[],\"features\":{}}", - "pkg-config_0.3.32": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1\"}],\"features\":{}}", - "ppv-lite86_0.2.20": "{\"dependencies\":[{\"features\":[\"simd\",\"derive\"],\"name\":\"zerocopy\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"std\"],\"no_simd\":[],\"simd\":[],\"std\":[]}}", - "ppv-lite86_0.2.21": "{\"dependencies\":[{\"features\":[\"simd\"],\"name\":\"zerocopy\",\"req\":\"^0.8.23\"}],\"features\":{\"default\":[\"std\"],\"no_simd\":[],\"simd\":[],\"std\":[]}}", - "prettyplease_0.2.37": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"default_features\":false,\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.105\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"extra-traits\",\"parsing\",\"printing\",\"visit-mut\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.105\"}],\"features\":{\"verbatim\":[\"syn/parsing\"]}}", - "proc-macro2_1.0.106": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4\"},{\"name\":\"unicode-ident\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"proc-macro\"],\"nightly\":[],\"proc-macro\":[],\"span-locations\":[]}}", - "proc-macro2_1.0.92": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4\"},{\"name\":\"unicode-ident\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"proc-macro\"],\"nightly\":[],\"proc-macro\":[],\"span-locations\":[]}}", - "prost-build_0.13.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"heck\",\"req\":\">=0.4, <=0.5\"},{\"default_features\":false,\"features\":[\"use_alloc\"],\"name\":\"itertools\",\"req\":\">=0.10, <=0.13\"},{\"name\":\"log\",\"req\":\"^0.4.4\"},{\"default_features\":false,\"name\":\"multimap\",\"req\":\">=0.8, <=0.10\"},{\"name\":\"once_cell\",\"req\":\"^1.17.1\"},{\"default_features\":false,\"name\":\"petgraph\",\"req\":\"^0.6\"},{\"name\":\"prettyplease\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"name\":\"prost\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-types\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"pulldown-cmark\",\"optional\":true,\"req\":\"^0.9.1\"},{\"name\":\"pulldown-cmark-to-cmark\",\"optional\":true,\"req\":\"^10.0.1\"},{\"default_features\":false,\"features\":[\"std\",\"unicode-bool\"],\"name\":\"regex\",\"req\":\"^1.8.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"optional\":true,\"req\":\"^2\"},{\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{\"cleanup-markdown\":[\"dep:pulldown-cmark\",\"dep:pulldown-cmark-to-cmark\"],\"default\":[\"format\"],\"format\":[\"dep:prettyplease\",\"dep:syn\"]}}", - "prost-derive_0.13.1": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.1\"},{\"name\":\"itertools\",\"req\":\">=0.10.1, <=0.13\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2\"}],\"features\":{}}", - "prost-types_0.13.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"prost-derive\"],\"name\":\"prost\",\"req\":\"^0.13.1\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"prost/std\"]}}", - "prost_0.13.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"name\":\"prost-derive\",\"optional\":true,\"req\":\"^0.13.1\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"derive\",\"std\"],\"derive\":[\"dep:prost-derive\"],\"no-recursion-limit\":[],\"prost-derive\":[\"derive\"],\"std\":[]}}", - "protoc-gen-prost_0.4.0": "{\"dependencies\":[{\"name\":\"once_cell\",\"req\":\"^1.10.0\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"prost\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-build\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-types\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"regex\",\"req\":\"^1.5.5\"}],\"features\":{}}", - "protoc-gen-tonic_0.4.1": "{\"dependencies\":[{\"name\":\"heck\",\"req\":\"^0.5.0\"},{\"name\":\"prettyplease\",\"req\":\"^0.2.9\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"prost\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-build\",\"req\":\"^0.13.1\"},{\"default_features\":false,\"name\":\"prost-types\",\"req\":\"^0.13.1\"},{\"name\":\"protoc-gen-prost\",\"req\":\"^0.4.0\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"regex\",\"req\":\"^1.5.5\"},{\"features\":[\"parsing\",\"full\"],\"name\":\"syn\",\"req\":\"^2.0.22\"},{\"name\":\"tonic-build\",\"req\":\"^0.12.0\"}],\"features\":{}}", - "quote_1.0.37": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.66\"}],\"features\":{\"default\":[\"proc-macro\"],\"proc-macro\":[\"proc-macro2/proc-macro\"]}}", - "quote_1.0.45": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"}],\"features\":{\"default\":[\"proc-macro\"],\"proc-macro\":[\"proc-macro2/proc-macro\"]}}", - "r-efi_5.3.0": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"efiapi\":[],\"examples\":[\"native\"],\"native\":[],\"rustc-dep-of-std\":[\"core\"]}}", - "r-efi_6.0.0": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"native\":[],\"rustc-dep-of-std\":[\"core\"]}}", - "rand_0.8.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.2.1\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.22\",\"target\":\"cfg(unix)\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.4\"},{\"features\":[\"into_bits\"],\"name\":\"packed_simd\",\"optional\":true,\"package\":\"packed_simd_2\",\"req\":\"^0.3.7\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"rand_core\",\"req\":\"^0.6.0\"},{\"kind\":\"dev\",\"name\":\"rand_pcg\",\"req\":\"^0.3.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"}],\"features\":{\"alloc\":[\"rand_core/alloc\"],\"default\":[\"std\",\"std_rng\"],\"getrandom\":[\"rand_core/getrandom\"],\"min_const_gen\":[],\"nightly\":[],\"serde1\":[\"serde\",\"rand_core/serde1\"],\"simd_support\":[\"packed_simd\"],\"small_rng\":[],\"std\":[\"rand_core/std\",\"rand_chacha/std\",\"alloc\",\"getrandom\",\"libc\"],\"std_rng\":[\"rand_chacha\"]}}", - "rand_0.9.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.2.1\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.4\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"optional\":true,\"req\":\"^0.9.0\"},{\"default_features\":false,\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rand_pcg\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.7\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"std_rng\",\"os_rng\",\"small_rng\",\"thread_rng\"],\"log\":[\"dep:log\"],\"nightly\":[],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\",\"rand_core/serde\"],\"simd_support\":[],\"small_rng\":[],\"std\":[\"rand_core/std\",\"rand_chacha?/std\",\"alloc\"],\"std_rng\":[\"dep:rand_chacha\"],\"thread_rng\":[\"std\",\"std_rng\",\"os_rng\"],\"unbiased\":[]}}", - "rand_chacha_0.3.1": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"ppv-lite86\",\"req\":\"^0.2.8\"},{\"name\":\"rand_core\",\"req\":\"^0.6.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"serde1\":[\"serde\"],\"simd\":[],\"std\":[\"ppv-lite86/std\"]}}", - "rand_chacha_0.9.0": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"ppv-lite86\",\"req\":\"^0.2.14\"},{\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"features\":[\"os_rng\"],\"kind\":\"dev\",\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\"],\"std\":[\"ppv-lite86/std\",\"rand_core/std\"]}}", - "rand_core_0.6.4": "{\"dependencies\":[{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.2\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"serde1\":[\"serde\"],\"std\":[\"alloc\",\"getrandom\",\"getrandom/std\"]}}", - "rand_core_0.9.3": "{\"dependencies\":[{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.3.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"os_rng\":[\"dep:getrandom\"],\"serde\":[\"dep:serde\"],\"std\":[\"getrandom?/std\"]}}", - "rayon-cond_0.4.0": "{\"dependencies\":[{\"name\":\"either\",\"req\":\"^1.6\"},{\"name\":\"itertools\",\"req\":\"^0.14\"},{\"name\":\"rayon\",\"req\":\"^1.7\"}],\"features\":{}}", - "rayon-core_1.12.1": "{\"dependencies\":[{\"name\":\"crossbeam-deque\",\"req\":\"^0.8.1\"},{\"name\":\"crossbeam-utils\",\"req\":\"^0.8.0\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"scoped-tls\",\"req\":\"^1.0\"},{\"name\":\"wasm_sync\",\"optional\":true,\"req\":\"^0.1.0\"}],\"features\":{\"web_spin_lock\":[\"dep:wasm_sync\"]}}", - "rayon_1.10.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"name\":\"rayon-core\",\"req\":\"^1.12.1\"},{\"name\":\"wasm_sync\",\"optional\":true,\"req\":\"^0.1.0\"}],\"features\":{\"web_spin_lock\":[\"dep:wasm_sync\",\"rayon-core/web_spin_lock\"]}}", - "redox_syscall_0.5.18": "{\"dependencies\":[{\"name\":\"bitflags\",\"req\":\"^2.4\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"loom\",\"req\":\"^0.7\",\"target\":\"cfg(loom)\"}],\"features\":{\"default\":[\"userspace\"],\"rustc-dep-of-std\":[\"core\",\"bitflags/rustc-dep-of-std\"],\"std\":[],\"userspace\":[]}}", - "regex-automata_0.4.14": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"bstr\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.14\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"optional\":true,\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"syntax\",\"perf\",\"unicode\",\"meta\",\"nfa\",\"dfa\",\"hybrid\"],\"dfa\":[\"dfa-build\",\"dfa-search\",\"dfa-onepass\"],\"dfa-build\":[\"nfa-thompson\",\"dfa-search\"],\"dfa-onepass\":[\"nfa-thompson\"],\"dfa-search\":[],\"hybrid\":[\"alloc\",\"nfa-thompson\"],\"internal-instrument\":[\"internal-instrument-pikevm\"],\"internal-instrument-pikevm\":[\"logging\",\"std\"],\"logging\":[\"dep:log\",\"aho-corasick?/logging\",\"memchr?/logging\"],\"meta\":[\"syntax\",\"nfa-pikevm\"],\"nfa\":[\"nfa-thompson\",\"nfa-pikevm\",\"nfa-backtrack\"],\"nfa-backtrack\":[\"nfa-thompson\"],\"nfa-pikevm\":[\"nfa-thompson\"],\"nfa-thompson\":[\"alloc\"],\"perf\":[\"perf-inline\",\"perf-literal\"],\"perf-inline\":[],\"perf-literal\":[\"perf-literal-substring\",\"perf-literal-multisubstring\"],\"perf-literal-multisubstring\":[\"dep:aho-corasick\"],\"perf-literal-substring\":[\"aho-corasick?/perf-literal\",\"dep:memchr\"],\"std\":[\"regex-syntax?/std\",\"memchr?/std\",\"aho-corasick?/std\",\"alloc\"],\"syntax\":[\"dep:regex-syntax\",\"alloc\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"unicode-word-boundary\",\"regex-syntax?/unicode\"],\"unicode-age\":[\"regex-syntax?/unicode-age\"],\"unicode-bool\":[\"regex-syntax?/unicode-bool\"],\"unicode-case\":[\"regex-syntax?/unicode-case\"],\"unicode-gencat\":[\"regex-syntax?/unicode-gencat\"],\"unicode-perl\":[\"regex-syntax?/unicode-perl\"],\"unicode-script\":[\"regex-syntax?/unicode-script\"],\"unicode-segment\":[\"regex-syntax?/unicode-segment\"],\"unicode-word-boundary\":[]}}", - "regex-automata_0.4.9": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"bstr\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.14\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"optional\":true,\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"syntax\",\"perf\",\"unicode\",\"meta\",\"nfa\",\"dfa\",\"hybrid\"],\"dfa\":[\"dfa-build\",\"dfa-search\",\"dfa-onepass\"],\"dfa-build\":[\"nfa-thompson\",\"dfa-search\"],\"dfa-onepass\":[\"nfa-thompson\"],\"dfa-search\":[],\"hybrid\":[\"alloc\",\"nfa-thompson\"],\"internal-instrument\":[\"internal-instrument-pikevm\"],\"internal-instrument-pikevm\":[\"logging\",\"std\"],\"logging\":[\"dep:log\",\"aho-corasick?/logging\",\"memchr?/logging\"],\"meta\":[\"syntax\",\"nfa-pikevm\"],\"nfa\":[\"nfa-thompson\",\"nfa-pikevm\",\"nfa-backtrack\"],\"nfa-backtrack\":[\"nfa-thompson\"],\"nfa-pikevm\":[\"nfa-thompson\"],\"nfa-thompson\":[\"alloc\"],\"perf\":[\"perf-inline\",\"perf-literal\"],\"perf-inline\":[],\"perf-literal\":[\"perf-literal-substring\",\"perf-literal-multisubstring\"],\"perf-literal-multisubstring\":[\"std\",\"dep:aho-corasick\"],\"perf-literal-substring\":[\"aho-corasick?/perf-literal\",\"dep:memchr\"],\"std\":[\"regex-syntax?/std\",\"memchr?/std\",\"aho-corasick?/std\",\"alloc\"],\"syntax\":[\"dep:regex-syntax\",\"alloc\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"unicode-word-boundary\",\"regex-syntax?/unicode\"],\"unicode-age\":[\"regex-syntax?/unicode-age\"],\"unicode-bool\":[\"regex-syntax?/unicode-bool\"],\"unicode-case\":[\"regex-syntax?/unicode-case\"],\"unicode-gencat\":[\"regex-syntax?/unicode-gencat\"],\"unicode-perl\":[\"regex-syntax?/unicode-perl\"],\"unicode-script\":[\"regex-syntax?/unicode-script\"],\"unicode-segment\":[\"regex-syntax?/unicode-segment\"],\"unicode-word-boundary\":[]}}", - "regex-syntax_0.8.10": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.3.0\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"default\":[\"std\",\"unicode\"],\"std\":[],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\"],\"unicode-age\":[],\"unicode-bool\":[],\"unicode-case\":[],\"unicode-gencat\":[],\"unicode-perl\":[],\"unicode-script\":[],\"unicode-segment\":[]}}", - "regex-syntax_0.8.5": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.3.0\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"default\":[\"std\",\"unicode\"],\"std\":[],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\"],\"unicode-age\":[],\"unicode-bool\":[],\"unicode-case\":[],\"unicode-gencat\":[],\"unicode-perl\":[],\"unicode-script\":[],\"unicode-segment\":[]}}", - "regex_1.11.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.17.1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"features\":[\"alloc\",\"syntax\",\"meta\",\"nfa-pikevm\"],\"name\":\"regex-automata\",\"req\":\"^0.4.8\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"default\":[\"std\",\"perf\",\"unicode\",\"regex-syntax/default\"],\"logging\":[\"aho-corasick?/logging\",\"memchr?/logging\",\"regex-automata/logging\"],\"pattern\":[],\"perf\":[\"perf-cache\",\"perf-dfa\",\"perf-onepass\",\"perf-backtrack\",\"perf-inline\",\"perf-literal\"],\"perf-backtrack\":[\"regex-automata/nfa-backtrack\"],\"perf-cache\":[],\"perf-dfa\":[\"regex-automata/hybrid\"],\"perf-dfa-full\":[\"regex-automata/dfa-build\",\"regex-automata/dfa-search\"],\"perf-inline\":[\"regex-automata/perf-inline\"],\"perf-literal\":[\"dep:aho-corasick\",\"dep:memchr\",\"regex-automata/perf-literal\"],\"perf-onepass\":[\"regex-automata/dfa-onepass\"],\"std\":[\"aho-corasick?/std\",\"memchr?/std\",\"regex-automata/std\",\"regex-syntax/std\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"regex-automata/unicode\",\"regex-syntax/unicode\"],\"unicode-age\":[\"regex-automata/unicode-age\",\"regex-syntax/unicode-age\"],\"unicode-bool\":[\"regex-automata/unicode-bool\",\"regex-syntax/unicode-bool\"],\"unicode-case\":[\"regex-automata/unicode-case\",\"regex-syntax/unicode-case\"],\"unicode-gencat\":[\"regex-automata/unicode-gencat\",\"regex-syntax/unicode-gencat\"],\"unicode-perl\":[\"regex-automata/unicode-perl\",\"regex-automata/unicode-word-boundary\",\"regex-syntax/unicode-perl\"],\"unicode-script\":[\"regex-automata/unicode-script\",\"regex-syntax/unicode-script\"],\"unicode-segment\":[\"regex-automata/unicode-segment\",\"regex-syntax/unicode-segment\"],\"unstable\":[\"pattern\"],\"use_std\":[\"std\"]}}", - "regex_1.12.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"features\":[\"alloc\",\"syntax\",\"meta\",\"nfa-pikevm\"],\"name\":\"regex-automata\",\"req\":\"^0.4.12\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"default\":[\"std\",\"perf\",\"unicode\",\"regex-syntax/default\"],\"logging\":[\"aho-corasick?/logging\",\"memchr?/logging\",\"regex-automata/logging\"],\"pattern\":[],\"perf\":[\"perf-cache\",\"perf-dfa\",\"perf-onepass\",\"perf-backtrack\",\"perf-inline\",\"perf-literal\"],\"perf-backtrack\":[\"regex-automata/nfa-backtrack\"],\"perf-cache\":[],\"perf-dfa\":[\"regex-automata/hybrid\"],\"perf-dfa-full\":[\"regex-automata/dfa-build\",\"regex-automata/dfa-search\"],\"perf-inline\":[\"regex-automata/perf-inline\"],\"perf-literal\":[\"dep:aho-corasick\",\"dep:memchr\",\"regex-automata/perf-literal\"],\"perf-onepass\":[\"regex-automata/dfa-onepass\"],\"std\":[\"aho-corasick?/std\",\"memchr?/std\",\"regex-automata/std\",\"regex-syntax/std\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"regex-automata/unicode\",\"regex-syntax/unicode\"],\"unicode-age\":[\"regex-automata/unicode-age\",\"regex-syntax/unicode-age\"],\"unicode-bool\":[\"regex-automata/unicode-bool\",\"regex-syntax/unicode-bool\"],\"unicode-case\":[\"regex-automata/unicode-case\",\"regex-syntax/unicode-case\"],\"unicode-gencat\":[\"regex-automata/unicode-gencat\",\"regex-syntax/unicode-gencat\"],\"unicode-perl\":[\"regex-automata/unicode-perl\",\"regex-automata/unicode-word-boundary\",\"regex-syntax/unicode-perl\"],\"unicode-script\":[\"regex-automata/unicode-script\",\"regex-syntax/unicode-script\"],\"unicode-segment\":[\"regex-automata/unicode-segment\",\"regex-syntax/unicode-segment\"],\"unstable\":[\"pattern\"],\"use_std\":[\"std\"]}}", - "rustc-demangle_0.1.27": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"compiler_builtins\":[],\"rustc-dep-of-std\":[\"core\"],\"std\":[]}}", - "rustix_1.1.4": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bitflags\",\"req\":\"^2.4.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4\",\"target\":\"cfg(all(criterion, not(any(target_os = \\\"emscripten\\\", target_os = \\\"wasi\\\"))))\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.182\",\"target\":\"cfg(all(not(windows), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.182\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.171\"},{\"default_features\":false,\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(all(not(windows), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(windows)\"},{\"default_features\":false,\"name\":\"libc_errno\",\"optional\":true,\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\"},{\"default_features\":false,\"features\":[\"general\",\"ioctl\",\"no_std\"],\"name\":\"linux-raw-sys\",\"req\":\"^0.12\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"features\":[\"auxvec\",\"general\",\"errno\",\"ioctl\",\"no_std\",\"elf\"],\"name\":\"linux-raw-sys\",\"req\":\"^0.12\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"kind\":\"dev\",\"name\":\"memoffset\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.20.3\",\"target\":\"cfg(windows)\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"serial_test\",\"req\":\"^2.0.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.5.0\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\"],\"name\":\"windows-sys\",\"req\":\">=0.52, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"all-apis\":[\"event\",\"fs\",\"io_uring\",\"mm\",\"mount\",\"net\",\"param\",\"pipe\",\"process\",\"pty\",\"rand\",\"runtime\",\"shm\",\"stdio\",\"system\",\"termios\",\"thread\",\"time\"],\"alloc\":[],\"default\":[\"std\"],\"event\":[],\"fs\":[],\"io_uring\":[\"event\",\"fs\",\"net\",\"thread\",\"linux-raw-sys/io_uring\"],\"linux_4_11\":[],\"linux_5_1\":[\"linux_4_11\"],\"linux_5_11\":[\"linux_5_1\"],\"linux_latest\":[\"linux_5_11\"],\"mm\":[],\"mount\":[],\"net\":[\"linux-raw-sys/net\",\"linux-raw-sys/netlink\",\"linux-raw-sys/if_ether\",\"linux-raw-sys/xdp\"],\"param\":[],\"pipe\":[],\"process\":[\"linux-raw-sys/prctl\"],\"pty\":[\"fs\"],\"rand\":[],\"runtime\":[\"linux-raw-sys/prctl\"],\"rustc-dep-of-std\":[\"core\",\"rustc-std-workspace-alloc\",\"linux-raw-sys/rustc-dep-of-std\",\"bitflags/rustc-dep-of-std\"],\"shm\":[\"fs\"],\"std\":[\"bitflags/std\",\"alloc\",\"libc?/std\",\"libc_errno?/std\"],\"stdio\":[],\"system\":[\"linux-raw-sys/system\"],\"termios\":[],\"thread\":[\"linux-raw-sys/prctl\"],\"time\":[],\"try_close\":[],\"use-explicitly-provided-auxv\":[],\"use-libc\":[\"libc_errno\",\"libc\"],\"use-libc-auxv\":[]}}", - "rustversion_1.0.21": "{\"dependencies\":[{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", - "rustversion_1.0.22": "{\"dependencies\":[{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", - "ryu_1.0.18": "{\"dependencies\":[{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"num_cpus\",\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"}],\"features\":{\"small\":[]}}", - "scopeguard_1.2.0": "{\"dependencies\":[],\"features\":{\"default\":[\"use_std\"],\"use_std\":[]}}", - "semver_1.0.27": "{\"dependencies\":[{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"package\":\"serde_core\",\"req\":\"^1.0.220\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"}],\"features\":{\"default\":[\"std\"],\"serde\":[\"dep:serde\"],\"std\":[]}}", - "serde_1.0.216": "{\"dependencies\":[{\"name\":\"serde_derive\",\"req\":\"=1.0.216\",\"target\":\"cfg(any())\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"derive\":[\"serde_derive\"],\"rc\":[],\"std\":[],\"unstable\":[]}}", - "serde_1.0.228": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"result\"],\"name\":\"serde_core\",\"req\":\"=1.0.228\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[\"serde_core/alloc\"],\"default\":[\"std\"],\"derive\":[\"serde_derive\"],\"rc\":[\"serde_core/rc\"],\"std\":[\"serde_core/std\"],\"unstable\":[\"serde_core/unstable\"]}}", - "serde_core_1.0.228": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"name\":\"serde_derive\",\"req\":\"=1.0.228\",\"target\":\"cfg(any())\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"result\"],\"rc\":[],\"result\":[],\"std\":[],\"unstable\":[]}}", - "serde_derive_1.0.216": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"derive\",\"parsing\",\"printing\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0.81\"}],\"features\":{\"default\":[],\"deserialize_in_place\":[]}}", - "serde_derive_1.0.228": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"derive\",\"parsing\",\"printing\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0.81\"}],\"features\":{\"default\":[],\"deserialize_in_place\":[]}}", - "serde_json_1.0.134": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0.11\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.2.3\"},{\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2.0.2\"},{\"name\":\"itoa\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"name\":\"ryu\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"kind\":\"dev\",\"name\":\"serde_bytes\",\"req\":\"^0.11.10\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_stacker\",\"req\":\"^0.1.8\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{\"alloc\":[\"serde/alloc\"],\"arbitrary_precision\":[],\"default\":[\"std\"],\"float_roundtrip\":[],\"preserve_order\":[\"indexmap\",\"std\"],\"raw_value\":[],\"std\":[\"memchr/std\",\"serde/std\"],\"unbounded_depth\":[]}}", - "serde_json_1.0.149": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0.11\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.2.3\"},{\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2.0.2\"},{\"name\":\"itoa\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"kind\":\"dev\",\"name\":\"serde_bytes\",\"req\":\"^0.11.10\"},{\"default_features\":false,\"name\":\"serde_core\",\"req\":\"^1.0.220\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_stacker\",\"req\":\"^0.1.8\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"},{\"name\":\"zmij\",\"req\":\"^1.0\"}],\"features\":{\"alloc\":[\"serde_core/alloc\"],\"arbitrary_precision\":[],\"default\":[\"std\"],\"float_roundtrip\":[],\"preserve_order\":[\"indexmap\",\"std\"],\"raw_value\":[],\"std\":[\"memchr/std\",\"serde_core/std\"],\"unbounded_depth\":[]}}", - "shlex_1.3.0": "{\"dependencies\":[],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "signal-hook-registry_1.4.8": "{\"dependencies\":[{\"name\":\"errno\",\"req\":\">=0.2, <0.4\"},{\"name\":\"libc\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"signal-hook\",\"req\":\"~0.3\"}],\"features\":{}}", - "slab_0.4.12": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.95\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "smallvec_1.13.2": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.0.1\"},{\"kind\":\"dev\",\"name\":\"debugger_test\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"debugger_test_parser\",\"req\":\"^0.1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"const_generics\":[],\"const_new\":[\"const_generics\"],\"debugger_visualizer\":[],\"drain_filter\":[],\"drain_keep_rest\":[\"drain_filter\"],\"may_dangle\":[],\"specialization\":[],\"union\":[],\"write\":[]}}", - "smallvec_1.15.1": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"name\":\"bincode\",\"optional\":true,\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"bincode1\",\"package\":\"bincode\",\"req\":\"^1.0.1\"},{\"kind\":\"dev\",\"name\":\"debugger_test\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"debugger_test_parser\",\"req\":\"^0.1.0\"},{\"default_features\":false,\"name\":\"malloc_size_of\",\"optional\":true,\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"name\":\"unty\",\"optional\":true,\"req\":\"^0.0.4\"}],\"features\":{\"const_generics\":[],\"const_new\":[\"const_generics\"],\"debugger_visualizer\":[],\"drain_filter\":[],\"drain_keep_rest\":[\"drain_filter\"],\"impl_bincode\":[\"bincode\",\"unty\"],\"may_dangle\":[],\"specialization\":[],\"union\":[],\"write\":[]}}", - "socket2_0.5.10": "{\"dependencies\":[{\"name\":\"libc\",\"req\":\"^0.2.171\",\"target\":\"cfg(unix)\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\",\"Win32_System_IO\",\"Win32_System_Threading\",\"Win32_System_WindowsProgramming\"],\"name\":\"windows-sys\",\"req\":\"^0.52\",\"target\":\"cfg(windows)\"}],\"features\":{\"all\":[]}}", - "socket2_0.6.3": "{\"dependencies\":[{\"name\":\"libc\",\"req\":\"^0.2.172\",\"target\":\"cfg(any(unix, target_os = \\\"wasi\\\"))\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\",\"Win32_System_IO\",\"Win32_System_Threading\",\"Win32_System_WindowsProgramming\"],\"name\":\"windows-sys\",\"req\":\">=0.60, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"all\":[]}}", - "spm_precompiled_0.1.4": "{\"dependencies\":[{\"name\":\"base64\",\"req\":\"^0.13\"},{\"name\":\"nom\",\"req\":\"^7.1.1\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"unicode-segmentation\",\"req\":\"^1.9\"}],\"features\":{}}", - "static_assertions_1.1.0": "{\"dependencies\":[],\"features\":{\"nightly\":[]}}", - "strsim_0.11.1": "{\"dependencies\":[],\"features\":{}}", - "syn_2.0.117": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"insta\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.91\"},{\"default_features\":false,\"name\":\"quote\",\"optional\":true,\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1\"},{\"features\":[\"blocking\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.13\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"syn-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4.16\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"termcolor\",\"req\":\"^1\"},{\"name\":\"unicode-ident\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\",\"target\":\"cfg(not(miri))\"}],\"features\":{\"clone-impls\":[],\"default\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\"],\"derive\":[],\"extra-traits\":[],\"fold\":[],\"full\":[],\"parsing\":[],\"printing\":[\"dep:quote\"],\"proc-macro\":[\"proc-macro2/proc-macro\",\"quote?/proc-macro\"],\"test\":[\"syn-test-suite/all-features\"],\"visit\":[],\"visit-mut\":[]}}", - "syn_2.0.91": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"insta\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.91\"},{\"default_features\":false,\"name\":\"quote\",\"optional\":true,\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1\"},{\"features\":[\"blocking\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.12\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"syn-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4.16\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"termcolor\",\"req\":\"^1\"},{\"name\":\"unicode-ident\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\",\"target\":\"cfg(not(miri))\"}],\"features\":{\"clone-impls\":[],\"default\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\"],\"derive\":[],\"extra-traits\":[],\"fold\":[],\"full\":[],\"parsing\":[],\"printing\":[\"dep:quote\"],\"proc-macro\":[\"proc-macro2/proc-macro\",\"quote?/proc-macro\"],\"test\":[\"syn-test-suite/all-features\"],\"visit\":[],\"visit-mut\":[]}}", - "sync_wrapper_1.0.2": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"}],\"features\":{\"futures\":[\"futures-core\"]}}", - "tempfile_3.27.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"name\":\"fastrand\",\"req\":\"^2.1.1\"},{\"default_features\":false,\"name\":\"getrandom\",\"optional\":true,\"req\":\">=0.3.0, <0.5\",\"target\":\"cfg(any(unix, windows, target_os = \\\"wasi\\\"))\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"once_cell\",\"req\":\"^1.19.0\"},{\"features\":[\"fs\"],\"name\":\"rustix\",\"req\":\"^1.1.4\",\"target\":\"cfg(any(unix, target_os = \\\"wasi\\\"))\"},{\"features\":[\"Win32_Storage_FileSystem\",\"Win32_Foundation\"],\"name\":\"windows-sys\",\"req\":\">=0.52, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[\"getrandom\"],\"nightly\":[]}}", - "thiserror-impl_2.0.12": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"name\":\"syn\",\"req\":\"^2.0.87\"}],\"features\":{}}", - "thiserror_2.0.12": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.73\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"name\":\"thiserror-impl\",\"req\":\"=2.0.12\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "tokenizers_0.22.2": "{\"dependencies\":[{\"features\":[\"serde\"],\"name\":\"ahash\",\"req\":\"^0.8.11\"},{\"name\":\"aho-corasick\",\"req\":\"^1.1\"},{\"kind\":\"dev\",\"name\":\"assert_approx_eq\",\"req\":\"^1.1\"},{\"features\":[\"serde\"],\"name\":\"compact_str\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.6\"},{\"features\":[\"serde\"],\"name\":\"dary_heap\",\"req\":\"^0.3.6\"},{\"name\":\"derive_builder\",\"req\":\"^0.20\"},{\"default_features\":false,\"name\":\"esaxx-rs\",\"req\":\"^0.1.10\"},{\"name\":\"fancy-regex\",\"optional\":true,\"req\":\"^0.14\"},{\"name\":\"getrandom\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"ureq\"],\"name\":\"hf-hub\",\"optional\":true,\"req\":\"^0.4.1\"},{\"name\":\"indicatif\",\"optional\":true,\"req\":\"^0.18\"},{\"name\":\"itertools\",\"req\":\"^0.14\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"macro_rules_attribute\",\"req\":\"^0.2.0\"},{\"name\":\"monostate\",\"req\":\"^0.1.12\"},{\"default_features\":false,\"name\":\"onig\",\"optional\":true,\"req\":\"^6.5.1\"},{\"name\":\"paste\",\"req\":\"^1.0.14\"},{\"name\":\"rand\",\"req\":\"^0.9\"},{\"name\":\"rayon\",\"req\":\"^1.10\"},{\"name\":\"rayon-cond\",\"req\":\"^0.4\"},{\"name\":\"regex\",\"req\":\"^1.10\"},{\"name\":\"regex-syntax\",\"req\":\"^0.8\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"spm_precompiled\",\"req\":\"^0.1.3\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.10\"},{\"name\":\"thiserror\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3.18\"},{\"name\":\"unicode-normalization-alignments\",\"req\":\"^0.1\"},{\"name\":\"unicode-segmentation\",\"req\":\"^1.11\"},{\"name\":\"unicode_categories\",\"req\":\"^0.1\"}],\"features\":{\"default\":[\"progressbar\",\"onig\",\"esaxx_fast\"],\"esaxx_fast\":[\"esaxx-rs/cpp\"],\"http\":[\"hf-hub\"],\"progressbar\":[\"indicatif\"],\"rustls-tls\":[\"hf-hub?/rustls-tls\"],\"unstable_wasm\":[\"fancy-regex\",\"getrandom/wasm_js\"]}}", - "tokio-macros_2.4.0": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0.0\"}],\"features\":{}}", - "tokio-stream_0.1.15": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-stream\",\"req\":\"^0.3\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"name\":\"futures-core\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"parking_lot\",\"req\":\"^0.12.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.11\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"req\":\"^1.15.0\"},{\"features\":[\"full\",\"test-util\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.2.0\"},{\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7.0\"}],\"features\":{\"default\":[\"time\"],\"fs\":[\"tokio/fs\"],\"full\":[\"time\",\"net\",\"io-util\",\"fs\",\"sync\",\"signal\"],\"io-util\":[\"tokio/io-util\"],\"net\":[\"tokio/net\"],\"signal\":[\"tokio/signal\"],\"sync\":[\"tokio/sync\",\"tokio-util\"],\"time\":[\"tokio/time\"]}}", - "tokio-util_0.7.17": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-stream\",\"req\":\"^0.3.0\"},{\"name\":\"bytes\",\"req\":\"^1.5.0\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.0\"},{\"name\":\"futures-core\",\"req\":\"^0.3.0\"},{\"name\":\"futures-io\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"futures-sink\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"futures-test\",\"req\":\"^0.3.5\"},{\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.0\"},{\"default_features\":false,\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.15.0\"},{\"kind\":\"dev\",\"name\":\"parking_lot\",\"req\":\"^0.12.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.11\"},{\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4.4\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.1.0\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"req\":\"^1.28.0\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.29\"}],\"features\":{\"__docs_rs\":[\"futures-util\"],\"codec\":[],\"compat\":[\"futures-io\"],\"default\":[],\"full\":[\"codec\",\"compat\",\"io-util\",\"time\",\"net\",\"rt\",\"join-map\"],\"io\":[],\"io-util\":[\"io\",\"tokio/rt\",\"tokio/io-util\"],\"join-map\":[\"rt\",\"hashbrown\"],\"net\":[\"tokio/net\"],\"rt\":[\"tokio/rt\",\"tokio/sync\",\"futures-util\"],\"time\":[\"tokio/time\",\"slab\"]}}", - "tokio_1.39.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-stream\",\"req\":\"^0.3\"},{\"name\":\"backtrace\",\"req\":\"^0.3.58\",\"target\":\"cfg(tokio_taskdump)\"},{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1.0.0\"},{\"features\":[\"async-await\"],\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.0\"},{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.149\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.149\",\"target\":\"cfg(unix)\"},{\"features\":[\"futures\",\"checkpoint\"],\"kind\":\"dev\",\"name\":\"loom\",\"req\":\"^0.7\",\"target\":\"cfg(loom)\"},{\"default_features\":false,\"name\":\"mio\",\"optional\":true,\"req\":\"^1.0.1\"},{\"features\":[\"tokio\"],\"kind\":\"dev\",\"name\":\"mio-aio\",\"req\":\"^0.9.0\",\"target\":\"cfg(target_os = \\\"freebsd\\\")\"},{\"kind\":\"dev\",\"name\":\"mockall\",\"req\":\"^0.11.1\"},{\"default_features\":false,\"features\":[\"aio\",\"fs\",\"socket\"],\"kind\":\"dev\",\"name\":\"nix\",\"req\":\"^0.29.0\",\"target\":\"cfg(unix)\"},{\"name\":\"parking_lot\",\"optional\":true,\"req\":\"^0.12.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.11\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.0\",\"target\":\"cfg(not(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\")))\"},{\"name\":\"signal-hook-registry\",\"optional\":true,\"req\":\"^1.1.1\",\"target\":\"cfg(unix)\"},{\"features\":[\"all\"],\"name\":\"socket2\",\"optional\":true,\"req\":\"^0.5.5\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"kind\":\"dev\",\"name\":\"socket2\",\"req\":\"^0.5.5\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.1.0\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"name\":\"tokio-macros\",\"optional\":true,\"req\":\"~2.4.0\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.25\",\"target\":\"cfg(tokio_unstable)\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.0\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", not(target_os = \\\"wasi\\\")))\"},{\"name\":\"windows-sys\",\"optional\":true,\"req\":\"^0.52\",\"target\":\"cfg(windows)\"},{\"features\":[\"Win32_Foundation\",\"Win32_Security_Authorization\"],\"kind\":\"dev\",\"name\":\"windows-sys\",\"req\":\"^0.52\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[],\"fs\":[],\"full\":[\"fs\",\"io-util\",\"io-std\",\"macros\",\"net\",\"parking_lot\",\"process\",\"rt\",\"rt-multi-thread\",\"signal\",\"sync\",\"time\"],\"io-std\":[],\"io-util\":[\"bytes\"],\"macros\":[\"tokio-macros\"],\"net\":[\"libc\",\"mio/os-poll\",\"mio/os-ext\",\"mio/net\",\"socket2\",\"windows-sys/Win32_Foundation\",\"windows-sys/Win32_Security\",\"windows-sys/Win32_Storage_FileSystem\",\"windows-sys/Win32_System_Pipes\",\"windows-sys/Win32_System_SystemServices\"],\"process\":[\"bytes\",\"libc\",\"mio/os-poll\",\"mio/os-ext\",\"mio/net\",\"signal-hook-registry\",\"windows-sys/Win32_Foundation\",\"windows-sys/Win32_System_Threading\",\"windows-sys/Win32_System_WindowsProgramming\"],\"rt\":[],\"rt-multi-thread\":[\"rt\"],\"signal\":[\"libc\",\"mio/os-poll\",\"mio/net\",\"mio/os-ext\",\"signal-hook-registry\",\"windows-sys/Win32_Foundation\",\"windows-sys/Win32_System_Console\"],\"sync\":[],\"test-util\":[\"rt\",\"sync\",\"time\"],\"time\":[]}}", - "tonic-build_0.12.3": "{\"dependencies\":[{\"name\":\"prettyplease\",\"req\":\"^0.2\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"name\":\"prost-build\",\"optional\":true,\"req\":\"^0.13\"},{\"name\":\"prost-types\",\"optional\":true,\"req\":\"^0.13\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"name\":\"syn\",\"req\":\"^2.0\"}],\"features\":{\"cleanup-markdown\":[\"prost\",\"prost-build/cleanup-markdown\"],\"default\":[\"transport\",\"prost\"],\"prost\":[\"prost-build\",\"dep:prost-types\"],\"transport\":[]}}", - "tonic_0.12.1": "{\"dependencies\":[{\"name\":\"async-stream\",\"optional\":true,\"req\":\"^0.3\"},{\"name\":\"async-trait\",\"optional\":true,\"req\":\"^0.1.13\"},{\"default_features\":false,\"name\":\"axum\",\"optional\":true,\"req\":\"^0.7\"},{\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.5\"},{\"name\":\"bytes\",\"req\":\"^1.0\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"h2\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"http-body\",\"req\":\"^1\"},{\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"features\":[\"http1\",\"http2\"],\"name\":\"hyper\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"hyper-timeout\",\"optional\":true,\"req\":\"^0.5\"},{\"features\":[\"tokio\"],\"name\":\"hyper-util\",\"optional\":true,\"req\":\"^0.1.4\"},{\"name\":\"percent-encoding\",\"req\":\"^2.1\"},{\"name\":\"pin-project\",\"req\":\"^1.0.11\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"prost\",\"optional\":true,\"req\":\"^0.13\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"name\":\"rustls-native-certs\",\"optional\":true,\"req\":\"^0.7\"},{\"name\":\"rustls-pemfile\",\"optional\":true,\"req\":\"^2.0\"},{\"features\":[\"all\"],\"name\":\"socket2\",\"optional\":true,\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"rt\",\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"logging\",\"tls12\",\"ring\"],\"name\":\"tokio-rustls\",\"optional\":true,\"req\":\"^0.26\"},{\"default_features\":false,\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"tower\",\"optional\":true,\"req\":\"^0.4.7\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tower\",\"req\":\"^0.4.7\"},{\"name\":\"tower-layer\",\"req\":\"^0.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"name\":\"tracing\",\"req\":\"^0.1\"},{\"name\":\"webpki-roots\",\"optional\":true,\"req\":\"^0.26\"},{\"name\":\"zstd\",\"optional\":true,\"req\":\"^0.13.0\"}],\"features\":{\"channel\":[\"dep:hyper\",\"hyper?/client\",\"dep:hyper-util\",\"hyper-util?/client-legacy\",\"dep:tower\",\"tower?/balance\",\"tower?/buffer\",\"tower?/discover\",\"tower?/limit\",\"dep:tokio\",\"tokio?/time\",\"dep:hyper-timeout\"],\"codegen\":[\"dep:async-trait\"],\"default\":[\"transport\",\"codegen\",\"prost\"],\"gzip\":[\"dep:flate2\"],\"prost\":[\"dep:prost\"],\"router\":[\"dep:axum\",\"dep:tower\",\"tower?/util\"],\"server\":[\"router\",\"dep:async-stream\",\"dep:h2\",\"dep:hyper\",\"hyper?/server\",\"dep:hyper-util\",\"hyper-util?/service\",\"hyper-util?/server-auto\",\"dep:socket2\",\"dep:tokio\",\"tokio?/macros\",\"tokio?/net\",\"tokio?/time\",\"tokio-stream/net\",\"dep:tower\",\"tower?/util\",\"tower?/limit\"],\"tls\":[\"dep:rustls-pemfile\",\"dep:tokio-rustls\",\"dep:tokio\",\"tokio?/rt\",\"tokio?/macros\"],\"tls-roots\":[\"tls\",\"channel\",\"dep:rustls-native-certs\"],\"tls-webpki-roots\":[\"tls\",\"channel\",\"dep:webpki-roots\"],\"transport\":[\"server\",\"channel\"],\"zstd\":[\"dep:zstd\"]}}", - "tower-layer_0.3.3": "{\"dependencies\":[],\"features\":{}}", - "tower-service_0.3.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.22\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^0.2\"},{\"features\":[\"macros\",\"time\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.6.2\"},{\"kind\":\"dev\",\"name\":\"tower-layer\",\"req\":\"^0.3\"}],\"features\":{}}", - "tower_0.4.13": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"hdrhistogram\",\"optional\":true,\"req\":\"^7.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"hdrhistogram\",\"req\":\"^7.0\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^0.2\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^1.0.2\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4.0\"},{\"name\":\"pin-project\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"pin-project-lite\",\"optional\":true,\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"},{\"features\":[\"small_rng\"],\"name\":\"rand\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.6\"},{\"features\":[\"macros\",\"sync\",\"test-util\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.6.2\"},{\"name\":\"tokio-stream\",\"optional\":true,\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7.0\"},{\"name\":\"tower-layer\",\"req\":\"^0.3.1\"},{\"name\":\"tower-service\",\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"tower-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.2\"},{\"default_features\":false,\"features\":[\"fmt\",\"ansi\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"}],\"features\":{\"__common\":[\"futures-core\",\"pin-project-lite\"],\"balance\":[\"discover\",\"load\",\"ready-cache\",\"make\",\"rand\",\"slab\"],\"buffer\":[\"__common\",\"tokio/sync\",\"tokio/rt\",\"tokio-util\",\"tracing\"],\"default\":[\"log\"],\"discover\":[\"__common\"],\"filter\":[\"__common\",\"futures-util\"],\"full\":[\"balance\",\"buffer\",\"discover\",\"filter\",\"hedge\",\"limit\",\"load\",\"load-shed\",\"make\",\"ready-cache\",\"reconnect\",\"retry\",\"spawn-ready\",\"steer\",\"timeout\",\"util\"],\"hedge\":[\"util\",\"filter\",\"futures-util\",\"hdrhistogram\",\"tokio/time\",\"tracing\"],\"limit\":[\"__common\",\"tokio/time\",\"tokio/sync\",\"tokio-util\",\"tracing\"],\"load\":[\"__common\",\"tokio/time\",\"tracing\"],\"load-shed\":[\"__common\"],\"log\":[\"tracing/log\"],\"make\":[\"futures-util\",\"pin-project-lite\",\"tokio/io-std\"],\"ready-cache\":[\"futures-core\",\"futures-util\",\"indexmap\",\"tokio/sync\",\"tracing\",\"pin-project-lite\"],\"reconnect\":[\"make\",\"tokio/io-std\",\"tracing\"],\"retry\":[\"__common\",\"tokio/time\"],\"spawn-ready\":[\"__common\",\"futures-util\",\"tokio/sync\",\"tokio/rt\",\"util\",\"tracing\"],\"steer\":[],\"timeout\":[\"pin-project-lite\",\"tokio/time\"],\"util\":[\"__common\",\"futures-util\",\"pin-project\"]}}", - "tower_0.5.3": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.22\"},{\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3.22\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.22\"},{\"default_features\":false,\"features\":[\"async-await-macro\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3.22\"},{\"default_features\":false,\"name\":\"hdrhistogram\",\"optional\":true,\"req\":\"^7.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"hdrhistogram\",\"req\":\"^7.0\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.0.2\"},{\"name\":\"pin-project-lite\",\"optional\":true,\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9\"},{\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4.9\"},{\"name\":\"sync_wrapper\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.6.2\"},{\"features\":[\"macros\",\"sync\",\"test-util\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.6.2\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1.1\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7.0\"},{\"name\":\"tower-layer\",\"req\":\"^0.3.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3.3\"},{\"kind\":\"dev\",\"name\":\"tower-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.2\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1.2\"},{\"default_features\":false,\"features\":[\"fmt\",\"ansi\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"}],\"features\":{\"balance\":[\"discover\",\"load\",\"ready-cache\",\"make\",\"slab\",\"util\"],\"buffer\":[\"tokio/sync\",\"tokio/rt\",\"tokio-util\",\"tracing\",\"pin-project-lite\"],\"discover\":[\"futures-core\",\"pin-project-lite\"],\"filter\":[\"futures-util\",\"pin-project-lite\"],\"full\":[\"balance\",\"buffer\",\"discover\",\"filter\",\"hedge\",\"limit\",\"load\",\"load-shed\",\"make\",\"ready-cache\",\"reconnect\",\"retry\",\"spawn-ready\",\"steer\",\"timeout\",\"util\"],\"hedge\":[\"util\",\"filter\",\"futures-util\",\"hdrhistogram\",\"tokio/time\",\"tracing\"],\"limit\":[\"tokio/time\",\"tokio/sync\",\"tokio-util\",\"tracing\",\"pin-project-lite\"],\"load\":[\"tokio/time\",\"tracing\",\"pin-project-lite\"],\"load-shed\":[\"pin-project-lite\"],\"log\":[\"tracing/log\"],\"make\":[\"pin-project-lite\",\"tokio\"],\"ready-cache\":[\"futures-core\",\"futures-util\",\"indexmap\",\"tokio/sync\",\"tracing\",\"pin-project-lite\"],\"reconnect\":[\"make\",\"tracing\"],\"retry\":[\"tokio/time\",\"util\"],\"spawn-ready\":[\"futures-util\",\"tokio/sync\",\"tokio/rt\",\"util\",\"tracing\"],\"steer\":[],\"timeout\":[\"pin-project-lite\",\"tokio/time\"],\"tokio-stream\":[],\"util\":[\"futures-core\",\"futures-util\",\"pin-project-lite\",\"sync_wrapper\"]}}", - "tracing-attributes_0.1.31": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-trait\",\"req\":\"^0.1.67\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1.0.20\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.9\"},{\"default_features\":false,\"features\":[\"full\",\"parsing\",\"printing\",\"visit-mut\",\"clone-impls\",\"extra-traits\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4.2\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1.35\"},{\"features\":[\"env-filter\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.64\"}],\"features\":{\"async-await\":[]}}", - "tracing-core_0.1.36": "{\"dependencies\":[{\"name\":\"once_cell\",\"optional\":true,\"req\":\"^1.13.0\"},{\"default_features\":false,\"name\":\"valuable\",\"optional\":true,\"req\":\"^0.1.0\",\"target\":\"cfg(tracing_unstable)\"}],\"features\":{\"default\":[\"std\",\"valuable?/std\"],\"std\":[\"once_cell\"]}}", - "tracing_0.1.44": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.6\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.21\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4.17\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.9\"},{\"name\":\"tracing-attributes\",\"optional\":true,\"req\":\"^0.1.31\"},{\"default_features\":false,\"name\":\"tracing-core\",\"req\":\"^0.1.36\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.38\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"}],\"features\":{\"async-await\":[],\"attributes\":[\"tracing-attributes\"],\"default\":[\"std\",\"attributes\"],\"log-always\":[\"log\"],\"max_level_debug\":[],\"max_level_error\":[],\"max_level_info\":[],\"max_level_off\":[],\"max_level_trace\":[],\"max_level_warn\":[],\"release_max_level_debug\":[],\"release_max_level_error\":[],\"release_max_level_info\":[],\"release_max_level_off\":[],\"release_max_level_trace\":[],\"release_max_level_warn\":[],\"std\":[\"tracing-core/std\"],\"valuable\":[\"tracing-core/valuable\"]}}", - "try-lock_0.2.5": "{\"dependencies\":[],\"features\":{}}", - "unicode-ident_1.0.14": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"fst\",\"req\":\"^0.4\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"roaring\",\"req\":\"^0.10\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"ucd-trie\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"unicode-xid\",\"req\":\"^0.2.6\"}],\"features\":{}}", - "unicode-ident_1.0.24": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"fst\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"roaring\",\"req\":\"^0.11\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"ucd-trie\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"unicode-xid\",\"req\":\"^0.2.6\"}],\"features\":{}}", - "unicode-normalization-alignments_0.1.12": "{\"dependencies\":[{\"name\":\"smallvec\",\"req\":\"^1.0\"}],\"features\":{}}", - "unicode-segmentation_1.12.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.7\"}],\"features\":{\"no_std\":[]}}", - "unicode-xid_0.2.6": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3\"}],\"features\":{\"bench\":[],\"default\":[],\"no_std\":[]}}", - "unicode_categories_0.1.1": "{\"dependencies\":[],\"features\":{}}", - "version_check_0.9.5": "{\"dependencies\":[],\"features\":{}}", - "want_0.3.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"tokio-executor\",\"req\":\"^0.2.0-alpha.2\"},{\"kind\":\"dev\",\"name\":\"tokio-sync\",\"req\":\"^0.2.0-alpha.2\"},{\"name\":\"try-lock\",\"req\":\"^0.2.4\"}],\"features\":{}}", - "wasi_0.11.1+wasi-snapshot-preview1": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"rustc-dep-of-std\":[\"core\",\"rustc-std-workspace-alloc\"],\"std\":[]}}", - "wasi_0.14.2+wasi-0.2.4": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"bitflags\"],\"name\":\"wit-bindgen-rt\",\"req\":\"^0.39.0\"}],\"features\":{\"default\":[\"std\"],\"rustc-dep-of-std\":[\"compiler_builtins\",\"core\",\"rustc-std-workspace-alloc\"],\"std\":[]}}", - "wasip2_1.0.2+wasi-0.2.9": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"wit-bindgen\",\"req\":\"^0.51.0\"}],\"features\":{\"bitflags\":[\"wit-bindgen/bitflags\"],\"default\":[\"std\",\"bitflags\"],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"wit-bindgen/rustc-dep-of-std\"],\"std\":[]}}", - "wasip3_0.4.0+wasi-0.3.0-rc-2026-01-06": "{\"dependencies\":[{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1.10.1\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.31\"},{\"name\":\"http\",\"optional\":true,\"req\":\"^1.3.1\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^1.3.1\"},{\"name\":\"http-body\",\"optional\":true,\"req\":\"^1.0.1\"},{\"name\":\"thiserror\",\"optional\":true,\"req\":\"^2.0.17\"},{\"default_features\":false,\"features\":[\"async\"],\"name\":\"wit-bindgen\",\"req\":\"^0.51.0\"},{\"default_features\":false,\"features\":[\"async-spawn\"],\"kind\":\"dev\",\"name\":\"wit-bindgen\",\"req\":\"^0.51.0\"}],\"features\":{\"http-compat\":[\"dep:bytes\",\"dep:http-body\",\"dep:http\",\"dep:thiserror\",\"wit-bindgen/async-spawn\"]}}", - "wasm-encoder_0.244.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"default_features\":false,\"name\":\"leb128fmt\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.2.0\"},{\"default_features\":false,\"features\":[\"simd\",\"simd\"],\"name\":\"wasmparser\",\"optional\":true,\"req\":\"^0.244.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"wasmprinter\",\"req\":\"^0.244.0\"}],\"features\":{\"component-model\":[\"wasmparser?/component-model\"],\"default\":[\"std\",\"component-model\"],\"std\":[\"wasmparser?/std\"]}}", - "wasm-metadata_0.244.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"name\":\"auditable-serde\",\"optional\":true,\"req\":\"^0.8.0\"},{\"features\":[\"derive\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4.0.0\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1.1.0\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"indexmap\",\"req\":\"^2.7.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"spdx\",\"optional\":true,\"req\":\"^0.10.1\"},{\"name\":\"url\",\"optional\":true,\"req\":\"^2.0.0\"},{\"default_features\":false,\"features\":[\"std\",\"component-model\"],\"name\":\"wasm-encoder\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"simd\",\"std\",\"component-model\",\"hash-collections\"],\"name\":\"wasmparser\",\"req\":\"^0.244.0\"}],\"features\":{\"default\":[\"oci\",\"serde\"],\"oci\":[\"dep:auditable-serde\",\"dep:flate2\",\"dep:url\",\"dep:spdx\",\"dep:serde_json\",\"serde\"],\"serde\":[\"dep:serde_derive\",\"dep:serde\"]}}", - "wasmparser_0.244.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"name\":\"bitflags\",\"req\":\"^2.4.1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5.1\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"default-hasher\"],\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.15.2\"},{\"default_features\":false,\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.7.0\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.13.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.3\"},{\"default_features\":false,\"name\":\"semver\",\"optional\":true,\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.166\"}],\"features\":{\"component-model\":[\"dep:semver\"],\"default\":[\"std\",\"validate\",\"serde\",\"features\",\"component-model\",\"hash-collections\",\"simd\"],\"features\":[],\"hash-collections\":[\"dep:hashbrown\",\"dep:indexmap\"],\"prefer-btree-collections\":[],\"serde\":[\"dep:serde\",\"indexmap?/serde\",\"hashbrown?/serde\"],\"simd\":[],\"std\":[\"indexmap?/std\"],\"validate\":[]}}", - "windows-link_0.2.1": "{\"dependencies\":[],\"features\":{}}", - "windows-sys_0.52.0": "{\"dependencies\":[{\"name\":\"windows-targets\",\"req\":\"^0.52.0\"}],\"features\":{\"Wdk\":[],\"Wdk_Foundation\":[\"Wdk\"],\"Wdk_Graphics\":[\"Wdk\"],\"Wdk_Graphics_Direct3D\":[\"Wdk_Graphics\"],\"Wdk_Storage\":[\"Wdk\"],\"Wdk_Storage_FileSystem\":[\"Wdk_Storage\"],\"Wdk_Storage_FileSystem_Minifilters\":[\"Wdk_Storage_FileSystem\"],\"Wdk_System\":[\"Wdk\"],\"Wdk_System_IO\":[\"Wdk_System\"],\"Wdk_System_OfflineRegistry\":[\"Wdk_System\"],\"Wdk_System_Registry\":[\"Wdk_System\"],\"Wdk_System_SystemInformation\":[\"Wdk_System\"],\"Wdk_System_SystemServices\":[\"Wdk_System\"],\"Wdk_System_Threading\":[\"Wdk_System\"],\"Win32\":[],\"Win32_Data\":[\"Win32\"],\"Win32_Data_HtmlHelp\":[\"Win32_Data\"],\"Win32_Data_RightsManagement\":[\"Win32_Data\"],\"Win32_Devices\":[\"Win32\"],\"Win32_Devices_AllJoyn\":[\"Win32_Devices\"],\"Win32_Devices_BiometricFramework\":[\"Win32_Devices\"],\"Win32_Devices_Bluetooth\":[\"Win32_Devices\"],\"Win32_Devices_Communication\":[\"Win32_Devices\"],\"Win32_Devices_DeviceAndDriverInstallation\":[\"Win32_Devices\"],\"Win32_Devices_DeviceQuery\":[\"Win32_Devices\"],\"Win32_Devices_Display\":[\"Win32_Devices\"],\"Win32_Devices_Enumeration\":[\"Win32_Devices\"],\"Win32_Devices_Enumeration_Pnp\":[\"Win32_Devices_Enumeration\"],\"Win32_Devices_Fax\":[\"Win32_Devices\"],\"Win32_Devices_HumanInterfaceDevice\":[\"Win32_Devices\"],\"Win32_Devices_PortableDevices\":[\"Win32_Devices\"],\"Win32_Devices_Properties\":[\"Win32_Devices\"],\"Win32_Devices_Pwm\":[\"Win32_Devices\"],\"Win32_Devices_Sensors\":[\"Win32_Devices\"],\"Win32_Devices_SerialCommunication\":[\"Win32_Devices\"],\"Win32_Devices_Tapi\":[\"Win32_Devices\"],\"Win32_Devices_Usb\":[\"Win32_Devices\"],\"Win32_Devices_WebServicesOnDevices\":[\"Win32_Devices\"],\"Win32_Foundation\":[\"Win32\"],\"Win32_Gaming\":[\"Win32\"],\"Win32_Globalization\":[\"Win32\"],\"Win32_Graphics\":[\"Win32\"],\"Win32_Graphics_Dwm\":[\"Win32_Graphics\"],\"Win32_Graphics_Gdi\":[\"Win32_Graphics\"],\"Win32_Graphics_GdiPlus\":[\"Win32_Graphics\"],\"Win32_Graphics_Hlsl\":[\"Win32_Graphics\"],\"Win32_Graphics_OpenGL\":[\"Win32_Graphics\"],\"Win32_Graphics_Printing\":[\"Win32_Graphics\"],\"Win32_Graphics_Printing_PrintTicket\":[\"Win32_Graphics_Printing\"],\"Win32_Management\":[\"Win32\"],\"Win32_Management_MobileDeviceManagementRegistration\":[\"Win32_Management\"],\"Win32_Media\":[\"Win32\"],\"Win32_Media_Audio\":[\"Win32_Media\"],\"Win32_Media_DxMediaObjects\":[\"Win32_Media\"],\"Win32_Media_KernelStreaming\":[\"Win32_Media\"],\"Win32_Media_Multimedia\":[\"Win32_Media\"],\"Win32_Media_Streaming\":[\"Win32_Media\"],\"Win32_Media_WindowsMediaFormat\":[\"Win32_Media\"],\"Win32_NetworkManagement\":[\"Win32\"],\"Win32_NetworkManagement_Dhcp\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Dns\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_InternetConnectionWizard\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_IpHelper\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Multicast\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Ndis\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetBios\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetManagement\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetShell\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetworkDiagnosticsFramework\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_P2P\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_QoS\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Rras\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Snmp\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WNet\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WebDav\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WiFi\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsConnectionManager\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsFilteringPlatform\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsFirewall\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsNetworkVirtualization\":[\"Win32_NetworkManagement\"],\"Win32_Networking\":[\"Win32\"],\"Win32_Networking_ActiveDirectory\":[\"Win32_Networking\"],\"Win32_Networking_Clustering\":[\"Win32_Networking\"],\"Win32_Networking_HttpServer\":[\"Win32_Networking\"],\"Win32_Networking_Ldap\":[\"Win32_Networking\"],\"Win32_Networking_WebSocket\":[\"Win32_Networking\"],\"Win32_Networking_WinHttp\":[\"Win32_Networking\"],\"Win32_Networking_WinInet\":[\"Win32_Networking\"],\"Win32_Networking_WinSock\":[\"Win32_Networking\"],\"Win32_Networking_WindowsWebServices\":[\"Win32_Networking\"],\"Win32_Security\":[\"Win32\"],\"Win32_Security_AppLocker\":[\"Win32_Security\"],\"Win32_Security_Authentication\":[\"Win32_Security\"],\"Win32_Security_Authentication_Identity\":[\"Win32_Security_Authentication\"],\"Win32_Security_Authorization\":[\"Win32_Security\"],\"Win32_Security_Credentials\":[\"Win32_Security\"],\"Win32_Security_Cryptography\":[\"Win32_Security\"],\"Win32_Security_Cryptography_Catalog\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_Certificates\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_Sip\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_UI\":[\"Win32_Security_Cryptography\"],\"Win32_Security_DiagnosticDataQuery\":[\"Win32_Security\"],\"Win32_Security_DirectoryServices\":[\"Win32_Security\"],\"Win32_Security_EnterpriseData\":[\"Win32_Security\"],\"Win32_Security_ExtensibleAuthenticationProtocol\":[\"Win32_Security\"],\"Win32_Security_Isolation\":[\"Win32_Security\"],\"Win32_Security_LicenseProtection\":[\"Win32_Security\"],\"Win32_Security_NetworkAccessProtection\":[\"Win32_Security\"],\"Win32_Security_WinTrust\":[\"Win32_Security\"],\"Win32_Security_WinWlx\":[\"Win32_Security\"],\"Win32_Storage\":[\"Win32\"],\"Win32_Storage_Cabinets\":[\"Win32_Storage\"],\"Win32_Storage_CloudFilters\":[\"Win32_Storage\"],\"Win32_Storage_Compression\":[\"Win32_Storage\"],\"Win32_Storage_DistributedFileSystem\":[\"Win32_Storage\"],\"Win32_Storage_FileHistory\":[\"Win32_Storage\"],\"Win32_Storage_FileSystem\":[\"Win32_Storage\"],\"Win32_Storage_Imapi\":[\"Win32_Storage\"],\"Win32_Storage_IndexServer\":[\"Win32_Storage\"],\"Win32_Storage_InstallableFileSystems\":[\"Win32_Storage\"],\"Win32_Storage_IscsiDisc\":[\"Win32_Storage\"],\"Win32_Storage_Jet\":[\"Win32_Storage\"],\"Win32_Storage_Nvme\":[\"Win32_Storage\"],\"Win32_Storage_OfflineFiles\":[\"Win32_Storage\"],\"Win32_Storage_OperationRecorder\":[\"Win32_Storage\"],\"Win32_Storage_Packaging\":[\"Win32_Storage\"],\"Win32_Storage_Packaging_Appx\":[\"Win32_Storage_Packaging\"],\"Win32_Storage_ProjectedFileSystem\":[\"Win32_Storage\"],\"Win32_Storage_StructuredStorage\":[\"Win32_Storage\"],\"Win32_Storage_Vhd\":[\"Win32_Storage\"],\"Win32_Storage_Xps\":[\"Win32_Storage\"],\"Win32_System\":[\"Win32\"],\"Win32_System_AddressBook\":[\"Win32_System\"],\"Win32_System_Antimalware\":[\"Win32_System\"],\"Win32_System_ApplicationInstallationAndServicing\":[\"Win32_System\"],\"Win32_System_ApplicationVerifier\":[\"Win32_System\"],\"Win32_System_ClrHosting\":[\"Win32_System\"],\"Win32_System_Com\":[\"Win32_System\"],\"Win32_System_Com_Marshal\":[\"Win32_System_Com\"],\"Win32_System_Com_StructuredStorage\":[\"Win32_System_Com\"],\"Win32_System_Com_Urlmon\":[\"Win32_System_Com\"],\"Win32_System_ComponentServices\":[\"Win32_System\"],\"Win32_System_Console\":[\"Win32_System\"],\"Win32_System_CorrelationVector\":[\"Win32_System\"],\"Win32_System_DataExchange\":[\"Win32_System\"],\"Win32_System_DeploymentServices\":[\"Win32_System\"],\"Win32_System_DeveloperLicensing\":[\"Win32_System\"],\"Win32_System_Diagnostics\":[\"Win32_System\"],\"Win32_System_Diagnostics_Ceip\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_Debug\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_Debug_Extensions\":[\"Win32_System_Diagnostics_Debug\"],\"Win32_System_Diagnostics_Etw\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_ProcessSnapshotting\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_ToolHelp\":[\"Win32_System_Diagnostics\"],\"Win32_System_DistributedTransactionCoordinator\":[\"Win32_System\"],\"Win32_System_Environment\":[\"Win32_System\"],\"Win32_System_ErrorReporting\":[\"Win32_System\"],\"Win32_System_EventCollector\":[\"Win32_System\"],\"Win32_System_EventLog\":[\"Win32_System\"],\"Win32_System_EventNotificationService\":[\"Win32_System\"],\"Win32_System_GroupPolicy\":[\"Win32_System\"],\"Win32_System_HostCompute\":[\"Win32_System\"],\"Win32_System_HostComputeNetwork\":[\"Win32_System\"],\"Win32_System_HostComputeSystem\":[\"Win32_System\"],\"Win32_System_Hypervisor\":[\"Win32_System\"],\"Win32_System_IO\":[\"Win32_System\"],\"Win32_System_Iis\":[\"Win32_System\"],\"Win32_System_Ioctl\":[\"Win32_System\"],\"Win32_System_JobObjects\":[\"Win32_System\"],\"Win32_System_Js\":[\"Win32_System\"],\"Win32_System_Kernel\":[\"Win32_System\"],\"Win32_System_LibraryLoader\":[\"Win32_System\"],\"Win32_System_Mailslots\":[\"Win32_System\"],\"Win32_System_Mapi\":[\"Win32_System\"],\"Win32_System_Memory\":[\"Win32_System\"],\"Win32_System_Memory_NonVolatile\":[\"Win32_System_Memory\"],\"Win32_System_MessageQueuing\":[\"Win32_System\"],\"Win32_System_MixedReality\":[\"Win32_System\"],\"Win32_System_Ole\":[\"Win32_System\"],\"Win32_System_PasswordManagement\":[\"Win32_System\"],\"Win32_System_Performance\":[\"Win32_System\"],\"Win32_System_Performance_HardwareCounterProfiling\":[\"Win32_System_Performance\"],\"Win32_System_Pipes\":[\"Win32_System\"],\"Win32_System_Power\":[\"Win32_System\"],\"Win32_System_ProcessStatus\":[\"Win32_System\"],\"Win32_System_Recovery\":[\"Win32_System\"],\"Win32_System_Registry\":[\"Win32_System\"],\"Win32_System_RemoteDesktop\":[\"Win32_System\"],\"Win32_System_RemoteManagement\":[\"Win32_System\"],\"Win32_System_RestartManager\":[\"Win32_System\"],\"Win32_System_Restore\":[\"Win32_System\"],\"Win32_System_Rpc\":[\"Win32_System\"],\"Win32_System_Search\":[\"Win32_System\"],\"Win32_System_Search_Common\":[\"Win32_System_Search\"],\"Win32_System_SecurityCenter\":[\"Win32_System\"],\"Win32_System_Services\":[\"Win32_System\"],\"Win32_System_SetupAndMigration\":[\"Win32_System\"],\"Win32_System_Shutdown\":[\"Win32_System\"],\"Win32_System_StationsAndDesktops\":[\"Win32_System\"],\"Win32_System_SubsystemForLinux\":[\"Win32_System\"],\"Win32_System_SystemInformation\":[\"Win32_System\"],\"Win32_System_SystemServices\":[\"Win32_System\"],\"Win32_System_Threading\":[\"Win32_System\"],\"Win32_System_Time\":[\"Win32_System\"],\"Win32_System_TpmBaseServices\":[\"Win32_System\"],\"Win32_System_UserAccessLogging\":[\"Win32_System\"],\"Win32_System_Variant\":[\"Win32_System\"],\"Win32_System_VirtualDosMachines\":[\"Win32_System\"],\"Win32_System_WindowsProgramming\":[\"Win32_System\"],\"Win32_System_Wmi\":[\"Win32_System\"],\"Win32_UI\":[\"Win32\"],\"Win32_UI_Accessibility\":[\"Win32_UI\"],\"Win32_UI_ColorSystem\":[\"Win32_UI\"],\"Win32_UI_Controls\":[\"Win32_UI\"],\"Win32_UI_Controls_Dialogs\":[\"Win32_UI_Controls\"],\"Win32_UI_HiDpi\":[\"Win32_UI\"],\"Win32_UI_Input\":[\"Win32_UI\"],\"Win32_UI_Input_Ime\":[\"Win32_UI_Input\"],\"Win32_UI_Input_KeyboardAndMouse\":[\"Win32_UI_Input\"],\"Win32_UI_Input_Pointer\":[\"Win32_UI_Input\"],\"Win32_UI_Input_Touch\":[\"Win32_UI_Input\"],\"Win32_UI_Input_XboxController\":[\"Win32_UI_Input\"],\"Win32_UI_InteractionContext\":[\"Win32_UI\"],\"Win32_UI_Magnification\":[\"Win32_UI\"],\"Win32_UI_Shell\":[\"Win32_UI\"],\"Win32_UI_Shell_PropertiesSystem\":[\"Win32_UI_Shell\"],\"Win32_UI_TabletPC\":[\"Win32_UI\"],\"Win32_UI_TextServices\":[\"Win32_UI\"],\"Win32_UI_WindowsAndMessaging\":[\"Win32_UI\"],\"Win32_Web\":[\"Win32\"],\"Win32_Web_InternetExplorer\":[\"Win32_Web\"],\"default\":[],\"docs\":[]}}", - "windows-sys_0.61.2": "{\"dependencies\":[{\"default_features\":false,\"name\":\"windows-link\",\"req\":\"^0.2.1\"}],\"features\":{\"Wdk\":[\"Win32_Foundation\"],\"Wdk_Devices\":[\"Wdk\"],\"Wdk_Devices_Bluetooth\":[\"Wdk_Devices\"],\"Wdk_Devices_HumanInterfaceDevice\":[\"Wdk_Devices\"],\"Wdk_Foundation\":[\"Wdk\"],\"Wdk_Graphics\":[\"Wdk\"],\"Wdk_Graphics_Direct3D\":[\"Wdk_Graphics\"],\"Wdk_NetworkManagement\":[\"Wdk\"],\"Wdk_NetworkManagement_Ndis\":[\"Wdk_NetworkManagement\"],\"Wdk_NetworkManagement_WindowsFilteringPlatform\":[\"Wdk_NetworkManagement\"],\"Wdk_Storage\":[\"Wdk\"],\"Wdk_Storage_FileSystem\":[\"Wdk_Storage\"],\"Wdk_Storage_FileSystem_Minifilters\":[\"Wdk_Storage_FileSystem\"],\"Wdk_System\":[\"Wdk\"],\"Wdk_System_IO\":[\"Wdk_System\"],\"Wdk_System_Memory\":[\"Wdk_System\"],\"Wdk_System_OfflineRegistry\":[\"Wdk_System\"],\"Wdk_System_Registry\":[\"Wdk_System\"],\"Wdk_System_SystemInformation\":[\"Wdk_System\"],\"Wdk_System_SystemServices\":[\"Wdk_System\"],\"Wdk_System_Threading\":[\"Wdk_System\"],\"Win32\":[\"Win32_Foundation\"],\"Win32_Data\":[\"Win32\"],\"Win32_Data_HtmlHelp\":[\"Win32_Data\"],\"Win32_Data_RightsManagement\":[\"Win32_Data\"],\"Win32_Devices\":[\"Win32\"],\"Win32_Devices_AllJoyn\":[\"Win32_Devices\"],\"Win32_Devices_Beep\":[\"Win32_Devices\"],\"Win32_Devices_BiometricFramework\":[\"Win32_Devices\"],\"Win32_Devices_Bluetooth\":[\"Win32_Devices\"],\"Win32_Devices_Cdrom\":[\"Win32_Devices\"],\"Win32_Devices_Communication\":[\"Win32_Devices\"],\"Win32_Devices_DeviceAndDriverInstallation\":[\"Win32_Devices\"],\"Win32_Devices_DeviceQuery\":[\"Win32_Devices\"],\"Win32_Devices_Display\":[\"Win32_Devices\"],\"Win32_Devices_Dvd\":[\"Win32_Devices\"],\"Win32_Devices_Enumeration\":[\"Win32_Devices\"],\"Win32_Devices_Enumeration_Pnp\":[\"Win32_Devices_Enumeration\"],\"Win32_Devices_Fax\":[\"Win32_Devices\"],\"Win32_Devices_HumanInterfaceDevice\":[\"Win32_Devices\"],\"Win32_Devices_Nfc\":[\"Win32_Devices\"],\"Win32_Devices_Nfp\":[\"Win32_Devices\"],\"Win32_Devices_PortableDevices\":[\"Win32_Devices\"],\"Win32_Devices_Properties\":[\"Win32_Devices\"],\"Win32_Devices_Pwm\":[\"Win32_Devices\"],\"Win32_Devices_Sensors\":[\"Win32_Devices\"],\"Win32_Devices_SerialCommunication\":[\"Win32_Devices\"],\"Win32_Devices_Tapi\":[\"Win32_Devices\"],\"Win32_Devices_Usb\":[\"Win32_Devices\"],\"Win32_Devices_WebServicesOnDevices\":[\"Win32_Devices\"],\"Win32_Foundation\":[\"Win32\"],\"Win32_Gaming\":[\"Win32\"],\"Win32_Globalization\":[\"Win32\"],\"Win32_Graphics\":[\"Win32\"],\"Win32_Graphics_Dwm\":[\"Win32_Graphics\"],\"Win32_Graphics_Gdi\":[\"Win32_Graphics\"],\"Win32_Graphics_GdiPlus\":[\"Win32_Graphics\"],\"Win32_Graphics_Hlsl\":[\"Win32_Graphics\"],\"Win32_Graphics_OpenGL\":[\"Win32_Graphics\"],\"Win32_Graphics_Printing\":[\"Win32_Graphics\"],\"Win32_Graphics_Printing_PrintTicket\":[\"Win32_Graphics_Printing\"],\"Win32_Management\":[\"Win32\"],\"Win32_Management_MobileDeviceManagementRegistration\":[\"Win32_Management\"],\"Win32_Media\":[\"Win32\"],\"Win32_Media_Audio\":[\"Win32_Media\"],\"Win32_Media_DxMediaObjects\":[\"Win32_Media\"],\"Win32_Media_KernelStreaming\":[\"Win32_Media\"],\"Win32_Media_Multimedia\":[\"Win32_Media\"],\"Win32_Media_Streaming\":[\"Win32_Media\"],\"Win32_Media_WindowsMediaFormat\":[\"Win32_Media\"],\"Win32_NetworkManagement\":[\"Win32\"],\"Win32_NetworkManagement_Dhcp\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Dns\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_InternetConnectionWizard\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_IpHelper\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Multicast\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Ndis\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetBios\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetManagement\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetShell\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_NetworkDiagnosticsFramework\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_P2P\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_QoS\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Rras\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_Snmp\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WNet\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WebDav\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WiFi\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsConnectionManager\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsFilteringPlatform\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsFirewall\":[\"Win32_NetworkManagement\"],\"Win32_NetworkManagement_WindowsNetworkVirtualization\":[\"Win32_NetworkManagement\"],\"Win32_Networking\":[\"Win32\"],\"Win32_Networking_ActiveDirectory\":[\"Win32_Networking\"],\"Win32_Networking_Clustering\":[\"Win32_Networking\"],\"Win32_Networking_HttpServer\":[\"Win32_Networking\"],\"Win32_Networking_Ldap\":[\"Win32_Networking\"],\"Win32_Networking_WebSocket\":[\"Win32_Networking\"],\"Win32_Networking_WinHttp\":[\"Win32_Networking\"],\"Win32_Networking_WinInet\":[\"Win32_Networking\"],\"Win32_Networking_WinSock\":[\"Win32_Networking\"],\"Win32_Networking_WindowsWebServices\":[\"Win32_Networking\"],\"Win32_Security\":[\"Win32\"],\"Win32_Security_AppLocker\":[\"Win32_Security\"],\"Win32_Security_Authentication\":[\"Win32_Security\"],\"Win32_Security_Authentication_Identity\":[\"Win32_Security_Authentication\"],\"Win32_Security_Authorization\":[\"Win32_Security\"],\"Win32_Security_Credentials\":[\"Win32_Security\"],\"Win32_Security_Cryptography\":[\"Win32_Security\"],\"Win32_Security_Cryptography_Catalog\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_Certificates\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_Sip\":[\"Win32_Security_Cryptography\"],\"Win32_Security_Cryptography_UI\":[\"Win32_Security_Cryptography\"],\"Win32_Security_DiagnosticDataQuery\":[\"Win32_Security\"],\"Win32_Security_DirectoryServices\":[\"Win32_Security\"],\"Win32_Security_EnterpriseData\":[\"Win32_Security\"],\"Win32_Security_ExtensibleAuthenticationProtocol\":[\"Win32_Security\"],\"Win32_Security_Isolation\":[\"Win32_Security\"],\"Win32_Security_LicenseProtection\":[\"Win32_Security\"],\"Win32_Security_NetworkAccessProtection\":[\"Win32_Security\"],\"Win32_Security_WinTrust\":[\"Win32_Security\"],\"Win32_Security_WinWlx\":[\"Win32_Security\"],\"Win32_Storage\":[\"Win32\"],\"Win32_Storage_Cabinets\":[\"Win32_Storage\"],\"Win32_Storage_CloudFilters\":[\"Win32_Storage\"],\"Win32_Storage_Compression\":[\"Win32_Storage\"],\"Win32_Storage_DistributedFileSystem\":[\"Win32_Storage\"],\"Win32_Storage_FileHistory\":[\"Win32_Storage\"],\"Win32_Storage_FileSystem\":[\"Win32_Storage\"],\"Win32_Storage_Imapi\":[\"Win32_Storage\"],\"Win32_Storage_IndexServer\":[\"Win32_Storage\"],\"Win32_Storage_InstallableFileSystems\":[\"Win32_Storage\"],\"Win32_Storage_IscsiDisc\":[\"Win32_Storage\"],\"Win32_Storage_Jet\":[\"Win32_Storage\"],\"Win32_Storage_Nvme\":[\"Win32_Storage\"],\"Win32_Storage_OfflineFiles\":[\"Win32_Storage\"],\"Win32_Storage_OperationRecorder\":[\"Win32_Storage\"],\"Win32_Storage_Packaging\":[\"Win32_Storage\"],\"Win32_Storage_Packaging_Appx\":[\"Win32_Storage_Packaging\"],\"Win32_Storage_ProjectedFileSystem\":[\"Win32_Storage\"],\"Win32_Storage_StructuredStorage\":[\"Win32_Storage\"],\"Win32_Storage_Vhd\":[\"Win32_Storage\"],\"Win32_Storage_Xps\":[\"Win32_Storage\"],\"Win32_System\":[\"Win32\"],\"Win32_System_AddressBook\":[\"Win32_System\"],\"Win32_System_Antimalware\":[\"Win32_System\"],\"Win32_System_ApplicationInstallationAndServicing\":[\"Win32_System\"],\"Win32_System_ApplicationVerifier\":[\"Win32_System\"],\"Win32_System_ClrHosting\":[\"Win32_System\"],\"Win32_System_Com\":[\"Win32_System\"],\"Win32_System_Com_Marshal\":[\"Win32_System_Com\"],\"Win32_System_Com_StructuredStorage\":[\"Win32_System_Com\"],\"Win32_System_Com_Urlmon\":[\"Win32_System_Com\"],\"Win32_System_ComponentServices\":[\"Win32_System\"],\"Win32_System_Console\":[\"Win32_System\"],\"Win32_System_CorrelationVector\":[\"Win32_System\"],\"Win32_System_DataExchange\":[\"Win32_System\"],\"Win32_System_DeploymentServices\":[\"Win32_System\"],\"Win32_System_DeveloperLicensing\":[\"Win32_System\"],\"Win32_System_Diagnostics\":[\"Win32_System\"],\"Win32_System_Diagnostics_Ceip\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_Debug\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_Debug_Extensions\":[\"Win32_System_Diagnostics_Debug\"],\"Win32_System_Diagnostics_Etw\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_ProcessSnapshotting\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_ToolHelp\":[\"Win32_System_Diagnostics\"],\"Win32_System_Diagnostics_TraceLogging\":[\"Win32_System_Diagnostics\"],\"Win32_System_DistributedTransactionCoordinator\":[\"Win32_System\"],\"Win32_System_Environment\":[\"Win32_System\"],\"Win32_System_ErrorReporting\":[\"Win32_System\"],\"Win32_System_EventCollector\":[\"Win32_System\"],\"Win32_System_EventLog\":[\"Win32_System\"],\"Win32_System_EventNotificationService\":[\"Win32_System\"],\"Win32_System_GroupPolicy\":[\"Win32_System\"],\"Win32_System_HostCompute\":[\"Win32_System\"],\"Win32_System_HostComputeNetwork\":[\"Win32_System\"],\"Win32_System_HostComputeSystem\":[\"Win32_System\"],\"Win32_System_Hypervisor\":[\"Win32_System\"],\"Win32_System_IO\":[\"Win32_System\"],\"Win32_System_Iis\":[\"Win32_System\"],\"Win32_System_Ioctl\":[\"Win32_System\"],\"Win32_System_JobObjects\":[\"Win32_System\"],\"Win32_System_Js\":[\"Win32_System\"],\"Win32_System_Kernel\":[\"Win32_System\"],\"Win32_System_LibraryLoader\":[\"Win32_System\"],\"Win32_System_Mailslots\":[\"Win32_System\"],\"Win32_System_Mapi\":[\"Win32_System\"],\"Win32_System_Memory\":[\"Win32_System\"],\"Win32_System_Memory_NonVolatile\":[\"Win32_System_Memory\"],\"Win32_System_MessageQueuing\":[\"Win32_System\"],\"Win32_System_MixedReality\":[\"Win32_System\"],\"Win32_System_Ole\":[\"Win32_System\"],\"Win32_System_PasswordManagement\":[\"Win32_System\"],\"Win32_System_Performance\":[\"Win32_System\"],\"Win32_System_Performance_HardwareCounterProfiling\":[\"Win32_System_Performance\"],\"Win32_System_Pipes\":[\"Win32_System\"],\"Win32_System_Power\":[\"Win32_System\"],\"Win32_System_ProcessStatus\":[\"Win32_System\"],\"Win32_System_Recovery\":[\"Win32_System\"],\"Win32_System_Registry\":[\"Win32_System\"],\"Win32_System_RemoteDesktop\":[\"Win32_System\"],\"Win32_System_RemoteManagement\":[\"Win32_System\"],\"Win32_System_RestartManager\":[\"Win32_System\"],\"Win32_System_Restore\":[\"Win32_System\"],\"Win32_System_Rpc\":[\"Win32_System\"],\"Win32_System_Search\":[\"Win32_System\"],\"Win32_System_Search_Common\":[\"Win32_System_Search\"],\"Win32_System_SecurityCenter\":[\"Win32_System\"],\"Win32_System_Services\":[\"Win32_System\"],\"Win32_System_SetupAndMigration\":[\"Win32_System\"],\"Win32_System_Shutdown\":[\"Win32_System\"],\"Win32_System_StationsAndDesktops\":[\"Win32_System\"],\"Win32_System_SubsystemForLinux\":[\"Win32_System\"],\"Win32_System_SystemInformation\":[\"Win32_System\"],\"Win32_System_SystemServices\":[\"Win32_System\"],\"Win32_System_Threading\":[\"Win32_System\"],\"Win32_System_Time\":[\"Win32_System\"],\"Win32_System_TpmBaseServices\":[\"Win32_System\"],\"Win32_System_UserAccessLogging\":[\"Win32_System\"],\"Win32_System_Variant\":[\"Win32_System\"],\"Win32_System_VirtualDosMachines\":[\"Win32_System\"],\"Win32_System_WindowsProgramming\":[\"Win32_System\"],\"Win32_System_Wmi\":[\"Win32_System\"],\"Win32_UI\":[\"Win32\"],\"Win32_UI_Accessibility\":[\"Win32_UI\"],\"Win32_UI_ColorSystem\":[\"Win32_UI\"],\"Win32_UI_Controls\":[\"Win32_UI\"],\"Win32_UI_Controls_Dialogs\":[\"Win32_UI_Controls\"],\"Win32_UI_HiDpi\":[\"Win32_UI\"],\"Win32_UI_Input\":[\"Win32_UI\"],\"Win32_UI_Input_Ime\":[\"Win32_UI_Input\"],\"Win32_UI_Input_KeyboardAndMouse\":[\"Win32_UI_Input\"],\"Win32_UI_Input_Pointer\":[\"Win32_UI_Input\"],\"Win32_UI_Input_Touch\":[\"Win32_UI_Input\"],\"Win32_UI_Input_XboxController\":[\"Win32_UI_Input\"],\"Win32_UI_InteractionContext\":[\"Win32_UI\"],\"Win32_UI_Magnification\":[\"Win32_UI\"],\"Win32_UI_Shell\":[\"Win32_UI\"],\"Win32_UI_Shell_Common\":[\"Win32_UI_Shell\"],\"Win32_UI_Shell_PropertiesSystem\":[\"Win32_UI_Shell\"],\"Win32_UI_TabletPC\":[\"Win32_UI\"],\"Win32_UI_TextServices\":[\"Win32_UI\"],\"Win32_UI_WindowsAndMessaging\":[\"Win32_UI\"],\"Win32_Web\":[\"Win32\"],\"Win32_Web_InternetExplorer\":[\"Win32_Web\"],\"default\":[],\"docs\":[]}}", - "windows-targets_0.52.6": "{\"dependencies\":[{\"name\":\"windows_aarch64_gnullvm\",\"req\":\"^0.52.6\",\"target\":\"aarch64-pc-windows-gnullvm\"},{\"name\":\"windows_aarch64_msvc\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(target_arch = \\\"aarch64\\\", target_env = \\\"msvc\\\", not(windows_raw_dylib)))\"},{\"name\":\"windows_i686_gnu\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(target_arch = \\\"x86\\\", target_env = \\\"gnu\\\", not(target_abi = \\\"llvm\\\"), not(windows_raw_dylib)))\"},{\"name\":\"windows_i686_gnullvm\",\"req\":\"^0.52.6\",\"target\":\"i686-pc-windows-gnullvm\"},{\"name\":\"windows_i686_msvc\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(target_arch = \\\"x86\\\", target_env = \\\"msvc\\\", not(windows_raw_dylib)))\"},{\"name\":\"windows_x86_64_gnu\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(target_arch = \\\"x86_64\\\", target_env = \\\"gnu\\\", not(target_abi = \\\"llvm\\\"), not(windows_raw_dylib)))\"},{\"name\":\"windows_x86_64_gnullvm\",\"req\":\"^0.52.6\",\"target\":\"x86_64-pc-windows-gnullvm\"},{\"name\":\"windows_x86_64_msvc\",\"req\":\"^0.52.6\",\"target\":\"cfg(all(any(target_arch = \\\"x86_64\\\", target_arch = \\\"arm64ec\\\"), target_env = \\\"msvc\\\", not(windows_raw_dylib)))\"}],\"features\":{}}", - "windows_aarch64_gnullvm_0.52.6": "{\"dependencies\":[],\"features\":{}}", - "windows_aarch64_msvc_0.52.6": "{\"dependencies\":[],\"features\":{}}", - "windows_i686_gnu_0.52.6": "{\"dependencies\":[],\"features\":{}}", - "windows_i686_gnullvm_0.52.6": "{\"dependencies\":[],\"features\":{}}", - "windows_i686_msvc_0.52.6": "{\"dependencies\":[],\"features\":{}}", - "windows_x86_64_gnu_0.52.6": "{\"dependencies\":[],\"features\":{}}", - "windows_x86_64_gnullvm_0.52.6": "{\"dependencies\":[],\"features\":{}}", - "windows_x86_64_msvc_0.52.6": "{\"dependencies\":[],\"features\":{}}", - "wit-bindgen-core_0.51.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.72\"},{\"features\":[\"derive\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4.3.19\"},{\"name\":\"heck\",\"req\":\"^0.5\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.218\"},{\"name\":\"wit-parser\",\"req\":\"^0.244.0\"}],\"features\":{\"clap\":[\"dep:clap\"],\"serde\":[\"dep:serde\"]}}", - "wit-bindgen-rt_0.39.0": "{\"dependencies\":[{\"name\":\"bitflags\",\"optional\":true,\"req\":\"^2.3.3\"},{\"name\":\"futures\",\"optional\":true,\"req\":\"^0.3.30\"},{\"name\":\"once_cell\",\"optional\":true,\"req\":\"^1.19.0\"}],\"features\":{\"async\":[\"dep:futures\",\"dep:once_cell\"]}}", - "wit-bindgen-rust-macro_0.51.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.72\"},{\"name\":\"prettyplease\",\"req\":\"^0.2.20\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"printing\"],\"name\":\"syn\",\"req\":\"^2.0.89\"},{\"name\":\"wit-bindgen-core\",\"req\":\"^0.51.0\"},{\"name\":\"wit-bindgen-rust\",\"req\":\"^0.51.0\"}],\"features\":{\"async\":[]}}", - "wit-bindgen-rust_0.51.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.72\"},{\"kind\":\"dev\",\"name\":\"bytes\",\"req\":\"^1\"},{\"features\":[\"derive\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4.3.19\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.31\"},{\"name\":\"heck\",\"req\":\"^0.5\"},{\"name\":\"indexmap\",\"req\":\"^2.0.0\"},{\"name\":\"prettyplease\",\"req\":\"^0.2.20\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.218\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"features\":[\"printing\"],\"name\":\"syn\",\"req\":\"^2.0.89\"},{\"default_features\":false,\"name\":\"wasm-metadata\",\"req\":\"^0.244.0\"},{\"name\":\"wit-bindgen-core\",\"req\":\"^0.51.0\"},{\"name\":\"wit-component\",\"req\":\"^0.244.0\"}],\"features\":{\"clap\":[\"dep:clap\",\"wit-bindgen-core/clap\"],\"serde\":[\"dep:serde\",\"wit-bindgen-core/serde\"]}}", - "wit-bindgen_0.51.0": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0\"},{\"name\":\"bitflags\",\"optional\":true,\"req\":\"^2.3.3\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"futures\",\"optional\":true,\"req\":\"^0.3.30\"},{\"name\":\"wit-bindgen-rust-macro\",\"optional\":true,\"req\":\"^0.51.0\"}],\"features\":{\"async\":[\"std\",\"wit-bindgen-rust-macro?/async\"],\"async-spawn\":[\"async\",\"dep:futures\"],\"bitflags\":[\"dep:bitflags\"],\"default\":[\"macros\",\"realloc\",\"async\",\"std\",\"bitflags\"],\"inter-task-wakeup\":[\"async\"],\"macros\":[\"dep:wit-bindgen-rust-macro\"],\"realloc\":[],\"rustc-dep-of-std\":[\"dep:core\",\"dep:alloc\"],\"std\":[]}}", - "wit-component_0.244.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"name\":\"bitflags\",\"req\":\"^2.3.3\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"kind\":\"dev\",\"name\":\"glob\",\"req\":\"^0.3.0\"},{\"default_features\":false,\"name\":\"indexmap\",\"req\":\"^2.7.0\"},{\"kind\":\"dev\",\"name\":\"libtest-mimic\",\"req\":\"^0.8.1\"},{\"name\":\"log\",\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.3.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"req\":\"^1.0.166\"},{\"name\":\"serde_derive\",\"req\":\"^1.0.166\"},{\"name\":\"serde_json\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"std\",\"wasmparser\"],\"name\":\"wasm-encoder\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"name\":\"wasm-metadata\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"oci\"],\"kind\":\"dev\",\"name\":\"wasm-metadata\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"simd\",\"std\",\"component-model\",\"simd\"],\"name\":\"wasmparser\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"simd\",\"std\",\"component-model\",\"features\"],\"kind\":\"dev\",\"name\":\"wasmparser\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"component-model\"],\"kind\":\"dev\",\"name\":\"wasmprinter\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"cranelift\",\"component-model\",\"runtime\",\"gc-drc\"],\"kind\":\"dev\",\"name\":\"wasmtime\",\"req\":\"^34.0.1\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"default_features\":false,\"name\":\"wast\",\"optional\":true,\"req\":\"^244.0.0\"},{\"default_features\":false,\"name\":\"wat\",\"optional\":true,\"req\":\"^1.244.0\"},{\"default_features\":false,\"features\":[\"component-model\"],\"kind\":\"dev\",\"name\":\"wat\",\"req\":\"^1.244.0\"},{\"features\":[\"decoding\",\"serde\"],\"name\":\"wit-parser\",\"req\":\"^0.244.0\"}],\"features\":{\"dummy-module\":[\"dep:wat\"],\"semver-check\":[\"dummy-module\"],\"wat\":[\"dep:wast\",\"dep:wat\"]}}", - "wit-parser_0.244.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"id-arena\",\"req\":\"^2\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"indexmap\",\"req\":\"^2.7.0\"},{\"kind\":\"dev\",\"name\":\"libtest-mimic\",\"req\":\"^0.8.1\"},{\"name\":\"log\",\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.3.0\"},{\"default_features\":false,\"name\":\"semver\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"name\":\"unicode-xid\",\"req\":\"^0.2.2\"},{\"default_features\":false,\"features\":[\"simd\",\"std\",\"validate\",\"component-model\",\"features\"],\"name\":\"wasmparser\",\"optional\":true,\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"component-model\"],\"name\":\"wat\",\"optional\":true,\"req\":\"^1.244.0\"}],\"features\":{\"decoding\":[\"dep:wasmparser\"],\"default\":[\"serde\",\"decoding\"],\"serde\":[\"dep:serde\",\"dep:serde_derive\",\"indexmap/serde\",\"serde_json\"],\"wat\":[\"decoding\",\"dep:wat\"]}}", - "zerocopy-derive_0.7.35": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.10\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"name\":\"syn\",\"req\":\"^2.0.31\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.85\"}],\"features\":{}}", - "zerocopy-derive_0.8.26": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"dissimilar\",\"req\":\"^1.0.9\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"=0.2.163\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"=1.9\"},{\"kind\":\"dev\",\"name\":\"prettyplease\",\"req\":\"=0.2.17\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.10\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.89\"}],\"features\":{}}", - "zerocopy-derive_0.8.42": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"dissimilar\",\"req\":\"^1.0.9\"},{\"kind\":\"dev\",\"name\":\"prettyplease\",\"req\":\"=0.2.17\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.40\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"features\":[\"visit\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.46\"}],\"features\":{}}", - "zerocopy_0.7.35": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"assert_matches\",\"req\":\"^1.5\"},{\"default_features\":false,\"name\":\"byteorder\",\"optional\":true,\"req\":\"^1.3\"},{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.85\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.7.35\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.7.35\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.7.35\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\"],\"alloc\":[],\"default\":[\"byteorder\"],\"derive\":[\"zerocopy-derive\"],\"simd\":[],\"simd-nightly\":[\"simd\"]}}", - "zerocopy_0.8.26": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"either\",\"req\":\"=1.13.0\"},{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.89\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.8.26\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\",\"std\"],\"alloc\":[],\"derive\":[\"zerocopy-derive\"],\"float-nightly\":[],\"simd\":[],\"simd-nightly\":[\"simd\"],\"std\":[\"alloc\"]}}", - "zerocopy_0.8.42": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.8.42\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.8.42\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.8.42\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\",\"std\"],\"alloc\":[],\"derive\":[\"zerocopy-derive\"],\"float-nightly\":[],\"simd\":[],\"simd-nightly\":[\"simd\"],\"std\":[\"alloc\"]}}", - "zmij_1.0.21": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\",\"target\":\"cfg(not(miri))\"},{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1.36\"},{\"kind\":\"dev\",\"name\":\"num-bigint\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"num-integer\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"num_cpus\",\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"opt-level\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"ryu\",\"req\":\"^1\"}],\"features\":{}}" - }, - "@@rules_rs+//rs/experimental/toolchains:module_extension.bzl%toolchains": { - "cargo-1.94.0-aarch64-apple-darwin.tar.xz": "7497ab873993cf38b806548c85bf818ffd71d1c3cf77b5bb4c3d3f1769588a08", - "cargo-1.94.0-aarch64-pc-windows-msvc.tar.xz": "0eacb6af29cacb850af4b390e698b2c8060f639a70e6b9f0575cdcc82b0822b2", - "cargo-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "d57ab7b23b52cba46e58c78327904d0c7db19892988677b0d59495c8af4bb790", - "cargo-1.94.0-x86_64-apple-darwin.tar.xz": "3beb7874010ebd757780284c0c9da47fc9f0b18091d7415d7b7231a8c19ec44f", - "cargo-1.94.0-x86_64-pc-windows-msvc.tar.xz": "6cf7113e1d5633721739a3bd6fee7148d6394bbab748fbede941210100374741", - "cargo-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "8e17624f3de39e079845bfb25ed15a042f4b50ceca78e37c56c4b9b15949b9f7", - "clippy-1.94.0-aarch64-apple-darwin.tar.xz": "42a683cbac0b81b96ad3b887cdade36a57ef3a131c203583ca7e3ce0be98f415", - "clippy-1.94.0-aarch64-pc-windows-msvc.tar.xz": "44468016e27457640c7ab62b7ceaddb44308e8ec77ee4344c1bd36113448c8eb", - "clippy-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "f7450c414e394450b53be602d37c374e2a2c5a2f51be2b54faf3d1e3272160b1", - "clippy-1.94.0-x86_64-apple-darwin.tar.xz": "1393cc688bc4d784a9d66619d2eec683a1ddb30b263f03ab615c0a2b18053126", - "clippy-1.94.0-x86_64-pc-windows-msvc.tar.xz": "e4dcc5a796857589e364064493abb3afb469e0e36e90155e7b0a2c5ce4bb7503", - "clippy-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "935529b0b9150084fa9cc1734c6598efe86b6e7d76f31ad3f8d4bb79ebf6031a", - "rust-std-1.94.0-aarch64-apple-darwin.tar.xz": "d3293668128b0a7838ef2544ae341eadd4437460ca22098af723349885a10ba2", - "rust-std-1.94.0-aarch64-apple-ios-macabi.tar.xz": "914f085065539ea1038d2832a5af1c22852cd5e9a063356b20e46698e72e770e", - "rust-std-1.94.0-aarch64-apple-ios-sim.tar.xz": "b4d16842b8f98ff2f431d6ab4aced298f4028ca00f0f5d588bc6efc94faa0680", - "rust-std-1.94.0-aarch64-apple-ios.tar.xz": "41ec1d18f4762f85231a19cf8aba3385da78fece2ff900a435c62881a3a3b44b", - "rust-std-1.94.0-aarch64-linux-android.tar.xz": "402c80df672a4b9fc948b6dc8bde21a63e1093eb1c8a2c2bd42221c309d9eba2", - "rust-std-1.94.0-aarch64-pc-windows-gnullvm.tar.xz": "8c89521f6826a1d2bf07b27c95405f4d6bf7c24f3de832c02758c41d4c1a511f", - "rust-std-1.94.0-aarch64-pc-windows-msvc.tar.xz": "d7964debd444dca92e771c953c3cd582fb927031b536c1439e1a831014d5f0d1", - "rust-std-1.94.0-aarch64-unknown-fuchsia.tar.xz": "87e11d97055540a0675bdc3dacf3d3d8cca5f806171ad0b51d0427f7cb9510c9", - "rust-std-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "c781b3ef4fefa5508fbe05820eddc95e46351d905a30921cc020febd9c596a2e", - "rust-std-1.94.0-aarch64-unknown-linux-musl.tar.xz": "3513032b2953d795bd3ac1489de4e8174e72500a31b4e7d1b349a23ab78a7a7b", - "rust-std-1.94.0-aarch64-unknown-none-softfloat.tar.xz": "375c496af31cf208ce67613c0aa1bde1894b5d674b00050cd46f7f954ed4a668", - "rust-std-1.94.0-aarch64-unknown-none.tar.xz": "854a9f8cea0ba889f553a7ea664c2a6479fc1fc49a4b56f84f99fc37001c4322", - "rust-std-1.94.0-aarch64-unknown-uefi.tar.xz": "f038d9e3df557d4aecdcf65989896179bcf0ecbc4d62bde71d33d7b914d3c253", - "rust-std-1.94.0-arm-linux-androideabi.tar.xz": "d2113ef50ed0b974f33b41102b961588c26abaf34f45a289a293b3d31ba01113", - "rust-std-1.94.0-arm-unknown-linux-gnueabi.tar.xz": "ad364b54b668e866315900ebff0d225616c556ab5228599792470aede05ba42c", - "rust-std-1.94.0-arm-unknown-linux-gnueabihf.tar.xz": "d5db943974fe05306ae43d100b0b7f3540108b93ef52de670675ce966af8a0fd", - "rust-std-1.94.0-arm-unknown-linux-musleabi.tar.xz": "c724b1a1b5dddeca563cda9b9695610bb56e681727b3af0a352174960087069a", - "rust-std-1.94.0-arm-unknown-linux-musleabihf.tar.xz": "9f8b707760a632b80864e91ea81a1b19c3f5452ff5e18b96da75792be1cf0a3c", - "rust-std-1.94.0-armv7-linux-androideabi.tar.xz": "e0b6883364f939300c281d96d6778ecf7fd97fe867b11936f0aa5c9a29f488ad", - "rust-std-1.94.0-armv7-unknown-linux-gnueabi.tar.xz": "9b1d8143881d510785d89413ea09b9fe910b64e3f300ab30e52e7fdd652ea66a", - "rust-std-1.94.0-armv7-unknown-linux-gnueabihf.tar.xz": "4b1744a8e48f58c2741afc1e076d0629a86051d785bd36e2c97f329eab6a5b49", - "rust-std-1.94.0-armv7-unknown-linux-musleabi.tar.xz": "fc2754f50ece0c7d3d32b2e0034b19c8b8bba9b471e0eb00f8ece7fa0bcb2837", - "rust-std-1.94.0-armv7-unknown-linux-musleabihf.tar.xz": "abfc0e7c1eaff1e3332b5c6262d12da309035a228261d07155add44464ed52f4", - "rust-std-1.94.0-i686-linux-android.tar.xz": "9093a894d87568a4b5eb4a144758bf49d4b62743677f747848afc3807c23f126", - "rust-std-1.94.0-i686-pc-windows-gnu.tar.xz": "188490bde6649163347050a70fd81c6d9e2e972ef12ba646f7f4fa106e21a956", - "rust-std-1.94.0-i686-pc-windows-gnullvm.tar.xz": "ba61dc038c3f4102a832a3381935192e88b0a1f79ff50b6d5590d476bdfa6879", - "rust-std-1.94.0-i686-pc-windows-msvc.tar.xz": "388cc2bb17ef9f12864c08e462cdbe3222d5ddafd84969dcf4ed944da27efcbd", - "rust-std-1.94.0-i686-unknown-freebsd.tar.xz": "3ce634092bcecb000c793c9d4bb0ccc6fd45c66fa8259fbb4f18c59844c4b13b", - "rust-std-1.94.0-i686-unknown-linux-gnu.tar.xz": "528c1dda491b25bb855426f48757773f4740c75d3c7271efffe7412c2fd5730d", - "rust-std-1.94.0-i686-unknown-linux-musl.tar.xz": "90ff16d879f774048c9b81d07188af4b63ed73a31848759cf560a9f51baa175f", - "rust-std-1.94.0-i686-unknown-uefi.tar.xz": "8c095069ec09d6e1d0b7e00f340fd7ea925a31acbad845d5956e1bc4a7d47733", - "rust-std-1.94.0-powerpc-unknown-linux-gnu.tar.xz": "23e32cab484d7787b7966a0508c49baf231bcdb4a79a421ee85ca69e81f2a06f", - "rust-std-1.94.0-powerpc64-unknown-linux-gnu.tar.xz": "7713b676a37fbd0a52a711c2e3c205a917f8b05614e17cd9f7f37874a88469a3", - "rust-std-1.94.0-powerpc64le-unknown-linux-gnu.tar.xz": "094f3955828d6630b70a9c7c71e3165c3ffc7a301929601dde08501def623347", - "rust-std-1.94.0-powerpc64le-unknown-linux-musl.tar.xz": "f3956b29463ec58d730f054ff2334755f929c602181ad67a25af9725c8500f68", - "rust-std-1.94.0-riscv32imc-unknown-none-elf.tar.xz": "5d10c7ab74cd5d10451480ec894ac5b98940a8527c333465eeec4a45b1125d43", - "rust-std-1.94.0-riscv64gc-unknown-linux-gnu.tar.xz": "32e9b7006f1269e48fb65af6298f6e8a222e1a8a2a20b112528bfb3dad2f65a8", - "rust-std-1.94.0-riscv64gc-unknown-linux-musl.tar.xz": "2c63d21143c7f1d44f3e3b1c9a82f3101576344c3fd1feb2d43ffc9d035e94f1", - "rust-std-1.94.0-riscv64gc-unknown-none-elf.tar.xz": "071a1ae54f7dead1b8147b9bae7d831a5554a37a1133f8cd52b265a765595a12", - "rust-std-1.94.0-s390x-unknown-linux-gnu.tar.xz": "ea69833addbc9521664464418c7468871f7a80b16fd2c385f6213a6c99cedfe9", - "rust-std-1.94.0-thumbv6m-none-eabi.tar.xz": "b022df94f6efa3bd2a6c4042472e114d2325bf058983b3ac7d586e24c8d77f25", - "rust-std-1.94.0-thumbv7em-none-eabi.tar.xz": "41894db05c4d8f421bf0dba7d0bed995685032393779f92b4ea1f3e6468badfb", - "rust-std-1.94.0-thumbv7em-none-eabihf.tar.xz": "be7e81535e7118fbad58a5dc31a1acfa7f2a36a2348aed23595169418559d67f", - "rust-std-1.94.0-thumbv7m-none-eabi.tar.xz": "4702e615be264df97b120fd075bed4e46710cc9a03c48115ce2ee3abfbf3822e", - "rust-std-1.94.0-thumbv8m.main-none-eabi.tar.xz": "5c1fb5ed8ad31580241b6e095583a2cac0ef90ea29dad7705c8a0ce0933a7797", - "rust-std-1.94.0-thumbv8m.main-none-eabihf.tar.xz": "573e9570bea039316ae3b55e323cbc741617ace1e0b87df8f3ecbbdf34b3de8a", - "rust-std-1.94.0-wasm32-unknown-emscripten.tar.xz": "0a607ba5d78afb47e9f5f134633ed5a954ece5558298e081084abe44f5265463", - "rust-std-1.94.0-wasm32-unknown-unknown.tar.xz": "bc65af3c0bfa2e5b20e299358710a26ce33a3674c762a2a010bc383c5b7fc202", - "rust-std-1.94.0-wasm32-wasip1-threads.tar.xz": "311b9fe1e94c90c16695899e670f0d76a366d493cdda29008191ce2d436cd22c", - "rust-std-1.94.0-wasm32-wasip1.tar.xz": "acbacacdca3e0ec0d93ff46d5f3579ae8cd5133b86e31a893da22cd94e28d0bf", - "rust-std-1.94.0-wasm32-wasip2.tar.xz": "e947e730da3fea212453b2835765802d9b0c80e19405cab188a6854834196e6c", - "rust-std-1.94.0-x86_64-apple-darwin.tar.xz": "cf716b5a5fab007398ce76536a85a3449df074b66e7c762c0f5de1d6c56a1342", - "rust-std-1.94.0-x86_64-apple-ios-macabi.tar.xz": "f39f4c1121dafcf8ea70a21c91be9552e30ea76debcad9bff28be945aaa79282", - "rust-std-1.94.0-x86_64-apple-ios.tar.xz": "60e70b80469dd3900dd8b1705831f5f02ac44fa97088b8a81fdda110ed421091", - "rust-std-1.94.0-x86_64-linux-android.tar.xz": "488c8a581612ac4779a05f05967b76a96e0ff0bdd1ab8d99c13a859c19056b01", - "rust-std-1.94.0-x86_64-pc-windows-gnu.tar.xz": "c9f98d62213d0559c8d2ee6d7eafbafbf5dfe04921c9e7cb15b66a770903bc33", - "rust-std-1.94.0-x86_64-pc-windows-gnullvm.tar.xz": "54c28d0030fab585457a431c555e5d23502132364d0483e5b3b2dfbfc3929cc6", - "rust-std-1.94.0-x86_64-pc-windows-msvc.tar.xz": "f8e2b43e52957e8224d97de68d6ec5da01613eb98bce4199de80071678b5fda0", - "rust-std-1.94.0-x86_64-unknown-freebsd.tar.xz": "081972f2f386ee859c15ed619695a9440896a80d1814d0e29d7620b7ef96207c", - "rust-std-1.94.0-x86_64-unknown-fuchsia.tar.xz": "6c18112553cf72cf991352208cce2bc96c712ca7b209acdd098abac894d674ef", - "rust-std-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "dd33653107c36e040082050d9e547e64dac5b456ba74069430d838c00c189a05", - "rust-std-1.94.0-x86_64-unknown-linux-musl.tar.xz": "b6786b5c490df2dcef76829039177799058f94e17725b42f912a057c0f6224f2", - "rust-std-1.94.0-x86_64-unknown-netbsd.tar.xz": "f763e56ca1ce3f819fb2136638d774777dd696f836cc52e6564431859a0f5159", - "rust-std-1.94.0-x86_64-unknown-none.tar.xz": "8509489165a8aa60a0590aeaa668f63423b3e7a33bb93e8e5719ff5404b6f6aa", - "rust-std-1.94.0-x86_64-unknown-uefi.tar.xz": "31b56e8178dd2319e51ccc474959d247db2767cda7149fe3704d636df9aee1c9", - "rustc-1.94.0-aarch64-apple-darwin.tar.xz": "6ba00a4f486af11826335cd7fe405592d959f50afb9489176fc1463cbe75dbf2", - "rustc-1.94.0-aarch64-pc-windows-msvc.tar.xz": "6d88d89c200a147357ded65e9bc426e14ebb6f5046d9772f3916b6a1ac60cdd5", - "rustc-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "7261b2fcdb88aef8b21f4b9f915d9a75c959674193f5b38e13fab4569684fc5b", - "rustc-1.94.0-x86_64-apple-darwin.tar.xz": "b1374b0fb63f5abcd8eb65607e43a499bfdc5e8160746d1f6f993aee9313f320", - "rustc-1.94.0-x86_64-pc-windows-msvc.tar.xz": "e0f59503778b563d3ee6d4edecd52b44056779ffe456181bad1fb4dc069802e6", - "rustc-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "31a0d3ac9383dfdeb4fce86eeed5ade3230131c635264c0eab7252dbf235f28e", - "rustfmt-1.94.0-aarch64-apple-darwin.tar.xz": "0eb716d201dc0d6d24ef8207d3ef27611d9d15d2212e3b27aa11e97286ed02ac", - "rustfmt-1.94.0-aarch64-pc-windows-msvc.tar.xz": "705ac3e797c4abed9e722e0864902f7a03872a8a201dcd7b70ff1b749eed2327", - "rustfmt-1.94.0-aarch64-unknown-linux-gnu.tar.xz": "1ac17dbe7f8f6c98db5b62cb149fcd10c7565d624a59471115fc40055db9af28", - "rustfmt-1.94.0-x86_64-apple-darwin.tar.xz": "e58601a10ee85e956611ea2814bf26de88934a8224c1663bb11aa23f727663ec", - "rustfmt-1.94.0-x86_64-pc-windows-msvc.tar.xz": "7b32d8b0dd99f9bf6ddaa4866d39b7b96e8c2db46644d8527096f7f34b4fbe26", - "rustfmt-1.94.0-x86_64-unknown-linux-gnu.tar.xz": "903e768f7626f78ed2d6cb82c1820b93bd339afec8575102e847fd3a379d33e5" - } - } + "facts": {} } diff --git a/kernels/common/BUILD.bazel b/kernels/common/BUILD.bazel new file mode 100644 index 00000000..11151f04 --- /dev/null +++ b/kernels/common/BUILD.bazel @@ -0,0 +1,22 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "common", + srcs = [ + "args.zig", + "control_flow.zig", + ], + import_name = "kernels/common", + main = "dsl.zig", + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "//mlir/dialects", + "//stdx", + ], +) + +zig_test( + name = "test", + deps = [":common"], +) diff --git a/kernels/common/args.zig b/kernels/common/args.zig new file mode 100644 index 00000000..6d89de34 --- /dev/null +++ b/kernels/common/args.zig @@ -0,0 +1,6 @@ +pub fn NamedArgs(comptime Spec: type, comptime ValueT: type) type { + const in = @typeInfo(Spec).@"struct".fields; + comptime var names: [in.len][]const u8 = undefined; + for (in, 0..) |f, i| names[i] = f.name; + return @Struct(.auto, null, &names, &@splat(ValueT), &@splat(.{})); +} diff --git a/kernels/common/control_flow.zig b/kernels/common/control_flow.zig new file mode 100644 index 00000000..2fdfffbd --- /dev/null +++ b/kernels/common/control_flow.zig @@ -0,0 +1,188 @@ +//! `scf` scope wrappers, generic over the frontend's `BuilderT` and `ValueT`. +//! +//! `BuilderT` must expose `ctx`, `loc()`, `pushBlock`, `popBlock`, `currentBlock`. +//! `ValueT` must have an `inner: *const mlir.Value` field and a `kernel: ?*BuilderT` field. + +const std = @import("std"); +const mlir = @import("mlir"); +const dialects = @import("mlir/dialects"); + +const scf = dialects.scf; + +pub fn tupleArity(comptime T: type, comptime what: []const u8) comptime_int { + const info = @typeInfo(T); + if (info != .@"struct" or !info.@"struct".is_tuple) + @compileError(what ++ " must be a tuple literal like `.{ v1, v2 }`"); + return info.@"struct".fields.len; +} + +pub fn emitScfYield( + comptime ValueT: type, + kernel: anytype, + block: *mlir.Block, + comptime N: usize, + values: anytype, + comptime what: []const u8, +) void { + const info = @typeInfo(@TypeOf(values)); + if (info != .@"struct" or !info.@"struct".is_tuple) + @compileError(what ++ " expects a tuple literal"); + if (info.@"struct".fields.len != N) + @compileError(what ++ ": yield arity must match the scope's declared arity"); + var buf: [N]*const mlir.Value = undefined; + inline for (info.@"struct".fields, 0..) |f, i| { + if (f.type != ValueT) + @compileError(what ++ ": every tuple element must be a Value"); + buf[i] = @field(values, f.name).inner; + } + _ = scf.yield(kernel.ctx, &buf, kernel.loc()).appendTo(block); +} + +pub fn ForScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize) type { + return struct { + kernel: *BuilderT, + body: *mlir.Block, + lb_inner: *const mlir.Value, + ub_inner: *const mlir.Value, + step_inner: *const mlir.Value, + inits_inner: [N]*const mlir.Value, + iv: ValueT, + carried: [N]ValueT, + results: [N]ValueT = undefined, + + const Self = @This(); + + pub fn yield(self: *Self, values: anytype) void { + const k = self.kernel; + emitScfYield(ValueT, k, self.body, N, values, "ForScope.yield"); + k.popBlock(); + const for_op = scf.for_( + k.ctx, + self.lb_inner, + self.ub_inner, + self.step_inner, + &self.inits_inner, + self.body, + .{}, + k.loc(), + ); + _ = for_op.appendTo(k.currentBlock()); + for (0..N) |i| self.results[i] = .{ .inner = for_op.result(i), .kernel = k }; + } + }; +} + +pub fn IfOnlyScope(comptime BuilderT: type, comptime ValueT: type) type { + return struct { + kernel: *BuilderT, + cond_inner: *const mlir.Value, + then_block: *mlir.Block, + + const Self = @This(); + + pub fn yieldThen(self: *Self, values: anytype) void { + const k = self.kernel; + emitScfYield(ValueT, k, self.then_block, 0, values, "IfOnlyScope.yieldThen"); + k.popBlock(); + const if_op = scf.if_( + k.ctx, + self.cond_inner, + &.{}, + self.then_block, + null, + k.loc(), + ); + _ = if_op.appendTo(k.currentBlock()); + } + }; +} + +pub fn IfScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize) type { + return struct { + kernel: *BuilderT, + cond_inner: *const mlir.Value, + then_block: *mlir.Block, + else_block: *mlir.Block, + result_types: [N]*const mlir.Type, + results: [N]ValueT = undefined, + + const Self = @This(); + + pub fn yieldThen(self: *Self, values: anytype) void { + const k = self.kernel; + emitScfYield(ValueT, k, self.then_block, N, values, "IfScope.yieldThen"); + k.popBlock(); + k.pushBlock(self.else_block); + } + + pub fn yieldElse(self: *Self, values: anytype) void { + const k = self.kernel; + emitScfYield(ValueT, k, self.else_block, N, values, "IfScope.yieldElse"); + k.popBlock(); + const if_op = scf.if_( + k.ctx, + self.cond_inner, + &self.result_types, + self.then_block, + self.else_block, + k.loc(), + ); + _ = if_op.appendTo(k.currentBlock()); + for (0..N) |i| self.results[i] = .{ .inner = if_op.result(i), .kernel = k }; + } + }; +} + +pub fn WhileScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize, comptime M: usize) type { + return struct { + kernel: *BuilderT, + before_block: *mlir.Block, + after_block: *mlir.Block, + inits_inner: [N]*const mlir.Value, + after_types: [M]*const mlir.Type, + before_carried: [N]ValueT, + after_carried: [M]ValueT = undefined, + results: [M]ValueT = undefined, + + const Self = @This(); + + pub fn yieldBefore(self: *Self, cond: ValueT, forwarded: anytype) void { + const info = @typeInfo(@TypeOf(forwarded)); + if (info != .@"struct" or !info.@"struct".is_tuple) + @compileError("WhileScope.yieldBefore: forwarded must be a tuple literal"); + if (info.@"struct".fields.len != M) + @compileError("WhileScope.yieldBefore: forwarded arity must match after_types arity"); + const k = self.kernel; + var buf: [M]*const mlir.Value = undefined; + inline for (info.@"struct".fields, 0..) |f, i| { + if (f.type != ValueT) + @compileError("WhileScope.yieldBefore: every forwarded element must be a Value"); + buf[i] = @field(forwarded, f.name).inner; + } + _ = scf.condition(k.ctx, cond.inner, &buf, k.loc()).appendTo(self.before_block); + k.popBlock(); + k.pushBlock(self.after_block); + for (0..M) |i| self.after_carried[i] = .{ .inner = self.after_block.argument(i), .kernel = k }; + } + + pub fn yieldAfter(self: *Self, values: anytype) void { + const k = self.kernel; + emitScfYield(ValueT, k, self.after_block, N, values, "WhileScope.yieldAfter"); + k.popBlock(); + const w = scf.while_( + k.ctx, + &self.inits_inner, + &self.after_types, + self.before_block, + self.after_block, + k.loc(), + ); + _ = w.appendTo(k.currentBlock()); + for (0..M) |i| self.results[i] = .{ .inner = w.result(i), .kernel = k }; + } + }; +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/kernels/common/dsl.zig b/kernels/common/dsl.zig new file mode 100644 index 00000000..acbf9e19 --- /dev/null +++ b/kernels/common/dsl.zig @@ -0,0 +1,17 @@ +const std = @import("std"); + +pub const control_flow = @import("control_flow.zig"); +pub const args = @import("args.zig"); + +pub const tupleArity = control_flow.tupleArity; +pub const emitScfYield = control_flow.emitScfYield; +pub const ForScope = control_flow.ForScope; +pub const IfOnlyScope = control_flow.IfOnlyScope; +pub const IfScope = control_flow.IfScope; +pub const WhileScope = control_flow.WhileScope; + +pub const NamedArgs = args.NamedArgs; + +test { + std.testing.refAllDecls(@This()); +} diff --git a/kernels/mosaic_tpu/BUILD.bazel b/kernels/mosaic_tpu/BUILD.bazel new file mode 100644 index 00000000..3506ede5 --- /dev/null +++ b/kernels/mosaic_tpu/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "mosaic_tpu_builder", + tags = ["manual"], + srcs = [ + "dtype.zig", + ], + import_name = "kernels/mosaic_tpu/builder", + main = "builder.zig", + visibility = ["//visibility:public"], + deps = [ + "//kernels/common", + "//mlir", + "//mlir/dialects", + "//mlir/dialects/mosaic_tpu", + "//stdx", + ], +) + +zig_test( + name = "test", + deps = [":mosaic_tpu_builder"], +) diff --git a/kernels/mosaic_tpu/README.md b/kernels/mosaic_tpu/README.md new file mode 100644 index 00000000..6c2c9433 --- /dev/null +++ b/kernels/mosaic_tpu/README.md @@ -0,0 +1,827 @@ +# ZML Mosaic TPU DSL — Author's Guide + +A Zig builder for Mosaic TPU IR (the `tpu` dialect shipped by JAX) that tries +to read like `pallas.tpu` Python while staying in pure Zig. Used by +`examples/mosaic_gdn` and by anything else that wants to emit a TPU kernel +op-for-op against what `pallas_call` produces. + +The DSL talks to the `tpu` dialect through `mlir.Operation.make(ctx, "tpu.", …)`. +The dialect must be registered into the MLIR context — see +`mlir/dialects/mosaic_tpu`. Until the JAX repo is wired into `MODULE.bazel`, the +whole stack is `manual`-tagged in Bazel. + +Two layers define the surface: + +- **Low layer** — `kernels/mosaic_tpu/builder.zig`: the IR builder primitives. + `Builder.open(allocator, ctx, name)` → `b.declareArgsOpts(spec, results, opts)` + → body via `b.*` helpers → `b.finishOpts(results, .{ .canonicalize = true })` + for the final IR string. (`Builder.buildOpts(...)` is a convenience that + bundles `open + declareArgs` into one call and returns a heap-allocated + `*Built(Spec)`; reach for it in escape-hatch code that drives the + lifecycle manually.) +- **High layer** — `zml.kernel.mosaic_tpu.Kernel(Config, spec)`: the + declarative kernel form, mirroring the Triton DSL. Bundles a config + type, a typed spec literal (name + named `inputs` / `outputs` + a typed + `run` function pointer), and produces a kernel type. Exposes + `.emit(allocator, ctx, cfg)` (IR string) and + `.call(inputs, outputs, opts)` (IR string + `stablehlo.custom_call` + targeting `tpu_custom_call` in one shot — the preferred form for + production model code). + +## Table of contents + +1. [Skeleton of a kernel](#skeleton-of-a-kernel) +2. [Argument specs](#argument-specs) +3. [Pallas `pallas_call` metadata](#pallas-pallas_call-metadata) +4. [The `Value` type](#the-value-type) +5. [Fluent methods on `Value`](#fluent-methods-on-value) +6. [Polymorphic scalars](#polymorphic-scalars) +7. [Constants and constant hoisting](#constants-and-constant-hoisting) +8. [Loads and stores](#loads-and-stores) +9. [Vector and shape helpers](#vector-and-shape-helpers) +10. [Reductions, scans, sorts](#reductions-scans-sorts) +11. [Compute primitives](#compute-primitives) +12. [Memref ops](#memref-ops) +13. [DMAs, semaphores, and barriers](#dmas-semaphores-and-barriers) +14. [Control flow](#control-flow) +15. [Escape hatches](#escape-hatches) +16. [Pallas Python ↔ Zig DSL cheat-sheet](#pallas-python--zig-dsl-cheat-sheet) +17. [Pitfalls](#pitfalls) + +--- + +## Skeleton of a kernel + +```zig +const mlir = @import("mlir"); +const mtt = @import("zml").kernel.mosaic_tpu; + +pub const MyKernel = struct { + pub const Cfg = struct { /* shape/dtype/grid knobs */ }; + pub const Kernel = mtt.Kernel(Cfg, .{ + .name = "my_kernel", + .inputs = &.{ "x", "cu_seqlens" }, + .outputs = &.{"y"}, + .run = run, + }); + fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { + _ = cfg; + const a = try b.declareArgsOpts(.{ + // Grid program-ids — i32 scalars. `pl.program_id(0)`, `pl.program_id(1)`. + .i_b = .{ .scalar = .i32 }, + + // Inputs: VMEM memrefs. + .x = .{ .ref = .{ .shape = &.{ 128, 256 }, .dtype = .bf16 } }, + + // SMEM scalar-prefetch. + .cu_seqlens = .{ .ref = .{ .shape = &.{4}, .dtype = .i32, .memory_space = .smem } }, + + // Outputs. + .y = .{ .ref = .{ .shape = &.{ 128, 256 }, .dtype = .bf16, .role = .output } }, + }, &.{}, .{ + .dimension_semantics = &.{.arbitrary}, + .iteration_bounds = &.{ /* grid */ 4 }, + .pallas_window_params = true, + }); + _ = a; + + // ... emit body via b.* helpers; reach args through `a.`. + } +}; +``` + +A kernel is a `struct {…}` wrapper bundling three decls: + +1. **`pub const Cfg`** — config type, an inline `struct { ... }` of + comptime/runtime parameters. +2. **`pub const Kernel = mtt.Kernel(Cfg, .{ .name, .inputs, .outputs, .run })`** — + the spec literal that turns into a kernel type. `inputs` / `outputs` are + slices of `[:0]const u8` field names; the factory generates `Inputs` + (struct of `Tensor`), `Outputs` (struct of `Shape`), and `Results` + (struct of `Tensor`) types from those names. The `run` field is a typed + function pointer — wrong signature is a compile error here. +3. **`fn run(b, cfg)`** — the kernel body, mirroring the Triton form. First + call `try b.declareArgsOpts(.{…}, &.{}, opts)` to create the `func.func` + op and bind a `NamedArgs(Spec)` struct of `Value`s. The factory has + already opened the builder; it'll terminate, verify, run + `canonicalize,cse,canonicalize`, and serialize after `run` returns — + no manual `b.finish(...)` call. + +The names in `spec.inputs` / `spec.outputs` are the **call-site** names +(used to build the `Inputs` / `Outputs` structs) and don't have to match +the names in `b.declareArgsOpts(...)` (which are the **MLIR-level** arg +names — what shows up in the `func.func` signature). By convention they +often do — pick whatever reads best at the boundary. + +The arg spec inside `b.declareArgsOpts(...)` is a struct literal — its +variant *tags* (`.scalar`, `.iter`, `.ref`, `.sem`) are comptime-known per +call site, but the *payload values* (shapes, dtypes, memory spaces, +windows) can be runtime values from a config. + +> **Why the wrapper struct.** `Cfg` has to be a *named* type so the typed +> `run: *const fn (*Builder, Cfg) FinishError!void` field in the spec can +> reference it. Wrapping the kernel in `pub const MyKernel = struct {...}` +> keeps `Cfg`, `Kernel`, and `run` scoped together and exposes them all to +> callers (`MyKernel.Cfg`, `MyKernel.Kernel`, plus everything the inner +> Kernel re-exports — `MyKernel.Kernel.Config`, `.Inputs`, `.Outputs`, +> `.Results`, `.CallOpts`). +> +> Single-kernel files (e.g. `zml/attention/mosaic_tpu_kernels/ragged_attention.zig`) +> can skip the wrapper and put `pub const Cfg` / `pub const Kernel` at +> module scope directly — the wrapper is just convenient when several +> kernels share a file. + +### Two ways to use a kernel + +**Offline / tooling** — get the MLIR string and print or diff it: + +```zig +const ir = try MyKernel.Kernel.emit(allocator, ctx, .{ /* cfg */ }); +defer allocator.free(ir); +try stdout.print("{s}\n", .{ir}); +``` + +**Inside a ZML model** — emit IR and drop a `stablehlo.custom_call` +(targeting `tpu_custom_call`) into the model in one shot: + +```zig +const out = MyKernel.Kernel.call( + .{ .x = x_tensor, .cu_seqlens = cu }, // Inputs (by name) + .{ .y = x_tensor.shape() }, // Outputs (Shapes) + .{ + .cfg = .{ /* cfg */ }, + .extras = .{ .vmem_limit_bytes = 1 << 20 }, + }, +).y; +``` + +`Kernel.call` takes a `mosaic_tpu.CallExtras` via `opts.extras` for +TPU-specific knobs (`vmem_limit_bytes`, `disable_bounds_checks`, +`disable_semaphore_checks`, `has_communication`, plus +`additional_attributes` and `output_operand_aliases` for in-place +updates). + +--- + +## Argument specs + +Each field passed to `b.declareArgsOpts(...)` (or to the lower-layer +`Builder.buildOpts(...)`) is an `ArgSpec.Kind` tagged-union literal. The +four kinds mirror what a Mosaic `func.func` can carry: + +| Kind literal | MLIR type | Use case | +|--------------------------------------------------------|--------------------------------------|-----------------------------------------| +| `.{ .scalar = .i32 }` | plain `i32` | `program_id`, `num_programs`, sizes | +| `.{ .iter = .{ .semantics = .parallel } }` | `index` | Loop / grid iter index with optional `tpu.dimension_semantics` | +| `.{ .ref = .{ .shape = &.{...}, .dtype = .bf16 } }` | `memref<… x T, #tpu.memory_space<...>>` | Pallas-style operand | +| `.{ .sem_array = .{ .shape = &.{2}, .kind = .dma } }` | `memref<…x!tpu.(dma_)semaphore, semaphore_mem>` | Scratch sem pool (e.g. DMA double-buffer) | +| `.{ .sem = .regular }` / `.{ .sem = .dma }` | `!tpu.semaphore` / `!tpu.dma_semaphore` | Single semaphore handle | + +### Roles + +`RefSpec.role` mirrors what Pallas `pallas_call` calls each operand: + +| Role | Where it appears | `window_params` / `transform_N` | +|--------------------|-----------------------------------------------------|---------------------------------| +| `.input` (default) | Inputs declared via `in_specs` | one entry per input | +| `.output` | Outputs declared via `out_specs` | one entry per output | +| `.scalar_prefetch` | Leading SMEM refs (Pallas's `scalar_prefetch`) | none — but appended to every `transform_N` body's args | +| `.scratch` | Trailing scratch refs (kv_bufs, l/m/acc, sem_array) | none | + +The DSL filters `window_params` to only `.input`/`.output` entries (matching +Pallas's `len(grid_mapping.block_mappings)`), and re-emits the +`scalar_prefetch` refs as trailing args on every emitted `transform_N` stub. + +The default `memory_space` for a `.ref` is `.vmem`. Override with `.smem`, +`.cmem`, `.hbm`, `.semaphore_mem`, `.vmem_shared`, or `.any` per the +`MemorySpace` enum. Iteration semantics (`.parallel`, `.arbitrary`, +`.sequential`) attach `tpu.dimension_semantics` to the iter arg. + +**Runtime payloads:** the variant tags are fixed per kernel, but shape / +dtype / memory_space values can be runtime — typical for kernels +parameterized by a `Config` struct. + +--- + +## Pallas `pallas_call` metadata + +The wrapper attributes Pallas attaches to the `func.func` are exposed through +`Builder.Opts`: + +| Field | Type | Meaning | +|--------------------------|---------------------------|--------------------------------------------------------------| +| `.@"noinline"` | `bool` (default false) | `no_inline` attribute on the function. | +| `.core_type` | `?CoreType` (default `.tc`) | `tpu.core_type = #tpu.core_type<…>`. | +| `.dimension_semantics` | `[]const DimensionSemantics` | `dimension_semantics = [...]` — one entry per grid dim. | +| `.scalar_prefetch` | `i64` (default 0) | `scalar_prefetch = N : i64`. | +| `.scratch_operands` | `i64` (default 0) | `scratch_operands = N : i64`. | +| `.iteration_bounds` | `?[]const i64` | `iteration_bounds = array` — the grid shape. | +| `.pallas_window_params` | `bool` (default false) | Auto-emit `window_params = [...]` and trailing `transform_N` stub funcs. | + +Per-arg `window_params` behavior follows +`jax/_src/pallas/mosaic/lowering.py:1014–1131` exactly. With +`pallas_window_params = true`: + +- **`.hbm` / `.any` / `.semaphore_mem`** memory spaces ⇒ empty `{}` window + entry, **no `transform_N`**. Anything else gets a populated dict. +- **VMEM trivial-window** (no `RefSpec.window`, or `window.block_shape == ref.shape`) + ⇒ auto-force `pipeline_mode = synchronous` (matching Pallas's + `Buffered(1)` injection). +- **Otherwise** ⇒ emit only attributes the user explicitly set on + `RefSpec.window`. +- **`window_bounds`** = `block_shape` (defaults to operand shape). +- **`transform_N` body** returns `block_rank` i32 zeros, mirroring a trivial + index map. + +Override per ref-arg via `RefSpec.window`: + +```zig +.x = .{ .ref = .{ + .shape = &.{ 256, 512 }, + .dtype = .bf16, + .window = mtt.ArgSpec.WindowSpec{ + .block_shape = &.{ 128, 512 }, + .pipeline_mode = .double_buffered, + .revisit_mode = .any, + .element_window = .{ .pad_low = &.{ 0, 0 }, .pad_high = &.{ 0, 0 } }, + // Custom transform_N return — by default the body returns + // `block_rank` zeros (a trivial index_map). Override per slot. + .transform_returns = &.{ + .{ .program_id = 1 }, // q_blk_idx + .{ .program_id = 0 }, // heads_blk_idx + .zero, + }, + }, +} }, +``` + +The DSL treats the window as **non-trivial** (so it does *not* auto-force +`pipeline_mode = synchronous` for VMEM) when `transform_returns` is non-null +and contains anything other than all `.zero`s. This matches Pallas's +`bm.has_trivial_window()` check. + +`PipelineMode` is `{ synchronous, double_buffered }` (matching Pallas's +`Buffered(1)` / `Buffered(2)`). `RevisitMode` is `{ immediate, any }`. + +> **Anonymous-struct gotcha.** Anonymous-struct literals (`.{ ... }`) don't +> coerce through the optional-pointer wrapping that Zig 0.16 expects for +> `RefSpec.window`. Always write the type explicitly: +> `.window = mtt.ArgSpec.WindowSpec{ ... }`. + +--- + +## The `Value` type + +```zig +pub const Value = struct { + inner: *const mlir.Value, + kernel: ?*Builder = null, + ... +}; +``` + +- `inner` is the underlying MLIR SSA handle; reach for it when calling raw + `tpu.*` / `arith.*` / `vector.*` constructors. +- `kernel` is the owning `Builder` pointer; all fluent methods route through + it. The DSL populates `kernel` automatically for every value it creates + (`emit`, `arg`, loop-region args, etc.). If you construct a `Value` by hand + (`.{ .inner = raw }`), the fluent methods will panic — use `Builder.*` + static helpers instead. + +Introspection methods always available: + +| Method | Returns | Notes | +|-------------------|----------------------------------|------------------------------------------------| +| `type_()` | `*const mlir.Type` | Full MLIR type (scalar, vector, or memref). | +| `elemType()` | `*const mlir.Type` | Element type; same as `type_()` for scalars. | +| `isVector()` | `bool` | `true` for `vector<…>`. | +| `isMemRef()` | `bool` | `true` for `memref<…>`. | +| `isShaped()` | `bool` | Vector or memref. | +| `rank()` | `usize` | `0` for scalars. | +| `dim(i)` | `i64` | Traps on scalars. | +| `shape()` | `Shape` (BoundedArray, MAX_RANK) | Use `.constSlice()` to feed `[]const i64`. | +| `isFloatElem()` | `bool` | True for any float element type. | +| `isIntElem()` | `bool` | True for any integer element type. | + +--- + +## Fluent methods on `Value` + +Every fluent method returns a new `Value`. They dispatch to +`arith.addi` / `arith.addf` / … based on the element type. The RHS is +`anytype` — see [Polymorphic scalars](#polymorphic-scalars). + +### Arithmetic + +| Method | Int op | Float op | +|----------------|--------------|------------------| +| `.add(rhs)` | `arith.addi` | `arith.addf` | +| `.sub(rhs)` | `arith.subi` | `arith.subf` | +| `.mul(rhs)` | `arith.muli` | `arith.mulf` | +| `.div(rhs)` | `arith.divsi`| `arith.divf` | +| `.rem(rhs)` | `arith.remsi`| `arith.remf` | + +For Python-style floor-division (`a // b` matches `jax.numpy.floor_divide`'s +sign-correcting expansion — `divsi` plus a `(sign(a) != sign(b)) && (a % b != +0)` correction), use `b.divFloor(lhs, rhs)`. `divsi` alone truncates toward +zero, which differs for mixed-sign operands. +| `.bitAnd(rhs)` | `arith.andi` | — | +| `.bitOr(rhs)` | `arith.ori` | — | +| `.minimum(rhs)`| `arith.minsi`| `arith.minnumf` | +| `.maximum(rhs)`| `arith.maxsi`| `arith.maxnumf` | + +### Comparisons + +All return i1 (scalar) or `vector<… x i1>`. Signed integer predicates by +default; ordered float predicates for floats. + +| Method | Int predicate | Float predicate | +|------------|---------------|-----------------| +| `.lt(rhs)` | `.slt` | `.olt` | +| `.le(rhs)` | `.sle` | `.ole` | +| `.gt(rhs)` | `.sgt` | `.ogt` | +| `.ge(rhs)` | `.sge` | `.oge` | +| `.eq(rhs)` | `.eq` | `.oeq` | +| `.ne(rhs)` | `.ne` | `.one` | + +### Casts + +| Method | Effect | +|-----------------------|-----------------------------------------------------| +| `.to(dtype)` | Numeric cast preserving shape; auto-dispatches int↔float. | +| `.bitcast(dtype)` | `tpu.bitcast` — same-bitwidth element-type swap. | +| `.abs()` | `math.absf` (float) / `math.absi` (int). | + +`.to(...)` chooses the right MLIR op based on the source vs target kind: + +| Source kind | Target kind | MLIR op | +|-------------|----------------|--------------------| +| int | wider int | `arith.extsi` | +| int | narrower int | `arith.trunci` | +| int | float | `arith.sitofp` | +| float | int | `arith.fptosi` | +| float | wider float | `arith.extf` | +| float | narrower float | `arith.truncf` | +| same type | same type | no-op | + +For `index`-typed values use `b.indexCast(v, target_ty)` or `b.toIndex(v)` +explicitly — `.to(...)` is for numeric types. + +--- + +## Polymorphic scalars + +Most `Builder.*` helpers and every fluent method's RHS take `anytype`. The +lift rule: + +- `Value` → pass through. +- `comptime_int` / `comptime_float` → a constant whose element type matches + the LHS operand. `v_i64.mul(16)` lifts `16` to i64, `v_f32.add(1.0)` lifts + `1.0` to f32. +- Runtime `i8/i16/i32`, `i64`, `f16/f32/f64`, etc. → a constant that + preserves the source Zig width. + +If you need to force a specific width for a runtime variable, cast at the +call site: `pid.mul(@as(i32, @intCast(block_size)))`. + +### When do you need `b.lift` / `b.liftAs` / `b.cIndex`? + +| Helper | Effect | +|--------------------------------|----------------------------------------------------------| +| `b.lift(value)` | Wrap a Zig scalar; dtype inferred from source. | +| `b.liftAs(value, dtype)` | Wrap a Zig scalar as a specific `DType`. | +| `b.cIndex(value)` | `arith.constant N : index` — for memref offsets. | + +Reach for these when the DSL surface demands an existing `Value` and there's +no binop sibling to match against — typically loop seeds for +`openWhile(.{inits}, ...)`, `openFor(0, N, 1, .{init})`, or `var x: Value = +b.liftAs(0, .i32);` that a later branch may reassign. + +If the constant flows through any `.add` / `.mul` / `splat` etc., skip the +lift and let auto-lift do it. + +--- + +## Constants and constant hoisting + +The DSL provides splat-style constructors for the dense-splat fast-path that +Pallas/Mosaic uses: + +| Helper | Emits | +|----------------------------------------|---------------------------------------------| +| `b.zeros(shape, dtype)` | `arith.constant dense<0.0> : vector<…>` | +| `b.ones(shape, dtype)` | Ones vector via dense splat. | +| `b.full(shape, value, dtype)` | `arith.constant dense : vector<…>` | +| `b.splat(value, shape, dtype)` | Same as `full` for comptime values; `vector.broadcast` for runtime Values. | + +These emit a single `arith.constant` (instead of a scalar-const + +`vector.broadcast` pair) so the IR diff vs Pallas stays empty for splat-only +constants. + +--- + +## Loads and stores + +Mosaic separates **vector** loads/stores from **scalar** memref loads/stores. +Helpers infer everything from the memref's type — there is no `result_type` +argument. + +### Vector + +| Helper | Pallas / Mosaic equivalent | +|---------------------------------------------------------|-------------------------------------------------------| +| `b.refLoad(ref)` | `vector.load ref[0, …, 0] : memref<…>, vector<…>` | +| `b.vectorLoadAt(ref, indices)` | `vector.load ref[indices...]` with full ref shape. | +| `b.vectorLoadShape(ref, indices, shape)` | Sliced `vector.load` — `ref[i, j, :, k:k+BLOCK]`. | +| `b.tpuVectorLoad(ref, indices, .{ .strides, .mask })` | `tpu.vector_load` — sublane stride + mask form. | +| `b.refStore(ref, value)` | Full-shape `tpu.vector_store ref[0, …]`. | +| `b.vectorStoreAt(ref, value, indices)` | `tpu.vector_store ref[indices…]`. | +| `b.refStoreOpts(ref, value, indices, .{ .strides, .mask, .add })` | Strided / masked / atomic-add store. | +| `b.stridedLoad(ref, indices, strides)` | `tpu.strided_load` — result shape = `ref.shape / strides` (per-dim ceil). | +| `b.stridedLoadShape(ref, indices, strides, result_shape)` | `tpu.strided_load` with explicit result shape — for cropped loads (e.g. half-rows). | +| `b.stridedStore(ref, value, indices, strides)` | `tpu.strided_store`. | + +> **`refLoad` vs `tpuVectorLoad`.** Pallas's default emit is +> `vector.load`, which is what `refLoad` / `vectorLoadAt` produce. Reach for +> `tpuVectorLoad` only when you need sublane stride or a per-element mask +> (the `tpu.vector_load` form). Same on the store side. + +`indices` must be `index`-typed Values (use `b.toIndex(v)` to convert from +i32). If you pass `&.{}`, the helper pads with a cached `arith.constant 0 : +index` to match the ref's rank — the same SSA value is reused per block, so +no `%c0` duplication shows up in the IR. + +### Scalar (SMEM) + +| Helper | MLIR op | +|-------------------------------------------------|------------------------------------------| +| `b.scalarLoad(ref, indices)` | `memref.load` | +| `b.scalarStore(ref, value, indices)` | `memref.store` | + +Use these for SMEM args (e.g. `cu_seqlens`) where Pallas emits scalar +`memref.load`. + +--- + +## Vector and shape helpers + +| Helper | Effect | +|-----------------------------------------------------|----------------------------------------------------------| +| `b.broadcastTo(src, shape)` | `vector.broadcast` — leading-dim broadcast or rank-prepend. | +| `b.shapeCast(src, new_shape)` | `vector.shape_cast` — reshape preserving total lanes. | +| `b.reshape(src, new_shape)` | `tpu.reshape` (TPU-flavored reshape; differs from `shapeCast`). | +| `b.repeat(src, dimension, times)` | `tpu.repeat`. | +| `b.concatenate(sources, dimension)` | `tpu.concatenate`. | +| `b.transpose(src, permutation)` | `tpu.transpose`. | +| `b.broadcastInSublanes(src, lane)` | `tpu.broadcast_in_sublanes`. | +| `b.rotate(src, .{ .amount, .dimension, .stride, .stride_dimension })` | `tpu.rotate`. | +| `b.vectorExtract(src, position)` | `vector.extract` — scalar / lower-rank slice. | +| `b.iota(shape, dtype, dimensions)` | `tpu.iota` over a chosen dim. | +| `b.arange(n, dtype)` | Pallas-style 1-D iota: `<1xN>` `tpu.iota` + `vector.shape_cast` to ``. Mirrors `_iota_lowering_rule`. | + +> **Use `arange`, not `iota(&.{N})`.** The TPU layout pass requires vectors +> to be 2-D with the lane dim trailing; a bare 1-D `tpu.iota` is rejected. +> `arange` emits the `<1xN>` + `shape_cast` pair Pallas's lowering produces. + +> **`shapeCast` vs `reshape`.** `shapeCast` emits the upstream +> `vector.shape_cast` Pallas uses everywhere; `reshape` emits TPU's +> `tpu.reshape`. Stick to `shapeCast` when matching Pallas IR. + +### Auto-broadcast + +There is **no implicit auto-broadcast** in the Mosaic DSL. Mosaic kernels run +inside the TPU's tile semantics and the layout pass is strict about +`vector<…>` ranks/shapes — surprise-broadcasts would be footguns. Build the +broadcasts you need explicitly with `shapeCast` + `broadcastTo`: + +```zig +// b_k[:, None] broadcast to +const bk_2d = k.shapeCast(b_k, &.{ BK, 1 }); +const bk_bc = k.broadcastTo(bk_2d, &.{ BK, BV }); +``` + +For scalar→vector splatting, use `b.broadcastTo(scalar, shape)` (it routes +through `vector.broadcast`) or `b.full / b.splat` for comptime constants. + +--- + +## Reductions, scans, sorts + +### Vector-dialect reductions + +| Helper | Effect | +|-----------------------------------------------------|----------------------------------------------------------| +| `b.multiReduction(kind, src, acc, axes)` | `vector.multi_reduction` — multi-D reduce along `axes`. | +| `b.reduceToScalar(kind, src, acc)` | 1-D vec → scalar via the Pallas idiom: `shape_cast→<1xN>`, `multi_reduction[1]`, `extract[0]`. | +| `b.vectorReductionFlat(kind, src)` | `vector.reduction` — bare 1-D reduce. Layout pass may reject; prefer `reduceToScalar`. | + +`kind` is a `CombiningKind` (`.add`, `.mul`, `.minsi`, `.maxsi`, `.minnumf`, +`.maxnumf`, `.and_`, `.or_`, `.xor`). + +> **Pallas's reduce-to-scalar idiom.** Pallas never emits a bare +> `vector.reduction` for the L2-norm / sum-to-scalar pattern — it always +> goes through `multi_reduction` after a leading-dim `shape_cast`. The +> `reduceToScalar` wrapper produces exactly that shape, so kernels stay +> byte-equivalent to `pallas_call`'s output. See +> `jax/_src/pallas/mosaic/lowering.py:reduce_lowering_rule`. + +### TPU-dialect reductions / scan / sort + +| Helper | MLIR op | +|-----------------------------------------------------|----------------------------------------------------------| +| `b.allReduce(input, kind, dim)` | `tpu.all_reduce` — keeps `dim` as size 1. | +| `b.reduceIndex(input, kind, axis)` | `tpu.reduce_index` — drop axis, replace with `index`. | +| `b.scan(input, kind, mask)` | `tpu.scan`. | +| `b.sort(keys, values, mask, .{ .descending })` | `tpu.sort` — returns `[3]Value` (sorted_keys, indices, sorted_values). | + +--- + +## Compute primitives + +### Math (scalar / elementwise) + +| Helper | MLIR op | +|------------------------------------------------|--------------------| +| `b.exp(x)` | `math.exp` | +| `b.exp2(x)` | `exp(x * ln2)` — matches Pallas's `_exp2_lowering_rule` (forward-compat). | +| `b.mathExp2(x)` | Raw `math.exp2` — only legal on cloud TPUs after 2025-07-26. | +| `b.log(x)` / `b.log2(x)` | `math.log` / `math.log2` | +| `b.sqrt(x)` / `b.rsqrt(x)` | `math.sqrt` / `math.rsqrt` | +| `b.sin(x)` / `b.cos(x)` / `b.tanh(x)` / `b.erf(x)` | `math.*` | +| `b.absf(x)` / `b.absi(x)` / `b.abs(x)` | `math.absf` / `math.absi` (auto) | +| `b.floor(x)` / `b.ceil(x)` | `math.floor` / `math.ceil` | +| `b.powf(x, y)` | `math.powf` | +| `b.fma(a, b, c)` | `math.fma` | +| `b.reciprocal(x)` / `b.reciprocalOpts(x, .{ .approximate })` | `tpu.reciprocal` | + +### Matmul + +```zig +// y = lhs @ rhs + acc +const y = k.matmul(lhs, rhs, acc); +const y2 = k.matmulOpts(lhs, rhs, acc, .{ + .transpose_lhs = false, + .transpose_rhs = true, + .precision = .bf16, +}); +``` + +`matmul` lowers to `tpu.matmul`; result type is taken from `acc`. Use +`matmulOpts` to set transposes, `tpu.contract_precision`, or a custom +`#tpu.dot_dimension_numbers<…>` attribute. + +### Selects / where + +| Helper | MLIR op | +|---------------------------------|--------------------------| +| `b.select(cond, t, f)` | `arith.select` | +| `b.where(cond, x, y)` | Alias of `select`. | + +### Masks + +| Helper | Effect | +|----------------------------------------------------------|-----------------------------------------| +| `b.createMask(low_bounds, high_bounds, shape)` | `vector.create_mask`. | +| `b.rangeMask(limits, shape)` | `vector.create_mask` from `0..limit`. | + +--- + +## Memref ops + +For carving sub-references out of a larger memref or reinterpreting an +existing one: + +| Helper | MLIR op | +|-----------------------------------------------------------------|--------------------------| +| `b.memRefSlice(mem_ref, offsets, sizes, strides, result_shape)` | `memref.subview`. | +| `b.memRefSqueeze(mem_ref, result_shape)` | `memref.collapse_shape` style. | +| `b.memRefReshape(mem_ref, result_shape)` | `tpu.memref_reshape`. | +| `b.memRefBitcast(mem_ref, dtype)` | `tpu.memref_bitcast` — same shape, new element dtype. | +| `b.memRefBitcastShape(mem_ref, result_shape, dtype)` | `tpu.memref_bitcast` with explicit result shape — for the bf16↔i32 packing trick (`<64x128xbf16>` → `<32x128xi32>`). | +| `b.reinterpretCast(mem_ref, result_shape, dtype)` | `memref.reinterpret_cast`. | +| `b.assumeLayout(src)` | `tpu.assume_layout`. | +| `b.eraseMemRefLayout(mem_ref)` | `memref.cast` to no-layout. | + +--- + +## DMAs, semaphores, and barriers + +For multi-core / async DMA orchestration: + +| Helper | MLIR op | +|------------------------------------------------------------------------------|------------------------| +| `b.semAlloc(.regular)` / `b.semAlloc(.dma)` | `tpu.sem_alloc`. | +| `b.semBarrier()` | `tpu.sem_barrier`. | +| `b.semRead(sem)` | `tpu.sem_read`. | +| `b.semWait(sem, amount)` | `tpu.sem_wait`. | +| `b.semSignal(sem, amount)` / `b.semSignalOpts(sem, amount, .{ .device_id, .core_id })` | `tpu.sem_signal`. | +| `b.barrier(barrier_id)` | `tpu.barrier`. | +| `b.enqueueDma(src, dst, target_sem, .{ .source_semaphore, .device_id, .core_id, .priority, .strict_ordering })` | `tpu.enqueue_dma`. | +| `b.waitDma2(sem, src, dst, .{ .device_id, .core_id, .strict_ordering })` | `tpu.wait_dma2`. | +| `b.deviceId()` | `tpu.device_id`. | +| `b.delay(cycles)` | `tpu.delay`. | +| `b.traceStart(level, message)` / `b.traceStop()` | `tpu.trace_start` / `tpu.trace_stop` — Pallas's einsum boundaries. | +| `b.dotDimensionNumbers(lhs_c, rhs_c, lhs_nc, rhs_nc, output_order, lhs_b, rhs_b)` | Build a `#tpu.dot_dimension_numbers<...>` attr for `matmulOpts`. | + +--- + +## Control flow + +SCF regions are built with **scope-based** builders — `b.openFor(...)`, +`b.openIf(...)` / `b.openIfElse(...)`, `b.openWhile(...)` return a typed +scope value. You emit body ops into a bare `{}` block, terminate with +`yield` / `yieldThen` / `yieldAfter`, and read results from `scope.results` +afterward. No `ctx` struct, no function literal — lexical scope carries +captures. + +### `openFor` — scf.for with iter_args + +```zig +var loop = k.openFor(c0_i32, T, c1_i32, .{ b_h_init }); +{ + const t = loop.iv; + const b_h = loop.carried[0]; + + // ... body ops ... + + loop.yield(.{ b_h_updated }); +} +const final_b_h = loop.results[0]; +``` + +- `lower` / `upper` / `step` are `anytype`: pass Values, comptime ints, or + runtime Zig ints. Use `i32` constants (`b.lift(@as(i32, 0))`) when you + want the loop counter to be `i32` (matching Pallas's `scf.for ... : i32`). + The default for `index`-typed counters is `b.cIndex(N)`. +- `inits` is a tuple literal `.{v1, v2, ...}`. Arity is comptime — `carried`, + `results`, and the `yield(...)` call are all fixed-size arrays of the + same length. +- Empty loops: `b.openFor(0, N, 1, .{})` + `loop.yield(.{})`. + +### `openIf` — scf.if without else (side-effects only) + +```zig +var i = k.openIf(cond); +{ + // then body — store, increment, etc. + i.yieldThen(.{}); +} +``` + +Use for conditional side-effects (no results). The else block is auto-built +empty. + +### `openIfElse` — scf.if with else and optional results + +```zig +var i = k.openIfElse(cond, .{ k.scalarTy(.f32) }); +{ + i.yieldThen(.{ x }); +} +{ + i.yieldElse(.{ y }); +} +const r = i.results[0]; +``` + +- `result_types` is a tuple of `*const mlir.Type` — use `k.scalarTy(dtype)` + / `k.vectorTy(shape, dtype)` / `k.memRefTy(shape, dtype, mem_space)`. + Pass `.{}` for an if/else with no results. + +### `openWhile` — scf.while + +```zig +var w = k.openWhile(.{ i0 }, .{ k.scalarTy(.i32) }); +{ + const s = w.before_carried[0]; + w.yieldBefore(s.lt(10), .{ s }); +} +{ + const s = w.after_carried[0]; + w.yieldAfter(.{ s.add(1) }); +} +const r = w.results[0]; +``` + +- `inits` — tuple of `Value`s; before-region arg types = their types. +- `after_types` — tuple of `*const mlir.Type`; after-region arg types + the + scf.while's result types. +- `yieldBefore(cond, forwarded)` — `forwarded` arity must match `after_types`. +- `yieldAfter(values)` — arity must match `inits`. + +### General tips + +- Scopes stack naturally: `openFor { openIf { openFor { ... } } }` all just + push/pop the kernel's current-block stack. +- Empty yields are legal: `loop.yield(.{})`, `i.yieldThen(.{})`, etc. +- After any `yield*`, the scope's `.results` holds the scf op's results as + a `[N]Value`. + +--- + +## Escape hatches + +Everything the DSL doesn't expose is still reachable — `Builder` is a +book-keeping layer, not a walled garden: + +- `v.inner` — the raw `*const mlir.Value`; feed it to any `tpu.*` / + `arith.*` / `vector.*` / `memref.*` / `scf.*` / `math.*` constructor + directly. +- `b.ctx` — the owning `*mlir.Context`. +- `b.currentBlock()` — the active `mlir.Block` (respects `pushBlock` / + `popBlock`). +- `b.emit(op)` / `b.emitMulti(op, n)` — if you've built an `mlir.Operation` + by hand, use these so the result comes back as a `Value` with its + `kernel` field populated. +- `b.scalarTy(dtype)` / `b.vectorTy(shape, dtype)` / + `b.memRefTy(shape, dtype, mem_space)` — type constructors used by raw + ops and by `openIfElse.result_types` / `openWhile.after_types`. + +When you drop to raw ops, remember to still **append them to the current +block** — `b.emit(...)` does that for you; `_ = op.appendTo(b.currentBlock())` +is the manual form. + +--- + +## Pallas Python ↔ Zig DSL cheat-sheet + +| Python | Zig | +|-------------------------------------------------------|--------------------------------------------------------------| +| `pl.program_id(0)` | `a.i_v` (declared as `.{ .scalar = .i32 }`) | +| `cu_seqlens[i]` | `k.scalarLoad(a.cu_seqlens, &.{ k.toIndex(i) })` | +| `lax.div(a, b)` | `k.divsi(a, b)` | +| `lax.rem(a, b)` | `k.remsi(a, b)` | +| `jnp.arange(N, dtype=jnp.int32)` | `k.arange(N, .i32)` | +| `o_k < KEY_DIM` | `k.cmpi(.slt, o_k, dim_splat)` / `o_k.lt(dim_splat)` | +| `mask_k[:, None] & mask_v[None, :]` | `k.andi(k.broadcastTo(k.shapeCast(mask_k, &.{BK,1}), &.{BK,BV}), k.broadcastTo(mask_v, &.{BK,BV}))` | +| `jnp.where(mask, x, 0.0)` | `k.select(mask, x, k.zeros(shape, dtype))` | +| `q_ref[0, t, h, :].astype(jnp.float32)` | `k.vectorLoadShape(a.q, &.{c0, t_idx, h_idx, c0}, &.{1,1,1,BK})` then `.shapeCast(&.{BK})` then `.to(.f32)` | +| `q_ref[0, t, h, :] = b_o.astype(o_ref.dtype)` | `k.vectorStoreAt(a.q, k.shapeCast(b_o, &.{1,1,1,BK}), &.{c0, t_idx, h_idx, c0})` | +| `jnp.sqrt(jnp.sum(b_q * b_q) + 1e-6)` | `k.sqrt(k.addf(k.reduceToScalar(.add, k.mulf(b_q, b_q), cst_zero_1), cst_eps))` | +| `jnp.exp(b_g)` | `k.exp(b_g)` | +| `jnp.sum(b_h * b_k[:, None], axis=0)` | `k.multiReduction(.add, k.mulf(b_h, bk_bc), cst_zero_v, &.{0})` | +| `b_h * jnp.exp(b_g)` (broadcast to ) | `k.mulf(b_h, k.broadcastTo(exp_g, &.{BK, BV}))` | +| `b_beta * (b_v - predicted_v)` | `k.mulf(k.broadcastTo(b_beta, &.{BV}), k.subf(b_v, predicted_v))` | +| `lax.fori_loop(0, T, body, init)` | `var loop = k.openFor(c0_i32, T, c1_i32, .{init}); { …; loop.yield(.{…}); }` | +| `if cond: …` (side-effects only) | `var i = k.openIf(cond); { …; i.yieldThen(.{}); }` | +| `if cond: x = a else: x = b` | `var i = k.openIfElse(cond, .{ty}); { …; i.yieldThen(.{a}); }; { …; i.yieldElse(.{b}); }` | +| `while cond:` | `var w = k.openWhile(.{inits}, .{after_types}); { …; w.yieldBefore(cond, .{…}); }; { …; w.yieldAfter(.{…}); }` | +| `pltpu.MemorySpace.SMEM` | `.memory_space = .smem` on the `RefSpec` | +| `pl.BlockSpec(memory_space=…)` (trivial window) | Default `RefSpec` — auto-`synchronous` on VMEM | +| `pl.BlockSpec(block_shape=(BM, BK), …)` | `.window = .{ .block_shape = &.{BM, BK} }` | +| `pl.BlockSpec(..., pipeline_mode=Buffered(2))` | `.window = .{ .pipeline_mode = .double_buffered }` | + +--- + +## Pitfalls + +1. **TPU vectors are 2-D minimum.** A bare `tpu.iota : vector` or + `vector.reduction` on `vector` is rejected by the layout pass. + Use `b.arange(N, dtype)` (emits the `<1xN>` iota + `shape_cast`) and + `b.reduceToScalar(...)` (emits the `shape_cast→<1xN>` + + `multi_reduction[1]` + `extract[0]` triple). Match Pallas's emit pattern + verbatim and the layout pass is happy. + +2. **No implicit auto-broadcast.** Unlike the Triton DSL, Mosaic won't + silently `expand_dims` / `broadcast` a rank-mismatched RHS. Build the + broadcasts explicitly (`shapeCast` + `broadcastTo`). The Pallas Python + DSL emits the same explicit broadcast pair — mirroring it keeps the IR + diff empty. + +3. **`vector.load` indices must be `index`-typed.** Convert from i32 with + `k.toIndex(v)` (emits `arith.index_cast`). JAX does no CSE inside + `pallas_call` lowering, so re-emitting the same `index_cast` per use is + the byte-equivalent thing to do. + +4. **Re-emit derived offsets per use.** Pallas re-runs its index_map + computation per operand load/store rather than CSE-ing intermediate + `arith.muli` / `arith.addi`. To match its IR exactly, do the same: emit + `iv_off = i_v * BV` again at every site that needs it. The MLIR + canonicalizer collapses duplicates downstream. + +5. **`refLoad` vs `tpuVectorLoad`.** Pallas's default `ref[...]` lowers to + `vector.load` (not `tpu.vector_load`). Use `refLoad` / `vectorLoadAt` / + `vectorLoadShape` for byte-equivalence; reach for `tpuVectorLoad` only + when you need `tpu.vector_load`-specific features (sublane stride or + per-element mask). + +6. **`shapeCast` vs `reshape`.** `shapeCast` emits the upstream + `vector.shape_cast` Pallas uses everywhere; `reshape` emits TPU's + `tpu.reshape` (different op, different layout semantics). Stick with + `shapeCast` when matching Pallas IR. + +7. **SMEM args take scalar `memref.load`, not `vector.load`.** Use + `k.scalarLoad(a.cu_seqlens, ...)` for SMEM operands (typically the + scalar-prefetch path). VMEM args go through the vector-load helpers. + +8. **`pallas_window_params = true` only adds metadata.** It does not change + the kernel body — it just appends `iteration_bounds`, `window_params`, + and the `transform_N` stub funcs to the `func.func` so downstream passes + that consume `window_params` (DMA scheduling, double-buffering) work. + For non-trivial windows or non-VMEM operands, attach a `RefSpec.window` + to override the auto-`synchronous` default. + +9. **`finish` verifies before serializing.** A verifier failure here almost + always means an operand-type mismatch or a missing block terminator — + `scf.yield` operand count must match the loop's result types, and + `func.return` arity must match the declared result types. diff --git a/kernels/mosaic_tpu/builder.zig b/kernels/mosaic_tpu/builder.zig new file mode 100644 index 00000000..e0afffd3 --- /dev/null +++ b/kernels/mosaic_tpu/builder.zig @@ -0,0 +1,2200 @@ +//! ZML Mosaic TPU DSL — a Zig builder for Mosaic TPU IR (the `tpu` dialect). +//! +//! Args are memrefs (`!memref`) plus optional scalars, +//! iteration-index, and semaphore args. The dialect must be registered into +//! the MLIR context — see `mlir/dialects/mosaic_tpu`. The entire Mosaic stack is +//! `manual`-tagged in Bazel until the JAX repo is wired into `MODULE.bazel`. + +const std = @import("std"); + +const mlir = @import("mlir"); +const dialects = @import("mlir/dialects"); +const dsl = @import("kernels/common"); +const dtypes = @import("dtype.zig"); +const stdx = @import("stdx"); +const tpu = @import("mlir/dialects/mosaic_tpu"); + +const arith = dialects.arith; +const cf = dialects.cf; +const func = dialects.func; +const math = dialects.math; +const memref = dialects.memref; +const scf = dialects.scf; +const vector = dialects.vector; + +pub const DType = dtypes.DType; + +const isFloatDtype = dtypes.isFloatDtype; +const dtypeBitwidth = dtypes.dtypeBitwidth; +const intBitwidth = dtypes.intBitwidth; + +/// `tpu`-dialect enums re-exported for convenience. +pub const MemorySpace = tpu.MemorySpace; +pub const ReductionKind = tpu.ReductionKind; +pub const ContractPrecision = tpu.ContractPrecision; +pub const RoundingMode = tpu.RoundingMode; +pub const DimensionSemantics = tpu.DimensionSemantics; +pub const CoreType = tpu.CoreType; +pub const PipelineMode = tpu.PipelineMode; +pub const RevisitMode = tpu.RevisitMode; + +/// `vector`-dialect enums re-exported for convenience. +/// Used by `vectorReduction` / `multiReduction`. +pub const CombiningKind = vector.CombiningKind; + +// ============================================================================= +// Value — a handle to an MLIR value inside the kernel body. +// ============================================================================= + +/// MLIR SSA value with an optional back-pointer to its `Builder` for fluent ops. +pub const Value = struct { + inner: *const mlir.Value, + kernel: ?*Builder = null, + + pub const Shape = stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK); + + pub fn type_(self: Value) *const mlir.Type { + return self.inner.type_(); + } + + fn kern(self: Value) *Builder { + return self.kernel orelse @panic("Value has no owning kernel; use Builder.* helpers instead"); + } + + /// Element type of a vector / memref value, or the value's own type if + /// scalar. + pub fn elemType(self: Value) *const mlir.Type { + if (self.type_().isA(mlir.ShapedType)) |shaped| return shaped.elementType(); + return self.type_(); + } + + /// True when this value is a vector (rank ≥ 1). + pub fn isVector(self: Value) bool { + return self.type_().isA(mlir.VectorType) != null; + } + + /// True when this value is a memref. + pub fn isMemRef(self: Value) bool { + return self.type_().isA(mlir.MemRefType) != null; + } + + /// True when the value is shaped (vector / memref / tensor). + pub fn isShaped(self: Value) bool { + return self.type_().isA(mlir.ShapedType) != null; + } + + /// Rank of this value. Scalars have rank 0. + pub fn rank(self: Value) usize { + if (self.type_().isA(mlir.ShapedType)) |shaped| return shaped.rank(); + return 0; + } + + /// `i`-th dim size of a shaped value; traps on scalar. + pub fn dim(self: Value, i: usize) i64 { + return self.type_().isA(mlir.ShapedType).?.dimension(i); + } + + /// Shape as a stack-allocated BoundedArray. Use `.constSlice()` for `[]const i64`. + pub fn shape(self: Value) Shape { + var out: Shape = .empty; + if (self.type_().isA(mlir.ShapedType)) |shaped| { + const r = shaped.rank(); + for (0..r) |i| out.appendAssumeCapacity(shaped.dimension(i)); + } + return out; + } + + /// True if the element type is a floating-point type (f16/bf16/f32/f64/fp8/…). + pub fn isFloatElem(self: Value) bool { + const et = self.elemType(); + inline for (std.meta.fields(mlir.FloatTypes)) |f| { + if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, f.name))) != null) return true; + } + return false; + } + + /// True if the element type is an integer type. + pub fn isIntElem(self: Value) bool { + return self.elemType().isA(mlir.IntegerType) != null; + } + + // -------- fluent arithmetic (int/float auto-dispatch from elem type) -------- + + pub fn add(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.addf(l, r) else k.addi(l, r); + } + + pub fn sub(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.subf(l, r) else k.subi(l, r); + } + + pub fn mul(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.mulf(l, r) else k.muli(l, r); + } + + pub fn div(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.divf(l, r) else k.divsi(l, r); + } + + pub fn rem(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.remf(l, r) else k.remsi(l, r); + } + + pub fn bitAnd(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.andi(l, r); + } + + pub fn bitOr(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.ori(l, r); + } + + pub fn minimum(self: Value, rhs: anytype) Value { + return self.kern().minimum(self, rhs); + } + pub fn maximum(self: Value, rhs: anytype) Value { + return self.kern().maximum(self, rhs); + } + + // -------- fluent comparisons (int predicates default; float uses cmpf) -------- + + pub fn lt(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.cmpf(.olt, l, r) else k.cmpi(.slt, l, r); + } + pub fn le(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.cmpf(.ole, l, r) else k.cmpi(.sle, l, r); + } + pub fn gt(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.cmpf(.ogt, l, r) else k.cmpi(.sgt, l, r); + } + pub fn ge(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.cmpf(.oge, l, r) else k.cmpi(.sge, l, r); + } + pub fn eq(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.cmpf(.oeq, l, r) else k.cmpi(.eq, l, r); + } + pub fn ne(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.cmpf(.one, l, r) else k.cmpi(.ne, l, r); + } + + // -------- casts -------- + + /// Numeric cast preserving shape (vector or scalar). Auto-dispatches on + /// source / destination float-vs-int. + pub fn to(self: Value, dtype: DType) Value { + return self.kern().cast(self, dtype); + } + + /// `tpu.bitcast` — same-bitwidth element-type swap. + pub fn bitcast(self: Value, dtype: DType) Value { + return self.kern().bitcastTo(self, dtype); + } + + /// Absolute value (float→absf, int→absi). + pub fn abs(self: Value) Value { + return self.kern().abs(self); + } +}; + +// ============================================================================= +// Arg spec — describes one function parameter. +// ============================================================================= + +/// Per-arg specification for building `func.func`'s signature. +pub const ArgSpec = struct { + name: []const u8, + kind: Kind, + + pub const Kind = union(enum) { + /// Plain scalar — typically used for grid sizes / dynamic dims passed + /// in by the runtime. `.{ .scalar = .i32 }`. + scalar: DType, + /// Loop / grid iteration index — `index` type with optional + /// `tpu.dimension_semantics` annotation. + /// `.{ .iter = .{ .semantics = .parallel } }`. + iter: IterOpts, + /// Memref (the bread-and-butter kernel arg). The arg is typed as + /// `memref>`. + /// `.{ .ref = .{ .shape = &.{128, 256}, .dtype = .bf16, .memory_space = .vmem } }`. + ref: RefSpec, + /// `memref` — the + /// scratch-array form Pallas uses for semaphore pools. Pallas + /// classifies it as `.scratch`, so it never appears in + /// `window_params` / `transform_N`. + sem_array: SemArraySpec, + /// `!tpu.semaphore` or `!tpu.dma_semaphore`. + /// `.{ .sem = .regular }` / `.{ .sem = .dma }`. + sem: SemKind, + }; + + pub const IterOpts = struct { + semantics: ?DimensionSemantics = null, + }; + + pub const RefSpec = struct { + shape: []const i64, + dtype: DType, + memory_space: MemorySpace = .vmem, + /// Block window. `null` ⇒ trivial window (block == operand shape); + /// VMEM trivial windows auto-get `pipeline_mode = synchronous`. + window: ?WindowSpec = null, + /// Pallas role of this operand. Only `.input`/`.output` get a + /// `window_params` entry and a `transform_N` stub. + /// `.scalar_prefetch` refs become trailing args of every + /// `transform_N` body (Pallas's index_map sees them). + /// `.scratch` refs are not visible to transforms or window_params. + role: Role = .input, + }; + + pub const Role = enum { input, output, scratch, scalar_prefetch }; + + pub const WindowSpec = struct { + /// `null` ⇒ use the operand's full shape (trivial window). + /// Determines `window_bounds` and the `transform_N` arity. + block_shape: ?[]const i64 = null, + /// `#tpu.pipeline_mode<…>`. `null` ⇒ auto-`synchronous` for VMEM trivial windows, omitted otherwise. + pipeline_mode: ?PipelineMode = null, + /// `#tpu.revisit_mode<…>`. `null` ⇒ omit. + revisit_mode: ?RevisitMode = null, + /// `#tpu.element_window<[pad_low], [pad_high]>`. `null` ⇒ omit. + element_window: ?ElementWindowSpec = null, + /// What `transform_N` returns. `null` ⇒ all-zero `block_rank` i32s + /// (matching a trivial index_map). One entry per result; length + /// must match `block_shape.len` (or operand rank if `block_shape` + /// is `null`). + transform_returns: ?[]const TransformReturn = null, + }; + + /// One return slot of a `transform_N` body. Mirrors what + /// `lower_jaxpr_to_transform_func` produces — typically a select of + /// program-ids, scalar-prefetch loads, or constants. We only encode + /// the most common forms; drop to a custom transform body for anything + /// fancier. + pub const TransformReturn = union(enum) { + /// Return `arith.constant 0 : i32`. + zero, + /// Return the Nth program-id (`%argN`). + program_id: usize, + }; + + pub const ElementWindowSpec = struct { + pad_low: []const i64, + pad_high: []const i64, + }; + + pub const SemKind = enum { regular, dma }; + + pub const SemArraySpec = struct { + shape: []const i64, + kind: SemKind = .dma, + }; +}; + +pub const FinishError = error{ InvalidMlir, MlirUnexpected } || std.mem.Allocator.Error || std.Io.Writer.Error; + +// ============================================================================= +// Named-param structs. +// ============================================================================= + +pub const VectorLoadOpts = struct { + /// Per-dim load stride. Pass `&.{}` for unit-strided. + strides: []const i32 = &.{}, + /// Optional element-wise mask. Must be a `vector<...xi1>` matching the + /// load result shape. + mask: ?Value = null, +}; + +pub const VectorStoreOpts = struct { + strides: []const i32 = &.{}, + mask: ?Value = null, + /// If true emits `tpu.vector_store ... add = true` (atomic-add semantics + /// where supported). + add: bool = false, +}; + +pub const MatmulOpts = struct { + transpose_lhs: bool = false, + transpose_rhs: bool = false, + precision: ?ContractPrecision = null, + /// Optional `#tpu.dot_dimension_numbers<...>` attribute. When omitted the + /// canonicalizer derives one. Build with `mlir.Attribute.parse`. + dimension_numbers: ?*const mlir.Attribute = null, +}; + +pub const ReduceOpts = struct { + /// Reduction dim. Defaults to 0. + dim: i64 = 0, +}; + +pub const SortOpts = struct { + descending: bool = false, +}; + +pub const SemSignalOpts = struct { + device_id: ?Value = null, + core_id: ?Value = null, +}; + +pub const EnqueueDmaOpts = struct { + source_semaphore: ?Value = null, + device_id: ?Value = null, + core_id: ?Value = null, + priority: i32 = 0, + strict_ordering: bool = false, +}; + +pub const WaitDma2Opts = struct { + device_id: ?Value = null, + core_id: ?Value = null, + strict_ordering: bool = false, +}; + +pub const ReciprocalOpts = struct { + approx: bool = false, + full_range: bool = false, +}; + +pub const RotateOpts = struct { + amount: i32, + dimension: i32, + stride: ?i32 = null, + stride_dimension: ?i32 = null, +}; + +pub const CastOpts = struct { + /// Forwarded to `tpu.fptosi` / `tpu.fptoui` / etc. + rounding: ?RoundingMode = null, + /// If true, perform a same-bitwidth `tpu.bitcast` instead of a numeric cast. + bitcast: bool = false, +}; + +pub const MinMaxOpts = struct { + /// IEEE NaN-propagation toggle. Default is non-propagating (maxnumf/minnumf). + propagate_nan: bool = false, +}; + +// ============================================================================= +// Builder — the main DSL builder. +// ============================================================================= + +/// The main DSL builder. Create with `init`, populate the body with helpers, +/// then call `finish` to get the IR string. +pub const Builder = struct { + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + ctx: *mlir.Context, + module: *mlir.Module, + /// Name of the kernel — used when `declareArgs` creates the `func.func`. + name: []const u8, + /// `func.func` op for this kernel. `null` between `open` and the first + /// `declareArgs` call; populated thereafter. + func_op: ?*mlir.Operation, + /// Entry block for the `func.func`. `null` between `open` and the first + /// `declareArgs` call; populated thereafter. + entry_block: ?*mlir.Block, + block_stack: std.ArrayList(*mlir.Block), + /// Cached `arith.constant 0 : index` Value (and the block it lives + /// in) so repeat no-index loads / stores share an SSA value. See + /// `zeroIndex` for the lookup logic. + zero_index: ?Value = null, + zero_index_block: ?*mlir.Block = null, + /// Snapshot of the kernel's ref-args saved at `declareArgsLowOpts` + /// time. `finish` walks this list to emit `transform_N` stubs when + /// `pallas_window_params=true`. Allocated in `arena`. + ref_args: []RefArgSnapshot = &.{}, + /// SMEM scalar-prefetch ref types — appended to every `transform_N` + /// body's argument list. Empty when no `.scalar_prefetch` refs were + /// declared. + prefetch_args: []const *const mlir.Type = &.{}, + /// Number of grid program-ids (i32) at the head of every `transform_N` + /// body. Equals `Opts.dimension_semantics.len`. + grid_dim_count: usize = 0, + /// Whether to emit Pallas-style `transform_N` stubs at `finish` time. + /// Set by `declareArgsLowOpts` from `Opts.pallas_window_params`. + emit_transform_stubs: bool = false, + + /// Per-ref-arg snapshot for `finish`-time `transform_N` stub emission. + /// `skipped` = ANY/HBM/SEMAPHORE memory space (empty window, no transform). + pub const RefArgSnapshot = struct { + block_rank: i64, + skipped: bool, + role: ArgSpec.Role, + transform_returns: ?[]const ArgSpec.TransformReturn = null, + }; + + /// Snapshot of the SMEM scalar-prefetch ref types (after the program-ids) + /// — repeated as trailing args of every `transform_N` body to mirror + /// Pallas's `lower_jaxpr_to_transform_func`. + pub const PrefetchSnapshot = struct { + /// Pre-built MLIR memref types (cached pointers from declareArgs time). + types: []const *const mlir.Type, + }; + + /// Builder-level options. `noinline` is a Zig keyword — write `.{ .@"noinline" = true }`. + pub const Opts = struct { + @"noinline": bool = false, + /// `tpu.core_type` on the `func.func`. `null` skips the attr. + core_type: ?CoreType = .tc, + /// One entry per grid dim. Empty list emits an empty array attr + /// (matching Pallas's `dimension_semantics = []`). + dimension_semantics: []const DimensionSemantics = &.{}, + /// Number of leading scalar-prefetch operands. + scalar_prefetch: i64 = 0, + /// Number of trailing scratch refs (memrefs holding kernel-local + /// scratch state). + scratch_operands: i64 = 0, + /// Pallas's `iteration_bounds = array` — the grid shape + /// (one entry per `dimension_semantics`). `null` omits the attr. + iteration_bounds: ?[]const i64 = null, + /// When `true`, auto-emit `window_params = [...]` and trailing + /// `transform_N` stub funcs at `finish` time. Per-arg behavior + /// matches `lowering.py:1014–1131`: ANY/HBM/SEMAPHORE refs get + /// an empty `{}` and no transform; VMEM trivial windows + /// auto-force `pipeline_mode = synchronous`. Override per-arg + /// via `RefSpec.window`. + pallas_window_params: bool = false, + }; + + /// Create a sub-module containing a single public `func.func` of the + /// given name, signature, and result types. + pub fn init( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + args: []const ArgSpec, + result_types: []const *const mlir.Type, + ) !Builder { + return initOpts(allocator, ctx, name, args, result_types, .{}); + } + + pub fn initOpts( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + args: []const ArgSpec, + result_types: []const *const mlir.Type, + opts: Opts, + ) !Builder { + var b = try open(allocator, ctx, name); + errdefer b.deinit(); + try b.declareArgsLowOpts(args, result_types, opts); + return b; + } + + /// Create an empty `Builder` with no `func.func` yet. + pub fn open( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + ) !Builder { + const unknown_loc: *const mlir.Location = .unknown(ctx); + const module: *mlir.Module = .init(unknown_loc); + errdefer module.deinit(); + + var arena: std.heap.ArenaAllocator = .init(allocator); + errdefer arena.deinit(); + + return .{ + .allocator = allocator, + .arena = arena, + .ctx = ctx, + .module = module, + .name = name, + .func_op = null, + .entry_block = null, + .block_stack = .empty, + }; + } + + /// Declare the kernel's args from a named-field struct literal and + /// return a `NamedArgs(Spec)`. + pub fn declareArgs(self: *Builder, spec: anytype) !dsl.NamedArgs(@TypeOf(spec), Value) { + return self.declareArgsOpts(spec, &.{}, .{}); + } + + pub fn declareArgsOpts( + self: *Builder, + spec: anytype, + result_types: []const *const mlir.Type, + opts: Opts, + ) !dsl.NamedArgs(@TypeOf(spec), Value) { + const Spec = @TypeOf(spec); + const fields = @typeInfo(Spec).@"struct".fields; + + var arg_specs: [fields.len]ArgSpec = undefined; + inline for (fields, 0..) |f, i| { + const raw = @field(spec, f.name); + const kind: ArgSpec.Kind = if (@TypeOf(raw) == ArgSpec.Kind) raw else blk: { + const variant = @typeInfo(@TypeOf(raw)).@"struct".fields[0].name; + const tag = @field(std.meta.Tag(ArgSpec.Kind), variant); + const inner = @field(raw, variant); + break :blk switch (tag) { + .scalar => .{ .scalar = inner }, + .iter => .{ .iter = .{ + .semantics = if (@hasField(@TypeOf(inner), "semantics")) inner.semantics else null, + } }, + .ref => .{ .ref = .{ + .shape = inner.shape, + .dtype = inner.dtype, + .memory_space = if (@hasField(@TypeOf(inner), "memory_space")) inner.memory_space else .vmem, + .window = if (@hasField(@TypeOf(inner), "window")) inner.window else null, + .role = if (@hasField(@TypeOf(inner), "role")) inner.role else .input, + } }, + .sem_array => .{ .sem_array = .{ + .shape = inner.shape, + .kind = if (@hasField(@TypeOf(inner), "kind")) inner.kind else .dma, + } }, + .sem => .{ .sem = inner }, + }; + }; + arg_specs[i] = .{ .name = f.name, .kind = kind }; + } + + try self.declareArgsLowOpts(&arg_specs, result_types, opts); + + var named: dsl.NamedArgs(Spec, Value) = undefined; + inline for (fields, 0..) |f, i| { + @field(named, f.name) = self.arg(i); + } + return named; + } + + fn declareArgsLowOpts( + self: *Builder, + args: []const ArgSpec, + result_types: []const *const mlir.Type, + opts: Opts, + ) !void { + std.debug.assert(self.entry_block == null); + const ctx = self.ctx; + const unknown_loc: *const mlir.Location = .unknown(ctx); + const scratch = self.arena.allocator(); + + const arg_types = try scratch.alloc(*const mlir.Type, args.len); + const arg_locs = try scratch.alloc(*const mlir.Location, args.len); + for (args, 0..) |a, i| { + arg_types[i] = switch (a.kind) { + .scalar => |dt| dt.toMlir(ctx), + .iter => mlir.indexType(ctx), + .ref => |r| mlir.memRefType( + r.dtype.toMlir(ctx), + r.shape, + null, + r.memory_space.attribute(ctx), + ), + .sem => |s| switch (s) { + .regular => tpu.semaphoreType(ctx), + .dma => tpu.dmaSemaphoreType(ctx), + }, + .sem_array => |sa| mlir.memRefType( + switch (sa.kind) { + .regular => tpu.semaphoreType(ctx), + .dma => tpu.dmaSemaphoreType(ctx), + }, + sa.shape, + null, + MemorySpace.semaphore_mem.attribute(ctx), + ), + }; + arg_locs[i] = unknown_loc.named(ctx, a.name); + } + + const entry = mlir.Block.init(arg_types, arg_locs); + + // Build per-arg attribute dicts. The only ones we currently emit are + // `tpu.dimension_semantics` on iter args. + const arg_attrs = try scratch.alloc(*const mlir.Attribute, args.len); + var any_arg_attr = false; + for (args, 0..) |a, i| { + const empty_dict: *const mlir.Attribute = mlir.dictionaryAttribute(ctx, &.{}); + switch (a.kind) { + .iter => |io| { + if (io.semantics) |sem| { + arg_attrs[i] = mlir.dictionaryAttribute(ctx, &.{ + .named(ctx, "tpu.dimension_semantics", sem.attribute(ctx)), + }); + any_arg_attr = true; + } else { + arg_attrs[i] = empty_dict; + } + }, + else => arg_attrs[i] = empty_dict, + } + } + + // Kernel-level func.func attributes. MLIR sorts attrs alphabetically on print. + var extra_attrs: stdx.BoundedArray(mlir.NamedAttribute, 8) = .empty; + const dim_sem_buf = try scratch.alloc(*const mlir.Attribute, opts.dimension_semantics.len); + for (opts.dimension_semantics, 0..) |s, i| dim_sem_buf[i] = s.attribute(ctx); + extra_attrs.appendAssumeCapacity(.named(ctx, "dimension_semantics", mlir.arrayAttribute(ctx, dim_sem_buf))); + if (opts.iteration_bounds) |bounds| { + extra_attrs.appendAssumeCapacity(.named(ctx, "iteration_bounds", mlir.denseArrayAttribute(ctx, .i64, bounds))); + } + extra_attrs.appendAssumeCapacity(.named(ctx, "scalar_prefetch", mlir.integerAttribute(ctx, .i64, opts.scalar_prefetch))); + extra_attrs.appendAssumeCapacity(.named(ctx, "scratch_operands", mlir.integerAttribute(ctx, .i64, opts.scratch_operands))); + if (opts.core_type) |ct| { + extra_attrs.appendAssumeCapacity(.named(ctx, "tpu.core_type", ct.attribute(ctx))); + } + + // Snapshot the ref args for `finish`-time stub emission. Only + // `.input` / `.output` refs participate in `window_params` and + // `transform_N` — `.scalar_prefetch` and `.scratch` are ignored + // (they're tracked separately for transform-arg expansion). + const arena_alloc = self.arena.allocator(); + var ref_args_buf: std.ArrayList(RefArgSnapshot) = .empty; + var prefetch_types_buf: std.ArrayList(*const mlir.Type) = .empty; + for (args, 0..) |a, i| switch (a.kind) { + .ref => |r| { + if (r.role == .input or r.role == .output) { + const block_shape = if (r.window) |w| + (w.block_shape orelse r.shape) + else + r.shape; + try ref_args_buf.append(arena_alloc, .{ + .block_rank = @intCast(block_shape.len), + .skipped = isSkippedWindowSpace(r.memory_space), + .role = r.role, + .transform_returns = if (r.window) |w| w.transform_returns else null, + }); + } else if (r.role == .scalar_prefetch) { + try prefetch_types_buf.append(arena_alloc, arg_types[i]); + } + }, + else => {}, + }; + self.ref_args = try arena_alloc.dupe(RefArgSnapshot, ref_args_buf.items); + self.prefetch_args = try arena_alloc.dupe(*const mlir.Type, prefetch_types_buf.items); + self.grid_dim_count = opts.dimension_semantics.len; + self.emit_transform_stubs = opts.pallas_window_params; + + // window_params: one entry per `.input` / `.output` ref. + // ANY/HBM/SEMAPHORE → empty dict, no transform_N. + // Otherwise → {transform_indices, window_bounds, [pipeline_mode], [revisit_mode], [window_kind]}. + // VMEM trivial-window → auto-force pipeline_mode = synchronous. + if (opts.pallas_window_params) { + var idx: usize = 0; + const wp_buf = try scratch.alloc(*const mlir.Attribute, self.ref_args.len); + for (args) |a| switch (a.kind) { + .ref => |r| { + if (r.role != .input and r.role != .output) continue; + if (isSkippedWindowSpace(r.memory_space)) { + wp_buf[idx] = mlir.dictionaryAttribute(ctx, &.{}); + idx += 1; + continue; + } + + const w = r.window orelse ArgSpec.WindowSpec{}; + const block_shape = w.block_shape orelse r.shape; + // `bm.has_trivial_window()` in pallas — block_shape == + // operand_shape AND the index_map is the identity + // (returns all zeros). + const has_trivial_index_map = if (w.transform_returns) |trs| blk: { + for (trs) |tr| { + if (tr != .zero) break :blk false; + } + break :blk true; + } else true; + const trivial = std.mem.eql(i64, block_shape, r.shape) and has_trivial_index_map; + + const resolved_pipeline: ?PipelineMode = + w.pipeline_mode orelse blk: { + // VMEM + trivial window ⇒ Buffered(1) ⇒ synchronous. + if (r.memory_space == .vmem and trivial) break :blk .synchronous; + break :blk null; + }; + + var sym_buf: [32]u8 = undefined; + const sym_name = std.fmt.bufPrint(&sym_buf, "transform_{d}", .{idx}) catch unreachable; + + var inner: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (resolved_pipeline) |pm| { + inner.appendAssumeCapacity(.named(ctx, "pipeline_mode", pm.attribute(ctx))); + } + if (w.revisit_mode) |rm| { + inner.appendAssumeCapacity(.named(ctx, "revisit_mode", rm.attribute(ctx))); + } + inner.appendAssumeCapacity(.named(ctx, "transform_indices", mlir.flatSymbolRefAttribute(ctx, sym_name))); + inner.appendAssumeCapacity(.named(ctx, "window_bounds", mlir.denseArrayAttribute(ctx, .i64, block_shape))); + if (w.element_window) |ew| { + inner.appendAssumeCapacity(.named(ctx, "window_kind", tpu.elementWindowAttribute(ctx, ew.pad_low, ew.pad_high))); + } + wp_buf[idx] = mlir.dictionaryAttribute(ctx, inner.constSlice()); + idx += 1; + }, + else => {}, + }; + extra_attrs.appendAssumeCapacity(.named(ctx, "window_params", mlir.arrayAttribute(ctx, wp_buf))); + } + + const func_op = func.func(ctx, .{ + .name = self.name, + .args = arg_types, + .args_attributes = if (any_arg_attr) arg_attrs else null, + .results = result_types, + .block = entry, + .location = unknown_loc, + .no_inline = opts.@"noinline", + // Body is empty at declare time — verify only at `finish`. + .verify = false, + .extra_attributes = extra_attrs.constSlice(), + // sym_visibility omitted — defaults to public, elided when unset. + .visibility = null, + }); + _ = func_op.appendTo(self.module.body()); + + self.func_op = func_op; + self.entry_block = entry; + } + + pub fn deinit(self: *Builder) void { + self.module.deinit(); + self.arena.deinit(); + } + + /// The i-th entry block argument. + pub fn arg(self: *Builder, i: usize) Value { + const eb = self.entry_block orelse @panic("Builder.arg called before declareArgs"); + return .{ .inner = eb.argument(i), .kernel = self }; + } + + pub fn pushBlock(self: *Builder, b: *mlir.Block) void { + self.block_stack.append(self.arena.allocator(), b) catch @panic("Builder.pushBlock OOM"); + } + + pub fn popBlock(self: *Builder) void { + _ = self.block_stack.pop(); + } + + pub fn currentBlock(self: *Builder) *mlir.Block { + if (self.block_stack.items.len > 0) { + return self.block_stack.items[self.block_stack.items.len - 1]; + } + return self.entry_block orelse @panic("Builder has no current block — call declareArgs first"); + } + + /// Unpack a slice of `Value`s into an arena-allocated slice of raw MLIR + /// value pointers. + fn innerSlice(self: *Builder, values: []const Value) []const *const mlir.Value { + const out = self.arena.allocator().alloc(*const mlir.Value, values.len) catch @panic("Builder.innerSlice OOM"); + for (values, 0..) |v, i| out[i] = v.inner; + return out; + } + + /// Emit `op` into the current block and return its first result as a Value. + pub fn emit(self: *Builder, op: *mlir.Operation) Value { + _ = op.appendTo(self.currentBlock()); + return .{ .inner = op.result(0), .kernel = self }; + } + + pub fn emitMulti(self: *Builder, op: *mlir.Operation, n: usize) []Value { + _ = op.appendTo(self.currentBlock()); + const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.emitMulti OOM"); + for (0..n) |i| out[i] = .{ .inner = op.result(i), .kernel = self }; + return out; + } + + pub fn yield(self: *Builder, values: anytype) []const Value { + const T = @TypeOf(values); + const info = @typeInfo(T); + if (info != .@"struct" or !info.@"struct".is_tuple) + @compileError("Builder.yield expects a tuple literal like `.{ v1, v2 }`"); + const n = info.@"struct".fields.len; + const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.yield OOM"); + inline for (info.@"struct".fields, 0..) |f, i| { + if (f.type != Value) + @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(f.type)); + out[i] = @field(values, f.name); + } + return out; + } + + pub fn loc(self: *const Builder) *const mlir.Location { + return .unknown(self.ctx); + } + + // ==================== type helpers ==================== + + pub fn scalarTy(self: *const Builder, dtype: DType) *const mlir.Type { + return dtype.toMlir(self.ctx); + } + + pub fn vectorTy(self: *const Builder, shape: []const i64, dtype: DType) *const mlir.Type { + return mlir.vectorType(shape, dtype.toMlir(self.ctx)); + } + + pub fn memRefTy(self: *const Builder, shape: []const i64, dtype: DType, memory_space: MemorySpace) *const mlir.Type { + return mlir.memRefType(dtype.toMlir(self.ctx), shape, null, memory_space.attribute(self.ctx)); + } + + fn mlirElemToDType(self: *const Builder, elem: *const mlir.Type) DType { + return dtypes.mlirElemToDType(self.ctx, elem); + } + + /// Replace `src`'s element type with `out_dtype`, preserving rank/shape. + fn swapElem(self: *const Builder, src: Value, out_dtype: DType) *const mlir.Type { + const out_elem = out_dtype.toMlir(self.ctx); + if (src.isVector()) return mlir.vectorType(src.shape().constSlice(), out_elem); + if (src.isMemRef()) { + const mr = src.type_().isA(mlir.MemRefType).?; + return mlir.memRefType(out_elem, src.shape().constSlice(), null, mr.memorySpace()); + } + return out_elem; + } + + // ==================== constants ==================== + + fn emitInt(self: *Builder, dtype: DType, value: i64) Value { + return self.emit(arith.constant_int(self.ctx, value, dtype.toMlir(self.ctx), self.loc())); + } + + fn emitFloat(self: *Builder, dtype: DType, value: f64) Value { + return switch (dtype) { + .f16 => self.emit(arith.constant_float(self.ctx, value, .f16, self.loc())), + .bf16 => self.emit(arith.constant_float(self.ctx, value, .bf16, self.loc())), + .f32 => self.emit(arith.constant_float(self.ctx, value, .f32, self.loc())), + .f64 => self.emit(arith.constant_float(self.ctx, value, .f64, self.loc())), + // fp8 has no direct arith.constant; emit f32 then truncf. + else => self.arithTruncf(self.emitFloat(.f32, value), dtype), + }; + } + + /// Constant `index`-typed value (for memref / loop bounds). + pub fn cIndex(self: *Builder, value: i64) Value { + return self.emit(arith.constant_index(self.ctx, value, self.loc())); + } + + /// Lift a Zig scalar (or pass-through Value) to a DSL constant. + pub fn lift(self: *Builder, value: anytype) Value { + const T = @TypeOf(value); + if (T == Value) return value; + return switch (@typeInfo(T)) { + .comptime_int => blk: { + if (value >= std.math.minInt(i32) and value <= std.math.maxInt(i32)) + break :blk self.emitInt(.i32, value); + if (value >= std.math.minInt(i64) and value <= std.math.maxInt(i64)) + break :blk self.emitInt(.i64, value); + if (value >= 0 and value <= std.math.maxInt(u64)) + break :blk self.emitInt(.i64, @bitCast(@as(u64, value))); + @compileError("Builder.lift: integer literal out of 64-bit range"); + }, + .int => |info| blk: { + const signed = info.signedness == .signed; + if (info.bits > 32) { + const v: i64 = if (signed) @intCast(value) else @bitCast(@as(u64, @intCast(value))); + break :blk self.emitInt(.i64, v); + } + const v32: i32 = if (signed) @intCast(value) else @bitCast(@as(u32, @intCast(value))); + break :blk self.emitInt(.i32, v32); + }, + .comptime_float => self.emitFloat(.f32, @floatCast(value)), + .float => |info| switch (info.bits) { + 16 => self.emitFloat(.f16, @floatCast(value)), + 32 => self.emitFloat(.f32, @floatCast(value)), + 64 => self.emitFloat(.f64, value), + else => @compileError("Builder.lift: unsupported float bitwidth"), + }, + else => @compileError("Builder.lift: unsupported type " ++ @typeName(T)), + }; + } + + /// Lift `value` and pin its dtype to `dtype`. Unsigned ints bit-cast + /// preserving the source bit-pattern. + pub fn liftAs(self: *Builder, value: anytype, dtype: DType) Value { + const T = @TypeOf(value); + if (T == Value) return value; + if (isFloatDtype(dtype)) { + const v_f64: f64 = switch (@typeInfo(T)) { + .comptime_int, .int => @floatFromInt(value), + .comptime_float, .float => @floatCast(value), + else => @compileError("Builder.liftAs: unsupported scalar " ++ @typeName(T)), + }; + return self.emitFloat(dtype, v_f64); + } + const v_i64: i64 = switch (@typeInfo(T)) { + .comptime_int => blk: { + if (value >= std.math.minInt(i64) and value <= std.math.maxInt(i64)) + break :blk @intCast(value); + if (value >= 0 and value <= std.math.maxInt(u64)) + break :blk @bitCast(@as(u64, value)); + @compileError("Builder.liftAs: integer literal out of 64-bit range"); + }, + .int => |info| if (info.signedness == .signed) @intCast(value) else @bitCast(@as(u64, @intCast(value))), + .comptime_float, .float => @intFromFloat(value), + else => @compileError("Builder.liftAs: unsupported scalar " ++ @typeName(T)), + }; + return self.emitInt(dtype, v_i64); + } + + /// Build an arith.constant of the given MLIR element type from a Zig + /// numeric. Used for matching the type of an existing reference value. + pub fn constMatching(self: *Builder, value: anytype, elem: *const mlir.Type) Value { + if (elem.eql(mlir.indexType(self.ctx))) return self.cIndex(@intCast(value)); + return self.liftAs(value, self.mlirElemToDType(elem)); + } + + fn liftMatching(self: *Builder, value: anytype, ref_elem: *const mlir.Type) Value { + const T = @TypeOf(value); + if (T == Value) return value; + return switch (@typeInfo(T)) { + .comptime_int, .comptime_float => self.constMatching(value, ref_elem), + else => self.lift(value), + }; + } + + /// Splat a scalar to a vector. Zig scalars take the dense-splat fast path + /// (`arith.constant dense`); a `Value` falls back to `vector.broadcast`. + pub fn splat(self: *Builder, value: anytype, shape: []const i64, dtype: DType) Value { + const T = @TypeOf(value); + const ty = mlir.vectorType(shape, dtype.toMlir(self.ctx)); + if (T == Value) return self.emit(vector.broadcast(self.ctx, value.inner, ty, self.loc())); + if (isFloatDtype(dtype)) { + const v_f64: f64 = switch (@typeInfo(T)) { + .comptime_int, .int => @floatFromInt(value), + .comptime_float, .float => @floatCast(value), + else => @compileError("Builder.splat: unsupported scalar " ++ @typeName(T)), + }; + return self.emit(arith.constant_dense_splat_f(self.ctx, v_f64, ty, self.loc())); + } + const v_i64: i64 = switch (@typeInfo(T)) { + .comptime_int => @intCast(value), + .int => |info| if (info.signedness == .signed) @intCast(value) else @bitCast(@as(u64, @intCast(value))), + .comptime_float, .float => @intFromFloat(value), + else => @compileError("Builder.splat: unsupported scalar " ++ @typeName(T)), + }; + return self.emit(arith.constant_dense_splat_i(self.ctx, v_i64, ty, self.loc())); + } + + /// Zero-valued vector of `shape x dtype`. + pub fn zeros(self: *Builder, shape: []const i64, dtype: DType) Value { + return self.splat(@as(i64, 0), shape, dtype); + } + + pub fn ones(self: *Builder, shape: []const i64, dtype: DType) Value { + return self.splat(@as(i64, 1), shape, dtype); + } + + pub fn full(self: *Builder, shape: []const i64, value: anytype, dtype: DType) Value { + return self.splat(value, shape, dtype); + } + + /// Lift scalars and broadcast scalar → vector to match the other operand. + pub fn coerce(self: *Builder, a: anytype, b: anytype) struct { Value, Value } { + const a_ref: ?*const mlir.Type = if (@TypeOf(b) == Value) b.elemType() else null; + const b_ref: ?*const mlir.Type = if (@TypeOf(a) == Value) a.elemType() else null; + var av = if (a_ref) |t| self.liftMatching(a, t) else self.lift(a); + var bv = if (b_ref) |t| self.liftMatching(b, t) else self.lift(b); + + // integer-width promotion (int family only). + if (av.isIntElem() and bv.isIntElem()) { + const a_dt = self.mlirElemToDType(av.elemType()); + const b_dt = self.mlirElemToDType(bv.elemType()); + const aw = intBitwidth(a_dt); + const bw = intBitwidth(b_dt); + if (aw < bw) av = av.to(b_dt); + if (bw < aw) bv = bv.to(a_dt); + } + + // scalar↔vector splat. + if (av.isVector() and !bv.isVector()) { + const dt = self.mlirElemToDType(av.elemType()); + bv = self.splat(bv, av.shape().constSlice(), dt); + } + if (!av.isVector() and bv.isVector()) { + const dt = self.mlirElemToDType(bv.elemType()); + av = self.splat(av, bv.shape().constSlice(), dt); + } + + return .{ av, bv }; + } + + // ==================== arith: binary ==================== + + pub fn addi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.addi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn subi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.subi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn muli(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.muli(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn addf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.addf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn subf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.subf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn mulf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.mulf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn divf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.divf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn divsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.divsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn divui(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.divui(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + /// Mirrors `jax.numpy.floor_divide` (`jax/_src/numpy/ufuncs.py:2487`): + /// truncating `divsi` plus the sign-correcting expansion + /// `q - 1 if (sign(a) != sign(b) and rem != 0) else q`. + /// `lhs` must be a vector of i32. `rhs` is `anytype` — passing a + /// scalar comptime int produces the dense<...> splat Pallas emits. + /// JAX's MLIR canonicalizer does not fold the expansion away even + /// when one operand is a constant, so this helper is needed for + /// IR-equivalent floor-div with `pallas_call`. + pub fn divFloor(self: *Builder, lhs: Value, rhs: anytype) Value { + const shape = lhs.shape().constSlice(); + const rhs_v: Value = if (@TypeOf(rhs) == Value) rhs else self.splat(rhs, shape, .i32); + const zero_v = self.zeros(shape, .i32); + const one_v = self.ones(shape, .i32); + const q = self.divsi(lhs, rhs_v); + const sign_a = self.subi( + self.extui(self.cmpi(.sgt, lhs, zero_v), .i32), + self.extui(self.cmpi(.slt, lhs, zero_v), .i32), + ); + const sign_b = self.subi( + self.extui(self.cmpi(.sgt, rhs_v, zero_v), .i32), + self.extui(self.cmpi(.slt, rhs_v, zero_v), .i32), + ); + const sign_diff = self.cmpi(.ne, sign_a, sign_b); + const rem_nz = self.cmpi(.ne, self.remsi(lhs, rhs_v), zero_v); + const cond = self.andi(sign_diff, rem_nz); + return self.select(cond, self.subi(q, one_v), q); + } + + pub fn remsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.remsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn remui(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.remui(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn remf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.remf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn andi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.andi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn ori(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.ori(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn xori(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.xori(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn shli(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.shli(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn shrsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.shrsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn shrui(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.shrui(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn maxsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.maxsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn minsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.minsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn maximumf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.maximumf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn minimumf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.minimumf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn maxnumf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.maxnumf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn minnumf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.minnumf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn negf(self: *Builder, src: Value) Value { + return self.emit(arith.negf(self.ctx, src.inner, self.loc())); + } + pub fn cmpi(self: *Builder, predicate: arith.CmpIPredicate, lhs: Value, rhs: Value) Value { + return self.emit(arith.cmpi(self.ctx, predicate, lhs.inner, rhs.inner, self.loc())); + } + pub fn cmpf(self: *Builder, predicate: arith.CmpFPredicate, lhs: Value, rhs: Value) Value { + return self.emit(arith.cmpf(self.ctx, predicate, lhs.inner, rhs.inner, self.loc())); + } + pub fn select(self: *Builder, cond: Value, t: Value, f: Value) Value { + return self.emit(arith.select(self.ctx, cond.inner, t.inner, f.inner, self.loc())); + } + pub fn where(self: *Builder, condition: Value, x: Value, y: Value) Value { + const xv, const yv = self.coerce(x, y); + const c = if (condition.isVector() == xv.isVector()) condition else blk: { + // Splat a scalar predicate to match a vector x/y. + if (xv.isVector()) { + break :blk self.splat(condition, xv.shape().constSlice(), .i1); + } + break :blk condition; + }; + return self.select(c, xv, yv); + } + + /// Elementwise max with auto-broadcast scalar→vector. Use `maxOpts` + /// for NaN-propagation control. + pub fn maximum(self: *Builder, a: anytype, b: anytype) Value { + return self.maximumOpts(a, b, .{}); + } + pub fn maximumOpts(self: *Builder, a: anytype, b: anytype, opts: MinMaxOpts) Value { + const l, const r = self.coerce(a, b); + if (l.isFloatElem()) { + return if (opts.propagate_nan) self.maximumf(l, r) else self.maxnumf(l, r); + } + return self.maxsi(l, r); + } + pub fn minimum(self: *Builder, a: anytype, b: anytype) Value { + return self.minimumOpts(a, b, .{}); + } + pub fn minimumOpts(self: *Builder, a: anytype, b: anytype, opts: MinMaxOpts) Value { + const l, const r = self.coerce(a, b); + if (l.isFloatElem()) { + return if (opts.propagate_nan) self.minimumf(l, r) else self.minnumf(l, r); + } + return self.minsi(l, r); + } + + // ==================== arith casts ==================== + + pub fn extsi(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.extsi(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn extui(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.extui(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn arithExtf(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.extf(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn trunci(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.trunci(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn arithTruncf(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.truncf(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn arithSitofp(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.sitofp(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn arithUitofp(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.uitofp(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn arithFptosi(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.fptosi(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn arithFptoui(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.fptoui(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn arithBitcast(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.bitcast(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + + /// `arith.index_cast` — cast between integer and `index` types. + pub fn indexCast(self: *Builder, src: Value, target: *const mlir.Type) Value { + return self.emit(arith.index_cast(self.ctx, src.inner, target, self.loc())); + } + + /// Cast a scalar integer (i32 / i64 / …) to the `index` type. Common + /// pattern for using i32 program-ids and scalar-prefetch values as + /// memref offsets. + pub fn toIndex(self: *Builder, src: Value) Value { + return self.indexCast(src, mlir.indexType(self.ctx)); + } + + // ==================== TPU casts ==================== + + /// `tpu.bitcast` — same-bitwidth element-type swap. Shape preserved. + pub fn bitcastTo(self: *Builder, src: Value, dt: DType) Value { + return self.emit(tpu.bitcast(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn extf(self: *Builder, src: Value, dt: DType) Value { + return self.emit(tpu.extf(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn truncf(self: *Builder, src: Value, dt: DType) Value { + return self.emit(tpu.truncf(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn sitofp(self: *Builder, src: Value, dt: DType) Value { + return self.emit(tpu.sitofp(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn uitofp(self: *Builder, src: Value, dt: DType) Value { + return self.emit(tpu.uitofp(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn fptosi(self: *Builder, src: Value, dt: DType) Value { + return self.emit(tpu.fptosi(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn fptoui(self: *Builder, src: Value, dt: DType) Value { + return self.emit(tpu.fptoui(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + + /// Numeric cast with auto-dispatch. Routes through `arith.*` ops for the + /// common float / int conversions (which don't carry a rounding-mode + /// attribute). The `tpu.*` cast variants (`tpuTruncf`, `tpuFptosi`, …) + /// are exposed separately for callers that need rounding-mode control — + /// pass `opts.rounding` to opt in. + pub fn cast(self: *Builder, src: Value, dt: DType) Value { + return self.castOpts(src, dt, .{}); + } + pub fn castOpts(self: *Builder, src: Value, dt: DType, opts: CastOpts) Value { + if (opts.bitcast) return self.bitcastTo(src, dt); + const cur_elem = src.elemType(); + const tgt_elem = dt.toMlir(self.ctx); + if (cur_elem.eql(tgt_elem)) return src; + const cur_dtype = self.mlirElemToDType(cur_elem); + const cur_is_float = src.isFloatElem(); + const tgt_is_float = isFloatDtype(dt); + if (cur_is_float and tgt_is_float) { + const cur_bw = dtypeBitwidth(cur_dtype); + const tgt_bw = dtypeBitwidth(dt); + if (tgt_bw > cur_bw) return self.arithExtf(src, dt); + return self.arithTruncf(src, dt); + } + if (cur_is_float and !tgt_is_float) return self.arithFptosi(src, dt); + if (!cur_is_float and tgt_is_float) { + if (cur_dtype == .i1) return self.arithUitofp(src, dt); + return self.arithSitofp(src, dt); + } + const cur_bw = dtypeBitwidth(cur_dtype); + const tgt_bw = dtypeBitwidth(dt); + if (tgt_bw > cur_bw) return self.extsi(src, dt); + if (tgt_bw < cur_bw) return self.trunci(src, dt); + return self.arithBitcast(src, dt); + } + + // ==================== math dialect ==================== + + pub fn exp(self: *Builder, x: Value) Value { + return self.emit(math.exp(self.ctx, x.inner, self.loc())); + } + /// `exp2(x)` → `exp(x * ln(2))`. Matches Pallas's TPU lowering rule + /// (`jax/_src/pallas/mosaic/lowering.py:_exp2_lowering_rule`), which + /// expands `lax.exp2_p` into the same `x * ln2` pattern in forward-compat + /// mode. Materializes as an `arith.constant` (matching x's element type) + /// + `arith.mulf x, ln2` + `math.exp`. For the raw `math.exp2` op + /// (post-2025-07-26 cloud TPUs), use `mathExp2`. + pub fn exp2(self: *Builder, x: Value) Value { + const ln2 = self.liftAs(@as(f64, 0.6931471805599453), self.mlirElemToDType(x.elemType())); + const ln2_bc = if (x.isShaped()) self.broadcastTo(ln2, x.shape().constSlice()) else ln2; + return self.exp(self.mulf(x, ln2_bc)); + } + pub fn mathExp2(self: *Builder, x: Value) Value { + return self.emit(math.exp2(self.ctx, x.inner, self.loc())); + } + pub fn log(self: *Builder, x: Value) Value { + return self.emit(math.log(self.ctx, x.inner, self.loc())); + } + pub fn log2(self: *Builder, x: Value) Value { + return self.emit(math.log2(self.ctx, x.inner, self.loc())); + } + pub fn sqrt(self: *Builder, x: Value) Value { + return self.emit(math.sqrt(self.ctx, x.inner, self.loc())); + } + pub fn rsqrt(self: *Builder, x: Value) Value { + return self.emit(math.rsqrt(self.ctx, x.inner, self.loc())); + } + pub fn sin(self: *Builder, x: Value) Value { + return self.emit(math.sin(self.ctx, x.inner, self.loc())); + } + pub fn cos(self: *Builder, x: Value) Value { + return self.emit(math.cos(self.ctx, x.inner, self.loc())); + } + pub fn tanh(self: *Builder, x: Value) Value { + return self.emit(math.tanh(self.ctx, x.inner, self.loc())); + } + pub fn erf(self: *Builder, x: Value) Value { + return self.emit(math.erf(self.ctx, x.inner, self.loc())); + } + pub fn absf(self: *Builder, x: Value) Value { + return self.emit(math.absf(self.ctx, x.inner, self.loc())); + } + pub fn absi(self: *Builder, x: Value) Value { + return self.emit(math.absi(self.ctx, x.inner, self.loc())); + } + pub fn abs(self: *Builder, x: Value) Value { + if (x.isFloatElem()) return self.absf(x); + return self.absi(x); + } + pub fn floor(self: *Builder, x: Value) Value { + return self.emit(math.floor(self.ctx, x.inner, self.loc())); + } + pub fn ceil(self: *Builder, x: Value) Value { + return self.emit(math.ceil(self.ctx, x.inner, self.loc())); + } + pub fn powf(self: *Builder, x: Value, y: Value) Value { + return self.emit(math.powf(self.ctx, x.inner, y.inner, self.loc())); + } + pub fn fma(self: *Builder, a: Value, b: Value, c_: Value) Value { + return self.emit(math.fma(self.ctx, a.inner, b.inner, c_.inner, self.loc())); + } + + // ==================== TPU memory: vector_load / vector_store / scalar load/store ==================== + + /// Build the result vector type for a full-shape `tpu.vector_load` from + /// the input ref's shape + element type. + fn refVectorType(self: *Builder, ref: Value) *const mlir.Type { + _ = self; + const mr = ref.type_().isA(mlir.MemRefType) orelse @panic("Builder.refVectorType: not a memref"); + var s: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..mr.rank()) |i| s.appendAssumeCapacity(mr.dimension(i)); + return mlir.vectorType(s.constSlice(), mr.elementType()); + } + + /// Full-shape `vector.load`. Use `tpuVectorLoad` for sublane stride / masking. + pub fn refLoad(self: *Builder, ref: Value) Value { + const result_ty = self.refVectorType(ref); + const idx_inner = self.indicesOrZero(ref, &.{}); + return self.emit(vector.load(self.ctx, ref.inner, idx_inner, result_ty, .{}, self.loc())); + } + + /// `vector.load(ref, indices)` — explicit `index`-typed offsets. + pub fn vectorLoadAt(self: *Builder, ref: Value, indices: []const Value) Value { + const result_ty = self.refVectorType(ref); + const idx_inner = self.indicesOrZero(ref, indices); + return self.emit(vector.load(self.ctx, ref.inner, idx_inner, result_ty, .{}, self.loc())); + } + + /// `tpu.vector_load` — sublane stride + optional element-wise mask. + pub fn tpuVectorLoad(self: *Builder, ref: Value, indices: []const Value, opts: VectorLoadOpts) Value { + const ctx = self.ctx; + const result_ty = self.refVectorType(ref); + const idx_inner = self.indicesOrZero(ref, indices); + const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; + return self.emit(tpu.vector_load( + ctx, + ref.inner, + idx_inner, + mask_inner, + .{ .strides = opts.strides }, + result_ty, + self.loc(), + )); + } + + /// `tpu.vector_store(value, ref)` — full-shape vector store. + pub fn refStore(self: *Builder, ref: Value, value: Value) void { + self.refStoreOpts(ref, value, &.{}, .{}); + } + + pub fn vectorStoreAt(self: *Builder, ref: Value, value: Value, indices: []const Value) void { + self.refStoreOpts(ref, value, indices, .{}); + } + + pub fn refStoreOpts(self: *Builder, ref: Value, value: Value, indices: []const Value, opts: VectorStoreOpts) void { + const idx_inner = self.indicesOrZero(ref, indices); + const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; + _ = tpu.vector_store( + self.ctx, + value.inner, + ref.inner, + idx_inner, + mask_inner, + .{ .strides = opts.strides, .add = opts.add }, + self.loc(), + ).appendTo(self.currentBlock()); + } + + /// `memref.load` — scalar load. Indices must be `index`-typed. + pub fn scalarLoad(self: *Builder, ref: Value, indices: []const Value) Value { + const idx_inner = self.indicesOrZero(ref, indices); + return self.emit(memref.load(self.ctx, ref.inner, idx_inner, .{}, self.loc())); + } + + /// `memref.store` — scalar store. + pub fn scalarStore(self: *Builder, ref: Value, value: Value, indices: []const Value) void { + const idx_inner = self.indicesOrZero(ref, indices); + _ = memref.store(self.ctx, value.inner, ref.inner, idx_inner, .{}, self.loc()).appendTo(self.currentBlock()); + } + + /// Returns a `*const mlir.Value` slice of length `ref.rank()`, padding with + /// a cached `arith.constant 0 : index` when `indices` is empty. + fn indicesOrZero(self: *Builder, ref: Value, indices: []const Value) []const *const mlir.Value { + const r = ref.rank(); + if (indices.len == r) return self.innerSlice(indices); + std.debug.assert(indices.len == 0); + const zero = self.zeroIndex(); + const out = self.arena.allocator().alloc(*const mlir.Value, r) catch @panic("Builder.indicesOrZero OOM"); + for (out) |*p| p.* = zero.inner; + return out; + } + + /// Return a cached `arith.constant 0 : index` Value, lazily emitting + /// it the first time it's needed. The cache is invalidated whenever + /// the insertion block changes — `index` constants are scoped to the + /// block they're emitted in. + fn zeroIndex(self: *Builder) Value { + const block = self.currentBlock(); + if (self.zero_index) |z| { + if (self.zero_index_block == block) return z; + } + const z = self.cIndex(0); + self.zero_index = z; + self.zero_index_block = block; + return z; + } + + /// `tpu.strided_load` — multi-dim strided load. Result shape is derived + /// from `ref.shape / strides` (per-dim ceil divide). + pub fn stridedLoad(self: *Builder, ref: Value, indices: []const Value, strides: []const i32) Value { + const mr = ref.type_().isA(mlir.MemRefType) orelse @panic("stridedLoad: not a memref"); + var s: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..mr.rank()) |i| { + const stride: i64 = if (i < strides.len) strides[i] else 1; + const dim = mr.dimension(i); + s.appendAssumeCapacity(@divTrunc(dim + stride - 1, stride)); + } + const result_ty = mlir.vectorType(s.constSlice(), mr.elementType()); + return self.emit(tpu.strided_load(self.ctx, ref.inner, self.innerSlice(indices), strides, result_ty, self.loc())); + } + + /// `tpu.strided_load` with an explicit result shape — when you know the + /// load shape doesn't match the simple `ref.shape / strides` form (e.g. + /// inner-dim crop). + pub fn stridedLoadShape(self: *Builder, ref: Value, indices: []const Value, strides: []const i32, result_shape: []const i64) Value { + const mr = ref.type_().isA(mlir.MemRefType) orelse @panic("stridedLoadShape: not a memref"); + const result_ty = mlir.vectorType(result_shape, mr.elementType()); + return self.emit(tpu.strided_load(self.ctx, ref.inner, self.innerSlice(indices), strides, result_ty, self.loc())); + } + + /// `tpu.strided_store` — multi-dim strided store. + pub fn stridedStore(self: *Builder, ref: Value, value: Value, indices: []const Value, strides: []const i32) void { + _ = tpu.strided_store(self.ctx, value.inner, ref.inner, self.innerSlice(indices), strides, self.loc()).appendTo(self.currentBlock()); + } + + // ==================== TPU compute: matmul / iota / reciprocal ==================== + + /// `tpu.matmul(lhs, rhs, acc)`. Result type is taken from `acc`. Use + /// `matmulOpts` to set transposes / precision / dimension_numbers. + pub fn matmul(self: *Builder, lhs: Value, rhs: Value, acc: Value) Value { + return self.matmulOpts(lhs, rhs, acc, .{}); + } + + pub fn matmulOpts(self: *Builder, lhs: Value, rhs: Value, acc: Value, opts: MatmulOpts) Value { + return self.emit(tpu.matmul( + self.ctx, + lhs.inner, + rhs.inner, + acc.inner, + .{ + .transpose_lhs = opts.transpose_lhs, + .transpose_rhs = opts.transpose_rhs, + .precision = opts.precision, + .dimension_numbers = opts.dimension_numbers, + }, + acc.type_(), + self.loc(), + )); + } + + /// `tpu.iota` over the given shape / dimensions. Pass `null` for + /// `dimensions` to use the default. + pub fn iota(self: *Builder, shape: []const i64, dtype: DType, dimensions: ?[]const i32) Value { + const ty = mlir.vectorType(shape, dtype.toMlir(self.ctx)); + return self.emit(tpu.iota(self.ctx, dimensions, ty, self.loc())); + } + + /// 1-D `arange(0, n)`. TPU layout pass requires rank ≥ 2 with the lane dim + /// trailing, so this emits a `<1xN>` `tpu.iota` then `shape_cast` to ``. + pub fn arange(self: *Builder, n: i64, dtype: DType) Value { + const elem = dtype.toMlir(self.ctx); + const ty_2d = mlir.vectorType(&.{ 1, n }, elem); + const iota_2d = self.emit(tpu.iota(self.ctx, &.{1}, ty_2d, self.loc())); + return self.shapeCast(iota_2d, &.{n}); + } + + /// `tpu.reciprocal(x)` with default options. + pub fn reciprocal(self: *Builder, x: Value) Value { + return self.reciprocalOpts(x, .{}); + } + + pub fn reciprocalOpts(self: *Builder, x: Value, opts: ReciprocalOpts) Value { + return self.emit(tpu.reciprocal(self.ctx, x.inner, .{ + .approx = opts.approx, + .full_range = opts.full_range, + }, self.loc())); + } + + // ==================== Vector dialect helpers ==================== + // + // The `tpu.*` shape ops (`reshape`, `repeat`, `transpose`, + // `broadcastInSublanes`) are Mosaic-specific layout-aware variants. + + /// `vector.broadcast` to `shape`. For Zig scalar literals prefer `splat` (dense-splat fast path). + pub fn broadcastTo(self: *Builder, src: Value, shape: []const i64) Value { + const ty = mlir.vectorType(shape, src.elemType()); + return self.emit(vector.broadcast(self.ctx, src.inner, ty, self.loc())); + } + + /// `vector.shape_cast` — reshape preserving total lanes. `reshape` emits `tpu.reshape` instead. + pub fn shapeCast(self: *Builder, src: Value, new_shape: []const i64) Value { + const ty = mlir.vectorType(new_shape, src.elemType()); + return self.emit(vector.shape_cast(self.ctx, src.inner, ty, self.loc())); + } + + /// `vector.extract` — extract a scalar (or lower-rank vector) at a + /// static position. Result shape = `src.shape[position.len..]`. + pub fn vectorExtract(self: *Builder, src: Value, position: []const i64) Value { + const sh = src.shape(); + var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (position.len..sh.len) |i| out.appendAssumeCapacity(sh.get(i)); + const ty: *const mlir.Type = if (out.len == 0) + src.elemType() + else + mlir.vectorType(out.constSlice(), src.elemType()); + return self.emit(vector.extract(self.ctx, src.inner, position, &.{}, ty, self.loc())); + } + + /// `vector.reduction ` — reduce a 1-D vector to a scalar. + /// Emits the bare upstream op; on real TPU the layout pass may reject + /// 1-D vectors, so prefer `reduceToScalar` for kernel-level reductions. + pub fn vectorReductionFlat(self: *Builder, kind: CombiningKind, src: Value) Value { + return self.emit(vector.reduction(self.ctx, kind, src.inner, src.elemType(), .{}, self.loc())); + } + + /// 1-D vector → scalar. TPU vectors are 2-D, so this adds a leading + /// singleton before reducing: shape_cast→<1xN>, multi_reduction[1]→<1>, extract[0]. + /// `acc` is the reduction identity (e.g. `k.zeros(&.{1}, .f32)` for sum). + pub fn reduceToScalar(self: *Builder, kind: CombiningKind, src: Value, acc: Value) Value { + std.debug.assert(src.rank() == 1); + const n = src.shape().get(0); + const src_2d = self.shapeCast(src, &.{ 1, n }); + const reduced = self.multiReduction(kind, src_2d, acc, &.{1}); + return self.vectorExtract(reduced, &.{0}); + } + + /// `vector.multi_reduction ` — reduce a multi-D vector along + /// the given axes with an explicit accumulator. Result rank = + /// `src.rank - axes.len`. + pub fn multiReduction( + self: *Builder, + kind: CombiningKind, + src: Value, + acc: Value, + axes: []const i64, + ) Value { + const sh = src.shape(); + var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + outer: for (0..sh.len) |i| { + for (axes) |a| if (a == @as(i64, @intCast(i))) continue :outer; + out.appendAssumeCapacity(sh.get(i)); + } + const ty: *const mlir.Type = if (out.len == 0) + src.elemType() + else + mlir.vectorType(out.constSlice(), src.elemType()); + return self.emit(vector.multi_reduction(self.ctx, kind, src.inner, acc.inner, axes, ty, self.loc())); + } + + /// `vector.load` with an explicit result shape — for `ref[i, j, :, k:k+BLOCK]`-style sliced loads. + pub fn vectorLoadShape(self: *Builder, ref: Value, indices: []const Value, shape: []const i64) Value { + const idx_inner = self.indicesOrZero(ref, indices); + const ty = mlir.vectorType(shape, ref.elemType()); + return self.emit(vector.load(self.ctx, ref.inner, idx_inner, ty, .{}, self.loc())); + } + + // ==================== TPU shape ==================== + + pub fn reshape(self: *Builder, src: Value, new_shape: []const i64) Value { + const ty = mlir.vectorType(new_shape, src.elemType()); + return self.emit(tpu.reshape(self.ctx, src.inner, ty, self.loc())); + } + + pub fn repeat(self: *Builder, src: Value, dimension: i32, times: i32) Value { + const in_shape = src.shape(); + var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..in_shape.len) |i| { + const d = in_shape.get(i); + out.appendAssumeCapacity(if (i == @as(usize, @intCast(dimension))) d * times else d); + } + const ty = mlir.vectorType(out.constSlice(), src.elemType()); + return self.emit(tpu.repeat(self.ctx, src.inner, dimension, times, ty, self.loc())); + } + + pub fn concatenate(self: *Builder, sources: []const Value, dimension: i32) Value { + std.debug.assert(sources.len > 0); + const ax: usize = @intCast(dimension); + var sum: i64 = 0; + for (sources) |s| sum += s.dim(ax); + const head = sources[0].shape(); + var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..head.len) |i| out.appendAssumeCapacity(if (i == ax) sum else head.get(i)); + const ty = mlir.vectorType(out.constSlice(), sources[0].elemType()); + return self.emit(tpu.concatenate(self.ctx, self.innerSlice(sources), dimension, ty, self.loc())); + } + + pub fn transpose(self: *Builder, src: Value, permutation: []const i32) Value { + const in_shape = src.shape(); + std.debug.assert(permutation.len == in_shape.len); + var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (permutation) |p| out.appendAssumeCapacity(in_shape.get(@intCast(p))); + const ty = mlir.vectorType(out.constSlice(), src.elemType()); + return self.emit(tpu.transpose(self.ctx, src.inner, permutation, ty, self.loc())); + } + + pub fn broadcastInSublanes(self: *Builder, src: Value, lane: i32) Value { + // Result shape = src shape (the op replicates within sublanes). + return self.emit(tpu.broadcast_in_sublanes(self.ctx, src.inner, lane, src.type_(), self.loc())); + } + + pub fn rotate(self: *Builder, value: Value, opts: RotateOpts) Value { + return self.emit(tpu.rotate(self.ctx, value.inner, .{ + .amount = opts.amount, + .dimension = opts.dimension, + .stride = opts.stride, + .stride_dimension = opts.stride_dimension, + }, self.loc())); + } + + // ==================== TPU reductions / scan / sort ==================== + + /// `tpu.all_reduce(input, dim, kind)` over a single dim. + pub fn allReduce(self: *Builder, input: Value, kind: ReductionKind, dim: i64) Value { + const in_shape = input.shape(); + var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..in_shape.len) |i| out.appendAssumeCapacity(if (i == @as(usize, @intCast(dim))) 1 else in_shape.get(i)); + const ty = mlir.vectorType(out.constSlice(), input.elemType()); + return self.emit(tpu.all_reduce(self.ctx, input.inner, dim, kind, ty, self.loc())); + } + + /// `tpu.reduce_index(input, kind, axis)` — drop axis, replace with + /// `index`-valued indices (integer). + pub fn reduceIndex(self: *Builder, input: Value, kind: ReductionKind, axis: i32) Value { + const in_shape = input.shape(); + var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (in_shape.constSlice(), 0..) |d, i| { + if (@as(i32, @intCast(i)) != axis) out.appendAssumeCapacity(d); + } + const ty = mlir.vectorType(out.constSlice(), mlir.integerType(self.ctx, .i32)); + return self.emit(tpu.reduce_index(self.ctx, input.inner, axis, kind, ty, self.loc())); + } + + /// `tpu.scan(input, kind, mask?)` — same shape as input. + pub fn scan(self: *Builder, input: Value, kind: ReductionKind, mask: ?Value) Value { + const m_inner: ?*const mlir.Value = if (mask) |m| m.inner else null; + return self.emit(tpu.scan(self.ctx, input.inner, kind, m_inner, input.type_(), self.loc())); + } + + /// `tpu.sort(keys, values, mask?)` — returns (output_mask, sorted_keys, sorted_values). + pub fn sort(self: *Builder, keys: Value, values: Value, mask: ?Value, opts: SortOpts) [3]Value { + const m_inner: ?*const mlir.Value = if (mask) |m| m.inner else null; + const out_mask_ty = if (mask) |m| m.type_() else blk: { + // Default: vector. + break :blk mlir.vectorType(keys.shape().constSlice(), mlir.integerType(self.ctx, .i1)); + }; + const op = tpu.sort( + self.ctx, + keys.inner, + values.inner, + m_inner, + .{ .descending = opts.descending }, + out_mask_ty, + keys.type_(), + values.type_(), + self.loc(), + ); + _ = op.appendTo(self.currentBlock()); + return .{ + .{ .inner = op.result(0), .kernel = self }, + .{ .inner = op.result(1), .kernel = self }, + .{ .inner = op.result(2), .kernel = self }, + }; + } + + // ==================== TPU memref-shape ops ==================== + + pub fn memRefSlice( + self: *Builder, + mem_ref: Value, + base_idx: []const Value, + result_shape: []const i64, + dynamic_sizes: []const Value, + ) Value { + const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefSlice: not a memref"); + const result_ty = mlir.memRefType(mr.elementType(), result_shape, null, mr.memorySpace()); + return self.emit(tpu.memref_slice(self.ctx, mem_ref.inner, .{ + .base_idx = self.innerSlice(base_idx), + .dynamic_sizes = self.innerSlice(dynamic_sizes), + }, result_ty, self.loc())); + } + + pub fn memRefSqueeze(self: *Builder, mem_ref: Value, result_shape: []const i64) Value { + const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefSqueeze: not a memref"); + const result_ty = mlir.memRefType(mr.elementType(), result_shape, null, mr.memorySpace()); + return self.emit(tpu.memref_squeeze(self.ctx, mem_ref.inner, result_ty, self.loc())); + } + + pub fn memRefReshape(self: *Builder, mem_ref: Value, result_shape: []const i64) Value { + const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefReshape: not a memref"); + const result_ty = mlir.memRefType(mr.elementType(), result_shape, null, mr.memorySpace()); + return self.emit(tpu.memref_reshape(self.ctx, mem_ref.inner, result_ty, self.loc())); + } + + pub fn memRefBitcast(self: *Builder, mem_ref: Value, dt: DType) Value { + const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefBitcast: not a memref"); + var s: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..mr.rank()) |i| s.appendAssumeCapacity(mr.dimension(i)); + const result_ty = mlir.memRefType(dt.toMlir(self.ctx), s.constSlice(), null, mr.memorySpace()); + return self.emit(tpu.memref_bitcast(self.ctx, mem_ref.inner, result_ty, self.loc())); + } + + /// `tpu.memref_bitcast` with an explicit result shape — needed for the + /// `bf16 → i32` packing trick in ragged-paged-attention's + /// `strided_load_kv` path (`<64x128xbf16>` → `<32x128xi32>`). + pub fn memRefBitcastShape(self: *Builder, mem_ref: Value, result_shape: []const i64, dt: DType) Value { + const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefBitcastShape: not a memref"); + const result_ty = mlir.memRefType(dt.toMlir(self.ctx), result_shape, null, mr.memorySpace()); + return self.emit(tpu.memref_bitcast(self.ctx, mem_ref.inner, result_ty, self.loc())); + } + + pub fn reinterpretCast(self: *Builder, mem_ref: Value, result_shape: []const i64, dt: DType) Value { + const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("reinterpretCast: not a memref"); + const result_ty = mlir.memRefType(dt.toMlir(self.ctx), result_shape, null, mr.memorySpace()); + return self.emit(tpu.reinterpret_cast(self.ctx, mem_ref.inner, result_ty, self.loc())); + } + + pub fn assumeLayout(self: *Builder, src: Value) Value { + return self.emit(tpu.assume_layout(self.ctx, src.inner, src.type_(), self.loc())); + } + + pub fn eraseMemRefLayout(self: *Builder, mem_ref: Value) Value { + const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("eraseMemRefLayout: not a memref"); + var s: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..mr.rank()) |i| s.appendAssumeCapacity(mr.dimension(i)); + // Result drops the layout but keeps shape, element type, memory space. + const result_ty = mlir.memRefType(mr.elementType(), s.constSlice(), null, mr.memorySpace()); + return self.emit(tpu.erase_memref_layout(self.ctx, mem_ref.inner, result_ty, self.loc())); + } + + // ==================== Mask building ==================== + + /// `tpu.create_mask` — 2-sided mask `[low, high)` per dim. + pub fn createMask(self: *Builder, low_bounds: []const Value, high_bounds: []const Value, shape: []const i64) Value { + const ty = mlir.vectorType(shape, mlir.integerType(self.ctx, .i1)); + return self.emit(tpu.create_mask(self.ctx, self.innerSlice(low_bounds), self.innerSlice(high_bounds), ty, self.loc())); + } + + /// Convenience: 1-sided range mask `(0 <= idx < limit)` — emits two + /// `index` constants and a `tpu.create_mask`. + pub fn rangeMask(self: *Builder, limits: []const Value, shape: []const i64) Value { + const zero = self.cIndex(0); + const lows = self.arena.allocator().alloc(Value, limits.len) catch @panic("rangeMask OOM"); + for (lows) |*l| l.* = zero; + return self.createMask(lows, limits, shape); + } + + // ==================== Semaphores / DMA / barriers ==================== + + pub fn semAlloc(self: *Builder, kind: ArgSpec.SemKind) Value { + const ty = switch (kind) { + .regular => tpu.semaphoreType(self.ctx), + .dma => tpu.dmaSemaphoreType(self.ctx), + }; + return self.emit(tpu.sem_alloc(self.ctx, ty, self.loc())); + } + + pub fn semBarrier(self: *Builder) Value { + const ty = tpu.dmaSemaphoreType(self.ctx); + return self.emit(tpu.sem_barrier(self.ctx, ty, self.loc())); + } + + pub fn semRead(self: *Builder, semaphore: Value) Value { + return self.emit(tpu.sem_read(self.ctx, semaphore.inner, self.loc())); + } + + pub fn semWait(self: *Builder, semaphore: Value, amount: anytype) void { + const a = if (@TypeOf(amount) == Value) amount else self.lift(amount); + _ = tpu.sem_wait(self.ctx, semaphore.inner, a.inner, self.loc()).appendTo(self.currentBlock()); + } + + pub fn semSignal(self: *Builder, semaphore: Value, amount: anytype) void { + self.semSignalOpts(semaphore, amount, .{}); + } + + pub fn semSignalOpts(self: *Builder, semaphore: Value, amount: anytype, opts: SemSignalOpts) void { + const a = if (@TypeOf(amount) == Value) amount else self.lift(amount); + const dev: ?*const mlir.Value = if (opts.device_id) |d| d.inner else null; + const core: ?*const mlir.Value = if (opts.core_id) |c_| c_.inner else null; + _ = tpu.sem_signal(self.ctx, semaphore.inner, a.inner, .{ + .device_id = dev, + .core_id = core, + }, self.loc()).appendTo(self.currentBlock()); + } + + pub fn barrier(self: *Builder, barrier_id: Value) void { + _ = tpu.barrier(self.ctx, barrier_id.inner, self.loc()).appendTo(self.currentBlock()); + } + + pub fn enqueueDma(self: *Builder, source: Value, target: Value, target_semaphore: Value, opts: EnqueueDmaOpts) void { + const src_sem: ?*const mlir.Value = if (opts.source_semaphore) |s| s.inner else null; + const dev: ?*const mlir.Value = if (opts.device_id) |d| d.inner else null; + const core: ?*const mlir.Value = if (opts.core_id) |c_| c_.inner else null; + _ = tpu.enqueue_dma(self.ctx, source.inner, target.inner, target_semaphore.inner, .{ + .source_semaphore = src_sem, + .device_id = dev, + .core_id = core, + .priority = opts.priority, + .strict_ordering = opts.strict_ordering, + }, self.loc()).appendTo(self.currentBlock()); + } + + pub fn waitDma2(self: *Builder, semaphore: Value, src: Value, dst: Value, opts: WaitDma2Opts) void { + const dev: ?*const mlir.Value = if (opts.device_id) |d| d.inner else null; + const core: ?*const mlir.Value = if (opts.core_id) |c_| c_.inner else null; + _ = tpu.wait_dma2(self.ctx, semaphore.inner, src.inner, dst.inner, .{ + .device_id = dev, + .core_id = core, + .strict_ordering = opts.strict_ordering, + }, self.loc()).appendTo(self.currentBlock()); + } + + pub fn deviceId(self: *Builder) Value { + return self.emit(tpu.device_id(self.ctx, self.loc())); + } + + /// `tpu.trace_start` — pair with `traceStop`. Used by Pallas to mark + /// einsum boundaries (e.g. `"nd,md->nm"`). + pub fn traceStart(self: *Builder, level: i32, message: []const u8) void { + _ = tpu.trace_start(self.ctx, level, message, self.loc()).appendTo(self.currentBlock()); + } + + pub fn traceStop(self: *Builder) void { + _ = tpu.trace_stop(self.ctx, self.loc()).appendTo(self.currentBlock()); + } + + /// Build a `#tpu.dot_dimension_numbers<...>` attribute for `matmulOpts`. + pub fn dotDimensionNumbers( + self: *Builder, + lhs_contracting: []const i64, + rhs_contracting: []const i64, + lhs_non_contracting: []const i64, + rhs_non_contracting: []const i64, + output_dim_order: []const i64, + lhs_batch: []const i64, + rhs_batch: []const i64, + ) *const mlir.Attribute { + return tpu.dotDimensionNumbers( + self.ctx, + lhs_contracting, + rhs_contracting, + lhs_non_contracting, + rhs_non_contracting, + output_dim_order, + lhs_batch, + rhs_batch, + ); + } + + pub fn delay(self: *Builder, cycles: anytype) void { + const c_ = if (@TypeOf(cycles) == Value) cycles else self.lift(cycles); + _ = tpu.delay(self.ctx, c_.inner, self.loc()).appendTo(self.currentBlock()); + } + + // ==================== SCF regions ==================== + + /// Open a scoped `scf.for`. `lower`/`upper`/`step` accept Values, comptime + /// ints, or runtime Zig ints — coerced to `index`. `inits` is `.{v1, v2, ...}`. + pub fn openFor( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + ) dsl.ForScope(Builder, Value, dsl.tupleArity(@TypeOf(inits), "openFor: inits")) { + const N = comptime dsl.tupleArity(@TypeOf(inits), "openFor: inits"); + const fields = @typeInfo(@TypeOf(inits)).@"struct".fields; + + // scf.for bounds default to `index`. + const lb_v: Value = if (@TypeOf(lower) == Value) lower else self.cIndex(@intCast(lower)); + const ub_v: Value = if (@TypeOf(upper) == Value) upper else self.cIndex(@intCast(upper)); + const step_v: Value = if (@TypeOf(step) == Value) step else self.cIndex(@intCast(step)); + + var block_types: [N + 1]*const mlir.Type = undefined; + var block_locs: [N + 1]*const mlir.Location = undefined; + block_types[0] = lb_v.type_(); + block_locs[0] = self.loc(); + var inits_inner: [N]*const mlir.Value = undefined; + inline for (fields, 0..) |f, i| { + const raw = @field(inits, f.name); + const v: Value = if (f.type == Value) raw else self.lift(raw); + block_types[i + 1] = v.type_(); + block_locs[i + 1] = self.loc(); + inits_inner[i] = v.inner; + } + + const body = mlir.Block.init(&block_types, &block_locs); + self.pushBlock(body); + + var carried: [N]Value = undefined; + for (0..N) |i| carried[i] = .{ .inner = body.argument(i + 1), .kernel = self }; + + return .{ + .kernel = self, + .body = body, + .lb_inner = lb_v.inner, + .ub_inner = ub_v.inner, + .step_inner = step_v.inner, + .inits_inner = inits_inner, + .iv = .{ .inner = body.argument(0), .kernel = self }, + .carried = carried, + }; + } + + pub fn openIf(self: *Builder, cond: Value) dsl.IfOnlyScope(Builder, Value) { + const then_block = mlir.Block.init(&.{}, &.{}); + self.pushBlock(then_block); + return .{ + .kernel = self, + .cond_inner = cond.inner, + .then_block = then_block, + }; + } + + pub fn openIfElse( + self: *Builder, + cond: Value, + result_types: anytype, + ) dsl.IfScope(Builder, Value, dsl.tupleArity(@TypeOf(result_types), "openIfElse: result_types")) { + const N = comptime dsl.tupleArity(@TypeOf(result_types), "openIfElse: result_types"); + const fields = @typeInfo(@TypeOf(result_types)).@"struct".fields; + + var types: [N]*const mlir.Type = undefined; + inline for (fields, 0..) |f, i| { + if (f.type != *const mlir.Type) + @compileError("openIfElse: every result_type must be *const mlir.Type (use k.scalarTy/vectorTy)"); + types[i] = @field(result_types, f.name); + } + + const then_block = mlir.Block.init(&.{}, &.{}); + const else_block = mlir.Block.init(&.{}, &.{}); + self.pushBlock(then_block); + + return .{ + .kernel = self, + .cond_inner = cond.inner, + .then_block = then_block, + .else_block = else_block, + .result_types = types, + }; + } + + pub fn openWhile( + self: *Builder, + inits: anytype, + after_types: anytype, + ) dsl.WhileScope( + Builder, + Value, + dsl.tupleArity(@TypeOf(inits), "openWhile: inits"), + dsl.tupleArity(@TypeOf(after_types), "openWhile: after_types"), + ) { + const N = comptime dsl.tupleArity(@TypeOf(inits), "openWhile: inits"); + const M = comptime dsl.tupleArity(@TypeOf(after_types), "openWhile: after_types"); + const init_fields = @typeInfo(@TypeOf(inits)).@"struct".fields; + const type_fields = @typeInfo(@TypeOf(after_types)).@"struct".fields; + + var before_types: [N]*const mlir.Type = undefined; + var before_locs: [N]*const mlir.Location = undefined; + var inits_inner: [N]*const mlir.Value = undefined; + inline for (init_fields, 0..) |f, i| { + if (f.type != Value) + @compileError("openWhile: every init must be a Value"); + const v: Value = @field(inits, f.name); + before_types[i] = v.type_(); + before_locs[i] = self.loc(); + inits_inner[i] = v.inner; + } + const before_block = mlir.Block.init(&before_types, &before_locs); + + var after_tys: [M]*const mlir.Type = undefined; + var after_locs: [M]*const mlir.Location = undefined; + inline for (type_fields, 0..) |f, i| { + if (f.type != *const mlir.Type) + @compileError("openWhile: every after_type must be *const mlir.Type"); + after_tys[i] = @field(after_types, f.name); + after_locs[i] = self.loc(); + } + const after_block = mlir.Block.init(&after_tys, &after_locs); + + self.pushBlock(before_block); + var before_carried: [N]Value = undefined; + for (0..N) |i| before_carried[i] = .{ .inner = before_block.argument(i), .kernel = self }; + + return .{ + .kernel = self, + .before_block = before_block, + .after_block = after_block, + .inits_inner = inits_inner, + .after_types = after_tys, + .before_carried = before_carried, + }; + } + + // ==================== finalization ==================== + + /// Append `func.return` with the given results, verify the module, and + /// serialize to a NUL-terminated MLIR string owned by the kernel's + /// allocator. Caller must `defer allocator.free(ir)`. + pub fn finish(self: *Builder, results: []const Value) FinishError![:0]const u8 { + return self.finishOpts(results, .{}); + } + + pub const FinishOpts = struct { + /// Run `canonicalize,cse` on the module before serializing — lets + /// the user code stay one-line-per-Pallas-line and lean on MLIR + /// to dedupe duplicated constants / identical subexpressions, the + /// same way Pallas does (its lowering doesn't dedupe either; the + /// IR you read in `debug=True` is post-canonicalize). + canonicalize: bool = false, + }; + + pub fn finishOpts(self: *Builder, results: []const Value, opts: FinishOpts) FinishError![:0]const u8 { + const current = self.currentBlock(); + _ = func.returns(self.ctx, self.innerSlice(results), self.loc()).appendTo(current); + + if (self.emit_transform_stubs) self.appendTransformStubs(); + + if (!self.module.operation().verify()) { + return error.InvalidMlir; + } + + if (opts.canonicalize) { + const pm = mlir.PassManager.init(self.ctx); + defer pm.deinit(); + // `canonicalize` folds `x * 1`, `x + 0`, etc.; `cse` then + // dedupes identical constants and subexpressions. Same + // pipeline `zml/module.zig` runs on every compiled module. + const opm = pm.asOpPassManager(); + inline for (.{ "canonicalize", "cse", "canonicalize" }) |pass| { + try opm.addPipeline(pass); + } + try pm.runOnOp(self.module.operation()); + } + + var al: std.Io.Writer.Allocating = .init(self.allocator); + defer al.deinit(); + try al.writer.print("{f}", .{self.module.operation().fmt(.{ + .debug_info = false, + })}); + + return try self.allocator.dupeZ(u8, al.written()); + } + + /// Emit `transform_N` stub funcs — one per non-skipped i/o ref-arg. + /// Each takes `(i32 ×grid_dim_count, prefetch_refs...)` and returns + /// `block_rank` i32 values per `RefArgSnapshot.transform_returns` + /// (default: all zeros, matching a trivial index_map). + fn appendTransformStubs(self: *Builder) void { + const ctx = self.ctx; + const unknown_loc: *const mlir.Location = .unknown(ctx); + const i32_ty = mlir.integerType(ctx, .i32); + + const arena_alloc = self.arena.allocator(); + const total_args = self.grid_dim_count + self.prefetch_args.len; + const arg_types = arena_alloc.alloc(*const mlir.Type, total_args) catch @panic("OOM"); + const arg_locs = arena_alloc.alloc(*const mlir.Location, total_args) catch @panic("OOM"); + for (0..self.grid_dim_count) |i| { + arg_types[i] = i32_ty; + arg_locs[i] = unknown_loc; + } + for (self.prefetch_args, 0..) |ty, i| { + arg_types[self.grid_dim_count + i] = ty; + arg_locs[self.grid_dim_count + i] = unknown_loc; + } + + for (self.ref_args, 0..) |ra, idx| { + if (ra.skipped) continue; + + const rank: usize = @intCast(ra.block_rank); + const result_types = arena_alloc.alloc(*const mlir.Type, rank) catch @panic("OOM"); + for (result_types) |*r| r.* = i32_ty; + + const block = mlir.Block.init(arg_types, arg_locs); + + const c0 = arith.constant_int(ctx, 0, i32_ty, unknown_loc); + _ = c0.appendTo(block); + + const ret_vals = arena_alloc.alloc(*const mlir.Value, rank) catch @panic("OOM"); + const returns = ra.transform_returns; + for (ret_vals, 0..) |*v, i| { + if (returns) |r_list| { + if (i < r_list.len) { + v.* = switch (r_list[i]) { + .zero => c0.result(0), + .program_id => |pid| block.argument(pid), + }; + continue; + } + } + v.* = c0.result(0); + } + _ = func.returns(ctx, ret_vals, unknown_loc).appendTo(block); + + var name_buf: [32]u8 = undefined; + const name = std.fmt.bufPrint(&name_buf, "transform_{d}", .{idx}) catch unreachable; + + const stub_op = func.func(ctx, .{ + .name = name, + .args = arg_types, + .results = result_types, + .block = block, + .location = unknown_loc, + .visibility = null, + .verify = false, + }); + _ = stub_op.appendTo(self.module.body()); + } + } +}; + +/// Memory spaces that don't support windowing (empty window_params, no transform_N). +fn isSkippedWindowSpace(ms: MemorySpace) bool { + return switch (ms) { + .hbm, .any, .semaphore_mem => true, + else => false, + }; +} + +// ==================== scope-type aliases pinned to Mosaic's Builder ==================== + +const tupleArity = dsl.tupleArity; + +pub fn ForScope(comptime N: usize) type { + return dsl.ForScope(Builder, Value, N); +} + +pub const IfOnlyScope = dsl.IfOnlyScope(Builder, Value); + +pub fn IfScope(comptime N: usize) type { + return dsl.IfScope(Builder, Value, N); +} + +pub fn WhileScope(comptime N: usize, comptime M: usize) type { + return dsl.WhileScope(Builder, Value, N, M); +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/kernels/mosaic_tpu/dtype.zig b/kernels/mosaic_tpu/dtype.zig new file mode 100644 index 00000000..4d678c1f --- /dev/null +++ b/kernels/mosaic_tpu/dtype.zig @@ -0,0 +1,72 @@ +const std = @import("std"); +const mlir = @import("mlir"); + +pub const DType = enum { + i1, + i8, + i16, + i32, + i64, + f16, + bf16, + f32, + f64, + f8e4m3fn, + f8e5m2, + + pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { + return switch (self) { + .i1 => mlir.integerType(ctx, .i1), + .i8 => mlir.integerType(ctx, .i8), + .i16 => mlir.integerType(ctx, .i16), + .i32 => mlir.integerType(ctx, .i32), + .i64 => mlir.integerType(ctx, .i64), + .f16 => mlir.floatType(ctx, .f16), + .bf16 => mlir.floatType(ctx, .bf16), + .f32 => mlir.floatType(ctx, .f32), + .f64 => mlir.floatType(ctx, .f64), + .f8e4m3fn => mlir.floatType(ctx, .f8e4m3fn), + .f8e5m2 => mlir.floatType(ctx, .f8e5m2), + }; + } +}; + +pub fn isFloatDtype(dt: DType) bool { + return switch (dt) { + .f16, .bf16, .f32, .f64, .f8e4m3fn, .f8e5m2 => true, + else => false, + }; +} + +pub fn dtypeBitwidth(dt: DType) usize { + return switch (dt) { + .i1 => 1, + .i8, .f8e4m3fn, .f8e5m2 => 8, + .i16, .f16, .bf16 => 16, + .i32, .f32 => 32, + .i64, .f64 => 64, + }; +} + +pub fn intBitwidth(dt: DType) u32 { + return switch (dt) { + .i1 => 1, + .i8 => 8, + .i16 => 16, + .i32 => 32, + .i64 => 64, + else => std.debug.panic("intBitwidth: not an integer DType: {s}", .{@tagName(dt)}), + }; +} + +pub fn mlirElemToDType(ctx: *mlir.Context, elem: *const mlir.Type) DType { + inline for (std.meta.fields(DType)) |f| { + const dt = @field(DType, f.name); + if (elem.eql(dt.toMlir(ctx))) return dt; + } + @panic("element type not a recognized Mosaic-TPU DType"); +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/kernels/triton/BUILD.bazel b/kernels/triton/BUILD.bazel new file mode 100644 index 00000000..cc80054c --- /dev/null +++ b/kernels/triton/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "triton_builder", + tags = ["manual"], + srcs = [ + "dtype.zig", + ], + import_name = "kernels/triton/builder", + main = "builder.zig", + visibility = ["//visibility:public"], + deps = [ + "//kernels/common", + "//mlir", + "//mlir/dialects", + "//mlir/dialects/ttir", + "//stdx", + ], +) + +zig_test( + name = "test", + deps = [":triton_builder"], +) diff --git a/kernels/triton/PORTING.md b/kernels/triton/PORTING.md new file mode 100644 index 00000000..659baadf --- /dev/null +++ b/kernels/triton/PORTING.md @@ -0,0 +1,748 @@ +# Porting Triton kernels from Python to the Zig DSL + +This guide covers the day-to-day workflow for **adding a new Triton kernel +to ZML** (port from a Python `@triton.jit` source) and **fixing an existing +Zig kernel that diverges from Python** (using the comparison harness in +`examples/triton_emitter/`). + +## Goals (in priority order) + +1. **Lowered code matches.** The Zig version produces **the same LLVM IR + + PTX** as the Python version. TTIR/TTGIR may differ cosmetically (op + ordering, naming) — what matters is what actually runs on the GPU. + +2. **Source code looks alike.** A Python kernel and its Zig port should be + *visually transparent* side by side: same overall structure, same + variable names, same comment placement, same line ordering. Someone + reading both should be able to follow them in lockstep. + + Visual fidelity matters because: + - It makes future ports trivial — patterns transfer one-to-one. + - It makes upstream-Triton patches easy to apply (the Python diff + translates almost mechanically to a Zig diff). + - It catches porting bugs before they reach the comparison harness: + when the Zig code reads differently from Python, that's where the + real-vs-cosmetic divergence usually hides. + + Concretely: if a Python source line is + ```python + y_ptr_offset = (row.to(tl.int64) * y_row_stride) + (row_g_id.to(tl.int64) * group_size) + ``` + the Zig port should be one line, named the same way: + ```zig + const y_ptr_offset = row.to(.i64).mul(y_row_stride).add(row_g_id.to(.i64).mul(group_size)); + ``` + Not split across three temporaries with renamed locals — even if the + resulting LLIR is identical. Goal #1 is a hard requirement; goal #2 + is what makes this codebase maintainable. + +For the DSL surface (`zml.kernel.triton.Kernel`, `Builder`, `Value`, etc.), see +[`README.md`](./README.md). This file is about the *process*, not the API. + +--- + +## Prerequisites + +- CUDA or ROCm GPU (XLA's Triton backend is GPU-only). +- The Python venv at `examples/triton_emitter/.venv` activated, or a + system-level Python with `triton`, `torch`, `numpy` installed. +- Bazel + the standard ZML build deps. + +One-time: + +```sh +cd examples/triton_emitter +python -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +``` + +--- + +## Phase 1 — Initial port + +Start by copying the Python kernel verbatim into the Zig DSL form. The +goal is a kernel that *compiles*, *verifies*, **and reads as similarly +to the Python source as possible**. + +> **Author hint:** open the Python file in a left pane and the Zig file +> in a right pane. Translate **line by line**, top to bottom. Keep +> variable names, comment placement, and intermediate-binding granularity +> identical wherever the DSL allows. Don't refactor, don't rename, don't +> "improve" — that's a separate task you do after the LLIR matches. + +### Step 1.1 — Drop the Python source next to the Zig file + +For comparison purposes, the Python source has to live under +`examples/triton_emitter/kernels_py/`. The dump script auto-discovers +every `@triton.jit` function in that directory. + +If your kernel's already in `vllm/` or another upstream tree, copy the +relevant `@triton.jit` definition (and any `@triton.jit` helpers it +calls) into a single `.py` file there. Strip away launch wrappers, +autotune decorators, and any code that isn't actually inside the kernel +function. + +### Step 1.2 — Sketch the Zig declaration + +Inside `zml/moe/triton_kernels.zig` or `zml/attention/triton_kernels.zig` +(or whichever production file fits), add: + +```zig +pub const MyKernel = zml.kernel.triton.Kernel(.{ + .name = "my_kernel", // EXACTLY matches the Python `def my_kernel` + .config = struct { + // One field per `tl.constexpr` parameter. No defaults — every + // field is required at call time. Use the same names Python uses, + // converted to snake_case. + BLOCK_SIZE: usize, + // ... + }, +}, struct { + pub fn run(b: *tri.Builder, cfg: anytype) !void { + const a = try b.declareArgs(.{ + // One field per *runtime* parameter (pointers, dims that + // arrive as 0-d tensors). Use the same names + order Python + // uses; the cheat-sheet's "argument specs" table maps the + // tagged-union variants. + .x_ptr = .{ .ptr = .f32 }, + // ... + }); + + // Body — translate Python line by line. Cheat-sheet in the + // README's "Python Triton ↔ Zig DSL cheat-sheet" section is + // your best friend here. + } +}); +``` + +### Step 1.3 — Wire the production caller + +Inside the same `.zig` directory, find the function that launches your +kernel (e.g. `pagedAttention2d`, `fusedExpertsImpl`). Replace the +TTIR-emit + `ops.triton(...)` two-step with a single +`MyKernel.call(inputs, outputs, opts)` — see the [README](./README.md#calling-kernels-from-a-zml-model--kcall) +for the exact shape. + +If this is a brand-new kernel with no existing caller, write the caller +fresh. + +### Step 1.4 — Wire the comparison harness + +Add the kernel to `examples/triton_emitter/kernels_zig.zig`: + +```zig +withConfig(moe.MyKernel, .{ .BLOCK_SIZE = 64, /* ... */ }), +``` + +If the kernel takes runtime tensor inputs, also add a `forward` struct + +`KERNELS` entry to `examples/triton_emitter/dump_via_xla.zig` so XLA's +codegen pipeline can lower it. The tensor sizes are dummies — XLA +doesn't launch the kernel, it just runs the lowering passes. + +For the Python side, the auto-discovery in `dump_python_ir.py` works for +simple kernels (only `*_ptr` runtime args + `tl.constexpr` params). +**If the signature mixes runtime args and constexpr in the middle of the +parameter list**, write an override function and add it to `_OVERRIDES` +(follow `_per_token_group_quant_fp8` / `_kernel_unified_attention_2d_ptr` +as templates). Use the same constexpr values + tensor shapes you used in +the Zig `withConfig(...)` call. + +### Step 1.5 — Build it + +```sh +bazel build //examples/triton_emitter:dump_zig_ir # Zig side compiles + verifies +bazel build //zml:test # production side compiles +``` + +If the Zig DSL `b.finish()` rejects your IR, the most common causes are: +- **Width mismatches** in `arith.muli` / `addi` (e.g. `i32` × `i64`). + Cast one operand with `.to(.i64)` so both match. +- **Tensor-of-pointers `addPtr` requires same shape** — when the offset + is a tensor, splat the pointer first (`scalar_ptr.addPtr(tensor_off)` + auto-splats; `tensor_ptrs.addPtr(scalar)` doesn't — pass + `tensor_ptrs.addPtr(b.splat(s, &.{shape}))`). +- **`scf.yield` arity** — the tuple you pass to `loop.yield(...)` / + `i.yieldThen(...)` must match the loop's `inits` / `result_types` + arity exactly. + +### Step 1.6 — First dump + +```sh +bash examples/triton_emitter/run.sh +``` + +This runs the full pipeline (Python TTIR → XLA → per-stage IR; same on +the Zig side; diff each stage). At the end you get a per-kernel +verdict: + +``` +my_kernel + ttir +30/-30 + ttgir +30/-30 + llir +50/-50 + ptx +40/-40 +``` + +Don't worry yet about cosmetic diffs — go through Phase 2 to find +*real* divergences first. + +--- + +## Phase 2 — The comparison pipeline + +The harness has six steps (see `run.sh`), output appears under +`examples/triton_emitter/`: + +| Step | Output | What it is | +|------|------------------------------|-----------------------------------------------| +| 1 | `py_ir/*.ttir`/`.ttgir`/... | Python's per-stage IR (Triton's own pipeline) | +| 2 | `zig_ir/*.ttir` | Just the TTIR your DSL emits. | +| 3 | `xla_dump_py/` | XLA's per-pass dumps lowering Python's TTIR. | +| 4 | `xla_dump_zig/` | XLA's per-pass dumps lowering the Zig TTIR. | +| 5 | `xla_py/*.{ttir,ttgir,llir,ptx}` + `xla_zig/*.{...}` | Per-stage IR extracted by `extract_xla_dump.py`. Both sides go through XLA, not Triton-the-Python-frontend, so the comparison is apples-to-apples. | +| 6 | stdout | `compare_ir.py` diff summary per kernel. | + +### Useful flags + +```sh +# Just one kernel (faster iteration): +# (use ZIG_IR/PY_IR filter — but note dump_zig_ir's CLI expects --kernel=NAME). +bash run.sh # full run +python compare_ir.py --left xla_py --right xla_zig --kernel my_kernel +python compare_ir.py --left xla_py --right xla_zig --kernel my_kernel --stage ttir --show-diffs + +# Skip the slow XLA-lowering steps if only the TTIR changed: +python dump_python_ir.py --out-dir py_ir +bazel run //examples/triton_emitter:dump_zig_ir -- --out-dir=$PWD/zig_ir +diff py_ir/my_kernel.ttir zig_ir/my_kernel.ttir +``` + +### Reading the verdict + +``` +my_kernel + ttir +30/-30 <- TTIR diverges 30 lines + ttgir +30/-30 <- so does TTGIR (same shape) + llir ✓ <- but LLIR converges! cosmetic-only TTIR diff + ptx ✓ <- and PTX matches +``` + +This kernel is **good** — final code is identical, the TTIR diff is just +op-naming/ordering cosmetics that XLA's lowering canonicalizes away. + +``` +my_kernel + ttir +5/-5 + ttgir +5/-5 + llir +60/-58 <- LLIR diverges → real semantic difference + ptx +50/-48 +``` + +This kernel needs work: a real divergence appeared somewhere between +TTIR and LLIR. Drill into the LLIR diff to find which op got lowered +differently. + +--- + +## Phase 3 — Fixing divergences + +Workflow: pick the *smallest* diverging stage's diff and locate the +op-by-op mismatch. Most fixes fall into one of the categories below. + +### Step 3.1 — `--show-diffs` to locate the divergence + +```sh +python compare_ir.py --left xla_py --right xla_zig --kernel my_kernel --stage ttir --show-diffs +``` + +Read the diff carefully. The first diverging op tells you what to look +at; subsequent diffs are usually downstream consequences. + +### Step 3.2 — Common porting bugs + +These are the gotchas we've hit while porting kernels. If your diff +shows any of these patterns, fix the corresponding Zig DSL call. + +#### 3.2.1 — `arith.maxnumf` vs `arith.maximumf` (and `min*`) + +**Symptom:** +``` +- %v43 = arith.maxnumf %v37, %v14 : f32 (Python) ++ %v43 = arith.maximumf %v37, %v13 : f32 (Zig) +``` + +**Fix:** **Use `b.maximum` / `Value.maximum` / `b.minimum` / +`Value.minimum`** — they default to `maxnumf`/`minnumf` and match +Python's `tl.max`/`tl.min`/`tl.maximum`/`tl.minimum`. **Don't reach +for `b.maximumf` / `b.minimumf` directly** — those are the IEEE-754 +NaN-propagating escape hatches (`arith.maximumf`/`minimumf`), exposed +for the rare case where you genuinely want NaN propagation. The naming +is unfortunate (the `f` suffix reads like "the float version") — it's +actually "the NaN-propagating version." See README's "Reductions and +scans" gotcha table for the full mapping. + +#### 3.2.2 — 2-operand vs 3-operand `tt.load` + +**Symptom:** +``` +- %v33 = tt.load %v32, %v30, %v9 : tensor<128x!tt.ptr> (Python — has `other`) ++ %v33 = tt.load %v32, %v30 : tensor<128x!tt.ptr> (Zig — no `other`) +``` + +**Fix:** Match Python's exact form. The DSL has only one masked-load +API (`b.loadOpts`); whether it emits 2-operand or 3-operand `tt.load` +depends entirely on whether you pass `.other`: + +```zig +// Python: tl.load(ptr, mask=mask, other=0.0) → 3-operand +const zero = b.zeros(&.{BLOCK}, dtype); +const x = b.loadOpts(ptr, .{ .mask = mask, .other = zero }); + +// Python: tl.load(ptr, mask=mask) → 2-operand +const x = b.loadOpts(ptr, .{ .mask = mask }); +``` + +The two forms are *semantically* close but not identical (out-of-bounds +lanes are undefined for 2-operand, zero for 3-operand). Always copy +Python's form, even if you suspect the difference is harmless — XLA's +canonicalization may or may not erase it. + +#### 3.2.3 — `i32` vs `i64` width in pointer offsets + +**Symptom:** +``` +- %v26 = tt.addptr %v6, %v16 : !tt.ptr, i32 (Python — uses i32) ++ %v25 = tt.addptr %v6, %v16 : !tt.ptr, i64 (Zig — uses i64) +``` + +**Fix:** Python's pointer arithmetic preserves the original integer +width. `tl.program_id(0)` is i32; if Python writes +`y_s_ptr += tl.program_id(0)` (no cast), the offset stays i32. In Zig, +if you converted `pid` to i64 early to use it elsewhere, pass the raw +`b.programId(.x)` (i32) to that one `addPtr` site: + +```zig +// raw i32 program_id for one site, i64 elsewhere +const pid_i32 = b.programId(.x); +const g_id = pid_i32.to(.i64); +// ... +const y_s_ptr_shifted = a.y_s_ptr.addPtr(pid_i32); // i32, not g_id +``` + +#### 3.2.4 — Mask comparison width + +**Symptom:** +``` +- %v30 = arith.cmpi slt, %v28, %v29 : tensor<128xi64> (Python — i64 vs i64) ++ %v28 = arith.trunci %v10 : i64 to i32 ++ %v29 = tt.splat %v28 : i32 -> tensor<128xi32> ++ %v30 = arith.cmpi slt, %v26, %v29 : tensor<128xi32> (Zig — trunc + splat + i32 cmp) +``` + +**Fix:** Match Python's promotion direction. Python implicitly promotes +the narrower operand to the wider one. If `cols` is i32 and +`group_size` is i64, Python emits `extsi cols i32→i64` then compares +at i64. The Zig DSL doesn't auto-promote, so do it explicitly: + +```zig +const cols_i32 = b.arange(0, BLOCK, .i32); +const cols_i64 = cols_i32.to(.i64); +const mask = cols_i64.lt(group_size); // both i64, no trunc +``` + +#### 3.2.5 — Op emit order + +**Symptom:** TTIR diff shows the same ops in a different order: +``` +- %v18 = arith.divsi %v17, %v15 : i64 +- %v19 = arith.remsi %v17, %v15 : i64 +- %v20 = arith.muli %v18, %v13 : i64 +- %v21 = arith.muli %v19, %v11 : i64 +- %v22 = arith.addi %v20, %v21 : i64 ++ %v17 = arith.divsi %v16, %v14 : i64 ++ %v18 = arith.muli %v17, %v12 : i64 ++ %v19 = arith.remsi %v16, %v14 : i64 ++ %v20 = arith.muli %v19, %v10 : i64 ++ %v21 = arith.addi %v18, %v20 : i64 +``` + +This usually doesn't affect LLIR/PTX (compiler reorders), but if it +does (or if you want a clean TTIR diff), match Python's source order +op-by-op. Bind every intermediate to a named local in the same order +Python evaluates: + +```zig +// Python: y_ptr += (g_id // groups_per_row) * y_row_stride + (g_id % groups_per_row) * group_size +// eval order: divsi, remsi, muli, muli, addi +const row = g_id.div(groups_per_row); // divsi +const row_g_id = g_id.rem(groups_per_row); // remsi +const row_off = row.mul(y_row_stride); // muli +const grp_off = row_g_id.mul(group_size); // muli +const y_ptr_shifted = a.y_ptr.addPtr(row_off.add(grp_off)); // addi + addptr +``` + +#### 3.2.6 — `arith.maxnumf` (and `splat`) vs `tt.splat` of a runtime value + +**Symptom:** +``` +- %splat = tt.splat %scalar : i32 -> tensor<128xi32> (Python — runtime splat) ++ %const = arith.constant dense<...> : tensor<128xi32> (Zig — constant pool) +``` + +**Fix:** `b.splat(comptime_literal, shape)` materializes a constant +dense; `b.splat(runtime_value, shape)` emits `tt.splat`. If Python +uses a runtime value (loaded scalar, function arg) and you accidentally +hardcoded the constant, switch to the runtime path. + +#### 3.2.7 — `tl.constexpr` divisibility hints + +**Symptom:** TTIR header differs in `tt.divisibility = N : i32` attrs. + +**Fix:** Default `.{ .ptr = .dtype }` carries `tt.divisibility = 16`. +For scalars, switch to `.scalar_opts`: + +```zig +.n_elements = .{ .scalar_opts = .{ .dtype = .i32, .divisibility = 16 } }, +``` + +This matches what Python's JIT runtime auto-attaches when the runtime +value is a multiple of 16. + +#### 3.2.8 — Reductions over rank-1 tensors + +**Symptom:** Python emits a `tt.reshape allow_reorder` before the +`tt.reduce`; Zig doesn't. + +**Fix:** Use the `axis = null` (default) form: `b.max(x)`, not +`b.maxOpts(x, .{ .axis = 0 })`. The `axis = null` path inserts the +reshape; explicit-axis paths skip it. + +#### 3.2.9 — `tl.dot` precision and accumulator + +**Symptom:** TTIR shows different `tt.dot` `inputPrecision` attribute or +different `acc` type. + +**Fix:** Match Python's `tl.dot(a, b, acc, input_precision="...")` +explicitly: + +```zig +const new_acc = b.dotOpts(a_val, b_val, acc, .{ + .input_precision = .tf32, // matches `input_precision="tf32"` + .max_num_imprecise_acc = 0, +}); +``` + +Default `input_precision` differs across Triton versions — always be +explicit. + +#### 3.2.10 — `iter_args` order in `scf.for` + +**Symptom:** TTIR diff shows the loop signature reordered: +``` +- %v152:3 = scf.for ... iter_args(%v149 = %v131, %v150 = %v143, %v151 = %v37) +- -> (tensor<...!tt.ptr>, tensor<...!tt.ptr>, tensor<...xf32>) ++ %v152:3 = scf.for ... iter_args(%v149 = %v37, %v150 = %v131, %v151 = %v143) ++ -> (tensor<...xf32>, tensor<...!tt.ptr>, tensor<...!tt.ptr>) +``` + +**Fix:** Python's `scf.for` lowering captures `iter_args` in the order +they're **declared** above the loop, not the order they're mutated inside. +For `fused_moe_kernel`: + +```python +a_ptrs = ... # declared 1st +b_ptrs = ... # declared 2nd +accumulator = tl.zeros(...) # declared 3rd +for k in range(...): + accumulator += tl.dot(a, b) # mutated 1st + a_ptrs += BLOCK_SIZE_K # mutated 2nd + b_ptrs += BLOCK_SIZE_K # mutated 3rd +``` + +Python's `iter_args` order = declaration order = `(a_ptrs, b_ptrs, acc)`, +**not** mutation order. The DSL `openFor` follows the order you pass — so +match Python's declaration order: + +```zig +var loop = b.openFor(0, num_k_iters, 1, .{ a_ptrs_init, b_ptrs_init, acc_init }); +{ + const a_ptrs = loop.carried[0]; + const b_ptrs = loop.carried[1]; + const acc = loop.carried[2]; + // ... + loop.yield(.{ new_a_ptrs, new_b_ptrs, new_acc }); +} +const accumulator = loop.results[2]; +``` + +The wrong order survives all the way to PTX as different register-block +layouts. + +#### 3.2.11 — Operand order in commutative `arith.muli` + +**Symptom:** TTIR diff shows `arith.muli` operands swapped: +``` +- %v137 = arith.muli %v136, %v135 : tensor<1x64xi64> (Python: stride on LHS) ++ %v137 = arith.muli %v135, %v136 : tensor<1x64xi64> (Zig: offs on LHS) +``` + +**Fix:** `arith.muli` is commutative semantically, but the operand order +affects downstream layout decisions in TTGIR. Match Python's source order +exactly. For `stride * offs[None, :]`, Python emits `arith.muli %stride_splat, +%offs`: + +```zig +// Python: stride_cm * offs_token[:, None] — stride on LHS +const cm_col = stride_cm.mul(offs_token_col); +// NOT: offs_token_col.mul(stride_cm) — offs on LHS, wrong order +``` + +Same applies to `arith.addi`, `arith.subi`, `arith.andi`, etc. + +#### 3.2.12 — Two-step `addPtr` for `c_ptr + cm[:, None] + cn[None, :]` + +**Symptom:** Python emits two `tt.addptr` ops; the Zig port collapses +them into one big addptr with a pre-merged 2D offset. +``` +Python: + %ptr_col = tt.addptr %c_ptr_splat_Mx1, %cm_col_Mx1 : Mx1x!tt.ptr, Mx1xi64 + %ptr_2d = tt.broadcast %ptr_col → MxNx!tt.ptr + %final_ptr = tt.addptr %ptr_2d, %cn_off_2d : MxNx!tt.ptr, MxNxi64 +Zig: + %offset = arith.addi %cm_term, %cn_term : MxNxi64 + %final_ptr = tt.addptr %c_ptr_splat, %offset : MxNx!tt.ptr, MxNxi64 +``` + +**Fix:** Python evaluates `c_ptr + cm[:, None] + cn[None, :]` left-to-right, +so it emits two `tt.addptr` ops. Match by splitting: + +```zig +// First addptr: scalar c_ptr + (Mx1) cm offset → Mx1 ptr tensor +const cm_col = stride_cm.mul(b.expandDims(offs_token, 1)); +const c_ptrs_col = b.splat(c_ptr, &.{ block_size_m, 1 }).addPtr(cm_col); + +// Second addptr: broadcast to MxN, add cn offset +const cn_row = b.expandDims(offs_cn, 0); +const c_ptrs_2d = b.broadcastTo(c_ptrs_col, &.{ block_size_m, block_size_n }); +const c_ptrs = c_ptrs_2d.addPtr(b.broadcastTo(cn_row, &.{ block_size_m, block_size_n })); +``` + +This pattern shows up wherever Python writes `ptr + a[:, None] + b[None, :]` +(both A and B ptrs in fused_moe, the C-output store, etc.). + +#### 3.2.13 — `extsi` placement relative to sibling `expand_dims`/`mul` + +**Symptom:** Same set of ops, different positions: +``` +- %v134 = expand_dims %offs_k_i32 (axis=1) (Python: offs_k expand) +- %v135 = expand_dims %offs_bn_i64 (axis=0) (Python: offs_bn expand) +- %v136 = splat %stride_bn → 1x64xi64 (Python: bn stride) +- %v137 = muli %v135, %v136 (Python: bn_row mul) +- %v138 = extsi %v134 → 32x1xi64 (Python: extsi DEFERRED) ++ %v135 = extsi %v134 → 32x1xi64 (Zig: extsi EAGER) ++ ... (rest in different order) +``` + +**Fix:** When Python writes `offs_k[:, None]` and uses it later in an +`addi` against an `i64`-typed tensor, the implicit `extsi` happens at the +**use site** (where the auto-promotion fires), not at the `expand_dims`. +Mirror this — keep the chain at i32 until just before it joins the i64 +expression: + +```zig +// Wrong: extsi happens immediately after expand_dims +const offs_k_col = b.expandDims(offs_k_i32, 1).to(.i64); // extsi NOW +const offs_bn_row = b.expandDims(offs_bn, 0); +const bn_row = offs_bn_row.mul(stride_bn); + +// Right: extsi deferred to after offs_bn's mul +const offs_k_col_i32 = b.expandDims(offs_k_i32, 1); // 32x1xi32 +const offs_bn_row = b.expandDims(offs_bn, 0); +const bn_row = offs_bn_row.mul(stride_bn); // 1x64xi64 +const offs_k_col = offs_k_col_i32.to(.i64); // extsi HERE +``` + +Same idea for placing a `muli` before an `extsi` on a sibling load: + +```zig +// Python emits muli before the extsi at the cmpi: +const num_tokens_post_padded_i32 = b.load(...); // load +const pid_m_block = pid_m.mul(block_size_m); // muli FIRST +const num_tokens_post_padded = num_tokens_post_padded_i32.to(.i64); // extsi +const out_of_range = pid_m_block.ge(num_tokens_post_padded); +``` + +### Step 3.3 — Iterate + +After each fix: + +```sh +bazel build //examples/triton_emitter:dump_zig_ir +bash examples/triton_emitter/run.sh +``` + +Re-read the diff. The line counts should drop, and ideally the LLIR/PTX +columns flip to `✓`. If not, drill again — there's another mismatch +hiding behind the first. + +### Step 3.4 — Don't be a perfectionist about TTIR (but do read every line) + +You only need **LLIR + PTX** to match. If TTIR/TTGIR have +5 cosmetic +lines but LLIR is byte-identical, ship it. The XLA canonicalization + +CSE passes erase a lot of low-level differences before LLIR. + +A kernel like `count_and_sort_expert_tokens_kernel` (TTIR/TTGIR +8/-8, +LLIR/PTX ✓) is a clean port. Don't keep grinding it. + +**But** — when LLIR is **not** ✓ and the diff is small (tens of lines), +do **not** label the rest "scheduling noise" without a name-normalized +diff. Most lines in a small LLIR diff are downstream SSA renumbering +caused by 1–4 high-leverage TTIR-level fixes (typically the patterns in +3.2.10–3.2.13). Run: + +```sh +diff \ + <(sed -E 's/%[a-zA-Z_.][a-zA-Z0-9_.]*[0-9]+/%V/g; s/%[0-9]+/%V/g; s/^[0-9]+:/V:/g' xla_py/foo.llir) \ + <(sed -E 's/%[a-zA-Z_.][a-zA-Z0-9_.]*[0-9]+/%V/g; s/%[0-9]+/%V/g; s/^[0-9]+:/V:/g' xla_zig/foo.llir) +``` + +If this normalized diff is small (~tens of lines), every remaining line +is pointing at a structural fix. The "+100 LLIR is just XLA scheduling" +intuition has been wrong every time so far — `fused_moe_kernel` went from +LLIR +100 → ✓ via four TTIR-level fixes (iter_args order, muli operand +order, two-step addPtr, extsi placement) that were all visible in the ++100-line diff once read carefully. + +### Step 3.5 — Visual-transparency pass + +Once the LLIR matches, do one pass over the Zig source with the Python +source open beside it. Check (in order): + +- **Variable names.** Python `groups_per_row` → Zig `groups_per_row`, + not `gp_row` or `gpr`. Same for arg names — `y_ptr` not `output_ptr`. +- **Line order.** Each Python line should correspond to roughly one + Zig line, in the same order. If a single Python expression became + three Zig temporaries, ask: does the DSL force the split? If not, + collapse them onto one chained `.add(...).mul(...)` line. +- **Granularity of locals.** If Python writes + `_absmax = tl.maximum(tl.max(tl.abs(y)), eps)` as a one-liner, the + Zig port should also be one line: `const absmax = b.max(b.absf(y)).maximum(eps);`. +- **Comments.** Carry over Python's comments verbatim where they apply. + If Python writes `# Map the program id to the row of X and Y it + should compute.` above its block, paste the same comment above the + Zig block. They serve as anchors for future side-by-side reads. +- **Helper functions.** If Python factors something into a + `@triton.jit` helper (e.g. `find_seq_idx`), the Zig port should have + a matching `fn findSeqIdx(b: *Builder, ...) Value` helper, not inline + the body. Same arguments, same name (snake_case → camelCase). +- **Control flow shape.** Python `for k in range(...)` → `b.openFor`, + Python `if ... return` → `b.returnIf` / `b.openReturnIf`. Don't + flatten an `if` ladder into a `select` chain (or vice versa) just + because the DSL allows it. + +If a transparency fix changes the LLIR diff, **the LLIR-match goal +wins** — back out the visual change and leave a comment explaining the +divergence. Goal #1 always beats goal #2. + +A useful sanity check: open both files side by side in your editor and +scroll both at the same rate. If the cursors stay roughly aligned, the +port reads transparently. If you have to scroll one twice as fast as +the other, there's structural drift to clean up. + +--- + +## Phase 4 — Production wiring (post-port) + +Once the LLIR matches *and* the visual-transparency pass is done: + +1. **Side-by-side review.** Pull up Python and Zig in adjacent panes + one more time. Anything that still reads differently between them + either has a justification (DSL constraint, language constraint — + leave a brief comment explaining why) or needs another pass. +2. **Remove any `// TODO match python` comments** from the kernel body. +3. **Verify the production caller passes the right cfg** — the dump + tool's config and the production caller's config can differ (the + dump tool uses small dimensions; production uses real ones). + Production behavior depends on the runtime cfg, so make sure + `makeFooConfig(...)` in the caller produces sensible values. +4. **Run the model end-to-end** if possible. The comparison harness + only validates IR equivalence — runtime correctness still needs a + real test (mismatched grid, missing strides, etc. won't show up in + the diff but will crash at execution time). + +--- + +## Cheat-sheet: when to look at which file + +| Problem | File | +|---------------------------------------------------|----------------------------------------------| +| "How do I express `tl.X(...)` in Zig?" | `kernels/triton/README.md` (Python ↔ Zig table) | +| "What does `b.declareArgs(...)` accept?" | `kernels/triton/README.md` (Argument specs) | +| "How do I call this kernel from `forward`?" | `kernels/triton/README.md` (`K.call(...)`) | +| "My kernel diverges from Python at LLIR" | This file (Phase 3) | +| "How do I add a kernel to the comparison harness?"| Phase 1 above + `examples/triton_emitter/` | +| "I want to inspect raw TTIR ops" | `mlir/dialects/ttir/ttir.zig` (escape hatch) | + +--- + +## When you're stuck + +If a divergence resists all the standard fixes: + +1. **Print both TTIRs side by side.** The visual structure usually + reveals which control-flow region or loop body is misaligned. +2. **Check if the Python kernel uses any non-`@triton.jit` helpers** — + if it imports something from `triton.language.extra` or has a custom + `@triton.jit` helper, that helper might also need a port (or a + `_SKIP` entry if it's pure-comptime). +3. **Verify the Python override args match the Zig config.** Mismatched + constexpr values (e.g. `BLOCK_SIZE=64` in Zig vs `1024` auto-default + in Python) generate completely different IR. +4. **Compare against `zml/moe/triton_kernels.zig` MoE kernels** — they + went through this loop and are reasonable references for the patterns + above. + +--- + +## Before "fixing" the DSL + +Every time a divergence makes you suspect the DSL itself is buggy and you +reach for `kernels/triton/builder.zig` — **verify against the Triton sources +first** at `/Users/raph/Documents/Git-Repos/triton`. The DSL exists to +mirror what Python's frontend emits; the source of truth is the Python +trace, not what the rule "looks like" or what feels symmetric. + +Concretely: + +- If you think `Value.cdiv` is wrong, read + `triton/python/triton/language/standard.py` for the definition of + `tl.cdiv` and compare. Don't assume the desugaring — check. +- If you think a binop dispatches to the wrong arith op + (`maxnumf` vs `maximumf`, `divsi` vs `divui`, …), read + `triton/python/triton/language/semantic.py` and find the cast / + promotion rule that actually fires. +- If you think the type-promotion is wrong (i32 stays i32, doesn't + auto-extend to i64, …), read `semantic.py:cast()` and the binop + promotion helpers. + +A second pitfall: **the local kernel may define its own helper that +shadows the stdlib name.** `unified_attention.py` defines a private +`cdiv_fn(x, y) = (x + y - 1) // y` — left-to-right Python parses this +as `((x + y) - 1)`, so it emits `addi` then `subi`. Triton's *stdlib* +`tl.cdiv` is `(x + (div - 1)) // div` and emits `subi` then `addi`. +These desugar identically when the divisor is comptime (Triton folds +`div - 1` at trace time), but differ when the divisor is a runtime +tensor. If your divergence is just an `add↔sub` reorder, check whether +the Python kernel actually uses `tl.cdiv` or a private `cdiv`-shaped +helper, and either keep the DSL aligned with `tl.cdiv` and write a +local Zig helper for the kernel-private form, or the other way around. + +In short: when a "bug" in the DSL would also affect kernels that +already match Python, the bug is almost certainly elsewhere. + +**And if you do change the DSL**, run `examples/triton_emitter/run.sh` +afterwards and confirm no kernel that previously matched (or was +codegen-equivalent) regressed at LLIR/PTX. A DSL edit that fixes one +kernel by introducing a divergence in another is not a fix — it's a +trade. Always check the full table before committing the change. diff --git a/kernels/triton/README.md b/kernels/triton/README.md new file mode 100644 index 00000000..5cb406e2 --- /dev/null +++ b/kernels/triton/README.md @@ -0,0 +1,1091 @@ +# ZML Triton DSL — Author's Guide + +A Zig builder for Triton IR (TTIR) that tries to read like `triton.language` +Python while staying in pure Zig. Used by `zml/moe/triton_kernels.zig`, +`zml/attention/triton_kernels.zig`, and by anything else that wants to call +Triton kernels from a ZML model. + +Two layers define the surface: + +- **Low layer** — `kernels/triton/builder.zig`: the IR builder primitives. + `Builder`, `Value`, `DType`, `ArgSpec`. Use these directly when you need + full control (custom op insertion, dynamic-arity arg lists, escape hatches). +- **High layer** — `zml.kernel.triton.Kernel(Config, spec)`: the declarative + kernel form. Bundles a config type, a typed spec literal (name + named + `inputs` / `outputs` + a typed `run` function pointer), and produces a + kernel type. Exposes `.emit(allocator, ctx, cfg)` (TTIR string) and + `.call(inputs, outputs, opts)` (TTIR string + `stablehlo.custom_call` in + one shot — the preferred form for production model code). Inputs and + outputs are by-name structs generated from the spec, so missing / typo'd + / extra arguments are compile errors at the callsite. + +## Table of contents + +1. [Skeleton of a kernel](#skeleton-of-a-kernel) +2. [`zml.kernel.triton.Kernel(Config, spec)` — the declarative form](#zmlkerneltritonkernelconfig-spec--the-declarative-form) +3. [Calling kernels from a ZML model — `K.Kernel.call(...)`](#calling-kernels-from-a-zml-model--kkernelcall) +4. [Argument specs](#argument-specs) +5. [The `Value` type](#the-value-type) +6. [Fluent methods on `Value`](#fluent-methods-on-value) +7. [Polymorphic scalars](#polymorphic-scalars) +8. [Comptime vs runtime widths](#comptime-vs-runtime-widths) +9. [Shape helpers](#shape-helpers) +10. [Loads and stores](#loads-and-stores) +11. [Reductions and scans](#reductions-and-scans) +12. [Control flow](#control-flow) +13. [Escape hatches](#escape-hatches) +14. [Lower layer — `Builder.build`](#lower-layer--builderbuild) +15. [Python Triton ↔ Zig DSL cheat-sheet](#python-triton--zig-dsl-cheat-sheet) +16. [Pitfalls](#pitfalls) + +--- + +## Skeleton of a kernel + +```zig +const tri = @import("zml").kernel.triton; +const zml = @import("zml"); + +pub const MyKernel = struct { + pub const Cfg = struct { BLOCK: i32 = 64 }; + pub const Kernel = tri.Kernel(Cfg, .{ + .name = "my_kernel", + .inputs = &.{ "x_ptr", "y_ptr", "n" }, + .outputs = &.{"out"}, + .run = run, + }); + fn run(b: *tri.Builder, cfg: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ + .x_ptr = .{ .ptr = .f32 }, + .y_ptr = .{ .ptr = .f32 }, + .n = .{ .scalar = .i32 }, + .out_ptr = .{ .ptr = .f32 }, + }); + + const pid = b.programId(.x); + const offs = b.arange(0, cfg.BLOCK, .i32).add(pid.mul(cfg.BLOCK)); + const mask = offs.lt(a.n); + + const x = b.loadOpts(a.x_ptr.addPtr(offs), .{ .mask = mask }); + const y = x.mul(2.0); + b.storeOpts(a.out_ptr.addPtr(offs), y, .{ .mask = mask }); + } +}; +``` + +A kernel is a `struct {…}` wrapper bundling three decls: + +1. **`pub const Cfg`** — config type, an inline `struct { ... }` of + comptime/runtime parameters. Use field defaults for sensible defaults. +2. **`pub const Kernel = tri.Kernel(Cfg, .{ .name, .inputs, .outputs, .run })`** — + the spec literal that turns into a kernel type. `inputs` / `outputs` are + slices of `[:0]const u8` field names; the factory generates `Inputs` + (struct of `Tensor`), `Outputs` (struct of `Shape`), and `Results` + (struct of `Tensor`) types from those names. The `run` field is a typed + function pointer — wrong signature is a compile error here. +3. **`fn run(b, cfg)`** — the kernel body. First call `try b.declareArgs(.{…})` + to create the `tt.func` op and bind a struct of `Value`s named after the + spec's fields. The rest of `run` builds the body via `b.*` helpers. No + string return, no allocator, no manual `tt.return` — the factory + terminates, verifies, and serializes for you. + +The names in `spec.inputs` / `spec.outputs` are the **call-site** names +(used to build the `Inputs` / `Outputs` structs) and don't have to match +the names in `b.declareArgs(...)` (which are the **TTIR-level** arg names). +By convention they often do — pick whatever reads best at the boundary. + +The arg spec inside `b.declareArgs(...)` is a struct literal — its variant +*tags* (`.ptr`, `.scalar`, etc.) are comptime-known per call site, but the +*payload values* (`cfg.input_dtype`, `cfg.output_dtype`, …) can be runtime +values. This is what makes the same form work for both comptime-pinned +configs (offline tools, simple kernels) and runtime configs (production +models where dtypes/sizes come from `Tensor.dim()` / `Options`). + +> **Why the wrapper struct.** `Cfg` has to be a *named* type so the typed +> `run: *const fn (*Builder, Cfg) FinishError!void` field in the spec can +> reference it. Wrapping the kernel in `pub const MyKernel = struct {...}` +> keeps `Cfg`, `Kernel`, and `run` scoped together and exposes them all to +> callers (`MyKernel.Cfg`, `MyKernel.Kernel`, plus everything the inner +> Kernel re-exports — `MyKernel.Kernel.Config`, `.Inputs`, `.Outputs`, +> `.Results`, `.CallOpts`). + +### Two ways to use a kernel + +**Offline / tooling** — get the TTIR string: + +```zig +const ttir = try MyKernel.Kernel.emit(allocator, ctx, .{ .BLOCK = 1024 }); +defer allocator.free(ttir); +``` + +**Inside a ZML `forward(...)`** — emit TTIR and drop a `stablehlo.custom_call` +into the model in one shot: + +```zig +const out = MyKernel.Kernel.call( + .{ .x_ptr = x_tensor, .y_ptr = y_tensor, .n = n_tensor }, // Inputs + .{ .out = x_tensor.shape() }, // Outputs (Shapes) + .{ + .cfg = .{ .BLOCK = 1024 }, + .grid = .{ ceilDiv(n, 1024), 1, 1 }, + .num_warps = 4, + .num_stages = 2, + }, +).out; +``` + +`Kernel.call` returns a `Results` struct (one `Tensor` per output name); +read fields by name (`out.out`, `align_outs.sorted_token_ids`, …). + +--- + +## `zml.kernel.triton.Kernel(Config, spec)` — the declarative form + +`zml.kernel.triton.Kernel(Config, spec)` is a comptime function returning a +kernel type. `Config` is the per-call config type (used both for the +generated `Inputs`/`Outputs`/`CallOpts` and for the typed `run` signature); +`spec` is a `Spec(Config)` literal: + +| Field | Type | Meaning | +|-------------|---------------------------------------------------|---------------------------------------------------------------------| +| `.name` | `[:0]const u8` | Kernel name (used as the `tt.func` symbol). | +| `.inputs` | `[]const [:0]const u8` | Field names for the call-site `Inputs` struct (fields are `Tensor`). Order must match the `tt.func` arg order for input pointers. | +| `.outputs` | `[]const [:0]const u8` | Field names for the `Outputs` (struct of `Shape`) and `Results` (struct of `Tensor`). Order must match the `tt.func` arg order for output pointers — XLA's lowering passes inputs first, then outputs. | +| `.run` | `*const fn (*Builder, Config) FinishError!void` | Typed function pointer to the kernel body. Wrong signature → compile error here, not at first call. | + +Wrong arity, missing field, typo'd name on the call site → compile error +at the call site (because `Inputs` / `Outputs` are real struct types, not +tuples). + +`run` is responsible for declaring the kernel's args (via +`b.declareArgs(...)`) and emitting the body. It emits ops via `b.*` helpers +and never returns a string; `Kernel.emit(...)` does the +terminate-verify-serialize dance after `run` returns. + +--- + +## Calling kernels from a ZML model — `K.Kernel.call(...)` + +Inside a model's `forward(...)`, drop in a Triton kernel alongside regular +ZML ops with `K.Kernel.call(...)`: + +```zig +pub fn forward(self: Model, x: zml.Tensor, y: zml.Tensor) zml.Tensor { + const x_pre = x.mul(2.0); // stablehlo + + const out = MyKernel.Kernel.call( + .{ .x_ptr = x_pre, .y_ptr = y, .n = n_tensor }, // Inputs (by name) + .{ .out = x_pre.shape() }, // Outputs (Shapes) + .{ + .cfg = .{ .BLOCK = 1024 }, // runtime config + .grid = .{ ceilDiv(x.dim(0), 1024), 1, 1 }, + .num_warps = 4, + .num_stages = 2, + }, + ).out; // .out: Tensor + + return out.relu(); // stablehlo +} +``` + +`call` is a thin wrapper over `Kernel.emit(...)` + `zml.ops.triton(...)`. +It must be called from inside a `CompilationContext` (i.e., during +`zml.module.compile(...)`). For offline tools without a CompilationContext +(e.g., dump-to-disk diff harnesses), use `Kernel.emit(allocator, ctx, cfg)` +directly to get the TTIR string. + +Signature: `Kernel.call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results`. + +| Positional arg | Type | Meaning | +|----------------|------------------|-------------------------------------------------------------------------| +| `inputs` | `Kernel.Inputs` | Struct of `Tensor`, one field per name in `spec.inputs`. | +| `outputs` | `Kernel.Outputs` | Struct of `Shape`, one field per name in `spec.outputs`. | + +`Kernel.CallOpts` (typed struct, all fields named): + +| Field | Type | Meaning | +|---------------------------|-----------------------|------------------------------------------| +| `.cfg` | `Kernel.Config` | Per-call kernel config. | +| `.grid` | `[3]i32` | Triton launch grid `(grid_x, grid_y, grid_z)`. | +| `.num_warps` | `i32` | Triton launch param. | +| `.num_stages` | `i32` | Triton launch param. | +| `.output_operand_aliases` | `[]const ...` (default `&.{}`) | For in-place updates (still indexed by output position — `.{ .output_index = 0, .operand_index = 1 }`). | +| `.debug` | `bool` (default false)| Emit `tt.debug = true` on the custom call. | + +Returns `Kernel.Results` — a struct of `Tensor` named like `spec.outputs`. +Read by name: `out.out`, `attn_output.segm_max`, etc. + +The `cfg` value can be a comptime literal *or* a runtime expression built +from `Tensor.dim()` / `Options` fields. The kernel TTIR is regenerated per +call (cached by the MLIR canonicalizer / CSE pass at the XLA layer). + +> **Why named `Inputs` / `Outputs`.** The previous form took +> `anytype` tuples and matched by position; misordering 27 args (typical +> in `unified_attention.zig`) was a silent corruption hazard. With +> generated structs (`Inputs = StructOf(spec.inputs, Tensor)`, +> `Outputs = StructOf(spec.outputs, Shape)`), Zig refuses unknown fields, +> missing fields, and wrong-typed values at the callsite. + +--- + +## Argument specs + +Each field passed to `b.declareArgs(...)` is an `ArgSpec.Kind` tagged-union +literal. + +The four arg kinds mirror what TTIR's `tt.func` can carry: + +| Kind literal | MLIR type | Use case | +|---------------------------------------------------------|--------------------------|---------------------------------| +| `.{ .ptr = .f32 }` | `!tt.ptr` | Pointers (default divisibility) | +| `.{ .scalar = .i32 }` | plain `i32` | Strides, sizes, flags | +| `.{ .tensor = .{ &.{64, 128}, .f32 } }` | `tensor<64x128 x f32>` | Pre-built tensor input (rare) | +| `.{ .ptr_opts = .{ .dtype = .f32, .divisibility = 16 } }` | `!tt.ptr` with override | Custom `address_space`/divisibility | +| `.{ .scalar_opts = .{ .dtype = .i32, .divisibility = 16 } }` | plain `i32` with `tt.divisibility = 16 : i32` | Scalar with specialization hint | + +`.ptr` gets a default `tt.divisibility = 16` hint (matching Python Triton's +default). Use `.ptr_opts` to change that (or set `divisibility = null` to +suppress). `.scalar` carries no extra hint; switch to `.scalar_opts` when +you want to declare the same `tt.divisibility` Python's JIT auto-attaches +to int args whose runtime value is a multiple of 16. + +**Runtime dtypes:** the variant *tags* (`.ptr`, `.scalar`, etc.) must be +fixed per kernel — every call to `b.declareArgs(...)` for a given kernel +uses the same struct shape. The variant *payloads* (the dtype enum, the +divisibility int) can be runtime values from `cfg`: +`.a_ptr = .{ .ptr = cfg.a_dtype }` works fine even when `cfg.a_dtype` is a +runtime field. + +--- + +## The `Value` type + +```zig +pub const Value = struct { + inner: *const mlir.Value, + kernel: ?*Builder = null, + ... +}; +``` + +- `inner` is the underlying MLIR SSA handle; reach for it when you need to call + into raw `ttir.*` / `arith.*` constructors. +- `kernel` is the owning `Builder` pointer; all fluent methods route through it. + The DSL populates `kernel` automatically for every value it creates (`emit`, + `arg`, loop-region args, reduction-region args, etc.). If you construct a + `Value` by hand (`.{ .inner = raw }`), the fluent methods will panic — use + `Builder.*` static helpers instead. + +Introspection methods always available: + +| Method | Returns | Notes | +|-------------------|-----------------------------------|-----------------------------------------| +| `type_()` | `*const mlir.Type` | Full MLIR type (scalar or tensor). | +| `elemType()` | `*const mlir.Type` | Element type; same as `type_()` for scalars. | +| `isTensor()` | `bool` | `true` for ranked tensors. | +| `rank()` | `usize` | `0` for scalars. | +| `dim(i)` | `i64` | Traps on scalars. | +| `shape()` | `Shape` (BoundedArray, `MAX_RANK`)| Use `.constSlice()` to feed `[]const i64`. | +| `isFloatElem()` | `bool` | True for any float element type. | +| `isIntElem()` | `bool` | True for any integer element type. | + +--- + +## Fluent methods on `Value` + +Every fluent method returns a new `Value`. They dispatch to +`arith.addi` / `arith.addf` / ... based on the element type. The RHS is +`anytype` — see [Polymorphic scalars](#polymorphic-scalars). + +### Arithmetic + +| Method | Int op | Float op | Notes | +|----------------|--------------|--------------|------------------------------------| +| `.add(rhs)` | `arith.addi` | `arith.addf` | | +| `.sub(rhs)` | `arith.subi` | `arith.subf` | | +| `.mul(rhs)` | `arith.muli` | `arith.mulf` | | +| `.div(rhs)` | `arith.divsi`| `arith.divf` | Signed integer divide. | +| `.rem(rhs)` | `arith.remsi`| `arith.remf` | Signed integer remainder. | +| `.cdiv(rhs)` | `arith.ceildivsi` | — | Int only — for `tl.cdiv`. | +| `.minimum(rhs)`| `arith.minsi`| `arith.minnumf` | Matches `tl.minimum` (non-NaN-prop). | +| `.maximum(rhs)`| `arith.maxsi`| `arith.maxnumf` | Matches `tl.maximum` (non-NaN-prop). | +| `.bitAnd(rhs)` | `arith.andi` | — | Also handles i1 masks. | +| `.bitOr(rhs)` | `arith.ori` | — | | + +### Comparisons + +All return i1 (scalar) or `tensor<... x i1>`. Signed integer predicates by +default; `.olt` / `.oge` / ... for floats. + +| Method | Int predicate | Float predicate | +|--------------|---------------|-----------------| +| `.lt(rhs)` | `.slt` | `.olt` | +| `.le(rhs)` | `.sle` | `.ole` | +| `.gt(rhs)` | `.sgt` | `.ogt` | +| `.ge(rhs)` | `.sge` | `.oge` | +| `.eq(rhs)` | `.eq` | `.oeq` | +| `.ne(rhs)` | `.ne` | `.one` | + +### Casts — `.to(dtype)` + +`v.to(.f32)` chooses the right MLIR op based on the current vs target kind: + +| Source kind | Target kind | MLIR op | +|-------------|------------------|-------------------------------| +| int | wider int | `arith.extsi` | +| int | narrower int | `arith.trunci` | +| int | float | `arith.sitofp` | +| **i1** | **float** | **`arith.uitofp`** (i1 is unsigned bool; sign-extending `1` would land at `-1.0`) | +| float | int | `arith.fptosi` | +| float | wider float | `arith.extf` (no rounding) | +| float | narrower float | fp8 involved or custom rounding → `tt.fp_to_fp`; otherwise `arith.truncf` | +| same type | same type | no-op (returns `self`) | + +Triton's verifier rejects `tt.fp_to_fp` with a rounding attribute when the +target is not strictly narrower than the source (e.g. `bf16 → f32`, or +same-width `bf16 ↔ f16`). The DSL routes strict upcasts through +`arith.extf` to match Python Triton's own lowering; only genuine +downcasts carry the `rtne` rounding attribute. + +For narrowing FP→FP, the dispatch follows +`triton/python/triton/language/semantic.py:cast()`: standard FP narrowing +between fp64 / fp32 / fp16 / bf16 lowers to `arith.truncf` (rank-independent +— scalar and tensor both), while `tt.fp_to_fp` is reserved for cases where +fp8 is on either side or the caller requested a non-default +`fp_downcast_rounding`. + +### Constant hoisting and CSE + +Every `arith.constant` emitted by the DSL — whether through `lift`, `splat` +of a comptime literal, `zeros` / `ones` / `full`, or any internal helper — +is deduped against a per-`Builder` cache keyed by `:`. On a hit +the existing SSA value is reused; on a miss the new op is inserted at the +*top* of the entry block (before any non-constant op already there) so the +final IR has all constants gathered above the function body, in roughly +first-use order. + +This mirrors what Triton's Python frontend produces and keeps the +`make_ttgir` layout pass from seeing duplicate constants — duplicates can +change the layout-slot assignment for kernels with `tt.dot`, since layout +inference walks the IR and the *first* tensor it sees with a given shape +gets `#blocked`, the second gets `#blocked2`, etc. + +`zeros(shape, dtype)`, `ones(shape, dtype)`, and `full(shape, value, dtype)` +emit a single `arith.constant dense:tensor<...>` (instead of the +old scalar-const + `tt.splat` two-op pattern); same for `splat(comptime, +shape)`. `splat(Value, shape)` still emits `tt.splat` since the operand is +a runtime value, not a literal. + +### Shape / pointer helpers + +| Method | Effect | +|------------------------------|-------------------------------------------------------------| +| `v.expandDims(axis)` / `b.expandDims(v, axis)` | `tt.expand_dims` — insert a size-1 axis at `axis`. Mirrors `v[:, None]` (`axis=1`) / `v[None, :]` (`axis=0`). | +| `b.broadcastTo(v, shape)` | `tt.broadcast` — explicit broadcast (size-1 dims → target). Rarely needed; auto-broadcast handles it. | +| `.splatTo(shape)` | `tt.splat` this scalar to the given shape. | +| `.addPtr(offset)` | `tt.addptr`; `offset` can be a Value or a comptime int. | + +> **Auto-broadcast is everywhere.** Mirrors Triton's +> `semantic.broadcast_impl_value` (`triton/python/triton/language/semantic.py:724`): +> rank-unification (prepend size-1 axes) + size-1 → size-N expansion. +> Applied transparently at: +> - every fluent binop (`.add`, `.mul`, `.lt`, `.bitAnd`, …) +> - `.addPtr(offset)` (either side may be scalar or rank-mismatched) +> - `b.where(cond, x, y)` / `b.select(...)` +> - `mask` and `other` of `b.loadOpts` / `b.storeOpts` +> +> So `v[:, None] * stride + w[None, :] * other_stride` is just: +> +> ```zig +> v.expandDims(1).mul(stride).add(w.expandDims(0).mul(other_stride)) +> ``` +> +> No `broadcastTo`, no full-shape `tt.splat` — the broadcast materializes +> implicitly at the `addi`. This matches Python's emit pattern op-for-op +> (Python's frontend never emits an eager `tt.broadcast` followed by a +> `arith.muli` either; it emits `expand_dims` + `splat + muli`). +> +> Reach for `b.broadcastTo` only when the consumer is a low-level op that +> bypasses the DSL's auto-broadcast — direct `arith.*` / raw `ttir.*` +> escape-hatch paths. + +--- + +## Polymorphic scalars + +Every fluent method's RHS (and every `b.splat` / `b.lift` input) is `anytype`. +The lift rule: + +- `Value` → pass through. +- `comptime_int` / `comptime_float` → a constant whose element type matches + the LHS operand. `v_i64.mul(16)` lifts `16` to i64, `v_f32.add(1.0)` lifts + `1.0` to f32, and so on. +- Runtime `i8/i16/i32`, `i64`, `f16/f32/f64`, etc. → a constant that preserves + the source Zig width. i8/i16/i32 (signed or unsigned) → i32 const; i64/u64 → + i64 const; f16 → f16; f32 → f32; f64 → f64. Unsigned ints are bit-cast + (bit pattern preserved), so `@as(u32, 0xFFFF_FFFF)` becomes an i32 with the + same all-ones bit pattern. + +This is why the common Python idiom `offs < 64` works in Zig as +`offs.lt(64)` without any wrapper — as long as the comptime scalar can be +represented in `offs`'s element type. + +If you need to force a specific width for a runtime variable, cast at the call +site: `pid.mul(@as(i32, @intCast(block_size)))`. + +### When is `@as` actually needed? (Rule of thumb) + +Auto-lift covers most cases. `@as(...)` around a binop rhs is **redundant** +in all of these — drop it: + +| Pattern | Auto-lifts to… | +|------------------------------------------|---------------------------------------| +| `v_i64.eq(0)` / `.div(16)` / `.add(-1)` | comptime_int → match lhs elem type | +| `v_f32.mul(1.0 / config.fp8_max)` | already-f32 expression → f32 const | +| `v_i64.lt(NUM_QUERY_HEADS)` (Zig i64) | runtime i64 → i64 const | +| `v_i32.add(pad_term)` (Zig i32) | runtime i32 → i32 const | +| `b.splat(-1, shape)` / `b.openFor(0, N, 1, ...)` | comptime → matches elem / lower | +| `b.liftAs(0, .i32)` | liftAs takes anytype — wrap nothing | + +`@as(...)` **is** required exactly when you're narrowing a runtime Zig int +to a smaller width that `lift` won't pick on its own. The idiom is +`@as(i32, @intCast(X_i64))` — you need the `@as` because `@intCast` has no +target type to infer, and without the narrowing `lift` would emit an i64 +constant that mismatches the i32 lhs: + +```zig +// BLOCK_Q is a Zig i64 (from config). lhs is i32. Must narrow. +q_start_raw.div(@as(i32, @intCast(BLOCK_Q))).add(seq_idx) +``` + +The alternative — `q_start_raw.to(.i64).div(BLOCK_Q).to(.i32)` — mirrors +Python's `.to(tl.int64)` style but costs two extra Value-level casts; prefer +the narrowing `@as` or a pre-declared `const FOO_I32: i32 = @intCast(FOO);` +when the same width cast recurs. + +### When do you need `b.lift` / `b.liftAs`? + +Almost never. The DSL auto-lifts rhs for every fluent binop, `b.splat`, +`b.full`, `b.openFor` bounds, `.addPtr` offsets, etc. Reach for `b.lift` / +`b.liftAs` **only** when the DSL surface demands an existing `Value` and +there's no binop sibling to match against — typically: + +- Loop seeds for `openWhile(.{inits}, ...)` where you start with a raw + comptime literal: `const left_init = b.liftAs(0, .i32);`. +- Mutable `var x: Value = b.liftAs(0, .i32);` that a later branch may + reassign — the var needs an initial Value even if the comptime path would + work on its own. + +If the constant is about to flow through any `.add` / `.mul` / `.splatTo` +etc., skip the lift and let auto-lift do it. And if a stride is hardcoded +to 1, **delete it**: `x.mul(1)` is a no-op, the DSL isn't Python `constexpr` +where such names are required. + +### Scalar ↔ tensor auto-broadcast + +Fluent arithmetic and comparison ops (`.add`, `.sub`, `.mul`, `.div`, `.rem`, +`.cdiv`, `.bitAnd`, `.bitOr`, `.min`, `.max`, `.lt`, `.le`, `.gt`, `.ge`, +`.eq`, `.ne`) are symmetric: if one operand is a tensor and the other a +scalar, the scalar is splatted to match. Either direction works: + +```zig +const offs = iv.add(cols); // scalar iv + tensor cols → tensor +const offs = cols.add(iv); // tensor cols + scalar iv → tensor +``` + +This mirrors Triton/NumPy's `i + cols`. **Size-1 and rank-unification +broadcasting also work** (mirroring Python's +`semantic.broadcast_impl_value`): a `(M, 1)` and a `(1, N)` operand +broadcast to `(M, N)` automatically, and a `(M,)` lined up against a +`(M, N)` is first `expand_dims`'d to `(1, M)` and then broadcast. So +`offs_token[:, None] + offs_k[None, :]` is just: + +```zig +const offs = b.expandDims(offs_token, 1).add(b.expandDims(offs_k, 0)); +``` + +with no `b.broadcastTo` calls needed. `b.where` / `b.select` and the +`mask` / `other` operands of `b.loadOpts` / `b.storeOpts` also auto-broadcast, +so the only time `b.broadcastTo` is needed is when the consumer bypasses +the DSL (raw `arith.*` / `ttir.*` escape-hatch paths). + +`.addPtr` auto-splats in **both directions**, matching Triton's +`ptr + offset` semantics: a scalar pointer is splatted to match a tensor +offset (`x_ptr.addPtr(tensor_offs)`), and a scalar offset is splatted to +match a tensor pointer (`a_ptrs.addPtr(BLOCK_K)`). Pre-splatting via +`b.splat` is no longer required for the loop-stride case. + +### When is `scalar.splatTo(shape)` redundant? + +Because fluent binops (`.add` / `.mul` / …) already auto-splat scalars +against tensor siblings, and `.addPtr` auto-splats either side (scalar +pointer or scalar offset) against a tensor partner, a pre-emptive +`scalar.splatTo(shape)` is usually noise: + +```zig +// redundant +pid_n.mul(N).splatTo(shape).add(b.arange(0, N, .i64)).rem(n_blocb.splatTo(shape)) +// equivalent — scalars auto-splat against the tensor sibling +pid_n.mul(N).add(b.arange(0, N, .i64)).rem(n_block) +``` + +Keep the explicit `splatTo` only when the scalar feeds into a raw +`arith.*` / `ttir.*` escape-hatch op that doesn't go through the DSL's +auto-broadcast. + +--- + +## Comptime vs runtime widths + +TTIR has no implicit int-width conversion — `arith.muli(i32, i64)` is a verifier +error. The DSL mirrors Python and **auto-promotes** integer binops when the +two runtime operands have different widths: it `extsi`s the narrower side +to the wider, matching `semantic.integer_promote_impl`. So +`v_i32.mul(v_i64)` works without a manual `.to(.i64)` — it emits exactly +the same `extsi` + `muli` pair Python would. + +For the rare case where you want explicit control, you can still cast +yourself; the auto-promotion is a no-op when widths already agree. + +Recurring example from `triton_kernels.zig`: + +```zig +// pid (i32) vs block (i64 because it came from a usize config) +const block_i32: i32 = @intCast(block); +const token_start_init = pid.mul(block_i32); +// alternative: pid.to(.i64).mul(block) +``` + +For pointer math it is often cleaner to promote everything to i64 up front: + +```zig +const pid = b.programId(.x).to(.i64); +const offs = b.arange(BLOCK, .i64); +``` + +--- + +## Shape helpers + +| Helper | Effect | +|----------------------------------------|--------------------------------------------------| +| `b.arange(start, end, dtype)` | `tl.arange(start, end).to(dtype)`. Pass `.i32` for no-op cast. | +| `b.zeros(shape, dtype)` | `tl.zeros(shape, dtype=dtype)`. | +| `b.ones(shape, dtype)` | Ones tensor. | +| `b.splat(value, shape)` | `tl.splat`; accepts Value or comptime scalar. | +| `b.expandDims(v, axis)` / `v.expandDims(axis)` | `tt.expand_dims` — insert a size-1 axis. Mirrors `v[:, None]` (`axis=1`) / `v[None, :]` (`axis=0`). | +| `b.mask2d(cond_m, cond_n, _, _)` | `cond_m[:, None] & cond_n[None, :]` — emits the `(m,1) & (1,n)` form; auto-broadcast at the next consumer. | +| `b.lift(value)` | Wrap a Zig scalar as a DSL Value; dtype inferred from source. | +| `b.liftAs(value, dtype)` | Wrap a Zig scalar as a DSL Value of a specific `DType`. | + +--- + +## Loads and stores + +Load helpers infer everything from the pointer's type — scalar `!tt.ptr` +loads a scalar `T`, `tensor<... x !tt.ptr>` loads a `tensor<... x T>`. +No `result_type` / `shape` / `dtype` arguments required. + +| Helper | Python equivalent | +|-------------------------------------------------|-------------------------------------------| +| `b.load(ptr)` | `tl.load(ptr)` — scalar or tensor. | +| `b.loadOpts(ptr, .{ .mask, .other, ... })` | `tl.load(ptrs, mask=..., other=..., cache_modifier=..., eviction_policy=..., volatile=...)`. | +| `b.store(ptr, value)` | `tl.store(ptr, value)`. | +| `b.storeOpts(ptr, value, .{ .mask, ... })` | `tl.store(ptr, value, mask=..., cache_modifier=..., eviction_policy=...)`. | + +`LoadOpts` / `StoreOpts` fields (`.mask`, `.other`, `.cache_modifier`, +`.eviction_policy`, `.@"volatile"`) take DSL-level `Value`s / enum literals — +no raw MLIR types needed. + +> **Match Python's emit form for masked loads.** `tl.load(ptr, mask=m, +> other=0.0)` lowers to a 3-operand `tt.load` op. To match it byte-for-byte +> in the Zig DSL, pass an explicit `.other`: +> +> ```zig +> const zero = b.zeros(&.{BLOCK}, dtype); +> const x = b.loadOpts(ptr, .{ .mask = mask, .other = zero }); +> ``` +> +> Calling `loadOpts` with `.mask` only (no `.other`) emits a 2-operand +> `tt.load` — semantically equivalent (out-of-bounds lanes are undefined, +> not zero), but the TTIR/TTGIR diff vs Python will not be empty. Triton's +> `tl.load(ptr, mask=...)` without `other=` lowers to the 2-operand form, +> so you only need the `.other` operand when the Python source explicitly +> passes `other=`. + +### Result-type inference + +Most helpers that produce a new Value infer their result type from the inputs, +so user code never constructs `*const mlir.Type` manually: + +| Helper | How the result type is derived | +|-----------------------------------------------------|---------------------------------------------| +| `b.load(ptr)` / `b.loadOpts(ptr, opts)` | From `ptr.type_()` (scalar or tensor of ptrs). | +| `b.dot(a, b, acc)` / `b.dotOpts(a, b, acc, opts)` | `acc.type_()`. | +| `b.dotScaled(...)` / `b.dotScaledOpts(...)` | `acc.type_()`. | +| `b.reshape(src, new_shape)` / `b.reshapeOpts(...)` | `new_shape` + `src` element type. | +| `b.gather(src, indices, axis)` / `b.gatherOpts(...)`| `indices` shape + `src` element type. | +| `b.histogram(src, num_bins)` / `b.histogramOpts(...)` | `tensor`. | +| `b.fpToFp(src, out_dtype)` / `b.fpToFpOpts(...)` | `src` shape + `out_dtype`. | +| `b.convertf(src, out_dtype)` / `b.convertfOpts(...)`| `src` shape + `out_dtype`. | +| `b.scalingTruncf(src, scale, out_dtype)` / …Opts | `src` shape + `out_dtype`. | +| `b.descriptorLoad(desc, indices, shape, dtype)` | `shape` + `dtype`. | + +Rule of thumb: you only supply things the op needs beyond what inputs carry — +a `new_shape`, a `num_bins`, an `out_dtype`, a TMA tile descriptor. The DSL +never asks you to restate information it can read off a `Value`. + +`b.scalarTy(dtype)` / `b.tensorTy(shape, dtype)` are still available but only +needed by `openIf.result_types` / `openWhile.after_types` (where block-arg +types can be arbitrary) and by the raw `externElementwise` / +`inlineAsmElementwise` escape hatches. + +### `fn X / fn XOpts` pattern + +Every Builder method whose Python Triton counterpart has named keyword-only +arguments comes in two flavours: + +- `b.X(non_named_args...)` — defaults applied; matches `tl.X(args...)` in + Python, which in Python would use the zero-arg defaults. +- `b.XOpts(non_named_args..., struct { named... })` — full named-param set + as a typed Zig struct literal with field defaults matching Triton. + +Pairs in place today: `load` / `loadOpts`, `store` / `storeOpts`, +`dot` / `dotOpts`, `reshape` / `reshapeOpts`, +`atomicRmw` / `atomicRmwOpts`, `atomicCas` / `atomicCasOpts`, +`fpToFp` / `fpToFpOpts`, `clampf` / `clampfOpts`, +`cast` / `castOpts`, `cat` / `catOpts`, +`convertf` / `convertfOpts`, `scalingTruncf` / `scalingTruncfOpts`, +`dotScaled` / `dotScaledOpts`, +`inlineAsmElementwise` / `inlineAsmElementwiseOpts`, +`externElementwise` / `externElementwiseOpts`, +`descriptorLoad` / `descriptorLoadOpts`, `histogram` / `histogramOpts`, +`devicePrint` / `devicePrintOpts`, +`deviceAssert` / `deviceAssertOpts`, +`maximum` / `maximumOpts`, `minimum` / `minimumOpts`, +`sum` / `sumOpts`, `max` / `maxOpts`, `min` / `minOpts`, +`cumsum` / `cumsumOpts`, `cumprod` / `cumprodOpts`. + +Ops with no kwargs in Python (pure positional): `gather(src, indices, axis)`, +`trans(src, order)`, `permute(src, order)`, `cat(lhs, rhs)` (dim=0 default), +`split(src)`, `join(lhs, rhs)`, `item(src)`, `broadcast(a, b)` (symmetric), +`broadcastTo(value, shape)`, `where(cond, x, y)`, `zeros(shape, dtype)`, +`full(shape, value, dtype)`. + +--- + +## Reductions and scans + +```zig +const absmax = b.max(abs_y); // tl.max(abs_y) — axis=None (all dims) +const total = b.sum(padded_counts); // tl.sum(padded_counts) +const cumul = b.cumsum(x); // tl.cumsum(x, axis=0) + +// Keyword arguments → the *Opts variant with a typed struct literal. +const axis0_max = b.maxOpts(abs_y, .{ .axis = 0 }); // tl.max(x, axis=0) +const absmax_kd = b.maxOpts(abs_y, .{ .axis = 0, .keep_dims = true }); // tl.max(x, 0, keep_dims=True) +const rev_cum = b.cumsumOpts(x, .{ .reverse = true }); // tl.cumsum(x, 0, reverse=True) +``` + +Under the hood these build the required `tt.reduce` / `tt.scan` region with a +matching `arith.addf` / `arith.addi` / `maxnumf` / `maxsi` combine function — +no more hand-rolled combiner closures for the common cases. + +> **Gotcha — `maxnumf` vs `maximumf` (and `minnumf` vs `minimumf`):** +> Triton's `tl.max`, `tl.min`, `tl.maximum`, `tl.minimum` all default to +> the **non-NaN-propagating** form (`arith.maxnumf` / `arith.minnumf`). +> The DSL fluent forms match Python: +> +> | Python | DSL | MLIR op | +> |-------------------------------------|--------------------------------------|-------------------| +> | `tl.max(x)` | `b.max(x)` / `x.max()` | `arith.maxnumf` | +> | `tl.min(x)` | `b.min(x)` / `x.min()` | `arith.minnumf` | +> | `tl.maximum(a, b)` | `a.maximum(b)` / `b.maximum(a, b)` | `arith.maxnumf` | +> | `tl.minimum(a, b)` | `a.minimum(b)` / `b.minimum(a, b)` | `arith.minnumf` | +> | `tl.maximum(a, b, propagate_nan=NAN_ALL)` | `a.maximumOpts(b, .{ .propagate_nan = .all })` | `arith.maximumf` | +> +> The lower-level builders **`b.maximumf(a, b)` / `b.minimumf(a, b)` skip +> Python's defaults** and emit IEEE-754 NaN-propagating ops directly. Reach +> for them only when you genuinely want NaN propagation; otherwise stick +> with `b.maximum` / `b.minimum` (or the fluent `Value.maximum` / +> `Value.minimum`). Mismatched ops here cause subtle TTIR/LLIR diffs vs +> Python and are a frequent porting bug. + +When `.keep_dims = true`, the DSL inserts a trailing +`tt.expand_dims` / `splat` so the result shape matches the input rank with a +size-1 reduced axis (matching Python Triton). For `axis = null` (the default), +the reduction goes through a `tt.reshape allow_reorder` first — even on rank-1 +input — so the layout-assignment pass picks the same reduction-friendly +layout Python's frontend gets. + +For custom combiners, use the low-level `b.reduce(ctx, .{ .src, .axis, .elem, +.result, .combine })` and `b.scan(ctx, .{ .src, .axis, .elem, .result, +.combine, .reverse })` — they take a `fn(*Builder, Value, Value, CtxT) Value` +closure for `combine` and give you full control. + +--- + +## Control flow + +SCF regions are built with **scope-based** builders — `b.openFor(...)`, +`b.openIf(...)` / `b.openIfElse(...)`, `b.openWhile(...)` return a typed scope +value. You emit body ops into a bare `{}` block, terminate with `yield` / +`yieldThen` / `yieldAfter`, and read results from `scope.results` afterward. +No `ctx` struct, no function literal, no `kk`/`k` split — lexical scope +carries captures because everything runs in the same function. + +### `openFor` — scf.for with iter_args + +```zig +var loop = b.openFor(0, N, BLOCK, .{acc0}); +{ + const iv = loop.iv; + const acc = loop.carried[0]; + const offs = iv.add(cols); + const tile = b.loadOpts(x_ptr.addPtr(offs), .{ .mask = offs.lt(N) }); + const partial = tile.sum(); + loop.yield(.{ acc.add(partial) }); +} +const total = loop.results[0]; +``` + +- `lower`/`upper`/`step` are `anytype`: pass Values, comptime ints, or runtime + Zig ints. Comptime literals adapt to the first operand's type; cast explicitly + to force a specific width (e.g. `@as(i32, @intCast(config.block))`). +- `inits` is a tuple literal `.{v1, v2, ...}`. Arity is comptime — `carried`, + `results`, and the `yield(...)` call are all fixed-size arrays of the same + length. +- Empty loops: `b.openFor(0, N, S, .{})` + `loop.yield(.{})` (no iter_args). + +### `openIf` — scf.if without else (side-effects only) + +```zig +var i = b.openIf(cond); +{ + // then body + i.yieldThen(.{}); // closes scope, builds scf.if with empty else +} +``` + +Use for conditional side-effects (no results). There is no `yieldElse` — the +else block is auto-built empty. + +### `openIfElse` — scf.if with else and optional results + +```zig +var i = b.openIfElse(cond, .{ b.scalarTy(.f32) }); // result types +{ + // then body + i.yieldThen(.{ x }); +} +{ + // else body + i.yieldElse(.{ y }); +} +const r = i.results[0]; +``` + +- `result_types` is a tuple of `*const mlir.Type` — use `b.scalarTy(dtype)` or + `b.tensorTy(shape, dtype)`. Pass `.{}` for an if/else with no results but + distinct then/else bodies. +- `yieldThen` swaps to the else block; `yieldElse` builds the `scf.if` op. +- Both branches must yield arity-matching tuples. + +### `openWhile` — scf.while + +```zig +var w = b.openWhile(.{ i0 }, .{ b.scalarTy(.i32) }); +{ + // before region (condition) + const s = w.before_carried[0]; + w.yieldBefore(s.lt(10), .{ s }); // scf.condition: cond + values forwarded to after +} +{ + // after region (body) + const s = w.after_carried[0]; + w.yieldAfter(.{ s.add(1) }); // values forwarded back to before +} +const r = w.results[0]; +``` + +- `inits` — tuple of `Value`s; before-region arg types = their types. +- `after_types` — tuple of `*const mlir.Type`; after-region arg types + the + scf.while's result types. +- `yieldBefore(cond, forwarded)` — `forwarded` arity must match `after_types`. +- `yieldAfter(values)` — arity must match `inits`. + +### `returnIf` — early `tt.return` via cf.cond_br + +```zig +// Python: if pid_m * BLOCK >= num_tokens_post_padded: return +const out_of_range = pid_m.mul(block).ge(num_tokens); +b.returnIf(out_of_range, .{}); +// ... ops here live in a fresh fall-through block ... +``` + +Lowers to: + +```mlir +cf.cond_br %out_of_range, ^ret, ^cont +^ret: + tt.return +^cont: + // subsequent ops +``` + +- `values` is a tuple of `Value`s matching the `tt.func`'s declared result + types (pass `.{}` for a void kernel). +- Works **only at the top-level `tt.func` body** — don't call inside an + `scf.if`/`scf.for`/`scf.while` region (those must stay structured; cf + branches can't escape their parent scf region). +- After `returnIf`, the kernel's current block is swapped to a fresh + fall-through block — any subsequent `b.*` calls emit into that block, and + `b.finish(...)` appends the final `tt.return` there. +- Use this to match Python Triton's `if cond: return` lowering exactly. + Python emits `cf.cond_br` for early-return idioms while using `scf.for` / + `scf.if` for structured loops and value-carrying branches; our DSL does + the same. + +### `openReturnIf` — early `tt.return` with side-effects in the taken branch + +```zig +// Python: if off_experts == -1: write_zeros_to_output(...); return +var scope = b.openReturnIf(is_dead); +{ + writeZerosToOutput(k, c_ptr, ...); // ops here emit into ^ret + scope.yieldReturn(.{}); // closes ^ret with tt.return +} +// ... ops here emit into ^cont (fall-through) ... +``` + +Lowers to the same `cf.cond_br ^ret, ^cont` shape as `returnIf`, but the +taken branch can carry arbitrary side-effects between the `cf.cond_br` and +the `tt.return`. Use this instead of the +`openIf(cond){body}yieldThen; returnIf(cond, values)` pair — that pattern +duplicates the predicate (emits both `scf.if` and `cf.cond_br` on the same +boolean) and drifts from Triton's reference IR. + +Same restrictions as `returnIf`: top-level `tt.func` body only. + +### When to use `cf` (returnIf / openReturnIf) vs `scf` (openIf / openIfElse / openFor) + +| Python source | Emits | DSL builder | +|----------------------------------------|-------|---------------------------------| +| `if cond: return` | `cf` | `b.returnIf(cond, .{})` | +| `if cond: ; return` | `cf` | `var s = b.openReturnIf(cond); {...; s.yieldReturn(.{})}` | +| `if cond: ` | `scf` | `b.openIf(cond) {...; yieldThen(.{})}` | +| `if cond: x=A else: x=B` (value-carrying) | `scf` | `b.openIfElse(cond, .{ty}) {...; yieldThen(.{a})} {...; yieldElse(.{b})}` | +| `for i in range(...)` | `scf` | `b.openFor(lo, hi, step, .{inits})` | +| `while cond:` | `scf` | `b.openWhile(.{inits}, .{after_types})` | + +Rule of thumb: **`scf` for structured, value-carrying control. `cf` for +early-exits.** Both lower to identical PTX after the +`convert-scf-to-cf` pass — the difference only matters at the TTIR layer +(pattern-matching passes that run before lowering). + +### `reduce` / `scan` — custom combiners (still callback-based) + +```zig +b.reduce(ctx, .{ .src, .axis, .elem, .result, .combine = fn (*Builder, Value, Value, Ctx) Value }); +b.scan (ctx, .{ .src, .axis, .reverse = false, .elem, .result, .combine }); +``` + +These stay on the callback pattern because the combine fn is a tiny pure +function (`(a, b) -> a ⊕ b`) where a `ctx` struct is a cleaner fit than a +scoped builder. Reach for `b.sum` / `b.max` / `b.cumsum` / etc. first — the +built-in reductions cover the common cases. + +### General tips + +- Scopes stack naturally: `openFor { openIf { openFor { ... } } }` all just + push/pop the kernel's current-block stack (same for `openIfElse` / + `openWhile`). +- Empty yields are legal: `loop.yield(.{})`, `i.yieldThen(.{})`, etc. +- After any `yield*`, the scope's `.results` field holds the scf op's results + as a `[N]Value`. + +--- + +## Escape hatches + +Everything the DSL doesn't expose is still reachable — `Builder` is a +book-keeping layer, not a walled garden: + +- `v.inner` — the raw `*const mlir.Value`; feed it to any `ttir.*` / + `arith.*` / `math.*` / `scf.*` constructor directly. +- `b.ctx` — the owning `*mlir.Context`. +- `b.currentBlock()` — the active `mlir.Block` (respects `pushBlock` / + `popBlock`). +- `b.emit(op)` / `b.emitMulti(op, n)` — if you've built an `mlir.Operation` + by hand, use these so the result comes back as a `Value` with its `kernel` + field populated (you'll need that for any subsequent fluent calls). + +When you drop to raw ops, remember to still **append them to the current +block** — `b.emit(...)` does that for you; `_ = op.appendTo(b.currentBlock())` +is the manual form. + +--- + +## Lower layer — `Builder.build` + +The declarative `zml.kernel.triton.Kernel(Config, spec)` form covers most +cases. For existing code, dynamic-arity arg lists, or escape-hatch scenarios +where you want to drive the IR construction yourself, the lower-layer +`Builder.build` API is still available: + +```zig +const Builder = tri.Builder; + +pub fn fusedMoeKernel( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + config: GenerationConfig, +) ![:0]const u8 { + var spec = try Builder.build(allocator, ctx, "fused_moe_kernel", .{ + .a_ptr = .{ .ptr = config.a_dtype }, + .b_ptr = .{ .ptr = config.b_dtype }, + // ... + }, &.{}); + defer spec.deinit(); + const b = &spec.kernel; + const a = spec.args; + + // ... emit body via b.* helpers; same DSL as inside `Kernel(...)` ... + + return b.finish(&.{}); +} +``` + +`Builder.build(allocator, ctx, name, spec, result_types)` returns a +heap-allocated `*Built(Spec)`: + +- `spec.kernel` — the runtime `Builder`; pass `&spec.kernel` to helpers. +- `spec.args` — the same `NamedArgs(Spec)` struct that the declarative + `b.declareArgs(...)` returns. +- `spec.deinit()` — frees the module + bundle. +- `b.finish(results)` — terminates with `tt.return`, verifies, serializes + to `[:0]const u8`. Pass `&.{}` for kernels with no return values. + +For dynamic-arg-count construction, `Builder.init(allocator, ctx, name, +args: []const ArgSpec, result_types)` takes a runtime slice of arg specs +instead of a comptime struct literal; you grab block args with `b.arg(i)`. + +You can also use `Builder.open(allocator, ctx, name)` to create an empty +builder and call `b.declareArgs(spec)` mid-stream — that's exactly what +`zml.kernel.triton.Kernel(...)` does internally. + +Prefer `zml.kernel.triton.Kernel(Config, spec)` for the common case. + +--- + +## Python Triton ↔ Zig DSL cheat-sheet + +| Python | Zig | +|-----------------------------------------------|--------------------------------------------------------------------| +| `tl.program_id(0)` | `b.programId(.x)` | +| `tl.num_programs(0)` | `b.numPrograms(.x)` | +| `tl.arange(0, N)` | `b.arange(0, N, .i32)` | +| `x.to(tl.int64)` | `x.to(.i64)` | +| `a + b`, `a * b`, `a // b` | `a.add(b)`, `a.mul(b)`, `a.div(b)` | +| `a < b` | `a.lt(b)` | +| `mask & other` | `masb.bitAnd(other)` | +| `tl.where(c, a, b)` | `b.where(c, a, b)` (alias of `b.select`) | +| `vec[:, None]` | `vec.expandDims(1)` | +| `vec[None, :]` | `vec.expandDims(0)` | +| `m[:, None] & (c[None, :] < N)` | `b.mask2d(m, c.lt(N), m_sz, n_sz)` | +| `ptr + offs` | `ptr.addPtr(offs)` (auto-splats scalar↔tensor either way) | +| `tl.load(ptr)` | `b.load(ptrs)` | +| `tl.load(ptr, mask=m)` | `b.loadOpts(ptrs, .{ .mask = m })` | +| `tl.load(ptr, mask=m, other=o)` | `b.loadOpts(ptrs, .{ .mask = m, .other = o })` | +| `tl.store(ptr, v)` | `b.store(ptr, v)` | +| `tl.store(ptr, v, mask=mask)` | `b.storeOpts(ptr, v, .{ .mask = mask })` | +| `tl.zeros((M, N), dtype=tl.float32)` | `b.zeros(&.{ M, N }, .f32)` | +| `tl.full((M, N), 3, tl.int32)` | `b.full(&.{ M, N }, 3, .i32)` | +| `tl.sum(x)` | `b.sum(x)` — axis=None default | +| `tl.sum(x, axis=0)` | `b.sumOpts(x, .{ .axis = 0 })` | +| `tl.sum(x, axis=0, keep_dims=True)` | `b.sumOpts(x, .{ .axis = 0, .keep_dims = true })` | +| `tl.max(x)` / `tl.min(x)` | `b.max(x)` / `b.min(x)` | +| `tl.max(x, axis=0)` | `b.maxOpts(x, .{ .axis = 0 })` | +| `tl.maximum(a, b)` / `tl.minimum(a, b)` | `a.maximum(b)` / `a.minimum(b)` or `b.maximum(a, b)` | +| `tl.cumsum(x)` / `tl.cumprod(x)` | `b.cumsum(x)` / `b.cumprod(x)` | +| `tl.cumsum(x, axis=0, reverse=True)` | `b.cumsumOpts(x, .{ .reverse = true })` | +| `tl.dot(a, b, acc=acc)` | `b.dot(a, b, acc)` — defaults to `input_precision = tf32` (NVIDIA) | +| `tl.dot(a, b, acc, input_precision="ieee")` | `b.dotOpts(a, b, acc, .{ .input_precision = .ieee })` | +| `tl.clamp(x, mn, mx)` | `b.clampf(x, mn, mx)` | +| `tl.clamp(x, mn, mx, propagate_nan=NAN_ALL)` | `b.clampfOpts(x, mn, mx, .{ .propagate_nan = .all })` | +| `tl.atomic_add(p, v)` | `b.atomicRmw(.add, p, v)` | +| `tl.atomic_add(p, v, mask=m, sem="relaxed")` | `b.atomicRmwOpts(.add, p, v, .{ .mask = m, .sem = .relaxed })` | +| `tl.histogram(x, num_bins)` | `b.histogram(x, num_bins)` | +| `tl.histogram(x, num_bins, mask=m)` | `b.histogramOpts(x, num_bins, .{ .mask = m })` | +| `tl.reshape(x, shape)` | `b.reshape(x, &.{ ...shape })` | +| `tl.reshape(x, shape, can_reorder=True)` | `b.reshapeOpts(x, &.{ ...shape }, .{ .can_reorder = true })` | +| `tl.cast(x, tl.bfloat16)` | `b.cast(x, .bf16)` (or fluent `x.to(.bf16)`) | +| `tl.cast(x, tl.bf16, fp_downcast_rounding="rtne")` | `b.castOpts(x, .bf16, .{ .fp_downcast_rounding = .rtne })` | +| `tl.cast(x, tl.int32, bitcast=True)` | `b.castOpts(x, .i32, .{ .bitcast = true })` | +| `src[indices]` (gather) | `b.gather(src, indices, axis)` | +| `tl.trans(x, dims)` / `tl.permute(x, dims)` | `b.trans(x, order)` / `b.permute(x, order)` | +| `tl.cat(a, b)` | `b.cat(a, b)` (dim=0 default) | +| `tl.cat(a, b, can_reorder=True)` | `b.catOpts(a, b, .{ .can_reorder = true })` | +| `tl.join(a, b)` / `tl.split(x)` | `b.join(a, b)` / `b.split(x)` | +| `tl.item(x)` | `b.item(x)` | +| `tl.abs(x)` | `b.abs(x)` (auto-dispatch) or fluent `x.abs()` | +| `tl.broadcast(a, b)` (symmetric) | `b.broadcast(a, b)` → `.{ Value, Value }` | +| `tl.broadcast_to(v, shape)` | `b.broadcastTo(v, &.{ ...shape })` | +| `tl.umulhi(a, b)` | `b.umulhi(a, b)` | +| `tl.sqrt_rn(x)` / `tl.div_rn(x, y)` | `b.sqrtRn(x)` / `b.divRn(x, y)` | +| `tl.device_print("dbg", args...)` | `b.devicePrint("dbg", args, is_signed)` | +| `tl.device_assert(cond, "msg")` | `b.deviceAssert(cond, "msg")` | +| `tl.inline_asm_elementwise(...)` | `b.inlineAsmElementwise(...)` | +| `tl.extern_elementwise(...)` | `b.externElementwise(srcs, shape, dtype, lib, path, sym)` | +| `for i in range(0, N, S): ...` | `var loop = b.openFor(0, N, S, .{inits...}); { ...; loop.yield(.{...}); }` | +| `while cond: ...` | `var w = b.openWhile(.{inits}, .{after_types}); { ...; w.yieldBefore(cond, .{...}); }; { ...; w.yieldAfter(.{...}); }` | +| `if cond: ...` (side-effects, no else) | `var i = b.openIf(cond); { ...; i.yieldThen(.{}); }` | +| `if cond: ... else: ...` | `var i = b.openIfElse(cond, .{result_types}); { ...; i.yieldThen(.{...}); }; { ...; i.yieldElse(.{...}); }` | +| `if cond: return` (early exit) | `b.returnIf(cond, .{});` — emits `cf.cond_br` + `tt.return`, continues in fall-through block | +| `if cond: ; return` (early exit, with side-effects) | `var s = b.openReturnIf(cond); { ...; s.yieldReturn(.{}); }` — single `cf.cond_br`; body emits into `^ret` before the `tt.return` | + +--- + +## Pitfalls + +1. **Width mismatches between runtime operands.** `v_i32.mul(v_i64)` will hit + `arith.muli requires the same type` at verify time. Either cast one with + `.to(...)` or reshape the kernel to stay within a single int width. Comptime + literals adapt automatically — runtime values don't. + +2. **Tensor-of-pointers via `.addPtr`.** Auto-splats either side: scalar + `!tt.ptr` + tensor offset (`x_ptr.addPtr(tensor_offs)`) and tensor + `tensor<...x!tt.ptr>` + scalar offset (`a_ptrs.addPtr(BLOCK_K)`) + both work without manual `.splatTo`. + +3. **Scope-based control flow — no closures needed.** `openFor` / `openIf` / + `openIfElse` / `openWhile` push a block onto the kernel's insertion stack; + body ops emitted in the bare `{}` that follows go into that blocb. Captures + are plain lexical scope, so you can reference outer Values, sizes, etc. + directly — no `ctx` struct, no function literal. + +4. **`Value.kernel == null` panics fluent calls.** Only happens if you build a + `Value` by hand via `.{ .inner = raw }`. Route through `b.emit(...)` to keep + the back-pointer populated — or just stick with the explicit `Builder.*` + helpers for that one op. + +5. **Don't worry about duplicate constants.** Every lifted literal (from + `.add(42)`, `.lt(0)`, `b.splat(1.0, shape)`, an `openFor(0, N, 1, …)` + bound, etc.) emits a fresh `arith.constant`. This matches Python Triton's + own frontend — the MLIR canonicalizer + CSE pass collapses duplicates on the + way to PTX, so the generated code is identical whether you emit one + constant or a hundred. Optimize for readability at the builder level. + +6. **`finish(results, allocator)` verifies before serializing.** A verifier + failure here almost always means an operand-type mismatch (see #1) or a + missing block terminator — a common one is forgetting that `scf.yield` + operand count must match the loop's result types. + +7. **Yielding from loop / if / while bodies.** Inside a scope (`openFor`, + `openIf`, `openIfElse`, `openWhile`), you *don't* return values — you call + `scope.yield(...)` (or `yieldThen` / `yieldElse` / `yieldBefore` / + `yieldAfter`) with a tuple of Values matching the scope's declared arity. + Scopes with results (`openFor`, `openIfElse`, `openWhile`) store the scf + op's results in `scope.results` as a fixed-size `[N]Value` array. `openIf` + has no results — `yieldThen(.{})` closes and builds scf.if with an empty + else block in one step. diff --git a/kernels/triton/builder.zig b/kernels/triton/builder.zig new file mode 100644 index 00000000..46945679 --- /dev/null +++ b/kernels/triton/builder.zig @@ -0,0 +1,2220 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const dialects = @import("mlir/dialects"); +const dsl = @import("kernels/common"); +const dtypes = @import("dtype.zig"); +const stdx = @import("stdx"); + +const arith = dialects.arith; +const cf = dialects.cf; +const math = dialects.math; +const scf = dialects.scf; +const ttir = dialects.ttir; + +pub const DType = dtypes.DType; + +pub const Value = struct { + inner: *const mlir.Value, + kernel: ?*Builder = null, + + pub const Shape = stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK); + + pub fn type_(self: Value) *const mlir.Type { + return self.inner.type_(); + } + + fn kern(self: Value) *Builder { + return self.kernel orelse @panic("Value has no owning kernel; use Builder.* helpers instead"); + } + + pub fn elemType(self: Value) *const mlir.Type { + if (self.type_().isA(mlir.ShapedType)) |shaped| return shaped.elementType(); + return self.type_(); + } + + pub fn isTensor(self: Value) bool { + return self.type_().isA(mlir.ShapedType) != null; + } + + pub fn rank(self: Value) usize { + if (self.type_().isA(mlir.ShapedType)) |shaped| return shaped.rank(); + return 0; + } + + pub fn dim(self: Value, i: usize) i64 { + return self.type_().isA(mlir.ShapedType).?.dimension(i); + } + + pub fn shape(self: Value) Shape { + var out: Shape = .empty; + if (self.type_().isA(mlir.ShapedType)) |shaped| { + const r = shaped.rank(); + for (0..r) |i| out.appendAssumeCapacity(shaped.dimension(i)); + } + return out; + } + + pub fn isFloatElem(self: Value) bool { + const et = self.elemType(); + inline for (std.meta.fields(mlir.FloatTypes)) |f| { + if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, f.name))) != null) return true; + } + return false; + } + + pub fn isIntElem(self: Value) bool { + return self.elemType().isA(mlir.IntegerType) != null; + } + + pub fn add(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.addf(l, r) else k.addi(l, r); + } + + pub fn sub(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.subf(l, r) else k.subi(l, r); + } + + pub fn mul(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.mulf(l, r) else k.muli(l, r); + } + + pub fn div(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.divf(l, r) else k.divsi(l, r); + } + + pub fn rem(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.remf(l, r) else k.remsi(l, r); + } + + /// Ceil-div: `(x + (div - 1)) // div`. Comptime rhs folds `rhs - 1`. + pub fn cdiv(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return switch (@typeInfo(@TypeOf(rhs))) { + .comptime_int, .comptime_float => k.divsi(l.add(rhs - 1), r), + else => k.divsi(k.addi(l, r.sub(1)), r), + }; + } + + pub fn bitAnd(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return k.andi(l, r); + } + + pub fn bitOr(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return k.ori(l, r); + } + + pub fn minimum(self: Value, rhs: anytype) Value { + return self.kern().minimum(self, rhs); + } + pub fn minimumOpts(self: Value, rhs: anytype, opts: MinMaxOpts) Value { + return self.kern().minimumOpts(self, rhs, opts); + } + + pub fn maximum(self: Value, rhs: anytype) Value { + return self.kern().maximum(self, rhs); + } + pub fn maximumOpts(self: Value, rhs: anytype, opts: MinMaxOpts) Value { + return self.kern().maximumOpts(self, rhs, opts); + } + + pub fn lt(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.cmpf(.olt, l, r) else k.cmpi(.slt, l, r); + } + pub fn le(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.cmpf(.ole, l, r) else k.cmpi(.sle, l, r); + } + pub fn gt(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.cmpf(.ogt, l, r) else k.cmpi(.sgt, l, r); + } + pub fn ge(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.cmpf(.oge, l, r) else k.cmpi(.sge, l, r); + } + pub fn eq(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.cmpf(.oeq, l, r) else k.cmpi(.eq, l, r); + } + pub fn ne(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.cmpf(.one, l, r) else k.cmpi(.ne, l, r); + } + + pub fn to(self: Value, dtype: DType) Value { + return self.kern().cast(self, dtype); + } + pub fn toOpts(self: Value, dtype: DType, opts: CastOpts) Value { + return self.kern().castOpts(self, dtype, opts); + } + + pub fn expandDims(self: Value, axis: i32) Value { + return self.kern().expandDims(self, axis); + } + + pub fn splatTo(self: Value, shape_: []const i64) Value { + return self.kern().splat(self, shape_); + } + + pub fn sum(self: Value) Value { + return self.kern().sum(self); + } + pub fn sumOpts(self: Value, opts: ReduceOpts) Value { + return self.kern().sumOpts(self, opts); + } + + pub fn max(self: Value) Value { + return self.kern().max(self); + } + pub fn maxOpts(self: Value, opts: ReduceOpts) Value { + return self.kern().maxOpts(self, opts); + } + + pub fn min(self: Value) Value { + return self.kern().min(self); + } + pub fn minOpts(self: Value, opts: ReduceOpts) Value { + return self.kern().minOpts(self, opts); + } + + pub fn cumsum(self: Value) Value { + return self.kern().cumsum(self); + } + pub fn cumsumOpts(self: Value, opts: ScanOpts) Value { + return self.kern().cumsumOpts(self, opts); + } + + pub fn cumprod(self: Value) Value { + return self.kern().cumprod(self); + } + pub fn cumprodOpts(self: Value, opts: ScanOpts) Value { + return self.kern().cumprodOpts(self, opts); + } + + pub fn gather(self: Value, indices: Value, axis: i32) Value { + return self.kern().gather(self, indices, axis); + } + + pub fn abs(self: Value) Value { + return self.kern().abs(self); + } + + pub fn addPtr(self: Value, offset: anytype) Value { + const k = self.kern(); + const off: Value = if (@TypeOf(offset) == Value) offset else k.lift(offset); + const ptr = if (!self.isTensor() and off.isTensor()) k.splat(self, off.shape().constSlice()) else self; + const off2 = if (self.isTensor() and !off.isTensor()) k.splat(off, self.shape().constSlice()) else off; + return k.addptr(ptr, off2); + } +}; + +const isFloatDtype = dtypes.isFloatDtype; +const dtypeBitwidth = dtypes.dtypeBitwidth; +const intBitwidth = dtypes.intBitwidth; + +fn computeReducedType(src_shape: []const i64, axis: i32, elem: *const mlir.Type) *const mlir.Type { + if (src_shape.len <= 1) return elem; + var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (src_shape, 0..) |d, i| { + if (@as(i32, @intCast(i)) != axis) out.appendAssumeCapacity(d); + } + return mlir.rankedTensorType(out.constSlice(), elem); +} + +pub const ArgSpec = struct { + name: []const u8, + kind: Kind, + + pub const Kind = union(enum) { + ptr: DType, + scalar: DType, + tensor: struct { []const i64, DType }, + ptr_opts: PtrOpts, + scalar_opts: ScalarOpts, + }; + + pub const PtrOpts = struct { + dtype: DType, + address_space: i32 = 1, + divisibility: ?u32 = 16, + }; + + pub const ScalarOpts = struct { + dtype: DType, + divisibility: ?u32 = null, + }; +}; + +pub const FinishError = error{InvalidMlir} || std.mem.Allocator.Error || std.Io.Writer.Error; + +pub const ClampOpts = struct { + propagate_nan: ttir.PropagateNan = .none, +}; + +pub const HistogramOpts = struct { + mask: ?Value = null, +}; + +pub const PrintOpts = struct { + hex: bool = false, +}; + +/// `axis=null` reduces all dims. +pub const ReduceOpts = struct { + axis: ?i32 = null, + keep_dims: bool = false, +}; + +pub const ScanOpts = struct { + axis: i32 = 0, + reverse: bool = false, +}; + +pub const LoadOpts = struct { + mask: ?Value = null, + other: ?Value = null, + cache_modifier: ttir.CacheModifier = .none, + eviction_policy: ttir.EvictionPolicy = .normal, + @"volatile": bool = false, +}; + +pub const StoreOpts = struct { + mask: ?Value = null, + cache_modifier: ttir.CacheModifier = .none, + eviction_policy: ttir.EvictionPolicy = .normal, +}; + +pub const ReshapeOpts = struct { + can_reorder: bool = false, + efficient_layout: bool = false, +}; + +pub const DotOpts = struct { + input_precision: ttir.InputPrecision = .tf32, + max_num_imprecise_acc: i32 = 0, +}; + +pub const FpToFpOpts = struct { + rounding: ?ttir.RoundingMode = null, +}; + +pub const AtomicRMWOpts = struct { + mask: ?Value = null, + sem: ttir.MemSemantic = .acq_rel, + scope: ttir.MemSyncScope = .gpu, +}; + +pub const AtomicCasOpts = struct { + sem: ttir.MemSemantic = .acq_rel, + scope: ttir.MemSyncScope = .gpu, +}; + +pub const DotScaledOpts = struct { + fast_math: bool = false, + lhs_k_pack: bool = true, + rhs_k_pack: bool = true, +}; + +pub const ExternElementwiseOpts = struct { + is_pure: bool = true, +}; + +pub const DescriptorLoadOpts = struct { + cache_modifier: ttir.CacheModifier = .none, + eviction_policy: ttir.EvictionPolicy = .normal, +}; + +pub const InlineAsmOpts = struct { + is_pure: bool = true, + pack: i32 = 1, +}; + +pub const CatOpts = struct { + can_reorder: bool = false, + dim: i32 = 0, +}; + +pub const CastOpts = struct { + fp_downcast_rounding: ?ttir.RoundingMode = null, + bitcast: bool = false, +}; + +pub const MinMaxOpts = struct { + propagate_nan: ttir.PropagateNan = .none, +}; + +pub const DeviceAssertOpts = struct { + mask: ?Value = null, +}; + +const tupleArity = dsl.tupleArity; + +pub fn ForScope(comptime N: usize) type { + return dsl.ForScope(Builder, Value, N); +} + +pub const ReturnIfScope = struct { + kernel: *Builder, + ret_block: *mlir.Block, + cont_block: *mlir.Block, + + pub fn yieldReturn(self: *ReturnIfScope, values: anytype) void { + const k = self.kernel; + const fields = @typeInfo(@TypeOf(values)).@"struct".fields; + + if (k.exit_block == null) { + k.exit_block = mlir.Block.init(&.{}, &.{}); + + var ret_operands: [fields.len]*const mlir.Value = undefined; + inline for (fields, 0..) |f, i| { + if (f.type != Value) + @compileError("ReturnIfScope.yieldReturn: every value must be a Value"); + ret_operands[i] = @field(values, f.name).inner; + } + _ = ttir.return_(k.ctx, &ret_operands, k.loc()).appendTo(k.exit_block.?); + } + + _ = cf.br(k.ctx, k.exit_block.?, &.{}, k.loc()).appendTo(self.ret_block); + k.popBlock(); + + if (k.block_stack.items.len == 0) { + k.pushBlock(self.cont_block); + } else { + k.block_stack.items[k.block_stack.items.len - 1] = self.cont_block; + } + } +}; + +pub const IfOnlyScope = dsl.IfOnlyScope(Builder, Value); + +pub fn IfScope(comptime N: usize) type { + return dsl.IfScope(Builder, Value, N); +} + +pub fn WhileScope(comptime N: usize, comptime M: usize) type { + return dsl.WhileScope(Builder, Value, N, M); +} + +pub fn ReduceArgs(comptime CtxT: type) type { + return struct { + src: Value, + axis: i32, + elem: *const mlir.Type, + result: *const mlir.Type, + combine: *const fn (*Builder, Value, Value, CtxT) Value, + }; +} + +pub fn ScanArgs(comptime CtxT: type) type { + return struct { + src: Value, + axis: i32, + reverse: bool = false, + elem: *const mlir.Type, + result: *const mlir.Type, + combine: *const fn (*Builder, Value, Value, CtxT) Value, + }; +} + +pub const Builder = struct { + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + ctx: *mlir.Context, + module: *mlir.Module, + name: []const u8, + func_op: ?*mlir.Operation, + entry_block: ?*mlir.Block, + exit_block: ?*mlir.Block = null, + block_stack: std.ArrayList(*mlir.Block), + + /// `noinline` is a Zig keyword — write `.{ .@"noinline" = true }`. + pub const Opts = struct { + @"noinline": bool = false, + }; + + pub fn init( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + args: []const ArgSpec, + result_types: []const *const mlir.Type, + ) !Builder { + return initOpts(allocator, ctx, name, args, result_types, .{}); + } + + pub fn initOpts( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + args: []const ArgSpec, + result_types: []const *const mlir.Type, + opts: Opts, + ) !Builder { + var b = try open(allocator, ctx, name); + errdefer b.deinit(); + try b.declareArgsLowOpts(args, result_types, opts); + return b; + } + + pub fn open( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + ) !Builder { + const unknown_loc: *const mlir.Location = .unknown(ctx); + const module: *mlir.Module = .init(unknown_loc); + errdefer module.deinit(); + + var arena: std.heap.ArenaAllocator = .init(allocator); + errdefer arena.deinit(); + + return .{ + .allocator = allocator, + .arena = arena, + .ctx = ctx, + .module = module, + .name = name, + .func_op = null, + .entry_block = null, + .block_stack = .empty, + }; + } + + pub fn declareArgs(self: *Builder, spec: anytype) !dsl.NamedArgs(@TypeOf(spec), Value) { + return self.declareArgsOpts(spec, &.{}, .{}); + } + + pub fn declareArgsOpts( + self: *Builder, + spec: anytype, + result_types: []const *const mlir.Type, + opts: Opts, + ) !dsl.NamedArgs(@TypeOf(spec), Value) { + const Spec = @TypeOf(spec); + const fields = @typeInfo(Spec).@"struct".fields; + + var arg_specs: [fields.len]ArgSpec = undefined; + inline for (fields, 0..) |f, i| { + const raw = @field(spec, f.name); + const kind: ArgSpec.Kind = if (@TypeOf(raw) == ArgSpec.Kind) raw else blk: { + const variant = @typeInfo(@TypeOf(raw)).@"struct".fields[0].name; + const tag = @field(std.meta.Tag(ArgSpec.Kind), variant); + const inner = @field(raw, variant); + break :blk switch (tag) { + .ptr => .{ .ptr = inner }, + .scalar => .{ .scalar = inner }, + .tensor => .{ .tensor = inner }, + .ptr_opts => .{ .ptr_opts = .{ + .dtype = inner.dtype, + .address_space = if (@hasField(@TypeOf(inner), "address_space")) inner.address_space else 1, + .divisibility = if (@hasField(@TypeOf(inner), "divisibility")) inner.divisibility else 16, + } }, + .scalar_opts => .{ .scalar_opts = .{ + .dtype = inner.dtype, + .divisibility = if (@hasField(@TypeOf(inner), "divisibility")) inner.divisibility else null, + } }, + }; + }; + arg_specs[i] = .{ .name = f.name, .kind = kind }; + } + + try self.declareArgsLowOpts(&arg_specs, result_types, opts); + + var named: dsl.NamedArgs(Spec, Value) = undefined; + inline for (fields, 0..) |f, i| { + @field(named, f.name) = self.arg(i); + } + return named; + } + + fn declareArgsLowOpts( + self: *Builder, + args: []const ArgSpec, + result_types: []const *const mlir.Type, + opts: Opts, + ) !void { + std.debug.assert(self.entry_block == null); + const ctx = self.ctx; + const unknown_loc: *const mlir.Location = .unknown(ctx); + const scratch = self.arena.allocator(); + + const arg_types = try scratch.alloc(*const mlir.Type, args.len); + const arg_locs = try scratch.alloc(*const mlir.Location, args.len); + for (args, 0..) |a, i| { + arg_types[i] = switch (a.kind) { + .ptr => |dt| ttir.pointerType(dt.toMlir(ctx), 1), + .ptr_opts => |p| ttir.pointerType(p.dtype.toMlir(ctx), p.address_space), + .scalar => |dt| dt.toMlir(ctx), + .scalar_opts => |s| s.dtype.toMlir(ctx), + .tensor => |t| mlir.rankedTensorType(t[0], t[1].toMlir(ctx)), + }; + arg_locs[i] = unknown_loc.named(ctx, a.name); + } + + const entry = mlir.Block.init(arg_types, arg_locs); + + const arg_attrs = try scratch.alloc(*const mlir.Attribute, args.len); + var any_arg_attr = false; + for (args, 0..) |a, i| { + const empty_dict: *const mlir.Attribute = mlir.dictionaryAttribute(ctx, &.{}); + switch (a.kind) { + .ptr => { + const div_attr = mlir.dictionaryAttribute(ctx, &.{ + .named(ctx, "tt.divisibility", mlir.integerAttribute(ctx, .i32, 16)), + }); + arg_attrs[i] = div_attr; + any_arg_attr = true; + }, + .ptr_opts => |p| { + if (p.divisibility) |v| { + const div_attr = mlir.dictionaryAttribute(ctx, &.{ + .named(ctx, "tt.divisibility", mlir.integerAttribute(ctx, .i32, v)), + }); + arg_attrs[i] = div_attr; + any_arg_attr = true; + } else { + arg_attrs[i] = empty_dict; + } + }, + .scalar_opts => |s| { + if (s.divisibility) |v| { + const div_attr = mlir.dictionaryAttribute(ctx, &.{ + .named(ctx, "tt.divisibility", mlir.integerAttribute(ctx, .i32, v)), + }); + arg_attrs[i] = div_attr; + any_arg_attr = true; + } else { + arg_attrs[i] = empty_dict; + } + }, + else => arg_attrs[i] = empty_dict, + } + } + + const func_op = ttir.func(ctx, .{ + .name = self.name, + .args = arg_types, + .args_attributes = if (any_arg_attr) arg_attrs else null, + .results = result_types, + .block = entry, + .location = unknown_loc, + .no_inline = opts.@"noinline", + }); + _ = func_op.appendTo(self.module.body()); + + self.func_op = func_op; + self.entry_block = entry; + } + + pub fn deinit(self: *Builder) void { + self.module.deinit(); + self.arena.deinit(); + } + + pub fn arg(self: *Builder, i: usize) Value { + const eb = self.entry_block orelse @panic("Builder.arg called before declareArgs"); + return .{ .inner = eb.argument(i), .kernel = self }; + } + + pub fn pushBlock(self: *Builder, b: *mlir.Block) void { + self.block_stack.append(self.arena.allocator(), b) catch @panic("Builder.pushBlock OOM"); + } + + pub fn popBlock(self: *Builder) void { + _ = self.block_stack.pop(); + } + + pub fn currentBlock(self: *Builder) *mlir.Block { + if (self.block_stack.items.len > 0) { + return self.block_stack.items[self.block_stack.items.len - 1]; + } + return self.entry_block orelse @panic("Builder has no current block — call declareArgs first"); + } + + fn innerSlice(self: *Builder, values: []const Value) []const *const mlir.Value { + const out = self.arena.allocator().alloc(*const mlir.Value, values.len) catch @panic("Builder.innerSlice OOM"); + for (values, 0..) |v, i| out[i] = v.inner; + return out; + } + + pub fn emit(self: *Builder, op: *mlir.Operation) Value { + _ = op.appendTo(self.currentBlock()); + return .{ .inner = op.result(0), .kernel = self }; + } + + pub fn emitMulti(self: *Builder, op: *mlir.Operation, n: usize) []Value { + _ = op.appendTo(self.currentBlock()); + const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.emitMulti OOM"); + for (0..n) |i| out[i] = .{ .inner = op.result(i), .kernel = self }; + return out; + } + + pub fn yield(self: *Builder, values: anytype) []const Value { + const T = @TypeOf(values); + const info = @typeInfo(T); + if (info != .@"struct" or !info.@"struct".is_tuple) + @compileError("Builder.yield expects a tuple literal like `.{ v1, v2 }`"); + const n = info.@"struct".fields.len; + const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.yield OOM"); + inline for (info.@"struct".fields, 0..) |f, i| { + if (f.type != Value) + @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(f.type)); + out[i] = @field(values, f.name); + } + return out; + } + + pub fn loc(self: *const Builder) *const mlir.Location { + return .unknown(self.ctx); + } + + pub fn scalarTy(self: *const Builder, dtype: DType) *const mlir.Type { + return dtype.toMlir(self.ctx); + } + + pub fn tensorTy(self: *const Builder, shape: []const i64, dtype: DType) *const mlir.Type { + return mlir.rankedTensorType(shape, dtype.toMlir(self.ctx)); + } + + fn loadResultType(self: *const Builder, ptr: Value) *const mlir.Type { + _ = self; + const pt = ptr.type_(); + if (pt.isA(mlir.ShapedType)) |shaped| { + const elem_ptr_ty = shaped.elementType(); + const pointee = elem_ptr_ty.isA(ttir.PointerType).?.pointee(); + var shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..shaped.rank()) |i| shape.appendAssumeCapacity(shaped.dimension(i)); + return mlir.rankedTensorType(shape.constSlice(), pointee); + } + return pt.isA(ttir.PointerType).?.pointee(); + } + + fn swapElem(self: *const Builder, src: Value, out_dtype: DType) *const mlir.Type { + const out_elem = out_dtype.toMlir(self.ctx); + if (src.isTensor()) return mlir.rankedTensorType(src.shape().constSlice(), out_elem); + return out_elem; + } + + // ==================== SPMD ==================== + + pub fn programId(self: *Builder, dim: ttir.ProgramIDDim) Value { + return self.emit(ttir.get_program_id(self.ctx, dim, self.loc())); + } + + pub fn numPrograms(self: *Builder, dim: ttir.ProgramIDDim) Value { + return self.emit(ttir.get_num_programs(self.ctx, dim, self.loc())); + } + + // ==================== constants ==================== + + fn emitInt(self: *Builder, dtype: DType, value: i64) Value { + return self.emit(arith.constant_int(self.ctx, value, dtype.toMlir(self.ctx), self.loc())); + } + + fn emitFloat(self: *Builder, dtype: DType, value: f64) Value { + return switch (dtype) { + .f32 => self.emit(arith.constant_float(self.ctx, value, .f32, self.loc())), + .f64 => self.emit(arith.constant_float(self.ctx, value, .f64, self.loc())), + else => self.fpToFp(self.emitFloat(.f32, value), dtype), + }; + } + + // ==================== ranges and shape manipulation ==================== + + /// `[start, end)` as `tensor<(end-start)xi32>`. Both must fit in i32. + pub fn makeRange(self: *Builder, start: anytype, end: anytype) Value { + const s: i32 = @intCast(start); + const e: i32 = @intCast(end); + std.debug.assert(e >= s); + const len: i64 = @intCast(e - s); + const ty = mlir.rankedTensorType(&.{len}, mlir.integerType(self.ctx, .i32)); + return self.emit(ttir.make_range(self.ctx, s, e, ty, self.loc())); + } + + pub fn arange(self: *Builder, start: anytype, end: anytype, dtype: DType) Value { + const r = self.makeRange(start, end); + return switch (dtype) { + .i32 => r, + .i64 => self.extsi(r, dtype), + .i16, .i8 => self.trunci(r, dtype), + .f16, .bf16, .f32, .f64, .f8e4m3fn, .f8e5m2 => self.sitofp(r, dtype), + else => @panic("Builder.arange: unsupported dtype"), + }; + } + + /// Broadcast a scalar Value or literal across `shape`. Literals lift first via `lift`. + pub fn splat(self: *Builder, value: anytype, shape: []const i64) Value { + const v: Value = if (@TypeOf(value) == Value) value else self.lift(value); + const ty = mlir.rankedTensorType(shape, v.type_()); + return self.emit(ttir.splat(self.ctx, v.inner, ty, self.loc())); + } + + /// Lift a Zig scalar to a DSL constant. Dtype follows the source type; + /// unsigned ints are bit-cast (signless MLIR). Use `liftAs` for explicit dtype. + pub fn lift(self: *Builder, value: anytype) Value { + const T = @TypeOf(value); + if (T == Value) return value; + return switch (@typeInfo(T)) { + .comptime_int => blk: { + if (value >= std.math.minInt(i32) and value <= std.math.maxInt(i32)) + break :blk self.emitInt(.i32, value); + if (value >= std.math.minInt(i64) and value <= std.math.maxInt(i64)) + break :blk self.emitInt(.i64, value); + if (value >= 0 and value <= std.math.maxInt(u64)) + break :blk self.emitInt(.i64, @bitCast(@as(u64, value))); + @compileError("Builder.lift: integer literal out of 64-bit range"); + }, + .int => |info| blk: { + const signed = info.signedness == .signed; + if (info.bits > 32) { + const v: i64 = if (signed) @intCast(value) else @bitCast(@as(u64, @intCast(value))); + break :blk self.emitInt(.i64, v); + } + const v32: i32 = if (signed) @intCast(value) else @bitCast(@as(u32, @intCast(value))); + break :blk self.emitInt(.i32, v32); + }, + .comptime_float => self.emitFloat(.f32, @floatCast(value)), + .float => |info| switch (info.bits) { + 16 => self.emitFloat(.f16, @floatCast(value)), + 32 => self.emitFloat(.f32, @floatCast(value)), + 64 => self.emitFloat(.f64, value), + else => @compileError("Builder.lift: unsupported float bitwidth"), + }, + else => @compileError("Builder.lift: unsupported type " ++ @typeName(T)), + }; + } + + /// `lift` but comptime scalars match `ref_elem`. Runtime Zig scalars keep their source width. + fn liftMatching(self: *Builder, value: anytype, ref_elem: *const mlir.Type) Value { + const T = @TypeOf(value); + if (T == Value) return value; + return switch (@typeInfo(T)) { + .comptime_int, .comptime_float => self.constMatching(value, ref_elem), + else => self.lift(value), + }; + } + + /// Lift `value` and splat to `ref.shape` when `ref` is a tensor. + pub fn broadcastLike(self: *Builder, value: anytype, ref: Value) Value { + const v = self.liftMatching(value, ref.elemType()); + if (ref.isTensor() and !v.isTensor()) return self.splat(v, ref.shape().constSlice()); + return v; + } + + /// Binary-op coercion: lift, int-width promote, scalar↔tensor splat, rank-align, size-1 broadcast. + pub fn broadcast(self: *Builder, a: anytype, b: anytype) struct { Value, Value } { + const a_ref: ?*const mlir.Type = if (@TypeOf(b) == Value) b.elemType() else null; + const b_ref: ?*const mlir.Type = if (@TypeOf(a) == Value) a.elemType() else null; + var av = if (a_ref) |t| self.liftMatching(a, t) else self.lift(a); + var bv = if (b_ref) |t| self.liftMatching(b, t) else self.lift(b); + + if (av.isIntElem() and bv.isIntElem()) { + const a_dt = self.mlirElemToDType(av.elemType()); + const b_dt = self.mlirElemToDType(bv.elemType()); + const aw = intBitwidth(a_dt); + const bw = intBitwidth(b_dt); + if (aw < bw) av = av.to(b_dt); + if (bw < aw) bv = bv.to(a_dt); + } + + if (av.isTensor() and !bv.isTensor()) bv = self.splat(bv, av.shape().constSlice()); + if (!av.isTensor() and bv.isTensor()) av = self.splat(av, bv.shape().constSlice()); + + if (av.isTensor() and bv.isTensor()) { + const a_rank = av.rank(); + const b_rank = bv.rank(); + if (a_rank < b_rank) { + var n = b_rank - a_rank; + while (n > 0) : (n -= 1) av = self.expandDims(av, 0); + } else if (b_rank < a_rank) { + var n = a_rank - b_rank; + while (n > 0) : (n -= 1) bv = self.expandDims(bv, 0); + } + const a_sh = av.shape(); + const b_sh = bv.shape(); + var ret_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + var grow_a = false; + var grow_b = false; + for (0..a_sh.len) |i| { + const l = a_sh.get(i); + const r = b_sh.get(i); + if (l == r) { + ret_shape.appendAssumeCapacity(l); + } else if (l == 1) { + ret_shape.appendAssumeCapacity(r); + grow_a = true; + } else if (r == 1) { + ret_shape.appendAssumeCapacity(l); + grow_b = true; + } else { + std.debug.panic( + "Builder.broadcast: incompatible shapes {any} vs {any} at axis {d}", + .{ a_sh.constSlice(), b_sh.constSlice(), i }, + ); + } + } + if (grow_a) av = self.broadcastTo(av, ret_shape.constSlice()); + if (grow_b) bv = self.broadcastTo(bv, ret_shape.constSlice()); + } + + return .{ av, bv }; + } + + fn mlirElemToDType(self: *const Builder, elem: *const mlir.Type) DType { + inline for (std.meta.fields(DType)) |f| { + const dt = @field(DType, f.name); + if (elem.eql(dt.toMlir(self.ctx))) return dt; + } + @panic("element type not a recognized DSL DType"); + } + + /// Lift a Zig scalar to a DSL constant of `dtype`. Unsigned ints are bit-cast. + pub fn liftAs(self: *Builder, value: anytype, dtype: DType) Value { + const T = @TypeOf(value); + if (T == Value) return value; + if (isFloatDtype(dtype)) { + const v_f64: f64 = switch (@typeInfo(T)) { + .comptime_int, .int => @floatFromInt(value), + .comptime_float, .float => @floatCast(value), + else => @compileError("Builder.liftAs: unsupported scalar " ++ @typeName(T)), + }; + return self.emitFloat(dtype, v_f64); + } + const v_i64: i64 = switch (@typeInfo(T)) { + .comptime_int => blk: { + if (value >= std.math.minInt(i64) and value <= std.math.maxInt(i64)) + break :blk @intCast(value); + if (value >= 0 and value <= std.math.maxInt(u64)) + break :blk @bitCast(@as(u64, value)); + @compileError("Builder.liftAs: integer literal out of 64-bit range"); + }, + .int => |info| if (info.signedness == .signed) @intCast(value) else @bitCast(@as(u64, @intCast(value))), + .comptime_float, .float => @intFromFloat(value), + else => @compileError("Builder.liftAs: unsupported scalar " ++ @typeName(T)), + }; + return self.emitInt(dtype, v_i64); + } + + fn constMatching(self: *Builder, value: anytype, elem: *const mlir.Type) Value { + return self.liftAs(value, self.mlirElemToDType(elem)); + } + + pub fn zeros(self: *Builder, shape: []const i64, dtype: DType) Value { + return self.full(shape, 0, dtype); + } + + pub fn ones(self: *Builder, shape: []const i64, dtype: DType) Value { + return self.full(shape, 1, dtype); + } + + pub fn full(self: *Builder, shape: []const i64, value: anytype, dtype: DType) Value { + return self.splat(self.liftAs(value, dtype), shape); + } + + /// 2-D mask `(m, 1) & (1, n)` — auto-broadcasts to `(m, n)`. + pub fn mask2d(self: *Builder, cond_m: Value, cond_n: Value, _: i64, _: i64) Value { + const cm2 = self.expandDims(cond_m, 1); + const cn2 = self.expandDims(cond_n, 0); + return cm2.bitAnd(cn2); + } + + pub fn sum(self: *Builder, src: Value) Value { + return self.sumOpts(src, .{}); + } + + pub fn sumOpts(self: *Builder, src: Value, opts: ReduceOpts) Value { + return self.reduceDispatch(src, opts, .sum); + } + + pub fn max(self: *Builder, src: Value) Value { + return self.maxOpts(src, .{}); + } + + pub fn maxOpts(self: *Builder, src: Value, opts: ReduceOpts) Value { + return self.reduceDispatch(src, opts, .max); + } + + pub fn min(self: *Builder, src: Value) Value { + return self.minOpts(src, .{}); + } + + pub fn minOpts(self: *Builder, src: Value, opts: ReduceOpts) Value { + return self.reduceDispatch(src, opts, .min); + } + + fn reduceDispatch(self: *Builder, src: Value, opts: ReduceOpts, comptime kind: enum { sum, max, min }) Value { + const src_shape = src.shape(); + const input: Value, const axis: i32 = if (opts.axis) |a| .{ src, a } else blk: { + var total: i64 = 1; + for (0..src_shape.len) |i| total *= src_shape.get(i); + break :blk .{ self.reshapeOpts(src, &.{total}, .{ .can_reorder = true }), 0 }; + }; + + const elem = input.elemType(); + const in_shape = input.shape(); + const result_ty: *const mlir.Type = computeReducedType(in_shape.constSlice(), axis, elem); + const is_float = input.isFloatElem(); + const Ctx = struct { is_float: bool, kind: @TypeOf(kind) }; + const combine = struct { + fn c(kk: *Builder, lhs: Value, rhs: Value, cc: Ctx) Value { + return switch (cc.kind) { + .sum => if (cc.is_float) kk.addf(lhs, rhs) else kk.addi(lhs, rhs), + .max => if (cc.is_float) kk.maxnumf(lhs, rhs) else kk.maxsi(lhs, rhs), + .min => if (cc.is_float) kk.minnumf(lhs, rhs) else kk.minsi(lhs, rhs), + }; + } + }.c; + const out = self.reduce(Ctx{ .is_float = is_float, .kind = kind }, .{ + .src = input, + .axis = axis, + .elem = elem, + .result = result_ty, + .combine = combine, + }); + + if (!opts.keep_dims) return out; + if (opts.axis == null) { + var ones_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..src_shape.len) |_| ones_shape.appendAssumeCapacity(1); + return self.splat(out, ones_shape.constSlice()); + } + if (in_shape.len <= 1) return self.splat(out, &.{1}); + return self.expandDims(out, axis); + } + + pub fn cumsum(self: *Builder, src: Value) Value { + return self.cumsumOpts(src, .{}); + } + + pub fn cumsumOpts(self: *Builder, src: Value, opts: ScanOpts) Value { + return self.scanDispatch(src, opts, .sum); + } + + pub fn cumprod(self: *Builder, src: Value) Value { + return self.cumprodOpts(src, .{}); + } + + pub fn cumprodOpts(self: *Builder, src: Value, opts: ScanOpts) Value { + return self.scanDispatch(src, opts, .prod); + } + + fn scanDispatch(self: *Builder, src: Value, opts: ScanOpts, comptime kind: enum { sum, prod }) Value { + const elem = src.elemType(); + const is_float = src.isFloatElem(); + const Ctx = struct { is_float: bool, kind: @TypeOf(kind) }; + const combine = struct { + fn c(kk: *Builder, lhs: Value, rhs: Value, cc: Ctx) Value { + return switch (cc.kind) { + .sum => if (cc.is_float) kk.addf(lhs, rhs) else kk.addi(lhs, rhs), + .prod => if (cc.is_float) kk.mulf(lhs, rhs) else kk.muli(lhs, rhs), + }; + } + }.c; + return self.scan(Ctx{ .is_float = is_float, .kind = kind }, .{ + .src = src, + .axis = opts.axis, + .reverse = opts.reverse, + .elem = elem, + .result = src.type_(), + .combine = combine, + }); + } + + pub fn expandDims(self: *Builder, value: Value, axis: i32) Value { + const in_shape = value.shape(); + var out_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + const ax: usize = @intCast(axis); + for (0..in_shape.len + 1) |i| { + if (i == ax) out_shape.appendAssumeCapacity(1) else out_shape.appendAssumeCapacity(in_shape.get(if (i < ax) i else i - 1)); + } + const ty = mlir.rankedTensorType(out_shape.constSlice(), value.elemType()); + return self.emit(ttir.expand_dims(self.ctx, value.inner, axis, ty, self.loc())); + } + + pub fn broadcastTo(self: *Builder, value: Value, result_shape: []const i64) Value { + const elem = mlir.RankedTensorType.fromShaped(value.inner.type_().isA(mlir.ShapedType).?).elementType(); + const ty = mlir.rankedTensorType(result_shape, elem); + return self.emit(ttir.broadcast(self.ctx, value.inner, ty, self.loc())); + } + + // ==================== pointer arithmetic ==================== + + pub fn addptr(self: *Builder, ptr: Value, offset: Value) Value { + return self.emit(ttir.addptr(self.ctx, ptr.inner, offset.inner, self.loc())); + } + + pub fn load(self: *Builder, ptr: Value) Value { + return self.loadOpts(ptr, .{}); + } + + pub fn loadOpts(self: *Builder, ptr: Value, opts: LoadOpts) Value { + var mask_v = opts.mask; + var other_v = opts.other; + if (ptr.isTensor()) { + if (mask_v) |m| { + _, mask_v = self.broadcast(ptr, m); + } + if (other_v) |o| { + _, other_v = self.broadcast(ptr, o); + } + } + const mask_inner: ?*const mlir.Value = if (mask_v) |m| m.inner else null; + const other_inner: ?*const mlir.Value = if (other_v) |o| o.inner else null; + return self.emit(ttir.load(self.ctx, ptr.inner, self.loadResultType(ptr), mask_inner, other_inner, opts.cache_modifier, opts.eviction_policy, opts.@"volatile", self.loc())); + } + + pub fn store(self: *Builder, ptr: Value, value: Value) void { + self.storeOpts(ptr, value, .{}); + } + + pub fn storeOpts(self: *Builder, ptr: Value, value: Value, opts: StoreOpts) void { + var mask_v = opts.mask; + if (ptr.isTensor()) { + if (mask_v) |m| { + _, mask_v = self.broadcast(ptr, m); + } + } + const mask_inner: ?*const mlir.Value = if (mask_v) |m| m.inner else null; + _ = ttir.store(self.ctx, ptr.inner, value.inner, mask_inner, opts.cache_modifier, opts.eviction_policy, self.loc()).appendTo(self.currentBlock()); + } + + // ==================== arith: binary ==================== + + pub fn addi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.addi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn subi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.subi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn muli(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.muli(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn addf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.addf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn subf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.subf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn mulf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.mulf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn divf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.divf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + + pub fn cmpi(self: *Builder, predicate: arith.CmpIPredicate, lhs: Value, rhs: Value) Value { + return self.emit(arith.cmpi(self.ctx, predicate, lhs.inner, rhs.inner, self.loc())); + } + pub fn cmpf(self: *Builder, predicate: arith.CmpFPredicate, lhs: Value, rhs: Value) Value { + return self.emit(arith.cmpf(self.ctx, predicate, lhs.inner, rhs.inner, self.loc())); + } + + pub fn select(self: *Builder, cond: Value, t: Value, f: Value) Value { + return self.emit(arith.select(self.ctx, cond.inner, t.inner, f.inner, self.loc())); + } + + /// Select `x` where condition is true, `y` elsewhere. Auto-broadcasts. + pub fn where(self: *Builder, condition: Value, x: Value, y: Value) Value { + var c = condition; + var xv, var yv = .{ x, y }; + xv, yv = self.broadcast(xv, yv); + if (c.isTensor()) { + c, xv = self.broadcast(c, xv); + xv, yv = self.broadcast(xv, yv); + } else { + c, _ = self.broadcast(c, xv); + } + return self.select(c, xv, yv); + } + + pub fn divsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.divsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn divui(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.divui(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn remsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.remsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn remui(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.remui(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn ceildivsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.ceildivsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn floordivsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.floordivsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + + pub fn andi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.andi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn ori(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.ori(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn xori(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.xori(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn shli(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.shli(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn shrsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.shrsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn shrui(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.shrui(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + + pub fn maxsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.maxsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn minsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.minsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + + pub fn remf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.remf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn maximumf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.maximumf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn minimumf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.minimumf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn maxnumf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.maxnumf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn minnumf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.minnumf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn negf(self: *Builder, src: Value) Value { + return self.emit(arith.negf(self.ctx, src.inner, self.loc())); + } + + pub fn maximum(self: *Builder, a: anytype, b: anytype) Value { + return self.maximumOpts(a, b, .{}); + } + + pub fn maximumOpts(self: *Builder, a: anytype, b: anytype, opts: MinMaxOpts) Value { + const l, const r = self.broadcast(a, b); + if (l.isFloatElem()) { + return switch (opts.propagate_nan) { + .none => self.maxnumf(l, r), + .all => self.maximumf(l, r), + }; + } + return self.maxsi(l, r); + } + + pub fn minimum(self: *Builder, a: anytype, b: anytype) Value { + return self.minimumOpts(a, b, .{}); + } + + pub fn minimumOpts(self: *Builder, a: anytype, b: anytype, opts: MinMaxOpts) Value { + const l, const r = self.broadcast(a, b); + if (l.isFloatElem()) { + return switch (opts.propagate_nan) { + .none => self.minnumf(l, r), + .all => self.minimumf(l, r), + }; + } + return self.minsi(l, r); + } + + // Casts — each preserves the src shape and swaps the element type to `out_dtype`. + pub fn extsi(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.extsi(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn extui(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.extui(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn extf(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.extf(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn trunci(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.trunci(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn truncf(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.truncf(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn sitofp(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.sitofp(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn uitofp(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.uitofp(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn fptosi(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.fptosi(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn fptoui(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.fptoui(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn arithBitcast(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.bitcast(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + + // ==================== Triton-specific ops ==================== + + /// tt.bitcast — same-bitwidth cast. Shape preserved from src. + pub fn bitcast(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(ttir.bitcast(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + + /// Numeric cast with auto-dispatch. Use `castOpts` for rounding / bitcast. + pub fn cast(self: *Builder, src: Value, dtype: DType) Value { + return self.castOpts(src, dtype, .{}); + } + + /// Numeric cast with options. Dispatch: + /// - fp↔fp narrow: truncf, except fp8 or custom rounding → tt.fp_to_fp. + /// - fp↔fp wide: extf. i1→fp: uitofp. int↔int: extsi/trunci/bitcast by width. + pub fn castOpts(self: *Builder, src: Value, dtype: DType, opts: CastOpts) Value { + if (opts.bitcast) return self.bitcast(src, dtype); + const cur_elem = src.elemType(); + const tgt_elem = dtype.toMlir(self.ctx); + if (cur_elem.eql(tgt_elem)) return src; + const cur_dtype = self.mlirElemToDType(cur_elem); + const cur_is_float = src.isFloatElem(); + const tgt_is_float = isFloatDtype(dtype); + if (cur_is_float and tgt_is_float) { + const cur_bw = dtypeBitwidth(cur_dtype); + const tgt_bw = dtypeBitwidth(dtype); + if (tgt_bw > cur_bw) return self.extf(src, dtype); + const fp8_involved = (cur_dtype == .f8e4m3fn or cur_dtype == .f8e5m2 or + dtype == .f8e4m3fn or dtype == .f8e5m2); + const custom_rounding = opts.fp_downcast_rounding != null and opts.fp_downcast_rounding.? != .rtne; + if (fp8_involved or custom_rounding) { + return self.fpToFpOpts(src, dtype, .{ .rounding = opts.fp_downcast_rounding orelse .rtne }); + } + return self.truncf(src, dtype); + } + if (cur_is_float and !tgt_is_float) return self.fptosi(src, dtype); + if (!cur_is_float and tgt_is_float) { + if (cur_dtype == .i1) return self.uitofp(src, dtype); + return self.sitofp(src, dtype); + } + const cur_bw = dtypeBitwidth(cur_dtype); + const tgt_bw = dtypeBitwidth(dtype); + if (tgt_bw > cur_bw) return self.extsi(src, dtype); + if (tgt_bw < cur_bw) return self.trunci(src, dtype); + return self.arithBitcast(src, dtype); + } + /// tt.int_to_ptr — cast int (scalar or tensor) to pointer, preserving shape. + pub fn intToPtr(self: *Builder, src: Value, pointee: DType, address_space: i32) Value { + const ptr_elem = ttir.pointerType(pointee.toMlir(self.ctx), address_space); + const result_type: *const mlir.Type = if (src.isTensor()) + mlir.rankedTensorType(src.shape().constSlice(), ptr_elem) + else + ptr_elem; + return self.emit(ttir.int_to_ptr(self.ctx, src.inner, result_type, self.loc())); + } + /// tt.ptr_to_int — cast pointer (scalar or tensor) to integer, preserving shape. + pub fn ptrToInt(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(ttir.ptr_to_int(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + /// tt.fp_to_fp with default rtne rounding. Use `fpToFpOpts` for other rounding modes. + pub fn fpToFp(self: *Builder, src: Value, out_dtype: DType) Value { + return self.fpToFpOpts(src, out_dtype, .{ .rounding = .rtne }); + } + /// tt.fp_to_fp with rounding option. + pub fn fpToFpOpts(self: *Builder, src: Value, out_dtype: DType, opts: FpToFpOpts) Value { + return self.emit(ttir.fp_to_fp(self.ctx, src.inner, self.swapElem(src, out_dtype), opts.rounding, self.loc())); + } + /// tt.clampf with propagate_nan=.none. Use `clampfOpts` to override. + pub fn clampf(self: *Builder, x: Value, lo: Value, hi: Value) Value { + return self.clampfOpts(x, lo, hi, .{}); + } + /// tt.clampf with full options. + pub fn clampfOpts(self: *Builder, x: Value, lo: Value, hi: Value, opts: ClampOpts) Value { + return self.emit(ttir.clampf(self.ctx, x.inner, lo.inner, hi.inner, opts.propagate_nan, self.loc())); + } + /// `tl.sqrt_rn(x)` — precise square root (IEEE round-to-nearest). + pub fn sqrtRn(self: *Builder, x: Value) Value { + return self.emit(ttir.precise_sqrt(self.ctx, x.inner, self.loc())); + } + /// `tl.div_rn(x, y)` — precise division (IEEE round-to-nearest). + pub fn divRn(self: *Builder, x: Value, y: Value) Value { + return self.emit(ttir.precise_divf(self.ctx, x.inner, y.inner, self.loc())); + } + /// `tl.umulhi(x, y)` — most-significant N bits of the 2N-bit unsigned product. + pub fn umulhi(self: *Builder, x: Value, y: Value) Value { + return self.emit(ttir.mulhiui(self.ctx, x.inner, y.inner, self.loc())); + } + + /// tt.dot — matmul with accumulator. Result type = c_acc. Use `dotOpts` for precision. + pub fn dot(self: *Builder, a: Value, b: Value, c_acc: Value) Value { + return self.dotOpts(a, b, c_acc, .{}); + } + + /// tt.dot with full options. Result type = `c_acc.type_()`. + pub fn dotOpts(self: *Builder, a: Value, b: Value, c_acc: Value, opts: DotOpts) Value { + return self.emit(ttir.dot(self.ctx, a.inner, b.inner, c_acc.inner, c_acc.type_(), opts.input_precision, opts.max_num_imprecise_acc, self.loc())); + } + + /// tt.reshape, element type preserved. Use `reshapeOpts` for can_reorder / efficient_layout. + pub fn reshape(self: *Builder, src: Value, new_shape: []const i64) Value { + return self.reshapeOpts(src, new_shape, .{}); + } + + /// tt.reshape with full options. + pub fn reshapeOpts(self: *Builder, src: Value, new_shape: []const i64, opts: ReshapeOpts) Value { + const result_ty = mlir.rankedTensorType(new_shape, src.elemType()); + return self.emit(ttir.reshape(self.ctx, src.inner, result_ty, opts.can_reorder, opts.efficient_layout, self.loc())); + } + + /// tt.trans — permute dimensions. Result shape is src.shape permuted by order. + pub fn trans(self: *Builder, src: Value, order: []const i32) Value { + const src_shape = src.shape(); + std.debug.assert(order.len == src_shape.len); + var out_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (order) |i| out_shape.appendAssumeCapacity(src_shape.get(@intCast(i))); + const result_type = mlir.rankedTensorType(out_shape.constSlice(), src.elemType()); + return self.emit(ttir.trans(self.ctx, src.inner, order, result_type, self.loc())); + } + + /// `tl.permute(src, *dims)` — alias for `trans`. + pub fn permute(self: *Builder, src: Value, order: []const i32) Value { + return self.trans(src, order); + } + + /// tt.cat — concatenate along dim 0. Use `catOpts` for can_reorder / dim. + pub fn cat(self: *Builder, lhs: Value, rhs: Value) Value { + return self.catOpts(lhs, rhs, .{}); + } + + /// tt.cat with full options. dim != 0 is not yet supported. + pub fn catOpts(self: *Builder, lhs: Value, rhs: Value, opts: CatOpts) Value { + std.debug.assert(opts.dim == 0); // TODO: dim != 0 requires join+permute+reshape + _ = opts.can_reorder; // forwarded to ttir.cat when supported + const ls = lhs.shape(); + const rs = rhs.shape(); + std.debug.assert(ls.len == rs.len); + var out_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + out_shape.appendAssumeCapacity(ls.get(0) + rs.get(0)); + for (1..ls.len) |i| out_shape.appendAssumeCapacity(ls.get(i)); + const result_type = mlir.rankedTensorType(out_shape.constSlice(), lhs.elemType()); + return self.emit(ttir.cat(self.ctx, lhs.inner, rhs.inner, result_type, self.loc())); + } + + /// `tt.join` — stack two equal-shape tensors along a new minor dim of size 2. + pub fn join(self: *Builder, lhs: Value, rhs: Value) Value { + const ls = lhs.shape(); + var out_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..ls.len) |i| out_shape.appendAssumeCapacity(ls.get(i)); + out_shape.appendAssumeCapacity(2); + const result_type = mlir.rankedTensorType(out_shape.constSlice(), lhs.elemType()); + return self.emit(ttir.join(self.ctx, lhs.inner, rhs.inner, result_type, self.loc())); + } + + /// `tt.split` — split a tensor whose last dim is 2 into two tensors with + /// that last dim dropped. Returns (lhs, rhs). + pub fn split(self: *Builder, src: Value) [2]Value { + const ss = src.shape(); + std.debug.assert(ss.len >= 1 and ss.get(ss.len - 1) == 2); + var out_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..ss.len - 1) |i| out_shape.appendAssumeCapacity(ss.get(i)); + const result_type = mlir.rankedTensorType(out_shape.constSlice(), src.elemType()); + const op = ttir.split(self.ctx, src.inner, result_type, self.loc()); + _ = op.appendTo(self.currentBlock()); + return .{ + .{ .inner = op.result(0), .kernel = self }, + .{ .inner = op.result(1), .kernel = self }, + }; + } + + /// `tl.item(src)` / `tt.unsplat` — 1-element tensor → scalar of the same element type. + pub fn item(self: *Builder, src: Value) Value { + return self.emit(ttir.unsplat(self.ctx, src.inner, src.elemType(), self.loc())); + } + + /// tt.gather — output shape matches indices, element type matches src. + pub fn gather(self: *Builder, src: Value, indices: Value, axis: i32) Value { + const result_ty = mlir.rankedTensorType(indices.shape().constSlice(), src.elemType()); + return self.emit(ttir.gather(self.ctx, src.inner, indices.inner, axis, result_ty, false, self.loc())); + } + + /// tt.histogram — output is `tensor`. Use `histogramOpts` for mask. + pub fn histogram(self: *Builder, src: Value, num_bins: i64) Value { + return self.histogramOpts(src, num_bins, .{}); + } + + /// tt.histogram with mask option. + pub fn histogramOpts(self: *Builder, src: Value, num_bins: i64, opts: HistogramOpts) Value { + const result_ty = mlir.rankedTensorType(&.{num_bins}, DType.i32.toMlir(self.ctx)); + const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; + return self.emit(ttir.histogram(self.ctx, src.inner, mask_inner, result_ty, self.loc())); + } + + /// tt.assert — device-side assert (i1 or tensor<...xi1>). Use `deviceAssertOpts` for mask. + pub fn deviceAssert(self: *Builder, condition: Value, message: []const u8) void { + self.deviceAssertOpts(condition, message, .{}); + } + + /// tt.assert with mask option. + pub fn deviceAssertOpts(self: *Builder, condition: Value, message: []const u8, opts: DeviceAssertOpts) void { + // TODO: threading mask through ttir.assert_ when supported. + _ = opts.mask; + _ = ttir.assert_(self.ctx, condition.inner, message, self.loc()).appendTo(self.currentBlock()); + } + + /// tt.atomic_rmw — returns old value. Use `atomicRmwOpts` for mask / sem / scope. + pub fn atomicRmw(self: *Builder, rmw: ttir.RMWOp, ptr: Value, val: Value) Value { + return self.atomicRmwOpts(rmw, ptr, val, .{}); + } + + /// tt.atomic_rmw with full options. + pub fn atomicRmwOpts(self: *Builder, rmw: ttir.RMWOp, ptr: Value, val: Value, opts: AtomicRMWOpts) Value { + const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; + return self.emit(ttir.atomic_rmw(self.ctx, rmw, ptr.inner, val.inner, mask_inner, opts.sem, opts.scope, self.loc())); + } + + /// tt.atomic_cas — returns old value. Use `atomicCasOpts` for sem / scope. + pub fn atomicCas(self: *Builder, ptr: Value, cmp: Value, val: Value) Value { + return self.atomicCasOpts(ptr, cmp, val, .{}); + } + + /// tt.atomic_cas with full options. + pub fn atomicCasOpts(self: *Builder, ptr: Value, cmp: Value, val: Value, opts: AtomicCasOpts) Value { + return self.emit(ttir.atomic_cas(self.ctx, ptr.inner, cmp.inner, val.inner, opts.sem, opts.scope, self.loc())); + } + + /// `tt.call` — call another tt.func in this module by symbol name. + pub fn call(self: *Builder, callee: []const u8, operands: []const Value, result_types: []const *const mlir.Type) []Value { + const op = ttir.call(self.ctx, callee, self.innerSlice(operands), result_types, null, null, self.loc()); + return self.emitMulti(op, result_types.len); + } + + /// tt.dot_scaled — dot with microscaling factors. Use `dotScaledOpts` for options. + pub fn dotScaled( + self: *Builder, + a: Value, + b: Value, + c_acc: Value, + a_scale: ?Value, + b_scale: ?Value, + a_elem_type: ttir.ScaleDotElemType, + b_elem_type: ttir.ScaleDotElemType, + ) Value { + return self.dotScaledOpts(a, b, c_acc, a_scale, b_scale, a_elem_type, b_elem_type, .{}); + } + + /// `tt.dot_scaled` with the full named-param set. Result type = `c_acc.type_()`. + pub fn dotScaledOpts( + self: *Builder, + a: Value, + b: Value, + c_acc: Value, + a_scale: ?Value, + b_scale: ?Value, + a_elem_type: ttir.ScaleDotElemType, + b_elem_type: ttir.ScaleDotElemType, + opts: DotScaledOpts, + ) Value { + const a_s: ?*const mlir.Value = if (a_scale) |s| s.inner else null; + const b_s: ?*const mlir.Value = if (b_scale) |s| s.inner else null; + return self.emit(ttir.dot_scaled( + self.ctx, + a.inner, + b.inner, + c_acc.inner, + a_s, + b_s, + a_elem_type, + b_elem_type, + c_acc.type_(), + opts.fast_math, + opts.lhs_k_pack, + opts.rhs_k_pack, + self.loc(), + )); + } + + /// tt.extern_elementwise — call a library symbol pointwise. + /// Result is `tensor`; pass `&.{}` for scalar. + pub fn externElementwise( + self: *Builder, + srcs: []const Value, + result_shape: []const i64, + result_dtype: DType, + libname: []const u8, + libpath: []const u8, + symbol: []const u8, + ) Value { + return self.externElementwiseOpts(srcs, result_shape, result_dtype, libname, libpath, symbol, .{}); + } + + /// `tt.extern_elementwise` with the full named-param set. + pub fn externElementwiseOpts( + self: *Builder, + srcs: []const Value, + result_shape: []const i64, + result_dtype: DType, + libname: []const u8, + libpath: []const u8, + symbol: []const u8, + opts: ExternElementwiseOpts, + ) Value { + const result_type: *const mlir.Type = if (result_shape.len == 0) + result_dtype.toMlir(self.ctx) + else + mlir.rankedTensorType(result_shape, result_dtype.toMlir(self.ctx)); + return self.emit(ttir.extern_elementwise( + self.ctx, + self.innerSlice(srcs), + result_type, + libname, + libpath, + symbol, + opts.is_pure, + self.loc(), + )); + } + + /// tt.print — device-side print. `is_signed` has one entry per arg (1=signed, 0=unsigned). + /// Use `devicePrintOpts` for hex. + pub fn devicePrint( + self: *Builder, + prefix: []const u8, + args: []const Value, + is_signed: []const i32, + ) void { + self.devicePrintOpts(prefix, args, is_signed, .{}); + } + + /// tt.print with full options. + pub fn devicePrintOpts( + self: *Builder, + prefix: []const u8, + args: []const Value, + is_signed: []const i32, + opts: PrintOpts, + ) void { + _ = ttir.print(self.ctx, prefix, opts.hex, self.innerSlice(args), is_signed, self.loc()).appendTo(self.currentBlock()); + } + + /// tt.make_tensor_descriptor — build a TMA descriptor. + /// shape/strides are runtime; block_shape/dtype describe the compile-time tile. + pub fn makeTensorDescriptor( + self: *Builder, + base: Value, + shape: []const Value, + strides: []const Value, + padding: ttir.PaddingOption, + block_shape: []const i64, + dtype: DType, + ) Value { + const result_type = ttir.tensorDescType(block_shape, dtype.toMlir(self.ctx), null); + return self.emit(ttir.make_tensor_descriptor( + self.ctx, + base.inner, + self.innerSlice(shape), + self.innerSlice(strides), + padding, + result_type, + self.loc(), + )); + } + + /// tt.descriptor_load — TMA load. Use `descriptorLoadOpts` for cache / eviction. + pub fn descriptorLoad( + self: *Builder, + desc: Value, + indices: []const Value, + shape: []const i64, + dtype: DType, + ) Value { + return self.descriptorLoadOpts(desc, indices, shape, dtype, .{}); + } + + /// `tt.descriptor_load` with the full named-param set. + pub fn descriptorLoadOpts( + self: *Builder, + desc: Value, + indices: []const Value, + shape: []const i64, + dtype: DType, + opts: DescriptorLoadOpts, + ) Value { + const result_ty = mlir.rankedTensorType(shape, dtype.toMlir(self.ctx)); + return self.emit(ttir.descriptor_load(self.ctx, desc.inner, self.innerSlice(indices), result_ty, opts.cache_modifier, opts.eviction_policy, self.loc())); + } + + /// `tt.descriptor_store` — TMA store. + pub fn descriptorStore(self: *Builder, desc: Value, src: Value, indices: []const Value) void { + _ = ttir.descriptor_store(self.ctx, desc.inner, src.inner, self.innerSlice(indices), self.loc()).appendTo(self.currentBlock()); + } + + /// `tt.descriptor_reduce` — TMA reducing store. + pub fn descriptorReduce( + self: *Builder, + kind: ttir.DescriptorReduceKind, + desc: Value, + src: Value, + indices: []const Value, + ) void { + _ = ttir.descriptor_reduce(self.ctx, kind, desc.inner, src.inner, self.innerSlice(indices), self.loc()).appendTo(self.currentBlock()); + } + + /// tt.descriptor_gather — TMA gather by x_offsets + y_offset. + pub fn descriptorGather( + self: *Builder, + desc: Value, + x_offsets: Value, + y_offset: Value, + shape: []const i64, + dtype: DType, + ) Value { + const result_ty = mlir.rankedTensorType(shape, dtype.toMlir(self.ctx)); + return self.emit(ttir.descriptor_gather(self.ctx, desc.inner, x_offsets.inner, y_offset.inner, result_ty, self.loc())); + } + + /// `tt.descriptor_scatter` — TMA scatter. + pub fn descriptorScatter(self: *Builder, desc: Value, x_offsets: Value, y_offset: Value, src: Value) void { + _ = ttir.descriptor_scatter(self.ctx, desc.inner, x_offsets.inner, y_offset.inner, src.inner, self.loc()).appendTo(self.currentBlock()); + } + + // ==================== math dialect ==================== + + pub fn exp(self: *Builder, x: Value) Value { + return self.emit(math.exp(self.ctx, x.inner, self.loc())); + } + pub fn exp2(self: *Builder, x: Value) Value { + return self.emit(math.exp2(self.ctx, x.inner, self.loc())); + } + pub fn log(self: *Builder, x: Value) Value { + return self.emit(math.log(self.ctx, x.inner, self.loc())); + } + pub fn log2(self: *Builder, x: Value) Value { + return self.emit(math.log2(self.ctx, x.inner, self.loc())); + } + pub fn sqrt(self: *Builder, x: Value) Value { + return self.emit(math.sqrt(self.ctx, x.inner, self.loc())); + } + pub fn rsqrt(self: *Builder, x: Value) Value { + return self.emit(math.rsqrt(self.ctx, x.inner, self.loc())); + } + pub fn sin(self: *Builder, x: Value) Value { + return self.emit(math.sin(self.ctx, x.inner, self.loc())); + } + pub fn cos(self: *Builder, x: Value) Value { + return self.emit(math.cos(self.ctx, x.inner, self.loc())); + } + pub fn tan(self: *Builder, x: Value) Value { + return self.emit(math.tan(self.ctx, x.inner, self.loc())); + } + pub fn tanh(self: *Builder, x: Value) Value { + return self.emit(math.tanh(self.ctx, x.inner, self.loc())); + } + pub fn erf(self: *Builder, x: Value) Value { + return self.emit(math.erf(self.ctx, x.inner, self.loc())); + } + pub fn absf(self: *Builder, x: Value) Value { + return self.emit(math.absf(self.ctx, x.inner, self.loc())); + } + pub fn absi(self: *Builder, x: Value) Value { + return self.emit(math.absi(self.ctx, x.inner, self.loc())); + } + /// `tl.abs(x)` — absolute value, auto-dispatch by element type (float→`absf`, + /// signed int→`absi`, unsigned int→no-op). + pub fn abs(self: *Builder, x: Value) Value { + if (x.isFloatElem()) return self.absf(x); + return self.absi(x); + } + pub fn floor(self: *Builder, x: Value) Value { + return self.emit(math.floor(self.ctx, x.inner, self.loc())); + } + pub fn ceil(self: *Builder, x: Value) Value { + return self.emit(math.ceil(self.ctx, x.inner, self.loc())); + } + pub fn powf(self: *Builder, x: Value, y: Value) Value { + return self.emit(math.powf(self.ctx, x.inner, y.inner, self.loc())); + } + pub fn fma(self: *Builder, a: Value, b: Value, c_: Value) Value { + return self.emit(math.fma(self.ctx, a.inner, b.inner, c_.inner, self.loc())); + } + /// `math.clampf` — distinct from `tt.clampf` (no propagateNan attr). + pub fn mathClampf(self: *Builder, value: Value, lo: Value, hi: Value) Value { + return self.emit(math.clampf(self.ctx, value.inner, lo.inner, hi.inner, self.loc())); + } + /// `math.sincos` — returns (sin, cos). + pub fn sincos(self: *Builder, x: Value) [2]Value { + const op = math.sincos(self.ctx, x.inner, self.loc()); + _ = op.appendTo(self.currentBlock()); + return .{ .{ .inner = op.result(0), .kernel = self }, .{ .inner = op.result(1), .kernel = self } }; + } + /// `math.fpowi` — float base, integer exponent. + pub fn fpowi(self: *Builder, base: Value, power: Value) Value { + return self.emit(math.fpowi(self.ctx, base.inner, power.inner, self.loc())); + } + pub fn ipowi(self: *Builder, base: Value, power: Value) Value { + return self.emit(math.ipowi(self.ctx, base.inner, power.inner, self.loc())); + } + pub fn atan2(self: *Builder, y: Value, x: Value) Value { + return self.emit(math.atan2(self.ctx, y.inner, x.inner, self.loc())); + } + pub fn copysign(self: *Builder, mag: Value, sign: Value) Value { + return self.emit(math.copysign(self.ctx, mag.inner, sign.inner, self.loc())); + } + pub fn mathRound(self: *Builder, x: Value) Value { + return self.emit(math.round(self.ctx, x.inner, self.loc())); + } + pub fn mathRoundEven(self: *Builder, x: Value) Value { + return self.emit(math.roundeven(self.ctx, x.inner, self.loc())); + } + pub fn mathTrunc(self: *Builder, x: Value) Value { + return self.emit(math.trunc(self.ctx, x.inner, self.loc())); + } + + /// `arith.convertf` — same-bitwidth float cast (e.g. f16 ↔ bf16) with + /// default rounding. Shape preserved from `src`. Use `convertfOpts` for + /// `fastMath`. + pub fn convertf(self: *Builder, src: Value, out_dtype: DType) Value { + return self.convertfOpts(src, out_dtype, .{}); + } + /// `arith.convertf` with the full named-param set. + pub fn convertfOpts(self: *Builder, src: Value, out_dtype: DType, opts: arith.ConvertFOpts) Value { + return self.emit(arith.convertf(self.ctx, src.inner, self.swapElem(src, out_dtype), opts, self.loc())); + } + /// `arith.scaling_truncf` — MXFP downcast with per-block scales, default + /// options. Shape preserved from `src`. Use `scalingTruncfOpts` for + /// `fastMath`. + pub fn scalingTruncf(self: *Builder, src: Value, scale: Value, out_dtype: DType) Value { + return self.scalingTruncfOpts(src, scale, out_dtype, .{}); + } + /// `arith.scaling_truncf` with the full named-param set. + pub fn scalingTruncfOpts(self: *Builder, src: Value, scale: Value, out_dtype: DType, opts: arith.ConvertFOpts) Value { + return self.emit(arith.scaling_truncf(self.ctx, src.inner, scale.inner, self.swapElem(src, out_dtype), opts, self.loc())); + } + + // ==================== tt.map_elementwise / elementwise_inline_asm ==================== + + /// `tt.map_elementwise` — body builder receives one scalar Value per + /// `srcs` tensor (repeated `pack` times, flattened), and must return the + /// scalar results to feed back via `tt.map_elementwise.return`. + pub fn mapElementwise( + self: *Builder, + srcs: []const Value, + pack: i32, + scalar_arg_types: []const *const mlir.Type, + result_tensor_types: []const *const mlir.Type, + comptime body_fn: anytype, + body_ctx: anytype, + ) []Value { + const scratch = self.arena.allocator(); + const locs = scratch.alloc(*const mlir.Location, scalar_arg_types.len) catch @panic("Builder.mapElementwise OOM"); + for (locs) |*l| l.* = self.loc(); + const body_block = mlir.Block.init(scalar_arg_types, locs); + + self.pushBlock(body_block); + const block_args = scratch.alloc(Value, scalar_arg_types.len) catch @panic("Builder.mapElementwise OOM"); + for (block_args, 0..) |*v, i| v.* = .{ .inner = body_block.argument(i), .kernel = self }; + const out_vals: []const Value = body_fn(self, block_args, body_ctx); + _ = ttir.map_elementwise_return(self.ctx, self.innerSlice(out_vals), self.loc()).appendTo(body_block); + self.popBlock(); + + const op = ttir.map_elementwise( + self.ctx, + self.innerSlice(srcs), + pack, + body_block, + result_tensor_types, + self.loc(), + ); + return self.emitMulti(op, result_tensor_types.len); + } + + /// `tl.inline_asm_elementwise(asm, constraints, args, dtype)` — inline asm + /// pointwise op with default options. Use `inlineAsmElementwiseOpts` for + /// `is_pure` / `pack`. + pub fn inlineAsmElementwise( + self: *Builder, + asm_string: []const u8, + constraints: []const u8, + args: []const Value, + result_types: []const *const mlir.Type, + ) []Value { + return self.inlineAsmElementwiseOpts(asm_string, constraints, args, result_types, .{}); + } + + /// `tl.inline_asm_elementwise(asm, constraints, args, dtype, is_pure=..., pack=...)` + /// with the full named-param set. + pub fn inlineAsmElementwiseOpts( + self: *Builder, + asm_string: []const u8, + constraints: []const u8, + args: []const Value, + result_types: []const *const mlir.Type, + opts: InlineAsmOpts, + ) []Value { + const op = ttir.elementwise_inline_asm(self.ctx, asm_string, constraints, self.innerSlice(args), result_types, opts.is_pure, opts.pack, self.loc()); + return self.emitMulti(op, result_types.len); + } + + // ==================== scf.parallel / scf.reduce / scf.forall ==================== + + pub fn parallel( + self: *Builder, + lbs: []const Value, + ubs: []const Value, + steps: []const Value, + inits: []const Value, + reduction_result_types: []const *const mlir.Type, + comptime body_fn: anytype, + comptime combiners: anytype, + body_ctx: anytype, + ) []Value { + std.debug.assert(lbs.len == ubs.len); + std.debug.assert(inits.len == reduction_result_types.len); + + const scratch = self.arena.allocator(); + const idx_ty = mlir.indexType(self.ctx); + const iv_types = scratch.alloc(*const mlir.Type, lbs.len) catch @panic("Builder.parallel OOM"); + const iv_locs = scratch.alloc(*const mlir.Location, lbs.len) catch @panic("Builder.parallel OOM"); + for (iv_types, iv_locs) |*t, *l| { + t.* = idx_ty; + l.* = self.loc(); + } + const body_block = mlir.Block.init(iv_types, iv_locs); + + self.pushBlock(body_block); + const iv_args = scratch.alloc(Value, lbs.len) catch @panic("Builder.parallel OOM"); + for (iv_args, 0..) |*v, i| v.* = .{ .inner = body_block.argument(i), .kernel = self }; + const reduce_operands: []const Value = body_fn(self, iv_args, body_ctx); + std.debug.assert(reduce_operands.len == reduction_result_types.len); + + // Build scf.reduce with one region per reduction. + const red_blocks = scratch.alloc(*mlir.Block, combiners.len) catch @panic("Builder.parallel OOM"); + inline for (combiners, 0..) |comb, i| { + const arg_ty = reduce_operands[i].type_(); + const arg_locs: [2]*const mlir.Location = .{ self.loc(), self.loc() }; + const red_block = mlir.Block.init(&.{ arg_ty, arg_ty }, &arg_locs); + self.pushBlock(red_block); + const lhs_v: Value = .{ .inner = red_block.argument(0), .kernel = self }; + const rhs_v: Value = .{ .inner = red_block.argument(1), .kernel = self }; + const combined: Value = comb(self, lhs_v, rhs_v, body_ctx); + _ = scf.reduce_return(self.ctx, combined.inner, self.loc()).appendTo(red_block); + self.popBlock(); + red_blocks[i] = red_block; + } + + _ = scf.reduce(self.ctx, self.innerSlice(reduce_operands), red_blocks, self.loc()).appendTo(body_block); + self.popBlock(); + + const op = scf.parallel( + self.ctx, + self.innerSlice(lbs), + self.innerSlice(ubs), + self.innerSlice(steps), + self.innerSlice(inits), + body_block, + reduction_result_types, + self.loc(), + ); + _ = op.appendTo(self.currentBlock()); + + const out = self.arena.allocator().alloc(Value, reduction_result_types.len) catch @panic("Builder.parallel OOM"); + for (0..reduction_result_types.len) |i| out[i] = .{ .inner = op.result(i), .kernel = self }; + return out; + } + + // ==================== reduce / scan ==================== + + /// `tt.reduce` along `axis`, single-input. See `ReduceArgs` for fields. + /// `ctx` is separate from `args` so `ReduceArgs(@TypeOf(ctx))` can be + /// inferred. + pub fn reduce(self: *Builder, ctx: anytype, args: ReduceArgs(@TypeOf(ctx))) Value { + const region = mlir.Block.init(&.{ args.elem, args.elem }, &.{ self.loc(), self.loc() }); + + self.pushBlock(region); + const lhs: Value = .{ .inner = region.argument(0), .kernel = self }; + const rhs: Value = .{ .inner = region.argument(1), .kernel = self }; + const combined: Value = args.combine(self, lhs, rhs, ctx); + _ = ttir.reduce_return(self.ctx, &.{combined.inner}, self.loc()).appendTo(region); + self.popBlock(); + + const op = ttir.reduce(self.ctx, &.{args.src.inner}, args.axis, region, &.{args.result}, self.loc()); + _ = op.appendTo(self.currentBlock()); + return .{ .inner = op.result(0), .kernel = self }; + } + + /// `tt.scan` along `axis`, single-input. See `ScanArgs` for fields. + pub fn scan(self: *Builder, ctx: anytype, args: ScanArgs(@TypeOf(ctx))) Value { + const region = mlir.Block.init(&.{ args.elem, args.elem }, &.{ self.loc(), self.loc() }); + + self.pushBlock(region); + const lhs: Value = .{ .inner = region.argument(0), .kernel = self }; + const rhs: Value = .{ .inner = region.argument(1), .kernel = self }; + const combined: Value = args.combine(self, lhs, rhs, ctx); + _ = ttir.scan_return(self.ctx, &.{combined.inner}, self.loc()).appendTo(region); + self.popBlock(); + + const op = ttir.scan(self.ctx, &.{args.src.inner}, args.axis, args.reverse, region, &.{args.result}, self.loc()); + _ = op.appendTo(self.currentBlock()); + return .{ .inner = op.result(0), .kernel = self }; + } + + // ==================== SCF regions ==================== + + pub fn openFor( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + ) ForScope(tupleArity(@TypeOf(inits), "openFor: inits")) { + const N = comptime tupleArity(@TypeOf(inits), "openFor: inits"); + const fields = @typeInfo(@TypeOf(inits)).@"struct".fields; + + const lb_v: Value = if (@TypeOf(lower) == Value) lower else self.lift(lower); + const ub_v: Value = if (@TypeOf(upper) == Value) upper else self.constMatching(upper, lb_v.type_()); + const step_v: Value = if (@TypeOf(step) == Value) step else self.constMatching(step, lb_v.type_()); + + var block_types: [N + 1]*const mlir.Type = undefined; + var block_locs: [N + 1]*const mlir.Location = undefined; + block_types[0] = lb_v.type_(); + block_locs[0] = self.loc(); + var inits_inner: [N]*const mlir.Value = undefined; + inline for (fields, 0..) |f, i| { + const raw = @field(inits, f.name); + const v: Value = if (f.type == Value) raw else self.lift(raw); + block_types[i + 1] = v.type_(); + block_locs[i + 1] = self.loc(); + inits_inner[i] = v.inner; + } + + const body = mlir.Block.init(&block_types, &block_locs); + self.pushBlock(body); + + var carried: [N]Value = undefined; + for (0..N) |i| carried[i] = .{ .inner = body.argument(i + 1), .kernel = self }; + + return .{ + .kernel = self, + .body = body, + .lb_inner = lb_v.inner, + .ub_inner = ub_v.inner, + .step_inner = step_v.inner, + .inits_inner = inits_inner, + .iv = .{ .inner = body.argument(0), .kernel = self }, + .carried = carried, + }; + } + + pub fn returnIf(self: *Builder, cond: Value, values: anytype) void { + const current = self.currentBlock(); + const region = current.parentRegion(); + + const fields = @typeInfo(@TypeOf(values)).@"struct".fields; + const cont_block = mlir.Block.init(&.{}, &.{}); + + // Use a shared exit block for all returns. This ensures that every + // return path (early or natural) goes through a pure tt.return block, + // allowing MLIR's Canonicalizer/BlockMerging to create a single join + // point. + if (self.exit_block == null) { + self.exit_block = mlir.Block.init(&.{}, &.{}); + + var ret_operands: [fields.len]*const mlir.Value = undefined; + inline for (fields, 0..) |f, i| { + if (f.type != Value) + @compileError("returnIf: every value must be a Value"); + ret_operands[i] = @field(values, f.name).inner; + } + _ = ttir.return_(self.ctx, &ret_operands, self.loc()).appendTo(self.exit_block.?); + } + region.appendOwnedBlock(cont_block); + + _ = cf.cond_br( + self.ctx, + cond.inner, + self.exit_block.?, + &.{}, + cont_block, + &.{}, + null, + self.loc(), + ).appendTo(current); + + if (self.block_stack.items.len == 0) { + self.pushBlock(cont_block); + } else { + self.block_stack.items[self.block_stack.items.len - 1] = cont_block; + } + } + + pub fn openReturnIf(self: *Builder, cond: Value) ReturnIfScope { + const current = self.currentBlock(); + const region = current.parentRegion(); + + const ret_block = mlir.Block.init(&.{}, &.{}); + const cont_block = mlir.Block.init(&.{}, &.{}); + region.appendOwnedBlock(ret_block); + region.appendOwnedBlock(cont_block); + + _ = cf.cond_br( + self.ctx, + cond.inner, + ret_block, + &.{}, + cont_block, + &.{}, + null, + self.loc(), + ).appendTo(current); + + self.pushBlock(ret_block); + + return .{ + .kernel = self, + .ret_block = ret_block, + .cont_block = cont_block, + }; + } + + pub fn openIf(self: *Builder, cond: Value) IfOnlyScope { + const then_block = mlir.Block.init(&.{}, &.{}); + self.pushBlock(then_block); + return .{ + .kernel = self, + .cond_inner = cond.inner, + .then_block = then_block, + }; + } + + pub fn openIfElse( + self: *Builder, + cond: Value, + result_types: anytype, + ) IfScope(tupleArity(@TypeOf(result_types), "openIfElse: result_types")) { + const N = comptime tupleArity(@TypeOf(result_types), "openIfElse: result_types"); + const fields = @typeInfo(@TypeOf(result_types)).@"struct".fields; + + var types: [N]*const mlir.Type = undefined; + inline for (fields, 0..) |f, i| { + if (f.type != *const mlir.Type) + @compileError("openIfElse: every result_type must be *const mlir.Type (use k.scalarTy/tensorTy)"); + types[i] = @field(result_types, f.name); + } + + const then_block = mlir.Block.init(&.{}, &.{}); + const else_block = mlir.Block.init(&.{}, &.{}); + self.pushBlock(then_block); + + return .{ + .kernel = self, + .cond_inner = cond.inner, + .then_block = then_block, + .else_block = else_block, + .result_types = types, + }; + } + + pub fn openWhile( + self: *Builder, + inits: anytype, + after_types: anytype, + ) WhileScope( + tupleArity(@TypeOf(inits), "openWhile: inits"), + tupleArity(@TypeOf(after_types), "openWhile: after_types"), + ) { + const N = comptime tupleArity(@TypeOf(inits), "openWhile: inits"); + const M = comptime tupleArity(@TypeOf(after_types), "openWhile: after_types"); + const init_fields = @typeInfo(@TypeOf(inits)).@"struct".fields; + const type_fields = @typeInfo(@TypeOf(after_types)).@"struct".fields; + + var before_types: [N]*const mlir.Type = undefined; + var before_locs: [N]*const mlir.Location = undefined; + var inits_inner: [N]*const mlir.Value = undefined; + inline for (init_fields, 0..) |f, i| { + if (f.type != Value) + @compileError("openWhile: every init must be a Value"); + const v: Value = @field(inits, f.name); + before_types[i] = v.type_(); + before_locs[i] = self.loc(); + inits_inner[i] = v.inner; + } + const before_block = mlir.Block.init(&before_types, &before_locs); + + var after_tys: [M]*const mlir.Type = undefined; + var after_locs: [M]*const mlir.Location = undefined; + inline for (type_fields, 0..) |f, i| { + if (f.type != *const mlir.Type) + @compileError("openWhile: every after_type must be *const mlir.Type"); + after_tys[i] = @field(after_types, f.name); + after_locs[i] = self.loc(); + } + const after_block = mlir.Block.init(&after_tys, &after_locs); + + self.pushBlock(before_block); + var before_carried: [N]Value = undefined; + for (0..N) |i| before_carried[i] = .{ .inner = before_block.argument(i), .kernel = self }; + + return .{ + .kernel = self, + .before_block = before_block, + .after_block = after_block, + .inits_inner = inits_inner, + .after_types = after_tys, + .before_carried = before_carried, + }; + } + + // ==================== finalization ==================== + + /// Append `tt.return` with the given results, verify the module, and + /// serialize to a NUL-terminated TTIR string owned by the kernel's + /// allocator (passed to `init`). Caller must `defer allocator.free(ir)`. + pub fn finish(self: *Builder, results: []const Value) FinishError![:0]const u8 { + // After any `returnIf` call, the entry block already has a + // `cf.cond_br` terminator; the fall-through block sits on the stack + // as the new insertion point. `currentBlock()` returns that block + // (or `entry_block` if no cf was emitted), so the final terminator + // goes to the correct "tail" block in either case. + const current = self.currentBlock(); + if (self.exit_block) |exit| { + const region = self.entry_block.?.parentRegion(); + region.appendOwnedBlock(exit); + _ = cf.br(self.ctx, exit, &.{}, self.loc()).appendTo(current); + } else { + _ = ttir.return_(self.ctx, self.innerSlice(results), self.loc()).appendTo(current); + } + + if (!self.module.operation().verify()) { + return error.InvalidMlir; + } + + var al: std.Io.Writer.Allocating = .init(self.allocator); + defer al.deinit(); + + try al.writer.print("{f}", .{self.module.operation().fmt(.{ + .debug_info = true, + .debug_info_pretty_form = false, + .print_name_loc_as_prefix = true, + })}); + + return try self.allocator.dupeZ(u8, al.written()); + } +}; diff --git a/kernels/triton/dtype.zig b/kernels/triton/dtype.zig new file mode 100644 index 00000000..56d6ed64 --- /dev/null +++ b/kernels/triton/dtype.zig @@ -0,0 +1,72 @@ +const std = @import("std"); +const mlir = @import("mlir"); + +pub const DType = enum { + i1, + i8, + i16, + i32, + i64, + f16, + bf16, + f32, + f64, + f8e4m3fn, + f8e5m2, + + pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { + return switch (self) { + .i1 => mlir.integerType(ctx, .i1), + .i8 => mlir.integerType(ctx, .i8), + .i16 => mlir.integerType(ctx, .i16), + .i32 => mlir.integerType(ctx, .i32), + .i64 => mlir.integerType(ctx, .i64), + .f16 => mlir.floatType(ctx, .f16), + .bf16 => mlir.floatType(ctx, .bf16), + .f32 => mlir.floatType(ctx, .f32), + .f64 => mlir.floatType(ctx, .f64), + .f8e4m3fn => mlir.floatType(ctx, .f8e4m3fn), + .f8e5m2 => mlir.floatType(ctx, .f8e5m2), + }; + } +}; + +pub fn isFloatDtype(dt: DType) bool { + return switch (dt) { + .f16, .bf16, .f32, .f64, .f8e4m3fn, .f8e5m2 => true, + else => false, + }; +} + +pub fn dtypeBitwidth(dt: DType) usize { + return switch (dt) { + .i1 => 1, + .i8, .f8e4m3fn, .f8e5m2 => 8, + .i16, .f16, .bf16 => 16, + .i32, .f32 => 32, + .i64, .f64 => 64, + }; +} + +pub fn intBitwidth(dt: DType) u32 { + return switch (dt) { + .i1 => 1, + .i8 => 8, + .i16 => 16, + .i32 => 32, + .i64 => 64, + else => std.debug.panic("intBitwidth: not an integer DType: {s}", .{@tagName(dt)}), + }; +} + +pub fn mlirElemToDType(ctx: *mlir.Context, elem: *const mlir.Type) DType { + inline for (std.meta.fields(DType)) |f| { + const dt = @field(DType, f.name); + if (elem.eql(dt.toMlir(ctx))) return dt; + } + @panic("element type not a recognized Triton DType"); +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/mlir/BUILD.bazel b/mlir/BUILD.bazel index fe081091..8e38ce82 100644 --- a/mlir/BUILD.bazel +++ b/mlir/BUILD.bazel @@ -8,11 +8,15 @@ cc_library( hdrs = ["mlir.h"], visibility = ["//mlir:__subpackages__"], deps = [ + "@llvm-project//mlir:AffineDialect", "@llvm-project//mlir:CAPIArith", + "@llvm-project//mlir:CAPICF", "@llvm-project//mlir:CAPIIR", "@llvm-project//mlir:CAPIMath", + "@llvm-project//mlir:CAPIMemRef", "@llvm-project//mlir:CAPISCF", "@llvm-project//mlir:CAPITransforms", + "@llvm-project//mlir:CAPIVector", "@llvm-project//mlir:FuncExtensions", ], ) diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/BUILD.bazel index 38e70da7..4dd16c1a 100644 --- a/mlir/dialects/BUILD.bazel +++ b/mlir/dialects/BUILD.bazel @@ -4,14 +4,23 @@ zig_library( name = "dialects", tags = ["manual"], srcs = [ + "affine.zig", + "arith.zig", + "cf.zig", "func.zig", + "math.zig", + "memref.zig", + "scf.zig", + "vector.zig", ], import_name = "mlir/dialects", main = "dialects.zig", visibility = ["//visibility:public"], deps = [ "//mlir", + "//mlir/dialects/mosaic_tpu", "//mlir/dialects/stablehlo", + "//mlir/dialects/ttir", "//stdx", "@shardy//shardy/integrations/c:sdy_capi", ], diff --git a/mlir/dialects/affine.zig b/mlir/dialects/affine.zig new file mode 100644 index 00000000..685248f7 --- /dev/null +++ b/mlir/dialects/affine.zig @@ -0,0 +1,414 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +// ============================================================================= +// affine.apply / affine.min / affine.max +// ============================================================================= + +/// affine.apply — apply a 1-result affine map to a list of dim+symbol +/// operands. Result is `index`. +pub fn apply( + ctx: *mlir.Context, + map: *const mlir.AffineMap, + map_operands: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "affine.apply", .{ + .operands = .{ .flat = map_operands }, + .results = .{ .flat = &.{mlir.indexType(ctx)} }, + .attributes = &.{ + .named(ctx, "map", mlir.affineMapAttribute(map)), + }, + .location = location, + }); +} + +/// affine.min — minimum of all results of `map` evaluated at `map_operands`. +pub fn min( + ctx: *mlir.Context, + map: *const mlir.AffineMap, + map_operands: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "affine.min", .{ + .operands = .{ .flat = map_operands }, + .results = .{ .flat = &.{mlir.indexType(ctx)} }, + .attributes = &.{ + .named(ctx, "map", mlir.affineMapAttribute(map)), + }, + .location = location, + }); +} + +/// affine.max — maximum of all results of `map`. +pub fn max( + ctx: *mlir.Context, + map: *const mlir.AffineMap, + map_operands: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "affine.max", .{ + .operands = .{ .flat = map_operands }, + .results = .{ .flat = &.{mlir.indexType(ctx)} }, + .attributes = &.{ + .named(ctx, "map", mlir.affineMapAttribute(map)), + }, + .location = location, + }); +} + +// ============================================================================= +// affine.for / affine.yield +// ============================================================================= + +pub const ForArgs = struct { + /// Constant lower bound. Pass `null` when supplying a `lower_bound_map`. + lower_bound: ?i64 = null, + /// Constant upper bound. Pass `null` when supplying an `upper_bound_map`. + upper_bound: ?i64 = null, + /// Optional map-based bounds. When non-null they take precedence over the + /// constant counterpart. + lower_bound_map: ?*const mlir.AffineMap = null, + upper_bound_map: ?*const mlir.AffineMap = null, + /// Operands feeding `lower_bound_map` (in dim+symbol order). + lower_bound_operands: []const *const mlir.Value = &.{}, + /// Operands feeding `upper_bound_map`. + upper_bound_operands: []const *const mlir.Value = &.{}, + /// Loop step (positive, defaults to 1). + step: i64 = 1, + /// Loop-carried initial values. + inits: []const *const mlir.Value = &.{}, +}; + +/// affine.for — `for %i = lower to upper step S iter_args(...)`. The body +/// block takes `index, *init_arg_types` and must terminate with `affine.yield`. +pub fn for_( + ctx: *mlir.Context, + args: ForArgs, + body: *mlir.Block, + location: *const mlir.Location, +) *mlir.Operation { + // Resolve final maps. + const lb_map = args.lower_bound_map orelse blk: { + const cst = mlir.AffineExpr.constant(ctx, args.lower_bound orelse 0); + break :blk mlir.AffineMap.get(ctx, 0, 0, &.{cst}); + }; + const ub_map = args.upper_bound_map orelse blk: { + const cst = mlir.AffineExpr.constant(ctx, args.upper_bound orelse 0); + break :blk mlir.AffineMap.get(ctx, 0, 0, &.{cst}); + }; + + var operands: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + operands.appendSliceAssumeCapacity(args.lower_bound_operands); + operands.appendSliceAssumeCapacity(args.upper_bound_operands); + operands.appendSliceAssumeCapacity(args.inits); + + const seg_sizes = [3]i32{ + @intCast(args.lower_bound_operands.len), + @intCast(args.upper_bound_operands.len), + @intCast(args.inits.len), + }; + + var result_types: stdx.BoundedArray(*const mlir.Type, 64) = .empty; + for (args.inits) |v| result_types.appendAssumeCapacity(v.type_()); + + return mlir.Operation.make(ctx, "affine.for", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = result_types.constSlice() }, + .blocks = &.{body}, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), + .named(ctx, "lowerBoundMap", mlir.affineMapAttribute(lb_map)), + .named(ctx, "upperBoundMap", mlir.affineMapAttribute(ub_map)), + .named(ctx, "step", mlir.integerAttribute(ctx, .i64, args.step)), + }, + // The for body may reference outer SSA values that only become + // reachable once it's appended to its parent — defer to module-level + // verify (same trick used by scf.for). + .verify = false, + .location = location, + }); +} + +/// affine.yield — terminator for affine.for / affine.if / affine.parallel. +pub fn yield(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "affine.yield", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +// ============================================================================= +// affine.if +// ============================================================================= + +/// affine.if — `if condition[ ... ] { then } else { else }`. `condition` is an +/// IntegerSetAttribute (a conjunction of affine constraints). The result types +/// are inferred from the yielded values; pass `&.{}` for the no-result form. +pub fn if_( + ctx: *mlir.Context, + condition: *const mlir.IntegerSet, + operands: []const *const mlir.Value, + result_types: []const *const mlir.Type, + then_block: *mlir.Block, + else_block: ?*mlir.Block, + location: *const mlir.Location, +) *mlir.Operation { + var state: mlir.OperationState = .init("affine.if", location); + state.addOperands(operands); + state.addResults(result_types); + state.addAttributes(&.{ + .named(ctx, "condition", mlir.integerSetAttribute(condition)), + }); + + const then_region = mlir.Region.init(); + then_region.appendOwnedBlock(then_block); + const else_region = mlir.Region.init(); + if (else_block) |b| else_region.appendOwnedBlock(b); + state.addOwnedRegions(&.{ then_region, else_region }); + + return mlir.Operation.init(&state) catch @panic("Failed to create affine.if"); +} + +// ============================================================================= +// affine.load / affine.store / affine.vector_load / affine.vector_store +// ============================================================================= + +/// affine.load — load with affine indexing. +pub fn load( + ctx: *mlir.Context, + memref: *const mlir.Value, + map: *const mlir.AffineMap, + map_operands: []const *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + operands_buf.appendAssumeCapacity(memref); + operands_buf.appendSliceAssumeCapacity(map_operands); + + return mlir.Operation.make(ctx, "affine.load", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "map", mlir.affineMapAttribute(map)), + }, + .location = location, + }); +} + +pub fn store( + ctx: *mlir.Context, + value: *const mlir.Value, + memref: *const mlir.Value, + map: *const mlir.AffineMap, + map_operands: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 17) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ value, memref }); + operands_buf.appendSliceAssumeCapacity(map_operands); + + return mlir.Operation.make(ctx, "affine.store", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = &.{ + .named(ctx, "map", mlir.affineMapAttribute(map)), + }, + .location = location, + }); +} + +pub fn vector_load( + ctx: *mlir.Context, + memref: *const mlir.Value, + map: *const mlir.AffineMap, + map_operands: []const *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + operands_buf.appendAssumeCapacity(memref); + operands_buf.appendSliceAssumeCapacity(map_operands); + + return mlir.Operation.make(ctx, "affine.vector_load", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "map", mlir.affineMapAttribute(map)), + }, + .location = location, + }); +} + +pub fn vector_store( + ctx: *mlir.Context, + value: *const mlir.Value, + memref: *const mlir.Value, + map: *const mlir.AffineMap, + map_operands: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 17) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ value, memref }); + operands_buf.appendSliceAssumeCapacity(map_operands); + + return mlir.Operation.make(ctx, "affine.vector_store", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = &.{ + .named(ctx, "map", mlir.affineMapAttribute(map)), + }, + .location = location, + }); +} + +// ============================================================================= +// affine.parallel +// ============================================================================= + +pub const ParallelArgs = struct { + /// Per-loop-dim reductions, one entry per result. Use the same kinds as + /// `mlir.dialects.memref.AtomicRMWKind` (the dialect uses + /// `arith::AtomicRMWKind`). Pass `&.{}` for parallel loops with no + /// loop-carried reductions. + reductions: []const i64 = &.{}, + /// Map producing all lower-bound values; a single dim's bound is the + /// max of the results in its `lower_bounds_groups[i]` group. + lower_bounds_map: *const mlir.AffineMap, + /// One i32 per loop dim — number of map results that contribute to the + /// lower bound of that dim. Sum must equal `lower_bounds_map.numResults()`. + lower_bounds_groups: []const i32, + upper_bounds_map: *const mlir.AffineMap, + upper_bounds_groups: []const i32, + /// Per-dim integer steps (default 1). + steps: []const i64, + /// Operands feeding both bound maps. + map_operands: []const *const mlir.Value = &.{}, +}; + +/// affine.parallel — multi-dim parallel band. `body_block` takes one `index` +/// argument per loop dim and must terminate with `affine.yield`. +pub fn parallel( + ctx: *mlir.Context, + args: ParallelArgs, + body_block: *mlir.Block, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + // I32ElementsAttr — dense<[...]> : tensor. + const lb_groups_ty = mlir.RankedTensorType.get( + &.{@intCast(args.lower_bounds_groups.len)}, + mlir.integerType(ctx, .i32), + null, + ).shaped(); + const ub_groups_ty = mlir.RankedTensorType.get( + &.{@intCast(args.upper_bounds_groups.len)}, + mlir.integerType(ctx, .i32), + null, + ).shaped(); + + // I64SmallVectorArrayAttr — built as ArrayAttr>. + var steps_attrs: stdx.BoundedArray(*const mlir.Attribute, 16) = .empty; + for (args.steps) |s| { + steps_attrs.appendAssumeCapacity(mlir.integerAttribute(ctx, .i64, s)); + } + + // Reductions are AtomicRMWKindAttr (I64EnumAttr). We let callers pass + // raw int values to avoid coupling against the enum table. + var reductions_attrs: stdx.BoundedArray(*const mlir.Attribute, 16) = .empty; + for (args.reductions) |k| { + reductions_attrs.appendAssumeCapacity(mlir.integerAttribute(ctx, .i64, k)); + } + + return mlir.Operation.make(ctx, "affine.parallel", .{ + .operands = .{ .flat = args.map_operands }, + .results = .{ .flat = result_types }, + .blocks = &.{body_block}, + .attributes = &.{ + .named(ctx, "reductions", mlir.arrayAttribute(ctx, reductions_attrs.constSlice())), + .named(ctx, "lowerBoundsMap", mlir.affineMapAttribute(args.lower_bounds_map)), + .named(ctx, "lowerBoundsGroups", mlir.denseElementsAttribute(lb_groups_ty, args.lower_bounds_groups)), + .named(ctx, "upperBoundsMap", mlir.affineMapAttribute(args.upper_bounds_map)), + .named(ctx, "upperBoundsGroups", mlir.denseElementsAttribute(ub_groups_ty, args.upper_bounds_groups)), + .named(ctx, "steps", mlir.arrayAttribute(ctx, steps_attrs.constSlice())), + }, + .verify = false, + .location = location, + }); +} + +// ============================================================================= +// affine.delinearize_index / affine.linearize_index +// ============================================================================= + +pub const DelinearizeArgs = struct { + static_basis: []const i64, + dynamic_basis: []const *const mlir.Value = &.{}, +}; + +pub fn delinearize_index( + ctx: *mlir.Context, + linear_index: *const mlir.Value, + args: DelinearizeArgs, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + operands_buf.appendAssumeCapacity(linear_index); + operands_buf.appendSliceAssumeCapacity(args.dynamic_basis); + + return mlir.Operation.make(ctx, "affine.delinearize_index", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = result_types }, + .attributes = &.{ + .named(ctx, "static_basis", mlir.denseArrayAttribute(ctx, .i64, args.static_basis)), + }, + .location = location, + }); +} + +pub const LinearizeArgs = struct { + multi_index: []const *const mlir.Value, + static_basis: []const i64, + dynamic_basis: []const *const mlir.Value = &.{}, + disjoint: bool = false, +}; + +pub fn linearize_index( + ctx: *mlir.Context, + args: LinearizeArgs, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 32) = .empty; + operands_buf.appendSliceAssumeCapacity(args.multi_index); + operands_buf.appendSliceAssumeCapacity(args.dynamic_basis); + + const seg_sizes = [2]i32{ + @intCast(args.multi_index.len), + @intCast(args.dynamic_basis.len), + }; + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), + .named(ctx, "static_basis", mlir.denseArrayAttribute(ctx, .i64, args.static_basis)), + }); + if (args.disjoint) { + attrs.appendAssumeCapacity(.named(ctx, "disjoint", mlir.unitAttribute(ctx))); + } + + return mlir.Operation.make(ctx, "affine.linearize_index", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/mlir/dialects/arith.zig b/mlir/dialects/arith.zig new file mode 100644 index 00000000..e1e96026 --- /dev/null +++ b/mlir/dialects/arith.zig @@ -0,0 +1,408 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +// ============================================================================= +// Predicate enums (match ArithBase.td exactly) +// ============================================================================= + +/// Arith cmpi predicates. Values match mlir::arith::CmpIPredicate (I64EnumAttr). +pub const CmpIPredicate = enum(i64) { + eq = 0, + ne = 1, + slt = 2, + sle = 3, + sgt = 4, + sge = 5, + ult = 6, + ule = 7, + ugt = 8, + uge = 9, +}; + +/// Arith cmpf predicates. Values match mlir::arith::CmpFPredicate. +pub const CmpFPredicate = enum(i64) { + always_false = 0, + oeq = 1, + ogt = 2, + oge = 3, + olt = 4, + ole = 5, + one = 6, + ord = 7, + ueq = 8, + ugt = 9, + uge = 10, + ult = 11, + ule = 12, + une = 13, + uno = 14, + always_true = 15, +}; + +// ============================================================================= +// Constants +// ============================================================================= + +/// arith.constant for an integer scalar. `value` is reinterpreted to `int_type`'s bit width. +pub fn constant_int( + ctx: *mlir.Context, + value: i64, + int_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + const attr: *const mlir.Attribute = @ptrCast(c.mlirIntegerAttrGet(int_type.ptr(), value).ptr); + return mlir.Operation.make(ctx, "arith.constant", .{ + .results = .{ .flat = &.{int_type} }, + .attributes = &.{.named(ctx, "value", attr)}, + .location = location, + }); +} + +/// arith.constant for a float scalar of the given FloatTypes kind. +pub fn constant_float( + ctx: *mlir.Context, + value: f64, + comptime ft: mlir.FloatTypes, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.constant", .{ + .results = .{ .flat = &.{mlir.floatType(ctx, ft)} }, + .attributes = &.{.named(ctx, "value", mlir.floatAttribute(ctx, ft, value))}, + .location = location, + }); +} + +/// arith.constant for an `index`-typed integer. +pub fn constant_index(ctx: *mlir.Context, value: i64, location: *const mlir.Location) *mlir.Operation { + const idx_ty = mlir.indexType(ctx); + const attr: *const mlir.Attribute = @ptrCast(c.mlirIntegerAttrGet(idx_ty.ptr(), value).ptr); + return mlir.Operation.make(ctx, "arith.constant", .{ + .results = .{ .flat = &.{idx_ty} }, + .attributes = &.{.named(ctx, "value", attr)}, + .location = location, + }); +} + +// ============================================================================= +// Binary ops (generated from shared template) +// ============================================================================= + +fn binaryOp(comptime op_name: []const u8) type { + return struct { + pub fn call( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + location: *const mlir.Location, + ) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); + } + }; +} + +// Integer arithmetic +pub const addi = binaryOp("arith.addi").call; +pub const subi = binaryOp("arith.subi").call; +pub const muli = binaryOp("arith.muli").call; +pub const divsi = binaryOp("arith.divsi").call; +pub const divui = binaryOp("arith.divui").call; +pub const remsi = binaryOp("arith.remsi").call; +pub const remui = binaryOp("arith.remui").call; + +// Integer bitwise / shifts +pub const andi = binaryOp("arith.andi").call; +pub const ori = binaryOp("arith.ori").call; +pub const xori = binaryOp("arith.xori").call; +pub const shli = binaryOp("arith.shli").call; +pub const shrsi = binaryOp("arith.shrsi").call; +pub const shrui = binaryOp("arith.shrui").call; + +// Integer min/max +pub const maxsi = binaryOp("arith.maxsi").call; +pub const maxui = binaryOp("arith.maxui").call; +pub const minsi = binaryOp("arith.minsi").call; +pub const minui = binaryOp("arith.minui").call; + +// Ceil/floor div +pub const ceildivsi = binaryOp("arith.ceildivsi").call; +pub const ceildivui = binaryOp("arith.ceildivui").call; +pub const floordivsi = binaryOp("arith.floordivsi").call; + +// Float arithmetic +pub const addf = binaryOp("arith.addf").call; +pub const subf = binaryOp("arith.subf").call; +pub const mulf = binaryOp("arith.mulf").call; +pub const divf = binaryOp("arith.divf").call; +pub const remf = binaryOp("arith.remf").call; + +// Float min/max +pub const maximumf = binaryOp("arith.maximumf").call; +pub const maxnumf = binaryOp("arith.maxnumf").call; +pub const minimumf = binaryOp("arith.minimumf").call; +pub const minnumf = binaryOp("arith.minnumf").call; + +// ============================================================================= +// Unary ops +// ============================================================================= + +pub fn negf(ctx: *mlir.Context, src: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.negf", .{ + .operands = .{ .flat = &.{src} }, + .result_type_inference = true, + .location = location, + }); +} + +/// arith.flush_denormals — clamp subnormals to zero (SameOperandsAndResultType). +pub fn flush_denormals(ctx: *mlir.Context, src: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.flush_denormals", .{ + .operands = .{ .flat = &.{src} }, + .result_type_inference = true, + .location = location, + }); +} + +/// arith.constant for a dense splat: builds `arith.constant dense : T` +/// where `T` is a ranked tensor type. Useful for tensor-typed zero/one constants. +/// For float splats, use `constant_dense_splat_f`; for ints, `constant_dense_splat_i`. +pub fn constant_dense_splat_f( + ctx: *mlir.Context, + value: f64, + tensor_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + const elem_c = c.mlirShapedTypeGetElementType(tensor_type.ptr()); + const elem_attr = c.mlirFloatAttrDoubleGet(ctx.ptr(), elem_c, value); + const dense_c = c.mlirDenseElementsAttrSplatGet(tensor_type.ptr(), elem_attr); + const dense_attr: *const mlir.Attribute = @ptrCast(dense_c.ptr); + return mlir.Operation.make(ctx, "arith.constant", .{ + .results = .{ .flat = &.{tensor_type} }, + .attributes = &.{.named(ctx, "value", dense_attr)}, + .location = location, + }); +} + +pub fn constant_dense_splat_i( + ctx: *mlir.Context, + value: i64, + tensor_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + const elem_c = c.mlirShapedTypeGetElementType(tensor_type.ptr()); + const elem_attr = c.mlirIntegerAttrGet(elem_c, value); + const dense_c = c.mlirDenseElementsAttrSplatGet(tensor_type.ptr(), elem_attr); + const dense_attr: *const mlir.Attribute = @ptrCast(dense_c.ptr); + return mlir.Operation.make(ctx, "arith.constant", .{ + .results = .{ .flat = &.{tensor_type} }, + .attributes = &.{.named(ctx, "value", dense_attr)}, + .location = location, + }); +} + +// ============================================================================= +// Extended-result binary ops — return two values +// ============================================================================= + +/// arith.addui_extended — unsigned add, returns (sum, overflow: i1 / i1-tensor). +/// The overflow type is typically i1 (scalar) or tensor<...xi1> matching lhs shape. +pub fn addui_extended( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + overflow_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.addui_extended", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{ lhs.type_(), overflow_type } }, + .location = location, + }); +} + +/// arith.mulsi_extended — signed multiply, returns (low, high) both same type as lhs. +pub fn mulsi_extended( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.mulsi_extended", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{ lhs.type_(), lhs.type_() } }, + .location = location, + }); +} + +/// arith.mului_extended — unsigned multiply, returns (low, high) both same type as lhs. +pub fn mului_extended( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.mului_extended", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{ lhs.type_(), lhs.type_() } }, + .location = location, + }); +} + +// ============================================================================= +// Cast ops (1 operand, explicit result type) +// ============================================================================= + +fn castOp(comptime op_name: []const u8) type { + return struct { + pub fn call( + ctx: *mlir.Context, + value: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, + ) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{value} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); + } + }; +} + +pub const extsi = castOp("arith.extsi").call; +pub const extui = castOp("arith.extui").call; +pub const extf = castOp("arith.extf").call; +pub const trunci = castOp("arith.trunci").call; +pub const truncf = castOp("arith.truncf").call; +pub const sitofp = castOp("arith.sitofp").call; +pub const uitofp = castOp("arith.uitofp").call; +pub const fptosi = castOp("arith.fptosi").call; +pub const fptoui = castOp("arith.fptoui").call; +pub const bitcast = castOp("arith.bitcast").call; +pub const index_cast = castOp("arith.index_cast").call; +pub const index_castui = castOp("arith.index_castui").call; + +// ============================================================================= +// convertf — same-bitwidth float cast (e.g. f16↔bf16). Not expressible via +// arith.extf or arith.truncf. +// ============================================================================= + +/// Matches `Arith_RoundingModeAttr` (I32EnumAttr; see arith_dialect.td). +pub const RoundingMode = enum(i32) { + to_nearest_even = 0, + downward = 1, + upward = 2, + toward_zero = 3, + to_nearest_away = 4, +}; + +pub const ConvertFOpts = struct { + rounding: ?RoundingMode = null, +}; + +pub fn convertf( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + opts: ConvertFOpts, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.rounding) |rm| { + attrs.appendAssumeCapacity(.named(ctx, "roundingmode", mlir.integerAttribute(ctx, .i32, @intFromEnum(rm)))); + } + return mlir.Operation.make(ctx, "arith.convertf", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// arith.scaling_truncf — OCP MXFP downcast with per-block scale factors. +pub fn scaling_truncf( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + result_type: *const mlir.Type, + opts: ConvertFOpts, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.rounding) |rm| { + attrs.appendAssumeCapacity(.named(ctx, "roundingmode", mlir.integerAttribute(ctx, .i32, @intFromEnum(rm)))); + } + return mlir.Operation.make(ctx, "arith.scaling_truncf", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +// ============================================================================= +// Compare +// ============================================================================= + +pub fn cmpi( + ctx: *mlir.Context, + predicate: CmpIPredicate, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.cmpi", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "predicate", mlir.integerAttribute(ctx, .i64, @intFromEnum(predicate))), + }, + .location = location, + }); +} + +pub fn cmpf( + ctx: *mlir.Context, + predicate: CmpFPredicate, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.cmpf", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "predicate", mlir.integerAttribute(ctx, .i64, @intFromEnum(predicate))), + }, + .location = location, + }); +} + +// ============================================================================= +// Select +// ============================================================================= + +/// arith.select — result type matches the true/false operand type. +pub fn select( + ctx: *mlir.Context, + cond: *const mlir.Value, + t: *const mlir.Value, + f: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.select", .{ + .operands = .{ .flat = &.{ cond, t, f } }, + .results = .{ .flat = &.{t.type_()} }, + .location = location, + }); +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/mlir/dialects/cf.zig b/mlir/dialects/cf.zig new file mode 100644 index 00000000..4e75dcce --- /dev/null +++ b/mlir/dialects/cf.zig @@ -0,0 +1,166 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +// ============================================================================= +// cf.assert +// ============================================================================= + +/// cf.assert %cond, "message" — aborts execution with `msg` when `cond` is +/// false. `cond` is i1. +pub fn assert( + ctx: *mlir.Context, + cond: *const mlir.Value, + msg: []const u8, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cf.assert", .{ + .operands = .{ .flat = &.{cond} }, + .attributes = &.{ + .named(ctx, "msg", mlir.stringAttribute(ctx, msg)), + }, + .location = location, + }); +} + +// ============================================================================= +// cf.br +// ============================================================================= + +/// cf.br ^dest(operands...) — unconditional branch. Terminator of its block. +/// The types of `dest_operands` must match the block arguments of `dest`. +pub fn br( + ctx: *mlir.Context, + dest: *mlir.Block, + dest_operands: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + _ = ctx; + var state: mlir.OperationState = .init("cf.br", location); + state.addOperands(dest_operands); + var successors: [1]*const mlir.Block = .{dest}; + state.addSuccessors(&successors); + return mlir.Operation.init(&state) catch @panic("Failed to create cf.br"); +} + +// ============================================================================= +// cf.cond_br +// ============================================================================= + +/// cf.cond_br %cond, ^trueDest(trueOps...), ^falseDest(falseOps...) — branch +/// on an i1. Terminator of its block. +/// +/// Operand layout (flat) is `[cond, *trueOperands, *falseOperands]` with an +/// `operandSegmentSizes = [1, |trueOperands|, |falseOperands|]` attribute per +/// the `AttrSizedOperandSegments` trait on the op. +/// +/// `branch_weights` (optional) lets profile-guided lowering bias the branch; +/// when provided, its length must be 2 (true weight, false weight). +pub fn cond_br( + ctx: *mlir.Context, + cond: *const mlir.Value, + true_dest: *mlir.Block, + true_operands: []const *const mlir.Value, + false_dest: *mlir.Block, + false_operands: []const *const mlir.Value, + branch_weights: ?[]const i32, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 128) = .empty; + operands_buf.appendAssumeCapacity(cond); + operands_buf.appendSliceAssumeCapacity(true_operands); + operands_buf.appendSliceAssumeCapacity(false_operands); + + const seg_sizes = [3]i32{ 1, @intCast(true_operands.len), @intCast(false_operands.len) }; + + var state: mlir.OperationState = .init("cf.cond_br", location); + state.addOperands(operands_buf.constSlice()); + var successors: [2]*const mlir.Block = .{ true_dest, false_dest }; + state.addSuccessors(&successors); + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attrs.appendAssumeCapacity(.named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes))); + if (branch_weights) |w| { + std.debug.assert(w.len == 2); + attrs.appendAssumeCapacity(.named(ctx, "branch_weights", mlir.denseArrayAttribute(ctx, .i32, w))); + } + state.addAttributes(attrs.constSlice()); + + return mlir.Operation.init(&state) catch @panic("Failed to create cf.cond_br"); +} + +// ============================================================================= +// cf.switch +// ============================================================================= + +/// cf.switch %flag : iN, [ default: ^def(defOps...), caseN: ^bbN(opsN...) ] +/// +/// `flag` is any integer type. `case_values` are matched signed; the matching +/// case's block is jumped to (with the corresponding operand segment +/// forwarded). If no case matches, `default_dest` is taken with +/// `default_operands`. +/// +/// Operand layout: `[flag, *defaultOperands, *caseOperands[0], ..., +/// *caseOperands[N-1]]`. `case_operand_segments[i]` carries the length of +/// each case's operand segment. The `AttrSizedOperandSegments` trait tracks +/// the top-level split between `defaultOperands` and the lumped +/// `caseOperands`. +pub fn switch_( + ctx: *mlir.Context, + flag: *const mlir.Value, + default_dest: *mlir.Block, + default_operands: []const *const mlir.Value, + case_values: []const i32, + case_dests: []const *mlir.Block, + case_operands: []const []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + std.debug.assert(case_values.len == case_dests.len); + std.debug.assert(case_values.len == case_operands.len); + + var operands_buf: stdx.BoundedArray(*const mlir.Value, 256) = .empty; + operands_buf.appendAssumeCapacity(flag); + operands_buf.appendSliceAssumeCapacity(default_operands); + + var case_seg_sizes: stdx.BoundedArray(i32, 64) = .empty; + var case_total: i32 = 0; + for (case_operands) |segment| { + operands_buf.appendSliceAssumeCapacity(segment); + case_seg_sizes.appendAssumeCapacity(@intCast(segment.len)); + case_total += @intCast(segment.len); + } + + // Top-level AttrSizedOperandSegments: [flag=1, defaultOperands, caseOperands lumped]. + const top_seg_sizes = [3]i32{ 1, @intCast(default_operands.len), case_total }; + + var state: mlir.OperationState = .init("cf.switch", location); + state.addOperands(operands_buf.constSlice()); + + var successors_buf: stdx.BoundedArray(*const mlir.Block, 64) = .empty; + successors_buf.appendAssumeCapacity(default_dest); + for (case_dests) |d| successors_buf.appendAssumeCapacity(d); + state.addSuccessors(@constCast(successors_buf.constSlice())); + + // case_values is stored as a DenseIntElementsAttr (tensor); + // the td uses `AnyIntElementsAttr`, and the parser prints decimal literals + // matching the flag's bit width. We use i32 for the common path (matches + // Python Triton's flag type for pid compares). + const shaped = mlir.RankedTensorType.get( + &.{@intCast(case_values.len)}, + mlir.integerType(ctx, .i32), + null, + ).shaped(); + + state.addAttributes(&.{ + .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &top_seg_sizes)), + .named(ctx, "case_values", mlir.denseElementsAttribute(shaped, case_values)), + .named(ctx, "case_operand_segments", mlir.denseArrayAttribute(ctx, .i32, case_seg_sizes.constSlice())), + }); + + return mlir.Operation.init(&state) catch @panic("Failed to create cf.switch"); +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/mlir/dialects/dialects.zig b/mlir/dialects/dialects.zig index 4f4780d4..fa5e1245 100644 --- a/mlir/dialects/dialects.zig +++ b/mlir/dialects/dialects.zig @@ -1,8 +1,17 @@ const std = @import("std"); +pub const mosaic_tpu = @import("mlir/dialects/mosaic_tpu"); pub const stablehlo = @import("mlir/dialects/stablehlo"); +pub const ttir = @import("mlir/dialects/ttir"); +pub const affine = @import("affine.zig"); +pub const arith = @import("arith.zig"); +pub const cf = @import("cf.zig"); pub const func = @import("func.zig"); +pub const math = @import("math.zig"); +pub const memref = @import("memref.zig"); +pub const scf = @import("scf.zig"); +pub const vector = @import("vector.zig"); test { std.testing.refAllDecls(@This()); diff --git a/mlir/dialects/func.zig b/mlir/dialects/func.zig index e9246ec3..00b060d2 100644 --- a/mlir/dialects/func.zig +++ b/mlir/dialects/func.zig @@ -17,8 +17,9 @@ const FuncOpArgs = struct { block: *mlir.Block, location: *const mlir.Location, no_inline: bool = false, - visibility: Visibility = .public, + visibility: ?Visibility = .public, verify: bool = true, + extra_attributes: []const mlir.NamedAttribute = &.{}, }; pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { @@ -26,9 +27,11 @@ pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { var results_buffer: stdx.BoundedArray(*const mlir.Type, 32) = .empty; var attr_tuples_buffer: stdx.BoundedArray(mlir.NamedAttribute, 16) = .empty; + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "sym_name", mlir.stringAttribute(ctx, args.name))); + if (args.visibility) |v| { + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "sym_visibility", mlir.stringAttribute(ctx, @tagName(v)))); + } attr_tuples_buffer.appendSliceAssumeCapacity(&.{ - .named(ctx, "sym_name", mlir.stringAttribute(ctx, args.name)), - .named(ctx, "sym_visibility", mlir.stringAttribute(ctx, @tagName(args.visibility))), .named(ctx, "function_type", mlir.typeAttribute(mlir.functionType( ctx, args.args orelse args: { @@ -58,6 +61,9 @@ pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { if (args.no_inline) { attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "no_inline", mlir.unitAttribute(ctx))); } + for (args.extra_attributes) |a| { + attr_tuples_buffer.appendAssumeCapacity(a); + } return mlir.Operation.make(ctx, "func.func", .{ .blocks = &.{args.block}, diff --git a/mlir/dialects/math.zig b/mlir/dialects/math.zig new file mode 100644 index 00000000..39ce5666 --- /dev/null +++ b/mlir/dialects/math.zig @@ -0,0 +1,212 @@ +const std = @import("std"); + +const mlir = @import("mlir"); + +// ============================================================================= +// Float unary — SameOperandsAndResultType. Result type inferred from operand. +// ============================================================================= + +fn floatUnaryOp(comptime op_name: []const u8) type { + return struct { + pub fn call( + ctx: *mlir.Context, + src: *const mlir.Value, + location: *const mlir.Location, + ) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{src} }, + .result_type_inference = true, + .location = location, + }); + } + }; +} + +// Exponentials / logs +pub const exp = floatUnaryOp("math.exp").call; +pub const exp2 = floatUnaryOp("math.exp2").call; +pub const expm1 = floatUnaryOp("math.expm1").call; +pub const log = floatUnaryOp("math.log").call; +pub const log2 = floatUnaryOp("math.log2").call; +pub const log10 = floatUnaryOp("math.log10").call; +pub const log1p = floatUnaryOp("math.log1p").call; + +// Trig / hyperbolic +pub const sin = floatUnaryOp("math.sin").call; +pub const cos = floatUnaryOp("math.cos").call; +pub const tan = floatUnaryOp("math.tan").call; +pub const asin = floatUnaryOp("math.asin").call; +pub const acos = floatUnaryOp("math.acos").call; +pub const atan = floatUnaryOp("math.atan").call; +pub const sinh = floatUnaryOp("math.sinh").call; +pub const cosh = floatUnaryOp("math.cosh").call; +pub const tanh = floatUnaryOp("math.tanh").call; +pub const asinh = floatUnaryOp("math.asinh").call; +pub const acosh = floatUnaryOp("math.acosh").call; +pub const atanh = floatUnaryOp("math.atanh").call; + +// Roots / errors / abs +pub const sqrt = floatUnaryOp("math.sqrt").call; +pub const rsqrt = floatUnaryOp("math.rsqrt").call; +pub const cbrt = floatUnaryOp("math.cbrt").call; +pub const erf = floatUnaryOp("math.erf").call; +pub const erfc = floatUnaryOp("math.erfc").call; +pub const absf = floatUnaryOp("math.absf").call; + +// Rounding +pub const floor = floatUnaryOp("math.floor").call; +pub const ceil = floatUnaryOp("math.ceil").call; +pub const round = floatUnaryOp("math.round").call; +pub const roundeven = floatUnaryOp("math.roundeven").call; +pub const trunc = floatUnaryOp("math.trunc").call; + +// ============================================================================= +// Float binary — SameOperandsAndResultType. +// ============================================================================= + +fn floatBinaryOp(comptime op_name: []const u8) type { + return struct { + pub fn call( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + location: *const mlir.Location, + ) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); + } + }; +} + +pub const powf = floatBinaryOp("math.powf").call; +pub const atan2 = floatBinaryOp("math.atan2").call; +pub const copysign = floatBinaryOp("math.copysign").call; + +// ============================================================================= +// Float ternary — SameOperandsAndResultType. +// ============================================================================= + +pub fn fma( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "math.fma", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .result_type_inference = true, + .location = location, + }); +} + +/// math.clampf value, min, max — clamp to [min, max], same type throughout. +/// Distinct from `tt.clampf` (which has an explicit `propagateNan` attr); the +/// math version uses fastmath for NaN behaviour. +pub fn clampf( + ctx: *mlir.Context, + value: *const mlir.Value, + min: *const mlir.Value, + max: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "math.clampf", .{ + .operands = .{ .flat = &.{ value, min, max } }, + .result_type_inference = true, + .location = location, + }); +} + +// ============================================================================= +// Sincos — operand → (sin, cos), both same type as operand. +// ============================================================================= + +pub fn sincos( + ctx: *mlir.Context, + operand: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "math.sincos", .{ + .operands = .{ .flat = &.{operand} }, + .results = .{ .flat = &.{ operand.type_(), operand.type_() } }, + .location = location, + }); +} + +// ============================================================================= +// FPowI — float base, signless integer exponent → float (same type as base). +// ============================================================================= + +pub fn fpowi( + ctx: *mlir.Context, + base: *const mlir.Value, + power: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "math.fpowi", .{ + .operands = .{ .flat = &.{ base, power } }, + .results = .{ .flat = &.{base.type_()} }, + .location = location, + }); +} + +// ============================================================================= +// Integer unary +// ============================================================================= + +fn intUnaryOp(comptime op_name: []const u8) type { + return struct { + pub fn call( + ctx: *mlir.Context, + src: *const mlir.Value, + location: *const mlir.Location, + ) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{src} }, + .result_type_inference = true, + .location = location, + }); + } + }; +} + +pub const absi = intUnaryOp("math.absi").call; +pub const ctlz = intUnaryOp("math.ctlz").call; +pub const cttz = intUnaryOp("math.cttz").call; +pub const ctpop = intUnaryOp("math.ctpop").call; + +/// math.ipowi(base, exp) — integer power. Result type matches base. +pub const ipowi = floatBinaryOp("math.ipowi").call; + +// ============================================================================= +// Float classification — operand is float-like, result is i1-like. +// ============================================================================= + +fn classifyOp(comptime op_name: []const u8) type { + return struct { + pub fn call( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, + ) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); + } + }; +} + +pub const isfinite = classifyOp("math.isfinite").call; +pub const isinf = classifyOp("math.isinf").call; +pub const isnan = classifyOp("math.isnan").call; +pub const isnormal = classifyOp("math.isnormal").call; + +test { + std.testing.refAllDecls(@This()); +} diff --git a/mlir/dialects/memref.zig b/mlir/dialects/memref.zig new file mode 100644 index 00000000..42ff2237 --- /dev/null +++ b/mlir/dialects/memref.zig @@ -0,0 +1,665 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +/// Sentinel matching `mlir::ShapedType::kDynamic` (== INT64_MIN). +/// Use this in static-size/offset/stride arrays to mark a slot as dynamic. +pub const kDynamic: i64 = std.math.minInt(i64); + +// ============================================================================= +// Allocation — alloc / alloca / dealloc / realloc +// ============================================================================= + +pub const AllocOpts = struct { + alignment: ?i64 = null, +}; + +/// memref.alloc — heap allocation. `dynamic_sizes` provides one Index value per +/// dynamic dim of `result_type` (left-to-right). `symbol_operands` binds to +/// the symbols of the layout map (rare; pass &.{} when none). +pub fn alloc( + ctx: *mlir.Context, + result_type: *const mlir.Type, + dynamic_sizes: []const *const mlir.Value, + symbol_operands: []const *const mlir.Value, + opts: AllocOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + operands_buf.appendSliceAssumeCapacity(dynamic_sizes); + operands_buf.appendSliceAssumeCapacity(symbol_operands); + + const seg_sizes = [2]i32{ @intCast(dynamic_sizes.len), @intCast(symbol_operands.len) }; + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attrs.appendAssumeCapacity(.named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes))); + if (opts.alignment) |a| { + attrs.appendAssumeCapacity(.named(ctx, "alignment", mlir.integerAttribute(ctx, .i64, a))); + } + + return mlir.Operation.make(ctx, "memref.alloc", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// memref.alloca — stack allocation; same operand/attribute layout as alloc. +pub fn alloca( + ctx: *mlir.Context, + result_type: *const mlir.Type, + dynamic_sizes: []const *const mlir.Value, + symbol_operands: []const *const mlir.Value, + opts: AllocOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + operands_buf.appendSliceAssumeCapacity(dynamic_sizes); + operands_buf.appendSliceAssumeCapacity(symbol_operands); + + const seg_sizes = [2]i32{ @intCast(dynamic_sizes.len), @intCast(symbol_operands.len) }; + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attrs.appendAssumeCapacity(.named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes))); + if (opts.alignment) |a| { + attrs.appendAssumeCapacity(.named(ctx, "alignment", mlir.integerAttribute(ctx, .i64, a))); + } + + return mlir.Operation.make(ctx, "memref.alloca", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub fn dealloc( + ctx: *mlir.Context, + memref: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.dealloc", .{ + .operands = .{ .flat = &.{memref} }, + .location = location, + }); +} + +/// memref.realloc — grow/shrink an existing allocation. `dynamic_size` is +/// optional; pass null to keep the size encoded in `result_type`. +pub fn realloc( + ctx: *mlir.Context, + src: *const mlir.Value, + dynamic_size: ?*const mlir.Value, + result_type: *const mlir.Type, + opts: AllocOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 2) = .empty; + operands_buf.appendAssumeCapacity(src); + if (dynamic_size) |s| operands_buf.appendAssumeCapacity(s); + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.alignment) |a| { + attrs.appendAssumeCapacity(.named(ctx, "alignment", mlir.integerAttribute(ctx, .i64, a))); + } + + return mlir.Operation.make(ctx, "memref.realloc", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +// ============================================================================= +// Load / Store — plain (non-affine) memref load/store +// ============================================================================= + +pub const LoadOpts = struct { + /// Hint that the lowering may use a non-temporal load. + nontemporal: bool = false, +}; + +/// memref.load — load a single element from a memref. +pub fn load( + ctx: *mlir.Context, + memref: *const mlir.Value, + indices: []const *const mlir.Value, + opts: LoadOpts, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendAssumeCapacity(memref); + buf.appendSliceAssumeCapacity(indices); + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.nontemporal) { + attrs.appendAssumeCapacity(.named(ctx, "nontemporal", mlir.boolAttribute(ctx, true))); + } + + return mlir.Operation.make(ctx, "memref.load", .{ + .operands = .{ .flat = buf.constSlice() }, + .result_type_inference = true, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub const StoreOpts = struct { + nontemporal: bool = false, +}; + +/// memref.store — store a single element into a memref. +pub fn store( + ctx: *mlir.Context, + value: *const mlir.Value, + memref: *const mlir.Value, + indices: []const *const mlir.Value, + opts: StoreOpts, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendSliceAssumeCapacity(&.{ value, memref }); + buf.appendSliceAssumeCapacity(indices); + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.nontemporal) { + attrs.appendAssumeCapacity(.named(ctx, "nontemporal", mlir.boolAttribute(ctx, true))); + } + + return mlir.Operation.make(ctx, "memref.store", .{ + .operands = .{ .flat = buf.constSlice() }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +// ============================================================================= +// Atomic RMW +// ============================================================================= + +/// Matches `mlir::arith::AtomicRMWKind` (I64EnumAttr) in upstream `Arith` td. +/// memref.atomic_rmw and affine.parallel both encode reductions through this. +pub const AtomicRMWKind = enum(i64) { + addf = 0, + addi = 1, + assign = 2, + maximumf = 3, + maxs = 4, + maxu = 5, + minimumf = 6, + mins = 7, + minu = 8, + mulf = 9, + muli = 10, + ori = 11, + andi = 12, + maxnumf = 13, + minnumf = 14, +}; + +/// memref.atomic_rmw — atomic read-modify-write. +pub fn atomic_rmw( + ctx: *mlir.Context, + kind: AtomicRMWKind, + value: *const mlir.Value, + memref: *const mlir.Value, + indices: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendSliceAssumeCapacity(&.{ value, memref }); + buf.appendSliceAssumeCapacity(indices); + + return mlir.Operation.make(ctx, "memref.atomic_rmw", .{ + .operands = .{ .flat = buf.constSlice() }, + .results = .{ .flat = &.{value.type_()} }, + .attributes = &.{ + .named(ctx, "kind", mlir.integerAttribute(ctx, .i64, @intFromEnum(kind))), + }, + .location = location, + }); +} + +// ============================================================================= +// Casts / metadata-only ops +// ============================================================================= + +/// memref.cast — change layout/element-shape compatibility (e.g. unranked ↔ +/// ranked, dynamic ↔ static). Element type and total memory must match. +pub fn cast( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.cast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// memref.memory_space_cast — change the memory-space attribute on a memref. +pub fn memory_space_cast( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.memory_space_cast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// memref.assume_alignment — pure, returns the same memref with an alignment +/// hint attached. +pub fn assume_alignment( + ctx: *mlir.Context, + src: *const mlir.Value, + alignment: i32, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.assume_alignment", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{src.type_()} }, + .attributes = &.{ + .named(ctx, "alignment", mlir.integerAttribute(ctx, .i32, alignment)), + }, + .location = location, + }); +} + +// ============================================================================= +// Shape introspection +// ============================================================================= + +/// memref.dim — returns the size of dim `index` of `memref` as `index`. +pub fn dim( + ctx: *mlir.Context, + memref: *const mlir.Value, + index: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.dim", .{ + .operands = .{ .flat = &.{ memref, index } }, + .results = .{ .flat = &.{mlir.indexType(ctx)} }, + .location = location, + }); +} + +/// memref.rank — returns rank of `memref` as `index`. +pub fn rank( + ctx: *mlir.Context, + memref: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.rank", .{ + .operands = .{ .flat = &.{memref} }, + .results = .{ .flat = &.{mlir.indexType(ctx)} }, + .location = location, + }); +} + +/// memref.extract_aligned_pointer_as_index — get the aligned base pointer of +/// a memref as an `index`. +pub fn extract_aligned_pointer_as_index( + ctx: *mlir.Context, + src: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.extract_aligned_pointer_as_index", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{mlir.indexType(ctx)} }, + .location = location, + }); +} + +// ============================================================================= +// View / SubView / Reinterpret +// ============================================================================= + +/// memref.view — type-changing view of a flat 1-D `memref` as a higher- +/// rank memref. `byte_shift` is an `index` byte offset; `dynamic_sizes` +/// supplies one operand per dynamic dim of `result_type`. +pub fn view( + ctx: *mlir.Context, + source: *const mlir.Value, + byte_shift: *const mlir.Value, + dynamic_sizes: []const *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendSliceAssumeCapacity(&.{ source, byte_shift }); + buf.appendSliceAssumeCapacity(dynamic_sizes); + + return mlir.Operation.make(ctx, "memref.view", .{ + .operands = .{ .flat = buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// Mixed static/dynamic offsets-sizes-strides — `static_*` slots holding +/// `kDynamic` consume one entry from the corresponding `dynamic_*` slice. +pub const SubViewArgs = struct { + static_offsets: []const i64, + static_sizes: []const i64, + static_strides: []const i64, + dynamic_offsets: []const *const mlir.Value = &.{}, + dynamic_sizes: []const *const mlir.Value = &.{}, + dynamic_strides: []const *const mlir.Value = &.{}, +}; + +/// memref.subview — strided view into a source memref. The result type must +/// be precomputed by the caller (the dialect provides `inferResultType`, +/// but we don't expose that yet — pass an explicit `result_type`). +pub fn subview( + ctx: *mlir.Context, + source: *const mlir.Value, + args: SubViewArgs, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + operands_buf.appendAssumeCapacity(source); + operands_buf.appendSliceAssumeCapacity(args.dynamic_offsets); + operands_buf.appendSliceAssumeCapacity(args.dynamic_sizes); + operands_buf.appendSliceAssumeCapacity(args.dynamic_strides); + + const seg_sizes = [4]i32{ + 1, + @intCast(args.dynamic_offsets.len), + @intCast(args.dynamic_sizes.len), + @intCast(args.dynamic_strides.len), + }; + + return mlir.Operation.make(ctx, "memref.subview", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), + .named(ctx, "static_offsets", mlir.denseArrayAttribute(ctx, .i64, args.static_offsets)), + .named(ctx, "static_sizes", mlir.denseArrayAttribute(ctx, .i64, args.static_sizes)), + .named(ctx, "static_strides", mlir.denseArrayAttribute(ctx, .i64, args.static_strides)), + }, + .location = location, + }); +} + +pub const ReinterpretCastArgs = struct { + static_offsets: []const i64, + static_sizes: []const i64, + static_strides: []const i64, + dynamic_offsets: []const *const mlir.Value = &.{}, + dynamic_sizes: []const *const mlir.Value = &.{}, + dynamic_strides: []const *const mlir.Value = &.{}, +}; + +/// memref.reinterpret_cast — changes offset/sizes/strides of a memref without +/// touching the underlying memory. +pub fn reinterpret_cast( + ctx: *mlir.Context, + source: *const mlir.Value, + args: ReinterpretCastArgs, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + operands_buf.appendAssumeCapacity(source); + operands_buf.appendSliceAssumeCapacity(args.dynamic_offsets); + operands_buf.appendSliceAssumeCapacity(args.dynamic_sizes); + operands_buf.appendSliceAssumeCapacity(args.dynamic_strides); + + const seg_sizes = [4]i32{ + 1, + @intCast(args.dynamic_offsets.len), + @intCast(args.dynamic_sizes.len), + @intCast(args.dynamic_strides.len), + }; + + return mlir.Operation.make(ctx, "memref.reinterpret_cast", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), + .named(ctx, "static_offsets", mlir.denseArrayAttribute(ctx, .i64, args.static_offsets)), + .named(ctx, "static_sizes", mlir.denseArrayAttribute(ctx, .i64, args.static_sizes)), + .named(ctx, "static_strides", mlir.denseArrayAttribute(ctx, .i64, args.static_strides)), + }, + .location = location, + }); +} + +// ============================================================================= +// Reshape — expand_shape / collapse_shape / reshape +// ============================================================================= + +/// memref.reshape — reshape a memref using a separate `shape` memref of `index` +/// elements. `result_type` carries the new shape. +pub fn reshape( + ctx: *mlir.Context, + source: *const mlir.Value, + shape: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.reshape", .{ + .operands = .{ .flat = &.{ source, shape } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `reassociation[i]` lists the source dimensions that map into result dim i +/// (for collapse_shape) or the result dimensions that come from source dim i +/// (for expand_shape). Each inner slice is stored as an `IntegerAttr` array. +pub const ReassociationArgs = struct { + /// Index groupings. + reassociation: []const []const i64, + /// For expand_shape only: per-result-dim sizes, with `kDynamic` for slots + /// resolved from `dynamic_output_shape`. Pass &.{} for collapse_shape. + static_output_shape: []const i64 = &.{}, + /// For expand_shape only: the dynamic dim values referenced by `kDynamic` + /// entries in `static_output_shape`. + dynamic_output_shape: []const *const mlir.Value = &.{}, +}; + +fn buildReassociationAttr(ctx: *mlir.Context, reassociation: []const []const i64) *const mlir.Attribute { + var outer: stdx.BoundedArray(*const mlir.Attribute, 32) = .empty; + var inner_buf: stdx.BoundedArray(*const mlir.Attribute, 64) = .empty; + for (reassociation) |group| { + const start = inner_buf.len; + for (group) |idx| { + inner_buf.appendAssumeCapacity(mlir.integerAttribute(ctx, .i64, idx)); + } + const inner_attr = mlir.arrayAttribute(ctx, inner_buf.constSlice()[start..]); + outer.appendAssumeCapacity(inner_attr); + } + return mlir.arrayAttribute(ctx, outer.constSlice()); +} + +/// memref.expand_shape — split source dimensions into multiple result +/// dimensions. `static_output_shape` must have one entry per result dim; +/// dynamic entries are filled from `dynamic_output_shape` in order. +pub fn expand_shape( + ctx: *mlir.Context, + source: *const mlir.Value, + args: ReassociationArgs, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 32) = .empty; + operands_buf.appendAssumeCapacity(source); + operands_buf.appendSliceAssumeCapacity(args.dynamic_output_shape); + + return mlir.Operation.make(ctx, "memref.expand_shape", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "reassociation", buildReassociationAttr(ctx, args.reassociation)), + .named(ctx, "static_output_shape", mlir.denseArrayAttribute(ctx, .i64, args.static_output_shape)), + }, + .location = location, + }); +} + +/// memref.collapse_shape — merge consecutive source dimensions into single +/// result dimensions. Reassociation map is canonical. +pub fn collapse_shape( + ctx: *mlir.Context, + source: *const mlir.Value, + reassociation: []const []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.collapse_shape", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "reassociation", buildReassociationAttr(ctx, reassociation)), + }, + .location = location, + }); +} + +// ============================================================================= +// Transpose — metadata-only permutation. Permutation is an AffineMapAttr. +// ============================================================================= + +/// memref.transpose — permute the dimensions of `src` according to +/// `permutation` (which must be a permutation of `(d0, ..., dN-1)`). Build +/// the map with `mlir.AffineMap.get` or parse via +/// `mlir.parseAffineMapAttribute`. +pub fn transpose( + ctx: *mlir.Context, + src: *const mlir.Value, + permutation: *const mlir.Attribute, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.transpose", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "permutation", permutation), + }, + .location = location, + }); +} + +// ============================================================================= +// Copy / Prefetch +// ============================================================================= + +/// memref.copy — copy source memref into target memref (same element type, +/// same total element count). +pub fn copy( + ctx: *mlir.Context, + source: *const mlir.Value, + target: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.copy", .{ + .operands = .{ .flat = &.{ source, target } }, + .location = location, + }); +} + +pub const PrefetchOpts = struct { + /// True for a write prefetch, false for a read prefetch. + is_write: bool = false, + /// 0 = no locality, 3 = extreme locality (keep in cache). + locality_hint: i32 = 3, + /// True for data cache, false for instruction cache. + is_data_cache: bool = true, +}; + +/// memref.prefetch — non-binding hint to bring `memref[indices]` into cache. +pub fn prefetch( + ctx: *mlir.Context, + memref: *const mlir.Value, + indices: []const *const mlir.Value, + opts: PrefetchOpts, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendAssumeCapacity(memref); + buf.appendSliceAssumeCapacity(indices); + + return mlir.Operation.make(ctx, "memref.prefetch", .{ + .operands = .{ .flat = buf.constSlice() }, + .attributes = &.{ + .named(ctx, "isWrite", mlir.boolAttribute(ctx, opts.is_write)), + .named(ctx, "localityHint", mlir.integerAttribute(ctx, .i32, opts.locality_hint)), + .named(ctx, "isDataCache", mlir.boolAttribute(ctx, opts.is_data_cache)), + }, + .location = location, + }); +} + +// ============================================================================= +// Globals +// ============================================================================= + +pub const GlobalArgs = struct { + pub const Visibility = enum { public, private, nested }; + + name: []const u8, + type_: *const mlir.Type, + /// Optional dense initial value. `null` ⇒ uninitialised global. + initial_value: ?*const mlir.Attribute = null, + visibility: Visibility = .public, + constant: bool = false, + alignment: ?i64 = null, +}; + +/// memref.global — declare a module-scope memref global (matches the symbol +/// referenced by `memref.get_global`). +pub fn global(ctx: *mlir.Context, args: GlobalArgs, location: *const mlir.Location) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "sym_name", mlir.stringAttribute(ctx, args.name)), + .named(ctx, "sym_visibility", mlir.stringAttribute(ctx, @tagName(args.visibility))), + .named(ctx, "type", mlir.typeAttribute(args.type_)), + }); + if (args.initial_value) |iv| { + attrs.appendAssumeCapacity(.named(ctx, "initial_value", iv)); + } + if (args.constant) { + attrs.appendAssumeCapacity(.named(ctx, "constant", mlir.unitAttribute(ctx))); + } + if (args.alignment) |a| { + attrs.appendAssumeCapacity(.named(ctx, "alignment", mlir.integerAttribute(ctx, .i64, a))); + } + + return mlir.Operation.make(ctx, "memref.global", .{ + .attributes = attrs.constSlice(), + .verify = false, + .location = location, + }); +} + +/// memref.get_global — fetch the value of a `memref.global` by symbol name. +pub fn get_global( + ctx: *mlir.Context, + name: []const u8, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.get_global", .{ + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "name", mlir.flatSymbolRefAttribute(ctx, name)), + }, + .location = location, + }); +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/mlir/dialects/mosaic_tpu/BUILD.bazel b/mlir/dialects/mosaic_tpu/BUILD.bazel new file mode 100644 index 00000000..5ae15776 --- /dev/null +++ b/mlir/dialects/mosaic_tpu/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "mosaic_tpu", + import_name = "mlir/dialects/mosaic_tpu", + main = "mosaic_tpu.zig", + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "//mlir:c", + "//stdx", + "@mosaic_tpu//:tpu_dialect_capi", + ], +) + +zig_test( + name = "test", + deps = [":mosaic_tpu"], +) diff --git a/mlir/dialects/mosaic_tpu/mosaic_tpu.zig b/mlir/dialects/mosaic_tpu/mosaic_tpu.zig new file mode 100644 index 00000000..c437f234 --- /dev/null +++ b/mlir/dialects/mosaic_tpu/mosaic_tpu.zig @@ -0,0 +1,1178 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +/// Latest Mosaic-serde format version. Pallas's `tpu_custom_call.py:CustomCallBackendConfig.to_json` +/// surfaces this through `serialization_format`. Bumped in lockstep with +/// `jaxlib/mosaic/dialect/tpu/transforms/serde.h:kVersion`. +pub const SERDE_VERSION: i32 = 11; + +/// Register the `mosaic-serde` pass so `PassManager.parse` can pick it up by +/// name. Idempotent — safe to call repeatedly. Required before running the +/// `mosaic-serde{serialize=true}` pipeline that produces the bytecode embedded +/// in `tpu_custom_call`'s backend_config. +pub fn registerMosaicSerdePass() void { + c.mlirTpuRegisterMosaicSerdePass(); +} + +// ============================================================================= +// Types — !tpu.semaphore, !tpu.dma_semaphore, !tpu.float8_exmy<...> +// ============================================================================= + +/// `!tpu.semaphore` element type. +pub fn semaphoreType(ctx: *mlir.Context) *const mlir.Type { + return mlir.Type.parse(ctx, "!tpu.semaphore") catch + @panic("failed to parse !tpu.semaphore — is the tpu dialect registered?"); +} + +/// Build `!tpu.dma_semaphore`. +pub fn dmaSemaphoreType(ctx: *mlir.Context) *const mlir.Type { + return mlir.Type.parse(ctx, "!tpu.dma_semaphore") catch + @panic("failed to parse !tpu.dma_semaphore — is the tpu dialect registered?"); +} + +/// `!tpu.float8_exmy` — Mosaic Float8 type. +pub const Float8EXMYType = opaque { + const M = mlir.Methods(Float8EXMYType, c.MlirType); + + pub const isAFn = c.mlirTpuIsAFloat8EXMYType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn get(ctx: *mlir.Context, underlying: *const mlir.Type) *const Float8EXMYType { + return @ptrCast(c.mlirTpuFloat8EXMYTypeGet(ctx.ptr(), underlying.ptr()).ptr); + } + + pub fn underlyingType(self: *const Float8EXMYType) *const mlir.Type { + return @ptrCast(c.mlirTpuFloat8EXMYTypeGetUnderlyingType(self.ptr()).ptr); + } +}; + +pub fn float8ExmyType(ctx: *mlir.Context, underlying: *const mlir.Type) *const mlir.Type { + return @ptrCast(Float8EXMYType.get(ctx, underlying)); +} + +// ============================================================================= +// Enum / dialect-attribute helpers — parsed from their textual form. +// ============================================================================= + +/// Reduction kind for `tpu.all_reduce`, `tpu.reduce_index`, `tpu.scan`. +pub const ReductionKind = enum { + sum, + max, + min, + @"and", + @"or", + xor, + + pub fn attribute(self: ReductionKind, ctx: *mlir.Context) *const mlir.Attribute { + var buf: [64]u8 = undefined; + const text = std.fmt.bufPrint(&buf, "#tpu.reduction_kind<{s}>", .{@tagName(self)}) catch unreachable; + return mlir.Attribute.parse(ctx, text) catch + std.debug.panic("failed to parse tpu.reduction_kind '{s}'", .{text}); + } +}; + +/// `tpu.contract_precision` — matmul precision setting. +pub const ContractPrecision = enum { + bf16, + fp32, + + pub fn attribute(self: ContractPrecision, ctx: *mlir.Context) *const mlir.Attribute { + const text = switch (self) { + .bf16 => "#tpu.contract_precision", + .fp32 => "#tpu.contract_precision", + }; + return mlir.Attribute.parse(ctx, text) catch @panic("contract_precision parse"); + } +}; + +/// `tpu.rounding_mode` — used by tpu.fptosi / tpu.fptoui / etc. +pub const RoundingMode = enum { + to_nearest_even, + to_nearest_away, + upward, + downward, + toward_zero, + + pub fn attribute(self: RoundingMode, ctx: *mlir.Context) *const mlir.Attribute { + var buf: [64]u8 = undefined; + const text = std.fmt.bufPrint(&buf, "#tpu.rounding_mode<{s}>", .{@tagName(self)}) catch unreachable; + return mlir.Attribute.parse(ctx, text) catch + std.debug.panic("failed to parse tpu.rounding_mode '{s}'", .{text}); + } +}; + +/// `#tpu.memory_space<...>` — passed as the `memory_space` of a memref's type. +pub const MemorySpace = enum { + vmem, + smem, + cmem, + hbm, + semaphore_mem, + vmem_shared, + any, + + pub fn attribute(self: MemorySpace, ctx: *mlir.Context) *const mlir.Attribute { + var buf: [64]u8 = undefined; + const text = std.fmt.bufPrint(&buf, "#tpu.memory_space<{s}>", .{@tagName(self)}) catch unreachable; + return mlir.Attribute.parse(ctx, text) catch + std.debug.panic("failed to parse tpu.memory_space '{s}'", .{text}); + } +}; + +/// `#tpu.core_type<...>` — used by remote DMAs / sem_signal target hints. +pub const CoreType = enum { + tc, + sc_scalar_subcore, + sc_vector_subcore, + + pub fn attribute(self: CoreType, ctx: *mlir.Context) *const mlir.Attribute { + var buf: [64]u8 = undefined; + const text = std.fmt.bufPrint(&buf, "#tpu.core_type<{s}>", .{@tagName(self)}) catch unreachable; + return mlir.Attribute.parse(ctx, text) catch + std.debug.panic("failed to parse tpu.core_type '{s}'", .{text}); + } +}; + +/// `#tpu.dimension_semantics<...>` — grid dimension semantics annotation. +pub const DimensionSemantics = enum { + parallel, + arbitrary, + sequential, + + pub fn attribute(self: DimensionSemantics, ctx: *mlir.Context) *const mlir.Attribute { + var buf: [64]u8 = undefined; + const text = std.fmt.bufPrint(&buf, "#tpu.dimension_semantics<{s}>", .{@tagName(self)}) catch unreachable; + return mlir.Attribute.parse(ctx, text) catch + std.debug.panic("failed to parse tpu.dimension_semantics '{s}'", .{text}); + } +}; + +/// `#tpu.pipeline_mode<...>` — double-buffering mode in `window_params`. +pub const PipelineMode = enum { + synchronous, + double_buffered, + + pub fn attribute(self: PipelineMode, ctx: *mlir.Context) *const mlir.Attribute { + var buf: [64]u8 = undefined; + const text = std.fmt.bufPrint(&buf, "#tpu.pipeline_mode<{s}>", .{@tagName(self)}) catch unreachable; + return mlir.Attribute.parse(ctx, text) catch + std.debug.panic("failed to parse tpu.pipeline_mode '{s}'", .{text}); + } +}; + +/// `#tpu.revisit_mode<...>` — controls block revisit for multi-pass outputs. +pub const RevisitMode = enum { + immediate, + any, + + pub fn attribute(self: RevisitMode, ctx: *mlir.Context) *const mlir.Attribute { + var buf: [64]u8 = undefined; + const text = std.fmt.bufPrint(&buf, "#tpu.revisit_mode<{s}>", .{@tagName(self)}) catch unreachable; + return mlir.Attribute.parse(ctx, text) catch + std.debug.panic("failed to parse tpu.revisit_mode '{s}'", .{text}); + } +}; + +/// `#tpu.element_window<[pad_low], [pad_high]>` — per-arg window padding. +pub fn elementWindowAttribute( + ctx: *mlir.Context, + pad_low: []const i64, + pad_high: []const i64, +) *const mlir.Attribute { + var buf: [256]u8 = undefined; + var len: usize = 0; + len += (std.fmt.bufPrint(buf[len..], "#tpu.element_window<[", .{}) catch unreachable).len; + for (pad_low, 0..) |p, i| { + const sep = if (i == 0) "" else ", "; + len += (std.fmt.bufPrint(buf[len..], "{s}{d}", .{ sep, p }) catch unreachable).len; + } + len += (std.fmt.bufPrint(buf[len..], "], [", .{}) catch unreachable).len; + for (pad_high, 0..) |p, i| { + const sep = if (i == 0) "" else ", "; + len += (std.fmt.bufPrint(buf[len..], "{s}{d}", .{ sep, p }) catch unreachable).len; + } + len += (std.fmt.bufPrint(buf[len..], "]>", .{}) catch unreachable).len; + const text = buf[0..len]; + return mlir.Attribute.parse(ctx, text) catch + std.debug.panic("failed to parse tpu.element_window '{s}'", .{text}); +} + +// ============================================================================= +// Reductions / scan / sort +// ============================================================================= + +pub fn all_reduce( + ctx: *mlir.Context, + input: *const mlir.Value, + dim: i64, + kind: ReductionKind, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.all_reduce", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "dim", mlir.integerAttribute(ctx, .i64, dim)), + .named(ctx, "kind", kind.attribute(ctx)), + }, + .location = location, + }); +} + +pub fn reduce_index( + ctx: *mlir.Context, + input: *const mlir.Value, + axis: i32, + kind: ReductionKind, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.reduce_index", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "axis", mlir.integerAttribute(ctx, .i32, axis)), + .named(ctx, "kind", kind.attribute(ctx)), + }, + .location = location, + }); +} + +pub fn scan( + ctx: *mlir.Context, + input: *const mlir.Value, + kind: ReductionKind, + mask: ?*const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 2) = .empty; + operands_buf.appendAssumeCapacity(input); + if (mask) |m| operands_buf.appendAssumeCapacity(m); + + return mlir.Operation.make(ctx, "tpu.scan", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "kind", kind.attribute(ctx)), + }, + .location = location, + }); +} + +pub const SortOpts = struct { + descending: bool = false, +}; + +pub fn sort( + ctx: *mlir.Context, + keys: *const mlir.Value, + values: *const mlir.Value, + mask: ?*const mlir.Value, + opts: SortOpts, + output_mask_type: *const mlir.Type, + sorted_keys_type: *const mlir.Type, + sorted_values_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 3) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ keys, values }); + if (mask) |m| operands_buf.appendAssumeCapacity(m); + + return mlir.Operation.make(ctx, "tpu.sort", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{ output_mask_type, sorted_keys_type, sorted_values_type } }, + .attributes = &.{ + .named(ctx, "descending", mlir.boolAttribute(ctx, opts.descending)), + }, + .location = location, + }); +} + +// ============================================================================= +// Memory — load / store / vector_load / vector_store / strided_* +// ============================================================================= + +pub const LoadOpts = struct { + sublane_mask: []const bool, + sublane_stride: i32 = 1, +}; + +/// tpu.load — sublane-aware load into a vreg. +pub fn load( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + opts: LoadOpts, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + + return mlir.Operation.make(ctx, "tpu.load", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "sublane_mask", denseBoolArrayAttribute(ctx, opts.sublane_mask)), + .named(ctx, "sublane_stride", mlir.integerAttribute(ctx, .i32, opts.sublane_stride)), + }, + .location = location, + }); +} + +pub const StoreOpts = struct { + sublane_mask: []const bool, + sublane_stride: i32 = 1, + add: bool = false, +}; + +/// tpu.store — sublane-aware store from a vreg. Optional elementwise mask. +pub fn store( + ctx: *mlir.Context, + value: *const mlir.Value, + base: *const mlir.Value, + indices: []const *const mlir.Value, + mask: ?*const mlir.Value, + opts: StoreOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ value, base }); + operands_buf.appendSliceAssumeCapacity(indices); + const mask_len: i32 = if (mask) |m| blk: { + operands_buf.appendAssumeCapacity(m); + break :blk 1; + } else 0; + + const seg_sizes = [4]i32{ 1, 1, @intCast(indices.len), mask_len }; + return mlir.Operation.make(ctx, "tpu.store", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), + .named(ctx, "sublane_mask", denseBoolArrayAttribute(ctx, opts.sublane_mask)), + .named(ctx, "sublane_stride", mlir.integerAttribute(ctx, .i32, opts.sublane_stride)), + .named(ctx, "add", mlir.boolAttribute(ctx, opts.add)), + }, + .location = location, + }); +} + +pub const VectorLoadOpts = struct { + /// Per-dim load stride. Pass `&.{}` for unit-strided. + strides: []const i32 = &.{}, +}; + +/// tpu.vector_load — multi-dim strided load with optional elementwise mask. +pub fn vector_load( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + mask: ?*const mlir.Value, + opts: VectorLoadOpts, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 17) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + const mask_len: i32 = if (mask) |m| blk: { + operands_buf.appendAssumeCapacity(m); + break :blk 1; + } else 0; + + const seg_sizes = [3]i32{ 1, @intCast(indices.len), mask_len }; + return mlir.Operation.make(ctx, "tpu.vector_load", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), + .named(ctx, "strides", mlir.denseArrayAttribute(ctx, .i32, opts.strides)), + }, + .location = location, + }); +} + +pub const VectorStoreOpts = struct { + strides: []const i32 = &.{}, + add: bool = false, +}; + +pub fn vector_store( + ctx: *mlir.Context, + value: *const mlir.Value, + base: *const mlir.Value, + indices: []const *const mlir.Value, + mask: ?*const mlir.Value, + opts: VectorStoreOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ value, base }); + operands_buf.appendSliceAssumeCapacity(indices); + const mask_len: i32 = if (mask) |m| blk: { + operands_buf.appendAssumeCapacity(m); + break :blk 1; + } else 0; + + const seg_sizes = [4]i32{ 1, 1, @intCast(indices.len), mask_len }; + return mlir.Operation.make(ctx, "tpu.vector_store", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), + .named(ctx, "strides", mlir.denseArrayAttribute(ctx, .i32, opts.strides)), + .named(ctx, "add", mlir.boolAttribute(ctx, opts.add)), + }, + .location = location, + }); +} + +pub fn strided_load( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + strides: []const i32, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + + return mlir.Operation.make(ctx, "tpu.strided_load", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "strides", mlir.denseArrayAttribute(ctx, .i32, strides)), + }, + .location = location, + }); +} + +pub fn strided_store( + ctx: *mlir.Context, + value: *const mlir.Value, + base: *const mlir.Value, + indices: []const *const mlir.Value, + strides: []const i32, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 17) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ value, base }); + operands_buf.appendSliceAssumeCapacity(indices); + + return mlir.Operation.make(ctx, "tpu.strided_store", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = &.{ + .named(ctx, "strides", mlir.denseArrayAttribute(ctx, .i32, strides)), + }, + .location = location, + }); +} + +// ============================================================================= +// Compute — matmul, iota, reciprocal, casts +// ============================================================================= + +pub const MatmulOpts = struct { + /// Deprecated when `dimension_numbers` is provided. + transpose_lhs: bool = false, + transpose_rhs: bool = false, + /// Optional precision; pass null to omit. + precision: ?ContractPrecision = null, + /// Optional `#tpu.dot_dimension_numbers<...>` attribute. When omitted the + /// canonicalizer derives one. Build with `mlir.Attribute.parse`. + dimension_numbers: ?*const mlir.Attribute = null, +}; + +pub fn matmul( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + acc: *const mlir.Value, + opts: MatmulOpts, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "transpose_lhs", mlir.boolAttribute(ctx, opts.transpose_lhs)), + .named(ctx, "transpose_rhs", mlir.boolAttribute(ctx, opts.transpose_rhs)), + }); + if (opts.precision) |p| { + attrs.appendAssumeCapacity(.named(ctx, "precision", p.attribute(ctx))); + } + if (opts.dimension_numbers) |dn| { + attrs.appendAssumeCapacity(.named(ctx, "dimension_numbers", dn)); + } + + return mlir.Operation.make(ctx, "tpu.matmul", .{ + .operands = .{ .flat = &.{ lhs, rhs, acc } }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub fn iota( + ctx: *mlir.Context, + dimensions: ?[]const i32, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (dimensions) |d| { + attrs.appendAssumeCapacity(.named(ctx, "dimensions", mlir.denseArrayAttribute(ctx, .i32, d))); + } + return mlir.Operation.make(ctx, "tpu.iota", .{ + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub const ReciprocalOpts = struct { + approx: bool = false, + full_range: bool = false, +}; + +pub fn reciprocal( + ctx: *mlir.Context, + input: *const mlir.Value, + opts: ReciprocalOpts, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.reciprocal", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{input.type_()} }, + .attributes = &.{ + .named(ctx, "approx", mlir.boolAttribute(ctx, opts.approx)), + .named(ctx, "full_range", mlir.boolAttribute(ctx, opts.full_range)), + }, + .location = location, + }); +} + +/// Generic 1-operand → 1-result builder for the various TPU casts. +fn castOp(comptime op_name: []const u8) type { + return struct { + pub fn call( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, + ) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); + } + }; +} + +pub const fptosi = castOp("tpu.fptosi").call; +pub const fptoui = castOp("tpu.fptoui").call; +pub const sitofp = castOp("tpu.sitofp").call; +pub const uitofp = castOp("tpu.uitofp").call; +pub const extf = castOp("tpu.extf").call; +pub const truncf = castOp("tpu.truncf").call; +pub const bitcast = castOp("tpu.bitcast").call; +pub const bitcast_vreg = castOp("tpu.bitcast_vreg").call; +pub const mask_cast = castOp("tpu.mask_cast").call; +pub const relayout = castOp("tpu.relayout").call; + +// ============================================================================= +// Shape — reshape / repeat / concatenate / transpose / broadcast_in_sublanes +// ============================================================================= + +pub fn reshape( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.reshape", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn repeat( + ctx: *mlir.Context, + src: *const mlir.Value, + dimension: i32, + times: i32, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.repeat", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "dimension", mlir.integerAttribute(ctx, .i32, dimension)), + .named(ctx, "times", mlir.integerAttribute(ctx, .i32, times)), + }, + .location = location, + }); +} + +pub fn concatenate( + ctx: *mlir.Context, + sources: []const *const mlir.Value, + dimension: i32, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.concatenate", .{ + .operands = .{ .flat = sources }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "dimension", mlir.integerAttribute(ctx, .i32, dimension)), + }, + .location = location, + }); +} + +pub fn transpose( + ctx: *mlir.Context, + src: *const mlir.Value, + permutation: []const i32, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.transpose", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "permutation", mlir.denseArrayAttribute(ctx, .i32, permutation)), + }, + .location = location, + }); +} + +pub fn broadcast_in_sublanes( + ctx: *mlir.Context, + src: *const mlir.Value, + lane: i32, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.broadcast_in_sublanes", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "lane", mlir.integerAttribute(ctx, .i32, lane)), + }, + .location = location, + }); +} + +pub const RotateOpts = struct { + amount: i32, + dimension: i32, + stride: ?i32 = null, + stride_dimension: ?i32 = null, +}; + +pub fn rotate( + ctx: *mlir.Context, + value: *const mlir.Value, + opts: RotateOpts, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "amount", mlir.integerAttribute(ctx, .i32, opts.amount)), + .named(ctx, "dimension", mlir.integerAttribute(ctx, .i32, opts.dimension)), + }); + if (opts.stride) |s| attrs.appendAssumeCapacity(.named(ctx, "stride", mlir.integerAttribute(ctx, .i32, s))); + if (opts.stride_dimension) |sd| attrs.appendAssumeCapacity(.named(ctx, "stride_dimension", mlir.integerAttribute(ctx, .i32, sd))); + + return mlir.Operation.make(ctx, "tpu.rotate", .{ + .operands = .{ .flat = &.{value} }, + .results = .{ .flat = &.{value.type_()} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +// ============================================================================= +// Memref-shaped ops — slice / squeeze / reshape / bitcast / reinterpret_cast +// ============================================================================= + +pub const MemRefSliceArgs = struct { + /// `base_idx` count must equal source memref rank. + base_idx: []const *const mlir.Value, + /// Optional dynamic sizes; left empty when target shape is fully static. + dynamic_sizes: []const *const mlir.Value = &.{}, +}; + +pub fn memref_slice( + ctx: *mlir.Context, + mem_ref: *const mlir.Value, + args: MemRefSliceArgs, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 32) = .empty; + operands_buf.appendAssumeCapacity(mem_ref); + operands_buf.appendSliceAssumeCapacity(args.base_idx); + operands_buf.appendSliceAssumeCapacity(args.dynamic_sizes); + + const seg_sizes = [3]i32{ 1, @intCast(args.base_idx.len), @intCast(args.dynamic_sizes.len) }; + return mlir.Operation.make(ctx, "tpu.memref_slice", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), + }, + .location = location, + }); +} + +pub fn memref_squeeze( + ctx: *mlir.Context, + mem_ref: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.memref_squeeze", .{ + .operands = .{ .flat = &.{mem_ref} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn memref_reshape( + ctx: *mlir.Context, + mem_ref: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.memref_reshape", .{ + .operands = .{ .flat = &.{mem_ref} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn memref_bitcast( + ctx: *mlir.Context, + mem_ref: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.memref_bitcast", .{ + .operands = .{ .flat = &.{mem_ref} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn reinterpret_cast( + ctx: *mlir.Context, + mem_ref: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.reinterpret_cast", .{ + .operands = .{ .flat = &.{mem_ref} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn assume_layout( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.assume_layout", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn erase_memref_layout( + ctx: *mlir.Context, + mem_ref: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.erase_memref_layout", .{ + .operands = .{ .flat = &.{mem_ref} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +// ============================================================================= +// Mask building — create_mask / create_subelement_mask / mask_cast +// ============================================================================= + +/// `tpu.create_mask` — two equal-length operand groups (low, high) per dim. +pub fn create_mask( + ctx: *mlir.Context, + low_bounds: []const *const mlir.Value, + high_bounds: []const *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + std.debug.assert(low_bounds.len == high_bounds.len); + var operands_buf: stdx.BoundedArray(*const mlir.Value, 32) = .empty; + operands_buf.appendSliceAssumeCapacity(low_bounds); + operands_buf.appendSliceAssumeCapacity(high_bounds); + + return mlir.Operation.make(ctx, "tpu.create_mask", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +// ============================================================================= +// Semaphores / DMA / barriers +// ============================================================================= + +pub fn sem_alloc( + ctx: *mlir.Context, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.sem_alloc", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn sem_barrier( + ctx: *mlir.Context, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.sem_barrier", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn sem_read( + ctx: *mlir.Context, + semaphore: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.sem_read", .{ + .operands = .{ .flat = &.{semaphore} }, + .results = .{ .flat = &.{mlir.integerType(ctx, .i32)} }, + .location = location, + }); +} + +pub fn sem_wait( + ctx: *mlir.Context, + semaphore: *const mlir.Value, + amount: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.sem_wait", .{ + .operands = .{ .flat = &.{ semaphore, amount } }, + .location = location, + }); +} + +pub const SemSignalOpts = struct { + device_id: ?*const mlir.Value = null, + core_id: ?*const mlir.Value = null, +}; + +pub fn sem_signal( + ctx: *mlir.Context, + semaphore: *const mlir.Value, + amount: *const mlir.Value, + opts: SemSignalOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 4) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ semaphore, amount }); + const dev_len: i32 = if (opts.device_id) |d| blk: { + operands_buf.appendAssumeCapacity(d); + break :blk 1; + } else 0; + const core_len: i32 = if (opts.core_id) |c_| blk: { + operands_buf.appendAssumeCapacity(c_); + break :blk 1; + } else 0; + + const seg_sizes = [4]i32{ 1, 1, dev_len, core_len }; + return mlir.Operation.make(ctx, "tpu.sem_signal", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), + }, + .location = location, + }); +} + +pub fn barrier( + ctx: *mlir.Context, + barrier_id: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.barrier", .{ + .operands = .{ .flat = &.{barrier_id} }, + .location = location, + }); +} + +pub const EnqueueDmaOpts = struct { + source_semaphore: ?*const mlir.Value = null, + device_id: ?*const mlir.Value = null, + core_id: ?*const mlir.Value = null, + priority: i32 = 0, + strict_ordering: bool = false, +}; + +pub fn enqueue_dma( + ctx: *mlir.Context, + source: *const mlir.Value, + target: *const mlir.Value, + target_semaphore: *const mlir.Value, + opts: EnqueueDmaOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 6) = .empty; + operands_buf.appendAssumeCapacity(source); + const src_sem_len: i32 = if (opts.source_semaphore) |s| blk: { + operands_buf.appendAssumeCapacity(s); + break :blk 1; + } else 0; + operands_buf.appendAssumeCapacity(target); + operands_buf.appendAssumeCapacity(target_semaphore); + const dev_len: i32 = if (opts.device_id) |d| blk: { + operands_buf.appendAssumeCapacity(d); + break :blk 1; + } else 0; + const core_len: i32 = if (opts.core_id) |c_| blk: { + operands_buf.appendAssumeCapacity(c_); + break :blk 1; + } else 0; + + // Source / source_semaphore? / target / target_semaphore / device_id? / core_id? + const seg_sizes = [6]i32{ 1, src_sem_len, 1, 1, dev_len, core_len }; + return mlir.Operation.make(ctx, "tpu.enqueue_dma", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), + .named(ctx, "priority", mlir.integerAttribute(ctx, .i32, opts.priority)), + .named(ctx, "strict_ordering", mlir.boolAttribute(ctx, opts.strict_ordering)), + }, + .location = location, + }); +} + +pub const WaitDma2Opts = struct { + device_id: ?*const mlir.Value = null, + core_id: ?*const mlir.Value = null, + strict_ordering: bool = false, +}; + +pub fn wait_dma2( + ctx: *mlir.Context, + semaphore: *const mlir.Value, + src: *const mlir.Value, + dst: *const mlir.Value, + opts: WaitDma2Opts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 5) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ semaphore, src, dst }); + const dev_len: i32 = if (opts.device_id) |d| blk: { + operands_buf.appendAssumeCapacity(d); + break :blk 1; + } else 0; + const core_len: i32 = if (opts.core_id) |c_| blk: { + operands_buf.appendAssumeCapacity(c_); + break :blk 1; + } else 0; + + const seg_sizes = [5]i32{ 1, 1, 1, dev_len, core_len }; + return mlir.Operation.make(ctx, "tpu.wait_dma2", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), + .named(ctx, "strict_ordering", mlir.boolAttribute(ctx, opts.strict_ordering)), + }, + .location = location, + }); +} + +// ============================================================================= +// Region-bearing — region / trace / yield +// ============================================================================= + +/// tpu.region — single-block region with an implicit `tpu.yield` terminator. +pub fn region( + ctx: *mlir.Context, + body: *mlir.Block, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.region", .{ + .results = .{ .flat = result_types }, + .blocks = &.{body}, + .verify = false, + .location = location, + }); +} + +pub fn trace( + ctx: *mlir.Context, + body: *mlir.Block, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.trace", .{ + .results = .{ .flat = result_types }, + .blocks = &.{body}, + .verify = false, + .location = location, + }); +} + +pub fn yield( + ctx: *mlir.Context, + values: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.yield", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +pub fn trace_start( + ctx: *mlir.Context, + level: i32, + message: []const u8, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.trace_start", .{ + .attributes = &.{ + .named(ctx, "level", mlir.integerAttribute(ctx, .i32, level)), + .named(ctx, "message", mlir.stringAttribute(ctx, message)), + }, + .verify = false, + .location = location, + }); +} + +pub fn trace_stop(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.trace_stop", .{ + .verify = false, + .location = location, + }); +} + +/// Build a `#tpu.dot_dimension_numbers<...>` attribute. Mirrors +/// `tpu_ops.td:TPU_DotDimensionNumbersAttr`. Pass empty slices for the +/// fields you don't use (most dot operations leave batch dims empty). +pub fn dotDimensionNumbers( + ctx: *mlir.Context, + lhs_contracting: []const i64, + rhs_contracting: []const i64, + lhs_non_contracting: []const i64, + rhs_non_contracting: []const i64, + output_dim_order: []const i64, + lhs_batch: []const i64, + rhs_batch: []const i64, +) *const mlir.Attribute { + var buf: [512]u8 = undefined; + var len: usize = 0; + len += (std.fmt.bufPrint(buf[len..], "#tpu.dot_dimension_numbers<", .{}) catch unreachable).len; + inline for (.{ + lhs_contracting, + rhs_contracting, + lhs_non_contracting, + rhs_non_contracting, + output_dim_order, + lhs_batch, + rhs_batch, + }, 0..) |arr, i| { + if (i != 0) len += (std.fmt.bufPrint(buf[len..], ", ", .{}) catch unreachable).len; + len += (std.fmt.bufPrint(buf[len..], "[", .{}) catch unreachable).len; + for (arr, 0..) |v, j| { + const sep = if (j == 0) "" else ", "; + len += (std.fmt.bufPrint(buf[len..], "{s}{d}", .{ sep, v }) catch unreachable).len; + } + len += (std.fmt.bufPrint(buf[len..], "]", .{}) catch unreachable).len; + } + len += (std.fmt.bufPrint(buf[len..], ">", .{}) catch unreachable).len; + return mlir.Attribute.parse(ctx, buf[0..len]) catch + std.debug.panic("failed to parse tpu.dot_dimension_numbers '{s}'", .{buf[0..len]}); +} + +// ============================================================================= +// Misc — device_id / delay / assume_multiple +// ============================================================================= + +pub fn device_id(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.device_id", .{ + .results = .{ .flat = &.{mlir.integerType(ctx, .i32)} }, + .location = location, + }); +} + +pub fn delay( + ctx: *mlir.Context, + cycles: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.delay", .{ + .operands = .{ .flat = &.{cycles} }, + .location = location, + }); +} + +pub fn assume_multiple( + ctx: *mlir.Context, + src: *const mlir.Value, + multiple: i64, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.assume_multiple", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{src.type_()} }, + .attributes = &.{ + .named(ctx, "multiple", mlir.integerAttribute(ctx, .i64, multiple)), + }, + .location = location, + }); +} + +// ============================================================================= +// Internal helpers +// ============================================================================= + +/// `DenseBoolArrayAttr` is distinct from `DenseArrayAttr`; pack bools as i8. +fn denseBoolArrayAttribute(ctx: *mlir.Context, values: []const bool) *const mlir.Attribute { + var bytes: stdx.BoundedArray(i32, 64) = .empty; + for (values) |b| bytes.appendAssumeCapacity(@intFromBool(b)); + return mlir.denseArrayAttribute(ctx, .bool, bytes.constSlice()); +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/mlir/dialects/scf.zig b/mlir/dialects/scf.zig new file mode 100644 index 00000000..4e90e579 --- /dev/null +++ b/mlir/dialects/scf.zig @@ -0,0 +1,355 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub const ForOpts = struct { + /// Per td: `scf.for` uses signed comparison by default. Setting this emits + /// the `unsignedCmp` UnitAttr so the induction variable advances with + /// unsigned comparison instead. + unsigned_cmp: bool = false, +}; + +/// scf.for — operands are (lower, upper, step, init_args...) as a flat list. +/// Result arity equals init_args arity; result types must match init_args types. +/// +/// The body block must have arguments `[iv_type, *init_arg_types]` and be +/// terminated with `scf.yield`. +pub fn for_( + ctx: *mlir.Context, + lower: *const mlir.Value, + upper: *const mlir.Value, + step: *const mlir.Value, + init_args: []const *const mlir.Value, + body: *mlir.Block, + opts: ForOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + operands.appendSliceAssumeCapacity(&.{ lower, upper, step }); + operands.appendSliceAssumeCapacity(init_args); + + var result_types: stdx.BoundedArray(*const mlir.Type, 64) = .empty; + for (init_args) |v| result_types.appendAssumeCapacity(v.type_()); + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.unsigned_cmp) { + attrs.appendAssumeCapacity(.named(ctx, "unsignedCmp", mlir.unitAttribute(ctx))); + } + + return mlir.Operation.make(ctx, "scf.for", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = result_types.constSlice() }, + .blocks = &.{body}, + .attributes = attrs.constSlice(), + // Per-op verify fails with "operand #0 does not dominate this use" when + // the scf.for references outer SSA values that only become reachable + // once it's appended to the parent block. Defer to module-level verify. + .verify = false, + .location = location, + }); +} + +/// scf.if — cond is i1; then/else blocks must be terminated with `scf.yield` +/// (or implicitly for no-result forms). +/// +/// Built via raw OperationState because it has two separate regions. +pub fn if_( + ctx: *mlir.Context, + cond: *const mlir.Value, + result_types: []const *const mlir.Type, + then_block: *mlir.Block, + else_block: ?*mlir.Block, + location: *const mlir.Location, +) *mlir.Operation { + _ = ctx; + var state: mlir.OperationState = .init("scf.if", location); + state.addOperands(&.{cond}); + state.addResults(result_types); + + const then_region = mlir.Region.init(); + then_region.appendOwnedBlock(then_block); + const else_region = mlir.Region.init(); + if (else_block) |b| else_region.appendOwnedBlock(b); + state.addOwnedRegions(&.{ then_region, else_region }); + + return mlir.Operation.init(&state) catch @panic("Failed to create scf.if"); +} + +/// scf.yield — variadic result terminator for scf.for / scf.if regions. +pub fn yield(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "scf.yield", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +/// scf.while — two regions. The `before` region computes the condition and +/// terminates with `scf.condition`. The `after` region runs when the condition +/// is true and terminates with `scf.yield`. The arguments of `before` have the +/// types of `inits`. The trailing operands of `scf.condition` feed both the +/// `after` region's arguments and the `scf.while` op's results. +pub fn while_( + ctx: *mlir.Context, + inits: []const *const mlir.Value, + result_types: []const *const mlir.Type, + before_block: *mlir.Block, + after_block: *mlir.Block, + location: *const mlir.Location, +) *mlir.Operation { + _ = ctx; + var state: mlir.OperationState = .init("scf.while", location); + state.addOperands(inits); + state.addResults(result_types); + + const before_region = mlir.Region.init(); + before_region.appendOwnedBlock(before_block); + const after_region = mlir.Region.init(); + after_region.appendOwnedBlock(after_block); + state.addOwnedRegions(&.{ before_region, after_region }); + + return mlir.Operation.init(&state) catch @panic("Failed to create scf.while"); +} + +/// scf.condition — terminator of the `before` region of scf.while. +/// First operand is i1 (the continuation condition). The remaining operands +/// are forwarded to either the `after` region's arguments (if true) or the +/// scf.while's results (if false). +pub fn condition( + ctx: *mlir.Context, + cond: *const mlir.Value, + args: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + buf.appendAssumeCapacity(cond); + buf.appendSliceAssumeCapacity(args); + return mlir.Operation.make(ctx, "scf.condition", .{ + .operands = .{ .flat = buf.constSlice() }, + .verify = false, + .location = location, + }); +} + +pub const ExecuteRegionOpts = struct { + no_inline: bool = false, +}; + +/// scf.execute_region — runs `body` exactly once, returns the yielded values. +/// `body` has no arguments but may span multiple blocks (unlike for/if). +pub fn execute_region( + ctx: *mlir.Context, + result_types: []const *const mlir.Type, + body: *mlir.Block, + opts: ExecuteRegionOpts, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.no_inline) { + attrs.appendAssumeCapacity(.named(ctx, "no_inline", mlir.unitAttribute(ctx))); + } + return mlir.Operation.make(ctx, "scf.execute_region", .{ + .results = .{ .flat = result_types }, + .blocks = &.{body}, + .attributes = attrs.constSlice(), + // execute_region has its own region-verify; defer to the module-level + // verify so unresolved SSA uses don't block construction. + .verify = false, + .location = location, + }); +} + +/// scf.parallel — multi-dim parallel for loop with optional reductions. +/// Operands are (lbs, ubs, steps, inits). `body_block` takes one `index` arg +/// per loop dim and must be terminated with `scf.reduce` (or no-op if inits is +/// empty — the parser auto-inserts an empty reduce). +pub fn parallel( + ctx: *mlir.Context, + lbs: []const *const mlir.Value, + ubs: []const *const mlir.Value, + steps: []const *const mlir.Value, + inits: []const *const mlir.Value, + body_block: *mlir.Block, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + std.debug.assert(lbs.len == ubs.len); + std.debug.assert(lbs.len == steps.len); + + var operands: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + operands.appendSliceAssumeCapacity(lbs); + operands.appendSliceAssumeCapacity(ubs); + operands.appendSliceAssumeCapacity(steps); + operands.appendSliceAssumeCapacity(inits); + + const seg_sizes = [4]i32{ + @intCast(lbs.len), + @intCast(ubs.len), + @intCast(steps.len), + @intCast(inits.len), + }; + return mlir.Operation.make(ctx, "scf.parallel", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = result_types }, + .blocks = &.{body_block}, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), + }, + .verify = false, + .location = location, + }); +} + +/// scf.reduce — terminator of `scf.parallel`. Each region is a reduction body +/// taking two args (lhs, rhs) matching the corresponding operand's type, +/// terminated with `scf.reduce.return`. +pub fn reduce( + ctx: *mlir.Context, + operands: []const *const mlir.Value, + reduction_blocks: []const *mlir.Block, + location: *const mlir.Location, +) *mlir.Operation { + _ = ctx; + std.debug.assert(operands.len == reduction_blocks.len); + var state: mlir.OperationState = .init("scf.reduce", location); + state.addOperands(operands); + + var regions: stdx.BoundedArray(*mlir.Region, 16) = .empty; + for (reduction_blocks) |b| { + const r = mlir.Region.init(); + r.appendOwnedBlock(b); + regions.appendAssumeCapacity(r); + } + state.addOwnedRegions(regions.constSlice()); + + return mlir.Operation.init(&state) catch @panic("Failed to create scf.reduce"); +} + +pub fn reduce_return(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "scf.reduce.return", .{ + .operands = .{ .flat = &.{value} }, + .verify = false, + .location = location, + }); +} + +/// scf.forall — multi-dim parallel region, similar to `scf.parallel` but using +/// tensor-based `shared_outs` instead of init values. This wrapper builds the +/// fully-dynamic form (all lb/ub/step are runtime values). +/// +/// `body_block` must have `rank + shared_outs.len` arguments (one index per +/// loop dim, plus one per shared_out) and must be terminated with +/// `scf.forall.in_parallel`. +pub fn forall( + ctx: *mlir.Context, + lbs: []const *const mlir.Value, + ubs: []const *const mlir.Value, + steps: []const *const mlir.Value, + shared_outs: []const *const mlir.Value, + mapping: ?*const mlir.Attribute, + body_block: *mlir.Block, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + std.debug.assert(lbs.len == ubs.len); + std.debug.assert(lbs.len == steps.len); + + var operands: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + operands.appendSliceAssumeCapacity(lbs); + operands.appendSliceAssumeCapacity(ubs); + operands.appendSliceAssumeCapacity(steps); + operands.appendSliceAssumeCapacity(shared_outs); + + const seg_sizes = [4]i32{ + @intCast(lbs.len), + @intCast(ubs.len), + @intCast(steps.len), + @intCast(shared_outs.len), + }; + + // Fully-dynamic form: the static arrays use the kDynamic sentinel + // (mlir::ShapedType::kDynamic == INT64_MIN). We build one slot per loop + // dim, all dynamic. This matches how MLIR printers emit + // `scf.forall (%iv) = (%lb) to (%ub) step (%step)` when no static + // constants are folded in. + const k_dynamic: i64 = std.math.minInt(i64); + var sentinel: stdx.BoundedArray(i64, 16) = .empty; + for (0..lbs.len) |_| sentinel.appendAssumeCapacity(k_dynamic); + const sentinel_slice = sentinel.constSlice(); + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), + .named(ctx, "staticLowerBound", mlir.denseArrayAttribute(ctx, .i64, sentinel_slice)), + .named(ctx, "staticUpperBound", mlir.denseArrayAttribute(ctx, .i64, sentinel_slice)), + .named(ctx, "staticStep", mlir.denseArrayAttribute(ctx, .i64, sentinel_slice)), + }); + if (mapping) |m| attrs.appendAssumeCapacity(.named(ctx, "mapping", m)); + + return mlir.Operation.make(ctx, "scf.forall", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = result_types }, + .blocks = &.{body_block}, + .attributes = attrs.constSlice(), + .verify = false, + .location = location, + }); +} + +/// scf.forall.in_parallel — terminator for scf.forall. The inner region is +/// a single block containing a flat list of ops (typically +/// tensor.parallel_insert_slice). `body_block` must be built by the caller. +pub fn forall_in_parallel( + ctx: *mlir.Context, + body_block: *mlir.Block, + location: *const mlir.Location, +) *mlir.Operation { + _ = ctx; + var state: mlir.OperationState = .init("scf.forall.in_parallel", location); + const region = mlir.Region.init(); + region.appendOwnedBlock(body_block); + state.addOwnedRegions(&.{region}); + return mlir.Operation.init(&state) catch @panic("Failed to create scf.forall.in_parallel"); +} + +/// scf.index_switch — branches to case_blocks[i] if arg == cases[i], else +/// falls through to default_block. Blocks must be terminated with scf.yield. +/// The op's results have the types yielded by each region. +pub fn index_switch( + ctx: *mlir.Context, + arg: *const mlir.Value, + cases: []const i64, + default_block: *mlir.Block, + case_blocks: []const *mlir.Block, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + std.debug.assert(cases.len == case_blocks.len); + var state: mlir.OperationState = .init("scf.index_switch", location); + state.addOperands(&.{arg}); + state.addResults(result_types); + state.addAttributes(&.{ + .named(ctx, "cases", mlir.denseArrayAttribute(ctx, .i64, cases)), + }); + + // First region is the default, then one per case — this is how + // `VariadicRegion>:$caseRegions` is positioned in the td. + const default_region = mlir.Region.init(); + default_region.appendOwnedBlock(default_block); + + var regions: stdx.BoundedArray(*mlir.Region, 64) = .empty; + regions.appendAssumeCapacity(default_region); + for (case_blocks) |b| { + const r = mlir.Region.init(); + r.appendOwnedBlock(b); + regions.appendAssumeCapacity(r); + } + state.addOwnedRegions(regions.constSlice()); + + return mlir.Operation.init(&state) catch @panic("Failed to create scf.index_switch"); +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/mlir/dialects/ttir/BUILD.bazel b/mlir/dialects/ttir/BUILD.bazel new file mode 100644 index 00000000..0bea003e --- /dev/null +++ b/mlir/dialects/ttir/BUILD.bazel @@ -0,0 +1,37 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "ttir_capi", + srcs = ["ttir_capi.cc"], + hdrs = ["ttir_capi.h"], + strip_include_prefix = ".", + tags = ["manual"], + visibility = ["//visibility:private"], + deps = [ + "@llvm-project//mlir:CAPIIR", + "@triton//:TritonDialects", + ], + # MLIR_DEFINE_CAPI_DIALECT_REGISTRATION expands to a global static + # initializer that the linker would otherwise dead-strip. + alwayslink = True, +) + +zig_library( + name = "ttir", + tags = ["manual"], + import_name = "mlir/dialects/ttir", + main = "ttir.zig", + visibility = ["//visibility:public"], + deps = [ + ":ttir_capi", + "//mlir", + "//mlir:c", + "//stdx", + ], +) + +zig_test( + name = "test", + deps = [":ttir"], +) diff --git a/mlir/dialects/ttir/ttir.zig b/mlir/dialects/ttir/ttir.zig new file mode 100644 index 00000000..dde46822 --- /dev/null +++ b/mlir/dialects/ttir/ttir.zig @@ -0,0 +1,1293 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +// ============================================================================= +// Types — !tt.ptr +// ============================================================================= + +pub const PointerType = opaque { + const M = mlir.Methods(PointerType, c.MlirType); + + pub const isAFn = c.mlirTritonTypeIsAPointer; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn get(pointee_type: *const mlir.Type, address_space: i32) *const PointerType { + return @ptrCast(c.mlirTritonPointerTypeGet(pointee_type.ptr(), address_space).ptr); + } + + pub fn pointee(self: *const PointerType) *const mlir.Type { + return @ptrCast(c.mlirTritonPointerTypeGetPointee(self.ptr()).ptr); + } + + pub fn addressSpace(self: *const PointerType) i32 { + return c.mlirTritonPointerTypeGetAddressSpace(self.ptr()); + } +}; + +/// Convenience alias: `!tt.ptr` as a plain *const mlir.Type. +pub fn pointerType(pointee_type: *const mlir.Type, address_space: i32) *const mlir.Type { + return @ptrCast(PointerType.get(pointee_type, address_space)); +} + +// ----------------------------------------------------------------------------- +// !tt.tensordesc> — TMA descriptor type. +// ----------------------------------------------------------------------------- + +pub const TensorDescType = opaque { + const M = mlir.Methods(TensorDescType, c.MlirType); + + pub const isAFn = c.mlirTritonTypeIsATensorDesc; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + /// `shared_layout` may be null for an unlayout'd descriptor. + pub fn get( + shape: []const i64, + element_type: *const mlir.Type, + shared_layout: ?*const mlir.Attribute, + ) *const TensorDescType { + const attr_ptr = if (shared_layout) |a| a.ptr() else c.MlirAttribute{ .ptr = null }; + return @ptrCast(c.mlirTritonTensorDescTypeGet( + @intCast(shape.len), + shape.ptr, + element_type.ptr(), + attr_ptr, + ).ptr); + } +}; + +pub fn tensorDescType( + shape: []const i64, + element_type: *const mlir.Type, + shared_layout: ?*const mlir.Attribute, +) *const mlir.Type { + return @ptrCast(TensorDescType.get(shape, element_type, shared_layout)); +} + +// ============================================================================= +// Enum attributes +// ============================================================================= + +pub const ProgramIDDim = enum(i32) { x = 0, y = 1, z = 2 }; +pub const CacheModifier = enum(i32) { none = 1, ca = 2, cg = 3, wb = 4, cs = 5, wt = 6, cv = 7 }; +pub const EvictionPolicy = enum(i32) { normal = 1, evict_first = 2, evict_last = 3 }; +pub const PaddingOption = enum(i32) { pad_zero = 1, pad_nan = 2 }; +pub const RoundingMode = enum(i32) { rtz = 0, rtne = 1 }; +pub const InputPrecision = enum(i32) { tf32 = 0, tf32x3 = 1, ieee = 2, bf16x3 = 3, bf16x6 = 4 }; + +/// Atomic read-modify-write operations. +pub const RMWOp = enum(i32) { + and_ = 1, + or_ = 2, + xor_ = 3, + add = 4, + fadd = 5, + max = 6, + min = 7, + umax = 8, + umin = 9, + xchg = 10, +}; + +/// Memory ordering semantics. +pub const MemSemantic = enum(i32) { + relaxed = 1, + acquire = 2, + release = 3, + acq_rel = 4, +}; + +/// Memory sync scope. +pub const MemSyncScope = enum(i32) { + gpu = 1, + cta = 2, + sys = 3, +}; + +/// NaN propagation for clampf. 0xFFFF is the exact .td encoding. +pub const PropagateNan = enum(i32) { + none = 0, + all = 0xFFFF, +}; + +/// Packed FP element types for tt.dot_scaled. +pub const ScaleDotElemType = enum(i32) { + e4m3 = 0, + e5m2 = 1, + e2m3 = 2, + e3m2 = 3, + e2m1 = 4, + bf16 = 5, + fp16 = 6, +}; + +/// Reduction kind for tt.descriptor_reduce. +pub const DescriptorReduceKind = enum(i32) { + add = 1, + min = 2, + max = 3, + inc = 4, + dec = 5, + and_ = 6, + or_ = 7, + xor_ = 8, +}; + +pub fn programDim(ctx: *mlir.Context, v: ProgramIDDim) *const mlir.Attribute { + return @ptrCast(c.mlirTritonProgramDimGet(ctx.ptr(), @intFromEnum(v)).ptr); +} + +pub fn cacheModifier(ctx: *mlir.Context, v: CacheModifier) *const mlir.Attribute { + return @ptrCast(c.mlirTritonCacheModifierGet(ctx.ptr(), @intFromEnum(v)).ptr); +} + +pub fn evictionPolicy(ctx: *mlir.Context, v: EvictionPolicy) *const mlir.Attribute { + return @ptrCast(c.mlirTritonEvictionPolicyGet(ctx.ptr(), @intFromEnum(v)).ptr); +} + +pub fn paddingOption(ctx: *mlir.Context, v: PaddingOption) *const mlir.Attribute { + return @ptrCast(c.mlirTritonPaddingOptionGet(ctx.ptr(), @intFromEnum(v)).ptr); +} + +pub fn roundingMode(ctx: *mlir.Context, v: RoundingMode) *const mlir.Attribute { + return @ptrCast(c.mlirTritonRoundingModeGet(ctx.ptr(), @intFromEnum(v)).ptr); +} + +pub fn inputPrecision(ctx: *mlir.Context, v: InputPrecision) *const mlir.Attribute { + return @ptrCast(c.mlirTritonInputPrecisionGet(ctx.ptr(), @intFromEnum(v)).ptr); +} + +pub fn rmwOp(ctx: *mlir.Context, v: RMWOp) *const mlir.Attribute { + return @ptrCast(c.mlirTritonRMWOpGet(ctx.ptr(), @intFromEnum(v)).ptr); +} + +pub fn memSemantic(ctx: *mlir.Context, v: MemSemantic) *const mlir.Attribute { + return @ptrCast(c.mlirTritonMemSemanticGet(ctx.ptr(), @intFromEnum(v)).ptr); +} + +pub fn memSyncScope(ctx: *mlir.Context, v: MemSyncScope) *const mlir.Attribute { + return @ptrCast(c.mlirTritonMemSyncScopeGet(ctx.ptr(), @intFromEnum(v)).ptr); +} + +pub fn propagateNan(ctx: *mlir.Context, v: PropagateNan) *const mlir.Attribute { + return @ptrCast(c.mlirTritonPropagateNanGet(ctx.ptr(), @intFromEnum(v)).ptr); +} + +pub fn scaleDotElemType(ctx: *mlir.Context, v: ScaleDotElemType) *const mlir.Attribute { + return @ptrCast(c.mlirTritonScaleDotElemTypeGet(ctx.ptr(), @intFromEnum(v)).ptr); +} + +pub fn descriptorReduceKind(ctx: *mlir.Context, v: DescriptorReduceKind) *const mlir.Attribute { + return @ptrCast(c.mlirTritonDescriptorReduceKindGet(ctx.ptr(), @intFromEnum(v)).ptr); +} + +// ============================================================================= +// Function ops — tt.func / tt.return +// ============================================================================= + +/// Args for building a `tt.func` op. +pub const FuncOpArgs = struct { + pub const Visibility = enum { public, private }; + + name: []const u8, + args: ?[]const *const mlir.Type = null, + args_attributes: ?[]const *const mlir.Attribute = null, + results: ?[]const *const mlir.Type = null, + results_attributes: ?[]const *const mlir.Attribute = null, + block: *mlir.Block, + location: *const mlir.Location, + no_inline: bool = false, + visibility: Visibility = .public, + /// `tt.func` has `HasParent<"ModuleOp">`, so verification before appending + /// to a module always fails. Defer to the module-level verify instead. + verify: bool = false, +}; + +pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { + var args_buffer: stdx.BoundedArray(*const mlir.Type, 1024) = .empty; + var results_buffer: stdx.BoundedArray(*const mlir.Type, 32) = .empty; + + var attr_tuples_buffer: stdx.BoundedArray(mlir.NamedAttribute, 16) = .empty; + attr_tuples_buffer.appendSliceAssumeCapacity(&.{ + .named(ctx, "sym_name", mlir.stringAttribute(ctx, args.name)), + .named(ctx, "sym_visibility", mlir.stringAttribute(ctx, @tagName(args.visibility))), + .named(ctx, "function_type", mlir.typeAttribute(mlir.functionType( + ctx, + args.args orelse args_: { + for (0..args.block.numArguments()) |i| { + args_buffer.appendAssumeCapacity(args.block.argument(i).type_()); + } + break :args_ args_buffer.constSlice(); + }, + args.results orelse results_: { + const terminator = args.block.terminator() orelse { + @panic("block has no terminator"); + }; + for (0..terminator.numOperands()) |i| { + results_buffer.appendAssumeCapacity(terminator.operand(i).type_()); + } + break :results_ results_buffer.constSlice(); + }, + ))), + }); + + if (args.args_attributes) |args_attributes| { + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "arg_attrs", mlir.arrayAttribute(ctx, args_attributes))); + } + if (args.results_attributes) |results_attributes| { + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "res_attrs", mlir.arrayAttribute(ctx, results_attributes))); + } + // Always emit noinline even when false — matches Triton's IR convention. + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "noinline", mlir.boolAttribute(ctx, args.no_inline))); + + return mlir.Operation.make(ctx, "tt.func", .{ + .blocks = &.{args.block}, + .attributes = attr_tuples_buffer.constSlice(), + .location = args.location, + .verify = args.verify, + }); +} + +pub fn return_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.return", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +// ============================================================================= +// SPMD ops +// ============================================================================= + +pub fn get_program_id(ctx: *mlir.Context, axis: ProgramIDDim, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.get_program_id", .{ + .results = .{ .flat = &.{mlir.integerType(ctx, .i32)} }, + .attributes = &.{ + .named(ctx, "axis", programDim(ctx, axis)), + }, + .location = location, + }); +} + +pub fn get_num_programs(ctx: *mlir.Context, axis: ProgramIDDim, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.get_num_programs", .{ + .results = .{ .flat = &.{mlir.integerType(ctx, .i32)} }, + .attributes = &.{ + .named(ctx, "axis", programDim(ctx, axis)), + }, + .location = location, + }); +} + +// ============================================================================= +// Range +// ============================================================================= + +pub fn make_range( + ctx: *mlir.Context, + start: i32, + end: i32, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.make_range", .{ + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "start", mlir.integerAttribute(ctx, .i32, start)), + .named(ctx, "end", mlir.integerAttribute(ctx, .i32, end)), + }, + .location = location, + }); +} + +// ============================================================================= +// Shape manipulation +// ============================================================================= + +pub fn splat( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.splat", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn expand_dims( + ctx: *mlir.Context, + src: *const mlir.Value, + axis: i32, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.expand_dims", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "axis", mlir.integerAttribute(ctx, .i32, axis)), + }, + .location = location, + }); +} + +pub fn broadcast( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.broadcast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn reshape( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + allow_reorder: bool, + efficient_layout: bool, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (allow_reorder) { + attrs.appendAssumeCapacity(.named(ctx, "allow_reorder", mlir.unitAttribute(ctx))); + } + if (efficient_layout) { + attrs.appendAssumeCapacity(.named(ctx, "efficient_layout", mlir.unitAttribute(ctx))); + } + return mlir.Operation.make(ctx, "tt.reshape", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub fn trans( + ctx: *mlir.Context, + src: *const mlir.Value, + order: []const i32, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.trans", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "order", mlir.denseArrayAttribute(ctx, .i32, order)), + }, + .location = location, + }); +} + +// ============================================================================= +// Pointer arithmetic & memory +// ============================================================================= + +/// tt.addptr — result type matches ptr type (TypesMatchWith in TritonOps.td). +pub fn addptr( + ctx: *mlir.Context, + ptr_val: *const mlir.Value, + offset: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.addptr", .{ + .operands = .{ .flat = &.{ ptr_val, offset } }, + .results = .{ .flat = &.{ptr_val.type_()} }, + .location = location, + }); +} + +/// tt.load — AttrSizedOperandSegments: (ptr, [mask], [other]). Uses flat operand list + +/// explicit operandSegmentSizes because Operation.make variadic coercion zeros mixed-length segments. +pub fn load( + ctx: *mlir.Context, + ptr_val: *const mlir.Value, + result_type: *const mlir.Type, + mask: ?*const mlir.Value, + other: ?*const mlir.Value, + cache: CacheModifier, + evict: EvictionPolicy, + is_volatile: bool, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 3) = .empty; + operands_buf.appendAssumeCapacity(ptr_val); + const mask_len: i32 = if (mask) |m| blk: { + operands_buf.appendAssumeCapacity(m); + break :blk 1; + } else 0; + const other_len: i32 = if (other) |o| blk: { + operands_buf.appendAssumeCapacity(o); + break :blk 1; + } else 0; + + const seg_sizes = [3]i32{ 1, mask_len, other_len }; + return mlir.Operation.make(ctx, "tt.load", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), + .named(ctx, "cache", cacheModifier(ctx, cache)), + .named(ctx, "evict", evictionPolicy(ctx, evict)), + .named(ctx, "isVolatile", mlir.boolAttribute(ctx, is_volatile)), + }, + .location = location, + }); +} + +/// tt.store — operands: (ptr, value, [mask]). Flat operand list; mask is trailing variadic. +pub fn store( + ctx: *mlir.Context, + ptr_val: *const mlir.Value, + value: *const mlir.Value, + mask: ?*const mlir.Value, + cache: CacheModifier, + evict: EvictionPolicy, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 3) = .empty; + buf.appendSliceAssumeCapacity(&.{ ptr_val, value }); + if (mask) |m| buf.appendAssumeCapacity(m); + return mlir.Operation.make(ctx, "tt.store", .{ + .operands = .{ .flat = buf.constSlice() }, + .attributes = &.{ + .named(ctx, "cache", cacheModifier(ctx, cache)), + .named(ctx, "evict", evictionPolicy(ctx, evict)), + }, + .location = location, + }); +} + +// ============================================================================= +// Compute +// ============================================================================= + +pub fn dot( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c_acc: *const mlir.Value, + result_type: *const mlir.Type, + input_precision_: InputPrecision, + max_num_imprecise_acc: i32, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.dot", .{ + .operands = .{ .flat = &.{ a, b, c_acc } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "inputPrecision", inputPrecision(ctx, input_precision_)), + .named(ctx, "maxNumImpreciseAcc", mlir.integerAttribute(ctx, .i32, max_num_imprecise_acc)), + }, + .location = location, + }); +} + +/// tt.reduce — combine_block must have args [elem, elem] matching srcs' element types, +/// terminated with tt.reduce.return. +pub fn reduce( + ctx: *mlir.Context, + srcs: []const *const mlir.Value, + axis: i32, + combine_block: *mlir.Block, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + // Single variadic operand group — flat list, no operandSegmentSizes needed. + return mlir.Operation.make(ctx, "tt.reduce", .{ + .operands = .{ .flat = srcs }, + .results = .{ .flat = result_types }, + .blocks = &.{combine_block}, + .attributes = &.{ + .named(ctx, "axis", mlir.integerAttribute(ctx, .i32, axis)), + }, + .location = location, + }); +} + +pub fn reduce_return( + ctx: *mlir.Context, + values: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.reduce.return", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +// ============================================================================= +// Elementwise casts specific to Triton (arith.bitcast doesn't support ptrs) +// ============================================================================= + +/// tt.int_to_ptr — cast int-like to ptr-like (element-wise, same shape). +pub fn int_to_ptr( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.int_to_ptr", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// tt.ptr_to_int — cast ptr-like to int-like (element-wise, same shape). +pub fn ptr_to_int( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.ptr_to_int", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// tt.bitcast — cast between types of the same bit-width (also works on ptrs, +/// unlike arith.bitcast). +pub fn bitcast( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.bitcast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// tt.fp_to_fp — float to float cast, optionally with a rounding mode. +/// Pass `rounding = null` to omit the rounding attribute. +pub fn fp_to_fp( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + rounding: ?RoundingMode, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (rounding) |rm| { + attrs.appendAssumeCapacity(.named(ctx, "rounding", roundingMode(ctx, rm))); + } + return mlir.Operation.make(ctx, "tt.fp_to_fp", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +// ============================================================================= +// Math ops +// ============================================================================= + +/// tt.clampf — clamp float tensor to [min, max] element-wise. +pub fn clampf( + ctx: *mlir.Context, + x: *const mlir.Value, + min: *const mlir.Value, + max: *const mlir.Value, + propagate_nan: PropagateNan, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.clampf", .{ + .operands = .{ .flat = &.{ x, min, max } }, + .results = .{ .flat = &.{x.type_()} }, + .attributes = &.{ + .named(ctx, "propagateNan", propagateNan(ctx, propagate_nan)), + }, + .location = location, + }); +} + +/// tt.precise_sqrt — precise sqrt (SameOperandsAndResultType). +pub fn precise_sqrt( + ctx: *mlir.Context, + x: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.precise_sqrt", .{ + .operands = .{ .flat = &.{x} }, + .results = .{ .flat = &.{x.type_()} }, + .location = location, + }); +} + +/// tt.precise_divf — precise float div (SameOperandsAndResultType). +pub fn precise_divf( + ctx: *mlir.Context, + x: *const mlir.Value, + y: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.precise_divf", .{ + .operands = .{ .flat = &.{ x, y } }, + .results = .{ .flat = &.{x.type_()} }, + .location = location, + }); +} + +/// tt.mulhiui — most significant N bits of 2N-bit unsigned product. +pub fn mulhiui( + ctx: *mlir.Context, + x: *const mlir.Value, + y: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.mulhiui", .{ + .operands = .{ .flat = &.{ x, y } }, + .results = .{ .flat = &.{x.type_()} }, + .location = location, + }); +} + +// ============================================================================= +// Shape manipulation — more +// ============================================================================= + +/// tt.unsplat — tensor-of-1-element → scalar. +pub fn unsplat( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.unsplat", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// tt.cat — concatenate two tensors; may reorder elements. +pub fn cat( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.cat", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// tt.join — join two tensors along a new minor dimension (shape gains +1 dim of size 2). +pub fn join( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.join", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// tt.split — split a tensor along its last dimension (size must be 2). Returns (lhs, rhs). +pub fn split( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.split", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{ result_type, result_type } }, + .location = location, + }); +} + +// ============================================================================= +// Gather / Histogram +// ============================================================================= + +/// tt.gather — gather elements from `src` using `indices` along `axis`. +/// Result shape matches `indices` shape. +pub fn gather( + ctx: *mlir.Context, + src: *const mlir.Value, + indices: *const mlir.Value, + axis: i32, + result_type: *const mlir.Type, + efficient_layout: bool, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attrs.appendAssumeCapacity(.named(ctx, "axis", mlir.integerAttribute(ctx, .i32, axis))); + if (efficient_layout) { + attrs.appendAssumeCapacity(.named(ctx, "efficient_layout", mlir.unitAttribute(ctx))); + } + return mlir.Operation.make(ctx, "tt.gather", .{ + .operands = .{ .flat = &.{ src, indices } }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// tt.histogram — build a histogram of `src`. Output size = number of bins. +pub fn histogram( + ctx: *mlir.Context, + src: *const mlir.Value, + mask: ?*const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 2) = .empty; + buf.appendAssumeCapacity(src); + if (mask) |m| buf.appendAssumeCapacity(m); + return mlir.Operation.make(ctx, "tt.histogram", .{ + .operands = .{ .flat = buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +// ============================================================================= +// Scan — parallel scan with combine region (like reduce) +// ============================================================================= + +/// tt.scan — combine_block must already contain the tt.scan.return terminator +/// and have arguments with element types matching `srcs`. +pub fn scan( + ctx: *mlir.Context, + srcs: []const *const mlir.Value, + axis: i32, + reverse: bool, + combine_block: *mlir.Block, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.scan", .{ + .operands = .{ .variadic = &.{srcs} }, + .results = .{ .flat = result_types }, + .blocks = &.{combine_block}, + .attributes = &.{ + .named(ctx, "axis", mlir.integerAttribute(ctx, .i32, axis)), + .named(ctx, "reverse", mlir.boolAttribute(ctx, reverse)), + }, + .location = location, + }); +} + +pub fn scan_return( + ctx: *mlir.Context, + values: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.scan.return", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +// ============================================================================= +// Atomics +// ============================================================================= + +/// tt.atomic_rmw — (ptr, val, [mask]). Result type matches val's type. +pub fn atomic_rmw( + ctx: *mlir.Context, + rmw: RMWOp, + ptr_val: *const mlir.Value, + val: *const mlir.Value, + mask: ?*const mlir.Value, + sem: MemSemantic, + scope: MemSyncScope, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 3) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ ptr_val, val }); + if (mask) |m| operands_buf.appendAssumeCapacity(m); + + return mlir.Operation.make(ctx, "tt.atomic_rmw", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{val.type_()} }, + .attributes = &.{ + .named(ctx, "atomic_rmw_op", rmwOp(ctx, rmw)), + .named(ctx, "sem", memSemantic(ctx, sem)), + .named(ctx, "scope", memSyncScope(ctx, scope)), + }, + .location = location, + }); +} + +/// tt.atomic_cas — compare-and-swap. Result type matches val's type. +pub fn atomic_cas( + ctx: *mlir.Context, + ptr_val: *const mlir.Value, + cmp: *const mlir.Value, + val: *const mlir.Value, + sem: MemSemantic, + scope: MemSyncScope, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.atomic_cas", .{ + .operands = .{ .flat = &.{ ptr_val, cmp, val } }, + .results = .{ .flat = &.{val.type_()} }, + .attributes = &.{ + .named(ctx, "sem", memSemantic(ctx, sem)), + .named(ctx, "scope", memSyncScope(ctx, scope)), + }, + .location = location, + }); +} + +// ============================================================================= +// Debug +// ============================================================================= + +/// tt.assert — device-side assert. `condition` must be i1 or tensor<...xi1>. +pub fn assert_( + ctx: *mlir.Context, + condition: *const mlir.Value, + message: []const u8, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.assert", .{ + .operands = .{ .flat = &.{condition} }, + .attributes = &.{ + .named(ctx, "message", mlir.stringAttribute(ctx, message)), + }, + .location = location, + }); +} + +// ============================================================================= +// Call +// ============================================================================= + +/// tt.call — direct call to another tt.func in the same module. +/// arg_attrs / res_attrs propagate hints like tt.divisibility; pass null to omit. +pub fn call( + ctx: *mlir.Context, + callee: []const u8, + operands: []const *const mlir.Value, + result_types: []const *const mlir.Type, + arg_attrs: ?[]const *const mlir.Attribute, + res_attrs: ?[]const *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attrs.appendAssumeCapacity(.named(ctx, "callee", mlir.flatSymbolRefAttribute(ctx, callee))); + if (arg_attrs) |aa| { + attrs.appendAssumeCapacity(.named(ctx, "arg_attrs", mlir.arrayAttribute(ctx, aa))); + } + if (res_attrs) |ra| { + attrs.appendAssumeCapacity(.named(ctx, "res_attrs", mlir.arrayAttribute(ctx, ra))); + } + return mlir.Operation.make(ctx, "tt.call", .{ + .operands = .{ .flat = operands }, + .results = .{ .flat = result_types }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +// ============================================================================= +// Scaled dot (microscaling spec). +// ============================================================================= + +/// tt.dot_scaled — dot with optional per-block scale factors. +/// a_scale/b_scale may be null. AttrSizedOperandSegments: (a, b, c, [a_scale], [b_scale]). +pub fn dot_scaled( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c_acc: *const mlir.Value, + a_scale: ?*const mlir.Value, + b_scale: ?*const mlir.Value, + a_elem_type: ScaleDotElemType, + b_elem_type: ScaleDotElemType, + result_type: *const mlir.Type, + fast_math: bool, + lhs_k_pack: bool, + rhs_k_pack: bool, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 5) = .empty; + buf.appendSliceAssumeCapacity(&.{ a, b, c_acc }); + const a_scale_len: i32 = if (a_scale) |s| blk: { + buf.appendAssumeCapacity(s); + break :blk 1; + } else 0; + const b_scale_len: i32 = if (b_scale) |s| blk: { + buf.appendAssumeCapacity(s); + break :blk 1; + } else 0; + + const seg_sizes = [5]i32{ 1, 1, 1, a_scale_len, b_scale_len }; + return mlir.Operation.make(ctx, "tt.dot_scaled", .{ + .operands = .{ .flat = buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), + .named(ctx, "a_elem_type", scaleDotElemType(ctx, a_elem_type)), + .named(ctx, "b_elem_type", scaleDotElemType(ctx, b_elem_type)), + .named(ctx, "fastMath", mlir.boolAttribute(ctx, fast_math)), + .named(ctx, "lhs_k_pack", mlir.boolAttribute(ctx, lhs_k_pack)), + .named(ctx, "rhs_k_pack", mlir.boolAttribute(ctx, rhs_k_pack)), + }, + .location = location, + }); +} + +// ============================================================================= +// Extern elementwise (call library function pointwise) +// ============================================================================= + +pub fn extern_elementwise( + ctx: *mlir.Context, + srcs: []const *const mlir.Value, + result_type: *const mlir.Type, + libname: []const u8, + libpath: []const u8, + symbol: []const u8, + pure: bool, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.extern_elementwise", .{ + .operands = .{ .flat = srcs }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "libname", mlir.stringAttribute(ctx, libname)), + .named(ctx, "libpath", mlir.stringAttribute(ctx, libpath)), + .named(ctx, "symbol", mlir.stringAttribute(ctx, symbol)), + .named(ctx, "pure", mlir.boolAttribute(ctx, pure)), + }, + .location = location, + }); +} + +// ============================================================================= +// Device-side print +// ============================================================================= + +/// tt.print — device-side print. `is_signed` must be `args.len` long and +/// indicates signedness per arg (1 = signed, 0 = unsigned) for printf-style +/// integer formatting. +pub fn print( + ctx: *mlir.Context, + prefix: []const u8, + hex: bool, + args: []const *const mlir.Value, + is_signed: []const i32, + location: *const mlir.Location, +) *mlir.Operation { + std.debug.assert(args.len == is_signed.len); + return mlir.Operation.make(ctx, "tt.print", .{ + .operands = .{ .flat = args }, + .attributes = &.{ + .named(ctx, "prefix", mlir.stringAttribute(ctx, prefix)), + .named(ctx, "hex", mlir.boolAttribute(ctx, hex)), + .named(ctx, "isSigned", mlir.denseArrayAttribute(ctx, .i32, is_signed)), + }, + .location = location, + }); +} + +// ============================================================================= +// Tensor descriptors (TMA) +// ============================================================================= + +/// tt.make_tensor_descriptor — build a TMA descriptor from a base ptr, runtime +/// shape/strides, and the compile-time block shape baked into `result_type`. +pub fn make_tensor_descriptor( + ctx: *mlir.Context, + base: *const mlir.Value, + shape: []const *const mlir.Value, + strides: []const *const mlir.Value, + padding: PaddingOption, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendAssumeCapacity(base); + buf.appendSliceAssumeCapacity(shape); + buf.appendSliceAssumeCapacity(strides); + // SameVariadicOperandSize — flatten all operands. + return mlir.Operation.make(ctx, "tt.make_tensor_descriptor", .{ + .operands = .{ .flat = buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "padding", paddingOption(ctx, padding)), + }, + .location = location, + }); +} + +pub fn descriptor_load( + ctx: *mlir.Context, + desc: *const mlir.Value, + indices: []const *const mlir.Value, + result_type: *const mlir.Type, + cache: CacheModifier, + evict: EvictionPolicy, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendAssumeCapacity(desc); + buf.appendSliceAssumeCapacity(indices); + return mlir.Operation.make(ctx, "tt.descriptor_load", .{ + .operands = .{ .flat = buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "cache", cacheModifier(ctx, cache)), + .named(ctx, "evict", evictionPolicy(ctx, evict)), + }, + .location = location, + }); +} + +pub fn descriptor_store( + ctx: *mlir.Context, + desc: *const mlir.Value, + src: *const mlir.Value, + indices: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendSliceAssumeCapacity(&.{ desc, src }); + buf.appendSliceAssumeCapacity(indices); + return mlir.Operation.make(ctx, "tt.descriptor_store", .{ + .operands = .{ .flat = buf.constSlice() }, + .location = location, + }); +} + +pub fn descriptor_reduce( + ctx: *mlir.Context, + kind: DescriptorReduceKind, + desc: *const mlir.Value, + src: *const mlir.Value, + indices: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendSliceAssumeCapacity(&.{ desc, src }); + buf.appendSliceAssumeCapacity(indices); + return mlir.Operation.make(ctx, "tt.descriptor_reduce", .{ + .operands = .{ .flat = buf.constSlice() }, + .attributes = &.{ + .named(ctx, "kind", descriptorReduceKind(ctx, kind)), + }, + .location = location, + }); +} + +pub fn descriptor_gather( + ctx: *mlir.Context, + desc: *const mlir.Value, + x_offsets: *const mlir.Value, + y_offset: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.descriptor_gather", .{ + .operands = .{ .flat = &.{ desc, x_offsets, y_offset } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn descriptor_scatter( + ctx: *mlir.Context, + desc: *const mlir.Value, + x_offsets: *const mlir.Value, + y_offset: *const mlir.Value, + src: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.descriptor_scatter", .{ + .operands = .{ .flat = &.{ desc, x_offsets, y_offset, src } }, + .location = location, + }); +} + +// ============================================================================= +// Map elementwise — like a pointwise `map` with a user-defined scalar region. +// ============================================================================= + +/// tt.map_elementwise — `scalar_block` receives `pack * srcs.len` scalar args +/// (packed groups of `pack` elements per src tensor) and must be terminated +/// with `tt.map_elementwise.return`. +pub fn map_elementwise( + ctx: *mlir.Context, + srcs: []const *const mlir.Value, + pack: i32, + scalar_block: *mlir.Block, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.map_elementwise", .{ + .operands = .{ .flat = srcs }, + .results = .{ .flat = result_types }, + .blocks = &.{scalar_block}, + .attributes = &.{ + .named(ctx, "pack", mlir.integerAttribute(ctx, .i32, pack)), + }, + .location = location, + }); +} + +pub fn map_elementwise_return( + ctx: *mlir.Context, + values: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.map_elementwise.return", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +// ============================================================================= +// Inline assembly +// ============================================================================= + +/// tt.elementwise_inline_asm — pointwise inline asm returning one or more +/// tensors. +pub fn elementwise_inline_asm( + ctx: *mlir.Context, + asm_string: []const u8, + constraints: []const u8, + args: []const *const mlir.Value, + result_types: []const *const mlir.Type, + pure: bool, + packed_element: i32, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tt.elementwise_inline_asm", .{ + .operands = .{ .flat = args }, + .results = .{ .flat = result_types }, + .attributes = &.{ + .named(ctx, "asm_string", mlir.stringAttribute(ctx, asm_string)), + .named(ctx, "constraints", mlir.stringAttribute(ctx, constraints)), + .named(ctx, "pure", mlir.boolAttribute(ctx, pure)), + .named(ctx, "packed_element", mlir.integerAttribute(ctx, .i32, packed_element)), + }, + .location = location, + }); +} + +test { + std.testing.refAllDecls(@This()); +} + +test "tt.func round-trip" { + // Build: module { tt.func public @id(%arg0: !tt.ptr) { tt.return } } + // by constructing it with the Zig wrappers, printing to string, and re-parsing. + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + inline for (.{ "func", "arith", "scf", "math", "tt" }) |d| { + mlir.DialectHandle.fromString(d).insertDialect(registry); + } + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + + try std.testing.expect(ctx.isRegisteredOperation("tt.func")); + try std.testing.expect(ctx.isRegisteredOperation("tt.return")); + try std.testing.expect(ctx.isRegisteredOperation("tt.load")); + try std.testing.expect(ctx.isRegisteredOperation("tt.store")); + try std.testing.expect(ctx.isRegisteredOperation("tt.get_program_id")); + try std.testing.expect(ctx.isRegisteredOperation("tt.splat")); + try std.testing.expect(ctx.isRegisteredOperation("tt.addptr")); + try std.testing.expect(ctx.isRegisteredOperation("scf.for")); + try std.testing.expect(ctx.isRegisteredOperation("arith.constant")); + + const loc: *const mlir.Location = .unknown(ctx); + const module: *mlir.Module = .init(loc); + defer module.deinit(); + + const f32_ty = mlir.floatType(ctx, .f32); + const ptr_ty = pointerType(f32_ty, 1); + const arg_types: []const *const mlir.Type = &.{ ptr_ty, ptr_ty }; + const arg_locs: []const *const mlir.Location = &.{ loc, loc }; + + const entry = mlir.Block.init(arg_types, arg_locs); + + const arg0 = entry.argument(0); + const arg1 = entry.argument(1); + + // %0 = tt.load %arg0 : !tt.ptr + const loaded = load(ctx, arg0, f32_ty, null, null, .none, .normal, false, loc); + _ = loaded.appendTo(entry); + + // %1 = arith.constant 1.0 : f32 — built with Operation.make to avoid cross-dialect import + const one_attr: *const mlir.Attribute = mlir.floatAttribute(ctx, .f32, 1.0); + const one = mlir.Operation.make(ctx, "arith.constant", .{ + .results = .{ .flat = &.{f32_ty} }, + .attributes = &.{.named(ctx, "value", one_attr)}, + .location = loc, + }); + _ = one.appendTo(entry); + + // %2 = arith.addf %0, %1 : f32 + const summed = mlir.Operation.make(ctx, "arith.addf", .{ + .operands = .{ .flat = &.{ loaded.result(0), one.result(0) } }, + .result_type_inference = true, + .location = loc, + }); + _ = summed.appendTo(entry); + + // tt.store %arg1, %2 + _ = store(ctx, arg1, summed.result(0), null, .none, .normal, loc).appendTo(entry); + + // tt.return + _ = return_(ctx, &.{}, loc).appendTo(entry); + + const f = func(ctx, .{ + .name = "add_one", + .block = entry, + .location = loc, + }); + _ = f.appendTo(module.body()); + + // Verify the operation. + try std.testing.expect(module.operation().verify()); + + // Print IR and re-parse it to confirm it's valid text. + var al: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer al.deinit(); + try al.writer.print("{f}", .{module.operation()}); + + const parsed = try mlir.Module.parse(ctx, al.written()); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} diff --git a/mlir/dialects/ttir/ttir_capi.cc b/mlir/dialects/ttir/ttir_capi.cc new file mode 100644 index 00000000..ed7d533e --- /dev/null +++ b/mlir/dialects/ttir/ttir_capi.cc @@ -0,0 +1,113 @@ +#include "ttir_capi.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" +#include "triton/Dialect/Triton/IR/Dialect.h" +#include "triton/Dialect/Triton/IR/Types.h" + +// Note: the second arg `tt` controls the generated symbol name +// `mlirGetDialectHandle__tt__`, which `mlir.DialectHandle.fromString("tt")` on +// the Zig side resolves at comptime (see mlir/mlir.zig:696-698). Must match the +// dialect's declared name (`let name = "tt"` in TritonDialect.td). +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Triton, tt, mlir::triton::TritonDialect) + +using mlir::cast; +using mlir::isa; + +extern "C" { + +MlirType mlirTritonPointerTypeGet(MlirType pointee, int32_t address_space) { + return wrap(mlir::triton::PointerType::get(unwrap(pointee), address_space)); +} + +MlirType mlirTritonPointerTypeGetPointee(MlirType ptr) { + return wrap(cast(unwrap(ptr)).getPointeeType()); +} + +int32_t mlirTritonPointerTypeGetAddressSpace(MlirType ptr) { + return cast(unwrap(ptr)).getAddressSpace(); +} + +bool mlirTritonTypeIsAPointer(MlirType t) { + return isa(unwrap(t)); +} + +MlirType mlirTritonTensorDescTypeGet(intptr_t rank, const int64_t* shape, + MlirType element_type, + MlirAttribute shared_layout) { + mlir::Attribute layout_attr; + if (!mlirAttributeIsNull(shared_layout)) { + layout_attr = unwrap(shared_layout); + } + // TensorDescType::get is a TypeBuilderWithInferredContext: the context is + // inferred from elementType. Signature: (shape, elementType, sharedLayout). + return wrap(mlir::triton::TensorDescType::get( + llvm::ArrayRef(shape, rank), + unwrap(element_type), layout_attr)); +} + +bool mlirTritonTypeIsATensorDesc(MlirType t) { + return isa(unwrap(t)); +} + +MlirAttribute mlirTritonProgramDimGet(MlirContext ctx, int32_t value) { + return wrap(mlir::triton::ProgramIDDimAttr::get( + unwrap(ctx), static_cast(value))); +} + +MlirAttribute mlirTritonCacheModifierGet(MlirContext ctx, int32_t value) { + return wrap(mlir::triton::CacheModifierAttr::get( + unwrap(ctx), static_cast(value))); +} + +MlirAttribute mlirTritonEvictionPolicyGet(MlirContext ctx, int32_t value) { + return wrap(mlir::triton::EvictionPolicyAttr::get( + unwrap(ctx), static_cast(value))); +} + +MlirAttribute mlirTritonPaddingOptionGet(MlirContext ctx, int32_t value) { + return wrap(mlir::triton::PaddingOptionAttr::get( + unwrap(ctx), static_cast(value))); +} + +MlirAttribute mlirTritonRoundingModeGet(MlirContext ctx, int32_t value) { + return wrap(mlir::triton::RoundingModeAttr::get( + unwrap(ctx), static_cast(value))); +} + +MlirAttribute mlirTritonInputPrecisionGet(MlirContext ctx, int32_t value) { + return wrap(mlir::triton::InputPrecisionAttr::get( + unwrap(ctx), static_cast(value))); +} + +MlirAttribute mlirTritonRMWOpGet(MlirContext ctx, int32_t value) { + return wrap(mlir::triton::RMWOpAttr::get( + unwrap(ctx), static_cast(value))); +} + +MlirAttribute mlirTritonMemSemanticGet(MlirContext ctx, int32_t value) { + return wrap(mlir::triton::MemSemanticAttr::get( + unwrap(ctx), static_cast(value))); +} + +MlirAttribute mlirTritonMemSyncScopeGet(MlirContext ctx, int32_t value) { + return wrap(mlir::triton::MemSyncScopeAttr::get( + unwrap(ctx), static_cast(value))); +} + +MlirAttribute mlirTritonPropagateNanGet(MlirContext ctx, int32_t value) { + return wrap(mlir::triton::PropagateNanAttr::get( + unwrap(ctx), static_cast(value))); +} + +MlirAttribute mlirTritonScaleDotElemTypeGet(MlirContext ctx, int32_t value) { + return wrap(mlir::triton::ScaleDotElemTypeAttr::get( + unwrap(ctx), static_cast(value))); +} + +MlirAttribute mlirTritonDescriptorReduceKindGet(MlirContext ctx, int32_t value) { + return wrap(mlir::triton::DescriptorReduceKindAttr::get( + unwrap(ctx), static_cast(value))); +} + +} // extern "C" diff --git a/mlir/dialects/ttir/ttir_capi.h b/mlir/dialects/ttir/ttir_capi.h new file mode 100644 index 00000000..7e5813c2 --- /dev/null +++ b/mlir/dialects/ttir/ttir_capi.h @@ -0,0 +1,47 @@ +#ifndef ZML_MLIR_DIALECTS_TTIR_TTIR_CAPI_H +#define ZML_MLIR_DIALECTS_TTIR_TTIR_CAPI_H + +#include +#include + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Triton, tt); + +// !tt.ptr +MlirType mlirTritonPointerTypeGet(MlirType pointee, int32_t address_space); +MlirType mlirTritonPointerTypeGetPointee(MlirType ptr); +int32_t mlirTritonPointerTypeGetAddressSpace(MlirType ptr); +bool mlirTritonTypeIsAPointer(MlirType t); + +// !tt.tensordesc> — result of tt.make_tensor_descriptor. +// `shape` is a `rank`-element array of int64_t block dims; `element_type` is +// the scalar element type; `shared_layout` may be a null MlirAttribute for none. +MlirType mlirTritonTensorDescTypeGet(intptr_t rank, const int64_t* shape, + MlirType element_type, + MlirAttribute shared_layout); +bool mlirTritonTypeIsATensorDesc(MlirType t); + +// I32EnumAttr getters +MlirAttribute mlirTritonProgramDimGet(MlirContext ctx, int32_t value); +MlirAttribute mlirTritonCacheModifierGet(MlirContext ctx, int32_t value); +MlirAttribute mlirTritonEvictionPolicyGet(MlirContext ctx, int32_t value); +MlirAttribute mlirTritonPaddingOptionGet(MlirContext ctx, int32_t value); +MlirAttribute mlirTritonRoundingModeGet(MlirContext ctx, int32_t value); +MlirAttribute mlirTritonInputPrecisionGet(MlirContext ctx, int32_t value); +MlirAttribute mlirTritonRMWOpGet(MlirContext ctx, int32_t value); +MlirAttribute mlirTritonMemSemanticGet(MlirContext ctx, int32_t value); +MlirAttribute mlirTritonMemSyncScopeGet(MlirContext ctx, int32_t value); +MlirAttribute mlirTritonPropagateNanGet(MlirContext ctx, int32_t value); +MlirAttribute mlirTritonScaleDotElemTypeGet(MlirContext ctx, int32_t value); +MlirAttribute mlirTritonDescriptorReduceKindGet(MlirContext ctx, int32_t value); + +#ifdef __cplusplus +} +#endif + +#endif // ZML_MLIR_DIALECTS_TTIR_TTIR_CAPI_H diff --git a/mlir/dialects/vector.zig b/mlir/dialects/vector.zig new file mode 100644 index 00000000..ab1dfa19 --- /dev/null +++ b/mlir/dialects/vector.zig @@ -0,0 +1,776 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +// ============================================================================= +// Enum attributes — parsed from their textual `#vector.<...>` form. +// ============================================================================= + +/// Matches `mlir::vector::CombiningKind` (I32EnumAttr in VectorAttributes.td). +pub const CombiningKind = enum { + add, + mul, + minui, + minsi, + minnumf, + maxui, + maxsi, + maxnumf, + @"and", + @"or", + xor, + minimumf, + maximumf, + + pub fn attribute(self: CombiningKind, ctx: *mlir.Context) *const mlir.Attribute { + var buf: [64]u8 = undefined; + const text = std.fmt.bufPrint(&buf, "#vector.kind<{s}>", .{@tagName(self)}) catch unreachable; + return mlir.Attribute.parse(ctx, text) catch + std.debug.panic("failed to parse CombiningKind '{s}'", .{text}); + } +}; + +/// Matches `mlir::vector::IteratorType` — used by `vector.contract`. +pub const IteratorType = enum { + parallel, + reduction, + + pub fn attribute(self: IteratorType, ctx: *mlir.Context) *const mlir.Attribute { + const text = switch (self) { + .parallel => "#vector.iterator_type", + .reduction => "#vector.iterator_type", + }; + return mlir.Attribute.parse(ctx, text) catch + std.debug.panic("failed to parse IteratorType", .{}); + } +}; + +// ============================================================================= +// Element / lane access — extract / insert / from_elements / to_elements +// ============================================================================= + +/// vector.extract — pull a single element or sub-vector out of `src`. Static +/// position entries equal to `kPoison` (== `INT64_MIN` per VectorOps.td) come +/// from `dynamic_position`. The result type is provided by the caller. +pub const kPoison: i64 = std.math.minInt(i64); + +pub fn extract( + ctx: *mlir.Context, + src: *const mlir.Value, + static_position: []const i64, + dynamic_position: []const *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + operands_buf.appendAssumeCapacity(src); + operands_buf.appendSliceAssumeCapacity(dynamic_position); + + return mlir.Operation.make(ctx, "vector.extract", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "static_position", mlir.denseArrayAttribute(ctx, .i64, static_position)), + }, + .location = location, + }); +} + +/// vector.insert — write `value` into `dest` at the given position. The result +/// has the same type as `dest`. +pub fn insert( + ctx: *mlir.Context, + value: *const mlir.Value, + dest: *const mlir.Value, + static_position: []const i64, + dynamic_position: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 17) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ value, dest }); + operands_buf.appendSliceAssumeCapacity(dynamic_position); + + return mlir.Operation.make(ctx, "vector.insert", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{dest.type_()} }, + .attributes = &.{ + .named(ctx, "static_position", mlir.denseArrayAttribute(ctx, .i64, static_position)), + }, + .location = location, + }); +} + +/// vector.from_elements — build a vector from an exact-size list of scalars. +pub fn from_elements( + ctx: *mlir.Context, + elements: []const *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.from_elements", .{ + .operands = .{ .flat = elements }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// vector.to_elements — destructure a vector into its scalar lanes. +pub fn to_elements( + ctx: *mlir.Context, + src: *const mlir.Value, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.to_elements", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = result_types }, + .location = location, + }); +} + +// ============================================================================= +// Strided slice — extract / insert with offsets/sizes/strides +// ============================================================================= + +pub const StridedSliceArgs = struct { + offsets: []const i64, + sizes: []const i64, + strides: []const i64, +}; + +pub fn extract_strided_slice( + ctx: *mlir.Context, + src: *const mlir.Value, + args: StridedSliceArgs, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.extract_strided_slice", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "offsets", mlir.denseArrayAttribute(ctx, .i64, args.offsets)), + .named(ctx, "sizes", mlir.denseArrayAttribute(ctx, .i64, args.sizes)), + .named(ctx, "strides", mlir.denseArrayAttribute(ctx, .i64, args.strides)), + }, + .location = location, + }); +} + +pub fn insert_strided_slice( + ctx: *mlir.Context, + src: *const mlir.Value, + dest: *const mlir.Value, + offsets: []const i64, + strides: []const i64, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.insert_strided_slice", .{ + .operands = .{ .flat = &.{ src, dest } }, + .results = .{ .flat = &.{dest.type_()} }, + .attributes = &.{ + .named(ctx, "offsets", mlir.denseArrayAttribute(ctx, .i64, offsets)), + .named(ctx, "strides", mlir.denseArrayAttribute(ctx, .i64, strides)), + }, + .location = location, + }); +} + +// ============================================================================= +// Shape manipulation +// ============================================================================= + +/// vector.broadcast — broadcast a scalar / lower-rank vector to a wider one. +pub fn broadcast( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.broadcast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// vector.shape_cast — reshape a vector preserving total number of lanes. +pub fn shape_cast( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.shape_cast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// vector.bitcast — same total bits, possibly different element type / shape. +pub fn bitcast( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.bitcast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// vector.type_cast — cast a `memref>` to `memref<...>` (and vice- +/// versa). Used in vector lowering / unrolling. +pub fn type_cast( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.type_cast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// vector.transpose — permute dims of a vector. +pub fn transpose( + ctx: *mlir.Context, + src: *const mlir.Value, + permutation: []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.transpose", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "permutation", mlir.denseArrayAttribute(ctx, .i64, permutation)), + }, + .location = location, + }); +} + +// ============================================================================= +// Lane shuffle — shuffle / interleave / deinterleave +// ============================================================================= + +/// vector.shuffle — concat lhs and rhs into a logical vector (rank 1) and +/// pick `mask`-listed lanes to form the result. +pub fn shuffle( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + mask: []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.shuffle", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "mask", mlir.denseArrayAttribute(ctx, .i64, mask)), + }, + .location = location, + }); +} + +/// vector.interleave — interleave even/odd lanes from lhs/rhs. +pub fn interleave( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.interleave", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// vector.deinterleave — split a vector into (even-lanes, odd-lanes). +pub fn deinterleave( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.deinterleave", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{ result_type, result_type } }, + .location = location, + }); +} + +// ============================================================================= +// Compute — fma / step / vscale +// ============================================================================= + +/// vector.fma — element-wise fused multiply-add `lhs * rhs + acc`. +pub fn fma( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + acc: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.fma", .{ + .operands = .{ .flat = &.{ lhs, rhs, acc } }, + .result_type_inference = true, + .location = location, + }); +} + +/// vector.step — produce `<0, 1, ..., N-1>` of the result type. +pub fn step( + ctx: *mlir.Context, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.step", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// vector.vscale — runtime scalable-vector size factor (index-typed). +pub fn vscale(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.vscale", .{ + .results = .{ .flat = &.{mlir.indexType(ctx)} }, + .location = location, + }); +} + +/// vector.yield — terminator for vector.mask / vector.warp regions. +pub fn yield(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.yield", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +// ============================================================================= +// Reductions — reduction / multi_reduction / contract +// ============================================================================= + +pub const ReductionOpts = struct { + /// Optional accumulator (scalar of result type). When provided the op is + /// `result = combine(reduce(src), acc)`. + acc: ?*const mlir.Value = null, +}; + +/// vector.reduction — reduce a 1-D vector to a scalar with `kind`. +pub fn reduction( + ctx: *mlir.Context, + kind: CombiningKind, + src: *const mlir.Value, + result_type: *const mlir.Type, + opts: ReductionOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 2) = .empty; + operands_buf.appendAssumeCapacity(src); + if (opts.acc) |a| operands_buf.appendAssumeCapacity(a); + + return mlir.Operation.make(ctx, "vector.reduction", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "kind", kind.attribute(ctx)), + }, + .location = location, + }); +} + +/// vector.multi_reduction — reduce a multi-D vector along `reduction_dims`. +pub fn multi_reduction( + ctx: *mlir.Context, + kind: CombiningKind, + src: *const mlir.Value, + acc: *const mlir.Value, + reduction_dims: []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.multi_reduction", .{ + .operands = .{ .flat = &.{ src, acc } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "kind", kind.attribute(ctx)), + .named(ctx, "reduction_dims", mlir.denseArrayAttribute(ctx, .i64, reduction_dims)), + }, + .location = location, + }); +} + +pub const ContractOpts = struct { + kind: CombiningKind = .add, +}; + +/// vector.contract — generalized contraction (matmul, dot, batched matmul, +/// reductions). `indexing_maps` is one AffineMapAttr per (lhs, rhs, acc). +/// `iterator_types` is one IteratorType per dim of the iteration space. +pub fn contract( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + acc: *const mlir.Value, + indexing_maps: []const *const mlir.Attribute, + iterator_types: []const IteratorType, + result_type: *const mlir.Type, + opts: ContractOpts, + location: *const mlir.Location, +) *mlir.Operation { + var iter_attrs: stdx.BoundedArray(*const mlir.Attribute, 16) = .empty; + for (iterator_types) |it| iter_attrs.appendAssumeCapacity(it.attribute(ctx)); + + return mlir.Operation.make(ctx, "vector.contract", .{ + .operands = .{ .flat = &.{ lhs, rhs, acc } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "indexing_maps", mlir.arrayAttribute(ctx, indexing_maps)), + .named(ctx, "iterator_types", mlir.arrayAttribute(ctx, iter_attrs.constSlice())), + .named(ctx, "kind", opts.kind.attribute(ctx)), + }, + .location = location, + }); +} + +// ============================================================================= +// Memory transfers — load / store / masked / gather / scatter / transfer_* +// ============================================================================= + +pub const LoadStoreOpts = struct { + nontemporal: bool = false, + alignment: ?i64 = null, +}; + +fn loadStoreAttrs(ctx: *mlir.Context, opts: LoadStoreOpts, dst: *stdx.BoundedArray(mlir.NamedAttribute, 2)) void { + if (opts.nontemporal) { + dst.appendAssumeCapacity(.named(ctx, "nontemporal", mlir.boolAttribute(ctx, true))); + } + if (opts.alignment) |a| { + dst.appendAssumeCapacity(.named(ctx, "alignment", mlir.integerAttribute(ctx, .i64, a))); + } +} + +/// vector.load — strided contiguous load from `base[indices...]` into the +/// result vector. +pub fn load( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + result_type: *const mlir.Type, + opts: LoadStoreOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + loadStoreAttrs(ctx, opts, &attrs); + + return mlir.Operation.make(ctx, "vector.load", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// vector.store — strided contiguous store of `value` to `base[indices...]`. +pub fn store( + ctx: *mlir.Context, + value: *const mlir.Value, + base: *const mlir.Value, + indices: []const *const mlir.Value, + opts: LoadStoreOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 17) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ value, base }); + operands_buf.appendSliceAssumeCapacity(indices); + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + loadStoreAttrs(ctx, opts, &attrs); + + return mlir.Operation.make(ctx, "vector.store", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// vector.maskedload — like load but lanes whose `mask` bit is 0 are read +/// from `passthru` instead. +pub fn masked_load( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + mask: *const mlir.Value, + passthru: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + operands_buf.appendSliceAssumeCapacity(&.{ mask, passthru }); + + return mlir.Operation.make(ctx, "vector.maskedload", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn masked_store( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + mask: *const mlir.Value, + value: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + operands_buf.appendSliceAssumeCapacity(&.{ mask, value }); + + return mlir.Operation.make(ctx, "vector.maskedstore", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .location = location, + }); +} + +/// vector.gather — gather scalar elements from `base[indices...]` plus per-lane +/// `index_vec` offsets. +pub fn gather( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + index_vec: *const mlir.Value, + mask: *const mlir.Value, + passthru: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 20) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + operands_buf.appendSliceAssumeCapacity(&.{ index_vec, mask, passthru }); + + return mlir.Operation.make(ctx, "vector.gather", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn scatter( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + index_vec: *const mlir.Value, + mask: *const mlir.Value, + value: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 20) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + operands_buf.appendSliceAssumeCapacity(&.{ index_vec, mask, value }); + + return mlir.Operation.make(ctx, "vector.scatter", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .location = location, + }); +} + +pub fn expand_load( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + mask: *const mlir.Value, + passthru: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + operands_buf.appendSliceAssumeCapacity(&.{ mask, passthru }); + + return mlir.Operation.make(ctx, "vector.expandload", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn compress_store( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + mask: *const mlir.Value, + value: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + operands_buf.appendSliceAssumeCapacity(&.{ mask, value }); + + return mlir.Operation.make(ctx, "vector.compressstore", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .location = location, + }); +} + +// ============================================================================= +// Transfer ops — vector.transfer_read / vector.transfer_write +// ============================================================================= + +pub const TransferReadArgs = struct { + /// `affine_map<(d0,...) -> (...)>` attribute. Build with + /// `mlir.parseAffineMapAttribute` or `mlir.affineMapAttribute(...)`. + permutation_map: *const mlir.Attribute, + /// One bool per result-vector dim. Length must equal vector rank. + in_bounds: []const bool, + /// Optional i1 vector mask. + mask: ?*const mlir.Value = null, +}; + +/// vector.transfer_read — supervector load with permutation map and optional +/// mask. AttrSizedOperandSegments: (base, indices, padding, [mask]). +pub fn transfer_read( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + padding: *const mlir.Value, + args: TransferReadArgs, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 20) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + operands_buf.appendAssumeCapacity(padding); + const mask_len: i32 = if (args.mask) |m| blk: { + operands_buf.appendAssumeCapacity(m); + break :blk 1; + } else 0; + + const seg_sizes = [4]i32{ 1, @intCast(indices.len), 1, mask_len }; + + var in_bounds_attrs: stdx.BoundedArray(*const mlir.Attribute, 16) = .empty; + for (args.in_bounds) |b| in_bounds_attrs.appendAssumeCapacity(mlir.boolAttribute(ctx, b)); + + return mlir.Operation.make(ctx, "vector.transfer_read", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), + .named(ctx, "permutation_map", args.permutation_map), + .named(ctx, "in_bounds", mlir.arrayAttribute(ctx, in_bounds_attrs.constSlice())), + }, + .location = location, + }); +} + +pub const TransferWriteArgs = struct { + permutation_map: *const mlir.Attribute, + in_bounds: []const bool, + mask: ?*const mlir.Value = null, + /// When `dest` is a tensor, the op produces a new tensor — pass its type + /// here. For memref destinations leave null. + result_type: ?*const mlir.Type = null, +}; + +/// vector.transfer_write — supervector store with permutation map and optional +/// mask. AttrSizedOperandSegments: (vector, base, indices, [mask]). +pub fn transfer_write( + ctx: *mlir.Context, + value: *const mlir.Value, + base: *const mlir.Value, + indices: []const *const mlir.Value, + args: TransferWriteArgs, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 20) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ value, base }); + operands_buf.appendSliceAssumeCapacity(indices); + const mask_len: i32 = if (args.mask) |m| blk: { + operands_buf.appendAssumeCapacity(m); + break :blk 1; + } else 0; + + const seg_sizes = [4]i32{ 1, 1, @intCast(indices.len), mask_len }; + + var in_bounds_attrs: stdx.BoundedArray(*const mlir.Attribute, 16) = .empty; + for (args.in_bounds) |b| in_bounds_attrs.appendAssumeCapacity(mlir.boolAttribute(ctx, b)); + + var result_types: stdx.BoundedArray(*const mlir.Type, 1) = .empty; + if (args.result_type) |rt| result_types.appendAssumeCapacity(rt); + + return mlir.Operation.make(ctx, "vector.transfer_write", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = result_types.constSlice() }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), + .named(ctx, "permutation_map", args.permutation_map), + .named(ctx, "in_bounds", mlir.arrayAttribute(ctx, in_bounds_attrs.constSlice())), + }, + .location = location, + }); +} + +// ============================================================================= +// Masks — constant_mask / create_mask +// ============================================================================= + +/// vector.constant_mask — constant boolean mask. Each entry of +/// `mask_dim_sizes` is the number of leading true lanes along that dim. +pub fn constant_mask( + ctx: *mlir.Context, + mask_dim_sizes: []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.constant_mask", .{ + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "mask_dim_sizes", mlir.denseArrayAttribute(ctx, .i64, mask_dim_sizes)), + }, + .location = location, + }); +} + +/// vector.create_mask — runtime mask from per-dim count operands. `operands` +/// length must equal the rank of `result_type`. +pub fn create_mask( + ctx: *mlir.Context, + operands: []const *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.create_mask", .{ + .operands = .{ .flat = operands }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/mlir/mlir.h b/mlir/mlir.h index d3e2719e..8d079000 100644 --- a/mlir/mlir.h +++ b/mlir/mlir.h @@ -1,10 +1,16 @@ +#include +#include #include #include #include +#include #include #include +#include #include +#include #include +#include #include #include @@ -14,6 +20,8 @@ extern "C" { void mlirRegisterFuncExtensions(MlirDialectRegistry registry); +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Affine, affine); + #ifdef __cplusplus } #endif diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 2cd086b4..8aa8a5b0 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -253,6 +253,14 @@ pub const Context = opaque { return c.mlirContextIsRegisteredOperation(self.ptr(), stringRef(op)); } + pub fn setAllowUnregisteredDialects(self: *Context, allow: bool) void { + c.mlirContextSetAllowUnregisteredDialects(self.ptr(), allow); + } + + pub fn allowUnregisteredDialects(self: *const Context) bool { + return c.mlirContextGetAllowUnregisteredDialects(self.ptr()); + } + pub fn format(self: *const Context, writer: *std.Io.Writer) !void { try writer.print("{s}{{ .numThreads = {d}, .numRegisteredDialects = {d}, .numLoadedDialects = {d} }}", .{ @typeName(Context), @@ -801,10 +809,24 @@ pub const Block = opaque { } } + pub fn firstOperation(self: *const Block) ?*Operation { + const op = c.mlirBlockGetFirstOperation(self.ptr()); + if (op.ptr == null) return null; + return @ptrCast(op.ptr); + } + + pub fn insertOwnedOperationBefore(self: *Block, reference: *Operation, op: *Operation) void { + c.mlirBlockInsertOwnedOperationBefore(self.ptr(), reference.ptr(), op.ptr()); + } + pub fn parentOperation(self: *const Block) ?*Operation { return @ptrCast(c.mlirBlockGetParentOperation(self.ptr()).ptr); } + pub fn parentRegion(self: *const Block) *Region { + return @ptrCast(c.mlirBlockGetParentRegion(self.ptr()).ptr); + } + pub fn detach(self: *Block) void { c.mlirBlockDetach(self.ptr()); } @@ -1046,6 +1068,14 @@ pub const Operation = opaque { return self; } + pub fn numRegions(self: *const Operation) usize { + return @intCast(c.mlirOperationGetNumRegions(self.ptr())); + } + + pub fn region(self: *const Operation, index: usize) *Region { + return @ptrCast(c.mlirOperationGetRegion(self.ptr(), @intCast(index)).ptr); + } + pub fn parent(self: *const Operation) ?*Operation { return @ptrCast(c.mlirOperationGetParentOperation(self.ptr()).ptr); } @@ -1093,6 +1123,7 @@ pub const Operation = opaque { print_generic_op_form: bool = false, use_local_scope: bool = false, assume_verified: bool = false, + print_name_loc_as_prefix: bool = false, const Ctx = struct { self: *const Operation, @@ -1118,6 +1149,9 @@ pub const Operation = opaque { if (self.assume_verified) { c.mlirOpPrintingFlagsAssumeVerified(flags); } + if (self.print_name_loc_as_prefix) { + c.mlirOpPrintingFlagsPrintNameLocAsPrefix(flags); + } return flags; } }; @@ -1196,11 +1230,11 @@ pub const Operation = opaque { state.addAttributes(attrs); } if (args.blocks) |blocks| { - const region = Region.init(); + const body_region = Region.init(); for (blocks) |block_| { - region.appendOwnedBlock(block_); + body_region.appendOwnedBlock(block_); } - state.addOwnedRegions(&.{region}); + state.addOwnedRegions(&.{body_region}); } const new_op = try Operation.init(&state); errdefer new_op.deinit(); @@ -1400,6 +1434,7 @@ pub const RankedTensorType = opaque { return get( dims.constSlice(), other.elementType(), + null, ); } @@ -1602,17 +1637,18 @@ pub fn unitAttribute(ctx: *Context) *const Attribute { pub const MemRefType = opaque { const M = Methods(MemRefType, c.MlirType); - pub const isAFn = c.mlirTypeIsAInteger; + pub const isAFn = c.mlirTypeIsAMemRef; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; - pub fn init(element_type: *const Type, shape: []const i64, layout: ?*const Attribute, memory_space: ?*const Attribute) *const MemRefType { + pub fn init(element_type: *const Type, shape: []const i64, layout_attr: ?*const Attribute, memory_space: ?*const Attribute) *const MemRefType { return @ptrCast(c.mlirMemRefTypeGet( element_type.ptr(), @intCast(shape.len), @ptrCast(shape), - if (layout) |l| l.ptr() else .{ .ptr = null }, + if (layout_attr) |l| l.ptr() else .{ .ptr = null }, if (memory_space) |m| m.ptr() else .{ .ptr = null }, ).ptr); } @@ -1630,6 +1666,27 @@ pub const MemRefType = opaque { ); } + pub fn elementType(self: *const MemRefType) *const Type { + return @ptrCast(c.mlirShapedTypeGetElementType(self.ptr()).ptr); + } + + pub fn rank(self: *const MemRefType) usize { + return @intCast(c.mlirShapedTypeGetRank(self.ptr())); + } + + pub fn dimension(self: *const MemRefType, dim: usize) i64 { + return c.mlirShapedTypeGetDimSize(self.ptr(), @intCast(dim)); + } + + pub fn layout(self: *const MemRefType) *const Attribute { + return @ptrCast(c.mlirMemRefTypeGetLayout(self.ptr()).ptr); + } + + pub fn memorySpace(self: *const MemRefType) ?*const Attribute { + const a = c.mlirMemRefTypeGetMemorySpace(self.ptr()); + return if (a.ptr == null) null else @ptrCast(a.ptr); + } + pub fn shaped(self: *const MemRefType) *const ShapedType { return @ptrCast(self); } @@ -1638,3 +1695,243 @@ pub const MemRefType = opaque { pub fn memRefType(element_type: *const Type, shape: []const i64, layout: ?*const Attribute, memory_space: ?*const Attribute) *const Type { return @ptrCast(MemRefType.init(element_type, shape, layout, memory_space)); } + +pub const VectorType = opaque { + const M = Methods(VectorType, c.MlirType); + + pub const isAFn = c.mlirTypeIsAVector; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + /// Plain (non-scalable) vector type. + pub fn init(shape: []const i64, element_type: *const Type) *const VectorType { + return @ptrCast(c.mlirVectorTypeGet( + @intCast(shape.len), + @ptrCast(shape), + element_type.ptr(), + ).ptr); + } + + /// Vector type with explicit per-dim scalability flags. `scalable.len` + /// must equal `shape.len`. + pub fn initScalable( + shape: []const i64, + scalable: []const bool, + element_type: *const Type, + ) *const VectorType { + std.debug.assert(shape.len == scalable.len); + return @ptrCast(c.mlirVectorTypeGetScalable( + @intCast(shape.len), + @ptrCast(shape), + @ptrCast(scalable), + element_type.ptr(), + ).ptr); + } + + pub fn elementType(self: *const VectorType) *const Type { + return @ptrCast(c.mlirShapedTypeGetElementType(self.ptr()).ptr); + } + + pub fn rank(self: *const VectorType) usize { + return @intCast(c.mlirShapedTypeGetRank(self.ptr())); + } + + pub fn dimension(self: *const VectorType, dim: usize) i64 { + return c.mlirShapedTypeGetDimSize(self.ptr(), @intCast(dim)); + } + + pub fn shaped(self: *const VectorType) *const ShapedType { + return @ptrCast(self); + } +}; + +pub fn vectorType(shape: []const i64, element_type: *const Type) *const Type { + return @ptrCast(VectorType.init(shape, element_type)); +} + +pub const UnrankedMemRefType = opaque { + const M = Methods(UnrankedMemRefType, c.MlirType); + + pub const isAFn = c.mlirTypeIsAUnrankedMemRef; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn init(element_type: *const Type, memory_space: ?*const Attribute) *const UnrankedMemRefType { + return @ptrCast(c.mlirUnrankedMemRefTypeGet( + element_type.ptr(), + if (memory_space) |m| m.ptr() else .{ .ptr = null }, + ).ptr); + } +}; + +pub fn unrankedMemRefType(element_type: *const Type, memory_space: ?*const Attribute) *const Type { + return @ptrCast(UnrankedMemRefType.init(element_type, memory_space)); +} + +pub const AffineExpr = opaque { + const M = Methods(AffineExpr, c.MlirAffineExpr); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAffineExprEqual); + pub const format = M.format(c.mlirAffineExprPrint); + + pub fn dim(ctx: *Context, position: usize) *const AffineExpr { + return @ptrCast(c.mlirAffineDimExprGet(ctx.ptr(), @intCast(position)).ptr); + } + + pub fn symbol(ctx: *Context, position: usize) *const AffineExpr { + return @ptrCast(c.mlirAffineSymbolExprGet(ctx.ptr(), @intCast(position)).ptr); + } + + pub fn constant(ctx: *Context, value: i64) *const AffineExpr { + return @ptrCast(c.mlirAffineConstantExprGet(ctx.ptr(), value).ptr); + } + + pub fn add(self: *const AffineExpr, other: *const AffineExpr) *const AffineExpr { + return @ptrCast(c.mlirAffineAddExprGet(self.ptr(), other.ptr()).ptr); + } + + pub fn mul(self: *const AffineExpr, other: *const AffineExpr) *const AffineExpr { + return @ptrCast(c.mlirAffineMulExprGet(self.ptr(), other.ptr()).ptr); + } + + pub fn floorDiv(self: *const AffineExpr, other: *const AffineExpr) *const AffineExpr { + return @ptrCast(c.mlirAffineFloorDivExprGet(self.ptr(), other.ptr()).ptr); + } + + pub fn ceilDiv(self: *const AffineExpr, other: *const AffineExpr) *const AffineExpr { + return @ptrCast(c.mlirAffineCeilDivExprGet(self.ptr(), other.ptr()).ptr); + } + + pub fn mod(self: *const AffineExpr, other: *const AffineExpr) *const AffineExpr { + return @ptrCast(c.mlirAffineModExprGet(self.ptr(), other.ptr()).ptr); + } +}; + +pub const AffineMap = opaque { + const M = Methods(AffineMap, c.MlirAffineMap); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAffineMapEqual); + pub const format = M.format(c.mlirAffineMapPrint); + + /// Empty `() -> ()` map. + pub fn empty(ctx: *Context) *const AffineMap { + return @ptrCast(c.mlirAffineMapEmptyGet(ctx.ptr()).ptr); + } + + /// Zero-result map: `(d0, d1, ...) -> ()`. + pub fn zeroResult(ctx: *Context, dim_count: usize, symbol_count: usize) *const AffineMap { + return @ptrCast(c.mlirAffineMapZeroResultGet( + ctx.ptr(), + @intCast(dim_count), + @intCast(symbol_count), + ).ptr); + } + + /// Identity map: `(d0, d1, ..., dN-1) -> (d0, d1, ..., dN-1)`. + pub fn multiDimIdentity(ctx: *Context, num_dims: usize) *const AffineMap { + return @ptrCast(c.mlirAffineMapMultiDimIdentityGet(ctx.ptr(), @intCast(num_dims)).ptr); + } + + /// Build a map from explicit results. + pub fn get( + ctx: *Context, + dim_count: usize, + symbol_count: usize, + results: []const *const AffineExpr, + ) *const AffineMap { + return @ptrCast(c.mlirAffineMapGet( + ctx.ptr(), + @intCast(dim_count), + @intCast(symbol_count), + @intCast(results.len), + @ptrCast(@constCast(results.ptr)), + ).ptr); + } + + pub fn numDims(self: *const AffineMap) usize { + return @intCast(c.mlirAffineMapGetNumDims(self.ptr())); + } + + pub fn numSymbols(self: *const AffineMap) usize { + return @intCast(c.mlirAffineMapGetNumSymbols(self.ptr())); + } + + pub fn numResults(self: *const AffineMap) usize { + return @intCast(c.mlirAffineMapGetNumResults(self.ptr())); + } +}; + +/// AffineMapAttr — `affine_map<(d0,...) -> (...)>` wrapped as an attribute. +pub const AffineMapAttribute = opaque { + const M = Methods(AffineMapAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAAffineMap; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(map: *const AffineMap) *const AffineMapAttribute { + return @ptrCast(c.mlirAffineMapAttrGet(map.ptr()).ptr); + } + + pub fn value(self: *const AffineMapAttribute) *const AffineMap { + return @ptrCast(c.mlirAffineMapAttrGetValue(self.ptr()).ptr); + } +}; + +pub fn affineMapAttribute(map: *const AffineMap) *const Attribute { + return @ptrCast(AffineMapAttribute.init(map)); +} + +pub fn parseAffineMapAttribute(ctx: *Context, src: []const u8) Error!*const Attribute { + return Attribute.parse(ctx, src); +} + +pub const IntegerSet = opaque { + const M = Methods(IntegerSet, c.MlirIntegerSet); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirIntegerSetEqual); + pub const format = M.format(c.mlirIntegerSetPrint); + + pub fn get( + ctx: *Context, + num_dims: usize, + num_symbols: usize, + constraints: []const *const AffineExpr, + eqs: []const bool, + ) *const IntegerSet { + std.debug.assert(constraints.len == eqs.len); + return @ptrCast(c.mlirIntegerSetGet( + ctx.ptr(), + @intCast(num_dims), + @intCast(num_symbols), + @intCast(constraints.len), + @ptrCast(constraints), + @ptrCast(eqs), + ).ptr); + } +}; + +pub const IntegerSetAttribute = opaque { + const M = Methods(IntegerSetAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAIntegerSet; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(set: *const IntegerSet) *const IntegerSetAttribute { + return @ptrCast(c.mlirIntegerSetAttrGet(set.ptr()).ptr); + } +}; + +pub fn integerSetAttribute(set: *const IntegerSet) *const Attribute { + return @ptrCast(IntegerSetAttribute.init(set)); +} diff --git a/mlir/register.cc b/mlir/register.cc index 0b5ef061..ee6b5dfe 100644 --- a/mlir/register.cc +++ b/mlir/register.cc @@ -1,6 +1,13 @@ +#include "mlir.h" + #include +#include +#include #include extern "C" void mlirRegisterFuncExtensions(MlirDialectRegistry registry) { mlir::func::registerAllExtensions(*unwrap(registry)); } + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Affine, affine, + mlir::affine::AffineDialect) diff --git a/platforms/tpu/BUILD.bazel b/platforms/tpu/BUILD.bazel index c5159842..3c1a8dff 100644 --- a/platforms/tpu/BUILD.bazel +++ b/platforms/tpu/BUILD.bazel @@ -1,8 +1,5 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_python//python:py_binary.bzl", "py_binary") -load("@rules_python//python/uv:lock.bzl", uv_lock = "lock") -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") -load("@zml//bazel:runfiles.bzl", "runfiles_to_default") +load("@rules_zig//zig:defs.bzl", "zig_library") cc_library( name = "empty", @@ -23,10 +20,6 @@ zig_library( import_name = "platforms/tpu", main = "tpu.zig", visibility = ["//visibility:public"], - data = select({ - "//platforms:tpu.enabled": [":gen_ir_files"], - "//conditions:default": [], - }), deps = [ "//bazel", "//pjrt", @@ -39,69 +32,14 @@ zig_library( }), ) -# bazel run //platforms/tpu:requirements.update -uv_lock( - name = "requirements", - srcs = ["requirements.in"], - out = "requirements.lock.txt", - args = [ - "--emit-index-url", - "--emit-find-links", - "--index-strategy=unsafe-best-match", - "--upgrade", - "--universal", - "--prerelease=allow", - ], - python_version = "3.12", +zig_library( + name = "ragged_paged", tags = ["manual"], -) - -py_binary( - name = "gen_ir", - srcs = ["gen_ir.py", "kernel.py"], - main = "gen_ir.py", - python_version = "3.12", - tags = ["manual"], - target_compatible_with = ["@platforms//os:linux"], - deps = [ - "@tpu_py_deps//jax", - "@tpu_py_deps//absl_py", - # "@tpu_py_deps//tpu_inference", - ], -) - -runfiles_to_default( - name = "gen_ir_files", - target_compatible_with = ["@platforms//os:linux"], + import_name = "platforms/tpu/ragged_paged", + main = "ragged_paged.zig", visibility = ["//visibility:public"], deps = [ - ":gen_ir", - ":gen_ir_zig", - ], -) - -zig_binary( - name = "gen_ir_zig", - data = [":gen_ir"], - linkopts = select({ - "@platforms//os:macos": ["-Wl,-rpath,@executable_path/../lib"], - "//conditions:default": ["-Wl,-rpath,$$ORIGIN/../lib"], - }), - main = "gen_ir.zig", - tags = ["manual"], - target_compatible_with = ["@platforms//os:linux"], - deps = [ - ":libpython", - "//bazel", - "@rules_zig//zig/runfiles", - ], -) - -cc_library( - name = "libpython", - hdrs = ["libpython.h"], - deps = [ - "@rules_python//python/cc:current_py_cc_headers", - "@rules_python//python/cc:current_py_cc_libs", + "//kernels/mosaic_tpu:mosaic_tpu_builder", + "//mlir", ], ) diff --git a/platforms/tpu/gen_ir.py b/platforms/tpu/gen_ir.py deleted file mode 100644 index 8e94f88f..00000000 --- a/platforms/tpu/gen_ir.py +++ /dev/null @@ -1,636 +0,0 @@ -import contextlib -import json -import math -import os -import signal -import sys - -# This IR generator only needs CPU lowering. Forcing CPU here avoids JAX -# probing local TPU runtime state and emitting a fallback warning. -os.environ.setdefault("JAX_PLATFORMS", "cpu") - -import jax -import jax.numpy as jnp -from jax._src.interpreters import mlir -from jax._src.pallas import pallas_call -from jax.experimental import pallas as pallas_mod -from jax.experimental.pallas.ops.tpu.flash_attention import BlockSizes, flash_attention -from jax.experimental.pallas.ops.tpu.paged_attention import paged_attention -from kernel import ragged_paged_attention - -RAGGED_KERNEL_ENV = "ZML_TPU_RAGGED_KERNEL" -_TPU_INFERENCE_RAGGED_PATCHED = False - - -def _cpu_lowering_tpu_wrapper(ctx, *in_nodes, **params): - # Reuse TPU lowering on CPU-only envs used by this IR generator. - params.pop("backend", None) - params.pop("which_linear", None) - if "out_shapes" in params: - params["out_avals"] = params.pop("out_shapes") - from jax._src.pallas.mosaic.pallas_call_registration import ( - pallas_call_tpu_lowering_rule, - ) - - return pallas_call_tpu_lowering_rule(ctx, *in_nodes, **params) - - -mlir.register_lowering( - pallas_call.pallas_call_p, - _cpu_lowering_tpu_wrapper, - platform="cpu", -) - - -def _abstract_tpu_device_kind() -> str: - return ( - os.environ.get("ZML_TPU_IR_DEVICE_KIND") - or os.environ.get("TPU_DEVICE_KIND") - or "TPU v5 lite" - ) - - -def _abstract_tpu_num_cores() -> int: - raw = os.environ.get("ZML_TPU_IR_NUM_CORES", "1") - try: - cores = int(raw) - except ValueError: - cores = 1 - return max(1, cores) - - -def _tpu_version_from_kind(device_kind: str) -> int: - kind = device_kind.strip() - if "TPU" not in kind: - return -1 - if kind == "TPU7x": - return 7 - if kind.endswith(" lite"): - kind = kind[: -len(" lite")] - if not kind.startswith("TPU v"): - return -1 - version = [] - for ch in kind[len("TPU v"):]: - if not ch.isdigit(): - break - version.append(ch) - if not version: - return -1 - return int("".join(version)) - - -def _tpu_device_name_from_kind(device_kind: str, num_devices: int | None = None) -> str: - parts = device_kind.split() - name = " ".join(parts[:2]) if len(parts) >= 2 else device_kind - if num_devices is not None: - name += f"-{num_devices}" - return name - - -def _tpu_inference_device_name(num_devices: int | None = None) -> str: - kind = _abstract_tpu_device_kind() - if "TPU" not in kind: - return kind - suffix = "" - if kind.endswith(" lite"): - kind = kind[: -len(" lite")] - suffix = "e" - elif kind.endswith("e"): - kind = kind[:-1] - suffix = "e" - elif kind.endswith("p"): - kind = kind[:-1] - suffix = "p" - elif kind == "TPU7x": - kind = "TPU v7" - if not kind.startswith("TPU v"): - return kind - kind += suffix - if num_devices is not None: - kind += f"-{num_devices}" - return kind - - -def _ensure_vllm_logger_shim() -> None: - if "vllm.logger" in sys.modules: - return - import logging - import types - - vllm_mod = sys.modules.get("vllm") - if vllm_mod is None: - vllm_mod = types.ModuleType("vllm") - vllm_mod.__path__ = [] - sys.modules["vllm"] = vllm_mod - - logger_mod = types.ModuleType("vllm.logger") - - class _VllmLogger(logging.Logger): - def info_once(self, msg, *args, **kwargs): - self.info(msg, *args, **kwargs) - - def warning_once(self, msg, *args, **kwargs): - self.warning(msg, *args, **kwargs) - - def warn_once(self, msg, *args, **kwargs): - self.warning(msg, *args, **kwargs) - - def error_once(self, msg, *args, **kwargs): - self.error(msg, *args, **kwargs) - - def debug_once(self, msg, *args, **kwargs): - self.debug(msg, *args, **kwargs) - - def init_logger(name: str) -> _VllmLogger: - logger = logging.getLogger(name) - if not isinstance(logger, _VllmLogger): - logger = _VllmLogger(name) - logging.Logger.manager.loggerDict[name] = logger - return logger - - logger_mod._VllmLogger = _VllmLogger - logger_mod.init_logger = init_logger - sys.modules["vllm.logger"] = logger_mod - setattr(vllm_mod, "logger", logger_mod) - - -def _patch_tpu_inference_ragged_v3_device_probe() -> None: - global _TPU_INFERENCE_RAGGED_PATCHED - if _TPU_INFERENCE_RAGGED_PATCHED: - return - - _ensure_vllm_logger_shim() - - try: - from tpu_inference.kernels.ragged_paged_attention.v3 import kernel as v3_kernel - from tpu_inference.kernels.ragged_paged_attention.v3 import util as v3_util - except Exception: - return - try: - from tpu_inference.kernels.ragged_paged_attention.v3 import tuned_block_sizes as v3_tuned - except Exception: - v3_tuned = None - try: - from tpu_inference.kernels.ragged_paged_attention.v3 import tuned_block_sizes_hd64 as v3_tuned_hd64 - except Exception: - v3_tuned_hd64 = None - try: - from tpu_inference import utils as tpu_utils - except Exception: - tpu_utils = None - - tpu_version_fn = lambda: _tpu_version_from_kind(_abstract_tpu_device_kind()) - device_name_fn = lambda num_devices=None: _tpu_inference_device_name(num_devices) - - v3_util.get_tpu_version = tpu_version_fn - v3_kernel.get_tpu_version = tpu_version_fn - if v3_tuned is not None: - v3_tuned.get_tpu_version = tpu_version_fn - if hasattr(v3_tuned, "get_device_name"): - v3_tuned.get_device_name = device_name_fn - if v3_tuned_hd64 is not None: - v3_tuned_hd64.get_tpu_version = tpu_version_fn - if hasattr(v3_tuned_hd64, "get_device_name"): - v3_tuned_hd64.get_device_name = device_name_fn - if tpu_utils is not None: - tpu_utils.get_device_name = device_name_fn - - _TPU_INFERENCE_RAGGED_PATCHED = True - - -def _effective_backend_kind(requested_backend_kind: str) -> str: - if requested_backend_kind != "ragged_paged": - return requested_backend_kind - - raw_selector = os.environ.get(RAGGED_KERNEL_ENV) - selector = (raw_selector or "").strip().lower() - - if selector in ("", "mosaic", "jax"): - return "ragged_paged" - if selector in ("v3", "tpu_inference", "vllm", "v2"): - return "ragged_paged_v3" - - raise ValueError( - f"Invalid {RAGGED_KERNEL_ENV}={raw_selector!r}; expected one of: mosaic, v3" - ) - - -def _patch_ragged_tpu_tuning_device_probe() -> None: - # The tuned block-size helper probes jax.devices()[0] directly, which is - # CPU in this process by design. Override it with our abstract TPU kind. - from jax.experimental.pallas.ops.tpu.ragged_paged_attention import tuned_block_sizes - - tuned_block_sizes.get_tpu_version = lambda: _tpu_version_from_kind(_abstract_tpu_device_kind()) - tuned_block_sizes.get_device_name = ( - lambda num_devices=None: _tpu_device_name_from_kind(_abstract_tpu_device_kind(), num_devices) - ) - - -@contextlib.contextmanager -def _tpu_abstract_mesh_context(): - # Force TPU chip metadata lookup to use an abstract TPU device while we - # still lower on CPU-only JAX backends. - mesh = jax.sharding.AbstractMesh( - axis_sizes=(1,), - axis_names=("tpu_core",), - abstract_device=jax.sharding.AbstractDevice( - device_kind=_abstract_tpu_device_kind(), - num_cores=_abstract_tpu_num_cores(), - ), - ) - with jax.sharding.use_abstract_mesh(mesh): - yield - - -_patch_ragged_tpu_tuning_device_probe() - - -def _extract_backend_configs(op) -> list[str]: - configs: list[str] = [] - backend = op.attributes.get("backend_config") - if backend is not None: - configs.append(backend.value if hasattr(backend, "value") else str(backend)) - for region in op.regions: - for block in region.blocks: - for sub_op in block.operations: - configs.extend(_extract_backend_configs(sub_op)) - return configs - - -def _extract_backend_configs_with_kernel_name(op) -> list[tuple[str | None, str]]: - configs: list[tuple[str | None, str]] = [] - backend = op.attributes.get("backend_config") - if backend is not None: - kernel_attr = op.attributes.get("kernel_name") - kernel_name: str | None = None - if kernel_attr is not None: - kernel_name = kernel_attr.value if hasattr(kernel_attr, "value") else str(kernel_attr) - configs.append((kernel_name, backend.value if hasattr(backend, "value") else str(backend))) - for region in op.regions: - for block in region.blocks: - for sub_op in block.operations: - configs.extend(_extract_backend_configs_with_kernel_name(sub_op)) - return configs - - -def _extract_backend_config(op) -> str | None: - configs = _extract_backend_configs(op) - return configs[0] if configs else None - - -def flash_attention_on_tpu(kernel_params: dict) -> str | None: - batch_size = kernel_params["batch_size"] - num_heads = kernel_params["num_heads"] - q_seq_len = kernel_params["q_seq_len"] - kv_seq_len = kernel_params["kv_seq_len"] - d_model = kernel_params["d_model"] - - q = jax.ShapeDtypeStruct((batch_size, num_heads, q_seq_len, d_model), jnp.bfloat16) - k = jax.ShapeDtypeStruct((batch_size, num_heads, kv_seq_len, d_model), jnp.bfloat16) - v = jax.ShapeDtypeStruct((batch_size, num_heads, kv_seq_len, d_model), jnp.bfloat16) - - block = BlockSizes( - block_q=min(128, q_seq_len), - block_k_major=min(128, kv_seq_len), - block_k=min(128, kv_seq_len), - block_b=1, - ) - sm_scale = kernel_params.get("sm_scale", 1.0 / math.sqrt(d_model)) - - with _tpu_abstract_mesh_context(): - lowered = jax.jit( - flash_attention, - backend="cpu", - static_argnames=["block_sizes", "causal", "sm_scale"], - ).lower(q, k, v, block_sizes=block, causal=True, sm_scale=sm_scale) - configs_with_name = _extract_backend_configs_with_kernel_name(lowered.compiler_ir().operation) - for kernel_name, cfg in configs_with_name: - if kernel_name and "RPAm" in kernel_name: - return cfg - configs = [cfg for _, cfg in configs_with_name] - for cfg in configs: - if "RPAm" in cfg: - return cfg - return configs[-1] if configs else None - - -def _fix_paged_attention_acc_specs(fn, **kwargs): - grid_spec = kwargs.get("grid_spec") - out_shape = kwargs.get("out_shape") - out_specs = getattr(grid_spec, "out_specs", None) - - if ( - isinstance(out_specs, tuple) - and isinstance(out_shape, list) - and len(out_specs) == 3 - and len(out_shape) == 3 - and out_shape[1].shape[-1] == 1 - ): - q_block_shape = out_specs[0].block_shape - if q_block_shape is not None and q_block_shape[-1] not in (None, 1): - fixed = pallas_mod.BlockSpec((*q_block_shape[:-1], 1), out_specs[0].index_map) - grid_spec.out_specs = (out_specs[0], fixed, fixed) - - return _ORIG_PALLAS_CALL(fn, **kwargs) - - -def paged_attention_on_tpu(kernel_params: dict) -> str | None: - batch_size = kernel_params["batch_size"] - pages_per_sequence = kernel_params["pages_per_sequence"] - dtype = jnp.float32 - - q = jax.ShapeDtypeStruct((batch_size, kernel_params["num_q_heads"], kernel_params["head_dim"]), dtype) - k = jax.ShapeDtypeStruct( - (kernel_params["num_kv_heads"], kernel_params["total_num_pages"], kernel_params["page_size"], kernel_params["head_dim"]), - jnp.bfloat16, - ) - v = jax.ShapeDtypeStruct( - (kernel_params["num_kv_heads"], kernel_params["total_num_pages"], kernel_params["page_size"], kernel_params["head_dim"]), - jnp.bfloat16, - ) - lengths = jax.ShapeDtypeStruct((batch_size,), jnp.int32) - page_indices_flat = jax.ShapeDtypeStruct((batch_size * pages_per_sequence,), jnp.int32) - - def _paged_attention_with_flat_indices(q, k, v, lengths, page_indices_flat, *, pages_per_compute_block): - indices = page_indices_flat.reshape(batch_size, pages_per_sequence) - return paged_attention(q, k, v, lengths, indices, pages_per_compute_block=pages_per_compute_block) - - comp_target = jax.jit(_paged_attention_with_flat_indices, static_argnames=["pages_per_compute_block"]) - - pallas_mod.pallas_call = _fix_paged_attention_acc_specs - try: - with _tpu_abstract_mesh_context(): - lowered = comp_target.lower( - q, - k, - v, - lengths, - page_indices_flat, - pages_per_compute_block=kernel_params["pages_per_compute_block"], - ) - finally: - pallas_mod.pallas_call = _ORIG_PALLAS_CALL - - return _extract_backend_config(lowered.compiler_ir().operation) - - -def _parse_jnp_dtype(dtype_name: str): - allowed_dtypes = { - "jnp.bfloat16": jnp.bfloat16, - "jnp.float32": jnp.float32, - } - try: - return allowed_dtypes[dtype_name] - except KeyError as exc: - raise ValueError(f"Unsupported dtype: {dtype_name}") from exc - - -def ragged_paged_attention_on_tpu(kernel_params: dict) -> str | None: - q_dtype = _parse_jnp_dtype(kernel_params["q_dtype"]) - kv_dtype = _parse_jnp_dtype(kernel_params["kv_dtype"]) - q = jax.ShapeDtypeStruct( - ( - kernel_params["num_q_tokens"], - kernel_params["num_q_heads"], - kernel_params["head_dim"], - ), - q_dtype, - ) - kv_pages = jax.ShapeDtypeStruct( - ( - kernel_params["total_num_pages"], - kernel_params["page_size"], - kernel_params["num_kv_heads"] * 2, - kernel_params["head_dim"], - ), - kv_dtype, - ) - kv_lens = jax.ShapeDtypeStruct((kernel_params["max_num_seqs"],), jnp.int32) - page_indices = jax.ShapeDtypeStruct( - (kernel_params["max_num_seqs"], kernel_params["pages_per_seq"]), - jnp.int32, - ) - cu_q_lens = jax.ShapeDtypeStruct((kernel_params["max_num_seqs"] + 1,), jnp.int32) - num_seqs = jax.ShapeDtypeStruct((1,), jnp.int32) - - with _tpu_abstract_mesh_context(): - lowered = jax.jit( - ragged_paged_attention, - backend="cpu", - static_argnames=[ - "sm_scale", - "sliding_window", - "soft_cap", - "mask_value", - "k_scale", - "v_scale", - "num_kv_pages_per_block", - "num_queries_per_block", - "vmem_limit_bytes", - ], - ).lower( - q, - kv_pages, - kv_lens, - page_indices, - cu_q_lens, - num_seqs, - sm_scale=kernel_params.get("sm_scale", 1.0), - sliding_window=kernel_params.get("sliding_window"), - soft_cap=kernel_params.get("soft_cap"), - mask_value=kernel_params.get("mask_value"), - k_scale=kernel_params.get("k_scale"), - v_scale=kernel_params.get("v_scale"), - num_kv_pages_per_block=kernel_params.get("num_kv_pages_per_block"), - num_queries_per_block=kernel_params.get("num_queries_per_block"), - vmem_limit_bytes=kernel_params.get("vmem_limit_bytes"), - ) - return _extract_backend_config(lowered.compiler_ir().operation) - - -def ragged_paged_attention_v3_on_tpu(kernel_params: dict) -> str | None: - _ensure_vllm_logger_shim() - try: - from tpu_inference.kernels.ragged_paged_attention.v3 import kernel as v3_kernel - except Exception as exc: - raise ValueError(f"Failed to import tpu-inference v3 ragged kernel: {exc}") - - _patch_tpu_inference_ragged_v3_device_probe() - - q_dtype = _parse_jnp_dtype(kernel_params["q_dtype"]) - kv_dtype = _parse_jnp_dtype(kernel_params["kv_dtype"]) - - num_q_tokens = kernel_params["num_q_tokens"] - num_q_heads = kernel_params["num_q_heads"] - num_kv_heads = kernel_params["num_kv_heads"] - head_dim = kernel_params["head_dim"] - total_num_pages = kernel_params["total_num_pages"] - page_size = kernel_params["page_size"] - max_num_seqs = kernel_params["max_num_seqs"] - pages_per_seq = kernel_params["pages_per_seq"] - - kv_cache_shape = v3_kernel.get_kv_cache_shape( - total_num_pages, - page_size, - num_kv_heads, - head_dim, - kv_dtype, - ) - - q = jax.ShapeDtypeStruct((num_q_tokens, num_q_heads, head_dim), q_dtype) - k = jax.ShapeDtypeStruct((num_q_tokens, num_kv_heads, head_dim), kv_dtype) - v = jax.ShapeDtypeStruct((num_q_tokens, num_kv_heads, head_dim), kv_dtype) - kv_cache = jax.ShapeDtypeStruct(kv_cache_shape, kv_dtype) - kv_lens = jax.ShapeDtypeStruct((max_num_seqs,), jnp.int32) - page_indices = jax.ShapeDtypeStruct((max_num_seqs * pages_per_seq,), jnp.int32) - cu_q_lens = jax.ShapeDtypeStruct((max_num_seqs + 1,), jnp.int32) - distribution = jax.ShapeDtypeStruct((3,), jnp.int32) - - def _block_sizes(name: str): - value = kernel_params.get(name) - if value is None: - return None - if isinstance(value, (list, tuple)) and len(value) == 4: - return tuple(int(v) for v in value) - raise ValueError(f"Invalid {name}: expected 4-element list or tuple") - - out_dtype = kernel_params.get("out_dtype") - if isinstance(out_dtype, str): - out_dtype = _parse_jnp_dtype(out_dtype) - - with _tpu_abstract_mesh_context(): - lowered = jax.jit( - v3_kernel.ragged_paged_attention, - backend="cpu", - static_argnames=[ - "use_causal_mask", - "skip_kv_mask", - "sm_scale", - "sliding_window", - "soft_cap", - "out_dtype", - "mask_value", - "q_scale", - "k_scale", - "v_scale", - "chunk_prefill_size", - "d_block_sizes", - "p_block_sizes", - "m_block_sizes", - "vmem_limit_bytes", - "debug_mode", - "disable_bounds_checks", - "disable_semaphore_checks", - ], - ).lower( - q, - k, - v, - kv_cache, - kv_lens, - page_indices, - cu_q_lens, - distribution, - use_causal_mask=kernel_params.get("use_causal_mask", True), - skip_kv_mask=kernel_params.get("skip_kv_mask", False), - sm_scale=kernel_params.get("sm_scale", 1.0), - sliding_window=kernel_params.get("sliding_window"), - soft_cap=kernel_params.get("soft_cap"), - out_dtype=out_dtype, - mask_value=kernel_params.get("mask_value"), - q_scale=kernel_params.get("q_scale"), - k_scale=kernel_params.get("k_scale"), - v_scale=kernel_params.get("v_scale"), - chunk_prefill_size=kernel_params.get("chunk_prefill_size"), - d_block_sizes=_block_sizes("d_block_sizes"), - p_block_sizes=_block_sizes("p_block_sizes"), - m_block_sizes=_block_sizes("m_block_sizes"), - vmem_limit_bytes=kernel_params.get("vmem_limit_bytes"), - debug_mode=kernel_params.get("debug_mode", False), - disable_bounds_checks=kernel_params.get("disable_bounds_checks", True), - disable_semaphore_checks=kernel_params.get("disable_semaphore_checks", True), - ) - - configs_with_names = _extract_backend_configs_with_kernel_name(lowered.compiler_ir().operation) - if os.environ.get("ZML_TPU_IR_DEBUG"): - print(f"debug: v3 custom call configs={len(configs_with_names)}", file=sys.stderr) - for idx, (name, cfg) in enumerate(configs_with_names): - snippet = cfg[:160].replace("\n", " ") - print( - f"debug: cfg[{idx}] kernel_name={name!r} len={len(cfg)} snippet={snippet!r}", - file=sys.stderr, - ) - - if configs_with_names: - for name, cfg in configs_with_names: - if name and "RPAm" in name: - return cfg - # If kernel_name isn't present or doesn't mention RPAm, use the last - # custom call (mixed is emitted last in v3). - return configs_with_names[-1][1] - - configs = _extract_backend_configs(lowered.compiler_ir().operation) - if not configs: - return None - return configs[-1] - - -def handle_sigint(signum, frame): - sys.exit(0) - - -def main(): - signal.signal(signal.SIGINT, handle_sigint) - - for raw_line in sys.stdin: - line = raw_line.strip() - if not line: - continue - - try: - request = json.loads(line) - if not isinstance(request, dict): - raise ValueError("request must be a JSON object") - - backend_kind = _effective_backend_kind(request["backend_config"]) - params = request["params"] - - if backend_kind not in {"paged", "flash", "ragged_paged", "ragged_paged_v3"}: - raise ValueError(f"Unsupported backend_config: {backend_kind}") - if not isinstance(params, dict): - raise ValueError("Invalid request: 'params' must be a JSON object") - - tpu_code = None - - if backend_kind == "ragged_paged": - tpu_code = ragged_paged_attention_on_tpu(params) - if tpu_code is None: - raise ValueError("Failed to extract TPU backend config from ragged paged attention IR") - if backend_kind == "ragged_paged_v3": - tpu_code = ragged_paged_attention_v3_on_tpu(params) - if tpu_code is None: - raise ValueError("Failed to extract TPU backend config from v3 ragged paged attention IR") - if backend_kind == "paged": - # tpu_code = paged_attention_on_tpu(params) - # if tpu_code is None: - # raise ValueError("Failed to extract TPU backend config from paged attention IR") - raise ValueError(f"Unsupported backend_config: {backend_kind} for now") - if backend_kind == "flash": - # tpu_code = flash_attention_on_tpu(params) - # if tpu_code is None: - # raise ValueError("Failed to extract TPU backend config from flash attention IR") - raise ValueError(f"Unsupported backend_config: {backend_kind} for now") - - if tpu_code is None: - raise ValueError(f"Unsupported backend_config: {backend_kind}") - response = {"ok": True, "result": tpu_code} - except Exception as exc: - response = {"ok": False, "error": str(exc)} - - print(json.dumps(response), flush=True) - - -_ORIG_PALLAS_CALL = pallas_mod.pallas_call - -if __name__ == "__main__": - main() diff --git a/platforms/tpu/gen_ir.zig b/platforms/tpu/gen_ir.zig deleted file mode 100644 index 199ed67c..00000000 --- a/platforms/tpu/gen_ir.zig +++ /dev/null @@ -1,117 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); - -fn pyStatusCheck(status: c.PyStatus) void { - if (c.PyStatus_Exception(status) != 0) { - if (c.PyStatus_IsExit(status) != 0) { - std.process.exit(@intCast(status.exitcode)); - } - c.Py_ExitStatusException(status); - } -} - -pub fn toPosixPathW(file_path: []const u8) error{ InvalidUtf8, NameTooLong, CodepointTooLarge }![std.posix.PATH_MAX - 1:0]c.wchar_t { - var view = std.unicode.Utf8View.init(file_path) catch return error.InvalidUtf8; - var it = view.iterator(); - - var path_with_null: [std.posix.PATH_MAX - 1:0]c.wchar_t = undefined; - var len: usize = 0; - while (it.nextCodepoint()) |cp| { - if (len + 1 >= path_with_null.len) return error.NameTooLong; - path_with_null[len] = std.math.cast(c.wchar_t, cp) orelse return error.CodepointTooLarge; - len += 1; - } - path_with_null[len] = 0; - return path_with_null; -} - -pub fn main(init: std.process.Init.Minimal) !void { - const allocator = std.heap.c_allocator; - - var threaded: std.Io.Threaded = .init(allocator, .{}); - defer threaded.deinit(); - - const io = threaded.io(); - - { - var preconfig: c.PyPreConfig = undefined; - c.PyPreConfig_InitIsolatedConfig(&preconfig); - preconfig.utf8_mode = 1; - pyStatusCheck(c.Py_PreInitialize(&preconfig)); - } - - var config: c.PyConfig = undefined; - c.PyConfig_InitIsolatedConfig(&config); - defer c.PyConfig_Clear(&config); - - config.module_search_paths_set = 1; - config.optimization_level = 2; - config.write_bytecode = 0; - - pyStatusCheck(c.PyConfig_SetBytesArgv(&config, @intCast(init.args.vector.len), @ptrCast(init.args.vector.ptr))); - - var self_exe_dir_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const self_exe_dir = blk: { - const n = try std.process.executableDirPath(io, &self_exe_dir_buf); - break :blk self_exe_dir_buf[0..n]; - }; - - { - var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const home = try std.fmt.bufPrintZ(&buf, "{f}{d}.{d}", .{ - std.Io.Dir.path.fmtJoin(&.{ - self_exe_dir, - "..", - "lib", - "python", - }), - c.PY_MAJOR_VERSION, - c.PY_MINOR_VERSION, - }); - pyStatusCheck(c.PyConfig_SetBytesString(&config, &config.home, home)); - pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(home))); - } - - { - var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const site_packages = try std.fmt.bufPrint(&buf, "{f}", .{ - std.Io.Dir.path.fmtJoin(&.{ - self_exe_dir, - "..", - "site-packages", - }), - }); - pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(site_packages))); - } - - pyStatusCheck(c.Py_InitializeFromConfig(&config)); - defer c.Py_Finalize(); - - const module = c.PyImport_ImportModule("gen_ir"); - if (module == null) { - c.PyErr_Print(); - return error.PythonModuleImportFailed; - } - defer c.Py_DecRef(module); - - const func = c.PyObject_GetAttrString(module, "main"); - if (func == null) { - std.log.err("Python module 'gen_ir' is missing required attribute 'main'", .{}); - c.PyErr_Print(); - return error.PythonAttributeLookupFailed; - } - defer c.Py_DecRef(func); - - if (c.PyCallable_Check(func) == 0) { - std.log.err("Python attribute 'gen_ir.main' is not callable", .{}); - return error.PythonAttributeNotCallable; - } - - const result = c.PyObject_CallNoArgs(func); - if (result == null) { - c.PyErr_Print(); - return error.PythonExecutionFailed; - } - defer c.Py_DecRef(result); -} diff --git a/platforms/tpu/kernel.py b/platforms/tpu/kernel.py deleted file mode 100644 index 97cbfe16..00000000 --- a/platforms/tpu/kernel.py +++ /dev/null @@ -1,910 +0,0 @@ -# Copyright 2025 The JAX Authors. -# Copyright 2026 ZML -# -# Vendored from JAX and modified by ZML. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""TPU-Friendly Ragged Paged Attention kernel. - -This kernel offers a highly optimized implementation of ragged paged attention, -specifically designed for TPU and compatible with a wide range of model -specifications. It supports mixed prefill and decoding, enhancing throughput -during inference. -""" -import functools -import jax -from jax import lax -from jax._src import dtypes -from jax.experimental import pallas as pl -from jax.experimental.pallas import tpu as pltpu -from jax.experimental.pallas.ops.tpu.ragged_paged_attention.tuned_block_sizes import get_tuned_block_sizes -import jax.numpy as jnp - - -DEFAULT_MASK_VALUE = -0.7 * float(jnp.finfo(jnp.dtype("float32")).max) - - -class MultiPageAsyncCopyDescriptor: - """Descriptor for async copy of multiple K/V pages from HBM.""" - - def __init__( - self, - pages_hbm_ref, # [total_num_pages, page_size, num_combined_kv_heads_per_blk, head_dim] - vmem_buf, # [num_kv_pages_per_blk, page_size, num_combined_kv_heads_per_blk, head_dim] - sem, - page_indices_ref, # i32[max_num_seqs, pages_per_seq] - metadata, # [seq_idx, start_page_idx, end_page_idx] - ): - self._vmem_buf = vmem_buf - seq_id, start_page_idx, end_page_idx = metadata - self._async_copies = [] - # TODO(jevinjiang): Only fetch dynamic shape in need! This will insert - # a bunch of if-ops. Check the performance when we have benchmarking setup. - for i in range(vmem_buf.shape[0]): - page_idx = start_page_idx + i - page_idx = jax.lax.select(page_idx < end_page_idx, page_idx, 0) - self._async_copies.append( - pltpu.make_async_copy( - pages_hbm_ref.at[page_indices_ref[seq_id, page_idx]], - vmem_buf.at[i], - sem, - ) - ) - - def start(self): - """Starts the async copies.""" - for async_copy in self._async_copies: - async_copy.start() - - def wait(self): - for async_copy in self._async_copies: - async_copy.wait() - return self._vmem_buf - - -def ref_ragged_paged_attention( - queries: jax.Array, # [max_num_batched_tokens, num_q_heads, head_dim] - kv_pages: jax.Array, # [total_num_pages, page_size, num_combined_kv_heads, head_dim] - kv_lens: jax.Array, # i32[max_num_seqs] - page_indices: jax.Array, # i32[max_num_seqs, pages_per_seq] - cu_q_lens: jax.Array, # i32[max_num_seqs + 1] - num_seqs: jax.Array, # i32[1], - *, - sm_scale: float = 1.0, - sliding_window: int | None = None, - soft_cap: float | None = None, - mask_value: float | None = DEFAULT_MASK_VALUE, - k_scale: float | None = None, - v_scale: float | None = None, -): - static_validate_inputs( - queries, - kv_pages, - kv_lens, - page_indices, - cu_q_lens, - num_seqs, - sm_scale=sm_scale, - k_scale=k_scale, - v_scale=v_scale, - sliding_window=sliding_window, - soft_cap=soft_cap, - mask_value=mask_value, - ) - if mask_value is None: - mask_value = DEFAULT_MASK_VALUE - _, _, num_combined_kv_heads, head_dim = kv_pages.shape - assert num_combined_kv_heads % 2 == 0 - num_kv_heads = num_combined_kv_heads // 2 - num_q_heads = queries.shape[1] - assert num_q_heads % num_kv_heads == 0 - num_query_per_kv = num_q_heads // num_kv_heads - outputs = [] - for i in range(num_seqs[0]): - q_start = cu_q_lens[i] - q_end = cu_q_lens[i + 1] - q_len = q_end - q_start - kv_len = kv_lens[i] - indices = page_indices[i] - q = queries[q_start:q_end] - k = kv_pages[indices, :, 0::2, :].reshape(-1, num_kv_heads, head_dim)[ - :kv_len - ] - v = kv_pages[indices, :, 1::2, :].reshape(-1, num_kv_heads, head_dim)[ - :kv_len - ] - if k_scale is not None: - k = k.astype(jnp.float32) * k_scale - k = k.astype(q.dtype) - if v_scale is not None: - v = v.astype(jnp.float32) * v_scale - v = v.astype(q.dtype) - k = jnp.repeat(k, num_query_per_kv, axis=1) - v = jnp.repeat(v, num_query_per_kv, axis=1) - attn = jnp.einsum("qhd,khd->hqk", q, k, preferred_element_type=jnp.float32) - attn *= sm_scale - q_span = (kv_len - q_len) + jax.lax.broadcasted_iota( - jnp.int32, attn.shape, 1 - ) - kv_span = jax.lax.broadcasted_iota(jnp.int32, attn.shape, 2) - mask = q_span < kv_span - if sliding_window is not None: - mask = jnp.logical_or(mask, q_span - sliding_window >= kv_span) - if soft_cap is not None: - attn = soft_cap * jnp.tanh(attn / soft_cap) - attn += jnp.where(mask, mask_value, 0.0) - attn = jax.nn.softmax(attn, axis=-1).astype(v.dtype) - out = jnp.einsum("hqk,khd->qhd", attn, v).astype(queries.dtype) - outputs.append(out) - - return jnp.concatenate(outputs, axis=0) - - -# Expect to run these checks during runtime. -def dynamic_validate_inputs( - q: jax.Array, # [max_num_batched_tokens, num_q_heads, head_dim] - kv_pages: jax.Array, # [total_num_pages, page_size, num_combined_kv_heads, head_dim] - kv_lens: jax.Array, # i32[max_num_seqs] - page_indices: jax.Array, # i32[max_num_seqs, pages_per_seq] - cu_q_lens: jax.Array, # i32[max_num_seqs + 1] - num_seqs: jax.Array, # i32[1] - *, - # These inputs are optional. If not specified, we will not validate them. - sm_scale: float | None = None, - sliding_window: int | None = None, - soft_cap: float | None = None, - mask_value: float | None = None, - k_scale: float | None = None, - v_scale: float | None = None, - # Kernel tuning params. - num_kv_pages_per_block: int | None = None, - num_queries_per_block: int | None = None, - vmem_limit_bytes: int | None = None, -): - static_validate_inputs( - q, - kv_pages, - kv_lens, - page_indices, - cu_q_lens, - num_seqs, - sm_scale=sm_scale, - sliding_window=sliding_window, - soft_cap=soft_cap, - mask_value=mask_value, - k_scale=k_scale, - v_scale=v_scale, - num_kv_pages_per_block=num_kv_pages_per_block, - num_queries_per_block=num_queries_per_block, - vmem_limit_bytes=vmem_limit_bytes, - ) - max_num_batched_tokens = q.shape[0] - page_size = kv_pages.shape[1] - max_num_seqs, pages_per_seq = page_indices.shape - if num_seqs[0] > max_num_seqs: - raise ValueError(f"{num_seqs[0]=} must be less or equal to {max_num_seqs=}") - max_kv_len = jnp.max(kv_lens) - min_pages_per_seq = pl.cdiv(max_kv_len, page_size) - if pages_per_seq < min_pages_per_seq: - raise ValueError( - f"{pages_per_seq=} must be greater or equal to" - f" {min_pages_per_seq=} given {max_kv_len=} and {page_size=}." - ) - if cu_q_lens[num_seqs[0]] > max_num_batched_tokens: - raise ValueError( - f"Total q tokens {cu_q_lens[num_seqs[0]]} must be less or equal to" - f" {max_num_batched_tokens=}." - ) - for i in range(num_seqs[0]): - q_len = cu_q_lens[i + 1] - cu_q_lens[i] - kv_len = kv_lens[i] - if q_len > kv_len: - raise ValueError( - f"{q_len=} must be less or equal to {kv_len=} at sequence {i}." - ) - - -# Expect to run these checks during compile time. -def static_validate_inputs( - q: jax.Array, # [max_num_batched_tokens, num_q_heads, head_dim] - kv_pages: jax.Array, # [total_num_pages, page_size, num_combined_kv_heads, head_dim] - kv_lens: jax.Array, # i32[max_num_seqs] - page_indices: jax.Array, # i32[max_num_seqs, pages_per_seq] - cu_q_lens: jax.Array, # i32[max_num_seqs + 1] - num_seqs: jax.Array, # i32[1] - *, - # These inputs are optional. If not specified, we will not validate them. - sm_scale: float | None = None, - sliding_window: int | None = None, - soft_cap: float | None = None, - mask_value: float | None = None, - k_scale: float | None = None, - v_scale: float | None = None, - # Kernel tuning params. - num_kv_pages_per_block: int | None = None, - num_queries_per_block: int | None = None, - vmem_limit_bytes: int | None = None, -): - _, num_q_heads, head_dim = q.shape - _, _, num_combined_kv_heads, head_dim_k = kv_pages.shape - assert num_combined_kv_heads % 2 == 0 - assert isinstance(k_scale, float) or k_scale is None - assert isinstance(v_scale, float) or v_scale is None - num_kv_heads = num_combined_kv_heads // 2 - max_num_seqs, pages_per_seq = page_indices.shape - if num_seqs.shape != (1,): - raise ValueError(f"{num_seqs.shape=} must be (1,)") - if head_dim_k != head_dim: - raise ValueError( - f"Q head_dim {head_dim} must be the same as that of K/V {head_dim_k}." - ) - if kv_lens.shape != (max_num_seqs,): - raise ValueError( - f"Expected {kv_lens.shape=} to be ({max_num_seqs},) where" - " `max_num_seqs` is `page_indices.shape[0]`." - ) - if cu_q_lens.shape != (max_num_seqs + 1,): - raise ValueError( - f"Expected {cu_q_lens.shape=} to be ({max_num_seqs + 1},) where" - " `max_num_seqs` is `page_indices.shape[0]`." - ) - if ( - kv_lens.dtype != jnp.int32 - or page_indices.dtype != jnp.int32 - or cu_q_lens.dtype != jnp.int32 - ): - raise ValueError( - "The dtype of `kv_lens`, `page_indices`, and `cu_q_lens` must be" - f" int32. Got {kv_lens.dtype=}, {page_indices.dtype=}," - f" {cu_q_lens.dtype=}." - ) - if num_q_heads % num_kv_heads != 0: - raise ValueError(f"{num_q_heads=} must be divisible by {num_kv_heads=}") - if sliding_window is not None and sliding_window <= 0: - raise ValueError(f"{sliding_window=} must be positive.") - if soft_cap is not None and soft_cap == 0.0: - raise ValueError(f"{soft_cap=} must not be 0.0.") - if ( - num_kv_pages_per_block is not None - and not 0 < num_kv_pages_per_block <= pages_per_seq - ): - raise ValueError( - f"{num_kv_pages_per_block=} must be in range (0, {pages_per_seq}]." - ) - if num_queries_per_block is not None and num_queries_per_block <= 0: - raise ValueError(f"{num_queries_per_block=} must be positive.") - if vmem_limit_bytes is not None and vmem_limit_bytes <= 0: - raise ValueError(f"{vmem_limit_bytes=} must be positive.") - del sm_scale # No constraints on sm_scale. - del mask_value # No consstraints on mask_value. - - -def ragged_paged_attention_kernel( - # Prefetch - kv_lens_ref, # [max_num_seqs] - page_indices_ref, # [max_num_seqs, pages_per_seq] - cu_q_lens_ref, # [max_num_seqs + 1] - seq_buf_idx_ref, - # TODO(jevinjiang): if OOM in SMEM, consider pack to other scalar refs. - num_seqs_ref, - # Input - q_ref, # [num_q_per_blk, num_q_heads_per_blk, head_dim] - kv_pages_hbm_ref, # [total_num_pages, page_size, num_combined_kv_heads, head_dim] - # Output - o_ref, # [num_q_per_blk, num_q_heads_per_blk, head_dim] - # Scratch - kv_bufs, # [2, num_kv_pages_per_blk, page_size, num_combined_kv_heads_per_blk, head_dim] - sems, # [2, 2] - l_ref, # [num_kv_heads_per_blk, num_q_per_blk * num_q_heads_per_kv_head, 128] - m_ref, # [num_kv_heads_per_blk, num_q_per_blk * num_q_heads_per_kv_head, 128] - acc_ref, # [num_q_per_blk, num_q_heads_per_blk, head_dim] - *, - sm_scale: float, - sliding_window: int | None = None, - soft_cap: float | None = None, - mask_value: float | None = DEFAULT_MASK_VALUE, - k_scale: float | None = None, - v_scale: float | None = None, -): - if mask_value is None: - mask_value = DEFAULT_MASK_VALUE - num_q_per_blk, num_q_heads_per_blk, head_dim = q_ref.shape - pages_per_seq = page_indices_ref.shape[-1] - num_seqs = num_seqs_ref[0] - _, num_kv_pages_per_blk, page_size, num_combined_kv_heads_per_blk, _ = ( - kv_bufs.shape - ) - num_kv_heads_per_blk = num_combined_kv_heads_per_blk // 2 - num_kv_per_blk = num_kv_pages_per_blk * page_size - num_q_heads_per_kv_head = num_q_heads_per_blk // num_kv_heads_per_blk - heads_blk_idx, q_blk_idx = ( - pl.program_id(0), - pl.program_id(1), - ) - num_heads_blks = pl.num_programs(0) - init_seq_idx = seq_buf_idx_ref[0] - init_buf_idx = seq_buf_idx_ref[1] - q_len_start = q_blk_idx * num_q_per_blk - q_len_end = q_len_start + num_q_per_blk - - def create_kv_async_copy_descriptors( - heads_blk_idx, seq_idx, kv_blk_idx, buf_idx - ): - start_kv_page_idx = kv_blk_idx * num_kv_pages_per_blk - end_kv_page_idx = jnp.minimum( - pages_per_seq, pl.cdiv(kv_lens_ref[seq_idx], page_size) - ) - metadata = (seq_idx, start_kv_page_idx, end_kv_page_idx) - heads_start = heads_blk_idx * num_combined_kv_heads_per_blk - async_copy_kv = MultiPageAsyncCopyDescriptor( - kv_pages_hbm_ref.at[ - :, :, pl.ds(heads_start, num_combined_kv_heads_per_blk), : - ], - kv_bufs.at[buf_idx], - sems.at[buf_idx], - page_indices_ref, - metadata, - ) - return async_copy_kv - - def strided_load_kv(ref, start, step): - packing = get_dtype_packing(ref.dtype) - if packing == 1: - if ref.shape[-1] <= 128: - return [ref[start::step, :]], [ref[start + 1 :: step, :]] - num_rows = ref.shape[0] // step - k = jnp.stack([ref[start + i * step, :] for i in range(num_rows)], axis=0) - v = jnp.stack( - [ref[start + 1 + i * step, :] for i in range(num_rows)], axis=0 - ) - return [k], [v] - assert packing in (2, 4, 8) - assert step % packing == 0 - k_list, v_list = [], [] - b_start = start // packing - b_step = step // packing - b_ref = ref.bitcast(jnp.uint32) - if b_ref.shape[-1] <= 128: - b = b_ref[b_start::b_step, :] - else: - num_rows = ref.shape[0] // step - b = jnp.stack( - [b_ref[b_start + i * b_step, :] for i in range(num_rows)], axis=0 - ) - - if ref.dtype == jnp.bfloat16: - bk = b << 16 - bv = b & jnp.uint32(0xFFFF0000) - k = pltpu.bitcast(bk, jnp.float32).astype(jnp.bfloat16) - v = pltpu.bitcast(bv, jnp.float32).astype(jnp.bfloat16) - k_list.append(k) - v_list.append(v) - else: - bitwidth = 32 // packing - bitcast_dst_dtype = jnp.dtype(f"uint{bitwidth}") - for i in range(0, packing, 2): - bk = b >> (i * bitwidth) - k = pltpu.bitcast(bk.astype(bitcast_dst_dtype), ref.dtype) - k_list.append(k) - bv = b >> ((i + 1) * bitwidth) - v = pltpu.bitcast(bv.astype(bitcast_dst_dtype), ref.dtype) - v_list.append(v) - - return k_list, v_list - - def fold_on_2nd_minor(vec): - assert vec.dtype == jnp.bfloat16 or vec.dtype == jnp.float32 - assert len(vec.shape) >= 2 - last_dim = vec.shape[-1] - packing = get_dtype_packing(vec.dtype) - if vec.shape[-2] % packing != 0: - vec = vec.astype(jnp.float32) - return vec.reshape(-1, last_dim) - - @pl.when(heads_blk_idx + q_blk_idx == 0) - def prefetch_first_kv_blk(): - async_copy_kv = create_kv_async_copy_descriptors( - heads_blk_idx, init_seq_idx, 0, init_buf_idx - ) - async_copy_kv.start() - - def is_cur_q_blk_needed(q_states): - done, cur_seq_idx, _ = q_states - should_run = jnp.logical_and(q_len_start < cu_q_lens_ref[num_seqs], - cur_seq_idx < num_seqs) - return jnp.logical_and(done == 0, should_run) - - def compute_with_cur_q_blk(q_states): - done, cur_seq_idx, cur_buf_idx = q_states - q_start = cu_q_lens_ref[cur_seq_idx] - q_end = cu_q_lens_ref[cur_seq_idx + 1] - q_len = q_end - q_start - kv_len = kv_lens_ref[cur_seq_idx] - - def get_next_prefetch_ids( - heads_blk_idx, cur_seq_idx, kv_blk_idx, cur_buf_idx - ): - next_kv_blk_idx = kv_blk_idx + 1 - is_last_kv_blk = next_kv_blk_idx * num_kv_per_blk >= kv_len - next_kv_blk_idx = lax.select( - is_last_kv_blk, - 0, - next_kv_blk_idx, - ) - is_cur_seq_end_in_cur_q_blk = q_end <= q_len_end - next_seq_idx = lax.select( - is_last_kv_blk, - lax.select(is_cur_seq_end_in_cur_q_blk, cur_seq_idx + 1, cur_seq_idx), - cur_seq_idx, - ) - is_last_seq = next_seq_idx == num_seqs - next_seq_idx = lax.select( - is_last_seq, - 0, - next_seq_idx, - ) - next_heads_blk_idx = lax.select( - is_last_seq, - heads_blk_idx + 1, - heads_blk_idx, - ) - next_buf_idx = lax.select(cur_buf_idx == 0, 1, 0) - return next_heads_blk_idx, next_seq_idx, next_kv_blk_idx, next_buf_idx - - def flash_attention( - q, # [num_q_per_blk * num_q_heads_per_kv_head, head_dim] - k, # [num_kv_per_blk, head_dim] - v, # [num_kv_per_blk, head_dim] - head_l_ref, # [num_q_per_blk * num_q_heads_per_kv_head, 128] - head_m_ref, # [num_q_per_blk * num_q_heads_per_kv_head, 128] - head_acc_ref, # [num_q_per_blk, num_q_heads_per_kv_head, head_dim] - *, - kv_blk_idx, - ): - assert q.shape == ( - num_q_per_blk * num_q_heads_per_kv_head, - head_dim, - ) - assert ( - k.shape - == v.shape - == ( - num_kv_per_blk, - head_dim, - ) - ) - assert k.dtype == v.dtype - assert ( - head_m_ref.shape - == head_l_ref.shape - == ( - num_q_per_blk * num_q_heads_per_kv_head, - 128, - ) - ) - assert head_acc_ref.shape == ( - num_q_per_blk, - num_q_heads_per_kv_head, - head_dim, - ) - kv_len_start = kv_blk_idx * num_kv_per_blk - - def masked_store(ref, val, start, end, group=1): - iota = lax.broadcasted_iota(jnp.int32, ref.shape, 0) // group - pltpu.store(ref, val, mask=jnp.logical_and(iota >= start, iota < end)) - - def load_with_init(ref, init_val): - return jnp.where( - kv_blk_idx == 0, jnp.full_like(ref, init_val), ref[...] - ) - - # kv lens will be contracting dim, we should mask out the NaNs. - kv_mask = ( - lax.broadcasted_iota(jnp.int32, k.shape, 0) < kv_len - kv_len_start - ) - k = jnp.where(kv_mask, k.astype(jnp.float32), 0).astype(k.dtype) - v = jnp.where(kv_mask, v.astype(jnp.float32), 0).astype(v.dtype) - - qk = ( - jnp.einsum("nd,md->nm", q, k, preferred_element_type=jnp.float32) - * sm_scale - ) - store_start = jnp.maximum(q_start - q_len_start, 0) - store_end = jnp.minimum(q_end - q_len_start, num_q_per_blk) - - row_ids = ( - (kv_len - q_len) - + q_len_start - - q_start - + jax.lax.broadcasted_iota( - jnp.int32, - (num_q_per_blk * num_q_heads_per_kv_head, num_kv_per_blk), - 0, - ) - // num_q_heads_per_kv_head - ) - col_ids = kv_len_start + jax.lax.broadcasted_iota( - jnp.int32, - (num_q_per_blk * num_q_heads_per_kv_head, num_kv_per_blk), - 1, - ) - causal_mask = row_ids < col_ids - if sliding_window is not None: - causal_mask = jnp.logical_or(causal_mask, - row_ids - sliding_window >= col_ids) - if soft_cap is not None: - qk = soft_cap * jnp.tanh(qk / soft_cap) - qk += jnp.where(causal_mask, mask_value, 0.0) - m_curr = jnp.max(qk, axis=1, keepdims=True) - s_curr = jnp.exp(qk - m_curr) - qkv = jnp.dot(s_curr, v, preferred_element_type=jnp.float32) - lm_store_shape = head_m_ref.shape - m_curr = jnp.broadcast_to(m_curr, lm_store_shape) - l_curr = jnp.broadcast_to( - s_curr.sum(axis=1, keepdims=True), lm_store_shape - ) - m_prev = load_with_init(head_m_ref, -jnp.inf) - l_prev = load_with_init(head_l_ref, 0.0) - m_next = jnp.maximum(m_prev, m_curr) - masked_store( - head_m_ref, m_next, store_start, store_end, num_q_heads_per_kv_head - ) - alpha = jnp.exp(m_prev - m_next) - beta = jnp.exp(m_curr - m_next) - l_alpha = alpha * l_prev - l_next = l_alpha + beta * l_curr - l_next_safe = jnp.where(l_next == 0.0, 1.0, l_next) - masked_store( - head_l_ref, - l_next_safe, - store_start, - store_end, - num_q_heads_per_kv_head, - ) - - def broadcast_to_shape(arr, shape): - if arr.shape == shape: - return arr - assert len(arr.shape) == len(shape) - assert arr.shape[0] == shape[0] - assert shape[1] % arr.shape[1] == 0 - # no-op concatenation. - return jnp.concatenate( - [arr for _ in range(shape[1] // arr.shape[1])], axis=1 - ) - - o_curr = load_with_init(head_acc_ref, 0.0).reshape(-1, head_dim) - l_alpha = broadcast_to_shape(l_alpha, qkv.shape) - beta = broadcast_to_shape(beta, qkv.shape) - l_next_safe = broadcast_to_shape(l_next_safe, qkv.shape) - out = lax.div( - l_alpha * o_curr + beta * qkv, - l_next_safe, - ) - masked_store( - head_acc_ref, - out.reshape(head_acc_ref.shape), - store_start, - store_end, - ) - - def is_valid_kv_blk_in_cur_seq(kv_states): - kv_blk_idx, _ = kv_states - return kv_blk_idx * num_kv_per_blk < kv_len - - def compute_with_kv_blk_in_cur_seq(kv_states): - kv_blk_idx, cur_buf_idx = kv_states - next_heads_blk_idx, next_seq_idx, next_kv_blk_idx, next_buf_idx = ( - get_next_prefetch_ids( - heads_blk_idx, cur_seq_idx, kv_blk_idx, cur_buf_idx - ) - ) - - @pl.when(next_heads_blk_idx < num_heads_blks) - def prefetch_next_kv_blk(): - # TODO(jevinjiang): reuse the same buffer if it is already prefetched! - # TODO(jevinjiang): only fetch effective dynamic size to hold kv_len and - # DMA to fixed size buffer! - next_async_copy_kv = create_kv_async_copy_descriptors( - next_heads_blk_idx, next_seq_idx, next_kv_blk_idx, next_buf_idx - ) - next_async_copy_kv.start() - - cur_async_copy_kv = create_kv_async_copy_descriptors( - heads_blk_idx, cur_seq_idx, kv_blk_idx, cur_buf_idx - ) - kv_ref = cur_async_copy_kv.wait().reshape( - num_kv_pages_per_blk * page_size * num_combined_kv_heads_per_blk, - head_dim, - ) - kv_packing = get_dtype_packing(kv_ref.dtype) - # NOTE: kv_packing is divided by 2 because k and v are packed together. - kv_load_step = max(1, kv_packing // 2) - for kv_head_chunk_idx in range(0, num_kv_heads_per_blk, kv_load_step): - k_list, v_list = strided_load_kv( - kv_ref, kv_head_chunk_idx * 2, num_combined_kv_heads_per_blk - ) - for step_idx in range(kv_load_step): - k = k_list[step_idx] - v = v_list[step_idx] - if k_scale is not None: - # NOTE: Conversion between arbitrary data types is not supported. - # That's why it is converted to float32 first. - k = k.astype(jnp.float32) * k_scale - k = k.astype(q_ref.dtype) - if v_scale is not None: - v = v.astype(jnp.float32) * v_scale - v = v.astype(q_ref.dtype) - kv_head_idx = kv_head_chunk_idx + step_idx - q_head_idx = kv_head_idx * num_q_heads_per_kv_head - # TODO(jevinjiang): extra handling for packed type that can start at - # unaligned position! - q = fold_on_2nd_minor( - q_ref[:, q_head_idx : q_head_idx + num_q_heads_per_kv_head, :] - ) - flash_attention( - q, - k, - v, - l_ref.at[kv_head_idx], - m_ref.at[kv_head_idx], - acc_ref.at[ - :, q_head_idx : q_head_idx + num_q_heads_per_kv_head, : - ], - kv_blk_idx=kv_blk_idx, - ) - return kv_blk_idx + 1, next_buf_idx - - _, next_buf_idx = lax.while_loop( - is_valid_kv_blk_in_cur_seq, - compute_with_kv_blk_in_cur_seq, - (0, cur_buf_idx), # (kv_blk_idx, buf_idx) - ) - next_seq_idx = lax.select(q_end <= q_len_end, cur_seq_idx + 1, cur_seq_idx) - done = lax.select(q_end < q_len_end, done, 1) - return done, next_seq_idx, next_buf_idx - - _, seq_idx, buf_idx = lax.while_loop( - is_cur_q_blk_needed, - compute_with_cur_q_blk, - (0, init_seq_idx, init_buf_idx), # (done, seq_idx, buf_idx) - ) - # Reset seq_idx for next kv_heads_blk if run out of seqs! - seq_buf_idx_ref[0] = lax.select(seq_idx < num_seqs, seq_idx, 0) - seq_buf_idx_ref[1] = buf_idx - o_ref[...] = acc_ref[...].astype(q_ref.dtype) - - -def get_dtype_packing(dtype): - bits = dtypes.itemsize_bits(dtype) - return 32 // bits - - -def get_min_heads_per_blk( - num_q_heads, num_combined_kv_heads, q_dtype, kv_dtype -): - q_packing = get_dtype_packing(q_dtype) - kv_packing = get_dtype_packing(kv_dtype) - - def can_be_xla_fully_tiled(x, packing): - if x % packing != 0: - return False - x //= packing - return x in (1, 2, 4, 8) or x % 8 == 0 - - # TODO(jevinjiang): support unaligned number of heads! - if not can_be_xla_fully_tiled(num_combined_kv_heads, kv_packing): - raise ValueError( - f"Not implemented: {num_combined_kv_heads=} can not be XLA fully tiled." - ) - assert num_combined_kv_heads % 2 == 0 - num_kv_heads = num_combined_kv_heads // 2 - assert num_q_heads % num_kv_heads == 0 - ratio = num_q_heads // num_kv_heads - # TODO(jevinjiang): we can choose smaller tiling for packed type if large - # second minor tiling is not on. - max_combined_kv_tiling = 8 * kv_packing - min_combined_kv_heads = ( - max_combined_kv_tiling - if num_combined_kv_heads % max_combined_kv_tiling == 0 - else num_combined_kv_heads - ) - min_q_heads = min_combined_kv_heads // 2 * ratio - if can_be_xla_fully_tiled(min_q_heads, q_packing): - return min_q_heads, min_combined_kv_heads - return num_q_heads, num_combined_kv_heads - - -@functools.partial( - jax.jit, - static_argnames=[ - "sm_scale", - "mask_value", - "num_kv_pages_per_block", - "num_queries_per_block", - "vmem_limit_bytes", - "sliding_window", - "soft_cap", - "k_scale", - "v_scale", - ], -) -def ragged_paged_attention( - q: jax.Array, # [max_num_batched_tokens, num_q_heads, head_dim] - # TODO(jevinjiang): create a write_to_kv_cache kernel! - kv_pages: jax.Array, # [total_num_pages, page_size, num_combined_kv_heads, head_dim] - kv_lens: jax.Array, # i32[max_num_seqs] - page_indices: jax.Array, # i32[max_num_seqs, pages_per_seq] - cu_q_lens: jax.Array, # i32[max_num_seqs + 1] - num_seqs: jax.Array, # i32[1] - *, - sm_scale: float = 1.0, - sliding_window: int | None = None, - soft_cap: float | None = None, - mask_value: float | None = DEFAULT_MASK_VALUE, - k_scale: float | None = None, - v_scale: float | None = None, - num_kv_pages_per_block: int | None = None, - num_queries_per_block: int | None = None, - vmem_limit_bytes: int | None = None, -): - """Ragged paged attention that supports mixed prefill and decode. - - Args: - q: concatenated all sequences' queries. - kv_pages: paged KV cache. Normally in HBM. - kv_lens: padded kv lengths. Only the first num_seqs values are valid. - page_indices: the first index indicates which page to use in the kv cache - for each sequence. Only the first num_seqs values are valid. - cu_q_lens: the cumulative sum of the effective query lengths. Similar to - kv_lens, only the first num_seqs+1 values are valid. - num_seqs: the dynamic number of sequences. - sm_scale: the softmax scale which will be applied to the Q@K^T. - sliding_window: the sliding window size for the attention. - soft_cap: the logit soft cap for the attention. - mask_value: mask value for causal mask. - k_scale: the scale for the key cache. - v_scale: the scale for the value cache. - num_kv_pages_per_block: number of kv pages to be processed in one flash - attention block in the pallas kernel. - num_queries_per_block: number of kv pages to be processed in one flash - attention block in the pallas kernel. - vmem_limit_bytes: the vmem limit for the pallas kernel. - - Returns: - The output of the attention. - """ - static_validate_inputs( - q, - kv_pages, - kv_lens, - page_indices, - cu_q_lens, - num_seqs, - sm_scale=sm_scale, - sliding_window=sliding_window, - soft_cap=soft_cap, - mask_value=mask_value, - k_scale=k_scale, - v_scale=v_scale, - num_kv_pages_per_block=num_kv_pages_per_block, - num_queries_per_block=num_queries_per_block, - vmem_limit_bytes=vmem_limit_bytes, - ) - if mask_value is None: - mask_value = DEFAULT_MASK_VALUE - num_q_tokens, num_q_heads, head_dim = q.shape - _, page_size, num_combined_kv_heads, _ = kv_pages.shape - assert num_combined_kv_heads % 2 == 0 - num_kv_heads = num_combined_kv_heads // 2 - _, pages_per_seq = page_indices.shape - num_q_heads_per_blk, num_combined_kv_heads_per_blk = get_min_heads_per_blk( - num_q_heads, num_combined_kv_heads, q.dtype, kv_pages.dtype - ) - num_q_per_blk = num_queries_per_block - num_kv_pages_per_blk = num_kv_pages_per_block - if num_q_per_blk is None or num_kv_pages_per_blk is None: - num_kv_pages_per_blk, num_q_per_blk = get_tuned_block_sizes( - q.dtype, - kv_pages.dtype, - num_q_heads_per_blk, - num_combined_kv_heads_per_blk // 2, - head_dim, - page_size, - num_q_tokens, - pages_per_seq, - ) - num_q_heads_per_kv_head = num_q_heads // num_kv_heads - num_q_blks = pl.cdiv(num_q_tokens, num_q_per_blk) - assert num_combined_kv_heads_per_blk % 2 == 0 - num_kv_heads_per_blk = num_combined_kv_heads_per_blk // 2 - assert num_q_heads_per_blk % num_q_heads_per_kv_head == 0 - num_heads_blks = num_q_heads // num_q_heads_per_blk - grid = (num_heads_blks, num_q_blks) - - def q_index_map(heads_blk_idx, q_blk_idx, *_): - return (q_blk_idx, heads_blk_idx, 0) - - q_block_spec = pl.BlockSpec( - (num_q_per_blk, num_q_heads_per_blk, head_dim), - q_index_map, - ) - in_specs = [ - q_block_spec, - pl.BlockSpec(memory_space=pl.ANY), - ] - out_specs = q_block_spec - lm_scratch = pltpu.VMEM( - # TODO(jevinjiang): use 128 instead of 1 is due to Mosaic does not support - # unaligned slicing! - (num_kv_heads_per_blk, num_q_per_blk * num_q_heads_per_kv_head, 128), - jnp.float32, - ) - acc_scratch = pltpu.VMEM( - (num_q_per_blk, num_q_heads_per_blk, head_dim), - jnp.float32, - ) - double_buf_scratch = pltpu.VMEM( - ( - 2, # For double buffering during DMA copies. - num_kv_pages_per_blk, - page_size, - num_combined_kv_heads_per_blk, - head_dim, - ), - kv_pages.dtype, - ) - scratch_shapes = [ - double_buf_scratch, # kv_bufs - pltpu.SemaphoreType.DMA((2,)), # Semaphores for double buffers. - lm_scratch, # l_ref - lm_scratch, # m_ref - acc_scratch, - ] - scalar_prefetches = ( - kv_lens, - page_indices, - cu_q_lens, - jnp.array((0, 0), jnp.int32), # seq_idx, buf_idx - num_seqs, - ) - kernel = pl.pallas_call( - functools.partial( - ragged_paged_attention_kernel, - sm_scale=sm_scale, - sliding_window=sliding_window, - soft_cap=soft_cap, - mask_value=mask_value, - k_scale=k_scale, - v_scale=v_scale, - ), - grid_spec=pltpu.PrefetchScalarGridSpec( - num_scalar_prefetch=len(scalar_prefetches), - in_specs=in_specs, - out_specs=out_specs, - grid=grid, - scratch_shapes=scratch_shapes, - ), - compiler_params=pltpu.CompilerParams( - dimension_semantics=( - "arbitrary", - "arbitrary", - ), - vmem_limit_bytes=vmem_limit_bytes, - ), - out_shape=jax.ShapeDtypeStruct(shape=q.shape, dtype=q.dtype), - name="ragged_paged_attention_kernel", - ) - - return kernel(*scalar_prefetches, q, kv_pages) diff --git a/platforms/tpu/libpjrt_tpu.BUILD.bazel b/platforms/tpu/libpjrt_tpu.BUILD.bazel index 910daf39..8574fe7e 100644 --- a/platforms/tpu/libpjrt_tpu.BUILD.bazel +++ b/platforms/tpu/libpjrt_tpu.BUILD.bazel @@ -11,7 +11,6 @@ copy_to_directory( name = "sandbox", srcs = [ "lib/libpjrt_tpu.so", - "@zml//platforms/tpu:gen_ir_files", ], add_directory_to_runfiles = False, allow_overwrites = True, @@ -22,17 +21,9 @@ copy_to_directory( hardlink = "auto", include_external_repositories = ["**"], include_srcs_patterns = [ - "platforms/tpu/gen_ir.py", - "platforms/tpu/kernel.py", - "platforms/tpu/gen_ir_zig", "**/lib**", - "lib/python*/**", - "site-packages/**", ], replace_prefixes = { - "platforms/tpu/gen_ir.py": "site-packages/gen_ir.py", - "platforms/tpu/kernel.py": "site-packages/kernel.py", - "platforms/tpu/gen_ir_zig": "bin/gen_ir_zig", "lib/x86_64-linux-gnu": "lib", }, ) diff --git a/platforms/tpu/libpython.h b/platforms/tpu/libpython.h deleted file mode 100644 index 605f9fc5..00000000 --- a/platforms/tpu/libpython.h +++ /dev/null @@ -1,2 +0,0 @@ -#define PY_SSIZE_T_CLEAN -#include diff --git a/platforms/tpu/ragged_paged.zig b/platforms/tpu/ragged_paged.zig new file mode 100644 index 00000000..c8fdf9d5 --- /dev/null +++ b/platforms/tpu/ragged_paged.zig @@ -0,0 +1,662 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const mtt = @import("kernels/mosaic_tpu/builder"); + +pub const Cfg = struct { + // Shapes that vary at compile-of-kernel time. + num_q_tokens: i64, + num_q_heads: i64, + num_kv_heads: i64, + head_dim: i64, + total_num_pages: i64, + page_size: i64, + max_num_seqs: i64, + pages_per_seq: i64, + + // Block-level tuning knobs. + num_kv_pages_per_block: i64, + num_queries_per_block: i64, + + // Element types. + q_dtype: mtt.DType = .bf16, + kv_dtype: mtt.DType = .bf16, + + // Numeric knobs. + sm_scale: f32, + mask_value: f32 = -2.38197633e+38, + /// Pre-mask softcap: `qk = soft_cap * tanh(qk / soft_cap)`. + soft_cap: ?f32 = null, + /// K dequant scale applied right after `stridedLoadKv` (mirrors Pallas's + /// `k.astype(f32) * k_scale → k.astype(q_dtype)`). + k_scale: ?f32 = null, + v_scale: ?f32 = null, + + /// Sliding-window mask added on top of the causal mask: any `(row, col)` + /// with `row - sliding_window >= col` is masked out. Mirrors + /// `kernel.py:391-393`. + sliding_window: ?i64 = null, + vmem_limit_bytes: ?i64 = null, + + // Derived shape helpers — keeps the body readable. + pub fn numCombinedKvHeads(self: Cfg) i64 { + return 2 * self.num_kv_heads; + } + pub fn numQHeadsPerKvHead(self: Cfg) i64 { + return @divExact(self.num_q_heads, self.num_kv_heads); + } + pub fn numQHeadsPerBlk(self: Cfg) i64 { + return self.num_q_heads; + } + pub fn numCombinedKvHeadsPerBlk(self: Cfg) i64 { + return self.numCombinedKvHeads(); + } + pub fn numKvHeadsPerBlk(self: Cfg) i64 { + return @divExact(self.numCombinedKvHeadsPerBlk(), 2); + } + pub fn numQPerBlk(self: Cfg) i64 { + return self.num_queries_per_block; + } + pub fn numKvPerBlk(self: Cfg) i64 { + return self.num_kv_pages_per_block * self.page_size; + } + pub fn numHeadsBlks(self: Cfg) i64 { + return @divExact(self.num_q_heads, self.numQHeadsPerBlk()); + } + pub fn numQBlks(self: Cfg) i64 { + return @divExact(self.num_q_tokens, self.numQPerBlk()); + } + pub fn flashQRows(self: Cfg) i64 { + return self.numQPerBlk() * self.numQHeadsPerKvHead(); + } + pub fn flashKRows(self: Cfg) i64 { + return self.numKvPerBlk(); + } + + /// Pallas's `get_dtype_packing(dtype) = 32 // bitwidth(dtype)`. Mirrors + /// `kernel.py:get_dtype_packing` exactly. + pub fn kvPackingFactor(self: Cfg) i64 { + return switch (self.kv_dtype) { + .i32, .f32 => 1, + .i16, .f16, .bf16 => 2, + .i8, .f8e4m3fn, .f8e5m2 => 4, + else => std.debug.panic("ragged_paged: unsupported KV dtype {s}", .{@tagName(self.kv_dtype)}), + }; + } + /// Same packing rule applied to `q_dtype` — used by `fold_on_2nd_minor` + /// to decide whether to cast q to f32 before the trailing-dim reshape. + pub fn qPackingFactor(self: Cfg) i64 { + return switch (self.q_dtype) { + .f32 => 1, + .f16, .bf16 => 2, + else => std.debug.panic("ragged_paged: unsupported Q dtype {s}", .{@tagName(self.q_dtype)}), + }; + } + /// `kv_load_step = max(1, kv_packing // 2)` — number of (k, v) pairs the + /// caller unpacks from a single `b` strided-load (Pallas's outer-loop step). + pub fn kvLoadStep(self: Cfg) i64 { + const p = self.kvPackingFactor(); + return @max(@as(i64, 1), @divFloor(p, 2)); + } + pub fn kvFlatRows(self: Cfg) i64 { + return self.num_kv_pages_per_block * self.page_size * self.numCombinedKvHeadsPerBlk(); + } + pub fn kvFlatI32Rows(self: Cfg) i64 { + return @divExact(self.kvFlatRows(), self.kvPackingFactor()); + } +}; + +fn maskedStoreLM( + k: *mtt.Builder, + ref: mtt.Value, + val: mtt.Value, + kv_head_idx_idx: mtt.Value, + start: mtt.Value, + end: mtt.Value, + group: i64, +) void { + const shape = val.shape().constSlice(); + const iota = k.iota(shape, .i32, &.{0}); + const iota_div = k.divFloor(iota, @as(i32, @intCast(group))); + const ge = k.cmpi(.sge, iota_div, k.broadcastTo(start, shape)); + const lt = k.cmpi(.slt, iota_div, k.broadcastTo(end, shape)); + const mask = k.andi(ge, lt); + const slot_shape = [_]i64{ 1, shape[0], shape[1] }; + const val_3d = k.shapeCast(val, &slot_shape); + const mask_3d = k.shapeCast(mask, &slot_shape); + k.refStoreOpts(ref, val_3d, &.{ kv_head_idx_idx, k.cIndex(0), k.cIndex(0) }, .{ .mask = mask_3d, .strides = &.{} }); +} + +/// `pltpu.store(acc_ref, val, mask=…)` — group=1 so no floor-div. +fn maskedStoreAcc( + k: *mtt.Builder, + ref: mtt.Value, + val: mtt.Value, + q_head_offset_idx: mtt.Value, + start: mtt.Value, + end: mtt.Value, +) void { + const shape = val.shape().constSlice(); + const iota = k.iota(shape, .i32, &.{0}); + const ge = k.cmpi(.sge, iota, k.broadcastTo(start, shape)); + const lt = k.cmpi(.slt, iota, k.broadcastTo(end, shape)); + const mask = k.andi(ge, lt); + k.refStoreOpts(ref, val, &.{ k.cIndex(0), q_head_offset_idx, k.cIndex(0) }, .{ .mask = mask, .strides = &.{} }); +} + +/// `MultiPageAsyncCopyDescriptor.__init__` — `tpu.memref_slice → squeeze +/// → enqueue_dma` chain that Pallas emits for one async page-copy. +fn createKvAsyncCopyDescriptors( + k: *mtt.Builder, + cfg: Cfg, + a: anytype, + heads_blk_idx: mtt.Value, // i32 + seq_idx: mtt.Value, // i32 + kv_blk_idx: mtt.Value, // i32 — the block-relative page index + buf_idx: mtt.Value, // i32 +) void { + const start_kv_page_idx = k.muli(kv_blk_idx, k.lift(@as(i32, @intCast(cfg.num_kv_pages_per_block)))); + const kv_at_seq = k.scalarLoad(a.kv_lens, &.{k.toIndex(seq_idx)}); + const end_kv_page_idx = k.minsi( + k.divsi(k.addi(kv_at_seq, k.lift(@as(i32, @intCast(cfg.page_size - 1)))), k.lift(@as(i32, @intCast(cfg.page_size)))), + k.lift(@as(i32, @intCast(cfg.pages_per_seq))), + ); + const heads_start = k.muli(heads_blk_idx, k.lift(@as(i32, @intCast(cfg.numCombinedKvHeadsPerBlk())))); + + const within = k.cmpi(.slt, start_kv_page_idx, end_kv_page_idx); + const page_idx_safe = k.select(within, start_kv_page_idx, k.lift(@as(i32, 0))); + const page_table = k.scalarLoad(a.page_indices, &.{ k.toIndex(seq_idx), k.toIndex(page_idx_safe) }); + + const src0 = k.memRefSlice(a.kv_pages, &.{ k.lift(@as(i32, 0)), k.lift(@as(i32, 0)), heads_start, k.lift(@as(i32, 0)) }, &.{ cfg.total_num_pages, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const src1 = k.memRefSlice(src0, &.{ page_table, k.lift(@as(i32, 0)), k.lift(@as(i32, 0)), k.lift(@as(i32, 0)) }, &.{ 1, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const src = k.memRefSqueeze(src1, &.{ cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); + + const dst0 = k.memRefSlice(a.kv_bufs, &.{ buf_idx, k.lift(@as(i32, 0)), k.lift(@as(i32, 0)), k.lift(@as(i32, 0)), k.lift(@as(i32, 0)) }, &.{ 1, cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const dst1 = k.memRefSqueeze(dst0, &.{ cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); + const dst = k.memRefSqueeze(dst1, &.{ cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); + + const sem_slice = k.memRefSlice(a.sems, &.{buf_idx}, &.{1}, &.{}); + const sem = k.memRefSqueeze(sem_slice, &.{}); + + k.enqueueDma(src, dst, sem, .{}); +} + +fn waitKvDma( + k: *mtt.Builder, + cfg: Cfg, + a: anytype, + heads_blk_idx: mtt.Value, + seq_idx: mtt.Value, + kv_blk_idx: mtt.Value, + buf_idx: mtt.Value, +) void { + const start_kv_page_idx = k.muli(kv_blk_idx, k.lift(@as(i32, @intCast(cfg.num_kv_pages_per_block)))); + const kv_at_seq = k.scalarLoad(a.kv_lens, &.{k.toIndex(seq_idx)}); + const end_kv_page_idx = k.minsi( + k.divsi(k.addi(kv_at_seq, k.lift(@as(i32, @intCast(cfg.page_size - 1)))), k.lift(@as(i32, @intCast(cfg.page_size)))), + k.lift(@as(i32, @intCast(cfg.pages_per_seq))), + ); + const heads_start = k.muli(heads_blk_idx, k.lift(@as(i32, @intCast(cfg.numCombinedKvHeadsPerBlk())))); + const within = k.cmpi(.slt, start_kv_page_idx, end_kv_page_idx); + const page_idx_safe = k.select(within, start_kv_page_idx, k.lift(@as(i32, 0))); + const page_table = k.scalarLoad(a.page_indices, &.{ k.toIndex(seq_idx), k.toIndex(page_idx_safe) }); + + const src0 = k.memRefSlice(a.kv_pages, &.{ k.lift(@as(i32, 0)), k.lift(@as(i32, 0)), heads_start, k.lift(@as(i32, 0)) }, &.{ cfg.total_num_pages, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const src1 = k.memRefSlice(src0, &.{ page_table, k.lift(@as(i32, 0)), k.lift(@as(i32, 0)), k.lift(@as(i32, 0)) }, &.{ 1, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const src = k.memRefSqueeze(src1, &.{ cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); + + const dst0 = k.memRefSlice(a.kv_bufs, &.{ buf_idx, k.lift(@as(i32, 0)), k.lift(@as(i32, 0)), k.lift(@as(i32, 0)), k.lift(@as(i32, 0)) }, &.{ 1, cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const dst1 = k.memRefSqueeze(dst0, &.{ cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); + const dst = k.memRefSqueeze(dst1, &.{ cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); + + const sem_slice = k.memRefSlice(a.sems, &.{buf_idx}, &.{1}, &.{}); + const sem = k.memRefSqueeze(sem_slice, &.{}); + + k.waitDma2(sem, src, dst, .{}); +} + +const KvPair = struct { k: mtt.Value, v: mtt.Value }; + +/// Mirrors Pallas's `b_ref[b_start::b_step, :]` lowering: +/// - `b_step == 1` → `vector.load` (the slice is contiguous). +/// - `last_dim ≤ 128` and `b_step > 1` → `tpu.strided_load`. +/// - `last_dim > 128` → per-row `vector.load` + `tpu.concatenate` +/// (`jnp.stack([ref[start + i*step, :] for i in range(num_rows)])`). +fn loadStridedOrStack( + k: *mtt.Builder, + ref: mtt.Value, + start_row: i64, + stride: i32, + out_rows: i64, + out_last_dim: i64, +) mtt.Value { + if (out_last_dim <= 128) { + if (stride == 1) { + return k.vectorLoadShape(ref, &.{ k.cIndex(start_row), k.cIndex(0) }, &.{ out_rows, out_last_dim }); + } + return k.stridedLoadShape(ref, &.{ k.cIndex(start_row), k.cIndex(0) }, &.{ stride, 1 }, &.{ out_rows, out_last_dim }); + } + const rows_count: usize = @intCast(out_rows); + const rows = k.arena.allocator().alloc(mtt.Value, rows_count) catch @panic("loadStridedOrStack OOM"); + for (0..rows_count) |i| { + const row_idx: i64 = start_row + @as(i64, @intCast(i)) * @as(i64, stride); + rows[i] = k.vectorLoadShape(ref, &.{ k.cIndex(row_idx), k.cIndex(0) }, &.{ 1, out_last_dim }); + } + return k.concatenate(rows, 0); +} + +/// Pallas's `broadcast_to_shape(arr, shape)` — emit `tpu.concatenate` of +/// `shape[1] / arr.shape[1]` copies when expanding axis 1 (e.g., +/// `` → `` for `head_dim > 128`). +fn broadcastToShape(k: *mtt.Builder, arr: mtt.Value, shape: []const i64) mtt.Value { + const arr_shape = arr.shape(); + std.debug.assert(arr_shape.len == 2 and shape.len == 2); + if (arr_shape.get(1) == shape[1]) return arr; + std.debug.assert(arr_shape.get(0) == shape[0]); + std.debug.assert(@mod(shape[1], arr_shape.get(1)) == 0); + const repeats: usize = @intCast(@divExact(shape[1], arr_shape.get(1))); + const sources = k.arena.allocator().alloc(mtt.Value, repeats) catch @panic("broadcastToShape OOM"); + for (sources) |*s| s.* = arr; + return k.concatenate(sources, 1); +} + +/// `strided_load_kv(ref, kv_head_chunk_idx * 2, num_combined_kv_heads_per_blk)` +/// — full Pallas parity. +/// +/// - `packing == 1` (i32/f32): two strided loads in original dtype. +/// - `packing == 2` bf16: special-case shli + and + tpu.bitcast(f32) + truncf. +/// - `packing == 2` f16/i16, `packing == 4` (i8/u8/f8): generic shrui + trunci +/// + tpu.bitcast unpack producing `kv_load_step` (k, v) pairs. +fn stridedLoadKv( + k: *mtt.Builder, + cfg: Cfg, + a: anytype, + buf_idx: mtt.Value, + kv_head_chunk_idx: i64, +) []const KvPair { + const packing = cfg.kvPackingFactor(); + const start: i64 = kv_head_chunk_idx * 2; + const step: i32 = @intCast(cfg.numCombinedKvHeadsPerBlk()); + + const dst0 = k.memRefSlice(a.kv_bufs, &.{ buf_idx, k.lift(@as(i32, 0)), k.lift(@as(i32, 0)), k.lift(@as(i32, 0)), k.lift(@as(i32, 0)) }, &.{ 1, cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const dst1 = k.memRefSqueeze(dst0, &.{ cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); + const flat = k.memRefReshape(dst1, &.{ cfg.kvFlatRows(), cfg.head_dim }); + + if (packing == 1) { + const k_v = loadStridedOrStack(k, flat, start, step, cfg.flashKRows(), cfg.head_dim); + const v_v = loadStridedOrStack(k, flat, start + 1, step, cfg.flashKRows(), cfg.head_dim); + const result = k.arena.allocator().alloc(KvPair, 1) catch @panic("stridedLoadKv OOM"); + result[0] = .{ .k = k_v, .v = v_v }; + return result; + } + + std.debug.assert(packing == 2 or packing == 4 or packing == 8); + std.debug.assert(@mod(step, packing) == 0); + + const ref_i32 = k.memRefBitcastShape(flat, &.{ cfg.kvFlatI32Rows(), cfg.head_dim }, .i32); + const b_stride: i32 = @intCast(@divExact(cfg.numCombinedKvHeadsPerBlk(), packing)); + const b_start: i64 = @divExact(start, packing); + const b = loadStridedOrStack(k, ref_i32, b_start, b_stride, cfg.flashKRows(), cfg.head_dim); + + const splat_shape = &[_]i64{ cfg.flashKRows(), cfg.head_dim }; + + if (cfg.kv_dtype == .bf16) { + const bk = k.shli(b, k.splat(@as(i32, 16), splat_shape, .i32)); + const bv = k.andi(b, k.splat(@as(i32, @bitCast(@as(u32, 0xFFFF0000))), splat_shape, .i32)); + const k_v = k.arithTruncf(k.bitcastTo(bk, .f32), cfg.kv_dtype); + const v_v = k.arithTruncf(k.bitcastTo(bv, .f32), cfg.kv_dtype); + const result = k.arena.allocator().alloc(KvPair, 1) catch @panic("stridedLoadKv OOM"); + result[0] = .{ .k = k_v, .v = v_v }; + return result; + } + + const bitwidth: i64 = @divExact(@as(i64, 32), packing); + const trunc_dtype: mtt.DType = switch (bitwidth) { + 16 => .i16, + 8 => .i8, + else => std.debug.panic("ragged_paged: unsupported packing bitwidth {d}", .{bitwidth}), + }; + const kv_load_step: usize = @intCast(cfg.kvLoadStep()); + const result = k.arena.allocator().alloc(KvPair, kv_load_step) catch @panic("stridedLoadKv OOM"); + var i: usize = 0; + while (i < kv_load_step) : (i += 1) { + const shift_k: i32 = @intCast(@as(i64, @intCast(i)) * 2 * bitwidth); + const shift_v: i32 = @intCast((@as(i64, @intCast(i)) * 2 + 1) * bitwidth); + const bk = k.shrui(b, k.splat(shift_k, splat_shape, .i32)); + const bv = k.shrui(b, k.splat(shift_v, splat_shape, .i32)); + const bk_trunc = k.trunci(bk, trunc_dtype); + const bv_trunc = k.trunci(bv, trunc_dtype); + result[i] = .{ + .k = k.bitcastTo(bk_trunc, cfg.kv_dtype), + .v = k.bitcastTo(bv_trunc, cfg.kv_dtype), + }; + } + return result; +} + +/// `flash_attention(q, k, v, ...)` — one online-softmax step over `num_kv_per_blk` +/// keys. +fn flashAttention( + k: *mtt.Builder, + cfg: Cfg, + a: anytype, + q: mtt.Value, + kv_k: mtt.Value, + v: mtt.Value, + /// Element type of `kv_k` as currently held — equal to `cfg.kv_dtype` + /// unless `cfg.k_scale` was applied upstream (then `cfg.q_dtype`). + /// Used for the masked-zero round-trip Pallas's `.astype(k.dtype)` does. + k_input_dtype: mtt.DType, + /// Same role as `k_input_dtype`, for `v`. + v_input_dtype: mtt.DType, + kv_head_idx: i64, + q_head_idx: i64, + kv_blk_idx: mtt.Value, + kv_len: mtt.Value, + q_start: mtt.Value, + q_end: mtt.Value, + q_len: mtt.Value, + q_len_start: mtt.Value, +) void { + const flash_q: i64 = cfg.flashQRows(); + const flash_k: i64 = cfg.flashKRows(); + const head_dim: i64 = cfg.head_dim; + const num_q_per_blk: i64 = cfg.numQPerBlk(); + const num_q_heads_per_kv_head: i64 = cfg.numQHeadsPerKvHead(); + + const kv_len_start = k.muli(kv_blk_idx, k.lift(@as(i32, @intCast(cfg.numKvPerBlk())))); + + const kv_mask = k.cmpi( + .slt, + k.iota(&.{ flash_k, head_dim }, .i32, &.{0}), + k.broadcastTo(k.subi(kv_len, kv_len_start), &.{ flash_k, head_dim }), + ); + const k_zeroed = k.select(kv_mask, kv_k.to(.f32), k.zeros(&.{ flash_k, head_dim }, .f32)).to(k_input_dtype); + const v_zeroed = k.select(kv_mask, v.to(.f32), k.zeros(&.{ flash_k, head_dim }, .f32)).to(v_input_dtype); + + k.traceStart(10, "nd,md->nm"); + const qk_raw = k.matmulOpts( + q, + k_zeroed, + k.zeros(&.{ flash_q, flash_k }, .f32), + .{ .dimension_numbers = k.dotDimensionNumbers(&.{1}, &.{1}, &.{0}, &.{0}, &.{ 0, 0, 1, 0 }, &.{}, &.{}) }, + ); + k.traceStop(); + var qk = k.mulf(qk_raw, k.splat(cfg.sm_scale, &.{ flash_q, flash_k }, .f32)); + + const store_start = k.maxsi(k.subi(q_start, q_len_start), k.lift(@as(i32, 0))); + const store_end = k.minsi(k.subi(q_end, q_len_start), k.lift(@as(i32, @intCast(num_q_per_blk)))); + + const row_offset = k.subi(k.addi(k.subi(kv_len, q_len), q_len_start), q_start); + const row_iota = k.iota(&.{ flash_q, flash_k }, .i32, &.{0}); + const row_q = k.divFloor(row_iota, @as(i32, @intCast(num_q_heads_per_kv_head))); + const row_ids = k.addi(k.broadcastTo(row_offset, &.{ flash_q, flash_k }), row_q); + const col_ids = k.addi( + k.broadcastTo(kv_len_start, &.{ flash_q, flash_k }), + k.iota(&.{ flash_q, flash_k }, .i32, &.{1}), + ); + var causal_mask = k.cmpi(.slt, row_ids, col_ids); + if (cfg.sliding_window) |sw| { + const sw_splat = k.splat(@as(i32, @intCast(sw)), &.{ flash_q, flash_k }, .i32); + const window_mask = k.cmpi(.sge, k.subi(row_ids, sw_splat), col_ids); + causal_mask = k.ori(causal_mask, window_mask); + } + if (cfg.soft_cap) |sc| { + const sc_splat = k.splat(sc, &.{ flash_q, flash_k }, .f32); + qk = k.mulf(sc_splat, k.tanh(k.divf(qk, sc_splat))); + } + const qk_masked = k.addf( + qk, + k.select(causal_mask, k.splat(cfg.mask_value, &.{ flash_q, flash_k }, .f32), k.zeros(&.{ flash_q, flash_k }, .f32)), + ); + + // jnp.max → maximumf (NaN-propagating). + const m_curr_v = k.multiReduction(.maximumf, qk_masked, k.splat(@as(f32, -std.math.inf(f32)), &.{flash_q}, .f32), &.{1}); + const m_curr_2d = k.shapeCast(m_curr_v, &.{ flash_q, 1 }); + const s_curr = k.exp(k.subf(qk_masked, k.broadcastTo(m_curr_2d, &.{ flash_q, flash_k }))); + + const qkv = k.matmulOpts( + s_curr, + v_zeroed, + k.zeros(&.{ flash_q, head_dim }, .f32), + .{ .dimension_numbers = k.dotDimensionNumbers(&.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}) }, + ); + + // Pallas hardcodes the m/l-ref trailing dim to 128 (the TPU lane width) + // regardless of `head_dim`. Broadcast m/l values to `head_dim` only + // immediately before mixing with `qkv` / `o_curr`. + const lm_lane: i64 = 128; + + const m_curr_bc = k.broadcastTo(m_curr_2d, &.{ flash_q, lm_lane }); + const l_curr_v = k.multiReduction(.add, s_curr, k.zeros(&.{flash_q}, .f32), &.{1}); + const l_curr_bc = k.broadcastTo(k.shapeCast(l_curr_v, &.{ flash_q, 1 }), &.{ flash_q, lm_lane }); + + const head_l_ref_idx = k.cIndex(kv_head_idx); + const m_prev_load = k.shapeCast( + k.vectorLoadShape(a.m_ref, &.{ head_l_ref_idx, k.cIndex(0), k.cIndex(0) }, &.{ 1, flash_q, lm_lane }), + &.{ flash_q, lm_lane }, + ); + const m_prev = k.select(k.cmpi(.eq, kv_blk_idx, k.lift(@as(i32, 0))), k.splat(@as(f32, -std.math.inf(f32)), &.{ flash_q, lm_lane }, .f32), m_prev_load); + const l_prev_load = k.shapeCast( + k.vectorLoadShape(a.l_ref, &.{ head_l_ref_idx, k.cIndex(0), k.cIndex(0) }, &.{ 1, flash_q, lm_lane }), + &.{ flash_q, lm_lane }, + ); + const l_prev = k.select(k.cmpi(.eq, kv_blk_idx, k.lift(@as(i32, 0))), k.zeros(&.{ flash_q, lm_lane }, .f32), l_prev_load); + + const m_next = k.maximumf(m_prev, m_curr_bc); + maskedStoreLM(k, a.m_ref, m_next, head_l_ref_idx, store_start, store_end, num_q_heads_per_kv_head); + + const alpha = k.exp(k.subf(m_prev, m_next)); + const beta = k.exp(k.subf(m_curr_bc, m_next)); + const l_alpha = k.mulf(alpha, l_prev); + const l_next = k.addf(l_alpha, k.mulf(beta, l_curr_bc)); + const l_next_safe = k.select( + k.cmpf(.oeq, l_next, k.zeros(&.{ flash_q, lm_lane }, .f32)), + k.splat(@as(f32, 1.0), &.{ flash_q, lm_lane }, .f32), + l_next, + ); + maskedStoreLM(k, a.l_ref, l_next_safe, head_l_ref_idx, store_start, store_end, num_q_heads_per_kv_head); + + const q_head_idx_idx = k.cIndex(q_head_idx); + const o_curr_3d = k.vectorLoadShape(a.acc_ref, &.{ k.cIndex(0), q_head_idx_idx, k.cIndex(0) }, &.{ num_q_per_blk, num_q_heads_per_kv_head, head_dim }); + const o_curr_pre = k.select( + k.cmpi(.eq, kv_blk_idx, k.lift(@as(i32, 0))), + k.zeros(&.{ num_q_per_blk, num_q_heads_per_kv_head, head_dim }, .f32), + o_curr_3d, + ); + const o_curr = k.shapeCast(o_curr_pre, &.{ flash_q, head_dim }); + const l_alpha_bc = broadcastToShape(k, l_alpha, &.{ flash_q, head_dim }); + const beta_bc = broadcastToShape(k, beta, &.{ flash_q, head_dim }); + const l_next_safe_bc = broadcastToShape(k, l_next_safe, &.{ flash_q, head_dim }); + const out = k.divf(k.addf(k.mulf(l_alpha_bc, o_curr), k.mulf(beta_bc, qkv)), l_next_safe_bc); + const out_3d = k.shapeCast(out, &.{ num_q_per_blk, num_q_heads_per_kv_head, head_dim }); + maskedStoreAcc(k, a.acc_ref, out_3d, q_head_idx_idx, store_start, store_end); +} + +/// Emit the kernel body into a pre-opened `mtt.Builder`. The factory in +/// `zml/kernel.zig` opens the builder, calls this `run`, and finalizes the +/// IR with `b.finishOpts(&.{}, .{ .canonicalize = true })`. +pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { + const q_transform: []const mtt.ArgSpec.TransformReturn = &.{ + .{ .program_id = 1 }, + .{ .program_id = 0 }, + .zero, + }; + + const a = try b.declareArgsOpts(.{ + .heads_blk_idx = .{ .scalar = .i32 }, + .q_blk_idx = .{ .scalar = .i32 }, + + .kv_lens = .{ .ref = .{ .shape = &.{cfg.max_num_seqs}, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, + .page_indices = .{ .ref = .{ .shape = &.{ cfg.max_num_seqs, cfg.pages_per_seq }, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, + .cu_q_lens = .{ .ref = .{ .shape = &.{cfg.max_num_seqs + 1}, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, + .seq_buf_idx = .{ .ref = .{ .shape = &.{2}, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, + .num_seqs = .{ .ref = .{ .shape = &.{1}, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, + + .q = .{ .ref = .{ + .shape = &.{ cfg.numQPerBlk(), cfg.numQHeadsPerBlk(), cfg.head_dim }, + .dtype = cfg.q_dtype, + .window = mtt.ArgSpec.WindowSpec{ .transform_returns = q_transform }, + } }, + .kv_pages = .{ .ref = .{ + .shape = &.{ cfg.total_num_pages, cfg.page_size, cfg.numCombinedKvHeads(), cfg.head_dim }, + .dtype = cfg.kv_dtype, + .memory_space = .any, + } }, + .o = .{ .ref = .{ + .shape = &.{ cfg.numQPerBlk(), cfg.numQHeadsPerBlk(), cfg.head_dim }, + .dtype = cfg.q_dtype, + .role = .output, + .window = mtt.ArgSpec.WindowSpec{ .transform_returns = q_transform }, + } }, + + .kv_bufs = .{ .ref = .{ + .shape = &.{ 2, cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, + .dtype = cfg.kv_dtype, + .role = .scratch, + } }, + .sems = .{ .sem_array = .{ .shape = &.{2}, .kind = .dma } }, + // Pallas hardcodes the trailing dim to 128 (TPU lane width); see + // `kernel.py:lm_scratch`. + .l_ref = .{ .ref = .{ .shape = &.{ cfg.numKvHeadsPerBlk(), cfg.flashQRows(), 128 }, .dtype = .f32, .role = .scratch } }, + .m_ref = .{ .ref = .{ .shape = &.{ cfg.numKvHeadsPerBlk(), cfg.flashQRows(), 128 }, .dtype = .f32, .role = .scratch } }, + .acc_ref = .{ .ref = .{ .shape = &.{ cfg.numQPerBlk(), cfg.numQHeadsPerBlk(), cfg.head_dim }, .dtype = .f32, .role = .scratch } }, + }, &.{}, .{ + .dimension_semantics = &.{ .arbitrary, .arbitrary }, + .iteration_bounds = &.{ cfg.numHeadsBlks(), cfg.numQBlks() }, + .scalar_prefetch = 5, + .scratch_operands = 5, + .pallas_window_params = true, + }); + + const k = b; + + const num_seqs = k.scalarLoad(a.num_seqs, &.{k.cIndex(0)}); + const init_seq_idx = k.scalarLoad(a.seq_buf_idx, &.{k.cIndex(0)}); + const init_buf_idx = k.scalarLoad(a.seq_buf_idx, &.{k.cIndex(1)}); + const q_len_start = k.muli(a.q_blk_idx, k.lift(@as(i32, @intCast(cfg.numQPerBlk())))); + const q_len_end = k.addi(q_len_start, k.lift(@as(i32, @intCast(cfg.numQPerBlk())))); + + { + var i = k.openIf(k.cmpi(.eq, k.addi(a.heads_blk_idx, a.q_blk_idx), k.lift(@as(i32, 0)))); + createKvAsyncCopyDescriptors(k, cfg, a, a.heads_blk_idx, init_seq_idx, k.lift(@as(i32, 0)), init_buf_idx); + i.yieldThen(.{}); + } + + var q_loop = k.openWhile(.{ k.lift(@as(i32, 0)), init_seq_idx, init_buf_idx }, .{ k.scalarTy(.i32), k.scalarTy(.i32), k.scalarTy(.i32) }); + { + const done = q_loop.before_carried[0]; + const cur_seq_idx = q_loop.before_carried[1]; + const cur_buf_idx = q_loop.before_carried[2]; + const cu_at_num_seqs = k.scalarLoad(a.cu_q_lens, &.{k.toIndex(num_seqs)}); + const should_run = k.andi( + k.cmpi(.slt, q_len_start, cu_at_num_seqs), + k.cmpi(.slt, cur_seq_idx, num_seqs), + ); + const cond = k.andi(k.cmpi(.eq, done, k.lift(@as(i32, 0))), should_run); + q_loop.yieldBefore(cond, .{ done, cur_seq_idx, cur_buf_idx }); + } + { + const done = q_loop.after_carried[0]; + const cur_seq_idx = q_loop.after_carried[1]; + const cur_buf_idx = q_loop.after_carried[2]; + + const q_start = k.scalarLoad(a.cu_q_lens, &.{k.toIndex(cur_seq_idx)}); + const q_end = k.scalarLoad(a.cu_q_lens, &.{k.toIndex(k.addi(cur_seq_idx, k.lift(@as(i32, 1))))}); + const q_len = k.subi(q_end, q_start); + const kv_len = k.scalarLoad(a.kv_lens, &.{k.toIndex(cur_seq_idx)}); + + var kv_loop = k.openWhile(.{ k.lift(@as(i32, 0)), cur_buf_idx }, .{ k.scalarTy(.i32), k.scalarTy(.i32) }); + { + const kv_blk = kv_loop.before_carried[0]; + const buf = kv_loop.before_carried[1]; + const cond = k.cmpi(.slt, k.muli(kv_blk, k.lift(@as(i32, @intCast(cfg.numKvPerBlk())))), kv_len); + kv_loop.yieldBefore(cond, .{ kv_blk, buf }); + } + { + const kv_blk = kv_loop.after_carried[0]; + const cur_buf = kv_loop.after_carried[1]; + + const next_kv_blk_raw = k.addi(kv_blk, k.lift(@as(i32, 1))); + const is_last_kv = k.cmpi(.sge, k.muli(next_kv_blk_raw, k.lift(@as(i32, @intCast(cfg.numKvPerBlk())))), kv_len); + const next_kv_blk = k.select(is_last_kv, k.lift(@as(i32, 0)), next_kv_blk_raw); + const cur_seq_end = k.cmpi(.sle, q_end, q_len_end); + const next_seq_a = k.select(cur_seq_end, k.addi(cur_seq_idx, k.lift(@as(i32, 1))), cur_seq_idx); + const next_seq_b = k.select(is_last_kv, next_seq_a, cur_seq_idx); + const is_last_seq = k.cmpi(.eq, next_seq_b, num_seqs); + const next_seq_idx = k.select(is_last_seq, k.lift(@as(i32, 0)), next_seq_b); + const next_heads_blk_idx = k.select(is_last_seq, k.addi(a.heads_blk_idx, k.lift(@as(i32, 1))), a.heads_blk_idx); + const next_buf = k.extui(k.cmpi(.eq, cur_buf, k.lift(@as(i32, 0))), .i32); + + { + var ii = k.openIf(k.cmpi(.slt, next_heads_blk_idx, k.lift(@as(i32, @intCast(cfg.numHeadsBlks()))))); + createKvAsyncCopyDescriptors(k, cfg, a, next_heads_blk_idx, next_seq_idx, next_kv_blk, next_buf); + ii.yieldThen(.{}); + } + + waitKvDma(k, cfg, a, a.heads_blk_idx, cur_seq_idx, kv_blk, cur_buf); + + // for kv_head_chunk_idx in range(0, num_kv_heads_per_blk, kv_load_step) — unrolled at IR-emit time. + const num_kv_heads_per_blk: usize = @intCast(cfg.numKvHeadsPerBlk()); + const kv_load_step: usize = @intCast(cfg.kvLoadStep()); + var kv_head_chunk_idx: usize = 0; + while (kv_head_chunk_idx < num_kv_heads_per_blk) : (kv_head_chunk_idx += kv_load_step) { + const kv_pairs = stridedLoadKv(k, cfg, a, cur_buf, @intCast(kv_head_chunk_idx)); + var step_idx: usize = 0; + while (step_idx < kv_load_step) : (step_idx += 1) { + var kv_k_val = kv_pairs[step_idx].k; + var kv_v_val = kv_pairs[step_idx].v; + var k_input_dtype = cfg.kv_dtype; + var v_input_dtype = cfg.kv_dtype; + if (cfg.k_scale) |ks| { + const ks_splat = k.splat(ks, &.{ cfg.flashKRows(), cfg.head_dim }, .f32); + kv_k_val = k.mulf(kv_k_val.to(.f32), ks_splat).to(cfg.q_dtype); + k_input_dtype = cfg.q_dtype; + } + if (cfg.v_scale) |vs| { + const vs_splat = k.splat(vs, &.{ cfg.flashKRows(), cfg.head_dim }, .f32); + kv_v_val = k.mulf(kv_v_val.to(.f32), vs_splat).to(cfg.q_dtype); + v_input_dtype = cfg.q_dtype; + } + const kv_head_idx: i64 = @intCast(kv_head_chunk_idx + step_idx); + const q_head_idx = kv_head_idx * cfg.numQHeadsPerKvHead(); + + // `fold_on_2nd_minor` (kernel.py:255-262): when the + // 2nd-to-last dim is not a multiple of q's packing + // factor, cast q to f32 before the trailing-dim reshape + // so the layout pass can fold it cleanly. + var q_block = k.vectorLoadShape(a.q, &.{ k.cIndex(0), k.cIndex(q_head_idx), k.cIndex(0) }, &.{ cfg.numQPerBlk(), cfg.numQHeadsPerKvHead(), cfg.head_dim }); + if (@mod(cfg.numQHeadsPerKvHead(), cfg.qPackingFactor()) != 0) { + q_block = q_block.to(.f32); + } + const q_2d = k.shapeCast(q_block, &.{ cfg.flashQRows(), cfg.head_dim }); + flashAttention(k, cfg, a, q_2d, kv_k_val, kv_v_val, k_input_dtype, v_input_dtype, kv_head_idx, q_head_idx, kv_blk, kv_len, q_start, q_end, q_len, q_len_start); + } + } + + kv_loop.yieldAfter(.{ k.addi(kv_blk, k.lift(@as(i32, 1))), next_buf }); + } + const next_buf_idx = kv_loop.results[1]; + + const next_seq_idx_outer = k.select(k.cmpi(.sle, q_end, q_len_end), k.addi(cur_seq_idx, k.lift(@as(i32, 1))), cur_seq_idx); + const next_done = k.select(k.cmpi(.slt, q_end, q_len_end), done, k.lift(@as(i32, 1))); + q_loop.yieldAfter(.{ next_done, next_seq_idx_outer, next_buf_idx }); + } + const seq_idx_out = q_loop.results[1]; + const buf_idx_out = q_loop.results[2]; + + k.scalarStore(a.seq_buf_idx, k.select(k.cmpi(.slt, seq_idx_out, num_seqs), seq_idx_out, k.lift(@as(i32, 0))), &.{k.cIndex(0)}); + k.scalarStore(a.seq_buf_idx, buf_idx_out, &.{k.cIndex(1)}); + + k.vectorStoreAt(a.o, k.refLoad(a.acc_ref).to(cfg.q_dtype), &.{ k.cIndex(0), k.cIndex(0), k.cIndex(0) }); +} + +/// Open a builder, run the kernel body, and return the canonicalized MLIR IR. +/// Used by offline tooling (`examples/mosaic_ragged_paged`); production model +/// code goes through `zml.kernel.mosaic_tpu.Kernel(...).emit(...)` instead. +pub fn buildIr(allocator: std.mem.Allocator, ctx: *mlir.Context, cfg: Cfg) ![:0]const u8 { + var b = try mtt.Builder.open(allocator, ctx, "ragged_paged_attention_kernel"); + defer b.deinit(); + try run(&b, cfg); + return b.finishOpts(&.{}, .{ .canonicalize = true }); +} diff --git a/platforms/tpu/requirements.in b/platforms/tpu/requirements.in deleted file mode 100644 index 59bf4118..00000000 --- a/platforms/tpu/requirements.in +++ /dev/null @@ -1,3 +0,0 @@ -absl-py # required for jax -jax==0.9.2 -# tpu-inference @ https://github.com/vllm-project/tpu-inference/archive/ac940e32079763405e9af1e67fedb890011ad84f.tar.gz diff --git a/platforms/tpu/requirements.lock.txt b/platforms/tpu/requirements.lock.txt deleted file mode 100644 index 762a625d..00000000 --- a/platforms/tpu/requirements.lock.txt +++ /dev/null @@ -1,226 +0,0 @@ -# This file was autogenerated by uv via the following command: -# bazel run //platforms/tpu:requirements.update ---index-url https://pypi.org/simple - -absl-py==2.4.0 \ - --hash=sha256:88476fd881ca8aab94ffa78b7b6c632a782ab3ba1cd19c9bd423abc4fb4cd28d \ - --hash=sha256:8c6af82722b35cf71e0f4d1d47dcaebfff286e27110a99fc359349b247dfb5d4 - # via -r platforms/tpu/requirements.in -jax==0.9.2 \ - --hash=sha256:42b28017b3e6b57a44b0274cc15f5153239c4873959030399ac1afc009c22365 \ - --hash=sha256:822a8ae155ab42e7bc59f2ae7a28705bcfccb01a7e76abfc8ae996190cdc5598 - # via -r platforms/tpu/requirements.in -jaxlib==0.9.2 \ - --hash=sha256:2d445dab57debd8c26b416c8bc91a4704ba6d7169788a961e4b15419bc3f4254 \ - --hash=sha256:306de54a1de7386c806c723e356ce332d923ef748f8a72d674fefb748121d4dc \ - --hash=sha256:35b473df72dbc2cfda0cb1b3de7521a2150a0aa5ef57ed7583eeceb012dc17c0 \ - --hash=sha256:3d7151140a4936f3218b2d1b1343dd237bd2865cf51442884b6d82fe884a3de7 \ - --hash=sha256:501df74472437ffc11aa3bd8f7fc8b1da274f80bd176d33012cf0d604093667d \ - --hash=sha256:52a0032508f8cf5791c7a7bee142531ee706c3c05518117fb0b6ee8d5e17fde7 \ - --hash=sha256:6949d7ecd869c117e7ea8361866e60cf229c3cd9d6afdc37425a43cf83fc89e9 \ - --hash=sha256:6ffb22eccf07bfc8c9760bfbcdaa268df9b3745739e8397bfce5daee5d79cb51 \ - --hash=sha256:785f177c3eb78cb7dc797c55ed5c4b6312141845c9a686957e484bacbfce5e88 \ - --hash=sha256:87bd42c9f18c9cc9a45371d02ecdbdb574ea1e2277149601a92e14a24c4bbc86 \ - --hash=sha256:88b276a71f4f2071b1fd2e922abfd67c87c6977a551a1036febcea78d5ef7e22 \ - --hash=sha256:97c2fbe58cbee4a27d94ca735d709d231b299ab6ed8b3b1075f52d864dfd32c1 \ - --hash=sha256:9ac995b4ba1aaeedae0d69f319987d515dcaecd4505b642b6312f9e15439351f \ - --hash=sha256:b6d5003e3add5c346a34ae9edc47058cbc2db60c8ed5c50096522176daf01c9f \ - --hash=sha256:b8998f9fa6e67bf956044c310023f6a7bbfaa0d8955f11d928404c8f6eb02fcf \ - --hash=sha256:bab168d25555464461bd077323484f690c471e69ce8b0c39a39fb81b3e3a8bf0 \ - --hash=sha256:bbe59bdef668ff5fd998c6d88e8df9a32ab95bec0dea3d2b5f7a11b86a9a6788 \ - --hash=sha256:be4627c42d44add7fe17d284ef579ff8d159e3cb6947f6437758f34177e878e6 \ - --hash=sha256:bef61eef36ed38cec1069ea973f88af9e03335e884f6501ec3fe7f6222a1555b \ - --hash=sha256:c2f0837cc0788746301e68ae9eda468e6a8a7734dc4d529f26a2cb60fb56c657 \ - --hash=sha256:e8e8165f0f647933f0ff9e1e4d9937d541841d3672a20db73f5ccb5e842b0edc \ - --hash=sha256:fef02d846863b726e72452993883a8596eac325f22a2ec7ea921da0fbc5509b4 - # via jax -ml-dtypes==0.5.4 \ - --hash=sha256:0d2ffd05a2575b1519dc928c0b93c06339eb67173ff53acb00724502cda231cf \ - --hash=sha256:11942cbf2cf92157db91e5022633c0d9474d4dfd813a909383bd23ce828a4b7d \ - --hash=sha256:14a4fd3228af936461db66faccef6e4f41c1d82fcc30e9f8d58a08916b1d811f \ - --hash=sha256:19b9a53598f21e453ea2fbda8aa783c20faff8e1eeb0d7ab899309a0053f1483 \ - --hash=sha256:2314892cdc3fcf05e373d76d72aaa15fda9fb98625effa73c1d646f331fcecb7 \ - --hash=sha256:2b857d3af6ac0d39db1de7c706e69c7f9791627209c3d6dedbfca8c7e5faec22 \ - --hash=sha256:304ad47faa395415b9ccbcc06a0350800bc50eda70f0e45326796e27c62f18b6 \ - --hash=sha256:35f29491a3e478407f7047b8a4834e4640a77d2737e0b294d049746507af5175 \ - --hash=sha256:388d399a2152dd79a3f0456a952284a99ee5c93d3e2f8dfe25977511e0515270 \ - --hash=sha256:3bbbe120b915090d9dd1375e4684dd17a20a2491ef25d640a908281da85e73f1 \ - --hash=sha256:3d277bf3637f2a62176f4575512e9ff9ef51d00e39626d9fe4a161992f355af2 \ - --hash=sha256:4381fe2f2452a2d7589689693d3162e876b3ddb0a832cde7a414f8e1adf7eab1 \ - --hash=sha256:4ff7f3e7ca2972e7de850e7b8fcbb355304271e2933dd90814c1cb847414d6e2 \ - --hash=sha256:531eff30e4d368cb6255bc2328d070e35836aa4f282a0fb5f3a0cd7260257298 \ - --hash=sha256:533ce891ba774eabf607172254f2e7260ba5f57bdd64030c9a4fcfbd99815d0d \ - --hash=sha256:557a31a390b7e9439056644cb80ed0735a6e3e3bb09d67fd5687e4b04238d1de \ - --hash=sha256:5a0f68ca8fd8d16583dfa7793973feb86f2fbb56ce3966daf9c9f748f52a2049 \ - --hash=sha256:6a0df4223b514d799b8a1629c65ddc351b3efa833ccf7f8ea0cf654a61d1e35d \ - --hash=sha256:6c7ecb74c4bd71db68a6bea1edf8da8c34f3d9fe218f038814fd1d310ac76c90 \ - --hash=sha256:7c23c54a00ae43edf48d44066a7ec31e05fdc2eee0be2b8b50dd1903a1db94bb \ - --hash=sha256:805cef3a38f4eafae3a5bf9ebdcdb741d0bcfd9e1bd90eb54abd24f928cd2465 \ - --hash=sha256:88c982aac7cb1cbe8cbb4e7f253072b1df872701fcaf48d84ffbb433b6568f24 \ - --hash=sha256:8ab06a50fb9bf9666dd0fe5dfb4676fa2b0ac0f31ecff72a6c3af8e22c063453 \ - --hash=sha256:8c6a2dcebd6f3903e05d51960a8058d6e131fe69f952a5397e5dbabc841b6d56 \ - --hash=sha256:8c760d85a2f82e2bed75867079188c9d18dae2ee77c25a54d60e9cc79be1bc48 \ - --hash=sha256:9ad459e99793fa6e13bd5b7e6792c8f9190b4e5a1b45c63aba14a4d0a7f1d5ff \ - --hash=sha256:9bad06436568442575beb2d03389aa7456c690a5b05892c471215bfd8cf39460 \ - --hash=sha256:a174837a64f5b16cab6f368171a1a03a27936b31699d167684073ff1c4237dac \ - --hash=sha256:a7f7c643e8b1320fd958bf098aa7ecf70623a42ec5154e3be3be673f4c34d900 \ - --hash=sha256:a9b61c19040397970d18d7737375cffd83b1f36a11dd4ad19f83a016f736c3ef \ - --hash=sha256:b4b801ebe0b477be666696bda493a9be8356f1f0057a57f1e35cd26928823e5a \ - --hash=sha256:b95e97e470fe60ed493fd9ae3911d8da4ebac16bd21f87ffa2b7c588bf22ea2c \ - --hash=sha256:bc11d7e8c44a65115d05e2ab9989d1e045125d7be8e05a071a48bc76eb6d6040 \ - --hash=sha256:bfc534409c5d4b0bf945af29e5d0ab075eae9eecbb549ff8a29280db822f34f9 \ - --hash=sha256:c1a953995cccb9e25a4ae19e34316671e4e2edaebe4cf538229b1fc7109087b7 \ - --hash=sha256:cb73dccfc991691c444acc8c0012bee8f2470da826a92e3a20bb333b1a7894e6 \ - --hash=sha256:ce756d3a10d0c4067172804c9cc276ba9cc0ff47af9078ad439b075d1abdc29b \ - --hash=sha256:d81fdb088defa30eb37bf390bb7dde35d3a83ec112ac8e33d75ab28cc29dd8b0 \ - --hash=sha256:f21c9219ef48ca5ee78402d5cc831bd58ea27ce89beda894428bc67a52da5328 - # via - # jax - # jaxlib -numpy==2.4.4 \ - --hash=sha256:07077278157d02f65c43b1b26a3886bce886f95d20aabd11f87932750dfb14ed \ - --hash=sha256:08f2e31ed5e6f04b118e49821397f12767934cfdd12a1ce86a058f91e004ee50 \ - --hash=sha256:0aec54fd785890ecca25a6003fd9a5aed47ad607bbac5cd64f836ad8666f4959 \ - --hash=sha256:0d35aea54ad1d420c812bfa0385c71cd7cc5bcf7c65fed95fc2cd02fe8c79827 \ - --hash=sha256:0d4e437e295f18ec29bc79daf55e8a47a9113df44d66f702f02a293d93a2d6dd \ - --hash=sha256:0dfd3f9d3adbe2920b68b5cd3d51444e13a10792ec7154cd0a2f6e74d4ab3233 \ - --hash=sha256:1378871da56ca8943c2ba674530924bb8ca40cd228358a3b5f302ad60cf875fc \ - --hash=sha256:15716cfef24d3a9762e3acdf87e27f58dc823d1348f765bbea6bef8c639bfa1b \ - --hash=sha256:19710a9ca9992d7174e9c52f643d4272dcd1558c5f7af7f6f8190f633bd651a7 \ - --hash=sha256:23cbfd4c17357c81021f21540da84ee282b9c8fba38a03b7b9d09ba6b951421e \ - --hash=sha256:2483e4584a1cb3092da4470b38866634bafb223cbcd551ee047633fd2584599a \ - --hash=sha256:27a8d92cd10f1382a67d7cf4db7ce18341b66438bdd9f691d7b0e48d104c2a9d \ - --hash=sha256:28a650663f7314afc3e6ec620f44f333c386aad9f6fc472030865dc0ebb26ee3 \ - --hash=sha256:2aa0613a5177c264ff5921051a5719d20095ea586ca88cc802c5c218d1c67d3e \ - --hash=sha256:2c194dd721e54ecad9ad387c1d35e63dce5c4450c6dc7dd5611283dda239aabb \ - --hash=sha256:2d19e6e2095506d1736b7d80595e0f252d76b89f5e715c35e06e937679ea7d7a \ - --hash=sha256:2d390634c5182175533585cc89f3608a4682ccb173cc9bb940b2881c8d6f8fa0 \ - --hash=sha256:30caa73029a225b2d40d9fae193e008e24b2026b7ee1a867b7ee8d96ca1a448e \ - --hash=sha256:42c16925aa5a02362f986765f9ebabf20de75cdefdca827d14315c568dcab113 \ - --hash=sha256:45dbed2ab436a9e826e302fcdcbe9133f9b0006e5af7168afb8963a6520da103 \ - --hash=sha256:4636de7fd195197b7535f231b5de9e4b36d2c440b6e566d2e4e4746e6af0ca93 \ - --hash=sha256:4a19d9dba1a76618dd86b164d608566f393f8ec6ac7c44f0cc879011c45e65af \ - --hash=sha256:4bbc7f303d125971f60ec0aaad5e12c62d0d2c925f0ab1273debd0e4ba37aba5 \ - --hash=sha256:4d6d57903571f86180eb98f8f0c839fa9ebbfb031356d87f1361be91e433f5b7 \ - --hash=sha256:4e874c976154687c1f71715b034739b45c7711bec81db01914770373d125e392 \ - --hash=sha256:51fc224f7ca4d92656d5a5eb315f12eb5fe2c97a66249aa7b5f562528a3be38c \ - --hash=sha256:58c8b5929fcb8287cbd6f0a3fae19c6e03a5c48402ae792962ac465224a629a4 \ - --hash=sha256:5a285b3b96f951841799528cd1f4f01cd70e7e0204b4abebac9463eecfcf2a40 \ - --hash=sha256:5c70f1cc1c4efbe316a572e2d8b9b9cc44e89b95f79ca3331553fbb63716e2bf \ - --hash=sha256:62d6b0f03b694173f9fcb1fb317f7222fd0b0b103e784c6549f5e53a27718c44 \ - --hash=sha256:6a246d5914aa1c820c9443ddcee9c02bec3e203b0c080349533fae17727dfd1b \ - --hash=sha256:6aa3236c78803afbcb255045fbef97a9e25a1f6c9888357d205ddc42f4d6eba5 \ - --hash=sha256:6bbe4eb67390b0a0265a2c25458f6b90a409d5d069f1041e6aff1e27e3d9a79e \ - --hash=sha256:715d1c092715954784bc79e1174fc2a90093dc4dc84ea15eb14dad8abdcdeb74 \ - --hash=sha256:72944b19f2324114e9dc86a159787333b77874143efcf89a5167ef83cfee8af0 \ - --hash=sha256:81f4a14bee47aec54f883e0cad2d73986640c1590eb9bfaaba7ad17394481e6e \ - --hash=sha256:846300f379b5b12cc769334464656bc882e0735d27d9726568bc932fdc49d5ec \ - --hash=sha256:86b6f55f5a352b48d7fbfd2dbc3d5b780b2d79f4d3c121f33eb6efb22e9a2015 \ - --hash=sha256:874f200b2a981c647340f841730fc3a2b54c9d940566a3c4149099591e2c4c3d \ - --hash=sha256:8a87ec22c87be071b6bdbd27920b129b94f2fc964358ce38f3822635a3e2e03d \ - --hash=sha256:8b3b60bb7cba2c8c81837661c488637eee696f59a877788a396d33150c35d842 \ - --hash=sha256:8e3ed142f2728df44263aaf5fb1f5b0b99f4070c553a0d7f033be65338329150 \ - --hash=sha256:93e15038125dc1e5345d9b5b68aa7f996ec33b98118d18c6ca0d0b7d6198b7e8 \ - --hash=sha256:989824e9faf85f96ec9c7761cd8d29c531ad857bfa1daa930cba85baaecf1a9a \ - --hash=sha256:99d838547ace2c4aace6c4f76e879ddfe02bb58a80c1549928477862b7a6d6ed \ - --hash=sha256:9b2aec6af35c113b05695ebb5749a787acd63cafc83086a05771d1e1cd1e555f \ - --hash=sha256:9c585a1790d5436a5374bac930dad6ed244c046ed91b2b2a3634eb2971d21008 \ - --hash=sha256:a7164afb23be6e37ad90b2f10426149fd75aee07ca55653d2aa41e66c4ef697e \ - --hash=sha256:ac6b31e35612a26483e20750126d30d0941f949426974cace8e6b5c58a3657b0 \ - --hash=sha256:ad2e2ef14e0b04e544ea2fa0a36463f847f113d314aa02e5b402fdf910ef309e \ - --hash=sha256:b268594bccac7d7cf5844c7732e3f20c50921d94e36d7ec9b79e9857694b1b2f \ - --hash=sha256:b5f0362dc928a6ecd9db58868fca5e48485205e3855957bdedea308f8672ea4a \ - --hash=sha256:ba1f4fc670ed79f876f70082eff4f9583c15fb9a4b89d6188412de4d18ae2f40 \ - --hash=sha256:ba203255017337d39f89bdd58417f03c4426f12beed0440cfd933cb15f8669c7 \ - --hash=sha256:c901b15172510173f5cb310eae652908340f8dede90fff9e3bf6c0d8dfd92f83 \ - --hash=sha256:c9b39d38a9bd2ae1becd7eac1303d031c5c110ad31f2b319c6e7d98b135c934d \ - --hash=sha256:d2a8490669bfe99a233298348acc2d824d496dee0e66e31b66a6022c2ad74a5c \ - --hash=sha256:dddbbd259598d7240b18c9d87c56a9d2fb3b02fe266f49a7c101532e78c1d871 \ - --hash=sha256:df3775294accfdd75f32c74ae39fcba920c9a378a2fc18a12b6820aa8c1fb502 \ - --hash=sha256:e44319a2953c738205bf3354537979eaa3998ed673395b964c1176083dd46252 \ - --hash=sha256:e4a010c27ff6f210ff4c6ef34394cd61470d01014439b192ec22552ee867f2a8 \ - --hash=sha256:e823b8b6edc81e747526f70f71a9c0a07ac4e7ad13020aa736bb7c9d67196115 \ - --hash=sha256:e892aff75639bbef0d2a2cfd55535510df26ff92f63c92cd84ef8d4ba5a5557f \ - --hash=sha256:eea7ac5d2dce4189771cedb559c738a71512768210dc4e4753b107a2048b3d0e \ - --hash=sha256:ef4059d6e5152fa1a39f888e344c73fdc926e1b2dd58c771d67b0acfbf2aa67d \ - --hash=sha256:f169b9a863d34f5d11b8698ead99febeaa17a13ca044961aa8e2662a6c7766a0 \ - --hash=sha256:f2cf083b324a467e1ab358c105f6cad5ea950f50524668a80c486ff1db24e119 \ - --hash=sha256:f8474c4241bc18b750be2abea9d7a9ec84f46ef861dbacf86a4f6e043401f79e \ - --hash=sha256:f983334aea213c99992053ede6168500e5f086ce74fbc4acc3f2b00f5762e9db \ - --hash=sha256:f9e75681b59ddaa5e659898085ae0eaea229d054f2ac0c7e563a62205a700121 \ - --hash=sha256:fbc356aae7adf9e6336d336b9c8111d390a05df88f1805573ebb0807bd06fd1d \ - --hash=sha256:fcfe2045fd2e8f3cb0ce9d4ba6dba6333b8fa05bb8a4939c908cd43322d14c7e - # via - # jax - # jaxlib - # ml-dtypes - # scipy -opt-einsum==3.4.0 \ - --hash=sha256:69bb92469f86a1565195ece4ac0323943e83477171b91d24c35afe028a90d7cd \ - --hash=sha256:96ca72f1b886d148241348783498194c577fa30a8faac108586b14f1ba4473ac - # via jax -scipy==1.17.1 \ - --hash=sha256:010f4333c96c9bb1a4516269e33cb5917b08ef2166d5556ca2fd9f082a9e6ea0 \ - --hash=sha256:02ae3b274fde71c5e92ac4d54bc06c42d80e399fec704383dcd99b301df37458 \ - --hash=sha256:08b900519463543aa604a06bec02461558a6e1cef8fdbb8098f77a48a83c8118 \ - --hash=sha256:131f5aaea57602008f9822e2115029b55d4b5f7c070287699fe45c661d051e39 \ - --hash=sha256:158dd96d2207e21c966063e1635b1063cd7787b627b6f07305315dd73d9c679e \ - --hash=sha256:1cc682cea2ae55524432f3cdff9e9a3be743d52a7443d0cba9017c23c87ae2f6 \ - --hash=sha256:1f95b894f13729334fb990162e911c9e5dc1ab390c58aa6cbecb389c5b5e28ec \ - --hash=sha256:200e1050faffacc162be6a486a984a0497866ec54149a01270adc8a59b7c7d21 \ - --hash=sha256:2040ad4d1795a0ae89bfc7e8429677f365d45aa9fd5e4587cf1ea737f927b4a1 \ - --hash=sha256:2b64ca7d4aee0102a97f3ba22124052b4bd2152522355073580bf4845e2550b6 \ - --hash=sha256:2ceb2d3e01c5f1d83c4189737a42d9cb2fc38a6eeed225e7515eef71ad301dce \ - --hash=sha256:35c3a56d2ef83efc372eaec584314bd0ef2e2f0d2adb21c55e6ad5b344c0dcb8 \ - --hash=sha256:37425bc9175607b0268f493d79a292c39f9d001a357bebb6b88fdfaff13f6448 \ - --hash=sha256:3877ac408e14da24a6196de0ddcace62092bfc12a83823e92e49e40747e52c19 \ - --hash=sha256:3fd1fcdab3ea951b610dc4cef356d416d5802991e7e32b5254828d342f7b7e0b \ - --hash=sha256:41b71f4a3a4cab9d366cd9065b288efc4d4f3c0b37a91a8e0947fb5bd7f31d87 \ - --hash=sha256:43af8d1f3bea642559019edfe64e9b11192a8978efbd1539d7bc2aaa23d92de4 \ - --hash=sha256:45abad819184f07240d8a696117a7aacd39787af9e0b719d00285549ed19a1e9 \ - --hash=sha256:4b400bdc6f79fa02a4d86640310dde87a21fba0c979efff5248908c6f15fad1b \ - --hash=sha256:4eb6c25dd62ee8d5edf68a8e1c171dd71c292fdae95d8aeb3dd7d7de4c364082 \ - --hash=sha256:581b2264fc0aa555f3f435a5944da7504ea3a065d7029ad60e7c3d1ae09c5464 \ - --hash=sha256:5cf36e801231b6a2059bf354720274b7558746f3b1a4efb43fcf557ccd484a87 \ - --hash=sha256:5e3c5c011904115f88a39308379c17f91546f77c1667cea98739fe0fccea804c \ - --hash=sha256:6609bc224e9568f65064cfa72edc0f24ee6655b47575954ec6339534b2798369 \ - --hash=sha256:6e3dcd57ab780c741fde8dc68619de988b966db759a3c3152e8e9142c26295ad \ - --hash=sha256:6fac755ca3d2c3edcb22f479fceaa241704111414831ddd3bc6056e18516892f \ - --hash=sha256:744b2bf3640d907b79f3fd7874efe432d1cf171ee721243e350f55234b4cec4c \ - --hash=sha256:74cbb80d93260fe2ffa334efa24cb8f2f0f622a9b9febf8b483c0b865bfb3475 \ - --hash=sha256:766e0dc5a616d026a3a1cffa379af959671729083882f50307e18175797b3dfd \ - --hash=sha256:7bdf2da170b67fdf10bca777614b1c7d96ae3ca5794fd9587dce41eb2966e866 \ - --hash=sha256:7ff200bf9d24f2e4d5dc6ee8c3ac64d739d3a89e2326ba68aaf6c4a2b838fd7d \ - --hash=sha256:844e165636711ef41f80b4103ed234181646b98a53c8f05da12ca5ca289134f6 \ - --hash=sha256:8a604bae87c6195d8b1045eddece0514d041604b14f2727bbc2b3020172045eb \ - --hash=sha256:94055a11dfebe37c656e70317e1996dc197e1a15bbcc351bcdd4610e128fe1ca \ - --hash=sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0 \ - --hash=sha256:9cdc1a2fcfd5c52cfb3045feb399f7b3ce822abdde3a193a6b9a60b3cb5854ca \ - --hash=sha256:9ecb4efb1cd6e8c4afea0daa91a87fbddbce1b99d2895d151596716c0b2e859d \ - --hash=sha256:a3472cfbca0a54177d0faa68f697d8ba4c80bbdc19908c3465556d9f7efce9ee \ - --hash=sha256:a4328d245944d09fd639771de275701ccadf5f781ba0ff092ad141e017eccda4 \ - --hash=sha256:a48a72c77a310327f6a3a920092fa2b8fd03d7deaa60f093038f22d98e096717 \ - --hash=sha256:a720477885a9d2411f94a93d16f9d89bad0f28ca23c3f8daa521e2dcc3f44d49 \ - --hash=sha256:a77cbd07b940d326d39a1d1b37817e2ee4d79cb30e7338f3d0cddffae70fcaa2 \ - --hash=sha256:a9956e4d4f4a301ebf6cde39850333a6b6110799d470dbbb1e25326ac447f52a \ - --hash=sha256:adb2642e060a6549c343603a3851ba76ef0b74cc8c079a9a58121c7ec9fe2350 \ - --hash=sha256:beeda3d4ae615106d7094f7e7cef6218392e4465cc95d25f900bebabfded0950 \ - --hash=sha256:c80be5ede8f3f8eded4eff73cc99a25c388ce98e555b17d31da05287015ffa5b \ - --hash=sha256:cc90d2e9c7e5c7f1a482c9875007c095c3194b1cfedca3c2f3291cdc2bc7c086 \ - --hash=sha256:cd96a1898c0a47be4520327e01f874acfd61fb48a9420f8aa9f6483412ffa444 \ - --hash=sha256:d2650c1fb97e184d12d8ba010493ee7b322864f7d3d00d3f9bb97d9c21de4068 \ - --hash=sha256:d30e57c72013c2a4fe441c2fcb8e77b14e152ad48b5464858e07e2ad9fbfceff \ - --hash=sha256:d59c30000a16d8edc7e64152e30220bfbd724c9bbb08368c054e24c651314f0a \ - --hash=sha256:dbc12c9f3d185f5c737d801da555fb74b3dcfa1a50b66a1a93e09190f41fab50 \ - --hash=sha256:e18f12c6b0bc5a592ed23d3f7b891f68fd7f8241d69b7883769eb5d5dfb52696 \ - --hash=sha256:e19ebea31758fac5893a2ac360fedd00116cbb7628e650842a6691ba7ca28a21 \ - --hash=sha256:e30bdeaa5deed6bc27b4cc490823cd0347d7dae09119b8803ae576ea0ce52e4c \ - --hash=sha256:eb092099205ef62cd1782b006658db09e2fed75bffcae7cc0d44052d8aa0f484 \ - --hash=sha256:eee2cfda04c00a857206a4330f0c5e3e56535494e30ca445eb19ec624ae75118 \ - --hash=sha256:f4115102802df98b2b0db3cce5cb9b92572633a1197c77b7553e5203f284a5b3 \ - --hash=sha256:f590cd684941912d10becc07325a3eeb77886fe981415660d9265c4c418d0bea \ - --hash=sha256:f8885db0bc2bffa59d5c1b72fad7a6a92d3e80e7257f967dd81abb553a90d293 \ - --hash=sha256:fcb310ddb270a06114bb64bbe53c94926b943f5b7f0842194d585c65eb4edd76 - # via - # jax - # jaxlib diff --git a/third_party/mosaic_tpu/BUILD.bazel b/third_party/mosaic_tpu/BUILD.bazel new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/third_party/mosaic_tpu/BUILD.bazel @@ -0,0 +1 @@ + diff --git a/third_party/mosaic_tpu/mosaic_tpu.bazel b/third_party/mosaic_tpu/mosaic_tpu.bazel new file mode 100644 index 00000000..ad2d1eb9 --- /dev/null +++ b/third_party/mosaic_tpu/mosaic_tpu.bazel @@ -0,0 +1,275 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") + +# Copyright 2023 The JAX Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +licenses(["notice"]) + +package( + default_applicable_licenses = [], + default_visibility = [ + "//visibility:public", + ], +) + +################################################################################ +# TPU dialect + +cc_library( + name = "tpu_dialect", + srcs = [ + "jaxlib/mosaic/dialect/tpu/array_util.cc", + "jaxlib/mosaic/dialect/tpu/layout.cc", + "jaxlib/mosaic/dialect/tpu/tpu_dialect.cc", + "jaxlib/mosaic/dialect/tpu/tpu_ops.cc", + "jaxlib/mosaic/dialect/tpu/util.cc", + "jaxlib/mosaic/dialect/tpu/vreg_util.cc", + ], + hdrs = [ + "jaxlib/mosaic/dialect/tpu/array_util.h", + "jaxlib/mosaic/dialect/tpu/layout.h", + "jaxlib/mosaic/dialect/tpu/tpu_dialect.h", + "jaxlib/mosaic/dialect/tpu/util.h", + "jaxlib/mosaic/dialect/tpu/vreg_util.h", + ], + # compatible with libtpu + deps = [ + ":stringify_util", + ":tpu_inc_gen", + "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/hash", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/types:span", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:CommonFolders", + "@llvm-project//mlir:DialectUtils", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:MathDialect", + "@llvm-project//mlir:MemRefDialect", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:VectorDialect", + "@xla//xla:array", + "@xla//xla:shape_util", + "@xla//xla/tsl/platform:statusor", + ], +) + +cc_library( + name = "tpu_serde_pass", + srcs = ["jaxlib/mosaic/dialect/tpu/transforms/serde.cc"], + hdrs = ["jaxlib/mosaic/dialect/tpu/transforms/serde.h"], + # compatible with libtpu + deps = [ + ":pass_boilerplate", + ":serde", + ":tpu_dialect", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:DataLayoutInterfaces", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:VectorDialect", + ], +) + +gentbl_cc_library( + name = "tpu_inc_gen", + # compatible with libtpu + tbl_outs = { + "jaxlib/mosaic/dialect/tpu/tpu_ops.h.inc": [ + "-gen-op-decls", + "-dialect=tpu", + ], + "jaxlib/mosaic/dialect/tpu/tpu_ops.cc.inc": [ + "-gen-op-defs", + "-dialect=tpu", + ], + "jaxlib/mosaic/dialect/tpu/tpu_dialect.h.inc": [ + "-gen-dialect-decls", + "-dialect=tpu", + ], + "jaxlib/mosaic/dialect/tpu/tpu_dialect.cc.inc": [ + "-gen-dialect-defs", + "-dialect=tpu", + ], + "jaxlib/mosaic/dialect/tpu/tpu_enums.h.inc": [ + "-gen-enum-decls", + "-dialect=tpu", + ], + "jaxlib/mosaic/dialect/tpu/tpu_enums.cc.inc": [ + "-gen-enum-defs", + "-dialect=tpu", + ], + "jaxlib/mosaic/dialect/tpu/tpu_attr_defs.h.inc": [ + "-gen-attrdef-decls", + "-dialect=tpu", + "--attrdefs-dialect=tpu", + ], + "jaxlib/mosaic/dialect/tpu/tpu_attr_defs.cc.inc": [ + "-gen-attrdef-defs", + "-dialect=tpu", + "--attrdefs-dialect=tpu", + ], + "jaxlib/mosaic/dialect/tpu/tpu_type_defs.h.inc": [ + "-gen-typedef-decls", + "-dialect=tpu", + "--typedefs-dialect=tpu", + ], + "jaxlib/mosaic/dialect/tpu/tpu_type_defs.cc.inc": [ + "-gen-typedef-defs", + "-dialect=tpu", + "--typedefs-dialect=tpu", + ], + "jaxlib/mosaic/dialect/tpu/tpu_passes.h.inc": [ + "-gen-pass-decls", + "-name=TPU", + ], + "jaxlib/mosaic/dialect/tpu/integrations/c/tpu_passes.capi.h.inc": [ + "-gen-pass-capi-header", + "--prefix=TPU", + ], + "jaxlib/mosaic/dialect/tpu/integrations/c/tpu_passes.capi.cc.inc": [ + "-gen-pass-capi-impl", + "--prefix=TPU", + ], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "jaxlib/mosaic/dialect/tpu/tpu_ops.td", + deps = [":tpu_ops_td_files"], +) + +td_library( + name = "tpu_td_files", + srcs = [ + "jaxlib/mosaic/dialect/tpu/tpu.td", + ], + # compatible with libtpu + deps = [ + "@llvm-project//mlir:BuiltinDialectTdFiles", + ], +) + +td_library( + name = "tpu_ops_td_files", + srcs = [ + "jaxlib/mosaic/dialect/tpu/tpu_ops.td", + ], + # compatible with libtpu + deps = [ + ":tpu_td_files", + "@llvm-project//mlir:BuiltinDialectTdFiles", + "@llvm-project//mlir:ControlFlowInterfacesTdFiles", + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:PassBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +# C API targets + +TPU_CAPI_SOURCES = [ + "jaxlib/mosaic/dialect/tpu/integrations/c/tpu_dialect.cc", + "jaxlib/mosaic/dialect/tpu/integrations/c/tpu_passes.capi.cc.inc", +] + +TPU_CAPI_HEADERS = [ + "jaxlib/mosaic/dialect/tpu/integrations/c/tpu_dialect.h", + "jaxlib/mosaic/dialect/tpu/integrations/c/tpu_passes.capi.h.inc", +] + +cc_library( + name = "tpu_dialect_capi", + srcs = TPU_CAPI_SOURCES, + hdrs = TPU_CAPI_HEADERS, + deps = [ + ":tpu_dialect", + ":tpu_inc_gen", + ":tpu_serde_pass", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:CAPIIR", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + ], +) + +# Header-only target, used when using the C API from a separate shared library. +cc_library( + name = "tpu_dialect_capi_headers", + hdrs = TPU_CAPI_HEADERS, + deps = [ + ":tpu_inc_gen", + "@llvm-project//mlir:CAPIIRHeaders", + ], +) + +# Alwayslink target, used when exporting the C API from a shared library. +cc_library( + name = "tpu_dialect_capi_objects", + srcs = TPU_CAPI_SOURCES, + hdrs = TPU_CAPI_HEADERS, + deps = [ + ":tpu_dialect", + ":tpu_inc_gen", + ":tpu_serde_pass", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:CAPIIR", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + ], + alwayslink = True, +) + +cc_library( + name = "pass_boilerplate", + hdrs = ["jaxlib/mosaic/pass_boilerplate.h"], + # compatible with libtpu + deps = [ + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + ], +) + +cc_library( + name = "serde", + srcs = ["jaxlib/mosaic/serde.cc"], + hdrs = ["jaxlib/mosaic/serde.h"], + # compatible with libtpu + deps = [ + "@llvm-project//llvm:Support", + "@llvm-project//mlir:DataLayoutInterfaces", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + ], +) + +cc_library( + name = "stringify_util", + hdrs = ["jaxlib/mosaic/dialect/tpu/stringify_util.h"], + # compatible with libtpu + deps = ["@llvm-project//llvm:Support"], +) diff --git a/third_party/mosaic_tpu/repo.bzl b/third_party/mosaic_tpu/repo.bzl new file mode 100644 index 00000000..e3fc240e --- /dev/null +++ b/third_party/mosaic_tpu/repo.bzl @@ -0,0 +1,11 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + git_repository( + name = "mosaic_tpu", + remote = "https://github.com/jax-ml/jax.git", + commit = "92348e8631f89bc9e4071f918524b432bcb73f97", + build_file = "//third_party/mosaic_tpu:mosaic_tpu.bazel", + sparse_checkout_patterns = ["jaxlib/mosaic/**"], + patch_cmds = ["find . -name BUILD -delete"], + ) diff --git a/third_party/non_module_deps.bzl b/third_party/non_module_deps.bzl index 4c513371..5cdbd542 100644 --- a/third_party/non_module_deps.bzl +++ b/third_party/non_module_deps.bzl @@ -1,17 +1,18 @@ load("//third_party/arocc:repo.bzl", arocc = "repo") load("//third_party/cloud_accelerator_diagnostics:repo.bzl", cloud_accelerator_diagnostics = "repo") load("//third_party/com_google_sentencepiece:repo.bzl", com_google_sentencepiece = "repo") -load("//third_party/iree:repo.bzl", iree = "repo") -load("//third_party/mnist:repo.bzl", mnist = "repo") -load("//third_party/org_swig_swig:repo.bzl", org_swig_swig = "repo") -load("//third_party/translate-c:repo.bzl", translate_c = "repo") -load("//third_party/xla:repo.bzl", xla = "repo") load("//third_party/flashattn:repo.bzl", flashattn = "repo") -load("//third_party/linenoise:repo.bzl", linenoise = "repo") -load("//third_party/stb:repo.bzl", stb = "repo") -load("//third_party/zigimg:repo.bzl", zigimg = "repo") -load("//third_party/uucode:repo.bzl", uucode = "repo") +load("//third_party/iree:repo.bzl", iree = "repo") load("//third_party/libvaxis:repo.bzl", libvaxis = "repo") +load("//third_party/linenoise:repo.bzl", linenoise = "repo") +load("//third_party/mnist:repo.bzl", mnist = "repo") +load("//third_party/mosaic_tpu:repo.bzl", mosaic_tpu = "repo") +load("//third_party/org_swig_swig:repo.bzl", org_swig_swig = "repo") +load("//third_party/stb:repo.bzl", stb = "repo") +load("//third_party/translate-c:repo.bzl", translate_c = "repo") +load("//third_party/uucode:repo.bzl", uucode = "repo") +load("//third_party/xla:repo.bzl", xla = "repo") +load("//third_party/zigimg:repo.bzl", zigimg = "repo") def _non_module_deps_impl(mctx): cloud_accelerator_diagnostics() @@ -27,6 +28,7 @@ def _non_module_deps_impl(mctx): zigimg() uucode() libvaxis() + mosaic_tpu() iree() return mctx.extension_metadata( diff --git a/third_party/xla/xla.bzl b/third_party/xla/xla.bzl index e0b20989..d324abc2 100644 --- a/third_party/xla/xla.bzl +++ b/third_party/xla/xla.bzl @@ -1,10 +1,12 @@ load("@xla//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls", "tf_vendored") -load("@xla//third_party/farmhash:workspace.bzl", farmhash = "repo") load("@xla//third_party/eigen3:workspace.bzl", eigen3 = "repo") +load("@xla//third_party/farmhash:workspace.bzl", farmhash = "repo") +load("@xla//third_party/highwayhash:workspace.bzl", highwayhash = "repo") +load("@xla//third_party/hwloc:workspace.bzl", hwloc = "repo") load("@xla//third_party/llvm:workspace.bzl", llvm = "repo") load("@xla//third_party/py/ml_dtypes:workspace.bzl", ml_dtypes = "repo") -load("@xla//third_party/stablehlo:workspace.bzl", stablehlo = "repo") load("@xla//third_party/shardy:workspace.bzl", shardy = "repo") +load("@xla//third_party/stablehlo:workspace.bzl", stablehlo = "repo") load("@xla//third_party/triton:workspace.bzl", triton = "repo") _BZL_HELPERS = """\ @@ -193,6 +195,8 @@ def _xla_impl(mctx): shardy() triton() farmhash() + highwayhash() + hwloc() eigen3() ml_dtypes() @@ -205,6 +209,14 @@ def _xla_impl(mctx): ) tf_vendored(name = "tsl", path = "third_party/tsl") + tf_http_archive( + name = "snappy", + build_file = "//third_party:snappy.BUILD", + sha256 = "736aeb64d86566d2236ddffa2865ee5d7a82d26c9016b36218fcc27ea4f09f86", + strip_prefix = "snappy-1.2.1", + urls = tf_mirror_urls("https://github.com/google/snappy/archive/refs/tags/1.2.1.tar.gz"), + ) + _dummy_repos(mctx) return mctx.extension_metadata( diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 8c41c76c..4daa80ef 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -33,9 +33,12 @@ zig_library( "attention.zig", "attention/attention.zig", "attention/flashattn.zig", + "attention/mosaic_tpu_kernels/ragged_attention.zig", "attention/paged_attention.zig", "attention/tpu_attention.zig", - "attention/triton.zig", + "attention/triton_attention.zig", + "attention/triton_kernels/unified_attention.zig", + "kernel.zig", "buffer.zig", "constants.zig", "dtype.zig", @@ -60,10 +63,6 @@ zig_library( "tracer.zig", "zml.zig", ], - data = select({ - "@platforms//os:macos": [], - "//conditions:default": ["//zml/attention/triton:sandbox"], - }), main = "zml.zig", visibility = ["//visibility:public"], zigopts = [ @@ -77,10 +76,14 @@ zig_library( "//bazel", "//ffi:zig", ":profiler_options", + "//kernels/mosaic_tpu:mosaic_tpu_builder", + "//kernels/triton:triton_builder", "//mlir", "//mlir/dialects", + "//mlir/dialects/mosaic_tpu", "//pjrt", "//platforms", + "//platforms/tpu:ragged_paged", "//stdx", "//tools:xspace_to_perfetto_lib", "//upb", diff --git a/zml/attention.zig b/zml/attention.zig index 67869b3f..ced93be2 100644 --- a/zml/attention.zig +++ b/zml/attention.zig @@ -2,4 +2,5 @@ pub const attention = @import("attention/attention.zig"); pub const paged_attention = @import("attention/paged_attention.zig"); pub const flashattn = @import("attention/flashattn.zig"); pub const tpu = @import("attention/tpu_attention.zig"); -pub const triton = @import("attention/triton.zig"); +pub const triton = @import("attention/triton_attention.zig"); +pub const triton_kernels = @import("attention/triton_kernels/unified_attention.zig"); diff --git a/zml/attention/mosaic_tpu_kernels/ragged_attention.zig b/zml/attention/mosaic_tpu_kernels/ragged_attention.zig new file mode 100644 index 00000000..a02ad2f1 --- /dev/null +++ b/zml/attention/mosaic_tpu_kernels/ragged_attention.zig @@ -0,0 +1,9 @@ +const zml = @import("../../zml.zig"); +const ragged_paged = @import("platforms/tpu/ragged_paged"); + +pub const Kernel = zml.kernel.mosaic_tpu.Kernel(ragged_paged.Cfg, .{ + .name = "ragged_paged_attention_kernel", + .inputs = &.{ "kv_lens", "page_indices", "cu_q_lens", "seq_buf_idx", "num_seqs", "q", "kv_pages" }, + .outputs = &.{"o"}, + .run = ragged_paged.run, +}); diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 10cc88c1..4cc7014d 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -5,7 +5,7 @@ const stdx = @import("stdx"); const zml = @import("../zml.zig"); const flashattn = @import("flashattn.zig"); const tpu = @import("tpu_attention.zig"); -const triton = @import("triton.zig"); +const triton = @import("triton_attention.zig"); const PagedAttention = @This(); @@ -265,8 +265,6 @@ test "Backend.auto selects mosaic_tpu on TPU" { .devices = &[_]zml.platform.Device{}, .memories = &[_]zml.platform.Memory{}, .physical_mesh = undefined, - .triton_runtime = null, - .tpu_ir_runtime = null, }; try std.testing.expectEqual(Backend.mosaic_tpu, Backend.auto(&platform)); diff --git a/zml/attention/tpu_attention.zig b/zml/attention/tpu_attention.zig index 264a554f..2791819a 100644 --- a/zml/attention/tpu_attention.zig +++ b/zml/attention/tpu_attention.zig @@ -1,132 +1,18 @@ const std = @import("std"); -const bazel = @import("bazel"); -const bazel_builtin = @import("bazel_builtin"); -const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); const stdx = @import("stdx"); +const ragged_paged = @import("platforms/tpu/ragged_paged"); + const CompilationContext = @import("../module.zig").CompilationContext; const zml = @import("../zml.zig"); const AttentionOptions = @import("paged_attention.zig").AttentionOptions; -const paged_attention = @import("paged_attention.zig"); +const ragged_attention = @import("mosaic_tpu_kernels/ragged_attention.zig"); const log = std.log.scoped(.tpu_attention); -pub const Runtime = struct { - process: std.process.Child, - request_mutex: std.Io.Mutex = .init, - - fn getGenerateBinPath(allocator: std.mem.Allocator) ![]const u8 { - const runfiles = try bazel.runfiles(bazel_builtin.current_repository); - - var sandbox_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = try runfiles.rlocation("libpjrt_tpu/sandbox", &sandbox_path_buf) orelse return error.FileNotFound; - - return std.fs.path.join(allocator, &.{ sandbox_path, "bin", "gen_ir_zig" }); - } - - pub fn init(allocator: std.mem.Allocator, io: std.Io) !Runtime { - const path = try getGenerateBinPath(allocator); - defer allocator.free(path); - - var process = try std.process.spawn(io, .{ - .argv = &.{path}, - .stdin = .pipe, - .stdout = .pipe, - }); - errdefer process.kill(io); - - return .{ .process = process }; - } - - pub fn request(self: *Runtime, allocator: std.mem.Allocator, io: std.Io, request_json: []const u8) ![]u8 { - self.request_mutex.lockUncancelable(io); - defer self.request_mutex.unlock(io); - - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - - const writer_buffer = try arena.allocator().alloc(u8, 4096); - var writer = self.process.stdin.?.writer(io, writer_buffer); - try writer.interface.print("{s}\n", .{request_json}); - try writer.interface.flush(); - - const reader_buffer = try arena.allocator().alloc(u8, 4096); - var reader = self.process.stdout.?.reader(io, reader_buffer); - var allocating: std.Io.Writer.Allocating = .init(arena.allocator()); - _ = try reader.interface.streamDelimiter(&allocating.writer, '\n'); - _ = try reader.interface.discardShort(1); - - const response: std.json.Value = try std.json.parseFromSliceLeaky(std.json.Value, arena.allocator(), allocating.written(), .{}); - const response_object = switch (response) { - .object => |object| object, - else => { - log.err("TPU backend config response is not a JSON object: {s}", .{allocating.written()}); - return error.InvalidResponse; - }, - }; - const ok = response_object.get("ok") orelse { - log.err("TPU backend config response is missing `ok`: {s}", .{allocating.written()}); - return error.InvalidResponse; - }; - const ok_bool = switch (ok) { - .bool => |value| value, - else => { - log.err("TPU backend config response has non-bool `ok`: {s}", .{allocating.written()}); - return error.InvalidResponse; - }, - }; - - if (ok_bool) { - const result = response_object.get("result") orelse { - log.err("TPU backend config response is missing `result`: {s}", .{allocating.written()}); - return error.InvalidResponse; - }; - return switch (result) { - .string => |value| try allocator.dupe(u8, value), - else => { - log.err("TPU backend config response has non-string `result`: {s}", .{allocating.written()}); - return error.InvalidResponse; - }, - }; - } - - if (response_object.get("error")) |value| { - switch (value) { - .string => |message| log.err("TPU backend config generation failed: {s}", .{message}), - else => log.err("TPU backend config generation failed with invalid error payload: {s}", .{allocating.written()}), - } - } else { - log.err("TPU backend config generation failed without an error payload: {s}", .{allocating.written()}); - } - return error.GenerationFailed; - } - - pub fn deinit(self: *Runtime, io: std.Io) void { - _ = std.c.kill(self.process.id.?, .INT); - _ = self.process.wait(io) catch unreachable; - } -}; - pub const mosaic_tpu = struct { - const RaggedPagedRequest = struct { - num_q_tokens: i64, - num_q_heads: i64, - num_kv_heads: i64, - head_dim: i64, - q_dtype: []const u8, - kv_dtype: []const u8, - max_num_seqs: i64, - pages_per_seq: i64, - total_num_pages: i64, - page_size: i64, - sm_scale: f32, - sliding_window: ?i64 = null, - num_kv_pages_per_block: u32, - num_queries_per_block: u32, - }; - const PreparedInputs = struct { q: zml.Tensor, kv_pages: zml.Tensor, @@ -203,35 +89,36 @@ pub const mosaic_tpu = struct { block_table: zml.Tensor, query_start_len: zml.Tensor, num_seqs: zml.Tensor, - backend_config: []const u8, + cfg: ragged_paged.Cfg, ) zml.Tensor { const seq_buf_idx = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{2}, .i32)); - const comp_ctx = CompilationContext.current(); - const op = dialects.stablehlo.custom_call( - comp_ctx.mlir_ctx, - &.{ - seq_lens.value(), - block_table.value(), - query_start_len.value(), - seq_buf_idx.value(), - num_seqs.value(), - q.value(), - kv_pages.value(), - }, - &.{q.value().type_()}, + + const additional_attrs = [_]mlir.NamedAttribute{ + mlir.NamedAttribute.named(comp_ctx.mlir_ctx, "kernel_name", mlir.stringAttribute(comp_ctx.mlir_ctx, "ragged_paged_attention_kernel")), + mlir.NamedAttribute.named(comp_ctx.mlir_ctx, "mhlo.frontend_attributes", kernelFrontendAttrs(comp_ctx)), + }; + + const out = ragged_attention.Kernel.call( .{ - .call_target_name = "tpu_custom_call", - .backend_config = .{ .original = backend_config }, - .has_side_effect = false, - .additional_attributes = &.{ - mlir.NamedAttribute.named(comp_ctx.mlir_ctx, "kernel_name", mlir.stringAttribute(comp_ctx.mlir_ctx, "ragged_paged_attention_kernel")), - mlir.NamedAttribute.named(comp_ctx.mlir_ctx, "mhlo.frontend_attributes", kernelFrontendAttrs(comp_ctx)), + .kv_lens = seq_lens, + .page_indices = block_table, + .cu_q_lens = query_start_len, + .seq_buf_idx = seq_buf_idx, + .num_seqs = num_seqs, + .q = q, + .kv_pages = kv_pages, + }, + .{ .o = q.shape() }, + .{ + .cfg = cfg, + .extras = .{ + .vmem_limit_bytes = cfg.vmem_limit_bytes, + .additional_attributes = &additional_attrs, }, }, - .unknown(comp_ctx.mlir_ctx), - ).appendTo(comp_ctx.currentScope().block); - return zml.Tensor._result(q.shape(), op.result(0)); + ); + return out.o; } fn activeSequenceCount(query_start_len: zml.Tensor) zml.Tensor { @@ -245,33 +132,11 @@ pub const mosaic_tpu = struct { .reshape(zml.Shape.init(.{1}, .i32)); } - fn jnpDTypeExpr(dtype: zml.DataType) []const u8 { - return switch (dtype) { - .bf16 => "jnp.bfloat16", - .f32 => "jnp.float32", - else => stdx.debug.panic("mosaic_tpu ragged paged attention expects q/kv dtype bf16 or f32, got {}", .{dtype}), - }; - } - - fn renderBackendConfig(allocator: std.mem.Allocator, io: std.Io, request: RaggedPagedRequest) []u8 { - const compilation_context = CompilationContext.current(); - - const json_string = std.fmt.allocPrint( - allocator, - "{{\"backend_config\":\"ragged_paged\",\"params\":{f}}}", - .{std.json.fmt(request, .{})}, - ) catch |err| stdx.debug.panic("Failed to allocate ragged paged attention TPU params: {}", .{err}); - defer allocator.free(json_string); - - const platform = @constCast(compilation_context.platform); - const runtime = &(platform.tpu_ir_runtime orelse stdx.debug.panic("TPU backend config runtime is not initialized", .{})); - const backend_config = runtime.request(allocator, io, json_string) catch |err| { - stdx.debug.panic("Failed to generate TPU backend config through persistent runtime: {}", .{err}); - }; - defer allocator.free(backend_config); - - return allocator.dupe(u8, backend_config) catch |err| { - stdx.debug.panic("Failed to persist TPU backend config in compilation arena: {}", .{err}); + fn cfgDtype(dtype: zml.DataType) @FieldType(ragged_paged.Cfg, "q_dtype") { + const cast = zml.kernel.mosaic_tpu.from(dtype); + return switch (cast) { + .bf16, .f16, .f32 => cast, + else => stdx.debug.panic("mosaic_tpu ragged paged attention expects q/kv dtype bf16/f16/f32, got {}", .{dtype}), }; } @@ -310,8 +175,6 @@ pub const mosaic_tpu = struct { const q_ragged = alignQueryHeadDimForKernel(q_merged, kv_pages_partitioned.dim(.hd)); const query_start_len = parameters.query_start_len.withPartitioning(.{ .b = .replicated }); - // Note - // activeSequenceCount could probably be computed when preparing the inputs and injected through Parameters, but we'll keep that for later. return .{ .q = q_ragged, .kv_pages = kv_pages_partitioned, @@ -322,12 +185,7 @@ pub const mosaic_tpu = struct { }; } - fn checkedU32(value: usize, comptime field_name: []const u8) u32 { - stdx.debug.assert(value <= std.math.maxInt(u32), "mosaic_tpu ragged paged attention field '{s}' exceeds u32 range: {}", .{ field_name, value }); - return @intCast(value); - } - - fn buildRequest(prepared_inputs: PreparedInputs, parameters: Parameters, opts: AttentionOptions) RaggedPagedRequest { + fn buildCfg(prepared_inputs: PreparedInputs, parameters: Parameters, opts: AttentionOptions) ragged_paged.Cfg { const q_shape = prepared_inputs.q.shape(); const kv_pages_shape = prepared_inputs.kv_pages.shape(); const block_table_shape = prepared_inputs.block_table.shape(); @@ -341,11 +199,8 @@ pub const mosaic_tpu = struct { .{ q_shape.dim(.hd), kv_pages_shape.dim(.hd) }, ); stdx.debug.assert(@mod(kv_pages_shape.dim(.hkv), 2) == 0, "Expected fused KV pages .hkv dimension to be even, got {}", .{kv_pages_shape.dim(.hkv)}); - const num_kv_heads = std.math.divExact(i64, kv_pages_shape.dim(.hkv), 2) catch |err| switch (err) { - error.UnexpectedRemainder => stdx.debug.panic("Expected fused KV pages .hkv dimension to be even, got {}", .{kv_pages_shape.dim(.hkv)}), - error.DivisionByZero => stdx.debug.panic("KV pages .hkv dimension must be non-zero, got {}", .{kv_pages_shape.dim(.hkv)}), - error.Overflow => stdx.debug.panic("KV pages .hkv dimension is too large to fit in i64: {}", .{kv_pages_shape.dim(.hkv)}), - }; + const num_kv_heads = std.math.divExact(i64, kv_pages_shape.dim(.hkv), 2) catch + stdx.debug.panic("Expected fused KV pages .hkv dimension to be even, got {}", .{kv_pages_shape.dim(.hkv)}); const num_queries_per_block = @max(@as(i64, 1), @min(q_token_count, max_num_seqs)); const logical_head_dim: f64 = @floatFromInt(parameters.opts.head_dim); @@ -354,16 +209,16 @@ pub const mosaic_tpu = struct { .num_q_heads = num_q_heads, .num_kv_heads = num_kv_heads, .head_dim = q_shape.dim(.hd), - .q_dtype = jnpDTypeExpr(q_shape.dtype()), - .kv_dtype = jnpDTypeExpr(kv_pages_shape.dtype()), - .max_num_seqs = max_num_seqs, - .pages_per_seq = @intCast(parameters.opts.max_num_pages), .total_num_pages = kv_pages_shape.dim(.page), .page_size = kv_pages_shape.dim(.k_chunk), + .max_num_seqs = max_num_seqs, + .pages_per_seq = @intCast(parameters.opts.max_num_pages), + .num_kv_pages_per_block = @intCast(parameters.opts.pages_per_compute_block), + .num_queries_per_block = num_queries_per_block, + .q_dtype = cfgDtype(q_shape.dtype()), + .kv_dtype = cfgDtype(kv_pages_shape.dtype()), .sm_scale = opts.scale orelse @floatCast(1.0 / @sqrt(logical_head_dim)), .sliding_window = if (opts.sliding_window < 0) null else @intCast(opts.sliding_window), - .num_kv_pages_per_block = checkedU32(parameters.opts.pages_per_compute_block, "num_kv_pages_per_block"), - .num_queries_per_block = checkedU32(@intCast(num_queries_per_block), "num_queries_per_block"), }; } @@ -383,9 +238,6 @@ pub const mosaic_tpu = struct { const prepared = prepareInputs(parameters, q, kv_cache); - // Keep sharding intent on the `manualComputation` boundary only. The - // TPU smoke compile regressed when the body restated - // `withPartitioning` constraints on intermediate tensors. const q_out = zml.ops.manualComputation( .{ prepared.q, @@ -403,6 +255,7 @@ pub const mosaic_tpu = struct { }, (struct { fn body(body_context: anytype, allocator: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output: zml.Shape) zml.Tensor { + _ = allocator; stdx.debug.assert(sharded_inputs.len == 6, "mosaic_tpu ragged paged manualComputation expects 6 inputs, got {}", .{sharded_inputs.len}); const prepared_inputs: PreparedInputs = .{ @@ -414,7 +267,7 @@ pub const mosaic_tpu = struct { .num_seqs = sharded_inputs[5], }; - const backend_config = renderBackendConfig(allocator, CompilationContext.current().io, buildRequest(prepared_inputs, body_context.parameters, body_context.opts)); + const cfg = buildCfg(prepared_inputs, body_context.parameters, body_context.opts); const q_out = raggedPagedKernelCall( prepared_inputs.q, prepared_inputs.kv_pages, @@ -422,7 +275,7 @@ pub const mosaic_tpu = struct { prepared_inputs.block_table, prepared_inputs.query_start_len, prepared_inputs.num_seqs, - backend_config, + cfg, ); stdx.debug.assert(q_out.shape().eql(output), "mosaic_tpu ragged paged attention output shape mismatch, got {f}, expected {f}", .{ q_out.shape(), output }); return q_out; @@ -443,7 +296,7 @@ test "mosaic_tpu aligns kernel head dimensions to 128" { try std.testing.expectEqual(@as(usize, 256), mosaic_tpu.alignedKernelHeadDim(256)); } -test "mosaic_tpu request uses kernel head dim and logical scale" { +test "mosaic_tpu cfg uses kernel head dim and logical scale" { const parameters = mosaic_tpu.Parameters.init(.{ .is_prefill = false, .batch_size = 1, @@ -463,11 +316,11 @@ test "mosaic_tpu request uses kernel head dim and logical scale" { .num_seqs = zml.Tensor.init(.{1}, .i32), }; - const request = mosaic_tpu.buildRequest(prepared_inputs, parameters, .{}); - try std.testing.expectEqual(@as(i64, 128), request.head_dim); + const cfg = mosaic_tpu.buildCfg(prepared_inputs, parameters, .{}); + try std.testing.expectEqual(@as(i64, 128), cfg.head_dim); try std.testing.expectApproxEqAbs( @as(f32, 1.0 / @sqrt(@as(f32, 64.0))), - request.sm_scale, + cfg.sm_scale, 0.000001, ); } diff --git a/zml/attention/triton/BUILD.bazel b/zml/attention/triton/BUILD.bazel deleted file mode 100644 index 10e0894e..00000000 --- a/zml/attention/triton/BUILD.bazel +++ /dev/null @@ -1,124 +0,0 @@ -load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") -load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_python//python:py_library.bzl", "py_library") -load("@rules_python//python/uv:lock.bzl", uv_lock = "lock") -load("@rules_zig//zig:defs.bzl", "zig_binary") -load("@zml//bazel:runfiles.bzl", "runfiles_to_default") - -uv_lock( - name = "requirements", - srcs = ["requirements.in"], - out = "requirements.lock.txt", - args = [ - "--emit-index-url", - "--emit-find-links", - "--index-strategy=unsafe-best-match", - "--upgrade", - ], - python_version = "3.12", - tags = ["manual"], -) - -py_library( - name = "fake_plugin", - srcs = [ - "fake_plugin/__init__.py", - "fake_plugin/compiler.py", - "fake_plugin/driver.py", - ], - target_compatible_with = ["@platforms//os:linux"], - deps = ["@triton_py_deps//triton"], -) - -py_library( - name = "triton_kernels", - srcs = [ - "triton_kernels/__init__.py", - "triton_kernels/unified_attention.py", - ], - target_compatible_with = ["@platforms//os:linux"], - deps = ["@triton_py_deps//triton"], -) - -py_binary( - name = "generate_py", - srcs = ["generate.py"], - main = "generate.py", - python_version = "3.12", - target_compatible_with = ["@platforms//os:linux"], - deps = [ - ":fake_plugin", - ":triton_kernels", - "@triton_py_deps//triton", - ], -) - -cc_library( - name = "libpython", - hdrs = ["libpython.h"], - deps = [ - "@python_3_12//:libpython", - "@python_3_12//:python_headers", - ], -) - -zig_binary( - name = "generate", - data = [":generate_py"], - linkopts = ["-Wl,-rpath,$$ORIGIN/../lib"], - main = "generate.zig", - tags = ["manual"], - target_compatible_with = ["@platforms//os:linux"], - deps = [":libpython"], -) - -runfiles_to_default( - name = "generate_files", - target_compatible_with = ["@platforms//os:linux"], - visibility = ["//visibility:public"], - deps = [ - ":generate", - ":generate_py", - ], -) - -copy_to_directory( - name = "sandbox", - srcs = [ - ":generate_files", - ] + select({ - "@llvm//platforms/config:linux_x86_64": [ - "@zlib1g_linux_amd64//:zlib1g", - ], - "@llvm//platforms/config:linux_aarch64": [ - "@zlib1g_linux_arm64//:zlib1g", - ], - }), - target_compatible_with = ["@platforms//os:linux"], - add_directory_to_runfiles = False, - allow_overwrites = True, - exclude_srcs_patterns = [ - "**/include/**", - "**/_solib_*/**", - ], - hardlink = "auto", - include_external_repositories = ["**"], - include_srcs_patterns = [ - "generate", - "generate.py", - "fake_plugin/*", - "triton_kernels/*", - "**/lib**", - "lib/python*/**", - "site-packages/**", - ], - replace_prefixes = { - "fake_plugin": "site-packages/fake_plugin", - "triton_kernels": "site-packages/triton_kernels", - "generate.py": "site-packages/generate.py", - "generate": "bin/generate", - "lib/x86_64-linux-gnu": "lib", - "lib/aarch64-linux-gnu": "lib", - }, - visibility = ["//visibility:public"], -) diff --git a/zml/attention/triton/fake_plugin/__init__.py b/zml/attention/triton/fake_plugin/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/zml/attention/triton/fake_plugin/compiler.py b/zml/attention/triton/fake_plugin/compiler.py deleted file mode 100755 index 2e0a6c36..00000000 --- a/zml/attention/triton/fake_plugin/compiler.py +++ /dev/null @@ -1,97 +0,0 @@ -from dataclasses import dataclass -from types import ModuleType -from typing import Any, Dict - -from triton.backends.compiler import BaseBackend, GPUTarget, Language - - -@dataclass(frozen=True) -class FakeOptions: - num_warps: int = 4 - num_ctas: int = 1 - num_stages: int = 2 - enable_fp_fusion: bool = True - launch_cooperative_grid: bool = False - extern_libs: tuple = () - sanitize_overflow: bool = True - debug: bool = False - instrumentation_mode: str = "" - default_dot_input_precision: str = "tf32" - allowed_dot_input_precisions: tuple = ("tf32", "tf32x3", "ieee", "bf16x3", "bf16x6") - max_num_imprecise_acc_default: int = 0 - backend_name: str = "cpu" - - def hash(self) -> str: - return "fake-options-v1" - - -class Backend(BaseBackend): - def __init__(self, target: GPUTarget) -> None: - super().__init__(target) - self.binary_ext = "fakebin" - - @staticmethod - def supports_target(target: GPUTarget): - return target.backend == "cpu" - - def hash(self) -> str: - return "fake-backend-v1" - - def parse_options(self, options: dict) -> object: - args = { - k: options[k] for k in FakeOptions.__dataclass_fields__.keys() if k in options and options[k] is not None - } - return FakeOptions(**args) - - def add_stages(self, stages: dict, options: object, language: Language = Language.TRITON) -> None: - del options, language - stages["ttir"] = lambda src, metadata: self._keep_ttir(src, metadata) - stages[self.binary_ext] = lambda src, metadata: self._emit_fake_binary(src, metadata) - - def _keep_ttir(self, src, metadata): - metadata.setdefault("shared", 0) - metadata.setdefault("name", "fake_kernel") - return src - - def _emit_fake_binary(self, src, metadata) -> bytes: - del src - metadata.setdefault("shared", 0) - metadata.setdefault("name", "fake_kernel") - return b"fake-binary" - - def pack_metadata(self, metadata): - return ( - getattr(metadata, "num_warps", 1), - getattr(metadata, "num_ctas", 1), - getattr(metadata, "shared", 0), - ) - - def get_codegen_implementation(self, options: Any): - del options - return {"min_dot_size": lambda lhs_type, rhs_type: (1, 1, 1)} - - def load_dialects(self, context): - del context - - def get_module_map(self) -> Dict[str, ModuleType]: - return {} - - @staticmethod - def parse_attr(desc): - assert isinstance(desc, str) - ret = [] - if "D" in desc: - ret += [["tt.divisibility", 16]] - return ret - - @staticmethod - def get_int_specialization(arg, **kwargs): - if arg % 16 == 0 and kwargs.get("align", False): - return "D" - return "" - - @staticmethod - def get_tensor_specialization(arg, **kwargs): - if arg.data_ptr() % 16 == 0 and kwargs.get("align", False): - return "D" - return "" diff --git a/zml/attention/triton/fake_plugin/driver.py b/zml/attention/triton/fake_plugin/driver.py deleted file mode 100755 index 44d339dd..00000000 --- a/zml/attention/triton/fake_plugin/driver.py +++ /dev/null @@ -1,55 +0,0 @@ -from triton.backends.compiler import GPUTarget -from triton.backends.driver import Benchmarker, DriverBase - - -_TYPE_MAP = { - "i1": "int8_t", - "i8": "int8_t", - "i16": "int16_t", - "i32": "int32_t", - "i64": "int64_t", - "u1": "uint8_t", - "u8": "uint8_t", - "u16": "uint16_t", - "u32": "uint32_t", - "u64": "uint64_t", - "fp16": "double", - "bf16": "double", - "fp32": "double", - "f32": "double", - "fp64": "double", -} - - -class Driver(DriverBase): - @classmethod - def is_active(cls): - return True - - def map_python_to_cpp_type(self, ty: str) -> str: - if ty.startswith("*"): - return "void*" - return _TYPE_MAP[ty] - - def get_current_target(self): - return GPUTarget("cpu", "generic", 32) - - def get_current_device(self): - return "cpu" - - def get_current_stream(self, device): - del device - return None - - def get_active_torch_device(self): - return "cpu" - - def get_benchmarker(self) -> Benchmarker: - def _benchmarker(kernel_call, *, quantiles, **kwargs): - del kernel_call, kwargs - return [0.0 for _ in quantiles] - - return _benchmarker - - def __init__(self) -> None: - pass diff --git a/zml/attention/triton/generate.py b/zml/attention/triton/generate.py deleted file mode 100755 index 06b9cc55..00000000 --- a/zml/attention/triton/generate.py +++ /dev/null @@ -1,365 +0,0 @@ -import signal -import sys -import json -import math -import triton - -import triton.backends as triton_backends -from triton.backends import Backend as BackendRegistration -from triton.runtime.driver import driver as runtime_driver - -from fake_plugin.compiler import Backend as FakeCompilerBackend -from fake_plugin.driver import Driver as FakeDriver -from triton_kernels.unified_attention import ( - kernel_unified_attention_2d_ptr, - kernel_unified_attention_3d_ptr, - reduce_segments_ptr, -) - - -class FakeTensor: - def __init__(self, dtype: str, shape, strides=None): - self.dtype = dtype - self.shape = tuple(shape) - self._strides = tuple(strides) if strides is not None else contiguous_strides(self.shape) - - def stride(self, dim: int) -> int: - return self._strides[dim] - - @staticmethod - def data_ptr() -> int: - return 0 - - -def contiguous_strides(shape: tuple[int, ...]) -> tuple[int, ...]: - stride = 1 - out = [] - for size in reversed(shape): - out.append(stride) - stride *= int(size) - return tuple(reversed(out)) - - -def next_power_of_2(x: int) -> int: - return 1 if x <= 1 else 1 << (x - 1).bit_length() - - -def select_3d_config( - block_size: int, - max_seqlen_k: int, - target_num_prgms: int, - num_2d_prgms: int, -) -> tuple[dict, dict]: - reduce_num_warps = 2 - attn_warps = 2 - tile_size = block_size - - num_segments = math.ceil(target_num_prgms / max(1, num_2d_prgms)) - num_segments = next_power_of_2(max(1, num_segments)) - num_segments = min(num_segments, 128) - min_segments = 16 if tile_size <= 16 else 8 - num_segments = max(num_segments, min_segments) - - if num_segments == min_segments: - reduce_num_warps = 1 - - attn_config = { - "TILE_SIZE": tile_size, - "NUM_SEGMENTS_PER_SEQ": num_segments, - "num_warps": attn_warps, - "num_stages": 1, - } - reduce_config = { - "TILE_SIZE": tile_size, - "NUM_SEGMENTS_PER_SEQ": num_segments, - "num_warps": reduce_num_warps, - "num_stages": 1, - } - return attn_config, reduce_config - - -def register_fake_backend() -> None: - triton_backends.backends["mybackend_runtime"] = BackendRegistration( - compiler=FakeCompilerBackend, - driver=FakeDriver, - ) - runtime_driver.set_active(FakeDriver()) - - -def compile_2d_ptr(cfg: dict) -> str: - dims = cfg["dimensions"] - - num_tokens = dims["num_tokens"] - num_heads = dims["num_heads"] - num_kv_heads = dims["num_kv_heads"] - head_dim = dims["head_dim"] - padded_head_dim = next_power_of_2(head_dim) - num_blocks = dims["num_blocks"] - block_size = dims["block_size"] - batch_size = dims["batch_size"] - num_blocks_per_seq = dims["num_blocks_per_seq"] - num_qq_tokens = dims.get("num_qq_tokens", 1) - max_seqlen_q = dims["max_seqlen_q"] - - num_queries_per_kv = num_heads // num_kv_heads - - q_shape = (num_tokens, num_heads, head_dim) - kv_shape = (num_blocks, block_size, num_kv_heads, head_dim) - - flags = cfg["feature_flags"] - use_alibi_slopes = flags["use_alibi_slopes"] - use_softcap = flags["use_softcap"] - use_sinks = flags["use_sinks"] - use_fp8 = flags["use_fp8"] - use_qq_bias = "num_qq_tokens" in dims - all_decode = flags["all_decode"] - sliding_window = flags["sliding_window"] - - cfg_2d = cfg["config"] - block_q = cfg_2d["block_q"] - block_m = cfg_2d["block_m"] - tile_size = cfg_2d["tile_size"] - num_warps = cfg_2d["num_warps"] - num_stages = cfg_2d["num_stages"] - total_q_blocks = cfg_2d["total_q_blocks"] - - kwargs = { - "output_ptr": FakeTensor("bf16", q_shape), - "query_ptr": FakeTensor("bf16", q_shape), - "key_cache_ptr": FakeTensor("bf16", kv_shape), - "value_cache_ptr": FakeTensor("bf16", kv_shape), - "sink_ptr": FakeTensor("fp32", (num_heads,)), - "block_tables_ptr": FakeTensor("i32", (batch_size, num_blocks_per_seq)), - "seq_lens_ptr": FakeTensor("i32", (batch_size,)), - "alibi_slopes_ptr": FakeTensor("fp32", (num_heads,)), - "qq_bias_ptr": FakeTensor("fp32", (num_qq_tokens, num_qq_tokens)), - "scale_ptr": FakeTensor("fp32", (1,)), - "k_scale_ptr": FakeTensor("fp32", (1,)), - "v_scale_ptr": FakeTensor("fp32", (1,)), - "out_scale_ptr": FakeTensor("fp32", (1,)), - "softcap_ptr": FakeTensor("fp32", (1,)), - "block_table_stride_ptr": FakeTensor("i64", (1,)), - "query_stride_0_ptr": FakeTensor("i64", (1,)), - "query_stride_1_ptr": FakeTensor("i64", (1,)), - "output_stride_0_ptr": FakeTensor("i64", (1,)), - "output_stride_1_ptr": FakeTensor("i64", (1,)), - "qq_bias_stride_0_ptr": FakeTensor("i64", (1,)), - "stride_k_cache_0_ptr": FakeTensor("i64", (1,)), - "stride_k_cache_1_ptr": FakeTensor("i64", (1,)), - "stride_k_cache_2_ptr": FakeTensor("i64", (1,)), - "stride_v_cache_0_ptr": FakeTensor("i64", (1,)), - "stride_v_cache_1_ptr": FakeTensor("i64", (1,)), - "stride_v_cache_2_ptr": FakeTensor("i64", (1,)), - "query_start_len_ptr": FakeTensor("i32", (batch_size + 1,)), - "num_seqs_ptr": FakeTensor("i32", (1,)), - "num_query_heads": num_heads, - "num_queries_per_kv": num_queries_per_kv, - "BLOCK_SIZE": block_size, - "TILE_SIZE": tile_size, - "HEAD_SIZE": head_dim, - "HEAD_SIZE_PADDED": padded_head_dim, - "USE_ALIBI_SLOPES": use_alibi_slopes, - "USE_QQ_BIAS": use_qq_bias, - "USE_SOFTCAP": use_softcap, - "USE_SINKS": use_sinks, - "SLIDING_WINDOW": sliding_window, - "stride_k_cache_3": 1, - "stride_v_cache_3": 1, - "BLOCK_Q": block_q, - "BLOCK_M": block_m, - "USE_FP8": use_fp8, - "ALL_DECODE": all_decode, - # runtime options - "num_warps": num_warps, - "num_stages": num_stages, - } - - kernel = kernel_unified_attention_2d_ptr.warmup(grid=(num_kv_heads, total_q_blocks), **kwargs) - return kernel.asm["ttir"] - - -def compile_3d_ptr(cfg: dict) -> str: - dims = cfg["dimensions"] - - num_tokens = dims["num_tokens"] - num_heads = dims["num_heads"] - num_kv_heads = dims["num_kv_heads"] - head_dim = dims["head_dim"] - padded_head_dim = next_power_of_2(head_dim) - num_blocks = dims["num_blocks"] - block_size = dims["block_size"] - batch_size = dims["batch_size"] - num_blocks_per_seq = dims["num_blocks_per_seq"] - num_qq_tokens = dims.get("num_qq_tokens", 1) - cu_count = dims["cu_count"] - num_queries_per_kv = num_heads // num_kv_heads - - # unused - max_seqlen_k = 0 - attn_cfg = cfg["config"] - - block_m = attn_cfg["block_m"] - block_q = attn_cfg["block_q"] - tile_size = attn_cfg["tile_size"] - num_warps = attn_cfg["num_warps"] - num_stages = attn_cfg["num_stages"] - num_segments_per_seq = attn_cfg["num_segments_per_seq"] - total_q_blocks = attn_cfg["total_q_blocks"] - - q_shape = (num_tokens, num_heads, head_dim) - kv_shape = (num_blocks, block_size, num_kv_heads, head_dim) - segm_out_shape = (num_tokens, num_heads, num_segments_per_seq, triton.next_power_of_2(head_dim)) - segm_stat_shape = (num_tokens, num_heads, num_segments_per_seq) - - flags = cfg["feature_flags"] - use_alibi_slopes = flags["use_alibi_slopes"] - use_softcap = flags["use_softcap"] - use_sinks = flags["use_sinks"] - all_decode = flags["all_decode"] - sliding_window = flags["sliding_window"] - use_qq_bias = "num_qq_tokens" in dims - - kwargs = { - "query_ptr": FakeTensor("bf16", q_shape), - "key_cache_ptr": FakeTensor("bf16", kv_shape), - "value_cache_ptr": FakeTensor("bf16", kv_shape), - "sink_ptr": FakeTensor("fp32", (num_heads,)), - "block_tables_ptr": FakeTensor("i32", (batch_size, num_blocks_per_seq)), - "seq_lens_ptr": FakeTensor("i32", (batch_size,)), - "alibi_slopes_ptr": FakeTensor("fp32", (num_heads,)), - "qq_bias_ptr": FakeTensor("fp32", (num_qq_tokens, num_qq_tokens)), - "scale_ptr": FakeTensor("fp32", (1,)), - "k_scale_ptr": FakeTensor("fp32", (1,)), - "v_scale_ptr": FakeTensor("fp32", (1,)), - "softcap_ptr": FakeTensor("fp32", (1,)), - "num_query_heads": num_heads, - "num_queries_per_kv": num_queries_per_kv, - "block_table_stride_ptr": FakeTensor("i64", (1,)), - "query_stride_0_ptr": FakeTensor("i64", (1,)), - "query_stride_1_ptr": FakeTensor("i64", (1,)), - "qq_bias_stride_0_ptr": FakeTensor("i64", (1,)), - "BLOCK_SIZE": block_size, - "TILE_SIZE": tile_size, - "HEAD_SIZE": head_dim, - "HEAD_SIZE_PADDED": padded_head_dim, - "USE_ALIBI_SLOPES": use_alibi_slopes, - "USE_QQ_BIAS": use_qq_bias, - "USE_SOFTCAP": use_softcap, - "USE_SINKS": use_sinks, - "SLIDING_WINDOW": sliding_window, - "stride_k_cache_0_ptr": FakeTensor("i64", (1,)), - "stride_k_cache_1_ptr": FakeTensor("i64", (1,)), - "stride_k_cache_2_ptr": FakeTensor("i64", (1,)), - "stride_k_cache_3": 1, - "stride_v_cache_0_ptr": FakeTensor("i64", (1,)), - "stride_v_cache_1_ptr": FakeTensor("i64", (1,)), - "stride_v_cache_2_ptr": FakeTensor("i64", (1,)), - "stride_v_cache_3": 1, - "query_start_len_ptr": FakeTensor("i32", (batch_size + 1,)), - "BLOCK_Q": block_q, - "num_seqs_ptr": FakeTensor("i32", (1,)), - "BLOCK_M": block_m, - "NUM_SEGMENTS_PER_SEQ": num_segments_per_seq, - "ALL_DECODE": all_decode, - "segm_output_ptr": FakeTensor("fp32", segm_out_shape), - "segm_max_ptr": FakeTensor("fp32", segm_stat_shape), - "segm_expsum_ptr": FakeTensor("fp32", segm_stat_shape), - "num_warps": num_warps, - "num_stages": num_stages, - } - - kernel = kernel_unified_attention_3d_ptr.warmup( - grid=(total_q_blocks, num_kv_heads, num_segments_per_seq), - **kwargs, - ) - return kernel.asm["ttir"] - - -def compile_reduce_ptr(cfg: dict) -> str: - dims = cfg["dimensions"] - - num_tokens = dims["num_tokens"] - num_heads = dims["num_heads"] - head_dim = dims["head_dim"] - padded_head_dim = next_power_of_2(head_dim) - batch_size = dims["batch_size"] - block_size = dims["block_size"] - num_kv_heads = dims.get("num_kv_heads", 1) - cu_count = dims["cu_count"] - num_queries_per_kv = max(1, num_heads // num_kv_heads) - - # unused - max_seqlen_k = 0 - - - reduce_cfg = cfg["config"] - num_segments_per_seq = reduce_cfg["num_segments_per_seq"] - tile_size = reduce_cfg["tile_size"] - num_warps = reduce_cfg["num_warps"] - num_stages = reduce_cfg["num_stages"] - block_q = reduce_cfg["block_q"] - - out_shape = (num_tokens, num_heads, head_dim) - segm_out_shape = (num_tokens, num_heads, num_segments_per_seq, head_dim) - segm_stat_shape = (num_tokens, num_heads, num_segments_per_seq) - out_strides = contiguous_strides(out_shape) - - use_fp8 = cfg["feature_flags"]["use_fp8"] - - kwargs = { - "segm_output_ptr": FakeTensor("fp32", segm_out_shape), - "segm_max_ptr": FakeTensor("fp32", segm_stat_shape), - "segm_expsum_ptr": FakeTensor("fp32", segm_stat_shape), - "seq_lens_ptr": FakeTensor("i32", (batch_size,)), - "num_seqs_ptr": FakeTensor("i32", (1,)), - "num_query_heads": num_heads, - "out_scale_inv_ptr": FakeTensor("fp32", (1,)), - "output_stride_0_ptr": FakeTensor("i64", (1,)), - "output_stride_1_ptr": FakeTensor("i64", (1,)), - "block_table_stride_ptr": FakeTensor("i64", (1,)), - "TILE_SIZE": tile_size, - "HEAD_SIZE": head_dim, - "HEAD_SIZE_PADDED": padded_head_dim, - "query_start_len_ptr": FakeTensor("i32", (batch_size + 1,)), - "BLOCK_Q": block_q, - "NUM_SEGMENTS_PER_SEQ": num_segments_per_seq, - "USE_FP8": use_fp8, - "output_ptr": FakeTensor("bf16", out_shape), - "num_warps": num_warps, - "num_stages": num_stages, - } - - kernel = reduce_segments_ptr.warmup(grid=(num_tokens, num_heads), **kwargs) - return kernel.asm["ttir"] - - -def handle_sigint(signum, frame): - sys.exit(0) - - -def main() -> None: - register_fake_backend() - signal.signal(signal.SIGINT, handle_sigint) - - for raw_line in sys.stdin: - line = raw_line.strip() - if not line: - continue - - try: - cfg = json.loads(line) - if not isinstance(cfg, dict): - raise ValueError("request must be a JSON object") - - if "kernel_unified_attention_2d_ptr" in cfg: - ttir = compile_2d_ptr(cfg["kernel_unified_attention_2d_ptr"]) - elif "kernel_unified_attention_3d_ptr" in cfg: - ttir = compile_3d_ptr(cfg["kernel_unified_attention_3d_ptr"]) - elif "reduce_segments_ptr" in cfg: - ttir = compile_reduce_ptr(cfg["reduce_segments_ptr"]) - - response = {"ok": True, "result": ttir} - except Exception as exc: - response = {"ok": False, "error": str(exc)} - - print(json.dumps(response), flush=True) diff --git a/zml/attention/triton/generate.zig b/zml/attention/triton/generate.zig deleted file mode 100644 index 8e38044e..00000000 --- a/zml/attention/triton/generate.zig +++ /dev/null @@ -1,111 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const runfiles = @import("runfiles"); - -fn pyStatusCheck(status: c.PyStatus) void { - if (c.PyStatus_Exception(status) != 0) { - if (c.PyStatus_IsExit(status) != 0) { - std.process.exit(@intCast(status.exitcode)); - } - c.Py_ExitStatusException(status); - } -} - -pub fn toPosixPathW(file_path: []const u8) error{ InvalidUtf8, NameTooLong, CodepointTooLarge }![std.posix.PATH_MAX - 1:0]c.wchar_t { - var view = std.unicode.Utf8View.init(file_path) catch return error.InvalidUtf8; - var it = view.iterator(); - - var path_with_null: [std.posix.PATH_MAX - 1:0]c.wchar_t = undefined; - var len: usize = 0; - while (it.nextCodepoint()) |cp| { - if (len + 1 >= path_with_null.len) return error.NameTooLong; - path_with_null[len] = std.math.cast(c.wchar_t, cp) orelse return error.CodepointTooLarge; - len += 1; - } - path_with_null[len] = 0; - return path_with_null; -} - -pub fn main(init: std.process.Init.Minimal) !void { - const allocator = std.heap.c_allocator; - - var threaded: std.Io.Threaded = .init(allocator, .{}); - defer threaded.deinit(); - - const io = threaded.io(); - - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - - { - var preconfig: c.PyPreConfig = undefined; - c.PyPreConfig_InitIsolatedConfig(&preconfig); - preconfig.utf8_mode = 1; - pyStatusCheck(c.Py_PreInitialize(&preconfig)); - } - - var config: c.PyConfig = undefined; - c.PyConfig_InitIsolatedConfig(&config); - defer c.PyConfig_Clear(&config); - - config.module_search_paths_set = 1; - config.optimization_level = 2; - config.write_bytecode = 0; - - pyStatusCheck(c.PyConfig_SetBytesArgv(&config, @intCast(init.args.vector.len), @ptrCast(init.args.vector.ptr))); - - var self_exe_dir_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const self_exe_dir = blk: { - const n = try std.process.executableDirPath(io, &self_exe_dir_buf); - break :blk self_exe_dir_buf[0..n]; - }; - - { - var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const home = try std.fmt.bufPrintZ(&buf, "{f}{d}.{d}", .{ - std.Io.Dir.path.fmtJoin(&.{ - self_exe_dir, - "..", - "lib", - "python", - }), - c.PY_MAJOR_VERSION, - c.PY_MINOR_VERSION, - }); - pyStatusCheck(c.PyConfig_SetBytesString(&config, &config.home, home)); - pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(home))); - } - - { - var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const site_packages = try std.fmt.bufPrint(&buf, "{f}", .{ - std.Io.Dir.path.fmtJoin(&.{ - self_exe_dir, - "..", - "site-packages", - }), - }); - pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(site_packages))); - } - - pyStatusCheck(c.Py_InitializeFromConfig(&config)); - defer c.Py_Finalize(); - - const module = c.PyImport_ImportModule("generate"); - if (module == null) { - c.PyErr_Print(); - return; - } - defer c.Py_DecRef(module); - - const func = c.PyObject_GetAttrString(module, "main"); - defer c.Py_DecRef(func); - - const result = c.PyObject_CallNoArgs(func); - if (result == null) { - c.PyErr_Print(); - return; - } - defer c.Py_DecRef(result); -} diff --git a/zml/attention/triton/libpython.h b/zml/attention/triton/libpython.h deleted file mode 100644 index a5b5c4a1..00000000 --- a/zml/attention/triton/libpython.h +++ /dev/null @@ -1,3 +0,0 @@ -#define PY_SSIZE_T_CLEAN -#include - diff --git a/zml/attention/triton/requirements.in b/zml/attention/triton/requirements.in deleted file mode 100644 index e0b3eb03..00000000 --- a/zml/attention/triton/requirements.in +++ /dev/null @@ -1 +0,0 @@ -triton==3.6.0 diff --git a/zml/attention/triton/requirements.lock.txt b/zml/attention/triton/requirements.lock.txt deleted file mode 100644 index 2e3244c2..00000000 --- a/zml/attention/triton/requirements.lock.txt +++ /dev/null @@ -1,20 +0,0 @@ -# This file was autogenerated by uv via the following command: -# bazel run //zml/attention/triton:requirements.update ---index-url https://pypi.org/simple - -triton==3.6.0 \ - --hash=sha256:0b3a97e8ed304dfa9bd23bb41ca04cdf6b2e617d5e782a8653d616037a5d537d \ - --hash=sha256:10c7f76c6e72d2ef08df639e3d0d30729112f47a56b0c81672edc05ee5116ac9 \ - --hash=sha256:1722e172d34e32abc3eb7711d0025bb69d7959ebea84e3b7f7a341cd7ed694d6 \ - --hash=sha256:374f52c11a711fd062b4bfbb201fd9ac0a5febd28a96fb41b4a0f51dde3157f4 \ - --hash=sha256:448e02fe6dc898e9e5aa89cf0ee5c371e99df5aa5e8ad976a80b93334f3494fd \ - --hash=sha256:46bd1c1af4b6704e554cad2eeb3b0a6513a980d470ccfa63189737340c7746a7 \ - --hash=sha256:49df5ef37379c0c2b5c0012286f80174fcf0e073e5ade1ca9a86c36814553651 \ - --hash=sha256:6c723cfb12f6842a0ae94ac307dba7e7a44741d720a40cf0e270ed4a4e3be781 \ - --hash=sha256:74caf5e34b66d9f3a429af689c1c7128daba1d8208df60e81106b115c00d6fca \ - --hash=sha256:a17a5d5985f0ac494ed8a8e54568f092f7057ef60e1b0fa09d3fd1512064e803 \ - --hash=sha256:a6550fae429e0667e397e5de64b332d1e5695b73650ee75a6146e2e902770bea \ - --hash=sha256:d002e07d7180fd65e622134fbd980c9a3d4211fb85224b56a0a0efbd422ab72f \ - --hash=sha256:e8e323d608e3a9bfcc2d9efcc90ceefb764a82b99dea12a86d643c72539ad5d3 \ - --hash=sha256:ef5523241e7d1abca00f1d240949eebdd7c673b005edbbce0aca95b8191f1d43 - # via -r zml/attention/triton/requirements.in diff --git a/zml/attention/triton/triton_kernels/__init__.py b/zml/attention/triton/triton_kernels/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/zml/attention/triton/triton_kernels/unified_attention.py b/zml/attention/triton/triton_kernels/unified_attention.py deleted file mode 100755 index ad6242e7..00000000 --- a/zml/attention/triton/triton_kernels/unified_attention.py +++ /dev/null @@ -1,1073 +0,0 @@ -# The kernels in this file are adapted from vLLM: -# https://github.com/vllm-project/vllm/blob/main/vllm/attention/ops/triton_unified_attention.py -# -# NOTE: -# `scale` in `kernel_unified_attention_2d` is intentionally a runtime scalar -# (not `tl.constexpr`) so `kernel_unified_attention_2d_ptr` can load it from -# pointer arguments and forward it. If you see a performance regression, first -# thing to try is restoring `scale` to `tl.constexpr` and bypassing the ptr -# wrapper for that argument. -import triton -import triton.language as tl - -FP8_E4M3_MIN = -448.0 -FP8_E4M3_MAX = 448.0 - - -@triton.jit -def fast_exp(x): - RCP_LN2: tl.constexpr = 1.4426950408889634 - return tl.math.exp2(x * RCP_LN2) - - -@triton.jit -def cdiv_fn(x, y): - return (x + y - 1) // y - - -@triton.jit -def apply_softcap(S, x): - Sdiv = S / x - p1 = tl.math.exp2(Sdiv) - p2 = tl.math.exp2(-Sdiv) - return x * (p1 - p2) / (p1 + p2) - - -@triton.jit -def find_seq_idx( - query_start_len_ptr, - target_idx, - num_seqs, - BLOCK_Q: tl.constexpr, - use_q_block_mode: tl.constexpr, -): - left: tl.int32 = 0 - right = num_seqs - while left < right: - mid = (left + right) // 2 - val = tl.load(query_start_len_ptr + mid) - mid_val = val // BLOCK_Q + mid if use_q_block_mode else val - - if mid_val <= target_idx: - left = mid + 1 - else: - right = mid - - return left - 1 - - -@triton.jit -def kernel_unified_attention_2d( - output_ptr, # [num_tokens, num_query_heads, head_size] - query_ptr, # [num_tokens, num_query_heads, head_size] - key_cache_ptr, # [num_blks, blk_size, num_kv_heads, head_size] - value_cache_ptr, # [num_blks, blk_size, num_kv_heads, head_size] - sink_ptr, # [num_query_heads] - block_tables_ptr, # [num_seqs, max_num_blocks_per_seq] - seq_lens_ptr, # [num_seqs] - alibi_slopes_ptr, # [num_query_heads] - qq_bias_ptr, # [num_query_tokens, num_query_tokens] - scale, # float32 - k_scale, # float32 - v_scale, # float32 - out_scale, # float32 - softcap, # float32 - num_query_heads: tl.constexpr, # int - num_queries_per_kv: tl.constexpr, # int - block_table_stride: tl.int64, # int - query_stride_0: tl.int64, # int - query_stride_1: tl.int64, # int, should be equal to head_size - output_stride_0: tl.int64, # int - output_stride_1: tl.int64, # int, should be equal to head_size - qq_bias_stride_0: tl.int64, # int - BLOCK_SIZE: tl.constexpr, # int - TILE_SIZE: tl.constexpr, # int must be power of 2 - HEAD_SIZE: tl.constexpr, # int - HEAD_SIZE_PADDED: tl.constexpr, # int, must be power of 2 - USE_ALIBI_SLOPES: tl.constexpr, # bool - USE_QQ_BIAS: tl.constexpr, # bool - USE_SOFTCAP: tl.constexpr, # bool - USE_SINKS: tl.constexpr, # bool - SLIDING_WINDOW: tl.constexpr, # int - stride_k_cache_0: tl.int64, # int - stride_k_cache_1: tl.int64, # int - stride_k_cache_2: tl.int64, # int - stride_k_cache_3: tl.constexpr, # int - stride_v_cache_0: tl.int64, # int - stride_v_cache_1: tl.int64, # int - stride_v_cache_2: tl.int64, # int - stride_v_cache_3: tl.constexpr, # int - query_start_len_ptr, # [num_seqs+1] - BLOCK_Q: tl.constexpr, # int - num_seqs: tl.int32, - BLOCK_M: tl.constexpr, # int - USE_FP8: tl.constexpr, # bool - FP8_MIN: tl.constexpr = FP8_E4M3_MIN, - FP8_MAX: tl.constexpr = FP8_E4M3_MAX, - ALL_DECODE: tl.constexpr = False, # bool -): - kv_head_idx = tl.program_id(0) - q_block_global_idx = tl.program_id(1) - - # needed to use exp2 (exp2 -> exp conversion) - RCP_LN2 = 1.4426950408889634 - qk_scale = scale * RCP_LN2 - - seq_idx = find_seq_idx( - query_start_len_ptr, q_block_global_idx, num_seqs, BLOCK_Q, True - ) - - q_block_start_idx = tl.load(query_start_len_ptr + seq_idx) // BLOCK_Q + seq_idx - - q_block_local_idx = q_block_global_idx - q_block_start_idx - - cur_batch_in_all_start_index = tl.load(query_start_len_ptr + seq_idx) - cur_batch_in_all_stop_index = tl.load(query_start_len_ptr + seq_idx + 1) - - cur_batch_query_len = cur_batch_in_all_stop_index - cur_batch_in_all_start_index - - if q_block_local_idx * BLOCK_Q >= cur_batch_query_len: - return - - offs_m = tl.arange(0, BLOCK_M) - offs_d = tl.arange(0, HEAD_SIZE_PADDED) - offs_t = tl.arange(0, TILE_SIZE) - query_pos = q_block_local_idx * BLOCK_Q + offs_m // num_queries_per_kv - - query_offset_0 = cur_batch_in_all_start_index + query_pos - query_offset_1 = kv_head_idx * num_queries_per_kv + offs_m % num_queries_per_kv - query_offset = ( - query_offset_0[:, None] * query_stride_0 - + query_offset_1[:, None] * query_stride_1 - + offs_d[None, :] - ) - - if HEAD_SIZE_PADDED != HEAD_SIZE: - dim_mask = offs_d < HEAD_SIZE - else: - dim_mask = tl.full((1,), 1, dtype=tl.int1) - query_mask_0 = query_pos < cur_batch_query_len - query_mask_1 = query_offset_1 < num_query_heads - - if ALL_DECODE or BLOCK_M >= num_query_heads: - Q_cache_modifier: tl.constexpr = ".cg" - else: - Q_cache_modifier: tl.constexpr = "" - # Q : (BLOCK_M, HEAD_SIZE_PADDED) - Q = tl.load( - query_ptr + query_offset, - mask=dim_mask[None, :] & query_mask_0[:, None] & query_mask_1[:, None], - other=0.0, - cache_modifier=Q_cache_modifier, - ) - - block_table_offset = seq_idx * block_table_stride - - if not USE_SINKS: - M = tl.full([BLOCK_M], float("-inf"), dtype=tl.float32) - else: - # Prescale with RCP_LN2, needed for exp2 - M = ( - tl.load( - sink_ptr + query_offset_1, - mask=query_mask_1, - other=float("-inf"), - ).to(dtype=tl.float32) - * RCP_LN2 - ) - - L = tl.full([BLOCK_M], 1.0, dtype=tl.float32) - acc = tl.zeros([BLOCK_M, HEAD_SIZE_PADDED], dtype=tl.float32) - - # sequence len for this particular sequence - seq_len = tl.load(seq_lens_ptr + seq_idx) - - # context length for this particular sequences - context_len = seq_len - cur_batch_query_len - - # alibi slope for this head - if USE_ALIBI_SLOPES: - alibi_slope = tl.load( - alibi_slopes_ptr + query_offset_1, mask=query_mask_1, other=0.0 - ) - - # query-query attention bias - if USE_QQ_BIAS: - qq_bias_row_ptrs = ( - qq_bias_ptr + query_pos[:, None] * qq_bias_stride_0 - ) # shape: [BLOCK_M] - - # compute the length of the longest sequence prefix spanned by any - # query token in the current q_block (q_block_local_idx) - max_seq_prefix_len = ( - context_len - + q_block_local_idx * BLOCK_Q - + (BLOCK_M - 1) // num_queries_per_kv - + 1 - ) - - # adjust for potential padding in the last q_block by considering the - # actual sequence length - max_seq_prefix_len = tl.minimum(max_seq_prefix_len, seq_len) - - # calculate the number of tiles that need to be processed to - # cover the longest sequence prefix (due to causal masking, tiles beyond - # this prefix can be skipped) - num_tiles = cdiv_fn(max_seq_prefix_len, TILE_SIZE) - - # ---- Sliding-window tile pruning -------------------- - # Default: keep previous global behavior - tile_start = 0 - tile_end = num_tiles - if SLIDING_WINDOW > 0: - # Query rows covered by this Q-block - qpos_lo = q_block_local_idx * BLOCK_Q - qpos_hi = tl.minimum( - qpos_lo + (BLOCK_M - 1) // num_queries_per_kv, - cur_batch_query_len - 1, - ) - # For sliding window, each query position q can only attend to - # keys in the range [q_abs - SLIDING_WINDOW + 1, q_abs] - # where q_abs = context_len + q - # The union of allowed key positions for this Q-block is: - # [context_len + qpos_lo - SLIDING_WINDOW + 1, context_len + qpos_hi] - first_allowed_key = context_len + qpos_lo - SLIDING_WINDOW + 1 - last_allowed_key = context_len + qpos_hi - # Convert to tile indices and clamp - tile_start = tl.maximum(0, first_allowed_key // TILE_SIZE) - tile_end = tl.minimum((last_allowed_key // TILE_SIZE) + 1, num_tiles) - - KV_cache_modifier: tl.constexpr = ".cg" if ALL_DECODE else "" - # iterate through tiles (now limited to the sliding window range) - for j in range(tile_start, tile_end): - seq_offset = j * TILE_SIZE + offs_t - # to reduce the masking effect when not needed - if TILE_SIZE == BLOCK_SIZE: - tile_mask = tl.full((1,), 1, dtype=tl.int1) - else: - tile_mask = seq_offset < max_seq_prefix_len - - physical_block_idx = tl.load( - block_tables_ptr + block_table_offset + seq_offset // BLOCK_SIZE - ).to(tl.int64) - - v_offset = ( - physical_block_idx[:, None] * stride_v_cache_0 - + kv_head_idx * stride_v_cache_2 - + offs_d[None, :] * stride_v_cache_3 - + (seq_offset % BLOCK_SIZE)[:, None] * stride_v_cache_1 - ) - - k_offset = ( - physical_block_idx[None, :] * stride_k_cache_0 - + kv_head_idx * stride_k_cache_2 - + offs_d[:, None] * stride_k_cache_3 - + (seq_offset % BLOCK_SIZE)[None, :] * stride_k_cache_1 - ) - - # K : (HEAD_SIZE, TILE_SIZE) - K_load = tl.load( - key_cache_ptr + k_offset, - mask=dim_mask[:, None] & tile_mask[None, :], - other=0.0, - cache_modifier=KV_cache_modifier, - ) - - if K_load.dtype.is_fp8(): - if Q.dtype.is_fp8(): - K = K_load - else: - K = (K_load.to(tl.float32) * k_scale).to(Q.dtype) - else: - K = K_load - - # V : (TILE_SIZE, HEAD_SIZE) - V_load = tl.load( - value_cache_ptr + v_offset, - mask=dim_mask[None, :] & tile_mask[:, None], - other=0.0, - cache_modifier=KV_cache_modifier, - ) - - if V_load.dtype.is_fp8(): - if Q.dtype.is_fp8(): - V = V_load - else: - V = (V_load.to(tl.float32) * v_scale).to(Q.dtype) - else: - V = V_load - - # S : (BLOCK_M, TILE_SIZE) - # qk_scale = scale * RCP_LN2 (log_2 e) so that we can use exp2 later - S = qk_scale * tl.dot(Q, K) - - if USE_SOFTCAP: - # softcap here uses exp2 and consumes RCP_LN2 conversion. - # multiply by RCP_LN2 again to be used in later exp2 - S = apply_softcap(S, softcap) * RCP_LN2 - seq_mask = seq_offset[None, :] < context_len + query_pos[:, None] + 1 - - S = tl.where( - query_mask_1[:, None] & query_mask_0[:, None] & seq_mask, S, float("-inf") - ) - - if SLIDING_WINDOW > 0: - S = tl.where( - (context_len + query_pos[:, None] - seq_offset) < SLIDING_WINDOW, - S, - float("-inf"), - ) - - if USE_ALIBI_SLOPES: - # prescale w. RCP_LN2 for later exp2 - S += alibi_slope[:, None] * (seq_offset - context_len) * RCP_LN2 - - if USE_QQ_BIAS: - # compute key positions relative to query section - key_rel_pos = seq_offset - context_len # shape: [BLOCK_SIZE] - # load bias only for keys that correspond to queries - is_query_key = key_rel_pos >= 0 and key_rel_pos < qq_bias_stride_0 - qq_bias = tl.load( - qq_bias_row_ptrs + key_rel_pos[None, :], - mask=is_query_key[None, :], # avoid OOB for context keys - other=0.0, - ) - # prescale w. RCP_LN2 for later exp2 - S += qq_bias * RCP_LN2 - - # compute running maximum - # m_j : (BLOCK_M,) - m_j = tl.maximum(M, tl.max(S, axis=1)) - - # For sliding window there's a chance the max is -inf due to masking of - # the entire row. In this case we need to set m_j 0 to avoid NaN - m_j = tl.where(m_j > float("-inf"), m_j, 0.0) - - # P : (BLOCK_M, TILE_SIZE) - P = tl.math.exp2(S - m_j[:, None]) - - # l_j : (BLOCK_M,) - l_j = tl.sum(P, axis=1) - - # alpha : (BLOCK_M, ) - alpha = tl.math.exp2(M - m_j) - - # acc : (BLOCK_M, HEAD_SIZE_PADDED) - acc = acc * alpha[:, None] - - # update constants - L = L * alpha + l_j - M = m_j - - # acc : (BLOCK_M, HEAD_SIZE_PADDED) - acc += tl.dot(P.to(V.dtype), V) - - # epilogue - # This helps the compiler do Newton Raphson on l_i vs on acc which is much larger. - one_over_L = 1.0 / L[:, None] - acc = acc * one_over_L - if USE_FP8: - acc = acc * out_scale - acc = tl.clamp(acc, FP8_MIN, FP8_MAX) - - output_offset = ( - query_offset_0[:, None] * output_stride_0 - + query_offset_1[:, None] * output_stride_1 - + offs_d[None, :] - ) - - tl.store( - output_ptr + output_offset, - acc, - mask=dim_mask[None, :] & query_mask_0[:, None] & query_mask_1[:, None], - ) - - -@triton.jit -def kernel_unified_attention_3d( - segm_output_ptr, - # [num_tokens, num_query_heads, num_segments, head_size] - segm_max_ptr, # [num_tokens, num_query_heads, num_segments] - segm_expsum_ptr, # [num_tokens, num_query_heads, num_segments] - query_ptr, # [num_tokens, num_query_heads, head_size] - key_cache_ptr, # [num_blks, blk_size, num_kv_heads, head_size] - value_cache_ptr, # [num_blks, blk_size, num_kv_heads, head_size] - sink_ptr, # [num_query_heads] - block_tables_ptr, # [num_seqs, max_num_blocks_per_seq] - seq_lens_ptr, # [num_seqs] - alibi_slopes_ptr, # [num_query_heads] - qq_bias_ptr, # [num_query_tokens, num_query_tokens] - scale, # float32 - k_scale, # float32 - v_scale, # float32 - softcap, # float32 - num_query_heads: tl.constexpr, # int - num_queries_per_kv: tl.constexpr, # int - block_table_stride: tl.int64, # int - query_stride_0: tl.int64, # int - query_stride_1: tl.int64, # int, should be equal to head_size - qq_bias_stride_0: tl.int64, # int - BLOCK_SIZE: tl.constexpr, # int - TILE_SIZE: tl.constexpr, # int, must be power of 2 - HEAD_SIZE: tl.constexpr, # int - HEAD_SIZE_PADDED: tl.constexpr, # int, must be power of 2 - USE_ALIBI_SLOPES: tl.constexpr, # bool - USE_QQ_BIAS: tl.constexpr, # bool - USE_SOFTCAP: tl.constexpr, # bool - USE_SINKS: tl.constexpr, # bool - SLIDING_WINDOW: tl.constexpr, # int - stride_k_cache_0: tl.int64, # int - stride_k_cache_1: tl.int64, # int - stride_k_cache_2: tl.int64, # int - stride_k_cache_3: tl.constexpr, # int - stride_v_cache_0: tl.int64, # int - stride_v_cache_1: tl.int64, # int - stride_v_cache_2: tl.int64, # int - stride_v_cache_3: tl.constexpr, # int - query_start_len_ptr, # [num_seqs+1] - BLOCK_Q: tl.constexpr, # int - num_seqs: tl.int32, - BLOCK_M: tl.constexpr, # int - NUM_SEGMENTS_PER_SEQ: tl.constexpr, # int - ALL_DECODE: tl.constexpr = False, # bool -): - q_block_global_idx = tl.program_id(0) - kv_head_idx = tl.program_id(1) - segm_idx = tl.program_id(2) - - # needed to use exp2 (exp2 -> exp conversion) - RCP_LN2 = 1.4426950408889634 - qk_scale = scale * RCP_LN2 - - seq_idx = find_seq_idx( - query_start_len_ptr, q_block_global_idx, num_seqs, BLOCK_Q, True - ) - - q_block_start_idx = tl.load(query_start_len_ptr + seq_idx) // BLOCK_Q + seq_idx - - q_block_local_idx = q_block_global_idx - q_block_start_idx - - cur_batch_in_all_start_index = tl.load(query_start_len_ptr + seq_idx) - cur_batch_in_all_stop_index = tl.load(query_start_len_ptr + seq_idx + 1) - - cur_batch_query_len = cur_batch_in_all_stop_index - cur_batch_in_all_start_index - - if q_block_local_idx * BLOCK_Q >= cur_batch_query_len: - return - - # sequence len for this particular sequence - seq_len = tl.load(seq_lens_ptr + seq_idx) - - # number of segments for this particular sequence - num_segments = NUM_SEGMENTS_PER_SEQ - tiles_per_segment = cdiv_fn(seq_len, num_segments * TILE_SIZE) - - if segm_idx * tiles_per_segment * TILE_SIZE >= seq_len: - return - - offs_m = tl.arange(0, BLOCK_M) - offs_d = tl.arange(0, HEAD_SIZE_PADDED) - offs_t = tl.arange(0, TILE_SIZE) - query_pos = q_block_local_idx * BLOCK_Q + offs_m // num_queries_per_kv - - query_offset_0 = cur_batch_in_all_start_index + query_pos - query_offset_1 = kv_head_idx * num_queries_per_kv + offs_m % num_queries_per_kv - query_offset = ( - query_offset_0[:, None] * query_stride_0 - + query_offset_1[:, None] * query_stride_1 - + offs_d[None, :] - ) - - if HEAD_SIZE_PADDED != HEAD_SIZE: - dim_mask = offs_d < HEAD_SIZE - else: - dim_mask = tl.full((1,), 1, dtype=tl.int1) - query_mask_0 = query_pos < cur_batch_query_len - query_mask_1 = query_offset_1 < num_query_heads - - # Q : (BLOCK_M, HEAD_SIZE_PADDED) - Q = tl.load( - query_ptr + query_offset, - mask=dim_mask[None, :] & query_mask_0[:, None] & query_mask_1[:, None], - other=0.0, - ) - - block_table_offset = seq_idx * block_table_stride - - if USE_SINKS: - if segm_idx == 0: - # Prescale with RCP_LN2, needed for exp2 - M = ( - tl.load( - sink_ptr + query_offset_1, - mask=query_mask_1, - other=float("-inf"), - ).to(dtype=tl.float32) - * RCP_LN2 - ) - else: - M = tl.full([BLOCK_M], float("-inf"), dtype=tl.float32) - else: - M = tl.full([BLOCK_M], float("-inf"), dtype=tl.float32) - - L = tl.full([BLOCK_M], 1.0, dtype=tl.float32) - acc = tl.zeros([BLOCK_M, HEAD_SIZE_PADDED], dtype=tl.float32) - - # context length for this particular sequences - context_len = seq_len - cur_batch_query_len - - # alibi slope for this head - if USE_ALIBI_SLOPES: - alibi_slope = tl.load( - alibi_slopes_ptr + query_offset_1, mask=query_mask_1, other=0.0 - ) - - # query-query attention bias - if USE_QQ_BIAS: - qq_bias_row_ptrs = ( - qq_bias_ptr + query_pos[:, None] * qq_bias_stride_0 - ) # shape: [BLOCK_M] - - # compute the length of the longest sequence prefix spanned by any - # query token in the current q_block (q_block_local_idx) - max_seq_prefix_len = ( - context_len - + q_block_local_idx * BLOCK_Q - + (BLOCK_M - 1) // num_queries_per_kv - + 1 - ) - - # adjust for potential padding in the last q_block by considering the - # actual sequence length - max_seq_prefix_len = tl.minimum(max_seq_prefix_len, seq_len) - - # calculate the number of tiles that need to be processed to - # cover the longest sequence prefix (due to causal masking, tiles beyond - # this prefix can be skipped) - num_tiles = cdiv_fn(max_seq_prefix_len, TILE_SIZE) - - KV_cache_modifier: tl.constexpr = ".cg" if ALL_DECODE else "" - # iterate through tiles within current segment - for j in range( - segm_idx * tiles_per_segment, - min((segm_idx + 1) * tiles_per_segment, num_tiles), - ): - seq_offset = j * TILE_SIZE + offs_t - if TILE_SIZE == BLOCK_SIZE: - tile_mask = tl.full((1,), 1, dtype=tl.int1) - else: - tile_mask = seq_offset < max_seq_prefix_len - - physical_block_idx = tl.load( - block_tables_ptr + block_table_offset + seq_offset // BLOCK_SIZE - ).to(tl.int64) - - v_offset = ( - physical_block_idx[:, None] * stride_v_cache_0 - + kv_head_idx * stride_v_cache_2 - + offs_d[None, :] * stride_v_cache_3 - + (seq_offset % BLOCK_SIZE)[:, None] * stride_v_cache_1 - ) - - k_offset = ( - physical_block_idx[None, :] * stride_k_cache_0 - + kv_head_idx * stride_k_cache_2 - + offs_d[:, None] * stride_k_cache_3 - + (seq_offset % BLOCK_SIZE)[None, :] * stride_k_cache_1 - ) - - # K : (HEAD_SIZE, TILE_SIZE) - K_load = tl.load( - key_cache_ptr + k_offset, - mask=dim_mask[:, None] & tile_mask[None, :], - other=0.0, - cache_modifier=KV_cache_modifier, - ) - - if K_load.dtype.is_fp8(): - if Q.dtype.is_fp8(): - K = K_load - else: - K = (K_load.to(tl.float32) * k_scale).to(Q.dtype) - else: - K = K_load - - # V : (TILE_SIZE, HEAD_SIZE) - V_load = tl.load( - value_cache_ptr + v_offset, - mask=dim_mask[None, :] & tile_mask[:, None], - other=0.0, - cache_modifier=KV_cache_modifier, - ) - - if V_load.dtype.is_fp8(): - if Q.dtype.is_fp8(): - V = V_load - else: - V = (V_load.to(tl.float32) * v_scale).to(Q.dtype) - else: - V = V_load - - seq_mask = seq_offset[None, :] < context_len + query_pos[:, None] + 1 - - # S : (BLOCK_M, TILE_SIZE) - # qk_scale = scale * RCP_LN2 (log_2 e) so that we can use exp2 later - S = qk_scale * tl.dot(Q, K) - - if USE_SOFTCAP: - # softcap here uses exp2 and consumes RCP_LN2 conversion. - # multiply by RCP_LN2 again to be used in later exp2 - S = apply_softcap(S, softcap) * RCP_LN2 - - S = tl.where( - query_mask_1[:, None] & query_mask_0[:, None] & seq_mask, S, float("-inf") - ) - - if SLIDING_WINDOW > 0: - S = tl.where( - (context_len + query_pos[:, None] - seq_offset) < SLIDING_WINDOW, - S, - float("-inf"), - ) - - if USE_ALIBI_SLOPES: - # prescale w. RCP_LN2 for later exp2 - S += alibi_slope[:, None] * (seq_offset - context_len) * RCP_LN2 - - if USE_QQ_BIAS: - # compute key positions relative to query section - key_rel_pos = seq_offset - context_len # shape: [BLOCK_SIZE] - # load bias only for keys that correspond to queries - is_query_key = key_rel_pos >= 0 and key_rel_pos < qq_bias_stride_0 - qq_bias = tl.load( - qq_bias_row_ptrs + key_rel_pos[None, :], - mask=is_query_key[None, :], # avoid OOB for context keys - other=0.0, - ) - # prescale w. RCP_LN2 for later exp2 - S += qq_bias * RCP_LN2 - - # compute running maximum - # m_j : (BLOCK_M,) - m_j = tl.maximum(M, tl.max(S, axis=1)) - - # For sliding window there's a chance the max is -inf due to masking of - # the entire row. In this case we need to set m_j 0 to avoid NaN - m_j = tl.where(m_j > float("-inf"), m_j, 0.0) - - # P : (BLOCK_M, TILE_SIZE,) - P = tl.math.exp2(S - m_j[:, None]) - - # l_j : (BLOCK_M,) - l_j = tl.sum(P, axis=1) - - # alpha : (BLOCK_M, ) - alpha = tl.math.exp2(M - m_j) - - # acc : (BLOCK_M, HEAD_SIZE_PADDED) - acc = acc * alpha[:, None] - - # update constants - L = L * alpha + l_j - M = m_j - - # acc : (BLOCK_M, HEAD_SIZE_PADDED) - acc += tl.dot(P.to(V.dtype), V) - - segm_output_offset = ( - query_offset_0[:, None].to(tl.int64) - * (num_query_heads * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) - + query_offset_1[:, None] * (NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) - + segm_idx * HEAD_SIZE_PADDED - + tl.arange(0, HEAD_SIZE_PADDED)[None, :] - ) - tl.store( - segm_output_ptr + segm_output_offset, - acc, - mask=dim_mask[None, :] & query_mask_0[:, None] & query_mask_1[:, None], - ) - segm_offset = ( - query_offset_0.to(tl.int64) * (num_query_heads * NUM_SEGMENTS_PER_SEQ) - + query_offset_1 * NUM_SEGMENTS_PER_SEQ - + segm_idx - ) - tl.store(segm_max_ptr + segm_offset, M, mask=query_mask_0 & query_mask_1) - tl.store(segm_expsum_ptr + segm_offset, L, mask=query_mask_0 & query_mask_1) - - -@triton.jit -def reduce_segments( - output_ptr, # [num_tokens, num_query_heads, head_size] - segm_output_ptr, - # [num_tokens, num_query_heads, max_num_segments, head_size] - segm_max_ptr, # [num_tokens, num_query_heads, max_num_segments] - segm_expsum_ptr, # [num_tokens, num_query_heads, max_num_segments] - seq_lens_ptr, # [num_seqs] - num_seqs, # int - num_query_heads: tl.constexpr, # int - out_scale_inv, # float32 - output_stride_0: tl.int64, # int - output_stride_1: tl.int64, # int, should be equal to head_size - block_table_stride: tl.int64, # int - TILE_SIZE: tl.constexpr, # int - HEAD_SIZE: tl.constexpr, # int, must be power of 2 - HEAD_SIZE_PADDED: tl.constexpr, # int, must be power of 2 - query_start_len_ptr, # [num_seqs+1] - BLOCK_Q: tl.constexpr, # int - NUM_SEGMENTS_PER_SEQ: tl.constexpr, # int - USE_FP8: tl.constexpr, # bool - FP8_MIN: tl.constexpr = FP8_E4M3_MIN, - FP8_MAX: tl.constexpr = FP8_E4M3_MAX, -): - query_token_idx = tl.program_id(0) - query_head_idx = tl.program_id(1) - - seq_idx = find_seq_idx( - query_start_len_ptr, query_token_idx, num_seqs, BLOCK_Q, False - ) - - # sequence len for this particular sequence - seq_len = tl.load(seq_lens_ptr + seq_idx) - - # number of segments for this particular sequence - num_segments = NUM_SEGMENTS_PER_SEQ - tiles_per_segment = cdiv_fn(seq_len, num_segments * TILE_SIZE) - - # create masks for subsequent loads - act_num_segments = cdiv_fn(seq_len, tiles_per_segment * TILE_SIZE) - segm_mask = tl.arange(0, NUM_SEGMENTS_PER_SEQ) < tl.full( - [NUM_SEGMENTS_PER_SEQ], act_num_segments, dtype=tl.int32 - ) - - if HEAD_SIZE_PADDED != HEAD_SIZE: - dim_mask = offs_d < HEAD_SIZE - else: - dim_mask = tl.full((1,), 1, dtype=tl.int1) - - # load segment maxima - segm_offset = ( - query_token_idx.to(tl.int64) * (num_query_heads * NUM_SEGMENTS_PER_SEQ) - + query_head_idx * NUM_SEGMENTS_PER_SEQ - + tl.arange(0, NUM_SEGMENTS_PER_SEQ) - ) - segm_max = tl.load(segm_max_ptr + segm_offset, mask=segm_mask, other=float("-inf")) - overall_max = tl.max(segm_max) - - # load and rescale segment exp sums - segm_expsum = tl.load(segm_expsum_ptr + segm_offset, mask=segm_mask, other=0.0) - segm_expsum = segm_expsum * tl.math.exp2(segm_max - overall_max) - overall_expsum = tl.sum(segm_expsum) - - # load, rescale, and add segment attention outputs - segm_output_offset = ( - query_token_idx.to(tl.int64) - * (num_query_heads * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) - + query_head_idx * (NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) - + tl.arange(0, NUM_SEGMENTS_PER_SEQ)[:, None] * HEAD_SIZE_PADDED - + tl.arange(0, HEAD_SIZE_PADDED)[None, :] - ) - segm_output = tl.load( - segm_output_ptr + segm_output_offset, - mask=segm_mask[:, None] & dim_mask[None, :], - other=0.0, - ) - segm_output *= tl.math.exp2(segm_max - overall_max)[:, None] - acc_sum = tl.sum(segm_output, axis=0) - # safely divide by overall_expsum, returning 0.0 if overall_expsum is 0 - acc = tl.where(overall_expsum == 0.0, 0.0, acc_sum / overall_expsum) - - if USE_FP8: - acc = acc * out_scale_inv - acc = tl.clamp(acc, FP8_MIN, FP8_MAX) - - # write result - output_offset = ( - query_token_idx * output_stride_0 - + query_head_idx * output_stride_1 - + tl.arange(0, HEAD_SIZE_PADDED) - ) - tl.store(output_ptr + output_offset, acc, mask=dim_mask) - - -@triton.jit -def kernel_unified_attention_2d_ptr( - query_ptr, - key_cache_ptr, - value_cache_ptr, - sink_ptr, - block_tables_ptr, - seq_lens_ptr, - alibi_slopes_ptr, - qq_bias_ptr, - scale_ptr, - k_scale_ptr, - v_scale_ptr, - out_scale_ptr, - softcap_ptr, - num_query_heads: tl.constexpr, - num_queries_per_kv: tl.constexpr, - block_table_stride_ptr, - query_stride_0_ptr, - query_stride_1_ptr, - output_stride_0_ptr, - output_stride_1_ptr, - qq_bias_stride_0_ptr, - BLOCK_SIZE: tl.constexpr, - TILE_SIZE: tl.constexpr, - HEAD_SIZE: tl.constexpr, - HEAD_SIZE_PADDED: tl.constexpr, - USE_ALIBI_SLOPES: tl.constexpr, - USE_QQ_BIAS: tl.constexpr, - USE_SOFTCAP: tl.constexpr, - USE_SINKS: tl.constexpr, - SLIDING_WINDOW: tl.constexpr, - stride_k_cache_0_ptr, - stride_k_cache_1_ptr, - stride_k_cache_2_ptr, - stride_k_cache_3: tl.constexpr, - stride_v_cache_0_ptr, - stride_v_cache_1_ptr, - stride_v_cache_2_ptr, - stride_v_cache_3: tl.constexpr, - query_start_len_ptr, - BLOCK_Q: tl.constexpr, - num_seqs_ptr, - BLOCK_M: tl.constexpr, - USE_FP8: tl.constexpr, - FP8_MIN: tl.constexpr = FP8_E4M3_MIN, - FP8_MAX: tl.constexpr = FP8_E4M3_MAX, - ALL_DECODE: tl.constexpr = False, - output_ptr=None, -): - scale = tl.load(scale_ptr) - k_scale = tl.load(k_scale_ptr) - v_scale = tl.load(v_scale_ptr) - out_scale = tl.load(out_scale_ptr) - softcap = tl.load(softcap_ptr) - block_table_stride = tl.load(block_table_stride_ptr) - query_stride_0 = tl.load(query_stride_0_ptr) - query_stride_1 = tl.load(query_stride_1_ptr) - output_stride_0 = tl.load(output_stride_0_ptr) - output_stride_1 = tl.load(output_stride_1_ptr) - qq_bias_stride_0 = tl.load(qq_bias_stride_0_ptr) - stride_k_cache_0 = tl.load(stride_k_cache_0_ptr) - stride_k_cache_1 = tl.load(stride_k_cache_1_ptr) - stride_k_cache_2 = tl.load(stride_k_cache_2_ptr) - stride_v_cache_0 = tl.load(stride_v_cache_0_ptr) - stride_v_cache_1 = tl.load(stride_v_cache_1_ptr) - stride_v_cache_2 = tl.load(stride_v_cache_2_ptr) - num_seqs = tl.load(num_seqs_ptr) - - kernel_unified_attention_2d( - output_ptr, - query_ptr, - key_cache_ptr, - value_cache_ptr, - sink_ptr, - block_tables_ptr, - seq_lens_ptr, - alibi_slopes_ptr, - qq_bias_ptr, - scale, - k_scale, - v_scale, - out_scale, - softcap, - num_query_heads=num_query_heads, - num_queries_per_kv=num_queries_per_kv, - block_table_stride=block_table_stride, - query_stride_0=query_stride_0, - query_stride_1=query_stride_1, - output_stride_0=output_stride_0, - output_stride_1=output_stride_1, - qq_bias_stride_0=qq_bias_stride_0, - BLOCK_SIZE=BLOCK_SIZE, - TILE_SIZE=TILE_SIZE, - HEAD_SIZE=HEAD_SIZE, - HEAD_SIZE_PADDED=HEAD_SIZE_PADDED, - USE_ALIBI_SLOPES=USE_ALIBI_SLOPES, - USE_QQ_BIAS=USE_QQ_BIAS, - USE_SOFTCAP=USE_SOFTCAP, - USE_SINKS=USE_SINKS, - SLIDING_WINDOW=SLIDING_WINDOW, - stride_k_cache_0=stride_k_cache_0, - stride_k_cache_1=stride_k_cache_1, - stride_k_cache_2=stride_k_cache_2, - stride_k_cache_3=stride_k_cache_3, - stride_v_cache_0=stride_v_cache_0, - stride_v_cache_1=stride_v_cache_1, - stride_v_cache_2=stride_v_cache_2, - stride_v_cache_3=stride_v_cache_3, - query_start_len_ptr=query_start_len_ptr, - BLOCK_Q=BLOCK_Q, - num_seqs=num_seqs, - BLOCK_M=BLOCK_M, - USE_FP8=USE_FP8, - FP8_MIN=FP8_MIN, - FP8_MAX=FP8_MAX, - ALL_DECODE=ALL_DECODE, - ) - - -@triton.jit -def kernel_unified_attention_3d_ptr( - query_ptr, - key_cache_ptr, - value_cache_ptr, - sink_ptr, - block_tables_ptr, - seq_lens_ptr, - alibi_slopes_ptr, - qq_bias_ptr, - scale_ptr, - k_scale_ptr, - v_scale_ptr, - softcap_ptr, - num_query_heads: tl.constexpr, - num_queries_per_kv: tl.constexpr, - block_table_stride_ptr, - query_stride_0_ptr, - query_stride_1_ptr, - qq_bias_stride_0_ptr, - BLOCK_SIZE: tl.constexpr, - TILE_SIZE: tl.constexpr, - HEAD_SIZE: tl.constexpr, - HEAD_SIZE_PADDED: tl.constexpr, - USE_ALIBI_SLOPES: tl.constexpr, - USE_QQ_BIAS: tl.constexpr, - USE_SOFTCAP: tl.constexpr, - USE_SINKS: tl.constexpr, - SLIDING_WINDOW: tl.constexpr, - stride_k_cache_0_ptr, - stride_k_cache_1_ptr, - stride_k_cache_2_ptr, - stride_k_cache_3: tl.constexpr, - stride_v_cache_0_ptr, - stride_v_cache_1_ptr, - stride_v_cache_2_ptr, - stride_v_cache_3: tl.constexpr, - query_start_len_ptr, - BLOCK_Q: tl.constexpr, - num_seqs_ptr, - BLOCK_M: tl.constexpr, - NUM_SEGMENTS_PER_SEQ: tl.constexpr, - ALL_DECODE: tl.constexpr = False, - segm_output_ptr=None, - segm_max_ptr=None, - segm_expsum_ptr=None, -): - scale = tl.load(scale_ptr) - k_scale = tl.load(k_scale_ptr) - v_scale = tl.load(v_scale_ptr) - softcap = tl.load(softcap_ptr) - block_table_stride = tl.load(block_table_stride_ptr) - query_stride_0 = tl.load(query_stride_0_ptr) - query_stride_1 = tl.load(query_stride_1_ptr) - qq_bias_stride_0 = tl.load(qq_bias_stride_0_ptr) - stride_k_cache_0 = tl.load(stride_k_cache_0_ptr) - stride_k_cache_1 = tl.load(stride_k_cache_1_ptr) - stride_k_cache_2 = tl.load(stride_k_cache_2_ptr) - stride_v_cache_0 = tl.load(stride_v_cache_0_ptr) - stride_v_cache_1 = tl.load(stride_v_cache_1_ptr) - stride_v_cache_2 = tl.load(stride_v_cache_2_ptr) - num_seqs = tl.load(num_seqs_ptr) - - kernel_unified_attention_3d( - segm_output_ptr, - segm_max_ptr, - segm_expsum_ptr, - query_ptr, - key_cache_ptr, - value_cache_ptr, - sink_ptr, - block_tables_ptr, - seq_lens_ptr, - alibi_slopes_ptr, - qq_bias_ptr, - scale, - k_scale, - v_scale, - softcap, - num_query_heads=num_query_heads, - num_queries_per_kv=num_queries_per_kv, - block_table_stride=block_table_stride, - query_stride_0=query_stride_0, - query_stride_1=query_stride_1, - qq_bias_stride_0=qq_bias_stride_0, - BLOCK_SIZE=BLOCK_SIZE, - TILE_SIZE=TILE_SIZE, - HEAD_SIZE=HEAD_SIZE, - HEAD_SIZE_PADDED=HEAD_SIZE_PADDED, - USE_ALIBI_SLOPES=USE_ALIBI_SLOPES, - USE_QQ_BIAS=USE_QQ_BIAS, - USE_SOFTCAP=USE_SOFTCAP, - USE_SINKS=USE_SINKS, - SLIDING_WINDOW=SLIDING_WINDOW, - stride_k_cache_0=stride_k_cache_0, - stride_k_cache_1=stride_k_cache_1, - stride_k_cache_2=stride_k_cache_2, - stride_k_cache_3=stride_k_cache_3, - stride_v_cache_0=stride_v_cache_0, - stride_v_cache_1=stride_v_cache_1, - stride_v_cache_2=stride_v_cache_2, - stride_v_cache_3=stride_v_cache_3, - query_start_len_ptr=query_start_len_ptr, - BLOCK_Q=BLOCK_Q, - num_seqs=num_seqs, - BLOCK_M=BLOCK_M, - NUM_SEGMENTS_PER_SEQ=NUM_SEGMENTS_PER_SEQ, - ALL_DECODE=ALL_DECODE, - ) - - -@triton.jit -def reduce_segments_ptr( - segm_output_ptr, - segm_max_ptr, - segm_expsum_ptr, - seq_lens_ptr, - num_seqs_ptr, - num_query_heads: tl.constexpr, - out_scale_inv_ptr, - output_stride_0_ptr, - output_stride_1_ptr, - block_table_stride_ptr, - TILE_SIZE: tl.constexpr, - HEAD_SIZE: tl.constexpr, - HEAD_SIZE_PADDED: tl.constexpr, - query_start_len_ptr, - BLOCK_Q: tl.constexpr, - NUM_SEGMENTS_PER_SEQ: tl.constexpr, - USE_FP8: tl.constexpr, - FP8_MIN: tl.constexpr = FP8_E4M3_MIN, - FP8_MAX: tl.constexpr = FP8_E4M3_MAX, - output_ptr=None, -): - num_seqs = tl.load(num_seqs_ptr) - out_scale_inv = tl.load(out_scale_inv_ptr) - output_stride_0 = tl.load(output_stride_0_ptr) - output_stride_1 = tl.load(output_stride_1_ptr) - block_table_stride = tl.load(block_table_stride_ptr) - - reduce_segments( - output_ptr, - segm_output_ptr, - segm_max_ptr, - segm_expsum_ptr, - seq_lens_ptr, - num_seqs, - num_query_heads=num_query_heads, - out_scale_inv=out_scale_inv, - output_stride_0=output_stride_0, - output_stride_1=output_stride_1, - block_table_stride=block_table_stride, - TILE_SIZE=TILE_SIZE, - HEAD_SIZE=HEAD_SIZE, - HEAD_SIZE_PADDED=HEAD_SIZE_PADDED, - query_start_len_ptr=query_start_len_ptr, - BLOCK_Q=BLOCK_Q, - NUM_SEGMENTS_PER_SEQ=NUM_SEGMENTS_PER_SEQ, - USE_FP8=USE_FP8, - FP8_MIN=FP8_MIN, - FP8_MAX=FP8_MAX, - ) diff --git a/zml/attention/triton.zig b/zml/attention/triton_attention.zig similarity index 53% rename from zml/attention/triton.zig rename to zml/attention/triton_attention.zig index cb8fb65d..f361944d 100644 --- a/zml/attention/triton.zig +++ b/zml/attention/triton_attention.zig @@ -1,14 +1,17 @@ const std = @import("std"); -const bazel = @import("bazel"); -const bazel_builtin = @import("bazel_builtin"); const stdx = @import("stdx"); const zml = @import("../zml.zig"); const AttentionOptions = @import("paged_attention.zig").AttentionOptions; +const kernels = @import("triton_kernels/unified_attention.zig"); +const tri = zml.kernel.triton; + const log = std.log.scoped(.@"zml/attention/triton"); +const toDType = tri.from; + fn use2dKernel(head_size: usize, sliding_window: usize, all_decode: bool, max_seqlen_q: usize, max_seqlen_k: usize, target_num_prgms: usize, num_2d_prgms: usize) bool { _ = head_size; _ = all_decode; @@ -113,125 +116,6 @@ fn select3dConfig(options: paged.PagedAttentionOptions) Config3D { }; } -pub const KernelKind = enum { - kernel_unified_attention_2d_ptr, - kernel_unified_attention_3d_ptr, - reduce_segments_ptr, -}; - -pub const GenerationConfig2D = struct { - pub const Dimensions = struct { - num_tokens: usize, - num_heads: usize, - num_kv_heads: usize, - head_dim: usize, - num_blocks: usize, - block_size: usize, - batch_size: usize, - num_blocks_per_seq: usize, - num_qq_tokens: ?usize = null, - max_seqlen_q: usize, - }; - - pub const FeatureFlags = struct { - use_alibi_slopes: bool, - use_softcap: bool, - use_sinks: bool, - sliding_window: usize, - use_fp8: bool, - all_decode: bool, - }; - - dimensions: Dimensions, - config: Config2D, - feature_flags: FeatureFlags, -}; - -pub const GenerationConfig3D = struct { - pub const Dimensions = struct { - num_tokens: usize, - num_heads: usize, - num_kv_heads: usize, - head_dim: usize, - num_blocks: usize, - block_size: usize, - batch_size: usize, - num_blocks_per_seq: usize, - num_qq_tokens: ?usize = null, - cu_count: usize, - }; - - pub const FeatureFlags = struct { - use_alibi_slopes: bool, - use_softcap: bool, - use_sinks: bool, - sliding_window: usize, - all_decode: bool, - }; - - dimensions: Dimensions, - config: Config3D.AttentionConfig, - feature_flags: FeatureFlags, -}; - -pub const GenerationConfigReduce = struct { - pub const Dimensions = struct { - num_tokens: usize, - num_heads: usize, - num_kv_heads: usize, - head_dim: usize, - block_size: usize, - batch_size: usize, - num_blocks_per_seq: usize, - cu_count: usize, - }; - - pub const FeatureFlags = struct { - use_fp8: bool, - }; - - dimensions: Dimensions, - config: Config3D.ReduceConfig, - feature_flags: FeatureFlags, -}; - -pub const GenerationConfig = union(KernelKind) { - kernel_unified_attention_2d_ptr: GenerationConfig2D, - kernel_unified_attention_3d_ptr: GenerationConfig3D, - reduce_segments_ptr: GenerationConfigReduce, -}; - -fn generateTtir(allocator: std.mem.Allocator, io: std.Io, config: GenerationConfig) ![:0]const u8 { - var arena: std.heap.ArenaAllocator = .init(allocator); - defer arena.deinit(); - const writer_buffer = try arena.allocator().alloc(u8, 4096); - const reader_buffer = try arena.allocator().alloc(u8, 4096); - - const compilation_context = zml.module.CompilationContext.current(); - const platform = compilation_context.platform; - const triton_runtime = platform.triton_runtime.?; - - var allocating: std.Io.Writer.Allocating = .init(arena.allocator()); - { - try triton_runtime.process_mutex.lock(compilation_context.io); - defer triton_runtime.process_mutex.unlock(compilation_context.io); - - var writer = triton_runtime.process.stdin.?.writer(io, writer_buffer); - try writer.interface.print("{f}\n", .{std.json.fmt(config, .{ .emit_null_optional_fields = false })}); - try writer.interface.flush(); - - var reader = triton_runtime.process.stdout.?.reader(io, reader_buffer); - _ = try reader.interface.streamDelimiter(&allocating.writer, '\n'); - _ = try reader.interface.discardShort(1); - } - - const response: std.json.Value = try std.json.parseFromSliceLeaky(std.json.Value, arena.allocator(), allocating.written(), .{}); - return if (response.object.get("ok").?.bool) - try allocator.dupeZ(u8, response.object.get("result").?.string) - else - error.GenerationFailed; -} - fn getCuCount() usize { const platform = zml.module.CompilationContext.current().platform; @@ -408,37 +292,28 @@ pub const paged = struct { _ = context; _ = opts; const config = select2dConfig(paged_attention_opts); - const generation_config: GenerationConfig = .{ - .kernel_unified_attention_2d_ptr = .{ - .dimensions = .{ - .batch_size = paged_attention_opts.batch_size, - .block_size = paged_attention_opts.block_size, - .head_dim = paged_attention_opts.head_dim, - .num_blocks = paged_attention_opts.num_blocks, - .num_blocks_per_seq = paged_attention_opts.max_num_block_per_seq, - .num_heads = paged_attention_opts.num_heads, - .num_kv_heads = paged_attention_opts.num_kv_heads, - .num_tokens = paged_attention_opts.num_tokens, - .max_seqlen_q = parameters.options_.max_seqlen_q, - }, - .config = config, - .feature_flags = .{ - .all_decode = paged_attention_opts.all_decode, - .sliding_window = paged_attention_opts.sliding_window, - .use_alibi_slopes = false, - .use_fp8 = false, - .use_sinks = false, - .use_softcap = false, - }, - }, + + const kernel_config: kernels.KernelUnifiedAttention2dPtr.Config = .{ + .q_dtype = toDType(q.dtype()), + .kv_dtype = toDType(k_cache.dtype()), + .o_dtype = toDType(q.dtype()), + .num_query_heads = @intCast(paged_attention_opts.num_heads), + .num_queries_per_kv = @intCast(paged_attention_opts.numQueriesPerKv()), + .block_size = @intCast(paged_attention_opts.block_size), + .tile_size = @intCast(config.tile_size), + .head_size = @intCast(paged_attention_opts.head_dim), + .head_size_padded = @intCast(std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim)), + .use_alibi_slopes = false, + .use_qq_bias = false, + .use_softcap = false, + .use_sinks = false, + .sliding_window = @intCast(paged_attention_opts.sliding_window), + .block_q = @intCast(config.block_q), + .block_m = @intCast(config.block_m), + .use_fp8 = false, + .all_decode = paged_attention_opts.all_decode, }; - - log.debug("pagedAttention2d config: {any}", .{generation_config}); - - const io = zml.module.CompilationContext.current().io; - const allocator = zml.module.CompilationContext.current().allocator; - const ttir = generateTtir(allocator, io, generation_config) catch unreachable; - defer allocator.free(ttir); + log.debug("pagedAttention2d config: {any}", .{kernel_config}); const dummy = zml.Tensor.constant(zml.DataType.i8.zero()); const block_table_strides = parameters.block_table.shape().computeElementStrides().constSlice(); @@ -459,43 +334,45 @@ pub const paged = struct { const scale: f32 = paged_attention_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))); const scale_ptr = zml.Tensor.constant(zml.DataType.f32.constant(scale)); - const output = zml.ops.triton(.{ - q, - k_cache, - v_cache, - dummy, // sink_ptr - parameters.block_table, - parameters.seq_lens, - dummy, // alibi_slopes_ptr - dummy, // qq_bias_ptr - scale_ptr, - dummy, // k_scale_ptr - dummy, // v_scale_ptr - dummy, // out_scale_ptr - dummy, // softcap_ptr - block_table_strides_ptr, - q_strides_0_ptr, - q_strides_1_ptr, - q_strides_0_ptr, - q_strides_1_ptr, - dummy, // qq_bias_stride_0_ptr - k_strides_0_ptr, - k_strides_1_ptr, - k_strides_2_ptr, - v_strides_0_ptr, - v_strides_1_ptr, - v_strides_2_ptr, - parameters.query_start_len, - num_seqs_ptr, - }, .{q.shape()}, .{ - .debug = false, - .name = "kernel_unified_attention_2d_ptr", - .ir = ttir, - .grid = .{ @intCast(paged_attention_opts.num_kv_heads), @intCast(config.total_q_blocks), 1 }, - .num_stages = @intCast(config.num_stages), - .num_warps = @intCast(config.num_warps), - }); - return output[0]; + const output = kernels.KernelUnifiedAttention2dPtr.Kernel.call( + .{ + .query_ptr = q, + .key_cache_ptr = k_cache, + .value_cache_ptr = v_cache, + .sink_ptr = dummy, + .block_tables_ptr = parameters.block_table, + .seq_lens_ptr = parameters.seq_lens, + .alibi_slopes_ptr = dummy, + .qq_bias_ptr = dummy, + .scale_ptr = scale_ptr, + .k_scale_ptr = dummy, + .v_scale_ptr = dummy, + .out_scale_ptr = dummy, + .softcap_ptr = dummy, + .block_table_stride_ptr = block_table_strides_ptr, + .query_stride_0_ptr = q_strides_0_ptr, + .query_stride_1_ptr = q_strides_1_ptr, + .output_stride_0_ptr = q_strides_0_ptr, + .output_stride_1_ptr = q_strides_1_ptr, + .qq_bias_stride_0_ptr = dummy, + .stride_k_cache_0_ptr = k_strides_0_ptr, + .stride_k_cache_1_ptr = k_strides_1_ptr, + .stride_k_cache_2_ptr = k_strides_2_ptr, + .stride_v_cache_0_ptr = v_strides_0_ptr, + .stride_v_cache_1_ptr = v_strides_1_ptr, + .stride_v_cache_2_ptr = v_strides_2_ptr, + .query_start_len_ptr = parameters.query_start_len, + .num_seqs_ptr = num_seqs_ptr, + }, + .{ .output = q.shape() }, + .{ + .cfg = kernel_config, + .grid = .{ @intCast(paged_attention_opts.num_kv_heads), @intCast(config.total_q_blocks), 1 }, + .num_stages = @intCast(config.num_stages), + .num_warps = @intCast(config.num_warps), + }, + ); + return output.output; } pub fn pagedAttention3d(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, paged_attention_opts: PagedAttentionOptions) zml.Tensor { @@ -504,58 +381,39 @@ pub const paged = struct { const config = select3dConfig(paged_attention_opts); - const attn_generation_config: GenerationConfig = .{ - .kernel_unified_attention_3d_ptr = .{ - .dimensions = .{ - .batch_size = paged_attention_opts.batch_size, - .block_size = paged_attention_opts.block_size, - .head_dim = paged_attention_opts.head_dim, - .num_blocks = paged_attention_opts.num_blocks, - .num_blocks_per_seq = paged_attention_opts.max_num_block_per_seq, - .num_heads = paged_attention_opts.num_heads, - .num_kv_heads = paged_attention_opts.num_kv_heads, - .num_tokens = paged_attention_opts.num_tokens, - .cu_count = paged_attention_opts.cu_count, - }, - .config = config.attention, - .feature_flags = .{ - .all_decode = paged_attention_opts.all_decode, - .sliding_window = paged_attention_opts.sliding_window, - .use_alibi_slopes = false, - .use_sinks = false, - .use_softcap = false, - }, - }, + const head_size_padded: i64 = @intCast(std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim)); + const attn_kernel_config: kernels.KernelUnifiedAttention3dPtr.Config = .{ + .q_dtype = toDType(q.dtype()), + .kv_dtype = toDType(k_cache.dtype()), + .num_query_heads = @intCast(paged_attention_opts.num_heads), + .num_queries_per_kv = @intCast(paged_attention_opts.numQueriesPerKv()), + .block_size = @intCast(paged_attention_opts.block_size), + .tile_size = @intCast(config.attention.tile_size), + .head_size = @intCast(paged_attention_opts.head_dim), + .head_size_padded = head_size_padded, + .use_alibi_slopes = false, + .use_qq_bias = false, + .use_softcap = false, + .use_sinks = false, + .sliding_window = @intCast(paged_attention_opts.sliding_window), + .block_q = @intCast(config.attention.block_q), + .block_m = @intCast(config.attention.block_m), + .num_segments_per_seq = @intCast(config.attention.num_segments_per_seq), + .all_decode = paged_attention_opts.all_decode, }; + log.debug("pagedAttention3d attention config: {any}", .{attn_kernel_config}); - log.debug("pagedAttention3d attention config: {any}", .{attn_generation_config}); - - const io = zml.module.CompilationContext.current().io; - const allocator = zml.module.CompilationContext.current().allocator; - const attn_ttir = generateTtir(allocator, io, attn_generation_config) catch unreachable; - defer allocator.free(attn_ttir); - - const reduce_generation_config: GenerationConfig = .{ - .reduce_segments_ptr = .{ - .dimensions = .{ - .num_tokens = paged_attention_opts.num_tokens, - .num_heads = paged_attention_opts.num_heads, - .num_kv_heads = paged_attention_opts.num_kv_heads, - .head_dim = paged_attention_opts.head_dim, - .block_size = paged_attention_opts.block_size, - .batch_size = paged_attention_opts.batch_size, - .num_blocks_per_seq = paged_attention_opts.max_num_block_per_seq, - .cu_count = paged_attention_opts.cu_count, - }, - .config = config.reduce, - .feature_flags = .{ - .use_fp8 = false, - }, - }, + const reduce_kernel_config: kernels.ReduceSegmentsPtr.Config = .{ + .o_dtype = toDType(q.dtype()), + .num_query_heads = @intCast(paged_attention_opts.num_heads), + .tile_size = @intCast(config.reduce.tile_size), + .head_size = @intCast(paged_attention_opts.head_dim), + .head_size_padded = head_size_padded, + .block_q = @intCast(config.reduce.block_q), + .num_segments_per_seq = @intCast(config.reduce.num_segments_per_seq), + .use_fp8 = false, }; - log.debug("pagedAttention3d reduce config: {any}", .{reduce_generation_config}); - const reduce_ttir = generateTtir(std.heap.c_allocator, io, reduce_generation_config) catch unreachable; - defer std.heap.c_allocator.free(reduce_ttir); + log.debug("pagedAttention3d reduce config: {any}", .{reduce_kernel_config}); const dummy = zml.Tensor.constant(zml.DataType.i8.zero()); const block_table_strides = parameters.block_table.shape().computeElementStrides().constSlice(); @@ -577,105 +435,68 @@ pub const paged = struct { const scale_ptr = zml.Tensor.constant(zml.DataType.f32.constant(scale)); const attn_grid: [3]i32 = .{ @intCast(config.attention.total_q_blocks), @intCast(paged_attention_opts.num_kv_heads), @intCast(config.attention.num_segments_per_seq) }; - const attn_output = zml.ops.triton(.{ - q, - k_cache, - v_cache, - dummy, // sink_ptr - parameters.block_table, - parameters.seq_lens, - dummy, // alibi_slopes_ptr - dummy, // qq_bias_ptr - scale_ptr, - dummy, // k_scale_ptr - dummy, // v_scale_ptr - dummy, // softcap_ptr - block_table_strides_ptr, - q_strides_0_ptr, - q_strides_1_ptr, - dummy, // qq_bias_stride_0_ptr - k_strides_0_ptr, - k_strides_1_ptr, - k_strides_2_ptr, - v_strides_0_ptr, - v_strides_1_ptr, - v_strides_2_ptr, - parameters.query_start_len, - num_seqs_ptr, - }, .{ - zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq, std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim) }, .f32), - zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq }, .f32), - zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq }, .f32), - }, .{ - .debug = false, - .name = "kernel_unified_attention_3d_ptr", - .ir = attn_ttir, - .grid = attn_grid, - .num_stages = @intCast(config.attention.num_stages), - .num_warps = @intCast(config.attention.num_warps), - }); + const attn_output = kernels.KernelUnifiedAttention3dPtr.Kernel.call( + .{ + .query_ptr = q, + .key_cache_ptr = k_cache, + .value_cache_ptr = v_cache, + .sink_ptr = dummy, + .block_tables_ptr = parameters.block_table, + .seq_lens_ptr = parameters.seq_lens, + .alibi_slopes_ptr = dummy, + .qq_bias_ptr = dummy, + .scale_ptr = scale_ptr, + .k_scale_ptr = dummy, + .v_scale_ptr = dummy, + .softcap_ptr = dummy, + .block_table_stride_ptr = block_table_strides_ptr, + .query_stride_0_ptr = q_strides_0_ptr, + .query_stride_1_ptr = q_strides_1_ptr, + .qq_bias_stride_0_ptr = dummy, + .stride_k_cache_0_ptr = k_strides_0_ptr, + .stride_k_cache_1_ptr = k_strides_1_ptr, + .stride_k_cache_2_ptr = k_strides_2_ptr, + .stride_v_cache_0_ptr = v_strides_0_ptr, + .stride_v_cache_1_ptr = v_strides_1_ptr, + .stride_v_cache_2_ptr = v_strides_2_ptr, + .query_start_len_ptr = parameters.query_start_len, + .num_seqs_ptr = num_seqs_ptr, + }, + .{ + .segm_output = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq, std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim) }, .f32), + .segm_max = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq }, .f32), + .segm_expsum = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq }, .f32), + }, + .{ + .cfg = attn_kernel_config, + .grid = attn_grid, + .num_stages = @intCast(config.attention.num_stages), + .num_warps = @intCast(config.attention.num_warps), + }, + ); - const output = zml.ops.triton(.{ - attn_output[0], - attn_output[1], - attn_output[2], - parameters.seq_lens, - num_seqs_ptr, - dummy, // out_scale_inv_ptr - q_strides_0_ptr, - q_strides_1_ptr, - block_table_strides_ptr, - parameters.query_start_len, - }, .{ - q.shape(), - }, .{ - .debug = false, - .name = "reduce_segments_ptr", - .ir = reduce_ttir, - .grid = .{ @intCast(paged_attention_opts.num_tokens), @intCast(paged_attention_opts.num_heads), 1 }, - .num_stages = @intCast(config.reduce.num_stages), - .num_warps = @intCast(config.reduce.num_warps), - }); + const output = kernels.ReduceSegmentsPtr.Kernel.call( + .{ + .segm_output_ptr = attn_output.segm_output, + .segm_max_ptr = attn_output.segm_max, + .segm_expsum_ptr = attn_output.segm_expsum, + .seq_lens_ptr = parameters.seq_lens, + .num_seqs_ptr = num_seqs_ptr, + .out_scale_inv_ptr = dummy, + .output_stride_0_ptr = q_strides_0_ptr, + .output_stride_1_ptr = q_strides_1_ptr, + .block_table_stride_ptr = block_table_strides_ptr, + .query_start_len_ptr = parameters.query_start_len, + }, + .{ .output = q.shape() }, + .{ + .cfg = reduce_kernel_config, + .grid = .{ @intCast(paged_attention_opts.num_tokens), @intCast(paged_attention_opts.num_heads), 1 }, + .num_stages = @intCast(config.reduce.num_stages), + .num_warps = @intCast(config.reduce.num_warps), + }, + ); - return output[0]; - } -}; - -pub const Runtime = struct { - process: std.process.Child, - process_mutex: *std.Io.Mutex, - - pub fn init(allocator: std.mem.Allocator, io: std.Io) !Runtime { - var arena: std.heap.ArenaAllocator = .init(allocator); - defer arena.deinit(); - - const runfiles = bazel.runfiles(bazel_builtin.current_repository) catch unreachable; - const sandbox_path = runfiles.rlocationAlloc(arena.allocator(), "zml/zml/attention/triton/sandbox") catch unreachable; - - var map: std.process.Environ.Map = .init(arena.allocator()); - try map.put("LD_LIBRARY_PATH", std.Io.Dir.path.join(arena.allocator(), &.{ sandbox_path.?, "lib" }) catch unreachable); - - const process_mutex = try allocator.create(std.Io.Mutex); - errdefer allocator.destroy(process_mutex); - process_mutex.* = .init; - - const process = try std.process.spawn(io, .{ - .argv = &.{std.Io.Dir.path.join(arena.allocator(), &.{ sandbox_path.?, "bin", "generate" }) catch unreachable}, - .stdin = .pipe, - .stdout = .pipe, - .environ_map = &map, - }); - - return .{ - .process = process, - .process_mutex = process_mutex, - }; - } - - pub fn deinit(self: *Runtime, allocator: std.mem.Allocator, io: std.Io) void { - allocator.destroy(self.process_mutex); - // NOTE(Corendos): This is not portable, but I couldn't find a better way with the current std.Io. - _ = std.c.kill(self.process.id.?, .INT); - _ = self.process.wait(io) catch unreachable; + return output.output; } }; diff --git a/zml/attention/triton_kernels/unified_attention.zig b/zml/attention/triton_kernels/unified_attention.zig new file mode 100644 index 00000000..38c4f961 --- /dev/null +++ b/zml/attention/triton_kernels/unified_attention.zig @@ -0,0 +1,1136 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const dialects = @import("mlir/dialects"); +const ttir = dialects.ttir; + +const zml = @import("../../zml.zig"); +const tri = zml.kernel.triton; +const Builder = tri.Builder; +const Value = tri.Value; +const DType = tri.DType; + +const log = std.log.scoped(.@"zml/attention/triton"); + +pub const FP8_E4M3_MIN: f32 = -448.0; +pub const FP8_E4M3_MAX: f32 = 448.0; + +/// `log_2(e)` — pre-multiply with this to turn `exp(x)` into `exp2(x * RCP_LN2)`. +const RCP_LN2: f32 = 1.4426950408889634; + +pub const KernelUnifiedAttention2dPtr = struct { + pub const Config = struct { + q_dtype: DType, + kv_dtype: DType, + o_dtype: DType, + scale_dtype: DType = .f32, + + num_query_heads: i64, + num_queries_per_kv: i64, + + block_size: i64, + tile_size: i64, + head_size: i64, + head_size_padded: i64, + + use_alibi_slopes: bool, + use_qq_bias: bool, + use_softcap: bool, + use_sinks: bool, + sliding_window: i64, + + stride_k_cache_3: i64 = 1, + stride_v_cache_3: i64 = 1, + + block_q: i64, + block_m: i64, + use_fp8: bool, + fp8_min: f32 = FP8_E4M3_MIN, + fp8_max: f32 = FP8_E4M3_MAX, + all_decode: bool = false, + }; + + pub const Kernel = tri.Kernel(Config, .{ + .name = "kernel_unified_attention_2d_ptr", + .inputs = &.{ + "query_ptr", "key_cache_ptr", "value_cache_ptr", "sink_ptr", + "block_tables_ptr", "seq_lens_ptr", "alibi_slopes_ptr", "qq_bias_ptr", + "scale_ptr", "k_scale_ptr", "v_scale_ptr", "out_scale_ptr", + "softcap_ptr", "block_table_stride_ptr", "query_stride_0_ptr", "query_stride_1_ptr", + "output_stride_0_ptr", "output_stride_1_ptr", "qq_bias_stride_0_ptr", "stride_k_cache_0_ptr", + "stride_k_cache_1_ptr", "stride_k_cache_2_ptr", "stride_v_cache_0_ptr", "stride_v_cache_1_ptr", + "stride_v_cache_2_ptr", "query_start_len_ptr", "num_seqs_ptr", + }, + .outputs = &.{"output"}, + .run = run, + }); + fn run(b: *Builder, cfg: Config) tri.FinishError!void { + const a = try b.declareArgs(.{ + .query_ptr = .{ .ptr = cfg.q_dtype }, + .key_cache_ptr = .{ .ptr = cfg.kv_dtype }, + .value_cache_ptr = .{ .ptr = cfg.kv_dtype }, + .sink_ptr = .{ .ptr = cfg.scale_dtype }, + .block_tables_ptr = .{ .ptr = .i32 }, + .seq_lens_ptr = .{ .ptr = .i32 }, + .alibi_slopes_ptr = .{ .ptr = cfg.scale_dtype }, + .qq_bias_ptr = .{ .ptr = cfg.scale_dtype }, + .scale_ptr = .{ .ptr = .f32 }, + .k_scale_ptr = .{ .ptr = .f32 }, + .v_scale_ptr = .{ .ptr = .f32 }, + .out_scale_ptr = .{ .ptr = .f32 }, + .softcap_ptr = .{ .ptr = .f32 }, + .block_table_stride_ptr = .{ .ptr = .i64 }, + .query_stride_0_ptr = .{ .ptr = .i64 }, + .query_stride_1_ptr = .{ .ptr = .i64 }, + .output_stride_0_ptr = .{ .ptr = .i64 }, + .output_stride_1_ptr = .{ .ptr = .i64 }, + .qq_bias_stride_0_ptr = .{ .ptr = .i64 }, + .stride_k_cache_0_ptr = .{ .ptr = .i64 }, + .stride_k_cache_1_ptr = .{ .ptr = .i64 }, + .stride_k_cache_2_ptr = .{ .ptr = .i64 }, + .stride_v_cache_0_ptr = .{ .ptr = .i64 }, + .stride_v_cache_1_ptr = .{ .ptr = .i64 }, + .stride_v_cache_2_ptr = .{ .ptr = .i64 }, + .query_start_len_ptr = .{ .ptr = .i32 }, + .num_seqs_ptr = .{ .ptr = .i32 }, + .output_ptr = .{ .ptr = cfg.o_dtype }, + }); + + const scale = b.load(a.scale_ptr); + const k_scale = b.load(a.k_scale_ptr); + const v_scale = b.load(a.v_scale_ptr); + const out_scale = b.load(a.out_scale_ptr); + const softcap = b.load(a.softcap_ptr); + const block_table_stride = b.load(a.block_table_stride_ptr); + const query_stride_0 = b.load(a.query_stride_0_ptr); + const query_stride_1 = b.load(a.query_stride_1_ptr); + const output_stride_0 = b.load(a.output_stride_0_ptr); + const output_stride_1 = b.load(a.output_stride_1_ptr); + const qq_bias_stride_0 = b.load(a.qq_bias_stride_0_ptr); + const stride_k_cache_0 = b.load(a.stride_k_cache_0_ptr); + const stride_k_cache_1 = b.load(a.stride_k_cache_1_ptr); + const stride_k_cache_2 = b.load(a.stride_k_cache_2_ptr); + const stride_v_cache_0 = b.load(a.stride_v_cache_0_ptr); + const stride_v_cache_1 = b.load(a.stride_v_cache_1_ptr); + const stride_v_cache_2 = b.load(a.stride_v_cache_2_ptr); + const num_seqs = b.load(a.num_seqs_ptr); + + kernelUnifiedAttention2d( + b, + a.output_ptr, + a.query_ptr, + a.key_cache_ptr, + a.value_cache_ptr, + a.sink_ptr, + a.block_tables_ptr, + a.seq_lens_ptr, + a.alibi_slopes_ptr, + a.qq_bias_ptr, + scale, + k_scale, + v_scale, + out_scale, + softcap, + block_table_stride, + query_stride_0, + query_stride_1, + output_stride_0, + output_stride_1, + qq_bias_stride_0, + stride_k_cache_0, + stride_k_cache_1, + stride_k_cache_2, + stride_v_cache_0, + stride_v_cache_1, + stride_v_cache_2, + a.query_start_len_ptr, + num_seqs, + cfg, + ); + } +}; + +/// Direct port of Python `kernel_unified_attention_3d_ptr`. +pub const KernelUnifiedAttention3dPtr = struct { + pub const Config = struct { + q_dtype: DType, + kv_dtype: DType, + scale_dtype: DType = .f32, + + num_query_heads: i64, + num_queries_per_kv: i64, + + block_size: i64, + tile_size: i64, + head_size: i64, + head_size_padded: i64, + + use_alibi_slopes: bool, + use_qq_bias: bool, + use_softcap: bool, + use_sinks: bool, + sliding_window: i64, + + stride_k_cache_3: i64 = 1, + stride_v_cache_3: i64 = 1, + + block_q: i64, + block_m: i64, + num_segments_per_seq: i64, + all_decode: bool = false, + }; + + pub const Kernel = tri.Kernel(Config, .{ + .name = "kernel_unified_attention_3d_ptr", + .inputs = &.{ + "query_ptr", "key_cache_ptr", "value_cache_ptr", "sink_ptr", + "block_tables_ptr", "seq_lens_ptr", "alibi_slopes_ptr", "qq_bias_ptr", + "scale_ptr", "k_scale_ptr", "v_scale_ptr", "softcap_ptr", + "block_table_stride_ptr", "query_stride_0_ptr", "query_stride_1_ptr", "qq_bias_stride_0_ptr", + "stride_k_cache_0_ptr", "stride_k_cache_1_ptr", "stride_k_cache_2_ptr", "stride_v_cache_0_ptr", + "stride_v_cache_1_ptr", "stride_v_cache_2_ptr", "query_start_len_ptr", "num_seqs_ptr", + }, + .outputs = &.{ "segm_output", "segm_max", "segm_expsum" }, + .run = run, + }); + fn run(b: *Builder, cfg: Config) tri.FinishError!void { + const a = try b.declareArgs(.{ + .query_ptr = .{ .ptr = cfg.q_dtype }, + .key_cache_ptr = .{ .ptr = cfg.kv_dtype }, + .value_cache_ptr = .{ .ptr = cfg.kv_dtype }, + .sink_ptr = .{ .ptr = cfg.scale_dtype }, + .block_tables_ptr = .{ .ptr = .i32 }, + .seq_lens_ptr = .{ .ptr = .i32 }, + .alibi_slopes_ptr = .{ .ptr = cfg.scale_dtype }, + .qq_bias_ptr = .{ .ptr = cfg.scale_dtype }, + .scale_ptr = .{ .ptr = .f32 }, + .k_scale_ptr = .{ .ptr = .f32 }, + .v_scale_ptr = .{ .ptr = .f32 }, + .softcap_ptr = .{ .ptr = .f32 }, + .block_table_stride_ptr = .{ .ptr = .i64 }, + .query_stride_0_ptr = .{ .ptr = .i64 }, + .query_stride_1_ptr = .{ .ptr = .i64 }, + .qq_bias_stride_0_ptr = .{ .ptr = .i64 }, + .stride_k_cache_0_ptr = .{ .ptr = .i64 }, + .stride_k_cache_1_ptr = .{ .ptr = .i64 }, + .stride_k_cache_2_ptr = .{ .ptr = .i64 }, + .stride_v_cache_0_ptr = .{ .ptr = .i64 }, + .stride_v_cache_1_ptr = .{ .ptr = .i64 }, + .stride_v_cache_2_ptr = .{ .ptr = .i64 }, + .query_start_len_ptr = .{ .ptr = .i32 }, + .num_seqs_ptr = .{ .ptr = .i32 }, + .segm_output_ptr = .{ .ptr = .f32 }, + .segm_max_ptr = .{ .ptr = .f32 }, + .segm_expsum_ptr = .{ .ptr = .f32 }, + }); + + const scale = b.load(a.scale_ptr); + const k_scale = b.load(a.k_scale_ptr); + const v_scale = b.load(a.v_scale_ptr); + const softcap = b.load(a.softcap_ptr); + const block_table_stride = b.load(a.block_table_stride_ptr); + const query_stride_0 = b.load(a.query_stride_0_ptr); + const query_stride_1 = b.load(a.query_stride_1_ptr); + const qq_bias_stride_0 = b.load(a.qq_bias_stride_0_ptr); + const stride_k_cache_0 = b.load(a.stride_k_cache_0_ptr); + const stride_k_cache_1 = b.load(a.stride_k_cache_1_ptr); + const stride_k_cache_2 = b.load(a.stride_k_cache_2_ptr); + const stride_v_cache_0 = b.load(a.stride_v_cache_0_ptr); + const stride_v_cache_1 = b.load(a.stride_v_cache_1_ptr); + const stride_v_cache_2 = b.load(a.stride_v_cache_2_ptr); + const num_seqs = b.load(a.num_seqs_ptr); + + kernelUnifiedAttention3d( + b, + a.segm_output_ptr, + a.segm_max_ptr, + a.segm_expsum_ptr, + a.query_ptr, + a.key_cache_ptr, + a.value_cache_ptr, + a.sink_ptr, + a.block_tables_ptr, + a.seq_lens_ptr, + a.alibi_slopes_ptr, + a.qq_bias_ptr, + scale, + k_scale, + v_scale, + softcap, + block_table_stride, + query_stride_0, + query_stride_1, + qq_bias_stride_0, + stride_k_cache_0, + stride_k_cache_1, + stride_k_cache_2, + stride_v_cache_0, + stride_v_cache_1, + stride_v_cache_2, + a.query_start_len_ptr, + num_seqs, + cfg, + ); + } +}; + +/// Direct port of Python `reduce_segments_ptr`. +pub const ReduceSegmentsPtr = struct { + pub const Config = struct { + o_dtype: DType, + scale_dtype: DType = .f32, + + num_query_heads: i64, + tile_size: i64, + head_size: i64, + head_size_padded: i64, + + block_q: i64, + num_segments_per_seq: i64, + + use_fp8: bool, + fp8_min: f32 = FP8_E4M3_MIN, + fp8_max: f32 = FP8_E4M3_MAX, + }; + + pub const Kernel = tri.Kernel(Config, .{ + .name = "reduce_segments_ptr", + .inputs = &.{ + "segm_output_ptr", "segm_max_ptr", "segm_expsum_ptr", "seq_lens_ptr", + "num_seqs_ptr", "out_scale_inv_ptr", "output_stride_0_ptr", "output_stride_1_ptr", + "block_table_stride_ptr", "query_start_len_ptr", + }, + .outputs = &.{"output"}, + .run = run, + }); + fn run(b: *Builder, cfg: Config) tri.FinishError!void { + const a = try b.declareArgs(.{ + .segm_output_ptr = .{ .ptr = .f32 }, + .segm_max_ptr = .{ .ptr = .f32 }, + .segm_expsum_ptr = .{ .ptr = .f32 }, + .seq_lens_ptr = .{ .ptr = .i32 }, + .num_seqs_ptr = .{ .ptr = .i32 }, + .out_scale_inv_ptr = .{ .ptr = .f32 }, + .output_stride_0_ptr = .{ .ptr = .i64 }, + .output_stride_1_ptr = .{ .ptr = .i64 }, + .block_table_stride_ptr = .{ .ptr = .i64 }, + .query_start_len_ptr = .{ .ptr = .i32 }, + .output_ptr = .{ .ptr = cfg.o_dtype }, + }); + + const num_seqs = b.load(a.num_seqs_ptr); + const out_scale_inv = b.load(a.out_scale_inv_ptr); + const output_stride_0 = b.load(a.output_stride_0_ptr); + const output_stride_1 = b.load(a.output_stride_1_ptr); + const block_table_stride = b.load(a.block_table_stride_ptr); + + reduceSegments( + b, + a.output_ptr, + a.segm_output_ptr, + a.segm_max_ptr, + a.segm_expsum_ptr, + a.seq_lens_ptr, + num_seqs, + out_scale_inv, + output_stride_0, + output_stride_1, + block_table_stride, + a.query_start_len_ptr, + cfg, + ); + } +}; + +// ============================================================================ +// Helpers +// ============================================================================ + +fn ctx() *mlir.Context { + return zml.module.CompilationContext.current().mlir_ctx; +} + +fn isFp8(dt: DType) bool { + return dt == .f8e4m3fn or dt == .f8e5m2; +} + +fn fastExp(k: *Builder, x: Value) Value { + return k.exp2(x.mul(RCP_LN2)); +} + +fn cdivFn(x: Value, y: anytype) Value { + return switch (@typeInfo(@TypeOf(y))) { + .comptime_int, .comptime_float => x.add(y - 1).div(y), + else => x.add(y).sub(1).div(y), + }; +} + +fn applySoftcap(k: *Builder, s_val: Value, x: Value) Value { + const sdiv = s_val.div(x); + const p1 = k.exp2(sdiv); + const p2 = k.exp2(k.negf(sdiv)); + return x.mul(p1.sub(p2)).div(p1.add(p2)); +} + +fn findSeqIdx( + k: *Builder, + query_start_len_ptr: Value, + target_idx: Value, + num_seqs: Value, + block_q: i64, + use_q_block_mode: bool, +) Value { + const left_init = k.liftAs(0, .i32); + + var w = k.openWhile(.{ left_init, num_seqs }, .{ k.scalarTy(.i32), k.scalarTy(.i32) }); + { + const left = w.before_carried[0]; + const right = w.before_carried[1]; + w.yieldBefore(left.lt(right), .{ left, right }); + } + { + const left = w.after_carried[0]; + const right = w.after_carried[1]; + const mid = left.add(right).div(2); + const val = k.load(query_start_len_ptr.addPtr(mid)); + const mid_val = if (use_q_block_mode) + val.div(@as(i32, @intCast(block_q))).add(mid) + else + val; + + var i = k.openIfElse(mid_val.le(target_idx), .{ k.scalarTy(.i32), k.scalarTy(.i32) }); + { + i.yieldThen(.{ mid.add(1), right }); + } + { + i.yieldElse(.{ left, mid }); + } + w.yieldAfter(.{ i.results[0], i.results[1] }); + } + + return w.results[0].sub(1); +} + +fn dequantKv(loaded: Value, scale: Value, kv_is_fp8: bool, q_dtype: DType) Value { + if (!kv_is_fp8 or isFp8(q_dtype)) return loaded; + return loaded.to(.f32).mul(scale).to(q_dtype); +} + +fn kernelUnifiedAttention2d( + k: *Builder, + output_ptr: Value, + query_ptr: Value, + key_cache_ptr: Value, + value_cache_ptr: Value, + sink_ptr: Value, + block_tables_ptr: Value, + seq_lens_ptr: Value, + alibi_slopes_ptr: Value, + qq_bias_ptr: Value, + scale: Value, + k_scale: Value, + v_scale: Value, + out_scale: Value, + softcap: Value, + block_table_stride: Value, + query_stride_0: Value, + query_stride_1: Value, + output_stride_0: Value, + output_stride_1: Value, + qq_bias_stride_0: Value, + stride_k_cache_0: Value, + stride_k_cache_1: Value, + stride_k_cache_2: Value, + stride_v_cache_0: Value, + stride_v_cache_1: Value, + stride_v_cache_2: Value, + query_start_len_ptr: Value, + num_seqs: Value, + config: KernelUnifiedAttention2dPtr.Config, +) void { + const BLOCK_M: i64 = config.block_m; + const BLOCK_Q: i64 = config.block_q; + const BLOCK_SIZE: i64 = config.block_size; + const TILE_SIZE: i64 = config.tile_size; + const HEAD_SIZE: i64 = config.head_size; + const HEAD_SIZE_PADDED: i64 = config.head_size_padded; + const NUM_QUERIES_PER_KV: i64 = config.num_queries_per_kv; + const NUM_QUERY_HEADS: i64 = config.num_query_heads; + const SLIDING_WINDOW: i64 = config.sliding_window; + + const kv_head_idx = k.programId(.x); + const q_block_global_idx = k.programId(.y); + + const qk_scale = scale.mul(RCP_LN2); + + const seq_idx = findSeqIdx(k, query_start_len_ptr, q_block_global_idx, num_seqs, BLOCK_Q, true); + + const q_start_raw = k.load(query_start_len_ptr.addPtr(seq_idx)); + const q_block_start_idx = q_start_raw.div(@as(i32, @intCast(BLOCK_Q))).add(seq_idx); + + const q_block_local_idx = q_block_global_idx.sub(q_block_start_idx); + + const cur_batch_in_all_start_index = q_start_raw; + // Match Python's left-to-right `query_start_len_ptr + seq_idx + 1`: two `tt.addptr`s. + const cur_batch_in_all_stop_index = k.load(query_start_len_ptr.addPtr(seq_idx).addPtr(1)); + const cur_batch_query_len = cur_batch_in_all_stop_index.sub(cur_batch_in_all_start_index); + + // Python: if q_block_local_idx * BLOCK_Q >= cur_batch_query_len: return + const out_of_range = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))).ge(cur_batch_query_len); + k.returnIf(out_of_range, .{}); + + const offs_m = k.arange(0, BLOCK_M, .i32); + const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); + const offs_t = k.arange(0, TILE_SIZE, .i32); + + // query_pos = q_block_local_idx * BLOCK_Q + offs_m // NUM_QUERIES_PER_KV + const q_local_bq = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))); + const query_pos = q_local_bq.add(offs_m.div(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); + + const query_offset_0 = cur_batch_in_all_start_index.add(query_pos); + const query_offset_1 = kv_head_idx.mul(@as(i32, @intCast(NUM_QUERIES_PER_KV))) + .add(offs_m.rem(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); + + const qo0_2d = k.expandDims(query_offset_0, 1).mul(query_stride_0); + const qo1_2d = k.expandDims(query_offset_1, 1).mul(query_stride_1); + const query_offset = qo0_2d.add(qo1_2d).add(k.expandDims(offs_d, 0)); + + const dim_mask: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + offs_d.lt(@as(i32, @intCast(HEAD_SIZE))) + else + k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); + const query_mask_0 = query_pos.lt(cur_batch_query_len); + const query_mask_1 = query_offset_1.lt(@as(i32, @intCast(NUM_QUERY_HEADS))); + + // Q : (BLOCK_M, HEAD_SIZE_PADDED) + const q_mask_ab = k.mask2d(query_mask_0, dim_mask, BLOCK_M, HEAD_SIZE_PADDED); + const q_mask_c = query_mask_1.expandDims(1); + const q_mask = q_mask_ab.bitAnd(q_mask_c); + const q_cache_mod: ttir.CacheModifier = + if (config.all_decode or BLOCK_M >= NUM_QUERY_HEADS) .cg else .none; + const Q = k.loadOpts(query_ptr.addPtr(query_offset), .{ + .mask = q_mask, + .other = k.zeros(&.{ BLOCK_M, HEAD_SIZE_PADDED }, config.q_dtype), + .cache_modifier = q_cache_mod, + }); + + const block_table_offset = seq_idx.to(.i64).mul(block_table_stride); + + // M init — sinks or -inf. + const m_init: Value = if (config.use_sinks) mb: { + const loaded = k.loadOpts(sink_ptr.addPtr(query_offset_1), .{ + .mask = query_mask_1, + .other = k.full(&.{BLOCK_M}, -std.math.inf(f32), config.scale_dtype), + }); + break :mb loaded.to(.f32).mul(RCP_LN2); + } else k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32); + + const l_init = k.full(&.{BLOCK_M}, 1.0, .f32); + const acc_init = k.zeros(&.{ BLOCK_M, HEAD_SIZE_PADDED }, .f32); + + const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); + const context_len = seq_len.sub(cur_batch_query_len); + + const alibi_slope: Value = if (config.use_alibi_slopes) + k.loadOpts(alibi_slopes_ptr.addPtr(query_offset_1), .{ + .mask = query_mask_1, + .other = k.full(&.{BLOCK_M}, 0.0, config.scale_dtype), + }) + else + k.zeros(&.{BLOCK_M}, config.scale_dtype); + + // In the `use_qq_bias = true` branch, the tensor offset auto-splats the + // scalar ptr via addPtr. In the else branch there is no offset, so we + // need an explicit splatTo to give expandDims a tensor to work on. + const qq_bias_row_ptrs: Value = if (config.use_qq_bias) + qq_bias_ptr.addPtr(query_pos.to(.i64).mul(qq_bias_stride_0)) + else + qq_bias_ptr.splatTo(&.{BLOCK_M}); + + // max_seq_prefix_len = context_len + q_block_local_idx*BLOCK_Q + (BLOCK_M-1)/NQ_PER_KV + 1 + const pad_term: i32 = @intCast(@divTrunc(BLOCK_M - 1, NUM_QUERIES_PER_KV) + 1); + const max_prefix_raw = context_len.add(q_local_bq).add(pad_term); + const max_seq_prefix_len = max_prefix_raw.minimum(seq_len); + + const num_tiles = cdivFn(max_seq_prefix_len, @as(i32, @intCast(TILE_SIZE))); + + // Sliding-window tile pruning — comptime branch. + var tile_start: Value = k.liftAs(0, .i32); + var tile_end: Value = num_tiles; + if (SLIDING_WINDOW > 0) { + const qpos_lo = q_local_bq; + const qpos_hi_raw = qpos_lo.add(pad_term - 1); + const qpos_hi = qpos_hi_raw.minimum(cur_batch_query_len.sub(1)); + const first_allowed_key = context_len.add(qpos_lo).sub(@as(i32, @intCast(SLIDING_WINDOW - 1))); + const last_allowed_key = context_len.add(qpos_hi); + tile_start = first_allowed_key.div(@as(i32, @intCast(TILE_SIZE))).maximum(0); + tile_end = last_allowed_key.div(@as(i32, @intCast(TILE_SIZE))).add(1).minimum(num_tiles); + } + + const kv_cache_mod: ttir.CacheModifier = + if (config.all_decode) .cg else .none; + + const block_tables_ptr_shifted = block_tables_ptr.addPtr(block_table_offset); + + // Iterate through tiles (loop carries M, L, acc). + var loop = k.openFor(tile_start, tile_end, 1, .{ m_init, l_init, acc_init }); + { + const j = loop.iv; + const M = loop.carried[0]; + const L = loop.carried[1]; + const acc = loop.carried[2]; + + const seq_offset = j.mul(@as(i32, @intCast(TILE_SIZE))).add(offs_t); + + const tile_mask: Value = if (TILE_SIZE == BLOCK_SIZE) + k.full(&.{TILE_SIZE}, 1, .i1) + else + seq_offset.lt(max_seq_prefix_len); + + const physical_block_idx = k.load( + block_tables_ptr_shifted.addPtr(seq_offset.div(@as(i32, @intCast(BLOCK_SIZE)))), + ).to(.i64); + + const v_offset = k.expandDims(physical_block_idx, 1).mul(stride_v_cache_0) + .add(kv_head_idx.to(.i64).mul(stride_v_cache_2)) + .add(k.expandDims(offs_d, 0).mul(config.stride_v_cache_3)) + .add(k.expandDims(seq_offset.rem(@as(i32, @intCast(BLOCK_SIZE))), 1).mul(stride_v_cache_1)); + + // k_offset : (HEAD_SIZE_PADDED, TILE_SIZE) + const k_offset = k.expandDims(physical_block_idx, 0).mul(stride_k_cache_0) + .add(kv_head_idx.to(.i64).mul(stride_k_cache_2)) + .add(k.expandDims(offs_d, 1).mul(config.stride_k_cache_3)) + .add(k.expandDims(seq_offset.rem(@as(i32, @intCast(BLOCK_SIZE))), 0).mul(stride_k_cache_1)); + + // K : (HEAD_SIZE_PADDED, TILE_SIZE) + const k_mask = k.mask2d(dim_mask, tile_mask, HEAD_SIZE_PADDED, TILE_SIZE); + const K_load = k.loadOpts(key_cache_ptr.addPtr(k_offset), .{ + .mask = k_mask, + .other = k.zeros(&.{ HEAD_SIZE_PADDED, TILE_SIZE }, config.kv_dtype), + .cache_modifier = kv_cache_mod, + }); + const K = dequantKv(K_load, k_scale, isFp8(config.kv_dtype), config.q_dtype); + + // V : (TILE_SIZE, HEAD_SIZE_PADDED) + const v_mask = k.mask2d(tile_mask, dim_mask, TILE_SIZE, HEAD_SIZE_PADDED); + const V_load = k.loadOpts(value_cache_ptr.addPtr(v_offset), .{ + .mask = v_mask, + .other = k.zeros(&.{ TILE_SIZE, HEAD_SIZE_PADDED }, config.kv_dtype), + .cache_modifier = kv_cache_mod, + }); + const V = dequantKv(V_load, v_scale, isFp8(config.kv_dtype), config.q_dtype); + + // S : (BLOCK_M, TILE_SIZE) + const acc_zero = k.zeros(&.{ BLOCK_M, TILE_SIZE }, .f32); + const qk = k.dot(Q, K, acc_zero); + var S = qk_scale.mul(qk); + + if (config.use_softcap) { + S = applySoftcap(k, S, softcap).mul(RCP_LN2); + } + + // seq_mask = seq_offset[None, :] < context_len + query_pos[:, None] + 1 + const ql_2d_i32 = query_pos.expandDims(1); + const so_2d = seq_offset.expandDims(0); + const rhs = ql_2d_i32.add(context_len).add(1); + const seq_mask = so_2d.lt(rhs); + + // S = tl.where(qmask_1 & qmask_0 & seq_mask, S, -inf) + const qm0_2d = query_mask_0.expandDims(1); + const qm1_2d = query_mask_1.expandDims(1); + const keep_mask = qm1_2d.bitAnd(qm0_2d).bitAnd(seq_mask); + S = k.where(keep_mask, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); + + if (SLIDING_WINDOW > 0) { + const diff = ql_2d_i32.add(context_len).sub(so_2d); + const in_win = diff.lt(@as(i32, @intCast(SLIDING_WINDOW))); + S = k.where(in_win, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); + } + + if (config.use_alibi_slopes) { + const alibi_2d = alibi_slope.expandDims(1); + const pos_diff = seq_offset.sub(context_len).to(config.scale_dtype).expandDims(0); + S = S.add(alibi_2d.mul(pos_diff).to(.f32).mul(RCP_LN2)); + } + + if (config.use_qq_bias) { + const key_rel_pos = seq_offset.sub(context_len); + const is_query_key = key_rel_pos.ge(0) + .bitAnd(key_rel_pos.to(.i64).lt(qq_bias_stride_0)); + const qq_ptrs = qq_bias_row_ptrs.expandDims(1) + .addPtr(key_rel_pos.to(.i64).expandDims(0)); + const qq_bias = k.loadOpts(qq_ptrs, .{ + .mask = is_query_key.expandDims(0), + .other = k.zeros(&.{ BLOCK_M, TILE_SIZE }, config.scale_dtype), + }); + S = S.add(qq_bias.to(.f32).mul(RCP_LN2)); + } + + // m_j = max(M, max(S, axis=1)) + var m_j = M.maximum(k.maxOpts(S, .{ .axis = 1 })); + m_j = k.where(m_j.gt(-std.math.inf(f32)), m_j, k.full(&.{BLOCK_M}, 0.0, .f32)); + + // P = exp2(S - m_j[:, None]); l_j = sum(P, axis=1); alpha = exp2(M - m_j) + const mj_2d = m_j.expandDims(1); + const P = k.exp2(S.sub(mj_2d)); + const l_j = k.sumOpts(P, .{ .axis = 1 }); + const alpha = k.exp2(M.sub(m_j)); + + const alpha_2d = alpha.expandDims(1); + const acc_scaled = acc.mul(alpha_2d); + const new_L = L.mul(alpha).add(l_j); + const new_M = m_j; + + const P_cast = P.to(config.q_dtype); + const new_acc = k.dot(P_cast, V, acc_scaled); + + loop.yield(.{ new_M, new_L, new_acc }); + } + const L_final = loop.results[1]; + var acc_final = loop.results[2]; + + const L_2d = k.expandDims(L_final, 1); + const one_over_L = k.full(&.{ BLOCK_M, 1 }, 1.0, .f32).div(L_2d); + acc_final = acc_final.mul(k.broadcastTo(one_over_L, &.{ BLOCK_M, HEAD_SIZE_PADDED })); + + if (config.use_fp8) { + acc_final = acc_final.mul(out_scale); + acc_final = k.clampf( + acc_final, + k.full(&.{ BLOCK_M, HEAD_SIZE_PADDED }, config.fp8_min, .f32), + k.full(&.{ BLOCK_M, HEAD_SIZE_PADDED }, config.fp8_max, .f32), + ); + } + + const oo0_2d = query_offset_0.expandDims(1).mul(output_stride_0); + const oo1_2d = query_offset_1.expandDims(1).mul(output_stride_1); + const output_offset = oo0_2d.add(oo1_2d).add(k.expandDims(offs_d, 0)); + + const store_mask = k.mask2d(query_mask_0, dim_mask, BLOCK_M, HEAD_SIZE_PADDED) + .bitAnd(query_mask_1.expandDims(1)); + k.storeOpts( + output_ptr.addPtr(output_offset), + acc_final.to(config.o_dtype), + .{ .mask = store_mask }, + ); +} + +fn kernelUnifiedAttention3d( + k: *Builder, + segm_output_ptr: Value, + segm_max_ptr: Value, + segm_expsum_ptr: Value, + query_ptr: Value, + key_cache_ptr: Value, + value_cache_ptr: Value, + sink_ptr: Value, + block_tables_ptr: Value, + seq_lens_ptr: Value, + alibi_slopes_ptr: Value, + qq_bias_ptr: Value, + scale: Value, + k_scale: Value, + v_scale: Value, + softcap: Value, + block_table_stride: Value, + query_stride_0: Value, + query_stride_1: Value, + qq_bias_stride_0: Value, + stride_k_cache_0: Value, + stride_k_cache_1: Value, + stride_k_cache_2: Value, + stride_v_cache_0: Value, + stride_v_cache_1: Value, + stride_v_cache_2: Value, + query_start_len_ptr: Value, + num_seqs: Value, + config: KernelUnifiedAttention3dPtr.Config, +) void { + const BLOCK_M: i64 = config.block_m; + const BLOCK_Q: i64 = config.block_q; + const BLOCK_SIZE: i64 = config.block_size; + const TILE_SIZE: i64 = config.tile_size; + const HEAD_SIZE: i64 = config.head_size; + const HEAD_SIZE_PADDED: i64 = config.head_size_padded; + const NUM_QUERIES_PER_KV: i64 = config.num_queries_per_kv; + const NUM_QUERY_HEADS: i64 = config.num_query_heads; + const NUM_SEGMENTS_PER_SEQ: i64 = config.num_segments_per_seq; + const SLIDING_WINDOW: i64 = config.sliding_window; + + const q_block_global_idx = k.programId(.x); + const kv_head_idx = k.programId(.y); + const segm_idx = k.programId(.z); + + const qk_scale = scale.mul(RCP_LN2); + + const seq_idx = findSeqIdx(k, query_start_len_ptr, q_block_global_idx, num_seqs, BLOCK_Q, true); + + const q_start_raw = k.load(query_start_len_ptr.addPtr(seq_idx)); + const q_block_start_idx = q_start_raw.div(@as(i32, @intCast(BLOCK_Q))).add(seq_idx); + const q_block_local_idx = q_block_global_idx.sub(q_block_start_idx); + + const cur_batch_in_all_start_index = q_start_raw; + // Match Python's left-to-right `query_start_len_ptr + seq_idx + 1`: two `tt.addptr`s. + const cur_batch_in_all_stop_index = k.load(query_start_len_ptr.addPtr(seq_idx).addPtr(1)); + const cur_batch_query_len = cur_batch_in_all_stop_index.sub(cur_batch_in_all_start_index); + + const q_out_of_range = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))).ge(cur_batch_query_len); + k.returnIf(q_out_of_range, .{}); + + const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); + const tiles_per_segment = cdivFn(seq_len, @as(i32, @intCast(NUM_SEGMENTS_PER_SEQ * TILE_SIZE))); + + const segm_lo = segm_idx.mul(tiles_per_segment).mul(@as(i32, @intCast(TILE_SIZE))); + const segm_out_of_range = segm_lo.ge(seq_len); + k.returnIf(segm_out_of_range, .{}); + + const offs_m = k.arange(0, BLOCK_M, .i32); + const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); + const offs_t = k.arange(0, TILE_SIZE, .i32); + + const q_local_bq = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))); + const query_pos = q_local_bq.add(offs_m.div(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); + + const query_offset_0 = cur_batch_in_all_start_index.add(query_pos); + const query_offset_1 = kv_head_idx.mul(@as(i32, @intCast(NUM_QUERIES_PER_KV))) + .add(offs_m.rem(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); + + // query_offset = qo_0[:, None]*qstride_0 + qo_1[:, None]*qstride_1 + offs_d[None, :] + // Inline `expandDims(offs_d, 0)` *inside* the chain so it's emitted after + // the qo0+qo1 addi + const qo0_2d = k.expandDims(query_offset_0, 1).mul(query_stride_0); + const qo1_2d = k.expandDims(query_offset_1, 1).mul(query_stride_1); + const query_offset = qo0_2d.add(qo1_2d).add(k.expandDims(offs_d, 0)); + + const dim_mask: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + offs_d.lt(@as(i32, @intCast(HEAD_SIZE))) + else + k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); + const query_mask_0 = query_pos.lt(cur_batch_query_len); + const query_mask_1 = query_offset_1.lt(@as(i32, @intCast(NUM_QUERY_HEADS))); + + const q_mask_pre = k.expandDims(query_mask_0, 1).bitAnd(k.expandDims(query_mask_1, 1)); + const q_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + q_mask_pre.bitAnd(k.expandDims(dim_mask, 0)) + else + q_mask_pre; + const q_ptrs = query_ptr.addPtr(query_offset); + const Q = k.loadOpts(q_ptrs, .{ + .mask = k.broadcastTo(q_mask_partial, &.{ BLOCK_M, HEAD_SIZE_PADDED }), + .other = k.zeros(&.{ BLOCK_M, HEAD_SIZE_PADDED }, config.q_dtype), + }); + + const block_table_offset = seq_idx.to(.i64).mul(block_table_stride); + + // M init: USE_SINKS + segm_idx==0 → loaded sinks; else -inf. + const segm_is_zero = segm_idx.eq(0); + const m_init: Value = if (config.use_sinks) mb: { + var mi = k.openIfElse(segm_is_zero, .{k.tensorTy(&.{BLOCK_M}, .f32)}); + { + const loaded = k.loadOpts(sink_ptr.addPtr(query_offset_1), .{ + .mask = query_mask_1, + .other = k.full(&.{BLOCK_M}, -std.math.inf(f32), config.scale_dtype), + }); + mi.yieldThen(.{loaded.to(.f32).mul(RCP_LN2)}); + } + { + mi.yieldElse(.{k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32)}); + } + break :mb mi.results[0]; + } else k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32); + + const l_init = k.full(&.{BLOCK_M}, 1.0, .f32); + const acc_init = k.zeros(&.{ BLOCK_M, HEAD_SIZE_PADDED }, .f32); + + const context_len = seq_len.sub(cur_batch_query_len); + + const alibi_slope: Value = if (config.use_alibi_slopes) + k.loadOpts(alibi_slopes_ptr.addPtr(query_offset_1), .{ + .mask = query_mask_1, + .other = k.full(&.{BLOCK_M}, 0.0, config.scale_dtype), + }) + else + k.zeros(&.{BLOCK_M}, config.scale_dtype); + + // See 2d-body note on qq_bias_row_ptrs: else branch needs an explicit splatTo + // (no sibling offset to auto-broadcast against). + const qq_bias_row_ptrs: Value = if (config.use_qq_bias) + qq_bias_ptr.addPtr(query_pos.to(.i64).mul(qq_bias_stride_0)) + else + qq_bias_ptr.splatTo(&.{BLOCK_M}); + + const pad_term: i32 = @intCast(@divTrunc(BLOCK_M - 1, NUM_QUERIES_PER_KV) + 1); + const max_prefix_raw = context_len.add(q_local_bq).add(pad_term); + const max_seq_prefix_len = max_prefix_raw.minimum(seq_len); + const num_tiles = cdivFn(max_seq_prefix_len, @as(i32, @intCast(TILE_SIZE))); + + const segm_tile_lo = segm_idx.mul(tiles_per_segment); + const segm_tile_hi = segm_idx.add(1).mul(tiles_per_segment).minimum(num_tiles); + + const kv_cache_mod: ttir.CacheModifier = + if (config.all_decode) .cg else .none; + + var loop = k.openFor(segm_tile_lo, segm_tile_hi, 1, .{ m_init, l_init, acc_init }); + { + const j = loop.iv; + const M = loop.carried[0]; + const L = loop.carried[1]; + const acc = loop.carried[2]; + + const seq_offset = j.mul(@as(i32, @intCast(TILE_SIZE))).add(offs_t); + const tile_mask: Value = if (TILE_SIZE == BLOCK_SIZE) + k.full(&.{TILE_SIZE}, 1, .i1) + else + seq_offset.lt(max_seq_prefix_len); + + const physical_block_idx = k.load( + block_tables_ptr.addPtr(block_table_offset).addPtr(seq_offset.div(@as(i32, @intCast(BLOCK_SIZE)))), + ).to(.i64); + + const v_offset = k.expandDims(physical_block_idx, 1).mul(stride_v_cache_0) + .add(kv_head_idx.to(.i64).mul(stride_v_cache_2)) + .add(k.expandDims(offs_d, 0).mul(config.stride_v_cache_3)) + .add(k.expandDims(seq_offset.rem(@as(i32, @intCast(BLOCK_SIZE))), 1).mul(stride_v_cache_1)); + + const k_offset = k.expandDims(physical_block_idx, 0).mul(stride_k_cache_0) + .add(kv_head_idx.to(.i64).mul(stride_k_cache_2)) + .add(k.expandDims(offs_d, 1).mul(config.stride_k_cache_3)) + .add(k.expandDims(seq_offset.rem(@as(i32, @intCast(BLOCK_SIZE))), 0).mul(stride_k_cache_1)); + + const k_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + k.expandDims(dim_mask, 1).bitAnd(k.expandDims(tile_mask, 0)) + else + k.expandDims(tile_mask, 0); + const K_load = k.loadOpts(key_cache_ptr.addPtr(k_offset), .{ + .mask = k.broadcastTo(k_mask_partial, &.{ HEAD_SIZE_PADDED, TILE_SIZE }), + .other = k.zeros(&.{ HEAD_SIZE_PADDED, TILE_SIZE }, config.kv_dtype), + .cache_modifier = kv_cache_mod, + }); + const K = dequantKv(K_load, k_scale, isFp8(config.kv_dtype), config.q_dtype); + + const v_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + k.expandDims(tile_mask, 1).bitAnd(k.expandDims(dim_mask, 0)) + else + k.expandDims(tile_mask, 1); + const V_load = k.loadOpts(value_cache_ptr.addPtr(v_offset), .{ + .mask = k.broadcastTo(v_mask_partial, &.{ TILE_SIZE, HEAD_SIZE_PADDED }), + .other = k.zeros(&.{ TILE_SIZE, HEAD_SIZE_PADDED }, config.kv_dtype), + .cache_modifier = kv_cache_mod, + }); + const V = dequantKv(V_load, v_scale, isFp8(config.kv_dtype), config.q_dtype); + + // seq_mask = seq_offset[None, :] < context_len + query_pos[:, None] + 1 + const ql_2d_i32 = query_pos.expandDims(1); + const so_2d = seq_offset.expandDims(0); + const rhs = context_len.add(ql_2d_i32).add(1); + const seq_mask = so_2d.lt(rhs); + + const acc_zero = k.zeros(&.{ BLOCK_M, TILE_SIZE }, .f32); + const qk = k.dot(Q, K, acc_zero); + var S = qk_scale.mul(qk); + + if (config.use_softcap) { + S = applySoftcap(k, S, softcap).mul(RCP_LN2); + } + + const qm0_2d = query_mask_0.expandDims(1); + const qm1_2d = query_mask_1.expandDims(1); + const keep_mask = qm1_2d.bitAnd(qm0_2d).bitAnd(seq_mask); + S = k.where(keep_mask, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); + + if (SLIDING_WINDOW > 0) { + const diff = ql_2d_i32.add(context_len).sub(so_2d); + const in_win = diff.lt(@as(i32, @intCast(SLIDING_WINDOW))); + S = k.where(in_win, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); + } + + if (config.use_alibi_slopes) { + const alibi_2d = alibi_slope.expandDims(1); + const pos_diff = seq_offset.sub(context_len).to(config.scale_dtype).expandDims(0); + S = S.add(alibi_2d.mul(pos_diff).to(.f32).mul(RCP_LN2)); + } + + if (config.use_qq_bias) { + const key_rel_pos = seq_offset.sub(context_len); + const is_query_key = key_rel_pos.ge(0) + .bitAnd(key_rel_pos.to(.i64).lt(qq_bias_stride_0)); + const qq_ptrs = qq_bias_row_ptrs.expandDims(1) + .addPtr(key_rel_pos.to(.i64).expandDims(0)); + const qq_bias = k.loadOpts(qq_ptrs, .{ + .mask = is_query_key.expandDims(0), + .other = k.zeros(&.{ BLOCK_M, TILE_SIZE }, config.scale_dtype), + }); + S = S.add(qq_bias.to(.f32).mul(RCP_LN2)); + } + + var m_j = M.maximum(k.maxOpts(S, .{ .axis = 1 })); + m_j = k.where(m_j.gt(-std.math.inf(f32)), m_j, k.full(&.{BLOCK_M}, 0.0, .f32)); + + const mj_2d = m_j.expandDims(1); + const P = k.exp2(S.sub(mj_2d)); + const l_j = k.sumOpts(P, .{ .axis = 1 }); + const alpha = k.exp2(M.sub(m_j)); + + const alpha_2d = alpha.expandDims(1); + const acc_scaled = acc.mul(alpha_2d); + const new_L = L.mul(alpha).add(l_j); + + const P_cast = P.to(config.q_dtype); + const new_acc = k.dot(P_cast, V, acc_scaled); + + loop.yield(.{ m_j, new_L, new_acc }); + } + const M_final = loop.results[0]; + const L_final = loop.results[1]; + const acc_final = loop.results[2]; + + // segm_output_offset = qo_0[:, None].to(i64) * (NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) + // + qo_1[:, None] * (NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) + // + segm_idx * HEAD_SIZE_PADDED + offs_d[None, :] + const segm_out_head_stride_i32: i32 = @intCast(NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED); + const segm_out_token_stride_i64: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED; + + // Inline `segm_idx * HEAD_SIZE_PADDED` and `offs_d[None, :]` inside the chain + // so each term's ops emit in source order + const oo0_2d = k.expandDims(query_offset_0, 1).to(.i64).mul(segm_out_token_stride_i64); + const oo1_2d = k.expandDims(query_offset_1, 1).mul(segm_out_head_stride_i32); + const segm_output_offset = oo0_2d.add(oo1_2d) + .add(segm_idx.mul(@as(i32, @intCast(HEAD_SIZE_PADDED)))) + .add(k.expandDims(offs_d, 0)); + + const store_mask_2d = k.mask2d(query_mask_0, dim_mask, BLOCK_M, HEAD_SIZE_PADDED) + .bitAnd(query_mask_1.expandDims(1)); + k.storeOpts( + segm_output_ptr.addPtr(segm_output_offset), + acc_final, + .{ .mask = store_mask_2d }, + ); + + // segm_offset = qo_0.to(i64) * (NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ) + // + qo_1 * NUM_SEGMENTS_PER_SEQ + segm_idx + const segm_head_stride_i32: i32 = @intCast(NUM_SEGMENTS_PER_SEQ); + const segm_token_stride_i64: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ; + const segm_offset = query_offset_0.to(.i64).mul(segm_token_stride_i64) + .add(query_offset_1.mul(segm_head_stride_i32)) + .add(segm_idx); + const store_mask_1d = query_mask_0.bitAnd(query_mask_1); + k.storeOpts(segm_max_ptr.addPtr(segm_offset), M_final, .{ .mask = store_mask_1d }); + k.storeOpts(segm_expsum_ptr.addPtr(segm_offset), L_final, .{ .mask = store_mask_1d }); +} + +fn reduceSegments( + k: *Builder, + output_ptr: Value, + segm_output_ptr: Value, + segm_max_ptr: Value, + segm_expsum_ptr: Value, + seq_lens_ptr: Value, + num_seqs: Value, + out_scale_inv: Value, + output_stride_0: Value, + output_stride_1: Value, + block_table_stride: Value, + query_start_len_ptr: Value, + config: ReduceSegmentsPtr.Config, +) void { + _ = block_table_stride; + + const TILE_SIZE: i64 = config.tile_size; + const HEAD_SIZE: i64 = config.head_size; + const HEAD_SIZE_PADDED: i64 = config.head_size_padded; + const NUM_QUERY_HEADS: i64 = config.num_query_heads; + const NUM_SEGMENTS_PER_SEQ: i64 = config.num_segments_per_seq; + const BLOCK_Q: i64 = config.block_q; + + const query_token_idx = k.programId(.x); + const query_head_idx = k.programId(.y); + + const seq_idx = findSeqIdx(k, query_start_len_ptr, query_token_idx, num_seqs, BLOCK_Q, false); + + const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); + const tiles_per_segment = cdivFn(seq_len, @as(i32, @intCast(NUM_SEGMENTS_PER_SEQ * TILE_SIZE))); + const act_num_segments = cdivFn(seq_len, tiles_per_segment.mul(@as(i32, @intCast(TILE_SIZE)))); + + const seg_range = k.arange(0, NUM_SEGMENTS_PER_SEQ, .i32); + const segm_mask = seg_range.lt(act_num_segments); + + const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); + const dim_mask: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + offs_d.lt(@as(i32, @intCast(HEAD_SIZE))) + else + k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); + + // segm_offset = query_token_idx*NQH*NSPS + query_head_idx*NSPS + arange(NSPS) + const tok_stride: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ; + const head_stride: i32 = @intCast(NUM_SEGMENTS_PER_SEQ); + const segm_offset = query_token_idx.to(.i64).mul(tok_stride) + .add(query_head_idx.mul(head_stride).to(.i64)) + .add(seg_range.to(.i64)); + + const segm_max = k.loadOpts(segm_max_ptr.addPtr(segm_offset), .{ + .mask = segm_mask, + .other = k.full(&.{NUM_SEGMENTS_PER_SEQ}, -std.math.inf(f32), .f32), + }); + const overall_max = k.max(segm_max); + + var segm_expsum = k.loadOpts(segm_expsum_ptr.addPtr(segm_offset), .{ + .mask = segm_mask, + .other = k.zeros(&.{NUM_SEGMENTS_PER_SEQ}, .f32), + }); + segm_expsum = segm_expsum.mul(k.exp2(segm_max.sub(overall_max))); + const overall_expsum = k.sum(segm_expsum); + + // segm_output_offset : (NSPS, HSP) + const out_tok_stride: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED; + const out_head_stride: i32 = @intCast(NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED); + const head_size_padded_i32: i32 = @intCast(HEAD_SIZE_PADDED); + const base = query_token_idx.to(.i64).mul(out_tok_stride) + .add(query_head_idx.mul(out_head_stride).to(.i64)); + const seg_off_2d = b: { + const seg_col_i32 = k.expandDims(seg_range, 1); // (NSPS, 1) i32 + const seg_col_scaled = seg_col_i32.mul(head_size_padded_i32); // (NSPS, 1) i32 + break :b seg_col_scaled.to(.i64); // extsi → (NSPS, 1) i64 + }; + const dim_off_2d = b: { + const dim_row_i32 = k.expandDims(offs_d, 0); // (1, HSP) i32 + break :b dim_row_i32.to(.i64); // extsi → (1, HSP) i64 + }; + const seg_offset_2d_full = k.broadcastTo(base.add(seg_off_2d), &.{ NUM_SEGMENTS_PER_SEQ, HEAD_SIZE_PADDED }); + const dim_offset_2d_full = k.broadcastTo(dim_off_2d, &.{ NUM_SEGMENTS_PER_SEQ, HEAD_SIZE_PADDED }); + const segm_output_offset = seg_offset_2d_full.add(dim_offset_2d_full); + + var segm_output = k.loadOpts( + segm_output_ptr.addPtr(segm_output_offset), + .{ + .mask = k.mask2d(segm_mask, dim_mask, NUM_SEGMENTS_PER_SEQ, HEAD_SIZE_PADDED), + .other = k.zeros(&.{ NUM_SEGMENTS_PER_SEQ, HEAD_SIZE_PADDED }, .f32), + }, + ); + const rescale = k.exp2(segm_max.sub(overall_max)); + segm_output = segm_output.mul(rescale.expandDims(1)); + const acc_sum = k.sumOpts(segm_output, .{ .axis = 0 }); + + var acc = k.where( + overall_expsum.eq(0.0), + k.full(&.{HEAD_SIZE_PADDED}, 0.0, .f32), + acc_sum.div(overall_expsum), + ); + + if (config.use_fp8) { + acc = acc.mul(out_scale_inv); + acc = k.clampf( + acc, + k.full(&.{HEAD_SIZE_PADDED}, config.fp8_min, .f32), + k.full(&.{HEAD_SIZE_PADDED}, config.fp8_max, .f32), + ); + } + + const output_offset = query_token_idx.to(.i64).mul(output_stride_0) + .add(query_head_idx.to(.i64).mul(output_stride_1)) + .add(offs_d.to(.i64)); + k.storeOpts( + output_ptr.addPtr(output_offset), + acc.to(config.o_dtype), + .{ .mask = dim_mask }, + ); +} diff --git a/zml/kernel.zig b/zml/kernel.zig new file mode 100644 index 00000000..8a96e0b0 --- /dev/null +++ b/zml/kernel.zig @@ -0,0 +1,309 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const dialects = @import("mlir/dialects"); + +const triton_builder = @import("kernels/triton/builder"); +const mosaic_tpu_builder = @import("kernels/mosaic_tpu/builder"); +const tpu_dialect = @import("mlir/dialects/mosaic_tpu"); + +const DataType = @import("dtype.zig").DataType; +const Shape = @import("shape.zig").Shape; +const ops = @import("ops.zig"); +const Tensor = @import("tensor.zig").Tensor; +const CompilationContext = @import("module.zig").CompilationContext; +const mlirx = @import("mlirx.zig"); + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +pub const triton = struct { + pub const Builder = triton_builder.Builder; + pub const Value = triton_builder.Value; + pub const DType = triton_builder.DType; + pub const FinishError = triton_builder.FinishError; + + pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + else => std.debug.panic("zml.kernel.triton.from: dtype {s} has no Triton equivalent", .{@tagName(dt)}), + }; + } + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + num_stages: i32, + num_warps: i32, + output_operand_aliases: @FieldType(ops.TritonOps, "output_operand_aliases") = &.{}, + debug: bool = false, + }; + + pub fn emit( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + cfg: ConfigT, + ) ![:0]const u8 { + var b = try triton_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + try spec.run(&b, cfg); + return b.finish(&.{}); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = CompilationContext.current(); + + const ttir = emit(cur.allocator, cur.mlir_ctx, opts.cfg) catch |err| + std.debug.panic("zml.kernel.triton.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ttir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const tensor_results = ops.triton(inputs_arr, outputs_arr, .{ + .debug = opts.debug, + .name = name, + .ir = ttir, + .grid = opts.grid, + .num_stages = opts.num_stages, + .num_warps = opts.num_warps, + .output_operand_aliases = opts.output_operand_aliases, + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; + } +}; + +pub const mosaic_tpu = struct { + const Builder = mosaic_tpu_builder.Builder; + const DType = mosaic_tpu_builder.DType; + const FinishError = mosaic_tpu_builder.FinishError; + + pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + else => std.debug.panic("zml.kernel.mosaic_tpu.from: dtype {s} has no Mosaic-TPU equivalent", .{@tagName(dt)}), + }; + } + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + extras: CallExtras = .{}, + }; + + pub fn emit( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + cfg: ConfigT, + ) ![:0]const u8 { + var b = try mosaic_tpu_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + try spec.run(&b, cfg); + return b.finishOpts(&.{}, .{ .canonicalize = true }); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = CompilationContext.current(); + + const ir = emit(cur.allocator, cur.mlir_ctx, opts.cfg) catch |err| + std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + const backend_config = buildBackendConfig(cur.allocator, cur.mlir_ctx, ir, opts.extras) catch |err| + std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: backend_config build failed: {}", .{ name, err }); + defer cur.allocator.free(backend_config); + + var values: [spec.inputs.len]*const mlir.Value = undefined; + inline for (spec.inputs, 0..) |fname, i| values[i] = @field(inputs, fname).value(); + + var res_types: [spec.outputs.len]*const mlir.Type = undefined; + inline for (spec.outputs, 0..) |fname, i| { + const out = @field(outputs, fname); + res_types[i] = mlir.rankedTensorType(out.dims(), mlirx.Type.fromDType(cur.mlir_ctx, out.dtype())); + } + + const op = callTpuCustomCall(&values, &res_types, backend_config, opts.extras); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| { + const out = @field(outputs, fname); + @field(results, fname) = Tensor._result(out, op.result(i)); + } + return results; + } + }; + } + + const CallExtras = struct { + vmem_limit_bytes: ?i64 = null, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + has_communication: ?bool = null, + additional_attributes: []const mlir.NamedAttribute = &.{}, + output_operand_aliases: []const dialects.stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, + }; + + const CustomCallConfig = struct { + body: []const u8, + has_communication: ?bool = null, + serialization_format: i64 = 1, + needs_layout_passes: bool = true, + shape_invariant_numerics: bool = false, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + }; + + const ScopedMemoryConfig = struct { + memory_space: i64 = 1, + offset: i64 = 0, + size: i64, + }; + + const BackendConfig = struct { + custom_call_config: CustomCallConfig, + scoped_memory_configs: ?[]const ScopedMemoryConfig = null, + }; + + fn buildBackendConfig( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + ir: [:0]const u8, + extras: CallExtras, + ) ![]u8 { + const module = try mlir.Module.parse(ctx, ir); + defer module.deinit(); + + tpu_dialect.registerMosaicSerdePass(); + const prev_allow_unregistered = ctx.allowUnregisteredDialects(); + ctx.setAllowUnregisteredDialects(true); + defer ctx.setAllowUnregisteredDialects(prev_allow_unregistered); + + const pm = mlir.PassManager.init(ctx); + defer pm.deinit(); + try pm.asOpPassManager().addPipeline("mosaic-serde{serialize=true}"); + try pm.runOnOp(module.operation()); + + const Encoder = std.base64.standard.Encoder; + const b64 = b64: { + var bytecode: std.Io.Writer.Allocating = .init(allocator); + defer bytecode.deinit(); + try module.operation().writeBytecode(.{ .desired_emit_version = 0 }, &bytecode.writer); + const buf = try allocator.alloc(u8, Encoder.calcSize(bytecode.written().len)); + break :b64 Encoder.encode(buf, bytecode.written()); + }; + defer allocator.free(b64); + + const config = BackendConfig{ + .custom_call_config = .{ + .body = b64, + .has_communication = extras.has_communication, + .disable_bounds_checks = extras.disable_bounds_checks, + .disable_semaphore_checks = extras.disable_semaphore_checks, + }, + .scoped_memory_configs = if (extras.vmem_limit_bytes) |vmem| + &.{.{ .size = vmem }} + else + null, + }; + + var json: std.Io.Writer.Allocating = .init(allocator); + errdefer json.deinit(); + + try json.writer.print("{f}", .{std.json.fmt(config, .{ .emit_null_optional_fields = false })}); + return try json.toOwnedSlice(); + } + + fn callTpuCustomCall( + inputs: []const *const mlir.Value, + result_types: []const *const mlir.Type, + backend_config: []const u8, + extras: CallExtras, + ) *mlir.Operation { + const cur = CompilationContext.current(); + return dialects.stablehlo.custom_call( + cur.mlir_ctx, + inputs, + result_types, + .{ + .call_target_name = "tpu_custom_call", + .backend_config = .{ .original = backend_config }, + .has_side_effect = false, + .api_version = .original, + .additional_attributes = extras.additional_attributes, + .output_operand_aliases = extras.output_operand_aliases, + }, + .unknown(cur.mlir_ctx), + ).appendTo(cur.currentScope().block); + } +}; diff --git a/zml/module.zig b/zml/module.zig index 61f32bae..aaf41016 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -33,7 +33,7 @@ fn mlirRegistry(io: std.Io) *mlir.DialectRegistry { mlir.registerPasses("Transforms"); const mlir_registry = mlir.DialectRegistry.init() catch unreachable; - inline for (.{ "func", "stablehlo", "sdy" }) |d| { + inline for (.{ "func", "stablehlo", "sdy", "arith", "scf", "math", "tt", "affine", "memref", "vector", "tpu", "cf" }) |d| { mlir.DialectHandle.fromString(d).insertDialect(mlir_registry); } mlir.registerFuncExtensions(mlir_registry); @@ -52,6 +52,7 @@ pub const CompilationOptions = struct { xla_dump_to: ?[]const u8 = null, xla_dump_fusion_visualization: bool = false, xla_dump_hlo_pass_re: ?[]const u8 = null, + xla_dump_emitter_re: ?[]const u8 = null, }; const AttributeList = stdx.BoundedArray(mlir.NamedAttribute, 3); @@ -609,6 +610,9 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: if (opts.xla_dump_hlo_pass_re) |re| { try setXlaOverrideFlag(overrides_map, "xla_dump_hlo_pass_re", re, upb_arena); } + if (opts.xla_dump_emitter_re) |re| { + try setXlaOverrideFlag(overrides_map, "xla_dump_emitter_re", re, upb_arena); + } } break :blk options; diff --git a/zml/platform.zig b/zml/platform.zig index 01361ca1..f049e71b 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -202,25 +202,6 @@ pub const Platform = struct { memories: []const Memory, physical_mesh: zml.sharding.PhysicalMesh, - triton_runtime: ?attention.triton.Runtime = null, - tpu_ir_runtime: ?attention.tpu.Runtime = null, - - pub fn initBackend(self: *Platform, allocator: std.mem.Allocator, io: std.Io, backend: attention.paged_attention.Backend) !void { - switch (backend) { - .triton => { - if (self.triton_runtime == null) { - self.triton_runtime = try zml.attention.triton.Runtime.init(allocator, io); - } - }, - .mosaic_tpu => { - if (self.tpu_ir_runtime == null) { - self.tpu_ir_runtime = try zml.attention.tpu.Runtime.init(allocator, io); - } - }, - else => {}, - } - } - pub const MAX_NUM_DEVICES: u16 = if (platforms.isEnabled(.tpu)) 64 else 32; pub fn init(allocator: std.mem.Allocator, io: std.Io, target: Target, options: CreateOptions) !*Platform { @@ -422,9 +403,8 @@ pub const Platform = struct { } pub fn deinit(self: *Platform, allocator: std.mem.Allocator, io: std.Io) void { + _ = io; self.pjrt_client.deinit(self.pjrt_api); - if (self.tpu_ir_runtime) |*rt| rt.deinit(io); - if (self.triton_runtime) |*rt| rt.deinit(allocator, io); self.arena_state.promote(allocator).deinit(); } diff --git a/zml/zml.zig b/zml/zml.zig index 4be94092..72304eac 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -46,6 +46,7 @@ pub const tensor = @import("tensor.zig"); pub const Tensor = tensor.Tensor; pub const testing = @import("testing.zig"); pub const tracer = @import("tracer.zig"); +pub const kernel = @import("kernel.zig"); test "zml" { std.testing.refAllDecls(@This()); From e2ab3022de5d538a8c61d50f7f64090ec953b9e3 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 30 Apr 2026 17:32:56 +0000 Subject: [PATCH 464/881] platforms: for aarch64 cpu flag when selecting the linux_arm64 platform this fixes highwayhash outdated build files that select on this sadly we can't update highwayhash just yet --- platforms/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index d4c9f450..bb5c804a 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -71,6 +71,7 @@ platform( ], flags = [ "--force_pic=true", # required for protobuf upb on Linux + "--cpu=aarch64", ], visibility = ["//visibility:public"], ) From 5e4d1b1d057c208b75135473f8707d0879169f20 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 30 Apr 2026 17:34:06 +0000 Subject: [PATCH 465/881] zml: bump allocation to 90% of the GPU memory up from 85% since we now usually compile _before_ loading models (at least on llmd) --- zml/platform.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/platform.zig b/zml/platform.zig index f049e71b..c83dd977 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -511,7 +511,7 @@ pub const CreateOptions = struct { // bump memory fraction from XLA defaults of 75% to 90%. // Even on a 8GB GPU it should leave enough space for the Cuda driver // https://github.com/openxla/xla/blob/3e87afa11a865cf91137522492918ad18bfe5b7c/xla/pjrt/plugin/xla_gpu/xla_gpu_allocator_config.h#L25-L60 - cuda: Cuda = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.85 } } }, + cuda: Cuda = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.90 } } }, rocm: struct {} = .{}, tpu: struct {} = .{}, neuron: struct {} = .{}, From b230be2f3f555e2589f1dc89a44423b39bcf823f Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 30 Apr 2026 17:34:49 +0000 Subject: [PATCH 466/881] platforms: bump PJRT=manual-2026-04-30T17-13-00Z notably, this bumps fixes TTIR gen problems now that we switched away from python based TTIR --- platforms/cpu/cpu.bzl | 12 ++++++------ platforms/cuda/cuda.bzl | 6 +++--- platforms/rocm/rocm.bzl | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 5e206fb2..99ce5d7c 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -23,22 +23,22 @@ def _cpu_pjrt_plugin_impl(mctx): http_archive( name = "libpjrt_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "05da8ee4d19edf327c0bcdc8a66811877bcd5eab75e44f1bbacc6b6b1e9947ca", - url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-04-21T10-00-00Z/pjrt-cpu_linux-amd64.tar.gz", + sha256 = "f71402ff75bb23afc49511d7b666a48371749e663d653da4fd70b95f6a5e77ab", + url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-04-30T17-13-00Z/pjrt-cpu_linux-amd64.tar.gz", ) http_archive( name = "libpjrt_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "7da35d4b81359f42b56e3b820cacb10c4569caeb4926f2e3b11da913cd8f7c11", - url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-04-21T10-00-00Z/pjrt-cpu_darwin-amd64.tar.gz", + sha256 = "8ba86d6a100e89b9ab8912d3ca85e0d1ba1ffb36d7961f8b41cdc1a29d9c5df8", + url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-04-30T17-13-00Z/pjrt-cpu_darwin-amd64.tar.gz", ) http_archive( name = "libpjrt_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "51513b318bd180d33ef88363b5c96fc20ee7ca9db325830ebf694888c6fbc4ec", - url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-04-21T10-00-00Z/pjrt-cpu_darwin-arm64.tar.gz", + sha256 = "abc8403005cf48c2014fa8417dae3e92f25fc97810f840bb5368e9a5df487fdd", + url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-04-30T17-13-00Z/pjrt-cpu_darwin-arm64.tar.gz", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index b13d0878..874cafc4 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -200,15 +200,15 @@ _UBUNTU_PACKAGES = { ], } -PJRT_CUDA_RELEASE = "manual-2026-04-21T10-00-00Z" +PJRT_CUDA_RELEASE = "manual-2026-04-30T17-13-00Z" _PJRT_CUDA_ASSETS = { "amd64": { - "sha256": "8e34f4ead657b697e1c670cb35acb562bee9f5ff31948411d1b8ad11416df417", + "sha256": "bab2fe379d8cfdd54f12cb537f44f72f8f023665cfe23524a06930f8d7d12010", "url": "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-cuda_linux-amd64.tar.gz", }, "arm64": { - "sha256": "54da63a8342f848e42fb26ea1f854db5211b55f9abc753e93ad06ae2964bcb44", + "sha256": "cce6524a263fcbfa4f40bd39419d26517629f7f0c288707b3af81fe84ad93c45", "url": "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-cuda_linux-arm64.tar.gz", }, } diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 32946833..744d61d1 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -254,8 +254,8 @@ def _rocm_impl(mctx): http_archive( name = "libpjrt_rocm", build_file = "libpjrt_rocm.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/nightly-2026-04-17/pjrt-rocm_linux-amd64.tar.gz", - sha256 = "b19db71d3d9aa43bda961a22e81b6a2f48ba2c633a18010c07a66b4ca48ec4d2", + url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-04-30T17-13-00Z/pjrt-rocm_linux-amd64.tar.gz", + sha256 = "e543e48f902656598234b5ffc05196dde84a87106be21ada126c7323572f792b", ) return mctx.extension_metadata( From 57d191d0929a434470efba5765e7bf4dcdff62ac Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 4 May 2026 13:04:11 +0000 Subject: [PATCH 467/881] zml/attention: make triton the default paged attention provider on cuda --- zml/attention/paged_attention.zig | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 4cc7014d..c9104a0d 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -17,17 +17,7 @@ pub const Backend = enum { pub fn auto(platform: *const zml.Platform) Backend { return switch (platform.target) { - .cuda => b: { - const first_device = platform.devices[0].pjrt_device; - - if (zml.platform.cuda.tryGetComputeCapabilities(platform, first_device)) |cc| { - if (std.mem.eql(u8, cc, "9.0")) { - break :b .cuda_fa3; - } - } - - break :b .cuda_fa2; - }, + .cuda => .triton, .rocm => .triton, .tpu => .mosaic_tpu, else => stdx.debug.panic("Paged attention is not supported on {s} yet", .{@tagName(platform.target)}), From 92b8e34c7bd885e16cdde9bdf330162c45a1f47b Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 5 May 2026 14:52:15 +0200 Subject: [PATCH 468/881] zml/profiler: post-process saved xplane profiles and clean up profiler FFI (#522) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change fixes profile timestamp handling so saved xplane.pb data is post-processed like JAX/XLA’s `ProfilerSession`, and also cleans up the Zig/C++ FFI used by the profile export bridge. Before this, we were writing raw profiler output from the PJRT extension directly to `profiling.xplane.pb`, which bypassed XLA’s single-host post-processing step done by JAX that we can find here: https://github.com/google/tsl/blob/efac222ebfdbd682d3a407ef165865c6b6759e2d/tsl/profiler/lib/profiler_session.cc#L100 That made ZML behave differently from JAX, including wrong timeline origins in downstream viewers like XProf. To fix that, we now run `tsl::profiler::PostProcessSingleHostXSpace(...)` before writing the protobuf, and we treat that post-processed xplane.pb as the source for subsequent Perfetto conversion. We also register start/stop timestamps in `zml/profiler.zig` so they match XLA’s expectations for session timestamps with `real` clock. While touching this path, I tightened the FFI boundary as well. Error text returned from the xspace export bridge is now allocated with the provided Zig allocator instead of malloc, C receives a pointer to the `zig_allocator wrapper` to avoid lifetime issues from passing allocator state by value, and error propagation in `tools/xspace_to_perfetto_lib.cc `now uses `zig_slice` outputs directly instead of building std::string error objects just to copy them back out. This removes mixed allocation ownership between C++ and Zig and fixes the allocator lifetime bug that could crash across the FFI boundary. --- ffi/zig_allocator.zig | 5 +- tools/BUILD.bazel | 2 + tools/xspace_to_perfetto.cc | 30 +++++- tools/xspace_to_perfetto.h | 12 ++- tools/xspace_to_perfetto_lib.cc | 159 ++++++++++++++++++++++++-------- zml/profiler.zig | 42 ++++++--- 6 files changed, 188 insertions(+), 62 deletions(-) diff --git a/ffi/zig_allocator.zig b/ffi/zig_allocator.zig index 35f5fc33..52c000bd 100644 --- a/ffi/zig_allocator.zig +++ b/ffi/zig_allocator.zig @@ -1,4 +1,5 @@ const std = @import("std"); + const c = @import("c"); pub const ZigAllocator = struct { @@ -12,7 +13,7 @@ pub const ZigAllocator = struct { pub fn alloc(ctx: ?*const anyopaque, elem: usize, nelems: usize, alignment: usize) callconv(.c) ?*anyopaque { const self: *const std.mem.Allocator = @ptrCast(@alignCast(ctx)); - const ret = self.rawAlloc(elem * nelems, std.math.log2_int(usize, alignment), @returnAddress()) orelse return null; + const ret = self.rawAlloc(elem * nelems, .fromByteUnits(alignment), @returnAddress()) orelse return null; return @ptrCast(ret); } @@ -20,6 +21,6 @@ pub const ZigAllocator = struct { const self: *const std.mem.Allocator = @ptrCast(@alignCast(ctx)); const memory: [*c]u8 = @ptrCast(ptr); const size = elem * nelems; - self.rawFree(memory[0..size], std.math.log2_int(usize, alignment), @returnAddress()); + self.rawFree(memory[0..size], .fromByteUnits(alignment), @returnAddress()); } }; diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 925a59db..d7532fcb 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -17,8 +17,10 @@ cc_library( deps = [ "//ffi:cc", "@tsl//tsl/profiler/protobuf:xplane_proto_cc", + "@xla//xla/tsl/profiler/convert:post_process_single_host_xplane", "@xla//xla/tsl/profiler/convert:trace_events_to_json", "@xla//xla/tsl/profiler/convert:xplane_to_trace_events", + "@xla//xla/tsl/profiler/utils:xplane_utils", ], ) diff --git a/tools/xspace_to_perfetto.cc b/tools/xspace_to_perfetto.cc index ce70cb6b..daa44f69 100644 --- a/tools/xspace_to_perfetto.cc +++ b/tools/xspace_to_perfetto.cc @@ -1,17 +1,30 @@ +#include #include #include #include +#include "ffi/zig_allocator.h" +#include "ffi/zig_slice.h" + namespace zml::tools { bool DumpXSpaceFileToTraceJsonFile(std::string_view input_path, std::string_view output_path, - std::string* error); + zig_allocator* error_allocator, + zig_slice* error); } void PrintUsage(std::string_view argv0) { std::cerr << "Usage: " << argv0 << " [output_trace.json]\n"; } +void* Alloc(const void*, size_t elem, size_t nelems, size_t) { + return std::malloc(elem * nelems); +} + +void Free(const void*, void* ptr, size_t, size_t, size_t) { + std::free(ptr); +} + int main(int argc, char** argv) { if (argc < 2 || argc > 3) { PrintUsage(argv[0]); @@ -21,10 +34,19 @@ int main(int argc, char** argv) { const std::string input_path = argv[1]; const std::string output_path = argc == 3 ? argv[2] : input_path + ".trace.json"; - std::string error; + zig_allocator error_allocator = { + .ctx = nullptr, + .alloc = &Alloc, + .free = &Free, + }; + zig_slice error = {.ptr = nullptr, .len = 0}; if (!zml::tools::DumpXSpaceFileToTraceJsonFile(input_path, output_path, - &error)) { - std::cerr << error << "\n"; + &error_allocator, &error)) { + std::cerr << std::string_view(static_cast(error.ptr), error.len) + << "\n"; + if (error.ptr != nullptr) { + error_allocator.deallocate(static_cast(error.ptr), error.len); + } return 1; } diff --git a/tools/xspace_to_perfetto.h b/tools/xspace_to_perfetto.h index 48794bb7..3546c8af 100644 --- a/tools/xspace_to_perfetto.h +++ b/tools/xspace_to_perfetto.h @@ -1,14 +1,22 @@ #pragma once +#include + +#include "ffi/zig_allocator.h" #include "ffi/zig_slice.h" #ifdef __cplusplus extern "C" { #endif -zig_slice zml_xspace_to_perfetto_dump(zig_slice xspace_proto, +zig_slice zml_xspace_to_perfetto_dump(zig_allocator* error_allocator, + zig_slice xspace_proto, zig_slice output_path); -void zml_xspace_to_perfetto_str_free(zig_slice text); +zig_slice zml_xspace_normalize_and_dump(zig_allocator* error_allocator, + zig_slice xspace_proto, + uint64_t start_time_ns, + uint64_t stop_time_ns, + zig_slice output_path); #ifdef __cplusplus } diff --git a/tools/xspace_to_perfetto_lib.cc b/tools/xspace_to_perfetto_lib.cc index 42823dfd..934c18ab 100644 --- a/tools/xspace_to_perfetto_lib.cc +++ b/tools/xspace_to_perfetto_lib.cc @@ -1,27 +1,88 @@ -#include #include #include #include #include #include +#include #include "tools/xspace_to_perfetto.h" +#include "xla/tsl/profiler/convert/post_process_single_host_xplane.h" #include "xla/tsl/profiler/convert/trace_events_to_json.h" #include "xla/tsl/profiler/convert/xplane_to_trace_events.h" +#include "xla/tsl/profiler/utils/xplane_utils.h" #include "tsl/profiler/protobuf/xplane.pb.h" namespace { +zig_slice EmptySlice() { return {.ptr = nullptr, .len = 0}; } + +std::string_view ToStringView(zig_slice slice) { + return std::string_view(static_cast(slice.ptr), slice.len); +} + +bool SetError(zig_allocator* error_allocator, + zig_slice* error, + std::string_view message) { + char* buffer = error_allocator->allocate(message.size()); + if (buffer == nullptr) { + *error = EmptySlice(); + return false; + } + + std::memcpy(buffer, message.data(), message.size()); + *error = {.ptr = buffer, .len = message.size()}; + return true; +} + +bool SetErrorWithPath(zig_allocator* error_allocator, + zig_slice* error, + std::string_view prefix, + std::string_view path) { + const size_t len = prefix.size() + path.size(); + char* buffer = error_allocator->allocate(len); + if (buffer == nullptr) { + *error = EmptySlice(); + return false; + } + + std::memcpy(buffer, prefix.data(), prefix.size()); + std::memcpy(buffer + prefix.size(), path.data(), path.size()); + *error = {.ptr = buffer, .len = len}; + return true; +} + +void PostProcessXSpace(tensorflow::profiler::XSpace* xspace, + uint64_t start_time_ns, + uint64_t stop_time_ns) { + tsl::profiler::SetXSpacePidIfNotSet(*xspace, getpid()); + tsl::profiler::PostProcessSingleHostXSpace( + xspace, start_time_ns, stop_time_ns); +} + +bool SerializeXSpace(const tensorflow::profiler::XSpace& xspace, + std::string* serialized_xspace, + zig_allocator* error_allocator, + zig_slice* error) { + if (!xspace.SerializeToString(serialized_xspace)) { + SetError(error_allocator, error, + "Failed to serialize normalized XSpace protobuf"); + return false; + } + return true; +} + bool ParseXSpace(std::string_view xspace_proto, tensorflow::profiler::XSpace* xspace, - std::string* error) { - if (xspace_proto.size() > static_cast(std::numeric_limits::max())) { - *error = "XSpace protobuf is too large to parse"; + zig_allocator* error_allocator, + zig_slice* error) { + if (xspace_proto.size() > + static_cast(std::numeric_limits::max())) { + SetError(error_allocator, error, "XSpace protobuf is too large to parse"); return false; } if (!xspace->ParseFromArray(xspace_proto.data(), static_cast(xspace_proto.size()))) { - *error = "Failed to parse XSpace protobuf"; + SetError(error_allocator, error, "Failed to parse XSpace protobuf"); return false; } return true; @@ -29,30 +90,34 @@ bool ParseXSpace(std::string_view xspace_proto, bool WriteFile(std::string_view output_path, std::string_view contents, - std::string* error) { + zig_allocator* error_allocator, + zig_slice* error) { std::ofstream output(std::string(output_path), std::ios::binary); if (!output.is_open()) { - *error = "Failed to open output file: " + std::string(output_path); + SetErrorWithPath(error_allocator, error, "Failed to open output file: ", + output_path); return false; } output.write(contents.data(), static_cast(contents.size())); if (!output) { - *error = "Failed to write output file: " + std::string(output_path); + SetErrorWithPath(error_allocator, error, "Failed to write output file: ", + output_path); return false; } return true; } -} +} // namespace namespace zml::tools { bool ConvertXSpaceProtoToTraceJson(std::string_view xspace_proto, std::string* json, - std::string* error) { + zig_allocator* error_allocator, + zig_slice* error) { tensorflow::profiler::XSpace xspace; - if (!ParseXSpace(xspace_proto, &xspace, error)) { + if (!ParseXSpace(xspace_proto, &xspace, error_allocator, error)) { return false; } @@ -64,62 +129,74 @@ bool ConvertXSpaceProtoToTraceJson(std::string_view xspace_proto, bool DumpXSpaceProtoToTraceJsonFile(std::string_view xspace_proto, std::string_view output_path, - std::string* error) { + zig_allocator* error_allocator, + zig_slice* error) { std::string json; - if (!ConvertXSpaceProtoToTraceJson(xspace_proto, &json, error)) { + if (!ConvertXSpaceProtoToTraceJson(xspace_proto, &json, error_allocator, + error)) { return false; } - return WriteFile(output_path, json, error); + return WriteFile(output_path, json, error_allocator, error); } bool DumpXSpaceFileToTraceJsonFile(std::string_view input_path, std::string_view output_path, - std::string* error) { + zig_allocator* error_allocator, + zig_slice* error) { std::ifstream input(std::string(input_path), std::ios::binary); if (!input.is_open()) { - *error = "Failed to open input file: " + std::string(input_path); + SetErrorWithPath(error_allocator, error, "Failed to open input file: ", + input_path); return false; } tensorflow::profiler::XSpace xspace; if (!xspace.ParseFromIstream(&input)) { - *error = "Failed to parse XSpace protobuf from: " + std::string(input_path); + SetErrorWithPath(error_allocator, error, + "Failed to parse XSpace protobuf from: ", input_path); return false; } const tsl::profiler::TraceContainer container = tsl::profiler::ConvertXSpaceToTraceContainer(xspace); const std::string json = tsl::profiler::TraceContainerToJson(container); - return WriteFile(output_path, json, error); + return WriteFile(output_path, json, error_allocator, error); } -} +} // namespace zml::tools -extern "C" zig_slice zml_xspace_to_perfetto_dump(zig_slice xspace_proto, - zig_slice output_path) { - std::string error; +extern "C" zig_slice zml_xspace_to_perfetto_dump( + zig_allocator* error_allocator, + zig_slice xspace_proto, + zig_slice output_path) { + zig_slice error = EmptySlice(); if (zml::tools::DumpXSpaceProtoToTraceJsonFile( - std::string_view(static_cast(xspace_proto.ptr), - xspace_proto.len), - std::string_view(static_cast(output_path.ptr), - output_path.len), + ToStringView(xspace_proto), + ToStringView(output_path), + error_allocator, &error)) { - return {.ptr = nullptr, .len = 0}; + return EmptySlice(); } - - if (error.empty()) { - return {.ptr = nullptr, .len = 0}; - } - - void* buffer = std::malloc(error.size()); - if (buffer == nullptr) { - return {.ptr = nullptr, .len = 0}; - } - - std::memcpy(buffer, error.data(), error.size()); - return {.ptr = buffer, .len = error.size()}; + return error; } -extern "C" void zml_xspace_to_perfetto_str_free(zig_slice text) { - std::free(text.ptr); +extern "C" zig_slice zml_xspace_normalize_and_dump( + zig_allocator* error_allocator, zig_slice xspace_proto, + uint64_t start_time_ns, uint64_t stop_time_ns, zig_slice output_path) { + zig_slice error = EmptySlice(); + tensorflow::profiler::XSpace xspace; + if (!ParseXSpace(ToStringView(xspace_proto), &xspace, error_allocator, + &error)) { + return error; + } + + PostProcessXSpace(&xspace, start_time_ns, stop_time_ns); + std::string normalized_xspace; + if (!SerializeXSpace(xspace, &normalized_xspace, error_allocator, &error) || + !WriteFile(ToStringView(output_path), normalized_xspace, error_allocator, + &error)) { + return error; + } + + return EmptySlice(); } diff --git a/zml/profiler.zig b/zml/profiler.zig index 5157a277..e5dbab26 100644 --- a/zml/profiler.zig +++ b/zml/profiler.zig @@ -142,6 +142,8 @@ pub const Profiler = struct { allocator: std.mem.Allocator, io: std.Io, inner: ?pjrt.Profiler, + start_timestamp_ns: ?u64, + stop_timestamp_ns: ?u64, session_dir: []const u8, profile: Profile, @@ -170,6 +172,8 @@ pub const Profiler = struct { .allocator = allocator, .io = io, .inner = inner, + .start_timestamp_ns = null, + .stop_timestamp_ns = null, .session_dir = session_dir, .profile = .{ .protobuf_path = protobuf_path, @@ -187,6 +191,7 @@ pub const Profiler = struct { pub fn start(self: *Profiler) !void { if (self.inner) |*inner| { + self.start_timestamp_ns = @intCast(std.Io.Timestamp.now(self.io, .real).nanoseconds); try inner.start(); } } @@ -195,30 +200,41 @@ pub const Profiler = struct { var inner = self.inner orelse return null; self.inner = null; try inner.stop(); + self.stop_timestamp_ns = @intCast(std.Io.Timestamp.now(self.io, .real).nanoseconds); const protobuf = try inner.collectData(self.arena.allocator()); + var error_allocator = zffi.ZigAllocator.from(self.arena.allocator()); + const normalization_error = c.zml_xspace_normalize_and_dump( + &error_allocator, + zffi.ZigSlice.from(protobuf), + self.start_timestamp_ns.?, + self.stop_timestamp_ns.?, + zffi.ZigSlice.from(self.profile.protobuf_path), + ); - try self.writeFile(self.profile.protobuf_path, protobuf); + if (normalization_error.len != 0) { + log.err("Failed to normalize profile protobuf: {s}", .{zffi.ZigSlice.to(u8, normalization_error)}); + return error.ProfileTraceNormalizationFailed; + } + + const normalized_protobuf = try std.Io.Dir.cwd().readFileAlloc( + self.io, + self.profile.protobuf_path, + self.arena.allocator(), + .unlimited, + ); const conversion_error = c.zml_xspace_to_perfetto_dump( - zffi.ZigSlice.from(protobuf), + &error_allocator, + zffi.ZigSlice.from(normalized_protobuf), zffi.ZigSlice.from(self.profile.perfetto_path), ); - defer if (conversion_error.len != 0) { - c.zml_xspace_to_perfetto_str_free(conversion_error); - }; + if (conversion_error.len != 0) { - log.err("Failed to convert profile protobuf to Perfetto trace: {s}", .{zffi.ZigSlice.to(u8, conversion_error)}); + log.err("Failed to convert normalized profile protobuf to Perfetto trace: {s}", .{zffi.ZigSlice.to(u8, conversion_error)}); return error.ProfileTraceConversionFailed; } return self.profile; } - - fn writeFile(self: *const Profiler, path: []const u8, contents: []const u8) !void { - const file = try std.Io.Dir.createFile(.cwd(), self.io, path, .{}); - defer file.close(self.io); - - try file.writePositionalAll(self.io, contents, 0); - } }; From c38179513d4374baa2ea88086a05ab08fb55973c Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 6 May 2026 12:26:09 +0200 Subject: [PATCH 469/881] bump xprof-nightly (#523) the previous nightly we depend on is not on pypi anymore. I wanted to switch to stable release but 2.23 isn't public yet: https://pypi.org/project/xprof/#history --- tools/xprof/requirements.lock.txt | 893 +++++++++++++++--------------- 1 file changed, 445 insertions(+), 448 deletions(-) diff --git a/tools/xprof/requirements.lock.txt b/tools/xprof/requirements.lock.txt index c62d58c0..7154ccd0 100644 --- a/tools/xprof/requirements.lock.txt +++ b/tools/xprof/requirements.lock.txt @@ -4,139 +4,139 @@ aiohappyeyeballs==2.6.1 \ --hash=sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558 \ --hash=sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 # via aiohttp -aiohttp==3.13.3 \ - --hash=sha256:01ad2529d4b5035578f5081606a465f3b814c542882804e2e8cda61adf5c71bf \ - --hash=sha256:042e9e0bcb5fba81886c8b4fbb9a09d6b8a00245fd8d88e4d989c1f96c74164c \ - --hash=sha256:05861afbbec40650d8a07ea324367cb93e9e8cc7762e04dd4405df99fa65159c \ - --hash=sha256:084911a532763e9d3dd95adf78a78f4096cd5f58cdc18e6fdbc1b58417a45423 \ - --hash=sha256:0add0900ff220d1d5c5ebbf99ed88b0c1bbf87aa7e4262300ed1376a6b13414f \ - --hash=sha256:0db318f7a6f065d84cb1e02662c526294450b314a02bd9e2a8e67f0d8564ce40 \ - --hash=sha256:10b47b7ba335d2e9b1239fa571131a87e2d8ec96b333e68b2a305e7a98b0bae2 \ - --hash=sha256:1449ceddcdbcf2e0446957863af03ebaaa03f94c090f945411b61269e2cb5daf \ - --hash=sha256:147e422fd1223005c22b4fe080f5d93ced44460f5f9c105406b753612b587821 \ - --hash=sha256:1cb93e166e6c28716c8c6aeb5f99dfb6d5ccf482d29fe9bf9a794110e6d0ab64 \ - --hash=sha256:215a685b6fbbfcf71dfe96e3eba7a6f58f10da1dfdf4889c7dd856abe430dca7 \ - --hash=sha256:2712039939ec963c237286113c68dbad80a82a4281543f3abf766d9d73228998 \ - --hash=sha256:27234ef6d85c914f9efeb77ff616dbf4ad2380be0cda40b4db086ffc7ddd1b7d \ - --hash=sha256:28e027cf2f6b641693a09f631759b4d9ce9165099d2b5d92af9bd4e197690eea \ - --hash=sha256:2b8d8ddba8f95ba17582226f80e2de99c7a7948e66490ef8d947e272a93e9463 \ - --hash=sha256:2ba0eea45eb5cc3172dbfc497c066f19c41bac70963ea1a67d51fc92e4cf9a80 \ - --hash=sha256:2be0e9ccf23e8a94f6f0650ce06042cefc6ac703d0d7ab6c7a917289f2539ad4 \ - --hash=sha256:2e41b18a58da1e474a057b3d35248d8320029f61d70a37629535b16a0c8f3767 \ - --hash=sha256:2eb752b102b12a76ca02dff751a801f028b4ffbbc478840b473597fc91a9ed43 \ - --hash=sha256:2fc82186fadc4a8316768d61f3722c230e2c1dcab4200d52d2ebdf2482e47592 \ - --hash=sha256:2fff83cfc93f18f215896e3a190e8e5cb413ce01553901aca925176e7568963a \ - --hash=sha256:31a83ea4aead760dfcb6962efb1d861db48c34379f2ff72db9ddddd4cda9ea2e \ - --hash=sha256:34749271508078b261c4abb1767d42b8d0c0cc9449c73a4df494777dc55f0687 \ - --hash=sha256:34bac00a67a812570d4a460447e1e9e06fae622946955f939051e7cc895cfab8 \ - --hash=sha256:37239e9f9a7ea9ac5bf6b92b0260b01f8a22281996da609206a84df860bc1261 \ - --hash=sha256:37da61e244d1749798c151421602884db5270faf479cf0ef03af0ff68954c9dd \ - --hash=sha256:3b61b7169ababd7802f9568ed96142616a9118dd2be0d1866e920e77ec8fa92a \ - --hash=sha256:3d9908a48eb7416dc1f4524e69f1d32e5d90e3981e4e37eb0aa1cd18f9cfa2a4 \ - --hash=sha256:3dd4dce1c718e38081c8f35f323209d4c1df7d4db4bab1b5c88a6b4d12b74587 \ - --hash=sha256:4021b51936308aeea0367b8f006dc999ca02bc118a0cc78c303f50a2ff6afb91 \ - --hash=sha256:40c5e40ecc29ba010656c18052b877a1c28f84344825efa106705e835c28530f \ - --hash=sha256:425c126c0dc43861e22cb1c14ba4c8e45d09516d0a3ae0a3f7494b79f5f233a3 \ - --hash=sha256:44531a36aa2264a1860089ffd4dce7baf875ee5a6079d5fb42e261c704ef7344 \ - --hash=sha256:48e377758516d262bde50c2584fc6c578af272559c409eecbdd2bae1601184d6 \ - --hash=sha256:49a03727c1bba9a97d3e93c9f93ca03a57300f484b6e935463099841261195d3 \ - --hash=sha256:4ae5b5a0e1926e504c81c5b84353e7a5516d8778fbbff00429fe7b05bb25cbce \ - --hash=sha256:4e239d501f73d6db1522599e14b9b321a7e3b1de66ce33d53a765d975e9f4808 \ - --hash=sha256:56339a36b9f1fc708260c76c87e593e2afb30d26de9ae1eb445b5e051b98a7a1 \ - --hash=sha256:568f416a4072fbfae453dcf9a99194bbb8bdeab718e08ee13dfa2ba0e4bebf29 \ - --hash=sha256:5b179331a481cb5529fca8b432d8d3c7001cb217513c94cd72d668d1248688a3 \ - --hash=sha256:5b6073099fb654e0a068ae678b10feff95c5cae95bbfcbfa7af669d361a8aa6b \ - --hash=sha256:5d2d94f1f5fcbe40838ac51a6ab5704a6f9ea42e72ceda48de5e6b898521da51 \ - --hash=sha256:5dff64413671b0d3e7d5918ea490bdccb97a4ad29b3f311ed423200b2203e01c \ - --hash=sha256:5e1d8c8b8f1d91cd08d8f4a3c2b067bfca6ec043d3ff36de0f3a715feeedf926 \ - --hash=sha256:5f8ca7f2bb6ba8348a3614c7918cc4bb73268c5ac2a207576b7afea19d3d9f64 \ - --hash=sha256:642f752c3eb117b105acbd87e2c143de710987e09860d674e068c4c2c441034f \ - --hash=sha256:65d2ccb7eabee90ce0503c17716fc77226be026dcc3e65cce859a30db715025b \ - --hash=sha256:693781c45a4033d31d4187d2436f5ac701e7bbfe5df40d917736108c1cc7436e \ - --hash=sha256:694976222c711d1d00ba131904beb60534f93966562f64440d0c9d41b8cdb440 \ - --hash=sha256:697753042d57f4bf7122cab985bf15d0cef23c770864580f5af4f52023a56bd6 \ - --hash=sha256:69c56fbc1993fa17043e24a546959c0178fe2b5782405ad4559e6c13975c15e3 \ - --hash=sha256:6de499a1a44e7de70735d0b39f67c8f25eb3d91eb3103be99ca0fa882cdd987d \ - --hash=sha256:6fc0e2337d1a4c3e6acafda6a78a39d4c14caea625124817420abceed36e2415 \ - --hash=sha256:75ca857eba4e20ce9f546cd59c7007b33906a4cd48f2ff6ccf1ccfc3b646f279 \ - --hash=sha256:7a4a94eb787e606d0a09404b9c38c113d3b099d508021faa615d70a0131907ce \ - --hash=sha256:7b5e8fe4de30df199155baaf64f2fcd604f4c678ed20910db8e2c66dc4b11603 \ - --hash=sha256:7bfdc049127717581866fa4708791220970ce291c23e28ccf3922c700740fdc0 \ - --hash=sha256:7e63f210bc1b57ef699035f2b4b6d9ce096b5914414a49b0997c839b2bd2223c \ - --hash=sha256:7f9120f7093c2a32d9647abcaf21e6ad275b4fbec5b55969f978b1a97c7c86bf \ - --hash=sha256:8057c98e0c8472d8846b9c79f56766bcc57e3e8ac7bfd510482332366c56c591 \ - --hash=sha256:80dd4c21b0f6237676449c6baaa1039abae86b91636b6c91a7f8e61c87f89540 \ - --hash=sha256:81e97251d9298386c2b7dbeb490d3d1badbdc69107fb8c9299dd04eb39bddc0e \ - --hash=sha256:82611aeec80eb144416956ec85b6ca45a64d76429c1ed46ae1b5f86c6e0c9a26 \ - --hash=sha256:8542f41a62bcc58fc7f11cf7c90e0ec324ce44950003feb70640fc2a9092c32a \ - --hash=sha256:859bd3f2156e81dd01432f5849fc73e2243d4a487c4fd26609b1299534ee1845 \ - --hash=sha256:87797e645d9d8e222e04160ee32aa06bc5c163e8499f24db719e7852ec23093a \ - --hash=sha256:87b9aab6d6ed88235aa2970294f496ff1a1f9adcd724d800e9b952395a80ffd9 \ - --hash=sha256:8a60e60746623925eab7d25823329941aee7242d559baa119ca2b253c88a7bd6 \ - --hash=sha256:90455115e5da1c3c51ab619ac57f877da8fd6d73c05aacd125c5ae9819582aba \ - --hash=sha256:90751b8eed69435bac9ff4e3d2f6b3af1f57e37ecb0fbeee59c0174c9e2d41df \ - --hash=sha256:947c26539750deeaee933b000fb6517cc770bbd064bad6033f1cff4803881e43 \ - --hash=sha256:96d604498a7c782cb15a51c406acaea70d8c027ee6b90c569baa6e7b93073679 \ - --hash=sha256:988a8c5e317544fdf0d39871559e67b6341065b87fceac641108c2096d5506b7 \ - --hash=sha256:9a9dc347e5a3dc7dfdbc1f82da0ef29e388ddb2ed281bfce9dd8248a313e62b7 \ - --hash=sha256:9ae8dd55c8e6c4257eae3a20fd2c8f41edaea5992ed67156642493b8daf3cecc \ - --hash=sha256:9af5e68ee47d6534d36791bbe9b646d2a7c7deb6fc24d7943628edfbb3581f29 \ - --hash=sha256:9b174f267b5cfb9a7dba9ee6859cecd234e9a681841eb85068059bc867fb8f02 \ - --hash=sha256:9bf9f7a65e7aa20dd764151fb3d616c81088f91f8df39c3893a536e279b4b984 \ - --hash=sha256:9d4c940f02f49483b18b079d1c27ab948721852b281f8b015c058100e9421dd1 \ - --hash=sha256:9ebf57d09e131f5323464bd347135a88622d1c0976e88ce15b670e7ad57e4bd6 \ - --hash=sha256:a19884d2ee70b06d9204b2727a7b9f983d0c684c650254679e716b0b77920632 \ - --hash=sha256:a1e53262fd202e4b40b70c3aff944a8155059beedc8a89bba9dc1f9ef06a1b56 \ - --hash=sha256:a2212ad43c0833a873d0fb3c63fa1bacedd4cf6af2fee62bf4b739ceec3ab239 \ - --hash=sha256:a45530014d7a1e09f4a55f4f43097ba0fd155089372e105e4bff4ca76cb1b168 \ - --hash=sha256:a949eee43d3782f2daae4f4a2819b2cb9b0c5d3b7f7a927067cc84dafdbb9f88 \ - --hash=sha256:add1da70de90a2569c5e15249ff76a631ccacfe198375eead4aadf3b8dc849dc \ - --hash=sha256:af71fff7bac6bb7508956696dce8f6eec2bbb045eceb40343944b1ae62b5ef11 \ - --hash=sha256:b04be762396457bef43f3597c991e192ee7da460a4953d7e647ee4b1c28e7046 \ - --hash=sha256:b0d95340658b9d2f11d9697f59b3814a9d3bb4b7a7c20b131df4bcef464037c0 \ - --hash=sha256:b1a6102b4d3ebc07dad44fbf07b45bb600300f15b552ddf1851b5390202ea2e3 \ - --hash=sha256:b46020d11d23fe16551466c77823df9cc2f2c1e63cc965daf67fa5eec6ca1877 \ - --hash=sha256:b556c85915d8efaed322bf1bdae9486aa0f3f764195a0fb6ee962e5c71ef5ce1 \ - --hash=sha256:b903a4dfee7d347e2d87697d0713be59e0b87925be030c9178c5faa58ea58d5c \ - --hash=sha256:b928f30fe49574253644b1ca44b1b8adbd903aa0da4b9054a6c20fc7f4092a25 \ - --hash=sha256:b99281b0704c103d4e11e72a76f1b543d4946fea7dd10767e7e1b5f00d4e5704 \ - --hash=sha256:bae5c2ed2eae26cc382020edad80d01f36cb8e746da40b292e68fec40421dc6a \ - --hash=sha256:bb4f7475e359992b580559e008c598091c45b5088f28614e855e42d39c2f1033 \ - --hash=sha256:bbe7d4cecacb439e2e2a8a1a7b935c25b812af7a5fd26503a66dadf428e79ec1 \ - --hash=sha256:bfc1cc2fe31a6026a8a88e4ecfb98d7f6b1fec150cfd708adbfd1d2f42257c29 \ - --hash=sha256:c014c7ea7fb775dd015b2d3137378b7be0249a448a1612268b5a90c2d81de04d \ - --hash=sha256:c048058117fd649334d81b4b526e94bde3ccaddb20463a815ced6ecbb7d11160 \ - --hash=sha256:c0e2d366af265797506f0283487223146af57815b388623f0357ef7eac9b209d \ - --hash=sha256:c19b90316ad3b24c69cd78d5c9b4f3aa4497643685901185b65166293d36a00f \ - --hash=sha256:c685f2d80bb67ca8c3837823ad76196b3694b0159d232206d1e461d3d434666f \ - --hash=sha256:c6b8568a3bb5819a0ad087f16d40e5a3fb6099f39ea1d5625a3edc1e923fc538 \ - --hash=sha256:d32764c6c9aafb7fb55366a224756387cd50bfa720f32b88e0e6fa45b27dcf29 \ - --hash=sha256:d5a372fd5afd301b3a89582817fdcdb6c34124787c70dbcc616f259013e7eef7 \ - --hash=sha256:d60ac9663f44168038586cab2157e122e46bdef09e9368b37f2d82d354c23f72 \ - --hash=sha256:dca68018bf48c251ba17c72ed479f4dafe9dbd5a73707ad8d28a38d11f3d42af \ - --hash=sha256:de2c184bb1fe2cbd2cefba613e9db29a5ab559323f994b6737e370d3da0ac455 \ - --hash=sha256:e3531d63d3bdfa7e3ac5e9b27b2dd7ec9df3206a98e0b3445fa906f233264c57 \ - --hash=sha256:e50a2e1404f063427c9d027378472316201a2290959a295169bcf25992d04558 \ - --hash=sha256:e636b3c5f61da31a92bf0d91da83e58fdfa96f178ba682f11d24f31944cdd28c \ - --hash=sha256:ea37047c6b367fd4bd632bff8077449b8fa034b69e812a18e0132a00fae6e808 \ - --hash=sha256:f33ed1a2bf1997a36661874b017f5c4b760f41266341af36febaf271d179f6d7 \ - --hash=sha256:f76c1e3fe7d7c8afad7ed193f89a292e1999608170dcc9751a7462a87dfd5bc0 \ - --hash=sha256:f9444f105664c4ce47a2a7171a2418bce5b7bae45fb610f4e2c36045d85911d3 \ - --hash=sha256:fc290605db2a917f6e81b0e1e0796469871f5af381ce15c604a3c5c7e51cb730 \ - --hash=sha256:fc353029f176fd2b3ec6cfc71be166aba1936fe5d73dd1992ce289ca6647a9aa \ - --hash=sha256:fee0c6bc7db1de362252affec009707a17478a00ec69f797d23ca256e36d5940 +aiohttp==3.13.5 \ + --hash=sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5 \ + --hash=sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b \ + --hash=sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9 \ + --hash=sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b \ + --hash=sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8 \ + --hash=sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a \ + --hash=sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2 \ + --hash=sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1 \ + --hash=sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0 \ + --hash=sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037 \ + --hash=sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416 \ + --hash=sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1 \ + --hash=sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9 \ + --hash=sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a \ + --hash=sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36 \ + --hash=sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f \ + --hash=sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b \ + --hash=sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174 \ + --hash=sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b \ + --hash=sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8 \ + --hash=sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e \ + --hash=sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6 \ + --hash=sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c \ + --hash=sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe \ + --hash=sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9 \ + --hash=sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc \ + --hash=sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800 \ + --hash=sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286 \ + --hash=sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf \ + --hash=sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a \ + --hash=sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc \ + --hash=sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9 \ + --hash=sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665 \ + --hash=sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832 \ + --hash=sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297 \ + --hash=sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f \ + --hash=sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73 \ + --hash=sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b \ + --hash=sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9 \ + --hash=sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090 \ + --hash=sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49 \ + --hash=sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d \ + --hash=sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46 \ + --hash=sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83 \ + --hash=sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796 \ + --hash=sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be \ + --hash=sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d \ + --hash=sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf \ + --hash=sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83 \ + --hash=sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25 \ + --hash=sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06 \ + --hash=sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3 \ + --hash=sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6 \ + --hash=sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb \ + --hash=sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88 \ + --hash=sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9 \ + --hash=sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be \ + --hash=sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14 \ + --hash=sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a \ + --hash=sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c \ + --hash=sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3 \ + --hash=sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f \ + --hash=sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d \ + --hash=sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670 \ + --hash=sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031 \ + --hash=sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8 \ + --hash=sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643 \ + --hash=sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d \ + --hash=sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8 \ + --hash=sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8 \ + --hash=sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1 \ + --hash=sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88 \ + --hash=sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb \ + --hash=sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61 \ + --hash=sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4 \ + --hash=sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7 \ + --hash=sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9 \ + --hash=sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b \ + --hash=sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500 \ + --hash=sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6 \ + --hash=sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2 \ + --hash=sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10 \ + --hash=sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1 \ + --hash=sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3 \ + --hash=sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e \ + --hash=sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a \ + --hash=sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5 \ + --hash=sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95 \ + --hash=sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074 \ + --hash=sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5 \ + --hash=sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b \ + --hash=sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772 \ + --hash=sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a \ + --hash=sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274 \ + --hash=sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94 \ + --hash=sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13 \ + --hash=sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac \ + --hash=sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67 \ + --hash=sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76 \ + --hash=sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f \ + --hash=sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8 \ + --hash=sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7 \ + --hash=sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8 \ + --hash=sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3 \ + --hash=sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be \ + --hash=sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b \ + --hash=sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c \ + --hash=sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258 \ + --hash=sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c \ + --hash=sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6 \ + --hash=sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56 \ + --hash=sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b \ + --hash=sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d \ + --hash=sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a \ + --hash=sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162 \ + --hash=sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1 \ + --hash=sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6 \ + --hash=sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5 \ + --hash=sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540 \ + --hash=sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254 # via gcsfs aiosignal==1.4.0 \ --hash=sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e \ --hash=sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7 # via aiohttp -attrs==25.4.0 \ - --hash=sha256:16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11 \ - --hash=sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373 +attrs==26.1.0 \ + --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 \ + --hash=sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32 # via aiohttp -certifi==2026.2.25 \ - --hash=sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa \ - --hash=sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7 +certifi==2026.4.22 \ + --hash=sha256:3cb2210c8f88ba2318d29b0388d1023c8492ff72ecdde4ebdaddbb13a31b1c4a \ + --hash=sha256:8d455352a37b71bf76a79caa83a3d6c25afee4a385d632127b6afb3963f1c580 # via requests cffi==2.0.0 \ --hash=sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb \ @@ -224,192 +224,194 @@ cffi==2.0.0 \ --hash=sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453 \ --hash=sha256:fe562eb1a64e67dd297ccc4f5addea2501664954f2692b69a76449ec7913ecbf # via cryptography -charset-normalizer==3.4.6 \ - --hash=sha256:06a7e86163334edfc5d20fe104db92fcd666e5a5df0977cb5680a506fe26cc8e \ - --hash=sha256:0c173ce3a681f309f31b87125fecec7a5d1347261ea11ebbb856fa6006b23c8c \ - --hash=sha256:0e28d62a8fc7a1fa411c43bd65e346f3bce9716dc51b897fbe930c5987b402d5 \ - --hash=sha256:0e901eb1049fdb80f5bd11ed5ea1e498ec423102f7a9b9e4645d5b8204ff2815 \ - --hash=sha256:11afb56037cbc4b1555a34dd69151e8e069bee82e613a73bef6e714ce733585f \ - --hash=sha256:150b8ce8e830eb7ccb029ec9ca36022f756986aaaa7956aad6d9ec90089338c0 \ - --hash=sha256:172985e4ff804a7ad08eebec0a1640ece87ba5041d565fff23c8f99c1f389484 \ - --hash=sha256:197c1a244a274bb016dd8b79204850144ef77fe81c5b797dc389327adb552407 \ - --hash=sha256:1ae6b62897110aa7c79ea2f5dd38d1abca6db663687c0b1ad9aed6f6bae3d9d6 \ - --hash=sha256:1cf0a70018692f85172348fe06d3a4b63f94ecb055e13a00c644d368eb82e5b8 \ - --hash=sha256:1ed80ff870ca6de33f4d953fda4d55654b9a2b340ff39ab32fa3adbcd718f264 \ - --hash=sha256:22c6f0c2fbc31e76c3b8a86fba1a56eda6166e238c29cdd3d14befdb4a4e4815 \ - --hash=sha256:231d4da14bcd9301310faf492051bee27df11f2bc7549bc0bb41fef11b82daa2 \ - --hash=sha256:259695e2ccc253feb2a016303543d691825e920917e31f894ca1a687982b1de4 \ - --hash=sha256:2a24157fa36980478dd1770b585c0f30d19e18f4fb0c47c13aa568f871718579 \ - --hash=sha256:2b1a63e8224e401cafe7739f77efd3f9e7f5f2026bda4aead8e59afab537784f \ - --hash=sha256:2bd9d128ef93637a5d7a6af25363cf5dec3fa21cf80e68055aad627f280e8afa \ - --hash=sha256:2e1d8ca8611099001949d1cdfaefc510cf0f212484fe7c565f735b68c78c3c95 \ - --hash=sha256:2ef7fedc7a6ecbe99969cd09632516738a97eeb8bd7258bf8a0f23114c057dab \ - --hash=sha256:2f7fdd9b6e6c529d6a2501a2d36b240109e78a8ceaef5687cfcfa2bbe671d297 \ - --hash=sha256:30f445ae60aad5e1f8bdbb3108e39f6fbc09f4ea16c815c66578878325f8f15a \ - --hash=sha256:31215157227939b4fb3d740cd23fe27be0439afef67b785a1eb78a3ae69cba9e \ - --hash=sha256:34315ff4fc374b285ad7f4a0bf7dcbfe769e1b104230d40f49f700d4ab6bbd84 \ - --hash=sha256:3516bbb8d42169de9e61b8520cbeeeb716f12f4ecfe3fd30a9919aa16c806ca8 \ - --hash=sha256:3778fd7d7cd04ae8f54651f4a7a0bd6e39a0cf20f801720a4c21d80e9b7ad6b0 \ - --hash=sha256:39f5068d35621da2881271e5c3205125cc456f54e9030d3f723288c873a71bf9 \ - --hash=sha256:404a1e552cf5b675a87f0651f8b79f5f1e6fd100ee88dc612f89aa16abd4486f \ - --hash=sha256:419a9d91bd238052642a51938af8ac05da5b3343becde08d5cdeab9046df9ee1 \ - --hash=sha256:423fb7e748a08f854a08a222b983f4df1912b1daedce51a72bd24fe8f26a1843 \ - --hash=sha256:4482481cb0572180b6fd976a4d5c72a30263e98564da68b86ec91f0fe35e8565 \ - --hash=sha256:461598cd852bfa5a61b09cae2b1c02e2efcd166ee5516e243d540ac24bfa68a7 \ - --hash=sha256:47955475ac79cc504ef2704b192364e51d0d473ad452caedd0002605f780101c \ - --hash=sha256:48696db7f18afb80a068821504296eb0787d9ce239b91ca15059d1d3eaacf13b \ - --hash=sha256:4be9f4830ba8741527693848403e2c457c16e499100963ec711b1c6f2049b7c7 \ - --hash=sha256:4d1d02209e06550bdaef34af58e041ad71b88e624f5d825519da3a3308e22687 \ - --hash=sha256:4f41da960b196ea355357285ad1316a00099f22d0929fe168343b99b254729c9 \ - --hash=sha256:517ad0e93394ac532745129ceabdf2696b609ec9f87863d337140317ebce1c14 \ - --hash=sha256:51fb3c322c81d20567019778cb5a4a6f2dc1c200b886bc0d636238e364848c89 \ - --hash=sha256:5273b9f0b5835ff0350c0828faea623c68bfa65b792720c453e22b25cc72930f \ - --hash=sha256:530d548084c4a9f7a16ed4a294d459b4f229db50df689bfe92027452452943a0 \ - --hash=sha256:530e8cebeea0d76bdcf93357aa5e41336f48c3dc709ac52da2bb167c5b8271d9 \ - --hash=sha256:54fae94be3d75f3e573c9a1b5402dc593de19377013c9a0e4285e3d402dd3a2a \ - --hash=sha256:572d7c822caf521f0525ba1bce1a622a0b85cf47ffbdae6c9c19e3b5ac3c4389 \ - --hash=sha256:58c948d0d086229efc484fe2f30c2d382c86720f55cd9bc33591774348ad44e0 \ - --hash=sha256:5d11595abf8dd942a77883a39d81433739b287b6aa71620f15164f8096221b30 \ - --hash=sha256:5f8ddd609f9e1af8c7bd6e2aca279c931aefecd148a14402d4e368f3171769fd \ - --hash=sha256:5feb91325bbceade6afab43eb3b508c63ee53579fe896c77137ded51c6b6958e \ - --hash=sha256:60c74963d8350241a79cb8feea80e54d518f72c26db618862a8f53e5023deaf9 \ - --hash=sha256:613f19aa6e082cf96e17e3ffd89383343d0d589abda756b7764cf78361fd41dc \ - --hash=sha256:659a1e1b500fac8f2779dd9e1570464e012f43e580371470b45277a27baa7532 \ - --hash=sha256:695f5c2823691a25f17bc5d5ffe79fa90972cc34b002ac6c843bb8a1720e950d \ - --hash=sha256:69dd852c2f0ad631b8b60cfbe25a28c0058a894de5abb566619c205ce0550eae \ - --hash=sha256:6cceb5473417d28edd20c6c984ab6fee6c6267d38d906823ebfe20b03d607dc2 \ - --hash=sha256:71be7e0e01753a89cf024abf7ecb6bca2c81738ead80d43004d9b5e3f1244e64 \ - --hash=sha256:74119174722c4349af9708993118581686f343adc1c8c9c007d59be90d077f3f \ - --hash=sha256:74a2e659c7ecbc73562e2a15e05039f1e22c75b7c7618b4b574a3ea9118d1557 \ - --hash=sha256:7504e9b7dc05f99a9bbb4525c67a2c155073b44d720470a148b34166a69c054e \ - --hash=sha256:79090741d842f564b1b2827c0b82d846405b744d31e84f18d7a7b41c20e473ff \ - --hash=sha256:7a6967aaf043bceabab5412ed6bd6bd26603dae84d5cb75bf8d9a74a4959d398 \ - --hash=sha256:7bda6eebafd42133efdca535b04ccb338ab29467b3f7bf79569883676fc628db \ - --hash=sha256:7edbed096e4a4798710ed6bc75dcaa2a21b68b6c356553ac4823c3658d53743a \ - --hash=sha256:7f9019c9cb613f084481bd6a100b12e1547cf2efe362d873c2e31e4035a6fa43 \ - --hash=sha256:802168e03fba8bbc5ce0d866d589e4b1ca751d06edee69f7f3a19c5a9fe6b597 \ - --hash=sha256:80d0a5615143c0b3225e5e3ef22c8d5d51f3f72ce0ea6fb84c943546c7b25b6c \ - --hash=sha256:82060f995ab5003a2d6e0f4ad29065b7672b6593c8c63559beefe5b443242c3e \ - --hash=sha256:836ab36280f21fc1a03c99cd05c6b7af70d2697e374c7af0b61ed271401a72a2 \ - --hash=sha256:8761ac29b6c81574724322a554605608a9960769ea83d2c73e396f3df896ad54 \ - --hash=sha256:87725cfb1a4f1f8c2fc9890ae2f42094120f4b44db9360be5d99a4c6b0e03a9e \ - --hash=sha256:899d28f422116b08be5118ef350c292b36fc15ec2daeb9ea987c89281c7bb5c4 \ - --hash=sha256:8bc5f0687d796c05b1e28ab0d38a50e6309906ee09375dd3aff6a9c09dd6e8f4 \ - --hash=sha256:8bea55c4eef25b0b19a0337dc4e3f9a15b00d569c77211fa8cde38684f234fb7 \ - --hash=sha256:8e5a94886bedca0f9b78fecd6afb6629142fd2605aa70a125d49f4edc6037ee6 \ - --hash=sha256:90ca27cd8da8118b18a52d5f547859cc1f8354a00cd1e8e5120df3e30d6279e5 \ - --hash=sha256:92734d4d8d187a354a556626c221cd1a892a4e0802ccb2af432a1d85ec012194 \ - --hash=sha256:947cf925bc916d90adba35a64c82aace04fa39b46b52d4630ece166655905a69 \ - --hash=sha256:95b52c68d64c1878818687a473a10547b3292e82b6f6fe483808fb1468e2f52f \ - --hash=sha256:97d0235baafca5f2b09cf332cc275f021e694e8362c6bb9c96fc9a0eb74fc316 \ - --hash=sha256:9ca4c0b502ab399ef89248a2c84c54954f77a070f28e546a85e91da627d1301e \ - --hash=sha256:9cc4fc6c196d6a8b76629a70ddfcd4635a6898756e2d9cac5565cf0654605d73 \ - --hash=sha256:9cc6e6d9e571d2f863fa77700701dae73ed5f78881efc8b3f9a4398772ff53e8 \ - --hash=sha256:a056d1ad2633548ca18ffa2f85c202cfb48b68615129143915b8dc72a806a923 \ - --hash=sha256:a26611d9987b230566f24a0a125f17fe0de6a6aff9f25c9f564aaa2721a5fb88 \ - --hash=sha256:a4474d924a47185a06411e0064b803c68be044be2d60e50e8bddcc2649957c1f \ - --hash=sha256:a4ea868bc28109052790eb2b52a9ab33f3aa7adc02f96673526ff47419490e21 \ - --hash=sha256:a9e68c9d88823b274cf1e72f28cb5dc89c990edf430b0bfd3e2fb0785bfeabf4 \ - --hash=sha256:aa9cccf4a44b9b62d8ba8b4dd06c649ba683e4bf04eea606d2e94cfc2d6ff4d6 \ - --hash=sha256:ab30e5e3e706e3063bc6de96b118688cb10396b70bb9864a430f67df98c61ecc \ - --hash=sha256:ac2393c73378fea4e52aa56285a3d64be50f1a12395afef9cce47772f60334c2 \ - --hash=sha256:ad8faf8df23f0378c6d527d8b0b15ea4a2e23c89376877c598c4870d1b2c7866 \ - --hash=sha256:b35b200d6a71b9839a46b9b7fff66b6638bb52fc9658aa58796b0326595d3021 \ - --hash=sha256:b3694e3f87f8ac7ce279d4355645b3c878d24d1424581b46282f24b92f5a4ae2 \ - --hash=sha256:b4ff1d35e8c5bd078be89349b6f3a845128e685e751b6ea1169cf2160b344c4d \ - --hash=sha256:bbc8c8650c6e51041ad1be191742b8b421d05bbd3410f43fa2a00c8db87678e8 \ - --hash=sha256:bc72863f4d9aba2e8fd9085e63548a324ba706d2ea2c83b260da08a59b9482de \ - --hash=sha256:bf625105bb9eef28a56a943fec8c8a98aeb80e7d7db99bd3c388137e6eb2d237 \ - --hash=sha256:c2274ca724536f173122f36c98ce188fd24ce3dad886ec2b7af859518ce008a4 \ - --hash=sha256:c45a03a4c69820a399f1dda9e1d8fbf3562eda46e7720458180302021b08f778 \ - --hash=sha256:c8ae56368f8cc97c7e40a7ee18e1cedaf8e780cd8bc5ed5ac8b81f238614facb \ - --hash=sha256:c907cdc8109f6c619e6254212e794d6548373cc40e1ec75e6e3823d9135d29cc \ - --hash=sha256:ca0276464d148c72defa8bb4390cce01b4a0e425f3b50d1435aa6d7a18107602 \ - --hash=sha256:cd5e2801c89992ed8c0a3f0293ae83c159a60d9a5d685005383ef4caca77f2c4 \ - --hash=sha256:d08ec48f0a1c48d75d0356cea971921848fb620fdeba805b28f937e90691209f \ - --hash=sha256:d1a2ee9c1499fc8f86f4521f27a973c914b211ffa87322f4ee33bb35392da2c5 \ - --hash=sha256:d5f5d1e9def3405f60e3ca8232d56f35c98fb7bf581efcc60051ebf53cb8b611 \ - --hash=sha256:d60377dce4511655582e300dc1e5a5f24ba0cb229005a1d5c8d0cb72bb758ab8 \ - --hash=sha256:d73beaac5e90173ac3deb9928a74763a6d230f494e4bfb422c217a0ad8e629bf \ - --hash=sha256:d7de2637729c67d67cf87614b566626057e95c303bc0a55ffe391f5205e7003d \ - --hash=sha256:dad6e0f2e481fffdcf776d10ebee25e0ef89f16d691f1e5dee4b586375fdc64b \ - --hash=sha256:dda86aba335c902b6149a02a55b38e96287157e609200811837678214ba2b1db \ - --hash=sha256:df01808ee470038c3f8dc4f48620df7225c49c2d6639e38f96e6d6ac6e6f7b0e \ - --hash=sha256:e1f6e2f00a6b8edb562826e4632e26d063ac10307e80f7461f7de3ad8ef3f077 \ - --hash=sha256:e25369dc110d58ddf29b949377a93e0716d72a24f62bad72b2b39f155949c1fd \ - --hash=sha256:e3c701e954abf6fc03a49f7c579cc80c2c6cc52525340ca3186c41d3f33482ef \ - --hash=sha256:e5bcc1a1ae744e0bb59641171ae53743760130600da8db48cbb6e4918e186e4e \ - --hash=sha256:e68c14b04827dd76dcbd1aeea9e604e3e4b78322d8faf2f8132c7138efa340a8 \ - --hash=sha256:e8aeb10fcbe92767f0fa69ad5a72deca50d0dca07fbde97848997d778a50c9fe \ - --hash=sha256:e985a16ff513596f217cee86c21371b8cd011c0f6f056d0920aa2d926c544058 \ - --hash=sha256:ecbbd45615a6885fe3240eb9db73b9e62518b611850fdf8ab08bd56de7ad2b17 \ - --hash=sha256:ee4ec14bc1680d6b0afab9aea2ef27e26d2024f18b24a2d7155a52b60da7e833 \ - --hash=sha256:ef5960d965e67165d75b7c7ffc60a83ec5abfc5c11b764ec13ea54fbef8b4421 \ - --hash=sha256:f0cdaecd4c953bfae0b6bb64910aaaca5a424ad9c72d85cb88417bb9814f7550 \ - --hash=sha256:f1ce721c8a7dfec21fcbdfe04e8f68174183cf4e8188e0645e92aa23985c57ff \ - --hash=sha256:f50498891691e0864dc3da965f340fada0771f6142a378083dc4608f4ea513e2 \ - --hash=sha256:f5ea69428fa1b49573eef0cc44a1d43bebd45ad0c611eb7d7eac760c7ae771bc \ - --hash=sha256:f61aa92e4aad0be58eb6eb4e0c21acf32cf8065f4b2cae5665da756c4ceef982 \ - --hash=sha256:f6e4333fb15c83f7d1482a76d45a0818897b3d33f00efd215528ff7c51b8e35d \ - --hash=sha256:f820f24b09e3e779fe84c3c456cb4108a7aa639b0d1f02c28046e11bfcd088ed \ - --hash=sha256:f98059e4fcd3e3e4e2d632b7cf81c2faae96c43c60b569e9c621468082f1d104 \ - --hash=sha256:fcce033e4021347d80ed9c66dcf1e7b1546319834b74445f561d2e2221de5659 +charset-normalizer==3.4.7 \ + --hash=sha256:007d05ec7321d12a40227aae9e2bc6dca73f3cb21058999a1df9e193555a9dcc \ + --hash=sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c \ + --hash=sha256:07d9e39b01743c3717745f4c530a6349eadbfa043c7577eef86c502c15df2c67 \ + --hash=sha256:08e721811161356f97b4059a9ba7bafb23ea5ee2255402c42881c214e173c6b4 \ + --hash=sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0 \ + --hash=sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c \ + --hash=sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5 \ + --hash=sha256:12a6fff75f6bc66711b73a2f0addfc4c8c15a20e805146a02d147a318962c444 \ + --hash=sha256:12d8baf840cc7889b37c7c770f478adea7adce3dcb3944d02ec87508e2dcf153 \ + --hash=sha256:14265bfe1f09498b9d8ec91e9ec9fa52775edf90fcbde092b25f4a33d444fea9 \ + --hash=sha256:16d971e29578a5e97d7117866d15889a4a07befe0e87e703ed63cd90cb348c01 \ + --hash=sha256:177a0ba5f0211d488e295aaf82707237e331c24788d8d76c96c5a41594723217 \ + --hash=sha256:1a87ca9d5df6fe460483d9a5bbf2b18f620cbed41b432e2bddb686228282d10b \ + --hash=sha256:1c2a768fdd44ee4a9339a9b0b130049139b8ce3c01d2ce09f67f5a68048d477c \ + --hash=sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a \ + --hash=sha256:1dc8b0ea451d6e69735094606991f32867807881400f808a106ee1d963c46a83 \ + --hash=sha256:1efde3cae86c8c273f1eb3b287be7d8499420cf2fe7585c41d370d3e790054a5 \ + --hash=sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7 \ + --hash=sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb \ + --hash=sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c \ + --hash=sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1 \ + --hash=sha256:2cd4a60d0e2fb04537162c62bbbb4182f53541fe0ede35cdf270a1c1e723cc42 \ + --hash=sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab \ + --hash=sha256:2fe249cb4651fd12605b7288b24751d8bfd46d35f12a20b1ba33dea122e690df \ + --hash=sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e \ + --hash=sha256:320ade88cfb846b8cd6b4ddf5ee9e80ee0c1f52401f2456b84ae1ae6a1a5f207 \ + --hash=sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18 \ + --hash=sha256:36836d6ff945a00b88ba1e4572d721e60b5b8c98c155d465f56ad19d68f23734 \ + --hash=sha256:38c0109396c4cfc574d502df99742a45c72c08eff0a36158b6f04000043dbf38 \ + --hash=sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110 \ + --hash=sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18 \ + --hash=sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44 \ + --hash=sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d \ + --hash=sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48 \ + --hash=sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e \ + --hash=sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5 \ + --hash=sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d \ + --hash=sha256:4e5163c14bffd570ef2affbfdd77bba66383890797df43dc8b4cc7d6f500bf53 \ + --hash=sha256:511ef87c8aec0783e08ac18565a16d435372bc1ac25a91e6ac7f5ef2b0bff790 \ + --hash=sha256:532bc9bf33a68613fd7d65e4b1c71a6a38d7d42604ecf239c77392e9b4e8998c \ + --hash=sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b \ + --hash=sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116 \ + --hash=sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d \ + --hash=sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10 \ + --hash=sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6 \ + --hash=sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2 \ + --hash=sha256:6370e8686f662e6a3941ee48ed4742317cafbe5707e36406e9df792cdb535776 \ + --hash=sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a \ + --hash=sha256:65bcd23054beab4d166035cabbc868a09c1a49d1efe458fe8e4361215df40265 \ + --hash=sha256:66671f93accb62ed07da56613636f3641f1a12c13046ce91ffc923721f23c008 \ + --hash=sha256:6696b7688f54f5af4462118f0bfa7c1621eeb87154f77fa04b9295ce7a8f2943 \ + --hash=sha256:6785f414ae0f3c733c437e0f3929197934f526d19dfaa75e18fdb4f94c6fb374 \ + --hash=sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246 \ + --hash=sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e \ + --hash=sha256:6e0d51f618228538a3e8f46bd246f87a6cd030565e015803691603f55e12afb5 \ + --hash=sha256:6ed74185b2db44f41ef35fd1617c5888e59792da9bbc9190d6c7300617182616 \ + --hash=sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15 \ + --hash=sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41 \ + --hash=sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960 \ + --hash=sha256:750e02e074872a3fad7f233b47734166440af3cdea0add3e95163110816d6752 \ + --hash=sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e \ + --hash=sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72 \ + --hash=sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7 \ + --hash=sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8 \ + --hash=sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b \ + --hash=sha256:813c0e0132266c08eb87469a642cb30aaff57c5f426255419572aaeceeaa7bf4 \ + --hash=sha256:82b271f5137d07749f7bf32f70b17ab6eaabedd297e75dce75081a24f76eb545 \ + --hash=sha256:84c018e49c3bf790f9c2771c45e9313a08c2c2a6342b162cd650258b57817706 \ + --hash=sha256:8751d2787c9131302398b11e6c8068053dcb55d5a8964e114b6e196cf16cb366 \ + --hash=sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb \ + --hash=sha256:87fad7d9ba98c86bcb41b2dc8dbb326619be2562af1f8ff50776a39e55721c5a \ + --hash=sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e \ + --hash=sha256:8e385e4267ab76874ae30db04c627faaaf0b509e1ccc11a95b3fc3e83f855c00 \ + --hash=sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f \ + --hash=sha256:94e1885b270625a9a828c9793b4d52a64445299baa1fea5a173bf1d3dd9a1a5a \ + --hash=sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1 \ + --hash=sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66 \ + --hash=sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356 \ + --hash=sha256:a6c5863edfbe888d9eff9c8b8087354e27618d9da76425c119293f11712a6319 \ + --hash=sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4 \ + --hash=sha256:adb2597b428735679446b46c8badf467b4ca5f5056aae4d51a19f9570301b1ad \ + --hash=sha256:ae196f021b5e7c78e918242d217db021ed2a6ace2bc6ae94c0fc596221c7f58d \ + --hash=sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5 \ + --hash=sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7 \ + --hash=sha256:aef65cd602a6d0e0ff6f9930fcb1c8fec60dd2cfcb6facaf4bdb0e5873042db0 \ + --hash=sha256:af21eb4409a119e365397b2adbaca4c9ccab56543a65d5dbd9f920d6ac29f686 \ + --hash=sha256:b14b2d9dac08e28bb8046a1a0434b1750eb221c8f5b87a68f4fa11a6f97b5e34 \ + --hash=sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49 \ + --hash=sha256:bb8cc7534f51d9a017b93e3e85b260924f909601c3df002bcdb58ddb4dc41a5c \ + --hash=sha256:bc17a677b21b3502a21f66a8cc64f5bfad4df8a0b8434d661666f8ce90ac3af1 \ + --hash=sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e \ + --hash=sha256:bd9b23791fe793e4968dba0c447e12f78e425c59fc0e3b97f6450f4781f3ee60 \ + --hash=sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0 \ + --hash=sha256:c0f081d69a6e58272819b70288d3221a6ee64b98df852631c80f293514d3b274 \ + --hash=sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d \ + --hash=sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0 \ + --hash=sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae \ + --hash=sha256:c593052c465475e64bbfe5dbd81680f64a67fdc752c56d7a0ae205dc8aeefe0f \ + --hash=sha256:cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d \ + --hash=sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe \ + --hash=sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3 \ + --hash=sha256:cf29836da5119f3c8a8a70667b0ef5fdca3bb12f80fd06487cfa575b3909b393 \ + --hash=sha256:d4a48e5b3c2a489fae013b7589308a40146ee081f6f509e047e0e096084ceca1 \ + --hash=sha256:d560742f3c0d62afaccf9f41fe485ed69bd7661a241f86a3ef0f0fb8b1a397af \ + --hash=sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44 \ + --hash=sha256:d61f00a0869d77422d9b2aba989e2d24afa6ffd552af442e0e58de4f35ea6d00 \ + --hash=sha256:d635aab80466bc95771bb78d5370e74d36d1fe31467b6b29b8b57b2a3cd7d22c \ + --hash=sha256:dca4bbc466a95ba9c0234ef56d7dd9509f63da22274589ebd4ed7f1f4d4c54e3 \ + --hash=sha256:dd915403e231e6b1809fe9b6d9fc55cf8fb5e02765ac625d9cd623342a7905d7 \ + --hash=sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd \ + --hash=sha256:e060d01aec0a910bdccb8be71faf34e7799ce36950f8294c8bf612cba65a2c9e \ + --hash=sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b \ + --hash=sha256:e17b8d5d6a8c47c85e68ca8379def1303fd360c3e22093a807cd34a71cd082b8 \ + --hash=sha256:e5f4d355f0a2b1a31bc3edec6795b46324349c9cb25eed068049e4f472fb4259 \ + --hash=sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859 \ + --hash=sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46 \ + --hash=sha256:e80c8378d8f3d83cd3164da1ad2df9e37a666cdde7b1cb2298ed0b558064be30 \ + --hash=sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b \ + --hash=sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46 \ + --hash=sha256:ed065083d0898c9d5b4bbec7b026fd755ff7454e6e8b73a67f8c744b13986e24 \ + --hash=sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a \ + --hash=sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24 \ + --hash=sha256:f22dec1690b584cea26fade98b2435c132c1b5f68e39f5a0b7627cd7ae31f1dc \ + --hash=sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215 \ + --hash=sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063 \ + --hash=sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832 \ + --hash=sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6 \ + --hash=sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79 \ + --hash=sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464 # via requests cheroot==11.1.2 \ --hash=sha256:0f6c0ba05c00fbc869fb46b1de4ec2384e1d85418ae963d3bc10ae83b688dbfa \ --hash=sha256:bfb70c49663f63b0440f2b54dbc6b0d1650e56dfe4e2641f59b2c6f727b44aca # via xprof-nightly -cryptography==46.0.5 \ - --hash=sha256:02f547fce831f5096c9a567fd41bc12ca8f11df260959ecc7c3202555cc47a72 \ - --hash=sha256:039917b0dc418bb9f6edce8a906572d69e74bd330b0b3fea4f79dab7f8ddd235 \ - --hash=sha256:1abfdb89b41c3be0365328a410baa9df3ff8a9110fb75e7b52e66803ddabc9a9 \ - --hash=sha256:2ae6971afd6246710480e3f15824ed3029a60fc16991db250034efd0b9fb4356 \ - --hash=sha256:2b7a67c9cd56372f3249b39699f2ad479f6991e62ea15800973b956f4b73e257 \ - --hash=sha256:351695ada9ea9618b3500b490ad54c739860883df6c1f555e088eaf25b1bbaad \ - --hash=sha256:38946c54b16c885c72c4f59846be9743d699eee2b69b6988e0a00a01f46a61a4 \ - --hash=sha256:3b4995dc971c9fb83c25aa44cf45f02ba86f71ee600d81091c2f0cbae116b06c \ - --hash=sha256:3ce58ba46e1bc2aac4f7d9290223cead56743fa6ab94a5d53292ffaac6a91614 \ - --hash=sha256:3ee190460e2fbe447175cda91b88b84ae8322a104fc27766ad09428754a618ed \ - --hash=sha256:4108d4c09fbbf2789d0c926eb4152ae1760d5a2d97612b92d508d96c861e4d31 \ - --hash=sha256:420d0e909050490d04359e7fdb5ed7e667ca5c3c402b809ae2563d7e66a92229 \ - --hash=sha256:47fb8a66058b80e509c47118ef8a75d14c455e81ac369050f20ba0d23e77fee0 \ - --hash=sha256:4c3341037c136030cb46e4b1e17b7418ea4cbd9dd207e4a6f3b2b24e0d4ac731 \ - --hash=sha256:4d7e3d356b8cd4ea5aff04f129d5f66ebdc7b6f8eae802b93739ed520c47c79b \ - --hash=sha256:4d8ae8659ab18c65ced284993c2265910f6c9e650189d4e3f68445ef82a810e4 \ - --hash=sha256:4e817a8920bfbcff8940ecfd60f23d01836408242b30f1a708d93198393a80b4 \ - --hash=sha256:50bfb6925eff619c9c023b967d5b77a54e04256c4281b0e21336a130cd7fc263 \ - --hash=sha256:556e106ee01aa13484ce9b0239bca667be5004efb0aabbed28d353df86445595 \ - --hash=sha256:582f5fcd2afa31622f317f80426a027f30dc792e9c80ffee87b993200ea115f1 \ - --hash=sha256:5be7bf2fb40769e05739dd0046e7b26f9d4670badc7b032d6ce4db64dddc0678 \ - --hash=sha256:60ee7e19e95104d4c03871d7d7dfb3d22ef8a9b9c6778c94e1c8fcc8365afd48 \ - --hash=sha256:61aa400dce22cb001a98014f647dc21cda08f7915ceb95df0c9eaf84b4b6af76 \ - --hash=sha256:68f68d13f2e1cb95163fa3b4db4bf9a159a418f5f6e7242564fc75fcae667fd0 \ - --hash=sha256:7d1f30a86d2757199cb2d56e48cce14deddf1f9c95f1ef1b64ee91ea43fe2e18 \ - --hash=sha256:7d731d4b107030987fd61a7f8ab512b25b53cef8f233a97379ede116f30eb67d \ - --hash=sha256:803812e111e75d1aa73690d2facc295eaefd4439be1023fefc4995eaea2af90d \ - --hash=sha256:80a8d7bfdf38f87ca30a5391c0c9ce4ed2926918e017c29ddf643d0ed2778ea1 \ - --hash=sha256:8293f3dea7fc929ef7240796ba231413afa7b68ce38fd21da2995549f5961981 \ - --hash=sha256:8456928655f856c6e1533ff59d5be76578a7157224dbd9ce6872f25055ab9ab7 \ - --hash=sha256:890bcb4abd5a2d3f852196437129eb3667d62630333aacc13dfd470fad3aaa82 \ - --hash=sha256:94a76daa32eb78d61339aff7952ea819b1734b46f73646a07decb40e5b3448e2 \ - --hash=sha256:9f16fbdf4da055efb21c22d81b89f155f02ba420558db21288b3d0035bafd5f4 \ - --hash=sha256:a3d1fae9863299076f05cb8a778c467578262fae09f9dc0ee9b12eb4268ce663 \ - --hash=sha256:a3d507bb6a513ca96ba84443226af944b0f7f47dcc9a399d110cd6146481d24c \ - --hash=sha256:abace499247268e3757271b2f1e244b36b06f8515cf27c4d49468fc9eb16e93d \ - --hash=sha256:ba2a27ff02f48193fc4daeadf8ad2590516fa3d0adeeb34336b96f7fa64c1e3a \ - --hash=sha256:bc84e875994c3b445871ea7181d424588171efec3e185dced958dad9e001950a \ - --hash=sha256:bfd56bb4b37ed4f330b82402f6f435845a5f5648edf1ad497da51a8452d5d62d \ - --hash=sha256:c18ff11e86df2e28854939acde2d003f7984f721eba450b56a200ad90eeb0e6b \ - --hash=sha256:c3bcce8521d785d510b2aad26ae2c966092b7daa8f45dd8f44734a104dc0bc1a \ - --hash=sha256:c4143987a42a2397f2fc3b4d7e3a7d313fbe684f67ff443999e803dd75a76826 \ - --hash=sha256:c69fd885df7d089548a42d5ec05be26050ebcd2283d89b3d30676eb32ff87dee \ - --hash=sha256:ced80795227d70549a411a4ab66e8ce307899fad2220ce5ab2f296e687eacde9 \ - --hash=sha256:d66e421495fdb797610a08f43b05269e0a5ea7f5e652a89bfd5a7d3c1dee3648 \ - --hash=sha256:d861ee9e76ace6cf36a6a89b959ec08e7bc2493ee39d07ffe5acb23ef46d27da \ - --hash=sha256:e9251e3be159d1020c4030bd2e5f84d6a43fe54b6c19c12f51cde9542a2817b2 \ - --hash=sha256:f145bba11b878005c496e93e257c1e88f154d278d2638e6450d17e0f31e558d2 \ - --hash=sha256:fe346b143ff9685e40192a4960938545c699054ba11d4f9029f94751e3f71d87 - # via google-auth +cryptography==48.0.0 \ + --hash=sha256:0890f502ddf7d9c6426129c3f49f5c0a39278ed7cd6322c8755ffca6ee675a13 \ + --hash=sha256:0c558d2cdffd8f4bbb30fc7134c74d2ca9a476f830bb053074498fbc86f41ed6 \ + --hash=sha256:16cd65b9330583e4619939b3a3843eec1e6e789744bb01e7c7e2e62e33c239c8 \ + --hash=sha256:18349bbc56f4743c8b12dc32e2bccb2cf83ee8b69a3bba74ef8ae857e26b3d25 \ + --hash=sha256:1e2d54c8be6152856a36f0882ab231e70f8ec7f14e93cf87db8a2ed056bf160c \ + --hash=sha256:22a5cb272895dce158b2cacdfdc3debd299019659f42947dbdac6f32d68fe832 \ + --hash=sha256:27241b1dc9962e056062a8eef1991d02c3a24569c95975bd2322a8a52c6e5e12 \ + --hash=sha256:2b4d59804e8408e2fea7d1fbaf218e5ec984325221db76e6a241a9abd6cdd95c \ + --hash=sha256:2eb992bbd4661238c5a397594c83f5b4dc2bc5b848c365c8f991b6780efcc5c7 \ + --hash=sha256:369a6348999f94bbd53435c894377b20ab95f25a9065c283570e70150d8abc3c \ + --hash=sha256:3cb07a3ed6431663cd321ea8a000a1314c74211f823e4177fefa2255e057d1ec \ + --hash=sha256:40ba1f85eaa6959837b1d51c9767e230e14612eea4ef110ee8854ada22da1bf5 \ + --hash=sha256:4defde8685ae324a9eb9d818717e93b4638ef67070ac9bc15b8ca85f63048355 \ + --hash=sha256:55b7718303bf06a5753dcdccf2f3945cf18ad7bffde41b61226e4db31ab89a9c \ + --hash=sha256:561215ea3879cb1cbbf272867e2efda62476f240fb58c64de6b393ae19246741 \ + --hash=sha256:58d00498e8933e4a194f3076aee1b4a97dfec1a6da444535755822fe5d8b0b86 \ + --hash=sha256:59baa2cb386c4f0b9905bd6eb4c2a79a69a128408fd31d32ca4d7102d4156321 \ + --hash=sha256:5a5ed8fde7a1d09376ca0b40e68cd59c69fe23b1f9768bd5824f54681626032a \ + --hash=sha256:5b012212e08b8dd5edc78ef54da83dd9892fd9105323b3993eff6bea65dc21d7 \ + --hash=sha256:5c3932f4436d1cccb036cb0eaef46e6e2db91035166f1ad6505c3c9d5a635920 \ + --hash=sha256:614d0949f4790582d2cc25553abd09dd723025f0c0e7c67376a1d77196743d6e \ + --hash=sha256:76341972e1eff8b4bea859f09c0d3e64b96ce931b084f9b9b7db8ef364c30eff \ + --hash=sha256:77a2ccbbe917f6710e05ba9adaa25fb5075620bf3ea6fb751997875aff4ae4bd \ + --hash=sha256:7995ef305d7165c3f11ae07f2517e5a4f1d5c18da1376a0a9ed496336b69e5f3 \ + --hash=sha256:7ce4bfae76319a532a2dc68f82cc32f5676ee792a983187dac07183690e5c66f \ + --hash=sha256:7e8eac43dfca5c4cccc6dad9a80504436fca53bb9bc3100a2386d730fbe6b602 \ + --hash=sha256:84cf79f0dc8b36ac5da873481716e87aef31fcfa0444f9e1d8b4b2cece142855 \ + --hash=sha256:8c7378637d7d88016fa6791c159f698b3d3eed28ebf844ac36b9dc04a14dae18 \ + --hash=sha256:8cd666227ef7af430aa5914a9910e0ddd703e75f039cef0825cd0da71b6b711a \ + --hash=sha256:906cbf0670286c6e0044156bc7d4af9cbb0ef6db9f73e52c3ec56ba6bdde5336 \ + --hash=sha256:9071196d81abc88b3516ac8cdfad32e2b66dd4a5393a8e68a961e9161ddc6239 \ + --hash=sha256:9249e3cd978541d665967ac2cb2787fd6a62bddf1e75b3e347a594d7dacf4f74 \ + --hash=sha256:984a20b0f62a26f48a3396c72e4bc34c66e356d356bf370053066b3b6d54634a \ + --hash=sha256:9be5aafa5736574f8f15f262adc81b2a9869e2cfe9014d52a44633905b40d52c \ + --hash=sha256:9c459db21422be75e2809370b829a87eb37f74cd785fc4aa9ea1e5f43b47cda4 \ + --hash=sha256:9ccdac7d40688ecb5a3b4a604b8a88c8002e3442d6c60aead1db2a89a041560c \ + --hash=sha256:a0e692c683f4df67815a2d258b324e66f4738bd7a96a218c826dce4f4bd05d8f \ + --hash=sha256:a5da777e32ffed6f85a7b2b3f7c5cbc88c146bfcd0a1d7baf5fcc6c52ee35dd4 \ + --hash=sha256:a64697c641c7b1b2178e573cbc31c7c6684cd56883a478d75143dbb7118036db \ + --hash=sha256:ad64688338ed4bc1a6618076ba75fd7194a5f1797ac60b47afe926285adb3166 \ + --hash=sha256:bd72e68b06bb1e96913f97dd4901119bc17f39d4586a5adf2d3e47bc2b9d58b5 \ + --hash=sha256:c17dfe85494deaeddc5ce251aebd1d60bbe6afc8b62071bb0b469431a000124f \ + --hash=sha256:c18684a7f0cc9a3cb60328f496b8e3372def7c5d2df39ac267878b05565aaaae \ + --hash=sha256:cc90c0b39b2e3c65ef52c804b72e3c58f8a04ab2a1871272798e5f9572c17d20 \ + --hash=sha256:db63bf618e5dea46c07de12e900fe1cdd2541e6dc9dbae772a70b7d4d4765f6a \ + --hash=sha256:ea8990436d914540a40ab24b6a77c0969695ed52f4a4874c5137ccf7045a7057 \ + --hash=sha256:ecde28a596bead48b0cfd2a1b4416c3d43074c2d785e3a398d7ec1fc4d0f7fbb \ + --hash=sha256:f5333311663ea94f75dd408665686aaf426563556bb5283554a3539177e03b8c \ + --hash=sha256:fdfef35d751d510fcef5252703621574364fec16418c4a1e5e1055248401054b + # via + # google-auth + # pyopenssl decorator==5.2.1 \ --hash=sha256:65f266143752f734b0a7cc83c46f4618af75b8c5911b00ccb61d0ac9b6da0360 \ --hash=sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a @@ -552,27 +554,27 @@ frozenlist==1.8.0 \ # via # aiohttp # aiosignal -fsspec==2026.2.0 \ - --hash=sha256:6544e34b16869f5aacd5b90bdf1a71acb37792ea3ddf6125ee69a22a53fb8bff \ - --hash=sha256:98de475b5cb3bd66bedd5c4679e87b4fdfe1a3bf4d707b151b3c07e58c9a2437 +fsspec==2026.4.0 \ + --hash=sha256:11ef7bb35dab8a394fde6e608221d5cf3e8499401c249bebaeaad760a1a8dec2 \ + --hash=sha256:301d8ac70ae90ef3ad05dcf94d6c3754a097f9b5fe4667d2787aa359ec7df7e4 # via # etils # gcsfs # xprof-nightly -gcsfs==2026.2.0 \ - --hash=sha256:407feaa2af0de81ebce44ea7e6f68598a3753e5e42257b61d6a9f8c0d6d4754e \ - --hash=sha256:d58a885d9e9c6227742b86da419c7a458e1f33c1de016e826ea2909f6338ed84 +gcsfs==2026.4.0 \ + --hash=sha256:d9e838834d8cce6cb623c6a6a5fad66a4d122dc5c609d4b1c1977b55f759dcc5 \ + --hash=sha256:f3d80dd1c98737798bc84472e6ff9c59873bd17a26e170d46ce90b3118db4390 # via xprof-nightly -google-api-core[grpc]==2.30.0 \ - --hash=sha256:02edfa9fab31e17fc0befb5f161b3bf93c9096d99aed584625f38065c511ad9b \ - --hash=sha256:80be49ee937ff9aba0fd79a6eddfde35fe658b9953ab9b79c57dd7061afa8df5 +google-api-core[grpc]==2.30.3 \ + --hash=sha256:a85761ba72c444dad5d611c2220633480b2b6be2521eca69cca2dbb3ffd6bfe8 \ + --hash=sha256:e601a37f148585319b26db36e219df68c5d07b6382cff2d580e83404e44d641b # via # google-cloud-core # google-cloud-storage # google-cloud-storage-control -google-auth==2.49.1 \ - --hash=sha256:16d40da1c3c5a0533f57d268fe72e0ebb0ae1cc3b567024122651c045d879b64 \ - --hash=sha256:195ebe3dca18eddd1b3db5edc5189b76c13e96f29e73043b923ebcf3f1a860f7 +google-auth==2.50.0 \ + --hash=sha256:04382175e28b94f49694977f0a792688b59a668def1499e9d8de996dc9ce5b15 \ + --hash=sha256:f35eafb191195328e8ce10a7883970877e7aeb49c2bfaa54aa0e394316d353d0 # via # gcsfs # google-api-core @@ -580,21 +582,21 @@ google-auth==2.49.1 \ # google-cloud-core # google-cloud-storage # google-cloud-storage-control -google-auth-oauthlib==1.3.0 \ - --hash=sha256:386b3fb85cf4a5b819c6ad23e3128d975216b4cac76324de1d90b128aaf38f29 \ - --hash=sha256:cd39e807ac7229d6b8b9c1e297321d36fcc8a9e4857dff4301870985df51a528 +google-auth-oauthlib==1.3.1 \ + --hash=sha256:14c22c7b3dd3d06dbe44264144409039465effdd1eef94f7ce3710e486cc4bfa \ + --hash=sha256:1a139ef23f1318756805b0e95f655c238bffd29655329a2978218248da4ee7f8 # via gcsfs -google-cloud-core==2.5.0 \ - --hash=sha256:67d977b41ae6c7211ee830c7912e41003ea8194bff15ae7d72fd6f51e57acabc \ - --hash=sha256:7c1b7ef5c92311717bd05301aa1a91ffbc565673d3b0b4163a52d8413a186963 +google-cloud-core==2.5.1 \ + --hash=sha256:3dc94bdec9d05a31d9f355045ed0f369fbc0d8c665076c734f065d729800f811 \ + --hash=sha256:ea62cdf502c20e3e14be8a32c05ed02113d7bef454e40ff3fab6fe1ec9f1f4e7 # via google-cloud-storage -google-cloud-storage==3.9.0 \ - --hash=sha256:2dce75a9e8b3387078cbbdad44757d410ecdb916101f8ba308abf202b6968066 \ - --hash=sha256:f2d8ca7db2f652be757e92573b2196e10fbc09649b5c016f8b422ad593c641cc +google-cloud-storage==3.10.1 \ + --hash=sha256:97db9aa4460727982040edd2bd13ff3d5e2260b5331ad22895802da1fc2a5286 \ + --hash=sha256:a72f656759b7b99bda700f901adcb3425a828d4a29f911bc26b3ea79c5b1217f # via gcsfs -google-cloud-storage-control==1.10.0 \ - --hash=sha256:2bcbfa4ca6530d25a5baa8dbe80caf0eeabe4c6804798f4f107279719c316bdb \ - --hash=sha256:81d9dc6b50106836733adca868501f879f0d7a1c41503d887a1a1b9b9ddbf508 +google-cloud-storage-control==1.11.0 \ + --hash=sha256:98d2a3be8ab124ae09c33c1b8e4aadc68597c171c60c33e62666bcc74493f8ee \ + --hash=sha256:e97a0f3c99bc5ec5a2a770298ca9e8d5feeaa41bb55f2f2d7b36df2b817ee950 # via gcsfs google-crc32c==1.8.0 \ --hash=sha256:014a7e68d623e9a4222d663931febc3033c5c7c9730785727de2a81f87d5bab8 \ @@ -633,100 +635,100 @@ google-crc32c==1.8.0 \ # via # google-cloud-storage # google-resumable-media -google-resumable-media==2.8.0 \ - --hash=sha256:dd14a116af303845a8d932ddae161a26e86cc229645bc98b39f026f9b1717582 \ - --hash=sha256:f1157ed8b46994d60a1bc432544db62352043113684d4e030ee02e77ebe9a1ae +google-resumable-media==2.8.2 \ + --hash=sha256:82b6d8ccd11765268cdd2a2123f417ec806b8eef3000a9a38dfe3033da5fb220 \ + --hash=sha256:f3354a182ebd193ae3f42e3ef95e6c9b10f128320de23ac7637236713b1acd70 # via google-cloud-storage -googleapis-common-protos[grpc]==1.73.0 \ - --hash=sha256:778d07cd4fbeff84c6f7c72102f0daf98fa2bfd3fa8bea426edc545588da0b5a \ - --hash=sha256:dfdaaa2e860f242046be561e6d6cb5c5f1541ae02cfbcb034371aadb2942b4e8 +googleapis-common-protos[grpc]==1.74.0 \ + --hash=sha256:57971e4eeeba6aad1163c1f0fc88543f965bb49129b8bb55b2b7b26ecab084f1 \ + --hash=sha256:702216f78610bb510e3f12ac3cafd281b7ac45cc5d86e90ad87e4d301a3426b5 # via # google-api-core # grpc-google-iam-v1 # grpcio-status -grpc-google-iam-v1==0.14.3 \ - --hash=sha256:7a7f697e017a067206a3dfef44e4c634a34d3dee135fe7d7a4613fe3e59217e6 \ - --hash=sha256:879ac4ef33136c5491a6300e27575a9ec760f6cdf9a2518798c1b8977a5dc389 +grpc-google-iam-v1==0.14.4 \ + --hash=sha256:392b3796947ed6334e61171d9ab06bf7eb357f554e5fc7556ad7aab6d0e17038 \ + --hash=sha256:412facc320fcbd94034b4df3d557662051d4d8adfa86e0ddb4dca70a3f739964 # via google-cloud-storage-control -grpcio==1.78.0 \ - --hash=sha256:082653eecbdf290e6e3e2c276ab2c54b9e7c299e07f4221872380312d8cf395e \ - --hash=sha256:10a9a644b5dd5aec3b82b5b0b90d41c0fa94c85ef42cb42cf78a23291ddb5e7d \ - --hash=sha256:12a771591ae40bc65ba67048fa52ef4f0e6db8279e595fd349f9dfddeef571f9 \ - --hash=sha256:185dea0d5260cbb2d224c507bf2a5444d5abbb1fa3594c1ed7e4c709d5eb8383 \ - --hash=sha256:1afa62af6e23f88629f2b29ec9e52ec7c65a7176c1e0a83292b93c76ca882558 \ - --hash=sha256:2045397e63a7a0ee7957c25f7dbb36ddc110e0cfb418403d110c0a7a68a844e9 \ - --hash=sha256:207db540302c884b8848036b80db352a832b99dfdf41db1eb554c2c2c7800f65 \ - --hash=sha256:271c73e6e5676afe4fc52907686670c7cea22ab2310b76a59b678403ed40d670 \ - --hash=sha256:2777b783f6c13b92bd7b716667452c329eefd646bfb3f2e9dabea2e05dbd34f6 \ - --hash=sha256:2bf5e2e163b356978b23652c4818ce4759d40f4712ee9ec5a83c4be6f8c23a3a \ - --hash=sha256:35eb275bf1751d2ffbd8f57cdbc46058e857cf3971041521b78b7db94bdaf127 \ - --hash=sha256:391e93548644e6b2726f1bb84ed60048d4bcc424ce5e4af0843d28ca0b754fec \ - --hash=sha256:3c586ac70e855c721bda8f548d38c3ca66ac791dc49b66a8281a1f99db85e452 \ - --hash=sha256:3f8904a8165ab21e07e58bf3e30a73f4dffc7a1e0dbc32d51c61b5360d26f43e \ - --hash=sha256:459ab414b35f4496138d0ecd735fed26f1318af5e52cb1efbc82a09f0d5aa911 \ - --hash=sha256:4c5533d03a6cbd7f56acfc9cfb44ea64f63d29091e40e44010d34178d392d7eb \ - --hash=sha256:51b13f9aed9d59ee389ad666b8c2214cc87b5de258fa712f9ab05f922e3896c6 \ - --hash=sha256:5361a0630a7fdb58a6a97638ab70e1dae2893c4d08d7aba64ded28bb9e7a29df \ - --hash=sha256:5397fff416b79e4b284959642a4e95ac4b0f1ece82c9993658e0e477d40551ec \ - --hash=sha256:57bab6deef2f4f1ca76cc04565df38dc5713ae6c17de690721bdf30cb1e0545c \ - --hash=sha256:6092beabe1966a3229f599d7088b38dfc8ffa1608b5b5cdda31e591e6500f856 \ - --hash=sha256:684083fd383e9dc04c794adb838d4faea08b291ce81f64ecd08e4577c7398adf \ - --hash=sha256:735e38e176a88ce41840c21bb49098ab66177c64c82426e24e0082500cc68af5 \ - --hash=sha256:7382b95189546f375c174f53a5fa873cef91c4b8005faa05cc5b3beea9c4f1c5 \ - --hash=sha256:748b6138585379c737adc08aeffd21222abbda1a86a0dca2a39682feb9196c20 \ - --hash=sha256:74be1268d1439eaaf552c698cdb11cd594f0c49295ae6bb72c34ee31abbe611b \ - --hash=sha256:7cc47943d524ee0096f973e1081cb8f4f17a4615f2116882a5f1416e4cfe92b5 \ - --hash=sha256:859b13906ce098c0b493af92142ad051bf64c7870fa58a123911c88606714996 \ - --hash=sha256:85f93781028ec63f383f6bc90db785a016319c561cc11151fbb7b34e0d012303 \ - --hash=sha256:86ce2371bfd7f212cf60d8517e5e854475c2c43ce14aa910e136ace72c6db6c1 \ - --hash=sha256:86f85dd7c947baa707078a236288a289044836d4b640962018ceb9cd1f899af5 \ - --hash=sha256:8dfffba826efcf366b1e3ccc37e67afe676f290e13a3b48d31a46739f80a8724 \ - --hash=sha256:8f2ac84905d12918e4e55a16da17939eb63e433dc11b677267c35568aa63fc84 \ - --hash=sha256:94309f498bcc07e5a7d16089ab984d42ad96af1d94b5a4eb966a266d9fcabf68 \ - --hash=sha256:94f95cf5d532d0e717eed4fc1810e8e6eded04621342ec54c89a7c2f14b581bf \ - --hash=sha256:9566fe4ababbb2610c39190791e5b829869351d14369603702e890ef3ad2d06e \ - --hash=sha256:9dca934f24c732750389ce49d638069c3892ad065df86cb465b3fa3012b70c9e \ - --hash=sha256:a9f136fbafe7ccf4ac7e8e0c28b31066e810be52d6e344ef954a3a70234e1702 \ - --hash=sha256:ab399ef5e3cd2a721b1038a0f3021001f19c5ab279f145e1146bb0b9f1b2b12c \ - --hash=sha256:b0c689c02947d636bc7fab3e30cc3a3445cca99c834dfb77cd4a6cabfc1c5597 \ - --hash=sha256:b2342d87af32790f934a79c3112641e7b27d63c261b8b4395350dad43eff1dc7 \ - --hash=sha256:b58f37edab4a3881bc6c9bca52670610e0c9ca14e2ea3cf9debf185b870457fb \ - --hash=sha256:bd8cb8026e5f5b50498a3c4f196f57f9db344dad829ffae16b82e4fdbaea2813 \ - --hash=sha256:be63c88b32e6c0f1429f1398ca5c09bc64b0d80950c8bb7807d7d7fb36fb84c7 \ - --hash=sha256:c3f293fdc675ccba4db5a561048cca627b5e7bd1c8a6973ffedabe7d116e22e2 \ - --hash=sha256:c41bc64626db62e72afec66b0c8a0da76491510015417c127bfc53b2fe6d7f7f \ - --hash=sha256:ce3a90455492bf8bfa38e56fbbe1dbd4f872a3d8eeaf7337dc3b1c8aa28c271b \ - --hash=sha256:ce7599575eeb25c0f4dc1be59cada6219f3b56176f799627f44088b21381a28a \ - --hash=sha256:dce09d6116df20a96acfdbf85e4866258c3758180e8c49845d6ba8248b6d0bbb \ - --hash=sha256:de8cb00d1483a412a06394b8303feec5dcb3b55f81d83aa216dbb6a0b86a94f5 \ - --hash=sha256:df2c8f3141f7cbd112a6ebbd760290b5849cda01884554f7c67acc14e7b1758a \ - --hash=sha256:e87cbc002b6f440482b3519e36e1313eb5443e9e9e73d6a52d43bd2004fcfd8e \ - --hash=sha256:e888474dee2f59ff68130f8a397792d8cb8e17e6b3434339657ba4ee90845a8c \ - --hash=sha256:f12857d24d98441af6a1d5c87442d624411db486f7ba12550b07788f74b67b04 \ - --hash=sha256:f2d4e43ee362adfc05994ed479334d5a451ab7bc3f3fee1b796b8ca66895acb4 \ - --hash=sha256:f3d6379493e18ad4d39537a82371c5281e153e963cecb13f953ebac155756525 \ - --hash=sha256:f8dff3d9777e5d2703a962ee5c286c239bf0ba173877cc68dc02c17d042e29de \ - --hash=sha256:f9ab915a267fc47c7e88c387a3a28325b58c898e23d4995f765728f4e3dedb97 \ - --hash=sha256:fbe6e89c7ffb48518384068321621b2a69cab509f58e40e4399fdd378fa6d074 \ - --hash=sha256:fd5f135b1bd58ab088930b3c613455796dfa0393626a6972663ccdda5b4ac6ce \ - --hash=sha256:ff870aebe9a93a85283837801d35cd5f8814fe2ad01e606861a7fb47c762a2b7 +grpcio==1.80.0 \ + --hash=sha256:00168469238b022500e486c1c33916acf2f2a9b2c022202cf8a1885d2e3073c1 \ + --hash=sha256:02e64bb0bb2da14d947a49e6f120a75e947250aebe65f9629b62bb1f5c14e6e9 \ + --hash=sha256:05d55e1798756282cddd52d56c896b3e7d673e3a8798c2f1cd05ba249a3bb4de \ + --hash=sha256:09e5e478b3d14afd23f12e49e8b44c8684ac3c5f08561c43a5b9691c54d136ab \ + --hash=sha256:0cb517eb1d0d0aaf1d87af7cc5b801d686557c1d88b2619f5e31fab3c2315921 \ + --hash=sha256:1b97cd29a8eda100b559b455331c487a80915b6ea6bd91cf3e89836c4ee8d957 \ + --hash=sha256:256507e2f524092f1473071a05e65a5b10d84b82e3ff24c5b571513cfaa61e2f \ + --hash=sha256:29aca15edd0688c22ba01d7cc01cb000d72b2033f4a3c72a81a19b56fd143257 \ + --hash=sha256:2bea16af2750fd0a899bf1abd9022244418b55d1f37da2202249ba4ba673838d \ + --hash=sha256:2dcc70e9f0ba987526e8e8603a610fb4f460e42899e74e7a518bf3c68fe1bf05 \ + --hash=sha256:2ed770b4c06984f3b47eb0517b1c69ad0b84ef3f40128f51448433be904634cd \ + --hash=sha256:31b9ac4ad1aa28ffee5503821fafd09e4da0a261ce1c1281c6c8da0423c83b6e \ + --hash=sha256:33eb763f18f006dc7fee1e69831d38d23f5eccd15b2e0f92a13ee1d9242e5e02 \ + --hash=sha256:367ce30ba67d05e0592470428f0ec1c31714cab9ef19b8f2e37be1f4c7d32fae \ + --hash=sha256:3b01e1f5464c583d2f567b2e46ff0d516ef979978f72091fd81f5ab7fa6e2e7f \ + --hash=sha256:3cb8130ba457d2aa09fa6b7c3ed6b6e4e6a2685fce63cb803d479576c4d80e21 \ + --hash=sha256:3d4147a97c8344d065d01bbf8b6acec2cf86fb0400d40696c8bdad34a64ffc0e \ + --hash=sha256:43168871f170d1e4ed16ae03d10cd21efa29f190e710a624cee7e5ae07da6f4f \ + --hash=sha256:448c884b668b868562b1bda833c5fce6272d26e1926ec46747cda05741d302c1 \ + --hash=sha256:4560cf0e86514595dbbd330cd65b7afad4b5c4b8c4905c041cfffa138d45e6fd \ + --hash=sha256:46c2390b59d67f84e882694d489f5b45707c657832d7934859ceb8c33f467069 \ + --hash=sha256:4e78c4ac0d97dc2e569b2f4bcbbb447491167cb358d1a389fc4af71ab6f70411 \ + --hash=sha256:4ed39fbdcf9b87370f6e8df4e39ca7b38b3e5e9d1b0013c7b6be9639d6578d14 \ + --hash=sha256:50a9871536d71c4fba24ee856abc03a87764570f0c457dd8db0b4018f379fed9 \ + --hash=sha256:51b4a7189b0bef2aa30adce3c78f09c83526cf3dddb24c6a96555e3b97340440 \ + --hash=sha256:52d143637e3872633fc7dd7c3c6a1c84e396b359f3a72e215f8bf69fd82084fc \ + --hash=sha256:5c07e82e822e1161354e32da2662f741a4944ea955f9f580ec8fb409dd6f6060 \ + --hash=sha256:627fb7312171cdc52828bd6fac8d7028ff2a64b89f1957b6f3416caa2218d141 \ + --hash=sha256:68e5851ac4b9afe07e7f84483803ad167852570d65326b34d54ca560bfa53fb6 \ + --hash=sha256:7b641fc3f1dc647bfd80bd713addc68f6d145956f64677e56d9ebafc0bd72388 \ + --hash=sha256:8502122a3cc1714038e39a0b071acb1207ca7844208d5ea0d091317555ee7106 \ + --hash=sha256:873ff5d17d68992ef6605330127425d2fc4e77e612fa3c3e0ed4e668685e3140 \ + --hash=sha256:886457a7768e408cdce226ad1ca67d2958917d306523a0e21e1a2fdaa75c9c9c \ + --hash=sha256:8ac393b58aa16991a2f1144ec578084d544038c12242da3a215966b512904d0f \ + --hash=sha256:8eb613f02d34721f1acf3626dfdb3545bd3c8505b0e52bf8b5710a28d02e8aa7 \ + --hash=sha256:92d787312e613754d4d8b9ca6d3297e69994a7912a32fa38c4c4e01c272974b0 \ + --hash=sha256:93b6f823810720912fd131f561f91f5fed0fda372b6b7028a2681b8194d5d294 \ + --hash=sha256:9a6284a5d907c37db53350645567c522be314bac859a64a7a5ca63b77bb7958f \ + --hash=sha256:9fe648599c0e37594c4809d81a9e77bd138cc82eb8baa71b6a86af65426723ff \ + --hash=sha256:a1dc80fe55685b4a543555e6eef975303b36c8db1023b1599b094b92aa77965f \ + --hash=sha256:a361c20ec1ccd3c3953d20fb6d7b4125093bdd10dff44c5e2bbb39e58917cedc \ + --hash=sha256:a72d84ad0514db063e21887fbacd1fd7acb4d494a564cae22227cd45c7fbf199 \ + --hash=sha256:aacdfb4ed3eb919ca997504d27e03d5dba403c85130b8ed450308590a738f7a4 \ + --hash=sha256:ba0915d51fd4ced2db5ff719f84e270afe0e2d4c45a7bdb1e8d036e4502928c2 \ + --hash=sha256:ba0db34f7e1d803a878284cd70e4c63cb6ae2510ba51937bf8f45ba997cefcf7 \ + --hash=sha256:bac1d573dfa84ce59a5547073e28fa7326d53352adda6912e362da0b917fcef4 \ + --hash=sha256:c51bf8ac4575af2e0678bccfb07e47321fc7acb5049b4482832c5c195e04e13a \ + --hash=sha256:c624cc9f1008361014378c9d776de7182b11fe8b2e5a81bc69f23a295f2a1ad0 \ + --hash=sha256:c71309cfce2f22be26aa4a847357c502db6c621f1a49825ae98aa0907595b193 \ + --hash=sha256:ce1794f4ea6cc3ca29463f42d665c32ba1b964b48958a66497917fe9069f26e6 \ + --hash=sha256:d334591df610ab94714048e0d5b4f3dd5ad1bee74dfec11eee344220077a79de \ + --hash=sha256:d8e11f167935b3eb089ac9038e1a063e6d7dbe995c0bb4a661e614583352e76f \ + --hash=sha256:dc053420fc75749c961e2a4c906398d7c15725d36ccc04ae6d16093167223b58 \ + --hash=sha256:deb10a1528473c11f72a0939eed36d83e847d7cbb63e8cc5611fb7a912d38614 \ + --hash=sha256:dfab85db094068ff42e2a3563f60ab3dddcc9d6488a35abf0132daec13209c8a \ + --hash=sha256:e172cf795a3ba5246d3529e4d34c53db70e888fa582a8ffebd2e6e48bc0cba50 \ + --hash=sha256:e9e408fc016dffd20661f0126c53d8a31c2821b5c13c5d67a0f5ed5de93319ad \ + --hash=sha256:ec0a592e926071b4abad50c1495cd0d0d513324b3ff5e7267067c33ba27506e4 \ + --hash=sha256:f14b618fc30de822681ee986cfdcc2d9327229dc4c98aed16896761cacd468b9 \ + --hash=sha256:f49eddcac43c3bf350c0385366a58f36bed8cc2c0ec35ef7b74b49e56552c0c2 \ + --hash=sha256:f7691a6788ad9196872f95716df5bc643ebba13c97140b7a5ee5c8e75d1dea81 # via # google-api-core # google-cloud-storage-control # googleapis-common-protos # grpc-google-iam-v1 # grpcio-status -grpcio-status==1.78.0 \ - --hash=sha256:a34cfd28101bfea84b5aa0f936b4b423019e9213882907166af6b3bddc59e189 \ - --hash=sha256:b492b693d4bf27b47a6c32590701724f1d3b9444b36491878fb71f6208857f34 +grpcio-status==1.62.3 \ + --hash=sha256:289bdd7b2459794a12cf95dc0cb727bd4a1742c37bd823f760236c937e53a485 \ + --hash=sha256:f9049b762ba8de6b1086789d8315846e094edac2c50beaf462338b301a8fd4b8 # via google-api-core gviz-api==1.10.0 \ --hash=sha256:846692dd8cc73224fc31b18e41589bd934e1cc05090c6576af4b4b26c2e71b90 \ --hash=sha256:a05055fed8c279f34f4b496eace7648c7fe9c1b06851e8a36e748541f1adbb05 # via xprof-nightly -idna==3.11 \ - --hash=sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea \ - --hash=sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902 +idna==3.13 \ + --hash=sha256:585ea8fe5d69b9181ec1afba340451fba6ba764af97026f92a91d4eef164a242 \ + --hash=sha256:892ea0cde124a99ce773decba204c5552b69c3c67ffd5f232eb7696135bc8bb3 # via # requests # yarl @@ -825,9 +827,9 @@ markupsafe==3.0.3 \ --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \ --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 # via werkzeug -more-itertools==10.8.0 \ - --hash=sha256:52d4362373dcf7c52546bc4af9a86ee7c4579df9a8dc268be0a2f949d376cc9b \ - --hash=sha256:f638ddf8a1a0d134181275fb5d58b086ead7c6a72429ad725c67503f13ba30bd +more-itertools==11.0.2 \ + --hash=sha256:392a9e1e362cbc106a2457d37cabf9b36e5e12efd4ebff1654630e76597df804 \ + --hash=sha256:6e35b35f818b01f691643c6c611bc0902f2e92b46c18fffa77ae1e7c46e912e4 # via # cheroot # jaraco-functools @@ -1111,23 +1113,21 @@ propcache==0.4.1 \ # via # aiohttp # yarl -proto-plus==1.27.1 \ - --hash=sha256:912a7460446625b792f6448bade9e55cd4e41e6ac10e27009ef71a7f317fa147 \ - --hash=sha256:e4643061f3a4d0de092d62aa4ad09fa4756b2cbb89d4627f3985018216f9fefc +proto-plus==1.27.2 \ + --hash=sha256:6432f75893d3b9e70b9c412f1d2f03f65b11fb164b793d14ae2ca01821d22718 \ + --hash=sha256:b2adde53adadf75737c44d3dcb0104fde65250dfc83ad59168b4aa3e574b6a24 # via # google-api-core # google-cloud-storage-control -protobuf==6.33.5 \ - --hash=sha256:3093804752167bcab3998bec9f1048baae6e29505adaf1afd14a37bddede533c \ - --hash=sha256:69915a973dd0f60f31a08b8318b73eab2bd6a392c79184b3612226b0a3f8ec02 \ - --hash=sha256:6ddcac2a081f8b7b9642c09406bc6a4290128fce5f471cddd165960bb9119e5c \ - --hash=sha256:8afa18e1d6d20af15b417e728e9f60f3aa108ee76f23c3b2c07a2c3b546d3afd \ - --hash=sha256:8f04fa32763dcdb4973d537d6b54e615cc61108c7cb38fe59310c3192d29510a \ - --hash=sha256:9b71e0281f36f179d00cbcb119cb19dec4d14a81393e5ea220f64b286173e190 \ - --hash=sha256:a3157e62729aafb8df6da2c03aa5c0937c7266c626ce11a278b6eb7963c4e37c \ - --hash=sha256:a5cb85982d95d906df1e2210e58f8e4f1e3cdc088e52c921a041f9c9a0386de5 \ - --hash=sha256:cbf16ba3350fb7b889fca858fb215967792dc125b35c7976ca4818bee3521cf0 \ - --hash=sha256:d71b040839446bac0f4d162e758bea99c8251161dae9d0983a3b88dee345153b +protobuf==7.34.1 \ + --hash=sha256:34b84ce27680df7cca9f231043ada0daa55d0c44a2ddfaa58ec1d0d89d8bf60a \ + --hash=sha256:403b093a6e28a960372b44e5eb081775c9b056e816a8029c61231743d63f881a \ + --hash=sha256:5185e0e948d07abe94bb76ec9b8416b604cfe5da6f871d67aad30cbf24c3110b \ + --hash=sha256:8ff40ce8cd688f7265326b38d5a1bed9bfdf5e6723d49961432f83e21d5713e4 \ + --hash=sha256:9ce42245e704cc5027be797c1db1eb93184d44d1cdd71811fb2d9b25ad541280 \ + --hash=sha256:bb3812cd53aefea2b028ef42bd780f5b96407247f20c6ef7c679807e9d188f11 \ + --hash=sha256:d8b2cc79c4d8f62b293ad9b11ec3aebce9af481fa73e64556969f7345ebf9fc7 \ + --hash=sha256:e97b55646e6ce5cbb0954a8c28cd39a5869b59090dfaa7df4598a7fba869468c # via # google-api-core # google-cloud-storage-control @@ -1148,9 +1148,13 @@ pycparser==3.0 \ --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 # via cffi -requests==2.32.5 \ - --hash=sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6 \ - --hash=sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf +pyopenssl==26.2.0 \ + --hash=sha256:4f9d971bc5298b8bc1fab282803da04bf000c755d4ad9d99b52de2569ca19a70 \ + --hash=sha256:8c6fcecd1183a7fc897548dfe388b0cdb7f37e018200d8409cf33959dbe35387 + # via xprof-nightly +requests==2.33.1 \ + --hash=sha256:18817f8c57c6263968bc123d237e3b8b08ac046f5456bd1e307ee8f4250d3517 \ + --hash=sha256:4e6d1ef462f3626a1f0a0a9c42dd93c63bad33f9f1c1937509b8c5c8718ab56a # via # gcsfs # google-api-core @@ -1177,27 +1181,20 @@ typing-extensions==4.15.0 \ # aiosignal # etils # grpcio + # pyopenssl urllib3==2.6.3 \ --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 # via requests -werkzeug==3.1.6 \ - --hash=sha256:210c6bede5a420a913956b4791a7f4d6843a43b6fcee4dfa08a65e93007d0d25 \ - --hash=sha256:7ddf3357bb9564e407607f988f683d72038551200c704012bb9a4c523d42f131 +werkzeug==3.1.8 \ + --hash=sha256:63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50 \ + --hash=sha256:9bad61a4268dac112f1c5cd4630a56ede601b6ed420300677a869083d70a4c44 # via xprof-nightly -xprof-nightly==2.23.0a20260317 \ - --hash=sha256:25d30efd382782f4e3689868e38cb963d0d152197c09fbde972c79ab4db06754 \ - --hash=sha256:295daa79f8ddf39e3575befa2ebf50b9b6f7356f19ec26f2a673dbf36f02e7d6 \ - --hash=sha256:3dd619c9aaf13b96c68b291e2857635407b37bdcb37bb943991fb42aaa2375c0 \ - --hash=sha256:5f49a4ebe94569316295fdbe501853b72a22c212d0d377dd9ff0dc96021b3fea \ - --hash=sha256:6364e51d1581d4d837be6b0b3783e4ba004200cde4266e5a28ad1f9c7049ddb0 \ - --hash=sha256:79743fae634e8cff896fd0be6c84bf726bd251a554a6b0c18e06b6c83e52953a \ - --hash=sha256:89a5ee026f670c409b5f880ebf02a24660736c2eef8d8f10da660d0b4438970f \ - --hash=sha256:a49a6f19d7505414196d6c06238161fbbfbc7bb2049095071e7427c79019f34e \ - --hash=sha256:a62a9c1a3670cb13446dcdc840a61370e19ceadc6f2d41307e2ab4677fd01d15 \ - --hash=sha256:ac61b96fb1a4accba93a4ede9a67ff2f6151466bcba37835c35a6b47501a19c7 \ - --hash=sha256:de1eff60d3164b06f9044f0c9ba12d69fc36f2abe55c81f7a76e6ec09e270c20 \ - --hash=sha256:ee7520f6ab3894ed08c4d77fe5fc1227b96aa8923452491d0104a2968c3d283e +xprof-nightly==2.23.2a20260505 \ + --hash=sha256:1cadc9aeb0063924da030774b9c491804bfb06458fc6622ab473adb785bb35bc \ + --hash=sha256:9619bac6f10b71afc64909d0641b732257b1ebc2bc9f9e80fdf5e15f53388baa \ + --hash=sha256:b2fafa447b221f58c2a848fb136ed00a48239331b5e9c68aeabb9d535ba8125d \ + --hash=sha256:e804d2f135711975b8b6d7f730fb97d1680e17e6b2a225a7ce7775795e835e77 # via -r tools/xprof/requirements.in yarl==1.23.0 \ --hash=sha256:03214408cfa590df47728b84c679ae4ef00be2428e11630277be0727eba2d7cc \ @@ -1329,7 +1326,7 @@ yarl==1.23.0 \ --hash=sha256:fe8f8f5e70e6dbdfca9882cd9deaac058729bcf323cf7a58660901e55c9c94f6 \ --hash=sha256:fffc45637bcd6538de8b85f51e3df3223e4ad89bccbfca0481c08c7fc8b7ed7d # via aiohttp -zipp==3.23.0 \ - --hash=sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e \ - --hash=sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166 +zipp==3.23.1 \ + --hash=sha256:0b3596c50a5c700c9cb40ba8d86d9f2cc4807e9bedb06bcdf7fac85633e444dc \ + --hash=sha256:32120e378d32cd9714ad503c1d024619063ec28aad2248dc6672ad13edfa5110 # via etils From 0979289ffe33628619cb85ca79414d4ba76a1fc1 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 6 May 2026 12:46:15 +0200 Subject: [PATCH 470/881] zml/sharding: pass sharding by pointer and simplify replicated usage (#514) This PR simplifies the common replicated case while making sharding intent explicit and use sharding by pointer ( sharding struct is quite big: ~600 bytes). Using pointers for shardings also avoids unnecessary sharding copies and makes it easier to share stable sharding instances across compilation, loading, and runtime paths. - `Platform` owns the canonical replicated sharding - `Buffer` APIs now take explicit sharding specs: - `.replicated` - `.{ .sharded = sharding_ptr }` - `createTensor` / `maybeCreateTensor `now require explicit partitioning instead of accepting null - compilation/loading now use pointer-based sharding lists This makes simple cases like mnist and benchmark much cleaner: callers no longer need to manually construct and thread a replicated sharding just to use the default case, but they still state placement explicitly at the call site. --------- Co-authored-by: Hugo Mano --- README.md | 2 +- docs/tutorials/write_first_model.md | 18 +- examples/benchmark/main.zig | 11 +- examples/io/main.zig | 17 +- examples/llm/models/common.zig | 12 +- examples/llm/models/lfm2/inference.zig | 5 +- examples/llm/models/lfm2/model.zig | 17 +- examples/llm/models/lfm2/session.zig | 20 +- examples/llm/models/lfm2_tests.zig | 14 +- examples/llm/models/llama/inference.zig | 8 +- examples/llm/models/llama/model.zig | 7 +- examples/llm/models/llama/session.zig | 16 +- examples/llm/models/llama_tests.zig | 13 +- examples/llm/models/qwen3_5/inference.zig | 12 +- examples/llm/models/qwen3_5/model.zig | 8 +- examples/llm/models/qwen3_5/session.zig | 15 +- examples/mnist/mnist.zig | 14 +- examples/sharding/main.zig | 23 +- zml/BUILD.bazel | 2 +- zml/{sharding.zig => Sharding.zig} | 347 +++++++++++----------- zml/attention/attention.zig | 2 +- zml/attention/flashattn.zig | 12 +- zml/attention/paged_attention.zig | 3 + zml/buffer.zig | 16 +- zml/exe.zig | 19 +- zml/io.zig | 90 +++--- zml/module.zig | 50 ++-- zml/nn.zig | 92 +++--- zml/ops.zig | 17 +- zml/platform.zig | 93 ++++-- zml/tensor.zig | 217 ++++++-------- zml/testing.zig | 21 +- zml/zml.zig | 4 +- 33 files changed, 581 insertions(+), 636 deletions(-) rename zml/{sharding.zig => Sharding.zig} (89%) diff --git a/README.md b/README.md index 528ce6b9..dd7bfe69 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ const Mnist = struct { io: std.Io, platform: *const zml.Platform, store: *const zml.io.TensorStore, - shardings: []const zml.sharding.Sharding, + shardings: []const zml.Sharding, ) !zml.Bufferized(Mnist) { return zml.io.load(Mnist, self, allocator, io, platform, store, .{ .shardings = shardings, diff --git a/docs/tutorials/write_first_model.md b/docs/tutorials/write_first_model.md index b88c495e..036776eb 100644 --- a/docs/tutorials/write_first_model.md +++ b/docs/tutorials/write_first_model.md @@ -171,9 +171,7 @@ We're only going to use the CPU for our simple model, but we need to compile the // Our computation require an input tensor const input: zml.Tensor = .init(.{3}, .f16); -const sharding = try zml.sharding.replicatedSharding(platform); - -var executable = try platform.compile(allocator, io, layer, .forward, .{input}, .{ .shardings = &.{sharding}}); +var executable = try platform.compile(allocator, io, layer, .forward, .{input}, .{}); defer executable.deinit(); ``` @@ -193,8 +191,8 @@ provides ways to have extra-low loading times). const weight_slice: zml.Slice = .init(layer.weight.shape(), std.mem.sliceAsBytes(&[3]f16{1.0, 2.0, 3.0})); const bias_slice: zml.Slice = .init(layer.bias.?.shape(), std.mem.sliceAsBytes(&[3]f16{1.0, 1.0, 1.0})); var layer_buffers: zml.Bufferized(Layer) = .{ - .weight = try zml.Buffer.fromSlice(io, platform, weight_slice, sharding), - .bias = try zml.Buffer.fromSlice(io, platform, bias_slice, sharding), + .weight = try zml.Buffer.fromSlice(io, platform, weight_slice, .replicated), + .bias = try zml.Buffer.fromSlice(io, platform, bias_slice, .replicated), }; defer layer_buffers.weight.deinit(); defer layer_buffers.bias.?.deinit(); @@ -359,23 +357,21 @@ pub fn main(init: std.process.Init) !void { // Our computation require an input tensor const input: zml.Tensor = .init(.{3}, .f16); - const sharding = try zml.sharding.replicatedSharding(platform); - - var executable = try platform.compile(allocator, io, layer, .forward, .{input}, .{ .shardings = &.{sharding} }); + var executable = try platform.compile(allocator, io, layer, .forward, .{input}, .{}); defer executable.deinit(); const weight_slice: zml.Slice = .init(layer.weight.shape(), std.mem.sliceAsBytes(&[3]f16{ 1.0, 2.0, 3.0 })); const bias_slice: zml.Slice = .init(layer.bias.?.shape(), std.mem.sliceAsBytes(&[3]f16{ 1.0, 1.0, 1.0 })); var layer_buffers: zml.Bufferized(Layer) = .{ - .weight = try zml.Buffer.fromSlice(io, platform, weight_slice, sharding), - .bias = try zml.Buffer.fromSlice(io, platform, bias_slice, sharding), + .weight = try zml.Buffer.fromSlice(io, platform, weight_slice, .replicated), + .bias = try zml.Buffer.fromSlice(io, platform, bias_slice, .replicated), }; defer layer_buffers.weight.deinit(); defer layer_buffers.bias.?.deinit(); // create the input buffer const input_slice: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[3]f16{ 5.0, 5.0, 5.0 })); - var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice, sharding); + var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice, .replicated); defer input_buffer.deinit(); // create the Args and Results structs diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index 953218d9..5a66613c 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -24,12 +24,9 @@ pub fn main(init: std.process.Init) !void { const platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator, io); - const logical_mesh: zml.sharding.LogicalMesh = .init("benchmark_mesh", .{ - .m = .low_bandwidth, - .n = .high_bandwidth, - }); - const strategy: zml.sharding.Strategy = .suggest(logical_mesh, platform.physical_mesh); - const benchmark_sharding: zml.sharding.Sharding = try .initFromStrategy(platform, logical_mesh, strategy); + const benchmark_sharding: zml.Sharding = try platform.registerSharding("benchmark_mesh", .mesh( + .{ .m = .low_bandwidth, .n = .high_bandwidth }, + )); const cli_args: CliArgs = stdx.flags.parse(init.minimal.args, CliArgs); @@ -97,7 +94,7 @@ pub fn main(init: std.process.Init) !void { }); } -fn createRandomBuffer(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, shape: zml.Shape, sharding: zml.sharding.Sharding, random: std.Random) !zml.Buffer { +fn createRandomBuffer(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, shape: zml.Shape, sharding: zml.Sharding, random: std.Random) !zml.Buffer { const slice = try zml.Slice.alloc(allocator, shape); defer slice.free(allocator); diff --git a/examples/io/main.zig b/examples/io/main.zig index 92f54aba..51032759 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -180,23 +180,20 @@ pub fn main(init: std.process.Init) !void { var load_count: usize = 0; while (registry_it.next()) |entry| : (load_count += 1) { tensors[load_count] = switch (sharding_type) { - .replicated => store.view().createTensor(entry.key_ptr.*, null, null), + .replicated => store.view().createTensor(entry.key_ptr.*, null, .replicated), .sharded => if (entry.value_ptr.shape.rank() > 0) store.view().createTensor(entry.key_ptr.*, null, .{ ._0 = .model }) else - store.view().createTensor(entry.key_ptr.*, null, null), + store.view().createTensor(entry.key_ptr.*, null, .replicated), }; } const model: AllTensorsModel = .{ .tensors = tensors }; - const replicated_sharding = try zml.sharding.replicatedSharding(platform); - - const sharded_sharding: zml.sharding.Sharding = blk: { - const model_logical_mesh: zml.sharding.LogicalMesh = .init("playground_model", .{ .model = .high_bandwidth }); - const model_strategy: zml.sharding.Strategy = .suggest(model_logical_mesh, platform.physical_mesh); - break :blk try .initFromStrategy(platform, model_logical_mesh, model_strategy); - }; + const sharded_sharding: zml.Sharding = try platform.registerSharding( + "playground_model", + .mesh(.{ .model = .high_bandwidth }), + ); var progress = std.Progress.start(io, .{ .root_name = "zml.examples.load" }); progress.increaseEstimatedTotalItems(load_count); @@ -211,7 +208,7 @@ pub fn main(init: std.process.Init) !void { } _ = try zml.io.load(AllTensorsModel, &model, init.arena.allocator(), io, platform, &store, .{ - .shardings = &.{ replicated_sharding, sharded_sharding }, + .shardings = &.{sharded_sharding}, .parallelism = 16, .dma_chunks = 16, .dma_chunk_size = 64 * zml.MiB, diff --git a/examples/llm/models/common.zig b/examples/llm/models/common.zig index 79112dcc..c547c761 100644 --- a/examples/llm/models/common.zig +++ b/examples/llm/models/common.zig @@ -13,20 +13,16 @@ pub const GenerationOptions = struct { }; pub const Shardings = struct { - replicated: zml.sharding.Sharding, - model: zml.sharding.Sharding, + model: zml.Sharding, pub fn init(platform: *zml.Platform) !Shardings { - const model_mesh: zml.sharding.LogicalMesh = .init("model", .{ .model = .high_bandwidth }); - const model_sharding_strategy: zml.sharding.Strategy = .suggest(model_mesh, platform.physical_mesh); return .{ - .replicated = try zml.sharding.replicatedSharding(platform), - .model = try .initFromStrategy(platform, model_mesh, model_sharding_strategy), + .model = try platform.registerSharding("model", .mesh(.{ .model = .high_bandwidth })), }; } - pub fn all(self: *const Shardings) [2]zml.sharding.Sharding { - return .{ self.replicated, self.model }; + pub fn all(self: Shardings) [1]zml.Sharding { + return .{self.model}; } }; diff --git a/examples/llm/models/lfm2/inference.zig b/examples/llm/models/lfm2/inference.zig index c75f22e4..599d8eb5 100644 --- a/examples/llm/models/lfm2/inference.zig +++ b/examples/llm/models/lfm2/inference.zig @@ -237,10 +237,9 @@ pub const ComposedKernelExe = struct { }; defer hidden_buf.deinit(); - const replicated_sharding = try zml.sharding.replicatedSharding(args.platform); - var conv_cache_index_buf: zml.Buffer = try .scalar(args.io, args.platform, @as(u32, 0), .u32, replicated_sharding); + var conv_cache_index_buf: zml.Buffer = try .scalar(args.io, args.platform, 0, .u32); defer conv_cache_index_buf.deinit(); - var kv_cache_index_buf: zml.Buffer = try .scalar(args.io, args.platform, @as(u32, 0), .u32, replicated_sharding); + var kv_cache_index_buf: zml.Buffer = try .scalar(args.io, args.platform, 0, .u32); defer kv_cache_index_buf.deinit(); for (args.model_buffers.layers) |layer_bufs| { diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index f7b009ed..0a0b6140 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -94,12 +94,11 @@ pub const LoadedModel = struct { log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); } - const all_shardings = shardings.all(); return zml.io.load(Model, &self.inner, allocator, io, platform, store, .{ .dma_chunks = 32, .dma_chunk_size = 128 * zml.MiB, .progress = progress, - .shardings = &all_shardings, + .shardings = &shardings.all(), .parallelism = 16, .total_bytes = &total_bytes, }); @@ -252,7 +251,7 @@ pub const TokenEmbedding = struct { weight: zml.Tensor, pub fn init(store: zml.io.TensorStore.View) TokenEmbedding { - return .{ .weight = store.createTensor("weight", .{ .voc, .d }, null) }; + return .{ .weight = store.createTensor("weight", .{ .voc, .d }, .replicated) }; } pub fn forward(self: TokenEmbedding, tokens: zml.Tensor) zml.Tensor { @@ -400,7 +399,7 @@ pub const ShortConv = struct { return .{ .in_proj = initLinear(store.withPrefix("in_proj"), .d), .out_proj = initLinear(store.withPrefix("out_proj"), .d), - .kernel = store.createTensor("conv.weight", .{ .out, .in, .kernel_size }, null), + .kernel = store.createTensor("conv.weight", .{ .out, .in, .kernel_size }, .replicated), .config = config, }; } @@ -540,7 +539,7 @@ pub const Cache = struct { conv: ConvCache, kv: KvCache, - pub fn initBuffers(self: Cache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !zml.Bufferized(Cache) { + pub fn initBuffers(self: Cache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(Cache) { return .{ .conv = try self.conv.initBuffers(allocator, io, platform, sharding), .kv = try self.kv.initBuffers(io, platform, sharding) }; } @@ -561,7 +560,7 @@ pub const ConvCache = struct { return .{ .state = .fromShape(shape) }; } - pub fn initBuffers(self: ConvCache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !zml.Bufferized(ConvCache) { + pub fn initBuffers(self: ConvCache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(ConvCache) { const sh = self.state.shape(); const host = try allocator.alloc(u8, sh.byteSize()); defer allocator.free(host); @@ -586,7 +585,7 @@ pub const KvCache = struct { return .{ .k = .fromShape(kv_shape), .v = .fromShape(kv_shape) }; } - pub fn initBuffers(self: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !zml.Bufferized(KvCache) { + pub fn initBuffers(self: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(KvCache) { return .{ .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), sharding, .{}), .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), sharding, .{}) }; } @@ -618,7 +617,7 @@ pub const KvCache = struct { }; fn initLinear(store: zml.io.TensorStore.View, tag: anytype) Linear { - return .init(store.createTensor("weight", .{ .out, tag }, null), null, tag); + return .init(store.createTensor("weight", .{ .out, tag }, .replicated), null, tag); } pub const Linear = struct { @@ -671,7 +670,7 @@ const RmsNorm = struct { tag: zml.Shape.Tag, pub fn init(store: zml.io.TensorStore.View, eps: f32, tag: anytype) RmsNorm { - return .{ .weight = store.createTensor("weight", .{tag}, null), .eps = eps, .tag = zml.Shape.toTag(tag) }; + return .{ .weight = store.createTensor("weight", .{tag}, .replicated), .eps = eps, .tag = zml.Shape.toTag(tag) }; } pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig index 1a756115..018ed738 100644 --- a/examples/llm/models/lfm2/session.zig +++ b/examples/llm/models/lfm2/session.zig @@ -40,9 +40,9 @@ pub const Session = struct { .tokenizer = tokenizer, .config = &compiled_model.loaded_model.parsed_config.value, .seqlen = compiled_model.params.seqlen, - .cache_buffers = try compiled_model.params.cache.initBuffers(allocator, io, platform, compiled_model.params.shardings.replicated), + .cache_buffers = try compiled_model.params.cache.initBuffers(allocator, io, platform, .replicated), .attention_metadata_buffers = try compiled_model.params.attention_metadata.initBuffer(io, platform, compiled_model.params.shardings.model), - .rng_buf = try zml.Tensor.Rng.initBuffer(io, platform, compiled_model.params.shardings.replicated, seed), + .rng_buf = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed), .generated_token_slice = try .alloc(allocator, zml.Shape.init(.{ .batch = 1, .seq = 1 }, .u32)), .think_start = tokenizer.tokenId("") orelse unreachable, .think_end = tokenizer.tokenId("") orelse unreachable, @@ -109,17 +109,15 @@ pub const Session = struct { @memset(tokens, self.config.pad_token_id); @memcpy(tokens[0..all_tokens.len], all_tokens); - const sharding = try zml.sharding.replicatedSharding(self.platform); - - var tokens_buf: zml.Buffer = try .fromSlice(self.io, self.platform, tokens_slice, sharding); + var tokens_buf: zml.Buffer = try .fromSlice(self.io, self.platform, tokens_slice, .replicated); defer tokens_buf.deinit(); const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{0})); - var tokens_pos_buf: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice, sharding); + var tokens_pos_buf: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice, .replicated); defer tokens_pos_buf.deinit(); const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(all_tokens.len)})); - var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice, sharding); + var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice, .replicated); defer actual_seq_len_buf.deinit(); try self.compiled_model.prefill.run(.{ @@ -143,13 +141,11 @@ pub const Session = struct { var decoder = try self.tokenizer.decoder(); defer decoder.deinit(); - const sharding = try zml.sharding.replicatedSharding(self.platform); - - var current_token_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, self.generated_token_slice, sharding); + var current_token_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, self.generated_token_slice, .replicated); defer current_token_buffer.deinit(); const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{0})); - var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice, sharding); + var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice, .replicated); defer actual_seq_len_buf.deinit(); const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); @@ -173,7 +169,7 @@ pub const Session = struct { if (all_tokens.items.len >= self.seqlen) break :generation; const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(all_tokens.items.len)})); - var token_pos_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice, sharding); + var token_pos_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice, .replicated); defer token_pos_buffer.deinit(); try self.compiled_model.decode.run(.{ diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig index 20e6893e..bb014984 100644 --- a/examples/llm/models/lfm2_tests.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -47,11 +47,8 @@ pub fn main(init: std.process.Init) !void { defer repo_model.deinit(allocator); var progress = std.Progress.start(io, .{ .root_name = args.model }); - const tp_mesh: zml.sharding.LogicalMesh = .init("tp_mesh", .{ .model = .high_bandwidth }); - const tp_strategy: zml.sharding.Strategy = .suggest(tp_mesh, platform.physical_mesh); const shardings: common.Shardings = .{ - .replicated = try zml.sharding.replicatedSharding(platform), - .model = try .initFromStrategy(platform, tp_mesh, tp_strategy), + .model = try platform.registerSharding("tp_mesh", .mesh(.{ .model = .high_bandwidth })), }; var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress, shardings); @@ -81,7 +78,6 @@ pub fn run( var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); defer activation_store.deinit(); - const sharding = try zml.sharding.replicatedSharding(platform); var ctx = TestContext{ .allocator = allocator, .io = io, @@ -89,7 +85,7 @@ pub fn run( .activations_store = &activation_store, .attention_metadata = attention_metadata, .attention_parameters = attention_parameters, - .sharding = sharding, + .sharding = platform.replicated_sharding, }; try ctx.testLayer("embed_tokens", .{ .batch, .seq }, mdl.embed_tokens, model_buffers.embed_tokens, .{}); @@ -134,7 +130,7 @@ const TestContext = struct { activations_store: *zml.io.TensorStore, attention_metadata: attention.Metadata, attention_parameters: attention.Parameters, - sharding: zml.sharding.Sharding, + sharding: zml.Sharding, fn testLayerPrint(self: *TestContext, comptime name_fmt: []const u8, name_args: anytype, tagz: anytype, layer: anytype, layer_buffers: anytype, opts: zml.testing.CompareOpts) !void { const name = try std.fmt.allocPrint(self.allocator, name_fmt, name_args); @@ -281,7 +277,7 @@ const TestContext = struct { var attention_metadata_buffers = try self.attention_metadata.initBuffer(self.io, self.platform, self.sharding); defer attention.Metadata.deinitBuffer(&attention_metadata_buffers); - var cache_index_buf: zml.Buffer = try .scalar(self.io, self.platform, cache_ix, .u32, self.sharding); + var cache_index_buf: zml.Buffer = try .scalar(self.io, self.platform, cache_ix, .u32); defer cache_index_buf.deinit(); args.set(.{ layer_buffers, in_buffer, cache_pos_buffer, kv_cache, cache_index_buf, attention_metadata_buffers }); @@ -299,7 +295,7 @@ const TestContext = struct { } }; -fn loadBufferFromStore(allocator: std.mem.Allocator, io: anytype, platform: *zml.Platform, store: *zml.io.TensorStore, key: []const u8, sharding: zml.sharding.Sharding) !zml.Buffer { +fn loadBufferFromStore(allocator: std.mem.Allocator, io: anytype, platform: *zml.Platform, store: *zml.io.TensorStore, key: []const u8, sharding: zml.Sharding) !zml.Buffer { const shape = store.view().getShape(key) orelse return error.NotFound; const host_bytes = try allocator.alloc(u8, shape.byteSize()); diff --git a/examples/llm/models/llama/inference.zig b/examples/llm/models/llama/inference.zig index 7597ce4b..8fed5ab9 100644 --- a/examples/llm/models/llama/inference.zig +++ b/examples/llm/models/llama/inference.zig @@ -89,7 +89,7 @@ fn compileModel( platform_: *const zml.Platform, llama_model_: model.Model, parameters_: CompilationParameters, - shardings_: [2]zml.sharding.Sharding, + shardings_: [1]zml.Sharding, progress_: *std.Progress.Node, ) !zml.Exe { progress_.increaseEstimatedTotalItems(1); @@ -123,7 +123,7 @@ fn compileModel( platform_: *const zml.Platform, llama_model_: model.Model, parameters_: CompilationParameters, - shardings_: [2]zml.sharding.Sharding, + shardings_: []const zml.Sharding, progress_: *std.Progress.Node, ) !zml.Exe { progress_.increaseEstimatedTotalItems(1); @@ -143,10 +143,10 @@ fn compileModel( parameters_.attention_metadata, parameters_.attention_parameters, }, - .{ .shardings = &shardings_ }, + .{ .shardings = shardings_ }, ); } - }.call, .{ allocator, io, platform, llama_model, parameters, all_shardings, progress }); + }.call, .{ allocator, io, platform, llama_model, parameters, &all_shardings, progress }); var decode_future_awaited = false; errdefer if (!decode_future_awaited) if (decode_future.cancel(io)) |v| v.deinit() else |_| {}; diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index a6fb2e1c..6f6c06d6 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -81,12 +81,11 @@ pub const LoadedModel = struct { std.log.scoped(.llama).info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); } - const all_shardings = shardings.all(); return zml.io.load(Model, &self.inner, allocator, io, platform, store, .{ .dma_chunks = 32, .dma_chunk_size = 128 * zml.MiB, .progress = progress, - .shardings = &all_shardings, + .shardings = &shardings.all(), .parallelism = 16, .total_bytes = &total_bytes, }); @@ -157,7 +156,7 @@ pub const Model = struct { io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, - shardings: []const zml.sharding.Sharding, + shardings: []const zml.Sharding, progress: *std.Progress.Node, ) !zml.Bufferized(Model) { progress.increaseEstimatedTotalItems(store.view().count()); @@ -552,7 +551,7 @@ pub const KvCache = struct { }; } - pub fn initBuffer(kv: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !Buffer { + pub fn initBuffer(kv: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !Buffer { return .{ .k = try zml.Buffer.uninitialized(io, platform, kv.k.shape(), sharding, .{}), .v = try zml.Buffer.uninitialized(io, platform, kv.v.shape(), sharding, .{}), diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig index 7c23f619..5b170272 100644 --- a/examples/llm/models/llama/session.zig +++ b/examples/llm/models/llama/session.zig @@ -27,7 +27,7 @@ pub const Session = struct { compiled_model: *const inference.CompiledModel, model_buffers: *model.Buffers, ) !Session { - const shardings = compiled_model.params.shardings; + const shardings = &compiled_model.params.shardings; var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform, shardings.model); errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); @@ -35,7 +35,7 @@ pub const Session = struct { errdefer zml.attention.attention.Metadata.deinitBuffer(&attention_metadata_buffers); const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); - var rng_buffers = try zml.Tensor.Rng.initBuffer(io, platform, shardings.replicated, seed); + var rng_buffers = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); return .{ @@ -118,12 +118,10 @@ pub const Session = struct { defer prefill_tokens_slice.free(self.allocator); @memcpy(prefill_tokens_slice.items(u32)[0..all_tokens.len], all_tokens); - const replicated_sharding = try zml.sharding.replicatedSharding(self.platform); - - var prefill_tokens_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, prefill_tokens_slice, replicated_sharding); + var prefill_tokens_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, prefill_tokens_slice, .replicated); defer prefill_tokens_buffer.deinit(); - var prefill_token_pos_buffer = try zml.Buffer.scalar(self.io, self.platform, 0, .u32, replicated_sharding); + var prefill_token_pos_buffer = try zml.Buffer.scalar(self.io, self.platform, 0, .u32); defer prefill_token_pos_buffer.deinit(); prefill_args.set(.{ @@ -157,10 +155,8 @@ pub const Session = struct { var decode_results = try self.compiled_model.decode_exe.results(self.allocator); defer decode_results.deinit(self.allocator); - const replicated_sharding = try zml.sharding.replicatedSharding(self.platform); - var last_token_id: u32 = self.last_generated_token; - var current_token_buffer: zml.Buffer = try .fromBytes(self.io, self.platform, .init(.{ .s = 1 }, .u32), replicated_sharding, @ptrCast(&last_token_id)); + var current_token_buffer: zml.Buffer = try .fromBytes(self.io, self.platform, .init(.{ .s = 1 }, .u32), .replicated, @ptrCast(&last_token_id)); defer current_token_buffer.deinit(); generation: while (true) { @@ -172,7 +168,7 @@ pub const Session = struct { try all_tokens.append(self.allocator, last_token_id); if (all_tokens.items.len >= self.seqlen) break :generation; - var token_pos_buffer: zml.Buffer = try .scalar(self.io, self.platform, all_tokens.items.len, .u32, replicated_sharding); + var token_pos_buffer: zml.Buffer = try .scalar(self.io, self.platform, all_tokens.items.len, .u32); defer token_pos_buffer.deinit(); decode_args.set(.{ diff --git a/examples/llm/models/llama_tests.zig b/examples/llm/models/llama_tests.zig index 14faba7f..fe1b9ec3 100644 --- a/examples/llm/models/llama_tests.zig +++ b/examples/llm/models/llama_tests.zig @@ -44,18 +44,15 @@ pub fn main(init: std.process.Init) !void { defer repo_model.deinit(allocator); var progress = std.Progress.start(io, .{ .root_name = args.model }); - const tp_mesh: zml.sharding.LogicalMesh = .init("tp_mesh", .{ .model = .high_bandwidth }); - const tp_strategy: zml.sharding.Strategy = .suggest(tp_mesh, platform.physical_mesh); const shardings: common.Shardings = .{ - .replicated = try zml.sharding.replicatedSharding(platform), - .model = try .initFromStrategy(platform, tp_mesh, tp_strategy), + .model = try platform.registerSharding("tp_mesh", .mesh(.{ .model = .high_bandwidth })), }; var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress, shardings); defer repo_model.unloadBuffers(&model_buffers, allocator); progress.end(); - try run(allocator, io, platform, args.activations, repo_model.inner, &model_buffers, shardings.replicated); + try run(allocator, io, platform, args.activations, repo_model.inner, &model_buffers, platform.replicated_sharding); } fn run( @@ -65,7 +62,7 @@ fn run( activations_path: []const u8, mdl: model.Model, model_buffers: *model.Buffers, - sharding: zml.sharding.Sharding, + sharding: zml.Sharding, ) !void { var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, activations_path); defer registry.deinit(); @@ -97,7 +94,7 @@ fn testLayer( name: []const u8, layer: anytype, layer_weights: zml.Bufferized(@TypeOf(layer)), - sharding: zml.sharding.Sharding, + sharding: zml.Sharding, opts: zml.testing.CompareOpts, ) !void { const in_key = try std.fmt.allocPrint(allocator, "{s}.in", .{name}); @@ -129,7 +126,7 @@ fn testLayer( try zml.testing.expectClose(io, out_result, out_buffer_expected, opts); } -fn loadBufferFromStore(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: zml.io.TensorStore.View, key: []const u8, sharding: zml.sharding.Sharding) !zml.Buffer { +fn loadBufferFromStore(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: zml.io.TensorStore.View, key: []const u8, sharding: zml.Sharding) !zml.Buffer { const shape = store.getShape(key) orelse return error.NotFound; const host_bytes = try allocator.alloc(u8, shape.byteSize()); diff --git a/examples/llm/models/qwen3_5/inference.zig b/examples/llm/models/qwen3_5/inference.zig index 650b57cb..2896caa4 100644 --- a/examples/llm/models/qwen3_5/inference.zig +++ b/examples/llm/models/qwen3_5/inference.zig @@ -85,7 +85,7 @@ fn compileModel( platform_: *const zml.Platform, qwen_model_: model.Model, parameters_: CompilationParameters, - shardings_: [2]zml.sharding.Sharding, + shardings_: []const zml.Sharding, progress_: *std.Progress.Node, ) !zml.Exe { progress_.increaseEstimatedTotalItems(1); @@ -106,10 +106,10 @@ fn compileModel( parameters_.kv_cache, parameters_.rng, }, - .{ .shardings = &shardings_ }, + .{ .shardings = shardings_ }, ); } - }.call, .{ allocator, io, platform, qwen_model, parameters, all_shardings, progress }); + }.call, .{ allocator, io, platform, qwen_model, parameters, &all_shardings, progress }); errdefer if (prefill_future.cancel(io)) |v| v.deinit() else |_| {}; var decode_future = try io.concurrent(struct { @@ -119,7 +119,7 @@ fn compileModel( platform_: *const zml.Platform, qwen_model_: model.Model, parameters_: CompilationParameters, - shardings_: [2]zml.sharding.Sharding, + shardings_: []const zml.Sharding, progress_: *std.Progress.Node, ) !zml.Exe { progress_.increaseEstimatedTotalItems(1); @@ -140,10 +140,10 @@ fn compileModel( parameters_.kv_cache, parameters_.rng, }, - .{ .shardings = &shardings_ }, + .{ .shardings = shardings_ }, ); } - }.call, .{ allocator, io, platform, qwen_model, parameters, all_shardings, progress }); + }.call, .{ allocator, io, platform, qwen_model, parameters, &all_shardings, progress }); errdefer if (decode_future.cancel(io)) |v| v.deinit() else |_| {}; const prefill_exe = try prefill_future.await(io); diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 22f031e6..2ec23dba 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -186,7 +186,7 @@ pub const Model = struct { io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, - shardings: []const zml.sharding.Sharding, + shardings: []const zml.Sharding, progress: *std.Progress.Node, ) !zml.Bufferized(Model) { progress.increaseEstimatedTotalItems(store.view().count()); @@ -882,7 +882,7 @@ pub const KvCache = struct { }; } - pub fn initBuffer(kv: SelfAttnCache, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !SelfAttnCache.Buffers { + pub fn initBuffer(kv: SelfAttnCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !SelfAttnCache.Buffers { return .{ .k = try zml.Buffer.uninitialized(io, platform, kv.k.shape(), sharding, .{}), .v = try zml.Buffer.uninitialized(io, platform, kv.v.shape(), sharding, .{}), @@ -968,7 +968,7 @@ pub const KvCache = struct { }; } - pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !GatedDeltaNetCache.Buffers { + pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !GatedDeltaNetCache.Buffers { return .{ .conv_state = try zml.Buffer.uninitialized(io, platform, self.conv_state.shape(), sharding, .{}), .recurrent_state = try zml.Buffer.uninitialized(io, platform, self.recurrent_state.shape(), sharding, .{}), @@ -1047,7 +1047,7 @@ pub const KvCache = struct { }; } - pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !zml.Bufferized(KvCache) { + pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(KvCache) { return .{ .self_attn = try self.self_attn.initBuffer(io, platform, sharding), .gated_delta_net = try self.gated_delta_net.initBuffer(io, platform, sharding), diff --git a/examples/llm/models/qwen3_5/session.zig b/examples/llm/models/qwen3_5/session.zig index 578a290f..56a87373 100644 --- a/examples/llm/models/qwen3_5/session.zig +++ b/examples/llm/models/qwen3_5/session.zig @@ -30,12 +30,11 @@ pub const Session = struct { compiled_model: *const inference.CompiledModel, model_buffers: *model.Buffers, ) !Session { - const replicated_sharding = compiled_model.params.shardings.replicated; var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform, compiled_model.params.shardings.model); errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); - var rng_buffers = try zml.Tensor.Rng.initBuffer(io, platform, replicated_sharding, seed); + var rng_buffers = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); return .{ @@ -79,12 +78,10 @@ pub const Session = struct { @memset(prefill_tokens_slice.items(u32), 0); @memcpy(prefill_tokens_slice.items(u32)[0..all_tokens.len], all_tokens); - const replicated_sharding = self.compiled_model.params.shardings.replicated; - - var prefill_tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, prefill_tokens_slice, replicated_sharding); + var prefill_tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, prefill_tokens_slice, .replicated); defer prefill_tokens_buffer.deinit(); - var prefill_token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 0), .u32, replicated_sharding); + var prefill_token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, 0, .u32); defer prefill_token_index_buffer.deinit(); var args = try self.compiled_model.prefill_exe.args(self.allocator); @@ -142,12 +139,10 @@ pub const Session = struct { fn runDecodeStep(self: *Session, token_id: u32, token_index: u32) !void { self.step_token_slice.items(u32)[0] = token_id; - const replicated_sharding = self.compiled_model.params.shardings.replicated; - - var token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.step_token_slice, replicated_sharding); + var token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.step_token_slice, .replicated); defer token_buffer.deinit(); - var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, token_index, .u32, replicated_sharding); + var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, token_index, .u32); defer token_index_buffer.deinit(); var args = try self.compiled_model.decode_exe.args(self.allocator); diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index c3f7c01a..718a2dd0 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -19,8 +19,8 @@ const Mnist = struct { pub fn init(store: zml.io.TensorStore.View) Layer { return .{ - .weight = store.createTensor("weight", .{ .d_out, .d }, null), - .bias = store.createTensor("bias", .{.d_out}, null), + .weight = store.createTensor("weight", .{ .d_out, .d }, .replicated), + .bias = store.createTensor("bias", .{.d_out}, .replicated), }; } @@ -42,10 +42,8 @@ const Mnist = struct { io: std.Io, platform: *const zml.Platform, store: *const zml.io.TensorStore, - shardings: []const zml.sharding.Sharding, ) !zml.Bufferized(Mnist) { return zml.io.load(Mnist, self, allocator, io, platform, store, .{ - .shardings = shardings, .parallelism = 1, .dma_chunks = 1, .dma_chunk_size = 16 * 1024 * 1024, @@ -93,15 +91,13 @@ pub fn main(init: std.process.Init) !void { const platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator, io); - const replicated_sharding = try zml.sharding.replicatedSharding(platform); - // // Compile model const input: zml.Tensor = .init(.{ 28, 28 }, .u8); var exe = blk: { log.info("Compiling model....", .{}); const start: std.Io.Timestamp = .now(io, .awake); defer log.info("✅ Compiled model [{f}]", .{start.untilNow(io, .awake)}); - break :blk try platform.compile(allocator, io, mnist_model, .forward, .{input}, .{ .shardings = &.{replicated_sharding} }); + break :blk try platform.compile(allocator, io, mnist_model, .forward, .{input}, .{}); }; defer exe.deinit(); @@ -112,7 +108,7 @@ pub fn main(init: std.process.Init) !void { defer log.info("✅ Transferred weights [{f}]", .{ start.untilNow(io, .awake), }); - break :blk try mnist_model.load(init.arena.allocator(), io, platform, &store, &.{replicated_sharding}); + break :blk try mnist_model.load(init.arena.allocator(), io, platform, &store); }; defer Mnist.unloadBuffers(&mnist_buffers); @@ -136,7 +132,7 @@ pub fn main(init: std.process.Init) !void { var sample: [28 * 28]u8 align(16) = undefined; _ = try dataset.readPositionalAll(io, &sample, 16 + (idx * 28 * 28)); - var input_buffer: zml.Buffer = try .fromSlice(io, platform, zml.Slice.init(input.shape(), &sample), replicated_sharding); + var input_buffer: zml.Buffer = try .fromSlice(io, platform, zml.Slice.init(input.shape(), &sample), .replicated); defer input_buffer.deinit(); printDigit(sample); diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index d31eb91e..097fd1cd 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -17,7 +17,7 @@ const CliMesh = enum { }; const CliArgs = struct { - partitioner: zml.sharding.Partitioning.Partitioner = .shardy, + partitioner: zml.Sharding.Partitioner = .shardy, mesh: CliMesh = .auto, }; @@ -112,9 +112,9 @@ fn buildMockMesh( allocator: std.mem.Allocator, target: zml.Target, devices: []const zml.platform.Device, -) !zml.sharding.PhysicalMesh { +) !zml.Sharding.PhysicalMesh { if (devices.len < 8) return error.NotEnoughDevicesForMockMesh; - const topology: zml.sharding.PhysicalMesh.Tree = .axis(.link_x, .{ .mesh = .torus }, &.{ + const topology: zml.Sharding.PhysicalMesh.Tree = .axis(.link_x, .{ .mesh = .torus }, &.{ .axis(.link_y, .{ .mesh = .torus }, &.{ .axis(.link_z, .{ .mesh = .torus }, &.{ .device(devices[3]), .device(devices[1]), @@ -132,7 +132,7 @@ fn buildMockMesh( }), }), }); - return zml.sharding.PhysicalMesh.fromTree(allocator, target, topology); + return zml.Sharding.PhysicalMesh.fromTree(allocator, target, topology); } fn createSequenceBuffer( @@ -140,7 +140,7 @@ fn createSequenceBuffer( io: std.Io, platform: *const zml.Platform, shape: zml.Shape, - sharding: zml.sharding.Sharding, + sharding: zml.Sharding, start: f32, ) !zml.Buffer { const slice = try zml.Slice.alloc(allocator, shape); @@ -188,15 +188,12 @@ pub fn main(init: std.process.Init) !void { log.info("Partitioner: {s}", .{@tagName(args.partitioner)}); log.info("{f}", .{platform.physical_mesh}); - const physical_mesh = platform.physical_mesh; - const mesh: zml.sharding.LogicalMesh = .init("demo_mesh", .{ - .data = .low_bandwidth, - .model = .high_bandwidth, - }); - const strategy: zml.sharding.Strategy = .suggest(mesh, physical_mesh); - const sharding: zml.sharding.Sharding = try .initFromStrategy(platform, mesh, strategy); + const sharding: zml.Sharding = try platform.registerSharding( + "demo_mesh", + .mesh(.{ .data = .low_bandwidth, .model = .high_bandwidth }), + ); - log.info("{f}", .{mesh}); + log.info("{f}", .{sharding.data.logical}); log.info("{f}", .{sharding}); const input_shape = zml.Shape.init(.{ .batch = 16, .feature = 32 }, .f32) diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 4daa80ef..175b91f3 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -50,7 +50,7 @@ zig_library( "mlirx.zig", "module.zig", "nn.zig", - "sharding.zig", + "Sharding.zig", "ops.zig", "pjrtx.zig", "platform.zig", diff --git a/zml/sharding.zig b/zml/Sharding.zig similarity index 89% rename from zml/sharding.zig rename to zml/Sharding.zig index 3d7f8a10..c284106d 100644 --- a/zml/sharding.zig +++ b/zml/Sharding.zig @@ -1,3 +1,4 @@ +//! Explain how to split a buffer across different devices. const std = @import("std"); const pjrt = @import("pjrt"); @@ -12,19 +13,40 @@ const Target = @import("platform.zig").Target; const log = std.log.scoped(.@"zml/sharding"); +const Sharding = @This(); + +data: *const Data, + +var _replicated: [11]u8 align(@alignOf(Data)) = "_replicated".*; + +// special value to make public apis more fluent. +pub const replicated: Sharding = .{ .data = @ptrCast(&_replicated) }; + +pub fn resolve(sharding: Sharding, platform: *const Platform) Sharding { + return if (sharding.data == replicated.data) platform.replicated_sharding else sharding; +} + +pub fn numPartitionsForLogicalAxis(sharding: Sharding, logical_axis: anytype) i64 { + return sharding.data.numPartitionsForLogicalAxis(logical_axis); +} + +pub fn format(sharding: Sharding, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return sharding.data.format(writer); +} + +pub const Partitioner = union(enum) { + shardy, + gspmd, + + pub fn fromTarget(target: Target) Partitioner { + return switch (target) { + .cpu, .cuda, .rocm, .tpu => .shardy, + .neuron => .gspmd, + }; + } +}; + pub const Partitioning = struct { - pub const Partitioner = union(enum) { - shardy, - gspmd, - - pub fn fromTarget(target: Target) Partitioner { - return switch (target) { - .cpu, .cuda, .rocm, .tpu => .shardy, - .neuron => .gspmd, - }; - } - }; - partitioner: Partitioner, shardings: []const Sharding, @@ -33,11 +55,11 @@ pub const Partitioning = struct { var plan: Partitioning = .{ .partitioner = partitioner, .shardings = shardings }; const first = plan.primarySharding(); - const partitions = first.numPartitions(); - const replicas = first.numReplicas(); + const partitions = first.data.numPartitions(); + const replicas = first.data.numReplicas(); for (shardings[1..]) |s| { - if (s.numPartitions() != partitions or s.numReplicas() != replicas) { + if (s.data.numPartitions() != partitions or s.data.numReplicas() != replicas) { // todo: deviceAssignments should also be checked for consistency here, but for simplicity we just check the cardinality numbers return error.InconsistentShardingCardinality; } @@ -48,12 +70,12 @@ pub const Partitioning = struct { pub fn numPartitions(self: Partitioning) i32 { const primary = self.primarySharding(); - return primary.numPartitions(); + return primary.data.numPartitions(); } pub fn numReplicas(self: Partitioning) i32 { const primary = self.primarySharding(); - return primary.numReplicas(); + return primary.data.numReplicas(); } pub fn numDevices(self: Partitioning) i32 { @@ -64,7 +86,7 @@ pub const Partitioning = struct { return switch (self.partitioner) { .shardy, .gspmd => blk: { const sharding = self.primarySharding(); - break :blk sharding.deviceAssignment(allocator); + break :blk sharding.data.deviceAssignment(allocator); }, }; } @@ -72,8 +94,8 @@ pub const Partitioning = struct { pub fn tensorShardingAttr(self: Partitioning, allocator: std.mem.Allocator, shape: Shape, sharding: ?Sharding) ![]const u8 { const selected_sharding = sharding orelse try self.selectSharding(shape); return switch (self.partitioner) { - .shardy => try selected_sharding.sdyShardingAttrForShape(allocator, shape), - .gspmd => try selected_sharding.gspmdShardingAttrForShape(allocator, shape), + .shardy => try selected_sharding.data.sdyShardingAttrForShape(allocator, shape), + .gspmd => try selected_sharding.data.gspmdShardingAttrForShape(allocator, shape), }; } @@ -85,7 +107,7 @@ pub const Partitioning = struct { pub fn numPartitionsForLogicalAxis(self: Partitioning, shape: Shape, logical_axis: anytype) !i64 { const sharding = try self.selectSharding(shape); - return sharding.numPartitionsForLogicalAxis(logical_axis); + return sharding.data.numPartitionsForLogicalAxis(logical_axis); } pub fn sdyPerValueShardingAttr(self: Partitioning, allocator: std.mem.Allocator, shapes: []const Shape) ![]const u8 { @@ -165,7 +187,7 @@ pub fn pickSharding(shardings: []const Sharding, shape: Shape, mode: SelectShard if (mode == .explicit_axis_binding and !shapeHasAxisPartition(shape)) return null; for (shardings) |sharding| { - if (shardingCoversShape(sharding, shape)) return sharding; + if (sharding.data.covers(shape)) return sharding; } return null; } @@ -177,16 +199,6 @@ fn shapeHasAxisPartition(shape: Shape) bool { return false; } -fn shardingCoversShape(sharding: Sharding, shape: Shape) bool { - for (0..shape.rank()) |ax| { - switch (shape.partition(ax)) { - .axis => |tag| if (sharding.binding(tag) == null) return false, - else => {}, - } - } - return true; -} - /// Device is the leaf representation in a PhysicalMesh. /// It carries identity, optional coordinates, and PJRT handle. pub const Device = struct { @@ -955,11 +967,10 @@ pub const LogicalMesh = struct { pub const Axes = stdx.BoundedArray(Shape.Tag, Shape.MAX_RANK); pub const Intents = stdx.BoundedArray(LogicalAxisIntent, Shape.MAX_RANK); - name: []const u8, axes: Axes, intents: Intents, - pub fn init(name: []const u8, axes_: anytype) LogicalMesh { + pub fn mesh(axes_: anytype) LogicalMesh { const T = @TypeOf(axes_); var axes: Axes = .empty; @@ -976,7 +987,6 @@ pub const LogicalMesh = struct { } return .{ - .name = name, .axes = axes, .intents = intents, }; @@ -1003,62 +1013,63 @@ pub const LogicalMesh = struct { } pub fn format(self: LogicalMesh, writer: *std.Io.Writer) !void { - try writer.print("LogicalMesh(name={s}", .{self.name}); + try writer.writeAll("LogicalMesh("); for (self.axes.constSlice(), self.intents.constSlice()) |axis, int| { try writer.print(" {s}={s}", .{ axis, @tagName(int) }); } - try writer.print(")", .{}); + try writer.writeAll(")"); } }; -pub const Sharding = struct { - pub const AxisList = stdx.BoundedArray(PhysicalAxisTag, Shape.MAX_RANK); - pub const Binding = struct { - logical: Shape.Tag, - physical: AxisList, - }; - pub const Bindings = stdx.BoundedArray(Binding, Shape.MAX_RANK); +pub const AxisList = stdx.BoundedArray(PhysicalAxisTag, Shape.MAX_RANK); +pub const Binding = struct { + logical: Shape.Tag, + physical: AxisList, +}; +pub const Bindings = stdx.BoundedArray(Binding, Shape.MAX_RANK); - pub const Fold = struct { - target: PhysicalAxisTag, - sources: AxisList, - }; - pub const Folds = stdx.BoundedArray(Fold, Shape.MAX_RANK); +pub const Fold = struct { + target: PhysicalAxisTag, + sources: AxisList, +}; +pub const Folds = stdx.BoundedArray(Fold, Shape.MAX_RANK); - pub const Axis = struct { - tag: PhysicalAxisTag, - size: i64, - geometry: ?AxisGeometry, - folded: stdx.BoundedArray(PhysicalAxisTag, Shape.MAX_RANK), +pub const Axis = struct { + tag: PhysicalAxisTag, + size: i64, + geometry: ?AxisGeometry, + folded: stdx.BoundedArray(PhysicalAxisTag, Shape.MAX_RANK), - pub fn contains(self: *const Axis, tag: PhysicalAxisTag) bool { - if (self.tag == tag) return true; - for (self.folded.constSlice()) |t| if (t == tag) return true; - return false; + pub fn contains(self: *const Axis, tag: PhysicalAxisTag) bool { + if (self.tag == tag) return true; + for (self.folded.constSlice()) |t| if (t == tag) return true; + return false; + } +}; + +pub const PhysicalView = struct { + axes: stdx.BoundedArray(Axis, Shape.MAX_RANK), + total_devices: i64, + + pub fn axisCoordFromLinearShard(self: *const PhysicalView, axis_index: usize, linear_idx: usize) usize { + const stride = self.axisStrideForLinear(axis_index); + const axis_size: usize = @intCast(self.axes.constSlice()[axis_index].size); + return (linear_idx / stride) % axis_size; + } + + pub fn axisStrideForLinear(self: *const PhysicalView, axis_index: usize) usize { + var stride: usize = 1; + var j = axis_index + 1; + while (j < self.axes.len) : (j += 1) { + stride *= @intCast(self.axes.constSlice()[j].size); } - }; + return stride; + } +}; - pub const PhysicalView = struct { - axes: stdx.BoundedArray(Axis, Shape.MAX_RANK), - total_devices: i64, - - pub fn axisCoordFromLinearShard(self: *const PhysicalView, axis_index: usize, linear_idx: usize) usize { - const stride = self.axisStrideForLinear(axis_index); - const axis_size: usize = @intCast(self.axes.constSlice()[axis_index].size); - return (linear_idx / stride) % axis_size; - } - - pub fn axisStrideForLinear(self: *const PhysicalView, axis_index: usize) usize { - var stride: usize = 1; - var j = axis_index + 1; - while (j < self.axes.len) : (j += 1) { - stride *= @intCast(self.axes.constSlice()[j].size); - } - return stride; - } - }; - - platform: *const Platform, +pub const Data = struct { + name: []const u8, + physical: PhysicalMesh, logical: LogicalMesh, /// Compact binding table: logical axis -> physical axes @@ -1068,7 +1079,7 @@ pub const Sharding = struct { folds: Folds, folds_consumed: std.EnumSet(PhysicalAxisTag), - pub fn binding(self: *const Sharding, tag: Shape.Tag) ?[]const PhysicalAxisTag { + pub fn binding(self: *const Data, tag: Shape.Tag) ?[]const PhysicalAxisTag { const target = std.mem.span(tag); for (self.bindings.constSlice()) |*b| { if (std.mem.eql(u8, std.mem.span(b.logical), target)) return b.physical.constSlice(); @@ -1076,12 +1087,12 @@ pub const Sharding = struct { return null; } - pub fn initFromStrategy( - platform: *const Platform, + pub fn init( + name: []const u8, + physical: PhysicalMesh, logical: LogicalMesh, strategy: Strategy, - ) !Sharding { - const physical = platform.physical_mesh; + ) !Data { const axis_order = physical.axisOrder().constSlice(); if (axis_order.len == 0) return error.InvalidPhysicalMesh; @@ -1101,23 +1112,24 @@ pub const Sharding = struct { } return .{ + .name = name, .logical = logical, - .platform = platform, + .physical = physical, .bindings = bindings, .folds = folds, .folds_consumed = folds_consumed, }; } - fn foldSources(self: *const Sharding, tag: PhysicalAxisTag) ?[]const PhysicalAxisTag { + fn foldSources(self: *const Data, tag: PhysicalAxisTag) ?[]const PhysicalAxisTag { for (self.folds.constSlice()) |*f| { if (f.target == tag) return f.sources.constSlice(); } return null; } - pub fn physicalView(self: Sharding) PhysicalView { - const physical = self.platform.physical_mesh; + pub fn physicalView(self: *const Data) PhysicalView { + const physical = self.physical; var view: PhysicalView = .{ .axes = .empty, .total_devices = 1, @@ -1154,19 +1166,15 @@ pub const Sharding = struct { return view; } - pub fn name(self: Sharding) []const u8 { - return self.logical.name; + pub fn logicalIndexFromCoords(self: *const Data, coords: []const usize) usize { + return self.physical.linearIndexFromCoords(coords); } - pub fn logicalIndexFromCoords(self: Sharding, coords: []const usize) usize { - return self.platform.physical_mesh.linearIndexFromCoords(coords); - } - - pub fn numPartitions(self: Sharding) i32 { + pub fn numPartitions(self: *const Data) i32 { return @intCast(self.physicalView().total_devices); } - pub fn numPartitionsForLogicalAxis(self: Sharding, logical_axis: anytype) i64 { + pub fn numPartitionsForLogicalAxis(self: *const Data, logical_axis: anytype) i64 { const logical_tag = Shape.toTag(logical_axis); const bound_axes = self.binding(logical_tag) orelse return 1; @@ -1182,21 +1190,31 @@ pub const Sharding = struct { var partitions: i64 = 1; for (std.enums.values(PhysicalAxisTag)) |physical_axis| { if (physical_axes.contains(physical_axis)) { - partitions *= self.platform.physical_mesh.axis(physical_axis); + partitions *= self.physical.axis(physical_axis); } } return partitions; } - pub fn numReplicas(_: Sharding) i32 { + pub fn numReplicas(_: *const Data) i32 { return 1; } - pub fn numDevices(self: Sharding) i32 { + pub fn numDevices(self: *const Data) i32 { return self.numPartitions() * self.numReplicas(); } - pub fn sdyMeshAttr(self: Sharding, allocator: std.mem.Allocator) ![]const u8 { + pub fn covers(sharding: *const Data, shape: Shape) bool { + for (shape._partitioning.constSlice()) |partitioning| { + switch (partitioning) { + .axis => |tag| if (sharding.binding(tag) == null) return false, + else => {}, + } + } + return true; + } + + pub fn sdyMeshAttr(self: *const Data, allocator: std.mem.Allocator) ![]const u8 { var out: std.Io.Writer.Allocating = .init(allocator); errdefer out.deinit(); @@ -1220,7 +1238,7 @@ pub const Sharding = struct { }; /// Common logic to map tensor dimensions to physical mesh indices. - fn getDimMapping(self: *const Sharding, shape: Shape) DimMapping { + fn getDimMapping(self: *const Data, shape: Shape) DimMapping { const view = self.physicalView(); var axes_per_dim: stdx.BoundedArray(stdx.BoundedArray(usize, Shape.MAX_RANK), Shape.MAX_RANK) = .empty; var used_mask: [Shape.MAX_RANK]bool = @splat(false); @@ -1259,7 +1277,8 @@ pub const Sharding = struct { }; } - pub fn sdyShardingAttrForShape(self: Sharding, allocator: std.mem.Allocator, shape: Shape) ![]const u8 { + // TODO: consider binding shardy dialect instead of generating strings then parsing later. + pub fn sdyShardingAttrForShape(self: *const Data, allocator: std.mem.Allocator, shape: Shape) ![]const u8 { var any_explicit = false; for (0..shape.rank()) |ax| { if (shape.partition(ax) != .unknown) { @@ -1273,7 +1292,7 @@ pub const Sharding = struct { var out: std.Io.Writer.Allocating = .init(allocator); errdefer out.deinit(); - try out.writer.print("#sdy.sharding<@{s}, [", .{self.name()}); + try out.writer.print("#sdy.sharding<@{s}, [", .{self.name}); for (0..shape.rank()) |ax| { if (ax > 0) try out.writer.writeAll(", "); const spec = shape.partition(ax); @@ -1320,7 +1339,7 @@ pub const Sharding = struct { return try out.toOwnedSlice(); } - pub fn gspmdShardingAttrForShape(self: Sharding, allocator: std.mem.Allocator, shape: Shape) ![]const u8 { + pub fn gspmdShardingAttrForShape(self: *const Data, allocator: std.mem.Allocator, shape: Shape) ![]const u8 { var has_sharding = false; for (0..shape.rank()) |ax| { if (shape.partition(ax) == .axis) { @@ -1384,7 +1403,7 @@ pub const Sharding = struct { return try out.toOwnedSlice(); } - pub fn deviceAssignment(self: Sharding, allocator: std.mem.Allocator) ![]usize { + pub fn deviceAssignment(self: *const Data, allocator: std.mem.Allocator) ![]usize { const view = self.physicalView(); const count: usize = @intCast(view.total_devices); @@ -1394,7 +1413,7 @@ pub const Sharding = struct { const ordered_devices = self.devicesInCanonicalOrder(); for (ordered_devices.constSlice()) |d| { const coords = d.coordsSlice() orelse return error.MissingDeviceCoords; - const idx = self.platform.physical_mesh.linearIndexFromCoords(coords); + const idx = self.physical.linearIndexFromCoords(coords); ids[idx] = d.id; } @@ -1405,8 +1424,8 @@ pub const Sharding = struct { return ids; } - fn devicesInCanonicalOrder(self: Sharding) Devices { - const physical = self.platform.physical_mesh; + fn devicesInCanonicalOrder(self: *const Data) Devices { + const physical = self.physical; var devices: Devices = physical.devices(); const Order = struct { @@ -1424,8 +1443,8 @@ pub const Sharding = struct { return devices; } - pub fn format(self: Sharding, writer: *std.Io.Writer) !void { - try writer.print("Sharding(name={s})\n", .{self.logical.name}); + pub fn format(self: Data, writer: *std.Io.Writer) !void { + try writer.print("Sharding(name={s})\n", .{self.name}); try writer.writeAll("Bindings:\n"); for (self.logical.axes.constSlice(), self.logical.intents.constSlice()) |l_tag, l_intent| { @@ -1477,27 +1496,20 @@ pub const Sharding = struct { } }; -pub fn replicatedSharding(platform: *const Platform) !Sharding { - const physical_mesh = platform.physical_mesh; - const logical_mesh: LogicalMesh = .init("replicated", .{ .x = .high_bandwidth }); - const strategy: Strategy = .suggest(logical_mesh, physical_mesh); - return try .initFromStrategy(platform, logical_mesh, strategy); -} - pub const Strategy = struct { pub const PhysicalList = stdx.BoundedArray(PhysicalAxisTag, Shape.MAX_RANK); pub const Binding = struct { logical: Shape.Tag, physical: PhysicalList, }; - pub const Bindings = stdx.BoundedArray(Binding, Shape.MAX_RANK); + pub const Bindings = stdx.BoundedArray(Strategy.Binding, Shape.MAX_RANK); pub const Fold = struct { target: PhysicalAxisTag, sources: PhysicalList, }; - pub const Folding = stdx.BoundedArray(Fold, Shape.MAX_RANK); + pub const Folding = stdx.BoundedArray(Strategy.Fold, Shape.MAX_RANK); - bindings: Bindings, + bindings: Strategy.Bindings, folding: Folding, pub const init: Strategy = .{ @@ -1677,7 +1689,7 @@ pub const Placement = struct { sharding: Sharding, shape: Shape, ) !Placement { - const ordered_devices = sharding.devicesInCanonicalOrder(); + const ordered_devices = sharding.data.devicesInCanonicalOrder(); var shards: Shards = .empty; for (ordered_devices.constSlice()) |d| { @@ -1730,7 +1742,7 @@ pub const Placement = struct { switch (spec) { .axis => |logical_tag| { - const binding = self.sharding.binding(logical_tag) orelse return error.MissingLogicalBinding; + const binding = self.sharding.data.binding(logical_tag) orelse return error.MissingLogicalBinding; // Calculate the split based on the physical coordinates of the current device const plan = try self.calculateSplit(dim, binding, device_coords, used_axes); @@ -1759,7 +1771,7 @@ pub const Placement = struct { for (binding) |p_tag| { // Expand the physical tag: check if it's a target for folded source axes - if (self.sharding.foldSources(p_tag)) |sources| { + if (self.sharding.data.foldSources(p_tag)) |sources| { for (sources) |src| { if (used_axes.contains(src)) continue; self.addPhysicalToSplit(&plan, src, device_coords); @@ -1782,7 +1794,7 @@ pub const Placement = struct { /// Extract coordinate data from the physical mesh for a specific axis. fn addPhysicalToSplit(self: *Placement, plan: *AxisSplit, tag: PhysicalAxisTag, device_coords: []const usize) void { - const physical = self.sharding.platform.physical_mesh; + const physical = self.sharding.data.physical; const info = physical.axisInfo(tag) orelse return; const depth = physical.axis_traversal.depth(tag) orelse return; const coord = device_coords[depth]; @@ -1834,9 +1846,7 @@ const ShardingTest = struct { }; pub const Scenario = struct { - platform: Platform, - logical: LogicalMesh, - strategy: Strategy, + sharding: Sharding.Data, shape: Shape, expected_sdy: ?[]const u8 = null, @@ -1880,21 +1890,8 @@ const ShardingTest = struct { return .{ .branch = .{ .tag = tags[depth], .geometry = geometry, .children = children } }; } - fn platformForMesh(self: ShardingTest, mesh: PhysicalMesh) Platform { - _ = self; - return .{ - .arena_state = .{}, - .target = mesh.target, - .pjrt_api = undefined, - .pjrt_client = undefined, - .devices = &.{}, - .memories = &.{}, - .physical_mesh = mesh, - }; - } - pub fn run(self: ShardingTest, s: Scenario) !void { - const sharding = try Sharding.initFromStrategy(&s.platform, s.logical, s.strategy); + const sharding = s.sharding; // Verify MLIR String if (s.expected_sdy) |expected_attr| { @@ -1905,21 +1902,19 @@ const ShardingTest = struct { // Verify Placement logic / Error if (s.expect_error) |err| { - try std.testing.expectError(err, Placement.init(sharding, s.shape)); + try std.testing.expectError(err, Placement.init(.{ .data = &sharding }, s.shape)); return; } // Verify Shard Slices (Math) if (s.expected_shards.len > 0) { - const placement = try Placement.init(sharding, s.shape); + const placement = try Placement.init(.{ .data = &sharding }, s.shape); try std.testing.expectEqual(s.expected_shards.len, placement.shards.len); - for (s.expected_shards, 0..) |expected, i| { - const actual = placement.shards.constSlice()[i]; + for (s.expected_shards, placement.shards.constSlice()) |expected, actual| { try std.testing.expectEqual(expected.device_id, actual.device_id); - for (expected.slices, 0..) |exp_slice, dim| { - const act_slice = actual.slices.constSlice()[dim]; - try std.testing.expectEqual(exp_slice[0], act_slice.start); - try std.testing.expectEqual(exp_slice[1], act_slice.size); + for (expected.slices, actual.slices.constSlice()) |exp_slice, actual_slice| { + try std.testing.expectEqual(exp_slice[0], actual_slice.start); + try std.testing.expectEqual(exp_slice[1], actual_slice.size); } } } @@ -1932,15 +1927,18 @@ test "sharding: unknown partitioning implies replication" { const runner: ShardingTest = .init(arena.allocator()); const physical: PhysicalMesh = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); - const logical: LogicalMesh = .init("folded_mesh", .{ .batch = .low_bandwidth }); + const logical: LogicalMesh = .mesh(.{ .batch = .low_bandwidth }); var strategy: Strategy = .init; strategy.addBinding(.batch, .link_x); const scenario: ShardingTest.Scenario = .{ - .platform = runner.platformForMesh(physical), - .logical = logical, - .strategy = strategy, + .sharding = try .init( + "folded_mesh", + physical, + logical, + strategy, + ), .shape = Shape.init(.{ .batch = 8 }, .f32), .expected_sdy = "#sdy.sharding<@folded_mesh, [{}], replicated={\"link_x\", \"link_y\"}>", .expected_shards = &.{ @@ -1960,7 +1958,7 @@ test "sharding: suggest strategy realization" { const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); - const logical: LogicalMesh = .init("suggested_mesh", .{ + const logical: LogicalMesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth, }); @@ -1968,9 +1966,7 @@ test "sharding: suggest strategy realization" { const strategy: Strategy = .suggest(logical, physical); const scenario: ShardingTest.Scenario = .{ - .platform = runner.platformForMesh(physical), - .logical = logical, - .strategy = strategy, + .sharding = try .init("suggested_mesh", physical, logical, strategy), .shape = Shape.init(.{ .batch = 4, .model = 4 }, .f32) .withPartitioning(.{ .batch = .batch, .model = .model }), .expected_sdy = "#sdy.sharding<@suggested_mesh, [{\"link_z\"}, {\"link_x\"}], replicated={\"link_y\"}>", @@ -1995,7 +1991,7 @@ test "sharding: suggest folds logical axes with same intent" { const physical = try runner.physical(.{4}, .point_to_point); - const logical: LogicalMesh = .init("fold_mesh", .{ + const logical: LogicalMesh = .mesh(.{ .model = .high_bandwidth, .experts = .high_bandwidth, }); @@ -2003,9 +1999,7 @@ test "sharding: suggest folds logical axes with same intent" { const strategy: Strategy = .suggest(logical, physical); const scenario: ShardingTest.Scenario = .{ - .platform = runner.platformForMesh(physical), - .logical = logical, - .strategy = strategy, + .sharding = try .init("fold_mesh", physical, logical, strategy), .shape = Shape.init(.{ .model = 4, .experts = 4 }, .f32) .withPartitioning(.{ .model = .model, .experts = .experts }), .expected_sdy = "#sdy.sharding<@fold_mesh, [{\"link_x\"}, {}]>", @@ -2025,16 +2019,14 @@ test "sharding: multiple physical axes on one logical dimension (folding)" { const runner: ShardingTest = .init(arena.allocator()); const physical: PhysicalMesh = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); - const logical: LogicalMesh = .init("folded_mesh", .{ .model = .high_bandwidth }); + const logical: LogicalMesh = .mesh(.{ .model = .high_bandwidth }); var strategy: Strategy = .init; strategy.addBinding(.model, .link_x); strategy.addBinding(.model, .link_y); const scenario: ShardingTest.Scenario = .{ - .platform = runner.platformForMesh(physical), - .logical = logical, - .strategy = strategy, + .sharding = try .init("folded_mesh", physical, logical, strategy), .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(.{ .model = .model }), .expected_sdy = "#sdy.sharding<@folded_mesh, [{\"link_x\", \"link_y\"}]>", .expected_shards = &.{ @@ -2053,16 +2045,14 @@ test "sharding: explicit strategy folding" { const runner: ShardingTest = .init(arena.allocator()); const physical: PhysicalMesh = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); - const logical: LogicalMesh = .init("strategy_fold", .{ .model = .high_bandwidth }); + const logical: LogicalMesh = .mesh(.{ .model = .high_bandwidth }); var strategy: Strategy = .init; strategy.addBinding(.model, .link_x); strategy.addFold(.link_x, &.{ .link_x, .link_z }); const scenario: ShardingTest.Scenario = .{ - .platform = runner.platformForMesh(physical), - .logical = logical, - .strategy = strategy, + .sharding = try .init("strategy_fold", physical, logical, strategy), .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(.{ .model = .model }), .expected_sdy = "#sdy.sharding<@strategy_fold, [{\"link_x\"}], replicated={\"link_y\"}>", .expected_shards = &.{ @@ -2085,16 +2075,14 @@ test "sharding: open and replicated dimension mix" { const runner: ShardingTest = .init(arena.allocator()); const physical: PhysicalMesh = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); - const logical: LogicalMesh = .init("mix_mesh", .{ .batch = .low_bandwidth, .model = .high_bandwidth }); + const logical: LogicalMesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth }); var strategy: Strategy = .init; strategy.addBinding(.batch, .link_x); strategy.addBinding(.model, .link_y); const scenario: ShardingTest.Scenario = .{ - .platform = runner.platformForMesh(physical), - .logical = logical, - .strategy = strategy, + .sharding = try .init("mix_mesh", physical, logical, strategy), .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(.{ .batch = .open, .model = .replicated }), .expected_sdy = "#sdy.sharding<@mix_mesh, [{?}, {}], replicated={\"link_x\", \"link_y\"}>", .expected_shards = &.{ @@ -2113,7 +2101,7 @@ test "sharding: full 3D cluster sharding" { const runner: ShardingTest = .init(arena.allocator()); const physical: PhysicalMesh = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); - const logical: LogicalMesh = .init("3d_mesh", .{ .batch = .low_bandwidth, .model = .high_bandwidth, .context = .balanced }); + const logical: LogicalMesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth, .context = .balanced }); var strategy: Strategy = .init; strategy.addBinding(.batch, .link_x); @@ -2121,9 +2109,7 @@ test "sharding: full 3D cluster sharding" { strategy.addBinding(.context, .link_z); const scenario: ShardingTest.Scenario = .{ - .platform = runner.platformForMesh(physical), - .logical = logical, - .strategy = strategy, + .sharding = try .init("3d_mesh", physical, logical, strategy), .shape = Shape.init(.{ .batch = 4, .model = 4, .context = 4 }, .f32) .withPartitioning(.{ .batch = .batch, .model = .model, .context = .context }), .expected_sdy = "#sdy.sharding<@3d_mesh, [{\"link_x\"}, {\"link_y\"}, {\"link_z\"}]>", @@ -2147,7 +2133,7 @@ test "sharding: num partitions for logical axis" { const runner: ShardingTest = .init(arena.allocator()); const physical: PhysicalMesh = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); - const logical: LogicalMesh = .init("axis_parts_mesh", .{ + const logical: LogicalMesh = .mesh(.{ .model = .high_bandwidth, .batch = .low_bandwidth, }); @@ -2156,8 +2142,7 @@ test "sharding: num partitions for logical axis" { strategy.addBinding(.model, .link_x); strategy.addFold(.link_x, &.{ .link_x, .link_z }); - var platform = runner.platformForMesh(physical); - const sharding: Sharding = try .initFromStrategy(&platform, logical, strategy); + const sharding: Sharding.Data = try .init("axis_parts_mesh", physical, logical, strategy); try std.testing.expectEqual(4, sharding.numPartitionsForLogicalAxis(.model)); try std.testing.expectEqual(1, sharding.numPartitionsForLogicalAxis(.batch)); diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index c50756ba..8809aaa7 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -84,7 +84,7 @@ pub const Metadata = union(Backend) { }; } - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !zml.Bufferized(Metadata) { + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(Metadata) { return switch (self) { .vanilla => .{ .vanilla = {} }, .cuda_fa2 => |v| .{ .cuda_fa2 = try v.initBuffer(io, platform, sharding) }, diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index eae20e04..24d03906 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -248,7 +248,7 @@ pub const fa2 = struct { }; } - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !zml.Bufferized(Metadata) { + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(Metadata) { return .{ .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), sharding, .{}), .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), sharding, .{}), @@ -444,7 +444,7 @@ pub const fa3 = struct { }; } - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.sharding.Sharding) !zml.Bufferized(Metadata) { + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(Metadata) { return .{ .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), sharding, .{}), .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), sharding, .{}), @@ -616,7 +616,7 @@ pub const paged_fa2 = struct { }; } - pub fn initBuffers(self: *const DecodeMetadata, io: std.Io, platform: zml.Platform, out_accum_sharding: zml.sharding.Sharding) !zml.Bufferized(DecodeMetadata) { + pub fn initBuffers(self: *const DecodeMetadata, io: std.Io, platform: zml.Platform, out_accum_sharding: zml.Sharding) !zml.Bufferized(DecodeMetadata) { return .{ .out_accum = try zml.Buffer.uninitialized(io, &platform, self.out_accum.shape(), out_accum_sharding, .{}), }; @@ -705,7 +705,7 @@ pub const paged_fa2 = struct { }; } - pub fn initBuffers(self: *const MixedMetadata, io: std.Io, platform: zml.Platform, out_accum_sharding: zml.sharding.Sharding) !zml.Bufferized(MixedMetadata) { + pub fn initBuffers(self: *const MixedMetadata, io: std.Io, platform: zml.Platform, out_accum_sharding: zml.Sharding) !zml.Bufferized(MixedMetadata) { return .{ .out_accum = try zml.Buffer.uninitialized(io, &platform, self.out_accum.shape(), out_accum_sharding, .{}), .host_metadata = undefined, @@ -1226,7 +1226,7 @@ pub const paged_fa3 = struct { }; } - pub fn initBuffers(self: *const DecodeMetadata, io: std.Io, platform: zml.Platform, out_accum_sharding: zml.sharding.Sharding) !zml.Bufferized(DecodeMetadata) { + pub fn initBuffers(self: *const DecodeMetadata, io: std.Io, platform: zml.Platform, out_accum_sharding: zml.Sharding) !zml.Bufferized(DecodeMetadata) { return .{ .out_accum = try zml.Buffer.uninitialized(io, &platform, self.out_accum.shape(), out_accum_sharding, .{}), }; @@ -1312,7 +1312,7 @@ pub const paged_fa3 = struct { }; } - pub fn initBuffers(self: *const MixedMetadata, io: std.Io, platform: zml.Platform, out_accum_sharding: zml.sharding.Sharding) !zml.Bufferized(MixedMetadata) { + pub fn initBuffers(self: *const MixedMetadata, io: std.Io, platform: zml.Platform, out_accum_sharding: zml.Sharding) !zml.Bufferized(MixedMetadata) { return .{ .out_accum = try zml.Buffer.uninitialized(io, &platform, self.out_accum.shape(), out_accum_sharding, .{}), .host_metadata = undefined, diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index c9104a0d..9f4b6036 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -255,6 +255,9 @@ test "Backend.auto selects mosaic_tpu on TPU" { .devices = &[_]zml.platform.Device{}, .memories = &[_]zml.platform.Memory{}, .physical_mesh = undefined, + .replicated_sharding = undefined, + .triton_runtime = null, + .tpu_ir_runtime = null, }; try std.testing.expectEqual(Backend.mosaic_tpu, Backend.auto(&platform)); diff --git a/zml/buffer.zig b/zml/buffer.zig index 23e25785..f5b33653 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -9,9 +9,7 @@ const Memory = @import("platform.zig").Memory; const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; -const sharding_ = @import("sharding.zig"); -const Sharding = sharding_.Sharding; -const Placement = sharding_.Placement; +const Sharding = @import("Sharding.zig"); const Slice = @import("slice.zig").Slice; const Target = @import("platform.zig").Target; const testing = @import("testing.zig"); @@ -100,7 +98,7 @@ pub const Buffer = struct { var res: Buffer = .{ ._platform = platform, ._shape = sh, - ._sharding = sharding, + ._sharding = sharding.resolve(platform), ._shards = .empty, }; @@ -167,9 +165,9 @@ pub const Buffer = struct { } /// Creates a Buffer with a single element. - pub fn scalar(io: std.Io, platform: *const Platform, val: anytype, dtype_: DataType, sharding: Sharding) !Buffer { + pub fn scalar(io: std.Io, platform: *const Platform, val: anytype, dtype_: DataType) !Buffer { const x = dtype_.constant(val); - return fromBytes(io, platform, .scalar(dtype_), sharding, x.asBytes()); + return fromBytes(io, platform, .scalar(dtype_), .replicated, x.asBytes()); } pub fn await(self: Buffer, io: std.Io) !void { @@ -192,7 +190,7 @@ pub const Buffer = struct { var res: Buffer = .{ ._platform = platform, ._shape = sh, - ._sharding = sharding, + ._sharding = sharding.resolve(platform), ._shards = .empty, }; errdefer for (res._shards.slice()) |shard| { @@ -310,7 +308,7 @@ pub const Buffer = struct { return byte_size; } - pub fn placement(self: Buffer) Placement { - return Placement.init(self._sharding, self._shape) catch unreachable; + pub fn placement(self: Buffer) Sharding.Placement { + return Sharding.Placement.init(self._sharding, self._shape) catch unreachable; } }; diff --git a/zml/exe.zig b/zml/exe.zig index 254921f0..4423a973 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -7,7 +7,7 @@ const Buffer = @import("buffer.zig").Buffer; const meta = @import("meta.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; -const Sharding = @import("sharding.zig").Sharding; +const Sharding = @import("Sharding.zig"); pub const Exe = struct { platform: *const Platform, @@ -43,6 +43,7 @@ pub const Exe = struct { const input_shapes_copy = try arena.allocator().dupe(Shape, input_shapes); const output_shapes_copy = try arena.allocator().dupe(Shape, output_shapes); + // Re-home sharding pointers into arena-owned values so exe doesn't depend on caller lifetimes. const input_shardings_copy = try arena.allocator().dupe(Sharding, input_shardings); const output_shardings_copy = try arena.allocator().dupe(Sharding, output_shardings); @@ -237,19 +238,19 @@ pub const Exe = struct { pub fn fill(self: *Results, v: anytype) void { const LocalContext = struct { - self: *Results, + results: *Results, current_index: usize = 0, }; - var context: LocalContext = .{ .self = self, .current_index = 0 }; + var context: LocalContext = .{ .results = self, .current_index = 0 }; meta.visit(struct { - fn cb(context_: *LocalContext, buffer: *Buffer) void { - //stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected result {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); + fn cb(ctx: *LocalContext, buffer: *Buffer) void { + //stdx.debug.assert(ctx.results.expected_shapes[ctx.current_index].eql(buffer.shape()), "Expected result {} to have shape {f}, got {f}", .{ ctx.current_index, ctx.results.expected_shapes[ctx.current_index], buffer.shape() }); var shards: Buffer.Shards = .empty; - for (0..context_.self.flat_buffers.num_devices) |device_index| { - shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); + for (0..ctx.results.flat_buffers.num_devices) |device_index| { + shards.appendAssumeCapacity(ctx.results.flat_buffers.buffers[device_index][ctx.current_index]); } - buffer.* = Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], context_.self.shardings[context_.current_index], shards.constSlice()); - context_.current_index += 1; + buffer.* = Buffer.fromPjrtBuffers(ctx.results.platform, ctx.results.expected_shapes[ctx.current_index], ctx.results.shardings[ctx.current_index], shards.constSlice()); + ctx.current_index += 1; } }.cb, &context, &v); } diff --git a/zml/io.zig b/zml/io.zig index 1d43b2d4..739a30d9 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -14,11 +14,8 @@ const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const safetensors = @import("safetensors.zig"); const Shape = @import("shape.zig").Shape; -const sharding_ = @import("sharding.zig"); -const Partitioner = sharding_.Partitioner; -const Partitioning = sharding_.Partitioning; -const Placement = sharding_.Placement; -const Sharding = sharding_.Sharding; +const Sharding = @import("Sharding.zig"); +const Placement = Sharding.Placement; const Slice = @import("slice.zig").Slice; const Tensor = @import("tensor.zig").Tensor; @@ -149,13 +146,17 @@ pub const TensorStore = struct { } } - if (@TypeOf(partitioning) != @TypeOf(null)) { - switch (@typeInfo(@TypeOf(partitioning))) { - .optional => if (partitioning) |p| { - ptr.shape = ptr.shape.withPartitioning(p); - }, - else => ptr.shape = ptr.shape.withPartitioning(partitioning), - } + if (@TypeOf(partitioning) == @TypeOf(null)) { + @compileError("TensorStore.View.createTensor partitioning cannot be null; pass .replicated or an explicit partitioning"); + } + + switch (@typeInfo(@TypeOf(partitioning))) { + .optional => @compileError("TensorStore.View.createTensor partitioning cannot be optional; pass .replicated or an explicit partitioning"), + .enum_literal => switch (partitioning) { + .replicated => ptr.shape = ptr.shape.withReplicatedPartitioning(), + else => @compileError("Only .replicated is supported as a standalone partitioning enum literal"), + }, + else => ptr.shape = ptr.shape.withPartitioning(partitioning), } const tensor: Tensor = .fromShape(ptr.shape); @@ -591,9 +592,9 @@ pub const DirectMemoryWriter = struct { allocator: std.mem.Allocator, shape: Shape, placement: Placement, - raw_segments: std.ArrayListUnmanaged(RawSegment) = .empty, - segments: std.ArrayListUnmanaged(StreamSegment) = .empty, - mirrors: std.ArrayListUnmanaged(usize) = .empty, + raw_segments: std.ArrayList(RawSegment) = .empty, + segments: std.ArrayList(StreamSegment) = .empty, + mirrors: std.ArrayList(usize) = .empty, fn init(allocator: std.mem.Allocator, shape: Shape, placement: Placement) StreamPlanner { return .{ @@ -1067,7 +1068,7 @@ pub const LoadOpts = struct { }; parallelism: usize, - shardings: []const Sharding, + shardings: []const Sharding = &.{}, progress: ?*std.Progress.Node = null, dma_chunks: usize, dma_chunk_size: usize, @@ -1101,8 +1102,6 @@ pub fn load( pool_.deinit(dma_allocators[i].allocator()); }; - const replicated_sharding = try sharding_.replicatedSharding(platform); - const Ctx = struct { allocator: std.mem.Allocator, dma_allocators: []const mem.DmaAllocator, @@ -1111,7 +1110,6 @@ pub fn load( platform: *const Platform, buffers: []*Buffer, shardings: []const Sharding, - replicated_sharding: Sharding, store: *const TensorStore, group: stdx.Io.LimitedGroup, total: std.atomic.Value(usize) = .init(0), @@ -1126,7 +1124,6 @@ pub fn load( .pinned_buffer_pools = buffer_pools, .io = io, .shardings = opts.shardings, - .replicated_sharding = replicated_sharding, .progress = opts.progress, .group = .init(opts.parallelism), }; @@ -1150,10 +1147,9 @@ pub fn load( defer reader.deinit(); const shape = reader.tensor.shape; - const select_sharding = sharding_.pickSharding(ctx_.shardings, shape, .explicit_axis_binding); - const sharding = if (select_sharding) |s| s else blk: { - log.debug("No sharding strategy found for tensor {s} with shape {f}, using replicated sharding:\n {f}", .{ reader.tensor.name, shape, ctx_.replicated_sharding }); - break :blk ctx_.replicated_sharding; + const sharding = Sharding.pickSharding(ctx_.shardings, shape, .explicit_axis_binding) orelse blk: { + log.debug("No sharding strategy found for tensor {s} with shape {f}, using replicated sharding", .{ reader.tensor.name, shape }); + break :blk ctx_.platform.replicated_sharding; }; var writer = MemoryWriter.init( @@ -1197,9 +1193,9 @@ fn buildMesh2x2( allocator: std.mem.Allocator, target: @import("platform.zig").Target, devices: []const @import("platform.zig").Device, -) !sharding_.PhysicalMesh { +) !Sharding.PhysicalMesh { if (devices.len < 4) return error.NotEnoughDevices; - const topology: sharding_.PhysicalMesh.Tree = .axis(.link_x, .{ .mesh = .torus }, &.{ + const topology: Sharding.PhysicalMesh.Tree = .axis(.link_x, .{ .mesh = .torus }, &.{ .axis(.link_y, .{ .mesh = .torus }, &.{ .device(devices[0]), .device(devices[1]), @@ -1210,16 +1206,16 @@ fn buildMesh2x2( }), }); - return sharding_.PhysicalMesh.fromTree(allocator, target, topology); + return Sharding.PhysicalMesh.fromTree(allocator, target, topology); } fn buildMesh2x2x2( allocator: std.mem.Allocator, target: @import("platform.zig").Target, devices: []const @import("platform.zig").Device, -) !sharding_.PhysicalMesh { +) !Sharding.PhysicalMesh { if (devices.len < 8) return error.NotEnoughDevices; - const topology: sharding_.PhysicalMesh.Tree = .axis(.link_x, .{ .mesh = .torus }, &.{ + const topology: Sharding.PhysicalMesh.Tree = .axis(.link_x, .{ .mesh = .torus }, &.{ .axis(.link_y, .{ .mesh = .torus }, &.{ .axis(.link_z, .{ .mesh = .torus }, &.{ .device(devices[0]), @@ -1242,7 +1238,7 @@ fn buildMesh2x2x2( }), }); - return sharding_.PhysicalMesh.fromTree(allocator, target, topology); + return Sharding.PhysicalMesh.fromTree(allocator, target, topology); } const DirectMemoryWriterDeviceTest = struct { @@ -1255,8 +1251,8 @@ const DirectMemoryWriterDeviceTest = struct { name: []const u8, create_options: CreateOptions, shape: Shape, - logical_mesh: sharding_.LogicalMesh, - strategy: sharding_.Strategy, + logical_mesh: Sharding.LogicalMesh, + strategy: Sharding.Strategy, write_mode: WriteMode = .stream_remaining, writable_slice_min_len: usize = 128, pool_chunks: usize = 4, @@ -1267,16 +1263,14 @@ const DirectMemoryWriterDeviceTest = struct { io: std.Io, fn run(self: DirectMemoryWriterDeviceTest, scenario: Scenario) !void { - _ = scenario.name; - var platform = Platform.auto(self.allocator, self.io, scenario.create_options) catch return error.SkipZigTest; defer platform.deinit(self.allocator, self.io); - const sharding: Sharding = try .initFromStrategy(platform, scenario.logical_mesh, scenario.strategy); + const sharding: Sharding.Data = try .init(scenario.name, platform.physical_mesh, scenario.logical_mesh, scenario.strategy); try self.runDirectMemoryWriter( platform, scenario.shape, - sharding, + .{ .data = &sharding }, scenario.write_mode, scenario.writable_slice_min_len, scenario.pool_chunks, @@ -1374,9 +1368,9 @@ test "DirectMemoryWriter: replicated with auto topology" { }, .shape = Shape.init(.{ .rows = 8, .cols = 128 }, .f32) .withPartitioning(.{ .rows = .replicated, .cols = .replicated }), - .logical_mesh = .init("replicated_cpu", .{ .x = .high_bandwidth }), + .logical_mesh = .mesh(.{ .x = .high_bandwidth }), .strategy = blk: { - var strategy: sharding_.Strategy = .init; + var strategy: Sharding.Strategy = .init; strategy.addBinding(.x, .link_x); break :blk strategy; @@ -1398,9 +1392,9 @@ test "DirectMemoryWriter: 1D model split with 2x2 physical mesh" { }, .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32) .withPartitioning(.{ .rows = .replicated, .cols = .model }), - .logical_mesh = .init("model_cpu", .{ .model = .high_bandwidth }), + .logical_mesh = .mesh(.{ .model = .high_bandwidth }), .strategy = blk: { - var strategy: sharding_.Strategy = .init; + var strategy: Sharding.Strategy = .init; strategy.addBinding(.model, .link_x); break :blk strategy; @@ -1422,12 +1416,12 @@ test "DirectMemoryWriter: 2D batch/model split with 2x2 physical mesh" { }, .shape = Shape.init(.{ .batch = 8, .model = 1024 }, .f32) .withPartitioning(.{ .batch = .batch, .model = .model }), - .logical_mesh = .init("batch_model_cpu", .{ + .logical_mesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth, }), .strategy = blk: { - var strategy: sharding_.Strategy = .init; + var strategy: Sharding.Strategy = .init; strategy.addBinding(.batch, .link_x); strategy.addBinding(.model, .link_y); @@ -1449,9 +1443,9 @@ test "DirectMemoryWriter: folded model sharding with 2x2 physical mesh" { .cpu = .{ .device_count = 4 }, }, .shape = Shape.init(.{ .model = 4096 }, .f32).withPartitioning(.{ .model = .model }), - .logical_mesh = .init("model_folded_cpu", .{ .model = .high_bandwidth }), + .logical_mesh = .mesh(.{ .model = .high_bandwidth }), .strategy = blk: { - var strategy: sharding_.Strategy = .init; + var strategy: Sharding.Strategy = .init; strategy.addBinding(.model, .link_x); strategy.addFold(.link_x, &.{ .link_x, .link_y }); @@ -1474,9 +1468,9 @@ test "DirectMemoryWriter: writableSliceGreedy with mirrored shards" { }, .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32) .withPartitioning(.{ .rows = .replicated, .cols = .model }), - .logical_mesh = .init("model_cpu_greedy", .{ .model = .high_bandwidth }), + .logical_mesh = .mesh(.{ .model = .high_bandwidth }), .strategy = blk: { - var strategy: sharding_.Strategy = .init; + var strategy: Sharding.Strategy = .init; strategy.addBinding(.model, .link_x); break :blk strategy; @@ -1501,12 +1495,12 @@ test "DirectMemoryWriter: 3D topology folded model + replicated batch" { }, .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32) .withPartitioning(.{ .batch = .replicated, .model = .model }), - .logical_mesh = .init("topology_3d_folded_model_mesh", .{ + .logical_mesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth, }), .strategy = blk: { - var strategy: sharding_.Strategy = .init; + var strategy: Sharding.Strategy = .init; strategy.addBinding(.model, .link_x); strategy.addFold(.link_x, &.{ .link_x, .link_z }); diff --git a/zml/module.zig b/zml/module.zig index aaf41016..90b1a46f 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -15,9 +15,8 @@ const mlirx = @import("mlirx.zig"); const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; -const sharding_ = @import("sharding.zig"); -const Sharding = sharding_.Sharding; -const Partitioning = sharding_.Partitioning; +const Sharding = @import("Sharding.zig"); +const Partitioning = Sharding.Partitioning; const Tensor = @import("tensor.zig").Tensor; const log = std.log.scoped(.@"zml/module"); @@ -44,9 +43,9 @@ fn mlirRegistry(io: std.Io) *mlir.DialectRegistry { return mlir_global_registry.?; } pub const CompilationOptions = struct { - shardings: []const Sharding, + shardings: []const Sharding = &.{}, // If null, will be initialized from the target - partitioner: ?Partitioning.Partitioner = null, + partitioner: ?Sharding.Partitioner = null, // Debugging options program_name: []const u8 = "zml", xla_dump_to: ?[]const u8 = null, @@ -90,7 +89,7 @@ pub const CompilationContext = struct { mlir_pass_manager: *mlir.PassManager, module: *mlir.Module, platform: *const Platform, - partitioning: Partitioning, + partitioning: Sharding.Partitioning, scopes: stdx.BoundedArray(Scope, 16) = .empty, manual_computation_depth: usize = 0, @@ -98,6 +97,7 @@ pub const CompilationContext = struct { threadlocal var _current: ?*CompilationContext = null; pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: CompilationOptions) CompilationContext { + var arena = std.heap.ArenaAllocator.init(allocator); const mlir_registry = mlirRegistry(io); var mlir_ctx = mlir.Context.init(.{ .registry = mlir_registry, .threading = false }) catch unreachable; mlir_ctx.loadAllAvailableDialects(); @@ -118,12 +118,21 @@ pub const CompilationContext = struct { } } - const partitioning = Partitioning.init(opts.partitioner orelse Partitioning.Partitioner.fromTarget(platform.target), opts.shardings) catch unreachable; + // Ensure replicated sharding is always included as a fallback option. + var shardings = std.ArrayList(Sharding).initCapacity(arena.allocator(), opts.shardings.len + 1) catch @panic("OOM"); + var needs_replicated: bool = true; + for (opts.shardings) |sharding| { + if (sharding.data == platform.replicated_sharding.data) needs_replicated = false; + shardings.appendAssumeCapacity(sharding.resolve(platform)); + } + if (needs_replicated) shardings.appendAssumeCapacity(platform.replicated_sharding); + + const partitioning = Sharding.Partitioning.init(opts.partitioner orelse .fromTarget(platform.target), shardings.items) catch @panic("OOM"); return .{ .allocator = allocator, .io = io, - .arena = std.heap.ArenaAllocator.init(allocator), + .arena = arena, .mlir_registry = mlir_registry, .mlir_ctx = mlir_ctx, .mlir_pass_manager = pass_manager, @@ -243,25 +252,26 @@ pub fn compile( return exe; } -fn addPartitionerOperations(compilation_context: *CompilationContext) !void { - const allocator = compilation_context.arena.allocator(); - const mlir_ctx = compilation_context.mlir_ctx; - const module = compilation_context.module; - const partitioning = compilation_context.partitioning; +fn addPartitionerOperations(ctx: *CompilationContext) !void { + const allocator = ctx.arena.allocator(); + const mlir_ctx = ctx.mlir_ctx; + const module = ctx.module; + const partitioning = ctx.partitioning; switch (partitioning.partitioner) { .gspmd => {}, .shardy => { for (partitioning.shardings) |sharding| { - const attr_str = try sharding.sdyMeshAttr(allocator); + const attr_str = try sharding.data.sdyMeshAttr(allocator); defer allocator.free(attr_str); + const name = sharding.data.name; const mesh_attr = try mlir.Attribute.parse(mlir_ctx, attr_str); const mesh_op = mlir.Operation.make(mlir_ctx, "sdy.mesh", .{ .attributes = &.{ - .named(mlir_ctx, "sym_name", mlir.stringAttribute(mlir_ctx, sharding.name())), - .named(mlir_ctx, "mesh", @ptrCast(mesh_attr)), + .named(mlir_ctx, "sym_name", mlir.stringAttribute(mlir_ctx, name)), + .named(mlir_ctx, "mesh", mesh_attr), }, .location = .unknown(mlir_ctx), .verify = false, @@ -289,14 +299,14 @@ pub const OutputInfo = struct { } }; -fn collectOutputInfo(allocator: std.mem.Allocator, partitioning: Partitioning, v: anytype) !OutputInfo { +fn collectOutputInfo(allocator: std.mem.Allocator, partitioning: Sharding.Partitioning, v: anytype) !OutputInfo { const LocalContext = struct { shape_list: *std.array_list.Managed(Shape), sharding_list: *std.array_list.Managed(Sharding), value_list: *std.array_list.Managed(*const mlir.Value), donation_list: *std.array_list.Managed(?usize), output_memory_kind_list: *std.array_list.Managed(Memory.Kind), - partitioning: Partitioning, + partitioning: Sharding.Partitioning, }; var shape_list = std.array_list.Managed(Shape).init(allocator); @@ -348,11 +358,11 @@ pub const InputInfo = struct { } }; -fn collectInputInfo(allocator: std.mem.Allocator, partitioning: Partitioning, v: anytype) !InputInfo { +fn collectInputInfo(allocator: std.mem.Allocator, partitioning: Sharding.Partitioning, v: anytype) !InputInfo { const LocalContext = struct { shape_list: *std.array_list.Managed(Shape), sharding_list: *std.array_list.Managed(Sharding), - partitioning: Partitioning, + partitioning: Sharding.Partitioning, }; var shape_list = std.array_list.Managed(Shape).init(allocator); diff --git a/zml/nn.zig b/zml/nn.zig index f7c9dd49..e0a564d0 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -127,10 +127,10 @@ test normalizeL2 { const input: zml.Tensor = .init(.{ 2, 2 }, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, normalizeL2, .{ input, 1e-12 }, platform, .{ .shardings = &.{zml.testing.replicatedSharding()} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, normalizeL2, .{ input, 1e-12 }, platform, .{}); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), zml.testing.replicatedSharding(), std.mem.sliceAsBytes(&[_]f32{ -0.9686, -1.0058, -1.7808, 0.6698 })); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&[_]f32{ -0.9686, -1.0058, -1.7808, 0.6698 })); defer input_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, normalizeL2, .{input_buffer}); @@ -616,7 +616,7 @@ test "real/img" { } }; { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.interleaved}, platform, .{ .shardings = &.{zml.testing.replicatedSharding()} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.interleaved}, platform, .{}); defer exe.deinit(); var d_interleaved = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitMergeIsId, {}); @@ -624,7 +624,7 @@ test "real/img" { try std.testing.expectEqual(20, try d_interleaved.getValue(i32, std.testing.io)); } { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.sequential}, platform, .{ .shardings = &.{zml.testing.replicatedSharding()} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.sequential}, platform, .{}); defer exe.deinit(); var d_sequential = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitMergeIsId, {}); @@ -634,7 +634,7 @@ test "real/img" { // test the function that accepts 1 void argument { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitSeqVoid, .{{}}, platform, .{ .shardings = &.{zml.testing.replicatedSharding()} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitSeqVoid, .{{}}, platform, .{}); defer exe.deinit(); var d_split_seq_void = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitSeqVoid, {}); @@ -644,7 +644,7 @@ test "real/img" { // test the function that takes NO arguments { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitSeq, .{}, platform, .{ .shardings = &.{zml.testing.replicatedSharding()} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitSeq, .{}, platform, .{}); defer exe.deinit(); var d_split_seq = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitSeq, {}); @@ -653,7 +653,7 @@ test "real/img" { } { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitInterleaved, .{}, platform, .{ .shardings = &.{zml.testing.replicatedSharding()} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitInterleaved, .{}, platform, .{}); defer exe.deinit(); var d_split_seq = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitInterleaved, {}); @@ -687,13 +687,13 @@ test rope { // x is made such as the interleaved and sequential reps are the same. // So the two implementations should give the same results. const x: zml.Tensor = .init(.{ .b = 1, .s = 5, .hd = 4 }, .f32); - var exe_interleaved = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .interleaved } }, platform, .{ .shardings = &.{zml.testing.replicatedSharding()} }); + var exe_interleaved = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .interleaved } }, platform, .{}); defer exe_interleaved.deinit(); - var exe_sequential = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .sequential } }, platform, .{ .shardings = &.{zml.testing.replicatedSharding()} }); + var exe_sequential = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .sequential } }, platform, .{}); defer exe_sequential.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), zml.testing.replicatedSharding(), std.mem.sliceAsBytes(&[_]f32{ 1.0, 0.1, -1.0, -0.5 } ** 5)); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]f32{ 1.0, 0.1, -1.0, -0.5 } ** 5)); defer x_buffer.deinit(); var res1 = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_interleaved, Local._fwd, .{x_buffer}); @@ -772,7 +772,7 @@ test nearest { upsample, .{ input_3d_basic, .{ .scale_factor = &.{3}, .mode = .nearest } }, platform, - .{ .shardings = &.{zml.testing.replicatedSharding()} }, + .{}, ); defer exe.deinit(); @@ -780,7 +780,7 @@ test nearest { std.testing.io, platform, input_3d_basic.shape(), - zml.testing.replicatedSharding(), + .replicated, std.mem.sliceAsBytes(&[1][1][2]i32{.{.{ 1, 2 }}}), ); defer input_3d_basic_buffer.deinit(); @@ -802,11 +802,11 @@ test nearest { upsample, .{ input_3d_advanced, .{ .scale_factor = &.{2}, .mode = .nearest } }, platform, - .{ .shardings = &.{zml.testing.replicatedSharding()} }, + .{}, ); defer exe.deinit(); - var input_3d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_3d_advanced.shape(), zml.testing.replicatedSharding(), std.mem.sliceAsBytes(&[2][3][4]i32{ + var input_3d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_3d_advanced.shape(), .replicated, std.mem.sliceAsBytes(&[2][3][4]i32{ .{ .{ 1, 2, 3, 4 }, .{ 5, 6, 7, 8 }, .{ 9, 10, 11, 12 } }, .{ .{ 13, 14, 15, 16 }, .{ 17, 18, 19, 20 }, .{ 21, 22, 23, 24 } }, })); @@ -840,11 +840,11 @@ test nearest { upsample, .{ input_4d_basic, .{ .scale_factor = &.{ 3, 3 }, .mode = .nearest } }, platform, - .{ .shardings = &.{zml.testing.replicatedSharding()} }, + .{}, ); defer exe.deinit(); - var input_4d_basic_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_basic.shape(), zml.testing.replicatedSharding(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); + var input_4d_basic_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_basic.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); defer input_4d_basic_buffer.deinit(); var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_4d_basic_buffer}); @@ -870,11 +870,11 @@ test nearest { upsample, .{ input_4d_advanced, .{ .scale_factor = &.{ 2, 2 }, .mode = .nearest } }, platform, - .{ .shardings = &.{zml.testing.replicatedSharding()} }, + .{}, ); defer exe.deinit(); - var input_4d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_advanced.shape(), zml.testing.replicatedSharding(), std.mem.sliceAsBytes(&[2][2][2][2]i32{ .{ + var input_4d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_advanced.shape(), .replicated, std.mem.sliceAsBytes(&[2][2][2][2]i32{ .{ .{ .{ 1, 2 }, .{ 3, 4 } }, .{ .{ 5, 6 }, .{ 7, 8 } }, }, .{ @@ -928,11 +928,11 @@ test nearest { upsample, .{ input_5d, .{ .scale_factor = &.{2}, .mode = .nearest } }, platform, - .{ .shardings = &.{zml.testing.replicatedSharding()} }, + .{}, ); defer exe.deinit(); - var input_5d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_5d.shape(), zml.testing.replicatedSharding(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); + var input_5d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_5d.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); defer input_5d_buffer.deinit(); var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_5d_buffer}); @@ -989,9 +989,9 @@ pub fn resizeBilinear(image: Tensor, resized_axes: anytype, opt: ResizeOpts) Ten test resizeBilinear { const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); + // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{ .shardings = &.{replicated_sharding} }); + var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -1056,9 +1056,9 @@ pub fn resizeBicubic(image: Tensor, resized_axes: anytype, opt: ResizeOpts) Tens test resizeBicubic { const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); + // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{ .shardings = &.{replicated_sharding} }); + var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -1406,7 +1406,6 @@ pub const GatedDeltaNet = struct { test "gated delta net" { const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); const queries: zml.Tensor = .init(.{ .s = 2, .h = 2, .k = 2 }, .f32); const keys: zml.Tensor = .init(.{ .s = 2, .h = 2, .k = 2 }, .f32); @@ -1421,7 +1420,7 @@ test "gated delta net" { GatedDeltaNet.forward, .{ queries, keys, values, alphas, betas, GatedDeltaNet.State{ .s = initial_s } }, platform, - .{ .shardings = &.{replicated_sharding} }, + .{}, ); defer exe.deinit(); @@ -1429,7 +1428,7 @@ test "gated delta net" { std.testing.io, platform, queries.shape(), - replicated_sharding, + .replicated, std.mem.sliceAsBytes(&[2][2][2]f32{ .{ .{ 1.0, 0.0 }, .{ 0.0, 1.0 } }, .{ .{ 1.0, 1.0 }, .{ 1.0, -1.0 } }, @@ -1440,7 +1439,7 @@ test "gated delta net" { std.testing.io, platform, keys.shape(), - replicated_sharding, + .replicated, std.mem.sliceAsBytes(&[2][2][2]f32{ .{ .{ 1.0, 2.0 }, .{ 0.0, 1.0 } }, .{ .{ 2.0, 1.0 }, .{ 1.0, 0.0 } }, @@ -1451,7 +1450,7 @@ test "gated delta net" { std.testing.io, platform, values.shape(), - replicated_sharding, + .replicated, std.mem.sliceAsBytes(&[2][2][2]f32{ .{ .{ 3.0, 1.0 }, .{ 2.0, 4.0 } }, .{ .{ 1.0, 5.0 }, .{ 3.0, 0.0 } }, @@ -1462,7 +1461,7 @@ test "gated delta net" { std.testing.io, platform, alphas.shape(), - replicated_sharding, + .replicated, std.mem.sliceAsBytes(&[2][2]f32{ .{ 0.5, 0.25 }, .{ 0.8, 0.6 }, @@ -1473,7 +1472,7 @@ test "gated delta net" { std.testing.io, platform, betas.shape(), - replicated_sharding, + .replicated, std.mem.sliceAsBytes(&[2][2]f32{ .{ 1.0, 0.5 }, .{ 0.75, 1.0 }, @@ -1484,7 +1483,7 @@ test "gated delta net" { std.testing.io, platform, initial_s.shape(), - replicated_sharding, + .replicated, std.mem.sliceAsBytes(&[2][2][2]f32{ .{ .{ 1.0, 0.0 }, .{ 0.0, 1.0 } }, .{ .{ 2.0, 1.0 }, .{ 1.0, 0.0 } }, @@ -1566,15 +1565,14 @@ pub fn sampleTokens(activations: Tensor, opts: SamplingStrategy, rng: Tensor.Rng test sampleTokens { const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); const rng: zml.Tensor.Rng = .init(); const activations: zml.Tensor = .init(.{ .voc = 4 }, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, sampleTokens, .{ activations, .{ .topk = 4, .temperature = 2.0 }, rng }, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, sampleTokens, .{ activations, .{ .topk = 4, .temperature = 2.0 }, rng }, platform, .{}); defer exe.deinit(); - var rng_buffer = try zml.Tensor.Rng.initBuffer(std.testing.io, platform, replicated_sharding, 0xdeadbeef); + var rng_buffer = try zml.Tensor.Rng.initBuffer(std.testing.io, platform, .replicated, 0xdeadbeef); defer rng_buffer._state.deinit(); const inf = std.math.inf(f32); @@ -1586,7 +1584,7 @@ test sampleTokens { }) |logits_expected| { const logits, const expected: i32 = logits_expected; - var activations_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, activations.shape(), replicated_sharding, std.mem.sliceAsBytes(&logits)); + var activations_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, activations.shape(), .replicated, std.mem.sliceAsBytes(&logits)); defer activations_buffer.deinit(); var sampled, rng_buffer = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, sampleTokens, .{ activations_buffer, rng_buffer }); @@ -1625,15 +1623,14 @@ pub const DynamicSamplingStrategy = struct { pub fn makeBuffers( io: std.Io, platform: *const zml.Platform, - sharding: zml.sharding.Sharding, dtype: zml.DataType, opts: Opts, ) !zml.Bufferized(DynamicSamplingStrategy) { return .{ - .top_k = try zml.Buffer.scalar(io, platform, opts.top_k, .i32, sharding), - .temperature = try zml.Buffer.scalar(io, platform, opts.temperature, dtype, sharding), - .top_p = try zml.Buffer.scalar(io, platform, opts.top_p, dtype, sharding), - .min_p = try zml.Buffer.scalar(io, platform, opts.min_p, dtype, sharding), + .top_k = try zml.Buffer.scalar(io, platform, opts.top_k, .i32), + .temperature = try zml.Buffer.scalar(io, platform, opts.temperature, dtype), + .top_p = try zml.Buffer.scalar(io, platform, opts.top_p, dtype), + .min_p = try zml.Buffer.scalar(io, platform, opts.min_p, dtype), }; } @@ -1704,7 +1701,6 @@ fn fixupLogits(logits: Tensor, opts: DynamicSamplingStrategy) [2]Tensor { test sampleTokensDynamic { const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); const ___ = -std.math.inf(f32); const logits_data = [_]f32{ @log(2.0), @log(1.0), @log(4.0), @log(3.0) }; @@ -1713,10 +1709,10 @@ test sampleTokensDynamic { const logits: zml.Tensor = .init(.{ .voc = logits_data.len }, .f32); const dynamic_sampling_strategy = DynamicSamplingStrategy.init(.f32, 0); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, fixupLogits, .{ logits, dynamic_sampling_strategy }, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, fixupLogits, .{ logits, dynamic_sampling_strategy }, platform, .{}); defer exe.deinit(); - var logits_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits.shape(), replicated_sharding, std.mem.sliceAsBytes(&logits_data)); + var logits_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits.shape(), .replicated, std.mem.sliceAsBytes(&logits_data)); defer logits_buffer.deinit(); const Args = struct { DynamicSamplingStrategy.Opts, [4]f32 }; @@ -1735,7 +1731,7 @@ test sampleTokensDynamic { .{ .{ .top_k = 4, .top_p = 0.901, .min_p = 0.6 }, [_]f32{ @log(4.0), @log(3.0), ___, ___ } }, }) |args_expected| { const args, const expected = args_expected; - var dynamic_sampling_strategy_buffers = try DynamicSamplingStrategy.makeBuffers(std.testing.io, platform, replicated_sharding, .f32, args); + var dynamic_sampling_strategy_buffers = try DynamicSamplingStrategy.makeBuffers(std.testing.io, platform, .f32, args); defer DynamicSamplingStrategy.deinitBuffers(&dynamic_sampling_strategy_buffers); var new_logits, var indices = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, fixupLogits, .{ logits_buffer, dynamic_sampling_strategy_buffers }); defer new_logits.deinit(); @@ -1750,15 +1746,15 @@ test sampleTokensDynamic { const logits_bf16: zml.Tensor = .init(.{ .voc = logits_data.len }, .bf16); const dynamic_sampling_strategy_bf16 = DynamicSamplingStrategy.init(.bf16, 0); - var exe_bf16 = try zml.module.compile(std.testing.allocator, std.testing.io, fixupLogits, .{ logits_bf16, dynamic_sampling_strategy_bf16 }, platform, .{ .shardings = &.{replicated_sharding} }); + var exe_bf16 = try zml.module.compile(std.testing.allocator, std.testing.io, fixupLogits, .{ logits_bf16, dynamic_sampling_strategy_bf16 }, platform, .{}); defer exe_bf16.deinit(); const boost = bf16.inf; const nerf = bf16.minus_inf; - var logits_bf16_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits_bf16.shape(), replicated_sharding, std.mem.sliceAsBytes(&[4]bf16{ boost, boost, bf16.fromF32(2), nerf })); + var logits_bf16_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits_bf16.shape(), .replicated, std.mem.sliceAsBytes(&[4]bf16{ boost, boost, bf16.fromF32(2), nerf })); defer logits_bf16_buffer.deinit(); - var dynamic_sampling_strategy_bf16_buffers = try DynamicSamplingStrategy.makeBuffers(std.testing.io, platform, replicated_sharding, .bf16, .{ .top_k = 4, .top_p = 0.9, .min_p = 0.1 }); + var dynamic_sampling_strategy_bf16_buffers = try DynamicSamplingStrategy.makeBuffers(std.testing.io, platform, .bf16, .{ .top_k = 4, .top_p = 0.9, .min_p = 0.1 }); defer DynamicSamplingStrategy.deinitBuffers(&dynamic_sampling_strategy_bf16_buffers); var new_logits, var indices = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_bf16, fixupLogits, .{ logits_bf16_buffer, dynamic_sampling_strategy_bf16_buffers }); diff --git a/zml/ops.zig b/zml/ops.zig index 10cc47ea..25d96479 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -392,7 +392,6 @@ pub const while_ = @"while"; test @"while" { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); const initial_i: zml.Tensor = .init(.{}, .i64); const initial_sum: zml.Tensor = .init(.{}, .i64); @@ -422,13 +421,13 @@ test @"while" { Local.forward, .{ initial_i, initial_sum }, platform, - .{ .shardings = &.{replicated_sharding} }, + .{}, ); defer exe.deinit(); - var i_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_i.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1]i64{1})); + var i_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_i.shape(), .replicated, std.mem.sliceAsBytes(&[1]i64{1})); defer i_buffer.deinit(); - var sum_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_sum.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1]i64{0})); + var sum_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_sum.shape(), .replicated, std.mem.sliceAsBytes(&[1]i64{0})); defer sum_buffer.deinit(); var results = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local.forward, .{ i_buffer, sum_buffer }); @@ -550,12 +549,12 @@ test "triton" { const a: zml.Tensor = .init(.{}, .f32); const b: zml.Tensor = .init(.{}, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, TritonMod.forward, .{ a, b }, platform, .{ .shardings = &.{zml.testing.replicatedSharding()} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, TritonMod.forward, .{ a, b }, platform, .{}); defer exe.deinit(); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), zml.testing.replicatedSharding(), std.mem.sliceAsBytes(&[1]f32{1})); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&[1]f32{1})); defer a_buffer.deinit(); - var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), zml.testing.replicatedSharding(), std.mem.sliceAsBytes(&[1]f32{3})); + var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), .replicated, std.mem.sliceAsBytes(&[1]f32{3})); defer b_buffer.deinit(); var results = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, TritonMod.forward, .{ a_buffer, b_buffer }); @@ -815,7 +814,7 @@ test scatterConfig { const zml = @import("zml.zig"); const platform = zml.testing.env(); - var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{ .shardings = &.{zml.testing.replicatedSharding()} }); + var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -1963,7 +1962,7 @@ test customCall { const zml = @import("zml.zig"); const platform = zml.testing.env(); - var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{ .shardings = &.{zml.testing.replicatedSharding()} }); + var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); diff --git a/zml/platform.zig b/zml/platform.zig index c83dd977..ccbe0e41 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -10,6 +10,7 @@ const attention = @import("attention.zig"); const Exe = @import("exe.zig").Exe; const pjrtx = @import("pjrtx.zig"); const profiler_ = @import("profiler.zig"); +const Sharding = @import("Sharding.zig"); const zml = @import("zml.zig"); const log = std.log.scoped(.zml); @@ -194,13 +195,15 @@ pub const Device = struct { }; pub const Platform = struct { - arena_state: std.heap.ArenaAllocator.State = .{}, + arena: std.heap.ArenaAllocator, target: Target, pjrt_api: *const pjrt.Api, pjrt_client: *pjrt.Client, devices: []const Device, memories: []const Memory, - physical_mesh: zml.sharding.PhysicalMesh, + physical_mesh: zml.Sharding.PhysicalMesh, + replicated_sharding: zml.Sharding, + shardings: std.StringArrayHashMapUnmanaged(zml.Sharding.Data), pub const MAX_NUM_DEVICES: u16 = if (platforms.isEnabled(.tpu)) 64 else 32; @@ -215,42 +218,63 @@ pub const Platform = struct { log.warn("platform {} got {} devices, but ZML only support up to {} devices. Some devices won't be used.", .{ target, pjrt_devices.len, MAX_NUM_DEVICES }); } - var arena = std.heap.ArenaAllocator.init(allocator); - errdefer arena.deinit(); - - const devices = try arena.allocator().alloc(Device, pjrt_devices.len); - const pjrt_memories = pjrt_client.addressableMemories(api); - const memories = try arena.allocator().alloc(Memory, pjrt_memories.len); - const platform = try arena.allocator().create(Platform); - platform.* = .{ - .target = target, - .pjrt_api = api, - .pjrt_client = pjrt_client, - .devices = devices, - .memories = memories, - .physical_mesh = undefined, + // Note: Platform is a self-owning struct. It contains the arena that created it in the first place + // But it does mean we have to be careful to pass the arena state that contains the node + const platform: *Platform = platform: { + var initial_arena = std.heap.ArenaAllocator.init(allocator); + errdefer initial_arena.deinit(); + + var initial_allocator = initial_arena.allocator(); + // Pre-heat the arena, to avoid fragmentation. + initial_allocator.free(try initial_allocator.alloc(u8, 8 * 1024)); + + const platform = try initial_allocator.create(Platform); + platform.* = .{ + .arena = initial_arena, + .target = target, + .pjrt_api = api, + .pjrt_client = pjrt_client, + .shardings = .empty, + // set below + .devices = undefined, + .memories = undefined, + .physical_mesh = undefined, + .replicated_sharding = undefined, + }; + break :platform platform; }; - defer platform.arena_state = arena.state; + + const arena = platform.arena.allocator(); + errdefer platform.arena.deinit(); + try platform.shardings.ensureTotalCapacity(arena, 8); { + const devices = try arena.alloc(Device, pjrt_devices.len); + platform.devices = devices; + const memories = try arena.alloc(Memory, pjrt_memories.len); + platform.memories = memories; + + // TODO: part of the complication here is that we layout the data in spaghetti mode, + // where devices and memories point to each other and also point to the platform. for (pjrt_devices, devices) |pjrt_device, *platform_device| { - platform_device.* = try .init(arena.allocator(), pjrt_device, platform, memories); + platform_device.* = try .init(arena, pjrt_device, platform, memories); } for (pjrt_memories, memories) |pjrt_memory, *platform_memory| { - platform_memory.* = try .init(arena.allocator(), pjrt_memory, platform, devices); + platform_memory.* = try .init(arena, pjrt_memory, platform, devices); } platform.physical_mesh = try switch (options.physical_mesh) { - .auto => zml.sharding.PhysicalMesh.auto(arena.allocator(), target, devices), - .custom => |builder| builder(arena.allocator(), target, devices), + .auto => zml.Sharding.PhysicalMesh.auto(arena, target, devices), + .custom => |builder| builder(arena, target, devices), }; + platform.replicated_sharding = try platform.registerSharding("replicated", .mesh(.{ .x = .high_bandwidth })); } switch (target) { .cuda => { - zml.attention.flashattn.load(arena.allocator(), io) catch { + zml.attention.flashattn.load(arena, io) catch { log.warn("Failed to load flashattn", .{}); }; zml.attention.flashattn.register(platform) catch { @@ -404,8 +428,9 @@ pub const Platform = struct { pub fn deinit(self: *Platform, allocator: std.mem.Allocator, io: std.Io) void { _ = io; + _ = allocator; self.pjrt_client.deinit(self.pjrt_api); - self.arena_state.promote(allocator).deinit(); + self.arena.deinit(); } pub fn compile( @@ -491,6 +516,26 @@ pub const Platform = struct { pub fn profiler(self: *const Platform, allocator: std.mem.Allocator, io: std.Io, options: ProfilerOptions) !Profiler { return try profiler_.profiler(self.pjrt_api, allocator, io, options); } + + /// Create a Sharding based on the given logical mesh and the default strategy. + /// Memory is owned by the platform, making it safe to copy around. + pub fn registerSharding(platform: *Platform, name: []const u8, logical: Sharding.LogicalMesh) !Sharding { + return platform.registerShardingWithStrategy(name, logical, .suggest(logical, platform.physical_mesh)); + } + + /// Create a Sharding based on the given logical mesh and a strategy. + /// Memory is owned by the platform, making it safe to copy around. + pub fn registerShardingWithStrategy(platform: *Platform, name: []const u8, logical: Sharding.LogicalMesh, strategy: Sharding.Strategy) !Sharding { + const arena = platform.arena.allocator(); + const entry = try platform.shardings.getOrPut(arena, name); + if (entry.found_existing) { + std.debug.panic("Another sharding already exists with this name: {s}", .{name}); + } + + entry.key_ptr.* = try arena.dupe(u8, name); + entry.value_ptr.* = try .init(name, platform.physical_mesh, logical, strategy); + return .{ .data = entry.value_ptr }; + } }; pub const CreateOptions = struct { @@ -498,7 +543,7 @@ pub const CreateOptions = struct { allocator: std.mem.Allocator, target: Target, devices: []const Device, - ) anyerror!zml.sharding.PhysicalMesh; + ) anyerror!zml.Sharding.PhysicalMesh; pub const PhysicalMesh = union(enum) { auto, diff --git a/zml/tensor.zig b/zml/tensor.zig index b660767d..33c21431 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -15,7 +15,7 @@ const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; -const Sharding = @import("sharding.zig").Sharding; +const Sharding = @import("Sharding.zig"); pub const Tensor = struct { var current_id: std.atomic.Value(usize) = .{ .raw = 1 }; @@ -346,7 +346,6 @@ pub const Tensor = struct { test fmod { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); const inputs: [2][6]f32 = .{ .{ -3.0, -2, -1, 1, 2, 3 }, .{ 1, 2, 3, 4, 5, -5 } }; const expectations: [2][6]f32 = .{ .{ -1.0, -0.0, -1.0, 1.0, 0.0, 1.0 }, .{ 1.0000, 0.5000, 0.0000, 1.0000, 0.5000, -0.5000 } }; @@ -361,11 +360,11 @@ pub const Tensor = struct { Tensor.fmod, .{ input, d }, platform, - .{ .shardings = &.{replicated_sharding} }, + .{}, ); defer exe.deinit(); - var input_buffer = try zml.Buffer.fromBytes(std.testing.io, platform, input.shape(), replicated_sharding, std.mem.sliceAsBytes(&i)); + var input_buffer = try zml.Buffer.fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&i)); defer input_buffer.deinit(); const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.fmod, .{input_buffer}); @@ -627,12 +626,11 @@ pub const Tensor = struct { }; const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); // Compute stats over a uniform distribution on [-2, 10]. - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.uniformStats, .{ Rng.init(), Shape.init(.{1024}, .f32), .{ .min = -2, .max = 10 } }, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.uniformStats, .{ Rng.init(), Shape.init(.{1024}, .f32), .{ .min = -2, .max = 10 } }, platform, .{}); defer exe.deinit(); - var rng_buffer = try Rng.initBuffer(std.testing.io, platform, replicated_sharding, 1234); + var rng_buffer = try Rng.initBuffer(std.testing.io, platform, .replicated, 1234); defer rng_buffer._state.deinit(); var rand, var stats = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Stats.uniformStats, .{rng_buffer}); @@ -733,17 +731,16 @@ pub const Tensor = struct { }; const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); const tgt_dist_data = [_]f32{ 2.0, 1.0, 4.0, 3.0 }; const tgt_dist: Tensor = .init(.{tgt_dist_data.len}, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.gumbelStats, .{ Rng.init(), tgt_dist }, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.gumbelStats, .{ Rng.init(), tgt_dist }, platform, .{}); defer exe.deinit(); - var rng_buffer = try Rng.initBuffer(std.testing.io, platform, replicated_sharding, 1234); + var rng_buffer = try Rng.initBuffer(std.testing.io, platform, .replicated, 1234); defer rng_buffer._state.deinit(); - var tgt_dist_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, tgt_dist.shape(), replicated_sharding, @ptrCast(&tgt_dist_data)); + var tgt_dist_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, tgt_dist.shape(), .replicated, @ptrCast(&tgt_dist_data)); defer tgt_dist_buffer.deinit(); var rand, var stats = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Stats.gumbelStats, .{ rng_buffer, tgt_dist_buffer }); @@ -1062,7 +1059,6 @@ pub const Tensor = struct { const floats = @import("floats.zig"); const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); // f4e2m1 { @@ -1077,11 +1073,11 @@ pub const Tensor = struct { Tensor.convert, .{ x_d, .f4e2m1 }, platform, - .{ .shardings = &.{replicated_sharding} }, + .{}, ); defer exe.deinit(); - var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), replicated_sharding, std.mem.sliceAsBytes(&x)); + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), .replicated, std.mem.sliceAsBytes(&x)); defer x_d_buffer.deinit(); var x_f4_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); @@ -1105,11 +1101,11 @@ pub const Tensor = struct { Tensor.convert, .{ x_d, .f8e3m4 }, platform, - .{ .shardings = &.{replicated_sharding} }, + .{}, ); defer exe.deinit(); - var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), replicated_sharding, std.mem.sliceAsBytes(&x)); + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), .replicated, std.mem.sliceAsBytes(&x)); defer x_d_buffer.deinit(); var x_f8e3_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); @@ -1158,7 +1154,6 @@ pub const Tensor = struct { test dot { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); inline for (.{ .{ .{ .c = 20 }, .{ .c = 20 }, .c, .{} }, @@ -1198,7 +1193,7 @@ pub const Tensor = struct { forward, .{ x, y, ctr }, platform, - .{ .shardings = &.{replicated_sharding} }, + .{}, ); defer exe.deinit(); @@ -1308,7 +1303,6 @@ pub const Tensor = struct { test leakyReLU { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); const input: Tensor = .init(.{2}, .f32); var exe = try zml.module.compile( @@ -1317,11 +1311,11 @@ pub const Tensor = struct { Tensor.leakyReLU, .{ input, 0.1 }, platform, - .{ .shardings = &.{replicated_sharding} }, + .{}, ); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), replicated_sharding, std.mem.sliceAsBytes(&[_]f32{ -0.6884, 1.6795 })); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&[_]f32{ -0.6884, 1.6795 })); defer input_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.leakyReLU, .{input_buffer}); @@ -1461,7 +1455,6 @@ pub const Tensor = struct { test cumulativeSum { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); const Local = struct { pub fn _cumsum(input: Tensor) Tensor { @@ -1481,11 +1474,11 @@ pub const Tensor = struct { Local._cumsum, .{x}, platform, - .{ .shardings = &.{replicated_sharding} }, + .{}, ); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } })); defer x_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._cumsum, .{x_buffer}); @@ -1682,11 +1675,10 @@ pub const Tensor = struct { test slice { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); const x: Tensor = .init(.{ 2, 5 }, .f32); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); defer x_buffer.deinit(); // Wrap slice1d to hide the anytype in the signature. @@ -1708,7 +1700,7 @@ pub const Tensor = struct { Local._slice1dAxis, .{ x, ax, slice_ }, platform, - .{ .shardings = &.{replicated_sharding} }, + .{}, ); defer exe.deinit(); @@ -1825,7 +1817,6 @@ pub const Tensor = struct { test repeat1d { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); const Local = struct { fn repeat1d(x: Tensor, axis_: u3, n_reps: u32) Tensor { @@ -1846,11 +1837,11 @@ pub const Tensor = struct { Local.repeat1d, .{ input, ax, reps }, platform, - .{ .shardings = &.{replicated_sharding} }, + .{}, ); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), replicated_sharding, std.mem.sliceAsBytes(&input_data)); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&input_data)); defer input_buffer.deinit(); const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local.repeat1d, .{input_buffer}); @@ -1887,7 +1878,6 @@ pub const Tensor = struct { test stutter1d { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); const Local = struct { fn stutter1d(x: Tensor, axis_: u3, n_reps: u32) Tensor { @@ -1909,11 +1899,11 @@ pub const Tensor = struct { Local.stutter1d, .{ input, ax, reps }, platform, - .{ .shardings = &.{replicated_sharding} }, + .{}, ); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), replicated_sharding, std.mem.sliceAsBytes(&input_data)); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&input_data)); defer input_buffer.deinit(); const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local.stutter1d, .{input_buffer}); @@ -2071,7 +2061,6 @@ pub const Tensor = struct { test scalar { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); const Local = struct { pub fn _fwd() [6]Tensor { @@ -2082,7 +2071,7 @@ pub const Tensor = struct { } }; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{}, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{}, platform, .{}); defer exe.deinit(); } @@ -2337,7 +2326,6 @@ pub const Tensor = struct { test gather { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); const Local = struct { pub fn _idx(idx_shape: anytype) Tensor { @@ -2349,7 +2337,7 @@ pub const Tensor = struct { { // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{ .shardings = &.{replicated_sharding} }); + var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -2492,7 +2480,6 @@ pub const Tensor = struct { test gatherSlices { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); const Local = struct { pub fn _gatherSlices(self: Tensor, slice_shape: Shape, indices: Tensor, opts: GatherOpts) Tensor { @@ -2502,7 +2489,7 @@ pub const Tensor = struct { { // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{ .shardings = &.{replicated_sharding} }); + var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -2551,11 +2538,11 @@ pub const Tensor = struct { Local._gatherSlices, .{ operand, Shape.init(.{ .b = 2, .c = 3 }, .u16), indices, .{} }, platform, - .{ .shardings = &.{replicated_sharding} }, + .{}, ); defer exe.deinit(); - var indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, indices.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2][2]i32{ .{ 2, 1 }, .{ 0, 3 } })); + var indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, indices.shape(), .replicated, std.mem.sliceAsBytes(&[2][2]i32{ .{ 2, 1 }, .{ 0, 3 } })); defer indices_buffer.deinit(); var operand_buffer: zml.Buffer = b: { const temp_func = struct { @@ -2563,7 +2550,7 @@ pub const Tensor = struct { return Tensor.arange(.{ .end = 2 * 4 * 6 }, .u16).reshape(.{ .a = 2, .b = 4, .c = 6 }); } }.forward; - var temp_exe = try zml.module.compile(std.testing.allocator, std.testing.io, temp_func, .{}, platform, .{ .shardings = &.{replicated_sharding} }); + var temp_exe = try zml.module.compile(std.testing.allocator, std.testing.io, temp_func, .{}, platform, .{}); defer temp_exe.deinit(); const buffer = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &temp_exe, temp_func, {}); @@ -2702,7 +2689,6 @@ pub const Tensor = struct { test scatterSlices { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); const Local = struct { pub fn _scatter(self: Tensor, indices: []const Tensor, updates: Tensor) Tensor { @@ -2728,7 +2714,7 @@ pub const Tensor = struct { { // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{ .shardings = &.{replicated_sharding} }); + var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -2771,14 +2757,14 @@ pub const Tensor = struct { const scatter_indices = Tensor.init(.{2}, .i32).withTags(.{.n}); const updates = Tensor.init(.{ 2, 3 }, .i32).withTags(.{ .n, .b }); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform, .{}); defer exe.deinit(); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), replicated_sharding, std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); defer a_buffer.deinit(); - var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), replicated_sharding, @ptrCast(&[2]i32{ 0, 2 })); + var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), .replicated, @ptrCast(&[2]i32{ 0, 2 })); defer scatter_indices_buffer.deinit(); - var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2][3]i32{ .{ 10, 20, 30 }, .{ 70, 80, 90 } })); + var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), .replicated, std.mem.sliceAsBytes(&[2][3]i32{ .{ 10, 20, 30 }, .{ 70, 80, 90 } })); defer updates_buffer.deinit(); var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatter, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); @@ -2795,14 +2781,14 @@ pub const Tensor = struct { const scatter_indices = Tensor.init(.{2}, .i32).withTags(.{.n}); const updates = Tensor.init(.{2}, .i32).withTags(.{.n}); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform, .{}); defer exe.deinit(); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), replicated_sharding, std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); defer a_buffer.deinit(); - var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2]i32{ 2, 7 })); + var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), .replicated, std.mem.sliceAsBytes(&[2]i32{ 2, 7 })); defer scatter_indices_buffer.deinit(); - var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2]i32{ 20, 70 })); + var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), .replicated, std.mem.sliceAsBytes(&[2]i32{ 20, 70 })); defer updates_buffer.deinit(); var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatter, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); @@ -2818,12 +2804,12 @@ pub const Tensor = struct { const start_indices: Tensor = .init(.{ .n = 2, .a = 2, .m = 3, .coord = 2 }, .i32); const values: Tensor = .init(.{ .n = 2, .a = 2, .m = 3, .c = 2, .d = 2 }, .u16); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatterCB, .{ operand, start_indices, values }, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatterCB, .{ operand, start_indices, values }, platform, .{}); defer exe.deinit(); - var operand_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, operand.shape(), replicated_sharding, std.mem.sliceAsBytes(&@as([2 * 3 * 4 * 2]u16, @splat(0)))); + var operand_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, operand.shape(), .replicated, std.mem.sliceAsBytes(&@as([2 * 3 * 4 * 2]u16, @splat(0)))); defer operand_buffer.deinit(); - var start_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, start_indices.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2][2][3][2]i32{ + var start_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, start_indices.shape(), .replicated, std.mem.sliceAsBytes(&[2][2][3][2]i32{ .{ .{ .{ 0, 0 }, .{ 1, 0 }, .{ 2, 1 } }, .{ .{ 0, 1 }, .{ 1, 1 }, .{ 0, 9 } }, @@ -2834,7 +2820,7 @@ pub const Tensor = struct { }, })); defer start_indices_buffer.deinit(); - var values_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, values.shape(), replicated_sharding, std.mem.sliceAsBytes(&@as([2 * 2 * 3 * 2 * 2]u16, @splat(1)))); + var values_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, values.shape(), .replicated, std.mem.sliceAsBytes(&@as([2 * 2 * 3 * 2 * 2]u16, @splat(1)))); defer values_buffer.deinit(); var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatterCB, .{ operand_buffer, start_indices_buffer, values_buffer }); @@ -2928,7 +2914,6 @@ pub const Tensor = struct { test argMax { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); const allocator = std.testing.allocator; const ArgMaxTest = struct { pub fn _fwd(x: Tensor) Tensor.ArgMaxRes { @@ -2937,11 +2922,11 @@ pub const Tensor = struct { }; const x: Tensor = .init(.{ 1, 5 }, .f32); - var exe = try zml.module.compile(allocator, std.testing.io, ArgMaxTest._fwd, .{x}, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(allocator, std.testing.io, ArgMaxTest._fwd, .{x}, platform, .{}); defer exe.deinit(); { - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, 4.1, 7.9, 0, 7.9 }})); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, 4.1, 7.9, 0, 7.9 }})); defer x_buffer.deinit(); var res = try zml.testing.autoCall(allocator, std.testing.io, &exe, ArgMaxTest._fwd, .{x_buffer}); @@ -2955,7 +2940,7 @@ pub const Tensor = struct { } { - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, std.math.nan(f32), 7.9, 0, 7.9 }})); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, std.math.nan(f32), 7.9, 0, 7.9 }})); defer x_buffer.deinit(); var res = try zml.testing.autoCall(allocator, std.testing.io, &exe, ArgMaxTest._fwd, .{x_buffer}); @@ -2995,7 +2980,6 @@ pub const Tensor = struct { test argsort { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); const Local = struct { pub fn _argsort(x: Tensor, axis_: u3, opts: ArgSortOpts) Tensor { @@ -3007,10 +2991,10 @@ pub const Tensor = struct { { const x: Tensor = .init(.{ 2, 5 }, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{} }, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{} }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 })); defer x_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); @@ -3026,10 +3010,10 @@ pub const Tensor = struct { { const x: Tensor = .init(.{ 1, 5, 10 }, .f16); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{ .descending = true } }, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{ .descending = true } }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[_]f16{ + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]f16{ -0.2505, 1.2520, -0.7041, 0.1066, 1.2773, -1.7246, 0.8389, 1.1094, 0.0601, 1.0684, 0.9619, 1.3916, 1.2246, -0.1406, 0.3674, -1.2480, -1.7051, -0.0934, 0.3435, 0.4373, 1.3809, 0.5444, -0.6079, 1.2031, -0.6880, 1.2979, -0.1869, 0.2991, 0.0156, 0.1847, @@ -3057,10 +3041,10 @@ pub const Tensor = struct { { const x: Tensor = .init(.{ 4, 2, 1, 4 }, .i32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 3, .{} }, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 3, .{} }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[_]i32{ + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 89, 31, 22, 42, 64, 39, 0, 30, 64, 71, 46, 31, @@ -3223,10 +3207,9 @@ pub const Tensor = struct { test chunkExact { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{ .shardings = &.{replicated_sharding} }); + var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -3285,10 +3268,9 @@ pub const Tensor = struct { test chunkAllowTrailing { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{ .shardings = &.{replicated_sharding} }); + var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -3423,7 +3405,6 @@ pub const Tensor = struct { test dynamicSlice { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); inline for (.{ .{ [10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, Shape.init(.{10}, .f32), [2]f32{ 4, 5 }, 4, 0 }, @@ -3433,12 +3414,12 @@ pub const Tensor = struct { const x: Tensor = .fromShape(x_shape); const z: Tensor = .init(.{}, .i32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.dynamicSlice1d, .{ x, ax, .{ .len = 2, .start = z } }, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.dynamicSlice1d, .{ x, ax, .{ .len = 2, .start = z } }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&x_data)); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&x_data)); defer x_buffer.deinit(); - var z_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, z.shape(), replicated_sharding, std.mem.asBytes(&z_value)); + var z_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, z.shape(), .replicated, std.mem.asBytes(&z_value)); defer z_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.dynamicSlice1d, .{ x_buffer, .{ .start = z_buffer } }); @@ -3547,7 +3528,6 @@ pub const Tensor = struct { test dynamicUpdateSlice { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); { const x = Tensor.init(.{10}, .f32).withTags(.{.a}); @@ -3558,14 +3538,14 @@ pub const Tensor = struct { return x_.dynamicUpdateSlice(idx_, y_); } }._fwd; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = ids }, y }, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = ids }, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); defer y_buffer.deinit(); - var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1]i32{4})); + var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{4})); defer ids_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ .a = ids_buffer }, y_buffer }); @@ -3584,14 +3564,14 @@ pub const Tensor = struct { return x_.dynamicUpdateSlice(.{ .b = idx_ }, y_); } }._fwd; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); defer y_buffer.deinit(); - var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1]i32{3})); + var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{3})); defer ids_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, ids_buffer, y_buffer }); @@ -3613,14 +3593,14 @@ pub const Tensor = struct { return x_.dynamicUpdateSlice(.{ zml.Tensor.scalar(0, .i32), idx_ }, y_); } }._fwd; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2][1]f32{ .{-1}, .{-1} })); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[2][1]f32{ .{-1}, .{-1} })); defer y_buffer.deinit(); - var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1]i32{3})); + var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{3})); defer ids_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, ids_buffer, y_buffer }); @@ -3643,16 +3623,16 @@ pub const Tensor = struct { return x_.dynamicUpdateSlice(idx_, y_); } }._fwd; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = idx_a, .b = idx_b }, y }, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = idx_a, .b = idx_b }, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1]f32{-1})); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[1]f32{-1})); defer y_buffer.deinit(); - var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1]i32{1})); + var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{1})); defer idx_a_buffer.deinit(); - var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1]i32{3})); + var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{3})); defer idx_b_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ .a = idx_a_buffer, .b = idx_b_buffer }, y_buffer }); @@ -3675,16 +3655,16 @@ pub const Tensor = struct { return x_.dynamicUpdateSlice(&idx_, y_); } }._fwd; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ idx_a, idx_b }, y }, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ idx_a, idx_b }, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1][1]f32{.{-1}})); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[1][1]f32{.{-1}})); defer y_buffer.deinit(); - var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1]i32{1})); + var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{1})); defer idx_a_buffer.deinit(); - var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), replicated_sharding, std.mem.sliceAsBytes(&[1]i32{3})); + var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{3})); defer idx_b_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ idx_a_buffer, idx_b_buffer }, y_buffer }); @@ -3744,7 +3724,6 @@ pub const Tensor = struct { test toDiagonal { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); const Local = struct { pub fn _toDiag(input: Tensor) Tensor { @@ -3756,10 +3735,10 @@ pub const Tensor = struct { }; const x: Tensor = .init(.{ 2, 2 }, .u8); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._toDiag, .{x}, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._toDiag, .{x}, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[2][2]u8{ .{ 1, 2 }, .{ 3, 4 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[2][2]u8{ .{ 1, 2 }, .{ 3, 4 } })); defer x_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._toDiag, .{x_buffer}); @@ -3798,7 +3777,6 @@ pub const Tensor = struct { test triangular { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); const Local = struct { pub fn _tri(input: Tensor, num_diagonals: i32) Tensor { @@ -3808,7 +3786,7 @@ pub const Tensor = struct { const x: Tensor = .init(.{ 3, 3 }, .u8); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[3][3]u8{ + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[3][3]u8{ .{ 1, 1, 1 }, .{ 1, 1, 1 }, .{ 1, 1, 1 }, @@ -3816,7 +3794,7 @@ pub const Tensor = struct { defer x_buffer.deinit(); { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 0 }, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 0 }, platform, .{}); defer exe.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); @@ -3832,7 +3810,7 @@ pub const Tensor = struct { ); } { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 1 }, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 1 }, platform, .{}); defer exe.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); @@ -3848,7 +3826,7 @@ pub const Tensor = struct { ); } { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, -1 }, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, -1 }, platform, .{}); defer exe.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); @@ -3973,7 +3951,6 @@ pub const Tensor = struct { test cartesianProduct { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); const x: Tensor = .init(.{6}, .i32); const y: Tensor = .init(.{4}, .i32); @@ -3984,12 +3961,12 @@ pub const Tensor = struct { } }; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cartesianProduct2, .{ x, y }, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cartesianProduct2, .{ x, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), replicated_sharding, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); defer y_buffer.deinit(); var xs, var ys = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._cartesianProduct2, .{ x_buffer, y_buffer }); @@ -4048,7 +4025,6 @@ pub const Tensor = struct { test cartesianProductStacked { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); const x: Tensor = .init(.{6}, .i32); const y: Tensor = .init(.{4}, .i32); @@ -4058,12 +4034,12 @@ pub const Tensor = struct { } }; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, y }, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), replicated_sharding, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); defer y_buffer.deinit(); var z = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._fwd, .{ x_buffer, y_buffer }); @@ -4190,7 +4166,6 @@ fn getComparisonType(ctx: *mlir.Context, dtype: DataType) *const dialects.stable test "Tensor.maxPool1d" { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); const MaxPool = struct { pub fn _fwd(x: zml.Tensor) Tensor.ArgMaxRes { @@ -4206,10 +4181,10 @@ test "Tensor.maxPool1d" { const x: Tensor = .init(.{ 2, 2, 5 }, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&data)); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&data)); defer x_buffer.deinit(); var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, MaxPool._fwd, .{x_buffer}); @@ -4230,7 +4205,6 @@ test "Tensor.maxPool1d" { test "Tensor.maxPool2d" { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); const MaxPool = struct { pub fn _fwd(x: Tensor) Tensor.ArgMaxRes { @@ -4246,10 +4220,10 @@ test "Tensor.maxPool2d" { const x: Tensor = .init(.{ 2, 2, 5, 5 }, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform, .{ .shardings = &.{replicated_sharding} }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), replicated_sharding, std.mem.sliceAsBytes(&data)); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&data)); defer x_buffer.deinit(); var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, MaxPool._fwd, .{x_buffer}); @@ -4337,7 +4311,6 @@ test transposeIsJustAReshape { test "unused tensor" { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const replicated_sharding = zml.testing.replicatedSharding(); const Local = struct { pub fn _fwd(x: Tensor) Tensor { @@ -4353,7 +4326,7 @@ test "unused tensor" { Local._fwd, .{Tensor.init(.{10}, .f32)}, platform, - .{ .shardings = &.{replicated_sharding} }, + .{}, ); defer exe.deinit(); } diff --git a/zml/testing.zig b/zml/testing.zig index 508cd820..84e409d4 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -4,14 +4,12 @@ const builtin = @import("builtin"); const stdx = @import("stdx"); const Platform = @import("platform.zig").Platform; -const sharding = @import("sharding.zig"); const Slice = @import("slice.zig").Slice; const zml = @import("zml.zig"); const log = std.log.scoped(.@"zml/testing"); var _platform: ?*const Platform = null; -var _replicated_sharding: ?sharding.Sharding = null; pub fn env() *const Platform { if (!builtin.is_test) @compileError("Cannot use zml.testing.env outside of a test block"); @@ -24,17 +22,6 @@ pub fn env() *const Platform { return _platform.?; } -pub fn physicalMesh() sharding.PhysicalMesh { - return env().physical_mesh; -} - -pub fn replicatedSharding() sharding.Sharding { - if (_replicated_sharding == null) { - _replicated_sharding = sharding.replicatedSharding(env()) catch unreachable; - } - return _replicated_sharding.?; -} - /// In neural network we generally care about the relative precision, /// but on a given dimension, if the output is close to 0, then the precision /// don't matter as much. @@ -302,7 +289,7 @@ pub fn testLayer( activation_store: zml.io.TensorStore.View, name: []const u8, layer_weights: zml.Bufferized(@TypeOf(layer)), - shardings: []const zml.sharding.Sharding, + shardings: []const *const zml.Sharding, opts: CompareOpts, ) !void { var arena = std.heap.ArenaAllocator.init(allocator); @@ -332,14 +319,12 @@ pub fn testLayer( var buffer: [256]u8 = undefined; const subkey = std.fmt.bufPrint(&buffer, "{d}", .{ctx_.index}) catch unreachable; - tensor.* = ctx_.activation_store.createTensor(subkey, null, null); + tensor.* = ctx_.activation_store.createTensor(subkey, null, .replicated); ctx_.index += 1; } }.cb, &ctx, &args); - const replicated_sharding = try zml.sharding.replicatedSharding(platform); - - const exe = try platform.compile(allocator, io, layer, func, args, .{ .shardings = if (shardings.len != 0) shardings else &.{replicated_sharding} }); + const exe = try platform.compile(allocator, io, layer, func, args, .{ .shardings = if (shardings.len != 0) shardings else &.{} }); defer exe.deinit(); const output_name = try std.fmt.allocPrint(arena.allocator(), "{s}.out", .{name}); diff --git a/zml/zml.zig b/zml/zml.zig index 72304eac..4a037716 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -23,6 +23,7 @@ pub const exe = @import("exe.zig"); pub const Exe = exe.Exe; pub const floats = @import("floats.zig"); pub const io = @import("io.zig"); +pub const kernel = @import("kernel.zig"); pub const mem = @import("mem.zig"); pub const Bufferized = mem.Bufferized; pub const meta = @import("meta.zig"); @@ -39,14 +40,13 @@ pub const Target = platform.Target; pub const safetensors = @import("safetensors.zig"); pub const shape = @import("shape.zig"); pub const Shape = shape.Shape; -pub const sharding = @import("sharding.zig"); +pub const Sharding = @import("Sharding.zig"); pub const slice = @import("slice.zig"); pub const Slice = slice.Slice; pub const tensor = @import("tensor.zig"); pub const Tensor = tensor.Tensor; pub const testing = @import("testing.zig"); pub const tracer = @import("tracer.zig"); -pub const kernel = @import("kernel.zig"); test "zml" { std.testing.refAllDecls(@This()); From 5d94f9b83cd3f94863f85d833b220f10978ac798 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 11 May 2026 09:37:05 +0200 Subject: [PATCH 471/881] mlir: add shardy dialect (#526) Add bindings to shardy dialect, remove the string generation code from zml.Sharding, and replace it with mlir calls. --- mlir/dialects/BUILD.bazel | 2 + mlir/dialects/dialects.zig | 1 + mlir/dialects/shardy.zig | 448 +++++++++++++++++++++++++++++++++++++ stdx/stdx.zig | 9 + zml/Sharding.zig | 154 +++++++------ zml/module.zig | 18 +- zml/ops.zig | 21 +- zml/tensor.zig | 7 +- 8 files changed, 560 insertions(+), 100 deletions(-) create mode 100644 mlir/dialects/shardy.zig diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/BUILD.bazel index 4dd16c1a..c60cd41c 100644 --- a/mlir/dialects/BUILD.bazel +++ b/mlir/dialects/BUILD.bazel @@ -10,6 +10,7 @@ zig_library( "func.zig", "math.zig", "memref.zig", + "shardy.zig", "scf.zig", "vector.zig", ], @@ -18,6 +19,7 @@ zig_library( visibility = ["//visibility:public"], deps = [ "//mlir", + "//mlir:c", "//mlir/dialects/mosaic_tpu", "//mlir/dialects/stablehlo", "//mlir/dialects/ttir", diff --git a/mlir/dialects/dialects.zig b/mlir/dialects/dialects.zig index fa5e1245..0c88eeaa 100644 --- a/mlir/dialects/dialects.zig +++ b/mlir/dialects/dialects.zig @@ -11,6 +11,7 @@ pub const func = @import("func.zig"); pub const math = @import("math.zig"); pub const memref = @import("memref.zig"); pub const scf = @import("scf.zig"); +pub const shardy = @import("shardy.zig"); pub const vector = @import("vector.zig"); test { diff --git a/mlir/dialects/shardy.zig b/mlir/dialects/shardy.zig new file mode 100644 index 00000000..41aa8e2b --- /dev/null +++ b/mlir/dialects/shardy.zig @@ -0,0 +1,448 @@ +//! Bindings for OpenXLA shardy dialect +//! +//! - dialect overview: https://openxla.org/shardy/sdy_dialect +//! - C API: https://github.com/openxla/shardy/blob/main/shardy/integrations/c/attributes.h +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); + +pub fn dialectHandle() *const mlir.DialectHandle { + return @ptrCast(c.mlirGetDialectHandle__sdy__().ptr); +} + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + dialectHandle().insertDialect(registry); +} + +pub const MeshAxisAttribute = opaque { + const M = mlir.Methods(MeshAxisAttribute, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsAMeshAxisAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn init(ctx: *mlir.Context, axis_name: []const u8, axis_size: i64) *const MeshAxisAttribute { + return @ptrCast(c.sdyMeshAxisAttrGet(ctx.ptr(), mlir.stringRef(axis_name), axis_size).ptr); + } + + pub fn asAttr(self: *const MeshAxisAttribute) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn name(self: *const MeshAxisAttribute) []const u8 { + return mlir.string(c.sdyMeshAxisAttrGetName(self.ptr())); + } + + pub fn size(self: *const MeshAxisAttribute) i64 { + return c.sdyMeshAxisAttrGetSize(self.ptr()); + } +}; + +pub const MeshAttribute = opaque { + const M = mlir.Methods(MeshAttribute, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsAMeshAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const Mesh = struct { + axes: []const *const MeshAxisAttribute, + device_ids: []const i64 = &.{}, + }; + + pub fn init(ctx: *mlir.Context, mesh: Mesh) *const MeshAttribute { + const axes: []const c.struct_MlirAttribute = @ptrCast(mesh.axes); + return @ptrCast(c.sdyMeshAttrGet( + ctx.ptr(), + @intCast(axes.len), + axes.ptr, + @intCast(mesh.device_ids.len), + mesh.device_ids.ptr, + ).ptr); + } + + pub fn asAttr(self: *const MeshAttribute) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub const SubAxisInfoAttribute = opaque { + const M = mlir.Methods(SubAxisInfoAttribute, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsASubAxisInfoAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn init(ctx: *mlir.Context, pre_size: i64, sub_axis_size: i64) *const SubAxisInfoAttribute { + return @ptrCast(c.sdySubAxisInfoAttrGet(ctx.ptr(), pre_size, sub_axis_size).ptr); + } + + pub fn asAttr(self: *const SubAxisInfoAttribute) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn preSize(self: *const SubAxisInfoAttribute) i64 { + return c.sdySubAxisInfoAttrGetPreSize(self.ptr()); + } + + pub fn size(self: *const SubAxisInfoAttribute) i64 { + return c.sdySubAxisInfoAttrGetSize(self.ptr()); + } +}; + +pub const AxisRefAttribute = opaque { + const M = mlir.Methods(AxisRefAttribute, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsAnAxisRefAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn named(ctx: *mlir.Context, axis_name: []const u8) *const AxisRefAttribute { + return @ptrCast(c.sdyAxisRefAttrGet(ctx.ptr(), mlir.stringRef(axis_name), .{ .ptr = null }).ptr); + } + + pub fn subAxis(ctx: *mlir.Context, axis_name: []const u8, pre_size: i64, size: i64) *const AxisRefAttribute { + const sub_axis: *const SubAxisInfoAttribute = .init(ctx, pre_size, size); + return @ptrCast(c.sdyAxisRefAttrGet(ctx.ptr(), mlir.stringRef(axis_name), sub_axis).ptr); + } + + pub fn asAttr(self: *const AxisRefAttribute) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn name(self: *const AxisRefAttribute) []const u8 { + return mlir.string(c.sdyAxisRefAttrGetName(self.ptr())); + } + + pub fn subAxisInfo(self: *const AxisRefAttribute) ?*const SubAxisInfoAttribute { + const attr = c.sdyAxisRefAttrGetSubAxisInfo(self.ptr()); + return if (attr.ptr == null) null else @ptrCast(attr.ptr); + } +}; + +pub const DimensionShardingAttribute = opaque { + const M = mlir.Methods(DimensionShardingAttribute, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsADimensionShardingAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn init( + ctx: *mlir.Context, + axes: []const *const AxisRefAttribute, + is_closed: bool, + priority_: ?i64, + ) *const DimensionShardingAttribute { + const attrs: []const c.struct_MlirAttribute = @ptrCast(axes); + return @ptrCast(c.sdyDimensionShardingAttrGet(ctx.ptr(), @intCast(attrs.len), attrs.ptr, is_closed, priority_ orelse -1).ptr); + } + + pub fn closed(ctx: *mlir.Context, axes: []const *const AxisRefAttribute) *const DimensionShardingAttribute { + return init(ctx, axes, true, null); + } + + pub fn open(ctx: *mlir.Context, axes: []const *const AxisRefAttribute) *const DimensionShardingAttribute { + return init(ctx, axes, false, null); + } + + pub fn replicated(ctx: *mlir.Context) *const DimensionShardingAttribute { + return closed(ctx, &.{}); + } + + pub fn asAttr(self: *const DimensionShardingAttribute) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn numAxes(self: *const DimensionShardingAttribute) usize { + return @intCast(c.sdyDimensionShardingAttrGetAxesSize(self.ptr())); + } + + pub fn axis(self: *const DimensionShardingAttribute, index: usize) *const AxisRefAttribute { + return @ptrCast(c.sdyDimensionShardingAttrGetAxesElem(self.ptr(), @intCast(index)).ptr); + } + + pub fn isClosed(self: *const DimensionShardingAttribute) bool { + return c.sdyDimensionShardingAttrGetIsClosed(self.ptr()); + } + + pub fn priority(self: *const DimensionShardingAttribute) ?i64 { + const value = c.sdyDimensionShardingAttrGetPriority(self.ptr()); + return if (value == -1) null else value; + } +}; + +pub const Sharding = struct { + mesh: []const u8, + dimensions: []const *const DimensionShardingAttribute, + replicated_axes: []const *const AxisRefAttribute = &.{}, + unreduced_axes: []const *const AxisRefAttribute = &.{}, +}; + +pub const TensorShardingAttribute = opaque { + const M = mlir.Methods(TensorShardingAttribute, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsATensorShardingAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn init(ctx: *mlir.Context, args: Sharding) *const TensorShardingAttribute { + comptime std.debug.assert(@sizeOf(c.struct_MlirAttribute) == @sizeOf(*const mlir.Attribute)); + const dimensions: []const c.struct_MlirAttribute = @ptrCast(args.dimensions); + const replicated: []const c.struct_MlirAttribute = @ptrCast(args.replicated_axes); + const unreduced: []const c.struct_MlirAttribute = @ptrCast(args.unreduced_axes); + return @ptrCast(c.sdyTensorShardingAttrGet( + ctx.ptr(), + mlir.flatSymbolRefAttribute(ctx, args.mesh).ptr(), + @intCast(dimensions.len), + dimensions.ptr, + @intCast(replicated.len), + replicated.ptr, + @intCast(unreduced.len), + unreduced.ptr, + ).ptr); + } + + pub fn asAttr(self: *const TensorShardingAttribute) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn meshOrRef(self: *const TensorShardingAttribute) *const mlir.Attribute { + return @ptrCast(c.sdyTensorShardingAttrGetMeshOrRef(self.ptr()).ptr); + } + + pub fn numDimensions(self: *const TensorShardingAttribute) usize { + return @intCast(c.sdyTensorShardingAttrGetDimShardingsSize(self.ptr())); + } + + pub fn dimension(self: *const TensorShardingAttribute, index: usize) *const DimensionShardingAttribute { + return @ptrCast(c.sdyTensorShardingAttrGetDimShardingsElem(self.ptr(), @intCast(index)).ptr); + } + + pub fn numReplicatedAxes(self: *const TensorShardingAttribute) usize { + return @intCast(c.sdyTensorShardingAttrGetReplicatedAxesSize(self.ptr())); + } + + pub fn replicatedAxis(self: *const TensorShardingAttribute, index: usize) *const AxisRefAttribute { + return @ptrCast(c.sdyTensorShardingAttrGetReplicatedAxesElem(self.ptr(), @intCast(index)).ptr); + } + + pub fn numUnreducedAxes(self: *const TensorShardingAttribute) usize { + return @intCast(c.sdyTensorShardingAttrGetUnreducedAxesSize(self.ptr())); + } + + pub fn unreducedAxis(self: *const TensorShardingAttribute, index: usize) *const AxisRefAttribute { + return @ptrCast(c.sdyTensorShardingAttrGetUnreducedAxesElem(self.ptr(), @intCast(index)).ptr); + } +}; + +test TensorShardingAttribute { + const registry: *mlir.DialectRegistry = try .init(); + defer registry.deinit(); + registry.registerDialect("sdy"); + mlir.registerFuncExtensions(registry); + + var ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + + var w: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer w.deinit(); + + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttribute = .init(ctx, .{ + .mesh = "folded_mesh", + .dimensions = &.{.replicated(ctx)}, + .replicated_axes = &.{ .named(ctx, "link_x"), .named(ctx, "link_y") }, + }); + try sharding.asAttr().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@folded_mesh, [{}], replicated={\"link_x\", \"link_y\"}>", w.written()); + } + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttribute = .init(ctx, .{ + .mesh = "suggested_mesh", + .dimensions = &.{ + .closed(ctx, &.{.named(ctx, "link_z")}), + .closed(ctx, &.{.named(ctx, "link_x")}), + }, + .replicated_axes = &.{.named(ctx, "link_y")}, + }); + try sharding.asAttr().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@suggested_mesh, [{\"link_z\"}, {\"link_x\"}], replicated={\"link_y\"}>", w.written()); + } + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttribute = .init(ctx, .{ + .mesh = "fold_mesh", + .dimensions = &.{ .closed(ctx, &.{.named(ctx, "link_x")}), .replicated(ctx) }, + }); + try sharding.asAttr().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@fold_mesh, [{\"link_x\"}, {}]>", w.written()); + } + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttribute = .init(ctx, .{ + .mesh = "folded_mesh", + .dimensions = &.{.closed(ctx, &.{ .named(ctx, "link_x"), .named(ctx, "link_y") })}, + }); + try sharding.asAttr().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@folded_mesh, [{\"link_x\", \"link_y\"}]>", w.written()); + } + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttribute = .init(ctx, .{ + .mesh = "strategy_fold", + .dimensions = &.{.closed(ctx, &.{.named(ctx, "link_x")})}, + .replicated_axes = &.{.named(ctx, "link_y")}, + }); + try sharding.asAttr().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@strategy_fold, [{\"link_x\"}], replicated={\"link_y\"}>", w.written()); + } + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttribute = .init(ctx, .{ + .mesh = "mix_mesh", + .dimensions = &.{ .open(ctx, &.{}), .replicated(ctx) }, + .replicated_axes = &.{ .named(ctx, "link_x"), .named(ctx, "link_y") }, + }); + try sharding.asAttr().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@mix_mesh, [{?}, {}], replicated={\"link_x\", \"link_y\"}>", w.written()); + } + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttribute = .init(ctx, .{ + .mesh = "3d_mesh", + .dimensions = &.{ + .closed(ctx, &.{.named(ctx, "link_x")}), + .closed(ctx, &.{.named(ctx, "link_y")}), + .closed(ctx, &.{.named(ctx, "link_z")}), + }, + }); + try sharding.asAttr().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@\"3d_mesh\", [{\"link_x\"}, {\"link_y\"}, {\"link_z\"}]>", w.written()); + } +} + +pub const TensorShardingPerValueAttribute = opaque { + const M = mlir.Methods(TensorShardingPerValueAttribute, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsATensorShardingPerValueAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn init(ctx: *mlir.Context, shardings: []const *const TensorShardingAttribute) *const TensorShardingPerValueAttribute { + const attrs: []const c.struct_MlirAttribute = @ptrCast(shardings); + return @ptrCast(c.sdyTensorShardingPerValueAttrGet(ctx.ptr(), @intCast(attrs.len), attrs.ptr).ptr); + } + + pub fn asAttr(self: *const TensorShardingPerValueAttribute) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +test TensorShardingPerValueAttribute { + const registry: *mlir.DialectRegistry = try .init(); + defer registry.deinit(); + registry.registerDialect("sdy"); + mlir.registerFuncExtensions(registry); + + var ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + + const shardings: [2]*const TensorShardingAttribute = .{ + .init(ctx, .{ + .mesh = "mesh", + .dimensions = &.{.closed(ctx, &.{.named(ctx, "link_x")})}, + }), + .init(ctx, .{ + .mesh = "mesh", + .dimensions = &.{.replicated(ctx)}, + .replicated_axes = &.{.named(ctx, "link_y")}, + }), + }; + + const per_value = TensorShardingPerValueAttribute.init(ctx, &shardings); + + var w: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer w.deinit(); + + try per_value.asAttr().format(&w.writer); + try std.testing.expectEqualSlices( + u8, + "#sdy.sharding_per_value<[<@mesh, [{\"link_x\"}]>, <@mesh, [{}], replicated={\"link_y\"}>]>", + w.written(), + ); +} + +pub const ManualAxesAttribute = opaque { + const M = mlir.Methods(ManualAxesAttribute, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsAManualAxesAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn init(ctx: *mlir.Context, axes: []const *const mlir.StringAttribute) *const ManualAxesAttribute { + comptime std.debug.assert(@sizeOf(c.struct_MlirAttribute) == @sizeOf(*const mlir.StringAttribute)); + const attrs: []const c.struct_MlirAttribute = @ptrCast(axes); + return @ptrCast(c.sdyManualAxesAttrGet(ctx.ptr(), @intCast(attrs.len), attrs.ptr).ptr); + } + + pub fn asAttr(self: *const ManualAxesAttribute) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn numAxes(self: *const ManualAxesAttribute) usize { + return @intCast(c.sdyManualAxesAttrGetAxesSize(self.ptr())); + } + + pub fn axis(self: *const ManualAxesAttribute, index: usize) []const u8 { + return mlir.string(c.sdyManualAxesAttrGetAxesElem(self.ptr(), @intCast(index))); + } +}; + +test ManualAxesAttribute { + const registry: *mlir.DialectRegistry = try .init(); + defer registry.deinit(); + registry.registerDialect("sdy"); + mlir.registerFuncExtensions(registry); + + var ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + + const axes: [2]*const mlir.StringAttribute = .{ + .init(ctx, "link_x"), + .init(ctx, "link_y"), + }; + + const manual_axes: *const ManualAxesAttribute = .init(ctx, &axes); + + try std.testing.expectEqual(2, manual_axes.numAxes()); + try std.testing.expectEqualStrings("link_x", manual_axes.axis(0)); + try std.testing.expectEqualStrings("link_y", manual_axes.axis(1)); + + var w: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer w.deinit(); + + try manual_axes.asAttr().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy", w.written()); +} diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 5ccf1cb4..40419397 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -29,6 +29,15 @@ pub inline fn stackSlice(comptime max_len: usize, T: type, len: usize) []T { pub const noalloc: std.mem.Allocator = if (builtin.mode == .ReleaseFast) undefined else std.testing.failing_allocator; +pub fn arenaWithCapacity(parent: std.mem.Allocator, initial_capacity: usize) std.mem.Allocator.Error!std.heap.ArenaAllocator { + var a: std.heap.ArenaAllocator = .init(parent); + + _ = try a.allocator().alloc(u8, initial_capacity); + std.debug.assert(a.state.used_list.?.end_index == initial_capacity); + a.state.used_list.?.end_index = 0; + return a; +} + pub fn pinToCore(core_id: usize) void { if (builtin.os.tag == .linux) { const CPUSet = std.bit_set.ArrayBitSet(usize, std.os.linux.CPU_SETSIZE * @sizeOf(usize)); diff --git a/zml/Sharding.zig b/zml/Sharding.zig index c284106d..43d61bc3 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -1,6 +1,8 @@ //! Explain how to split a buffer across different devices. const std = @import("std"); +const dialects = @import("mlir/dialects"); +const mlir = @import("mlir"); const pjrt = @import("pjrt"); const stdx = @import("stdx"); @@ -91,11 +93,11 @@ pub const Partitioning = struct { }; } - pub fn tensorShardingAttr(self: Partitioning, allocator: std.mem.Allocator, shape: Shape, sharding: ?Sharding) ![]const u8 { + pub fn tensorShardingAttr(self: Partitioning, allocator: std.mem.Allocator, ctx: *mlir.Context, shape: Shape, sharding: ?Sharding) !*const mlir.Attribute { const selected_sharding = sharding orelse try self.selectSharding(shape); return switch (self.partitioner) { - .shardy => try selected_sharding.data.sdyShardingAttrForShape(allocator, shape), - .gspmd => try selected_sharding.data.gspmdShardingAttrForShape(allocator, shape), + .shardy => (try selected_sharding.data.sdyShardingAttrForShape(allocator, ctx, shape)).asAttr(), + .gspmd => mlir.stringAttribute(ctx, try selected_sharding.data.gspmdShardingAttrForShape(allocator, shape)), }; } @@ -110,21 +112,18 @@ pub const Partitioning = struct { return sharding.data.numPartitionsForLogicalAxis(logical_axis); } - pub fn sdyPerValueShardingAttr(self: Partitioning, allocator: std.mem.Allocator, shapes: []const Shape) ![]const u8 { + pub fn sdyPerValueShardingAttr(self: Partitioning, allocator: std.mem.Allocator, ctx: *mlir.Context, shapes: []const Shape) !*const dialects.shardy.TensorShardingPerValueAttribute { stdx.debug.assert(self.partitioner == .shardy, "sdyPerValueShardingAttr requires shardy partitioner", .{}); - var out: std.Io.Writer.Allocating = .init(allocator); - out.writer.writeAll("#sdy.sharding_per_value<[") catch unreachable; + const shardings = try allocator.alloc(*const dialects.shardy.TensorShardingAttribute, shapes.len); for (shapes, 0..) |shape, i| { - const attr_str = try self.tensorShardingAttr(allocator, shape, null); - if (i > 0) out.writer.writeAll(", ") catch unreachable; - out.writer.writeAll(attr_str["#sdy.sharding".len..]) catch unreachable; + const sharding = try self.selectSharding(shape); + shardings[i] = try sharding.data.sdyShardingAttrForShape(allocator, ctx, shape); } - out.writer.writeAll("]>") catch unreachable; - return out.toOwnedSlice() catch unreachable; + return .init(ctx, shardings); } - pub fn sdyManualAxesAttr(self: Partitioning, allocator: std.mem.Allocator, in_shapes: []const Shape, out_shapes: []const Shape) ![]const u8 { + pub fn sdyManualAxesAttr(self: Partitioning, allocator: std.mem.Allocator, ctx: *mlir.Context, in_shapes: []const Shape, out_shapes: []const Shape) !*const dialects.shardy.ManualAxesAttribute { stdx.debug.assert(self.partitioner == .shardy, "sdyManualAxesAttr requires shardy partitioner", .{}); var axis_names = std.ArrayList([]const u8).empty; @@ -137,36 +136,41 @@ pub const Partitioning = struct { } list.append(allocator_, axis_name) catch unreachable; } - - fn scan(list: *std.ArrayList([]const u8), allocator_: std.mem.Allocator, sharding: []const u8) void { - var i: usize = 0; - while (i < sharding.len) : (i += 1) { - if (sharding[i] != '"') continue; - const start = i + 1; - i = start; - while (i < sharding.len and sharding[i] != '"') : (i += 1) {} - if (i > start) appendUnique(list, allocator_, sharding[start..i]); - } - } }; for (in_shapes) |shape| { - const attr_str = try self.tensorShardingAttr(allocator, shape, null); - Collect.scan(&axis_names, allocator, attr_str); + const sharding = try self.selectSharding(shape); + const attr = try sharding.data.sdyShardingAttrForShape(allocator, ctx, shape); + for (0..attr.numReplicatedAxes()) |i| { + Collect.appendUnique(&axis_names, allocator, attr.replicatedAxis(i).name()); + } + for (0..attr.numDimensions()) |i| { + const dim = attr.dimension(i); + for (0..dim.numAxes()) |j| { + Collect.appendUnique(&axis_names, allocator, dim.axis(j).name()); + } + } } for (out_shapes) |shape| { - const attr_str = try self.tensorShardingAttr(allocator, shape, null); - Collect.scan(&axis_names, allocator, attr_str); + const sharding = try self.selectSharding(shape); + const attr = try sharding.data.sdyShardingAttrForShape(allocator, ctx, shape); + for (0..attr.numReplicatedAxes()) |i| { + Collect.appendUnique(&axis_names, allocator, attr.replicatedAxis(i).name()); + } + for (0..attr.numDimensions()) |i| { + const dim = attr.dimension(i); + for (0..dim.numAxes()) |j| { + Collect.appendUnique(&axis_names, allocator, dim.axis(j).name()); + } + } } - var out: std.Io.Writer.Allocating = .init(allocator); - out.writer.writeAll("#sdy 0) out.writer.writeAll(", ") catch unreachable; - out.writer.print("\"{s}\"", .{axis_name}) catch unreachable; + axes[i] = mlir.StringAttribute.init(ctx, axis_name); } - out.writer.writeAll("}>") catch unreachable; - return out.toOwnedSlice() catch unreachable; + + return dialects.shardy.ManualAxesAttribute.init(ctx, axes); } pub fn selectSharding(self: Partitioning, shape: Shape) !Sharding { @@ -1277,8 +1281,15 @@ pub const Data = struct { }; } - // TODO: consider binding shardy dialect instead of generating strings then parsing later. - pub fn sdyShardingAttrForShape(self: *const Data, allocator: std.mem.Allocator, shape: Shape) ![]const u8 { + pub fn sdyShardingAttrForShape( + data: *const Data, + parent_allocator: std.mem.Allocator, + ctx: *mlir.Context, + shape: Shape, + ) !*const dialects.shardy.TensorShardingAttribute { + var arena = try stdx.arenaWithCapacity(parent_allocator, 1024); + defer arena.deinit(); + const allocator = arena.allocator(); var any_explicit = false; for (0..shape.rank()) |ax| { if (shape.partition(ax) != .unknown) { @@ -1288,55 +1299,39 @@ pub const Data = struct { } const all_replicated = !any_explicit; - const mapping = self.getDimMapping(shape); - var out: std.Io.Writer.Allocating = .init(allocator); - errdefer out.deinit(); + const mapping = data.getDimMapping(shape); + const dimensions = try allocator.alloc(*const dialects.shardy.DimensionShardingAttribute, shape.rank()); - try out.writer.print("#sdy.sharding<@{s}, [", .{self.name}); - for (0..shape.rank()) |ax| { - if (ax > 0) try out.writer.writeAll(", "); + for (0.., dimensions) |ax, *d| { const spec = shape.partition(ax); - - switch (spec) { - .axis => |logical_tag| { - if (self.binding(logical_tag)) |_| { + d.* = switch (spec) { + .axis => |logical_tag| d: { + if (data.binding(logical_tag)) |_| { const dim_phys_indices = mapping.axes_per_dim.get(ax); if (dim_phys_indices.len == 0) { - try out.writer.writeAll("{}"); + break :d .replicated(ctx); } else { - try out.writer.writeAll("{"); + const axes = try allocator.alloc(*const dialects.shardy.AxisRefAttribute, dim_phys_indices.len); for (dim_phys_indices.constSlice(), 0..) |p_idx, i| { - if (i > 0) try out.writer.writeAll(", "); - try out.writer.print("\"{s}\"", .{@tagName(mapping.view.axes.get(p_idx).tag)}); + axes[i] = .named(ctx, @tagName(mapping.view.axes.get(p_idx).tag)); } - try out.writer.writeAll("}"); + break :d .closed(ctx, axes); } } else { - try out.writer.writeAll("{?}"); + break :d .open(ctx, &.{}); } }, - .replicated => try out.writer.writeAll("{}"), - .open, .unknown => { - if (all_replicated) { - try out.writer.writeAll("{}"); - } else { - try out.writer.writeAll("{?}"); - } - }, - } + .replicated => .replicated(ctx), + .open, .unknown => if (all_replicated) .replicated(ctx) else .open(ctx, &.{}), + }; } - try out.writer.writeAll("]"); - if (mapping.replicated_axes.len > 0) { - try out.writer.writeAll(", replicated={"); - for (mapping.replicated_axes.constSlice(), 0..) |p_idx, i| { - if (i > 0) try out.writer.writeAll(", "); - try out.writer.print("\"{s}\"", .{@tagName(mapping.view.axes.get(p_idx).tag)}); - } - try out.writer.writeAll("}"); + const replicated_axes = try allocator.alloc(*const dialects.shardy.AxisRefAttribute, mapping.replicated_axes.len); + for (replicated_axes, mapping.replicated_axes.constSlice()) |*r, p_idx| { + r.* = .named(ctx, @tagName(mapping.view.axes.get(p_idx).tag)); } - try out.writer.writeAll(">"); - return try out.toOwnedSlice(); + + return .init(ctx, .{ .mesh = data.name, .dimensions = dimensions, .replicated_axes = replicated_axes }); } pub fn gspmdShardingAttrForShape(self: *const Data, allocator: std.mem.Allocator, shape: Shape) ![]const u8 { @@ -1895,9 +1890,20 @@ const ShardingTest = struct { // Verify MLIR String if (s.expected_sdy) |expected_attr| { - const actual_attr = try sharding.sdyShardingAttrForShape(self.allocator, s.shape); - defer self.allocator.free(actual_attr); - try std.testing.expectEqualStrings(expected_attr, actual_attr); + const registry: *mlir.DialectRegistry = try .init(); + defer registry.deinit(); + registry.registerDialect("sdy"); + mlir.registerFuncExtensions(registry); + + var ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + + const actual_attr = try sharding.sdyShardingAttrForShape(self.allocator, ctx, s.shape); + var writer: std.Io.Writer.Allocating = .init(self.allocator); + defer writer.deinit(); + try actual_attr.asAttr().format(&writer.writer); + try std.testing.expectEqualStrings(expected_attr, writer.written()); } // Verify Placement logic / Error @@ -2112,7 +2118,7 @@ test "sharding: full 3D cluster sharding" { .sharding = try .init("3d_mesh", physical, logical, strategy), .shape = Shape.init(.{ .batch = 4, .model = 4, .context = 4 }, .f32) .withPartitioning(.{ .batch = .batch, .model = .model, .context = .context }), - .expected_sdy = "#sdy.sharding<@3d_mesh, [{\"link_x\"}, {\"link_y\"}, {\"link_z\"}]>", + .expected_sdy = "#sdy.sharding<@\"3d_mesh\", [{\"link_x\"}, {\"link_y\"}, {\"link_z\"}]>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{ .{ 0, 2 }, .{ 0, 2 }, .{ 0, 2 } } }, .{ .device_id = 1, .slices = &.{ .{ 0, 2 }, .{ 0, 2 }, .{ 2, 2 } } }, diff --git a/zml/module.zig b/zml/module.zig index 90b1a46f..5b3782c0 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -475,22 +475,20 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar _ = dialects.func.returns(compilation_context.mlir_ctx, output_info.values, .unknown(compilation_context.mlir_ctx)).appendTo(compilation_context.currentScope().block); for (input_info.shapes, input_info.shardings, 0..) |shape, sharding, i| { - const attr_str = try compilation_context.partitioning.tensorShardingAttr(compilation_context.arena.allocator(), shape, sharding); - - const name, const attr = switch (compilation_context.partitioning.partitioner) { - .gspmd => .{ "mhlo.sharding", mlir.stringAttribute(compilation_context.mlir_ctx, attr_str) }, - .shardy => .{ "sdy.sharding", try mlir.Attribute.parse(compilation_context.mlir_ctx, attr_str) }, + const attr = try compilation_context.partitioning.tensorShardingAttr(compilation_context.arena.allocator(), compilation_context.mlir_ctx, shape, sharding); + const name = switch (compilation_context.partitioning.partitioner) { + .gspmd => "mhlo.sharding", + .shardy => "sdy.sharding", }; input_attributes[i].appendAssumeCapacity(.named(compilation_context.mlir_ctx, name, attr)); } for (output_info.shapes, output_info.shardings, 0..) |shape, sharding, i| { - const attr_str = try compilation_context.partitioning.tensorShardingAttr(compilation_context.arena.allocator(), shape, sharding); - - const name, const attr = switch (compilation_context.partitioning.partitioner) { - .gspmd => .{ "mhlo.sharding", mlir.stringAttribute(compilation_context.mlir_ctx, attr_str) }, - .shardy => .{ "sdy.sharding", try mlir.Attribute.parse(compilation_context.mlir_ctx, attr_str) }, + const attr = try compilation_context.partitioning.tensorShardingAttr(compilation_context.arena.allocator(), compilation_context.mlir_ctx, shape, sharding); + const name = switch (compilation_context.partitioning.partitioner) { + .gspmd => "mhlo.sharding", + .shardy => "sdy.sharding", }; output_attributes[i].appendAssumeCapacity(.named(compilation_context.mlir_ctx, name, attr)); diff --git a/zml/ops.zig b/zml/ops.zig index 25d96479..9212c63e 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1251,12 +1251,9 @@ fn manualComputationInternal( local_output_shapes[i] = ctx.partitioning.localShapeForShape(output_shape) catch unreachable; } - const in_shardings_str = ctx.partitioning.sdyPerValueShardingAttr(allocator, input_shapes) catch unreachable; - const out_shardings_str = ctx.partitioning.sdyPerValueShardingAttr(allocator, outputs) catch unreachable; - const manual_axes_str = ctx.partitioning.sdyManualAxesAttr(allocator, input_shapes, outputs) catch unreachable; - const in_shardings_attr = mlir.Attribute.parse(ctx.mlir_ctx, in_shardings_str) catch unreachable; - const out_shardings_attr = mlir.Attribute.parse(ctx.mlir_ctx, out_shardings_str) catch unreachable; - const manual_axes_attr = mlir.Attribute.parse(ctx.mlir_ctx, manual_axes_str) catch unreachable; + const in_shardings_attr = ctx.partitioning.sdyPerValueShardingAttr(allocator, ctx.mlir_ctx, input_shapes) catch unreachable; + const out_shardings_attr = ctx.partitioning.sdyPerValueShardingAttr(allocator, ctx.mlir_ctx, outputs) catch unreachable; + const manual_axes_attr = ctx.partitioning.sdyManualAxesAttr(allocator, ctx.mlir_ctx, input_shapes, outputs) catch unreachable; const block_types = allocator.alloc(*const mlir.Type, inputs.len) catch unreachable; const block_locs = allocator.alloc(*const mlir.Location, inputs.len) catch unreachable; @@ -1308,9 +1305,9 @@ fn manualComputationInternal( .results = .{ .flat = global_result_types }, .blocks = &.{manual_block}, .attributes = &.{ - .named(ctx.mlir_ctx, "in_shardings", in_shardings_attr), - .named(ctx.mlir_ctx, "out_shardings", out_shardings_attr), - .named(ctx.mlir_ctx, "manual_axes", manual_axes_attr), + .named(ctx.mlir_ctx, "in_shardings", in_shardings_attr.asAttr()), + .named(ctx.mlir_ctx, "out_shardings", out_shardings_attr.asAttr()), + .named(ctx.mlir_ctx, "manual_axes", manual_axes_attr.asAttr()), }, .verify = true, .location = .unknown(ctx.mlir_ctx), @@ -1324,7 +1321,7 @@ fn manualComputationInternal( }, .gspmd => blk: { const manual_sharding = "{manual}"; - const output_shardings = allocator.alloc([]const u8, outputs.len) catch unreachable; + const output_shardings = allocator.alloc(*const mlir.Attribute, outputs.len) catch unreachable; const local_input_shapes = allocator.alloc(Shape, inputs.len) catch unreachable; const local_output_shapes = allocator.alloc(Shape, outputs.len) catch unreachable; @@ -1333,7 +1330,7 @@ fn manualComputationInternal( } for (outputs, 0..) |output_shape, i| { local_output_shapes[i] = ctx.partitioning.localShapeForShape(output_shape) catch unreachable; - output_shardings[i] = ctx.partitioning.tensorShardingAttr(allocator, output_shape, null) catch unreachable; + output_shardings[i] = ctx.partitioning.tensorShardingAttr(allocator, ctx.mlir_ctx, output_shape, null) catch unreachable; } const local_input_values = allocator.alloc(*const mlir.Value, inputs.len) catch unreachable; @@ -1389,7 +1386,7 @@ fn manualComputationInternal( .has_side_effect = false, .backend_config = .{ .original = "" }, .additional_attributes = &.{ - .named(ctx.mlir_ctx, "mhlo.sharding", mlir.stringAttribute(ctx.mlir_ctx, output_shardings[i])), + .named(ctx.mlir_ctx, "mhlo.sharding", output_shardings[i]), }, }, .unknown(ctx.mlir_ctx), diff --git a/zml/tensor.zig b/zml/tensor.zig index 33c21431..eb84949d 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -146,8 +146,7 @@ pub const Tensor = struct { const ctx = CompilationContext.current(); const partitioned_shape = self._shape.withPartitioning(axes_); - const attr = ctx.partitioning.tensorShardingAttr(ctx.allocator, partitioned_shape, null) catch unreachable; - defer ctx.allocator.free(attr); + const attr = ctx.partitioning.tensorShardingAttr(ctx.allocator, ctx.mlir_ctx, partitioned_shape, null) catch unreachable; const op_result = switch (ctx.partitioning.partitioner) { .shardy => blk: { @@ -155,7 +154,7 @@ pub const Tensor = struct { .operands = .{ .flat = &.{self.value()} }, .results = .{ .flat = &.{self.value().type_()} }, .attributes = &.{ - .named(ctx.mlir_ctx, "sharding", mlir.Attribute.parse(ctx.mlir_ctx, attr) catch unreachable), + .named(ctx.mlir_ctx, "sharding", attr), }, }).appendTo(currentBlock()); break :blk op.result(0); @@ -170,7 +169,7 @@ pub const Tensor = struct { .has_side_effect = false, .backend_config = .{ .original = "" }, .additional_attributes = &.{ - .named(ctx.mlir_ctx, "mhlo.sharding", mlir.stringAttribute(ctx.mlir_ctx, attr)), + .named(ctx.mlir_ctx, "mhlo.sharding", attr), }, }, .unknown(ctx.mlir_ctx), From 13cee57bf26faa5fb7c19ac7a293074f0d073264 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Mon, 11 May 2026 09:52:11 +0200 Subject: [PATCH 472/881] zml/profiling: unify profiling flow and tracing (#443) This PR modernizes ZML profiling in four areas: - replaces the old C++ `xspace_to_perfetto` tool with a native Zig exporter built on upb - integrates trace export directly into `zml/profiler.zig` - introduces a scoped tracing API in `zml.tracer` and wires it through examples - adds platform-specific annotation support and Bazel-native profiler wrappers for CUDA and ROCm The result is a simpler profiling flow, better large-trace behavior, and a more consistent experience across Linux CUDA, Linux ROCm, and macOS. --- .bazelrc | 7 + MODULE.bazel | 5 +- docs/README.md | 2 +- docs/howtos/profiling.md | 325 ++++++++ examples/llm/chat.zig | 9 +- examples/sharding/main.zig | 4 +- platforms/cuda/BUILD.bazel | 9 + platforms/cuda/cuda.bzl | 7 + platforms/rocm/libpjrt_rocm.BUILD.bazel | 4 +- platforms/rocm/rocm.bzl | 23 +- third_party/xla/BUILD.bazel | 2 +- third_party/xla/repo.bzl | 2 - third_party/xla/xspace_to_perfetto.patch | 24 - tools/BUILD.bazel | 29 +- tools/nsys.sh | 31 + tools/rocprofv3.sh | 51 ++ tools/xspace_to_perfetto.cc | 55 -- tools/xspace_to_perfetto.h | 23 - tools/xspace_to_perfetto/BUILD.bazel | 32 + tools/xspace_to_perfetto/cli.zig | 34 + .../xspace_to_perfetto/xspace_to_perfetto.zig | 712 ++++++++++++++++++ tools/xspace_to_perfetto_lib.cc | 202 ----- zml/BUILD.bazel | 42 +- zml/exe.zig | 13 + zml/io.zig | 6 + zml/module.zig | 7 + zml/platform.zig | 2 +- zml/{ => profiling}/profiler.zig | 111 ++- zml/profiling/traceme.h | 27 + zml/profiling/traceme_lib.cc | 252 +++++++ zml/profiling/tracer.zig | 165 ++++ zml/tracer.zig | 77 -- zml/zml.zig | 2 +- 33 files changed, 1838 insertions(+), 458 deletions(-) create mode 100644 docs/howtos/profiling.md delete mode 100644 third_party/xla/xspace_to_perfetto.patch create mode 100755 tools/nsys.sh create mode 100755 tools/rocprofv3.sh delete mode 100644 tools/xspace_to_perfetto.cc delete mode 100644 tools/xspace_to_perfetto.h create mode 100644 tools/xspace_to_perfetto/BUILD.bazel create mode 100644 tools/xspace_to_perfetto/cli.zig create mode 100644 tools/xspace_to_perfetto/xspace_to_perfetto.zig delete mode 100644 tools/xspace_to_perfetto_lib.cc rename zml/{ => profiling}/profiler.zig (69%) create mode 100644 zml/profiling/traceme.h create mode 100644 zml/profiling/traceme_lib.cc create mode 100644 zml/profiling/tracer.zig delete mode 100644 zml/tracer.zig diff --git a/.bazelrc b/.bazelrc index 76e445c7..530aa158 100644 --- a/.bazelrc +++ b/.bazelrc @@ -143,3 +143,10 @@ common:remote --grpc_keepalive_time=30s common:remote --jobs=2000 common:remote --extra_execution_platforms=@zml//:rbe_platform common:remote --experimental_repo_remote_exec=true + +# Runtime profiling helpers for `bazel run`. +run:nsys --@zml//platforms:cuda=true +run:nsys --run_under=//tools:nsys + +run:rocprofv3 --@zml//platforms:rocm=true +run:rocprofv3 --run_under=//tools:rocprofv3 diff --git a/MODULE.bazel b/MODULE.bazel index a351a940..f5900280 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -36,7 +36,6 @@ git_override( bazel_dep(name = "tar.bzl", version = "0.9.0") bazel_dep(name = "llvm", version = "0.7.3") -bazel_dep(name = "jsoncpp", version = "1.9.6", repo_name = "jsoncpp_git") bazel_dep(name = "with_cfg.bzl", version = "0.14.4") bazel_dep(name = "grpc", version = "1.74.0") @@ -139,10 +138,10 @@ cpu = use_extension("//platforms/cpu:cpu.bzl", "cpu_pjrt_plugin") use_repo(cpu, "libpjrt_cpu_darwin_amd64", "libpjrt_cpu_darwin_arm64", "libpjrt_cpu_linux_amd64") cuda = use_extension("//platforms/cuda:cuda.bzl", "cuda_packages") -use_repo(cuda, "libpjrt_cuda_linux_amd64", "libpjrt_cuda_linux_arm64", "cuda_nvml_dev_linux_x86_64", "cuda_nvml_dev_linux_sbsa", "zlib1g_linux_arm64", "zlib1g_linux_amd64") +use_repo(cuda, "libpjrt_cuda_linux_amd64", "libpjrt_cuda_linux_arm64", "cuda_nvml_dev_linux_x86_64", "cuda_nvml_dev_linux_sbsa", "zlib1g_linux_arm64", "zlib1g_linux_amd64", "cuda_nvtx_linux_x86_64", "cuda_nvtx_linux_sbsa") rocm = use_extension("//platforms/rocm:rocm.bzl", "rocm_packages") -use_repo(rocm, "amd-smi-lib", "hipblaslt", "libdrm-amdgpu-amdgpu1", "libdrm2-amdgpu", "libpjrt_rocm", "rocblas") +use_repo(rocm, "amd-smi-lib", "hipblaslt", "libdrm-amdgpu-amdgpu1", "libdrm2-amdgpu", "libpjrt_rocm", "rocblas", "rocprofiler-sdk", "rocprofiler-sdk-roctx") tpu = use_extension("//platforms/tpu:tpu.bzl", "tpu_packages") use_repo(tpu, "libpjrt_tpu") diff --git a/docs/README.md b/docs/README.md index 3bcfb3f3..03369120 100644 --- a/docs/README.md +++ b/docs/README.md @@ -19,6 +19,7 @@ Ready to write some code? Try starting with [your first model in ZML](./tutorial - [Port Pytorch models to ZML](./howtos/howto_torch2zml.md) - [Cross Compile and Deploy Models on a Server](./howtos/deploy_on_server.md) - [Dockerize Models](./howtos/dockerize_models.md) +- [Profile ZML Programs](./howtos/profiling.md) ## Learn more... @@ -27,4 +28,3 @@ Ready to write some code? Try starting with [your first model in ZML](./tutorial ## Contribute - [Style Guide](./misc/style_guide.md) - diff --git a/docs/howtos/profiling.md b/docs/howtos/profiling.md new file mode 100644 index 00000000..d60fc63c --- /dev/null +++ b/docs/howtos/profiling.md @@ -0,0 +1,325 @@ +# Profiling ZML Programs + +ZML has two profiling layers: + +- host tracing through `zml.tracer.scope(...)` +- backend profiling through `zml.Platform.profiler(...)` + +The host tracing path is always based on `TraceMe`. Depending on the target +platform, the tracing bridge also emits device-visible annotations: + +- CUDA on Linux: NVTX ranges +- ROCm on Linux: ROCTx ranges +- macOS: `os_signpost` intervals for Instruments + +When backend profiling is enabled, ZML writes both the raw `XSpace` protobuf +and a streamed Perfetto / trace-viewer JSON file. + +## Profiling architecture + +ZML profiling is split into four layers: + +1. `zml.tracer.scope(...)` in Zig marks lexical regions of interest +2. `zml/profiling/traceme_lib.cc` bridges those scopes into platform-specific + tracing backends +3. `zml.Platform.profiler(...)` manages a profiling session and collects profile + data from PJRT +4. `tools/xspace_to_perfetto` converts the final `XSpace` protobuf into a + streamed trace-event JSON file + +The important architectural split is: + +- `zml.tracer.scope(...)` is for instrumentation inside your code +- `zml.Platform.profiler(...)` is for session lifecycle and artifact generation + +You can use scopes without a profiler session, but you only get exported +`profiling.xplane.pb` and `profiling.trace.json` when a profiler session is +started. + +### Scope instrumentation path + +`zml.tracer.scope(...)` is the lowest-level API you use directly in Zig code. +Each scope creates a `TraceMe` region, and the C bridge adds extra +platform-visible annotations when supported: + +- CUDA/Linux: NVTX ranges +- ROCm/Linux: ROCTx ranges +- macOS: `os_signpost` intervals + +Metadata passed to `zml.tracer.scope(...)` is encoded into the `TraceMe` scope +name. That is why the API only accepts small scalar or string-like fields: it is +designed to produce compact trace labels rather than structured protobuf +payloads. + +### Profiler session path + +`zml.Platform.profiler(...)` builds a `tensorflow.ProfileOptions` protobuf from +`zml.Platform.ProfilerOptions` and asks the active PJRT backend for a profiler +session. + +At `start()` time, ZML does two distinct things: + +- it tries to start the backend PJRT profiler session +- it configures local host-side `TraceMeRecorder` capture when enabled + +At `stop()` time, ZML: + +1. stops the PJRT profiler session if one exists +2. collects the backend `XSpace` protobuf +3. merges locally recorded host `TraceMe` events into the host plane +4. writes the final protobuf to disk +5. streams trace JSON from that final protobuf + +This separation is important because host scopes and backend profiler output are +not produced by the same subsystem. ZML joins them only at the end of the +session. + +### Export path + +The final export path is intentionally one-way: + +- source data format: `XSpace` protobuf +- derived export format: trace-event JSON for Perfetto / trace viewer + +ZML keeps `profiling.xplane.pb` as the source of truth and generates +`profiling.trace.json` from it afterwards. The converter is implemented in Zig +and streams JSON directly from the protobuf instead of materializing the whole +JSON document in memory first. + +## The generated artifacts + +By default, a profiling session is written under: + +```text +/tmp/xprof/plugins/profile/profiling/ +``` + +The relevant files are: + +- `profiling.xplane.pb`: raw `XSpace` protobuf +- `profiling.trace.json`: Perfetto / trace-viewer JSON export + +These paths come from `zml/profiling/profiler.zig` and can be changed through +`zml.Platform.ProfilerOptions`. + +## Add trace scopes in Zig + +Use `zml.tracer.scope(name, metadata)` to bracket the code you want to inspect. + +```zig +const zml = @import("zml"); + +var trace = try zml.tracer.scope("llm.compile", .{ + .seqlen = args.seqlen, + .interactive = args.prompt == null, +}); +defer trace.end(); + +try models.LoadedModel.compile(...); +``` + +Use `.{}` when there is no metadata: + +```zig +var trace = try zml.tracer.scope("llm.load_buffers", .{}); +defer trace.end(); +``` + +Metadata is encoded into the `TraceMe` scope name, so keep it small and +high-signal. + +## Record a ZML profile from code + +If you want ZML to collect a backend profile and export `profiling.trace.json`, +create a profiler from the current platform: + +```zig +const platform: *zml.Platform = try .auto(allocator, io, .{}); + +var profiler = try platform.profiler(allocator, io, .defaults); +defer profiler.deinit(); + +try profiler.start(); +defer { + if ((profiler.stop() catch unreachable)) |profile| { + std.log.info("Profile dumped: {s} and {s}", .{ + profile.protobuf_path, + profile.perfetto_path, + }); + } +} +``` + +This is the normal ZML flow: + +1. start the PJRT backend profiler +2. record local `TraceMe` scopes +3. merge the local host trace into the backend `XSpace` +4. write `profiling.xplane.pb` +5. stream `profiling.trace.json` + +If the backend profiler is unavailable, ZML still keeps the local host trace +path and logs a warning. + +## CUDA on Linux with Nsight Systems + +ZML provides a Bazel run config for `nsys`: + +```bash +bazel run --config=nsys //examples/benchmark:benchmark +``` + +Or for a real model: + +```bash +bazel run --config=nsys //examples/llm -- \ + --model=/path/to/model \ + --prompt="Write a story about a cat" +``` + +The wrapper target is `//tools:nsys`. By default it runs: + +```text +sudo -E nsys profile ... +``` + +with CUDA, cuBLAS, cuSPARSE, cuDNN, NVTX, syscall, and OS runtime tracing +enabled. + +If you need to install Nsight Systems manually, use NVIDIA's official download +page: . For the server +or target machine where `nsys` will run, choose the `Linux CLI only` package. +For the client machine where you open and inspect reports, choose the full host +package for your OS. NVIDIA's installation guide is here: +. + +Useful overrides: + +- `ZML_PROFILE_NO_SUDO=1`: run `nsys` without `sudo` +- `ZML_NSYS_ARGS='...'`: replace the default `nsys` arguments + +Example: + +```bash +ZML_PROFILE_NO_SUDO=1 bazel run --config=nsys //examples/benchmark:benchmark +``` + +### Important interaction with `zml.profiler` + +The `nsys` wrapper sets: + +```text +SKIP_PJRT_PROFILER=true +``` + +That tells `zml/profiling/profiler.zig` to skip nested PJRT backend profiling. This is +intentional. Under `nsys`, you normally want: + +- the external Nsight session +- local `TraceMe` / NVTX annotations from your scopes + +and not a second backend profile session layered on top. + +## ROCm on Linux with `rocprofv3` + +ZML also provides a Bazel run config for ROCm: + +```bash +bazel run --config=rocprofv3 //examples/benchmark:benchmark +``` + +Or: + +```bash +bazel run --config=rocprofv3 //examples/llm -- \ + --model=/path/to/model \ + --prompt="Write a story about a cat" +``` + +The wrapper target is `//tools:rocprofv3`. + +This path is intentionally sandboxed inside the repository's ROCm packaging: + +- it launches the Bazel-runfiles copy of `rocprofv3` +- it passes `--rocm-root` pointing at the sandboxed ROCm runtime + +So it does not depend on a host-installed `rocprofv3` layout. + +The default wrapper enables system, runtime, HIP, kernel, memory copy, +scratch-memory, HSA core, and marker tracing, and writes `pftrace` output. + +Useful overrides: + +- `ZML_ROCPROFV3_ARGS='...'`: replace the default `rocprofv3` arguments + +### Important interaction with `zml.profiler` + +Like the `nsys` wrapper, the `rocprofv3` wrapper sets: + +```text +SKIP_PJRT_PROFILER=true +``` + +That means: + +- the external `rocprofv3` session owns backend/device profiling +- ZML still records local host `TraceMe` scopes +- ROCTx ranges from `zml.tracer.scope(...)` stay visible as device annotations + +## macOS Instruments + +On macOS, `zml.tracer.scope(...)` emits `os_signpost` intervals. These show up +in Instruments, especially in the `Points of Interest` template. + +Build the binary first: + +```bash +bazel build //examples/llm:llm +``` + +Then either launch it from the Instruments UI or use `xctrace`: + +```bash +xctrace record \ + --template 'Points of Interest' \ + --output /tmp/llm.trace \ + --launch -- \ + /home/you/zml/bazel-bin/examples/llm/llm \ + --model=/path/to/model \ + --prompt="Write a story about a cat" +``` + +Use full Xcode for `xctrace`. Command Line Tools alone are not enough. + +## Open the exported trace + +The `profiling.trace.json` file can be opened in Perfetto or in tools that still +understand the trace-event JSON format. + +Typical flow: + +1. run a ZML program with `zml.Platform.profiler(...)` +2. wait for `profiling.trace.json` +3. open the JSON in Perfetto + +The converter is implemented in Zig and streams JSON directly from `XSpace`, +which keeps large traces much more manageable than the old in-memory path. + +## Platform summary + +- Plain ZML profiling: backend `XSpace` profile plus merged local host trace +- CUDA + `--config=nsys`: external Nsight session plus local `TraceMe` / NVTX +- ROCm + `--config=rocprofv3`: external `rocprofv3` session plus local `TraceMe` / ROCTx +- macOS Instruments: local `TraceMe` scopes exposed as `os_signpost` + +## Troubleshooting + +- If you run under `--config=nsys` or `--config=rocprofv3`, PJRT profiling is + skipped on purpose because the wrappers set `SKIP_PJRT_PROFILER=true`. +- If you do not see device annotations, first check whether you are on a + supported platform: + - CUDA/Linux for NVTX + - ROCm/Linux for ROCTx + - macOS for Instruments signposts +- If you only need host scopes, `zml.tracer.scope(...)` works independently of + whether backend profiling succeeds. diff --git a/examples/llm/chat.zig b/examples/llm/chat.zig index a755bd90..a976c796 100644 --- a/examples/llm/chat.zig +++ b/examples/llm/chat.zig @@ -87,15 +87,8 @@ pub const Chat = struct { try self.runPrefill(prompt_tokens, &stdout.interface); try self.runDecodeTurn(&stdout.interface); - const profile = if (profiler) |*p| try p.stop() else null; + _ = if (profiler) |*p| try p.stop() else null; - if (opts.profile) { - if (profile) |trace| { - try stdout.interface.print("\x1b[2mprofile: perfetto: {s} protobuf: {s})\x1b[0m\n\n", .{ trace.perfetto_path, trace.protobuf_path }); - } else { - try stdout.interface.writeAll("\x1b[2mprofile: unavailable on this PJRT plugin\x1b[0m\n\n"); - } - } try stdout.interface.flush(); } diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index 097fd1cd..2602c2da 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -180,9 +180,7 @@ pub fn main(init: std.process.Init) !void { try profiler.start(); defer { - if ((profiler.stop() catch unreachable)) |profile| { - log.info("Profile dumped: {s} and {s}", .{ profile.protobuf_path, profile.perfetto_path }); - } + _ = profiler.stop() catch unreachable; } log.info("Partitioner: {s}", .{@tagName(args.partitioner)}); diff --git a/platforms/cuda/BUILD.bazel b/platforms/cuda/BUILD.bazel index 7db26a1f..3122bdaa 100644 --- a/platforms/cuda/BUILD.bazel +++ b/platforms/cuda/BUILD.bazel @@ -32,6 +32,15 @@ cc_library( }), ) +cc_library( + name = "cuda_nvtx_headers", + visibility = ["//visibility:public"], + deps = select({ + "@llvm//platforms/config:linux_x86_64": ["@cuda_nvtx_linux_x86_64//:headers"], + "@llvm//platforms/config:linux_aarch64": ["@cuda_nvtx_linux_sbsa//:headers"], + }), +) + cc_library( name = "libpjrt_cuda", tags = ["manual"], diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 874cafc4..4937d70d 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -63,6 +63,11 @@ CUDA_PACKAGES = { ), ], "cuda_nvtx": [ + packages.cc_library_hdrs_glob( + name = "headers", + hdrs_glob = ["include/nvtx3/**"], + includes = ["include"], + ), packages.filegroup( name = "cuda_nvtx", srcs = ["lib/libnvtx3interop.so"], @@ -354,6 +359,8 @@ def _cuda_impl(mctx): "cuda_nvml_dev_linux_sbsa", "zlib1g_linux_arm64", "zlib1g_linux_amd64", + "cuda_nvtx_linux_x86_64", + "cuda_nvtx_linux_sbsa", ], root_module_direct_dev_deps = [], ) diff --git a/platforms/rocm/libpjrt_rocm.BUILD.bazel b/platforms/rocm/libpjrt_rocm.BUILD.bazel index 21967a6c..efffb73e 100644 --- a/platforms/rocm/libpjrt_rocm.BUILD.bazel +++ b/platforms/rocm/libpjrt_rocm.BUILD.bazel @@ -79,7 +79,7 @@ copy_to_directory( "@rocm-device-libs//:runfiles", "@rocm-smi-lib//:rocm_smi", "@rocprofiler-register", - "@rocprofiler-sdk//:rocprofiler-sdk", + "@rocprofiler-sdk//:rocprofv3_tool", "@rocprofiler-sdk-rocpd//:rocprofiler-sdk-rocpd", "@rocprofiler-sdk-roctx//:rocprofiler-sdk-roctx", "@rocsparse", @@ -114,6 +114,7 @@ copy_to_directory( "rocm-core": "lib", "rocprofiler-register": "lib", "rocprofiler-sdk_so": "lib", + "rocprofiler-sdk-roctx": "lib", "rocm_smi": "lib", "rocsolver": "lib", "rocroller": "lib", @@ -124,6 +125,7 @@ copy_to_directory( }, add_directory_to_runfiles = True, include_external_repositories = ["**"], + visibility = ["//visibility:public"], ) cc_library( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 744d61d1..040a1084 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -85,9 +85,28 @@ _ROCM_PACKAGES = { ":rocprofiler-sdk_so", ":rocprofiler-sdk-attach", ]), + """filegroup( + name = "rocprofv3_tool", + srcs = glob([ + "bin/**", + "lib/**", + "libexec/rocprofiler-sdk/**", + "share/rocprofiler-sdk/**", + ]), + )""", ]), "rocprofiler-sdk-rocpd": packages.filegroup(name = "rocprofiler-sdk-rocpd", srcs = ["lib/librocprofiler-sdk-rocpd.so.1"]), - "rocprofiler-sdk-roctx": packages.filegroup(name = "rocprofiler-sdk-roctx", srcs = ["lib/librocprofiler-sdk-roctx.so.1"]), + "rocprofiler-sdk-roctx": "\n".join([ + packages.cc_library_hdrs_glob( + name = "headers", + hdrs_glob = ["include/rocprofiler-sdk-roctx/**/*.h"], + includes = ["include"], + ), + packages.filegroup( + name = "rocprofiler-sdk-roctx", + srcs = ["lib/librocprofiler-sdk-roctx.so.1"], + ), + ]), "hsa-rocr": _rocm_dlopen_patchelf( name = "hsa-runtime", src = "lib/libhsa-runtime64.so.1", @@ -260,7 +279,7 @@ def _rocm_impl(mctx): return mctx.extension_metadata( reproducible = True, - root_module_direct_deps = ["amd-smi-lib", "libpjrt_rocm", "libdrm2-amdgpu", "libdrm-amdgpu-amdgpu1", "hipblaslt", "rocblas"], + root_module_direct_deps = ["amd-smi-lib", "libpjrt_rocm", "libdrm2-amdgpu", "libdrm-amdgpu-amdgpu1", "hipblaslt", "rocblas", "rocprofiler-sdk", "rocprofiler-sdk-roctx"], root_module_direct_dev_deps = [], ) diff --git a/third_party/xla/BUILD.bazel b/third_party/xla/BUILD.bazel index 8de71230..75f731ff 100644 --- a/third_party/xla/BUILD.bazel +++ b/third_party/xla/BUILD.bazel @@ -1 +1 @@ -exports_files(["xspace_to_perfetto.patch"]) +# Package marker for XLA patches diff --git a/third_party/xla/repo.bzl b/third_party/xla/repo.bzl index 9250c314..d1af45ac 100644 --- a/third_party/xla/repo.bzl +++ b/third_party/xla/repo.bzl @@ -5,6 +5,4 @@ def repo(): name = "xla", remote = "https://github.com/openxla/xla.git", commit = "47f005bb8150a13cb0217c2d7daf108bcdca34cc", - patch_args = ["-p0"], - patches = ["//third_party/xla:xspace_to_perfetto.patch"], ) diff --git a/third_party/xla/xspace_to_perfetto.patch b/third_party/xla/xspace_to_perfetto.patch deleted file mode 100644 index d42cb950..00000000 --- a/third_party/xla/xspace_to_perfetto.patch +++ /dev/null @@ -1,24 +0,0 @@ ---- xla/tsl/profiler/convert/BUILD -+++ xla/tsl/profiler/convert/BUILD -@@ -104,9 +104,7 @@ cc_library( - srcs = ["trace_events_to_json.cc"], - hdrs = ["trace_events_to_json.h"], - copts = tf_profiler_copts(), -- visibility = internal_visibility([ -- "//xla/tsl/profiler:internal", -- ]), -+ visibility = ["//visibility:public"], - deps = [ - ":trace_container", - "//xla/tsl/platform:types", -@@ -150,9 +148,7 @@ cc_library( - srcs = ["xplane_to_trace_events.cc"], - hdrs = ["xplane_to_trace_events.h"], - copts = tf_profiler_copts(), -- visibility = internal_visibility([ -- "//xla/tsl/profiler:internal", -- ]), -+ visibility = ["//visibility:public"], - deps = [ - ":trace_container", - "//xla/tsl/platform:types", diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index d7532fcb..41d8092c 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -1,5 +1,3 @@ -load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_cc//cc:cc_test.bzl", "cc_test") load("@rules_python//python:py_library.bzl", "py_library") py_library( @@ -8,25 +6,18 @@ py_library( visibility = ["//visibility:public"], ) -cc_library( - name = "xspace_to_perfetto_lib", - tags = ["manual"], - srcs = ["xspace_to_perfetto_lib.cc"], - hdrs = ["xspace_to_perfetto.h"], +sh_binary( + name = "nsys", + srcs = ["nsys.sh"], visibility = ["//visibility:public"], - deps = [ - "//ffi:cc", - "@tsl//tsl/profiler/protobuf:xplane_proto_cc", - "@xla//xla/tsl/profiler/convert:post_process_single_host_xplane", - "@xla//xla/tsl/profiler/convert:trace_events_to_json", - "@xla//xla/tsl/profiler/convert:xplane_to_trace_events", - "@xla//xla/tsl/profiler/utils:xplane_utils", - ], ) -cc_binary( - name = "xspace_to_perfetto", - srcs = ["xspace_to_perfetto.cc"], +sh_binary( + name = "rocprofv3", + data = [ + "@libpjrt_rocm//:sandbox", + "@rocprofiler-sdk//:rocprofv3_tool", + ], + srcs = ["rocprofv3.sh"], visibility = ["//visibility:public"], - deps = [":xspace_to_perfetto_lib"], ) diff --git a/tools/nsys.sh b/tools/nsys.sh new file mode 100755 index 00000000..b8d38e38 --- /dev/null +++ b/tools/nsys.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "usage: $0 [args...]" >&2 + exit 1 +fi + +sudo_cmd=() +if [[ "${ZML_PROFILE_NO_SUDO:-0}" != "1" ]]; then + sudo_cmd=(sudo -E) +fi + +declare -a nsys_args +if [[ -n "${ZML_NSYS_ARGS:-}" ]]; then + nsys_args=(${ZML_NSYS_ARGS}) +else + nsys_args=( + profile + -t + cuda,syscall,nvtx,cublas,cublas-verbose,cusparse,cusparse-verbose,cudnn,osrt + --inherit-environment=true + --cuda-memory-usage + true + --cuda-event-trace=false + --backtrace=dwarf + --cuda-graph-trace=node + ) +fi + +exec env SKIP_PJRT_PROFILER=true "${sudo_cmd[@]}" nsys "${nsys_args[@]}" "$@" diff --git a/tools/rocprofv3.sh b/tools/rocprofv3.sh new file mode 100755 index 00000000..f698d893 --- /dev/null +++ b/tools/rocprofv3.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "usage: $0 [args...]" >&2 + exit 1 +fi + +if [[ -n "${RUNFILES_DIR:-}" ]]; then + runfiles_root="${RUNFILES_DIR}" +elif [[ -d "$0.runfiles" ]]; then + runfiles_root="$0.runfiles" +else + echo "failed to locate Bazel runfiles for sandboxed rocprofv3" >&2 + exit 1 +fi + +rocprofv3_bin="${runfiles_root}/+rocm_packages+rocprofiler-sdk/bin/rocprofv3" +if [[ ! -x "${rocprofv3_bin}" ]]; then + echo "sandboxed rocprofv3 not found at ${rocprofv3_bin}" >&2 + exit 1 +fi + +rocm_root="${runfiles_root}/+rocm_packages+libpjrt_rocm/sandbox" +if [[ ! -d "${rocm_root}" ]]; then + echo "sandboxed ROCm runtime root not found at ${rocm_root}" >&2 + exit 1 +fi + +declare -a rocprof_args +if [[ -n "${ZML_ROCPROFV3_ARGS:-}" ]]; then + rocprof_args=(${ZML_ROCPROFV3_ARGS}) +else + rocprof_args=( + --sys-trace + --runtime-trace + --hip-trace + --kernel-trace + --memory-copy-trace + --hip-runtime-trace + --hsa-core-trace + --scratch-memory-trace + --marker-trace + --kernel-rename + --stats + --output-format + pftrace + ) +fi + +exec env SKIP_PJRT_PROFILER=true "${rocprofv3_bin}" --rocm-root "${rocm_root}" "${rocprof_args[@]}" -- "$@" diff --git a/tools/xspace_to_perfetto.cc b/tools/xspace_to_perfetto.cc deleted file mode 100644 index daa44f69..00000000 --- a/tools/xspace_to_perfetto.cc +++ /dev/null @@ -1,55 +0,0 @@ -#include -#include -#include -#include - -#include "ffi/zig_allocator.h" -#include "ffi/zig_slice.h" - -namespace zml::tools { -bool DumpXSpaceFileToTraceJsonFile(std::string_view input_path, - std::string_view output_path, - zig_allocator* error_allocator, - zig_slice* error); -} - -void PrintUsage(std::string_view argv0) { - std::cerr << "Usage: " << argv0 << " [output_trace.json]\n"; -} - -void* Alloc(const void*, size_t elem, size_t nelems, size_t) { - return std::malloc(elem * nelems); -} - -void Free(const void*, void* ptr, size_t, size_t, size_t) { - std::free(ptr); -} - -int main(int argc, char** argv) { - if (argc < 2 || argc > 3) { - PrintUsage(argv[0]); - return 1; - } - - const std::string input_path = argv[1]; - const std::string output_path = argc == 3 ? argv[2] : input_path + ".trace.json"; - - zig_allocator error_allocator = { - .ctx = nullptr, - .alloc = &Alloc, - .free = &Free, - }; - zig_slice error = {.ptr = nullptr, .len = 0}; - if (!zml::tools::DumpXSpaceFileToTraceJsonFile(input_path, output_path, - &error_allocator, &error)) { - std::cerr << std::string_view(static_cast(error.ptr), error.len) - << "\n"; - if (error.ptr != nullptr) { - error_allocator.deallocate(static_cast(error.ptr), error.len); - } - return 1; - } - - std::cout << output_path << "\n"; - return 0; -} diff --git a/tools/xspace_to_perfetto.h b/tools/xspace_to_perfetto.h deleted file mode 100644 index 3546c8af..00000000 --- a/tools/xspace_to_perfetto.h +++ /dev/null @@ -1,23 +0,0 @@ -#pragma once - -#include - -#include "ffi/zig_allocator.h" -#include "ffi/zig_slice.h" - -#ifdef __cplusplus -extern "C" { -#endif - -zig_slice zml_xspace_to_perfetto_dump(zig_allocator* error_allocator, - zig_slice xspace_proto, - zig_slice output_path); -zig_slice zml_xspace_normalize_and_dump(zig_allocator* error_allocator, - zig_slice xspace_proto, - uint64_t start_time_ns, - uint64_t stop_time_ns, - zig_slice output_path); - -#ifdef __cplusplus -} -#endif diff --git a/tools/xspace_to_perfetto/BUILD.bazel b/tools/xspace_to_perfetto/BUILD.bazel new file mode 100644 index 00000000..63658c51 --- /dev/null +++ b/tools/xspace_to_perfetto/BUILD.bazel @@ -0,0 +1,32 @@ +load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_c_proto_library") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") + +upb_c_proto_library( + name = "xplane_upb", + deps = ["@tsl//tsl/profiler/protobuf:xplane_proto"], + tags = ["manual"], +) + +zig_library( + name = "xspace_to_perfetto_lib", + tags = ["manual"], + import_name = "tools/xspace_to_perfetto", + main = "xspace_to_perfetto.zig", + visibility = ["//visibility:public"], + zigopts = ["-lc"], + deps = [ + ":xplane_upb", + "//upb", + ], +) + +zig_binary( + name = "xspace_to_perfetto", + main = "cli.zig", + visibility = ["//visibility:public"], + zigopts = ["-lc"], + deps = [ + ":xspace_to_perfetto_lib", + "//stdx", + ], +) diff --git a/tools/xspace_to_perfetto/cli.zig b/tools/xspace_to_perfetto/cli.zig new file mode 100644 index 00000000..f338855a --- /dev/null +++ b/tools/xspace_to_perfetto/cli.zig @@ -0,0 +1,34 @@ +const std = @import("std"); + +const stdx = @import("stdx"); +const xspace_to_perfetto = @import("tools/xspace_to_perfetto"); + +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + const io = threaded.io(); + + var upb_arena = std.heap.ArenaAllocator.init(allocator); + defer upb_arena.deinit(); + + const args = stdx.flags.parseProcessArgs(init.minimal, xspace_to_perfetto.Args); + const input_path = args.positional.input; + const output_path = args.positional.output orelse try std.fmt.allocPrint(allocator, "{s}.trace.json", .{input_path}); + defer if (args.positional.output == null) allocator.free(output_path); + + const stat = try std.Io.Dir.statFile(.cwd(), io, input_path, .{}); + + var file = try std.Io.Dir.openFile(.cwd(), io, input_path, .{ .mode = .read_only }); + defer file.close(io); + + const xspace = try allocator.alloc(u8, stat.size); + defer allocator.free(xspace); + _ = try file.readPositionalAll(io, xspace, 0); + + try xspace_to_perfetto.dumpXSpaceProtoToTraceJsonFile(allocator, io, xspace, output_path); + + var stdout = std.Io.File.stdout().writer(io, &.{}); + try stdout.interface.print("{s}\n", .{output_path}); +} diff --git a/tools/xspace_to_perfetto/xspace_to_perfetto.zig b/tools/xspace_to_perfetto/xspace_to_perfetto.zig new file mode 100644 index 00000000..46aec099 --- /dev/null +++ b/tools/xspace_to_perfetto/xspace_to_perfetto.zig @@ -0,0 +1,712 @@ +const std = @import("std"); + +const c = @import("c"); +const upb = @import("upb"); + +pub const Args = struct { + positional: struct { + input: []const u8, + output: ?[]const u8 = null, + }, + + pub const help = + \\Usage: xspace_to_perfetto [output_trace.json] + \\ + \\Converts an XSpace protobuf profile into trace-viewer JSON. + \\ + ; +}; + +// XPlane schema source: +// ../../xla/third_party/tsl/tsl/profiler/protobuf/xplane.proto +const kHostThreadsPlaneName = "/host:CPU"; +const kGpuPlanePrefix = "/device:GPU:"; +const kTpuPlanePrefix = "/device:TPU:"; +const kCustomPlanePrefix = "/device:CUSTOM:"; +const kXlaAsyncOpLineName = "Async XLA Ops"; +const kOpaqueBytesText = ""; + +const kPicosecondsPerNanosecond: u64 = 1_000; +const kPicosecondsPerMicrosecond: f64 = 1_000_000.0; +const kOutputBufferSizeBytes = 1024 * 1024; +const kHexDigits = "0123456789abcdef"; + +const kFirstDeviceId: u32 = 1; +const kLastDeviceId: u32 = 500; +const kFirstCustomPlaneDeviceId: u32 = kLastDeviceId + 1; +const kMaxCustomPlaneDevicesPerHost: u32 = 200; +const kLastCustomPlaneDeviceId: u32 = kFirstCustomPlaneDeviceId + kMaxCustomPlaneDevicesPerHost - 1; +const kHostThreadsDeviceId: u32 = kLastCustomPlaneDeviceId + 1; + +const StringValue = union(enum) { + text: []const u8, + int64: i64, + uint64: u64, + double: f64, + opaque_bytes, +}; + +const Arg = struct { + name: []const u8, + value: StringValue, +}; + +const StatBehavior = enum { + emit, + step_name, + skip, +}; + +const JsonArrayWriter = struct { + needs_comma: bool = false, + + fn nextItem(self: *JsonArrayWriter, writer: *std.Io.Writer) !void { + if (self.needs_comma) { + try writer.writeByte(','); + } + self.needs_comma = true; + } +}; + +const EventScratch = struct { + args: std.ArrayListUnmanaged(Arg) = .empty, + + fn reset(self: *EventScratch) void { + self.args.clearRetainingCapacity(); + } + + fn deinit(self: *EventScratch, allocator: std.mem.Allocator) void { + self.args.deinit(allocator); + } +}; + +const PlaneContext = struct { + plane: *const c.tensorflow_profiler_XPlane, + device_id: u32, + is_host: bool, + event_metadata_by_id: std.AutoHashMapUnmanaged(i64, *const c.tensorflow_profiler_XEventMetadata) = .{}, + stat_metadata_by_id: std.AutoHashMapUnmanaged(i64, *const c.tensorflow_profiler_XStatMetadata) = .{}, + event_is_internal_by_id: std.AutoHashMapUnmanaged(i64, bool) = .{}, + stat_behavior_by_id: std.AutoHashMapUnmanaged(i64, StatBehavior) = .{}, + + // Metadata caching mirrors the role of XPlaneVisitor lookups in: + // ../../xla/xla/tsl/profiler/utils/xplane_visitor.h + // ../../xla/xla/tsl/profiler/utils/xplane_visitor.cc + fn init(plane: *const c.tensorflow_profiler_XPlane, device_id: u32, is_host: bool) PlaneContext { + return .{ + .plane = plane, + .device_id = device_id, + .is_host = is_host, + }; + } + + fn deinit(self: *PlaneContext, allocator: std.mem.Allocator) void { + self.event_metadata_by_id.deinit(allocator); + self.stat_metadata_by_id.deinit(allocator); + self.event_is_internal_by_id.deinit(allocator); + self.stat_behavior_by_id.deinit(allocator); + } + + fn getEventMetadata( + self: *PlaneContext, + allocator: std.mem.Allocator, + metadata_id: i64, + ) !?*const c.tensorflow_profiler_XEventMetadata { + if (self.event_metadata_by_id.get(metadata_id)) |metadata| { + return metadata; + } + + var metadata: ?*c.tensorflow_profiler_XEventMetadata = null; + if (!c.tensorflow_profiler_XPlane_event_metadata_get(self.plane, metadata_id, &metadata)) { + return null; + } + const resolved = metadata orelse return null; + const result: *const c.tensorflow_profiler_XEventMetadata = resolved; + try self.event_metadata_by_id.put(allocator, metadata_id, result); + return result; + } + + fn getStatMetadata( + self: *PlaneContext, + allocator: std.mem.Allocator, + metadata_id: i64, + ) !?*const c.tensorflow_profiler_XStatMetadata { + if (self.stat_metadata_by_id.get(metadata_id)) |metadata| { + return metadata; + } + + var metadata: ?*c.tensorflow_profiler_XStatMetadata = null; + if (!c.tensorflow_profiler_XPlane_stat_metadata_get(self.plane, metadata_id, &metadata)) { + return null; + } + const resolved = metadata orelse return null; + const result: *const c.tensorflow_profiler_XStatMetadata = resolved; + try self.stat_metadata_by_id.put(allocator, metadata_id, result); + return result; + } + + fn isInternalEvent( + self: *PlaneContext, + allocator: std.mem.Allocator, + metadata_id: i64, + ) !bool { + if (self.event_is_internal_by_id.get(metadata_id)) |is_internal| { + return is_internal; + } + + const is_internal = if (try self.getEventMetadata(allocator, metadata_id)) |metadata| + isInternalEventName(stringView(c.tensorflow_profiler_XEventMetadata_name(metadata))) + else + false; + try self.event_is_internal_by_id.put(allocator, metadata_id, is_internal); + return is_internal; + } + + fn statBehavior( + self: *PlaneContext, + allocator: std.mem.Allocator, + metadata_id: i64, + ) !StatBehavior { + if (self.stat_behavior_by_id.get(metadata_id)) |behavior| { + return behavior; + } + + const behavior = if (try self.getStatMetadata(allocator, metadata_id)) |metadata| + classifyStatName(stringView(c.tensorflow_profiler_XStatMetadata_name(metadata))) + else + .emit; + try self.stat_behavior_by_id.put(allocator, metadata_id, behavior); + return behavior; + } +}; + +pub fn dumpXSpaceProtoToTraceJsonFile( + allocator: std.mem.Allocator, + io: std.Io, + xspace_proto: []const u8, + output_path: []const u8, +) !void { + var upb_backing_arena = std.heap.ArenaAllocator.init(allocator); + defer upb_backing_arena.deinit(); + + var upb_allocator: upb.Allocator = .init(upb_backing_arena.allocator()); + const upb_arena = c.upb_Arena_Init(null, 0, upb_allocator.inner()); + defer c.upb_Arena_Free(upb_arena); + + const xspace = c.tensorflow_profiler_XSpace_parse(xspace_proto.ptr, xspace_proto.len, upb_arena) orelse + return error.MalformedXSpace; + try writeTraceJsonFile(allocator, io, xspace, output_path); +} + +fn writeTraceJsonFile( + allocator: std.mem.Allocator, + io: std.Io, + xspace: *const c.tensorflow_profiler_XSpace, + output_path: []const u8, +) !void { + var output_file = try std.Io.Dir.createFile(.cwd(), io, output_path, .{}); + defer output_file.close(io); + + const output_buffer = try allocator.alloc(u8, kOutputBufferSizeBytes); + defer allocator.free(output_buffer); + + var file_writer = output_file.writer(io, output_buffer); + defer file_writer.interface.flush() catch {}; + + try streamXSpaceToTraceJson(allocator, xspace, &file_writer.interface); + try file_writer.interface.flush(); +} + +fn streamXSpaceToTraceJson( + allocator: std.mem.Allocator, + xspace: *const c.tensorflow_profiler_XSpace, + writer: *std.Io.Writer, +) !void { + // Keep the same trace JSON envelope as the streaming C++ implementation in: + // ../../xla/xla/tsl/profiler/convert/trace_events_to_json.cc + var planes = try collectSelectedPlanes(allocator, xspace); + defer { + for (planes.items) |*plane| { + plane.deinit(allocator); + } + planes.deinit(allocator); + } + + try writer.writeAll("{\"displayTimeUnit\":\"ns\",\"metadata\":{\"highres-ticks\":true},\"traceEvents\":["); + var array_writer: JsonArrayWriter = .{}; + + for (planes.items) |*plane| { + try streamPlaneMetadata(writer, &array_writer, plane); + } + for (planes.items) |*plane| { + try streamPlaneEvents(allocator, writer, &array_writer, plane); + } + + try writer.writeAll("]}"); +} + +fn collectSelectedPlanes( + allocator: std.mem.Allocator, + xspace: *const c.tensorflow_profiler_XSpace, +) !std.ArrayListUnmanaged(PlaneContext) { + // Plane selection intentionally follows: + // ../../xla/xla/tsl/profiler/convert/xplane_to_trace_events.cc + var result: std.ArrayListUnmanaged(PlaneContext) = .empty; + errdefer { + for (result.items) |*plane| { + plane.deinit(allocator); + } + result.deinit(allocator); + } + + var plane_count: usize = 0; + const planes = c.tensorflow_profiler_XSpace_planes(xspace, &plane_count) orelse return result; + const plane_slice = planes[0..plane_count]; + + var host_plane: ?*const c.tensorflow_profiler_XPlane = null; + for (plane_slice) |plane_c| { + const plane: *const c.tensorflow_profiler_XPlane = plane_c orelse continue; + if (std.mem.eql(u8, stringView(c.tensorflow_profiler_XPlane_name(plane)), kHostThreadsPlaneName)) { + host_plane = plane; + break; + } + } + + if (host_plane) |plane| { + try result.append(allocator, PlaneContext.init(plane, kHostThreadsDeviceId, true)); + } + + if (try appendPlanesWithPrefix(allocator, &result, plane_slice, kGpuPlanePrefix)) { + return result; + } + if (try appendPlanesWithPrefix(allocator, &result, plane_slice, kTpuPlanePrefix)) { + return result; + } + _ = try appendPlanesWithPrefix(allocator, &result, plane_slice, kCustomPlanePrefix); + return result; +} + +fn appendPlanesWithPrefix( + allocator: std.mem.Allocator, + result: *std.ArrayListUnmanaged(PlaneContext), + planes: []const [*c]const c.tensorflow_profiler_XPlane, + prefix: []const u8, +) !bool { + var found = false; + for (planes) |plane_c| { + const plane: *const c.tensorflow_profiler_XPlane = plane_c orelse continue; + const name = stringView(c.tensorflow_profiler_XPlane_name(plane)); + if (!std.mem.startsWith(u8, name, prefix)) continue; + found = true; + const device_ordinal = wrapI64ToU32(c.tensorflow_profiler_XPlane_id(plane)); + try result.append(allocator, PlaneContext.init(plane, kFirstDeviceId + device_ordinal, false)); + } + return found; +} + +fn streamPlaneMetadata( + writer: *std.Io.Writer, + array_writer: *JsonArrayWriter, + plane: *const PlaneContext, +) !void { + // Metadata event shape matches the resource/process metadata emitted in: + // ../../xla/xla/tsl/profiler/convert/xplane_to_trace_events.cc + const plane_name = stringView(c.tensorflow_profiler_XPlane_name(plane.plane)); + if (plane_name.len != 0) { + try array_writer.nextItem(writer); + try writer.writeAll("{\"ph\":\"M\",\"pid\":"); + try writer.print("{d}", .{plane.device_id}); + try writer.writeAll(",\"name\":\"process_name\",\"args\":{\"name\":"); + try writeJsonString(writer, plane_name); + try writer.writeAll("}}"); + } + + try array_writer.nextItem(writer); + try writer.writeAll("{\"ph\":\"M\",\"pid\":"); + try writer.print("{d}", .{plane.device_id}); + try writer.writeAll(",\"name\":\"process_sort_index\",\"args\":{\"sort_index\":"); + try writer.print("{d}", .{plane.device_id}); + try writer.writeAll("}}"); + + var ordinal: u32 = 0; + var line_count: usize = 0; + const lines = c.tensorflow_profiler_XPlane_lines(plane.plane, &line_count) orelse return; + for (lines[0..line_count]) |line_c| { + const line: *const c.tensorflow_profiler_XLine = line_c orelse continue; + const resource_id = try lineResourceId(line); + const resource_name = lineDisplayName(line); + const sort_index = if (plane.is_host) blk: { + ordinal += 1; + break :blk ordinal; + } else resource_id; + try streamResourceMetadata(writer, array_writer, plane.device_id, resource_id, resource_name, sort_index); + } +} + +fn streamResourceMetadata( + writer: *std.Io.Writer, + array_writer: *JsonArrayWriter, + device_id: u32, + resource_id: u32, + resource_name: []const u8, + sort_index: u32, +) !void { + if (resource_name.len != 0) { + try array_writer.nextItem(writer); + try writer.writeAll("{\"ph\":\"M\",\"pid\":"); + try writer.print("{d}", .{device_id}); + try writer.writeAll(",\"tid\":"); + try writer.print("{d}", .{resource_id}); + try writer.writeAll(",\"name\":\"thread_name\",\"args\":{\"name\":"); + try writeJsonString(writer, resource_name); + try writer.writeAll("}}"); + } + + try array_writer.nextItem(writer); + try writer.writeAll("{\"ph\":\"M\",\"pid\":"); + try writer.print("{d}", .{device_id}); + try writer.writeAll(",\"tid\":"); + try writer.print("{d}", .{resource_id}); + try writer.writeAll(",\"name\":\"thread_sort_index\",\"args\":{\"sort_index\":"); + try writer.print("{d}", .{sort_index}); + try writer.writeAll("}}"); +} + +fn streamPlaneEvents( + allocator: std.mem.Allocator, + writer: *std.Io.Writer, + array_writer: *JsonArrayWriter, + plane: *PlaneContext, +) !void { + var scratch: EventScratch = .{}; + defer scratch.deinit(allocator); + + var line_count: usize = 0; + const lines = c.tensorflow_profiler_XPlane_lines(plane.plane, &line_count) orelse return; + for (lines[0..line_count]) |line_c| { + const line: *const c.tensorflow_profiler_XLine = line_c orelse continue; + if (std.mem.eql(u8, lineDisplayName(line), kXlaAsyncOpLineName)) continue; + + const resource_id = try lineResourceId(line); + var event_count: usize = 0; + const events = c.tensorflow_profiler_XLine_events(line, &event_count) orelse continue; + for (events[0..event_count]) |event_c| { + const event: *const c.tensorflow_profiler_XEvent = event_c orelse continue; + try streamEvent(allocator, writer, array_writer, plane, line, event, resource_id, &scratch); + } + } +} + +fn streamEvent( + allocator: std.mem.Allocator, + writer: *std.Io.Writer, + array_writer: *JsonArrayWriter, + plane: *PlaneContext, + line: *const c.tensorflow_profiler_XLine, + event: *const c.tensorflow_profiler_XEvent, + resource_id: u32, + scratch: *EventScratch, +) !void { + const metadata_id = c.tensorflow_profiler_XEvent_metadata_id(event); + if (try plane.isInternalEvent(allocator, metadata_id)) return; + + // Display name, long_name, stat merge order, and step_name override mirror: + // ../../xla/xla/tsl/profiler/convert/xplane_to_trace_events.cc + const metadata = try plane.getEventMetadata(allocator, metadata_id); + const metadata_name = if (metadata) |value| stringView(c.tensorflow_profiler_XEventMetadata_name(value)) else ""; + const metadata_display_name = if (metadata) |value| stringView(c.tensorflow_profiler_XEventMetadata_display_name(value)) else ""; + + var event_name: StringValue = .{ + .text = if (metadata_display_name.len != 0) metadata_display_name else metadata_name, + }; + + scratch.reset(); + const args = &scratch.args; + + if (metadata_display_name.len != 0) { + try putArg(args, allocator, "long_name", .{ .text = metadata_name }); + } + + if (metadata) |value| { + try appendStats(allocator, plane, args, &event_name, metadataStats(value)); + } + try appendStats(allocator, plane, args, &event_name, eventStats(event)); + + std.mem.sort(Arg, args.items, {}, argLessThan); + + try array_writer.nextItem(writer); + try writer.writeAll("{\"ph\":\"X\",\"pid\":"); + try writer.print("{d}", .{plane.device_id}); + try writer.writeAll(",\"tid\":"); + try writer.print("{d}", .{resource_id}); + try writer.writeAll(",\"ts\":"); + var number_buf: [64]u8 = undefined; + try writer.writeAll(formatPicosToMicros(&number_buf, eventTimestampPs(line, event))); + try writer.writeAll(",\"dur\":"); + try writer.writeAll(formatPicosToMicros(&number_buf, eventDurationPs(event))); + try writer.writeAll(",\"name\":"); + try writeJsonStringValue(writer, event_name); + + if (args.items.len != 0) { + try writer.writeAll(",\"args\":{"); + for (args.items, 0..) |arg, index| { + if (index != 0) try writer.writeByte(','); + try writeJsonString(writer, arg.name); + try writer.writeByte(':'); + try writeJsonStringValue(writer, arg.value); + } + try writer.writeByte('}'); + } + + try writer.writeByte('}'); +} + +fn appendStats( + allocator: std.mem.Allocator, + plane: *PlaneContext, + args: *std.ArrayListUnmanaged(Arg), + event_name: *StringValue, + stats: []const [*c]const c.tensorflow_profiler_XStat, +) !void { + // Stat filtering and name lookup correspond to: + // ../../xla/xla/tsl/profiler/utils/xplane_schema.cc + for (stats) |stat_c| { + const stat: *const c.tensorflow_profiler_XStat = stat_c orelse continue; + const value_case = c.tensorflow_profiler_XStat_value_case(stat); + if (value_case == c.tensorflow_profiler_XStat_value_NOT_SET) continue; + + const metadata_id = c.tensorflow_profiler_XStat_metadata_id(stat); + const behavior = try plane.statBehavior(allocator, metadata_id); + if (behavior == .skip) continue; + + const stat_value = try stringValueFromStat(allocator, plane, stat); + if (behavior == .step_name) { + event_name.* = stat_value; + } + + const stat_name = if (try plane.getStatMetadata(allocator, metadata_id)) |metadata| + stringView(c.tensorflow_profiler_XStatMetadata_name(metadata)) + else + ""; + try putArg(args, allocator, stat_name, stat_value); + } +} + +fn putArg( + args: *std.ArrayListUnmanaged(Arg), + allocator: std.mem.Allocator, + name: []const u8, + value: StringValue, +) !void { + for (args.items) |*arg| { + if (std.mem.eql(u8, arg.name, name)) { + arg.value = value; + return; + } + } + try args.append(allocator, .{ .name = name, .value = value }); +} + +fn metadataStats(metadata: *const c.tensorflow_profiler_XEventMetadata) []const [*c]const c.tensorflow_profiler_XStat { + var stat_count: usize = 0; + const stats = c.tensorflow_profiler_XEventMetadata_stats(metadata, &stat_count) orelse return &.{}; + return stats[0..stat_count]; +} + +fn eventStats(event: *const c.tensorflow_profiler_XEvent) []const [*c]const c.tensorflow_profiler_XStat { + var stat_count: usize = 0; + const stats = c.tensorflow_profiler_XEvent_stats(event, &stat_count) orelse return &.{}; + return stats[0..stat_count]; +} + +fn stringValueFromStat( + allocator: std.mem.Allocator, + plane: *PlaneContext, + stat: *const c.tensorflow_profiler_XStat, +) !StringValue { + // Stringification intentionally matches XStatVisitor::ToString(): + // ../../xla/xla/tsl/profiler/utils/xplane_visitor.cc + return switch (c.tensorflow_profiler_XStat_value_case(stat)) { + c.tensorflow_profiler_XStat_value_int64_value => .{ .int64 = c.tensorflow_profiler_XStat_int64_value(stat) }, + c.tensorflow_profiler_XStat_value_uint64_value => .{ .uint64 = c.tensorflow_profiler_XStat_uint64_value(stat) }, + c.tensorflow_profiler_XStat_value_double_value => .{ .double = c.tensorflow_profiler_XStat_double_value(stat) }, + c.tensorflow_profiler_XStat_value_str_value => .{ .text = stringView(c.tensorflow_profiler_XStat_str_value(stat)) }, + c.tensorflow_profiler_XStat_value_bytes_value => .opaque_bytes, + c.tensorflow_profiler_XStat_value_ref_value => blk: { + const ref_metadata_id = std.math.cast(i64, c.tensorflow_profiler_XStat_ref_value(stat)) orelse return error.InvalidStatMetadataId; + const text = if (try plane.getStatMetadata(allocator, ref_metadata_id)) |metadata| + stringView(c.tensorflow_profiler_XStatMetadata_name(metadata)) + else + ""; + break :blk .{ .text = text }; + }, + else => unreachable, + }; +} + +fn writeJsonStringValue(writer: *std.Io.Writer, value: StringValue) !void { + switch (value) { + .text => |text| try writeJsonString(writer, text), + .opaque_bytes => try writeJsonString(writer, kOpaqueBytesText), + .int64 => |number| { + var buffer: [64]u8 = undefined; + const text = try std.fmt.bufPrint(&buffer, "{d}", .{number}); + try writeJsonString(writer, text); + }, + .uint64 => |number| { + var buffer: [64]u8 = undefined; + const text = try std.fmt.bufPrint(&buffer, "{d}", .{number}); + try writeJsonString(writer, text); + }, + .double => |number| { + var buffer: [64]u8 = undefined; + try writeJsonString(writer, formatDouble(&buffer, "%g", number)); + }, + } +} + +fn writeJsonString(writer: *std.Io.Writer, text: []const u8) !void { + try writer.writeByte('"'); + + var chunk_start: usize = 0; + for (text, 0..) |byte, index| { + const escape = switch (byte) { + '"' => "\\\"", + '\\' => "\\\\", + '\n' => "\\n", + '\r' => "\\r", + '\t' => "\\t", + 0x08 => "\\b", + 0x0c => "\\f", + else => null, + }; + if (escape) |escaped| { + if (chunk_start < index) { + try writer.writeAll(text[chunk_start..index]); + } + try writer.writeAll(escaped); + chunk_start = index + 1; + continue; + } + if (byte < 0x20) { + if (chunk_start < index) { + try writer.writeAll(text[chunk_start..index]); + } + try writeAsciiControlEscape(writer, byte); + chunk_start = index + 1; + } + } + + if (chunk_start < text.len) { + try writer.writeAll(text[chunk_start..]); + } + + try writer.writeByte('"'); +} + +fn lineResourceId(line: *const c.tensorflow_profiler_XLine) !u32 { + const display_id = c.tensorflow_profiler_XLine_display_id(line); + const effective_id = if (display_id != 0) display_id else c.tensorflow_profiler_XLine_id(line); + return wrapI64ToU32(effective_id); +} + +fn lineDisplayName(line: *const c.tensorflow_profiler_XLine) []const u8 { + const display_name = stringView(c.tensorflow_profiler_XLine_display_name(line)); + if (display_name.len != 0) return display_name; + return stringView(c.tensorflow_profiler_XLine_name(line)); +} + +fn eventTimestampPs(line: *const c.tensorflow_profiler_XLine, event: *const c.tensorflow_profiler_XEvent) u64 { + const line_timestamp_ns = reinterpretSignedAsUnsigned64(c.tensorflow_profiler_XLine_timestamp_ns(line)); + const event_offset_ps = reinterpretSignedAsUnsigned64(c.tensorflow_profiler_XEvent_offset_ps(event)); + return line_timestamp_ns *% kPicosecondsPerNanosecond +% event_offset_ps; +} + +fn eventDurationPs(event: *const c.tensorflow_profiler_XEvent) u64 { + const duration_ps = reinterpretSignedAsUnsigned64(c.tensorflow_profiler_XEvent_duration_ps(event)); + return @max(duration_ps, 1); +} + +fn formatPicosToMicros(buffer: *[64]u8, picoseconds: u64) []const u8 { + // The conversion and "%.17g" formatting are kept identical to: + // ../../xla/xla/tsl/profiler/utils/math_utils.h + // ../../xla/xla/tsl/profiler/utils/format_utils.h + const microseconds = @as(f64, @floatFromInt(picoseconds)) / kPicosecondsPerMicrosecond; + return formatDouble(buffer, "%.17g", microseconds); +} + +fn formatDouble(buffer: *[64]u8, comptime fmt: [*:0]const u8, value: f64) []const u8 { + const written = c.snprintf(buffer.ptr, buffer.len, fmt, value); + if (written <= 0 or written >= buffer.len) unreachable; + return buffer[0..@intCast(written)]; +} + +fn stringView(value: c.upb_StringView) []const u8 { + const text = upb.slice(value); + return text orelse ""; +} + +fn reinterpretSignedAsUnsigned64(value: i64) u64 { + const same_bits: u64 = @bitCast(value); + return same_bits; +} + +fn wrapI64ToU32(value: i64) u32 { + const unsigned_value = reinterpretSignedAsUnsigned64(value); + return @truncate(unsigned_value); +} + +fn writeAsciiControlEscape(writer: *std.Io.Writer, byte: u8) !void { + const high_nibble = @as(usize, byte / 16); + const low_nibble = @as(usize, byte % 16); + + try writer.writeAll("\\u00"); + try writer.writeByte(kHexDigits[high_nibble]); + try writer.writeByte(kHexDigits[low_nibble]); +} + +fn argLessThan(_: void, lhs: Arg, rhs: Arg) bool { + return std.mem.order(u8, lhs.name, rhs.name) == .lt; +} + +fn isInternalEventName(name: []const u8) bool { + // Internal host-event filtering is derived from: + // ../../xla/xla/tsl/profiler/utils/xplane_schema.cc + inline for ([_][]const u8{ + "MemoryAllocation", + "MemoryDeallocation", + "PrefetchProduce", + "PrefetchConsume", + "ParallelInterleaveProduce", + "ParallelInterleaveConsume", + "ParallelInterleaveInitializeInput", + "ParallelMapProduce", + "ParallelMapConsume", + "MapAndBatchProduce", + "MapAndBatchConsume", + "ParseExampleProduce", + "ParseExampleConsume", + }) |candidate| { + if (std.mem.eql(u8, name, candidate)) return true; + } + return false; +} + +fn classifyStatName(name: []const u8) StatBehavior { + // Internal stat filtering and step_name handling are derived from: + // ../../xla/xla/tsl/profiler/utils/xplane_schema.cc + if (std.mem.eql(u8, name, "step_name")) return .step_name; + inline for ([_][]const u8{ + "_pt", + "_ct", + "_p", + "_c", + "_r", + "flops", + "program_id", + "symbol_id", + }) |candidate| { + if (std.mem.eql(u8, name, candidate)) return .skip; + } + return .emit; +} diff --git a/tools/xspace_to_perfetto_lib.cc b/tools/xspace_to_perfetto_lib.cc deleted file mode 100644 index 934c18ab..00000000 --- a/tools/xspace_to_perfetto_lib.cc +++ /dev/null @@ -1,202 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include "tools/xspace_to_perfetto.h" -#include "xla/tsl/profiler/convert/post_process_single_host_xplane.h" -#include "xla/tsl/profiler/convert/trace_events_to_json.h" -#include "xla/tsl/profiler/convert/xplane_to_trace_events.h" -#include "xla/tsl/profiler/utils/xplane_utils.h" -#include "tsl/profiler/protobuf/xplane.pb.h" - -namespace { - -zig_slice EmptySlice() { return {.ptr = nullptr, .len = 0}; } - -std::string_view ToStringView(zig_slice slice) { - return std::string_view(static_cast(slice.ptr), slice.len); -} - -bool SetError(zig_allocator* error_allocator, - zig_slice* error, - std::string_view message) { - char* buffer = error_allocator->allocate(message.size()); - if (buffer == nullptr) { - *error = EmptySlice(); - return false; - } - - std::memcpy(buffer, message.data(), message.size()); - *error = {.ptr = buffer, .len = message.size()}; - return true; -} - -bool SetErrorWithPath(zig_allocator* error_allocator, - zig_slice* error, - std::string_view prefix, - std::string_view path) { - const size_t len = prefix.size() + path.size(); - char* buffer = error_allocator->allocate(len); - if (buffer == nullptr) { - *error = EmptySlice(); - return false; - } - - std::memcpy(buffer, prefix.data(), prefix.size()); - std::memcpy(buffer + prefix.size(), path.data(), path.size()); - *error = {.ptr = buffer, .len = len}; - return true; -} - -void PostProcessXSpace(tensorflow::profiler::XSpace* xspace, - uint64_t start_time_ns, - uint64_t stop_time_ns) { - tsl::profiler::SetXSpacePidIfNotSet(*xspace, getpid()); - tsl::profiler::PostProcessSingleHostXSpace( - xspace, start_time_ns, stop_time_ns); -} - -bool SerializeXSpace(const tensorflow::profiler::XSpace& xspace, - std::string* serialized_xspace, - zig_allocator* error_allocator, - zig_slice* error) { - if (!xspace.SerializeToString(serialized_xspace)) { - SetError(error_allocator, error, - "Failed to serialize normalized XSpace protobuf"); - return false; - } - return true; -} - -bool ParseXSpace(std::string_view xspace_proto, - tensorflow::profiler::XSpace* xspace, - zig_allocator* error_allocator, - zig_slice* error) { - if (xspace_proto.size() > - static_cast(std::numeric_limits::max())) { - SetError(error_allocator, error, "XSpace protobuf is too large to parse"); - return false; - } - if (!xspace->ParseFromArray(xspace_proto.data(), - static_cast(xspace_proto.size()))) { - SetError(error_allocator, error, "Failed to parse XSpace protobuf"); - return false; - } - return true; -} - -bool WriteFile(std::string_view output_path, - std::string_view contents, - zig_allocator* error_allocator, - zig_slice* error) { - std::ofstream output(std::string(output_path), std::ios::binary); - if (!output.is_open()) { - SetErrorWithPath(error_allocator, error, "Failed to open output file: ", - output_path); - return false; - } - - output.write(contents.data(), static_cast(contents.size())); - if (!output) { - SetErrorWithPath(error_allocator, error, "Failed to write output file: ", - output_path); - return false; - } - return true; -} - -} // namespace - -namespace zml::tools { - -bool ConvertXSpaceProtoToTraceJson(std::string_view xspace_proto, - std::string* json, - zig_allocator* error_allocator, - zig_slice* error) { - tensorflow::profiler::XSpace xspace; - if (!ParseXSpace(xspace_proto, &xspace, error_allocator, error)) { - return false; - } - - const tsl::profiler::TraceContainer container = - tsl::profiler::ConvertXSpaceToTraceContainer(xspace); - *json = tsl::profiler::TraceContainerToJson(container); - return true; -} - -bool DumpXSpaceProtoToTraceJsonFile(std::string_view xspace_proto, - std::string_view output_path, - zig_allocator* error_allocator, - zig_slice* error) { - std::string json; - if (!ConvertXSpaceProtoToTraceJson(xspace_proto, &json, error_allocator, - error)) { - return false; - } - return WriteFile(output_path, json, error_allocator, error); -} - -bool DumpXSpaceFileToTraceJsonFile(std::string_view input_path, - std::string_view output_path, - zig_allocator* error_allocator, - zig_slice* error) { - std::ifstream input(std::string(input_path), std::ios::binary); - if (!input.is_open()) { - SetErrorWithPath(error_allocator, error, "Failed to open input file: ", - input_path); - return false; - } - - tensorflow::profiler::XSpace xspace; - if (!xspace.ParseFromIstream(&input)) { - SetErrorWithPath(error_allocator, error, - "Failed to parse XSpace protobuf from: ", input_path); - return false; - } - - const tsl::profiler::TraceContainer container = - tsl::profiler::ConvertXSpaceToTraceContainer(xspace); - const std::string json = tsl::profiler::TraceContainerToJson(container); - return WriteFile(output_path, json, error_allocator, error); -} - -} // namespace zml::tools - -extern "C" zig_slice zml_xspace_to_perfetto_dump( - zig_allocator* error_allocator, - zig_slice xspace_proto, - zig_slice output_path) { - zig_slice error = EmptySlice(); - if (zml::tools::DumpXSpaceProtoToTraceJsonFile( - ToStringView(xspace_proto), - ToStringView(output_path), - error_allocator, - &error)) { - return EmptySlice(); - } - return error; -} - -extern "C" zig_slice zml_xspace_normalize_and_dump( - zig_allocator* error_allocator, zig_slice xspace_proto, - uint64_t start_time_ns, uint64_t stop_time_ns, zig_slice output_path) { - zig_slice error = EmptySlice(); - tensorflow::profiler::XSpace xspace; - if (!ParseXSpace(ToStringView(xspace_proto), &xspace, error_allocator, - &error)) { - return error; - } - - PostProcessXSpace(&xspace, start_time_ns, stop_time_ns); - std::string normalized_xspace; - if (!SerializeXSpace(xspace, &normalized_xspace, error_allocator, &error) || - !WriteFile(ToStringView(output_path), normalized_xspace, error_allocator, - &error)) { - return error; - } - - return EmptySlice(); -} diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 175b91f3..d6677539 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -26,6 +26,41 @@ cc_library( hdrs = ["posix.h"], ) +cc_library( + name = "profiling_traceme_lib", + tags = ["manual"], + srcs = ["profiling/traceme_lib.cc"], + hdrs = ["profiling/traceme.h"], + defines = select({ + "//platforms:cuda.enabled": ["ZML_RUNTIME_CUDA"], + "//platforms:rocm.enabled": ["ZML_RUNTIME_ROCM"], + "//conditions:default": [], + }), + linkopts = select({ + "//platforms:rocm.enabled": ["-ldl"], + "//conditions:default": [], + }), + visibility = ["//visibility:public"], + deps = [ + "//ffi:cc", + "@tsl//tsl/profiler/lib:traceme", + "@tsl//tsl/profiler/protobuf:xplane_proto_cc", + "@xla//xla/tsl/profiler/backends/cpu:host_tracer_utils", + "@xla//xla/tsl/profiler/backends/cpu:traceme_recorder", + "@xla//xla/tsl/profiler/convert:post_process_single_host_xplane", + "@xla//xla/tsl/profiler/utils:time_utils", + "@xla//xla/tsl/profiler/utils:xplane_utils", + ] + select({ + "//platforms:cuda.enabled": [ + "//platforms/cuda:cuda_nvtx_headers", + ], + "//platforms:rocm.enabled": [ + "@rocprofiler-sdk-roctx//:headers", + ], + "//conditions:default": [], + }), +) + zig_library( name = "zml", tags = ["manual"], @@ -54,13 +89,13 @@ zig_library( "ops.zig", "pjrtx.zig", "platform.zig", - "profiler.zig", + "profiling/profiler.zig", + "profiling/tracer.zig", "safetensors.zig", "shape.zig", "slice.zig", "tensor.zig", "testing.zig", - "tracer.zig", "zml.zig", ], main = "zml.zig", @@ -85,11 +120,12 @@ zig_library( "//platforms", "//platforms/tpu:ragged_paged", "//stdx", - "//tools:xspace_to_perfetto_lib", + "//tools/xspace_to_perfetto:xspace_to_perfetto_lib", "//upb", "//zml/io", "//zml/tokenizer", "@rules_zig//zig/runfiles", + ":profiling_traceme_lib", ] + select({ "//platforms:cuda.enabled": [ "//platforms/cuda/flashattn", diff --git a/zml/exe.zig b/zml/exe.zig index 4423a973..8bf9708e 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -6,6 +6,7 @@ const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; const meta = @import("meta.zig"); const Platform = @import("platform.zig").Platform; +const tracer = @import("profiling/tracer.zig"); const Shape = @import("shape.zig").Shape; const Sharding = @import("Sharding.zig"); @@ -290,10 +291,22 @@ pub const Exe = struct { }; pub fn callOpts(self: *const Exe, io: std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { + var trace = tracer.scope("zml.exe.call", .{ + .wait = opts.wait, + .arg_count = arguments.expected_shapes.len, + .result_count = results_.expected_shapes.len, + }) catch unreachable; + defer trace.end(); return self.internalCall(io, arguments, results_, opts); } pub fn call(self: *const Exe, arguments: Arguments, results_: *Results) void { + var trace = tracer.scope("zml.exe.call", .{ + .wait = false, + .arg_count = arguments.expected_shapes.len, + .result_count = results_.expected_shapes.len, + }) catch unreachable; + defer trace.end(); return self.internalCall(null, arguments, results_, .{}); } }; diff --git a/zml/io.zig b/zml/io.zig index 739a30d9..d6c9e94d 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -12,6 +12,7 @@ const Memory = @import("platform.zig").Memory; const meta = @import("meta.zig"); const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; +const tracer = @import("profiling/tracer.zig"); const safetensors = @import("safetensors.zig"); const Shape = @import("shape.zig").Shape; const Sharding = @import("Sharding.zig"); @@ -1084,6 +1085,11 @@ pub fn load( store: *const TensorStore, opts: LoadOpts, ) !Bufferized(ModelType) { + var trace = try tracer.scope("zml.io.load", .{ + .tensor_count = meta.count(Tensor, model), + }); + defer trace.end(); + var bufferized = try mem.bufferize(allocator, ModelType, model); const pool_count = platform.devices.len; diff --git a/zml/module.zig b/zml/module.zig index 5b3782c0..fb828fbd 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -14,6 +14,7 @@ const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; +const tracer = @import("profiling/tracer.zig"); const Shape = @import("shape.zig").Shape; const Sharding = @import("Sharding.zig"); const Partitioning = Sharding.Partitioning; @@ -199,6 +200,12 @@ pub fn compile( var st_io: std.Io.Threaded = .init_single_threaded; defer st_io.deinit(); + var trace = try tracer.scope("zml.module.compile", .{ + .program_name = opts.program_name, + .arg_count = args.len, + }); + defer trace.end(); + var compilation_context: CompilationContext = .init(allocator, st_io.io(), platform, opts); defer compilation_context.deinit(); diff --git a/zml/platform.zig b/zml/platform.zig index ccbe0e41..c64af15b 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -9,7 +9,7 @@ const stdx = @import("stdx"); const attention = @import("attention.zig"); const Exe = @import("exe.zig").Exe; const pjrtx = @import("pjrtx.zig"); -const profiler_ = @import("profiler.zig"); +const profiler_ = @import("profiling/profiler.zig"); const Sharding = @import("Sharding.zig"); const zml = @import("zml.zig"); diff --git a/zml/profiler.zig b/zml/profiling/profiler.zig similarity index 69% rename from zml/profiler.zig rename to zml/profiling/profiler.zig index e5dbab26..3f3c8409 100644 --- a/zml/profiler.zig +++ b/zml/profiling/profiler.zig @@ -4,6 +4,7 @@ const log = std.log; const c = @import("c"); const pjrt = @import("pjrt"); const upb = @import("upb"); +const xspace_to_perfetto = @import("tools/xspace_to_perfetto"); const zffi = @import("ffi"); pub fn profiler(api: *const pjrt.Api, allocator: std.mem.Allocator, io: std.Io, options: ProfilerOptions) !Profiler { @@ -144,6 +145,9 @@ pub const Profiler = struct { inner: ?pjrt.Profiler, start_timestamp_ns: ?u64, stop_timestamp_ns: ?u64, + raise_error_on_start_failure: bool, + local_host_tracer_level: ?u32, + local_host_traceme_filter_mask: ?u64, session_dir: []const u8, profile: Profile, @@ -174,6 +178,9 @@ pub const Profiler = struct { .inner = inner, .start_timestamp_ns = null, .stop_timestamp_ns = null, + .raise_error_on_start_failure = options.raise_error_on_start_failure orelse false, + .local_host_tracer_level = options.host_tracer_level, + .local_host_traceme_filter_mask = if (options.trace_options) |trace_options| trace_options.host_traceme_filter_mask else null, .session_dir = session_dir, .profile = .{ .protobuf_path = protobuf_path, @@ -190,51 +197,91 @@ pub const Profiler = struct { } pub fn start(self: *Profiler) !void { - if (self.inner) |*inner| { - self.start_timestamp_ns = @intCast(std.Io.Timestamp.now(self.io, .real).nanoseconds); - try inner.start(); + self.start_timestamp_ns = @intCast(std.Io.Timestamp.now(self.io, .real).nanoseconds); + self.stop_timestamp_ns = null; + + if (shouldSkipPjrtProfiler()) { + log.warn("Detected SKIP_PJRT_PROFILER=true; disabling nested backend profiling and keeping local host tracing only.", .{}); + self.inner = null; + } else if (self.inner) |*inner| { + inner.start() catch |err| { + if (self.raise_error_on_start_failure) { + return err; + } + log.warn("Profiler backend failed to start; continuing with local host tracing only: {}", .{err}); + self.inner = null; + }; + } + if (self.local_host_tracer_level) |level| { + c.zml_traceme_session_start( + @intCast(level), + self.local_host_traceme_filter_mask orelse 0, + self.local_host_traceme_filter_mask != null, + ); } } pub fn stop(self: *Profiler) !?Profile { - var inner = self.inner orelse return null; - self.inner = null; - try inner.stop(); self.stop_timestamp_ns = @intCast(std.Io.Timestamp.now(self.io, .real).nanoseconds); - const protobuf = try inner.collectData(self.arena.allocator()); - var error_allocator = zffi.ZigAllocator.from(self.arena.allocator()); - const normalization_error = c.zml_xspace_normalize_and_dump( - &error_allocator, - zffi.ZigSlice.from(protobuf), - self.start_timestamp_ns.?, - self.stop_timestamp_ns.?, - zffi.ZigSlice.from(self.profile.protobuf_path), - ); + const protobuf = if (self.inner) |saved_inner| blk: { + var inner = saved_inner; + self.inner = null; + try inner.stop(); + break :blk try inner.collectData(self.arena.allocator()); + } else &.{}; + const merged_protobuf = c.zml_traceme_session_merge(zffi.ZigSlice.from(protobuf)); + defer if (merged_protobuf.len != 0) { + c.zml_traceme_str_free(merged_protobuf); + }; + const final_protobuf = if (merged_protobuf.len != 0) zffi.ZigSlice.to(u8, merged_protobuf) else protobuf; + if (final_protobuf.len == 0) { + return null; + } - if (normalization_error.len != 0) { - log.err("Failed to normalize profile protobuf: {s}", .{zffi.ZigSlice.to(u8, normalization_error)}); + const normalized_protobuf = c.zml_traceme_session_postprocess( + zffi.ZigSlice.from(final_protobuf), + self.start_timestamp_ns orelse self.stop_timestamp_ns.?, + self.stop_timestamp_ns.?, + ); + defer if (normalized_protobuf.len != 0) { + c.zml_traceme_str_free(normalized_protobuf); + }; + if (normalized_protobuf.len == 0) { + log.err("Failed to normalize profile protobuf", .{}); return error.ProfileTraceNormalizationFailed; } - const normalized_protobuf = try std.Io.Dir.cwd().readFileAlloc( + const final_normalized_protobuf = zffi.ZigSlice.to(u8, normalized_protobuf); + + try self.writeFile(self.profile.protobuf_path, final_normalized_protobuf); + + xspace_to_perfetto.dumpXSpaceProtoToTraceJsonFile( + self.allocator, self.io, - self.profile.protobuf_path, - self.arena.allocator(), - .unlimited, - ); - - const conversion_error = c.zml_xspace_to_perfetto_dump( - &error_allocator, - zffi.ZigSlice.from(normalized_protobuf), - zffi.ZigSlice.from(self.profile.perfetto_path), - ); - - if (conversion_error.len != 0) { - log.err("Failed to convert normalized profile protobuf to Perfetto trace: {s}", .{zffi.ZigSlice.to(u8, conversion_error)}); + final_normalized_protobuf, + self.profile.perfetto_path, + ) catch |err| { + log.err("Failed to convert profile protobuf to Perfetto trace: {}", .{err}); return error.ProfileTraceConversionFailed; - } + }; + log.info("Profile dumped: {s} and {s}", .{ + self.profile.protobuf_path, + self.profile.perfetto_path, + }); return self.profile; } + + fn writeFile(self: *const Profiler, path: []const u8, contents: []const u8) !void { + const file = try std.Io.Dir.createFile(.cwd(), self.io, path, .{}); + defer file.close(self.io); + + try file.writePositionalAll(self.io, contents, 0); + } }; + +fn shouldSkipPjrtProfiler() bool { + const value = std.c.getenv("SKIP_PJRT_PROFILER") orelse return false; + return std.mem.eql(u8, std.mem.span(value), "true"); +} diff --git a/zml/profiling/traceme.h b/zml/profiling/traceme.h new file mode 100644 index 00000000..a1df3d04 --- /dev/null +++ b/zml/profiling/traceme.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include + +#include "ffi/zig_slice.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct zml_traceme zml_traceme; + +zml_traceme *zml_traceme_start(zig_slice name); +void zml_traceme_stop(zml_traceme *traceme); +bool zml_traceme_enabled(void); +void zml_traceme_session_start(int level, uint64_t filter_mask, + bool enable_filter); +zig_slice zml_traceme_session_merge(zig_slice xspace); +zig_slice zml_traceme_session_postprocess(zig_slice xspace, + uint64_t start_time_ns, + uint64_t stop_time_ns); +void zml_traceme_str_free(zig_slice text); + +#ifdef __cplusplus +} +#endif diff --git a/zml/profiling/traceme_lib.cc b/zml/profiling/traceme_lib.cc new file mode 100644 index 00000000..ca37dee7 --- /dev/null +++ b/zml/profiling/traceme_lib.cc @@ -0,0 +1,252 @@ +#include "zml/profiling/traceme.h" + +#include +#include +#include + +#include "xla/tsl/profiler/convert/post_process_single_host_xplane.h" +#include "xla/tsl/profiler/backends/cpu/host_tracer_utils.h" +#include "xla/tsl/profiler/backends/cpu/traceme_recorder.h" +#include "xla/tsl/profiler/utils/time_utils.h" +#include "xla/tsl/profiler/utils/xplane_utils.h" +#include "tsl/profiler/lib/traceme.h" +#include "tsl/profiler/protobuf/xplane.pb.h" + +#if defined(__APPLE__) +#include +#include +#endif + +#if defined(ZML_RUNTIME_CUDA) +#include "nvtx3/nvToolsExt.h" +#endif + +#if defined(ZML_RUNTIME_ROCM) +#include "rocprofiler-sdk-roctx/roctx.h" +#endif + +namespace { + +struct LocalHostTraceSession { + bool active = false; + uint64_t start_timestamp_ns = 0; +}; + +LocalHostTraceSession g_local_host_trace_session; + +tensorflow::profiler::XPlane* FindOrAddHostPlane( + tensorflow::profiler::XSpace* space) { + for (auto& plane : *space->mutable_planes()) { + if (plane.name() == "/host:CPU") { + return &plane; + } + } + tensorflow::profiler::XPlane* plane = space->add_planes(); + plane->set_name("/host:CPU"); + return plane; +} + +zig_slice CopyToOwnedSlice(const std::string& data) { + if (data.empty()) { + return {}; + } + void* ptr = std::malloc(data.size()); + if (ptr == nullptr) { + return {}; + } + std::memcpy(ptr, data.data(), data.size()); + return {ptr, data.size()}; +} + +#if defined(ZML_RUNTIME_CUDA) +struct CudaNvtxRangeBackend { + explicit CudaNvtxRangeBackend(const std::string& name) + : domain(TslNvtxDomain()), active(domain != nullptr) { + if (!active) { + return; + } + + nvtxEventAttributes_t attrs{}; + attrs.version = NVTX_VERSION; + attrs.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE; + attrs.messageType = NVTX_MESSAGE_TYPE_ASCII; + attrs.message.ascii = name.c_str(); + nvtxDomainRangePushEx(domain, &attrs); + } + + ~CudaNvtxRangeBackend() { + if (active) { + nvtxDomainRangePop(domain); + } + } + + static bool IsEnabled() { return TslNvtxDomain() != nullptr; } + + private: + static nvtxDomainHandle_t TslNvtxDomain() { + static nvtxDomainHandle_t domain = nvtxDomainCreateA("TSL"); + return domain; + } + + nvtxDomainHandle_t domain = nullptr; + bool active = false; +}; +#else +struct CudaNvtxRangeBackend { + explicit CudaNvtxRangeBackend(const std::string&) {} + static bool IsEnabled() { return false; } +}; +#endif + +#if defined(ZML_RUNTIME_ROCM) +struct RocmRoctxRangeBackend { + explicit RocmRoctxRangeBackend(const std::string& name) + : active(RangePush() != nullptr && RangePop() != nullptr) { + if (!active) { + return; + } + + const int level = RangePush()(name.c_str()); + active = level >= 0; + } + + ~RocmRoctxRangeBackend() { + if (active) { + RangePop()(); + } + } + + static bool IsEnabled() { + return RangePush() != nullptr && RangePop() != nullptr; + } + + private: + using RangePushFn = decltype(&roctxRangePushA); + using RangePopFn = decltype(&roctxRangePop); + + static RangePushFn RangePush() { + return reinterpret_cast(dlsym(RTLD_DEFAULT, "roctxRangePushA")); + } + + static RangePopFn RangePop() { + return reinterpret_cast(dlsym(RTLD_DEFAULT, "roctxRangePop")); + } + + bool active = false; +}; +#else +struct RocmRoctxRangeBackend { + explicit RocmRoctxRangeBackend(const std::string&) {} + static bool IsEnabled() { return false; } +}; +#endif + +#if defined(__APPLE__) +struct MacOsSignpostRangeBackend { + explicit MacOsSignpostRangeBackend(const std::string& name) + : log(Logger()), signpost_id(os_signpost_id_generate(log)) { + os_signpost_interval_begin(log, signpost_id, "zml.scope", "%{public}s", + name.c_str()); + } + + ~MacOsSignpostRangeBackend() { + os_signpost_interval_end(log, signpost_id, "zml.scope", "%{public}s", + ""); + } + + static bool IsEnabled() { return true; } + + private: + static os_log_t Logger() { + static os_log_t log = + os_log_create("ai.zml", OS_LOG_CATEGORY_POINTS_OF_INTEREST); + return log; + } + + os_log_t log; + os_signpost_id_t signpost_id; +}; +#else +struct MacOsSignpostRangeBackend { + explicit MacOsSignpostRangeBackend(const std::string&) {} + static bool IsEnabled() { return false; } +}; +#endif + +} // namespace + +struct zml_traceme { + explicit zml_traceme(std::string encoded_name) + : name(std::move(encoded_name)), + traceme([&] { return name; }), + cuda_backend(name), + rocm_backend(name), + macos_backend(name) {} + + std::string name; + tsl::profiler::TraceMe traceme; + CudaNvtxRangeBackend cuda_backend; + RocmRoctxRangeBackend rocm_backend; + MacOsSignpostRangeBackend macos_backend; +}; + +extern "C" zml_traceme* zml_traceme_start(zig_slice name) { + return new zml_traceme( + std::string(static_cast(name.ptr), name.len)); +} + +extern "C" void zml_traceme_stop(zml_traceme* traceme) { delete traceme; } + +extern "C" bool zml_traceme_enabled(void) { + return tsl::profiler::TraceMe::Active() || + CudaNvtxRangeBackend::IsEnabled() || + RocmRoctxRangeBackend::IsEnabled() || + MacOsSignpostRangeBackend::IsEnabled(); +} + +extern "C" void zml_traceme_session_start(int level, uint64_t filter_mask, + bool enable_filter) { + g_local_host_trace_session.start_timestamp_ns = + tsl::profiler::GetCurrentTimeNanos(); + g_local_host_trace_session.active = + enable_filter ? tsl::profiler::TraceMeRecorder::Start(level, filter_mask) + : tsl::profiler::TraceMeRecorder::Start(level); +} + +extern "C" zig_slice zml_traceme_session_merge(zig_slice xspace) { + tensorflow::profiler::XSpace space; + if (xspace.len != 0 && + !space.ParseFromArray(xspace.ptr, static_cast(xspace.len))) { + return CopyToOwnedSlice( + std::string(static_cast(xspace.ptr), xspace.len)); + } + + if (g_local_host_trace_session.active) { + auto events = tsl::profiler::TraceMeRecorder::Stop(); + g_local_host_trace_session.active = false; + if (!events.empty()) { + tsl::profiler::ConvertCompleteEventsToXPlane( + g_local_host_trace_session.start_timestamp_ns, std::move(events), + FindOrAddHostPlane(&space)); + } + } + + return CopyToOwnedSlice(space.SerializeAsString()); +} + +extern "C" zig_slice zml_traceme_session_postprocess(zig_slice xspace, + uint64_t start_time_ns, + uint64_t stop_time_ns) { + tensorflow::profiler::XSpace space; + if (xspace.len != 0 && + !space.ParseFromArray(xspace.ptr, static_cast(xspace.len))) { + return {}; + } + + tsl::profiler::SetXSpacePidIfNotSet(space, getpid()); + tsl::profiler::PostProcessSingleHostXSpace(&space, start_time_ns, + stop_time_ns); + return CopyToOwnedSlice(space.SerializeAsString()); +} + +extern "C" void zml_traceme_str_free(zig_slice text) { std::free(text.ptr); } diff --git a/zml/profiling/tracer.zig b/zml/profiling/tracer.zig new file mode 100644 index 00000000..4b93fd80 --- /dev/null +++ b/zml/profiling/tracer.zig @@ -0,0 +1,165 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const c = @import("c"); +const platforms = @import("platforms"); +const zffi = @import("ffi"); + +pub const Scope = struct { + inner: ?*c.zml_traceme = null, + + pub fn end(self: *Scope) void { + if (self.inner) |inner| { + c.zml_traceme_stop(inner); + self.inner = null; + } + } + + pub fn deinit(self: *Scope) void { + self.end(); + } +}; + +// TraceMe is always the base host tracing path. The C tracing bridge adds NVTX +// ranges on CUDA/Linux builds, roctx ranges on ROCm/Linux builds, and +// os_signpost intervals on macOS. +pub const supportsDeviceAnnotations = switch (builtin.os.tag) { + .macos => true, + .linux => platforms.target == .cuda or platforms.target == .rocm, + else => false, +}; +const inline_scope_name_buffer_size = 1024; + +pub fn enabled() bool { + return c.zml_traceme_enabled(); +} + +pub fn scope(name: []const u8, metadata: anytype) !Scope { + const Metadata = @TypeOf(metadata); + comptime assertMetadataStruct(Metadata); + + if (comptime metadataFieldCount(Metadata) == 0) { + return .{ + .inner = c.zml_traceme_start(zffi.ZigSlice.from(name)), + }; + } + + var inline_buffer: [inline_scope_name_buffer_size]u8 = undefined; + var fixed_buffer_allocator: std.heap.FixedBufferAllocator = .init(&inline_buffer); + const fixed_allocator = fixed_buffer_allocator.allocator(); + + var encoded: std.ArrayList(u8) = .empty; + defer encoded.deinit(fixed_allocator); + + try appendEncodedName(&encoded, fixed_allocator, name, metadata); + return .{ + .inner = c.zml_traceme_start(zffi.ZigSlice.from(encoded.items)), + }; +} + +fn appendEncodedName(buffer: *std.ArrayList(u8), allocator: std.mem.Allocator, name: []const u8, metadata: anytype) !void { + const Metadata = @TypeOf(metadata); + try buffer.appendSlice(allocator, name); + if (comptime metadataFieldCount(Metadata) == 0) return; + + var field_count: usize = 0; + try buffer.append(allocator, '#'); + inline for (std.meta.fields(Metadata)) |field| { + if (shouldEncodeField(@TypeOf(@field(metadata, field.name)), @field(metadata, field.name), field.name)) { + if (field_count != 0) { + try buffer.append(allocator, ','); + } + field_count += 1; + + const key = comptime metadataKey(field.name); + validateMetadataToken(key); + try buffer.appendSlice(allocator, key); + try buffer.append(allocator, '='); + try appendMetadataValue(buffer, allocator, @field(metadata, field.name), field.name); + } + } + if (field_count == 0) { + buffer.items.len = name.len; + return; + } + try buffer.append(allocator, '#'); +} + +fn validateMetadataToken(token: []const u8) void { + if (std.mem.indexOfScalar(u8, token, '#') != null) { + @panic("trace metadata cannot contain '#'"); + } +} + +fn assertMetadataStruct(comptime T: type) void { + switch (@typeInfo(T)) { + .@"struct" => {}, + else => @compileError("trace metadata must be a struct literal like .{ .step_num = 42 }"), + } +} + +fn metadataFieldCount(comptime T: type) usize { + assertMetadataStruct(T); + return std.meta.fields(T).len; +} + +fn metadataKey(comptime field_name: []const u8) []const u8 { + return if (comptime std.mem.eql(u8, field_name, "root")) "_r" else field_name; +} + +fn shouldEncodeField(comptime T: type, value: T, comptime field_name: []const u8) bool { + return switch (@typeInfo(T)) { + .optional => value != null, + .bool => if (comptime std.mem.eql(u8, field_name, "root")) value else true, + else => true, + }; +} + +fn appendMetadataValue(buffer: *std.ArrayList(u8), allocator: std.mem.Allocator, value: anytype, comptime field_name: []const u8) !void { + const T = @TypeOf(value); + switch (@typeInfo(T)) { + .optional => { + try appendMetadataValue(buffer, allocator, value.?, field_name); + }, + .bool => { + if (comptime std.mem.eql(u8, field_name, "root")) { + try buffer.append(allocator, '1'); + } else { + try buffer.appendSlice(allocator, if (value) "1" else "0"); + } + }, + .int, .comptime_int => try buffer.print(allocator, "{}", .{value}), + .float, .comptime_float => try buffer.print(allocator, "{d}", .{value}), + .enum_literal => try buffer.appendSlice(allocator, @tagName(value)), + .@"enum" => try buffer.appendSlice(allocator, @tagName(value)), + .pointer => |ptr| { + if (ptr.size == .slice and ptr.child == u8) { + validateMetadataToken(value); + try buffer.appendSlice(allocator, value); + return; + } + + if (ptr.size == .one) switch (@typeInfo(ptr.child)) { + .array => |arr| { + if (arr.child == u8) { + const slice = value[0..]; + validateMetadataToken(slice); + try buffer.appendSlice(allocator, slice); + return; + } + }, + else => {}, + }; + + @compileError("trace metadata pointers must be []const u8 or string literals"); + }, + .array => |arr| { + if (arr.child != u8) { + @compileError("trace metadata arrays must be [N]u8"); + } + validateMetadataToken(value[0..]); + try buffer.appendSlice(allocator, value[0..]); + }, + else => @compileError("unsupported trace metadata type for field '" ++ field_name ++ "'"), + } +} diff --git a/zml/tracer.zig b/zml/tracer.zig deleted file mode 100644 index e2e173ef..00000000 --- a/zml/tracer.zig +++ /dev/null @@ -1,77 +0,0 @@ -const builtin = @import("builtin"); - -const c = @import("c"); - -pub const Tracer = switch (builtin.os.tag) { - // TODO(cerisier): fix MacOsTracer - // .macos => MacOsTracer, - .linux => if (@hasDecl(c, "ZML_RUNTIME_CUDA")) CudaTracer else FakeTracer, - else => FakeTracer, -}; - -const CudaTracer = struct { - - // Those symbols are defined in cudaProfiler.h but their implementation is in libcuda.so - // They will be bound at call time after libcuda.so is loaded (as a needed dependency of libpjrt_cuda.so). - const cuProfilerStart = @extern(*const fn () callconv(.c) c_int, .{ .name = "cuProfilerStart", .linkage = .weak }) orelse unreachable; - const cuProfilerStop = @extern(*const fn () callconv(.c) c_int, .{ .name = "cuProfilerStop", .linkage = .weak }) orelse unreachable; - - // Those symbols are defined in nvToolsExt.h which we don't want to provide. - // However, we link with libnvToolsExt.so which provides them. - // They will be bound at call time after libnvToolsExt.so is loaded (manually dlopen'ed by us). - const nvtxMarkA = @extern(*const fn ([*:0]const u8) callconv(.c) void, .{ .name = "nvtxMarkA", .linkage = .weak }) orelse unreachable; - const nvtxRangeStartA = @extern(*const fn ([*:0]const u8) callconv(.c) c_int, .{ .name = "nvtxRangeStartA", .linkage = .weak }) orelse unreachable; - const nvtxRangeEnd = @extern(*const fn (c_int) callconv(.c) void, .{ .name = "nvtxRangeEnd", .linkage = .weak }) orelse unreachable; - - pub fn event(message: [:0]const u8) void { - nvtxMarkA(message.ptr); - } - - pub fn frameStart(message: [:0]const u8) u64 { - return @intCast(nvtxRangeStartA(message.ptr)); - } - - pub fn frameEnd(interval_id: u64, message: [:0]const u8) void { - _ = message; - nvtxRangeEnd(@intCast(interval_id)); - return; - } -}; - -const MacOsTracer = struct { - logger: c.os_log_t, - - pub fn event(self: *const MacOsTracer, message: [:0]const u8) void { - const interval_id = c.os_signpost_id_generate(self.logger); - c.zml_os_signpost_event(self.logger, interval_id, message); - } - - pub fn frameStart(self: *const MacOsTracer, message: [:0]const u8) c.os_signpost_id_t { - const interval_id = c.os_signpost_id_generate(self.logger); - c.zml_os_signpost_interval_begin(self.logger, interval_id, message); - return interval_id; - } - - pub fn frameEnd(self: *const MacOsTracer, interval_id: c.os_signpost_id_t, message: [:0]const u8) void { - c.zml_os_signpost_interval_end(self.logger, interval_id, message); - } -}; - -/// Mock tracer for OS which don't have an impl. -const FakeTracer = struct { - pub fn event(message: [:0]const u8) void { - _ = message; - return; - } - - pub fn frameStart(message: [:0]const u8) u64 { - _ = message; - return 0; - } - - pub fn frameEnd(interval_id: u64, message: [:0]const u8) void { - _ = interval_id; - _ = message; - return; - } -}; diff --git a/zml/zml.zig b/zml/zml.zig index 4a037716..0c788ebf 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -37,6 +37,7 @@ pub const platform = @import("platform.zig"); pub const Memory = platform.Memory; pub const Platform = platform.Platform; pub const Target = platform.Target; +pub const tracer = @import("profiling/tracer.zig"); pub const safetensors = @import("safetensors.zig"); pub const shape = @import("shape.zig"); pub const Shape = shape.Shape; @@ -46,7 +47,6 @@ pub const Slice = slice.Slice; pub const tensor = @import("tensor.zig"); pub const Tensor = tensor.Tensor; pub const testing = @import("testing.zig"); -pub const tracer = @import("tracer.zig"); test "zml" { std.testing.refAllDecls(@This()); From 86aeed775e790edb3a4beaed65cc169b3d48253d Mon Sep 17 00:00:00 2001 From: Reese Chong Date: Mon, 11 May 2026 10:37:52 +0200 Subject: [PATCH 473/881] docs: fix excerpts don't match final code (#524) Two fixes to the `write_first_model.md` tutorial: - `allocator` on line 221 is deinitialized, however, `io` is not. `io` should be deinitialized as well. - `sharding` on line 216, is an undeclared identifier. It should be replaced with `.replicated`, which is used in the full code example. --- docs/tutorials/write_first_model.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tutorials/write_first_model.md b/docs/tutorials/write_first_model.md index 036776eb..f6174b36 100644 --- a/docs/tutorials/write_first_model.md +++ b/docs/tutorials/write_first_model.md @@ -213,7 +213,7 @@ To create the _Args_ and _Results_, the _Executable_ exposes two conveniences. ```zig // create the input buffer const input_slice: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[3]f16{ 5.0, 5.0, 5.0 })); -var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice, sharding); +var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice, .replicated); defer input_buffer.deinit(); // create the Args and Results structs @@ -347,7 +347,7 @@ pub fn main(init: std.process.Init) !void { const io = init.io; var platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator); + defer platform.deinit(allocator, io); const layer: Layer = .{ .weight = zml.Tensor.init(.{3}, .f16), From 10536817a4a9aad5a726e92e3cd4bfa06ae2952d Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Mon, 11 May 2026 11:16:56 +0200 Subject: [PATCH 474/881] zml/sharding: fix leak and use mlir.Attribute instead of concrete types (#527) Co-authored-by: Guillaume Wenzek --- zml/Sharding.zig | 14 +++++++++----- zml/ops.zig | 6 +++--- zml/platform.zig | 13 +++++++++---- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/zml/Sharding.zig b/zml/Sharding.zig index 43d61bc3..83067a17 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -112,7 +112,7 @@ pub const Partitioning = struct { return sharding.data.numPartitionsForLogicalAxis(logical_axis); } - pub fn sdyPerValueShardingAttr(self: Partitioning, allocator: std.mem.Allocator, ctx: *mlir.Context, shapes: []const Shape) !*const dialects.shardy.TensorShardingPerValueAttribute { + pub fn sdyPerValueShardingAttr(self: Partitioning, allocator: std.mem.Allocator, ctx: *mlir.Context, shapes: []const Shape) !*const mlir.Attribute { stdx.debug.assert(self.partitioner == .shardy, "sdyPerValueShardingAttr requires shardy partitioner", .{}); const shardings = try allocator.alloc(*const dialects.shardy.TensorShardingAttribute, shapes.len); @@ -120,10 +120,10 @@ pub const Partitioning = struct { const sharding = try self.selectSharding(shape); shardings[i] = try sharding.data.sdyShardingAttrForShape(allocator, ctx, shape); } - return .init(ctx, shardings); + return dialects.shardy.TensorShardingPerValueAttribute.init(ctx, shardings).asAttr(); } - pub fn sdyManualAxesAttr(self: Partitioning, allocator: std.mem.Allocator, ctx: *mlir.Context, in_shapes: []const Shape, out_shapes: []const Shape) !*const dialects.shardy.ManualAxesAttribute { + pub fn sdyManualAxesAttr(self: Partitioning, allocator: std.mem.Allocator, ctx: *mlir.Context, in_shapes: []const Shape, out_shapes: []const Shape) !*const mlir.Attribute { stdx.debug.assert(self.partitioner == .shardy, "sdyManualAxesAttr requires shardy partitioner", .{}); var axis_names = std.ArrayList([]const u8).empty; @@ -170,7 +170,7 @@ pub const Partitioning = struct { axes[i] = mlir.StringAttribute.init(ctx, axis_name); } - return dialects.shardy.ManualAxesAttribute.init(ctx, axes); + return dialects.shardy.ManualAxesAttribute.init(ctx, axes).asAttr(); } pub fn selectSharding(self: Partitioning, shape: Shape) !Sharding { @@ -1334,7 +1334,11 @@ pub const Data = struct { return .init(ctx, .{ .mesh = data.name, .dimensions = dimensions, .replicated_axes = replicated_axes }); } - pub fn gspmdShardingAttrForShape(self: *const Data, allocator: std.mem.Allocator, shape: Shape) ![]const u8 { + pub fn gspmdShardingAttrForShape(self: *const Data, parent_allocator: std.mem.Allocator, shape: Shape) ![]const u8 { + var arena = try stdx.arenaWithCapacity(parent_allocator, 1024); + defer arena.deinit(); + const allocator = arena.allocator(); + var has_sharding = false; for (0..shape.rank()) |ax| { if (shape.partition(ax) == .axis) { diff --git a/zml/ops.zig b/zml/ops.zig index 9212c63e..3196309d 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1305,9 +1305,9 @@ fn manualComputationInternal( .results = .{ .flat = global_result_types }, .blocks = &.{manual_block}, .attributes = &.{ - .named(ctx.mlir_ctx, "in_shardings", in_shardings_attr.asAttr()), - .named(ctx.mlir_ctx, "out_shardings", out_shardings_attr.asAttr()), - .named(ctx.mlir_ctx, "manual_axes", manual_axes_attr.asAttr()), + .named(ctx.mlir_ctx, "in_shardings", in_shardings_attr), + .named(ctx.mlir_ctx, "out_shardings", out_shardings_attr), + .named(ctx.mlir_ctx, "manual_axes", manual_axes_attr), }, .verify = true, .location = .unknown(ctx.mlir_ctx), diff --git a/zml/platform.zig b/zml/platform.zig index c64af15b..9eb021b5 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -203,7 +203,7 @@ pub const Platform = struct { memories: []const Memory, physical_mesh: zml.Sharding.PhysicalMesh, replicated_sharding: zml.Sharding, - shardings: std.StringArrayHashMapUnmanaged(zml.Sharding.Data), + shardings: std.StringArrayHashMapUnmanaged(zml.Sharding), pub const MAX_NUM_DEVICES: u16 = if (platforms.isEnabled(.tpu)) 64 else 32; @@ -532,9 +532,14 @@ pub const Platform = struct { std.debug.panic("Another sharding already exists with this name: {s}", .{name}); } - entry.key_ptr.* = try arena.dupe(u8, name); - entry.value_ptr.* = try .init(name, platform.physical_mesh, logical, strategy); - return .{ .data = entry.value_ptr }; + const owned_name = try arena.dupe(u8, name); + const owned_data = try arena.create(Sharding.Data); + owned_data.* = try .init(owned_name, platform.physical_mesh, logical, strategy); + const sharding: Sharding = .{ .data = owned_data }; + entry.key_ptr.* = owned_name; + entry.value_ptr.* = sharding; + + return sharding; } }; From 532c22955dcad74763b4c9f72c26d017c17aa0e3 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Mon, 11 May 2026 18:15:42 +0200 Subject: [PATCH 475/881] zml/ops: Merge the two customCall and add compute_on_host (#511) Add `compute_on_host` to force host offloading and merging the `customCallInternal` and `typeCustomCall` function into one to avoid duplication. I've slightly simplified the signature of some of the functions I wrote initially to avoid passing the types explicitly. --- zml/ops.zig | 332 +++++++++++++++++++------------------------------ zml/pjrtx.zig | 28 +++++ zml/tensor.zig | 2 +- 3 files changed, 156 insertions(+), 206 deletions(-) diff --git a/zml/ops.zig b/zml/ops.zig index 3196309d..2bc0ddea 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -15,7 +15,9 @@ const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; +pub const ShapeToCustomCallBuffer = @import("pjrtx.zig").ShapeToCustomCallBuffer; const Tensor = @import("tensor.zig").Tensor; +pub const TensorToCustomCallBuffer = @import("pjrtx.zig").TensorToCustomCallBuffer; pub const ReduceArgs = struct { left: Tensor, @@ -1052,8 +1054,21 @@ fn CustomCallResultTypeFromOutputSpec(comptime OutputSpecT: type) type { pub const CustomCallOptions = struct { has_side_effect: bool, output_operand_aliases: ?[]const dialects.stablehlo.CustomCallOpts.OutputOperandAlias = null, + compute_on_host: bool = false, }; +fn customCallAdditionalAttributes(ctx: *CompilationContext, opts: CustomCallOptions) []const mlir.NamedAttribute { + if (!opts.compute_on_host or ctx.platform.target == .cpu) return &.{}; + + const frontend_attributes = mlir.dictionaryAttribute(ctx.mlir_ctx, &.{ + .named(ctx.mlir_ctx, "_xla_compute_type", mlir.stringAttribute(ctx.mlir_ctx, "host")), + }); + + const additional_attributes = ctx.arena.allocator().alloc(mlir.NamedAttribute, 1) catch unreachable; + additional_attributes[0] = .named(ctx.mlir_ctx, "mhlo.frontend_attributes", frontend_attributes); + return additional_attributes; +} + fn customCallOutputOperandAliases( comptime I: type, comptime O: type, @@ -1133,7 +1148,13 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, else => @compileError("Unsupported type: " ++ @typeName(@TypeOf(outputs))), }; - const outputs_flat = customCallInternal(target_name, inputs_, output_shapes, metadata, opts); + const outputs_flat = typedCustomCall( + target_name, + opts, + inputs_, + output_shapes, + metadata, + ); if (comptime CustomCallResultTypeFromOutputSpec(@TypeOf(outputs)) == void) { stdx.debug.assert(outputs_flat.len == 0, "customCall expected zero outputs for void return, got {}", .{outputs_flat.len}); return; @@ -1527,6 +1548,10 @@ fn metadataFieldToMlirAttribute(mlir_ctx: *mlir.Context, comptime T: type, value const type_info = @typeInfo(T); return switch (type_info) { .comptime_int => mlir.integerAttribute(mlir_ctx, .u64, @as(u64, value)), + .@"enum" => |enum_field| switch (@typeInfo(enum_field.tag_type)) { + .int => |int_tag| metadataIntegerFieldToMlirAttribute(mlir_ctx, int_tag, @intFromEnum(value)), + else => @compileError("Unsupported tag type for enum metadata: " ++ @typeName(enum_field.tag_type)), + }, .int => |int_field| metadataIntegerFieldToMlirAttribute(mlir_ctx, int_field, value), .float => |float_field| metadataFloatFieldToMlirAttribute(mlir_ctx, float_field, value), .bool => mlir.boolAttribute(mlir_ctx, value), @@ -1547,77 +1572,6 @@ fn metadataFieldToMlirAttribute(mlir_ctx: *mlir.Context, comptime T: type, value }; } -fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs: []const Shape, metadata: anytype, opts: CustomCallOptions) []Tensor { - const ctx = CompilationContext.current(); - const allocator = ctx.arena.allocator(); - - stdx.debug.assert(!opts.has_side_effect or ctx.manual_computation_depth > 0, "side-effect customCall '{s}' must be emitted inside manualComputation", .{target_name}); - - const values = allocator.alloc(*const mlir.Value, inputs.len) catch unreachable; - const input_shapes = allocator.alloc(Shape, inputs.len) catch unreachable; - for (inputs, 0..) |input, i| { - values[i] = input.value(); - input_shapes[i] = input.shape(); - } - - const result_types = allocator.alloc(*const mlir.Type, outputs.len) catch unreachable; - for (outputs, 0..) |shape, i| { - result_types[i] = mlir.rankedTensorType(shape.dims(), mlirx.Type.fromDType(ctx.mlir_ctx, shape.dtype())); - } - - const metadata_type_info = @typeInfo(@TypeOf(metadata)); - var metadata_attribute_list = std.ArrayList(mlir.NamedAttribute).initCapacity(allocator, metadata_type_info.@"struct".fields.len + 2) catch unreachable; - inline for (metadata_type_info.@"struct".fields) |field| { - if (metadataFieldToMlirAttribute(ctx.mlir_ctx, field.type, @field(metadata, field.name))) |attribute| { - metadata_attribute_list.appendAssumeCapacity(mlir.NamedAttribute.named(ctx.mlir_ctx, field.name, attribute)); - } - } - - metadata_attribute_list.appendSliceAssumeCapacity(&[_]mlir.NamedAttribute{ - .named(ctx.mlir_ctx, "pjrt_api", mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_api))))), - .named(ctx.mlir_ctx, "pjrt_client", mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_client))))), - }); - - const backend_config = mlir.dictionaryAttribute(ctx.mlir_ctx, metadata_attribute_list.items); - - const operand_layouts = allocator.alloc([]const usize, input_shapes.len) catch unreachable; - for (input_shapes, 0..) |shape, i| { - operand_layouts[i] = allocator.dupe(usize, toUsize(constants.minorToMajor(shape.rank())).constSlice()) catch unreachable; - } - - const result_layouts = allocator.alloc([]const usize, outputs.len) catch unreachable; - for (outputs, 0..) |shape, i| { - result_layouts[i] = allocator.dupe(usize, toUsize(constants.minorToMajor(shape.rank())).constSlice()) catch unreachable; - } - - const op = dialects.stablehlo.custom_call( - ctx.mlir_ctx, - values, - result_types, - .{ - .call_target_name = target_name, - .backend_config = .{ .typed_ffi = backend_config }, - .has_side_effect = opts.has_side_effect, - .operand_layouts = operand_layouts, - .result_layouts = result_layouts, - .output_operand_aliases = opts.output_operand_aliases orelse &.{}, - .additional_attributes = &.{}, - }, - .unknown(ctx.mlir_ctx), - ).appendTo(ctx.currentScope().block); - - if (ctx.manual_computation_depth > 0 and ctx.partitioning.partitioner == .gspmd) { - op.setAttributeByName("mhlo.sharding", mlir.stringAttribute(ctx.mlir_ctx, "{manual}")); - } - - const outputs_ = allocator.alloc(Tensor, outputs.len) catch unreachable; - for (outputs, 0..) |output_shape, i| { - outputs_[i] = Tensor._result(output_shape, op.result(i)); - } - - return outputs_; -} - pub fn CustomCall( // A, I and O are assumed to be simple struct without any nesting like: { a: zml.Tensor } I: type, @@ -1640,6 +1594,9 @@ pub fn CustomCall( /// Similar to reuseBuffer on tensors, tells XLA that the custom call re-uses an input buffer for its output. /// Use the input/output field names to express the mapping: `{ . = . }` output_operand_aliases: ?CustomCallOutputOperandAliases(I, O) = null, + /// Request XLA host-compute offload for this custom call on accelerator backends. + /// Input/Output buffers will automatically be moved d2h/h2d without the need for an explicit `.toMemory(...)`. + compute_on_host: bool = false, }, ) type { return struct { @@ -1651,7 +1608,7 @@ pub fn CustomCall( pub fn register(platform: *const Platform) !void { try platform.registerFfi(.{ .name = params.name, - .platform_name = null, + .platform_name = if (params.compute_on_host) "Host" else null, .handler = @This().handler, .traits = .{ .command_buffer_compatible = true, @@ -1660,32 +1617,26 @@ pub fn CustomCall( } pub fn call(input_tensors: I, output_shapes: O, attributes: A) ShapeToTensor(O) { - const args: CustomCallArgs(I, O, A) = .{ - .input_tensors = input_tensors, - .output_shapes = output_shapes, - .attributes = attributes, - }; const opts: CustomCallOptions = .{ .has_side_effect = params.has_side_effect, .output_operand_aliases = comptime customCallOutputOperandAliases(I, O, params.output_operand_aliases), + .compute_on_host = params.compute_on_host, }; if (params.sharding_aware) { return shardingAwareTypedCustomCall( - I, - O, - A, params.name, opts, - args, + input_tensors, + output_shapes, + attributes, ); } else { return typedCustomCall( - I, - O, - A, params.name, opts, - args, + input_tensors, + output_shapes, + attributes, ); } } @@ -1700,171 +1651,137 @@ pub fn CustomCall( }; } -fn CustomCallArgs(I: type, O: type, A: type) type { - return struct { - input_tensors: I, - output_shapes: O, - attributes: A, - }; -} - -pub fn TensorToCustomCallBuffer(T: type) type { - return meta.MapRestrict(Tensor, CustomCallBuffer).map(T); -} - -pub fn ShapeToCustomCallBuffer(T: type) type { - return meta.MapRestrict(Shape, CustomCallBuffer).map(T); -} - pub fn ShapeToTensor(T: type) type { return meta.MapRestrict(Shape, Tensor).map(T); } pub fn CustomCallOutputOperandAliases(I: type, O: type) type { - return meta.MapRestrict(Shape, ?std.meta.FieldEnum(I)).map(O); + const output_fields = @typeInfo(O).@"struct".fields; + var field_names: [output_fields.len][]const u8 = undefined; + inline for (output_fields, 0..) |field, i| { + field_names[i] = field.name; + } + return @Struct( + .auto, + null, + &field_names, + &@splat(?std.meta.FieldEnum(I)), + &@splat(.{ .default_value_ptr = &@as(?std.meta.FieldEnum(I), null) }), + ); } pub fn shardingAwareTypedCustomCall( - I: type, - O: type, - A: type, comptime target_name: [:0]const u8, comptime opts: CustomCallOptions, - args: CustomCallArgs(I, O, A), -) ShapeToTensor(O) { + input: anytype, + output: anytype, + attributes: anytype, +) ShapeToTensor(@TypeOf(output)) { + const Input = @TypeOf(input); + const Output = @TypeOf(output); + const Attributes = @TypeOf(attributes); + // Convert to slices so that manualComputationInternal can inspect the input/output. - var input_tensors: [@typeInfo(@TypeOf(args.input_tensors)).@"struct".fields.len]Tensor = undefined; - inline for (@typeInfo(@TypeOf(args.input_tensors)).@"struct".fields, 0..) |field, i| { - input_tensors[i] = @field(args.input_tensors, field.name); + var input_tensors: [@typeInfo(Input).@"struct".fields.len]Tensor = undefined; + inline for (@typeInfo(Input).@"struct".fields, 0..) |field, i| { + input_tensors[i] = @field(input, field.name); } - var output_shapes: [@typeInfo(@TypeOf(args.output_shapes)).@"struct".fields.len]Shape = undefined; - inline for (@typeInfo(@TypeOf(args.output_shapes)).@"struct".fields, 0..) |field, i| { - output_shapes[i] = @field(args.output_shapes, field.name); + var output_shapes: [@typeInfo(Output).@"struct".fields.len]Shape = undefined; + inline for (@typeInfo(Output).@"struct".fields, 0..) |field, i| { + output_shapes[i] = @field(output, field.name); } - const output_tensors = manualComputationInternal(&input_tensors, &output_shapes, args.attributes, (struct { - fn body(attributes: A, _: std.mem.Allocator, sharded_input_tensors: []const Tensor, sharded_output_shapes: []const Shape) []const Tensor { + const output_tensors = manualComputationInternal(&input_tensors, &output_shapes, attributes, (struct { + fn body(attributes_: Attributes, _: std.mem.Allocator, sharded_input_tensors: []const Tensor, sharded_output_shapes: []const Shape) []const Tensor { return typedCustomCall( - []const Tensor, - []const Shape, - A, target_name, opts, - .{ - .input_tensors = sharded_input_tensors, - .output_shapes = sharded_output_shapes, - .attributes = attributes, - }, + sharded_input_tensors, + sharded_output_shapes, + attributes_, ); } }).body); // Convert the slice back to a struct - var out: ShapeToTensor(O) = undefined; - inline for (@typeInfo(@TypeOf(args.output_shapes)).@"struct".fields, 0..) |field, i| { + var out: ShapeToTensor(Output) = undefined; + inline for (@typeInfo(Output).@"struct".fields, 0..) |field, i| { @field(out, field.name) = output_tensors[i]; } return out; } pub fn typedCustomCall( - I: type, - O: type, - A: type, - comptime target_name: [:0]const u8, - comptime opts: CustomCallOptions, - args: CustomCallArgs(I, O, A), -) ShapeToTensor(O) { + target_name: [:0]const u8, + opts: CustomCallOptions, + input: anytype, + output: anytype, + attributes: anytype, +) ShapeToTensor(@TypeOf(output)) { + const Input = @TypeOf(input); + const Output = @TypeOf(output); + const Attributes = @TypeOf(attributes); + const ctx = CompilationContext.current(); const allocator = ctx.arena.allocator(); stdx.debug.assert(!opts.has_side_effect or ctx.manual_computation_depth > 0, "side-effect customCall '{s}' must be emitted inside manualComputation", .{target_name}); - const input_tensors: []const Tensor = switch (@typeInfo(@TypeOf(args.input_tensors))) { + const input_tensors: []const Tensor = switch (@typeInfo(Input)) { .@"struct" => |struct_info| b: { var input_tensors: [struct_info.fields.len]Tensor = undefined; inline for (struct_info.fields, 0..) |field, i| { - input_tensors[i] = @field(args.input_tensors, field.name); + input_tensors[i] = @field(input, field.name); } break :b &input_tensors; }, // Extra case to support []const Tensor from shardingAwareTypedCustomCall .pointer => |pointer_info| b: { if (pointer_info.size != .slice) @compileError("Expected input slice"); - break :b args.input_tensors; + break :b input; }, - else => @compileError("Unsupported input type: " ++ @typeName(@TypeOf(args.input_tensor))), + else => @compileError("Unsupported input type: " ++ @typeName(Input)), }; - const output_shapes: []const Shape = switch (@typeInfo(@TypeOf(args.output_shapes))) { + const output_shapes: []const Shape = switch (@typeInfo(Output)) { .@"struct" => |struct_info| b: { var output_shapes: [struct_info.fields.len]Shape = undefined; inline for (struct_info.fields, 0..) |field, i| { - output_shapes[i] = @field(args.output_shapes, field.name); + output_shapes[i] = @field(output, field.name); } break :b &output_shapes; }, // Extra case to support []const Shape from shardingAwareTypedCustomCall .pointer => |pointer_info| b: { if (pointer_info.size != .slice) @compileError("Expected input slice"); - break :b args.output_shapes; + break :b output; }, - else => @compileError("Unsupported input type: " ++ @typeName(@TypeOf(args.input_tensor))), + else => @compileError("Unsupported output type: " ++ @typeName(Output)), }; - const metadata_attributes = - switch (@typeInfo(@TypeOf(args.attributes))) { - .@"struct" => |struct_info| b: { - var mlir_attributes_array: [struct_info.fields.len]mlir.NamedAttribute = undefined; - inline for (struct_info.fields, 0..) |field, i| { - const attribute: *const mlir.Attribute = switch (@typeInfo(field.type)) { - .comptime_int => mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @field(args.attributes, field.name))), - .int => |int_field| switch (int_field.signedness) { - .signed => switch (int_field.bits) { - 8 => mlir.integerAttribute(ctx.mlir_ctx, .i8, @field(args.attributes, field.name)), - 16 => mlir.integerAttribute(ctx.mlir_ctx, .i16, @field(args.attributes, field.name)), - 32 => mlir.integerAttribute(ctx.mlir_ctx, .i32, @field(args.attributes, field.name)), - 64 => mlir.integerAttribute(ctx.mlir_ctx, .i64, @field(args.attributes, field.name)), - else => @panic("Unsupported DataType"), - }, - .unsigned => switch (int_field.bits) { - 8 => mlir.integerAttribute(ctx.mlir_ctx, .u8, @field(args.attributes, field.name)), - 16 => mlir.integerAttribute(ctx.mlir_ctx, .u16, @field(args.attributes, field.name)), - 32 => mlir.integerAttribute(ctx.mlir_ctx, .u32, @field(args.attributes, field.name)), - 64 => mlir.integerAttribute(ctx.mlir_ctx, .u64, @field(args.attributes, field.name)), - else => @panic("Unsupported DataType"), - }, - }, - .float => |float_field| switch (float_field.bits) { - 16 => mlir.floatAttribute(ctx.mlir_ctx, .f16, @as(f64, @field(args.attributes, field.name))), - 32 => mlir.floatAttribute(ctx.mlir_ctx, .f32, @as(f64, @field(args.attributes, field.name))), - 64 => mlir.floatAttribute(ctx.mlir_ctx, .f64, @as(f64, @field(args.attributes, field.name))), - else => @panic("Unsupported DataType"), - }, - .bool => mlir.boolAttribute(ctx.mlir_ctx, @field(args.attributes, field.name)), - .pointer => |pointer_info| switch (pointer_info.size) { - .slice => if (pointer_info.child == u8) - mlir.stringAttribute(ctx.mlir_ctx, @field(args.attributes, field.name)) - else - @panic("Unsupported pointer type in metadata"), - else => @panic("Unsupported pointer type in metadata"), - }, - else => @compileError("Unsupported metadata type: " ++ @typeName(field.type)), - }; - mlir_attributes_array[i] = mlir.NamedAttribute.named(ctx.mlir_ctx, field.name, attribute); - } - - break :b mlir_attributes_array; - }, - else => @compileError("Unsupported type: " ++ @typeName(@TypeOf(args.attributes))), - }; + var metadata_attribute_list = switch (@typeInfo(Attributes)) { + .void => std.ArrayList(mlir.NamedAttribute).initCapacity(allocator, 2) catch unreachable, + .@"struct" => |struct_info| std.ArrayList(mlir.NamedAttribute).initCapacity(allocator, struct_info.fields.len + 2) catch unreachable, + else => @compileError("Unsupported type: " ++ @typeName(Attributes)), + }; + if (@typeInfo(Attributes) == .@"struct") { + inline for (@typeInfo(Attributes).@"struct".fields) |field| { + if (metadataFieldToMlirAttribute(ctx.mlir_ctx, field.type, @field(attributes, field.name))) |attribute| { + metadata_attribute_list.appendAssumeCapacity(mlir.NamedAttribute.named(ctx.mlir_ctx, field.name, attribute)); + } + } + } + metadata_attribute_list.appendSliceAssumeCapacity(&[_]mlir.NamedAttribute{ + .named(ctx.mlir_ctx, "pjrt_api", mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_api))))), + .named(ctx.mlir_ctx, "pjrt_client", mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_client))))), + }); const values = allocator.alloc(*const mlir.Value, input_tensors.len) catch unreachable; const input_shapes = allocator.alloc(Shape, input_tensors.len) catch unreachable; - for (input_tensors, 0..) |input, i| { - values[i] = input.value(); - input_shapes[i] = input.shape(); + for (input_tensors, 0..) |input_tensor, i| { + values[i] = input_tensor.value(); + input_shapes[i] = input_tensor.shape(); } const result_types = allocator.alloc(*const mlir.Type, output_shapes.len) catch unreachable; @@ -1872,10 +1789,7 @@ pub fn typedCustomCall( result_types[i] = mlir.rankedTensorType(shape.dims(), mlirx.Type.fromDType(ctx.mlir_ctx, shape.dtype())); } - const backend_config = mlir.dictionaryAttribute(ctx.mlir_ctx, &(metadata_attributes ++ [_]mlir.NamedAttribute{ - .named(ctx.mlir_ctx, "pjrt_api", mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_api))))), - .named(ctx.mlir_ctx, "pjrt_client", mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_client))))), - })); + const backend_config = mlir.dictionaryAttribute(ctx.mlir_ctx, metadata_attribute_list.items); const operand_layouts = allocator.alloc([]const usize, input_shapes.len) catch unreachable; for (input_shapes, 0..) |shape, i| { @@ -1898,7 +1812,7 @@ pub fn typedCustomCall( .operand_layouts = operand_layouts, .result_layouts = result_layouts, .output_operand_aliases = opts.output_operand_aliases orelse &.{}, - .additional_attributes = &.{}, + .additional_attributes = customCallAdditionalAttributes(ctx, opts), }, .unknown(ctx.mlir_ctx), ).appendTo(ctx.currentScope().block); @@ -1907,9 +1821,9 @@ pub fn typedCustomCall( op.setAttributeByName("mhlo.sharding", mlir.stringAttribute(ctx.mlir_ctx, "{manual}")); } - switch (@typeInfo(@TypeOf(args.output_shapes))) { + switch (@typeInfo(Output)) { .@"struct" => |struct_info| { - var out: ShapeToTensor(O) = undefined; + var out: ShapeToTensor(Output) = undefined; inline for (output_shapes, struct_info.fields, 0..) |output_shape, field, i| { @field(out, field.name) = Tensor._result(output_shape, op.result(i)); } @@ -1918,13 +1832,13 @@ pub fn typedCustomCall( // Extra case to support []const Shape from shardingAwareTypedCustomCall .pointer => |pointer_info| { if (pointer_info.size != .slice) @compileError("Expected input slice"); - var out: []Tensor = allocator.alloc(Tensor, args.output_shapes.len) catch unreachable; + var out: []Tensor = allocator.alloc(Tensor, output_shapes.len) catch unreachable; for (output_shapes, 0..) |output_shape, i| { out[i] = Tensor._result(output_shape, op.result(i)); } return out; }, - else => @compileError("Unsupported input type: " ++ @typeName(@TypeOf(args.input_tensor))), + else => @compileError("Unsupported output type: " ++ @typeName(Output)), } } @@ -1945,12 +1859,20 @@ fn customCallArgsFromPjrtCallFrame(I: type, O: type, A: type, call_frame: *pjrt. @field(output, field.name) = .fromPjrt(shape_buf); } - var attributes: A = undefined; - inline for (@typeInfo(A).@"struct".fields) |field| { - const attribute = call_frame.attrs.getByName(.scalar, field.name) orelse - @panic("Attribute not found: " ++ field.name); - @field(attributes, field.name) = attribute.get(field.type); - } + const attributes: A = switch (@typeInfo(A)) { + .void => {}, + .@"struct" => blk: { + var attrs: A = undefined; + inline for (@typeInfo(A).@"struct".fields) |field| { + // TODO: Use getByIndex instead? + const attribute = call_frame.attrs.getByName(.scalar, field.name) orelse + @panic("Attribute not found: " ++ field.name); + @field(attrs, field.name) = attribute.get(field.type); + } + break :blk attrs; + }, + else => @compileError("Unsupported attributes type: " ++ @typeName(A)), + }; return .{ input, output, attributes }; } diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index e328c5de..038d5682 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -5,7 +5,9 @@ const mlir = @import("mlir"); const pjrt = @import("pjrt"); pub const DataType = @import("dtype.zig").DataType; +const meta = @import("meta.zig"); const Shape = @import("shape.zig").Shape; +const Tensor = @import("tensor.zig").Tensor; const log = std.log.scoped(.@"zml/pjrtx"); @@ -43,6 +45,14 @@ pub const Client = opaque { } }; +pub fn TensorToCustomCallBuffer(T: type) type { + return meta.MapRestrict(Tensor, CustomCallBuffer).map(T); +} + +pub fn ShapeToCustomCallBuffer(T: type) type { + return meta.MapRestrict(Shape, CustomCallBuffer).map(T); +} + pub const CustomCallBuffer = struct { shape: Shape, ptr: *anyopaque, @@ -65,6 +75,24 @@ pub const CustomCallBuffer = struct { .shape = .init(buffer.dims(), dt), }; } + + // Assumes as host-side buffer + pub fn asHostSlice(self: CustomCallBuffer) []u8 { + const bytes: [*]u8 = @ptrCast(@alignCast(self.ptr)); + return bytes[0..self.shape.byteSize()]; + } + + // Assumes as host-side buffer + pub fn asHostConstSlice(self: CustomCallBuffer) []const u8 { + const bytes: [*]const u8 = @ptrCast(@alignCast(self.ptr)); + return bytes[0..self.shape.byteSize()]; + } + + // Assumes as host-side buffer + pub fn asHostScalar(self: CustomCallBuffer, T: type) T { + std.debug.assert(self.shape.byteSize() == @sizeOf(T)); + return std.mem.bytesAsValue(T, self.asHostConstSlice()[0..@sizeOf(T)]).*; + } }; pub fn bufferTypeFromDtype(dt: DataType) pjrt.BufferType { diff --git a/zml/tensor.zig b/zml/tensor.zig index eb84949d..e7dae22e 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -181,7 +181,7 @@ pub const Tensor = struct { return _result(partitioned_shape, op_result); } - pub fn toMemory(self: Tensor, kind: Memory) Tensor { + pub fn toMemory(self: Tensor, kind: Memory.Kind) Tensor { const ctx = CompilationContext.current(); if (ctx.platform.target == .cpu) return self; From 144f3eb0afc2fe050768a9480bd1c6de8d32946a Mon Sep 17 00:00:00 2001 From: elogir Date: Tue, 12 May 2026 09:58:32 +0200 Subject: [PATCH 476/881] platforms/neuron: bump toolchain and stabilize support (#476) - bump Neuron dependencies in `platforms/neuron/requirements.lock.txt` - `libneuronxla` `2.2.15515.0+50c26cbd` -> `2.2.16408.0+50c26cbd` - `neuronx-cc` `2.23.6484.0+3b612583` -> `2.24.8799.0+6f62ff7c` - make `neuronx-cc` invocation more robust by fixing path resolution, releasing the Python GIL during compilation, checking the compiler process exit status, and propagating NEFF wrapping/read failures - resolve PJRT devices and memories by PJRT handles instead of assuming stable numeric ids during platform initialization, including Neuron cases where memory ids are reused across devices - avoid unsupported Neuron synchronization/event paths by always requesting execution events on Neuron, deinitializing them without awaiting, and removing the explicit await from the sharding example - avoid unsupported Neuron runtime paths by: - falling back to host-backed buffer initialization for `Buffer.uninitialized` - using passthrough DMA allocation on Neuron - skipping `Tensor.print()` custom calls on Neuron - add NRT-backed Neuron physical mesh topology construction, including chip/core placement, ring/torus/point-to-point topology inference, and Neuron local-hardware-id based shard placement - let the sharding example pick the default partitioner from the current target --- examples/sharding/main.zig | 12 +- platforms/neuron/BUILD.bazel | 7 +- platforms/neuron/libneuronxla.zig | 80 +- platforms/neuron/neuronx-cc.zig | 4 + platforms/neuron/requirements.lock.txt | 974 +++++++++++++++++++------ platforms/neuron/topology/BUILD.bazel | 37 + platforms/neuron/topology/topology.zig | 287 ++++++++ stdx/process.zig | 4 +- zml/BUILD.bazel | 5 + zml/Sharding.zig | 132 +++- zml/buffer.zig | 15 +- zml/exe.zig | 25 +- zml/mem.zig | 4 +- zml/platform.zig | 50 +- zml/tensor.zig | 15 +- 15 files changed, 1346 insertions(+), 305 deletions(-) create mode 100644 platforms/neuron/topology/BUILD.bazel create mode 100644 platforms/neuron/topology/topology.zig diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index 2602c2da..2a407b67 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -17,7 +17,7 @@ const CliMesh = enum { }; const CliArgs = struct { - partitioner: zml.Sharding.Partitioner = .shardy, + partitioner: ?zml.Sharding.Partitioner = null, mesh: CliMesh = .auto, }; @@ -156,7 +156,7 @@ fn createSequenceBuffer( pub fn main(init: std.process.Init) !void { const allocator = init.gpa; const io = init.io; - const args = try parseArgs(init); + var args = try parseArgs(init); const create_options: zml.platform.CreateOptions = switch (args.mesh) { .auto => .{}, @@ -183,7 +183,12 @@ pub fn main(init: std.process.Init) !void { _ = profiler.stop() catch unreachable; } - log.info("Partitioner: {s}", .{@tagName(args.partitioner)}); + if (args.partitioner) |partitioner| { + log.info("Partitioner: {s}", .{@tagName(partitioner)}); + } else { + args.partitioner = .fromTarget(platform.target); + log.info("Partitioner: {s} (default)", .{@tagName(args.partitioner.?)}); + } log.info("{f}", .{platform.physical_mesh}); const sharding: zml.Sharding = try platform.registerSharding( @@ -239,7 +244,6 @@ pub fn main(init: std.process.Init) !void { exe.call(exe_args, &exe_results); var out = exe_results.get(zml.Buffer); defer out.deinit(); - _ = try out.await(io); const out_slice = try out.toSliceAlloc(allocator, io); defer out_slice.free(allocator); diff --git a/platforms/neuron/BUILD.bazel b/platforms/neuron/BUILD.bazel index 91cd6cc9..1e815ff9 100644 --- a/platforms/neuron/BUILD.bazel +++ b/platforms/neuron/BUILD.bazel @@ -54,19 +54,24 @@ cc_library( name = "empty", tags = ["manual"], defines = ["ZML_RUNTIME_NEURON_DISABLED"], + visibility = ["//platforms/neuron/topology:__pkg__"], ) cc_library( name = "zmlxneuron", tags = ["manual"], defines = ["ZML_RUNTIME_NEURON"], + visibility = ["//platforms/neuron/topology:__pkg__"], ) cc_library( name = "libnrt_headers", tags = ["manual"], hdrs = ["nrt.h"], - visibility = ["//bin/zml-smi:__subpackages__"], + visibility = [ + "//bin/zml-smi:__subpackages__", + "//platforms/neuron/topology:__pkg__", + ], deps = ["@libpjrt_neuron//:libnrt_headers"], ) diff --git a/platforms/neuron/libneuronxla.zig b/platforms/neuron/libneuronxla.zig index e11907cb..56a67ab0 100644 --- a/platforms/neuron/libneuronxla.zig +++ b/platforms/neuron/libneuronxla.zig @@ -191,37 +191,55 @@ fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) const neff_file = try std.Io.Dir.path.join(arena.allocator(), &.{ tmp_dir, "file.neff" }); - var neuronx_cc_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - var child = try std.process.spawn(io, .{ - .argv = &.{ - try stdx.Io.Dir.path.bufJoin(&neuronx_cc_buf, &.{ - stdx.process.selfSharedObjectDirPath(), - "..", - "bin", - "neuronx-cc", - }), - "compile", - "--framework=XLA", - "--target", - target, - "--verbose=35", - "--enable-internal-neff-wrapper", - "--output", - neff_file, - "--optlevel=1", - // generic is the default, but it fails on transformers, force it - "--model-type=transformer", - // disable it, we do our own - "--auto-cast=none", - "--enable-fast-loading-neuron-binaries", - code_file, - }, - .stdin = .ignore, - .stdout = .inherit, - .stderr = .inherit, - .cwd = .{ .path = tmp_dir }, - }); - _ = try child.wait(io); + { + const gil_state = c.PyEval_SaveThread(); + defer c.PyEval_RestoreThread(gil_state); + + var neuronx_cc_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + var child = try std.process.spawn(io, .{ + .argv = &.{ + try stdx.Io.Dir.path.bufJoin(&neuronx_cc_buf, &.{ + stdx.process.selfSharedObjectDirPath(), + "..", + "bin", + "neuronx-cc", + }), + "compile", + "--framework=XLA", + "--target", + target, + "--verbose=35", + "--enable-internal-neff-wrapper", + "--output", + neff_file, + "--optlevel=1", + // generic is the default, but it fails on transformers, force it + "--model-type=transformer", + // disable it, we do our own + "--auto-cast=none", + "--enable-fast-loading-neuron-binaries", + code_file, + }, + .stdin = .ignore, + .stdout = .ignore, + .stderr = .ignore, + .cwd = .{ .path = tmp_dir }, + }); + + const term = try child.wait(io); + switch (term) { + .exited => |exit_code| { + if (exit_code != 0) { + log.err("neuronx-cc failed: {}", .{exit_code}); + return error.NeuronxCcFailed; + } + }, + else => { + log.err("neuronx-cc failed, unknown error", .{}); + return error.NeuronxCcFailedUnknown; + }, + } + } const neff_hlo_bytes = wrapNeffAsCustomCall(arena.allocator(), io, code, neff_file) catch |err| { log.err("Error wrapping NEFF as custom call: {}\n", .{err}); diff --git a/platforms/neuron/neuronx-cc.zig b/platforms/neuron/neuronx-cc.zig index d57094b7..1ac09e2d 100644 --- a/platforms/neuron/neuronx-cc.zig +++ b/platforms/neuron/neuronx-cc.zig @@ -101,5 +101,9 @@ pub fn main(init: std.process.Init) !void { defer c.Py_DecRef(neuronxcc_main); const result = c.PyObject_CallNoArgs(neuronxcc_main); + if (result == null) { + c.PyErr_Print(); + std.process.exit(1); + } defer c.Py_DecRef(result); } diff --git a/platforms/neuron/requirements.lock.txt b/platforms/neuron/requirements.lock.txt index b84f682a..e18a16f4 100644 --- a/platforms/neuron/requirements.lock.txt +++ b/platforms/neuron/requirements.lock.txt @@ -4,159 +4,293 @@ --extra-index-url https://pip.repos.neuron.amazonaws.com --find-links https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html -boto3==1.42.75 \ - --hash=sha256:16bc657d16403ee8e11c8b6920c245629e37a36ea60352b919da566f82b4cb4c \ - --hash=sha256:3c7fd95a50c69271bd7707b7eda07dcfddb30e961a392613010f7ee81d91acb3 +boto3==1.43.3 \ + --hash=sha256:7c7777862ffc898f05efa566032bbabfe226dbb810e35ec11125817f128bc5c5 \ + --hash=sha256:fb9fe51849ef2a78198d582756fc06f14f7de27f73e0fa90275d6aa4171eb4d0 # via libneuronxla -botocore==1.42.75 \ - --hash=sha256:915e43b7ac8f50cf3dbc937ba713de5acb999ea48ad8fecd1589d92ad415f787 \ - --hash=sha256:95c8e716b6be903ee1601531caa4f50217400aa877c18fe9a2c3047d2945d477 +botocore==1.43.3 \ + --hash=sha256:eac6da0fffccf87888ebf4d89f0b2378218a707efa748cd955b838995e944695 \ + --hash=sha256:ec0769eb0f7c5034856bb406a92698dbc02a3d4be0f78a384747106b161d8ea3 # via # boto3 # libneuronxla # s3transfer -certifi==2026.2.25 \ - --hash=sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa \ - --hash=sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7 +certifi==2026.4.22 \ + --hash=sha256:3cb2210c8f88ba2318d29b0388d1023c8492ff72ecdde4ebdaddbb13a31b1c4a \ + --hash=sha256:8d455352a37b71bf76a79caa83a3d6c25afee4a385d632127b6afb3963f1c580 # via requests -charset-normalizer==3.4.6 \ - --hash=sha256:06a7e86163334edfc5d20fe104db92fcd666e5a5df0977cb5680a506fe26cc8e \ - --hash=sha256:0c173ce3a681f309f31b87125fecec7a5d1347261ea11ebbb856fa6006b23c8c \ - --hash=sha256:0e28d62a8fc7a1fa411c43bd65e346f3bce9716dc51b897fbe930c5987b402d5 \ - --hash=sha256:0e901eb1049fdb80f5bd11ed5ea1e498ec423102f7a9b9e4645d5b8204ff2815 \ - --hash=sha256:11afb56037cbc4b1555a34dd69151e8e069bee82e613a73bef6e714ce733585f \ - --hash=sha256:150b8ce8e830eb7ccb029ec9ca36022f756986aaaa7956aad6d9ec90089338c0 \ - --hash=sha256:172985e4ff804a7ad08eebec0a1640ece87ba5041d565fff23c8f99c1f389484 \ - --hash=sha256:197c1a244a274bb016dd8b79204850144ef77fe81c5b797dc389327adb552407 \ - --hash=sha256:1ae6b62897110aa7c79ea2f5dd38d1abca6db663687c0b1ad9aed6f6bae3d9d6 \ - --hash=sha256:1cf0a70018692f85172348fe06d3a4b63f94ecb055e13a00c644d368eb82e5b8 \ - --hash=sha256:1ed80ff870ca6de33f4d953fda4d55654b9a2b340ff39ab32fa3adbcd718f264 \ - --hash=sha256:22c6f0c2fbc31e76c3b8a86fba1a56eda6166e238c29cdd3d14befdb4a4e4815 \ - --hash=sha256:231d4da14bcd9301310faf492051bee27df11f2bc7549bc0bb41fef11b82daa2 \ - --hash=sha256:259695e2ccc253feb2a016303543d691825e920917e31f894ca1a687982b1de4 \ - --hash=sha256:2a24157fa36980478dd1770b585c0f30d19e18f4fb0c47c13aa568f871718579 \ - --hash=sha256:2b1a63e8224e401cafe7739f77efd3f9e7f5f2026bda4aead8e59afab537784f \ - --hash=sha256:2bd9d128ef93637a5d7a6af25363cf5dec3fa21cf80e68055aad627f280e8afa \ - --hash=sha256:2e1d8ca8611099001949d1cdfaefc510cf0f212484fe7c565f735b68c78c3c95 \ - --hash=sha256:2ef7fedc7a6ecbe99969cd09632516738a97eeb8bd7258bf8a0f23114c057dab \ - --hash=sha256:2f7fdd9b6e6c529d6a2501a2d36b240109e78a8ceaef5687cfcfa2bbe671d297 \ - --hash=sha256:30f445ae60aad5e1f8bdbb3108e39f6fbc09f4ea16c815c66578878325f8f15a \ - --hash=sha256:31215157227939b4fb3d740cd23fe27be0439afef67b785a1eb78a3ae69cba9e \ - --hash=sha256:34315ff4fc374b285ad7f4a0bf7dcbfe769e1b104230d40f49f700d4ab6bbd84 \ - --hash=sha256:3516bbb8d42169de9e61b8520cbeeeb716f12f4ecfe3fd30a9919aa16c806ca8 \ - --hash=sha256:3778fd7d7cd04ae8f54651f4a7a0bd6e39a0cf20f801720a4c21d80e9b7ad6b0 \ - --hash=sha256:39f5068d35621da2881271e5c3205125cc456f54e9030d3f723288c873a71bf9 \ - --hash=sha256:404a1e552cf5b675a87f0651f8b79f5f1e6fd100ee88dc612f89aa16abd4486f \ - --hash=sha256:419a9d91bd238052642a51938af8ac05da5b3343becde08d5cdeab9046df9ee1 \ - --hash=sha256:423fb7e748a08f854a08a222b983f4df1912b1daedce51a72bd24fe8f26a1843 \ - --hash=sha256:4482481cb0572180b6fd976a4d5c72a30263e98564da68b86ec91f0fe35e8565 \ - --hash=sha256:461598cd852bfa5a61b09cae2b1c02e2efcd166ee5516e243d540ac24bfa68a7 \ - --hash=sha256:47955475ac79cc504ef2704b192364e51d0d473ad452caedd0002605f780101c \ - --hash=sha256:48696db7f18afb80a068821504296eb0787d9ce239b91ca15059d1d3eaacf13b \ - --hash=sha256:4be9f4830ba8741527693848403e2c457c16e499100963ec711b1c6f2049b7c7 \ - --hash=sha256:4d1d02209e06550bdaef34af58e041ad71b88e624f5d825519da3a3308e22687 \ - --hash=sha256:4f41da960b196ea355357285ad1316a00099f22d0929fe168343b99b254729c9 \ - --hash=sha256:517ad0e93394ac532745129ceabdf2696b609ec9f87863d337140317ebce1c14 \ - --hash=sha256:51fb3c322c81d20567019778cb5a4a6f2dc1c200b886bc0d636238e364848c89 \ - --hash=sha256:5273b9f0b5835ff0350c0828faea623c68bfa65b792720c453e22b25cc72930f \ - --hash=sha256:530d548084c4a9f7a16ed4a294d459b4f229db50df689bfe92027452452943a0 \ - --hash=sha256:530e8cebeea0d76bdcf93357aa5e41336f48c3dc709ac52da2bb167c5b8271d9 \ - --hash=sha256:54fae94be3d75f3e573c9a1b5402dc593de19377013c9a0e4285e3d402dd3a2a \ - --hash=sha256:572d7c822caf521f0525ba1bce1a622a0b85cf47ffbdae6c9c19e3b5ac3c4389 \ - --hash=sha256:58c948d0d086229efc484fe2f30c2d382c86720f55cd9bc33591774348ad44e0 \ - --hash=sha256:5d11595abf8dd942a77883a39d81433739b287b6aa71620f15164f8096221b30 \ - --hash=sha256:5f8ddd609f9e1af8c7bd6e2aca279c931aefecd148a14402d4e368f3171769fd \ - --hash=sha256:5feb91325bbceade6afab43eb3b508c63ee53579fe896c77137ded51c6b6958e \ - --hash=sha256:60c74963d8350241a79cb8feea80e54d518f72c26db618862a8f53e5023deaf9 \ - --hash=sha256:613f19aa6e082cf96e17e3ffd89383343d0d589abda756b7764cf78361fd41dc \ - --hash=sha256:659a1e1b500fac8f2779dd9e1570464e012f43e580371470b45277a27baa7532 \ - --hash=sha256:695f5c2823691a25f17bc5d5ffe79fa90972cc34b002ac6c843bb8a1720e950d \ - --hash=sha256:69dd852c2f0ad631b8b60cfbe25a28c0058a894de5abb566619c205ce0550eae \ - --hash=sha256:6cceb5473417d28edd20c6c984ab6fee6c6267d38d906823ebfe20b03d607dc2 \ - --hash=sha256:71be7e0e01753a89cf024abf7ecb6bca2c81738ead80d43004d9b5e3f1244e64 \ - --hash=sha256:74119174722c4349af9708993118581686f343adc1c8c9c007d59be90d077f3f \ - --hash=sha256:74a2e659c7ecbc73562e2a15e05039f1e22c75b7c7618b4b574a3ea9118d1557 \ - --hash=sha256:7504e9b7dc05f99a9bbb4525c67a2c155073b44d720470a148b34166a69c054e \ - --hash=sha256:79090741d842f564b1b2827c0b82d846405b744d31e84f18d7a7b41c20e473ff \ - --hash=sha256:7a6967aaf043bceabab5412ed6bd6bd26603dae84d5cb75bf8d9a74a4959d398 \ - --hash=sha256:7bda6eebafd42133efdca535b04ccb338ab29467b3f7bf79569883676fc628db \ - --hash=sha256:7edbed096e4a4798710ed6bc75dcaa2a21b68b6c356553ac4823c3658d53743a \ - --hash=sha256:7f9019c9cb613f084481bd6a100b12e1547cf2efe362d873c2e31e4035a6fa43 \ - --hash=sha256:802168e03fba8bbc5ce0d866d589e4b1ca751d06edee69f7f3a19c5a9fe6b597 \ - --hash=sha256:80d0a5615143c0b3225e5e3ef22c8d5d51f3f72ce0ea6fb84c943546c7b25b6c \ - --hash=sha256:82060f995ab5003a2d6e0f4ad29065b7672b6593c8c63559beefe5b443242c3e \ - --hash=sha256:836ab36280f21fc1a03c99cd05c6b7af70d2697e374c7af0b61ed271401a72a2 \ - --hash=sha256:8761ac29b6c81574724322a554605608a9960769ea83d2c73e396f3df896ad54 \ - --hash=sha256:87725cfb1a4f1f8c2fc9890ae2f42094120f4b44db9360be5d99a4c6b0e03a9e \ - --hash=sha256:899d28f422116b08be5118ef350c292b36fc15ec2daeb9ea987c89281c7bb5c4 \ - --hash=sha256:8bc5f0687d796c05b1e28ab0d38a50e6309906ee09375dd3aff6a9c09dd6e8f4 \ - --hash=sha256:8bea55c4eef25b0b19a0337dc4e3f9a15b00d569c77211fa8cde38684f234fb7 \ - --hash=sha256:8e5a94886bedca0f9b78fecd6afb6629142fd2605aa70a125d49f4edc6037ee6 \ - --hash=sha256:90ca27cd8da8118b18a52d5f547859cc1f8354a00cd1e8e5120df3e30d6279e5 \ - --hash=sha256:92734d4d8d187a354a556626c221cd1a892a4e0802ccb2af432a1d85ec012194 \ - --hash=sha256:947cf925bc916d90adba35a64c82aace04fa39b46b52d4630ece166655905a69 \ - --hash=sha256:95b52c68d64c1878818687a473a10547b3292e82b6f6fe483808fb1468e2f52f \ - --hash=sha256:97d0235baafca5f2b09cf332cc275f021e694e8362c6bb9c96fc9a0eb74fc316 \ - --hash=sha256:9ca4c0b502ab399ef89248a2c84c54954f77a070f28e546a85e91da627d1301e \ - --hash=sha256:9cc4fc6c196d6a8b76629a70ddfcd4635a6898756e2d9cac5565cf0654605d73 \ - --hash=sha256:9cc6e6d9e571d2f863fa77700701dae73ed5f78881efc8b3f9a4398772ff53e8 \ - --hash=sha256:a056d1ad2633548ca18ffa2f85c202cfb48b68615129143915b8dc72a806a923 \ - --hash=sha256:a26611d9987b230566f24a0a125f17fe0de6a6aff9f25c9f564aaa2721a5fb88 \ - --hash=sha256:a4474d924a47185a06411e0064b803c68be044be2d60e50e8bddcc2649957c1f \ - --hash=sha256:a4ea868bc28109052790eb2b52a9ab33f3aa7adc02f96673526ff47419490e21 \ - --hash=sha256:a9e68c9d88823b274cf1e72f28cb5dc89c990edf430b0bfd3e2fb0785bfeabf4 \ - --hash=sha256:aa9cccf4a44b9b62d8ba8b4dd06c649ba683e4bf04eea606d2e94cfc2d6ff4d6 \ - --hash=sha256:ab30e5e3e706e3063bc6de96b118688cb10396b70bb9864a430f67df98c61ecc \ - --hash=sha256:ac2393c73378fea4e52aa56285a3d64be50f1a12395afef9cce47772f60334c2 \ - --hash=sha256:ad8faf8df23f0378c6d527d8b0b15ea4a2e23c89376877c598c4870d1b2c7866 \ - --hash=sha256:b35b200d6a71b9839a46b9b7fff66b6638bb52fc9658aa58796b0326595d3021 \ - --hash=sha256:b3694e3f87f8ac7ce279d4355645b3c878d24d1424581b46282f24b92f5a4ae2 \ - --hash=sha256:b4ff1d35e8c5bd078be89349b6f3a845128e685e751b6ea1169cf2160b344c4d \ - --hash=sha256:bbc8c8650c6e51041ad1be191742b8b421d05bbd3410f43fa2a00c8db87678e8 \ - --hash=sha256:bc72863f4d9aba2e8fd9085e63548a324ba706d2ea2c83b260da08a59b9482de \ - --hash=sha256:bf625105bb9eef28a56a943fec8c8a98aeb80e7d7db99bd3c388137e6eb2d237 \ - --hash=sha256:c2274ca724536f173122f36c98ce188fd24ce3dad886ec2b7af859518ce008a4 \ - --hash=sha256:c45a03a4c69820a399f1dda9e1d8fbf3562eda46e7720458180302021b08f778 \ - --hash=sha256:c8ae56368f8cc97c7e40a7ee18e1cedaf8e780cd8bc5ed5ac8b81f238614facb \ - --hash=sha256:c907cdc8109f6c619e6254212e794d6548373cc40e1ec75e6e3823d9135d29cc \ - --hash=sha256:ca0276464d148c72defa8bb4390cce01b4a0e425f3b50d1435aa6d7a18107602 \ - --hash=sha256:cd5e2801c89992ed8c0a3f0293ae83c159a60d9a5d685005383ef4caca77f2c4 \ - --hash=sha256:d08ec48f0a1c48d75d0356cea971921848fb620fdeba805b28f937e90691209f \ - --hash=sha256:d1a2ee9c1499fc8f86f4521f27a973c914b211ffa87322f4ee33bb35392da2c5 \ - --hash=sha256:d5f5d1e9def3405f60e3ca8232d56f35c98fb7bf581efcc60051ebf53cb8b611 \ - --hash=sha256:d60377dce4511655582e300dc1e5a5f24ba0cb229005a1d5c8d0cb72bb758ab8 \ - --hash=sha256:d73beaac5e90173ac3deb9928a74763a6d230f494e4bfb422c217a0ad8e629bf \ - --hash=sha256:d7de2637729c67d67cf87614b566626057e95c303bc0a55ffe391f5205e7003d \ - --hash=sha256:dad6e0f2e481fffdcf776d10ebee25e0ef89f16d691f1e5dee4b586375fdc64b \ - --hash=sha256:dda86aba335c902b6149a02a55b38e96287157e609200811837678214ba2b1db \ - --hash=sha256:df01808ee470038c3f8dc4f48620df7225c49c2d6639e38f96e6d6ac6e6f7b0e \ - --hash=sha256:e1f6e2f00a6b8edb562826e4632e26d063ac10307e80f7461f7de3ad8ef3f077 \ - --hash=sha256:e25369dc110d58ddf29b949377a93e0716d72a24f62bad72b2b39f155949c1fd \ - --hash=sha256:e3c701e954abf6fc03a49f7c579cc80c2c6cc52525340ca3186c41d3f33482ef \ - --hash=sha256:e5bcc1a1ae744e0bb59641171ae53743760130600da8db48cbb6e4918e186e4e \ - --hash=sha256:e68c14b04827dd76dcbd1aeea9e604e3e4b78322d8faf2f8132c7138efa340a8 \ - --hash=sha256:e8aeb10fcbe92767f0fa69ad5a72deca50d0dca07fbde97848997d778a50c9fe \ - --hash=sha256:e985a16ff513596f217cee86c21371b8cd011c0f6f056d0920aa2d926c544058 \ - --hash=sha256:ecbbd45615a6885fe3240eb9db73b9e62518b611850fdf8ab08bd56de7ad2b17 \ - --hash=sha256:ee4ec14bc1680d6b0afab9aea2ef27e26d2024f18b24a2d7155a52b60da7e833 \ - --hash=sha256:ef5960d965e67165d75b7c7ffc60a83ec5abfc5c11b764ec13ea54fbef8b4421 \ - --hash=sha256:f0cdaecd4c953bfae0b6bb64910aaaca5a424ad9c72d85cb88417bb9814f7550 \ - --hash=sha256:f1ce721c8a7dfec21fcbdfe04e8f68174183cf4e8188e0645e92aa23985c57ff \ - --hash=sha256:f50498891691e0864dc3da965f340fada0771f6142a378083dc4608f4ea513e2 \ - --hash=sha256:f5ea69428fa1b49573eef0cc44a1d43bebd45ad0c611eb7d7eac760c7ae771bc \ - --hash=sha256:f61aa92e4aad0be58eb6eb4e0c21acf32cf8065f4b2cae5665da756c4ceef982 \ - --hash=sha256:f6e4333fb15c83f7d1482a76d45a0818897b3d33f00efd215528ff7c51b8e35d \ - --hash=sha256:f820f24b09e3e779fe84c3c456cb4108a7aa639b0d1f02c28046e11bfcd088ed \ - --hash=sha256:f98059e4fcd3e3e4e2d632b7cf81c2faae96c43c60b569e9c621468082f1d104 \ - --hash=sha256:fcce033e4021347d80ed9c66dcf1e7b1546319834b74445f561d2e2221de5659 +charset-normalizer==3.4.7 \ + --hash=sha256:007d05ec7321d12a40227aae9e2bc6dca73f3cb21058999a1df9e193555a9dcc \ + --hash=sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c \ + --hash=sha256:07d9e39b01743c3717745f4c530a6349eadbfa043c7577eef86c502c15df2c67 \ + --hash=sha256:08e721811161356f97b4059a9ba7bafb23ea5ee2255402c42881c214e173c6b4 \ + --hash=sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0 \ + --hash=sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c \ + --hash=sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5 \ + --hash=sha256:12a6fff75f6bc66711b73a2f0addfc4c8c15a20e805146a02d147a318962c444 \ + --hash=sha256:12d8baf840cc7889b37c7c770f478adea7adce3dcb3944d02ec87508e2dcf153 \ + --hash=sha256:14265bfe1f09498b9d8ec91e9ec9fa52775edf90fcbde092b25f4a33d444fea9 \ + --hash=sha256:16d971e29578a5e97d7117866d15889a4a07befe0e87e703ed63cd90cb348c01 \ + --hash=sha256:177a0ba5f0211d488e295aaf82707237e331c24788d8d76c96c5a41594723217 \ + --hash=sha256:1a87ca9d5df6fe460483d9a5bbf2b18f620cbed41b432e2bddb686228282d10b \ + --hash=sha256:1c2a768fdd44ee4a9339a9b0b130049139b8ce3c01d2ce09f67f5a68048d477c \ + --hash=sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a \ + --hash=sha256:1dc8b0ea451d6e69735094606991f32867807881400f808a106ee1d963c46a83 \ + --hash=sha256:1efde3cae86c8c273f1eb3b287be7d8499420cf2fe7585c41d370d3e790054a5 \ + --hash=sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7 \ + --hash=sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb \ + --hash=sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c \ + --hash=sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1 \ + --hash=sha256:2cd4a60d0e2fb04537162c62bbbb4182f53541fe0ede35cdf270a1c1e723cc42 \ + --hash=sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab \ + --hash=sha256:2fe249cb4651fd12605b7288b24751d8bfd46d35f12a20b1ba33dea122e690df \ + --hash=sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e \ + --hash=sha256:320ade88cfb846b8cd6b4ddf5ee9e80ee0c1f52401f2456b84ae1ae6a1a5f207 \ + --hash=sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18 \ + --hash=sha256:36836d6ff945a00b88ba1e4572d721e60b5b8c98c155d465f56ad19d68f23734 \ + --hash=sha256:38c0109396c4cfc574d502df99742a45c72c08eff0a36158b6f04000043dbf38 \ + --hash=sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110 \ + --hash=sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18 \ + --hash=sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44 \ + --hash=sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d \ + --hash=sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48 \ + --hash=sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e \ + --hash=sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5 \ + --hash=sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d \ + --hash=sha256:4e5163c14bffd570ef2affbfdd77bba66383890797df43dc8b4cc7d6f500bf53 \ + --hash=sha256:511ef87c8aec0783e08ac18565a16d435372bc1ac25a91e6ac7f5ef2b0bff790 \ + --hash=sha256:532bc9bf33a68613fd7d65e4b1c71a6a38d7d42604ecf239c77392e9b4e8998c \ + --hash=sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b \ + --hash=sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116 \ + --hash=sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d \ + --hash=sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10 \ + --hash=sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6 \ + --hash=sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2 \ + --hash=sha256:6370e8686f662e6a3941ee48ed4742317cafbe5707e36406e9df792cdb535776 \ + --hash=sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a \ + --hash=sha256:65bcd23054beab4d166035cabbc868a09c1a49d1efe458fe8e4361215df40265 \ + --hash=sha256:66671f93accb62ed07da56613636f3641f1a12c13046ce91ffc923721f23c008 \ + --hash=sha256:6696b7688f54f5af4462118f0bfa7c1621eeb87154f77fa04b9295ce7a8f2943 \ + --hash=sha256:6785f414ae0f3c733c437e0f3929197934f526d19dfaa75e18fdb4f94c6fb374 \ + --hash=sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246 \ + --hash=sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e \ + --hash=sha256:6e0d51f618228538a3e8f46bd246f87a6cd030565e015803691603f55e12afb5 \ + --hash=sha256:6ed74185b2db44f41ef35fd1617c5888e59792da9bbc9190d6c7300617182616 \ + --hash=sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15 \ + --hash=sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41 \ + --hash=sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960 \ + --hash=sha256:750e02e074872a3fad7f233b47734166440af3cdea0add3e95163110816d6752 \ + --hash=sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e \ + --hash=sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72 \ + --hash=sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7 \ + --hash=sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8 \ + --hash=sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b \ + --hash=sha256:813c0e0132266c08eb87469a642cb30aaff57c5f426255419572aaeceeaa7bf4 \ + --hash=sha256:82b271f5137d07749f7bf32f70b17ab6eaabedd297e75dce75081a24f76eb545 \ + --hash=sha256:84c018e49c3bf790f9c2771c45e9313a08c2c2a6342b162cd650258b57817706 \ + --hash=sha256:8751d2787c9131302398b11e6c8068053dcb55d5a8964e114b6e196cf16cb366 \ + --hash=sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb \ + --hash=sha256:87fad7d9ba98c86bcb41b2dc8dbb326619be2562af1f8ff50776a39e55721c5a \ + --hash=sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e \ + --hash=sha256:8e385e4267ab76874ae30db04c627faaaf0b509e1ccc11a95b3fc3e83f855c00 \ + --hash=sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f \ + --hash=sha256:94e1885b270625a9a828c9793b4d52a64445299baa1fea5a173bf1d3dd9a1a5a \ + --hash=sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1 \ + --hash=sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66 \ + --hash=sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356 \ + --hash=sha256:a6c5863edfbe888d9eff9c8b8087354e27618d9da76425c119293f11712a6319 \ + --hash=sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4 \ + --hash=sha256:adb2597b428735679446b46c8badf467b4ca5f5056aae4d51a19f9570301b1ad \ + --hash=sha256:ae196f021b5e7c78e918242d217db021ed2a6ace2bc6ae94c0fc596221c7f58d \ + --hash=sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5 \ + --hash=sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7 \ + --hash=sha256:aef65cd602a6d0e0ff6f9930fcb1c8fec60dd2cfcb6facaf4bdb0e5873042db0 \ + --hash=sha256:af21eb4409a119e365397b2adbaca4c9ccab56543a65d5dbd9f920d6ac29f686 \ + --hash=sha256:b14b2d9dac08e28bb8046a1a0434b1750eb221c8f5b87a68f4fa11a6f97b5e34 \ + --hash=sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49 \ + --hash=sha256:bb8cc7534f51d9a017b93e3e85b260924f909601c3df002bcdb58ddb4dc41a5c \ + --hash=sha256:bc17a677b21b3502a21f66a8cc64f5bfad4df8a0b8434d661666f8ce90ac3af1 \ + --hash=sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e \ + --hash=sha256:bd9b23791fe793e4968dba0c447e12f78e425c59fc0e3b97f6450f4781f3ee60 \ + --hash=sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0 \ + --hash=sha256:c0f081d69a6e58272819b70288d3221a6ee64b98df852631c80f293514d3b274 \ + --hash=sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d \ + --hash=sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0 \ + --hash=sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae \ + --hash=sha256:c593052c465475e64bbfe5dbd81680f64a67fdc752c56d7a0ae205dc8aeefe0f \ + --hash=sha256:cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d \ + --hash=sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe \ + --hash=sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3 \ + --hash=sha256:cf29836da5119f3c8a8a70667b0ef5fdca3bb12f80fd06487cfa575b3909b393 \ + --hash=sha256:d4a48e5b3c2a489fae013b7589308a40146ee081f6f509e047e0e096084ceca1 \ + --hash=sha256:d560742f3c0d62afaccf9f41fe485ed69bd7661a241f86a3ef0f0fb8b1a397af \ + --hash=sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44 \ + --hash=sha256:d61f00a0869d77422d9b2aba989e2d24afa6ffd552af442e0e58de4f35ea6d00 \ + --hash=sha256:d635aab80466bc95771bb78d5370e74d36d1fe31467b6b29b8b57b2a3cd7d22c \ + --hash=sha256:dca4bbc466a95ba9c0234ef56d7dd9509f63da22274589ebd4ed7f1f4d4c54e3 \ + --hash=sha256:dd915403e231e6b1809fe9b6d9fc55cf8fb5e02765ac625d9cd623342a7905d7 \ + --hash=sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd \ + --hash=sha256:e060d01aec0a910bdccb8be71faf34e7799ce36950f8294c8bf612cba65a2c9e \ + --hash=sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b \ + --hash=sha256:e17b8d5d6a8c47c85e68ca8379def1303fd360c3e22093a807cd34a71cd082b8 \ + --hash=sha256:e5f4d355f0a2b1a31bc3edec6795b46324349c9cb25eed068049e4f472fb4259 \ + --hash=sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859 \ + --hash=sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46 \ + --hash=sha256:e80c8378d8f3d83cd3164da1ad2df9e37a666cdde7b1cb2298ed0b558064be30 \ + --hash=sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b \ + --hash=sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46 \ + --hash=sha256:ed065083d0898c9d5b4bbec7b026fd755ff7454e6e8b73a67f8c744b13986e24 \ + --hash=sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a \ + --hash=sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24 \ + --hash=sha256:f22dec1690b584cea26fade98b2435c132c1b5f68e39f5a0b7627cd7ae31f1dc \ + --hash=sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215 \ + --hash=sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063 \ + --hash=sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832 \ + --hash=sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6 \ + --hash=sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79 \ + --hash=sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464 # via requests +contourpy==1.3.3 \ + --hash=sha256:023b44101dfe49d7d53932be418477dba359649246075c996866106da069af69 \ + --hash=sha256:07ce5ed73ecdc4a03ffe3e1b3e3c1166db35ae7584be76f65dbbe28a7791b0cc \ + --hash=sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880 \ + --hash=sha256:0bf67e0e3f482cb69779dd3061b534eb35ac9b17f163d851e2a547d56dba0a3a \ + --hash=sha256:0c1fc238306b35f246d61a1d416a627348b5cf0648648a031e14bb8705fcdfe8 \ + --hash=sha256:13b68d6a62db8eafaebb8039218921399baf6e47bf85006fd8529f2a08ef33fc \ + --hash=sha256:15ff10bfada4bf92ec8b31c62bf7c1834c244019b4a33095a68000d7075df470 \ + --hash=sha256:177fb367556747a686509d6fef71d221a4b198a3905fe824430e5ea0fda54eb5 \ + --hash=sha256:1cadd8b8969f060ba45ed7c1b714fe69185812ab43bd6b86a9123fe8f99c3263 \ + --hash=sha256:1fd43c3be4c8e5fd6e4f2baeae35ae18176cf2e5cced681cca908addf1cdd53b \ + --hash=sha256:22e9b1bd7a9b1d652cd77388465dc358dafcd2e217d35552424aa4f996f524f5 \ + --hash=sha256:23416f38bfd74d5d28ab8429cc4d63fa67d5068bd711a85edb1c3fb0c3e2f381 \ + --hash=sha256:283edd842a01e3dcd435b1c5116798d661378d83d36d337b8dde1d16a5fc9ba3 \ + --hash=sha256:2a2a8b627d5cc6b7c41a4beff6c5ad5eb848c88255fda4a8745f7e901b32d8e4 \ + --hash=sha256:2b7e9480ffe2b0cd2e787e4df64270e3a0440d9db8dc823312e2c940c167df7e \ + --hash=sha256:322ab1c99b008dad206d406bb61d014cf0174df491ae9d9d0fac6a6fda4f977f \ + --hash=sha256:33c82d0138c0a062380332c861387650c82e4cf1747aaa6938b9b6516762e772 \ + --hash=sha256:348ac1f5d4f1d66d3322420f01d42e43122f43616e0f194fc1c9f5d830c5b286 \ + --hash=sha256:3519428f6be58431c56581f1694ba8e50626f2dd550af225f82fb5f5814d2a42 \ + --hash=sha256:3c30273eb2a55024ff31ba7d052dde990d7d8e5450f4bbb6e913558b3d6c2301 \ + --hash=sha256:3d1a3799d62d45c18bafd41c5fa05120b96a28079f2393af559b843d1a966a77 \ + --hash=sha256:451e71b5a7d597379ef572de31eeb909a87246974d960049a9848c3bc6c41bf7 \ + --hash=sha256:459c1f020cd59fcfe6650180678a9993932d80d44ccde1fa1868977438f0b411 \ + --hash=sha256:4d00e655fcef08aba35ec9610536bfe90267d7ab5ba944f7032549c55a146da1 \ + --hash=sha256:4debd64f124ca62069f313a9cb86656ff087786016d76927ae2cf37846b006c9 \ + --hash=sha256:4feffb6537d64b84877da813a5c30f1422ea5739566abf0bd18065ac040e120a \ + --hash=sha256:50ed930df7289ff2a8d7afeb9603f8289e5704755c7e5c3bbd929c90c817164b \ + --hash=sha256:51e79c1f7470158e838808d4a996fa9bac72c498e93d8ebe5119bc1e6becb0db \ + --hash=sha256:556dba8fb6f5d8742f2923fe9457dbdd51e1049c4a43fd3986a0b14a1d815fc6 \ + --hash=sha256:598c3aaece21c503615fd59c92a3598b428b2f01bfb4b8ca9c4edeecc2438620 \ + --hash=sha256:5ed3657edf08512fc3fe81b510e35c2012fbd3081d2e26160f27ca28affec989 \ + --hash=sha256:626d60935cf668e70a5ce6ff184fd713e9683fb458898e4249b63be9e28286ea \ + --hash=sha256:644a6853d15b2512d67881586bd03f462c7ab755db95f16f14d7e238f2852c67 \ + --hash=sha256:655456777ff65c2c548b7c454af9c6f33f16c8884f11083244b5819cc214f1b5 \ + --hash=sha256:66c8a43a4f7b8df8b71ee1840e4211a3c8d93b214b213f590e18a1beca458f7d \ + --hash=sha256:6afc576f7b33cf00996e5c1102dc2a8f7cc89e39c0b55df93a0b78c1bd992b36 \ + --hash=sha256:6c3d53c796f8647d6deb1abe867daeb66dcc8a97e8455efa729516b997b8ed99 \ + --hash=sha256:709a48ef9a690e1343202916450bc48b9e51c049b089c7f79a267b46cffcdaa1 \ + --hash=sha256:70f9aad7de812d6541d29d2bbf8feb22ff7e1c299523db288004e3157ff4674e \ + --hash=sha256:8153b8bfc11e1e4d75bcb0bff1db232f9e10b274e0929de9d608027e0d34ff8b \ + --hash=sha256:87acf5963fc2b34825e5b6b048f40e3635dd547f590b04d2ab317c2619ef7ae8 \ + --hash=sha256:88df9880d507169449d434c293467418b9f6cbe82edd19284aa0409e7fdb933d \ + --hash=sha256:929ddf8c4c7f348e4c0a5a3a714b5c8542ffaa8c22954862a46ca1813b667ee7 \ + --hash=sha256:92d9abc807cf7d0e047b95ca5d957cf4792fcd04e920ca70d48add15c1a90ea7 \ + --hash=sha256:95b181891b4c71de4bb404c6621e7e2390745f887f2a026b2d99e92c17892339 \ + --hash=sha256:9e999574eddae35f1312c2b4b717b7885d4edd6cb46700e04f7f02db454e67c1 \ + --hash=sha256:a15459b0f4615b00bbd1e91f1b9e19b7e63aea7483d03d804186f278c0af2659 \ + --hash=sha256:a22738912262aa3e254e4f3cb079a95a67132fc5a063890e224393596902f5a4 \ + --hash=sha256:ab2fd90904c503739a75b7c8c5c01160130ba67944a7b77bbf36ef8054576e7f \ + --hash=sha256:ab3074b48c4e2cf1a960e6bbeb7f04566bf36b1861d5c9d4d8ac04b82e38ba20 \ + --hash=sha256:afe5a512f31ee6bd7d0dda52ec9864c984ca3d66664444f2d72e0dc4eb832e36 \ + --hash=sha256:b08a32ea2f8e42cf1d4be3169a98dd4be32bafe4f22b6c4cb4ba810fa9e5d2cb \ + --hash=sha256:b20c7c9a3bf701366556e1b1984ed2d0cedf999903c51311417cf5f591d8c78d \ + --hash=sha256:b2e8faa0ed68cb29af51edd8e24798bb661eac3bd9f65420c1887b6ca89987c8 \ + --hash=sha256:b7301b89040075c30e5768810bc96a8e8d78085b47d8be6e4c3f5a0b4ed478a0 \ + --hash=sha256:b7448cb5a725bb1e35ce88771b86fba35ef418952474492cf7c764059933ff8b \ + --hash=sha256:ca0fdcd73925568ca027e0b17ab07aad764be4706d0a925b89227e447d9737b7 \ + --hash=sha256:ca658cd1a680a5c9ea96dc61cdbae1e85c8f25849843aa799dfd3cb370ad4fbe \ + --hash=sha256:cbedb772ed74ff5be440fa8eee9bd49f64f6e3fc09436d9c7d8f1c287b121d77 \ + --hash=sha256:cd5dfcaeb10f7b7f9dc8941717c6c2ade08f587be2226222c12b25f0483ed497 \ + --hash=sha256:cf9022ef053f2694e31d630feaacb21ea24224be1c3ad0520b13d844274614fd \ + --hash=sha256:d002b6f00d73d69333dac9d0b8d5e84d9724ff9ef044fd63c5986e62b7c9e1b1 \ + --hash=sha256:d06bb1f751ba5d417047db62bca3c8fde202b8c11fb50742ab3ab962c81e8216 \ + --hash=sha256:d304906ecc71672e9c89e87c4675dc5c2645e1f4269a5063b99b0bb29f232d13 \ + --hash=sha256:e4e6b05a45525357e382909a4c1600444e2a45b4795163d3b22669285591c1ae \ + --hash=sha256:e74a9a0f5e3fff48fb5a7f2fd2b9b70a3fe014a67522f79b7cca4c0c7e43c9ae \ + --hash=sha256:ea37e7b45949df430fe649e5de8351c423430046a2af20b1c1961cae3afcda77 \ + --hash=sha256:f64836de09927cba6f79dcd00fdd7d5329f3fccc633468507079c829ca4db4e3 \ + --hash=sha256:fd6ec6be509c787f1caf6b247f0b1ca598bef13f4ddeaa126b7658215529ba0f \ + --hash=sha256:fd907ae12cd483cd83e414b12941c632a969171bf90fc937d0c9f268a31cafff \ + --hash=sha256:fd914713266421b7536de2bfa8181aa8c699432b6763a0ea64195ebe28bff6a9 \ + --hash=sha256:fde6c716d51c04b1c25d0b90364d0be954624a0ee9d60e23e850e8d48353d07a + # via matplotlib +cycler==0.12.1 \ + --hash=sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30 \ + --hash=sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c + # via matplotlib ec2-metadata==2.17.0 \ --hash=sha256:a696a899cb8ee1f0101eb9ec7bc2050e1ffc8574ee4cd748a52d9c748fb19d9e \ --hash=sha256:ad98a5afd8f4f49f688e4899dc505257da3237348761eceeb0fc7fc7d00cc90d # via neuronx-cc -idna==3.11 \ - --hash=sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea \ - --hash=sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902 +einops==0.8.1 \ + --hash=sha256:919387eb55330f5757c6bea9165c5ff5cfe63a642682ea788a6d472576d81737 \ + --hash=sha256:de5d960a7a761225532e0f1959e5315ebeafc0cd43394732f103ca44b9837e84 + # via nki +fonttools==4.62.1 \ + --hash=sha256:0aa72c43a601cfa9273bb1ae0518f1acadc01ee181a6fc60cd758d7fdadffc04 \ + --hash=sha256:0b3ae47e8636156a9accff64c02c0924cbebad62854c4a6dbdc110cd5b4b341a \ + --hash=sha256:12859ff0b47dd20f110804c3e0d0970f7b832f561630cd879969011541a464a9 \ + --hash=sha256:149f7d84afca659d1a97e39a4778794a2f83bf344c5ee5134e09995086cc2392 \ + --hash=sha256:1596aeaddf7f78e21e68293c011316a25267b3effdaccaf4d59bc9159d681b82 \ + --hash=sha256:19177c8d96c7c36359266e571c5173bcee9157b59cfc8cb0153c5673dc5a3a7d \ + --hash=sha256:1c5c25671ce8805e0d080e2ffdeca7f1e86778c5cbfbeae86d7f866d8830517b \ + --hash=sha256:1eecc128c86c552fb963fe846ca4e011b1be053728f798185a1687502f6d398e \ + --hash=sha256:268abb1cb221e66c014acc234e872b7870d8b5d4657a83a8f4205094c32d2416 \ + --hash=sha256:2d850f66830a27b0d498ee05adb13a3781637b1826982cd7e2b3789ef0cc71ae \ + --hash=sha256:2e7abd2b1e11736f58c1de27819e1955a53267c21732e78243fa2fa2e5c1e069 \ + --hash=sha256:403d28ce06ebfc547fbcb0cb8b7f7cc2f7a2d3e1a67ba9a34b14632df9e080f9 \ + --hash=sha256:40975849bac44fb0b9253d77420c6d8b523ac4dcdcefeff6e4d706838a5b80f7 \ + --hash=sha256:486f32c8047ccd05652aba17e4a8819a3a9d78570eb8a0e3b4503142947880ed \ + --hash=sha256:49a445d2f544ce4a69338694cad575ba97b9a75fff02720da0882d1a73f12800 \ + --hash=sha256:59b372b4f0e113d3746b88985f1c796e7bf830dd54b28374cd85c2b8acd7583e \ + --hash=sha256:5a648bde915fba9da05ae98856987ca91ba832949a9e2888b48c47ef8b96c5a9 \ + --hash=sha256:5f37df1cac61d906e7b836abe356bc2f34c99d4477467755c216b72aa3dc748b \ + --hash=sha256:6706d1cb1d5e6251a97ad3c1b9347505c5615c112e66047abbef0f8545fa30d1 \ + --hash=sha256:68959f5fc58ed4599b44aad161c2837477d7f35f5f79402d97439974faebfebe \ + --hash=sha256:6acb4109f8bee00fec985c8c7afb02299e35e9c94b57287f3ea542f28bd0b0a7 \ + --hash=sha256:7487782e2113861f4ddcc07c3436450659e3caa5e470b27dc2177cade2d8e7fd \ + --hash=sha256:7aa21ff53e28a9c2157acbc44e5b401149d3c9178107130e82d74ceb500e5056 \ + --hash=sha256:7bca7a1c1faf235ffe25d4f2e555246b4750220b38de8261d94ebc5ce8a23c23 \ + --hash=sha256:8d337fdd49a79b0d51c4da87bc38169d21c3abbf0c1aa9367eff5c6656fb6dae \ + --hash=sha256:8f8fca95d3bb3208f59626a4b0ea6e526ee51f5a8ad5d91821c165903e8d9260 \ + --hash=sha256:90365821debbd7db678809c7491ca4acd1e0779b9624cdc6ddaf1f31992bf974 \ + --hash=sha256:92bb00a947e666169c99b43753c4305fc95a890a60ef3aeb2a6963e07902cc87 \ + --hash=sha256:93c316e0f5301b2adbe6a5f658634307c096fd5aae60a5b3412e4f3e1728ab24 \ + --hash=sha256:942b03094d7edbb99bdf1ae7e9090898cad7bf9030b3d21f33d7072dbcb51a53 \ + --hash=sha256:9c125ffa00c3d9003cdaaf7f2c79e6e535628093e14b5de1dccb08859b680936 \ + --hash=sha256:9dde91633f77fa576879a0c76b1d89de373cae751a98ddf0109d54e173b40f14 \ + --hash=sha256:9e7863e10b3de72376280b515d35b14f5eeed639d1aa7824f4cf06779ec65e42 \ + --hash=sha256:a24decd24d60744ee8b4679d38e88b8303d86772053afc29b19d23bb8207803c \ + --hash=sha256:a5d8825e1140f04e6c99bb7d37a9e31c172f3bc208afbe02175339e699c710e1 \ + --hash=sha256:aa69d10ed420d8121118e628ad47d86e4caa79ba37f968597b958f6cceab7eca \ + --hash=sha256:ad5cca75776cd453b1b035b530e943334957ae152a36a88a320e779d61fc980c \ + --hash=sha256:b4e0fcf265ad26e487c56cb12a42dffe7162de708762db951e1b3f755319507d \ + --hash=sha256:b820fcb92d4655513d8402d5b219f94481c4443d825b4372c75a2072aa4b357a \ + --hash=sha256:bd13b7999d59c5eb1c2b442eb2d0c427cb517a0b7a1f5798fc5c9e003f5ff782 \ + --hash=sha256:bdfe592802ef939a0e33106ea4a318eeb17822c7ee168c290273cbd5fabd746c \ + --hash=sha256:c05557a78f8fa514da0f869556eeda40887a8abc77c76ee3f74cf241778afd5a \ + --hash=sha256:c22b1014017111c401469e3acc5433e6acf6ebcc6aa9efb538a533c800971c79 \ + --hash=sha256:c9b9e288b4da2f64fd6180644221749de651703e8d0c16bd4b719533a3a7d6e3 \ + --hash=sha256:d241cdc4a67b5431c6d7f115fdf63335222414995e3a1df1a41e1182acd4bcc7 \ + --hash=sha256:e54c75fd6041f1122476776880f7c3c3295ffa31962dc6ebe2543c00dca58b5d \ + --hash=sha256:e8514f4924375f77084e81467e63238b095abda5107620f49421c368a6017ed2 \ + --hash=sha256:ee91628c08e76f77b533d65feb3fbe6d9dad699f95be51cf0d022db94089cdc4 \ + --hash=sha256:ef46db46c9447103b8f3ff91e8ba009d5fe181b1920a83757a5762551e32bb68 \ + --hash=sha256:fa1d16210b6b10a826d71bed68dd9ec24a9e218d5a5e2797f37c573e7ec215ca + # via matplotlib +idna==3.13 \ + --hash=sha256:585ea8fe5d69b9181ec1afba340451fba6ba764af97026f92a91d4eef164a242 \ + --hash=sha256:892ea0cde124a99ce773decba204c5552b69c3c67ffd5f232eb7696135bc8bb3 # via requests islpy==2023.2.5 \ --hash=sha256:06446395028b88701fd84b685b1c82b9de493728906f96f576a4dca69d12dd1f \ @@ -206,13 +340,226 @@ jmespath==1.1.0 \ # via # boto3 # botocore -libneuronxla==2.2.15515.0+50c26cbd \ - --hash=sha256:770d6ed89be152c2dc777b042ccf66c7af7c71e3204521f0b84d65705a6958e2 +kiwisolver==1.5.0 \ + --hash=sha256:012b1eb16e28718fa782b5e61dc6f2da1f0792ca73bd05d54de6cb9561665fc9 \ + --hash=sha256:01808c6d15f4c3e8559595d6d1fe6411c68e4a3822b4b9972b44473b24f4e679 \ + --hash=sha256:0255a027391d52944eae1dbb5d4cc5903f57092f3674e8e544cdd2622826b3f0 \ + --hash=sha256:0b85aad90cea8ac6797a53b5d5f2e967334fa4d1149f031c4537569972596cb8 \ + --hash=sha256:0bf3acf1419fa93064a4c2189ac0b58e3be7872bf6ee6177b0d4c63dc4cea276 \ + --hash=sha256:0c50b89ffd3e1a911c69a1dd3de7173c0cd10b130f56222e57898683841e4f96 \ + --hash=sha256:0cbe94b69b819209a62cb27bdfa5dc2a8977d8de2f89dfd97ba4f53ed3af754e \ + --hash=sha256:0df54df7e686afa55e6f21fb86195224a6d9beb71d637e8d7920c95cf0f89aac \ + --hash=sha256:0e3aafb33aed7479377e5e9a82e9d4bf87063741fc99fc7ae48b0f16e32bdd6f \ + --hash=sha256:12e91c215a96e39f57989c8912ae761286ac5a9584d04030ceb3368a357f017a \ + --hash=sha256:1465387ac63576c3e125e5337a6892b9e99e0627d52317f3ca79e6930d889d15 \ + --hash=sha256:16b85d37c2cbb3253226d26e64663f755d88a03439a9c47df6246b35defbdfb7 \ + --hash=sha256:1b0feb50971481a2cc44d94e88bdb02cdd497618252ae226b8eb1201b957e368 \ + --hash=sha256:1d49a49ac4cbfb7c1375301cd1ec90169dfeae55ff84710d782260ce77a75a02 \ + --hash=sha256:1d9daea4ea6b9be74fe2f01f7fbade8d6ffab263e781274cffca0dba9be9eec9 \ + --hash=sha256:1dd9b0b119a350976a6d781e7278ec7aca0b201e1a9e2d23d9804afecb6ca681 \ + --hash=sha256:1f1489f769582498610e015a8ef2d36f28f505ab3096d0e16b4858a9ec214f57 \ + --hash=sha256:2517e24d7315eb51c10664cdb865195df38ab74456c677df67bb47f12d088a27 \ + --hash=sha256:295d9ffe712caa9f8a3081de8d32fc60191b4b51c76f02f951fd8407253528f4 \ + --hash=sha256:2a075bd7bd19c70cf67c8badfa36cf7c5d8de3c9ddb8420c51e10d9c50e94920 \ + --hash=sha256:32cc0a5365239a6ea0c6ed461e8838d053b57e397443c0ca894dcc8e388d4374 \ + --hash=sha256:332b4f0145c30b5f5ad9374881133e5aa64320428a57c2c2b61e9d891a51c2f3 \ + --hash=sha256:377815a8616074cabbf3f53354e1d040c35815a134e01d7614b7692e4bf8acfa \ + --hash=sha256:38f4a703656f493b0ad185211ccfca7f0386120f022066b018eb5296d8613e23 \ + --hash=sha256:3ac2360e93cb41be81121755c6462cff3beaa9967188c866e5fce5cf13170859 \ + --hash=sha256:3c4923e404d6bcd91b6779c009542e5647fef32e4a5d75e115e3bbac6f2335eb \ + --hash=sha256:3cdcb35dc9d807259c981a85531048ede628eabcffb3239adf3d17463518992d \ + --hash=sha256:41024ed50e44ab1a60d3fe0a9d15a4ccc9f5f2b1d814ff283c8d01134d5b81bc \ + --hash=sha256:413b820229730d358efd838ecbab79902fe97094565fdc80ddb6b0a18c18a581 \ + --hash=sha256:4432b835675f0ea7414aab3d37d119f7226d24869b7a829caeab49ebda407b0c \ + --hash=sha256:4db576bb8c3ef9365f8b40fe0f671644de6736ae2c27a2c62d7d8a1b4329f099 \ + --hash=sha256:4e7f886f47ab881692f278ae901039a234e4025a68e6dfab514263a0b1c4ae05 \ + --hash=sha256:4e9750bc21b886308024f8a54ccb9a2cc38ac9fa813bf4348434e3d54f337ff9 \ + --hash=sha256:5060731cc3ed12ca3a8b57acd4aeca5bbc2f49216dd0bec1650a1acd89486bcd \ + --hash=sha256:50847dca5d197fcbd389c805aa1a1cf32f25d2e7273dc47ab181a517666b68cc \ + --hash=sha256:5092eb5b1172947f57d6ea7d89b2f29650414e4293c47707eb499ec07a0ac796 \ + --hash=sha256:5124d1ea754509b09e53738ec185584cc609aae4a3b510aaf4ed6aa047ef9303 \ + --hash=sha256:51e8c4084897de9f05898c2c2a39af6318044ae969d46ff7a34ed3f96274adca \ + --hash=sha256:530a3fd64c87cffa844d4b6b9768774763d9caa299e9b75d8eca6a4423b31314 \ + --hash=sha256:56fa888f10d0f367155e76ce849fa1166fc9730d13bd2d65a2aa13b6f5424489 \ + --hash=sha256:58f812017cd2985c21fbffb4864d59174d4903dd66fa23815e74bbc7a0e2dd57 \ + --hash=sha256:59cd8683f575d96df5bb48f6add94afc055012c29e28124fcae2b63661b9efb1 \ + --hash=sha256:5ae8e62c147495b01a0f4765c878e9bfdf843412446a247e28df59936e99e797 \ + --hash=sha256:5b233ea3e165e43e35dba1d2b8ecc21cf070b45b65ae17dd2747d2713d942021 \ + --hash=sha256:6176c1811d9d5a04fa391c490cc44f451e240697a16977f11c6f722efb9041db \ + --hash=sha256:62f59da443c4f4849f73a51a193b1d9d258dcad0c41bc4d1b8fb2bcc04bfeb22 \ + --hash=sha256:6783e069732715ad0c3ce96dbf21dbc2235ab0593f2baf6338101f70371f4028 \ + --hash=sha256:6ab8ba9152203feec73758dad83af9a0bbe05001eb4639e547207c40cfb52083 \ + --hash=sha256:70d593af6a6ca332d1df73d519fddb5148edb15cd90d5f0155e3746a6d4fcc65 \ + --hash=sha256:72ec46b7eba5b395e0a7b63025490d3214c11013f4aacb4f5e8d6c3041829588 \ + --hash=sha256:7a32f72973f0f950c1920475d5c5ea3d971b81b6f0ec53b8d0a956cc965f22e0 \ + --hash=sha256:7a4aa69609f40fce3cbc3f87b2061f042eee32f94b8f11db707b66a26461591a \ + --hash=sha256:7c60d3c9b06fb23bd9c6139281ccbdc384297579ae037f08ae90c69f6845c0b1 \ + --hash=sha256:800ee55980c18545af444d93fdd60c56b580db5cc54867d8cbf8a1dc0829938c \ + --hash=sha256:80aa065ffd378ff784822a6d7c3212f2d5f5e9c3589614b5c228b311fd3063ac \ + --hash=sha256:86e0287879f75621ae85197b0877ed2f8b7aa57b511c7331dce2eb6f4de7d476 \ + --hash=sha256:893ff3a711d1b515ba9da14ee090519bad4610ed1962fbe298a434e8c5f8db53 \ + --hash=sha256:89fc958c702ee9a745e4700378f5d23fddbc46ff89e8fdbf5395c24d5c1452a3 \ + --hash=sha256:8c63c91f95173f9c2a67c7c526b2cea976828a0e7fced9cdcead2802dc10f8a4 \ + --hash=sha256:8df31fe574b8b3993cc61764f40941111b25c2d9fea13d3ce24a49907cd2d615 \ + --hash=sha256:8f9baf6f0a6e7571c45c8863010b45e837c3ee1c2c77fcd6ef423be91b21fedb \ + --hash=sha256:9027d773c4ff81487181a925945743413f6069634d0b122d0b37684ccf4f1e18 \ + --hash=sha256:9190426b7aa26c5229501fa297b8d0653cfd3f5a36f7990c264e157cbf886b3b \ + --hash=sha256:940dda65d5e764406b9fb92761cbf462e4e63f712ab60ed98f70552e496f3bf1 \ + --hash=sha256:94eff26096eb5395136634622515b234ecb6c9979824c1f5004c6e3c3c85ccd2 \ + --hash=sha256:9eed0f7edbb274413b6ee781cca50541c8c0facd3d6fd289779e494340a2b85c \ + --hash=sha256:ad4ae4ffd1ee9cd11357b4c66b612da9888f4f4daf2f36995eda64bd45370cac \ + --hash=sha256:b0f172dc8ffaccb8522d7c5d899de00133f2f1ca7b0a49b7da98e901de87bf2d \ + --hash=sha256:b2af221f268f5af85e776a73d62b0845fc8baf8ef0abfae79d29c77d0e776aaf \ + --hash=sha256:b7d335370ae48a780c6e6a6bbfa97342f563744c39c35562f3f367665f5c1de2 \ + --hash=sha256:b83af57bdddef03c01a9138034c6ff03181a3028d9a1003b301eb1a55e161a3f \ + --hash=sha256:bb5136fb5352d3f422df33f0c879a1b0c204004324150cc3b5e3c4f310c9049f \ + --hash=sha256:bc4d8e252f532ab46a1de9349e2d27b91fce46736a9eedaa37beaca66f574ed4 \ + --hash=sha256:bdd3e53429ff02aa319ba59dfe4ceeec345bf46cf180ec2cf6fd5b942e7975e9 \ + --hash=sha256:be12f931839a3bdfe28b584db0e640a65a8bcbc24560ae3fdb025a449b3d754e \ + --hash=sha256:be4a51a55833dc29ab5d7503e7bcb3b3af3402d266018137127450005cdfe737 \ + --hash=sha256:beb7f344487cdcb9e1efe4b7a29681b74d34c08f0043a327a74da852a6749e7b \ + --hash=sha256:bf4679a3d71012a7c2bf360e5cd878fbd5e4fcac0896b56393dec239d81529ed \ + --hash=sha256:c0e1403fd7c26d77c1f03e096dc58a5c726503fa0db0456678b8668f76f521e3 \ + --hash=sha256:c31c13da98624f957b0fb1b5bae5383b2333c2c3f6793d9825dd5ce79b525cb7 \ + --hash=sha256:c438f6ca858697c9ab67eb28246c92508af972e114cac34e57a6d4ba17a3ac08 \ + --hash=sha256:c8277104ded0a51e699c8c3aff63ce2c56d4ed5519a5f73e0fd7057f959a2b9e \ + --hash=sha256:c95cab08d1965db3d84a121f1c7ce7479bdd4072c9b3dafd8fecce48a2e6b902 \ + --hash=sha256:cc0b66c1eec9021353a4b4483afb12dfd50e3669ffbb9152d6842eb34c7e29fd \ + --hash=sha256:cdee07c4d7f6d72008d3f73b9bf027f4e11550224c7c50d8df1ae4a37c1402a6 \ + --hash=sha256:ce9bf03dad3b46408c08649c6fbd6ca28a9fce0eb32fdfffa6775a13103b5310 \ + --hash=sha256:cff8e5383db4989311f99e814feeb90c4723eb4edca425b9d5d9c3fefcdd9537 \ + --hash=sha256:d168fda2dbff7b9b5f38e693182d792a938c31db4dac3a80a4888de603c99554 \ + --hash=sha256:d1ffeb80b5676463d7a7d56acbe8e37a20ce725570e09549fe738e02ca6b7e1e \ + --hash=sha256:d36ca54cb4c6c4686f7cbb7b817f66f5911c12ddb519450bbe86707155028f87 \ + --hash=sha256:d4193f3d9dc3f6f79aaed0e5637f45d98850ebf01f7ca20e69457f3e8946b66a \ + --hash=sha256:d5cd5189fc2b6a538b75ae45433140c4823463918f7b1617c31e68b085c0022c \ + --hash=sha256:d618fd27420381a4f6044faa71f46d8bfd911bd077c555f7138ed88729bfbe79 \ + --hash=sha256:d76e2d8c75051d58177e762164d2e9ab92886534e3a12e795f103524f221dd8e \ + --hash=sha256:daae526907e262de627d8f70058a0f64acc9e2641c164c99c8f594b34a799a16 \ + --hash=sha256:db485b3847d182b908b483b2ed133c66d88d49cacf98fd278fadafe11b4478d1 \ + --hash=sha256:dd952e03bfbb096cfe2dd35cd9e00f269969b67536cb4370994afc20ff2d0875 \ + --hash=sha256:dda366d548e89a90d88a86c692377d18d8bd64b39c1fb2b92cb31370e2896bbd \ + --hash=sha256:e315e5ec90d88e140f57696ff85b484ff68bb311e36f2c414aa4286293e6dee0 \ + --hash=sha256:e4415a8db000bf49a6dd1c478bf70062eaacff0f462b92b0ba68791a905861f9 \ + --hash=sha256:e7a116ae737f0000343218c4edf5bd45893bfeaff0993c0b215d7124c9f77646 \ + --hash=sha256:e7c4c09a490dc4d4a7f8cbee56c606a320f9dc28cf92a7157a39d1ce7676a657 \ + --hash=sha256:ebae99ed6764f2b5771c522477b311be313e8841d2e0376db2b10922daebbba4 \ + --hash=sha256:ec4c85dc4b687c7f7f15f553ff26a98bfe8c58f5f7f0ac8905f0ba4c7be60232 \ + --hash=sha256:ed3a984b31da7481b103f68776f7128a89ef26ed40f4dc41a2223cda7fb24819 \ + --hash=sha256:f18c2d9782259a6dc132fdc7a63c168cbc74b35284b6d75c673958982a378384 \ + --hash=sha256:f1f9f4121ec58628c96baa3de1a55a4e3a333c5102c8e94b64e23bf7b2083309 \ + --hash=sha256:f42c23db5d1521218a3276bb08666dcb662896a0be7347cba864eca45ff64ede \ + --hash=sha256:f443b4825c50a51ee68585522ab4a1d1257fac65896f282b4c6763337ac9f5d2 \ + --hash=sha256:f6764a4ccab3078db14a632420930f6186058750df066b8ea2a7106df91d3203 \ + --hash=sha256:f7c7553b13f69c1b29a5bde08ddc6d9d0c8bfb84f9ed01c30db25944aeb852a7 \ + --hash=sha256:fa6248cd194edff41d7ea9425ced8ca3a6f838bfb295f6f1d6e6bb694a8518df \ + --hash=sha256:fa8eb9ecdb7efb0b226acec134e0d709e87a909fa4971a54c0c4f6e88635484c \ + --hash=sha256:fc20894c3d21194d8041a28b65622d5b86db786da6e3cfe73f0c762951a61167 \ + --hash=sha256:fc4d3f1fb9ca0ae9f97b095963bc6326f1dbfd3779d6679a1e016b9baaa153d3 \ + --hash=sha256:fd40bb9cd0891c4c3cb1ddf83f8bbfa15731a248fdc8162669405451e2724b09 \ + --hash=sha256:ff710414307fefa903e0d9bdf300972f892c23477829f49504e59834f4195398 + # via matplotlib +lets-plot==4.9.0 \ + --hash=sha256:103b09b9594448896bf31438b2f3866ed3d94a0e2f5df93047af27ae851c6951 \ + --hash=sha256:1563b52971670ec8fbf989980d39706ecf560072454001afac605cd77886c64a \ + --hash=sha256:1ba7aee59622fb444468c6cdb0c933504f36d884803da0a3b5e6d598f70cf1ad \ + --hash=sha256:1d64bdeb0c05146dfe0b5ebe26fc998f90224419beecf6354e2cb54207775ce0 \ + --hash=sha256:25873696e08e7e9df90aa42fe112e65112ca817da4341d04c54a74cdea74fd34 \ + --hash=sha256:2af3fee5f773d268e81daeaa346233e9bf6fee7aa9aa40dc01693bea986d8456 \ + --hash=sha256:318c0eae10218b7f8a153cac68f984ef4c0c313ef439b3db41fea9261097693b \ + --hash=sha256:3c2296a8dc550fa6f8906878e0fdb25bf25bb3acde22281b942528b441a52c1f \ + --hash=sha256:4310baa5cb8374e96a3c2b54152220603bc89851dfc57fc2c43b5d38fdb0832a \ + --hash=sha256:48359b96e5528c5cdbeed799e7e80a7dfa1b3af76cedb997abfd9f6f9ec0e506 \ + --hash=sha256:52b7569f91b8a44438fead2ec28b9907e1d23bdfeb4451589bc80cb3fba91384 \ + --hash=sha256:536bd8ea19b6f79c29b7998538b63bf602fe8e743e434f64c872c027cb9fb9f7 \ + --hash=sha256:5449e25a7a20c091ae4ee6f54d1711458cd2cc5e1963455c2293052249eda824 \ + --hash=sha256:5c9f2815ec9d3566145b98af486ed4ec0614e331238b3adbba10dd46c5bea472 \ + --hash=sha256:606f34d0401589c45a0cb7b1ab69823a8a10cd128d01d7ef8d464e343d128d18 \ + --hash=sha256:83269b428c6334bbcd5e873ea250ac6d2bb52aa763572eeba78988566786a5de \ + --hash=sha256:88f095a33f87bc64b87da9ed225dae6a0af6023e8a8c39e309e0bba40a8ca6f2 \ + --hash=sha256:9083b193748ae89bb4c118eeae944efface9777dd7b4d784af674106f8b5f2d8 \ + --hash=sha256:93a1ffea4736f49637c5d9ffeb18f69950343e15896333512c96c8f952f23656 \ + --hash=sha256:9793cc367b06fbe1664d6955bce0f7392369a8c318030d3a0cec419f00cc3b4a \ + --hash=sha256:a40036a03c98622cca9e316410feb8e29ddad5a941c3d63890f720821c8687c5 \ + --hash=sha256:a6cda1c544f91bef063b8dbea4876b21e8e6a1b709e89246976c7d422b1c3df2 \ + --hash=sha256:af591a4ee4af3cd2a3730d4e2ae4db74c0ceaa47da1c1ec2d33518431d86cd4b \ + --hash=sha256:b6d9666baab9810b54df30076c4918e6c28c17b98a54fea271b87b0aec3ec1ca \ + --hash=sha256:bc16a74a50395f9683ce3ea810e4fa154923e10406936f3756d6b32663124ae8 \ + --hash=sha256:be9766eeffd3690c80194faa7110e9f85235179da2e970ef8898791df1f97615 \ + --hash=sha256:ca34487455d7cb57f1a1d606419554bb5a8d3fa39692904d8b6e56342c67e9e1 \ + --hash=sha256:d26e09ea6fc8e5623d17ee4daa577ad06f70034badd13afad81e3fddb29d44ec \ + --hash=sha256:d2925f897963686df42e49a244e3b26e86f6acd5e1bddc6a0c24a6b70f9eb4ba \ + --hash=sha256:d420aa3b32b713bd6bb28a9474162d76a66ee4f3554d0bd0e0ef7685de0befe9 \ + --hash=sha256:dfd40b3aebfd54fd55e580223bfa8cdca91d66dfc3567c494c602c3f1918d960 \ + --hash=sha256:eb4442ae410d8b4be80432b182d0149d8efd049a00dc25b6c96b1ea9b9768a54 \ + --hash=sha256:ee643bb4adb8a3fe29126eab76c9661a4db87bf44ac0d8d01635fd2911465ca4 \ + --hash=sha256:ef61ba3b5f8613c48bd898a0fecfb40f1a76f15c6eee9030853a58bd4f7c8fd5 \ + --hash=sha256:f1dc14065a1ae08a2354971cc1704d2af236f8b5c69e2bade7cd3efa5aa5f1c8 + # via nki +libneuronxla==2.2.16408.0+50c26cbd \ + --hash=sha256:23b22819565826c27c89e660efc9e7745dfae1255d2ac72fe9fea1e34655b446 # via -r platforms/neuron/requirements.in lockfile==0.12.2 \ --hash=sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799 \ --hash=sha256:6c3cb24f344923d30b2785d5ad75182c8ea7ac1b6171b08657258ec7429d50fa # via python-daemon +matplotlib==3.10.9 \ + --hash=sha256:09218df8a93712bd6ea133e83a153c755448cf7868316c531cffcc43f69d1cc9 \ + --hash=sha256:10cc5ce06d10231c36f40e875f3c7e8050362a4ee8f0ee5d29a6b3277d57bb42 \ + --hash=sha256:172db52c9e683f5d12eaf57f0f54834190e12581fe1cc2a19595a8f5acb4e77d \ + --hash=sha256:1872fb212a05b729e649754a72d5da61d03e0554d76e80303b6f83d1d2c0552b \ + --hash=sha256:1aa972116abb4c9d201bf245620b433726cb6856f3bef6a78f776a00f5c92d37 \ + --hash=sha256:1e7698ac9868428e84d2c967424803b2472ff7167d9d6590d4204ed775343c3b \ + --hash=sha256:2dc9477819ffd78ad12a20df1d9d6a6bd4fec6aaa9072681465fddca052f1456 \ + --hash=sha256:3225f4e1edcb8c86c884ddf79ebe20ecd0a67d30188f279897554ccd8fded4dc \ + --hash=sha256:336b9acc64d309063126edcdaca00db9373af3c476bb94388fe9c5a53ad13e6f \ + --hash=sha256:345f6f68ecc8da0ca56fad2ea08fde1a115eda530079eca185d50a7bc3e146c6 \ + --hash=sha256:34cf8167e023ad956c15f36302911d5406bd99a9862c1a8499ea6f7c0e015dc2 \ + --hash=sha256:3fc0364dfbe1d07f6d15c5ebd0c5bf89e126916e5a8667dd4a7a6e84c36653d4 \ + --hash=sha256:41cb28c2bd769aa3e98322c6ab09854cbcc52ab69d2759d681bba3e327b2b320 \ + --hash=sha256:42fb814efabe95c06c1994d8ab5a8385f43a249e23badd3ba931d4308e5bca20 \ + --hash=sha256:4e42042d54db34fda4e95a7bd3e5789c2a995d2dad3eb8850232ee534092fbbf \ + --hash=sha256:4edcfbd8565339aa62f1cd4012f7180926fdbe71850f7b0d3c379c175cd6b66c \ + --hash=sha256:51bf0ddbdc598e060d46c16b5590708f81a1624cefbaaf62f6a81bf9285b8c80 \ + --hash=sha256:56fc0bd271b00025c6edfdc7c2dcd247372c8e1544971d62e1dc7c17367e8bf9 \ + --hash=sha256:59476c6d29d612b8e9bb6ce8c5b631be6ba8f9e3a2421f22a02b192c7dd28716 \ + --hash=sha256:6640f75af2c6148293caa0a2b39dd806a492dd66c8a8b04035813e33d0fd2585 \ + --hash=sha256:68cfdcede415f7c8f5577b03303dd94526cdb6d11036cecdc205e08733b2d2bb \ + --hash=sha256:6b63d9c7c769b88ab81e10dc86e4e0607cf56817b9f9e6cf24b2a5f1693b8e38 \ + --hash=sha256:6be157fe17fc37cb95ac1d7374cf717ce9259616edec911a78d9d26dae8522d4 \ + --hash=sha256:6c63ebcd8b4b169eb2f5c200552ae6b8be8999a005b6b507ed76fb8d7d674fe2 \ + --hash=sha256:77210dce9cb8153dffc967efaae990543392563d5a376d4dd8539bebcb0ed217 \ + --hash=sha256:7a8d66a55def891c33147ba3ba9bfcabf0b526a43764c818acbb4525e5ed0838 \ + --hash=sha256:82368699727bfb7b0182e1aa13082e3c08e092fa1a25d3e1fd92405bff96f6d4 \ + --hash=sha256:82834c3c292d24d3a8aae77cd2d20019de69d692a34a970e4fdb8d33e2ea3dda \ + --hash=sha256:8e436d155fa8a3399dc62683f8f5d0e2e50d25d0144a73edd73f82eec8f4abfb \ + --hash=sha256:8f3bcac1ca5ed000a6f4337d47ba67dfddf37ed6a46c15fd7f014997f7bf865f \ + --hash=sha256:97e35e8d39ccc85859095e01a53847432ba9a53ddf7986f7a54a11b73d0e143f \ + --hash=sha256:985f2238880e2e69093f588f5fe2e46771747febf0649f3cf7f7b7480875317f \ + --hash=sha256:a49f1eadc84ca85fd72fa4e89e70e61bf86452df6f971af04b12c60761a0772c \ + --hash=sha256:a5a6104ed666402ba5106d7f36e0e0cdca4e8d7fa4d39708ca88019e2835a2eb \ + --hash=sha256:aba1615dabe83188e19d4f75a253c6a08423e04c1425e64039f800050a69de6b \ + --hash=sha256:ae20801130378b82d647ff5047c07316295b68dc054ca6b3c13519d0ea624285 \ + --hash=sha256:ae2f11957b27ce53497dd4d7b235c4d4f1faf383dfb39d0c5beb833bff883294 \ + --hash=sha256:b049278ddce116aaa1c1377ebf58adea909132dfce0281cf7e3a1ea9fc2e2c65 \ + --hash=sha256:b1b745c489cd1a77a0dc1120a05dc87af9798faebc913601feb8c73d89bf2d1e \ + --hash=sha256:b2b9516251cb89ff618d757daec0e2ed1bf21248013844a853d87ef85ab3081d \ + --hash=sha256:b580440f1ff81a0e34122051a3dfabb7e4b7f9e380629929bde0eff9af72165f \ + --hash=sha256:ba7b3b8ef09eab7df0e86e9ae086faa433efbfbdb46afcb3aa16aabf779469a8 \ + --hash=sha256:c27df8b3848f32a83d1767566595e43cfaa4460380974da06f4279a7ec143c39 \ + --hash=sha256:d091f9d758b34aaaaa6331d13574bf01891d903b3dec59bfff458ef7551de5d6 \ + --hash=sha256:d730e984eddf56974c3e72b6129c7ca462ac38dc624338f4b0b23eb23ecba00f \ + --hash=sha256:d75d11c949914165976c621b2324f9ef162af7ebf4b057ddf95dd1dba7e5edcf \ + --hash=sha256:d843374407c4017a6403b59c6c81606773d136f3259d5b6da3131bc814542cc2 \ + --hash=sha256:da4e09638420548f31c354032a6250e473c68e5a4e96899b4844cf39ddea23fe \ + --hash=sha256:de2445a0c6690d21b7eb6ce071cebad6d40a2e9bdf10d039074a96ba19797b99 \ + --hash=sha256:dfca0129678bd56379db26c52b5d77ed7de314c047492fbdc763aa7501710cfb \ + --hash=sha256:e9fae004b941b23ff2edcf1567a857ed77bafc8086ffa258190462328434faf8 \ + --hash=sha256:f0c3c28d9fbcc1fe7a03be236d73430cf6409c41fb2383a7ac52fe932b072cb1 \ + --hash=sha256:f4399f64b3e94cd500195490972ae1ee81170df1636fa15364d157d5bdd7b921 \ + --hash=sha256:f76e640a5268850bfda54b5131b1b1941cc685e42c5fa98ed9f2d64038308cba \ + --hash=sha256:fd66508e8c6877d98e586654b608a0456db8d7e8a546eb1e2600efd957302358 + # via nki ml-dtypes==0.5.4 \ --hash=sha256:0d2ffd05a2575b1519dc928c0b93c06339eb67173ff53acb00724502cda231cf \ --hash=sha256:11942cbf2cf92157db91e5022633c0d9474d4dfd813a909383bd23ce828a4b7d \ @@ -260,96 +607,252 @@ networkx==2.8.8 \ --hash=sha256:230d388117af870fce5647a3c52401fcf753e94720e6ea6b4197a5355648885e \ --hash=sha256:e435dfa75b1d7195c7b8378c3859f0445cd88c6b0375c181ed66823a9ceb7524 # via neuronx-cc -neuronx-cc==2.23.6484.0+3b612583 \ - --hash=sha256:91089eb2b745ac6022dde04e71f4e08f2ccd5b61c860403ab18df8d5058a5c97 +neuronx-cc==2.24.8799.0+6f62ff7c \ + --hash=sha256:d34bad1aea02a7eb033e374f7086b24b9f502da6b017c3cdd10eee6c9f872e76 # via # -r platforms/neuron/requirements.in # libneuronxla -nki==0.2.0+g82fdb402 \ - --hash=sha256:42e2ffa5dfd9b5d22d4cca0f9bfb69a8ba8f2cac7f6eff801780798c05d07fb5 +nki==0.3.0+23928721754.g18aa1271 \ + --hash=sha256:fe45d60d70c7977b4af61193c88cc91ce04ba4bd4b606cf047622bed9b5f2143 # via neuronx-cc -numpy==2.4.3 \ - --hash=sha256:0200b25c687033316fb39f0ff4e3e690e8957a2c3c8d22499891ec58c37a3eb5 \ - --hash=sha256:0448e7f9caefb34b4b7dd2b77f21e8906e5d6f0365ad525f9f4f530b13df2afc \ - --hash=sha256:0a195f4216be9305a73c0e91c9b026a35f2161237cf1c6de9b681637772ea657 \ - --hash=sha256:0a60e17a14d640f49146cb38e3f105f571318db7826d9b6fef7e4dce758faecd \ - --hash=sha256:120df8c0a81ebbf5b9020c91439fccd85f5e018a927a39f624845be194a2be02 \ - --hash=sha256:148d59127ac95979d6f07e4d460f934ebdd6eed641db9c0db6c73026f2b2101a \ - --hash=sha256:1ec84fd7c8e652b0f4aaaf2e6e9cc8eaa9b1b80a537e06b2e3a2fb176eedcb26 \ - --hash=sha256:22654fe6be0e5206f553a9250762c653d3698e46686eee53b399ab90da59bd92 \ - --hash=sha256:22c31dc07025123aedf7f2db9e91783df13f1776dc52c6b22c620870dc0fab22 \ - --hash=sha256:23b46bb6d8ecb68b58c09944483c135ae5f0e9b8d8858ece5e4ead783771d2a9 \ - --hash=sha256:2629289168f4897a3c4e23dc98d6f1731f0fc0fe52fb9db19f974041e4cc12b9 \ - --hash=sha256:26952e18d82a1dbbc2f008d402021baa8d6fc8e84347a2072a25e08b46d698b9 \ - --hash=sha256:29363fbfa6f8ee855d7569c96ce524845e3d726d6c19b29eceec7dd555dab152 \ - --hash=sha256:297837823f5bc572c5f9379b0c9f3a3365f08492cbdc33bcc3af174372ebb168 \ - --hash=sha256:2abad5c7fef172b3377502bde47892439bae394a71bc329f31df0fd829b41a9e \ - --hash=sha256:2b3f8d2c4589b1a2028d2a770b0fc4d1f332fb5e01521f4de3199a896d158ddd \ - --hash=sha256:2ddb7919366ee468342b91dea2352824c25b55814a987847b6c52003a7c97f15 \ - --hash=sha256:2e03c05abaee1f672e9d67bc858f300b5ccba1c21397211e8d77d98350972093 \ - --hash=sha256:32e3bef222ad6b052280311d1d60db8e259e4947052c3ae7dd6817451fc8a4c5 \ - --hash=sha256:33b3bf58ee84b172c067f56aeadc7ee9ab6de69c5e800ab5b10295d54c581adb \ - --hash=sha256:45f003dbdffb997a03da2d1d0cb41fbd24a87507fb41605c0420a3db5bd4667b \ - --hash=sha256:483a201202b73495f00dbc83796c6ae63137a9bdade074f7648b3e32613412dd \ - --hash=sha256:48da3a4ee1336454b07497ff7ec83903efa5505792c4e6d9bf83d99dc07a1e18 \ - --hash=sha256:4b42639cdde6d24e732ff823a3fa5b701d8acad89c4142bc1d0bd6dc85200ba5 \ - --hash=sha256:4bd4741a6a676770e0e97fe9ab2e51de01183df3dcbcec591d26d331a40de950 \ - --hash=sha256:4d382735cecd7bcf090172489a525cd7d4087bc331f7df9f60ddc9a296cf208e \ - --hash=sha256:52077feedeff7c76ed7c9f1a0428558e50825347b7545bbb8523da2cd55c547a \ - --hash=sha256:54f29b877279d51e210e0c80709ee14ccbbad647810e8f3d375561c45ef613dd \ - --hash=sha256:5884ce5c7acfae1e4e1b6fde43797d10aa506074d25b531b4f54bde33c0c31d4 \ - --hash=sha256:5e10da9e93247e554bb1d22f8edc51847ddd7dde52d85ce31024c1b4312bfba0 \ - --hash=sha256:61b0cbabbb6126c8df63b9a3a0c4b1f44ebca5e12ff6997b80fcf267fb3150ef \ - --hash=sha256:65f3c2455188f09678355f5cae1f959a06b778bc66d535da07bf2ef20cd319d5 \ - --hash=sha256:679f2a834bae9020f81534671c56fd0cc76dd7e5182f57131478e23d0dc59e24 \ - --hash=sha256:6bd06731541f89cdc01b261ba2c9e037f1543df7472517836b78dfb15bd6e476 \ - --hash=sha256:715de7f82e192e8cae5a507a347d97ad17598f8e026152ca97233e3666daaa71 \ - --hash=sha256:737f630a337364665aba3b5a77e56a68cc42d350edd010c345d65a3efa3addcc \ - --hash=sha256:7395e69ff32526710748f92cd8c9849b361830968ea3e24a676f272653e8983e \ - --hash=sha256:76dbb9d4e43c16cf9aa711fcd8de1e2eeb27539dcefb60a1d5e9f12fae1d1ed8 \ - --hash=sha256:76f0f283506c28b12bba319c0fab98217e9f9b54e6160e9c79e9f7348ba32e9c \ - --hash=sha256:77e76d932c49a75617c6d13464e41203cd410956614d0a0e999b25e9e8d27eec \ - --hash=sha256:7aa4e54f6469300ebca1d9eb80acd5253cdfa36f2c03d79a35883687da430875 \ - --hash=sha256:7d1ce23cce91fcea443320a9d0ece9b9305d4368875bab09538f7a5b4131938a \ - --hash=sha256:7e58765ad74dcebd3ef0208a5078fba32dc8ec3578fe84a604432950cd043d79 \ - --hash=sha256:7f3408ff897f8ab07a07fbe2823d7aee6ff644c097cc1f90382511fe982f647f \ - --hash=sha256:8ba7b51e71c05aa1f9bc3641463cd82308eab40ce0d5c7e1fd4038cbf9938147 \ - --hash=sha256:8e236dbda4e1d319d681afcbb136c0c4a8e0f1a5c58ceec2adebb547357fe857 \ - --hash=sha256:94f3c4a151a2e529adf49c1d54f0f57ff8f9b233ee4d44af623a81553ab86368 \ - --hash=sha256:9684823a78a6cd6ad7511fc5e25b07947d1d5b5e2812c93fe99d7d4195130720 \ - --hash=sha256:a016db5c5dba78fa8fe9f5d80d6708f9c42ab087a739803c0ac83a43d686a470 \ - --hash=sha256:a111698b4a3f8dcbe54c64a7708f049355abd603e619013c346553c1fd4ca90b \ - --hash=sha256:a1988292870c7cb9d0ebb4cc96b4d447513a9644801de54606dc7aabf2b7d920 \ - --hash=sha256:a315e5234d88067f2d97e1f2ef670a7569df445d55400f1e33d117418d008d52 \ - --hash=sha256:a749547700de0a20a6718293396ec237bb38218049cfce788e08fcb716e8cf73 \ - --hash=sha256:a97cbf7e905c435865c2d939af3d93f99d18eaaa3cabe4256f4304fb51604349 \ - --hash=sha256:abdce0f71dcb4a00e4e77f3faf05e4616ceccfe72ccaa07f47ee79cda3b7b0f4 \ - --hash=sha256:b346845443716c8e542d54112966383b448f4a3ba5c66409771b8c0889485dd3 \ - --hash=sha256:b44fd60341c4d9783039598efadd03617fa28d041fc37d22b62d08f2027fa0e7 \ - --hash=sha256:bb2e3cf95854233799013779216c57e153c1ee67a0bf92138acca0e429aefaee \ - --hash=sha256:bc71942c789ef415a37f0d4eab90341425a00d538cd0642445d30b41023d3395 \ - --hash=sha256:be3b8487d725a77acccc9924f65fd8bce9af7fac8c9820df1049424a2115af6c \ - --hash=sha256:c59020932feb24ed49ffd03704fbab89f22aa9c0d4b180ff45542fe8918f5611 \ - --hash=sha256:c6b124bfcafb9e8d3ed09130dbee44848c20b3e758b6bbf006e641778927c028 \ - --hash=sha256:c9619741e9da2059cd9c3f206110b97583c7152c1dc9f8aafd4beb450ac1c89d \ - --hash=sha256:cd32fbacb9fd1bf041bf8e89e4576b6f00b895f06d00914820ae06a616bdfef7 \ - --hash=sha256:d1b90d840b25874cf5cd20c219af10bac3667db3876d9a495609273ebe679070 \ - --hash=sha256:d213c7e6e8d211888cc359bab7199670a00f5b82c0978b9d1c75baf1eddbeac0 \ - --hash=sha256:d5f51900414fc9204a0e0da158ba2ac52b75656e7dce7e77fb9f84bfa343b4cc \ - --hash=sha256:d71e379452a2f670ccb689ec801b1218cd3983e253105d6e83780967e899d687 \ - --hash=sha256:d84f0f881cb2225c2dfd7f78a10a5645d487a496c6668d6cc39f0f114164f3d0 \ - --hash=sha256:decb0eb8a53c3b009b0962378065589685d66b23467ef5dac16cbe818afde27f \ - --hash=sha256:e7dd01a46700b1967487141a66ac1a3cf0dd8ebf1f08db37d46389401512ca97 \ - --hash=sha256:eb610595dd91560905c132c709412b512135a60f1851ccbd2c959e136431ff67 +numpy==2.4.4 \ + --hash=sha256:07077278157d02f65c43b1b26a3886bce886f95d20aabd11f87932750dfb14ed \ + --hash=sha256:08f2e31ed5e6f04b118e49821397f12767934cfdd12a1ce86a058f91e004ee50 \ + --hash=sha256:0aec54fd785890ecca25a6003fd9a5aed47ad607bbac5cd64f836ad8666f4959 \ + --hash=sha256:0d35aea54ad1d420c812bfa0385c71cd7cc5bcf7c65fed95fc2cd02fe8c79827 \ + --hash=sha256:0d4e437e295f18ec29bc79daf55e8a47a9113df44d66f702f02a293d93a2d6dd \ + --hash=sha256:0dfd3f9d3adbe2920b68b5cd3d51444e13a10792ec7154cd0a2f6e74d4ab3233 \ + --hash=sha256:1378871da56ca8943c2ba674530924bb8ca40cd228358a3b5f302ad60cf875fc \ + --hash=sha256:15716cfef24d3a9762e3acdf87e27f58dc823d1348f765bbea6bef8c639bfa1b \ + --hash=sha256:19710a9ca9992d7174e9c52f643d4272dcd1558c5f7af7f6f8190f633bd651a7 \ + --hash=sha256:23cbfd4c17357c81021f21540da84ee282b9c8fba38a03b7b9d09ba6b951421e \ + --hash=sha256:2483e4584a1cb3092da4470b38866634bafb223cbcd551ee047633fd2584599a \ + --hash=sha256:27a8d92cd10f1382a67d7cf4db7ce18341b66438bdd9f691d7b0e48d104c2a9d \ + --hash=sha256:28a650663f7314afc3e6ec620f44f333c386aad9f6fc472030865dc0ebb26ee3 \ + --hash=sha256:2aa0613a5177c264ff5921051a5719d20095ea586ca88cc802c5c218d1c67d3e \ + --hash=sha256:2c194dd721e54ecad9ad387c1d35e63dce5c4450c6dc7dd5611283dda239aabb \ + --hash=sha256:2d19e6e2095506d1736b7d80595e0f252d76b89f5e715c35e06e937679ea7d7a \ + --hash=sha256:2d390634c5182175533585cc89f3608a4682ccb173cc9bb940b2881c8d6f8fa0 \ + --hash=sha256:30caa73029a225b2d40d9fae193e008e24b2026b7ee1a867b7ee8d96ca1a448e \ + --hash=sha256:42c16925aa5a02362f986765f9ebabf20de75cdefdca827d14315c568dcab113 \ + --hash=sha256:45dbed2ab436a9e826e302fcdcbe9133f9b0006e5af7168afb8963a6520da103 \ + --hash=sha256:4636de7fd195197b7535f231b5de9e4b36d2c440b6e566d2e4e4746e6af0ca93 \ + --hash=sha256:4a19d9dba1a76618dd86b164d608566f393f8ec6ac7c44f0cc879011c45e65af \ + --hash=sha256:4bbc7f303d125971f60ec0aaad5e12c62d0d2c925f0ab1273debd0e4ba37aba5 \ + --hash=sha256:4d6d57903571f86180eb98f8f0c839fa9ebbfb031356d87f1361be91e433f5b7 \ + --hash=sha256:4e874c976154687c1f71715b034739b45c7711bec81db01914770373d125e392 \ + --hash=sha256:51fc224f7ca4d92656d5a5eb315f12eb5fe2c97a66249aa7b5f562528a3be38c \ + --hash=sha256:58c8b5929fcb8287cbd6f0a3fae19c6e03a5c48402ae792962ac465224a629a4 \ + --hash=sha256:5a285b3b96f951841799528cd1f4f01cd70e7e0204b4abebac9463eecfcf2a40 \ + --hash=sha256:5c70f1cc1c4efbe316a572e2d8b9b9cc44e89b95f79ca3331553fbb63716e2bf \ + --hash=sha256:62d6b0f03b694173f9fcb1fb317f7222fd0b0b103e784c6549f5e53a27718c44 \ + --hash=sha256:6a246d5914aa1c820c9443ddcee9c02bec3e203b0c080349533fae17727dfd1b \ + --hash=sha256:6aa3236c78803afbcb255045fbef97a9e25a1f6c9888357d205ddc42f4d6eba5 \ + --hash=sha256:6bbe4eb67390b0a0265a2c25458f6b90a409d5d069f1041e6aff1e27e3d9a79e \ + --hash=sha256:715d1c092715954784bc79e1174fc2a90093dc4dc84ea15eb14dad8abdcdeb74 \ + --hash=sha256:72944b19f2324114e9dc86a159787333b77874143efcf89a5167ef83cfee8af0 \ + --hash=sha256:81f4a14bee47aec54f883e0cad2d73986640c1590eb9bfaaba7ad17394481e6e \ + --hash=sha256:846300f379b5b12cc769334464656bc882e0735d27d9726568bc932fdc49d5ec \ + --hash=sha256:86b6f55f5a352b48d7fbfd2dbc3d5b780b2d79f4d3c121f33eb6efb22e9a2015 \ + --hash=sha256:874f200b2a981c647340f841730fc3a2b54c9d940566a3c4149099591e2c4c3d \ + --hash=sha256:8a87ec22c87be071b6bdbd27920b129b94f2fc964358ce38f3822635a3e2e03d \ + --hash=sha256:8b3b60bb7cba2c8c81837661c488637eee696f59a877788a396d33150c35d842 \ + --hash=sha256:8e3ed142f2728df44263aaf5fb1f5b0b99f4070c553a0d7f033be65338329150 \ + --hash=sha256:93e15038125dc1e5345d9b5b68aa7f996ec33b98118d18c6ca0d0b7d6198b7e8 \ + --hash=sha256:989824e9faf85f96ec9c7761cd8d29c531ad857bfa1daa930cba85baaecf1a9a \ + --hash=sha256:99d838547ace2c4aace6c4f76e879ddfe02bb58a80c1549928477862b7a6d6ed \ + --hash=sha256:9b2aec6af35c113b05695ebb5749a787acd63cafc83086a05771d1e1cd1e555f \ + --hash=sha256:9c585a1790d5436a5374bac930dad6ed244c046ed91b2b2a3634eb2971d21008 \ + --hash=sha256:a7164afb23be6e37ad90b2f10426149fd75aee07ca55653d2aa41e66c4ef697e \ + --hash=sha256:ac6b31e35612a26483e20750126d30d0941f949426974cace8e6b5c58a3657b0 \ + --hash=sha256:ad2e2ef14e0b04e544ea2fa0a36463f847f113d314aa02e5b402fdf910ef309e \ + --hash=sha256:b268594bccac7d7cf5844c7732e3f20c50921d94e36d7ec9b79e9857694b1b2f \ + --hash=sha256:b5f0362dc928a6ecd9db58868fca5e48485205e3855957bdedea308f8672ea4a \ + --hash=sha256:ba1f4fc670ed79f876f70082eff4f9583c15fb9a4b89d6188412de4d18ae2f40 \ + --hash=sha256:ba203255017337d39f89bdd58417f03c4426f12beed0440cfd933cb15f8669c7 \ + --hash=sha256:c901b15172510173f5cb310eae652908340f8dede90fff9e3bf6c0d8dfd92f83 \ + --hash=sha256:c9b39d38a9bd2ae1becd7eac1303d031c5c110ad31f2b319c6e7d98b135c934d \ + --hash=sha256:d2a8490669bfe99a233298348acc2d824d496dee0e66e31b66a6022c2ad74a5c \ + --hash=sha256:dddbbd259598d7240b18c9d87c56a9d2fb3b02fe266f49a7c101532e78c1d871 \ + --hash=sha256:df3775294accfdd75f32c74ae39fcba920c9a378a2fc18a12b6820aa8c1fb502 \ + --hash=sha256:e44319a2953c738205bf3354537979eaa3998ed673395b964c1176083dd46252 \ + --hash=sha256:e4a010c27ff6f210ff4c6ef34394cd61470d01014439b192ec22552ee867f2a8 \ + --hash=sha256:e823b8b6edc81e747526f70f71a9c0a07ac4e7ad13020aa736bb7c9d67196115 \ + --hash=sha256:e892aff75639bbef0d2a2cfd55535510df26ff92f63c92cd84ef8d4ba5a5557f \ + --hash=sha256:eea7ac5d2dce4189771cedb559c738a71512768210dc4e4753b107a2048b3d0e \ + --hash=sha256:ef4059d6e5152fa1a39f888e344c73fdc926e1b2dd58c771d67b0acfbf2aa67d \ + --hash=sha256:f169b9a863d34f5d11b8698ead99febeaa17a13ca044961aa8e2662a6c7766a0 \ + --hash=sha256:f2cf083b324a467e1ab358c105f6cad5ea950f50524668a80c486ff1db24e119 \ + --hash=sha256:f8474c4241bc18b750be2abea9d7a9ec84f46ef861dbacf86a4f6e043401f79e \ + --hash=sha256:f983334aea213c99992053ede6168500e5f086ce74fbc4acc3f2b00f5762e9db \ + --hash=sha256:f9e75681b59ddaa5e659898085ae0eaea229d054f2ac0c7e563a62205a700121 \ + --hash=sha256:fbc356aae7adf9e6336d336b9c8111d390a05df88f1805573ebb0807bd06fd1d \ + --hash=sha256:fcfe2045fd2e8f3cb0ce9d4ba6dba6333b8fa05bb8a4939c908cd43322d14c7e # via + # contourpy + # matplotlib # ml-dtypes # neuronx-cc # nki + # pandas # scipy +packaging==26.2 \ + --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e \ + --hash=sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661 + # via matplotlib +palettable==3.3.3 \ + --hash=sha256:094dd7d9a5fc1cca4854773e5c1fc6a315b33bd5b3a8f47064928facaf0490a8 \ + --hash=sha256:74e9e7d7fe5a9be065e02397558ed1777b2df0b793a6f4ce1a5ee74f74fb0caa + # via lets-plot +pandas==3.0.2 \ + --hash=sha256:01f31a546acd5574ef77fe199bc90b55527c225c20ccda6601cf6b0fd5ed597c \ + --hash=sha256:0555c5882688a39317179ab4a0ed41d3ebc8812ab14c69364bbee8fb7a3f6288 \ + --hash=sha256:07a10f5c36512eead51bc578eb3354ad17578b22c013d89a796ab5eee90cd991 \ + --hash=sha256:08504503f7101300107ecdc8df73658e4347586db5cfdadabc1592e9d7e7a0fd \ + --hash=sha256:0f48afd9bb13300ffb5a3316973324c787054ba6665cda0da3fbd67f451995db \ + --hash=sha256:140f0cffb1fa2524e874dde5b477d9defe10780d8e9e220d259b2c0874c89d9d \ + --hash=sha256:232a70ebb568c0c4d2db4584f338c1577d81e3af63292208d615907b698a0f18 \ + --hash=sha256:32cc41f310ebd4a296d93515fcac312216adfedb1894e879303987b8f1e2b97d \ + --hash=sha256:339dda302bd8369dedeae979cb750e484d549b563c3f54f3922cb8ff4978c5eb \ + --hash=sha256:4544c7a54920de8eeacaa1466a6b7268ecfbc9bc64ab4dbb89c6bbe94d5e0660 \ + --hash=sha256:4d888a5c678a419a5bb41a2a93818e8ed9fd3172246555c0b37b7cc27027effd \ + --hash=sha256:5371b72c2d4d415d08765f32d689217a43227484e81b2305b52076e328f6f482 \ + --hash=sha256:57a07209bebcbcf768d2d13c9b78b852f9a15978dac41b9e6421a81ad4cdd276 \ + --hash=sha256:5880314e69e763d4c8b27937090de570f1fb8d027059a7ada3f7f8e98bdcb677 \ + --hash=sha256:5d3cfe227c725b1f3dff4278b43d8c784656a42a9325b63af6b1492a8232209e \ + --hash=sha256:5fdbfa05931071aba28b408e59226186b01eb5e92bea2ab78b65863ca3228d84 \ + --hash=sha256:60a80bb4feacbef5e1447a3f82c33209c8b7e07f28d805cfd1fb951e5cb443aa \ + --hash=sha256:61c2fd96d72b983a9891b2598f286befd4ad262161a609c92dc1652544b46b76 \ + --hash=sha256:63d141b56ef686f7f0d714cfb8de4e320475b86bf4b620aa0b7da89af8cbdbbb \ + --hash=sha256:6c4d8458b97a35717b62469a4ea0e85abd5ed8687277f5ccfc67f8a5126f8c53 \ + --hash=sha256:710246ba0616e86891b58ab95f2495143bb2bc83ab6b06747c74216f583a6ac9 \ + --hash=sha256:734be7551687c00fbd760dc0522ed974f82ad230d4a10f54bf51b80d44a08702 \ + --hash=sha256:7cadd7e9a44ec13b621aec60f9150e744cfc7a3dd32924a7e2f45edff31823b0 \ + --hash=sha256:81526c4afd31971f8b62671442a4b2b51e0aa9acc3819c9f0f12a28b6fcf85f1 \ + --hash=sha256:970762605cff1ca0d3f71ed4f3a769ea8f85fc8e6348f6e110b8fea7e6eb5a14 \ + --hash=sha256:a3096110bf9eac0070b7208465f2740e2d8a670d5cb6530b5bb884eca495fd39 \ + --hash=sha256:a4785e1d6547d8427c5208b748ae2efb64659a21bd82bf440d4262d02bfa02a4 \ + --hash=sha256:a727a73cbdba2f7458dc82449e2315899d5140b449015d822f515749a46cbbe0 \ + --hash=sha256:ae37e833ff4fed0ba352f6bdd8b73ba3ab3256a85e54edfd1ab51ae40cca0af8 \ + --hash=sha256:aff4e6f4d722e0652707d7bcb190c445fe58428500c6d16005b02401764b1b3d \ + --hash=sha256:b35d14bb5d8285d9494fe93815a9e9307c0876e10f1e8e89ac5b88f728ec8dcf \ + --hash=sha256:b444dc64c079e84df91baa8bf613d58405645461cabca929d9178f2cd392398d \ + --hash=sha256:b5329e26898896f06035241a626d7c335daa479b9bbc82be7c2742d048e41172 \ + --hash=sha256:b5918ba197c951dec132b0c5929a00c0bf05d5942f590d3c10a807f6e15a57d3 \ + --hash=sha256:b75c347eff42497452116ce05ef461822d97ce5b9ff8df6edacb8076092c855d \ + --hash=sha256:c3b723df9087a9a9a840e263ebd9f88b64a12075d1bf2ea401a5a42f254f084d \ + --hash=sha256:c934008c733b8bbea273ea308b73b3156f0181e5b72960790b09c18a2794fe1e \ + --hash=sha256:d1478075142e83a5571782ad007fb201ed074bdeac7ebcc8890c71442e96adf7 \ + --hash=sha256:d606a041c89c0a474a4702d532ab7e73a14fe35c8d427b972a625c8e46373668 \ + --hash=sha256:db0dbfd2a6cdf3770aa60464d50333d8f3d9165b2f2671bcc299b72de5a6677b \ + --hash=sha256:dbbd4aa20ca51e63b53bbde6a0fa4254b1aaabb74d2f542df7a7959feb1d760c \ + --hash=sha256:dbc20dea3b9e27d0e66d74c42b2d0c1bed9c2ffe92adea33633e3bedeb5ac235 \ + --hash=sha256:deeca1b5a931fdf0c2212c8a659ade6d3b1edc21f0914ce71ef24456ca7a6535 \ + --hash=sha256:ed72cb3f45190874eb579c64fa92d9df74e98fd63e2be7f62bce5ace0ade61df \ + --hash=sha256:ef8b27695c3d3dc78403c9a7d5e59a62d5464a7e1123b4e0042763f7104dc74f \ + --hash=sha256:f12b1a9e332c01e09510586f8ca9b108fd631fd656af82e452d7315ef6df5f9f \ + --hash=sha256:f4753e73e34c8d83221ba58f232433fca2748be8b18dbca02d242ed153945043 \ + --hash=sha256:f8d68083e49e16b84734eb1a4dcae4259a75c90fb6e2251ab9a00b61120c06ab + # via nki pgzip==0.4.0 \ --hash=sha256:41e44c03a9f1bbbd390dd6b6e37a11484113f8c159a049fa81cd8a9ddde82a21 \ --hash=sha256:91f6526f0c6e1f6c2d3522707777e5843fad7c791401745593fd01bd869b7a05 # via neuronx-cc +pillow==12.2.0 \ + --hash=sha256:00a2865911330191c0b818c59103b58a5e697cae67042366970a6b6f1b20b7f9 \ + --hash=sha256:01afa7cf67f74f09523699b4e88c73fb55c13346d212a59a2db1f86b0a63e8c5 \ + --hash=sha256:03e7e372d5240cc23e9f07deca4d775c0817bffc641b01e9c3af208dbd300987 \ + --hash=sha256:03f6fab9219220f041c74aeaa2939ff0062bd5c364ba9ce037197f4c6d498cd9 \ + --hash=sha256:042db20a421b9bafecc4b84a8b6e444686bd9d836c7fd24542db3e7df7baad9b \ + --hash=sha256:0538bd5e05efec03ae613fd89c4ce0368ecd2ba239cc25b9f9be7ed426b0af1f \ + --hash=sha256:0a34329707af4f73cf1782a36cd2289c0368880654a2c11f027bcee9052d35dd \ + --hash=sha256:0c838a5125cee37e68edec915651521191cef1e6aa336b855f495766e77a366e \ + --hash=sha256:144748b3af2d1b358d41286056d0003f47cb339b8c43a9ea42f5fea4d8c66b6e \ + --hash=sha256:1610dd6c61621ae1cf811bef44d77e149ce3f7b95afe66a4512f8c59f25d9ebe \ + --hash=sha256:1e1757442ed87f4912397c6d35a0db6a7b52592156014706f17658ff58bbf795 \ + --hash=sha256:22db17c68434de69d8ecfc2fe821569195c0c373b25cccb9cbdacf2c6e53c601 \ + --hash=sha256:25373b66e0dd5905ed63fa3cae13c82fbddf3079f2c8bf15c6fb6a35586324c1 \ + --hash=sha256:2bb4a8d594eacdfc59d9e5ad972aa8afdd48d584ffd5f13a937a664c3e7db0ed \ + --hash=sha256:2c727a6d53cb0018aadd8018c2b938376af27914a68a492f59dfcaca650d5eea \ + --hash=sha256:2d192a155bbcec180f8564f693e6fd9bccff5a7af9b32e2e4bf8c9c69dbad6b5 \ + --hash=sha256:2e589959f10d9824d39b350472b92f0ce3b443c0a3442ebf41c40cb8361c5b97 \ + --hash=sha256:2e5a76d03a6c6dcef67edabda7a52494afa4035021a79c8558e14af25313d453 \ + --hash=sha256:325ca0528c6788d2a6c3d40e3568639398137346c3d6e66bb61db96b96511c98 \ + --hash=sha256:34c0d99ecccea270c04882cb3b86e7b57296079c9a4aff88cb3b33563d95afaa \ + --hash=sha256:390ede346628ccc626e5730107cde16c42d3836b89662a115a921f28440e6a3b \ + --hash=sha256:394167b21da716608eac917c60aa9b969421b5dcbbe02ae7f013e7b85811c69d \ + --hash=sha256:3997232e10d2920a68d25191392e3a4487d8183039e1c74c2297f00ed1c50705 \ + --hash=sha256:3adc9215e8be0448ed6e814966ecf3d9952f0ea40eb14e89a102b87f450660d8 \ + --hash=sha256:3e080565d8d7c671db5802eedfb438e5565ffa40115216eabb8cd52d0ecce024 \ + --hash=sha256:4a6c9fa44005fa37a91ebfc95d081e8079757d2e904b27103f4f5fa6f0bf78c0 \ + --hash=sha256:4bfd07bc812fbd20395212969e41931001fd59eb55a60658b0e5710872e95286 \ + --hash=sha256:4e6c62e9d237e9b65fac06857d511e90d8461a32adcc1b9065ea0c0fa3a28150 \ + --hash=sha256:50d8520da2a6ce0af445fa6d648c4273c3eeefbc32d7ce049f22e8b5c3daecc2 \ + --hash=sha256:51c4167c34b0d8ba05b547a3bb23578d0ba17b80a5593f93bd8ecb123dd336a3 \ + --hash=sha256:56a3f9c60a13133a98ecff6197af34d7824de9b7b38c3654861a725c970c197b \ + --hash=sha256:56b25336f502b6ed02e889f4ece894a72612fe885889a6e8c4c80239ff6e5f5f \ + --hash=sha256:57850958fe9c751670e49b2cecf6294acc99e562531f4bd317fa5ddee2068463 \ + --hash=sha256:58f62cc0f00fd29e64b29f4fd923ffdb3859c9f9e6105bfc37ba1d08994e8940 \ + --hash=sha256:5c0a9f29ca8e79f09de89293f82fc9b0270bb4af1d58bc98f540cc4aedf03166 \ + --hash=sha256:5cdfebd752ec52bf5bb4e35d9c64b40826bc5b40a13df7c3cda20a2c03a0f5ed \ + --hash=sha256:5d04bfa02cc2d23b497d1e90a0f927070043f6cbf303e738300532379a4b4e0f \ + --hash=sha256:5d2fd0fa6b5d9d1de415060363433f28da8b1526c1c129020435e186794b3795 \ + --hash=sha256:62f5409336adb0663b7caa0da5c7d9e7bdbaae9ce761d34669420c2a801b2780 \ + --hash=sha256:632ff19b2778e43162304d50da0181ce24ac5bb8180122cbe1bf4673428328c7 \ + --hash=sha256:6562ace0d3fb5f20ed7290f1f929cae41b25ae29528f2af1722966a0a02e2aa1 \ + --hash=sha256:673aa32138f3e7531ccdbca7b3901dba9b70940a19ccecc6a37c77d5fdeb05b5 \ + --hash=sha256:6a6e67ea2e6feda684ed370f9a1c52e7a243631c025ba42149a2cc5934dec295 \ + --hash=sha256:6a9adfc6d24b10f89588096364cc726174118c62130c817c2837c60cf08a392b \ + --hash=sha256:6bb77b2dcb06b20f9f4b4a8454caa581cd4dd0643a08bacf821216a16d9c8354 \ + --hash=sha256:6e6b2a0c538fc200b38ff9eb6628228b77908c319a005815f2dde585a0664b60 \ + --hash=sha256:71cde9a1e1551df7d34a25462fc60325e8a11a82cc2e2f54578e5e9a1e153d65 \ + --hash=sha256:7371b48c4fa448d20d2714c9a1f775a81155050d383333e0a6c15b1123dda005 \ + --hash=sha256:766cef22385fa1091258ad7e6216792b156dc16d8d3fa607e7545b2b72061f1c \ + --hash=sha256:7b14cc0106cd9aecda615dd6903840a058b4700fcb817687d0ee4fc8b6e389be \ + --hash=sha256:7f84204dee22a783350679a0333981df803dac21a0190d706a50475e361c93f5 \ + --hash=sha256:8023abc91fba39036dbce14a7d6535632f99c0b857807cbbbf21ecc9f4717f06 \ + --hash=sha256:80b2da48193b2f33ed0c32c38140f9d3186583ce7d516526d462645fd98660ae \ + --hash=sha256:8297651f5b5679c19968abefd6bb84d95fe30ef712eb1b2d9b2d31ca61267f4c \ + --hash=sha256:88d387ff40b3ff7c274947ed3125dedf5262ec6919d83946753b5f3d7c67ea4c \ + --hash=sha256:88ddbc66737e277852913bd1e07c150cc7bb124539f94c4e2df5344494e0a612 \ + --hash=sha256:8bd7903a5f2a4545f6fd5935c90058b89d30045568985a71c79f5fd6edf9b91e \ + --hash=sha256:8be29e59487a79f173507c30ddf57e733a357f67881430449bb32614075a40ab \ + --hash=sha256:8c984051042858021a54926eb597d6ee3012393ce9c181814115df4c60b9a808 \ + --hash=sha256:8cbeb542b2ebc6fcdacabf8aca8c1a97c9b3ad3927d46b8723f9d4f033288a0f \ + --hash=sha256:8e9c4f5b3c546fa3458a29ab22646c1c6c787ea8f5ef51300e5a60300736905e \ + --hash=sha256:90e6f81de50ad6b534cab6e5aef77ff6e37722b2f5d908686f4a5c9eba17a909 \ + --hash=sha256:975385f4776fafde056abb318f612ef6285b10a1f12b8570f3647ad0d74b48ec \ + --hash=sha256:9a8a34cc89c67a65ea7437ce257cea81a9dad65b29805f3ecee8c8fe8ff25ffe \ + --hash=sha256:9aba9a17b623ef750a4d11b742cbafffeb48a869821252b30ee21b5e91392c50 \ + --hash=sha256:9f08483a632889536b8139663db60f6724bfcb443c96f1b18855860d7d5c0fd4 \ + --hash=sha256:a4e8f36e677d3336f35089648c8955c51c6d386a13cf6ee9c189c5f5bd713a9f \ + --hash=sha256:a52edc8bfff4429aaabdf4d9ee0daadbbf8562364f940937b941f87a4290f5ff \ + --hash=sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5 \ + --hash=sha256:aa88ccfe4e32d362816319ed727a004423aab09c5cea43c01a4b435643fa34eb \ + --hash=sha256:af73337013e0b3b46f175e79492d96845b16126ddf79c438d7ea7ff27783a414 \ + --hash=sha256:b1c1fbd8a5a1af3412a0810d060a78b5136ec0836c8a4ef9aa11807f2a22f4e1 \ + --hash=sha256:b85f66ae9eb53e860a873b858b789217ba505e5e405a24b85c0464822fe88032 \ + --hash=sha256:b86024e52a1b269467a802258c25521e6d742349d760728092e1bc2d135b4d76 \ + --hash=sha256:bd9c0c7a0c681a347b3194c500cb1e6ca9cab053ea4d82a5cf45b6b754560136 \ + --hash=sha256:bfa9c230d2fe991bed5318a5f119bd6780cda2915cca595393649fc118ab895e \ + --hash=sha256:d362d1878f00c142b7e1a16e6e5e780f02be8195123f164edf7eddd911eefe7c \ + --hash=sha256:d5d38f1411c0ed9f97bcb49b7bd59b6b7c314e0e27420e34d99d844b9ce3b6f3 \ + --hash=sha256:dac8d77255a37e81a2efcbd1fc05f1c15ee82200e6c240d7e127e25e365c39ea \ + --hash=sha256:dd025009355c926a84a612fecf58bb315a3f6814b17ead51a8e48d3823d9087f \ + --hash=sha256:deede7c263feb25dba4e82ea23058a235dcc2fe1f6021025dc71f2b618e26104 \ + --hash=sha256:e74473c875d78b8e9d5da2a70f7099549f9eb37ded4e2f6a463e60125bccd176 \ + --hash=sha256:ee3120ae9dff32f121610bb08e4313be87e03efeadfc6c0d18f89127e24d0c24 \ + --hash=sha256:eedf4b74eda2b5a4b2b2fb4c006d6295df3bf29e459e198c90ea48e130dc75c3 \ + --hash=sha256:efd8c21c98c5cc60653bcb311bef2ce0401642b7ce9d09e03a7da87c878289d4 \ + --hash=sha256:f1c943e96e85df3d3478f7b691f229887e143f81fedab9b20205349ab04d73ed \ + --hash=sha256:f278f034eb75b4e8a13a54a876cc4a5ab39173d2cdd93a638e1b467fc545ac43 \ + --hash=sha256:f3f40b3c5a968281fd507d519e444c35f0ff171237f4fdde090dd60699458421 \ + --hash=sha256:f490f9368b6fc026f021db16d7ec2fbf7d89e2edb42e8ec09d2c60505f5729c7 \ + --hash=sha256:fb043ee2f06b41473269765c2feae53fc2e2fbf96e5e22ca94fb5ad677856f06 \ + --hash=sha256:fc3d34d4a8fbec3e88a79b92e5465e0f9b842b628675850d860b8bd300b159f5 + # via + # lets-plot + # matplotlib protobuf==7.34.1 \ --hash=sha256:34b84ce27680df7cca9f231043ada0daa55d0c44a2ddfaa58ec1d0d89d8bf60a \ --hash=sha256:403b093a6e28a960372b44e5eb081775c9b056e816a8029c61231743d63f881a \ @@ -383,6 +886,14 @@ psutil==7.2.2 \ --hash=sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00 \ --hash=sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8 # via neuronx-cc +pyparsing==3.3.2 \ + --hash=sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d \ + --hash=sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc + # via matplotlib +pypng==0.20220715.0 \ + --hash=sha256:4a43e969b8f5aaafb2a415536c1a8ec7e341cd6a3f957fd5b5f32a4cfeed902c \ + --hash=sha256:739c433ba96f078315de54c0db975aee537cbc3e1d0ae4ed9aab0ca1e427e2c1 + # via lets-plot python-daemon==3.1.2 \ --hash=sha256:b906833cef63502994ad48e2eab213259ed9bb18d54fa8774dcba2ff7864cec6 \ --hash=sha256:f7b04335adc473de877f5117e26d5f1142f4c9f7cd765408f0877757be5afbf4 @@ -390,10 +901,13 @@ python-daemon==3.1.2 \ python-dateutil==2.9.0.post0 \ --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 - # via botocore -requests==2.32.5 \ - --hash=sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6 \ - --hash=sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf + # via + # botocore + # matplotlib + # pandas +requests==2.33.1 \ + --hash=sha256:18817f8c57c6263968bc123d237e3b8b08ac046f5456bd1e307ee8f4250d3517 \ + --hash=sha256:4e6d1ef462f3626a1f0a0a9c42dd93c63bad33f9f1c1937509b8c5c8718ab56a # via # ec2-metadata # requests-unixsocket @@ -401,9 +915,9 @@ requests-unixsocket==0.4.1 \ --hash=sha256:60c4942e9dbecc2f64d611039fb1dfc25da382083c6434ac0316dca3ff908f4d \ --hash=sha256:b2596158c356ecee68d27ba469a52211230ac6fb0cde8b66afb19f0ed47a1995 # via neuronx-cc -s3transfer==0.16.0 \ - --hash=sha256:18e25d66fed509e3868dc1572b3f427ff947dd2c56f844a5bf09481ad3f3b2fe \ - --hash=sha256:8e990f13268025792229cd52fa10cb7163744bf56e719e0b9cb925ab79abf920 +s3transfer==0.17.0 \ + --hash=sha256:9edeb6d1c3c2f89d6050348548834ad8289610d886e5bf7b7207728bd43ce33a \ + --hash=sha256:ce3801712acf4ad3e89fb9990df97b4972e93f4b3b0004d214be5bce12814c20 # via boto3 scipy==1.17.1 \ --hash=sha256:010f4333c96c9bb1a4516269e33cb5917b08ef2166d5556ca2fd9f082a9e6ea0 \ @@ -467,9 +981,7 @@ scipy==1.17.1 \ --hash=sha256:f590cd684941912d10becc07325a3eeb77886fe981415660d9265c4c418d0bea \ --hash=sha256:f8885db0bc2bffa59d5c1b72fad7a6a92d3e80e7257f967dd81abb553a90d293 \ --hash=sha256:fcb310ddb270a06114bb64bbe53c94926b943f5b7f0842194d585c65eb4edd76 - # via - # neuronx-cc - # nki + # via neuronx-cc six==1.17.0 \ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 diff --git a/platforms/neuron/topology/BUILD.bazel b/platforms/neuron/topology/BUILD.bazel new file mode 100644 index 00000000..8c0885c4 --- /dev/null +++ b/platforms/neuron/topology/BUILD.bazel @@ -0,0 +1,37 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") +load("@zml//bazel:cc_import.bzl", "cc_import") +load("@zml//bazel:runfiles.bzl", "runfiles_to_default") + +cc_import( + name = "neuron_ncfw", + tags = ["manual"], + shared_library = "@aws-neuronx-runtime-lib//:libncfw.patchelf", +) + +cc_import( + name = "neuron_nrt", + tags = ["manual"], + shared_library = "@aws-neuronx-runtime-lib//:libnrt.patchelf", + deps = [":neuron_ncfw"], +) + +runfiles_to_default( + name = "sandbox", + target_compatible_with = ["@platforms//os:linux"], + visibility = ["//visibility:public"], + deps = ["@libpjrt_neuron//:libpjrt_neuron"], +) + +zig_library( + name = "topology", + tags = ["manual"], + import_name = "platforms/neuron/topology", + main = "topology.zig", + visibility = ["//visibility:public"], + data = [":sandbox"], + deps = [ + ":neuron_nrt", + "//platforms/neuron:libnrt_headers", + "//platforms/neuron:zmlxneuron", + ], +) diff --git a/platforms/neuron/topology/topology.zig b/platforms/neuron/topology/topology.zig new file mode 100644 index 00000000..fb1ba01c --- /dev/null +++ b/platforms/neuron/topology/topology.zig @@ -0,0 +1,287 @@ +const std = @import("std"); + +const c = @import("c"); + +// References: +// - NRT exposes `nrt_instance_info.family` and `.size`, which map to the +// https://github.com/aws-neuron/aws-neuron-sdk/blob/master/src/libnrt/include/nrt/nrt.h +// Neuron instance families and sizes documented here: +// https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/index.html +// - Inf1: +// https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/neuron-hardware/inf1-arch.html +// - Inf2: +// https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/neuron-hardware/inf2-arch.html +// - Trn1 / Trn1n: +// https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/neuron-hardware/trn1-arch.html +// - Trn2 / Trn2 UltraServer: +// https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/neuron-hardware/trn2-arch.html +// - Trn3: +// https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/neuron-hardware/trn3-arch.html +// +// The same instance-architecture pages are also the basis for: +// - `InterChipTopology`, including the 2D torus and ring-over-torus fabrics. +// - `Instance.coresPerChip()`, using documented NeuronCore counts per chip generation: +// Inferentia v1 = 4, Inferentia2 / Trainium v1 = 2, Trainium2 / Trainium3 = 8. +pub const Family = enum(u32) { + unknown = 0, + inf1 = 1, + trn1 = 2, + trn1n = 3, + inf2 = 4, + trn2 = 5, + trn2n = 6, + inf2e = 7, + trn2p = 8, + trn2u = 9, + trn2e = 10, + trn2eu = 11, + trn2ac = 12, + trn2uac = 13, + trn3 = 14, + _, +}; + +pub const Size = enum(u32) { + xl1 = 0, + xl2 = 1, + xl4 = 2, + xl6 = 3, + xl8 = 4, + xl24 = 5, + xl32 = 6, + xl48 = 7, + xl3 = 8, + unknown = 9, + _, +}; + +pub const Instance = struct { + family: Family, + size: Size, + + pub fn current() !Instance { + var info = std.mem.zeroes(c.struct_nrt_instance_info); + info.family = @intFromEnum(Family.unknown); + info.size = @intFromEnum(Size.unknown); + + if (c.nrt_get_instance_info(&info, @sizeOf(c.struct_nrt_instance_info)) != c.NRT_SUCCESS) { + return error.Unavailable; + } + + return .{ + .family = @enumFromInt(info.family), + .size = @enumFromInt(info.size), + }; + } + + pub fn coresPerChip(self: Instance) !usize { + return switch (self.family) { + .inf1 => 4, + .trn1, .trn1n, .inf2, .inf2e => 2, + .trn2, .trn2n, .trn2p, .trn2u, .trn2e, .trn2eu, .trn2ac, .trn2uac, .trn3 => 8, + else => error.Unavailable, + }; + } + + pub fn interChipTopology(self: Instance, chip_count: usize) InterChipTopology { + if (chip_count == 1) return .none; + + // References for chip-count and fabric-shape inference: + // - Inf1 instance sizes and 1/4/16-chip layouts + // - Inf2 instance sizes and 1/6/12-chip layouts + // - Trn1 / Trn1n 16-chip 2D torus + // - Trn2 16-chip torus and Trn2 UltraServer 64-chip ring-over-torus + // - Trn3 all-to-all / point-to-point scale-up + return switch (self.family) { + .inf1 => switch (chip_count) { + 4 => .{ .linear = 4 }, + else => .{ .ring = chip_count }, + }, + .inf2, .inf2e => switch (chip_count) { + 6 => .{ .torus2d = .{ .x = 3, .y = 2 } }, + 12 => .{ .torus2d = .{ .x = 4, .y = 3 } }, + else => switch (self.size) { + .xl24 => .{ .torus2d = .{ .x = 3, .y = 2 } }, + .xl48 => .{ .torus2d = .{ .x = 4, .y = 3 } }, + else => .{ .ring = chip_count }, + }, + }, + .trn1, .trn1n => switch (chip_count) { + 16 => .{ .torus2d = .{ .x = 4, .y = 4 } }, + else => .{ .ring = chip_count }, + }, + .trn2, .trn2n, .trn2p, .trn2u, .trn2e, .trn2eu, .trn2ac, .trn2uac => switch (chip_count) { + 16 => .{ .torus2d = .{ .x = 4, .y = 4 } }, + 64 => .{ .torus2d_ring = .{ .x = 4, .y = 4, .z = 4 } }, + else => .{ .ring = chip_count }, + }, + .trn3 => .{ .point_to_point = chip_count }, + else => .{ .ring = chip_count }, + }; + } +}; + +pub const InterChipTopology = union(enum) { + none: void, + linear: usize, + ring: usize, + torus2d: struct { + x: usize, + y: usize, + }, + torus2d_ring: struct { + x: usize, + y: usize, + z: usize, + }, + point_to_point: usize, +}; + +pub const Axis = struct { + fabric: Fabric, + size: usize, + geometry: Geometry, +}; + +pub const Fabric = enum { + core, + chip_x, + chip_y, + chip_z, +}; + +pub const Geometry = enum { + linear_ring, + closed_ring, + torus, + point_to_point, +}; + +pub const Placement = struct { + input_index: usize, + nc_id: usize, + chip: usize, + core: usize, +}; + +pub const VisibleMesh = struct { + instance: Instance, + chips: usize, + cores_per_chip: usize, + inter_chip_topology: InterChipTopology, + placements: []Placement, + axes_buffer: [4]Axis, + axes_len: usize, + + pub fn deinit(self: *VisibleMesh, allocator: std.mem.Allocator) void { + allocator.free(self.placements); + self.* = undefined; + } + + pub fn axes(self: *const VisibleMesh) []const Axis { + return self.axes_buffer[0..self.axes_len]; + } + + pub fn format(self: VisibleMesh, writer: *std.Io.Writer) !void { + try writer.print( + "VisibleMesh(family={s} size={s} chips={d} cores_per_chip={d} inter_chip_topology=", + .{ + @tagName(self.instance.family), + @tagName(self.instance.size), + self.chips, + self.cores_per_chip, + }, + ); + switch (self.inter_chip_topology) { + .none => try writer.writeAll("none"), + .linear => |size| try writer.print("linear({d})", .{size}), + .ring => |size| try writer.print("ring({d})", .{size}), + .torus2d => |dims| try writer.print("torus2d({d}x{d})", .{ dims.x, dims.y }), + .torus2d_ring => |dims| try writer.print("torus2d_ring({d}x{d}x{d})", .{ dims.x, dims.y, dims.z }), + .point_to_point => |size| try writer.print("point_to_point({d})", .{size}), + } + try writer.writeAll(")"); + } +}; + +pub fn visibleMeshFromNcIds(allocator: std.mem.Allocator, nc_ids: []const usize) !VisibleMesh { + if (nc_ids.len == 0) return error.InvalidDeviceTopology; + + const instance = try Instance.current(); + const physical_cores_per_chip = try instance.coresPerChip(); + const placements = try allocator.alloc(Placement, nc_ids.len); + errdefer allocator.free(placements); + + for (nc_ids, placements, 0..) |nc_id, *placement, input_index| { + placement.* = .{ + .input_index = input_index, + .nc_id = nc_id, + .chip = chipOf(nc_id, physical_cores_per_chip), + .core = coreOf(nc_id, physical_cores_per_chip), + }; + } + + const Sort = struct { + fn lessThan(_: void, a: Placement, b: Placement) bool { + return if (a.chip == b.chip) a.core < b.core else a.chip < b.chip; + } + }; + std.mem.sort(Placement, placements, {}, Sort.lessThan); + + const first_chip = placements[0].chip; + const last_chip = placements[placements.len - 1].chip; + const selected_chip_count = last_chip - first_chip + 1; + const selected_cores_per_chip = try std.math.divExact(usize, placements.len, selected_chip_count); + const inter_chip_topology = instance.interChipTopology(selected_chip_count); + + var res: VisibleMesh = .{ + .instance = instance, + .chips = selected_chip_count, + .cores_per_chip = selected_cores_per_chip, + .inter_chip_topology = inter_chip_topology, + .placements = placements, + .axes_buffer = undefined, + .axes_len = 0, + }; + + switch (inter_chip_topology) { + .none => {}, + .linear => |size| appendAxis(&res, .chip_x, size, .linear_ring), + .ring => |size| appendAxis(&res, .chip_x, size, .closed_ring), + .torus2d => |dims| { + appendAxis(&res, .chip_x, dims.x, .torus); + appendAxis(&res, .chip_y, dims.y, .torus); + }, + .torus2d_ring => |dims| { + appendAxis(&res, .chip_x, dims.x, .torus); + appendAxis(&res, .chip_y, dims.y, .torus); + appendAxis(&res, .chip_z, dims.z, .closed_ring); + }, + .point_to_point => |size| appendAxis(&res, .chip_x, size, .point_to_point), + } + + // Inter-chip topology only describes NeuronLink fabric between chips. Add a + // local core axis for multi-core chips, and keep single-chip meshes non-empty. + if (res.cores_per_chip > 1 or res.axes_len == 0) { + appendAxis(&res, .core, res.cores_per_chip, .point_to_point); + } + + return res; +} + +pub fn chipOf(nc_id: usize, physical_cores_per_chip: usize) usize { + return @divFloor(nc_id, physical_cores_per_chip); +} + +pub fn coreOf(nc_id: usize, physical_cores_per_chip: usize) usize { + return @mod(nc_id, physical_cores_per_chip); +} + +fn appendAxis(self: *VisibleMesh, fabric: Fabric, size: usize, geometry: Geometry) void { + self.axes_buffer[self.axes_len] = .{ + .fabric = fabric, + .size = size, + .geometry = geometry, + }; + self.axes_len += 1; +} diff --git a/stdx/process.zig b/stdx/process.zig index 434a5bf7..91cf5586 100644 --- a/stdx/process.zig +++ b/stdx/process.zig @@ -15,10 +15,10 @@ fn selfSharedObjectPathImpl(addr: usize) []const u8 { return std.mem.span(info.dli_fname); } -pub fn selfSharedObjectPath() []const u8 { +pub noinline fn selfSharedObjectPath() []const u8 { return selfSharedObjectPathImpl(@returnAddress()); } -pub fn selfSharedObjectDirPath() []const u8 { +pub noinline fn selfSharedObjectDirPath() []const u8 { return std.fs.path.dirname(selfSharedObjectPathImpl(@returnAddress())).?; } diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index d6677539..9adc472a 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -127,6 +127,11 @@ zig_library( "@rules_zig//zig/runfiles", ":profiling_traceme_lib", ] + select({ + "//platforms:neuron.enabled": [ + "//platforms/neuron/topology", + ], + "//conditions:default": [], + }) + select({ "//platforms:cuda.enabled": [ "//platforms/cuda/flashattn", ], diff --git a/zml/Sharding.zig b/zml/Sharding.zig index 83067a17..dce1239e 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -1,9 +1,11 @@ //! Explain how to split a buffer across different devices. const std = @import("std"); +const c = @import("c"); const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); +const platforms = @import("platforms"); const stdx = @import("stdx"); const Memory = @import("platform.zig").Memory; @@ -13,8 +15,6 @@ const Shape = @import("shape.zig").Shape; const Slice = @import("slice.zig").Slice; const Target = @import("platform.zig").Target; -const log = std.log.scoped(.@"zml/sharding"); - const Sharding = @This(); data: *const Data, @@ -232,7 +232,7 @@ pub const Devices = stdx.BoundedArray(Device, Platform.MAX_NUM_DEVICES); /// Physical axis tags represent hardware interconnect tiers. pub const PhysicalAxisTag = enum { - link, // 1D Interconnect (Inf1, Inf2 Islands) + link, // Intra-device / island-local 1D interconnect link_x, // Torus/Mesh Dim X (TPU, Trainium) link_y, // Torus/Mesh Dim Y (TPU, Trainium) link_z, // Torus/Mesh Dim Z (TPU v3/v4/v5) @@ -634,7 +634,7 @@ pub const PhysicalMesh = struct { pub fn shardableAxes(self: PhysicalMesh) []const PhysicalAxisTag { return switch (self.target) { .tpu => &.{ .link_x, .link_y, .link_z }, - .neuron => &.{ .link_x, .link_y, .link_z, .link }, + .neuron => &.{ .link, .link_x, .link_y, .link_z }, .cuda, .rocm => &.{.link}, .cpu => &.{.bus}, }; @@ -932,8 +932,118 @@ pub const PhysicalMesh = struct { return CoordsTopology.buildMeshNode(allocator, indexed, axis_sizes[0..rank], axis_tags[0..rank], axis_strides[0..rank], 0, 0); } - pub fn neuron(_: std.mem.Allocator, _: []const PlatformDevice) !PhysicalNode { - stdx.debug.panic("neuron topology not implemented", .{}); + pub fn neuron(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) !Tree { + if (comptime !platforms.isEnabled(.neuron)) { + return error.UnsupportedPlatform; + } + + const neuron_topology = @import("platforms/neuron/topology"); + + const nc_ids = try allocator.alloc(usize, platform_devices.len); + defer allocator.free(nc_ids); + for (platform_devices, nc_ids) |device, *nc_id| { + nc_id.* = @intCast(device.localHardwareId()); + } + + var topology = try neuron_topology.visibleMeshFromNcIds(allocator, nc_ids); + defer topology.deinit(allocator); + + var axis_tags: stdx.BoundedArray(PhysicalAxisTag, Shape.MAX_RANK) = .empty; + var axis_sizes: stdx.BoundedArray(usize, Shape.MAX_RANK) = .empty; + var axis_geometries: stdx.BoundedArray(AxisGeometry, Shape.MAX_RANK) = .empty; + + for (topology.axes()) |mesh_axis| { + axis_tags.appendAssumeCapacity(switch (mesh_axis.fabric) { + .core => .link, + .chip_x => .link_x, + .chip_y => .link_y, + .chip_z => .link_z, + }); + axis_sizes.appendAssumeCapacity(mesh_axis.size); + axis_geometries.appendAssumeCapacity(switch (mesh_axis.geometry) { + .linear_ring => .{ .ring = .linear }, + .closed_ring => .{ .ring = .closed_ring }, + .torus => .{ .mesh = .torus }, + .point_to_point => .point_to_point, + }); + } + + var coords_buf: [Shape.MAX_RANK]usize = [_]usize{0} ** Shape.MAX_RANK; + var next_device: usize = 0; + + const Node = struct { + fn build( + allocator_: std.mem.Allocator, + platform_devices_: []const PlatformDevice, + placements: []const neuron_topology.Placement, + axis_tags_: []const PhysicalAxisTag, + axis_sizes_: []const usize, + axis_geometries_: []const AxisGeometry, + depth: usize, + coords_buf_: *[Shape.MAX_RANK]usize, + next_device_: *usize, + ) !PhysicalNode { + if (depth == axis_tags_.len) { + const placement = placements[next_device_.*]; + const device = platform_devices_[placement.input_index]; + next_device_.* += 1; + + var coords: stdx.BoundedArray(usize, Shape.MAX_RANK) = .empty; + for (coords_buf_[0..axis_tags_.len]) |coord| coords.appendAssumeCapacity(coord); + + return .{ + .leaf = .{ + .id = placement.nc_id, + .coords = coords, + .pjrt_device = device.pjrt_device, + }, + }; + } + + const children = try allocator_.alloc(PhysicalNode, axis_sizes_[depth]); + var built: usize = 0; + errdefer { + for (children[0..built]) |child| freeNode(allocator_, child); + allocator_.free(children); + } + + for (children, 0..) |*child, i| { + coords_buf_[depth] = i; + child.* = try @This().build( + allocator_, + platform_devices_, + placements, + axis_tags_, + axis_sizes_, + axis_geometries_, + depth + 1, + coords_buf_, + next_device_, + ); + built += 1; + } + + return .{ + .branch = .{ + .tag = axis_tags_[depth], + .geometry = axis_geometries_[depth], + .children = children, + }, + }; + } + }; + + return Node.build( + allocator, + platform_devices, + topology.placements, + axis_tags.constSlice(), + axis_sizes.constSlice(), + axis_geometries.constSlice(), + 0, + &coords_buf, + &next_device, + ); } }; @@ -1611,7 +1721,11 @@ pub const Placement = struct { pub fn platformDeviceIndex(self: *const Shard, platform: *const Platform) ?usize { for (platform.devices, 0..) |d, device_index| { - if (d.id() == self.device_id) return device_index; + const device_id = switch (platform.target) { + .neuron => @as(usize, @intCast(d.localHardwareId())), + .cuda, .rocm, .tpu, .cpu => d.id(), + }; + if (device_id == self.device_id) return device_index; } return null; @@ -1673,8 +1787,8 @@ pub const Placement = struct { pub fn linearIndex(self: AxisSplit) i64 { var linear: i64 = 0; - for (self.counts.constSlice(), self.indices.constSlice()) |c, iidx| { - linear = linear * c + iidx; + for (self.counts.constSlice(), self.indices.constSlice()) |c_, iidx| { + linear = linear * c_ + iidx; } return linear; } diff --git a/zml/buffer.zig b/zml/buffer.zig index f5b33653..fc3a1ce2 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -5,6 +5,7 @@ const stdx = @import("stdx"); const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; +const mem = @import("mem.zig"); const Memory = @import("platform.zig").Memory; const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; @@ -181,12 +182,24 @@ pub const Buffer = struct { pub const UnitializedOptions = struct { memory: Memory.Kind = .default }; pub fn uninitialized( - _: std.Io, + io: std.Io, platform: *const Platform, sh: Shape, sharding: Sharding, opts: UnitializedOptions, ) !Buffer { + switch (platform.target) { + .neuron => { + const allocator = std.heap.smp_allocator; + + const host = try allocator.alloc(u8, sh.byteSize()); + defer allocator.free(host); + + return try Buffer.from(io, platform, sh, sharding, host, .{ .wait = true, .memory = opts.memory }); + }, + .cpu, .cuda, .rocm, .tpu => {}, + } + var res: Buffer = .{ ._platform = platform, ._shape = sh, diff --git a/zml/exe.zig b/zml/exe.zig index 8bf9708e..cf305efe 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -261,7 +261,11 @@ pub const Exe = struct { stdx.debug.assert(opts.wait == false or io != null, "io should not be null when waiting for execution completion", .{}); var events = [_]?*pjrt.Event{null} ** Platform.MAX_NUM_DEVICES; - const events_slice: ?[]?*pjrt.Event = if (opts.wait) events[0..@intCast(self.num_partitions)] else null; + const partition_events = events[0..@intCast(self.num_partitions)]; + const events_slice: ?[]?*pjrt.Event = switch (self.platform.target) { + .neuron => partition_events, + .cpu, .cuda, .rocm, .tpu => if (opts.wait) partition_events else null, + }; self.exe.execute(self.platform.pjrt_api, .{ .arguments = arguments.flat_buffers.buffers, @@ -277,12 +281,21 @@ pub const Exe = struct { std.debug.panic("PJRT_LoadedExecutable_Execute failed with: {}", .{err}); }; - if (opts.wait) { - for (events_slice.?) |e| { - if (e) |ev| { - ev.await(self.platform.pjrt_api, io.?) catch unreachable; + switch (self.platform.target) { + .neuron => { + for (events_slice.?) |e| { + if (e) |ev| { + ev.deinit(self.platform.pjrt_api); + } } - } + }, + .cpu, .cuda, .rocm, .tpu => if (opts.wait) { + for (events_slice.?) |e| { + if (e) |ev| { + ev.await(self.platform.pjrt_api, io.?) catch unreachable; + } + } + }, } } diff --git a/zml/mem.zig b/zml/mem.zig index 2d4fa6a8..a620e4bb 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -22,8 +22,8 @@ pub const DmaAllocator = union(enum) { pub fn init(parent: std.mem.Allocator, device: *const Device) DmaAllocator { return switch (device.platform.target) { .cuda => .{ .dmam = .init(parent, device.platform) }, - .tpu, .neuron => .{ .uib = .init(device.memory(.host_pinned)) }, - .rocm, .cpu => .{ .passthrough = parent }, + .tpu => .{ .uib = .init(device.memory(.host_pinned)) }, + .rocm, .cpu, .neuron => .{ .passthrough = parent }, }; } diff --git a/zml/platform.zig b/zml/platform.zig index 9eb021b5..61b66f25 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -81,12 +81,9 @@ pub const Memory = struct { platform: *const Platform, addressable_by_devices: []*const Device, - fn init(allocator: std.mem.Allocator, pjrt_memory: *const pjrt.Memory, platform: *Platform, all_devices: []const Device) !Memory { + fn init(allocator: std.mem.Allocator, pjrt_memory: *const pjrt.Memory, platform: *Platform) !Memory { const pjrt_addressable_by_devices = pjrt_memory.addressableByDevices(platform.pjrt_api); const addressable_by_devices = try allocator.alloc(*const Device, pjrt_addressable_by_devices.len); - for (pjrt_addressable_by_devices, addressable_by_devices) |pjrt_device, *addressable_by_device| { - addressable_by_device.* = &all_devices[pjrt_device.localHardwareId(platform.pjrt_api)]; - } return .{ .pjrt_memory = pjrt_memory, @@ -117,6 +114,13 @@ pub const Memory = struct { fn deinit(self: *Memory, allocator: std.mem.Allocator) void { allocator.free(self.addressable_devices); } + + fn populateAddressableByDevices(self: *Memory) void { + const pjrt_addressable_by_devices = self.pjrt_memory.addressableByDevices(self.platform.pjrt_api); + for (pjrt_addressable_by_devices, self.addressable_by_devices) |pjrt_device, *addressable_by_device| { + addressable_by_device.* = self.platform.deviceFromPjrt(pjrt_device); + } + } }; pub const Device = struct { @@ -125,11 +129,11 @@ pub const Device = struct { pjrt_desc: *const pjrt.DeviceDescription, addressable_memories: []*const Memory, - fn init(allocator: std.mem.Allocator, pjrt_device_: *const pjrt.Device, platform: *const Platform, all_addressable_memories: []const Memory) !Device { + fn init(allocator: std.mem.Allocator, pjrt_device_: *const pjrt.Device, platform: *const Platform) !Device { const pjrt_addressable_memories = pjrt_device_.addressableMemories(platform.pjrt_api); const addressable_memories = try allocator.alloc(*const Memory, pjrt_addressable_memories.len); for (pjrt_addressable_memories, addressable_memories) |pjrt_memory, *addressable_memory| { - addressable_memory.* = &all_addressable_memories[pjrt_memory.id(platform.pjrt_api)]; + addressable_memory.* = platform.memoryFromPjrt(pjrt_memory); } return .{ @@ -153,7 +157,7 @@ pub const Device = struct { } pub fn localHardwareId(self: Device) i32 { - return self.device.localHardwareId(self.platform.pjrt_api); + return @intCast(self.pjrt_device.localHardwareId(self.platform.pjrt_api)); } pub fn kind(self: Device) []const u8 { @@ -181,8 +185,11 @@ pub const Device = struct { pub fn memory(self: Device, memory_kind: Memory.Kind) *const Memory { if (memory_kind == .default) { - const mem = self.pjrt_device.defaultMemory(self.platform.pjrt_api); - return &self.platform.memories[mem.id(self.platform.pjrt_api)]; + const pjrt_memory = self.pjrt_device.defaultMemory(self.platform.pjrt_api); + for (self.addressable_memories) |mem| { + if (mem.pjrt_memory == pjrt_memory) return mem; + } + return self.platform.memoryFromPjrt(pjrt_memory); } for (self.addressable_memories) |mem| { @@ -258,11 +265,14 @@ pub const Platform = struct { // TODO: part of the complication here is that we layout the data in spaghetti mode, // where devices and memories point to each other and also point to the platform. - for (pjrt_devices, devices) |pjrt_device, *platform_device| { - platform_device.* = try .init(arena, pjrt_device, platform, memories); - } for (pjrt_memories, memories) |pjrt_memory, *platform_memory| { - platform_memory.* = try .init(arena, pjrt_memory, platform, devices); + platform_memory.* = try .init(arena, pjrt_memory, platform); + } + for (pjrt_devices, devices) |pjrt_device, *platform_device| { + platform_device.* = try .init(arena, pjrt_device, platform); + } + for (memories) |*platform_memory| { + platform_memory.populateAddressableByDevices(); } platform.physical_mesh = try switch (options.physical_mesh) { @@ -541,6 +551,20 @@ pub const Platform = struct { return sharding; } + + fn memoryFromPjrt(self: *const Platform, pjrt_memory: *const pjrt.Memory) *const Memory { + for (self.memories) |*mem| { + if (mem.pjrt_memory == pjrt_memory) return mem; + } + unreachable; + } + + fn deviceFromPjrt(self: *const Platform, pjrt_device: *const pjrt.Device) *const Device { + for (self.devices) |*device| { + if (device.pjrt_device == pjrt_device) return device; + } + unreachable; + } }; pub const CreateOptions = struct { diff --git a/zml/tensor.zig b/zml/tensor.zig index e7dae22e..87166295 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -4095,11 +4095,16 @@ pub const Tensor = struct { /// Only for debug purpose, it inserts device to host synchronization /// so it will slow down the program execution. pub fn print(input: Tensor, name: []const u8) void { - ops.manualComputation(input, {}, .{ .name = name }, (struct { - fn body(ctx_: anytype, _: std.mem.Allocator, sharded_input: Tensor, _: void) void { - ops.customCall("zml$print", sharded_input, {}, .{ .name = ctx_.name }, .{ .has_side_effect = true }); - } - }).body); + switch (CompilationContext.current().platform.target) { + .cpu, .cuda, .rocm, .tpu => { + ops.manualComputation(input, {}, .{ .name = name }, (struct { + fn body(ctx_: anytype, _: std.mem.Allocator, sharded_input: Tensor, _: void) void { + ops.customCall("zml$print", sharded_input, {}, .{ .name = ctx_.name }, .{ .has_side_effect = true }); + } + }).body); + }, + .neuron => {}, + } } fn mlirCtx() *mlir.Context { From 31887689a9dace01a9c0307b5c13aa600d8fb81e Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Tue, 12 May 2026 10:27:18 +0200 Subject: [PATCH 477/881] workspace: add several bazel 9 required changed (#530) Just a few stuff to get us closer to Bazel 9 XLA is the main blocker and i'll tackle this as soon as other dependencies merge my PRs. --- MODULE.bazel | 11 +- MODULE.bazel.lock | 929 +++++++++++++++++++--------------------- bazel/cc_import.bzl | 1 + bazel/swig.bzl | 3 +- third_party/xla/xla.bzl | 7 +- tools/BUILD.bazel | 1 + 6 files changed, 461 insertions(+), 491 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index f5900280..75ad9a4e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -18,9 +18,9 @@ single_version_override( version = "33.6", # important, until grpc-cpp is fixed ) -bazel_dep(name = "pybind11_bazel", version = "2.13.6") +bazel_dep(name = "pybind11_bazel", version = "3.0.1") bazel_dep(name = "re2", version = "2025-11-05", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_cc", version = "0.2.16") +bazel_dep(name = "rules_cc", version = "0.2.18") bazel_dep(name = "rules_distroless", version = "0.6.2") bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep(name = "rules_oci", version = "2.2.7") @@ -38,6 +38,13 @@ bazel_dep(name = "tar.bzl", version = "0.9.0") bazel_dep(name = "llvm", version = "0.7.3") bazel_dep(name = "with_cfg.bzl", version = "0.14.4") bazel_dep(name = "grpc", version = "1.74.0") +# Added to support bazel 9+ since an old version is pulled by GRPC +# TODO: remove when GRPC is updated to v1.80+ +bazel_dep(name = "rules_foreign_cc", version = "0.15.1") +# Same as rules_foreign_cc +bazel_dep(name = "rules_go", version = "0.60.0") +# Same as rules_foreign_cc +bazel_dep(name = "jsoncpp", version = "1.9.6.bcr.2", repo_name = "jsoncpp_git") bazel_dep(name = "buildifier_prebuilt", version = "8.5.1", dev_dependency = True) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 3f2e5294..2b6f470b 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -51,6 +51,7 @@ "https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87", "https://bcr.bazel.build/modules/bazel_features/1.33.0/MODULE.bazel": "8b8dc9d2a4c88609409c3191165bccec0e4cb044cd7a72ccbe826583303459f6", "https://bcr.bazel.build/modules/bazel_features/1.34.0/MODULE.bazel": "e8475ad7c8965542e0c7aac8af68eb48c4af904be3d614b6aa6274c092c2ea1e", + "https://bcr.bazel.build/modules/bazel_features/1.36.0/MODULE.bazel": "596cb62090b039caf1cad1d52a8bc35cf188ca9a4e279a828005e7ee49a1bec3", "https://bcr.bazel.build/modules/bazel_features/1.38.0/MODULE.bazel": "f9b8a9c890ebd216b4049fd12a31d3c2602e3403c7af636b04fbbd7453edc9c9", "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", "https://bcr.bazel.build/modules/bazel_features/1.42.0/MODULE.bazel": "e8ca15cb2639c5f12183db6dcb678735555d0cdd739b32a0418b6532b5e565f8", @@ -144,8 +145,9 @@ "https://bcr.bazel.build/modules/grpc/1.74.0/source.json": "87c009e2f3226fd43170ce4896a3f17c819bd4c7388ff1336158b12b636a0ed9", "https://bcr.bazel.build/modules/jq.bzl/0.1.0/MODULE.bazel": "2ce69b1af49952cd4121a9c3055faa679e748ce774c7f1fda9657f936cae902f", "https://bcr.bazel.build/modules/jq.bzl/0.1.0/source.json": "746bf13cac0860f091df5e4911d0c593971cd8796b5ad4e809b2f8e133eee3d5", + "https://bcr.bazel.build/modules/jsoncpp/1.9.6.bcr.2/MODULE.bazel": "64f508885f907ac2518039b3d0c5c1703a8f6137d9f5d635067ba93d33c2670a", + "https://bcr.bazel.build/modules/jsoncpp/1.9.6.bcr.2/source.json": "13199fa0a267ca46814e9e6ab313a71890c8f1822e57376fdc23a2d2cd384051", "https://bcr.bazel.build/modules/jsoncpp/1.9.6/MODULE.bazel": "2f8d20d3b7d54143213c4dfc3d98225c42de7d666011528dc8fe91591e2e17b0", - "https://bcr.bazel.build/modules/jsoncpp/1.9.6/source.json": "a04756d367a2126c3541682864ecec52f92cdee80a35735a3cb249ce015ca000", "https://bcr.bazel.build/modules/libbacktrace/1.0.0-20250926-7939218/MODULE.bazel": "4cd5b9355f1c2533c30f01904b71ac1b1a9bf61c43c81ba9972a5bf8f216476b", "https://bcr.bazel.build/modules/libbacktrace/1.0.0-20250926-7939218/source.json": "83239989d09748f63dee4bdd430d261e217a012670e066e2238925a303a65d89", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", @@ -206,7 +208,8 @@ "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e", "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/MODULE.bazel": "e6f4c20442eaa7c90d7190d8dc539d0ab422f95c65a57cc59562170c58ae3d34", "https://bcr.bazel.build/modules/pybind11_bazel/2.13.6/MODULE.bazel": "2d746fda559464b253b2b2e6073cb51643a2ac79009ca02100ebbc44b4548656", - "https://bcr.bazel.build/modules/pybind11_bazel/2.13.6/source.json": "6aa0703de8efb20cc897bbdbeb928582ee7beaf278bcd001ac253e1605bddfae", + "https://bcr.bazel.build/modules/pybind11_bazel/3.0.1/MODULE.bazel": "67899804c3dbdf169d107ce7c00e60a38a46d097c7ff3460bdebe192cbc77f56", + "https://bcr.bazel.build/modules/pybind11_bazel/3.0.1/source.json": "5bcb3ae55c0d5a159bd910154d7ee8873bdf61dd7b13cd0a2bb456011ec706ae", "https://bcr.bazel.build/modules/rapidjson/1.1.0.bcr.20241007/MODULE.bazel": "82fbcb2e42f9e0040e76ccc74c06c3e46dfd33c64ca359293f8b84df0e6dff4c", "https://bcr.bazel.build/modules/rapidjson/1.1.0.bcr.20241007/source.json": "5c42389ad0e21fc06b95ad7c0b730008271624a2fa3292e0eab5f30e15adeee3", "https://bcr.bazel.build/modules/re2/2021-09-01/MODULE.bazel": "bcb6b96f3b071e6fe2d8bed9cc8ada137a105f9d2c5912e91d27528b3d123833", @@ -239,6 +242,7 @@ "https://bcr.bazel.build/modules/rules_cc/0.2.0/MODULE.bazel": "b5c17f90458caae90d2ccd114c81970062946f49f355610ed89bebf954f5783c", "https://bcr.bazel.build/modules/rules_cc/0.2.15/MODULE.bazel": "6a0a4a75a57aa6dc888300d848053a58c6b12a29f89d4304e1c41448514ec6e8", "https://bcr.bazel.build/modules/rules_cc/0.2.16/MODULE.bazel": "9242fa89f950c6ef7702801ab53922e99c69b02310c39fb6e62b2bd30df2a1d4", + "https://bcr.bazel.build/modules/rules_cc/0.2.17/MODULE.bazel": "1849602c86cb60da8613d2de887f9566a6d354a6df6d7009f9d04a14402f9a84", "https://bcr.bazel.build/modules/rules_cc/0.2.18/MODULE.bazel": "4460ec36adc8f722a6a2a4ac9374cb91f2acebadaa93fc37966129afb3dece87", "https://bcr.bazel.build/modules/rules_cc/0.2.18/source.json": "abad668ff2fd63ada1ac49bf386d37e27048b89a3465a6fd968bb832b00a09d3", "https://bcr.bazel.build/modules/rules_cc/0.2.4/MODULE.bazel": "1ff1223dfd24f3ecf8f028446d4a27608aa43c3f41e346d22838a4223980b8cc", @@ -246,7 +250,8 @@ "https://bcr.bazel.build/modules/rules_distroless/0.6.2/MODULE.bazel": "eafe8b473c0c78038d9a6e1dbd9278b7bf83457ddc78207297483bfbe1012d56", "https://bcr.bazel.build/modules/rules_distroless/0.6.2/source.json": "830d7a25ce382e571e58791fbed3b47e6da3237805624bab8997ae59d763bdc2", "https://bcr.bazel.build/modules/rules_foreign_cc/0.10.1/MODULE.bazel": "b9527010e5fef060af92b6724edb3691970a5b1f76f74b21d39f7d433641be60", - "https://bcr.bazel.build/modules/rules_foreign_cc/0.10.1/source.json": "9300e71df0cdde0952f10afff1401fa664e9fc5d9ae6204660ba1b158d90d6a6", + "https://bcr.bazel.build/modules/rules_foreign_cc/0.15.1/MODULE.bazel": "c2c60d26c79fda484acb95cdbec46e89d6b28b4845cb277160ce1e0c8622bb88", + "https://bcr.bazel.build/modules/rules_foreign_cc/0.15.1/source.json": "a161811a63ba8a859086da3b7ff3ad04f2e9c255d7727b41087103fc0eb22f55", "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", "https://bcr.bazel.build/modules/rules_go/0.33.0/MODULE.bazel": "a2b11b64cd24bf94f57454f53288a5dacfe6cb86453eee7761b7637728c1910c", "https://bcr.bazel.build/modules/rules_go/0.38.1/MODULE.bazel": "fb8e73dd3b6fc4ff9d260ceacd830114891d49904f5bda1c16bc147bcc254f71", @@ -257,7 +262,8 @@ "https://bcr.bazel.build/modules/rules_go/0.46.0/MODULE.bazel": "3477df8bdcc49e698b9d25f734c4f3a9f5931ff34ee48a2c662be168f5f2d3fd", "https://bcr.bazel.build/modules/rules_go/0.48.0/MODULE.bazel": "d00ebcae0908ee3f5e6d53f68677a303d6d59a77beef879598700049c3980a03", "https://bcr.bazel.build/modules/rules_go/0.50.1/MODULE.bazel": "b91a308dc5782bb0a8021ad4330c81fea5bda77f96b9e4c117b9b9c8f6665ee0", - "https://bcr.bazel.build/modules/rules_go/0.50.1/source.json": "205765fd30216c70321f84c9a967267684bdc74350af3f3c46c857d9f80a4fa2", + "https://bcr.bazel.build/modules/rules_go/0.60.0/MODULE.bazel": "4a57ff2ffc2a3570e3c5646575c5a4b07287e91bcdac5d1f72383d51502b48cb", + "https://bcr.bazel.build/modules/rules_go/0.60.0/source.json": "1e21368c5e0c3013a110bd79a8fcff8ca46b5bcb2b561713a7273cbfcff7c464", "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", "https://bcr.bazel.build/modules/rules_java/5.1.0/MODULE.bazel": "324b6478b0343a3ce7a9add8586ad75d24076d6d43d2f622990b9c1cfd8a1b15", "https://bcr.bazel.build/modules/rules_java/5.3.5/MODULE.bazel": "a4ec4f2db570171e3e5eb753276ee4b389bae16b96207e9d3230895c99644b86", @@ -320,6 +326,7 @@ "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", "https://bcr.bazel.build/modules/rules_python/0.40.0/MODULE.bazel": "9d1a3cd88ed7d8e39583d9ffe56ae8a244f67783ae89b60caafc9f5cf318ada7", "https://bcr.bazel.build/modules/rules_python/1.0.0/MODULE.bazel": "898a3d999c22caa585eb062b600f88654bf92efb204fa346fb55f6f8edffca43", + "https://bcr.bazel.build/modules/rules_python/1.2.0/MODULE.bazel": "5aeeb48b2a6c19d668b48adf2b8a2b209a6310c230db0ce77450f148a89846e4", "https://bcr.bazel.build/modules/rules_python/1.5.1/MODULE.bazel": "acfe65880942d44a69129d4c5c3122d57baaf3edf58ae5a6bd4edea114906bf5", "https://bcr.bazel.build/modules/rules_python/1.6.0/MODULE.bazel": "7e04ad8f8d5bea40451cf80b1bd8262552aa73f841415d20db96b7241bd027d8", "https://bcr.bazel.build/modules/rules_python/1.7.0/MODULE.bazel": "d01f995ecd137abf30238ad9ce97f8fc3ac57289c8b24bd0bf53324d937a14f8", @@ -473,487 +480,6 @@ ] } }, - "@@googleapis+//:extensions.bzl%switched_rules": { - "general": { - "bzlTransitiveDigest": "JLpojYeGHLdgnwSUuQWOrtkL9UMygBUUEFJZh7ASlmA=", - "usagesDigest": "66rLsrgz6YqDpU0pD+02mTGNxYxhb/of6Fh0ulHzjdE=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "com_google_googleapis_imports": { - "repoRuleId": "@@googleapis+//:repository_rules.bzl%switched_rules", - "attributes": { - "rules": { - "proto_library_with_info": [ - "", - "" - ], - "moved_proto_library": [ - "", - "" - ], - "java_proto_library": [ - "", - "" - ], - "java_grpc_library": [ - "", - "" - ], - "java_gapic_library": [ - "", - "" - ], - "java_gapic_test": [ - "", - "" - ], - "java_gapic_assembly_gradle_pkg": [ - "", - "" - ], - "py_proto_library": [ - "", - "" - ], - "py_grpc_library": [ - "", - "" - ], - "py_gapic_library": [ - "", - "" - ], - "py_test": [ - "", - "" - ], - "py_gapic_assembly_pkg": [ - "", - "" - ], - "py_import": [ - "", - "" - ], - "go_proto_library": [ - "", - "" - ], - "go_grpc_library": [ - "", - "" - ], - "go_library": [ - "", - "" - ], - "go_test": [ - "", - "" - ], - "go_gapic_library": [ - "", - "" - ], - "go_gapic_assembly_pkg": [ - "", - "" - ], - "cc_proto_library": [ - "", - "" - ], - "cc_grpc_library": [ - "", - "" - ], - "cc_gapic_library": [ - "", - "" - ], - "php_proto_library": [ - "", - "php_proto_library" - ], - "php_grpc_library": [ - "", - "php_grpc_library" - ], - "php_gapic_library": [ - "", - "php_gapic_library" - ], - "php_gapic_assembly_pkg": [ - "", - "php_gapic_assembly_pkg" - ], - "nodejs_gapic_library": [ - "", - "typescript_gapic_library" - ], - "nodejs_gapic_assembly_pkg": [ - "", - "typescript_gapic_assembly_pkg" - ], - "ruby_proto_library": [ - "", - "" - ], - "ruby_grpc_library": [ - "", - "" - ], - "ruby_ads_gapic_library": [ - "", - "" - ], - "ruby_cloud_gapic_library": [ - "", - "" - ], - "ruby_gapic_assembly_pkg": [ - "", - "" - ], - "csharp_proto_library": [ - "", - "" - ], - "csharp_grpc_library": [ - "", - "" - ], - "csharp_gapic_library": [ - "", - "" - ], - "csharp_gapic_assembly_pkg": [ - "", - "" - ] - } - } - } - }, - "recordedRepoMappingEntries": [] - } - }, - "@@rules_foreign_cc+//foreign_cc:extensions.bzl%tools": { - "general": { - "bzlTransitiveDigest": "b41JwymKkuHbovxSrZD6eawffiN8+iwKcSQZrLCkvbw=", - "usagesDigest": "9LXdVp01HkdYQT8gYPjYLO6VLVJHo9uFfxWaU1ymiRE=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "rules_foreign_cc_framework_toolchain_linux": { - "repoRuleId": "@@rules_foreign_cc+//foreign_cc/private/framework:toolchain.bzl%framework_toolchain_repository", - "attributes": { - "commands_src": "@rules_foreign_cc//foreign_cc/private/framework/toolchains:linux_commands.bzl", - "exec_compatible_with": [ - "@platforms//os:linux" - ] - } - }, - "rules_foreign_cc_framework_toolchain_freebsd": { - "repoRuleId": "@@rules_foreign_cc+//foreign_cc/private/framework:toolchain.bzl%framework_toolchain_repository", - "attributes": { - "commands_src": "@rules_foreign_cc//foreign_cc/private/framework/toolchains:freebsd_commands.bzl", - "exec_compatible_with": [ - "@platforms//os:freebsd" - ] - } - }, - "rules_foreign_cc_framework_toolchain_windows": { - "repoRuleId": "@@rules_foreign_cc+//foreign_cc/private/framework:toolchain.bzl%framework_toolchain_repository", - "attributes": { - "commands_src": "@rules_foreign_cc//foreign_cc/private/framework/toolchains:windows_commands.bzl", - "exec_compatible_with": [ - "@platforms//os:windows" - ] - } - }, - "rules_foreign_cc_framework_toolchain_macos": { - "repoRuleId": "@@rules_foreign_cc+//foreign_cc/private/framework:toolchain.bzl%framework_toolchain_repository", - "attributes": { - "commands_src": "@rules_foreign_cc//foreign_cc/private/framework/toolchains:macos_commands.bzl", - "exec_compatible_with": [ - "@platforms//os:macos" - ] - } - }, - "rules_foreign_cc_framework_toolchains": { - "repoRuleId": "@@rules_foreign_cc+//foreign_cc/private/framework:toolchain.bzl%framework_toolchain_repository_hub", - "attributes": {} - }, - "cmake_src": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "build_file_content": "filegroup(\n name = \"all_srcs\",\n srcs = glob([\"**\"]),\n visibility = [\"//visibility:public\"],\n)\n", - "sha256": "f316b40053466f9a416adf981efda41b160ca859e97f6a484b447ea299ff26aa", - "strip_prefix": "cmake-3.23.2", - "urls": [ - "https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2.tar.gz" - ] - } - }, - "gnumake_src": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "build_file_content": "filegroup(\n name = \"all_srcs\",\n srcs = glob([\"**\"]),\n visibility = [\"//visibility:public\"],\n)\n", - "sha256": "581f4d4e872da74b3941c874215898a7d35802f03732bdccee1d4a7979105d18", - "strip_prefix": "make-4.4", - "urls": [ - "https://mirror.bazel.build/ftpmirror.gnu.org/gnu/make/make-4.4.tar.gz", - "http://ftpmirror.gnu.org/gnu/make/make-4.4.tar.gz" - ] - } - }, - "ninja_build_src": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "build_file_content": "filegroup(\n name = \"all_srcs\",\n srcs = glob([\"**\"]),\n visibility = [\"//visibility:public\"],\n)\n", - "sha256": "31747ae633213f1eda3842686f83c2aa1412e0f5691d1c14dbbcc67fe7400cea", - "strip_prefix": "ninja-1.11.1", - "urls": [ - "https://github.com/ninja-build/ninja/archive/v1.11.1.tar.gz" - ] - } - }, - "meson_src": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "build_file_content": "exports_files([\"meson.py\"])\n\nfilegroup(\n name = \"runtime\",\n srcs = glob([\"mesonbuild/**\"]),\n visibility = [\"//visibility:public\"],\n)\n", - "strip_prefix": "meson-1.1.1", - "url": "https://github.com/mesonbuild/meson/releases/download/1.1.1/meson-1.1.1.tar.gz" - } - }, - "glib_dev": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "build_file_content": "\nload(\"@rules_cc//cc:defs.bzl\", \"cc_library\")\n\ncc_import(\n name = \"glib_dev\",\n hdrs = glob([\"include/**\"]),\n shared_library = \"@glib_runtime//:bin/libglib-2.0-0.dll\",\n visibility = [\"//visibility:public\"],\n)\n ", - "sha256": "bdf18506df304d38be98a4b3f18055b8b8cca81beabecad0eece6ce95319c369", - "urls": [ - "https://download.gnome.org/binaries/win64/glib/2.26/glib-dev_2.26.1-1_win64.zip" - ] - } - }, - "glib_src": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "build_file_content": "\ncc_import(\n name = \"msvc_hdr\",\n hdrs = [\"msvc_recommended_pragmas.h\"],\n visibility = [\"//visibility:public\"],\n)\n ", - "sha256": "bc96f63112823b7d6c9f06572d2ad626ddac7eb452c04d762592197f6e07898e", - "strip_prefix": "glib-2.26.1", - "urls": [ - "https://download.gnome.org/sources/glib/2.26/glib-2.26.1.tar.gz" - ] - } - }, - "glib_runtime": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "build_file_content": "\nexports_files(\n [\n \"bin/libgio-2.0-0.dll\",\n \"bin/libglib-2.0-0.dll\",\n \"bin/libgmodule-2.0-0.dll\",\n \"bin/libgobject-2.0-0.dll\",\n \"bin/libgthread-2.0-0.dll\",\n ],\n visibility = [\"//visibility:public\"],\n)\n ", - "sha256": "88d857087e86f16a9be651ee7021880b3f7ba050d34a1ed9f06113b8799cb973", - "urls": [ - "https://download.gnome.org/binaries/win64/glib/2.26/glib_2.26.1-1_win64.zip" - ] - } - }, - "gettext_runtime": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "build_file_content": "\ncc_import(\n name = \"gettext_runtime\",\n shared_library = \"bin/libintl-8.dll\",\n visibility = [\"//visibility:public\"],\n)\n ", - "sha256": "1f4269c0e021076d60a54e98da6f978a3195013f6de21674ba0edbc339c5b079", - "urls": [ - "https://download.gnome.org/binaries/win64/dependencies/gettext-runtime_0.18.1.1-2_win64.zip" - ] - } - }, - "pkgconfig_src": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "build_file_content": "filegroup(\n name = \"all_srcs\",\n srcs = glob([\"**\"]),\n visibility = [\"//visibility:public\"],\n)\n", - "sha256": "6fc69c01688c9458a57eb9a1664c9aba372ccda420a02bf4429fe610e7e7d591", - "strip_prefix": "pkg-config-0.29.2", - "patches": [ - "@@rules_foreign_cc+//toolchains:pkgconfig-detectenv.patch", - "@@rules_foreign_cc+//toolchains:pkgconfig-makefile-vc.patch" - ], - "urls": [ - "https://pkgconfig.freedesktop.org/releases/pkg-config-0.29.2.tar.gz" - ] - } - }, - "bazel_skylib": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.2.1/bazel-skylib-1.2.1.tar.gz", - "https://github.com/bazelbuild/bazel-skylib/releases/download/1.2.1/bazel-skylib-1.2.1.tar.gz" - ], - "sha256": "f7be3474d42aae265405a592bb7da8e171919d74c16f082a5457840f06054728" - } - }, - "rules_python": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "sha256": "84aec9e21cc56fbc7f1335035a71c850d1b9b5cc6ff497306f84cced9a769841", - "strip_prefix": "rules_python-0.23.1", - "url": "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.23.1.tar.gz" - } - }, - "cmake-3.23.2-linux-aarch64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-linux-aarch64.tar.gz" - ], - "sha256": "f2654bf780b53f170bbbec44d8ac67d401d24788e590faa53036a89476efa91e", - "strip_prefix": "cmake-3.23.2-linux-aarch64", - "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"cmake_data\",\n srcs = glob(\n [\n \"**\",\n ],\n exclude = [\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n \"BUILD\",\n \"BUILD.bazel\",\n ],\n ),\n)\n\nnative_tool_toolchain(\n name = \"cmake_tool\",\n path = \"bin/cmake\",\n target = \":cmake_data\",\n)\n" - } - }, - "cmake-3.23.2-linux-x86_64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-linux-x86_64.tar.gz" - ], - "sha256": "aaced6f745b86ce853661a595bdac6c5314a60f8181b6912a0a4920acfa32708", - "strip_prefix": "cmake-3.23.2-linux-x86_64", - "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"cmake_data\",\n srcs = glob(\n [\n \"**\",\n ],\n exclude = [\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n \"BUILD\",\n \"BUILD.bazel\",\n ],\n ),\n)\n\nnative_tool_toolchain(\n name = \"cmake_tool\",\n path = \"bin/cmake\",\n target = \":cmake_data\",\n)\n" - } - }, - "cmake-3.23.2-macos-universal": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-macos-universal.tar.gz" - ], - "sha256": "853a0f9af148c5ef47282ffffee06c4c9f257be2635936755f39ca13c3286c88", - "strip_prefix": "cmake-3.23.2-macos-universal/CMake.app/Contents", - "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"cmake_data\",\n srcs = glob(\n [\n \"**\",\n ],\n exclude = [\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n \"BUILD\",\n \"BUILD.bazel\",\n ],\n ),\n)\n\nnative_tool_toolchain(\n name = \"cmake_tool\",\n path = \"bin/cmake\",\n target = \":cmake_data\",\n)\n" - } - }, - "cmake-3.23.2-windows-i386": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-windows-i386.zip" - ], - "sha256": "6a4fcd6a2315b93cb23c93507efccacc30c449c2bf98f14d6032bb226c582e07", - "strip_prefix": "cmake-3.23.2-windows-i386", - "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"cmake_data\",\n srcs = glob(\n [\n \"**\",\n ],\n exclude = [\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n \"BUILD\",\n \"BUILD.bazel\",\n ],\n ),\n)\n\nnative_tool_toolchain(\n name = \"cmake_tool\",\n path = \"bin/cmake.exe\",\n target = \":cmake_data\",\n)\n" - } - }, - "cmake-3.23.2-windows-x86_64": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-windows-x86_64.zip" - ], - "sha256": "2329387f3166b84c25091c86389fb891193967740c9bcf01e7f6d3306f7ffda0", - "strip_prefix": "cmake-3.23.2-windows-x86_64", - "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"cmake_data\",\n srcs = glob(\n [\n \"**\",\n ],\n exclude = [\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n \"BUILD\",\n \"BUILD.bazel\",\n ],\n ),\n)\n\nnative_tool_toolchain(\n name = \"cmake_tool\",\n path = \"bin/cmake.exe\",\n target = \":cmake_data\",\n)\n" - } - }, - "cmake_3.23.2_toolchains": { - "repoRuleId": "@@rules_foreign_cc+//toolchains:prebuilt_toolchains_repository.bzl%prebuilt_toolchains_repository", - "attributes": { - "repos": { - "cmake-3.23.2-linux-aarch64": [ - "@platforms//cpu:aarch64", - "@platforms//os:linux" - ], - "cmake-3.23.2-linux-x86_64": [ - "@platforms//cpu:x86_64", - "@platforms//os:linux" - ], - "cmake-3.23.2-macos-universal": [ - "@platforms//os:macos" - ], - "cmake-3.23.2-windows-i386": [ - "@platforms//cpu:x86_32", - "@platforms//os:windows" - ], - "cmake-3.23.2-windows-x86_64": [ - "@platforms//cpu:x86_64", - "@platforms//os:windows" - ] - }, - "tool": "cmake" - } - }, - "ninja_1.11.1_linux": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-linux.zip" - ], - "sha256": "b901ba96e486dce377f9a070ed4ef3f79deb45f4ffe2938f8e7ddc69cfb3df77", - "strip_prefix": "", - "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"ninja_bin\",\n srcs = [\"ninja\"],\n)\n\nnative_tool_toolchain(\n name = \"ninja_tool\",\n env = {\"NINJA\": \"$(execpath :ninja_bin)\"},\n path = \"$(execpath :ninja_bin)\",\n target = \":ninja_bin\",\n)\n" - } - }, - "ninja_1.11.1_mac": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-mac.zip" - ], - "sha256": "482ecb23c59ae3d4f158029112de172dd96bb0e97549c4b1ca32d8fad11f873e", - "strip_prefix": "", - "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"ninja_bin\",\n srcs = [\"ninja\"],\n)\n\nnative_tool_toolchain(\n name = \"ninja_tool\",\n env = {\"NINJA\": \"$(execpath :ninja_bin)\"},\n path = \"$(execpath :ninja_bin)\",\n target = \":ninja_bin\",\n)\n" - } - }, - "ninja_1.11.1_win": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-win.zip" - ], - "sha256": "524b344a1a9a55005eaf868d991e090ab8ce07fa109f1820d40e74642e289abc", - "strip_prefix": "", - "build_file_content": "load(\"@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl\", \"native_tool_toolchain\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nfilegroup(\n name = \"ninja_bin\",\n srcs = [\"ninja.exe\"],\n)\n\nnative_tool_toolchain(\n name = \"ninja_tool\",\n env = {\"NINJA\": \"$(execpath :ninja_bin)\"},\n path = \"$(execpath :ninja_bin)\",\n target = \":ninja_bin\",\n)\n" - } - }, - "ninja_1.11.1_toolchains": { - "repoRuleId": "@@rules_foreign_cc+//toolchains:prebuilt_toolchains_repository.bzl%prebuilt_toolchains_repository", - "attributes": { - "repos": { - "ninja_1.11.1_linux": [ - "@platforms//cpu:x86_64", - "@platforms//os:linux" - ], - "ninja_1.11.1_mac": [ - "@platforms//cpu:x86_64", - "@platforms//os:macos" - ], - "ninja_1.11.1_win": [ - "@platforms//cpu:x86_64", - "@platforms//os:windows" - ] - }, - "tool": "ninja" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_foreign_cc+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_foreign_cc+", - "rules_foreign_cc", - "rules_foreign_cc+" - ] - ] - } - }, "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { "general": { "bzlTransitiveDigest": "AEsQvKwdstptzt98AO1bJYfoWpPUn2upx53ra6o/Rc8=", @@ -1723,5 +1249,436 @@ } } }, - "facts": {} + "facts": { + "@@rules_go+//go:extensions.bzl%go_sdk": { + "1.19.1": { + "darwin_amd64": [ + "go1.19.1.darwin-amd64.tar.gz", + "b2828a2b05f0d2169afc74c11ed010775bf7cf0061822b275697b2f470495fb7" + ], + "darwin_arm64": [ + "go1.19.1.darwin-arm64.tar.gz", + "e46aecce83a9289be16ce4ba9b8478a5b89b8aa0230171d5c6adbc0c66640548" + ], + "freebsd_386": [ + "go1.19.1.freebsd-386.tar.gz", + "cfaca8c1d5784d2bc21e12d8893cfd2dc885a60db4c1a9a95e4ffc694d0925ce" + ], + "freebsd_amd64": [ + "go1.19.1.freebsd-amd64.tar.gz", + "db5b8f232e12c655cc6cde6af1adf4d27d842541807802d747c86161e89efa0a" + ], + "linux_386": [ + "go1.19.1.linux-386.tar.gz", + "9acc57342400c5b0c2da07b5b01b50da239dd4a7fad41a1fb56af8363ef4133f" + ], + "linux_amd64": [ + "go1.19.1.linux-amd64.tar.gz", + "acc512fbab4f716a8f97a8b3fbaa9ddd39606a28be6c2515ef7c6c6311acffde" + ], + "linux_arm64": [ + "go1.19.1.linux-arm64.tar.gz", + "49960821948b9c6b14041430890eccee58c76b52e2dbaafce971c3c38d43df9f" + ], + "linux_armv6l": [ + "go1.19.1.linux-armv6l.tar.gz", + "efe93f5671621ee84ce5e262e1e21acbc72acefbaba360f21778abd083d4ad16" + ], + "linux_ppc64le": [ + "go1.19.1.linux-ppc64le.tar.gz", + "4137984aa353de9c5ec1bd8fb3cd00a0624b75eafa3d4ec13d2f3f48261dba2e" + ], + "linux_s390x": [ + "go1.19.1.linux-s390x.tar.gz", + "ca1005cc80a3dd726536b4c6ea5fef0318939351ff273eff420bd66a377c74eb" + ], + "windows_386": [ + "go1.19.1.windows-386.zip", + "bc7043e7a9a8d34aacd06f8c2f70e166d1d148f6800814cff790c45b9ab31cee" + ], + "windows_amd64": [ + "go1.19.1.windows-amd64.zip", + "b33584c1d93b0e9c783de876b7aa99d3018bdeccd396aeb6d516a74e9d88d55f" + ], + "windows_arm64": [ + "go1.19.1.windows-arm64.zip", + "d8cf3f04762fa7d5d9c82dfa15b5adaae2404463af3bc8dcd7f89837512501fe" + ] + }, + "1.20.2": { + "darwin_amd64": [ + "go1.20.2.darwin-amd64.tar.gz", + "c93b8ced9517d07e1cd4c362c6e2d5242cb139e29b417a328fbf19aded08764c" + ], + "darwin_arm64": [ + "go1.20.2.darwin-arm64.tar.gz", + "7343c87f19e79c0063532e82e1c4d6f42175a32d99f7a4d15e658e88bf97f885" + ], + "freebsd_386": [ + "go1.20.2.freebsd-386.tar.gz", + "14f9be2004e042b3a64d0facb0c020756a9084a5c7333e33b0752b393b6016ea" + ], + "freebsd_amd64": [ + "go1.20.2.freebsd-amd64.tar.gz", + "b41b67b4f1b56797a7cecf6ee7f47fcf4f93960b2788a3683c07dd009d30b2a4" + ], + "linux_386": [ + "go1.20.2.linux-386.tar.gz", + "ee240ed33ae57504c41f04c12236aeaa17fbeb6ea9fcd096cd9dc7a89d10d4db" + ], + "linux_amd64": [ + "go1.20.2.linux-amd64.tar.gz", + "4eaea32f59cde4dc635fbc42161031d13e1c780b87097f4b4234cfce671f1768" + ], + "linux_arm64": [ + "go1.20.2.linux-arm64.tar.gz", + "78d632915bb75e9a6356a47a42625fd1a785c83a64a643fedd8f61e31b1b3bef" + ], + "linux_armv6l": [ + "go1.20.2.linux-armv6l.tar.gz", + "d79d56bafd6b52b8d8cbe3f8e967caaac5383a23d7a4fa9ac0e89778cd16a076" + ], + "linux_ppc64le": [ + "go1.20.2.linux-ppc64le.tar.gz", + "850564ddb760cb703db63bf20182dc4407abd2ff090a95fa66d6634d172fd095" + ], + "linux_s390x": [ + "go1.20.2.linux-s390x.tar.gz", + "8da24c5c4205fe8115f594237e5db7bcb1d23df67bc1fa9a999954b1976896e8" + ], + "windows_386": [ + "go1.20.2.windows-386.zip", + "31838b291117495bbb93683603e98d5118bfabd2eb318b4d07540bfd524bab86" + ], + "windows_amd64": [ + "go1.20.2.windows-amd64.zip", + "fe439f0e438f7555a7f5f7194ddb6f4a07b0de1fa414385d19f2aeb26d9f43db" + ], + "windows_arm64": [ + "go1.20.2.windows-arm64.zip", + "ac5010c8b8b22849228a8dea698d58b9c7be2195d30c6d778cce0f709858fa64" + ] + }, + "1.21.1": { + "aix_ppc64": [ + "go1.21.1.aix-ppc64.tar.gz", + "ad5c160e2ff7989f818572f2391ca000d587569ff68d8591a695382a863719d0" + ], + "darwin_amd64": [ + "go1.21.1.darwin-amd64.tar.gz", + "809f5b0ef4f7dcdd5f51e9630a5b2e5a1006f22a047126d61560cdc365678a19" + ], + "darwin_arm64": [ + "go1.21.1.darwin-arm64.tar.gz", + "ffd40391a1e995855488b008ad9326ff8c2e81803a6e80894401003bae47fcf1" + ], + "dragonfly_amd64": [ + "go1.21.1.dragonfly-amd64.tar.gz", + "646f5db8ba479032aea07c597ccd187bc75c161f18e7a1785c62ab8cb103cb3e" + ], + "freebsd_386": [ + "go1.21.1.freebsd-386.tar.gz", + "9919a9a4dc82371aba3da5b7c830bcb6249fc1502cd26d959eb340a60e41ee01" + ], + "freebsd_amd64": [ + "go1.21.1.freebsd-amd64.tar.gz", + "2571f10f6047e04d87c1f5986a05e5e8f7b511faf98803ef12b66d563845d2a1" + ], + "freebsd_arm64": [ + "go1.21.1.freebsd-arm64.tar.gz", + "4946118e72deccf194a9bfcfedbafe52e45b5fe9441fd3acfced760b27a20f9a" + ], + "freebsd_armv6l": [ + "go1.21.1.freebsd-arm.tar.gz", + "6d4dab314ed6adff68a74e3bce516e36fd89c7bf4523c2c105f96e04933fc982" + ], + "freebsd_riscv64": [ + "go1.21.1.freebsd-riscv64.tar.gz", + "591bd28e2dac581facb20c1529f8c29590d7eaaccbcdc93d12a234461b7bfe89" + ], + "illumos_amd64": [ + "go1.21.1.illumos-amd64.tar.gz", + "829548b591675df5f113317ac011bfc3381da7a96bd6a1c748694370818b4497" + ], + "linux_386": [ + "go1.21.1.linux-386.tar.gz", + "b93850666cdadbd696a986cf7b03111fe99db8c34a9aaa113d7c96d0081e1901" + ], + "linux_amd64": [ + "go1.21.1.linux-amd64.tar.gz", + "b3075ae1ce5dab85f89bc7905d1632de23ca196bd8336afd93fa97434cfa55ae" + ], + "linux_arm64": [ + "go1.21.1.linux-arm64.tar.gz", + "7da1a3936a928fd0b2602ed4f3ef535b8cd1990f1503b8d3e1acc0fa0759c967" + ], + "linux_armv6l": [ + "go1.21.1.linux-armv6l.tar.gz", + "f3716a43f59ae69999841d6007b42c9e286e8d8ce470656fb3e70d7be2d7ca85" + ], + "linux_loong64": [ + "go1.21.1.linux-loong64.tar.gz", + "2273cc4ed398d8b341aee01fd62f3845045b2f439b84b28720a7f3d3552f5d41" + ], + "linux_mips": [ + "go1.21.1.linux-mips.tar.gz", + "2ba529210bae7ba7b8c48b0084f713b41e8c109ae75637820fd5b72be9f3faff" + ], + "linux_mips64": [ + "go1.21.1.linux-mips64.tar.gz", + "f2cef18856b17cc5c81cc8b0a35b1ca08d2b04d4b88adc07d190560cb464e2f6" + ], + "linux_mips64le": [ + "go1.21.1.linux-mips64le.tar.gz", + "3aa007a41f533b50eae2491bbd29926ada09357367a8aa05e7e50ec50c78acf9" + ], + "linux_mipsle": [ + "go1.21.1.linux-mipsle.tar.gz", + "c0059f49c3ea31bebe972f47d460d49addb78641081a7000e79e730464bcce95" + ], + "linux_ppc64": [ + "go1.21.1.linux-ppc64.tar.gz", + "6871fd3e6dae3897112dc4a98988406461ef1018f4a17c87689d977eba8dbf20" + ], + "linux_ppc64le": [ + "go1.21.1.linux-ppc64le.tar.gz", + "eddf018206f8a5589bda75252b72716d26611efebabdca5d0083ec15e9e41ab7" + ], + "linux_riscv64": [ + "go1.21.1.linux-riscv64.tar.gz", + "fac64ed26e003f49f1d77f6d2c4cf951422aecbce12232d9ec1bf4585fc44ee1" + ], + "linux_s390x": [ + "go1.21.1.linux-s390x.tar.gz", + "a83b3e8eb4dbf76294e773055eb51397510ff4d612a247bad9903560267bba6d" + ], + "netbsd_386": [ + "go1.21.1.netbsd-386.tar.gz", + "c919f2ba4755e6aebeea6e7f19abf1c5ed13d37c2930867d10a86adfd0686eba" + ], + "netbsd_amd64": [ + "go1.21.1.netbsd-amd64.tar.gz", + "b2dd9913063caa2b5f960e8ebc6c225293b4155e9ebd88d43afd55bb81ca8931" + ], + "netbsd_arm64": [ + "go1.21.1.netbsd-arm64.tar.gz", + "5d245ac932d421389c1bfcedc38733367ae407dbdc162a6e7bfca5677fb15269" + ], + "netbsd_armv6l": [ + "go1.21.1.netbsd-arm.tar.gz", + "4d1756a9845917b07c8ec4999b8e019a2bd2a8b3d5d16da5da0ef3ce84d2139c" + ], + "openbsd_386": [ + "go1.21.1.openbsd-386.tar.gz", + "ce094e551227522aaf3111a94a7f115c5a2b4f862c42d349026503544a40fb96" + ], + "openbsd_amd64": [ + "go1.21.1.openbsd-amd64.tar.gz", + "321c31c42131db2bd0e9d67dd98450f55b281e8e6d8eb681594c39e6688ccbb2" + ], + "openbsd_arm64": [ + "go1.21.1.openbsd-arm64.tar.gz", + "4d65bcb383895be8d6c05507f2b410e15448b0610e1f5fb23ece9ff4e045f7a8" + ], + "openbsd_armv6l": [ + "go1.21.1.openbsd-arm.tar.gz", + "481254b99b9861b65c9eaab137af15311fca618f9f17f2bd872795368c726666" + ], + "plan9_386": [ + "go1.21.1.plan9-386.tar.gz", + "93ea25cbb8063cd4d5917b8ca58a72c26b6cb9e6c1d785e30867ae8ad8290484" + ], + "plan9_amd64": [ + "go1.21.1.plan9-amd64.tar.gz", + "14d61633059651d12cc02b2f53739016f1b07063a4140147e44a196413950553" + ], + "plan9_armv6l": [ + "go1.21.1.plan9-arm.tar.gz", + "adfacbcfb5e7415802187fb6aa8b5397d589fdedcd5d0eac3b86352da7390603" + ], + "solaris_amd64": [ + "go1.21.1.solaris-amd64.tar.gz", + "b86a223e52e41de3ff180f90f1d7e705d30f96271f9a81005e20f25be2658990" + ], + "windows_386": [ + "go1.21.1.windows-386.zip", + "170256c820f466f29d64876f25f4dfa4029ed9902a0a9095d8bd603aecf4d83b" + ], + "windows_amd64": [ + "go1.21.1.windows-amd64.zip", + "10a4f5b63215d11d1770453733dbcbf024f3f74872f84e28d7ea59f0250316c6" + ], + "windows_arm64": [ + "go1.21.1.windows-arm64.zip", + "41135ce6e0ced4bc1e459cb96bd4090c9dc2062e24179c3f337d855af9b560ef" + ], + "windows_armv6l": [ + "go1.21.1.windows-arm.zip", + "d7a2472999a80dbf6bbcb6783a2b4299916055473263117295022f18e7fad487" + ] + }, + "1.25.0": { + "aix_ppc64": [ + "go1.25.0.aix-ppc64.tar.gz", + "e5234a7dac67bc86c528fe9752fc9d63557918627707a733ab4cac1a6faed2d4" + ], + "darwin_amd64": [ + "go1.25.0.darwin-amd64.tar.gz", + "5bd60e823037062c2307c71e8111809865116714d6f6b410597cf5075dfd80ef" + ], + "darwin_arm64": [ + "go1.25.0.darwin-arm64.tar.gz", + "544932844156d8172f7a28f77f2ac9c15a23046698b6243f633b0a0b00c0749c" + ], + "dragonfly_amd64": [ + "go1.25.0.dragonfly-amd64.tar.gz", + "5ed3cf9a810a1483822538674f1336c06b51aa1b94d6d545a1a0319a48177120" + ], + "freebsd_386": [ + "go1.25.0.freebsd-386.tar.gz", + "abea5d5c6697e6b5c224731f2158fe87c602996a2a233ac0c4730cd57bf8374e" + ], + "freebsd_amd64": [ + "go1.25.0.freebsd-amd64.tar.gz", + "86e6fe0a29698d7601c4442052dac48bd58d532c51cccb8f1917df648138730b" + ], + "freebsd_arm": [ + "go1.25.0.freebsd-arm.tar.gz", + "d90b78e41921f72f30e8bbc81d9dec2cff7ff384a33d8d8debb24053e4336bfe" + ], + "freebsd_arm64": [ + "go1.25.0.freebsd-arm64.tar.gz", + "451d0da1affd886bfb291b7c63a6018527b269505db21ce6e14724f22ab0662e" + ], + "freebsd_riscv64": [ + "go1.25.0.freebsd-riscv64.tar.gz", + "7b565f76bd8bda46549eeaaefe0e53b251e644c230577290c0f66b1ecdb3cdbe" + ], + "illumos_amd64": [ + "go1.25.0.illumos-amd64.tar.gz", + "b1e1fdaab1ad25aa1c08d7a36c97d45d74b98b89c3f78c6d2145f77face54a2c" + ], + "linux_386": [ + "go1.25.0.linux-386.tar.gz", + "8c602dd9d99bc9453b3995d20ce4baf382cc50855900a0ece5de9929df4a993a" + ], + "linux_amd64": [ + "go1.25.0.linux-amd64.tar.gz", + "2852af0cb20a13139b3448992e69b868e50ed0f8a1e5940ee1de9e19a123b613" + ], + "linux_arm64": [ + "go1.25.0.linux-arm64.tar.gz", + "05de75d6994a2783699815ee553bd5a9327d8b79991de36e38b66862782f54ae" + ], + "linux_armv6l": [ + "go1.25.0.linux-armv6l.tar.gz", + "a5a8f8198fcf00e1e485b8ecef9ee020778bf32a408a4e8873371bfce458cd09" + ], + "linux_loong64": [ + "go1.25.0.linux-loong64.tar.gz", + "cab86b1cf761b1cb3bac86a8877cfc92e7b036fc0d3084123d77013d61432afc" + ], + "linux_mips": [ + "go1.25.0.linux-mips.tar.gz", + "d66b6fb74c3d91b9829dc95ec10ca1f047ef5e89332152f92e136cf0e2da5be1" + ], + "linux_mips64": [ + "go1.25.0.linux-mips64.tar.gz", + "4082e4381a8661bc2a839ff94ba3daf4f6cde20f8fb771b5b3d4762dc84198a2" + ], + "linux_mips64le": [ + "go1.25.0.linux-mips64le.tar.gz", + "70002c299ec7f7175ac2ef673b1b347eecfa54ae11f34416a6053c17f855afcc" + ], + "linux_mipsle": [ + "go1.25.0.linux-mipsle.tar.gz", + "b00a3a39eff099f6df9f1c7355bf28e4589d0586f42d7d4a394efb763d145a73" + ], + "linux_ppc64": [ + "go1.25.0.linux-ppc64.tar.gz", + "df166f33bd98160662560a72ff0b4ba731f969a80f088922bddcf566a88c1ec1" + ], + "linux_ppc64le": [ + "go1.25.0.linux-ppc64le.tar.gz", + "0f18a89e7576cf2c5fa0b487a1635d9bcbf843df5f110e9982c64df52a983ad0" + ], + "linux_riscv64": [ + "go1.25.0.linux-riscv64.tar.gz", + "c018ff74a2c48d55c8ca9b07c8e24163558ffec8bea08b326d6336905d956b67" + ], + "linux_s390x": [ + "go1.25.0.linux-s390x.tar.gz", + "34e5a2e19f2292fbaf8783e3a241e6e49689276aef6510a8060ea5ef54eee408" + ], + "netbsd_386": [ + "go1.25.0.netbsd-386.tar.gz", + "f8586cdb7aa855657609a5c5f6dbf523efa00c2bbd7c76d3936bec80aa6c0aba" + ], + "netbsd_amd64": [ + "go1.25.0.netbsd-amd64.tar.gz", + "ae8dc1469385b86a157a423bb56304ba45730de8a897615874f57dd096db2c2a" + ], + "netbsd_arm": [ + "go1.25.0.netbsd-arm.tar.gz", + "1ff7e4cc764425fc9dd6825eaee79d02b3c7cafffbb3691687c8d672ade76cb7" + ], + "netbsd_arm64": [ + "go1.25.0.netbsd-arm64.tar.gz", + "e1b310739f26724216aa6d7d7208c4031f9ff54c9b5b9a796ddc8bebcb4a5f16" + ], + "openbsd_386": [ + "go1.25.0.openbsd-386.tar.gz", + "4802a9b20e533da91adb84aab42e94aa56cfe3e5475d0550bed3385b182e69d8" + ], + "openbsd_amd64": [ + "go1.25.0.openbsd-amd64.tar.gz", + "c016cd984bebe317b19a4f297c4f50def120dc9788490540c89f28e42f1dabe1" + ], + "openbsd_arm": [ + "go1.25.0.openbsd-arm.tar.gz", + "a1e31d0bf22172ddde42edf5ec811ef81be43433df0948ece52fecb247ccfd8d" + ], + "openbsd_arm64": [ + "go1.25.0.openbsd-arm64.tar.gz", + "343ea8edd8c218196e15a859c6072d0dd3246fbbb168481ab665eb4c4140458d" + ], + "openbsd_ppc64": [ + "go1.25.0.openbsd-ppc64.tar.gz", + "694c14da1bcaeb5e3332d49bdc2b6d155067648f8fe1540c5de8f3cf8e157154" + ], + "openbsd_riscv64": [ + "go1.25.0.openbsd-riscv64.tar.gz", + "aa510ad25cf54c06cd9c70b6d80ded69cb20188ac6e1735655eef29ff7e7885f" + ], + "plan9_386": [ + "go1.25.0.plan9-386.tar.gz", + "46f8cef02086cf04bf186c5912776b56535178d4cb319cd19c9fdbdd29231986" + ], + "plan9_amd64": [ + "go1.25.0.plan9-amd64.tar.gz", + "29b34391d84095e44608a228f63f2f88113a37b74a79781353ec043dfbcb427b" + ], + "plan9_arm": [ + "go1.25.0.plan9-arm.tar.gz", + "0a047107d13ebe7943aaa6d54b1d7bbd2e45e68ce449b52915a818da715799c2" + ], + "solaris_amd64": [ + "go1.25.0.solaris-amd64.tar.gz", + "9977f9e4351984364a3b2b78f8b88bfd1d339812356d5237678514594b7d3611" + ], + "windows_386": [ + "go1.25.0.windows-386.zip", + "df9f39db82a803af0db639e3613a36681ab7a42866b1384b3f3a1045663961a7" + ], + "windows_amd64": [ + "go1.25.0.windows-amd64.zip", + "89efb4f9b30812eee083cc1770fdd2913c14d301064f6454851428f9707d190b" + ], + "windows_arm64": [ + "go1.25.0.windows-arm64.zip", + "27bab004c72b3d7bd05a69b6ec0fc54a309b4b78cc569dd963d8b3ec28bfdb8c" + ] + } + } + } } diff --git a/bazel/cc_import.bzl b/bazel/cc_import.bzl index 7c0e8de5..09d3f2e1 100644 --- a/bazel/cc_import.bzl +++ b/bazel/cc_import.bzl @@ -1,4 +1,5 @@ load("@bazel_tools//tools/build_defs/cc:cc_import.bzl", _cc_import = "cc_import") +load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") load(":patchelf.bzl", "patchelf") def _cc_import_runfiles_impl(ctx): diff --git a/bazel/swig.bzl b/bazel/swig.bzl index d6274860..cd4f994f 100644 --- a/bazel/swig.bzl +++ b/bazel/swig.bzl @@ -1,6 +1,7 @@ load("@rules_cc//cc:action_names.bzl", "C_COMPILE_ACTION_NAME") load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_cc//cc:defs.bzl", "CcInfo", "cc_common") +load("@rules_cc//cc/common:cc_common.bzl", "cc_common") +load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") load("@rules_cc//cc:find_cc_toolchain.bzl", "find_cc_toolchain", "use_cc_toolchain") def _swig_cc_library_impl(ctx): diff --git a/third_party/xla/xla.bzl b/third_party/xla/xla.bzl index d324abc2..84301e07 100644 --- a/third_party/xla/xla.bzl +++ b/third_party/xla/xla.bzl @@ -111,10 +111,13 @@ filegroup( ) """, "crosstool/BUILD.bazel": "", - "sycl/build_defs.bzl": _BZL_HELPERS + """\ + "sycl/build_defs.bzl": """\ +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +""" + _BZL_HELPERS + """\ if_sycl = always_if_false if_sycl_is_configured = always_if_false -sycl_library = native.cc_library +sycl_library = cc_library """, }) simple_files(name = "local_config_remote_execution", files = { diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 41d8092c..f1483aa0 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -1,3 +1,4 @@ +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") load("@rules_python//python:py_library.bzl", "py_library") py_library( From e67f60fe25fae18dcd2337014763cf8537716563 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Tue, 12 May 2026 10:32:51 +0200 Subject: [PATCH 478/881] workspace: remove custom registry (#531) Unused and noisy --- .bazelignore | 1 - .../aspect_bazel_lib/2.8.1.1/MODULE.bazel | 78 - ...-fix-add-repo-mapping-to-tar-archive.patch | 50 - .../2.8.1.1/patches/go_dev_dep.patch | 27 - .../patches/module_dot_bazel_version.patch | 14 - .../aspect_bazel_lib/2.8.1.1/source.json | 11 - .../modules/aspect_bazel_lib/metadata.json | 25 - .../libxev/20240825.0-dbe2291/MODULE.bazel | 7 - .../20240825.0-dbe2291/overlay/BUILD.bazel | 13 - .../20240825.0-dbe2291/overlay/MODULE.bazel | 1 - .../libxev/20240825.0-dbe2291/source.json | 9 - .../libxev/20240910.0-a2d9b31/MODULE.bazel | 7 - .../20240910.0-a2d9b31/overlay/BUILD.bazel | 13 - .../20240910.0-a2d9b31/overlay/MODULE.bazel | 1 - .../libxev/20240910.0-a2d9b31/source.json | 9 - .../libxev/20241119.0-6afcde9/MODULE.bazel | 7 - .../20241119.0-6afcde9/overlay/BUILD.bazel | 13 - .../20241119.0-6afcde9/overlay/MODULE.bazel | 1 - .../20241119.0-6afcde9/overlay/main2.zig | 22 - .../20241119.0-6afcde9/patches/128.patch | 119 -- .../libxev/20241119.0-6afcde9/source.json | 14 - .../libxev/20241208.0-db6a52b/MODULE.bazel | 7 - .../20241208.0-db6a52b/overlay/BUILD.bazel | 13 - .../20241208.0-db6a52b/overlay/MODULE.bazel | 7 - .../20241208.0-db6a52b/overlay/main2.zig | 22 - .../20241208.0-db6a52b/patches/128.patch | 348 ---- .../libxev/20241208.0-db6a52b/source.json | 14 - .../libxev/20241208.1-db6a52b/MODULE.bazel | 7 - .../20241208.1-db6a52b/overlay/BUILD.bazel | 13 - .../20241208.1-db6a52b/overlay/MODULE.bazel | 7 - .../20241208.1-db6a52b/overlay/main2.zig | 22 - .../20241208.1-db6a52b/patches/128.patch | 323 --- .../libxev/20241208.1-db6a52b/source.json | 14 - .../libxev/20241208.2-db6a52b/MODULE.bazel | 7 - .../20241208.2-db6a52b/overlay/BUILD.bazel | 13 - .../20241208.2-db6a52b/overlay/MODULE.bazel | 7 - .../20241208.2-db6a52b/overlay/main2.zig | 22 - .../20241208.2-db6a52b/patches/128.patch | 359 ---- .../libxev/20241208.2-db6a52b/source.json | 14 - .../libxev/20250124.0-31eed4e/MODULE.bazel | 7 - .../20250124.0-31eed4e/overlay/BUILD.bazel | 13 - .../20250124.0-31eed4e/overlay/MODULE.bazel | 1 - .../20250124.0-31eed4e/overlay/main2.zig | 22 - .../20250124.0-31eed4e/patches/128.patch | 359 ---- .../20250124.0-31eed4e/patches/141.patch | 35 - .../libxev/20250124.0-31eed4e/source.json | 15 - .../libxev/20250222.0-07bcffa/MODULE.bazel | 7 - .../20250222.0-07bcffa/overlay/BUILD.bazel | 13 - .../20250222.0-07bcffa/overlay/MODULE.bazel | 1 - .../libxev/20250222.0-07bcffa/source.json | 9 - .../libxev/20250313.0-5773f46/MODULE.bazel | 7 - .../20250313.0-5773f46/overlay/BUILD.bazel | 13 - .../20250313.0-5773f46/overlay/MODULE.bazel | 1 - .../libxev/20250313.0-5773f46/source.json | 10 - .../libxev/20250718.0-9f785d2/MODULE.bazel | 1 - .../20250718.0-9f785d2/overlay/BUILD.bazel | 13 - .../20250718.0-9f785d2/overlay/MODULE.bazel | 7 - .../libxev/20250718.0-9f785d2/source.json | 11 - .../libxev/20251010.0-9f785d2/MODULE.bazel | 1 - .../20251010.0-9f785d2/overlay/BUILD.bazel | 13 - .../20251010.0-9f785d2/overlay/MODULE.bazel | 7 - .../libxev/20251010.0-9f785d2/source.json | 11 - third_party/modules/libxev/metadata.json | 27 - .../llvm-raw/20240823.0-f142f8a/MODULE.bazel | 10 - .../20240823.0-f142f8a/overlay/BUILD.bazel | 0 .../20240823.0-f142f8a/overlay/MODULE.bazel | 1 - .../overlay/utils/bazel/extension.bzl | 28 - .../llvm-raw/20240823.0-f142f8a/source.json | 10 - .../llvm-raw/20240919.0-94c024a/MODULE.bazel | 10 - .../20240919.0-94c024a/overlay/BUILD.bazel | 0 .../20240919.0-94c024a/overlay/MODULE.bazel | 1 - .../overlay/utils/bazel/extension.bzl | 28 - .../llvm-raw/20240919.0-94c024a/source.json | 10 - .../llvm-raw/20241022.0-6c4267f/MODULE.bazel | 11 - .../20241022.0-6c4267f/overlay/BUILD.bazel | 0 .../20241022.0-6c4267f/overlay/MODULE.bazel | 1 - .../overlay/utils/bazel/extension.bzl | 28 - .../llvm-raw/20241022.0-6c4267f/source.json | 10 - .../llvm-raw/20250102.0-f739aa4/MODULE.bazel | 11 - .../20250102.0-f739aa4/overlay/BUILD.bazel | 0 .../20250102.0-f739aa4/overlay/MODULE.bazel | 1 - .../overlay/utils/bazel/extension.bzl | 28 - .../llvm-raw/20250102.0-f739aa4/source.json | 10 - .../llvm-raw/20250117.0-bf17016/MODULE.bazel | 11 - .../20250117.0-bf17016/overlay/BUILD.bazel | 0 .../20250117.0-bf17016/overlay/MODULE.bazel | 1 - .../overlay/utils/bazel/extension.bzl | 28 - .../llvm-raw/20250117.0-bf17016/source.json | 10 - .../llvm-raw/20250201.0-386af4a/MODULE.bazel | 11 - .../20250201.0-386af4a/overlay/BUILD.bazel | 0 .../20250201.0-386af4a/overlay/MODULE.bazel | 1 - .../overlay/utils/bazel/extension.bzl | 28 - .../llvm-raw/20250201.0-386af4a/source.json | 10 - .../llvm-raw/20250217.0-0e779ad/MODULE.bazel | 11 - .../20250217.0-0e779ad/overlay/BUILD.bazel | 0 .../20250217.0-0e779ad/overlay/MODULE.bazel | 1 - .../overlay/utils/bazel/extension.bzl | 28 - .../llvm-raw/20250217.0-0e779ad/source.json | 10 - third_party/modules/llvm-raw/metadata.json | 22 - .../rules_zig/20240904.0-010da15/MODULE.bazel | 68 - .../rules_zig/20240904.0-010da15/source.json | 5 - .../rules_zig/20240909.0-37f17ff/MODULE.bazel | 68 - .../rules_zig/20240909.0-37f17ff/source.json | 5 - .../rules_zig/20240912.0-41bfe84/MODULE.bazel | 68 - .../rules_zig/20240912.0-41bfe84/source.json | 5 - .../rules_zig/20240913.0-1957d05/MODULE.bazel | 68 - .../rules_zig/20240913.0-1957d05/source.json | 5 - .../rules_zig/20250314.0-b9739c6/MODULE.bazel | 68 - .../rules_zig/20250314.0-b9739c6/source.json | 5 - .../rules_zig/20250519.0-233b207/MODULE.bazel | 68 - .../rules_zig/20250519.0-233b207/source.json | 5 - .../rules_zig/20250530.0-5084f1f/MODULE.bazel | 68 - .../rules_zig/20250530.0-5084f1f/source.json | 5 - .../rules_zig/20250613.0-567662a/MODULE.bazel | 68 - .../rules_zig/20250613.0-567662a/source.json | 5 - .../rules_zig/20250714.0-b14a4f1/MODULE.bazel | 68 - .../rules_zig/20250714.0-b14a4f1/source.json | 5 - .../rules_zig/20250821.0-be53625/MODULE.bazel | 75 - .../rules_zig/20250821.0-be53625/source.json | 5 - .../rules_zig/20250827.0-35b6d57/MODULE.bazel | 75 - .../rules_zig/20250827.0-35b6d57/source.json | 5 - third_party/modules/rules_zig/metadata.json | 24 - .../20240618.0-d7ace0a/MODULE.bazel | 7 - .../20240618.0-d7ace0a/overlay/BUILD.bazel | 7 - .../20240618.0-d7ace0a/overlay/MODULE.bazel | 1 - .../20240618.0-d7ace0a/source.json | 9 - .../modules/sentencepiece/metadata.json | 17 - .../stablehlo/20240829.0-54aa1a5/MODULE.bazel | 15 - .../20240829.0-54aa1a5/overlay/MODULE.bazel | 1 - ...duplicated-symbols-in-StablehloApi.h.patch | 83 - .../patches/temporary.patch | 401 ---- .../stablehlo/20240829.0-54aa1a5/source.json | 12 - .../stablehlo/20240917.0-78c753a/MODULE.bazel | 15 - .../20240917.0-78c753a/overlay/MODULE.bazel | 1 - .../stablehlo/20240917.0-78c753a/source.json | 8 - .../stablehlo/20241021.0-1c0b606/MODULE.bazel | 15 - .../20241021.0-1c0b606/overlay/MODULE.bazel | 1 - .../stablehlo/20241021.0-1c0b606/source.json | 8 - .../stablehlo/20241220.0-38bb2f9/MODULE.bazel | 15 - .../20241220.0-38bb2f9/overlay/MODULE.bazel | 1 - .../stablehlo/20241220.0-38bb2f9/source.json | 8 - .../stablehlo/20250117.0-c125b32/MODULE.bazel | 15 - .../20250117.0-c125b32/overlay/MODULE.bazel | 1 - .../stablehlo/20250117.0-c125b32/source.json | 8 - .../stablehlo/20250201.0-7775e3e/MODULE.bazel | 15 - .../20250201.0-7775e3e/overlay/MODULE.bazel | 1 - .../stablehlo/20250201.0-7775e3e/source.json | 8 - .../stablehlo/20250217.0-4598975/MODULE.bazel | 15 - .../20250217.0-4598975/overlay/MODULE.bazel | 1 - .../stablehlo/20250217.0-4598975/source.json | 8 - third_party/modules/stablehlo/metadata.json | 22 - .../xla/20240902.0-d18cd64/MODULE.bazel | 34 - .../20240902.0-d18cd64/overlay/MODULE.bazel | 1 - .../xla/20240902.0-d18cd64/overlay/tsl.bzl | 13 - .../20240902.0-d18cd64/overlay/workspace.bzl | 52 - ...e-C-compliance-for-Profiler-Extensio.patch | 27 - .../xla/20240902.0-d18cd64/source.json | 14 - .../xla/20240919.0-1b18dd6/MODULE.bazel | 34 - .../20240919.0-1b18dd6/overlay/MODULE.bazel | 1 - .../xla/20240919.0-1b18dd6/overlay/tsl.bzl | 13 - .../20240919.0-1b18dd6/overlay/workspace.bzl | 52 - ...e-C-compliance-for-Profiler-Extensio.patch | 27 - .../xla/20240919.0-1b18dd6/source.json | 14 - .../xla/20241025.0-4663f04/MODULE.bazel | 34 - .../20241025.0-4663f04/overlay/MODULE.bazel | 1 - .../xla/20241025.0-4663f04/overlay/tsl.bzl | 19 - .../20241025.0-4663f04/overlay/workspace.bzl | 52 - ...e-C-compliance-for-Profiler-Extensio.patch | 27 - .../xla/20241025.0-4663f04/source.json | 14 - .../xla/20250103.0-5f1fe6a/MODULE.bazel | 34 - .../20250103.0-5f1fe6a/overlay/MODULE.bazel | 1 - .../xla/20250103.0-5f1fe6a/overlay/tsl.bzl | 19 - .../20250103.0-5f1fe6a/overlay/workspace.bzl | 52 - ...e-C-compliance-for-Profiler-Extensio.patch | 27 - .../xla/20250103.0-5f1fe6a/source.json | 14 - .../xla/20250122.0-cc075be/MODULE.bazel | 34 - .../20250122.0-cc075be/overlay/MODULE.bazel | 1 - .../xla/20250122.0-cc075be/overlay/tsl.bzl | 19 - .../20250122.0-cc075be/overlay/workspace.bzl | 52 - ...e-C-compliance-for-Profiler-Extensio.patch | 27 - .../xla/20250122.0-cc075be/source.json | 14 - .../xla/20250123.0-cc075be/MODULE.bazel | 34 - .../20250123.0-cc075be/overlay/MODULE.bazel | 1 - .../xla/20250123.0-cc075be/overlay/tsl.bzl | 19 - .../20250123.0-cc075be/overlay/workspace.bzl | 52 - .../0001-bazel-migration-to-bazel-8.0.1.patch | 42 - ...e-C-compliance-for-Profiler-Extensio.patch | 27 - .../xla/20250123.0-cc075be/source.json | 15 - .../xla/20250204-0-6789523/MODULE.bazel | 34 - .../20250204-0-6789523/overlay/MODULE.bazel | 1 - .../xla/20250204-0-6789523/overlay/tsl.bzl | 19 - .../20250204-0-6789523/overlay/workspace.bzl | 52 - .../0001-bazel-migration-to-bazel-8.0.1.patch | 41 - ...e-C-compliance-for-Profiler-Extensio.patch | 26 - .../xla/20250204-0-6789523/source.json | 15 - .../xla/20250204.1-6789523/MODULE.bazel | 34 - .../20250204.1-6789523/overlay/MODULE.bazel | 1 - .../xla/20250204.1-6789523/overlay/tsl.bzl | 19 - .../20250204.1-6789523/overlay/workspace.bzl | 52 - .../0001-bazel-migration-to-bazel-8.0.1.patch | 41 - ...e-C-compliance-for-Profiler-Extensio.patch | 26 - .../xla/20250204.1-6789523/source.json | 15 - .../xla/20250317.0-71c67e2/MODULE.bazel | 37 - .../20250317.0-71c67e2/overlay/MODULE.bazel | 1 - .../xla/20250317.0-71c67e2/overlay/tsl.bzl | 19 - .../20250317.0-71c67e2/overlay/workspace.bzl | 60 - .../0001-bazel-migration-to-bazel-8.1.1.patch | 41 - .../xla/20250317.0-71c67e2/source.json | 14 - .../xla/20250317.1-71c67e2/MODULE.bazel | 37 - .../20250317.1-71c67e2/overlay/MODULE.bazel | 1 - .../xla/20250317.1-71c67e2/overlay/tsl.bzl | 19 - .../20250317.1-71c67e2/overlay/workspace.bzl | 60 - .../0001-bazel-migration-to-bazel-8.1.1.patch | 41 - ...ler-registration-API-to-the-FFI-PjRt.patch | 131 -- .../xla/20250317.1-71c67e2/source.json | 15 - .../xla/20250317.2-71c67e2/MODULE.bazel | 37 - .../20250317.2-71c67e2/overlay/MODULE.bazel | 1 - .../xla/20250317.2-71c67e2/overlay/llvm.bzl | 13 - .../xla/20250317.2-71c67e2/overlay/tsl.bzl | 19 - .../20250317.2-71c67e2/overlay/workspace.bzl | 60 - .../0001-bazel-migration-to-bazel-8.1.1.patch | 41 - ...ler-registration-API-to-the-FFI-PjRt.patch | 131 -- .../xla/20250317.2-71c67e2/source.json | 16 - .../xla/20250527.0-cb67f2f/MODULE.bazel | 52 - .../20250527.0-cb67f2f/overlay/MODULE.bazel | 1 - .../xla/20250527.0-cb67f2f/overlay/llvm.bzl | 30 - .../overlay/workspace_private.bzl | 71 - .../xla/20250527.0-cb67f2f/overlay/xla.bzl | 17 - .../0001-bazel-migration-to-bazel-8.1.1.patch | 41 - ...ler-registration-API-to-the-FFI-PjRt.patch | 135 -- ...nal-allowOtherDialects-field-to-stab.patch | 186 -- .../xla/20250527.0-cb67f2f/source.json | 17 - .../xla/20250612.0-6e48cbb/MODULE.bazel | 52 - .../20250612.0-6e48cbb/overlay/MODULE.bazel | 52 - .../xla/20250612.0-6e48cbb/overlay/llvm.bzl | 30 - .../overlay/workspace_private.bzl | 71 - .../xla/20250612.0-6e48cbb/overlay/xla.bzl | 17 - .../0001-bazel-migration-to-bazel-8.1.1.patch | 41 - ...ler-registration-API-to-the-FFI-PjRt.patch | 135 -- ...nal-allowOtherDialects-field-to-stab.patch | 1855 ----------------- .../xla/20250612.0-6e48cbb/source.json | 17 - .../xla/20250710.0-22ea002/MODULE.bazel | 58 - .../20250710.0-22ea002/overlay/MODULE.bazel | 1 - .../xla/20250710.0-22ea002/overlay/llvm.bzl | 30 - .../overlay/toolchains_private.bzl | 21 - .../overlay/workspace_private.bzl | 73 - .../xla/20250710.0-22ea002/overlay/xla.bzl | 17 - .../0001-bazel-migration-to-bazel-8.1.1.patch | 41 - ...ler-registration-API-to-the-FFI-PjRt.patch | 135 -- ...ove-unconventional-C-code-in-headers.patch | 124 -- .../xla/20250710.0-22ea002/source.json | 18 - .../xla/20250718.0-6319f0d/MODULE.bazel | 58 - .../20250718.0-6319f0d/overlay/MODULE.bazel | 1 - .../xla/20250718.0-6319f0d/overlay/llvm.bzl | 30 - .../overlay/toolchains_private.bzl | 21 - .../overlay/workspace_private.bzl | 73 - .../xla/20250718.0-6319f0d/overlay/xla.bzl | 17 - .../0001-bazel-migration-to-bazel-8.1.1.patch | 41 - ...ler-registration-API-to-the-FFI-PjRt.patch | 135 -- ...ove-unconventional-C-code-in-headers.patch | 124 -- .../xla/20250718.0-6319f0d/source.json | 18 - third_party/modules/xla/metadata.json | 30 - .../20240722.0-c644d11/MODULE.bazel | 8 - .../20240722.0-c644d11/overlay/BUILD.bazel | 32 - .../20240722.0-c644d11/overlay/MODULE.bazel | 1 - .../patches/0001-USE_MODULES-true.patch | 11 - .../20240722.0-c644d11/source.json | 13 - .../20250213.0-5304067/MODULE.bazel | 8 - .../20250213.0-5304067/overlay/BUILD.bazel | 32 - .../20250213.0-5304067/overlay/MODULE.bazel | 1 - .../20250213.0-5304067/source.json | 9 - .../20250318.0-930153e/MODULE.bazel | 8 - .../20250318.0-930153e/overlay/BUILD.bazel | 32 - .../20250318.0-930153e/overlay/MODULE.bazel | 1 - .../20250318.0-930153e/source.json | 9 - .../20250716.0-97f1e31/MODULE.bazel | 8 - .../20250716.0-97f1e31/overlay/BUILD.bazel | 32 - .../20250716.0-97f1e31/overlay/MODULE.bazel | 8 - .../20250716.0-97f1e31/source.json | 9 - .../modules/zig-protobuf/metadata.json | 20 - .../zig-yaml/20240903.0-83d5fdf/MODULE.bazel | 7 - .../20240903.0-83d5fdf/overlay/BUILD.bazel | 12 - .../20240903.0-83d5fdf/overlay/MODULE.bazel | 1 - .../zig-yaml/20240903.0-83d5fdf/source.json | 9 - third_party/modules/zig-yaml/metadata.json | 17 - .../zigcoro/20240829.0-fc1db29/MODULE.bazel | 8 - .../20240829.0-fc1db29/overlay/BUILD.bazel | 29 - .../20240829.0-fc1db29/overlay/MODULE.bazel | 1 - ...Handle-generic-function-in-Signature.patch | 48 - ...turn-error-super-set-in-pread-pwrite.patch | 34 - .../zigcoro/20240829.0-fc1db29/source.json | 14 - third_party/modules/zigcoro/metadata.json | 17 - 292 files changed, 10603 deletions(-) delete mode 100644 third_party/modules/aspect_bazel_lib/2.8.1.1/MODULE.bazel delete mode 100644 third_party/modules/aspect_bazel_lib/2.8.1.1/patches/0001-fix-add-repo-mapping-to-tar-archive.patch delete mode 100644 third_party/modules/aspect_bazel_lib/2.8.1.1/patches/go_dev_dep.patch delete mode 100644 third_party/modules/aspect_bazel_lib/2.8.1.1/patches/module_dot_bazel_version.patch delete mode 100644 third_party/modules/aspect_bazel_lib/2.8.1.1/source.json delete mode 100644 third_party/modules/aspect_bazel_lib/metadata.json delete mode 100644 third_party/modules/libxev/20240825.0-dbe2291/MODULE.bazel delete mode 100644 third_party/modules/libxev/20240825.0-dbe2291/overlay/BUILD.bazel delete mode 120000 third_party/modules/libxev/20240825.0-dbe2291/overlay/MODULE.bazel delete mode 100644 third_party/modules/libxev/20240825.0-dbe2291/source.json delete mode 100644 third_party/modules/libxev/20240910.0-a2d9b31/MODULE.bazel delete mode 100644 third_party/modules/libxev/20240910.0-a2d9b31/overlay/BUILD.bazel delete mode 120000 third_party/modules/libxev/20240910.0-a2d9b31/overlay/MODULE.bazel delete mode 100644 third_party/modules/libxev/20240910.0-a2d9b31/source.json delete mode 100644 third_party/modules/libxev/20241119.0-6afcde9/MODULE.bazel delete mode 100644 third_party/modules/libxev/20241119.0-6afcde9/overlay/BUILD.bazel delete mode 120000 third_party/modules/libxev/20241119.0-6afcde9/overlay/MODULE.bazel delete mode 100644 third_party/modules/libxev/20241119.0-6afcde9/overlay/main2.zig delete mode 100644 third_party/modules/libxev/20241119.0-6afcde9/patches/128.patch delete mode 100644 third_party/modules/libxev/20241119.0-6afcde9/source.json delete mode 100644 third_party/modules/libxev/20241208.0-db6a52b/MODULE.bazel delete mode 100644 third_party/modules/libxev/20241208.0-db6a52b/overlay/BUILD.bazel delete mode 100644 third_party/modules/libxev/20241208.0-db6a52b/overlay/MODULE.bazel delete mode 100644 third_party/modules/libxev/20241208.0-db6a52b/overlay/main2.zig delete mode 100644 third_party/modules/libxev/20241208.0-db6a52b/patches/128.patch delete mode 100644 third_party/modules/libxev/20241208.0-db6a52b/source.json delete mode 100644 third_party/modules/libxev/20241208.1-db6a52b/MODULE.bazel delete mode 100644 third_party/modules/libxev/20241208.1-db6a52b/overlay/BUILD.bazel delete mode 100644 third_party/modules/libxev/20241208.1-db6a52b/overlay/MODULE.bazel delete mode 100644 third_party/modules/libxev/20241208.1-db6a52b/overlay/main2.zig delete mode 100644 third_party/modules/libxev/20241208.1-db6a52b/patches/128.patch delete mode 100644 third_party/modules/libxev/20241208.1-db6a52b/source.json delete mode 100644 third_party/modules/libxev/20241208.2-db6a52b/MODULE.bazel delete mode 100644 third_party/modules/libxev/20241208.2-db6a52b/overlay/BUILD.bazel delete mode 100644 third_party/modules/libxev/20241208.2-db6a52b/overlay/MODULE.bazel delete mode 100644 third_party/modules/libxev/20241208.2-db6a52b/overlay/main2.zig delete mode 100644 third_party/modules/libxev/20241208.2-db6a52b/patches/128.patch delete mode 100644 third_party/modules/libxev/20241208.2-db6a52b/source.json delete mode 100644 third_party/modules/libxev/20250124.0-31eed4e/MODULE.bazel delete mode 100644 third_party/modules/libxev/20250124.0-31eed4e/overlay/BUILD.bazel delete mode 120000 third_party/modules/libxev/20250124.0-31eed4e/overlay/MODULE.bazel delete mode 100644 third_party/modules/libxev/20250124.0-31eed4e/overlay/main2.zig delete mode 100644 third_party/modules/libxev/20250124.0-31eed4e/patches/128.patch delete mode 100644 third_party/modules/libxev/20250124.0-31eed4e/patches/141.patch delete mode 100644 third_party/modules/libxev/20250124.0-31eed4e/source.json delete mode 100644 third_party/modules/libxev/20250222.0-07bcffa/MODULE.bazel delete mode 100644 third_party/modules/libxev/20250222.0-07bcffa/overlay/BUILD.bazel delete mode 120000 third_party/modules/libxev/20250222.0-07bcffa/overlay/MODULE.bazel delete mode 100644 third_party/modules/libxev/20250222.0-07bcffa/source.json delete mode 100644 third_party/modules/libxev/20250313.0-5773f46/MODULE.bazel delete mode 100644 third_party/modules/libxev/20250313.0-5773f46/overlay/BUILD.bazel delete mode 120000 third_party/modules/libxev/20250313.0-5773f46/overlay/MODULE.bazel delete mode 100644 third_party/modules/libxev/20250313.0-5773f46/source.json delete mode 120000 third_party/modules/libxev/20250718.0-9f785d2/MODULE.bazel delete mode 100644 third_party/modules/libxev/20250718.0-9f785d2/overlay/BUILD.bazel delete mode 100644 third_party/modules/libxev/20250718.0-9f785d2/overlay/MODULE.bazel delete mode 100644 third_party/modules/libxev/20250718.0-9f785d2/source.json delete mode 120000 third_party/modules/libxev/20251010.0-9f785d2/MODULE.bazel delete mode 100644 third_party/modules/libxev/20251010.0-9f785d2/overlay/BUILD.bazel delete mode 100644 third_party/modules/libxev/20251010.0-9f785d2/overlay/MODULE.bazel delete mode 100644 third_party/modules/libxev/20251010.0-9f785d2/source.json delete mode 100644 third_party/modules/libxev/metadata.json delete mode 100644 third_party/modules/llvm-raw/20240823.0-f142f8a/MODULE.bazel delete mode 100644 third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/BUILD.bazel delete mode 120000 third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/MODULE.bazel delete mode 100644 third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/utils/bazel/extension.bzl delete mode 100644 third_party/modules/llvm-raw/20240823.0-f142f8a/source.json delete mode 100644 third_party/modules/llvm-raw/20240919.0-94c024a/MODULE.bazel delete mode 100644 third_party/modules/llvm-raw/20240919.0-94c024a/overlay/BUILD.bazel delete mode 120000 third_party/modules/llvm-raw/20240919.0-94c024a/overlay/MODULE.bazel delete mode 100644 third_party/modules/llvm-raw/20240919.0-94c024a/overlay/utils/bazel/extension.bzl delete mode 100644 third_party/modules/llvm-raw/20240919.0-94c024a/source.json delete mode 100644 third_party/modules/llvm-raw/20241022.0-6c4267f/MODULE.bazel delete mode 100644 third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/BUILD.bazel delete mode 120000 third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/MODULE.bazel delete mode 100644 third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/utils/bazel/extension.bzl delete mode 100644 third_party/modules/llvm-raw/20241022.0-6c4267f/source.json delete mode 100644 third_party/modules/llvm-raw/20250102.0-f739aa4/MODULE.bazel delete mode 100644 third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/BUILD.bazel delete mode 120000 third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/MODULE.bazel delete mode 100644 third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/utils/bazel/extension.bzl delete mode 100644 third_party/modules/llvm-raw/20250102.0-f739aa4/source.json delete mode 100644 third_party/modules/llvm-raw/20250117.0-bf17016/MODULE.bazel delete mode 100644 third_party/modules/llvm-raw/20250117.0-bf17016/overlay/BUILD.bazel delete mode 120000 third_party/modules/llvm-raw/20250117.0-bf17016/overlay/MODULE.bazel delete mode 100644 third_party/modules/llvm-raw/20250117.0-bf17016/overlay/utils/bazel/extension.bzl delete mode 100644 third_party/modules/llvm-raw/20250117.0-bf17016/source.json delete mode 100644 third_party/modules/llvm-raw/20250201.0-386af4a/MODULE.bazel delete mode 100644 third_party/modules/llvm-raw/20250201.0-386af4a/overlay/BUILD.bazel delete mode 120000 third_party/modules/llvm-raw/20250201.0-386af4a/overlay/MODULE.bazel delete mode 100644 third_party/modules/llvm-raw/20250201.0-386af4a/overlay/utils/bazel/extension.bzl delete mode 100644 third_party/modules/llvm-raw/20250201.0-386af4a/source.json delete mode 100644 third_party/modules/llvm-raw/20250217.0-0e779ad/MODULE.bazel delete mode 100644 third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/BUILD.bazel delete mode 120000 third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/MODULE.bazel delete mode 100644 third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/utils/bazel/extension.bzl delete mode 100644 third_party/modules/llvm-raw/20250217.0-0e779ad/source.json delete mode 100644 third_party/modules/llvm-raw/metadata.json delete mode 100644 third_party/modules/rules_zig/20240904.0-010da15/MODULE.bazel delete mode 100644 third_party/modules/rules_zig/20240904.0-010da15/source.json delete mode 100644 third_party/modules/rules_zig/20240909.0-37f17ff/MODULE.bazel delete mode 100644 third_party/modules/rules_zig/20240909.0-37f17ff/source.json delete mode 100644 third_party/modules/rules_zig/20240912.0-41bfe84/MODULE.bazel delete mode 100644 third_party/modules/rules_zig/20240912.0-41bfe84/source.json delete mode 100644 third_party/modules/rules_zig/20240913.0-1957d05/MODULE.bazel delete mode 100644 third_party/modules/rules_zig/20240913.0-1957d05/source.json delete mode 100644 third_party/modules/rules_zig/20250314.0-b9739c6/MODULE.bazel delete mode 100644 third_party/modules/rules_zig/20250314.0-b9739c6/source.json delete mode 100644 third_party/modules/rules_zig/20250519.0-233b207/MODULE.bazel delete mode 100644 third_party/modules/rules_zig/20250519.0-233b207/source.json delete mode 100644 third_party/modules/rules_zig/20250530.0-5084f1f/MODULE.bazel delete mode 100644 third_party/modules/rules_zig/20250530.0-5084f1f/source.json delete mode 100644 third_party/modules/rules_zig/20250613.0-567662a/MODULE.bazel delete mode 100644 third_party/modules/rules_zig/20250613.0-567662a/source.json delete mode 100644 third_party/modules/rules_zig/20250714.0-b14a4f1/MODULE.bazel delete mode 100644 third_party/modules/rules_zig/20250714.0-b14a4f1/source.json delete mode 100644 third_party/modules/rules_zig/20250821.0-be53625/MODULE.bazel delete mode 100644 third_party/modules/rules_zig/20250821.0-be53625/source.json delete mode 100644 third_party/modules/rules_zig/20250827.0-35b6d57/MODULE.bazel delete mode 100644 third_party/modules/rules_zig/20250827.0-35b6d57/source.json delete mode 100644 third_party/modules/rules_zig/metadata.json delete mode 100644 third_party/modules/sentencepiece/20240618.0-d7ace0a/MODULE.bazel delete mode 100644 third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/BUILD.bazel delete mode 120000 third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/MODULE.bazel delete mode 100644 third_party/modules/sentencepiece/20240618.0-d7ace0a/source.json delete mode 100644 third_party/modules/sentencepiece/metadata.json delete mode 100644 third_party/modules/stablehlo/20240829.0-54aa1a5/MODULE.bazel delete mode 120000 third_party/modules/stablehlo/20240829.0-54aa1a5/overlay/MODULE.bazel delete mode 100644 third_party/modules/stablehlo/20240829.0-54aa1a5/patches/0001-Remove-duplicated-symbols-in-StablehloApi.h.patch delete mode 100644 third_party/modules/stablehlo/20240829.0-54aa1a5/patches/temporary.patch delete mode 100644 third_party/modules/stablehlo/20240829.0-54aa1a5/source.json delete mode 100644 third_party/modules/stablehlo/20240917.0-78c753a/MODULE.bazel delete mode 120000 third_party/modules/stablehlo/20240917.0-78c753a/overlay/MODULE.bazel delete mode 100644 third_party/modules/stablehlo/20240917.0-78c753a/source.json delete mode 100644 third_party/modules/stablehlo/20241021.0-1c0b606/MODULE.bazel delete mode 120000 third_party/modules/stablehlo/20241021.0-1c0b606/overlay/MODULE.bazel delete mode 100644 third_party/modules/stablehlo/20241021.0-1c0b606/source.json delete mode 100644 third_party/modules/stablehlo/20241220.0-38bb2f9/MODULE.bazel delete mode 120000 third_party/modules/stablehlo/20241220.0-38bb2f9/overlay/MODULE.bazel delete mode 100644 third_party/modules/stablehlo/20241220.0-38bb2f9/source.json delete mode 100644 third_party/modules/stablehlo/20250117.0-c125b32/MODULE.bazel delete mode 120000 third_party/modules/stablehlo/20250117.0-c125b32/overlay/MODULE.bazel delete mode 100644 third_party/modules/stablehlo/20250117.0-c125b32/source.json delete mode 100644 third_party/modules/stablehlo/20250201.0-7775e3e/MODULE.bazel delete mode 120000 third_party/modules/stablehlo/20250201.0-7775e3e/overlay/MODULE.bazel delete mode 100644 third_party/modules/stablehlo/20250201.0-7775e3e/source.json delete mode 100644 third_party/modules/stablehlo/20250217.0-4598975/MODULE.bazel delete mode 120000 third_party/modules/stablehlo/20250217.0-4598975/overlay/MODULE.bazel delete mode 100644 third_party/modules/stablehlo/20250217.0-4598975/source.json delete mode 100644 third_party/modules/stablehlo/metadata.json delete mode 100644 third_party/modules/xla/20240902.0-d18cd64/MODULE.bazel delete mode 120000 third_party/modules/xla/20240902.0-d18cd64/overlay/MODULE.bazel delete mode 100644 third_party/modules/xla/20240902.0-d18cd64/overlay/tsl.bzl delete mode 100644 third_party/modules/xla/20240902.0-d18cd64/overlay/workspace.bzl delete mode 100644 third_party/modules/xla/20240902.0-d18cd64/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch delete mode 100644 third_party/modules/xla/20240902.0-d18cd64/source.json delete mode 100644 third_party/modules/xla/20240919.0-1b18dd6/MODULE.bazel delete mode 120000 third_party/modules/xla/20240919.0-1b18dd6/overlay/MODULE.bazel delete mode 100644 third_party/modules/xla/20240919.0-1b18dd6/overlay/tsl.bzl delete mode 100644 third_party/modules/xla/20240919.0-1b18dd6/overlay/workspace.bzl delete mode 100644 third_party/modules/xla/20240919.0-1b18dd6/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch delete mode 100644 third_party/modules/xla/20240919.0-1b18dd6/source.json delete mode 100644 third_party/modules/xla/20241025.0-4663f04/MODULE.bazel delete mode 120000 third_party/modules/xla/20241025.0-4663f04/overlay/MODULE.bazel delete mode 100644 third_party/modules/xla/20241025.0-4663f04/overlay/tsl.bzl delete mode 100644 third_party/modules/xla/20241025.0-4663f04/overlay/workspace.bzl delete mode 100644 third_party/modules/xla/20241025.0-4663f04/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch delete mode 100644 third_party/modules/xla/20241025.0-4663f04/source.json delete mode 100644 third_party/modules/xla/20250103.0-5f1fe6a/MODULE.bazel delete mode 120000 third_party/modules/xla/20250103.0-5f1fe6a/overlay/MODULE.bazel delete mode 100644 third_party/modules/xla/20250103.0-5f1fe6a/overlay/tsl.bzl delete mode 100644 third_party/modules/xla/20250103.0-5f1fe6a/overlay/workspace.bzl delete mode 100644 third_party/modules/xla/20250103.0-5f1fe6a/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch delete mode 100644 third_party/modules/xla/20250103.0-5f1fe6a/source.json delete mode 100644 third_party/modules/xla/20250122.0-cc075be/MODULE.bazel delete mode 120000 third_party/modules/xla/20250122.0-cc075be/overlay/MODULE.bazel delete mode 100644 third_party/modules/xla/20250122.0-cc075be/overlay/tsl.bzl delete mode 100644 third_party/modules/xla/20250122.0-cc075be/overlay/workspace.bzl delete mode 100644 third_party/modules/xla/20250122.0-cc075be/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch delete mode 100644 third_party/modules/xla/20250122.0-cc075be/source.json delete mode 100644 third_party/modules/xla/20250123.0-cc075be/MODULE.bazel delete mode 120000 third_party/modules/xla/20250123.0-cc075be/overlay/MODULE.bazel delete mode 100644 third_party/modules/xla/20250123.0-cc075be/overlay/tsl.bzl delete mode 100644 third_party/modules/xla/20250123.0-cc075be/overlay/workspace.bzl delete mode 100644 third_party/modules/xla/20250123.0-cc075be/patches/0001-bazel-migration-to-bazel-8.0.1.patch delete mode 100644 third_party/modules/xla/20250123.0-cc075be/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch delete mode 100644 third_party/modules/xla/20250123.0-cc075be/source.json delete mode 100644 third_party/modules/xla/20250204-0-6789523/MODULE.bazel delete mode 120000 third_party/modules/xla/20250204-0-6789523/overlay/MODULE.bazel delete mode 100644 third_party/modules/xla/20250204-0-6789523/overlay/tsl.bzl delete mode 100644 third_party/modules/xla/20250204-0-6789523/overlay/workspace.bzl delete mode 100644 third_party/modules/xla/20250204-0-6789523/patches/0001-bazel-migration-to-bazel-8.0.1.patch delete mode 100644 third_party/modules/xla/20250204-0-6789523/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch delete mode 100644 third_party/modules/xla/20250204-0-6789523/source.json delete mode 100644 third_party/modules/xla/20250204.1-6789523/MODULE.bazel delete mode 120000 third_party/modules/xla/20250204.1-6789523/overlay/MODULE.bazel delete mode 100644 third_party/modules/xla/20250204.1-6789523/overlay/tsl.bzl delete mode 100644 third_party/modules/xla/20250204.1-6789523/overlay/workspace.bzl delete mode 100644 third_party/modules/xla/20250204.1-6789523/patches/0001-bazel-migration-to-bazel-8.0.1.patch delete mode 100644 third_party/modules/xla/20250204.1-6789523/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch delete mode 100644 third_party/modules/xla/20250204.1-6789523/source.json delete mode 100644 third_party/modules/xla/20250317.0-71c67e2/MODULE.bazel delete mode 120000 third_party/modules/xla/20250317.0-71c67e2/overlay/MODULE.bazel delete mode 100644 third_party/modules/xla/20250317.0-71c67e2/overlay/tsl.bzl delete mode 100644 third_party/modules/xla/20250317.0-71c67e2/overlay/workspace.bzl delete mode 100644 third_party/modules/xla/20250317.0-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch delete mode 100644 third_party/modules/xla/20250317.0-71c67e2/source.json delete mode 100644 third_party/modules/xla/20250317.1-71c67e2/MODULE.bazel delete mode 120000 third_party/modules/xla/20250317.1-71c67e2/overlay/MODULE.bazel delete mode 100644 third_party/modules/xla/20250317.1-71c67e2/overlay/tsl.bzl delete mode 100644 third_party/modules/xla/20250317.1-71c67e2/overlay/workspace.bzl delete mode 100644 third_party/modules/xla/20250317.1-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch delete mode 100644 third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch delete mode 100644 third_party/modules/xla/20250317.1-71c67e2/source.json delete mode 100644 third_party/modules/xla/20250317.2-71c67e2/MODULE.bazel delete mode 120000 third_party/modules/xla/20250317.2-71c67e2/overlay/MODULE.bazel delete mode 100644 third_party/modules/xla/20250317.2-71c67e2/overlay/llvm.bzl delete mode 100644 third_party/modules/xla/20250317.2-71c67e2/overlay/tsl.bzl delete mode 100644 third_party/modules/xla/20250317.2-71c67e2/overlay/workspace.bzl delete mode 100644 third_party/modules/xla/20250317.2-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch delete mode 100644 third_party/modules/xla/20250317.2-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch delete mode 100644 third_party/modules/xla/20250317.2-71c67e2/source.json delete mode 100644 third_party/modules/xla/20250527.0-cb67f2f/MODULE.bazel delete mode 120000 third_party/modules/xla/20250527.0-cb67f2f/overlay/MODULE.bazel delete mode 100644 third_party/modules/xla/20250527.0-cb67f2f/overlay/llvm.bzl delete mode 100644 third_party/modules/xla/20250527.0-cb67f2f/overlay/workspace_private.bzl delete mode 100644 third_party/modules/xla/20250527.0-cb67f2f/overlay/xla.bzl delete mode 100644 third_party/modules/xla/20250527.0-cb67f2f/patches/0001-bazel-migration-to-bazel-8.1.1.patch delete mode 100644 third_party/modules/xla/20250527.0-cb67f2f/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch delete mode 100644 third_party/modules/xla/20250527.0-cb67f2f/patches/0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch delete mode 100644 third_party/modules/xla/20250527.0-cb67f2f/source.json delete mode 100644 third_party/modules/xla/20250612.0-6e48cbb/MODULE.bazel delete mode 100644 third_party/modules/xla/20250612.0-6e48cbb/overlay/MODULE.bazel delete mode 100644 third_party/modules/xla/20250612.0-6e48cbb/overlay/llvm.bzl delete mode 100644 third_party/modules/xla/20250612.0-6e48cbb/overlay/workspace_private.bzl delete mode 100644 third_party/modules/xla/20250612.0-6e48cbb/overlay/xla.bzl delete mode 100644 third_party/modules/xla/20250612.0-6e48cbb/patches/0001-bazel-migration-to-bazel-8.1.1.patch delete mode 100644 third_party/modules/xla/20250612.0-6e48cbb/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch delete mode 100644 third_party/modules/xla/20250612.0-6e48cbb/patches/0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch delete mode 100644 third_party/modules/xla/20250612.0-6e48cbb/source.json delete mode 100644 third_party/modules/xla/20250710.0-22ea002/MODULE.bazel delete mode 120000 third_party/modules/xla/20250710.0-22ea002/overlay/MODULE.bazel delete mode 100644 third_party/modules/xla/20250710.0-22ea002/overlay/llvm.bzl delete mode 100644 third_party/modules/xla/20250710.0-22ea002/overlay/toolchains_private.bzl delete mode 100644 third_party/modules/xla/20250710.0-22ea002/overlay/workspace_private.bzl delete mode 100644 third_party/modules/xla/20250710.0-22ea002/overlay/xla.bzl delete mode 100644 third_party/modules/xla/20250710.0-22ea002/patches/0001-bazel-migration-to-bazel-8.1.1.patch delete mode 100644 third_party/modules/xla/20250710.0-22ea002/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch delete mode 100644 third_party/modules/xla/20250710.0-22ea002/patches/0003-Remove-unconventional-C-code-in-headers.patch delete mode 100644 third_party/modules/xla/20250710.0-22ea002/source.json delete mode 100644 third_party/modules/xla/20250718.0-6319f0d/MODULE.bazel delete mode 120000 third_party/modules/xla/20250718.0-6319f0d/overlay/MODULE.bazel delete mode 100644 third_party/modules/xla/20250718.0-6319f0d/overlay/llvm.bzl delete mode 100644 third_party/modules/xla/20250718.0-6319f0d/overlay/toolchains_private.bzl delete mode 100644 third_party/modules/xla/20250718.0-6319f0d/overlay/workspace_private.bzl delete mode 100644 third_party/modules/xla/20250718.0-6319f0d/overlay/xla.bzl delete mode 100644 third_party/modules/xla/20250718.0-6319f0d/patches/0001-bazel-migration-to-bazel-8.1.1.patch delete mode 100644 third_party/modules/xla/20250718.0-6319f0d/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch delete mode 100644 third_party/modules/xla/20250718.0-6319f0d/patches/0003-Remove-unconventional-C-code-in-headers.patch delete mode 100644 third_party/modules/xla/20250718.0-6319f0d/source.json delete mode 100644 third_party/modules/xla/metadata.json delete mode 100644 third_party/modules/zig-protobuf/20240722.0-c644d11/MODULE.bazel delete mode 100644 third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/BUILD.bazel delete mode 120000 third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/MODULE.bazel delete mode 100644 third_party/modules/zig-protobuf/20240722.0-c644d11/patches/0001-USE_MODULES-true.patch delete mode 100644 third_party/modules/zig-protobuf/20240722.0-c644d11/source.json delete mode 100644 third_party/modules/zig-protobuf/20250213.0-5304067/MODULE.bazel delete mode 100644 third_party/modules/zig-protobuf/20250213.0-5304067/overlay/BUILD.bazel delete mode 120000 third_party/modules/zig-protobuf/20250213.0-5304067/overlay/MODULE.bazel delete mode 100644 third_party/modules/zig-protobuf/20250213.0-5304067/source.json delete mode 100644 third_party/modules/zig-protobuf/20250318.0-930153e/MODULE.bazel delete mode 100644 third_party/modules/zig-protobuf/20250318.0-930153e/overlay/BUILD.bazel delete mode 120000 third_party/modules/zig-protobuf/20250318.0-930153e/overlay/MODULE.bazel delete mode 100644 third_party/modules/zig-protobuf/20250318.0-930153e/source.json delete mode 100644 third_party/modules/zig-protobuf/20250716.0-97f1e31/MODULE.bazel delete mode 100644 third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/BUILD.bazel delete mode 100644 third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/MODULE.bazel delete mode 100644 third_party/modules/zig-protobuf/20250716.0-97f1e31/source.json delete mode 100644 third_party/modules/zig-protobuf/metadata.json delete mode 100644 third_party/modules/zig-yaml/20240903.0-83d5fdf/MODULE.bazel delete mode 100644 third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/BUILD.bazel delete mode 120000 third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/MODULE.bazel delete mode 100644 third_party/modules/zig-yaml/20240903.0-83d5fdf/source.json delete mode 100644 third_party/modules/zig-yaml/metadata.json delete mode 100644 third_party/modules/zigcoro/20240829.0-fc1db29/MODULE.bazel delete mode 100644 third_party/modules/zigcoro/20240829.0-fc1db29/overlay/BUILD.bazel delete mode 120000 third_party/modules/zigcoro/20240829.0-fc1db29/overlay/MODULE.bazel delete mode 100644 third_party/modules/zigcoro/20240829.0-fc1db29/patches/0001-Handle-generic-function-in-Signature.patch delete mode 100644 third_party/modules/zigcoro/20240829.0-fc1db29/patches/0002-asyncio-return-error-super-set-in-pread-pwrite.patch delete mode 100644 third_party/modules/zigcoro/20240829.0-fc1db29/source.json delete mode 100644 third_party/modules/zigcoro/metadata.json diff --git a/.bazelignore b/.bazelignore index 084d45c8..e0df23f9 100644 --- a/.bazelignore +++ b/.bazelignore @@ -1,5 +1,4 @@ .* -third_party/modules .zig-cache .devenv venv/ diff --git a/third_party/modules/aspect_bazel_lib/2.8.1.1/MODULE.bazel b/third_party/modules/aspect_bazel_lib/2.8.1.1/MODULE.bazel deleted file mode 100644 index 7f1750b4..00000000 --- a/third_party/modules/aspect_bazel_lib/2.8.1.1/MODULE.bazel +++ /dev/null @@ -1,78 +0,0 @@ -"aspect-build/bazel-lib" - -module( - name = "aspect_bazel_lib", - version = "2.8.1.1", - bazel_compatibility = [">=6.0.0"], - compatibility_level = 1, -) - -# Lower-bounds (minimum) versions for direct runtime dependencies -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "platforms", version = "0.0.8") - -# 0.5.4 is the first version with bzlmod support -bazel_dep(name = "stardoc", version = "0.6.2", repo_name = "io_bazel_stardoc") - -bazel_lib_toolchains = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "toolchains") -bazel_lib_toolchains.copy_directory() -bazel_lib_toolchains.copy_to_directory() -bazel_lib_toolchains.jq() -bazel_lib_toolchains.yq() -bazel_lib_toolchains.coreutils() -bazel_lib_toolchains.tar() -bazel_lib_toolchains.zstd() -bazel_lib_toolchains.expand_template() -bazel_lib_toolchains.bats() -use_repo(bazel_lib_toolchains, "bats_toolchains", "bsd_tar_toolchains", "copy_directory_toolchains", "copy_to_directory_toolchains", "coreutils_toolchains", "expand_template_toolchains", "jq_toolchains", "yq_toolchains", "zstd_toolchains") - -register_toolchains( - "@copy_directory_toolchains//:all", - "@copy_to_directory_toolchains//:all", - "@jq_toolchains//:all", - "@yq_toolchains//:all", - "@coreutils_toolchains//:all", - "@expand_template_toolchains//:all", - "@bats_toolchains//:all", - "@bsd_tar_toolchains//:all", - "@zstd_toolchains//:all", -) - -####### Dev dependencies ######## - -# To allow /tools to be built from source -# NOTE: when publishing to BCR, we patch this to be dev_dependency, as we publish pre-built binaries -# along with our releases. - -bazel_dep( - name = "gazelle", - version = "0.36.0", - dev_dependency = True, -) -bazel_dep( - name = "rules_go", - version = "0.46.0", - repo_name = "io_bazel_rules_go", - dev_dependency = True, -) - -go_deps = use_extension( - "@gazelle//:extensions.bzl", - "go_deps", - dev_dependency = True, -) -go_deps.from_file(go_mod = "//:go.mod") -use_repo( - go_deps, - "com_github_bmatcuk_doublestar_v4", - "org_golang_x_exp", - "org_golang_x_sys", -) - -host = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "host", dev_dependency = True) -host.host() -use_repo(host, "aspect_bazel_lib_host") - -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.5.0", dev_dependency = True) -bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True) -bazel_dep(name = "bazel_features", version = "0.2.0", dev_dependency = True) diff --git a/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/0001-fix-add-repo-mapping-to-tar-archive.patch b/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/0001-fix-add-repo-mapping-to-tar-archive.patch deleted file mode 100644 index 96aa8dfc..00000000 --- a/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/0001-fix-add-repo-mapping-to-tar-archive.patch +++ /dev/null @@ -1,50 +0,0 @@ -From 5dbe6a2604e440b684add7b44531898acc8631b4 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Sun, 8 Sep 2024 21:17:29 +0200 -Subject: [PATCH] fix: add repo mapping to tar archive - -When using bzlmod, runfiles lookup will fail without it. ---- - lib/private/tar.bzl | 17 +++++++++++++---- - 1 file changed, 13 insertions(+), 4 deletions(-) - -diff --git a/lib/private/tar.bzl b/lib/private/tar.bzl -index 733ff60..29434f6 100644 ---- a/lib/private/tar.bzl -+++ b/lib/private/tar.bzl -@@ -147,12 +147,19 @@ def _tar_impl(ctx): - args.add(ctx.file.mtree, format = "@%s") - inputs.append(ctx.file.mtree) - -+ src_runfiles = [] -+ for src in ctx.attr.srcs: -+ src_di = src[DefaultInfo] -+ if getattr(src_di.files_to_run, "repo_mapping_manifest", None) != None: -+ src_runfiles.append(depset( -+ direct = [src_di.files_to_run.repo_mapping_manifest], -+ transitive = [src_di.default_runfiles.files], -+ )) -+ else: -+ src_runfiles.append(src_di.default_runfiles.files) - ctx.actions.run( - executable = bsdtar.tarinfo.binary, -- inputs = depset(direct = inputs, transitive = [bsdtar.default.files] + [ -- src[DefaultInfo].default_runfiles.files -- for src in ctx.attr.srcs -- ]), -+ inputs = depset(direct = inputs, transitive = [bsdtar.default.files] + src_runfiles), - outputs = [out], - arguments = [args], - mnemonic = "Tar", -@@ -234,6 +241,8 @@ def _mtree_impl(ctx): - workspace_name = str(ctx.workspace_name) - - content.add(_mtree_line(runfiles_dir, type = "dir")) -+ if getattr(default_info.files_to_run, "repo_mapping_manifest", None) != None: -+ content.add(_mtree_line("{}/_repo_mapping".format(runfiles_dir), type = "file", content = default_info.files_to_run.repo_mapping_manifest.path)) - content.add_all( - s.default_runfiles.files, - expand_directories = True, --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/go_dev_dep.patch b/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/go_dev_dep.patch deleted file mode 100644 index ffe1c1d3..00000000 --- a/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/go_dev_dep.patch +++ /dev/null @@ -1,27 +0,0 @@ -diff --git a/MODULE.bazel b/MODULE.bazel -index e63fa5b..9d78a88 100644 ---- a/MODULE.bazel -+++ b/MODULE.bazel -@@ -50,19 +50,19 @@ use_repo(host, "aspect_bazel_lib_host") - bazel_dep( - name = "gazelle", - version = "0.36.0", -- # In released versions: dev_dependency = True -+ dev_dependency = True, - ) - bazel_dep( - name = "rules_go", - version = "0.46.0", - repo_name = "io_bazel_rules_go", -- # In released versions: dev_dependency = True -+ dev_dependency = True, - ) - - go_deps = use_extension( - "@gazelle//:extensions.bzl", - "go_deps", -- # In released versions: dev_dependency = True -+ dev_dependency = True, - ) - go_deps.from_file(go_mod = "//:go.mod") - use_repo( diff --git a/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/module_dot_bazel_version.patch b/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/module_dot_bazel_version.patch deleted file mode 100644 index fe11475e..00000000 --- a/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/module_dot_bazel_version.patch +++ /dev/null @@ -1,14 +0,0 @@ -=================================================================== ---- a/MODULE.bazel -+++ b/MODULE.bazel -@@ -1,9 +1,9 @@ - "aspect-build/bazel-lib" - - module( - name = "aspect_bazel_lib", -- version = "0.0.0", -+ version = "2.8.1", - bazel_compatibility = [">=6.0.0"], - compatibility_level = 1, - ) - diff --git a/third_party/modules/aspect_bazel_lib/2.8.1.1/source.json b/third_party/modules/aspect_bazel_lib/2.8.1.1/source.json deleted file mode 100644 index 3b7d500e..00000000 --- a/third_party/modules/aspect_bazel_lib/2.8.1.1/source.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "integrity": "sha256-aINU7mvuunGUJD1z6wmSuaEujt7u7FtlRPS1MaMRIjc=", - "strip_prefix": "bazel-lib-2.8.1", - "url": "https://github.com/aspect-build/bazel-lib/releases/download/v2.8.1/bazel-lib-v2.8.1.tar.gz", - "patches": { - "go_dev_dep.patch": "sha256-DTc/hk+etl4D50M0BLRik2vHbrgDb6rds+Dj4xphWb4=", - "module_dot_bazel_version.patch": "sha256-cEMv6bY7Sc5dERv8YG0lq45zuZCsVPNn4oAN9aOkf40=", - "0001-fix-add-repo-mapping-to-tar-archive.patch": "" - }, - "patch_strip": 1 -} diff --git a/third_party/modules/aspect_bazel_lib/metadata.json b/third_party/modules/aspect_bazel_lib/metadata.json deleted file mode 100644 index db6bde15..00000000 --- a/third_party/modules/aspect_bazel_lib/metadata.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "homepage": "https://docs.aspect.build/rules/aspect_bazel_lib", - "maintainers": [ - { - "name": "Alex Eagle", - "email": "alex@aspect.dev", - "github": "alexeagle" - }, - { - "name": "Derek Cormier", - "email": "derek@aspect.dev", - "github": "kormide" - } - ], - "repository": [ - "github:aspect-build/bazel-lib" - ], - "versions": [ - "2.8.1.1" - ], - "yanked_versions": { - "1.31.0": "1.31.0 has a breaking change to the default yq version", - "1.34.2": "The version field has a leading 'v' due to a release automation bug" - } -} diff --git a/third_party/modules/libxev/20240825.0-dbe2291/MODULE.bazel b/third_party/modules/libxev/20240825.0-dbe2291/MODULE.bazel deleted file mode 100644 index 8f8806c6..00000000 --- a/third_party/modules/libxev/20240825.0-dbe2291/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20240825.0-dbe2291", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20240825.0-dbe2291/overlay/BUILD.bazel b/third_party/modules/libxev/20240825.0-dbe2291/overlay/BUILD.bazel deleted file mode 100644 index 872ef7fb..00000000 --- a/third_party/modules/libxev/20240825.0-dbe2291/overlay/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "xev", - srcs = glob([ - "src/*.zig", - "src/backend/*.zig", - "src/linux/*.zig", - "src/watcher/*.zig", - ]), - main = "src/main.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/libxev/20240825.0-dbe2291/overlay/MODULE.bazel b/third_party/modules/libxev/20240825.0-dbe2291/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/libxev/20240825.0-dbe2291/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20240825.0-dbe2291/source.json b/third_party/modules/libxev/20240825.0-dbe2291/source.json deleted file mode 100644 index 81950288..00000000 --- a/third_party/modules/libxev/20240825.0-dbe2291/source.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "strip_prefix": "libxev-dbe22910a43e9e8ec9948d3cbd73d8488a074967", - "url": "https://github.com/mitchellh/libxev/archive/dbe22910a43e9e8ec9948d3cbd73d8488a074967.tar.gz", - "integrity": "sha256-JM70bkRUyuAQXKV1piJ6I0IClzCrXV43sdLfIKVC9Lo=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - } -} diff --git a/third_party/modules/libxev/20240910.0-a2d9b31/MODULE.bazel b/third_party/modules/libxev/20240910.0-a2d9b31/MODULE.bazel deleted file mode 100644 index 99daae70..00000000 --- a/third_party/modules/libxev/20240910.0-a2d9b31/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20240910.0-a2d9b31", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20240910.0-a2d9b31/overlay/BUILD.bazel b/third_party/modules/libxev/20240910.0-a2d9b31/overlay/BUILD.bazel deleted file mode 100644 index 872ef7fb..00000000 --- a/third_party/modules/libxev/20240910.0-a2d9b31/overlay/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "xev", - srcs = glob([ - "src/*.zig", - "src/backend/*.zig", - "src/linux/*.zig", - "src/watcher/*.zig", - ]), - main = "src/main.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/libxev/20240910.0-a2d9b31/overlay/MODULE.bazel b/third_party/modules/libxev/20240910.0-a2d9b31/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/libxev/20240910.0-a2d9b31/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20240910.0-a2d9b31/source.json b/third_party/modules/libxev/20240910.0-a2d9b31/source.json deleted file mode 100644 index c56bc6a8..00000000 --- a/third_party/modules/libxev/20240910.0-a2d9b31/source.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "strip_prefix": "libxev-a2d9b3154f1e5ed463c9f2fb98a0d739057796ce", - "url": "https://github.com/zml/libxev/archive/a2d9b3154f1e5ed463c9f2fb98a0d739057796ce.tar.gz", - "integrity": "sha256-xdMjrGSB3telt52iRqHgem0WSRPjfGlTKBRyDMXLlaQ=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - } -} diff --git a/third_party/modules/libxev/20241119.0-6afcde9/MODULE.bazel b/third_party/modules/libxev/20241119.0-6afcde9/MODULE.bazel deleted file mode 100644 index c9b03066..00000000 --- a/third_party/modules/libxev/20241119.0-6afcde9/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20241119.0-6afcde9", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20241119.0-6afcde9/overlay/BUILD.bazel b/third_party/modules/libxev/20241119.0-6afcde9/overlay/BUILD.bazel deleted file mode 100644 index 7c0c4ce9..00000000 --- a/third_party/modules/libxev/20241119.0-6afcde9/overlay/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "xev", - srcs = glob([ - "src/*.zig", - "src/backend/*.zig", - "src/linux/*.zig", - "src/watcher/*.zig", - ]), - main = "main2.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/libxev/20241119.0-6afcde9/overlay/MODULE.bazel b/third_party/modules/libxev/20241119.0-6afcde9/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/libxev/20241119.0-6afcde9/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20241119.0-6afcde9/overlay/main2.zig b/third_party/modules/libxev/20241119.0-6afcde9/overlay/main2.zig deleted file mode 100644 index 4166925a..00000000 --- a/third_party/modules/libxev/20241119.0-6afcde9/overlay/main2.zig +++ /dev/null @@ -1,22 +0,0 @@ -const builtin = @import("builtin"); -const root = @import("root"); - -const main = @import("src/main.zig"); - -pub const ThreadPool = main.ThreadPool; -pub const stream = main.stream; - -pub const Options = struct { - linux_backend: main.Backend = .epoll, -}; - -pub const options: Options = if (@hasDecl(root, "xev_options")) root.xev_options else .{}; - -const default: main.Backend = switch (builtin.os.tag) { - .ios, .macos => .kqueue, - .linux => options.linux_backend, - .wasi => .wasi_poll, - .windows => .iocp, - else => @compileError("Unsupported OS"), -}; -pub usingnamespace default.Api(); diff --git a/third_party/modules/libxev/20241119.0-6afcde9/patches/128.patch b/third_party/modules/libxev/20241119.0-6afcde9/patches/128.patch deleted file mode 100644 index 6643419d..00000000 --- a/third_party/modules/libxev/20241119.0-6afcde9/patches/128.patch +++ /dev/null @@ -1,119 +0,0 @@ -From 0d1c2f8258072148459d3114b9ccaf43c02e0958 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Tue, 19 Nov 2024 16:14:14 +0100 -Subject: [PATCH] backend/epoll: implement eventfd wakeup notification - -Tries to mimic what happens in backend/kqueue. - -Closes #4 ---- - src/backend/epoll.zig | 42 ++++++++++++++++++++++++++++++++++++++++++ - 1 file changed, 42 insertions(+) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index ae4ec7d..f44d326 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -21,6 +21,12 @@ pub const Loop = struct { - - fd: posix.fd_t, - -+ /// The eventfd that this epoll queue always has a filter for. Writing -+ /// an empty message to this eventfd can be used to wake up the loop -+ /// at any time. Waking up the loop via this eventfd won't trigger any -+ /// particular completion, it just forces tick to cycle. -+ eventfd: xev.Async, -+ - /// The number of active completions. This DOES NOT include completions that - /// are queued in the submissions queue. - active: usize = 0, -@@ -56,8 +62,12 @@ pub const Loop = struct { - } = .{}, - - pub fn init(options: xev.Options) !Loop { -+ var eventfd = try xev.Async.init(); -+ errdefer eventfd.deinit(); -+ - var res: Loop = .{ - .fd = try posix.epoll_create1(std.os.linux.EPOLL.CLOEXEC), -+ .eventfd = eventfd, - .thread_pool = options.thread_pool, - .thread_pool_completions = undefined, - .cached_now = undefined, -@@ -68,6 +78,7 @@ pub const Loop = struct { - - pub fn deinit(self: *Loop) void { - posix.close(self.fd); -+ self.eventfd.deinit(); - } - - /// Run the event loop. See RunMode documentation for details on modes. -@@ -262,9 +273,26 @@ pub const Loop = struct { - // Initialize - if (!self.flags.init) { - self.flags.init = true; -+ - if (self.thread_pool != null) { - self.thread_pool_completions.init(); - } -+ -+ var ev: linux.epoll_event = .{ -+ .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, -+ .data = .{ .ptr = 0 }, -+ }; -+ posix.epoll_ctl( -+ self.fd, -+ linux.EPOLL.CTL_ADD, -+ self.eventfd.fd, -+ &ev, -+ ) catch |err| { -+ // We reset initialization because we can't do anything -+ // safely unless we get this mach port registered! -+ self.flags.init = false; -+ return err; -+ }; - } - - // Submit all the submissions. We copy the submission queue so that -@@ -369,6 +397,10 @@ pub const Loop = struct { - - // Process all our events and invoke their completion handlers - for (events[0..n]) |ev| { -+ // Zero data values are internal events that we do nothing -+ // on such as the eventfd wakeup. -+ if (ev.data.ptr == 0) continue; -+ - const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); - - // We get the fd and mark this as in progress we can properly -@@ -415,6 +447,7 @@ pub const Loop = struct { - const pool = self.thread_pool orelse return error.ThreadPoolRequired; - - // Setup our completion state so that thread_perform can do stuff -+ c.task_loop = self; - c.task_completions = &self.thread_pool_completions; - c.task = .{ .callback = Loop.thread_perform }; - -@@ -436,6 +469,14 @@ pub const Loop = struct { - - // Add to our completion queue - c.task_completions.push(c); -+ -+ // Wake up our main loop -+ c.task_loop.wakeup() catch {}; -+ } -+ -+ /// Sends an empty message to this loop's eventfd so that it wakes up. -+ fn wakeup(self: *Loop) !void { -+ try self.eventfd.notify(); - } - - fn start(self: *Loop, completion: *Completion) void { -@@ -800,6 +841,7 @@ pub const Completion = struct { - /// reliable way to get access to the loop and shouldn't be used - /// except internally. - task: ThreadPool.Task = undefined, -+ task_loop: *Loop = undefined, - task_completions: *Loop.TaskCompletionQueue = undefined, - task_result: Result = undefined, - diff --git a/third_party/modules/libxev/20241119.0-6afcde9/source.json b/third_party/modules/libxev/20241119.0-6afcde9/source.json deleted file mode 100644 index de3c2417..00000000 --- a/third_party/modules/libxev/20241119.0-6afcde9/source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "strip_prefix": "libxev-690c76fd792f001c5776716f1e7b04be2cc50b52", - "url": "https://github.com/zml/libxev/archive/690c76fd792f001c5776716f1e7b04be2cc50b52.tar.gz", - "integrity": "sha256-DV66ic8PcRnG3EdimswCleiHo/dDztgebz/1EY5XDXg=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "", - "main2.zig": "" - }, - "patches": { - "128.patch": "" - }, - "patch_strip": 1 -} diff --git a/third_party/modules/libxev/20241208.0-db6a52b/MODULE.bazel b/third_party/modules/libxev/20241208.0-db6a52b/MODULE.bazel deleted file mode 100644 index ce556ab1..00000000 --- a/third_party/modules/libxev/20241208.0-db6a52b/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20241208.0-db6a52b", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20241208.0-db6a52b/overlay/BUILD.bazel b/third_party/modules/libxev/20241208.0-db6a52b/overlay/BUILD.bazel deleted file mode 100644 index 7c0c4ce9..00000000 --- a/third_party/modules/libxev/20241208.0-db6a52b/overlay/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "xev", - srcs = glob([ - "src/*.zig", - "src/backend/*.zig", - "src/linux/*.zig", - "src/watcher/*.zig", - ]), - main = "main2.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/libxev/20241208.0-db6a52b/overlay/MODULE.bazel b/third_party/modules/libxev/20241208.0-db6a52b/overlay/MODULE.bazel deleted file mode 100644 index ce556ab1..00000000 --- a/third_party/modules/libxev/20241208.0-db6a52b/overlay/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20241208.0-db6a52b", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20241208.0-db6a52b/overlay/main2.zig b/third_party/modules/libxev/20241208.0-db6a52b/overlay/main2.zig deleted file mode 100644 index 4166925a..00000000 --- a/third_party/modules/libxev/20241208.0-db6a52b/overlay/main2.zig +++ /dev/null @@ -1,22 +0,0 @@ -const builtin = @import("builtin"); -const root = @import("root"); - -const main = @import("src/main.zig"); - -pub const ThreadPool = main.ThreadPool; -pub const stream = main.stream; - -pub const Options = struct { - linux_backend: main.Backend = .epoll, -}; - -pub const options: Options = if (@hasDecl(root, "xev_options")) root.xev_options else .{}; - -const default: main.Backend = switch (builtin.os.tag) { - .ios, .macos => .kqueue, - .linux => options.linux_backend, - .wasi => .wasi_poll, - .windows => .iocp, - else => @compileError("Unsupported OS"), -}; -pub usingnamespace default.Api(); diff --git a/third_party/modules/libxev/20241208.0-db6a52b/patches/128.patch b/third_party/modules/libxev/20241208.0-db6a52b/patches/128.patch deleted file mode 100644 index 6db29cc8..00000000 --- a/third_party/modules/libxev/20241208.0-db6a52b/patches/128.patch +++ /dev/null @@ -1,348 +0,0 @@ -From 7b345e8d423f6cb1907f0dcba1d3c2f82e4c6b4d Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Tue, 19 Nov 2024 16:14:14 +0100 -Subject: [PATCH 1/5] backend/epoll: implement eventfd wakeup notification - -Tries to mimic what happens in backend/kqueue. - -Closes #4 ---- - src/backend/epoll.zig | 42 ++++++++++++++++++++++++++++++++++++++++++ - 1 file changed, 42 insertions(+) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index ae4ec7d..f44d326 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -21,6 +21,12 @@ pub const Loop = struct { - - fd: posix.fd_t, - -+ /// The eventfd that this epoll queue always has a filter for. Writing -+ /// an empty message to this eventfd can be used to wake up the loop -+ /// at any time. Waking up the loop via this eventfd won't trigger any -+ /// particular completion, it just forces tick to cycle. -+ eventfd: xev.Async, -+ - /// The number of active completions. This DOES NOT include completions that - /// are queued in the submissions queue. - active: usize = 0, -@@ -56,8 +62,12 @@ pub const Loop = struct { - } = .{}, - - pub fn init(options: xev.Options) !Loop { -+ var eventfd = try xev.Async.init(); -+ errdefer eventfd.deinit(); -+ - var res: Loop = .{ - .fd = try posix.epoll_create1(std.os.linux.EPOLL.CLOEXEC), -+ .eventfd = eventfd, - .thread_pool = options.thread_pool, - .thread_pool_completions = undefined, - .cached_now = undefined, -@@ -68,6 +78,7 @@ pub const Loop = struct { - - pub fn deinit(self: *Loop) void { - posix.close(self.fd); -+ self.eventfd.deinit(); - } - - /// Run the event loop. See RunMode documentation for details on modes. -@@ -262,9 +273,26 @@ pub const Loop = struct { - // Initialize - if (!self.flags.init) { - self.flags.init = true; -+ - if (self.thread_pool != null) { - self.thread_pool_completions.init(); - } -+ -+ var ev: linux.epoll_event = .{ -+ .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, -+ .data = .{ .ptr = 0 }, -+ }; -+ posix.epoll_ctl( -+ self.fd, -+ linux.EPOLL.CTL_ADD, -+ self.eventfd.fd, -+ &ev, -+ ) catch |err| { -+ // We reset initialization because we can't do anything -+ // safely unless we get this mach port registered! -+ self.flags.init = false; -+ return err; -+ }; - } - - // Submit all the submissions. We copy the submission queue so that -@@ -369,6 +397,10 @@ pub const Loop = struct { - - // Process all our events and invoke their completion handlers - for (events[0..n]) |ev| { -+ // Zero data values are internal events that we do nothing -+ // on such as the eventfd wakeup. -+ if (ev.data.ptr == 0) continue; -+ - const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); - - // We get the fd and mark this as in progress we can properly -@@ -415,6 +447,7 @@ pub const Loop = struct { - const pool = self.thread_pool orelse return error.ThreadPoolRequired; - - // Setup our completion state so that thread_perform can do stuff -+ c.task_loop = self; - c.task_completions = &self.thread_pool_completions; - c.task = .{ .callback = Loop.thread_perform }; - -@@ -436,6 +469,14 @@ pub const Loop = struct { - - // Add to our completion queue - c.task_completions.push(c); -+ -+ // Wake up our main loop -+ c.task_loop.wakeup() catch {}; -+ } -+ -+ /// Sends an empty message to this loop's eventfd so that it wakes up. -+ fn wakeup(self: *Loop) !void { -+ try self.eventfd.notify(); - } - - fn start(self: *Loop, completion: *Completion) void { -@@ -800,6 +841,7 @@ pub const Completion = struct { - /// reliable way to get access to the loop and shouldn't be used - /// except internally. - task: ThreadPool.Task = undefined, -+ task_loop: *Loop = undefined, - task_completions: *Loop.TaskCompletionQueue = undefined, - task_result: Result = undefined, - - -From 40dc25fa922fe436418f9c01acd9b76a74593ddb Mon Sep 17 00:00:00 2001 -From: Corentin Godeau -Date: Tue, 14 Jan 2025 14:43:54 +0000 -Subject: [PATCH 2/5] backend/epoll: read the wakeup eventfd to avoid being - awaken again - ---- - src/backend/epoll.zig | 11 +++++++---- - 1 file changed, 7 insertions(+), 4 deletions(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index f44d326..f84c687 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -280,7 +280,7 @@ pub const Loop = struct { - - var ev: linux.epoll_event = .{ - .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, -- .data = .{ .ptr = 0 }, -+ .data = .{ .fd = self.eventfd.fd }, - }; - posix.epoll_ctl( - self.fd, -@@ -397,9 +397,12 @@ pub const Loop = struct { - - // Process all our events and invoke their completion handlers - for (events[0..n]) |ev| { -- // Zero data values are internal events that we do nothing -- // on such as the eventfd wakeup. -- if (ev.data.ptr == 0) continue; -+ // Handle wakeup eventfd -+ if (ev.data.fd == self.eventfd.fd) { -+ var buffer: u64 = undefined; -+ _ = posix.read(self.eventfd.fd, std.mem.asBytes(&buffer)) catch {}; -+ continue; -+ } - - const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); - - -From c082ed1270a59d4276d7ac6752833b6730fa73d5 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Fri, 17 Jan 2025 20:47:42 +0000 -Subject: [PATCH 3/5] epoll: use infinite timeout for epoll_wait - -Since eventfd is now implemented. ---- - src/backend/epoll.zig | 4 +--- - 1 file changed, 1 insertion(+), 3 deletions(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index f84c687..e3eee20 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -375,9 +375,7 @@ pub const Loop = struct { - const timeout: i32 = if (wait_rem == 0) 0 else timeout: { - // If we have a timer, we want to set the timeout to our next - // timer value. If we have no timer, we wait forever. -- // TODO: do not wait 100ms here, use an eventfd for our -- // thread pool to wake us up. -- const t = self.timers.peek() orelse break :timeout 100; -+ const t = self.timers.peek() orelse break :timeout -1; - - // Determine the time in milliseconds. - const ms_now = @as(u64, @intCast(self.cached_now.tv_sec)) * std.time.ms_per_s + - -From e8f23275bf736bcc39cb0c9e4617f6e1338c5880 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Fri, 17 Jan 2025 20:48:27 +0000 -Subject: [PATCH 4/5] epoll,kqueue: dispatch close in threadpool - -Close might block, so dispatch it inside a threadpool. ---- - src/backend/epoll.zig | 15 +++++++++++++-- - src/backend/kqueue.zig | 6 +++++- - src/watcher/stream.zig | 16 ++++++++++++++++ - 3 files changed, 34 insertions(+), 3 deletions(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index e3eee20..0f4a2ac 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -700,6 +700,13 @@ pub const Loop = struct { - }, - - .close => |v| res: { -+ if (completion.flags.threadpool) { -+ if (self.thread_schedule(completion)) |_| -+ return -+ else |err| -+ break :res .{ .close = err }; -+ } -+ - posix.close(v.fd); - break :res .{ .close = {} }; - }, -@@ -909,7 +916,6 @@ pub const Completion = struct { - // This should never happen because we always do these synchronously - // or in another location. - .cancel, -- .close, - .noop, - .shutdown, - .timer, -@@ -1015,6 +1021,11 @@ pub const Completion = struct { - err, - }; - }, -+ -+ .close => |*op| res: { -+ posix.close(op.fd); -+ break :res .{ .close = {} }; -+ }, - }; - } - -@@ -1277,7 +1288,7 @@ pub const AcceptError = posix.EpollCtlError || error{ - Unknown, - }; - --pub const CloseError = posix.EpollCtlError || error{ -+pub const CloseError = posix.EpollCtlError || ThreadPoolError || error{ - Unknown, - }; - -diff --git a/src/backend/kqueue.zig b/src/backend/kqueue.zig -index 456f64f..9914309 100644 ---- a/src/backend/kqueue.zig -+++ b/src/backend/kqueue.zig -@@ -1102,7 +1102,6 @@ pub const Completion = struct { - fn perform(self: *Completion, ev_: ?*const Kevent) Result { - return switch (self.op) { - .cancel, -- .close, - .noop, - .timer, - .shutdown, -@@ -1232,6 +1231,11 @@ pub const Completion = struct { - - break :res .{ .proc = 0 }; - }, -+ -+ .close => |*op| res: { -+ posix.close(op.fd); -+ break :res .{ .close = {} }; -+ }, - }; - } - -diff --git a/src/watcher/stream.zig b/src/watcher/stream.zig -index 7f5df6f..bc95282 100644 ---- a/src/watcher/stream.zig -+++ b/src/watcher/stream.zig -@@ -80,6 +80,22 @@ pub fn Closeable(comptime xev: type, comptime T: type, comptime options: Options - }).callback, - }; - -+ // If we're dup-ing, then we ask the backend to manage the fd. -+ switch (xev.backend) { -+ .io_uring, -+ .wasi_poll, -+ .iocp, -+ => {}, -+ -+ .epoll => { -+ c.flags.threadpool = true; -+ }, -+ -+ .kqueue => { -+ c.flags.threadpool = true; -+ }, -+ } -+ - loop.add(c); - } - }; - -From 378a3968b8820aef5a6ab64d1893d022b1b6d70c Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Fri, 17 Jan 2025 20:59:18 +0000 -Subject: [PATCH 5/5] epoll: don't count immediate actions - -If an immediate action is dispatched, the loop might block -on epoll_wait even though only one action was requested. ---- - src/backend/epoll.zig | 15 ++++++++++++++- - 1 file changed, 14 insertions(+), 1 deletion(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index 0f4a2ac..fb7e59e 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -297,6 +297,7 @@ pub const Loop = struct { - - // Submit all the submissions. We copy the submission queue so that - // any resubmits don't cause an infinite loop. -+ var wait_rem: usize = @intCast(wait); - var queued = self.submissions; - self.submissions = .{}; - while (queued.pop()) |c| { -@@ -304,6 +305,19 @@ pub const Loop = struct { - // This usually means that we switched them to be deleted or - // something. - if (c.flags.state != .adding) continue; -+ -+ // These operations happen synchronously. Ensure they are -+ // decremented from wait_rem. -+ switch (c.op) { -+ .cancel, -+ // should noop be counted? -+ // .noop, -+ .shutdown, -+ .timer, -+ => wait_rem -|= 1, -+ else => {}, -+ } -+ - self.start(c); - } - -@@ -322,7 +336,6 @@ pub const Loop = struct { - - // Wait and process events. We only do this if we have any active. - var events: [1024]linux.epoll_event = undefined; -- var wait_rem: usize = @intCast(wait); - while (self.active > 0 and (wait == 0 or wait_rem > 0)) { - self.update_now(); - const now_timer: Operation.Timer = .{ .next = self.cached_now }; diff --git a/third_party/modules/libxev/20241208.0-db6a52b/source.json b/third_party/modules/libxev/20241208.0-db6a52b/source.json deleted file mode 100644 index f08c9517..00000000 --- a/third_party/modules/libxev/20241208.0-db6a52b/source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "strip_prefix": "libxev-db6a52bafadf00360e675fefa7926e8e6c0e9931", - "url": "https://github.com/zml/libxev/archive/db6a52bafadf00360e675fefa7926e8e6c0e9931.tar.gz", - "integrity": "sha256-4GT5wkfkZnIjNv20yDiWEzHAhbIiwHHJfS7A4u/LoNQ=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "", - "main2.zig": "" - }, - "patches": { - "128.patch": "" - }, - "patch_strip": 1 -} diff --git a/third_party/modules/libxev/20241208.1-db6a52b/MODULE.bazel b/third_party/modules/libxev/20241208.1-db6a52b/MODULE.bazel deleted file mode 100644 index ea8c55d7..00000000 --- a/third_party/modules/libxev/20241208.1-db6a52b/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20241208.1-db6a52b", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20241208.1-db6a52b/overlay/BUILD.bazel b/third_party/modules/libxev/20241208.1-db6a52b/overlay/BUILD.bazel deleted file mode 100644 index 7c0c4ce9..00000000 --- a/third_party/modules/libxev/20241208.1-db6a52b/overlay/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "xev", - srcs = glob([ - "src/*.zig", - "src/backend/*.zig", - "src/linux/*.zig", - "src/watcher/*.zig", - ]), - main = "main2.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/libxev/20241208.1-db6a52b/overlay/MODULE.bazel b/third_party/modules/libxev/20241208.1-db6a52b/overlay/MODULE.bazel deleted file mode 100644 index ce556ab1..00000000 --- a/third_party/modules/libxev/20241208.1-db6a52b/overlay/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20241208.0-db6a52b", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20241208.1-db6a52b/overlay/main2.zig b/third_party/modules/libxev/20241208.1-db6a52b/overlay/main2.zig deleted file mode 100644 index 4166925a..00000000 --- a/third_party/modules/libxev/20241208.1-db6a52b/overlay/main2.zig +++ /dev/null @@ -1,22 +0,0 @@ -const builtin = @import("builtin"); -const root = @import("root"); - -const main = @import("src/main.zig"); - -pub const ThreadPool = main.ThreadPool; -pub const stream = main.stream; - -pub const Options = struct { - linux_backend: main.Backend = .epoll, -}; - -pub const options: Options = if (@hasDecl(root, "xev_options")) root.xev_options else .{}; - -const default: main.Backend = switch (builtin.os.tag) { - .ios, .macos => .kqueue, - .linux => options.linux_backend, - .wasi => .wasi_poll, - .windows => .iocp, - else => @compileError("Unsupported OS"), -}; -pub usingnamespace default.Api(); diff --git a/third_party/modules/libxev/20241208.1-db6a52b/patches/128.patch b/third_party/modules/libxev/20241208.1-db6a52b/patches/128.patch deleted file mode 100644 index 361bc9d1..00000000 --- a/third_party/modules/libxev/20241208.1-db6a52b/patches/128.patch +++ /dev/null @@ -1,323 +0,0 @@ -From 8927108937e23ce2ad547a310dabd976ddcaa39d Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Tue, 19 Nov 2024 16:14:14 +0100 -Subject: [PATCH 1/5] backend/epoll: implement eventfd wakeup notification - -Tries to mimic what happens in backend/kqueue. - -Closes #4 ---- - src/backend/epoll.zig | 42 ++++++++++++++++++++++++++++++++++++++++++ - 1 file changed, 42 insertions(+) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index ae4ec7d..f44d326 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -21,6 +21,12 @@ pub const Loop = struct { - - fd: posix.fd_t, - -+ /// The eventfd that this epoll queue always has a filter for. Writing -+ /// an empty message to this eventfd can be used to wake up the loop -+ /// at any time. Waking up the loop via this eventfd won't trigger any -+ /// particular completion, it just forces tick to cycle. -+ eventfd: xev.Async, -+ - /// The number of active completions. This DOES NOT include completions that - /// are queued in the submissions queue. - active: usize = 0, -@@ -56,8 +62,12 @@ pub const Loop = struct { - } = .{}, - - pub fn init(options: xev.Options) !Loop { -+ var eventfd = try xev.Async.init(); -+ errdefer eventfd.deinit(); -+ - var res: Loop = .{ - .fd = try posix.epoll_create1(std.os.linux.EPOLL.CLOEXEC), -+ .eventfd = eventfd, - .thread_pool = options.thread_pool, - .thread_pool_completions = undefined, - .cached_now = undefined, -@@ -68,6 +78,7 @@ pub const Loop = struct { - - pub fn deinit(self: *Loop) void { - posix.close(self.fd); -+ self.eventfd.deinit(); - } - - /// Run the event loop. See RunMode documentation for details on modes. -@@ -262,9 +273,26 @@ pub const Loop = struct { - // Initialize - if (!self.flags.init) { - self.flags.init = true; -+ - if (self.thread_pool != null) { - self.thread_pool_completions.init(); - } -+ -+ var ev: linux.epoll_event = .{ -+ .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, -+ .data = .{ .ptr = 0 }, -+ }; -+ posix.epoll_ctl( -+ self.fd, -+ linux.EPOLL.CTL_ADD, -+ self.eventfd.fd, -+ &ev, -+ ) catch |err| { -+ // We reset initialization because we can't do anything -+ // safely unless we get this mach port registered! -+ self.flags.init = false; -+ return err; -+ }; - } - - // Submit all the submissions. We copy the submission queue so that -@@ -369,6 +397,10 @@ pub const Loop = struct { - - // Process all our events and invoke their completion handlers - for (events[0..n]) |ev| { -+ // Zero data values are internal events that we do nothing -+ // on such as the eventfd wakeup. -+ if (ev.data.ptr == 0) continue; -+ - const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); - - // We get the fd and mark this as in progress we can properly -@@ -415,6 +447,7 @@ pub const Loop = struct { - const pool = self.thread_pool orelse return error.ThreadPoolRequired; - - // Setup our completion state so that thread_perform can do stuff -+ c.task_loop = self; - c.task_completions = &self.thread_pool_completions; - c.task = .{ .callback = Loop.thread_perform }; - -@@ -436,6 +469,14 @@ pub const Loop = struct { - - // Add to our completion queue - c.task_completions.push(c); -+ -+ // Wake up our main loop -+ c.task_loop.wakeup() catch {}; -+ } -+ -+ /// Sends an empty message to this loop's eventfd so that it wakes up. -+ fn wakeup(self: *Loop) !void { -+ try self.eventfd.notify(); - } - - fn start(self: *Loop, completion: *Completion) void { -@@ -800,6 +841,7 @@ pub const Completion = struct { - /// reliable way to get access to the loop and shouldn't be used - /// except internally. - task: ThreadPool.Task = undefined, -+ task_loop: *Loop = undefined, - task_completions: *Loop.TaskCompletionQueue = undefined, - task_result: Result = undefined, - - -From bf6c19eb1c06d1fb1b3cb5a66ea2ce8f743da5ba Mon Sep 17 00:00:00 2001 -From: Corentin Godeau -Date: Tue, 14 Jan 2025 14:43:54 +0000 -Subject: [PATCH 2/5] backend/epoll: read the wakeup eventfd to avoid being - awaken again - ---- - src/backend/epoll.zig | 11 +++++++---- - 1 file changed, 7 insertions(+), 4 deletions(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index f44d326..f84c687 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -280,7 +280,7 @@ pub const Loop = struct { - - var ev: linux.epoll_event = .{ - .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, -- .data = .{ .ptr = 0 }, -+ .data = .{ .fd = self.eventfd.fd }, - }; - posix.epoll_ctl( - self.fd, -@@ -397,9 +397,12 @@ pub const Loop = struct { - - // Process all our events and invoke their completion handlers - for (events[0..n]) |ev| { -- // Zero data values are internal events that we do nothing -- // on such as the eventfd wakeup. -- if (ev.data.ptr == 0) continue; -+ // Handle wakeup eventfd -+ if (ev.data.fd == self.eventfd.fd) { -+ var buffer: u64 = undefined; -+ _ = posix.read(self.eventfd.fd, std.mem.asBytes(&buffer)) catch {}; -+ continue; -+ } - - const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); - - -From c0ae753a9874bf78a48bf25b4362b6cca4de1da1 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Fri, 17 Jan 2025 20:47:42 +0000 -Subject: [PATCH 3/5] epoll: use infinite timeout for epoll_wait - -Since eventfd is now implemented. ---- - src/backend/epoll.zig | 4 +--- - 1 file changed, 1 insertion(+), 3 deletions(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index f84c687..e3eee20 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -375,9 +375,7 @@ pub const Loop = struct { - const timeout: i32 = if (wait_rem == 0) 0 else timeout: { - // If we have a timer, we want to set the timeout to our next - // timer value. If we have no timer, we wait forever. -- // TODO: do not wait 100ms here, use an eventfd for our -- // thread pool to wake us up. -- const t = self.timers.peek() orelse break :timeout 100; -+ const t = self.timers.peek() orelse break :timeout -1; - - // Determine the time in milliseconds. - const ms_now = @as(u64, @intCast(self.cached_now.tv_sec)) * std.time.ms_per_s + - -From 84e75a10ec1c10ea2a13edfcd8a31e582458fc88 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Fri, 17 Jan 2025 20:48:27 +0000 -Subject: [PATCH 4/5] epoll: dispatch close in threadpool - -Close might block, so dispatch it inside a threadpool. ---- - src/backend/epoll.zig | 15 +++++++++++++-- - src/watcher/stream.zig | 16 ++++++++++++++++ - 2 files changed, 29 insertions(+), 2 deletions(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index e3eee20..0f4a2ac 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -700,6 +700,13 @@ pub const Loop = struct { - }, - - .close => |v| res: { -+ if (completion.flags.threadpool) { -+ if (self.thread_schedule(completion)) |_| -+ return -+ else |err| -+ break :res .{ .close = err }; -+ } -+ - posix.close(v.fd); - break :res .{ .close = {} }; - }, -@@ -909,7 +916,6 @@ pub const Completion = struct { - // This should never happen because we always do these synchronously - // or in another location. - .cancel, -- .close, - .noop, - .shutdown, - .timer, -@@ -1015,6 +1021,11 @@ pub const Completion = struct { - err, - }; - }, -+ -+ .close => |*op| res: { -+ posix.close(op.fd); -+ break :res .{ .close = {} }; -+ }, - }; - } - -@@ -1277,7 +1288,7 @@ pub const AcceptError = posix.EpollCtlError || error{ - Unknown, - }; - --pub const CloseError = posix.EpollCtlError || error{ -+pub const CloseError = posix.EpollCtlError || ThreadPoolError || error{ - Unknown, - }; - -diff --git a/src/watcher/stream.zig b/src/watcher/stream.zig -index 7f5df6f..bc95282 100644 ---- a/src/watcher/stream.zig -+++ b/src/watcher/stream.zig -@@ -80,6 +80,22 @@ pub fn Closeable(comptime xev: type, comptime T: type, comptime options: Options - }).callback, - }; - -+ // If we're dup-ing, then we ask the backend to manage the fd. -+ switch (xev.backend) { -+ .io_uring, -+ .wasi_poll, -+ .iocp, -+ => {}, -+ -+ .epoll => { -+ c.flags.threadpool = true; -+ }, -+ -+ .kqueue => { -+ c.flags.threadpool = true; -+ }, -+ } -+ - loop.add(c); - } - }; - -From 5bc3ad9f23a03ff3af7a4ec33c8bfa4f3cc6bae9 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Fri, 17 Jan 2025 20:59:18 +0000 -Subject: [PATCH 5/5] epoll: don't count immediate actions - -If an immediate action is dispatched, the loop might block -on epoll_wait even though only one action was requested. ---- - src/backend/epoll.zig | 15 ++++++++++++++- - 1 file changed, 14 insertions(+), 1 deletion(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index 0f4a2ac..fb7e59e 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -297,6 +297,7 @@ pub const Loop = struct { - - // Submit all the submissions. We copy the submission queue so that - // any resubmits don't cause an infinite loop. -+ var wait_rem: usize = @intCast(wait); - var queued = self.submissions; - self.submissions = .{}; - while (queued.pop()) |c| { -@@ -304,6 +305,19 @@ pub const Loop = struct { - // This usually means that we switched them to be deleted or - // something. - if (c.flags.state != .adding) continue; -+ -+ // These operations happen synchronously. Ensure they are -+ // decremented from wait_rem. -+ switch (c.op) { -+ .cancel, -+ // should noop be counted? -+ // .noop, -+ .shutdown, -+ .timer, -+ => wait_rem -|= 1, -+ else => {}, -+ } -+ - self.start(c); - } - -@@ -322,7 +336,6 @@ pub const Loop = struct { - - // Wait and process events. We only do this if we have any active. - var events: [1024]linux.epoll_event = undefined; -- var wait_rem: usize = @intCast(wait); - while (self.active > 0 and (wait == 0 or wait_rem > 0)) { - self.update_now(); - const now_timer: Operation.Timer = .{ .next = self.cached_now }; diff --git a/third_party/modules/libxev/20241208.1-db6a52b/source.json b/third_party/modules/libxev/20241208.1-db6a52b/source.json deleted file mode 100644 index f08c9517..00000000 --- a/third_party/modules/libxev/20241208.1-db6a52b/source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "strip_prefix": "libxev-db6a52bafadf00360e675fefa7926e8e6c0e9931", - "url": "https://github.com/zml/libxev/archive/db6a52bafadf00360e675fefa7926e8e6c0e9931.tar.gz", - "integrity": "sha256-4GT5wkfkZnIjNv20yDiWEzHAhbIiwHHJfS7A4u/LoNQ=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "", - "main2.zig": "" - }, - "patches": { - "128.patch": "" - }, - "patch_strip": 1 -} diff --git a/third_party/modules/libxev/20241208.2-db6a52b/MODULE.bazel b/third_party/modules/libxev/20241208.2-db6a52b/MODULE.bazel deleted file mode 100644 index 1d593b7d..00000000 --- a/third_party/modules/libxev/20241208.2-db6a52b/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20241208.2-db6a52b", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20241208.2-db6a52b/overlay/BUILD.bazel b/third_party/modules/libxev/20241208.2-db6a52b/overlay/BUILD.bazel deleted file mode 100644 index 7c0c4ce9..00000000 --- a/third_party/modules/libxev/20241208.2-db6a52b/overlay/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "xev", - srcs = glob([ - "src/*.zig", - "src/backend/*.zig", - "src/linux/*.zig", - "src/watcher/*.zig", - ]), - main = "main2.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/libxev/20241208.2-db6a52b/overlay/MODULE.bazel b/third_party/modules/libxev/20241208.2-db6a52b/overlay/MODULE.bazel deleted file mode 100644 index ce556ab1..00000000 --- a/third_party/modules/libxev/20241208.2-db6a52b/overlay/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20241208.0-db6a52b", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20241208.2-db6a52b/overlay/main2.zig b/third_party/modules/libxev/20241208.2-db6a52b/overlay/main2.zig deleted file mode 100644 index 4166925a..00000000 --- a/third_party/modules/libxev/20241208.2-db6a52b/overlay/main2.zig +++ /dev/null @@ -1,22 +0,0 @@ -const builtin = @import("builtin"); -const root = @import("root"); - -const main = @import("src/main.zig"); - -pub const ThreadPool = main.ThreadPool; -pub const stream = main.stream; - -pub const Options = struct { - linux_backend: main.Backend = .epoll, -}; - -pub const options: Options = if (@hasDecl(root, "xev_options")) root.xev_options else .{}; - -const default: main.Backend = switch (builtin.os.tag) { - .ios, .macos => .kqueue, - .linux => options.linux_backend, - .wasi => .wasi_poll, - .windows => .iocp, - else => @compileError("Unsupported OS"), -}; -pub usingnamespace default.Api(); diff --git a/third_party/modules/libxev/20241208.2-db6a52b/patches/128.patch b/third_party/modules/libxev/20241208.2-db6a52b/patches/128.patch deleted file mode 100644 index 18349433..00000000 --- a/third_party/modules/libxev/20241208.2-db6a52b/patches/128.patch +++ /dev/null @@ -1,359 +0,0 @@ -From 7b345e8d423f6cb1907f0dcba1d3c2f82e4c6b4d Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Tue, 19 Nov 2024 16:14:14 +0100 -Subject: [PATCH 1/5] backend/epoll: implement eventfd wakeup notification - -Tries to mimic what happens in backend/kqueue. - -Closes #4 ---- - src/backend/epoll.zig | 42 ++++++++++++++++++++++++++++++++++++++++++ - 1 file changed, 42 insertions(+) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index ae4ec7d..f44d326 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -21,6 +21,12 @@ pub const Loop = struct { - - fd: posix.fd_t, - -+ /// The eventfd that this epoll queue always has a filter for. Writing -+ /// an empty message to this eventfd can be used to wake up the loop -+ /// at any time. Waking up the loop via this eventfd won't trigger any -+ /// particular completion, it just forces tick to cycle. -+ eventfd: xev.Async, -+ - /// The number of active completions. This DOES NOT include completions that - /// are queued in the submissions queue. - active: usize = 0, -@@ -56,8 +62,12 @@ pub const Loop = struct { - } = .{}, - - pub fn init(options: xev.Options) !Loop { -+ var eventfd = try xev.Async.init(); -+ errdefer eventfd.deinit(); -+ - var res: Loop = .{ - .fd = try posix.epoll_create1(std.os.linux.EPOLL.CLOEXEC), -+ .eventfd = eventfd, - .thread_pool = options.thread_pool, - .thread_pool_completions = undefined, - .cached_now = undefined, -@@ -68,6 +78,7 @@ pub const Loop = struct { - - pub fn deinit(self: *Loop) void { - posix.close(self.fd); -+ self.eventfd.deinit(); - } - - /// Run the event loop. See RunMode documentation for details on modes. -@@ -262,9 +273,26 @@ pub const Loop = struct { - // Initialize - if (!self.flags.init) { - self.flags.init = true; -+ - if (self.thread_pool != null) { - self.thread_pool_completions.init(); - } -+ -+ var ev: linux.epoll_event = .{ -+ .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, -+ .data = .{ .ptr = 0 }, -+ }; -+ posix.epoll_ctl( -+ self.fd, -+ linux.EPOLL.CTL_ADD, -+ self.eventfd.fd, -+ &ev, -+ ) catch |err| { -+ // We reset initialization because we can't do anything -+ // safely unless we get this mach port registered! -+ self.flags.init = false; -+ return err; -+ }; - } - - // Submit all the submissions. We copy the submission queue so that -@@ -369,6 +397,10 @@ pub const Loop = struct { - - // Process all our events and invoke their completion handlers - for (events[0..n]) |ev| { -+ // Zero data values are internal events that we do nothing -+ // on such as the eventfd wakeup. -+ if (ev.data.ptr == 0) continue; -+ - const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); - - // We get the fd and mark this as in progress we can properly -@@ -415,6 +447,7 @@ pub const Loop = struct { - const pool = self.thread_pool orelse return error.ThreadPoolRequired; - - // Setup our completion state so that thread_perform can do stuff -+ c.task_loop = self; - c.task_completions = &self.thread_pool_completions; - c.task = .{ .callback = Loop.thread_perform }; - -@@ -436,6 +469,14 @@ pub const Loop = struct { - - // Add to our completion queue - c.task_completions.push(c); -+ -+ // Wake up our main loop -+ c.task_loop.wakeup() catch {}; -+ } -+ -+ /// Sends an empty message to this loop's eventfd so that it wakes up. -+ fn wakeup(self: *Loop) !void { -+ try self.eventfd.notify(); - } - - fn start(self: *Loop, completion: *Completion) void { -@@ -800,6 +841,7 @@ pub const Completion = struct { - /// reliable way to get access to the loop and shouldn't be used - /// except internally. - task: ThreadPool.Task = undefined, -+ task_loop: *Loop = undefined, - task_completions: *Loop.TaskCompletionQueue = undefined, - task_result: Result = undefined, - --- -2.39.5 (Apple Git-154) - -From 40dc25fa922fe436418f9c01acd9b76a74593ddb Mon Sep 17 00:00:00 2001 -From: Corentin Godeau -Date: Tue, 14 Jan 2025 14:43:54 +0000 -Subject: [PATCH 2/5] backend/epoll: read the wakeup eventfd to avoid being - awaken again - ---- - src/backend/epoll.zig | 11 +++++++---- - 1 file changed, 7 insertions(+), 4 deletions(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index f44d326..f84c687 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -280,7 +280,7 @@ pub const Loop = struct { - - var ev: linux.epoll_event = .{ - .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, -- .data = .{ .ptr = 0 }, -+ .data = .{ .fd = self.eventfd.fd }, - }; - posix.epoll_ctl( - self.fd, -@@ -397,9 +397,12 @@ pub const Loop = struct { - - // Process all our events and invoke their completion handlers - for (events[0..n]) |ev| { -- // Zero data values are internal events that we do nothing -- // on such as the eventfd wakeup. -- if (ev.data.ptr == 0) continue; -+ // Handle wakeup eventfd -+ if (ev.data.fd == self.eventfd.fd) { -+ var buffer: u64 = undefined; -+ _ = posix.read(self.eventfd.fd, std.mem.asBytes(&buffer)) catch {}; -+ continue; -+ } - - const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); - --- -2.39.5 (Apple Git-154) - -From c082ed1270a59d4276d7ac6752833b6730fa73d5 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Fri, 17 Jan 2025 20:47:42 +0000 -Subject: [PATCH 3/5] epoll: use infinite timeout for epoll_wait - -Since eventfd is now implemented. ---- - src/backend/epoll.zig | 4 +--- - 1 file changed, 1 insertion(+), 3 deletions(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index f84c687..e3eee20 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -375,9 +375,7 @@ pub const Loop = struct { - const timeout: i32 = if (wait_rem == 0) 0 else timeout: { - // If we have a timer, we want to set the timeout to our next - // timer value. If we have no timer, we wait forever. -- // TODO: do not wait 100ms here, use an eventfd for our -- // thread pool to wake us up. -- const t = self.timers.peek() orelse break :timeout 100; -+ const t = self.timers.peek() orelse break :timeout -1; - - // Determine the time in milliseconds. - const ms_now = @as(u64, @intCast(self.cached_now.tv_sec)) * std.time.ms_per_s + --- -2.39.5 (Apple Git-154) - -From e8f23275bf736bcc39cb0c9e4617f6e1338c5880 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Fri, 17 Jan 2025 20:48:27 +0000 -Subject: [PATCH 4/5] epoll,kqueue: dispatch close in threadpool - -Close might block, so dispatch it inside a threadpool. ---- - src/backend/epoll.zig | 15 +++++++++++++-- - src/backend/kqueue.zig | 6 +++++- - src/watcher/stream.zig | 16 ++++++++++++++++ - 3 files changed, 34 insertions(+), 3 deletions(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index e3eee20..0f4a2ac 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -700,6 +700,13 @@ pub const Loop = struct { - }, - - .close => |v| res: { -+ if (completion.flags.threadpool) { -+ if (self.thread_schedule(completion)) |_| -+ return -+ else |err| -+ break :res .{ .close = err }; -+ } -+ - posix.close(v.fd); - break :res .{ .close = {} }; - }, -@@ -909,7 +916,6 @@ pub const Completion = struct { - // This should never happen because we always do these synchronously - // or in another location. - .cancel, -- .close, - .noop, - .shutdown, - .timer, -@@ -1015,6 +1021,11 @@ pub const Completion = struct { - err, - }; - }, -+ -+ .close => |*op| res: { -+ posix.close(op.fd); -+ break :res .{ .close = {} }; -+ }, - }; - } - -@@ -1277,7 +1288,7 @@ pub const AcceptError = posix.EpollCtlError || error{ - Unknown, - }; - --pub const CloseError = posix.EpollCtlError || error{ -+pub const CloseError = posix.EpollCtlError || ThreadPoolError || error{ - Unknown, - }; - -diff --git a/src/backend/kqueue.zig b/src/backend/kqueue.zig -index 456f64f..9914309 100644 ---- a/src/backend/kqueue.zig -+++ b/src/backend/kqueue.zig -@@ -1102,7 +1102,6 @@ pub const Completion = struct { - fn perform(self: *Completion, ev_: ?*const Kevent) Result { - return switch (self.op) { - .cancel, -- .close, - .noop, - .timer, - .shutdown, -@@ -1232,6 +1231,11 @@ pub const Completion = struct { - - break :res .{ .proc = 0 }; - }, -+ -+ .close => |*op| res: { -+ posix.close(op.fd); -+ break :res .{ .close = {} }; -+ }, - }; - } - -diff --git a/src/watcher/stream.zig b/src/watcher/stream.zig -index 7f5df6f..bc95282 100644 ---- a/src/watcher/stream.zig -+++ b/src/watcher/stream.zig -@@ -80,6 +80,22 @@ pub fn Closeable(comptime xev: type, comptime T: type, comptime options: Options - }).callback, - }; - -+ // If we're dup-ing, then we ask the backend to manage the fd. -+ switch (xev.backend) { -+ .io_uring, -+ .wasi_poll, -+ .iocp, -+ => {}, -+ -+ .epoll => { -+ c.flags.threadpool = true; -+ }, -+ -+ .kqueue => { -+ c.flags.threadpool = true; -+ }, -+ } -+ - loop.add(c); - } - }; --- -2.39.5 (Apple Git-154) - -From 378a3968b8820aef5a6ab64d1893d022b1b6d70c Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Fri, 17 Jan 2025 20:59:18 +0000 -Subject: [PATCH 5/5] epoll: don't count immediate actions - -If an immediate action is dispatched, the loop might block -on epoll_wait even though only one action was requested. ---- - src/backend/epoll.zig | 15 ++++++++++++++- - 1 file changed, 14 insertions(+), 1 deletion(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index 0f4a2ac..fb7e59e 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -297,6 +297,7 @@ pub const Loop = struct { - - // Submit all the submissions. We copy the submission queue so that - // any resubmits don't cause an infinite loop. -+ var wait_rem: usize = @intCast(wait); - var queued = self.submissions; - self.submissions = .{}; - while (queued.pop()) |c| { -@@ -304,6 +305,19 @@ pub const Loop = struct { - // This usually means that we switched them to be deleted or - // something. - if (c.flags.state != .adding) continue; -+ -+ // These operations happen synchronously. Ensure they are -+ // decremented from wait_rem. -+ switch (c.op) { -+ .cancel, -+ // should noop be counted? -+ // .noop, -+ .shutdown, -+ .timer, -+ => wait_rem -|= 1, -+ else => {}, -+ } -+ - self.start(c); - } - -@@ -322,7 +336,6 @@ pub const Loop = struct { - - // Wait and process events. We only do this if we have any active. - var events: [1024]linux.epoll_event = undefined; -- var wait_rem: usize = @intCast(wait); - while (self.active > 0 and (wait == 0 or wait_rem > 0)) { - self.update_now(); - const now_timer: Operation.Timer = .{ .next = self.cached_now }; --- -2.39.5 (Apple Git-154) - diff --git a/third_party/modules/libxev/20241208.2-db6a52b/source.json b/third_party/modules/libxev/20241208.2-db6a52b/source.json deleted file mode 100644 index f08c9517..00000000 --- a/third_party/modules/libxev/20241208.2-db6a52b/source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "strip_prefix": "libxev-db6a52bafadf00360e675fefa7926e8e6c0e9931", - "url": "https://github.com/zml/libxev/archive/db6a52bafadf00360e675fefa7926e8e6c0e9931.tar.gz", - "integrity": "sha256-4GT5wkfkZnIjNv20yDiWEzHAhbIiwHHJfS7A4u/LoNQ=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "", - "main2.zig": "" - }, - "patches": { - "128.patch": "" - }, - "patch_strip": 1 -} diff --git a/third_party/modules/libxev/20250124.0-31eed4e/MODULE.bazel b/third_party/modules/libxev/20250124.0-31eed4e/MODULE.bazel deleted file mode 100644 index 6102ba67..00000000 --- a/third_party/modules/libxev/20250124.0-31eed4e/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20250124.0-31eed4e", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20250124.0-31eed4e/overlay/BUILD.bazel b/third_party/modules/libxev/20250124.0-31eed4e/overlay/BUILD.bazel deleted file mode 100644 index 7c0c4ce9..00000000 --- a/third_party/modules/libxev/20250124.0-31eed4e/overlay/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "xev", - srcs = glob([ - "src/*.zig", - "src/backend/*.zig", - "src/linux/*.zig", - "src/watcher/*.zig", - ]), - main = "main2.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/libxev/20250124.0-31eed4e/overlay/MODULE.bazel b/third_party/modules/libxev/20250124.0-31eed4e/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/libxev/20250124.0-31eed4e/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20250124.0-31eed4e/overlay/main2.zig b/third_party/modules/libxev/20250124.0-31eed4e/overlay/main2.zig deleted file mode 100644 index 4166925a..00000000 --- a/third_party/modules/libxev/20250124.0-31eed4e/overlay/main2.zig +++ /dev/null @@ -1,22 +0,0 @@ -const builtin = @import("builtin"); -const root = @import("root"); - -const main = @import("src/main.zig"); - -pub const ThreadPool = main.ThreadPool; -pub const stream = main.stream; - -pub const Options = struct { - linux_backend: main.Backend = .epoll, -}; - -pub const options: Options = if (@hasDecl(root, "xev_options")) root.xev_options else .{}; - -const default: main.Backend = switch (builtin.os.tag) { - .ios, .macos => .kqueue, - .linux => options.linux_backend, - .wasi => .wasi_poll, - .windows => .iocp, - else => @compileError("Unsupported OS"), -}; -pub usingnamespace default.Api(); diff --git a/third_party/modules/libxev/20250124.0-31eed4e/patches/128.patch b/third_party/modules/libxev/20250124.0-31eed4e/patches/128.patch deleted file mode 100644 index 18349433..00000000 --- a/third_party/modules/libxev/20250124.0-31eed4e/patches/128.patch +++ /dev/null @@ -1,359 +0,0 @@ -From 7b345e8d423f6cb1907f0dcba1d3c2f82e4c6b4d Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Tue, 19 Nov 2024 16:14:14 +0100 -Subject: [PATCH 1/5] backend/epoll: implement eventfd wakeup notification - -Tries to mimic what happens in backend/kqueue. - -Closes #4 ---- - src/backend/epoll.zig | 42 ++++++++++++++++++++++++++++++++++++++++++ - 1 file changed, 42 insertions(+) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index ae4ec7d..f44d326 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -21,6 +21,12 @@ pub const Loop = struct { - - fd: posix.fd_t, - -+ /// The eventfd that this epoll queue always has a filter for. Writing -+ /// an empty message to this eventfd can be used to wake up the loop -+ /// at any time. Waking up the loop via this eventfd won't trigger any -+ /// particular completion, it just forces tick to cycle. -+ eventfd: xev.Async, -+ - /// The number of active completions. This DOES NOT include completions that - /// are queued in the submissions queue. - active: usize = 0, -@@ -56,8 +62,12 @@ pub const Loop = struct { - } = .{}, - - pub fn init(options: xev.Options) !Loop { -+ var eventfd = try xev.Async.init(); -+ errdefer eventfd.deinit(); -+ - var res: Loop = .{ - .fd = try posix.epoll_create1(std.os.linux.EPOLL.CLOEXEC), -+ .eventfd = eventfd, - .thread_pool = options.thread_pool, - .thread_pool_completions = undefined, - .cached_now = undefined, -@@ -68,6 +78,7 @@ pub const Loop = struct { - - pub fn deinit(self: *Loop) void { - posix.close(self.fd); -+ self.eventfd.deinit(); - } - - /// Run the event loop. See RunMode documentation for details on modes. -@@ -262,9 +273,26 @@ pub const Loop = struct { - // Initialize - if (!self.flags.init) { - self.flags.init = true; -+ - if (self.thread_pool != null) { - self.thread_pool_completions.init(); - } -+ -+ var ev: linux.epoll_event = .{ -+ .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, -+ .data = .{ .ptr = 0 }, -+ }; -+ posix.epoll_ctl( -+ self.fd, -+ linux.EPOLL.CTL_ADD, -+ self.eventfd.fd, -+ &ev, -+ ) catch |err| { -+ // We reset initialization because we can't do anything -+ // safely unless we get this mach port registered! -+ self.flags.init = false; -+ return err; -+ }; - } - - // Submit all the submissions. We copy the submission queue so that -@@ -369,6 +397,10 @@ pub const Loop = struct { - - // Process all our events and invoke their completion handlers - for (events[0..n]) |ev| { -+ // Zero data values are internal events that we do nothing -+ // on such as the eventfd wakeup. -+ if (ev.data.ptr == 0) continue; -+ - const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); - - // We get the fd and mark this as in progress we can properly -@@ -415,6 +447,7 @@ pub const Loop = struct { - const pool = self.thread_pool orelse return error.ThreadPoolRequired; - - // Setup our completion state so that thread_perform can do stuff -+ c.task_loop = self; - c.task_completions = &self.thread_pool_completions; - c.task = .{ .callback = Loop.thread_perform }; - -@@ -436,6 +469,14 @@ pub const Loop = struct { - - // Add to our completion queue - c.task_completions.push(c); -+ -+ // Wake up our main loop -+ c.task_loop.wakeup() catch {}; -+ } -+ -+ /// Sends an empty message to this loop's eventfd so that it wakes up. -+ fn wakeup(self: *Loop) !void { -+ try self.eventfd.notify(); - } - - fn start(self: *Loop, completion: *Completion) void { -@@ -800,6 +841,7 @@ pub const Completion = struct { - /// reliable way to get access to the loop and shouldn't be used - /// except internally. - task: ThreadPool.Task = undefined, -+ task_loop: *Loop = undefined, - task_completions: *Loop.TaskCompletionQueue = undefined, - task_result: Result = undefined, - --- -2.39.5 (Apple Git-154) - -From 40dc25fa922fe436418f9c01acd9b76a74593ddb Mon Sep 17 00:00:00 2001 -From: Corentin Godeau -Date: Tue, 14 Jan 2025 14:43:54 +0000 -Subject: [PATCH 2/5] backend/epoll: read the wakeup eventfd to avoid being - awaken again - ---- - src/backend/epoll.zig | 11 +++++++---- - 1 file changed, 7 insertions(+), 4 deletions(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index f44d326..f84c687 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -280,7 +280,7 @@ pub const Loop = struct { - - var ev: linux.epoll_event = .{ - .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, -- .data = .{ .ptr = 0 }, -+ .data = .{ .fd = self.eventfd.fd }, - }; - posix.epoll_ctl( - self.fd, -@@ -397,9 +397,12 @@ pub const Loop = struct { - - // Process all our events and invoke their completion handlers - for (events[0..n]) |ev| { -- // Zero data values are internal events that we do nothing -- // on such as the eventfd wakeup. -- if (ev.data.ptr == 0) continue; -+ // Handle wakeup eventfd -+ if (ev.data.fd == self.eventfd.fd) { -+ var buffer: u64 = undefined; -+ _ = posix.read(self.eventfd.fd, std.mem.asBytes(&buffer)) catch {}; -+ continue; -+ } - - const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); - --- -2.39.5 (Apple Git-154) - -From c082ed1270a59d4276d7ac6752833b6730fa73d5 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Fri, 17 Jan 2025 20:47:42 +0000 -Subject: [PATCH 3/5] epoll: use infinite timeout for epoll_wait - -Since eventfd is now implemented. ---- - src/backend/epoll.zig | 4 +--- - 1 file changed, 1 insertion(+), 3 deletions(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index f84c687..e3eee20 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -375,9 +375,7 @@ pub const Loop = struct { - const timeout: i32 = if (wait_rem == 0) 0 else timeout: { - // If we have a timer, we want to set the timeout to our next - // timer value. If we have no timer, we wait forever. -- // TODO: do not wait 100ms here, use an eventfd for our -- // thread pool to wake us up. -- const t = self.timers.peek() orelse break :timeout 100; -+ const t = self.timers.peek() orelse break :timeout -1; - - // Determine the time in milliseconds. - const ms_now = @as(u64, @intCast(self.cached_now.tv_sec)) * std.time.ms_per_s + --- -2.39.5 (Apple Git-154) - -From e8f23275bf736bcc39cb0c9e4617f6e1338c5880 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Fri, 17 Jan 2025 20:48:27 +0000 -Subject: [PATCH 4/5] epoll,kqueue: dispatch close in threadpool - -Close might block, so dispatch it inside a threadpool. ---- - src/backend/epoll.zig | 15 +++++++++++++-- - src/backend/kqueue.zig | 6 +++++- - src/watcher/stream.zig | 16 ++++++++++++++++ - 3 files changed, 34 insertions(+), 3 deletions(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index e3eee20..0f4a2ac 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -700,6 +700,13 @@ pub const Loop = struct { - }, - - .close => |v| res: { -+ if (completion.flags.threadpool) { -+ if (self.thread_schedule(completion)) |_| -+ return -+ else |err| -+ break :res .{ .close = err }; -+ } -+ - posix.close(v.fd); - break :res .{ .close = {} }; - }, -@@ -909,7 +916,6 @@ pub const Completion = struct { - // This should never happen because we always do these synchronously - // or in another location. - .cancel, -- .close, - .noop, - .shutdown, - .timer, -@@ -1015,6 +1021,11 @@ pub const Completion = struct { - err, - }; - }, -+ -+ .close => |*op| res: { -+ posix.close(op.fd); -+ break :res .{ .close = {} }; -+ }, - }; - } - -@@ -1277,7 +1288,7 @@ pub const AcceptError = posix.EpollCtlError || error{ - Unknown, - }; - --pub const CloseError = posix.EpollCtlError || error{ -+pub const CloseError = posix.EpollCtlError || ThreadPoolError || error{ - Unknown, - }; - -diff --git a/src/backend/kqueue.zig b/src/backend/kqueue.zig -index 456f64f..9914309 100644 ---- a/src/backend/kqueue.zig -+++ b/src/backend/kqueue.zig -@@ -1102,7 +1102,6 @@ pub const Completion = struct { - fn perform(self: *Completion, ev_: ?*const Kevent) Result { - return switch (self.op) { - .cancel, -- .close, - .noop, - .timer, - .shutdown, -@@ -1232,6 +1231,11 @@ pub const Completion = struct { - - break :res .{ .proc = 0 }; - }, -+ -+ .close => |*op| res: { -+ posix.close(op.fd); -+ break :res .{ .close = {} }; -+ }, - }; - } - -diff --git a/src/watcher/stream.zig b/src/watcher/stream.zig -index 7f5df6f..bc95282 100644 ---- a/src/watcher/stream.zig -+++ b/src/watcher/stream.zig -@@ -80,6 +80,22 @@ pub fn Closeable(comptime xev: type, comptime T: type, comptime options: Options - }).callback, - }; - -+ // If we're dup-ing, then we ask the backend to manage the fd. -+ switch (xev.backend) { -+ .io_uring, -+ .wasi_poll, -+ .iocp, -+ => {}, -+ -+ .epoll => { -+ c.flags.threadpool = true; -+ }, -+ -+ .kqueue => { -+ c.flags.threadpool = true; -+ }, -+ } -+ - loop.add(c); - } - }; --- -2.39.5 (Apple Git-154) - -From 378a3968b8820aef5a6ab64d1893d022b1b6d70c Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Fri, 17 Jan 2025 20:59:18 +0000 -Subject: [PATCH 5/5] epoll: don't count immediate actions - -If an immediate action is dispatched, the loop might block -on epoll_wait even though only one action was requested. ---- - src/backend/epoll.zig | 15 ++++++++++++++- - 1 file changed, 14 insertions(+), 1 deletion(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index 0f4a2ac..fb7e59e 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -297,6 +297,7 @@ pub const Loop = struct { - - // Submit all the submissions. We copy the submission queue so that - // any resubmits don't cause an infinite loop. -+ var wait_rem: usize = @intCast(wait); - var queued = self.submissions; - self.submissions = .{}; - while (queued.pop()) |c| { -@@ -304,6 +305,19 @@ pub const Loop = struct { - // This usually means that we switched them to be deleted or - // something. - if (c.flags.state != .adding) continue; -+ -+ // These operations happen synchronously. Ensure they are -+ // decremented from wait_rem. -+ switch (c.op) { -+ .cancel, -+ // should noop be counted? -+ // .noop, -+ .shutdown, -+ .timer, -+ => wait_rem -|= 1, -+ else => {}, -+ } -+ - self.start(c); - } - -@@ -322,7 +336,6 @@ pub const Loop = struct { - - // Wait and process events. We only do this if we have any active. - var events: [1024]linux.epoll_event = undefined; -- var wait_rem: usize = @intCast(wait); - while (self.active > 0 and (wait == 0 or wait_rem > 0)) { - self.update_now(); - const now_timer: Operation.Timer = .{ .next = self.cached_now }; --- -2.39.5 (Apple Git-154) - diff --git a/third_party/modules/libxev/20250124.0-31eed4e/patches/141.patch b/third_party/modules/libxev/20250124.0-31eed4e/patches/141.patch deleted file mode 100644 index 70cee31c..00000000 --- a/third_party/modules/libxev/20250124.0-31eed4e/patches/141.patch +++ /dev/null @@ -1,35 +0,0 @@ -From 283a911561385e0cc9278dfd350bdc5c5132c9c4 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Wed, 5 Feb 2025 11:17:01 +0100 -Subject: [PATCH] backend/kqueue: fix empty udata crash - -On certain occasions, internal kqueue events are dispatched with an empty udata field. -This causes a crash when trying to access the udata field in the event handler. - -This patch adds a check to ensure that the udata field is not empty before accessing it -just like in the other events consumer. ---- - src/backend/kqueue.zig | 6 +++++- - 1 file changed, 5 insertions(+), 1 deletion(-) - -diff --git a/src/backend/kqueue.zig b/src/backend/kqueue.zig -index 456f64f..7d919ce 100644 ---- a/src/backend/kqueue.zig -+++ b/src/backend/kqueue.zig -@@ -206,11 +206,15 @@ pub const Loop = struct { - // event list to zero length) because it was leading to - // memory corruption we need to investigate. - for (events[0..completed]) |ev| { -- const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.udata))); -+ // Zero udata values are internal events that we do nothing -+ // on such as the mach port wakeup. -+ if (ev.udata == 0) continue; - - // We handle deletions separately. - if (ev.flags & posix.system.EV_DELETE != 0) continue; - -+ const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.udata))); -+ - // If EV_ERROR is set, then submission failed for this - // completion. We get the syscall errorcode from data and - // store it. diff --git a/third_party/modules/libxev/20250124.0-31eed4e/source.json b/third_party/modules/libxev/20250124.0-31eed4e/source.json deleted file mode 100644 index 77af3d8f..00000000 --- a/third_party/modules/libxev/20250124.0-31eed4e/source.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "strip_prefix": "libxev-31eed4e337fed7b0149319e5cdbb62b848c24fbd", - "url": "https://github.com/zml/libxev/archive/31eed4e337fed7b0149319e5cdbb62b848c24fbd.tar.gz", - "integrity": "sha256-VHP90NTytIZ8UZsYRKOOxN490/I6yv6ec40sP8y5MJ8=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "", - "main2.zig": "" - }, - "patches": { - "128.patch": "", - "141.patch": "" - }, - "patch_strip": 1 -} diff --git a/third_party/modules/libxev/20250222.0-07bcffa/MODULE.bazel b/third_party/modules/libxev/20250222.0-07bcffa/MODULE.bazel deleted file mode 100644 index 424792ed..00000000 --- a/third_party/modules/libxev/20250222.0-07bcffa/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20250222.0-07bcffa", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20250222.0-07bcffa/overlay/BUILD.bazel b/third_party/modules/libxev/20250222.0-07bcffa/overlay/BUILD.bazel deleted file mode 100644 index 872ef7fb..00000000 --- a/third_party/modules/libxev/20250222.0-07bcffa/overlay/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "xev", - srcs = glob([ - "src/*.zig", - "src/backend/*.zig", - "src/linux/*.zig", - "src/watcher/*.zig", - ]), - main = "src/main.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/libxev/20250222.0-07bcffa/overlay/MODULE.bazel b/third_party/modules/libxev/20250222.0-07bcffa/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/libxev/20250222.0-07bcffa/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20250222.0-07bcffa/source.json b/third_party/modules/libxev/20250222.0-07bcffa/source.json deleted file mode 100644 index 1483ad30..00000000 --- a/third_party/modules/libxev/20250222.0-07bcffa/source.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "strip_prefix": "libxev-07bcffa0f63054152a9883baa42bce5faad297e6", - "url": "https://github.com/mitchellh/libxev/archive/07bcffa0f63054152a9883baa42bce5faad297e6.tar.gz", - "integrity": "sha256-H80IoKcTeJFwpL5i4S9zyMuyWaQCK173ppoAO/CYXjc=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - } -} diff --git a/third_party/modules/libxev/20250313.0-5773f46/MODULE.bazel b/third_party/modules/libxev/20250313.0-5773f46/MODULE.bazel deleted file mode 100644 index bfe80aeb..00000000 --- a/third_party/modules/libxev/20250313.0-5773f46/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20250313.0-5773f46", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20250313.0-5773f46/overlay/BUILD.bazel b/third_party/modules/libxev/20250313.0-5773f46/overlay/BUILD.bazel deleted file mode 100644 index 872ef7fb..00000000 --- a/third_party/modules/libxev/20250313.0-5773f46/overlay/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "xev", - srcs = glob([ - "src/*.zig", - "src/backend/*.zig", - "src/linux/*.zig", - "src/watcher/*.zig", - ]), - main = "src/main.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/libxev/20250313.0-5773f46/overlay/MODULE.bazel b/third_party/modules/libxev/20250313.0-5773f46/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/libxev/20250313.0-5773f46/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20250313.0-5773f46/source.json b/third_party/modules/libxev/20250313.0-5773f46/source.json deleted file mode 100644 index 65786582..00000000 --- a/third_party/modules/libxev/20250313.0-5773f46/source.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "strip_prefix": "libxev-5773f46de3937e848db3ff87577add8ef7e8222d", - "url": "https://github.com/mitchellh/libxev/archive/5773f46de3937e848db3ff87577add8ef7e8222d.tar.gz", - "integrity": "sha256-k9cyG0DPhEKwM+/dKiLFT9WeM+c2kCokN8VftOriSxI=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - }, - "patch_strip": 1 -} diff --git a/third_party/modules/libxev/20250718.0-9f785d2/MODULE.bazel b/third_party/modules/libxev/20250718.0-9f785d2/MODULE.bazel deleted file mode 120000 index c8a52909..00000000 --- a/third_party/modules/libxev/20250718.0-9f785d2/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -./overlay/MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20250718.0-9f785d2/overlay/BUILD.bazel b/third_party/modules/libxev/20250718.0-9f785d2/overlay/BUILD.bazel deleted file mode 100644 index 872ef7fb..00000000 --- a/third_party/modules/libxev/20250718.0-9f785d2/overlay/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "xev", - srcs = glob([ - "src/*.zig", - "src/backend/*.zig", - "src/linux/*.zig", - "src/watcher/*.zig", - ]), - main = "src/main.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/libxev/20250718.0-9f785d2/overlay/MODULE.bazel b/third_party/modules/libxev/20250718.0-9f785d2/overlay/MODULE.bazel deleted file mode 100644 index 82907dd7..00000000 --- a/third_party/modules/libxev/20250718.0-9f785d2/overlay/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20250718.0-9f785d2", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20250613.0-567662a") diff --git a/third_party/modules/libxev/20250718.0-9f785d2/source.json b/third_party/modules/libxev/20250718.0-9f785d2/source.json deleted file mode 100644 index daf74448..00000000 --- a/third_party/modules/libxev/20250718.0-9f785d2/source.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "strip_prefix": "libxev-9f785d202ddccf7a625e799250579253977978b6", - "url": "https://github.com/mitchellh/libxev/archive/9f785d202ddccf7a625e799250579253977978b6.tar.gz", - "integrity": "sha256-HaExxsAsATjH/pCP9NMo/EjVrG2KVX3SiTtZzmS8R90=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - }, - "patch_strip": 1 -} - diff --git a/third_party/modules/libxev/20251010.0-9f785d2/MODULE.bazel b/third_party/modules/libxev/20251010.0-9f785d2/MODULE.bazel deleted file mode 120000 index c8a52909..00000000 --- a/third_party/modules/libxev/20251010.0-9f785d2/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -./overlay/MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20251010.0-9f785d2/overlay/BUILD.bazel b/third_party/modules/libxev/20251010.0-9f785d2/overlay/BUILD.bazel deleted file mode 100644 index 872ef7fb..00000000 --- a/third_party/modules/libxev/20251010.0-9f785d2/overlay/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "xev", - srcs = glob([ - "src/*.zig", - "src/backend/*.zig", - "src/linux/*.zig", - "src/watcher/*.zig", - ]), - main = "src/main.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/libxev/20251010.0-9f785d2/overlay/MODULE.bazel b/third_party/modules/libxev/20251010.0-9f785d2/overlay/MODULE.bazel deleted file mode 100644 index 66f5a554..00000000 --- a/third_party/modules/libxev/20251010.0-9f785d2/overlay/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20251010.0-9f785d2", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "0.12.1") diff --git a/third_party/modules/libxev/20251010.0-9f785d2/source.json b/third_party/modules/libxev/20251010.0-9f785d2/source.json deleted file mode 100644 index daf74448..00000000 --- a/third_party/modules/libxev/20251010.0-9f785d2/source.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "strip_prefix": "libxev-9f785d202ddccf7a625e799250579253977978b6", - "url": "https://github.com/mitchellh/libxev/archive/9f785d202ddccf7a625e799250579253977978b6.tar.gz", - "integrity": "sha256-HaExxsAsATjH/pCP9NMo/EjVrG2KVX3SiTtZzmS8R90=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - }, - "patch_strip": 1 -} - diff --git a/third_party/modules/libxev/metadata.json b/third_party/modules/libxev/metadata.json deleted file mode 100644 index 1fd34d18..00000000 --- a/third_party/modules/libxev/metadata.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "homepage": "https://github.com/mitchellh/libxev", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": [ - "github:mitchellh/libxev" - ], - "versions": [ - "20240825.0-dbe2291", - "20240910.0-a2d9b31", - "20241119.0-6afcde9", - "20241208.0-db6a52b", - "20241208.1-db6a52b", - "20241208.2-db6a52b", - "20250124.0-31eed4e", - "20250222.0-07bcffa", - "20250313.0-5773f46", - "20250716.0-9f785d2", - "20251010.0-9f785d2" - ], - "yanked_versions": {} -} diff --git a/third_party/modules/llvm-raw/20240823.0-f142f8a/MODULE.bazel b/third_party/modules/llvm-raw/20240823.0-f142f8a/MODULE.bazel deleted file mode 100644 index 6bf421a8..00000000 --- a/third_party/modules/llvm-raw/20240823.0-f142f8a/MODULE.bazel +++ /dev/null @@ -1,10 +0,0 @@ -module( - name = "llvm-raw", - version = "20240823.0-f142f8a", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") -bazel_dep(name = "zstd", version = "1.5.6", repo_name = "llvm_zstd") -bazel_dep(name = "zlib", version = "1.3.1.bcr.3", repo_name = "llvm_zlib") diff --git a/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/BUILD.bazel b/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/BUILD.bazel deleted file mode 100644 index e69de29b..00000000 diff --git a/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/MODULE.bazel b/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/utils/bazel/extension.bzl b/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/utils/bazel/extension.bzl deleted file mode 100644 index f247ed4b..00000000 --- a/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/utils/bazel/extension.bzl +++ /dev/null @@ -1,28 +0,0 @@ -load("//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") - -def _llvm_impl(mctx): - _targets = {} - for mod in mctx.modules: - for conf in mod.tags.configure: - for target in conf.targets: - _targets[target] = True - _llvm_configure( - name = "llvm-project", - targets = _targets.keys(), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -llvm = module_extension( - implementation = _llvm_impl, - tag_classes = { - "configure": tag_class( - attrs = { - "targets": attr.string_list(mandatory = True), - }, - ), - }, -) diff --git a/third_party/modules/llvm-raw/20240823.0-f142f8a/source.json b/third_party/modules/llvm-raw/20240823.0-f142f8a/source.json deleted file mode 100644 index 6edc73dc..00000000 --- a/third_party/modules/llvm-raw/20240823.0-f142f8a/source.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "strip_prefix": "llvm-project-f142f8afe21bceb00fb495468aa0b5043e98c419", - "url": "https://github.com/llvm/llvm-project/archive/f142f8afe21bceb00fb495468aa0b5043e98c419.tar.gz", - "integrity": "sha256-8ftYOnMGq99igt2I1n9SOZpYLq79+F7HiKL/7lOwcHs=", - "overlay": { - "BUILD.bazel": "", - "MODULE.bazel": "", - "utils/bazel/extension.bzl": "" - } -} diff --git a/third_party/modules/llvm-raw/20240919.0-94c024a/MODULE.bazel b/third_party/modules/llvm-raw/20240919.0-94c024a/MODULE.bazel deleted file mode 100644 index c9130b80..00000000 --- a/third_party/modules/llvm-raw/20240919.0-94c024a/MODULE.bazel +++ /dev/null @@ -1,10 +0,0 @@ -module( - name = "llvm-raw", - version = "20240919.0-94c024a", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") -bazel_dep(name = "zstd", version = "1.5.6", repo_name = "llvm_zstd") -bazel_dep(name = "zlib", version = "1.3.1.bcr.3", repo_name = "llvm_zlib") diff --git a/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/BUILD.bazel b/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/BUILD.bazel deleted file mode 100644 index e69de29b..00000000 diff --git a/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/MODULE.bazel b/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/utils/bazel/extension.bzl b/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/utils/bazel/extension.bzl deleted file mode 100644 index f247ed4b..00000000 --- a/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/utils/bazel/extension.bzl +++ /dev/null @@ -1,28 +0,0 @@ -load("//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") - -def _llvm_impl(mctx): - _targets = {} - for mod in mctx.modules: - for conf in mod.tags.configure: - for target in conf.targets: - _targets[target] = True - _llvm_configure( - name = "llvm-project", - targets = _targets.keys(), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -llvm = module_extension( - implementation = _llvm_impl, - tag_classes = { - "configure": tag_class( - attrs = { - "targets": attr.string_list(mandatory = True), - }, - ), - }, -) diff --git a/third_party/modules/llvm-raw/20240919.0-94c024a/source.json b/third_party/modules/llvm-raw/20240919.0-94c024a/source.json deleted file mode 100644 index 2aec201a..00000000 --- a/third_party/modules/llvm-raw/20240919.0-94c024a/source.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "strip_prefix": "llvm-project-94c024adedcb53059c29d7c2d62982053b60e86a", - "url": "https://github.com/llvm/llvm-project/archive/94c024adedcb53059c29d7c2d62982053b60e86a.tar.gz", - "integrity": "sha256-IEzt6quG8GXvZMs4id0ukt3UqPXVtrwctLJ2aU+2p5g=", - "overlay": { - "BUILD.bazel": "", - "MODULE.bazel": "", - "utils/bazel/extension.bzl": "" - } -} diff --git a/third_party/modules/llvm-raw/20241022.0-6c4267f/MODULE.bazel b/third_party/modules/llvm-raw/20241022.0-6c4267f/MODULE.bazel deleted file mode 100644 index b3ed250a..00000000 --- a/third_party/modules/llvm-raw/20241022.0-6c4267f/MODULE.bazel +++ /dev/null @@ -1,11 +0,0 @@ -module( - name = "llvm-raw", - version = "20241022.0-6c4267f", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") -bazel_dep(name = "zstd", version = "1.5.6", repo_name = "llvm_zstd") -bazel_dep(name = "zlib", version = "1.3.1.bcr.3", repo_name = "llvm_zlib") -bazel_dep(name = "rules_python", version = "0.29.0") diff --git a/third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/BUILD.bazel b/third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/BUILD.bazel deleted file mode 100644 index e69de29b..00000000 diff --git a/third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/MODULE.bazel b/third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/utils/bazel/extension.bzl b/third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/utils/bazel/extension.bzl deleted file mode 100644 index f247ed4b..00000000 --- a/third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/utils/bazel/extension.bzl +++ /dev/null @@ -1,28 +0,0 @@ -load("//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") - -def _llvm_impl(mctx): - _targets = {} - for mod in mctx.modules: - for conf in mod.tags.configure: - for target in conf.targets: - _targets[target] = True - _llvm_configure( - name = "llvm-project", - targets = _targets.keys(), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -llvm = module_extension( - implementation = _llvm_impl, - tag_classes = { - "configure": tag_class( - attrs = { - "targets": attr.string_list(mandatory = True), - }, - ), - }, -) diff --git a/third_party/modules/llvm-raw/20241022.0-6c4267f/source.json b/third_party/modules/llvm-raw/20241022.0-6c4267f/source.json deleted file mode 100644 index f2286dc8..00000000 --- a/third_party/modules/llvm-raw/20241022.0-6c4267f/source.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "strip_prefix": "llvm-project-6c4267fb1779bc5550bb413f33250f9365acfbc6", - "url": "https://github.com/llvm/llvm-project/archive/6c4267fb1779bc5550bb413f33250f9365acfbc6.tar.gz", - "integrity": "sha256-cBDuj+hiRvq8rtvtIfqawr0lQuDSrWFypEgeApT981Q=", - "overlay": { - "BUILD.bazel": "", - "MODULE.bazel": "", - "utils/bazel/extension.bzl": "" - } -} diff --git a/third_party/modules/llvm-raw/20250102.0-f739aa4/MODULE.bazel b/third_party/modules/llvm-raw/20250102.0-f739aa4/MODULE.bazel deleted file mode 100644 index 102b0a38..00000000 --- a/third_party/modules/llvm-raw/20250102.0-f739aa4/MODULE.bazel +++ /dev/null @@ -1,11 +0,0 @@ -module( - name = "llvm-raw", - version = "20250102.0-f739aa4", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") -bazel_dep(name = "zstd", version = "1.5.6", repo_name = "llvm_zstd") -bazel_dep(name = "zlib", version = "1.3.1.bcr.3", repo_name = "llvm_zlib") -bazel_dep(name = "rules_python", version = "0.29.0") diff --git a/third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/BUILD.bazel b/third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/BUILD.bazel deleted file mode 100644 index e69de29b..00000000 diff --git a/third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/MODULE.bazel b/third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/utils/bazel/extension.bzl b/third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/utils/bazel/extension.bzl deleted file mode 100644 index f247ed4b..00000000 --- a/third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/utils/bazel/extension.bzl +++ /dev/null @@ -1,28 +0,0 @@ -load("//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") - -def _llvm_impl(mctx): - _targets = {} - for mod in mctx.modules: - for conf in mod.tags.configure: - for target in conf.targets: - _targets[target] = True - _llvm_configure( - name = "llvm-project", - targets = _targets.keys(), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -llvm = module_extension( - implementation = _llvm_impl, - tag_classes = { - "configure": tag_class( - attrs = { - "targets": attr.string_list(mandatory = True), - }, - ), - }, -) diff --git a/third_party/modules/llvm-raw/20250102.0-f739aa4/source.json b/third_party/modules/llvm-raw/20250102.0-f739aa4/source.json deleted file mode 100644 index 146f68ab..00000000 --- a/third_party/modules/llvm-raw/20250102.0-f739aa4/source.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "strip_prefix": "llvm-project-f739aa4004165dc64d3a1f418d5ad3c84886f01a", - "url": "https://github.com/llvm/llvm-project/archive/f739aa4004165dc64d3a1f418d5ad3c84886f01a.tar.gz", - "integrity": "sha256-hdoTTnugRO9Q68AJ0aV6h/sOLbIIoEZQ7y/lZOlWSqc=", - "overlay": { - "BUILD.bazel": "", - "MODULE.bazel": "", - "utils/bazel/extension.bzl": "" - } -} diff --git a/third_party/modules/llvm-raw/20250117.0-bf17016/MODULE.bazel b/third_party/modules/llvm-raw/20250117.0-bf17016/MODULE.bazel deleted file mode 100644 index b1171a0a..00000000 --- a/third_party/modules/llvm-raw/20250117.0-bf17016/MODULE.bazel +++ /dev/null @@ -1,11 +0,0 @@ -module( - name = "llvm-raw", - version = "20250117.0-bf17016", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") -bazel_dep(name = "zstd", version = "1.5.6", repo_name = "llvm_zstd") -bazel_dep(name = "zlib", version = "1.3.1.bcr.3", repo_name = "llvm_zlib") -bazel_dep(name = "rules_python", version = "0.29.0") diff --git a/third_party/modules/llvm-raw/20250117.0-bf17016/overlay/BUILD.bazel b/third_party/modules/llvm-raw/20250117.0-bf17016/overlay/BUILD.bazel deleted file mode 100644 index e69de29b..00000000 diff --git a/third_party/modules/llvm-raw/20250117.0-bf17016/overlay/MODULE.bazel b/third_party/modules/llvm-raw/20250117.0-bf17016/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/llvm-raw/20250117.0-bf17016/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/llvm-raw/20250117.0-bf17016/overlay/utils/bazel/extension.bzl b/third_party/modules/llvm-raw/20250117.0-bf17016/overlay/utils/bazel/extension.bzl deleted file mode 100644 index f247ed4b..00000000 --- a/third_party/modules/llvm-raw/20250117.0-bf17016/overlay/utils/bazel/extension.bzl +++ /dev/null @@ -1,28 +0,0 @@ -load("//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") - -def _llvm_impl(mctx): - _targets = {} - for mod in mctx.modules: - for conf in mod.tags.configure: - for target in conf.targets: - _targets[target] = True - _llvm_configure( - name = "llvm-project", - targets = _targets.keys(), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -llvm = module_extension( - implementation = _llvm_impl, - tag_classes = { - "configure": tag_class( - attrs = { - "targets": attr.string_list(mandatory = True), - }, - ), - }, -) diff --git a/third_party/modules/llvm-raw/20250117.0-bf17016/source.json b/third_party/modules/llvm-raw/20250117.0-bf17016/source.json deleted file mode 100644 index 851e85aa..00000000 --- a/third_party/modules/llvm-raw/20250117.0-bf17016/source.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "strip_prefix": "llvm-project-bf17016a92bc8a23d2cdd2b51355dd4eb5019c68", - "url": "https://github.com/llvm/llvm-project/archive/bf17016a92bc8a23d2cdd2b51355dd4eb5019c68.tar.gz", - "integrity": "sha256-ugnxLlAZ9aylMbFzMnXwoQsYHW+JTesaRhDgF/drFyo=", - "overlay": { - "BUILD.bazel": "", - "MODULE.bazel": "", - "utils/bazel/extension.bzl": "" - } -} diff --git a/third_party/modules/llvm-raw/20250201.0-386af4a/MODULE.bazel b/third_party/modules/llvm-raw/20250201.0-386af4a/MODULE.bazel deleted file mode 100644 index c213afe0..00000000 --- a/third_party/modules/llvm-raw/20250201.0-386af4a/MODULE.bazel +++ /dev/null @@ -1,11 +0,0 @@ -module( - name = "llvm-raw", - version = "20250201.0-386af4a", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.11") -bazel_dep(name = "zstd", version = "1.5.6", repo_name = "llvm_zstd") -bazel_dep(name = "zlib", version = "1.3.1.bcr.3", repo_name = "llvm_zlib") -bazel_dep(name = "rules_python", version = "0.29.0") diff --git a/third_party/modules/llvm-raw/20250201.0-386af4a/overlay/BUILD.bazel b/third_party/modules/llvm-raw/20250201.0-386af4a/overlay/BUILD.bazel deleted file mode 100644 index e69de29b..00000000 diff --git a/third_party/modules/llvm-raw/20250201.0-386af4a/overlay/MODULE.bazel b/third_party/modules/llvm-raw/20250201.0-386af4a/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/llvm-raw/20250201.0-386af4a/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/llvm-raw/20250201.0-386af4a/overlay/utils/bazel/extension.bzl b/third_party/modules/llvm-raw/20250201.0-386af4a/overlay/utils/bazel/extension.bzl deleted file mode 100644 index f247ed4b..00000000 --- a/third_party/modules/llvm-raw/20250201.0-386af4a/overlay/utils/bazel/extension.bzl +++ /dev/null @@ -1,28 +0,0 @@ -load("//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") - -def _llvm_impl(mctx): - _targets = {} - for mod in mctx.modules: - for conf in mod.tags.configure: - for target in conf.targets: - _targets[target] = True - _llvm_configure( - name = "llvm-project", - targets = _targets.keys(), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -llvm = module_extension( - implementation = _llvm_impl, - tag_classes = { - "configure": tag_class( - attrs = { - "targets": attr.string_list(mandatory = True), - }, - ), - }, -) diff --git a/third_party/modules/llvm-raw/20250201.0-386af4a/source.json b/third_party/modules/llvm-raw/20250201.0-386af4a/source.json deleted file mode 100644 index e3a7a1ac..00000000 --- a/third_party/modules/llvm-raw/20250201.0-386af4a/source.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "strip_prefix": "llvm-project-386af4a5c64ab75eaee2448dc38f2e34a40bfed0", - "url": "https://github.com/llvm/llvm-project/archive/386af4a5c64ab75eaee2448dc38f2e34a40bfed0.tar.gz", - "integrity": "sha256-S3UqLfteTlXPmBKjU7zn5DfghS9O72SSn0jXRzIIDFA=", - "overlay": { - "BUILD.bazel": "", - "MODULE.bazel": "", - "utils/bazel/extension.bzl": "" - } -} diff --git a/third_party/modules/llvm-raw/20250217.0-0e779ad/MODULE.bazel b/third_party/modules/llvm-raw/20250217.0-0e779ad/MODULE.bazel deleted file mode 100644 index 245f034b..00000000 --- a/third_party/modules/llvm-raw/20250217.0-0e779ad/MODULE.bazel +++ /dev/null @@ -1,11 +0,0 @@ -module( - name = "llvm-raw", - version = "20250217.0-0e779ad", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.11") -bazel_dep(name = "zstd", version = "1.5.6", repo_name = "llvm_zstd") -bazel_dep(name = "zlib", version = "1.3.1.bcr.3", repo_name = "llvm_zlib") -bazel_dep(name = "rules_python", version = "0.29.0") diff --git a/third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/BUILD.bazel b/third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/BUILD.bazel deleted file mode 100644 index e69de29b..00000000 diff --git a/third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/MODULE.bazel b/third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/utils/bazel/extension.bzl b/third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/utils/bazel/extension.bzl deleted file mode 100644 index f247ed4b..00000000 --- a/third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/utils/bazel/extension.bzl +++ /dev/null @@ -1,28 +0,0 @@ -load("//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") - -def _llvm_impl(mctx): - _targets = {} - for mod in mctx.modules: - for conf in mod.tags.configure: - for target in conf.targets: - _targets[target] = True - _llvm_configure( - name = "llvm-project", - targets = _targets.keys(), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -llvm = module_extension( - implementation = _llvm_impl, - tag_classes = { - "configure": tag_class( - attrs = { - "targets": attr.string_list(mandatory = True), - }, - ), - }, -) diff --git a/third_party/modules/llvm-raw/20250217.0-0e779ad/source.json b/third_party/modules/llvm-raw/20250217.0-0e779ad/source.json deleted file mode 100644 index 6add20e8..00000000 --- a/third_party/modules/llvm-raw/20250217.0-0e779ad/source.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "strip_prefix": "llvm-project-0e779ad4998ef65907502101c5b82ede05ddfa4e", - "url": "https://github.com/llvm/llvm-project/archive/0e779ad4998ef65907502101c5b82ede05ddfa4e.tar.gz", - "integrity": "sha256-1cJWCy2c487XlRET8rXR6kKGZWePTcsfuHgOsSGcphU=", - "overlay": { - "BUILD.bazel": "", - "MODULE.bazel": "", - "utils/bazel/extension.bzl": "" - } -} diff --git a/third_party/modules/llvm-raw/metadata.json b/third_party/modules/llvm-raw/metadata.json deleted file mode 100644 index 32713e58..00000000 --- a/third_party/modules/llvm-raw/metadata.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "homepage": "https://github.com/llvm/llvm-project", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": [ - "github:llvm/llvm-project" - ], - "versions": [ - "20240823.0-f142f8a", - "20240919.0-94c024a", - "20241022.0-6c4267f", - "20250102.0-f739aa4", - "20250117.0-bf17016", - "20250201.0-386af4a", - ], - "yanked_versions": {} -} diff --git a/third_party/modules/rules_zig/20240904.0-010da15/MODULE.bazel b/third_party/modules/rules_zig/20240904.0-010da15/MODULE.bazel deleted file mode 100644 index 723c6a98..00000000 --- a/third_party/modules/rules_zig/20240904.0-010da15/MODULE.bazel +++ /dev/null @@ -1,68 +0,0 @@ -module( - name = "rules_zig", - version = "20240904.0-010da15", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9", dev_dependency = True) -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20240904.0-010da15/source.json b/third_party/modules/rules_zig/20240904.0-010da15/source.json deleted file mode 100644 index 253af196..00000000 --- a/third_party/modules/rules_zig/20240904.0-010da15/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "type": "git_repository", - "remote": "https://github.com/zml/rules_zig.git", - "commit": "010da15abb4335479778d6b4fb2ca752a0ab80e3" -} diff --git a/third_party/modules/rules_zig/20240909.0-37f17ff/MODULE.bazel b/third_party/modules/rules_zig/20240909.0-37f17ff/MODULE.bazel deleted file mode 100644 index 4ded2bb6..00000000 --- a/third_party/modules/rules_zig/20240909.0-37f17ff/MODULE.bazel +++ /dev/null @@ -1,68 +0,0 @@ -module( - name = "rules_zig", - version = "20240909.0-37f17ff", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9", dev_dependency = True) -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20240909.0-37f17ff/source.json b/third_party/modules/rules_zig/20240909.0-37f17ff/source.json deleted file mode 100644 index 19d05aef..00000000 --- a/third_party/modules/rules_zig/20240909.0-37f17ff/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "type": "git_repository", - "remote": "https://github.com/zml/rules_zig.git", - "commit": "37f17ffd2f6d04fc4bf9b583f9a1b117856a88d1" -} diff --git a/third_party/modules/rules_zig/20240912.0-41bfe84/MODULE.bazel b/third_party/modules/rules_zig/20240912.0-41bfe84/MODULE.bazel deleted file mode 100644 index e288be56..00000000 --- a/third_party/modules/rules_zig/20240912.0-41bfe84/MODULE.bazel +++ /dev/null @@ -1,68 +0,0 @@ -module( - name = "rules_zig", - version = "20240912.0-41bfe84", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20240912.0-41bfe84/source.json b/third_party/modules/rules_zig/20240912.0-41bfe84/source.json deleted file mode 100644 index 3d576fb1..00000000 --- a/third_party/modules/rules_zig/20240912.0-41bfe84/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "type": "git_repository", - "remote": "https://github.com/zml/rules_zig.git", - "commit": "41bfe84e4d9a43cbe55281dddc80b683cc6fc6eb" -} diff --git a/third_party/modules/rules_zig/20240913.0-1957d05/MODULE.bazel b/third_party/modules/rules_zig/20240913.0-1957d05/MODULE.bazel deleted file mode 100644 index f2ff768e..00000000 --- a/third_party/modules/rules_zig/20240913.0-1957d05/MODULE.bazel +++ /dev/null @@ -1,68 +0,0 @@ -module( - name = "rules_zig", - version = "20240913.0-1957d05", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20240913.0-1957d05/source.json b/third_party/modules/rules_zig/20240913.0-1957d05/source.json deleted file mode 100644 index 9da3ae38..00000000 --- a/third_party/modules/rules_zig/20240913.0-1957d05/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "type": "git_repository", - "remote": "https://github.com/zml/rules_zig.git", - "commit": "1957d0572193fb859e721a0fab8bd8f0fb57f3ff" -} diff --git a/third_party/modules/rules_zig/20250314.0-b9739c6/MODULE.bazel b/third_party/modules/rules_zig/20250314.0-b9739c6/MODULE.bazel deleted file mode 100644 index 392e8aff..00000000 --- a/third_party/modules/rules_zig/20250314.0-b9739c6/MODULE.bazel +++ /dev/null @@ -1,68 +0,0 @@ -module( - name = "rules_zig", - version = "20250314.0-b9739c6", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20250314.0-b9739c6/source.json b/third_party/modules/rules_zig/20250314.0-b9739c6/source.json deleted file mode 100644 index ceea72c3..00000000 --- a/third_party/modules/rules_zig/20250314.0-b9739c6/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "strip_prefix": "rules_zig-b9739c615ce62b64ee595ac4bcd9ee7cc06b0422", - "url": "https://github.com/zml/rules_zig/archive/b9739c615ce62b64ee595ac4bcd9ee7cc06b0422.tar.gz", - "integrity": "sha256-p1rYD9gvYS3DgBjN6jo1pB86Kr4k2wojKjVpYRTZ3Us=" -} diff --git a/third_party/modules/rules_zig/20250519.0-233b207/MODULE.bazel b/third_party/modules/rules_zig/20250519.0-233b207/MODULE.bazel deleted file mode 100644 index b5dacc58..00000000 --- a/third_party/modules/rules_zig/20250519.0-233b207/MODULE.bazel +++ /dev/null @@ -1,68 +0,0 @@ -module( - name = "rules_zig", - version = "20250519.0-233b207", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20250519.0-233b207/source.json b/third_party/modules/rules_zig/20250519.0-233b207/source.json deleted file mode 100644 index 6082ccd8..00000000 --- a/third_party/modules/rules_zig/20250519.0-233b207/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "strip_prefix": "rules_zig-233b207499826715a3bf1a6eeea6e3052ed49542", - "url": "https://github.com/zml/rules_zig/archive/233b207499826715a3bf1a6eeea6e3052ed49542.tar.gz", - "integrity": "sha256-4gloFoSMPkWCV0Jxikqs1UkfOgQL+NmoXMMnVhtgVvg=" -} diff --git a/third_party/modules/rules_zig/20250530.0-5084f1f/MODULE.bazel b/third_party/modules/rules_zig/20250530.0-5084f1f/MODULE.bazel deleted file mode 100644 index 06b1e94e..00000000 --- a/third_party/modules/rules_zig/20250530.0-5084f1f/MODULE.bazel +++ /dev/null @@ -1,68 +0,0 @@ -module( - name = "rules_zig", - version = "20250530.0-5084f1f", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20250530.0-5084f1f/source.json b/third_party/modules/rules_zig/20250530.0-5084f1f/source.json deleted file mode 100644 index 081142c0..00000000 --- a/third_party/modules/rules_zig/20250530.0-5084f1f/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "strip_prefix": "rules_zig-5084f1f3a151dfc697900274c19813a4260217c6", - "url": "https://github.com/zml/rules_zig/archive/5084f1f3a151dfc697900274c19813a4260217c6.tar.gz", - "integrity": "sha256-KpZkIDnKG0sXtRhsE0SDoj3QeeHRKQCRjm6LzxmueRA=" -} diff --git a/third_party/modules/rules_zig/20250613.0-567662a/MODULE.bazel b/third_party/modules/rules_zig/20250613.0-567662a/MODULE.bazel deleted file mode 100644 index 4189634c..00000000 --- a/third_party/modules/rules_zig/20250613.0-567662a/MODULE.bazel +++ /dev/null @@ -1,68 +0,0 @@ -module( - name = "rules_zig", - version = "20250613.0-567662a", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20250613.0-567662a/source.json b/third_party/modules/rules_zig/20250613.0-567662a/source.json deleted file mode 100644 index 4dbdafee..00000000 --- a/third_party/modules/rules_zig/20250613.0-567662a/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "strip_prefix": "rules_zig-567662a3ce5e87894950d56c69d8de5ad6e0b5f0", - "url": "https://github.com/zml/rules_zig/archive/567662a3ce5e87894950d56c69d8de5ad6e0b5f0.tar.gz", - "integrity": "sha256-PjFpDJXO0BGx2CAZ54Ppv6d4TwTDOWS+mVangKQFvZc=" -} diff --git a/third_party/modules/rules_zig/20250714.0-b14a4f1/MODULE.bazel b/third_party/modules/rules_zig/20250714.0-b14a4f1/MODULE.bazel deleted file mode 100644 index 69a0dc9b..00000000 --- a/third_party/modules/rules_zig/20250714.0-b14a4f1/MODULE.bazel +++ /dev/null @@ -1,68 +0,0 @@ -module( - name = "rules_zig", - version = "20250714.0-b14a4f1", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20250714.0-b14a4f1/source.json b/third_party/modules/rules_zig/20250714.0-b14a4f1/source.json deleted file mode 100644 index 5eb58847..00000000 --- a/third_party/modules/rules_zig/20250714.0-b14a4f1/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "strip_prefix": "rules_zig-b14a4f1bba1a766311f75513e922cc8c2179e1e3", - "url": "https://github.com/zml/rules_zig/archive/b14a4f1bba1a766311f75513e922cc8c2179e1e3.tar.gz", - "integrity": "sha256-XEdYlvjs54SVoLGM4x9J+uofCZk2UIO1nbsqwV+Blqs=" -} diff --git a/third_party/modules/rules_zig/20250821.0-be53625/MODULE.bazel b/third_party/modules/rules_zig/20250821.0-be53625/MODULE.bazel deleted file mode 100644 index 3c64eb32..00000000 --- a/third_party/modules/rules_zig/20250821.0-be53625/MODULE.bazel +++ /dev/null @@ -1,75 +0,0 @@ -module( - name = "rules_zig", - version = "20250821.0-be53625", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9") - -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) - -# Hack to get around a cc_common.link limitation. -# See https://github.com/bazelbuild/bazel/pull/23838 -cc_common_link = use_repo_rule("//zig:extensions.bzl", "cc_common_link") - -cc_common_link(name = "build_bazel_rules_android") diff --git a/third_party/modules/rules_zig/20250821.0-be53625/source.json b/third_party/modules/rules_zig/20250821.0-be53625/source.json deleted file mode 100644 index 959b2dbc..00000000 --- a/third_party/modules/rules_zig/20250821.0-be53625/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "strip_prefix": "rules_zig-be53625afb13e73856ee4cab38d5aad9f86f63ef", - "url": "https://github.com/zml/rules_zig/archive/be53625afb13e73856ee4cab38d5aad9f86f63ef.tar.gz", - "integrity": "sha256-IsqEl3BHW/vttyHpVBx5WzO3an/K3eZZx8RJk7nVx8s=" -} diff --git a/third_party/modules/rules_zig/20250827.0-35b6d57/MODULE.bazel b/third_party/modules/rules_zig/20250827.0-35b6d57/MODULE.bazel deleted file mode 100644 index 7620c53c..00000000 --- a/third_party/modules/rules_zig/20250827.0-35b6d57/MODULE.bazel +++ /dev/null @@ -1,75 +0,0 @@ -module( - name = "rules_zig", - version = "20250827.0-35b6d57", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9") - -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) - -# Hack to get around a cc_common.link limitation. -# See https://github.com/bazelbuild/bazel/pull/23838 -cc_common_link = use_repo_rule("//zig:extensions.bzl", "cc_common_link") - -cc_common_link(name = "build_bazel_rules_android") diff --git a/third_party/modules/rules_zig/20250827.0-35b6d57/source.json b/third_party/modules/rules_zig/20250827.0-35b6d57/source.json deleted file mode 100644 index e42d62ba..00000000 --- a/third_party/modules/rules_zig/20250827.0-35b6d57/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "strip_prefix": "rules_zig-35b6d57e94f3eb08e978c1133314606f4f38e216", - "url": "https://github.com/zml/rules_zig/archive/35b6d57e94f3eb08e978c1133314606f4f38e216.tar.gz", - "integrity": "sha256-FDnAqynTD2LB3W/IaBgocmnLz8CA9nyHZYYntC4plUU=" -} diff --git a/third_party/modules/rules_zig/metadata.json b/third_party/modules/rules_zig/metadata.json deleted file mode 100644 index e7937f1e..00000000 --- a/third_party/modules/rules_zig/metadata.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "homepage": "https://github.com/zml/rules_zig", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": ["github:zml/rules_zig"], - "versions": [ - "20240904.0-010da15", - "20240909.0-37f17ff", - "20240912.0-41bfe84", - "20240913.0-1957d05", - "20250314.0-b9739c6", - "20250519.0-233b207", - "20250613.0-567662a", - "20250714.0-b14a4f1", - "20250821.0-be53625", - "20250827.0-35b6d57" - ], - "yanked_versions": {} -} diff --git a/third_party/modules/sentencepiece/20240618.0-d7ace0a/MODULE.bazel b/third_party/modules/sentencepiece/20240618.0-d7ace0a/MODULE.bazel deleted file mode 100644 index af94d93f..00000000 --- a/third_party/modules/sentencepiece/20240618.0-d7ace0a/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "sentencepiece", - version = "20240618.0-d7ace0a", - compatibility_level = 1, -) - -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") diff --git a/third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/BUILD.bazel b/third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/BUILD.bazel deleted file mode 100644 index 43639e89..00000000 --- a/third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/BUILD.bazel +++ /dev/null @@ -1,7 +0,0 @@ -load("@rules_proto//proto:defs.bzl", "proto_library") - -proto_library( - name = "sentencepiece_model_proto", - srcs = ["src/sentencepiece_model.proto"], - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/MODULE.bazel b/third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/sentencepiece/20240618.0-d7ace0a/source.json b/third_party/modules/sentencepiece/20240618.0-d7ace0a/source.json deleted file mode 100644 index 588c9bcb..00000000 --- a/third_party/modules/sentencepiece/20240618.0-d7ace0a/source.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "strip_prefix": "sentencepiece-d7ace0a4df17af17d73dc8fe6ff92cfd8d08e2b8", - "url": "https://github.com/google/sentencepiece/archive/d7ace0a4df17af17d73dc8fe6ff92cfd8d08e2b8.tar.gz", - "integrity": "sha256-bt8QFNNIpqcJoCNMUrmWfUf0uNobtk8pW+XbHL+ZRLg=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - } -} diff --git a/third_party/modules/sentencepiece/metadata.json b/third_party/modules/sentencepiece/metadata.json deleted file mode 100644 index 2c9758c9..00000000 --- a/third_party/modules/sentencepiece/metadata.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "homepage": "https://github.com/google/sentencepiece", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": [ - "google/sentencepiece" - ], - "versions": [ - "20240618.0-d7ace0a" - ], - "yanked_versions": {} -} diff --git a/third_party/modules/stablehlo/20240829.0-54aa1a5/MODULE.bazel b/third_party/modules/stablehlo/20240829.0-54aa1a5/MODULE.bazel deleted file mode 100644 index 29e45e04..00000000 --- a/third_party/modules/stablehlo/20240829.0-54aa1a5/MODULE.bazel +++ /dev/null @@ -1,15 +0,0 @@ -module( - name = "stablehlo", - version = "20240829.0-54aa1a5", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "llvm-raw", version = "20240823.0-94c024a") - -llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") -llvm.configure( - targets = ["AArch64", "X86", "NVPTX"], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/stablehlo/20240829.0-54aa1a5/overlay/MODULE.bazel b/third_party/modules/stablehlo/20240829.0-54aa1a5/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/stablehlo/20240829.0-54aa1a5/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/stablehlo/20240829.0-54aa1a5/patches/0001-Remove-duplicated-symbols-in-StablehloApi.h.patch b/third_party/modules/stablehlo/20240829.0-54aa1a5/patches/0001-Remove-duplicated-symbols-in-StablehloApi.h.patch deleted file mode 100644 index 82ca0c4d..00000000 --- a/third_party/modules/stablehlo/20240829.0-54aa1a5/patches/0001-Remove-duplicated-symbols-in-StablehloApi.h.patch +++ /dev/null @@ -1,83 +0,0 @@ -From f11d9bd7639f63cba681caa39cea27af114a4a71 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Mon, 2 Sep 2024 23:02:28 +0200 -Subject: [PATCH 1/2] Remove duplicated symbols in StablehloApi.h - -This causes C compilers to choke. - -Refs #2494 ---- - stablehlo/integrations/c/StablehloApi.cpp | 2 +- - stablehlo/integrations/c/StablehloApi.h | 15 +-------------- - stablehlo/integrations/python/StablehloApi.cpp | 2 +- - 3 files changed, 3 insertions(+), 16 deletions(-) - -diff --git a/stablehlo/integrations/c/StablehloApi.cpp b/stablehlo/integrations/c/StablehloApi.cpp -index 8d922198..9b41ff21 100644 ---- a/stablehlo/integrations/c/StablehloApi.cpp -+++ b/stablehlo/integrations/c/StablehloApi.cpp -@@ -98,7 +98,7 @@ MlirLogicalResult stablehloSerializePortableArtifact( - return mlirLogicalResultSuccess(); - } - --MlirLogicalResult stablehloDeserializePortableArtifact( -+MlirLogicalResult stablehloDeserializePortableArtifactToBytecode( - MlirStringRef artifactStr, MlirStringCallback callback, void *userData) { - mlir::detail::CallbackOstream stream(callback, userData); - if (failed(mlir::stablehlo::deserializePortableArtifact(unwrap(artifactStr), -diff --git a/stablehlo/integrations/c/StablehloApi.h b/stablehlo/integrations/c/StablehloApi.h -index 4c542508..f94dca8d 100644 ---- a/stablehlo/integrations/c/StablehloApi.h -+++ b/stablehlo/integrations/c/StablehloApi.h -@@ -76,16 +76,6 @@ MLIR_CAPI_EXPORTED MlirLogicalResult stablehloSerializePortableArtifact( - MlirStringRef moduleStr, MlirStringRef targetVersion, - MlirStringCallback callback, void* userData); - --// Write a StableHLO program expressed as a string (either prettyprinted MLIR --// module or MLIR bytecode) to a portable artifact. --// Can fail if `moduleStr` cannot be parsed, or if it cannot be expressed in the --// `targetVersion` version of StableHLO, e.g. if it's using new or removed --// features, or if it involves unsupported dialects. --// Returns false on failure. --MLIR_CAPI_EXPORTED MlirLogicalResult stablehloSerializePortableArtifact( -- MlirModule moduleStr, MlirStringRef targetVersion, -- MlirStringCallback callback, void* userData); -- - // Read a StableHLO program from a portable artifact, returning the module as - // MLIR bytecode. Note, this bytecode returned is not a portable artifact, - // and has the stability of returning textual assembly format. Bytecode is -@@ -93,7 +83,7 @@ MLIR_CAPI_EXPORTED MlirLogicalResult stablehloSerializePortableArtifact( - // Can fail if `artifactStr` cannot be expressed in the current version of - // StableHLO, e.g. if it's using incompatible features. - // Returns false on failure. --MLIR_CAPI_EXPORTED MlirLogicalResult stablehloDeserializePortableArtifact( -+MLIR_CAPI_EXPORTED MlirLogicalResult stablehloDeserializePortableArtifactAsBytecode( - MlirStringRef artifactStr, MlirStringCallback callback, void* userData); - - // Read a StableHLO program from a portable artifact, returning the module as -@@ -109,9 +99,6 @@ MLIR_CAPI_EXPORTED MlirModule stablehloDeserializePortableArtifact( - - // Call the Interpreter, returns MlirArrayAttr of dense element - // MlirAttribute results --MLIR_CAPI_EXPORTED MlirModule stablehloDeserializePortableArtifact( -- MlirStringRef artifactStr, MlirContext ctx); -- - // Entrypoint for calling the StableHLO reference interpreter. - // Returns an array attribute of dense element attributes for results. - // Sets error code to non-zero on failure. -diff --git a/stablehlo/integrations/python/StablehloApi.cpp b/stablehlo/integrations/python/StablehloApi.cpp -index 46a640e1..4229ef76 100644 ---- a/stablehlo/integrations/python/StablehloApi.cpp -+++ b/stablehlo/integrations/python/StablehloApi.cpp -@@ -213,7 +213,7 @@ void AddPortableApi(py::module &m) { - "deserialize_portable_artifact_str", - [](std::string_view artifact) -> py::bytes { - StringWriterHelper accumulator; -- if (mlirLogicalResultIsFailure(stablehloDeserializePortableArtifact( -+ if (mlirLogicalResultIsFailure(stablehloDeserializePortableArtifactToBytecode( - toMlirStringRef(artifact), accumulator.getMlirStringCallback(), - accumulator.getUserData()))) { - PyErr_SetString(PyExc_ValueError, "failed to deserialize module"); --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/stablehlo/20240829.0-54aa1a5/patches/temporary.patch b/third_party/modules/stablehlo/20240829.0-54aa1a5/patches/temporary.patch deleted file mode 100644 index 5240a14c..00000000 --- a/third_party/modules/stablehlo/20240829.0-54aa1a5/patches/temporary.patch +++ /dev/null @@ -1,401 +0,0 @@ -diff --ruN a/stablehlo/BUILD.bazel b/stablehlo/BUILD.bazel ---- stablehlo/BUILD.bazel -+++ stablehlo/BUILD.bazel -@@ -340,6 +340,21 @@ - ], - ) - -+gentbl_cc_library( -+ name = "stablehlo_create_compatibility_expander_inc_gen", -+ tbl_outs = [ -+ ( -+ ["--gen-rewriters"], -+ "stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.h.inc", -+ ), -+ ], -+ tblgen = "@llvm-project//mlir:mlir-tblgen", -+ td_file = "stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.td", -+ deps = [ -+ ":stablehlo_ops_td_files", -+ ], -+) -+ - cc_library( - name = "interpreter_ops", - srcs = [ -@@ -1086,6 +1101,7 @@ - "stablehlo/transforms/StablehloAggressiveSimplification.cpp", - "stablehlo/transforms/StablehloCanonicalizeDynamism.cpp", - "stablehlo/transforms/StablehloConvertToSignless.cpp", -+ "stablehlo/transforms/StablehloCreateCompatibilityExpander.cpp", - "stablehlo/transforms/StablehloLegalizeCompositeToCall.cpp", - "stablehlo/transforms/StablehloLegalizeDeprecatedOps.cpp", - "stablehlo/transforms/StablehloLegalizeQDQToQuantizedOp.cpp", -@@ -1109,6 +1125,7 @@ - ":chlo_ops", - ":chlo_rewriters_inc_gen", - ":linalg_passes", -+ ":stablehlo_create_compatibility_expander_inc_gen", - ":stablehlo_legalize_deprecated_ops_inc_gen", - ":stablehlo_ops", - ":stablehlo_ops_inc_gen", -diff --ruN a/stablehlo/stablehlo/dialect/Version.cpp b/stablehlo/stablehlo/dialect/Version.cpp ---- stablehlo/stablehlo/dialect/Version.cpp -+++ stablehlo/stablehlo/dialect/Version.cpp -@@ -82,7 +82,7 @@ - case CompatibilityRequirement::WEEK_4: - return Version(1, 3, 0); // v1.3.0 - Jul 15, 2024 - case CompatibilityRequirement::WEEK_12: -- return Version(1, 0, 0); // v1.0.0 - May 14, 2024 -+ return Version(1, 1, 0); // v1.1.0 - May 30, 2024 - case CompatibilityRequirement::MAX: - return Version::getMinimumVersion(); - } -diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_create_compatibility_expander.mlir b/stablehlo/stablehlo/tests/transforms/stablehlo_create_compatibility_expander.mlir ---- stablehlo/stablehlo/tests/transforms/stablehlo_create_compatibility_expander.mlir -+++ stablehlo/stablehlo/tests/transforms/stablehlo_create_compatibility_expander.mlir -@@ -0,0 +1,43 @@ -+// RUN: stablehlo-opt %s -verify-diagnostics -split-input-file -allow-unregistered-dialect --stablehlo-create-compatibility-expander='target=1.0.0' | FileCheck %s --check-prefixes=CHECK -+// RUN: stablehlo-opt %s -verify-diagnostics -split-input-file --stablehlo-create-compatibility-expander='target=1.6.0' | FileCheck %s --check-prefixes=CHECK-NO-DOWNGRADE -+ -+// ----- -+ -+// CHECK-LABEL @tan_op_non_complex -+// CHECK: %[[sine0:.*]] = stablehlo.sine %arg0 : tensor<4xf64> -+// CHECK-NEXT: %[[cosine1:.*]] = stablehlo.cosine %arg0 : tensor<4xf64> -+// CHECK-NEXT: %[[div2:.*]] = stablehlo.divide %[[sine0]], %[[cosine1]] : tensor<4xf64> -+// CHECK-NEXT: return %[[div2]] : tensor<4xf64> -+func.func @tan_op_non_complex(%arg0: tensor<4xf64>) -> tensor<4xf64> { -+ // CHECK-NO-DOWNGRADE: stablehlo.tan %arg0 : tensor<4xf64> -+ %1 = stablehlo.tan %arg0 : tensor<4xf64> -+ func.return %1 : tensor<4xf64> -+} -+ -+// ----- -+ -+// CHECK-LABEL: @tan_op_complex -+// CHECK: %[[cst:.*]] = stablehlo.constant dense<1.000000e+00> : tensor<4xf64> -+// CHECK: %[[complex:.*]] = stablehlo.complex %arg0, %arg1 : tensor<4xcomplex> -+// CHECK: %[[real:.*]] = stablehlo.real %[[complex]] : (tensor<4xcomplex>) -> tensor<4xf64> -+// CHECK: %[[sine:.*]] = stablehlo.sine %[[real]] : tensor<4xf64> -+// CHECK: %[[cosine:.*]] = stablehlo.cosine %[[real]] : tensor<4xf64> -+// CHECK: %[[divide1:.*]] = stablehlo.divide %[[sine]], %[[cosine]] : tensor<4xf64> -+// CHECK: %[[imag:.*]] = stablehlo.imag %[[complex]] : (tensor<4xcomplex>) -> tensor<4xf64> -+// CHECK: %[[tanh:.*]] = stablehlo.tanh %[[imag]] : tensor<4xf64> -+// CHECK: %[[complex2:.*]] = stablehlo.complex %[[divide1]], %[[tanh]] : tensor<4xcomplex> -+// CHECK: %[[multiply:.*]] = stablehlo.multiply %[[divide1]], %[[tanh]] : tensor<4xf64> -+// CHECK: %[[negate:.*]] = stablehlo.negate %[[multiply]] : tensor<4xf64> -+// CHECK: %[[complex3:.*]] = stablehlo.complex %[[cst]], %[[negate]] : tensor<4xcomplex> -+// CHECK: %[[divide2:.*]] = stablehlo.divide %[[complex2]], %[[complex3]] : tensor<4xcomplex> -+// CHECK: %[[real2:.*]] = stablehlo.real %[[divide2]] : (tensor<4xcomplex>) -> tensor<4xf64> -+// CHECK: %[[imag2:.*]] = stablehlo.imag %[[divide2]] : (tensor<4xcomplex>) -> tensor<4xf64> -+// CHECK: return %[[real2]], %[[imag2]] : tensor<4xf64>, tensor<4xf64> -+func.func @tan_op_complex(%arg0: tensor<4xf64>, %arg1: tensor<4xf64>) -> (tensor<4xf64>, tensor<4xf64>) { -+ %0 = stablehlo.complex %arg0, %arg1 : tensor<4xcomplex> -+ // CHECK-NO-DOWNGRADE: stablehlo.tan %0 : tensor<4xcomplex> -+ %1 = stablehlo.tan %0 : tensor<4xcomplex> -+ %2 = stablehlo.real %1 : (tensor<4xcomplex>) -> tensor<4xf64> -+ %3 = stablehlo.imag %1 : (tensor<4xcomplex>) -> tensor<4xf64> -+ func.return %2, %3 : tensor<4xf64>, tensor<4xf64> -+} -diff --ruN a/stablehlo/stablehlo/transforms/CMakeLists.txt b/stablehlo/stablehlo/transforms/CMakeLists.txt ---- stablehlo/stablehlo/transforms/CMakeLists.txt -+++ stablehlo/stablehlo/transforms/CMakeLists.txt -@@ -20,6 +20,10 @@ - mlir_tablegen(ChloDecompositionPatterns.h.inc --gen-rewriters) - add_public_tablegen_target(ChloDecompositionPatternsIncGen) - -+set(LLVM_TARGET_DEFINITIONS StablehloCreateCompatibilityExpanderPatterns.td) -+mlir_tablegen(StablehloCreateCompatibilityExpanderPatterns.h.inc --gen-rewriters) -+add_public_tablegen_target(StablehloCreateCompatibilityExpanderPatternsIncGen) -+ - set(LLVM_TARGET_DEFINITIONS StablehloLegalizeDeprecatedOpsPatterns.td) - mlir_tablegen(StablehloLegalizeDeprecatedOpsPatterns.h.inc --gen-rewriters) - add_public_tablegen_target(StablehloLegalizeDeprecatedOpsPatternsIncGen) -@@ -27,6 +31,7 @@ - set(LLVM_TARGET_DEFINITIONS VhloToVersionPatterns.td) - mlir_tablegen(VhloToVersionPatterns.h.inc --gen-rewriters) - add_public_tablegen_target(VhloToVersionPatterns) -+ - - add_mlir_dialect_library(StablehloPasses - PARTIAL_SOURCES_INTENDED -@@ -37,6 +42,7 @@ - StablehloAggressiveSimplification.cpp - StablehloCanonicalizeDynamism.cpp - StablehloConvertToSignless.cpp -+ StablehloCreateCompatibilityExpander.cpp - StablehloLegalizeCompositeToCall.cpp - StablehloLegalizeDeprecatedOps.cpp - StablehloLegalizeQuantToMath.cpp -@@ -53,6 +59,7 @@ - StablehloLegalizeDeprecatedOpsPatternsIncGen - PassesIncGen - VhloToVersionPatterns -+ StablehloCreateCompatibilityExpanderPatternsIncGen - - LINK_LIBS PUBLIC - ChloOps -diff --ruN a/stablehlo/stablehlo/transforms/Passes.h b/stablehlo/stablehlo/transforms/Passes.h ---- stablehlo/stablehlo/transforms/Passes.h -+++ stablehlo/stablehlo/transforms/Passes.h -@@ -25,6 +25,7 @@ - #include "mlir/Pass/Pass.h" - #include "mlir/Support/LogicalResult.h" - #include "mlir/Transforms/DialectConversion.h" -+#include "stablehlo/dialect/Version.h" - - namespace mlir { - namespace stablehlo { -@@ -96,6 +97,12 @@ - void populateShapeToStablehloPatterns(MLIRContext *context, - RewritePatternSet *patterns); - -+/// Collection of patterns to create compatibility expander for StableHLO -+/// operations. -+void populateStablehloCreateCompatibilityExpanderPatterns( -+ RewritePatternSet *patterns, MLIRContext *context, -+ vhlo::Version targetVersion); -+ - //// Additional pass constructors //// - - std::unique_ptr> createStablehloRefineArgumentsPass( -diff --ruN a/stablehlo/stablehlo/transforms/Passes.td b/stablehlo/stablehlo/transforms/Passes.td ---- stablehlo/stablehlo/transforms/Passes.td -+++ stablehlo/stablehlo/transforms/Passes.td -@@ -292,3 +292,51 @@ - "mlir::stablehlo::StablehloDialect", - ]; - } -+ -+def StablehloCreateCompatibilityExpanderPass : Pass<"stablehlo-create-compatibility-expander", "mlir::func::FuncOp"> { -+ let summary = "Create compatibility expander for StableHLO operations."; -+ -+ let description = [{ -+ StableHLO ops gets updates or new op is introduced in the latest versions. -+ This opt-in pass expands backward compatibility with older StableHLO -+ versions by decomposing newer StableHLO operations into equivalent -+ operations supported by those older versions. -+ -+ Why is this an opt-in pass? -+ -+ Occasionally, StableHLO op enhancements are used to greatly simplify the -+ handling of certain common patterns in the OpenXLA ecosystem. This -+ includes things like TanOp, which has high framework and compiler support, -+ as well as gather/scatter batching dimensions, which can be represented -+ using slices, but makes sharding much more difficult. For this category of -+ new features, we do not offer automatic downgrade, since it may throw away -+ important information used in subsequent optimizations. This pass can be -+ used to expand these ops based on a target version to maximize compatibility -+ at the expense of potentially less optimal compilation. -+ -+ ```mlir -+ func.func @tan_op_non_complex(%arg0: tensor<4xf64>) -> tensor<4xf64> { -+ %1 = stablehlo.tan %arg0 : tensor<4xf64> -+ func.return %1 : tensor<4xf64> -+ } -+ ``` -+ -+ will become: -+ -+ ```mlir -+ func.func @tan_op_non_complex(%arg0: tensor<4xf64>) -> tensor<4xf64> { -+ %0 = stablehlo.sine %arg0 : tensor<4xf64> -+ %1 = stablehlo.cosine %arg0 : tensor<4xf64> -+ %2 = stablehlo.divide %0, %1 : tensor<4xf64> -+ return %2 : tensor<4xf64> -+ } -+ ``` -+ }]; -+ let options = [ -+ Option<"targetVersionOption", "target", "std::string", "", -+ "The target version. Must be a version of the form #.#.#.">, -+ ]; -+ let dependentDialects = [ -+ "mlir::stablehlo::StablehloDialect", -+ ]; -+} -diff --ruN a/stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpander.cpp b/stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpander.cpp ---- stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpander.cpp -+++ stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpander.cpp -@@ -0,0 +1,128 @@ -+/* Copyright 2024 The StableHLO Authors. All Rights Reserved. -+Licensed under the Apache License, Version 2.0 (the "License"); -+you may not use this file except in compliance with the License. -+You may obtain a copy of the License at -+ http://www.apache.org/licenses/LICENSE-2.0 -+Unless required by applicable law or agreed to in writing, software -+distributed under the License is distributed on an "AS IS" BASIS, -+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+See the License for the specific language governing permissions and -+limitations under the License. -+==============================================================================*/ -+ -+#include -+ -+#include -+ -+#include "llvm/ADT/APFloat.h" -+#include "llvm/Support/ErrorHandling.h" -+#include "mlir/Dialect/Func/IR/FuncOps.h" -+#include "mlir/IR/BuiltinAttributes.h" -+#include "mlir/IR/PatternMatch.h" -+#include "mlir/Support/LLVM.h" -+#include "mlir/Transforms/DialectConversion.h" -+#include "mlir/Transforms/GreedyPatternRewriteDriver.h" -+#include "stablehlo/dialect/StablehloOps.h" -+#include "stablehlo/dialect/Version.h" -+#include "stablehlo/transforms/Passes.h" -+ -+namespace mlir { -+namespace stablehlo { -+#define GEN_PASS_DEF_STABLEHLOCREATECOMPATIBILITYEXPANDERPASS -+#include "stablehlo/transforms/Passes.h.inc" -+ -+namespace { -+ -+//===----------------------------------------------------------------------===// -+// Helpers. -+//===----------------------------------------------------------------------===// -+ -+// Creates a constant with all ones. -+static Value createConstantWithAllOnes(OpBuilder &b, Location loc, Value val) { -+ auto shapedTy = dyn_cast(val.getType()); -+ if (!shapedTy) llvm_unreachable("Unsupported shaped type."); -+ -+ mlir::DenseElementsAttr elementsAttr = -+ mlir::DenseElementsAttr::get(shapedTy, 1.0); -+ -+ return b.create(loc, val.getType(), -+ elementsAttr); -+} -+ -+// Check user-specified target version. -+vhlo::Version validateTargetVersion(llvm::StringRef versionRef) { -+ auto failOrVersion = vhlo::Version::fromString(versionRef); -+ if (failed(failOrVersion)) { -+ assert(!versionRef.empty() && -+ "No target version specified. Target version must be of the form " -+ "`#.#.#`."); -+ assert(versionRef.empty() && -+ "Invalid target version argument. Target version must be of the " -+ "form `#.#.#`."); -+ } -+ vhlo::Version targetVersion = *failOrVersion; -+ assert((vhlo::Version::getMinimumVersion() <= targetVersion) && -+ "target version is less than minimum supported."); -+ assert((targetVersion <= vhlo::Version::getCurrentVersion()) && -+ "target version is greater than current version."); -+ return targetVersion; -+} -+ -+//===----------------------------------------------------------------------===// -+// Pass -+//===----------------------------------------------------------------------===// -+ -+struct StablehloCreateCompatibilityExpanderPass -+ : public impl::StablehloCreateCompatibilityExpanderPassBase< -+ StablehloCreateCompatibilityExpanderPass> { -+ StablehloCreateCompatibilityExpanderPass() -+ : StablehloCreateCompatibilityExpanderPassBase< -+ StablehloCreateCompatibilityExpanderPass>() {} -+ StablehloCreateCompatibilityExpanderPass( -+ const StablehloCreateCompatibilityExpanderPassOptions &opts) -+ : StablehloCreateCompatibilityExpanderPassBase< -+ StablehloCreateCompatibilityExpanderPass>(opts) {} -+ -+ public: -+ LogicalResult initialize(MLIRContext *context) override { -+ auto targetVersion = validateTargetVersion(targetVersionOption); -+ -+ config.useTopDownTraversal = true; -+ RewritePatternSet patterns_(context); -+ populateStablehloCreateCompatibilityExpanderPatterns(&patterns_, context, -+ targetVersion); -+ patterns = std::move(patterns_); -+ return success(); -+ } -+ -+ void runOnOperation() override { -+ auto func = getOperation(); -+ if (failed(applyPatternsAndFoldGreedily(func, patterns, config))) { -+ func.emitError( -+ "Failed to converge StableHLOCreateCompatibilityExpanderPass in ") -+ << config.maxIterations << " iterations"; -+ signalPassFailure(); -+ } -+ } -+ -+ private: -+ FrozenRewritePatternSet patterns; -+ GreedyRewriteConfig config; -+}; -+ -+#include "stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.h.inc" -+ -+} // namespace -+ -+void populateStablehloCreateCompatibilityExpanderPatterns( -+ RewritePatternSet *patterns, MLIRContext *context, -+ vhlo::Version targetVersion) { -+ // StableHLO TanOp is introduced in v1.4.0. -+ if (targetVersion < vhlo::Version(1, 4, 0)) { -+ patterns->add(context); -+ patterns->add(context); -+ } -+} -+ -+} // namespace stablehlo -+} // namespace mlir -diff --ruN a/stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.td b/stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.td ---- stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.td -+++ stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.td -@@ -0,0 +1,47 @@ -+/* Copyright 2022 The StableHLO Authors. -+ -+Licensed under the Apache License, Version 2.0 (the "License"); -+you may not use this file except in compliance with the License. -+You may obtain a copy of the License at -+ -+ http://www.apache.org/licenses/LICENSE-2.0 -+ -+Unless required by applicable law or agreed to in writing, software -+distributed under the License is distributed on an "AS IS" BASIS, -+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+See the License for the specific language governing permissions and -+limitations under the License. -+==============================================================================*/ -+ -+include "mlir/IR/OpBase.td" -+include "stablehlo/dialect/StablehloOps.td" -+ -+def ComplexElementType : Type< -+ CPred<"isa(cast($_self).getElementType())">, -+ "Complex element type">; -+ -+def NonComplexElementType : Type< -+ CPred<"!isa(cast($_self).getElementType())">, -+ "Non-complex element type">; -+ -+def createConstantWithAllOnes : NativeCodeCall<"createConstantWithAllOnes($_builder, $_loc, $0)">; -+ -+// Express `tan` as -+// sine(x) / cosine(x) -+def TanOp_CompatiblityExpander : Pat<(StableHLO_TanOp NonComplexElementType:$input), -+ (StableHLO_DivOp -+ (StableHLO_SineOp $input), -+ (StableHLO_CosineOp $input) -+ )>; -+ -+// Express `tan(a + bi)` as -+// (tan(a) + i tanh(b)) / (1 - i tan(a) * tanh(b)) -+def TanOp_ComplexElementType_CompatiblityExpander : Pat<(StableHLO_TanOp ComplexElementType:$input), -+ (StableHLO_DivOp -+ (StableHLO_ComplexOp -+ (StableHLO_TanOp:$tan (StableHLO_RealOp $input)), -+ (StableHLO_TanhOp:$tanh (StableHLO_ImagOp $input))), -+ (StableHLO_ComplexOp -+ (createConstantWithAllOnes $tan), -+ (StableHLO_NegOp (StableHLO_MulOp $tan, $tanh))) -+ )>; - diff --git a/third_party/modules/stablehlo/20240829.0-54aa1a5/source.json b/third_party/modules/stablehlo/20240829.0-54aa1a5/source.json deleted file mode 100644 index 6d271a53..00000000 --- a/third_party/modules/stablehlo/20240829.0-54aa1a5/source.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "strip_prefix": "stablehlo-54aa1a57178251981da616b877dda1a88d840d11", - "url": "https://github.com/openxla/stablehlo/archive/54aa1a57178251981da616b877dda1a88d840d11.tar.gz", - "integrity": "sha256-PFXXs3vQ+WlMiCC8twbGEFbsBEF/67Z4qtmq8CtUSUU=", - "overlay": { - "MODULE.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0001-Remove-duplicated-symbols-in-StablehloApi.h.patch": "" - } -} diff --git a/third_party/modules/stablehlo/20240917.0-78c753a/MODULE.bazel b/third_party/modules/stablehlo/20240917.0-78c753a/MODULE.bazel deleted file mode 100644 index 7cbfb936..00000000 --- a/third_party/modules/stablehlo/20240917.0-78c753a/MODULE.bazel +++ /dev/null @@ -1,15 +0,0 @@ -module( - name = "stablehlo", - version = "20240917.0-78c753a", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "llvm-raw", version = "20240919.0-94c024a") - -llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") -llvm.configure( - targets = ["AArch64", "X86", "NVPTX"], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/stablehlo/20240917.0-78c753a/overlay/MODULE.bazel b/third_party/modules/stablehlo/20240917.0-78c753a/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/stablehlo/20240917.0-78c753a/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/stablehlo/20240917.0-78c753a/source.json b/third_party/modules/stablehlo/20240917.0-78c753a/source.json deleted file mode 100644 index cd677bd8..00000000 --- a/third_party/modules/stablehlo/20240917.0-78c753a/source.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "strip_prefix": "stablehlo-78c753ad13ad8205cacc5fcc12418c1ac97276c7", - "url": "https://github.com/openxla/stablehlo/archive/78c753ad13ad8205cacc5fcc12418c1ac97276c7.tar.gz", - "integrity": "sha256-2qFcPLXs7glf7RFDUiqbKXiBBamW7n8a30egR4Yu/bc=", - "overlay": { - "MODULE.bazel": "" - } -} diff --git a/third_party/modules/stablehlo/20241021.0-1c0b606/MODULE.bazel b/third_party/modules/stablehlo/20241021.0-1c0b606/MODULE.bazel deleted file mode 100644 index 637e0e76..00000000 --- a/third_party/modules/stablehlo/20241021.0-1c0b606/MODULE.bazel +++ /dev/null @@ -1,15 +0,0 @@ -module( - name = "stablehlo", - version = "20241021.0-1c0b606", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "llvm-raw", version = "20241022.0-6c4267f") - -llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") -llvm.configure( - targets = ["AArch64", "X86", "NVPTX"], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/stablehlo/20241021.0-1c0b606/overlay/MODULE.bazel b/third_party/modules/stablehlo/20241021.0-1c0b606/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/stablehlo/20241021.0-1c0b606/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/stablehlo/20241021.0-1c0b606/source.json b/third_party/modules/stablehlo/20241021.0-1c0b606/source.json deleted file mode 100644 index e67596eb..00000000 --- a/third_party/modules/stablehlo/20241021.0-1c0b606/source.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "strip_prefix": "stablehlo-f7f8e4e35296deeff2e12e39421ac8d9599ba340", - "url": "https://github.com/openxla/stablehlo/archive/f7f8e4e35296deeff2e12e39421ac8d9599ba340.tar.gz", - "integrity": "sha256-/chY6BYV7KcrXIuqXTNIUknISCiA1JYhWE8gG5cEeV4=", - "overlay": { - "MODULE.bazel": "" - } -} diff --git a/third_party/modules/stablehlo/20241220.0-38bb2f9/MODULE.bazel b/third_party/modules/stablehlo/20241220.0-38bb2f9/MODULE.bazel deleted file mode 100644 index a045302f..00000000 --- a/third_party/modules/stablehlo/20241220.0-38bb2f9/MODULE.bazel +++ /dev/null @@ -1,15 +0,0 @@ -module( - name = "stablehlo", - version = "20241220.0-38bb2f9", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "llvm-raw", version = "20250102.0-f739aa4") - -llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") -llvm.configure( - targets = ["AArch64", "X86", "NVPTX"], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/stablehlo/20241220.0-38bb2f9/overlay/MODULE.bazel b/third_party/modules/stablehlo/20241220.0-38bb2f9/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/stablehlo/20241220.0-38bb2f9/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/stablehlo/20241220.0-38bb2f9/source.json b/third_party/modules/stablehlo/20241220.0-38bb2f9/source.json deleted file mode 100644 index 9370183f..00000000 --- a/third_party/modules/stablehlo/20241220.0-38bb2f9/source.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "strip_prefix": "stablehlo-38bb2f9bf63b714e8a49fe34a478139058ee1660", - "url": "https://github.com/openxla/stablehlo/archive/38bb2f9bf63b714e8a49fe34a478139058ee1660.tar.gz", - "integrity": "sha256-TuaG8q6WCdJGl4ECpu8YO8ORYcRo0NSff3xHKVj8Ixw=", - "overlay": { - "MODULE.bazel": "" - } -} diff --git a/third_party/modules/stablehlo/20250117.0-c125b32/MODULE.bazel b/third_party/modules/stablehlo/20250117.0-c125b32/MODULE.bazel deleted file mode 100644 index 2aabd323..00000000 --- a/third_party/modules/stablehlo/20250117.0-c125b32/MODULE.bazel +++ /dev/null @@ -1,15 +0,0 @@ -module( - name = "stablehlo", - version = "20250117.0-c125b32", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "llvm-raw", version = "20250117.0-bf17016") - -llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") -llvm.configure( - targets = ["AArch64", "X86", "NVPTX"], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/stablehlo/20250117.0-c125b32/overlay/MODULE.bazel b/third_party/modules/stablehlo/20250117.0-c125b32/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/stablehlo/20250117.0-c125b32/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/stablehlo/20250117.0-c125b32/source.json b/third_party/modules/stablehlo/20250117.0-c125b32/source.json deleted file mode 100644 index fd8426be..00000000 --- a/third_party/modules/stablehlo/20250117.0-c125b32/source.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "strip_prefix": "stablehlo-c125b3284819fec57120231cf4430657dab7b881", - "url": "https://github.com/openxla/stablehlo/archive/c125b3284819fec57120231cf4430657dab7b881.tar.gz", - "integrity": "sha256-h4iBDLVbK2JZRDY7qdmNdns25AoVjpLekKdmsETEDf8=", - "overlay": { - "MODULE.bazel": "" - } -} diff --git a/third_party/modules/stablehlo/20250201.0-7775e3e/MODULE.bazel b/third_party/modules/stablehlo/20250201.0-7775e3e/MODULE.bazel deleted file mode 100644 index 7e9accd0..00000000 --- a/third_party/modules/stablehlo/20250201.0-7775e3e/MODULE.bazel +++ /dev/null @@ -1,15 +0,0 @@ -module( - name = "stablehlo", - version = "20250201.0-7775e3e", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "llvm-raw", version = "20250201.0-386af4a") - -llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") -llvm.configure( - targets = ["AArch64", "X86", "NVPTX"], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/stablehlo/20250201.0-7775e3e/overlay/MODULE.bazel b/third_party/modules/stablehlo/20250201.0-7775e3e/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/stablehlo/20250201.0-7775e3e/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/stablehlo/20250201.0-7775e3e/source.json b/third_party/modules/stablehlo/20250201.0-7775e3e/source.json deleted file mode 100644 index e3e17ea2..00000000 --- a/third_party/modules/stablehlo/20250201.0-7775e3e/source.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "strip_prefix": "stablehlo-7775e3e2f98fe458f227cdc41aec567535d1c796", - "url": "https://github.com/openxla/stablehlo/archive/7775e3e2f98fe458f227cdc41aec567535d1c796.tar.gz", - "integrity": "sha256-HM/MIs3FeN4PxcYgFsjj04tVJ4u4usGiYGSa9BwT5bg=", - "overlay": { - "MODULE.bazel": "" - } -} diff --git a/third_party/modules/stablehlo/20250217.0-4598975/MODULE.bazel b/third_party/modules/stablehlo/20250217.0-4598975/MODULE.bazel deleted file mode 100644 index 8c462655..00000000 --- a/third_party/modules/stablehlo/20250217.0-4598975/MODULE.bazel +++ /dev/null @@ -1,15 +0,0 @@ -module( - name = "stablehlo", - version = "20250217.0-4598975", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "llvm-raw", version = "20250217.0-0e779ad") - -llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") -llvm.configure( - targets = ["AArch64", "X86", "NVPTX"], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/stablehlo/20250217.0-4598975/overlay/MODULE.bazel b/third_party/modules/stablehlo/20250217.0-4598975/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/stablehlo/20250217.0-4598975/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/stablehlo/20250217.0-4598975/source.json b/third_party/modules/stablehlo/20250217.0-4598975/source.json deleted file mode 100644 index 6988aba7..00000000 --- a/third_party/modules/stablehlo/20250217.0-4598975/source.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "strip_prefix": "stablehlo-459897561d365ef97caba46984847f9184d472ec", - "url": "https://github.com/openxla/stablehlo/archive/459897561d365ef97caba46984847f9184d472ec.tar.gz", - "integrity": "sha256-J2KXzmX3uAJiwuMLl06n8dyb9l1TCzGZ62IULyAUcec=", - "overlay": { - "MODULE.bazel": "" - } -} diff --git a/third_party/modules/stablehlo/metadata.json b/third_party/modules/stablehlo/metadata.json deleted file mode 100644 index d0e5fef6..00000000 --- a/third_party/modules/stablehlo/metadata.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "homepage": "https://github.com/openxla/stablehlo", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": [ - "github:openxla/stablehlo" - ], - "versions": [ - "20240829.0-54aa1a5", - "20240917.0-78c753a", - "20241021.0-1c0b606", - "20241220.0-38bb2f9", - "20250117.0-c125b32", - "20250201.0-7775e3e", - ], - "yanked_versions": {} -} diff --git a/third_party/modules/xla/20240902.0-d18cd64/MODULE.bazel b/third_party/modules/xla/20240902.0-d18cd64/MODULE.bazel deleted file mode 100644 index 739fe835..00000000 --- a/third_party/modules/xla/20240902.0-d18cd64/MODULE.bazel +++ /dev/null @@ -1,34 +0,0 @@ -module( - name = "xla", - version = "20240902.0-d18cd64", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "rules_apple", version = "3.2.1", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.29.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-02-01", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") - -bazel_dep(name = "stablehlo", version = "20240829.0-54aa1a5") - -tsl = use_extension("//:tsl.bzl", "tsl") -use_repo(tsl, "tsl") - -xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") -use_repo( - xla_workspace, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", -) diff --git a/third_party/modules/xla/20240902.0-d18cd64/overlay/MODULE.bazel b/third_party/modules/xla/20240902.0-d18cd64/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20240902.0-d18cd64/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20240902.0-d18cd64/overlay/tsl.bzl b/third_party/modules/xla/20240902.0-d18cd64/overlay/tsl.bzl deleted file mode 100644 index 50c1df12..00000000 --- a/third_party/modules/xla/20240902.0-d18cd64/overlay/tsl.bzl +++ /dev/null @@ -1,13 +0,0 @@ -load("//third_party:repo.bzl", "tf_vendored") - -def _tsl_impl(mctx): - tf_vendored(name = "tsl", relpath = "third_party/tsl") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -tsl = module_extension( - implementation = _tsl_impl, -) diff --git a/third_party/modules/xla/20240902.0-d18cd64/overlay/workspace.bzl b/third_party/modules/xla/20240902.0-d18cd64/overlay/workspace.bzl deleted file mode 100644 index 87c035b7..00000000 --- a/third_party/modules/xla/20240902.0-d18cd64/overlay/workspace.bzl +++ /dev/null @@ -1,52 +0,0 @@ -load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _xla_workspace_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "@tsl//third_party/grpc:generate_cc_env_fix.patch", - "@tsl//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", - "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla_workspace = module_extension( - implementation = _xla_workspace_impl, -) diff --git a/third_party/modules/xla/20240902.0-d18cd64/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20240902.0-d18cd64/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch deleted file mode 100644 index 0f58f107..00000000 --- a/third_party/modules/xla/20240902.0-d18cd64/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch +++ /dev/null @@ -1,27 +0,0 @@ -From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Mon, 18 Mar 2024 17:17:34 +0100 -Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension - ---- - xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -index c821916ad..89a596123 100644 ---- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -@@ -16,8 +16,10 @@ limitations under the License. - #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - -+#ifdef __cplusplus - #include - #include -+#endif - - #include "xla/backends/profiler/plugin/profiler_c_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/xla/20240902.0-d18cd64/source.json b/third_party/modules/xla/20240902.0-d18cd64/source.json deleted file mode 100644 index 359f9b12..00000000 --- a/third_party/modules/xla/20240902.0-d18cd64/source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "strip_prefix": "xla-d18cd64b7cd61a2ade10089665ac104f639101b1", - "url": "https://github.com/openxla/xla/archive/d18cd64b7cd61a2ade10089665ac104f639101b1.tar.gz", - "integrity": "sha256-EtKhjU91STBceUmg0TUE6cPeRkeSz3LI2S1i3EFMj/E=", - "overlay": { - "tsl.bzl": "", - "workspace.bzl": "", - "MODULE.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" - } -} diff --git a/third_party/modules/xla/20240919.0-1b18dd6/MODULE.bazel b/third_party/modules/xla/20240919.0-1b18dd6/MODULE.bazel deleted file mode 100644 index bd3a0046..00000000 --- a/third_party/modules/xla/20240919.0-1b18dd6/MODULE.bazel +++ /dev/null @@ -1,34 +0,0 @@ -module( - name = "xla", - version = "20240919.0-1b18dd6", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "rules_apple", version = "3.2.1", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.29.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-02-01", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") - -bazel_dep(name = "stablehlo", version = "20240917.0-78c753a") - -tsl = use_extension("//:tsl.bzl", "tsl") -use_repo(tsl, "tsl") - -xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") -use_repo( - xla_workspace, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", -) diff --git a/third_party/modules/xla/20240919.0-1b18dd6/overlay/MODULE.bazel b/third_party/modules/xla/20240919.0-1b18dd6/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20240919.0-1b18dd6/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20240919.0-1b18dd6/overlay/tsl.bzl b/third_party/modules/xla/20240919.0-1b18dd6/overlay/tsl.bzl deleted file mode 100644 index 50c1df12..00000000 --- a/third_party/modules/xla/20240919.0-1b18dd6/overlay/tsl.bzl +++ /dev/null @@ -1,13 +0,0 @@ -load("//third_party:repo.bzl", "tf_vendored") - -def _tsl_impl(mctx): - tf_vendored(name = "tsl", relpath = "third_party/tsl") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -tsl = module_extension( - implementation = _tsl_impl, -) diff --git a/third_party/modules/xla/20240919.0-1b18dd6/overlay/workspace.bzl b/third_party/modules/xla/20240919.0-1b18dd6/overlay/workspace.bzl deleted file mode 100644 index 87c035b7..00000000 --- a/third_party/modules/xla/20240919.0-1b18dd6/overlay/workspace.bzl +++ /dev/null @@ -1,52 +0,0 @@ -load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _xla_workspace_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "@tsl//third_party/grpc:generate_cc_env_fix.patch", - "@tsl//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", - "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla_workspace = module_extension( - implementation = _xla_workspace_impl, -) diff --git a/third_party/modules/xla/20240919.0-1b18dd6/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20240919.0-1b18dd6/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch deleted file mode 100644 index 0f58f107..00000000 --- a/third_party/modules/xla/20240919.0-1b18dd6/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch +++ /dev/null @@ -1,27 +0,0 @@ -From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Mon, 18 Mar 2024 17:17:34 +0100 -Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension - ---- - xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -index c821916ad..89a596123 100644 ---- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -@@ -16,8 +16,10 @@ limitations under the License. - #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - -+#ifdef __cplusplus - #include - #include -+#endif - - #include "xla/backends/profiler/plugin/profiler_c_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/xla/20240919.0-1b18dd6/source.json b/third_party/modules/xla/20240919.0-1b18dd6/source.json deleted file mode 100644 index 9853728a..00000000 --- a/third_party/modules/xla/20240919.0-1b18dd6/source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "strip_prefix": "xla-d391119197eab771a84c1f8a59a7f50b7da4b43d", - "url": "https://github.com/openxla/xla/archive/d391119197eab771a84c1f8a59a7f50b7da4b43d.tar.gz", - "integrity": "sha256-ToPcvPhoGX6Ny3yLx6kJlQSwj7k5Xy4T+BjIp3DAj/s=", - "overlay": { - "tsl.bzl": "", - "workspace.bzl": "", - "MODULE.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" - } -} diff --git a/third_party/modules/xla/20241025.0-4663f04/MODULE.bazel b/third_party/modules/xla/20241025.0-4663f04/MODULE.bazel deleted file mode 100644 index 22019c9e..00000000 --- a/third_party/modules/xla/20241025.0-4663f04/MODULE.bazel +++ /dev/null @@ -1,34 +0,0 @@ -module( - name = "xla", - version = "20241025.0-4663f04", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "rules_apple", version = "3.2.1", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.29.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-02-01", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") - -bazel_dep(name = "stablehlo", version = "20241021.0-1c0b606") - -tsl = use_extension("//:tsl.bzl", "tsl") -use_repo(tsl, "tsl") - -xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") -use_repo( - xla_workspace, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", -) diff --git a/third_party/modules/xla/20241025.0-4663f04/overlay/MODULE.bazel b/third_party/modules/xla/20241025.0-4663f04/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20241025.0-4663f04/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20241025.0-4663f04/overlay/tsl.bzl b/third_party/modules/xla/20241025.0-4663f04/overlay/tsl.bzl deleted file mode 100644 index dc25c9d6..00000000 --- a/third_party/modules/xla/20241025.0-4663f04/overlay/tsl.bzl +++ /dev/null @@ -1,19 +0,0 @@ -load("//third_party:repo.bzl", "tf_vendored") -load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") - -def _tsl_impl(mctx): - python_init_repositories( - requirements = { - "3.11": "//:requirements_lock_3_11.txt", - }, - ) - tf_vendored(name = "tsl", relpath = "third_party/tsl") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = ["tsl"], - root_module_direct_dev_deps = [], - ) - -tsl = module_extension( - implementation = _tsl_impl, -) diff --git a/third_party/modules/xla/20241025.0-4663f04/overlay/workspace.bzl b/third_party/modules/xla/20241025.0-4663f04/overlay/workspace.bzl deleted file mode 100644 index 87c035b7..00000000 --- a/third_party/modules/xla/20241025.0-4663f04/overlay/workspace.bzl +++ /dev/null @@ -1,52 +0,0 @@ -load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _xla_workspace_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "@tsl//third_party/grpc:generate_cc_env_fix.patch", - "@tsl//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", - "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla_workspace = module_extension( - implementation = _xla_workspace_impl, -) diff --git a/third_party/modules/xla/20241025.0-4663f04/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20241025.0-4663f04/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch deleted file mode 100644 index 0f58f107..00000000 --- a/third_party/modules/xla/20241025.0-4663f04/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch +++ /dev/null @@ -1,27 +0,0 @@ -From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Mon, 18 Mar 2024 17:17:34 +0100 -Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension - ---- - xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -index c821916ad..89a596123 100644 ---- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -@@ -16,8 +16,10 @@ limitations under the License. - #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - -+#ifdef __cplusplus - #include - #include -+#endif - - #include "xla/backends/profiler/plugin/profiler_c_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/xla/20241025.0-4663f04/source.json b/third_party/modules/xla/20241025.0-4663f04/source.json deleted file mode 100644 index 33327387..00000000 --- a/third_party/modules/xla/20241025.0-4663f04/source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "strip_prefix": "xla-4663f04d2653d686833f9c306bd0d899c3127358", - "url": "https://github.com/openxla/xla/archive/4663f04d2653d686833f9c306bd0d899c3127358.tar.gz", - "integrity": "sha256-tQRn0TXkPcmKpCcIuTYM10XNB1qC/d6++4rA1mlnnKw=", - "overlay": { - "tsl.bzl": "", - "workspace.bzl": "", - "MODULE.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" - } -} diff --git a/third_party/modules/xla/20250103.0-5f1fe6a/MODULE.bazel b/third_party/modules/xla/20250103.0-5f1fe6a/MODULE.bazel deleted file mode 100644 index 5261911d..00000000 --- a/third_party/modules/xla/20250103.0-5f1fe6a/MODULE.bazel +++ /dev/null @@ -1,34 +0,0 @@ -module( - name = "xla", - version = "20250103.0-5f1fe6a", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "rules_apple", version = "3.2.1", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.29.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-02-01", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") - -bazel_dep(name = "stablehlo", version = "20241220.0-38bb2f9") - -tsl = use_extension("//:tsl.bzl", "tsl") -use_repo(tsl, "tsl") - -xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") -use_repo( - xla_workspace, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", -) diff --git a/third_party/modules/xla/20250103.0-5f1fe6a/overlay/MODULE.bazel b/third_party/modules/xla/20250103.0-5f1fe6a/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20250103.0-5f1fe6a/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250103.0-5f1fe6a/overlay/tsl.bzl b/third_party/modules/xla/20250103.0-5f1fe6a/overlay/tsl.bzl deleted file mode 100644 index dc25c9d6..00000000 --- a/third_party/modules/xla/20250103.0-5f1fe6a/overlay/tsl.bzl +++ /dev/null @@ -1,19 +0,0 @@ -load("//third_party:repo.bzl", "tf_vendored") -load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") - -def _tsl_impl(mctx): - python_init_repositories( - requirements = { - "3.11": "//:requirements_lock_3_11.txt", - }, - ) - tf_vendored(name = "tsl", relpath = "third_party/tsl") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = ["tsl"], - root_module_direct_dev_deps = [], - ) - -tsl = module_extension( - implementation = _tsl_impl, -) diff --git a/third_party/modules/xla/20250103.0-5f1fe6a/overlay/workspace.bzl b/third_party/modules/xla/20250103.0-5f1fe6a/overlay/workspace.bzl deleted file mode 100644 index 87c035b7..00000000 --- a/third_party/modules/xla/20250103.0-5f1fe6a/overlay/workspace.bzl +++ /dev/null @@ -1,52 +0,0 @@ -load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _xla_workspace_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "@tsl//third_party/grpc:generate_cc_env_fix.patch", - "@tsl//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", - "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla_workspace = module_extension( - implementation = _xla_workspace_impl, -) diff --git a/third_party/modules/xla/20250103.0-5f1fe6a/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20250103.0-5f1fe6a/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch deleted file mode 100644 index 0f58f107..00000000 --- a/third_party/modules/xla/20250103.0-5f1fe6a/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch +++ /dev/null @@ -1,27 +0,0 @@ -From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Mon, 18 Mar 2024 17:17:34 +0100 -Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension - ---- - xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -index c821916ad..89a596123 100644 ---- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -@@ -16,8 +16,10 @@ limitations under the License. - #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - -+#ifdef __cplusplus - #include - #include -+#endif - - #include "xla/backends/profiler/plugin/profiler_c_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/xla/20250103.0-5f1fe6a/source.json b/third_party/modules/xla/20250103.0-5f1fe6a/source.json deleted file mode 100644 index 690bbb4b..00000000 --- a/third_party/modules/xla/20250103.0-5f1fe6a/source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "strip_prefix": "xla-5f1fe6a2f35756238e4a0a0c4e754d6454296efe", - "url": "https://github.com/openxla/xla/archive/5f1fe6a2f35756238e4a0a0c4e754d6454296efe.tar.gz", - "integrity": "sha256-0+kdYIla4F63oPfACtw6LmmMLdKTFlBBrgvc9PmxFoI=", - "overlay": { - "tsl.bzl": "", - "workspace.bzl": "", - "MODULE.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" - } -} diff --git a/third_party/modules/xla/20250122.0-cc075be/MODULE.bazel b/third_party/modules/xla/20250122.0-cc075be/MODULE.bazel deleted file mode 100644 index fd99777d..00000000 --- a/third_party/modules/xla/20250122.0-cc075be/MODULE.bazel +++ /dev/null @@ -1,34 +0,0 @@ -module( - name = "xla", - version = "20250122.0-cc075be", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "rules_apple", version = "3.2.1", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.29.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-02-01", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") - -bazel_dep(name = "stablehlo", version = "20250117.0-c125b32") - -tsl = use_extension("//:tsl.bzl", "tsl") -use_repo(tsl, "tsl") - -xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") -use_repo( - xla_workspace, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", -) diff --git a/third_party/modules/xla/20250122.0-cc075be/overlay/MODULE.bazel b/third_party/modules/xla/20250122.0-cc075be/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20250122.0-cc075be/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250122.0-cc075be/overlay/tsl.bzl b/third_party/modules/xla/20250122.0-cc075be/overlay/tsl.bzl deleted file mode 100644 index dc25c9d6..00000000 --- a/third_party/modules/xla/20250122.0-cc075be/overlay/tsl.bzl +++ /dev/null @@ -1,19 +0,0 @@ -load("//third_party:repo.bzl", "tf_vendored") -load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") - -def _tsl_impl(mctx): - python_init_repositories( - requirements = { - "3.11": "//:requirements_lock_3_11.txt", - }, - ) - tf_vendored(name = "tsl", relpath = "third_party/tsl") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = ["tsl"], - root_module_direct_dev_deps = [], - ) - -tsl = module_extension( - implementation = _tsl_impl, -) diff --git a/third_party/modules/xla/20250122.0-cc075be/overlay/workspace.bzl b/third_party/modules/xla/20250122.0-cc075be/overlay/workspace.bzl deleted file mode 100644 index 87c035b7..00000000 --- a/third_party/modules/xla/20250122.0-cc075be/overlay/workspace.bzl +++ /dev/null @@ -1,52 +0,0 @@ -load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _xla_workspace_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "@tsl//third_party/grpc:generate_cc_env_fix.patch", - "@tsl//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", - "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla_workspace = module_extension( - implementation = _xla_workspace_impl, -) diff --git a/third_party/modules/xla/20250122.0-cc075be/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20250122.0-cc075be/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch deleted file mode 100644 index 0f58f107..00000000 --- a/third_party/modules/xla/20250122.0-cc075be/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch +++ /dev/null @@ -1,27 +0,0 @@ -From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Mon, 18 Mar 2024 17:17:34 +0100 -Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension - ---- - xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -index c821916ad..89a596123 100644 ---- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -@@ -16,8 +16,10 @@ limitations under the License. - #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - -+#ifdef __cplusplus - #include - #include -+#endif - - #include "xla/backends/profiler/plugin/profiler_c_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/xla/20250122.0-cc075be/source.json b/third_party/modules/xla/20250122.0-cc075be/source.json deleted file mode 100644 index 4946d805..00000000 --- a/third_party/modules/xla/20250122.0-cc075be/source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "strip_prefix": "xla-cc075beb6148c2777da2b6749c63830856ee6c2a", - "url": "https://github.com/openxla/xla/archive/cc075beb6148c2777da2b6749c63830856ee6c2a.tar.gz", - "integrity": "sha256-oB8S38WZKEXBtZ6rARd0oL9SAtSmfsPM2xTuj3ylexc=", - "overlay": { - "tsl.bzl": "", - "workspace.bzl": "", - "MODULE.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" - } -} diff --git a/third_party/modules/xla/20250123.0-cc075be/MODULE.bazel b/third_party/modules/xla/20250123.0-cc075be/MODULE.bazel deleted file mode 100644 index 6a94e649..00000000 --- a/third_party/modules/xla/20250123.0-cc075be/MODULE.bazel +++ /dev/null @@ -1,34 +0,0 @@ -module( - name = "xla", - version = "20250123.0-cc075be", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.17") -bazel_dep(name = "rules_apple", version = "3.2.1", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.29.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-02-01", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") - -bazel_dep(name = "stablehlo", version = "20250117.0-c125b32") - -tsl = use_extension("//:tsl.bzl", "tsl") -use_repo(tsl, "tsl") - -xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") -use_repo( - xla_workspace, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", -) diff --git a/third_party/modules/xla/20250123.0-cc075be/overlay/MODULE.bazel b/third_party/modules/xla/20250123.0-cc075be/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20250123.0-cc075be/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250123.0-cc075be/overlay/tsl.bzl b/third_party/modules/xla/20250123.0-cc075be/overlay/tsl.bzl deleted file mode 100644 index dc25c9d6..00000000 --- a/third_party/modules/xla/20250123.0-cc075be/overlay/tsl.bzl +++ /dev/null @@ -1,19 +0,0 @@ -load("//third_party:repo.bzl", "tf_vendored") -load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") - -def _tsl_impl(mctx): - python_init_repositories( - requirements = { - "3.11": "//:requirements_lock_3_11.txt", - }, - ) - tf_vendored(name = "tsl", relpath = "third_party/tsl") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = ["tsl"], - root_module_direct_dev_deps = [], - ) - -tsl = module_extension( - implementation = _tsl_impl, -) diff --git a/third_party/modules/xla/20250123.0-cc075be/overlay/workspace.bzl b/third_party/modules/xla/20250123.0-cc075be/overlay/workspace.bzl deleted file mode 100644 index 87c035b7..00000000 --- a/third_party/modules/xla/20250123.0-cc075be/overlay/workspace.bzl +++ /dev/null @@ -1,52 +0,0 @@ -load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _xla_workspace_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "@tsl//third_party/grpc:generate_cc_env_fix.patch", - "@tsl//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", - "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla_workspace = module_extension( - implementation = _xla_workspace_impl, -) diff --git a/third_party/modules/xla/20250123.0-cc075be/patches/0001-bazel-migration-to-bazel-8.0.1.patch b/third_party/modules/xla/20250123.0-cc075be/patches/0001-bazel-migration-to-bazel-8.0.1.patch deleted file mode 100644 index 221fa393..00000000 --- a/third_party/modules/xla/20250123.0-cc075be/patches/0001-bazel-migration-to-bazel-8.0.1.patch +++ /dev/null @@ -1,42 +0,0 @@ -From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 -From: Jean-Baptiste Dalido -Date: Mon, 6 Jan 2025 13:33:13 +0100 -Subject: [PATCH] bazel: migration to bazel 8.0.1 - ---- - .bazelversion | 2 +- - third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/.bazelversion b/.bazelversion -index f22d756da3..fa5fce04b3 100644 ---- a/.bazelversion -+++ b/.bazelversion -@@ -1 +1 @@ --6.5.0 -+8.0.1 -\ No newline at end of file -diff --git a/third_party/tsl/third_party/gpus/cuda_configure.bzl b/third_party/tsl/third_party/gpus/cuda_configure.bzl -index d62531152d..71d80a5a99 100644 ---- a/third_party/tsl/third_party/gpus/cuda_configure.bzl -+++ b/third_party/tsl/third_party/gpus/cuda_configure.bzl -@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. - load( - "@bazel_tools//tools/cpp:lib_cc_configure.bzl", - "escape_string", -- "get_env_var", - ) - load( - "@bazel_tools//tools/cpp:windows_cc_configure.bzl", -- "find_msvc_tool", - "find_vc_path", - "setup_vc_env_vars", - ) -+load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") -+load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") - load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") - load( - "//third_party/remote_config:common.bzl", --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/xla/20250123.0-cc075be/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20250123.0-cc075be/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch deleted file mode 100644 index 0f58f107..00000000 --- a/third_party/modules/xla/20250123.0-cc075be/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch +++ /dev/null @@ -1,27 +0,0 @@ -From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Mon, 18 Mar 2024 17:17:34 +0100 -Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension - ---- - xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -index c821916ad..89a596123 100644 ---- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -@@ -16,8 +16,10 @@ limitations under the License. - #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - -+#ifdef __cplusplus - #include - #include -+#endif - - #include "xla/backends/profiler/plugin/profiler_c_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/xla/20250123.0-cc075be/source.json b/third_party/modules/xla/20250123.0-cc075be/source.json deleted file mode 100644 index 02869dd7..00000000 --- a/third_party/modules/xla/20250123.0-cc075be/source.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "strip_prefix": "xla-cc075beb6148c2777da2b6749c63830856ee6c2a", - "url": "https://github.com/openxla/xla/archive/cc075beb6148c2777da2b6749c63830856ee6c2a.tar.gz", - "integrity": "sha256-oB8S38WZKEXBtZ6rARd0oL9SAtSmfsPM2xTuj3ylexc=", - "overlay": { - "tsl.bzl": "", - "workspace.bzl": "", - "MODULE.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0001-bazel-migration-to-bazel-8.0.1.patch": "", - "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" - } -} diff --git a/third_party/modules/xla/20250204-0-6789523/MODULE.bazel b/third_party/modules/xla/20250204-0-6789523/MODULE.bazel deleted file mode 100644 index 7d2ebfb7..00000000 --- a/third_party/modules/xla/20250204-0-6789523/MODULE.bazel +++ /dev/null @@ -1,34 +0,0 @@ -module( - name = "xla", - version = "20250204-0-6789523", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.17") -bazel_dep(name = "rules_apple", version = "3.17.0", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.29.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") - -bazel_dep(name = "stablehlo", version = "20250201.0-7775e3e") - -tsl = use_extension("//:tsl.bzl", "tsl") -use_repo(tsl, "tsl") - -xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") -use_repo( - xla_workspace, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", -) diff --git a/third_party/modules/xla/20250204-0-6789523/overlay/MODULE.bazel b/third_party/modules/xla/20250204-0-6789523/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20250204-0-6789523/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250204-0-6789523/overlay/tsl.bzl b/third_party/modules/xla/20250204-0-6789523/overlay/tsl.bzl deleted file mode 100644 index dc25c9d6..00000000 --- a/third_party/modules/xla/20250204-0-6789523/overlay/tsl.bzl +++ /dev/null @@ -1,19 +0,0 @@ -load("//third_party:repo.bzl", "tf_vendored") -load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") - -def _tsl_impl(mctx): - python_init_repositories( - requirements = { - "3.11": "//:requirements_lock_3_11.txt", - }, - ) - tf_vendored(name = "tsl", relpath = "third_party/tsl") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = ["tsl"], - root_module_direct_dev_deps = [], - ) - -tsl = module_extension( - implementation = _tsl_impl, -) diff --git a/third_party/modules/xla/20250204-0-6789523/overlay/workspace.bzl b/third_party/modules/xla/20250204-0-6789523/overlay/workspace.bzl deleted file mode 100644 index 87c035b7..00000000 --- a/third_party/modules/xla/20250204-0-6789523/overlay/workspace.bzl +++ /dev/null @@ -1,52 +0,0 @@ -load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _xla_workspace_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "@tsl//third_party/grpc:generate_cc_env_fix.patch", - "@tsl//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", - "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla_workspace = module_extension( - implementation = _xla_workspace_impl, -) diff --git a/third_party/modules/xla/20250204-0-6789523/patches/0001-bazel-migration-to-bazel-8.0.1.patch b/third_party/modules/xla/20250204-0-6789523/patches/0001-bazel-migration-to-bazel-8.0.1.patch deleted file mode 100644 index a29cbd9e..00000000 --- a/third_party/modules/xla/20250204-0-6789523/patches/0001-bazel-migration-to-bazel-8.0.1.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 -From: Jean-Baptiste Dalido -Date: Mon, 6 Jan 2025 13:33:13 +0100 -Subject: [PATCH] bazel: migration to bazel 8.0.1 - ---- - .bazelversion | 2 +- - third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/.bazelversion b/.bazelversion -index f22d756da3..fa5fce04b3 100644 ---- a/.bazelversion -+++ b/.bazelversion -@@ -1 +1 @@ --6.5.0 -+8.0.1 -\ No newline at end of file -diff --git a/third_party/tsl/third_party/gpus/cuda_configure.bzl b/third_party/tsl/third_party/gpus/cuda_configure.bzl -index d62531152d..71d80a5a99 100644 ---- a/third_party/tsl/third_party/gpus/cuda_configure.bzl -+++ b/third_party/tsl/third_party/gpus/cuda_configure.bzl -@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. - load( - "@bazel_tools//tools/cpp:lib_cc_configure.bzl", - "escape_string", -- "get_env_var", - ) - load( - "@bazel_tools//tools/cpp:windows_cc_configure.bzl", -- "find_msvc_tool", - "find_vc_path", - "setup_vc_env_vars", - ) -+load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") -+load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") - load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") - load( - "//third_party/remote_config:common.bzl", --- -2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250204-0-6789523/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20250204-0-6789523/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch deleted file mode 100644 index 4b6a65ce..00000000 --- a/third_party/modules/xla/20250204-0-6789523/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch +++ /dev/null @@ -1,26 +0,0 @@ -From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Mon, 18 Mar 2024 17:17:34 +0100 -Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension - ---- - xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -index c821916ad..89a596123 100644 ---- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -@@ -16,8 +16,10 @@ limitations under the License. - #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - -+#ifdef __cplusplus - #include - #include -+#endif - - #include "xla/backends/profiler/plugin/profiler_c_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" --- -2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250204-0-6789523/source.json b/third_party/modules/xla/20250204-0-6789523/source.json deleted file mode 100644 index ab3abf73..00000000 --- a/third_party/modules/xla/20250204-0-6789523/source.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "strip_prefix": "xla-6789523d0a42c2cf83309a263f417122c4c63811", - "url": "https://github.com/openxla/xla/archive/6789523d0a42c2cf83309a263f417122c4c63811.tar.gz", - "integrity": "sha256-7z0MdXQBoGApSYpTvYjKFmg18tGkjJ3P/PPQ487cPc8=", - "overlay": { - "tsl.bzl": "", - "workspace.bzl": "", - "MODULE.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0001-bazel-migration-to-bazel-8.0.1.patch": "", - "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" - } -} diff --git a/third_party/modules/xla/20250204.1-6789523/MODULE.bazel b/third_party/modules/xla/20250204.1-6789523/MODULE.bazel deleted file mode 100644 index f0ad4f19..00000000 --- a/third_party/modules/xla/20250204.1-6789523/MODULE.bazel +++ /dev/null @@ -1,34 +0,0 @@ -module( - name = "xla", - version = "20250204.1-6789523", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.17") -bazel_dep(name = "rules_apple", version = "3.17.0", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.29.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") - -bazel_dep(name = "stablehlo", version = "20250217.0-4598975") - -tsl = use_extension("//:tsl.bzl", "tsl") -use_repo(tsl, "tsl") - -xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") -use_repo( - xla_workspace, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", -) diff --git a/third_party/modules/xla/20250204.1-6789523/overlay/MODULE.bazel b/third_party/modules/xla/20250204.1-6789523/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20250204.1-6789523/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250204.1-6789523/overlay/tsl.bzl b/third_party/modules/xla/20250204.1-6789523/overlay/tsl.bzl deleted file mode 100644 index dc25c9d6..00000000 --- a/third_party/modules/xla/20250204.1-6789523/overlay/tsl.bzl +++ /dev/null @@ -1,19 +0,0 @@ -load("//third_party:repo.bzl", "tf_vendored") -load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") - -def _tsl_impl(mctx): - python_init_repositories( - requirements = { - "3.11": "//:requirements_lock_3_11.txt", - }, - ) - tf_vendored(name = "tsl", relpath = "third_party/tsl") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = ["tsl"], - root_module_direct_dev_deps = [], - ) - -tsl = module_extension( - implementation = _tsl_impl, -) diff --git a/third_party/modules/xla/20250204.1-6789523/overlay/workspace.bzl b/third_party/modules/xla/20250204.1-6789523/overlay/workspace.bzl deleted file mode 100644 index 87c035b7..00000000 --- a/third_party/modules/xla/20250204.1-6789523/overlay/workspace.bzl +++ /dev/null @@ -1,52 +0,0 @@ -load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _xla_workspace_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "@tsl//third_party/grpc:generate_cc_env_fix.patch", - "@tsl//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", - "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla_workspace = module_extension( - implementation = _xla_workspace_impl, -) diff --git a/third_party/modules/xla/20250204.1-6789523/patches/0001-bazel-migration-to-bazel-8.0.1.patch b/third_party/modules/xla/20250204.1-6789523/patches/0001-bazel-migration-to-bazel-8.0.1.patch deleted file mode 100644 index a29cbd9e..00000000 --- a/third_party/modules/xla/20250204.1-6789523/patches/0001-bazel-migration-to-bazel-8.0.1.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 -From: Jean-Baptiste Dalido -Date: Mon, 6 Jan 2025 13:33:13 +0100 -Subject: [PATCH] bazel: migration to bazel 8.0.1 - ---- - .bazelversion | 2 +- - third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/.bazelversion b/.bazelversion -index f22d756da3..fa5fce04b3 100644 ---- a/.bazelversion -+++ b/.bazelversion -@@ -1 +1 @@ --6.5.0 -+8.0.1 -\ No newline at end of file -diff --git a/third_party/tsl/third_party/gpus/cuda_configure.bzl b/third_party/tsl/third_party/gpus/cuda_configure.bzl -index d62531152d..71d80a5a99 100644 ---- a/third_party/tsl/third_party/gpus/cuda_configure.bzl -+++ b/third_party/tsl/third_party/gpus/cuda_configure.bzl -@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. - load( - "@bazel_tools//tools/cpp:lib_cc_configure.bzl", - "escape_string", -- "get_env_var", - ) - load( - "@bazel_tools//tools/cpp:windows_cc_configure.bzl", -- "find_msvc_tool", - "find_vc_path", - "setup_vc_env_vars", - ) -+load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") -+load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") - load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") - load( - "//third_party/remote_config:common.bzl", --- -2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250204.1-6789523/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20250204.1-6789523/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch deleted file mode 100644 index 4b6a65ce..00000000 --- a/third_party/modules/xla/20250204.1-6789523/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch +++ /dev/null @@ -1,26 +0,0 @@ -From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Mon, 18 Mar 2024 17:17:34 +0100 -Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension - ---- - xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -index c821916ad..89a596123 100644 ---- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -@@ -16,8 +16,10 @@ limitations under the License. - #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - -+#ifdef __cplusplus - #include - #include -+#endif - - #include "xla/backends/profiler/plugin/profiler_c_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" --- -2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250204.1-6789523/source.json b/third_party/modules/xla/20250204.1-6789523/source.json deleted file mode 100644 index ab3abf73..00000000 --- a/third_party/modules/xla/20250204.1-6789523/source.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "strip_prefix": "xla-6789523d0a42c2cf83309a263f417122c4c63811", - "url": "https://github.com/openxla/xla/archive/6789523d0a42c2cf83309a263f417122c4c63811.tar.gz", - "integrity": "sha256-7z0MdXQBoGApSYpTvYjKFmg18tGkjJ3P/PPQ487cPc8=", - "overlay": { - "tsl.bzl": "", - "workspace.bzl": "", - "MODULE.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0001-bazel-migration-to-bazel-8.0.1.patch": "", - "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" - } -} diff --git a/third_party/modules/xla/20250317.0-71c67e2/MODULE.bazel b/third_party/modules/xla/20250317.0-71c67e2/MODULE.bazel deleted file mode 100644 index 19a58980..00000000 --- a/third_party/modules/xla/20250317.0-71c67e2/MODULE.bazel +++ /dev/null @@ -1,37 +0,0 @@ -module( - name = "xla", - version = "20250317.0-71c67e2", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.17") -bazel_dep(name = "rules_apple", version = "3.17.0", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.29.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") - -tsl = use_extension("//:tsl.bzl", "tsl") -use_repo(tsl, "tsl", "python_version_repo") - -xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") -use_repo( - xla_workspace, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", - "llvm-raw", - "stablehlo", -) - -llvm_configure = use_repo_rule("@llvm-raw//utils/bazel:configure.bzl", "llvm_configure") -llvm_configure(name = "llvm-project") diff --git a/third_party/modules/xla/20250317.0-71c67e2/overlay/MODULE.bazel b/third_party/modules/xla/20250317.0-71c67e2/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20250317.0-71c67e2/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250317.0-71c67e2/overlay/tsl.bzl b/third_party/modules/xla/20250317.0-71c67e2/overlay/tsl.bzl deleted file mode 100644 index dc25c9d6..00000000 --- a/third_party/modules/xla/20250317.0-71c67e2/overlay/tsl.bzl +++ /dev/null @@ -1,19 +0,0 @@ -load("//third_party:repo.bzl", "tf_vendored") -load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") - -def _tsl_impl(mctx): - python_init_repositories( - requirements = { - "3.11": "//:requirements_lock_3_11.txt", - }, - ) - tf_vendored(name = "tsl", relpath = "third_party/tsl") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = ["tsl"], - root_module_direct_dev_deps = [], - ) - -tsl = module_extension( - implementation = _tsl_impl, -) diff --git a/third_party/modules/xla/20250317.0-71c67e2/overlay/workspace.bzl b/third_party/modules/xla/20250317.0-71c67e2/overlay/workspace.bzl deleted file mode 100644 index 192e06f8..00000000 --- a/third_party/modules/xla/20250317.0-71c67e2/overlay/workspace.bzl +++ /dev/null @@ -1,60 +0,0 @@ -load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("//third_party/llvm:workspace.bzl", llvm = "repo") -load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") -load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") -load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("//third_party/triton:workspace.bzl", triton = "repo") -load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _xla_workspace_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - pybind11_bazel() - triton() - llvm("llvm-raw") - stablehlo() - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "//third_party/grpc:generate_cc_env_fix.patch", - "//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "//third_party/systemlibs:BUILD.bazel": "bazel/BUILD.bazel", - "//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla_workspace = module_extension( - implementation = _xla_workspace_impl, -) diff --git a/third_party/modules/xla/20250317.0-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/modules/xla/20250317.0-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch deleted file mode 100644 index 8924cf4b..00000000 --- a/third_party/modules/xla/20250317.0-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 -From: Jean-Baptiste Dalido -Date: Mon, 6 Jan 2025 13:33:13 +0100 -Subject: [PATCH] bazel: migration to bazel 8.0.1 - ---- - .bazelversion | 2 +- - third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/.bazelversion b/.bazelversion -index f22d756da3..fa5fce04b3 100644 ---- a/.bazelversion -+++ b/.bazelversion -@@ -1 +1 @@ --7.4.1 -+8.1.1 -\ No newline at end of file -diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl -index d62531152d..71d80a5a99 100644 ---- a/third_party/gpus/cuda_configure.bzl -+++ b/third_party/gpus/cuda_configure.bzl -@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. - load( - "@bazel_tools//tools/cpp:lib_cc_configure.bzl", - "escape_string", -- "get_env_var", - ) - load( - "@bazel_tools//tools/cpp:windows_cc_configure.bzl", -- "find_msvc_tool", - "find_vc_path", - "setup_vc_env_vars", - ) -+load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") -+load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") - load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") - load( - "//third_party/remote_config:common.bzl", --- -2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250317.0-71c67e2/source.json b/third_party/modules/xla/20250317.0-71c67e2/source.json deleted file mode 100644 index c239d180..00000000 --- a/third_party/modules/xla/20250317.0-71c67e2/source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "strip_prefix": "xla-71c67e2a4f40267115a0d4ea7c36748bbe7e750e", - "url": "https://github.com/openxla/xla/archive/71c67e2a4f40267115a0d4ea7c36748bbe7e750e.tar.gz", - "integrity": "sha256-j6D1MC7+WsbZ+Ve3hPmDlCzZX1yV6RIP2BWDgQcbcYc=", - "overlay": { - "tsl.bzl": "", - "workspace.bzl": "", - "MODULE.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0001-bazel-migration-to-bazel-8.1.1.patch": "" - } -} diff --git a/third_party/modules/xla/20250317.1-71c67e2/MODULE.bazel b/third_party/modules/xla/20250317.1-71c67e2/MODULE.bazel deleted file mode 100644 index 516cef14..00000000 --- a/third_party/modules/xla/20250317.1-71c67e2/MODULE.bazel +++ /dev/null @@ -1,37 +0,0 @@ -module( - name = "xla", - version = "20250317.1-71c67e2", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.17") -bazel_dep(name = "rules_apple", version = "3.17.0", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.29.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") - -tsl = use_extension("//:tsl.bzl", "tsl") -use_repo(tsl, "tsl", "python_version_repo") - -xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") -use_repo( - xla_workspace, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", - "llvm-raw", - "stablehlo", -) - -llvm_configure = use_repo_rule("@llvm-raw//utils/bazel:configure.bzl", "llvm_configure") -llvm_configure(name = "llvm-project") diff --git a/third_party/modules/xla/20250317.1-71c67e2/overlay/MODULE.bazel b/third_party/modules/xla/20250317.1-71c67e2/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20250317.1-71c67e2/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250317.1-71c67e2/overlay/tsl.bzl b/third_party/modules/xla/20250317.1-71c67e2/overlay/tsl.bzl deleted file mode 100644 index dc25c9d6..00000000 --- a/third_party/modules/xla/20250317.1-71c67e2/overlay/tsl.bzl +++ /dev/null @@ -1,19 +0,0 @@ -load("//third_party:repo.bzl", "tf_vendored") -load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") - -def _tsl_impl(mctx): - python_init_repositories( - requirements = { - "3.11": "//:requirements_lock_3_11.txt", - }, - ) - tf_vendored(name = "tsl", relpath = "third_party/tsl") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = ["tsl"], - root_module_direct_dev_deps = [], - ) - -tsl = module_extension( - implementation = _tsl_impl, -) diff --git a/third_party/modules/xla/20250317.1-71c67e2/overlay/workspace.bzl b/third_party/modules/xla/20250317.1-71c67e2/overlay/workspace.bzl deleted file mode 100644 index 192e06f8..00000000 --- a/third_party/modules/xla/20250317.1-71c67e2/overlay/workspace.bzl +++ /dev/null @@ -1,60 +0,0 @@ -load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("//third_party/llvm:workspace.bzl", llvm = "repo") -load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") -load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") -load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("//third_party/triton:workspace.bzl", triton = "repo") -load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _xla_workspace_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - pybind11_bazel() - triton() - llvm("llvm-raw") - stablehlo() - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "//third_party/grpc:generate_cc_env_fix.patch", - "//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "//third_party/systemlibs:BUILD.bazel": "bazel/BUILD.bazel", - "//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla_workspace = module_extension( - implementation = _xla_workspace_impl, -) diff --git a/third_party/modules/xla/20250317.1-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/modules/xla/20250317.1-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch deleted file mode 100644 index 8924cf4b..00000000 --- a/third_party/modules/xla/20250317.1-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 -From: Jean-Baptiste Dalido -Date: Mon, 6 Jan 2025 13:33:13 +0100 -Subject: [PATCH] bazel: migration to bazel 8.0.1 - ---- - .bazelversion | 2 +- - third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/.bazelversion b/.bazelversion -index f22d756da3..fa5fce04b3 100644 ---- a/.bazelversion -+++ b/.bazelversion -@@ -1 +1 @@ --7.4.1 -+8.1.1 -\ No newline at end of file -diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl -index d62531152d..71d80a5a99 100644 ---- a/third_party/gpus/cuda_configure.bzl -+++ b/third_party/gpus/cuda_configure.bzl -@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. - load( - "@bazel_tools//tools/cpp:lib_cc_configure.bzl", - "escape_string", -- "get_env_var", - ) - load( - "@bazel_tools//tools/cpp:windows_cc_configure.bzl", -- "find_msvc_tool", - "find_vc_path", - "setup_vc_env_vars", - ) -+load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") -+load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") - load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") - load( - "//third_party/remote_config:common.bzl", --- -2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch deleted file mode 100644 index 58e3874d..00000000 --- a/third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch +++ /dev/null @@ -1,131 +0,0 @@ -From 367df40470c00b9a4f83e3c5bc5553e7b0878351 Mon Sep 17 00:00:00 2001 -From: Hugo Mano -Date: Wed, 5 Feb 2025 19:25:03 +0100 -Subject: [PATCH 1/8] Added FFI handler registration API to the FFI PjRt - -PR: https://github.com/openxla/xla/pull/13420 ---- - xla/pjrt/c/BUILD | 5 ++++ - xla/pjrt/c/pjrt_c_api_ffi_extension.h | 16 ++++++++++++ - xla/pjrt/c/pjrt_c_api_ffi_internal.cc | 35 +++++++++++++++++++++++++-- - 3 files changed, 54 insertions(+), 2 deletions(-) - -diff --git a/xla/pjrt/c/BUILD b/xla/pjrt/c/BUILD -index ad2ed95bce..0e7c35c30f 100644 ---- a/xla/pjrt/c/BUILD -+++ b/xla/pjrt/c/BUILD -@@ -69,7 +69,12 @@ cc_library( - ":pjrt_c_api_wrapper_impl", - "//xla/ffi:execution_context", - "//xla/ffi:type_id_registry", -+ "//xla/ffi:ffi_api", -+ "//xla/ffi/api:c_api", -+ "//xla/ffi/api:ffi", -+ "//xla/service:custom_call_target_registry", - "@com_google_absl//absl/status", -+ "@com_google_absl//absl/strings:str_format", - ], - ) - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -index a33bd4aa9c..3309194538 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -@@ -66,12 +66,28 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); - // Adds a user data to the execute context. - typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); - -+struct PJRT_FFI_Register_Handler_Args { -+ size_t struct_size; -+ const char* target_name; -+ size_t target_name_size; -+ int api_version; // 0 for an untyped call, 1 -- for typed -+ void* handler; -+ const char* platform_name; -+ size_t platform_name_size; -+}; -+PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, handler); -+ -+// Registers an FFI call handler for a specific platform. -+typedef PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args); -+ - typedef struct PJRT_FFI_Extension { - size_t struct_size; - PJRT_Extension_Type type; - PJRT_Extension_Base* next; - PJRT_FFI_TypeID_Register* type_id_register; - PJRT_FFI_UserData_Add* user_data_add; -+ PJRT_FFI_Register_Handler* register_handler; - } PJRT_FFI; - PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Extension, user_data_add); - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -index 0375b39d0b..3527a0756e 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -+++ b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -@@ -13,15 +13,20 @@ See the License for the specific language governing permissions and - limitations under the License. - ==============================================================================*/ - --#include "xla/pjrt/c/pjrt_c_api_ffi_internal.h" -+#include - - #include "absl/status/status.h" -+#include "absl/strings/str_format.h" -+#include "xla/ffi/api/c_api.h" -+#include "xla/ffi/api/ffi.h" - #include "xla/ffi/execution_context.h" --#include "xla/ffi/type_id_registry.h" -+ #include "xla/ffi/type_id_registry.h" -+#include "xla/ffi/ffi_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" - #include "xla/pjrt/c/pjrt_c_api_ffi_extension.h" - #include "xla/pjrt/c/pjrt_c_api_helpers.h" - #include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" -+#include "xla/service/custom_call_target_registry.h" - - namespace pjrt { - -@@ -55,6 +60,31 @@ static PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args) { - return nullptr; - } - -+static PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args) { -+ PJRT_RETURN_IF_ERROR(ActualStructSizeIsGreaterOrEqual( -+ "PJRT_FFI_Register_Handler_Args", -+ PJRT_FFI_Register_Handler_Args_STRUCT_SIZE, args->struct_size)); -+ std::string target_name(args->target_name, args->target_name_size); -+ std::string platform_name(args->platform_name, args->platform_name_size); -+ switch (args->api_version) { -+ case 0: -+ xla::CustomCallTargetRegistry::Global()->Register( -+ target_name, args->handler, platform_name); -+ return nullptr; -+ case 1: -+ xla::ffi::Ffi::RegisterStaticHandler( -+ xla::ffi::GetXlaFfiApi(), target_name, platform_name, -+ reinterpret_cast(args->handler)); -+ return nullptr; -+ default: -+ return new PJRT_Error{absl::UnimplementedError( -+ absl::StrFormat("API version %d not supported for PJRT GPU plugin. " -+ "Supported versions are 0 and 1.", -+ args->api_version))}; -+ } -+} -+ - PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - return { - /*struct_size=*/PJRT_FFI_Extension_STRUCT_SIZE, -@@ -62,6 +92,7 @@ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - /*next=*/next, - /*type_id_register=*/PJRT_FFI_TypeID_Register, - /*user_data_add=*/PJRT_FFI_UserData_Add, -+ /*register_handler=*/PJRT_FFI_Register_Handler, - }; - } - --- -2.43.0 diff --git a/third_party/modules/xla/20250317.1-71c67e2/source.json b/third_party/modules/xla/20250317.1-71c67e2/source.json deleted file mode 100644 index 5577d247..00000000 --- a/third_party/modules/xla/20250317.1-71c67e2/source.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "strip_prefix": "xla-71c67e2a4f40267115a0d4ea7c36748bbe7e750e", - "url": "https://github.com/openxla/xla/archive/71c67e2a4f40267115a0d4ea7c36748bbe7e750e.tar.gz", - "integrity": "sha256-j6D1MC7+WsbZ+Ve3hPmDlCzZX1yV6RIP2BWDgQcbcYc=", - "overlay": { - "tsl.bzl": "", - "workspace.bzl": "", - "MODULE.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0001-bazel-migration-to-bazel-8.1.1.patch": "", - "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "" - } -} diff --git a/third_party/modules/xla/20250317.2-71c67e2/MODULE.bazel b/third_party/modules/xla/20250317.2-71c67e2/MODULE.bazel deleted file mode 100644 index 042d899c..00000000 --- a/third_party/modules/xla/20250317.2-71c67e2/MODULE.bazel +++ /dev/null @@ -1,37 +0,0 @@ -module( - name = "xla", - version = "20250317.2-71c67e2", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.17") -bazel_dep(name = "rules_apple", version = "3.17.0", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.29.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") - -tsl = use_extension("//:tsl.bzl", "tsl") -use_repo(tsl, "tsl", "python_version_repo") - -xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") -use_repo( - xla_workspace, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", - "llvm-raw", - "stablehlo", -) - -llvm_configure = use_extension("//:llvm.bzl", "llvm_configure") -use_repo(llvm_configure, "llvm-project") diff --git a/third_party/modules/xla/20250317.2-71c67e2/overlay/MODULE.bazel b/third_party/modules/xla/20250317.2-71c67e2/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20250317.2-71c67e2/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250317.2-71c67e2/overlay/llvm.bzl b/third_party/modules/xla/20250317.2-71c67e2/overlay/llvm.bzl deleted file mode 100644 index 57e77361..00000000 --- a/third_party/modules/xla/20250317.2-71c67e2/overlay/llvm.bzl +++ /dev/null @@ -1,13 +0,0 @@ -load("@llvm-raw//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") - -def _llvm_configure_impl(mctx): - _llvm_configure(name = "llvm-project") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -llvm_configure = module_extension( - implementation = _llvm_configure_impl, -) diff --git a/third_party/modules/xla/20250317.2-71c67e2/overlay/tsl.bzl b/third_party/modules/xla/20250317.2-71c67e2/overlay/tsl.bzl deleted file mode 100644 index dc25c9d6..00000000 --- a/third_party/modules/xla/20250317.2-71c67e2/overlay/tsl.bzl +++ /dev/null @@ -1,19 +0,0 @@ -load("//third_party:repo.bzl", "tf_vendored") -load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") - -def _tsl_impl(mctx): - python_init_repositories( - requirements = { - "3.11": "//:requirements_lock_3_11.txt", - }, - ) - tf_vendored(name = "tsl", relpath = "third_party/tsl") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = ["tsl"], - root_module_direct_dev_deps = [], - ) - -tsl = module_extension( - implementation = _tsl_impl, -) diff --git a/third_party/modules/xla/20250317.2-71c67e2/overlay/workspace.bzl b/third_party/modules/xla/20250317.2-71c67e2/overlay/workspace.bzl deleted file mode 100644 index 192e06f8..00000000 --- a/third_party/modules/xla/20250317.2-71c67e2/overlay/workspace.bzl +++ /dev/null @@ -1,60 +0,0 @@ -load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("//third_party/llvm:workspace.bzl", llvm = "repo") -load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") -load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") -load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("//third_party/triton:workspace.bzl", triton = "repo") -load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _xla_workspace_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - pybind11_bazel() - triton() - llvm("llvm-raw") - stablehlo() - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "//third_party/grpc:generate_cc_env_fix.patch", - "//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "//third_party/systemlibs:BUILD.bazel": "bazel/BUILD.bazel", - "//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla_workspace = module_extension( - implementation = _xla_workspace_impl, -) diff --git a/third_party/modules/xla/20250317.2-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/modules/xla/20250317.2-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch deleted file mode 100644 index 8924cf4b..00000000 --- a/third_party/modules/xla/20250317.2-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 -From: Jean-Baptiste Dalido -Date: Mon, 6 Jan 2025 13:33:13 +0100 -Subject: [PATCH] bazel: migration to bazel 8.0.1 - ---- - .bazelversion | 2 +- - third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/.bazelversion b/.bazelversion -index f22d756da3..fa5fce04b3 100644 ---- a/.bazelversion -+++ b/.bazelversion -@@ -1 +1 @@ --7.4.1 -+8.1.1 -\ No newline at end of file -diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl -index d62531152d..71d80a5a99 100644 ---- a/third_party/gpus/cuda_configure.bzl -+++ b/third_party/gpus/cuda_configure.bzl -@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. - load( - "@bazel_tools//tools/cpp:lib_cc_configure.bzl", - "escape_string", -- "get_env_var", - ) - load( - "@bazel_tools//tools/cpp:windows_cc_configure.bzl", -- "find_msvc_tool", - "find_vc_path", - "setup_vc_env_vars", - ) -+load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") -+load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") - load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") - load( - "//third_party/remote_config:common.bzl", --- -2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250317.2-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250317.2-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch deleted file mode 100644 index 58e3874d..00000000 --- a/third_party/modules/xla/20250317.2-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch +++ /dev/null @@ -1,131 +0,0 @@ -From 367df40470c00b9a4f83e3c5bc5553e7b0878351 Mon Sep 17 00:00:00 2001 -From: Hugo Mano -Date: Wed, 5 Feb 2025 19:25:03 +0100 -Subject: [PATCH 1/8] Added FFI handler registration API to the FFI PjRt - -PR: https://github.com/openxla/xla/pull/13420 ---- - xla/pjrt/c/BUILD | 5 ++++ - xla/pjrt/c/pjrt_c_api_ffi_extension.h | 16 ++++++++++++ - xla/pjrt/c/pjrt_c_api_ffi_internal.cc | 35 +++++++++++++++++++++++++-- - 3 files changed, 54 insertions(+), 2 deletions(-) - -diff --git a/xla/pjrt/c/BUILD b/xla/pjrt/c/BUILD -index ad2ed95bce..0e7c35c30f 100644 ---- a/xla/pjrt/c/BUILD -+++ b/xla/pjrt/c/BUILD -@@ -69,7 +69,12 @@ cc_library( - ":pjrt_c_api_wrapper_impl", - "//xla/ffi:execution_context", - "//xla/ffi:type_id_registry", -+ "//xla/ffi:ffi_api", -+ "//xla/ffi/api:c_api", -+ "//xla/ffi/api:ffi", -+ "//xla/service:custom_call_target_registry", - "@com_google_absl//absl/status", -+ "@com_google_absl//absl/strings:str_format", - ], - ) - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -index a33bd4aa9c..3309194538 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -@@ -66,12 +66,28 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); - // Adds a user data to the execute context. - typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); - -+struct PJRT_FFI_Register_Handler_Args { -+ size_t struct_size; -+ const char* target_name; -+ size_t target_name_size; -+ int api_version; // 0 for an untyped call, 1 -- for typed -+ void* handler; -+ const char* platform_name; -+ size_t platform_name_size; -+}; -+PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, handler); -+ -+// Registers an FFI call handler for a specific platform. -+typedef PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args); -+ - typedef struct PJRT_FFI_Extension { - size_t struct_size; - PJRT_Extension_Type type; - PJRT_Extension_Base* next; - PJRT_FFI_TypeID_Register* type_id_register; - PJRT_FFI_UserData_Add* user_data_add; -+ PJRT_FFI_Register_Handler* register_handler; - } PJRT_FFI; - PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Extension, user_data_add); - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -index 0375b39d0b..3527a0756e 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -+++ b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -@@ -13,15 +13,20 @@ See the License for the specific language governing permissions and - limitations under the License. - ==============================================================================*/ - --#include "xla/pjrt/c/pjrt_c_api_ffi_internal.h" -+#include - - #include "absl/status/status.h" -+#include "absl/strings/str_format.h" -+#include "xla/ffi/api/c_api.h" -+#include "xla/ffi/api/ffi.h" - #include "xla/ffi/execution_context.h" --#include "xla/ffi/type_id_registry.h" -+ #include "xla/ffi/type_id_registry.h" -+#include "xla/ffi/ffi_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" - #include "xla/pjrt/c/pjrt_c_api_ffi_extension.h" - #include "xla/pjrt/c/pjrt_c_api_helpers.h" - #include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" -+#include "xla/service/custom_call_target_registry.h" - - namespace pjrt { - -@@ -55,6 +60,31 @@ static PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args) { - return nullptr; - } - -+static PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args) { -+ PJRT_RETURN_IF_ERROR(ActualStructSizeIsGreaterOrEqual( -+ "PJRT_FFI_Register_Handler_Args", -+ PJRT_FFI_Register_Handler_Args_STRUCT_SIZE, args->struct_size)); -+ std::string target_name(args->target_name, args->target_name_size); -+ std::string platform_name(args->platform_name, args->platform_name_size); -+ switch (args->api_version) { -+ case 0: -+ xla::CustomCallTargetRegistry::Global()->Register( -+ target_name, args->handler, platform_name); -+ return nullptr; -+ case 1: -+ xla::ffi::Ffi::RegisterStaticHandler( -+ xla::ffi::GetXlaFfiApi(), target_name, platform_name, -+ reinterpret_cast(args->handler)); -+ return nullptr; -+ default: -+ return new PJRT_Error{absl::UnimplementedError( -+ absl::StrFormat("API version %d not supported for PJRT GPU plugin. " -+ "Supported versions are 0 and 1.", -+ args->api_version))}; -+ } -+} -+ - PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - return { - /*struct_size=*/PJRT_FFI_Extension_STRUCT_SIZE, -@@ -62,6 +92,7 @@ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - /*next=*/next, - /*type_id_register=*/PJRT_FFI_TypeID_Register, - /*user_data_add=*/PJRT_FFI_UserData_Add, -+ /*register_handler=*/PJRT_FFI_Register_Handler, - }; - } - --- -2.43.0 diff --git a/third_party/modules/xla/20250317.2-71c67e2/source.json b/third_party/modules/xla/20250317.2-71c67e2/source.json deleted file mode 100644 index 9d579abc..00000000 --- a/third_party/modules/xla/20250317.2-71c67e2/source.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "strip_prefix": "xla-71c67e2a4f40267115a0d4ea7c36748bbe7e750e", - "url": "https://github.com/openxla/xla/archive/71c67e2a4f40267115a0d4ea7c36748bbe7e750e.tar.gz", - "integrity": "sha256-j6D1MC7+WsbZ+Ve3hPmDlCzZX1yV6RIP2BWDgQcbcYc=", - "overlay": { - "llvm.bzl": "", - "MODULE.bazel": "", - "tsl.bzl": "", - "workspace.bzl": "" - }, - "patch_strip": 1, - "patches": { - "0001-bazel-migration-to-bazel-8.1.1.patch": "", - "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "" - } -} diff --git a/third_party/modules/xla/20250527.0-cb67f2f/MODULE.bazel b/third_party/modules/xla/20250527.0-cb67f2f/MODULE.bazel deleted file mode 100644 index 74c9eaad..00000000 --- a/third_party/modules/xla/20250527.0-cb67f2f/MODULE.bazel +++ /dev/null @@ -1,52 +0,0 @@ -module( - name = "xla", - version = "20250527.0-cb67f2f", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.17") -bazel_dep(name = "rules_apple", version = "3.22.0", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.39.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") -bazel_dep(name = "rules_shell", version = "0.4.1") -bazel_dep(name = "bazel_features", version = "1.25.0", repo_name = "proto_bazel_features") - -workspace_private = use_extension("//:workspace_private.bzl", "workspace_private") -use_repo( - workspace_private, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", - "python_version_repo", - "tsl", -) - -workspace_public = use_extension("//:xla.bzl", "xla") -use_repo( - workspace_public, - "llvm-raw", - "stablehlo", - "triton", -) - -llvm = use_extension("//:llvm.bzl", "llvm") -llvm.configure( - targets = [ - "AArch64", - "AMDGPU", - "NVPTX", - "X86", - ], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/xla/20250527.0-cb67f2f/overlay/MODULE.bazel b/third_party/modules/xla/20250527.0-cb67f2f/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20250527.0-cb67f2f/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250527.0-cb67f2f/overlay/llvm.bzl b/third_party/modules/xla/20250527.0-cb67f2f/overlay/llvm.bzl deleted file mode 100644 index b4a2fe4f..00000000 --- a/third_party/modules/xla/20250527.0-cb67f2f/overlay/llvm.bzl +++ /dev/null @@ -1,30 +0,0 @@ -load("@llvm-raw//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") - -def _llvm_impl(mctx): - _targets = {} - for mod in mctx.modules: - for conf in mod.tags.configure: - for target in conf.targets: - _targets[target] = True - _llvm_configure( - name = "llvm-project", - targets = _targets.keys(), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -llvm = module_extension( - implementation = _llvm_impl, - tag_classes = { - "configure": tag_class( - attrs = { - "targets": attr.string_list( - default = [], - ), - }, - ), - }, -) diff --git a/third_party/modules/xla/20250527.0-cb67f2f/overlay/workspace_private.bzl b/third_party/modules/xla/20250527.0-cb67f2f/overlay/workspace_private.bzl deleted file mode 100644 index b234d122..00000000 --- a/third_party/modules/xla/20250527.0-cb67f2f/overlay/workspace_private.bzl +++ /dev/null @@ -1,71 +0,0 @@ -load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("//third_party/llvm:workspace.bzl", llvm = "repo") -load("//third_party/py:python_repo.bzl", "python_repository") -load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") -load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") -load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("//third_party/triton:workspace.bzl", triton = "repo") -load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("//third_party:repo.bzl", "tf_vendored") -load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _workspace_private_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_vendored(name = "tsl", relpath = "third_party/tsl") - pybind11_bazel() - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "//third_party/grpc:generate_cc_env_fix.patch", - "//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "//third_party/systemlibs:BUILD.bazel": "bazel/BUILD.bazel", - "//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - python_repository( - name = "python_version_repo", - requirements_versions = ["3.11"], - requirements_locks = ["//:requirements_lock_3_11.txt"], - local_wheel_workspaces = [], - local_wheel_dist_folder = None, - default_python_version = None, - local_wheel_inclusion_list = ["*"], - local_wheel_exclusion_list = [], - ) - - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -workspace_private = module_extension( - implementation = _workspace_private_impl, -) diff --git a/third_party/modules/xla/20250527.0-cb67f2f/overlay/xla.bzl b/third_party/modules/xla/20250527.0-cb67f2f/overlay/xla.bzl deleted file mode 100644 index f14bf2a9..00000000 --- a/third_party/modules/xla/20250527.0-cb67f2f/overlay/xla.bzl +++ /dev/null @@ -1,17 +0,0 @@ -load("//third_party/llvm:workspace.bzl", llvm = "repo") -load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") -load("//third_party/triton:workspace.bzl", triton = "repo") - -def _xla_impl(mctx): - triton() - llvm("llvm-raw") - stablehlo() - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla = module_extension( - implementation = _xla_impl, -) diff --git a/third_party/modules/xla/20250527.0-cb67f2f/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/modules/xla/20250527.0-cb67f2f/patches/0001-bazel-migration-to-bazel-8.1.1.patch deleted file mode 100644 index 8924cf4b..00000000 --- a/third_party/modules/xla/20250527.0-cb67f2f/patches/0001-bazel-migration-to-bazel-8.1.1.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 -From: Jean-Baptiste Dalido -Date: Mon, 6 Jan 2025 13:33:13 +0100 -Subject: [PATCH] bazel: migration to bazel 8.0.1 - ---- - .bazelversion | 2 +- - third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/.bazelversion b/.bazelversion -index f22d756da3..fa5fce04b3 100644 ---- a/.bazelversion -+++ b/.bazelversion -@@ -1 +1 @@ --7.4.1 -+8.1.1 -\ No newline at end of file -diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl -index d62531152d..71d80a5a99 100644 ---- a/third_party/gpus/cuda_configure.bzl -+++ b/third_party/gpus/cuda_configure.bzl -@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. - load( - "@bazel_tools//tools/cpp:lib_cc_configure.bzl", - "escape_string", -- "get_env_var", - ) - load( - "@bazel_tools//tools/cpp:windows_cc_configure.bzl", -- "find_msvc_tool", - "find_vc_path", - "setup_vc_env_vars", - ) -+load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") -+load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") - load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") - load( - "//third_party/remote_config:common.bzl", --- -2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250527.0-cb67f2f/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250527.0-cb67f2f/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch deleted file mode 100644 index 938ef401..00000000 --- a/third_party/modules/xla/20250527.0-cb67f2f/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch +++ /dev/null @@ -1,135 +0,0 @@ -From 2ae9bb9d24b569c2c6bfab3c54b428103614944d Mon Sep 17 00:00:00 2001 -From: Hugo Mano -Date: Tue, 27 May 2025 11:48:17 +0200 -Subject: [PATCH 1/8] Added FFI handler registration API to the FFI PjRt - -PR: https://github.com/openxla/xla/pull/13420 ---- - xla/pjrt/c/BUILD | 5 +++++ - xla/pjrt/c/pjrt_c_api_ffi_extension.h | 21 ++++++++++++++++++ - xla/pjrt/c/pjrt_c_api_ffi_internal.cc | 32 ++++++++++++++++++++++++++- - 3 files changed, 57 insertions(+), 1 deletion(-) - -diff --git a/xla/pjrt/c/BUILD b/xla/pjrt/c/BUILD -index 79f18fa0bc..0f33dd8a6e 100644 ---- a/xla/pjrt/c/BUILD -+++ b/xla/pjrt/c/BUILD -@@ -69,8 +69,13 @@ cc_library( - ":pjrt_c_api_wrapper_impl", - "//xla/ffi:execution_context", - "//xla/ffi:type_id_registry", -+ "//xla/ffi:ffi_api", -+ "//xla/ffi/api:c_api", -+ "//xla/ffi/api:ffi", -+ "//xla/service:custom_call_target_registry", - "@com_google_absl//absl/status", - "@com_google_absl//absl/strings:string_view", -+ "@com_google_absl//absl/strings:str_format", - ], - ) - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -index 995a2c7e50..b8f10bc2f7 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -@@ -69,10 +69,31 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); - // Adds a user data to the execute context. - typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); - -+typedef enum PJRT_FFI_Handler_TraitsBits { -+ PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE = 1u << 0, -+} PJRT_FFI_Handler_TraitsBits; -+ -+struct PJRT_FFI_Register_Handler_Args { -+ size_t struct_size; -+ const char* target_name; -+ size_t target_name_size; -+ int api_version; // 0 for an untyped call, 1 -- for typed -+ void* handler; -+ const char* platform_name; -+ size_t platform_name_size; -+ PJRT_FFI_Handler_TraitsBits traits; -+}; -+PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, traits); -+ -+// Registers an FFI call handler for a specific platform. -+typedef PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args); -+ - typedef struct PJRT_FFI_Extension { - PJRT_Extension_Base base; - PJRT_FFI_TypeID_Register* type_id_register; - PJRT_FFI_UserData_Add* user_data_add; -+ PJRT_FFI_Register_Handler* register_handler; - } PJRT_FFI; - PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Extension, user_data_add); - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -index 5fa88eab33..763270331b 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -+++ b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -@@ -13,16 +13,20 @@ See the License for the specific language governing permissions and - limitations under the License. - ==============================================================================*/ - --#include "xla/pjrt/c/pjrt_c_api_ffi_internal.h" -+#include - - #include "absl/status/status.h" -+#include "absl/strings/str_format.h" - #include "absl/strings/string_view.h" -+#include "xla/ffi/api/c_api.h" - #include "xla/ffi/execution_context.h" - #include "xla/ffi/type_id_registry.h" -+#include "xla/ffi/ffi_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" - #include "xla/pjrt/c/pjrt_c_api_ffi_extension.h" - #include "xla/pjrt/c/pjrt_c_api_helpers.h" - #include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" -+#include "xla/service/custom_call_target_registry.h" - - namespace pjrt { - -@@ -68,6 +72,31 @@ static PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args) { - return nullptr; - } - -+static PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args) { -+ PJRT_RETURN_IF_ERROR(ActualStructSizeIsGreaterOrEqual( -+ "PJRT_FFI_Register_Handler_Args", -+ PJRT_FFI_Register_Handler_Args_STRUCT_SIZE, args->struct_size)); -+ std::string target_name(args->target_name, args->target_name_size); -+ std::string platform_name(args->platform_name, args->platform_name_size); -+ switch (args->api_version) { -+ case 0: -+ xla::CustomCallTargetRegistry::Global()->Register( -+ target_name, args->handler, platform_name); -+ return nullptr; -+ case 1: -+ xla::ffi::Ffi::RegisterStaticHandler( -+ xla::ffi::GetXlaFfiApi(), target_name, platform_name, -+ reinterpret_cast(args->handler)); -+ return nullptr; -+ default: -+ return new PJRT_Error{absl::UnimplementedError( -+ absl::StrFormat("API version %d not supported for PJRT GPU plugin. " -+ "Supported versions are 0 and 1.", -+ args->api_version))}; -+ } -+} -+ - PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - return { - PJRT_Extension_Base{ -@@ -77,6 +106,7 @@ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - }, - /*type_id_register=*/PJRT_FFI_TypeID_Register, - /*user_data_add=*/PJRT_FFI_UserData_Add, -+ /*register_handler=*/PJRT_FFI_Register_Handler, - }; - } - --- -2.39.5 (Apple Git-154) - diff --git a/third_party/modules/xla/20250527.0-cb67f2f/patches/0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch b/third_party/modules/xla/20250527.0-cb67f2f/patches/0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch deleted file mode 100644 index 4173db86..00000000 --- a/third_party/modules/xla/20250527.0-cb67f2f/patches/0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch +++ /dev/null @@ -1,186 +0,0 @@ -From 2a62bc5df9774810313142eb0c9390aab3cd18f8 Mon Sep 17 00:00:00 2001 -From: Hugo Mano -Date: Thu, 29 May 2025 08:02:32 +0200 -Subject: [PATCH] Revert "Add optional allowOtherDialects field to - stablehlo.serialize_portable_artifact." - -Commit: https://github.com/openxla/xla/commit/e7137a383809a24875a95237b1d1f6485acdf710 - -Issue: C does not support default arguments with ZigTranslateC ---- - third_party/stablehlo/temporary.patch | 151 ++++---------------------- - 1 file changed, 21 insertions(+), 130 deletions(-) - -diff --git a/third_party/stablehlo/temporary.patch b/third_party/stablehlo/temporary.patch -index 6e1fd159f9..d17c141b18 100755 ---- a/third_party/stablehlo/temporary.patch -+++ b/third_party/stablehlo/temporary.patch -@@ -1,3 +1,23 @@ -+diff --ruN a/stablehlo/stablehlo/dialect/StablehloOps.cpp b/stablehlo/stablehlo/dialect/StablehloOps.cpp -+--- stablehlo/stablehlo/dialect/StablehloOps.cpp -++++ stablehlo/stablehlo/dialect/StablehloOps.cpp -+@@ -511,12 +511,10 @@ -+ void CustomCallOp::getEffects( -+ SmallVectorImpl>& -+ effects) { -+- // Note: `has_side_effect` "defaults" to `false` but isn't required to exist. -+- // This semantic contradiction means, in practical terms, that the attribute -+- // won't exist by default but should be *treated* as `false` if missing. -+- // `getHasSideEffect()` abstracts this nuance away and returns `false` by -+- // default, whereas `getHasSideEffectAttr()` may return a null attribute. -+- if (!getHasSideEffect()) return; -++ // CustomCall has "all possible effects" unless the has_side_effect is present -++ // and set to false. -++ auto hasSideEffect = getHasSideEffectAttr(); -++ if (hasSideEffect && !hasSideEffect.getValue()) return; -+ effects.emplace_back(MemoryEffects::Allocate::get()); -+ effects.emplace_back(MemoryEffects::Free::get()); -+ effects.emplace_back(MemoryEffects::Write::get()); - diff --ruN a/stablehlo/stablehlo/dialect/StablehloOps.h b/stablehlo/stablehlo/dialect/StablehloOps.h - --- stablehlo/stablehlo/dialect/StablehloOps.h - +++ stablehlo/stablehlo/dialect/StablehloOps.h -@@ -82,135 +102,6 @@ diff --ruN a/stablehlo/stablehlo/dialect/StablehloOps.td b/stablehlo/stablehlo/d - ]> { - let summary = "Recv operation"; - let description = [{ --diff --ruN a/stablehlo/stablehlo/dialect/TypeInference.cpp b/stablehlo/stablehlo/dialect/TypeInference.cpp ----- stablehlo/stablehlo/dialect/TypeInference.cpp --+++ stablehlo/stablehlo/dialect/TypeInference.cpp --@@ -879,7 +879,8 @@ -- -- auto replicaIds = replicaGroups.getValues(); -- --- llvm::SmallSet replicaIdsSeen; --+ // Large programs can have many replicas, use a set with efficient lookup. --+ llvm::DenseSet replicaIdsSeen; -- for (int64_t replicaId : replicaIds) { -- // Replica groups are stored in a 2D tensor. If the op supports non-uniform -- // groups, null replica IDs are stored as -1. --@@ -1841,6 +1842,7 @@ -- /*allGroupsMustHaveSameSize=*/true, -- /*useGlobalDeviceIds=*/false, splitCount))) -- return failure(); --+ -- for (const Value& operand : operands) { -- auto operandType = cast(operand.getType()); -- --@@ -3562,6 +3564,19 @@ -- DenseIntElementsAttr replicaGroups, -- int64_t channelId, bool useGlobalDeviceIds, -- ValueRange results) { --+ // all_gather_i3, all_gather_c2, all_gather_c4 --+ if (failed(verifyReplicaGroups(location, replicaGroups, --+ /*allGroupsMustHaveSameSize=*/true, --+ useGlobalDeviceIds, --+ /*expectedGroupSize=*/std::nullopt))) --+ return failure(); --+ --+ // all_gather_c5 --+ if (useGlobalDeviceIds && channelId < 0) --+ return emitOptionalError( --+ location, --+ "channel_id cannot be negative when useGlobalDeviceIds is set"); --+ -- for (const auto& [operand, result] : llvm::zip(operands, results)) { -- auto operandType = cast(operand.getType()); -- auto resultType = cast(result.getType()); --@@ -3576,19 +3591,6 @@ -- return emitOptionalError( -- location, -- "dimension size of operand at 'all_gather_dim' cannot be zero"); --- --- // all_gather_i3, all_gather_c2, all_gather_c4 --- if (failed(verifyReplicaGroups(location, replicaGroups, --- /*allGroupsMustHaveSameSize=*/true, --- useGlobalDeviceIds, --- /*expectedGroupSize=*/std::nullopt))) --- return failure(); --- --- // all_gather_c5 --- if (useGlobalDeviceIds && channelId < 0) --- return emitOptionalError( --- location, --- "channel_id cannot be negative when useGlobalDeviceIds is set"); -- -- // all_gather_c6 -- if (resultType.getRank() != operandType.getRank()) --@@ -3788,7 +3790,7 @@ -- "but instead it is of rank ", replicaGroupType.getRank()); -- -- auto replicaIds = replicaGroups.getValues(); --- llvm::SmallSet replicaIdsSeen; --+ llvm::DenseSet replicaIdsSeen; -- for (int64_t replicaId : replicaIds) { -- // collective_broadcast_c2 -- // We only check that is is not negative, as it is impossible --diff --ruN a/stablehlo/stablehlo/integrations/c/StablehloDialectApi.cpp b/stablehlo/stablehlo/integrations/c/StablehloDialectApi.cpp ----- stablehlo/stablehlo/integrations/c/StablehloDialectApi.cpp --+++ stablehlo/stablehlo/integrations/c/StablehloDialectApi.cpp --@@ -78,10 +78,11 @@ -- -- MlirLogicalResult stablehloSerializePortableArtifactFromModule( -- MlirModule moduleStr, MlirStringRef targetVersion, --- MlirStringCallback callback, void *userData) { --+ MlirStringCallback callback, void *userData, bool allowOtherDialects) { -- mlir::detail::CallbackOstream stream(callback, userData); -- if (failed(mlir::stablehlo::serializePortableArtifact( --- unwrap(moduleStr), unwrap(targetVersion), stream))) --+ unwrap(moduleStr), unwrap(targetVersion), stream, --+ allowOtherDialects))) -- return mlirLogicalResultFailure(); -- return mlirLogicalResultSuccess(); -- } --diff --ruN a/stablehlo/stablehlo/integrations/c/StablehloDialectApi.h b/stablehlo/stablehlo/integrations/c/StablehloDialectApi.h ----- stablehlo/stablehlo/integrations/c/StablehloDialectApi.h --+++ stablehlo/stablehlo/integrations/c/StablehloDialectApi.h --@@ -92,7 +92,8 @@ -- stablehloSerializePortableArtifactFromModule(MlirModule moduleStr, -- MlirStringRef targetVersion, -- MlirStringCallback callback, --- void* userData); --+ void* userData, --+ bool allowOtherDialects = false); -- -- // Read a StableHLO program from a portable artifact, returning the module as -- // MLIR bytecode. Note, this bytecode returned is not a portable artifact, --diff --ruN a/stablehlo/stablehlo/integrations/python/StablehloApi.cpp b/stablehlo/stablehlo/integrations/python/StablehloApi.cpp ----- stablehlo/stablehlo/integrations/python/StablehloApi.cpp --+++ stablehlo/stablehlo/integrations/python/StablehloApi.cpp --@@ -102,20 +102,22 @@ -- // -- m.def( -- "serialize_portable_artifact", --- [](MlirModule module, std::string_view target) -> nb::bytes { --+ [](MlirModule module, std::string_view target, --+ bool allowOtherDialects) -> nb::bytes { -- StringWriterHelper accumulator; -- if (mlirLogicalResultIsFailure( -- stablehloSerializePortableArtifactFromModule( -- module, toMlirStringRef(target), -- accumulator.getMlirStringCallback(), --- accumulator.getUserData()))) { --+ accumulator.getUserData(), allowOtherDialects))) { -- throw nb::value_error("failed to serialize module"); -- } -- -- std::string serialized = accumulator.toString(); -- return nb::bytes(serialized.data(), serialized.size()); -- }, --- nb::arg("module"), nb::arg("target")); --+ nb::arg("module"), nb::arg("target"), --+ nb::arg("allow_other_dialects") = false); -- -- m.def( -- "deserialize_portable_artifact", - diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_convert_to_signless.mlir b/stablehlo/stablehlo/tests/transforms/stablehlo_convert_to_signless.mlir - --- stablehlo/stablehlo/tests/transforms/stablehlo_convert_to_signless.mlir - +++ stablehlo/stablehlo/tests/transforms/stablehlo_convert_to_signless.mlir -@@ -223,4 +114,4 @@ diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_convert_to_signless. - %7 = builtin.unrealized_conversion_cast %6 : memref to memref - func.return %7 : memref - } -- -+ --- -2.39.5 (Apple Git-154) - diff --git a/third_party/modules/xla/20250527.0-cb67f2f/source.json b/third_party/modules/xla/20250527.0-cb67f2f/source.json deleted file mode 100644 index 86bdd83a..00000000 --- a/third_party/modules/xla/20250527.0-cb67f2f/source.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "strip_prefix": "xla-cb67f2f7ce4787f63f5fc80dc5c30cd3dee8f4e3", - "url": "https://github.com/openxla/xla/archive/cb67f2f7ce4787f63f5fc80dc5c30cd3dee8f4e3.tar.gz", - "integrity": "sha256-SDAAOY6cjcCQ5e1JMob5G4+BYHk7spDb5zZEDrVeA4I=", - "overlay": { - "llvm.bzl": "", - "MODULE.bazel": "", - "workspace_private.bzl": "", - "xla.bzl": "" - }, - "patch_strip": 1, - "patches": { - "0001-bazel-migration-to-bazel-8.1.1.patch": "", - "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "", - "0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch": "" - } -} diff --git a/third_party/modules/xla/20250612.0-6e48cbb/MODULE.bazel b/third_party/modules/xla/20250612.0-6e48cbb/MODULE.bazel deleted file mode 100644 index 46eb7d18..00000000 --- a/third_party/modules/xla/20250612.0-6e48cbb/MODULE.bazel +++ /dev/null @@ -1,52 +0,0 @@ -module( - name = "xla", - version = "20250612.0-6e48cbb", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.17") -bazel_dep(name = "rules_apple", version = "3.22.0", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.39.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") -bazel_dep(name = "rules_shell", version = "0.4.1") -bazel_dep(name = "bazel_features", version = "1.25.0", repo_name = "proto_bazel_features") - -workspace_private = use_extension("//:workspace_private.bzl", "workspace_private") -use_repo( - workspace_private, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", - "python_version_repo", - "tsl", -) - -workspace_public = use_extension("//:xla.bzl", "xla") -use_repo( - workspace_public, - "llvm-raw", - "stablehlo", - "triton", -) - -llvm = use_extension("//:llvm.bzl", "llvm") -llvm.configure( - targets = [ - "AArch64", - "AMDGPU", - "NVPTX", - "X86", - ], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/xla/20250612.0-6e48cbb/overlay/MODULE.bazel b/third_party/modules/xla/20250612.0-6e48cbb/overlay/MODULE.bazel deleted file mode 100644 index 74c9eaad..00000000 --- a/third_party/modules/xla/20250612.0-6e48cbb/overlay/MODULE.bazel +++ /dev/null @@ -1,52 +0,0 @@ -module( - name = "xla", - version = "20250527.0-cb67f2f", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.17") -bazel_dep(name = "rules_apple", version = "3.22.0", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.39.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") -bazel_dep(name = "rules_shell", version = "0.4.1") -bazel_dep(name = "bazel_features", version = "1.25.0", repo_name = "proto_bazel_features") - -workspace_private = use_extension("//:workspace_private.bzl", "workspace_private") -use_repo( - workspace_private, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", - "python_version_repo", - "tsl", -) - -workspace_public = use_extension("//:xla.bzl", "xla") -use_repo( - workspace_public, - "llvm-raw", - "stablehlo", - "triton", -) - -llvm = use_extension("//:llvm.bzl", "llvm") -llvm.configure( - targets = [ - "AArch64", - "AMDGPU", - "NVPTX", - "X86", - ], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/xla/20250612.0-6e48cbb/overlay/llvm.bzl b/third_party/modules/xla/20250612.0-6e48cbb/overlay/llvm.bzl deleted file mode 100644 index b4a2fe4f..00000000 --- a/third_party/modules/xla/20250612.0-6e48cbb/overlay/llvm.bzl +++ /dev/null @@ -1,30 +0,0 @@ -load("@llvm-raw//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") - -def _llvm_impl(mctx): - _targets = {} - for mod in mctx.modules: - for conf in mod.tags.configure: - for target in conf.targets: - _targets[target] = True - _llvm_configure( - name = "llvm-project", - targets = _targets.keys(), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -llvm = module_extension( - implementation = _llvm_impl, - tag_classes = { - "configure": tag_class( - attrs = { - "targets": attr.string_list( - default = [], - ), - }, - ), - }, -) diff --git a/third_party/modules/xla/20250612.0-6e48cbb/overlay/workspace_private.bzl b/third_party/modules/xla/20250612.0-6e48cbb/overlay/workspace_private.bzl deleted file mode 100644 index b234d122..00000000 --- a/third_party/modules/xla/20250612.0-6e48cbb/overlay/workspace_private.bzl +++ /dev/null @@ -1,71 +0,0 @@ -load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("//third_party/llvm:workspace.bzl", llvm = "repo") -load("//third_party/py:python_repo.bzl", "python_repository") -load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") -load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") -load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("//third_party/triton:workspace.bzl", triton = "repo") -load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("//third_party:repo.bzl", "tf_vendored") -load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _workspace_private_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_vendored(name = "tsl", relpath = "third_party/tsl") - pybind11_bazel() - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "//third_party/grpc:generate_cc_env_fix.patch", - "//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "//third_party/systemlibs:BUILD.bazel": "bazel/BUILD.bazel", - "//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - python_repository( - name = "python_version_repo", - requirements_versions = ["3.11"], - requirements_locks = ["//:requirements_lock_3_11.txt"], - local_wheel_workspaces = [], - local_wheel_dist_folder = None, - default_python_version = None, - local_wheel_inclusion_list = ["*"], - local_wheel_exclusion_list = [], - ) - - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -workspace_private = module_extension( - implementation = _workspace_private_impl, -) diff --git a/third_party/modules/xla/20250612.0-6e48cbb/overlay/xla.bzl b/third_party/modules/xla/20250612.0-6e48cbb/overlay/xla.bzl deleted file mode 100644 index f14bf2a9..00000000 --- a/third_party/modules/xla/20250612.0-6e48cbb/overlay/xla.bzl +++ /dev/null @@ -1,17 +0,0 @@ -load("//third_party/llvm:workspace.bzl", llvm = "repo") -load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") -load("//third_party/triton:workspace.bzl", triton = "repo") - -def _xla_impl(mctx): - triton() - llvm("llvm-raw") - stablehlo() - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla = module_extension( - implementation = _xla_impl, -) diff --git a/third_party/modules/xla/20250612.0-6e48cbb/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/modules/xla/20250612.0-6e48cbb/patches/0001-bazel-migration-to-bazel-8.1.1.patch deleted file mode 100644 index 8924cf4b..00000000 --- a/third_party/modules/xla/20250612.0-6e48cbb/patches/0001-bazel-migration-to-bazel-8.1.1.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 -From: Jean-Baptiste Dalido -Date: Mon, 6 Jan 2025 13:33:13 +0100 -Subject: [PATCH] bazel: migration to bazel 8.0.1 - ---- - .bazelversion | 2 +- - third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/.bazelversion b/.bazelversion -index f22d756da3..fa5fce04b3 100644 ---- a/.bazelversion -+++ b/.bazelversion -@@ -1 +1 @@ --7.4.1 -+8.1.1 -\ No newline at end of file -diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl -index d62531152d..71d80a5a99 100644 ---- a/third_party/gpus/cuda_configure.bzl -+++ b/third_party/gpus/cuda_configure.bzl -@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. - load( - "@bazel_tools//tools/cpp:lib_cc_configure.bzl", - "escape_string", -- "get_env_var", - ) - load( - "@bazel_tools//tools/cpp:windows_cc_configure.bzl", -- "find_msvc_tool", - "find_vc_path", - "setup_vc_env_vars", - ) -+load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") -+load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") - load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") - load( - "//third_party/remote_config:common.bzl", --- -2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250612.0-6e48cbb/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250612.0-6e48cbb/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch deleted file mode 100644 index 938ef401..00000000 --- a/third_party/modules/xla/20250612.0-6e48cbb/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch +++ /dev/null @@ -1,135 +0,0 @@ -From 2ae9bb9d24b569c2c6bfab3c54b428103614944d Mon Sep 17 00:00:00 2001 -From: Hugo Mano -Date: Tue, 27 May 2025 11:48:17 +0200 -Subject: [PATCH 1/8] Added FFI handler registration API to the FFI PjRt - -PR: https://github.com/openxla/xla/pull/13420 ---- - xla/pjrt/c/BUILD | 5 +++++ - xla/pjrt/c/pjrt_c_api_ffi_extension.h | 21 ++++++++++++++++++ - xla/pjrt/c/pjrt_c_api_ffi_internal.cc | 32 ++++++++++++++++++++++++++- - 3 files changed, 57 insertions(+), 1 deletion(-) - -diff --git a/xla/pjrt/c/BUILD b/xla/pjrt/c/BUILD -index 79f18fa0bc..0f33dd8a6e 100644 ---- a/xla/pjrt/c/BUILD -+++ b/xla/pjrt/c/BUILD -@@ -69,8 +69,13 @@ cc_library( - ":pjrt_c_api_wrapper_impl", - "//xla/ffi:execution_context", - "//xla/ffi:type_id_registry", -+ "//xla/ffi:ffi_api", -+ "//xla/ffi/api:c_api", -+ "//xla/ffi/api:ffi", -+ "//xla/service:custom_call_target_registry", - "@com_google_absl//absl/status", - "@com_google_absl//absl/strings:string_view", -+ "@com_google_absl//absl/strings:str_format", - ], - ) - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -index 995a2c7e50..b8f10bc2f7 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -@@ -69,10 +69,31 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); - // Adds a user data to the execute context. - typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); - -+typedef enum PJRT_FFI_Handler_TraitsBits { -+ PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE = 1u << 0, -+} PJRT_FFI_Handler_TraitsBits; -+ -+struct PJRT_FFI_Register_Handler_Args { -+ size_t struct_size; -+ const char* target_name; -+ size_t target_name_size; -+ int api_version; // 0 for an untyped call, 1 -- for typed -+ void* handler; -+ const char* platform_name; -+ size_t platform_name_size; -+ PJRT_FFI_Handler_TraitsBits traits; -+}; -+PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, traits); -+ -+// Registers an FFI call handler for a specific platform. -+typedef PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args); -+ - typedef struct PJRT_FFI_Extension { - PJRT_Extension_Base base; - PJRT_FFI_TypeID_Register* type_id_register; - PJRT_FFI_UserData_Add* user_data_add; -+ PJRT_FFI_Register_Handler* register_handler; - } PJRT_FFI; - PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Extension, user_data_add); - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -index 5fa88eab33..763270331b 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -+++ b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -@@ -13,16 +13,20 @@ See the License for the specific language governing permissions and - limitations under the License. - ==============================================================================*/ - --#include "xla/pjrt/c/pjrt_c_api_ffi_internal.h" -+#include - - #include "absl/status/status.h" -+#include "absl/strings/str_format.h" - #include "absl/strings/string_view.h" -+#include "xla/ffi/api/c_api.h" - #include "xla/ffi/execution_context.h" - #include "xla/ffi/type_id_registry.h" -+#include "xla/ffi/ffi_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" - #include "xla/pjrt/c/pjrt_c_api_ffi_extension.h" - #include "xla/pjrt/c/pjrt_c_api_helpers.h" - #include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" -+#include "xla/service/custom_call_target_registry.h" - - namespace pjrt { - -@@ -68,6 +72,31 @@ static PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args) { - return nullptr; - } - -+static PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args) { -+ PJRT_RETURN_IF_ERROR(ActualStructSizeIsGreaterOrEqual( -+ "PJRT_FFI_Register_Handler_Args", -+ PJRT_FFI_Register_Handler_Args_STRUCT_SIZE, args->struct_size)); -+ std::string target_name(args->target_name, args->target_name_size); -+ std::string platform_name(args->platform_name, args->platform_name_size); -+ switch (args->api_version) { -+ case 0: -+ xla::CustomCallTargetRegistry::Global()->Register( -+ target_name, args->handler, platform_name); -+ return nullptr; -+ case 1: -+ xla::ffi::Ffi::RegisterStaticHandler( -+ xla::ffi::GetXlaFfiApi(), target_name, platform_name, -+ reinterpret_cast(args->handler)); -+ return nullptr; -+ default: -+ return new PJRT_Error{absl::UnimplementedError( -+ absl::StrFormat("API version %d not supported for PJRT GPU plugin. " -+ "Supported versions are 0 and 1.", -+ args->api_version))}; -+ } -+} -+ - PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - return { - PJRT_Extension_Base{ -@@ -77,6 +106,7 @@ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - }, - /*type_id_register=*/PJRT_FFI_TypeID_Register, - /*user_data_add=*/PJRT_FFI_UserData_Add, -+ /*register_handler=*/PJRT_FFI_Register_Handler, - }; - } - --- -2.39.5 (Apple Git-154) - diff --git a/third_party/modules/xla/20250612.0-6e48cbb/patches/0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch b/third_party/modules/xla/20250612.0-6e48cbb/patches/0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch deleted file mode 100644 index 27d52df5..00000000 --- a/third_party/modules/xla/20250612.0-6e48cbb/patches/0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch +++ /dev/null @@ -1,1855 +0,0 @@ -From 4c0819ac9fb9dfc6156ae4de83fb29e987ade780 Mon Sep 17 00:00:00 2001 -From: Corentin Godeau -Date: Mon, 16 Jun 2025 14:27:47 +0000 -Subject: [PATCH] Update Stablehlo patch - ---- - third_party/stablehlo/temporary.patch | 841 +++++++++++++------------- - 1 file changed, 432 insertions(+), 409 deletions(-) - mode change 100755 => 100644 third_party/stablehlo/temporary.patch - -diff --git a/third_party/stablehlo/temporary.patch b/third_party/stablehlo/temporary.patch -old mode 100755 -new mode 100644 -index f54e8341ba..6b25e78467 ---- a/third_party/stablehlo/temporary.patch -+++ b/third_party/stablehlo/temporary.patch -@@ -1,7 +1,47 @@ --diff --ruN a/stablehlo/stablehlo/dialect/VhloAttrs.td b/stablehlo/stablehlo/dialect/VhloAttrs.td ----- stablehlo/stablehlo/dialect/VhloAttrs.td --+++ stablehlo/stablehlo/dialect/VhloAttrs.td --@@ -45,14 +45,6 @@ -+From c48aa0650bf27b81c42799f9218dd90779e36e3b Mon Sep 17 00:00:00 2001 -+From: Corentin Godeau -+Date: Mon, 16 Jun 2025 14:08:48 +0000 -+Subject: [PATCH] Remove default argument that is not valid C -+ -+--- -+ stablehlo/dialect/Version.h | 2 +- -+ stablehlo/dialect/VhloAttrs.td | 14 +- -+ stablehlo/dialect/VhloDialect.td | 1 + -+ stablehlo/dialect/VhloTypes.h | 24 +- -+ .../integrations/c/StablehloDialectApi.h | 2 +- -+ .../stablehlo_aggressive_folder.mlir | 37 +- -+ .../stablehlo_aggressive_simplification.mlir | 2 +- -+ .../transforms/stablehlo_refine_shapes.mlir | 10 +- -+ .../stablehlo_legalize_to_vhlo_mixed.mlir | 38 +- -+ .../tests/vhlo/vhlo_attributes_invalid.mlir | 13 +- -+ stablehlo/tools/StablehloTranslateMain.cpp | 8 +- -+ stablehlo/transforms/Passes.h | 7 +- -+ .../transforms/StablehloLegalizeToVhlo.cpp | 48 +- -+ .../transforms/StablehloRefineShapes.cpp | 4 + -+ .../transforms/VhloLegalizeToStablehlo.cpp | 100 ++- -+ stablehlo/transforms/VhloToVersion.cpp | 38 +- -+ .../StablehloAggressiveFolder.cpp | 816 ++++++++++-------- -+ ...ablehloAggressiveSimplificationPatterns.td | 6 +- -+ 18 files changed, 696 insertions(+), 474 deletions(-) -+ -+diff --git a/stablehlo/dialect/Version.h b/stablehlo/dialect/Version.h -+index eb35ad52..6aaef985 100644 -+--- a/stablehlo/dialect/Version.h -++++ b/stablehlo/dialect/Version.h -+@@ -38,7 +38,7 @@ class Version { -+ static FailureOr fromString(llvm::StringRef versionRef); -+ -+ /// Return a Version representing the current VHLO dialect version. -+- static Version getCurrentVersion() { return Version(1, 10, 10); } -++ static Version getCurrentVersion() { return Version(1, 11, 0); } -+ -+ /// Return a Version representing the minimum supported VHLO dialect version. -+ static Version getMinimumVersion() { return Version(0, 9, 0); } -+diff --git a/stablehlo/dialect/VhloAttrs.td b/stablehlo/dialect/VhloAttrs.td -+index bf75ad27..ab48630a 100644 -+--- a/stablehlo/dialect/VhloAttrs.td -++++ b/stablehlo/dialect/VhloAttrs.td -+@@ -45,14 +45,6 @@ class VHLO_AttrDef - def VHLO_ArrayAttrV1 : VHLO_AttrDef<"ArrayV1", "0.9.0", "current"> { - let mnemonic = "array_v1"; - let parameters = (ins ArrayRefParameter<"mlir::Attribute">:$value); -@@ -16,7 +56,7 @@ diff --ruN a/stablehlo/stablehlo/dialect/VhloAttrs.td b/stablehlo/stablehlo/dial - let assemblyFormat = "`<` custom($value) `>`"; - } - --@@ -75,9 +67,9 @@ -+@@ -75,9 +67,9 @@ def VHLO_DictionaryAttrV1 : VHLO_AttrDef<"DictionaryV1", "0.9.0", "current"> { - LogicalResult DictionaryV1Attr::verify( - llvm::function_ref errFn, - ArrayRef> value) { -@@ -29,36 +69,48 @@ diff --ruN a/stablehlo/stablehlo/dialect/VhloAttrs.td b/stablehlo/stablehlo/dial - return success(); - } - }]; --diff --ruN a/stablehlo/stablehlo/dialect/VhloTypes.h b/stablehlo/stablehlo/dialect/VhloTypes.h ----- stablehlo/stablehlo/dialect/VhloTypes.h --+++ stablehlo/stablehlo/dialect/VhloTypes.h --@@ -27,20 +27,23 @@ -+diff --git a/stablehlo/dialect/VhloDialect.td b/stablehlo/dialect/VhloDialect.td -+index 4e50cb32..edd220e3 100644 -+--- a/stablehlo/dialect/VhloDialect.td -++++ b/stablehlo/dialect/VhloDialect.td -+@@ -49,6 +49,7 @@ def VHLO_Dialect : Dialect { -+ 1.8.0: Introduce `f4E2M1FN`, `f6E2M3FN`, `f6E3M2FN` and `f8E8M0FNU` types. -+ 1.9.0: Add `ResultAccuracy` attribute to `exp` op. -+ 1.10.0: Add `ResultAccuracy` attribute to `cbrt`, `cosine`, `exponential`, `exponential_minus_one`, `log`, `log_plus_one`, `logistic`, `rsqrt`, `sine`, `sqrt`, `tan` and `tanh` ops. -++ 1.11.0: Allow (de)serializing VHLO programs mixed with potentially unstable dialects. -+ }]; -+ -+ let useDefaultAttributePrinterParser = 0; -+diff --git a/stablehlo/dialect/VhloTypes.h b/stablehlo/dialect/VhloTypes.h -+index e5aee254..f3a8d68d 100644 -+--- a/stablehlo/dialect/VhloTypes.h -++++ b/stablehlo/dialect/VhloTypes.h -+@@ -27,20 +27,23 @@ limitations under the License. - namespace mlir { - namespace vhlo { - - -class VhloTypeConverterBase : public TypeConverter { --- public: --- VhloTypeConverterBase() : TypeConverter(){}; --- --- virtual ~VhloTypeConverterBase() = default; --- --- virtual Attribute convertEncoding(Attribute attr) const = 0; ---}; -- -- // This class is used to manage conversions between VHLO and Builtin -- // dialects. ---class VhloTypeConverter : public VhloTypeConverterBase { -++ -++// This class is used to manage conversions between VHLO and Builtin -++// dialects. - +class VhloTypeConverter : public TypeConverter { - public: --- VhloTypeConverter() : VhloTypeConverterBase() {} -+- VhloTypeConverterBase() : TypeConverter(){}; - + VhloTypeConverter() : TypeConverter(), allowOtherDialects(false) {} - + VhloTypeConverter(bool allowOtherDialects) - + : TypeConverter(), allowOtherDialects(allowOtherDialects) {} --+ -+ -+- virtual ~VhloTypeConverterBase() = default; - + virtual ~VhloTypeConverter() = default; --+ --+ virtual Attribute convertEncoding(Attribute attr) const = 0; --+ -+ -+ virtual Attribute convertEncoding(Attribute attr) const = 0; -+-}; -+ -+-// This class is used to manage conversions between VHLO and Builtin -+-// dialects. -+-class VhloTypeConverter : public VhloTypeConverterBase { -+- public: -+- VhloTypeConverter() : VhloTypeConverterBase() {} - + Attribute convertUnknownAttribute(Attribute attr) const { - + if (allowOtherDialects) return attr; - + return {}; -@@ -66,7 +118,7 @@ diff --ruN a/stablehlo/stablehlo/dialect/VhloTypes.h b/stablehlo/stablehlo/diale - - // A subclass can call this method to add conversions from VHLO -> Builtin - // types. Note that conversions are applied in reverse order, with the most --@@ -58,6 +61,9 @@ -+@@ -58,6 +61,9 @@ class VhloTypeConverter : public VhloTypeConverterBase { - - // Mark unrealized casts as legal. Useful for dialect mixing. - void addUnrealizedMaterializations(); -@@ -76,19 +128,34 @@ diff --ruN a/stablehlo/stablehlo/dialect/VhloTypes.h b/stablehlo/stablehlo/diale - }; - - // Autogenerated VHLO type printers and parsers. --diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_folder.mlir b/stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_folder.mlir ----- stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_folder.mlir --+++ stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_folder.mlir -+diff --git a/stablehlo/integrations/c/StablehloDialectApi.h b/stablehlo/integrations/c/StablehloDialectApi.h -+index 385156bf..24d11c1d 100644 -+--- a/stablehlo/integrations/c/StablehloDialectApi.h -++++ b/stablehlo/integrations/c/StablehloDialectApi.h -+@@ -93,7 +93,7 @@ stablehloSerializePortableArtifactFromModule(MlirModule moduleStr, -+ MlirStringRef targetVersion, -+ MlirStringCallback callback, -+ void* userData, -+- bool allowOtherDialects = false); -++ bool allowOtherDialects); -+ -+ // Read a StableHLO program from a portable artifact, returning the module as -+ // MLIR bytecode. Note, this bytecode returned is not a portable artifact, -+diff --git a/stablehlo/tests/transforms/stablehlo_aggressive_folder.mlir b/stablehlo/tests/transforms/stablehlo_aggressive_folder.mlir -+index 758965d0..c5239671 100644 -+--- a/stablehlo/tests/transforms/stablehlo_aggressive_folder.mlir -++++ b/stablehlo/tests/transforms/stablehlo_aggressive_folder.mlir - @@ -1,4 +1,4 @@ - -// RUN: stablehlo-opt --stablehlo-aggressive-folder --split-input-file --verify-diagnostics %s | FileCheck %s - +// RUN: stablehlo-opt --stablehlo-aggressive-folder=fold-op-element-limit=100 --split-input-file --verify-diagnostics %s | FileCheck %s - - //////// - // AddOp --@@ -42,6 +42,21 @@ -+@@ -41,6 +41,21 @@ func.func @broadcast_in_dim_fold_splat(%arg0: tensor<3x3xi32>) -+ - // ----- - -- //////// -++//////// - +// ClampOp - + - +// CHECK-LABEL: func.func @clamp_fold -@@ -103,18 +170,13 @@ diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_folder.ml - + - +// ----- - + --+//////// -+ //////// - // CompareOp - -- // CHECK-LABEL: func.func @compare_folds --@@ -98,6 +113,26 @@ -- // CHECK-DAG: [[R3:%.+]] = stablehlo.constant dense<{{\[\[0, 1, 2, 11, 12\], \[3, 4, 5, 13, 14\]\]}}> : tensor<2x5xi32> -- // CHECK-NEXT: return [[R0]], [[R1]], [[R2]], [[R3]] -- return %0, %1, %2, %3 : tensor<6xi32>, tensor<3xi32>, tensor<3x3xi32>, tensor<2x5xi32> --+} --+ --+// ----- --+ -+@@ -102,6 +117,26 @@ func.func @concatenate_fold() -> (tensor<6xi32>, tensor<3xi32>, tensor<3x3xi32>, -+ -+ // ----- -+ - +//////// - +// DivOp - + -@@ -131,32 +193,37 @@ diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_folder.ml - + %1 = stablehlo.divide %cst_1, %cst_1 : tensor - + %2 = stablehlo.divide %cst_2, %cst_2 : tensor - + return %0, %1, %2 : tensor, tensor, tensor -- } -++} -++ -++// ----- -++ -+ //////// -+ // MulOp - -- // ----- --diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_simplification.mlir b/stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_simplification.mlir ----- stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_simplification.mlir --+++ stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_simplification.mlir -+diff --git a/stablehlo/tests/transforms/stablehlo_aggressive_simplification.mlir b/stablehlo/tests/transforms/stablehlo_aggressive_simplification.mlir -+index 4921a224..a1eed944 100644 -+--- a/stablehlo/tests/transforms/stablehlo_aggressive_simplification.mlir -++++ b/stablehlo/tests/transforms/stablehlo_aggressive_simplification.mlir - @@ -1,4 +1,4 @@ - -// RUN: stablehlo-opt --stablehlo-aggressive-simplification --allow-unregistered-dialect --split-input-file %s | FileCheck %s - +// RUN: stablehlo-opt --stablehlo-aggressive-simplification=fold-op-element-limit=100 --allow-unregistered-dialect --split-input-file %s | FileCheck %s - - ///////// - // AddOp --diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir b/stablehlo/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir ----- stablehlo/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir --+++ stablehlo/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir --@@ -521,16 +521,16 @@ -+diff --git a/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir b/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir -+index b262bf09..01b5dd8f 100644 -+--- a/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir -++++ b/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir -+@@ -521,16 +521,16 @@ func.func @eval_slice_zerodim() -> tensor<0x2x1xi64> { - // ----- - - // CHECK-LABEL: func @eval_slice_zerorank - -func.func @eval_slice_zerorank() -> tensor { - - // CHECK: [[RESULT:%.*]] = stablehlo.constant dense<3.300000e+01> : tensor --- // CHECK: return [[RESULT]] --- %0 = stablehlo.constant dense<33.0> : tensor - +func.func @eval_slice_zerorank() -> tensor { - + // CHECK: [[RESULT:%.*]] = stablehlo.constant dense<33> : tensor --+ // CHECK: return [[RESULT]] -+ // CHECK: return [[RESULT]] -+- %0 = stablehlo.constant dense<33.0> : tensor - + %0 = stablehlo.constant dense<33> : tensor - %1 = "stablehlo.slice"(%0) { - start_indices = array, -@@ -169,9 +236,10 @@ diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir b - } - - // ----- --diff --ruN a/stablehlo/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mlir b/stablehlo/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mlir ----- stablehlo/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mlir --+++ stablehlo/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mlir -+diff --git a/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mlir b/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mlir -+index 6340898e..a5d344ac 100644 -+--- a/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mlir -++++ b/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mlir - @@ -5,15 +5,9 @@ - // about what constitutes a good test! The CHECK should be - // minimized and named to reflect the test intent. -@@ -190,14 +258,10 @@ diff --ruN a/stablehlo/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mli - // RUN: diff %t.0 %t.1 - - // CHECK-LABEL: vhlo.func_v1 @op_other( --@@ -26,6 +20,34 @@ -- func.func @op_other(%arg0: tensor) -> tensor { -- %0 = arith.addf %arg0, %arg0 : tensor -- return %0 : tensor --+} --+ --+// ----- --+ -+@@ -30,6 +24,34 @@ func.func @op_other(%arg0: tensor) -> tensor { -+ -+ // ----- -+ - +// CHECK-LABEL: vhlo.func_v1 @func_attributes( - +// CHECK-SAME: %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !vhlo.tensor_v1) -> (!vhlo.tensor_v1) { - +// CHECK: "vhlo.return_v1"(%[[VAL_0]]) : (!vhlo.tensor_v1) -> () -@@ -222,12 +286,17 @@ diff --ruN a/stablehlo/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mli - + vhlo.mixed_dict = {affine_map = affine_map<(d0) -> (d0)>, str_attr = "STR_ATTR"} - +} { - + return %arg0 : tensor -- } -- -- // ----- --diff --ruN a/stablehlo/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir b/stablehlo/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir ----- stablehlo/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir --+++ stablehlo/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir -++} -++ -++// ----- -++ -+ // CHECK-LABEL: vhlo.func_v1 @op_shlo( -+ // CHECK-SAME: %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !vhlo.tensor_v1) -> (!vhlo.tensor_v1) { -+ // CHECK: %[[VAL_1:.*]] = "vhlo.add_v1"(%[[VAL_0]], %[[VAL_0]]) : (!vhlo.tensor_v1, !vhlo.tensor_v1) -> !vhlo.tensor_v1 -+diff --git a/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir b/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir -+index b73d1005..f4330e67 100644 -+--- a/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir -++++ b/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir - @@ -1,17 +1,8 @@ - // RUN: stablehlo-opt --vhlo-to-version=target=1.9.0 -verify-diagnostics --split-input-file %s - -@@ -248,10 +317,11 @@ diff --ruN a/stablehlo/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir b/stabl - } { - return - } --diff --ruN a/stablehlo/stablehlo/tools/StablehloTranslateMain.cpp b/stablehlo/stablehlo/tools/StablehloTranslateMain.cpp ----- stablehlo/stablehlo/tools/StablehloTranslateMain.cpp --+++ stablehlo/stablehlo/tools/StablehloTranslateMain.cpp --@@ -76,6 +76,11 @@ -+diff --git a/stablehlo/tools/StablehloTranslateMain.cpp b/stablehlo/tools/StablehloTranslateMain.cpp -+index e3171b2d..fdf0d6a9 100644 -+--- a/stablehlo/tools/StablehloTranslateMain.cpp -++++ b/stablehlo/tools/StablehloTranslateMain.cpp -+@@ -76,6 +76,11 @@ llvm::cl::opt targetOption( - "target", llvm::cl::desc("Target version for serialization"), - llvm::cl::init("")); - -@@ -263,7 +333,7 @@ diff --ruN a/stablehlo/stablehlo/tools/StablehloTranslateMain.cpp b/stablehlo/st - llvm::cl::opt argsOption( - "args", llvm::cl::desc("Arguments to pass to the interpreter"), - llvm::cl::init("")); --@@ -317,7 +322,8 @@ -+@@ -317,7 +322,8 @@ TranslateFromMLIRRegistration serializeRegistration( - return module.emitError("failed to strip debuginfo"); - } - -@@ -273,10 +343,11 @@ diff --ruN a/stablehlo/stablehlo/tools/StablehloTranslateMain.cpp b/stablehlo/st - }, - [](DialectRegistry ®istry) { - mlir::registerAllDialects(registry); --diff --ruN a/stablehlo/stablehlo/transforms/Passes.h b/stablehlo/stablehlo/transforms/Passes.h ----- stablehlo/stablehlo/transforms/Passes.h --+++ stablehlo/stablehlo/transforms/Passes.h --@@ -35,6 +35,7 @@ -+diff --git a/stablehlo/transforms/Passes.h b/stablehlo/transforms/Passes.h -+index 0e4406d2..a5a4aa27 100644 -+--- a/stablehlo/transforms/Passes.h -++++ b/stablehlo/transforms/Passes.h -+@@ -35,6 +35,7 @@ limitations under the License. - #include "mlir/Transforms/DialectConversion.h" - #include "stablehlo/dialect/StablehloOps.h" - #include "stablehlo/dialect/Version.h" -@@ -284,7 +355,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/Passes.h b/stablehlo/stablehlo/trans - - namespace mlir { - namespace stablehlo { --@@ -54,17 +55,17 @@ -+@@ -54,17 +55,17 @@ void populateStablehloRefineShapesPatterns(MLIRContext *context, - // Populates StableHLO ops to VHLO ops rewriting patterns. - void populateStablehloToVhloPatterns(MLIRContext *context, - RewritePatternSet *patterns, -@@ -305,10 +376,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/Passes.h b/stablehlo/stablehlo/trans - - /// Collection of rewrite patterns for lowering of CHLO ops to StableHLO and - /// Shape ops. --diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp ----- stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp --+++ stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp --@@ -57,7 +57,7 @@ -+diff --git a/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stablehlo/transforms/StablehloLegalizeToVhlo.cpp -+index e768ded6..25903399 100644 -+--- a/stablehlo/transforms/StablehloLegalizeToVhlo.cpp -++++ b/stablehlo/transforms/StablehloLegalizeToVhlo.cpp -+@@ -57,7 +57,7 @@ namespace { - class StablehloToVhloTypeConverter : public vhlo::VhloTypeConverter { - public: - StablehloToVhloTypeConverter(bool allowOtherDialects) -@@ -317,7 +389,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable - LLVM_DEBUG( - llvm::dbgs() - << "[StablehloToVhloTypeConverter] Creating with allowOtherDialects: " --@@ -82,7 +82,15 @@ -+@@ -82,7 +82,15 @@ class StablehloToVhloTypeConverter : public vhlo::VhloTypeConverter { - return vhlo::TokenV1Type::get(token.getContext()); - }); - addBuiltinToVhloConversions(); -@@ -334,7 +406,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable - } - - Attribute convertEncoding(Attribute attr) const final { --@@ -114,7 +122,7 @@ -+@@ -114,7 +122,7 @@ class StablehloToVhloTypeConverter : public vhlo::VhloTypeConverter { - return vhlo::Name##Version##Attr::get(attr.getContext(), vhloValue.value()) - - Attribute convertGeneric(Attribute stablehloAttr, -@@ -343,7 +415,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable - LLVM_DEBUG(llvm::dbgs() << "Convert generic: " << stablehloAttr << '\n'); - - // Handle StableHLO attributes. --@@ -241,6 +249,10 @@ -+@@ -241,6 +249,10 @@ Attribute convertGeneric(Attribute stablehloAttr, - return vhlo::TypeV1Attr::get(attr.getContext(), vhloType); - } - -@@ -354,7 +426,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable - LLVM_DEBUG(llvm::dbgs() << "Failed to convert: " << stablehloAttr << '\n'); - return {}; // Failed to convert attribute. - } --@@ -268,13 +280,15 @@ -+@@ -268,13 +280,15 @@ SpecialResult notSpecial() { return SpecialResult::NOT_SPECIAL; } - Attribute convertBool(const ConversionPattern& pattern, int64_t stablehloDim) { - auto stablehloType = IntegerType::get(pattern.getContext(), 1); - auto stablehloAttr = IntegerAttr::get(stablehloType, stablehloDim); -@@ -372,7 +444,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable - } - - Attribute convertInts(const ConversionPattern& pattern, --@@ -282,7 +296,8 @@ -+@@ -282,7 +296,8 @@ Attribute convertInts(const ConversionPattern& pattern, - auto stablehloType = RankedTensorType::get( - stablehloDims.size(), IntegerType::get(pattern.getContext(), 64)); - auto stablehloAttr = DenseIntElementsAttr::get(stablehloType, stablehloDims); -@@ -382,7 +454,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable - } - - Attribute convertSymbol(const ConversionPattern& pattern, --@@ -290,7 +305,7 @@ -+@@ -290,7 +305,7 @@ Attribute convertSymbol(const ConversionPattern& pattern, - auto stablehloSymbolAttr = dyn_cast(stablehloAttr); - if (!stablehloSymbolAttr) return {}; - return convertGeneric(stablehloSymbolAttr.getAttr(), -@@ -391,7 +463,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable - } - - SpecialResult convertChannelHandle(const ConversionPattern& pattern, --@@ -445,15 +460,15 @@ -+@@ -445,15 +460,15 @@ SpecialResult convertDotAlgorithm(const ConversionPattern& pattern, - vhloAttrs.emplace_back( - StringAttr::get(pattern.getContext(), "lhs_precision_type"), - convertGeneric(TypeAttr::get(attr.getLhsPrecisionType()), -@@ -410,7 +482,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable - - // Components - auto vhloLhsComponentCount = convertInt(pattern, attr.getLhsComponentCount()); --@@ -712,7 +727,9 @@ -+@@ -712,7 +727,9 @@ LogicalResult addDefaults(const OpConversionPattern& pattern, - auto addDefaultAttr = [&](StringRef vhloName, Attribute stablehloAttr) { - vhloAttrs.emplace_back( - StringAttr::get(pattern.getContext(), vhloName), -@@ -421,7 +493,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable - }; - if constexpr (std::is_same::value) { - if (!stablehloOp.getSymVisibilityAttr()) --@@ -987,8 +1004,9 @@ -+@@ -987,8 +1004,9 @@ class StablehloToVhloOpConverter : public OpConversionPattern { - case SpecialResult::SPECIAL_FAILURE: - return failure(); - case SpecialResult::NOT_SPECIAL: -@@ -433,7 +505,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable - if (!vhloAttr) return failure(); - vhloAttrs.push_back({stablehloAttr.getName(), vhloAttr}); - break; --@@ -1075,14 +1093,14 @@ -+@@ -1075,14 +1093,14 @@ struct StablehloLegalizeToVhloPass - } - - private: -@@ -450,10 +522,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable - populateStablehloToVhloPatterns< - #define GET_OP_LIST - #include "stablehlo/dialect/StablehloOps.cpp.inc" --diff --ruN a/stablehlo/stablehlo/transforms/StablehloRefineShapes.cpp b/stablehlo/stablehlo/transforms/StablehloRefineShapes.cpp ----- stablehlo/stablehlo/transforms/StablehloRefineShapes.cpp --+++ stablehlo/stablehlo/transforms/StablehloRefineShapes.cpp --@@ -16,6 +16,7 @@ -+diff --git a/stablehlo/transforms/StablehloRefineShapes.cpp b/stablehlo/transforms/StablehloRefineShapes.cpp -+index 64eaa3c0..4b585b46 100644 -+--- a/stablehlo/transforms/StablehloRefineShapes.cpp -++++ b/stablehlo/transforms/StablehloRefineShapes.cpp -+@@ -16,6 +16,7 @@ limitations under the License. - - #include - #include -@@ -461,7 +534,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloRefineShapes.cpp b/stablehl - #include - #include - --@@ -1038,8 +1039,11 @@ -+@@ -1038,8 +1039,11 @@ LogicalResult applyShapeRefinementPatterns(func::FuncOp func, - // Populate additional patterns for StableHLO extensions. - state.addAdditionalPatterns(patterns); - -@@ -473,10 +546,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloRefineShapes.cpp b/stablehl - - // The folding patterns implement partial evaluation of shape computations - // which is a critical part of implementing type refinement for ops like --diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp ----- stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp --+++ stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp --@@ -57,7 +57,10 @@ -+diff --git a/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stablehlo/transforms/VhloLegalizeToStablehlo.cpp -+index 5662ad9d..41973a0e 100644 -+--- a/stablehlo/transforms/VhloLegalizeToStablehlo.cpp -++++ b/stablehlo/transforms/VhloLegalizeToStablehlo.cpp -+@@ -57,7 +57,10 @@ namespace { - - class VhloToStablehloTypeConverter : public vhlo::VhloTypeConverter { - public: -@@ -488,7 +562,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - addConversion([](Type type) -> Type { return type; }); - addConversion([](vhlo::TokenV1Type token) -> Type { - LLVM_DEBUG(llvm::dbgs() << "Converting TokenType\n"); --@@ -90,7 +93,7 @@ -+@@ -90,7 +93,7 @@ class VhloToStablehloTypeConverter : public vhlo::VhloTypeConverter { - return stablehlo::Name##Attr::get(attr.getContext(), stablehloValue.value()) - - Attribute convertGeneric(Attribute vhloAttr, -@@ -497,18 +571,18 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - LLVM_DEBUG(llvm::dbgs() << "Converting attr " << vhloAttr); - if (auto vhloAttrs = dyn_cast(vhloAttr)) { - SmallVector stablehloAttrs; --@@ -189,6 +192,10 @@ -- // All VHLO attributes must have counterparts in StableHLO. -+@@ -190,6 +193,10 @@ Attribute convertGeneric(Attribute vhloAttr, - return {}; - } --+ -+ - + // Fall back to type converter for unknown attributes. - + auto unknownAttr = typeConverter->convertUnknownAttribute(vhloAttr); - + if (unknownAttr) return unknownAttr; -- -++ - // This should be unreachable unless program is a mix of VHLO and other - // dialects, e.g. due to user edits to textual assembly format. --@@ -229,7 +236,7 @@ -+ return {}; -+@@ -229,7 +236,7 @@ bool isNoneType(Attribute vhloAttr) { - } - - LogicalResult convertTypeAttr(Attribute vhloAttr, Type& result, -@@ -517,7 +591,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - auto stablehloAttr = convertGeneric(vhloAttr, typeConverter); - if (!stablehloAttr || !isa(stablehloAttr)) return failure(); - result = cast(stablehloAttr).getValue(); --@@ -244,7 +251,7 @@ -+@@ -244,7 +251,7 @@ LogicalResult convertInt(Attribute vhloAttr, int64_t& result) { - } - - LogicalResult convertInts(Attribute vhloAttr, -@@ -526,7 +600,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - SmallVector& result) { - auto vhloTensorAttr = dyn_cast(vhloAttr); - if (!vhloTensorAttr) return failure(); --@@ -256,7 +263,7 @@ -+@@ -256,7 +263,7 @@ LogicalResult convertInts(Attribute vhloAttr, - } - - Attribute convertSymbol(Attribute vhloAttr, -@@ -535,7 +609,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - auto vhloStringAttr = dyn_cast(vhloAttr); - if (!vhloStringAttr) return {}; - auto stablehloStringAttr = dyn_cast_or_null( --@@ -267,7 +274,7 @@ -+@@ -267,7 +274,7 @@ Attribute convertSymbol(Attribute vhloAttr, - - template - Attribute convertChannelHandle(OpType vhloOp, -@@ -544,7 +618,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - int64_t channelId, channelType; - if (failed(convertInt(vhloOp.getChannelId(), channelId)) || - failed(convertInt(vhloOp.getChannelType(), channelType))) --@@ -277,7 +284,7 @@ -+@@ -277,7 +284,7 @@ Attribute convertChannelHandle(OpType vhloOp, - } - - Attribute convertChannelId(Attribute vhloAttr, -@@ -553,7 +627,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - int64_t channelId; - if (failed(convertInt(vhloAttr, channelId))) return {}; - return stablehlo::ChannelHandleAttr::get(vhloAttr.getContext(), channelId, --@@ -285,8 +292,8 @@ -+@@ -285,8 +292,8 @@ Attribute convertChannelId(Attribute vhloAttr, - } - - template -@@ -564,7 +638,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - int64_t stablehloInputBatchDimension, stablehloInputFeatureDimension; - SmallVector stablehloInputSpatialDimensions; - int64_t stablehloKernelInputFeatureDimension, --@@ -323,7 +330,7 @@ -+@@ -323,7 +330,7 @@ Attribute convertConvDimensionNumbers(OpType vhloOp, - } - - Attribute convertCustomCallCalledComputations( -@@ -573,7 +647,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - if (auto vhloArrayAttr = dyn_cast(vhloAttr)) { - SmallVector stablehloAttrs; - for (auto vhloAttr : vhloArrayAttr.getValue()) { --@@ -336,8 +343,8 @@ -+@@ -336,8 +343,8 @@ Attribute convertCustomCallCalledComputations( - return {}; - } - -@@ -584,7 +658,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - Type lhsPrecisionType, rhsPrecisionType, accumulationType; - if (isNoneType(vhloOp.getLhsComponentCount())) { - // All must be nonetype --@@ -373,8 +380,8 @@ -+@@ -373,8 +380,8 @@ FailureOr convertDotAlgorithm(vhlo::DotGeneralOpV2 vhloOp, - numPrimitiveOperations, allowImpreciseAccumulation); - } - -@@ -595,7 +669,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - SmallVector stablehloLhsBatchingDimensions, - stablehloRhsBatchingDimensions, stablehloLhsContractingDimensions, - stablehloRhsContractingDimensions; --@@ -394,13 +401,13 @@ -+@@ -394,13 +401,13 @@ Attribute convertDotDimensionNumbers(vhlo::DotGeneralOpV2 vhloOp, - } - - Attribute convertFuncCallee(Attribute vhloAttr, -@@ -612,7 +686,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - SmallVector stablehloOffsetDims, stablehloCollapsedSliceDims, - stablehloOperandBatchingDims, stablehloStartIndicesBatchingDims, - stablehloStartIndexMap; --@@ -423,8 +430,8 @@ -+@@ -423,8 +430,8 @@ Attribute convertGatherDimensionNumbers(OpType vhloOp, - stablehloStartIndexMap, stablehloIndexVectorDim); - } - -@@ -623,7 +697,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - SmallVector stablehloUpdateWindowDims, stablehloInsertedWindowDims, - stablehloInputBatchingDims, stablehloScatterIndicesBatchingDims, - stablehloScatterDimsToOperandDims; --@@ -463,10 +470,11 @@ -+@@ -463,10 +470,11 @@ LogicalResult implodeSpecial(const OpConversionPattern& pattern, - VhloOpTy vhloOp, - SmallVector& vhloAttrs, - SmallVector& stablehloAttrs) { -@@ -637,7 +711,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - if (!stablehloAttr) return failure(); - stablehloAttrs.emplace_back( - StringAttr::get(pattern.getContext(), "dimension_numbers"), --@@ -480,7 +488,7 @@ -+@@ -480,7 +488,7 @@ LogicalResult implodeSpecial(const OpConversionPattern& pattern, - if constexpr (std::is_same::value) { - // Dot Dimension Numbers - auto stablehloDotDimAttr = -@@ -646,7 +720,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - if (!stablehloDotDimAttr) return failure(); - stablehloAttrs.emplace_back( - StringAttr::get(pattern.getContext(), "dot_dimension_numbers"), --@@ -489,8 +497,7 @@ -+@@ -489,8 +497,7 @@ LogicalResult implodeSpecial(const OpConversionPattern& pattern, - "lhs_contracting_dimensions", "rhs_contracting_dimensions"); - - // Dot Algorithm -@@ -656,7 +730,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - if (failed(stablehloDotAlgorithmAttr)) return failure(); - if (stablehloDotAlgorithmAttr.value()) - stablehloAttrs.emplace_back( --@@ -503,8 +510,7 @@ -+@@ -503,8 +510,7 @@ LogicalResult implodeSpecial(const OpConversionPattern& pattern, - } - if constexpr (std::is_same::value || - std::is_same::value) { -@@ -666,7 +740,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - if (!stablehloAttr) return failure(); - stablehloAttrs.emplace_back( - StringAttr::get(pattern.getContext(), "dimension_numbers"), --@@ -514,8 +520,7 @@ -+@@ -514,8 +520,7 @@ LogicalResult implodeSpecial(const OpConversionPattern& pattern, - "start_index_map", "index_vector_dim"); - } - if constexpr (std::is_same::value) { -@@ -676,7 +750,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - if (!stablehloAttr) return failure(); - stablehloAttrs.emplace_back( - StringAttr::get(pattern.getContext(), "scatter_dimension_numbers"), --@@ -526,8 +531,7 @@ -+@@ -526,8 +531,7 @@ LogicalResult implodeSpecial(const OpConversionPattern& pattern, - } - if constexpr (std::is_same::value || - std::is_same::value) { -@@ -686,7 +760,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - if (!stablehloAttr) return failure(); - stablehloAttrs.emplace_back( - StringAttr::get(pattern.getContext(), "channel_handle"), stablehloAttr); --@@ -537,7 +541,7 @@ -+@@ -537,7 +541,7 @@ LogicalResult implodeSpecial(const OpConversionPattern& pattern, - } - - template -@@ -695,7 +769,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - StringAttr vhloName, Attribute vhloAttr, - SmallVector& stablehloAttrs) { - auto tensorAttr = dyn_cast(vhloAttr); --@@ -556,15 +560,15 @@ -+@@ -556,15 +560,15 @@ SpecialResult convertDenseArray(const TypeConverter* typeConverter, - } - - SpecialResult convertDenseBoolArray( -@@ -715,7 +789,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - return convertDenseArray( - typeConverter, vhloName, vhloAttr, stablehloAttrs); - } --@@ -575,7 +579,8 @@ -+@@ -575,7 +579,8 @@ SpecialResult convertSpecial(const OpConversionPattern& pattern, - SmallVector& stablehloAttrs) { - StringAttr stablehloName = vhloName; - Attribute stablehloAttr; -@@ -725,7 +799,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - - if constexpr (std::is_same::value || - std::is_same::value || --@@ -585,7 +590,7 @@ -+@@ -585,7 +590,7 @@ SpecialResult convertSpecial(const OpConversionPattern& pattern, - std::is_same::value) { - if (vhloName == "channel_id") { - stablehloName = StringAttr::get(pattern.getContext(), "channel_handle"); -@@ -734,7 +808,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - if (!stablehloAttr) return specialFailure(); - } - if (vhloName == "use_global_device_ids") { --@@ -597,20 +602,20 @@ -+@@ -597,20 +602,20 @@ SpecialResult convertSpecial(const OpConversionPattern& pattern, - } - if constexpr (std::is_same::value) { - if (vhloName == "called_computations") { -@@ -759,7 +833,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - if (!stablehloAttr) return specialFailure(); - } - } --@@ -760,8 +765,8 @@ -+@@ -760,8 +765,8 @@ bool isDefaultResultAccuracyAttribute(Attribute vhloAttr) { - template - bool isSplatTensor(const ConversionPattern& pattern, Attribute vhloAttr, - T splatValue) { -@@ -770,7 +844,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - return attr && attr.isSplat() && - attr.template getSplatValue() == splatValue; - } --@@ -977,8 +982,9 @@ -+@@ -977,8 +982,9 @@ class VhloToStablehloOpConverter : public OpConversionPattern { - case SpecialResult::SPECIAL_FAILURE: - return failure(); - case SpecialResult::NOT_SPECIAL: -@@ -782,7 +856,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - if (!stablehloAttr) return failure(); - stablehloAttrs.push_back({vhloAttr.getName(), stablehloAttr}); - break; --@@ -1056,7 +1062,7 @@ -+@@ -1056,7 +1062,7 @@ struct ReconcileUnrealizedConversionCasts - template - void populateVhloToStablehloPatterns(MLIRContext* context, - RewritePatternSet* patterns, -@@ -791,7 +865,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - patterns - ->add>...>( - *converter, context); --@@ -1104,7 +1110,7 @@ -+@@ -1104,7 +1110,7 @@ struct VhloLegalizeToStablehloPass - - void populateVhloToStablehloPatterns(MLIRContext* context, - RewritePatternSet* patterns, -@@ -800,10 +874,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - populateVhloToStablehloPatterns< - #define GET_OP_LIST - #include "stablehlo/dialect/StablehloOps.cpp.inc" --diff --ruN a/stablehlo/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/stablehlo/transforms/VhloToVersion.cpp ----- stablehlo/stablehlo/transforms/VhloToVersion.cpp --+++ stablehlo/stablehlo/transforms/VhloToVersion.cpp --@@ -56,15 +56,23 @@ -+diff --git a/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/transforms/VhloToVersion.cpp -+index dfbf2877..7818e601 100644 -+--- a/stablehlo/transforms/VhloToVersion.cpp -++++ b/stablehlo/transforms/VhloToVersion.cpp -+@@ -56,15 +56,23 @@ namespace { - - // Currently there are no type-to-version conversions so this class - // simply validates that all types are from the VHLO dialect. -@@ -833,7 +908,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/stable - }; - - // Check user-specified target version. Emit error if invalid. --@@ -111,6 +119,10 @@ -+@@ -111,6 +119,10 @@ bool isLegalVersion(VersionedInterface& interface, const Version& target) { - LogicalResult isLegalType(Type type, const Version& targetVersion); - - LogicalResult isLegalAttribute(const Attribute& attr, Version targetVersion) { -@@ -844,7 +919,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/stable - auto attrInterface = dyn_cast(attr); - if (!attrInterface || !isLegalVersion(attrInterface, targetVersion)) { - LLVM_DEBUG(llvm::dbgs() << "failed to legalize attribute " << attr --@@ -119,10 +131,11 @@ -+@@ -119,10 +131,11 @@ LogicalResult isLegalAttribute(const Attribute& attr, Version targetVersion) { - } - - // Recursively check attrs if VHLO attr is a container -@@ -857,7 +932,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/stable - if (auto arrAttr = dyn_cast(attr)) { - return success(llvm::all_of( - arrAttr.getValue(), [&](std::pair entry) { --@@ -146,6 +159,10 @@ -+@@ -146,6 +159,10 @@ LogicalResult isLegalAttribute(const Attribute& attr, Version targetVersion) { - } - - LogicalResult isLegalType(Type type, const Version& targetVersion) { -@@ -868,7 +943,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/stable - // All valid VHLO types must have versioned type interface. - auto typeInterface = dyn_cast(type); - if (!typeInterface || !isLegalVersion(typeInterface, targetVersion)) { --@@ -170,10 +187,11 @@ -+@@ -170,10 +187,11 @@ LogicalResult isLegalType(Type type, const Version& targetVersion) { - return failure(); - return isLegalType(ranked.getElementType(), targetVersion); - } -@@ -881,7 +956,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/stable - if (auto quant = dyn_cast(type)) - return success( - succeeded(isLegalType(quant.getStorageType(), targetVersion)) && --@@ -213,7 +231,7 @@ -+@@ -213,7 +231,7 @@ bool isLegalLocation(Location loc, const Version& targetVersion) { - bool isLegalOperation(Operation* op, const Version& targetVersion) { - // Validate op - auto opInterface = dyn_cast(op); -@@ -890,7 +965,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/stable - if (!isLegalVersion(opInterface, targetVersion)) return false; - LLVM_DEBUG(llvm::dbgs() << "Legal op version for target. " << op << '\n'); - --@@ -454,7 +472,7 @@ -+@@ -454,7 +472,7 @@ struct AllReduceOpV2ToV1 : public OpRewritePattern { - namespace stablehlo { - void populateVhloToVersionPatterns(MLIRContext* context, - RewritePatternSet* patterns, -@@ -899,10 +974,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/stable - vhlo::populateWithGenerated(*patterns); - patterns->add(context); - patterns->add(context); --diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFolder.cpp b/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFolder.cpp ----- stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFolder.cpp --+++ stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFolder.cpp --@@ -19,6 +19,7 @@ -+diff --git a/stablehlo/transforms/optimization/StablehloAggressiveFolder.cpp b/stablehlo/transforms/optimization/StablehloAggressiveFolder.cpp -+index 9cfac0ba..09bfb783 100644 -+--- a/stablehlo/transforms/optimization/StablehloAggressiveFolder.cpp -++++ b/stablehlo/transforms/optimization/StablehloAggressiveFolder.cpp -+@@ -19,6 +19,7 @@ limitations under the License. - #include - #include - #include -@@ -910,7 +986,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - #include - - #include "llvm/ADT/APInt.h" --@@ -47,6 +48,7 @@ -+@@ -47,6 +48,7 @@ limitations under the License. - #include "mlir/IR/TypeUtilities.h" - #include "mlir/IR/Types.h" - #include "mlir/IR/Value.h" -@@ -918,7 +994,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - #include "mlir/Interfaces/SideEffectInterfaces.h" - #include "mlir/Pass/Pass.h" - #include "mlir/Rewrite/FrozenRewritePatternSet.h" --@@ -98,29 +100,88 @@ -+@@ -98,29 +100,88 @@ LogicalResult validateStaticShapeResult(PatternRewriter& rewriter, - return success(); - } - -@@ -982,8 +1058,8 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - if (res) return cast(res); - - return nullptr; --+} --+ -+ } -+ - + - +/// Binary constant folder that used a generic folder function to handle both - +/// ints and floats. -@@ -1005,8 +1081,8 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + if (!res) return rewriter.notifyMatchFailure(op, "folding failed"); - + - + return res; -- } -- -++} -++ - template - -LogicalResult evalConvertHelper(PatternRewriter& rewriter, OpType op, -@@ -1014,7 +1090,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - DenseIntOrFPElementsAttr elements, Type resType, - CalculationT&& calculate) { - auto result = constFoldCastOp( - elements, resType, calculate); - -@@ -1036,7 +1112,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - DenseIntOrFPElementsAttr elements, - RankedTensorType resultType) { - auto oldType = getElementTypeOrSelf(elements); --@@ -153,7 +215,7 @@ -+@@ -153,7 +215,7 @@ LogicalResult evalConvert(PatternRewriter& rewriter, OpType op, - if (auto newFloatType = dyn_cast(newType)) { - // Float -> Float - const auto& targetSemantics = newFloatType.getFloatSemantics(); -@@ -1045,7 +1121,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - rewriter, op, elements, resultType, - [&targetSemantics](const APFloat& operand, bool& castStatus) { - bool losesInfo; --@@ -167,7 +229,7 @@ -+@@ -167,7 +229,7 @@ LogicalResult evalConvert(PatternRewriter& rewriter, OpType op, - } - - // Float -> Int -@@ -1054,7 +1130,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - rewriter, op, elements, resultType, - [&newBitWidth, &isNewTypeUnsigned](const APFloat& operand, - bool& castStatus) { --@@ -186,7 +248,7 @@ -+@@ -186,7 +248,7 @@ LogicalResult evalConvert(PatternRewriter& rewriter, OpType op, - - if (auto newFloatType = dyn_cast(newType)) { - // Int -> Float -@@ -1063,7 +1139,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - rewriter, op, elements, resultType, - [&newFloatType, &isOldTypeUnsigned](const APInt& operand, - bool& /*castStatus*/) { --@@ -199,64 +261,12 @@ -+@@ -199,7 +261,7 @@ LogicalResult evalConvert(PatternRewriter& rewriter, OpType op, - } - - // Int -> Int -@@ -1072,10 +1148,10 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - rewriter, op, elements, resultType, - [&newBitWidth, &isOldTypeUnsigned](const APInt& operand, - bool& /*castStatus*/) { -- return APSInt(operand, isOldTypeUnsigned).extOrTrunc(newBitWidth); -+@@ -207,58 +269,6 @@ LogicalResult evalConvert(PatternRewriter& rewriter, OpType op, - }); ---} --- -+ } -+ - -// The patterns below implement partial evaluation of shape computations which - -// is a critical part of implementing type refinement for ops like - -// dynamic_broadcast_in_dim, dynamic_iota and dynamic_reshape whose shape -@@ -1126,10 +1202,12 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - - rewriter.replaceOpWithNewOp(op, - - getTensorAttr(resultType, result)); - - return success(); -- } -- -+-} -+- - template --@@ -275,29 +285,18 @@ -+ struct FoldOpRewritePattern : OpRewritePattern { -+ FoldOpRewritePattern(MLIRContext* context, -+@@ -275,29 +285,18 @@ struct FoldOpRewritePattern : OpRewritePattern { - ArrayRef generatedNames = {}) = delete; - - const StablehloAggressiveFolderPassOptions& options; -@@ -1154,9 +1232,8 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - - rewriter.replaceOpWithNewOp(op, res); - - return success(); - - } --- -+ - - return failure(); --+ - + LogicalResult validateElementCountForFold(PatternRewriter& rewriter, - + Operation* op, - + ShapedType resultType) const { -@@ -1171,31 +1248,21 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - } - }; - --@@ -318,100 +317,102 @@ -+@@ -318,100 +317,102 @@ struct ShapeOpRewritePattern : public FoldOpRewritePattern { - } - }; - - -struct EvalAddOpShapePattern : public FoldOpRewritePattern { - - using FoldOpRewritePattern::FoldOpRewritePattern; --- --- LogicalResult matchAndRewrite(AddOp op, --- PatternRewriter& rewriter) const override { --- return evalElementwise(rewriter, op, --- [&](APSInt lhs, APSInt rhs) { return lhs + rhs; }); --- } ---}; --- ---struct EvalAndOpPattern : public FoldOpRewritePattern { --- using FoldOpRewritePattern::FoldOpRewritePattern; --- --- LogicalResult matchAndRewrite(AndOp op, --- PatternRewriter& rewriter) const override { - +struct FoldAddOpPattern final - + : public ShapeOpRewritePattern { - + using ShapeOpRewritePattern::ShapeOpRewritePattern; --+ -+ -+- LogicalResult matchAndRewrite(AddOp op, - + LogicalResult matchAndRewrite(mlir::stablehlo::AddOp op, --+ PatternRewriter& rewriter) const override { -+ PatternRewriter& rewriter) const override { -+- return evalElementwise(rewriter, op, -+- [&](APSInt lhs, APSInt rhs) { return lhs + rhs; }); - + if (failed(validateShapeFoldDtype(rewriter, op, op.getType()))) - + return failure(); - + -@@ -1203,14 +1270,17 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + if (failed(res)) return failure(); - + rewriter.replaceOpWithNewOp(op, res.value()); - + return success(); --+ } --+}; --+ -+ } -+ }; -+ -+-struct EvalAndOpPattern : public FoldOpRewritePattern { -+- using FoldOpRewritePattern::FoldOpRewritePattern; - +struct FoldAndOpPattern : public ShapeOpRewritePattern { - + using ShapeOpRewritePattern::ShapeOpRewritePattern; --+ -+ -+- LogicalResult matchAndRewrite(AndOp op, - + LogicalResult matchAndRewrite(mlir::stablehlo::AndOp op, --+ PatternRewriter& rewriter) const override { -+ PatternRewriter& rewriter) const override { - + // TODO: Support more int types - auto resultType = op.getType(); - if (!resultType.getElementType().isInteger(1)) -@@ -1219,24 +1289,14 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - - return evalElementwise(rewriter, op, [&](APSInt lhsInt, APSInt rhsInt) { - - return getAPSInt(resultType.getElementType(), lhsInt != 0 && rhsInt != 0); - - }); --- } - + auto res = foldBinaryOpIntOrFloat(rewriter, op, FoldAnd{}); - + if (failed(res)) return failure(); - + rewriter.replaceOpWithNewOp(op, res.value()); - + return success(); --+ } --+ --+ struct FoldAnd { --+ APInt operator()(APInt lhs, APInt rhs) const { --+ return APInt(lhs.getBitWidth(), !lhs.isZero() && !rhs.isZero()); --+ } --+ std::optional operator()(APFloat lhs, APFloat rhs) const { --+ return std::nullopt; --+ } --+ }; -- }; -+ } -+-}; - -- // Pattern: broadcast_in_dim(splat, _) -> constant(splat) -+-// Pattern: broadcast_in_dim(splat, _) -> constant(splat) - -struct FoldBroadcastInDimSplatPattern final - - : FoldOpRewritePattern { - - using FoldOpRewritePattern::FoldOpRewritePattern; -@@ -1251,14 +1311,22 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - - op, SplatElementsAttr::get(op.getType(), - - cstAttr.getSplatValue())); - - return success(); --- } -++ struct FoldAnd { -++ APInt operator()(APInt lhs, APInt rhs) const { -++ return APInt(lhs.getBitWidth(), !lhs.isZero() && !rhs.isZero()); -+ } - - return failure(); - - } ---}; --- -++ std::optional operator()(APFloat lhs, APFloat rhs) const { -++ return std::nullopt; -++ } -++ }; -+ }; -+ - -struct EvalBroadcastInDimOpPattern - - : public FoldOpRewritePattern { - - using FoldOpRewritePattern::FoldOpRewritePattern; -++// Pattern: broadcast_in_dim(splat, _) -> constant(splat) - +struct FoldBroadcastInDimOpSplatPattern - + : public ShapeOpRewritePattern { - + using ShapeOpRewritePattern::ShapeOpRewritePattern; -@@ -1267,8 +1335,10 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - PatternRewriter& rewriter) const override { - auto resultType = op.getType(); - - if (failed(validateStaticShapeResult(rewriter, op, resultType))) --- return failure(); --- -++ if (failed(validateStaticShapeResult(rewriter, op, resultType)) || -++ failed(validateShapeFoldDtype(rewriter, op, resultType))) -+ return failure(); -+ - - auto operandType = op.getOperand().getType(); - - if (operandType.getRank() != 0) - - return rewriter.notifyMatchFailure(op, "expected 0-dimensional type"); -@@ -1277,52 +1347,34 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - - if (failed(hlo::matchInts(op.getOperand(), operand))) - - return rewriter.notifyMatchFailure(op, "expected constant operands"); - - auto scalar = operand[0]; --- -++ SplatElementsAttr cstAttr; -++ matchPattern(op.getOperand(), m_Constant(&cstAttr)); -++ if (!cstAttr) return rewriter.notifyMatchFailure(op, "operand not splat"); -+ - - rewriter.replaceOpWithNewOp( - - op, getTensorAttr(op.getType(), scalar)); --- return success(); --- } ---}; --- -++ rewriter.replaceOpWithNewOp( -++ op, SplatElementsAttr::get(op.getType(), -++ cstAttr.getSplatValue())); -+ return success(); -+ } -+ }; -+ - -struct EvalClampOpPattern : public FoldOpRewritePattern { - - using FoldOpRewritePattern::FoldOpRewritePattern; --- -++struct FoldCompareOpPattern : public ShapeOpRewritePattern { -++ using ShapeOpRewritePattern::ShapeOpRewritePattern; -+ - - LogicalResult matchAndRewrite(ClampOp op, --- PatternRewriter& rewriter) const override { -++ LogicalResult matchAndRewrite(CompareOp op, -+ PatternRewriter& rewriter) const override { - - return evalElementwise(rewriter, op, - - [&](APSInt min, APSInt operand, APSInt max) { - - if (operand < min) return min; - - if (max < operand) return max; - - return operand; - - }); --- } ---}; --- ---struct EvalCompareOpPattern : public FoldOpRewritePattern { --- using FoldOpRewritePattern::FoldOpRewritePattern; --+ if (failed(validateStaticShapeResult(rewriter, op, resultType)) || --+ failed(validateShapeFoldDtype(rewriter, op, resultType))) --+ return failure(); --+ --+ SplatElementsAttr cstAttr; --+ matchPattern(op.getOperand(), m_Constant(&cstAttr)); --+ if (!cstAttr) return rewriter.notifyMatchFailure(op, "operand not splat"); --+ --+ rewriter.replaceOpWithNewOp( --+ op, SplatElementsAttr::get(op.getType(), --+ cstAttr.getSplatValue())); --+ return success(); --+ } --+}; --+ --+struct FoldCompareOpPattern : public ShapeOpRewritePattern { --+ using ShapeOpRewritePattern::ShapeOpRewritePattern; -- -- LogicalResult matchAndRewrite(CompareOp op, -- PatternRewriter& rewriter) const override { -- auto resultType = op.getType(); --- auto kind = op.getCompareType(); --- return evalElementwise(rewriter, op, [&](APInt lhs, APInt rhs) { -++ auto resultType = op.getType(); - + if (failed(validateShapeFoldDtype(rewriter, op, resultType))) - + return failure(); - + -@@ -1332,15 +1384,23 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + if (failed(res)) return failure(); - + rewriter.replaceOpWithNewOp(op, res.value()); - + return success(); --+ } --+ -+ } -+-}; -+ -+-struct EvalCompareOpPattern : public FoldOpRewritePattern { -+- using FoldOpRewritePattern::FoldOpRewritePattern; - + struct FoldCompare { - + FoldCompare(ComparisonDirection direction, - + std::optional kind) - + : direction(direction), kind(kind) {} - + ComparisonDirection direction; - + std::optional kind; --+ -+ -+- LogicalResult matchAndRewrite(CompareOp op, -+- PatternRewriter& rewriter) const override { -+- auto resultType = op.getType(); -+- auto kind = op.getCompareType(); -+- return evalElementwise(rewriter, op, [&](APInt lhs, APInt rhs) { - + // TODO: Enable float folding. - + std::optional operator()(APFloat lhs, APFloat rhs) { - + return std::nullopt; -@@ -1352,7 +1412,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - case ComparisonDirection::EQ: - result = lhs == rhs; - break; --@@ -431,9 +432,9 @@ -+@@ -431,9 +432,9 @@ struct EvalCompareOpPattern : public FoldOpRewritePattern { - result = kind == ComparisonType::SIGNED ? lhs.slt(rhs) : lhs.ult(rhs); - break; - } -@@ -1365,7 +1425,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - }; - - ////////////////////////////////// --@@ -441,16 +442,15 @@ -+@@ -441,16 +442,15 @@ struct EvalCompareOpPattern : public FoldOpRewritePattern { - ///////////////////////////////// - - struct FoldConcatenateOpPattern final -@@ -1387,7 +1447,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - return failure(); - - // Fold concatenate when all inputs are constants. --@@ -466,6 +466,7 @@ -+@@ -466,6 +466,7 @@ struct FoldConcatenateOpPattern final - int64_t{1}, std::multiplies<>{}); - - SmallVector newElems; -@@ -1395,7 +1455,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - newElems.reserve(numElems); - - for (int64_t i = 0; i != topSize; ++i) { --@@ -485,31 +486,7 @@ -+@@ -485,31 +486,7 @@ struct FoldConcatenateOpPattern final - int64_t foldOpElementLimit; - }; - -@@ -1428,20 +1488,17 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - using ShapeOpRewritePattern::ShapeOpRewritePattern; - - LogicalResult matchAndRewrite(ConvertOp op, --@@ -532,28 +509,50 @@ -+@@ -532,28 +509,50 @@ struct EvalConvertOpPattern : public ShapeOpRewritePattern { - return rewriter.notifyMatchFailure( - op, "expected constant integer or float operand"); - - - return evalConvert(rewriter, op, elements, resultType); --- } ---}; --- -++ return foldConvert(rewriter, op, elements, resultType); -+ } -+ }; -+ - -struct EvalDivOpPattern : public FoldOpRewritePattern { - - using FoldOpRewritePattern::FoldOpRewritePattern; --+ return foldConvert(rewriter, op, elements, resultType); --+ } --+}; --+ - +struct FoldDivOpPattern : public ShapeOpRewritePattern { - + using ShapeOpRewritePattern::ShapeOpRewritePattern; - -@@ -1449,12 +1506,6 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - PatternRewriter& rewriter) const override { - - return evalElementwise(rewriter, op, - - [&](APSInt lhs, APSInt rhs) { return lhs / rhs; }); --- } ---}; --- ---struct EvalGetDimensionSizeOpPattern --- : public FoldOpRewritePattern { --- using FoldOpRewritePattern::FoldOpRewritePattern; - + auto resultType = op.getType(); - + if (failed(validateShapeFoldDtype(rewriter, op, resultType))) - + return failure(); -@@ -1464,7 +1515,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + if (failed(res)) return failure(); - + rewriter.replaceOpWithNewOp(op, res.value()); - + return success(); --+ } -+ } - + - + struct FoldDivide { - + FoldDivide(bool isUnsignedInt) -@@ -1479,8 +1530,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + static APInt foldUint(APInt lhs, APInt rhs) { return lhs.udiv(rhs); } - + static APInt foldSint(APInt lhs, APInt rhs) { return lhs.sdiv(rhs); } - + }; --+}; --+ -+ }; -+ -+-struct EvalGetDimensionSizeOpPattern -+- : public FoldOpRewritePattern { -+- using FoldOpRewritePattern::FoldOpRewritePattern; - +struct FoldGetDimensionSizeOpPattern - + : public ShapeOpRewritePattern { - + using ShapeOpRewritePattern::ShapeOpRewritePattern; -@@ -1494,7 +1548,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - return failure(); - - auto operandType = op.getOperand().getType(); --@@ -567,86 +566,187 @@ -+@@ -567,86 +566,187 @@ struct EvalGetDimensionSizeOpPattern - } - }; - -@@ -1549,11 +1603,6 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - - return evalElementwise(rewriter, op, [&](APSInt lhs, APSInt rhs) { - - return lhs >= rhs ? lhs : rhs; - - }); --- } ---}; --- ---struct EvalMinOpPattern : public FoldOpRewritePattern { --- using FoldOpRewritePattern::FoldOpRewritePattern; - + auto resultType = op.getType(); - + if (failed(validateShapeFoldDtype(rewriter, op, resultType))) - + return failure(); -@@ -1563,9 +1612,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + if (failed(res)) return failure(); - + rewriter.replaceOpWithNewOp(op, res.value()); - + return success(); --+ } --+}; --+ -+ } -+ }; -+ -+-struct EvalMinOpPattern : public FoldOpRewritePattern { -+- using FoldOpRewritePattern::FoldOpRewritePattern; - +struct FoldMinOpPattern : public ShapeOpRewritePattern { - + using ShapeOpRewritePattern::ShapeOpRewritePattern; - -@@ -1574,11 +1625,6 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - - return evalElementwise(rewriter, op, [&](APSInt lhs, APSInt rhs) { - - return lhs <= rhs ? lhs : rhs; - - }); --- } ---}; --- ---struct FoldMulOpPattern final : FoldOpRewritePattern { --- using FoldOpRewritePattern::FoldOpRewritePattern; - + auto resultType = op.getType(); - + if (failed(validateShapeFoldDtype(rewriter, op, resultType))) - + return failure(); -@@ -1588,19 +1634,35 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + if (failed(res)) return failure(); - + rewriter.replaceOpWithNewOp(op, res.value()); - + return success(); --+ } --+}; --+ -+ } -+ }; -+ -+-struct FoldMulOpPattern final : FoldOpRewritePattern { -+- using FoldOpRewritePattern::FoldOpRewritePattern; - +// Clamp is folded using Min and Max folders. - +struct FoldClampOpPattern : public ShapeOpRewritePattern { - + using ShapeOpRewritePattern::ShapeOpRewritePattern; --+ -+ -+- LogicalResult matchAndRewrite(mlir::stablehlo::MulOp op, - + LogicalResult matchAndRewrite(ClampOp op, --+ PatternRewriter& rewriter) const override { -+ PatternRewriter& rewriter) const override { -+- TypedAttr lhsAttr; -+- matchPattern(op.getLhs(), m_Constant(&lhsAttr)); -+- -+- TypedAttr rhsAttr; -+- matchPattern(op.getRhs(), m_Constant(&rhsAttr)); -+- -+- if (TypedAttr res; -+- lhsAttr && rhsAttr && -+- (res = foldBinaryOpIntOrFloat(lhsAttr, rhsAttr, std::multiplies<>{}))) { -+- rewriter.replaceOpWithNewOp(op, res); -+- return success(); -+- } - + auto resultType = op.getType(); - + if (failed(validateShapeFoldDtype(rewriter, op, resultType))) - + return failure(); --+ -+ -+- return failure(); - + TypedAttr minAttr, operandAttr, maxAttr; - + matchPattern(op.getMin(), m_Constant(&minAttr)); - + matchPattern(op.getOperand(), m_Constant(&operandAttr)); -@@ -1620,43 +1682,19 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + if (!res) return rewriter.notifyMatchFailure(op, "failed to fold clamp"); - + rewriter.replaceOpWithNewOp(op, res); - + return success(); --+ } --+}; --+ --+struct FoldMulOpPattern final : ShapeOpRewritePattern { --+ using ShapeOpRewritePattern::ShapeOpRewritePattern; -+ } -+ }; - -- LogicalResult matchAndRewrite(mlir::stablehlo::MulOp op, -- PatternRewriter& rewriter) const override { --- TypedAttr lhsAttr; --- matchPattern(op.getLhs(), m_Constant(&lhsAttr)); --- --- TypedAttr rhsAttr; --- matchPattern(op.getRhs(), m_Constant(&rhsAttr)); --- --- if (TypedAttr res; --- lhsAttr && rhsAttr && --- (res = foldBinaryOpIntOrFloat(lhsAttr, rhsAttr, std::multiplies<>{}))) { --- rewriter.replaceOpWithNewOp(op, res); --- return success(); --- } --- --- return failure(); --- } ---}; --- - -struct EvalMulOpPattern : public FoldOpRewritePattern { - - using FoldOpRewritePattern::FoldOpRewritePattern; --- -++struct FoldMulOpPattern final : ShapeOpRewritePattern { -++ using ShapeOpRewritePattern::ShapeOpRewritePattern; -+ - - LogicalResult matchAndRewrite(MulOp op, --- PatternRewriter& rewriter) const override { -++ LogicalResult matchAndRewrite(mlir::stablehlo::MulOp op, -+ PatternRewriter& rewriter) const override { - - return evalElementwise(rewriter, op, - - [&](APSInt lhs, APSInt rhs) { return lhs * rhs; }); --- } ---}; --- ---struct EvalOrOpPattern : public FoldOpRewritePattern { --- using FoldOpRewritePattern::FoldOpRewritePattern; - + if (failed(validateShapeFoldDtype(rewriter, op, op.getType()))) - + return failure(); - + -@@ -1664,9 +1702,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + if (failed(res)) return failure(); - + rewriter.replaceOpWithNewOp(op, res.value()); - + return success(); --+ } --+}; --+ -+ } -+ }; -+ -+-struct EvalOrOpPattern : public FoldOpRewritePattern { -+- using FoldOpRewritePattern::FoldOpRewritePattern; - +struct FoldOrOpPattern : public ShapeOpRewritePattern { - + using ShapeOpRewritePattern::ShapeOpRewritePattern; - -@@ -1680,16 +1720,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - - return evalElementwise(rewriter, op, [&](APSInt lhsInt, APSInt rhsInt) { - - return getAPSInt(resultType.getElementType(), lhsInt != 0 || rhsInt != 0); - - }); --- } ---}; --- ---struct EvalRemOpPattern : public FoldOpRewritePattern { --- using FoldOpRewritePattern::FoldOpRewritePattern; - + auto res = foldBinaryOpIntOrFloat(rewriter, op, FoldOr{}); - + if (failed(res)) return failure(); - + rewriter.replaceOpWithNewOp(op, res.value()); - + return success(); --+ } -+ } - + - + struct FoldOr { - + APInt operator()(APInt lhs, APInt rhs) const { -@@ -1699,8 +1734,10 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + return std::nullopt; - + } - + }; --+}; --+ -+ }; -+ -+-struct EvalRemOpPattern : public FoldOpRewritePattern { -+- using FoldOpRewritePattern::FoldOpRewritePattern; - +struct FoldRemOpPattern : public ShapeOpRewritePattern { - + using ShapeOpRewritePattern::ShapeOpRewritePattern; - -@@ -1708,10 +1745,6 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - PatternRewriter& rewriter) const override { - - return evalElementwise(rewriter, op, - - [&](APSInt lhs, APSInt rhs) { return lhs % rhs; }); --- } ---}; --- ---struct EvalReshapeOpPattern : public ShapeOpRewritePattern { - + auto resultType = op.getType(); - + if (failed(validateShapeFoldDtype(rewriter, op, resultType))) - + return failure(); -@@ -1721,7 +1754,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + if (failed(res)) return failure(); - + rewriter.replaceOpWithNewOp(op, res.value()); - + return success(); --+ } -+ } - + - + struct FoldRem { - + FoldRem(bool isUnsignedInt) -@@ -1736,14 +1769,15 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + static APInt foldUint(APInt lhs, APInt rhs) { return lhs.urem(rhs); } - + static APInt foldSint(APInt lhs, APInt rhs) { return lhs.srem(rhs); } - + }; --+}; --+ -+ }; -+ -+-struct EvalReshapeOpPattern : public ShapeOpRewritePattern { - +// Pattern: reshape(cst, shape) -> cst - +struct FoldReshapeOpPattern : public ShapeOpRewritePattern { - using ShapeOpRewritePattern::ShapeOpRewritePattern; - - LogicalResult matchAndRewrite(ReshapeOp op, --@@ -656,7 +756,6 @@ -+@@ -656,7 +756,6 @@ struct EvalReshapeOpPattern : public ShapeOpRewritePattern { - failed(validateShapeFoldDtype(rewriter, op, resultType))) - return failure(); - -@@ -1751,7 +1785,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - DenseIntOrFPElementsAttr attr; - if (!matchPattern(op.getOperand(), m_Constant(&attr))) - return rewriter.notifyMatchFailure(op, "expected constant operand"); --@@ -665,53 +764,98 @@ -+@@ -665,53 +764,98 @@ struct EvalReshapeOpPattern : public ShapeOpRewritePattern { - } - }; - -@@ -1764,16 +1798,14 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - PatternRewriter& rewriter) const override { - auto resultType = op.getType(); - - if (failed(validateStaticShapeResult(rewriter, op, resultType))) --- return failure(); --- -++ if (failed(validateStaticShapeResult(rewriter, op, resultType)) || -++ failed(validateShapeFoldDtype(rewriter, op, resultType))) -+ return failure(); -+ - - SmallVector pred, onTrue, onFalse; - - if (failed(hlo::matchInts(op.getPred(), pred)) || - - failed(hlo::matchInts(op.getOnTrue(), onTrue)) || - - failed(hlo::matchInts(op.getOnFalse(), onFalse))) --+ if (failed(validateStaticShapeResult(rewriter, op, resultType)) || --+ failed(validateShapeFoldDtype(rewriter, op, resultType))) --+ return failure(); --+ - + DenseIntElementsAttr predAttr; - + DenseElementsAttr onTrueAttr, onFalseAttr; - + matchPattern(op.getPred(), m_Constant(&predAttr)); -@@ -1783,14 +1815,17 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - return rewriter.notifyMatchFailure(op, "expected constant operands"); - - - SmallVector result; -+- for (auto [predEl, onTrueEl, onFalseEl] : -+- llvm::zip(pred, onTrue, onFalse)) { -+- result.push_back(predEl != 0 ? onTrueEl : onFalseEl); - + // Optimization, handle splat predicate - + if (isa(predAttr)) { - + auto pred = predAttr.getSplatValue(); - + rewriter.replaceOpWithNewOp( - + op, pred.isZero() ? onFalseAttr : onTrueAttr); - + return success(); --+ } --+ -+ } -+ - + // TODO: Enable float folding. - + if (op.getType().getElementType().isFloat()) - + return rewriter.notifyMatchFailure(op, "float select not supported yet"); -@@ -1800,27 +1835,17 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + return failure(); - + - + SmallVector result; -- for (auto [predEl, onTrueEl, onFalseEl] : --- llvm::zip(pred, onTrue, onFalse)) { --- result.push_back(predEl != 0 ? onTrueEl : onFalseEl); --- } --- -++ for (auto [predEl, onTrueEl, onFalseEl] : - + llvm::zip(predAttr.getValues(), onTrueAttr.getValues(), - + onFalseAttr.getValues())) { - + result.push_back(!predEl.isZero() ? onTrueEl : onFalseEl); - + } - rewriter.replaceOpWithNewOp( - - op, getTensorAttr(op.getType(), result)); --- return success(); --- } ---}; --- ---struct EvalSignOpPattern : public FoldOpRewritePattern { --- using FoldOpRewritePattern::FoldOpRewritePattern; - + op, DenseIntElementsAttr::get(resultType, result)); - + --+ return success(); --+ } -+ return success(); -+ } - + - + struct FoldSelect { - + std::optional operator()(APFloat pred, APFloat onTrue, -@@ -1832,8 +1857,10 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + return pred != 0 ? onTrue : onFalse; - + } - + }; --+}; --+ -+ }; -+ -+-struct EvalSignOpPattern : public FoldOpRewritePattern { -+- using FoldOpRewritePattern::FoldOpRewritePattern; - +struct FoldSignOpPattern : public ShapeOpRewritePattern { - + using ShapeOpRewritePattern::ShapeOpRewritePattern; - -@@ -1881,7 +1908,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - }; - - template --@@ -749,13 +893,14 @@ -+@@ -749,13 +893,14 @@ DenseElementsAttr sliceType(SliceOp& op, const RangeType& data) { - ArrayRef(result)); - } - -@@ -1899,7 +1926,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - return failure(); - - auto operand = op.getOperand(); --@@ -784,36 +929,18 @@ -+@@ -784,36 +929,18 @@ struct EvalSliceOpPattern : public FoldOpRewritePattern { - }; - - struct FoldSubtractOpPattern final -@@ -1930,14 +1957,13 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - - - -struct EvalSubtractOpPattern : public FoldOpRewritePattern { - - using FoldOpRewritePattern::FoldOpRewritePattern; --- -++ if (failed(validateShapeFoldDtype(rewriter, op, op.getType()))) -++ return failure(); -+ - - LogicalResult matchAndRewrite(SubtractOp op, - - PatternRewriter& rewriter) const override { - - return evalElementwise(rewriter, op, - - [&](APSInt lhs, APSInt rhs) { return lhs - rhs; }); --+ if (failed(validateShapeFoldDtype(rewriter, op, op.getType()))) --+ return failure(); --+ - + auto res = foldBinaryOpIntOrFloat(rewriter, op, std::minus<>{}); - + if (failed(res)) return failure(); - + rewriter.replaceOpWithNewOp(op, res.value()); -@@ -1945,23 +1971,35 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - } - }; - --@@ -823,42 +950,38 @@ -+@@ -823,42 +950,38 @@ struct FoldSqrtOpPattern - - LogicalResult matchAndRewrite(mlir::stablehlo::SqrtOp op, - PatternRewriter& rewriter) const final { - - TypedAttr lhsAttr; - - matchPattern(op.getOperand(), m_Constant(&lhsAttr)); --- -++ auto res = foldUnaryOpIntOrFloat(rewriter, op, FoldSqrt()); -++ if (failed(res)) return failure(); -++ rewriter.replaceOpWithNewOp(op, res.value()); -++ return success(); -++ } -+ - - if (!lhsAttr) - - return rewriter.notifyMatchFailure(op, "operand not constant"); --- -++ struct FoldSqrt { -++ std::optional operator()(APFloat operand) { -++ if (operand.getSizeInBits(operand.getSemantics()) == 64) -++ return APFloat(std::sqrt(operand.convertToDouble())); -+ - - if (auto res = constFoldUnaryOp( - - lhsAttr, foldSqrt)) { - - rewriter.replaceOpWithNewOp( - - op, op.getType(), llvm::cast(res)); - - return success(); --- } --- -++ if (operand.getSizeInBits(operand.getSemantics()) == 32) -++ return APFloat(sqrtf(operand.convertToFloat())); -++ return std::nullopt; -+ } -+ - - return rewriter.notifyMatchFailure(op, "unable to fold sqrt"); - - } - - -@@ -1973,32 +2011,14 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - - return APFloat(sqrtf(a.convertToFloat())); - - return {}; - - } ---}; --- ---struct EvalIotaOpPattern : public FoldOpRewritePattern { --+ auto res = foldUnaryOpIntOrFloat(rewriter, op, FoldSqrt()); --+ if (failed(res)) return failure(); --+ rewriter.replaceOpWithNewOp(op, res.value()); --+ return success(); --+ } --+ --+ struct FoldSqrt { --+ std::optional operator()(APFloat operand) { --+ if (operand.getSizeInBits(operand.getSemantics()) == 64) --+ return APFloat(std::sqrt(operand.convertToDouble())); --+ --+ if (operand.getSizeInBits(operand.getSemantics()) == 32) --+ return APFloat(sqrtf(operand.convertToFloat())); --+ return std::nullopt; --+ } --+ - + // TODO: Enable int folding. - + std::optional operator()(APInt operand) { - + return std::nullopt; - + } - + }; --+}; --+ -+ }; -+ -+-struct EvalIotaOpPattern : public FoldOpRewritePattern { - +struct FoldIotaOpPattern : public FoldOpRewritePattern { - using FoldOpRewritePattern::FoldOpRewritePattern; - -@@ -2015,7 +2035,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - - auto elementType = resultType.getElementType(); - --@@ -929,7 +1052,7 @@ -+@@ -929,7 +1052,7 @@ DenseElementsAttr transposeType(TransposeOp& op, const RangeType& data) { - // transpose(constant) => constant with permuted dimensions - // This covers ranked tensor types with 0 dimensions(zero elements) and 0 - // rank(scalar), as well as splat values. -@@ -2024,7 +2044,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - using FoldOpRewritePattern::FoldOpRewritePattern; - - LogicalResult matchAndRewrite(TransposeOp op, --@@ -943,6 +1066,7 @@ -+@@ -943,6 +1066,7 @@ struct EvalTransposeOpPattern : public FoldOpRewritePattern { - return rewriter.notifyMatchFailure( - op, "expected constant integer or float operand"); - -@@ -2032,7 +2052,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - DenseElementsAttr resAttr; - if (auto data = els.tryGetValues()) - resAttr = transposeType(op, *data); --@@ -957,6 +1081,7 @@ -+@@ -957,6 +1081,7 @@ struct EvalTransposeOpPattern : public FoldOpRewritePattern { - } - }; - -@@ -2040,7 +2060,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - struct LowerBoolSplatConstantsIntoReduceOpRegion - : public FoldOpRewritePattern { - using FoldOpRewritePattern::FoldOpRewritePattern; --@@ -1160,7 +1285,7 @@ -+@@ -1160,7 +1285,7 @@ bool hasNoDeclaredSideEffects(Operation* op) { - return true; - } - -@@ -2049,7 +2069,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - : public FoldOpRewritePattern { - using FoldOpRewritePattern::FoldOpRewritePattern; - --@@ -1233,23 +1358,16 @@ -+@@ -1233,23 +1358,16 @@ void populateStablehloAggressiveFolderPatterns( - PatternBenefit benefit) { - populateStablehloShapeFolderPatterns(context, patterns, options, benefit); - -@@ -2083,7 +2103,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - } - - class StablehloTargetIndependentOptimizationPass { --@@ -1266,25 +1384,25 @@ -+@@ -1266,25 +1384,25 @@ void populateStablehloShapeFolderPatterns( - MLIRContext* context, RewritePatternSet* patterns, - const StablehloAggressiveFolderPassOptions& options, - PatternBenefit benefit) { -@@ -2128,28 +2148,29 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - } - - void populateStablehloShapeFolderPatterns(MLIRContext* context, --diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveSimplificationPatterns.td b/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveSimplificationPatterns.td ----- stablehlo/stablehlo/transforms/optimization/StablehloAggressiveSimplificationPatterns.td --+++ stablehlo/stablehlo/transforms/optimization/StablehloAggressiveSimplificationPatterns.td --@@ -48,6 +48,8 @@ -- def RankEqual : Constraint< -+diff --git a/stablehlo/transforms/optimization/StablehloAggressiveSimplificationPatterns.td b/stablehlo/transforms/optimization/StablehloAggressiveSimplificationPatterns.td -+index 8adb6dbc..3a8edc61 100644 -+--- a/stablehlo/transforms/optimization/StablehloAggressiveSimplificationPatterns.td -++++ b/stablehlo/transforms/optimization/StablehloAggressiveSimplificationPatterns.td -+@@ -49,6 +49,8 @@ def RankEqual : Constraint< - CPred<"llvm::cast($0.getType()).getRank() == llvm::cast($1.getType()).getRank()">, - "same rank">; --+ --+def TensorDimsAllOne : Constraint, "all tensor dims are 1">; - -++def TensorDimsAllOne : Constraint, "all tensor dims are 1">; -++ - def TypesEqual : Constraint, "operands are equal">; - --@@ -100,6 +102,8 @@ -- def ZeroExtent : AttrConstraint< -+ /////////// -+@@ -101,6 +103,8 @@ def ZeroExtent : AttrConstraint< - CPred<"cast($_self).getNumElements() == 0">, - "is zero extent">; --+ --+def AnyStaticShapeIntTensor : StaticShapeTensorOf<[HLO_Int]>; - -++def AnyStaticShapeIntTensor : StaticShapeTensorOf<[HLO_Int]>; -++ - /////////// - //// Native Code Call Utilities --@@ -503,7 +507,7 @@ -+ -+@@ -503,7 +507,7 @@ def SelectOp_InvertBroadcastPredicateAndSwap - // Must be static shape, otherwise would require broadcasting via - // CHLO_ConstantLike. - def SubtractOp_FoldToZero -@@ -2158,4 +2179,6 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveSimp - (StableHLO_ConstantLike<"0"> $operand)>; - - // Pattern: subtract(X, 0) -> X -+-- -+2.48.1 - --- -2.48.1 - diff --git a/third_party/modules/xla/20250612.0-6e48cbb/source.json b/third_party/modules/xla/20250612.0-6e48cbb/source.json deleted file mode 100644 index 6a4e208b..00000000 --- a/third_party/modules/xla/20250612.0-6e48cbb/source.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "strip_prefix": "xla-6e48cbb8d33d771c964697e39bfaf678bcc6de31", - "url": "https://github.com/openxla/xla/archive/6e48cbb8d33d771c964697e39bfaf678bcc6de31.tar.gz", - "integrity": "sha256-i9lYvZ2MkzfyVW2Iu3qIucXIgGEhkbwsYXCrUZ6Yze8=", - "overlay": { - "llvm.bzl": "", - "MODULE.bazel": "", - "workspace_private.bzl": "", - "xla.bzl": "" - }, - "patch_strip": 1, - "patches": { - "0001-bazel-migration-to-bazel-8.1.1.patch": "", - "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "", - "0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch": "" - } -} diff --git a/third_party/modules/xla/20250710.0-22ea002/MODULE.bazel b/third_party/modules/xla/20250710.0-22ea002/MODULE.bazel deleted file mode 100644 index 09bc2307..00000000 --- a/third_party/modules/xla/20250710.0-22ea002/MODULE.bazel +++ /dev/null @@ -1,58 +0,0 @@ -module( - name = "xla", - version = "20250710.0-22ea002", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.17") -bazel_dep(name = "rules_apple", version = "3.22.0", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.39.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") -bazel_dep(name = "rules_shell", version = "0.4.1") -bazel_dep(name = "bazel_features", version = "1.25.0", repo_name = "proto_bazel_features") - -toolchains_private = use_extension("//:toolchains_private.bzl", "toolchains_private") -use_repo( - toolchains_private, - "rules_ml_toolchain", -) - -workspace_private = use_extension("//:workspace_private.bzl", "workspace_private") -use_repo( - workspace_private, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", - "python_version_repo", - "tsl", -) - -workspace_public = use_extension("//:xla.bzl", "xla") -use_repo( - workspace_public, - "llvm-raw", - "stablehlo", - "triton", -) - -llvm = use_extension("//:llvm.bzl", "llvm") -llvm.configure( - targets = [ - "AArch64", - "AMDGPU", - "NVPTX", - "X86", - ], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/xla/20250710.0-22ea002/overlay/MODULE.bazel b/third_party/modules/xla/20250710.0-22ea002/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20250710.0-22ea002/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250710.0-22ea002/overlay/llvm.bzl b/third_party/modules/xla/20250710.0-22ea002/overlay/llvm.bzl deleted file mode 100644 index b4a2fe4f..00000000 --- a/third_party/modules/xla/20250710.0-22ea002/overlay/llvm.bzl +++ /dev/null @@ -1,30 +0,0 @@ -load("@llvm-raw//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") - -def _llvm_impl(mctx): - _targets = {} - for mod in mctx.modules: - for conf in mod.tags.configure: - for target in conf.targets: - _targets[target] = True - _llvm_configure( - name = "llvm-project", - targets = _targets.keys(), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -llvm = module_extension( - implementation = _llvm_impl, - tag_classes = { - "configure": tag_class( - attrs = { - "targets": attr.string_list( - default = [], - ), - }, - ), - }, -) diff --git a/third_party/modules/xla/20250710.0-22ea002/overlay/toolchains_private.bzl b/third_party/modules/xla/20250710.0-22ea002/overlay/toolchains_private.bzl deleted file mode 100644 index b8c79070..00000000 --- a/third_party/modules/xla/20250710.0-22ea002/overlay/toolchains_private.bzl +++ /dev/null @@ -1,21 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -def _toolchains_private_impl(mctx): - http_archive( - name = "rules_ml_toolchain", - sha256 = "fb78d09234528aef2be856820b69b76486829f65e4eb3c7ffaa5803b667fa441", - strip_prefix = "rules_ml_toolchain-f4ad89fa906be2c1374785a79335c8a7dcd49df7", - urls = [ - "https://github.com/zml/rules_ml_toolchain/archive/f4ad89fa906be2c1374785a79335c8a7dcd49df7.tar.gz", - ], - ) - - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -toolchains_private = module_extension( - implementation = _toolchains_private_impl, -) diff --git a/third_party/modules/xla/20250710.0-22ea002/overlay/workspace_private.bzl b/third_party/modules/xla/20250710.0-22ea002/overlay/workspace_private.bzl deleted file mode 100644 index 9dfe0b16..00000000 --- a/third_party/modules/xla/20250710.0-22ea002/overlay/workspace_private.bzl +++ /dev/null @@ -1,73 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls", "tf_vendored") -load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("//third_party/py:python_repo.bzl", "python_repository") -load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") -load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _workspace_private_impl(mctx): - http_archive( - name = "rules_ml_toolchain", - sha256 = "fb78d09234528aef2be856820b69b76486829f65e4eb3c7ffaa5803b667fa441", - strip_prefix = "rules_ml_toolchain-f4ad89fa906be2c1374785a79335c8a7dcd49df7", - urls = [ - "https://github.com/zml/rules_ml_toolchain/archive/f4ad89fa906be2c1374785a79335c8a7dcd49df7.tar.gz", - ], - ) - - # Use cuda_configure from XLA to make it work with bzlmod. - # A pure bzlmod solution for rules_ml_toolchain is impossible because of the legacy design. - # It relies on a "generate-then-load" pattern that creates a deadlock in Bazel's architecture: - # - Generate: First, it runs a rule to generate a .bzl file containing configuration data. - # - Load: Then, it requires a load() statement to load that same file to continue the setup. - # This fails in bzlmod because Bazel's Loading Phase (when load() statements are processed) happens before - # the Analysis Phase (when repository rules are run). - # This creates a fundamental chicken-and-egg problem: the build tries to load a file that has not been generated yet. - # Without using the official WORKSPACE.bzlmod escape hatch, - # this incompatibility cannot be resolved without modifying the upstream rules. - - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_vendored(name = "tsl", relpath = "third_party/tsl") - pybind11_bazel() - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "afbc5d78d6ba6d509cc6e264de0d49dcd7304db435cbf2d630385bacf49e066c", - strip_prefix = "grpc-1.68.2", - patch_file = [ - "//third_party/grpc:grpc.patch", - ], - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/refs/tags/v1.68.2.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["//third_party/protobuf:protobuf.patch"], - sha256 = "f645e6e42745ce922ca5388b1883ca583bafe4366cc74cf35c3c9299005136e2", - strip_prefix = "protobuf-5.28.3", - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/refs/tags/v5.28.3.zip"), - ) - - python_repository( - name = "python_version_repo", - requirements_versions = ["3.11"], - requirements_locks = ["//:requirements_lock_3_11.txt"], - local_wheel_workspaces = [], - local_wheel_dist_folder = None, - default_python_version = None, - local_wheel_inclusion_list = ["*"], - local_wheel_exclusion_list = [], - ) - - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -workspace_private = module_extension( - implementation = _workspace_private_impl, -) diff --git a/third_party/modules/xla/20250710.0-22ea002/overlay/xla.bzl b/third_party/modules/xla/20250710.0-22ea002/overlay/xla.bzl deleted file mode 100644 index f14bf2a9..00000000 --- a/third_party/modules/xla/20250710.0-22ea002/overlay/xla.bzl +++ /dev/null @@ -1,17 +0,0 @@ -load("//third_party/llvm:workspace.bzl", llvm = "repo") -load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") -load("//third_party/triton:workspace.bzl", triton = "repo") - -def _xla_impl(mctx): - triton() - llvm("llvm-raw") - stablehlo() - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla = module_extension( - implementation = _xla_impl, -) diff --git a/third_party/modules/xla/20250710.0-22ea002/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/modules/xla/20250710.0-22ea002/patches/0001-bazel-migration-to-bazel-8.1.1.patch deleted file mode 100644 index 8924cf4b..00000000 --- a/third_party/modules/xla/20250710.0-22ea002/patches/0001-bazel-migration-to-bazel-8.1.1.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 -From: Jean-Baptiste Dalido -Date: Mon, 6 Jan 2025 13:33:13 +0100 -Subject: [PATCH] bazel: migration to bazel 8.0.1 - ---- - .bazelversion | 2 +- - third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/.bazelversion b/.bazelversion -index f22d756da3..fa5fce04b3 100644 ---- a/.bazelversion -+++ b/.bazelversion -@@ -1 +1 @@ --7.4.1 -+8.1.1 -\ No newline at end of file -diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl -index d62531152d..71d80a5a99 100644 ---- a/third_party/gpus/cuda_configure.bzl -+++ b/third_party/gpus/cuda_configure.bzl -@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. - load( - "@bazel_tools//tools/cpp:lib_cc_configure.bzl", - "escape_string", -- "get_env_var", - ) - load( - "@bazel_tools//tools/cpp:windows_cc_configure.bzl", -- "find_msvc_tool", - "find_vc_path", - "setup_vc_env_vars", - ) -+load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") -+load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") - load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") - load( - "//third_party/remote_config:common.bzl", --- -2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250710.0-22ea002/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250710.0-22ea002/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch deleted file mode 100644 index 938ef401..00000000 --- a/third_party/modules/xla/20250710.0-22ea002/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch +++ /dev/null @@ -1,135 +0,0 @@ -From 2ae9bb9d24b569c2c6bfab3c54b428103614944d Mon Sep 17 00:00:00 2001 -From: Hugo Mano -Date: Tue, 27 May 2025 11:48:17 +0200 -Subject: [PATCH 1/8] Added FFI handler registration API to the FFI PjRt - -PR: https://github.com/openxla/xla/pull/13420 ---- - xla/pjrt/c/BUILD | 5 +++++ - xla/pjrt/c/pjrt_c_api_ffi_extension.h | 21 ++++++++++++++++++ - xla/pjrt/c/pjrt_c_api_ffi_internal.cc | 32 ++++++++++++++++++++++++++- - 3 files changed, 57 insertions(+), 1 deletion(-) - -diff --git a/xla/pjrt/c/BUILD b/xla/pjrt/c/BUILD -index 79f18fa0bc..0f33dd8a6e 100644 ---- a/xla/pjrt/c/BUILD -+++ b/xla/pjrt/c/BUILD -@@ -69,8 +69,13 @@ cc_library( - ":pjrt_c_api_wrapper_impl", - "//xla/ffi:execution_context", - "//xla/ffi:type_id_registry", -+ "//xla/ffi:ffi_api", -+ "//xla/ffi/api:c_api", -+ "//xla/ffi/api:ffi", -+ "//xla/service:custom_call_target_registry", - "@com_google_absl//absl/status", - "@com_google_absl//absl/strings:string_view", -+ "@com_google_absl//absl/strings:str_format", - ], - ) - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -index 995a2c7e50..b8f10bc2f7 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -@@ -69,10 +69,31 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); - // Adds a user data to the execute context. - typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); - -+typedef enum PJRT_FFI_Handler_TraitsBits { -+ PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE = 1u << 0, -+} PJRT_FFI_Handler_TraitsBits; -+ -+struct PJRT_FFI_Register_Handler_Args { -+ size_t struct_size; -+ const char* target_name; -+ size_t target_name_size; -+ int api_version; // 0 for an untyped call, 1 -- for typed -+ void* handler; -+ const char* platform_name; -+ size_t platform_name_size; -+ PJRT_FFI_Handler_TraitsBits traits; -+}; -+PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, traits); -+ -+// Registers an FFI call handler for a specific platform. -+typedef PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args); -+ - typedef struct PJRT_FFI_Extension { - PJRT_Extension_Base base; - PJRT_FFI_TypeID_Register* type_id_register; - PJRT_FFI_UserData_Add* user_data_add; -+ PJRT_FFI_Register_Handler* register_handler; - } PJRT_FFI; - PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Extension, user_data_add); - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -index 5fa88eab33..763270331b 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -+++ b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -@@ -13,16 +13,20 @@ See the License for the specific language governing permissions and - limitations under the License. - ==============================================================================*/ - --#include "xla/pjrt/c/pjrt_c_api_ffi_internal.h" -+#include - - #include "absl/status/status.h" -+#include "absl/strings/str_format.h" - #include "absl/strings/string_view.h" -+#include "xla/ffi/api/c_api.h" - #include "xla/ffi/execution_context.h" - #include "xla/ffi/type_id_registry.h" -+#include "xla/ffi/ffi_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" - #include "xla/pjrt/c/pjrt_c_api_ffi_extension.h" - #include "xla/pjrt/c/pjrt_c_api_helpers.h" - #include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" -+#include "xla/service/custom_call_target_registry.h" - - namespace pjrt { - -@@ -68,6 +72,31 @@ static PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args) { - return nullptr; - } - -+static PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args) { -+ PJRT_RETURN_IF_ERROR(ActualStructSizeIsGreaterOrEqual( -+ "PJRT_FFI_Register_Handler_Args", -+ PJRT_FFI_Register_Handler_Args_STRUCT_SIZE, args->struct_size)); -+ std::string target_name(args->target_name, args->target_name_size); -+ std::string platform_name(args->platform_name, args->platform_name_size); -+ switch (args->api_version) { -+ case 0: -+ xla::CustomCallTargetRegistry::Global()->Register( -+ target_name, args->handler, platform_name); -+ return nullptr; -+ case 1: -+ xla::ffi::Ffi::RegisterStaticHandler( -+ xla::ffi::GetXlaFfiApi(), target_name, platform_name, -+ reinterpret_cast(args->handler)); -+ return nullptr; -+ default: -+ return new PJRT_Error{absl::UnimplementedError( -+ absl::StrFormat("API version %d not supported for PJRT GPU plugin. " -+ "Supported versions are 0 and 1.", -+ args->api_version))}; -+ } -+} -+ - PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - return { - PJRT_Extension_Base{ -@@ -77,6 +106,7 @@ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - }, - /*type_id_register=*/PJRT_FFI_TypeID_Register, - /*user_data_add=*/PJRT_FFI_UserData_Add, -+ /*register_handler=*/PJRT_FFI_Register_Handler, - }; - } - --- -2.39.5 (Apple Git-154) - diff --git a/third_party/modules/xla/20250710.0-22ea002/patches/0003-Remove-unconventional-C-code-in-headers.patch b/third_party/modules/xla/20250710.0-22ea002/patches/0003-Remove-unconventional-C-code-in-headers.patch deleted file mode 100644 index 6df5b1b3..00000000 --- a/third_party/modules/xla/20250710.0-22ea002/patches/0003-Remove-unconventional-C-code-in-headers.patch +++ /dev/null @@ -1,124 +0,0 @@ -From 6078da86a46b6f0d983dccb9ae4f36fc90640247 Mon Sep 17 00:00:00 2001 -From: Hugo Mano -Date: Fri, 11 Jul 2025 14:05:16 +0200 -Subject: [PATCH] zml patch - ---- - third_party/stablehlo/workspace.bzl | 1 + - third_party/stablehlo/zml.patch | 93 +++++++++++++++++++++++++++++ - 2 files changed, 94 insertions(+) - create mode 100644 third_party/stablehlo/zml.patch - -diff --git a/third_party/stablehlo/workspace.bzl b/third_party/stablehlo/workspace.bzl -index d9d5063744..44980948d0 100644 ---- a/third_party/stablehlo/workspace.bzl -+++ b/third_party/stablehlo/workspace.bzl -@@ -15,5 +15,6 @@ def repo(): - urls = tf_mirror_urls("https://github.com/openxla/stablehlo/archive/{commit}.zip".format(commit = STABLEHLO_COMMIT)), - patch_file = [ - "//third_party/stablehlo:temporary.patch", # Autogenerated, don't remove. -+ "//third_party/stablehlo:zml.patch", # Autogenerated, don't remove. - ], - ) -diff --git a/third_party/stablehlo/zml.patch b/third_party/stablehlo/zml.patch -new file mode 100644 -index 0000000000..2a09384582 ---- /dev/null -+++ b/third_party/stablehlo/zml.patch -@@ -0,0 +1,93 @@ -+From e38ab68376dd8a17ebf4469d2c8350f521310182 Mon Sep 17 00:00:00 2001 -+From: Hugo Mano -+Date: Fri, 11 Jul 2025 12:08:35 +0200 -+Subject: [PATCH] zml patch -+ -+--- -+ stablehlo/dialect/Serialization.cpp | 5 ++--- -+ stablehlo/dialect/Serialization.h | 3 +-- -+ stablehlo/integrations/c/StablehloDialectApi.cpp | 3 +-- -+ stablehlo/integrations/c/StablehloDialectApi.h | 2 +- -+ stablehlo/tools/StablehloTranslateMain.cpp | 2 +- -+ 5 files changed, 6 insertions(+), 9 deletions(-) -+ -+diff --git a/stablehlo/dialect/Serialization.cpp b/stablehlo/dialect/Serialization.cpp -+index cb89d673..4370d588 100644 -+--- a/stablehlo/dialect/Serialization.cpp -++++ b/stablehlo/dialect/Serialization.cpp -+@@ -39,8 +39,7 @@ namespace stablehlo { -+ -+ LogicalResult serializePortableArtifact(ModuleOp module, -+ StringRef targetVersion, -+- raw_ostream& os, -+- bool allowOtherDialects) { -++ raw_ostream& os) { -+ MLIRContext* context = module.getContext(); -+ -+ // Convert StableHLO --> VHLO. -+@@ -49,7 +48,7 @@ LogicalResult serializePortableArtifact(ModuleOp module, -+ { -+ PassManager pm(context); -+ StablehloLegalizeToVhloPassOptions options; -+- options.allowOtherDialects = allowOtherDialects; -++ options.allowOtherDialects = false; -+ pm.addPass(stablehlo::createStablehloLegalizeToVhloPass(options)); -+ if (!succeeded(pm.run(module))) { -+ return failure(); -+diff --git a/stablehlo/dialect/Serialization.h b/stablehlo/dialect/Serialization.h -+index 811ca97b..abe95e63 100644 -+--- a/stablehlo/dialect/Serialization.h -++++ b/stablehlo/dialect/Serialization.h -+@@ -34,8 +34,7 @@ namespace stablehlo { -+ // unsupported dialects. -+ LogicalResult serializePortableArtifact(ModuleOp module, -+ StringRef targetVersion, -+- raw_ostream& os, -+- bool allowOtherDialects = false); -++ raw_ostream& os); -+ -+ // Read StableHLO portable artifact -+ // -+diff --git a/stablehlo/integrations/c/StablehloDialectApi.cpp b/stablehlo/integrations/c/StablehloDialectApi.cpp -+index 343f8d0b..8f52e4d5 100644 -+--- a/stablehlo/integrations/c/StablehloDialectApi.cpp -++++ b/stablehlo/integrations/c/StablehloDialectApi.cpp -+@@ -81,8 +81,7 @@ MlirLogicalResult stablehloSerializePortableArtifactFromModule( -+ MlirStringCallback callback, void *userData, bool allowOtherDialects) { -+ mlir::detail::CallbackOstream stream(callback, userData); -+ if (failed(mlir::stablehlo::serializePortableArtifact( -+- unwrap(moduleStr), unwrap(targetVersion), stream, -+- allowOtherDialects))) -++ unwrap(moduleStr), unwrap(targetVersion), stream))) -+ return mlirLogicalResultFailure(); -+ return mlirLogicalResultSuccess(); -+ } -+diff --git a/stablehlo/integrations/c/StablehloDialectApi.h b/stablehlo/integrations/c/StablehloDialectApi.h -+index 385156bf..24d11c1d 100644 -+--- a/stablehlo/integrations/c/StablehloDialectApi.h -++++ b/stablehlo/integrations/c/StablehloDialectApi.h -+@@ -93,7 +93,7 @@ stablehloSerializePortableArtifactFromModule(MlirModule moduleStr, -+ MlirStringRef targetVersion, -+ MlirStringCallback callback, -+ void* userData, -+- bool allowOtherDialects = false); -++ bool allowOtherDialects); -+ -+ // Read a StableHLO program from a portable artifact, returning the module as -+ // MLIR bytecode. Note, this bytecode returned is not a portable artifact, -+diff --git a/stablehlo/tools/StablehloTranslateMain.cpp b/stablehlo/tools/StablehloTranslateMain.cpp -+index fdf0d6a9..8d5c8752 100644 -+--- a/stablehlo/tools/StablehloTranslateMain.cpp -++++ b/stablehlo/tools/StablehloTranslateMain.cpp -+@@ -323,7 +323,7 @@ TranslateFromMLIRRegistration serializeRegistration( -+ } -+ -+ return stablehlo::serializePortableArtifact( -+- module, targetVersion, os, allowOtherDialectsOption.getValue()); -++ module, targetVersion, os); -+ }, -+ [](DialectRegistry ®istry) { -+ mlir::registerAllDialects(registry); -+-- -+2.39.5 (Apple Git-154) -+ --- -2.39.5 (Apple Git-154) - diff --git a/third_party/modules/xla/20250710.0-22ea002/source.json b/third_party/modules/xla/20250710.0-22ea002/source.json deleted file mode 100644 index d01c18a3..00000000 --- a/third_party/modules/xla/20250710.0-22ea002/source.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "strip_prefix": "xla-ef07e787ea1303fa2f8d8a175d24d434bfb84107", - "url": "https://github.com/openxla/xla/archive/ef07e787ea1303fa2f8d8a175d24d434bfb84107.tar.gz", - "integrity": "sha256-OL0e3Y+dgu/DM5MV+LsWvtOI9rUGLvXbwIU/9bnHGXc=", - "overlay": { - "toolchains_private.bzl": "", - "llvm.bzl": "", - "MODULE.bazel": "", - "workspace_private.bzl": "", - "xla.bzl": "" - }, - "patch_strip": 1, - "patches": { - "0001-bazel-migration-to-bazel-8.1.1.patch": "", - "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "", - "0003-Remove-unconventional-C-code-in-headers.patch": "" - } -} diff --git a/third_party/modules/xla/20250718.0-6319f0d/MODULE.bazel b/third_party/modules/xla/20250718.0-6319f0d/MODULE.bazel deleted file mode 100644 index 6c1bde70..00000000 --- a/third_party/modules/xla/20250718.0-6319f0d/MODULE.bazel +++ /dev/null @@ -1,58 +0,0 @@ -module( - name = "xla", - version = "20250718.0-6319f0d", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.17") -bazel_dep(name = "rules_apple", version = "3.22.0", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.39.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") -bazel_dep(name = "rules_shell", version = "0.4.1") -bazel_dep(name = "bazel_features", version = "1.25.0", repo_name = "proto_bazel_features") - -toolchains_private = use_extension("//:toolchains_private.bzl", "toolchains_private") -use_repo( - toolchains_private, - "rules_ml_toolchain", -) - -workspace_private = use_extension("//:workspace_private.bzl", "workspace_private") -use_repo( - workspace_private, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", - "python_version_repo", - "tsl", -) - -workspace_public = use_extension("//:xla.bzl", "xla") -use_repo( - workspace_public, - "llvm-raw", - "stablehlo", - "triton", -) - -llvm = use_extension("//:llvm.bzl", "llvm") -llvm.configure( - targets = [ - "AArch64", - "AMDGPU", - "NVPTX", - "X86", - ], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/xla/20250718.0-6319f0d/overlay/MODULE.bazel b/third_party/modules/xla/20250718.0-6319f0d/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20250718.0-6319f0d/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250718.0-6319f0d/overlay/llvm.bzl b/third_party/modules/xla/20250718.0-6319f0d/overlay/llvm.bzl deleted file mode 100644 index b4a2fe4f..00000000 --- a/third_party/modules/xla/20250718.0-6319f0d/overlay/llvm.bzl +++ /dev/null @@ -1,30 +0,0 @@ -load("@llvm-raw//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") - -def _llvm_impl(mctx): - _targets = {} - for mod in mctx.modules: - for conf in mod.tags.configure: - for target in conf.targets: - _targets[target] = True - _llvm_configure( - name = "llvm-project", - targets = _targets.keys(), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -llvm = module_extension( - implementation = _llvm_impl, - tag_classes = { - "configure": tag_class( - attrs = { - "targets": attr.string_list( - default = [], - ), - }, - ), - }, -) diff --git a/third_party/modules/xla/20250718.0-6319f0d/overlay/toolchains_private.bzl b/third_party/modules/xla/20250718.0-6319f0d/overlay/toolchains_private.bzl deleted file mode 100644 index b8c79070..00000000 --- a/third_party/modules/xla/20250718.0-6319f0d/overlay/toolchains_private.bzl +++ /dev/null @@ -1,21 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -def _toolchains_private_impl(mctx): - http_archive( - name = "rules_ml_toolchain", - sha256 = "fb78d09234528aef2be856820b69b76486829f65e4eb3c7ffaa5803b667fa441", - strip_prefix = "rules_ml_toolchain-f4ad89fa906be2c1374785a79335c8a7dcd49df7", - urls = [ - "https://github.com/zml/rules_ml_toolchain/archive/f4ad89fa906be2c1374785a79335c8a7dcd49df7.tar.gz", - ], - ) - - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -toolchains_private = module_extension( - implementation = _toolchains_private_impl, -) diff --git a/third_party/modules/xla/20250718.0-6319f0d/overlay/workspace_private.bzl b/third_party/modules/xla/20250718.0-6319f0d/overlay/workspace_private.bzl deleted file mode 100644 index 9dfe0b16..00000000 --- a/third_party/modules/xla/20250718.0-6319f0d/overlay/workspace_private.bzl +++ /dev/null @@ -1,73 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls", "tf_vendored") -load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("//third_party/py:python_repo.bzl", "python_repository") -load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") -load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _workspace_private_impl(mctx): - http_archive( - name = "rules_ml_toolchain", - sha256 = "fb78d09234528aef2be856820b69b76486829f65e4eb3c7ffaa5803b667fa441", - strip_prefix = "rules_ml_toolchain-f4ad89fa906be2c1374785a79335c8a7dcd49df7", - urls = [ - "https://github.com/zml/rules_ml_toolchain/archive/f4ad89fa906be2c1374785a79335c8a7dcd49df7.tar.gz", - ], - ) - - # Use cuda_configure from XLA to make it work with bzlmod. - # A pure bzlmod solution for rules_ml_toolchain is impossible because of the legacy design. - # It relies on a "generate-then-load" pattern that creates a deadlock in Bazel's architecture: - # - Generate: First, it runs a rule to generate a .bzl file containing configuration data. - # - Load: Then, it requires a load() statement to load that same file to continue the setup. - # This fails in bzlmod because Bazel's Loading Phase (when load() statements are processed) happens before - # the Analysis Phase (when repository rules are run). - # This creates a fundamental chicken-and-egg problem: the build tries to load a file that has not been generated yet. - # Without using the official WORKSPACE.bzlmod escape hatch, - # this incompatibility cannot be resolved without modifying the upstream rules. - - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_vendored(name = "tsl", relpath = "third_party/tsl") - pybind11_bazel() - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "afbc5d78d6ba6d509cc6e264de0d49dcd7304db435cbf2d630385bacf49e066c", - strip_prefix = "grpc-1.68.2", - patch_file = [ - "//third_party/grpc:grpc.patch", - ], - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/refs/tags/v1.68.2.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["//third_party/protobuf:protobuf.patch"], - sha256 = "f645e6e42745ce922ca5388b1883ca583bafe4366cc74cf35c3c9299005136e2", - strip_prefix = "protobuf-5.28.3", - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/refs/tags/v5.28.3.zip"), - ) - - python_repository( - name = "python_version_repo", - requirements_versions = ["3.11"], - requirements_locks = ["//:requirements_lock_3_11.txt"], - local_wheel_workspaces = [], - local_wheel_dist_folder = None, - default_python_version = None, - local_wheel_inclusion_list = ["*"], - local_wheel_exclusion_list = [], - ) - - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -workspace_private = module_extension( - implementation = _workspace_private_impl, -) diff --git a/third_party/modules/xla/20250718.0-6319f0d/overlay/xla.bzl b/third_party/modules/xla/20250718.0-6319f0d/overlay/xla.bzl deleted file mode 100644 index f14bf2a9..00000000 --- a/third_party/modules/xla/20250718.0-6319f0d/overlay/xla.bzl +++ /dev/null @@ -1,17 +0,0 @@ -load("//third_party/llvm:workspace.bzl", llvm = "repo") -load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") -load("//third_party/triton:workspace.bzl", triton = "repo") - -def _xla_impl(mctx): - triton() - llvm("llvm-raw") - stablehlo() - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla = module_extension( - implementation = _xla_impl, -) diff --git a/third_party/modules/xla/20250718.0-6319f0d/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/modules/xla/20250718.0-6319f0d/patches/0001-bazel-migration-to-bazel-8.1.1.patch deleted file mode 100644 index 8924cf4b..00000000 --- a/third_party/modules/xla/20250718.0-6319f0d/patches/0001-bazel-migration-to-bazel-8.1.1.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 -From: Jean-Baptiste Dalido -Date: Mon, 6 Jan 2025 13:33:13 +0100 -Subject: [PATCH] bazel: migration to bazel 8.0.1 - ---- - .bazelversion | 2 +- - third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/.bazelversion b/.bazelversion -index f22d756da3..fa5fce04b3 100644 ---- a/.bazelversion -+++ b/.bazelversion -@@ -1 +1 @@ --7.4.1 -+8.1.1 -\ No newline at end of file -diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl -index d62531152d..71d80a5a99 100644 ---- a/third_party/gpus/cuda_configure.bzl -+++ b/third_party/gpus/cuda_configure.bzl -@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. - load( - "@bazel_tools//tools/cpp:lib_cc_configure.bzl", - "escape_string", -- "get_env_var", - ) - load( - "@bazel_tools//tools/cpp:windows_cc_configure.bzl", -- "find_msvc_tool", - "find_vc_path", - "setup_vc_env_vars", - ) -+load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") -+load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") - load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") - load( - "//third_party/remote_config:common.bzl", --- -2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250718.0-6319f0d/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250718.0-6319f0d/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch deleted file mode 100644 index 938ef401..00000000 --- a/third_party/modules/xla/20250718.0-6319f0d/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch +++ /dev/null @@ -1,135 +0,0 @@ -From 2ae9bb9d24b569c2c6bfab3c54b428103614944d Mon Sep 17 00:00:00 2001 -From: Hugo Mano -Date: Tue, 27 May 2025 11:48:17 +0200 -Subject: [PATCH 1/8] Added FFI handler registration API to the FFI PjRt - -PR: https://github.com/openxla/xla/pull/13420 ---- - xla/pjrt/c/BUILD | 5 +++++ - xla/pjrt/c/pjrt_c_api_ffi_extension.h | 21 ++++++++++++++++++ - xla/pjrt/c/pjrt_c_api_ffi_internal.cc | 32 ++++++++++++++++++++++++++- - 3 files changed, 57 insertions(+), 1 deletion(-) - -diff --git a/xla/pjrt/c/BUILD b/xla/pjrt/c/BUILD -index 79f18fa0bc..0f33dd8a6e 100644 ---- a/xla/pjrt/c/BUILD -+++ b/xla/pjrt/c/BUILD -@@ -69,8 +69,13 @@ cc_library( - ":pjrt_c_api_wrapper_impl", - "//xla/ffi:execution_context", - "//xla/ffi:type_id_registry", -+ "//xla/ffi:ffi_api", -+ "//xla/ffi/api:c_api", -+ "//xla/ffi/api:ffi", -+ "//xla/service:custom_call_target_registry", - "@com_google_absl//absl/status", - "@com_google_absl//absl/strings:string_view", -+ "@com_google_absl//absl/strings:str_format", - ], - ) - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -index 995a2c7e50..b8f10bc2f7 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -@@ -69,10 +69,31 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); - // Adds a user data to the execute context. - typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); - -+typedef enum PJRT_FFI_Handler_TraitsBits { -+ PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE = 1u << 0, -+} PJRT_FFI_Handler_TraitsBits; -+ -+struct PJRT_FFI_Register_Handler_Args { -+ size_t struct_size; -+ const char* target_name; -+ size_t target_name_size; -+ int api_version; // 0 for an untyped call, 1 -- for typed -+ void* handler; -+ const char* platform_name; -+ size_t platform_name_size; -+ PJRT_FFI_Handler_TraitsBits traits; -+}; -+PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, traits); -+ -+// Registers an FFI call handler for a specific platform. -+typedef PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args); -+ - typedef struct PJRT_FFI_Extension { - PJRT_Extension_Base base; - PJRT_FFI_TypeID_Register* type_id_register; - PJRT_FFI_UserData_Add* user_data_add; -+ PJRT_FFI_Register_Handler* register_handler; - } PJRT_FFI; - PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Extension, user_data_add); - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -index 5fa88eab33..763270331b 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -+++ b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -@@ -13,16 +13,20 @@ See the License for the specific language governing permissions and - limitations under the License. - ==============================================================================*/ - --#include "xla/pjrt/c/pjrt_c_api_ffi_internal.h" -+#include - - #include "absl/status/status.h" -+#include "absl/strings/str_format.h" - #include "absl/strings/string_view.h" -+#include "xla/ffi/api/c_api.h" - #include "xla/ffi/execution_context.h" - #include "xla/ffi/type_id_registry.h" -+#include "xla/ffi/ffi_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" - #include "xla/pjrt/c/pjrt_c_api_ffi_extension.h" - #include "xla/pjrt/c/pjrt_c_api_helpers.h" - #include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" -+#include "xla/service/custom_call_target_registry.h" - - namespace pjrt { - -@@ -68,6 +72,31 @@ static PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args) { - return nullptr; - } - -+static PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args) { -+ PJRT_RETURN_IF_ERROR(ActualStructSizeIsGreaterOrEqual( -+ "PJRT_FFI_Register_Handler_Args", -+ PJRT_FFI_Register_Handler_Args_STRUCT_SIZE, args->struct_size)); -+ std::string target_name(args->target_name, args->target_name_size); -+ std::string platform_name(args->platform_name, args->platform_name_size); -+ switch (args->api_version) { -+ case 0: -+ xla::CustomCallTargetRegistry::Global()->Register( -+ target_name, args->handler, platform_name); -+ return nullptr; -+ case 1: -+ xla::ffi::Ffi::RegisterStaticHandler( -+ xla::ffi::GetXlaFfiApi(), target_name, platform_name, -+ reinterpret_cast(args->handler)); -+ return nullptr; -+ default: -+ return new PJRT_Error{absl::UnimplementedError( -+ absl::StrFormat("API version %d not supported for PJRT GPU plugin. " -+ "Supported versions are 0 and 1.", -+ args->api_version))}; -+ } -+} -+ - PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - return { - PJRT_Extension_Base{ -@@ -77,6 +106,7 @@ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - }, - /*type_id_register=*/PJRT_FFI_TypeID_Register, - /*user_data_add=*/PJRT_FFI_UserData_Add, -+ /*register_handler=*/PJRT_FFI_Register_Handler, - }; - } - --- -2.39.5 (Apple Git-154) - diff --git a/third_party/modules/xla/20250718.0-6319f0d/patches/0003-Remove-unconventional-C-code-in-headers.patch b/third_party/modules/xla/20250718.0-6319f0d/patches/0003-Remove-unconventional-C-code-in-headers.patch deleted file mode 100644 index 6df5b1b3..00000000 --- a/third_party/modules/xla/20250718.0-6319f0d/patches/0003-Remove-unconventional-C-code-in-headers.patch +++ /dev/null @@ -1,124 +0,0 @@ -From 6078da86a46b6f0d983dccb9ae4f36fc90640247 Mon Sep 17 00:00:00 2001 -From: Hugo Mano -Date: Fri, 11 Jul 2025 14:05:16 +0200 -Subject: [PATCH] zml patch - ---- - third_party/stablehlo/workspace.bzl | 1 + - third_party/stablehlo/zml.patch | 93 +++++++++++++++++++++++++++++ - 2 files changed, 94 insertions(+) - create mode 100644 third_party/stablehlo/zml.patch - -diff --git a/third_party/stablehlo/workspace.bzl b/third_party/stablehlo/workspace.bzl -index d9d5063744..44980948d0 100644 ---- a/third_party/stablehlo/workspace.bzl -+++ b/third_party/stablehlo/workspace.bzl -@@ -15,5 +15,6 @@ def repo(): - urls = tf_mirror_urls("https://github.com/openxla/stablehlo/archive/{commit}.zip".format(commit = STABLEHLO_COMMIT)), - patch_file = [ - "//third_party/stablehlo:temporary.patch", # Autogenerated, don't remove. -+ "//third_party/stablehlo:zml.patch", # Autogenerated, don't remove. - ], - ) -diff --git a/third_party/stablehlo/zml.patch b/third_party/stablehlo/zml.patch -new file mode 100644 -index 0000000000..2a09384582 ---- /dev/null -+++ b/third_party/stablehlo/zml.patch -@@ -0,0 +1,93 @@ -+From e38ab68376dd8a17ebf4469d2c8350f521310182 Mon Sep 17 00:00:00 2001 -+From: Hugo Mano -+Date: Fri, 11 Jul 2025 12:08:35 +0200 -+Subject: [PATCH] zml patch -+ -+--- -+ stablehlo/dialect/Serialization.cpp | 5 ++--- -+ stablehlo/dialect/Serialization.h | 3 +-- -+ stablehlo/integrations/c/StablehloDialectApi.cpp | 3 +-- -+ stablehlo/integrations/c/StablehloDialectApi.h | 2 +- -+ stablehlo/tools/StablehloTranslateMain.cpp | 2 +- -+ 5 files changed, 6 insertions(+), 9 deletions(-) -+ -+diff --git a/stablehlo/dialect/Serialization.cpp b/stablehlo/dialect/Serialization.cpp -+index cb89d673..4370d588 100644 -+--- a/stablehlo/dialect/Serialization.cpp -++++ b/stablehlo/dialect/Serialization.cpp -+@@ -39,8 +39,7 @@ namespace stablehlo { -+ -+ LogicalResult serializePortableArtifact(ModuleOp module, -+ StringRef targetVersion, -+- raw_ostream& os, -+- bool allowOtherDialects) { -++ raw_ostream& os) { -+ MLIRContext* context = module.getContext(); -+ -+ // Convert StableHLO --> VHLO. -+@@ -49,7 +48,7 @@ LogicalResult serializePortableArtifact(ModuleOp module, -+ { -+ PassManager pm(context); -+ StablehloLegalizeToVhloPassOptions options; -+- options.allowOtherDialects = allowOtherDialects; -++ options.allowOtherDialects = false; -+ pm.addPass(stablehlo::createStablehloLegalizeToVhloPass(options)); -+ if (!succeeded(pm.run(module))) { -+ return failure(); -+diff --git a/stablehlo/dialect/Serialization.h b/stablehlo/dialect/Serialization.h -+index 811ca97b..abe95e63 100644 -+--- a/stablehlo/dialect/Serialization.h -++++ b/stablehlo/dialect/Serialization.h -+@@ -34,8 +34,7 @@ namespace stablehlo { -+ // unsupported dialects. -+ LogicalResult serializePortableArtifact(ModuleOp module, -+ StringRef targetVersion, -+- raw_ostream& os, -+- bool allowOtherDialects = false); -++ raw_ostream& os); -+ -+ // Read StableHLO portable artifact -+ // -+diff --git a/stablehlo/integrations/c/StablehloDialectApi.cpp b/stablehlo/integrations/c/StablehloDialectApi.cpp -+index 343f8d0b..8f52e4d5 100644 -+--- a/stablehlo/integrations/c/StablehloDialectApi.cpp -++++ b/stablehlo/integrations/c/StablehloDialectApi.cpp -+@@ -81,8 +81,7 @@ MlirLogicalResult stablehloSerializePortableArtifactFromModule( -+ MlirStringCallback callback, void *userData, bool allowOtherDialects) { -+ mlir::detail::CallbackOstream stream(callback, userData); -+ if (failed(mlir::stablehlo::serializePortableArtifact( -+- unwrap(moduleStr), unwrap(targetVersion), stream, -+- allowOtherDialects))) -++ unwrap(moduleStr), unwrap(targetVersion), stream))) -+ return mlirLogicalResultFailure(); -+ return mlirLogicalResultSuccess(); -+ } -+diff --git a/stablehlo/integrations/c/StablehloDialectApi.h b/stablehlo/integrations/c/StablehloDialectApi.h -+index 385156bf..24d11c1d 100644 -+--- a/stablehlo/integrations/c/StablehloDialectApi.h -++++ b/stablehlo/integrations/c/StablehloDialectApi.h -+@@ -93,7 +93,7 @@ stablehloSerializePortableArtifactFromModule(MlirModule moduleStr, -+ MlirStringRef targetVersion, -+ MlirStringCallback callback, -+ void* userData, -+- bool allowOtherDialects = false); -++ bool allowOtherDialects); -+ -+ // Read a StableHLO program from a portable artifact, returning the module as -+ // MLIR bytecode. Note, this bytecode returned is not a portable artifact, -+diff --git a/stablehlo/tools/StablehloTranslateMain.cpp b/stablehlo/tools/StablehloTranslateMain.cpp -+index fdf0d6a9..8d5c8752 100644 -+--- a/stablehlo/tools/StablehloTranslateMain.cpp -++++ b/stablehlo/tools/StablehloTranslateMain.cpp -+@@ -323,7 +323,7 @@ TranslateFromMLIRRegistration serializeRegistration( -+ } -+ -+ return stablehlo::serializePortableArtifact( -+- module, targetVersion, os, allowOtherDialectsOption.getValue()); -++ module, targetVersion, os); -+ }, -+ [](DialectRegistry ®istry) { -+ mlir::registerAllDialects(registry); -+-- -+2.39.5 (Apple Git-154) -+ --- -2.39.5 (Apple Git-154) - diff --git a/third_party/modules/xla/20250718.0-6319f0d/source.json b/third_party/modules/xla/20250718.0-6319f0d/source.json deleted file mode 100644 index a34f9a9e..00000000 --- a/third_party/modules/xla/20250718.0-6319f0d/source.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "strip_prefix": "xla-6319f0d3bdfd3078e04bb984a759c890b7116484", - "url": "https://github.com/openxla/xla/archive/6319f0d3bdfd3078e04bb984a759c890b7116484.tar.gz", - "integrity": "sha256-0XAMbJ4tn5yFllPGVCSh8st7lgTOUoyeypoudCB8tsY=", - "overlay": { - "toolchains_private.bzl": "", - "llvm.bzl": "", - "MODULE.bazel": "", - "workspace_private.bzl": "", - "xla.bzl": "" - }, - "patch_strip": 1, - "patches": { - "0001-bazel-migration-to-bazel-8.1.1.patch": "", - "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "", - "0003-Remove-unconventional-C-code-in-headers.patch": "" - } -} diff --git a/third_party/modules/xla/metadata.json b/third_party/modules/xla/metadata.json deleted file mode 100644 index 4fa86cba..00000000 --- a/third_party/modules/xla/metadata.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "homepage": "https://github.com/openxla/xla", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": [ - "github:openxla/xla" - ], - "versions": [ - "20240902.0-d18cd64", - "20240919.0-1b18dd6", - "20241025.0-4663f04", - "20250103.0-5f1fe6a", - "20250122.0-cc075be", - "20250123.0-cc075be", - "20250204-0-6789523", - "20250317.0-71c67e2", - "20250317.1-71c67e2", - "20250317.2-71c67e2", - "20250527.0-cb67f2f", - "20250612.0-6e48cbb", - "20250710.0-22ea002", - "20250718.0-6319f0d" - ], - "yanked_versions": {} -} diff --git a/third_party/modules/zig-protobuf/20240722.0-c644d11/MODULE.bazel b/third_party/modules/zig-protobuf/20240722.0-c644d11/MODULE.bazel deleted file mode 100644 index 4fb39921..00000000 --- a/third_party/modules/zig-protobuf/20240722.0-c644d11/MODULE.bazel +++ /dev/null @@ -1,8 +0,0 @@ -module( - name = "zig-protobuf", - version = "20240722.0-c644d11", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") diff --git a/third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/BUILD.bazel b/third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/BUILD.bazel deleted file mode 100644 index 832c7591..00000000 --- a/third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/BUILD.bazel +++ /dev/null @@ -1,32 +0,0 @@ -load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") -load("@rules_zig//zig:defs.bzl", "BINARY_KIND", "zig_binary", "zig_library") - -zig_library( - name = "protobuf", - import_name = "protobuf", - main = "src/protobuf.zig", - visibility = ["//visibility:public"], -) - -zig_binary( - name = "generator", - srcs = [ - "bootstrapped-generator/FullName.zig", - "bootstrapped-generator/google/protobuf/compiler/plugin.pb.zig", - "bootstrapped-generator/google/protobuf/descriptor.pb.zig", - ], - kind = BINARY_KIND.exe, - main = "bootstrapped-generator/main.zig", - visibility = ["//visibility:public"], - deps = [":protobuf"], -) - -proto_lang_toolchain( - name = "zig_toolchain", - command_line = "--zig_out=$(OUT)", - output_files = "multiple", - plugin = ":generator", - plugin_format_flag = "--plugin=protoc-gen-zig=%s", - runtime = ":protobuf", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/MODULE.bazel b/third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/zig-protobuf/20240722.0-c644d11/patches/0001-USE_MODULES-true.patch b/third_party/modules/zig-protobuf/20240722.0-c644d11/patches/0001-USE_MODULES-true.patch deleted file mode 100644 index 67353922..00000000 --- a/third_party/modules/zig-protobuf/20240722.0-c644d11/patches/0001-USE_MODULES-true.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- a/bootstrapped-generator/main.zig -+++ b/bootstrapped-generator/main.zig -@@ -6,7 +6,7 @@ const descriptor = @import("google/protobuf/descriptor.pb.zig"); - const mem = std.mem; - const FullName = @import("./FullName.zig").FullName; - --const USE_MODULES = false; -+const USE_MODULES = true; - - const allocator = std.heap.page_allocator; - diff --git a/third_party/modules/zig-protobuf/20240722.0-c644d11/source.json b/third_party/modules/zig-protobuf/20240722.0-c644d11/source.json deleted file mode 100644 index f985b249..00000000 --- a/third_party/modules/zig-protobuf/20240722.0-c644d11/source.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "strip_prefix": "zig-protobuf-c644d1105caaca4d9a48d3536e253687052f74c8", - "url": "https://github.com/gwenzek/zig-protobuf/archive/c644d1105caaca4d9a48d3536e253687052f74c8.tar.gz", - "integrity": "sha256-KoZp8MLLEQQgS7Uip4sdjw7TYmhzKPdpilQYKp5iYV4=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0001-USE_MODULES-true.patch": "" - } -} diff --git a/third_party/modules/zig-protobuf/20250213.0-5304067/MODULE.bazel b/third_party/modules/zig-protobuf/20250213.0-5304067/MODULE.bazel deleted file mode 100644 index 0b9ec156..00000000 --- a/third_party/modules/zig-protobuf/20250213.0-5304067/MODULE.bazel +++ /dev/null @@ -1,8 +0,0 @@ -module( - name = "zig-protobuf", - version = "20250213.0-5304067", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") diff --git a/third_party/modules/zig-protobuf/20250213.0-5304067/overlay/BUILD.bazel b/third_party/modules/zig-protobuf/20250213.0-5304067/overlay/BUILD.bazel deleted file mode 100644 index 832c7591..00000000 --- a/third_party/modules/zig-protobuf/20250213.0-5304067/overlay/BUILD.bazel +++ /dev/null @@ -1,32 +0,0 @@ -load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") -load("@rules_zig//zig:defs.bzl", "BINARY_KIND", "zig_binary", "zig_library") - -zig_library( - name = "protobuf", - import_name = "protobuf", - main = "src/protobuf.zig", - visibility = ["//visibility:public"], -) - -zig_binary( - name = "generator", - srcs = [ - "bootstrapped-generator/FullName.zig", - "bootstrapped-generator/google/protobuf/compiler/plugin.pb.zig", - "bootstrapped-generator/google/protobuf/descriptor.pb.zig", - ], - kind = BINARY_KIND.exe, - main = "bootstrapped-generator/main.zig", - visibility = ["//visibility:public"], - deps = [":protobuf"], -) - -proto_lang_toolchain( - name = "zig_toolchain", - command_line = "--zig_out=$(OUT)", - output_files = "multiple", - plugin = ":generator", - plugin_format_flag = "--plugin=protoc-gen-zig=%s", - runtime = ":protobuf", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/zig-protobuf/20250213.0-5304067/overlay/MODULE.bazel b/third_party/modules/zig-protobuf/20250213.0-5304067/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/zig-protobuf/20250213.0-5304067/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/zig-protobuf/20250213.0-5304067/source.json b/third_party/modules/zig-protobuf/20250213.0-5304067/source.json deleted file mode 100644 index e4395bdd..00000000 --- a/third_party/modules/zig-protobuf/20250213.0-5304067/source.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "strip_prefix": "zig-protobuf-5304067205135532c0ad57f78243bfe86dc1ad3f", - "url": "https://github.com/gwenzek/zig-protobuf/archive/5304067205135532c0ad57f78243bfe86dc1ad3f.tar.gz", - "integrity": "sha256-cdby+J1CGNOvLRvdeASPNlB7y1q59LYcgjFOooBvIcA=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - } -} diff --git a/third_party/modules/zig-protobuf/20250318.0-930153e/MODULE.bazel b/third_party/modules/zig-protobuf/20250318.0-930153e/MODULE.bazel deleted file mode 100644 index 40830df4..00000000 --- a/third_party/modules/zig-protobuf/20250318.0-930153e/MODULE.bazel +++ /dev/null @@ -1,8 +0,0 @@ -module( - name = "zig-protobuf", - version = "20250318.0-930153e", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") diff --git a/third_party/modules/zig-protobuf/20250318.0-930153e/overlay/BUILD.bazel b/third_party/modules/zig-protobuf/20250318.0-930153e/overlay/BUILD.bazel deleted file mode 100644 index 832c7591..00000000 --- a/third_party/modules/zig-protobuf/20250318.0-930153e/overlay/BUILD.bazel +++ /dev/null @@ -1,32 +0,0 @@ -load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") -load("@rules_zig//zig:defs.bzl", "BINARY_KIND", "zig_binary", "zig_library") - -zig_library( - name = "protobuf", - import_name = "protobuf", - main = "src/protobuf.zig", - visibility = ["//visibility:public"], -) - -zig_binary( - name = "generator", - srcs = [ - "bootstrapped-generator/FullName.zig", - "bootstrapped-generator/google/protobuf/compiler/plugin.pb.zig", - "bootstrapped-generator/google/protobuf/descriptor.pb.zig", - ], - kind = BINARY_KIND.exe, - main = "bootstrapped-generator/main.zig", - visibility = ["//visibility:public"], - deps = [":protobuf"], -) - -proto_lang_toolchain( - name = "zig_toolchain", - command_line = "--zig_out=$(OUT)", - output_files = "multiple", - plugin = ":generator", - plugin_format_flag = "--plugin=protoc-gen-zig=%s", - runtime = ":protobuf", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/zig-protobuf/20250318.0-930153e/overlay/MODULE.bazel b/third_party/modules/zig-protobuf/20250318.0-930153e/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/zig-protobuf/20250318.0-930153e/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/zig-protobuf/20250318.0-930153e/source.json b/third_party/modules/zig-protobuf/20250318.0-930153e/source.json deleted file mode 100644 index fa33ec5e..00000000 --- a/third_party/modules/zig-protobuf/20250318.0-930153e/source.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "strip_prefix": "zig-protobuf-930153ec955853ff9fcf8a55ba632b2b8051f731", - "url": "https://github.com/zml/zig-protobuf/archive/930153ec955853ff9fcf8a55ba632b2b8051f731.tar.gz", - "integrity": "sha256-ORRDxK/cLQ2E+c9l2lcj8FgIUHH+uEk/+qWVWuHZMEY=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - } -} diff --git a/third_party/modules/zig-protobuf/20250716.0-97f1e31/MODULE.bazel b/third_party/modules/zig-protobuf/20250716.0-97f1e31/MODULE.bazel deleted file mode 100644 index b71798cc..00000000 --- a/third_party/modules/zig-protobuf/20250716.0-97f1e31/MODULE.bazel +++ /dev/null @@ -1,8 +0,0 @@ -module( - name = "zig-protobuf", - version = "20250716.0-97f1e31", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20250613.0-567662a") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") diff --git a/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/BUILD.bazel b/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/BUILD.bazel deleted file mode 100644 index 832c7591..00000000 --- a/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/BUILD.bazel +++ /dev/null @@ -1,32 +0,0 @@ -load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") -load("@rules_zig//zig:defs.bzl", "BINARY_KIND", "zig_binary", "zig_library") - -zig_library( - name = "protobuf", - import_name = "protobuf", - main = "src/protobuf.zig", - visibility = ["//visibility:public"], -) - -zig_binary( - name = "generator", - srcs = [ - "bootstrapped-generator/FullName.zig", - "bootstrapped-generator/google/protobuf/compiler/plugin.pb.zig", - "bootstrapped-generator/google/protobuf/descriptor.pb.zig", - ], - kind = BINARY_KIND.exe, - main = "bootstrapped-generator/main.zig", - visibility = ["//visibility:public"], - deps = [":protobuf"], -) - -proto_lang_toolchain( - name = "zig_toolchain", - command_line = "--zig_out=$(OUT)", - output_files = "multiple", - plugin = ":generator", - plugin_format_flag = "--plugin=protoc-gen-zig=%s", - runtime = ":protobuf", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/MODULE.bazel b/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/MODULE.bazel deleted file mode 100644 index b71798cc..00000000 --- a/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/MODULE.bazel +++ /dev/null @@ -1,8 +0,0 @@ -module( - name = "zig-protobuf", - version = "20250716.0-97f1e31", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20250613.0-567662a") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") diff --git a/third_party/modules/zig-protobuf/20250716.0-97f1e31/source.json b/third_party/modules/zig-protobuf/20250716.0-97f1e31/source.json deleted file mode 100644 index 4975cdb8..00000000 --- a/third_party/modules/zig-protobuf/20250716.0-97f1e31/source.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "strip_prefix": "zig-protobuf-97f1e314d9bbf600fbaadee5bb08c8856e0006de", - "url": "https://github.com/zml/zig-protobuf/archive/97f1e314d9bbf600fbaadee5bb08c8856e0006de.tar.gz", - "integrity": "sha256-iAqsLIKhl15JInJvwMY8NGVll6eUBXzdLW6Xv8wANGY=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - } -} diff --git a/third_party/modules/zig-protobuf/metadata.json b/third_party/modules/zig-protobuf/metadata.json deleted file mode 100644 index 4a038360..00000000 --- a/third_party/modules/zig-protobuf/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "homepage": "https://github.com/gwenzek/zig-protobuf", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": [ - "zml/zig-protobuf" - ], - "versions": [ - "20240722.0-c644d11", - "20250213.0-5304067", - "20250318.0-930153e", - "20250716.0-97f1e31", - ], - "yanked_versions": {} -} diff --git a/third_party/modules/zig-yaml/20240903.0-83d5fdf/MODULE.bazel b/third_party/modules/zig-yaml/20240903.0-83d5fdf/MODULE.bazel deleted file mode 100644 index 151f3337..00000000 --- a/third_party/modules/zig-yaml/20240903.0-83d5fdf/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "zig-yaml", - version = "20240903.0-83d5fdf", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/BUILD.bazel b/third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/BUILD.bazel deleted file mode 100644 index 94897f78..00000000 --- a/third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/BUILD.bazel +++ /dev/null @@ -1,12 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "zig-yaml", - srcs = glob([ - "src/*.zig", - "src/parse/*.zig", - "src/yaml/*.zig", - ]), - main = "src/yaml.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/MODULE.bazel b/third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/zig-yaml/20240903.0-83d5fdf/source.json b/third_party/modules/zig-yaml/20240903.0-83d5fdf/source.json deleted file mode 100644 index 643aa238..00000000 --- a/third_party/modules/zig-yaml/20240903.0-83d5fdf/source.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "strip_prefix": "zig-yaml-83d5fdfdd6c847b4bd2a8cad73b05a5c6447984b", - "url": "https://github.com/gwenzek/zig-yaml/archive/83d5fdfdd6c847b4bd2a8cad73b05a5c6447984b.tar.gz", - "integrity": "sha256-uusblAIZQgyKlg0qVmuHWD2JG+dMM+N6oKNGZsBdpEk=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - } -} diff --git a/third_party/modules/zig-yaml/metadata.json b/third_party/modules/zig-yaml/metadata.json deleted file mode 100644 index 3aa2c43c..00000000 --- a/third_party/modules/zig-yaml/metadata.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "homepage": "https://github.com/gwenzek/zig-yaml", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": [ - "github:gwenzek/zig-yaml" - ], - "versions": [ - "20240903.0-83d5fdf" - ], - "yanked_versions": {} -} diff --git a/third_party/modules/zigcoro/20240829.0-fc1db29/MODULE.bazel b/third_party/modules/zigcoro/20240829.0-fc1db29/MODULE.bazel deleted file mode 100644 index a59b2d87..00000000 --- a/third_party/modules/zigcoro/20240829.0-fc1db29/MODULE.bazel +++ /dev/null @@ -1,8 +0,0 @@ -module( - name = "zigcoro", - version = "20240829.0-fc1db29", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") -bazel_dep(name = "libxev", version = "20240825.0-dbe2291") diff --git a/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/BUILD.bazel b/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/BUILD.bazel deleted file mode 100644 index 2aa934ca..00000000 --- a/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/BUILD.bazel +++ /dev/null @@ -1,29 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -genrule( - name = "libcoro_options_gen", - outs = ["options.zig"], - cmd = """\ -cat < $@ -pub const default_stack_size: usize = 4096; -pub const debug_log_level: usize = 0; -EOF -""", -) - -zig_library( - name = "libcoro_options", - main = ":libcoro_options_gen", -) - -zig_library( - name = "libcoro", - srcs = glob(["src/*.zig"]), - extra_srcs = glob(["src/asm/*.s"]), - main = "src/main.zig", - visibility = ["//visibility:public"], - deps = [ - ":libcoro_options", - "@libxev", - ], -) diff --git a/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/MODULE.bazel b/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/zigcoro/20240829.0-fc1db29/patches/0001-Handle-generic-function-in-Signature.patch b/third_party/modules/zigcoro/20240829.0-fc1db29/patches/0001-Handle-generic-function-in-Signature.patch deleted file mode 100644 index 6072de07..00000000 --- a/third_party/modules/zigcoro/20240829.0-fc1db29/patches/0001-Handle-generic-function-in-Signature.patch +++ /dev/null @@ -1,48 +0,0 @@ -From e2416ff5ec9cad5572e6a5edac3f949b75024c00 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Tue, 6 Aug 2024 11:34:35 +0200 -Subject: [PATCH 1/2] Handle generic function in Signature - ---- - src/coro.zig | 7 +++++-- - 1 file changed, 5 insertions(+), 2 deletions(-) - -diff --git a/src/coro.zig b/src/coro.zig -index 9285489..8a34717 100644 ---- a/src/coro.zig -+++ b/src/coro.zig -@@ -266,6 +266,7 @@ const CoroT = struct { - YieldT: type = void, - InjectT: type = void, - ArgsT: type, -+ _ReturnT: type, - - // If the function this signature represents is compile-time known, - // it can be held here. -@@ -273,19 +274,21 @@ const CoroT = struct { - - fn init(comptime Func: anytype, comptime options: CoroT.Options) @This() { - const FuncT = if (@TypeOf(Func) == type) Func else @TypeOf(Func); -+ const ArgsT = options.ArgsT orelse ArgsTuple(FuncT); - return .{ - .Func = FuncT, - .YieldT = options.YieldT, - .InjectT = options.InjectT, -- .ArgsT = options.ArgsT orelse ArgsTuple(FuncT), -+ .ArgsT = ArgsT, - .func_ptr = if (@TypeOf(Func) == type) null else struct { - const val = Func; - }, -+ ._ReturnT = @TypeOf(@call(.auto, Func, @as(ArgsT, undefined))), - }; - } - - pub fn ReturnT(comptime self: @This()) type { -- return @typeInfo(self.Func).Fn.return_type.?; -+ return self._ReturnT; - } - }; - --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/zigcoro/20240829.0-fc1db29/patches/0002-asyncio-return-error-super-set-in-pread-pwrite.patch b/third_party/modules/zigcoro/20240829.0-fc1db29/patches/0002-asyncio-return-error-super-set-in-pread-pwrite.patch deleted file mode 100644 index f1e68622..00000000 --- a/third_party/modules/zigcoro/20240829.0-fc1db29/patches/0002-asyncio-return-error-super-set-in-pread-pwrite.patch +++ /dev/null @@ -1,34 +0,0 @@ -From d8a4773ee9fa0ae5c5fd91e84c322e75a634a077 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Tue, 6 Aug 2024 13:14:59 +0200 -Subject: [PATCH 2/2] asyncio: return error super set in pread/pwrite - ---- - src/asyncio.zig | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/src/asyncio.zig b/src/asyncio.zig -index 68cc7d3..8487754 100644 ---- a/src/asyncio.zig -+++ b/src/asyncio.zig -@@ -354,7 +354,7 @@ pub const File = struct { - } - - const PReadResult = xev.ReadError!usize; -- pub fn pread(self: Self, buf: xev.ReadBuffer, offset: u64) PReadResult { -+ pub fn pread(self: Self, buf: xev.ReadBuffer, offset: u64) !usize { - const ResultT = PReadResult; - const Data = struct { - result: ResultT = undefined, -@@ -391,7 +391,7 @@ pub const File = struct { - } - - const PWriteResult = xev.WriteError!usize; -- pub fn pwrite(self: Self, buf: xev.WriteBuffer, offset: u64) PWriteResult { -+ pub fn pwrite(self: Self, buf: xev.WriteBuffer, offset: u64) !usize { - const ResultT = PWriteResult; - const Data = struct { - result: ResultT = undefined, --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/zigcoro/20240829.0-fc1db29/source.json b/third_party/modules/zigcoro/20240829.0-fc1db29/source.json deleted file mode 100644 index 17e66d4e..00000000 --- a/third_party/modules/zigcoro/20240829.0-fc1db29/source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "strip_prefix": "zigcoro-fc1db29dee049ce572a516f1586cce0eca4fcb2a", - "url": "https://github.com/rsepassi/zigcoro/archive/fc1db29dee049ce572a516f1586cce0eca4fcb2a.tar.gz", - "integrity": "sha256-2ynCPLSRFPto3HIOhDTBDyUS770PNMI9ioMkj8Z7Y+s=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0001-Handle-generic-function-in-Signature.patch": "", - "0002-asyncio-return-error-super-set-in-pread-pwrite.patch": "" - } -} diff --git a/third_party/modules/zigcoro/metadata.json b/third_party/modules/zigcoro/metadata.json deleted file mode 100644 index 533957ce..00000000 --- a/third_party/modules/zigcoro/metadata.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "homepage": "https://github.com/rsepassi/zigcoro", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": [ - "github:rsepassi/zigcoro" - ], - "versions": [ - "20240829.0-fc1db29" - ], - "yanked_versions": {} -} From eae31a51b80d90a407b8168fa61fe0874e4fd671 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 12 May 2026 13:34:08 +0200 Subject: [PATCH 479/881] zml/sharding: fix segfault in tensorShardingAttr for gspmd (#532) --- zml/Sharding.zig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zml/Sharding.zig b/zml/Sharding.zig index dce1239e..366d9e03 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -97,7 +97,7 @@ pub const Partitioning = struct { const selected_sharding = sharding orelse try self.selectSharding(shape); return switch (self.partitioner) { .shardy => (try selected_sharding.data.sdyShardingAttrForShape(allocator, ctx, shape)).asAttr(), - .gspmd => mlir.stringAttribute(ctx, try selected_sharding.data.gspmdShardingAttrForShape(allocator, shape)), + .gspmd => try selected_sharding.data.gspmdShardingAttrForShape(allocator, ctx, shape), }; } @@ -1444,7 +1444,7 @@ pub const Data = struct { return .init(ctx, .{ .mesh = data.name, .dimensions = dimensions, .replicated_axes = replicated_axes }); } - pub fn gspmdShardingAttrForShape(self: *const Data, parent_allocator: std.mem.Allocator, shape: Shape) ![]const u8 { + pub fn gspmdShardingAttrForShape(self: *const Data, parent_allocator: std.mem.Allocator, ctx: *mlir.Context, shape: Shape) !*const mlir.Attribute { var arena = try stdx.arenaWithCapacity(parent_allocator, 1024); defer arena.deinit(); const allocator = arena.allocator(); @@ -1456,7 +1456,7 @@ pub const Data = struct { break; } } - if (!has_sharding) return try allocator.dupe(u8, "{replicated}"); + if (!has_sharding) return mlir.stringAttribute(ctx, try allocator.dupe(u8, "{replicated}")); const mapping = self.getDimMapping(shape); var tile_shape: stdx.BoundedArray(i64, Shape.MAX_RANK + 1) = .empty; @@ -1509,7 +1509,7 @@ pub const Data = struct { try out.writer.writeAll("}"); - return try out.toOwnedSlice(); + return mlir.stringAttribute(ctx, try out.toOwnedSlice()); } pub fn deviceAssignment(self: *const Data, allocator: std.mem.Allocator) ![]usize { From c917acb572f5733e0ceb93edb4a276e254a0c0d2 Mon Sep 17 00:00:00 2001 From: elogir Date: Tue, 12 May 2026 14:54:42 +0200 Subject: [PATCH 480/881] kernels/triton: add LLVM dialect, pipelining hints and fixes (#525) This PR introduces the following: - Wire up the LLVM dialect so the Triton builder can emit `llvm.intr.assume` so we can match [`triton.language.assume`](https://triton-lang.org/main/python-api/generated/triton.language.assume.html#triton.language.assume) - Thread `tt.num_stages` through `scf.for` for pipelining hints - Add an inline-return mode to `ReturnIfScope`. Not necessary but better for IR matching with Python reference - Build DSL kernels IR in a new MLIR context Also fixes i1 extension to use `extui` instead of `extsi`. --- kernels/common/control_flow.zig | 3 +- kernels/mosaic_tpu/README.md | 25 ++++++---- kernels/mosaic_tpu/builder.zig | 2 + kernels/triton/README.md | 67 +++++++++++++++++++------ kernels/triton/builder.zig | 87 ++++++++++++++++++++++++--------- mlir/BUILD.bazel | 1 + mlir/dialects/scf.zig | 6 ++- mlir/mlir.h | 1 + zml/kernel.zig | 58 ++++++++++++++++------ zml/module.zig | 2 +- 10 files changed, 187 insertions(+), 65 deletions(-) diff --git a/kernels/common/control_flow.zig b/kernels/common/control_flow.zig index 2fdfffbd..252184f3 100644 --- a/kernels/common/control_flow.zig +++ b/kernels/common/control_flow.zig @@ -49,6 +49,7 @@ pub fn ForScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usiz iv: ValueT, carried: [N]ValueT, results: [N]ValueT = undefined, + opts: scf.ForOpts = .{}, const Self = @This(); @@ -63,7 +64,7 @@ pub fn ForScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usiz self.step_inner, &self.inits_inner, self.body, - .{}, + self.opts, k.loc(), ); _ = for_op.appendTo(k.currentBlock()); diff --git a/kernels/mosaic_tpu/README.md b/kernels/mosaic_tpu/README.md index 6c2c9433..8ebf3d06 100644 --- a/kernels/mosaic_tpu/README.md +++ b/kernels/mosaic_tpu/README.md @@ -6,24 +6,28 @@ to read like `pallas.tpu` Python while staying in pure Zig. Used by op-for-op against what `pallas_call` produces. The DSL talks to the `tpu` dialect through `mlir.Operation.make(ctx, "tpu.", …)`. -The dialect must be registered into the MLIR context — see -`mlir/dialects/mosaic_tpu`. Until the JAX repo is wired into `MODULE.bazel`, the -whole stack is `manual`-tagged in Bazel. +A Mosaic kernel is a pure IR generator: it builds its MLIR in a throwaway +context, canonicalizes it, serializes it to a string, and hands that to a +`stablehlo.custom_call` targeting `tpu_custom_call` — it never touches ZML's +compilation context (which carries only `func`/`stablehlo`/`sdy`). Until the +JAX repo is wired into `MODULE.bazel`, the whole stack is `manual`-tagged in Bazel. Two layers define the surface: - **Low layer** — `kernels/mosaic_tpu/builder.zig`: the IR builder primitives. `Builder.open(allocator, ctx, name)` → `b.declareArgsOpts(spec, results, opts)` → body via `b.*` helpers → `b.finishOpts(results, .{ .canonicalize = true })` - for the final IR string. (`Builder.buildOpts(...)` is a convenience that - bundles `open + declareArgs` into one call and returns a heap-allocated - `*Built(Spec)`; reach for it in escape-hatch code that drives the - lifecycle manually.) + for the final IR string. Pass `Builder.open` / `Builder.init` a + `zml.kernel.mosaic_tpu.newContext()` (a throwaway context with the dialects + the DSL emits), `defer ctx.deinit()` once you have the string — not a + `CompilationContext`'s `mlir_ctx`. (`Builder.buildOpts(...)` is a convenience + that bundles `open + declareArgs` into one call and returns a heap-allocated + `*Built(Spec)`; reach for it in escape-hatch code that drives the lifecycle manually.) - **High layer** — `zml.kernel.mosaic_tpu.Kernel(Config, spec)`: the declarative kernel form, mirroring the Triton DSL. Bundles a config type, a typed spec literal (name + named `inputs` / `outputs` + a typed `run` function pointer), and produces a kernel type. Exposes - `.emit(allocator, ctx, cfg)` (IR string) and + `.emit(allocator, cfg)` (IR string; manages its own context) and `.call(inputs, outputs, opts)` (IR string + `stablehlo.custom_call` targeting `tpu_custom_call` in one shot — the preferred form for production model code). @@ -133,10 +137,11 @@ windows) can be runtime values from a config. ### Two ways to use a kernel -**Offline / tooling** — get the MLIR string and print or diff it: +**Offline / tooling** — get the MLIR string and print or diff it (no MLIR +context needed; `emit` spins up and tears down its own): ```zig -const ir = try MyKernel.Kernel.emit(allocator, ctx, .{ /* cfg */ }); +const ir = try MyKernel.Kernel.emit(allocator, .{ /* cfg */ }); defer allocator.free(ir); try stdout.print("{s}\n", .{ir}); ``` diff --git a/kernels/mosaic_tpu/builder.zig b/kernels/mosaic_tpu/builder.zig index e0afffd3..963ac907 100644 --- a/kernels/mosaic_tpu/builder.zig +++ b/kernels/mosaic_tpu/builder.zig @@ -402,6 +402,8 @@ pub const MinMaxOpts = struct { // Builder — the main DSL builder. // ============================================================================= +pub const dialects_needed = [_][]const u8{ "func", "tpu", "arith", "scf", "math", "cf", "memref", "vector", "affine" }; + /// The main DSL builder. Create with `init`, populate the body with helpers, /// then call `finish` to get the IR string. pub const Builder = struct { diff --git a/kernels/triton/README.md b/kernels/triton/README.md index 5cb406e2..d36fae08 100644 --- a/kernels/triton/README.md +++ b/kernels/triton/README.md @@ -10,14 +10,19 @@ Two layers define the surface: - **Low layer** — `kernels/triton/builder.zig`: the IR builder primitives. `Builder`, `Value`, `DType`, `ArgSpec`. Use these directly when you need full control (custom op insertion, dynamic-arity arg lists, escape hatches). + `Builder.open` / `Builder.init` take an `*mlir.Context` — pass them a + `zml.kernel.triton.newContext()` (a throwaway context with the dialects the + DSL emits), `defer ctx.deinit()` once you have the IR string. Not a + `CompilationContext`'s `mlir_ctx`: that one carries only `func`/`stablehlo`/`sdy` + (a registered `llvm` dialect makes XLA:TPU choke). - **High layer** — `zml.kernel.triton.Kernel(Config, spec)`: the declarative kernel form. Bundles a config type, a typed spec literal (name + named `inputs` / `outputs` + a typed `run` function pointer), and produces a - kernel type. Exposes `.emit(allocator, ctx, cfg)` (TTIR string) and - `.call(inputs, outputs, opts)` (TTIR string + `stablehlo.custom_call` in - one shot — the preferred form for production model code). Inputs and - outputs are by-name structs generated from the spec, so missing / typo'd - / extra arguments are compile errors at the callsite. + kernel type. Exposes `.emit(allocator, cfg)` (TTIR string; manages its own + throwaway context) and `.call(inputs, outputs, opts)` (TTIR string + + `stablehlo.custom_call` in one shot — the preferred form for production + model code). Inputs and outputs are by-name structs generated from the + spec, so missing / typo'd / extra arguments are compile errors at the callsite. ## Table of contents @@ -111,10 +116,11 @@ models where dtypes/sizes come from `Tensor.dim()` / `Options`). ### Two ways to use a kernel -**Offline / tooling** — get the TTIR string: +**Offline / tooling** — get the TTIR string (no MLIR context needed; `emit` +spins up and tears down its own): ```zig -const ttir = try MyKernel.Kernel.emit(allocator, ctx, .{ .BLOCK = 1024 }); +const ttir = try MyKernel.Kernel.emit(allocator, .{ .BLOCK = 1024 }); defer allocator.free(ttir); ``` @@ -190,9 +196,10 @@ pub fn forward(self: Model, x: zml.Tensor, y: zml.Tensor) zml.Tensor { `call` is a thin wrapper over `Kernel.emit(...)` + `zml.ops.triton(...)`. It must be called from inside a `CompilationContext` (i.e., during -`zml.module.compile(...)`). For offline tools without a CompilationContext -(e.g., dump-to-disk diff harnesses), use `Kernel.emit(allocator, ctx, cfg)` -directly to get the TTIR string. +`zml.module.compile(...)`) — `emit` builds the TTIR string in its own +throwaway context; `zml.ops.triton(...)` drops the `stablehlo.custom_call` +into the model's context. For offline tools, `Kernel.emit(allocator, cfg)` +gives you just the TTIR string. Signature: `Kernel.call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results`. @@ -666,6 +673,10 @@ Pairs in place today: `load` / `loadOpts`, `store` / `storeOpts`, `sum` / `sumOpts`, `max` / `maxOpts`, `min` / `minOpts`, `cumsum` / `cumsumOpts`, `cumprod` / `cumprodOpts`. +DSL-only pair (no Python kwarg counterpart — `.inline_return` is about IR +block layout, see the `returnIf` / `returnIfOpts` section under Control flow): +`returnIf` / `returnIfOpts`. + Ops with no kwargs in Python (pure positional): `gather(src, indices, axis)`, `trans(src, order)`, `permute(src, order)`, `cat(lhs, rhs)` (dim=0 default), `split(src)`, `join(lhs, rhs)`, `item(src)`, `broadcast(a, b)` (symmetric), @@ -814,7 +825,7 @@ const r = w.results[0]; - `yieldBefore(cond, forwarded)` — `forwarded` arity must match `after_types`. - `yieldAfter(values)` — arity must match `inits`. -### `returnIf` — early `tt.return` via cf.cond_br +### `returnIf` / `returnIfOpts` — early `tt.return` via cf.cond_br ```zig // Python: if pid_m * BLOCK >= num_tokens_post_padded: return @@ -846,14 +857,32 @@ cf.cond_br %out_of_range, ^ret, ^cont `scf.if` for structured loops and value-carrying branches; our DSL does the same. +**`returnIfOpts(cond, values, .{ .inline_return = true })` — match Python's +per-`return` block layout.** By default `returnIf` routes every return path +(every early `returnIf` / `openReturnIf` plus the natural fall-through from +`b.finish`) through one **shared exit block** holding the single `tt.return`. +Python's frontend does the opposite: each `if cond: return` (and the natural +exit) gets its **own** `tt.return`-only block, and LLVM's SimplifyCFG later +merges them into a single `common.ret`. The two are semantically identical +but differ in the function-exit block's label and position — which renumbers +every `$L__BB` label downstream and blows up the PTX diff. When the Python source contains *any* literal `return` statement (it then +has ≥2 `ret` blocks in LLVM — the explicit one(s) plus the implicit +function-end one — which SimplifyCFG merges into `common.ret`), pass +`.{ .inline_return = true }` on *every* return site, and set +`scope.inline_return = true` on each `openReturnIf` (see below). Each one +then emits `tt.return` inline and SimplifyCFG reproduces Python's +`common.ret`. The default shared exit block emits exactly one `tt.return`, +so the merge never fires and the function-exit block lands somewhere else. + ### `openReturnIf` — early `tt.return` with side-effects in the taken branch ```zig // Python: if off_experts == -1: write_zeros_to_output(...); return var scope = b.openReturnIf(is_dead); +scope.inline_return = true; // optional — see `returnIfOpts` above { writeZerosToOutput(k, c_ptr, ...); // ops here emit into ^ret - scope.yieldReturn(.{}); // closes ^ret with tt.return + scope.yieldReturn(.{}); // closes ^ret with tt.return (or cf.br ^exit) } // ... ops here emit into ^cont (fall-through) ... ``` @@ -865,6 +894,12 @@ the `tt.return`. Use this instead of the duplicates the predicate (emits both `scf.if` and `cf.cond_br` on the same boolean) and drifts from Triton's reference IR. +`scope.inline_return` is the per-scope twin of `returnIfOpts`'s +`.inline_return` (default `false`): set it before `yieldReturn` to make +`^ret` close with `tt.return` directly instead of `cf.br ^exit`. Flip it on +every `returnIf` / `openReturnIf` of a kernel when matching a Python source +that has literal `return` statements. + Same restrictions as `returnIf`: top-level `tt.func` body only. ### When to use `cf` (returnIf / openReturnIf) vs `scf` (openIf / openIfElse / openFor) @@ -974,6 +1009,10 @@ You can also use `Builder.open(allocator, ctx, name)` to create an empty builder and call `b.declareArgs(spec)` mid-stream — that's exactly what `zml.kernel.triton.Kernel(...)` does internally. +Pass these constructors a `zml.kernel.triton.newContext()`, not a +`CompilationContext`'s `mlir_ctx`; `defer ctx.deinit()` once you have the IR +string. `Kernel.emit` does this for you, which is why it takes no context. + Prefer `zml.kernel.triton.Kernel(Config, spec)` for the common case. --- @@ -1041,8 +1080,8 @@ Prefer `zml.kernel.triton.Kernel(Config, spec)` for the common case. | `while cond: ...` | `var w = b.openWhile(.{inits}, .{after_types}); { ...; w.yieldBefore(cond, .{...}); }; { ...; w.yieldAfter(.{...}); }` | | `if cond: ...` (side-effects, no else) | `var i = b.openIf(cond); { ...; i.yieldThen(.{}); }` | | `if cond: ... else: ...` | `var i = b.openIfElse(cond, .{result_types}); { ...; i.yieldThen(.{...}); }; { ...; i.yieldElse(.{...}); }` | -| `if cond: return` (early exit) | `b.returnIf(cond, .{});` — emits `cf.cond_br` + `tt.return`, continues in fall-through block | -| `if cond: ; return` (early exit, with side-effects) | `var s = b.openReturnIf(cond); { ...; s.yieldReturn(.{}); }` — single `cf.cond_br`; body emits into `^ret` before the `tt.return` | +| `if cond: return` (early exit) | `b.returnIf(cond, .{});` — emits `cf.cond_br` + `tt.return`, continues in fall-through block (add `b.returnIfOpts(cond, .{}, .{ .inline_return = true })` when matching a Python source that has literal `return`s) | +| `if cond: ; return` (early exit, with side-effects) | `var s = b.openReturnIf(cond); { ...; s.yieldReturn(.{}); }` — single `cf.cond_br`; body emits into `^ret` before the `tt.return` (set `s.inline_return = true` to match, alongside `returnIfOpts`) | --- diff --git a/kernels/triton/builder.zig b/kernels/triton/builder.zig index 46945679..711c692a 100644 --- a/kernels/triton/builder.zig +++ b/kernels/triton/builder.zig @@ -370,6 +370,10 @@ pub const DeviceAssertOpts = struct { mask: ?Value = null, }; +pub const ReturnIfOpts = struct { + inline_return: bool = false, +}; + const tupleArity = dsl.tupleArity; pub fn ForScope(comptime N: usize) type { @@ -380,24 +384,34 @@ pub const ReturnIfScope = struct { kernel: *Builder, ret_block: *mlir.Block, cont_block: *mlir.Block, + inline_return: bool = false, pub fn yieldReturn(self: *ReturnIfScope, values: anytype) void { const k = self.kernel; const fields = @typeInfo(@TypeOf(values)).@"struct".fields; - if (k.exit_block == null) { - k.exit_block = mlir.Block.init(&.{}, &.{}); - + if (self.inline_return) { var ret_operands: [fields.len]*const mlir.Value = undefined; inline for (fields, 0..) |f, i| { if (f.type != Value) @compileError("ReturnIfScope.yieldReturn: every value must be a Value"); ret_operands[i] = @field(values, f.name).inner; } - _ = ttir.return_(k.ctx, &ret_operands, k.loc()).appendTo(k.exit_block.?); - } + _ = ttir.return_(k.ctx, &ret_operands, k.loc()).appendTo(self.ret_block); + } else { + if (k.exit_block == null) { + k.exit_block = mlir.Block.init(&.{}, &.{}); - _ = cf.br(k.ctx, k.exit_block.?, &.{}, k.loc()).appendTo(self.ret_block); + var ret_operands: [fields.len]*const mlir.Value = undefined; + inline for (fields, 0..) |f, i| { + if (f.type != Value) + @compileError("ReturnIfScope.yieldReturn: every value must be a Value"); + ret_operands[i] = @field(values, f.name).inner; + } + _ = ttir.return_(k.ctx, &ret_operands, k.loc()).appendTo(k.exit_block.?); + } + _ = cf.br(k.ctx, k.exit_block.?, &.{}, k.loc()).appendTo(self.ret_block); + } k.popBlock(); if (k.block_stack.items.len == 0) { @@ -439,6 +453,8 @@ pub fn ScanArgs(comptime CtxT: type) type { }; } +pub const dialects_needed = [_][]const u8{ "func", "tt", "arith", "scf", "math", "cf", "llvm" }; + pub const Builder = struct { allocator: std.mem.Allocator, arena: std.heap.ArenaAllocator, @@ -728,6 +744,18 @@ pub const Builder = struct { return self.emit(ttir.get_num_programs(self.ctx, dim, self.loc())); } + pub fn assume(self: *Builder, cond: Value) void { + const op = mlir.Operation.make(self.ctx, "llvm.intr.assume", .{ + .operands = .{ .flat = &.{cond.inner} }, + .attributes = &.{ + .named(self.ctx, "op_bundle_sizes", mlir.denseArrayAttribute(self.ctx, .i32, &.{})), + .named(self.ctx, "op_bundle_tags", mlir.arrayAttribute(self.ctx, &.{})), + }, + .location = self.loc(), + }); + _ = op.appendTo(self.currentBlock()); + } + // ==================== constants ==================== fn emitInt(self: *Builder, dtype: DType, value: i64) Value { @@ -1318,7 +1346,10 @@ pub const Builder = struct { } const cur_bw = dtypeBitwidth(cur_dtype); const tgt_bw = dtypeBitwidth(dtype); - if (tgt_bw > cur_bw) return self.extsi(src, dtype); + if (tgt_bw > cur_bw) { + if (cur_dtype == .i1) return self.extui(src, dtype); + return self.extsi(src, dtype); + } if (tgt_bw < cur_bw) return self.trunci(src, dtype); return self.arithBitcast(src, dtype); } @@ -2024,33 +2055,41 @@ pub const Builder = struct { } pub fn returnIf(self: *Builder, cond: Value, values: anytype) void { + self.returnIfOpts(cond, values, .{}); + } + + pub fn returnIfOpts(self: *Builder, cond: Value, values: anytype, opts: ReturnIfOpts) void { const current = self.currentBlock(); const region = current.parentRegion(); - - const fields = @typeInfo(@TypeOf(values)).@"struct".fields; const cont_block = mlir.Block.init(&.{}, &.{}); - // Use a shared exit block for all returns. This ensures that every - // return path (early or natural) goes through a pure tt.return block, - // allowing MLIR's Canonicalizer/BlockMerging to create a single join - // point. - if (self.exit_block == null) { - self.exit_block = mlir.Block.init(&.{}, &.{}); - - var ret_operands: [fields.len]*const mlir.Value = undefined; - inline for (fields, 0..) |f, i| { - if (f.type != Value) - @compileError("returnIf: every value must be a Value"); - ret_operands[i] = @field(values, f.name).inner; - } - _ = ttir.return_(self.ctx, &ret_operands, self.loc()).appendTo(self.exit_block.?); + const fields = @typeInfo(@TypeOf(values)).@"struct".fields; + var ret_operands: [fields.len]*const mlir.Value = undefined; + inline for (fields, 0..) |f, i| { + if (f.type != Value) + @compileError("returnIfOpts: every value must be a Value"); + ret_operands[i] = @field(values, f.name).inner; } + + const ret_target: *mlir.Block = if (opts.inline_return) blk: { + const rb = mlir.Block.init(&.{}, &.{}); + _ = ttir.return_(self.ctx, &ret_operands, self.loc()).appendTo(rb); + region.appendOwnedBlock(rb); + break :blk rb; + } else blk: { + if (self.exit_block == null) { + self.exit_block = mlir.Block.init(&.{}, &.{}); + _ = ttir.return_(self.ctx, &ret_operands, self.loc()).appendTo(self.exit_block.?); + } + break :blk self.exit_block.?; + }; + region.appendOwnedBlock(cont_block); _ = cf.cond_br( self.ctx, cond.inner, - self.exit_block.?, + ret_target, &.{}, cont_block, &.{}, diff --git a/mlir/BUILD.bazel b/mlir/BUILD.bazel index 8e38ce82..750b0285 100644 --- a/mlir/BUILD.bazel +++ b/mlir/BUILD.bazel @@ -12,6 +12,7 @@ cc_library( "@llvm-project//mlir:CAPIArith", "@llvm-project//mlir:CAPICF", "@llvm-project//mlir:CAPIIR", + "@llvm-project//mlir:CAPILLVM", "@llvm-project//mlir:CAPIMath", "@llvm-project//mlir:CAPIMemRef", "@llvm-project//mlir:CAPISCF", diff --git a/mlir/dialects/scf.zig b/mlir/dialects/scf.zig index 4e90e579..6dbc337a 100644 --- a/mlir/dialects/scf.zig +++ b/mlir/dialects/scf.zig @@ -8,6 +8,7 @@ pub const ForOpts = struct { /// the `unsignedCmp` UnitAttr so the induction variable advances with /// unsigned comparison instead. unsigned_cmp: bool = false, + num_stages: ?i32 = null, }; /// scf.for — operands are (lower, upper, step, init_args...) as a flat list. @@ -32,10 +33,13 @@ pub fn for_( var result_types: stdx.BoundedArray(*const mlir.Type, 64) = .empty; for (init_args) |v| result_types.appendAssumeCapacity(v.type_()); - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; if (opts.unsigned_cmp) { attrs.appendAssumeCapacity(.named(ctx, "unsignedCmp", mlir.unitAttribute(ctx))); } + if (opts.num_stages) |n| { + attrs.appendAssumeCapacity(.named(ctx, "tt.num_stages", mlir.integerAttribute(ctx, .i32, n))); + } return mlir.Operation.make(ctx, "scf.for", .{ .operands = .{ .flat = operands.constSlice() }, diff --git a/mlir/mlir.h b/mlir/mlir.h index 8d079000..0effcda9 100644 --- a/mlir/mlir.h +++ b/mlir/mlir.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/zml/kernel.zig b/zml/kernel.zig index 8a96e0b0..23d95de8 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -21,12 +21,34 @@ fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); } +fn makeKernelContext(comptime dialects_needed: []const []const u8) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + inline for (dialects_needed) |d| { + mlir.DialectHandle.fromString(d).insertDialect(registry); + } + + mlir.registerFuncExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + pub const triton = struct { pub const Builder = triton_builder.Builder; pub const Value = triton_builder.Value; pub const DType = triton_builder.DType; pub const FinishError = triton_builder.FinishError; + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(&triton_builder.dialects_needed); + } + pub fn from(dt: DataType) DType { return switch (dt) { .bool => .i1, @@ -73,21 +95,22 @@ pub const triton = struct { debug: bool = false, }; - pub fn emit( - allocator: std.mem.Allocator, - ctx: *mlir.Context, - cfg: ConfigT, - ) ![:0]const u8 { + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + var b = try triton_builder.Builder.open(allocator, ctx, name); defer b.deinit(); + try spec.run(&b, cfg); + return b.finish(&.{}); } pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { const cur = CompilationContext.current(); - const ttir = emit(cur.allocator, cur.mlir_ctx, opts.cfg) catch |err| + const ttir = emit(cur.allocator, opts.cfg) catch |err| std.debug.panic("zml.kernel.triton.Kernel({s}).call: emit failed: {}", .{ name, err }); defer cur.allocator.free(ttir); @@ -120,6 +143,10 @@ pub const mosaic_tpu = struct { const DType = mosaic_tpu_builder.DType; const FinishError = mosaic_tpu_builder.FinishError; + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(&mosaic_tpu_builder.dialects_needed); + } + pub fn from(dt: DataType) DType { return switch (dt) { .bool => .i1, @@ -162,25 +189,26 @@ pub const mosaic_tpu = struct { extras: CallExtras = .{}, }; - pub fn emit( - allocator: std.mem.Allocator, - ctx: *mlir.Context, - cfg: ConfigT, - ) ![:0]const u8 { + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + var b = try mosaic_tpu_builder.Builder.open(allocator, ctx, name); defer b.deinit(); + try spec.run(&b, cfg); + return b.finishOpts(&.{}, .{ .canonicalize = true }); } pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { const cur = CompilationContext.current(); - const ir = emit(cur.allocator, cur.mlir_ctx, opts.cfg) catch |err| + const ir = emit(cur.allocator, opts.cfg) catch |err| std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: emit failed: {}", .{ name, err }); defer cur.allocator.free(ir); - const backend_config = buildBackendConfig(cur.allocator, cur.mlir_ctx, ir, opts.extras) catch |err| + const backend_config = buildBackendConfig(cur.allocator, ir, opts.extras) catch |err| std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: backend_config build failed: {}", .{ name, err }); defer cur.allocator.free(backend_config); @@ -237,10 +265,12 @@ pub const mosaic_tpu = struct { fn buildBackendConfig( allocator: std.mem.Allocator, - ctx: *mlir.Context, ir: [:0]const u8, extras: CallExtras, ) ![]u8 { + const ctx = try newContext(); + defer ctx.deinit(); + const module = try mlir.Module.parse(ctx, ir); defer module.deinit(); diff --git a/zml/module.zig b/zml/module.zig index fb828fbd..9ef99e6d 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -33,7 +33,7 @@ fn mlirRegistry(io: std.Io) *mlir.DialectRegistry { mlir.registerPasses("Transforms"); const mlir_registry = mlir.DialectRegistry.init() catch unreachable; - inline for (.{ "func", "stablehlo", "sdy", "arith", "scf", "math", "tt", "affine", "memref", "vector", "tpu", "cf" }) |d| { + inline for (.{ "func", "stablehlo", "sdy" }) |d| { mlir.DialectHandle.fromString(d).insertDialect(mlir_registry); } mlir.registerFuncExtensions(mlir_registry); From ca8599aea0ba3c72b93c004ad96716015e1fdf7b Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 12 May 2026 17:27:37 +0200 Subject: [PATCH 481/881] zml/attention: attnd (#450) Simple reference implementation of attnd client as a attention backend relying on POSIX. I've tried various things with `sendmmsg`/`recvmmsg` and `io_uring` out of curiosity but they weren't worth the extra code here. The integration as a attention backend is a bit clunky, we're still allocating a KV cache and there are a few panics here and there because the current initialization mechanism of attention backend doesn't work for attnd, but that's something for later. --- examples/llm/main.zig | 18 +- examples/llm/models/llama/inference.zig | 40 +- examples/llm/models/llama/model.zig | 12 +- examples/llm/models/llama/session.zig | 24 +- zml/BUILD.bazel | 1 + zml/attention.zig | 1 + zml/attention/attention.zig | 13 + zml/attention/attnd.zig | 501 ++++++++++++++++++++++++ zml/attention/flashattn.zig | 4 +- 9 files changed, 596 insertions(+), 18 deletions(-) create mode 100644 zml/attention/attnd.zig diff --git a/examples/llm/main.zig b/examples/llm/main.zig index 747d569a..9cea2b67 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -19,6 +19,7 @@ const Args = struct { seqlen: u32 = 2048, topk: u32 = 4, backend: ?zml.attention.attention.Backend = null, + attnd_ip: ?[]const u8 = null, single: bool = false, profile: bool = false, @@ -32,7 +33,8 @@ const Args = struct { \\ --prompt= Prompt to use for generation (default: none) \\ --seqlen= Sequence length (default: 2048) \\ --topk= Top-k sampling cutoff (default: 4) - \\ --backend= Attention backend to use ([vanilla, cuda_fa2, cuda_fa3], default: auto-selection) + \\ --backend= Attention backend to use ([vanilla, attnd, cuda_fa2, cuda_fa3], default: auto-selection) + \\ --attnd-ip= Register and prefer the `attnd` backend at the provided `IP:PORT` \\ --single Create a single kernel encompassing all the layers when supported \\ (only used by LFM2 which uses multiple kernels by default) \\ --profile Capture a PJRT profile for non-interactive runs and write a Perfetto trace @@ -85,11 +87,15 @@ pub fn main(init: std.process.Init) !void { log.info("\n{f}", .{platform.fmtVerbose()}); - const backend = args.backend orelse b: { - const selected = zml.attention.attention.Backend.auto(platform); - log.info("Selected backend: {}", .{selected}); - break :b selected; - }; + const backend = args.backend orelse if (args.attnd_ip) |attnd_ip| b: { + try zml.attention.attnd.register(allocator, io, platform, .{ + .destination = try .parseLiteral(attnd_ip), + .mtu = 9000, + }); + break :b zml.attention.attention.Backend.attnd; + } else zml.attention.attention.Backend.auto(platform); + defer if (args.attnd_ip) |_| zml.attention.attnd.deinit(); + log.info("Selected backend: {}", .{backend}); // // Model initialization diff --git a/examples/llm/models/llama/inference.zig b/examples/llm/models/llama/inference.zig index 8fed5ab9..893a330f 100644 --- a/examples/llm/models/llama/inference.zig +++ b/examples/llm/models/llama/inference.zig @@ -15,11 +15,13 @@ pub const CompilationParameters = struct { kv_cache: model.KvCache, rng: zml.Tensor.Rng, attention_metadata: attention.Metadata, - attention_parameters: attention.Parameters, + prefill_attention_parameters: attention.Parameters, + decode_attention_parameters: attention.Parameters, seqlen: usize, shardings: common.Shardings, pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: attention.Backend, shardings: common.Shardings) CompilationParameters { + const head_dim = config.head_dim orelse @divExact(config.hidden_size, config.num_attention_heads); return .{ .prefill_tokens = .init(.{ .s = seqlen }, .u32), .decode_tokens = .init(.{ .s = 1 }, .u32), @@ -28,11 +30,33 @@ pub const CompilationParameters = struct { .layer = mdl.model.layers.len, .k = seqlen, .h = config.num_key_value_heads, - .hd = config.head_dim orelse @divExact(config.hidden_size, config.num_attention_heads), + .hd = head_dim, }, mdl.model.embed_tokens.weight.dtype())), .rng = .init(), - .attention_metadata = .init(.fromBackend(backend, @intCast(seqlen), @intCast(config.num_attention_heads))), - .attention_parameters = .init(.fromBackend(backend)), + .attention_metadata = switch (backend) { + .attnd => .{ .attnd = .init() }, + else => .init(.fromBackend(backend, @intCast(seqlen), @intCast(config.num_attention_heads))), + }, + .prefill_attention_parameters = switch (backend) { + .attnd => .{ .attnd = .init(.{ + .model_id = .@"llama-3.1-8B", + .head_dim = head_dim, + .num_attention_heads = config.num_attention_heads, + .num_kv_heads = @intCast(config.num_key_value_heads), + .is_prefill = true, + }) }, + else => .init(.fromBackend(backend)), + }, + .decode_attention_parameters = switch (backend) { + .attnd => .{ .attnd = .init(.{ + .model_id = .@"llama-3.1-8B", + .head_dim = head_dim, + .num_attention_heads = config.num_attention_heads, + .num_kv_heads = @intCast(config.num_key_value_heads), + .is_prefill = false, + }) }, + else => .init(.fromBackend(backend)), + }, .seqlen = seqlen, .shardings = shardings, }; @@ -107,9 +131,11 @@ fn compileModel( parameters_.kv_cache, parameters_.rng, parameters_.attention_metadata, - parameters_.attention_parameters, + parameters_.prefill_attention_parameters, + }, + .{ + .shardings = &shardings_, }, - .{ .shardings = &shardings_ }, ); } }.call, .{ allocator, io, platform, llama_model, parameters, all_shardings, progress }); @@ -141,7 +167,7 @@ fn compileModel( parameters_.kv_cache, parameters_.rng, parameters_.attention_metadata, - parameters_.attention_parameters, + parameters_.decode_attention_parameters, }, .{ .shardings = shardings_ }, ); diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index 6f6c06d6..ae736777 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -293,11 +293,21 @@ pub const Llama = struct { var updated_kv_cache = kv_cache; for (self.layers, 0..) |layer, i| { + const layer_attention_metadata: zml.attention.attention.Metadata = switch (attention_parameters) { + .attnd => .{ .attnd = .{ + .layer_id = zml.Tensor.scalar(i, .u16), + .conversation_id = attention_metadata.attnd.conversation_id, + .num_tokens = attention_metadata.attnd.num_tokens, + } }, + .vanilla => attention_metadata, + .cuda_fa2 => attention_metadata, + .cuda_fa3 => attention_metadata, + }; hidden, updated_kv_cache = layer.forward( hidden, token_index, updated_kv_cache.atLayer(i), - attention_metadata, + layer_attention_metadata, attention_parameters, ); } diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig index 5b170272..464e36b7 100644 --- a/examples/llm/models/llama/session.zig +++ b/examples/llm/models/llama/session.zig @@ -18,6 +18,7 @@ pub const Session = struct { config: *const model.Config, seqlen: u32, last_generated_token: u32 = 0, + conversation_id: u64, pub fn init( allocator: std.mem.Allocator, @@ -50,6 +51,7 @@ pub const Session = struct { .tokenizer = tokenizer, .config = &compiled_model.loaded_model.parsed_config.value, .seqlen = @intCast(compiled_model.params.seqlen), + .conversation_id = @bitCast(std.Io.Clock.now(.real, io).toMicroseconds()), }; } @@ -124,13 +126,22 @@ pub const Session = struct { var prefill_token_pos_buffer = try zml.Buffer.scalar(self.io, self.platform, 0, .u32); defer prefill_token_pos_buffer.deinit(); + const attention_metadata_buffers: zml.Bufferized(zml.attention.attention.Metadata) = switch (self.compiled_model.params.prefill_attention_parameters) { + .attnd => .{ .attnd = .{ + .conversation_id = try zml.Buffer.scalar(self.io, self.platform, self.conversation_id, .u64), + .layer_id = try zml.Buffer.scalar(self.io, self.platform, 0, .u16), + .num_tokens = try zml.Buffer.scalar(self.io, self.platform, all_tokens.len, .u32), + } }, + .vanilla, .cuda_fa2, .cuda_fa3 => self.attention_metadata_buffers, + }; + prefill_args.set(.{ self.model_buffers, prefill_tokens_buffer, prefill_token_pos_buffer, &self.kv_cache_buffers, &self.rng_buffers, - &self.attention_metadata_buffers, + &attention_metadata_buffers, }); self.compiled_model.prefill_exe.call(prefill_args, &prefill_results); @@ -171,12 +182,21 @@ pub const Session = struct { var token_pos_buffer: zml.Buffer = try .scalar(self.io, self.platform, all_tokens.items.len, .u32); defer token_pos_buffer.deinit(); + const attention_metadata_buffers: zml.Bufferized(zml.attention.attention.Metadata) = switch (self.compiled_model.params.decode_attention_parameters) { + .attnd => .{ .attnd = .{ + .conversation_id = try zml.Buffer.scalar(self.io, self.platform, self.conversation_id, .u64), + .layer_id = try zml.Buffer.scalar(self.io, self.platform, 0, .u16), + .num_tokens = try zml.Buffer.scalar(self.io, self.platform, 1, .u32), + } }, + .vanilla, .cuda_fa2, .cuda_fa3 => self.attention_metadata_buffers, + }; + decode_args.set(.{ current_token_buffer, token_pos_buffer, &self.kv_cache_buffers, &self.rng_buffers, - &self.attention_metadata_buffers, + &attention_metadata_buffers, }); self.compiled_model.decode_exe.call(decode_args, &decode_results); diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 9adc472a..7af1de68 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -67,6 +67,7 @@ zig_library( srcs = [ "attention.zig", "attention/attention.zig", + "attention/attnd.zig", "attention/flashattn.zig", "attention/mosaic_tpu_kernels/ragged_attention.zig", "attention/paged_attention.zig", diff --git a/zml/attention.zig b/zml/attention.zig index ced93be2..ddcbd3fa 100644 --- a/zml/attention.zig +++ b/zml/attention.zig @@ -1,4 +1,5 @@ pub const attention = @import("attention/attention.zig"); +pub const attnd = @import("attention/attnd.zig"); pub const paged_attention = @import("attention/paged_attention.zig"); pub const flashattn = @import("attention/flashattn.zig"); pub const tpu = @import("attention/tpu_attention.zig"); diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index 8809aaa7..ca6345c6 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -1,12 +1,14 @@ const std = @import("std"); const zml = @import("../zml.zig"); +const attnd = @import("attnd.zig"); const flashattn = @import("flashattn.zig"); const Attention = @This(); pub const Backend = enum { vanilla, + attnd, cuda_fa2, cuda_fa3, @@ -31,17 +33,20 @@ pub const Backend = enum { pub const Parameters = union(Backend) { vanilla: void, + attnd: attnd.Parameters, cuda_fa2: flashattn.fa2.Parameters, cuda_fa3: flashattn.fa3.Parameters, pub const InitOptions = union(Backend) { vanilla: void, + attnd: void, cuda_fa2: flashattn.fa2.Parameters.InitOptions, cuda_fa3: flashattn.fa3.Parameters.InitOptions, pub fn fromBackend(backend: Backend) InitOptions { return switch (backend) { .vanilla => .{ .vanilla = {} }, + .attnd => @panic("Must be initialized manually"), .cuda_fa2 => .{ .cuda_fa2 = .{} }, .cuda_fa3 => .{ .cuda_fa3 = .{} }, }; @@ -51,6 +56,7 @@ pub const Parameters = union(Backend) { pub fn init(opts: InitOptions) Parameters { return switch (opts) { .vanilla => .{ .vanilla = {} }, + .attnd => @panic("Must be initialized manually"), .cuda_fa2 => |v| .{ .cuda_fa2 = .init(v) }, .cuda_fa3 => |v| .{ .cuda_fa3 = .init(v) }, }; @@ -59,17 +65,20 @@ pub const Parameters = union(Backend) { pub const Metadata = union(Backend) { vanilla: void, + attnd: attnd.Metadata, cuda_fa2: flashattn.fa2.Metadata, cuda_fa3: flashattn.fa3.Metadata, pub const InitOptions = union(Backend) { vanilla: void, + attnd: void, cuda_fa2: flashattn.fa2.Metadata.InitOptions, cuda_fa3: flashattn.fa3.Metadata.InitOptions, pub fn fromBackend(backend: Backend, seqlen: i64, num_heads: i64) InitOptions { return switch (backend) { .vanilla => .{ .vanilla = {} }, + .attnd => .{ .attnd = {} }, .cuda_fa2 => .{ .cuda_fa2 = .{ .seqlen = seqlen, .num_heads = num_heads } }, .cuda_fa3 => .{ .cuda_fa3 = .{ .seqlen = seqlen, .num_heads = num_heads } }, }; @@ -79,6 +88,7 @@ pub const Metadata = union(Backend) { pub fn init(opts: InitOptions) Metadata { return switch (opts) { .vanilla => .{ .vanilla = {} }, + .attnd => @panic("Must be initialized manually"), .cuda_fa2 => |o| .{ .cuda_fa2 = flashattn.fa2.Metadata.init(o) }, .cuda_fa3 => |o| .{ .cuda_fa3 = flashattn.fa3.Metadata.init(o) }, }; @@ -87,6 +97,7 @@ pub const Metadata = union(Backend) { pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(Metadata) { return switch (self) { .vanilla => .{ .vanilla = {} }, + .attnd => |v| .{ .attnd = try v.initBuffer(io, platform, sharding) }, .cuda_fa2 => |v| .{ .cuda_fa2 = try v.initBuffer(io, platform, sharding) }, .cuda_fa3 => |v| .{ .cuda_fa3 = try v.initBuffer(io, platform, sharding) }, }; @@ -95,6 +106,7 @@ pub const Metadata = union(Backend) { pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { switch (self.*) { .vanilla => {}, + .attnd => |*v| attnd.Metadata.deinitBuffer(v), .cuda_fa2 => |*v| flashattn.fa2.Metadata.deinitBuffer(v), .cuda_fa3 => |*v| flashattn.fa3.Metadata.deinitBuffer(v), } @@ -114,6 +126,7 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask, .allow_cudnn = true }); break :b attn_output; }, + .attnd => attnd.causalAttention(q, k, v, token_index, metadata.attnd, parameters.attnd), .cuda_fa2 => flashattn.fa2.attention(q, k, v, token_index, metadata.cuda_fa2, parameters.cuda_fa2), .cuda_fa3 => flashattn.fa3.attention(q, k, v, token_index, metadata.cuda_fa3, parameters.cuda_fa3), }; diff --git a/zml/attention/attnd.zig b/zml/attention/attnd.zig new file mode 100644 index 00000000..ddf9c04f --- /dev/null +++ b/zml/attention/attnd.zig @@ -0,0 +1,501 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const stdx = @import("stdx"); + +const zml = @import("../zml.zig"); + +const log = std.log.scoped(.@"zml/attnd"); +var context: ?*Context = null; + +pub const Config = struct { + destination: std.Io.net.IpAddress, + mtu: u16, +}; + +pub const Parameters = struct { + model_id: ModelId, + head_dim: u32, + head_bytes: u32, + num_kv_heads: u8, + num_q_per_head: u8, + num_q_per_packet: u8, + is_prefill: bool, + + pub const Init = struct { + model_id: ModelId, + head_dim: u32, + num_attention_heads: u32, + num_kv_heads: u8, + is_prefill: bool, + }; + + pub fn init(args: Init) Parameters { + const mtu: u32 = if (context) |ctx| @intCast(ctx.mtu) else @panic("attnd wasn't registered yet, use register() first."); + const num_q_per_head: u8 = @intCast(@divExact(args.num_attention_heads, args.num_kv_heads)); + const head_bytes = args.head_dim * @sizeOf(u16); // TODO: Here we assume bf16 or equivalent. + const kv_bytes = 2 * head_bytes; + + var max_q_per_packet: u8 = @intCast(std.math.divFloor(u32, mtu - @sizeOf(Header) - kv_bytes, head_bytes) catch unreachable); + max_q_per_packet = @min(max_q_per_packet, num_q_per_head); + + const num_packets_per_head: u8 = @intCast(std.math.divCeil(u32, num_q_per_head, max_q_per_packet) catch unreachable); + const num_q_per_packet = @divExact(num_q_per_head, num_packets_per_head); + if (num_q_per_packet < num_q_per_head) { + log.warn("Will split attnd requests in {d} packets of {d} queries", .{ num_packets_per_head, max_q_per_packet }); + } + + return .{ + .model_id = args.model_id, + .head_dim = args.head_dim, + .head_bytes = head_bytes, + .num_kv_heads = args.num_kv_heads, + .num_q_per_head = num_q_per_head, + .num_q_per_packet = num_q_per_packet, + .is_prefill = args.is_prefill, + }; + } +}; + +pub const Metadata = struct { + conversation_id: zml.Tensor, + layer_id: zml.Tensor, + num_tokens: zml.Tensor, + + pub fn init() Metadata { + return .{ + .conversation_id = .fromShape(.scalar(.u64)), + .layer_id = .fromShape(.scalar(.u16)), + .num_tokens = .fromShape(.scalar(.u32)), + }; + } + + pub fn initBuffer( + self: Metadata, + io: std.Io, + platform: *const zml.Platform, + sharding: zml.Sharding, + ) !zml.Bufferized(Metadata) { + return .{ + .conversation_id = try zml.Buffer.scalar(io, platform, 749, .u64), + .layer_id = try zml.Buffer.uninitialized(io, platform, self.layer_id.shape(), sharding, .{}), + .num_tokens = try zml.Buffer.uninitialized(io, platform, self.num_tokens.shape(), sharding, .{}), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { + self.conversation_id.deinit(); + self.layer_id.deinit(); + self.num_tokens.deinit(); + } +}; + +pub fn register(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, config: Config) !void { + var ctx = try allocator.create(Context); + errdefer allocator.destroy(ctx); + + try ctx.initialize(io, allocator, platform, config); + + context = ctx; + + try targets.attnd.register(platform); +} + +pub fn deinit() void { + if (context) |ctx| { + ctx.deinit(); + context = null; + } +} + +const Context = struct { + allocator: std.mem.Allocator, + io: std.Io, + mtu: u16, + // One client per device. + clients: []Client, + + fn initialize(ctx: *Context, io: std.Io, allocator: std.mem.Allocator, platform: *const zml.Platform, config: Config) !void { + ctx.allocator = allocator; + ctx.io = io; + ctx.mtu = config.mtu; + ctx.clients = try allocator.alloc(Client, platform.devices.len); + errdefer allocator.free(ctx.clients); + + var c_len: usize = 0; + errdefer for (ctx.clients[0..c_len]) |*c| c.deinit(allocator, io); + for (ctx.clients) |*c| { + try c.initialize(allocator, io, config); + c_len += 1; + } + } + + fn deinit(self: *Context) void { + for (self.clients) |*c| { + c.deinit(self.allocator, self.io); + } + self.allocator.free(self.clients); + self.allocator.destroy(self); + } +}; + +pub fn causalAttention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_offset: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { + const ctx = zml.module.CompilationContext.current(); + const num_partitions = ctx.partitioning.numPartitionsForLogicalAxis(q.shape(), .model) catch unreachable; + + const actual_k, const actual_v = if (parameters.is_prefill) + .{ k, v } + else + .{ + k.dynamicSlice(.{ .k = zml.Tensor.DynSlice{ .start = token_offset, .len = 1 } }), + v.dynamicSlice(.{ .k = zml.Tensor.DynSlice{ .start = token_offset, .len = 1 } }), + }; + + // Concatenate q/k/v and metadata into a single byte buffer on the GPU. + // This reduces 7 separate D2H transfers to 1 during host-compute offload. + const packed_input = zml.Tensor.concatenate(&.{ + q.bytes(), + actual_k.bytes(), + actual_v.bytes(), + metadata.conversation_id.bytes(), + metadata.layer_id.bytes(), + token_offset.bytes(), + metadata.num_tokens.bytes(), + }, .bytes); + + const out = targets.attnd.call( + .{ .mqkv = packed_input }, + .{ + .attn = q.shape(), + }, + .{ + .model_id = parameters.model_id, + .head_dim = parameters.head_dim, + .head_size_in_bytes = parameters.head_bytes, + .num_kv_heads = parameters.num_kv_heads, + .num_q_per_head = parameters.num_q_per_head, + .num_q_per_packet = parameters.num_q_per_packet, + .num_partitions = @intCast(num_partitions), + .q_byte_size = @intCast(q.shape().byteSize()), + .k_byte_size = @intCast(actual_k.shape().byteSize()), + .v_byte_size = @intCast(actual_v.shape().byteSize()), + }, + ); + + return out.attn; +} + +pub const targets = struct { + const NAME = "attnd"; + + pub const attnd = zml.ops.CustomCall(Input, Output, Attributes, attndCall, .{ + .name = NAME, + .sharding_aware = true, + .has_side_effect = false, + .compute_on_host = true, + }); +}; + +const Input = struct { + mqkv: zml.Tensor, +}; + +const Output = struct { + attn: zml.Shape, +}; + +const Attributes = struct { + model_id: ModelId, + head_dim: u32, + head_size_in_bytes: u32, + num_kv_heads: u8, + num_q_per_head: u8, + num_q_per_packet: u8, + num_partitions: u32, + /// Byte sizes for unpacking the concatenated packed buffer. + q_byte_size: u32, + k_byte_size: u32, + v_byte_size: u32, +}; + +fn attndCall( + call_frame: *zml.pjrt.ffi.CallFrame, + input: zml.pjrtx.TensorToCustomCallBuffer(Input), + output: zml.pjrtx.ShapeToCustomCallBuffer(Output), + attrs: Attributes, +) !?*zml.pjrt.ffi.Error { + const ctx: *Context = context.?; + const device_ordinal: u64 = @intCast(try call_frame.ctx.getDeviceOrdinal(call_frame.api)); + + const client = &ctx.clients[device_ordinal]; + // Unpack the concatenated byte buffer: [q_bytes | k_bytes | v_bytes | conversation_id(8) | layer_id(2) | token_offset(4) | num_tokens(4)] + const out_full = output.attn.asHostSlice(); + const mqkv = input.mqkv.asHostConstSlice(); + const q_end = attrs.q_byte_size; + const k_end = q_end + attrs.k_byte_size; + const v_end = k_end + attrs.v_byte_size; + const q_full = mqkv[0..q_end]; + const k_full = mqkv[q_end..k_end]; + const v_full = mqkv[k_end..v_end]; + const conversation_id = std.mem.bytesAsValue(u64, mqkv[v_end..][0..@sizeOf(u64)]).*; + const layer_id = std.mem.bytesAsValue(u16, mqkv[v_end + @sizeOf(u64) ..][0..@sizeOf(u16)]).*; + const token_offset = std.mem.bytesAsValue(u32, mqkv[v_end + @sizeOf(u64) + @sizeOf(u16) ..][0..@sizeOf(u32)]).*; + const num_tokens = std.mem.bytesAsValue(u32, mqkv[v_end + @sizeOf(u64) + @sizeOf(u16) + @sizeOf(u32) ..][0..@sizeOf(u32)]).*; + + const q_size_in_bytes = attrs.num_q_per_head * attrs.head_size_in_bytes; + const head_size_in_bytes = attrs.head_size_in_bytes; + const kv_heads_per_partitions = @divExact(attrs.num_kv_heads, attrs.num_partitions); + const kv_head_offset = kv_heads_per_partitions * device_ordinal; + for (0..num_tokens) |t| { + const q = q_full[t * kv_heads_per_partitions * q_size_in_bytes ..][0 .. kv_heads_per_partitions * q_size_in_bytes]; + const k = k_full[t * kv_heads_per_partitions * head_size_in_bytes ..][0 .. kv_heads_per_partitions * head_size_in_bytes]; + const v = v_full[t * kv_heads_per_partitions * head_size_in_bytes ..][0 .. kv_heads_per_partitions * head_size_in_bytes]; + + var num_send_messages: usize = 0; + for (0..kv_heads_per_partitions) |kv_head_id| { + const k_offset = kv_head_id * head_size_in_bytes; + const v_offset = kv_head_id * head_size_in_bytes; + var q_offset = kv_head_id * head_size_in_bytes * attrs.num_q_per_head; + var q_sent: u8 = 0; + + while (q_sent < attrs.num_q_per_head) { + const nq = @min(attrs.num_q_per_head - q_sent, attrs.num_q_per_packet); + const header: Header = .{ + .type = .attn, + .kv_head_id = @intCast(kv_head_offset + kv_head_id), + .conversation_id = conversation_id, + .token_pos = token_offset + @as(u32, @intCast(t)), + .layer_id = layer_id, + .model_id = attrs.model_id, + .first_q_id = q_sent, + .num_queries = nq, + }; + try client.send( + header, + q[q_offset..][0 .. head_size_in_bytes * nq], + k[k_offset..][0..head_size_in_bytes], + v[v_offset..][0..head_size_in_bytes], + ); + + q_offset += head_size_in_bytes * nq; + q_sent += nq; + num_send_messages += 1; + } + } + + var out = out_full[t * kv_heads_per_partitions * q_size_in_bytes ..][0 .. kv_heads_per_partitions * q_size_in_bytes]; + var recv_q: usize = 0; + while (recv_q < kv_heads_per_partitions * attrs.num_q_per_head) { + const response = client.receive() catch |err| { + log.err("Failed to receive response: {any}", .{err}); + return err; + }; + const header = response.header; + const payload = response.payload; + + std.debug.assert(header.num_queries <= attrs.num_q_per_head); + std.debug.assert(header.kv_head_id < attrs.num_kv_heads); + std.debug.assert(header.first_q_id + header.num_queries <= attrs.num_q_per_head); + + const n = header.num_queries * attrs.head_size_in_bytes; + std.debug.assert(payload.len == n); + const kv_head_id = header.kv_head_id - kv_head_offset; + const q_offset = (@as(usize, kv_head_id) * attrs.num_q_per_head + header.first_q_id) * attrs.head_size_in_bytes; + @memcpy(out[q_offset .. q_offset + n], payload[0..n]); + recv_q += header.num_queries; + num_send_messages -= 1; + } + std.debug.assert(num_send_messages == 0); + } + + return null; +} + +const MAX_CONCURRENT_PACKETS_PER_CLIENT = 32; + +const Client = struct { + socket: std.Io.net.Socket, + destination: std.Io.net.IpAddress, + packet_buffer: []u8, + send_addr: std.Io.Threaded.PosixAddress, + send_namelen: std.posix.socklen_t, + + fn initialize(self: *Client, allocator: std.mem.Allocator, io: std.Io, config: Config) !void { + const src: std.Io.net.IpAddress = .{ .ip4 = .unspecified(0) }; + const socket = try src.bind(io, .{ .mode = .dgram, .protocol = .udp }); + errdefer socket.close(io); + + const socket_buffer_size = @as(u32, @intCast(config.mtu)) * MAX_CONCURRENT_PACKETS_PER_CLIENT; + try std.posix.setsockopt(socket.handle, std.posix.SOL.SOCKET, std.posix.SO.RCVBUF, @ptrCast(&socket_buffer_size)); + try std.posix.setsockopt(socket.handle, std.posix.SOL.SOCKET, std.posix.SO.SNDBUF, @ptrCast(&socket_buffer_size)); + + // Set the Don't Fragment bit to avoid fragmentation on IPv4 + switch (builtin.os.tag) { + .linux => { + // With PMTUDISC_DO the kernel will keep track of the ICMP messages to detect the real MTU. + // Best practice [1] seems to be to use IP_PMTUDISC_PROBE and do DPLPMTUD to avoid Blind Performance-Degrading Attack [2] + // But we assume to be within an internal network, otherwise attnd is useless anyway. + // [1] https://seemann.io/posts/2025-02-19---ip-fragmentation/ + // [2] https://www.rfc-editor.org/rfc/rfc5927#section-7 + // TODO: Do we actually try to use the max MTU? If we just assume it to be fine, we could just as well use IP_PMTUDISC_PROBE + const mtu_discover: u32 = std.os.linux.IP.PMTUDISC_DO; + try std.posix.setsockopt(socket.handle, std.posix.IPPROTO.IP, std.os.linux.IP.MTU_DISCOVER, @ptrCast(&mtu_discover)); + }, + .macos => { + // We could set IP_DONTFRAG on the socket, but Zig doesn't expose this option (yet). + }, + else => {}, + } + + const packet_buffer = try allocator.alloc(u8, @as(usize, @intCast(config.mtu))); + errdefer allocator.free(packet_buffer); + + self.socket = socket; + self.destination = config.destination; + self.packet_buffer = packet_buffer; + self.send_namelen = std.Io.Threaded.addressToPosix(&self.destination, &self.send_addr); + } + + pub fn send(self: *Client, header: Header, q: []const u8, k: []const u8, v: []const u8) !void { + const iovecs = [_]std.posix.iovec_const{ + .{ .base = std.mem.asBytes(&header).ptr, .len = @sizeOf(Header) }, + .{ .base = q.ptr, .len = q.len }, + .{ .base = k.ptr, .len = k.len }, + .{ .base = v.ptr, .len = v.len }, + }; + const msg: std.posix.msghdr_const = .{ + .name = &self.send_addr.any, + .namelen = self.send_namelen, + .iov = &iovecs, + .iovlen = iovecs.len, + .control = null, + .controllen = 0, + .flags = 0, + }; + + const rc = std.c.sendmsg(self.socket.handle, &msg, std.posix.MSG.DONTWAIT); + switch (std.posix.errno(rc)) { + .SUCCESS => { + std.debug.assert(rc == @sizeOf(Header) + q.len + k.len + v.len); + }, + else => |err| { + log.err("Failed to send response: {any}", .{err}); + @panic("Attnd failed"); + }, + } + } + + pub fn receive(self: *Client) !Response { + var iovecs = [_]std.posix.iovec{ + .{ .base = self.packet_buffer.ptr, .len = self.packet_buffer.len }, + }; + var msg: std.posix.msghdr = .{ + .name = null, + .namelen = 0, + .iov = &iovecs, + .iovlen = iovecs.len, + .control = null, + .controllen = 0, + .flags = 0, + }; + + const rc = std.c.recvmsg( + self.socket.handle, + &msg, + 0, + ); + switch (std.posix.errno(rc)) { + .SUCCESS => { + const n: usize = @intCast(rc); + return Response.parse(self.packet_buffer[0..n]); + }, + else => |err| { + log.err("Failed to receive response: {any}", .{err}); + @panic("Attnd failed"); + }, + } + } + + pub fn deinit(self: *Client, allocator: std.mem.Allocator, io: std.Io) void { + allocator.free(self.packet_buffer); + self.socket.close(io); + } +}; + +pub const Message = struct { + header: *align(1) const Header, + payload: []const u8, + + pub fn parse(bytes: []const u8) !Message { + const n = @sizeOf(Header); + if (bytes.len < n) return error.ShortMessage; + + const msg: Message = .{ + .header = std.mem.bytesAsValue(Header, bytes[0..n]), + .payload = bytes[n..], + }; + + if (!(msg.header.magic[0] == 'Z' and msg.header.magic[1] == 'M' and msg.header.magic[2] == 'L')) { + return error.InvalidHeader; + } + + return msg; + } +}; + +pub const Request = struct { + header: *align(1) const Header, + q: []const u8, + k: []const u8, + v: []const u8, +}; + +pub const Response = Message; + +pub const Header = extern struct { + pub const MAGIC: [3]u8 = .{ 'Z', 'M', 'L' }; // 0x5A4D4C + + pub const Type = enum(u8) { + // Compute attention + attn = 'a', + // Starts a conversation + start = 's', + // Ping: the server answers immediately skipping all compute + ping = 'p', + _, + }; + + type: Type, + kv_head_id: u8, + + conversation_id: u64 align(1), + token_pos: u32 align(1), + layer_id: u16 align(1), + + model_id: ModelId, + first_q_id: u8, + num_queries: u8, + magic: [3]u8 align(1) = MAGIC, + + comptime { + // destination MAC (6) + source MAC (6) + Ethertype (2) [+ 802.1Q / VLAN tag (4)] + // We the optional VLAN, we can only guarantee an alignment of 4 for the payload after the prefix + header. + const ETH_HEADER_SIZE = 6 + 6 + 2; + const IP_HEADER_SIZE = 20; + const UDP_HEADER_SIZE = 8; + const PREFIX_SIZE = ETH_HEADER_SIZE + IP_HEADER_SIZE + UDP_HEADER_SIZE; + stdx.debug.assertComptime( + (PREFIX_SIZE + @sizeOf(Header)) % 4 == 0, + "ETH_HEADER_SIZE ({}) + @sizeOf(AttnRequest) ({}) == {d} != k * 4", + .{ PREFIX_SIZE, @sizeOf(Header), PREFIX_SIZE + @sizeOf(Header) }, + ); + } +}; + +pub const ModelId = enum(u8) { + @"llama-3.1-8B" = 0, + @"llama-3.2-1B" = 1, + @"qwen3-14B" = 2, + @"qwen3-32B" = 3, + _, // Unsupported models +}; diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 24d03906..6d6ada25 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -170,8 +170,8 @@ pub const fa2 = struct { fn ffiCall( call_frame: *zml.pjrt.ffi.CallFrame, - input: zml.ops.TensorToCustomCallBuffer(Input), - output: zml.ops.ShapeToCustomCallBuffer(Output), + input: zml.pjrtx.TensorToCustomCallBuffer(Input), + output: zml.pjrtx.ShapeToCustomCallBuffer(Output), attributes: Attributes, ) !?*zml.pjrt.ffi.Error { const params: flashattn.FA2MhaVarlenFwdParams = .{ From 8418c9762488e2c9f0d7ad07ea6cb352c1a49883 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 13 May 2026 10:39:07 +0200 Subject: [PATCH 482/881] zml/tensor: add `Tensor.onMemory()` (#534) It is now required with newer plugin to annotate the executable arguments with their location. Like `Tensor.reuseBuffer()`, you can now use `Tensor.onMemory()` to specify on which memory a tensor is located. --- zml/module.zig | 42 +++++++++++++++++++++++++++++++++--------- zml/tensor.zig | 17 +++++++++++++++++ 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/zml/module.zig b/zml/module.zig index 9ef99e6d..83678d85 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -63,6 +63,7 @@ pub const CompilationContext = struct { id_to_argument: std.AutoArrayHashMapUnmanaged(usize, usize), id_to_donation: std.AutoArrayHashMapUnmanaged(usize, usize), id_to_output_memory_kind: std.AutoArrayHashMapUnmanaged(usize, Memory.Kind), + id_to_input_memory_kind: std.AutoArrayHashMapUnmanaged(usize, Memory.Kind), arena: std.heap.ArenaAllocator, pub fn initFromBlock(allocator: std.mem.Allocator, block: *mlir.Block) Scope { @@ -72,6 +73,7 @@ pub const CompilationContext = struct { .id_to_argument = .empty, .id_to_donation = .empty, .id_to_output_memory_kind = .empty, + .id_to_input_memory_kind = .empty, .arena = arena, }; } @@ -358,10 +360,12 @@ fn collectOutputInfo(allocator: std.mem.Allocator, partitioning: Sharding.Partit pub const InputInfo = struct { shapes: []Shape, shardings: []Sharding, + memory_kinds: []?Memory.Kind, pub fn deinit(self: InputInfo, allocator: std.mem.Allocator) void { allocator.free(self.shapes); allocator.free(self.shardings); + allocator.free(self.memory_kinds); } }; @@ -369,6 +373,7 @@ fn collectInputInfo(allocator: std.mem.Allocator, partitioning: Sharding.Partiti const LocalContext = struct { shape_list: *std.array_list.Managed(Shape), sharding_list: *std.array_list.Managed(Sharding), + memory_kind_list: *std.array_list.Managed(?Memory.Kind), partitioning: Sharding.Partitioning, }; @@ -378,9 +383,13 @@ fn collectInputInfo(allocator: std.mem.Allocator, partitioning: Sharding.Partiti var sharding_list = std.array_list.Managed(Sharding).init(allocator); errdefer sharding_list.deinit(); + var memory_kind_list = std.array_list.Managed(?Memory.Kind).init(allocator); + errdefer memory_kind_list.deinit(); + var context: LocalContext = .{ .shape_list = &shape_list, .sharding_list = &sharding_list, + .memory_kind_list = &memory_kind_list, .partitioning = partitioning, }; @@ -388,12 +397,14 @@ fn collectInputInfo(allocator: std.mem.Allocator, partitioning: Sharding.Partiti fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { try ctx_.shape_list.append(tensor.shape()); try ctx_.sharding_list.append(try ctx_.partitioning.selectSharding(tensor.shape())); + try ctx_.memory_kind_list.append(tensor.inputMemoryKind()); } }.cb, &context, v); return .{ .shapes = try shape_list.toOwnedSlice(), .shardings = try sharding_list.toOwnedSlice(), + .memory_kinds = try memory_kind_list.toOwnedSlice(), }; } @@ -443,25 +454,26 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar } }.cb, &context, &args); - const input_info = try collectInputInfo(compilation_context.allocator, compilation_context.partitioning, &args); - errdefer input_info.deinit(compilation_context.allocator); - - const input_attributes = try arena.allocator().alloc(AttributeList, input_info.shapes.len); - @memset(input_attributes, .empty); - - const output_info = b: { + const output_info, const input_info = b: { compilation_context.activate(); defer compilation_context.deactivate(); const result = @call(.auto, func, args); + const input_info = try collectInputInfo(compilation_context.allocator, compilation_context.partitioning, &args); + errdefer input_info.deinit(compilation_context.allocator); + const output_info = try collectOutputInfo(compilation_context.allocator, compilation_context.partitioning, &result); errdefer output_info.deinit(compilation_context.allocator); - break :b output_info; + break :b .{ output_info, input_info }; }; + errdefer input_info.deinit(compilation_context.allocator); errdefer output_info.deinit(compilation_context.allocator); + const input_attributes = try arena.allocator().alloc(AttributeList, input_info.shapes.len); + @memset(input_attributes, .empty); + const output_attributes = try arena.allocator().alloc(AttributeList, output_info.shapes.len); @memset(output_attributes, .empty); @@ -481,7 +493,7 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar } _ = dialects.func.returns(compilation_context.mlir_ctx, output_info.values, .unknown(compilation_context.mlir_ctx)).appendTo(compilation_context.currentScope().block); - for (input_info.shapes, input_info.shardings, 0..) |shape, sharding, i| { + for (input_info.shapes, input_info.shardings, input_info.memory_kinds, 0..) |shape, sharding, maybe_memory_kind, i| { const attr = try compilation_context.partitioning.tensorShardingAttr(compilation_context.arena.allocator(), compilation_context.mlir_ctx, shape, sharding); const name = switch (compilation_context.partitioning.partitioner) { .gspmd => "mhlo.sharding", @@ -489,6 +501,18 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar }; input_attributes[i].appendAssumeCapacity(.named(compilation_context.mlir_ctx, name, attr)); + + if (maybe_memory_kind) |memory_kind| { + if (memory_kind == .device) continue; + input_attributes[i].appendAssumeCapacity(.named( + compilation_context.mlir_ctx, + "mhlo.memory_kind", + mlir.stringAttribute( + compilation_context.mlir_ctx, + compilation_context.platform.memoryKind(memory_kind), + ), + )); + } } for (output_info.shapes, output_info.shardings, 0..) |shape, sharding, i| { diff --git a/zml/tensor.zig b/zml/tensor.zig index 87166295..a1e82740 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -212,6 +212,19 @@ pub const Tensor = struct { return res; } + pub fn onMemory(self: Tensor, kind: Memory.Kind) Tensor { + const ctx = CompilationContext.current(); + if (ctx.platform.target == .cpu) return self; + + if (ctx.currentScope().id_to_argument.get(self.id) == null) { + return self; + } + + ctx.currentScope().id_to_input_memory_kind.put(ctx.currentScope().arena.allocator(), self.id, kind) catch unreachable; + + return self; + } + /// Returns a Tensor with new tag names. pub fn rename(self: Tensor, renames: anytype) Tensor { var res = self; @@ -4124,6 +4137,10 @@ pub const Tensor = struct { pub fn outputMemoryKind(self: Tensor) Memory.Kind { return CompilationContext.current().currentScope().id_to_output_memory_kind.get(self.id) orelse .device; } + + pub fn inputMemoryKind(self: Tensor) ?Memory.Kind { + return CompilationContext.current().currentScope().id_to_input_memory_kind.get(self.id); + } }; fn initPoolArg(rank: usize, data: []const i64) [constants.MAX_RANK]i64 { From 3b718a5a895e150ccab6416d5a98fe83ce2eba84 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 13 May 2026 10:40:20 +0200 Subject: [PATCH 483/881] zml/tensor: add `Tensor.uninitialized()` (#535) This PR exposes the XLA-specific custom calls that allow to create uninitialized tensors (thus not paying the cost of a kernel to essentially do a memset). The `stablehlo` part of it was already done, so I added a wrapper in `tensor.zig` --- mlir/dialects/stablehlo/stablehlo.zig | 28 +++++++++++++++++++-------- zml/tensor.zig | 15 ++++++++++++++ 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 180cc2b0..bc2de9e4 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -883,17 +883,29 @@ pub fn createBuffer(ctx: *mlir.Context, value: *const mlir.Type, location: *cons } pub fn pin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return custom_call(ctx, &.{}, &.{@ptrCast(mlir.MemRefType.fromShaped(value.type_().isA(mlir.ShapedType).?))}, .{ - .call_target_name = "Pin", - .api_version = .typed_ffi, - }, location); + return custom_call( + ctx, + &.{value}, + &.{@ptrCast(mlir.MemRefType.fromShaped(value.type_().isA(mlir.ShapedType).?))}, + .{ + .call_target_name = "Pin", + .api_version = .typed_ffi, + }, + location, + ); } pub fn unpin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return custom_call(ctx, &.{}, &.{@ptrCast(mlir.RankedTensorType.fromShaped(value.type_().isA(mlir.ShapedType).?))}, .{ - .call_target_name = "Unpin", - .api_version = .typed_ffi, - }, location); + return custom_call( + ctx, + &.{value}, + &.{@ptrCast(mlir.RankedTensorType.fromShaped(value.type_().isA(mlir.ShapedType).?))}, + .{ + .call_target_name = "Unpin", + .api_version = .typed_ffi, + }, + location, + ); } pub fn partition_id(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { diff --git a/zml/tensor.zig b/zml/tensor.zig index a1e82740..f805b80f 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -2103,6 +2103,21 @@ pub const Tensor = struct { return Tensor.constant(sh.dtype().zero()).broad(sh); } + pub fn uninitialized(sh: Shape) Tensor { + const ctx = CompilationContext.current(); + const buffer_type = mlir.memRefType( + mlirx.Type.fromDType(ctx.mlir_ctx, sh.dtype()), + sh.dims(), + null, + null, + ); + const buffer = dialects.stablehlo.createBuffer(ctx.mlir_ctx, buffer_type, .unknown(ctx.mlir_ctx)) + .appendTo(currentBlock()); + const tensor = dialects.stablehlo.unpin(ctx.mlir_ctx, buffer.result(0), .unknown(ctx.mlir_ctx)) + .appendTo(currentBlock()); + return _result(sh, tensor.result(0)); + } + /// Embeds a buffer with concrete values into an Mlir program. pub fn constantTensor(sh: Shape, bytes_: []const u8) Tensor { const elem_type = mlirx.Type.fromDType(mlirCtx(), sh.dtype()); From c8cc4760428cb48bf184aa02084501b0d903a739 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Wed, 13 May 2026 11:35:34 +0000 Subject: [PATCH 484/881] third_party/zls: adapt to rules_zig changes notably when using with_cfg targets --- third_party/zls/zls_write_build_config.bzl | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/third_party/zls/zls_write_build_config.bzl b/third_party/zls/zls_write_build_config.bzl index fc4a04f9..bb260349 100644 --- a/third_party/zls/zls_write_build_config.bzl +++ b/third_party/zls/zls_write_build_config.bzl @@ -13,12 +13,20 @@ load( "zig_target_platform", ) +def _is_zig_target(kind): + if not kind.startswith("zig_"): + return False + for end in ("_binary", "_static_library", "_shared_library", "_test"): + if kind.endswith(end): + return True + return False + def _zls_construct_zig_module_info_impl(target, ctx): """Aspect that constructs ZigModuleInfo for zig_binary and zig_library rules.""" if ZigModuleInfo in target: return [] - if ctx.rule.kind not in ("zig_binary", "zig_static_library", "zig_shared_library", "zig_test"): + if _is_zig_target(ctx.rule.kind) == False: return [] cdeps = [] @@ -89,7 +97,6 @@ def _zls_write_build_config_impl(ctx): name = "c", canonical_name = "c", zigtoolchaininfo = zigtoolchaininfo, - global_args = global_args, cc_infos = [cc_info], ) From 37ef0a87c01742915ec3d21f4a44b0b25bbe376b Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 15 May 2026 10:53:54 +0200 Subject: [PATCH 485/881] mlir: fluent api (#533) --- kernels/mosaic_tpu/builder.zig | 125 +++++----- kernels/mosaic_tpu/dtype.zig | 22 +- kernels/triton/builder.zig | 128 +++++----- kernels/triton/dtype.zig | 22 +- mlir/dialects/affine.zig | 58 ++--- mlir/dialects/arith.zig | 17 +- mlir/dialects/cf.zig | 14 +- mlir/dialects/func.zig | 14 +- mlir/dialects/memref.zig | 71 +++--- mlir/dialects/mosaic_tpu/mosaic_tpu.zig | 82 +++---- mlir/dialects/scf.zig | 18 +- mlir/dialects/shardy.zig | 2 +- mlir/dialects/stablehlo/stablehlo.zig | 120 ++++----- mlir/dialects/ttir/ttir.zig | 188 +++++++------- mlir/dialects/vector.zig | 46 ++-- mlir/mlir.zig | 231 +++++++++--------- zml/Sharding.zig | 4 +- zml/attention.zig | 7 +- .../mosaic_tpu_kernels/ragged_attention.zig | 7 +- zml/attention/paged_attention.zig | 9 +- zml/attention/tpu_attention.zig | 20 +- zml/kernel.zig | 18 +- zml/mlirx.zig | 106 ++++---- zml/module.zig | 21 +- zml/ops.zig | 112 ++++----- zml/tensor.zig | 55 +++-- 26 files changed, 751 insertions(+), 766 deletions(-) diff --git a/kernels/mosaic_tpu/builder.zig b/kernels/mosaic_tpu/builder.zig index 963ac907..b6a0d4f4 100644 --- a/kernels/mosaic_tpu/builder.zig +++ b/kernels/mosaic_tpu/builder.zig @@ -7,13 +7,7 @@ const std = @import("std"); -const mlir = @import("mlir"); const dialects = @import("mlir/dialects"); -const dsl = @import("kernels/common"); -const dtypes = @import("dtype.zig"); -const stdx = @import("stdx"); -const tpu = @import("mlir/dialects/mosaic_tpu"); - const arith = dialects.arith; const cf = dialects.cf; const func = dialects.func; @@ -21,13 +15,14 @@ const math = dialects.math; const memref = dialects.memref; const scf = dialects.scf; const vector = dialects.vector; - -pub const DType = dtypes.DType; - -const isFloatDtype = dtypes.isFloatDtype; -const dtypeBitwidth = dtypes.dtypeBitwidth; -const intBitwidth = dtypes.intBitwidth; - +/// `vector`-dialect enums re-exported for convenience. +/// Used by `vectorReduction` / `multiReduction`. +pub const CombiningKind = vector.CombiningKind; +const dsl = @import("kernels/common"); +const tupleArity = dsl.tupleArity; +const mlir = @import("mlir"); +const stdx = @import("stdx"); +const tpu = @import("mlir/dialects/mosaic_tpu"); /// `tpu`-dialect enums re-exported for convenience. pub const MemorySpace = tpu.MemorySpace; pub const ReductionKind = tpu.ReductionKind; @@ -38,9 +33,11 @@ pub const CoreType = tpu.CoreType; pub const PipelineMode = tpu.PipelineMode; pub const RevisitMode = tpu.RevisitMode; -/// `vector`-dialect enums re-exported for convenience. -/// Used by `vectorReduction` / `multiReduction`. -pub const CombiningKind = vector.CombiningKind; +const dtypes = @import("dtype.zig"); +pub const DType = dtypes.DType; +const isFloatDtype = dtypes.isFloatDtype; +const dtypeBitwidth = dtypes.dtypeBitwidth; +const intBitwidth = dtypes.intBitwidth; // ============================================================================= // Value — a handle to an MLIR value inside the kernel body. @@ -602,8 +599,8 @@ pub const Builder = struct { for (args, 0..) |a, i| { arg_types[i] = switch (a.kind) { .scalar => |dt| dt.toMlir(ctx), - .iter => mlir.indexType(ctx), - .ref => |r| mlir.memRefType( + .iter => .index(ctx), + .ref => |r| .memRef( r.dtype.toMlir(ctx), r.shape, null, @@ -613,7 +610,7 @@ pub const Builder = struct { .regular => tpu.semaphoreType(ctx), .dma => tpu.dmaSemaphoreType(ctx), }, - .sem_array => |sa| mlir.memRefType( + .sem_array => |sa| .memRef( switch (sa.kind) { .regular => tpu.semaphoreType(ctx), .dma => tpu.dmaSemaphoreType(ctx), @@ -633,11 +630,11 @@ pub const Builder = struct { const arg_attrs = try scratch.alloc(*const mlir.Attribute, args.len); var any_arg_attr = false; for (args, 0..) |a, i| { - const empty_dict: *const mlir.Attribute = mlir.dictionaryAttribute(ctx, &.{}); + const empty_dict: *const mlir.Attribute = .dict(ctx, &.{}); switch (a.kind) { .iter => |io| { if (io.semantics) |sem| { - arg_attrs[i] = mlir.dictionaryAttribute(ctx, &.{ + arg_attrs[i] = .dict(ctx, &.{ .named(ctx, "tpu.dimension_semantics", sem.attribute(ctx)), }); any_arg_attr = true; @@ -653,12 +650,12 @@ pub const Builder = struct { var extra_attrs: stdx.BoundedArray(mlir.NamedAttribute, 8) = .empty; const dim_sem_buf = try scratch.alloc(*const mlir.Attribute, opts.dimension_semantics.len); for (opts.dimension_semantics, 0..) |s, i| dim_sem_buf[i] = s.attribute(ctx); - extra_attrs.appendAssumeCapacity(.named(ctx, "dimension_semantics", mlir.arrayAttribute(ctx, dim_sem_buf))); + extra_attrs.appendAssumeCapacity(.named(ctx, "dimension_semantics", .array(ctx, dim_sem_buf))); if (opts.iteration_bounds) |bounds| { - extra_attrs.appendAssumeCapacity(.named(ctx, "iteration_bounds", mlir.denseArrayAttribute(ctx, .i64, bounds))); + extra_attrs.appendAssumeCapacity(.named(ctx, "iteration_bounds", .denseArray(ctx, .i64, bounds))); } - extra_attrs.appendAssumeCapacity(.named(ctx, "scalar_prefetch", mlir.integerAttribute(ctx, .i64, opts.scalar_prefetch))); - extra_attrs.appendAssumeCapacity(.named(ctx, "scratch_operands", mlir.integerAttribute(ctx, .i64, opts.scratch_operands))); + extra_attrs.appendAssumeCapacity(.named(ctx, "scalar_prefetch", .int(ctx, .i64, opts.scalar_prefetch))); + extra_attrs.appendAssumeCapacity(.named(ctx, "scratch_operands", .int(ctx, .i64, opts.scratch_operands))); if (opts.core_type) |ct| { extra_attrs.appendAssumeCapacity(.named(ctx, "tpu.core_type", ct.attribute(ctx))); } @@ -705,7 +702,7 @@ pub const Builder = struct { .ref => |r| { if (r.role != .input and r.role != .output) continue; if (isSkippedWindowSpace(r.memory_space)) { - wp_buf[idx] = mlir.dictionaryAttribute(ctx, &.{}); + wp_buf[idx] = .dict(ctx, &.{}); idx += 1; continue; } @@ -740,17 +737,17 @@ pub const Builder = struct { if (w.revisit_mode) |rm| { inner.appendAssumeCapacity(.named(ctx, "revisit_mode", rm.attribute(ctx))); } - inner.appendAssumeCapacity(.named(ctx, "transform_indices", mlir.flatSymbolRefAttribute(ctx, sym_name))); - inner.appendAssumeCapacity(.named(ctx, "window_bounds", mlir.denseArrayAttribute(ctx, .i64, block_shape))); + inner.appendAssumeCapacity(.named(ctx, "transform_indices", .flatSymbolRef(ctx, sym_name))); + inner.appendAssumeCapacity(.named(ctx, "window_bounds", .denseArray(ctx, .i64, block_shape))); if (w.element_window) |ew| { inner.appendAssumeCapacity(.named(ctx, "window_kind", tpu.elementWindowAttribute(ctx, ew.pad_low, ew.pad_high))); } - wp_buf[idx] = mlir.dictionaryAttribute(ctx, inner.constSlice()); + wp_buf[idx] = .dict(ctx, inner.constSlice()); idx += 1; }, else => {}, }; - extra_attrs.appendAssumeCapacity(.named(ctx, "window_params", mlir.arrayAttribute(ctx, wp_buf))); + extra_attrs.appendAssumeCapacity(.named(ctx, "window_params", .array(ctx, wp_buf))); } const func_op = func.func(ctx, .{ @@ -846,11 +843,11 @@ pub const Builder = struct { } pub fn vectorTy(self: *const Builder, shape: []const i64, dtype: DType) *const mlir.Type { - return mlir.vectorType(shape, dtype.toMlir(self.ctx)); + return .vector(shape, dtype.toMlir(self.ctx)); } pub fn memRefTy(self: *const Builder, shape: []const i64, dtype: DType, memory_space: MemorySpace) *const mlir.Type { - return mlir.memRefType(dtype.toMlir(self.ctx), shape, null, memory_space.attribute(self.ctx)); + return .memRef(dtype.toMlir(self.ctx), shape, null, memory_space.attribute(self.ctx)); } fn mlirElemToDType(self: *const Builder, elem: *const mlir.Type) DType { @@ -860,10 +857,10 @@ pub const Builder = struct { /// Replace `src`'s element type with `out_dtype`, preserving rank/shape. fn swapElem(self: *const Builder, src: Value, out_dtype: DType) *const mlir.Type { const out_elem = out_dtype.toMlir(self.ctx); - if (src.isVector()) return mlir.vectorType(src.shape().constSlice(), out_elem); + if (src.isVector()) return .vector(src.shape().constSlice(), out_elem); if (src.isMemRef()) { const mr = src.type_().isA(mlir.MemRefType).?; - return mlir.memRefType(out_elem, src.shape().constSlice(), null, mr.memorySpace()); + return .memRef(out_elem, src.shape().constSlice(), null, mr.memorySpace()); } return out_elem; } @@ -955,7 +952,7 @@ pub const Builder = struct { /// Build an arith.constant of the given MLIR element type from a Zig /// numeric. Used for matching the type of an existing reference value. pub fn constMatching(self: *Builder, value: anytype, elem: *const mlir.Type) Value { - if (elem.eql(mlir.indexType(self.ctx))) return self.cIndex(@intCast(value)); + if (elem.eql(.index(self.ctx))) return self.cIndex(@intCast(value)); return self.liftAs(value, self.mlirElemToDType(elem)); } @@ -972,7 +969,7 @@ pub const Builder = struct { /// (`arith.constant dense`); a `Value` falls back to `vector.broadcast`. pub fn splat(self: *Builder, value: anytype, shape: []const i64, dtype: DType) Value { const T = @TypeOf(value); - const ty = mlir.vectorType(shape, dtype.toMlir(self.ctx)); + const ty = mlir.Type.vector(shape, dtype.toMlir(self.ctx)); if (T == Value) return self.emit(vector.broadcast(self.ctx, value.inner, ty, self.loc())); if (isFloatDtype(dtype)) { const v_f64: f64 = switch (@typeInfo(T)) { @@ -1225,7 +1222,7 @@ pub const Builder = struct { /// pattern for using i32 program-ids and scalar-prefetch values as /// memref offsets. pub fn toIndex(self: *Builder, src: Value) Value { - return self.indexCast(src, mlir.indexType(self.ctx)); + return self.indexCast(src, .index(self.ctx)); } // ==================== TPU casts ==================== @@ -1362,7 +1359,7 @@ pub const Builder = struct { const mr = ref.type_().isA(mlir.MemRefType) orelse @panic("Builder.refVectorType: not a memref"); var s: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; for (0..mr.rank()) |i| s.appendAssumeCapacity(mr.dimension(i)); - return mlir.vectorType(s.constSlice(), mr.elementType()); + return .vector(s.constSlice(), mr.elementType()); } /// Full-shape `vector.load`. Use `tpuVectorLoad` for sublane stride / masking. @@ -1468,7 +1465,7 @@ pub const Builder = struct { const dim = mr.dimension(i); s.appendAssumeCapacity(@divTrunc(dim + stride - 1, stride)); } - const result_ty = mlir.vectorType(s.constSlice(), mr.elementType()); + const result_ty = mlir.Type.vector(s.constSlice(), mr.elementType()); return self.emit(tpu.strided_load(self.ctx, ref.inner, self.innerSlice(indices), strides, result_ty, self.loc())); } @@ -1477,7 +1474,7 @@ pub const Builder = struct { /// inner-dim crop). pub fn stridedLoadShape(self: *Builder, ref: Value, indices: []const Value, strides: []const i32, result_shape: []const i64) Value { const mr = ref.type_().isA(mlir.MemRefType) orelse @panic("stridedLoadShape: not a memref"); - const result_ty = mlir.vectorType(result_shape, mr.elementType()); + const result_ty = mlir.Type.vector(result_shape, mr.elementType()); return self.emit(tpu.strided_load(self.ctx, ref.inner, self.innerSlice(indices), strides, result_ty, self.loc())); } @@ -1514,7 +1511,7 @@ pub const Builder = struct { /// `tpu.iota` over the given shape / dimensions. Pass `null` for /// `dimensions` to use the default. pub fn iota(self: *Builder, shape: []const i64, dtype: DType, dimensions: ?[]const i32) Value { - const ty = mlir.vectorType(shape, dtype.toMlir(self.ctx)); + const ty = mlir.Type.vector(shape, dtype.toMlir(self.ctx)); return self.emit(tpu.iota(self.ctx, dimensions, ty, self.loc())); } @@ -1522,7 +1519,7 @@ pub const Builder = struct { /// trailing, so this emits a `<1xN>` `tpu.iota` then `shape_cast` to ``. pub fn arange(self: *Builder, n: i64, dtype: DType) Value { const elem = dtype.toMlir(self.ctx); - const ty_2d = mlir.vectorType(&.{ 1, n }, elem); + const ty_2d = .vector(&.{ 1, n }, elem); const iota_2d = self.emit(tpu.iota(self.ctx, &.{1}, ty_2d, self.loc())); return self.shapeCast(iota_2d, &.{n}); } @@ -1546,13 +1543,13 @@ pub const Builder = struct { /// `vector.broadcast` to `shape`. For Zig scalar literals prefer `splat` (dense-splat fast path). pub fn broadcastTo(self: *Builder, src: Value, shape: []const i64) Value { - const ty = mlir.vectorType(shape, src.elemType()); + const ty = mlir.Type.vector(shape, src.elemType()); return self.emit(vector.broadcast(self.ctx, src.inner, ty, self.loc())); } /// `vector.shape_cast` — reshape preserving total lanes. `reshape` emits `tpu.reshape` instead. pub fn shapeCast(self: *Builder, src: Value, new_shape: []const i64) Value { - const ty = mlir.vectorType(new_shape, src.elemType()); + const ty = mlir.Type.vector(new_shape, src.elemType()); return self.emit(vector.shape_cast(self.ctx, src.inner, ty, self.loc())); } @@ -1565,7 +1562,7 @@ pub const Builder = struct { const ty: *const mlir.Type = if (out.len == 0) src.elemType() else - mlir.vectorType(out.constSlice(), src.elemType()); + .vector(out.constSlice(), src.elemType()); return self.emit(vector.extract(self.ctx, src.inner, position, &.{}, ty, self.loc())); } @@ -1606,21 +1603,21 @@ pub const Builder = struct { const ty: *const mlir.Type = if (out.len == 0) src.elemType() else - mlir.vectorType(out.constSlice(), src.elemType()); + .vector(out.constSlice(), src.elemType()); return self.emit(vector.multi_reduction(self.ctx, kind, src.inner, acc.inner, axes, ty, self.loc())); } /// `vector.load` with an explicit result shape — for `ref[i, j, :, k:k+BLOCK]`-style sliced loads. pub fn vectorLoadShape(self: *Builder, ref: Value, indices: []const Value, shape: []const i64) Value { const idx_inner = self.indicesOrZero(ref, indices); - const ty = mlir.vectorType(shape, ref.elemType()); + const ty = mlir.Type.vector(shape, ref.elemType()); return self.emit(vector.load(self.ctx, ref.inner, idx_inner, ty, .{}, self.loc())); } // ==================== TPU shape ==================== pub fn reshape(self: *Builder, src: Value, new_shape: []const i64) Value { - const ty = mlir.vectorType(new_shape, src.elemType()); + const ty = mlir.Type.vector(new_shape, src.elemType()); return self.emit(tpu.reshape(self.ctx, src.inner, ty, self.loc())); } @@ -1631,7 +1628,7 @@ pub const Builder = struct { const d = in_shape.get(i); out.appendAssumeCapacity(if (i == @as(usize, @intCast(dimension))) d * times else d); } - const ty = mlir.vectorType(out.constSlice(), src.elemType()); + const ty = mlir.Type.vector(out.constSlice(), src.elemType()); return self.emit(tpu.repeat(self.ctx, src.inner, dimension, times, ty, self.loc())); } @@ -1643,7 +1640,7 @@ pub const Builder = struct { const head = sources[0].shape(); var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; for (0..head.len) |i| out.appendAssumeCapacity(if (i == ax) sum else head.get(i)); - const ty = mlir.vectorType(out.constSlice(), sources[0].elemType()); + const ty = mlir.Type.vector(out.constSlice(), sources[0].elemType()); return self.emit(tpu.concatenate(self.ctx, self.innerSlice(sources), dimension, ty, self.loc())); } @@ -1652,7 +1649,7 @@ pub const Builder = struct { std.debug.assert(permutation.len == in_shape.len); var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; for (permutation) |p| out.appendAssumeCapacity(in_shape.get(@intCast(p))); - const ty = mlir.vectorType(out.constSlice(), src.elemType()); + const ty = mlir.Type.vector(out.constSlice(), src.elemType()); return self.emit(tpu.transpose(self.ctx, src.inner, permutation, ty, self.loc())); } @@ -1677,7 +1674,7 @@ pub const Builder = struct { const in_shape = input.shape(); var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; for (0..in_shape.len) |i| out.appendAssumeCapacity(if (i == @as(usize, @intCast(dim))) 1 else in_shape.get(i)); - const ty = mlir.vectorType(out.constSlice(), input.elemType()); + const ty = mlir.Type.vector(out.constSlice(), input.elemType()); return self.emit(tpu.all_reduce(self.ctx, input.inner, dim, kind, ty, self.loc())); } @@ -1689,7 +1686,7 @@ pub const Builder = struct { for (in_shape.constSlice(), 0..) |d, i| { if (@as(i32, @intCast(i)) != axis) out.appendAssumeCapacity(d); } - const ty = mlir.vectorType(out.constSlice(), mlir.integerType(self.ctx, .i32)); + const ty = mlir.Type.vector(out.constSlice(), .integer(self.ctx, .i32)); return self.emit(tpu.reduce_index(self.ctx, input.inner, axis, kind, ty, self.loc())); } @@ -1704,7 +1701,7 @@ pub const Builder = struct { const m_inner: ?*const mlir.Value = if (mask) |m| m.inner else null; const out_mask_ty = if (mask) |m| m.type_() else blk: { // Default: vector. - break :blk mlir.vectorType(keys.shape().constSlice(), mlir.integerType(self.ctx, .i1)); + break :blk .vector(keys.shape().constSlice(), .integer(self.ctx, .i1)); }; const op = tpu.sort( self.ctx, @@ -1735,7 +1732,7 @@ pub const Builder = struct { dynamic_sizes: []const Value, ) Value { const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefSlice: not a memref"); - const result_ty = mlir.memRefType(mr.elementType(), result_shape, null, mr.memorySpace()); + const result_ty = mlir.Type.memRef(mr.elementType(), result_shape, null, mr.memorySpace()); return self.emit(tpu.memref_slice(self.ctx, mem_ref.inner, .{ .base_idx = self.innerSlice(base_idx), .dynamic_sizes = self.innerSlice(dynamic_sizes), @@ -1744,13 +1741,13 @@ pub const Builder = struct { pub fn memRefSqueeze(self: *Builder, mem_ref: Value, result_shape: []const i64) Value { const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefSqueeze: not a memref"); - const result_ty = mlir.memRefType(mr.elementType(), result_shape, null, mr.memorySpace()); + const result_ty = mlir.Type.memRef(mr.elementType(), result_shape, null, mr.memorySpace()); return self.emit(tpu.memref_squeeze(self.ctx, mem_ref.inner, result_ty, self.loc())); } pub fn memRefReshape(self: *Builder, mem_ref: Value, result_shape: []const i64) Value { const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefReshape: not a memref"); - const result_ty = mlir.memRefType(mr.elementType(), result_shape, null, mr.memorySpace()); + const result_ty = mlir.Type.memRef(mr.elementType(), result_shape, null, mr.memorySpace()); return self.emit(tpu.memref_reshape(self.ctx, mem_ref.inner, result_ty, self.loc())); } @@ -1758,7 +1755,7 @@ pub const Builder = struct { const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefBitcast: not a memref"); var s: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; for (0..mr.rank()) |i| s.appendAssumeCapacity(mr.dimension(i)); - const result_ty = mlir.memRefType(dt.toMlir(self.ctx), s.constSlice(), null, mr.memorySpace()); + const result_ty = .memRef(dt.toMlir(self.ctx), s.constSlice(), null, mr.memorySpace()); return self.emit(tpu.memref_bitcast(self.ctx, mem_ref.inner, result_ty, self.loc())); } @@ -1767,13 +1764,13 @@ pub const Builder = struct { /// `strided_load_kv` path (`<64x128xbf16>` → `<32x128xi32>`). pub fn memRefBitcastShape(self: *Builder, mem_ref: Value, result_shape: []const i64, dt: DType) Value { const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefBitcastShape: not a memref"); - const result_ty = mlir.memRefType(dt.toMlir(self.ctx), result_shape, null, mr.memorySpace()); + const result_ty = mlir.Type.memRef(dt.toMlir(self.ctx), result_shape, null, mr.memorySpace()); return self.emit(tpu.memref_bitcast(self.ctx, mem_ref.inner, result_ty, self.loc())); } pub fn reinterpretCast(self: *Builder, mem_ref: Value, result_shape: []const i64, dt: DType) Value { const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("reinterpretCast: not a memref"); - const result_ty = mlir.memRefType(dt.toMlir(self.ctx), result_shape, null, mr.memorySpace()); + const result_ty = mlir.Type.memRef(dt.toMlir(self.ctx), result_shape, null, mr.memorySpace()); return self.emit(tpu.reinterpret_cast(self.ctx, mem_ref.inner, result_ty, self.loc())); } @@ -1786,7 +1783,7 @@ pub const Builder = struct { var s: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; for (0..mr.rank()) |i| s.appendAssumeCapacity(mr.dimension(i)); // Result drops the layout but keeps shape, element type, memory space. - const result_ty = mlir.memRefType(mr.elementType(), s.constSlice(), null, mr.memorySpace()); + const result_ty = .memRef(mr.elementType(), s.constSlice(), null, mr.memorySpace()); return self.emit(tpu.erase_memref_layout(self.ctx, mem_ref.inner, result_ty, self.loc())); } @@ -1794,7 +1791,7 @@ pub const Builder = struct { /// `tpu.create_mask` — 2-sided mask `[low, high)` per dim. pub fn createMask(self: *Builder, low_bounds: []const Value, high_bounds: []const Value, shape: []const i64) Value { - const ty = mlir.vectorType(shape, mlir.integerType(self.ctx, .i1)); + const ty = mlir.Type.vector(shape, .integer(self.ctx, .i1)); return self.emit(tpu.create_mask(self.ctx, self.innerSlice(low_bounds), self.innerSlice(high_bounds), ty, self.loc())); } @@ -2111,7 +2108,7 @@ pub const Builder = struct { fn appendTransformStubs(self: *Builder) void { const ctx = self.ctx; const unknown_loc: *const mlir.Location = .unknown(ctx); - const i32_ty = mlir.integerType(ctx, .i32); + const i32_ty: *const mlir.Type = .int(ctx, .i32); const arena_alloc = self.arena.allocator(); const total_args = self.grid_dim_count + self.prefetch_args.len; @@ -2181,8 +2178,6 @@ fn isSkippedWindowSpace(ms: MemorySpace) bool { // ==================== scope-type aliases pinned to Mosaic's Builder ==================== -const tupleArity = dsl.tupleArity; - pub fn ForScope(comptime N: usize) type { return dsl.ForScope(Builder, Value, N); } diff --git a/kernels/mosaic_tpu/dtype.zig b/kernels/mosaic_tpu/dtype.zig index 4d678c1f..d22370f6 100644 --- a/kernels/mosaic_tpu/dtype.zig +++ b/kernels/mosaic_tpu/dtype.zig @@ -16,17 +16,17 @@ pub const DType = enum { pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { return switch (self) { - .i1 => mlir.integerType(ctx, .i1), - .i8 => mlir.integerType(ctx, .i8), - .i16 => mlir.integerType(ctx, .i16), - .i32 => mlir.integerType(ctx, .i32), - .i64 => mlir.integerType(ctx, .i64), - .f16 => mlir.floatType(ctx, .f16), - .bf16 => mlir.floatType(ctx, .bf16), - .f32 => mlir.floatType(ctx, .f32), - .f64 => mlir.floatType(ctx, .f64), - .f8e4m3fn => mlir.floatType(ctx, .f8e4m3fn), - .f8e5m2 => mlir.floatType(ctx, .f8e5m2), + .i1 => .int(ctx, .i1), + .i8 => .int(ctx, .i8), + .i16 => .int(ctx, .i16), + .i32 => .int(ctx, .i32), + .i64 => .int(ctx, .i64), + .f16 => .float(ctx, .f16), + .bf16 => .float(ctx, .bf16), + .f32 => .float(ctx, .f32), + .f64 => .float(ctx, .f64), + .f8e4m3fn => .float(ctx, .f8e4m3fn), + .f8e5m2 => .float(ctx, .f8e5m2), }; } }; diff --git a/kernels/triton/builder.zig b/kernels/triton/builder.zig index 711c692a..bcae6421 100644 --- a/kernels/triton/builder.zig +++ b/kernels/triton/builder.zig @@ -1,18 +1,27 @@ const std = @import("std"); -const mlir = @import("mlir"); const dialects = @import("mlir/dialects"); -const dsl = @import("kernels/common"); -const dtypes = @import("dtype.zig"); -const stdx = @import("stdx"); - const arith = dialects.arith; const cf = dialects.cf; const math = dialects.math; const scf = dialects.scf; const ttir = dialects.ttir; +const dsl = @import("kernels/common"); +const tupleArity = dsl.tupleArity; +const mlir = @import("mlir"); +const stdx = @import("stdx"); +const dtypes = @import("dtype.zig"); pub const DType = dtypes.DType; +const isFloatDtype = dtypes.isFloatDtype; +const dtypeBitwidth = dtypes.dtypeBitwidth; +const intBitwidth = dtypes.intBitwidth; + +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(Builder); + std.testing.refAllDecls(Value); +} pub const Value = struct { inner: *const mlir.Value, @@ -231,17 +240,13 @@ pub const Value = struct { } }; -const isFloatDtype = dtypes.isFloatDtype; -const dtypeBitwidth = dtypes.dtypeBitwidth; -const intBitwidth = dtypes.intBitwidth; - fn computeReducedType(src_shape: []const i64, axis: i32, elem: *const mlir.Type) *const mlir.Type { if (src_shape.len <= 1) return elem; var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; for (src_shape, 0..) |d, i| { if (@as(i32, @intCast(i)) != axis) out.appendAssumeCapacity(d); } - return mlir.rankedTensorType(out.constSlice(), elem); + return .rankedTensor(out.constSlice(), elem); } pub const ArgSpec = struct { @@ -374,8 +379,6 @@ pub const ReturnIfOpts = struct { inline_return: bool = false, }; -const tupleArity = dsl.tupleArity; - pub fn ForScope(comptime N: usize) type { return dsl.ForScope(Builder, Value, N); } @@ -579,55 +582,38 @@ pub const Builder = struct { const arg_types = try scratch.alloc(*const mlir.Type, args.len); const arg_locs = try scratch.alloc(*const mlir.Location, args.len); - for (args, 0..) |a, i| { - arg_types[i] = switch (a.kind) { + for (arg_types, arg_locs, args) |*ty, *a_loc, a| { + ty.* = switch (a.kind) { .ptr => |dt| ttir.pointerType(dt.toMlir(ctx), 1), .ptr_opts => |p| ttir.pointerType(p.dtype.toMlir(ctx), p.address_space), .scalar => |dt| dt.toMlir(ctx), .scalar_opts => |s| s.dtype.toMlir(ctx), - .tensor => |t| mlir.rankedTensorType(t[0], t[1].toMlir(ctx)), + .tensor => |t| .rankedTensor(t[0], t[1].toMlir(ctx)), }; - arg_locs[i] = unknown_loc.named(ctx, a.name); + a_loc.* = unknown_loc.named(ctx, a.name); } const entry = mlir.Block.init(arg_types, arg_locs); const arg_attrs = try scratch.alloc(*const mlir.Attribute, args.len); + const empty_dict: *const mlir.Attribute = .dict(ctx, &.{}); var any_arg_attr = false; - for (args, 0..) |a, i| { - const empty_dict: *const mlir.Attribute = mlir.dictionaryAttribute(ctx, &.{}); - switch (a.kind) { - .ptr => { - const div_attr = mlir.dictionaryAttribute(ctx, &.{ - .named(ctx, "tt.divisibility", mlir.integerAttribute(ctx, .i32, 16)), - }); - arg_attrs[i] = div_attr; - any_arg_attr = true; - }, - .ptr_opts => |p| { - if (p.divisibility) |v| { - const div_attr = mlir.dictionaryAttribute(ctx, &.{ - .named(ctx, "tt.divisibility", mlir.integerAttribute(ctx, .i32, v)), - }); - arg_attrs[i] = div_attr; - any_arg_attr = true; - } else { - arg_attrs[i] = empty_dict; - } - }, - .scalar_opts => |s| { - if (s.divisibility) |v| { - const div_attr = mlir.dictionaryAttribute(ctx, &.{ - .named(ctx, "tt.divisibility", mlir.integerAttribute(ctx, .i32, v)), - }); - arg_attrs[i] = div_attr; - any_arg_attr = true; - } else { - arg_attrs[i] = empty_dict; - } - }, - else => arg_attrs[i] = empty_dict, - } + for (arg_attrs, args) |*attr, a| { + attr.* = switch (a.kind) { + .ptr => .dict(ctx, &.{ + .named(ctx, "tt.divisibility", .int(ctx, .i32, 16)), + }), + .ptr_opts => |p| if (p.divisibility) |v| + .dict(ctx, &.{.named(ctx, "tt.divisibility", .int(ctx, .i32, v))}) + else + empty_dict, + .scalar_opts => |s| if (s.divisibility) |v| + .dict(ctx, &.{.named(ctx, "tt.divisibility", .int(ctx, .i32, v))}) + else + empty_dict, + else => empty_dict, + }; + if (attr.* != empty_dict) any_arg_attr = true; } const func_op = ttir.func(ctx, .{ @@ -712,7 +698,7 @@ pub const Builder = struct { } pub fn tensorTy(self: *const Builder, shape: []const i64, dtype: DType) *const mlir.Type { - return mlir.rankedTensorType(shape, dtype.toMlir(self.ctx)); + return .rankedTensor(shape, dtype.toMlir(self.ctx)); } fn loadResultType(self: *const Builder, ptr: Value) *const mlir.Type { @@ -723,14 +709,14 @@ pub const Builder = struct { const pointee = elem_ptr_ty.isA(ttir.PointerType).?.pointee(); var shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; for (0..shaped.rank()) |i| shape.appendAssumeCapacity(shaped.dimension(i)); - return mlir.rankedTensorType(shape.constSlice(), pointee); + return .rankedTensor(shape.constSlice(), pointee); } return pt.isA(ttir.PointerType).?.pointee(); } fn swapElem(self: *const Builder, src: Value, out_dtype: DType) *const mlir.Type { const out_elem = out_dtype.toMlir(self.ctx); - if (src.isTensor()) return mlir.rankedTensorType(src.shape().constSlice(), out_elem); + if (src.isTensor()) return .rankedTensor(src.shape().constSlice(), out_elem); return out_elem; } @@ -748,8 +734,8 @@ pub const Builder = struct { const op = mlir.Operation.make(self.ctx, "llvm.intr.assume", .{ .operands = .{ .flat = &.{cond.inner} }, .attributes = &.{ - .named(self.ctx, "op_bundle_sizes", mlir.denseArrayAttribute(self.ctx, .i32, &.{})), - .named(self.ctx, "op_bundle_tags", mlir.arrayAttribute(self.ctx, &.{})), + .named(self.ctx, "op_bundle_sizes", .denseArray(self.ctx, .i32, &.{})), + .named(self.ctx, "op_bundle_tags", .array(self.ctx, &.{})), }, .location = self.loc(), }); @@ -778,7 +764,7 @@ pub const Builder = struct { const e: i32 = @intCast(end); std.debug.assert(e >= s); const len: i64 = @intCast(e - s); - const ty = mlir.rankedTensorType(&.{len}, mlir.integerType(self.ctx, .i32)); + const ty = mlir.Type.rankedTensor(&.{len}, .int(self.ctx, .i32)); return self.emit(ttir.make_range(self.ctx, s, e, ty, self.loc())); } @@ -796,7 +782,7 @@ pub const Builder = struct { /// Broadcast a scalar Value or literal across `shape`. Literals lift first via `lift`. pub fn splat(self: *Builder, value: anytype, shape: []const i64) Value { const v: Value = if (@TypeOf(value) == Value) value else self.lift(value); - const ty = mlir.rankedTensorType(shape, v.type_()); + const ty: *const mlir.Type = .rankedTensor(shape, v.type_()); return self.emit(ttir.splat(self.ctx, v.inner, ty, self.loc())); } @@ -1078,13 +1064,13 @@ pub const Builder = struct { for (0..in_shape.len + 1) |i| { if (i == ax) out_shape.appendAssumeCapacity(1) else out_shape.appendAssumeCapacity(in_shape.get(if (i < ax) i else i - 1)); } - const ty = mlir.rankedTensorType(out_shape.constSlice(), value.elemType()); + const ty = mlir.Type.rankedTensor(out_shape.constSlice(), value.elemType()); return self.emit(ttir.expand_dims(self.ctx, value.inner, axis, ty, self.loc())); } pub fn broadcastTo(self: *Builder, value: Value, result_shape: []const i64) Value { const elem = mlir.RankedTensorType.fromShaped(value.inner.type_().isA(mlir.ShapedType).?).elementType(); - const ty = mlir.rankedTensorType(result_shape, elem); + const ty = mlir.Type.rankedTensor(result_shape, elem); return self.emit(ttir.broadcast(self.ctx, value.inner, ty, self.loc())); } @@ -1357,7 +1343,7 @@ pub const Builder = struct { pub fn intToPtr(self: *Builder, src: Value, pointee: DType, address_space: i32) Value { const ptr_elem = ttir.pointerType(pointee.toMlir(self.ctx), address_space); const result_type: *const mlir.Type = if (src.isTensor()) - mlir.rankedTensorType(src.shape().constSlice(), ptr_elem) + .rankedTensor(src.shape().constSlice(), ptr_elem) else ptr_elem; return self.emit(ttir.int_to_ptr(self.ctx, src.inner, result_type, self.loc())); @@ -1412,7 +1398,7 @@ pub const Builder = struct { /// tt.reshape with full options. pub fn reshapeOpts(self: *Builder, src: Value, new_shape: []const i64, opts: ReshapeOpts) Value { - const result_ty = mlir.rankedTensorType(new_shape, src.elemType()); + const result_ty = mlir.Type.rankedTensor(new_shape, src.elemType()); return self.emit(ttir.reshape(self.ctx, src.inner, result_ty, opts.can_reorder, opts.efficient_layout, self.loc())); } @@ -1422,7 +1408,7 @@ pub const Builder = struct { std.debug.assert(order.len == src_shape.len); var out_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; for (order) |i| out_shape.appendAssumeCapacity(src_shape.get(@intCast(i))); - const result_type = mlir.rankedTensorType(out_shape.constSlice(), src.elemType()); + const result_type = mlir.Type.rankedTensor(out_shape.constSlice(), src.elemType()); return self.emit(ttir.trans(self.ctx, src.inner, order, result_type, self.loc())); } @@ -1446,7 +1432,7 @@ pub const Builder = struct { var out_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; out_shape.appendAssumeCapacity(ls.get(0) + rs.get(0)); for (1..ls.len) |i| out_shape.appendAssumeCapacity(ls.get(i)); - const result_type = mlir.rankedTensorType(out_shape.constSlice(), lhs.elemType()); + const result_type = mlir.Type.rankedTensor(out_shape.constSlice(), lhs.elemType()); return self.emit(ttir.cat(self.ctx, lhs.inner, rhs.inner, result_type, self.loc())); } @@ -1456,7 +1442,7 @@ pub const Builder = struct { var out_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; for (0..ls.len) |i| out_shape.appendAssumeCapacity(ls.get(i)); out_shape.appendAssumeCapacity(2); - const result_type = mlir.rankedTensorType(out_shape.constSlice(), lhs.elemType()); + const result_type = mlir.Type.rankedTensor(out_shape.constSlice(), lhs.elemType()); return self.emit(ttir.join(self.ctx, lhs.inner, rhs.inner, result_type, self.loc())); } @@ -1467,7 +1453,7 @@ pub const Builder = struct { std.debug.assert(ss.len >= 1 and ss.get(ss.len - 1) == 2); var out_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; for (0..ss.len - 1) |i| out_shape.appendAssumeCapacity(ss.get(i)); - const result_type = mlir.rankedTensorType(out_shape.constSlice(), src.elemType()); + const result_type = mlir.Type.rankedTensor(out_shape.constSlice(), src.elemType()); const op = ttir.split(self.ctx, src.inner, result_type, self.loc()); _ = op.appendTo(self.currentBlock()); return .{ @@ -1483,7 +1469,7 @@ pub const Builder = struct { /// tt.gather — output shape matches indices, element type matches src. pub fn gather(self: *Builder, src: Value, indices: Value, axis: i32) Value { - const result_ty = mlir.rankedTensorType(indices.shape().constSlice(), src.elemType()); + const result_ty = mlir.Type.rankedTensor(indices.shape().constSlice(), src.elemType()); return self.emit(ttir.gather(self.ctx, src.inner, indices.inner, axis, result_ty, false, self.loc())); } @@ -1494,7 +1480,7 @@ pub const Builder = struct { /// tt.histogram with mask option. pub fn histogramOpts(self: *Builder, src: Value, num_bins: i64, opts: HistogramOpts) Value { - const result_ty = mlir.rankedTensorType(&.{num_bins}, DType.i32.toMlir(self.ctx)); + const result_ty = mlir.Type.rankedTensor(&.{num_bins}, DType.i32.toMlir(self.ctx)); const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; return self.emit(ttir.histogram(self.ctx, src.inner, mask_inner, result_ty, self.loc())); } @@ -1611,7 +1597,7 @@ pub const Builder = struct { const result_type: *const mlir.Type = if (result_shape.len == 0) result_dtype.toMlir(self.ctx) else - mlir.rankedTensorType(result_shape, result_dtype.toMlir(self.ctx)); + .rankedTensor(result_shape, result_dtype.toMlir(self.ctx)); return self.emit(ttir.extern_elementwise( self.ctx, self.innerSlice(srcs), @@ -1689,7 +1675,7 @@ pub const Builder = struct { dtype: DType, opts: DescriptorLoadOpts, ) Value { - const result_ty = mlir.rankedTensorType(shape, dtype.toMlir(self.ctx)); + const result_ty = mlir.Type.rankedTensor(shape, dtype.toMlir(self.ctx)); return self.emit(ttir.descriptor_load(self.ctx, desc.inner, self.innerSlice(indices), result_ty, opts.cache_modifier, opts.eviction_policy, self.loc())); } @@ -1718,7 +1704,7 @@ pub const Builder = struct { shape: []const i64, dtype: DType, ) Value { - const result_ty = mlir.rankedTensorType(shape, dtype.toMlir(self.ctx)); + const result_ty = mlir.Type.rankedTensor(shape, dtype.toMlir(self.ctx)); return self.emit(ttir.descriptor_gather(self.ctx, desc.inner, x_offsets.inner, y_offset.inner, result_ty, self.loc())); } @@ -1921,7 +1907,7 @@ pub const Builder = struct { std.debug.assert(inits.len == reduction_result_types.len); const scratch = self.arena.allocator(); - const idx_ty = mlir.indexType(self.ctx); + const idx_ty = .index(self.ctx); const iv_types = scratch.alloc(*const mlir.Type, lbs.len) catch @panic("Builder.parallel OOM"); const iv_locs = scratch.alloc(*const mlir.Location, lbs.len) catch @panic("Builder.parallel OOM"); for (iv_types, iv_locs) |*t, *l| { diff --git a/kernels/triton/dtype.zig b/kernels/triton/dtype.zig index 56d6ed64..3ac696b8 100644 --- a/kernels/triton/dtype.zig +++ b/kernels/triton/dtype.zig @@ -16,17 +16,17 @@ pub const DType = enum { pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { return switch (self) { - .i1 => mlir.integerType(ctx, .i1), - .i8 => mlir.integerType(ctx, .i8), - .i16 => mlir.integerType(ctx, .i16), - .i32 => mlir.integerType(ctx, .i32), - .i64 => mlir.integerType(ctx, .i64), - .f16 => mlir.floatType(ctx, .f16), - .bf16 => mlir.floatType(ctx, .bf16), - .f32 => mlir.floatType(ctx, .f32), - .f64 => mlir.floatType(ctx, .f64), - .f8e4m3fn => mlir.floatType(ctx, .f8e4m3fn), - .f8e5m2 => mlir.floatType(ctx, .f8e5m2), + .i1 => .int(ctx, .i1), + .i8 => .int(ctx, .i8), + .i16 => .int(ctx, .i16), + .i32 => .int(ctx, .i32), + .i64 => .int(ctx, .i64), + .f16 => .float(ctx, .f16), + .bf16 => .float(ctx, .bf16), + .f32 => .float(ctx, .f32), + .f64 => .float(ctx, .f64), + .f8e4m3fn => .float(ctx, .f8e4m3fn), + .f8e5m2 => .float(ctx, .f8e5m2), }; } }; diff --git a/mlir/dialects/affine.zig b/mlir/dialects/affine.zig index 685248f7..be0c560c 100644 --- a/mlir/dialects/affine.zig +++ b/mlir/dialects/affine.zig @@ -18,9 +18,9 @@ pub fn apply( ) *mlir.Operation { return mlir.Operation.make(ctx, "affine.apply", .{ .operands = .{ .flat = map_operands }, - .results = .{ .flat = &.{mlir.indexType(ctx)} }, + .results = .{ .flat = &.{.index(ctx)} }, .attributes = &.{ - .named(ctx, "map", mlir.affineMapAttribute(map)), + .named(ctx, "map", .affineMap(map)), }, .location = location, }); @@ -35,9 +35,9 @@ pub fn min( ) *mlir.Operation { return mlir.Operation.make(ctx, "affine.min", .{ .operands = .{ .flat = map_operands }, - .results = .{ .flat = &.{mlir.indexType(ctx)} }, + .results = .{ .flat = &.{.index(ctx)} }, .attributes = &.{ - .named(ctx, "map", mlir.affineMapAttribute(map)), + .named(ctx, "map", .affineMap(map)), }, .location = location, }); @@ -52,9 +52,9 @@ pub fn max( ) *mlir.Operation { return mlir.Operation.make(ctx, "affine.max", .{ .operands = .{ .flat = map_operands }, - .results = .{ .flat = &.{mlir.indexType(ctx)} }, + .results = .{ .flat = &.{.index(ctx)} }, .attributes = &.{ - .named(ctx, "map", mlir.affineMapAttribute(map)), + .named(ctx, "map", .affineMap(map)), }, .location = location, }); @@ -120,10 +120,10 @@ pub fn for_( .results = .{ .flat = result_types.constSlice() }, .blocks = &.{body}, .attributes = &.{ - .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), - .named(ctx, "lowerBoundMap", mlir.affineMapAttribute(lb_map)), - .named(ctx, "upperBoundMap", mlir.affineMapAttribute(ub_map)), - .named(ctx, "step", mlir.integerAttribute(ctx, .i64, args.step)), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "lowerBoundMap", .affineMap(lb_map)), + .named(ctx, "upperBoundMap", .affineMap(ub_map)), + .named(ctx, "step", .int(ctx, .i64, args.step)), }, // The for body may reference outer SSA values that only become // reachable once it's appended to its parent — defer to module-level @@ -162,7 +162,7 @@ pub fn if_( state.addOperands(operands); state.addResults(result_types); state.addAttributes(&.{ - .named(ctx, "condition", mlir.integerSetAttribute(condition)), + .named(ctx, "condition", .integerSet(condition)), }); const then_region = mlir.Region.init(); @@ -195,7 +195,7 @@ pub fn load( .operands = .{ .flat = operands_buf.constSlice() }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "map", mlir.affineMapAttribute(map)), + .named(ctx, "map", .affineMap(map)), }, .location = location, }); @@ -216,7 +216,7 @@ pub fn store( return mlir.Operation.make(ctx, "affine.store", .{ .operands = .{ .flat = operands_buf.constSlice() }, .attributes = &.{ - .named(ctx, "map", mlir.affineMapAttribute(map)), + .named(ctx, "map", .affineMap(map)), }, .location = location, }); @@ -238,7 +238,7 @@ pub fn vector_load( .operands = .{ .flat = operands_buf.constSlice() }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "map", mlir.affineMapAttribute(map)), + .named(ctx, "map", .affineMap(map)), }, .location = location, }); @@ -259,7 +259,7 @@ pub fn vector_store( return mlir.Operation.make(ctx, "affine.vector_store", .{ .operands = .{ .flat = operands_buf.constSlice() }, .attributes = &.{ - .named(ctx, "map", mlir.affineMapAttribute(map)), + .named(ctx, "map", .affineMap(map)), }, .location = location, }); @@ -301,26 +301,26 @@ pub fn parallel( // I32ElementsAttr — dense<[...]> : tensor. const lb_groups_ty = mlir.RankedTensorType.get( &.{@intCast(args.lower_bounds_groups.len)}, - mlir.integerType(ctx, .i32), + .int(ctx, .i32), null, ).shaped(); const ub_groups_ty = mlir.RankedTensorType.get( &.{@intCast(args.upper_bounds_groups.len)}, - mlir.integerType(ctx, .i32), + .int(ctx, .i32), null, ).shaped(); // I64SmallVectorArrayAttr — built as ArrayAttr>. var steps_attrs: stdx.BoundedArray(*const mlir.Attribute, 16) = .empty; for (args.steps) |s| { - steps_attrs.appendAssumeCapacity(mlir.integerAttribute(ctx, .i64, s)); + steps_attrs.appendAssumeCapacity(.int(ctx, .i64, s)); } // Reductions are AtomicRMWKindAttr (I64EnumAttr). We let callers pass // raw int values to avoid coupling against the enum table. var reductions_attrs: stdx.BoundedArray(*const mlir.Attribute, 16) = .empty; for (args.reductions) |k| { - reductions_attrs.appendAssumeCapacity(mlir.integerAttribute(ctx, .i64, k)); + reductions_attrs.appendAssumeCapacity(.int(ctx, .i64, k)); } return mlir.Operation.make(ctx, "affine.parallel", .{ @@ -328,12 +328,12 @@ pub fn parallel( .results = .{ .flat = result_types }, .blocks = &.{body_block}, .attributes = &.{ - .named(ctx, "reductions", mlir.arrayAttribute(ctx, reductions_attrs.constSlice())), - .named(ctx, "lowerBoundsMap", mlir.affineMapAttribute(args.lower_bounds_map)), - .named(ctx, "lowerBoundsGroups", mlir.denseElementsAttribute(lb_groups_ty, args.lower_bounds_groups)), - .named(ctx, "upperBoundsMap", mlir.affineMapAttribute(args.upper_bounds_map)), - .named(ctx, "upperBoundsGroups", mlir.denseElementsAttribute(ub_groups_ty, args.upper_bounds_groups)), - .named(ctx, "steps", mlir.arrayAttribute(ctx, steps_attrs.constSlice())), + .named(ctx, "reductions", .array(ctx, reductions_attrs.constSlice())), + .named(ctx, "lowerBoundsMap", .affineMap(args.lower_bounds_map)), + .named(ctx, "lowerBoundsGroups", .denseElements(lb_groups_ty, args.lower_bounds_groups)), + .named(ctx, "upperBoundsMap", .affineMap(args.upper_bounds_map)), + .named(ctx, "upperBoundsGroups", .denseElements(ub_groups_ty, args.upper_bounds_groups)), + .named(ctx, "steps", .array(ctx, steps_attrs.constSlice())), }, .verify = false, .location = location, @@ -364,7 +364,7 @@ pub fn delinearize_index( .operands = .{ .flat = operands_buf.constSlice() }, .results = .{ .flat = result_types }, .attributes = &.{ - .named(ctx, "static_basis", mlir.denseArrayAttribute(ctx, .i64, args.static_basis)), + .named(ctx, "static_basis", .denseArray(ctx, .i64, args.static_basis)), }, .location = location, }); @@ -394,11 +394,11 @@ pub fn linearize_index( var attrs: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; attrs.appendSliceAssumeCapacity(&.{ - .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), - .named(ctx, "static_basis", mlir.denseArrayAttribute(ctx, .i64, args.static_basis)), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "static_basis", .denseArray(ctx, .i64, args.static_basis)), }); if (args.disjoint) { - attrs.appendAssumeCapacity(.named(ctx, "disjoint", mlir.unitAttribute(ctx))); + attrs.appendAssumeCapacity(.named(ctx, "disjoint", .unit(ctx))); } return mlir.Operation.make(ctx, "affine.linearize_index", .{ diff --git a/mlir/dialects/arith.zig b/mlir/dialects/arith.zig index e1e96026..14e2bf4b 100644 --- a/mlir/dialects/arith.zig +++ b/mlir/dialects/arith.zig @@ -53,10 +53,9 @@ pub fn constant_int( int_type: *const mlir.Type, location: *const mlir.Location, ) *mlir.Operation { - const attr: *const mlir.Attribute = @ptrCast(c.mlirIntegerAttrGet(int_type.ptr(), value).ptr); return mlir.Operation.make(ctx, "arith.constant", .{ .results = .{ .flat = &.{int_type} }, - .attributes = &.{.named(ctx, "value", attr)}, + .attributes = &.{.named(ctx, "value", .intFromType(@ptrCast(int_type), value))}, .location = location, }); } @@ -69,15 +68,15 @@ pub fn constant_float( location: *const mlir.Location, ) *mlir.Operation { return mlir.Operation.make(ctx, "arith.constant", .{ - .results = .{ .flat = &.{mlir.floatType(ctx, ft)} }, - .attributes = &.{.named(ctx, "value", mlir.floatAttribute(ctx, ft, value))}, + .results = .{ .flat = &.{.float(ctx, ft)} }, + .attributes = &.{.named(ctx, "value", .float(ctx, ft, value))}, .location = location, }); } /// arith.constant for an `index`-typed integer. pub fn constant_index(ctx: *mlir.Context, value: i64, location: *const mlir.Location) *mlir.Operation { - const idx_ty = mlir.indexType(ctx); + const idx_ty = mlir.Type.index(ctx); const attr: *const mlir.Attribute = @ptrCast(c.mlirIntegerAttrGet(idx_ty.ptr(), value).ptr); return mlir.Operation.make(ctx, "arith.constant", .{ .results = .{ .flat = &.{idx_ty} }, @@ -315,7 +314,7 @@ pub fn convertf( ) *mlir.Operation { var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; if (opts.rounding) |rm| { - attrs.appendAssumeCapacity(.named(ctx, "roundingmode", mlir.integerAttribute(ctx, .i32, @intFromEnum(rm)))); + attrs.appendAssumeCapacity(.named(ctx, "roundingmode", .int(ctx, .i32, @intFromEnum(rm)))); } return mlir.Operation.make(ctx, "arith.convertf", .{ .operands = .{ .flat = &.{src} }, @@ -336,7 +335,7 @@ pub fn scaling_truncf( ) *mlir.Operation { var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; if (opts.rounding) |rm| { - attrs.appendAssumeCapacity(.named(ctx, "roundingmode", mlir.integerAttribute(ctx, .i32, @intFromEnum(rm)))); + attrs.appendAssumeCapacity(.named(ctx, "roundingmode", .int(ctx, .i32, @intFromEnum(rm)))); } return mlir.Operation.make(ctx, "arith.scaling_truncf", .{ .operands = .{ .flat = &.{ src, scale } }, @@ -361,7 +360,7 @@ pub fn cmpi( .operands = .{ .flat = &.{ lhs, rhs } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "predicate", mlir.integerAttribute(ctx, .i64, @intFromEnum(predicate))), + .named(ctx, "predicate", .int(ctx, .i64, @intFromEnum(predicate))), }, .location = location, }); @@ -378,7 +377,7 @@ pub fn cmpf( .operands = .{ .flat = &.{ lhs, rhs } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "predicate", mlir.integerAttribute(ctx, .i64, @intFromEnum(predicate))), + .named(ctx, "predicate", .int(ctx, .i64, @intFromEnum(predicate))), }, .location = location, }); diff --git a/mlir/dialects/cf.zig b/mlir/dialects/cf.zig index 4e75dcce..37df9737 100644 --- a/mlir/dialects/cf.zig +++ b/mlir/dialects/cf.zig @@ -18,7 +18,7 @@ pub fn assert( return mlir.Operation.make(ctx, "cf.assert", .{ .operands = .{ .flat = &.{cond} }, .attributes = &.{ - .named(ctx, "msg", mlir.stringAttribute(ctx, msg)), + .named(ctx, "msg", .string(ctx, msg)), }, .location = location, }); @@ -80,10 +80,10 @@ pub fn cond_br( state.addSuccessors(&successors); var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attrs.appendAssumeCapacity(.named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes))); + attrs.appendAssumeCapacity(.named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes))); if (branch_weights) |w| { std.debug.assert(w.len == 2); - attrs.appendAssumeCapacity(.named(ctx, "branch_weights", mlir.denseArrayAttribute(ctx, .i32, w))); + attrs.appendAssumeCapacity(.named(ctx, "branch_weights", .denseArray(ctx, .i32, w))); } state.addAttributes(attrs.constSlice()); @@ -148,14 +148,14 @@ pub fn switch_( // Python Triton's flag type for pid compares). const shaped = mlir.RankedTensorType.get( &.{@intCast(case_values.len)}, - mlir.integerType(ctx, .i32), + .int(ctx, .i32), null, ).shaped(); state.addAttributes(&.{ - .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &top_seg_sizes)), - .named(ctx, "case_values", mlir.denseElementsAttribute(shaped, case_values)), - .named(ctx, "case_operand_segments", mlir.denseArrayAttribute(ctx, .i32, case_seg_sizes.constSlice())), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &top_seg_sizes)), + .named(ctx, "case_values", .denseElements(shaped, case_values)), + .named(ctx, "case_operand_segments", .denseArray(ctx, .i32, case_seg_sizes.constSlice())), }); return mlir.Operation.init(&state) catch @panic("Failed to create cf.switch"); diff --git a/mlir/dialects/func.zig b/mlir/dialects/func.zig index 00b060d2..aee70d49 100644 --- a/mlir/dialects/func.zig +++ b/mlir/dialects/func.zig @@ -27,12 +27,12 @@ pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { var results_buffer: stdx.BoundedArray(*const mlir.Type, 32) = .empty; var attr_tuples_buffer: stdx.BoundedArray(mlir.NamedAttribute, 16) = .empty; - attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "sym_name", mlir.stringAttribute(ctx, args.name))); + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "sym_name", .string(ctx, args.name))); if (args.visibility) |v| { - attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "sym_visibility", mlir.stringAttribute(ctx, @tagName(v)))); + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "sym_visibility", .string(ctx, @tagName(v)))); } attr_tuples_buffer.appendSliceAssumeCapacity(&.{ - .named(ctx, "function_type", mlir.typeAttribute(mlir.functionType( + .named(ctx, "function_type", .typeAttr(.function( ctx, args.args orelse args: { for (0..args.block.numArguments()) |i| { @@ -53,13 +53,13 @@ pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { }); if (args.args_attributes) |args_attributes| { - attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "arg_attrs", mlir.arrayAttribute(ctx, args_attributes))); + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "arg_attrs", .array(ctx, args_attributes))); } if (args.results_attributes) |results_attributes| { - attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "res_attrs", mlir.arrayAttribute(ctx, results_attributes))); + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "res_attrs", .array(ctx, results_attributes))); } if (args.no_inline) { - attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "no_inline", mlir.unitAttribute(ctx))); + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "no_inline", .unit(ctx))); } for (args.extra_attributes) |a| { attr_tuples_buffer.appendAssumeCapacity(a); @@ -78,7 +78,7 @@ pub fn call(ctx: *mlir.Context, name: []const u8, values: []const *const mlir.Va .operands = .{ .variadic = &.{values} }, .results = .{ .flat = results }, .attributes = &.{ - .named(ctx, "callee", mlir.flatSymbolRefAttribute(ctx, name)), + .named(ctx, "callee", .flatSymbolRef(ctx, name)), }, .location = location, }); diff --git a/mlir/dialects/memref.zig b/mlir/dialects/memref.zig index 42ff2237..77acbc5d 100644 --- a/mlir/dialects/memref.zig +++ b/mlir/dialects/memref.zig @@ -33,9 +33,9 @@ pub fn alloc( const seg_sizes = [2]i32{ @intCast(dynamic_sizes.len), @intCast(symbol_operands.len) }; var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attrs.appendAssumeCapacity(.named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes))); + attrs.appendAssumeCapacity(.named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes))); if (opts.alignment) |a| { - attrs.appendAssumeCapacity(.named(ctx, "alignment", mlir.integerAttribute(ctx, .i64, a))); + attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, a))); } return mlir.Operation.make(ctx, "memref.alloc", .{ @@ -61,9 +61,9 @@ pub fn alloca( const seg_sizes = [2]i32{ @intCast(dynamic_sizes.len), @intCast(symbol_operands.len) }; var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attrs.appendAssumeCapacity(.named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes))); + attrs.appendAssumeCapacity(.named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes))); if (opts.alignment) |a| { - attrs.appendAssumeCapacity(.named(ctx, "alignment", mlir.integerAttribute(ctx, .i64, a))); + attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, a))); } return mlir.Operation.make(ctx, "memref.alloca", .{ @@ -101,7 +101,7 @@ pub fn realloc( var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; if (opts.alignment) |a| { - attrs.appendAssumeCapacity(.named(ctx, "alignment", mlir.integerAttribute(ctx, .i64, a))); + attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, a))); } return mlir.Operation.make(ctx, "memref.realloc", .{ @@ -135,7 +135,7 @@ pub fn load( var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; if (opts.nontemporal) { - attrs.appendAssumeCapacity(.named(ctx, "nontemporal", mlir.boolAttribute(ctx, true))); + attrs.appendAssumeCapacity(.named(ctx, "nontemporal", .boolean(ctx, true))); } return mlir.Operation.make(ctx, "memref.load", .{ @@ -165,7 +165,7 @@ pub fn store( var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; if (opts.nontemporal) { - attrs.appendAssumeCapacity(.named(ctx, "nontemporal", mlir.boolAttribute(ctx, true))); + attrs.appendAssumeCapacity(.named(ctx, "nontemporal", .boolean(ctx, true))); } return mlir.Operation.make(ctx, "memref.store", .{ @@ -216,7 +216,7 @@ pub fn atomic_rmw( .operands = .{ .flat = buf.constSlice() }, .results = .{ .flat = &.{value.type_()} }, .attributes = &.{ - .named(ctx, "kind", mlir.integerAttribute(ctx, .i64, @intFromEnum(kind))), + .named(ctx, "kind", .int(ctx, .i64, @intFromEnum(kind))), }, .location = location, }); @@ -267,7 +267,7 @@ pub fn assume_alignment( .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{src.type_()} }, .attributes = &.{ - .named(ctx, "alignment", mlir.integerAttribute(ctx, .i32, alignment)), + .named(ctx, "alignment", .int(ctx, .i32, alignment)), }, .location = location, }); @@ -286,7 +286,7 @@ pub fn dim( ) *mlir.Operation { return mlir.Operation.make(ctx, "memref.dim", .{ .operands = .{ .flat = &.{ memref, index } }, - .results = .{ .flat = &.{mlir.indexType(ctx)} }, + .results = .{ .flat = &.{.index(ctx)} }, .location = location, }); } @@ -299,7 +299,7 @@ pub fn rank( ) *mlir.Operation { return mlir.Operation.make(ctx, "memref.rank", .{ .operands = .{ .flat = &.{memref} }, - .results = .{ .flat = &.{mlir.indexType(ctx)} }, + .results = .{ .flat = &.{.index(ctx)} }, .location = location, }); } @@ -313,7 +313,7 @@ pub fn extract_aligned_pointer_as_index( ) *mlir.Operation { return mlir.Operation.make(ctx, "memref.extract_aligned_pointer_as_index", .{ .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{mlir.indexType(ctx)} }, + .results = .{ .flat = &.{.index(ctx)} }, .location = location, }); } @@ -382,10 +382,10 @@ pub fn subview( .operands = .{ .flat = operands_buf.constSlice() }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), - .named(ctx, "static_offsets", mlir.denseArrayAttribute(ctx, .i64, args.static_offsets)), - .named(ctx, "static_sizes", mlir.denseArrayAttribute(ctx, .i64, args.static_sizes)), - .named(ctx, "static_strides", mlir.denseArrayAttribute(ctx, .i64, args.static_strides)), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "static_offsets", .denseArray(ctx, .i64, args.static_offsets)), + .named(ctx, "static_sizes", .denseArray(ctx, .i64, args.static_sizes)), + .named(ctx, "static_strides", .denseArray(ctx, .i64, args.static_strides)), }, .location = location, }); @@ -426,10 +426,10 @@ pub fn reinterpret_cast( .operands = .{ .flat = operands_buf.constSlice() }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), - .named(ctx, "static_offsets", mlir.denseArrayAttribute(ctx, .i64, args.static_offsets)), - .named(ctx, "static_sizes", mlir.denseArrayAttribute(ctx, .i64, args.static_sizes)), - .named(ctx, "static_strides", mlir.denseArrayAttribute(ctx, .i64, args.static_strides)), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "static_offsets", .denseArray(ctx, .i64, args.static_offsets)), + .named(ctx, "static_sizes", .denseArray(ctx, .i64, args.static_sizes)), + .named(ctx, "static_strides", .denseArray(ctx, .i64, args.static_strides)), }, .location = location, }); @@ -471,16 +471,15 @@ pub const ReassociationArgs = struct { fn buildReassociationAttr(ctx: *mlir.Context, reassociation: []const []const i64) *const mlir.Attribute { var outer: stdx.BoundedArray(*const mlir.Attribute, 32) = .empty; - var inner_buf: stdx.BoundedArray(*const mlir.Attribute, 64) = .empty; + var inner_buf: stdx.BoundedArray(*const mlir.Attribute, 32) = .empty; for (reassociation) |group| { - const start = inner_buf.len; + inner_buf = .empty; for (group) |idx| { - inner_buf.appendAssumeCapacity(mlir.integerAttribute(ctx, .i64, idx)); + inner_buf.appendAssumeCapacity(.int(ctx, .i64, idx)); } - const inner_attr = mlir.arrayAttribute(ctx, inner_buf.constSlice()[start..]); - outer.appendAssumeCapacity(inner_attr); + outer.appendAssumeCapacity(.array(ctx, inner_buf.constSlice())); } - return mlir.arrayAttribute(ctx, outer.constSlice()); + return .array(ctx, outer.constSlice()); } /// memref.expand_shape — split source dimensions into multiple result @@ -502,7 +501,7 @@ pub fn expand_shape( .results = .{ .flat = &.{result_type} }, .attributes = &.{ .named(ctx, "reassociation", buildReassociationAttr(ctx, args.reassociation)), - .named(ctx, "static_output_shape", mlir.denseArrayAttribute(ctx, .i64, args.static_output_shape)), + .named(ctx, "static_output_shape", .denseArray(ctx, .i64, args.static_output_shape)), }, .location = location, }); @@ -594,9 +593,9 @@ pub fn prefetch( return mlir.Operation.make(ctx, "memref.prefetch", .{ .operands = .{ .flat = buf.constSlice() }, .attributes = &.{ - .named(ctx, "isWrite", mlir.boolAttribute(ctx, opts.is_write)), - .named(ctx, "localityHint", mlir.integerAttribute(ctx, .i32, opts.locality_hint)), - .named(ctx, "isDataCache", mlir.boolAttribute(ctx, opts.is_data_cache)), + .named(ctx, "isWrite", .boolean(ctx, opts.is_write)), + .named(ctx, "localityHint", .int(ctx, .i32, opts.locality_hint)), + .named(ctx, "isDataCache", .boolean(ctx, opts.is_data_cache)), }, .location = location, }); @@ -623,18 +622,18 @@ pub const GlobalArgs = struct { pub fn global(ctx: *mlir.Context, args: GlobalArgs, location: *const mlir.Location) *mlir.Operation { var attrs: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; attrs.appendSliceAssumeCapacity(&.{ - .named(ctx, "sym_name", mlir.stringAttribute(ctx, args.name)), - .named(ctx, "sym_visibility", mlir.stringAttribute(ctx, @tagName(args.visibility))), - .named(ctx, "type", mlir.typeAttribute(args.type_)), + .named(ctx, "sym_name", .string(ctx, args.name)), + .named(ctx, "sym_visibility", .string(ctx, @tagName(args.visibility))), + .named(ctx, "type", .typeAttr(args.type_)), }); if (args.initial_value) |iv| { attrs.appendAssumeCapacity(.named(ctx, "initial_value", iv)); } if (args.constant) { - attrs.appendAssumeCapacity(.named(ctx, "constant", mlir.unitAttribute(ctx))); + attrs.appendAssumeCapacity(.named(ctx, "constant", .unit(ctx))); } if (args.alignment) |a| { - attrs.appendAssumeCapacity(.named(ctx, "alignment", mlir.integerAttribute(ctx, .i64, a))); + attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, a))); } return mlir.Operation.make(ctx, "memref.global", .{ @@ -654,7 +653,7 @@ pub fn get_global( return mlir.Operation.make(ctx, "memref.get_global", .{ .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "name", mlir.flatSymbolRefAttribute(ctx, name)), + .named(ctx, "name", .flatSymbolRef(ctx, name)), }, .location = location, }); diff --git a/mlir/dialects/mosaic_tpu/mosaic_tpu.zig b/mlir/dialects/mosaic_tpu/mosaic_tpu.zig index c437f234..35598957 100644 --- a/mlir/dialects/mosaic_tpu/mosaic_tpu.zig +++ b/mlir/dialects/mosaic_tpu/mosaic_tpu.zig @@ -219,7 +219,7 @@ pub fn all_reduce( .operands = .{ .flat = &.{input} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "dim", mlir.integerAttribute(ctx, .i64, dim)), + .named(ctx, "dim", .int(ctx, .i64, dim)), .named(ctx, "kind", kind.attribute(ctx)), }, .location = location, @@ -238,7 +238,7 @@ pub fn reduce_index( .operands = .{ .flat = &.{input} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "axis", mlir.integerAttribute(ctx, .i32, axis)), + .named(ctx, "axis", .int(ctx, .i32, axis)), .named(ctx, "kind", kind.attribute(ctx)), }, .location = location, @@ -290,7 +290,7 @@ pub fn sort( .operands = .{ .flat = operands_buf.constSlice() }, .results = .{ .flat = &.{ output_mask_type, sorted_keys_type, sorted_values_type } }, .attributes = &.{ - .named(ctx, "descending", mlir.boolAttribute(ctx, opts.descending)), + .named(ctx, "descending", .boolean(ctx, opts.descending)), }, .location = location, }); @@ -323,7 +323,7 @@ pub fn load( .results = .{ .flat = &.{result_type} }, .attributes = &.{ .named(ctx, "sublane_mask", denseBoolArrayAttribute(ctx, opts.sublane_mask)), - .named(ctx, "sublane_stride", mlir.integerAttribute(ctx, .i32, opts.sublane_stride)), + .named(ctx, "sublane_stride", .int(ctx, .i32, opts.sublane_stride)), }, .location = location, }); @@ -357,10 +357,10 @@ pub fn store( return mlir.Operation.make(ctx, "tpu.store", .{ .operands = .{ .flat = operands_buf.constSlice() }, .attributes = &.{ - .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), .named(ctx, "sublane_mask", denseBoolArrayAttribute(ctx, opts.sublane_mask)), - .named(ctx, "sublane_stride", mlir.integerAttribute(ctx, .i32, opts.sublane_stride)), - .named(ctx, "add", mlir.boolAttribute(ctx, opts.add)), + .named(ctx, "sublane_stride", .int(ctx, .i32, opts.sublane_stride)), + .named(ctx, "add", .boolean(ctx, opts.add)), }, .location = location, }); @@ -394,8 +394,8 @@ pub fn vector_load( .operands = .{ .flat = operands_buf.constSlice() }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), - .named(ctx, "strides", mlir.denseArrayAttribute(ctx, .i32, opts.strides)), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "strides", .denseArray(ctx, .i32, opts.strides)), }, .location = location, }); @@ -427,9 +427,9 @@ pub fn vector_store( return mlir.Operation.make(ctx, "tpu.vector_store", .{ .operands = .{ .flat = operands_buf.constSlice() }, .attributes = &.{ - .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), - .named(ctx, "strides", mlir.denseArrayAttribute(ctx, .i32, opts.strides)), - .named(ctx, "add", mlir.boolAttribute(ctx, opts.add)), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "strides", .denseArray(ctx, .i32, opts.strides)), + .named(ctx, "add", .boolean(ctx, opts.add)), }, .location = location, }); @@ -451,7 +451,7 @@ pub fn strided_load( .operands = .{ .flat = operands_buf.constSlice() }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "strides", mlir.denseArrayAttribute(ctx, .i32, strides)), + .named(ctx, "strides", .denseArray(ctx, .i32, strides)), }, .location = location, }); @@ -472,7 +472,7 @@ pub fn strided_store( return mlir.Operation.make(ctx, "tpu.strided_store", .{ .operands = .{ .flat = operands_buf.constSlice() }, .attributes = &.{ - .named(ctx, "strides", mlir.denseArrayAttribute(ctx, .i32, strides)), + .named(ctx, "strides", .denseArray(ctx, .i32, strides)), }, .location = location, }); @@ -504,8 +504,8 @@ pub fn matmul( ) *mlir.Operation { var attrs: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; attrs.appendSliceAssumeCapacity(&.{ - .named(ctx, "transpose_lhs", mlir.boolAttribute(ctx, opts.transpose_lhs)), - .named(ctx, "transpose_rhs", mlir.boolAttribute(ctx, opts.transpose_rhs)), + .named(ctx, "transpose_lhs", .boolean(ctx, opts.transpose_lhs)), + .named(ctx, "transpose_rhs", .boolean(ctx, opts.transpose_rhs)), }); if (opts.precision) |p| { attrs.appendAssumeCapacity(.named(ctx, "precision", p.attribute(ctx))); @@ -530,7 +530,7 @@ pub fn iota( ) *mlir.Operation { var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; if (dimensions) |d| { - attrs.appendAssumeCapacity(.named(ctx, "dimensions", mlir.denseArrayAttribute(ctx, .i32, d))); + attrs.appendAssumeCapacity(.named(ctx, "dimensions", .denseArray(ctx, .i32, d))); } return mlir.Operation.make(ctx, "tpu.iota", .{ .results = .{ .flat = &.{result_type} }, @@ -554,8 +554,8 @@ pub fn reciprocal( .operands = .{ .flat = &.{input} }, .results = .{ .flat = &.{input.type_()} }, .attributes = &.{ - .named(ctx, "approx", mlir.boolAttribute(ctx, opts.approx)), - .named(ctx, "full_range", mlir.boolAttribute(ctx, opts.full_range)), + .named(ctx, "approx", .boolean(ctx, opts.approx)), + .named(ctx, "full_range", .boolean(ctx, opts.full_range)), }, .location = location, }); @@ -619,8 +619,8 @@ pub fn repeat( .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "dimension", mlir.integerAttribute(ctx, .i32, dimension)), - .named(ctx, "times", mlir.integerAttribute(ctx, .i32, times)), + .named(ctx, "dimension", .int(ctx, .i32, dimension)), + .named(ctx, "times", .int(ctx, .i32, times)), }, .location = location, }); @@ -637,7 +637,7 @@ pub fn concatenate( .operands = .{ .flat = sources }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "dimension", mlir.integerAttribute(ctx, .i32, dimension)), + .named(ctx, "dimension", .int(ctx, .i32, dimension)), }, .location = location, }); @@ -654,7 +654,7 @@ pub fn transpose( .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "permutation", mlir.denseArrayAttribute(ctx, .i32, permutation)), + .named(ctx, "permutation", .denseArray(ctx, .i32, permutation)), }, .location = location, }); @@ -671,7 +671,7 @@ pub fn broadcast_in_sublanes( .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "lane", mlir.integerAttribute(ctx, .i32, lane)), + .named(ctx, "lane", .int(ctx, .i32, lane)), }, .location = location, }); @@ -692,11 +692,11 @@ pub fn rotate( ) *mlir.Operation { var attrs: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; attrs.appendSliceAssumeCapacity(&.{ - .named(ctx, "amount", mlir.integerAttribute(ctx, .i32, opts.amount)), - .named(ctx, "dimension", mlir.integerAttribute(ctx, .i32, opts.dimension)), + .named(ctx, "amount", .int(ctx, .i32, opts.amount)), + .named(ctx, "dimension", .int(ctx, .i32, opts.dimension)), }); - if (opts.stride) |s| attrs.appendAssumeCapacity(.named(ctx, "stride", mlir.integerAttribute(ctx, .i32, s))); - if (opts.stride_dimension) |sd| attrs.appendAssumeCapacity(.named(ctx, "stride_dimension", mlir.integerAttribute(ctx, .i32, sd))); + if (opts.stride) |s| attrs.appendAssumeCapacity(.named(ctx, "stride", .int(ctx, .i32, s))); + if (opts.stride_dimension) |sd| attrs.appendAssumeCapacity(.named(ctx, "stride_dimension", .int(ctx, .i32, sd))); return mlir.Operation.make(ctx, "tpu.rotate", .{ .operands = .{ .flat = &.{value} }, @@ -734,7 +734,7 @@ pub fn memref_slice( .operands = .{ .flat = operands_buf.constSlice() }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), }, .location = location, }); @@ -875,7 +875,7 @@ pub fn sem_read( ) *mlir.Operation { return mlir.Operation.make(ctx, "tpu.sem_read", .{ .operands = .{ .flat = &.{semaphore} }, - .results = .{ .flat = &.{mlir.integerType(ctx, .i32)} }, + .results = .{ .flat = &.{.int(ctx, .i32)} }, .location = location, }); } @@ -919,7 +919,7 @@ pub fn sem_signal( return mlir.Operation.make(ctx, "tpu.sem_signal", .{ .operands = .{ .flat = operands_buf.constSlice() }, .attributes = &.{ - .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), }, .location = location, }); @@ -974,9 +974,9 @@ pub fn enqueue_dma( return mlir.Operation.make(ctx, "tpu.enqueue_dma", .{ .operands = .{ .flat = operands_buf.constSlice() }, .attributes = &.{ - .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), - .named(ctx, "priority", mlir.integerAttribute(ctx, .i32, opts.priority)), - .named(ctx, "strict_ordering", mlir.boolAttribute(ctx, opts.strict_ordering)), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "priority", .int(ctx, .i32, opts.priority)), + .named(ctx, "strict_ordering", .boolean(ctx, opts.strict_ordering)), }, .location = location, }); @@ -1011,8 +1011,8 @@ pub fn wait_dma2( return mlir.Operation.make(ctx, "tpu.wait_dma2", .{ .operands = .{ .flat = operands_buf.constSlice() }, .attributes = &.{ - .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), - .named(ctx, "strict_ordering", mlir.boolAttribute(ctx, opts.strict_ordering)), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "strict_ordering", .boolean(ctx, opts.strict_ordering)), }, .location = location, }); @@ -1071,8 +1071,8 @@ pub fn trace_start( ) *mlir.Operation { return mlir.Operation.make(ctx, "tpu.trace_start", .{ .attributes = &.{ - .named(ctx, "level", mlir.integerAttribute(ctx, .i32, level)), - .named(ctx, "message", mlir.stringAttribute(ctx, message)), + .named(ctx, "level", .int(ctx, .i32, level)), + .named(ctx, "message", .string(ctx, message)), }, .verify = false, .location = location, @@ -1130,7 +1130,7 @@ pub fn dotDimensionNumbers( pub fn device_id(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "tpu.device_id", .{ - .results = .{ .flat = &.{mlir.integerType(ctx, .i32)} }, + .results = .{ .flat = &.{.int(ctx, .i32)} }, .location = location, }); } @@ -1156,7 +1156,7 @@ pub fn assume_multiple( .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{src.type_()} }, .attributes = &.{ - .named(ctx, "multiple", mlir.integerAttribute(ctx, .i64, multiple)), + .named(ctx, "multiple", .int(ctx, .i64, multiple)), }, .location = location, }); @@ -1170,7 +1170,7 @@ pub fn assume_multiple( fn denseBoolArrayAttribute(ctx: *mlir.Context, values: []const bool) *const mlir.Attribute { var bytes: stdx.BoundedArray(i32, 64) = .empty; for (values) |b| bytes.appendAssumeCapacity(@intFromBool(b)); - return mlir.denseArrayAttribute(ctx, .bool, bytes.constSlice()); + return .denseArray(ctx, .bool, bytes.constSlice()); } test { diff --git a/mlir/dialects/scf.zig b/mlir/dialects/scf.zig index 6dbc337a..2feb5268 100644 --- a/mlir/dialects/scf.zig +++ b/mlir/dialects/scf.zig @@ -35,10 +35,10 @@ pub fn for_( var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; if (opts.unsigned_cmp) { - attrs.appendAssumeCapacity(.named(ctx, "unsignedCmp", mlir.unitAttribute(ctx))); + attrs.appendAssumeCapacity(.named(ctx, "unsignedCmp", .unit(ctx))); } if (opts.num_stages) |n| { - attrs.appendAssumeCapacity(.named(ctx, "tt.num_stages", mlir.integerAttribute(ctx, .i32, n))); + attrs.appendAssumeCapacity(.named(ctx, "tt.num_stages", .int(ctx, .i32, n))); } return mlir.Operation.make(ctx, "scf.for", .{ @@ -151,7 +151,7 @@ pub fn execute_region( ) *mlir.Operation { var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; if (opts.no_inline) { - attrs.appendAssumeCapacity(.named(ctx, "no_inline", mlir.unitAttribute(ctx))); + attrs.appendAssumeCapacity(.named(ctx, "no_inline", .unit(ctx))); } return mlir.Operation.make(ctx, "scf.execute_region", .{ .results = .{ .flat = result_types }, @@ -198,7 +198,7 @@ pub fn parallel( .results = .{ .flat = result_types }, .blocks = &.{body_block}, .attributes = &.{ - .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), }, .verify = false, .location = location, @@ -284,10 +284,10 @@ pub fn forall( var attrs: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; attrs.appendSliceAssumeCapacity(&.{ - .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), - .named(ctx, "staticLowerBound", mlir.denseArrayAttribute(ctx, .i64, sentinel_slice)), - .named(ctx, "staticUpperBound", mlir.denseArrayAttribute(ctx, .i64, sentinel_slice)), - .named(ctx, "staticStep", mlir.denseArrayAttribute(ctx, .i64, sentinel_slice)), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "staticLowerBound", .denseArray(ctx, .i64, sentinel_slice)), + .named(ctx, "staticUpperBound", .denseArray(ctx, .i64, sentinel_slice)), + .named(ctx, "staticStep", .denseArray(ctx, .i64, sentinel_slice)), }); if (mapping) |m| attrs.appendAssumeCapacity(.named(ctx, "mapping", m)); @@ -334,7 +334,7 @@ pub fn index_switch( state.addOperands(&.{arg}); state.addResults(result_types); state.addAttributes(&.{ - .named(ctx, "cases", mlir.denseArrayAttribute(ctx, .i64, cases)), + .named(ctx, "cases", .denseArray(ctx, .i64, cases)), }); // First region is the default, then one per case — this is how diff --git a/mlir/dialects/shardy.zig b/mlir/dialects/shardy.zig index 41aa8e2b..376820a5 100644 --- a/mlir/dialects/shardy.zig +++ b/mlir/dialects/shardy.zig @@ -205,7 +205,7 @@ pub const TensorShardingAttribute = opaque { const unreduced: []const c.struct_MlirAttribute = @ptrCast(args.unreduced_axes); return @ptrCast(c.sdyTensorShardingAttrGet( ctx.ptr(), - mlir.flatSymbolRefAttribute(ctx, args.mesh).ptr(), + mlir.Attribute.flatSymbolRef(ctx, args.mesh).ptr(), @intCast(dimensions.len), dimensions.ptr, @intCast(replicated.len), diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index bc2de9e4..25890b74 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -93,7 +93,7 @@ pub fn cholesky(ctx: *mlir.Context, value: *const mlir.Value, lower: bool, locat .operands = .{ .flat = &.{value} }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "lower", mlir.integerAttribute(ctx, .i1, @intFromBool(lower))), + .named(ctx, "lower", .int(ctx, .i1, @intFromBool(lower))), }, .location = location, }); @@ -154,7 +154,7 @@ pub fn dotAlgorithmAttribute(ctx: *mlir.Context, dot_algorithm: DotAlgorithm, el ctx.ptr(), element_type.ptr(), element_type.ptr(), - mlir.floatType(ctx, dot_algorithm.accumulation).ptr(), + mlir.Type.float(ctx, dot_algorithm.accumulation).ptr(), dot_algorithm.component_count, dot_algorithm.component_count, dot_algorithm.num_primitive_operations, @@ -190,7 +190,7 @@ pub fn dot_general( .rhs_contracting_dimensions = opts.rhs_contracting_dimensions, }), ), - .named(ctx, "precision_config", mlir.arrayAttribute(ctx, &precisions)), + .named(ctx, "precision_config", .array(ctx, &precisions)), // keep algorithm as the last attribute so we can omit it when it's not set. .named(ctx, "algorithm", switch (opts.dot_precision) { .algorithm => |v| dotAlgorithmAttribute(ctx, v, lhs.type_()), @@ -215,9 +215,9 @@ pub fn constant( ) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.constant", .{ .operands = .{ .flat = &.{} }, - .results = .{ .flat = &.{mlir.rankedTensorType(dims, elem_type)} }, + .results = .{ .flat = &.{.rankedTensor(dims, elem_type)} }, .attributes = &.{ - .named(ctx, "value", mlir.denseElementsAttribute(mlir.RankedTensorType.get(dims, elem_type, null).shaped(), raw_bytes)), + .named(ctx, "value", .denseElements(.rankedTensor(dims, elem_type), raw_bytes)), }, .location = location, }); @@ -234,7 +234,7 @@ pub fn broadcast_in_dim( .operands = .{ .flat = &.{operand} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "broadcast_dimensions", mlir.denseArrayAttribute(ctx, .i64, dims)), + .named(ctx, "broadcast_dimensions", .denseArray(ctx, .i64, dims)), }, .location = location, }); @@ -251,7 +251,7 @@ pub fn transpose( .operands = .{ .flat = &.{value} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "permutation", mlir.denseArrayAttribute(ctx, .i64, opts.permutation)), + .named(ctx, "permutation", .denseArray(ctx, .i64, opts.permutation)), }, .location = location, }); @@ -270,9 +270,9 @@ pub fn slice( .operands = .{ .flat = &.{operand} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "start_indices", mlir.denseArrayAttribute(ctx, .i64, start_indices)), - .named(ctx, "limit_indices", mlir.denseArrayAttribute(ctx, .i64, limit_indices)), - .named(ctx, "strides", mlir.denseArrayAttribute(ctx, .i64, strides)), + .named(ctx, "start_indices", .denseArray(ctx, .i64, start_indices)), + .named(ctx, "limit_indices", .denseArray(ctx, .i64, limit_indices)), + .named(ctx, "strides", .denseArray(ctx, .i64, strides)), }, .location = location, }); @@ -283,7 +283,7 @@ pub fn concatenate(ctx: *mlir.Context, inputs: []const *const mlir.Value, dimens .operands = .{ .flat = inputs }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "dimension", mlir.integerAttribute(ctx, .i64, dimension)), + .named(ctx, "dimension", .int(ctx, .i64, dimension)), }, .location = location, }); @@ -335,8 +335,8 @@ pub fn gather( args.start_index_map, args.index_vector_dim, )), - .named(ctx, "slice_sizes", mlir.denseArrayAttribute(ctx, .i64, slice_sizes)), - .named(ctx, "indices_are_sorted", mlir.boolAttribute(ctx, args.indices_are_sorted)), + .named(ctx, "slice_sizes", .denseArray(ctx, .i64, slice_sizes)), + .named(ctx, "indices_are_sorted", .boolean(ctx, args.indices_are_sorted)), }, .location = location, }, @@ -388,8 +388,8 @@ pub fn scatter( .blocks = &.{update_block}, .attributes = &.{ .named(ctx, "scatter_dimension_numbers", args.getScatterDimensionNumbers(ctx)), - .named(ctx, "indices_are_sorted", mlir.boolAttribute(ctx, args.indices_are_sorted)), - .named(ctx, "unique_indices", mlir.boolAttribute(ctx, args.unique_indices)), + .named(ctx, "indices_are_sorted", .boolean(ctx, args.indices_are_sorted)), + .named(ctx, "unique_indices", .boolean(ctx, args.unique_indices)), }, .result_type_inference = true, .location = location, @@ -402,7 +402,7 @@ pub fn iota(ctx: *mlir.Context, dimension: i64, result_type: *const mlir.Type, l .operands = .{ .flat = &.{} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "iota_dimension", mlir.integerAttribute(ctx, .i64, dimension)), + .named(ctx, "iota_dimension", .int(ctx, .i64, dimension)), }, .location = location, }); @@ -413,7 +413,7 @@ pub fn reverse(ctx: *mlir.Context, operand: *const mlir.Value, dimensions: []con .operands = .{ .flat = &.{operand} }, .results = .{ .flat = &.{operand.type_()} }, .attributes = &.{ - .named(ctx, "dimensions", mlir.denseArrayAttribute(ctx, .i64, dimensions)), + .named(ctx, "dimensions", .denseArray(ctx, .i64, dimensions)), }, .location = location, }); @@ -449,7 +449,7 @@ pub fn dynamic_slice( .operands = .{ .variadic = &.{ &.{operand}, start_indices } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "slice_sizes", mlir.denseArrayAttribute(ctx, .i64, new_dims)), + .named(ctx, "slice_sizes", .denseArray(ctx, .i64, new_dims)), }, .location = location, }); @@ -466,9 +466,9 @@ pub fn pad(ctx: *mlir.Context, value: *const mlir.Value, padding_value: *const m .operands = .{ .flat = &.{ value, padding_value } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "edge_padding_low", mlir.denseArrayAttribute(ctx, .i64, opts.low)), - .named(ctx, "edge_padding_high", mlir.denseArrayAttribute(ctx, .i64, opts.high)), - .named(ctx, "interior_padding", mlir.denseArrayAttribute(ctx, .i64, opts.interior)), + .named(ctx, "edge_padding_low", .denseArray(ctx, .i64, opts.low)), + .named(ctx, "edge_padding_high", .denseArray(ctx, .i64, opts.high)), + .named(ctx, "interior_padding", .denseArray(ctx, .i64, opts.interior)), }, .location = location, }); @@ -486,9 +486,9 @@ pub fn triangular_solve(ctx: *mlir.Context, value: *const mlir.Value, other: *co .operands = .{ .flat = &.{ value, other } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "left_side", mlir.integerAttribute(ctx, .i1, @intFromBool(opts.left_side))), - .named(ctx, "lower", mlir.integerAttribute(ctx, .i1, @intFromBool(opts.lower))), - .named(ctx, "unit_diagonal", mlir.integerAttribute(ctx, .i1, @intFromBool(opts.unit_diagonal))), + .named(ctx, "left_side", .int(ctx, .i1, @intFromBool(opts.left_side))), + .named(ctx, "lower", .int(ctx, .i1, @intFromBool(opts.lower))), + .named(ctx, "unit_diagonal", .int(ctx, .i1, @intFromBool(opts.unit_diagonal))), .named(ctx, "transpose_a", transposeAttribute(ctx, opts.transpose_a)), }, .location = location, @@ -506,7 +506,7 @@ pub fn fft(ctx: *mlir.Context, value: *const mlir.Value, opts: FftOpts, location .result_type_inference = true, .attributes = &.{ .named(ctx, "fft_type", fftType(ctx, opts.kind)), - .named(ctx, "fft_length", mlir.denseArrayAttribute(ctx, .i64, opts.length)), + .named(ctx, "fft_length", .denseArray(ctx, .i64, opts.length)), }, .location = location, }); @@ -605,8 +605,8 @@ pub fn reduce_precision(ctx: *mlir.Context, value: *const mlir.Value, exponent_b .operands = .{ .flat = &.{value} }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "exponent_bits", mlir.integerAttribute(ctx, .i32, exponent_bits)), - .named(ctx, "mantissa_bits", mlir.integerAttribute(ctx, .i32, mantissa_bits)), + .named(ctx, "exponent_bits", .int(ctx, .i32, exponent_bits)), + .named(ctx, "mantissa_bits", .int(ctx, .i32, mantissa_bits)), }, .location = location, }); @@ -647,7 +647,7 @@ pub fn get_tuple_element(ctx: *mlir.Context, tuple_value: *const mlir.Value, ind .operands = .{ .flat = &.{tuple_value} }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "index", mlir.integerAttribute(ctx, .i32, index)), + .named(ctx, "index", .int(ctx, .i32, index)), }, .location = location, }); @@ -694,11 +694,11 @@ pub fn convolution( .operands = .{ .flat = &.{ lhs, rhs } }, .results = .{ .flat = &.{res_type} }, .attributes = &.{ - .named(ctx, "window_strides", mlir.denseArrayAttribute(ctx, .i64, opts.window_strides)), - .named(ctx, "padding", mlir.denseElementsAttribute(mlir.RankedTensorType.get(opts.pad_shape, mlir.integerType(ctx, .i64), null).shaped(), opts.pad_value)), - .named(ctx, "lhs_dilation", mlir.denseArrayAttribute(ctx, .i64, opts.lhs_dilation)), - .named(ctx, "rhs_dilation", mlir.denseArrayAttribute(ctx, .i64, opts.rhs_dilation)), - .named(ctx, "window_reversal", mlir.denseArrayAttribute(ctx, .bool, window_reversal[0..opts.window_reversal.len])), + .named(ctx, "window_strides", .denseArray(ctx, .i64, opts.window_strides)), + .named(ctx, "padding", .denseElements(.rankedTensor(opts.pad_shape, .int(ctx, .i64)), opts.pad_value)), + .named(ctx, "lhs_dilation", .denseArray(ctx, .i64, opts.lhs_dilation)), + .named(ctx, "rhs_dilation", .denseArray(ctx, .i64, opts.rhs_dilation)), + .named(ctx, "window_reversal", .denseArray(ctx, .bool, window_reversal[0..opts.window_reversal.len])), .named(ctx, "dimension_numbers", convDimensionNumbersAttribute(ctx, .{ .input_batch_dimension = opts.input_batch_dimension, .input_feature_dimension = opts.input_feature_dimension, @@ -710,9 +710,9 @@ pub fn convolution( .output_feature_dimension = opts.output_feature_dimension, .output_spatial_dimensions = opts.output_spatial_dimensions, })), - .named(ctx, "feature_group_count", mlir.integerAttribute(ctx, .i64, opts.feature_group_count)), - .named(ctx, "batch_group_count", mlir.integerAttribute(ctx, .i64, opts.batch_group_count)), - .named(ctx, "precision_config", mlir.arrayAttribute(ctx, &max_precisions)), + .named(ctx, "feature_group_count", .int(ctx, .i64, opts.feature_group_count)), + .named(ctx, "batch_group_count", .int(ctx, .i64, opts.batch_group_count)), + .named(ctx, "precision_config", .array(ctx, &max_precisions)), }, .location = location, }); @@ -761,27 +761,27 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result var attrs: stdx.BoundedArray(mlir.NamedAttribute, 32) = .empty; attrs.appendSliceAssumeCapacity(&.{ - .named(ctx, "call_target_name", mlir.stringAttribute(ctx, opts.call_target_name)), + .named(ctx, "call_target_name", .string(ctx, opts.call_target_name)), }); attrs.appendSliceAssumeCapacity(opts.additional_attributes); if (opts.has_side_effect) |has_side_effects| { attrs.appendAssumeCapacity( - .named(ctx, "has_side_effect", mlir.boolAttribute(ctx, has_side_effects)), + .named(ctx, "has_side_effect", .boolean(ctx, has_side_effects)), ); } const backend_config_api_version: ?CustomCallOpts.ApiVersion = if (opts.backend_config) |bc| bc else null; if (opts.api_version orelse backend_config_api_version) |v| { attrs.appendAssumeCapacity( - .named(ctx, "api_version", mlir.integerAttribute(ctx, .i32, @intFromEnum(v))), + .named(ctx, "api_version", .int(ctx, .i32, @intFromEnum(v))), ); } if (opts.backend_config) |backend_config| { attrs.appendAssumeCapacity( .named(ctx, "backend_config", switch (backend_config) { .typed_ffi => |v| @ptrCast(v), - inline else => |v| mlir.stringAttribute(ctx, v), + inline else => |v| .string(ctx, v), }), ); } @@ -795,7 +795,7 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result ); } attrs.appendAssumeCapacity( - .named(ctx, "output_operand_aliases", mlir.arrayAttribute(ctx, buffer.constSlice())), + .named(ctx, "output_operand_aliases", .array(ctx, buffer.constSlice())), ); } @@ -803,12 +803,8 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result if (opts.operand_layouts) |layouts| { for (layouts) |ol| { layouts_buffer.appendAssumeCapacity( - mlir.denseElementsAttribute( - mlir.RankedTensorType.get( - &.{@intCast(ol.len)}, - mlir.indexType(ctx), - null, - ).shaped(), + .denseElements( + .rankedTensor(&.{@intCast(ol.len)}, .index(ctx)), ol, ), ); @@ -818,31 +814,23 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result const shaped_type = input.type_().isA(mlir.ShapedType).?; const ol = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - shaped_type.rank() ..]; layouts_buffer.appendAssumeCapacity( - mlir.denseElementsAttribute( - mlir.RankedTensorType.get( - &.{@intCast(ol.len)}, - mlir.indexType(ctx), - null, - ).shaped(), + .denseElements( + .rankedTensor(&.{@intCast(ol.len)}, .index(ctx)), ol, ), ); } } attrs.appendAssumeCapacity( - .named(ctx, "operand_layouts", mlir.arrayAttribute(ctx, layouts_buffer.constSlice())), + .named(ctx, "operand_layouts", .array(ctx, layouts_buffer.constSlice())), ); layouts_buffer.clear(); if (opts.result_layouts) |layouts| { for (layouts) |rl| { layouts_buffer.appendAssumeCapacity( - mlir.denseElementsAttribute( - mlir.RankedTensorType.get( - &.{@intCast(rl.len)}, - mlir.indexType(ctx), - null, - ).shaped(), + .denseElements( + .rankedTensor(&.{@intCast(rl.len)}, .index(ctx)), rl, ), ); @@ -852,19 +840,15 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result const shaped_type = result_type.isA(mlir.ShapedType).?; const rl = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - shaped_type.rank() ..]; layouts_buffer.appendAssumeCapacity( - mlir.denseElementsAttribute( - mlir.RankedTensorType.get( - &.{@intCast(rl.len)}, - mlir.indexType(ctx), - null, - ).shaped(), + .denseElements( + .rankedTensor(&.{@intCast(rl.len)}, .index(ctx)), rl, ), ); } } attrs.appendAssumeCapacity( - .named(ctx, "result_layouts", mlir.arrayAttribute(ctx, layouts_buffer.constSlice())), + .named(ctx, "result_layouts", .array(ctx, layouts_buffer.constSlice())), ); return mlir.Operation.make(ctx, "stablehlo.custom_call", .{ @@ -910,7 +894,7 @@ pub fn unpin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir pub fn partition_id(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.partition_id", .{ - .results = .{ .flat = &.{mlir.rankedTensorType(&.{}, mlir.integerType(ctx, .u32))} }, + .results = .{ .flat = &.{.rankedTensor(&.{}, mlir.integerType(ctx, .u32))} }, .location = location, }); } diff --git a/mlir/dialects/ttir/ttir.zig b/mlir/dialects/ttir/ttir.zig index dde46822..81d926dc 100644 --- a/mlir/dialects/ttir/ttir.zig +++ b/mlir/dialects/ttir/ttir.zig @@ -217,9 +217,9 @@ pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { var attr_tuples_buffer: stdx.BoundedArray(mlir.NamedAttribute, 16) = .empty; attr_tuples_buffer.appendSliceAssumeCapacity(&.{ - .named(ctx, "sym_name", mlir.stringAttribute(ctx, args.name)), - .named(ctx, "sym_visibility", mlir.stringAttribute(ctx, @tagName(args.visibility))), - .named(ctx, "function_type", mlir.typeAttribute(mlir.functionType( + .named(ctx, "sym_name", .string(ctx, args.name)), + .named(ctx, "sym_visibility", .string(ctx, @tagName(args.visibility))), + .named(ctx, "function_type", .typeAttr(.function( ctx, args.args orelse args_: { for (0..args.block.numArguments()) |i| { @@ -240,15 +240,15 @@ pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { }); if (args.args_attributes) |args_attributes| { - attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "arg_attrs", mlir.arrayAttribute(ctx, args_attributes))); + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "arg_attrs", .array(ctx, args_attributes))); } if (args.results_attributes) |results_attributes| { - attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "res_attrs", mlir.arrayAttribute(ctx, results_attributes))); + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "res_attrs", .array(ctx, results_attributes))); } // Always emit noinline even when false — matches Triton's IR convention. - attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "noinline", mlir.boolAttribute(ctx, args.no_inline))); + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "noinline", .boolean(ctx, args.no_inline))); - return mlir.Operation.make(ctx, "tt.func", .{ + return .make(ctx, "tt.func", .{ .blocks = &.{args.block}, .attributes = attr_tuples_buffer.constSlice(), .location = args.location, @@ -257,7 +257,7 @@ pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { } pub fn return_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.return", .{ + return .make(ctx, "tt.return", .{ .operands = .{ .flat = values }, .verify = false, .location = location, @@ -269,8 +269,8 @@ pub fn return_(ctx: *mlir.Context, values: []const *const mlir.Value, location: // ============================================================================= pub fn get_program_id(ctx: *mlir.Context, axis: ProgramIDDim, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.get_program_id", .{ - .results = .{ .flat = &.{mlir.integerType(ctx, .i32)} }, + return .make(ctx, "tt.get_program_id", .{ + .results = .{ .flat = &.{.int(ctx, .i32)} }, .attributes = &.{ .named(ctx, "axis", programDim(ctx, axis)), }, @@ -279,8 +279,8 @@ pub fn get_program_id(ctx: *mlir.Context, axis: ProgramIDDim, location: *const m } pub fn get_num_programs(ctx: *mlir.Context, axis: ProgramIDDim, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.get_num_programs", .{ - .results = .{ .flat = &.{mlir.integerType(ctx, .i32)} }, + return .make(ctx, "tt.get_num_programs", .{ + .results = .{ .flat = &.{.int(ctx, .i32)} }, .attributes = &.{ .named(ctx, "axis", programDim(ctx, axis)), }, @@ -299,11 +299,11 @@ pub fn make_range( result_type: *const mlir.Type, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.make_range", .{ + return .make(ctx, "tt.make_range", .{ .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "start", mlir.integerAttribute(ctx, .i32, start)), - .named(ctx, "end", mlir.integerAttribute(ctx, .i32, end)), + .named(ctx, "start", .int(ctx, .i32, start)), + .named(ctx, "end", .int(ctx, .i32, end)), }, .location = location, }); @@ -319,7 +319,7 @@ pub fn splat( result_type: *const mlir.Type, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.splat", .{ + return .make(ctx, "tt.splat", .{ .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{result_type} }, .location = location, @@ -333,11 +333,11 @@ pub fn expand_dims( result_type: *const mlir.Type, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.expand_dims", .{ + return .make(ctx, "tt.expand_dims", .{ .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "axis", mlir.integerAttribute(ctx, .i32, axis)), + .named(ctx, "axis", .int(ctx, .i32, axis)), }, .location = location, }); @@ -349,7 +349,7 @@ pub fn broadcast( result_type: *const mlir.Type, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.broadcast", .{ + return .make(ctx, "tt.broadcast", .{ .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{result_type} }, .location = location, @@ -366,12 +366,12 @@ pub fn reshape( ) *mlir.Operation { var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; if (allow_reorder) { - attrs.appendAssumeCapacity(.named(ctx, "allow_reorder", mlir.unitAttribute(ctx))); + attrs.appendAssumeCapacity(.named(ctx, "allow_reorder", .unit(ctx))); } if (efficient_layout) { - attrs.appendAssumeCapacity(.named(ctx, "efficient_layout", mlir.unitAttribute(ctx))); + attrs.appendAssumeCapacity(.named(ctx, "efficient_layout", .unit(ctx))); } - return mlir.Operation.make(ctx, "tt.reshape", .{ + return .make(ctx, "tt.reshape", .{ .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{result_type} }, .attributes = attrs.constSlice(), @@ -386,11 +386,11 @@ pub fn trans( result_type: *const mlir.Type, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.trans", .{ + return .make(ctx, "tt.trans", .{ .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "order", mlir.denseArrayAttribute(ctx, .i32, order)), + .named(ctx, "order", .denseArray(ctx, .i32, order)), }, .location = location, }); @@ -407,7 +407,7 @@ pub fn addptr( offset: *const mlir.Value, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.addptr", .{ + return .make(ctx, "tt.addptr", .{ .operands = .{ .flat = &.{ ptr_val, offset } }, .results = .{ .flat = &.{ptr_val.type_()} }, .location = location, @@ -439,14 +439,14 @@ pub fn load( } else 0; const seg_sizes = [3]i32{ 1, mask_len, other_len }; - return mlir.Operation.make(ctx, "tt.load", .{ + return .make(ctx, "tt.load", .{ .operands = .{ .flat = operands_buf.constSlice() }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), .named(ctx, "cache", cacheModifier(ctx, cache)), .named(ctx, "evict", evictionPolicy(ctx, evict)), - .named(ctx, "isVolatile", mlir.boolAttribute(ctx, is_volatile)), + .named(ctx, "isVolatile", .boolean(ctx, is_volatile)), }, .location = location, }); @@ -465,7 +465,7 @@ pub fn store( var buf: stdx.BoundedArray(*const mlir.Value, 3) = .empty; buf.appendSliceAssumeCapacity(&.{ ptr_val, value }); if (mask) |m| buf.appendAssumeCapacity(m); - return mlir.Operation.make(ctx, "tt.store", .{ + return .make(ctx, "tt.store", .{ .operands = .{ .flat = buf.constSlice() }, .attributes = &.{ .named(ctx, "cache", cacheModifier(ctx, cache)), @@ -489,12 +489,12 @@ pub fn dot( max_num_imprecise_acc: i32, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.dot", .{ + return .make(ctx, "tt.dot", .{ .operands = .{ .flat = &.{ a, b, c_acc } }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ .named(ctx, "inputPrecision", inputPrecision(ctx, input_precision_)), - .named(ctx, "maxNumImpreciseAcc", mlir.integerAttribute(ctx, .i32, max_num_imprecise_acc)), + .named(ctx, "maxNumImpreciseAcc", .int(ctx, .i32, max_num_imprecise_acc)), }, .location = location, }); @@ -511,12 +511,12 @@ pub fn reduce( location: *const mlir.Location, ) *mlir.Operation { // Single variadic operand group — flat list, no operandSegmentSizes needed. - return mlir.Operation.make(ctx, "tt.reduce", .{ + return .make(ctx, "tt.reduce", .{ .operands = .{ .flat = srcs }, .results = .{ .flat = result_types }, .blocks = &.{combine_block}, .attributes = &.{ - .named(ctx, "axis", mlir.integerAttribute(ctx, .i32, axis)), + .named(ctx, "axis", .int(ctx, .i32, axis)), }, .location = location, }); @@ -527,7 +527,7 @@ pub fn reduce_return( values: []const *const mlir.Value, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.reduce.return", .{ + return .make(ctx, "tt.reduce.return", .{ .operands = .{ .flat = values }, .verify = false, .location = location, @@ -545,7 +545,7 @@ pub fn int_to_ptr( result_type: *const mlir.Type, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.int_to_ptr", .{ + return .make(ctx, "tt.int_to_ptr", .{ .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{result_type} }, .location = location, @@ -559,7 +559,7 @@ pub fn ptr_to_int( result_type: *const mlir.Type, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.ptr_to_int", .{ + return .make(ctx, "tt.ptr_to_int", .{ .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{result_type} }, .location = location, @@ -574,7 +574,7 @@ pub fn bitcast( result_type: *const mlir.Type, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.bitcast", .{ + return .make(ctx, "tt.bitcast", .{ .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{result_type} }, .location = location, @@ -594,7 +594,7 @@ pub fn fp_to_fp( if (rounding) |rm| { attrs.appendAssumeCapacity(.named(ctx, "rounding", roundingMode(ctx, rm))); } - return mlir.Operation.make(ctx, "tt.fp_to_fp", .{ + return .make(ctx, "tt.fp_to_fp", .{ .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{result_type} }, .attributes = attrs.constSlice(), @@ -615,7 +615,7 @@ pub fn clampf( propagate_nan: PropagateNan, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.clampf", .{ + return .make(ctx, "tt.clampf", .{ .operands = .{ .flat = &.{ x, min, max } }, .results = .{ .flat = &.{x.type_()} }, .attributes = &.{ @@ -631,7 +631,7 @@ pub fn precise_sqrt( x: *const mlir.Value, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.precise_sqrt", .{ + return .make(ctx, "tt.precise_sqrt", .{ .operands = .{ .flat = &.{x} }, .results = .{ .flat = &.{x.type_()} }, .location = location, @@ -645,7 +645,7 @@ pub fn precise_divf( y: *const mlir.Value, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.precise_divf", .{ + return .make(ctx, "tt.precise_divf", .{ .operands = .{ .flat = &.{ x, y } }, .results = .{ .flat = &.{x.type_()} }, .location = location, @@ -659,7 +659,7 @@ pub fn mulhiui( y: *const mlir.Value, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.mulhiui", .{ + return .make(ctx, "tt.mulhiui", .{ .operands = .{ .flat = &.{ x, y } }, .results = .{ .flat = &.{x.type_()} }, .location = location, @@ -677,7 +677,7 @@ pub fn unsplat( result_type: *const mlir.Type, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.unsplat", .{ + return .make(ctx, "tt.unsplat", .{ .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{result_type} }, .location = location, @@ -692,7 +692,7 @@ pub fn cat( result_type: *const mlir.Type, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.cat", .{ + return .make(ctx, "tt.cat", .{ .operands = .{ .flat = &.{ lhs, rhs } }, .results = .{ .flat = &.{result_type} }, .location = location, @@ -707,7 +707,7 @@ pub fn join( result_type: *const mlir.Type, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.join", .{ + return .make(ctx, "tt.join", .{ .operands = .{ .flat = &.{ lhs, rhs } }, .results = .{ .flat = &.{result_type} }, .location = location, @@ -721,7 +721,7 @@ pub fn split( result_type: *const mlir.Type, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.split", .{ + return .make(ctx, "tt.split", .{ .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{ result_type, result_type } }, .location = location, @@ -744,11 +744,11 @@ pub fn gather( location: *const mlir.Location, ) *mlir.Operation { var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attrs.appendAssumeCapacity(.named(ctx, "axis", mlir.integerAttribute(ctx, .i32, axis))); + attrs.appendAssumeCapacity(.named(ctx, "axis", .int(ctx, .i32, axis))); if (efficient_layout) { - attrs.appendAssumeCapacity(.named(ctx, "efficient_layout", mlir.unitAttribute(ctx))); + attrs.appendAssumeCapacity(.named(ctx, "efficient_layout", .unit(ctx))); } - return mlir.Operation.make(ctx, "tt.gather", .{ + return .make(ctx, "tt.gather", .{ .operands = .{ .flat = &.{ src, indices } }, .results = .{ .flat = &.{result_type} }, .attributes = attrs.constSlice(), @@ -767,7 +767,7 @@ pub fn histogram( var buf: stdx.BoundedArray(*const mlir.Value, 2) = .empty; buf.appendAssumeCapacity(src); if (mask) |m| buf.appendAssumeCapacity(m); - return mlir.Operation.make(ctx, "tt.histogram", .{ + return .make(ctx, "tt.histogram", .{ .operands = .{ .flat = buf.constSlice() }, .results = .{ .flat = &.{result_type} }, .location = location, @@ -789,13 +789,13 @@ pub fn scan( result_types: []const *const mlir.Type, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.scan", .{ + return .make(ctx, "tt.scan", .{ .operands = .{ .variadic = &.{srcs} }, .results = .{ .flat = result_types }, .blocks = &.{combine_block}, .attributes = &.{ - .named(ctx, "axis", mlir.integerAttribute(ctx, .i32, axis)), - .named(ctx, "reverse", mlir.boolAttribute(ctx, reverse)), + .named(ctx, "axis", .int(ctx, .i32, axis)), + .named(ctx, "reverse", .boolean(ctx, reverse)), }, .location = location, }); @@ -806,7 +806,7 @@ pub fn scan_return( values: []const *const mlir.Value, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.scan.return", .{ + return .make(ctx, "tt.scan.return", .{ .operands = .{ .flat = values }, .verify = false, .location = location, @@ -832,7 +832,7 @@ pub fn atomic_rmw( operands_buf.appendSliceAssumeCapacity(&.{ ptr_val, val }); if (mask) |m| operands_buf.appendAssumeCapacity(m); - return mlir.Operation.make(ctx, "tt.atomic_rmw", .{ + return .make(ctx, "tt.atomic_rmw", .{ .operands = .{ .flat = operands_buf.constSlice() }, .results = .{ .flat = &.{val.type_()} }, .attributes = &.{ @@ -854,7 +854,7 @@ pub fn atomic_cas( scope: MemSyncScope, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.atomic_cas", .{ + return .make(ctx, "tt.atomic_cas", .{ .operands = .{ .flat = &.{ ptr_val, cmp, val } }, .results = .{ .flat = &.{val.type_()} }, .attributes = &.{ @@ -876,10 +876,10 @@ pub fn assert_( message: []const u8, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.assert", .{ + return .make(ctx, "tt.assert", .{ .operands = .{ .flat = &.{condition} }, .attributes = &.{ - .named(ctx, "message", mlir.stringAttribute(ctx, message)), + .named(ctx, "message", .string(ctx, message)), }, .location = location, }); @@ -901,14 +901,14 @@ pub fn call( location: *const mlir.Location, ) *mlir.Operation { var attrs: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attrs.appendAssumeCapacity(.named(ctx, "callee", mlir.flatSymbolRefAttribute(ctx, callee))); + attrs.appendAssumeCapacity(.named(ctx, "callee", .flatSymbolRef(ctx, callee))); if (arg_attrs) |aa| { - attrs.appendAssumeCapacity(.named(ctx, "arg_attrs", mlir.arrayAttribute(ctx, aa))); + attrs.appendAssumeCapacity(.named(ctx, "arg_attrs", .array(ctx, aa))); } if (res_attrs) |ra| { - attrs.appendAssumeCapacity(.named(ctx, "res_attrs", mlir.arrayAttribute(ctx, ra))); + attrs.appendAssumeCapacity(.named(ctx, "res_attrs", .array(ctx, ra))); } - return mlir.Operation.make(ctx, "tt.call", .{ + return .make(ctx, "tt.call", .{ .operands = .{ .flat = operands }, .results = .{ .flat = result_types }, .attributes = attrs.constSlice(), @@ -949,16 +949,16 @@ pub fn dot_scaled( } else 0; const seg_sizes = [5]i32{ 1, 1, 1, a_scale_len, b_scale_len }; - return mlir.Operation.make(ctx, "tt.dot_scaled", .{ + return .make(ctx, "tt.dot_scaled", .{ .operands = .{ .flat = buf.constSlice() }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), .named(ctx, "a_elem_type", scaleDotElemType(ctx, a_elem_type)), .named(ctx, "b_elem_type", scaleDotElemType(ctx, b_elem_type)), - .named(ctx, "fastMath", mlir.boolAttribute(ctx, fast_math)), - .named(ctx, "lhs_k_pack", mlir.boolAttribute(ctx, lhs_k_pack)), - .named(ctx, "rhs_k_pack", mlir.boolAttribute(ctx, rhs_k_pack)), + .named(ctx, "fastMath", .boolean(ctx, fast_math)), + .named(ctx, "lhs_k_pack", .boolean(ctx, lhs_k_pack)), + .named(ctx, "rhs_k_pack", .boolean(ctx, rhs_k_pack)), }, .location = location, }); @@ -978,14 +978,14 @@ pub fn extern_elementwise( pure: bool, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.extern_elementwise", .{ + return .make(ctx, "tt.extern_elementwise", .{ .operands = .{ .flat = srcs }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "libname", mlir.stringAttribute(ctx, libname)), - .named(ctx, "libpath", mlir.stringAttribute(ctx, libpath)), - .named(ctx, "symbol", mlir.stringAttribute(ctx, symbol)), - .named(ctx, "pure", mlir.boolAttribute(ctx, pure)), + .named(ctx, "libname", .string(ctx, libname)), + .named(ctx, "libpath", .string(ctx, libpath)), + .named(ctx, "symbol", .string(ctx, symbol)), + .named(ctx, "pure", .boolean(ctx, pure)), }, .location = location, }); @@ -1007,12 +1007,12 @@ pub fn print( location: *const mlir.Location, ) *mlir.Operation { std.debug.assert(args.len == is_signed.len); - return mlir.Operation.make(ctx, "tt.print", .{ + return .make(ctx, "tt.print", .{ .operands = .{ .flat = args }, .attributes = &.{ - .named(ctx, "prefix", mlir.stringAttribute(ctx, prefix)), - .named(ctx, "hex", mlir.boolAttribute(ctx, hex)), - .named(ctx, "isSigned", mlir.denseArrayAttribute(ctx, .i32, is_signed)), + .named(ctx, "prefix", .string(ctx, prefix)), + .named(ctx, "hex", .boolean(ctx, hex)), + .named(ctx, "isSigned", .denseArray(ctx, .i32, is_signed)), }, .location = location, }); @@ -1038,7 +1038,7 @@ pub fn make_tensor_descriptor( buf.appendSliceAssumeCapacity(shape); buf.appendSliceAssumeCapacity(strides); // SameVariadicOperandSize — flatten all operands. - return mlir.Operation.make(ctx, "tt.make_tensor_descriptor", .{ + return .make(ctx, "tt.make_tensor_descriptor", .{ .operands = .{ .flat = buf.constSlice() }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ @@ -1060,7 +1060,7 @@ pub fn descriptor_load( var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; buf.appendAssumeCapacity(desc); buf.appendSliceAssumeCapacity(indices); - return mlir.Operation.make(ctx, "tt.descriptor_load", .{ + return .make(ctx, "tt.descriptor_load", .{ .operands = .{ .flat = buf.constSlice() }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ @@ -1081,7 +1081,7 @@ pub fn descriptor_store( var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; buf.appendSliceAssumeCapacity(&.{ desc, src }); buf.appendSliceAssumeCapacity(indices); - return mlir.Operation.make(ctx, "tt.descriptor_store", .{ + return .make(ctx, "tt.descriptor_store", .{ .operands = .{ .flat = buf.constSlice() }, .location = location, }); @@ -1098,7 +1098,7 @@ pub fn descriptor_reduce( var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; buf.appendSliceAssumeCapacity(&.{ desc, src }); buf.appendSliceAssumeCapacity(indices); - return mlir.Operation.make(ctx, "tt.descriptor_reduce", .{ + return .make(ctx, "tt.descriptor_reduce", .{ .operands = .{ .flat = buf.constSlice() }, .attributes = &.{ .named(ctx, "kind", descriptorReduceKind(ctx, kind)), @@ -1115,7 +1115,7 @@ pub fn descriptor_gather( result_type: *const mlir.Type, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.descriptor_gather", .{ + return .make(ctx, "tt.descriptor_gather", .{ .operands = .{ .flat = &.{ desc, x_offsets, y_offset } }, .results = .{ .flat = &.{result_type} }, .location = location, @@ -1130,7 +1130,7 @@ pub fn descriptor_scatter( src: *const mlir.Value, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.descriptor_scatter", .{ + return .make(ctx, "tt.descriptor_scatter", .{ .operands = .{ .flat = &.{ desc, x_offsets, y_offset, src } }, .location = location, }); @@ -1151,12 +1151,12 @@ pub fn map_elementwise( result_types: []const *const mlir.Type, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.map_elementwise", .{ + return .make(ctx, "tt.map_elementwise", .{ .operands = .{ .flat = srcs }, .results = .{ .flat = result_types }, .blocks = &.{scalar_block}, .attributes = &.{ - .named(ctx, "pack", mlir.integerAttribute(ctx, .i32, pack)), + .named(ctx, "pack", .int(ctx, .i32, pack)), }, .location = location, }); @@ -1167,7 +1167,7 @@ pub fn map_elementwise_return( values: []const *const mlir.Value, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.map_elementwise.return", .{ + return .make(ctx, "tt.map_elementwise.return", .{ .operands = .{ .flat = values }, .verify = false, .location = location, @@ -1190,14 +1190,14 @@ pub fn elementwise_inline_asm( packed_element: i32, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "tt.elementwise_inline_asm", .{ + return .make(ctx, "tt.elementwise_inline_asm", .{ .operands = .{ .flat = args }, .results = .{ .flat = result_types }, .attributes = &.{ - .named(ctx, "asm_string", mlir.stringAttribute(ctx, asm_string)), - .named(ctx, "constraints", mlir.stringAttribute(ctx, constraints)), - .named(ctx, "pure", mlir.boolAttribute(ctx, pure)), - .named(ctx, "packed_element", mlir.integerAttribute(ctx, .i32, packed_element)), + .named(ctx, "asm_string", .string(ctx, asm_string)), + .named(ctx, "constraints", .string(ctx, constraints)), + .named(ctx, "pure", .boolean(ctx, pure)), + .named(ctx, "packed_element", .int(ctx, .i32, packed_element)), }, .location = location, }); @@ -1235,12 +1235,10 @@ test "tt.func round-trip" { const module: *mlir.Module = .init(loc); defer module.deinit(); - const f32_ty = mlir.floatType(ctx, .f32); + const f32_ty = mlir.Type.float(ctx, .f32); const ptr_ty = pointerType(f32_ty, 1); - const arg_types: []const *const mlir.Type = &.{ ptr_ty, ptr_ty }; - const arg_locs: []const *const mlir.Location = &.{ loc, loc }; - const entry = mlir.Block.init(arg_types, arg_locs); + const entry = mlir.Block.init(&.{ ptr_ty, ptr_ty }, &.{ loc, loc }); const arg0 = entry.argument(0); const arg1 = entry.argument(1); @@ -1250,7 +1248,7 @@ test "tt.func round-trip" { _ = loaded.appendTo(entry); // %1 = arith.constant 1.0 : f32 — built with Operation.make to avoid cross-dialect import - const one_attr: *const mlir.Attribute = mlir.floatAttribute(ctx, .f32, 1.0); + const one_attr: *const mlir.Attribute = .float(ctx, .f32, 1.0); const one = mlir.Operation.make(ctx, "arith.constant", .{ .results = .{ .flat = &.{f32_ty} }, .attributes = &.{.named(ctx, "value", one_attr)}, diff --git a/mlir/dialects/vector.zig b/mlir/dialects/vector.zig index ab1dfa19..34ee5d78 100644 --- a/mlir/dialects/vector.zig +++ b/mlir/dialects/vector.zig @@ -72,7 +72,7 @@ pub fn extract( .operands = .{ .flat = operands_buf.constSlice() }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "static_position", mlir.denseArrayAttribute(ctx, .i64, static_position)), + .named(ctx, "static_position", .denseArray(ctx, .i64, static_position)), }, .location = location, }); @@ -96,7 +96,7 @@ pub fn insert( .operands = .{ .flat = operands_buf.constSlice() }, .results = .{ .flat = &.{dest.type_()} }, .attributes = &.{ - .named(ctx, "static_position", mlir.denseArrayAttribute(ctx, .i64, static_position)), + .named(ctx, "static_position", .denseArray(ctx, .i64, static_position)), }, .location = location, }); @@ -151,9 +151,9 @@ pub fn extract_strided_slice( .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "offsets", mlir.denseArrayAttribute(ctx, .i64, args.offsets)), - .named(ctx, "sizes", mlir.denseArrayAttribute(ctx, .i64, args.sizes)), - .named(ctx, "strides", mlir.denseArrayAttribute(ctx, .i64, args.strides)), + .named(ctx, "offsets", .denseArray(ctx, .i64, args.offsets)), + .named(ctx, "sizes", .denseArray(ctx, .i64, args.sizes)), + .named(ctx, "strides", .denseArray(ctx, .i64, args.strides)), }, .location = location, }); @@ -171,8 +171,8 @@ pub fn insert_strided_slice( .operands = .{ .flat = &.{ src, dest } }, .results = .{ .flat = &.{dest.type_()} }, .attributes = &.{ - .named(ctx, "offsets", mlir.denseArrayAttribute(ctx, .i64, offsets)), - .named(ctx, "strides", mlir.denseArrayAttribute(ctx, .i64, strides)), + .named(ctx, "offsets", .denseArray(ctx, .i64, offsets)), + .named(ctx, "strides", .denseArray(ctx, .i64, strides)), }, .location = location, }); @@ -251,7 +251,7 @@ pub fn transpose( .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "permutation", mlir.denseArrayAttribute(ctx, .i64, permutation)), + .named(ctx, "permutation", .denseArray(ctx, .i64, permutation)), }, .location = location, }); @@ -275,7 +275,7 @@ pub fn shuffle( .operands = .{ .flat = &.{ lhs, rhs } }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "mask", mlir.denseArrayAttribute(ctx, .i64, mask)), + .named(ctx, "mask", .denseArray(ctx, .i64, mask)), }, .location = location, }); @@ -344,7 +344,7 @@ pub fn step( /// vector.vscale — runtime scalable-vector size factor (index-typed). pub fn vscale(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "vector.vscale", .{ - .results = .{ .flat = &.{mlir.indexType(ctx)} }, + .results = .{ .flat = &.{.index(ctx)} }, .location = location, }); } @@ -406,7 +406,7 @@ pub fn multi_reduction( .results = .{ .flat = &.{result_type} }, .attributes = &.{ .named(ctx, "kind", kind.attribute(ctx)), - .named(ctx, "reduction_dims", mlir.denseArrayAttribute(ctx, .i64, reduction_dims)), + .named(ctx, "reduction_dims", .denseArray(ctx, .i64, reduction_dims)), }, .location = location, }); @@ -437,8 +437,8 @@ pub fn contract( .operands = .{ .flat = &.{ lhs, rhs, acc } }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "indexing_maps", mlir.arrayAttribute(ctx, indexing_maps)), - .named(ctx, "iterator_types", mlir.arrayAttribute(ctx, iter_attrs.constSlice())), + .named(ctx, "indexing_maps", .array(ctx, indexing_maps)), + .named(ctx, "iterator_types", .array(ctx, iter_attrs.constSlice())), .named(ctx, "kind", opts.kind.attribute(ctx)), }, .location = location, @@ -456,10 +456,10 @@ pub const LoadStoreOpts = struct { fn loadStoreAttrs(ctx: *mlir.Context, opts: LoadStoreOpts, dst: *stdx.BoundedArray(mlir.NamedAttribute, 2)) void { if (opts.nontemporal) { - dst.appendAssumeCapacity(.named(ctx, "nontemporal", mlir.boolAttribute(ctx, true))); + dst.appendAssumeCapacity(.named(ctx, "nontemporal", .boolean(ctx, true))); } if (opts.alignment) |a| { - dst.appendAssumeCapacity(.named(ctx, "alignment", mlir.integerAttribute(ctx, .i64, a))); + dst.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, a))); } } @@ -643,7 +643,7 @@ pub fn compress_store( pub const TransferReadArgs = struct { /// `affine_map<(d0,...) -> (...)>` attribute. Build with - /// `mlir.parseAffineMapAttribute` or `mlir.affineMapAttribute(...)`. + /// `mlir.parseAffineMapAttribute` or `mlir.Attribute.affineMap(...)`. permutation_map: *const mlir.Attribute, /// One bool per result-vector dim. Length must equal vector rank. in_bounds: []const bool, @@ -674,15 +674,15 @@ pub fn transfer_read( const seg_sizes = [4]i32{ 1, @intCast(indices.len), 1, mask_len }; var in_bounds_attrs: stdx.BoundedArray(*const mlir.Attribute, 16) = .empty; - for (args.in_bounds) |b| in_bounds_attrs.appendAssumeCapacity(mlir.boolAttribute(ctx, b)); + for (args.in_bounds) |b| in_bounds_attrs.appendAssumeCapacity(.boolean(ctx, b)); return mlir.Operation.make(ctx, "vector.transfer_read", .{ .operands = .{ .flat = operands_buf.constSlice() }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), .named(ctx, "permutation_map", args.permutation_map), - .named(ctx, "in_bounds", mlir.arrayAttribute(ctx, in_bounds_attrs.constSlice())), + .named(ctx, "in_bounds", .array(ctx, in_bounds_attrs.constSlice())), }, .location = location, }); @@ -718,7 +718,7 @@ pub fn transfer_write( const seg_sizes = [4]i32{ 1, 1, @intCast(indices.len), mask_len }; var in_bounds_attrs: stdx.BoundedArray(*const mlir.Attribute, 16) = .empty; - for (args.in_bounds) |b| in_bounds_attrs.appendAssumeCapacity(mlir.boolAttribute(ctx, b)); + for (args.in_bounds) |b| in_bounds_attrs.appendAssumeCapacity(.boolean(ctx, b)); var result_types: stdx.BoundedArray(*const mlir.Type, 1) = .empty; if (args.result_type) |rt| result_types.appendAssumeCapacity(rt); @@ -727,9 +727,9 @@ pub fn transfer_write( .operands = .{ .flat = operands_buf.constSlice() }, .results = .{ .flat = result_types.constSlice() }, .attributes = &.{ - .named(ctx, "operandSegmentSizes", mlir.denseArrayAttribute(ctx, .i32, &seg_sizes)), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), .named(ctx, "permutation_map", args.permutation_map), - .named(ctx, "in_bounds", mlir.arrayAttribute(ctx, in_bounds_attrs.constSlice())), + .named(ctx, "in_bounds", .array(ctx, in_bounds_attrs.constSlice())), }, .location = location, }); @@ -750,7 +750,7 @@ pub fn constant_mask( return mlir.Operation.make(ctx, "vector.constant_mask", .{ .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "mask_dim_sizes", mlir.denseArrayAttribute(ctx, .i64, mask_dim_sizes)), + .named(ctx, "mask_dim_sizes", .denseArray(ctx, .i64, mask_dim_sizes)), }, .location = location, }); diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 8aa8a5b0..86d22b11 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -341,6 +341,52 @@ pub const Type = opaque { pub fn parse(ctx: *Context, str: []const u8) Error!*const Type { return @ptrCast(c.mlirTypeParseGet(ctx.ptr(), stringRef(str)).ptr orelse return Error.InvalidMlir); } + + pub fn index(ctx: *Context) *const Type { + return @ptrCast(IndexType.get(ctx)); + } + + pub fn int(ctx: *Context, it: IntegerTypes) *const Type { + return @ptrCast(IntegerType.get(ctx, it)); + } + + pub fn float(ctx: *Context, ft: FloatTypes) *const Type { + return switch (ft) { + inline else => |v| @ptrCast(FloatType(v).get(ctx)), + }; + } + + pub fn complex(ctx: *Context, complex_type: ComplexTypes) *const Type { + return @ptrCast(ComplexType.get(ctx, complex_type)); + } + + pub fn rankedTensor(dimensions: []const i64, elem_type: *const Type) *const Type { + return @ptrCast(RankedTensorType.get(dimensions, elem_type, null)); + } + + pub fn rankedTensorWithEncoding(dimensions: []const i64, elem_type: *const Type, encoding: *Attribute) *const Type { + return @ptrCast(RankedTensorType.get(dimensions, elem_type, encoding)); + } + + pub fn function(ctx: *Context, args: []const *const Type, results: []const *const Type) *const Type { + return @ptrCast(FunctionType.init(ctx, args, results)); + } + + pub fn memRef(element_type: *const Type, shape: []const i64, layout: ?*const Attribute, memory_space: ?*const Attribute) *const Type { + return @ptrCast(MemRefType.init(element_type, shape, layout, memory_space)); + } + + pub fn element(self: *const VectorType) *const Type { + return @ptrCast(c.mlirShapedTypeGetElementType(self.ptr()).ptr); + } + + pub fn vector(shape: []const i64, element_type: *const Type) *const Type { + return @ptrCast(VectorType.init(shape, element_type)); + } + + pub fn unrankedMemRef(element_type: *const Type, memory_space: ?*const Attribute) *const Type { + return @ptrCast(UnrankedMemRefType.init(element_type, memory_space)); + } }; pub const IndexType = opaque { @@ -356,10 +402,6 @@ pub const IndexType = opaque { } }; -pub fn indexType(ctx: *Context) *const Type { - return @ptrCast(IndexType.get(ctx)); -} - pub const IntegerTypes = enum { i1, i2, @@ -434,10 +476,6 @@ pub const IntegerType = opaque { } }; -pub fn integerType(ctx: *Context, it: IntegerTypes) *const Type { - return @ptrCast(IntegerType.get(ctx, it)); -} - pub const FloatTypes = enum { f4e2m1fn, f6e2m3fn, @@ -494,12 +532,6 @@ pub fn FloatType(comptime ft: FloatTypes) type { }; } -pub fn floatType(ctx: *Context, ft: FloatTypes) *const Type { - return switch (ft) { - inline else => |v| @ptrCast(FloatType(v).get(ctx)), - }; -} - pub const ComplexTypes = enum { c64, c128, @@ -515,16 +547,12 @@ pub const ComplexType = opaque { pub fn get(ctx: *Context, complex_type: ComplexTypes) *const ComplexType { return @ptrCast(c.mlirComplexTypeGet(switch (complex_type) { - .c64 => floatType(ctx, .f32).ptr(), - .c128 => floatType(ctx, .f64).ptr(), + .c64 => Type.float(ctx, .f32).ptr(), + .c128 => Type.float(ctx, .f64).ptr(), }).ptr); } }; -pub fn complexType(ctx: *Context, complex_type: ComplexTypes) *const Type { - return @ptrCast(ComplexType.get(ctx, complex_type)); -} - pub const Attribute = opaque { const M = Methods(Attribute, c.MlirAttribute); @@ -544,6 +572,60 @@ pub const Attribute = opaque { pub fn dialect(self: *const Context) *const Dialect { return @ptrCast(c.mlirAttributeGetDialect(self.ptr()).ptr); } + + pub fn string(ctx: *Context, str: []const u8) *const Attribute { + return @ptrCast(StringAttribute.init(ctx, str)); + } + + pub fn int(ctx: *Context, it: IntegerTypes, value_: anytype) *const Attribute { + return @ptrCast(IntegerAttribute.init(ctx, it, value_)); + } + + pub fn intFromType(it: *const IntegerType, value_: i64) *const Attribute { + return @ptrCast(IntegerAttribute.fromType(it, value_)); + } + + pub fn float(ctx: *Context, comptime ft: FloatTypes, value_: anytype) *const Attribute { + return @ptrCast(FloatAttribute.init(ctx, ft, value_)); + } + pub fn boolean(ctx: *Context, value: bool) *const Attribute { + return @ptrCast(BoolAttribute.init(ctx, value)); + } + + pub fn typeAttr(type_: *const Type) *const Attribute { + return @ptrCast(TypeAttribute.init(type_)); + } + + pub fn dict(ctx: *Context, attributes: []const NamedAttribute) *const Attribute { + return @ptrCast(DictionaryAttribute.init(ctx, attributes)); + } + + pub fn denseElements(shaped_type: *const ShapedType, values: anytype) *const Attribute { + return @ptrCast(DenseElementsAttribute.init(shaped_type, values)); + } + + pub fn denseArray(ctx: *Context, comptime ElementType: DenseArrayTypes, values: []const ElementType.ZigType()) *const Attribute { + return @ptrCast(DenseArrayAttribute(ElementType).init(ctx, @ptrCast(values))); + } + pub fn array(ctx: *Context, attrs: []const *const Attribute) *const Attribute { + return @ptrCast(ArrayAttribute.init(ctx, attrs)); + } + + pub fn flatSymbolRef(ctx: *Context, symbol: []const u8) *const Attribute { + return @ptrCast(FlatSymbolRefAttribute.init(ctx, symbol)); + } + + pub fn unit(ctx: *Context) *const Attribute { + return @ptrCast(UnitAttribute.get(ctx)); + } + + pub fn affineMap(map: *const AffineMap) *const Attribute { + return @ptrCast(AffineMapAttribute.init(map)); + } + + pub fn integerSet(set: *const IntegerSet) *const Attribute { + return @ptrCast(IntegerSetAttribute.init(set)); + } }; pub const StringAttribute = opaque { @@ -563,10 +645,6 @@ pub const StringAttribute = opaque { } }; -pub fn stringAttribute(ctx: *Context, str: []const u8) *const Attribute { - return @ptrCast(StringAttribute.init(ctx, str)); -} - pub const IntegerAttribute = opaque { const M = Methods(IntegerAttribute, c.MlirAttribute); @@ -576,10 +654,11 @@ pub const IntegerAttribute = opaque { pub const format = M.format(c.mlirAttributePrint); pub fn init(ctx: *Context, it: IntegerTypes, value_: anytype) *const IntegerAttribute { - return @ptrCast(c.mlirIntegerAttrGet( - IntegerType.get(ctx, it).ptr(), - @intCast(value_), - ).ptr); + return .fromType(.get(ctx, it), @intCast(value_)); + } + + pub fn fromType(it: *const IntegerType, value_: i64) *const IntegerAttribute { + return @ptrCast(c.mlirIntegerAttrGet(it.ptr(), value_).ptr); } pub fn value(self: *const IntegerAttribute, comptime T: type) T { @@ -591,10 +670,6 @@ pub const IntegerAttribute = opaque { } }; -pub fn integerAttribute(ctx: *Context, it: IntegerTypes, value_: anytype) *const Attribute { - return @ptrCast(IntegerAttribute.init(ctx, it, value_)); -} - pub const FloatAttribute = opaque { const M = Methods(FloatAttribute, c.MlirAttribute); @@ -612,10 +687,6 @@ pub const FloatAttribute = opaque { } }; -pub fn floatAttribute(ctx: *Context, comptime ft: FloatTypes, value_: anytype) *const Attribute { - return @ptrCast(FloatAttribute.init(ctx, ft, value_)); -} - pub const BoolAttribute = opaque { const M = Methods(BoolAttribute, c.MlirAttribute); @@ -633,10 +704,6 @@ pub const BoolAttribute = opaque { } }; -pub fn boolAttribute(ctx: *Context, value: bool) *const Attribute { - return @ptrCast(BoolAttribute.init(ctx, value)); -} - pub const TypeAttribute = opaque { const M = Methods(TypeAttribute, c.MlirAttribute); @@ -654,10 +721,6 @@ pub const TypeAttribute = opaque { } }; -pub fn typeAttribute(type_: *const Type) *const Attribute { - return @ptrCast(TypeAttribute.init(type_)); -} - pub const Dialect = struct { const M = Methods(Dialect, c.MlirDialect); @@ -907,10 +970,6 @@ pub const DictionaryAttribute = opaque { } }; -pub fn dictionaryAttribute(ctx: *Context, attributes: []const NamedAttribute) *const Attribute { - return @ptrCast(DictionaryAttribute.init(ctx, attributes)); -} - pub const Value = opaque { const M = Methods(Value, c.MlirValue); @@ -1209,7 +1268,7 @@ pub const Operation = opaque { sizes.appendAssumeCapacity(@intCast(segment_operands.len)); } state.addAttributes(&.{ - .named(ctx, "operandSegmentSizes", denseElementsAttribute(RankedTensorType.get(&.{@intCast(sizes.len)}, integerType(ctx, .i32), null).shaped(), sizes.constSlice())), + .named(ctx, "operandSegmentSizes", .denseElements(RankedTensorType.get(&.{@intCast(sizes.len)}, .int(ctx, .i32), null).shaped(), sizes.constSlice())), }); }, }; @@ -1222,7 +1281,7 @@ pub const Operation = opaque { sizes.appendAssumeCapacity(@intCast(segment_results.len)); } state.addAttributes(&.{ - .named(ctx, "resultSegmentSizes", denseElementsAttribute(RankedTensorType.get(&.{@intCast(sizes.len)}, integerType(ctx, .i32), null).shaped(), sizes.constSlice())), + .named(ctx, "resultSegmentSizes", .denseElements(RankedTensorType.get(&.{@intCast(sizes.len)}, .int(ctx, .i32), null).shaped(), sizes.constSlice())), }); }, }; @@ -1407,6 +1466,14 @@ pub const ShapedType = opaque { pub fn dimension(self: *const ShapedType, dim: usize) i64 { return c.mlirShapedTypeGetDimSize(self.ptr(), @intCast(dim)); } + + pub fn rankedTensor(dimensions: []const i64, elem_type: *const Type) *const ShapedType { + return @ptrCast(RankedTensorType.get(dimensions, elem_type, null)); + } + + pub fn rankedTensorWithEncoding(dimensions: []const i64, elem_type: *const Type, encoding: *Attribute) *const ShapedType { + return @ptrCast(RankedTensorType.get(dimensions, elem_type, encoding)); + } }; pub const RankedTensorType = opaque { @@ -1417,11 +1484,11 @@ pub const RankedTensorType = opaque { pub const eql = M.eql(c.mlirTypeEqual); pub const format = M.format(c.mlirTypePrint); - pub fn get(dimensions: []const i64, elemType: *const Type, encoding: ?*Attribute) *const RankedTensorType { + pub fn get(dimensions: []const i64, elem_type: *const Type, encoding: ?*Attribute) *const RankedTensorType { return @ptrCast(c.mlirRankedTensorTypeGet( @intCast(dimensions.len), @ptrCast(dimensions), - elemType.ptr(), + elem_type.ptr(), if (encoding) |e| e.ptr() else c.mlirAttributeGetNull(), ).ptr); } @@ -1455,14 +1522,6 @@ pub const RankedTensorType = opaque { } }; -pub fn rankedTensorType(dimensions: []const i64, elem_type: *const Type) *const Type { - return @ptrCast(RankedTensorType.get(dimensions, elem_type, null)); -} - -pub fn rankedTensorTypeWithEncoding(dimensions: []const i64, elem_type: *const Type, encoding: *Attribute) *const Type { - return @ptrCast(RankedTensorType.get(dimensions, elem_type, encoding)); -} - pub const DenseElementsAttribute = opaque { const M = Methods(DenseElementsAttribute, c.MlirAttribute); @@ -1481,10 +1540,6 @@ pub const DenseElementsAttribute = opaque { } }; -pub fn denseElementsAttribute(shaped_type: *const ShapedType, values: anytype) *const Attribute { - return @ptrCast(DenseElementsAttribute.init(shaped_type, values)); -} - pub const DenseArrayTypes = enum { bool, i8, @@ -1544,10 +1599,6 @@ pub fn DenseArrayAttribute(comptime ElementType: DenseArrayTypes) type { }; } -pub fn denseArrayAttribute(ctx: *Context, comptime ElementType: DenseArrayTypes, values: []const ElementType.ZigType()) *const Attribute { - return @ptrCast(DenseArrayAttribute(ElementType).init(ctx, @ptrCast(values))); -} - pub const ArrayAttribute = opaque { const M = Methods(ArrayAttribute, c.MlirAttribute); @@ -1569,10 +1620,6 @@ pub const ArrayAttribute = opaque { } }; -pub fn arrayAttribute(ctx: *Context, attrs: []const *const Attribute) *const Attribute { - return @ptrCast(ArrayAttribute.init(ctx, attrs)); -} - pub const FunctionType = opaque { const M = Methods(FunctionType, c.MlirType); pub const isAFn = c.mlirTypeIsAFunction; @@ -1592,10 +1639,6 @@ pub const FunctionType = opaque { } }; -pub fn functionType(ctx: *Context, args: []const *const Type, results: []const *const Type) *const Type { - return @ptrCast(FunctionType.init(ctx, args, results)); -} - pub const FlatSymbolRefAttribute = opaque { const M = Methods(FlatSymbolRefAttribute, c.MlirAttribute); @@ -1613,10 +1656,6 @@ pub const FlatSymbolRefAttribute = opaque { } }; -pub fn flatSymbolRefAttribute(ctx: *Context, symbol: []const u8) *const Attribute { - return @ptrCast(FlatSymbolRefAttribute.init(ctx, symbol)); -} - pub const UnitAttribute = opaque { const M = Methods(UnitAttribute, c.MlirAttribute); @@ -1630,10 +1669,6 @@ pub const UnitAttribute = opaque { } }; -pub fn unitAttribute(ctx: *Context) *const Attribute { - return @ptrCast(UnitAttribute.get(ctx)); -} - pub const MemRefType = opaque { const M = Methods(MemRefType, c.MlirType); @@ -1692,10 +1727,6 @@ pub const MemRefType = opaque { } }; -pub fn memRefType(element_type: *const Type, shape: []const i64, layout: ?*const Attribute, memory_space: ?*const Attribute) *const Type { - return @ptrCast(MemRefType.init(element_type, shape, layout, memory_space)); -} - pub const VectorType = opaque { const M = Methods(VectorType, c.MlirType); @@ -1730,10 +1761,6 @@ pub const VectorType = opaque { ).ptr); } - pub fn elementType(self: *const VectorType) *const Type { - return @ptrCast(c.mlirShapedTypeGetElementType(self.ptr()).ptr); - } - pub fn rank(self: *const VectorType) usize { return @intCast(c.mlirShapedTypeGetRank(self.ptr())); } @@ -1747,10 +1774,6 @@ pub const VectorType = opaque { } }; -pub fn vectorType(shape: []const i64, element_type: *const Type) *const Type { - return @ptrCast(VectorType.init(shape, element_type)); -} - pub const UnrankedMemRefType = opaque { const M = Methods(UnrankedMemRefType, c.MlirType); @@ -1768,10 +1791,6 @@ pub const UnrankedMemRefType = opaque { } }; -pub fn unrankedMemRefType(element_type: *const Type, memory_space: ?*const Attribute) *const Type { - return @ptrCast(UnrankedMemRefType.init(element_type, memory_space)); -} - pub const AffineExpr = opaque { const M = Methods(AffineExpr, c.MlirAffineExpr); @@ -1885,14 +1904,6 @@ pub const AffineMapAttribute = opaque { } }; -pub fn affineMapAttribute(map: *const AffineMap) *const Attribute { - return @ptrCast(AffineMapAttribute.init(map)); -} - -pub fn parseAffineMapAttribute(ctx: *Context, src: []const u8) Error!*const Attribute { - return Attribute.parse(ctx, src); -} - pub const IntegerSet = opaque { const M = Methods(IntegerSet, c.MlirIntegerSet); @@ -1931,7 +1942,3 @@ pub const IntegerSetAttribute = opaque { return @ptrCast(c.mlirIntegerSetAttrGet(set.ptr()).ptr); } }; - -pub fn integerSetAttribute(set: *const IntegerSet) *const Attribute { - return @ptrCast(IntegerSetAttribute.init(set)); -} diff --git a/zml/Sharding.zig b/zml/Sharding.zig index 366d9e03..9d90eaae 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -1456,7 +1456,7 @@ pub const Data = struct { break; } } - if (!has_sharding) return mlir.stringAttribute(ctx, try allocator.dupe(u8, "{replicated}")); + if (!has_sharding) return .string(ctx, try allocator.dupe(u8, "{replicated}")); const mapping = self.getDimMapping(shape); var tile_shape: stdx.BoundedArray(i64, Shape.MAX_RANK + 1) = .empty; @@ -1509,7 +1509,7 @@ pub const Data = struct { try out.writer.writeAll("}"); - return mlir.stringAttribute(ctx, try out.toOwnedSlice()); + return .string(ctx, try out.toOwnedSlice()); } pub fn deviceAssignment(self: *const Data, allocator: std.mem.Allocator) ![]usize { diff --git a/zml/attention.zig b/zml/attention.zig index ddcbd3fa..74cd28e6 100644 --- a/zml/attention.zig +++ b/zml/attention.zig @@ -1,7 +1,12 @@ pub const attention = @import("attention/attention.zig"); pub const attnd = @import("attention/attnd.zig"); -pub const paged_attention = @import("attention/paged_attention.zig"); pub const flashattn = @import("attention/flashattn.zig"); +pub const paged_attention = @import("attention/paged_attention.zig"); pub const tpu = @import("attention/tpu_attention.zig"); pub const triton = @import("attention/triton_attention.zig"); pub const triton_kernels = @import("attention/triton_kernels/unified_attention.zig"); + +test { + const std = @import("std"); + std.testing.refAllDecls(@This()); +} diff --git a/zml/attention/mosaic_tpu_kernels/ragged_attention.zig b/zml/attention/mosaic_tpu_kernels/ragged_attention.zig index a02ad2f1..a402a7e5 100644 --- a/zml/attention/mosaic_tpu_kernels/ragged_attention.zig +++ b/zml/attention/mosaic_tpu_kernels/ragged_attention.zig @@ -1,6 +1,11 @@ -const zml = @import("../../zml.zig"); const ragged_paged = @import("platforms/tpu/ragged_paged"); +const zml = @import("../../zml.zig"); + +test { + _ = Kernel; +} + pub const Kernel = zml.kernel.mosaic_tpu.Kernel(ragged_paged.Cfg, .{ .name = "ragged_paged_attention_kernel", .inputs = &.{ "kv_lens", "page_indices", "cu_q_lens", "seq_buf_idx", "num_seqs", "q", "kv_pages" }, diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 9f4b6036..d36c2bca 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -248,16 +248,15 @@ test "Options.fromBackend configures mosaic_tpu ragged backend" { test "Backend.auto selects mosaic_tpu on TPU" { const platform: zml.Platform = .{ - .arena_state = .{}, + .arena = undefined, .target = .tpu, .pjrt_api = undefined, .pjrt_client = undefined, - .devices = &[_]zml.platform.Device{}, - .memories = &[_]zml.platform.Memory{}, + .devices = &.{}, + .memories = &.{}, .physical_mesh = undefined, .replicated_sharding = undefined, - .triton_runtime = null, - .tpu_ir_runtime = null, + .shardings = .empty, }; try std.testing.expectEqual(Backend.mosaic_tpu, Backend.auto(&platform)); diff --git a/zml/attention/tpu_attention.zig b/zml/attention/tpu_attention.zig index 2791819a..a1046c75 100644 --- a/zml/attention/tpu_attention.zig +++ b/zml/attention/tpu_attention.zig @@ -1,9 +1,8 @@ const std = @import("std"); const mlir = @import("mlir"); -const stdx = @import("stdx"); - const ragged_paged = @import("platforms/tpu/ragged_paged"); +const stdx = @import("stdx"); const CompilationContext = @import("../module.zig").CompilationContext; const zml = @import("../zml.zig"); @@ -12,6 +11,11 @@ const ragged_attention = @import("mosaic_tpu_kernels/ragged_attention.zig"); const log = std.log.scoped(.tpu_attention); +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(ragged_attention); +} + pub const mosaic_tpu = struct { const PreparedInputs = struct { q: zml.Tensor, @@ -77,8 +81,8 @@ pub const mosaic_tpu = struct { }; fn kernelFrontendAttrs(comp_ctx: *CompilationContext) *const mlir.Attribute { - return mlir.dictionaryAttribute(comp_ctx.mlir_ctx, &.{ - mlir.NamedAttribute.named(comp_ctx.mlir_ctx, "kernel_metadata", mlir.stringAttribute(comp_ctx.mlir_ctx, "{}")), + return .dict(comp_ctx.mlir_ctx, &.{ + mlir.NamedAttribute.named(comp_ctx.mlir_ctx, "kernel_metadata", .string(comp_ctx.mlir_ctx, "{}")), }); } @@ -95,8 +99,8 @@ pub const mosaic_tpu = struct { const comp_ctx = CompilationContext.current(); const additional_attrs = [_]mlir.NamedAttribute{ - mlir.NamedAttribute.named(comp_ctx.mlir_ctx, "kernel_name", mlir.stringAttribute(comp_ctx.mlir_ctx, "ragged_paged_attention_kernel")), - mlir.NamedAttribute.named(comp_ctx.mlir_ctx, "mhlo.frontend_attributes", kernelFrontendAttrs(comp_ctx)), + .named(comp_ctx.mlir_ctx, "kernel_name", .string(comp_ctx.mlir_ctx, "ragged_paged_attention_kernel")), + .named(comp_ctx.mlir_ctx, "mhlo.frontend_attributes", kernelFrontendAttrs(comp_ctx)), }; const out = ragged_attention.Kernel.call( @@ -126,10 +130,10 @@ pub const mosaic_tpu = struct { const end = query_start_len.slice1d(.b, .{ .start = 1 }); const query_lens = end.sub(start); return query_lens - .cmp(.GT, zml.Tensor.constant(query_lens.dtype().zero()).broad(query_lens.shape())) + .cmp(.GT, .zeroes(query_lens.shape())) .convert(.i32) .sum(.b) - .reshape(zml.Shape.init(.{1}, .i32)); + .reshape(.{1}); } fn cfgDtype(dtype: zml.DataType) @FieldType(ragged_paged.Cfg, "q_dtype") { diff --git a/zml/kernel.zig b/zml/kernel.zig index 23d95de8..89bbb924 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -1,18 +1,17 @@ const std = @import("std"); -const mlir = @import("mlir"); const dialects = @import("mlir/dialects"); - -const triton_builder = @import("kernels/triton/builder"); +const mlir = @import("mlir"); const mosaic_tpu_builder = @import("kernels/mosaic_tpu/builder"); const tpu_dialect = @import("mlir/dialects/mosaic_tpu"); +const triton_builder = @import("kernels/triton/builder"); -const DataType = @import("dtype.zig").DataType; -const Shape = @import("shape.zig").Shape; -const ops = @import("ops.zig"); -const Tensor = @import("tensor.zig").Tensor; const CompilationContext = @import("module.zig").CompilationContext; +const DataType = @import("dtype.zig").DataType; const mlirx = @import("mlirx.zig"); +const ops = @import("ops.zig"); +const Shape = @import("shape.zig").Shape; +const Tensor = @import("tensor.zig").Tensor; /// Build a struct type with one field per name, all of type `FieldT`. fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { @@ -216,9 +215,8 @@ pub const mosaic_tpu = struct { inline for (spec.inputs, 0..) |fname, i| values[i] = @field(inputs, fname).value(); var res_types: [spec.outputs.len]*const mlir.Type = undefined; - inline for (spec.outputs, 0..) |fname, i| { - const out = @field(outputs, fname); - res_types[i] = mlir.rankedTensorType(out.dims(), mlirx.Type.fromDType(cur.mlir_ctx, out.dtype())); + inline for (&res_types, spec.outputs) |*r, fname| { + r.* = mlirx.Type.rankedTensor(cur.mlir_ctx, @field(outputs, fname)); } const op = callTpuCustomCall(&values, &res_types, backend_config, opts.extras); diff --git a/zml/mlirx.zig b/zml/mlirx.zig index 748d988c..a6e86522 100644 --- a/zml/mlirx.zig +++ b/zml/mlirx.zig @@ -8,36 +8,40 @@ const Shape = @import("shape.zig").Shape; const mlirx = @This(); pub const Type = struct { + pub fn rankedTensor(ctx: *mlir.Context, shape: Shape) *const mlir.Type { + return .rankedTensor(shape.dims(), mlirx.Type.fromDType(ctx, shape.dtype())); + } + pub fn fromDType(ctx: *mlir.Context, dt: dtype.DataType) *const mlir.Type { return switch (dt) { - .bool => mlir.integerType(ctx, .i1), - .f4e2m1 => mlir.floatType(ctx, .f4e2m1fn), - .f8e3m4 => mlir.floatType(ctx, .f8e3m4), - .f8e4m3 => mlir.floatType(ctx, .f8e4m3), - .f8e4m3fn => mlir.floatType(ctx, .f8e4m3fn), - .f8e4m3fnuz => mlir.floatType(ctx, .f8e4m3fnuz), - .f8e4m3b11fnuz => mlir.floatType(ctx, .f8e4m3b11fnuz), - .f8e5m2 => mlir.floatType(ctx, .f8e5m2), - .f8e5m2fnuz => mlir.floatType(ctx, .f8e5m2fnuz), + .bool => .int(ctx, .i1), + .f4e2m1 => .float(ctx, .f4e2m1fn), + .f8e3m4 => .float(ctx, .f8e3m4), + .f8e4m3 => .float(ctx, .f8e4m3), + .f8e4m3fn => .float(ctx, .f8e4m3fn), + .f8e4m3fnuz => .float(ctx, .f8e4m3fnuz), + .f8e4m3b11fnuz => .float(ctx, .f8e4m3b11fnuz), + .f8e5m2 => .float(ctx, .f8e5m2), + .f8e5m2fnuz => .float(ctx, .f8e5m2fnuz), .f8e8m0 => @panic("Unsupported type"), - .bf16 => mlir.floatType(ctx, .bf16), - .f16 => mlir.floatType(ctx, .f16), - .f32 => mlir.floatType(ctx, .f32), - .f64 => mlir.floatType(ctx, .f64), - .i2 => mlir.integerType(ctx, .i2), - .i4 => mlir.integerType(ctx, .i4), - .i8 => mlir.integerType(ctx, .i8), - .i16 => mlir.integerType(ctx, .i16), - .i32 => mlir.integerType(ctx, .i32), - .i64 => mlir.integerType(ctx, .i64), - .u2 => mlir.integerType(ctx, .u2), - .u4 => mlir.integerType(ctx, .u4), - .u8 => mlir.integerType(ctx, .u8), - .u16 => mlir.integerType(ctx, .u16), - .u32 => mlir.integerType(ctx, .u32), - .u64 => mlir.integerType(ctx, .u64), - .c64 => mlir.complexType(ctx, .c64), - .c128 => mlir.complexType(ctx, .c128), + .bf16 => .float(ctx, .bf16), + .f16 => .float(ctx, .f16), + .f32 => .float(ctx, .f32), + .f64 => .float(ctx, .f64), + .i2 => .int(ctx, .i2), + .i4 => .int(ctx, .i4), + .i8 => .int(ctx, .i8), + .i16 => .int(ctx, .i16), + .i32 => .int(ctx, .i32), + .i64 => .int(ctx, .i64), + .u2 => .int(ctx, .u2), + .u4 => .int(ctx, .u4), + .u8 => .int(ctx, .u8), + .u16 => .int(ctx, .u16), + .u32 => .int(ctx, .u32), + .u64 => .int(ctx, .u64), + .c64 => .complex(ctx, .c64), + .c128 => .complex(ctx, .c128), }; } @@ -45,34 +49,34 @@ pub const Type = struct { // It's weird to have to provide a mlir context to get a zml.DataType from a mlir.Type. pub fn toDType(ctx: *mlir.Context, mlir_type: *const mlir.Type) dtype.DataType { const mapping = .{ - .{ .bool, mlir.integerType(ctx, .i1) }, + .{ .bool, mlir.Type.int(ctx, .i1) }, - .{ .f8e4m3b11fnuz, mlir.floatType(ctx, .f8e4m3b11fnuz) }, - .{ .f8e4m3fn, mlir.floatType(ctx, .f8e4m3fn) }, - .{ .f8e4m3fnuz, mlir.floatType(ctx, .f8e4m3fnuz) }, - .{ .f8e3m4, mlir.floatType(ctx, .f8e3m4) }, - .{ .f8e5m2, mlir.floatType(ctx, .f8e5m2) }, - .{ .f8e5m2fnuz, mlir.floatType(ctx, .f8e5m2fnuz) }, - .{ .f4e2m1, mlir.floatType(ctx, .f4e2m1fn) }, - .{ .bf16, mlir.floatType(ctx, .bf16) }, - .{ .f16, mlir.floatType(ctx, .f16) }, - .{ .f32, mlir.floatType(ctx, .f32) }, - .{ .f64, mlir.floatType(ctx, .f64) }, + .{ .f8e4m3b11fnuz, mlir.Type.float(ctx, .f8e4m3b11fnuz) }, + .{ .f8e4m3fn, mlir.Type.float(ctx, .f8e4m3fn) }, + .{ .f8e4m3fnuz, mlir.Type.float(ctx, .f8e4m3fnuz) }, + .{ .f8e3m4, mlir.Type.float(ctx, .f8e3m4) }, + .{ .f8e5m2, mlir.Type.float(ctx, .f8e5m2) }, + .{ .f8e5m2fnuz, mlir.Type.float(ctx, .f8e5m2fnuz) }, + .{ .f4e2m1, mlir.Type.float(ctx, .f4e2m1fn) }, + .{ .bf16, mlir.Type.float(ctx, .bf16) }, + .{ .f16, mlir.Type.float(ctx, .f16) }, + .{ .f32, mlir.Type.float(ctx, .f32) }, + .{ .f64, mlir.Type.float(ctx, .f64) }, - .{ .i4, mlir.integerType(ctx, .i4) }, - .{ .i8, mlir.integerType(ctx, .i8) }, - .{ .i16, mlir.integerType(ctx, .i16) }, - .{ .i32, mlir.integerType(ctx, .i32) }, - .{ .i64, mlir.integerType(ctx, .i64) }, + .{ .i4, mlir.Type.int(ctx, .i4) }, + .{ .i8, mlir.Type.int(ctx, .i8) }, + .{ .i16, mlir.Type.int(ctx, .i16) }, + .{ .i32, mlir.Type.int(ctx, .i32) }, + .{ .i64, mlir.Type.int(ctx, .i64) }, - .{ .u4, mlir.integerType(ctx, .u4) }, - .{ .u8, mlir.integerType(ctx, .u8) }, - .{ .u16, mlir.integerType(ctx, .u16) }, - .{ .u32, mlir.integerType(ctx, .u32) }, - .{ .u64, mlir.integerType(ctx, .u64) }, + .{ .u4, mlir.Type.int(ctx, .u4) }, + .{ .u8, mlir.Type.int(ctx, .u8) }, + .{ .u16, mlir.Type.int(ctx, .u16) }, + .{ .u32, mlir.Type.int(ctx, .u32) }, + .{ .u64, mlir.Type.int(ctx, .u64) }, - .{ .c64, mlir.complexType(ctx, .c64) }, - .{ .c128, mlir.complexType(ctx, .c128) }, + .{ .c64, mlir.Type.complex(ctx, .c64) }, + .{ .c128, mlir.Type.complex(ctx, .c128) }, }; inline for (mapping) |entry| { diff --git a/zml/module.zig b/zml/module.zig index 83678d85..de774f31 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -106,7 +106,7 @@ pub const CompilationContext = struct { mlir_ctx.loadAllAvailableDialects(); const module = mlir.Module.init(.unknown(mlir_ctx)); - module.operation().setAttributeByName("sym_name", mlir.stringAttribute(mlir_ctx, opts.program_name)); + module.operation().setAttributeByName("sym_name", .string(mlir_ctx, opts.program_name)); const pass_manager = mlir.PassManager.init(mlir_ctx); { @@ -225,11 +225,11 @@ pub fn compile( compilation_context.module.operation().setAttributeByName( "mhlo.num_partitions", - mlir.integerAttribute(compilation_context.mlir_ctx, .i32, num_partitions), + .int(compilation_context.mlir_ctx, .i32, num_partitions), ); compilation_context.module.operation().setAttributeByName( "mhlo.num_replicas", - mlir.integerAttribute(compilation_context.mlir_ctx, .i32, num_replicas), + .int(compilation_context.mlir_ctx, .i32, num_replicas), ); compilation_context.mlir_pass_manager.runOnOp(compilation_context.module.operation()) catch |err| switch (err) { @@ -279,7 +279,7 @@ fn addPartitionerOperations(ctx: *CompilationContext) !void { const mesh_op = mlir.Operation.make(mlir_ctx, "sdy.mesh", .{ .attributes = &.{ - .named(mlir_ctx, "sym_name", mlir.stringAttribute(mlir_ctx, name)), + .named(mlir_ctx, "sym_name", .string(mlir_ctx, name)), .named(mlir_ctx, "mesh", mesh_attr), }, .location = .unknown(mlir_ctx), @@ -417,7 +417,7 @@ const EmitMlirResult = struct { fn finalizeAttributeList(allocator_: std.mem.Allocator, mlir_ctx: *mlir.Context, attributes: []AttributeList) ![]*const mlir.Attribute { const res = try allocator_.alloc(*const mlir.Attribute, attributes.len); for (res, attributes) |*r, attr| { - r.* = mlir.dictionaryAttribute(mlir_ctx, attr.constSlice()); + r.* = .dict(mlir_ctx, attr.constSlice()); } return res; } @@ -444,10 +444,7 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar }; meta.visit(struct { fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { - const mlir_type = mlir.rankedTensorType( - tensor.dims(), - mlirx.Type.fromDType(ctx_.compilation_context.mlir_ctx, tensor.dtype()), - ); + const mlir_type = mlirx.Type.rankedTensor(ctx_.compilation_context.mlir_ctx, tensor.shape()); _ = ctx_.compilation_context.currentScope().block.addArgument(mlir_type, .unknown(ctx_.compilation_context.mlir_ctx)); ctx_.compilation_context.currentScope().id_to_argument.put(ctx_.compilation_context.currentScope().arena.allocator(), tensor.id, ctx_.current_argument_id) catch unreachable; ctx_.current_argument_id += 1; @@ -478,14 +475,14 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar @memset(output_attributes, .empty); for (output_info.donations, 0..) |donation, index| if (donation) |argument_index| { - input_attributes[argument_index].appendAssumeCapacity(.named(compilation_context.mlir_ctx, "tf.aliasing_output", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, index))); + input_attributes[argument_index].appendAssumeCapacity(.named(compilation_context.mlir_ctx, "tf.aliasing_output", .int(compilation_context.mlir_ctx, .i32, index))); }; for (output_info.output_memory_kinds, 0..) |output_memory_kind, index| { if (output_memory_kind == .device) continue; output_attributes[index].appendAssumeCapacity(.named( compilation_context.mlir_ctx, "mhlo.memory_kind", - mlir.stringAttribute( + .string( compilation_context.mlir_ctx, compilation_context.platform.memoryKind(output_memory_kind), ), @@ -507,7 +504,7 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar input_attributes[i].appendAssumeCapacity(.named( compilation_context.mlir_ctx, "mhlo.memory_kind", - mlir.stringAttribute( + .string( compilation_context.mlir_ctx, compilation_context.platform.memoryKind(memory_kind), ), diff --git a/zml/ops.zig b/zml/ops.zig index 2bc0ddea..cc63c925 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -39,8 +39,8 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func args[i].left = .fromShape(inits[i].shape()); args[i].right = .fromShape(inits[i].shape()); - block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); - block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); + block_types[i] = mlirx.Type.rankedTensor(mlir_ctx, args[i].left.shape()); + block_types[i + inits.len] = mlirx.Type.rankedTensor(mlir_ctx, args[i].right.shape()); } const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); @@ -79,7 +79,7 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func .result_type_inference = true, .blocks = &.{reduce_block}, .attributes = &.{ - .named(mlir_ctx, "dimensions", mlir.denseArrayAttribute(mlir_ctx, .i64, axes_)), + .named(mlir_ctx, "dimensions", .denseArray(mlir_ctx, .i64, axes_)), }, .verify = true, .location = .unknown(mlir_ctx), @@ -103,12 +103,11 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func reduced_shape = reduced_shape.setDim(a, 1); } - const tensor_type = mlir.rankedTensorType(reduced_shape.dims(), mlirx.Type.fromDType(mlir_ctx, reduced_shape.dtype())); const broad_op = dialects.stablehlo.broadcast_in_dim( mlir_ctx, reduce_op.result(i), broadcasting_axes.slice()[0 .. reduced_shape.rank() - axes_.len], - tensor_type, + mlirx.Type.rankedTensor(mlir_ctx, reduced_shape), .unknown(mlir_ctx), ).appendTo(CompilationContext.current().currentScope().block); @@ -142,8 +141,8 @@ pub fn reduceWindow(inputs: anytype, inits: anytype, opts: ReduceWindowOpts, com args[i].left = .fromShape(inits[i].shape()); args[i].right = .fromShape(inits[i].shape()); - block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); - block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); + block_types[i] = mlirx.Type.rankedTensor(mlir_ctx, args[i].left.shape()); + block_types[i + inits.len] = mlirx.Type.rankedTensor(mlir_ctx, args[i].right.shape()); } const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); @@ -182,12 +181,15 @@ pub fn reduceWindow(inputs: anytype, inits: anytype, opts: ReduceWindowOpts, com .result_type_inference = true, .blocks = &.{reduce_block}, .attributes = &.{ - .named(mlir_ctx, "window_dimensions", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.window_dimensions)), - .named(mlir_ctx, "window_strides", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.window_strides)), - .named(mlir_ctx, "base_dilations", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.base_dilations)), - .named(mlir_ctx, "window_dilations", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.window_dilations)), + .named(mlir_ctx, "window_dimensions", .denseArray(mlir_ctx, .i64, opts.window_dimensions)), + .named(mlir_ctx, "window_strides", .denseArray(mlir_ctx, .i64, opts.window_strides)), + .named(mlir_ctx, "base_dilations", .denseArray(mlir_ctx, .i64, opts.base_dilations)), + .named(mlir_ctx, "window_dilations", .denseArray(mlir_ctx, .i64, opts.window_dilations)), // Cast the [][2]i64 to []i64 (safe) - .named(mlir_ctx, "padding", mlir.denseElementsAttribute(mlir.RankedTensorType.get(&.{ @intCast(opts.padding.len), 2 }, mlir.integerType(mlir_ctx, .i64), null).shaped(), @as([]const i64, @ptrCast(opts.padding)))), + .named(mlir_ctx, "padding", .denseElements( + .rankedTensor(&.{ @intCast(opts.padding.len), 2 }, .int(mlir_ctx, .i64)), + @as([]const i64, @ptrCast(opts.padding)), + )), }, .verify = true, .location = .unknown(mlir_ctx), @@ -220,8 +222,8 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytyp args[i].left = Tensor.init(.{}, inputs[i].shape().dtype()); args[i].right = Tensor.init(.{}, inputs[i].shape().dtype()); - block_types[2 * i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); - block_types[2 * i + 1] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); + block_types[2 * i] = mlirx.Type.rankedTensor(mlir_ctx, args[i].left.shape()); + block_types[2 * i + 1] = mlirx.Type.rankedTensor(mlir_ctx, args[i].right.shape()); } const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); @@ -253,8 +255,8 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytyp .result_type_inference = true, .blocks = &.{sort_block}, .attributes = &.{ - .named(mlir_ctx, "dimension", mlir.integerAttribute(mlir_ctx, .i64, axis_)), - .named(mlir_ctx, "is_stable", mlir.boolAttribute(mlir_ctx, is_stable)), + .named(mlir_ctx, "dimension", .int(mlir_ctx, .i64, axis_)), + .named(mlir_ctx, "is_stable", .boolean(mlir_ctx, is_stable)), }, .verify = true, .location = .unknown(mlir_ctx), @@ -292,7 +294,7 @@ pub fn @"while"(operands: anytype, comptime cond: anytype, comptime body: anytyp inline for (0..operands.len) |i| { operand_shapes[i] = operands[i].shape(); - block_types[i] = mlir.rankedTensorType(operand_shapes[i].dims(), mlirx.Type.fromDType(mlir_ctx, operand_shapes[i].dtype())); + block_types[i] = mlirx.Type.rankedTensor(mlir_ctx, operand_shapes[i]); operand_values[i] = operands[i].value(); } @@ -468,17 +470,17 @@ pub fn triton(inputs: anytype, outputs: anytype, opts: TritonOps) [outputs.len]T var res_types: [outputs.len]*const mlir.Type = undefined; inline for (outputs, 0..) |output, i| { - res_types[i] = mlir.rankedTensorType(output.dims(), mlirx.Type.fromDType(mlir_ctx, output.dtype())); + res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); } - const backend_config = mlir.dictionaryAttribute(mlir_ctx, &.{ - mlir.NamedAttribute.named(mlir_ctx, "name", mlir.stringAttribute(mlir_ctx, opts.name)), - mlir.NamedAttribute.named(mlir_ctx, "ir", mlir.stringAttribute(mlir_ctx, opts.ir)), - mlir.NamedAttribute.named(mlir_ctx, "grid_x", mlir.integerAttribute(mlir_ctx, .i32, opts.grid[0])), - mlir.NamedAttribute.named(mlir_ctx, "grid_y", mlir.integerAttribute(mlir_ctx, .i32, opts.grid[1])), - mlir.NamedAttribute.named(mlir_ctx, "grid_z", mlir.integerAttribute(mlir_ctx, .i32, opts.grid[2])), - mlir.NamedAttribute.named(mlir_ctx, "num_stages", mlir.integerAttribute(mlir_ctx, .i32, opts.num_stages)), - mlir.NamedAttribute.named(mlir_ctx, "num_warps", mlir.integerAttribute(mlir_ctx, .i32, opts.num_warps)), + const backend_config: *const mlir.Attribute = .dict(mlir_ctx, &.{ + .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), + .named(mlir_ctx, "ir", .string(mlir_ctx, opts.ir)), + .named(mlir_ctx, "grid_x", .int(mlir_ctx, .i32, opts.grid[0])), + .named(mlir_ctx, "grid_y", .int(mlir_ctx, .i32, opts.grid[1])), + .named(mlir_ctx, "grid_z", .int(mlir_ctx, .i32, opts.grid[2])), + .named(mlir_ctx, "num_stages", .int(mlir_ctx, .i32, opts.num_stages)), + .named(mlir_ctx, "num_warps", .int(mlir_ctx, .i32, opts.num_warps)), }); var operands_layouts: [inputs.len][]const usize = undefined; @@ -611,8 +613,8 @@ pub fn scatter( args[i].input = Tensor.init(.{}, inputs[i].dtype()); args[i].update = Tensor.init(.{}, inputs[i].dtype()); - block_types[i] = mlir.rankedTensorType(args[i].input.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].input.dtype())); - block_types[i + inputs.len] = mlir.rankedTensorType(args[i].update.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].update.dtype())); + block_types[i] = mlirx.Type.rankedTensor(mlir_ctx, args[i].input.shape()); + block_types[i + inputs.len] = mlirx.Type.rankedTensor(mlir_ctx, args[i].update.shape()); } const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); @@ -1060,8 +1062,8 @@ pub const CustomCallOptions = struct { fn customCallAdditionalAttributes(ctx: *CompilationContext, opts: CustomCallOptions) []const mlir.NamedAttribute { if (!opts.compute_on_host or ctx.platform.target == .cpu) return &.{}; - const frontend_attributes = mlir.dictionaryAttribute(ctx.mlir_ctx, &.{ - .named(ctx.mlir_ctx, "_xla_compute_type", mlir.stringAttribute(ctx.mlir_ctx, "host")), + const frontend_attributes = mlir.Attribute.dict(ctx.mlir_ctx, &.{ + .named(ctx.mlir_ctx, "_xla_compute_type", .string(ctx.mlir_ctx, "host")), }); const additional_attributes = ctx.arena.allocator().alloc(mlir.NamedAttribute, 1) catch unreachable; @@ -1279,7 +1281,7 @@ fn manualComputationInternal( const block_types = allocator.alloc(*const mlir.Type, inputs.len) catch unreachable; const block_locs = allocator.alloc(*const mlir.Location, inputs.len) catch unreachable; for (local_input_shapes, 0..) |input_shape, i| { - block_types[i] = mlir.rankedTensorType(input_shape.dims(), mlirx.Type.fromDType(ctx.mlir_ctx, input_shape.dtype())); + block_types[i] = mlirx.Type.rankedTensor(ctx.mlir_ctx, input_shape); block_locs[i] = mlir.Location.unknown(ctx.mlir_ctx); } @@ -1318,7 +1320,7 @@ fn manualComputationInternal( const global_result_types = allocator.alloc(*const mlir.Type, outputs.len) catch unreachable; for (outputs, 0..) |output_shape, i| { - global_result_types[i] = mlir.rankedTensorType(output_shape.dims(), mlirx.Type.fromDType(ctx.mlir_ctx, output_shape.dtype())); + global_result_types[i] = mlirx.Type.rankedTensor(ctx.mlir_ctx, output_shape); } const op = mlir.Operation.make(ctx.mlir_ctx, "sdy.manual_computation", .{ @@ -1356,7 +1358,7 @@ fn manualComputationInternal( const local_input_values = allocator.alloc(*const mlir.Value, inputs.len) catch unreachable; for (0..inputs.len) |i| { - const local_type = mlir.rankedTensorType(local_input_shapes[i].dims(), mlirx.Type.fromDType(ctx.mlir_ctx, local_input_shapes[i].dtype())); + const local_type = mlirx.Type.rankedTensor(ctx.mlir_ctx, local_input_shapes[i]); const full_to_shard = dialects.stablehlo.custom_call( ctx.mlir_ctx, &.{input_values[i]}, @@ -1366,7 +1368,7 @@ fn manualComputationInternal( .has_side_effect = false, .backend_config = .{ .original = "" }, .additional_attributes = &.{ - .named(ctx.mlir_ctx, "mhlo.sharding", mlir.stringAttribute(ctx.mlir_ctx, manual_sharding)), + .named(ctx.mlir_ctx, "mhlo.sharding", .string(ctx.mlir_ctx, manual_sharding)), }, }, .unknown(ctx.mlir_ctx), @@ -1397,7 +1399,7 @@ fn manualComputationInternal( const global_values = allocator.alloc(*const mlir.Value, outputs.len) catch unreachable; const global_types = allocator.alloc(*const mlir.Type, outputs.len) catch unreachable; for (outputs, 0..) |output_shape, i| { - global_types[i] = mlir.rankedTensorType(output_shape.dims(), mlirx.Type.fromDType(ctx.mlir_ctx, output_shape.dtype())); + global_types[i] = mlirx.Type.rankedTensor(ctx.mlir_ctx, output_shape); const shard_to_full = dialects.stablehlo.custom_call( ctx.mlir_ctx, &.{local_outputs[i].value()}, @@ -1519,17 +1521,17 @@ fn manualComputationSliceToReturn(comptime ReturnT: type, outputs: []Tensor) Ret fn metadataIntegerFieldToMlirAttribute(mlir_ctx: *mlir.Context, int_field: std.builtin.Type.Int, value: anytype) *const mlir.Attribute { return switch (int_field.signedness) { .signed => switch (int_field.bits) { - 8 => mlir.integerAttribute(mlir_ctx, .i8, value), - 16 => mlir.integerAttribute(mlir_ctx, .i16, value), - 32 => mlir.integerAttribute(mlir_ctx, .i32, value), - 64 => mlir.integerAttribute(mlir_ctx, .i64, value), + 8 => .int(mlir_ctx, .i8, value), + 16 => .int(mlir_ctx, .i16, value), + 32 => .int(mlir_ctx, .i32, value), + 64 => .int(mlir_ctx, .i64, value), else => @panic("Unsupported DataType"), }, .unsigned => switch (int_field.bits) { - 8 => mlir.integerAttribute(mlir_ctx, .u8, value), - 16 => mlir.integerAttribute(mlir_ctx, .u16, value), - 32 => mlir.integerAttribute(mlir_ctx, .u32, value), - 64 => mlir.integerAttribute(mlir_ctx, .u64, value), + 8 => .int(mlir_ctx, .u8, value), + 16 => .int(mlir_ctx, .u16, value), + 32 => .int(mlir_ctx, .u32, value), + 64 => .int(mlir_ctx, .u64, value), else => @panic("Unsupported DataType"), }, }; @@ -1537,9 +1539,9 @@ fn metadataIntegerFieldToMlirAttribute(mlir_ctx: *mlir.Context, int_field: std.b fn metadataFloatFieldToMlirAttribute(mlir_ctx: *mlir.Context, float_field: std.builtin.Type.Float, value: anytype) *const mlir.Attribute { return switch (float_field.bits) { - 16 => mlir.floatAttribute(mlir_ctx, .f16, @as(f64, value)), - 32 => mlir.floatAttribute(mlir_ctx, .f32, @as(f64, value)), - 64 => mlir.floatAttribute(mlir_ctx, .f64, @as(f64, value)), + 16 => .float(mlir_ctx, .f16, @as(f64, value)), + 32 => .float(mlir_ctx, .f32, @as(f64, value)), + 64 => .float(mlir_ctx, .f64, @as(f64, value)), else => @panic("Unsupported DataType"), }; } @@ -1547,17 +1549,17 @@ fn metadataFloatFieldToMlirAttribute(mlir_ctx: *mlir.Context, float_field: std.b fn metadataFieldToMlirAttribute(mlir_ctx: *mlir.Context, comptime T: type, value: anytype) ?*const mlir.Attribute { const type_info = @typeInfo(T); return switch (type_info) { - .comptime_int => mlir.integerAttribute(mlir_ctx, .u64, @as(u64, value)), + .comptime_int => .int(mlir_ctx, .u64, @as(u64, value)), .@"enum" => |enum_field| switch (@typeInfo(enum_field.tag_type)) { .int => |int_tag| metadataIntegerFieldToMlirAttribute(mlir_ctx, int_tag, @intFromEnum(value)), else => @compileError("Unsupported tag type for enum metadata: " ++ @typeName(enum_field.tag_type)), }, .int => |int_field| metadataIntegerFieldToMlirAttribute(mlir_ctx, int_field, value), .float => |float_field| metadataFloatFieldToMlirAttribute(mlir_ctx, float_field, value), - .bool => mlir.boolAttribute(mlir_ctx, value), + .bool => .boolean(mlir_ctx, value), .pointer => |pointer_info| switch (pointer_info.size) { .slice => if (pointer_info.child == u8) - mlir.stringAttribute(mlir_ctx, value) + .string(mlir_ctx, value) else @panic("Unsupported pointer type in metadata"), else => @panic("Unsupported pointer type in metadata"), @@ -1565,7 +1567,7 @@ fn metadataFieldToMlirAttribute(mlir_ctx: *mlir.Context, comptime T: type, value .optional => |optional_info| if (value) |wrapped_value| switch (@typeInfo(optional_info.child)) { .int => |int_field| metadataIntegerFieldToMlirAttribute(mlir_ctx, int_field, wrapped_value), .float => |float_field| metadataFloatFieldToMlirAttribute(mlir_ctx, float_field, wrapped_value), - .bool => mlir.boolAttribute(mlir_ctx, wrapped_value), + .bool => .boolean(mlir_ctx, wrapped_value), else => @panic("Unsupported optional child type in metadata"), } else null, else => @compileError("Unsupported metadata type: " ++ @typeName(T)), @@ -1773,8 +1775,8 @@ pub fn typedCustomCall( } } metadata_attribute_list.appendSliceAssumeCapacity(&[_]mlir.NamedAttribute{ - .named(ctx.mlir_ctx, "pjrt_api", mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_api))))), - .named(ctx.mlir_ctx, "pjrt_client", mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_client))))), + .named(ctx.mlir_ctx, "pjrt_api", .int(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_api))))), + .named(ctx.mlir_ctx, "pjrt_client", .int(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_client))))), }); const values = allocator.alloc(*const mlir.Value, input_tensors.len) catch unreachable; @@ -1786,10 +1788,10 @@ pub fn typedCustomCall( const result_types = allocator.alloc(*const mlir.Type, output_shapes.len) catch unreachable; for (output_shapes, 0..) |shape, i| { - result_types[i] = mlir.rankedTensorType(shape.dims(), mlirx.Type.fromDType(ctx.mlir_ctx, shape.dtype())); + result_types[i] = .rankedTensor(shape.dims(), mlirx.Type.fromDType(ctx.mlir_ctx, shape.dtype())); } - const backend_config = mlir.dictionaryAttribute(ctx.mlir_ctx, metadata_attribute_list.items); + const backend_config: *const mlir.Attribute = .dict(ctx.mlir_ctx, metadata_attribute_list.items); const operand_layouts = allocator.alloc([]const usize, input_shapes.len) catch unreachable; for (input_shapes, 0..) |shape, i| { @@ -1818,7 +1820,7 @@ pub fn typedCustomCall( ).appendTo(ctx.currentScope().block); if (ctx.manual_computation_depth > 0 and ctx.partitioning.partitioner == .gspmd) { - op.setAttributeByName("mhlo.sharding", mlir.stringAttribute(ctx.mlir_ctx, "{manual}")); + op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); } switch (@typeInfo(Output)) { diff --git a/zml/tensor.zig b/zml/tensor.zig index f805b80f..d1faf710 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -17,6 +17,11 @@ const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const Sharding = @import("Sharding.zig"); +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(Tensor); +} + pub const Tensor = struct { var current_id: std.atomic.Value(usize) = .{ .raw = 1 }; @@ -185,11 +190,12 @@ pub const Tensor = struct { const ctx = CompilationContext.current(); if (ctx.platform.target == .cpu) return self; - const frontend_attributes = mlir.dictionaryAttribute(ctx.mlir_ctx, &.{ - .named(ctx.mlir_ctx, "_xla_buffer_placement", mlir.stringAttribute( + const frontend_attributes: *const mlir.Attribute = .dict(ctx.mlir_ctx, &.{ + .named( ctx.mlir_ctx, - ctx.platform.memoryKind(kind), - )), + "_xla_buffer_placement", + .string(ctx.mlir_ctx, ctx.platform.memoryKind(kind)), + ), }); const op = dialects.stablehlo.custom_call( @@ -311,7 +317,7 @@ pub const Tensor = struct { const op = dialects.stablehlo.bitcast_convert( mlirCtx(), self.value(), - mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), + mlirx.Type.rankedTensor(mlirCtx(), res_shape), .unknown(mlirCtx()), ).appendTo(currentBlock()); @@ -557,8 +563,8 @@ pub const Tensor = struct { mlirCtx(), self.algorithm, self._state.value(), - mlir.rankedTensorType(self._state.dims(), mlirx.Type.fromDType(mlirCtx(), self._state.dtype())), - mlir.rankedTensorType(sh.dims(), mlirx.Type.fromDType(mlirCtx(), sh.dtype())), + mlirx.Type.rankedTensor(mlirCtx(), self._state.shape()), + mlirx.Type.rankedTensor(mlirCtx(), sh), .unknown(mlirCtx()), ).appendTo(currentBlock()); return .{ self.update(op.result(0)), _result(sh, op.result(1)) }; @@ -880,7 +886,7 @@ pub const Tensor = struct { mlirCtx(), self.value(), other.value(), - mlir.rankedTensorType(new_shape.dims(), mlirx.Type.fromDType(mlirCtx(), new_shape.dtype())), + mlirx.Type.rankedTensor(mlirCtx(), new_shape), used_opts, .unknown(mlirCtx()), ).appendTo(currentBlock()); @@ -1062,7 +1068,7 @@ pub const Tensor = struct { return self; } - const res_type = mlir.rankedTensorType(self.shape().dims(), mlirx.Type.fromDType(mlirCtx(), to)); + const res_type = mlir.Type.rankedTensor(self.shape().dims(), mlirx.Type.fromDType(mlirCtx(), to)); const op = dialects.stablehlo.convert(mlirCtx(), self.value(), res_type, .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape.withDtype(to), op.result(0)); } @@ -1277,7 +1283,7 @@ pub const Tensor = struct { mlirCtx(), lhs.value(), rhs.value(), - mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), + mlirx.Type.rankedTensor(mlirCtx(), res_shape), .{ .lhs_batching_dimensions = lhs_batching_axes.constSlice(), .rhs_batching_dimensions = rhs_batching_axes.constSlice(), @@ -1528,7 +1534,7 @@ pub const Tensor = struct { const op = dialects.stablehlo.transpose( mlirCtx(), self.value(), - mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), + mlirx.Type.rankedTensor(mlirCtx(), res_shape), .{ .permutation = toI64(permutation).constSlice() }, .unknown(mlirCtx()), ).appendTo(currentBlock()); @@ -1560,7 +1566,7 @@ pub const Tensor = struct { const reshaped_val = dialects.stablehlo.reshape( mlirCtx(), self.value(), - mlir.rankedTensorType(new_shape.dims(), mlirx.Type.fromDType(mlirCtx(), new_shape.dtype())), + mlirx.Type.rankedTensor(mlirCtx(), new_shape), .unknown(mlirCtx()), ).appendTo(currentBlock()); return _result(new_shape, reshaped_val.result(0)); @@ -1576,7 +1582,7 @@ pub const Tensor = struct { const reshaped_val = dialects.stablehlo.reshape( mlirCtx(), self.value(), - mlir.rankedTensorType(new_shape.dims(), mlirx.Type.fromDType(mlirCtx(), new_shape.dtype())), + mlirx.Type.rankedTensor(mlirCtx(), new_shape), .unknown(mlirCtx()), ).appendTo(currentBlock()); return _result(new_shape, reshaped_val.result(0)); @@ -1665,15 +1671,15 @@ pub const Tensor = struct { res_shape = res_shape.setDim(a, std.math.divCeil(i64, args.end - args.start, args.step) catch unreachable); } - const result_type = mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())); + const ctx = mlirCtx(); const slice_op = dialects.stablehlo.slice( - mlirCtx(), + ctx, self.value(), start_indices[0..self.rank()], limit_indices[0..self.rank()], strides[0..self.rank()], - result_type, - .unknown(mlirCtx()), + mlirx.Type.rankedTensor(ctx, res_shape), + .unknown(ctx), ).appendTo(currentBlock()); var res = _result(res_shape, slice_op.result(0)); @@ -1993,7 +1999,7 @@ pub const Tensor = struct { var op = dialects.stablehlo.iota( mlirCtx(), 0, - mlir.rankedTensorType(sh.dims(), mlirx.Type.fromDType(mlirCtx(), sh.dtype())), + mlirx.Type.rankedTensor(mlirCtx(), sh), .unknown(mlirCtx()), ).appendTo(currentBlock()); var res = _result(sh, op.result(0)); @@ -2022,7 +2028,7 @@ pub const Tensor = struct { var op = dialects.stablehlo.iota( mlirCtx(), a, - mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), + mlirx.Type.rankedTensor(mlirCtx(), res_shape), .unknown(mlirCtx()), ).appendTo(currentBlock()); return _result(res_shape, op.result(0)); @@ -2044,7 +2050,7 @@ pub const Tensor = struct { var iota_op = dialects.stablehlo.iota( mlirCtx(), 0, - mlir.rankedTensorType(sh.dims(), mlirx.Type.fromDType(mlirCtx(), sh.dtype())), + mlirx.Type.rankedTensor(mlirCtx(), sh), .unknown(mlirCtx()), ).appendTo(currentBlock()); var res = _result(sh, iota_op.result(0)); @@ -2105,7 +2111,7 @@ pub const Tensor = struct { pub fn uninitialized(sh: Shape) Tensor { const ctx = CompilationContext.current(); - const buffer_type = mlir.memRefType( + const buffer_type = mlir.Type.memRef( mlirx.Type.fromDType(ctx.mlir_ctx, sh.dtype()), sh.dims(), null, @@ -2157,10 +2163,7 @@ pub const Tensor = struct { // but we propagate output_shape tags. return _result(res_shape, self.value()); } - const result_type = mlir.rankedTensorType( - res_shape.dims(), - mlirx.Type.fromDType(mlirCtx(), res_shape.dtype()), - ); + const result_type = mlirx.Type.rankedTensor(mlirCtx(), res_shape); const broadcast_op = dialects.stablehlo.broadcast_in_dim(mlirCtx(), self.value(), axes_, result_type, .unknown(mlirCtx())).appendTo(currentBlock()); return _result(res_shape, broadcast_op.result(0)); } @@ -2215,7 +2218,7 @@ pub const Tensor = struct { /// Reshapes the input Tensor with the given shape. pub fn reshape(self: Tensor, output_shape_: anytype) Tensor { const output_shape = self._shape.reshape(output_shape_); - const tensor_type = mlir.rankedTensorType(output_shape.dims(), mlirx.Type.fromDType(mlirCtx(), output_shape.dtype())); + const tensor_type = mlirx.Type.rankedTensor(mlirCtx(), output_shape); const reshape_value = dialects.stablehlo.reshape(mlirCtx(), self.value(), tensor_type, .unknown(mlirCtx())).appendTo(currentBlock()); return _result(output_shape, reshape_value.result(0)); } From dd0ab6c3341cfe8dc00175a5fe1e2758d6466e7f Mon Sep 17 00:00:00 2001 From: loupicaaa Date: Fri, 15 May 2026 11:06:02 +0200 Subject: [PATCH 486/881] zml/moe : Moe backend triton zig, qwen3_5_moe model (#529) ### Overview Context: Modern models utilize a MLP called Mixture of Experts (MoE). In a MoE, the MLP is organized into "experts," where specific sub-tensors are specialized to process particular linguistic patterns or elements. During inference, the hidden states of each token are routed to the most relevant experts, ensuring that matrix multiplications (matmul) are performed only on the active parameters. Problem: Due to the sparse and dynamic nature of MoE where token assignment is determined at runtime via routing, treating an MoE layer as a standard matmul results in significant performance degradation. Standard kernels are not optimized for the irregular memory access patterns in sparse routing. Approach: MoE is implemented as a primitive in the ZML operation set. The generic MoE Backend is designed to support various specialized kernels tailored to specific hardware and weight precisions. The primary implementation currently features a Triton MoE backend optimized for NVIDIA Hopper architecture using BF16 and FP8 weights. This implementation is utilized in the Qwen 3.5 MoE model. ### Features **Generic MoE Backend (zml/moe/moe.zig)** - Integrate specialized MoE backends (e.g., Triton). - Select the optimal backend based on hardware and precision. - Implements essential utilities and structures for registration, parameter loading, and metadata management. - The forwardMoe() function acts as the entry point to invoke the appropriate backend. **Triton MoE Backend (zml/moe/triton.zig)** - Generates TTIR (Triton Intermediate Representation) at compile time. - Leverages Triton's custom calls exposed via XLA. Execution Flow: 1. Select the optimal configuration for the fused MoE kernel. 2. Determine token distribution: use naive assignment if the distribution is sparse; otherwise, sort and align tokens for efficiency. 3. Quantization: quantize activations if using FP8 weights. 4. Fused Kernel: execute the fused MoE kernel for the gate and up-projection layers. 5. Post-Processing: split the output and apply the activation function. 6. Quantization : re-quantize activations for the next stage if required by the FP8 path. 7. Fused Kernel: execute the fused MoE kernel for the down-projection layer. 8. Aggregation: sum the Top-K results for each token. Qwen 3.5 MoE - Fully utilizes the Triton MoE backend. - BF16 weights implementation (also tested with FP8 but not implemented here) --------- Co-authored-by: loupicaaa Co-authored-by: Raphael NAHOUM --- MODULE.bazel.lock | 822 ++++++++++ examples/llm/models.zig | 33 + examples/llm/models/qwen3_5/model.zig | 4 +- examples/llm/models/qwen3_5_moe.zig | 11 + examples/llm/models/qwen3_5_moe/inference.zig | 464 ++++++ examples/llm/models/qwen3_5_moe/model.zig | 1376 +++++++++++++++++ examples/llm/models/qwen3_5_moe/session.zig | 459 ++++++ third_party/zls/zls_write_build_config.bzl | 1 + zml/BUILD.bazel | 6 + zml/moe/moe.zig | 135 ++ zml/moe/triton.zig | 654 ++++++++ zml/moe/triton_kernels/config.json | 146 ++ zml/moe/triton_kernels/triton_kernels.zig | 652 ++++++++ zml/zml.zig | 1 + 14 files changed, 4762 insertions(+), 2 deletions(-) create mode 100644 examples/llm/models/qwen3_5_moe.zig create mode 100644 examples/llm/models/qwen3_5_moe/inference.zig create mode 100644 examples/llm/models/qwen3_5_moe/model.zig create mode 100644 examples/llm/models/qwen3_5_moe/session.zig create mode 100644 zml/moe/moe.zig create mode 100644 zml/moe/triton.zig create mode 100644 zml/moe/triton_kernels/config.json create mode 100644 zml/moe/triton_kernels/triton_kernels.zig diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 2b6f470b..66181253 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -448,6 +448,56 @@ ] } }, + "@@aspect_rules_py+//uv/private/tomltool:extension.bzl%tomltool": { + "general": { + "bzlTransitiveDigest": "9BC6q6vdlG+lqF9L13wVxxakVkZBx8buKPFIndN6mHk=", + "usagesDigest": "tQ76lH+Nn5OlAFLvBbnfvnCzNIiwGgY6tOumYZXH44w=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "toml2json_aarch64_osx_libsystem": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_darwin_arm64", + "sha256": "9adc44d976e8f5baf5a6d613ceb3db6e5f56b2e22e75ac56521fdce62f227d88", + "executable": true + } + }, + "toml2json_x86_64_osx_libsystem": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_darwin_amd64", + "sha256": "425bac4015394fd9840fabcd52ac60fc796a9c10655b36c57e36ffb7dc9f3dd4", + "executable": true + } + }, + "toml2json_aarch64_linux_gnu": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_linux_arm64", + "sha256": "be8376c8e3232a242eae0d187f741a475498a949b942361a9af6e95072ef5670", + "executable": true + } + }, + "toml2json_x86_64_linux_gnu": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_linux_amd64", + "sha256": "f3dd54fabf2d27d0c027b0421860e5d9d909080be1613f0ffd87057633b65e9a", + "executable": true + } + } + }, + "recordedRepoMappingEntries": [ + [ + "aspect_rules_py+", + "bazel_tools", + "bazel_tools" + ] + ] + } + }, "@@aspect_tools_telemetry+//:extension.bzl%telemetry": { "general": { "bzlTransitiveDigest": "MePriaXmQNSqUfE+YQvEtzBc2bU1mjITINIffFiZYgo=", @@ -480,6 +530,545 @@ ] } }, + "@@cel-spec+//:extensions.bzl%non_module_dependencies": { + "general": { + "bzlTransitiveDigest": "VN89nCHLUuqo5KktrpNZqv8O+QKrrsGIqdoLPn91hhY=", + "usagesDigest": "HFQJtQrL9nKaFZEjgwaHVMHALMW+cafu696xy7J4ueM=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "com_google_googleapis": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "sha256": "bd8e735d881fb829751ecb1a77038dda4a8d274c45490cb9fcf004583ee10571", + "strip_prefix": "googleapis-07c27163ac591955d736f3057b1619ece66f5b99", + "urls": [ + "https://github.com/googleapis/googleapis/archive/07c27163ac591955d736f3057b1619ece66f5b99.tar.gz" + ] + } + } + }, + "recordedRepoMappingEntries": [ + [ + "cel-spec+", + "bazel_tools", + "bazel_tools" + ] + ] + } + }, + "@@cel-spec+//:googleapis_ext.bzl%googleapis_ext": { + "general": { + "bzlTransitiveDigest": "65z4sd8o73L2a8X2Pd3yAjn73l9HBG45JVnVY2DUbzI=", + "usagesDigest": "Ek7VfZ+tuyRBx/1h5wcmtnW9EGpOb0dkXUwBluZbD8k=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "com_google_googleapis_imports": { + "repoRuleId": "@@cel-spec++non_module_dependencies+com_google_googleapis//:repository_rules.bzl%switched_rules", + "attributes": { + "rules": { + "proto_library_with_info": [ + "", + "" + ], + "moved_proto_library": [ + "", + "" + ], + "java_proto_library": [ + "", + "" + ], + "java_grpc_library": [ + "", + "" + ], + "java_gapic_library": [ + "", + "" + ], + "java_gapic_test": [ + "", + "" + ], + "java_gapic_assembly_gradle_pkg": [ + "", + "" + ], + "py_proto_library": [ + "", + "" + ], + "py_grpc_library": [ + "", + "" + ], + "py_gapic_library": [ + "", + "" + ], + "py_test": [ + "", + "" + ], + "py_gapic_assembly_pkg": [ + "", + "" + ], + "py_import": [ + "", + "" + ], + "go_proto_library": [ + "", + "" + ], + "go_library": [ + "", + "" + ], + "go_test": [ + "", + "" + ], + "go_gapic_library": [ + "", + "" + ], + "go_gapic_assembly_pkg": [ + "", + "" + ], + "cc_proto_library": [ + "native.cc_proto_library", + "" + ], + "cc_grpc_library": [ + "", + "" + ], + "cc_gapic_library": [ + "", + "" + ], + "php_proto_library": [ + "", + "php_proto_library" + ], + "php_grpc_library": [ + "", + "php_grpc_library" + ], + "php_gapic_library": [ + "", + "php_gapic_library" + ], + "php_gapic_assembly_pkg": [ + "", + "php_gapic_assembly_pkg" + ], + "nodejs_gapic_library": [ + "", + "typescript_gapic_library" + ], + "nodejs_gapic_assembly_pkg": [ + "", + "typescript_gapic_assembly_pkg" + ], + "ruby_proto_library": [ + "", + "" + ], + "ruby_grpc_library": [ + "", + "" + ], + "ruby_ads_gapic_library": [ + "", + "" + ], + "ruby_cloud_gapic_library": [ + "", + "" + ], + "ruby_gapic_assembly_pkg": [ + "", + "" + ], + "csharp_proto_library": [ + "", + "" + ], + "csharp_grpc_library": [ + "", + "" + ], + "csharp_gapic_library": [ + "", + "" + ], + "csharp_gapic_assembly_pkg": [ + "", + "" + ] + } + } + } + }, + "recordedRepoMappingEntries": [ + [ + "cel-spec+", + "com_google_googleapis", + "cel-spec++non_module_dependencies+com_google_googleapis" + ] + ] + } + }, + "@@envoy_api+//bazel:repositories.bzl%non_module_deps": { + "general": { + "bzlTransitiveDigest": "JQHl7KCG4m+2+/1IddYzhdiav5qs2iUe+gkg5f+kUPA=", + "usagesDigest": "OP2p9QPxt4WFY/lah8Iecqt6SOUwZHs89hYRyaV+rSg=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "prometheus_metrics_model": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/prometheus/client_model/archive/v0.6.1.tar.gz" + ], + "sha256": "b9b690bc35d80061f255faa7df7621eae39fe157179ccd78ff6409c3b004f05e", + "strip_prefix": "client_model-0.6.1", + "build_file_content": "\nload(\"@envoy_api//bazel:api_build_system.bzl\", \"api_cc_py_proto_library\")\nload(\"@io_bazel_rules_go//proto:def.bzl\", \"go_proto_library\")\n\napi_cc_py_proto_library(\n name = \"client_model\",\n srcs = [\n \"io/prometheus/client/metrics.proto\",\n ],\n visibility = [\"//visibility:public\"],\n)\n\ngo_proto_library(\n name = \"client_model_go_proto\",\n importpath = \"github.com/prometheus/client_model/go\",\n proto = \":client_model\",\n visibility = [\"//visibility:public\"],\n)\n" + } + }, + "com_github_bufbuild_buf": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/bufbuild/buf/releases/download/v1.49.0/buf-Linux-x86_64.tar.gz" + ], + "sha256": "ee8da9748249f7946d79191e36469ce7bc3b8ba80019bff1fa4289a44cbc23bf", + "strip_prefix": "buf", + "build_file_content": "\npackage(\n default_visibility = [\"//visibility:public\"],\n)\n\nfilegroup(\n name = \"buf\",\n srcs = [\n \"@com_github_bufbuild_buf//:bin/buf\",\n ],\n tags = [\"manual\"], # buf is downloaded as a linux binary; tagged manual to prevent build for non-linux users\n)\n" + } + }, + "envoy_toolshed": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/envoyproxy/toolshed/archive/bazel-v0.2.2.tar.gz" + ], + "sha256": "443fe177aba0cef8c17b7a48905c925c67b09005b10dd70ff12cd9f729a72d51", + "strip_prefix": "toolshed-bazel-v0.2.2/bazel" + } + } + }, + "recordedRepoMappingEntries": [ + [ + "envoy_api+", + "bazel_tools", + "bazel_tools" + ], + [ + "envoy_api+", + "envoy_api", + "envoy_api+" + ] + ] + } + }, + "@@googleapis+//:extensions.bzl%switched_rules": { + "general": { + "bzlTransitiveDigest": "JLpojYeGHLdgnwSUuQWOrtkL9UMygBUUEFJZh7ASlmA=", + "usagesDigest": "66rLsrgz6YqDpU0pD+02mTGNxYxhb/of6Fh0ulHzjdE=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "com_google_googleapis_imports": { + "repoRuleId": "@@googleapis+//:repository_rules.bzl%switched_rules", + "attributes": { + "rules": { + "proto_library_with_info": [ + "", + "" + ], + "moved_proto_library": [ + "", + "" + ], + "java_proto_library": [ + "", + "" + ], + "java_grpc_library": [ + "", + "" + ], + "java_gapic_library": [ + "", + "" + ], + "java_gapic_test": [ + "", + "" + ], + "java_gapic_assembly_gradle_pkg": [ + "", + "" + ], + "py_proto_library": [ + "", + "" + ], + "py_grpc_library": [ + "", + "" + ], + "py_gapic_library": [ + "", + "" + ], + "py_test": [ + "", + "" + ], + "py_gapic_assembly_pkg": [ + "", + "" + ], + "py_import": [ + "", + "" + ], + "go_proto_library": [ + "", + "" + ], + "go_grpc_library": [ + "", + "" + ], + "go_library": [ + "", + "" + ], + "go_test": [ + "", + "" + ], + "go_gapic_library": [ + "", + "" + ], + "go_gapic_assembly_pkg": [ + "", + "" + ], + "cc_proto_library": [ + "", + "" + ], + "cc_grpc_library": [ + "", + "" + ], + "cc_gapic_library": [ + "", + "" + ], + "php_proto_library": [ + "", + "php_proto_library" + ], + "php_grpc_library": [ + "", + "php_grpc_library" + ], + "php_gapic_library": [ + "", + "php_gapic_library" + ], + "php_gapic_assembly_pkg": [ + "", + "php_gapic_assembly_pkg" + ], + "nodejs_gapic_library": [ + "", + "typescript_gapic_library" + ], + "nodejs_gapic_assembly_pkg": [ + "", + "typescript_gapic_assembly_pkg" + ], + "ruby_proto_library": [ + "", + "" + ], + "ruby_grpc_library": [ + "", + "" + ], + "ruby_ads_gapic_library": [ + "", + "" + ], + "ruby_cloud_gapic_library": [ + "", + "" + ], + "ruby_gapic_assembly_pkg": [ + "", + "" + ], + "csharp_proto_library": [ + "", + "" + ], + "csharp_grpc_library": [ + "", + "" + ], + "csharp_gapic_library": [ + "", + "" + ], + "csharp_gapic_assembly_pkg": [ + "", + "" + ] + } + } + } + }, + "recordedRepoMappingEntries": [] + } + }, + "@@rules_apple+//apple:apple.bzl%provisioning_profile_repository_extension": { + "general": { + "bzlTransitiveDigest": "uPXHPr6/YdHK89/x69cJN1hdhUF74hBsmCHaOkor2Fg=", + "usagesDigest": "vsJl8Rw5NL+5Ag2wdUDoTeRF/5klkXO8545Iy7U1Q08=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "local_provisioning_profiles": { + "repoRuleId": "@@rules_apple+//apple/internal:local_provisioning_profiles.bzl%provisioning_profile_repository", + "attributes": {} + } + }, + "recordedRepoMappingEntries": [ + [ + "apple_support+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "bazel_tools", + "rules_cc", + "rules_cc+" + ], + [ + "rules_apple+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "rules_apple+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_apple+", + "build_bazel_apple_support", + "apple_support+" + ], + [ + "rules_apple+", + "build_bazel_rules_swift", + "rules_swift+" + ], + [ + "rules_cc+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_cc+", + "cc_compatibility_proxy", + "rules_cc++compatibility_proxy+cc_compatibility_proxy" + ], + [ + "rules_cc+", + "rules_cc", + "rules_cc+" + ], + [ + "rules_cc++compatibility_proxy+cc_compatibility_proxy", + "rules_cc", + "rules_cc+" + ], + [ + "rules_swift+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "rules_swift+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_swift+", + "build_bazel_apple_support", + "apple_support+" + ], + [ + "rules_swift+", + "build_bazel_rules_swift", + "rules_swift+" + ], + [ + "rules_swift+", + "build_bazel_rules_swift_local_config", + "rules_swift++non_module_deps+build_bazel_rules_swift_local_config" + ] + ] + } + }, + "@@rules_apple+//apple:extensions.bzl%non_module_deps": { + "general": { + "bzlTransitiveDigest": "fe3Htkk5VXHdsIrdLtxsT4oUgADmqgmxwx83bja4UqY=", + "usagesDigest": "M3VqFpeTCo4qmrNKGZw0dxBHvTYDrfV3cscGzlSAhQ4=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "xctestrunner": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/google/xctestrunner/archive/b7698df3d435b6491b4b4c0f9fc7a63fbed5e3a6.tar.gz" + ], + "strip_prefix": "xctestrunner-b7698df3d435b6491b4b4c0f9fc7a63fbed5e3a6", + "sha256": "ae3a063c985a8633cb7eb566db21656f8db8eb9a0edb8c182312c7f0db53730d" + } + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_apple+", + "bazel_tools", + "bazel_tools" + ] + ] + } + }, "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { "general": { "bzlTransitiveDigest": "AEsQvKwdstptzt98AO1bJYfoWpPUn2upx53ra6o/Rc8=", @@ -764,6 +1353,80 @@ ] } }, + "@@rules_perl+//perl:extensions.bzl%perl_repositories": { + "general": { + "bzlTransitiveDigest": "hVM1gWzR35NfpY87MWdNF1xPgjhXCgJv3lHedzj33TY=", + "usagesDigest": "qSSNDdCNVxNhY36wMndEAFacdhR0ooLTmumfad0km9s=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "perl_darwin_arm64": { + "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", + "attributes": { + "strip_prefix": "perl-darwin-arm64", + "sha256": "285769f3c50c339fb59a3987b216ae3c5c573b95babe6875a1ef56fb178433da", + "urls": [ + "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-darwin-arm64.tar.xz" + ] + } + }, + "perl_darwin_amd64": { + "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", + "attributes": { + "strip_prefix": "perl-darwin-amd64", + "sha256": "63bc5ee36f5394d71c50cca6cafdd333ee58f9eaa40bca63c85f9bd06f2c1fd6", + "urls": [ + "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-darwin-amd64.tar.xz" + ] + } + }, + "perl_linux_amd64": { + "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", + "attributes": { + "strip_prefix": "perl-linux-amd64", + "sha256": "3bdffa9d7a3f97c0207314637b260ba5115b1d0829f97e3e2e301191a4d4d076", + "urls": [ + "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-linux-amd64.tar.xz" + ] + } + }, + "perl_linux_arm64": { + "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", + "attributes": { + "strip_prefix": "perl-linux-arm64", + "sha256": "6fa4ece99e790ecbc2861f6ecb7b52694c01c2eeb215b4370f16a3b12d952117", + "urls": [ + "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-linux-arm64.tar.xz" + ] + } + }, + "perl_windows_x86_64": { + "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", + "attributes": { + "strip_prefix": "", + "sha256": "aeb973da474f14210d3e1a1f942dcf779e2ae7e71e4c535e6c53ebabe632cc98", + "urls": [ + "https://mirror.bazel.build/strawberryperl.com/download/5.32.1.1/strawberry-perl-5.32.1.1-64bit.zip", + "https://strawberryperl.com/download/5.32.1.1/strawberry-perl-5.32.1.1-64bit.zip" + ] + } + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_perl+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_perl+", + "rules_perl", + "rules_perl+" + ] + ] + } + }, "@@rules_python+//python/uv:uv.bzl%uv": { "general": { "bzlTransitiveDigest": "LM8+g04lV+afYP67YcE7wenTJMrNqvF5u/ZnOctwctY=", @@ -936,6 +1599,165 @@ ] } }, + "@@rules_swift+//swift:extensions.bzl%non_module_deps": { + "general": { + "bzlTransitiveDigest": "vwewHB1JdoOb38MyyeRQPIdGOkEKD52YNUYgGwmeOy8=", + "usagesDigest": "mhACFnrdMv9Wi0Mt67bxocJqviRkDSV+Ee5Mqdj5akA=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "com_github_apple_swift_protobuf": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-protobuf/archive/1.20.2.tar.gz" + ], + "sha256": "3fb50bd4d293337f202d917b6ada22f9548a0a0aed9d9a4d791e6fbd8a246ebb", + "strip_prefix": "swift-protobuf-1.20.2/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_protobuf/BUILD.overlay" + } + }, + "com_github_grpc_grpc_swift": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/grpc/grpc-swift/archive/1.16.0.tar.gz" + ], + "sha256": "58b60431d0064969f9679411264b82e40a217ae6bd34e17096d92cc4e47556a5", + "strip_prefix": "grpc-swift-1.16.0/", + "build_file": "@@rules_swift+//third_party:com_github_grpc_grpc_swift/BUILD.overlay" + } + }, + "com_github_apple_swift_docc_symbolkit": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-docc-symbolkit/archive/refs/tags/swift-5.10-RELEASE.tar.gz" + ], + "sha256": "de1d4b6940468ddb53b89df7aa1a81323b9712775b0e33e8254fa0f6f7469a97", + "strip_prefix": "swift-docc-symbolkit-swift-5.10-RELEASE", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_docc_symbolkit/BUILD.overlay" + } + }, + "com_github_apple_swift_nio": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio/archive/2.42.0.tar.gz" + ], + "sha256": "e3304bc3fb53aea74a3e54bd005ede11f6dc357117d9b1db642d03aea87194a0", + "strip_prefix": "swift-nio-2.42.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_http2": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-http2/archive/1.26.0.tar.gz" + ], + "sha256": "f0edfc9d6a7be1d587e5b403f2d04264bdfae59aac1d74f7d974a9022c6d2b25", + "strip_prefix": "swift-nio-http2-1.26.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_http2/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_transport_services": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-transport-services/archive/1.15.0.tar.gz" + ], + "sha256": "f3498dafa633751a52b9b7f741f7ac30c42bcbeb3b9edca6d447e0da8e693262", + "strip_prefix": "swift-nio-transport-services-1.15.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_transport_services/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_extras": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-extras/archive/1.4.0.tar.gz" + ], + "sha256": "4684b52951d9d9937bb3e8ccd6b5daedd777021ef2519ea2f18c4c922843b52b", + "strip_prefix": "swift-nio-extras-1.4.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_extras/BUILD.overlay" + } + }, + "com_github_apple_swift_log": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-log/archive/1.4.4.tar.gz" + ], + "sha256": "48fe66426c784c0c20031f15dc17faf9f4c9037c192bfac2f643f65cb2321ba0", + "strip_prefix": "swift-log-1.4.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_log/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_ssl": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-ssl/archive/2.23.0.tar.gz" + ], + "sha256": "4787c63f61dd04d99e498adc3d1a628193387e41efddf8de19b8db04544d016d", + "strip_prefix": "swift-nio-ssl-2.23.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_ssl/BUILD.overlay" + } + }, + "com_github_apple_swift_collections": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-collections/archive/1.0.4.tar.gz" + ], + "sha256": "d9e4c8a91c60fb9c92a04caccbb10ded42f4cb47b26a212bc6b39cc390a4b096", + "strip_prefix": "swift-collections-1.0.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_collections/BUILD.overlay" + } + }, + "com_github_apple_swift_atomics": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-atomics/archive/1.1.0.tar.gz" + ], + "sha256": "1bee7f469f7e8dc49f11cfa4da07182fbc79eab000ec2c17bfdce468c5d276fb", + "strip_prefix": "swift-atomics-1.1.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_atomics/BUILD.overlay" + } + }, + "build_bazel_rules_swift_index_import": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file": "@@rules_swift+//third_party:build_bazel_rules_swift_index_import/BUILD.overlay", + "canonical_id": "index-import-5.8", + "urls": [ + "https://github.com/MobileNativeFoundation/index-import/releases/download/5.8.0.1/index-import.tar.gz" + ], + "sha256": "28c1ffa39d99e74ed70623899b207b41f79214c498c603915aef55972a851a15" + } + }, + "build_bazel_rules_swift_local_config": { + "repoRuleId": "@@rules_swift+//swift/internal:swift_autoconfiguration.bzl%swift_autoconfiguration", + "attributes": {} + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_swift+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_swift+", + "build_bazel_rules_swift", + "rules_swift+" + ] + ] + } + }, "@@rules_zig+//zig:extensions.bzl%zig": { "general": { "bzlTransitiveDigest": "xx2lb1ba6RcnnXdfUyaxI5j2wFLM03GTQfTv2UUie3Q=", diff --git a/examples/llm/models.zig b/examples/llm/models.zig index 2c5c7e26..4f364f22 100644 --- a/examples/llm/models.zig +++ b/examples/llm/models.zig @@ -9,6 +9,7 @@ pub const parseConfig = common.parseConfig; pub const lfm2 = @import("models/lfm2.zig"); pub const llama = @import("models/llama.zig"); pub const qwen3_5 = @import("models/qwen3_5.zig"); +pub const qwen3_5_moe = @import("models/qwen3_5_moe.zig"); const log = std.log.scoped(.llm); @@ -16,6 +17,7 @@ pub const ModelType = enum { lfm2, llama, qwen3_5, + qwen3_5_moe, }; const RawConfig = struct { @@ -26,6 +28,7 @@ pub const LoadedModel = union(ModelType) { lfm2: lfm2.LoadedModel, llama: llama.LoadedModel, qwen3_5: qwen3_5.LoadedModel, + qwen3_5_moe: qwen3_5_moe.LoadedModel, pub fn load( allocator: std.mem.Allocator, @@ -41,6 +44,7 @@ pub const LoadedModel = union(ModelType) { .lfm2 => .{ .lfm2 = try lfm2.LoadedModel.init(allocator, io, repo, store, generation) }, .llama => .{ .llama = try llama.LoadedModel.init(allocator, io, repo, store, generation) }, .qwen3_5 => .{ .qwen3_5 = try qwen3_5.LoadedModel.init(allocator, io, repo, store, generation) }, + .qwen3_5_moe => .{ .qwen3_5_moe = try qwen3_5_moe.LoadedModel.init(allocator, io, repo, store, generation) }, }; } @@ -55,6 +59,7 @@ pub const LoadedModel = union(ModelType) { .lfm2 => |*m| .{ .lfm2 = try m.loadBuffers(allocator, io, platform, store, progress, shardings) }, .llama => |*m| .{ .llama = try m.loadBuffers(allocator, io, platform, store, progress, shardings) }, .qwen3_5 => |*m| .{ .qwen3_5 = try m.loadBuffers(allocator, io, platform, store, progress, shardings) }, + .qwen3_5_moe => |*m| .{ .qwen3_5_moe = try m.loadBuffers(allocator, io, platform, store, progress, shardings) }, }; } @@ -72,6 +77,10 @@ pub const LoadedModel = union(ModelType) { .qwen3_5 => |*loaded_buffers| loaded_model.unloadBuffers(loaded_buffers, allocator), else => unreachable, }, + .qwen3_5_moe => |*loaded_model| switch (buffers.*) { + .qwen3_5_moe => |*loaded_buffers| loaded_model.unloadBuffers(loaded_buffers, allocator), + else => unreachable, + }, } } @@ -113,6 +122,15 @@ pub const LoadedModel = union(ModelType) { seqlen, progress, ) }, + .qwen3_5_moe => |*m| .{ .qwen3_5_moe = try m.compile( + allocator, + io, + platform, + backend, + shardings, + seqlen, + progress, + ) }, }; return .{ .inner = inner, @@ -126,6 +144,7 @@ pub const CompiledModel = struct { lfm2: lfm2.inference.CompiledModel, llama: llama.inference.CompiledModel, qwen3_5: qwen3_5.inference.CompiledModel, + qwen3_5_moe: qwen3_5_moe.inference.CompiledModel, }; inner: Inner, @@ -136,6 +155,7 @@ pub const CompiledModel = struct { .lfm2 => |*b| b.deinit(), .llama => |*b| b.deinit(), .qwen3_5 => |*b| b.deinit(), + .qwen3_5_moe => |*b| b.deinit(), } } @@ -181,6 +201,17 @@ pub const CompiledModel = struct { ) }, .seqlen = self.seqlen, }, + .qwen3_5_moe => |*compiled| .{ + .inner = .{ .qwen3_5_moe = try qwen3_5_moe.Session.init( + allocator, + io, + platform, + tokenizer, + compiled, + &model_buffers.qwen3_5_moe, + ) }, + .seqlen = self.seqlen, + }, }; } }; @@ -189,6 +220,7 @@ pub const Buffers = union(ModelType) { lfm2: lfm2.Buffers, llama: llama.Buffers, qwen3_5: qwen3_5.Buffers, + qwen3_5_moe: qwen3_5_moe.Buffers, }; pub const Session = struct { @@ -196,6 +228,7 @@ pub const Session = struct { lfm2: lfm2.Session, llama: llama.Session, qwen3_5: qwen3_5.Session, + qwen3_5_moe: qwen3_5_moe.Session, }; inner: Inner, diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 2ec23dba..f6a5c2c4 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -671,7 +671,7 @@ pub const GatedDeltaNet = struct { RmsNormGated.unloadBuffers(&self.norm); } - fn recurrent_gated_delta_rule(query: zml.Tensor, key: zml.Tensor, value: zml.Tensor, g: zml.Tensor, beta: zml.Tensor, initial_state: ?zml.Tensor) struct { zml.Tensor, zml.Tensor } { + fn recurrentGatedDeltaRule(query: zml.Tensor, key: zml.Tensor, value: zml.Tensor, g: zml.Tensor, beta: zml.Tensor, initial_state: ?zml.Tensor) struct { zml.Tensor, zml.Tensor } { const scale: f32 = 1.0 / @sqrt(@as(f32, @floatFromInt(query.dim(.khd)))); const query_norm = zml.nn.normalizeL2(query.rename(.{ .kh = .vh }), 1e-6); const key_norm = zml.nn.normalizeL2(key.rename(.{ .kh = .vh }), 1e-6); @@ -782,7 +782,7 @@ pub const GatedDeltaNet = struct { const query_for_rule = if (self.qk_head_repetition == 1) query else query.stutter1d(@intCast(query.axis(.kh)), @intCast(self.qk_head_repetition)); const key_for_rule = if (self.qk_head_repetition == 1) key else key.stutter1d(@intCast(key.axis(.kh)), @intCast(self.qk_head_repetition)); - const core_attn_out, const last_recurrent_state = recurrent_gated_delta_rule( + const core_attn_out, const last_recurrent_state = recurrentGatedDeltaRule( query_for_rule, key_for_rule, value, diff --git a/examples/llm/models/qwen3_5_moe.zig b/examples/llm/models/qwen3_5_moe.zig new file mode 100644 index 00000000..a97a3207 --- /dev/null +++ b/examples/llm/models/qwen3_5_moe.zig @@ -0,0 +1,11 @@ +pub const inference = @import("qwen3_5_moe/inference.zig"); +pub const CompilationParameters = inference.CompilationParameters; +pub const CompilationOptions = inference.CompilationParameters; +pub const CompiledModel = inference.CompiledModel; +pub const model = @import("qwen3_5_moe/model.zig"); +pub const Config = model.Config; +pub const Buffers = model.Buffers; +pub const Model = model.Model; +pub const LoadedModel = model.LoadedModel; +pub const session = @import("qwen3_5_moe/session.zig"); +pub const Session = session.Session; diff --git a/examples/llm/models/qwen3_5_moe/inference.zig b/examples/llm/models/qwen3_5_moe/inference.zig new file mode 100644 index 00000000..9a8ecc35 --- /dev/null +++ b/examples/llm/models/qwen3_5_moe/inference.zig @@ -0,0 +1,464 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const common = @import("../common.zig"); +const model = @import("model.zig"); + +const log = std.log.scoped(.qwen3_5_moe); + +const CompileModelResult = struct { + prefill_embedding_exe: zml.Exe, + decode_embedding_exe: zml.Exe, + prefill_full_layer_exe: ?zml.Exe, + decode_full_layer_exe: ?zml.Exe, + prefill_linear_layer_exe: ?zml.Exe, + decode_linear_layer_exe: ?zml.Exe, + prefill_sampling_exe: zml.Exe, + decode_sampling_exe: zml.Exe, +}; + +pub const CompilationParameters = struct { + kv_cache: model.KvCache, + rng: zml.Tensor.Rng, + prefill_moe_metadata: zml.moe.Metadata, + decode_moe_metadata: zml.moe.Metadata, + moe_parameters: zml.moe.Parameters, + seqlen: u32, + shardings: common.Shardings, + xla_dump_to: ?[]const u8, + + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, moe_backend: zml.moe.Backend, shardings: common.Shardings) CompilationParameters { + const dtype = mdl.text_model.embed_tokens.weight.dtype(); + return .{ + .kv_cache = .init(config, 1, seqlen, dtype, .f32), + .rng = .init(), + .prefill_moe_metadata = initMoeMetadata(mdl, @intCast(seqlen), 1, moe_backend), + .decode_moe_metadata = initMoeMetadata(mdl, 1, 1, moe_backend), + .moe_parameters = .init(.fromBackend(moe_backend, config.text_config.num_experts_per_tok, zml.moe.triton.Parameters.ActivationMode.silu)), + .seqlen = seqlen, + .shardings = shardings, + .xla_dump_to = "/home/ubuntu/xla_dump", + }; + } +}; + +pub const CompilationOptions = CompilationParameters; + +pub const CompiledModel = struct { + loaded_model: *const model.LoadedModel, + prefill_embedding_exe: zml.Exe, + decode_embedding_exe: zml.Exe, + prefill_full_layer_exe: ?zml.Exe, + decode_full_layer_exe: ?zml.Exe, + prefill_linear_layer_exe: ?zml.Exe, + decode_linear_layer_exe: ?zml.Exe, + prefill_sampling_exe: zml.Exe, + decode_sampling_exe: zml.Exe, + params: CompilationParameters, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + loaded_model: *const model.LoadedModel, + qwen_model: model.Model, + parameters: CompilationParameters, + progress: *std.Progress.Node, + ) !CompiledModel { + const compile_result = try compileModel(allocator, io, platform, qwen_model, parameters, progress); + return .{ + .loaded_model = loaded_model, + .prefill_embedding_exe = compile_result.prefill_embedding_exe, + .decode_embedding_exe = compile_result.decode_embedding_exe, + .prefill_full_layer_exe = compile_result.prefill_full_layer_exe, + .decode_full_layer_exe = compile_result.decode_full_layer_exe, + .prefill_linear_layer_exe = compile_result.prefill_linear_layer_exe, + .decode_linear_layer_exe = compile_result.decode_linear_layer_exe, + .prefill_sampling_exe = compile_result.prefill_sampling_exe, + .decode_sampling_exe = compile_result.decode_sampling_exe, + .params = parameters, + }; + } + + pub fn deinit(self: *CompiledModel) void { + self.prefill_embedding_exe.deinit(); + self.decode_embedding_exe.deinit(); + if (self.prefill_full_layer_exe) |exe| exe.deinit(); + if (self.decode_full_layer_exe) |exe| exe.deinit(); + if (self.prefill_linear_layer_exe) |exe| exe.deinit(); + if (self.decode_linear_layer_exe) |exe| exe.deinit(); + self.prefill_sampling_exe.deinit(); + self.decode_sampling_exe.deinit(); + } +}; + +pub const Inference = CompiledModel; + +fn compileSelfAttnLayerExe( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + mdl: model.TransformerLayer, + hidden: zml.Tensor, + token_index: zml.Tensor, + cache: model.KvCache.SelfAttnCache, + config: model.Config, + moe_metadata: zml.moe.Metadata, + moe_parameters: zml.moe.Parameters, + shardings: []const zml.Sharding, + xla_dump_to: ?[]const u8, + progress: *std.Progress.Node, + label: []const u8, +) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + const compiling_label = try std.fmt.allocPrint(allocator, "Compiling {s}...", .{label}); + defer allocator.free(compiling_label); + var node = progress.start(compiling_label, 1); + defer node.end(); + + const now: std.Io.Timestamp = .now(io, .awake); + defer log.info("Compiled {s} [{f}]", .{ label, now.untilNow(io, .awake) }); + + return platform.compile(allocator, io, mdl, .forwardSelfAttn, .{ hidden, token_index, cache, config, moe_metadata, moe_parameters }, .{ + .shardings = shardings, + .xla_dump_to = xla_dump_to, + }); +} + +fn compileLinearAttnLayerExe( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + mdl: model.TransformerLayer, + hidden: zml.Tensor, + token_index: zml.Tensor, + cache: model.KvCache.GatedDeltaNetCache, + config: model.Config, + moe_metadata: zml.moe.Metadata, + moe_parameters: zml.moe.Parameters, + shardings: []const zml.Sharding, + xla_dump_to: ?[]const u8, + progress: *std.Progress.Node, + label: []const u8, +) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + const compiling_label = try std.fmt.allocPrint(allocator, "Compiling {s}...", .{label}); + defer allocator.free(compiling_label); + var node = progress.start(compiling_label, 1); + defer node.end(); + + const now: std.Io.Timestamp = .now(io, .awake); + defer log.info("Compiled {s} [{f}]", .{ label, now.untilNow(io, .awake) }); + + return platform.compile(allocator, io, mdl, .forwardLinearAttn, .{ hidden, token_index, cache, config, moe_metadata, moe_parameters }, .{ + .shardings = shardings, + .xla_dump_to = xla_dump_to, + }); +} + +fn findFirstLayerIndex(layer_types: []const model.LayerType, target: model.LayerType) ?usize { + for (layer_types, 0..) |layer_type, index| { + if (layer_type == target) return index; + } + return null; +} + +fn compileModel( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + qwen35_model: model.Model, + parameters: CompilationParameters, + progress: *std.Progress.Node, +) !CompileModelResult { + const now: std.Io.Timestamp = .now(io, .awake); + const all_shardings = parameters.shardings.all(); + const prefill_len: usize = @intCast(parameters.seqlen); + defer log.info("Compiled model [{f}]", .{now.untilNow(io, .awake)}); + log.info("Compiling model for platform {any} with prefill length {d}...", .{ platform.target, prefill_len }); + + const prefill_tokens = zml.Tensor.init(.{ .b = 1, .s = prefill_len }, .u32); + const decode_tokens = zml.Tensor.init(.{ .b = 1, .s = 1 }, .u32); + const prefill_hidden = zml.Tensor.init(.{ .b = 1, .s = prefill_len, .d = qwen35_model.config.text_config.hidden_size }, qwen35_model.text_model.embed_tokens.weight.dtype()); + const decode_hidden = zml.Tensor.init(.{ .b = 1, .s = 1, .d = qwen35_model.config.text_config.hidden_size }, qwen35_model.text_model.embed_tokens.weight.dtype()); + const token_index = zml.Tensor.init(.{}, .u32); + const self_attn_cache: model.KvCache.SelfAttnCache = .{ + .k = parameters.kv_cache.self_attn.k, + .v = parameters.kv_cache.self_attn.v, + .layer_index = zml.Tensor.init(.{}, .u32), + }; + const linear_attn_cache: model.KvCache.GatedDeltaNetCache = .{ + .conv_state = parameters.kv_cache.gated_delta_net.conv_state, + .recurrent_state = parameters.kv_cache.gated_delta_net.recurrent_state, + .layer_index = zml.Tensor.init(.{}, .u32), + }; + + var prefill_embedding_future = try io.concurrent(struct { + fn call( + allocator_: std.mem.Allocator, + io_: std.Io, + platform_: *const zml.Platform, + model_: zml.nn.TokenEmbedding, + prefill_tokens_: zml.Tensor, + shardings_: []const zml.Sharding, + progress_: *std.Progress.Node, + ) !zml.Exe { + progress_.increaseEstimatedTotalItems(1); + var node_ = progress_.start("Compiling prefill embedding...", 1); + defer node_.end(); + const now_: std.Io.Timestamp = .now(io_, .awake); + defer log.info("Compiled prefill embedding [{f}]", .{now_.untilNow(io_, .awake)}); + return platform_.compile(allocator_, io_, model_, .forward, .{prefill_tokens_}, .{ .shardings = shardings_ }); + } + }.call, .{ allocator, io, platform, qwen35_model.text_model.embed_tokens, prefill_tokens, &all_shardings, progress }); + errdefer if (prefill_embedding_future.cancel(io)) |v| { + v.deinit(); + } else |_| {}; + + var decode_embedding_future = try io.concurrent(struct { + fn call( + allocator_: std.mem.Allocator, + io_: std.Io, + platform_: *const zml.Platform, + model_: zml.nn.TokenEmbedding, + decode_tokens_: zml.Tensor, + shardings_: []const zml.Sharding, + progress_: *std.Progress.Node, + ) !zml.Exe { + progress_.increaseEstimatedTotalItems(1); + var node_ = progress_.start("Compiling decode embedding...", 1); + defer node_.end(); + const now_: std.Io.Timestamp = .now(io_, .awake); + defer log.info("Compiled decode embedding [{f}]", .{now_.untilNow(io_, .awake)}); + return platform_.compile(allocator_, io_, model_, .forward, .{decode_tokens_}, .{ .shardings = shardings_ }); + } + }.call, .{ allocator, io, platform, qwen35_model.text_model.embed_tokens, decode_tokens, &all_shardings, progress }); + errdefer if (decode_embedding_future.cancel(io)) |v| { + v.deinit(); + } else |_| {}; + + const prefill_embedding_exe = try prefill_embedding_future.await(io); + const decode_embedding_exe = try decode_embedding_future.await(io); + + const full_layer_index = findFirstLayerIndex(qwen35_model.config.text_config.layer_types, .full_attention) orelse return error.MissingFullAttentionLayer; + const linear_layer_index = findFirstLayerIndex(qwen35_model.config.text_config.layer_types, .linear_attention) orelse return error.MissingLinearAttentionLayer; + const full_layer_model = qwen35_model.text_model.layers[full_layer_index]; + const linear_layer_model = qwen35_model.text_model.layers[linear_layer_index]; + + var prefill_full_layer_future = try io.concurrent(struct { + fn call( + allocator_: std.mem.Allocator, + io_: std.Io, + platform_: *const zml.Platform, + layer_model_: model.TransformerLayer, + hidden_: zml.Tensor, + token_index_: zml.Tensor, + cache_: model.KvCache.SelfAttnCache, + config_: model.Config, + moe_metadata_: zml.moe.Metadata, + moe_parameters_: zml.moe.Parameters, + shardings_: []const zml.Sharding, + xla_dump_to_: ?[]const u8, + progress_: *std.Progress.Node, + ) !zml.Exe { + return compileSelfAttnLayerExe(allocator_, io_, platform_, layer_model_, hidden_, token_index_, cache_, config_, moe_metadata_, moe_parameters_, shardings_, xla_dump_to_, progress_, "prefill full-attention layer..."); + } + }.call, .{ allocator, io, platform, full_layer_model, prefill_hidden, token_index, self_attn_cache, qwen35_model.config, parameters.prefill_moe_metadata, parameters.moe_parameters, &all_shardings, parameters.xla_dump_to, progress }); + errdefer if (prefill_full_layer_future.cancel(io)) |v| v.deinit() else |_| {}; + + var decode_full_layer_future = try io.concurrent(struct { + fn call( + allocator_: std.mem.Allocator, + io_: std.Io, + platform_: *const zml.Platform, + layer_model_: model.TransformerLayer, + hidden_: zml.Tensor, + token_index_: zml.Tensor, + cache_: model.KvCache.SelfAttnCache, + config_: model.Config, + moe_metadata_: zml.moe.Metadata, + moe_parameters_: zml.moe.Parameters, + shardings_: []const zml.Sharding, + xla_dump_to_: ?[]const u8, + progress_: *std.Progress.Node, + ) !zml.Exe { + return compileSelfAttnLayerExe(allocator_, io_, platform_, layer_model_, hidden_, token_index_, cache_, config_, moe_metadata_, moe_parameters_, shardings_, xla_dump_to_, progress_, "decode full-attention layer..."); + } + }.call, .{ allocator, io, platform, full_layer_model, decode_hidden, token_index, self_attn_cache, qwen35_model.config, parameters.decode_moe_metadata, parameters.moe_parameters, &all_shardings, parameters.xla_dump_to, progress }); + errdefer if (decode_full_layer_future.cancel(io)) |v| v.deinit() else |_| {}; + + var prefill_full_layer_exe: ?zml.Exe = null; + errdefer if (prefill_full_layer_exe) |exe| exe.deinit(); + var decode_full_layer_exe: ?zml.Exe = null; + errdefer if (decode_full_layer_exe) |exe| exe.deinit(); + prefill_full_layer_exe = try prefill_full_layer_future.await(io); + decode_full_layer_exe = try decode_full_layer_future.await(io); + + var prefill_linear_layer_future = try io.concurrent(struct { + fn call( + allocator_: std.mem.Allocator, + io_: std.Io, + platform_: *const zml.Platform, + layer_model_: model.TransformerLayer, + hidden_: zml.Tensor, + token_index_: zml.Tensor, + cache_: model.KvCache.GatedDeltaNetCache, + config_: model.Config, + moe_metadata_: zml.moe.Metadata, + moe_parameters_: zml.moe.Parameters, + shardings_: []const zml.Sharding, + xla_dump_to_: ?[]const u8, + progress_: *std.Progress.Node, + ) !zml.Exe { + return compileLinearAttnLayerExe(allocator_, io_, platform_, layer_model_, hidden_, token_index_, cache_, config_, moe_metadata_, moe_parameters_, shardings_, xla_dump_to_, progress_, "prefill linear-attention layer..."); + } + }.call, .{ allocator, io, platform, linear_layer_model, prefill_hidden, token_index, linear_attn_cache, qwen35_model.config, parameters.prefill_moe_metadata, parameters.moe_parameters, &all_shardings, parameters.xla_dump_to, progress }); + errdefer if (prefill_linear_layer_future.cancel(io)) |v| v.deinit() else |_| {}; + + var decode_linear_layer_future = try io.concurrent(struct { + fn call( + allocator_: std.mem.Allocator, + io_: std.Io, + platform_: *const zml.Platform, + layer_model_: model.TransformerLayer, + hidden_: zml.Tensor, + token_index_: zml.Tensor, + cache_: model.KvCache.GatedDeltaNetCache, + config_: model.Config, + moe_metadata_: zml.moe.Metadata, + moe_parameters_: zml.moe.Parameters, + shardings_: []const zml.Sharding, + xla_dump_to_: ?[]const u8, + progress_: *std.Progress.Node, + ) !zml.Exe { + return compileLinearAttnLayerExe(allocator_, io_, platform_, layer_model_, hidden_, token_index_, cache_, config_, moe_metadata_, moe_parameters_, shardings_, xla_dump_to_, progress_, "decode linear-attention layer..."); + } + }.call, .{ allocator, io, platform, linear_layer_model, decode_hidden, token_index, linear_attn_cache, qwen35_model.config, parameters.decode_moe_metadata, parameters.moe_parameters, &all_shardings, parameters.xla_dump_to, progress }); + errdefer if (decode_linear_layer_future.cancel(io)) |v| v.deinit() else |_| {}; + + var prefill_linear_layer_exe: ?zml.Exe = null; + errdefer if (prefill_linear_layer_exe) |exe| exe.deinit(); + var decode_linear_layer_exe: ?zml.Exe = null; + errdefer if (decode_linear_layer_exe) |exe| exe.deinit(); + + prefill_linear_layer_exe = try prefill_linear_layer_future.await(io); + decode_linear_layer_exe = try decode_linear_layer_future.await(io); + + var prefill_sampling_future = try io.concurrent(struct { + fn call( + allocator_: std.mem.Allocator, + io_: std.Io, + platform_: *const zml.Platform, + sampler_: model.Sampler, + prefill_hidden_: zml.Tensor, + rng_: zml.Tensor.Rng, + shardings_: []const zml.Sharding, + progress_: *std.Progress.Node, + ) !zml.Exe { + progress_.increaseEstimatedTotalItems(1); + var node_ = progress_.start("Compiling prefill sampling...", 1); + defer node_.end(); + const now_: std.Io.Timestamp = .now(io_, .awake); + defer log.info("Compiled prefill sampling [{f}]", .{now_.untilNow(io_, .awake)}); + return platform_.compile(allocator_, io_, sampler_, .sampleTokens, .{ prefill_hidden_, rng_, null }, .{ .shardings = shardings_ }); + } + }.call, .{ allocator, io, platform, qwen35_model.text_model.sampler(), prefill_hidden, parameters.rng, &all_shardings, progress }); + errdefer if (prefill_sampling_future.cancel(io)) |v| { + v.deinit(); + } else |_| {}; + const prefill_sampling_exe = try prefill_sampling_future.await(io); + + var decode_sampling_future = try io.concurrent(struct { + fn call( + allocator_: std.mem.Allocator, + io_: std.Io, + platform_: *const zml.Platform, + sampler_: model.Sampler, + decode_hidden_: zml.Tensor, + rng_: zml.Tensor.Rng, + token_index_: zml.Tensor, + shardings_: []const zml.Sharding, + progress_: *std.Progress.Node, + ) !zml.Exe { + progress_.increaseEstimatedTotalItems(1); + var node_ = progress_.start("Compiling decode sampling...", 1); + defer node_.end(); + const now_: std.Io.Timestamp = .now(io_, .awake); + defer log.info("Compiled decode sampling [{f}]", .{now_.untilNow(io_, .awake)}); + return platform_.compile(allocator_, io_, sampler_, .sampleTokens, .{ decode_hidden_, rng_, token_index_ }, .{ .shardings = shardings_ }); + } + }.call, .{ allocator, io, platform, qwen35_model.text_model.sampler(), decode_hidden, parameters.rng, token_index, &all_shardings, progress }); + errdefer if (decode_sampling_future.cancel(io)) |v| { + v.deinit(); + } else |_| {}; + + const decode_sampling_exe = try decode_sampling_future.await(io); + + return .{ + .prefill_embedding_exe = prefill_embedding_exe, + .decode_embedding_exe = decode_embedding_exe, + .prefill_full_layer_exe = prefill_full_layer_exe, + .decode_full_layer_exe = decode_full_layer_exe, + .prefill_linear_layer_exe = prefill_linear_layer_exe, + .decode_linear_layer_exe = decode_linear_layer_exe, + .prefill_sampling_exe = prefill_sampling_exe, + .decode_sampling_exe = decode_sampling_exe, + }; +} + +fn initMoeMetadata(qwen_model: model.Model, token_len: usize, batch_size: u32, backend: zml.moe.Backend) zml.moe.Metadata { + if (qwen_model.config.text_config.num_experts_per_tok == null) { + return .init(.fromBackend(backend)); + } + + var w1_zero_bias_shape: ?zml.Shape = null; + var w2_zero_bias_shape: ?zml.Shape = null; + var first_out_shape: ?zml.Shape = null; + var second_out_shape: ?zml.Shape = null; + + const num_experts_per_tok = qwen_model.config.text_config.num_experts_per_tok.?; + const num_experts = qwen_model.config.text_config.num_experts.?; + + for (qwen_model.text_model.layers) |layer| { + const gate_up_shape = zml.Shape.init(.{ + .expert = num_experts, + .out = layer.moe.shared_expert.gate_proj.weight.dim(.dout), + }, .bf16); + const down_shape = zml.Shape.init(.{ + .expert = num_experts, + .out = layer.moe.shared_expert.gate_proj.weight.dim(.d), + }, .bf16); + const first_out = zml.Shape.init(.{ + .total_tokens = batch_size * token_len * num_experts_per_tok, + .out = layer.moe.shared_expert.gate_proj.weight.dim(.dout) * 2, + }, .bf16); + const second_out = zml.Shape.init(.{ + .token = batch_size * token_len, + .topk = num_experts_per_tok, + .out = layer.moe.shared_expert.down_proj.weight.dim(.d), + }, .bf16); + + if (w1_zero_bias_shape == null) { + w1_zero_bias_shape = gate_up_shape; + w2_zero_bias_shape = down_shape; + first_out_shape = first_out; + second_out_shape = second_out; + continue; + } + + if (!w1_zero_bias_shape.?.eql(gate_up_shape) or !w2_zero_bias_shape.?.eql(down_shape) or !first_out_shape.?.eql(first_out) or !second_out_shape.?.eql(second_out)) { + log.warn("MoE bias shapes differ across layers; using shapes from the first layer", .{}); + break; + } + } + + return switch (backend) { + .triton => .init(.{ + .triton = .{ + .w1_zero_bias_shape = w1_zero_bias_shape, + .w2_zero_bias_shape = w2_zero_bias_shape, + }, + }), + }; +} diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig new file mode 100644 index 00000000..96f9c9d2 --- /dev/null +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -0,0 +1,1376 @@ +const std = @import("std"); + +const zml = @import("zml"); +const CompilationContext = zml.module.CompilationContext; + +const stdx = zml.stdx; + +const common = @import("../common.zig"); +const inference = @import("inference.zig"); + +const log = std.log.scoped(.qwen3_5); + +pub const Config = struct { + text_config: TextConfig, +}; + +pub const TextConfig = struct { + // General + num_hidden_layers: i64, + layer_types: []const LayerType, + hidden_size: i64, + max_position_embeddings: i64, + rms_norm_eps: f32, + // Self attention + head_dim: i64, + num_attention_heads: i64, + num_key_value_heads: i64, + rope_parameters: RopeParameters, + // Linear attention + linear_conv_kernel_dim: i64, + linear_key_head_dim: i64, + linear_num_key_heads: i64, + linear_num_value_heads: i64, + linear_value_head_dim: i64, + // MoE + num_experts: ?i64 = null, + num_experts_per_tok: ?u32 = null, +}; + +// Each layer uses either: full attention (SelfAttn) or linear attention (GatedDeltaNet). +pub const LayerType = enum { + linear_attention, + full_attention, +}; + +pub const RopeParameters = struct { + mrope_section: [3]i64, + partial_rotary_factor: f32, + rope_theta: f32, +}; + +fn kvHeadsAreReplicated(axis_size: i64, model_partitions: i64) bool { + return axis_size > 0 and axis_size < model_partitions and @rem(model_partitions, axis_size) == 0; +} + +fn partitionProjectedKv(tensor: zml.Tensor, replicate_kv_heads: bool) zml.Tensor { + return if (replicate_kv_heads) + tensor.withPartitioning(.{ .s = .replicated, .h = .replicated, .hd = .replicated }) + else + tensor.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); +} + +fn partitionCachedKv(tensor: zml.Tensor, replicate_kv_heads: bool) zml.Tensor { + return if (replicate_kv_heads) + tensor.rename(.{ .s = .k }).withPartitioning(.{ .k = .replicated, .h = .replicated, .hd = .replicated }) + else + tensor.rename(.{ .s = .k }).withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); +} + +pub const LoadedModel = struct { + inner: Model, + parsed_config: std.json.Parsed(Config), + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + repo: std.Io.Dir, + store: zml.io.TensorStore.View, + generation: common.GenerationOptions, + ) !LoadedModel { + const parsed_config = try common.parseConfig(Config, allocator, io, repo); + errdefer parsed_config.deinit(); + + const options: Model.GenOptions = .{ + .sampling_strategy = generation.sampling_strategy, + .max_seq_len = parsed_config.value.text_config.max_position_embeddings, + }; + + return .{ + .inner = try .init(allocator, store, parsed_config.value, options), + .parsed_config = parsed_config, + }; + } + + pub fn deinit(self: *LoadedModel, allocator: std.mem.Allocator) void { + self.inner.deinit(allocator); + self.parsed_config.deinit(); + } + + pub fn loadBuffers( + self: *LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + shardings: common.Shardings, + ) !Buffers { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + var total_bytes: usize = 0; + defer { + const took = now.untilNow(io, .awake); + const bytes_per_sec: u64 = @intFromFloat( + @as(f64, @floatFromInt(total_bytes)) / + (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s), + ); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + } + + const all_shardings = shardings.all(); + var buffers = try zml.io.load(Model, &self.inner, allocator, io, platform, store, .{ + .dma_chunks = 32, + .dma_chunk_size = 128 * zml.MiB, + .progress = progress, + .shardings = &all_shardings, + .parallelism = 16, + .total_bytes = &total_bytes, + }); + errdefer TextModel.unloadBuffers(&buffers.text_model, allocator); + return buffers; + } + + pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { + _ = self; + TextModel.unloadBuffers(&buffers.text_model, allocator); + } + + pub fn compile( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + backend: zml.attention.attention.Backend, + shardings: common.Shardings, + seqlen: usize, + progress: *std.Progress.Node, + ) !inference.CompiledModel { + _ = backend; + const moe_dtype = self.inner.text_model.layers[0].moe.gate_up_proj.dtype(); + log.info("Moe dtype : {}", .{moe_dtype}); + const moe_backend = try zml.moe.Backend.auto(platform, moe_dtype); + const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), moe_backend, shardings); + return inference.CompiledModel.init(allocator, io, platform, self, self.inner, params, progress); + } +}; + +pub const Buffers = zml.Bufferized(Model); + +pub const Model = struct { + pub const GenOptions = struct { sampling_strategy: zml.nn.SamplingStrategy = .{}, max_seq_len: i64 }; + + pub const SpecialTokens = struct { + im_start_token_id: u32, + im_end_token_id: u32, + end_of_text_token_id: u32, + }; + + text_model: TextModel, + + config: Config, + special_tokens: SpecialTokens = .{ + .im_start_token_id = 248045, + .im_end_token_id = 248046, + .end_of_text_token_id = 248044, + }, + + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config, gen_options: GenOptions) !Model { + // For some Qwen3.5 versions, the output projection lm_head has a standalone weight tensor, while for others it's the same as the input embedding layer + return .{ + .text_model = try .init(allocator, store.withPrefix("model.language_model"), config, gen_options), + .config = config, + }; + } + + pub fn deinit(self: Model, allocator: std.mem.Allocator) void { + self.text_model.deinit(allocator); + } + + pub fn load( + self: *const Model, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + shardings: []const zml.Sharding, + progress: *std.Progress.Node, + ) !zml.Bufferized(Model) { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + var total_bytes: usize = 0; + defer { + const took = now.untilNow(io, .awake); + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + } + return zml.io.load(Model, self, allocator, io, platform, store, .{ + .dma_chunks = 32, + .dma_chunk_size = 128 * zml.MiB, + .progress = progress, + .shardings = shardings, + .parallelism = 16, + .total_bytes = &total_bytes, + }); + } + + pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { + TextModel.unloadBuffers(&self.text_model, allocator); + } + pub fn forward( + self: Model, + tokens_: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache, + rng: zml.Tensor.Rng, + moe_metadata: zml.moe.Metadata, + moe_parameters: zml.moe.Parameters, + ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { + const tokens = tokens_.withPartialTags(.{.s}); + const new_tokens, const updated_kv_cache, const new_rng = self.text_model.forward(tokens, token_index, kv_cache, self.config, rng, moe_metadata, moe_parameters); + return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; + } +}; + +//========================Text model======================== + +// Use this intermediate struct to compile sampleTokens without taking the whole TextModel as input +pub const Sampler = struct { + norm: RmsNorm, + lm_head: zml.nn.Linear, + gen_options: Model.GenOptions, + + pub fn sampleTokens( + self: Sampler, + out: zml.Tensor, + rng: zml.Tensor.Rng, + token_index: ?zml.Tensor, + ) struct { zml.Tensor, zml.Tensor.Rng, ?zml.Tensor } { + const x = self.norm.forward(out); + const logits = self.lm_head.forward(x.withPartialTags(.{.d})).rename(.{ .dout = .voc }); + const next_tokens, const new_rng = zml.nn.sampleTokens(logits, self.gen_options.sampling_strategy, rng); + if (token_index) |token_idx| { + return .{ next_tokens.convert(.u32), new_rng, token_idx.addConstant(1) }; + } + return .{ next_tokens.convert(.u32), new_rng, null }; + } +}; + +pub const TextModel = struct { + embed_tokens: zml.nn.TokenEmbedding, + layers: []TransformerLayer, + norm: RmsNorm, + lm_head: zml.nn.Linear, + gen_options: Model.GenOptions, + + pub fn init( + allocator: std.mem.Allocator, + store: zml.io.TensorStore.View, + config: Config, + gen_options: Model.GenOptions, + ) !TextModel { + const lm_head_prefix = if (store.root().hasKey("lm_head.weight")) "lm_head" else "model.language_model.embed_tokens"; + + const layers = try allocator.alloc(TransformerLayer, @intCast(config.text_config.num_hidden_layers)); + errdefer allocator.free(layers); + + for (layers, 0..) |*layer, i| { + errdefer for (layers[0..i]) |previous_layer| previous_layer.deinit(allocator); + layer.* = try .init(allocator, store.withPrefix("layers").withLayer(i), config, i); + } + + return .{ + .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight", .{ .voc, .d }, .{ .voc = .replicated, .d = .model }) }, + .layers = layers, + .norm = RmsNorm.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), + .lm_head = .init(store.root().withPrefix(lm_head_prefix).createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), null, .d), + .gen_options = gen_options, + }; + } + + pub fn deinit(self: TextModel, allocator: std.mem.Allocator) void { + for (self.layers) |layer| { + layer.deinit(allocator); + } + allocator.free(self.layers); + } + + pub fn unloadBuffers(self: *zml.Bufferized(TextModel), allocator: std.mem.Allocator) void { + self.embed_tokens.weight.deinit(); + for (self.layers) |*layer| { + TransformerLayer.unloadBuffers(layer, allocator); + } + allocator.free(self.layers); + RmsNorm.unloadBuffers(&self.norm); + self.lm_head.weight.deinit(); + } + + pub fn sampler(self: TextModel) Sampler { + return .{ + .norm = self.norm, + .lm_head = self.lm_head, + .gen_options = self.gen_options, + }; + } + + pub fn forward( + self: TextModel, + tokens: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache, + config: Config, + rng: zml.Tensor.Rng, + moe_metadata: zml.moe.Metadata, + moe_parameters: zml.moe.Parameters, + ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { + var hidden_states = self.embed_tokens.weight.gather(.{ .voc = tokens }, .{}); + + var updated_kv_cache = kv_cache; + for (self.layers, 0..) |layer, i| { + hidden_states, updated_kv_cache = layer.forward(hidden_states, token_index, updated_kv_cache.atLayer(i), config, moe_metadata, moe_parameters); + } + + const new_tokens, const new_rng = self.sampleTokens(hidden_states, rng); + return .{ new_tokens, updated_kv_cache.reuseBuffer(kv_cache), new_rng }; + } + + pub fn sampleTokens( + self: TextModel, + out: zml.Tensor, + rng: zml.Tensor.Rng, + token_index: ?zml.Tensor, + ) struct { zml.Tensor, zml.Tensor.Rng, ?zml.Tensor } { + return self.sampler().sampleTokens(out, rng, token_index); + } +}; + +pub const TransformerLayer = struct { + const Attn = union(enum) { + self_attn: SelfAttn, + linear_attn: GatedDeltaNet, + }; + + input_layernorm: RmsNorm, + attn: Attn, + moe: Moe, + post_attention_layernorm: RmsNorm, + + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config, layer_index: usize) !TransformerLayer { + const is_full_attention = config.text_config.layer_types[layer_index] == .full_attention; + return .{ + .input_layernorm = RmsNorm.init(store.withPrefix("input_layernorm"), config.text_config.rms_norm_eps), + .attn = if (is_full_attention) + .{ .self_attn = try .init(store.withPrefix("self_attn"), config) } + else + .{ .linear_attn = .init(store.withPrefix("linear_attn"), config) }, + .moe = try .init(allocator, store.withPrefix("mlp"), config), + .post_attention_layernorm = RmsNorm.init(store.withPrefix("post_attention_layernorm"), config.text_config.rms_norm_eps), + }; + } + + pub fn deinit(self: TransformerLayer, allocator: std.mem.Allocator) void { + self.moe.deinit(allocator); + } + + pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer), allocator: std.mem.Allocator) void { + RmsNorm.unloadBuffers(&self.input_layernorm); + switch (self.attn) { + .self_attn => |*self_attn| SelfAttn.unloadBuffers(self_attn), + .linear_attn => |*linear_attn| GatedDeltaNet.unloadBuffers(linear_attn), + } + Moe.unloadBuffers(&self.moe, allocator); + RmsNorm.unloadBuffers(&self.post_attention_layernorm); + } + + pub fn forwardSelfAttn( + self: TransformerLayer, + x0: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache.SelfAttnCache, + config: Config, + moe_metadata: zml.moe.Metadata, + moe_parameters: zml.moe.Parameters, + ) struct { zml.Tensor, KvCache.SelfAttnCache } { + _ = config; + const residual0 = x0; + const normalized_x0 = self.input_layernorm.forward(x0); + + const self_attn = switch (self.attn) { + .self_attn => |self_attn| self_attn, + .linear_attn => unreachable, + }; + const attention_output, const updated_kv_cache = self_attn.forward(normalized_x0, token_index, kv_cache); + + const x1 = attention_output.add(residual0); + const residual1 = x1; + const normalized_hidden = self.post_attention_layernorm.forward(x1); + + const moe_output = self.moe.forward(normalized_hidden, moe_metadata, moe_parameters); + + return .{ moe_output.add(residual1).reuseBuffer(x0), updated_kv_cache }; + } + + pub fn forwardLinearAttn( + self: TransformerLayer, + x0: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache.GatedDeltaNetCache, + config: Config, + moe_metadata: zml.moe.Metadata, + moe_parameters: zml.moe.Parameters, + ) struct { zml.Tensor, KvCache.GatedDeltaNetCache } { + _ = config; + _ = token_index; + const residual0 = x0; + const normalized_x0 = self.input_layernorm.forward(x0); + + const linear_attn = switch (self.attn) { + .linear_attn => |linear_attn| linear_attn, + .self_attn => unreachable, + }; + const attention_output, const updated_kv_cache = linear_attn.forward(normalized_x0, kv_cache); + + const x1 = attention_output.add(residual0); + const residual1 = x1; + const normalized_hidden = self.post_attention_layernorm.forward(x1); + + const moe_output = self.moe.forward(normalized_hidden, moe_metadata, moe_parameters); + + return .{ moe_output.add(residual1).reuseBuffer(x0), updated_kv_cache }; + } + + pub fn forward( + self: TransformerLayer, + x0: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache.LayerView, + config: Config, + moe_metadata: zml.moe.Metadata, + moe_parameters: zml.moe.Parameters, + ) struct { zml.Tensor, KvCache } { + _ = config; + const residual0 = x0; + const normalized_x0 = self.input_layernorm.forward(x0); + + var attention_output: zml.Tensor = undefined; + var updated_kv_cache: KvCache = kv_cache.parent; + switch (self.attn) { + .self_attn => |*self_attn| { + const result = self_attn.forward(normalized_x0, token_index, kv_cache.cache.self_attn); + attention_output = result[0]; + updated_kv_cache.self_attn = result[1]; + }, + .linear_attn => |*linear_attn| { + const result = linear_attn.forward(normalized_x0, kv_cache.cache.linear_attn); + attention_output = result[0]; + updated_kv_cache.gated_delta_net = result[1]; + }, + } + + const x1 = attention_output.add(residual0); + const residual1 = x1; + const normalized_hidden = self.post_attention_layernorm.forward(x1); + + const moe_output = self.moe.forward(normalized_hidden, moe_metadata, moe_parameters); + + return .{ moe_output.add(residual1), updated_kv_cache }; + } +}; + +pub const SelfAttn = struct { + q_proj: zml.nn.Linear, + q_proj_scale: ?zml.Tensor, + k_proj: zml.nn.Linear, + k_proj_scale: ?zml.Tensor, + v_proj: zml.nn.Linear, + v_proj_scale: ?zml.Tensor, + + q_norm: RmsNorm, + k_norm: RmsNorm, + + num_heads: i64, + num_kv_heads: i64, + head_dim: i64, + rotary_dim: i64, + rotary_embed: TextRotaryEmbedding, + o_proj: zml.nn.Linear, + + o_proj_scale: ?zml.Tensor, + + fn initProj(store: zml.io.TensorStore.View, partitions: anytype, bias_partitions: anytype) zml.nn.Linear { + return .init( + store.createTensor("weight", .{ .dout, .d }, partitions), + store.maybeCreateTensor("bias", .{.dout}, bias_partitions), + .d, + ); + } + + pub fn init(store: zml.io.TensorStore.View, config: Config) !SelfAttn { + const rotary_dim: i64 = @intFromFloat(@as(f32, @floatFromInt(config.text_config.head_dim)) * + config.text_config.rope_parameters.partial_rotary_factor); + return .{ + .q_proj = initProj(store.withPrefix("q_proj"), .{ .dout = .model, .d = .replicated }, .{ .dout = .model }), + .q_proj_scale = null, + .k_proj = initProj(store.withPrefix("k_proj"), .{ .dout = .model, .d = .replicated }, .{ .dout = .model }), + .k_proj_scale = null, + .v_proj = initProj(store.withPrefix("v_proj"), .{ .dout = .model, .d = .replicated }, .{ .dout = .model }), + .v_proj_scale = null, + .o_proj = initProj(store.withPrefix("o_proj"), .{ .dout = .replicated, .d = .model }, .{ .dout = .replicated }), + .o_proj_scale = null, + .q_norm = RmsNorm.init(store.withPrefix("q_norm"), config.text_config.rms_norm_eps), + .k_norm = RmsNorm.init(store.withPrefix("k_norm"), config.text_config.rms_norm_eps), + .num_heads = config.text_config.num_attention_heads, + .num_kv_heads = config.text_config.num_key_value_heads, + .head_dim = config.text_config.head_dim, + .rotary_dim = rotary_dim, + .rotary_embed = .init(rotary_dim, config.text_config.rope_parameters.rope_theta, config.text_config.rope_parameters.mrope_section), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { + self.q_proj.weight.deinit(); + if (self.q_proj.bias) |*bias| bias.deinit(); + if (self.q_proj_scale) |*scale| scale.deinit(); + self.k_proj.weight.deinit(); + if (self.k_proj.bias) |*bias| bias.deinit(); + if (self.k_proj_scale) |*scale| scale.deinit(); + self.v_proj.weight.deinit(); + if (self.v_proj.bias) |*bias| bias.deinit(); + if (self.v_proj_scale) |*scale| scale.deinit(); + self.o_proj.weight.deinit(); + if (self.o_proj.bias) |*bias| bias.deinit(); + if (self.o_proj_scale) |*scale| scale.deinit(); + RmsNorm.unloadBuffers(&self.q_norm); + RmsNorm.unloadBuffers(&self.k_norm); + } + + fn projectQAndGate(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { + const q_proj = self.q_proj.forward(x) + .splitAxis(.dout, .{ .h = self.num_heads, .hd = 2 * self.head_dim }); + const q, var gate = q_proj.chunkExact(.hd, 2); + gate = gate.merge(.{ .d_out_proj = .{ .h, .hd } }); + return .{ q, gate }; + } + + fn projectKV(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { + const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; + const k = self.k_proj.forward(x) + .splitAxis(.dout, .{ .h = num_kv_heads, .hd = self.head_dim }); + const v = self.v_proj.forward(x) + .splitAxis(.dout, .{ .h = num_kv_heads, .hd = self.head_dim }); + return .{ k, v }; + } + + pub fn forward( + self: SelfAttn, + x: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache.SelfAttnCache, + ) struct { zml.Tensor, KvCache.SelfAttnCache } { + const model_partitions = zml.module.CompilationContext.current().partitioning.numPartitionsForLogicalAxis(self.q_proj.weight.shape(), .model) catch unreachable; + const replicate_kv_heads = kvHeadsAreReplicated(self.num_kv_heads, model_partitions); + const x_qkv = x.withPartitioning(.{ .d = .replicated }); + + var q, var gate = self.projectQAndGate(x_qkv); + var k, var v = self.projectKV(x_qkv); + q = q.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); + gate = gate.withPartitioning(.{ .s = .replicated, .d_out_proj = .model }); + k = partitionProjectedKv(k, replicate_kv_heads); + v = partitionProjectedKv(v, replicate_kv_heads); + q = self.q_norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); + k = self.k_norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); + + const dtype = q.dtype(); + const position_ids = zml.Tensor.arange(.{ .end = x.dim(.s) }, .i64) + .withTags(.{.s}).insertAxes(.s, .{.b}).broad(zml.Shape.init(.{ .b = x.dim(.b), .s = x.dim(.s) }, .i64)) + .add(token_index.convert(.i64).broad(zml.Shape.init(.{ .b = x.dim(.b), .s = x.dim(.s) }, .i64))); + + const cos, const sin = self.rotary_embed.getCosAndSin(position_ids, dtype); + q = self.rotary_embed.applyRope(q, cos, sin); + k = self.rotary_embed.applyRope(k, cos, sin); + q = q.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); + k = partitionProjectedKv(k, replicate_kv_heads); + v = partitionProjectedKv(v, replicate_kv_heads); + + const new_kv_cache = kv_cache.update(k, v, token_index.convert(.u32)); + k = new_kv_cache.keys().convert(dtype); + v = new_kv_cache.values().convert(dtype); + q = q.rename(.{ .s = .q }).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }); + k = partitionCachedKv(k, replicate_kv_heads); + v = partitionCachedKv(v, replicate_kv_heads); + + const attn_output = zml.attention.attention.attention( + q, + k, + v, + token_index, + zml.attention.attention.Metadata.init(.fromBackend(.vanilla, x.dim(.s), self.num_heads)), + zml.attention.attention.Parameters.init(.fromBackend(.vanilla)), + ).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); + + const gated_output = attn_output.mul(gate.sigmoid()); + const projected_output = self.o_proj.forward(gated_output.rename(.{ .d_out_proj = .d })).rename(.{ .dout = .d }).withPartitioning(.{ .d = .replicated }); + + return .{ projected_output, new_kv_cache }; + } +}; + +pub const Mlp = struct { + up_proj: zml.nn.Linear, + up_proj_scale: ?zml.Tensor, + gate_proj: zml.nn.Linear, + gate_proj_scale: ?zml.Tensor, + down_proj: zml.nn.Linear, + down_proj_scale: ?zml.Tensor, + + pub fn init(store: zml.io.TensorStore.View) Mlp { + return .{ + .up_proj = .init( + store.withPrefix("up_proj").createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), + store.withPrefix("up_proj").maybeCreateTensor("bias", .{.dout}, .{ .dout = .model }), + .d, + ), + .up_proj_scale = null, + .gate_proj = .init( + store.withPrefix("gate_proj").createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), + store.withPrefix("gate_proj").maybeCreateTensor("bias", .{.dout}, .{ .dout = .model }), + .d, + ), + .gate_proj_scale = null, + .down_proj = .init( + store.withPrefix("down_proj").createTensor("weight", .{ .dout, .d }, .{ .dout = .replicated, .d = .model }), + store.withPrefix("down_proj").maybeCreateTensor("bias", .{.d}, .{ .d = .replicated }), + .d, + ), + .down_proj_scale = null, + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { + self.up_proj.weight.deinit(); + if (self.up_proj.bias) |*bias| bias.deinit(); + if (self.up_proj_scale) |*scale| scale.deinit(); + self.gate_proj.weight.deinit(); + if (self.gate_proj.bias) |*bias| bias.deinit(); + if (self.gate_proj_scale) |*scale| scale.deinit(); + self.down_proj.weight.deinit(); + if (self.down_proj.bias) |*bias| bias.deinit(); + if (self.down_proj_scale) |*scale| scale.deinit(); + } + + pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { + const up_projed = self.up_proj.forward(x); + const gate = self.gate_proj.forward(x); + const hidden = gate.silu().mul(up_projed).rename(.{ .dout = .d }); + + const output = self.down_proj.forward(hidden); + return output; + } +}; + +const Router = struct { + router: zml.nn.Linear, + num_experts_per_tok: u32, + + pub fn init(store: zml.io.TensorStore.View, num_experts_per_tok: u32) Router { + return .{ + .router = .init( + store.createTensor("weight", .{ .expert, .d }, .{ .expert = .replicated, .d = .replicated }), + store.maybeCreateTensor("bias", .{.expert}, .{ .expert = .replicated }), + .d, + ), + .num_experts_per_tok = num_experts_per_tok, + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Router)) void { + self.router.weight.deinit(); + if (self.router.bias) |*bias| bias.deinit(); + } + + pub fn forward(self: Router, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { + const router_logits = self.router.forward(x).convert(.f32); + const routing = router_logits.topK(.{ .top_expert = .expert }, self.num_experts_per_tok, .{}); + const topk_ids = routing.indices.convert(.i32); + const router_scores = routing.values.softmax(.top_expert); + return .{ router_scores, topk_ids }; + } +}; + +pub const Moe = struct { + shared_expert: Mlp, + shared_expert_gate: zml.nn.Linear, + gate_up_proj: zml.Tensor, + down_proj: zml.Tensor, + router: Router, + + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config) !Moe { + _ = allocator; + const experts_store = store.withPrefix("experts"); + const gate_up_proj_tensor = experts_store.createTensor( + "gate_up_proj", + .{ .expert, .dout, .d }, + .{ .expert = .replicated, .dout = .model, .d = .replicated }, + ); + const down_proj_tensor = experts_store.createTensor( + "down_proj", + .{ .expert, .d, .dout }, + .{ .expert = .replicated, .d = .replicated, .dout = .model }, + ); + + return .{ + .shared_expert = Mlp.init(store.withPrefix("shared_expert")), + .shared_expert_gate = .init( + store.withPrefix("shared_expert_gate").createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), + store.withPrefix("shared_expert_gate").maybeCreateTensor("bias", .{.dout}, .{ .dout = .model }), + .d, + ), + .gate_up_proj = gate_up_proj_tensor, + .down_proj = down_proj_tensor, + .router = Router.init(store.withPrefix("gate"), config.text_config.num_experts_per_tok.?), + }; + } + + pub fn deinit(self: Moe, allocator: std.mem.Allocator) void { + _ = self; + _ = allocator; + } + + pub fn forward(self: Moe, x: zml.Tensor, moe_metadata: zml.moe.Metadata, moe_parameters: zml.moe.Parameters) zml.Tensor { + const routing_scores, const topk_ids = self.router.forward(x); + + const moe_output = zml.moe.forwardMoe( + x, + topk_ids, + routing_scores, + self.gate_up_proj, + null, + null, + self.down_proj, + null, + null, + moe_metadata, + moe_parameters, + ) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); + + const shared_gate = self.shared_expert_gate.forward(x).sigmoid().broad(x.shape()); + const shared = self.shared_expert.forward(x).mul(shared_gate); + + return moe_output.add(shared); + } + + pub fn unloadBuffers(self: *zml.Bufferized(Moe), allocator: std.mem.Allocator) void { + _ = allocator; + Mlp.unloadBuffers(&self.shared_expert); + self.shared_expert_gate.weight.deinit(); + if (self.shared_expert_gate.bias) |*bias| bias.deinit(); + self.gate_up_proj.deinit(); + self.down_proj.deinit(); + Router.unloadBuffers(&self.router); + } +}; + +pub const TextRotaryEmbedding = struct { + rope_opts: zml.nn.RopeOpts, + rotary_dim: i64, + mrope_section: [3]i64, + + pub fn init(rotary_dim: i64, theta: f32, mrope_section: [3]i64) TextRotaryEmbedding { + return .{ + .rope_opts = .{ + .layout = .sequential, + .scaling = .{ .default = .{ .rope_theta = theta } }, + }, + .rotary_dim = rotary_dim, + .mrope_section = mrope_section, + }; + } + + pub fn getCosAndSin(self: TextRotaryEmbedding, position_ids: zml.Tensor, dtype: zml.DataType) struct { zml.Tensor, zml.Tensor } { + const inv_freq = zml.nn.invFreq(self.rotary_dim, self.rope_opts).withTags(.{.hd}); + + const freqs_t = position_ids.convert(.f32).outer(inv_freq); + + const emb = zml.Tensor.concatenate(&.{ freqs_t, freqs_t }, -1); + const cos = emb.cos().convert(dtype); + const sin = emb.sin().convert(dtype); + + return .{ cos, sin }; + } + + pub fn getCosAndSinInterleaved(self: TextRotaryEmbedding, position_ids: zml.Tensor, dtype: zml.DataType) struct { zml.Tensor, zml.Tensor } { // To be used later in image extension + const stacked_position_ids = zml.Tensor.stack(&.{ position_ids, position_ids, position_ids }, 0, .g).convert(.f32); + const inv_freq = zml.nn.InvFreq(self.rotary_dim, self.rope_opts).withTags(.{.hd}); + + var freqs = stacked_position_ids.outer(inv_freq); + var freqs_t, var freqs_h, var freqs_w = freqs.chunkExact(.g, 3); + freqs_t = freqs_t.squeeze(.g); + freqs_h = freqs_h.squeeze(.g); + freqs_w = freqs_w.squeeze(.g); + + const h_indices = zml.Tensor.iota(zml.Shape.init(.{ .h = self.mrope_section[1] }, .i32), .h).scale(3).addConstant(1); + const w_indices = zml.Tensor.iota(zml.Shape.init(.{ .h = self.mrope_section[2] }, .i32), .h).scale(3).addConstant(2); + + const h_input = freqs_h.gather(.{ .dh = h_indices }, .{ .indices_are_sorted = true }); + const w_input = freqs_w.gather(.{ .dh = w_indices }, .{ .indices_are_sorted = true }); + freqs_t = freqs_t.scatterSlices(.{ .dh = h_indices }, h_input, .{ .update_fn = zml.Tensor.ScatterOpts.override }); + freqs = freqs_t.scatterSlices(.{ .dh = w_indices }, w_input, .{ .update_fn = zml.Tensor.ScatterOpts.override }); + + const emb = zml.Tensor.concatenate(&.{ freqs, freqs }, -1); + const cos = emb.cos().convert(dtype); + const sin = emb.sin().convert(dtype); + + return .{ cos, sin }; + } + + fn rotateHalf(x: zml.Tensor) zml.Tensor { + const half_dim = @divExact(x.dim(-1), 2); + const x1 = x.slice1d(-1, .{ .start = 0, .end = half_dim }); + const x2 = x.slice1d(-1, .{ .start = half_dim, .end = x.dim(-1) }); + return zml.Tensor.concatenate(&.{ x2.negate(), x1 }, -1); + } + + pub fn applyRope(self: TextRotaryEmbedding, x: zml.Tensor, cos: zml.Tensor, sin: zml.Tensor) zml.Tensor { + const x_rot = x.slice1d(-1, .{ .start = 0, .end = self.rotary_dim }); + const x_pass = x.slice1d(-1, .{ .start = self.rotary_dim, .end = x.dim(-1) }); + + const cos_x = cos.insertAxes(.hd, .{.h}).broad(x_rot.shape()); + const sin_x = sin.insertAxes(.hd, .{.h}).broad(x_rot.shape()); + + const rotated = x_rot.mul(cos_x).add(rotateHalf(x_rot).mul(sin_x)); + + return zml.Tensor.concatenate(&.{ rotated, x_pass }, -1); + } +}; + +pub const GatedDeltaNet = struct { + in_proj_qkv: zml.nn.Linear, + in_proj_qkv_scale: ?zml.Tensor, + in_proj_z: zml.nn.Linear, + in_proj_z_scale: ?zml.Tensor, + in_proj_b: zml.nn.Linear, + in_proj_a: zml.nn.Linear, + out_proj: zml.nn.Linear, + out_proj_scale: ?zml.Tensor, + conv1d_weight: zml.Tensor, + dt_bias: zml.Tensor, + aLog: zml.Tensor, + norm: RmsNormGated, + + num_k_heads: i64, + num_v_heads: i64, + qk_head_repetition: i64, + head_k_dim: i64, + head_v_dim: i64, + conv_kernel_size: i64, + + fn initProj(store: zml.io.TensorStore.View, partitions: anytype) zml.nn.Linear { + return .init(store.createTensor("weight", .{ .dout, .d }, partitions), null, .d); + } + + pub fn init(store: zml.io.TensorStore.View, config: Config) GatedDeltaNet { + const qk_head_repetition = + @divExact(config.text_config.linear_num_value_heads, config.text_config.linear_num_key_heads); + return .{ + .in_proj_qkv = initProj(store.withPrefix("in_proj_qkv"), .{ .dout = .model, .d = .replicated }), + .in_proj_qkv_scale = null, + .in_proj_z = initProj(store.withPrefix("in_proj_z"), .{ .dout = .model, .d = .replicated }), + .in_proj_z_scale = null, + .in_proj_b = initProj(store.withPrefix("in_proj_b"), .{ .dout = .model, .d = .replicated }), + .in_proj_a = initProj(store.withPrefix("in_proj_a"), .{ .dout = .model, .d = .replicated }), + .out_proj = initProj(store.withPrefix("out_proj"), .{ .dout = .replicated, .d = .model }), + .out_proj_scale = null, + .conv1d_weight = store.withPrefix("conv1d").createTensor("weight", .{ .out, .in, .kernel_size }, .{ .out = .model, .in = .replicated, .kernel_size = .replicated }), + .dt_bias = store.createTensor("dt_bias", .{.vh}, .{ .vh = .model }), + .aLog = store.createTensor("A_log", .{.vh}, .{ .vh = .model }), + .norm = RmsNormGated.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), + .num_k_heads = config.text_config.linear_num_key_heads, + .num_v_heads = config.text_config.linear_num_value_heads, + .qk_head_repetition = qk_head_repetition, + .head_k_dim = config.text_config.linear_key_head_dim, + .head_v_dim = config.text_config.linear_value_head_dim, + .conv_kernel_size = config.text_config.linear_conv_kernel_dim, + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(GatedDeltaNet)) void { + self.in_proj_qkv.weight.deinit(); + if (self.in_proj_qkv_scale) |*scale| scale.deinit(); + self.in_proj_z.weight.deinit(); + if (self.in_proj_z_scale) |*scale| scale.deinit(); + self.in_proj_b.weight.deinit(); + self.in_proj_a.weight.deinit(); + self.out_proj.weight.deinit(); + if (self.out_proj_scale) |*scale| scale.deinit(); + self.conv1d_weight.deinit(); + self.dt_bias.deinit(); + self.aLog.deinit(); + RmsNormGated.unloadBuffers(&self.norm); + } + + fn recurrentGatedDeltaRule(query: zml.Tensor, key: zml.Tensor, value: zml.Tensor, g: zml.Tensor, beta: zml.Tensor, initial_state: ?zml.Tensor) struct { zml.Tensor, zml.Tensor } { + const scale: f32 = 1.0 / @sqrt(@as(f32, @floatFromInt(query.dim(.khd)))); + const query_norm = zml.nn.normalizeL2(query.rename(.{ .kh = .vh }), 1e-6); + const key_norm = zml.nn.normalizeL2(key.rename(.{ .kh = .vh }), 1e-6); + + const query_f32, const key_f32, const value_f32, const alpha_f32, const beta_f32 = .{ + query_norm.convert(.f32).scale(scale).rename(.{ .vh = .h, .khd = .k }), + key_norm.convert(.f32).rename(.{ .vh = .h, .khd = .k }), + value.convert(.f32).rename(.{ .vh = .h, .vhd = .v }), + g.convert(.f32).exp().rename(.{ .vh = .h }), + beta.convert(.f32).rename(.{ .vh = .h }), + }; + + const initial_recurrent_state = if (initial_state) |state| + state.convert(.f32).transpose(.{ .b, .vh, .vhd, .khd }).rename(.{ .vh = .h, .vhd = .v, .khd = .k }) + else + zml.Tensor.constant(zml.DataType.zero(.f32)).broad(zml.Shape.init(.{ + .b = value.dim(.b), + .h = value.dim(.vh), + .v = value.dim(.vhd), + .k = query.dim(.khd), + }, .f32)); + + const result = zml.nn.GatedDeltaNet.forward( + query_f32, + key_f32, + value_f32, + alpha_f32, + beta_f32, + .{ .s = initial_recurrent_state }, + ); + + return .{ + result.outputs.rename(.{ .h = .vh, .v = .vhd }).convert(query.dtype()), + result.state.s.transpose(.{ .b, .h, .k, .v }).rename(.{ .h = .vh, .k = .khd, .v = .vhd }), + }; + } + + fn buildUpdatedConvState(input: zml.Tensor, left_pad: i64) zml.Tensor { + const copy_len = @min(input.dim(.s), left_pad); + const tail = input.slice1d(.s, .{ .start = input.dim(.s) - copy_len, .end = input.dim(.s) }); + if (copy_len == left_pad) return tail; + + const padding_shape = zml.Shape.init(.{ .b = input.dim(.b), .s = left_pad - copy_len, .mix = input.dim(.mix) }, input.dtype()); + const padding = zml.Tensor.constant(input.dtype().zero()).broad(padding_shape); + return zml.Tensor.concatenate(&.{ padding, tail }, .s); + } + + pub fn forward(self: GatedDeltaNet, x: zml.Tensor, cache: KvCache.GatedDeltaNetCache) struct { zml.Tensor, KvCache.GatedDeltaNetCache } { + const key_dim = self.num_k_heads * self.head_k_dim; + const value_dim = self.num_v_heads * self.head_v_dim; + const conv_dim = 2 * key_dim + value_dim; + const left_pad = self.conv_kernel_size - 1; + + const x_in = x.withPartitioning(.{ .d = .replicated }); + const projected_qkv = self.in_proj_qkv.forward(x_in) + .rename(.{ .dout = .mix }).withPartitioning(.{ .s = .replicated, .mix = .model }); + const use_cached_state = x.dim(.s) == 1 and left_pad > 0; + const conv_input = if (use_cached_state) + zml.Tensor.concatenate(&.{ cache.convState(), projected_qkv }, .s) + else + projected_qkv; + + const kernel = self.conv1d_weight; + var mixed_qkv = zml.Tensor.conv1d( + conv_input, + kernel, + .{ + .padding = &.{ left_pad, 0 }, + .input_batch_dimension = 0, + .input_feature_dimension = 2, + .input_spatial_dimensions = 1, + .kernel_output_feature_dimension = 0, + .kernel_input_feature_dimension = 1, + .kernel_spatial_dimensions = 2, + .output_batch_dimension = 0, + .output_feature_dimension = 2, + .output_spatial_dimensions = 1, + .feature_group_count = conv_dim, + }, + ) + .silu(); + + if (use_cached_state) { + mixed_qkv = mixed_qkv.slice1d(.s, .{ .start = mixed_qkv.dim(.s) - 1, .end = mixed_qkv.dim(.s) }); + } + mixed_qkv = mixed_qkv.withPartitioning(.{ .s = .replicated, .mix = .model }); + + const z = self.in_proj_z.forward(x_in) + .splitAxis(.dout, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }) + .withPartitioning(.{ .s = .replicated, .vh = .model, .vhd = .replicated }); + const b = self.in_proj_b.forward(x_in).rename(.{ .dout = .vh }).withPartitioning(.{ .s = .replicated, .vh = .model }); + const a = self.in_proj_a.forward(x_in).rename(.{ .dout = .vh }).withPartitioning(.{ .s = .replicated, .vh = .model }); + + const query = mixed_qkv + .slice1d(.mix, .{ .start = 0, .end = key_dim }) + .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }) + .withPartitioning(.{ .s = .replicated, .kh = .model, .khd = .replicated }); + const key = mixed_qkv + .slice1d(.mix, .{ .start = key_dim, .end = 2 * key_dim }) + .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }) + .withPartitioning(.{ .s = .replicated, .kh = .model, .khd = .replicated }); + const value = mixed_qkv + .slice1d(.mix, .{ .start = 2 * key_dim, .end = 2 * key_dim + value_dim }) + .splitAxis(.mix, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }) + .withPartitioning(.{ .s = .replicated, .vh = .model, .vhd = .replicated }); + + const beta = b.sigmoid(); + const aLog_type = self.aLog.dtype(); + const g = self.aLog.broad(a.shape()).exp().mul(softplus(a.convert(aLog_type).add(self.dt_bias.convert(aLog_type).broad(a.shape())))).negate(); + + const query_for_rule = if (self.qk_head_repetition == 1) query else query.stutter1d(@intCast(query.axis(.kh)), @intCast(self.qk_head_repetition)); + const key_for_rule = if (self.qk_head_repetition == 1) key else key.stutter1d(@intCast(key.axis(.kh)), @intCast(self.qk_head_repetition)); + + const core_attn_out, const last_recurrent_state = recurrentGatedDeltaRule( + query_for_rule, + key_for_rule, + value, + g, + beta, + if (use_cached_state) cache.recurrentState() else null, + ); + + const core_attn_out_normed = self.norm + .forward( + core_attn_out.rename(.{ .vhd = .d }), + z.rename(.{ .vhd = .d }), + ) + .rename(.{ .d = .vhd }) + .withPartitioning(.{ .s = .replicated, .vh = .model, .vhd = .replicated }); + + const output = self.out_proj.forward(core_attn_out_normed.merge(.{ .d = .{ .vh, .vhd } })) + .rename(.{ .dout = .d }).withPartitioning(.{ .d = .replicated }); + const updated_cache = cache.update( + buildUpdatedConvState(conv_input, left_pad), + last_recurrent_state, + ); + return .{ output, updated_cache }; + } +}; + +pub const RmsNorm = struct { + weight: zml.Tensor, + eps: f32 = 1e-6, + + pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { + return .{ .weight = store.createTensor("weight", .{.d}, .{ .d = .replicated }), .eps = eps }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { + self.weight.deinit(); + } + + pub fn forward(self: RmsNorm, x: zml.Tensor) zml.Tensor { + const x_f32 = x.convert(.f32); + const weight_f32 = self.weight.convert(.f32); + + const normalized = zml.nn.rmsNorm(x_f32, .d, self.eps); + return normalized.mul(weight_f32.broad(x.shape())).add(normalized).convert(x.dtype()); + } +}; + +pub const RmsNormGated = struct { + weight: zml.Tensor, + eps: f32 = 1e-6, + + pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNormGated { + return .{ .weight = store.createTensor("weight", .{.d}, .{ .d = .replicated }), .eps = eps }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(RmsNormGated)) void { + self.weight.deinit(); + } + + pub fn forward(self: RmsNormGated, x: zml.Tensor, gate: zml.Tensor) zml.Tensor { + const x_f32 = x.convert(.f32); + const gate_f32 = gate.convert(.f32); + + const normalized = zml.nn.rmsNorm(x_f32, .d, self.eps); + const output = normalized.mul(self.weight.broad(x.shape())); + + const gated_output = output.mul(gate_f32.silu()); + return gated_output.convert(x.dtype()); + } +}; + +pub const KvCache = struct { + layer_types: []const LayerType, + self_attn: SelfAttnCache, + gated_delta_net: GatedDeltaNetCache, + + pub const SelfAttnCache = struct { + k: zml.Tensor, + v: zml.Tensor, + layer_index: zml.Tensor, + + pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType) SelfAttnCache { + const num_self_attn_layers = countLayers(config.text_config.layer_types, .full_attention); + const kv_shape = zml.Shape.init(.{ + .b = batch_dim, + .layer = num_self_attn_layers, + .s = max_seq_len, + .h = config.text_config.num_key_value_heads, + .hd = config.text_config.head_dim, + }, dtype); + const sharded_kv_shape = kv_shape.withPartitioning(.{ .h = .model }); + return .{ + .k = .fromShape(sharded_kv_shape), + .v = .fromShape(sharded_kv_shape), + .layer_index = .init(.{}, .u32), + }; + } + + pub fn initBuffer(self: SelfAttnCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(SelfAttnCache) { + return .{ + .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), sharding, .{}), + .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), sharding, .{}), + .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(SelfAttnCache)) void { + self.k.deinit(); + self.v.deinit(); + self.layer_index.deinit(); + } + + pub fn keys(self: SelfAttnCache) zml.Tensor { + return self.k.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn values(self: SelfAttnCache) zml.Tensor { + return self.v.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn update(self: SelfAttnCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: ?zml.Tensor) SelfAttnCache { + const k_shape = self.k.shape().drop(.layer); + var layer = self.layer_index; + layer = if (token_index) |idx| layer.broad(idx.shape()) else layer; + + return if (token_index) |idx| .{ + .k = self.k.scatterSlices( + .{ .layer = layer, .s = idx }, + new_k.convert(self.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.k), + .v = self.v.scatterSlices( + .{ .layer = layer, .s = idx }, + new_v.convert(self.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.v), + .layer_index = self.layer_index, + } else .{ + .k = self.k.scatterSlices( + .{ .layer = layer }, + new_k.convert(self.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.k), + .v = self.v.scatterSlices( + .{ .layer = layer }, + new_v.convert(self.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.v), + .layer_index = self.layer_index, + }; + } + + pub fn atLayer(self: SelfAttnCache, layer_index: usize) SelfAttnCache { + return .{ + .k = self.k, + .v = self.v, + .layer_index = zml.Tensor.scalar(layer_index, .u32), + }; + } + + pub fn reuseBuffer(self: SelfAttnCache, other: SelfAttnCache) SelfAttnCache { + return .{ + .k = self.k.reuseBuffer(other.k), + .v = self.v.reuseBuffer(other.v), + .layer_index = self.layer_index.reuseBuffer(other.layer_index), + }; + } + }; + + pub const GatedDeltaNetCache = struct { + conv_state: zml.Tensor, + recurrent_state: zml.Tensor, + layer_index: zml.Tensor, + + pub fn init(config: Config, batch_dim: i64, conv_dtype: zml.DataType, recurrent_dtype: zml.DataType) GatedDeltaNetCache { + const num_linear_attn_layers = countLayers(config.text_config.layer_types, .linear_attention); + const conv_dim = 2 * config.text_config.linear_num_key_heads * config.text_config.linear_key_head_dim + config.text_config.linear_num_value_heads * config.text_config.linear_value_head_dim; + const conv_state_shape = zml.Shape.init(.{ + .b = batch_dim, + .layer = num_linear_attn_layers, + .s = config.text_config.linear_conv_kernel_dim - 1, + .mix = conv_dim, + }, conv_dtype); + const recurrent_state_shape = zml.Shape.init(.{ + .b = batch_dim, + .layer = num_linear_attn_layers, + .vh = config.text_config.linear_num_value_heads, + .khd = config.text_config.linear_key_head_dim, + .vhd = config.text_config.linear_value_head_dim, + }, recurrent_dtype); + const sharded_conv_state_shape = conv_state_shape.withPartitioning(.{ .mix = .model }); + const sharded_recurrent_state_shape = recurrent_state_shape.withPartitioning(.{ .vh = .model }); + return .{ + .conv_state = .fromShape(sharded_conv_state_shape), + .recurrent_state = .fromShape(sharded_recurrent_state_shape), + .layer_index = .init(.{}, .u32), + }; + } + + pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(GatedDeltaNetCache) { + return .{ + .conv_state = try zml.Buffer.uninitialized(io, platform, self.conv_state.shape(), sharding, .{}), + .recurrent_state = try zml.Buffer.uninitialized(io, platform, self.recurrent_state.shape(), sharding, .{}), + .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(GatedDeltaNetCache)) void { + self.conv_state.deinit(); + self.recurrent_state.deinit(); + self.layer_index.deinit(); + } + + pub fn convState(self: GatedDeltaNetCache) zml.Tensor { + return self.conv_state.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn recurrentState(self: GatedDeltaNetCache) zml.Tensor { + return self.recurrent_state.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn update(self: GatedDeltaNetCache, new_conv_state: ?zml.Tensor, new_recurrent_state: ?zml.Tensor) GatedDeltaNetCache { + const conv_state = if (new_conv_state) |state| + self.conv_state.scatterSlices( + .{ .layer = self.layer_index }, + state.convert(self.conv_state.dtype()).transpose(self.conv_state.shape().drop(.layer)), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.conv_state) + else + self.conv_state; + + const recurrent_state = if (new_recurrent_state) |state| + self.recurrent_state.scatterSlices( + .{ .layer = self.layer_index }, + state.convert(self.recurrent_state.dtype()).transpose(self.recurrent_state.shape().drop(.layer)), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.recurrent_state) + else + self.recurrent_state; + + return .{ + .conv_state = conv_state, + .recurrent_state = recurrent_state, + .layer_index = self.layer_index, + }; + } + + pub fn atLayer(self: GatedDeltaNetCache, layer_index: usize) GatedDeltaNetCache { + return .{ + .conv_state = self.conv_state, + .recurrent_state = self.recurrent_state, + .layer_index = zml.Tensor.scalar(layer_index, .u32), + }; + } + + pub fn reuseBuffer(self: GatedDeltaNetCache, other: GatedDeltaNetCache) GatedDeltaNetCache { + return .{ + .conv_state = self.conv_state.reuseBuffer(other.conv_state), + .recurrent_state = self.recurrent_state.reuseBuffer(other.recurrent_state), + .layer_index = self.layer_index.reuseBuffer(other.layer_index), + }; + } + }; + + pub fn init( + config: Config, + batch_dim: i64, + max_seq_len: i64, + cache_dtype: zml.DataType, + recurrent_dtype: zml.DataType, + ) KvCache { + return .{ + .layer_types = config.text_config.layer_types, + .self_attn = SelfAttnCache.init(config, batch_dim, max_seq_len, cache_dtype), + .gated_delta_net = GatedDeltaNetCache.init(config, batch_dim, cache_dtype, recurrent_dtype), + }; + } + + pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(KvCache) { + return .{ + .self_attn = try self.self_attn.initBuffer(io, platform, sharding), + .gated_delta_net = try self.gated_delta_net.initBuffer(io, platform, sharding), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(KvCache)) void { + SelfAttnCache.deinitBuffer(&self.self_attn); + GatedDeltaNetCache.deinitBuffer(&self.gated_delta_net); + } + + pub const LayerView = struct { + parent: KvCache, + cache: union(enum) { + self_attn: SelfAttnCache, + linear_attn: GatedDeltaNetCache, + }, + }; + + pub fn atLayer(self: KvCache, layer_index: usize) LayerView { + return switch (getDenseIndex(self.layer_types, layer_index)) { + .full_attention => |dense_index| .{ + .parent = self, + .cache = .{ .self_attn = self.self_attn.atLayer(dense_index.layer_dense_index) }, + }, + .linear_attention => |dense_index| .{ + .parent = self, + .cache = .{ .linear_attn = self.gated_delta_net.atLayer(dense_index.layer_dense_index) }, + }, + }; + } + + pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { + return .{ + .layer_types = self.layer_types, + .self_attn = self.self_attn.reuseBuffer(other.self_attn), + .gated_delta_net = self.gated_delta_net.reuseBuffer(other.gated_delta_net), + }; + } + + fn countLayers(layer_types: []const LayerType, layer_type: LayerType) i64 { + var count: i64 = 0; + for (layer_types) |registered_layer_type| { + if (registered_layer_type == layer_type) count += 1; + } + return count; + } + + fn getDenseIndex(layer_types: []const LayerType, layer_index: usize) union(enum) { + full_attention: struct { layer_dense_index: usize }, + linear_attention: struct { layer_dense_index: usize }, + } { + var self_attn_layer_index: usize = 0; + var linear_attn_layer_index: usize = 0; + for (layer_types[0..layer_index]) |layer_type| { + switch (layer_type) { + .full_attention => self_attn_layer_index += 1, + .linear_attention => linear_attn_layer_index += 1, + } + } + return switch (layer_types[layer_index]) { + .full_attention => .{ .full_attention = .{ .layer_dense_index = self_attn_layer_index } }, + .linear_attention => .{ .linear_attention = .{ .layer_dense_index = linear_attn_layer_index } }, + }; + } +}; + +//========================Utils======================== + +fn softplus(x: zml.Tensor) zml.Tensor { + return x.exp().addConstant(1).log(); +} diff --git a/examples/llm/models/qwen3_5_moe/session.zig b/examples/llm/models/qwen3_5_moe/session.zig new file mode 100644 index 00000000..90133d47 --- /dev/null +++ b/examples/llm/models/qwen3_5_moe/session.zig @@ -0,0 +1,459 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const inference = @import("inference.zig"); +const model = @import("model.zig"); + +const LayerIndexBuffer = union(enum) { + self_attn: zml.Buffer, + linear_attn: zml.Buffer, +}; + +pub const Session = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + model_buffers: *model.Buffers, + compiled_model: *const inference.CompiledModel, + kv_cache_buffers: zml.Bufferized(model.KvCache), + prefill_moe_metadata_buffers: zml.Bufferized(zml.moe.Metadata), + decode_moe_metadata_buffers: zml.Bufferized(zml.moe.Metadata), + rng_buffers: zml.Bufferized(zml.Tensor.Rng), + layer_index_buffers: []LayerIndexBuffer, + decode_token_index_buffer: zml.Buffer, + self_attn_layers_caches: []zml.Bufferized(model.KvCache.SelfAttnCache), + linear_attn_layers_caches: []zml.Bufferized(model.KvCache.GatedDeltaNetCache), + step_token_slice: zml.Slice, + generated_token_slice: zml.Slice, + tokenizer: zml.tokenizer.Tokenizer, + seqlen: u32, + eos_token_id: u32, + special_tokens: model.Model.SpecialTokens, + think_start: ?u32, + think_end: ?u32, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + tokenizer: zml.tokenizer.Tokenizer, + compiled_model: *const inference.CompiledModel, + model_buffers: *model.Buffers, + ) !Session { + const shardings = compiled_model.params.shardings; + var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform, shardings.model); + errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); + + var prefill_moe_metadata_buffers = try compiled_model.params.prefill_moe_metadata.initBuffer(io, platform); + errdefer zml.moe.Metadata.deinitBuffer(&prefill_moe_metadata_buffers); + + var decode_moe_metadata_buffers = try compiled_model.params.decode_moe_metadata.initBuffer(io, platform); + errdefer zml.moe.Metadata.deinitBuffer(&decode_moe_metadata_buffers); + + const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); + var rng_buffers = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed); + errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); + + const layer_types = compiled_model.loaded_model.inner.config.text_config.layer_types; + var layer_index_buffers = try allocator.alloc(LayerIndexBuffer, compiled_model.loaded_model.inner.text_model.layers.len); + errdefer { + for (layer_index_buffers) |*layer_index_buffer| { + switch (layer_index_buffer.*) { + .self_attn => |*buffer| buffer.deinit(), + .linear_attn => |*buffer| buffer.deinit(), + } + } + allocator.free(layer_index_buffers); + } + + var self_attn_layer_index: usize = 0; + var linear_attn_layer_index: usize = 0; + for (layer_types, 0..) |layer_type, layer_index| { + switch (layer_type) { + .full_attention => { + const layer_index_buffer = try zml.Buffer.scalar(io, platform, @as(u32, @intCast(self_attn_layer_index)), .u32); + layer_index_buffers[layer_index] = .{ .self_attn = layer_index_buffer }; + self_attn_layer_index += 1; + }, + .linear_attention => { + const layer_index_buffer = try zml.Buffer.scalar(io, platform, @as(u32, @intCast(linear_attn_layer_index)), .u32); + layer_index_buffers[layer_index] = .{ .linear_attn = layer_index_buffer }; + linear_attn_layer_index += 1; + }, + } + } + + var decode_token_index_buffer = try zml.Buffer.scalar(io, platform, @as(u32, 0), .u32); + errdefer decode_token_index_buffer.deinit(); + + var self_attn_layers_caches = try allocator.alloc(zml.Bufferized(model.KvCache.SelfAttnCache), self_attn_layer_index); + errdefer allocator.free(self_attn_layers_caches); + + var linear_attn_layers_caches = try allocator.alloc(zml.Bufferized(model.KvCache.GatedDeltaNetCache), linear_attn_layer_index); + errdefer allocator.free(linear_attn_layers_caches); + + var self_attn_cache_index: usize = 0; + var linear_attn_cache_index: usize = 0; + for (layer_types, 0..) |layer_type, layer_index| { + switch (layer_type) { + .full_attention => { + const layer_index_buffer = switch (layer_index_buffers[layer_index]) { + .self_attn => |buffer| buffer, + .linear_attn => unreachable, + }; + self_attn_layers_caches[self_attn_cache_index] = .{ + .k = kv_cache_buffers.self_attn.k, + .v = kv_cache_buffers.self_attn.v, + .layer_index = layer_index_buffer, + }; + self_attn_cache_index += 1; + }, + .linear_attention => { + const layer_index_buffer = switch (layer_index_buffers[layer_index]) { + .linear_attn => |buffer| buffer, + .self_attn => unreachable, + }; + linear_attn_layers_caches[linear_attn_cache_index] = .{ + .conv_state = kv_cache_buffers.gated_delta_net.conv_state, + .recurrent_state = kv_cache_buffers.gated_delta_net.recurrent_state, + .layer_index = layer_index_buffer, + }; + linear_attn_cache_index += 1; + }, + } + } + + return .{ + .allocator = allocator, + .io = io, + .platform = platform, + .model_buffers = model_buffers, + .compiled_model = compiled_model, + .kv_cache_buffers = kv_cache_buffers, + .prefill_moe_metadata_buffers = prefill_moe_metadata_buffers, + .decode_moe_metadata_buffers = decode_moe_metadata_buffers, + .rng_buffers = rng_buffers, + .layer_index_buffers = layer_index_buffers, + .decode_token_index_buffer = decode_token_index_buffer, + .self_attn_layers_caches = self_attn_layers_caches, + .linear_attn_layers_caches = linear_attn_layers_caches, + .step_token_slice = try .alloc(allocator, zml.Shape.init(.{ .b = 1, .s = 1 }, .u32)), + .generated_token_slice = try .alloc(allocator, zml.Shape.init(.{ .b = 1, .s = 1 }, .u32)), + .tokenizer = tokenizer, + .seqlen = compiled_model.params.seqlen, + .eos_token_id = compiled_model.loaded_model.inner.special_tokens.end_of_text_token_id, + .special_tokens = compiled_model.loaded_model.inner.special_tokens, + .think_start = tokenizer.tokenId(""), + .think_end = tokenizer.tokenId(""), + }; + } + + pub fn deinit(self: *Session) void { + model.KvCache.deinitBuffer(&self.kv_cache_buffers); + zml.moe.Metadata.deinitBuffer(&self.prefill_moe_metadata_buffers); + zml.moe.Metadata.deinitBuffer(&self.decode_moe_metadata_buffers); + zml.Tensor.Rng.deinitBuffer(&self.rng_buffers); + self.decode_token_index_buffer.deinit(); + for (self.layer_index_buffers) |*layer_index_buffer| { + switch (layer_index_buffer.*) { + .self_attn => |*buffer| buffer.deinit(), + .linear_attn => |*buffer| buffer.deinit(), + } + } + self.allocator.free(self.layer_index_buffers); + self.allocator.free(self.self_attn_layers_caches); + self.allocator.free(self.linear_attn_layers_caches); + self.step_token_slice.free(self.allocator); + self.generated_token_slice.free(self.allocator); + } + + pub fn tokenizePrompt(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + return tokenizeChatPrompt(allocator, self.tokenizer, prompt, self.special_tokens, true); + } + + pub fn tokenizeTurn(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + return tokenizeChatPrompt(allocator, self.tokenizer, prompt, self.special_tokens, false); + } + + fn storeSelfAttnLayerCache( + self: *Session, + dense_layer_index: usize, + layer_index: usize, + layer_cache: zml.Bufferized(model.KvCache.SelfAttnCache), + ) void { + self.kv_cache_buffers.self_attn.k = layer_cache.k; + self.kv_cache_buffers.self_attn.v = layer_cache.v; + self.self_attn_layers_caches[dense_layer_index] = layer_cache; + self.layer_index_buffers[layer_index] = .{ .self_attn = layer_cache.layer_index }; + } + + fn storeLinearAttnLayerCache( + self: *Session, + dense_layer_index: usize, + layer_index: usize, + layer_cache: zml.Bufferized(model.KvCache.GatedDeltaNetCache), + ) void { + self.kv_cache_buffers.gated_delta_net.conv_state = layer_cache.conv_state; + self.kv_cache_buffers.gated_delta_net.recurrent_state = layer_cache.recurrent_state; + self.linear_attn_layers_caches[dense_layer_index] = layer_cache; + self.layer_index_buffers[layer_index] = .{ .linear_attn = layer_cache.layer_index }; + } + + pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { + const hidden_size = self.compiled_model.loaded_model.inner.config.text_config.hidden_size; + const model_dtype = self.compiled_model.loaded_model.inner.text_model.embed_tokens.weight.dtype(); + + const prefill_tokens_shape = zml.Shape.init(.{ .b = 1, .s = self.seqlen }, .u32); + const prefill_hidden_shape = zml.Shape.init(.{ .b = 1, .s = self.seqlen, .d = hidden_size }, model_dtype); + + const prefill_tokens_slice = try zml.Slice.alloc(self.allocator, prefill_tokens_shape); + defer prefill_tokens_slice.free(self.allocator); + @memset(prefill_tokens_slice.items(u32), 0); + @memcpy(prefill_tokens_slice.items(u32)[0..all_tokens.len], all_tokens); + + const replicated_sharding: zml.Sharding = .replicated; + + var prefill_tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, prefill_tokens_slice, replicated_sharding); + defer prefill_tokens_buffer.deinit(); + + var prefill_token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 0), .u32); + defer prefill_token_index_buffer.deinit(); + + var prefill_hidden_buffer = try zml.Buffer.uninitialized(self.io, self.platform, prefill_hidden_shape, replicated_sharding, .{}); + defer prefill_hidden_buffer.deinit(); + + var embedding_prefill_args = try self.compiled_model.prefill_embedding_exe.args(self.allocator); + defer embedding_prefill_args.deinit(self.allocator); + var embedding_prefill_results = try self.compiled_model.prefill_embedding_exe.results(self.allocator); + defer embedding_prefill_results.deinit(self.allocator); + + embedding_prefill_args.set(.{ self.model_buffers.text_model.embed_tokens, prefill_tokens_buffer }); + self.compiled_model.prefill_embedding_exe.call(embedding_prefill_args, &embedding_prefill_results); + embedding_prefill_results.fill(.{&prefill_hidden_buffer}); + + var prefill_full_layer_args: ?zml.Exe.Arguments = if (self.compiled_model.prefill_full_layer_exe) |exe| try exe.args(self.allocator) else null; + defer if (prefill_full_layer_args) |*args| args.deinit(self.allocator); + var prefill_full_layer_results: ?zml.Exe.Results = if (self.compiled_model.prefill_full_layer_exe) |exe| try exe.results(self.allocator) else null; + defer if (prefill_full_layer_results) |*results| results.deinit(self.allocator); + + var prefill_linear_layer_args: ?zml.Exe.Arguments = if (self.compiled_model.prefill_linear_layer_exe) |exe| try exe.args(self.allocator) else null; + defer if (prefill_linear_layer_args) |*args| args.deinit(self.allocator); + var prefill_linear_layer_results: ?zml.Exe.Results = if (self.compiled_model.prefill_linear_layer_exe) |exe| try exe.results(self.allocator) else null; + defer if (prefill_linear_layer_results) |*results| results.deinit(self.allocator); + + var self_attn_cache_index: usize = 0; + var linear_attn_cache_index: usize = 0; + + for (self.model_buffers.text_model.layers, 0..) |layer_weights, i| { + switch (self.layer_index_buffers[i]) { + .self_attn => { + const exe = self.compiled_model.prefill_full_layer_exe orelse unreachable; + self.self_attn_layers_caches[self_attn_cache_index].k = self.kv_cache_buffers.self_attn.k; + self.self_attn_layers_caches[self_attn_cache_index].v = self.kv_cache_buffers.self_attn.v; + prefill_full_layer_args.?.set(.{ layer_weights, prefill_hidden_buffer, prefill_token_index_buffer, self.self_attn_layers_caches[self_attn_cache_index], self.compiled_model.loaded_model.inner.config, self.prefill_moe_metadata_buffers }); + exe.call(prefill_full_layer_args.?, &prefill_full_layer_results.?); + prefill_full_layer_results.?.fill(.{ &prefill_hidden_buffer, &self.kv_cache_buffers.self_attn }); + self_attn_cache_index += 1; + }, + .linear_attn => { + const exe = self.compiled_model.prefill_linear_layer_exe orelse unreachable; + self.linear_attn_layers_caches[linear_attn_cache_index].conv_state = self.kv_cache_buffers.gated_delta_net.conv_state; + self.linear_attn_layers_caches[linear_attn_cache_index].recurrent_state = self.kv_cache_buffers.gated_delta_net.recurrent_state; + prefill_linear_layer_args.?.set(.{ layer_weights, prefill_hidden_buffer, prefill_token_index_buffer, self.linear_attn_layers_caches[linear_attn_cache_index], self.compiled_model.loaded_model.inner.config, self.prefill_moe_metadata_buffers }); + exe.call(prefill_linear_layer_args.?, &prefill_linear_layer_results.?); + prefill_linear_layer_results.?.fill(.{ &prefill_hidden_buffer, &self.kv_cache_buffers.gated_delta_net }); + linear_attn_cache_index += 1; + }, + } + } + + var sampling_prefill_args = try self.compiled_model.prefill_sampling_exe.args(self.allocator); + defer sampling_prefill_args.deinit(self.allocator); + var sampling_prefill_results = try self.compiled_model.prefill_sampling_exe.results(self.allocator); + defer sampling_prefill_results.deinit(self.allocator); + + sampling_prefill_args.set(.{ .{ + .norm = self.model_buffers.text_model.norm, + .lm_head = self.model_buffers.text_model.lm_head, + .gen_options = self.compiled_model.loaded_model.inner.text_model.gen_options, + }, prefill_hidden_buffer, self.rng_buffers }); + self.compiled_model.prefill_sampling_exe.call(sampling_prefill_args, &sampling_prefill_results); + sampling_prefill_results.fill(.{ &prefill_tokens_buffer, &self.rng_buffers }); + + try prefill_tokens_buffer.toSlice(self.io, prefill_tokens_slice); + const generated_token = prefill_tokens_slice.items(u32)[all_tokens.len - 1]; + self.generated_token_slice.items(u32)[0] = generated_token; + } + + pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { + var decoder = try self.tokenizer.decoder(); + defer decoder.deinit(); + + const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); + defer self.allocator.free(out_tokens_buffer); + + const hidden_size = self.compiled_model.loaded_model.inner.config.text_config.hidden_size; + const model_dtype = self.compiled_model.loaded_model.inner.text_model.embed_tokens.weight.dtype(); + + const decode_hidden_shape = zml.Shape.init(.{ .b = 1, .s = 1, .d = hidden_size }, model_dtype); + const replicated_sharding: zml.Sharding = .replicated; + + var embedding_decode_args = try self.compiled_model.decode_embedding_exe.args(self.allocator); + defer embedding_decode_args.deinit(self.allocator); + var embedding_decode_results = try self.compiled_model.decode_embedding_exe.results(self.allocator); + defer embedding_decode_results.deinit(self.allocator); + + var decode_full_layer_args: ?zml.Exe.Arguments = if (self.compiled_model.decode_full_layer_exe) |exe| try exe.args(self.allocator) else null; + defer if (decode_full_layer_args) |*args| args.deinit(self.allocator); + var decode_full_layer_results: ?zml.Exe.Results = if (self.compiled_model.decode_full_layer_exe) |exe| try exe.results(self.allocator) else null; + defer if (decode_full_layer_results) |*results| results.deinit(self.allocator); + + var decode_linear_layer_args: ?zml.Exe.Arguments = if (self.compiled_model.decode_linear_layer_exe) |exe| try exe.args(self.allocator) else null; + defer if (decode_linear_layer_args) |*args| args.deinit(self.allocator); + var decode_linear_layer_results: ?zml.Exe.Results = if (self.compiled_model.decode_linear_layer_exe) |exe| try exe.results(self.allocator) else null; + defer if (decode_linear_layer_results) |*results| results.deinit(self.allocator); + + var sampling_decode_args = try self.compiled_model.decode_sampling_exe.args(self.allocator); + defer sampling_decode_args.deinit(self.allocator); + var sampling_decode_results = try self.compiled_model.decode_sampling_exe.results(self.allocator); + defer sampling_decode_results.deinit(self.allocator); + + var decode_hidden_buffer = try zml.Buffer.uninitialized(self.io, self.platform, decode_hidden_shape, replicated_sharding, .{}); + defer decode_hidden_buffer.deinit(); + + var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice, replicated_sharding); + defer current_token_buffer.deinit(); + + var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.items.len)), .u32); + defer token_index_buffer.deinit(); + + var self_attn_layer_index: usize = 0; + var linear_attn_layer_index: usize = 0; + + generation: while (true) { + const token_id = self.generated_token_slice.items(u32)[0]; + if (token_id == self.eos_token_id) break :generation; + + const token = try decoder.feedOne(token_id, out_tokens_buffer); + if (self.think_start) |think_start| if (token_id == think_start) { + try stdout.writeAll("\x1b[2m"); + }; + try stdout.writeAll(token); + if (self.think_end) |think_end| if (token_id == think_end) { + try stdout.writeAll("\x1b[0m"); + }; + try stdout.flush(); + + try all_tokens.append(self.allocator, token_id); + if (all_tokens.items.len >= self.seqlen) break :generation; + + self_attn_layer_index = 0; + linear_attn_layer_index = 0; + + embedding_decode_args.set(.{ self.model_buffers.text_model.embed_tokens, current_token_buffer }); + self.compiled_model.decode_embedding_exe.call(embedding_decode_args, &embedding_decode_results); + embedding_decode_results.fill(.{&decode_hidden_buffer}); + + for (self.model_buffers.text_model.layers, 0..) |layer_weights, layer_index| { + switch (self.layer_index_buffers[layer_index]) { + .self_attn => { + const exe = self.compiled_model.decode_full_layer_exe orelse unreachable; + self.self_attn_layers_caches[self_attn_layer_index].k = self.kv_cache_buffers.self_attn.k; + self.self_attn_layers_caches[self_attn_layer_index].v = self.kv_cache_buffers.self_attn.v; + + decode_full_layer_args.?.set(.{ + layer_weights, + decode_hidden_buffer, + token_index_buffer, + self.self_attn_layers_caches[self_attn_layer_index], + self.compiled_model.loaded_model.inner.config, + self.decode_moe_metadata_buffers, + }); + + exe.call(decode_full_layer_args.?, &decode_full_layer_results.?); + decode_full_layer_results.?.fill(.{ &decode_hidden_buffer, &self.kv_cache_buffers.self_attn }); + + self_attn_layer_index += 1; + }, + .linear_attn => { + const exe = self.compiled_model.decode_linear_layer_exe orelse unreachable; + + self.linear_attn_layers_caches[linear_attn_layer_index].conv_state = self.kv_cache_buffers.gated_delta_net.conv_state; + self.linear_attn_layers_caches[linear_attn_layer_index].recurrent_state = self.kv_cache_buffers.gated_delta_net.recurrent_state; + + decode_linear_layer_args.?.set(.{ + layer_weights, + decode_hidden_buffer, + token_index_buffer, + self.linear_attn_layers_caches[linear_attn_layer_index], + self.compiled_model.loaded_model.inner.config, + self.decode_moe_metadata_buffers, + }); + + exe.call(decode_linear_layer_args.?, &decode_linear_layer_results.?); + decode_linear_layer_results.?.fill(.{ &decode_hidden_buffer, &self.kv_cache_buffers.gated_delta_net }); + + linear_attn_layer_index += 1; + }, + } + } + + sampling_decode_args.set(.{ .{ + .norm = self.model_buffers.text_model.norm, + .lm_head = self.model_buffers.text_model.lm_head, + .gen_options = self.compiled_model.loaded_model.inner.text_model.gen_options, + }, decode_hidden_buffer, self.rng_buffers, token_index_buffer }); + self.compiled_model.decode_sampling_exe.call(sampling_decode_args, &sampling_decode_results); + sampling_decode_results.fill(.{ ¤t_token_buffer, &self.rng_buffers, &token_index_buffer }); + + try current_token_buffer.toSlice(self.io, self.generated_token_slice); + } + + try stdout.writeAll(try decoder.finalize(out_tokens_buffer)); + try stdout.flush(); + } +}; + +fn tokenizeChatPrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8, special_tokens: model.Model.SpecialTokens, is_first_turn: bool) ![]const u32 { + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const im_start = tokenizer.tokenId("<|im_start|>") orelse special_tokens.im_start_token_id; + const im_end = tokenizer.tokenId("<|im_end|>") orelse special_tokens.im_end_token_id; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len + 32); + if (!is_first_turn) { + try tokens.append(allocator, im_end); + const newline = try encoder.encodeAlloc(allocator, "\n"); + try tokens.appendSlice(allocator, newline); + allocator.free(newline); + } + + try tokens.append(allocator, im_start); + const user = try encoder.encodeAlloc(allocator, "user\n"); + try tokens.appendSlice(allocator, user); + allocator.free(user); + const prompt_encoded = try encoder.encodeAlloc(allocator, prompt); + try tokens.appendSlice(allocator, prompt_encoded); + allocator.free(prompt_encoded); + try tokens.append(allocator, im_end); + const newline = try encoder.encodeAlloc(allocator, "\n"); + try tokens.appendSlice(allocator, newline); + allocator.free(newline); + try tokens.append(allocator, im_start); + const assistant = try encoder.encodeAlloc(allocator, "assistant\n"); + try tokens.appendSlice(allocator, assistant); + allocator.free(assistant); + const think = try encoder.encodeAlloc(allocator, "\n"); + try tokens.appendSlice(allocator, think); + allocator.free(think); + + return tokens.toOwnedSlice(allocator); +} + +fn encodeSingleToken(encoder: *zml.tokenizer.Tokenizer.Encoder, text: []const u8) !u32 { + const encoded = try encoder.encode(text); + if (encoded.len != 1) return error.InvalidTokenizerEncoding; + return encoded[0]; +} diff --git a/third_party/zls/zls_write_build_config.bzl b/third_party/zls/zls_write_build_config.bzl index bb260349..b65b968b 100644 --- a/third_party/zls/zls_write_build_config.bzl +++ b/third_party/zls/zls_write_build_config.bzl @@ -97,6 +97,7 @@ def _zls_write_build_config_impl(ctx): name = "c", canonical_name = "c", zigtoolchaininfo = zigtoolchaininfo, + global_args = global_args, cc_infos = [cc_info], ) diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 7af1de68..215ea188 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -64,6 +64,9 @@ cc_library( zig_library( name = "zml", tags = ["manual"], + data = [ + "moe/triton_kernels/config.json", + ], srcs = [ "attention.zig", "attention/attention.zig", @@ -74,6 +77,9 @@ zig_library( "attention/tpu_attention.zig", "attention/triton_attention.zig", "attention/triton_kernels/unified_attention.zig", + "moe/moe.zig", + "moe/triton.zig", + "moe/triton_kernels/triton_kernels.zig", "kernel.zig", "buffer.zig", "constants.zig", diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig new file mode 100644 index 00000000..325476fb --- /dev/null +++ b/zml/moe/moe.zig @@ -0,0 +1,135 @@ +const std = @import("std"); +const zml = @import("../zml.zig"); +pub const triton = @import("triton.zig"); + +pub const triton_kernels = @import("triton_kernels/triton_kernels.zig"); + +pub const Backend = enum { + // Could select a more specific name like "triton_sm90_bf16" + triton, + + pub fn auto(platform: *const zml.Platform, weights_dtype: zml.DataType) !Backend { + return switch (platform.target) { + .cuda => b: { + const first_device = platform.pjrt_client.devices(platform.pjrt_api)[0]; + if (zml.platform.cuda.tryGetComputeCapabilities(platform, first_device)) |cc| { + if (std.mem.eql(u8, cc, "9.0") or + std.mem.eql(u8, cc, "10.0")) + { + break :b switch (weights_dtype) { + .bf16, .f16, .f8e4m3fn => .triton, + else => error.UnsupportedDataType, + }; + } + break :b error.UnsupportedComputeCapability; + } + break :b error.UnsupportedComputeCapability; + }, + else => error.UnimplementedMoEBackend, + }; + } + + pub fn load(backend: Backend, allocator: std.mem.Allocator) !void { + _ = allocator; + return switch (backend) { + .triton => {}, + }; + } + + pub fn register(backend: Backend, platform: *zml.Platform) !void { + _ = platform; + return switch (backend) { + .triton => {}, + }; + } +}; + +pub const Parameters = union(Backend) { + triton: triton.Parameters, + + pub const InitOptions = union(Backend) { + triton: triton.Parameters.InitOptions, + + pub fn fromBackend(backend: Backend, num_experts_per_tok: ?u32, activation: triton.Parameters.ActivationMode) InitOptions { + return switch (backend) { + .triton => .{ .triton = .{ .num_experts_per_tok = num_experts_per_tok.?, .activation = activation } }, + }; + } + }; + + pub fn init(opts: InitOptions) Parameters { + return switch (opts) { + .triton => |v| .{ .triton = triton.Parameters.init(v) }, + }; + } +}; + +pub const Metadata = union(Backend) { + triton: triton.Metadata, + + pub const InitOptions = union(Backend) { + triton: triton.Metadata.InitOptions, + + pub fn fromBackend(backend: Backend) InitOptions { + return switch (backend) { + .triton => .{ .triton = .{} }, + }; + } + }; + + pub fn init(opts: InitOptions) Metadata { + return switch (opts) { + .triton => |v| .{ .triton = triton.Metadata.init(v) }, + }; + } + + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { + return switch (self) { + .triton => |metadata| .{ .triton = try metadata.initBuffer(io, platform) }, + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { + switch (self.*) { + .triton => |*metadata| triton.deinitBuffer(metadata), + } + } +}; + +pub fn forwardMoe( + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + weights_gate_up: zml.Tensor, + scales_gate_up: ?zml.Tensor, + bias_gate_up: ?zml.Tensor, + weights_down: zml.Tensor, + scales_down: ?zml.Tensor, + bias_down: ?zml.Tensor, + metadata: Metadata, + parameters: Parameters, +) !zml.Tensor { + return switch (parameters) { + .triton => b: { + const triton_metadata = switch (metadata) { + .triton => |v| v, + }; + + break :b try triton.fusedExpertsImpl( + input, + weights_gate_up, + weights_down, + topk_weights, + topk_ids, + triton_metadata, + .{ + .activation = parameters.triton.activation, + .w1_scale = scales_gate_up, + .w2_scale = scales_down, + .w1_bias = bias_gate_up, + .w2_bias = bias_down, + }, + ); + }, + }; +} diff --git a/zml/moe/triton.zig b/zml/moe/triton.zig new file mode 100644 index 00000000..f951eb9f --- /dev/null +++ b/zml/moe/triton.zig @@ -0,0 +1,654 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../zml.zig"); +const DataType = zml.DataType; +const Tensor = zml.Tensor; +const Shape = zml.Shape; +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); + +const tri = zml.kernel.triton; +const DType = tri.DType; + +const kernels = @import("triton_kernels/triton_kernels.zig"); + +const toDType = tri.from; + +const log = std.log.scoped(.moe_triton); + +// ============================================================================= +// Public API +// ============================================================================= + +pub const Options = struct { + inplace: bool = false, + activation: Parameters.ActivationMode = .silu, + apply_router_weight_on_input: bool = false, + use_fp8_w8a8: bool = false, + use_int8_w8a8: bool = false, + use_int8_w8a16: bool = false, + use_int4_w4a16: bool = false, + ocp_mx_scheme: ?[]const u8 = null, + per_channel_quant: bool = false, + global_num_experts: i64 = -1, + expert_map: ?Tensor = null, + w1_scale: ?Tensor = null, + w2_scale: ?Tensor = null, + w1_zp: ?Tensor = null, + w2_zp: ?Tensor = null, + a1_scale: ?Tensor = null, + a2_scale: ?Tensor = null, + block_shape: ?[]const i64 = null, + w1_bias: ?Tensor = null, + w2_bias: ?Tensor = null, + block_size_m: i64 = 16, + block_size_n: i64 = 64, + block_size_k: i64 = 32, + group_size_m: i64 = 1, + num_warps: i64 = 8, + num_stages: i64 = 4, + dynamic_launch_by_num_tokens: bool = true, +}; + +pub const Parameters = struct { + num_experts_per_tok: u32, + activation: ActivationMode, + + pub const ActivationMode = enum { + silu, + relu, + }; + + pub const InitOptions = struct { + num_experts_per_tok: u32, + activation: ActivationMode, + }; + + pub fn init(opts: InitOptions) Parameters { + return .{ + .num_experts_per_tok = opts.num_experts_per_tok, + .activation = opts.activation, + }; + } +}; + +pub const Metadata = struct { + w1_zero_bias: ?Tensor = null, + w2_zero_bias: ?Tensor = null, + + pub const InitOptions = struct { + w1_zero_bias_shape: ?Shape = null, + w2_zero_bias_shape: ?Shape = null, + }; + + pub fn init(opts: InitOptions) Metadata { + return .{ + .w1_zero_bias = if (opts.w1_zero_bias_shape) |shape| Tensor.fromShape(shape) else null, + .w2_zero_bias = if (opts.w2_zero_bias_shape) |shape| Tensor.fromShape(shape) else null, + }; + } + + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { + const replicated_sharding = platform.replicated_sharding; + return .{ + .w1_zero_bias = if (self.w1_zero_bias) |tensor| try initZeroBiasBuffer(io, platform, replicated_sharding, tensor.shape()) else null, + .w2_zero_bias = if (self.w2_zero_bias) |tensor| try initZeroBiasBuffer(io, platform, replicated_sharding, tensor.shape()) else null, + }; + } +}; + +pub fn deinitBuffer(bufferized: *zml.Bufferized(Metadata)) void { + if (bufferized.w1_zero_bias) |*buffer| buffer.deinit(); + if (bufferized.w2_zero_bias) |*buffer| buffer.deinit(); +} + +fn initZeroBiasBuffer(io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding, shape: Shape) !zml.Buffer { + var zero_slice: zml.Slice = try .alloc(std.heap.c_allocator, shape); + defer zero_slice.free(std.heap.c_allocator); + @memset(zero_slice.data(), 0); + return zml.Buffer.fromSlice(io, platform, zero_slice, sharding); +} + +// ============================================================================= +// Top-level entry point +// ============================================================================= + +pub fn fusedExpertsImpl( + hidden_states: Tensor, + w1: Tensor, + w2: Tensor, + topk_weights: Tensor, + topk_ids: Tensor, + metadata: Metadata, + opts: Options, +) !Tensor { + try validateOptions(opts); + const options = applyJsonTokenConfig(opts, hidden_states.dim(0)) catch |err| fallback: { + log.warn("Failed to load MoE launch config from JSON ({}), falling back to built-in token heuristic", .{err}); + break :fallback applyDefaultTokenConfig(opts, hidden_states.dim(0), w1.dim(0)); + }; + const b = hidden_states.dim(.b); + const s = hidden_states.dim(.s); + + const hidden = hidden_states.reshape(.{ .token = b * s, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); + const gate_up = w1.withTags(.{ .expert, .out, .in }); + const down = w2.withTags(.{ .expert, .out, .mid }); + const weights = topk_weights.reshape(.{ .token = b * s, .in = topk_weights.dim(.top_expert) }).withTags(.{ .token, .topk }); + const ids = topk_ids.reshape(.{ .token = b * s, .in = topk_ids.dim(.top_expert) }).withTags(.{ .token, .topk }); + + try validateInputs(hidden, gate_up, down, weights, ids); + + const block_size_m = options.block_size_m; + const num_experts = gate_up.dim(.expert); + const num_assignments = hidden.dim(.token) * ids.dim(.topk); + const sparsity_factor: i64 = 4; + const naive_block_assignment = num_assignments * sparsity_factor <= num_experts; + + const max_num_tokens_padded = if (naive_block_assignment) + num_assignments * block_size_m + else if (num_assignments < num_experts) + num_assignments * block_size_m + else + num_assignments + num_experts * (block_size_m - 1); + + const sorted_token_ids, const expert_ids, const num_tokens_post_padded = if (naive_block_assignment) blk: { + log.info("Using naive block assignment for MoE kernels. Num assignments: {d}, Num experts: {d}", .{ num_assignments, num_experts }); + const naive_sorted_ids = Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)); + const naive_expert_ids = ids.reshape(.{ .g = num_assignments }); + const naive_num_tokens_post_padded = Tensor.constant(.{ .i32 = @as(i32, @intCast(max_num_tokens_padded)) }).reshape(.{1}); + break :blk .{ naive_sorted_ids, naive_expert_ids, naive_num_tokens_post_padded }; + } else alignBlockSize(ids, num_experts, block_size_m); + + var hidden_quant = hidden; + var a_scale = opts.a1_scale orelse Tensor.scalar(1.0, .f32); + + if (gate_up.dtype() == .f8e4m3fn) { + hidden_quant, a_scale = quantizePerTokenGroupFp8(hidden, fp8ActivationGroupSize(hidden)); + } + + const first_cfg = makeFusedMoeConfig( + hidden_quant, + gate_up, + options, + naive_block_assignment, + ids.dim(.topk), + false, + false, + .bf16, + ); + + const b_bias_1 = + opts.w1_bias orelse + metadata.w1_zero_bias orelse + Tensor.zeroes(Shape.init(.{ .expert = gate_up.dim(.expert), .out = gate_up.dim(.out) }, .bf16)); + + const b_scale_1 = opts.w1_scale orelse Tensor.scalar(1.0, .f32); + + const first_out = callFusedMoe( + hidden_quant, + gate_up, + b_bias_1, + a_scale, + b_scale_1, + weights, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + first_cfg, + options, + max_num_tokens_padded, + num_assignments, + Shape.init(.{ .token = num_assignments, .out = gate_up.dim(.out) }, .bf16), + ); + + const activated = switch (opts.activation) { + .silu => b: { + const gate, const up = zml.nn.splitRealImg(first_out, .sequential); + break :b gate.silu().mul(up); + }, + .relu => first_out.relu().powByConst(2), + }; + + var activated_quant = activated; + a_scale = opts.a2_scale orelse Tensor.scalar(1.0, .f32); + if (down.dtype() == .f8e4m3fn) { + activated_quant, a_scale = quantizePerTokenGroupFp8(activated, fp8ActivationGroupSize(activated)); + } + + const second_cfg = makeFusedMoeConfig( + activated_quant, + down, + options, + naive_block_assignment, + 1, + true, + false, + .bf16, + ); + + const b_bias_2 = + opts.w2_bias orelse + metadata.w2_zero_bias orelse + Tensor.zeroes(Shape.init(.{ .expert = down.dim(.expert), .out = down.dim(.out) }, .bf16)); + + const b_scale_2 = opts.w2_scale orelse Tensor.scalar(1.0, .f32); + + const second_out = callFusedMoe( + activated_quant, + down, + b_bias_2, + a_scale, + b_scale_2, + weights, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + second_cfg, + options, + max_num_tokens_padded, + num_assignments, + Shape.init(.{ .token = b * s, .topk = ids.dim(.topk), .out = down.dim(.out) }, .bf16), + ); + + const output = second_out.sum(.topk).squeeze(.topk); + + return output.reshape(.{ .b = b, .token = s, .out = down.dim(.out) }); +} + +/// Build the inputs tuple for FusedMoe and invoke it via `K.call(...)`. +fn callFusedMoe( + a: Tensor, + b: Tensor, + b_bias: Tensor, + a_scale: Tensor, + b_scale: Tensor, + topk_weights: Tensor, + sorted_token_ids: Tensor, + expert_ids: Tensor, + num_tokens_post_padded: Tensor, + cfg: kernels.FusedMoe.Cfg, + options: Options, + max_num_tokens_padded: i64, + num_valid_tokens: i64, + output_shape: Shape, +) Tensor { + const block_size_m: i64 = @intCast(cfg.block_size_m); + const block_size_n: i64 = @intCast(cfg.block_size_n); + const m_tokens = a.dim(0); + const em_effective = if (m_tokens < block_size_m) + @min(max_num_tokens_padded, num_valid_tokens * block_size_m) + else + max_num_tokens_padded; + const grid_x = + (std.math.divCeil(i64, em_effective, block_size_m) catch unreachable) * + (std.math.divCeil(i64, b.dim(1), block_size_n) catch unreachable); + + const stride_asm: i64 = if (cfg.b_scale_dtype != null and a_scale.rank() == 2) a_scale.dim(1) else 0; + const stride_ask: i64 = if (cfg.b_scale_dtype != null and a_scale.rank() == 2) 1 else 0; + const stride_bse: i64 = if (cfg.b_scale_dtype != null and b_scale.rank() == 3) + b_scale.dim(1) * b_scale.dim(2) + else + 0; + const stride_bsk: i64 = if (cfg.b_scale_dtype != null and b_scale.rank() == 3) 1 else 0; + const stride_bsn: i64 = if (cfg.b_scale_dtype != null and b_scale.rank() == 3) b_scale.dim(2) else 0; + + return kernels.FusedMoe.Kernel.call( + .{ + .a_ptr = a, + .b_ptr = b, + .b_bias_ptr = b_bias, + .a_scale_ptr = a_scale, + .b_scale_ptr = b_scale, + .topk_weights_ptr = topk_weights, + .sorted_token_ids_ptr = sorted_token_ids, + .expert_ids_ptr = expert_ids, + .num_tokens_post_padded_ptr = num_tokens_post_padded, + .N_ptr = Tensor.constant(.{ .i64 = b.dim(1) }).reshape(.{1}), + .K_ptr = Tensor.constant(.{ .i64 = b.dim(2) }).reshape(.{1}), + .EM_ptr = Tensor.constant(.{ .i64 = em_effective }).reshape(.{1}), + .num_valid_tokens_ptr = Tensor.constant(.{ .i64 = num_valid_tokens }).reshape(.{1}), + .stride_am_ptr = Tensor.constant(.{ .i64 = a.dim(1) }).reshape(.{1}), + .stride_be_ptr = Tensor.constant(.{ .i64 = b.dim(1) * b.dim(2) }).reshape(.{1}), + .stride_bn_ptr = Tensor.constant(.{ .i64 = b.dim(2) }).reshape(.{1}), + .stride_cm_ptr = Tensor.constant(.{ .i64 = b.dim(.out) }).reshape(.{1}), + .stride_asm_ptr = Tensor.constant(.{ .i64 = stride_asm }).reshape(.{1}), + .stride_ask_ptr = Tensor.constant(.{ .i64 = stride_ask }).reshape(.{1}), + .stride_bse_ptr = Tensor.constant(.{ .i64 = stride_bse }).reshape(.{1}), + .stride_bsk_ptr = Tensor.constant(.{ .i64 = stride_bsk }).reshape(.{1}), + .stride_bsn_ptr = Tensor.constant(.{ .i64 = stride_bsn }).reshape(.{1}), + .stride_bbe_ptr = Tensor.constant(.{ .i64 = 0 }).reshape(.{1}), + .stride_bbn_ptr = Tensor.constant(.{ .i64 = 0 }).reshape(.{1}), + }, + .{ .c = output_shape }, + .{ + .cfg = cfg, + .grid = .{ @intCast(grid_x), 1, 1 }, + .num_warps = @intCast(options.num_warps), + .num_stages = @intCast(options.num_stages), + }, + ).c; +} + +fn alignBlockSize(topk_ids: Tensor, num_experts: i64, block_size_m: i64) struct { Tensor, Tensor, Tensor } { + log.info("Using triton kernels to sort and align tokens to experts with block size {d}", .{block_size_m}); + const topk_ids_ = topk_ids.withTags(.{ .token, .topk }).convert(.i32); + const num_tokens = topk_ids_.dim(.token); + const topk = topk_ids_.dim(.topk); + const num_assignments = num_tokens * topk; + const max_num_tokens_padded = if (num_assignments < num_experts) + num_assignments * block_size_m + else + num_assignments + num_experts * (block_size_m - 1); + const max_num_m_blocks = std.math.divCeil(i64, max_num_tokens_padded, block_size_m) catch unreachable; + const warp_size: i64 = 32; + const padded_num_experts = (std.math.divCeil(i64, num_experts, warp_size) catch unreachable) * warp_size; + const sort_block_size: i64 = 256; + const sort_grid_x: i64 = @min(std.math.divCeil(i64, num_assignments, sort_block_size) catch unreachable, 65535); + + const flat_experts = topk_ids_.reshape(.{ .g = num_assignments }); + var cumsums = Tensor.zeroes(Shape.init(.{ .g = num_experts + 1 }, .i32)); + var expert_ids = Tensor.zeroes(Shape.init(.{ .g = max_num_m_blocks }, .i32)); + var sorted_token_ids = Tensor.zeroes(Shape.init(.{ .g = max_num_tokens_padded }, .i32)); + var num_tokens_post_padded = Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)); + + { + const align_outs = kernels.MoeAlignBlockSize.Kernel.call( + .{ + .topk_ids_ptr = flat_experts, + .sorted_token_ids_ptr = sorted_token_ids, + .expert_ids_ptr = expert_ids, + .num_tokens_post_pad_ptr = num_tokens_post_padded, + .cumsum_ptr = cumsums, + }, + .{ + .sorted_token_ids = sorted_token_ids.shape(), + .expert_ids = expert_ids.shape(), + .num_tokens_post_pad = num_tokens_post_padded.shape(), + .cumsum = cumsums.shape(), + }, + .{ + .cfg = .{ + .numel = @intCast(num_assignments), + .num_experts = @intCast(num_experts), + .padded_num_experts = @intCast(padded_num_experts), + .max_num_tokens_padded = @intCast(max_num_tokens_padded), + .max_num_m_blocks = @intCast(max_num_m_blocks), + .block_size_m = @intCast(block_size_m), + .hist_block = 256, + }, + .grid = .{ 2, 1, 1 }, + .num_stages = 1, + .num_warps = 8, + .output_operand_aliases = &.{ + .{ .output_index = 0, .operand_index = 1 }, + .{ .output_index = 1, .operand_index = 2 }, + .{ .output_index = 2, .operand_index = 3 }, + .{ .output_index = 3, .operand_index = 4 }, + }, + }, + ); + sorted_token_ids = align_outs.sorted_token_ids; + expert_ids = align_outs.expert_ids; + num_tokens_post_padded = align_outs.num_tokens_post_pad; + cumsums = align_outs.cumsum; + } + + { + const sort_outs = kernels.CountAndSortExpertTokens.Kernel.call( + .{ + .topk_ids_ptr = flat_experts, + .sorted_token_ids_ptr = sorted_token_ids, + .cumsum_ptr = cumsums, + }, + .{ + .sorted_token_ids = sorted_token_ids.shape(), + .cumsum = cumsums.shape(), + }, + .{ + .cfg = .{ + .numel = @intCast(num_assignments), + .num_experts = @intCast(num_experts), + .sort_block_size = @intCast(sort_block_size), + }, + .grid = .{ @intCast(sort_grid_x), 1, 1 }, + .num_stages = 1, + .num_warps = 4, + .output_operand_aliases = &.{ + .{ .output_index = 0, .operand_index = 1 }, + .{ .output_index = 1, .operand_index = 2 }, + }, + }, + ); + sorted_token_ids = sort_outs.sorted_token_ids; + cumsums = sort_outs.cumsum; + } + + return .{ sorted_token_ids, expert_ids, num_tokens_post_padded }; +} + +fn quantizePerTokenGroupFp8(x: Tensor, group_size: i64) struct { Tensor, Tensor } { + stdx.debug.assert(x.rank() == 2, "expected a rank-2 activation matrix, got {f}", .{x.shape()}); + stdx.debug.assert(@mod(x.dim(1), group_size) == 0, "activation width must be divisible by group size {d}, got {d}", .{ group_size, x.dim(1) }); + + const groups_per_row = @divExact(x.dim(1), group_size); + const quantized = Tensor.zeroes(Shape.init(.{ .token = x.dim(0), .feature = x.dim(1) }, .f8e4m3fn)); + const scales = Tensor.zeroes(Shape.init(.{ .token = x.dim(0), .group = groups_per_row }, .bf16)); + + const outs = kernels.PerTokenGroupQuantFp8.Kernel.call( + .{ + .y_ptr = x, + .group_size_ptr = Tensor.constant(.{ .i64 = group_size }).reshape(.{1}), + .y_num_columns_ptr = Tensor.constant(.{ .i64 = x.dim(1) }).reshape(.{1}), + .y_row_stride_ptr = Tensor.constant(.{ .i64 = x.dim(1) }).reshape(.{1}), + .eps_ptr = Tensor.scalar(1e-6, .f32), + }, + .{ .y_q = quantized.shape(), .y_s = scales.shape() }, + .{ + .cfg = .{ + .input_dtype = toDType(x.dtype()), + .output_dtype = .f8e4m3fn, + .scale_dtype = .bf16, + .block = @intCast(group_size), + .fp8_min = -448.0, + .fp8_max = 448.0, + .use_ue8m0 = false, + }, + .grid = .{ @intCast(x.dim(0) * groups_per_row), 1, 1 }, + .num_stages = 1, + .num_warps = 1, + }, + ); + + return .{ outs.y_q, outs.y_s }; +} + +// ============================================================================= +// Config / validation helpers +// ============================================================================= + +fn makeFusedMoeConfig( + a: Tensor, + b: Tensor, + opts: Options, + naive_block_assignment: bool, + top_k: i64, + mul_routed_weight: bool, + has_bias: bool, + output_dtype: DataType, +) kernels.FusedMoe.Cfg { + var use_fp8 = opts.use_fp8_w8a8; + if (b.dtype() == .f8e4m3fn) use_fp8 = true; + return .{ + .a_dtype = toDType(a.dtype()), + .b_dtype = toDType(b.dtype()), + .c_dtype = toDType(output_dtype), + .a_scale_dtype = if (use_fp8) .bf16 else null, + .b_scale_dtype = if (use_fp8) .bf16 else null, + .b_bias_dtype = null, + .topk_weights_dtype = null, + .block_size_m = @intCast(opts.block_size_m), + .block_size_n = @intCast(opts.block_size_n), + .block_size_k = @intCast(opts.block_size_k), + .group_size_m = @intCast(opts.group_size_m), + .top_k = @intCast(top_k), + .naive_block_assignment = naive_block_assignment, + .mul_routed_weight = mul_routed_weight, + .compute_type = .bf16, + .use_fp8_w8a8 = use_fp8, + .use_int8_w8a8 = false, + .use_int8_w8a16 = false, + .per_channel_quant = false, + .has_bias = has_bias, + }; +} + +const DefaultTokenBucket = struct { + tokens: i64, + block_size_m: i64, + block_size_n: i64, + block_size_k: i64, + group_size_m: i64, + num_warps: i64, + num_stages: i64, +}; + +fn applyDefaultTokenConfig(opts: Options, num_tokens: i64, num_experts: i64) Options { + var out = opts; + if (!opts.dynamic_launch_by_num_tokens) return out; + + // General default policy for NVIDIA bf16/fp16 and fp8 per-tensor. + // Tile sizes scale with batch size: small batches are more memory-bound, + // while larger batches benefit from wider M/N tiles and more warps. + if (num_tokens <= 32) { + out.block_size_m = 16; + } else if (num_tokens <= 96) { + out.block_size_m = 32; + } else if (num_tokens <= 512) { + out.block_size_m = 64; + } else { + out.block_size_m = 128; + } + + out.block_size_n = if (num_tokens <= 64) 64 else 128; + out.block_size_k = if (opts.use_fp8_w8a8 or num_tokens <= 64) 128 else 64; + + const tokens_per_expert = @divFloor(num_tokens, @max(num_experts, 1)); + out.group_size_m = if (tokens_per_expert > 128) 16 else 1; + out.num_warps = if (num_tokens <= 128) 4 else 8; + out.num_stages = if (num_tokens <= 32) 4 else 3; + + return out; +} + +fn getLaunchConfigJsonPath(allocator: std.mem.Allocator) ![]const u8 { + const runfiles = bazel.runfiles(bazel_builtin.current_repository) catch |err| { + log.err("Failed to initialize runfiles for MoE launch config: {}", .{err}); + return err; + }; + + var config_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const config_path = + runfiles.rlocation("zml/zml/moe/triton_kernels/config.json", &config_path_buf) catch null orelse + runfiles.rlocation("zml/zml/moe/triton/triton_kernels/config.json", &config_path_buf) catch |err| { + log.err("Failed to resolve MoE launch config in runfiles: {}", .{err}); + return err; + }; + + const config_json = config_path orelse { + log.warn("MoE launch config is missing from runfiles at both zml/zml/moe/triton_kernels/config.json and zml/zml/moe/triton/triton_kernels/config.json", .{}); + return error.MissingLaunchConfigRunfile; + }; + + return try allocator.dupe(u8, config_json); +} + +const JsonConfigFields = struct { + BLOCK_SIZE_M: i64, + BLOCK_SIZE_N: i64, + BLOCK_SIZE_K: i64, + GROUP_SIZE_M: i64, + num_warps: i64, + num_stages: i64, +}; + +fn applyJsonTokenConfig(opts: Options, num_tokens: i64) !Options { + var out = opts; + if (!opts.dynamic_launch_by_num_tokens) return out; + + const compilation_context = zml.module.CompilationContext.current(); + const io = compilation_context.io; + const allocator = compilation_context.allocator; + + const config_path = try getLaunchConfigJsonPath(allocator); + defer allocator.free(config_path); + const config_json = try std.Io.Dir.cwd().readFileAlloc(io, config_path, allocator, .unlimited); + defer allocator.free(config_json); + const parsed = try std.json.parseFromSlice( + std.json.ArrayHashMap(JsonConfigFields), + allocator, + config_json, + .{}, + ); + defer parsed.deinit(); + + var best_diff: u64 = std.math.maxInt(u64); + var best_m: ?i64 = null; + var best_config: ?JsonConfigFields = null; + + var it = parsed.value.map.iterator(); + while (it.next()) |entry| { + const m = try std.fmt.parseInt(i64, entry.key_ptr.*, 10); + + const diff: u64 = @intCast(@abs(m - num_tokens)); + if (best_m == null or diff < best_diff or (diff == best_diff and m < best_m.?)) { + best_m = m; + best_config = entry.value_ptr.*; + best_diff = diff; + } + } + + const cfg = best_config orelse return error.NoMatchingLaunchConfig; + out.block_size_m = cfg.BLOCK_SIZE_M; + out.block_size_n = cfg.BLOCK_SIZE_N; + out.block_size_k = cfg.BLOCK_SIZE_K; + out.group_size_m = cfg.GROUP_SIZE_M; + out.num_warps = cfg.num_warps; + out.num_stages = cfg.num_stages; + + return out; +} + +fn fp8ActivationGroupSize(x: Tensor) i64 { + const group_size: i64 = 128; + stdx.debug.assert(@mod(x.dim(1), group_size) == 0, "FP8 activation width must be divisible by {d}, got {d}", .{ group_size, x.dim(1) }); + return group_size; +} + +fn validateOptions(opts: Options) !void { + if (opts.inplace) return error.Unimplemented; + if (opts.activation != .silu and opts.activation != .relu) return error.UnsupportedActivation; + if (opts.apply_router_weight_on_input) return error.UnsupportedOption; + if (opts.use_fp8_w8a8 or opts.use_int8_w8a8 or opts.use_int8_w8a16 or opts.use_int4_w4a16) return error.UnsupportedQuantization; + if (opts.ocp_mx_scheme != null or opts.per_channel_quant) return error.UnsupportedOption; + if (opts.global_num_experts != -1) return error.UnsupportedOption; + if (opts.expert_map != null) return error.UnsupportedOption; + if (opts.w1_zp != null or opts.w2_zp != null) return error.UnsupportedOption; + if (opts.a1_scale != null or opts.a2_scale != null or opts.block_shape != null) return error.UnsupportedOption; + if (opts.w1_bias != null or opts.w2_bias != null) return error.UnsupportedOption; +} + +fn validateInputs(hidden: Tensor, gate_up: Tensor, down: Tensor, weights: Tensor, ids: Tensor) !void { + if (hidden.dtype() != .bf16) return error.UnsupportedType; + if (gate_up.dtype() != .bf16 and gate_up.dtype() != .f8e4m3fn) return error.UnsupportedType; + if (down.dtype() != .bf16 and down.dtype() != .f8e4m3fn) return error.UnsupportedType; + if (weights.dtype() != .f32 and weights.dtype() != .bf16) return error.UnsupportedType; + if (ids.dtype() != .i32) return error.UnsupportedType; + if (hidden.dim(.in) != gate_up.dim(.in)) return error.InvalidShape; + if (@rem(gate_up.dim(.out), 2) != 0) return error.InvalidShape; + if (down.dim(.mid) != @divFloor(gate_up.dim(.out), 2)) return error.InvalidShape; + if (ids.dim(.token) != hidden.dim(.token) or weights.dim(.token) != hidden.dim(.token)) return error.InvalidShape; + if (ids.dim(.topk) != weights.dim(.topk)) return error.InvalidShape; + if (gate_up.dim(.expert) != down.dim(.expert)) return error.InvalidShape; +} diff --git a/zml/moe/triton_kernels/config.json b/zml/moe/triton_kernels/config.json new file mode 100644 index 00000000..cc853947 --- /dev/null +++ b/zml/moe/triton_kernels/config.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/zml/moe/triton_kernels/triton_kernels.zig b/zml/moe/triton_kernels/triton_kernels.zig new file mode 100644 index 00000000..cb913f46 --- /dev/null +++ b/zml/moe/triton_kernels/triton_kernels.zig @@ -0,0 +1,652 @@ +//! Triton MoE kernel declarations — direct ports of the `@triton.jit` +//! functions in `triton_kernels/moe.py`. Each kernel here is a +//! `tri.Kernel(...)` declaration with its own inline config struct (no +//! defaults — every field is required at call time). Production callers +//! invoke these via `K.call(inputs, outputs, opts)`; offline tooling reaches the TTIR +//! string via `K.emit(allocator, ctx, cfg)`. + +const std = @import("std"); + +const zml = @import("../../zml.zig"); +const tri = zml.kernel.triton; +const Builder = tri.Builder; +const Value = tri.Value; +const DType = tri.DType; + +const log = std.log.scoped(.moe_triton); + +/// Floor to a multiple of 16 — matches the Python `(v // 16) * 16` guards +/// that keep dynamic strides aligned for tt.load/store. +fn blockFloor16(v: Value) Value { + return v.div(16).mul(16); +} + +// ============================================================================= +// per_token_group_quant_fp8 +// ============================================================================= + +/// Direct port of Python `per_token_group_quant_fp8` — per-token-group +/// float8 quantization. +pub const PerTokenGroupQuantFp8 = struct { + pub const Cfg = struct { + input_dtype: DType, + output_dtype: DType, + scale_dtype: DType, + block: usize, + fp8_min: f32, + fp8_max: f32, + use_ue8m0: bool, + }; + pub const Kernel = tri.Kernel(Cfg, .{ + .name = "per_token_group_quant_fp8", + .inputs = &.{ "y_ptr", "group_size_ptr", "y_num_columns_ptr", "y_row_stride_ptr", "eps_ptr" }, + .outputs = &.{ "y_q", "y_s" }, + .run = run, + }); + fn run(b: *Builder, cfg: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ + .y_ptr = .{ .ptr = cfg.input_dtype }, + .group_size_ptr = .{ .ptr = .i64 }, + .y_num_columns_ptr = .{ .ptr = .i64 }, + .y_row_stride_ptr = .{ .ptr = .i64 }, + .eps_ptr = .{ .ptr = .f32 }, + .y_q_ptr = .{ .ptr = cfg.output_dtype }, + .y_s_ptr = .{ .ptr = cfg.scale_dtype }, + }); + + const block: i64 = @intCast(cfg.block); + const out_dt = cfg.output_dtype; + const scale_dt = cfg.scale_dtype; + + const group_size = b.load(a.group_size_ptr); + const y_num_columns = b.load(a.y_num_columns_ptr); + const y_row_stride = b.load(a.y_row_stride_ptr); + const eps = b.load(a.eps_ptr); + + const groups_per_row = y_num_columns.div(group_size); + // g_id is i32 from `tl.program_id`; the i64 promotion only happens + // inside the `(row|row_g_id).to(tl.int64)` casts in the Python source. + // y_s_ptr_shifted reuses the raw i32 `g_id` directly (no extsi). + const pid_i32 = b.programId(.x); + const g_id = pid_i32.to(.i64); + + // y_ptr += (g_id // groups_per_row) * y_row_stride + (g_id % groups_per_row) * group_size + // Python emit order: divsi, remsi, muli (row * stride), muli (row_g_id * group_size), addi. + const row = g_id.div(groups_per_row); + const row_g_id = g_id.rem(groups_per_row); + const row_off = row.mul(y_row_stride); + const grp_off = row_g_id.mul(group_size); + const y_ptr_shifted = a.y_ptr.addPtr(row_off.add(grp_off)); + const y_q_ptr_shifted = a.y_q_ptr.addPtr(g_id.mul(group_size)); + // y_s_ptr += g_id (raw i32, no cast — matches `y_s_ptr += g_id` in Python). + const y_s_ptr_shifted = a.y_s_ptr.addPtr(pid_i32); + + // cols = arange(0, BLOCK) — i64 for both pointer offsets and mask + // comparison against the i64 `group_size` (matches Python's + // `cols < group_size` after Triton's auto-promotion). + const cols_i32 = b.arange(0, block, .i32); + const cols_i64 = cols_i32.to(.i64); + const mask = cols_i64.lt(group_size); + + // Python: `tl.load(y_ptr + cols, mask=mask, other=0.0)` — explicit + // 3-operand form. Use `.other` to materialize the const-zero tensor + // (without it, `tt.load` is 2-operand and Python's emit doesn't match). + const y_load_ptrs = y_ptr_shifted.addPtr(cols_i64); + const y_other = b.zeros(&.{block}, cfg.input_dtype); + const y = b.loadOpts(y_load_ptrs, .{ .mask = mask, .other = y_other }).to(.f32); + + // _absmax = max(tl.max(tl.abs(y)), eps) — `b.max(...).maximum(eps)` + // resolves to `arith.maxnumf` (matches Python's `tl.maximum`). + const absmax = b.max(b.absf(y)).maximum(eps); + + // scale_raw = _absmax * (1.0 / fp8_max) + const scale_raw = absmax.mul(1.0 / cfg.fp8_max); + const y_s = if (cfg.use_ue8m0) b.exp2(b.ceil(b.log2(scale_raw))) else scale_raw; + + // y_q = clamp(y / y_s, fp8_min, fp8_max).to(output_dtype) + const y_div = y.div(y_s); + const clamped = b.clampf( + y_div, + b.splat(cfg.fp8_min, &.{block}), + b.splat(cfg.fp8_max, &.{block}), + ); + const y_q = clamped.to(out_dt); + + const y_q_ptrs = y_q_ptr_shifted.addPtr(cols_i64); + b.storeOpts(y_q_ptrs, y_q, .{ .mask = mask }); + b.store(y_s_ptr_shifted, y_s.to(scale_dt)); + } +}; + +// ============================================================================= +// write_zeros_to_output — helper called by FusedMoe +// ============================================================================= + +/// Direct port of Python `write_zeros_to_output`. Stores a masked zero-tile +/// for blocks whose expert is `-1` (no expert assigned). `stride_cn` is +/// hardcoded to 1 (contiguous last dim), so the N-axis offset is just +/// `offs_cn` itself. +fn writeZerosToOutput( + b: *Builder, + c_ptr: Value, + stride_cm: Value, + pid_n: Value, + n: Value, + offs_token: Value, + token_mask: Value, + block_size_m: i64, + block_size_n: i64, + compute_type: DType, +) void { + const accumulator = b.zeros(&.{ block_size_m, block_size_n }, compute_type); + const offs_cn = pid_n.mul(block_size_n).add(b.arange(0, block_size_n, .i64)); + + // c_ptrs in two addptrs to match Python's `c_ptr + cm[:, None] + cn[None, :]` + // left-to-right evaluation. `stride_cm * offs_token[:, None]` puts the + // splatted stride on the LHS of arith.muli (Python source order). + const offs_token_col = b.expandDims(offs_token, 1); + const cm_col = stride_cm.mul(offs_token_col); + const c_ptrs_col = b.splat(c_ptr, &.{ block_size_m, 1 }).addPtr(cm_col); + const offs_cn_row = b.expandDims(offs_cn, 0); + const c_ptrs_2d = b.broadcastTo(c_ptrs_col, &.{ block_size_m, block_size_n }); + const cn_off_2d = b.broadcastTo(offs_cn_row, &.{ block_size_m, block_size_n }); + const c_ptrs = c_ptrs_2d.addPtr(cn_off_2d); + + // c_mask: Python emit order — expand_dims, splat, cmpi, broadcast, broadcast, andi. + const token_mask_col = b.expandDims(token_mask, 1); + const n_splat = b.splat(n, &.{ 1, block_size_n }); + const offs_cn_lt_2d = offs_cn_row.lt(n_splat); + const token_mask_full = b.broadcastTo(token_mask_col, &.{ block_size_m, block_size_n }); + const offs_cn_lt_full = b.broadcastTo(offs_cn_lt_2d, &.{ block_size_m, block_size_n }); + b.storeOpts(c_ptrs, accumulator, .{ .mask = token_mask_full.bitAnd(offs_cn_lt_full) }); +} + +// ============================================================================= +// fused_moe_kernel +// ============================================================================= + +/// Direct port of Python `fused_moe_kernel`. Bf16 / no-quant / no-bias path +/// only — errors out for the feature flags that aren't implemented yet. +pub const FusedMoe = struct { + pub const Cfg = struct { + a_dtype: DType, + b_dtype: DType, + c_dtype: DType, + a_scale_dtype: ?DType, + b_scale_dtype: ?DType, + b_bias_dtype: ?DType, + topk_weights_dtype: ?DType, + block_size_m: usize, + block_size_n: usize, + block_size_k: usize, + group_size_m: usize, + top_k: usize, + naive_block_assignment: bool, + mul_routed_weight: bool, + compute_type: DType, + // Validation flags — the kernel rejects configs that set any of + // these to true (the body only implements the bf16 / no-quant / + // no-bias path). + use_fp8_w8a8: bool, + use_int8_w8a8: bool, + use_int8_w8a16: bool, + per_channel_quant: bool, + has_bias: bool, + }; + pub const Kernel = tri.Kernel(Cfg, .{ + .name = "fused_moe_kernel", + .inputs = &.{ + "a_ptr", "b_ptr", "b_bias_ptr", "a_scale_ptr", "b_scale_ptr", + "topk_weights_ptr", "sorted_token_ids_ptr", "expert_ids_ptr", "num_tokens_post_padded_ptr", "N_ptr", + "K_ptr", "EM_ptr", "num_valid_tokens_ptr", "stride_am_ptr", "stride_be_ptr", + "stride_bn_ptr", "stride_cm_ptr", "stride_asm_ptr", "stride_ask_ptr", "stride_bse_ptr", + "stride_bsk_ptr", "stride_bsn_ptr", "stride_bbe_ptr", "stride_bbn_ptr", + }, + .outputs = &.{"c"}, + .run = run, + }); + fn run(b: *Builder, cfg: Cfg) tri.FinishError!void { + if (cfg.use_fp8_w8a8 or cfg.use_int8_w8a8 or cfg.use_int8_w8a16 or cfg.has_bias or cfg.per_channel_quant) { + log.err("fused_moe_kernel: unsupported config (fp8/int8/bias/per_channel)", .{}); + return error.InvalidMlir; + } + + const a = try b.declareArgs(.{ + .a_ptr = .{ .ptr = cfg.a_dtype }, + .b_ptr = .{ .ptr = cfg.b_dtype }, + .b_bias_ptr = .{ .ptr = cfg.b_bias_dtype orelse cfg.c_dtype }, + .a_scale_ptr = .{ .ptr = cfg.a_scale_dtype orelse .f32 }, + .b_scale_ptr = .{ .ptr = cfg.b_scale_dtype orelse .f32 }, + .topk_weights_ptr = .{ .ptr = cfg.topk_weights_dtype orelse .f32 }, + .sorted_token_ids_ptr = .{ .ptr = .i32 }, + .expert_ids_ptr = .{ .ptr = .i32 }, + .num_tokens_post_padded_ptr = .{ .ptr = .i32 }, + .N_ptr = .{ .ptr = .i64 }, + .K_ptr = .{ .ptr = .i64 }, + .EM_ptr = .{ .ptr = .i64 }, + .num_valid_tokens_ptr = .{ .ptr = .i64 }, + .stride_am_ptr = .{ .ptr = .i64 }, + .stride_be_ptr = .{ .ptr = .i64 }, + .stride_bn_ptr = .{ .ptr = .i64 }, + .stride_cm_ptr = .{ .ptr = .i64 }, + .stride_asm_ptr = .{ .ptr = .i64 }, + .stride_ask_ptr = .{ .ptr = .i64 }, + .stride_bse_ptr = .{ .ptr = .i64 }, + .stride_bsk_ptr = .{ .ptr = .i64 }, + .stride_bsn_ptr = .{ .ptr = .i64 }, + .stride_bbe_ptr = .{ .ptr = .i64 }, + .stride_bbn_ptr = .{ .ptr = .i64 }, + .c_ptr = .{ .ptr = cfg.c_dtype }, + }); + + const block_size_m: i64 = @intCast(cfg.block_size_m); + const block_size_n: i64 = @intCast(cfg.block_size_n); + const block_size_k: i64 = @intCast(cfg.block_size_k); + const group_size_m: i64 = @intCast(cfg.group_size_m); + const top_k: i64 = @intCast(cfg.top_k); + const compute_type = cfg.compute_type; + + // Runtime scalars — load all first, then floor to multiples of 16 + // (Python loads them as a group then runs the // 16 * 16 sequence). + const n_raw = b.load(a.N_ptr); + const k_raw = b.load(a.K_ptr); + const em_raw = b.load(a.EM_ptr); + const num_valid_tokens = b.load(a.num_valid_tokens_ptr); + const stride_am_raw = b.load(a.stride_am_ptr); + const stride_be_raw = b.load(a.stride_be_ptr); + const stride_bn_raw = b.load(a.stride_bn_ptr); + const stride_cm_raw = b.load(a.stride_cm_ptr); + const n_block = blockFloor16(n_raw); + const k_block = blockFloor16(k_raw); + const em_block = blockFloor16(em_raw); + const stride_am_block = blockFloor16(stride_am_raw); + const stride_be_block = blockFloor16(stride_be_raw); + const stride_bn_block = blockFloor16(stride_bn_raw); + const stride_cm_block = blockFloor16(stride_cm_raw); + // stride_ak / stride_bk / stride_cn are all 1 in this port — the A/B/C + // inner dimension is contiguous, so those mul-by-1 terms are elided. + + // pid grouping. Python keeps `pid` as i32 and lets the divsi auto- + // promote at the use site — emit the cdivs first, then extsi at the + // first i64 use. Manual order keeps `extsi pid` after the cdivs in TTIR. + const pid_i32 = b.programId(.x); + const num_pid_m = em_block.cdiv(block_size_m); + const num_pid_n = n_block.cdiv(block_size_n); + const num_pid_in_group = num_pid_n.mul(group_size_m); + const pid = pid_i32.to(.i64); + const group_id = pid.div(num_pid_in_group); + const first_pid_m = group_id.mul(group_size_m); + const gsm_actual = num_pid_m.sub(first_pid_m).minimum(group_size_m); + const pid_mod_in_group = pid.rem(num_pid_in_group); + const pid_m = first_pid_m.add(pid_mod_in_group.rem(gsm_actual)); + const pid_n = pid_mod_in_group.div(gsm_actual); + + // Python source order: arange first, then load + early-return. + // Match emit order so the make_range op lands before the cf.cond_br. + const offs = b.arange(0, block_size_m, .i64); + // Order: load (i32), muli, extsi (Python emits the muli before the + // i32→i64 extsi at the cmpi). + const num_tokens_post_padded_i32 = b.load(a.num_tokens_post_padded_ptr); + const pid_m_block = pid_m.mul(block_size_m); + const num_tokens_post_padded = num_tokens_post_padded_i32.to(.i64); + const out_of_range = pid_m_block.ge(num_tokens_post_padded); + + b.returnIfOpts(out_of_range, .{}, .{ .inline_return = true }); + + const offs_token = if (cfg.naive_block_assignment) + b.select( + offs.eq(0), + pid_m.splatTo(&.{block_size_m}), + num_valid_tokens.splatTo(&.{block_size_m}), + ) + else off: { + const ids_ptrs = a.sorted_token_ids_ptr + .addPtr(pid_m.mul(block_size_m).add(offs)); + break :off b.load(ids_ptrs).to(.i64); + }; + const token_mask = offs_token.lt(num_valid_tokens); + + // off_experts = expert_ids[pid_m].to(i64). pid_m is i64; Python's + // `expert_ids_ptr + pid_m` keeps the offset as i64. + const off_experts = b.load(a.expert_ids_ptr.addPtr(pid_m)).to(.i64); + const is_dead = off_experts.eq(-1); + + // Python: if off_experts == -1: write_zeros_to_output(...); return + var dead_ret = b.openReturnIf(is_dead); + dead_ret.inline_return = true; + { + writeZerosToOutput( + b, + a.c_ptr, + stride_cm_block, + pid_n, + n_block, + offs_token, + token_mask, + block_size_m, + block_size_n, + compute_type, + ); + dead_ret.yieldReturn(.{}); + } + + // CSE `pid_n * BLOCK_SIZE_N` so it reuses across offs_bn / offs_cn. + const pid_n_block_n = pid_n.mul(block_size_n); + const offs_bn = pid_n_block_n.add(b.arange(0, block_size_n, .i64)).rem(n_block); + + // offs_k stays i32 through expand_dims; extsi to i64 happens on the 2D form. + const offs_k_i32 = b.arange(0, block_size_k, .i32); + + // a_ptrs: `offs_token[:, None] // top_k * stride_am + offs_k[None, :]`. + // (stride_ak == 1 here, so `* stride_ak` folds away.) Build with explicit + // broadcastTo to match Python's emit shape. + const offs_token_col = b.expandDims(offs_token, 1); + const am_col = offs_token_col.div(top_k).mul(stride_am_block); + const offs_k_row = b.expandDims(offs_k_i32, 0).to(.i64); + const am_term = b.broadcastTo(am_col, &.{ block_size_m, block_size_k }); + const ak_term = b.broadcastTo(offs_k_row, &.{ block_size_m, block_size_k }); + const a_ptrs_init = a.a_ptr.addPtr(am_term.add(ak_term)); + + // b_ptrs: `b_ptr + off_experts * stride_be` (scalar addptr) THEN + // `+ (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn)` (2D + // addptr) — matches Python's left-to-right evaluation. Defer extsi + // for offs_k to after offs_bn's expand_dims/mul (Python emit order). + const b_ptr_shifted = a.b_ptr.addPtr(off_experts.mul(stride_be_block)); + const offs_k_col_i32 = b.expandDims(offs_k_i32, 1); + const offs_bn_row = b.expandDims(offs_bn, 0); + const bn_row = offs_bn_row.mul(stride_bn_block); + const offs_k_col = offs_k_col_i32.to(.i64); + const bk_term = b.broadcastTo(offs_k_col, &.{ block_size_k, block_size_n }); + const bn_term = b.broadcastTo(bn_row, &.{ block_size_k, block_size_n }); + const b_ptrs_init = b_ptr_shifted.addPtr(bk_term.add(bn_term)); + + const acc_init = b.zeros(&.{ block_size_m, block_size_n }, .f32); + + // Loop bounds stay i64 because `k_block` is i64. iter_args order + // matches Python's TTIR (a_ptrs, b_ptrs, accumulator). + const num_k_iters = k_block.cdiv(block_size_k); + var loop = b.openFor(@as(i64, 0), num_k_iters, @as(i64, 1), .{ a_ptrs_init, b_ptrs_init, acc_init }); + { + const k_iter = loop.iv; + const a_ptrs = loop.carried[0]; + const b_ptrs = loop.carried[1]; + const acc = loop.carried[2]; + + const k_remaining = k_block.sub(k_iter.mul(block_size_k)); + + // mask_a = token_mask[:, None] & (offs_k[None, :] < k_remaining). + const token_mask_col_a = b.expandDims(token_mask, 1); + const offs_k_row_a = b.expandDims(offs_k_i32, 0).to(.i64); + const offs_k_lt_a = offs_k_row_a.lt(b.splat(k_remaining, &.{ 1, block_size_k })); + const token_mask_full_a = b.broadcastTo(token_mask_col_a, &.{ block_size_m, block_size_k }); + const offs_k_lt_a_full = b.broadcastTo(offs_k_lt_a, &.{ block_size_m, block_size_k }); + const mask_a = token_mask_full_a.bitAnd(offs_k_lt_a_full); + + const a_val = b.loadOpts(a_ptrs, .{ + .mask = mask_a, + .other = b.zeros(&.{ block_size_m, block_size_k }, cfg.a_dtype), + }); + + // mask_b = offs_k[:, None] < k_remaining (broadcast to KxN). + const offs_k_col_b = b.expandDims(offs_k_i32, 1).to(.i64); + const offs_k_lt_b = offs_k_col_b.lt(b.splat(k_remaining, &.{ block_size_k, 1 })); + const mask_b = b.broadcastTo(offs_k_lt_b, &.{ block_size_k, block_size_n }); + + const b_val = b.loadOpts(b_ptrs, .{ + .mask = mask_b, + .other = b.zeros(&.{ block_size_k, block_size_n }, cfg.b_dtype), + }); + + const new_acc = b.dotOpts(a_val, b_val, acc, .{ + .input_precision = .tf32, + .max_num_imprecise_acc = 0, + }); + + // BLOCK_SIZE_K is `tl.constexpr` → i32 dense splat (stride_ak == 1). + const bsk_i32: i32 = @intCast(cfg.block_size_k); + const new_a_ptrs = a_ptrs.addPtr(b.splat(bsk_i32, &.{ block_size_m, block_size_k })); + const new_b_ptrs = b_ptrs.addPtr(b.splat(bsk_i32, &.{ block_size_k, block_size_n })); + + loop.yield(.{ new_a_ptrs, new_b_ptrs, new_acc }); + } + var accumulator = loop.results[2]; + + if (cfg.mul_routed_weight) { + const tw_dtype = cfg.topk_weights_dtype orelse .f32; + const tw_other = b.zeros(&.{block_size_m}, tw_dtype); + const tw = b.loadOpts(a.topk_weights_ptr.addPtr(offs_token), .{ .mask = token_mask, .other = tw_other }); + accumulator = accumulator.mul(tw.expandDims(1)); + } + + accumulator = accumulator.to(compute_type); + + // c_ptrs: two addptrs to match Python's `c_ptr + cm[:, None] + cn[None, :]` + // left-to-right eval. `stride_cm * offs_token[:, None]` puts the splatted + // stride on the LHS of arith.muli (Python source order). + const offs_cn = pid_n_block_n.add(b.arange(0, block_size_n, .i64)); + const offs_token_col_c = b.expandDims(offs_token, 1); + const cm_col = stride_cm_block.mul(offs_token_col_c); + const c_ptrs_col = b.splat(a.c_ptr, &.{ block_size_m, 1 }).addPtr(cm_col); + const offs_cn_row = b.expandDims(offs_cn, 0); + const c_ptrs_2d = b.broadcastTo(c_ptrs_col, &.{ block_size_m, block_size_n }); + const c_ptrs = c_ptrs_2d.addPtr(b.broadcastTo(offs_cn_row, &.{ block_size_m, block_size_n })); + + // c_mask = token_mask[:, None] & (offs_cn[None, :] < n_block). + const token_mask_col_c = b.expandDims(token_mask, 1); + const offs_cn_lt_2d = offs_cn_row.lt(b.splat(n_block, &.{ 1, block_size_n })); + const token_mask_full_c = b.broadcastTo(token_mask_col_c, &.{ block_size_m, block_size_n }); + const offs_cn_lt_full = b.broadcastTo(offs_cn_lt_2d, &.{ block_size_m, block_size_n }); + b.storeOpts(c_ptrs, accumulator, .{ .mask = token_mask_full_c.bitAnd(offs_cn_lt_full) }); + } +}; + +// ============================================================================= +// moe_align_block_size_kernel +// ============================================================================= + +/// Direct port of Python `moe_align_block_size_kernel`. pid==0 runs the +/// histogram + cumsum + block-to-expert assignment pass; pid==1 fills +/// sorted_token_ids with NUMEL. +pub const MoeAlignBlockSize = struct { + pub const Cfg = struct { + numel: usize, + num_experts: usize, + padded_num_experts: usize, + max_num_tokens_padded: usize, + max_num_m_blocks: usize, + block_size_m: usize, + hist_block: usize, + }; + pub const Kernel = tri.Kernel(Cfg, .{ + .name = "moe_align_block_size_kernel", + .inputs = &.{ "topk_ids_ptr", "sorted_token_ids_ptr", "expert_ids_ptr", "num_tokens_post_pad_ptr", "cumsum_ptr" }, + .outputs = &.{ "sorted_token_ids", "expert_ids", "num_tokens_post_pad", "cumsum" }, + .run = run, + }); + fn run(b: *Builder, cfg: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ + .topk_ids_ptr = .{ .ptr = .i32 }, + .sorted_token_ids_ptr = .{ .ptr = .i32 }, + .expert_ids_ptr = .{ .ptr = .i32 }, + .num_tokens_post_pad_ptr = .{ .ptr = .i32 }, + .cumsum_ptr = .{ .ptr = .i32 }, + .out0_ptr = .{ .ptr = .i32 }, + .out1_ptr = .{ .ptr = .i32 }, + .out2_ptr = .{ .ptr = .i32 }, + .out3_ptr = .{ .ptr = .i32 }, + }); + + const block_size_m: i32 = @intCast(cfg.block_size_m); + const numel: i32 = @intCast(cfg.numel); + const num_experts: i32 = @intCast(cfg.num_experts); + const padded_num_experts: i64 = @intCast(cfg.padded_num_experts); + const max_num_tokens_padded: i32 = @intCast(cfg.max_num_tokens_padded); + const max_num_m_blocks: i64 = @intCast(cfg.max_num_m_blocks); + const hist_block: i64 = @intCast(cfg.hist_block); + + const pid = b.programId(.x); + const fill_offs = b.arange(0, hist_block, .i32); + + // if pid == 1: fill sorted_token_ids with NUMEL; return + var fill_branch = b.openReturnIf(pid.eq(1)); + fill_branch.inline_return = true; + { + var fill_loop = b.openFor(0, max_num_tokens_padded, hist_block, .{}); + { + const offs = fill_loop.iv.add(fill_offs); + const mask = offs.lt(max_num_tokens_padded); + b.storeOpts( + a.sorted_token_ids_ptr.addPtr(offs), + b.splat(numel, &.{hist_block}), + .{ .mask = mask }, + ); + fill_loop.yield(.{}); + } + fill_branch.yieldReturn(.{}); + } + + // pid != 1: histogram + cumsum + block assignment. + const expert_offs = b.arange(0, padded_num_experts, .i32); + const token_offs = b.arange(0, hist_block, .i32); + const expert_mask = expert_offs.lt(num_experts); + const counts_init = b.zeros(&.{padded_num_experts}, .i32); + + var hist_loop = b.openFor(0, numel, hist_block, .{counts_init}); + { + const offs = hist_loop.iv.add(token_offs); + const mask = offs.lt(numel); + const expert_vals = b.loadOpts(a.topk_ids_ptr.addPtr(offs), .{ + .mask = mask, + .other = b.splat(num_experts, &.{hist_block}), + }); + const valid = mask.bitAnd(expert_vals.lt(num_experts)); + const h = b.histogramOpts(expert_vals, padded_num_experts, .{ .mask = valid }); + hist_loop.yield(.{hist_loop.carried[0].add(h)}); + } + const counts = hist_loop.results[0]; + + // padded_counts = where(expert_mask, cdiv(counts, BLOCK_SIZE_M) * BLOCK_SIZE_M, 0) + const padded_counts = b.select( + expert_mask, + counts.cdiv(block_size_m).mul(block_size_m), + b.zeros(&.{padded_num_experts}, .i32), + ); + const padded_cumsum = b.cumsumOpts(padded_counts, .{ .axis = 0 }); + const starts = padded_cumsum.sub(padded_counts); + const total_tokens_post_pad = b.sumOpts(padded_counts, .{ .axis = 0 }); + + b.storeOpts(a.cumsum_ptr.addPtr(expert_offs), starts, .{ .mask = expert_mask }); + b.store(a.cumsum_ptr.addPtr(num_experts), total_tokens_post_pad); + b.store(a.num_tokens_post_pad_ptr, total_tokens_post_pad); + + const block_offs = b.arange(0, hist_block, .i32); + var assign_loop = b.openFor(0, max_num_m_blocks, hist_block, .{}); + { + const block_ids = assign_loop.iv.add(block_offs); + const block_mask = block_ids.lt(@as(i32, @intCast(max_num_m_blocks))); + const block_offsets = block_ids.mul(block_size_m); + const block_expert_init = b.full(&.{hist_block}, -1, .i32); + + var exp_loop = b.openFor(0, num_experts, 1, .{block_expert_init}); + { + const e_iv = exp_loop.iv; + const start_v = b.load(a.cumsum_ptr.addPtr(e_iv)); + const end_v = b.load(a.cumsum_ptr.addPtr(e_iv.add(1))); + const in_range = block_mask + .bitAnd(block_offsets.ge(start_v)) + .bitAnd(block_offsets.lt(end_v)); + exp_loop.yield(.{ + b.select(in_range, e_iv.splatTo(&.{hist_block}), exp_loop.carried[0]), + }); + } + b.storeOpts( + a.expert_ids_ptr.addPtr(block_ids), + exp_loop.results[0], + .{ .mask = block_mask }, + ); + assign_loop.yield(.{}); + } + } +}; + +// ============================================================================= +// count_and_sort_expert_tokens_kernel +// ============================================================================= + +/// Direct port of Python `count_and_sort_expert_tokens_kernel`. Each program +/// atomically bumps `cumsum[expert]`, writing its block's token offsets into +/// `sorted_token_ids` at the returned rank. +pub const CountAndSortExpertTokens = struct { + pub const Cfg = struct { + numel: usize, + num_experts: usize, + sort_block_size: usize, + }; + pub const Kernel = tri.Kernel(Cfg, .{ + .name = "count_and_sort_expert_tokens_kernel", + .inputs = &.{ "topk_ids_ptr", "sorted_token_ids_ptr", "cumsum_ptr" }, + .outputs = &.{ "sorted_token_ids", "cumsum" }, + .run = run, + }); + fn run(b: *Builder, cfg: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ + .topk_ids_ptr = .{ .ptr = .i32 }, + .sorted_token_ids_ptr = .{ .ptr = .i32 }, + .cumsum_ptr = .{ .ptr = .i32 }, + .out0_ptr = .{ .ptr = .i32 }, + .out1_ptr = .{ .ptr = .i32 }, + }); + + const block: i64 = @intCast(cfg.sort_block_size); + const numel: i32 = @intCast(cfg.numel); + const num_experts: i32 = @intCast(cfg.num_experts); + const block_i32: i32 = @intCast(block); + + const pid = b.programId(.x); + const num_progs = b.numPrograms(.x); + const token_offs = b.makeRange(0, block); + + const token_start_init = pid.mul(block_i32); + + // while token_start < NUMEL: + var w = b.openWhile(.{token_start_init}, .{b.scalarTy(.i32)}); + { + const ts = w.before_carried[0]; + w.yieldBefore(ts.lt(numel), .{ts}); + } + { + const ts = w.after_carried[0]; + + // offs = ts + token_offs; mask = offs < NUMEL + const offs = ts.add(token_offs); + const mask = offs.lt(numel); + + // expert_vals = load(topk_ids + offs, mask=mask, other=NUM_EXPERTS) + const topk_ptrs = a.topk_ids_ptr.addPtr(offs); + const expert_vals = b.loadOpts(topk_ptrs, .{ + .mask = mask, + .other = b.splat(num_experts, &.{block}), + }); + const valid = mask.bitAnd(expert_vals.lt(num_experts)); + + // rank = atomic_add(cumsum + expert_vals, 1, mask=valid, sem="relaxed") + const cumsum_ptrs = a.cumsum_ptr.addPtr(expert_vals); + const rank = b.atomicRmwOpts(.add, cumsum_ptrs, b.ones(&.{block}, .i32), .{ + .mask = valid, + .sem = .relaxed, + .scope = .gpu, + }); + + // store(sorted_token_ids + rank, offs, mask=valid) + const sorted_ptrs = a.sorted_token_ids_ptr.addPtr(rank); + b.storeOpts(sorted_ptrs, offs, .{ .mask = valid }); + + // token_start += num_programs * BLOCK_SIZE + // Compute step INSIDE the loop body (Python's source position). + // It's loop-invariant, so XLA's LICM hoists it out — the hoisted + // op-order then matches Python's IR (splats before step). + const step = num_progs.mul(block_i32); + w.yieldAfter(.{ts.add(step)}); + } + } +}; diff --git a/zml/zml.zig b/zml/zml.zig index 0c788ebf..7c216fcf 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -30,6 +30,7 @@ pub const meta = @import("meta.zig"); pub const mlir = @import("mlirx.zig"); pub const module = @import("module.zig"); pub const CompilationOptions = module.CompilationOptions; +pub const moe = @import("moe/moe.zig"); pub const nn = @import("nn.zig"); pub const ops = @import("ops.zig"); pub const pjrtx = @import("pjrtx.zig"); From 886d9000be40145e1333455c9585402ba155356b Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Fri, 15 May 2026 21:01:50 +0200 Subject: [PATCH 487/881] workspace: use latest version of upstreamable rules_zig (#537) The upstreamable branch now contains only 2 minor commits. Everything else inccluding test-obj, translatec and bootstrap have been upstreamed. --- .bazelrc | 2 + BUILD.bazel | 18 +++ MODULE.bazel | 4 +- MODULE.bazel.lock | 135 +++++++++++++----- third_party/arocc/repo.bzl | 2 +- third_party/translate-c/repo.bzl | 2 +- third_party/translate-c/translate-c.bazel | 19 ++- .../uucode/config-storage-module.patch | 33 +++++ third_party/uucode/repo.bzl | 3 + third_party/uucode/uucode.bazel | 26 +--- third_party/zls/zls_write_runner_zig_src.bzl | 17 ++- 11 files changed, 200 insertions(+), 61 deletions(-) create mode 100644 third_party/uucode/config-storage-module.patch diff --git a/.bazelrc b/.bazelrc index 530aa158..3f52fe6d 100644 --- a/.bazelrc +++ b/.bazelrc @@ -78,6 +78,8 @@ common --@rules_zig//zig/settings:use_cc_common_link=true # Zig self hosted backend is too fragile for now common --@rules_zig//zig/settings:zigopt=-fllvm +common --@rules_zig//zig/settings:use_standalone_translate_c=True + common --@llvm//config:experimental_stub_libgcc_s=true common --macos_minimum_os=13.0 diff --git a/BUILD.bazel b/BUILD.bazel index f049fbe8..67502496 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,4 +1,5 @@ load("//third_party/zls:zls_completion.bzl", "zls_completion") +load("@rules_zig//zig/translate-c:toolchain.bzl", "translate_c_toolchain") alias( name = "translate-c", @@ -6,6 +7,23 @@ alias( visibility = ["//visibility:public"], ) +translate_c_toolchain( + name = "translate_c_toolchain_impl", + runtime_modules = [ + "@translate-c//:c_builtins", + "@translate-c//:helpers", + ], + translate_c = "@translate-c//:translate-c", +) + +toolchain( + name = "translate_c_toolchain", + target_settings = ["@rules_zig//zig/config/use_standalone_translate_c:enabled"], + toolchain = ":translate_c_toolchain_impl", + toolchain_type = "@rules_zig//zig/translate-c:toolchain_type", + visibility = ["//visibility:public"], +) + zls_completion( name = "completion", visibility = ["//visibility:public"], diff --git a/MODULE.bazel b/MODULE.bazel index 75ad9a4e..7f5a686c 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -30,7 +30,7 @@ bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "rules_zig", version = "0.12.2") git_override( module_name = "rules_zig", - commit = "b65a2bb460b9dbcbcc3231ba648c3c1a266c4eb5", + commit = "6e9d876037ecb01ce4354c9c9e0c93e026e84e37", # upstreamable-20260515.1 remote = "https://github.com/zml/rules_zig.git", ) @@ -65,7 +65,6 @@ use_repo( zig = use_extension("@rules_zig//zig:extensions.bzl", "zig") zig.index(file = "//bazel:zig_index.json") zig.toolchain( - translate_c = "@translate-c", zig_version = "0.16.0", ) zig.mirrors(urls = [ @@ -89,6 +88,7 @@ use_repo(zig, "zig_toolchains") register_toolchains("@rules_zig//zig/target:all") register_toolchains("@zig_toolchains//:all") +register_toolchains("//:translate_c_toolchain") ### ### LLVM Bootstrapped toolchains diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 66181253..ca07200a 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -22,12 +22,12 @@ "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", "https://bcr.bazel.build/modules/apple_support/1.17.1/MODULE.bazel": "655c922ab1209978a94ef6ca7d9d43e940cd97d9c172fb55f94d91ac53f8610b", "https://bcr.bazel.build/modules/apple_support/1.23.1/MODULE.bazel": "53763fed456a968cf919b3240427cf3a9d5481ec5466abc9d5dc51bc70087442", - "https://bcr.bazel.build/modules/apple_support/2.2.0/MODULE.bazel": "5e32bc42e413a4544df7afee7fa4c7aff73e670a44e56e8e45ce1954332034ad", - "https://bcr.bazel.build/modules/apple_support/2.2.0/source.json": "368bf1de5dace2a411e792e6df2ea5384eb8db3899680ed51b37eb48b88fd8ac", + "https://bcr.bazel.build/modules/apple_support/2.5.4/MODULE.bazel": "964e9268b37edb64e9a0fa525e5b5e535ba56bf63cadc68375f92e69fcb734f6", + "https://bcr.bazel.build/modules/apple_support/2.5.4/source.json": "df85e46410ff71c4006a13dbdeeeee488feba563be78f3a9ed8497c1fb93fb5d", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.3/MODULE.bazel": "253d739ba126f62a5767d832765b12b59e9f8d2bc88cc1572f4a73e46eb298ca", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.21.2/MODULE.bazel": "276347663a25b0d5bd6cad869252bea3e160c4d980e764b15f3bae7f80b30624", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.21.2/source.json": "f42051fa42629f0e59b7ac2adf0a55749144b11f1efcd8c697f0ee247181e526", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.22.5/MODULE.bazel": "004ba890363d05372a97248c37205ae64b6fa31047629cd2c0895a9d0c7779e8", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.22.5/source.json": "ac2c3213df8f985785f1d0aeb7f0f73d5324e6e67d593d9b9470fb74a25d4a9b", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "780d1a6522b28f5edb7ea09630748720721dfe27690d65a2d33aa7509de77e07", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "812d2dd42f65dca362152101fbec418029cc8fd34cbad1a2fde905383d705838", "https://bcr.bazel.build/modules/aspect_rules_py/1.8.4/MODULE.bazel": "4ab4a78e9d65cf15e755807f7ae504655e1771cefd7c3343ad34dbbb625b9564", @@ -179,7 +179,8 @@ "https://bcr.bazel.build/modules/opentracing-cpp/1.6.0/MODULE.bazel": "b3925269f63561b8b880ae7cf62ccf81f6ece55b62cd791eda9925147ae116ec", "https://bcr.bazel.build/modules/opentracing-cpp/1.6.0/source.json": "da1cb1add160f5e5074b7272e9db6fd8f1b3336c15032cd0a653af9d2f484aed", "https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92", - "https://bcr.bazel.build/modules/package_metadata/0.0.2/source.json": "e53a759a72488d2c0576f57491ef2da0cf4aab05ac0997314012495935531b73", + "https://bcr.bazel.build/modules/package_metadata/0.0.3/MODULE.bazel": "77890552ecea9e284b5424c9de827a58099348763a4359e975c359a83d4faa83", + "https://bcr.bazel.build/modules/package_metadata/0.0.3/source.json": "742075a428ad12a3fa18a69014c2f57f01af910c6d9d18646c990200853e641a", "https://bcr.bazel.build/modules/patchelf/0.18.0.bcr.1/MODULE.bazel": "35320668481400503dea7c75313473b9ffd6dea4e7ba583b96a896199b0088d5", "https://bcr.bazel.build/modules/patchelf/0.18.0.bcr.1/source.json": "d877d271ef5d531407885322054c99c7f0617a9a85ff99c0a70f4ced7f13e13a", "https://bcr.bazel.build/modules/pcre2/10.45/MODULE.bazel": "9bdc838aa2e21be030d7181385e45a5c87b14575bb7ff35d10b5be3d56913bed", @@ -193,7 +194,8 @@ "https://bcr.bazel.build/modules/platforms/0.0.8/MODULE.bazel": "9f142c03e348f6d263719f5074b21ef3adf0b139ee4c5133e2aa35664da9eb2d", "https://bcr.bazel.build/modules/platforms/0.0.9/MODULE.bazel": "4a87a60c927b56ddd67db50c89acaa62f4ce2a1d2149ccb63ffd871d5ce29ebc", "https://bcr.bazel.build/modules/platforms/1.0.0/MODULE.bazel": "f05feb42b48f1b3c225e4ccf351f367be0371411a803198ec34a389fb22aa580", - "https://bcr.bazel.build/modules/platforms/1.0.0/source.json": "f4ff1fd412e0246fd38c82328eb209130ead81d62dcd5a9e40910f867f733d96", + "https://bcr.bazel.build/modules/platforms/1.1.0/MODULE.bazel": "1c0c09f5bdcf4b3f924720d2478a3711cb39f4977019ca5988685e5b7e18b3d2", + "https://bcr.bazel.build/modules/platforms/1.1.0/source.json": "fcf351c47596c939140ab0d333dfdd08ed1ea6ce33c2fe70c12493a301cf1344", "https://bcr.bazel.build/modules/prometheus-cpp/1.2.4/MODULE.bazel": "0fbe5dcff66311947a3f6b86ebc6a6d9328e31a28413ca864debc4a043f371e5", "https://bcr.bazel.build/modules/prometheus-cpp/1.3.0.bcr.1/MODULE.bazel": "116ad46e97c1d2aeb020fe2899a342a7e703574ce7c0faf7e4810f938c974a9a", "https://bcr.bazel.build/modules/prometheus-cpp/1.3.0.bcr.1/source.json": "e813cce2d450708cfcb26e309c5172583a7440776edf354e83e6788c768e5cca", @@ -1135,7 +1137,7 @@ }, "@@rules_oci+//oci:extensions.bzl%oci": { "general": { - "bzlTransitiveDigest": "rM01g4kd1EG6Z+4iqFqd+ZkaNCU9oVCGJf+Kg9jgr5k=", + "bzlTransitiveDigest": "7oR14uSSRXhIDozspnn5ya0JWJlnqcG+JmT5yO1BsLw=", "usagesDigest": "JLbamplho20rviuebatMGqnQnaoj32ewCzLuRVusGoU=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -1760,11 +1762,11 @@ }, "@@rules_zig+//zig:extensions.bzl%zig": { "general": { - "bzlTransitiveDigest": "xx2lb1ba6RcnnXdfUyaxI5j2wFLM03GTQfTv2UUie3Q=", - "usagesDigest": "3YLtfwSfkXudL3+Coim7hrTFktQHln4CiUbszoYuRzE=", + "bzlTransitiveDigest": "mBSDTk+QEb2UlXBSWQWCw8A/tUSarS8PCSbhFYCkotc=", + "usagesDigest": "TA6Tzd8UNrxyL67PtNy0LKhpR1wAStj+mb2UDC/QiyI=", "recordedFileInputs": { "@@//bazel/zig_index.json": "2e3b4b83f1add5d3ca12fd05797c04c38af21d86c7fc7c1a9b356560141a4797", - "@@rules_zig+//zig/private/versions.json": "6eb85ebaee72c4e6fbc82d8ad73f69e0899d839b70a7e19caa792ab6a5fa0c36" + "@@rules_zig+//zig/private/versions.json": "3e4e370dee9c3f3fdc8bfc7a600e76743278fde40815eead5d0bcda3aec3e2bd" }, "recordedDirentsInputs": {}, "envVariables": {}, @@ -1787,12 +1789,24 @@ "https://zig.karearl.com/zig", "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds", - "https://ziglang.org/builds/" + "https://ziglang.org/builds/", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds" ], "sha256": "ea4b09bfb22ec6f6c6ceac57ab63efb6b46e17ab08d21f69f3a48b38e1534f17", "zig_version": "0.16.0", - "platform": "aarch64-linux", - "translate_c": "@@+non_module_deps+translate-c//:translate-c" + "platform": "aarch64-linux" } }, "zig_0.16.0_aarch64-macos": { @@ -1813,12 +1827,24 @@ "https://zig.karearl.com/zig", "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds", - "https://ziglang.org/builds/" + "https://ziglang.org/builds/", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds" ], "sha256": "b23d70deaa879b5c2d486ed3316f7eaa53e84acf6fc9cc747de152450d401489", "zig_version": "0.16.0", - "platform": "aarch64-macos", - "translate_c": "@@+non_module_deps+translate-c//:translate-c" + "platform": "aarch64-macos" } }, "zig_0.16.0_aarch64-windows": { @@ -1839,12 +1865,24 @@ "https://zig.karearl.com/zig", "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds", - "https://ziglang.org/builds/" + "https://ziglang.org/builds/", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds" ], "sha256": "aee38316ee4111717900f45dd3130145c39289e105541d737eb8c5ed653c78ef", "zig_version": "0.16.0", - "platform": "aarch64-windows", - "translate_c": "@@+non_module_deps+translate-c//:translate-c" + "platform": "aarch64-windows" } }, "zig_0.16.0_x86_64-linux": { @@ -1865,12 +1903,24 @@ "https://zig.karearl.com/zig", "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds", - "https://ziglang.org/builds/" + "https://ziglang.org/builds/", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds" ], "sha256": "70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00", "zig_version": "0.16.0", - "platform": "x86_64-linux", - "translate_c": "@@+non_module_deps+translate-c//:translate-c" + "platform": "x86_64-linux" } }, "zig_0.16.0_x86_64-macos": { @@ -1891,12 +1941,24 @@ "https://zig.karearl.com/zig", "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds", - "https://ziglang.org/builds/" + "https://ziglang.org/builds/", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds" ], "sha256": "0387557ed1877bc6a2e1802c8391953baddba76081876301c522f52977b52ba7", "zig_version": "0.16.0", - "platform": "x86_64-macos", - "translate_c": "@@+non_module_deps+translate-c//:translate-c" + "platform": "x86_64-macos" } }, "zig_0.16.0_x86_64-windows": { @@ -1917,12 +1979,24 @@ "https://zig.karearl.com/zig", "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds", - "https://ziglang.org/builds/" + "https://ziglang.org/builds/", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds" ], "sha256": "68659eb5f1e4eb1437a722f1dd889c5a322c9954607f5edcf337bc3684a75a7e", "zig_version": "0.16.0", - "platform": "x86_64-windows", - "translate_c": "@@+non_module_deps+translate-c//:translate-c" + "platform": "x86_64-windows" } }, "zig_toolchains": { @@ -1978,11 +2052,6 @@ } }, "recordedRepoMappingEntries": [ - [ - "", - "translate-c", - "+non_module_deps+translate-c" - ], [ "rules_zig+", "bazel_skylib", diff --git a/third_party/arocc/repo.bzl b/third_party/arocc/repo.bzl index 10ff796b..9f574c98 100644 --- a/third_party/arocc/repo.bzl +++ b/third_party/arocc/repo.bzl @@ -5,5 +5,5 @@ def repo(): name = "arocc", remote = "https://github.com/Vexu/arocc.git", commit = "5f5a050569a95ecc40a426f0c3666ae7ef987ede", - build_file = "//third_party/arocc:arocc.bazel", + build_file = Label("//third_party/arocc:arocc.bazel"), ) diff --git a/third_party/translate-c/repo.bzl b/third_party/translate-c/repo.bzl index d3a4bd19..db21bd0d 100644 --- a/third_party/translate-c/repo.bzl +++ b/third_party/translate-c/repo.bzl @@ -5,5 +5,5 @@ def repo(): name = "translate-c", remote = "https://codeberg.org/ziglang/translate-c", commit = "46b5609b5ac4c0a896217d1d984f3ae50e4810b5", - build_file = "//third_party/translate-c:translate-c.bazel", + build_file = Label("//third_party/translate-c:translate-c.bazel"), ) diff --git a/third_party/translate-c/translate-c.bazel b/third_party/translate-c/translate-c.bazel index 84a0e059..b2ff9c98 100644 --- a/third_party/translate-c/translate-c.bazel +++ b/third_party/translate-c/translate-c.bazel @@ -1,17 +1,19 @@ -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_configure_binary", "zig_library") zig_library( name = "c_builtins", main = "lib/c_builtins.zig", + visibility = ["//visibility:public"], ) zig_library( name = "helpers", main = "lib/helpers.zig", + visibility = ["//visibility:public"], ) zig_binary( - name = "translate-c", + name = "translate-c_raw", data = glob(["lib/**/*.zig"]), srcs = glob(["src/**/*.zig"]), main = "src/main.zig", @@ -23,3 +25,16 @@ zig_binary( ], visibility = ["//visibility:public"], ) + +# TODO(cerisier): Generate 1 translate-c toolchain per target platform +# So that the choice of doing this for a given platform is given to users. +# +# Right now we do this because the arocc source code has the same support +# for platforms than the official zig compiler and in the case of using a +# custom zig version with support for newer chips, arocc cannot be built. +zig_configure_binary( + name = "translate-c", + actual = ":translate-c_raw", + bootstrapped = 0, + visibility = ["//visibility:public"], +) diff --git a/third_party/uucode/config-storage-module.patch b/third_party/uucode/config-storage-module.patch new file mode 100644 index 00000000..d390986c --- /dev/null +++ b/third_party/uucode/config-storage-module.patch @@ -0,0 +1,33 @@ +diff --git a/src/config.zig b/src/config.zig +index 2cadab4..aa46c5d 100644 +--- a/src/config.zig ++++ b/src/config.zig +@@ -1,5 +1,5 @@ + const std = @import("std"); +-const storage = @import("storage.zig"); ++pub const storage = @import("storage.zig"); + const multi_slice = @import("multi_slice.zig"); + pub const quirks = @import("quirks.zig"); + pub const components = @import("components.zig"); +diff --git a/src/generate.zig b/src/generate.zig +index 6f4dcf5..8ef1d83 100644 +--- a/src/generate.zig ++++ b/src/generate.zig +@@ -1,7 +1,7 @@ + const std = @import("std"); +-const storage = @import("storage.zig"); + const config = @import("config.zig"); + const build_config = @import("build_config"); ++const storage = config.storage; + const inlineAssert = config.quirks.inlineAssert; + + pub const std_options: std.Options = .{ +@@ -339,7 +339,7 @@ pub fn main(init: std.process.Init.Minimal) !void { + \\const std = @import("std"); + \\const config = @import("config.zig"); + \\const types = config.types; +- \\const storage = @import("storage.zig"); ++ \\const storage = config.storage; + \\const build_config = @import("build_config"); + \\ + \\pub const get_components = build_config.get_components; diff --git a/third_party/uucode/repo.bzl b/third_party/uucode/repo.bzl index 6b913bbc..20a2f080 100644 --- a/third_party/uucode/repo.bzl +++ b/third_party/uucode/repo.bzl @@ -3,6 +3,7 @@ def _uucode_repo_impl(ctx): url = "https://github.com/jacobsandlund/uucode/archive/{}.tar.gz".format(ctx.attr.commit), stripPrefix = "uucode-{}".format(ctx.attr.commit), ) + ctx.patch(ctx.attr.config_storage_patch, strip = 1) ctx.symlink(ctx.attr.build_config, "build_config.zig") ctx.symlink(ctx.attr.build_file, "BUILD.bazel") @@ -13,6 +14,7 @@ _uucode_repo = repository_rule( "commit": attr.string(mandatory = True), "build_config": attr.label(mandatory = True, allow_single_file = True), "build_file": attr.label(mandatory = True, allow_single_file = True), + "config_storage_patch": attr.label(mandatory = True, allow_single_file = True), }, ) @@ -22,4 +24,5 @@ def repo(): commit = "f1c687f09174f9f11eda14d3e3ce497d68ab09a6", build_config = "//third_party/uucode:build_config.zig", build_file = "//third_party/uucode:uucode.bazel", + config_storage_patch = "//third_party/uucode:config-storage-module.patch", ) diff --git a/third_party/uucode/uucode.bazel b/third_party/uucode/uucode.bazel index 67b7eb39..f2b5634d 100644 --- a/third_party/uucode/uucode.bazel +++ b/third_party/uucode/uucode.bazel @@ -11,29 +11,22 @@ zig_library( "src/fields.zig", "src/multi_slice.zig", "src/quirks.zig", + "src/storage.zig", "src/types.zig", ], - zigopts = ["--dep", "storage.zig=_storage_gen"], # circular: config.zig <-> storage.zig -) - -zig_library( - name = "_storage_gen", - import_name = "storage.zig", - main = "src/storage.zig", - deps = [":_config_gen"], ) zig_library( name = "_build_config_gen", import_name = "build_config", main = "build_config.zig", - deps = [":_config_gen", ":_storage_gen"], + deps = [":_config_gen"], ) zig_binary( name = "gen_tables", main = "src/generate.zig", - deps = [":_build_config_gen", ":_config_gen", ":_storage_gen"], + deps = [":_build_config_gen", ":_config_gen"], ) genrule( @@ -61,30 +54,23 @@ zig_library( "src/fields.zig", "src/multi_slice.zig", "src/quirks.zig", + "src/storage.zig", ], deps = [":types"], - zigopts = ["--dep", "storage.zig=storage"], # circular: config.zig <-> storage.zig -) - -zig_library( - name = "storage", - import_name = "storage.zig", - main = "src/storage.zig", - deps = [":config"], ) zig_library( name = "build_config", import_name = "build_config", main = "build_config.zig", - deps = [":config", ":storage"], + deps = [":config"], ) zig_library( name = "tables_lib", import_name = "tables", main = ":tables", - deps = [":build_config", ":config", ":storage"], + deps = [":build_config", ":config"], ) zig_library( diff --git a/third_party/zls/zls_write_runner_zig_src.bzl b/third_party/zls/zls_write_runner_zig_src.bzl index 224e732d..2217db20 100644 --- a/third_party/zls/zls_write_runner_zig_src.bzl +++ b/third_party/zls/zls_write_runner_zig_src.bzl @@ -2,17 +2,30 @@ load("@bazel_lib//lib:paths.bzl", "to_rlocation_path") +def zig_toolchain_executable_rpath(ctx, zigtoolchaininfo): + if zigtoolchaininfo.zig_exe.file != None: + return to_rlocation_path(ctx, zigtoolchaininfo.zig_exe.file) + return zigtoolchaininfo.zig_exe.path + +def zig_toolchain_lib_rpath(ctx, zigtoolchaininfo): + if zigtoolchaininfo.zig_lib.file != None: + return to_rlocation_path(ctx, zigtoolchaininfo.zig_lib.file) + return zigtoolchaininfo.zig_lib.path + def _zls_write_runner_zig_src_impl(ctx): zigtoolchaininfo = ctx.toolchains["@rules_zig//zig:toolchain_type"].zigtoolchaininfo zlstoolchaininfo = ctx.toolchains["//third_party/zls:toolchain_type"].zlstoolchaininfo + zig_exe_rpath = zig_toolchain_executable_rpath(ctx, zigtoolchaininfo) + zig_lib_rpath = zig_toolchain_lib_rpath(ctx, zigtoolchaininfo) + zls_runner = ctx.outputs.out ctx.actions.expand_template( output = zls_runner, template = ctx.file._runner_tpl, substitutions = { - "@@__ZIG_EXE_RPATH__@@": to_rlocation_path(ctx, zigtoolchaininfo.zig_exe), - "@@__ZIG_LIB_RPATH__@@": to_rlocation_path(ctx, zigtoolchaininfo.zig_lib), + "@@__ZIG_EXE_RPATH__@@": zig_exe_rpath, + "@@__ZIG_LIB_RPATH__@@": zig_lib_rpath, "@@__ZLS_BIN_RPATH__@@": to_rlocation_path(ctx, zlstoolchaininfo.bin), "@@__ZLS_BUILD_RUNNER_RPATH__@@": to_rlocation_path(ctx, ctx.file.build_runner), "@@__GLOBAL_CACHE_PATH__@@": zigtoolchaininfo.zig_cache, From 55474ea31fc401a7dd44cc92a39942a7953b267c Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Sat, 16 May 2026 09:58:34 +0200 Subject: [PATCH 488/881] zls: resolve translate-c toolchain in zls rules (#538) This one slipped from #537 It worked because if there is no translate-c toolchain, zig uses zig translate-c. But we want to respect user choice here too. --- third_party/zls/zls_write_build_config.bzl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/third_party/zls/zls_write_build_config.bzl b/third_party/zls/zls_write_build_config.bzl index b65b968b..d5d863d8 100644 --- a/third_party/zls/zls_write_build_config.bzl +++ b/third_party/zls/zls_write_build_config.bzl @@ -69,6 +69,8 @@ def format_main_file(main): def _zls_write_build_config_impl(ctx): zigtoolchaininfo = ctx.toolchains["@rules_zig//zig:toolchain_type"].zigtoolchaininfo zigtargetinfo = ctx.toolchains["@rules_zig//zig/target:toolchain_type"].zigtargetinfo + translate_c_toolchain = ctx.toolchains["@rules_zig//zig/translate-c:toolchain_type"] + translatectoolchaininfo = translate_c_toolchain.translatectoolchaininfo if translate_c_toolchain else None c_module_contexts = [] c_module_inputs = [] @@ -99,6 +101,7 @@ def _zls_write_build_config_impl(ctx): zigtoolchaininfo = zigtoolchaininfo, global_args = global_args, cc_infos = [cc_info], + translatectoolchaininfo = translatectoolchaininfo, ) c_module_contexts = [depset( @@ -168,6 +171,7 @@ zls_write_build_config = rule( toolchains = [ "@rules_zig//zig:toolchain_type", "@rules_zig//zig/target:toolchain_type", + config_common.toolchain_type("@rules_zig//zig/translate-c:toolchain_type", mandatory = False), ] + use_cc_toolchain(mandatory = False), fragments = ["cpp", "apple"], ) From 57424e6c41f087328fde0793eee43e34943c7e60 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 18 May 2026 15:31:00 +0200 Subject: [PATCH 489/881] zml/buffer: added `fromBytesOpts` (#544) Since there is `fromSliceOpts`, there should be `fromBytesOpts` --- zml/buffer.zig | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index fc3a1ce2..f15b411f 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -152,7 +152,11 @@ pub const Buffer = struct { /// Copies the given Zig bytes to the accelerator memory and /// return a Buffer with the given dimensions. pub fn fromBytes(io: std.Io, platform: *const Platform, sh: Shape, sharding: Sharding, data: []const u8) !Buffer { - return from(io, platform, sh, sharding, data, .{}); + return fromBytesOpts(io, platform, sh, sharding, data, .{}); + } + + pub fn fromBytesOpts(io: std.Io, platform: *const Platform, sh: Shape, sharding: Sharding, data: []const u8, opts: FromOptions) !Buffer { + return from(io, platform, sh, sharding, data, opts); } /// Copies the given zml.Slice to the accelerator memory and From 61ed004556f37aa1416d448b36b1b96da906c645 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 18 May 2026 15:40:04 +0200 Subject: [PATCH 490/881] zml/attention: big rework of paged attention (#536) List of changes: * Removed `Context` * Simplified metadata to not contain scratch buffers. They are created uninitialized on the fly instead (thanks to #535) * Added helper for memory movement (host_pinned -> device) (thanks to #534) * FA2/FA3 don't read the `max_seqlen_q` from host anymore. This was error prone. Instead rely on a fixed value for now. (It's error prone because it's baked in the command buffer and can result in wrong value if the value changes but the command buffer is not recorded again). --- MODULE.bazel.lock | 655 ----------------------------- zml/attention/flashattn.zig | 517 +++++++++++++++-------- zml/attention/paged_attention.zig | 39 +- zml/attention/tpu_attention.zig | 32 +- zml/attention/triton_attention.zig | 38 +- 5 files changed, 394 insertions(+), 887 deletions(-) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index ca07200a..5e176c22 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -450,56 +450,6 @@ ] } }, - "@@aspect_rules_py+//uv/private/tomltool:extension.bzl%tomltool": { - "general": { - "bzlTransitiveDigest": "9BC6q6vdlG+lqF9L13wVxxakVkZBx8buKPFIndN6mHk=", - "usagesDigest": "tQ76lH+Nn5OlAFLvBbnfvnCzNIiwGgY6tOumYZXH44w=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "toml2json_aarch64_osx_libsystem": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_darwin_arm64", - "sha256": "9adc44d976e8f5baf5a6d613ceb3db6e5f56b2e22e75ac56521fdce62f227d88", - "executable": true - } - }, - "toml2json_x86_64_osx_libsystem": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_darwin_amd64", - "sha256": "425bac4015394fd9840fabcd52ac60fc796a9c10655b36c57e36ffb7dc9f3dd4", - "executable": true - } - }, - "toml2json_aarch64_linux_gnu": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_linux_arm64", - "sha256": "be8376c8e3232a242eae0d187f741a475498a949b942361a9af6e95072ef5670", - "executable": true - } - }, - "toml2json_x86_64_linux_gnu": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_linux_amd64", - "sha256": "f3dd54fabf2d27d0c027b0421860e5d9d909080be1613f0ffd87057633b65e9a", - "executable": true - } - } - }, - "recordedRepoMappingEntries": [ - [ - "aspect_rules_py+", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, "@@aspect_tools_telemetry+//:extension.bzl%telemetry": { "general": { "bzlTransitiveDigest": "MePriaXmQNSqUfE+YQvEtzBc2bU1mjITINIffFiZYgo=", @@ -532,258 +482,6 @@ ] } }, - "@@cel-spec+//:extensions.bzl%non_module_dependencies": { - "general": { - "bzlTransitiveDigest": "VN89nCHLUuqo5KktrpNZqv8O+QKrrsGIqdoLPn91hhY=", - "usagesDigest": "HFQJtQrL9nKaFZEjgwaHVMHALMW+cafu696xy7J4ueM=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "com_google_googleapis": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "sha256": "bd8e735d881fb829751ecb1a77038dda4a8d274c45490cb9fcf004583ee10571", - "strip_prefix": "googleapis-07c27163ac591955d736f3057b1619ece66f5b99", - "urls": [ - "https://github.com/googleapis/googleapis/archive/07c27163ac591955d736f3057b1619ece66f5b99.tar.gz" - ] - } - } - }, - "recordedRepoMappingEntries": [ - [ - "cel-spec+", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@cel-spec+//:googleapis_ext.bzl%googleapis_ext": { - "general": { - "bzlTransitiveDigest": "65z4sd8o73L2a8X2Pd3yAjn73l9HBG45JVnVY2DUbzI=", - "usagesDigest": "Ek7VfZ+tuyRBx/1h5wcmtnW9EGpOb0dkXUwBluZbD8k=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "com_google_googleapis_imports": { - "repoRuleId": "@@cel-spec++non_module_dependencies+com_google_googleapis//:repository_rules.bzl%switched_rules", - "attributes": { - "rules": { - "proto_library_with_info": [ - "", - "" - ], - "moved_proto_library": [ - "", - "" - ], - "java_proto_library": [ - "", - "" - ], - "java_grpc_library": [ - "", - "" - ], - "java_gapic_library": [ - "", - "" - ], - "java_gapic_test": [ - "", - "" - ], - "java_gapic_assembly_gradle_pkg": [ - "", - "" - ], - "py_proto_library": [ - "", - "" - ], - "py_grpc_library": [ - "", - "" - ], - "py_gapic_library": [ - "", - "" - ], - "py_test": [ - "", - "" - ], - "py_gapic_assembly_pkg": [ - "", - "" - ], - "py_import": [ - "", - "" - ], - "go_proto_library": [ - "", - "" - ], - "go_library": [ - "", - "" - ], - "go_test": [ - "", - "" - ], - "go_gapic_library": [ - "", - "" - ], - "go_gapic_assembly_pkg": [ - "", - "" - ], - "cc_proto_library": [ - "native.cc_proto_library", - "" - ], - "cc_grpc_library": [ - "", - "" - ], - "cc_gapic_library": [ - "", - "" - ], - "php_proto_library": [ - "", - "php_proto_library" - ], - "php_grpc_library": [ - "", - "php_grpc_library" - ], - "php_gapic_library": [ - "", - "php_gapic_library" - ], - "php_gapic_assembly_pkg": [ - "", - "php_gapic_assembly_pkg" - ], - "nodejs_gapic_library": [ - "", - "typescript_gapic_library" - ], - "nodejs_gapic_assembly_pkg": [ - "", - "typescript_gapic_assembly_pkg" - ], - "ruby_proto_library": [ - "", - "" - ], - "ruby_grpc_library": [ - "", - "" - ], - "ruby_ads_gapic_library": [ - "", - "" - ], - "ruby_cloud_gapic_library": [ - "", - "" - ], - "ruby_gapic_assembly_pkg": [ - "", - "" - ], - "csharp_proto_library": [ - "", - "" - ], - "csharp_grpc_library": [ - "", - "" - ], - "csharp_gapic_library": [ - "", - "" - ], - "csharp_gapic_assembly_pkg": [ - "", - "" - ] - } - } - } - }, - "recordedRepoMappingEntries": [ - [ - "cel-spec+", - "com_google_googleapis", - "cel-spec++non_module_dependencies+com_google_googleapis" - ] - ] - } - }, - "@@envoy_api+//bazel:repositories.bzl%non_module_deps": { - "general": { - "bzlTransitiveDigest": "JQHl7KCG4m+2+/1IddYzhdiav5qs2iUe+gkg5f+kUPA=", - "usagesDigest": "OP2p9QPxt4WFY/lah8Iecqt6SOUwZHs89hYRyaV+rSg=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "prometheus_metrics_model": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/prometheus/client_model/archive/v0.6.1.tar.gz" - ], - "sha256": "b9b690bc35d80061f255faa7df7621eae39fe157179ccd78ff6409c3b004f05e", - "strip_prefix": "client_model-0.6.1", - "build_file_content": "\nload(\"@envoy_api//bazel:api_build_system.bzl\", \"api_cc_py_proto_library\")\nload(\"@io_bazel_rules_go//proto:def.bzl\", \"go_proto_library\")\n\napi_cc_py_proto_library(\n name = \"client_model\",\n srcs = [\n \"io/prometheus/client/metrics.proto\",\n ],\n visibility = [\"//visibility:public\"],\n)\n\ngo_proto_library(\n name = \"client_model_go_proto\",\n importpath = \"github.com/prometheus/client_model/go\",\n proto = \":client_model\",\n visibility = [\"//visibility:public\"],\n)\n" - } - }, - "com_github_bufbuild_buf": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/bufbuild/buf/releases/download/v1.49.0/buf-Linux-x86_64.tar.gz" - ], - "sha256": "ee8da9748249f7946d79191e36469ce7bc3b8ba80019bff1fa4289a44cbc23bf", - "strip_prefix": "buf", - "build_file_content": "\npackage(\n default_visibility = [\"//visibility:public\"],\n)\n\nfilegroup(\n name = \"buf\",\n srcs = [\n \"@com_github_bufbuild_buf//:bin/buf\",\n ],\n tags = [\"manual\"], # buf is downloaded as a linux binary; tagged manual to prevent build for non-linux users\n)\n" - } - }, - "envoy_toolshed": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/envoyproxy/toolshed/archive/bazel-v0.2.2.tar.gz" - ], - "sha256": "443fe177aba0cef8c17b7a48905c925c67b09005b10dd70ff12cd9f729a72d51", - "strip_prefix": "toolshed-bazel-v0.2.2/bazel" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "envoy_api+", - "bazel_tools", - "bazel_tools" - ], - [ - "envoy_api+", - "envoy_api", - "envoy_api+" - ] - ] - } - }, "@@googleapis+//:extensions.bzl%switched_rules": { "general": { "bzlTransitiveDigest": "JLpojYeGHLdgnwSUuQWOrtkL9UMygBUUEFJZh7ASlmA=", @@ -951,126 +649,6 @@ "recordedRepoMappingEntries": [] } }, - "@@rules_apple+//apple:apple.bzl%provisioning_profile_repository_extension": { - "general": { - "bzlTransitiveDigest": "uPXHPr6/YdHK89/x69cJN1hdhUF74hBsmCHaOkor2Fg=", - "usagesDigest": "vsJl8Rw5NL+5Ag2wdUDoTeRF/5klkXO8545Iy7U1Q08=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "local_provisioning_profiles": { - "repoRuleId": "@@rules_apple+//apple/internal:local_provisioning_profiles.bzl%provisioning_profile_repository", - "attributes": {} - } - }, - "recordedRepoMappingEntries": [ - [ - "apple_support+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "bazel_tools", - "rules_cc", - "rules_cc+" - ], - [ - "rules_apple+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "rules_apple+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_apple+", - "build_bazel_apple_support", - "apple_support+" - ], - [ - "rules_apple+", - "build_bazel_rules_swift", - "rules_swift+" - ], - [ - "rules_cc+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_cc+", - "cc_compatibility_proxy", - "rules_cc++compatibility_proxy+cc_compatibility_proxy" - ], - [ - "rules_cc+", - "rules_cc", - "rules_cc+" - ], - [ - "rules_cc++compatibility_proxy+cc_compatibility_proxy", - "rules_cc", - "rules_cc+" - ], - [ - "rules_swift+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "rules_swift+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_swift+", - "build_bazel_apple_support", - "apple_support+" - ], - [ - "rules_swift+", - "build_bazel_rules_swift", - "rules_swift+" - ], - [ - "rules_swift+", - "build_bazel_rules_swift_local_config", - "rules_swift++non_module_deps+build_bazel_rules_swift_local_config" - ] - ] - } - }, - "@@rules_apple+//apple:extensions.bzl%non_module_deps": { - "general": { - "bzlTransitiveDigest": "fe3Htkk5VXHdsIrdLtxsT4oUgADmqgmxwx83bja4UqY=", - "usagesDigest": "M3VqFpeTCo4qmrNKGZw0dxBHvTYDrfV3cscGzlSAhQ4=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "xctestrunner": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/google/xctestrunner/archive/b7698df3d435b6491b4b4c0f9fc7a63fbed5e3a6.tar.gz" - ], - "strip_prefix": "xctestrunner-b7698df3d435b6491b4b4c0f9fc7a63fbed5e3a6", - "sha256": "ae3a063c985a8633cb7eb566db21656f8db8eb9a0edb8c182312c7f0db53730d" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_apple+", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { "general": { "bzlTransitiveDigest": "AEsQvKwdstptzt98AO1bJYfoWpPUn2upx53ra6o/Rc8=", @@ -1355,80 +933,6 @@ ] } }, - "@@rules_perl+//perl:extensions.bzl%perl_repositories": { - "general": { - "bzlTransitiveDigest": "hVM1gWzR35NfpY87MWdNF1xPgjhXCgJv3lHedzj33TY=", - "usagesDigest": "qSSNDdCNVxNhY36wMndEAFacdhR0ooLTmumfad0km9s=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "perl_darwin_arm64": { - "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", - "attributes": { - "strip_prefix": "perl-darwin-arm64", - "sha256": "285769f3c50c339fb59a3987b216ae3c5c573b95babe6875a1ef56fb178433da", - "urls": [ - "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-darwin-arm64.tar.xz" - ] - } - }, - "perl_darwin_amd64": { - "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", - "attributes": { - "strip_prefix": "perl-darwin-amd64", - "sha256": "63bc5ee36f5394d71c50cca6cafdd333ee58f9eaa40bca63c85f9bd06f2c1fd6", - "urls": [ - "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-darwin-amd64.tar.xz" - ] - } - }, - "perl_linux_amd64": { - "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", - "attributes": { - "strip_prefix": "perl-linux-amd64", - "sha256": "3bdffa9d7a3f97c0207314637b260ba5115b1d0829f97e3e2e301191a4d4d076", - "urls": [ - "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-linux-amd64.tar.xz" - ] - } - }, - "perl_linux_arm64": { - "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", - "attributes": { - "strip_prefix": "perl-linux-arm64", - "sha256": "6fa4ece99e790ecbc2861f6ecb7b52694c01c2eeb215b4370f16a3b12d952117", - "urls": [ - "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-linux-arm64.tar.xz" - ] - } - }, - "perl_windows_x86_64": { - "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", - "attributes": { - "strip_prefix": "", - "sha256": "aeb973da474f14210d3e1a1f942dcf779e2ae7e71e4c535e6c53ebabe632cc98", - "urls": [ - "https://mirror.bazel.build/strawberryperl.com/download/5.32.1.1/strawberry-perl-5.32.1.1-64bit.zip", - "https://strawberryperl.com/download/5.32.1.1/strawberry-perl-5.32.1.1-64bit.zip" - ] - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_perl+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_perl+", - "rules_perl", - "rules_perl+" - ] - ] - } - }, "@@rules_python+//python/uv:uv.bzl%uv": { "general": { "bzlTransitiveDigest": "LM8+g04lV+afYP67YcE7wenTJMrNqvF5u/ZnOctwctY=", @@ -1601,165 +1105,6 @@ ] } }, - "@@rules_swift+//swift:extensions.bzl%non_module_deps": { - "general": { - "bzlTransitiveDigest": "vwewHB1JdoOb38MyyeRQPIdGOkEKD52YNUYgGwmeOy8=", - "usagesDigest": "mhACFnrdMv9Wi0Mt67bxocJqviRkDSV+Ee5Mqdj5akA=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "com_github_apple_swift_protobuf": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-protobuf/archive/1.20.2.tar.gz" - ], - "sha256": "3fb50bd4d293337f202d917b6ada22f9548a0a0aed9d9a4d791e6fbd8a246ebb", - "strip_prefix": "swift-protobuf-1.20.2/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_protobuf/BUILD.overlay" - } - }, - "com_github_grpc_grpc_swift": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/grpc/grpc-swift/archive/1.16.0.tar.gz" - ], - "sha256": "58b60431d0064969f9679411264b82e40a217ae6bd34e17096d92cc4e47556a5", - "strip_prefix": "grpc-swift-1.16.0/", - "build_file": "@@rules_swift+//third_party:com_github_grpc_grpc_swift/BUILD.overlay" - } - }, - "com_github_apple_swift_docc_symbolkit": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-docc-symbolkit/archive/refs/tags/swift-5.10-RELEASE.tar.gz" - ], - "sha256": "de1d4b6940468ddb53b89df7aa1a81323b9712775b0e33e8254fa0f6f7469a97", - "strip_prefix": "swift-docc-symbolkit-swift-5.10-RELEASE", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_docc_symbolkit/BUILD.overlay" - } - }, - "com_github_apple_swift_nio": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio/archive/2.42.0.tar.gz" - ], - "sha256": "e3304bc3fb53aea74a3e54bd005ede11f6dc357117d9b1db642d03aea87194a0", - "strip_prefix": "swift-nio-2.42.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_http2": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-http2/archive/1.26.0.tar.gz" - ], - "sha256": "f0edfc9d6a7be1d587e5b403f2d04264bdfae59aac1d74f7d974a9022c6d2b25", - "strip_prefix": "swift-nio-http2-1.26.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_http2/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_transport_services": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-transport-services/archive/1.15.0.tar.gz" - ], - "sha256": "f3498dafa633751a52b9b7f741f7ac30c42bcbeb3b9edca6d447e0da8e693262", - "strip_prefix": "swift-nio-transport-services-1.15.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_transport_services/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_extras": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-extras/archive/1.4.0.tar.gz" - ], - "sha256": "4684b52951d9d9937bb3e8ccd6b5daedd777021ef2519ea2f18c4c922843b52b", - "strip_prefix": "swift-nio-extras-1.4.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_extras/BUILD.overlay" - } - }, - "com_github_apple_swift_log": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-log/archive/1.4.4.tar.gz" - ], - "sha256": "48fe66426c784c0c20031f15dc17faf9f4c9037c192bfac2f643f65cb2321ba0", - "strip_prefix": "swift-log-1.4.4/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_log/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_ssl": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-ssl/archive/2.23.0.tar.gz" - ], - "sha256": "4787c63f61dd04d99e498adc3d1a628193387e41efddf8de19b8db04544d016d", - "strip_prefix": "swift-nio-ssl-2.23.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_ssl/BUILD.overlay" - } - }, - "com_github_apple_swift_collections": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-collections/archive/1.0.4.tar.gz" - ], - "sha256": "d9e4c8a91c60fb9c92a04caccbb10ded42f4cb47b26a212bc6b39cc390a4b096", - "strip_prefix": "swift-collections-1.0.4/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_collections/BUILD.overlay" - } - }, - "com_github_apple_swift_atomics": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-atomics/archive/1.1.0.tar.gz" - ], - "sha256": "1bee7f469f7e8dc49f11cfa4da07182fbc79eab000ec2c17bfdce468c5d276fb", - "strip_prefix": "swift-atomics-1.1.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_atomics/BUILD.overlay" - } - }, - "build_bazel_rules_swift_index_import": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "build_file": "@@rules_swift+//third_party:build_bazel_rules_swift_index_import/BUILD.overlay", - "canonical_id": "index-import-5.8", - "urls": [ - "https://github.com/MobileNativeFoundation/index-import/releases/download/5.8.0.1/index-import.tar.gz" - ], - "sha256": "28c1ffa39d99e74ed70623899b207b41f79214c498c603915aef55972a851a15" - } - }, - "build_bazel_rules_swift_local_config": { - "repoRuleId": "@@rules_swift+//swift/internal:swift_autoconfiguration.bzl%swift_autoconfiguration", - "attributes": {} - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_swift+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_swift+", - "build_bazel_rules_swift", - "rules_swift+" - ] - ] - } - }, "@@rules_zig+//zig:extensions.bzl%zig": { "general": { "bzlTransitiveDigest": "mBSDTk+QEb2UlXBSWQWCw8A/tUSarS8PCSbhFYCkotc=", diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 6d6ada25..d80e558a 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -558,6 +558,20 @@ pub const paged_fa2 = struct { .mixed => |v| .{ .mixed = v.options }, }; } + + pub fn onMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { + return switch (self) { + .mixed => |s| .{ .mixed = s.onMemory(memory) }, + .decode => |s| .{ .decode = s.onMemory(memory) }, + }; + } + + pub fn toMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { + return switch (self) { + .mixed => |s| .{ .mixed = s.toMemory(memory) }, + .decode => |s| .{ .decode = s.toMemory(memory) }, + }; + } }; pub const DecodeOptions = struct { @@ -575,7 +589,6 @@ pub const paged_fa2 = struct { cu_seqlens_q: zml.Tensor, seqused_k: zml.Tensor, - metadata: DecodeMetadata, options: DecodeOptions, pub fn init(options: DecodeOptions) DecodeParameters { @@ -583,7 +596,6 @@ pub const paged_fa2 = struct { .block_table = zml.Tensor.init(.{ .b = options.batch_size, .p = options.max_num_pages }, .i32), .cu_seqlens_q = zml.Tensor.init(.{ .b = options.batch_size + 1 }, .i32), .seqused_k = zml.Tensor.init(.{ .b = options.batch_size }, .i32), - .metadata = DecodeMetadata.init(options), .options = options, }; } @@ -594,43 +606,51 @@ pub const paged_fa2 = struct { allocation_size += self.block_table.byteSize(); allocation_size += self.cu_seqlens_q.byteSize(); allocation_size += self.seqused_k.byteSize(); - allocation_size += self.metadata.allocationSize(); - return allocation_size; - } - }; - - pub const DecodeMetadata = struct { - out_accum: zml.Tensor, - - pub fn init(opts: DecodeOptions) DecodeMetadata { - const shape = zml.Shape.init(.{ + const out_accum_shape: zml.Shape = .init(.{ .splits = MAX_NUM_SPLITS, - .hkv = opts.num_kv_heads, - .hg = @divExact(opts.num_heads, opts.num_kv_heads), - .b = opts.max_token_count, - .hd = opts.head_dim, - }, .f32).withPartitioning(.{ .hkv = .model }); - return .{ - .out_accum = .fromShape(shape), - }; - } + .hkv = self.options.num_kv_heads, + .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), + .b = self.options.batch_size, + .hd = self.options.head_dim, + }, .f32); + allocation_size += out_accum_shape.byteSize(); - pub fn initBuffers(self: *const DecodeMetadata, io: std.Io, platform: zml.Platform, out_accum_sharding: zml.Sharding) !zml.Bufferized(DecodeMetadata) { - return .{ - .out_accum = try zml.Buffer.uninitialized(io, &platform, self.out_accum.shape(), out_accum_sharding, .{}), - }; - } + const softmax_lse_shape: zml.Shape = .init(.{ + .hkv = self.options.num_kv_heads, + .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), + .b = self.options.batch_size, + }, .f32); + allocation_size += softmax_lse_shape.byteSize(); - pub fn deinitBuffers(self: *const zml.Bufferized(DecodeMetadata)) void { - self.out_accum.deinit(); - } + const softmax_lse_accum_shape: zml.Shape = .init(.{ + .splits = MAX_NUM_SPLITS, + .hkv = self.options.num_kv_heads, + .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), + .b = self.options.batch_size, + }, .f32); + allocation_size += softmax_lse_accum_shape.byteSize(); - pub fn allocationSize(self: DecodeMetadata) usize { - var allocation_size: usize = 0; - allocation_size += self.out_accum.byteSize(); return allocation_size; } + + pub fn onMemory(self: DecodeParameters, memory: zml.platform.Memory.Kind) DecodeParameters { + return .{ + .block_table = self.block_table.onMemory(memory), + .cu_seqlens_q = self.cu_seqlens_q.onMemory(memory), + .seqused_k = self.seqused_k.onMemory(memory), + .options = self.options, + }; + } + + pub fn toMemory(self: DecodeParameters, memory: zml.platform.Memory.Kind) DecodeParameters { + return .{ + .block_table = self.block_table.toMemory(memory), + .cu_seqlens_q = self.cu_seqlens_q.toMemory(memory), + .seqused_k = self.seqused_k.toMemory(memory), + .options = self.options, + }; + } }; pub const MixedOptions = struct { @@ -638,6 +658,7 @@ pub const paged_fa2 = struct { batch_size_decode: usize, max_num_pages: usize, max_seqlen_k: usize, + max_seqlen_q: usize, max_token_count: usize, num_heads: usize, num_kv_heads: usize, @@ -683,64 +704,110 @@ pub const paged_fa2 = struct { allocation_size += self.seqused_k_decode.byteSize(); allocation_size += self.metadata.allocationSize(); + const out_accum_prefill_shape: zml.Shape = .init(.{ + .splits = MAX_NUM_SPLITS, + .hkv = self.options.num_kv_heads, + .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), + .b = self.options.max_token_count, + .hd = self.options.head_dim, + }, .f32); + allocation_size += out_accum_prefill_shape.byteSize(); + + const softmax_lse_prefill_shape: zml.Shape = .init(.{ + .hkv = self.options.num_kv_heads, + .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), + .b = self.options.max_token_count, + }, .f32); + allocation_size += softmax_lse_prefill_shape.byteSize(); + + const softmax_lse_accum_prefill_shape: zml.Shape = .init(.{ + .splits = MAX_NUM_SPLITS, + .hkv = self.options.num_kv_heads, + .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), + .b = self.options.max_token_count, + }, .f32); + allocation_size += softmax_lse_accum_prefill_shape.byteSize(); + + const out_accum_decode_shape: zml.Shape = .init(.{ + .splits = MAX_NUM_SPLITS, + .hkv = self.options.num_kv_heads, + .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), + .b = self.options.batch_size_decode, + .hd = self.options.head_dim, + }, .f32); + allocation_size += out_accum_decode_shape.byteSize(); + + const softmax_lse_decode_shape: zml.Shape = .init(.{ + .hkv = self.options.num_kv_heads, + .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), + .b = self.options.max_token_count, + }, .f32); + allocation_size += softmax_lse_decode_shape.byteSize(); + + const softmax_lse_accum_decode_shape: zml.Shape = .init(.{ + .splits = MAX_NUM_SPLITS, + .hkv = self.options.num_kv_heads, + .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), + .b = self.options.batch_size_decode, + }, .f32); + allocation_size += softmax_lse_accum_decode_shape.byteSize(); + return allocation_size; } + + pub fn onMemory(self: MixedParameters, memory: zml.platform.Memory.Kind) MixedParameters { + return .{ + .block_table_prefill = self.block_table_prefill.onMemory(memory), + .cu_seqlens_q_prefill = self.cu_seqlens_q_prefill.onMemory(memory), + .seqused_k_prefill = self.seqused_k_prefill.onMemory(memory), + .block_table_decode = self.block_table_decode.onMemory(memory), + .cu_seqlens_q_decode = self.cu_seqlens_q_decode.onMemory(memory), + .seqused_k_decode = self.seqused_k_decode.onMemory(memory), + .metadata = self.metadata.onMemory(memory), + .options = self.options, + }; + } + + pub fn toMemory(self: MixedParameters, memory: zml.platform.Memory.Kind) MixedParameters { + return .{ + .block_table_prefill = self.block_table_prefill.toMemory(memory), + .cu_seqlens_q_prefill = self.cu_seqlens_q_prefill.toMemory(memory), + .seqused_k_prefill = self.seqused_k_prefill.toMemory(memory), + .block_table_decode = self.block_table_decode.toMemory(memory), + .cu_seqlens_q_decode = self.cu_seqlens_q_decode.toMemory(memory), + .seqused_k_decode = self.seqused_k_decode.toMemory(memory), + .metadata = self.metadata.toMemory(memory), + .options = self.options, + }; + } }; pub const MixedMetadata = struct { - out_accum: zml.Tensor, - host_metadata: zml.Tensor, + decode_offset: zml.Tensor, pub fn init(opts: MixedOptions) MixedMetadata { - const shape = zml.Shape.init(.{ - .splits = MAX_NUM_SPLITS, - .hkv = opts.num_kv_heads, - .hg = @divExact(opts.num_heads, opts.num_kv_heads), - .b = opts.max_token_count, - .hd = opts.head_dim, - }, .f32).withPartitioning(.{ .hkv = .model }); + _ = opts; return .{ - .out_accum = .fromShape(shape), - .host_metadata = .init(.{2}, .i32), + .decode_offset = .init(.{}, .i32), }; } - pub fn initBuffers(self: *const MixedMetadata, io: std.Io, platform: zml.Platform, out_accum_sharding: zml.Sharding) !zml.Bufferized(MixedMetadata) { - return .{ - .out_accum = try zml.Buffer.uninitialized(io, &platform, self.out_accum.shape(), out_accum_sharding, .{}), - .host_metadata = undefined, - }; - } - - pub fn deinitBuffers(self: *const zml.Bufferized(MixedMetadata)) void { - self.out_accum.deinit(); - } - pub fn allocationSize(self: MixedMetadata) usize { var allocation_size: usize = 0; - allocation_size += self.out_accum.byteSize(); - allocation_size += self.host_metadata.byteSize(); + allocation_size += self.decode_offset.byteSize(); return allocation_size; } - }; - pub const Context = struct { - max_seqlen_k: usize, - decode_offset: ?zml.Tensor = null, + pub fn onMemory(self: MixedMetadata, memory: zml.platform.Memory.Kind) MixedMetadata { + return .{ + .decode_offset = self.decode_offset.onMemory(memory), + }; + } - pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { - _ = num_heads; - _ = num_kv_heads; - _ = head_dim; - _ = page_size; - const max_seqlen_k = switch (parameters) { - inline else => |v| v.options.max_seqlen_k, + pub fn toMemory(self: MixedMetadata, memory: zml.platform.Memory.Kind) MixedMetadata { + return .{ + .decode_offset = self.decode_offset.toMemory(memory), }; - const decode_offset = switch (parameters) { - .mixed => |mixed_parameters| mixed_parameters.metadata.host_metadata.slice1d(0, .{ .end = 1 }).squeeze(0), - .decode => null, - }; - return .{ .max_seqlen_k = max_seqlen_k, .decode_offset = decode_offset }; } }; @@ -764,10 +831,8 @@ pub const paged_fa2 = struct { const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[7]); const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[9]); - const host_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[10]); const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); - const max_seqlen_q = @as([*]i32, @ptrCast(@alignCast(host_metadata.ptr)))[1]; const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { const head_dim = q.shape.dim(2); break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); @@ -775,6 +840,7 @@ pub const paged_fa2 = struct { const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_q: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_q").?; const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; @@ -880,7 +946,7 @@ pub const paged_fa2 = struct { } }; - pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { + pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { stdx.debug.assert(q.shape().hasTags(.{ .b, .hg, .hkv, .hd }), "Expected q to have tags .b, .h, .hd", .{}); stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); @@ -901,19 +967,20 @@ pub const paged_fa2 = struct { const cu_seqlens_q = decode_parameters.cu_seqlens_q.withPartitioning(.{ .b = .replicated }); const seqused_k = decode_parameters.seqused_k.withPartitioning(.{ .b = .replicated }); - const out_accum = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + const out_accum = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, .hkv = num_kv_heads, .hg = num_head_groups, .b = q.dim(.b), .hd = head_dim, }, .f32)).withPartitioning(.{ .hkv = .model }); - const softmax_lse = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + + const softmax_lse = zml.Tensor.uninitialized(.init(.{ .hkv = num_kv_heads, .hg = num_head_groups, .b = q.dim(.b), }, .f32)).withPartitioning(.{ .hkv = .model }); - const softmax_lse_accum = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + const softmax_lse_accum = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, .hkv = num_kv_heads, .hg = num_head_groups, @@ -947,7 +1014,7 @@ pub const paged_fa2 = struct { .{ .metadata = .{ .is_causal = opts.is_causal, - .max_seqlen_k = context.max_seqlen_k, + .max_seqlen_k = decode_parameters.options.max_seqlen_k, .num_heads = num_heads_per_shard, .window_size_left = opts.sliding_window, .softmax_scale = opts.scale, @@ -980,21 +1047,20 @@ pub const paged_fa2 = struct { const block_table_decode = mixed_parameters.block_table_decode.withPartitioning(.{ .b = .replicated }); const cu_seqlens_q_decode = mixed_parameters.cu_seqlens_q_decode.withPartitioning(.{ .b = .replicated }); const seqused_k_decode = mixed_parameters.seqused_k_decode.withPartitioning(.{ .b = .replicated }); - const host_metadata = mixed_parameters.metadata.host_metadata.withTags(.{.x}).withPartitioning(.{ .x = .replicated }); - const out_accum_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + const out_accum_prefill = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, .hkv = num_kv_heads, .hg = num_head_groups, .b = q.dim(.b), .hd = head_dim, }, .f32)).withPartitioning(.{ .hkv = .model }); - const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + const softmax_lse_prefill = zml.Tensor.uninitialized(.init(.{ .hkv = num_kv_heads, .hg = num_head_groups, .b = q.dim(.b), }, .f32)).withPartitioning(.{ .hkv = .model }); - const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + const softmax_lse_accum_prefill = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, .hkv = num_kv_heads, .hg = num_head_groups, @@ -1018,13 +1084,13 @@ pub const paged_fa2 = struct { softmax_lse_prefill, softmax_lse_accum_prefill, out_accum_prefill, - host_metadata, }, output_shape, .{ .metadata = .{ .is_causal = opts.is_causal, - .max_seqlen_k = context.max_seqlen_k, + .max_seqlen_k = mixed_parameters.options.max_seqlen_k, + .max_seqlen_q = mixed_parameters.options.max_seqlen_q, .num_heads = num_heads_per_shard, .window_size_left = opts.sliding_window, .softmax_scale = opts.scale, @@ -1043,26 +1109,26 @@ pub const paged_fa2 = struct { o = o.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = num_head_groups }); const batch_dim_decode = block_table_decode.dim(0); - const out_accum_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + const out_accum_decode = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, .hkv = num_kv_heads, .hg = num_head_groups, .b = batch_dim_decode, .hd = head_dim, }, .f32)).withPartitioning(.{ .hkv = .model }); - const softmax_lse_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + const softmax_lse_decode = zml.Tensor.uninitialized(.init(.{ .hkv = num_kv_heads, .hg = num_head_groups, .b = batch_dim_decode, }, .f32)).withPartitioning(.{ .hkv = .model }); - const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + const softmax_lse_accum_decode = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, .hkv = num_kv_heads, .hg = num_head_groups, .b = batch_dim_decode, }, .f32)).withPartitioning(.{ .hkv = .model }); const dummy_cu_seqlens_k_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(cu_seqlens_q_decode.shape()); - var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_dim_decode }); + var q_decode = q.dynamicSlice1d(0, .{ .start = mixed_parameters.metadata.decode_offset, .len = batch_dim_decode }); if (seqlenq_ngroups_swapped) { q_decode = q_decode.transpose(.{ .b, .hg, .hkv, .hd }).merge(.{ .b = .{ .b, .hg } }).withPartitioning(.{ .hkv = .model }); @@ -1088,7 +1154,7 @@ pub const paged_fa2 = struct { .{ .metadata = .{ .is_causal = opts.is_causal, - .max_seqlen_k = context.max_seqlen_k, + .max_seqlen_k = mixed_parameters.options.max_seqlen_k, .num_heads = num_heads_per_shard, .window_size_left = opts.sliding_window, .softmax_scale = opts.scale, @@ -1110,7 +1176,7 @@ pub const paged_fa2 = struct { o_decode = o_decode.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = num_head_groups }); } - o = o.dynamicUpdateSlice1d(o_decode, 0, context.decode_offset.?); + o = o.dynamicUpdateSlice1d(o_decode, 0, mixed_parameters.metadata.decode_offset); break :b o; }, }; @@ -1168,6 +1234,20 @@ pub const paged_fa3 = struct { .mixed => |v| .{ .mixed = v.options }, }; } + + pub fn onMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { + return switch (self) { + .mixed => |s| .{ .mixed = s.onMemory(memory) }, + .decode => |s| .{ .decode = s.onMemory(memory) }, + }; + } + + pub fn toMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { + return switch (self) { + .mixed => |s| .{ .mixed = s.toMemory(memory) }, + .decode => |s| .{ .decode = s.toMemory(memory) }, + }; + } }; pub const DecodeOptions = struct { @@ -1185,7 +1265,6 @@ pub const paged_fa3 = struct { cu_seqlens_q: zml.Tensor, seqused_k: zml.Tensor, - metadata: DecodeMetadata, options: DecodeOptions, pub fn init(options: DecodeOptions) DecodeParameters { @@ -1193,7 +1272,6 @@ pub const paged_fa3 = struct { .block_table = .init(.{ .b = options.batch_size, .p = options.max_num_pages }, .i32), .cu_seqlens_q = .init(.{ .b = options.batch_size + 1 }, .i32), .seqused_k = .init(.{ .b = options.batch_size }, .i32), - .metadata = DecodeMetadata.init(options), .options = options, }; } @@ -1204,49 +1282,58 @@ pub const paged_fa3 = struct { allocation_size += self.block_table.byteSize(); allocation_size += self.cu_seqlens_q.byteSize(); allocation_size += self.seqused_k.byteSize(); - allocation_size += self.metadata.allocationSize(); - return allocation_size; - } - }; - - pub const DecodeMetadata = struct { - out_accum: zml.Tensor, - - pub fn init(opts: DecodeOptions) DecodeMetadata { - const shape = zml.Shape.init(.{ + const out_accum_shape: zml.Shape = .init(.{ .splits = MAX_NUM_SPLITS, - .hkv = opts.num_kv_heads, - .hg = @divExact(opts.num_heads, opts.num_kv_heads), - .b = opts.max_token_count, - .hd = opts.head_dim, - }, .f32).withPartitioning(.{ .hkv = .model }); - return .{ - .out_accum = .fromShape(shape), - }; - } + .hkv = self.options.num_kv_heads, + .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), + .b = self.options.batch_size, + .hd = self.options.head_dim, + }, .f32); + allocation_size += out_accum_shape.byteSize(); - pub fn initBuffers(self: *const DecodeMetadata, io: std.Io, platform: zml.Platform, out_accum_sharding: zml.Sharding) !zml.Bufferized(DecodeMetadata) { - return .{ - .out_accum = try zml.Buffer.uninitialized(io, &platform, self.out_accum.shape(), out_accum_sharding, .{}), - }; - } + const softmax_lse_shape: zml.Shape = .init(.{ + .hkv = self.options.num_kv_heads, + .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), + .b = self.options.batch_size, + }, .f32); + allocation_size += softmax_lse_shape.byteSize(); - pub fn deinitBuffers(self: *const zml.Bufferized(DecodeMetadata)) void { - self.out_accum.deinit(); - } + const softmax_lse_accum_shape: zml.Shape = .init(.{ + .splits = MAX_NUM_SPLITS, + .hkv = self.options.num_kv_heads, + .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), + .b = self.options.batch_size, + }, .f32); + allocation_size += softmax_lse_accum_shape.byteSize(); - pub fn allocationSize(self: DecodeMetadata) usize { - var allocation_size: usize = 0; - allocation_size += self.out_accum.byteSize(); return allocation_size; } + + pub fn onMemory(self: DecodeParameters, memory: zml.platform.Memory.Kind) DecodeParameters { + return .{ + .block_table = self.block_table.onMemory(memory), + .cu_seqlens_q = self.cu_seqlens_q.onMemory(memory), + .seqused_k = self.seqused_k.onMemory(memory), + .options = self.options, + }; + } + + pub fn toMemory(self: DecodeParameters, memory: zml.platform.Memory.Kind) DecodeParameters { + return .{ + .block_table = self.block_table.toMemory(memory), + .cu_seqlens_q = self.cu_seqlens_q.toMemory(memory), + .seqused_k = self.seqused_k.toMemory(memory), + .options = self.options, + }; + } }; pub const MixedOptions = struct { batch_size_prefill: usize, batch_size_decode: usize, max_num_pages: usize, + max_seqlen_q: usize, max_seqlen_k: usize, max_token_count: usize, num_heads: usize, @@ -1290,64 +1377,110 @@ pub const paged_fa3 = struct { allocation_size += self.seqused_k_decode.byteSize(); allocation_size += self.metadata.allocationSize(); + const out_accum_prefill_shape: zml.Shape = .init(.{ + .splits = MAX_NUM_SPLITS, + .hkv = self.options.num_kv_heads, + .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), + .b = self.options.max_token_count, + .hd = self.options.head_dim, + }, .f32); + allocation_size += out_accum_prefill_shape.byteSize(); + + const softmax_lse_prefill_shape: zml.Shape = .init(.{ + .hkv = self.options.num_kv_heads, + .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), + .b = self.options.max_token_count, + }, .f32); + allocation_size += softmax_lse_prefill_shape.byteSize(); + + const softmax_lse_accum_prefill_shape: zml.Shape = .init(.{ + .splits = MAX_NUM_SPLITS, + .hkv = self.options.num_kv_heads, + .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), + .b = self.options.max_token_count, + }, .f32); + allocation_size += softmax_lse_accum_prefill_shape.byteSize(); + + const out_accum_decode_shape: zml.Shape = .init(.{ + .splits = MAX_NUM_SPLITS, + .hkv = self.options.num_kv_heads, + .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), + .b = self.options.batch_size_decode, + .hd = self.options.head_dim, + }, .f32); + allocation_size += out_accum_decode_shape.byteSize(); + + const softmax_lse_decode_shape: zml.Shape = .init(.{ + .hkv = self.options.num_kv_heads, + .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), + .b = self.options.max_token_count, + }, .f32); + allocation_size += softmax_lse_decode_shape.byteSize(); + + const softmax_lse_accum_decode_shape: zml.Shape = .init(.{ + .splits = MAX_NUM_SPLITS, + .hkv = self.options.num_kv_heads, + .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), + .b = self.options.batch_size_decode, + }, .f32); + allocation_size += softmax_lse_accum_decode_shape.byteSize(); + return allocation_size; } + + pub fn onMemory(self: MixedParameters, memory: zml.platform.Memory.Kind) MixedParameters { + return .{ + .block_table_prefill = self.block_table_prefill.onMemory(memory), + .cu_seqlens_q_prefill = self.cu_seqlens_q_prefill.onMemory(memory), + .seqused_k_prefill = self.seqused_k_prefill.onMemory(memory), + .block_table_decode = self.block_table_decode.onMemory(memory), + .cu_seqlens_q_decode = self.cu_seqlens_q_decode.onMemory(memory), + .seqused_k_decode = self.seqused_k_decode.onMemory(memory), + .metadata = self.metadata.onMemory(memory), + .options = self.options, + }; + } + + pub fn toMemory(self: MixedParameters, memory: zml.platform.Memory.Kind) MixedParameters { + return .{ + .block_table_prefill = self.block_table_prefill.toMemory(memory), + .cu_seqlens_q_prefill = self.cu_seqlens_q_prefill.toMemory(memory), + .seqused_k_prefill = self.seqused_k_prefill.toMemory(memory), + .block_table_decode = self.block_table_decode.toMemory(memory), + .cu_seqlens_q_decode = self.cu_seqlens_q_decode.toMemory(memory), + .seqused_k_decode = self.seqused_k_decode.toMemory(memory), + .metadata = self.metadata.toMemory(memory), + .options = self.options, + }; + } }; pub const MixedMetadata = struct { - out_accum: zml.Tensor, - host_metadata: zml.Tensor, + decode_offset: zml.Tensor, pub fn init(opts: MixedOptions) MixedMetadata { - const shape = zml.Shape.init(.{ - .splits = MAX_NUM_SPLITS, - .hkv = opts.num_kv_heads, - .hg = @divExact(opts.num_heads, opts.num_kv_heads), - .b = opts.max_token_count, - .hd = opts.head_dim, - }, .f32).withPartitioning(.{ .hkv = .model }); + _ = opts; return .{ - .out_accum = .fromShape(shape), - .host_metadata = .init(.{2}, .i32), + .decode_offset = .init(.{}, .i32), }; } - pub fn initBuffers(self: *const MixedMetadata, io: std.Io, platform: zml.Platform, out_accum_sharding: zml.Sharding) !zml.Bufferized(MixedMetadata) { - return .{ - .out_accum = try zml.Buffer.uninitialized(io, &platform, self.out_accum.shape(), out_accum_sharding, .{}), - .host_metadata = undefined, - }; - } - - pub fn deinitBuffers(self: *const zml.Bufferized(MixedMetadata)) void { - self.out_accum.deinit(); - } - pub fn allocationSize(self: MixedMetadata) usize { var allocation_size: usize = 0; - allocation_size += self.out_accum.byteSize(); - allocation_size += self.host_metadata.byteSize(); + allocation_size += self.decode_offset.byteSize(); return allocation_size; } - }; - pub const Context = struct { - max_seqlen_k: usize, - decode_offset: ?zml.Tensor = null, + pub fn onMemory(self: MixedMetadata, memory: zml.platform.Memory.Kind) MixedMetadata { + return .{ + .decode_offset = self.decode_offset.onMemory(memory), + }; + } - pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { - _ = num_heads; - _ = num_kv_heads; - _ = head_dim; - _ = page_size; - const max_seqlen_k = switch (parameters) { - inline else => |v| v.options.max_seqlen_k, + pub fn toMemory(self: MixedMetadata, memory: zml.platform.Memory.Kind) MixedMetadata { + return .{ + .decode_offset = self.decode_offset.toMemory(memory), }; - const decode_offset = switch (parameters) { - .mixed => |mixed_parameters| mixed_parameters.metadata.host_metadata.slice1d(0, .{ .end = 1 }).squeeze(0), - .decode => null, - }; - return .{ .max_seqlen_k = max_seqlen_k, .decode_offset = decode_offset }; } }; @@ -1371,10 +1504,8 @@ pub const paged_fa3 = struct { const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); const scheduler_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[9]); - const host_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[10]); const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); - const max_seqlen_q = @as([*]i32, @ptrCast(@alignCast(host_metadata.ptr)))[1]; const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { const head_dim = q.shape.dim(2); break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); @@ -1382,6 +1513,7 @@ pub const paged_fa3 = struct { const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_q: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_q").?; const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); @@ -1505,13 +1637,14 @@ pub const paged_fa3 = struct { } }; - pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { + pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { stdx.debug.assert(q.shape().hasTags(.{ .b, .hg, .hkv, .hd }), "Expected q to have tags .b, .h, .hd", .{}); stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); const num_head_groups = q.dim(.hg); const num_kv_heads = q.dim(.hkv); + const head_dim = q.dim(.hd); const o = switch (parameters) { .decode => |decode_parameters| b: { const batch_size = decode_parameters.block_table.dim(0); @@ -1519,14 +1652,20 @@ pub const paged_fa3 = struct { const block_table = decode_parameters.block_table.withPartitioning(.{ .b = .replicated }); const cu_seqlens_q = decode_parameters.cu_seqlens_q.withPartitioning(.{ .b = .replicated }); const seqused_k = decode_parameters.seqused_k.withPartitioning(.{ .b = .replicated }); - const out_accum = decode_parameters.metadata.out_accum.withPartitioning(.{ .hkv = .model }); - const softmax_lse = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + const out_accum = zml.Tensor.uninitialized(.init(.{ + .splits = MAX_NUM_SPLITS, + .hkv = num_kv_heads, + .hg = num_head_groups, + .b = q.dim(.b), + .hd = head_dim, + }, .f32)).withPartitioning(.{ .hkv = .model }); + const softmax_lse = zml.Tensor.uninitialized(.init(.{ .hkv = num_kv_heads, .hg = num_head_groups, .q = q.dim(.b), }, .f32)).withPartitioning(.{ .hkv = .model }); - const softmax_lse_accum = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + const softmax_lse_accum = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, .q = q.dim(.b), .hkv = num_kv_heads, @@ -1554,7 +1693,7 @@ pub const paged_fa3 = struct { .{ .metadata = .{ .is_causal = opts.is_causal, - .max_seqlen_k = context.max_seqlen_k, + .max_seqlen_k = decode_parameters.options.max_seqlen_k, .window_size_left = opts.sliding_window, }, .opts = zml.ops.CustomCallOptions{ @@ -1581,15 +1720,20 @@ pub const paged_fa3 = struct { const block_table_decode = mixed_parameters.block_table_decode.withPartitioning(.{ .b = .replicated }); const cu_seqlens_q_decode = mixed_parameters.cu_seqlens_q_decode.withPartitioning(.{ .b = .replicated }); const seqused_k_decode = mixed_parameters.seqused_k_decode.withPartitioning(.{ .b = .replicated }); - const out_accum = mixed_parameters.metadata.out_accum.withPartitioning(.{ .hkv = .model }); - const host_metadata = mixed_parameters.metadata.host_metadata.withTags(.{.x}).withPartitioning(.{ .x = .replicated }); - const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + const out_accum_prefill = zml.Tensor.uninitialized(.init(.{ + .splits = MAX_NUM_SPLITS, + .hkv = num_kv_heads, + .hg = num_head_groups, + .b = q.dim(.b), + .hd = head_dim, + }, .f32)).withPartitioning(.{ .hkv = .model }); + const softmax_lse_prefill = zml.Tensor.uninitialized(.init(.{ .hkv = num_kv_heads, .hg = num_head_groups, .q = q.dim(.b), }, .f32)).withPartitioning(.{ .hkv = .model }); - const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + const softmax_lse_accum_prefill = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, .q = q.dim(.b), .hkv = num_kv_heads, @@ -1610,15 +1754,15 @@ pub const paged_fa3 = struct { block_table_prefill, softmax_lse_prefill, softmax_lse_accum_prefill, - out_accum, + out_accum_prefill, scheduler_metadata_prefill, - host_metadata, }, output_shape, .{ .metadata = .{ .is_causal = opts.is_causal, - .max_seqlen_k = context.max_seqlen_k, + .max_seqlen_k = mixed_parameters.options.max_seqlen_k, + .max_seqlen_q = mixed_parameters.options.max_seqlen_q, .window_size_left = opts.sliding_window, }, .opts = zml.ops.CustomCallOptions{ .has_side_effect = false }, @@ -1633,19 +1777,26 @@ pub const paged_fa3 = struct { o = o.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = num_head_groups }); const batch_size_decode = mixed_parameters.block_table_prefill.dim(0); - const softmax_lse_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + const out_accum_decode = zml.Tensor.uninitialized(.init(.{ + .splits = MAX_NUM_SPLITS, + .hkv = num_kv_heads, + .hg = num_head_groups, + .b = batch_size_decode, + .hd = head_dim, + }, .f32)).withPartitioning(.{ .hkv = .model }); + const softmax_lse_decode = zml.Tensor.uninitialized(.init(.{ .hkv = num_kv_heads, .hg = num_head_groups, .q = q.dim(.b), }, .f32)); - const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ + const softmax_lse_accum_decode = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, .q = q.dim(.b), .hkv = num_kv_heads, .hg = num_head_groups, }, .f32)); const scheduler_metadata_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = batch_size_decode + 1 }, .i32)).withPartitioning(.{ .b = .replicated }); - var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_size_decode }).withPartitioning(.{ .hkv = .model }); + var q_decode = q.dynamicSlice1d(0, .{ .start = mixed_parameters.metadata.decode_offset, .len = batch_size_decode }).withPartitioning(.{ .hkv = .model }); q_decode = q_decode.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); @@ -1660,14 +1811,14 @@ pub const paged_fa3 = struct { block_table_decode, softmax_lse_decode, softmax_lse_accum_decode, - out_accum, + out_accum_decode, scheduler_metadata_decode, }, decode_output_shape, .{ .metadata = .{ .is_causal = opts.is_causal, - .max_seqlen_k = context.max_seqlen_k, + .max_seqlen_k = mixed_parameters.options.max_seqlen_k, .window_size_left = opts.sliding_window, }, .opts = zml.ops.CustomCallOptions{ .has_side_effect = false }, @@ -1680,7 +1831,7 @@ pub const paged_fa3 = struct { ); o_decode = o_decode.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = num_head_groups }); - o = o.dynamicUpdateSlice1d(o_decode, 0, context.decode_offset.?); + o = o.dynamicUpdateSlice1d(o_decode, 0, mixed_parameters.metadata.decode_offset); break :b o; }, }; diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index d36c2bca..7ede3a71 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -54,6 +54,7 @@ pub const Options = union(Backend) { .batch_size_prefill = args.batch_size, .max_num_pages = max_num_pages, .max_seqlen_k = args.seq_len, + .max_seqlen_q = args.max_seqlen_q, .max_token_count = args.max_token_count, .num_heads = args.num_heads, .num_kv_heads = args.num_kv_heads, @@ -80,6 +81,7 @@ pub const Options = union(Backend) { .batch_size_prefill = args.batch_size, .max_num_pages = max_num_pages, .max_seqlen_k = args.seq_len, + .max_seqlen_q = args.max_seqlen_q, .max_token_count = args.max_token_count, .num_heads = args.num_heads, .num_kv_heads = args.num_kv_heads, @@ -164,21 +166,22 @@ pub const Parameters = union(Backend) { inline else => |v| v.allocationSize(), }; } -}; -/// Internal state that can be used inside the model code. It's derived from Parameters and Option. -pub const Context = union(Backend) { - cuda_fa2: flashattn.paged_fa2.Context, - cuda_fa3: flashattn.paged_fa3.Context, - triton: triton.paged.Context, - mosaic_tpu: tpu.mosaic_tpu.Context, + pub fn onMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { + return switch (self) { + .cuda_fa2 => |v| .{ .cuda_fa2 = v.onMemory(memory) }, + .cuda_fa3 => |v| .{ .cuda_fa3 = v.onMemory(memory) }, + .triton => |v| .{ .triton = v.onMemory(memory) }, + .mosaic_tpu => |v| .{ .mosaic_tpu = v.onMemory(memory) }, + }; + } - pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { - return switch (parameters) { - .cuda_fa2 => |cuda_fa2_parameters| .{ .cuda_fa2 = flashattn.paged_fa2.Context.init(cuda_fa2_parameters, num_heads, num_kv_heads, head_dim, page_size) }, - .cuda_fa3 => |cuda_fa3_parameters| .{ .cuda_fa3 = flashattn.paged_fa3.Context.init(cuda_fa3_parameters, num_heads, num_kv_heads, head_dim, page_size) }, - .triton => |triton_parameters| .{ .triton = triton.paged.Context.init(triton_parameters, num_heads, num_kv_heads, head_dim, page_size) }, - .mosaic_tpu => |mosaic_tpu_parameters| .{ .mosaic_tpu = tpu.mosaic_tpu.Context.init(mosaic_tpu_parameters, num_heads, num_kv_heads, head_dim, page_size) }, + pub fn toMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { + return switch (self) { + .cuda_fa2 => |v| .{ .cuda_fa2 = v.toMemory(memory) }, + .cuda_fa3 => |v| .{ .cuda_fa3 = v.toMemory(memory) }, + .triton => |v| .{ .triton = v.toMemory(memory) }, + .mosaic_tpu => |v| .{ .mosaic_tpu = v.toMemory(memory) }, }; } }; @@ -197,23 +200,23 @@ pub const KvCache = union(enum) { dense: zml.Tensor, }; -pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache, opts: AttentionOptions) zml.Tensor { +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache, opts: AttentionOptions) zml.Tensor { _ = k; _ = v; return switch (parameters) { .cuda_fa2 => |cuda_fa2_parameters| switch (kv_cache) { - .split => |split| flashattn.paged_fa2.pagedAttention(cuda_fa2_parameters, context.cuda_fa2, q, split.k, split.v, opts), + .split => |split| flashattn.paged_fa2.pagedAttention(cuda_fa2_parameters, q, split.k, split.v, opts), .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), }, .cuda_fa3 => |cuda_fa3_parameters| switch (kv_cache) { - .split => |split| flashattn.paged_fa3.pagedAttention(cuda_fa3_parameters, context.cuda_fa3, q, split.k, split.v, opts), + .split => |split| flashattn.paged_fa3.pagedAttention(cuda_fa3_parameters, q, split.k, split.v, opts), .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), }, .triton => |triton_parameters| switch (kv_cache) { - .split => |split| triton.paged.pagedAttention(triton_parameters, context.triton, q, split.k, split.v, opts), + .split => |split| triton.paged.pagedAttention(triton_parameters, q, split.k, split.v, opts), .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), }, - .mosaic_tpu => |mosaic_tpu_parameters| tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, context.mosaic_tpu, q, kv_cache.dense, opts), + .mosaic_tpu => |mosaic_tpu_parameters| tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, q, kv_cache.dense, opts), }; } diff --git a/zml/attention/tpu_attention.zig b/zml/attention/tpu_attention.zig index a1046c75..0ce40058 100644 --- a/zml/attention/tpu_attention.zig +++ b/zml/attention/tpu_attention.zig @@ -78,6 +78,24 @@ pub const mosaic_tpu = struct { allocation_size += self.query_start_len.byteSize(); return allocation_size; } + + pub fn onMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { + return .{ + .opts = self.opts, + .block_table = self.block_table.onMemory(memory), + .seq_lens = self.seq_lens.onMemory(memory), + .query_start_len = self.query_start_len.onMemory(memory), + }; + } + + pub fn toMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { + return .{ + .opts = self.opts, + .block_table = self.block_table.toMemory(memory), + .seq_lens = self.seq_lens.toMemory(memory), + .query_start_len = self.query_start_len.toMemory(memory), + }; + } }; fn kernelFrontendAttrs(comp_ctx: *CompilationContext) *const mlir.Attribute { @@ -226,18 +244,7 @@ pub const mosaic_tpu = struct { }; } - pub const Context = struct { - pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { - _ = parameters; - _ = num_heads; - _ = num_kv_heads; - _ = head_dim; - _ = page_size; - return .{}; - } - }; - - pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, kv_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { + pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, kv_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { stdx.debug.assert(opts.is_causal, "mosaic_tpu ragged paged attention currently only supports causal attention", .{}); const prepared = prepareInputs(parameters, q, kv_cache); @@ -255,7 +262,6 @@ pub const mosaic_tpu = struct { .{ .opts = opts, .parameters = parameters, - .context = context, }, (struct { fn body(body_context: anytype, allocator: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output: zml.Shape) zml.Tensor { diff --git a/zml/attention/triton_attention.zig b/zml/attention/triton_attention.zig index f361944d..2bfa6d4b 100644 --- a/zml/attention/triton_attention.zig +++ b/zml/attention/triton_attention.zig @@ -166,18 +166,23 @@ pub const paged = struct { pub fn options(self: Parameters) Options { return self.options_; } - }; - pub const Context = struct { - is_prefill: bool, - max_seqlen_q: usize, + pub fn onMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { + return .{ + .options_ = self.options_, + .block_table = self.block_table.onMemory(memory), + .seq_lens = self.seq_lens.onMemory(memory), + .query_start_len = self.query_start_len.onMemory(memory), + }; + } - pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { - _ = num_heads; - _ = num_kv_heads; - _ = head_dim; - _ = page_size; - return .{ .is_prefill = parameters.options_.is_prefill, .max_seqlen_q = parameters.options_.max_seqlen_q }; + pub fn toMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { + return .{ + .options_ = self.options_, + .block_table = self.block_table.toMemory(memory), + .seq_lens = self.seq_lens.toMemory(memory), + .query_start_len = self.query_start_len.toMemory(memory), + }; } }; @@ -210,7 +215,7 @@ pub const paged = struct { } }; - pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { + pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { const output = zml.ops.manualComputation( .{ q, @@ -224,7 +229,6 @@ pub const paged = struct { .{ .opts = opts, .options = parameters.options_, - .context = context, }, (struct { fn body(ctx_: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, _: zml.Shape) zml.Tensor { @@ -276,9 +280,9 @@ pub const paged = struct { paged_attention_opts.num_2d_prgms, ); const output = if (use_2d_kernel) - pagedAttention2d(parameters_, ctx_.context, q_, k_cache_, v_cache_, ctx_.opts, paged_attention_opts) + pagedAttention2d(parameters_, q_, k_cache_, v_cache_, ctx_.opts, paged_attention_opts) else - pagedAttention3d(parameters_, ctx_.context, q_, k_cache_, v_cache_, ctx_.opts, paged_attention_opts); + pagedAttention3d(parameters_, q_, k_cache_, v_cache_, ctx_.opts, paged_attention_opts); return output; } @@ -288,8 +292,7 @@ pub const paged = struct { return output; } - pub fn pagedAttention2d(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, paged_attention_opts: PagedAttentionOptions) zml.Tensor { - _ = context; + pub fn pagedAttention2d(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, paged_attention_opts: PagedAttentionOptions) zml.Tensor { _ = opts; const config = select2dConfig(paged_attention_opts); @@ -375,8 +378,7 @@ pub const paged = struct { return output.output; } - pub fn pagedAttention3d(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, paged_attention_opts: PagedAttentionOptions) zml.Tensor { - _ = context; + pub fn pagedAttention3d(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, paged_attention_opts: PagedAttentionOptions) zml.Tensor { _ = opts; const config = select3dConfig(paged_attention_opts); From 986532bb339e576c7995d8a3741015aa41c1044b Mon Sep 17 00:00:00 2001 From: Reese Chong Date: Mon, 18 May 2026 16:52:05 +0200 Subject: [PATCH 491/881] tensor/zml: fix SwiGLU (#539) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update SwiGLU implementation to match the SwiGLU paper. - Formula: `(xW + b) * sigmoid(β(xW + b)), outer product with (xV + c)` - Add test for SwiGLU --- zml/tensor.zig | 70 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 4 deletions(-) diff --git a/zml/tensor.zig b/zml/tensor.zig index d1faf710..cb4b55e0 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1344,11 +1344,73 @@ pub const Tensor = struct { } /// Returns a Tensor containing the SwiGLU activation function applied to the input Tensor. - pub fn swiglu(self: Tensor, beta: f32, w: Tensor, b: Tensor, tag: Shape.Tag) Tensor { - const sigmoid_tensor = self.mul(Tensor.constant(DataType.Value.init(self.dtype(), beta)).broad(.init(self._shape, self.dtype()))).sigmoid(); - const one_minus_sigmoid_tensor = Tensor.constant(.init(self.dtype(), 1)).broad(.init(self._shape, self.dtype())).sub(sigmoid_tensor); + pub fn swiglu(self: Tensor, beta: f32, w: Tensor, v: Tensor, b: Tensor, c: Tensor, tag: Shape.Tag) Tensor { + var value_path = self.dot(w, tag); + value_path = value_path.add(b.broad(value_path.shape())); - return self.mul(sigmoid_tensor).add(one_minus_sigmoid_tensor.mul(w.dot(self, tag).add(b))); + const sigmoid_tensor = value_path.scale(beta).sigmoid(); + + var gate_path = self.dot(v, tag); + gate_path = gate_path.add(c.broad(gate_path.shape())); + + return value_path.mul(sigmoid_tensor).outer(gate_path); + } + + test swiglu { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const input: Tensor = .init(.{ .d = 4 }, .f32); + const beta = 3; + const w: Tensor = .init(.{ .c = 3, .d = 4 }, .f32); + const v: Tensor = .init(.{ .e = 3, .d = 4 }, .f32); + const b: Tensor = .init(.{ .c = 3 }, .f32); + const c: Tensor = .init(.{ .e = 3 }, .f32); + + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.swiglu, + .{ input, beta, w, v, b, c, Shape.toTag(.d) }, + platform, + .{}, + ); + defer exe.deinit(); + + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&[4]f32{ 1, 0, 1, 2 })); + defer input_buffer.deinit(); + + var w_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, w.shape(), .replicated, std.mem.sliceAsBytes(&[3][4]f32{ + .{ 1, 0, 2, 0 }, + .{ 2, 1, 3, 3 }, + .{ 1, 1, 0, 0 }, + })); + defer w_buffer.deinit(); + + var v_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, v.shape(), .replicated, std.mem.sliceAsBytes(&[3][4]f32{ + .{ 0, 0, 2, 1 }, + .{ 1, 2, 1, 3 }, + .{ 0, 0, 2, 0 }, + })); + defer v_buffer.deinit(); + + var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), .replicated, std.mem.sliceAsBytes(&[3]f32{ 1, 1, 1 })); + defer b_buffer.deinit(); + + var c_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, c.shape(), .replicated, std.mem.sliceAsBytes(&[3]f32{ 2, 2, 2 })); + defer c_buffer.deinit(); + + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.swiglu, .{ input_buffer, w_buffer, v_buffer, b_buffer, c_buffer }); + defer res.deinit(); + + const output_shape = Shape.init(.{ .c = 3, .e = 3 }, .f32); + const expectation: zml.Slice = .init(output_shape, std.mem.sliceAsBytes(&[3][3]f32{ + .{ 23.999856, 39.99976, 15.999904 }, + .{ 72.000000, 120.000000, 48.000000 }, + .{ 11.970324, 19.950540, 7.980216 }, + })); + + try zml.testing.expectClose(std.testing.io, expectation, res, .{}); } /// Returns a Tensor containing the Gaussian Error Linear Units (GeLU) activation function applied to each element of the input Tensor. From 3f2666dba62cb2b10b0785a47e01ecd633170017 Mon Sep 17 00:00:00 2001 From: Reese Chong Date: Tue, 19 May 2026 09:19:56 +0200 Subject: [PATCH 492/881] zml/testing: resolve type of forward in testLayer() (#540) Change made in response to the following error: ``` zml/testing.zig:299:43: error: root source file struct 'meta' has no member named 'FnArgs' const ArgsT = stdx.meta.Tail(stdx.meta.FnArgs(forward)); ~~~~~~~~~^~~~~~~ stdx/meta.zig:1:1: note: struct declared here ``` --- zml/testing.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/testing.zig b/zml/testing.zig index 84e409d4..18dbea02 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -296,7 +296,7 @@ pub fn testLayer( defer arena.deinit(); const forward = @field(@TypeOf(layer), @tagName(func)); - const ArgsT = stdx.meta.Tail(stdx.meta.FnArgs(forward)); + const ArgsT = stdx.meta.Tail(std.meta.ArgsTuple(@TypeOf(forward))); var args: ArgsT = undefined; From 77711b949d8533654ad78146d758308d4e65ece8 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Wed, 20 May 2026 16:13:59 +0200 Subject: [PATCH 493/881] workspace: support zig-only local execution (#546) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR bumps rules_zig back to upstream/main where we now have all the necessary features to enable zig local actions while the reste on remote. It also adds the necessary configurable to actually enable this. Additionally, since rules_zig now ships ZLS support, we remove zls entirely from zml. ### How the Bazel resolution works Bazel resolves execution platforms and toolchains during analysis for each target that requires toolchains. If a rule defines exe, resolution happens separately for the default target execution group and for each `exec_group`; actions assigned to an exec group use that group’s selected execution platform and toolchains. Execution platforms come from an ordered set. In this repository’s remote configuration, the relevant order is: 1. `//:rbe_platform`, supplied by `--extra_execution_platforms=@zml//:rbe_platform` 2. `//bazel/zig_toolchain_config:local_execution_platform`, registered by this PR 3. the host platform, included automatically by Bazel Earlier platforms are selected when they can satisfy the requested toolchains. For each target or exec group, Bazel takes the required toolchain types, the target platform, the ordered execution platforms, and the ordered registered toolchains. It filters execution platforms by execution constraints, then tries to find a matching toolchain for every required toolchain type. A toolchain matches when: - its `exec_compatible_with` constraints are present on the candidate execution platform - its `target_compatible_with` constraints are present on the target platform - its `target_settings` match the current configuration Any execution platform missing a required toolchain is rejected. The first remaining execution platform wins, and the first matching toolchain for each type is selected. ### How this makes Zig local-only This PR defines a custom execution constraint: - `//bazel/zig_toolchain_config:local_zig_execution` It then defines a host-derived execution platform: - `//bazel/zig_toolchain_config:local_execution_platform` That platform inherits the host OS/CPU constraints from `@platforms//host`, adds `local_zig_execution`, and sets: ```starlark exec_properties = { "no-remote-exec": "1", } ``` The zig.toolchain(...) module extension call now passes: extra_exec_compatible_with = ["//bazel/zig_toolchain_config:local_zig_execution"] rules_zig appends that extra execution constraint to every generated Zig SDK toolchain. As a result, when rules_zig resolves @rules_zig//zig:toolchain_type for its zig execution group: - the remote execution platform is considered first, but it does not have local_zig_execution, so it cannot match the Zig SDK toolchain - the custom local execution platform has both the host OS/CPU constraints and local_zig_execution, so it matches - the selected platform carries no-remote-exec, so Zig actions execute locally even in remote builds Other toolchains are not constrained by local_zig_execution, so they can continue to resolve to the remote execution platform first. This keeps the build remote by default while carving out Zig actions as local-only. --------- Co-authored-by: Steeve Morin --- .bazelrc | 2 - BUILD.bazel | 2 +- MODULE.bazel | 48 +++-- MODULE.bazel.lock | 171 +++++++++++++++++- platforms/BUILD.bazel | 34 +++- third_party/zls/BUILD.bazel | 56 ------ third_party/zls/resolved_toolchain.bzl | 24 --- third_party/zls/string_replace_reader.zig | 28 --- third_party/zls/workspace_printer.zig | 169 ------------------ third_party/zls/workspace_printer_test.zig | 90 ---------- third_party/zls/zls.completion.tpl | 10 -- third_party/zls/zls_build_runner.zig | 14 -- third_party/zls/zls_completion.bzl | 103 ----------- third_party/zls/zls_runner.zig | 100 ----------- third_party/zls/zls_toolchain.bzl | 129 -------------- third_party/zls/zls_write_build_config.bzl | 177 ------------------- third_party/zls/zls_write_runner_zig_src.bzl | 58 ------ 17 files changed, 229 insertions(+), 986 deletions(-) delete mode 100644 third_party/zls/BUILD.bazel delete mode 100644 third_party/zls/resolved_toolchain.bzl delete mode 100644 third_party/zls/string_replace_reader.zig delete mode 100644 third_party/zls/workspace_printer.zig delete mode 100644 third_party/zls/workspace_printer_test.zig delete mode 100644 third_party/zls/zls.completion.tpl delete mode 100644 third_party/zls/zls_build_runner.zig delete mode 100644 third_party/zls/zls_completion.bzl delete mode 100644 third_party/zls/zls_runner.zig delete mode 100644 third_party/zls/zls_toolchain.bzl delete mode 100644 third_party/zls/zls_write_build_config.bzl delete mode 100644 third_party/zls/zls_write_runner_zig_src.bzl diff --git a/.bazelrc b/.bazelrc index 3f52fe6d..b60a1782 100644 --- a/.bazelrc +++ b/.bazelrc @@ -127,8 +127,6 @@ common:release --@rules_zig//zig/settings:mode=release_safe common:native --copt=-march=native common:native --@rules_zig//zig/settings:zigopt=-mcpu=native -common:linux --host_platform=//platforms:host_glibc - # Remote execution and caching with BuildBuddy common:remote --bes_results_url=https://app.buildbuddy.io/invocation/ common:remote --bes_backend=grpcs://remote.buildbuddy.io diff --git a/BUILD.bazel b/BUILD.bazel index 67502496..e2ffd349 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,4 +1,4 @@ -load("//third_party/zls:zls_completion.bzl", "zls_completion") +load("@rules_zig//zig/zls:zls_completion.bzl", "zls_completion") load("@rules_zig//zig/translate-c:toolchain.bzl", "translate_c_toolchain") alias( diff --git a/MODULE.bazel b/MODULE.bazel index 7f5a686c..3df68694 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -27,22 +27,20 @@ bazel_dep(name = "rules_oci", version = "2.2.7") bazel_dep(name = "rules_proto", version = "7.1.0") bazel_dep(name = "rules_python", version = "1.9.0") bazel_dep(name = "rules_shell", version = "0.6.1") -bazel_dep(name = "rules_zig", version = "0.12.2") -git_override( - module_name = "rules_zig", - commit = "6e9d876037ecb01ce4354c9c9e0c93e026e84e37", # upstreamable-20260515.1 - remote = "https://github.com/zml/rules_zig.git", -) +bazel_dep(name = "rules_zig", version = "0.15.1") bazel_dep(name = "tar.bzl", version = "0.9.0") bazel_dep(name = "llvm", version = "0.7.3") bazel_dep(name = "with_cfg.bzl", version = "0.14.4") bazel_dep(name = "grpc", version = "1.74.0") + # Added to support bazel 9+ since an old version is pulled by GRPC # TODO: remove when GRPC is updated to v1.80+ bazel_dep(name = "rules_foreign_cc", version = "0.15.1") + # Same as rules_foreign_cc bazel_dep(name = "rules_go", version = "0.60.0") + # Same as rules_foreign_cc bazel_dep(name = "jsoncpp", version = "1.9.6.bcr.2", repo_name = "jsoncpp_git") @@ -65,6 +63,8 @@ use_repo( zig = use_extension("@rules_zig//zig:extensions.bzl", "zig") zig.index(file = "//bazel:zig_index.json") zig.toolchain( + # Ensure Zig actions are run locally only + extra_exec_compatible_with = ["//platforms:local_execution"], zig_version = "0.16.0", ) zig.mirrors(urls = [ @@ -88,7 +88,28 @@ use_repo(zig, "zig_toolchains") register_toolchains("@rules_zig//zig/target:all") register_toolchains("@zig_toolchains//:all") + register_toolchains("//:translate_c_toolchain") + +# Add the local execution platform to the set of eligible execution platforms +# This is used so that Zig toolchains that are only declared as compatible with +# the constraints described by this platform be selected and matched with only +# this platform only. +# This ensures Zig actions only run locally because they're slower remotely +# due to the Zig compilation model which concentrates one action for the whole +# build graph. +register_execution_platforms("//platforms:local_execution_platform") + +zls = use_extension("@rules_zig//zig/zls:extensions.bzl", "zls") +zls.index(file = "@rules_zig//zig/zls/private:versions.json") +zls.toolchain( + zig_version = "0.16.0", + zls_version = "0.16.0", +) +use_repo(zls, "zls_toolchains") + +register_toolchains("@zls_toolchains//:all") + ### ### LLVM Bootstrapped toolchains @@ -145,7 +166,7 @@ cpu = use_extension("//platforms/cpu:cpu.bzl", "cpu_pjrt_plugin") use_repo(cpu, "libpjrt_cpu_darwin_amd64", "libpjrt_cpu_darwin_arm64", "libpjrt_cpu_linux_amd64") cuda = use_extension("//platforms/cuda:cuda.bzl", "cuda_packages") -use_repo(cuda, "libpjrt_cuda_linux_amd64", "libpjrt_cuda_linux_arm64", "cuda_nvml_dev_linux_x86_64", "cuda_nvml_dev_linux_sbsa", "zlib1g_linux_arm64", "zlib1g_linux_amd64", "cuda_nvtx_linux_x86_64", "cuda_nvtx_linux_sbsa") +use_repo(cuda, "cuda_nvml_dev_linux_sbsa", "cuda_nvml_dev_linux_x86_64", "cuda_nvtx_linux_sbsa", "cuda_nvtx_linux_x86_64", "libpjrt_cuda_linux_amd64", "libpjrt_cuda_linux_arm64", "zlib1g_linux_amd64", "zlib1g_linux_arm64") rocm = use_extension("//platforms/rocm:rocm.bzl", "rocm_packages") use_repo(rocm, "amd-smi-lib", "hipblaslt", "libdrm-amdgpu-amdgpu1", "libdrm2-amdgpu", "libpjrt_rocm", "rocblas", "rocprofiler-sdk", "rocprofiler-sdk-roctx") @@ -156,13 +177,8 @@ use_repo(tpu, "libpjrt_tpu") neuron = use_extension("//platforms/neuron:neuron.bzl", "neuron_packages") use_repo(neuron, "aws-neuronx-collectives", "aws-neuronx-runtime-lib", "libgomp1", "libpjrt_neuron") -zls = use_extension("//third_party/zls:zls_toolchain.bzl", "zls_toolchains") -use_repo(zls, "zls_toolchains") - -register_toolchains("@zls_toolchains//:all") - non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") -use_repo(non_module_deps, "arocc", "cloud_accelerator_diagnostics", "com_google_sentencepiece", "flashattn_linux_amd64", "flashattn_linux_arm64", "iree", "libvaxis", "linenoise", "mnist", "org_swig_swig", "stb", "translate-c", "uucode", "xla", "zigimg", "mosaic_tpu") +use_repo(non_module_deps, "arocc", "cloud_accelerator_diagnostics", "com_google_sentencepiece", "flashattn_linux_amd64", "flashattn_linux_arm64", "iree", "libvaxis", "linenoise", "mnist", "mosaic_tpu", "org_swig_swig", "stb", "translate-c", "uucode", "xla", "zigimg") xla = use_extension("//third_party/xla:xla.bzl", "xla") use_repo( @@ -171,6 +187,8 @@ use_repo( "eigen_archive", "farmhash_archive", "farmhash_gpu_archive", + "highwayhash", + "hwloc", "llvm-raw", "local_config_cuda", "local_config_remote_execution", @@ -181,14 +199,12 @@ use_repo( "python_version_repo", "rules_ml_toolchain", "shardy", + "snappy", "stablehlo", "sycl_configure", "sycl_configure_ext", "triton", "tsl", - "highwayhash", - "snappy", - "hwloc", ) llvm = use_extension("//third_party/xla:llvm.bzl", "llvm") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 5e176c22..d39cd41d 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -343,6 +343,8 @@ "https://bcr.bazel.build/modules/rules_swift/1.18.0/MODULE.bazel": "a6aba73625d0dc64c7b4a1e831549b6e375fbddb9d2dde9d80c9de6ec45b24c9", "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", "https://bcr.bazel.build/modules/rules_swift/2.1.1/source.json": "40fc69dfaac64deddbb75bd99cdac55f4427d9ca0afbe408576a65428427a186", + "https://bcr.bazel.build/modules/rules_zig/0.15.0/MODULE.bazel": "8464afbe0a1b59967bf3d56cb166f889f1e53599a1194b2c0e668aae651cb9f8", + "https://bcr.bazel.build/modules/rules_zig/0.15.0/source.json": "33e47f366fae6e81fb5398972af4208c3546be162ca58be9142c018b516d4252", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", "https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4", "https://bcr.bazel.build/modules/stardoc/0.5.6/MODULE.bazel": "c43dabc564990eeab55e25ed61c07a1aadafe9ece96a4efabb3f8bf9063b71ef", @@ -1107,8 +1109,8 @@ }, "@@rules_zig+//zig:extensions.bzl%zig": { "general": { - "bzlTransitiveDigest": "mBSDTk+QEb2UlXBSWQWCw8A/tUSarS8PCSbhFYCkotc=", - "usagesDigest": "TA6Tzd8UNrxyL67PtNy0LKhpR1wAStj+mb2UDC/QiyI=", + "bzlTransitiveDigest": "u7Psh7szBG3+lbRmEeBBbWlZRHJWhWtxQo/wtRqvnTg=", + "usagesDigest": "E3Q/pEwXzITKlwKks+3Y68QKa4eUgIuqZzlQCbDX7dU=", "recordedFileInputs": { "@@//bazel/zig_index.json": "2e3b4b83f1add5d3ca12fd05797c04c38af21d86c7fc7c1a9b356560141a4797", "@@rules_zig+//zig/private/versions.json": "3e4e370dee9c3f3fdc8bfc7a600e76743278fde40815eead5d0bcda3aec3e2bd" @@ -1371,6 +1373,166 @@ "0.16.0", "0.16.0" ], + "exec_lengths": [ + 3, + 3, + 3, + 3, + 3, + 3 + ], + "exec_constraints": [ + "@platforms//os:linux", + "@platforms//cpu:aarch64", + "'@@//platforms:local_execution'", + "@platforms//os:macos", + "@platforms//cpu:aarch64", + "'@@//platforms:local_execution'", + "@platforms//os:windows", + "@platforms//cpu:aarch64", + "'@@//platforms:local_execution'", + "@platforms//os:linux", + "@platforms//cpu:x86_64", + "'@@//platforms:local_execution'", + "@platforms//os:macos", + "@platforms//cpu:x86_64", + "'@@//platforms:local_execution'", + "@platforms//os:windows", + "@platforms//cpu:x86_64", + "'@@//platforms:local_execution'" + ], + "target_compatible_lengths": [ + 0, + 0, + 0, + 0, + 0, + 0 + ], + "target_compatible_constraints": [], + "target_settings_lengths": [ + 0, + 0, + 0, + 0, + 0, + 0 + ], + "target_settings": [] + } + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_zig+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "rules_zig+", + "bazel_tools", + "bazel_tools" + ] + ] + } + }, + "@@rules_zig+//zig/zls:extensions.bzl%zls": { + "general": { + "bzlTransitiveDigest": "8ZwSSF/7dE4rb0W0cZqil2SfGF1bPK04LTv1/zjtkTM=", + "usagesDigest": "FAKFOHWIN8RdVDOmVLmwjLdv/f9a2WDp48ekZEePDlA=", + "recordedFileInputs": { + "@@rules_zig+//zig/zls/private/versions.json": "127803289dfd79b5255db5f74be1d9a2f790f03292d6147009af702d888aba69" + }, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "zls_0.16.0_aarch64-linux": { + "repoRuleId": "@@rules_zig+//zig/zls/private/repo:zls_repository.bzl%zls_repository", + "attributes": { + "url": "https://builds.zigtools.org/zls-aarch64-linux-0.16.0.tar.xz", + "mirrors": [], + "sha256": "430cd293d201eb70ae2519dbc96c854bf8791b8df7fc9392e8d2dc9680a2bed7", + "zls_version": "0.16.0", + "platform": "aarch64-linux" + } + }, + "zls_0.16.0_aarch64-macos": { + "repoRuleId": "@@rules_zig+//zig/zls/private/repo:zls_repository.bzl%zls_repository", + "attributes": { + "url": "https://builds.zigtools.org/zls-aarch64-macos-0.16.0.tar.xz", + "mirrors": [], + "sha256": "b93ec549f8558a7e85984a840e9276d274f1059b54ade4254296ef4982958359", + "zls_version": "0.16.0", + "platform": "aarch64-macos" + } + }, + "zls_0.16.0_aarch64-windows": { + "repoRuleId": "@@rules_zig+//zig/zls/private/repo:zls_repository.bzl%zls_repository", + "attributes": { + "url": "https://builds.zigtools.org/zls-aarch64-windows-0.16.0.zip", + "mirrors": [], + "sha256": "ef4c5ccb93c80c9f023105c5f558ae8774ac6668d560ba6f92a2f87d95df2311", + "zls_version": "0.16.0", + "platform": "aarch64-windows" + } + }, + "zls_0.16.0_x86_64-linux": { + "repoRuleId": "@@rules_zig+//zig/zls/private/repo:zls_repository.bzl%zls_repository", + "attributes": { + "url": "https://builds.zigtools.org/zls-x86_64-linux-0.16.0.tar.xz", + "mirrors": [], + "sha256": "ded6d562a0b86ee878b1ddf70ffab2797ce3cdca3b02d6077548f9d56dff96b6", + "zls_version": "0.16.0", + "platform": "x86_64-linux" + } + }, + "zls_0.16.0_x86_64-macos": { + "repoRuleId": "@@rules_zig+//zig/zls/private/repo:zls_repository.bzl%zls_repository", + "attributes": { + "url": "https://builds.zigtools.org/zls-x86_64-macos-0.16.0.tar.xz", + "mirrors": [], + "sha256": "49f716ea96c1aadaecaa5d9c0a50874cbcf443dc42b825f1e7ee35499ad3eb96", + "zls_version": "0.16.0", + "platform": "x86_64-macos" + } + }, + "zls_0.16.0_x86_64-windows": { + "repoRuleId": "@@rules_zig+//zig/zls/private/repo:zls_repository.bzl%zls_repository", + "attributes": { + "url": "https://builds.zigtools.org/zls-x86_64-windows-0.16.0.zip", + "mirrors": [], + "sha256": "35cbb7163224e8cf92d21099c1b1391f2aba927f25d389f021b13a21d40b96dd", + "zls_version": "0.16.0", + "platform": "x86_64-windows" + } + }, + "zls_toolchains": { + "repoRuleId": "@@rules_zig+//zig/zls/private/repo:toolchains_repo.bzl%toolchains_repo", + "attributes": { + "names": [ + "0.16.0_for_0.16.0_aarch64-linux", + "0.16.0_for_0.16.0_aarch64-macos", + "0.16.0_for_0.16.0_aarch64-windows", + "0.16.0_for_0.16.0_x86_64-linux", + "0.16.0_for_0.16.0_x86_64-macos", + "0.16.0_for_0.16.0_x86_64-windows" + ], + "labels": [ + "@zls_0.16.0_aarch64-linux//:zls_toolchain", + "@zls_0.16.0_aarch64-macos//:zls_toolchain", + "@zls_0.16.0_aarch64-windows//:zls_toolchain", + "@zls_0.16.0_x86_64-linux//:zls_toolchain", + "@zls_0.16.0_x86_64-macos//:zls_toolchain", + "@zls_0.16.0_x86_64-windows//:zls_toolchain" + ], + "zig_versions": [ + "0.16.0", + "0.16.0", + "0.16.0", + "0.16.0", + "0.16.0", + "0.16.0" + ], "exec_lengths": [ 2, 2, @@ -1397,6 +1559,11 @@ } }, "recordedRepoMappingEntries": [ + [ + "", + "rules_zig", + "rules_zig+" + ], [ "rules_zig+", "bazel_skylib", diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index bb5c804a..744522cd 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -19,11 +19,11 @@ PLATFORMS = { config_setting( name = "rocm.enabled", - flag_values = {":rocm": "True"}, constraint_values = [ "@platforms//cpu:x86_64", "@platforms//os:linux", ], + flag_values = {":rocm": "True"}, visibility = ["//visibility:public"], ) @@ -33,13 +33,18 @@ config_setting( flag_values = {":{}".format(runtime): "True"}, visibility = ["//visibility:public"], ) - for runtime in ["cpu", "cuda", "tpu", "neuron"] + for runtime in [ + "cpu", + "cuda", + "tpu", + "neuron", + ] ] zig_library( name = "platforms", - tags = ["manual"], main = "platforms.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ "//pjrt", @@ -94,10 +99,25 @@ platform( visibility = ["//visibility:public"], ) +constraint_setting(name = "execution_location") + +constraint_value( + name = "local_execution", + constraint_setting = ":execution_location", + visibility = ["//visibility:public"], +) + +# Special constrained host platform that toolchain can use as exec_compatible_with +# if they want to only be used when executing on the local machine. +# +# This is currently used by Zig toolchains so that they are only resolved for +# the host platform and thus can be used for no-remote-exec actions. platform( - name = "host_glibc", + name = "local_execution_platform", + constraint_values = [":local_execution"], + exec_properties = { + "no-remote-exec": "1", + }, parents = ["@platforms//host"], - constraint_values = [ - "@llvm//constraints/libc:gnu.2.28", - ], + visibility = ["//visibility:public"], ) diff --git a/third_party/zls/BUILD.bazel b/third_party/zls/BUILD.bazel deleted file mode 100644 index c8bfd83d..00000000 --- a/third_party/zls/BUILD.bazel +++ /dev/null @@ -1,56 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_binary") -load(":resolved_toolchain.bzl", "resolved_toolchain") - -# This is the ZLS binary toolchain type. Zig rule authors should use this -# target in the `toolchains` parameter in order to access the appropriate ZLS -# binary for the current execution platform. -toolchain_type( - name = "toolchain_type", - visibility = ["//visibility:public"], -) - -resolved_toolchain( - name = "resolved_toolchain", - # Marked manual so that `bazel test //...` passes - # even if no toolchain is registered. - tags = ["manual"], - visibility = ["//visibility:public"], -) - -exports_files( - [ - "workspace_printer.zig", - "workspace_printer_test.zig", - "zls_build_runner.zig", - "zls_runner.zig", - "zls.completion.tpl", - ], -) - -# Execute `bazel run //util:update_filegroups` to update this target. -filegroup( - name = "all_files", - srcs = [ - ":BUILD.bazel", - ":workspace_printer_test.zig", - ":zls.completion.tpl", - ":zls_build_runner.zig", - ":zls_completion.bzl", - ":zls_runner.zig", - ":zls_toolchain.bzl", - ], - visibility = ["@rules_zig//zig:__pkg__"], -) - -zig_binary( - name = "workspace_printer", - main = "workspace_printer.zig", - deps = [ - "@rules_zig//zig/runfiles", - ], -) - -zig_binary( - name = "build_runner", - main = "zls_build_runner.zig", -) diff --git a/third_party/zls/resolved_toolchain.bzl b/third_party/zls/resolved_toolchain.bzl deleted file mode 100644 index c5e38e63..00000000 --- a/third_party/zls/resolved_toolchain.bzl +++ /dev/null @@ -1,24 +0,0 @@ -"""This module implements an alias rule to the resolved toolchain.""" - -DOC = """\ -Exposes a concrete toolchain which is the result of Bazel resolving the -toolchain for the execution or target platform. -Workaround for https://github.com/bazelbuild/bazel/issues/14009 -""" - -# Forward all the providers -def _resolved_toolchain_impl(ctx): - toolchain_info = ctx.toolchains["//third_party/zls:toolchain_type"] - return [ - toolchain_info, - toolchain_info.default, - toolchain_info.zlstoolchaininfo, - ] - -# Copied from java_toolchain_alias -# https://cs.opensource.google/bazel/bazel/+/master:tools/jdk/java_toolchain_alias.bzl -resolved_toolchain = rule( - implementation = _resolved_toolchain_impl, - toolchains = ["//third_party/zls:toolchain_type"], - doc = DOC, -) diff --git a/third_party/zls/string_replace_reader.zig b/third_party/zls/string_replace_reader.zig deleted file mode 100644 index 3b68bf67..00000000 --- a/third_party/zls/string_replace_reader.zig +++ /dev/null @@ -1,28 +0,0 @@ -const std = @import("std"); - -pub const StringReplaceReader = struct { - interface: std.Io.Reader = .{ - .buffer = &.{}, - .vtable = &.{ - .stream = stream, - }, - .seek = 0, - .end = 0, - }, - in: *std.Io.Reader, - pattern: []const u8, - replace_by: []const u8, - - pub fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { - const self: *StringReplaceReader = @fieldParentPtr("interface", r); - if (std.mem.eql(u8, try self.in.peek(self.pattern.len), self.pattern)) { - self.in.toss(self.pattern.len); - try w.writeAll(self.replace_by); - return self.replace_by.len; - } - return self.in.streamDelimiterLimit(w, self.pattern[0], limit) catch |err| switch (err) { - std.Io.Reader.StreamDelimiterLimitError.StreamTooLong => return 0, - else => |e| return e, - }; - } -}; diff --git a/third_party/zls/workspace_printer.zig b/third_party/zls/workspace_printer.zig deleted file mode 100644 index dd4b38e3..00000000 --- a/third_party/zls/workspace_printer.zig +++ /dev/null @@ -1,169 +0,0 @@ -const std = @import("std"); - -const bazel_builtin = @import("bazel_builtin"); -const runfiles = @import("runfiles"); - -/// https://github.com/zigtools/zls/blob/master/src/build_runner/shared.zig#L6 (2026-03-10) -pub const BuildConfig = struct { - /// The `dependencies` in `build.zig.zon`. - dependencies: std.json.ArrayHashMap([]const u8), - /// The key is the `root_source_file`. - /// All modules with the same root source file are merged. This limitation may be lifted in the future. - modules: std.json.ArrayHashMap(Module), - /// List of all compilations units. - compilations: []Compile, - /// The names of all top level steps. - top_level_steps: []const []const u8, - available_options: std.json.ArrayHashMap(AvailableOption), - - pub const Module = struct { - import_table: std.json.ArrayHashMap([]const u8), - c_macros: []const []const u8, - include_dirs: []const []const u8, - }; - - pub const Compile = struct { - /// Key in `BuildConfig.modules`. - root_module: []const u8, - - // may contain additional information in the future like `target` or `link_libc`. - }; - - /// Equivalent to `std.Build.AvailableOption` which is not accessible because it non-pub. - pub const AvailableOption = @FieldType(@FieldType(std.Build, "available_options_map").KV, "value"); -}; - -fn processConfigLeaky(allocator: std.mem.Allocator, io: std.Io, config_string: []const u8) !BuildConfig { - var config = try std.json.parseFromSliceLeaky(BuildConfig, allocator, config_string, .{}); - try canonicalizeModulesLeaky(allocator, io, &config); - return config; -} - -// Neovim canonicalizes the path it sends to ZLS, we need to do the same as ZLS will match against the config we generate. -// We canonicalize the following module paths: -// -// { -// "modules": { -// "": { -// "import_table": { -// "": "" -// }, -// }, -// }, -// } -fn canonicalizeModulesLeaky(allocator: std.mem.Allocator, io: std.Io, config: *BuildConfig) !void { - var old_modules = config.modules.map.iterator(); - var new_modules: std.StringArrayHashMapUnmanaged(BuildConfig.Module) = .empty; - try new_modules.ensureTotalCapacity(allocator, old_modules.len); - - while (old_modules.next()) |entry| { - const module_name = canonicalizePath(allocator, io, entry.key_ptr.*); - var module: BuildConfig.Module = entry.value_ptr.*; - for (module.import_table.map.values()) |*imported_module_path| { - imported_module_path.* = canonicalizePath(allocator, io, imported_module_path.*); - } - new_modules.put(allocator, module_name, entry.value_ptr.*) catch return; - } - - for (config.compilations) |*compile| { - compile.root_module = canonicalizePath(allocator, io, compile.root_module); - } - - config.modules.map = new_modules; -} - -fn canonicalizePath(allocator: std.mem.Allocator, io: std.Io, path: []const u8) []const u8 { - return std.Io.Dir.realPathFileAbsoluteAlloc(io, path, allocator) catch path; -} - -fn readBuildConfig( - allocator: std.mem.Allocator, - io: std.Io, - init: std.process.Init, - arg_path: []const u8, -) ![]u8 { - if (std.fs.path.isAbsolutePosix(arg_path)) { - return std.Io.Dir.cwd().readFileAlloc(io, arg_path, allocator, .unlimited); - } - - var r_ = try runfiles.Runfiles.create(.{ - .allocator = allocator, - .io = io, - .environ_map = init.environ_map, - .argv = init.minimal.args, - }) orelse return error.RunfilesNotFound; - - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const r = r_.withSourceRepo(bazel_builtin.current_repository); - - if (try r.rlocation(arg_path, &path_buf)) |config_path| { - return std.Io.Dir.cwd().readFileAlloc(io, config_path, allocator, .unlimited); - } - - const stripped = if (std.mem.startsWith(u8, arg_path, "./")) arg_path[2..] else arg_path; - if (try r.rlocation(stripped, &path_buf)) |config_path| { - return std.Io.Dir.cwd().readFileAlloc(io, config_path, allocator, .unlimited); - } - - // Convenience for testing. - const output = std.Io.Dir.cwd().readFileAlloc(io, arg_path, allocator, .unlimited) catch |err| switch (err) { - error.FileNotFound => null, - else => return err, - }; - if (output) |content| { - return content; - } - - return error.BuildConfigNotFound; -} - -pub fn main(init: std.process.Init) !void { - const arena = init.arena; - const io = init.io; - const args = try init.minimal.args.toSlice(arena.allocator()); - - const build_workspace_directory = init.environ_map.get("BUILD_WORKSPACE_DIRECTORY").?; - const execution_root = if (init.environ_map.get("BAZEL_EXECUTION_ROOT")) |value| - value - else blk: { - const output_base_result = try std.process.run(arena.allocator(), init.io, .{ - .argv = &.{ - "bazel", - "info", - "execution_root", - }, - .cwd = .{ .path = build_workspace_directory }, - }); - break :blk std.mem.trimEnd(u8, output_base_result.stdout, "\n"); - }; - - var output = try readBuildConfig( - arena.allocator(), - io, - init, - args[1], - ); - output = try std.mem.replaceOwned( - u8, - arena.allocator(), - output, - "@@__BUILD_WORKSPACE_DIRECTORY__@@", - build_workspace_directory, - ); - output = try std.mem.replaceOwned( - u8, - arena.allocator(), - output, - "@@__BAZEL_EXECUTION_ROOT__@@", - execution_root, - ); - - var stdout_buf: [4096]u8 = undefined; - var writer = std.Io.File.stdout().writer(io, &stdout_buf); - if (processConfigLeaky(arena.allocator(), io, output)) |config| { - try std.json.Stringify.value(config, .{}, &writer.interface); - } else |_| { - try writer.interface.writeAll(output); - } - try writer.flush(); -} diff --git a/third_party/zls/workspace_printer_test.zig b/third_party/zls/workspace_printer_test.zig deleted file mode 100644 index 71e81adb..00000000 --- a/third_party/zls/workspace_printer_test.zig +++ /dev/null @@ -1,90 +0,0 @@ -const std = @import("std"); - -const bazel_builtin = @import("bazel_builtin"); -const runfiles = @import("runfiles"); - -const workspace_printer = @import("workspace_printer.zig"); - -fn requireEnv(name: [:0]const u8) ![]const u8 { - return std.mem.span(std.c.getenv(name) orelse return error.MissingEnvironmentVariable); -} - -fn workspaceDirFromConfigPath(config_path: []const u8, package_name: []const u8) ![]const u8 { - var workspace_dir = std.fs.path.dirname(config_path) orelse - return error.InvalidWorkspacePath; - if (package_name.len == 0) { - return workspace_dir; - } - - var it = std.mem.splitScalar(u8, package_name, '/'); - while (it.next()) |_| { - workspace_dir = std.fs.path.dirname(workspace_dir) orelse - return error.InvalidWorkspacePath; - } - return workspace_dir; -} - -test "completion print_build_config emits valid json" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const allocator = arena.allocator(); - - var r_ = try runfiles.Runfiles.create(.{ - .allocator = allocator, - .io = std.testing.io, - .directory = if (std.c.getenv("RUNFILES_DIR")) |value| std.mem.span(value) else null, - .manifest = if (std.c.getenv("RUNFILES_MANIFEST_FILE")) |value| std.mem.span(value) else null, - }) orelse return error.RunfilesNotFound; - defer r_.deinit(allocator); - - const r = r_.withSourceRepo(bazel_builtin.current_repository); - - const printer_rpath = try requireEnv("COMPLETION_PRINTER_RLOCATION"); - const config_rpath = try requireEnv("COMPLETION_BUILD_CONFIG_RLOCATION"); - const package_name = try requireEnv("COMPLETION_PACKAGE"); - - const printer_path = try r.rlocationAlloc(allocator, printer_rpath) orelse - return error.RLocationNotFound; - const config_path = try r.rlocationAlloc(allocator, config_rpath) orelse - return error.RLocationNotFound; - const workspace_dir = try workspaceDirFromConfigPath(config_path, package_name); - const execution_root = workspace_dir; - - var child_env_map: std.process.Environ.Map = .init(std.testing.allocator); - defer child_env_map.deinit(); - try child_env_map.put("BUILD_WORKSPACE_DIRECTORY", workspace_dir); - try child_env_map.put("BAZEL_EXECUTION_ROOT", execution_root); - - const result = try std.process.run(std.testing.allocator, std.testing.io, .{ - .argv = &.{ printer_path, config_path }, - .environ_map = &child_env_map, - }); - defer std.testing.allocator.free(result.stdout); - defer std.testing.allocator.free(result.stderr); - - if (result.stderr.len > 0) { - std.log.warn("workspace_printer stderr: {s}", .{result.stderr}); - } - - try std.testing.expectEqual(@as(std.process.Child.Term, .{ .exited = 0 }), result.term); - - const config = try std.json.parseFromSliceLeaky(workspace_printer.BuildConfig, allocator, result.stdout, .{}); - try std.testing.expect(config.modules.map.count() > 0); - - var modules = config.modules.map.iterator(); - while (modules.next()) |entry| { - try std.testing.expect(entry.key_ptr.*.len > 0); - - const module = entry.value_ptr.*; - for (module.import_table.map.values()) |import_path| { - try std.testing.expect(import_path.len > 0); - } - for (module.include_dirs) |include_dir| { - try std.testing.expect(include_dir.len > 0); - } - } - - for (config.compilations) |compilation| { - try std.testing.expect(config.modules.map.contains(compilation.root_module)); - } -} diff --git a/third_party/zls/zls.completion.tpl b/third_party/zls/zls.completion.tpl deleted file mode 100644 index b3fb688e..00000000 --- a/third_party/zls/zls.completion.tpl +++ /dev/null @@ -1,10 +0,0 @@ -{ - "dependencies": {}, - "modules": [ - @@PACKAGES@@ - ], - "include_dirs": [], - "top_level_steps": [], - "available_options": {}, - "c_macros": [] -} diff --git a/third_party/zls/zls_build_runner.zig b/third_party/zls/zls_build_runner.zig deleted file mode 100644 index ce82a66d..00000000 --- a/third_party/zls/zls_build_runner.zig +++ /dev/null @@ -1,14 +0,0 @@ -const std = @import("std"); - -pub fn main(init: std.process.Init) !void { - const build_workspace_directory = init.environ_map.get("BUILD_WORKSPACE_DIRECTORY").?; - var child = try std.process.spawn(init.io, .{ - .argv = &.{ - "bazel", - "run", - "@@__TARGET__@@", - }, - .cwd = .{ .path = build_workspace_directory }, - }); - _ = try child.wait(init.io); -} diff --git a/third_party/zls/zls_completion.bzl b/third_party/zls/zls_completion.bzl deleted file mode 100644 index c6ccadfe..00000000 --- a/third_party/zls/zls_completion.bzl +++ /dev/null @@ -1,103 +0,0 @@ -"""Implementation of the zls_completion macro.""" - -load("@bazel_lib//lib:utils.bzl", "utils") -load("@bazel_skylib//rules:expand_template.bzl", "expand_template") -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_test") -load(":zls_write_build_config.bzl", "zls_write_build_config") -load(":zls_write_runner_zig_src.bzl", "zls_write_runner_zig_src") - -def zls_completion(name, deps, testonly = False, **kwargs): - """Entry point for ZLS completion. - - Args: - name: The name of the completion target. - deps: The List of Zig modules to include for completion. - **kwargs: Additional keyword arguments passed to the `zig_binary` - """ - - # Generate the ZLS BuildConfig file. - # It contains the list of Zig packages alongside their main Zig file paths. - build_config = name + ".build_config" - build_config_file = name + ".build_config.json" - zls_write_build_config( - name = build_config, - out = build_config_file, - deps = deps, - testonly = testonly, - ) - - # Create a target that will be invoked by ZLS using our customer build_runner. - build_config_printer = "{}.print_build_config".format(name) - zig_binary( - name = build_config_printer, - main = Label("//third_party/zls:workspace_printer.zig"), - data = [ - ":{}".format(build_config), - ":{}".format(build_config_file), - ], - args = [ - "$(rlocationpath :{})".format(build_config_file), - ], - deps = [ - "@rules_zig//zig/runfiles", - ], - visibility = ["//visibility:private"], - testonly = testonly, - ) - - # Used to verify that the build config printer works as expected. - # Prevent regressions. - zig_test( - name = "{}_test".format(build_config_printer), - main = Label("//third_party/zls:workspace_printer_test.zig"), - srcs = [Label("//third_party/zls:workspace_printer.zig")], - data = deps + [ - ":{}".format(build_config), - ":{}".format(build_config_file), - ":{}".format(build_config_printer), - ], - env = { - "COMPLETION_BUILD_CONFIG_RLOCATION": "$(rlocationpath :{})".format(build_config_file), - "COMPLETION_PACKAGE": native.package_name(), - "COMPLETION_PRINTER_RLOCATION": "$(rlocationpath :{})".format(build_config_printer), - }, - deps = [ - "@rules_zig//zig/runfiles", - ], - testonly = testonly, - ) - - # Generate the Zig build runner that will be used by ZLS to query the build config. - expand_template( - name = name + ".build_runner", - out = name + ".build_runner.zig", - substitutions = { - "@@__TARGET__@@": str(utils.to_label(build_config_printer)), - }, - template = Label(":zls_build_runner.zig"), - testonly = testonly, - ) - - # Generate the Zig source file for the ZLS runner binary which embeds the - # rlocationpath of all runtime dependencies of the ZLS runner binary. - zls_write_runner_zig_src( - name = name + ".runner", - out = name + ".runner.zig", - build_runner = ":" + name + ".build_runner.zig", - testonly = testonly, - ) - - zig_binary( - name = name, - main = name + ".runner", - data = [ - "@rules_zig//zig:resolved_toolchain", - "@zml//third_party/zls:resolved_toolchain", - name + ".build_runner", - ], - deps = [ - "@rules_zig//zig/runfiles", - ], - testonly = testonly, - **kwargs - ) diff --git a/third_party/zls/zls_runner.zig b/third_party/zls/zls_runner.zig deleted file mode 100644 index aac52646..00000000 --- a/third_party/zls/zls_runner.zig +++ /dev/null @@ -1,100 +0,0 @@ -/// Custom ZLS launcher. -/// -/// Sets up paths to ZLS dependencies with Bazel runfiles. -/// -/// This file is used as a template by `zls_write_runner_zig_src.bzl`. -const std = @import("std"); -const fs = std.fs; - -const bazel_builtin = @import("bazel_builtin"); -const runfiles = @import("runfiles"); - -fn getRandomFilename(io: std.Io, buf: []u8, extension: []const u8) ![]const u8 { - const now = std.Io.Clock.real.now(io); - return std.fmt.bufPrint(buf, "/tmp/{d}{s}", .{ now.nanoseconds, extension }) catch @panic("OOM"); -} - -const Config = struct { - /// Override the Zig library path. Will be automatically resolved using the 'zig_exe_path'. - zig_lib_path: ?[]const u8 = null, - - /// Specify the path to the Zig executable (not the directory). If unset, zig is looked up in `PATH`. e.g. `/path/to/zig-templeos-armless-1.0.0/zig`. - zig_exe_path: ?[]const u8 = null, - - /// Specify a custom build runner to resolve build system information. - build_runner_path: ?[]const u8 = null, - - /// Path to a directory that will be used as zig's cache. Will default to `${KnownFolders.Cache}/zls`. - global_cache_path: ?[]const u8 = null, -}; - -pub fn main(init: std.process.Init) !void { - const arena = init.arena; - const io = init.io; - - var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator(), .io = io, .environ_map = init.environ_map, .argv = init.minimal.args }) orelse - return error.RunfilesNotFound; - const r = r_.withSourceRepo(bazel_builtin.current_repository); - - const zls_bin_rpath = "@@__ZLS_BIN_RPATH__@@"; - var zls_bin_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const zls_bin_path = try r.rlocation(zls_bin_rpath, &zls_bin_path_buf) orelse - return error.RLocationNotFound; - - const zig_exe_rpath = "@@__ZIG_EXE_RPATH__@@"; - var zig_exe_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const zig_exe_path = try r.rlocation(zig_exe_rpath, &zig_exe_path_buf) orelse - return error.RLocationNotFound; - - const zig_lib_path = "@@__ZIG_LIB_RPATH__@@"; - var zig_lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const zig_lib_computed_path = try r.rlocation(zig_lib_path, &zig_lib_path_buf) orelse - return error.RLocationNotFound; - - const zls_build_runner_rpath = "@@__ZLS_BUILD_RUNNER_RPATH__@@"; - var zls_build_runner_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const zls_build_runner_path = try r.rlocation(zls_build_runner_rpath, &zls_build_runner_path_buf) orelse - return error.RLocationNotFound; - - const global_cache_path = "@@__GLOBAL_CACHE_PATH__@@"; - - const config: Config = .{ - .zig_exe_path = zig_exe_path, - .zig_lib_path = zig_lib_computed_path, - .build_runner_path = zls_build_runner_path, - .global_cache_path = global_cache_path, - }; - - var buf: [std.Io.Dir.max_name_bytes]u8 = undefined; - const tmp_file_path = try getRandomFilename(io, &buf, ".json"); - - { - var tmp_file = try std.Io.Dir.createFileAbsolute(io, tmp_file_path, .{}); - defer tmp_file.close(io); - - var out_buf: [4096]u8 = undefined; - var tmp_file_writer = tmp_file.writer(io, &out_buf); - - const formatter = std.json.fmt(config, .{ .whitespace = .indent_2 }); - - try tmp_file_writer.interface.print("{f}", .{formatter}); - try tmp_file_writer.interface.flush(); - } - - const args = init.minimal.args.toSlice(arena.allocator()) catch return error.ArgsAllocFailed; - - const exec_args_len = args.len - 1 + 3; // Skip args[0] and add "zls" + --config-path + tmp_file_path - var exec_args = try arena.allocator().alloc([]const u8, exec_args_len); - - exec_args[0] = zls_bin_path; // "zls" - - @memcpy(exec_args[1 .. exec_args.len - 2], args[1..]); - @memcpy(exec_args[exec_args.len - 2 ..], &[_][]const u8{ - "--config-path", - tmp_file_path, - }); - var child = try std.process.spawn(io, .{ - .argv = exec_args, - }); - _ = try child.wait(io); -} diff --git a/third_party/zls/zls_toolchain.bzl b/third_party/zls/zls_toolchain.bzl deleted file mode 100644 index 337be425..00000000 --- a/third_party/zls/zls_toolchain.bzl +++ /dev/null @@ -1,129 +0,0 @@ -"""Implementation of the zls_toolchain rule.""" - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -_VERSION = "0.16.0" - -_ZLS_PLATFORMS = { - "x86_64-linux": struct( - compatible_with = [ - "@platforms//os:linux", - "@platforms//cpu:x86_64", - ], - ), - "aarch64-linux": struct( - compatible_with = [ - "@platforms//os:linux", - "@platforms//cpu:aarch64", - ], - ), - "x86_64-macos": struct( - compatible_with = [ - "@platforms//os:macos", - "@platforms//cpu:x86_64", - ], - ), - "aarch64-macos": struct( - compatible_with = [ - "@platforms//os:macos", - "@platforms//cpu:aarch64", - ], - ), -} - -_ZLS_VERSIONS = { - "x86_64-linux": "ded6d562a0b86ee878b1ddf70ffab2797ce3cdca3b02d6077548f9d56dff96b6", - "aarch64-linux": "430cd293d201eb70ae2519dbc96c854bf8791b8df7fc9392e8d2dc9680a2bed7", - "x86_64-macos": "49f716ea96c1aadaecaa5d9c0a50874cbcf443dc42b825f1e7ee35499ad3eb96", - "aarch64-macos": "b93ec549f8558a7e85984a840e9276d274f1059b54ade4254296ef4982958359", -} - -ZlsToolchainInfo = provider( - doc = "Provide info for executing zls", - fields = { - "bin": "Executable ZLS binary", - }, -) - -def _zls_toolchain_impl(ctx): - binary = ctx.file.bin - - default_info = DefaultInfo( - files = depset([binary]), - runfiles = ctx.runfiles(files = [binary]), - ) - zlstoolchaininfo = ZlsToolchainInfo( - bin = binary, - ) - toolchain_info = platform_common.ToolchainInfo( - default = default_info, - zlstoolchaininfo = zlstoolchaininfo, - ) - - return [ - default_info, - zlstoolchaininfo, - toolchain_info, - ] - -zls_toolchain = rule( - implementation = _zls_toolchain_impl, - attrs = { - "bin": attr.label( - executable = True, - allow_single_file = True, - cfg = "exec", - ), - }, -) - -def _zls_toolchains_repos_impl(mctx): - for platform, _ in _ZLS_PLATFORMS.items(): - http_archive( - name = "zls_{}".format(platform), - url = "https://builds.zigtools.org/zls-{platform}-{version}.tar.xz".format( - version = _VERSION, - platform = platform, - ), - sha256 = _ZLS_VERSIONS[platform], - build_file_content = """\ -load("@zml//third_party/zls:zls_toolchain.bzl", "zls_toolchain") -zls_toolchain(name = "toolchain", bin = "zls") -""", - ) - - zls_toolchains_repo(name = "zls_toolchains") - - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = ["zls_toolchains"], - root_module_direct_dev_deps = [], - ) - -zls_toolchains = module_extension( - implementation = _zls_toolchains_repos_impl, -) - -def _zls_toolchains_repo_impl(rctx): - build_content = """# @generated by @zml//third_party/zls:zls_toolchain.bzl""" - - for platform, config in _ZLS_PLATFORMS.items(): - build_content += """ -toolchain( - name = "{platform}_toolchain", - exec_compatible_with = {compatible_with}, - toolchain = "@zls_{platform}//:toolchain", - toolchain_type = "@zml//third_party/zls:toolchain_type", -) -""".format( - platform = platform, - compatible_with = config.compatible_with, - ) - - rctx.file("BUILD.bazel", build_content) - -zls_toolchains_repo = repository_rule( - _zls_toolchains_repo_impl, - doc = """Creates a repository with toolchain definitions for all known platforms - which can be registered or selected.""", -) diff --git a/third_party/zls/zls_write_build_config.bzl b/third_party/zls/zls_write_build_config.bzl deleted file mode 100644 index d5d863d8..00000000 --- a/third_party/zls/zls_write_build_config.bzl +++ /dev/null @@ -1,177 +0,0 @@ -"""Generates the ZLS build config file.""" - -load("@rules_cc//cc:find_cc_toolchain.bzl", "use_cc_toolchain") -load("@rules_cc//cc/common:cc_common.bzl", "cc_common") -load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") -load("@rules_zig//zig/private:cc_helper.bzl", "need_translate_c") -load("@rules_zig//zig/private/common:translate_c.bzl", "zig_translate_c") -load("@rules_zig//zig/private/common:zig_cache.bzl", "zig_cache_output") -load("@rules_zig//zig/private/common:zig_lib_dir.bzl", "zig_lib_dir") -load("@rules_zig//zig/private/providers:zig_module_info.bzl", "ZigModuleInfo", "zig_module_info") -load( - "@rules_zig//zig/private/providers:zig_target_info.bzl", - "zig_target_platform", -) - -def _is_zig_target(kind): - if not kind.startswith("zig_"): - return False - for end in ("_binary", "_static_library", "_shared_library", "_test"): - if kind.endswith(end): - return True - return False - -def _zls_construct_zig_module_info_impl(target, ctx): - """Aspect that constructs ZigModuleInfo for zig_binary and zig_library rules.""" - if ZigModuleInfo in target: - return [] - - if _is_zig_target(ctx.rule.kind) == False: - return [] - - cdeps = [] - zdeps = [] - for dep in ctx.rule.attr.deps: - if ZigModuleInfo in dep: - zdeps.append(dep[ZigModuleInfo]) - elif CcInfo in dep: - cdeps.append(dep[CcInfo]) - - root_module = None - if ctx.rule.attr.main: - root_module = zig_module_info( - name = ctx.rule.attr.name, - canonical_name = target.label.name, - main = ctx.rule.file.main, - srcs = ctx.rule.files.srcs, - extra_srcs = ctx.rule.files.extra_srcs, - deps = zdeps, # [bazel_builtin_module(ctx)], - cdeps = cdeps, - zigopts = [], - ) - else: - root_module = zdeps[0] - - return [ - root_module, - ] - -zls_construct_zig_module_info = aspect( - implementation = _zls_construct_zig_module_info_impl, -) - -def format_main_file(main): - prefix = "@@__BUILD_WORKSPACE_DIRECTORY__@@/" - if (main.startswith("bazel-out/") or main.startswith("external/")): - prefix = "@@__BAZEL_EXECUTION_ROOT__@@/" - return prefix + main - -def _zls_write_build_config_impl(ctx): - zigtoolchaininfo = ctx.toolchains["@rules_zig//zig:toolchain_type"].zigtoolchaininfo - zigtargetinfo = ctx.toolchains["@rules_zig//zig/target:toolchain_type"].zigtargetinfo - translate_c_toolchain = ctx.toolchains["@rules_zig//zig/translate-c:toolchain_type"] - translatectoolchaininfo = translate_c_toolchain.translatectoolchaininfo if translate_c_toolchain else None - - c_module_contexts = [] - c_module_inputs = [] - cc_info = cc_common.merge_cc_infos(direct_cc_infos = [dep[ZigModuleInfo].cc_info for dep in ctx.attr.deps if dep[ZigModuleInfo].cc_info]) - if need_translate_c(cc_info): - global_args = ctx.actions.args() - global_args.use_param_file("@%s") - - zig_target_platform( - target = zigtargetinfo, - args = global_args, - ) - - zig_lib_dir( - zigtoolchaininfo = zigtoolchaininfo, - args = global_args, - ) - - zig_cache_output( - zigtoolchaininfo = zigtoolchaininfo, - args = global_args, - ) - - c_module = zig_translate_c( - ctx = ctx, - name = "c", - canonical_name = "c", - zigtoolchaininfo = zigtoolchaininfo, - global_args = global_args, - cc_infos = [cc_info], - translatectoolchaininfo = translatectoolchaininfo, - ) - - c_module_contexts = [depset( - direct = [c_module.module_context], - transitive = [c_module.transitive_module_contexts], - )] - c_module_inputs = [c_module.transitive_inputs] - - contexts = depset( - direct = [dep[ZigModuleInfo].module_context for dep in ctx.attr.deps], - transitive = [dep[ZigModuleInfo].transitive_module_contexts for dep in ctx.attr.deps] + c_module_contexts, - ) - - modules = { - mod.canonical_name: mod - for mod in contexts.to_list() - } - - output = { - "dependencies": {}, - "modules": { - format_main_file(mod.main): { - "import_table": { - dep.name: format_main_file(modules.get(dep.canonical_name, "").main) - for dep in mod.dependency_mappings - }, - "c_macros": [], - "include_dirs": [], - } - for mod in modules.values() - }, - "compilations": [], - "top_level_steps": [], - "available_options": {}, - } - - config = ctx.outputs.out - ctx.actions.write( - output = config, - content = json.encode(output), - ) - - return [ - DefaultInfo( - files = depset(direct = [config]), - # Not really runfiles but we need to pull the output of translate_c... - runfiles = ctx.runfiles(transitive_files = depset(transitive = c_module_inputs)), - ), - ] - -zls_write_build_config = rule( - implementation = _zls_write_build_config_impl, - attrs = { - "deps": attr.label_list( - providers = [ZigModuleInfo], - mandatory = True, - aspects = [zls_construct_zig_module_info], - ), - "out": attr.output( - mandatory = True, - ), - "_zls_completion_tpl": attr.label( - default = ":zls.completion.tpl", - allow_single_file = True, - ), - }, - toolchains = [ - "@rules_zig//zig:toolchain_type", - "@rules_zig//zig/target:toolchain_type", - config_common.toolchain_type("@rules_zig//zig/translate-c:toolchain_type", mandatory = False), - ] + use_cc_toolchain(mandatory = False), - fragments = ["cpp", "apple"], -) diff --git a/third_party/zls/zls_write_runner_zig_src.bzl b/third_party/zls/zls_write_runner_zig_src.bzl deleted file mode 100644 index 2217db20..00000000 --- a/third_party/zls/zls_write_runner_zig_src.bzl +++ /dev/null @@ -1,58 +0,0 @@ -"""Generates the ZLS runner source file.""" - -load("@bazel_lib//lib:paths.bzl", "to_rlocation_path") - -def zig_toolchain_executable_rpath(ctx, zigtoolchaininfo): - if zigtoolchaininfo.zig_exe.file != None: - return to_rlocation_path(ctx, zigtoolchaininfo.zig_exe.file) - return zigtoolchaininfo.zig_exe.path - -def zig_toolchain_lib_rpath(ctx, zigtoolchaininfo): - if zigtoolchaininfo.zig_lib.file != None: - return to_rlocation_path(ctx, zigtoolchaininfo.zig_lib.file) - return zigtoolchaininfo.zig_lib.path - -def _zls_write_runner_zig_src_impl(ctx): - zigtoolchaininfo = ctx.toolchains["@rules_zig//zig:toolchain_type"].zigtoolchaininfo - zlstoolchaininfo = ctx.toolchains["//third_party/zls:toolchain_type"].zlstoolchaininfo - - zig_exe_rpath = zig_toolchain_executable_rpath(ctx, zigtoolchaininfo) - zig_lib_rpath = zig_toolchain_lib_rpath(ctx, zigtoolchaininfo) - - zls_runner = ctx.outputs.out - ctx.actions.expand_template( - output = zls_runner, - template = ctx.file._runner_tpl, - substitutions = { - "@@__ZIG_EXE_RPATH__@@": zig_exe_rpath, - "@@__ZIG_LIB_RPATH__@@": zig_lib_rpath, - "@@__ZLS_BIN_RPATH__@@": to_rlocation_path(ctx, zlstoolchaininfo.bin), - "@@__ZLS_BUILD_RUNNER_RPATH__@@": to_rlocation_path(ctx, ctx.file.build_runner), - "@@__GLOBAL_CACHE_PATH__@@": zigtoolchaininfo.zig_cache, - }, - ) - - return [ - DefaultInfo(files = depset([zls_runner])), - ] - -zls_write_runner_zig_src = rule( - implementation = _zls_write_runner_zig_src_impl, - attrs = { - "out": attr.output( - mandatory = True, - ), - "build_runner": attr.label( - allow_single_file = True, - mandatory = True, - ), - "_runner_tpl": attr.label( - default = "zls_runner.zig", - allow_single_file = True, - ), - }, - toolchains = [ - "@rules_zig//zig:toolchain_type", - "//third_party/zls:toolchain_type", - ], -) From 5d372aa4c0045fe743d6b7d3ad07466ecd1d8dbb Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 20 May 2026 16:34:47 +0200 Subject: [PATCH 494/881] zml/Sharding: Reduce Placement size by allocating shards on the stack. (#547) `Placement` is 27KiB big, it led to stack allocation failure with gigantic struct sizes (~400KiB) in llmd. So changing the shards to be allocated on the heap with functions to avoid allocations when we don't need it (`format`, etc.). --- examples/sharding/main.zig | 13 ++- zml/Sharding.zig | 189 ++++++++++++++++++++++++++----------- zml/buffer.zig | 26 +++-- zml/io.zig | 7 +- 4 files changed, 168 insertions(+), 67 deletions(-) diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index 2a407b67..327244cb 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -231,9 +231,16 @@ pub fn main(init: std.process.Init) !void { var input_buf = try createSequenceBuffer(allocator, io, platform, input_shape, sharding, 1000.0); defer input_buf.deinit(); - log.info("input placement: {f}", .{input_buf.placement()}); - log.info("weight placement: {f}", .{w_buf.placement()}); - log.info("bias placement: {f}", .{b_buf.placement()}); + var input_placement = try input_buf.placement(allocator); + defer input_placement.deinit(allocator); + var w_placement = try w_buf.placement(allocator); + defer w_placement.deinit(allocator); + var b_placement = try b_buf.placement(allocator); + defer b_placement.deinit(allocator); + + log.info("input placement: {f}", .{input_placement}); + log.info("weight placement: {f}", .{w_placement}); + log.info("bias placement: {f}", .{b_placement}); var exe_args = try exe.args(allocator); defer exe_args.deinit(allocator); diff --git a/zml/Sharding.zig b/zml/Sharding.zig index 9d90eaae..6cfafb6d 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -103,8 +103,9 @@ pub const Partitioning = struct { pub fn localShapeForShape(self: Partitioning, shape: Shape) !Shape { const sharding = try self.selectSharding(shape); - const placement = try Placement.init(sharding, shape); - return placement.shards.get(0).shape.withDefaultPartitioning(); + var shards = Placement.shardIterator(sharding, shape); + const shard = (try shards.next()) orelse return error.InvalidPhysicalMesh; + return shard.shape.withDefaultPartitioning(); } pub fn numPartitionsForLogicalAxis(self: Partitioning, shape: Shape, logical_axis: anytype) !i64 { @@ -1766,7 +1767,7 @@ pub const Placement = struct { try writer.writeAll("]"); } }; - pub const Shards = stdx.BoundedArray(Shard, Platform.MAX_NUM_DEVICES); + pub const Shards = []Shard; const AxisSplit = struct { product: i64, @@ -1794,71 +1795,106 @@ pub const Placement = struct { } }; + pub const ShardIterator = struct { + sharding: Sharding, + shape: Shape, + ordered_devices: Devices, + index: usize = 0, + + pub fn count(self: *const ShardIterator) usize { + return self.ordered_devices.len; + } + + pub fn next(self: *ShardIterator) !?Shard { + if (self.index >= self.ordered_devices.len) return null; + defer self.index += 1; + + return try shardForDevice(self.sharding, self.shape, self.ordered_devices.get(self.index)); + } + }; + sharding: Sharding, shape: Shape, shards: Shards, + pub fn shardIterator(sharding: Sharding, shape: Shape) ShardIterator { + return .{ + .sharding = sharding, + .shape = shape, + .ordered_devices = sharding.data.devicesInCanonicalOrder(), + }; + } + + /// Caller owns `shards` and must call `deinit` with the same allocator. pub fn init( + allocator: std.mem.Allocator, sharding: Sharding, shape: Shape, ) !Placement { const ordered_devices = sharding.data.devicesInCanonicalOrder(); - var shards: Shards = .empty; + const shards = try allocator.alloc(Shard, ordered_devices.len); + errdefer allocator.free(shards); - for (ordered_devices.constSlice()) |d| { - if (d.coords.len == 0) return error.MissingDeviceCoords; - - shards.appendAssumeCapacity(.{ - .device_id = d.id, - .device_coords = d.coords, - .shape = undefined, // Calculated below - .global_shape = shape, - .slices = .empty, - }); + for (ordered_devices.constSlice(), 0..) |d, i| { + shards[i] = try shardForDevice(sharding, shape, d); } - var self: Placement = .{ + return .{ .sharding = sharding, .shape = shape, .shards = shards, }; + } - for (self.shards.slice()) |*s| { - var used_axes: std.EnumSet(PhysicalAxisTag) = .empty; + pub fn deinit(self: *Placement, allocator: std.mem.Allocator) void { + allocator.free(self.shards); + } - for (0..shape.rank()) |ax| { - const axis_index: u8 = @intCast(ax); - const slice = try self.sliceForDevice(s.device_coords.constSlice(), &used_axes, axis_index); - s.slices.appendAssumeCapacity(slice); - } + fn shardForDevice(sharding: Sharding, shape: Shape, device: Device) !Shard { + if (device.coords.len == 0) return error.MissingDeviceCoords; - s.shape = blk: { - var sh = shape; - for (s.slices.constSlice()) |slice| { - sh = sh.set(slice.axis, slice.size); - } - break :blk sh; - }; + var shard: Shard = .{ + .device_id = device.id, + .device_coords = device.coords, + .shape = undefined, + .global_shape = shape, + .slices = .empty, + }; + var used_axes: std.EnumSet(PhysicalAxisTag) = .empty; + + for (0..shape.rank()) |ax| { + const axis_index: u8 = @intCast(ax); + const slice = try sliceForDevice(sharding, shape, shard.device_coords.constSlice(), &used_axes, axis_index); + shard.slices.appendAssumeCapacity(slice); } - return self; + shard.shape = blk: { + var sh = shape; + for (shard.slices.constSlice()) |slice| { + sh = sh.set(slice.axis, slice.size); + } + break :blk sh; + }; + + return shard; } fn sliceForDevice( - self: *Placement, + sharding: Sharding, + shape: Shape, device_coords: []const usize, used_axes: *std.EnumSet(PhysicalAxisTag), axis_index: u8, ) !Slice1d { - const dim = self.shape.dim(axis_index); - const spec = self.shape.partition(axis_index); + const dim = shape.dim(axis_index); + const spec = shape.partition(axis_index); switch (spec) { .axis => |logical_tag| { - const binding = self.sharding.data.binding(logical_tag) orelse return error.MissingLogicalBinding; + const binding = sharding.data.binding(logical_tag) orelse return error.MissingLogicalBinding; // Calculate the split based on the physical coordinates of the current device - const plan = try self.calculateSplit(dim, binding, device_coords, used_axes); + const plan = try calculateSplit(sharding, dim, binding, device_coords, used_axes); const size = @divExact(dim, plan.product); const start = plan.linearIndex() * size; @@ -1874,7 +1910,7 @@ pub const Placement = struct { } fn calculateSplit( - self: *Placement, + sharding: Sharding, dim: i64, binding: []const PhysicalAxisTag, device_coords: []const usize, @@ -1884,16 +1920,16 @@ pub const Placement = struct { for (binding) |p_tag| { // Expand the physical tag: check if it's a target for folded source axes - if (self.sharding.data.foldSources(p_tag)) |sources| { + if (sharding.data.foldSources(p_tag)) |sources| { for (sources) |src| { if (used_axes.contains(src)) continue; - self.addPhysicalToSplit(&plan, src, device_coords); + addPhysicalToSplit(sharding, &plan, src, device_coords); used_axes.insert(src); } } else { // Standard case: directly bound, not part of an explicit fold if (used_axes.contains(p_tag)) continue; - self.addPhysicalToSplit(&plan, p_tag, device_coords); + addPhysicalToSplit(sharding, &plan, p_tag, device_coords); used_axes.insert(p_tag); } } @@ -1906,8 +1942,8 @@ pub const Placement = struct { } /// Extract coordinate data from the physical mesh for a specific axis. - fn addPhysicalToSplit(self: *Placement, plan: *AxisSplit, tag: PhysicalAxisTag, device_coords: []const usize) void { - const physical = self.sharding.data.physical; + fn addPhysicalToSplit(sharding: Sharding, plan: *AxisSplit, tag: PhysicalAxisTag, device_coords: []const usize) void { + const physical = sharding.data.physical; const info = physical.axisInfo(tag) orelse return; const depth = physical.axis_traversal.depth(tag) orelse return; const coord = device_coords[depth]; @@ -1916,17 +1952,45 @@ pub const Placement = struct { } pub fn format(self: Placement, writer: *std.Io.Writer) !void { - try writer.print("Placement(shape={f} shards={d})\n", .{ self.shape, self.shards.len }); + const Iterator = struct { + shards: []const Shard, + index: usize = 0, + + fn count(it: @This()) usize { + return it.shards.len; + } + + fn next(it: *@This()) !?Shard { + if (it.index >= it.shards.len) return null; + defer it.index += 1; + + return it.shards[it.index]; + } + }; + return formatFromShardSource(self.shape, Iterator{ .shards = self.shards }, writer); + } + + pub fn formatShards(sharding: Sharding, shape: Shape, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return formatFromShardSource(shape, shardIterator(sharding, shape), writer); + } + + fn formatFromShardSource(shape: Shape, shards_: anytype, writer: *std.Io.Writer) std.Io.Writer.Error!void { + var shards = shards_; + const first_shard = shards.next() catch |err| { + return formatShardError(shape, shards.count(), err, writer); + }; + + try writer.print("Placement(shape={f} shards={d})\n", .{ shape, shards.count() }); try writer.writeAll("Axis partitioning summary:\n"); - for (0..self.shape.rank()) |ax| { - const dim = self.shape.dim(ax); - const spec = self.shape.partition(ax); - const axis_label = self.shape.debugTag(ax); + for (0..shape.rank()) |ax| { + const dim = shape.dim(ax); + const spec = shape.partition(ax); + const axis_label = shape.debugTag(ax); var shard_size: i64 = dim; - if (self.shards.len > 0) { - for (self.shards.constSlice()[0].slices.constSlice()) |s| { + if (first_shard) |shard| { + for (shard.slices.constSlice()) |s| { if (s.axis == ax) { shard_size = s.size; break; @@ -1946,9 +2010,27 @@ pub const Placement = struct { } try writer.writeAll("Per-shard placement:\n"); - for (self.shards.constSlice(), 0..) |shard, i| { - try writer.print("└─ Shard[{d}] {f}\n", .{ i, shard }); + var shard_index: usize = 0; + if (first_shard) |shard| { + try writer.print("└─ Shard[{d}] {f}\n", .{ shard_index, shard }); + shard_index += 1; } + while (shards.next() catch |err| { + try writer.print("└─ Shard[{d}] error={s}\n", .{ shard_index, @errorName(err) }); + return; + }) |shard| : (shard_index += 1) { + try writer.print("└─ Shard[{d}] {f}\n", .{ shard_index, shard }); + } + } + + fn formatShardError( + shape: Shape, + shard_count: usize, + err: anyerror, + writer: *std.Io.Writer, + ) std.Io.Writer.Error!void { + try writer.print("Placement(shape={f} shards={d})\n", .{ shape, shard_count }); + try writer.print("Shard placement error: {s}\n", .{@errorName(err)}); } }; @@ -2026,15 +2108,16 @@ const ShardingTest = struct { // Verify Placement logic / Error if (s.expect_error) |err| { - try std.testing.expectError(err, Placement.init(.{ .data = &sharding }, s.shape)); + try std.testing.expectError(err, Placement.init(self.allocator, .{ .data = &sharding }, s.shape)); return; } // Verify Shard Slices (Math) if (s.expected_shards.len > 0) { - const placement = try Placement.init(.{ .data = &sharding }, s.shape); + var placement = try Placement.init(self.allocator, .{ .data = &sharding }, s.shape); + defer placement.deinit(self.allocator); try std.testing.expectEqual(s.expected_shards.len, placement.shards.len); - for (s.expected_shards, placement.shards.constSlice()) |expected, actual| { + for (s.expected_shards, placement.shards) |expected, actual| { try std.testing.expectEqual(expected.device_id, actual.device_id); for (expected.slices, actual.slices.constSlice()) |exp_slice, actual_slice| { try std.testing.expectEqual(exp_slice[0], actual_slice.start); diff --git a/zml/buffer.zig b/zml/buffer.zig index f15b411f..c6a34030 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -84,7 +84,7 @@ pub const Buffer = struct { } pub fn format(self: Buffer, writer: *std.Io.Writer) !void { - try writer.print("{f}", .{self.placement()}); + try Sharding.Placement.formatShards(self._sharding, self._shape, writer); } /// Copies the content of the given buffer from host memory to the accelerator memory. @@ -102,11 +102,15 @@ pub const Buffer = struct { ._sharding = sharding.resolve(platform), ._shards = .empty, }; + errdefer for (res._shards.slice()) |shard| { + shard.deinit(platform.pjrt_api); + }; const buffer_type = pjrtx.bufferTypeFromDtype(sh.dtype()); const slice = Slice.init(sh, data_); - for (res.placement().shards.constSlice()) |shard| { + var shard_iter = Sharding.Placement.shardIterator(res._sharding, res._shape); + while (try shard_iter.next()) |shard| { const sub_slice = shard.shardSlice(slice); var default_layout: ?pjrt.DefaultMemoryLayout = null; const layout: pjrt.MemoryLayout = switch (platform.target) { @@ -214,7 +218,8 @@ pub const Buffer = struct { shard.deinit(platform.pjrt_api); }; - for (res.placement().shards.constSlice()) |shard| { + var shard_iter = Sharding.Placement.shardIterator(res._sharding, res._shape); + while (try shard_iter.next()) |shard| { var default_layout: ?pjrt.DefaultMemoryLayout = null; const layout: pjrt.MemoryLayout = switch (platform.target) { .tpu => blk: { @@ -274,7 +279,9 @@ pub const Buffer = struct { pub fn toSlice(self: Buffer, io: std.Io, slice: Slice) !void { stdx.debug.assert(self._shape.eql(slice.shape), "Buffer shape {f} doesn't match destination slice {f}", .{ self._shape, slice.shape }); - for (self.placement().shards.constSlice(), 0..) |shard, shard_index| { + var shard_iter = Sharding.Placement.shardIterator(self._sharding, self._shape); + var shard_index: usize = 0; + while (try shard_iter.next()) |shard| : (shard_index += 1) { const sub_slice = shard.shardSlice(slice); if (!sub_slice.isContiguous()) return error.NonContiguousShardRead; @@ -298,7 +305,9 @@ pub const Buffer = struct { const slice = try Slice.alloc(allocator, self.shape()); errdefer slice.free(allocator); - for (self.placement().shards.constSlice(), 0..) |shard, shard_index| { + var shard_iter = Sharding.Placement.shardIterator(self._sharding, self._shape); + var shard_index: usize = 0; + while (try shard_iter.next()) |shard| : (shard_index += 1) { const sub_slice = shard.shardSlice(slice); var shard_slice = try Slice.alloc(allocator, sub_slice.shape); @@ -318,14 +327,15 @@ pub const Buffer = struct { pub fn byteSize(self: Buffer) usize { var byte_size: usize = 0; - for (self.placement().shards.constSlice()) |shard| { + var shard_iter = Sharding.Placement.shardIterator(self._sharding, self._shape); + while (shard_iter.next() catch unreachable) |shard| { byte_size += shard.shape.byteSize(); } return byte_size; } - pub fn placement(self: Buffer) Sharding.Placement { - return Sharding.Placement.init(self._sharding, self._shape) catch unreachable; + pub fn placement(self: Buffer, allocator: std.mem.Allocator) !Sharding.Placement { + return Sharding.Placement.init(allocator, self._sharding, self._shape); } }; diff --git a/zml/io.zig b/zml/io.zig index d6c9e94d..36a857e6 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -628,7 +628,7 @@ pub const DirectMemoryWriter = struct { } fn collectRawSegments(self: *StreamPlanner) !void { - for (self.placement.shards.constSlice(), 0..) |shard, writer_index| { + for (self.placement.shards, 0..) |shard, writer_index| { try self.appendShardSegments(shard, writer_index); } } @@ -788,7 +788,8 @@ pub const DirectMemoryWriter = struct { sharding: Sharding, buffer: *Buffer, ) !DirectMemoryWriter { - const placement = try Placement.init(sharding, shape); + var placement = try Placement.init(allocator, sharding, shape); + defer placement.deinit(allocator); var shard_writers = try allocator.alloc(DirectShardWriter, placement.shards.len); errdefer allocator.free(shard_writers); @@ -798,7 +799,7 @@ pub const DirectMemoryWriter = struct { }; var pjrt_buffers: Buffer.Shards = .empty; - for (placement.shards.constSlice(), 0..) |shard, i| { + for (placement.shards, 0..) |shard, i| { defer initialized += 1; const device_index = shard.platformDeviceIndex(platform) orelse return error.UnknownShardDevice; From 36300dee2eb3eb36593344147095b24f5535516c Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 22 May 2026 10:58:27 +0200 Subject: [PATCH 495/881] zml/attention: small changes required for llmd (#551) * Made the `MAX_NUM_SPLITS` public * Inject max num pages from args in `Options.fromBackend()` instead of computing it --- MODULE.bazel.lock | 8 +++--- zml/attention/flashattn.zig | 4 +-- zml/attention/paged_attention.zig | 44 +++++-------------------------- 3 files changed, 13 insertions(+), 43 deletions(-) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index d39cd41d..2ff0ccb9 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -343,8 +343,8 @@ "https://bcr.bazel.build/modules/rules_swift/1.18.0/MODULE.bazel": "a6aba73625d0dc64c7b4a1e831549b6e375fbddb9d2dde9d80c9de6ec45b24c9", "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", "https://bcr.bazel.build/modules/rules_swift/2.1.1/source.json": "40fc69dfaac64deddbb75bd99cdac55f4427d9ca0afbe408576a65428427a186", - "https://bcr.bazel.build/modules/rules_zig/0.15.0/MODULE.bazel": "8464afbe0a1b59967bf3d56cb166f889f1e53599a1194b2c0e668aae651cb9f8", - "https://bcr.bazel.build/modules/rules_zig/0.15.0/source.json": "33e47f366fae6e81fb5398972af4208c3546be162ca58be9142c018b516d4252", + "https://bcr.bazel.build/modules/rules_zig/0.15.1/MODULE.bazel": "f0b813d9191c8d201e0b3ecbe5c5220ddedb717a15009fc711a78b3e1db8710d", + "https://bcr.bazel.build/modules/rules_zig/0.15.1/source.json": "0bd91d7459f504196dfdc90cd0fffbf1f0df2667c9cd5eb708feb3fd27578e6a", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", "https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4", "https://bcr.bazel.build/modules/stardoc/0.5.6/MODULE.bazel": "c43dabc564990eeab55e25ed61c07a1aadafe9ece96a4efabb3f8bf9063b71ef", @@ -1110,7 +1110,7 @@ "@@rules_zig+//zig:extensions.bzl%zig": { "general": { "bzlTransitiveDigest": "u7Psh7szBG3+lbRmEeBBbWlZRHJWhWtxQo/wtRqvnTg=", - "usagesDigest": "E3Q/pEwXzITKlwKks+3Y68QKa4eUgIuqZzlQCbDX7dU=", + "usagesDigest": "4xBoe2eIScx2QI0jVieHWXdflezUUDIApwMqS+SWGbw=", "recordedFileInputs": { "@@//bazel/zig_index.json": "2e3b4b83f1add5d3ca12fd05797c04c38af21d86c7fc7c1a9b356560141a4797", "@@rules_zig+//zig/private/versions.json": "3e4e370dee9c3f3fdc8bfc7a600e76743278fde40815eead5d0bcda3aec3e2bd" @@ -1439,7 +1439,7 @@ "@@rules_zig+//zig/zls:extensions.bzl%zls": { "general": { "bzlTransitiveDigest": "8ZwSSF/7dE4rb0W0cZqil2SfGF1bPK04LTv1/zjtkTM=", - "usagesDigest": "FAKFOHWIN8RdVDOmVLmwjLdv/f9a2WDp48ekZEePDlA=", + "usagesDigest": "b/ERjMTWazfa8VFWuTc0PTtQQzLDuloWSHlzXj6Zb5w=", "recordedFileInputs": { "@@rules_zig+//zig/zls/private/versions.json": "127803289dfd79b5255db5f74be1d9a2f790f03292d6147009af702d888aba69" }, diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index d80e558a..723316c3 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -511,7 +511,7 @@ pub const fa3 = struct { pub const paged_fa2 = struct { // God knows why flash attention uses this number and not something else. - const MAX_NUM_SPLITS = 8; + pub const MAX_NUM_SPLITS = 8; pub const Variant = enum { decode, @@ -1187,7 +1187,7 @@ pub const paged_fa2 = struct { pub const paged_fa3 = struct { // God knows why flash attention uses this number and not something else. - const MAX_NUM_SPLITS = 32; + pub const MAX_NUM_SPLITS = 32; pub const Variant = enum { decode, diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 7ede3a71..603dbd1b 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -36,7 +36,7 @@ pub const Options = union(Backend) { is_prefill: bool, batch_size: u32, seq_len: u32, - page_chunk_size: u32, + max_num_pages: u32, max_token_count: u32, num_heads: u32, num_kv_heads: u32, @@ -45,14 +45,13 @@ pub const Options = union(Backend) { }; pub fn fromBackend(args: Args) Options { - const max_num_pages = std.math.divCeil(u32, args.seq_len, args.page_chunk_size) catch unreachable; return switch (args.backend) { .cuda_fa2 => if (args.is_prefill) .{ .cuda_fa2 = .{ .mixed = .{ .batch_size_decode = args.batch_size, .batch_size_prefill = args.batch_size, - .max_num_pages = max_num_pages, + .max_num_pages = args.max_num_pages, .max_seqlen_k = args.seq_len, .max_seqlen_q = args.max_seqlen_q, .max_token_count = args.max_token_count, @@ -65,7 +64,7 @@ pub const Options = union(Backend) { .cuda_fa2 = .{ .decode = .{ .batch_size = args.batch_size, - .max_num_pages = max_num_pages, + .max_num_pages = args.max_num_pages, .max_seqlen_k = args.seq_len, .max_token_count = args.max_token_count, .num_heads = args.num_heads, @@ -79,7 +78,7 @@ pub const Options = union(Backend) { .mixed = .{ .batch_size_decode = args.batch_size, .batch_size_prefill = args.batch_size, - .max_num_pages = max_num_pages, + .max_num_pages = args.max_num_pages, .max_seqlen_k = args.seq_len, .max_seqlen_q = args.max_seqlen_q, .max_token_count = args.max_token_count, @@ -92,7 +91,7 @@ pub const Options = union(Backend) { .cuda_fa3 = .{ .decode = .{ .batch_size = args.batch_size, - .max_num_pages = max_num_pages, + .max_num_pages = args.max_num_pages, .max_seqlen_k = args.seq_len, .max_token_count = args.max_token_count, .num_heads = args.num_heads, @@ -104,7 +103,7 @@ pub const Options = union(Backend) { .triton => .{ .triton = .{ .batch_size = args.batch_size, - .max_num_pages = max_num_pages, + .max_num_pages = args.max_num_pages, .max_seqlen_q = args.max_seqlen_q, .is_prefill = args.is_prefill, }, @@ -113,7 +112,7 @@ pub const Options = union(Backend) { .mosaic_tpu = .{ .is_prefill = args.is_prefill, .batch_size = args.batch_size, - .max_num_pages = max_num_pages, + .max_num_pages = args.max_num_pages, .max_seqlen_k = args.seq_len, .max_token_count = args.max_token_count, .num_heads = args.num_heads, @@ -220,35 +219,6 @@ pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: z }; } -test "Options.fromBackend configures mosaic_tpu ragged backend" { - const options = Options.fromBackend(.{ - .backend = .mosaic_tpu, - .is_prefill = false, - .batch_size = 4, - .seq_len = 512, - .page_chunk_size = 16, - .max_token_count = 4, - .num_heads = 16, - .num_kv_heads = 4, - .head_dim = 128, - .max_seqlen_q = 1, - }); - - switch (options) { - .mosaic_tpu => |tpu_option| { - try std.testing.expect(!tpu_option.is_prefill); - try std.testing.expectEqual(@as(usize, 4), tpu_option.batch_size); - try std.testing.expectEqual(@as(usize, 32), tpu_option.max_num_pages); - try std.testing.expectEqual(@as(usize, 512), tpu_option.max_seqlen_k); - try std.testing.expectEqual(@as(usize, 4), tpu_option.max_token_count); - try std.testing.expectEqual(@as(usize, 16), tpu_option.num_heads); - try std.testing.expectEqual(@as(usize, 4), tpu_option.num_kv_heads); - try std.testing.expectEqual(@as(usize, 128), tpu_option.head_dim); - }, - else => return error.UnexpectedBackendVariant, - } -} - test "Backend.auto selects mosaic_tpu on TPU" { const platform: zml.Platform = .{ .arena = undefined, From cebc4cebf8a18dcf87b0acdfbbc6f9f5bea3e66e Mon Sep 17 00:00:00 2001 From: elogir Date: Mon, 25 May 2026 09:41:08 +0200 Subject: [PATCH 496/881] kernels: extend Mosaic TPU + Triton kernel DSL (#545) This PR changes the following: - Name-based `output_operand_aliases` for MosaicTPU and Triton DSLs - New `kernels/mosaic_tpu/pipeline.zig`, a Zig port of `pltpu.emit_pipeline` - Some fixes to MosaicTPU --- kernels/mosaic_tpu/BUILD.bazel | 1 + kernels/mosaic_tpu/builder.zig | 42 +- kernels/mosaic_tpu/pipeline.zig | 872 ++++++++++++++++++++++++ mlir/dialects/mosaic_tpu/mosaic_tpu.zig | 16 +- mlir/dialects/stablehlo/stablehlo.zig | 7 +- mlir/dialects/vector.zig | 18 +- mlir/mlir.zig | 5 + zml/kernel.zig | 104 ++- zml/moe/triton.zig | 16 +- zml/ops.zig | 2 +- 10 files changed, 1032 insertions(+), 51 deletions(-) create mode 100644 kernels/mosaic_tpu/pipeline.zig diff --git a/kernels/mosaic_tpu/BUILD.bazel b/kernels/mosaic_tpu/BUILD.bazel index 3506ede5..651d9826 100644 --- a/kernels/mosaic_tpu/BUILD.bazel +++ b/kernels/mosaic_tpu/BUILD.bazel @@ -5,6 +5,7 @@ zig_library( tags = ["manual"], srcs = [ "dtype.zig", + "pipeline.zig", ], import_name = "kernels/mosaic_tpu/builder", main = "builder.zig", diff --git a/kernels/mosaic_tpu/builder.zig b/kernels/mosaic_tpu/builder.zig index b6a0d4f4..adb2f4e4 100644 --- a/kernels/mosaic_tpu/builder.zig +++ b/kernels/mosaic_tpu/builder.zig @@ -39,6 +39,8 @@ const isFloatDtype = dtypes.isFloatDtype; const dtypeBitwidth = dtypes.dtypeBitwidth; const intBitwidth = dtypes.intBitwidth; +pub const pipeline = @import("pipeline.zig"); + // ============================================================================= // Value — a handle to an MLIR value inside the kernel body. // ============================================================================= @@ -216,6 +218,44 @@ pub const Value = struct { pub fn abs(self: Value) Value { return self.kern().abs(self); } + + /// Best-effort fold to an `i64` for `arith.constant` (and `addi`/`muli` of folded + /// operands). Returns `null` for anything else; the caller decides what to do. + /// TODO(raph): check if there is no better way + pub fn asConstantInt(self: Value) ?i64 { + if (self.inner.isA(mlir.OpResult) == null) return null; + + const op = self.inner.owner(); + const op_name = op.name(); + + if (std.mem.eql(u8, op_name, "arith.constant")) { + const attr = op.attributeByName("value") orelse return null; + const iattr = attr.isA(mlir.IntegerAttribute) orelse return null; + return iattr.value(i64); + } + + if (std.mem.eql(u8, op_name, "arith.muli")) { + const lhs: Value = .{ .inner = op.operand(0), .kernel = self.kernel }; + const rhs: Value = .{ .inner = op.operand(1), .kernel = self.kernel }; + const lv = lhs.asConstantInt(); + const rv = rhs.asConstantInt(); + if (lv) |l| if (rv) |r| return l *% r; + // One side null, the other known: still folds if the known side is 0. + if (lv) |l| if (l == 0) return 0; + if (rv) |r| if (r == 0) return 0; + return null; + } + + if (std.mem.eql(u8, op_name, "arith.addi")) { + const lhs: Value = .{ .inner = op.operand(0), .kernel = self.kernel }; + const rhs: Value = .{ .inner = op.operand(1), .kernel = self.kernel }; + const lv = lhs.asConstantInt() orelse return null; + const rv = rhs.asConstantInt() orelse return null; + return lv +% rv; + } + + return null; + } }; // ============================================================================= @@ -1644,7 +1684,7 @@ pub const Builder = struct { return self.emit(tpu.concatenate(self.ctx, self.innerSlice(sources), dimension, ty, self.loc())); } - pub fn transpose(self: *Builder, src: Value, permutation: []const i32) Value { + pub fn transpose(self: *Builder, src: Value, permutation: []const i64) Value { const in_shape = src.shape(); std.debug.assert(permutation.len == in_shape.len); var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; diff --git a/kernels/mosaic_tpu/pipeline.zig b/kernels/mosaic_tpu/pipeline.zig new file mode 100644 index 00000000..da7644c8 --- /dev/null +++ b/kernels/mosaic_tpu/pipeline.zig @@ -0,0 +1,872 @@ +//! Zig port of `pltpu.emit_pipeline` (jax/_src/pallas/mosaic/pipeline.py) — +//! emits lowered `tpu`-dialect IR byte-equivalent to `pallas_call`. +//! +//! 1-D dynamic-bound grid; VMEM operands; `buffer_count ∈ {1,2}`; +//! `.blocked`/`.squeezed`/one `.bounded_slice` block dim; trivial-windowing +//! inputs peeled as sync copy; `trace_scopes`. Unsupported features panic. + +const std = @import("std"); +const mlir = @import("mlir"); +const dialects = @import("mlir/dialects"); +const builder = @import("builder.zig"); + +const tpu = @import("mlir/dialects/mosaic_tpu"); +const arith = dialects.arith; +const scf = dialects.scf; +const memref = dialects.memref; + +const Builder = builder.Builder; +const Value = builder.Value; +const DType = builder.DType; +const MemorySpace = builder.MemorySpace; +const FinishError = builder.FinishError; + +/// `ShapedType::kDynamic` — sentinel for a `?` dim in a memref shape. +const kDyn: i64 = std.math.minInt(i64); + +/// One entry of `pl.BlockSpec.block_shape`. +pub const BlockDim = union(enum) { + /// `pl.Blocked(n)` — windowed dim of size `n`. + blocked: i64, + /// `pl.Squeezed()` / `None` — dim dropped in the window. + squeezed, + /// `pl.BoundedSlice(n)` — index_map returns `.slice`; `n` is the max. + bounded_slice: i64, +}; + +/// One entry returned by `pl.BlockSpec.index_map`. +pub const BlockIndex = union(enum) { + /// `pl.ds(idx*block, block)` — for `.blocked`/`.squeezed` dims. + scalar: Value, + /// `pl.ds(start, size)` — for `.bounded_slice` dims. + slice: struct { start: Value, size: Value }, +}; + +/// `pl.BlockSpec.index_map`. `ctx` is opaque user data (Zig has no closures). +pub const IndexMapFn = *const fn (b: *Builder, indices: []const Value, ctx: ?*anyopaque) []const BlockIndex; + +/// Working window for one pipeline operand. Body must re-emit +/// `memref_slice`/`squeeze` at each use site. `slot == null` ⇒ trivial-windowing; +/// `buf == null` ⇒ no operand. +pub const RefWindow = struct { buf: ?Value = null, slot: ?Value = null }; + +/// The pipeline body — `pallas_call`'s inner kernel. +pub const BodyFn = *const fn (b: *Builder, grid_indices: []const Value, refs: []const RefWindow, scratches: []const Value, ctx: ?*anyopaque) FinishError!void; + +/// `pl.BlockSpec` for one pipeline operand. `null` slot ↔ `None` spec. +pub const PipeSpec = struct { + full_shape: []const i64, + dtype: DType, + /// `null` ⇒ trivial windowing (the whole array). + block_shape: ?[]const BlockDim = null, + /// `null` only allowed with trivial windowing. + index_map: ?IndexMapFn = null, + index_map_ctx: ?*anyopaque = null, + /// Only `.vmem` supported. + memory_space: MemorySpace = .vmem, + /// `null` ⇒ 2 (or 1 if trivial). + buffer_count: ?usize = null, + /// Tile that `_create_bounded_slice` rounds the DMA size up to. + bounded_slice_tiling: i32 = 1, + /// Pallas's `disable_bounds_checks`: skip OOB clamp, emit only `tpu.assume_multiple`. + bounded_slice_skip_clamp: bool = false, +}; + +pub const PipelineOpts = struct { + body: BodyFn, + body_ctx: ?*anyopaque = null, + /// Dynamic (`i32`) grid bounds. Comptime: pass `b.lift(@as(i32, N))`. + grid: []const Value, + /// One per HBM input ref (slot `null` <-> `None` spec). + in_specs: []const ?PipeSpec, + out_specs: []const ?PipeSpec, + trace_scopes: bool = true, +}; + +/// Emits `pltpu.emit_pipeline(body, grid=..., in_specs=..., out_specs=...)(*refs, scratches=...)`. +/// `refs` = `in_refs ++ out_refs`. Emits into `b.currentBlock()`. +pub fn emitPipeline( + b: *Builder, + refs: []const ?Value, + scratches: []const Value, + opts: PipelineOpts, +) FinishError!void { + std.debug.assert(opts.grid.len >= 1); + if (opts.grid.len != 1) @panic("emitPipeline: only 1-D grids supported (TODO: N-D `_next_index`)"); + const n_in = opts.in_specs.len; + std.debug.assert(refs.len == n_in + opts.out_specs.len); + + const a = b.arena.allocator(); + + var brefs: std.ArrayList(BufferedRef) = try .initCapacity(a, opts.in_specs.len + opts.out_specs.len); + for (opts.in_specs, 0..) |maybe_spec, i| { + if (maybe_spec) |spec| { + std.debug.assert(refs[i] != null); + brefs.appendAssumeCapacity(BufferedRef.create(b, spec, .input, refs[i].?, i)); + } + } + for (opts.out_specs, 0..) |maybe_spec, j| { + if (maybe_spec) |spec| { + std.debug.assert(refs[n_in + j] != null); + brefs.appendAssumeCapacity(BufferedRef.create(b, spec, .output, refs[n_in + j].?, n_in + j)); + } + } + + // Whole pipeline lives in a `tpu.region`. + const region_block = mlir.Block.init(&.{}, &.{}); + b.pushBlock(region_block); + + for (brefs.items) |*br| br.allocate(b); + + const num_steps = opts.grid[0]; + const c0_i32 = b.lift(@as(i32, 0)); + const c1_i32 = b.lift(@as(i32, 1)); + { + var when = b.openIf(b.cmpi(.sgt, num_steps, c0_i32)); + { + for (brefs.items) |*br| if (br.is_trivial and br.buffer_type == .input) br.syncCopyIn(b); + + const num_stages = blk: { + var m: usize = 2; + for (brefs.items) |br| if (br.buffer_count > m) { + m = br.buffer_count; + }; + break :blk m; + }; + var sched0 = Scheduler.init(b, c0_i32, opts.grid, num_steps, num_stages, opts.trace_scopes); + for (0..num_stages - 1) |step| { + if (opts.trace_scopes) { + var buf: [32]u8 = undefined; + b.traceStart(10, std.fmt.bufPrint(&buf, "ep_initialize_{d}", .{step}) catch unreachable); + } + for (brefs.items) |*br| sched0.initializeStep(b, br, step); + if (opts.trace_scopes) b.traceStop(); + } + + // iter_args: each bref's slots in spec order (see appendSlotInits), then grid_idx. + var inits: std.ArrayList(*const mlir.Value) = try .initCapacity(a, brefs.items.len * 2 + 1); + for (brefs.items) |*br| br.appendSlotInits(b, &inits); + inits.appendAssumeCapacity(c0_i32.inner); + const n_slots = inits.items.len - 1; + + const i32_ty = b.scalarTy(.i32); + const block_types = a.alloc(*const mlir.Type, inits.items.len + 1) catch unreachable; + const block_locs = a.alloc(*const mlir.Location, inits.items.len + 1) catch unreachable; + for (block_types) |*t| t.* = i32_ty; + for (block_locs) |*l| l.* = b.loc(); + const for_body = mlir.Block.init(block_types, block_locs); + b.pushBlock(for_body); + + const iv: Value = .{ .inner = for_body.argument(0), .kernel = b }; + { + var cur: usize = 1; + for (brefs.items) |*br| cur = br.bindSlotsAt(b, for_body, cur); + std.debug.assert(cur == n_slots + 1); + } + const grid_idx: Value = .{ .inner = for_body.argument(@intCast(n_slots + 1)), .kernel = b }; + + var sched = Scheduler.init(b, iv, opts.grid, num_steps, num_stages, opts.trace_scopes); + sched.indices_storage[0] = grid_idx; + sched.recomputeDerivedIndices(b); + + for (brefs.items) |*br| if (br.buffer_type == .input) sched.copyIn(b, br); + for (brefs.items) |*br| if (br.buffer_type == .input) sched.waitIn(b, br); + + // Emit slot `rem` ops BEFORE the body; body re-emits slice/squeeze per use. + var current_refs = a.alloc(RefWindow, refs.len) catch unreachable; + for (current_refs) |*r| r.* = .{}; + for (brefs.items) |*br| { + if (br.is_trivial or !br.isBuffered()) { + current_refs[br.spec_index] = .{ .buf = br.window_ref }; + } else { + const active = if (br.buffer_type == .output) br.copy_out_slot.? else br.wait_in_slot.?; + current_refs[br.spec_index] = .{ .buf = br.window_ref, .slot = b.remui(active, b.lift(@as(i32, @intCast(br.buffer_count)))) }; + } + } + + if (opts.trace_scopes) b.traceStart(10, "ep_run_kernel"); + try opts.body(b, sched.indices_storage[0..opts.grid.len], current_refs, scratches, opts.body_ctx); + if (opts.trace_scopes) b.traceStop(); + + for (brefs.items) |*br| if (br.buffer_type == .output) sched.copyOut(b, br); + for (brefs.items) |*br| if (br.buffer_type == .output) sched.waitOut(b, br); + + for (brefs.items) |*br| if (br.buffer_type == .input) sched.advanceSlots(b, br); + + var yields: std.ArrayList(*const mlir.Value) = try .initCapacity(a, brefs.items.len * 2 + 1); + for (brefs.items) |*br| br.appendSlotYields(b, &yields); + const next_idx = nextIndex1D(b, grid_idx, num_steps); + yields.appendAssumeCapacity(next_idx.inner); + _ = scf.yield(b.ctx, yields.items, b.loc()).appendTo(for_body); + b.popBlock(); + + const for_op = scf.for_(b.ctx, c0_i32.inner, num_steps.inner, c1_i32.inner, inits.items, for_body, .{}, b.loc()); + _ = for_op.appendTo(b.currentBlock()); + var k: usize = 0; + for (brefs.items) |*br| k = br.readbackSlots(b, for_op, k); + const final_grid_idx: Value = .{ .inner = for_op.result(@intCast(n_slots)), .kernel = b }; + + // prev_index BEFORE Scheduler.init so its inner `subi(num_steps,1)` CSEs with `step`. + const final_idx = prevIndex1D(b, final_grid_idx, num_steps); + var sched_fin = Scheduler.init(b, b.subi(num_steps, c1_i32), opts.grid, num_steps, num_stages, opts.trace_scopes); + sched_fin.indices_storage[0] = final_idx; + sched_fin.recomputeDerivedIndices(b); + for (brefs.items) |*br| sched_fin.finalize(b, br); + for (brefs.items) |*br| if (br.is_trivial and br.buffer_type == .output) br.syncCopyOut(b); + + when.yieldThen(.{}); + } + } + + _ = tpu.yield(b.ctx, &.{}, b.loc()).appendTo(region_block); + b.popBlock(); + _ = tpu.region(b.ctx, region_block, &.{}, b.loc()).appendTo(b.currentBlock()); +} + +const BufferType = enum { input, output }; + +/// VMEM double-buffer + DMA semaphores for one pipeline operand. +const BufferedRef = struct { + spec: PipeSpec, + buffer_type: BufferType, + buffer_count: usize, + is_trivial: bool, + /// Index into `refs` / `current_refs`. + spec_index: usize, + src: Value, + /// Block shape with `.squeezed` dims removed. + block_compact: []const i64, + /// Indices into `spec.block_shape` parallel to `block_compact`. + block_keep: []const usize, + + /// `memref` (or full shape if trivial). + window_ref: Value = undefined, + sem_recvs: ?Value = null, + sem_sends: ?Value = null, + + // Slot iter_arg state — see appendSlotInits for init values. + copy_in_slot: ?Value = null, + wait_in_slot: ?Value = null, + copy_out_slot: ?Value = null, + wait_out_slot: ?Value = null, + slot: ?Value = null, + + fn create(b: *Builder, spec: PipeSpec, bt: BufferType, src: Value, spec_index: usize) BufferedRef { + const a = b.arena.allocator(); + if (spec.memory_space != .vmem) @panic("emitPipeline: only VMEM operands supported"); + const trivial = specHasTrivialWindowing(spec); + var bc: usize = if (spec.buffer_count) |c| c else 2; + if (spec.buffer_count == null and trivial) bc = 1; + if (bt == .output and bc > 2) @panic("emitPipeline: output buffer_count > 2 unsupported"); + + const block_len = if (spec.block_shape) |bs| bs.len else 0; + var keep = std.ArrayList(usize).initCapacity(a, block_len) catch unreachable; + var compact = std.ArrayList(i64).initCapacity(a, block_len) catch unreachable; + if (spec.block_shape) |bs| { + for (bs, 0..) |bd, di| switch (bd) { + .blocked => |n| { + keep.appendAssumeCapacity(di); + compact.appendAssumeCapacity(n); + }, + .bounded_slice => |n| { + keep.appendAssumeCapacity(di); + compact.appendAssumeCapacity(n); + }, + .squeezed => {}, + }; + } + return .{ + .spec = spec, + .buffer_type = bt, + .buffer_count = bc, + .is_trivial = trivial, + .spec_index = spec_index, + .src = src, + .block_compact = compact.items, + .block_keep = keep.items, + }; + } + + fn isBuffered(self: *const BufferedRef) bool { + return self.buffer_count > 0; + } + + fn allocate(self: *BufferedRef, b: *Builder) void { + const a = b.arena.allocator(); + const vmem = MemorySpace.vmem.attribute(b.ctx); + const elem = self.spec.dtype.toMlir(b.ctx); + if (self.is_trivial) { + const ty = mlir.Type.memRef(elem, self.spec.full_shape, null, vmem); + self.window_ref = memRefAlloca(b, ty); + } else { + if (self.block_compact.len == 1) @panic("emitPipeline: 1-D flat block buffers unsupported (TODO)"); + var shape = a.alloc(i64, self.block_compact.len + 1) catch unreachable; + shape[0] = @intCast(self.buffer_count); + @memcpy(shape[1..], self.block_compact); + const ty = mlir.Type.memRef(elem, shape, null, vmem); + self.window_ref = memRefAlloca(b, ty); + } + if (!self.is_trivial) { + const sem_arr_ty = dmaSemRefTy(b, self.buffer_count); + const sem = semAllocOp(b, sem_arr_ty); + if (self.buffer_type == .input) self.sem_recvs = sem else self.sem_sends = sem; + } + } + + fn appendSlotInits(self: *BufferedRef, b: *Builder, out: *std.ArrayList(*const mlir.Value)) void { + const c0 = b.lift(@as(i32, 0)).inner; + const c1 = b.lift(@as(i32, 1)).inner; + switch (self.buffer_type) { + .input => { + if (self.is_trivial) { + out.appendAssumeCapacity(c0); + } else { + out.appendAssumeCapacity(c1); // copy_in_slot: initialize_step advanced 0→1 + out.appendAssumeCapacity(c0); + } + }, + .output => { + out.appendAssumeCapacity(c0); + out.appendAssumeCapacity(c0); + }, + } + } + + fn nSlots(self: *const BufferedRef) usize { + return switch (self.buffer_type) { + .input => if (self.is_trivial) 1 else 2, + .output => 2, + }; + } + + fn bindSlotsAt(self: *BufferedRef, b: *Builder, body: *mlir.Block, start: usize) usize { + var k = start; + const arg = struct { + fn f(b_: *Builder, body_: *mlir.Block, i: *usize) Value { + const v: Value = .{ .inner = body_.argument(@intCast(i.*)), .kernel = b_ }; + i.* += 1; + return v; + } + }.f; + switch (self.buffer_type) { + .input => { + if (self.is_trivial) { + self.slot = arg(b, body, &k); + } else { + self.copy_in_slot = arg(b, body, &k); + self.wait_in_slot = arg(b, body, &k); + } + }, + .output => { + self.copy_out_slot = arg(b, body, &k); + self.wait_out_slot = arg(b, body, &k); + }, + } + return k; + } + + fn appendSlotYields(self: *BufferedRef, b: *Builder, out: *std.ArrayList(*const mlir.Value)) void { + _ = b; + switch (self.buffer_type) { + .input => { + if (self.is_trivial) { + out.appendAssumeCapacity(self.slot.?.inner); + } else { + out.appendAssumeCapacity(self.copy_in_slot.?.inner); + out.appendAssumeCapacity(self.wait_in_slot.?.inner); + } + }, + .output => { + out.appendAssumeCapacity(self.copy_out_slot.?.inner); + out.appendAssumeCapacity(self.wait_out_slot.?.inner); + }, + } + } + + fn readbackSlots(self: *BufferedRef, b: *Builder, for_op: *mlir.Operation, start: usize) usize { + var k = start; + const take = struct { + fn f(b_: *Builder, op: *mlir.Operation, i: *usize) Value { + const v: Value = .{ .inner = op.result(@intCast(i.*)), .kernel = b_ }; + i.* += 1; + return v; + } + }.f; + switch (self.buffer_type) { + .input => { + if (self.is_trivial) { + self.slot = take(b, for_op, &k); + } else { + self.copy_in_slot = take(b, for_op, &k); + self.wait_in_slot = take(b, for_op, &k); + } + }, + .output => { + self.copy_out_slot = take(b, for_op, &k); + self.wait_out_slot = take(b, for_op, &k); + }, + } + return k; + } + + fn computeIndex(self: *const BufferedRef, b: *Builder, indices: []const Value) []const BlockIndex { + if (self.spec.index_map) |im| return im(b, indices, self.spec.index_map_ctx); + @panic("emitPipeline: index_map required for non-trivial windowing"); + } + + /// Per `block_shape` dim: `{base, size?}`. `size == null` ⇒ static-full. + /// `static_zero_start` enables prologue shortcut (Python-int 0 start folds). + const DmaSlice = struct { base: []Value, size: []?Value }; + fn getDmaSlice(self: *const BufferedRef, b: *Builder, indices: []const Value, static_zero_start: bool) DmaSlice { + const a = b.arena.allocator(); + const bs = self.spec.block_shape.?; + const bidx = self.computeIndex(b, indices); + std.debug.assert(bidx.len == bs.len); + var base = a.alloc(Value, bs.len) catch unreachable; + var size = a.alloc(?Value, bs.len) catch unreachable; + for (bs, 0..) |bd, d| { + switch (bd) { + .squeezed => { + base[d] = switch (bidx[d]) { + .scalar => |v| v, + .slice => @panic("emitPipeline: squeezed dim must get a scalar index"), + }; + size[d] = null; + }, + .blocked => |n| { + const idx = switch (bidx[d]) { + .scalar => |v| v, + .slice => @panic("emitPipeline: blocked dim must get a scalar index"), + }; + if (@mod(self.spec.full_shape[d], n) != 0) @panic("emitPipeline: non-dividing blocked dim unsupported"); + base[d] = if (n == 1) idx else idx.mul(@as(i32, @intCast(n))); + size[d] = null; + }, + .bounded_slice => { + const sl = switch (bidx[d]) { + .scalar => @panic("emitPipeline: bounded_slice dim must get a `.slice` index from index_map"), + .slice => |s| s, + }; + const tiling: i32 = self.spec.bounded_slice_tiling; + const dim_size = self.spec.full_shape[d]; + base[d] = sl.start; + if (self.spec.bounded_slice_skip_clamp) { + size[d] = b.emit(tpu.assume_multiple(b.ctx, sl.size.inner, tiling, b.loc())); + continue; + } + // Prologue shortcut: Python-int 0 start ⇒ `_round_up_to_nearest_multiple` folds. + if (static_zero_start and sl.start.asConstantInt() == @as(?i64, 0)) { + const is_oob = b.cmpi(.sgt, sl.size, b.lift(@as(i32, @intCast(dim_size)))); + const sz = b.select(is_oob, b.lift(@as(i32, @intCast(dim_size))), sl.size); + size[d] = b.emit(tpu.assume_multiple(b.ctx, sz.inner, tiling, b.loc())); + continue; + } + const is_oob = b.cmpi(.sgt, b.addi(sl.start, sl.size), b.lift(@as(i32, @intCast(dim_size)))); + const rounded = blk: { + if (tiling == 1) break :blk b.subi(b.lift(@as(i32, @intCast(dim_size + 1))), sl.start); + // lax sign-fixup for `jnp.remainder`. + const m = b.lift(tiling); + const c0 = b.lift(@as(i32, 0)); + const rem_arg = b.subi(b.lift(@as(i32, @intCast(dim_size))), sl.start); + const r = b.remsi(rem_arg, m); + const r_nz = b.cmpi(.ne, r, c0); + const r_neg = b.cmpi(.slt, r, c0); + const fixup = b.andi(r_neg, r_nz); + const r_plus = b.addi(r, m); + const floormod = b.select(fixup, r_plus, r); + const round_down = b.subi(rem_arg, floormod); + break :blk b.addi(round_down, m); + }; + const sz = b.select(is_oob, rounded, sl.size); + size[d] = b.emit(tpu.assume_multiple(b.ctx, sz.inner, tiling, b.loc())); + }, + } + } + return .{ .base = base, .size = size }; + } + + fn sliceSrc(self: *const BufferedRef, b: *Builder, dma: DmaSlice) Value { + const a = b.arena.allocator(); + const bs = self.spec.block_shape.?; + var base = a.alloc(Value, bs.len) catch unreachable; + var dyn = std.ArrayList(Value).initCapacity(a, bs.len) catch unreachable; + var result_shape = a.alloc(i64, bs.len) catch unreachable; + for (bs, 0..) |bd, d| { + base[d] = dma.base[d]; + if (dma.size[d]) |sz| { + result_shape[d] = kDyn; + dyn.appendAssumeCapacity(sz); + } else { + result_shape[d] = switch (bd) { + .squeezed => 1, + .blocked => |n| n, + .bounded_slice => unreachable, + }; + } + } + return b.memRefSlice(self.src, base, result_shape, dyn.items); + } + + /// Slices `alloca[slot, 0…]` (non-squeezed dims) then squeezes the slot dim. + fn sliceWindowAt(self: *const BufferedRef, b: *Builder, slot: Value, dma: DmaSlice) Value { + const a = b.arena.allocator(); + const bs = self.spec.block_shape.?; + const c0 = b.lift(@as(i32, 0)); + const nd = 1 + self.block_compact.len; + var base = a.alloc(Value, nd) catch unreachable; + var result_shape = a.alloc(i64, nd) catch unreachable; + var dyn = std.ArrayList(Value).initCapacity(a, self.block_keep.len) catch unreachable; + base[0] = slot; + result_shape[0] = 1; + for (self.block_keep, 0..) |sd, ci| { + base[ci + 1] = c0; + if (dma.size[sd]) |sz| { + result_shape[ci + 1] = kDyn; + dyn.appendAssumeCapacity(sz); + } else { + result_shape[ci + 1] = switch (bs[sd]) { + .blocked => |n| n, + else => unreachable, + }; + } + } + const sliced = b.memRefSlice(self.window_ref, base, result_shape, dyn.items); + const squeezed_shape = a.alloc(i64, self.block_compact.len) catch unreachable; + @memcpy(squeezed_shape, result_shape[1..]); + return b.memRefSqueeze(sliced, squeezed_shape); + } + + fn semAt(b: *Builder, sem_array: Value, i: Value) Value { + const sliced = b.memRefSlice(sem_array, &.{i}, &.{1}, &.{}); + return b.memRefSqueeze(sliced, &.{}); + } + + fn currentRef(self: *BufferedRef, b: *Builder) Value { + if (self.is_trivial or !self.isBuffered()) return self.window_ref; + const slot = switch (self.buffer_type) { + .output => b.remui(self.copy_out_slot.?, b.lift(@as(i32, @intCast(self.buffer_count)))), + .input => b.remui(self.wait_in_slot.?, b.lift(@as(i32, @intCast(self.buffer_count)))), + }; + const a = b.arena.allocator(); + const nd = 1 + self.block_compact.len; + var base = a.alloc(Value, nd) catch unreachable; + var result_shape = a.alloc(i64, nd) catch unreachable; + const c0 = b.lift(@as(i32, 0)); + base[0] = slot; + result_shape[0] = 1; + for (self.block_compact, 0..) |n, ci| { + base[ci + 1] = c0; + result_shape[ci + 1] = n; + } + const sliced = b.memRefSlice(self.window_ref, base, result_shape, &.{}); + return b.memRefSqueeze(sliced, self.block_compact); + } + + fn syncCopyIn(self: *BufferedRef, b: *Builder) void { + const blk = mlir.Block.init(&.{}, &.{}); + b.pushBlock(blk); + const sem = semAllocOp(b, dmaSemRefTy(b, null)); + b.enqueueDma(self.src, self.window_ref, sem, .{}); + b.waitDma2(sem, self.src, self.window_ref, .{}); + _ = tpu.yield(b.ctx, &.{}, b.loc()).appendTo(blk); + b.popBlock(); + _ = tpu.region(b.ctx, blk, &.{}, b.loc()).appendTo(b.currentBlock()); + } + + fn syncCopyOut(self: *BufferedRef, b: *Builder) void { + const blk = mlir.Block.init(&.{}, &.{}); + b.pushBlock(blk); + const sem = semAllocOp(b, dmaSemRefTy(b, null)); + b.enqueueDma(self.window_ref, self.src, sem, .{}); + b.waitDma2(sem, self.window_ref, self.src, .{}); + _ = tpu.yield(b.ctx, &.{}, b.loc()).appendTo(blk); + b.popBlock(); + _ = tpu.region(b.ctx, blk, &.{}, b.loc()).appendTo(b.currentBlock()); + } + + // Op order mirrors pipeline.py: copy_in/copy_out/wait_out take slot rem + // before get_dma_slice; wait_in does it after. + + fn copyIn(self: *BufferedRef, b: *Builder, indices: []const Value, slot_cumulative: Value, static_zero_start: bool) void { + const slot = b.remui(slot_cumulative, b.lift(@as(i32, @intCast(self.buffer_count)))); + const dma = self.getDmaSlice(b, indices, static_zero_start); + const src_sl = self.sliceSrc(b, dma); + const dst_sl = self.sliceWindowAt(b, slot, dma); + const sem = semAt(b, self.sem_recvs.?, slot); + b.enqueueDma(src_sl, dst_sl, sem, .{}); + } + + fn waitIn(self: *BufferedRef, b: *Builder, indices: []const Value) void { + const dma = self.getDmaSlice(b, indices, false); + const slot = b.remui(self.wait_in_slot.?, b.lift(@as(i32, @intCast(self.buffer_count)))); + const src_sl = self.sliceSrc(b, dma); + const dst_sl = self.sliceWindowAt(b, slot, dma); + const sem = semAt(b, self.sem_recvs.?, slot); + b.waitDma2(sem, src_sl, dst_sl, .{}); + } + + fn copyOut(self: *BufferedRef, b: *Builder, indices: []const Value) void { + if (self.buffer_count == 1) @panic("emitPipeline: single-buffered output sync_copy_out unsupported (TODO)"); + const slot = b.remui(self.copy_out_slot.?, b.lift(@as(i32, @intCast(self.buffer_count)))); + const dma = self.getDmaSlice(b, indices, false); + // Pallas evaluates `src` (VMEM window) before `dst` (HBM). + const src_sl = self.sliceWindowAt(b, slot, dma); + const dst_sl = self.sliceSrc(b, dma); + const sem = semAt(b, self.sem_sends.?, slot); + b.enqueueDma(src_sl, dst_sl, sem, .{}); + } + + fn waitOut(self: *BufferedRef, b: *Builder, indices: []const Value) void { + if (self.buffer_count <= 1) return; + const slot = b.remui(self.wait_out_slot.?, b.lift(@as(i32, @intCast(self.buffer_count)))); + const dma = self.getDmaSlice(b, indices, false); + const src_sl = self.sliceWindowAt(b, slot, dma); + const dst_sl = self.sliceSrc(b, dma); + const sem = semAt(b, self.sem_sends.?, slot); + b.waitDma2(sem, src_sl, dst_sl, .{}); + } +}; + +/// Prologue/loop/epilogue copy+wait schedule. +const Scheduler = struct { + step: Value, + grid: []const Value, + num_steps: Value, + num_stages: usize, + trace_scopes: bool, + first_step: Value, + last_step: Value, + indices_storage: [4]Value = undefined, + prev_storage: [4]Value = undefined, + next_storage: [4]Value = undefined, + indices_len: usize, + + fn init(b: *Builder, step: Value, grid: []const Value, num_steps: Value, num_stages: usize, trace_scopes: bool) Scheduler { + const c0 = b.lift(@as(i32, 0)); + const c1 = b.lift(@as(i32, 1)); + var s: Scheduler = .{ + .step = step, + .grid = grid, + .num_steps = num_steps, + .num_stages = num_stages, + .trace_scopes = trace_scopes, + .first_step = b.cmpi(.eq, step, c0), + .last_step = b.cmpi(.eq, step, b.subi(num_steps, c1)), + .indices_len = grid.len, + }; + for (0..grid.len) |i| s.indices_storage[i] = c0; + s.recomputeDerivedIndices(b); + return s; + } + + fn indices(self: *Scheduler) []const Value { + return self.indices_storage[0..self.indices_len]; + } + fn prevIndices(self: *Scheduler) []const Value { + return self.prev_storage[0..self.indices_len]; + } + fn nextIndices(self: *Scheduler) []const Value { + return self.next_storage[0..self.indices_len]; + } + + fn recomputeDerivedIndices(self: *Scheduler, b: *Builder) void { + const i = self.indices_storage[0]; + self.prev_storage[0] = prevIndex1D(b, i, self.num_steps); + self.next_storage[0] = nextIndex1D(b, i, self.num_steps); + } + + fn namedScope(self: *Scheduler, b: *Builder, name: []const u8) void { + if (self.trace_scopes) b.traceStart(10, name); + } + fn endScope(self: *Scheduler, b: *Builder) void { + if (self.trace_scopes) b.traceStop(); + } + + /// `~out_of_fetch` as `step < num_steps + (1 - buffer_count)`. + fn notOutOfFetch(self: *Scheduler, b: *Builder, br: *const BufferedRef) Value { + std.debug.assert(br.isBuffered()); + const off: i32 = 1 - @as(i32, @intCast(br.buffer_count)); + return b.cmpi(.slt, self.step, b.addi(self.num_steps, b.lift(off))); + } + + fn notFirstStep(self: *Scheduler, b: *Builder) Value { + return b.cmpi(.ne, self.step, b.lift(@as(i32, 0))); + } + + fn indicesDiffer(b: *Builder, br: *const BufferedRef, ai: []const Value, ci: []const Value) Value { + const xa = br.computeIndex(b, ai); + const xc = br.computeIndex(b, ci); + std.debug.assert(xa.len == xc.len and xa.len > 0); + var acc: ?Value = null; + for (xa, xc) |x, y| { + const d: Value = switch (x) { + .scalar => |xv| b.cmpi(.ne, xv, y.scalar), + .slice => |xs| b.ori(b.cmpi(.ne, xs.start, y.slice.start), b.cmpi(.ne, xs.size, y.slice.size)), + }; + acc = if (acc) |p| b.ori(p, d) else d; + } + return acc.?; + } + + fn hasChanged(self: *Scheduler, b: *Builder, br: *const BufferedRef) Value { + std.debug.assert(br.isBuffered() and !br.is_trivial); + return indicesDiffer(b, br, self.indices(), self.prevIndices()); + } + fn willChangeCurrent(self: *Scheduler, b: *Builder, br: *const BufferedRef) Value { + std.debug.assert(br.isBuffered() and !br.is_trivial); + return indicesDiffer(b, br, self.indices(), self.nextIndices()); + } + fn willChangeFetch(self: *Scheduler, b: *Builder, br: *const BufferedRef) Value { + std.debug.assert(br.isBuffered() and !br.is_trivial); + if (br.buffer_count < 2) return self.hasChanged(b, br); + std.debug.assert(br.buffer_count == 2); + return indicesDiffer(b, br, self.indices(), self.nextIndices()); + } + + fn advance(b: *Builder, slot: Value, pred: Value) Value { + return b.select(pred, b.addi(slot, b.lift(@as(i32, 1))), slot); + } + + fn initializeStep(self: *Scheduler, b: *Builder, br: *BufferedRef, step: usize) void { + if (br.buffer_type != .input or !br.isBuffered() or br.is_trivial) return; + if (step + 1 >= br.buffer_count) return; + if (step != 0) @panic("emitPipeline: num_stages>2 (multi-step initialize) unsupported (TODO)"); + // static_zero_start=true: Python-int 0 grid indices enable folds in getDmaSlice. + br.copyIn(b, self.indices(), b.lift(@as(i32, 0)), true); + } + + fn copyIn(self: *Scheduler, b: *Builder, br: *BufferedRef) void { + if (br.is_trivial) return; + var pred = b.andi(self.willChangeFetch(b, br), self.notOutOfFetch(b, br)); + if (br.isBuffered() and br.buffer_count < 2) pred = b.ori(pred, self.first_step); + if (br.buffer_type != .input) return; + { + var when = b.openIf(pred); + { + self.namedScope(b, "ep_copy_in"); + br.copyIn(b, self.nextIndices(), br.copy_in_slot.?, false); + self.endScope(b); + when.yieldThen(.{}); + } + } + br.copy_in_slot = advance(b, br.copy_in_slot.?, pred); + } + + fn waitIn(self: *Scheduler, b: *Builder, br: *BufferedRef) void { + if (br.is_trivial) return; + const pred = b.ori(self.hasChanged(b, br), self.first_step); + { + var when = b.openIf(pred); + { + self.namedScope(b, "ep_wait_in"); + if (br.buffer_type == .input) br.waitIn(b, self.indices()); + self.endScope(b); + when.yieldThen(.{}); + } + } + } + + fn copyOut(self: *Scheduler, b: *Builder, br: *BufferedRef) void { + if (br.is_trivial) return; + const pred = b.ori(self.willChangeCurrent(b, br), self.last_step); + { + var when = b.openIf(pred); + { + self.namedScope(b, "ep_copy_out"); + if (br.buffer_type == .output) br.copyOut(b, self.indices()); + self.endScope(b); + when.yieldThen(.{}); + } + } + br.copy_out_slot = advance(b, br.copy_out_slot.?, pred); + } + + fn waitOut(self: *Scheduler, b: *Builder, br: *BufferedRef) void { + if (br.is_trivial) return; + const pred = b.andi(self.hasChanged(b, br), self.notFirstStep(b)); + { + var when = b.openIf(pred); + { + self.namedScope(b, "ep_wait_out"); + // copy_out was issued the previous iteration. + if (br.buffer_type == .output) br.waitOut(b, self.prevIndices()); + self.endScope(b); + when.yieldThen(.{}); + } + } + br.wait_out_slot = advance(b, br.wait_out_slot.?, pred); + } + + fn finalize(self: *Scheduler, b: *Builder, br: *BufferedRef) void { + if (br.is_trivial or br.buffer_type != .output) return; + var when = b.openIf(self.last_step); + { + self.namedScope(b, "ep_finalize"); + br.waitOut(b, self.indices()); + self.endScope(b); + when.yieldThen(.{}); + } + } + + fn advanceSlots(self: *Scheduler, b: *Builder, br: *BufferedRef) void { + if (br.buffer_type != .input) return; + if (br.is_trivial) { + br.slot = advance(b, br.slot.?, self.last_step); + } else { + const pred = b.ori(self.willChangeCurrent(b, br), self.last_step); + br.wait_in_slot = advance(b, br.wait_in_slot.?, pred); + } + } +}; + +/// `verify = false`: `AutomaticAllocationScope` requirement is only satisfied +/// once it's inside `func.func`; full verify at `Builder.finish`. +pub fn memRefAlloca(b: *Builder, ty: *const mlir.Type) Value { + return b.emit(mlir.Operation.make(b.ctx, "memref.alloca", .{ + .results = .{ .flat = &.{ty} }, + .attributes = &.{.named(b.ctx, "operandSegmentSizes", .denseArray(b.ctx, .i32, &.{ 0, 0 }))}, + .verify = false, + .location = b.loc(), + })); +} + +pub fn semAllocOp(b: *Builder, ty: *const mlir.Type) Value { + return b.emit(mlir.Operation.make(b.ctx, "tpu.sem_alloc", .{ + .results = .{ .flat = &.{ty} }, + .verify = false, + .location = b.loc(), + })); +} + +/// `pl.SemaphoreType.DMA((n,))`, or rank-0 for `n == null`. +pub fn dmaSemRefTy(b: *Builder, n: ?usize) *const mlir.Type { + const shape: []const i64 = if (n) |m| &.{@intCast(m)} else &.{}; + return mlir.Type.memRef(tpu.dmaSemaphoreType(b.ctx), shape, null, MemorySpace.semaphore_mem.attribute(b.ctx)); +} + +fn nextIndex1D(b: *Builder, i: Value, n: Value) Value { + const inc = b.addi(i, b.lift(@as(i32, 1))); + const carry = b.cmpi(.eq, inc, n); + return b.select(carry, b.lift(@as(i32, 0)), inc); +} + +fn prevIndex1D(b: *Builder, i: Value, n: Value) Value { + const dec = b.subi(i, b.lift(@as(i32, 1))); + const borrow = b.cmpi(.eq, dec, b.lift(@as(i32, -1))); + return b.select(borrow, b.subi(n, b.lift(@as(i32, 1))), dec); +} + +/// `null` block_shape, or every dim is `.blocked(full_dim)`. +fn specHasTrivialWindowing(spec: PipeSpec) bool { + const bs = spec.block_shape orelse return true; + if (bs.len != spec.full_shape.len) return false; + for (bs, spec.full_shape) |bd, fs| switch (bd) { + .blocked => |n| if (n != fs) return false, + else => return false, + }; + return true; +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/mlir/dialects/mosaic_tpu/mosaic_tpu.zig b/mlir/dialects/mosaic_tpu/mosaic_tpu.zig index 35598957..5895a548 100644 --- a/mlir/dialects/mosaic_tpu/mosaic_tpu.zig +++ b/mlir/dialects/mosaic_tpu/mosaic_tpu.zig @@ -646,7 +646,7 @@ pub fn concatenate( pub fn transpose( ctx: *mlir.Context, src: *const mlir.Value, - permutation: []const i32, + permutation: []const i64, result_type: *const mlir.Type, location: *const mlir.Location, ) *mlir.Operation { @@ -654,7 +654,7 @@ pub fn transpose( .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "permutation", .denseArray(ctx, .i32, permutation)), + .named(ctx, "permutation", .denseArray(ctx, .i64, permutation)), }, .location = location, }); @@ -1149,14 +1149,14 @@ pub fn delay( pub fn assume_multiple( ctx: *mlir.Context, src: *const mlir.Value, - multiple: i64, + multiple: i32, location: *const mlir.Location, ) *mlir.Operation { return mlir.Operation.make(ctx, "tpu.assume_multiple", .{ .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{src.type_()} }, .attributes = &.{ - .named(ctx, "multiple", .int(ctx, .i64, multiple)), + .named(ctx, "multiple", .int(ctx, .i32, multiple)), }, .location = location, }); @@ -1166,11 +1166,11 @@ pub fn assume_multiple( // Internal helpers // ============================================================================= -/// `DenseBoolArrayAttr` is distinct from `DenseArrayAttr`; pack bools as i8. +/// `tpu.DenseBoolArrayAttr` stores one C `int` (i32) per bool, see `mlirDenseBoolArrayGet` fn denseBoolArrayAttribute(ctx: *mlir.Context, values: []const bool) *const mlir.Attribute { - var bytes: stdx.BoundedArray(i32, 64) = .empty; - for (values) |b| bytes.appendAssumeCapacity(@intFromBool(b)); - return .denseArray(ctx, .bool, bytes.constSlice()); + var ints: stdx.BoundedArray(i32, 64) = .empty; + for (values) |b| ints.appendAssumeCapacity(@intFromBool(b)); + return .denseArray(ctx, .bool, ints.constSlice()); } test { diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 25890b74..79c13b89 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -719,6 +719,9 @@ pub fn convolution( } pub const CustomCallOpts = struct { + pub const MAX_OPERANDS = 64; + pub const MAX_RESULTS = 16; + pub const ApiVersion = enum(i32) { original = 1, status_returning = 2, @@ -749,8 +752,8 @@ pub const CustomCallOpts = struct { }; pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result_types: []const *const mlir.Type, opts: CustomCallOpts, location: *const mlir.Location) *mlir.Operation { - const MAX_OPERANDS = 64; - const MAX_RESULTS = 16; + const MAX_OPERANDS = CustomCallOpts.MAX_OPERANDS; + const MAX_RESULTS = CustomCallOpts.MAX_RESULTS; const MINOR_TO_MAJOR = comptime blk: { var ret: [mlir.ShapedType.MAX_RANK]usize = undefined; for (0..ret.len) |i| { diff --git a/mlir/dialects/vector.zig b/mlir/dialects/vector.zig index 34ee5d78..d9bc904e 100644 --- a/mlir/dialects/vector.zig +++ b/mlir/dialects/vector.zig @@ -140,6 +140,14 @@ pub const StridedSliceArgs = struct { strides: []const i64, }; +/// extract/insert_strided_slice need offsets/sizes/strides as an ArrayAttr of IntegerAttr, not a DenseArrayAttr +fn intArrayAttribute(ctx: *mlir.Context, comptime T: type, values: []const T) *const mlir.Attribute { + const it: mlir.IntegerTypes = comptime @field(mlir.IntegerTypes, @typeName(T)); + var buf: stdx.BoundedArray(*const mlir.Attribute, mlir.ShapedType.MAX_RANK) = .empty; + for (values) |v| buf.appendAssumeCapacity(.int(ctx, it, v)); + return .array(ctx, buf.constSlice()); +} + pub fn extract_strided_slice( ctx: *mlir.Context, src: *const mlir.Value, @@ -151,9 +159,9 @@ pub fn extract_strided_slice( .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "offsets", .denseArray(ctx, .i64, args.offsets)), - .named(ctx, "sizes", .denseArray(ctx, .i64, args.sizes)), - .named(ctx, "strides", .denseArray(ctx, .i64, args.strides)), + .named(ctx, "offsets", intArrayAttribute(ctx, i64, args.offsets)), + .named(ctx, "sizes", intArrayAttribute(ctx, i64, args.sizes)), + .named(ctx, "strides", intArrayAttribute(ctx, i64, args.strides)), }, .location = location, }); @@ -171,8 +179,8 @@ pub fn insert_strided_slice( .operands = .{ .flat = &.{ src, dest } }, .results = .{ .flat = &.{dest.type_()} }, .attributes = &.{ - .named(ctx, "offsets", .denseArray(ctx, .i64, offsets)), - .named(ctx, "strides", .denseArray(ctx, .i64, strides)), + .named(ctx, "offsets", intArrayAttribute(ctx, i64, offsets)), + .named(ctx, "strides", intArrayAttribute(ctx, i64, strides)), }, .location = location, }); diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 86d22b11..a6efea3e 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -1175,6 +1175,11 @@ pub const Operation = opaque { c.mlirOperationSetAttributeByName(self.ptr(), stringRef(name_), attribute.ptr()); } + pub fn attributeByName(self: *const Operation, name_: []const u8) ?*const Attribute { + const attr = c.mlirOperationGetAttributeByName(self.ptr(), stringRef(name_)); + return if (attr.ptr == null) null else @ptrCast(attr.ptr); + } + pub const PrintFlags = struct { elide_large_elements_attrs: ?usize = null, debug_info: bool = false, diff --git a/zml/kernel.zig b/zml/kernel.zig index 89bbb924..9f8a98b9 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -1,5 +1,6 @@ const std = @import("std"); +const stdx = @import("stdx"); const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); const mosaic_tpu_builder = @import("kernels/mosaic_tpu/builder"); @@ -38,6 +39,24 @@ fn makeKernelContext(comptime dialects_needed: []const []const u8) std.mem.Alloc return ctx; } +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(dialects.stablehlo.CustomCallOpts.OutputOperandAlias, dialects.stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(dialects.stablehlo.CustomCallOpts.OutputOperandAlias, dialects.stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".fields, 0..) |field, output_index| { + if (@field(a, field.name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@intFromEnum(operand))) + operand_offset, + }); + } + } + + return out; +} + pub const triton = struct { pub const Builder = triton_builder.Builder; pub const Value = triton_builder.Value; @@ -90,7 +109,7 @@ pub const triton = struct { grid: [3]i32, num_stages: i32, num_warps: i32, - output_operand_aliases: @FieldType(ops.TritonOps, "output_operand_aliases") = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, debug: bool = false, }; @@ -119,6 +138,8 @@ pub const triton = struct { var outputs_arr: [spec.outputs.len]Shape = undefined; inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + const tensor_results = ops.triton(inputs_arr, outputs_arr, .{ .debug = opts.debug, .name = name, @@ -126,7 +147,7 @@ pub const triton = struct { .grid = opts.grid, .num_stages = opts.num_stages, .num_warps = opts.num_warps, - .output_operand_aliases = opts.output_operand_aliases, + .output_operand_aliases = aliases.constSlice(), }); var results: Results = undefined; @@ -183,6 +204,16 @@ pub const mosaic_tpu = struct { pub const Outputs = StructOf(spec.outputs, Shape); pub const Results = StructOf(spec.outputs, Tensor); + pub const CallExtras = struct { + vmem_limit_bytes: ?i64 = null, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + has_communication: ?bool = null, + additional_attributes: []const mlir.NamedAttribute = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + dynamic_grid_bounds: []const Tensor = &.{}, + }; + pub const CallOpts = struct { cfg: ConfigT, extras: CallExtras = .{}, @@ -207,7 +238,12 @@ pub const mosaic_tpu = struct { std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: emit failed: {}", .{ name, err }); defer cur.allocator.free(ir); - const backend_config = buildBackendConfig(cur.allocator, ir, opts.extras) catch |err| + const backend_config = buildBackendConfig(cur.allocator, ir, .{ + .vmem_limit_bytes = opts.extras.vmem_limit_bytes, + .disable_bounds_checks = opts.extras.disable_bounds_checks, + .disable_semaphore_checks = opts.extras.disable_semaphore_checks, + .has_communication = opts.extras.has_communication, + }) catch |err| std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: backend_config build failed: {}", .{ name, err }); defer cur.allocator.free(backend_config); @@ -219,7 +255,16 @@ pub const mosaic_tpu = struct { r.* = mlirx.Type.rankedTensor(cur.mlir_ctx, @field(outputs, fname)); } - const op = callTpuCustomCall(&values, &res_types, backend_config, opts.extras); + const aliases = resolveOutputOperandAliases(opts.extras.output_operand_aliases, @intCast(opts.extras.dynamic_grid_bounds.len)); + + const op = callTpuCustomCall(.{ + .inputs = &values, + .result_types = &res_types, + .backend_config = backend_config, + .aliases = aliases.constSlice(), + .dynamic_grid_bounds = opts.extras.dynamic_grid_bounds, + .additional_attributes = opts.extras.additional_attributes, + }); var results: Results = undefined; inline for (spec.outputs, 0..) |fname, i| { @@ -231,15 +276,6 @@ pub const mosaic_tpu = struct { }; } - const CallExtras = struct { - vmem_limit_bytes: ?i64 = null, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, - has_communication: ?bool = null, - additional_attributes: []const mlir.NamedAttribute = &.{}, - output_operand_aliases: []const dialects.stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, - }; - const CustomCallConfig = struct { body: []const u8, has_communication: ?bool = null, @@ -261,10 +297,17 @@ pub const mosaic_tpu = struct { scoped_memory_configs: ?[]const ScopedMemoryConfig = null, }; + const BackendConfigOpts = struct { + vmem_limit_bytes: ?i64 = null, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + has_communication: ?bool = null, + }; + fn buildBackendConfig( allocator: std.mem.Allocator, ir: [:0]const u8, - extras: CallExtras, + opts: BackendConfigOpts, ) ![]u8 { const ctx = try newContext(); defer ctx.deinit(); @@ -295,11 +338,11 @@ pub const mosaic_tpu = struct { const config = BackendConfig{ .custom_call_config = .{ .body = b64, - .has_communication = extras.has_communication, - .disable_bounds_checks = extras.disable_bounds_checks, - .disable_semaphore_checks = extras.disable_semaphore_checks, + .has_communication = opts.has_communication, + .disable_bounds_checks = opts.disable_bounds_checks, + .disable_semaphore_checks = opts.disable_semaphore_checks, }, - .scoped_memory_configs = if (extras.vmem_limit_bytes) |vmem| + .scoped_memory_configs = if (opts.vmem_limit_bytes) |vmem| &.{.{ .size = vmem }} else null, @@ -312,24 +355,33 @@ pub const mosaic_tpu = struct { return try json.toOwnedSlice(); } - fn callTpuCustomCall( + const TpuCustomCallArgs = struct { inputs: []const *const mlir.Value, result_types: []const *const mlir.Type, backend_config: []const u8, - extras: CallExtras, - ) *mlir.Operation { + aliases: []const dialects.stablehlo.CustomCallOpts.OutputOperandAlias, + dynamic_grid_bounds: []const Tensor, + additional_attributes: []const mlir.NamedAttribute, + }; + + fn callTpuCustomCall(args: TpuCustomCallArgs) *mlir.Operation { const cur = CompilationContext.current(); + + var all_inputs: stdx.BoundedArray(*const mlir.Value, dialects.stablehlo.CustomCallOpts.MAX_OPERANDS) = .empty; + for (args.dynamic_grid_bounds) |t| all_inputs.appendAssumeCapacity(t.value()); + all_inputs.appendSliceAssumeCapacity(args.inputs); + return dialects.stablehlo.custom_call( cur.mlir_ctx, - inputs, - result_types, + all_inputs.constSlice(), + args.result_types, .{ .call_target_name = "tpu_custom_call", - .backend_config = .{ .original = backend_config }, + .backend_config = .{ .original = args.backend_config }, .has_side_effect = false, .api_version = .original, - .additional_attributes = extras.additional_attributes, - .output_operand_aliases = extras.output_operand_aliases, + .additional_attributes = args.additional_attributes, + .output_operand_aliases = args.aliases, }, .unknown(cur.mlir_ctx), ).appendTo(cur.currentScope().block); diff --git a/zml/moe/triton.zig b/zml/moe/triton.zig index f951eb9f..5dfd8c96 100644 --- a/zml/moe/triton.zig +++ b/zml/moe/triton.zig @@ -381,11 +381,11 @@ fn alignBlockSize(topk_ids: Tensor, num_experts: i64, block_size_m: i64) struct .grid = .{ 2, 1, 1 }, .num_stages = 1, .num_warps = 8, - .output_operand_aliases = &.{ - .{ .output_index = 0, .operand_index = 1 }, - .{ .output_index = 1, .operand_index = 2 }, - .{ .output_index = 2, .operand_index = 3 }, - .{ .output_index = 3, .operand_index = 4 }, + .output_operand_aliases = .{ + .sorted_token_ids = .sorted_token_ids_ptr, + .expert_ids = .expert_ids_ptr, + .num_tokens_post_pad = .num_tokens_post_pad_ptr, + .cumsum = .cumsum_ptr, }, }, ); @@ -415,9 +415,9 @@ fn alignBlockSize(topk_ids: Tensor, num_experts: i64, block_size_m: i64) struct .grid = .{ @intCast(sort_grid_x), 1, 1 }, .num_stages = 1, .num_warps = 4, - .output_operand_aliases = &.{ - .{ .output_index = 0, .operand_index = 1 }, - .{ .output_index = 1, .operand_index = 2 }, + .output_operand_aliases = .{ + .sorted_token_ids = .sorted_token_ids_ptr, + .cumsum = .cumsum_ptr, }, }, ); diff --git a/zml/ops.zig b/zml/ops.zig index cc63c925..0433ca33 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1071,7 +1071,7 @@ fn customCallAdditionalAttributes(ctx: *CompilationContext, opts: CustomCallOpti return additional_attributes; } -fn customCallOutputOperandAliases( +pub fn customCallOutputOperandAliases( comptime I: type, comptime O: type, comptime aliases: ?CustomCallOutputOperandAliases(I, O), From 73fab92393764589730d64051b7fc30a71c882ae Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Mon, 25 May 2026 15:52:10 +0200 Subject: [PATCH 497/881] zml/platforms: add experimental oneapi (intel) (#550) Add experimental Intel GPU backend Add oneAPI as a ZML platform target with Bazel flags, module extension repositories, and PJRT runtime packaging for Linux Intel GPUs. - package libpjrt_oneapi with oneAPI runtime libraries, Level Zero loader, libsycl, and a zmlxoneapi dlopen shim for versioned shared libraries - register the oneAPI platform loader and detect available render devices before loading the packaged PJRT plugin from runfiles - thread the oneAPI target through platform selection, sharding, placement, buffers, execution, memory, IO, tensor behavior, and XLA compile flags - document oneAPI prerequisites and example LLM usage - update Bazel module lockfile entries for the new runtime dependencies --------- Co-authored-by: Steeve Morin Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: Steeve Morin --- MODULE.bazel | 3 + MODULE.bazel.lock | 655 ++++++++++++++++++++ examples/benchmark/main.zig | 2 + platforms/BUILD.bazel | 8 +- platforms/oneapi/BUILD.bazel | 44 ++ platforms/oneapi/README.md | 54 ++ platforms/oneapi/libpjrt_oneapi.BUILD.bazel | 60 ++ platforms/oneapi/oneapi.bzl | 110 ++++ platforms/oneapi/oneapi.zig | 56 ++ platforms/oneapi/zmlxoneapi.zig | 29 + platforms/platforms.zig | 4 + zml/Sharding.zig | 6 +- zml/buffer.zig | 2 +- zml/exe.zig | 6 +- zml/io.zig | 2 +- zml/mem.zig | 2 +- zml/module.zig | 5 + zml/ops.zig | 2 +- zml/platform.zig | 6 +- zml/tensor.zig | 2 +- 20 files changed, 1041 insertions(+), 17 deletions(-) create mode 100644 platforms/oneapi/BUILD.bazel create mode 100644 platforms/oneapi/README.md create mode 100644 platforms/oneapi/libpjrt_oneapi.BUILD.bazel create mode 100644 platforms/oneapi/oneapi.bzl create mode 100644 platforms/oneapi/oneapi.zig create mode 100644 platforms/oneapi/zmlxoneapi.zig diff --git a/MODULE.bazel b/MODULE.bazel index 3df68694..9ddc5751 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -171,6 +171,9 @@ use_repo(cuda, "cuda_nvml_dev_linux_sbsa", "cuda_nvml_dev_linux_x86_64", "cuda_n rocm = use_extension("//platforms/rocm:rocm.bzl", "rocm_packages") use_repo(rocm, "amd-smi-lib", "hipblaslt", "libdrm-amdgpu-amdgpu1", "libdrm2-amdgpu", "libpjrt_rocm", "rocblas", "rocprofiler-sdk", "rocprofiler-sdk-roctx") +oneapi = use_extension("//platforms/oneapi:oneapi.bzl", "oneapi_packages") +use_repo(oneapi, "libpjrt_oneapi") + tpu = use_extension("//platforms/tpu:tpu.bzl", "tpu_packages") use_repo(tpu, "libpjrt_tpu") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 2ff0ccb9..294b79c6 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -452,6 +452,56 @@ ] } }, + "@@aspect_rules_py+//uv/private/tomltool:extension.bzl%tomltool": { + "general": { + "bzlTransitiveDigest": "9BC6q6vdlG+lqF9L13wVxxakVkZBx8buKPFIndN6mHk=", + "usagesDigest": "tQ76lH+Nn5OlAFLvBbnfvnCzNIiwGgY6tOumYZXH44w=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "toml2json_aarch64_osx_libsystem": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_darwin_arm64", + "sha256": "9adc44d976e8f5baf5a6d613ceb3db6e5f56b2e22e75ac56521fdce62f227d88", + "executable": true + } + }, + "toml2json_x86_64_osx_libsystem": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_darwin_amd64", + "sha256": "425bac4015394fd9840fabcd52ac60fc796a9c10655b36c57e36ffb7dc9f3dd4", + "executable": true + } + }, + "toml2json_aarch64_linux_gnu": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_linux_arm64", + "sha256": "be8376c8e3232a242eae0d187f741a475498a949b942361a9af6e95072ef5670", + "executable": true + } + }, + "toml2json_x86_64_linux_gnu": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_linux_amd64", + "sha256": "f3dd54fabf2d27d0c027b0421860e5d9d909080be1613f0ffd87057633b65e9a", + "executable": true + } + } + }, + "recordedRepoMappingEntries": [ + [ + "aspect_rules_py+", + "bazel_tools", + "bazel_tools" + ] + ] + } + }, "@@aspect_tools_telemetry+//:extension.bzl%telemetry": { "general": { "bzlTransitiveDigest": "MePriaXmQNSqUfE+YQvEtzBc2bU1mjITINIffFiZYgo=", @@ -484,6 +534,258 @@ ] } }, + "@@cel-spec+//:extensions.bzl%non_module_dependencies": { + "general": { + "bzlTransitiveDigest": "VN89nCHLUuqo5KktrpNZqv8O+QKrrsGIqdoLPn91hhY=", + "usagesDigest": "HFQJtQrL9nKaFZEjgwaHVMHALMW+cafu696xy7J4ueM=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "com_google_googleapis": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "sha256": "bd8e735d881fb829751ecb1a77038dda4a8d274c45490cb9fcf004583ee10571", + "strip_prefix": "googleapis-07c27163ac591955d736f3057b1619ece66f5b99", + "urls": [ + "https://github.com/googleapis/googleapis/archive/07c27163ac591955d736f3057b1619ece66f5b99.tar.gz" + ] + } + } + }, + "recordedRepoMappingEntries": [ + [ + "cel-spec+", + "bazel_tools", + "bazel_tools" + ] + ] + } + }, + "@@cel-spec+//:googleapis_ext.bzl%googleapis_ext": { + "general": { + "bzlTransitiveDigest": "65z4sd8o73L2a8X2Pd3yAjn73l9HBG45JVnVY2DUbzI=", + "usagesDigest": "Ek7VfZ+tuyRBx/1h5wcmtnW9EGpOb0dkXUwBluZbD8k=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "com_google_googleapis_imports": { + "repoRuleId": "@@cel-spec++non_module_dependencies+com_google_googleapis//:repository_rules.bzl%switched_rules", + "attributes": { + "rules": { + "proto_library_with_info": [ + "", + "" + ], + "moved_proto_library": [ + "", + "" + ], + "java_proto_library": [ + "", + "" + ], + "java_grpc_library": [ + "", + "" + ], + "java_gapic_library": [ + "", + "" + ], + "java_gapic_test": [ + "", + "" + ], + "java_gapic_assembly_gradle_pkg": [ + "", + "" + ], + "py_proto_library": [ + "", + "" + ], + "py_grpc_library": [ + "", + "" + ], + "py_gapic_library": [ + "", + "" + ], + "py_test": [ + "", + "" + ], + "py_gapic_assembly_pkg": [ + "", + "" + ], + "py_import": [ + "", + "" + ], + "go_proto_library": [ + "", + "" + ], + "go_library": [ + "", + "" + ], + "go_test": [ + "", + "" + ], + "go_gapic_library": [ + "", + "" + ], + "go_gapic_assembly_pkg": [ + "", + "" + ], + "cc_proto_library": [ + "native.cc_proto_library", + "" + ], + "cc_grpc_library": [ + "", + "" + ], + "cc_gapic_library": [ + "", + "" + ], + "php_proto_library": [ + "", + "php_proto_library" + ], + "php_grpc_library": [ + "", + "php_grpc_library" + ], + "php_gapic_library": [ + "", + "php_gapic_library" + ], + "php_gapic_assembly_pkg": [ + "", + "php_gapic_assembly_pkg" + ], + "nodejs_gapic_library": [ + "", + "typescript_gapic_library" + ], + "nodejs_gapic_assembly_pkg": [ + "", + "typescript_gapic_assembly_pkg" + ], + "ruby_proto_library": [ + "", + "" + ], + "ruby_grpc_library": [ + "", + "" + ], + "ruby_ads_gapic_library": [ + "", + "" + ], + "ruby_cloud_gapic_library": [ + "", + "" + ], + "ruby_gapic_assembly_pkg": [ + "", + "" + ], + "csharp_proto_library": [ + "", + "" + ], + "csharp_grpc_library": [ + "", + "" + ], + "csharp_gapic_library": [ + "", + "" + ], + "csharp_gapic_assembly_pkg": [ + "", + "" + ] + } + } + } + }, + "recordedRepoMappingEntries": [ + [ + "cel-spec+", + "com_google_googleapis", + "cel-spec++non_module_dependencies+com_google_googleapis" + ] + ] + } + }, + "@@envoy_api+//bazel:repositories.bzl%non_module_deps": { + "general": { + "bzlTransitiveDigest": "JQHl7KCG4m+2+/1IddYzhdiav5qs2iUe+gkg5f+kUPA=", + "usagesDigest": "OP2p9QPxt4WFY/lah8Iecqt6SOUwZHs89hYRyaV+rSg=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "prometheus_metrics_model": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/prometheus/client_model/archive/v0.6.1.tar.gz" + ], + "sha256": "b9b690bc35d80061f255faa7df7621eae39fe157179ccd78ff6409c3b004f05e", + "strip_prefix": "client_model-0.6.1", + "build_file_content": "\nload(\"@envoy_api//bazel:api_build_system.bzl\", \"api_cc_py_proto_library\")\nload(\"@io_bazel_rules_go//proto:def.bzl\", \"go_proto_library\")\n\napi_cc_py_proto_library(\n name = \"client_model\",\n srcs = [\n \"io/prometheus/client/metrics.proto\",\n ],\n visibility = [\"//visibility:public\"],\n)\n\ngo_proto_library(\n name = \"client_model_go_proto\",\n importpath = \"github.com/prometheus/client_model/go\",\n proto = \":client_model\",\n visibility = [\"//visibility:public\"],\n)\n" + } + }, + "com_github_bufbuild_buf": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/bufbuild/buf/releases/download/v1.49.0/buf-Linux-x86_64.tar.gz" + ], + "sha256": "ee8da9748249f7946d79191e36469ce7bc3b8ba80019bff1fa4289a44cbc23bf", + "strip_prefix": "buf", + "build_file_content": "\npackage(\n default_visibility = [\"//visibility:public\"],\n)\n\nfilegroup(\n name = \"buf\",\n srcs = [\n \"@com_github_bufbuild_buf//:bin/buf\",\n ],\n tags = [\"manual\"], # buf is downloaded as a linux binary; tagged manual to prevent build for non-linux users\n)\n" + } + }, + "envoy_toolshed": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/envoyproxy/toolshed/archive/bazel-v0.2.2.tar.gz" + ], + "sha256": "443fe177aba0cef8c17b7a48905c925c67b09005b10dd70ff12cd9f729a72d51", + "strip_prefix": "toolshed-bazel-v0.2.2/bazel" + } + } + }, + "recordedRepoMappingEntries": [ + [ + "envoy_api+", + "bazel_tools", + "bazel_tools" + ], + [ + "envoy_api+", + "envoy_api", + "envoy_api+" + ] + ] + } + }, "@@googleapis+//:extensions.bzl%switched_rules": { "general": { "bzlTransitiveDigest": "JLpojYeGHLdgnwSUuQWOrtkL9UMygBUUEFJZh7ASlmA=", @@ -651,6 +953,126 @@ "recordedRepoMappingEntries": [] } }, + "@@rules_apple+//apple:apple.bzl%provisioning_profile_repository_extension": { + "general": { + "bzlTransitiveDigest": "4qt/sABmG3x78RRuugrSdt7saJl2/iSXQeuIJpyeVjY=", + "usagesDigest": "vsJl8Rw5NL+5Ag2wdUDoTeRF/5klkXO8545Iy7U1Q08=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "local_provisioning_profiles": { + "repoRuleId": "@@rules_apple+//apple/internal:local_provisioning_profiles.bzl%provisioning_profile_repository", + "attributes": {} + } + }, + "recordedRepoMappingEntries": [ + [ + "apple_support+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "bazel_tools", + "rules_cc", + "rules_cc+" + ], + [ + "rules_apple+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "rules_apple+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_apple+", + "build_bazel_apple_support", + "apple_support+" + ], + [ + "rules_apple+", + "build_bazel_rules_swift", + "rules_swift+" + ], + [ + "rules_cc+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_cc+", + "cc_compatibility_proxy", + "rules_cc++compatibility_proxy+cc_compatibility_proxy" + ], + [ + "rules_cc+", + "rules_cc", + "rules_cc+" + ], + [ + "rules_cc++compatibility_proxy+cc_compatibility_proxy", + "rules_cc", + "rules_cc+" + ], + [ + "rules_swift+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "rules_swift+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_swift+", + "build_bazel_apple_support", + "apple_support+" + ], + [ + "rules_swift+", + "build_bazel_rules_swift", + "rules_swift+" + ], + [ + "rules_swift+", + "build_bazel_rules_swift_local_config", + "rules_swift++non_module_deps+build_bazel_rules_swift_local_config" + ] + ] + } + }, + "@@rules_apple+//apple:extensions.bzl%non_module_deps": { + "general": { + "bzlTransitiveDigest": "fe3Htkk5VXHdsIrdLtxsT4oUgADmqgmxwx83bja4UqY=", + "usagesDigest": "M3VqFpeTCo4qmrNKGZw0dxBHvTYDrfV3cscGzlSAhQ4=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "xctestrunner": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/google/xctestrunner/archive/b7698df3d435b6491b4b4c0f9fc7a63fbed5e3a6.tar.gz" + ], + "strip_prefix": "xctestrunner-b7698df3d435b6491b4b4c0f9fc7a63fbed5e3a6", + "sha256": "ae3a063c985a8633cb7eb566db21656f8db8eb9a0edb8c182312c7f0db53730d" + } + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_apple+", + "bazel_tools", + "bazel_tools" + ] + ] + } + }, "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { "general": { "bzlTransitiveDigest": "AEsQvKwdstptzt98AO1bJYfoWpPUn2upx53ra6o/Rc8=", @@ -935,6 +1357,80 @@ ] } }, + "@@rules_perl+//perl:extensions.bzl%perl_repositories": { + "general": { + "bzlTransitiveDigest": "hVM1gWzR35NfpY87MWdNF1xPgjhXCgJv3lHedzj33TY=", + "usagesDigest": "qSSNDdCNVxNhY36wMndEAFacdhR0ooLTmumfad0km9s=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "perl_darwin_arm64": { + "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", + "attributes": { + "strip_prefix": "perl-darwin-arm64", + "sha256": "285769f3c50c339fb59a3987b216ae3c5c573b95babe6875a1ef56fb178433da", + "urls": [ + "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-darwin-arm64.tar.xz" + ] + } + }, + "perl_darwin_amd64": { + "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", + "attributes": { + "strip_prefix": "perl-darwin-amd64", + "sha256": "63bc5ee36f5394d71c50cca6cafdd333ee58f9eaa40bca63c85f9bd06f2c1fd6", + "urls": [ + "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-darwin-amd64.tar.xz" + ] + } + }, + "perl_linux_amd64": { + "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", + "attributes": { + "strip_prefix": "perl-linux-amd64", + "sha256": "3bdffa9d7a3f97c0207314637b260ba5115b1d0829f97e3e2e301191a4d4d076", + "urls": [ + "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-linux-amd64.tar.xz" + ] + } + }, + "perl_linux_arm64": { + "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", + "attributes": { + "strip_prefix": "perl-linux-arm64", + "sha256": "6fa4ece99e790ecbc2861f6ecb7b52694c01c2eeb215b4370f16a3b12d952117", + "urls": [ + "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-linux-arm64.tar.xz" + ] + } + }, + "perl_windows_x86_64": { + "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", + "attributes": { + "strip_prefix": "", + "sha256": "aeb973da474f14210d3e1a1f942dcf779e2ae7e71e4c535e6c53ebabe632cc98", + "urls": [ + "https://mirror.bazel.build/strawberryperl.com/download/5.32.1.1/strawberry-perl-5.32.1.1-64bit.zip", + "https://strawberryperl.com/download/5.32.1.1/strawberry-perl-5.32.1.1-64bit.zip" + ] + } + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_perl+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_perl+", + "rules_perl", + "rules_perl+" + ] + ] + } + }, "@@rules_python+//python/uv:uv.bzl%uv": { "general": { "bzlTransitiveDigest": "LM8+g04lV+afYP67YcE7wenTJMrNqvF5u/ZnOctwctY=", @@ -1107,6 +1603,165 @@ ] } }, + "@@rules_swift+//swift:extensions.bzl%non_module_deps": { + "general": { + "bzlTransitiveDigest": "vwewHB1JdoOb38MyyeRQPIdGOkEKD52YNUYgGwmeOy8=", + "usagesDigest": "mhACFnrdMv9Wi0Mt67bxocJqviRkDSV+Ee5Mqdj5akA=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "com_github_apple_swift_protobuf": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-protobuf/archive/1.20.2.tar.gz" + ], + "sha256": "3fb50bd4d293337f202d917b6ada22f9548a0a0aed9d9a4d791e6fbd8a246ebb", + "strip_prefix": "swift-protobuf-1.20.2/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_protobuf/BUILD.overlay" + } + }, + "com_github_grpc_grpc_swift": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/grpc/grpc-swift/archive/1.16.0.tar.gz" + ], + "sha256": "58b60431d0064969f9679411264b82e40a217ae6bd34e17096d92cc4e47556a5", + "strip_prefix": "grpc-swift-1.16.0/", + "build_file": "@@rules_swift+//third_party:com_github_grpc_grpc_swift/BUILD.overlay" + } + }, + "com_github_apple_swift_docc_symbolkit": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-docc-symbolkit/archive/refs/tags/swift-5.10-RELEASE.tar.gz" + ], + "sha256": "de1d4b6940468ddb53b89df7aa1a81323b9712775b0e33e8254fa0f6f7469a97", + "strip_prefix": "swift-docc-symbolkit-swift-5.10-RELEASE", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_docc_symbolkit/BUILD.overlay" + } + }, + "com_github_apple_swift_nio": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio/archive/2.42.0.tar.gz" + ], + "sha256": "e3304bc3fb53aea74a3e54bd005ede11f6dc357117d9b1db642d03aea87194a0", + "strip_prefix": "swift-nio-2.42.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_http2": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-http2/archive/1.26.0.tar.gz" + ], + "sha256": "f0edfc9d6a7be1d587e5b403f2d04264bdfae59aac1d74f7d974a9022c6d2b25", + "strip_prefix": "swift-nio-http2-1.26.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_http2/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_transport_services": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-transport-services/archive/1.15.0.tar.gz" + ], + "sha256": "f3498dafa633751a52b9b7f741f7ac30c42bcbeb3b9edca6d447e0da8e693262", + "strip_prefix": "swift-nio-transport-services-1.15.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_transport_services/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_extras": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-extras/archive/1.4.0.tar.gz" + ], + "sha256": "4684b52951d9d9937bb3e8ccd6b5daedd777021ef2519ea2f18c4c922843b52b", + "strip_prefix": "swift-nio-extras-1.4.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_extras/BUILD.overlay" + } + }, + "com_github_apple_swift_log": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-log/archive/1.4.4.tar.gz" + ], + "sha256": "48fe66426c784c0c20031f15dc17faf9f4c9037c192bfac2f643f65cb2321ba0", + "strip_prefix": "swift-log-1.4.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_log/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_ssl": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-ssl/archive/2.23.0.tar.gz" + ], + "sha256": "4787c63f61dd04d99e498adc3d1a628193387e41efddf8de19b8db04544d016d", + "strip_prefix": "swift-nio-ssl-2.23.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_ssl/BUILD.overlay" + } + }, + "com_github_apple_swift_collections": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-collections/archive/1.0.4.tar.gz" + ], + "sha256": "d9e4c8a91c60fb9c92a04caccbb10ded42f4cb47b26a212bc6b39cc390a4b096", + "strip_prefix": "swift-collections-1.0.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_collections/BUILD.overlay" + } + }, + "com_github_apple_swift_atomics": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-atomics/archive/1.1.0.tar.gz" + ], + "sha256": "1bee7f469f7e8dc49f11cfa4da07182fbc79eab000ec2c17bfdce468c5d276fb", + "strip_prefix": "swift-atomics-1.1.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_atomics/BUILD.overlay" + } + }, + "build_bazel_rules_swift_index_import": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file": "@@rules_swift+//third_party:build_bazel_rules_swift_index_import/BUILD.overlay", + "canonical_id": "index-import-5.8", + "urls": [ + "https://github.com/MobileNativeFoundation/index-import/releases/download/5.8.0.1/index-import.tar.gz" + ], + "sha256": "28c1ffa39d99e74ed70623899b207b41f79214c498c603915aef55972a851a15" + } + }, + "build_bazel_rules_swift_local_config": { + "repoRuleId": "@@rules_swift+//swift/internal:swift_autoconfiguration.bzl%swift_autoconfiguration", + "attributes": {} + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_swift+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_swift+", + "build_bazel_rules_swift", + "rules_swift+" + ] + ] + } + }, "@@rules_zig+//zig:extensions.bzl%zig": { "general": { "bzlTransitiveDigest": "u7Psh7szBG3+lbRmEeBBbWlZRHJWhWtxQo/wtRqvnTg=", diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index 5a66613c..d3ba9eee 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -24,6 +24,8 @@ pub fn main(init: std.process.Init) !void { const platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator, io); + log.info("\n{f}", .{platform.fmtVerbose()}); + const benchmark_sharding: zml.Sharding = try platform.registerSharding("benchmark_mesh", .mesh( .{ .m = .low_bandwidth, .n = .high_bandwidth }, )); diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index 744522cd..ec0e462f 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -7,6 +7,7 @@ PLATFORMS = { "rocm": False, "tpu": False, "neuron": False, + "oneapi": False, } [ @@ -33,12 +34,7 @@ config_setting( flag_values = {":{}".format(runtime): "True"}, visibility = ["//visibility:public"], ) - for runtime in [ - "cpu", - "cuda", - "tpu", - "neuron", - ] + for runtime in ["cpu", "cuda", "tpu", "neuron", "oneapi"] ] zig_library( diff --git a/platforms/oneapi/BUILD.bazel b/platforms/oneapi/BUILD.bazel new file mode 100644 index 00000000..f4d23804 --- /dev/null +++ b/platforms/oneapi/BUILD.bazel @@ -0,0 +1,44 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_shared_library") + +zig_shared_library( + name = "zmlxoneapi", + main = "zmlxoneapi.zig", + shared_lib_name = "libzmlxoneapi.so.0", + target_compatible_with = ["@platforms//os:linux"], + visibility = ["@libpjrt_oneapi//:__subpackages__"], + zigopts = ["-fno-stack-check"], + deps = ["//stdx"], +) + +cc_library( + name = "empty", + defines = ["ZML_RUNTIME_ONEAPI_DISABLED"], + tags = ["manual"], +) + +cc_library( + name = "libpjrt_oneapi", + defines = ["ZML_RUNTIME_ONEAPI"], + tags = ["manual"], + deps = ["@libpjrt_oneapi"], +) + +zig_library( + name = "oneapi", + import_name = "platforms/oneapi", + main = "oneapi.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//bazel", + "//pjrt", + "//stdx", + "@rules_zig//zig/runfiles", + ] + select({ + "//platforms:oneapi.enabled": [ + ":libpjrt_oneapi", + ], + "//conditions:default": [":empty"], + }), +) diff --git a/platforms/oneapi/README.md b/platforms/oneapi/README.md new file mode 100644 index 00000000..d450bba6 --- /dev/null +++ b/platforms/oneapi/README.md @@ -0,0 +1,54 @@ +# EXPERIMENTAL: oneAPI + + +# About + +This directory contains the oneAPI backend for ZML. It is currently in an early stage of development, and only supports a subset of ZML's features. The oneAPI backend is only available on Linux, and requires an Intel GPU. + +The sharding is not yet supported, so only models that fit in the GPU memory can be run. + +# Getting Started + +## Prerequisites + +- An Intel GPU +- bazel see [Getting Started](../README.md#getting-started) for installation instructions. + +### Linux Only + +One GPU is supported on Linux. +ONEAPI_DEVICE_SELECTOR environment variable must be set to select the GPU device. + +For example, to select the first GPU: +```bash +export ONEAPI_DEVICE_SELECTOR=level_zero:0 +``` + +#### Run + + +To run the LLM example with oneAPI and the Llama 3.2 1B model: +```bash +ONEAPI_DEVICE_SELECTOR=level_zero:0 \ +bazel run //examples/llm \ + --config=release \ + --@zml//platforms:cpu=false \ + --@zml//platforms:oneapi=true \ + -- \ + --model=hf://meta-llama/Llama-3.2-1B-Instruct \ + --topk=1 \ + --prompt="Tell me a story about a cat in 2 lines" +``` + +To run the LLM example with oneAPI and a local model: +```bash +ONEAPI_DEVICE_SELECTOR=level_zero:0 \ +bazel run //examples/llm \ + --config=release \ + --@zml//platforms:cpu=false \ + --@zml//platforms:oneapi=true \ + -- \ + --model=path/to/your/model/Llama-3.1-8B-Instruct \ + --topk=1 \ + --prompt="Tell me a story about a cat in 2 lines" +``` diff --git a/platforms/oneapi/libpjrt_oneapi.BUILD.bazel b/platforms/oneapi/libpjrt_oneapi.BUILD.bazel new file mode 100644 index 00000000..1b28a04f --- /dev/null +++ b/platforms/oneapi/libpjrt_oneapi.BUILD.bazel @@ -0,0 +1,60 @@ +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@zml//bazel:patchelf.bzl", "patchelf") + +ONEAPI_UMF_VERSION = "0.10.0" +ONEAPI_UR_VERSION = "0.11.10" + +patchelf( + name = "libpjrt_oneapi_so", + src = "libpjrt_oneapi.so", + add_needed = [ + "libzmlxoneapi.so.0", + ], + rename_dynamic_symbols = { + "dlopen": "zmlxoneapi_dlopen", + }, + set_rpath = "$ORIGIN", +) + +patchelf( + name = "libsycl_so", + src = "@oneapi//:libsycl_so", + set_rpath = "$ORIGIN", +) + +ONEAPI_VERSION = "2025.1" +ONEAPI_BASE_LIB = "{}/lib".format(ONEAPI_VERSION) +ONEAPI_COMPILER_LIB = "compiler/{}/lib".format(ONEAPI_VERSION) +ONEAPI_MKL_LIB = "mkl/{}/lib".format(ONEAPI_VERSION) + +copy_to_directory( + name = "sandbox", + srcs = [ + ":libpjrt_oneapi_so", + ":libsycl_so", + "@zml//platforms/oneapi:zmlxoneapi", + "@oneapi//:base_runtime", + "@oneapi//:compiler_runtime", + "@oneapi//:mkl_runtime", + "@zero_loader//:all", + ], + add_directory_to_runfiles = False, + allow_overwrites = True, + hardlink = "auto", + include_external_repositories = ["**"], + replace_prefixes = { + "libpjrt_oneapi_so": "lib", + "libsycl_so": "lib", + ONEAPI_BASE_LIB: "lib", + ONEAPI_COMPILER_LIB: "lib", + ONEAPI_MKL_LIB: "lib", + "platforms/oneapi/libzmlxoneapi": "lib/libzmlxoneapi", + }, + root_paths = ["."], +) + +cc_library( + name = "libpjrt_oneapi", + data = [":sandbox"], + visibility = ["@zml//platforms/oneapi:__subpackages__"], +) diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl new file mode 100644 index 00000000..9bee53f6 --- /dev/null +++ b/platforms/oneapi/oneapi.bzl @@ -0,0 +1,110 @@ +load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") + +PJRT_ONEAPI_RELEASE = "manual-2026-05-21T17-13-00Z" +PJRT_ONEAPI_ARTIFACT_SHA256 = "24c518768b5f4994cde22603c7451ce6c53e9d236c3e574e65ca73cd32c6368a" +PJRT_ONEAPI_ARTIFACT_URL = "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-oneapi_linux-amd64.tar.gz".format( + release = PJRT_ONEAPI_RELEASE, +) + +ONEAPI_VERSION = "2025.1" +ONEAPI_BASE_TOOLKIT_VERSION = "2025.1.3.7" + +ONEAPI_BASE_LIB = "{}/lib".format(ONEAPI_VERSION) +ONEAPI_COMPILER_LIB = "compiler/{}/lib".format(ONEAPI_VERSION) +ONEAPI_MKL_LIB = "mkl/{}/lib".format(ONEAPI_VERSION) + +ONEAPI_RUNTIME_URL = "https://tensorflow-file-hosting.s3.us-east-1.amazonaws.com/intel-oneapi-base-toolkit-{version}.tar.gz".format( + version = ONEAPI_BASE_TOOLKIT_VERSION, +) +ONEAPI_RUNTIME_SHA256 = "2213104bd122336551aa144512e7ab99e4a84220e77980b5f346edc14ebd458a" +ONEAPI_RUNTIME_STRIP_PREFIX = "oneapi" + +ONEAPI_ZERO_LOADER_URL = "https://tensorflow-file-hosting.s3.us-east-1.amazonaws.com/ze_loader_libs.tar.gz" +ONEAPI_ZERO_LOADER_SHA256 = "71cbfd8ac59e1231f013e827ea8efe6cf5da36fad771da2e75e202423bd6b82e" + +_ONEAPI_BUILD_FILE_CONTENT = """\ +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "base_runtime", + srcs = [ + "{ONEAPI_BASE_LIB}/libhwloc.so.15", + "{ONEAPI_BASE_LIB}/libumf.so.0", + "{ONEAPI_BASE_LIB}/libur_adapter_level_zero.so.0", + "{ONEAPI_BASE_LIB}/libur_adapter_opencl.so.0", + "{ONEAPI_BASE_LIB}/libur_loader.so.0", + ], +) + +filegroup( + name = "compiler_runtime", + srcs = glob(["{ONEAPI_COMPILER_LIB}/*.spv"]) + [ + "{ONEAPI_COMPILER_LIB}/libOpenCL.so.1", + "{ONEAPI_COMPILER_LIB}/libimf.so", + "{ONEAPI_COMPILER_LIB}/libintlc.so.5", + "{ONEAPI_COMPILER_LIB}/libirc.so", + "{ONEAPI_COMPILER_LIB}/libirng.so", + "{ONEAPI_COMPILER_LIB}/libsvml.so", + ], +) + +filegroup( + name = "libsycl_so", + srcs = ["{ONEAPI_COMPILER_LIB}/libsycl.so.8"] +) + +filegroup( + name = "mkl_runtime", + srcs = [ + "{ONEAPI_MKL_LIB}/libmkl_core.so.2", + "{ONEAPI_MKL_LIB}/libmkl_intel_ilp64.so.2", + "{ONEAPI_MKL_LIB}/libmkl_sequential.so.2", + "{ONEAPI_MKL_LIB}/libmkl_sycl_blas.so.5", + ], +) +""".format( + ONEAPI_BASE_LIB = ONEAPI_BASE_LIB, + ONEAPI_COMPILER_LIB = ONEAPI_COMPILER_LIB, + ONEAPI_MKL_LIB = ONEAPI_MKL_LIB, +) + +_ZERO_LOADER_BUILD_FILE_CONTENT = """\ +filegroup( + name = "all", + srcs = glob(["lib/**"]), + visibility = ["//visibility:public"], +) +""" + +def _oneapi_impl(mctx): + http_archive( + name = "libpjrt_oneapi", + build_file = "libpjrt_oneapi.BUILD.bazel", + sha256 = PJRT_ONEAPI_ARTIFACT_SHA256, + url = PJRT_ONEAPI_ARTIFACT_URL, + ) + http_archive( + name = "oneapi", + build_file_content = _ONEAPI_BUILD_FILE_CONTENT, + sha256 = ONEAPI_RUNTIME_SHA256, + strip_prefix = ONEAPI_RUNTIME_STRIP_PREFIX, + url = ONEAPI_RUNTIME_URL, + ) + http_archive( + name = "zero_loader", + build_file_content = _ZERO_LOADER_BUILD_FILE_CONTENT, + sha256 = ONEAPI_ZERO_LOADER_SHA256, + url = ONEAPI_ZERO_LOADER_URL, + ) + + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = [ + "libpjrt_oneapi", + ], + root_module_direct_dev_deps = [], + ) + +oneapi_packages = module_extension( + implementation = _oneapi_impl, +) diff --git a/platforms/oneapi/oneapi.zig b/platforms/oneapi/oneapi.zig new file mode 100644 index 00000000..c81b6356 --- /dev/null +++ b/platforms/oneapi/oneapi.zig @@ -0,0 +1,56 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); +const c = @import("c"); +const pjrt = @import("pjrt"); +const stdx = @import("stdx"); + +const log = std.log.scoped(.@"zml/platforms/oneapi"); + +pub fn isEnabled() bool { + return @hasDecl(c, "ZML_RUNTIME_ONEAPI"); +} + +fn hasOneApiDevices(io: std.Io) bool { + var dir = std.Io.Dir.openDirAbsolute(io, "/dev/dri", .{ .iterate = true }) catch return false; + defer dir.close(io); + + var it = dir.iterate(); + while (it.next(io) catch null) |entry| { + if (std.mem.startsWith(u8, entry.name, "renderD")) { + return true; + } + } + + return false; +} + +pub fn load(_: std.mem.Allocator, io: std.Io) !*const pjrt.Api { + if (comptime !isEnabled()) { + return error.Unavailable; + } + + if (comptime builtin.os.tag != .linux) { + return error.Unavailable; + } + + if (!hasOneApiDevices(io)) { + return error.Unavailable; + } + + const r = try bazel.runfiles(bazel_builtin.current_repository); + + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = try r.rlocation("libpjrt_oneapi/sandbox", &path_buf) orelse { + log.err("Failed to find sandbox path for oneAPI runtime", .{}); + return error.FileNotFound; + }; + + return blk: { + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_oneapi.so" }); + break :blk pjrt.Api.loadFrom(path); + }; +} diff --git a/platforms/oneapi/zmlxoneapi.zig b/platforms/oneapi/zmlxoneapi.zig new file mode 100644 index 00000000..e5d94e10 --- /dev/null +++ b/platforms/oneapi/zmlxoneapi.zig @@ -0,0 +1,29 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +pub export fn zmlxoneapi_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque { + const replacements: std.StaticStringMap([:0]const u8) = .initComptime(.{ + .{ "libOpenCL.so", "libOpenCL.so.1" }, + .{ "libumf.so", "libumf.so.0" }, + .{ "libur_loader.so", "libur_loader.so.0" }, + .{ "libur_adapter_level_zero.so", "libur_adapter_level_zero.so.0" }, + .{ "libur_adapter_opencl.so", "libur_adapter_opencl.so.0" }, + .{ "libsycl.so", "libsycl.so.8" }, + .{ "libmkl_core.so", "libmkl_core.so.2" }, + .{ "libmkl_intel_ilp64.so", "libmkl_intel_ilp64.so.2" }, + .{ "libmkl_sequential.so", "libmkl_sequential.so.2" }, + .{ "libmkl_sycl_blas.so", "libmkl_sycl_blas.so.5" }, + }); + + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const new_filename: [*c]const u8 = if (filename) |f| blk: { + const replacement = replacements.get(std.Io.Dir.path.basename(std.mem.span(f))) orelse break :blk f; + break :blk stdx.Io.Dir.path.bufJoinZ(&buf, &.{ + stdx.process.selfSharedObjectDirPath(), + replacement, + }) catch unreachable; + } else null; + + return std.c.dlopen(new_filename, @bitCast(flags)); +} diff --git a/platforms/platforms.zig b/platforms/platforms.zig index 4033415c..63a0a7b0 100644 --- a/platforms/platforms.zig +++ b/platforms/platforms.zig @@ -5,6 +5,7 @@ const cuda = @import("platforms/cuda"); const neuron = @import("platforms/neuron"); const pjrt = @import("pjrt"); const rocm = @import("platforms/rocm"); +const oneapi = @import("platforms/oneapi"); const tpu = @import("platforms/tpu"); pub const Platform = enum { @@ -13,6 +14,7 @@ pub const Platform = enum { rocm, tpu, neuron, + oneapi, }; pub fn load(allocator: std.mem.Allocator, io: std.Io, tag: Platform) !*const pjrt.Api { @@ -22,6 +24,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io, tag: Platform) !*const pjr .rocm => try rocm.load(allocator, io), .tpu => try tpu.load(allocator, io), .neuron => try neuron.load(allocator, io), + .oneapi => try oneapi.load(allocator, io), }; } @@ -32,5 +35,6 @@ pub fn isEnabled(tag: Platform) bool { .rocm => rocm.isEnabled(), .tpu => tpu.isEnabled(), .neuron => neuron.isEnabled(), + .oneapi => oneapi.isEnabled(), }; } diff --git a/zml/Sharding.zig b/zml/Sharding.zig index 6cfafb6d..27dc2845 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -42,7 +42,7 @@ pub const Partitioner = union(enum) { pub fn fromTarget(target: Target) Partitioner { return switch (target) { - .cpu, .cuda, .rocm, .tpu => .shardy, + .cpu, .cuda, .rocm, .tpu, .oneapi => .shardy, .neuron => .gspmd, }; } @@ -638,6 +638,7 @@ pub const PhysicalMesh = struct { .neuron => &.{ .link, .link_x, .link_y, .link_z }, .cuda, .rocm => &.{.link}, .cpu => &.{.bus}, + .oneapi => &.{ .link, .bus }, }; } @@ -861,6 +862,7 @@ pub const PhysicalMesh = struct { .cuda, .rocm => gpu(allocator, platform_devices), .tpu => tpu(allocator, platform_devices), .neuron => neuron(allocator, platform_devices), + .oneapi => cpu(allocator, platform_devices), }; errdefer freeNode(allocator, root); @@ -1724,7 +1726,7 @@ pub const Placement = struct { for (platform.devices, 0..) |d, device_index| { const device_id = switch (platform.target) { .neuron => @as(usize, @intCast(d.localHardwareId())), - .cuda, .rocm, .tpu, .cpu => d.id(), + .cuda, .rocm, .tpu, .cpu, .oneapi => d.id(), }; if (device_id == self.device_id) return device_index; } diff --git a/zml/buffer.zig b/zml/buffer.zig index c6a34030..8e0df09d 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -205,7 +205,7 @@ pub const Buffer = struct { return try Buffer.from(io, platform, sh, sharding, host, .{ .wait = true, .memory = opts.memory }); }, - .cpu, .cuda, .rocm, .tpu => {}, + .cpu, .cuda, .rocm, .tpu, .oneapi => {}, } var res: Buffer = .{ diff --git a/zml/exe.zig b/zml/exe.zig index cf305efe..f077b009 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -62,6 +62,8 @@ pub const Exe = struct { } pub fn deinit(self: *const Exe) void { + if (self.context) |context| context.deinit(self.platform.pjrt_api); + self.exe.deinit(self.platform.pjrt_api); self.arena.deinit(); } @@ -264,7 +266,7 @@ pub const Exe = struct { const partition_events = events[0..@intCast(self.num_partitions)]; const events_slice: ?[]?*pjrt.Event = switch (self.platform.target) { .neuron => partition_events, - .cpu, .cuda, .rocm, .tpu => if (opts.wait) partition_events else null, + .cpu, .cuda, .rocm, .tpu, .oneapi => if (opts.wait) partition_events else null, }; self.exe.execute(self.platform.pjrt_api, .{ @@ -289,7 +291,7 @@ pub const Exe = struct { } } }, - .cpu, .cuda, .rocm, .tpu => if (opts.wait) { + .cpu, .cuda, .rocm, .tpu, .oneapi => if (opts.wait) { for (events_slice.?) |e| { if (e) |ev| { ev.await(self.platform.pjrt_api, io.?) catch unreachable; diff --git a/zml/io.zig b/zml/io.zig index 36a857e6..33efdfa6 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -300,7 +300,7 @@ pub const MemoryWriter = union(enum) { buffer: *Buffer, ) !MemoryWriter { return switch (platform.target) { - .cuda => .{ .direct = try DirectMemoryWriter.init(allocator, io, platform, pools, dma_allocators, shape, sharding, buffer) }, + .cuda, .oneapi => .{ .direct = try DirectMemoryWriter.init(allocator, io, platform, pools, dma_allocators, shape, sharding, buffer) }, .rocm, .tpu, .neuron, .cpu => .{ .buffered = try BufferedMemoryWriter.init(allocator, io, platform, shape, sharding, buffer) }, }; } diff --git a/zml/mem.zig b/zml/mem.zig index a620e4bb..1a5d012d 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -22,7 +22,7 @@ pub const DmaAllocator = union(enum) { pub fn init(parent: std.mem.Allocator, device: *const Device) DmaAllocator { return switch (device.platform.target) { .cuda => .{ .dmam = .init(parent, device.platform) }, - .tpu => .{ .uib = .init(device.memory(.host_pinned)) }, + .oneapi, .tpu => .{ .uib = .init(device.memory(.host_pinned)) }, .rocm, .cpu, .neuron => .{ .passthrough = parent }, }; } diff --git a/zml/module.zig b/zml/module.zig index de774f31..f27c5944 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -634,6 +634,11 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: // This is what AMD recommendeds in the meantime. try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "CUBLAS,CUBLASLT,CUSTOM_CALL,CUDNN,DYNAMIC_SLICE_FUSION", upb_arena); }, + .oneapi => { + try setXlaOverrideFlag(overrides_map, "xla_gpu_autotune_level", 0, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_cublaslt", false, upb_arena); + }, else => {}, } diff --git a/zml/ops.zig b/zml/ops.zig index 0433ca33..4729c28b 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -520,7 +520,7 @@ test "triton" { const zml = @import("zml.zig"); const platform = zml.testing.env(); - if (platform.target != .cuda and platform.target != .rocm) return error.SkipZigTest; + if (platform.target != .cuda and platform.target != .rocm and platform.target != .oneapi) return error.SkipZigTest; const ir = \\ module { diff --git a/zml/platform.zig b/zml/platform.zig index 61b66f25..7e359d61 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -45,7 +45,7 @@ fn disableXlaLogs() void { fn validateDeviceCount(target: Target, num_devices: usize) !void { switch (target) { - .cpu, .cuda, .rocm, .tpu, .neuron => { + .cpu, .cuda, .rocm, .tpu, .neuron, .oneapi => { if (num_devices == 0) { log.err("Platform {} requires at least 1 device, got {}", .{ target, num_devices }); return error.ZeroVisibleDevices; @@ -98,7 +98,7 @@ pub const Memory = struct { pub fn isOfKind(self: Memory, kind_: Kind) bool { switch (self.platform.target) { - .cuda, .rocm, .tpu => { + .cuda, .rocm, .oneapi, .tpu => { const zml_kind: Memory.Kind = switch (self.kind().len) { "device".len => .device, "pinned_host".len => .host_pinned, @@ -313,6 +313,7 @@ pub const Platform = struct { .neuron, .rocm, .cuda, + .oneapi, .cpu, }; return for (ordered_targets) |target| { @@ -589,6 +590,7 @@ pub const CreateOptions = struct { rocm: struct {} = .{}, tpu: struct {} = .{}, neuron: struct {} = .{}, + oneapi: struct {} = .{}, pub const Cpu = struct { device_count: u32, diff --git a/zml/tensor.zig b/zml/tensor.zig index cb4b55e0..4759069c 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -4196,7 +4196,7 @@ pub const Tensor = struct { } }).body); }, - .neuron => {}, + .oneapi, .neuron => {}, } } From 289c29f365640a1cb1e7d2c9fce978e39295e3bf Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Mon, 25 May 2026 17:14:28 +0200 Subject: [PATCH 498/881] workspace: update AGENTS.md (#548) More concise AGENTS.md and add a command to help the agent find the sources of the Zig library it regularly struggles with it --- AGENTS.md | 166 +++++++----------------------------------------------- 1 file changed, 20 insertions(+), 146 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 834e379e..e3f5ef1e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,157 +1,31 @@ -# AGENTS.md - ZML Agent Guide +# Repository Guidelines -This guide is for autonomous coding agents working in this repository. +## Project Structure & Module Organization -## Scope and Environment +ZML is a Zig project built with Bazel. Core library code lives in `zml/`, with IO under `zml/io/`, tokenizers under `zml/tokenizer/`, attention implementations under `zml/attention/`, and MoE support under `zml/moe/`. Shared Zig utilities are in `stdx/`; FFI and C helpers are in `ffi/` and `upb/`. Bazel rules and workspace helpers live in `bazel/` and `third_party/`. Runnable examples are in `examples/`, while user-facing documentation is in `docs/`. Keep tests close to implementation files unless a package already has a separate test file. -- Primary language: Zig (dev snapshot), built and tested with Bazel. -- Main subsystems: `zml/`, `stdx/`, `mlir/`, `pjrt/`, `platforms/`, `examples/`. -- Bazel version is pinned in `.bazelversion` (`8.5.1`). -- Build mode defaults to debug via `.bazelrc` (`--compilation_mode=dbg`). +## Build, Test, and Development Commands -## External Instruction Files +- `bazel build //zml`: builds the main ZML library. +- `bazel test //zml:test`: runs the core Zig test target. +- `bazel test //stdx:test`: runs shared utility tests. +- `bazel test //zml/tokenizer:test`: runs tokenizer tests. +- `bazel run //examples/mnist`: runs the quick MNIST smoke test. +- `bazel run //examples/llm -- --model=hf://... --prompt="..."`: runs the LLM example. +- `bazel build //...` and `bazel test //...`: use before broad or shared changes. +- `./tools/buildifier.sh`: formats Bazel/Starlark files. +- `echo "$(bazel info output_base)/$(bazel cquery --output=files "$(bazel cquery "filter('zig_toolchain', deps(//zml))" 2>/dev/null | cut -d' ' -f1 | head -n1)" 2>/dev/null | rg "/lib\$")/std"`: find the current Zig standard library used by Bazel. -- Cursor rules: `.cursor/rules/**` and `.cursorrules` are not present. -- Copilot rules: `.github/copilot-instructions.md` is not present. -- If any of these files appear later, treat them as required constraints. +Append platform flags when relevant, for example `--@zml//platforms:cuda=true`, `--@zml//platforms:rocm=true`, `--@zml//platforms:tpu=true`, or `--@zml//platforms:cpu=false`. -## Core Build Commands +## Coding Style & Naming Conventions -```bash -# Build all targets -bazel build //... +Follow the Zig style guide and format Zig code with `zig fmt`; CI checks `zig fmt --check` outside `third_party/`. Use four-space indentation. Prefer `const x: Foo = .{ .bar = 1 };`, `pub fn method(self: Foo)`, and module imports such as `const foo = @import("foo.zig"); foo.bar()`. Use `PascalCase` for types and `lowerCamelCase` for functions, fields, and locals. Add comments only for non-obvious invariants, ownership, tensor shape semantics, or platform behavior. -# Build specific targets/packages -bazel build //zml -bazel build //examples/llama -bazel build //examples/mnist +## Testing Guidelines -# Alternate build configs -bazel build --config=debug //... -bazel build --config=alldebug //... -bazel build --config=release //... -bazel build --config=native //... -``` +Tests are generally inline Zig `test "..." {}` blocks near the code they cover. Use descriptive test names and narrow Bazel targets first, then broaden when touching shared APIs. For tensor or graph changes, cover shape/tag inference and runtime behavior when practical. CI builds all targets and runs tests on CPU, CUDA, and ROCm where available. -### Useful `.bazelrc` config meanings +## Commit & Pull Request Guidelines -- `--config=debug`: optimized backend, Zig frontend debug mode. -- `--config=release`: optimized (`-Ofast`, Zig `release_safe`). -- `--config=alldebug` / `--config=native` / `--config=silent`: debug-all, local CPU tuning, reduced UI noise. - -## Test Commands - -```bash -# Run everything -bazel test //... - -# Common package tests -bazel test //zml:test -bazel test //stdx:test -bazel test //mlir:test -bazel test //zml/tokenizer:test - -# Discover Zig test targets -bazel query 'kind(zig_test, //...)' -``` - -### Run a single test (inside a `zig_test` target) - -`zml/test_runner.zig` supports substring filtering from positional args. -Pass filter with `--test_arg`: - -```bash -# Runs tests whose name contains "normalizeL2" -bazel test //zml:test --test_arg=normalizeL2 - -# Example on another target -bazel test //stdx:test --test_arg=json -``` - -- Matching is substring-based on test function identifier; `.bazelrc` already sets `test --test_output=errors`. - -## Lint / Formatting Commands - -```bash -# Format Bazel/Starlark files -./tools/buildifier.sh - -# Run ZLS through Bazel (editor and language tooling) -./tools/zls.sh -``` - -## Platform / Accelerator Flags - -Append these to `bazel build|test|run` as needed: - -```bash ---@zml//platforms:cuda=true ---@zml//platforms:rocm=true ---@zml//platforms:tpu=true ---@zml//platforms:neuron=true ---@zml//platforms:cpu=false -``` - -## Code Style (Repository Conventions) - -### Imports and namespacing - -- Prefer module imports, then namespaced access. -- Prefer: `const foo = @import("foo.zig"); foo.bar()`. -- Avoid: `const bar = @import("foo.zig").bar` as default style. -- Import specific types directly only when heavily used. -- Keep local imports relative within a package. - -### Formatting and structure - -- Follow Zig style and existing local patterns first. -- Prefer typed anonymous initialization where clear: `const x: Foo = .{ .bar = 1 };`. -- Keep functions focused; pair resource acquisition with nearby `defer`/`errdefer`. -- Add comments only for non-obvious invariants, ownership, or shape semantics. - -### Types, naming, and tensor semantics - -- Types/structs: `PascalCase`. -- Functions/vars/fields: Zig lower camel case. -- Keep names descriptive; avoid short opaque identifiers unless very local. -- Use semantic tensor tags (`.b`, `.s`, `.d`, `.h`, `.hd`, etc.) over raw index assumptions. - -### Errors, assertions, and cleanup - -- Prefer `try`-based propagation. -- Use `catch |err|` when you add useful recovery or context. -- Use `errdefer` on partial initialization paths. -- Prefer `stdx.debug.assert` over bare `std.debug.assert` in this codebase. -- Avoid panic-style control flow except for unrecoverable programmer errors. - -### Logging and CLI entrypoints - -- Use scoped logs (for example `std.log.scoped(.@"zml/io")`). -- CLI binaries commonly define `pub const std_options`. -- Argument parsing usually follows `stdx.flags` conventions. - -## Testing Patterns - -- Tests are usually inline Zig `test "..." {}` blocks near implementation. -- Common helpers: `zml.testing.env`, `zml.testing.expectClose`, `zml.testing.approxEq`. -- `zig_test` targets often use `zml/test_runner.zig` for progress/filter/leak checks. - -## Critical Architecture Guardrails - -- `Tensor` ops in `forward` build an MLIR graph; they do not execute immediately. -- Preserve distinctions: - - `Shape` = metadata only - - `Tensor` = compile-time graph value - - `Buffer` = runtime device allocation - - `Slice` = host-side data view -- `zml.Bufferized(T)` maps tensor-typed model structures to runtime buffer structures. -- Weight loading paths are centered on safetensors + TensorStore/VFS. - -## Practical Agent Workflow - -- Read the target file, nearby tests, and owning `BUILD.bazel`. -- Check whether behavior lives in compile-time graph construction or runtime execution. - -- Run the narrowest relevant tests first (`--test_arg` when possible). -- Run broader package tests if shared code changed. -- Run `./tools/buildifier.sh` when any Bazel files changed. +Recent history uses scoped, imperative commit subjects such as `zml/tensor: add Tensor.onMemory()` or `workspace: use latest version of upstreamable rules_zig`. Keep subjects concise and mention the affected area first. Pull requests should include a short description, relevant linked issue, platform impact if any, and the exact `bazel build` or `bazel test` commands run. Include screenshots only for docs or UI-visible changes. From a06265264f7039d5276398ea4c3774701ba23434 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 26 May 2026 09:29:49 +0200 Subject: [PATCH 499/881] zml/Sharding: Rename Placement.Shard to Placement (#549) Keeping `devices_in_canonical_order: []Device` in the `PhysicalMesh` and inside `Sharding.Data` I've changed `physical: PhysicalMesh` to `physical: *const PhysicalMesh` to have the platform own it now that PhysicalMesh has a `deinit()`. --- devenv.lock | 18 +- examples/sharding/main.zig | 13 +- zml/Sharding.zig | 688 +++++++++++++++++-------------------- zml/buffer.zig | 62 ++-- zml/io.zig | 41 +-- zml/platform.zig | 5 +- 6 files changed, 377 insertions(+), 450 deletions(-) diff --git a/devenv.lock b/devenv.lock index afbf1114..bb3ef660 100644 --- a/devenv.lock +++ b/devenv.lock @@ -3,11 +3,11 @@ "devenv": { "locked": { "dir": "src/modules", - "lastModified": 1774261302, - "narHash": "sha256-P5noO+IXXDBL7l8dEVcig94OXfzLiN1K1KT0COjPcTU=", + "lastModified": 1779636362, + "narHash": "sha256-Q/hy/ybvbe91fjA1MbulUwPjJH9yKaFZnSaTkwLE8S0=", "owner": "cachix", "repo": "devenv", - "rev": "406f2ed97937995234c5d3de43cf77f31c42ca2b", + "rev": "9d6e68dac81fab1f7f2ec6784220151b4fba4b59", "type": "github" }, "original": { @@ -22,11 +22,11 @@ "nixpkgs-src": "nixpkgs-src" }, "locked": { - "lastModified": 1773704619, - "narHash": "sha256-LKtmit8Sr81z8+N2vpIaN/fyiQJ8f7XJ6tMSKyDVQ9s=", + "lastModified": 1778507786, + "narHash": "sha256-HzSQCKMsMr8r55LwM1JuzIOB+8bzk0FEv6sItKvsfoY=", "owner": "cachix", "repo": "devenv-nixpkgs", - "rev": "906534d75b0e2fe74a719559dfb1ad3563485f43", + "rev": "8f24a228a782e24576b155d1e39f0d914b380691", "type": "github" }, "original": { @@ -39,11 +39,11 @@ "nixpkgs-src": { "flake": false, "locked": { - "lastModified": 1773597492, - "narHash": "sha256-hQ284SkIeNaeyud+LS0WVLX+WL2rxcVZLFEaK0e03zg=", + "lastModified": 1778274207, + "narHash": "sha256-I4puXmX1iovcCHZlRmztO3vW0mAbbRvq4F8wgIMQ1MM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "a07d4ce6bee67d7c838a8a5796e75dff9caa21ef", + "rev": "b3da656039dc7a6240f27b2ef8cc6a3ef3bccae7", "type": "github" }, "original": { diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index 327244cb..cc028539 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -231,16 +231,9 @@ pub fn main(init: std.process.Init) !void { var input_buf = try createSequenceBuffer(allocator, io, platform, input_shape, sharding, 1000.0); defer input_buf.deinit(); - var input_placement = try input_buf.placement(allocator); - defer input_placement.deinit(allocator); - var w_placement = try w_buf.placement(allocator); - defer w_placement.deinit(allocator); - var b_placement = try b_buf.placement(allocator); - defer b_placement.deinit(allocator); - - log.info("input placement: {f}", .{input_placement}); - log.info("weight placement: {f}", .{w_placement}); - log.info("bias placement: {f}", .{b_placement}); + log.info("input placement: {f}", .{input_buf}); + log.info("weight placement: {f}", .{w_buf}); + log.info("bias placement: {f}", .{b_buf}); var exe_args = try exe.args(allocator); defer exe_args.deinit(allocator); diff --git a/zml/Sharding.zig b/zml/Sharding.zig index 27dc2845..d9746629 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -32,6 +32,10 @@ pub fn numPartitionsForLogicalAxis(sharding: Sharding, logical_axis: anytype) i6 return sharding.data.numPartitionsForLogicalAxis(logical_axis); } +pub fn devicesInCanonicalOrder(sharding: Sharding) []const Device { + return sharding.data.physical.devices_in_canonical_order; +} + pub fn format(sharding: Sharding, writer: *std.Io.Writer) std.Io.Writer.Error!void { return sharding.data.format(writer); } @@ -103,9 +107,10 @@ pub const Partitioning = struct { pub fn localShapeForShape(self: Partitioning, shape: Shape) !Shape { const sharding = try self.selectSharding(shape); - var shards = Placement.shardIterator(sharding, shape); - const shard = (try shards.next()) orelse return error.InvalidPhysicalMesh; - return shard.shape.withDefaultPartitioning(); + const ordered_devices = sharding.devicesInCanonicalOrder(); + if (ordered_devices.len == 0) return error.InvalidPhysicalMesh; + const placement_ = try sharding.placement(shape, ordered_devices[0]); + return placement_.shape.withDefaultPartitioning(); } pub fn numPartitionsForLogicalAxis(self: Partitioning, shape: Shape, logical_axis: anytype) !i64 { @@ -229,8 +234,6 @@ pub const Device = struct { } }; -pub const Devices = stdx.BoundedArray(Device, Platform.MAX_NUM_DEVICES); - /// Physical axis tags represent hardware interconnect tiers. pub const PhysicalAxisTag = enum { link, // Intra-device / island-local 1D interconnect @@ -418,31 +421,55 @@ pub const PhysicalMesh = struct { target: Target, root: PhysicalNode, axis_traversal: AxisTraversal, + devices_in_canonical_order: []Device, /// Build a PhysicalMesh from a tree: /// - clone the tree into `allocator` /// - validate geometry invariants /// - assign coordinates to leaves - /// - compute axis traversal + /// - compute axis traversal and canonical device order pub fn fromTree(allocator: std.mem.Allocator, target: Target, root: PhysicalNode) !PhysicalMesh { const cloned = try cloneNode(allocator, root); errdefer freeNode(allocator, cloned); - return try fromOwnedTree(target, cloned); + return try fromOwnedTree(allocator, target, cloned); } - fn fromOwnedTree(target: Target, root: PhysicalNode) !PhysicalMesh { + fn fromOwnedTree(allocator: std.mem.Allocator, target: Target, root: PhysicalNode) !PhysicalMesh { try validateGeometry(root); var owned_root = root; var path: [Shape.MAX_RANK]usize = @splat(0); assignCoords(&owned_root, &path, 0); - return .{ + var mesh: PhysicalMesh = .{ .target = target, .root = owned_root, .axis_traversal = .init(owned_root), + .devices_in_canonical_order = undefined, }; + try mesh.populateDevicesInCanonicalOrder(allocator); + return mesh; + } + + fn populateDevicesInCanonicalOrder(self: *PhysicalMesh, allocator: std.mem.Allocator) !void { + var devices_: std.ArrayList(Device) = .empty; + errdefer devices_.deinit(allocator); + try devicesNodeInto(allocator, self.root, &devices_); + self.devices_in_canonical_order = try devices_.toOwnedSlice(allocator); + + const Order = struct { + mesh: *const PhysicalMesh, + fn lessThan(ctx: @This(), a: Device, b: Device) bool { + const ca = a.coordsSlice() orelse unreachable; + const cb = b.coordsSlice() orelse unreachable; + const ia = ctx.mesh.linearIndexFromCoords(ca); + const ib = ctx.mesh.linearIndexFromCoords(cb); + return ia < ib; + } + }; + + std.mem.sort(Device, self.devices_in_canonical_order, Order{ .mesh = self }, Order.lessThan); } fn cloneNode(allocator: std.mem.Allocator, node: PhysicalNode) !PhysicalNode { @@ -481,6 +508,12 @@ pub const PhysicalMesh = struct { } } + pub fn deinit(self: *PhysicalMesh, allocator: std.mem.Allocator) void { + allocator.free(self.devices_in_canonical_order); + freeNode(allocator, self.root); + self.* = undefined; + } + fn validateGeometry(node: PhysicalNode) !void { switch (node) { .leaf => return, @@ -550,18 +583,12 @@ pub const PhysicalMesh = struct { return self.root.countDevices(); } - pub fn devices(self: PhysicalMesh) Devices { - var list: Devices = .empty; - devicesNodeInto(self.root, &list); - return list; - } - - fn devicesNodeInto(node: PhysicalNode, out: *Devices) void { + fn devicesNodeInto(allocator: std.mem.Allocator, node: PhysicalNode, out: *std.ArrayList(Device)) !void { switch (node) { - .leaf => |d| out.appendAssumeCapacity(d), + .leaf => |d| try out.append(allocator, d), .branch => |b| { for (b.children) |child| { - devicesNodeInto(child, out); + try devicesNodeInto(allocator, child, out); } }, } @@ -650,7 +677,7 @@ pub const PhysicalMesh = struct { return false; } - pub fn format(self: *const PhysicalMesh, writer: *std.Io.Writer) !void { + pub fn format(self: PhysicalMesh, writer: *std.Io.Writer) !void { try writer.print("\nPhysicalMesh(platform={s} shardable_axes={any} num_devices={d})\n", .{ @tagName(self.target), self.shardableAxes(), self.countDevices() }); try writer.print("├── {f}\n", .{self.axis_traversal}); try writer.print("│ \n", .{}); @@ -734,15 +761,15 @@ pub const PhysicalMesh = struct { if (rank == 0) return error.InvalidDeviceCoords; if (rank > max_rank) return error.UnsupportedDeviceCoordsRank; - for (placements[1..]) |placement| { - if (placement.coords.len != rank) return error.InvalidDeviceCoordsRank; + for (placements[1..]) |coord_placement| { + if (coord_placement.coords.len != rank) return error.InvalidDeviceCoordsRank; } var axis_sizes = [_]usize{1} ** Shape.MAX_RANK; for (0..rank) |ax_i| { var max_coord: usize = 0; - for (placements) |placement| { - max_coord = @max(max_coord, placement.coords.constSlice()[ax_i]); + for (placements) |coord_placement| { + max_coord = @max(max_coord, coord_placement.coords.constSlice()[ax_i]); } axis_sizes[ax_i] = max_coord + 1; } @@ -767,10 +794,10 @@ pub const PhysicalMesh = struct { pub fn sorted(allocator: std.mem.Allocator, placements: []const CoordPlacement, rank: usize, axis_sizes: []const usize) ![]IndexedCoord { const indexed = try allocator.alloc(IndexedCoord, placements.len); - for (placements, indexed) |placement, *out| { + for (placements, indexed) |coord_placement, *out| { out.* = .{ - .placement = placement, - .linear = linearIndex(placement.coords.constSlice(), rank, axis_sizes), + .placement = coord_placement, + .linear = linearIndex(coord_placement.coords.constSlice(), rank, axis_sizes), }; } @@ -788,12 +815,12 @@ pub const PhysicalMesh = struct { return indexed; } - pub fn leaf(placement: *const CoordPlacement, coords_slice: []const usize) !PhysicalNode { + pub fn leaf(coord_placement: *const CoordPlacement, coords_slice: []const usize) !PhysicalNode { return .{ .leaf = .{ - .id = placement.device_id, + .id = coord_placement.device_id, .coords = try .fromSlice(coords_slice), - .pjrt_device = placement.pjrt_device, + .pjrt_device = coord_placement.pjrt_device, }, }; } @@ -822,8 +849,8 @@ pub const PhysicalMesh = struct { base_offset: usize, ) !PhysicalNode { if (depth == axis_sizes.len) { - const placement = &indexed[base_offset].placement; - return leaf(placement, placement.coords.constSlice()[0..axis_sizes.len]); + const coord_placement = &indexed[base_offset].placement; + return leaf(coord_placement, coord_placement.coords.constSlice()[0..axis_sizes.len]); } const children = try allocator.alloc(PhysicalNode, axis_sizes[depth]); @@ -866,7 +893,7 @@ pub const PhysicalMesh = struct { }; errdefer freeNode(allocator, root); - return try fromOwnedTree(target, root); + return try fromOwnedTree(allocator, target, root); } pub fn cpu(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) !Tree { @@ -987,8 +1014,8 @@ pub const PhysicalMesh = struct { next_device_: *usize, ) !PhysicalNode { if (depth == axis_tags_.len) { - const placement = placements[next_device_.*]; - const device = platform_devices_[placement.input_index]; + const placement_ = placements[next_device_.*]; + const device = platform_devices_[placement_.input_index]; next_device_.* += 1; var coords: stdx.BoundedArray(usize, Shape.MAX_RANK) = .empty; @@ -996,7 +1023,7 @@ pub const PhysicalMesh = struct { return .{ .leaf = .{ - .id = placement.nc_id, + .id = placement_.nc_id, .coords = coords, .pjrt_device = device.pjrt_device, }, @@ -1186,7 +1213,7 @@ pub const PhysicalView = struct { pub const Data = struct { name: []const u8, - physical: PhysicalMesh, + physical: *const PhysicalMesh, logical: LogicalMesh, /// Compact binding table: logical axis -> physical axes @@ -1206,7 +1233,7 @@ pub const Data = struct { pub fn init( name: []const u8, - physical: PhysicalMesh, + physical: *const PhysicalMesh, logical: LogicalMesh, strategy: Strategy, ) !Data { @@ -1522,8 +1549,7 @@ pub const Data = struct { var ids = try allocator.alloc(usize, count); @memset(ids, std.math.maxInt(usize)); - const ordered_devices = self.devicesInCanonicalOrder(); - for (ordered_devices.constSlice()) |d| { + for (self.physical.devices_in_canonical_order) |d| { const coords = d.coordsSlice() orelse return error.MissingDeviceCoords; const idx = self.physical.linearIndexFromCoords(coords); ids[idx] = d.id; @@ -1536,25 +1562,6 @@ pub const Data = struct { return ids; } - fn devicesInCanonicalOrder(self: *const Data) Devices { - const physical = self.physical; - var devices: Devices = physical.devices(); - - const Order = struct { - mesh: PhysicalMesh, - fn lessThan(ctx: @This(), a: Device, b: Device) bool { - const ca = a.coordsSlice() orelse unreachable; - const cb = b.coordsSlice() orelse unreachable; - const ia = ctx.mesh.linearIndexFromCoords(ca); - const ib = ctx.mesh.linearIndexFromCoords(cb); - return ia < ib; - } - }; - - std.mem.sort(Device, devices.slice(), Order{ .mesh = physical }, Order.lessThan); - return devices; - } - pub fn format(self: Data, writer: *std.Io.Writer) !void { try writer.print("Sharding(name={s})\n", .{self.name}); @@ -1672,7 +1679,7 @@ pub const Strategy = struct { } /// suggest builds a Strategy from logical intents. - pub fn suggest(logical: LogicalMesh, physical: PhysicalMesh) Strategy { + pub fn suggest(logical: LogicalMesh, physical: *const PhysicalMesh) Strategy { var strategy: Strategy = .init; const base_order = physical.shardableAxes(); @@ -1708,6 +1715,35 @@ pub const Strategy = struct { } }; +pub fn placement(sharding: Sharding, shape: Shape, device: Device) !Placement { + if (device.coords.len == 0) return error.MissingDeviceCoords; + + var placement_: Placement = .{ + .device_id = device.id, + .device_coords = device.coords, + .shape = undefined, + .global_shape = shape, + .slices = .empty, + }; + var used_axes: std.EnumSet(PhysicalAxisTag) = .empty; + + for (0..shape.rank()) |ax| { + const axis_index: u8 = @intCast(ax); + const slice = try sliceForDevice(sharding, shape, placement_.device_coords.constSlice(), &used_axes, axis_index); + placement_.slices.appendAssumeCapacity(slice); + } + + placement_.shape = blk: { + var sh = shape; + for (placement_.slices.constSlice()) |slice| { + sh = sh.set(slice.axis, slice.size); + } + break :blk sh; + }; + + return placement_; +} + pub const Placement = struct { pub const Slice1d = struct { axis: u8, @@ -1715,327 +1751,225 @@ pub const Placement = struct { size: i64, }; - pub const Shard = struct { - device_id: usize, - device_coords: stdx.BoundedArray(usize, Shape.MAX_RANK), - shape: Shape, - global_shape: Shape, - slices: stdx.BoundedArray(Slice1d, Shape.MAX_RANK), - - pub fn platformDeviceIndex(self: *const Shard, platform: *const Platform) ?usize { - for (platform.devices, 0..) |d, device_index| { - const device_id = switch (platform.target) { - .neuron => @as(usize, @intCast(d.localHardwareId())), - .cuda, .rocm, .tpu, .cpu, .oneapi => d.id(), - }; - if (device_id == self.device_id) return device_index; - } - - return null; - } - - pub fn device(self: *const Shard, platform: *const Platform) PlatformDevice { - const device_index = self.platformDeviceIndex(platform) orelse unreachable; - return platform.devices[device_index]; - } - - pub fn memory(self: *const Shard, platform: *const Platform, kind: Memory.Kind) *const Memory { - return self.device(platform).memory(kind); - } - - pub fn shardSlice(self: *const Shard, slice: Slice) Slice { - stdx.debug.assert( - self.global_shape.eql(slice.shape), - "Placement global shape {f} doesn't match slice shape {f}", - .{ self.global_shape, slice.shape }, - ); - - var res = slice; - for (self.slices.constSlice()) |s| { - res = res.subSlice(s.axis, s.start, s.size); - } - return res; - } - - pub fn format(self: Shard, writer: *std.Io.Writer) !void { - try writer.print("device_id={d} coords={any}", .{ self.device_id, self.device_coords.constSlice() }); - - try writer.writeAll(" slices=["); - for (self.slices.constSlice(), 0..) |s, i| { - if (i > 0) try writer.writeAll(", "); - const axis_label = self.global_shape.debugTag(s.axis); - try writer.print("{s}:[{d}:{d}]", .{ axis_label, s.start, s.start + s.size }); - } - try writer.writeAll("]"); - } - }; - pub const Shards = []Shard; - - const AxisSplit = struct { - product: i64, - counts: stdx.BoundedArray(i64, Shape.MAX_RANK), - indices: stdx.BoundedArray(i64, Shape.MAX_RANK), - - pub const empty: AxisSplit = .{ - .product = 1, - .counts = .empty, - .indices = .empty, - }; - - pub fn add(split: *AxisSplit, size: i64, coord: usize) void { - split.counts.appendAssumeCapacity(size); - split.indices.appendAssumeCapacity(@intCast(coord)); - split.product *= size; - } - - pub fn linearIndex(self: AxisSplit) i64 { - var linear: i64 = 0; - for (self.counts.constSlice(), self.indices.constSlice()) |c_, iidx| { - linear = linear * c_ + iidx; - } - return linear; - } - }; - - pub const ShardIterator = struct { - sharding: Sharding, - shape: Shape, - ordered_devices: Devices, - index: usize = 0, - - pub fn count(self: *const ShardIterator) usize { - return self.ordered_devices.len; - } - - pub fn next(self: *ShardIterator) !?Shard { - if (self.index >= self.ordered_devices.len) return null; - defer self.index += 1; - - return try shardForDevice(self.sharding, self.shape, self.ordered_devices.get(self.index)); - } - }; - - sharding: Sharding, + device_id: usize, + device_coords: stdx.BoundedArray(usize, Shape.MAX_RANK), shape: Shape, - shards: Shards, + global_shape: Shape, + slices: stdx.BoundedArray(Slice1d, Shape.MAX_RANK), - pub fn shardIterator(sharding: Sharding, shape: Shape) ShardIterator { - return .{ - .sharding = sharding, - .shape = shape, - .ordered_devices = sharding.data.devicesInCanonicalOrder(), - }; - } - - /// Caller owns `shards` and must call `deinit` with the same allocator. - pub fn init( - allocator: std.mem.Allocator, - sharding: Sharding, - shape: Shape, - ) !Placement { - const ordered_devices = sharding.data.devicesInCanonicalOrder(); - const shards = try allocator.alloc(Shard, ordered_devices.len); - errdefer allocator.free(shards); - - for (ordered_devices.constSlice(), 0..) |d, i| { - shards[i] = try shardForDevice(sharding, shape, d); + pub fn platformDeviceIndex(self: *const Placement, platform: *const Platform) ?usize { + for (platform.devices, 0..) |d, device_index| { + const device_id = switch (platform.target) { + .neuron => @as(usize, @intCast(d.localHardwareId())), + .cuda, .rocm, .tpu, .cpu, .oneapi => d.id(), + }; + if (device_id == self.device_id) return device_index; } - return .{ - .sharding = sharding, - .shape = shape, - .shards = shards, - }; + return null; } - pub fn deinit(self: *Placement, allocator: std.mem.Allocator) void { - allocator.free(self.shards); + pub fn device(self: *const Placement, platform: *const Platform) PlatformDevice { + const device_index = self.platformDeviceIndex(platform) orelse unreachable; + return platform.devices[device_index]; } - fn shardForDevice(sharding: Sharding, shape: Shape, device: Device) !Shard { - if (device.coords.len == 0) return error.MissingDeviceCoords; + pub fn memory(self: *const Placement, platform: *const Platform, kind: Memory.Kind) *const Memory { + return self.device(platform).memory(kind); + } - var shard: Shard = .{ - .device_id = device.id, - .device_coords = device.coords, - .shape = undefined, - .global_shape = shape, - .slices = .empty, - }; - var used_axes: std.EnumSet(PhysicalAxisTag) = .empty; + pub fn shardSlice(self: *const Placement, slice: Slice) Slice { + stdx.debug.assert( + self.global_shape.eql(slice.shape), + "Placement global shape {f} doesn't match slice shape {f}", + .{ self.global_shape, slice.shape }, + ); - for (0..shape.rank()) |ax| { - const axis_index: u8 = @intCast(ax); - const slice = try sliceForDevice(sharding, shape, shard.device_coords.constSlice(), &used_axes, axis_index); - shard.slices.appendAssumeCapacity(slice); + var res = slice; + for (self.slices.constSlice()) |s| { + res = res.subSlice(s.axis, s.start, s.size); } - - shard.shape = blk: { - var sh = shape; - for (shard.slices.constSlice()) |slice| { - sh = sh.set(slice.axis, slice.size); - } - break :blk sh; - }; - - return shard; - } - - fn sliceForDevice( - sharding: Sharding, - shape: Shape, - device_coords: []const usize, - used_axes: *std.EnumSet(PhysicalAxisTag), - axis_index: u8, - ) !Slice1d { - const dim = shape.dim(axis_index); - const spec = shape.partition(axis_index); - - switch (spec) { - .axis => |logical_tag| { - const binding = sharding.data.binding(logical_tag) orelse return error.MissingLogicalBinding; - - // Calculate the split based on the physical coordinates of the current device - const plan = try calculateSplit(sharding, dim, binding, device_coords, used_axes); - - const size = @divExact(dim, plan.product); - const start = plan.linearIndex() * size; - - return .{ - .axis = axis_index, - .start = start, - .size = size, - }; - }, - else => return .{ .axis = axis_index, .start = 0, .size = dim }, - } - } - - fn calculateSplit( - sharding: Sharding, - dim: i64, - binding: []const PhysicalAxisTag, - device_coords: []const usize, - used_axes: *std.EnumSet(PhysicalAxisTag), - ) !AxisSplit { - var plan: AxisSplit = .empty; - - for (binding) |p_tag| { - // Expand the physical tag: check if it's a target for folded source axes - if (sharding.data.foldSources(p_tag)) |sources| { - for (sources) |src| { - if (used_axes.contains(src)) continue; - addPhysicalToSplit(sharding, &plan, src, device_coords); - used_axes.insert(src); - } - } else { - // Standard case: directly bound, not part of an explicit fold - if (used_axes.contains(p_tag)) continue; - addPhysicalToSplit(sharding, &plan, p_tag, device_coords); - used_axes.insert(p_tag); - } - } - - // Validate divisibility - if (plan.product > 0 and @rem(dim, plan.product) != 0) { - return error.IncompatibleSharding; - } - return plan; - } - - /// Extract coordinate data from the physical mesh for a specific axis. - fn addPhysicalToSplit(sharding: Sharding, plan: *AxisSplit, tag: PhysicalAxisTag, device_coords: []const usize) void { - const physical = sharding.data.physical; - const info = physical.axisInfo(tag) orelse return; - const depth = physical.axis_traversal.depth(tag) orelse return; - const coord = device_coords[depth]; - - plan.add(info.size, coord); + return res; } pub fn format(self: Placement, writer: *std.Io.Writer) !void { - const Iterator = struct { - shards: []const Shard, - index: usize = 0, + try writer.print("device_id={d} coords={any}", .{ self.device_id, self.device_coords.constSlice() }); - fn count(it: @This()) usize { - return it.shards.len; - } - - fn next(it: *@This()) !?Shard { - if (it.index >= it.shards.len) return null; - defer it.index += 1; - - return it.shards[it.index]; - } - }; - return formatFromShardSource(self.shape, Iterator{ .shards = self.shards }, writer); - } - - pub fn formatShards(sharding: Sharding, shape: Shape, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return formatFromShardSource(shape, shardIterator(sharding, shape), writer); - } - - fn formatFromShardSource(shape: Shape, shards_: anytype, writer: *std.Io.Writer) std.Io.Writer.Error!void { - var shards = shards_; - const first_shard = shards.next() catch |err| { - return formatShardError(shape, shards.count(), err, writer); - }; - - try writer.print("Placement(shape={f} shards={d})\n", .{ shape, shards.count() }); - - try writer.writeAll("Axis partitioning summary:\n"); - for (0..shape.rank()) |ax| { - const dim = shape.dim(ax); - const spec = shape.partition(ax); - const axis_label = shape.debugTag(ax); - - var shard_size: i64 = dim; - if (first_shard) |shard| { - for (shard.slices.constSlice()) |s| { - if (s.axis == ax) { - shard_size = s.size; - break; - } - } - } - - const shards_count: i64 = if (shard_size > 0 and @rem(dim, shard_size) == 0) - @divExact(dim, shard_size) - else - 0; - - try writer.print( - "└─ {s}: dim={d}, spec={s}, shard_size={d}, shards={d}\n", - .{ axis_label, dim, @tagName(spec), shard_size, shards_count }, - ); + try writer.writeAll(" slices=["); + for (self.slices.constSlice(), 0..) |s, i| { + if (i > 0) try writer.writeAll(", "); + const axis_label = self.global_shape.debugTag(s.axis); + try writer.print("{s}:[{d}:{d}]", .{ axis_label, s.start, s.start + s.size }); } - - try writer.writeAll("Per-shard placement:\n"); - var shard_index: usize = 0; - if (first_shard) |shard| { - try writer.print("└─ Shard[{d}] {f}\n", .{ shard_index, shard }); - shard_index += 1; - } - while (shards.next() catch |err| { - try writer.print("└─ Shard[{d}] error={s}\n", .{ shard_index, @errorName(err) }); - return; - }) |shard| : (shard_index += 1) { - try writer.print("└─ Shard[{d}] {f}\n", .{ shard_index, shard }); - } - } - - fn formatShardError( - shape: Shape, - shard_count: usize, - err: anyerror, - writer: *std.Io.Writer, - ) std.Io.Writer.Error!void { - try writer.print("Placement(shape={f} shards={d})\n", .{ shape, shard_count }); - try writer.print("Shard placement error: {s}\n", .{@errorName(err)}); + try writer.writeAll("]"); } }; +const AxisSplit = struct { + product: i64, + counts: stdx.BoundedArray(i64, Shape.MAX_RANK), + indices: stdx.BoundedArray(i64, Shape.MAX_RANK), + + pub const empty: AxisSplit = .{ + .product = 1, + .counts = .empty, + .indices = .empty, + }; + + pub fn add(split: *AxisSplit, size: i64, coord: usize) void { + split.counts.appendAssumeCapacity(size); + split.indices.appendAssumeCapacity(@intCast(coord)); + split.product *= size; + } + + pub fn linearIndex(self: AxisSplit) i64 { + var linear: i64 = 0; + for (self.counts.constSlice(), self.indices.constSlice()) |c_, iidx| { + linear = linear * c_ + iidx; + } + return linear; + } +}; + +fn sliceForDevice( + sharding: Sharding, + shape: Shape, + device_coords: []const usize, + used_axes: *std.EnumSet(PhysicalAxisTag), + axis_index: u8, +) !Placement.Slice1d { + const dim = shape.dim(axis_index); + const spec = shape.partition(axis_index); + + switch (spec) { + .axis => |logical_tag| { + const binding = sharding.data.binding(logical_tag) orelse return error.MissingLogicalBinding; + + // Calculate the split based on the physical coordinates of the current device. + const plan = try calculateSplit(sharding, dim, binding, device_coords, used_axes); + + const size = @divExact(dim, plan.product); + const start = plan.linearIndex() * size; + + return .{ + .axis = axis_index, + .start = start, + .size = size, + }; + }, + else => return .{ .axis = axis_index, .start = 0, .size = dim }, + } +} + +fn calculateSplit( + sharding: Sharding, + dim: i64, + binding: []const PhysicalAxisTag, + device_coords: []const usize, + used_axes: *std.EnumSet(PhysicalAxisTag), +) !AxisSplit { + var plan: AxisSplit = .empty; + + for (binding) |p_tag| { + // Expand the physical tag: check if it's a target for folded source axes. + if (sharding.data.foldSources(p_tag)) |sources| { + for (sources) |src| { + if (used_axes.contains(src)) continue; + addPhysicalToSplit(sharding, &plan, src, device_coords); + used_axes.insert(src); + } + } else { + // Standard case: directly bound, not part of an explicit fold. + if (used_axes.contains(p_tag)) continue; + addPhysicalToSplit(sharding, &plan, p_tag, device_coords); + used_axes.insert(p_tag); + } + } + + if (plan.product > 0 and @rem(dim, plan.product) != 0) { + return error.IncompatibleSharding; + } + return plan; +} + +/// Extract coordinate data from the physical mesh for a specific axis. +fn addPhysicalToSplit(sharding: Sharding, plan: *AxisSplit, tag: PhysicalAxisTag, device_coords: []const usize) void { + const physical = sharding.data.physical; + const info = physical.axisInfo(tag) orelse return; + const depth = physical.axis_traversal.depth(tag) orelse return; + const coord = device_coords[depth]; + + plan.add(info.size, coord); +} + +pub fn fmtPlacements(sharding: Sharding, shape: Shape) PlacementsFormatter { + return .{ .sharding = sharding, .shape = shape }; +} + +pub const PlacementsFormatter = struct { + sharding: Sharding, + shape: Shape, + + pub fn format(self: PlacementsFormatter, writer: *std.Io.Writer) std.Io.Writer.Error!void { + const ordered_devices = self.sharding.devicesInCanonicalOrder(); + const first_shard: ?Placement = if (ordered_devices.len > 0) + self.sharding.placement(self.shape, ordered_devices[0]) catch |err| return formatPlacementError(self.shape, ordered_devices.len, err, writer) + else + null; + + try formatPlacementSummary(self.shape, ordered_devices.len, first_shard, writer); + try writer.writeAll("Per-shard placement:\n"); + for (ordered_devices, 0..) |device, shard_index| { + const placement_ = self.sharding.placement(self.shape, device) catch |err| { + try writer.print("└─ Shard[{d}] error={s}\n", .{ shard_index, @errorName(err) }); + return; + }; + try writer.print("└─ Shard[{d}] {f}\n", .{ shard_index, placement_ }); + } + } +}; + +fn formatPlacementSummary(shape: Shape, shard_count: usize, first_shard: ?Placement, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("Placement(shape={f} shards={d})\n", .{ shape, shard_count }); + + try writer.writeAll("Axis partitioning summary:\n"); + for (0..shape.rank()) |ax| { + const dim = shape.dim(ax); + const spec = shape.partition(ax); + const axis_label = shape.debugTag(ax); + + var shard_size: i64 = dim; + if (first_shard) |shard| { + for (shard.slices.constSlice()) |s| { + if (s.axis == ax) { + shard_size = s.size; + break; + } + } + } + + const shards_count: i64 = if (shard_size > 0 and @rem(dim, shard_size) == 0) + @divExact(dim, shard_size) + else + 0; + + try writer.print( + "└─ {s}: dim={d}, spec={s}, shard_size={d}, shards={d}\n", + .{ axis_label, dim, @tagName(spec), shard_size, shards_count }, + ); + } +} + +fn formatPlacementError( + shape: Shape, + shard_count: usize, + err: anyerror, + writer: *std.Io.Writer, +) std.Io.Writer.Error!void { + try writer.print("Placement(shape={f} shards={d})\n", .{ shape, shard_count }); + try writer.print("Shard placement error: {s}\n", .{@errorName(err)}); +} + const ShardingTest = struct { pub const ExpectedShard = struct { device_id: usize, @@ -2109,17 +2043,19 @@ const ShardingTest = struct { } // Verify Placement logic / Error + const sharding_ref: Sharding = .{ .data = &sharding }; + const ordered_devices = sharding_ref.devicesInCanonicalOrder(); if (s.expect_error) |err| { - try std.testing.expectError(err, Placement.init(self.allocator, .{ .data = &sharding }, s.shape)); + try std.testing.expect(ordered_devices.len > 0); + try std.testing.expectError(err, sharding_ref.placement(s.shape, ordered_devices[0])); return; } // Verify Shard Slices (Math) if (s.expected_shards.len > 0) { - var placement = try Placement.init(self.allocator, .{ .data = &sharding }, s.shape); - defer placement.deinit(self.allocator); - try std.testing.expectEqual(s.expected_shards.len, placement.shards.len); - for (s.expected_shards, placement.shards) |expected, actual| { + try std.testing.expectEqual(s.expected_shards.len, ordered_devices.len); + for (s.expected_shards, ordered_devices) |expected, device| { + const actual = try sharding_ref.placement(s.shape, device); try std.testing.expectEqual(expected.device_id, actual.device_id); for (expected.slices, actual.slices.constSlice()) |exp_slice, actual_slice| { try std.testing.expectEqual(exp_slice[0], actual_slice.start); @@ -2144,7 +2080,7 @@ test "sharding: unknown partitioning implies replication" { const scenario: ShardingTest.Scenario = .{ .sharding = try .init( "folded_mesh", - physical, + &physical, logical, strategy, ), @@ -2172,10 +2108,10 @@ test "sharding: suggest strategy realization" { .model = .high_bandwidth, }); - const strategy: Strategy = .suggest(logical, physical); + const strategy: Strategy = .suggest(logical, &physical); const scenario: ShardingTest.Scenario = .{ - .sharding = try .init("suggested_mesh", physical, logical, strategy), + .sharding = try .init("suggested_mesh", &physical, logical, strategy), .shape = Shape.init(.{ .batch = 4, .model = 4 }, .f32) .withPartitioning(.{ .batch = .batch, .model = .model }), .expected_sdy = "#sdy.sharding<@suggested_mesh, [{\"link_z\"}, {\"link_x\"}], replicated={\"link_y\"}>", @@ -2205,10 +2141,10 @@ test "sharding: suggest folds logical axes with same intent" { .experts = .high_bandwidth, }); - const strategy: Strategy = .suggest(logical, physical); + const strategy: Strategy = .suggest(logical, &physical); const scenario: ShardingTest.Scenario = .{ - .sharding = try .init("fold_mesh", physical, logical, strategy), + .sharding = try .init("fold_mesh", &physical, logical, strategy), .shape = Shape.init(.{ .model = 4, .experts = 4 }, .f32) .withPartitioning(.{ .model = .model, .experts = .experts }), .expected_sdy = "#sdy.sharding<@fold_mesh, [{\"link_x\"}, {}]>", @@ -2235,7 +2171,7 @@ test "sharding: multiple physical axes on one logical dimension (folding)" { strategy.addBinding(.model, .link_y); const scenario: ShardingTest.Scenario = .{ - .sharding = try .init("folded_mesh", physical, logical, strategy), + .sharding = try .init("folded_mesh", &physical, logical, strategy), .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(.{ .model = .model }), .expected_sdy = "#sdy.sharding<@folded_mesh, [{\"link_x\", \"link_y\"}]>", .expected_shards = &.{ @@ -2261,7 +2197,7 @@ test "sharding: explicit strategy folding" { strategy.addFold(.link_x, &.{ .link_x, .link_z }); const scenario: ShardingTest.Scenario = .{ - .sharding = try .init("strategy_fold", physical, logical, strategy), + .sharding = try .init("strategy_fold", &physical, logical, strategy), .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(.{ .model = .model }), .expected_sdy = "#sdy.sharding<@strategy_fold, [{\"link_x\"}], replicated={\"link_y\"}>", .expected_shards = &.{ @@ -2291,7 +2227,7 @@ test "sharding: open and replicated dimension mix" { strategy.addBinding(.model, .link_y); const scenario: ShardingTest.Scenario = .{ - .sharding = try .init("mix_mesh", physical, logical, strategy), + .sharding = try .init("mix_mesh", &physical, logical, strategy), .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(.{ .batch = .open, .model = .replicated }), .expected_sdy = "#sdy.sharding<@mix_mesh, [{?}, {}], replicated={\"link_x\", \"link_y\"}>", .expected_shards = &.{ @@ -2318,7 +2254,7 @@ test "sharding: full 3D cluster sharding" { strategy.addBinding(.context, .link_z); const scenario: ShardingTest.Scenario = .{ - .sharding = try .init("3d_mesh", physical, logical, strategy), + .sharding = try .init("3d_mesh", &physical, logical, strategy), .shape = Shape.init(.{ .batch = 4, .model = 4, .context = 4 }, .f32) .withPartitioning(.{ .batch = .batch, .model = .model, .context = .context }), .expected_sdy = "#sdy.sharding<@\"3d_mesh\", [{\"link_x\"}, {\"link_y\"}, {\"link_z\"}]>", @@ -2351,7 +2287,7 @@ test "sharding: num partitions for logical axis" { strategy.addBinding(.model, .link_x); strategy.addFold(.link_x, &.{ .link_x, .link_z }); - const sharding: Sharding.Data = try .init("axis_parts_mesh", physical, logical, strategy); + const sharding: Sharding.Data = try .init("axis_parts_mesh", &physical, logical, strategy); try std.testing.expectEqual(4, sharding.numPartitionsForLogicalAxis(.model)); try std.testing.expectEqual(1, sharding.numPartitionsForLogicalAxis(.batch)); diff --git a/zml/buffer.zig b/zml/buffer.zig index 8e0df09d..db650907 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -84,7 +84,7 @@ pub const Buffer = struct { } pub fn format(self: Buffer, writer: *std.Io.Writer) !void { - try Sharding.Placement.formatShards(self._sharding, self._shape, writer); + try writer.print("{f}", .{self._sharding.fmtPlacements(self._shape)}); } /// Copies the content of the given buffer from host memory to the accelerator memory. @@ -109,22 +109,22 @@ pub const Buffer = struct { const buffer_type = pjrtx.bufferTypeFromDtype(sh.dtype()); const slice = Slice.init(sh, data_); - var shard_iter = Sharding.Placement.shardIterator(res._sharding, res._shape); - while (try shard_iter.next()) |shard| { - const sub_slice = shard.shardSlice(slice); + for (res._sharding.devicesInCanonicalOrder()) |device| { + const placement = try res._sharding.placement(res._shape, device); + const sub_slice = placement.shardSlice(slice); var default_layout: ?pjrt.DefaultMemoryLayout = null; const layout: pjrt.MemoryLayout = switch (platform.target) { .tpu => blk: { default_layout = try platform.pjrt_client.defaultMemoryLayout( platform.pjrt_api, - pjrtx.bufferTypeFromDtype(shard.shape.dtype()), - shard.shape.dims(), + pjrtx.bufferTypeFromDtype(placement.shape.dtype()), + placement.shape.dims(), ); break :blk default_layout.?.toMemoryLayout(); }, else => .{ .tiled = .{ - .minor_to_major = constants.minorToMajor(shard.shape.rank()), + .minor_to_major = constants.minorToMajor(placement.shape.rank()), .tile_dims = &.{}, .tile_dims_sizes = &.{}, }, @@ -133,11 +133,11 @@ pub const Buffer = struct { const args: pjrt.Client.BufferFromHostBufferArgs = .{ .data = sub_slice.constData().ptr, .buffer_type = buffer_type, - .dims = shard.shape.dims(), + .dims = placement.shape.dims(), .byte_strides = sub_slice.byte_strides.constSlice(), .layout = layout, .host_buffer_semantics = .ImmutableUntilTransferCompletes, - .dst = .{ .memory = shard.memory(platform, opts.memory).pjrt_memory }, + .dst = .{ .memory = placement.memory(platform, opts.memory).pjrt_memory }, }; const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); @@ -218,31 +218,33 @@ pub const Buffer = struct { shard.deinit(platform.pjrt_api); }; - var shard_iter = Sharding.Placement.shardIterator(res._sharding, res._shape); - while (try shard_iter.next()) |shard| { + const minor_to_major = constants.minorToMajor(res._shape.rank()); // Rank doesn't change with sharding. + const element_type = pjrtx.bufferTypeFromDtype(res._shape.dtype()); // dtype doesn't change with sharding. + for (res._sharding.devicesInCanonicalOrder()) |device| { + const placement = try res._sharding.placement(res._shape, device); var default_layout: ?pjrt.DefaultMemoryLayout = null; const layout: pjrt.MemoryLayout = switch (platform.target) { .tpu => blk: { default_layout = try platform.pjrt_client.defaultMemoryLayout( platform.pjrt_api, - pjrtx.bufferTypeFromDtype(shard.shape.dtype()), - shard.shape.dims(), + pjrtx.bufferTypeFromDtype(placement.shape.dtype()), + placement.shape.dims(), ); break :blk default_layout.?.toMemoryLayout(); }, else => .{ .tiled = .{ - .minor_to_major = constants.minorToMajor(shard.shape.rank()), + .minor_to_major = minor_to_major, .tile_dims = &.{}, .tile_dims_sizes = &.{}, }, }, }; const args: pjrt.Client.CreateUninitializedBufferArgs = .{ - .dims = shard.shape.dims(), - .element_type = pjrtx.bufferTypeFromDtype(shard.shape.dtype()), + .dims = placement.shape.dims(), + .element_type = element_type, .layout = layout, - .dst = .{ .memory = shard.memory(platform, opts.memory).pjrt_memory }, + .dst = .{ .memory = placement.memory(platform, opts.memory).pjrt_memory }, }; const shard_buffer = try platform.pjrt_client.createUninitializedBuffer(platform.pjrt_api, args); @@ -279,13 +281,12 @@ pub const Buffer = struct { pub fn toSlice(self: Buffer, io: std.Io, slice: Slice) !void { stdx.debug.assert(self._shape.eql(slice.shape), "Buffer shape {f} doesn't match destination slice {f}", .{ self._shape, slice.shape }); - var shard_iter = Sharding.Placement.shardIterator(self._sharding, self._shape); - var shard_index: usize = 0; - while (try shard_iter.next()) |shard| : (shard_index += 1) { - const sub_slice = shard.shardSlice(slice); + for (self._sharding.devicesInCanonicalOrder(), 0..) |device, shard_index| { + const placement = try self._sharding.placement(self._shape, device); + const sub_slice = placement.shardSlice(slice); if (!sub_slice.isContiguous()) return error.NonContiguousShardRead; - const size_bytes = shard.shape.byteSize(); + const size_bytes = placement.shape.byteSize(); const start = sub_slice.offset_bytes; const end = start + size_bytes; stdx.debug.assert(end <= slice.constData().len, "Shard sub_slice exceeds destination slice", .{}); @@ -305,10 +306,9 @@ pub const Buffer = struct { const slice = try Slice.alloc(allocator, self.shape()); errdefer slice.free(allocator); - var shard_iter = Sharding.Placement.shardIterator(self._sharding, self._shape); - var shard_index: usize = 0; - while (try shard_iter.next()) |shard| : (shard_index += 1) { - const sub_slice = shard.shardSlice(slice); + for (self._sharding.devicesInCanonicalOrder(), 0..) |device, shard_index| { + const placement = try self._sharding.placement(self._shape, device); + const sub_slice = placement.shardSlice(slice); var shard_slice = try Slice.alloc(allocator, sub_slice.shape); defer shard_slice.free(allocator); @@ -327,15 +327,11 @@ pub const Buffer = struct { pub fn byteSize(self: Buffer) usize { var byte_size: usize = 0; - var shard_iter = Sharding.Placement.shardIterator(self._sharding, self._shape); - while (shard_iter.next() catch unreachable) |shard| { - byte_size += shard.shape.byteSize(); + for (self._sharding.devicesInCanonicalOrder()) |device| { + const placement = self._sharding.placement(self._shape, device) catch unreachable; + byte_size += placement.shape.byteSize(); } return byte_size; } - - pub fn placement(self: Buffer, allocator: std.mem.Allocator) !Sharding.Placement { - return Sharding.Placement.init(allocator, self._sharding, self._shape); - } }; diff --git a/zml/io.zig b/zml/io.zig index 33efdfa6..638c9bea 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -592,16 +592,16 @@ pub const DirectMemoryWriter = struct { allocator: std.mem.Allocator, shape: Shape, - placement: Placement, + sharding: Sharding, raw_segments: std.ArrayList(RawSegment) = .empty, segments: std.ArrayList(StreamSegment) = .empty, mirrors: std.ArrayList(usize) = .empty, - fn init(allocator: std.mem.Allocator, shape: Shape, placement: Placement) StreamPlanner { + fn init(allocator: std.mem.Allocator, shape: Shape, sharding: Sharding) StreamPlanner { return .{ .allocator = allocator, .shape = shape, - .placement = placement, + .sharding = sharding, }; } @@ -628,8 +628,9 @@ pub const DirectMemoryWriter = struct { } fn collectRawSegments(self: *StreamPlanner) !void { - for (self.placement.shards, 0..) |shard, writer_index| { - try self.appendShardSegments(shard, writer_index); + for (self.sharding.devicesInCanonicalOrder(), 0..) |device, writer_index| { + const placement = try self.sharding.placement(self.shape, device); + try self.appendShardSegments(placement, writer_index); } } @@ -690,8 +691,8 @@ pub const DirectMemoryWriter = struct { }); } - fn axisRange(self: *const StreamPlanner, shard: Placement.Shard, axis: usize) AxisRange { - for (shard.slices.constSlice()) |slice| { + fn axisRange(self: *const StreamPlanner, placement: Placement, axis: usize) AxisRange { + for (placement.slices.constSlice()) |slice| { if (slice.axis == axis) { return .{ .start = slice.start, @@ -708,14 +709,14 @@ pub const DirectMemoryWriter = struct { fn appendShardSegmentsAtAxis( self: *StreamPlanner, - shard: Placement.Shard, + placement: Placement, writer_index: usize, rank: usize, axis: usize, base_start: i64, strides: []const i64, ) !void { - const range = self.axisRange(shard, axis); + const range = self.axisRange(placement, axis); if (range.size == 0) return; if (axis + 1 == rank) { @@ -728,11 +729,11 @@ pub const DirectMemoryWriter = struct { var i: i64 = 0; while (i < range.size) : (i += 1) { const child_start = base_start + (range.start + i) * strides[axis]; - try self.appendShardSegmentsAtAxis(shard, writer_index, rank, axis + 1, child_start, strides); + try self.appendShardSegmentsAtAxis(placement, writer_index, rank, axis + 1, child_start, strides); } } - fn appendShardSegments(self: *StreamPlanner, shard: Placement.Shard, writer_index: usize) !void { + fn appendShardSegments(self: *StreamPlanner, placement: Placement, writer_index: usize) !void { const rank = self.shape.rank(); if (rank == 0) { try self.appendRawSegment(writer_index, 0, self.shape.byteSize()); @@ -740,7 +741,7 @@ pub const DirectMemoryWriter = struct { } const strides = self.shape.computeByteStrides(); - try self.appendShardSegmentsAtAxis(shard, writer_index, rank, 0, 0, strides.constSlice()); + try self.appendShardSegmentsAtAxis(placement, writer_index, rank, 0, 0, strides.constSlice()); } }; @@ -788,9 +789,8 @@ pub const DirectMemoryWriter = struct { sharding: Sharding, buffer: *Buffer, ) !DirectMemoryWriter { - var placement = try Placement.init(allocator, sharding, shape); - defer placement.deinit(allocator); - var shard_writers = try allocator.alloc(DirectShardWriter, placement.shards.len); + const ordered_devices = sharding.devicesInCanonicalOrder(); + var shard_writers = try allocator.alloc(DirectShardWriter, ordered_devices.len); errdefer allocator.free(shard_writers); var initialized: usize = 0; @@ -799,21 +799,22 @@ pub const DirectMemoryWriter = struct { }; var pjrt_buffers: Buffer.Shards = .empty; - for (placement.shards, 0..) |shard, i| { + for (ordered_devices, 0..) |device, i| { defer initialized += 1; - const device_index = shard.platformDeviceIndex(platform) orelse return error.UnknownShardDevice; + const placement = try sharding.placement(shape, device); + const device_index = placement.platformDeviceIndex(platform) orelse return error.UnknownShardDevice; const pool = &pools[device_index]; const shard_dma_allocator = dma_allocators[device_index].allocator(); - shard_writers[i] = try DirectShardWriter.init(shard_dma_allocator, io, shard.memory(platform, .default), pool, shard.shape); + shard_writers[i] = try DirectShardWriter.init(shard_dma_allocator, io, placement.memory(platform, .default), pool, placement.shape); pjrt_buffers.appendAssumeCapacity(shard_writers[i].pjrt_buffer); } buffer.* = Buffer.fromPjrtBuffers(platform, shape, sharding, pjrt_buffers.constSlice()); - var planner = StreamPlanner.init(allocator, shape, placement); + var planner = StreamPlanner.init(allocator, shape, sharding); defer planner.deinit(); const stream_plan = try planner.build(); @@ -1273,7 +1274,7 @@ const DirectMemoryWriterDeviceTest = struct { var platform = Platform.auto(self.allocator, self.io, scenario.create_options) catch return error.SkipZigTest; defer platform.deinit(self.allocator, self.io); - const sharding: Sharding.Data = try .init(scenario.name, platform.physical_mesh, scenario.logical_mesh, scenario.strategy); + const sharding: Sharding.Data = try .init(scenario.name, &platform.physical_mesh, scenario.logical_mesh, scenario.strategy); try self.runDirectMemoryWriter( platform, scenario.shape, diff --git a/zml/platform.zig b/zml/platform.zig index 7e359d61..00a24215 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -440,6 +440,7 @@ pub const Platform = struct { pub fn deinit(self: *Platform, allocator: std.mem.Allocator, io: std.Io) void { _ = io; _ = allocator; + self.physical_mesh.deinit(self.arena.allocator()); self.pjrt_client.deinit(self.pjrt_api); self.arena.deinit(); } @@ -531,7 +532,7 @@ pub const Platform = struct { /// Create a Sharding based on the given logical mesh and the default strategy. /// Memory is owned by the platform, making it safe to copy around. pub fn registerSharding(platform: *Platform, name: []const u8, logical: Sharding.LogicalMesh) !Sharding { - return platform.registerShardingWithStrategy(name, logical, .suggest(logical, platform.physical_mesh)); + return platform.registerShardingWithStrategy(name, logical, .suggest(logical, &platform.physical_mesh)); } /// Create a Sharding based on the given logical mesh and a strategy. @@ -545,7 +546,7 @@ pub const Platform = struct { const owned_name = try arena.dupe(u8, name); const owned_data = try arena.create(Sharding.Data); - owned_data.* = try .init(owned_name, platform.physical_mesh, logical, strategy); + owned_data.* = try .init(owned_name, &platform.physical_mesh, logical, strategy); const sharding: Sharding = .{ .data = owned_data }; entry.key_ptr.* = owned_name; entry.value_ptr.* = sharding; From b6e2b026bc317337f3b84070e81fe53a80d86510 Mon Sep 17 00:00:00 2001 From: loupicaaa Date: Wed, 27 May 2026 10:13:38 +0200 Subject: [PATCH 500/881] zml/moe: Add sharding for triton MoE (#552) This PR introduces the sharding as expert parallelism for MoE : - Introduce a .experts logical sharding (required to express an other semantic than TP (.model)) - Wrap the whole MoE impl into a manualComputation - Implement a manual stableHlo all reduce since shardy can not infer the correct communication after a manualComputation (blackbox) - Partition qwen 3.5 moe example with TP for the rest of the model (non MoE) --------- Co-authored-by: loupicaaa --- examples/llm/models/common.zig | 6 +- examples/llm/models/lfm2/model.zig | 3 +- examples/llm/models/lfm2_tests.zig | 4 +- examples/llm/models/llama/inference.zig | 6 +- examples/llm/models/llama/model.zig | 3 +- examples/llm/models/llama_tests.zig | 4 +- examples/llm/models/qwen3_5_moe/inference.zig | 22 ++- examples/llm/models/qwen3_5_moe/model.zig | 85 +++++---- examples/llm/models/qwen3_5_moe/session.zig | 15 +- mlir/dialects/stablehlo/stablehlo.zig | 41 +++- zml/module.zig | 7 + zml/moe/moe.zig | 63 ++++++ zml/moe/triton.zig | 16 +- zml/ops.zig | 180 ++++++++++++++++++ 14 files changed, 393 insertions(+), 62 deletions(-) diff --git a/examples/llm/models/common.zig b/examples/llm/models/common.zig index c547c761..e4baf025 100644 --- a/examples/llm/models/common.zig +++ b/examples/llm/models/common.zig @@ -14,15 +14,17 @@ pub const GenerationOptions = struct { pub const Shardings = struct { model: zml.Sharding, + experts: zml.Sharding, pub fn init(platform: *zml.Platform) !Shardings { return .{ .model = try platform.registerSharding("model", .mesh(.{ .model = .high_bandwidth })), + .experts = try platform.registerSharding("experts", .mesh(.{ .experts = .high_bandwidth })), }; } - pub fn all(self: Shardings) [1]zml.Sharding { - return .{self.model}; + pub fn all(self: Shardings) [2]zml.Sharding { + return .{ self.model, self.experts }; } }; diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index 0a0b6140..9c09ed58 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -94,11 +94,12 @@ pub const LoadedModel = struct { log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); } + const all_shardings = shardings.all(); return zml.io.load(Model, &self.inner, allocator, io, platform, store, .{ .dma_chunks = 32, .dma_chunk_size = 128 * zml.MiB, .progress = progress, - .shardings = &shardings.all(), + .shardings = &all_shardings, .parallelism = 16, .total_bytes = &total_bytes, }); diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig index bb014984..4b110e50 100644 --- a/examples/llm/models/lfm2_tests.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -47,9 +47,7 @@ pub fn main(init: std.process.Init) !void { defer repo_model.deinit(allocator); var progress = std.Progress.start(io, .{ .root_name = args.model }); - const shardings: common.Shardings = .{ - .model = try platform.registerSharding("tp_mesh", .mesh(.{ .model = .high_bandwidth })), - }; + const shardings: common.Shardings = try .init(platform); var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress, shardings); defer repo_model.unloadBuffers(&model_buffers, allocator); diff --git a/examples/llm/models/llama/inference.zig b/examples/llm/models/llama/inference.zig index 893a330f..46445c48 100644 --- a/examples/llm/models/llama/inference.zig +++ b/examples/llm/models/llama/inference.zig @@ -113,7 +113,7 @@ fn compileModel( platform_: *const zml.Platform, llama_model_: model.Model, parameters_: CompilationParameters, - shardings_: [1]zml.Sharding, + shardings_: []const zml.Sharding, progress_: *std.Progress.Node, ) !zml.Exe { progress_.increaseEstimatedTotalItems(1); @@ -134,11 +134,11 @@ fn compileModel( parameters_.prefill_attention_parameters, }, .{ - .shardings = &shardings_, + .shardings = shardings_, }, ); } - }.call, .{ allocator, io, platform, llama_model, parameters, all_shardings, progress }); + }.call, .{ allocator, io, platform, llama_model, parameters, &all_shardings, progress }); var prefill_future_awaited = false; errdefer if (!prefill_future_awaited) if (prefill_future.cancel(io)) |v| v.deinit() else |_| {}; diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index ae736777..56d43c36 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -81,11 +81,12 @@ pub const LoadedModel = struct { std.log.scoped(.llama).info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); } + const all_shardings = shardings.all(); return zml.io.load(Model, &self.inner, allocator, io, platform, store, .{ .dma_chunks = 32, .dma_chunk_size = 128 * zml.MiB, .progress = progress, - .shardings = &shardings.all(), + .shardings = &all_shardings, .parallelism = 16, .total_bytes = &total_bytes, }); diff --git a/examples/llm/models/llama_tests.zig b/examples/llm/models/llama_tests.zig index fe1b9ec3..08d86ec8 100644 --- a/examples/llm/models/llama_tests.zig +++ b/examples/llm/models/llama_tests.zig @@ -44,9 +44,7 @@ pub fn main(init: std.process.Init) !void { defer repo_model.deinit(allocator); var progress = std.Progress.start(io, .{ .root_name = args.model }); - const shardings: common.Shardings = .{ - .model = try platform.registerSharding("tp_mesh", .mesh(.{ .model = .high_bandwidth })), - }; + const shardings: common.Shardings = try .init(platform); var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress, shardings); defer repo_model.unloadBuffers(&model_buffers, allocator); diff --git a/examples/llm/models/qwen3_5_moe/inference.zig b/examples/llm/models/qwen3_5_moe/inference.zig index 9a8ecc35..2d475ec3 100644 --- a/examples/llm/models/qwen3_5_moe/inference.zig +++ b/examples/llm/models/qwen3_5_moe/inference.zig @@ -30,8 +30,9 @@ pub const CompilationParameters = struct { pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, moe_backend: zml.moe.Backend, shardings: common.Shardings) CompilationParameters { const dtype = mdl.text_model.embed_tokens.weight.dtype(); + const model_partitions = shardings.model.numPartitionsForLogicalAxis(.model); return .{ - .kv_cache = .init(config, 1, seqlen, dtype, .f32), + .kv_cache = .init(config, 1, seqlen, dtype, .f32, model_partitions), .rng = .init(), .prefill_moe_metadata = initMoeMetadata(mdl, @intCast(seqlen), 1, moe_backend), .decode_moe_metadata = initMoeMetadata(mdl, 1, 1, moe_backend), @@ -180,8 +181,23 @@ fn compileModel( const prefill_tokens = zml.Tensor.init(.{ .b = 1, .s = prefill_len }, .u32); const decode_tokens = zml.Tensor.init(.{ .b = 1, .s = 1 }, .u32); - const prefill_hidden = zml.Tensor.init(.{ .b = 1, .s = prefill_len, .d = qwen35_model.config.text_config.hidden_size }, qwen35_model.text_model.embed_tokens.weight.dtype()); - const decode_hidden = zml.Tensor.init(.{ .b = 1, .s = 1, .d = qwen35_model.config.text_config.hidden_size }, qwen35_model.text_model.embed_tokens.weight.dtype()); + const hidden_dtype = qwen35_model.text_model.embed_tokens.weight.dtype(); + const prefill_hidden: zml.Tensor = .fromShape(zml.Shape.init( + .{ .b = 1, .s = prefill_len, .d = qwen35_model.config.text_config.hidden_size }, + hidden_dtype, + ).withPartitioning(.{ + .b = .replicated, + .s = .replicated, + .d = .replicated, + })); + const decode_hidden: zml.Tensor = .fromShape(zml.Shape.init( + .{ .b = 1, .s = 1, .d = qwen35_model.config.text_config.hidden_size }, + hidden_dtype, + ).withPartitioning(.{ + .b = .replicated, + .s = .replicated, + .d = .replicated, + })); const token_index = zml.Tensor.init(.{}, .u32); const self_attn_cache: model.KvCache.SelfAttnCache = .{ .k = parameters.kv_cache.self_attn.k, diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 96f9c9d2..fc27eea7 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -49,24 +49,39 @@ pub const RopeParameters = struct { rope_theta: f32, }; -fn kvHeadsAreReplicated(axis_size: i64, model_partitions: i64) bool { - return axis_size > 0 and axis_size < model_partitions and @rem(model_partitions, axis_size) == 0; +fn kvHeadsAreReplicated(axis_size: i64, model_partitions: i64) struct { bool, ?u32 } { + const is_replicated = axis_size > 0 and axis_size < model_partitions and @rem(model_partitions, axis_size) == 0; + const repeat_factor = if (is_replicated) @as(u32, @intCast(@divExact(model_partitions, axis_size))) else null; + return .{ is_replicated, repeat_factor }; } -fn partitionProjectedKv(tensor: zml.Tensor, replicate_kv_heads: bool) zml.Tensor { +fn partitionProjectedKv(tensor: zml.Tensor, replicate_kv_heads: bool, repeat_factor: ?u32) zml.Tensor { return if (replicate_kv_heads) - tensor.withPartitioning(.{ .s = .replicated, .h = .replicated, .hd = .replicated }) + tensor.repeat1d(.h, @as(u63, repeat_factor.?)).withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }) else tensor.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); } -fn partitionCachedKv(tensor: zml.Tensor, replicate_kv_heads: bool) zml.Tensor { +fn partitionCachedKv(tensor: zml.Tensor, replicate_kv_heads: bool, repeat_factor: ?u32) zml.Tensor { return if (replicate_kv_heads) - tensor.rename(.{ .s = .k }).withPartitioning(.{ .k = .replicated, .h = .replicated, .hd = .replicated }) + tensor.rename(.{ .s = .k }).repeat1d(.h, @as(u63, repeat_factor.?)).withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }) else tensor.rename(.{ .s = .k }).withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); } +fn partitionKvCacheShape(kv_shape: zml.Shape, kv_heads: i64, model_partitions: i64) zml.Shape { + const is_replicated, const repeat_factor = kvHeadsAreReplicated(kv_heads, model_partitions); + + if (is_replicated) { + const repeated_h = kv_shape.dim(.h) * @as(i64, @intCast(repeat_factor.?)); + return kv_shape + .setDim(.h, repeated_h) + .withPartitioning(.{ .h = .model }); + } + + return kv_shape.withPartitioning(.{ .h = .model }); +} + pub const LoadedModel = struct { inner: Model, parsed_config: std.json.Parsed(Config), @@ -392,8 +407,8 @@ pub const TransformerLayer = struct { moe_parameters: zml.moe.Parameters, ) struct { zml.Tensor, KvCache.SelfAttnCache } { _ = config; - const residual0 = x0; - const normalized_x0 = self.input_layernorm.forward(x0); + const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const normalized_x0 = self.input_layernorm.forward(x0_replicated); const self_attn = switch (self.attn) { .self_attn => |self_attn| self_attn, @@ -401,13 +416,12 @@ pub const TransformerLayer = struct { }; const attention_output, const updated_kv_cache = self_attn.forward(normalized_x0, token_index, kv_cache); - const x1 = attention_output.add(residual0); - const residual1 = x1; + const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); const moe_output = self.moe.forward(normalized_hidden, moe_metadata, moe_parameters); - return .{ moe_output.add(residual1).reuseBuffer(x0), updated_kv_cache }; + return .{ moe_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), updated_kv_cache }; } pub fn forwardLinearAttn( @@ -421,8 +435,8 @@ pub const TransformerLayer = struct { ) struct { zml.Tensor, KvCache.GatedDeltaNetCache } { _ = config; _ = token_index; - const residual0 = x0; - const normalized_x0 = self.input_layernorm.forward(x0); + const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const normalized_x0 = self.input_layernorm.forward(x0_replicated); const linear_attn = switch (self.attn) { .linear_attn => |linear_attn| linear_attn, @@ -430,13 +444,12 @@ pub const TransformerLayer = struct { }; const attention_output, const updated_kv_cache = linear_attn.forward(normalized_x0, kv_cache); - const x1 = attention_output.add(residual0); - const residual1 = x1; + const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); const moe_output = self.moe.forward(normalized_hidden, moe_metadata, moe_parameters); - return .{ moe_output.add(residual1).reuseBuffer(x0), updated_kv_cache }; + return .{ moe_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), updated_kv_cache }; } pub fn forward( @@ -449,8 +462,8 @@ pub const TransformerLayer = struct { moe_parameters: zml.moe.Parameters, ) struct { zml.Tensor, KvCache } { _ = config; - const residual0 = x0; - const normalized_x0 = self.input_layernorm.forward(x0); + const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const normalized_x0 = self.input_layernorm.forward(x0_replicated); var attention_output: zml.Tensor = undefined; var updated_kv_cache: KvCache = kv_cache.parent; @@ -467,13 +480,12 @@ pub const TransformerLayer = struct { }, } - const x1 = attention_output.add(residual0); - const residual1 = x1; + const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); const moe_output = self.moe.forward(normalized_hidden, moe_metadata, moe_parameters); - return .{ moe_output.add(residual1), updated_kv_cache }; + return .{ moe_output.add(x1).withPartitioning(.{ .d = .replicated }), updated_kv_cache }; } }; @@ -568,15 +580,15 @@ pub const SelfAttn = struct { kv_cache: KvCache.SelfAttnCache, ) struct { zml.Tensor, KvCache.SelfAttnCache } { const model_partitions = zml.module.CompilationContext.current().partitioning.numPartitionsForLogicalAxis(self.q_proj.weight.shape(), .model) catch unreachable; - const replicate_kv_heads = kvHeadsAreReplicated(self.num_kv_heads, model_partitions); + const replicate_kv_heads, const repeat_factor = kvHeadsAreReplicated(self.num_kv_heads, model_partitions); const x_qkv = x.withPartitioning(.{ .d = .replicated }); var q, var gate = self.projectQAndGate(x_qkv); var k, var v = self.projectKV(x_qkv); q = q.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); gate = gate.withPartitioning(.{ .s = .replicated, .d_out_proj = .model }); - k = partitionProjectedKv(k, replicate_kv_heads); - v = partitionProjectedKv(v, replicate_kv_heads); + k = partitionProjectedKv(k, replicate_kv_heads, repeat_factor); + v = partitionProjectedKv(v, replicate_kv_heads, repeat_factor); q = self.q_norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); k = self.k_norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); @@ -589,15 +601,15 @@ pub const SelfAttn = struct { q = self.rotary_embed.applyRope(q, cos, sin); k = self.rotary_embed.applyRope(k, cos, sin); q = q.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); - k = partitionProjectedKv(k, replicate_kv_heads); - v = partitionProjectedKv(v, replicate_kv_heads); + k = partitionProjectedKv(k, replicate_kv_heads, repeat_factor); + v = partitionProjectedKv(v, replicate_kv_heads, repeat_factor); const new_kv_cache = kv_cache.update(k, v, token_index.convert(.u32)); k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); q = q.rename(.{ .s = .q }).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }); - k = partitionCachedKv(k, replicate_kv_heads); - v = partitionCachedKv(v, replicate_kv_heads); + k = partitionCachedKv(k, replicate_kv_heads, repeat_factor); + v = partitionCachedKv(v, replicate_kv_heads, repeat_factor); const attn_output = zml.attention.attention.attention( q, @@ -710,19 +722,19 @@ pub const Moe = struct { const gate_up_proj_tensor = experts_store.createTensor( "gate_up_proj", .{ .expert, .dout, .d }, - .{ .expert = .replicated, .dout = .model, .d = .replicated }, + .{ .expert = .experts, .dout = .replicated, .d = .replicated }, ); const down_proj_tensor = experts_store.createTensor( "down_proj", .{ .expert, .d, .dout }, - .{ .expert = .replicated, .d = .replicated, .dout = .model }, + .{ .expert = .experts, .d = .replicated, .dout = .replicated }, ); return .{ .shared_expert = Mlp.init(store.withPrefix("shared_expert")), .shared_expert_gate = .init( - store.withPrefix("shared_expert_gate").createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), - store.withPrefix("shared_expert_gate").maybeCreateTensor("bias", .{.dout}, .{ .dout = .model }), + store.withPrefix("shared_expert_gate").createTensor("weight", .{ .dout, .d }, .{ .dout = .replicated, .d = .replicated }), + store.withPrefix("shared_expert_gate").maybeCreateTensor("bias", .{.dout}, .{ .dout = .replicated }), .d, ), .gate_up_proj = gate_up_proj_tensor, @@ -754,7 +766,7 @@ pub const Moe = struct { ) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); const shared_gate = self.shared_expert_gate.forward(x).sigmoid().broad(x.shape()); - const shared = self.shared_expert.forward(x).mul(shared_gate); + const shared = self.shared_expert.forward(x).rename(.{ .dout = .d }).mul(shared_gate); return moe_output.add(shared); } @@ -1104,7 +1116,7 @@ pub const KvCache = struct { v: zml.Tensor, layer_index: zml.Tensor, - pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType) SelfAttnCache { + pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, model_partitions: i64) SelfAttnCache { const num_self_attn_layers = countLayers(config.text_config.layer_types, .full_attention); const kv_shape = zml.Shape.init(.{ .b = batch_dim, @@ -1113,7 +1125,7 @@ pub const KvCache = struct { .h = config.text_config.num_key_value_heads, .hd = config.text_config.head_dim, }, dtype); - const sharded_kv_shape = kv_shape.withPartitioning(.{ .h = .model }); + const sharded_kv_shape = partitionKvCacheShape(kv_shape, config.text_config.num_key_value_heads, model_partitions); return .{ .k = .fromShape(sharded_kv_shape), .v = .fromShape(sharded_kv_shape), @@ -1293,10 +1305,11 @@ pub const KvCache = struct { max_seq_len: i64, cache_dtype: zml.DataType, recurrent_dtype: zml.DataType, + model_partitions: i64, ) KvCache { return .{ .layer_types = config.text_config.layer_types, - .self_attn = SelfAttnCache.init(config, batch_dim, max_seq_len, cache_dtype), + .self_attn = SelfAttnCache.init(config, batch_dim, max_seq_len, cache_dtype, model_partitions), .gated_delta_net = GatedDeltaNetCache.init(config, batch_dim, cache_dtype, recurrent_dtype), }; } diff --git a/examples/llm/models/qwen3_5_moe/session.zig b/examples/llm/models/qwen3_5_moe/session.zig index 90133d47..ac7281f1 100644 --- a/examples/llm/models/qwen3_5_moe/session.zig +++ b/examples/llm/models/qwen3_5_moe/session.zig @@ -205,7 +205,11 @@ pub const Session = struct { const model_dtype = self.compiled_model.loaded_model.inner.text_model.embed_tokens.weight.dtype(); const prefill_tokens_shape = zml.Shape.init(.{ .b = 1, .s = self.seqlen }, .u32); - const prefill_hidden_shape = zml.Shape.init(.{ .b = 1, .s = self.seqlen, .d = hidden_size }, model_dtype); + const prefill_hidden_shape = zml.Shape.init(.{ .b = 1, .s = self.seqlen, .d = hidden_size }, model_dtype).withPartitioning(.{ + .b = .replicated, + .s = .replicated, + .d = .replicated, + }); const prefill_tokens_slice = try zml.Slice.alloc(self.allocator, prefill_tokens_shape); defer prefill_tokens_slice.free(self.allocator); @@ -296,7 +300,11 @@ pub const Session = struct { const hidden_size = self.compiled_model.loaded_model.inner.config.text_config.hidden_size; const model_dtype = self.compiled_model.loaded_model.inner.text_model.embed_tokens.weight.dtype(); - const decode_hidden_shape = zml.Shape.init(.{ .b = 1, .s = 1, .d = hidden_size }, model_dtype); + const decode_hidden_shape = zml.Shape.init(.{ .b = 1, .s = 1, .d = hidden_size }, model_dtype).withPartitioning(.{ + .b = .replicated, + .s = .replicated, + .d = .replicated, + }); const replicated_sharding: zml.Sharding = .replicated; var embedding_decode_args = try self.compiled_model.decode_embedding_exe.args(self.allocator); @@ -445,9 +453,6 @@ fn tokenizeChatPrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tok const assistant = try encoder.encodeAlloc(allocator, "assistant\n"); try tokens.appendSlice(allocator, assistant); allocator.free(assistant); - const think = try encoder.encodeAlloc(allocator, "\n"); - try tokens.appendSlice(allocator, think); - allocator.free(think); return tokens.toOwnedSlice(allocator); } diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 79c13b89..487b40d8 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -897,7 +897,7 @@ pub fn unpin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir pub fn partition_id(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.partition_id", .{ - .results = .{ .flat = &.{.rankedTensor(&.{}, mlir.integerType(ctx, .u32))} }, + .results = .{ .flat = &.{.rankedTensor(&.{}, .int(ctx, .u32))} }, .location = location, }); } @@ -1413,3 +1413,42 @@ pub fn returns(ctx: *mlir.Context, values: []const *const mlir.Value, location: .location = location, }); } + +pub const ChannelType = enum(i64) { + collective = 0, + send_recv = 1, +}; + +pub fn channelHandle( + ctx: *mlir.Context, + handle: i64, + channel_type: ChannelType, +) *const mlir.Attribute { + return @ptrCast(c.stablehloChannelHandleGet( + ctx.ptr(), + handle, + @intFromEnum(channel_type), + ).ptr); +} + +pub fn allReduce( + ctx: *mlir.Context, + inputs: []const *const mlir.Value, + result_types: []const *const mlir.Type, + reducer_block: *mlir.Block, + replica_groups: *const mlir.Attribute, + channel_handle: *const mlir.Attribute, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.all_reduce", .{ + .operands = .{ .variadic = &.{inputs} }, + .results = .{ .flat = result_types }, + .blocks = &.{reducer_block}, + .attributes = &.{ + .named(ctx, "replica_groups", replica_groups), + .named(ctx, "channel_handle", channel_handle), + .named(ctx, "use_global_device_ids", .unit(ctx)), + }, + .verify = true, + .location = .unknown(ctx), + }); +} diff --git a/zml/module.zig b/zml/module.zig index f27c5944..6bdc2b39 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -97,6 +97,8 @@ pub const CompilationContext = struct { scopes: stdx.BoundedArray(Scope, 16) = .empty, manual_computation_depth: usize = 0, + channel_id: i64 = 0, + threadlocal var _current: ?*CompilationContext = null; pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: CompilationOptions) CompilationContext { @@ -181,6 +183,11 @@ pub const CompilationContext = struct { popped.deinit(); } } + + pub fn nextChannelId(self: *CompilationContext) i64 { + self.channel_id += 1; + return self.channel_id; + } }; pub fn compile( diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index 325476fb..909d93fd 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -1,5 +1,6 @@ const std = @import("std"); const zml = @import("../zml.zig"); +const stdx = zml.stdx; pub const triton = @import("triton.zig"); pub const triton_kernels = @import("triton_kernels/triton_kernels.zig"); @@ -115,6 +116,67 @@ pub fn forwardMoe( .triton => |v| v, }; + const expert_partition = weights_gate_up.shape().partition(.expert); + + if (expert_partition.eql(.init(.experts))) { + const expert_partitions = zml.module.CompilationContext.current().partitioning.numPartitionsForLogicalAxis( + weights_gate_up.shape(), + .experts, + ) catch unreachable; + + const global_num_experts = weights_down.shape().dim(.expert) * expert_partitions; + + const partial_output = zml.ops.manualComputation( + .{ input, topk_ids, topk_weights, weights_gate_up, weights_down }, + input.shape().withPartitioning(.{ .b = .replicated, .s = .replicated, .d = .replicated }), + .{ + .activation = parameters.triton.activation, + .global_num_experts = global_num_experts, + .scales_gate_up = scales_gate_up, + .bias_gate_up = bias_gate_up, + .scales_down = scales_down, + .bias_down = bias_down, + }, + (struct { + fn body(ctx: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output_shape: zml.Shape) zml.Tensor { + const local_num_experts = sharded_inputs[3].dim(.expert); + const partition_id = zml.ops.partitionId().convert(.i32); + const expert_start = partition_id.scale(local_num_experts).convert(.i32); + // List of global expert ids + const global_expert_ids = zml.Tensor.arange(.{ .end = ctx.global_num_experts }, .i32).withTags(.{.expert}); + + // Mapping of local experts to global expert ids, -1 if the global expert is not present in the local partition + const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) + .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); + const expert_map = local_expert_mask.select( + global_expert_ids.sub(expert_start), + zml.Tensor.scalar(-1, .i32), + ); + const local_output = triton.fusedExpertsImpl( + sharded_inputs[0], + sharded_inputs[3], + sharded_inputs[4], + sharded_inputs[2], + sharded_inputs[1], + .{}, + .{ + .activation = ctx.activation, + .global_num_experts = ctx.global_num_experts, + .expert_map = expert_map, + .w1_scale = ctx.scales_gate_up, + .w2_scale = ctx.scales_down, + .w1_bias = ctx.bias_gate_up, + .w2_bias = ctx.bias_down, + }, + ) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); + _ = output_shape; + return local_output.reshape(sharded_inputs[0].shape().dims()).withTags(.{ .b, .s, .d }); + } + }).body, + ); + break :b zml.ops.allReduce(partial_output, zml.Tensor.add); + } + break :b try triton.fusedExpertsImpl( input, weights_gate_up, @@ -124,6 +186,7 @@ pub fn forwardMoe( triton_metadata, .{ .activation = parameters.triton.activation, + .global_num_experts = weights_gate_up.dim(.expert), .w1_scale = scales_gate_up, .w2_scale = scales_down, .w1_bias = bias_gate_up, diff --git a/zml/moe/triton.zig b/zml/moe/triton.zig index 5dfd8c96..941c15dc 100644 --- a/zml/moe/triton.zig +++ b/zml/moe/triton.zig @@ -141,7 +141,11 @@ pub fn fusedExpertsImpl( try validateInputs(hidden, gate_up, down, weights, ids); const block_size_m = options.block_size_m; - const num_experts = gate_up.dim(.expert); + const num_experts = if (opts.global_num_experts != -1) opts.global_num_experts else gate_up.dim(.expert); + if (opts.expert_map) |expert_map| { + if (expert_map.dtype() != .i32) return error.UnsupportedType; + if (expert_map.rank() != 1 or expert_map.dim(.expert) != num_experts) return error.InvalidShape; + } const num_assignments = hidden.dim(.token) * ids.dim(.topk); const sparsity_factor: i64 = 4; const naive_block_assignment = num_assignments * sparsity_factor <= num_experts; @@ -153,7 +157,7 @@ pub fn fusedExpertsImpl( else num_assignments + num_experts * (block_size_m - 1); - const sorted_token_ids, const expert_ids, const num_tokens_post_padded = if (naive_block_assignment) blk: { + const sorted_token_ids, const expert_ids_global, const num_tokens_post_padded = if (naive_block_assignment) blk: { log.info("Using naive block assignment for MoE kernels. Num assignments: {d}, Num experts: {d}", .{ num_assignments, num_experts }); const naive_sorted_ids = Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)); const naive_expert_ids = ids.reshape(.{ .g = num_assignments }); @@ -161,6 +165,11 @@ pub fn fusedExpertsImpl( break :blk .{ naive_sorted_ids, naive_expert_ids, naive_num_tokens_post_padded }; } else alignBlockSize(ids, num_experts, block_size_m); + const expert_ids = if (opts.expert_map) |expert_map| + expert_map.gather(.{ .expert = expert_ids_global }, .{}).withTags(.{.g}) + else + expert_ids_global; + var hidden_quant = hidden; var a_scale = opts.a1_scale orelse Tensor.scalar(1.0, .f32); @@ -632,8 +641,7 @@ fn validateOptions(opts: Options) !void { if (opts.apply_router_weight_on_input) return error.UnsupportedOption; if (opts.use_fp8_w8a8 or opts.use_int8_w8a8 or opts.use_int8_w8a16 or opts.use_int4_w4a16) return error.UnsupportedQuantization; if (opts.ocp_mx_scheme != null or opts.per_channel_quant) return error.UnsupportedOption; - if (opts.global_num_experts != -1) return error.UnsupportedOption; - if (opts.expert_map != null) return error.UnsupportedOption; + if (opts.expert_map != null and opts.global_num_experts == -1) return error.InvalidShape; if (opts.w1_zp != null or opts.w2_zp != null) return error.UnsupportedOption; if (opts.a1_scale != null or opts.a2_scale != null or opts.block_shape != null) return error.UnsupportedOption; if (opts.w1_bias != null or opts.w2_bias != null) return error.UnsupportedOption; diff --git a/zml/ops.zig b/zml/ops.zig index 4729c28b..b14bb90c 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -19,6 +19,186 @@ pub const ShapeToCustomCallBuffer = @import("pjrtx.zig").ShapeToCustomCallBuffer const Tensor = @import("tensor.zig").Tensor; pub const TensorToCustomCallBuffer = @import("pjrtx.zig").TensorToCustomCallBuffer; +pub fn allReduce(inputs: anytype, comptime func: anytype) allReduceReturnType(@TypeOf(inputs)) { + const ctx = CompilationContext.current(); + const mlir_ctx = ctx.mlir_ctx; + + const InputsT = @TypeOf(inputs); + const n_inputs = switch (@typeInfo(InputsT)) { + .@"struct" => |struct_info| b: { + if (InputsT == Tensor) break :b 1; + if (!struct_info.is_tuple) { + @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"); + } + break :b struct_info.fields.len; + }, + .array => |array_info| b: { + if (array_info.child != Tensor) { + @compileError("zml.ops.allReduce expects [N]Tensor inputs"); + } + break :b array_info.len; + }, + else => @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"), + }; + comptime stdx.debug.assertComptime(n_inputs > 0, "zml.ops.allReduce requires at least one input tensor", .{}); + + const input_tensors: [n_inputs]Tensor = switch (@typeInfo(InputsT)) { + .@"struct" => |struct_info| b: { + if (InputsT == Tensor) { + break :b .{inputs}; + } + if (!struct_info.is_tuple) { + @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"); + } + + var flat_inputs: [n_inputs]Tensor = undefined; + meta.collectBuf((struct { + pub fn cb(t: Tensor) Tensor { + return t; + } + }).cb, {}, &inputs, &flat_inputs); + break :b flat_inputs; + }, + .array => inputs, + else => @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"), + }; + + const num_devices = ctx.partitioning.numPartitions(); + if (num_devices <= 1) return inputs; + + const reducer_block = b: { + var args: std.meta.Tuple(&[1]type{ReduceArgs} ** input_tensors.len) = undefined; + var block_types: [2 * input_tensors.len]*const mlir.Type = undefined; + + inline for (0..input_tensors.len) |i| { + const scalar_shape = Shape.init(.{}, input_tensors[i].dtype()); + args[i].left = .fromShape(scalar_shape); + args[i].right = .fromShape(scalar_shape); + block_types[i] = mlirx.Type.rankedTensor(mlir_ctx, scalar_shape); + block_types[i + input_tensors.len] = mlirx.Type.rankedTensor(mlir_ctx, scalar_shape); + } + + const block_locs: [2 * input_tensors.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); + + const block = mlir.Block.init(&block_types, &block_locs); + errdefer block.deinit(); + + ctx.pushBlock(block); + defer ctx.popBlock(); + + const scope = ctx.currentScope(); + inline for (0..input_tensors.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + input_tensors.len) catch unreachable; + } + + var reduced_values: [input_tensors.len]*const mlir.Value = undefined; + if (input_tensors.len == 1) { + const reduced = @call(.auto, func, .{ args[0].left, args[0].right }); + reduced_values[0] = reduced.value(); + } else { + var reduced = @call(.auto, func, args); + var reduced_tensors: [input_tensors.len]Tensor = undefined; + meta.collectBuf((struct { + pub fn cb(t: Tensor) Tensor { + return t; + } + }).cb, {}, &reduced, &reduced_tensors); + + inline for (0..input_tensors.len) |i| { + reduced_values[i] = reduced_tensors[i].value(); + } + } + + _ = dialects.stablehlo.returns(mlir_ctx, &reduced_values, .unknown(mlir_ctx)).appendTo(block); + break :b block; + }; + + var replica_group_storage: [constants.MAX_RANK]i64 = undefined; + for (0..@intCast(num_devices)) |i| { + replica_group_storage[i] = @intCast(i); + } + const replica_group_slice = replica_group_storage[0..@intCast(num_devices)]; + const replica_groups_attr = mlir.Attribute.denseElements( + .rankedTensor(&.{ 1, num_devices }, .int(mlir_ctx, .i64)), + replica_group_slice, + ); + + const handle = ctx.nextChannelId(); + const channel_handle_attr = dialects.stablehlo.channelHandle( + mlir_ctx, + handle, + .collective, + ); + + var input_values: [input_tensors.len]*const mlir.Value = undefined; + var result_types: [input_tensors.len]*const mlir.Type = undefined; + inline for (0..input_tensors.len) |i| { + input_values[i] = input_tensors[i].value(); + result_types[i] = input_values[i].type_(); + } + + const op = dialects.stablehlo.allReduce( + mlir_ctx, + &input_values, + &result_types, + reducer_block, + replica_groups_attr, + channel_handle_attr, + ).appendTo(ctx.currentScope().block); + + return switch (@typeInfo(@TypeOf(inputs))) { + .@"struct" => |struct_info| b: { + if (@TypeOf(inputs) == Tensor) { + break :b Tensor._result(input_tensors[0].shape(), op.result(0)); + } + + var out: @TypeOf(inputs) = undefined; + inline for (struct_info.fields, 0..) |field, i| { + @field(out, field.name) = Tensor._result(input_tensors[i].shape(), op.result(i)); + } + break :b out; + }, + .array => |array_info| b: { + var out: [array_info.len]Tensor = undefined; + inline for (0..array_info.len) |i| { + out[i] = Tensor._result(input_tensors[i].shape(), op.result(i)); + } + break :b out; + }, + else => unreachable, + }; +} + +fn allReduceReturnType(comptime InputsT: type) type { + if (InputsT == Tensor) return Tensor; + + return switch (@typeInfo(InputsT)) { + .@"struct" => |struct_info| b: { + if (!struct_info.is_tuple) { + @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor or [N]Tensor inputs"); + } + break :b InputsT; + }, + .array => |array_info| b: { + if (array_info.child != Tensor) { + @compileError("zml.ops.allReduce expects [N]Tensor inputs"); + } + break :b InputsT; + }, + else => @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"), + }; +} + +pub fn partitionId() Tensor { + const ctx = CompilationContext.current(); + const op = mlir.Operation.make(ctx.mlir_ctx, "stablehlo.partition_id", .{ + .results = .{ .flat = &.{mlirx.Type.rankedTensor(ctx.mlir_ctx, Shape.scalar(.u32))} }, + .location = .unknown(ctx.mlir_ctx), + }).appendTo(ctx.currentScope().block); + return Tensor._result(.init(.{}, .u32), op.result(0)); +} + pub const ReduceArgs = struct { left: Tensor, right: Tensor, From 6d596964ebdc645a25962917e1b99a4709b15313 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 27 May 2026 15:00:47 +0200 Subject: [PATCH 501/881] workspace: bump llvm to `0.8.3` (#554) --- MODULE.bazel | 2 +- MODULE.bazel.lock | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 9ddc5751..265d98f2 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -30,7 +30,7 @@ bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "rules_zig", version = "0.15.1") bazel_dep(name = "tar.bzl", version = "0.9.0") -bazel_dep(name = "llvm", version = "0.7.3") +bazel_dep(name = "llvm", version = "0.8.3") bazel_dep(name = "with_cfg.bzl", version = "0.14.4") bazel_dep(name = "grpc", version = "1.74.0") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 294b79c6..a296acfb 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -152,8 +152,8 @@ "https://bcr.bazel.build/modules/libbacktrace/1.0.0-20250926-7939218/source.json": "83239989d09748f63dee4bdd430d261e217a012670e066e2238925a303a65d89", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", "https://bcr.bazel.build/modules/libpfm/4.11.0/source.json": "caaffb3ac2b59b8aac456917a4ecf3167d40478ee79f15ab7a877ec9273937c9", - "https://bcr.bazel.build/modules/llvm/0.7.3/MODULE.bazel": "b8ffe1ec9d18c0deb6d6dc1a6f78a64bcfcec8648c97b92ed6cd98a7fe49c64f", - "https://bcr.bazel.build/modules/llvm/0.7.3/source.json": "44d694e1559683ea4cf96052c56c5426d2604016636133798fcd57cc78542984", + "https://bcr.bazel.build/modules/llvm/0.8.3/MODULE.bazel": "dbc1bc13171f8f9980a75524883abef1c491613e8d8d805cd4ffa3767ba9708e", + "https://bcr.bazel.build/modules/llvm/0.8.3/source.json": "4b11874c26a9de53c03a25a517ebc36dee6b458c920fe9ed65e3ca279ff19775", "https://bcr.bazel.build/modules/mbedtls/3.6.0/MODULE.bazel": "8e380e4698107c5f8766264d4df92e36766248447858db28187151d884995a09", "https://bcr.bazel.build/modules/mbedtls/3.6.0/source.json": "1dbe7eb5258050afcc3806b9d43050f71c6f539ce0175535c670df606790b30c", "https://bcr.bazel.build/modules/nlohmann_json/3.11.3/MODULE.bazel": "87023db2f55fc3a9949c7b08dc711fae4d4be339a80a99d04453c4bb3998eefc", @@ -355,12 +355,13 @@ "https://bcr.bazel.build/modules/stardoc/0.7.2/source.json": "58b029e5e901d6802967754adf0a9056747e8176f017cfe3607c0851f4d42216", "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "5e463fbfba7b1701d957555ed45097d7f984211330106ccd1352c6e0af0dcf91", "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/source.json": "32bd87e5f4d7acc57c5b2ff7c325ae3061d5e242c0c4c214ae87e0f1c13e54cb", + "https://bcr.bazel.build/modules/tar.bzl/0.10.4/MODULE.bazel": "e8f9ff79199e8d9eaad7f1b0a77ad74b30bb82d794b87d8ca942bead5de83ae9", + "https://bcr.bazel.build/modules/tar.bzl/0.10.4/source.json": "20143442376c03426f6135292ba02d825cb75308aa47e6bf42dd4cc5a435c2ff", "https://bcr.bazel.build/modules/tar.bzl/0.2.1/MODULE.bazel": "52d1c00a80a8cc67acbd01649e83d8dd6a9dc426a6c0b754a04fe8c219c76468", "https://bcr.bazel.build/modules/tar.bzl/0.5.1/MODULE.bazel": "7c2eb3dcfc53b0f3d6f9acdfd911ca803eaf92aadf54f8ca6e4c1f3aee288351", "https://bcr.bazel.build/modules/tar.bzl/0.5.5/MODULE.bazel": "4bfab9bbc7a1966c2c5f7371f5848f5e2d27c465951b4435adc9aaf00ed681da", "https://bcr.bazel.build/modules/tar.bzl/0.6.0/MODULE.bazel": "a3584b4edcfafcabd9b0ef9819808f05b372957bbdff41601429d5fd0aac2e7c", "https://bcr.bazel.build/modules/tar.bzl/0.9.0/MODULE.bazel": "452a22d7f02b1c9d7a22ab25edf20f46f3e1101f0f67dc4bfbf9a474ddf02445", - "https://bcr.bazel.build/modules/tar.bzl/0.9.0/source.json": "c732760a374831a2cf5b08839e4be75017196b4d796a5aa55235272ee17cd839", "https://bcr.bazel.build/modules/upb/0.0.0-20211020-160625a/MODULE.bazel": "6cced416be2dc5b9c05efd5b997049ba795e5e4e6fafbe1624f4587767638928", "https://bcr.bazel.build/modules/upb/0.0.0-20230516-61a97ef/MODULE.bazel": "c0df5e35ad55e264160417fd0875932ee3c9dda63d9fccace35ac62f45e1b6f9", "https://bcr.bazel.build/modules/upb/0.0.0-20230907-e7430e6/MODULE.bazel": "3a7dedadf70346e678dc059dbe44d05cbf3ab17f1ce43a1c7a42edc7cbf93fd9", From 3db711aa6fdf5234bbd2664a1057a855a9478c17 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 27 May 2026 17:52:33 +0200 Subject: [PATCH 502/881] zml/ops: fix typo in type name (#553) Names of types should be `PascalCase` --- zml/ops.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zml/ops.zig b/zml/ops.zig index b14bb90c..c5fac4e6 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -19,7 +19,7 @@ pub const ShapeToCustomCallBuffer = @import("pjrtx.zig").ShapeToCustomCallBuffer const Tensor = @import("tensor.zig").Tensor; pub const TensorToCustomCallBuffer = @import("pjrtx.zig").TensorToCustomCallBuffer; -pub fn allReduce(inputs: anytype, comptime func: anytype) allReduceReturnType(@TypeOf(inputs)) { +pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@TypeOf(inputs)) { const ctx = CompilationContext.current(); const mlir_ctx = ctx.mlir_ctx; @@ -170,7 +170,7 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) allReduceReturnType(@T }; } -fn allReduceReturnType(comptime InputsT: type) type { +fn AllReduceReturnType(comptime InputsT: type) type { if (InputsT == Tensor) return Tensor; return switch (@typeInfo(InputsT)) { From 950c80dee8f6a99b77cc57bff669067173143859 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 28 May 2026 10:10:29 +0200 Subject: [PATCH 503/881] examples/llm: split Llama and Qwen 3.5 with LFM cleanup (#555) This PR splits Llama in `examples/llm` into separate `embed_tokens`, `layer`, and `lm_head` kernels for both prefill and decode, and brings LFM2 onto the same composed execution path. And done same work for Qwen 3.5 . It removes the old LFM2 `--single` mode, reworks Llama KV-cache access to use explicit layer indices, and reuses more decode-time state through a persistent runner and prebuilt token index buffers. It also tightens sharding/partitioning in LFM2 for multi-device runs, adds clearer per-phase compile logging/program names, removes the unused `--single` CLI flag. --- examples/llm/main.zig | 3 - examples/llm/models/common.zig | 32 +- examples/llm/models/lfm2/inference.zig | 568 ++++++++--------- examples/llm/models/lfm2/model.zig | 73 ++- examples/llm/models/lfm2/session.zig | 2 +- examples/llm/models/lfm2_tests.zig | 8 +- examples/llm/models/llama/inference.zig | 608 +++++++++++++++--- examples/llm/models/llama/model.zig | 167 +++-- examples/llm/models/llama/session.zig | 115 ++-- examples/llm/models/qwen3_5/inference.zig | 745 +++++++++++++++++++--- examples/llm/models/qwen3_5/model.zig | 361 ++++++++--- examples/llm/models/qwen3_5/session.zig | 101 +-- zml/safetensors.zig | 2 +- 13 files changed, 1970 insertions(+), 815 deletions(-) diff --git a/examples/llm/main.zig b/examples/llm/main.zig index 9cea2b67..c1edf5b6 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -20,7 +20,6 @@ const Args = struct { topk: u32 = 4, backend: ?zml.attention.attention.Backend = null, attnd_ip: ?[]const u8 = null, - single: bool = false, profile: bool = false, pub const help = @@ -35,8 +34,6 @@ const Args = struct { \\ --topk= Top-k sampling cutoff (default: 4) \\ --backend= Attention backend to use ([vanilla, attnd, cuda_fa2, cuda_fa3], default: auto-selection) \\ --attnd-ip= Register and prefer the `attnd` backend at the provided `IP:PORT` - \\ --single Create a single kernel encompassing all the layers when supported - \\ (only used by LFM2 which uses multiple kernels by default) \\ --profile Capture a PJRT profile for non-interactive runs and write a Perfetto trace \\ ; diff --git a/examples/llm/models/common.zig b/examples/llm/models/common.zig index e4baf025..7de783c2 100644 --- a/examples/llm/models/common.zig +++ b/examples/llm/models/common.zig @@ -5,13 +5,43 @@ const zml = @import("zml"); pub const SessionOptions = struct { seqlen: u32, backend: zml.attention.attention.Backend, - single: bool, }; pub const GenerationOptions = struct { sampling_strategy: zml.nn.SamplingStrategy = .{}, }; +pub const Phase = enum { + prefill, + decode, + + pub fn isPrefill(self: Phase) bool { + return self == .prefill; + } + + pub fn label(self: Phase) []const u8 { + return @tagName(self); + } + + pub fn startMessage(self: Phase, comptime component: []const u8) []const u8 { + return switch (self) { + .prefill => "Compiling prefill " ++ component ++ "...", + .decode => "Compiling decode " ++ component ++ "...", + }; + } + + pub fn programName(self: Phase, comptime model_name: []const u8, comptime component: []const u8) []const u8 { + return switch (self) { + .prefill => "llm_" ++ model_name ++ "_prefill_" ++ component, + .decode => "llm_" ++ model_name ++ "_decode_" ++ component, + }; + } + + pub fn logCompileDone(self: Phase, logger: anytype, comptime component: []const u8, io: std.Io, from: std.Io.Timestamp) void { + logger.info("Compiled {s} " ++ component ++ " [{f}]", .{ self.label(), from.untilNow(io, .awake) }); + } +}; + pub const Shardings = struct { model: zml.Sharding, experts: zml.Sharding, diff --git a/examples/llm/models/lfm2/inference.zig b/examples/llm/models/lfm2/inference.zig index 599d8eb5..899c9100 100644 --- a/examples/llm/models/lfm2/inference.zig +++ b/examples/llm/models/lfm2/inference.zig @@ -8,11 +8,11 @@ const common = @import("../common.zig"); const model = @import("model.zig"); const log = std.log.scoped(.lfm); +const Phase = common.Phase; pub const CompilationParameters = struct { hidden_dim: usize, batch_dim: usize, - single: bool, rng: zml.Tensor.Rng, cache: model.Cache, attention_metadata: attention.Metadata, @@ -20,7 +20,7 @@ pub const CompilationParameters = struct { seqlen: u32, shardings: common.Shardings, - pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: attention.Backend, single: bool, shardings: common.Shardings) CompilationParameters { + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: attention.Backend, shardings: common.Shardings) CompilationParameters { stdx.debug.assert(seqlen >= config.conv_L_cache, "seqlen ({}) must be at least conv_L_cache ({})", .{ seqlen, config.conv_L_cache }); const cache: model.Cache = .{ .kv = .init(.init(.{ @@ -39,7 +39,6 @@ pub const CompilationParameters = struct { }; return .{ - .single = single, .hidden_dim = config.hidden_size, .batch_dim = 1, .rng = .init(), @@ -82,27 +81,12 @@ pub const CompiledModel = struct { opts: CompilationParameters, progress: *std.Progress.Node, ) !CompiledModel { - if (opts.single) { - const prefill_exe = try compileSingleKernelExe(allocator, io, platform, mdl, opts, opts.seqlen, true, progress, opts.shardings); - const decode_exe = try compileSingleKernelExe(allocator, io, platform, mdl, opts, 1, false, progress, opts.shardings); - - return .{ - .loaded_model = loaded_model, - .prefill = .{ .single = prefill_exe }, - .decode = .{ .single = decode_exe }, - .params = opts, - }; - } else { - const prefill_exe = try compileComposedKernelExe(allocator, io, platform, mdl, opts, opts.seqlen, true, progress, opts.shardings); - const decode_exe = try compileComposedKernelExe(allocator, io, platform, mdl, opts, 1, false, progress, opts.shardings); - - return .{ - .loaded_model = loaded_model, - .prefill = .{ .composed = prefill_exe }, - .decode = .{ .composed = decode_exe }, - .params = opts, - }; - } + return .{ + .loaded_model = loaded_model, + .prefill = try KernelExe.init(allocator, io, platform, mdl, opts, opts.seqlen, .prefill, progress), + .decode = try KernelExe.init(allocator, io, platform, mdl, opts, 1, .decode, progress), + .params = opts, + }; } pub fn deinit(self: *CompiledModel) void { @@ -113,110 +97,69 @@ pub const CompiledModel = struct { pub const Inference = CompiledModel; -pub const KernelExe = union(enum) { - single: SingleKernelExe, +pub const KernelExe = struct { composed: ComposedKernelExe, + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + mdl: model.Model, + opts: CompilationOptions, + seqlen: u32, + phase: Phase, + progress: *std.Progress.Node, + ) !KernelExe { + return .{ + .composed = try ComposedKernelExe.init(allocator, io, platform, mdl, opts, seqlen, phase, progress), + }; + } + pub fn deinit(self: KernelExe) void { - switch (self) { - .single => |exe| exe.deinit(), - .composed => |exe| exe.deinit(), - } + self.composed.deinit(); } pub fn run(self: *const KernelExe, args: Args) !void { - switch (self.*) { - .single => |*exe| try exe.run(args), - .composed => |*exe| try exe.run(args), - } + try self.composed.run(args); } }; -pub const SingleKernelExe = struct { - exe: zml.Exe, - - fn deinit(self: SingleKernelExe) void { - self.exe.deinit(); - } - - pub fn run(self: *const SingleKernelExe, args: Args) !void { - var exe_args = try self.exe.args(args.allocator); - defer exe_args.deinit(args.allocator); - - var results = try self.exe.results(args.allocator); - defer results.deinit(args.allocator); - - exe_args.set(.{ - args.model_buffers, - args.tokens_buf, - args.tokens_pos_buf, - args.actual_seq_len_buf, - args.rng_buf, - args.cache_buffers, - args.attention_metadata_buffers, - }); - self.exe.call(exe_args, &results); - - var new_tokens, var new_cache, var new_rng = results.get(struct { - zml.Buffer, - zml.Bufferized(model.Cache), - zml.Bufferized(zml.Tensor.Rng), - }); - replaceBuffer(args.tokens_buf, &new_tokens); - replaceCacheBuffers(args.cache_buffers, &new_cache); - replaceBuffer(&args.rng_buf._state, &new_rng._state); - } -}; - -fn compileSingleKernelExe( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - mdl: model.Model, - opts: CompilationOptions, - seqlen: u32, - is_prefill: bool, - progress: *std.Progress.Node, - shardings: common.Shardings, -) !SingleKernelExe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start("Compiling single kernel...", 1); - defer node.end(); - const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Compiled single kernel [{f}]", .{now.untilNow(io, .awake)}); - - const actual_seq_len: zml.Tensor = .init(.{}, .u32); - const tokens: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32); - const token_position_offset: zml.Tensor = .init(.{ .batch = opts.batch_dim }, .u32); - - const all_shardings = shardings.all(); - const exe = try platform.compile( - allocator, - io, - mdl, - .forward, - .{ - tokens, - token_position_offset, - actual_seq_len, - opts.rng, - opts.cache, - opts.attention_metadata, - opts.attention_parameters, - model.ConvParameters{ .is_prefill = is_prefill }, - }, - .{ .shardings = &all_shardings }, - ); - - return .{ .exe = exe }; -} - pub const ComposedKernelExe = struct { embed_tokens: zml.Exe, conv_layer: zml.Exe, attn_layer: zml.Exe, lm_head: zml.Exe, + fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + mdl: model.Model, + opts: CompilationOptions, + seqlen: u32, + phase: Phase, + progress: *std.Progress.Node, + ) !ComposedKernelExe { + const embed_tokens = try ComposedKernelExe.compileEmbedTokens(allocator, io, platform, mdl.embed_tokens, opts, seqlen, phase, progress); + errdefer embed_tokens.deinit(); + + const conv_layer = try ComposedKernelExe.compileConvLayer(allocator, io, platform, mdl, opts, seqlen, phase, progress); + errdefer conv_layer.deinit(); + + const attn_layer = try ComposedKernelExe.compileAttnLayer(allocator, io, platform, mdl, opts, seqlen, phase, progress); + errdefer attn_layer.deinit(); + + const lm_head = try ComposedKernelExe.compileLmHead(allocator, io, platform, mdl, opts, seqlen, phase, progress); + errdefer lm_head.deinit(); + + return .{ + .embed_tokens = embed_tokens, + .conv_layer = conv_layer, + .attn_layer = attn_layer, + .lm_head = lm_head, + }; + } + fn deinit(self: ComposedKernelExe) void { self.embed_tokens.deinit(); self.conv_layer.deinit(); @@ -228,6 +171,7 @@ pub const ComposedKernelExe = struct { var hidden_buf: zml.Buffer = b: { var exe_args = try self.embed_tokens.args(args.allocator); defer exe_args.deinit(args.allocator); + var results = try self.embed_tokens.results(args.allocator); defer results.deinit(args.allocator); @@ -263,203 +207,213 @@ pub const ComposedKernelExe = struct { &kv_cache_index_buf, args.attention_metadata_buffers, }); - exe.call(exe_args, &results); - - var new_hidden, var new_cache, var new_conv_cache_index, var new_kv_cache_index = results.get(struct { - zml.Buffer, - zml.Bufferized(model.Cache), - zml.Buffer, - zml.Buffer, - }); - replaceBuffer(&hidden_buf, &new_hidden); - replaceCacheBuffers(args.cache_buffers, &new_cache); - replaceBuffer(&conv_cache_index_buf, &new_conv_cache_index); - replaceBuffer(&kv_cache_index_buf, &new_kv_cache_index); + ComposedKernelExe.runLayer(exe, &exe_args, &results, args.cache_buffers, &hidden_buf, &conv_cache_index_buf, &kv_cache_index_buf); } - { - var exe_args = try self.lm_head.args(args.allocator); - defer exe_args.deinit(args.allocator); - var results = try self.lm_head.results(args.allocator); - defer results.deinit(args.allocator); + var exe_args = try self.lm_head.args(args.allocator); + defer exe_args.deinit(args.allocator); + var results = try self.lm_head.results(args.allocator); + defer results.deinit(args.allocator); - exe_args.set(.{ args.model_buffers.lm_head, hidden_buf, args.model_buffers.embed_tokens, args.tokens_buf, args.rng_buf }); - self.lm_head.call(exe_args, &results); - var new_tokens, var new_rng = results.get(struct { - zml.Buffer, - zml.Bufferized(zml.Tensor.Rng), - }); - replaceBuffer(args.tokens_buf, &new_tokens); - replaceBuffer(&args.rng_buf._state, &new_rng._state); + exe_args.set(.{ args.model_buffers.lm_head, hidden_buf, args.model_buffers.embed_tokens, args.tokens_buf, args.rng_buf }); + self.runLmHead(&exe_args, &results, args.tokens_buf, args.rng_buf); + } + + fn runLayer( + exe: *const zml.Exe, + exe_args: *zml.exe.Exe.Arguments, + results: *zml.exe.Exe.Results, + cache_buffers: *zml.Bufferized(model.Cache), + hidden_buf: *zml.Buffer, + conv_cache_index_buf: *zml.Buffer, + kv_cache_index_buf: *zml.Buffer, + ) void { + exe.call(exe_args.*, results); + + var new_hidden, var new_cache, var new_conv_cache_index, var new_kv_cache_index = results.get(struct { + zml.Buffer, + zml.Bufferized(model.Cache), + zml.Buffer, + zml.Buffer, + }); + ComposedKernelExe.replaceBuffer(hidden_buf, &new_hidden); + ComposedKernelExe.replaceCacheBuffers(cache_buffers, &new_cache); + ComposedKernelExe.replaceBuffer(conv_cache_index_buf, &new_conv_cache_index); + ComposedKernelExe.replaceBuffer(kv_cache_index_buf, &new_kv_cache_index); + } + + fn runLmHead( + self: *const ComposedKernelExe, + exe_args: *zml.exe.Exe.Arguments, + results: *zml.exe.Exe.Results, + tokens_buf: *zml.Buffer, + rng_buf: *zml.Bufferized(zml.Tensor.Rng), + ) void { + self.lm_head.call(exe_args.*, results); + + var new_tokens, var new_rng = results.get(struct { + zml.Buffer, + zml.Bufferized(zml.Tensor.Rng), + }); + ComposedKernelExe.replaceBuffer(tokens_buf, &new_tokens); + ComposedKernelExe.replaceBuffer(&rng_buf._state, &new_rng._state); + } + + fn replaceCacheBuffers(dst: *zml.Bufferized(model.Cache), src: *zml.Bufferized(model.Cache)) void { + ComposedKernelExe.replaceBuffer(&dst.conv.state, &src.conv.state); + ComposedKernelExe.replaceBuffer(&dst.kv.k, &src.kv.k); + ComposedKernelExe.replaceBuffer(&dst.kv.v, &src.kv.v); + } + + fn replaceBuffer(dst: *zml.Buffer, src: *zml.Buffer) void { + if (!ComposedKernelExe.sameBufferHandle(dst.*, src.*)) { + dst.deinit(); } + dst.* = src.*; + } + + fn sameBufferHandle(a: zml.Buffer, b: zml.Buffer) bool { + if (a._shards.len != b._shards.len) return false; + for (a._shards.constSlice(), b._shards.constSlice()) |a_shard, b_shard| { + if (a_shard != b_shard) return false; + } + return true; + } + + fn compileEmbedTokens( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + embed_tokens: model.TokenEmbedding, + opts: CompilationOptions, + seqlen: u32, + phase: Phase, + progress: *std.Progress.Node, + ) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("embed_tokens"), 1); + defer node.end(); + + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "embed_tokens", io, from); + + const tokens: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32); + + return platform.compile(allocator, io, embed_tokens, .forward, .{tokens}, .{ + .shardings = &opts.shardings.all(), + .program_name = phase.programName("lfm2", "embed_tokens"), + }); + } + + fn compileConvLayer( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + mdl: model.Model, + opts: CompilationOptions, + seqlen: u32, + phase: Phase, + progress: *std.Progress.Node, + ) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("conv layer"), 1); + defer node.end(); + + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "conv layer", io, from); + + const conv_layer = for (mdl.layers) |layer| { + if (layer.operator == .conv) break layer; + } else unreachable; + + const hidden: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()); + const token_position_offset: zml.Tensor = .init(.{ .batch = opts.batch_dim }, .u32); + const actual_seq_len: zml.Tensor = .init(.{}, .u32); + const conv_cache_index: zml.Tensor = .init(.{}, .u32); + const kv_cache_index: zml.Tensor = .init(.{}, .u32); + + return platform.compile(allocator, io, conv_layer, .forward, .{ + hidden, + token_position_offset, + actual_seq_len, + opts.cache, + conv_cache_index, + kv_cache_index, + opts.attention_metadata, + opts.attention_parameters, + model.ConvParameters{ .is_prefill = phase.isPrefill() }, + }, .{ + .shardings = &opts.shardings.all(), + .program_name = phase.programName("lfm2", "conv_layer"), + }); + } + + fn compileAttnLayer( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + mdl: model.Model, + opts: CompilationOptions, + seqlen: u32, + phase: Phase, + progress: *std.Progress.Node, + ) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("attn layer"), 1); + defer node.end(); + + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "attn layer", io, from); + + const attn_layer = for (mdl.layers) |layer| { + if (layer.operator == .self_attn) break layer; + } else unreachable; + + const hidden: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()); + const token_position_offset: zml.Tensor = .init(.{ .batch = opts.batch_dim }, .u32); + const actual_seq_len: zml.Tensor = .init(.{}, .u32); + const conv_cache_index: zml.Tensor = .init(.{}, .u32); + const kv_cache_index: zml.Tensor = .init(.{}, .u32); + + return platform.compile(allocator, io, attn_layer, .forward, .{ + hidden, + token_position_offset, + actual_seq_len, + opts.cache, + conv_cache_index, + kv_cache_index, + opts.attention_metadata, + opts.attention_parameters, + model.ConvParameters{ .is_prefill = phase.isPrefill() }, + }, .{ + .shardings = &opts.shardings.all(), + .program_name = phase.programName("lfm2", "attn_layer"), + }); + } + + fn compileLmHead( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + mdl: model.Model, + opts: CompilationOptions, + seqlen: u32, + phase: Phase, + progress: *std.Progress.Node, + ) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("lm_head"), 1); + defer node.end(); + + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "lm_head", io, from); + + const hidden: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()); + const tokens: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32); + + return platform.compile(allocator, io, mdl.lm_head, .forward, .{ hidden, mdl.embed_tokens, tokens, opts.rng }, .{ + .shardings = &opts.shardings.all(), + .program_name = phase.programName("lfm2", "lm_head"), + }); } }; - -fn replaceCacheBuffers(dst: *zml.Bufferized(model.Cache), src: *zml.Bufferized(model.Cache)) void { - replaceBuffer(&dst.conv.state, &src.conv.state); - replaceBuffer(&dst.kv.k, &src.kv.k); - replaceBuffer(&dst.kv.v, &src.kv.v); -} - -fn replaceBuffer(dst: *zml.Buffer, src: *zml.Buffer) void { - if (!sameBufferHandle(dst.*, src.*)) { - dst.deinit(); - } - dst.* = src.*; -} - -fn sameBufferHandle(a: zml.Buffer, b: zml.Buffer) bool { - if (a._shards.len != b._shards.len) return false; - for (a._shards.constSlice(), b._shards.constSlice()) |a_shard, b_shard| { - if (a_shard != b_shard) return false; - } - return true; -} - -fn compileComposedKernelExe( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - mdl: model.Model, - opts: CompilationOptions, - seqlen: u32, - is_prefill: bool, - progress: *std.Progress.Node, - shardings: common.Shardings, -) !ComposedKernelExe { - const embed_tokens_exe = try compileEmbedTokens(allocator, io, platform, mdl.embed_tokens, opts, seqlen, progress, shardings); - const conv_layer_exe = try compileConvLayer(allocator, io, platform, mdl, opts, seqlen, is_prefill, progress, shardings); - const attn_layer_exe = try compileAttnLayer(allocator, io, platform, mdl, opts, seqlen, is_prefill, progress, shardings); - const lm_head_exe = try compileLmHead(allocator, io, platform, mdl, opts, seqlen, progress, shardings); - return .{ - .embed_tokens = embed_tokens_exe, - .conv_layer = conv_layer_exe, - .attn_layer = attn_layer_exe, - .lm_head = lm_head_exe, - }; -} - -fn compileEmbedTokens( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - embed_tokens: model.TokenEmbedding, - opts: CompilationOptions, - seqlen: u32, - progress: *std.Progress.Node, - shardings: common.Shardings, -) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start("Compiling embed_tokens...", 1); - defer node.end(); - const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Compiled embed_tokens [{f}]", .{now.untilNow(io, .awake)}); - - const tokens: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32); - const all_shardings = shardings.all(); - return platform.compile(allocator, io, embed_tokens, .forward, .{tokens}, .{ .shardings = &all_shardings }); -} - -fn compileConvLayer( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - mdl: model.Model, - opts: CompilationOptions, - seqlen: u32, - is_prefill: bool, - progress: *std.Progress.Node, - shardings: common.Shardings, -) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start("Compiling conv layer...", 1); - defer node.end(); - const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Compiled conv layer [{f}]", .{now.untilNow(io, .awake)}); - - const conv_layer = for (mdl.layers) |layer| { - if (layer.operator == .conv) break layer; - } else unreachable; - - const hidden: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()); - const token_position_offset: zml.Tensor = .init(.{ .batch = opts.batch_dim }, .u32); - const actual_seq_len: zml.Tensor = .init(.{}, .u32); - const conv_cache_index: zml.Tensor = .init(.{}, .u32); - const kv_cache_index: zml.Tensor = .init(.{}, .u32); - - const all_shardings = shardings.all(); - return platform.compile(allocator, io, conv_layer, .forward, .{ - hidden, - token_position_offset, - actual_seq_len, - opts.cache, - conv_cache_index, - kv_cache_index, - opts.attention_metadata, - opts.attention_parameters, - model.ConvParameters{ .is_prefill = is_prefill }, - }, .{ .shardings = &all_shardings }); -} - -fn compileAttnLayer( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - mdl: model.Model, - opts: CompilationOptions, - seqlen: u32, - is_prefill: bool, - progress: *std.Progress.Node, - shardings: common.Shardings, -) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start("Compiling attn layer...", 1); - defer node.end(); - const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Compiled attn layer [{f}]", .{now.untilNow(io, .awake)}); - - const attn_layer = for (mdl.layers) |layer| { - if (layer.operator == .self_attn) break layer; - } else unreachable; - - const hidden: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()); - const token_position_offset: zml.Tensor = .init(.{ .batch = opts.batch_dim }, .u32); - const actual_seq_len: zml.Tensor = .init(.{}, .u32); - const conv_cache_index: zml.Tensor = .init(.{}, .u32); - const kv_cache_index: zml.Tensor = .init(.{}, .u32); - - const all_shardings = shardings.all(); - return platform.compile(allocator, io, attn_layer, .forward, .{ - hidden, - token_position_offset, - actual_seq_len, - opts.cache, - conv_cache_index, - kv_cache_index, - opts.attention_metadata, - opts.attention_parameters, - model.ConvParameters{ .is_prefill = is_prefill }, - }, .{ .shardings = &all_shardings }); -} - -fn compileLmHead( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - mdl: model.Model, - opts: CompilationOptions, - seqlen: u32, - progress: *std.Progress.Node, - shardings: common.Shardings, -) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start("Compiling lm_head...", 1); - defer node.end(); - const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Compiled lm_head [{f}]", .{now.untilNow(io, .awake)}); - - const hidden: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()); - const tokens: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32); - const all_shardings = shardings.all(); - return platform.compile(allocator, io, mdl.lm_head, .forward, .{ hidden, mdl.embed_tokens, tokens, opts.rng }, .{ .shardings = &all_shardings }); -} diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index 9c09ed58..988f62e5 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -120,7 +120,7 @@ pub const LoadedModel = struct { seqlen: usize, progress: *std.Progress.Node, ) !inference.CompiledModel { - const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), backend, false, shardings); + const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), backend, shardings); return inference.CompiledModel.init(allocator, io, @constCast(platform), self, self.inner, params, progress); } }; @@ -252,7 +252,7 @@ pub const TokenEmbedding = struct { weight: zml.Tensor, pub fn init(store: zml.io.TensorStore.View) TokenEmbedding { - return .{ .weight = store.createTensor("weight", .{ .voc, .d }, .replicated) }; + return .{ .weight = store.createTensor("weight", .{ .voc, .d }, .{ .voc = .replicated, .d = .model }) }; } pub fn forward(self: TokenEmbedding, tokens: zml.Tensor) zml.Tensor { @@ -398,8 +398,8 @@ pub const ShortConv = struct { pub fn init(config: Config, store: zml.io.TensorStore.View) ShortConv { stdx.debug.assert(!config.conv_bias, "conv_bias is not supported.", .{}); return .{ - .in_proj = initLinear(store.withPrefix("in_proj"), .d), - .out_proj = initLinear(store.withPrefix("out_proj"), .d), + .in_proj = .init(store.withPrefix("in_proj").createTensor("weight", .{ .out, .d }, .{ .out = .model, .d = .replicated }), null, .d), + .out_proj = .init(store.withPrefix("out_proj").createTensor("weight", .{ .out, .d }, .{ .out = .replicated, .d = .model }), null, .d), .kernel = store.createTensor("conv.weight", .{ .out, .in, .kernel_size }, .replicated), .config = config, }; @@ -407,7 +407,7 @@ pub const ShortConv = struct { pub fn forward(self: ShortConv, input: zml.Tensor, tokens_position_offset: zml.Tensor, actual_seq_len: zml.Tensor, cache_: ConvCache, cache_index: zml.Tensor, parameters: ConvParameters) struct { zml.Tensor, ConvCache } { var cache = cache_; - const BCx = self.in_proj.forward(input); + const BCx = self.in_proj.forward(input.withPartitioning(.{ .d = .replicated })); const B, const C, const x = BCx.chunkExact(.d, 3); const Bx = B.mul(x); @@ -473,10 +473,10 @@ pub const Attention = struct { const head_dim = config.hidden_size / config.num_attention_heads; const num_key_value_groups = config.num_attention_heads / config.num_key_value_heads; return .{ - .q_proj = initLinear(store.withPrefix("q_proj"), .d), - .k_proj = initLinear(store.withPrefix("k_proj"), .d), - .v_proj = initLinear(store.withPrefix("v_proj"), .d), - .out_proj = initLinear(store.withPrefix("out_proj"), .d), + .q_proj = .init(store.withPrefix("q_proj").createTensor("weight", .{ .out, .d }, .{ .out = .model, .d = .replicated }), null, .d), + .k_proj = .init(store.withPrefix("k_proj").createTensor("weight", .{ .out, .d }, .{ .out = .model, .d = .replicated }), null, .d), + .v_proj = .init(store.withPrefix("v_proj").createTensor("weight", .{ .out, .d }, .{ .out = .model, .d = .replicated }), null, .d), + .out_proj = .init(store.withPrefix("out_proj").createTensor("weight", .{ .out, .d }, .{ .out = .replicated, .d = .model }), null, .d), .q_layernorm = RmsNorm.init(store.withPrefix("q_layernorm"), config.norm_eps, .hd), .k_layernorm = RmsNorm.init(store.withPrefix("k_layernorm"), config.norm_eps, .hd), .head_dim = head_dim, @@ -494,9 +494,14 @@ pub const Attention = struct { attention_metadata: zml.attention.attention.Metadata, attention_parameters: zml.attention.attention.Parameters, ) struct { zml.Tensor, KvCache } { - var q = self.q_proj.forward(x).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); - var k = self.k_proj.forward(x).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); - var v = self.v_proj.forward(x).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); + const x_qkv = x.withPartitioning(.{ .d = .replicated }); + + var q = self.q_proj.forward(x_qkv).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); + var k = self.k_proj.forward(x_qkv).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); + var v = self.v_proj.forward(x_qkv).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); + q = q.withPartitioning(.{ .seq = .replicated, .h = .model, .hd = .replicated }); + k = k.withPartitioning(.{ .seq = .replicated, .h = .model, .hd = .replicated }); + v = v.withPartitioning(.{ .seq = .replicated, .h = .model, .hd = .replicated }); q = self.q_layernorm.forward(q); k = self.k_layernorm.forward(k); @@ -509,18 +514,35 @@ pub const Attention = struct { q = zml.nn.rope(q, token_positions, self.rope_opts); k = zml.nn.rope(k, token_positions, self.rope_opts); - q = q.rename(.{ .seq = .q }); - k = k.rename(.{ .seq = .k }); - v = v.rename(.{ .seq = .k }); + q = q.rename(.{ .seq = .q }).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }); + k = k.rename(.{ .seq = .k }).withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); + v = v.rename(.{ .seq = .k }).withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); const new_kv_cache = kv_cache.update(k, v, tokens_position_offset, cache_index); - k = new_kv_cache.keys(cache_index); - v = new_kv_cache.values(cache_index); + k = new_kv_cache.keys(cache_index).withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); + v = new_kv_cache.values(cache_index).withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); stdx.debug.assert(q.dim(.batch) == 1, "LFM attention currently expects batch size 1 for flash attention backend, got {}", .{q.dim(.batch)}); const attn = switch (attention_parameters) { - .vanilla => zml.attention.attention.attention(q, k, v, tokens_position_offset, attention_metadata, attention_parameters).merge(.{ .d = .{ .h, .hd } }), - else => zml.attention.attention.attention(q.squeeze(.batch), k.squeeze(.batch), v.squeeze(.batch), tokens_position_offset.squeeze(.batch), attention_metadata, attention_parameters).merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .seq }).insertAxes(.seq, .{.batch}), + .vanilla => zml.attention.attention.attention( + q, + k, + v, + tokens_position_offset, + attention_metadata, + attention_parameters, + ).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }).merge(.{ .d = .{ .h, .hd } }), + else => zml.attention.attention.attention( + q.squeeze(.batch), + k.squeeze(.batch), + v.squeeze(.batch), + tokens_position_offset.squeeze(.batch), + attention_metadata, + attention_parameters, + ).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }) + .merge(.{ .d = .{ .h, .hd } }) + .rename(.{ .q = .seq }) + .insertAxes(.seq, .{.batch}), }; return .{ self.out_proj.forward(attn).reuseBuffer(x), new_kv_cache.reuseBuffer(kv_cache) }; @@ -583,7 +605,8 @@ pub const KvCache = struct { v: zml.Tensor, pub fn init(kv_shape: zml.Shape) KvCache { - return .{ .k = .fromShape(kv_shape), .v = .fromShape(kv_shape) }; + const sharded_shape = kv_shape.withPartitioning(.{ .h = .model }); + return .{ .k = .fromShape(sharded_shape), .v = .fromShape(sharded_shape) }; } pub fn initBuffers(self: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(KvCache) { @@ -617,10 +640,6 @@ pub const KvCache = struct { } }; -fn initLinear(store: zml.io.TensorStore.View, tag: anytype) Linear { - return .init(store.createTensor("weight", .{ .out, tag }, .replicated), null, tag); -} - pub const Linear = struct { weight: zml.Tensor, bias: ?zml.Tensor = null, @@ -651,7 +670,11 @@ const Mlp = struct { w3: Linear, pub fn init(store: zml.io.TensorStore.View) Mlp { - return .{ .w1 = initLinear(store.withPrefix("w1"), .d), .w2 = initLinear(store.withPrefix("w2"), .d), .w3 = initLinear(store.withPrefix("w3"), .d) }; + return .{ + .w1 = .init(store.withPrefix("w1").createTensor("weight", .{ .out, .d }, .{ .out = .model, .d = .replicated }), null, .d), + .w2 = .init(store.withPrefix("w2").createTensor("weight", .{ .out, .d }, .{ .out = .replicated, .d = .model }), null, .d), + .w3 = .init(store.withPrefix("w3").createTensor("weight", .{ .out, .d }, .{ .out = .model, .d = .replicated }), null, .d), + }; } pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig index 018ed738..d367cbe6 100644 --- a/examples/llm/models/lfm2/session.zig +++ b/examples/llm/models/lfm2/session.zig @@ -40,7 +40,7 @@ pub const Session = struct { .tokenizer = tokenizer, .config = &compiled_model.loaded_model.parsed_config.value, .seqlen = compiled_model.params.seqlen, - .cache_buffers = try compiled_model.params.cache.initBuffers(allocator, io, platform, .replicated), + .cache_buffers = try compiled_model.params.cache.initBuffers(allocator, io, platform, compiled_model.params.shardings.model), .attention_metadata_buffers = try compiled_model.params.attention_metadata.initBuffer(io, platform, compiled_model.params.shardings.model), .rng_buf = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed), .generated_token_slice = try .alloc(allocator, zml.Shape.init(.{ .batch = 1, .seq = 1 }, .u32)), diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig index 4b110e50..8e60e9d5 100644 --- a/examples/llm/models/lfm2_tests.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -53,7 +53,13 @@ pub fn main(init: std.process.Init) !void { defer repo_model.unloadBuffers(&model_buffers, allocator); const backend = args.backend orelse attention.Backend.auto(platform); - const params = lfm2.CompilationParameters.init(repo_model.inner, repo_model.parsed_config.value, repo_model.parsed_config.value.max_position_embeddings, backend, false, shardings); + const params = lfm2.CompilationParameters.init( + repo_model.inner, + repo_model.parsed_config.value, + repo_model.parsed_config.value.max_position_embeddings, + backend, + shardings, + ); progress.end(); try run(allocator, io, platform, args.activations, repo_model.parsed_config.value, repo_model.inner, &model_buffers, params.attention_metadata, params.attention_parameters); diff --git a/examples/llm/models/llama/inference.zig b/examples/llm/models/llama/inference.zig index 46445c48..c2d13701 100644 --- a/examples/llm/models/llama/inference.zig +++ b/examples/llm/models/llama/inference.zig @@ -7,6 +7,7 @@ const common = @import("../common.zig"); const model = @import("model.zig"); const log = std.log.scoped(.llama); +const Phase = common.Phase; pub const CompilationParameters = struct { prefill_tokens: zml.Tensor, @@ -22,6 +23,7 @@ pub const CompilationParameters = struct { pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: attention.Backend, shardings: common.Shardings) CompilationParameters { const head_dim = config.head_dim orelse @divExact(config.hidden_size, config.num_attention_heads); + return .{ .prefill_tokens = .init(.{ .s = seqlen }, .u32), .decode_tokens = .init(.{ .s = 1 }, .u32), @@ -63,10 +65,24 @@ pub const CompilationParameters = struct { } }; +pub const CompilationOptions = CompilationParameters; + +pub const Args = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + model_buffers: *model.Buffers, + tokens_buf: *zml.Buffer, + token_index_buf: *zml.Buffer, + kv_cache_buffers: *zml.Bufferized(model.KvCache), + rng_buffers: *zml.Bufferized(zml.Tensor.Rng), + attention_metadata_buffers: *const zml.Bufferized(attention.Metadata), +}; + pub const CompiledModel = struct { loaded_model: *const model.LoadedModel, - prefill_exe: zml.Exe, - decode_exe: zml.Exe, + prefill: KernelExe, + decode: KernelExe, params: CompilationParameters, pub fn init( @@ -78,113 +94,505 @@ pub const CompiledModel = struct { parameters: CompilationParameters, progress: *std.Progress.Node, ) !CompiledModel { - const compiled = try compileModel(allocator, io, platform, llama_model, parameters, progress); - return .{ .loaded_model = loaded_model, .prefill_exe = compiled.prefill_exe, .decode_exe = compiled.decode_exe, .params = parameters }; + return .{ + .loaded_model = loaded_model, + .prefill = try KernelExe.init(allocator, io, platform, llama_model, parameters, @intCast(parameters.prefill_tokens.dim(.s)), parameters.prefill_attention_parameters, .prefill, progress), + .decode = try KernelExe.init(allocator, io, platform, llama_model, parameters, @intCast(parameters.decode_tokens.dim(.s)), parameters.decode_attention_parameters, .decode, progress), + .params = parameters, + }; } pub fn deinit(self: *CompiledModel) void { - self.prefill_exe.deinit(); - self.decode_exe.deinit(); + self.prefill.deinit(); + self.decode.deinit(); } }; -const CompileModelResult = struct { - prefill_exe: zml.Exe, - decode_exe: zml.Exe, +pub const Inference = CompiledModel; + +pub const KernelExe = struct { + composed: ComposedKernelExe, + + pub const Runner = struct { + exe: *const ComposedKernelExe, + embed_args: zml.exe.Exe.Arguments, + embed_results: zml.exe.Exe.Results, + layers: Layers, + lm_head_args: zml.exe.Exe.Arguments, + lm_head_results: zml.exe.Exe.Results, + + const Layers = struct { + args: []zml.exe.Exe.Arguments, + results: []zml.exe.Exe.Results, + kv_cache_indices: []zml.Buffer, + + fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + exe: *const ComposedKernelExe, + model_buffers: *model.Buffers, + ) !Layers { + const args = try allocator.alloc(zml.exe.Exe.Arguments, model_buffers.model.layers.len); + errdefer allocator.free(args); + + const results = try allocator.alloc(zml.exe.Exe.Results, model_buffers.model.layers.len); + errdefer allocator.free(results); + + const kv_cache_indices = try allocator.alloc(zml.Buffer, model_buffers.model.layers.len); + errdefer allocator.free(kv_cache_indices); + + var initialized_args: usize = 0; + errdefer { + for (args[0..initialized_args]) |*exe_args| { + exe_args.deinit(allocator); + } + } + + var initialized_results: usize = 0; + errdefer { + for (results[0..initialized_results]) |*exe_results| { + exe_results.deinit(allocator); + } + } + + var initialized_kv_cache_indices: usize = 0; + errdefer { + for (kv_cache_indices[0..initialized_kv_cache_indices]) |*kv_cache_index| { + kv_cache_index.deinit(); + } + } + + for (model_buffers.model.layers, 0..) |layer_bufs, i| { + args[i] = try exe.layer.args(allocator); + initialized_args = i + 1; + args[i].bake(layer_bufs); + + results[i] = try exe.layer.results(allocator); + initialized_results = i + 1; + + kv_cache_indices[i] = try zml.Buffer.scalar(io, platform, i, .u32); + initialized_kv_cache_indices = i + 1; + } + + return .{ .args = args, .results = results, .kv_cache_indices = kv_cache_indices }; + } + + fn deinit(self: *Layers, allocator: std.mem.Allocator) void { + for (self.args) |*exe_args| { + exe_args.deinit(allocator); + } + allocator.free(self.args); + + for (self.results) |*exe_results| { + exe_results.deinit(allocator); + } + allocator.free(self.results); + + for (self.kv_cache_indices) |*kv_cache_index| { + kv_cache_index.deinit(); + } + allocator.free(self.kv_cache_indices); + } + }; + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + exe: *const ComposedKernelExe, + model_buffers: *model.Buffers, + ) !Runner { + var embed_args = try exe.embed_tokens.args(allocator); + errdefer embed_args.deinit(allocator); + + embed_args.bake(ComposedKernelExe.embedTokensBuffers(model_buffers)); + + var embed_results = try exe.embed_tokens.results(allocator); + errdefer embed_results.deinit(allocator); + + var layers = try Layers.init(allocator, io, platform, exe, model_buffers); + errdefer layers.deinit(allocator); + + var lm_head_args = try exe.lm_head.args(allocator); + errdefer lm_head_args.deinit(allocator); + + lm_head_args.bake(ComposedKernelExe.lmHeadBuffers(model_buffers)); + + var lm_head_results = try exe.lm_head.results(allocator); + errdefer lm_head_results.deinit(allocator); + + return .{ + .exe = exe, + .embed_args = embed_args, + .embed_results = embed_results, + .layers = layers, + .lm_head_args = lm_head_args, + .lm_head_results = lm_head_results, + }; + } + + pub fn deinit(self: *Runner, allocator: std.mem.Allocator) void { + self.embed_args.deinit(allocator); + self.embed_results.deinit(allocator); + self.layers.deinit(allocator); + self.lm_head_args.deinit(allocator); + self.lm_head_results.deinit(allocator); + } + + pub fn run(self: *Runner, args: Args) !void { + var hidden_buf: zml.Buffer = b: { + self.embed_args.set(.{args.tokens_buf}); + self.exe.embed_tokens.call(self.embed_args, &self.embed_results); + + break :b self.embed_results.get(zml.Buffer); + }; + defer hidden_buf.deinit(); + + for (self.layers.args, self.layers.results, self.layers.kv_cache_indices) |*exe_args, *results, *kv_cache_index_buf| { + self.exe.runLayer(exe_args, results, args, &hidden_buf, kv_cache_index_buf); + } + + self.exe.runLmHead(&self.lm_head_args, &self.lm_head_results, args, &hidden_buf); + } + }; + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + llama_model: model.Model, + parameters: CompilationOptions, + seqlen: usize, + attention_parameters: attention.Parameters, + phase: Phase, + progress: *std.Progress.Node, + ) !KernelExe { + return .{ + .composed = try ComposedKernelExe.init(allocator, io, platform, llama_model, parameters, seqlen, attention_parameters, phase, progress), + }; + } + + pub fn deinit(self: KernelExe) void { + self.composed.deinit(); + } + + pub fn run(self: *const KernelExe, args: Args) !void { + try self.composed.run(args); + } + + pub fn initRunner( + self: *const KernelExe, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + model_buffers: *model.Buffers, + ) !Runner { + return .init(allocator, io, platform, &self.composed, model_buffers); + } }; -fn compileModel( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - llama_model: model.Model, - parameters: CompilationParameters, - progress: *std.Progress.Node, -) !CompileModelResult { - const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Compiled model [{f}]", .{now.untilNow(io, .awake)}); +const ComposedKernelExe = struct { + embed_tokens: zml.Exe, + layer: zml.Exe, + lm_head: zml.Exe, - const all_shardings = parameters.shardings.all(); + const EmbedTokens = struct { + embed_tokens: zml.nn.TokenEmbedding, - var prefill_future = try io.concurrent(struct { - fn call( - allocator_: std.mem.Allocator, - io_: std.Io, - platform_: *const zml.Platform, - llama_model_: model.Model, - parameters_: CompilationParameters, - shardings_: []const zml.Sharding, - progress_: *std.Progress.Node, - ) !zml.Exe { - progress_.increaseEstimatedTotalItems(1); - var node_ = progress_.start("Compiling prefill...", 1); - defer node_.end(); - - return platform_.compile( - allocator_, - io_, - llama_model_, - .forward, - .{ - parameters_.prefill_tokens, - parameters_.token_index, - parameters_.kv_cache, - parameters_.rng, - parameters_.attention_metadata, - parameters_.prefill_attention_parameters, - }, - .{ - .shardings = shardings_, - }, - ); + pub fn forward(self: EmbedTokens, tokens_: zml.Tensor) zml.Tensor { + const tokens = tokens_.withPartialTags(.{.s}); + return self.embed_tokens.forward(tokens) + .withPartialTags(.{.d}) + .withPartitioning(.{ .d = .replicated }); } - }.call, .{ allocator, io, platform, llama_model, parameters, &all_shardings, progress }); - var prefill_future_awaited = false; - errdefer if (!prefill_future_awaited) if (prefill_future.cancel(io)) |v| v.deinit() else |_| {}; - - var decode_future = try io.concurrent(struct { - fn call( - allocator_: std.mem.Allocator, - io_: std.Io, - platform_: *const zml.Platform, - llama_model_: model.Model, - parameters_: CompilationParameters, - shardings_: []const zml.Sharding, - progress_: *std.Progress.Node, - ) !zml.Exe { - progress_.increaseEstimatedTotalItems(1); - var node_ = progress_.start("Compiling decode...", 1); - defer node_.end(); - - return platform_.compile( - allocator_, - io_, - llama_model_, - .forward, - .{ - parameters_.decode_tokens, - parameters_.token_index, - parameters_.kv_cache, - parameters_.rng, - parameters_.attention_metadata, - parameters_.decode_attention_parameters, - }, - .{ .shardings = shardings_ }, - ); - } - }.call, .{ allocator, io, platform, llama_model, parameters, &all_shardings, progress }); - var decode_future_awaited = false; - errdefer if (!decode_future_awaited) if (decode_future.cancel(io)) |v| v.deinit() else |_| {}; - - const prefill_exe = try prefill_future.await(io); - prefill_future_awaited = true; - errdefer prefill_exe.deinit(); - - const decode_exe = try decode_future.await(io); - decode_future_awaited = true; - - return .{ - .prefill_exe = prefill_exe, - .decode_exe = decode_exe, }; -} + + fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + llama_model: model.Model, + parameters: CompilationOptions, + seqlen: usize, + attention_parameters: attention.Parameters, + phase: Phase, + progress: *std.Progress.Node, + ) !ComposedKernelExe { + const embed_tokens = try ComposedKernelExe.compileEmbedTokens(allocator, io, platform, llama_model.model.embed_tokens, parameters, seqlen, phase, progress); + errdefer embed_tokens.deinit(); + + const layer = try ComposedKernelExe.compileLayer(allocator, io, platform, llama_model, parameters, seqlen, attention_parameters, phase, progress); + errdefer layer.deinit(); + + const lm_head = try ComposedKernelExe.compileLmHead(allocator, io, platform, llama_model, parameters, seqlen, phase, progress); + errdefer lm_head.deinit(); + + return .{ + .embed_tokens = embed_tokens, + .layer = layer, + .lm_head = lm_head, + }; + } + + fn deinit(self: ComposedKernelExe) void { + self.embed_tokens.deinit(); + self.layer.deinit(); + self.lm_head.deinit(); + } + + fn run(self: *const ComposedKernelExe, args: Args) !void { + var hidden_buf: zml.Buffer = b: { + var exe_args = try self.embed_tokens.args(args.allocator); + defer exe_args.deinit(args.allocator); + + var results = try self.embed_tokens.results(args.allocator); + defer results.deinit(args.allocator); + + exe_args.bake(ComposedKernelExe.embedTokensBuffers(args.model_buffers)); + exe_args.set(.{args.tokens_buf}); + + self.embed_tokens.call(exe_args, &results); + + break :b results.get(zml.Buffer); + }; + defer hidden_buf.deinit(); + + for (args.model_buffers.model.layers, 0..) |layer_bufs, i| { + var exe_args = try self.layer.args(args.allocator); + defer exe_args.deinit(args.allocator); + + var results = try self.layer.results(args.allocator); + defer results.deinit(args.allocator); + + var kv_cache_index_buf: zml.Buffer = try .scalar(args.io, args.platform, i, .u32); + defer kv_cache_index_buf.deinit(); + + exe_args.bake(layer_bufs); + + self.runLayer(&exe_args, &results, args, &hidden_buf, &kv_cache_index_buf); + } + + { + var exe_args = try self.lm_head.args(args.allocator); + defer exe_args.deinit(args.allocator); + + var results = try self.lm_head.results(args.allocator); + defer results.deinit(args.allocator); + + exe_args.bake(ComposedKernelExe.lmHeadBuffers(args.model_buffers)); + + self.runLmHead(&exe_args, &results, args, &hidden_buf); + } + } + + fn runLayer( + self: *const ComposedKernelExe, + exe_args: *zml.exe.Exe.Arguments, + results: *zml.exe.Exe.Results, + args: Args, + hidden_buf: *zml.Buffer, + kv_cache_index_buf: *zml.Buffer, + ) void { + exe_args.set(.{ + hidden_buf, + args.token_index_buf, + args.kv_cache_buffers, + kv_cache_index_buf, + args.attention_metadata_buffers, + }); + + self.layer.call(exe_args.*, results); + + var new_hidden, var new_kv_cache = results.get(struct { + zml.Buffer, + zml.Bufferized(model.KvCache), + }); + + ComposedKernelExe.replaceBuffer(hidden_buf, &new_hidden); + ComposedKernelExe.replaceKvCacheBuffers(args.kv_cache_buffers, &new_kv_cache); + } + + fn runLmHead( + self: *const ComposedKernelExe, + exe_args: *zml.exe.Exe.Arguments, + results: *zml.exe.Exe.Results, + args: Args, + hidden_buf: *zml.Buffer, + ) void { + exe_args.set(.{ hidden_buf, args.tokens_buf, args.rng_buffers }); + self.lm_head.call(exe_args.*, results); + + var new_tokens, var new_rng = results.get(struct { + zml.Buffer, + zml.Bufferized(zml.Tensor.Rng), + }); + + ComposedKernelExe.replaceBuffer(args.tokens_buf, &new_tokens); + ComposedKernelExe.replaceBuffer(&args.rng_buffers._state, &new_rng._state); + } + + fn embedTokensBuffers(model_buffers: *const model.Buffers) zml.Bufferized(EmbedTokens) { + return .{ + .embed_tokens = model_buffers.model.embed_tokens, + }; + } + + fn lmHeadBuffers(model_buffers: *const model.Buffers) zml.Bufferized(model.LmHead) { + return .{ + .lm_head = model_buffers.lm_head, + .embed_tokens = model_buffers.model.embed_tokens, + .norm = model_buffers.model.norm, + }; + } + + fn replaceKvCacheBuffers(dst: *zml.Bufferized(model.KvCache), src: *zml.Bufferized(model.KvCache)) void { + ComposedKernelExe.replaceBuffer(&dst.k, &src.k); + ComposedKernelExe.replaceBuffer(&dst.v, &src.v); + } + + fn replaceBuffer(dst: *zml.Buffer, src: *zml.Buffer) void { + if (!ComposedKernelExe.sameBufferHandle(dst.*, src.*)) { + dst.deinit(); + } + + dst.* = src.*; + } + + fn sameBufferHandle(a: zml.Buffer, b: zml.Buffer) bool { + if (a._shards.len != b._shards.len) return false; + + for (a._shards.constSlice(), b._shards.constSlice()) |a_shard, b_shard| { + if (a_shard != b_shard) return false; + } + + return true; + } + + fn compileEmbedTokens( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + embed_tokens: zml.nn.TokenEmbedding, + parameters: CompilationOptions, + seqlen: usize, + phase: Phase, + progress: *std.Progress.Node, + ) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("embed_tokens"), 1); + defer node.end(); + + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "embed_tokens", io, from); + + const tokens: zml.Tensor = .init(.{ .s = seqlen }, .u32); + + return platform.compile(allocator, io, EmbedTokens{ .embed_tokens = embed_tokens }, .forward, .{tokens}, .{ + .shardings = ¶meters.shardings.all(), + .program_name = phase.programName("llama", "embed_tokens"), + }); + } + + fn compileLayer( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + llama_model: model.Model, + parameters: CompilationOptions, + seqlen: usize, + attention_parameters: attention.Parameters, + phase: Phase, + progress: *std.Progress.Node, + ) !zml.Exe { + const Layer = struct { + layer: model.TransformerLayer, + + pub fn forward( + self: @This(), + hidden: zml.Tensor, + token_index: zml.Tensor, + kv_cache: model.KvCache, + kv_cache_index: zml.Tensor, + attention_metadata: attention.Metadata, + attention_parameters_: attention.Parameters, + ) struct { zml.Tensor, model.KvCache } { + const new_hidden, const new_kv_cache, _ = self.layer.forward( + hidden, + token_index, + kv_cache, + kv_cache_index, + attention_metadata, + attention_parameters_, + ); + + return .{ new_hidden, new_kv_cache }; + } + }; + + progress.increaseEstimatedTotalItems(1); + + var node = progress.start(phase.startMessage("transformer layer"), 1); + defer node.end(); + + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "transformer layer", io, from); + + const hidden: zml.Tensor = .fromShape(zml.Shape.init( + .{ .s = seqlen, .d = llama_model.config.hidden_size }, + llama_model.model.embed_tokens.weight.dtype(), + ).withPartitioning(.{ .d = .replicated })); + + const kv_cache_index: zml.Tensor = .init(.{}, .u32); + + return platform.compile( + allocator, + io, + Layer{ .layer = llama_model.model.layers[0] }, + .forward, + .{ + hidden, + parameters.token_index, + parameters.kv_cache, + kv_cache_index, + parameters.attention_metadata, + attention_parameters, + }, + .{ + .shardings = ¶meters.shardings.all(), + .program_name = phase.programName("llama", "layer"), + }, + ); + } + + fn compileLmHead( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + llama_model: model.Model, + parameters: CompilationOptions, + seqlen: usize, + phase: Phase, + progress: *std.Progress.Node, + ) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + + var node = progress.start(phase.startMessage("lm_head"), 1); + defer node.end(); + + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "lm_head", io, from); + + const hidden: zml.Tensor = .fromShape(zml.Shape.init( + .{ .s = seqlen, .d = llama_model.config.hidden_size }, + llama_model.model.embed_tokens.weight.dtype(), + ).withPartitioning(.{ .d = .replicated })); + + const tokens: zml.Tensor = .init(.{ .s = seqlen }, .u32); + + return platform.compile(allocator, io, model.LmHead.init(llama_model), .forward, .{ hidden, tokens, parameters.rng }, .{ + .shardings = ¶meters.shardings.all(), + .program_name = phase.programName("llama", "lm_head"), + }); + } +}; diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index 56d43c36..b808a605 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -25,7 +25,7 @@ pub const Config = struct { rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = .{} }, }; -pub const Options = struct { +const Options = struct { sampling_strategy: ?zml.nn.SamplingStrategy, max_seq_len: u32, }; @@ -81,19 +81,17 @@ pub const LoadedModel = struct { std.log.scoped(.llama).info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); } - const all_shardings = shardings.all(); return zml.io.load(Model, &self.inner, allocator, io, platform, store, .{ .dma_chunks = 32, .dma_chunk_size = 128 * zml.MiB, .progress = progress, - .shardings = &all_shardings, + .shardings = &shardings.all(), .parallelism = 16, .total_bytes = &total_bytes, }); } - pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { - _ = self; + pub fn unloadBuffers(_: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { if (buffers.lm_head) |*lm_head| lm_head.weight.deinit(); Llama.unloadBuffers(&buffers.model, allocator); } @@ -109,6 +107,7 @@ pub const LoadedModel = struct { progress: *std.Progress.Node, ) !inference.CompiledModel { const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), backend, shardings); + return inference.CompiledModel.init(allocator, io, platform, self, self.inner, params, progress); } }; @@ -187,6 +186,11 @@ pub const Model = struct { if (self.lm_head) |*lm_head| lm_head.weight.deinit(); Llama.unloadBuffers(&self.model, allocator); } + + fn lmHead(self: Model) LmHead { + return .init(self); + } + /// Predicts the token at `token_index` position. /// Returns: /// - updated `tokens`, @@ -202,6 +206,7 @@ pub const Model = struct { attention_parameters: zml.attention.attention.Parameters, ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { const tokens = tokens_.withPartialTags(.{.s}); + const out, const updated_kv_cache = self.model.forward( tokens, token_index, @@ -209,37 +214,14 @@ pub const Model = struct { attention_metadata, attention_parameters, ); - const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); - return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; - } + const new_tokens, const new_rng = self.lmHead().forward(out, tokens, rng); - pub fn sampleTokens( - self: Model, - lm_head_: ?zml.nn.Linear, - out_: zml.Tensor, - rng: zml.Tensor.Rng, - opts: zml.nn.SamplingStrategy, - ) struct { zml.Tensor, zml.Tensor.Rng } { - const out = out_.withPartialTags(.{ .s, .d }); - - var logits = blk: { - if (lm_head_) |lm_head| { - break :blk lm_head.forward(out).rename(.{ .dout = .d }); - } else { - break :blk self.model.embed_tokens.weight.withTags(.{ .voc, .d }).dot(out, .d); - } - }; - - if (logits.shape().hasTag(.voc) == null) - logits = logits.rename(.{ .d = .voc }); - - const next_tokens, const new_rng = zml.nn.sampleTokens(logits, opts, rng); - return .{ next_tokens, new_rng }; + return .{ new_tokens, updated_kv_cache, new_rng }; } }; -pub const Llama = struct { +const Llama = struct { embed_tokens: zml.nn.TokenEmbedding, norm: RmsNorm, layers: []TransformerLayer, @@ -289,26 +271,18 @@ pub const Llama = struct { attention_metadata: zml.attention.attention.Metadata, attention_parameters: zml.attention.attention.Parameters, ) struct { zml.Tensor, KvCache } { - const embeds = embed_tokensForward(self.embed_tokens, tokens); + const embeds = self.embed_tokens.forward(tokens).withPartialTags(.{.d}); var hidden = embeds; var updated_kv_cache = kv_cache; + var kv_cache_index = zml.Tensor.scalar(0, .u32); - for (self.layers, 0..) |layer, i| { - const layer_attention_metadata: zml.attention.attention.Metadata = switch (attention_parameters) { - .attnd => .{ .attnd = .{ - .layer_id = zml.Tensor.scalar(i, .u16), - .conversation_id = attention_metadata.attnd.conversation_id, - .num_tokens = attention_metadata.attnd.num_tokens, - } }, - .vanilla => attention_metadata, - .cuda_fa2 => attention_metadata, - .cuda_fa3 => attention_metadata, - }; - hidden, updated_kv_cache = layer.forward( + for (self.layers) |layer| { + hidden, updated_kv_cache, kv_cache_index = layer.forward( hidden, token_index, - updated_kv_cache.atLayer(i), - layer_attention_metadata, + updated_kv_cache, + kv_cache_index, + attention_metadata, attention_parameters, ); } @@ -317,9 +291,40 @@ pub const Llama = struct { } }; -fn embed_tokensForward(embed_tokens_: zml.nn.TokenEmbedding, tokens_: zml.Tensor) zml.Tensor { - return embed_tokens_.forward(tokens_).withPartialTags(.{.d}); -} +pub const LmHead = struct { + lm_head: ?zml.nn.Linear, + embed_tokens: zml.nn.TokenEmbedding, + norm: RmsNorm, + gen_opts: zml.nn.SamplingStrategy, + + pub fn init(mdl: Model) LmHead { + return .{ + .lm_head = mdl.lm_head, + .embed_tokens = mdl.model.embed_tokens, + .norm = mdl.model.norm, + .gen_opts = mdl.gen_opts, + }; + } + + pub fn forward(self: LmHead, hidden_: zml.Tensor, tokens_: zml.Tensor, rng: zml.Tensor.Rng) struct { zml.Tensor, zml.Tensor.Rng } { + const tokens = tokens_.withPartialTags(.{.s}); + const hidden = self.norm.forward(hidden_.withPartialTags(.{ .s, .d })); + + var logits = blk: { + if (self.lm_head) |lm_head| { + break :blk lm_head.forward(hidden).rename(.{ .dout = .d }); + } else { + break :blk self.embed_tokens.weight.withTags(.{ .voc, .d }).dot(hidden, .d); + } + }; + + if (logits.shape().hasTag(.voc) == null) + logits = logits.rename(.{ .d = .voc }); + + const next_tokens, const new_rng = zml.nn.sampleTokens(logits, self.gen_opts, rng); + return .{ next_tokens.convert(tokens.dtype()).reuseBuffer(tokens), new_rng }; + } +}; pub const TransformerLayer = struct { input_layernorm: RmsNorm, @@ -348,9 +353,10 @@ pub const TransformerLayer = struct { x0: zml.Tensor, token_index: zml.Tensor, kv_cache: KvCache, + kv_cache_index: zml.Tensor, attention_metadata: zml.attention.attention.Metadata, attention_parameters: zml.attention.attention.Parameters, - ) struct { zml.Tensor, KvCache } { + ) struct { zml.Tensor, KvCache, zml.Tensor } { // Self Attention //log.debug("TransformerLayer({f}) -> {f}", .{ x0, self.input_layernorm.forward(x0) }); stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {f}", .{x0}); @@ -362,9 +368,11 @@ pub const TransformerLayer = struct { x0_normalized, token_index, kv_cache, + kv_cache_index, attention_metadata, attention_parameters, ); + const updated_kv_cache_index = kv_cache_index.add(zml.Tensor.scalar(@as(u32, 1), .u32)); // Fully Connected const x1 = x0_replicated.add(delta0).withPartitioning(.{ .d = .replicated }); @@ -375,7 +383,7 @@ pub const TransformerLayer = struct { .add(x1) .withPartitioning(.{ .d = .replicated }); - return .{ x2.reuseBuffer(x0), updated_kv_cache }; + return .{ x2.reuseBuffer(x0), updated_kv_cache, updated_kv_cache_index.reuseBuffer(kv_cache_index) }; } }; @@ -432,7 +440,7 @@ const Mlp = struct { } }; -pub const SelfAttn = struct { +const SelfAttn = struct { q_proj: zml.nn.Linear, k_proj: zml.nn.Linear, v_proj: zml.nn.Linear, @@ -491,6 +499,7 @@ pub const SelfAttn = struct { x: zml.Tensor, token_index: zml.Tensor, kv_cache: KvCache, + kv_cache_index: zml.Tensor, attention_metadata: zml.attention.attention.Metadata, attention_parameters: zml.attention.attention.Parameters, ) struct { zml.Tensor, KvCache } { @@ -522,18 +531,29 @@ pub const SelfAttn = struct { v = v.rename(.{ .s = .k }); const dtype = q.dtype(); - const new_kv_cache = kv_cache.update(k, v, token_index); - k = new_kv_cache.keys().convert(dtype); - v = new_kv_cache.values().convert(dtype); + const new_kv_cache = kv_cache.updateAt(k, v, token_index, kv_cache_index); + k = new_kv_cache.keysAt(kv_cache_index).convert(dtype); + v = new_kv_cache.valuesAt(kv_cache_index).convert(dtype); k = k.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); v = v.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); + const layer_attention_metadata: zml.attention.attention.Metadata = switch (attention_parameters) { + .attnd => .{ .attnd = .{ + .layer_id = kv_cache_index.convert(.u16), + .conversation_id = attention_metadata.attnd.conversation_id, + .num_tokens = attention_metadata.attnd.num_tokens, + } }, + .vanilla => attention_metadata, + .cuda_fa2 => attention_metadata, + .cuda_fa3 => attention_metadata, + }; + const attn_output = zml.attention.attention.attention( q, k, v, token_index, - attention_metadata, + layer_attention_metadata, attention_parameters, ).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }); @@ -548,7 +568,6 @@ pub const SelfAttn = struct { pub const KvCache = struct { k: zml.Tensor, v: zml.Tensor, - layer_index: ?u32, pub const Buffer = zml.Bufferized(KvCache); @@ -558,7 +577,6 @@ pub const KvCache = struct { return .{ .k = .fromShape(sharded_shape), .v = .fromShape(sharded_shape), - .layer_index = null, }; } @@ -574,37 +592,19 @@ pub const KvCache = struct { kv.v.deinit(); } - pub fn keys(kv: KvCache) zml.Tensor { - return kv.k.slice1d(.layer, .single(kv.layer_index orelse @panic("forgot to call atLayer"))); + pub fn keysAt(kv: KvCache, layer_index: zml.Tensor) zml.Tensor { + return kv.k.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = layer_index, .len = 1 } }).squeeze(.layer); } - pub fn values(kv: KvCache) zml.Tensor { - return kv.v.slice1d(.layer, .single(kv.layer_index orelse @panic("forgot to call atLayer"))); + pub fn valuesAt(kv: KvCache, layer_index: zml.Tensor) zml.Tensor { + return kv.v.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = layer_index, .len = 1 } }).squeeze(.layer); } - pub fn update(kv: KvCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: ?zml.Tensor) KvCache { + pub fn updateAt(kv: KvCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: zml.Tensor, layer_index: zml.Tensor) KvCache { const k_shape = kv.k.shape().drop(.layer); - const layer: zml.Tensor = .scalar(kv.layer_index orelse @panic("forgot to call atLayer"), .u32); - - return if (token_index) |idx| - .{ - .k = kv.k.scatterSlices(.{ .layer = layer, .k = idx }, new_k.convert(kv.k.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(kv.k), - .v = kv.v.scatterSlices(.{ .layer = layer, .k = idx }, new_v.convert(kv.v.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(kv.v), - .layer_index = kv.layer_index, - } - else - .{ - .k = kv.k.scatterSlices(.{ .layer = layer }, new_k.convert(kv.k.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(kv.k), - .v = kv.v.scatterSlices(.{ .layer = layer }, new_v.convert(kv.v.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(kv.v), - .layer_index = kv.layer_index, - }; - } - - pub fn atLayer(kv: KvCache, layer_index: usize) KvCache { return .{ - .k = kv.k, - .v = kv.v, - .layer_index = @intCast(layer_index), + .k = kv.k.scatterSlices(.{ .layer = layer_index, .k = token_index }, new_k.convert(kv.k.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(kv.k), + .v = kv.v.scatterSlices(.{ .layer = layer_index, .k = token_index }, new_v.convert(kv.v.dtype()).transpose(kv.v.shape().drop(.layer)), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(kv.v), }; } @@ -612,7 +612,6 @@ pub const KvCache = struct { return .{ .k = kv.k.reuseBuffer(other.k), .v = kv.v.reuseBuffer(other.v), - .layer_index = null, }; } }; diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig index 464e36b7..4d1ce119 100644 --- a/examples/llm/models/llama/session.zig +++ b/examples/llm/models/llama/session.zig @@ -11,8 +11,11 @@ pub const Session = struct { platform: *const zml.Platform, model_buffers: *model.Buffers, compiled_model: *const inference.CompiledModel, + decode_runner: inference.KernelExe.Runner, kv_cache_buffers: zml.Bufferized(model.KvCache), + token_index_buffers: []zml.Buffer, attention_metadata_buffers: zml.Bufferized(zml.attention.attention.Metadata), + decode_attention_metadata_buffers: ?zml.Bufferized(zml.attention.attention.Metadata), rng_buffers: zml.Bufferized(zml.Tensor.Rng), tokenizer: zml.tokenizer.Tokenizer, config: *const model.Config, @@ -32,32 +35,72 @@ pub const Session = struct { var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform, shardings.model); errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); + const token_index_buffers = try allocator.alloc(zml.Buffer, compiled_model.params.seqlen); + errdefer allocator.free(token_index_buffers); + var initialized_token_index_buffers: usize = 0; + errdefer { + for (token_index_buffers[0..initialized_token_index_buffers]) |*token_index_buffer| { + token_index_buffer.deinit(); + } + } + for (token_index_buffers, 0..) |*token_index_buffer, i| { + token_index_buffer.* = try zml.Buffer.scalar(io, platform, i, .u32); + initialized_token_index_buffers = i + 1; + } + var attention_metadata_buffers = try compiled_model.params.attention_metadata.initBuffer(io, platform, shardings.model); errdefer zml.attention.attention.Metadata.deinitBuffer(&attention_metadata_buffers); + const conversation_id: u64 = @bitCast(std.Io.Clock.now(.real, io).toMicroseconds()); + + var decode_attention_metadata_buffers: ?zml.Bufferized(zml.attention.attention.Metadata) = switch (compiled_model.params.decode_attention_parameters) { + .attnd => b: { + const buffers: zml.Bufferized(zml.attention.attention.Metadata) = .{ .attnd = .{ + .conversation_id = try zml.Buffer.scalar(io, platform, conversation_id, .u64), + .layer_id = try zml.Buffer.scalar(io, platform, 0, .u16), + .num_tokens = try zml.Buffer.scalar(io, platform, 1, .u32), + } }; + break :b buffers; + }, + .vanilla, .cuda_fa2, .cuda_fa3 => null, + }; + errdefer if (decode_attention_metadata_buffers) |*buffers| zml.attention.attention.Metadata.deinitBuffer(buffers); + const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); var rng_buffers = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); + var decode_runner = try compiled_model.decode.initRunner(allocator, io, platform, model_buffers); + errdefer decode_runner.deinit(allocator); + return .{ .allocator = allocator, .io = io, .platform = platform, .model_buffers = model_buffers, .compiled_model = compiled_model, + .decode_runner = decode_runner, .kv_cache_buffers = kv_cache_buffers, + .token_index_buffers = token_index_buffers, .attention_metadata_buffers = attention_metadata_buffers, + .decode_attention_metadata_buffers = decode_attention_metadata_buffers, .rng_buffers = rng_buffers, .tokenizer = tokenizer, .config = &compiled_model.loaded_model.parsed_config.value, .seqlen = @intCast(compiled_model.params.seqlen), - .conversation_id = @bitCast(std.Io.Clock.now(.real, io).toMicroseconds()), + .conversation_id = conversation_id, }; } pub fn deinit(self: *Session) void { + self.decode_runner.deinit(self.allocator); model.KvCache.deinitBuffer(&self.kv_cache_buffers); + for (self.token_index_buffers) |*token_index_buffer| { + token_index_buffer.deinit(); + } + self.allocator.free(self.token_index_buffers); zml.attention.attention.Metadata.deinitBuffer(&self.attention_metadata_buffers); + if (self.decode_attention_metadata_buffers) |*buffers| zml.attention.attention.Metadata.deinitBuffer(buffers); zml.Tensor.Rng.deinitBuffer(&self.rng_buffers); } @@ -110,22 +153,14 @@ pub const Session = struct { } pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { - var prefill_args = try self.compiled_model.prefill_exe.args(self.allocator); - defer prefill_args.deinit(self.allocator); - - var prefill_results = try self.compiled_model.prefill_exe.results(self.allocator); - defer prefill_results.deinit(self.allocator); - const prefill_tokens_slice: zml.Slice = try .alloc(self.allocator, .init(.{self.seqlen}, .u32)); defer prefill_tokens_slice.free(self.allocator); + @memset(prefill_tokens_slice.items(u32), 0); @memcpy(prefill_tokens_slice.items(u32)[0..all_tokens.len], all_tokens); var prefill_tokens_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, prefill_tokens_slice, .replicated); defer prefill_tokens_buffer.deinit(); - var prefill_token_pos_buffer = try zml.Buffer.scalar(self.io, self.platform, 0, .u32); - defer prefill_token_pos_buffer.deinit(); - const attention_metadata_buffers: zml.Bufferized(zml.attention.attention.Metadata) = switch (self.compiled_model.params.prefill_attention_parameters) { .attnd => .{ .attnd = .{ .conversation_id = try zml.Buffer.scalar(self.io, self.platform, self.conversation_id, .u64), @@ -135,17 +170,17 @@ pub const Session = struct { .vanilla, .cuda_fa2, .cuda_fa3 => self.attention_metadata_buffers, }; - prefill_args.set(.{ - self.model_buffers, - prefill_tokens_buffer, - prefill_token_pos_buffer, - &self.kv_cache_buffers, - &self.rng_buffers, - &attention_metadata_buffers, + try self.compiled_model.prefill.run(.{ + .allocator = self.allocator, + .io = self.io, + .platform = self.platform, + .model_buffers = self.model_buffers, + .tokens_buf = &prefill_tokens_buffer, + .token_index_buf = &self.token_index_buffers[0], + .kv_cache_buffers = &self.kv_cache_buffers, + .rng_buffers = &self.rng_buffers, + .attention_metadata_buffers = &attention_metadata_buffers, }); - self.compiled_model.prefill_exe.call(prefill_args, &prefill_results); - - prefill_results.fill(.{ &prefill_tokens_buffer, &self.kv_cache_buffers, &self.rng_buffers }); try prefill_tokens_buffer.toSlice(self.io, prefill_tokens_slice); self.last_generated_token = prefill_tokens_slice.items(u32)[all_tokens.len - 1]; @@ -158,14 +193,6 @@ pub const Session = struct { const decoder_out_buffer: []u8 = try self.allocator.alloc(u8, 256); defer self.allocator.free(decoder_out_buffer); - var decode_args = try self.compiled_model.decode_exe.args(self.allocator); - defer decode_args.deinit(self.allocator); - // Fix model buffers, since they are stable for the full gen. - decode_args.bake(self.model_buffers.*); - - var decode_results = try self.compiled_model.decode_exe.results(self.allocator); - defer decode_results.deinit(self.allocator); - var last_token_id: u32 = self.last_generated_token; var current_token_buffer: zml.Buffer = try .fromBytes(self.io, self.platform, .init(.{ .s = 1 }, .u32), .replicated, @ptrCast(&last_token_id)); defer current_token_buffer.deinit(); @@ -179,28 +206,20 @@ pub const Session = struct { try all_tokens.append(self.allocator, last_token_id); if (all_tokens.items.len >= self.seqlen) break :generation; - var token_pos_buffer: zml.Buffer = try .scalar(self.io, self.platform, all_tokens.items.len, .u32); - defer token_pos_buffer.deinit(); + const attention_metadata_buffers: zml.Bufferized(zml.attention.attention.Metadata) = + self.decode_attention_metadata_buffers orelse self.attention_metadata_buffers; - const attention_metadata_buffers: zml.Bufferized(zml.attention.attention.Metadata) = switch (self.compiled_model.params.decode_attention_parameters) { - .attnd => .{ .attnd = .{ - .conversation_id = try zml.Buffer.scalar(self.io, self.platform, self.conversation_id, .u64), - .layer_id = try zml.Buffer.scalar(self.io, self.platform, 0, .u16), - .num_tokens = try zml.Buffer.scalar(self.io, self.platform, 1, .u32), - } }, - .vanilla, .cuda_fa2, .cuda_fa3 => self.attention_metadata_buffers, - }; - - decode_args.set(.{ - current_token_buffer, - token_pos_buffer, - &self.kv_cache_buffers, - &self.rng_buffers, - &attention_metadata_buffers, + try self.decode_runner.run(.{ + .allocator = self.allocator, + .io = self.io, + .platform = self.platform, + .model_buffers = self.model_buffers, + .tokens_buf = ¤t_token_buffer, + .token_index_buf = &self.token_index_buffers[all_tokens.items.len], + .kv_cache_buffers = &self.kv_cache_buffers, + .rng_buffers = &self.rng_buffers, + .attention_metadata_buffers = &attention_metadata_buffers, }); - self.compiled_model.decode_exe.call(decode_args, &decode_results); - - decode_results.fill(.{ ¤t_token_buffer, &self.kv_cache_buffers, &self.rng_buffers }); last_token_id = try current_token_buffer.getValue(u32, self.io); } diff --git a/examples/llm/models/qwen3_5/inference.zig b/examples/llm/models/qwen3_5/inference.zig index 2896caa4..b856feee 100644 --- a/examples/llm/models/qwen3_5/inference.zig +++ b/examples/llm/models/qwen3_5/inference.zig @@ -6,11 +6,7 @@ const common = @import("../common.zig"); const model = @import("model.zig"); const log = std.log.scoped(.qwen3_5); - -const CompileModelResult = struct { - prefill_exe: ?zml.Exe = null, - decode_exe: ?zml.Exe = null, -}; +const Phase = common.Phase; pub const CompilationParameters = struct { prefill_tokens: zml.Tensor, @@ -38,10 +34,21 @@ pub const CompilationParameters = struct { pub const CompilationOptions = CompilationParameters; +pub const Args = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + model_buffers: *model.Buffers, + tokens_buf: *zml.Buffer, + token_index_buf: *zml.Buffer, + kv_cache_buffers: *zml.Bufferized(model.KvCache), + rng_buffers: *zml.Bufferized(zml.Tensor.Rng), +}; + pub const CompiledModel = struct { loaded_model: *const model.LoadedModel, - prefill_exe: zml.Exe, - decode_exe: zml.Exe, + prefill: KernelExe, + decode: KernelExe, params: CompilationParameters, pub fn init( @@ -53,104 +60,662 @@ pub const CompiledModel = struct { parameters: CompilationParameters, progress: *std.Progress.Node, ) !CompiledModel { - const compile_result = try compileModel(allocator, io, platform, qwen_model, parameters, progress); - return .{ .loaded_model = loaded_model, .prefill_exe = compile_result.prefill_exe.?, .decode_exe = compile_result.decode_exe.?, .params = parameters }; + return .{ + .loaded_model = loaded_model, + .prefill = try .init( + allocator, + io, + platform, + qwen_model, + parameters, + @intCast(parameters.prefill_tokens.dim(.s)), + .prefill, + progress, + ), + .decode = try .init( + allocator, + io, + platform, + qwen_model, + parameters, + @intCast(parameters.decode_tokens.dim(.s)), + .decode, + progress, + ), + .params = parameters, + }; } pub fn deinit(self: *CompiledModel) void { - self.prefill_exe.deinit(); - self.decode_exe.deinit(); + self.prefill.deinit(); + self.decode.deinit(); } }; pub const Inference = CompiledModel; -fn compileModel( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - qwen_model: model.Model, - parameters: CompilationParameters, - progress: *std.Progress.Node, -) !CompileModelResult { - const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Compiled model [{f}]", .{now.untilNow(io, .awake)}); +pub const KernelExe = struct { + composed: ComposedKernelExe, - const all_shardings = parameters.shardings.all(); + pub const Runner = struct { + exe: *const ComposedKernelExe, + embed_args: zml.exe.Exe.Arguments, + embed_results: zml.exe.Exe.Results, + layers: Layers, + sampler_args: zml.exe.Exe.Arguments, + sampler_results: zml.exe.Exe.Results, - var prefill_future = try io.concurrent(struct { - fn call( - allocator_: std.mem.Allocator, - io_: std.Io, - platform_: *const zml.Platform, - qwen_model_: model.Model, - parameters_: CompilationParameters, - shardings_: []const zml.Sharding, - progress_: *std.Progress.Node, - ) !zml.Exe { - progress_.increaseEstimatedTotalItems(1); - var node_ = progress_.start("Compiling prefill...", 1); - defer node_.end(); + const Layers = struct { + args: []zml.exe.Exe.Arguments, + results: []zml.exe.Exe.Results, + layer_indices: []zml.Buffer, + layer_types: []const model.LayerType, - const now_: std.Io.Timestamp = .now(io_, .awake); - defer log.info("Compiled prefill [{f}]", .{now_.untilNow(io_, .awake)}); + fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + exe: *const ComposedKernelExe, + model_buffers: *model.Buffers, + ) !Layers { + const args = try allocator.alloc(zml.exe.Exe.Arguments, model_buffers.text_model.layers.len); + errdefer allocator.free(args); - return platform_.compile( - allocator_, - io_, - qwen_model_, - .forward, - .{ - parameters_.prefill_tokens, - parameters_.token_index, - parameters_.kv_cache, - parameters_.rng, - }, - .{ .shardings = shardings_ }, - ); + const results = try allocator.alloc(zml.exe.Exe.Results, model_buffers.text_model.layers.len); + errdefer allocator.free(results); + + const layer_indices = try allocator.alloc(zml.Buffer, model_buffers.text_model.layers.len); + errdefer allocator.free(layer_indices); + + var initialized_args: usize = 0; + errdefer { + for (args[0..initialized_args]) |*exe_args| { + exe_args.deinit(allocator); + } + } + + var initialized_results: usize = 0; + errdefer { + for (results[0..initialized_results]) |*exe_results| { + exe_results.deinit(allocator); + } + } + + var initialized_layer_indices: usize = 0; + errdefer { + for (layer_indices[0..initialized_layer_indices]) |*layer_index| { + layer_index.deinit(); + } + } + + var self_attn_layer_index: usize = 0; + var linear_attn_layer_index: usize = 0; + for (model_buffers.text_model.layers, exe.layer_types, 0..) |layer_bufs, layer_type, i| { + args[i] = try switch (layer_type) { + .full_attention => (exe.full_attention_layer orelse unreachable).args(allocator), + .linear_attention => (exe.linear_attention_layer orelse unreachable).args(allocator), + }; + initialized_args = i + 1; + args[i].bake(layer_bufs); + + results[i] = try switch (layer_type) { + .full_attention => (exe.full_attention_layer orelse unreachable).results(allocator), + .linear_attention => (exe.linear_attention_layer orelse unreachable).results(allocator), + }; + initialized_results = i + 1; + + layer_indices[i] = try switch (layer_type) { + .full_attention => b: { + defer self_attn_layer_index += 1; + break :b zml.Buffer.scalar(io, platform, @as(u32, @intCast(self_attn_layer_index)), .u32); + }, + .linear_attention => b: { + defer linear_attn_layer_index += 1; + break :b zml.Buffer.scalar(io, platform, @as(u32, @intCast(linear_attn_layer_index)), .u32); + }, + }; + initialized_layer_indices = i + 1; + } + + return .{ + .args = args, + .results = results, + .layer_indices = layer_indices, + .layer_types = exe.layer_types, + }; + } + + fn deinit(self: *Layers, allocator: std.mem.Allocator) void { + for (self.args) |*exe_args| { + exe_args.deinit(allocator); + } + allocator.free(self.args); + + for (self.results) |*exe_results| { + exe_results.deinit(allocator); + } + allocator.free(self.results); + + for (self.layer_indices) |*layer_index| { + layer_index.deinit(); + } + allocator.free(self.layer_indices); + } + }; + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + exe: *const ComposedKernelExe, + model_buffers: *model.Buffers, + ) !Runner { + var embed_args = try exe.embed_tokens.args(allocator); + errdefer embed_args.deinit(allocator); + embed_args.bake(ComposedKernelExe.embedTokensBuffers(model_buffers)); + + var embed_results = try exe.embed_tokens.results(allocator); + errdefer embed_results.deinit(allocator); + + var layers = try Layers.init(allocator, io, platform, exe, model_buffers); + errdefer layers.deinit(allocator); + + var sampler_args = try exe.sampler.args(allocator); + errdefer sampler_args.deinit(allocator); + sampler_args.bake(ComposedKernelExe.samplerBuffers(model_buffers)); + + var sampler_results = try exe.sampler.results(allocator); + errdefer sampler_results.deinit(allocator); + + return .{ + .exe = exe, + .embed_args = embed_args, + .embed_results = embed_results, + .layers = layers, + .sampler_args = sampler_args, + .sampler_results = sampler_results, + }; } - }.call, .{ allocator, io, platform, qwen_model, parameters, &all_shardings, progress }); - errdefer if (prefill_future.cancel(io)) |v| v.deinit() else |_| {}; - var decode_future = try io.concurrent(struct { - fn call( - allocator_: std.mem.Allocator, - io_: std.Io, - platform_: *const zml.Platform, - qwen_model_: model.Model, - parameters_: CompilationParameters, - shardings_: []const zml.Sharding, - progress_: *std.Progress.Node, - ) !zml.Exe { - progress_.increaseEstimatedTotalItems(1); - var node_ = progress_.start("Compiling decode...", 1); - defer node_.end(); - - const now_: std.Io.Timestamp = .now(io_, .awake); - defer log.info("Compiled decode [{f}]", .{now_.untilNow(io_, .awake)}); - - return platform_.compile( - allocator_, - io_, - qwen_model_, - .forward, - .{ - parameters_.decode_tokens, - parameters_.token_index, - parameters_.kv_cache, - parameters_.rng, - }, - .{ .shardings = shardings_ }, - ); + pub fn deinit(self: *Runner, allocator: std.mem.Allocator) void { + self.embed_args.deinit(allocator); + self.embed_results.deinit(allocator); + self.layers.deinit(allocator); + self.sampler_args.deinit(allocator); + self.sampler_results.deinit(allocator); } - }.call, .{ allocator, io, platform, qwen_model, parameters, &all_shardings, progress }); - errdefer if (decode_future.cancel(io)) |v| v.deinit() else |_| {}; - const prefill_exe = try prefill_future.await(io); - const decode_exe = try decode_future.await(io); + pub fn run(self: *Runner, args: Args) !void { + var hidden_buf: zml.Buffer = b: { + self.embed_args.set(.{args.tokens_buf}); + self.exe.embed_tokens.call(self.embed_args, &self.embed_results); - return .{ - .prefill_exe = prefill_exe, - .decode_exe = decode_exe, + break :b self.embed_results.get(zml.Buffer); + }; + defer hidden_buf.deinit(); + + for ( + self.layers.args, + self.layers.results, + self.layers.layer_indices, + self.layers.layer_types, + ) |*exe_args, *results, *layer_index_buf, layer_type| { + self.exe.runLayer(exe_args, results, layer_type, args, &hidden_buf, layer_index_buf); + } + + self.exe.runSampler(&self.sampler_args, &self.sampler_results, args, &hidden_buf); + } }; -} + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + qwen_model: model.Model, + parameters: CompilationOptions, + seqlen: usize, + phase: Phase, + progress: *std.Progress.Node, + ) !KernelExe { + return .{ + .composed = try .init(allocator, io, platform, qwen_model, parameters, seqlen, phase, progress), + }; + } + + pub fn deinit(self: KernelExe) void { + self.composed.deinit(); + } + + pub fn run(self: *const KernelExe, args: Args) !void { + try self.composed.run(args); + } + + pub fn initRunner( + self: *const KernelExe, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + model_buffers: *model.Buffers, + ) !Runner { + return .init(allocator, io, platform, &self.composed, model_buffers); + } +}; + +const ComposedKernelExe = struct { + embed_tokens: zml.Exe, + full_attention_layer: ?zml.Exe, + linear_attention_layer: ?zml.Exe, + sampler: zml.Exe, + layer_types: []const model.LayerType, + phase: Phase, + + const EmbedTokens = struct { + embed_tokens: zml.nn.TokenEmbedding, + + pub fn forward(self: EmbedTokens, tokens_: zml.Tensor) zml.Tensor { + const tokens = tokens_.withPartialTags(.{.s}); + return self.embed_tokens.forward(tokens) + .withPartialTags(.{.d}) + .withPartitioning(.{ .d = .replicated }); + } + }; + + fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + qwen_model: model.Model, + parameters: CompilationOptions, + seqlen: usize, + phase: Phase, + progress: *std.Progress.Node, + ) !ComposedKernelExe { + const embed_tokens = try ComposedKernelExe.compileEmbedTokens(allocator, io, platform, qwen_model.text_model.embed_tokens, parameters, seqlen, phase, progress); + errdefer embed_tokens.deinit(); + + const full_attention_layer: ?zml.Exe = b: { + const index = ComposedKernelExe.findFirstLayerIndex(qwen_model.config.text_config.layer_types, .full_attention) orelse break :b null; + break :b try ComposedKernelExe.compileFullAttentionLayer(allocator, io, platform, qwen_model, parameters, seqlen, index, phase, progress); + }; + errdefer if (full_attention_layer) |exe| exe.deinit(); + + const linear_attention_layer: ?zml.Exe = b: { + const index = ComposedKernelExe.findFirstLayerIndex(qwen_model.config.text_config.layer_types, .linear_attention) orelse break :b null; + break :b try ComposedKernelExe.compileLinearAttentionLayer(allocator, io, platform, qwen_model, parameters, seqlen, index, phase, progress); + }; + errdefer if (linear_attention_layer) |exe| exe.deinit(); + + const sampler = try ComposedKernelExe.compileSampler(allocator, io, platform, qwen_model, parameters, seqlen, phase, progress); + errdefer sampler.deinit(); + + return .{ + .embed_tokens = embed_tokens, + .full_attention_layer = full_attention_layer, + .linear_attention_layer = linear_attention_layer, + .sampler = sampler, + .layer_types = qwen_model.config.text_config.layer_types, + .phase = phase, + }; + } + + fn deinit(self: ComposedKernelExe) void { + self.embed_tokens.deinit(); + if (self.full_attention_layer) |exe| exe.deinit(); + if (self.linear_attention_layer) |exe| exe.deinit(); + self.sampler.deinit(); + } + + fn run(self: *const ComposedKernelExe, args: Args) !void { + var hidden_buf: zml.Buffer = b: { + var exe_args = try self.embed_tokens.args(args.allocator); + defer exe_args.deinit(args.allocator); + + var results = try self.embed_tokens.results(args.allocator); + defer results.deinit(args.allocator); + + exe_args.bake(ComposedKernelExe.embedTokensBuffers(args.model_buffers)); + exe_args.set(.{args.tokens_buf}); + + self.embed_tokens.call(exe_args, &results); + + break :b results.get(zml.Buffer); + }; + defer hidden_buf.deinit(); + + var self_attn_layer_index: usize = 0; + var linear_attn_layer_index: usize = 0; + for (args.model_buffers.text_model.layers, self.layer_types) |layer_bufs, layer_type| { + var exe_args = try switch (layer_type) { + .full_attention => (self.full_attention_layer orelse unreachable).args(args.allocator), + .linear_attention => (self.linear_attention_layer orelse unreachable).args(args.allocator), + }; + defer exe_args.deinit(args.allocator); + exe_args.bake(layer_bufs); + + var results = try switch (layer_type) { + .full_attention => (self.full_attention_layer orelse unreachable).results(args.allocator), + .linear_attention => (self.linear_attention_layer orelse unreachable).results(args.allocator), + }; + defer results.deinit(args.allocator); + + var layer_index_buf: zml.Buffer = try switch (layer_type) { + .full_attention => b: { + defer self_attn_layer_index += 1; + break :b zml.Buffer.scalar(args.io, args.platform, @as(u32, @intCast(self_attn_layer_index)), .u32); + }, + .linear_attention => b: { + defer linear_attn_layer_index += 1; + break :b zml.Buffer.scalar(args.io, args.platform, @as(u32, @intCast(linear_attn_layer_index)), .u32); + }, + }; + defer layer_index_buf.deinit(); + + self.runLayer(&exe_args, &results, layer_type, args, &hidden_buf, &layer_index_buf); + } + + { + var exe_args = try self.sampler.args(args.allocator); + defer exe_args.deinit(args.allocator); + + var results = try self.sampler.results(args.allocator); + defer results.deinit(args.allocator); + + exe_args.bake(ComposedKernelExe.samplerBuffers(args.model_buffers)); + + self.runSampler(&exe_args, &results, args, &hidden_buf); + } + } + + fn runLayer( + self: *const ComposedKernelExe, + exe_args: *zml.exe.Exe.Arguments, + results: *zml.exe.Exe.Results, + layer_type: model.LayerType, + args: Args, + hidden_buf: *zml.Buffer, + layer_index_buf: *zml.Buffer, + ) void { + switch (layer_type) { + .full_attention => { + const layer_cache: zml.Bufferized(model.KvCache.SelfAttnCache) = .{ + .k = args.kv_cache_buffers.self_attn.k, + .v = args.kv_cache_buffers.self_attn.v, + .layer_index = layer_index_buf.*, + }; + exe_args.set(.{ hidden_buf, args.token_index_buf, layer_cache }); + (self.full_attention_layer orelse unreachable).call(exe_args.*, results); + + var new_hidden, var new_cache = results.get(struct { + zml.Buffer, + zml.Bufferized(model.KvCache.SelfAttnCache), + }); + ComposedKernelExe.replaceBuffer(hidden_buf, &new_hidden); + ComposedKernelExe.replaceBuffer(&args.kv_cache_buffers.self_attn.k, &new_cache.k); + ComposedKernelExe.replaceBuffer(&args.kv_cache_buffers.self_attn.v, &new_cache.v); + ComposedKernelExe.releaseBuffer(layer_index_buf.*, &new_cache.layer_index); + }, + .linear_attention => { + const layer_cache: zml.Bufferized(model.KvCache.GatedDeltaNetCache) = .{ + .conv_state = args.kv_cache_buffers.gated_delta_net.conv_state, + .recurrent_state = args.kv_cache_buffers.gated_delta_net.recurrent_state, + .layer_index = layer_index_buf.*, + }; + exe_args.set(.{ hidden_buf, args.token_index_buf, layer_cache }); + (self.linear_attention_layer orelse unreachable).call(exe_args.*, results); + + var new_hidden, var new_cache = results.get(struct { + zml.Buffer, + zml.Bufferized(model.KvCache.GatedDeltaNetCache), + }); + ComposedKernelExe.replaceBuffer(hidden_buf, &new_hidden); + ComposedKernelExe.replaceBuffer(&args.kv_cache_buffers.gated_delta_net.conv_state, &new_cache.conv_state); + ComposedKernelExe.replaceBuffer(&args.kv_cache_buffers.gated_delta_net.recurrent_state, &new_cache.recurrent_state); + ComposedKernelExe.releaseBuffer(layer_index_buf.*, &new_cache.layer_index); + }, + } + } + + fn runSampler( + self: *const ComposedKernelExe, + exe_args: *zml.exe.Exe.Arguments, + results: *zml.exe.Exe.Results, + args: Args, + hidden_buf: *zml.Buffer, + ) void { + switch (self.phase) { + .prefill => self.runPrefillSampler(exe_args, results, args, hidden_buf), + .decode => self.runDecodeSampler(exe_args, results, args, hidden_buf), + } + } + + fn runPrefillSampler( + self: *const ComposedKernelExe, + exe_args: *zml.exe.Exe.Arguments, + results: *zml.exe.Exe.Results, + args: Args, + hidden_buf: *zml.Buffer, + ) void { + exe_args.set(.{ hidden_buf, args.rng_buffers }); + self.sampler.call(exe_args.*, results); + + var new_tokens, var new_rng = results.get(struct { + zml.Buffer, + zml.Bufferized(zml.Tensor.Rng), + }); + + ComposedKernelExe.replaceBuffer(args.tokens_buf, &new_tokens); + ComposedKernelExe.replaceBuffer(&args.rng_buffers._state, &new_rng._state); + } + + fn runDecodeSampler( + self: *const ComposedKernelExe, + exe_args: *zml.exe.Exe.Arguments, + results: *zml.exe.Exe.Results, + args: Args, + hidden_buf: *zml.Buffer, + ) void { + exe_args.set(.{ hidden_buf, args.rng_buffers, args.token_index_buf }); + self.sampler.call(exe_args.*, results); + + var new_tokens, var new_rng, var new_token_index = results.get(struct { + zml.Buffer, + zml.Bufferized(zml.Tensor.Rng), + zml.Buffer, + }); + + ComposedKernelExe.replaceBuffer(args.tokens_buf, &new_tokens); + ComposedKernelExe.replaceBuffer(&args.rng_buffers._state, &new_rng._state); + ComposedKernelExe.replaceBuffer(args.token_index_buf, &new_token_index); + } + + fn embedTokensBuffers(model_buffers: *const model.Buffers) zml.Bufferized(EmbedTokens) { + return .{ + .embed_tokens = model_buffers.text_model.embed_tokens, + }; + } + + fn samplerBuffers(model_buffers: *const model.Buffers) zml.Bufferized(model.Sampler) { + return .{ + .norm = model_buffers.text_model.norm, + .lm_head = model_buffers.lm_head, + }; + } + + fn replaceBuffer(dst: *zml.Buffer, src: *zml.Buffer) void { + if (!ComposedKernelExe.sameBufferHandle(dst.*, src.*)) { + dst.deinit(); + } + + dst.* = src.*; + } + + fn releaseBuffer(expected: zml.Buffer, actual: *zml.Buffer) void { + if (!ComposedKernelExe.sameBufferHandle(expected, actual.*)) { + actual.deinit(); + } + } + + fn sameBufferHandle(a: zml.Buffer, b: zml.Buffer) bool { + if (a._shards.len != b._shards.len) return false; + + for (a._shards.constSlice(), b._shards.constSlice()) |a_shard, b_shard| { + if (a_shard != b_shard) return false; + } + + return true; + } + + fn compileEmbedTokens( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + embed_tokens: zml.nn.TokenEmbedding, + parameters: CompilationOptions, + seqlen: usize, + phase: Phase, + progress: *std.Progress.Node, + ) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("embed tokens"), 1); + defer node.end(); + + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "embed tokens", io, from); + + const tokens: zml.Tensor = .init(.{ .b = 1, .s = seqlen }, .u32); + + return platform.compile(allocator, io, EmbedTokens{ .embed_tokens = embed_tokens }, .forward, .{tokens}, .{ + .shardings = ¶meters.shardings.all(), + .program_name = phase.programName("qwen3_5", "embed_tokens"), + }); + } + + fn compileFullAttentionLayer( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + qwen_model: model.Model, + parameters: CompilationOptions, + seqlen: usize, + layer_index: usize, + phase: Phase, + progress: *std.Progress.Node, + ) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("full attention layer"), 1); + defer node.end(); + + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "full attention layer", io, from); + + const hidden_tensor = ComposedKernelExe.hidden(qwen_model, seqlen); + const self_attn_cache: model.KvCache.SelfAttnCache = .{ + .k = parameters.kv_cache.self_attn.k, + .v = parameters.kv_cache.self_attn.v, + .layer_index = zml.Tensor.init(.{}, .u32), + }; + + return platform.compile( + allocator, + io, + qwen_model.text_model.layers[layer_index], + .forwardSelfAttn, + .{ hidden_tensor, parameters.token_index, self_attn_cache }, + .{ + .shardings = ¶meters.shardings.all(), + .program_name = phase.programName("qwen3_5", "full_attention_layer"), + }, + ); + } + + fn compileLinearAttentionLayer( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + qwen_model: model.Model, + parameters: CompilationOptions, + seqlen: usize, + layer_index: usize, + phase: Phase, + progress: *std.Progress.Node, + ) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("linear attention layer"), 1); + defer node.end(); + + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "linear attention layer", io, from); + + const hidden_tensor = ComposedKernelExe.hidden(qwen_model, seqlen); + const linear_attn_cache: model.KvCache.GatedDeltaNetCache = .{ + .conv_state = parameters.kv_cache.gated_delta_net.conv_state, + .recurrent_state = parameters.kv_cache.gated_delta_net.recurrent_state, + .layer_index = zml.Tensor.init(.{}, .u32), + }; + + return platform.compile( + allocator, + io, + qwen_model.text_model.layers[layer_index], + .forwardLinearAttn, + .{ hidden_tensor, parameters.token_index, linear_attn_cache }, + .{ + .shardings = ¶meters.shardings.all(), + .program_name = phase.programName("qwen3_5", "linear_attention_layer"), + }, + ); + } + + fn compileSampler( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + qwen_model: model.Model, + parameters: CompilationOptions, + seqlen: usize, + phase: Phase, + progress: *std.Progress.Node, + ) !zml.Exe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("sampler"), 1); + defer node.end(); + + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "sampler", io, from); + + const token_index: ?zml.Tensor = switch (phase) { + .prefill => null, + .decode => parameters.token_index, + }; + + return platform.compile( + allocator, + io, + qwen_model.sampler(), + .sampleTokens, + .{ ComposedKernelExe.hidden(qwen_model, seqlen), parameters.rng, token_index }, + .{ + .shardings = ¶meters.shardings.all(), + .program_name = phase.programName("qwen3_5", "sampler"), + }, + ); + } + + fn hidden(qwen_model: model.Model, seqlen: usize) zml.Tensor { + return .fromShape(zml.Shape.init( + .{ .b = 1, .s = seqlen, .d = qwen_model.config.text_config.hidden_size }, + qwen_model.text_model.embed_tokens.weight.dtype(), + ).withPartitioning(.{ + .b = .replicated, + .s = .replicated, + .d = .replicated, + })); + } + + fn findFirstLayerIndex(layer_types: []const model.LayerType, target: model.LayerType) ?usize { + for (layer_types, 0..) |layer_type, index| { + if (layer_type == target) return index; + } + return null; + } +}; diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index f6a5c2c4..624d4ae5 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -49,10 +49,11 @@ fn kvHeadsAreReplicated(axis_size: i64, model_partitions: i64) bool { } fn partitionKvCacheShape(kv_shape: zml.Shape, kv_heads: i64, model_partitions: i64) zml.Shape { - return if (kvHeadsAreReplicated(kv_heads, model_partitions)) - kv_shape.withPartitioning(.{ .h = .replicated }) - else - kv_shape.withPartitioning(.{ .h = .model }); + if (kvHeadsAreReplicated(kv_heads, model_partitions)) { + return kv_shape.withPartitioning(.{ .h = .replicated }); + } + + return kv_shape.withPartitioning(.{ .h = .model }); } pub const LoadedModel = struct { @@ -163,7 +164,10 @@ pub const Model = struct { pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config, gen_options: GenOptions) !Model { // For some Qwen3.5 versions, the output projection lm_head has a standalone weight tensor, while for others it's the same as the input embedding layer - const lm_head_prefix = if (store.hasKey("lm_head.weight")) "lm_head" else "model.language_model.embed_tokens"; + const lm_head_prefix = b: { + if (store.hasKey("lm_head.weight")) break :b "lm_head"; + break :b "model.language_model.embed_tokens"; + }; return .{ .text_model = try .init(allocator, store.withPrefix("model.language_model"), config), .lm_head = .init( @@ -220,22 +224,39 @@ pub const Model = struct { ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { const tokens = tokens_.withPartialTags(.{.s}); const text_model_output, const updated_kv_cache = self.text_model.forward(tokens, token_index, kv_cache); - const new_tokens, const new_rng = self.sampleTokens(text_model_output, rng); + const new_tokens, const new_rng, _ = self.sampler().sampleTokens(text_model_output, rng, null); return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; } - pub fn sampleTokens( - self: Model, - out: zml.Tensor, - rng: zml.Tensor.Rng, - ) struct { zml.Tensor, zml.Tensor.Rng } { - const logits = self.lm_head.forward(out.withPartialTags(.{.d})).rename(.{ .dout = .voc }); - const next_tokens, const new_rng = zml.nn.sampleTokens(logits, self.gen_options.sampling_strategy, rng); - return .{ next_tokens, new_rng }; + pub fn sampler(self: Model) Sampler { + return .{ + .norm = self.text_model.norm, + .lm_head = self.lm_head, + .gen_options = self.gen_options, + }; } }; -//========================Text model======================== +pub const Sampler = struct { + norm: RmsNorm, + lm_head: zml.nn.Linear, + gen_options: Model.GenOptions, + + pub fn sampleTokens( + self: Sampler, + out: zml.Tensor, + rng: zml.Tensor.Rng, + token_index: ?zml.Tensor, + ) struct { zml.Tensor, zml.Tensor.Rng, ?zml.Tensor } { + const x = self.norm.forward(out); + const logits = self.lm_head.forward(x.withPartialTags(.{.d})).rename(.{ .dout = .voc }); + const next_tokens, const new_rng = zml.nn.sampleTokens(logits, self.gen_options.sampling_strategy, rng); + if (token_index) |token_idx| { + return .{ next_tokens.convert(.u32), new_rng, token_idx.addConstant(1) }; + } + return .{ next_tokens.convert(.u32), new_rng, null }; + } +}; pub const TextModel = struct { embed_tokens: zml.nn.TokenEmbedding, @@ -284,10 +305,9 @@ pub const TextModel = struct { var updated_kv_cache = kv_cache; for (self.layers, 0..) |layer, i| { - hidden_states, updated_kv_cache = layer.forward(hidden_states, token_index, updated_kv_cache, @intCast(i)); + hidden_states, updated_kv_cache = layer.forward(hidden_states, token_index, updated_kv_cache.atLayer(i)); } - hidden_states = self.norm.forward(hidden_states); return .{ hidden_states, updated_kv_cache.reuseBuffer(kv_cache) }; } }; @@ -305,12 +325,13 @@ pub const TransformerLayer = struct { pub fn init(store: zml.io.TensorStore.View, config: Config, layer_index: usize) !TransformerLayer { const is_full_attention = config.text_config.layer_types[layer_index] == .full_attention; + const attn: Attn = switch (is_full_attention) { + true => .{ .full_attention = try .init(store.withPrefix("self_attn"), config) }, + false => .{ .linear_attention = .init(store.withPrefix("linear_attn"), config) }, + }; return .{ .input_layernorm = RmsNorm.init(store.withPrefix("input_layernorm"), config.text_config.rms_norm_eps), - .attn = if (is_full_attention) - .{ .full_attention = try .init(store.withPrefix("self_attn"), config) } - else - .{ .linear_attention = .init(store.withPrefix("linear_attn"), config) }, + .attn = attn, .mlp = .init(store.withPrefix("mlp")), .post_attention_layernorm = RmsNorm.init(store.withPrefix("post_attention_layernorm"), config.text_config.rms_norm_eps), }; @@ -330,26 +351,31 @@ pub const TransformerLayer = struct { self: TransformerLayer, x0: zml.Tensor, token_index: zml.Tensor, - kv_cache: KvCache, - layer_id: u32, + kv_cache: KvCache.LayerView, ) struct { zml.Tensor, KvCache } { const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); var attention_output: zml.Tensor = undefined; - var updated_kv_cache: KvCache = kv_cache; - const layer_type, const dense_id = kv_cache.getDenseIndex(layer_id); - std.debug.assert(self.attn == layer_type); + var updated_kv_cache: KvCache = kv_cache.parent; switch (self.attn) { .full_attention => |*self_attn| { - const result = self_attn.forward(normalized_x0, token_index, kv_cache.self_attn.atLayer(dense_id)); + const cache = switch (kv_cache.cache) { + .self_attn => |cache| cache, + .linear_attn => unreachable, + }; + const result = self_attn.forward(normalized_x0, token_index, cache); attention_output = result[0]; - updated_kv_cache.self_attn = result[1]; + updated_kv_cache.self_attn = result[1].reuseBuffer(updated_kv_cache.self_attn); }, .linear_attention => |*linear_attn| { - const result = linear_attn.forward(normalized_x0, kv_cache.gated_delta_net.atLayer(dense_id)); + const cache = switch (kv_cache.cache) { + .linear_attn => |cache| cache, + .self_attn => unreachable, + }; + const result = linear_attn.forward(normalized_x0, cache); attention_output = result[0]; - updated_kv_cache.gated_delta_net = result[1]; + updated_kv_cache.gated_delta_net = result[1].reuseBuffer(updated_kv_cache.gated_delta_net); }, } @@ -360,6 +386,51 @@ pub const TransformerLayer = struct { return .{ mlp_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), updated_kv_cache }; } + + pub fn forwardSelfAttn( + self: TransformerLayer, + x0: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache.SelfAttnCache, + ) struct { zml.Tensor, KvCache.SelfAttnCache } { + const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const normalized_x0 = self.input_layernorm.forward(x0_replicated); + + const self_attn = switch (self.attn) { + .full_attention => |self_attn| self_attn, + .linear_attention => unreachable, + }; + const attention_output, const updated_kv_cache = self_attn.forward(normalized_x0, token_index, kv_cache); + + const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); + const normalized_hidden = self.post_attention_layernorm.forward(x1); + const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.{ .d = .replicated }); + + return .{ mlp_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), updated_kv_cache }; + } + + pub fn forwardLinearAttn( + self: TransformerLayer, + x0: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache.GatedDeltaNetCache, + ) struct { zml.Tensor, KvCache.GatedDeltaNetCache } { + _ = token_index; + const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const normalized_x0 = self.input_layernorm.forward(x0_replicated); + + const linear_attn = switch (self.attn) { + .linear_attention => |linear_attn| linear_attn, + .full_attention => unreachable, + }; + const attention_output, const updated_kv_cache = linear_attn.forward(normalized_x0, kv_cache); + + const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); + const normalized_hidden = self.post_attention_layernorm.forward(x1); + const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.{ .d = .replicated }); + + return .{ mlp_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), updated_kv_cache }; + } }; pub const Mlp = struct { @@ -468,24 +539,30 @@ pub const SelfAttn = struct { } fn projectKV(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { - const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; + const num_kv_heads = b: { + if (self.num_kv_heads > 0) break :b self.num_kv_heads; + break :b self.num_heads; + }; const k = self.k_proj.forward(x).splitAxis(.dout, .{ .h = num_kv_heads, .hd = self.head_dim }); const v = self.v_proj.forward(x).splitAxis(.dout, .{ .h = num_kv_heads, .hd = self.head_dim }); return .{ k, v }; } fn partitionProjectedKv(tensor: zml.Tensor, replicate_kv_heads: bool) zml.Tensor { - return if (replicate_kv_heads) - tensor.withPartitioning(.{ .s = .replicated, .h = .replicated, .hd = .replicated }) - else - tensor.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); + if (replicate_kv_heads) { + return tensor.withPartitioning(.{ .s = .replicated, .h = .replicated, .hd = .replicated }); + } + + return tensor.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); } fn partitionCachedKv(tensor: zml.Tensor, replicate_kv_heads: bool) zml.Tensor { - return if (replicate_kv_heads) - tensor.rename(.{ .s = .k }).withPartitioning(.{ .k = .replicated, .h = .replicated, .hd = .replicated }) - else - tensor.rename(.{ .s = .k }).withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); + const cache_tensor = tensor.rename(.{ .s = .k }); + if (replicate_kv_heads) { + return cache_tensor.withPartitioning(.{ .k = .replicated, .h = .replicated, .hd = .replicated }); + } + + return cache_tensor.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); } pub fn forward( @@ -570,7 +647,12 @@ pub const TextRotaryEmbedding = struct { return .{ cos, sin }; } - pub fn getCosAndSinInterleaved(self: TextRotaryEmbedding, position_ids: zml.Tensor, dtype: zml.DataType) struct { zml.Tensor, zml.Tensor } { // To be used later in image extension + pub fn getCosAndSinInterleaved( + self: TextRotaryEmbedding, + position_ids: zml.Tensor, + dtype: zml.DataType, + ) struct { zml.Tensor, zml.Tensor } { + // To be used later in image extension. const stacked_position_ids = zml.Tensor.stack(&.{ position_ids, position_ids, position_ids }, 0, .g).convert(.f32); const inv_freq = zml.nn.InvFreq(self.rotary_dim, self.rope_opts).withTags(.{.hd}); @@ -671,7 +753,14 @@ pub const GatedDeltaNet = struct { RmsNormGated.unloadBuffers(&self.norm); } - fn recurrentGatedDeltaRule(query: zml.Tensor, key: zml.Tensor, value: zml.Tensor, g: zml.Tensor, beta: zml.Tensor, initial_state: ?zml.Tensor) struct { zml.Tensor, zml.Tensor } { + fn recurrentGatedDeltaRule( + query: zml.Tensor, + key: zml.Tensor, + value: zml.Tensor, + g: zml.Tensor, + beta: zml.Tensor, + initial_state: ?zml.Tensor, + ) struct { zml.Tensor, zml.Tensor } { const scale: f32 = 1.0 / @sqrt(@as(f32, @floatFromInt(query.dim(.khd)))); const query_norm = zml.nn.normalizeL2(query.rename(.{ .kh = .vh }), 1e-6); const key_norm = zml.nn.normalizeL2(key.rename(.{ .kh = .vh }), 1e-6); @@ -684,15 +773,18 @@ pub const GatedDeltaNet = struct { beta.convert(.f32).rename(.{ .vh = .h }), }; - const initial_recurrent_state = if (initial_state) |state| - state.convert(.f32).transpose(.{ .b, .vh, .vhd, .khd }).rename(.{ .vh = .h, .vhd = .v, .khd = .k }) - else - zml.Tensor.constant(zml.DataType.zero(.f32)).broad(zml.Shape.init(.{ + const initial_recurrent_state = b: { + if (initial_state) |state| { + break :b state.convert(.f32).transpose(.{ .b, .vh, .vhd, .khd }).rename(.{ .vh = .h, .vhd = .v, .khd = .k }); + } + + break :b zml.Tensor.constant(zml.DataType.zero(.f32)).broad(zml.Shape.init(.{ .b = value.dim(.b), .h = value.dim(.vh), .v = value.dim(.vhd), .k = query.dim(.khd), }, .f32)); + }; const result = zml.nn.GatedDeltaNet.forward( query_f32, @@ -728,10 +820,10 @@ pub const GatedDeltaNet = struct { const x_in = x.withPartitioning(.{ .d = .replicated }); const projected_qkv = self.in_proj_qkv.forward(x_in).rename(.{ .dout = .mix }).withPartitioning(.{ .s = .replicated, .mix = .model }); const use_cached_state = x.dim(.s) == 1 and left_pad > 0; - const conv_input = if (use_cached_state) - zml.Tensor.concatenate(&.{ cache.convState(), projected_qkv }, .s) - else - projected_qkv; + const conv_input = b: { + if (use_cached_state) break :b zml.Tensor.concatenate(&.{ cache.convState(), projected_qkv }, .s); + break :b projected_qkv; + }; const kernel = self.conv1d_weight; var mixed_qkv = zml.Tensor.conv1d( @@ -758,7 +850,9 @@ pub const GatedDeltaNet = struct { } mixed_qkv = mixed_qkv.withPartitioning(.{ .s = .replicated, .mix = .model }); - const z = self.in_proj_z.forward(x_in).splitAxis(.dout, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }).withPartitioning(.{ .s = .replicated, .vh = .model, .vhd = .replicated }); + const z = self.in_proj_z.forward(x_in) + .splitAxis(.dout, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }) + .withPartitioning(.{ .s = .replicated, .vh = .model, .vhd = .replicated }); const b = self.in_proj_b.forward(x_in).rename(.{ .dout = .vh }).withPartitioning(.{ .s = .replicated, .vh = .model }); const a = self.in_proj_a.forward(x_in).rename(.{ .dout = .vh }).withPartitioning(.{ .s = .replicated, .vh = .model }); @@ -779,8 +873,14 @@ pub const GatedDeltaNet = struct { const aLog_type = self.aLog.dtype(); const g = self.aLog.broad(a.shape()).exp().mul(softplus(a.convert(aLog_type).add(self.dt_bias.convert(aLog_type).broad(a.shape())))).negate(); - const query_for_rule = if (self.qk_head_repetition == 1) query else query.stutter1d(@intCast(query.axis(.kh)), @intCast(self.qk_head_repetition)); - const key_for_rule = if (self.qk_head_repetition == 1) key else key.stutter1d(@intCast(key.axis(.kh)), @intCast(self.qk_head_repetition)); + const query_for_rule = b: { + if (self.qk_head_repetition == 1) break :b query; + break :b query.stutter1d(@intCast(query.axis(.kh)), @intCast(self.qk_head_repetition)); + }; + const key_for_rule = b: { + if (self.qk_head_repetition == 1) break :b key; + break :b key.stutter1d(@intCast(key.axis(.kh)), @intCast(self.qk_head_repetition)); + }; const core_attn_out, const last_recurrent_state = recurrentGatedDeltaRule( query_for_rule, @@ -788,7 +888,10 @@ pub const GatedDeltaNet = struct { value, g, beta, - if (use_cached_state) cache.recurrentState() else null, + b: { + if (use_cached_state) break :b cache.recurrentState(); + break :b null; + }, ); const core_attn_out_normed = self.norm @@ -861,7 +964,7 @@ pub const KvCache = struct { pub const SelfAttnCache = struct { k: zml.Tensor, v: zml.Tensor, - layer_index: ?u32, + layer_index: zml.Tensor, pub const Buffers = zml.Bufferized(SelfAttnCache); @@ -878,7 +981,7 @@ pub const KvCache = struct { return .{ .k = .fromShape(sharded_kv_shape), .v = .fromShape(sharded_kv_shape), - .layer_index = null, + .layer_index = .init(.{}, .u32), }; } @@ -886,44 +989,63 @@ pub const KvCache = struct { return .{ .k = try zml.Buffer.uninitialized(io, platform, kv.k.shape(), sharding, .{}), .v = try zml.Buffer.uninitialized(io, platform, kv.v.shape(), sharding, .{}), + .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), }; } pub fn deinitBuffer(kv: *SelfAttnCache.Buffers) void { kv.k.deinit(); kv.v.deinit(); + kv.layer_index.deinit(); } pub fn keys(kv: SelfAttnCache) zml.Tensor { - return kv.k.slice1d(.layer, .single(kv.layer_index orelse @panic("forgot to call atLayer"))); + return kv.k.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = kv.layer_index, .len = 1 } }).squeeze(.layer); } pub fn values(kv: SelfAttnCache) zml.Tensor { - return kv.v.slice1d(.layer, .single(kv.layer_index orelse @panic("forgot to call atLayer"))); + return kv.v.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = kv.layer_index, .len = 1 } }).squeeze(.layer); } pub fn update(kv: SelfAttnCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: ?zml.Tensor) SelfAttnCache { const k_shape = kv.k.shape().drop(.layer); - const layer: zml.Tensor = .scalar(kv.layer_index orelse @panic("forgot to call atLayer"), .u32); - - return if (token_index) |idx| - .{ - .k = kv.k.scatterSlices(.{ .layer = layer, .k = idx }, new_k.convert(kv.k.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(kv.k), - .v = kv.v.scatterSlices(.{ .layer = layer, .k = idx }, new_v.convert(kv.v.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(kv.v), - .layer_index = kv.layer_index, - } - else - .{ - .k = kv.k.scatterSlices(.{ .layer = layer }, new_k.convert(kv.k.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(kv.k), - .v = kv.v.scatterSlices(.{ .layer = layer }, new_v.convert(kv.v.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(kv.v), + var layer = kv.layer_index; + if (token_index) |idx| { + layer = layer.broad(idx.shape()); + return .{ + .k = kv.k.scatterSlices( + .{ .layer = layer, .s = idx }, + new_k.convert(kv.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(kv.k), + .v = kv.v.scatterSlices( + .{ .layer = layer, .s = idx }, + new_v.convert(kv.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(kv.v), .layer_index = kv.layer_index, }; + } + + return .{ + .k = kv.k.scatterSlices( + .{ .layer = layer }, + new_k.convert(kv.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(kv.k), + .v = kv.v.scatterSlices( + .{ .layer = layer }, + new_v.convert(kv.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(kv.v), + .layer_index = kv.layer_index, + }; } pub fn atLayer(kv: SelfAttnCache, layer_index: usize) SelfAttnCache { return .{ .k = kv.k, .v = kv.v, - .layer_index = @intCast(layer_index), + .layer_index = zml.Tensor.scalar(layer_index, .u32), }; } @@ -931,7 +1053,7 @@ pub const KvCache = struct { return .{ .k = kv.k.reuseBuffer(other.k), .v = kv.v.reuseBuffer(other.v), - .layer_index = kv.layer_index, + .layer_index = kv.layer_index.reuseBuffer(other.layer_index), }; } }; @@ -939,13 +1061,15 @@ pub const KvCache = struct { pub const GatedDeltaNetCache = struct { conv_state: zml.Tensor, recurrent_state: zml.Tensor, - layer_index: ?u32, + layer_index: zml.Tensor, pub const Buffers = zml.Bufferized(GatedDeltaNetCache); pub fn init(config: Config, batch_dim: i64, conv_dtype: zml.DataType, recurrent_dtype: zml.DataType) GatedDeltaNetCache { const num_linear_attn_layers = countLayers(config.text_config.layer_types, .linear_attention); - const conv_dim = 2 * config.text_config.linear_num_key_heads * config.text_config.linear_key_head_dim + config.text_config.linear_num_value_heads * config.text_config.linear_value_head_dim; + const conv_dim = + 2 * config.text_config.linear_num_key_heads * config.text_config.linear_key_head_dim + + config.text_config.linear_num_value_heads * config.text_config.linear_value_head_dim; const conv_state_shape = zml.Shape.init(.{ .b = batch_dim, .layer = num_linear_attn_layers, @@ -964,7 +1088,7 @@ pub const KvCache = struct { return .{ .conv_state = .fromShape(sharded_conv_state_shape), .recurrent_state = .fromShape(sharded_recurrent_state_shape), - .layer_index = null, + .layer_index = .init(.{}, .u32), }; } @@ -972,41 +1096,48 @@ pub const KvCache = struct { return .{ .conv_state = try zml.Buffer.uninitialized(io, platform, self.conv_state.shape(), sharding, .{}), .recurrent_state = try zml.Buffer.uninitialized(io, platform, self.recurrent_state.shape(), sharding, .{}), + .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), }; } pub fn deinitBuffer(self: *GatedDeltaNetCache.Buffers) void { self.conv_state.deinit(); self.recurrent_state.deinit(); + self.layer_index.deinit(); } pub fn convState(self: GatedDeltaNetCache) zml.Tensor { - return self.conv_state.slice1d(.layer, .single(self.layer_index orelse @panic("forgot to call atLayer"))); + return self.conv_state.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); } pub fn recurrentState(self: GatedDeltaNetCache) zml.Tensor { - return self.recurrent_state.slice1d(.layer, .single(self.layer_index orelse @panic("forgot to call atLayer"))); + return self.recurrent_state.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); } pub fn update(self: GatedDeltaNetCache, new_conv_state: ?zml.Tensor, new_recurrent_state: ?zml.Tensor) GatedDeltaNetCache { - const layer: zml.Tensor = .scalar(self.layer_index, .u32); - const conv_state = if (new_conv_state) |state| - self.conv_state.scatterSlices( - .{ .layer = layer }, - state.convert(self.conv_state.dtype()).transpose(self.conv_state.shape().drop(.layer)), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.conv_state) - else - self.conv_state; + const conv_state = b: { + if (new_conv_state) |state| { + break :b self.conv_state.scatterSlices( + .{ .layer = self.layer_index }, + state.convert(self.conv_state.dtype()).transpose(self.conv_state.shape().drop(.layer)), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.conv_state); + } - const recurrent_state = if (new_recurrent_state) |state| - self.recurrent_state.scatterSlices( - .{ .layer = layer }, - state.convert(self.recurrent_state.dtype()).transpose(self.recurrent_state.shape().drop(.layer)), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.recurrent_state) - else - self.recurrent_state; + break :b self.conv_state; + }; + + const recurrent_state = b: { + if (new_recurrent_state) |state| { + break :b self.recurrent_state.scatterSlices( + .{ .layer = self.layer_index }, + state.convert(self.recurrent_state.dtype()).transpose(self.recurrent_state.shape().drop(.layer)), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.recurrent_state); + } + + break :b self.recurrent_state; + }; return .{ .conv_state = conv_state, @@ -1019,7 +1150,7 @@ pub const KvCache = struct { return .{ .conv_state = self.conv_state, .recurrent_state = self.recurrent_state, - .layer_index = @intCast(layer_index), + .layer_index = zml.Tensor.scalar(layer_index, .u32), }; } @@ -1027,7 +1158,7 @@ pub const KvCache = struct { return .{ .conv_state = self.conv_state.reuseBuffer(other.conv_state), .recurrent_state = self.recurrent_state.reuseBuffer(other.recurrent_state), - .layer_index = null, + .layer_index = self.layer_index.reuseBuffer(other.layer_index), }; } }; @@ -1067,28 +1198,50 @@ pub const KvCache = struct { }; } + pub const LayerView = struct { + parent: KvCache, + cache: union(enum) { + self_attn: SelfAttnCache, + linear_attn: GatedDeltaNetCache, + }, + }; + + pub fn atLayer(self: KvCache, layer_index: usize) LayerView { + return switch (denseIndex(self.layer_types, layer_index)) { + .full_attention => |dense_index| .{ + .parent = self, + .cache = .{ .self_attn = self.self_attn.atLayer(dense_index.layer_dense_index) }, + }, + .linear_attention => |dense_index| .{ + .parent = self, + .cache = .{ .linear_attn = self.gated_delta_net.atLayer(dense_index.layer_dense_index) }, + }, + }; + } + fn countLayers(layer_types: []const LayerType, layer_type: LayerType) i64 { return @intCast(std.mem.countScalar(LayerType, layer_types, layer_type)); } - pub fn getDenseIndex(cache: KvCache, layer_index: usize) struct { LayerType, u32 } { - var self_attn_layer_index: u32 = 0; - var linear_attn_layer_index: u32 = 0; - for (cache.layer_types[0..layer_index]) |layer_type| { + fn denseIndex(layer_types: []const LayerType, layer_index: usize) union(enum) { + full_attention: struct { layer_dense_index: usize }, + linear_attention: struct { layer_dense_index: usize }, + } { + var self_attn_layer_index: usize = 0; + var linear_attn_layer_index: usize = 0; + for (layer_types[0..layer_index]) |layer_type| { switch (layer_type) { .full_attention => self_attn_layer_index += 1, .linear_attention => linear_attn_layer_index += 1, } } - return switch (cache.layer_types[layer_index]) { - .full_attention => .{ .full_attention, self_attn_layer_index }, - .linear_attention => .{ .linear_attention, linear_attn_layer_index }, + return switch (layer_types[layer_index]) { + .full_attention => .{ .full_attention = .{ .layer_dense_index = self_attn_layer_index } }, + .linear_attention => .{ .linear_attention = .{ .layer_dense_index = linear_attn_layer_index } }, }; } }; -//========================Utils======================== - fn softplus(x: zml.Tensor) zml.Tensor { return x.exp().addConstant(1).log(); } diff --git a/examples/llm/models/qwen3_5/session.zig b/examples/llm/models/qwen3_5/session.zig index 56a87373..d593e319 100644 --- a/examples/llm/models/qwen3_5/session.zig +++ b/examples/llm/models/qwen3_5/session.zig @@ -11,10 +11,10 @@ pub const Session = struct { platform: *const zml.Platform, model_buffers: *model.Buffers, compiled_model: *const inference.CompiledModel, + decode_runner: inference.KernelExe.Runner, kv_cache_buffers: zml.Bufferized(model.KvCache), rng_buffers: zml.Bufferized(zml.Tensor.Rng), tokenizer: zml.tokenizer.Tokenizer, - step_token_slice: zml.Slice, generated_token_slice: zml.Slice, seqlen: u32, eos_token_id: u32, @@ -37,16 +37,24 @@ pub const Session = struct { var rng_buffers = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); + var decode_runner = try compiled_model.decode.initRunner( + allocator, + io, + platform, + model_buffers, + ); + errdefer decode_runner.deinit(allocator); + return .{ .allocator = allocator, .io = io, .platform = platform, .model_buffers = model_buffers, .compiled_model = compiled_model, + .decode_runner = decode_runner, .kv_cache_buffers = kv_cache_buffers, .rng_buffers = rng_buffers, .tokenizer = tokenizer, - .step_token_slice = try .alloc(allocator, zml.Shape.init(.{ .b = 1, .s = 1 }, .u32)), .generated_token_slice = try .alloc(allocator, zml.Shape.init(.{ .b = 1, .s = 1 }, .u32)), .seqlen = compiled_model.params.seqlen, .eos_token_id = compiled_model.loaded_model.inner.special_tokens.end_of_text_token_id, @@ -57,9 +65,9 @@ pub const Session = struct { } pub fn deinit(self: *Session) void { + self.decode_runner.deinit(self.allocator); model.KvCache.deinitBuffer(&self.kv_cache_buffers); zml.Tensor.Rng.deinitBuffer(&self.rng_buffers); - self.step_token_slice.free(self.allocator); self.generated_token_slice.free(self.allocator); } @@ -78,30 +86,26 @@ pub const Session = struct { @memset(prefill_tokens_slice.items(u32), 0); @memcpy(prefill_tokens_slice.items(u32)[0..all_tokens.len], all_tokens); - var prefill_tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, prefill_tokens_slice, .replicated); + const replicated_sharding: zml.Sharding = .replicated; + + var prefill_tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, prefill_tokens_slice, replicated_sharding); defer prefill_tokens_buffer.deinit(); - var prefill_token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, 0, .u32); + var prefill_token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 0), .u32); defer prefill_token_index_buffer.deinit(); - var args = try self.compiled_model.prefill_exe.args(self.allocator); - defer args.deinit(self.allocator); - - var results = try self.compiled_model.prefill_exe.results(self.allocator); - defer results.deinit(self.allocator); - - args.set(.{ - self.model_buffers, - prefill_tokens_buffer, - prefill_token_index_buffer, - &self.kv_cache_buffers, - &self.rng_buffers, + try self.compiled_model.prefill.run(.{ + .allocator = self.allocator, + .io = self.io, + .platform = self.platform, + .model_buffers = self.model_buffers, + .tokens_buf = &prefill_tokens_buffer, + .token_index_buf = &prefill_token_index_buffer, + .kv_cache_buffers = &self.kv_cache_buffers, + .rng_buffers = &self.rng_buffers, }); - self.compiled_model.prefill_exe.call(args, &results); - results.fill(.{ &prefill_tokens_buffer, &self.kv_cache_buffers, &self.rng_buffers }); try prefill_tokens_buffer.toSlice(self.io, prefill_tokens_slice); - const generated_token = prefill_tokens_slice.items(u32)[all_tokens.len - 1]; self.generated_token_slice.items(u32)[0] = generated_token; } @@ -112,6 +116,14 @@ pub const Session = struct { const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); defer self.allocator.free(out_tokens_buffer); + const replicated_sharding: zml.Sharding = .replicated; + + var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice, replicated_sharding); + defer current_token_buffer.deinit(); + + var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.items.len)), .u32); + defer token_index_buffer.deinit(); + generation: while (true) { const token_id = self.generated_token_slice.items(u32)[0]; if (token_id == self.eos_token_id) break :generation; @@ -129,43 +141,32 @@ pub const Session = struct { try all_tokens.append(self.allocator, token_id); if (all_tokens.items.len >= self.seqlen) break :generation; - try self.runDecodeStep(token_id, @intCast(all_tokens.items.len)); + try self.decode_runner.run(.{ + .allocator = self.allocator, + .io = self.io, + .platform = self.platform, + .model_buffers = self.model_buffers, + .tokens_buf = ¤t_token_buffer, + .token_index_buf = &token_index_buffer, + .kv_cache_buffers = &self.kv_cache_buffers, + .rng_buffers = &self.rng_buffers, + }); + + try current_token_buffer.toSlice(self.io, self.generated_token_slice); } try stdout.writeAll(try decoder.finalize(out_tokens_buffer)); try stdout.flush(); } - - fn runDecodeStep(self: *Session, token_id: u32, token_index: u32) !void { - self.step_token_slice.items(u32)[0] = token_id; - - var token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.step_token_slice, .replicated); - defer token_buffer.deinit(); - - var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, token_index, .u32); - defer token_index_buffer.deinit(); - - var args = try self.compiled_model.decode_exe.args(self.allocator); - defer args.deinit(self.allocator); - - var results = try self.compiled_model.decode_exe.results(self.allocator); - defer results.deinit(self.allocator); - - args.set(.{ - self.model_buffers, - token_buffer, - token_index_buffer, - &self.kv_cache_buffers, - &self.rng_buffers, - }); - self.compiled_model.decode_exe.call(args, &results); - - results.fill(.{ &token_buffer, &self.kv_cache_buffers, &self.rng_buffers }); - try token_buffer.toSlice(self.io, self.generated_token_slice); - } }; -fn tokenizeChatPrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8, special_tokens: model.Model.SpecialTokens, is_first_turn: bool) ![]const u32 { +fn tokenizeChatPrompt( + allocator: std.mem.Allocator, + tokenizer: zml.tokenizer.Tokenizer, + prompt: []const u8, + special_tokens: model.Model.SpecialTokens, + is_first_turn: bool, +) ![]const u32 { var encoder = try tokenizer.encoder(); defer encoder.deinit(); diff --git a/zml/safetensors.zig b/zml/safetensors.zig index bbf3508a..377e8124 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -320,7 +320,7 @@ pub const TensorRegistry = struct { if (gop.found_existing) { gop.value_ptr.*.deinit(allocator); gop.value_ptr.* = value; - log.info("Overwrote existing metadata key={s} with value={f}", .{ key, value }); + log.debug("Overwrote existing metadata key={s} with value={f}", .{ key, value }); } else { gop.value_ptr.* = value; log.debug("Added new metadata key={s} with value={f}", .{ key, value }); From cd941143dcfe35a007a597d9cdd9b1ae7325765d Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 28 May 2026 20:47:02 +0200 Subject: [PATCH 504/881] examples/llm: remove useless sharding constraints (#560) Perf iso to https://github.com/zml/zml/pull/555 --- examples/llm/models/lfm2/model.zig | 18 +++++------- examples/llm/models/llama/model.zig | 7 +---- examples/llm/models/qwen3_5/model.zig | 40 +++++++++++++-------------- 3 files changed, 28 insertions(+), 37 deletions(-) diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index 988f62e5..8e4c0956 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -499,9 +499,6 @@ pub const Attention = struct { var q = self.q_proj.forward(x_qkv).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); var k = self.k_proj.forward(x_qkv).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); var v = self.v_proj.forward(x_qkv).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); - q = q.withPartitioning(.{ .seq = .replicated, .h = .model, .hd = .replicated }); - k = k.withPartitioning(.{ .seq = .replicated, .h = .model, .hd = .replicated }); - v = v.withPartitioning(.{ .seq = .replicated, .h = .model, .hd = .replicated }); q = self.q_layernorm.forward(q); k = self.k_layernorm.forward(k); @@ -514,13 +511,13 @@ pub const Attention = struct { q = zml.nn.rope(q, token_positions, self.rope_opts); k = zml.nn.rope(k, token_positions, self.rope_opts); - q = q.rename(.{ .seq = .q }).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }); - k = k.rename(.{ .seq = .k }).withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); - v = v.rename(.{ .seq = .k }).withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); + q = q.rename(.{ .seq = .q }); + k = k.rename(.{ .seq = .k }); + v = v.rename(.{ .seq = .k }); const new_kv_cache = kv_cache.update(k, v, tokens_position_offset, cache_index); - k = new_kv_cache.keys(cache_index).withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); - v = new_kv_cache.values(cache_index).withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); + k = new_kv_cache.keys(cache_index); + v = new_kv_cache.values(cache_index); stdx.debug.assert(q.dim(.batch) == 1, "LFM attention currently expects batch size 1 for flash attention backend, got {}", .{q.dim(.batch)}); const attn = switch (attention_parameters) { @@ -531,7 +528,7 @@ pub const Attention = struct { tokens_position_offset, attention_metadata, attention_parameters, - ).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }).merge(.{ .d = .{ .h, .hd } }), + ).merge(.{ .d = .{ .h, .hd } }), else => zml.attention.attention.attention( q.squeeze(.batch), k.squeeze(.batch), @@ -539,8 +536,7 @@ pub const Attention = struct { tokens_position_offset.squeeze(.batch), attention_metadata, attention_parameters, - ).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }) - .merge(.{ .d = .{ .h, .hd } }) + ).merge(.{ .d = .{ .h, .hd } }) .rename(.{ .q = .seq }) .insertAxes(.seq, .{.batch}), }; diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index b808a605..304cb26f 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -512,9 +512,6 @@ const SelfAttn = struct { var q = self.q_proj.forward(x_qkv).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); var k = self.k_proj.forward(x_qkv).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); var v = self.v_proj.forward(x_qkv).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); - q = q.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); - k = k.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); - v = v.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); // In self-attention, .s axis is used both for keys and queries. const pos_index = b: { @@ -534,8 +531,6 @@ const SelfAttn = struct { const new_kv_cache = kv_cache.updateAt(k, v, token_index, kv_cache_index); k = new_kv_cache.keysAt(kv_cache_index).convert(dtype); v = new_kv_cache.valuesAt(kv_cache_index).convert(dtype); - k = k.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); - v = v.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); const layer_attention_metadata: zml.attention.attention.Metadata = switch (attention_parameters) { .attnd => .{ .attnd = .{ @@ -555,7 +550,7 @@ const SelfAttn = struct { token_index, layer_attention_metadata, attention_parameters, - ).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }); + ); const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); const delta = self.o_proj.forward(attn) diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 624d4ae5..f5b996a0 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -577,8 +577,6 @@ pub const SelfAttn = struct { var q, var gate = self.projectQAndGate(x_qkv); var k, var v = self.projectKV(x_qkv); - q = q.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); - gate = gate.withPartitioning(.{ .s = .replicated, .d_out_proj = .model }); k = partitionProjectedKv(k, replicate_kv_heads); v = partitionProjectedKv(v, replicate_kv_heads); q = self.q_norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); @@ -592,14 +590,13 @@ pub const SelfAttn = struct { const cos, const sin = self.rotary_embed.getCosAndSin(position_ids, dtype); q = self.rotary_embed.applyRope(q, cos, sin); k = self.rotary_embed.applyRope(k, cos, sin); - q = q.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); k = partitionProjectedKv(k, replicate_kv_heads); v = partitionProjectedKv(v, replicate_kv_heads); const new_kv_cache = kv_cache.update(k, v, token_index.convert(.u32)); k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); - q = q.rename(.{ .s = .q }).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }); + q = q.rename(.{ .s = .q }); k = partitionCachedKv(k, replicate_kv_heads); v = partitionCachedKv(v, replicate_kv_heads); @@ -610,10 +607,13 @@ pub const SelfAttn = struct { token_index, zml.attention.attention.Metadata.init(.fromBackend(.vanilla, x.dim(.s), self.num_heads)), zml.attention.attention.Parameters.init(.fromBackend(.vanilla)), - ).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); + ).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); const gated_output = attn_output.mul(gate.sigmoid()); - const projected_output = self.o_proj.forward(gated_output.rename(.{ .d_out_proj = .d })).rename(.{ .dout = .d }).withPartitioning(.{ .d = .replicated }); + const projected_output = self.o_proj + .forward(gated_output.rename(.{ .d_out_proj = .d })) + .rename(.{ .dout = .d }) + .withPartitioning(.{ .d = .replicated }); return .{ projected_output, new_kv_cache }; } @@ -818,7 +818,9 @@ pub const GatedDeltaNet = struct { const left_pad = self.conv_kernel_size - 1; const x_in = x.withPartitioning(.{ .d = .replicated }); - const projected_qkv = self.in_proj_qkv.forward(x_in).rename(.{ .dout = .mix }).withPartitioning(.{ .s = .replicated, .mix = .model }); + const projected_qkv = self.in_proj_qkv.forward(x_in) + .rename(.{ .dout = .mix }) + .withPartitioning(.{ .s = .replicated, .mix = .model }); const use_cached_state = x.dim(.s) == 1 and left_pad > 0; const conv_input = b: { if (use_cached_state) break :b zml.Tensor.concatenate(&.{ cache.convState(), projected_qkv }, .s); @@ -851,23 +853,19 @@ pub const GatedDeltaNet = struct { mixed_qkv = mixed_qkv.withPartitioning(.{ .s = .replicated, .mix = .model }); const z = self.in_proj_z.forward(x_in) - .splitAxis(.dout, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }) - .withPartitioning(.{ .s = .replicated, .vh = .model, .vhd = .replicated }); - const b = self.in_proj_b.forward(x_in).rename(.{ .dout = .vh }).withPartitioning(.{ .s = .replicated, .vh = .model }); - const a = self.in_proj_a.forward(x_in).rename(.{ .dout = .vh }).withPartitioning(.{ .s = .replicated, .vh = .model }); + .splitAxis(.dout, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }); + const b = self.in_proj_b.forward(x_in).rename(.{ .dout = .vh }); + const a = self.in_proj_a.forward(x_in).rename(.{ .dout = .vh }); const query = mixed_qkv .slice1d(.mix, .{ .start = 0, .end = key_dim }) - .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }) - .withPartitioning(.{ .s = .replicated, .kh = .model, .khd = .replicated }); + .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }); const key = mixed_qkv .slice1d(.mix, .{ .start = key_dim, .end = 2 * key_dim }) - .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }) - .withPartitioning(.{ .s = .replicated, .kh = .model, .khd = .replicated }); + .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }); const value = mixed_qkv .slice1d(.mix, .{ .start = 2 * key_dim, .end = 2 * key_dim + value_dim }) - .splitAxis(.mix, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }) - .withPartitioning(.{ .s = .replicated, .vh = .model, .vhd = .replicated }); + .splitAxis(.mix, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }); const beta = b.sigmoid(); const aLog_type = self.aLog.dtype(); @@ -899,10 +897,12 @@ pub const GatedDeltaNet = struct { core_attn_out.rename(.{ .vhd = .d }), z.rename(.{ .vhd = .d }), ) - .rename(.{ .d = .vhd }) - .withPartitioning(.{ .s = .replicated, .vh = .model, .vhd = .replicated }); + .rename(.{ .d = .vhd }); - const output = self.out_proj.forward(core_attn_out_normed.merge(.{ .d = .{ .vh, .vhd } })).rename(.{ .dout = .d }).withPartitioning(.{ .d = .replicated }); + const output = self.out_proj + .forward(core_attn_out_normed.merge(.{ .d = .{ .vh, .vhd } })) + .rename(.{ .dout = .d }) + .withPartitioning(.{ .d = .replicated }); const updated_cache = cache.update( buildUpdatedConvState(conv_input, left_pad), last_recurrent_state, From 2530747949ec842bacb65aabff7aef6405620c22 Mon Sep 17 00:00:00 2001 From: loupicaaa Date: Fri, 29 May 2026 17:42:35 +0200 Subject: [PATCH 505/881] zml/attention: Fix the triton attention kernel by always masking the padding tokens (#563) Fix the triton attention kernel by avoiding the case where `TILE_SIZE == BLOCK_SIZE`. Now we enforce masking this way : `const tile_mask: Value = seq_offset.lt(max_seq_prefix_len);`. Co-authored-by: loupicaaa --- zml/attention/triton_kernels/unified_attention.zig | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/zml/attention/triton_kernels/unified_attention.zig b/zml/attention/triton_kernels/unified_attention.zig index 38c4f961..1353e1d9 100644 --- a/zml/attention/triton_kernels/unified_attention.zig +++ b/zml/attention/triton_kernels/unified_attention.zig @@ -582,10 +582,7 @@ fn kernelUnifiedAttention2d( const seq_offset = j.mul(@as(i32, @intCast(TILE_SIZE))).add(offs_t); - const tile_mask: Value = if (TILE_SIZE == BLOCK_SIZE) - k.full(&.{TILE_SIZE}, 1, .i1) - else - seq_offset.lt(max_seq_prefix_len); + const tile_mask: Value = seq_offset.lt(max_seq_prefix_len); const physical_block_idx = k.load( block_tables_ptr_shifted.addPtr(seq_offset.div(@as(i32, @intCast(BLOCK_SIZE)))), @@ -878,10 +875,7 @@ fn kernelUnifiedAttention3d( const acc = loop.carried[2]; const seq_offset = j.mul(@as(i32, @intCast(TILE_SIZE))).add(offs_t); - const tile_mask: Value = if (TILE_SIZE == BLOCK_SIZE) - k.full(&.{TILE_SIZE}, 1, .i1) - else - seq_offset.lt(max_seq_prefix_len); + const tile_mask: Value = seq_offset.lt(max_seq_prefix_len); const physical_block_idx = k.load( block_tables_ptr.addPtr(block_table_offset).addPtr(seq_offset.div(@as(i32, @intCast(BLOCK_SIZE)))), From 02c1a90c08d7d60d0f6e108cc6e9d954075a2479 Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Fri, 29 May 2026 18:14:38 +0200 Subject: [PATCH 506/881] bin/smi: add oneapi smi monitoring (#557) Add a oneAPI backend for `zml-smi` that detects Intel DRM render devices and reports GPU telemetry from sysfs, hwmon, and fdinfo. The new backend collects utilization, memory usage, power, temperature, clocks, PCIe link data, and per-process usage for Intel GPUs. It is wired into device detection, Bazel builds, Prometheus metrics, and the TUI, including Intel GPU imagery and GPU detail/process table support. Also clean up related TUI handling for GPU clock/PCIe fields and process table lifetime management. --- bin/zml-smi/BUILD.bazel | 2 + bin/zml-smi/info/device_info.zig | 2 + bin/zml-smi/main.zig | 30 +- bin/zml-smi/platforms/oneapi/BUILD.bazel | 24 + bin/zml-smi/platforms/oneapi/metrics.zig | 134 +++ bin/zml-smi/platforms/oneapi/monitor.zig | 849 ++++++++++++++++++++ bin/zml-smi/platforms/oneapi/oneapi.zig | 248 ++++++ bin/zml-smi/platforms/oneapi/process.zig | 45 ++ bin/zml-smi/prometheus/exposition.zig | 2 +- bin/zml-smi/tui/BUILD.bazel | 1 + bin/zml-smi/tui/assets/intel.png | Bin 0 -> 2124 bytes bin/zml-smi/tui/data.zig | 2 +- bin/zml-smi/tui/image_cache.zig | 1 + bin/zml-smi/tui/pages/detail.zig | 2 +- bin/zml-smi/tui/pages/detail/gpu_detail.zig | 22 +- bin/zml-smi/tui/top.zig | 1 + bin/zml-smi/tui/widgets/device_card.zig | 7 +- bin/zml-smi/tui/widgets/process_table.zig | 4 + 18 files changed, 1359 insertions(+), 17 deletions(-) create mode 100644 bin/zml-smi/platforms/oneapi/BUILD.bazel create mode 100644 bin/zml-smi/platforms/oneapi/metrics.zig create mode 100644 bin/zml-smi/platforms/oneapi/monitor.zig create mode 100644 bin/zml-smi/platforms/oneapi/oneapi.zig create mode 100644 bin/zml-smi/platforms/oneapi/process.zig create mode 100644 bin/zml-smi/tui/assets/intel.png diff --git a/bin/zml-smi/BUILD.bazel b/bin/zml-smi/BUILD.bazel index a1757150..0c21893a 100644 --- a/bin/zml-smi/BUILD.bazel +++ b/bin/zml-smi/BUILD.bazel @@ -87,6 +87,7 @@ zig_binary( "//bin/zml-smi/json", "//bin/zml-smi/api", "//bin/zml-smi/prometheus", + "//bin/zml-smi/utils:sysfs", "//bin/zml-smi/info", "//bin/zml-smi/tui", "//stdx", @@ -98,6 +99,7 @@ zig_binary( }) + select({ "@llvm//platforms/config:linux_x86_64": [ "//bin/zml-smi/platforms/amdsmi", + "//bin/zml-smi/platforms/oneapi", "//bin/zml-smi/platforms/neuron", "//bin/zml-smi/platforms/nvml", "//bin/zml-smi/platforms/tpu", diff --git a/bin/zml-smi/info/device_info.zig b/bin/zml-smi/info/device_info.zig index faab1c11..4b01c728 100644 --- a/bin/zml-smi/info/device_info.zig +++ b/bin/zml-smi/info/device_info.zig @@ -4,6 +4,7 @@ const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; pub const Target = enum { cuda, rocm, + oneapi, neuron, tpu, }; @@ -11,6 +12,7 @@ pub const Target = enum { pub const DeviceInfo = union(Target) { cuda: DoubleBuffer(GpuInfo), rocm: DoubleBuffer(GpuInfo), + oneapi: DoubleBuffer(GpuInfo), neuron: DoubleBuffer(NeuronInfo), tpu: DoubleBuffer(TpuInfo), diff --git a/bin/zml-smi/main.zig b/bin/zml-smi/main.zig index e7ffcf67..47c861ff 100644 --- a/bin/zml-smi/main.zig +++ b/bin/zml-smi/main.zig @@ -23,6 +23,7 @@ const csv = @import("zml-smi/csv"); const json = @import("zml-smi/json"); const api = @import("zml-smi/api"); const prometheus = @import("zml-smi/prometheus"); +const sysfs = @import("zml-smi/sysfs"); const smi_tui = @import("zml-smi/tui"); const tui = smi_tui.top; const static_print = smi_tui.print; @@ -64,6 +65,7 @@ const device_backends = switch (builtin.os.tag) { .x86_64 => .{ .{ .cuda, @import("zml-smi/platforms/nvml") }, .{ .rocm, @import("zml-smi/platforms/amdsmi") }, + .{ .oneapi, @import("zml-smi/platforms/oneapi") }, .{ .neuron, @import("zml-smi/platforms/neuron") }, .{ .tpu, @import("zml-smi/platforms/tpu") }, }, @@ -81,7 +83,7 @@ pub fn main(init: std.process.Init) !void { const io = init.io; const args = stdx.flags.parse(init.minimal.args, CliArgs); - const targets = detect(io); + const targets = detect(gpa, io); var collector: Collector = .init(arena, gpa, io, .{ .poll_interval_ms = args.poll_interval, @@ -167,7 +169,7 @@ pub fn main(init: std.process.Init) !void { } } -fn detect(io: std.Io) smi_info.Targets { +fn detect(allocator: std.mem.Allocator, io: std.Io) smi_info.Targets { if (comptime builtin.os.tag == .macos) { return .{}; } @@ -187,6 +189,10 @@ fn detect(io: std.Io) smi_info.Targets { targets.insert(.rocm); } + if (hasDrmRenderDevice(allocator, io, "0x8086")) { + targets.insert(.oneapi); + } + if (hasDevice(io, c.NEURON_DEVICE_PREFIX ++ "0")) { targets.insert(.neuron); } @@ -213,3 +219,23 @@ fn hasDevice(io: std.Io, path: []const u8) bool { std.Io.Dir.accessAbsolute(io, path, .{ .read = true }) catch return false; return true; } + +fn hasDrmRenderDevice(allocator: std.mem.Allocator, io: std.Io, comptime pci_vendor_id: []const u8) bool { + var dir = std.Io.Dir.openDirAbsolute(io, "/dev/dri", .{ .iterate = true }) catch return false; + defer dir.close(io); + + var it = dir.iterate(); + while (it.next(io) catch null) |entry| { + if (!std.mem.startsWith(u8, entry.name, "renderD")) continue; + + var vendor_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const vendor_path = std.fmt.bufPrint(&vendor_path_buf, "/sys/class/drm/{s}/device/vendor", .{entry.name}) catch continue; + const vendor = sysfs.readString(allocator, io, vendor_path) catch continue; + defer allocator.free(vendor); + if (std.mem.eql(u8, std.mem.trim(u8, vendor, &std.ascii.whitespace), pci_vendor_id)) { + return true; + } + } + + return false; +} diff --git a/bin/zml-smi/platforms/oneapi/BUILD.bazel b/bin/zml-smi/platforms/oneapi/BUILD.bazel new file mode 100644 index 00000000..c0b27d84 --- /dev/null +++ b/bin/zml-smi/platforms/oneapi/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "oneapi", + import_name = "zml-smi/platforms/oneapi", + main = "metrics.zig", + srcs = [ + "monitor.zig", + "oneapi.zig", + "process.zig", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + "//bin/zml-smi/utils:sysfs", + ], +) + +zig_test( + name = "test", + deps = [":oneapi"], +) diff --git a/bin/zml-smi/platforms/oneapi/metrics.zig b/bin/zml-smi/platforms/oneapi/metrics.zig new file mode 100644 index 00000000..ae2a86a3 --- /dev/null +++ b/bin/zml-smi/platforms/oneapi/metrics.zig @@ -0,0 +1,134 @@ +const std = @import("std"); +const OneApi = @import("oneapi.zig"); +const device_info = @import("zml-smi/info").device_info; +const GpuInfo = device_info.GpuInfo; +const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; +const Collector = @import("zml-smi/collector").Collector; +const poll_metrics = @import("zml-smi/info").poll_metrics; +const process = @import("process.zig"); + +pub fn start(collector: *Collector) !void { + const oneapi = try collector.arena.create(OneApi); + oneapi.* = try OneApi.init(collector.arena, collector.io); + + const count = oneapi.monitor.devices.len; + const dev_offset: u16 = @intCast(collector.device_infos.items.len); + + for (0..count) |i| { + const poll_arena = try collector.createPollArena(); + const dev = Device.open(oneapi, i, poll_arena) catch continue; + const initial: GpuInfo = .{ + .name = dev.name(collector.arena) catch null, + .driver_version = dev.driverVersion(collector.arena) catch null, + .power_limit_mw = dev.powerLimit() catch null, + .clock_graphics_max_mhz = dev.maxClockGraphics() catch null, + .mem_total_bytes = dev.memTotal() catch null, + .pcie_link_gen = dev.pcieLinkGen() catch null, + .pcie_link_width = dev.pcieLinkWidth() catch null, + .pcie_bandwidth_mbps = dev.pcieBandwidth() catch null, + }; + const info = try collector.addDevice(.{ .oneapi = .{ .values = .{ initial, initial } } }); + + try collector.spawnPoll(pollOnce, .{ poll_arena, &info.oneapi, dev }); + } + + const processes = try collector.createProcessList(); + try process.init(collector, processes, oneapi, dev_offset); +} + +const pollOnce = poll_metrics.poll(*DoubleBuffer(GpuInfo), Device, metrics); + +const Device = struct { + oneapi: *OneApi, + arena: *std.heap.ArenaAllocator, + handle: OneApi.Handle, + + fn open(oneapi: *OneApi, index: usize, arena: *std.heap.ArenaAllocator) !Device { + return .{ + .oneapi = oneapi, + .arena = arena, + .handle = try oneapi.handleByIndex(index), + }; + } + + fn name(self: Device, arena: std.mem.Allocator) ![]const u8 { + return self.oneapi.name(arena, self.handle); + } + + fn driverVersion(self: Device, arena: std.mem.Allocator) ![]const u8 { + return self.oneapi.driverVersion(arena); + } + + pub fn powerUsage(self: Device) !u64 { + return self.oneapi.powerUsage(self.arena.allocator(), self.handle); + } + + pub fn temperature(self: Device) !u64 { + return self.oneapi.temperature(self.arena.allocator(), self.handle); + } + + pub fn fanSpeed(self: Device) !u64 { + return self.oneapi.fanSpeed(self.arena.allocator(), self.handle); + } + + pub fn gpuUtil(self: Device) !u64 { + return self.oneapi.gpuUtil(self.arena.allocator(), self.handle); + } + + pub fn encoderUtil(self: Device) !u64 { + return self.oneapi.encoderUtil(self.arena.allocator(), self.handle); + } + + pub fn decoderUtil(self: Device) !u64 { + return self.oneapi.decoderUtil(self.arena.allocator(), self.handle); + } + + pub fn powerLimit(self: Device) !u64 { + return self.oneapi.powerLimit(self.arena.allocator(), self.handle); + } + + pub fn clockGraphics(self: Device) !u64 { + return self.oneapi.clockGraphics(self.arena.allocator(), self.handle); + } + + pub fn maxClockGraphics(self: Device) !u64 { + return self.oneapi.maxClockGraphics(self.arena.allocator(), self.handle); + } + + pub fn memUsed(self: Device) !u64 { + return self.oneapi.memUsed(self.arena.allocator(), self.handle); + } + + pub fn memTotal(self: Device) !u64 { + return self.oneapi.memTotal(self.arena.allocator(), self.handle); + } + + pub fn pcieLinkGen(self: Device) !u64 { + return self.oneapi.pcieLinkGen(self.arena.allocator(), self.handle); + } + + pub fn pcieLinkWidth(self: Device) !u64 { + return self.oneapi.pcieLinkWidth(self.arena.allocator(), self.handle); + } + + pub fn pcieBandwidth(self: Device) !u64 { + return self.oneapi.pcieBandwidth(self.arena.allocator(), self.handle); + } +}; + +const metrics = .{ + .{ .field = "power_mw", .query = Device.powerUsage }, + .{ .field = "power_limit_mw", .query = Device.powerLimit }, + .{ .field = "temperature", .query = Device.temperature }, + .{ .field = "fan_speed_percent", .query = Device.fanSpeed }, + .{ .field = "util_percent", .query = Device.gpuUtil }, + .{ .field = "encoder_util_percent", .query = Device.encoderUtil }, + .{ .field = "decoder_util_percent", .query = Device.decoderUtil }, + .{ .field = "clock_graphics_mhz", .query = Device.clockGraphics }, + .{ .field = "clock_graphics_max_mhz", .query = Device.maxClockGraphics }, + .{ .field = "mem_used_bytes", .query = Device.memUsed }, + .{ .field = "mem_total_bytes", .query = Device.memTotal }, + .{ .field = "pcie_bandwidth_mbps", .query = Device.pcieBandwidth }, + .{ .field = "pcie_link_gen", .query = Device.pcieLinkGen }, + .{ .field = "pcie_link_width", .query = Device.pcieLinkWidth }, +}; diff --git a/bin/zml-smi/platforms/oneapi/monitor.zig b/bin/zml-smi/platforms/oneapi/monitor.zig new file mode 100644 index 00000000..80a7a336 --- /dev/null +++ b/bin/zml-smi/platforms/oneapi/monitor.zig @@ -0,0 +1,849 @@ +const std = @import("std"); +const smi_sysfs = @import("zml-smi/sysfs"); +const Monitor = @This(); + +pub const bus_device_identifier_len = "0000:00:00.0".len; +pub const Handle = enum(u32) { _ }; + +const Device = struct { + render_idx: usize, + drm_path: []const u8, + dev_path: []const u8, + pcie_path: []const u8, + hwmon_path: ?[]const u8, + bus_device_identifier: ?[bus_device_identifier_len]u8, + activity_prev: ?EngineSample = null, + encoder_prev: ?EngineSample = null, + decoder_prev: ?EngineSample = null, + energy_prev: ?EnergySample = null, +}; + +const EnergySample = struct { + micro_joules: u64, + timestamp_ns: u64, +}; + +pub const EngineSample = struct { + engine_ns: u64, + timestamp_ns: u64, +}; + +const DeviceSample = struct { + mem_kib: u64 = 0, + engine: ?EngineSample = null, + encoder: ?EngineSample = null, + decoder: ?EngineSample = null, +}; + +pub const ProcessSample = struct { + pid: u32, + device_idx: u16, + mem_kib: ?u64 = null, + engine: ?EngineSample = null, + encoder: ?EngineSample = null, + decoder: ?EngineSample = null, +}; + +pub const ProcessUsage = std.AutoHashMapUnmanaged(u64, ProcessSample); +pub const ProcessPrevious = std.AutoHashMapUnmanaged(u64, EngineSample); +const DeviceUsage = std.AutoHashMapUnmanaged(u16, DeviceSample); + +allocator: std.mem.Allocator, +io: std.Io, +devices: []Device, + +pub fn init(allocator: std.mem.Allocator, io: std.Io) !Monitor { + var monitor: Monitor = .{ + .allocator = allocator, + .io = io, + .devices = try discoverIntelDevices(allocator, io), + }; + try monitor.seedDeviceMetrics(); + return monitor; +} + +pub fn deviceId(self: *const Monitor, allocator: std.mem.Allocator, handle: Handle) ![]const u8 { + const dev = try self.device(handle); + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/device", .{dev.dev_path}); + const raw = try smi_sysfs.readString(allocator, self.io, path); + return std.mem.trim(u8, raw, &std.ascii.whitespace); +} + +inline fn seedDeviceMetrics(self: *Monitor) !void { + var device_usage = collectDeviceUsage(self.allocator, self.io, self.devices) catch DeviceUsage{}; + defer device_usage.deinit(self.allocator); + + for (self.devices, 0..) |*dev, idx| { + dev.energy_prev = readEnergy(self.allocator, self.io, dev) catch null; + if (device_usage.get(@intCast(idx))) |sample| { + dev.activity_prev = sample.engine; + dev.encoder_prev = sample.encoder; + dev.decoder_prev = sample.decoder; + } + } +} + +pub fn saveProcessPrevious(allocator: std.mem.Allocator, previous: *ProcessPrevious, usage: *const ProcessUsage) !void { + previous.clearRetainingCapacity(); + try previous.ensureTotalCapacity(allocator, @intCast(usage.count())); + + var it = usage.iterator(); + while (it.next()) |entry| { + if (entry.value_ptr.engine) |engine| { + previous.putAssumeCapacity(entry.key_ptr.*, engine); + } + } +} + +inline fn discoverIntelDevices(allocator: std.mem.Allocator, io: std.Io) ![]Device { + var devices: std.ArrayList(Device) = .empty; + errdefer devices.deinit(allocator); + + var dir = std.Io.Dir.openDirAbsolute(io, "/dev/dri", .{ .iterate = true }) catch |err| switch (err) { + error.FileNotFound => return &.{}, + else => |e| return e, + }; + defer dir.close(io); + + var it = dir.iterate(); + while (it.next(io) catch null) |entry| { + const render_idx = parseRenderIndex(entry.name) orelse continue; + const dev = openDevice(allocator, io, render_idx) catch continue; + try devices.append(allocator, dev); + } + + std.mem.sort(Device, devices.items, {}, struct { + fn lessThan(_: void, lhs: Device, rhs: Device) bool { + return lhs.render_idx < rhs.render_idx; + } + }.lessThan); + return devices.toOwnedSlice(allocator); +} + +inline fn openDevice(allocator: std.mem.Allocator, io: std.Io, render_idx: usize) !Device { + const drm_path = try std.fmt.allocPrint(allocator, "/sys/class/drm/renderD{d}", .{render_idx}); + errdefer allocator.free(drm_path); + + const dev_path = try std.fmt.allocPrint(allocator, "{s}/device", .{drm_path}); + errdefer allocator.free(dev_path); + + var vendor_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const vendor_path = try std.fmt.bufPrint(&vendor_path_buf, "{s}/vendor", .{dev_path}); + const vendor = try smi_sysfs.readString(allocator, io, vendor_path); + if (!std.mem.eql(u8, std.mem.trim(u8, vendor, &std.ascii.whitespace), "0x8086")) { + return error.not_intel; + } + + const pcie_path = try findPcieLinkPath(allocator, io, dev_path); + errdefer if (pcie_path.ptr != dev_path.ptr) allocator.free(pcie_path); + + return .{ + .render_idx = render_idx, + .drm_path = drm_path, + .dev_path = dev_path, + .pcie_path = pcie_path, + .hwmon_path = findHwmon(allocator, io, dev_path) catch null, + .bus_device_identifier = readBusDeviceId(allocator, io, dev_path) catch null, + }; +} + +pub fn device(self: *const Monitor, handle: Handle) !*const Device { + const idx = @intFromEnum(handle); + if (idx >= self.devices.len) return error.NotFound; + return &self.devices[idx]; +} + +pub fn readTemperature(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + const hwmon = dev.hwmon_path orelse return error.NotFound; + const milli_c = readHwmonInput(allocator, io, hwmon, "temp", "pkg") catch + readHwmonInput(allocator, io, hwmon, "temp", "gpu") catch + readNumberedInput(allocator, io, hwmon, "temp", 2) catch + try readNumberedInput(allocator, io, hwmon, "temp", 1); + return (milli_c + 500) / 1000; +} + +pub fn readEnergy(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !EnergySample { + const hwmon = dev.hwmon_path orelse return error.NotFound; + const micro_joules = readHwmonInput(allocator, io, hwmon, "energy", "card") catch + readHwmonInput(allocator, io, hwmon, "energy", "pkg") catch + try readNumberedInput(allocator, io, hwmon, "energy", 1); + return .{ + .micro_joules = micro_joules, + .timestamp_ns = @intCast(std.Io.Timestamp.now(io, .awake).nanoseconds), + }; +} +pub fn readPowerLimit(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + const hwmon = dev.hwmon_path orelse return error.NotFound; + const microwatts = readNumberedSuffix(allocator, io, hwmon, "power", 1, "cap") catch + try readNumberedSuffix(allocator, io, hwmon, "power", 1, "max"); + return microwatts / 1000; +} + +pub fn readFanSpeed(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + const hwmon = dev.hwmon_path orelse return error.NotFound; + return readPwmFanSpeed(allocator, io, hwmon) catch readRpmFanSpeedPercent(allocator, io, hwmon); +} + +pub fn readClockGraphics(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + return readFreq(allocator, io, dev, "act_freq") catch readFreq(allocator, io, dev, "cur_freq"); +} + +pub fn readMaxClockGraphics(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + return readFreq(allocator, io, dev, "max_freq"); +} + +pub fn readMemTotal(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/resource", .{dev.dev_path}); + return readLargestResource(allocator, io, path); +} + +pub fn readPcieLinkGen(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/current_link_speed", .{dev.pcie_path}); + const raw = try smi_sysfs.readString(allocator, io, path); + return pcieGenFromSpeed(raw); +} + +pub fn readPcieLinkWidth(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/current_link_width", .{dev.pcie_path}); + return smi_sysfs.readInt(allocator, io, path); +} + +pub fn readPcieBandwidth(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + const gen = try readPcieLinkGen(allocator, io, dev); + const width = try readPcieLinkWidth(allocator, io, dev); + return pcieBandwidthFromLink(gen, width) orelse error.NotFound; +} + +pub fn collectDeviceUsage(allocator: std.mem.Allocator, io: std.Io, devices: []const Device) !DeviceUsage { + var processes = try collectProcessUsage(allocator, io, devices); + errdefer processes.deinit(allocator); + + var result: DeviceUsage = .{}; + errdefer result.deinit(allocator); + + var it = processes.iterator(); + while (it.next()) |entry| { + const sample = entry.value_ptr.*; + const gop = try result.getOrPut(allocator, sample.device_idx); + if (!gop.found_existing) gop.value_ptr.* = .{}; + if (sample.mem_kib) |mem| gop.value_ptr.mem_kib += mem; + addEngineSample(&gop.value_ptr.engine, sample.engine); + addEngineSample(&gop.value_ptr.encoder, sample.encoder); + addEngineSample(&gop.value_ptr.decoder, sample.decoder); + } + + return result; +} + +pub fn collectProcessUsage(allocator: std.mem.Allocator, io: std.Io, devices: []const Device) !ProcessUsage { + var result: ProcessUsage = .{}; + errdefer result.deinit(allocator); + + var proc_dir = try std.Io.Dir.openDirAbsolute(io, "/proc", .{ .iterate = true }); + defer proc_dir.close(io); + + const now_ns: u64 = @intCast(std.Io.Timestamp.now(io, .awake).nanoseconds); + var proc_it = proc_dir.iterate(); + while (proc_it.next(io) catch null) |proc_entry| { + const pid = std.fmt.parseInt(u32, proc_entry.name, 10) catch continue; + + var fdinfo_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const fdinfo_path = std.fmt.bufPrint(&fdinfo_path_buf, "/proc/{d}/fdinfo", .{pid}) catch continue; + var fdinfo_dir = std.Io.Dir.openDirAbsolute(io, fdinfo_path, .{ .iterate = true }) catch continue; + defer fdinfo_dir.close(io); + + var fd_it = fdinfo_dir.iterate(); + while (fd_it.next(io) catch null) |fd_entry| { + var file_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const file_path = std.fmt.bufPrint(&file_path_buf, "{s}/{s}", .{ fdinfo_path, fd_entry.name }) catch continue; + const parsed = parseFdinfoFile(io, file_path, now_ns) catch continue; + const bus_device_identifier = parsed.bus_device_identifier orelse continue; + const dev_idx = matchDevice(devices, bus_device_identifier) orelse continue; + const key = processKey(pid, dev_idx); + const gop = try result.getOrPut(allocator, key); + if (!gop.found_existing) { + gop.value_ptr.* = .{ .pid = pid, .device_idx = dev_idx }; + } + mergeProcessSample(gop.value_ptr, parsed.sample); + } + } + + return result; +} + +pub fn processUtil(previous: ?EngineSample, current: ?EngineSample) ?u16 { + const prev = previous orelse return null; + const cur = current orelse return null; + if (cur.timestamp_ns <= prev.timestamp_ns or cur.engine_ns < prev.engine_ns) return null; + const engine_delta = cur.engine_ns - prev.engine_ns; + const time_delta = cur.timestamp_ns - prev.timestamp_ns; + if (time_delta == 0) return null; + const pct = @min(engine_delta * 100 / time_delta, 100); + return @intCast(if (pct == 0 and engine_delta > 0) 0 else pct); +} + +pub fn powerMilliwatts(previous: ?EnergySample, current: ?EnergySample) ?u64 { + const prev = previous orelse return null; + const cur = current orelse return null; + return milliwattsFromEnergyDelta(prev.micro_joules, cur.micro_joules, prev.timestamp_ns, cur.timestamp_ns); +} + +inline fn milliwattsFromEnergyDelta(prev_uj: u64, cur_uj: u64, prev_ns: u64, cur_ns: u64) ?u64 { + if (cur_uj < prev_uj or cur_ns <= prev_ns) return null; + const energy_delta = cur_uj - prev_uj; + const time_delta = cur_ns - prev_ns; + if (time_delta == 0) return null; + return energy_delta * 1_000_000 / time_delta; +} + +inline fn readFreq(allocator: std.mem.Allocator, io: std.Io, dev: *const Device, file: []const u8) !u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/tile0/gt0/freq0/{s}", .{ dev.dev_path, file }); + return smi_sysfs.readInt(allocator, io, path); +} + +inline fn parseRenderIndex(name: []const u8) ?usize { + if (!std.mem.startsWith(u8, name, "renderD")) return null; + return std.fmt.parseInt(usize, name["renderD".len..], 10) catch null; +} + +inline fn findHwmon(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) !?[]const u8 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const hwmon_root = try std.fmt.bufPrint(&path_buf, "{s}/hwmon", .{dev_path}); + var dir = std.Io.Dir.openDirAbsolute(io, hwmon_root, .{ .iterate = true }) catch return null; + defer dir.close(io); + + var it = dir.iterate(); + while (it.next(io) catch null) |entry| { + if (!std.mem.startsWith(u8, entry.name, "hwmon")) continue; + const candidate = try std.fmt.allocPrint(allocator, "{s}/{s}", .{ hwmon_root, entry.name }); + var name_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const name_path = try std.fmt.bufPrint(&name_buf, "{s}/name", .{candidate}); + const name = smi_sysfs.readString(allocator, io, name_path) catch return candidate; + const trimmed = std.mem.trim(u8, name, &std.ascii.whitespace); + if (std.mem.eql(u8, trimmed, "xe") or std.mem.eql(u8, trimmed, "i915")) return candidate; + } + return null; +} + +inline fn readBusDeviceId(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) ![bus_device_identifier_len]u8 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/uevent", .{dev_path}); + const raw = try smi_sysfs.readFieldString(allocator, io, path, "PCI_SLOT_NAME="); + const trimmed = std.mem.trim(u8, raw, &std.ascii.whitespace); + if (trimmed.len != bus_device_identifier_len) return error.NotFound; + var out: [bus_device_identifier_len]u8 = undefined; + @memcpy(&out, trimmed); + return out; +} + +inline fn findPcieLinkPath(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) ![]const u8 { + var candidate = dev_path; + var best_path: ?[]const u8 = null; + + var best_bandwidth: u64 = 0; + var depth: usize = 0; + while (depth < 8) : (depth += 1) { + if (!try isIntelPciDevice(allocator, io, candidate)) break; + + const bandwidth = readMaxPcieBandwidth(allocator, io, candidate) catch 0; + if (bandwidth > best_bandwidth) { + best_path = candidate; + best_bandwidth = bandwidth; + } + + const parent = try std.fmt.allocPrint(allocator, "{s}/..", .{candidate}); + candidate = parent; + } + + return best_path orelse dev_path; +} + +inline fn isIntelPciDevice(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) !bool { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const vendor_path = try std.fmt.bufPrint(&path_buf, "{s}/vendor", .{dev_path}); + const vendor_raw = smi_sysfs.readString(allocator, io, vendor_path) catch return false; + + const vendor = std.mem.trim(u8, vendor_raw, &std.ascii.whitespace); + return std.mem.eql(u8, vendor, "0x8086"); +} + +inline fn readMaxPcieBandwidth(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) !u64 { + var speed_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const speed_path = try std.fmt.bufPrint(&speed_path_buf, "{s}/max_link_speed", .{dev_path}); + const raw_speed = try smi_sysfs.readString(allocator, io, speed_path); + + var width_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const width_path = try std.fmt.bufPrint(&width_path_buf, "{s}/max_link_width", .{dev_path}); + const width = try smi_sysfs.readInt(allocator, io, width_path); + const gen = try pcieGenFromSpeed(raw_speed); + return pcieBandwidthFromLink(gen, width) orelse error.NotFound; +} + +inline fn readHwmonInput(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8, comptime prefix: []const u8, label: []const u8) !u64 { + for (1..16) |idx| { + var label_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const label_path = try std.fmt.bufPrint(&label_path_buf, "{s}/{s}{d}_label", .{ hwmon, prefix, idx }); + const raw = smi_sysfs.readString(allocator, io, label_path) catch continue; + const trimmed = std.mem.trim(u8, raw, &std.ascii.whitespace); + if (!std.ascii.eqlIgnoreCase(trimmed, label)) continue; + return readNumberedInput(allocator, io, hwmon, prefix, idx); + } + return error.NotFound; +} + +inline fn readNumberedInput(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8, comptime prefix: []const u8, idx: usize) !u64 { + return readNumberedSuffix(allocator, io, hwmon, prefix, idx, "input"); +} + +inline fn readNumberedSuffix(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8, comptime prefix: []const u8, idx: usize, suffix: []const u8) !u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/{s}{d}_{s}", .{ hwmon, prefix, idx, suffix }); + return smi_sysfs.readInt(allocator, io, path); +} + +inline fn readNumberedRaw(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8, comptime prefix: []const u8, idx: usize) !u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/{s}{d}", .{ hwmon, prefix, idx }); + return smi_sysfs.readInt(allocator, io, path); +} + +inline fn readPwmFanSpeed(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8) !u64 { + for (1..16) |idx| { + const pwm = readNumberedRaw(allocator, io, hwmon, "pwm", idx) catch continue; + return pwmToPercent(pwm); + } + return error.NotFound; +} + +inline fn readRpmFanSpeedPercent(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8) !u64 { + for (1..16) |idx| { + const input = readNumberedSuffix(allocator, io, hwmon, "fan", idx, "input") catch continue; + const max = readNumberedSuffix(allocator, io, hwmon, "fan", idx, "max") catch continue; + if (max == 0) continue; + return @min(input * 100 / max, 100); + } + return error.NotFound; +} + +inline fn pwmToPercent(pwm: u64) u64 { + return @min((@min(pwm, 255) * 100 + 127) / 255, 100); +} + +inline fn readLargestResource(allocator: std.mem.Allocator, io: std.Io, path: []const u8) !u64 { + var file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); + defer file.close(io); + + var read_buf: [4096]u8 = undefined; + var reader = file.reader(io, &read_buf); + var line_writer: std.Io.Writer.Allocating = .init(allocator); + defer line_writer.deinit(); + + var largest: u64 = 0; + while (true) { + line_writer.clearRetainingCapacity(); + _ = reader.interface.streamDelimiter(&line_writer.writer, '\n') catch |err| switch (err) { + error.EndOfStream => break, + else => |e| return e, + }; + reader.interface.toss(1); + largest = @max(largest, resourceLineSize(line_writer.written()) orelse 0); + } + if (largest == 0) return error.NotFound; + return largest; +} + +inline fn resourceLineSize(line: []const u8) ?u64 { + var it = std.mem.tokenizeAny(u8, line, " \t"); + const base = parseHex(it.next() orelse return null) orelse return null; + const end = parseHex(it.next() orelse return null) orelse return null; + if (end < base) return null; + const size = end - base + 1; + return if (size >= 256 * 1024 * 1024) size else null; +} + +inline fn parseHex(raw: []const u8) ?u64 { + const trimmed = std.mem.trim(u8, raw, &std.ascii.whitespace); + const digits = if (std.mem.startsWith(u8, trimmed, "0x")) trimmed[2..] else trimmed; + return std.fmt.parseInt(u64, digits, 16) catch null; +} + +inline fn pcieGenFromSpeed(raw: []const u8) !u64 { + const trimmed = std.mem.trim(u8, raw, &std.ascii.whitespace); + if (std.mem.startsWith(u8, trimmed, "2.5")) return 1; + if (std.mem.startsWith(u8, trimmed, "5.0") or std.mem.startsWith(u8, trimmed, "5 ")) return 2; + if (std.mem.startsWith(u8, trimmed, "8.0") or std.mem.startsWith(u8, trimmed, "8 ")) return 3; + if (std.mem.startsWith(u8, trimmed, "16.0") or std.mem.startsWith(u8, trimmed, "16 ")) return 4; + if (std.mem.startsWith(u8, trimmed, "32.0") or std.mem.startsWith(u8, trimmed, "32 ")) return 5; + if (std.mem.startsWith(u8, trimmed, "64.0") or std.mem.startsWith(u8, trimmed, "64 ")) return 6; + return error.NotFound; +} + +inline fn pcieBandwidthFromLink(gen: u64, width: u64) ?u64 { + const lane_mb_s: u64 = switch (gen) { + 1 => 250, + 2 => 500, + 3 => 985, + 4 => 1969, + 5 => 3938, + 6 => 7563, + else => return null, + }; + return lane_mb_s * width; +} + +const FdinfoSample = struct { + mem_kib: ?u64 = null, + engine: ?EngineSample = null, + encoder: ?EngineSample = null, + decoder: ?EngineSample = null, +}; + +const ParsedFdinfo = struct { + bus_device_identifier: ?[bus_device_identifier_len]u8 = null, + sample: FdinfoSample = .{}, + resident_vram_kib: u64 = 0, + total_vram_kib: u64 = 0, + resident_memory_kib: u64 = 0, + total_memory_kib: u64 = 0, + cycle_counter: u64 = 0, + total_cycle_counter: u64 = 0, + engine_time_seen: bool = false, +}; + +inline fn parseFdinfoFile(io: std.Io, path: []const u8, timestamp_ns: u64) !ParsedFdinfo { + var read_buf: [8192]u8 = undefined; + const data = std.Io.Dir.readFile(.cwd(), io, path, &read_buf) catch return error.FileUnavailable; + + var parsed: ParsedFdinfo = .{}; + + var lines = std.mem.tokenizeScalar(u8, data, '\n'); + while (lines.next()) |line| { + parseFdinfoLine(&parsed, line, timestamp_ns); + } + + finishFdinfo(&parsed); + return parsed; +} + +inline fn finishFdinfo(self: *ParsedFdinfo) void { + if (self.resident_vram_kib > 0) { + self.sample.mem_kib = self.resident_vram_kib; + } else if (self.total_vram_kib > 0) { + self.sample.mem_kib = self.total_vram_kib; + } else if (self.resident_memory_kib > 0) { + self.sample.mem_kib = self.resident_memory_kib; + } else if (self.total_memory_kib > 0) { + self.sample.mem_kib = self.total_memory_kib; + } + + if (!self.engine_time_seen and self.total_cycle_counter > 0) { + self.sample.engine = .{ + .engine_ns = self.cycle_counter, + .timestamp_ns = self.total_cycle_counter, + }; + } +} + +inline fn parseFdinfoLine(parsed: *ParsedFdinfo, line: []const u8, timestamp_ns: u64) void { + if (std.mem.startsWith(u8, line, "drm-pdev:")) { + parsed.bus_device_identifier = parseBdf(line["drm-pdev:".len..]); + return; + } + + if (std.mem.startsWith(u8, line, "vram mem:")) { + parsed.total_vram_kib += memoryKiB(line["vram mem:".len..]); + return; + } + + if (std.mem.startsWith(u8, line, "drm-cycles-")) { + const colon = std.mem.indexOfScalar(u8, line, ':') orelse return; + parsed.cycle_counter += firstInt(line[colon + 1 ..]); + return; + } + + if (std.mem.startsWith(u8, line, "drm-total-cycles-")) { + const colon = std.mem.indexOfScalar(u8, line, ':') orelse return; + parsed.total_cycle_counter = @max(parsed.total_cycle_counter, firstInt(line[colon + 1 ..])); + return; + } + + if (std.mem.startsWith(u8, line, "drm-engine-capacity-")) { + return; + } + + if (std.mem.startsWith(u8, line, "drm-engine-")) { + const colon = std.mem.indexOfScalar(u8, line, ':') orelse return; + const engine_label = line["drm-engine-".len..colon]; + const value = firstInt(line[colon + 1 ..]); + parsed.engine_time_seen = true; + addEngineTime(&parsed.sample.engine, value, timestamp_ns); + if (containsAsciiIgnoreCase(engine_label, "encode")) { + addEngineTime(&parsed.sample.encoder, value, timestamp_ns); + } else if (containsAsciiIgnoreCase(engine_label, "decode")) { + addEngineTime(&parsed.sample.decoder, value, timestamp_ns); + } + return; + } + + if (std.mem.startsWith(u8, line, "drm-")) { + const colon = std.mem.indexOfScalar(u8, line, ':') orelse return; + const key = line[0..colon]; + const value = memoryKiB(line[colon + 1 ..]); + if (value == 0) return; + + if (std.mem.startsWith(u8, key, "drm-resident-vram")) { + parsed.resident_vram_kib += value; + return; + } + if (std.mem.startsWith(u8, key, "drm-total-vram") or std.mem.startsWith(u8, key, "drm-memory-vram")) { + parsed.total_vram_kib += value; + return; + } + if (std.mem.startsWith(u8, key, "drm-resident-")) { + parsed.resident_memory_kib += value; + return; + } + if (std.mem.startsWith(u8, key, "drm-total-")) { + parsed.total_memory_kib += value; + return; + } + if (std.mem.eql(u8, key, "drm-resident-memory")) { + parsed.resident_memory_kib += value; + return; + } + if (std.mem.eql(u8, key, "drm-total-memory")) { + parsed.total_memory_kib += value; + return; + } + } +} + +fn mergeProcessSample(dst: *ProcessSample, src: FdinfoSample) void { + if (src.mem_kib) |mem| dst.mem_kib = (dst.mem_kib orelse 0) + mem; + addEngineSample(&dst.engine, src.engine); + addEngineSample(&dst.encoder, src.encoder); + addEngineSample(&dst.decoder, src.decoder); +} + +fn addEngineSample(dst: *?EngineSample, src: ?EngineSample) void { + if (src) |engine| addEngineTime(dst, engine.engine_ns, engine.timestamp_ns); +} + +fn addEngineTime(dst: *?EngineSample, engine_ns: u64, timestamp_ns: u64) void { + const current = dst.* orelse EngineSample{ .engine_ns = 0, .timestamp_ns = timestamp_ns }; + dst.* = .{ + .engine_ns = current.engine_ns + engine_ns, + .timestamp_ns = timestamp_ns, + }; +} + +fn matchDevice(devices: []const Device, bus_device_identifier: [bus_device_identifier_len]u8) ?u16 { + for (devices, 0..) |dev, idx| { + if (dev.bus_device_identifier) |device_bus_device_identifier| { + if (std.mem.eql(u8, &device_bus_device_identifier, &bus_device_identifier)) return @intCast(idx); + } + } + return null; +} + +fn processKey(pid: u32, device_idx: u16) u64 { + return (@as(u64, device_idx) << 32) | pid; +} + +pub const BdfParts = struct { + domain: u32, + bus: u32, + device: u32, + function: u32, +}; + +pub fn formatBdf(parts: BdfParts) [bus_device_identifier_len]u8 { + var buf: [bus_device_identifier_len]u8 = undefined; + _ = std.fmt.bufPrint(&buf, "{x:0>4}:{x:0>2}:{x:0>2}.{x}", .{ + parts.domain, + parts.bus, + parts.device, + parts.function, + }) catch unreachable; + return buf; +} + +inline fn parseBdf(raw: []const u8) ?[bus_device_identifier_len]u8 { + const trimmed = std.mem.trim(u8, raw, " \t\n"); + var out: [bus_device_identifier_len]u8 = undefined; + if (trimmed.len == bus_device_identifier_len) { + @memcpy(&out, trimmed[0..bus_device_identifier_len]); + return out; + } + if (trimmed.len == "00:00.0".len) { + _ = std.fmt.bufPrint(&out, "0000:{s}", .{trimmed}) catch return null; + return out; + } + return null; +} + +inline fn firstInt(raw: []const u8) u64 { + var iter = std.mem.tokenizeAny(u8, raw, " \t"); + return std.fmt.parseInt(u64, iter.next() orelse return 0, 10) catch 0; +} + +inline fn memoryKiB(raw: []const u8) u64 { + var iter = std.mem.tokenizeAny(u8, raw, " \t"); + const value = std.fmt.parseInt(u64, iter.next() orelse return 0, 10) catch return 0; + const unit = iter.next() orelse return value; + if (std.ascii.eqlIgnoreCase(unit, "KiB")) return value; + if (std.ascii.eqlIgnoreCase(unit, "MiB")) return value * 1024; + if (std.ascii.eqlIgnoreCase(unit, "GiB")) return value * 1024 * 1024; + if (std.ascii.eqlIgnoreCase(unit, "B")) return value / 1024; + return value; +} + +inline fn containsAsciiIgnoreCase(haystack: []const u8, needle: []const u8) bool { + if (needle.len == 0) return true; + if (needle.len > haystack.len) return false; + for (0..haystack.len - needle.len + 1) |i| { + if (std.ascii.eqlIgnoreCase(haystack[i .. i + needle.len], needle)) return true; + } + return false; +} + +test "oneAPI fdinfo parser" { + var parsed: ParsedFdinfo = .{}; + parseFdinfoLine(&parsed, "drm-pdev:\t0000:03:00.0", 10); + parseFdinfoLine(&parsed, "drm-total-memory:\t2048 KiB", 10); + parseFdinfoLine(&parsed, "drm-engine-render:\t100 ns", 10); + parseFdinfoLine(&parsed, "drm-engine-compute:\t50 ns", 10); + finishFdinfo(&parsed); + + try std.testing.expect(parsed.bus_device_identifier != null); + try std.testing.expectEqualSlices(u8, "0000:03:00.0", &parsed.bus_device_identifier.?); + try std.testing.expectEqual(@as(?u64, 2048), parsed.sample.mem_kib); + try std.testing.expectEqual(@as(u64, 150), parsed.sample.engine.?.engine_ns); + try std.testing.expectEqual(@as(?EngineSample, null), parsed.sample.encoder); + try std.testing.expectEqual(@as(?EngineSample, null), parsed.sample.decoder); +} + +test "oneAPI fdinfo parser classifies explicit media engines" { + var parsed: ParsedFdinfo = .{}; + parseFdinfoLine(&parsed, "drm-engine-video-decode:\t100 ns", 10); + parseFdinfoLine(&parsed, "drm-engine-media-encode:\t50 ns", 10); + parseFdinfoLine(&parsed, "drm-engine-VIDEO-DECODE1:\t25 ns", 10); + finishFdinfo(&parsed); + + try std.testing.expectEqual(@as(u64, 175), parsed.sample.engine.?.engine_ns); + try std.testing.expectEqual(@as(u64, 50), parsed.sample.encoder.?.engine_ns); + try std.testing.expectEqual(@as(u64, 125), parsed.sample.decoder.?.engine_ns); +} + +test "oneAPI fdinfo parser leaves generic video engines unclassified" { + var parsed: ParsedFdinfo = .{}; + parseFdinfoLine(&parsed, "drm-engine-video:\t100 ns", 10); + parseFdinfoLine(&parsed, "drm-engine-vcs0:\t50 ns", 10); + parseFdinfoLine(&parsed, "drm-engine-vecs0:\t25 ns", 10); + finishFdinfo(&parsed); + + try std.testing.expectEqual(@as(u64, 175), parsed.sample.engine.?.engine_ns); + try std.testing.expectEqual(@as(?EngineSample, null), parsed.sample.encoder); + try std.testing.expectEqual(@as(?EngineSample, null), parsed.sample.decoder); +} + +test "oneAPI fdinfo parser cycle counters" { + var parsed: ParsedFdinfo = .{}; + parseFdinfoLine(&parsed, "drm-pdev:\t0000:03:00.0", 10); + parseFdinfoLine(&parsed, "drm-cycles-rcs:\t10", 10); + parseFdinfoLine(&parsed, "drm-total-cycles-rcs:\t100", 10); + parseFdinfoLine(&parsed, "drm-engine-capacity-vcs:\t2", 10); + parseFdinfoLine(&parsed, "drm-cycles-ccs:\t30", 10); + parseFdinfoLine(&parsed, "drm-total-cycles-ccs:\t100", 10); + finishFdinfo(&parsed); + + try std.testing.expectEqual(@as(u64, 40), parsed.sample.engine.?.engine_ns); + try std.testing.expectEqual(@as(u64, 100), parsed.sample.engine.?.timestamp_ns); +} + +test "oneAPI fdinfo parser xe vram regions" { + var parsed: ParsedFdinfo = .{}; + parseFdinfoLine(&parsed, "drm-pdev:\t0000:03:00.0", 10); + parseFdinfoLine(&parsed, "drm-total-system:\t184 MiB", 10); + parseFdinfoLine(&parsed, "drm-total-vram0:\t4096 KiB", 10); + parseFdinfoLine(&parsed, "drm-resident-vram0:\t3072 KiB", 10); + finishFdinfo(&parsed); + + try std.testing.expectEqual(@as(?u64, 3072), parsed.sample.mem_kib); +} + +test "oneAPI fdinfo parser falls back to total vram" { + var parsed: ParsedFdinfo = .{}; + parseFdinfoLine(&parsed, "drm-total-vram0:\t2 MiB", 10); + finishFdinfo(&parsed); + + try std.testing.expectEqual(@as(?u64, 2048), parsed.sample.mem_kib); +} + +test "oneAPI process utilization delta" { + try std.testing.expectEqual(@as(?u16, 50), processUtil(.{ .engine_ns = 100, .timestamp_ns = 1000 }, .{ .engine_ns = 600, .timestamp_ns = 2000 })); + try std.testing.expectEqual(@as(?u16, 1), processUtil(.{ .engine_ns = 100, .timestamp_ns = 1000 }, .{ .engine_ns = 101, .timestamp_ns = 2000 })); + try std.testing.expectEqual(@as(?u16, null), processUtil(null, .{ .engine_ns = 600, .timestamp_ns = 2000 })); +} + +test "oneAPI PWM fan speed conversion" { + try std.testing.expectEqual(@as(u64, 0), pwmToPercent(0)); + try std.testing.expectEqual(@as(u64, 50), pwmToPercent(128)); + try std.testing.expectEqual(@as(u64, 100), pwmToPercent(255)); + try std.testing.expectEqual(@as(u64, 100), pwmToPercent(300)); +} + +test "oneAPI process previous keeps engine samples" { + var usage: ProcessUsage = .{}; + defer usage.deinit(std.testing.allocator); + + const engine_key = processKey(10, 1); + const mem_only_key = processKey(20, 1); + try usage.put(std.testing.allocator, engine_key, .{ + .pid = 10, + .device_idx = 1, + .engine = .{ .engine_ns = 100, .timestamp_ns = 1000 }, + }); + try usage.put(std.testing.allocator, mem_only_key, .{ + .pid = 20, + .device_idx = 1, + .mem_kib = 2048, + }); + + var previous: ProcessPrevious = .{}; + defer previous.deinit(std.testing.allocator); + + try saveProcessPrevious(std.testing.allocator, &previous, &usage); + + try std.testing.expectEqual(@as(u32, 1), previous.count()); + try std.testing.expectEqual(@as(?EngineSample, .{ .engine_ns = 100, .timestamp_ns = 1000 }), previous.get(engine_key)); + try std.testing.expectEqual(@as(?EngineSample, null), previous.get(mem_only_key)); +} + +test "oneAPI power delta from hwmon energy" { + try std.testing.expectEqual(@as(?u64, 20_000), milliwattsFromEnergyDelta(1_000, 11_000, 1_000, 501_000)); + try std.testing.expectEqual(@as(?u64, null), milliwattsFromEnergyDelta(11_000, 1_000, 1_000, 501_000)); + try std.testing.expectEqual(@as(?u64, null), milliwattsFromEnergyDelta(1_000, 11_000, 501_000, 1_000)); +} + +test "oneAPI resource line chooses large memory BARs" { + try std.testing.expectEqual(@as(?u64, 34_359_738_368), resourceLineSize("0x0000048800000000 0x0000048fffffffff 0x000000000014220c")); + try std.testing.expectEqual(@as(?u64, null), resourceLineSize("0x00000000f6c00000 0x00000000f6dfffff 0x0000000000046200")); +} + +test "oneAPI PCIe speed maps to generation" { + try std.testing.expectEqual(@as(u64, 1), try pcieGenFromSpeed("2.5 GT/s PCIe")); + try std.testing.expectEqual(@as(u64, 4), try pcieGenFromSpeed("16.0 GT/s PCIe")); + try std.testing.expectError(error.NotFound, pcieGenFromSpeed("Unknown")); +} + +test "oneAPI PCIe bandwidth derives from generation and width" { + try std.testing.expectEqual(@as(?u64, 250), pcieBandwidthFromLink(1, 1)); + try std.testing.expectEqual(@as(?u64, 63_008), pcieBandwidthFromLink(5, 16)); + try std.testing.expectEqual(@as(?u64, null), pcieBandwidthFromLink(0, 16)); +} diff --git a/bin/zml-smi/platforms/oneapi/oneapi.zig b/bin/zml-smi/platforms/oneapi/oneapi.zig new file mode 100644 index 00000000..0e89a6a2 --- /dev/null +++ b/bin/zml-smi/platforms/oneapi/oneapi.zig @@ -0,0 +1,248 @@ +const std = @import("std"); +const Monitor = @import("monitor.zig"); +const smi_sysfs = @import("zml-smi/sysfs"); + +const OneApi = @This(); + +pub const Handle = Monitor.Handle; + +monitor: Monitor, +io: std.Io, + +pub fn init(allocator: std.mem.Allocator, io: std.Io) !OneApi { + return .{ + .monitor = try Monitor.init(allocator, io), + .io = io, + }; +} + +pub fn handleByIndex(self: *const OneApi, device_id: usize) !Handle { + if (device_id >= self.monitor.devices.len) return error.NotFound; + return @enumFromInt(@as(u32, @intCast(device_id))); +} + +pub fn name(self: *const OneApi, allocator: std.mem.Allocator, handle: Handle) ![]const u8 { + if (self.monitor.device(handle)) |dev| { + if (dev.bus_device_identifier) |bdf| { + if (try lspciName(allocator, self.io, bdf[0..])) |display_name| { + return display_name; + } + } + } else |_| {} + + const id = try self.monitor.deviceId(allocator, handle); + return try std.fmt.allocPrint(allocator, "Intel GPU {s}", .{id}); +} + +fn lspciName(allocator: std.mem.Allocator, io: std.Io, bdf: []const u8) !?[]const u8 { + const result = std.process.run(allocator, io, .{ + .argv = &.{ "lspci", "-Dnn", "-s", bdf }, + .stdout_limit = .limited(4096), + .stderr_limit = .limited(1024), + .reserve_amount = 256, + }) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => return null, + }; + defer allocator.free(result.stdout); + defer allocator.free(result.stderr); + + switch (result.term) { + .exited => |code| if (code != 0) return null, + else => return null, + } + + return parseLspciName(allocator, result.stdout); +} + +fn parseLspciName(allocator: std.mem.Allocator, output: []const u8) !?[]const u8 { + var lines = std.mem.tokenizeScalar(u8, output, '\n'); + while (lines.next()) |line| { + const trimmed = std.mem.trim(u8, line, &std.ascii.whitespace); + const header_end = std.mem.indexOf(u8, trimmed, "]: ") orelse continue; + const header = trimmed[0 .. header_end + 1]; + if (!containsAsciiIgnoreCase(header, "VGA") and !containsAsciiIgnoreCase(header, "Display")) continue; + + const name_start = header_end + "]: ".len; + const name_end = std.mem.lastIndexOf(u8, trimmed[name_start..], " [8086:") orelse + std.mem.indexOf(u8, trimmed[name_start..], " (rev ") orelse + trimmed[name_start..].len; + const raw_name = std.mem.trim(u8, trimmed[name_start .. name_start + name_end], &std.ascii.whitespace); + if (raw_name.len == 0 or std.mem.startsWith(u8, raw_name, "Intel Corporation Device ")) return null; + return try allocator.dupe(u8, raw_name); + } + return null; +} + +fn containsAsciiIgnoreCase(haystack: []const u8, needle: []const u8) bool { + if (needle.len == 0) return true; + if (needle.len > haystack.len) return false; + for (0..haystack.len - needle.len + 1) |idx| { + if (std.ascii.eqlIgnoreCase(haystack[idx .. idx + needle.len], needle)) return true; + } + return false; +} + +test "parse lspci display name" { + const output = + \\0000:84:00.0 VGA compatible controller [0300]: Intel Corporation Battlemage G31 [Arc Pro B70] [8086:e223] (rev 04) + ; + const name_ = (try parseLspciName(std.testing.allocator, output)).?; + defer std.testing.allocator.free(name_); + try std.testing.expectEqualStrings("Intel Corporation Battlemage G31 [Arc Pro B70]", name_); +} + +test "parse lspci display name skips generic device ids" { + const output = + \\0000:84:00.0 Display controller [0380]: Intel Corporation Device e223 [8086:e223] (rev 04) + ; + try std.testing.expect(try parseLspciName(std.testing.allocator, output) == null); +} + +pub fn driverVersion(self: *const OneApi, allocator: std.mem.Allocator) ![]const u8 { + const version = smi_sysfs.readString(allocator, self.io, "/sys/module/xe/version") catch + smi_sysfs.readString(allocator, self.io, "/sys/module/xe/srcversion") catch + smi_sysfs.readString(allocator, self.io, "/sys/module/i915/version") catch + try smi_sysfs.readString(allocator, self.io, "/sys/module/i915/srcversion"); + const trimmed = std.mem.trim(u8, version, &std.ascii.whitespace); + if (trimmed.len == 0) return error.NotFound; + return trimmed; +} + +pub fn powerUsage(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = block: { + const idx = @intFromEnum(handle); + if (idx >= self.monitor.devices.len) return error.NotFound; + break :block &self.monitor.devices[idx]; + }; + const current = Monitor.readEnergy(allocator, self.io, dev) catch |err| { + dev.energy_prev = null; + return err; + }; + const power = Monitor.powerMilliwatts(dev.energy_prev, current) orelse { + dev.energy_prev = current; + return error.NotFound; + }; + dev.energy_prev = current; + return power; +} + +pub fn powerLimit(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readPowerLimit(allocator, self.io, dev); +} + +pub fn temperature(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readTemperature(allocator, self.io, dev); +} + +pub fn fanSpeed(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readFanSpeed(allocator, self.io, dev); +} + +pub fn gpuUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = block: { + const idx = @intFromEnum(handle); + if (idx >= self.monitor.devices.len) return error.NotFound; + break :block &self.monitor.devices[idx]; + }; + + var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch { + dev.activity_prev = null; + return 0; + }; + defer usage.deinit(allocator); + + const dev_idx: u16 = @intCast(@intFromEnum(handle)); + const current = if (usage.get(dev_idx)) |sample| sample.engine else null; + const util = Monitor.processUtil(dev.activity_prev, current) orelse 0; + dev.activity_prev = current; + return util; +} + +pub fn encoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = block: { + const idx = @intFromEnum(handle); + if (idx >= self.monitor.devices.len) return error.NotFound; + break :block &self.monitor.devices[idx]; + }; + + var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch |err| { + dev.encoder_prev = null; + return err; + }; + defer usage.deinit(allocator); + + const dev_idx: u16 = @intCast(@intFromEnum(handle)); + const current = if (usage.get(dev_idx)) |sample| sample.encoder else null; + const util = Monitor.processUtil(dev.encoder_prev, current) orelse { + dev.encoder_prev = current; + return error.NotFound; + }; + dev.encoder_prev = current; + return util; +} + +pub fn decoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = block: { + const idx = @intFromEnum(handle); + if (idx >= self.monitor.devices.len) return error.NotFound; + break :block &self.monitor.devices[idx]; + }; + + var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch |err| { + dev.decoder_prev = null; + return err; + }; + defer usage.deinit(allocator); + + const dev_idx: u16 = @intCast(@intFromEnum(handle)); + const current = if (usage.get(dev_idx)) |sample| sample.decoder else null; + const util = Monitor.processUtil(dev.decoder_prev, current) orelse { + dev.decoder_prev = current; + return error.NotFound; + }; + dev.decoder_prev = current; + return util; +} + +pub fn clockGraphics(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readClockGraphics(allocator, self.io, dev); +} + +pub fn maxClockGraphics(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readMaxClockGraphics(allocator, self.io, dev); +} + +pub fn memUsed(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev_idx: u16 = @intCast(@intFromEnum(handle)); + + var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch return 0; + defer usage.deinit(allocator); + + return if (usage.get(dev_idx)) |sample| sample.mem_kib * 1024 else 0; +} + +pub fn memTotal(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readMemTotal(allocator, self.io, dev); +} + +pub fn pcieLinkGen(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readPcieLinkGen(allocator, self.io, dev); +} + +pub fn pcieLinkWidth(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readPcieLinkWidth(allocator, self.io, dev); +} + +pub fn pcieBandwidth(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readPcieBandwidth(allocator, self.io, dev); +} diff --git a/bin/zml-smi/platforms/oneapi/process.zig b/bin/zml-smi/platforms/oneapi/process.zig new file mode 100644 index 00000000..c4d0c719 --- /dev/null +++ b/bin/zml-smi/platforms/oneapi/process.zig @@ -0,0 +1,45 @@ +const std = @import("std"); +const OneApi = @import("oneapi.zig"); +const Monitor = @import("monitor.zig"); +const pi = @import("zml-smi/info").process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const Collector = @import("zml-smi/collector").Collector; + +pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, oneapi: *OneApi, dev_offset: u16) !void { + const state = try collector.arena.create(Monitor.ProcessPrevious); + state.* = .{}; + + const previous_usage: ?Monitor.ProcessUsage = Monitor.collectProcessUsage(collector.gpa, collector.io, oneapi.monitor.devices) catch null; + if (previous_usage) |usage_value| { + var usage = usage_value; + defer usage.deinit(collector.gpa); + Monitor.saveProcessPrevious(collector.arena, state, &usage) catch {}; + } + + try collector.spawnPoll(pollOnce, .{ collector.gpa, collector.arena, collector.io, list, oneapi, dev_offset, state }); +} + +fn pollOnce(allocator: std.mem.Allocator, arena: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, oneapi: *OneApi, dev_offset: u16, state: *Monitor.ProcessPrevious) void { + const back = list.back(); + back.clearRetainingCapacity(); + + var usage = Monitor.collectProcessUsage(allocator, io, oneapi.monitor.devices) catch { + list.swap(); + return; + }; + defer usage.deinit(allocator); + + var it = usage.iterator(); + while (it.next()) |entry| { + const sample = entry.value_ptr.*; + back.append(allocator, .{ + .pid = sample.pid, + .device_idx = @intCast(sample.device_idx + dev_offset), + .dev_mem_kib = sample.mem_kib, + .dev_util_percent = Monitor.processUtil(state.get(entry.key_ptr.*), sample.engine), + }) catch break; + } + + Monitor.saveProcessPrevious(arena, state, &usage) catch {}; + list.swap(); +} diff --git a/bin/zml-smi/prometheus/exposition.zig b/bin/zml-smi/prometheus/exposition.zig index ac04d70d..c955226f 100644 --- a/bin/zml-smi/prometheus/exposition.zig +++ b/bin/zml-smi/prometheus/exposition.zig @@ -8,7 +8,7 @@ const device_metrics = .{ .{ .field = "util_percent", .metric = "zml_device_utilization_percent", .help = "Device utilization percentage" }, .{ .field = "mem_used_bytes", .metric = "zml_device_memory_used_bytes", .help = "Device memory used in bytes" }, .{ .field = "mem_total_bytes", .metric = "zml_device_memory_total_bytes", .help = "Device memory total in bytes" }, - // GPU (CUDA/ROCm) + // GPU (CUDA/ROCm/oneAPI) .{ .field = "encoder_util_percent", .metric = "zml_device_encoder_utilization_percent", .help = "Encoder utilization percentage" }, .{ .field = "decoder_util_percent", .metric = "zml_device_decoder_utilization_percent", .help = "Decoder utilization percentage" }, .{ .field = "power_mw", .metric = "zml_device_power_watts", .help = "Power draw in watts", .fmt = fmtMilliwattsAsWatts }, diff --git a/bin/zml-smi/tui/BUILD.bazel b/bin/zml-smi/tui/BUILD.bazel index 7602cde4..e0eb6420 100644 --- a/bin/zml-smi/tui/BUILD.bazel +++ b/bin/zml-smi/tui/BUILD.bazel @@ -34,6 +34,7 @@ zig_library( ], extra_srcs = [ "assets/amd.png", + "assets/intel.png", "assets/logo.png", "assets/neuron.png", "assets/nvidia.png", diff --git a/bin/zml-smi/tui/assets/intel.png b/bin/zml-smi/tui/assets/intel.png new file mode 100644 index 0000000000000000000000000000000000000000..1f71fcd7292cbb7bc9c35b05cbaef809f2a8cd20 GIT binary patch literal 2124 zcmV-S2($NzP)c3aw|h2FqI8z_QYwS4n_8dKm85`T!7gea1b z7=NgLYJ>!pgqoO!2m~l#Fkqo#NePwKUg(w5?ZVz?cD{37e|+E0&a|`Jc6Xbo{Z2BO z$(iq5-sim6Gb22>RXH!40HIwsY(2Vq|~8U`k}Q4xIEv+)~CS-rRwfUO4yx$B8t zn8+|C4hl_?Vd2bNwW}DJGq$mbO<)A5cn~{vAE2Eqi?b z$XH+Af)zs9k7hgx~}5 za14nZ=M~cv6;@ubjC()VSvSXpGjiOtsFk&sxAWDFA21TMF6OI3 zN71ovMLPh$-Fh4pDkaZ#OIvyJ?j>B>SvY0RE9V#a&h>Nn@$-lH`SXW}iVlBHY5{=} zulS&pP_`sS=+N#12jfCWx)GL?c!0nf`@yOZ*#3i4Y)i|!qh(I~! z%8Ocfal`UD;fE`M{<5!gijUs0fbXs8qB0ntz1KU_C?IK~q3OHS#B^(f;32ZY#{(6< zzxe>K?dd0Pg*&gF%}>7Ag|msn`u4S*JhtTs-Q8s*bqGd;_l^zogUtu9kzwnH1Gpx~ z@76D-HJ6;pq23W5e)b@L+dn`Yl*{K7c=$^fa^poURJ`)o9SeBw{a$wN?VA#VNf$#R z$w}Y!`#o&g)(x!@1bF0`eavl**l@!fM!eEiu&nB8=B1;<)s2e4vHqB6-|mKjg-YPd z*UVHFKhMA0!>bRiWcA`!L}0^Bb9mr`PtOqmnYiB> zVaL&7w(RW1wI|NcMurfSKfZH<4L8iGdj5HBQIcKOOcPnbHal_!OQ~;!waaIsn!wq0 zyu{TP6lqyDgV+Z^h^$Z^sPO)=VOB3`MHN;rX@i2}3{#4;O@vP0zqhB1_<)k8vmp?G z-cnqR`=h4RXc`}s*ehy;h3$DnlD?alv~u(P%g4|BLISF@*+F4yA<;}$14sopqYSy> zLwZsTuXU;Igquza(?VnrGqsF*N*Zuh4nrZ-NM_QhA}7Q_*?*#(Bz#7AWB&l(eR?mg zEe;%cPi*Vv<$s>wox?+1)m0=9ytZLEk8e4`%e#6Rt^_XaEO5{D zbGT(`8~b|7+_z~Tuk7xl?8gT_DzpY2H(xLlnQfdVKygh`<7@i-I1WImAM1lcWcc%r zPY{8~2ojRhZ)`Kf}>g!9nWoaAb zSXneH&!g)WVvy?5)~*+;;v&PHt2(&n+D>K_9TFOAiWyIWyi2ODf-^WHL?&6Ry~*Ly z%4|l#HPo%q@2g|2K;E)#Z$CeL?oa~3j%_FQ3VFx72L`$I*E`v9bU4Wl8??L=Hf`_W z_FwI!ocffL*E2w&C1TO6Jo9JdnAe^|gEqX;a}6DKB`F3}8y(4`U zhKFL_Ixs-a84jH&BUTy)BWd_g(X=3+676eV68y%ao5G5Kk zT5S@Cu?qqxCzZnrr8Lja+hj}}V8nwFOu?eH(KV!d9bGVQ)5xRsml3B387gi(6A&lZ zC^{wP+a@=5OQSUO17ZZ*QZrV^@`a#^&0C}@MDkr#mQ+=3b5u>1LRwPWC~A0_Dn_u) zr>-Y8I)BztKo#N!!ktOhHR=S%YhHE2eB$KsqlX!v#_3KbW%-}@Y3yA7n>+h)Gy{+U z$N*#jG5{HX3_u1T1CRm80Av6%02zP`KnCD^YySa |*sv| { + .cuda, .rocm, .oneapi => |*sv| { const gpu = sv.front().*; self.history.util[i].push(gpu.util_percent orelse 0); const used = gpu.mem_used_bytes orelse 0; diff --git a/bin/zml-smi/tui/image_cache.zig b/bin/zml-smi/tui/image_cache.zig index a2340ba5..cc841691 100644 --- a/bin/zml-smi/tui/image_cache.zig +++ b/bin/zml-smi/tui/image_cache.zig @@ -27,6 +27,7 @@ pub fn loadAll(self: *ImageCache, vx: *vaxis.Vaxis, allocator: std.mem.Allocator self.load(vx, allocator, writer, "logo", @embedFile("assets/logo.png")); self.load(vx, allocator, writer, "gpu_cuda", @embedFile("assets/nvidia.png")); self.load(vx, allocator, writer, "gpu_rocm", @embedFile("assets/amd.png")); + self.load(vx, allocator, writer, "gpu_oneapi", @embedFile("assets/intel.png")); self.load(vx, allocator, writer, "gpu_neuron", @embedFile("assets/neuron.png")); self.load(vx, allocator, writer, "gpu_tpu", @embedFile("assets/tpu.png")); } diff --git a/bin/zml-smi/tui/pages/detail.zig b/bin/zml-smi/tui/pages/detail.zig index d307bf5e..86d4b7a3 100644 --- a/bin/zml-smi/tui/pages/detail.zig +++ b/bin/zml-smi/tui/pages/detail.zig @@ -21,7 +21,7 @@ pub fn draw(self: *const Detail, ctx: vxfw.DrawContext) std.mem.Allocator.Error! const wgt = ui.widget(self); return switch (dev.*) { - .cuda, .rocm => |*sv| try gpu_detail.draw(self.state, self.process_table, ctx, w, id, sv.front().*, wgt), + .cuda, .rocm, .oneapi => |*sv| try gpu_detail.draw(self.state, self.process_table, ctx, w, id, sv.front().*, wgt), .neuron => |*sv| try neuron_detail.draw(self.state, self.process_table, ctx, w, id, sv.front().*, wgt), .tpu => |*sv| try tpu_detail.draw(self.state, self.process_table, ctx, w, id, sv.front().*, wgt), }; diff --git a/bin/zml-smi/tui/pages/detail/gpu_detail.zig b/bin/zml-smi/tui/pages/detail/gpu_detail.zig index 5943cfd2..7884d6f8 100644 --- a/bin/zml-smi/tui/pages/detail/gpu_detail.zig +++ b/bin/zml-smi/tui/pages/detail/gpu_detail.zig @@ -99,24 +99,18 @@ pub fn draw( // ── Clocks + PCIe ─────────────────────────────────────── var clock_lines: std.ArrayList(MetricCard.LineEntry) = .empty; - try clock_lines.append(ctx.arena, .{ .label = "Graphics ", .value = try std.fmt.allocPrint(ctx.arena, " {d} / {d} MHz", .{ - gpu.clock_graphics_mhz orelse 0, gpu.clock_graphics_max_mhz orelse 0, - }) }); + try clock_lines.append(ctx.arena, .{ .label = "Graphics ", .value = try formatClockPair(ctx.arena, gpu.clock_graphics_mhz, gpu.clock_graphics_max_mhz) }); if (gpu.clock_sm_mhz) |sm| { try clock_lines.append(ctx.arena, .{ .label = "SM ", .value = try std.fmt.allocPrint(ctx.arena, " {d} MHz", .{sm}) }); } if (gpu.clock_soc_mhz) |soc| { try clock_lines.append(ctx.arena, .{ .label = "SOC ", .value = try std.fmt.allocPrint(ctx.arena, " {d} MHz", .{soc}) }); } - try clock_lines.append(ctx.arena, .{ .label = "Memory ", .value = try std.fmt.allocPrint(ctx.arena, " {d} / {d} MHz", .{ - gpu.clock_mem_mhz orelse 0, gpu.clock_mem_max_mhz orelse 0, - }) }); + try clock_lines.append(ctx.arena, .{ .label = "Memory ", .value = try formatClockPair(ctx.arena, gpu.clock_mem_mhz, gpu.clock_mem_max_mhz) }); const clocks_card: MetricCard = .{ .title = "Clocks", .lines = clock_lines.items }; var pcie_lines: std.ArrayList(MetricCard.LineEntry) = .empty; - try pcie_lines.append(ctx.arena, .{ .label = "Link ", .value = try std.fmt.allocPrint(ctx.arena, " Gen{d} x{d}", .{ - gpu.pcie_link_gen orelse 0, gpu.pcie_link_width orelse 0, - }) }); + try pcie_lines.append(ctx.arena, .{ .label = "Link ", .value = try formatPcieLink(ctx.arena, gpu.pcie_link_gen, gpu.pcie_link_width) }); if (gpu.pcie_tx_kbps != null or gpu.pcie_rx_kbps != null) { try pcie_lines.append(ctx.arena, .{ .label = "TX ", .value = try formatBandwidth(ctx.arena, gpu.pcie_tx_kbps orelse 0) }); try pcie_lines.append(ctx.arena, .{ .label = "RX ", .value = try formatBandwidth(ctx.arena, gpu.pcie_rx_kbps orelse 0) }); @@ -143,6 +137,16 @@ pub fn draw( }, w, parent_widget); } +fn formatClockPair(arena: std.mem.Allocator, current: ?u64, max: ?u64) std.mem.Allocator.Error![]const u8 { + if (current == null and max == null) return " N/A"; + return std.fmt.allocPrint(arena, " {d} / {d} MHz", .{ current orelse 0, max orelse 0 }); +} + +fn formatPcieLink(arena: std.mem.Allocator, gen: ?u64, width: ?u64) std.mem.Allocator.Error![]const u8 { + if (gen == null and width == null) return " N/A"; + return std.fmt.allocPrint(arena, " Gen{d} x{d}", .{ gen orelse 0, width orelse 0 }); +} + fn formatBandwidth(arena: std.mem.Allocator, kbps: u64) std.mem.Allocator.Error![]const u8 { if (kbps >= 1_000_000) { const gbps = @as(f64, @floatFromInt(kbps)) / 1_000_000.0; diff --git a/bin/zml-smi/tui/top.zig b/bin/zml-smi/tui/top.zig index 0ad41424..9a5b8624 100644 --- a/bin/zml-smi/tui/top.zig +++ b/bin/zml-smi/tui/top.zig @@ -49,6 +49,7 @@ const Model = struct { overview: Overview = undefined, fn deinit(self: *Model) void { + self.process_table.deinit(self.allocator); self.overview.deinit(self.allocator); image_cache.global.deinit(self.allocator); } diff --git a/bin/zml-smi/tui/widgets/device_card.zig b/bin/zml-smi/tui/widgets/device_card.zig index 9d56f1f2..c7f7ac38 100644 --- a/bin/zml-smi/tui/widgets/device_card.zig +++ b/bin/zml-smi/tui/widgets/device_card.zig @@ -57,7 +57,7 @@ pub fn draw(self: *DeviceCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vx const mem_pct: u8 = if (cf.mem_total > 0) @intCast(@min(cf.mem_used * 100 / cf.mem_total, 100)) else 0; const device_label, const util_label = switch (target) { - .cuda, .rocm => .{ "GPU", "GPU" }, + .cuda, .rocm, .oneapi => .{ "GPU", "GPU" }, .neuron => .{ "NC", "Core" }, .tpu => .{ "TPU", "Duty" }, }; @@ -75,14 +75,14 @@ pub fn draw(self: *DeviceCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vx else @as([]const u8, ""); }, - .rocm => |*sv| if (sv.front().driver_version) |v| try std.fmt.allocPrint(ctx.arena, " ({s})", .{v}) else "", + .rocm, .oneapi => |*sv| if (sv.front().driver_version) |v| try std.fmt.allocPrint(ctx.arena, " ({s})", .{v}) else "", .neuron => |*sv| if (sv.front().driver_version) |v| try std.fmt.allocPrint(ctx.arena, " ({s})", .{v}) else "", else => "", }; const card_title = try std.fmt.allocPrint(ctx.arena, "{s} {d}: {s}{s}", .{ device_label, i, cf.name, driver_ver }); const dev_suffix = switch (dev.*) { - .cuda, .rocm => |*sv| blk: { + .cuda, .rocm, .oneapi => |*sv| blk: { const gpu = sv.front().*; const power = (gpu.power_mw orelse 0) / 1000; const power_limit = (gpu.power_limit_mw orelse 0) / 1000; @@ -113,6 +113,7 @@ pub fn draw(self: *DeviceCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vx .title_image = image_cache.global.get(switch (dev.*) { .cuda => "gpu_cuda", .rocm => "gpu_rocm", + .oneapi => "gpu_oneapi", .neuron => "gpu_neuron", .tpu => "gpu_tpu", }), diff --git a/bin/zml-smi/tui/widgets/process_table.zig b/bin/zml-smi/tui/widgets/process_table.zig index 6a2d57aa..58b20d51 100644 --- a/bin/zml-smi/tui/widgets/process_table.zig +++ b/bin/zml-smi/tui/widgets/process_table.zig @@ -26,6 +26,10 @@ scroll_bars: vxfw.ScrollBars = .{ }, merged: std.ArrayList(ProcessInfo) = .empty, +pub fn deinit(self: *ProcessTable, allocator: std.mem.Allocator) void { + self.merged.deinit(allocator); +} + pub fn prepare(self: *ProcessTable, state: *data.SystemState, device_id: ?u16) void { self.scroll_bars.scroll_view.children.builder.userdata = self; self.merged.clearRetainingCapacity(); From 807a9116d4aaac7110fb070b67187e3983a5ce2a Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 1 Jun 2026 10:51:32 +0200 Subject: [PATCH 507/881] zml: make Buffer.from faster (#556) `Buffer.from` is in the hot path everytime we want to send smth to the GPU. Sometimes we can overlap it with GPU compute but not always, and sometimes it's quite complicated to do so. I've rewrote Sharding.Placement to be device agnostic instead of device specific. This allow to reuse most of the logic for multiple devices. According to LLVM MCA (with script included here), we got from: ``` Iterations: 100 Instructions: 324200 Total Cycles: 175191 Total uOps: 369700 ``` to ``` Iterations: 100 Instructions: 162900 Total Cycles: 85585 Total uOps: 182600 ``` So about twice as fast for this part of the code. I would need some help to confirm the numbers on TPU. (Once the PR is approved, I'll remove the mca script as well as the build.zig) --------- Co-authored-by: Hugo Mano --- stdx/bounded_array.zig | 11 +- zml/Sharding.zig | 798 +++++++++++++++++++++-------------------- zml/buffer.zig | 110 +++--- zml/io.zig | 77 +--- zml/mem.zig | 2 +- zml/platform.zig | 145 ++++++-- zml/shape.zig | 34 +- zml/slice.zig | 55 ++- zml/tensor.zig | 40 +++ 9 files changed, 676 insertions(+), 596 deletions(-) diff --git a/stdx/bounded_array.zig b/stdx/bounded_array.zig index 3f00f89e..190f30fc 100644 --- a/stdx/bounded_array.zig +++ b/stdx/bounded_array.zig @@ -49,6 +49,12 @@ pub fn BoundedArrayAligned( pub const empty: Self = .{ .buffer = undefined, .len = 0 }; + pub fn init(m: []const T) Self { + var buf: Self = .{ .buffer = undefined, .len = m.len }; + @memcpy(buf.buffer[0..m.len], m); + return buf; + } + /// View the internal array as a slice whose size was previously set. pub fn slice(self: anytype) switch (@TypeOf(&self.buffer)) { *align(alignment.toByteUnits()) [buffer_capacity]T => []align(alignment.toByteUnits()) T, @@ -78,10 +84,7 @@ pub fn BoundedArrayAligned( /// Copy the content of an existing slice. pub fn fromSlice(m: []const T) error{Overflow}!Self { if (m.len > buffer_capacity) return error.Overflow; - - var list: Self = .{ .buffer = undefined, .len = m.len }; - @memcpy(list.buffer[0..m.len], m); - return list; + return .init(m); } /// Return the element at index `i` of the slice. diff --git a/zml/Sharding.zig b/zml/Sharding.zig index d9746629..89c5b303 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -1,24 +1,25 @@ //! Explain how to split a buffer across different devices. const std = @import("std"); -const c = @import("c"); const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); const platforms = @import("platforms"); const stdx = @import("stdx"); -const Memory = @import("platform.zig").Memory; const Platform = @import("platform.zig").Platform; const PlatformDevice = @import("platform.zig").Device; const Shape = @import("shape.zig").Shape; const Slice = @import("slice.zig").Slice; const Target = @import("platform.zig").Target; +const builtin = @import("builtin"); const Sharding = @This(); data: *const Data, +pub const MAX_MESH_RANK = 4; + var _replicated: [11]u8 align(@alignOf(Data)) = "_replicated".*; // special value to make public apis more fluent. @@ -109,8 +110,7 @@ pub const Partitioning = struct { const sharding = try self.selectSharding(shape); const ordered_devices = sharding.devicesInCanonicalOrder(); if (ordered_devices.len == 0) return error.InvalidPhysicalMesh; - const placement_ = try sharding.placement(shape, ordered_devices[0]); - return placement_.shape.withDefaultPartitioning(); + return sharding.shardedShape(shape); } pub fn numPartitionsForLogicalAxis(self: Partitioning, shape: Shape, logical_axis: anytype) !i64 { @@ -209,27 +209,28 @@ fn shapeHasAxisPartition(shape: Shape) bool { return false; } -/// Device is the leaf representation in a PhysicalMesh. -/// It carries identity, optional coordinates, and PJRT handle. +/// Device as part of a PhysicalMesh. pub const Device = struct { /// Unique device identifier in the mesh - id: usize, + id: u32, /// Coordinates in the physical mesh - coords: stdx.BoundedArray(usize, Shape.MAX_RANK) = .empty, + coords: Coords, - /// Placeholder - pjrt_device: ?*const pjrt.Device = null, + const Coords = [MAX_MESH_RANK]u8; + // Placeholder coords, that would most likely trigger panic if used like that. + // Those are used when creating a mesh, before calling assignCoords + // Private so it's kept as implementation detail + const undefined_coords: Coords = @splat(0xff); - pub fn coordsSlice(self: *const Device) ?[]const usize { - if (self.coords.len == 0) return null; - return self.coords.constSlice(); + fn hasValidCoords(device: Device) bool { + return @as(u32, @bitCast(device.coords)) != @as(u32, @bitCast(undefined_coords)); } - pub fn format(self: Device, writer: *std.Io.Writer) !void { + pub fn format(device: Device, writer: *std.Io.Writer) !void { try writer.print( "Device(id={d} coords={any})", - .{ self.id, self.coords.constSlice() }, + .{ device.id, device.coords }, ); } }; @@ -299,8 +300,7 @@ pub const PhysicalNode = union(enum) { return .{ .leaf = .{ .id = @intCast(device_.id()), - .coords = .empty, - .pjrt_device = device_.pjrt_device, + .coords = Device.undefined_coords, }, }; } @@ -335,7 +335,7 @@ pub const PhysicalMesh = struct { /// AxisTraversal captures canonical axis order and depth mapping. pub const AxisTraversal = struct { - pub const Order = stdx.BoundedArray(PhysicalAxisTag, Shape.MAX_RANK); + pub const Order = stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK); pub const DepthByTag = std.EnumArray(PhysicalAxisTag, ?u8); /// Order of axes (DFS, first-child descent). @@ -349,7 +349,7 @@ pub const PhysicalMesh = struct { axisOrderNode(root, &order); var depth_by_tag: DepthByTag = .initFill(null); - for (order.constSlice(), 0..) |t, i| { + for (order.slice(), 0..) |t, i| { depth_by_tag.set(t, @intCast(i)); } @@ -366,7 +366,7 @@ pub const PhysicalMesh = struct { pub fn format(self: AxisTraversal, writer: *std.Io.Writer) !void { try writer.writeAll("AxisTraversal(order=["); - for (self.order.constSlice(), 0..) |tag, i| { + for (self.order.slice(), 0..) |tag, i| { if (i > 0) try writer.writeAll(", "); try writer.writeAll(@tagName(tag)); } @@ -439,7 +439,7 @@ pub const PhysicalMesh = struct { try validateGeometry(root); var owned_root = root; - var path: [Shape.MAX_RANK]usize = @splat(0); + var path: Device.Coords = @splat(0); assignCoords(&owned_root, &path, 0); var mesh: PhysicalMesh = .{ @@ -449,6 +449,9 @@ pub const PhysicalMesh = struct { .devices_in_canonical_order = undefined, }; try mesh.populateDevicesInCanonicalOrder(allocator); + for (mesh.devices_in_canonical_order) |device| { + std.debug.assert(device.hasValidCoords()); + } return mesh; } @@ -461,10 +464,8 @@ pub const PhysicalMesh = struct { const Order = struct { mesh: *const PhysicalMesh, fn lessThan(ctx: @This(), a: Device, b: Device) bool { - const ca = a.coordsSlice() orelse unreachable; - const cb = b.coordsSlice() orelse unreachable; - const ia = ctx.mesh.linearIndexFromCoords(ca); - const ib = ctx.mesh.linearIndexFromCoords(cb); + const ia = ctx.mesh.linearIndexFromCoords(a.coords); + const ib = ctx.mesh.linearIndexFromCoords(b.coords); return ia < ib; } }; @@ -478,7 +479,6 @@ pub const PhysicalMesh = struct { .leaf = .{ .id = d.id, .coords = d.coords, - .pjrt_device = d.pjrt_device, }, }, .branch => |b| blk: { @@ -562,17 +562,17 @@ pub const PhysicalMesh = struct { /// coords: /// (0,0) (0,1) /// (1,0) (1,1) - fn assignCoords(node: *PhysicalNode, path: *[Shape.MAX_RANK]usize, depth: usize) void { + fn assignCoords(node: *PhysicalNode, path: *Device.Coords, depth: usize) void { switch (node.*) { .leaf => |*d| { // Coords can already been set by caller - if (d.coords.len > 0) return; + if (d.hasValidCoords()) return; - d.coords = stdx.BoundedArray(usize, Shape.MAX_RANK).fromSlice(path[0..depth]) catch unreachable; + d.coords = path.*; }, .branch => |*b| { for (b.children, 0..) |*child, i| { - path[depth] = i; + path[depth] = @intCast(i); assignCoords(child, path, depth + 1); } }, @@ -598,10 +598,10 @@ pub const PhysicalMesh = struct { /// /// sizes: S0..Sk, coords: C0..Ck /// linear = (((C0 * S1) + C1) * S2 + C2) ... - pub fn linearIndexFromCoords(self: PhysicalMesh, coords: []const usize) usize { + pub fn linearIndexFromCoords(self: PhysicalMesh, coords: Device.Coords) usize { const order = self.axisOrder(); var idx: usize = 0; - for (order.constSlice()) |tag| { + for (order.slice()) |tag| { const depth = self.axis_traversal.depth(tag); if (depth) |d| { const coord = coords[@intCast(d)]; @@ -708,53 +708,44 @@ pub const PhysicalMesh = struct { /// CoordsTopology builds a mesh from PJRT `coords` attributes. const CoordsTopology = struct { - const CoordPlacement = struct { - device_id: usize, - pjrt_device: ?*const pjrt.Device, - coords: stdx.BoundedArray(usize, Shape.MAX_RANK), - }; - const CoordLayout = struct { rank: usize, - axis_sizes: [Shape.MAX_RANK]usize, + axis_sizes: [MAX_MESH_RANK]usize, }; const IndexedCoord = struct { - placement: CoordPlacement, + placement: Device, linear: usize, }; - fn parseDevice(device: PlatformDevice) !CoordPlacement { + fn parseDevice(device: PlatformDevice) !Device { const api = device.platform.pjrt_api; const coords_attr = device.pjrt_desc.attribute(api, "coords") orelse return error.MissingDeviceCoords; - var coords: stdx.BoundedArray(usize, Shape.MAX_RANK) = .empty; - switch (coords_attr) { + const coords: Device.Coords = coords: switch (coords_attr) { .int64list => |values| { - if (values.len == 0 or values.len > Shape.MAX_RANK) return error.InvalidDeviceCoords; - for (values) |value| { + var coords: Device.Coords = @splat(0); + if (values.len == 0 or values.len > MAX_MESH_RANK) return error.InvalidDeviceCoords; + for (coords[0..values.len], values) |*coord, value| { if (value < 0) return error.InvalidDeviceCoords; - coords.appendAssumeCapacity(@intCast(value)); + coord.* = @intCast(value); } + break :coords coords; }, else => return error.InvalidDeviceCoords, - } - - return .{ - .device_id = device.id(), - .pjrt_device = device.pjrt_device, - .coords = coords, }; + + return .{ .id = device.id(), .coords = coords }; } - pub fn collect(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) ![]CoordPlacement { - const placements = try allocator.alloc(CoordPlacement, platform_devices.len); + pub fn collect(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) ![]Device { + const placements = try allocator.alloc(Device, platform_devices.len); for (platform_devices, 0..) |device, i| { placements[i] = try parseDevice(device); } return placements; } - pub fn layout(placements: []const CoordPlacement, max_rank: usize) !CoordLayout { + pub fn layout(placements: []const Device, max_rank: usize) !CoordLayout { if (placements.len == 0) return error.InvalidPhysicalMesh; const rank = placements[0].coords.len; @@ -765,11 +756,11 @@ pub const PhysicalMesh = struct { if (coord_placement.coords.len != rank) return error.InvalidDeviceCoordsRank; } - var axis_sizes = [_]usize{1} ** Shape.MAX_RANK; + var axis_sizes = [_]usize{1} ** MAX_MESH_RANK; for (0..rank) |ax_i| { var max_coord: usize = 0; for (placements) |coord_placement| { - max_coord = @max(max_coord, coord_placement.coords.constSlice()[ax_i]); + max_coord = @max(max_coord, coord_placement.coords[ax_i]); } axis_sizes[ax_i] = max_coord + 1; } @@ -780,7 +771,7 @@ pub const PhysicalMesh = struct { }; } - fn linearIndex(coords: []const usize, rank: usize, axis_sizes: []const usize) usize { + fn linearIndex(coords: Device.Coords, rank: usize, axis_sizes: []const usize) usize { var linear: usize = 0; var stride: usize = 1; var ax = rank; @@ -792,12 +783,12 @@ pub const PhysicalMesh = struct { return linear; } - pub fn sorted(allocator: std.mem.Allocator, placements: []const CoordPlacement, rank: usize, axis_sizes: []const usize) ![]IndexedCoord { + pub fn sorted(allocator: std.mem.Allocator, placements: []const Device, rank: usize, axis_sizes: []const usize) ![]IndexedCoord { const indexed = try allocator.alloc(IndexedCoord, placements.len); for (placements, indexed) |coord_placement, *out| { out.* = .{ .placement = coord_placement, - .linear = linearIndex(coord_placement.coords.constSlice(), rank, axis_sizes), + .linear = linearIndex(coord_placement.coords, rank, axis_sizes), }; } @@ -815,18 +806,8 @@ pub const PhysicalMesh = struct { return indexed; } - pub fn leaf(coord_placement: *const CoordPlacement, coords_slice: []const usize) !PhysicalNode { - return .{ - .leaf = .{ - .id = coord_placement.device_id, - .coords = try .fromSlice(coords_slice), - .pjrt_device = coord_placement.pjrt_device, - }, - }; - } - - pub fn axisStrides(axis_sizes: []const usize) ![Shape.MAX_RANK]usize { - var strides: [Shape.MAX_RANK]usize = @splat(0); + pub fn axisStrides(axis_sizes: []const usize) ![MAX_MESH_RANK]usize { + var strides: [MAX_MESH_RANK]usize = @splat(0); var stride: usize = 1; var i = axis_sizes.len; @@ -849,8 +830,8 @@ pub const PhysicalMesh = struct { base_offset: usize, ) !PhysicalNode { if (depth == axis_sizes.len) { - const coord_placement = &indexed[base_offset].placement; - return leaf(coord_placement, coord_placement.coords.constSlice()[0..axis_sizes.len]); + const device = &indexed[base_offset].placement; + return .{ .leaf = device.* }; } const children = try allocator.alloc(PhysicalNode, axis_sizes[depth]); @@ -888,12 +869,23 @@ pub const PhysicalMesh = struct { .cpu => cpu(allocator, platform_devices), .cuda, .rocm => gpu(allocator, platform_devices), .tpu => tpu(allocator, platform_devices), - .neuron => neuron(allocator, platform_devices), + .neuron => return neuron(allocator, platform_devices), .oneapi => cpu(allocator, platform_devices), }; errdefer freeNode(allocator, root); - return try fromOwnedTree(allocator, target, root); + // We check a posteriori that the device id is a valid offset into platform.devices. + // This is generally true because platform.devices is sorted by id and ids are 0-N. + // For platforms where this isn't true like Neuron, + // the platform specific builder need to correctly update the ids so that it's true. + const mesh = try fromOwnedTree(allocator, target, root); + for (mesh.devices_in_canonical_order) |pl| { + const device = platform_devices[pl.id]; + std.debug.assert(pl.hasValidCoords()); + std.debug.assert(device.id() == pl.id); + } + + return mesh; } pub fn cpu(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) !Tree { @@ -914,14 +906,14 @@ pub const PhysicalMesh = struct { const placements = try CoordsTopology.collect(allocator, platform_devices); defer allocator.free(placements); - const layout = try CoordsTopology.layout(placements, Shape.MAX_RANK); + const layout = try CoordsTopology.layout(placements, MAX_MESH_RANK); const indexed = try CoordsTopology.sorted(allocator, placements, layout.rank, layout.axis_sizes[0..layout.rank]); defer allocator.free(indexed); const nodes = try allocator.alloc(PhysicalNode, platform_devices.len); for (nodes, indexed, 0..) |*n, entry, i| { - const linear_coord: [1]usize = .{i}; - n.* = try CoordsTopology.leaf(&entry.placement, linear_coord[0..]); + const linear_coord: Device.Coords = .{ @intCast(i), 0, 0, 0 }; + n.* = .{ .leaf = .{ .id = entry.placement.id, .coords = linear_coord } }; } return .{ @@ -962,7 +954,7 @@ pub const PhysicalMesh = struct { return CoordsTopology.buildMeshNode(allocator, indexed, axis_sizes[0..rank], axis_tags[0..rank], axis_strides[0..rank], 0, 0); } - pub fn neuron(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) !Tree { + pub fn neuron(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) !PhysicalMesh { if (comptime !platforms.isEnabled(.neuron)) { return error.UnsupportedPlatform; } @@ -978,9 +970,9 @@ pub const PhysicalMesh = struct { var topology = try neuron_topology.visibleMeshFromNcIds(allocator, nc_ids); defer topology.deinit(allocator); - var axis_tags: stdx.BoundedArray(PhysicalAxisTag, Shape.MAX_RANK) = .empty; - var axis_sizes: stdx.BoundedArray(usize, Shape.MAX_RANK) = .empty; - var axis_geometries: stdx.BoundedArray(AxisGeometry, Shape.MAX_RANK) = .empty; + var axis_tags: stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK) = .empty; + var axis_sizes: stdx.BoundedArray(usize, MAX_MESH_RANK) = .empty; + var axis_geometries: stdx.BoundedArray(AxisGeometry, MAX_MESH_RANK) = .empty; for (topology.axes()) |mesh_axis| { axis_tags.appendAssumeCapacity(switch (mesh_axis.fabric) { @@ -998,7 +990,7 @@ pub const PhysicalMesh = struct { }); } - var coords_buf: [Shape.MAX_RANK]usize = [_]usize{0} ** Shape.MAX_RANK; + var coords_buf: [MAX_MESH_RANK]usize = [_]usize{0} ** MAX_MESH_RANK; var next_device: usize = 0; const Node = struct { @@ -1010,24 +1002,17 @@ pub const PhysicalMesh = struct { axis_sizes_: []const usize, axis_geometries_: []const AxisGeometry, depth: usize, - coords_buf_: *[Shape.MAX_RANK]usize, + coords_buf_: *[MAX_MESH_RANK]usize, next_device_: *usize, ) !PhysicalNode { if (depth == axis_tags_.len) { const placement_ = placements[next_device_.*]; - const device = platform_devices_[placement_.input_index]; next_device_.* += 1; - var coords: stdx.BoundedArray(usize, Shape.MAX_RANK) = .empty; - for (coords_buf_[0..axis_tags_.len]) |coord| coords.appendAssumeCapacity(coord); + var coords: Device.Coords = @splat(0); + for (coords_buf_[0..axis_tags_.len], 0..) |coord, i| coords[i] = @intCast(coord); - return .{ - .leaf = .{ - .id = placement_.nc_id, - .coords = coords, - .pjrt_device = device.pjrt_device, - }, - }; + return .{ .leaf = .{ .id = @intCast(placement_.nc_id), .coords = coords } }; } const children = try allocator_.alloc(PhysicalNode, axis_sizes_[depth]); @@ -1063,17 +1048,31 @@ pub const PhysicalMesh = struct { } }; - return Node.build( + const root = try Node.build( allocator, platform_devices, topology.placements, - axis_tags.constSlice(), - axis_sizes.constSlice(), - axis_geometries.constSlice(), + axis_tags.slice(), + axis_sizes.slice(), + axis_geometries.slice(), 0, &coords_buf, &next_device, ); + const mesh: PhysicalMesh = try fromOwnedTree(allocator, .neuron, root); + for (mesh.devices_in_canonical_order) |*dev| { + // In Node.build we use nc_id as the Device id. + // But now we want to use the id to get the offset into platform.devices. + const dev_nc_id = dev.id; + for (0.., topology.placements) |order, placement_| { + if (placement_.nc_id == dev_nc_id) { + dev.id = @intCast(order); + std.debug.assert(platform_devices[order].localHardwareId() == dev_nc_id); + break; + } + } + } + return mesh; } }; @@ -1108,8 +1107,8 @@ pub const LogicalAxisIntent = enum { /// Limitations: /// - No sizes here; no divisibility checks here. pub const LogicalMesh = struct { - pub const Axes = stdx.BoundedArray(Shape.Tag, Shape.MAX_RANK); - pub const Intents = stdx.BoundedArray(LogicalAxisIntent, Shape.MAX_RANK); + pub const Axes = stdx.BoundedArray(Shape.Tag, MAX_MESH_RANK); + pub const Intents = stdx.BoundedArray(LogicalAxisIntent, MAX_MESH_RANK); axes: Axes, intents: Intents, @@ -1138,7 +1137,7 @@ pub const LogicalMesh = struct { pub fn intent(self: LogicalMesh, tag: Shape.Tag) ?LogicalAxisIntent { const target = std.mem.span(tag); - for (self.axes.constSlice(), self.intents.constSlice()) |t, i| { + for (self.axes.slice(), self.intents.slice()) |t, i| { if (std.mem.eql(u8, std.mem.span(t), target)) return i; } @@ -1158,46 +1157,46 @@ pub const LogicalMesh = struct { pub fn format(self: LogicalMesh, writer: *std.Io.Writer) !void { try writer.writeAll("LogicalMesh("); - for (self.axes.constSlice(), self.intents.constSlice()) |axis, int| { + for (self.axes.slice(), self.intents.slice()) |axis, int| { try writer.print(" {s}={s}", .{ axis, @tagName(int) }); } try writer.writeAll(")"); } }; -pub const AxisList = stdx.BoundedArray(PhysicalAxisTag, Shape.MAX_RANK); +pub const AxisList = stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK); pub const Binding = struct { logical: Shape.Tag, physical: AxisList, }; -pub const Bindings = stdx.BoundedArray(Binding, Shape.MAX_RANK); +pub const Bindings = stdx.BoundedArray(Binding, MAX_MESH_RANK); pub const Fold = struct { target: PhysicalAxisTag, sources: AxisList, }; -pub const Folds = stdx.BoundedArray(Fold, Shape.MAX_RANK); +pub const Folds = stdx.BoundedArray(Fold, MAX_MESH_RANK); pub const Axis = struct { tag: PhysicalAxisTag, size: i64, geometry: ?AxisGeometry, - folded: stdx.BoundedArray(PhysicalAxisTag, Shape.MAX_RANK), + folded: stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK), pub fn contains(self: *const Axis, tag: PhysicalAxisTag) bool { if (self.tag == tag) return true; - for (self.folded.constSlice()) |t| if (t == tag) return true; + for (self.folded.slice()) |t| if (t == tag) return true; return false; } }; pub const PhysicalView = struct { - axes: stdx.BoundedArray(Axis, Shape.MAX_RANK), + axes: stdx.BoundedArray(Axis, MAX_MESH_RANK), total_devices: i64, pub fn axisCoordFromLinearShard(self: *const PhysicalView, axis_index: usize, linear_idx: usize) usize { const stride = self.axisStrideForLinear(axis_index); - const axis_size: usize = @intCast(self.axes.constSlice()[axis_index].size); + const axis_size: usize = @intCast(self.axes.slice()[axis_index].size); return (linear_idx / stride) % axis_size; } @@ -1205,7 +1204,7 @@ pub const PhysicalView = struct { var stride: usize = 1; var j = axis_index + 1; while (j < self.axes.len) : (j += 1) { - stride *= @intCast(self.axes.constSlice()[j].size); + stride *= @intCast(self.axes.slice()[j].size); } return stride; } @@ -1224,9 +1223,8 @@ pub const Data = struct { folds_consumed: std.EnumSet(PhysicalAxisTag), pub fn binding(self: *const Data, tag: Shape.Tag) ?[]const PhysicalAxisTag { - const target = std.mem.span(tag); - for (self.bindings.constSlice()) |*b| { - if (std.mem.eql(u8, std.mem.span(b.logical), target)) return b.physical.constSlice(); + for (self.bindings.slice()) |*b| { + if (b.logical == tag) return b.physical.slice(); } return null; } @@ -1237,18 +1235,19 @@ pub const Data = struct { logical: LogicalMesh, strategy: Strategy, ) !Data { - const axis_order = physical.axisOrder().constSlice(); + const axis_order = physical.axisOrder().slice(); if (axis_order.len == 0) return error.InvalidPhysicalMesh; + if (strategy.bindings.len == 0) return error.InvalidStrategy; var bindings: Bindings = .empty; - for (strategy.bindings.constSlice()) |bind| { + for (strategy.bindings.slice()) |bind| { bindings.appendAssumeCapacity(.{ .logical = bind.logical, .physical = bind.physical }); } var folds: Folds = .empty; var folds_consumed: std.EnumSet(PhysicalAxisTag) = .empty; - for (strategy.folding.constSlice()) |entry| { - for (entry.sources.constSlice()) |src| { + for (strategy.folding.slice()) |entry| { + for (entry.sources.slice()) |src| { if (!physical.hasAxis(src)) return error.InvalidPhysicalAxis; folds_consumed.insert(src); } @@ -1266,8 +1265,8 @@ pub const Data = struct { } fn foldSources(self: *const Data, tag: PhysicalAxisTag) ?[]const PhysicalAxisTag { - for (self.folds.constSlice()) |*f| { - if (f.target == tag) return f.sources.constSlice(); + for (self.folds.slice()) |*f| { + if (f.target == tag) return f.sources.slice(); } return null; } @@ -1279,13 +1278,13 @@ pub const Data = struct { .total_devices = 1, }; - const axis_order = physical.axisOrder().constSlice(); + const axis_order = physical.axisOrder().slice(); for (axis_order) |tag| { if (!physical.hasAxis(tag)) continue; if (self.folds_consumed.contains(tag) and self.foldSources(tag) == null) continue; - var folded: stdx.BoundedArray(PhysicalAxisTag, Shape.MAX_RANK) = .empty; + var folded: stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK) = .empty; var size: i64 = 1; if (self.foldSources(tag)) |sources| { @@ -1310,7 +1309,7 @@ pub const Data = struct { return view; } - pub fn logicalIndexFromCoords(self: *const Data, coords: []const usize) usize { + pub fn logicalIndexFromCoords(self: *const Data, coords: []const u8) usize { return self.physical.linearIndexFromCoords(coords); } @@ -1349,7 +1348,7 @@ pub const Data = struct { } pub fn covers(sharding: *const Data, shape: Shape) bool { - for (shape._partitioning.constSlice()) |partitioning| { + for (shape._partitioning.slice()) |partitioning| { switch (partitioning) { .axis => |tag| if (sharding.binding(tag) == null) return false, else => {}, @@ -1364,7 +1363,7 @@ pub const Data = struct { const view = self.physicalView(); try out.writer.writeAll("#sdy.mesh<["); - for (view.axes.constSlice(), 0..) |p, i| { + for (view.axes.slice(), 0..) |p, i| { if (i > 0) try out.writer.writeAll(", "); try out.writer.print("\"{s}\"={d}", .{ @tagName(p.tag), p.size }); } @@ -1395,7 +1394,7 @@ pub const Data = struct { if (spec == .axis) { if (self.binding(spec.axis)) |binding_| { for (binding_) |p_tag| { - for (view.axes.constSlice(), 0..) |v_ax, i| { + for (view.axes.slice(), 0..) |v_ax, i| { // Only use the axis if it's bound and hasn't been consumed by a previous dimension if (v_ax.contains(p_tag) and !globally_used.contains(v_ax.tag)) { dim_axes.appendAssumeCapacity(i); @@ -1452,7 +1451,7 @@ pub const Data = struct { break :d .replicated(ctx); } else { const axes = try allocator.alloc(*const dialects.shardy.AxisRefAttribute, dim_phys_indices.len); - for (dim_phys_indices.constSlice(), 0..) |p_idx, i| { + for (dim_phys_indices.slice(), 0..) |p_idx, i| { axes[i] = .named(ctx, @tagName(mapping.view.axes.get(p_idx).tag)); } break :d .closed(ctx, axes); @@ -1467,7 +1466,7 @@ pub const Data = struct { } const replicated_axes = try allocator.alloc(*const dialects.shardy.AxisRefAttribute, mapping.replicated_axes.len); - for (replicated_axes, mapping.replicated_axes.constSlice()) |*r, p_idx| { + for (replicated_axes, mapping.replicated_axes.slice()) |*r, p_idx| { r.* = .named(ctx, @tagName(mapping.view.axes.get(p_idx).tag)); } @@ -1492,9 +1491,9 @@ pub const Data = struct { var tile_shape: stdx.BoundedArray(i64, Shape.MAX_RANK + 1) = .empty; // Calculate tile sizes per tensor dimension - for (mapping.axes_per_dim.constSlice()) |dim_axes| { + for (mapping.axes_per_dim.slice()) |dim_axes| { var combined: i64 = 1; - for (dim_axes.constSlice()) |idx| { + for (dim_axes.slice()) |idx| { combined *= mapping.view.axes.get(idx).size; } tile_shape.appendAssumeCapacity(combined); @@ -1504,7 +1503,7 @@ pub const Data = struct { const has_replication = mapping.replicated_axes.len > 0; if (has_replication) { var repl_size: i64 = 1; - for (mapping.replicated_axes.constSlice()) |idx| { + for (mapping.replicated_axes.slice()) |idx| { repl_size *= mapping.view.axes.get(idx).size; } tile_shape.appendAssumeCapacity(repl_size); @@ -1519,7 +1518,7 @@ pub const Data = struct { defer allocator.free(ids); try out.writer.writeAll("{devices=["); - for (tile_shape.constSlice(), 0..) |s, i| { + for (tile_shape.slice(), 0..) |s, i| { if (i > 0) try out.writer.writeAll(","); try out.writer.print("{d}", .{s}); } @@ -1550,7 +1549,7 @@ pub const Data = struct { @memset(ids, std.math.maxInt(usize)); for (self.physical.devices_in_canonical_order) |d| { - const coords = d.coordsSlice() orelse return error.MissingDeviceCoords; + const coords = d.coords; const idx = self.physical.linearIndexFromCoords(coords); ids[idx] = d.id; } @@ -1566,7 +1565,7 @@ pub const Data = struct { try writer.print("Sharding(name={s})\n", .{self.name}); try writer.writeAll("Bindings:\n"); - for (self.logical.axes.constSlice(), self.logical.intents.constSlice()) |l_tag, l_intent| { + for (self.logical.axes.slice(), self.logical.intents.slice()) |l_tag, l_intent| { try writer.print(" - {s} ({s}) -> ", .{ l_tag, @tagName(l_intent) }); if (self.binding(l_tag)) |axes| { @@ -1592,7 +1591,7 @@ pub const Data = struct { if (view.axes.len == 0) { try writer.writeAll("(none)\n"); } else { - for (view.axes.constSlice(), 0..) |axis, i| { + for (view.axes.slice(), 0..) |axis, i| { if (i > 0) try writer.writeAll(" × "); try writer.print("{s}[{d}]", .{ @tagName(axis.tag), axis.size }); } @@ -1603,7 +1602,7 @@ pub const Data = struct { if (view.axes.len == 0) { try writer.writeAll("(none)\n"); } else { - for (view.axes.constSlice(), 0..) |axis, i| { + for (view.axes.slice(), 0..) |axis, i| { if (i > 0) try writer.writeAll(", "); try writer.print("{s}={s}", .{ @tagName(axis.tag), @@ -1616,25 +1615,44 @@ pub const Data = struct { }; pub const Strategy = struct { - pub const PhysicalList = stdx.BoundedArray(PhysicalAxisTag, Shape.MAX_RANK); + pub const PhysicalList = stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK); pub const Binding = struct { logical: Shape.Tag, physical: PhysicalList, }; - pub const Bindings = stdx.BoundedArray(Strategy.Binding, Shape.MAX_RANK); + pub const Bindings = stdx.BoundedArray(Strategy.Binding, MAX_MESH_RANK); pub const Fold = struct { target: PhysicalAxisTag, sources: PhysicalList, }; - pub const Folding = stdx.BoundedArray(Strategy.Fold, Shape.MAX_RANK); + pub const Folding = stdx.BoundedArray(Strategy.Fold, MAX_MESH_RANK); bindings: Strategy.Bindings, folding: Folding, - pub const init: Strategy = .{ - .bindings = .empty, - .folding = .empty, - }; + // TODO: remove .init, promote .parseBindings to .init + // It's currently an error to call .init and never calling `addBinding` + pub const init: Strategy = .{ .bindings = .empty, .folding = .empty }; + + pub fn parseBindings(bindings: anytype) Strategy { + const err_msg = "Strategy.parseBindings excepts fields to be PhysicalAxisTag or tuple of PhysicalAxisTag, got {}"; + var res: Strategy = .{ .bindings = .empty, .folding = .empty }; + const fields = @typeInfo(@TypeOf(bindings)).@"struct".fields; + if (fields.len == 0) @compileError("Strategy.parseBindings requires at least one binding"); + inline for (fields) |field_info| { + switch (@typeInfo(field_info.type)) { + .enum_literal, .@"enum" => res.addBinding(field_info, @field(bindings, field_info.name)), + .@"struct" => |struct_info| { + stdx.debug.assertComptime(struct_info.is_tuple, err_msg, .{@TypeOf(bindings)}); + inline for (@field(bindings, field_info.name)) |axis_tag| { + res.addBinding(field_info, axis_tag); + } + }, + else => stdx.debug.compileError(err_msg, .{@TypeOf(bindings)}), + } + } + return res; + } pub fn addBinding(self: *Strategy, logical: anytype, physical: PhysicalAxisTag) void { const raw_tag = Shape.toTag(logical); @@ -1680,23 +1698,23 @@ pub const Strategy = struct { /// suggest builds a Strategy from logical intents. pub fn suggest(logical: LogicalMesh, physical: *const PhysicalMesh) Strategy { - var strategy: Strategy = .init; + var strategy: Strategy = .{ .bindings = .empty, .folding = .empty }; const base_order = physical.shardableAxes(); - var available: stdx.BoundedArray(PhysicalAxisTag, Shape.MAX_RANK) = .empty; + var available: stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK) = .empty; for (base_order) |tag| { if (physical.hasAxis(tag)) available.appendAssumeCapacity(tag); } if (available.len == 0) { - for (physical.axisOrder().constSlice()) |tag| available.appendAssumeCapacity(tag); + for (physical.axisOrder().slice()) |tag| available.appendAssumeCapacity(tag); } - const avail = available.constSlice(); + const avail = available.slice(); var counters = std.EnumArray(LogicalAxisIntent, usize).initFill(0); inline for (std.meta.tags(LogicalAxisIntent)) |target_intent| { - for (logical.axes.constSlice(), logical.intents.constSlice()) |l_tag, l_intent| { + for (logical.axes.slice(), logical.intents.slice()) |l_tag, l_intent| { if (l_intent != target_intent) continue; const i = counters.get(target_intent); @@ -1715,129 +1733,173 @@ pub const Strategy = struct { } }; -pub fn placement(sharding: Sharding, shape: Shape, device: Device) !Placement { - if (device.coords.len == 0) return error.MissingDeviceCoords; - - var placement_: Placement = .{ - .device_id = device.id, - .device_coords = device.coords, - .shape = undefined, - .global_shape = shape, - .slices = .empty, - }; - var used_axes: std.EnumSet(PhysicalAxisTag) = .empty; - - for (0..shape.rank()) |ax| { - const axis_index: u8 = @intCast(ax); - const slice = try sliceForDevice(sharding, shape, placement_.device_coords.constSlice(), &used_axes, axis_index); - placement_.slices.appendAssumeCapacity(slice); - } - - placement_.shape = blk: { - var sh = shape; - for (placement_.slices.constSlice()) |slice| { - sh = sh.set(slice.axis, slice.size); - } - break :blk sh; - }; - - return placement_; +/// For a given shape, compute the shape of the slice each shard will receive. +pub fn shardedShape(sharding: Sharding, shape: Shape) !Shape { + const pl = try sharding.placement(shape); + return pl.shape; } +pub fn placement(sharding: Sharding, shape: Shape) !Placement { + return .init(sharding, shape); +} + +/// Precompute sharding information common to all devices for a given shape. pub const Placement = struct { + pub const Slices = stdx.BoundedArray(Slice1d, Shape.MAX_RANK); pub const Slice1d = struct { - axis: u8, start: i64, size: i64, }; - device_id: usize, - device_coords: stdx.BoundedArray(usize, Shape.MAX_RANK), + sharding: Sharding, + // Note this shape is mainly use to indicate the dims of the sharded buffer. + // We may want to only store that. shape: Shape, - global_shape: Shape, - slices: stdx.BoundedArray(Slice1d, Shape.MAX_RANK), + global_shape: if (builtin.mode == .Debug) Shape else void, - pub fn platformDeviceIndex(self: *const Placement, platform: *const Platform) ?usize { - for (platform.devices, 0..) |d, device_index| { - const device_id = switch (platform.target) { - .neuron => @as(usize, @intCast(d.localHardwareId())), - .cuda, .rocm, .tpu, .cpu, .oneapi => d.id(), - }; - if (device_id == self.device_id) return device_index; + axis_plans: stdx.BoundedArray(AxisSplit, Shape.MAX_RANK), + + pub fn init(sharding: Sharding, shape: Shape) !Placement { + var pl: Placement = .{ + .sharding = sharding, + .shape = shape, // modified below + .global_shape = if (builtin.mode == .Debug) shape else {}, + .axis_plans = .empty, // set below + }; + var used_axes: std.EnumSet(PhysicalAxisTag) = .empty; + + for (0..shape.rank()) |ax| { + const axis_index: u8 = @intCast(ax); + pl.axis_plans.appendAssumeCapacity(try axisSplit(sharding, shape, &used_axes, axis_index)); } - return null; + for (pl.shape._dims.slice(), shape.dims(), pl.axis_plans.slice()) |*shard_dim, dim, plan| { + shard_dim.* = @divExact(dim, plan.num_devices); + } + return pl; } - pub fn device(self: *const Placement, platform: *const Platform) PlatformDevice { - const device_index = self.platformDeviceIndex(platform) orelse unreachable; - return platform.devices[device_index]; - } - - pub fn memory(self: *const Placement, platform: *const Platform, kind: Memory.Kind) *const Memory { - return self.device(platform).memory(kind); - } - - pub fn shardSlice(self: *const Placement, slice: Slice) Slice { - stdx.debug.assert( - self.global_shape.eql(slice.shape), - "Placement global shape {f} doesn't match slice shape {f}", - .{ self.global_shape, slice.shape }, - ); - - var res = slice; - for (self.slices.constSlice()) |s| { - res = res.subSlice(s.axis, s.start, s.size); + pub fn slices(pl: *const Placement, device: Device.Coords) Slices { + var res: Slices = .{ .buffer = undefined, .len = pl.shape.rank() }; + for (res.slice(), pl.shape.dims(), pl.axis_plans.slice()) |*r, size, plan| { + const start = plan.linearIndex(device) * size; + r.* = .{ + .start = start, + .size = size, + }; } return res; } - pub fn format(self: Placement, writer: *std.Io.Writer) !void { - try writer.print("device_id={d} coords={any}", .{ self.device_id, self.device_coords.constSlice() }); - - try writer.writeAll(" slices=["); - for (self.slices.constSlice(), 0..) |s, i| { - if (i > 0) try writer.writeAll(", "); - const axis_label = self.global_shape.debugTag(s.axis); - try writer.print("{s}:[{d}:{d}]", .{ axis_label, s.start, s.start + s.size }); + pub fn shardPtr(pl: *const Placement, device: Device.Coords, slice: Slice) [*]const u8 { + if (builtin.mode == .Debug) { + // there is a bug in caller code that used a placement for a different shape + std.debug.assert(pl.global_shape.eql(slice.shape)); + } + + var ptr: [*]const u8 = slice.constData().ptr; + for (pl.shape.dims(), pl.axis_plans.slice(), slice.byte_strides.slice()) |size, plan, stride| { + const start = plan.linearIndex(device) * size; + ptr = ptr[@intCast(start * stride)..]; + } + return ptr; + } + + /// Given a Slice, returns the subslice corresponding to a specific device. + /// Depending on the layout, the sub-slice may or may not be contiguous. + pub fn shardSlice(pl: *const Placement, device: Device.Coords, slice: Slice) Slice { + const shard_ptr = pl.shardPtr(device, slice); + const offset_bytes = @intFromPtr(shard_ptr) - @intFromPtr(slice.bytes.ptr); + return .{ + .bytes = slice.bytes, + .mutable = slice.mutable, + .shape = pl.shape, + .offset_bytes = offset_bytes, + .byte_strides = slice.byte_strides, + }; + } + + pub fn format(pl: Placement, writer: *std.Io.Writer) std.Io.Writer.Error!void { + const ordered_devices = pl.sharding.devicesInCanonicalOrder(); + + try pl.formatPlacementSummary(pl.shape, ordered_devices.len, writer); + try writer.writeAll("Per-shard placement:\n"); + for (ordered_devices, 0..) |device, shard_index| { + try writer.print("└─ Shard[{d}] device={d} coords={any}, slices=[", .{ shard_index, device.id, device.coords }); + for (0.., pl.slices(device.coords).slice()) |ax, s| { + if (ax > 0) try writer.writeAll(", "); + const axis_label = pl.shape.debugTag(ax); + try writer.print("{s}:[{d}:{d}]", .{ axis_label, s.start, s.start + s.size }); + } + try writer.writeAll("]\n"); + } + + for (0.., pl.axis_plans.slice()) |ax, plan| { + try writer.print("- axis{d} -> plan {f}\n", .{ ax, plan }); + } + } + + fn formatPlacementSummary(pl: Placement, shape: Shape, shard_count: usize, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("Placement(shape={f} shards={d})\n", .{ shape, shard_count }); + + try writer.writeAll("Axis partitioning summary:\n"); + for (0.., pl.slices(@splat(0)).slice()) |ax, slice| { + const dim = shape.dim(ax); + const spec = shape.partition(ax); + const axis_label = shape.debugTag(ax); + + const shard_size: i64 = slice.size; + const shards_count: i64 = if (shard_size > 0 and @rem(dim, shard_size) == 0) + @divExact(dim, shard_size) + else + 0; + + try writer.print( + "└─ {s}: dim={d}, spec={s}, shard_size={d}, shards={d}\n", + .{ axis_label, dim, @tagName(spec), shard_size, shards_count }, + ); } - try writer.writeAll("]"); } }; const AxisSplit = struct { - product: i64, - counts: stdx.BoundedArray(i64, Shape.MAX_RANK), - indices: stdx.BoundedArray(i64, Shape.MAX_RANK), + /// For each physical coordinate depth, the contribution of that coordinate + /// to the linear shard index. Unused depths have stride 0. + coord_strides: Device.Coords, + counts: Device.Coords, + num_devices: u32, pub const empty: AxisSplit = .{ - .product = 1, - .counts = .empty, - .indices = .empty, + .coord_strides = @splat(0), + .counts = @splat(0), + .num_devices = 1, }; - pub fn add(split: *AxisSplit, size: i64, coord: usize) void { - split.counts.appendAssumeCapacity(size); - split.indices.appendAssumeCapacity(@intCast(coord)); - split.product *= size; + pub fn add(split: *AxisSplit, size: u8, depth: u8) void { + for (&split.coord_strides) |*stride| stride.* *= size; + split.coord_strides[depth] = 1; + split.counts[depth] = size; + split.num_devices *= size; } - pub fn linearIndex(self: AxisSplit) i64 { - var linear: i64 = 0; - for (self.counts.constSlice(), self.indices.constSlice()) |c_, iidx| { - linear = linear * c_ + iidx; - } - return linear; + pub fn linearIndex(split: AxisSplit, device_coords: Device.Coords) u32 { + @setRuntimeSafety(false); + const coords_u8: @Vector(MAX_MESH_RANK, u8) = device_coords; + const strides: @Vector(MAX_MESH_RANK, u8) = split.coord_strides; + return @reduce(.Add, coords_u8 * strides); + } + + pub fn format(split: AxisSplit, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("Plan(counts={any},strides={any})", .{ split.counts, split.coord_strides }); } }; -fn sliceForDevice( +fn axisSplit( sharding: Sharding, shape: Shape, - device_coords: []const usize, used_axes: *std.EnumSet(PhysicalAxisTag), axis_index: u8, -) !Placement.Slice1d { +) !AxisSplit { const dim = shape.dim(axis_index); const spec = shape.partition(axis_index); @@ -1846,18 +1908,9 @@ fn sliceForDevice( const binding = sharding.data.binding(logical_tag) orelse return error.MissingLogicalBinding; // Calculate the split based on the physical coordinates of the current device. - const plan = try calculateSplit(sharding, dim, binding, device_coords, used_axes); - - const size = @divExact(dim, plan.product); - const start = plan.linearIndex() * size; - - return .{ - .axis = axis_index, - .start = start, - .size = size, - }; + return try calculateSplit(sharding, dim, binding, used_axes); }, - else => return .{ .axis = axis_index, .start = 0, .size = dim }, + else => return .empty, } } @@ -1865,7 +1918,6 @@ fn calculateSplit( sharding: Sharding, dim: i64, binding: []const PhysicalAxisTag, - device_coords: []const usize, used_axes: *std.EnumSet(PhysicalAxisTag), ) !AxisSplit { var plan: AxisSplit = .empty; @@ -1875,103 +1927,34 @@ fn calculateSplit( if (sharding.data.foldSources(p_tag)) |sources| { for (sources) |src| { if (used_axes.contains(src)) continue; - addPhysicalToSplit(sharding, &plan, src, device_coords); + addPhysicalToSplit(sharding, &plan, src); used_axes.insert(src); } } else { // Standard case: directly bound, not part of an explicit fold. if (used_axes.contains(p_tag)) continue; - addPhysicalToSplit(sharding, &plan, p_tag, device_coords); + addPhysicalToSplit(sharding, &plan, p_tag); used_axes.insert(p_tag); } } - if (plan.product > 0 and @rem(dim, plan.product) != 0) { + if (plan.num_devices > 0 and @rem(dim, plan.num_devices) != 0) { return error.IncompatibleSharding; } return plan; } /// Extract coordinate data from the physical mesh for a specific axis. -fn addPhysicalToSplit(sharding: Sharding, plan: *AxisSplit, tag: PhysicalAxisTag, device_coords: []const usize) void { +fn addPhysicalToSplit(sharding: Sharding, plan: *AxisSplit, tag: PhysicalAxisTag) void { const physical = sharding.data.physical; const info = physical.axisInfo(tag) orelse return; const depth = physical.axis_traversal.depth(tag) orelse return; - const coord = device_coords[depth]; - - plan.add(info.size, coord); -} - -pub fn fmtPlacements(sharding: Sharding, shape: Shape) PlacementsFormatter { - return .{ .sharding = sharding, .shape = shape }; -} - -pub const PlacementsFormatter = struct { - sharding: Sharding, - shape: Shape, - - pub fn format(self: PlacementsFormatter, writer: *std.Io.Writer) std.Io.Writer.Error!void { - const ordered_devices = self.sharding.devicesInCanonicalOrder(); - const first_shard: ?Placement = if (ordered_devices.len > 0) - self.sharding.placement(self.shape, ordered_devices[0]) catch |err| return formatPlacementError(self.shape, ordered_devices.len, err, writer) - else - null; - - try formatPlacementSummary(self.shape, ordered_devices.len, first_shard, writer); - try writer.writeAll("Per-shard placement:\n"); - for (ordered_devices, 0..) |device, shard_index| { - const placement_ = self.sharding.placement(self.shape, device) catch |err| { - try writer.print("└─ Shard[{d}] error={s}\n", .{ shard_index, @errorName(err) }); - return; - }; - try writer.print("└─ Shard[{d}] {f}\n", .{ shard_index, placement_ }); - } - } -}; - -fn formatPlacementSummary(shape: Shape, shard_count: usize, first_shard: ?Placement, writer: *std.Io.Writer) std.Io.Writer.Error!void { - try writer.print("Placement(shape={f} shards={d})\n", .{ shape, shard_count }); - - try writer.writeAll("Axis partitioning summary:\n"); - for (0..shape.rank()) |ax| { - const dim = shape.dim(ax); - const spec = shape.partition(ax); - const axis_label = shape.debugTag(ax); - - var shard_size: i64 = dim; - if (first_shard) |shard| { - for (shard.slices.constSlice()) |s| { - if (s.axis == ax) { - shard_size = s.size; - break; - } - } - } - - const shards_count: i64 = if (shard_size > 0 and @rem(dim, shard_size) == 0) - @divExact(dim, shard_size) - else - 0; - - try writer.print( - "└─ {s}: dim={d}, spec={s}, shard_size={d}, shards={d}\n", - .{ axis_label, dim, @tagName(spec), shard_size, shards_count }, - ); - } -} - -fn formatPlacementError( - shape: Shape, - shard_count: usize, - err: anyerror, - writer: *std.Io.Writer, -) std.Io.Writer.Error!void { - try writer.print("Placement(shape={f} shards={d})\n", .{ shape, shard_count }); - try writer.print("Shard placement error: {s}\n", .{@errorName(err)}); + plan.add(@intCast(info.size), depth); } const ShardingTest = struct { pub const ExpectedShard = struct { + // TODO: remove device_id from here, we always have ids from 0 to N device_id: usize, slices: []const [2]i64, }; @@ -1980,10 +1963,19 @@ const ShardingTest = struct { sharding: Sharding.Data, shape: Shape, + expect_error: ?anyerror = null, expected_sdy: ?[]const u8 = null, expected_shards: []const ExpectedShard = &.{}, - expect_error: ?anyerror = null, + pub fn format(scenario: Scenario, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("Scenario:\n", .{}); + if (scenario.expect_error) |err| try writer.print("- error {}\n", .{err}); + if (scenario.expected_sdy) |sdy| try writer.print("- shardy: {s}\n", .{sdy}); + if (scenario.expected_shards.len > 0) try writer.print("- shards:\n", .{}); + for (scenario.expected_shards) |shard| { + try writer.print("\t- device {d}: slices={any}\n", .{ shard.device_id, shard.slices }); + } + } }; allocator: std.mem.Allocator, @@ -2002,16 +1994,16 @@ const ShardingTest = struct { } const tags: [3]PhysicalAxisTag = .{ .link_x, .link_y, .link_z }; - var next_id: usize = 0; + var next_id: u32 = 0; const root = try self.buildNode(tags[0..N], sizes[0..N], 0, &next_id, geometry); return try .fromTree(self.allocator, .tpu, root); } - fn buildNode(self: ShardingTest, tags: []const PhysicalAxisTag, sizes: []const usize, depth: usize, next_id: *usize, geometry: AxisGeometry) !PhysicalNode { + fn buildNode(self: ShardingTest, tags: []const PhysicalAxisTag, sizes: []const usize, depth: usize, next_id: *u32, geometry: AxisGeometry) !PhysicalNode { if (depth == tags.len) { const id = next_id.*; next_id.* += 1; - return .{ .leaf = .{ .id = id } }; + return .{ .leaf = .{ .id = id, .coords = Device.undefined_coords } }; } const count = sizes[depth]; const children = try self.allocator.alloc(PhysicalNode, count); @@ -2023,9 +2015,11 @@ const ShardingTest = struct { pub fn run(self: ShardingTest, s: Scenario) !void { const sharding = s.sharding; + errdefer std.log.warn("Failed sharding {f}", .{s}); // Verify MLIR String if (s.expected_sdy) |expected_attr| { + errdefer std.log.warn("Expected shardy annotation failed", .{}); const registry: *mlir.DialectRegistry = try .init(); defer registry.deinit(); registry.registerDialect("sdy"); @@ -2046,20 +2040,42 @@ const ShardingTest = struct { const sharding_ref: Sharding = .{ .data = &sharding }; const ordered_devices = sharding_ref.devicesInCanonicalOrder(); if (s.expect_error) |err| { + errdefer std.log.warn("Expected error failed", .{}); try std.testing.expect(ordered_devices.len > 0); - try std.testing.expectError(err, sharding_ref.placement(s.shape, ordered_devices[0])); + try std.testing.expectError(err, sharding_ref.placement(s.shape)); return; } // Verify Shard Slices (Math) if (s.expected_shards.len > 0) { + const pl = try sharding_ref.placement(s.shape); + errdefer std.log.warn("Expected shards failed. Got {f}", .{pl}); try std.testing.expectEqual(s.expected_shards.len, ordered_devices.len); for (s.expected_shards, ordered_devices) |expected, device| { - const actual = try sharding_ref.placement(s.shape, device); - try std.testing.expectEqual(expected.device_id, actual.device_id); - for (expected.slices, actual.slices.constSlice()) |exp_slice, actual_slice| { - try std.testing.expectEqual(exp_slice[0], actual_slice.start); - try std.testing.expectEqual(exp_slice[1], actual_slice.size); + { + try std.testing.expectEqual(expected.device_id, device.id); + + // Check pl.slices(device) + const actual_slices = pl.slices(device.coords); + + // Consider rewriting test cases to use Slice1D. This requires passing the axis + const actual_start_len = try self.allocator.alloc([2]i64, actual_slices.len); + defer self.allocator.free(actual_start_len); + for (actual_start_len, actual_slices.slice()) |*s_l, slice| { + s_l.* = .{ slice.start, slice.size }; + } + + try std.testing.expectEqualSlices([2]i64, expected.slices, actual_start_len); + } + { + // Check pl.shape + const expected_sizes = try self.allocator.alloc(i64, expected.slices.len); + defer self.allocator.free(expected_sizes); + for (expected_sizes, expected.slices) |*expected_size, slice| { + expected_size.* = slice[1]; + } + + try std.testing.expectEqualSlices(i64, expected_sizes, pl.shape.dims()); } } } @@ -2074,16 +2090,11 @@ test "sharding: unknown partitioning implies replication" { const physical: PhysicalMesh = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); const logical: LogicalMesh = .mesh(.{ .batch = .low_bandwidth }); - var strategy: Strategy = .init; - strategy.addBinding(.batch, .link_x); + const strategy: Strategy = .parseBindings(.{ .batch = .link_x }); - const scenario: ShardingTest.Scenario = .{ - .sharding = try .init( - "folded_mesh", - &physical, - logical, - strategy, - ), + try runner.run(.{ + .sharding = try .init("folded_mesh", &physical, logical, strategy), + // No partitioning on shape .shape = Shape.init(.{ .batch = 8 }, .f32), .expected_sdy = "#sdy.sharding<@folded_mesh, [{}], replicated={\"link_x\", \"link_y\"}>", .expected_shards = &.{ @@ -2092,8 +2103,7 @@ test "sharding: unknown partitioning implies replication" { .{ .device_id = 2, .slices = &.{.{ 0, 8 }} }, .{ .device_id = 3, .slices = &.{.{ 0, 8 }} }, }, - }; - try runner.run(scenario); + }); } test "sharding: suggest strategy realization" { @@ -2110,7 +2120,7 @@ test "sharding: suggest strategy realization" { const strategy: Strategy = .suggest(logical, &physical); - const scenario: ShardingTest.Scenario = .{ + try runner.run(.{ .sharding = try .init("suggested_mesh", &physical, logical, strategy), .shape = Shape.init(.{ .batch = 4, .model = 4 }, .f32) .withPartitioning(.{ .batch = .batch, .model = .model }), @@ -2125,8 +2135,7 @@ test "sharding: suggest strategy realization" { .{ .device_id = 6, .slices = &.{ .{ 0, 2 }, .{ 2, 2 } } }, .{ .device_id = 7, .slices = &.{ .{ 2, 2 }, .{ 2, 2 } } }, }, - }; - try runner.run(scenario); + }); } test "sharding: suggest folds logical axes with same intent" { @@ -2143,7 +2152,7 @@ test "sharding: suggest folds logical axes with same intent" { const strategy: Strategy = .suggest(logical, &physical); - const scenario: ShardingTest.Scenario = .{ + try runner.run(.{ .sharding = try .init("fold_mesh", &physical, logical, strategy), .shape = Shape.init(.{ .model = 4, .experts = 4 }, .f32) .withPartitioning(.{ .model = .model, .experts = .experts }), @@ -2154,8 +2163,7 @@ test "sharding: suggest folds logical axes with same intent" { .{ .device_id = 2, .slices = &.{ .{ 2, 1 }, .{ 0, 4 } } }, .{ .device_id = 3, .slices = &.{ .{ 3, 1 }, .{ 0, 4 } } }, }, - }; - try runner.run(scenario); + }); } test "sharding: multiple physical axes on one logical dimension (folding)" { @@ -2166,22 +2174,37 @@ test "sharding: multiple physical axes on one logical dimension (folding)" { const physical: PhysicalMesh = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); const logical: LogicalMesh = .mesh(.{ .model = .high_bandwidth }); - var strategy: Strategy = .init; - strategy.addBinding(.model, .link_x); - strategy.addBinding(.model, .link_y); + { + const strategy: Strategy = .parseBindings(.{ .model = .{ .link_x, .link_y } }); + try runner.run(.{ + .sharding = try .init("folded_mesh", &physical, logical, strategy), + .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(.{ .model = .model }), + .expected_sdy = "#sdy.sharding<@folded_mesh, [{\"link_x\", \"link_y\"}]>", + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, + .{ .device_id = 1, .slices = &.{.{ 4, 4 }} }, + .{ .device_id = 2, .slices = &.{.{ 8, 4 }} }, + .{ .device_id = 3, .slices = &.{.{ 12, 4 }} }, + }, + }); + } - const scenario: ShardingTest.Scenario = .{ - .sharding = try .init("folded_mesh", &physical, logical, strategy), - .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(.{ .model = .model }), - .expected_sdy = "#sdy.sharding<@folded_mesh, [{\"link_x\", \"link_y\"}]>", - .expected_shards = &.{ - .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, - .{ .device_id = 1, .slices = &.{.{ 4, 4 }} }, - .{ .device_id = 2, .slices = &.{.{ 8, 4 }} }, - .{ .device_id = 3, .slices = &.{.{ 12, 4 }} }, - }, - }; - try runner.run(scenario); + { + // Swap link_x and link_y order: + // -> the roles of devices 1 (0,1) and 2 (1,0) are swapped + const strategy: Strategy = .parseBindings(.{ .model = .{ .link_y, .link_x } }); + try runner.run(.{ + .sharding = try .init("folded_mesh", &physical, logical, strategy), + .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(.{ .model = .model }), + .expected_sdy = "#sdy.sharding<@folded_mesh, [{\"link_y\", \"link_x\"}]>", + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, + .{ .device_id = 1, .slices = &.{.{ 8, 4 }} }, + .{ .device_id = 2, .slices = &.{.{ 4, 4 }} }, + .{ .device_id = 3, .slices = &.{.{ 12, 4 }} }, + }, + }); + } } test "sharding: explicit strategy folding" { @@ -2192,11 +2215,10 @@ test "sharding: explicit strategy folding" { const physical: PhysicalMesh = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); const logical: LogicalMesh = .mesh(.{ .model = .high_bandwidth }); - var strategy: Strategy = .init; - strategy.addBinding(.model, .link_x); + var strategy: Strategy = .parseBindings(.{ .model = .link_x }); strategy.addFold(.link_x, &.{ .link_x, .link_z }); - const scenario: ShardingTest.Scenario = .{ + try runner.run(.{ .sharding = try .init("strategy_fold", &physical, logical, strategy), .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(.{ .model = .model }), .expected_sdy = "#sdy.sharding<@strategy_fold, [{\"link_x\"}], replicated={\"link_y\"}>", @@ -2210,8 +2232,7 @@ test "sharding: explicit strategy folding" { .{ .device_id = 6, .slices = &.{.{ 8, 4 }} }, .{ .device_id = 7, .slices = &.{.{ 12, 4 }} }, }, - }; - try runner.run(scenario); + }); } test "sharding: open and replicated dimension mix" { @@ -2222,11 +2243,9 @@ test "sharding: open and replicated dimension mix" { const physical: PhysicalMesh = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); const logical: LogicalMesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth }); - var strategy: Strategy = .init; - strategy.addBinding(.batch, .link_x); - strategy.addBinding(.model, .link_y); + const strategy: Strategy = .parseBindings(.{ .batch = .link_x, .model = .link_y }); - const scenario: ShardingTest.Scenario = .{ + try runner.run(.{ .sharding = try .init("mix_mesh", &physical, logical, strategy), .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(.{ .batch = .open, .model = .replicated }), .expected_sdy = "#sdy.sharding<@mix_mesh, [{?}, {}], replicated={\"link_x\", \"link_y\"}>", @@ -2236,8 +2255,7 @@ test "sharding: open and replicated dimension mix" { .{ .device_id = 2, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, .{ .device_id = 3, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, }, - }; - try runner.run(scenario); + }); } test "sharding: full 3D cluster sharding" { @@ -2248,28 +2266,29 @@ test "sharding: full 3D cluster sharding" { const physical: PhysicalMesh = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); const logical: LogicalMesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth, .context = .balanced }); - var strategy: Strategy = .init; - strategy.addBinding(.batch, .link_x); - strategy.addBinding(.model, .link_y); - strategy.addBinding(.context, .link_z); + const strategy: Strategy = .parseBindings(.{ + .batch = .link_x, + .model = .link_y, + .context = .link_z, + }); - const scenario: ShardingTest.Scenario = .{ + try runner.run(.{ .sharding = try .init("3d_mesh", &physical, logical, strategy), - .shape = Shape.init(.{ .batch = 4, .model = 4, .context = 4 }, .f32) + // Note: .context and .model are swapped compared to .link_x and .link_z + .shape = Shape.init(.{ .batch = 4, .context = 4, .model = 4 }, .f32) .withPartitioning(.{ .batch = .batch, .model = .model, .context = .context }), - .expected_sdy = "#sdy.sharding<@\"3d_mesh\", [{\"link_x\"}, {\"link_y\"}, {\"link_z\"}]>", + .expected_sdy = "#sdy.sharding<@\"3d_mesh\", [{\"link_x\"}, {\"link_z\"}, {\"link_y\"}]>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{ .{ 0, 2 }, .{ 0, 2 }, .{ 0, 2 } } }, - .{ .device_id = 1, .slices = &.{ .{ 0, 2 }, .{ 0, 2 }, .{ 2, 2 } } }, - .{ .device_id = 2, .slices = &.{ .{ 0, 2 }, .{ 2, 2 }, .{ 0, 2 } } }, + .{ .device_id = 1, .slices = &.{ .{ 0, 2 }, .{ 2, 2 }, .{ 0, 2 } } }, + .{ .device_id = 2, .slices = &.{ .{ 0, 2 }, .{ 0, 2 }, .{ 2, 2 } } }, .{ .device_id = 3, .slices = &.{ .{ 0, 2 }, .{ 2, 2 }, .{ 2, 2 } } }, .{ .device_id = 4, .slices = &.{ .{ 2, 2 }, .{ 0, 2 }, .{ 0, 2 } } }, - .{ .device_id = 5, .slices = &.{ .{ 2, 2 }, .{ 0, 2 }, .{ 2, 2 } } }, - .{ .device_id = 6, .slices = &.{ .{ 2, 2 }, .{ 2, 2 }, .{ 0, 2 } } }, + .{ .device_id = 5, .slices = &.{ .{ 2, 2 }, .{ 2, 2 }, .{ 0, 2 } } }, + .{ .device_id = 6, .slices = &.{ .{ 2, 2 }, .{ 0, 2 }, .{ 2, 2 } } }, .{ .device_id = 7, .slices = &.{ .{ 2, 2 }, .{ 2, 2 }, .{ 2, 2 } } }, }, - }; - try runner.run(scenario); + }); } test "sharding: num partitions for logical axis" { @@ -2283,8 +2302,7 @@ test "sharding: num partitions for logical axis" { .batch = .low_bandwidth, }); - var strategy: Strategy = .init; - strategy.addBinding(.model, .link_x); + var strategy: Strategy = .parseBindings(.{ .model = .link_x }); strategy.addFold(.link_x, &.{ .link_x, .link_z }); const sharding: Sharding.Data = try .init("axis_parts_mesh", &physical, logical, strategy); diff --git a/zml/buffer.zig b/zml/buffer.zig index db650907..499ee650 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -1,6 +1,7 @@ const std = @import("std"); const pjrt = @import("pjrt"); +const platforms = @import("platforms"); const stdx = @import("stdx"); const constants = @import("constants.zig"); @@ -76,6 +77,10 @@ pub const Buffer = struct { return self._shape; } + pub fn numShards(self: Buffer) u32 { + return @intCast(self._shards.len); + } + pub fn shards(self: *const Buffer) ShardIterator { return .{ ._platform = self._platform, @@ -84,7 +89,10 @@ pub const Buffer = struct { } pub fn format(self: Buffer, writer: *std.Io.Writer) !void { - try writer.print("{f}", .{self._sharding.fmtPlacements(self._shape)}); + const placement = self._sharding.placement(self._shape) catch { + return try writer.print("sharding error {} vs {}", .{ self._sharding, self._shape }); + }; + try writer.print("{f}", .{placement}); } /// Copies the content of the given buffer from host memory to the accelerator memory. @@ -106,38 +114,26 @@ pub const Buffer = struct { shard.deinit(platform.pjrt_api); }; - const buffer_type = pjrtx.bufferTypeFromDtype(sh.dtype()); + stdx.debug.assert(platform.devices[0].memory(opts.memory) != null, "Device doesn't have {} memory", .{opts.memory}); const slice = Slice.init(sh, data_); + const buffer_type = pjrtx.bufferTypeFromDtype(sh.dtype()); - for (res._sharding.devicesInCanonicalOrder()) |device| { - const placement = try res._sharding.placement(res._shape, device); - const sub_slice = placement.shardSlice(slice); - var default_layout: ?pjrt.DefaultMemoryLayout = null; - const layout: pjrt.MemoryLayout = switch (platform.target) { - .tpu => blk: { - default_layout = try platform.pjrt_client.defaultMemoryLayout( - platform.pjrt_api, - pjrtx.bufferTypeFromDtype(placement.shape.dtype()), - placement.shape.dims(), - ); - break :blk default_layout.?.toMemoryLayout(); - }, - else => .{ - .tiled = .{ - .minor_to_major = constants.minorToMajor(placement.shape.rank()), - .tile_dims = &.{}, - .tile_dims_sizes = &.{}, - }, - }, - }; + const placement = try res._sharding.placement(sh); + const shard_dims: []const i64 = placement.shape.dims(); + const layout = platform.defaultMemoryLayout(shard_dims, sh.dtype()); + + for (platform.physical_mesh.devices_in_canonical_order) |device| { + const memory = platform.devices[device.id].memory(opts.memory).?; const args: pjrt.Client.BufferFromHostBufferArgs = .{ - .data = sub_slice.constData().ptr, - .buffer_type = buffer_type, - .dims = placement.shape.dims(), - .byte_strides = sub_slice.byte_strides.constSlice(), + // Change for each device + .data = placement.shardPtr(device.coords, slice), + .dst = .{ .memory = memory.pjrt_memory }, + // Constant across devices .layout = layout, + .dims = shard_dims, + .buffer_type = buffer_type, + .byte_strides = slice.byte_strides.constSlice(), .host_buffer_semantics = .ImmutableUntilTransferCompletes, - .dst = .{ .memory = placement.memory(platform, opts.memory).pjrt_memory }, }; const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); @@ -200,6 +196,7 @@ pub const Buffer = struct { .neuron => { const allocator = std.heap.smp_allocator; + // TODO: use the strides trick to avoid allocating so much memory const host = try allocator.alloc(u8, sh.byteSize()); defer allocator.free(host); @@ -218,33 +215,21 @@ pub const Buffer = struct { shard.deinit(platform.pjrt_api); }; - const minor_to_major = constants.minorToMajor(res._shape.rank()); // Rank doesn't change with sharding. - const element_type = pjrtx.bufferTypeFromDtype(res._shape.dtype()); // dtype doesn't change with sharding. - for (res._sharding.devicesInCanonicalOrder()) |device| { - const placement = try res._sharding.placement(res._shape, device); - var default_layout: ?pjrt.DefaultMemoryLayout = null; - const layout: pjrt.MemoryLayout = switch (platform.target) { - .tpu => blk: { - default_layout = try platform.pjrt_client.defaultMemoryLayout( - platform.pjrt_api, - pjrtx.bufferTypeFromDtype(placement.shape.dtype()), - placement.shape.dims(), - ); - break :blk default_layout.?.toMemoryLayout(); - }, - else => .{ - .tiled = .{ - .minor_to_major = minor_to_major, - .tile_dims = &.{}, - .tile_dims_sizes = &.{}, - }, - }, - }; + stdx.debug.assert(platform.devices[0].memory(opts.memory) != null, "Device doesn't have {} memory", .{opts.memory}); + const element_type = pjrtx.bufferTypeFromDtype(sh.dtype()); + const placement = try res._sharding.placement(sh); + const shard_dims: []const i64 = placement.shape.dims(); + const layout = platform.defaultMemoryLayout(shard_dims, sh.dtype()); + + for (platform.physical_mesh.devices_in_canonical_order) |device| { + const memory = platform.devices[device.id].memory(opts.memory).?; const args: pjrt.Client.CreateUninitializedBufferArgs = .{ - .dims = placement.shape.dims(), - .element_type = element_type, + // Change for each device + .dst = .{ .memory = memory.pjrt_memory }, + // Constant across devices .layout = layout, - .dst = .{ .memory = placement.memory(platform, opts.memory).pjrt_memory }, + .dims = shard_dims, + .element_type = element_type, }; const shard_buffer = try platform.pjrt_client.createUninitializedBuffer(platform.pjrt_api, args); @@ -281,17 +266,14 @@ pub const Buffer = struct { pub fn toSlice(self: Buffer, io: std.Io, slice: Slice) !void { stdx.debug.assert(self._shape.eql(slice.shape), "Buffer shape {f} doesn't match destination slice {f}", .{ self._shape, slice.shape }); + const placement = try self._sharding.placement(self._shape); for (self._sharding.devicesInCanonicalOrder(), 0..) |device, shard_index| { - const placement = try self._sharding.placement(self._shape, device); - const sub_slice = placement.shardSlice(slice); + // TODO: handle replicated information, we shouldn't iterate over all the devices unless needed + const sub_slice = placement.shardSlice(device.coords, slice); if (!sub_slice.isContiguous()) return error.NonContiguousShardRead; const size_bytes = placement.shape.byteSize(); - const start = sub_slice.offset_bytes; - const end = start + size_bytes; - stdx.debug.assert(end <= slice.constData().len, "Shard sub_slice exceeds destination slice", .{}); - - const destination = slice.data()[start..end]; + const destination = sub_slice.data()[0..size_bytes]; const maybe_event = try self._shards.get(shard_index).toHostBuffer(self._platform.pjrt_api, destination); if (maybe_event) |event| { @@ -306,9 +288,9 @@ pub const Buffer = struct { const slice = try Slice.alloc(allocator, self.shape()); errdefer slice.free(allocator); + const placement = try self._sharding.placement(self._shape); for (self._sharding.devicesInCanonicalOrder(), 0..) |device, shard_index| { - const placement = try self._sharding.placement(self._shape, device); - const sub_slice = placement.shardSlice(slice); + const sub_slice = placement.shardSlice(device.coords, slice); var shard_slice = try Slice.alloc(allocator, sub_slice.shape); defer shard_slice.free(allocator); @@ -334,4 +316,8 @@ pub const Buffer = struct { return byte_size; } + + pub fn opaqueDevicePtr(self: Buffer, device_id: usize) *anyopaque { + return self._shards.get(device_id).opaqueDeviceMemoryDataPointer(self._platform.pjrt_api) catch unreachable; + } }; diff --git a/zml/io.zig b/zml/io.zig index 638c9bea..8b61fbd9 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -628,9 +628,9 @@ pub const DirectMemoryWriter = struct { } fn collectRawSegments(self: *StreamPlanner) !void { + const placement = try self.sharding.placement(self.shape); for (self.sharding.devicesInCanonicalOrder(), 0..) |device, writer_index| { - const placement = try self.sharding.placement(self.shape, device); - try self.appendShardSegments(placement, writer_index); + try self.appendShardSegments(placement.slices(device.coords).constSlice(), writer_index); } } @@ -691,32 +691,16 @@ pub const DirectMemoryWriter = struct { }); } - fn axisRange(self: *const StreamPlanner, placement: Placement, axis: usize) AxisRange { - for (placement.slices.constSlice()) |slice| { - if (slice.axis == axis) { - return .{ - .start = slice.start, - .size = slice.size, - }; - } - } - - return .{ - .start = 0, - .size = self.shape.dim(axis), - }; - } - fn appendShardSegmentsAtAxis( self: *StreamPlanner, - placement: Placement, + slices: []const Placement.Slice1d, writer_index: usize, rank: usize, axis: usize, base_start: i64, strides: []const i64, ) !void { - const range = self.axisRange(placement, axis); + const range: AxisRange = .{ .start = slices[axis].start, .size = slices[axis].size }; if (range.size == 0) return; if (axis + 1 == rank) { @@ -729,11 +713,11 @@ pub const DirectMemoryWriter = struct { var i: i64 = 0; while (i < range.size) : (i += 1) { const child_start = base_start + (range.start + i) * strides[axis]; - try self.appendShardSegmentsAtAxis(placement, writer_index, rank, axis + 1, child_start, strides); + try self.appendShardSegmentsAtAxis(slices, writer_index, rank, axis + 1, child_start, strides); } } - fn appendShardSegments(self: *StreamPlanner, placement: Placement, writer_index: usize) !void { + fn appendShardSegments(self: *StreamPlanner, slices: []const Placement.Slice1d, writer_index: usize) !void { const rank = self.shape.rank(); if (rank == 0) { try self.appendRawSegment(writer_index, 0, self.shape.byteSize()); @@ -741,7 +725,7 @@ pub const DirectMemoryWriter = struct { } const strides = self.shape.computeByteStrides(); - try self.appendShardSegmentsAtAxis(placement, writer_index, rank, 0, 0, strides.constSlice()); + try self.appendShardSegmentsAtAxis(slices, writer_index, rank, 0, 0, strides.constSlice()); } }; @@ -799,15 +783,15 @@ pub const DirectMemoryWriter = struct { }; var pjrt_buffers: Buffer.Shards = .empty; + const placement = try sharding.placement(shape); for (ordered_devices, 0..) |device, i| { defer initialized += 1; - const placement = try sharding.placement(shape, device); - const device_index = placement.platformDeviceIndex(platform) orelse return error.UnknownShardDevice; - const pool = &pools[device_index]; - const shard_dma_allocator = dma_allocators[device_index].allocator(); + const pool = &pools[device.id]; + const shard_dma_allocator = dma_allocators[device.id].allocator(); + const dst_memory = platform.devices[device.id].memory(.default).?; - shard_writers[i] = try DirectShardWriter.init(shard_dma_allocator, io, placement.memory(platform, .default), pool, placement.shape); + shard_writers[i] = try DirectShardWriter.init(shard_dma_allocator, io, dst_memory, pool, placement.shape); pjrt_buffers.appendAssumeCapacity(shard_writers[i].pjrt_buffer); } @@ -1377,12 +1361,7 @@ test "DirectMemoryWriter: replicated with auto topology" { .shape = Shape.init(.{ .rows = 8, .cols = 128 }, .f32) .withPartitioning(.{ .rows = .replicated, .cols = .replicated }), .logical_mesh = .mesh(.{ .x = .high_bandwidth }), - .strategy = blk: { - var strategy: Sharding.Strategy = .init; - strategy.addBinding(.x, .link_x); - - break :blk strategy; - }, + .strategy = .parseBindings(.{ .x = .link_x }), }); } @@ -1401,12 +1380,7 @@ test "DirectMemoryWriter: 1D model split with 2x2 physical mesh" { .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32) .withPartitioning(.{ .rows = .replicated, .cols = .model }), .logical_mesh = .mesh(.{ .model = .high_bandwidth }), - .strategy = blk: { - var strategy: Sharding.Strategy = .init; - strategy.addBinding(.model, .link_x); - - break :blk strategy; - }, + .strategy = .parseBindings(.{ .model = .link_x }), }); } @@ -1428,13 +1402,7 @@ test "DirectMemoryWriter: 2D batch/model split with 2x2 physical mesh" { .batch = .low_bandwidth, .model = .high_bandwidth, }), - .strategy = blk: { - var strategy: Sharding.Strategy = .init; - strategy.addBinding(.batch, .link_x); - strategy.addBinding(.model, .link_y); - - break :blk strategy; - }, + .strategy = .parseBindings(.{ .batch = .link_x, .model = .link_y }), }); } @@ -1453,10 +1421,8 @@ test "DirectMemoryWriter: folded model sharding with 2x2 physical mesh" { .shape = Shape.init(.{ .model = 4096 }, .f32).withPartitioning(.{ .model = .model }), .logical_mesh = .mesh(.{ .model = .high_bandwidth }), .strategy = blk: { - var strategy: Sharding.Strategy = .init; - strategy.addBinding(.model, .link_x); + var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); strategy.addFold(.link_x, &.{ .link_x, .link_y }); - break :blk strategy; }, }); @@ -1477,12 +1443,7 @@ test "DirectMemoryWriter: writableSliceGreedy with mirrored shards" { .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32) .withPartitioning(.{ .rows = .replicated, .cols = .model }), .logical_mesh = .mesh(.{ .model = .high_bandwidth }), - .strategy = blk: { - var strategy: Sharding.Strategy = .init; - strategy.addBinding(.model, .link_x); - - break :blk strategy; - }, + .strategy = .parseBindings(.{ .model = .link_x }), .write_mode = .writable_slice_greedy, .writable_slice_min_len = 64, .pool_chunk_size = 1024, @@ -1508,10 +1469,8 @@ test "DirectMemoryWriter: 3D topology folded model + replicated batch" { .model = .high_bandwidth, }), .strategy = blk: { - var strategy: Sharding.Strategy = .init; - strategy.addBinding(.model, .link_x); + var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); strategy.addFold(.link_x, &.{ .link_x, .link_z }); - break :blk strategy; }, }); diff --git a/zml/mem.zig b/zml/mem.zig index 1a5d012d..4b7b236f 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -22,7 +22,7 @@ pub const DmaAllocator = union(enum) { pub fn init(parent: std.mem.Allocator, device: *const Device) DmaAllocator { return switch (device.platform.target) { .cuda => .{ .dmam = .init(parent, device.platform) }, - .oneapi, .tpu => .{ .uib = .init(device.memory(.host_pinned)) }, + .oneapi, .tpu => .{ .uib = .init(device.memory(.host_pinned).?) }, .rocm, .cpu, .neuron => .{ .passthrough = parent }, }; } diff --git a/zml/platform.zig b/zml/platform.zig index 00a24215..d9928721 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const builtin = @import("builtin"); const c = @import("c"); const pjrt = @import("pjrt"); @@ -12,21 +13,11 @@ const pjrtx = @import("pjrtx.zig"); const profiler_ = @import("profiling/profiler.zig"); const Sharding = @import("Sharding.zig"); const zml = @import("zml.zig"); +const constants = @import("constants.zig"); const log = std.log.scoped(.zml); -fn StaticPlatformMap(comptime E: type, comptime T: type) type { - const tag_count = @typeInfo(E).@"enum".fields.len; - var struct_field_names: [tag_count][]const u8 = undefined; - var struct_field_types: [tag_count]type = @splat(T); - const is_optional = @typeInfo(T) == .optional; - const default_value_ptr: ?*const anyopaque = if (is_optional) @ptrCast(&@as(T, null)) else null; - var struct_field_attrs: [tag_count]std.builtin.Type.StructField.Attributes = @splat(.{ .default_value_ptr = default_value_ptr }); - inline for (@typeInfo(E).@"enum".fields, 0..) |f, i| struct_field_names[i] = f.name; - return @Struct(.auto, null, &struct_field_names, &struct_field_types, &struct_field_attrs); -} - -var api_map: StaticPlatformMap(Target, ?*const pjrt.Api) = .{}; +var api_map: std.enums.EnumArray(Target, ?*const pjrt.Api) = .initFill(null); fn disableXlaLogs() void { // https://deepreg.readthedocs.io/en/latest/docs/logging.html#tensorflow-logging @@ -60,10 +51,10 @@ fn validateDeviceCount(target: Target, num_devices: usize) !void { fn loadOrGetApi(allocator: std.mem.Allocator, io: std.Io, target: Target) !*const pjrt.Api { return switch (target) { - inline else => |tag| @field(api_map, @tagName(tag)) orelse b: { + inline else => |tag| api_map.get(tag) orelse b: { disableXlaLogs(); const api = try platforms.load(allocator, io, tag); - @field(api_map, @tagName(tag)) = api; + api_map.set(tag, api); break :b api; }, }; @@ -128,6 +119,7 @@ pub const Device = struct { pjrt_device: *const pjrt.Device, pjrt_desc: *const pjrt.DeviceDescription, addressable_memories: []*const Memory, + memory_by_kind: std.EnumArray(Memory.Kind, ?*const Memory), fn init(allocator: std.mem.Allocator, pjrt_device_: *const pjrt.Device, platform: *const Platform) !Device { const pjrt_addressable_memories = pjrt_device_.addressableMemories(platform.pjrt_api); @@ -136,11 +128,21 @@ pub const Device = struct { addressable_memory.* = platform.memoryFromPjrt(pjrt_memory); } + // Cache memory lookups since they are expensive + const default_memory: *const Memory = resolveDefaultMemory(pjrt_device_, platform, addressable_memories); + const memory_by_kind: std.EnumArray(Memory.Kind, ?*const Memory) = .init(.{ + .default = default_memory, + .device = resolveMemory(addressable_memories, .device), + .host_pinned = resolveMemory(addressable_memories, .host_pinned), + .host_unpinned = resolveMemory(addressable_memories, .host_unpinned), + }); + return .{ .platform = platform, .pjrt_device = pjrt_device_, .pjrt_desc = pjrt_device_.getDescription(platform.pjrt_api), .addressable_memories = addressable_memories, + .memory_by_kind = memory_by_kind, }; } @@ -148,8 +150,27 @@ pub const Device = struct { allocator.free(self.addressable_memories); } - pub fn id(self: Device) usize { - return self.pjrt_desc.id(self.platform.pjrt_api); + fn resolveDefaultMemory(pjrt_device_: *const pjrt.Device, platform: *const Platform, addressable_memories: []*const Memory) *const Memory { + const pjrt_memory = pjrt_device_.defaultMemory(platform.pjrt_api); + for (addressable_memories) |mem| { + if (mem.pjrt_memory == pjrt_memory) return mem; + } + return platform.memoryFromPjrt(pjrt_memory); + } + + fn resolveMemory(addressable_memories: []*const Memory, memory_kind: Memory.Kind) ?*const Memory { + std.debug.assert(memory_kind != .default); + + for (addressable_memories) |mem| { + if (mem.isOfKind(memory_kind)) { + return mem; + } + } + return null; + } + + pub fn id(self: Device) u32 { + return @intCast(self.pjrt_desc.id(self.platform.pjrt_api)); } pub fn processIndex(self: Device) i32 { @@ -183,24 +204,36 @@ pub const Device = struct { }); } - pub fn memory(self: Device, memory_kind: Memory.Kind) *const Memory { - if (memory_kind == .default) { - const pjrt_memory = self.pjrt_device.defaultMemory(self.platform.pjrt_api); - for (self.addressable_memories) |mem| { - if (mem.pjrt_memory == pjrt_memory) return mem; - } - return self.platform.memoryFromPjrt(pjrt_memory); - } - - for (self.addressable_memories) |mem| { - if (mem.isOfKind(memory_kind)) { - return mem; - } - } - unreachable; + pub fn memory(self: *const Device, memory_kind: Memory.Kind) ?*const Memory { + return self.memory_by_kind.values[@intFromEnum(memory_kind)]; } }; +fn platformDeviceSortId(target: Target, device: Device) usize { + return switch (target) { + .neuron => @intCast(device.localHardwareId()), + .cuda, .rocm, .tpu, .cpu, .oneapi => device.id(), + }; +} + +fn sortDevicesById(target: Target, devices: []Device) void { + const Context = struct { + target: Target, + + fn lessThan(ctx: @This(), lhs: Device, rhs: Device) bool { + return platformDeviceSortId(ctx.target, lhs) < platformDeviceSortId(ctx.target, rhs); + } + }; + + std.mem.sort(Device, devices, Context{ .target = target }, Context.lessThan); + + if (builtin.mode == .Debug) { + for (devices, 0..) |device, expected_id| { + std.debug.assert(platformDeviceSortId(target, device) == expected_id); + } + } +} + pub const Platform = struct { arena: std.heap.ArenaAllocator, target: Target, @@ -271,6 +304,7 @@ pub const Platform = struct { for (pjrt_devices, devices) |pjrt_device, *platform_device| { platform_device.* = try .init(arena, pjrt_device, platform); } + sortDevicesById(target, devices); for (memories) |*platform_memory| { platform_memory.populateAddressableByDevices(); } @@ -567,6 +601,29 @@ pub const Platform = struct { } unreachable; } + + pub inline fn defaultMemoryLayout(platform: *const Platform, dims: []const i64, dtype: zml.DataType) pjrt.MemoryLayout { + // inline cause `default` is a huge ass struct allocated on the stack, + // and toMemoryLayout returns slices into it. + // There is probably a better way of doing this, + // but given it's compiled out (except for TPU), I'm not gonna care for now. + return switch (platform.target) { + .tpu => { + if (comptime !platforms.isEnabled(.tpu)) unreachable; + const element_type = pjrtx.bufferTypeFromDtype(dtype); + const default = platform.pjrt_client.defaultMemoryLayout(platform.pjrt_api, element_type, dims) catch @panic("Failed to get default memory layout"); + return default.toMemoryLayout(); + }, + .cuda, .rocm, .neuron, .oneapi, .cpu => .{ + // If this is the default layout on the platform, there is no point calling PJRT + .tiled = .{ + .minor_to_major = constants.minorToMajor(@intCast(dims.len)), + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }, + }; + } }; pub const CreateOptions = struct { @@ -766,3 +823,29 @@ fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { return null; } + +test "platform defaultMemoryLayout is boring" { + const platform = zml.testing.env(); + + const shapes = [_][]const i64{ + &.{4096}, + &.{ 4096, 4096 }, + &.{ 4096, 4096, 4096 }, + }; + for (shapes) |dims| { + // Checks that the PJRT client always return the same thing than `platform.defaultMemoryLayout` + // This allows to bypass the PJRT calls and the string of pjrt_client.defaultMemoryLayout. + const default_layout = try platform.pjrt_client.defaultMemoryLayout(platform.pjrt_api, pjrtx.bufferTypeFromDtype(.f32), dims); + const mem_layout = default_layout.toMemoryLayout(); + + // Note: I'm not just calling platform.defaultMemoryLayout because I'm investigating + // wether TPU requires its special branch. + try std.testing.expectEqualDeep(mem_layout, pjrt.MemoryLayout{ + .tiled = .{ + .minor_to_major = constants.minorToMajor(@intCast(dims.len)), + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }); + } +} diff --git a/zml/shape.zig b/zml/shape.zig index 01a2098b..8de83d95 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -302,9 +302,7 @@ pub const Shape = struct { fn isTagConvertible(comptime T: type) bool { return switch (T) { - @EnumLiteral() => true, - std.builtin.Type.StructField => true, - Tag => true, + @EnumLiteral(), std.builtin.Type.StructField, Tag => true, else => false, }; } @@ -319,8 +317,10 @@ pub const Shape = struct { }; } - inline fn ensureAttributesAreSync(self: Shape) void { - stdx.debug.assert(self._dims.len == self._tags.len and self._dims.len == self._partitioning.len, "Tags, dims and partitioning have diverged! dims={d} tags={d} partitioning={d}", .{ self._dims.len, self._tags.len, self._partitioning.len }); + fn ensureAttributesAreSync(self: Shape) void { + if (builtin.mode == .Debug) { + stdx.debug.assert(self._dims.len == self._tags.len and self._dims.len == self._partitioning.len, "Tags, dims and partitioning have diverged! dims={d} tags={d} partitioning={d}", .{ self._dims.len, self._tags.len, self._partitioning.len }); + } } pub fn tag(self: Shape, ax: anytype) Tag { @@ -387,15 +387,21 @@ pub const Shape = struct { self.ensureAttributesAreSync(); const T = @TypeOf(axis_); - if (comptime stdx.meta.isInteger(T)) { - return self.axisFromInt(@intCast(axis_)); - } - - if (comptime isTagConvertible(T)) { - return self.axisFromTag(toTag(axis_)); - } - - stdx.debug.compileError("Wrong axis type, expected .literal, or an integer, got: {any}", .{T}); + // Special handling of unsigned int, to remove runtime computations and favor inlining + return switch (@typeInfo(T)) { + .int => |info| if (info.signedness == .signed) + self.axisFromInt(axis_) + else + @intCast(axis_), + .comptime_int => if (axis_ < 0) + self.axisFromInt(axis_) + else + axis_, + else => if (comptime isTagConvertible(T)) + self.axisFromTag(toTag(axis_)) + else + stdx.debug.compileError("Wrong axis type, expected .literal, or an integer, got: {any}", .{T}), + }; } pub fn axes(self: Shape, axes_: anytype) AxesArray { diff --git a/zml/slice.zig b/zml/slice.zig index be604b0e..1fb363e5 100644 --- a/zml/slice.zig +++ b/zml/slice.zig @@ -18,10 +18,8 @@ pub fn isBytes(comptime T: type) bool { } pub const Slice = struct { - inner_data: union(enum) { - mutable: []u8, - immutable: []const u8, - }, + bytes: []const u8, + mutable: bool, shape: Shape, offset_bytes: usize = 0, byte_strides: stdx.BoundedArray(i64, constants.MAX_RANK), @@ -31,10 +29,13 @@ pub const Slice = struct { stdx.debug.assert(bytes.len == shape.byteSize(), "Expected \"bytes\" to have the same length as shape.byteSize() ({}), got {}", .{ shape.byteSize(), bytes.len }); const type_info = @typeInfo(@TypeOf(bytes)); const byte_strides = shape.computeByteStrides(); - return if (type_info.pointer.is_const) - .{ .inner_data = .{ .immutable = bytes }, .shape = shape, .offset_bytes = 0, .byte_strides = byte_strides } - else - .{ .inner_data = .{ .mutable = bytes }, .shape = shape, .offset_bytes = 0, .byte_strides = byte_strides }; + return .{ + .bytes = bytes, + .mutable = !type_info.pointer.is_const, + .shape = shape, + .offset_bytes = 0, + .byte_strides = byte_strides, + }; } pub fn alloc(allocator: std.mem.Allocator, shape_: Shape) !Slice { @@ -51,7 +52,8 @@ pub const Slice = struct { }; return .{ - .inner_data = .{ .mutable = bytes }, + .bytes = bytes, + .mutable = true, .shape = shape_, .offset_bytes = 0, .byte_strides = shape_.computeByteStrides(), @@ -59,8 +61,8 @@ pub const Slice = struct { } pub fn free(slice: Slice, allocator: std.mem.Allocator) void { - const d = slice.constData_(); - + const d = slice.bytes; + // TODO: It would be better to store the aligment, this can be error prone in the presence of type casting switch (slice.shape.dtype().alignOf()) { 1 => allocator.free(@as([]align(1) const u8, @alignCast(d))), 2 => allocator.free(@as([]align(2) const u8, @alignCast(d))), @@ -79,9 +81,8 @@ pub const Slice = struct { pub fn isContiguous(slice: Slice) bool { const expected = slice.shape.computeByteStrides(); - const rank = slice.shape.rank(); - for (0..rank) |ax| { - if (slice.byte_strides.get(ax) != expected.get(ax)) return false; + for (expected.slice(), slice.byte_strides.slice()) |exp, actual| { + if (exp != actual) return false; } return true; } @@ -122,36 +123,20 @@ pub const Slice = struct { } pub fn data(slice: Slice) []u8 { - const base = slice.data_(); - stdx.debug.assert(slice.offset_bytes <= base.len, "Slice offset exceeds data length", .{}); - return base[slice.offset_bytes..]; - } - - fn data_(slice: Slice) []u8 { - return switch (slice.inner_data) { - .mutable => |d| d, - else => stdx.debug.panic("Expected slice to be mutable but it's immutable", .{}), - }; + std.debug.assert(slice.mutable); + return @constCast(slice.bytes[slice.offset_bytes..]); } pub fn constData(slice: Slice) []const u8 { - const base = slice.constData_(); - stdx.debug.assert(slice.offset_bytes <= base.len, "Slice offset exceeds data length", .{}); - return base[slice.offset_bytes..]; - } - - fn constData_(slice: Slice) []const u8 { - return switch (slice.inner_data) { - inline else => |d| d, - }; + return slice.bytes[slice.offset_bytes..]; } pub fn items(slice: Slice, comptime T: type) []T { - return @alignCast(std.mem.bytesAsSlice(T, slice.data())); + return @ptrCast(@alignCast(slice.data())); } pub fn constItems(slice: Slice, comptime T: type) []const T { - return @alignCast(std.mem.bytesAsSlice(T, slice.constData())); + return @ptrCast(@alignCast(slice.constData())); } pub fn format( diff --git a/zml/tensor.zig b/zml/tensor.zig index 4759069c..b8ffe93b 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -281,6 +281,46 @@ pub const Tensor = struct { return self; } + test reuseBuffer { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const io = std.testing.io; + + const inputs: [2][6]f32 = .{ .{ -3.0, -2, -1, 1, 2, 3 }, .{ 1, 2, 3, 4, 5, -5 } }; + const left = Tensor.init(.{ 2, 6 }, .f32); + const right = Tensor.init(.{ 2, 6 }, .f32); + + const Local = struct { + pub fn memcopy(x: Tensor, y: Tensor) Tensor { + return x.addConstant(1).reuseBuffer(y); + } + }; + + const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.memcopy, .{ left, right }, platform, .{}); + defer exe.deinit(); + + var left_d = try zml.Buffer.fromBytes(io, platform, left.shape(), .replicated, @ptrCast(&inputs)); + defer left_d.deinit(); + + var right_d = try zml.Buffer.uninitialized(io, platform, left.shape(), .replicated, .{}); + defer right_d.deinit(); + + var right_memory: [Platform.MAX_NUM_DEVICES]*anyopaque = undefined; + for (0..right_d.numShards()) |dev| { + right_memory[dev] = right_d.opaqueDevicePtr(dev); + } + + const output_d = try zml.testing.autoCall(std.testing.allocator, io, &exe, Local.memcopy, .{ left_d, right_d }); + + var output_memory: [Platform.MAX_NUM_DEVICES]*anyopaque = undefined; + for (0..output_d.numShards()) |dev| { + output_memory[dev] = output_d.opaqueDevicePtr(dev); + } + + // Check that right_d and output_d point to the same addresses on the devices. + try std.testing.expectEqualSlices(*anyopaque, right_memory[0..right_d.numShards()], output_memory[0..output_d.numShards()]); + } + /// Returns a Tensor containing the absolute value of each element of the input Tensor. pub fn abs(self: Tensor) Tensor { const op = dialects.stablehlo.abs(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); From 8c03af9987cfe6cd07cd7e550c108de4796aae01 Mon Sep 17 00:00:00 2001 From: elogir Date: Mon, 1 Jun 2026 17:51:42 +0200 Subject: [PATCH 508/881] stdx/flags: Update typeInfo tag to Zig 0.16 `.@"union"`. (#565) --- stdx/flags.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdx/flags.zig b/stdx/flags.zig index c23ae55d..6ee06eaf 100644 --- a/stdx/flags.zig +++ b/stdx/flags.zig @@ -93,7 +93,7 @@ pub fn parseProcessArgs(init_: std.process.Init.Minimal, comptime CliArgs: type) } fn parse_commands(args: *std.process.Args.Iterator, comptime Commands: type) Commands { - comptime assert(@typeInfo(Commands) == .Union); + comptime assert(@typeInfo(Commands) == .@"union"); comptime assert(std.meta.fields(Commands).len >= 2); const first_arg = args.next() orelse fatal( From f9a282162a470a648bfd9bd637c0ce8a021b5685 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 1 Jun 2026 15:53:48 +0000 Subject: [PATCH 509/881] zml: fix multiple enabled platforms more than one platform can be enabled at the same time --- zml/BUILD.bazel | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 215ea188..393d3bff 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -33,6 +33,8 @@ cc_library( hdrs = ["profiling/traceme.h"], defines = select({ "//platforms:cuda.enabled": ["ZML_RUNTIME_CUDA"], + "//conditions:default": [], + }) + select({ "//platforms:rocm.enabled": ["ZML_RUNTIME_ROCM"], "//conditions:default": [], }), @@ -54,6 +56,8 @@ cc_library( "//platforms:cuda.enabled": [ "//platforms/cuda:cuda_nvtx_headers", ], + "//conditions:default": [], + }) + select({ "//platforms:rocm.enabled": [ "@rocprofiler-sdk-roctx//:headers", ], From 4b0c7ed1c0091ee2daa7e92d9cc8a36bb653d195 Mon Sep 17 00:00:00 2001 From: Louis Lechevalier Date: Wed, 3 Jun 2026 15:58:16 +0200 Subject: [PATCH 510/881] zml/moe : Mosaic GMM EP TPU kernel. DSL update. (#569) This PR introduce the new GMM Expert parallelism kernel (basis grouped gemm for moe on TPU). It induces some updates into the DSL mosaic TPU, such as Builders helpers or Pipelines and schedulers : - Buffer counts (for scheduling overlapping) > 2 allowed (until 7) - source_reshape_shape reshaped at DMA slice/use sites to match Pallas IR placement - More grid dimensionality for pipeline (up to 4) --- kernels/mosaic_tpu/builder.zig | 118 ++++++- kernels/mosaic_tpu/pipeline.zig | 157 ++++++--- zml/BUILD.bazel | 12 + zml/moe/mosaic_tpu_kernels/gmm_ep.zig | 450 ++++++++++++++++++++++++++ 4 files changed, 687 insertions(+), 50 deletions(-) create mode 100644 zml/moe/mosaic_tpu_kernels/gmm_ep.zig diff --git a/kernels/mosaic_tpu/builder.zig b/kernels/mosaic_tpu/builder.zig index adb2f4e4..2f6bd2ab 100644 --- a/kernels/mosaic_tpu/builder.zig +++ b/kernels/mosaic_tpu/builder.zig @@ -297,6 +297,9 @@ pub const ArgSpec = struct { shape: []const i64, dtype: DType, memory_space: MemorySpace = .vmem, + /// Emit `{sc.persistent}` on this memref argument. Sparse-core Pallas + /// kernels mark persistent HBM operands this way. + persistent: bool = false, /// Block window. `null` ⇒ trivial window (block == operand shape); /// VMEM trivial windows auto-get `pipeline_mode = synchronous`. window: ?WindowSpec = null, @@ -476,6 +479,10 @@ pub const Builder = struct { /// Whether to emit Pallas-style `transform_N` stubs at `finish` time. /// Set by `declareArgsLowOpts` from `Opts.pallas_window_params`. emit_transform_stubs: bool = false, + /// Optional prefix for generated transform function names. + transform_name_prefix: ?[]const u8 = null, + /// Whether generated transform functions carry `{sc.is_transform_indices}`. + sc_transform_indices: bool = false, /// Per-ref-arg snapshot for `finish`-time `transform_N` stub emission. /// `skipped` = ANY/HBM/SEMAPHORE memory space (empty window, no transform). @@ -510,6 +517,9 @@ pub const Builder = struct { /// Pallas's `iteration_bounds = array` — the grid shape /// (one entry per `dimension_semantics`). `null` omits the attr. iteration_bounds: ?[]const i64 = null, + /// Emit `scalar_prefetch = N : i64`. Sparse-core kernels with no + /// scalar prefetches omit the zero-valued attr. + emit_scalar_prefetch_attr: bool = true, /// When `true`, auto-emit `window_params = [...]` and trailing /// `transform_N` stub funcs at `finish` time. Per-arg behavior /// matches `lowering.py:1014–1131`: ANY/HBM/SEMAPHORE refs get @@ -517,6 +527,13 @@ pub const Builder = struct { /// auto-force `pipeline_mode = synchronous`. Override per-arg /// via `RefSpec.window`. pallas_window_params: bool = false, + /// Include HBM refs in Pallas-style `window_params`. Tensor-core + /// Pallas lowering skips HBM; sparse-core lowering includes it. + window_hbm_refs: bool = false, + /// Optional prefix for generated transform function names. + transform_name_prefix: ?[]const u8 = null, + /// Add `{sc.is_transform_indices}` to generated transform stubs. + sc_transform_indices: bool = false, }; /// Create a sub-module containing a single public `func.func` of the @@ -601,6 +618,7 @@ pub const Builder = struct { .shape = inner.shape, .dtype = inner.dtype, .memory_space = if (@hasField(@TypeOf(inner), "memory_space")) inner.memory_space else .vmem, + .persistent = if (@hasField(@TypeOf(inner), "persistent")) inner.persistent else false, .window = if (@hasField(@TypeOf(inner), "window")) inner.window else null, .role = if (@hasField(@TypeOf(inner), "role")) inner.role else .input, } }, @@ -665,8 +683,7 @@ pub const Builder = struct { const entry = mlir.Block.init(arg_types, arg_locs); - // Build per-arg attribute dicts. The only ones we currently emit are - // `tpu.dimension_semantics` on iter args. + // Build per-arg attribute dicts. const arg_attrs = try scratch.alloc(*const mlir.Attribute, args.len); var any_arg_attr = false; for (args, 0..) |a, i| { @@ -682,19 +699,31 @@ pub const Builder = struct { arg_attrs[i] = empty_dict; } }, + .ref => |r| { + if (r.persistent) { + arg_attrs[i] = .dict(ctx, &.{ + .named(ctx, "sc.persistent", .unit(ctx)), + }); + any_arg_attr = true; + } else { + arg_attrs[i] = empty_dict; + } + }, else => arg_attrs[i] = empty_dict, } } // Kernel-level func.func attributes. MLIR sorts attrs alphabetically on print. - var extra_attrs: stdx.BoundedArray(mlir.NamedAttribute, 8) = .empty; + var extra_attrs: stdx.BoundedArray(mlir.NamedAttribute, 12) = .empty; const dim_sem_buf = try scratch.alloc(*const mlir.Attribute, opts.dimension_semantics.len); for (opts.dimension_semantics, 0..) |s, i| dim_sem_buf[i] = s.attribute(ctx); extra_attrs.appendAssumeCapacity(.named(ctx, "dimension_semantics", .array(ctx, dim_sem_buf))); if (opts.iteration_bounds) |bounds| { extra_attrs.appendAssumeCapacity(.named(ctx, "iteration_bounds", .denseArray(ctx, .i64, bounds))); } - extra_attrs.appendAssumeCapacity(.named(ctx, "scalar_prefetch", .int(ctx, .i64, opts.scalar_prefetch))); + if (opts.emit_scalar_prefetch_attr) { + extra_attrs.appendAssumeCapacity(.named(ctx, "scalar_prefetch", .int(ctx, .i64, opts.scalar_prefetch))); + } extra_attrs.appendAssumeCapacity(.named(ctx, "scratch_operands", .int(ctx, .i64, opts.scratch_operands))); if (opts.core_type) |ct| { extra_attrs.appendAssumeCapacity(.named(ctx, "tpu.core_type", ct.attribute(ctx))); @@ -714,9 +743,10 @@ pub const Builder = struct { (w.block_shape orelse r.shape) else r.shape; + const skipped = isSkippedWindowSpace(r.memory_space) and !(opts.window_hbm_refs and r.memory_space == .hbm); try ref_args_buf.append(arena_alloc, .{ .block_rank = @intCast(block_shape.len), - .skipped = isSkippedWindowSpace(r.memory_space), + .skipped = skipped, .role = r.role, .transform_returns = if (r.window) |w| w.transform_returns else null, }); @@ -730,6 +760,8 @@ pub const Builder = struct { self.prefetch_args = try arena_alloc.dupe(*const mlir.Type, prefetch_types_buf.items); self.grid_dim_count = opts.dimension_semantics.len; self.emit_transform_stubs = opts.pallas_window_params; + self.transform_name_prefix = opts.transform_name_prefix; + self.sc_transform_indices = opts.sc_transform_indices; // window_params: one entry per `.input` / `.output` ref. // ANY/HBM/SEMAPHORE → empty dict, no transform_N. @@ -741,7 +773,7 @@ pub const Builder = struct { for (args) |a| switch (a.kind) { .ref => |r| { if (r.role != .input and r.role != .output) continue; - if (isSkippedWindowSpace(r.memory_space)) { + if (isSkippedWindowSpace(r.memory_space) and !(opts.window_hbm_refs and r.memory_space == .hbm)) { wp_buf[idx] = .dict(ctx, &.{}); idx += 1; continue; @@ -767,8 +799,10 @@ pub const Builder = struct { break :blk null; }; - var sym_buf: [32]u8 = undefined; - const sym_name = std.fmt.bufPrint(&sym_buf, "transform_{d}", .{idx}) catch unreachable; + const sym_name = if (opts.transform_name_prefix) |prefix| + try std.fmt.allocPrint(scratch, "{s}_transform_{d}", .{ prefix, idx }) + else + try std.fmt.allocPrint(scratch, "transform_{d}", .{idx}); var inner: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; if (resolved_pipeline) |pm| { @@ -1433,6 +1467,23 @@ pub const Builder = struct { )); } + /// `tpu.vector_load` with an explicit result vector shape. + pub fn tpuVectorLoadShape(self: *Builder, ref: Value, indices: []const Value, shape: []const i64, opts: VectorLoadOpts) Value { + const ctx = self.ctx; + const result_ty = mlir.Type.vector(shape, ref.elemType()); + const idx_inner = self.indicesOrZero(ref, indices); + const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; + return self.emit(tpu.vector_load( + ctx, + ref.inner, + idx_inner, + mask_inner, + .{ .strides = opts.strides }, + result_ty, + self.loc(), + )); + } + /// `tpu.vector_store(value, ref)` — full-shape vector store. pub fn refStore(self: *Builder, ref: Value, value: Value) void { self.refStoreOpts(ref, value, &.{}, .{}); @@ -1606,6 +1657,16 @@ pub const Builder = struct { return self.emit(vector.extract(self.ctx, src.inner, position, &.{}, ty, self.loc())); } + /// `vector.extract_strided_slice` — statically slice a vector. + pub fn vectorExtractStridedSlice(self: *Builder, src: Value, offsets: []const i64, sizes: []const i64, strides: []const i64) Value { + const ty = mlir.Type.vector(sizes, src.elemType()); + return self.emit(vector.extract_strided_slice(self.ctx, src.inner, .{ + .offsets = offsets, + .sizes = sizes, + .strides = strides, + }, ty, self.loc())); + } + /// `vector.reduction ` — reduce a 1-D vector to a scalar. /// Emits the bare upstream op; on real TPU the layout pass may reject /// 1-D vectors, so prefer `reduceToScalar` for kernel-level reductions. @@ -1726,7 +1787,7 @@ pub const Builder = struct { for (in_shape.constSlice(), 0..) |d, i| { if (@as(i32, @intCast(i)) != axis) out.appendAssumeCapacity(d); } - const ty = mlir.Type.vector(out.constSlice(), .integer(self.ctx, .i32)); + const ty = mlir.Type.vector(out.constSlice(), .int(self.ctx, .i32)); return self.emit(tpu.reduce_index(self.ctx, input.inner, axis, kind, ty, self.loc())); } @@ -1791,6 +1852,10 @@ pub const Builder = struct { return self.emit(tpu.memref_reshape(self.ctx, mem_ref.inner, result_ty, self.loc())); } + pub fn memRefAlloca(self: *Builder, shape: []const i64, dtype: DType, memory_space: MemorySpace) Value { + return self.emit(memref.alloca(self.ctx, self.memRefTy(shape, dtype, memory_space), &.{}, &.{}, .{}, self.loc())); + } + pub fn memRefBitcast(self: *Builder, mem_ref: Value, dt: DType) Value { const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefBitcast: not a memref"); var s: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; @@ -1847,15 +1912,16 @@ pub const Builder = struct { // ==================== Semaphores / DMA / barriers ==================== pub fn semAlloc(self: *Builder, kind: ArgSpec.SemKind) Value { - const ty = switch (kind) { + const elem = switch (kind) { .regular => tpu.semaphoreType(self.ctx), .dma => tpu.dmaSemaphoreType(self.ctx), }; + const ty = mlir.Type.memRef(elem, &.{}, null, MemorySpace.semaphore_mem.attribute(self.ctx)); return self.emit(tpu.sem_alloc(self.ctx, ty, self.loc())); } pub fn semBarrier(self: *Builder) Value { - const ty = tpu.dmaSemaphoreType(self.ctx); + const ty = mlir.Type.memRef(tpu.semaphoreType(self.ctx), &.{}, null, MemorySpace.semaphore_mem.attribute(self.ctx)); return self.emit(tpu.sem_barrier(self.ctx, ty, self.loc())); } @@ -1951,6 +2017,22 @@ pub const Builder = struct { _ = tpu.delay(self.ctx, c_.inner, self.loc()).appendTo(self.currentBlock()); } + pub fn assumeMultiple(self: *Builder, src: Value, multiple: i32) Value { + return self.emit(tpu.assume_multiple(self.ctx, src.inner, multiple, self.loc())); + } + + pub fn openRegion(self: *Builder) *mlir.Block { + const block = mlir.Block.init(&.{}, &.{}); + self.pushBlock(block); + return block; + } + + pub fn closeRegion(self: *Builder, block: *mlir.Block) void { + _ = tpu.yield(self.ctx, &.{}, self.loc()).appendTo(block); + self.popBlock(); + _ = tpu.region(self.ctx, block, &.{}, self.loc()).appendTo(self.currentBlock()); + } + // ==================== SCF regions ==================== /// Open a scoped `scf.for`. `lower`/`upper`/`step` accept Values, comptime @@ -2191,8 +2273,17 @@ pub const Builder = struct { } _ = func.returns(ctx, ret_vals, unknown_loc).appendTo(block); - var name_buf: [32]u8 = undefined; - const name = std.fmt.bufPrint(&name_buf, "transform_{d}", .{idx}) catch unreachable; + const name = if (self.transform_name_prefix) |prefix| + std.fmt.allocPrint(arena_alloc, "{s}_transform_{d}", .{ prefix, idx }) catch @panic("OOM") + else blk: { + var name_buf: [32]u8 = undefined; + break :blk arena_alloc.dupe(u8, std.fmt.bufPrint(&name_buf, "transform_{d}", .{idx}) catch unreachable) catch @panic("OOM"); + }; + + var extra_attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (self.sc_transform_indices) { + extra_attrs.appendAssumeCapacity(.named(ctx, "sc.is_transform_indices", .unit(ctx))); + } const stub_op = func.func(ctx, .{ .name = name, @@ -2202,6 +2293,7 @@ pub const Builder = struct { .location = unknown_loc, .visibility = null, .verify = false, + .extra_attributes = extra_attrs.constSlice(), }); _ = stub_op.appendTo(self.module.body()); } diff --git a/kernels/mosaic_tpu/pipeline.zig b/kernels/mosaic_tpu/pipeline.zig index da7644c8..235513a0 100644 --- a/kernels/mosaic_tpu/pipeline.zig +++ b/kernels/mosaic_tpu/pipeline.zig @@ -1,7 +1,6 @@ //! Zig port of `pltpu.emit_pipeline` (jax/_src/pallas/mosaic/pipeline.py) — //! emits lowered `tpu`-dialect IR byte-equivalent to `pallas_call`. //! -//! 1-D dynamic-bound grid; VMEM operands; `buffer_count ∈ {1,2}`; //! `.blocked`/`.squeezed`/one `.bounded_slice` block dim; trivial-windowing //! inputs peeled as sync copy; `trace_scopes`. Unsupported features panic. @@ -57,6 +56,8 @@ pub const BodyFn = *const fn (b: *Builder, grid_indices: []const Value, refs: [] pub const PipeSpec = struct { full_shape: []const i64, dtype: DType, + /// Optional HBM source/result view to materialize at each DMA use site. + source_reshape_shape: ?[]const i64 = null, /// `null` ⇒ trivial windowing (the whole array). block_shape: ?[]const BlockDim = null, /// `null` only allowed with trivial windowing. @@ -64,7 +65,7 @@ pub const PipeSpec = struct { index_map_ctx: ?*anyopaque = null, /// Only `.vmem` supported. memory_space: MemorySpace = .vmem, - /// `null` ⇒ 2 (or 1 if trivial). + buffer_count: ?usize = null, /// Tile that `_create_bounded_slice` rounds the DMA size up to. bounded_slice_tiling: i32 = 1, @@ -92,7 +93,7 @@ pub fn emitPipeline( opts: PipelineOpts, ) FinishError!void { std.debug.assert(opts.grid.len >= 1); - if (opts.grid.len != 1) @panic("emitPipeline: only 1-D grids supported (TODO: N-D `_next_index`)"); + if (opts.grid.len > 4) @panic("emitPipeline: grids with rank > 4 unsupported"); const n_in = opts.in_specs.len; std.debug.assert(refs.len == n_in + opts.out_specs.len); @@ -118,7 +119,7 @@ pub fn emitPipeline( for (brefs.items) |*br| br.allocate(b); - const num_steps = opts.grid[0]; + const num_steps = gridSize(b, opts.grid); const c0_i32 = b.lift(@as(i32, 0)); const c1_i32 = b.lift(@as(i32, 1)); { @@ -143,11 +144,11 @@ pub fn emitPipeline( if (opts.trace_scopes) b.traceStop(); } - // iter_args: each bref's slots in spec order (see appendSlotInits), then grid_idx. - var inits: std.ArrayList(*const mlir.Value) = try .initCapacity(a, brefs.items.len * 2 + 1); + // iter_args: each bref's slots in spec order (see appendSlotInits), then grid indices. + var inits: std.ArrayList(*const mlir.Value) = try .initCapacity(a, brefs.items.len * 2 + opts.grid.len); for (brefs.items) |*br| br.appendSlotInits(b, &inits); - inits.appendAssumeCapacity(c0_i32.inner); - const n_slots = inits.items.len - 1; + for (0..opts.grid.len) |_| inits.appendAssumeCapacity(c0_i32.inner); + const n_slots = inits.items.len - opts.grid.len; const i32_ty = b.scalarTy(.i32); const block_types = a.alloc(*const mlir.Type, inits.items.len + 1) catch unreachable; @@ -163,10 +164,13 @@ pub fn emitPipeline( for (brefs.items) |*br| cur = br.bindSlotsAt(b, for_body, cur); std.debug.assert(cur == n_slots + 1); } - const grid_idx: Value = .{ .inner = for_body.argument(@intCast(n_slots + 1)), .kernel = b }; + var grid_indices_buf: [4]Value = undefined; + for (0..opts.grid.len) |i| { + grid_indices_buf[i] = .{ .inner = for_body.argument(@intCast(n_slots + 1 + i)), .kernel = b }; + } var sched = Scheduler.init(b, iv, opts.grid, num_steps, num_stages, opts.trace_scopes); - sched.indices_storage[0] = grid_idx; + for (0..opts.grid.len) |i| sched.indices_storage[i] = grid_indices_buf[i]; sched.recomputeDerivedIndices(b); for (brefs.items) |*br| if (br.buffer_type == .input) sched.copyIn(b, br); @@ -193,10 +197,10 @@ pub fn emitPipeline( for (brefs.items) |*br| if (br.buffer_type == .input) sched.advanceSlots(b, br); - var yields: std.ArrayList(*const mlir.Value) = try .initCapacity(a, brefs.items.len * 2 + 1); + var yields: std.ArrayList(*const mlir.Value) = try .initCapacity(a, brefs.items.len * 2 + opts.grid.len); for (brefs.items) |*br| br.appendSlotYields(b, &yields); - const next_idx = nextIndex1D(b, grid_idx, num_steps); - yields.appendAssumeCapacity(next_idx.inner); + const next_indices = nextIndexND(b, sched.indices(), opts.grid); + for (next_indices) |idx| yields.appendAssumeCapacity(idx.inner); _ = scf.yield(b.ctx, yields.items, b.loc()).appendTo(for_body); b.popBlock(); @@ -204,12 +208,15 @@ pub fn emitPipeline( _ = for_op.appendTo(b.currentBlock()); var k: usize = 0; for (brefs.items) |*br| k = br.readbackSlots(b, for_op, k); - const final_grid_idx: Value = .{ .inner = for_op.result(@intCast(n_slots)), .kernel = b }; + var final_grid_indices_buf: [4]Value = undefined; + for (0..opts.grid.len) |i| { + final_grid_indices_buf[i] = .{ .inner = for_op.result(@intCast(n_slots + i)), .kernel = b }; + } - // prev_index BEFORE Scheduler.init so its inner `subi(num_steps,1)` CSEs with `step`. - const final_idx = prevIndex1D(b, final_grid_idx, num_steps); + // prev_index BEFORE Scheduler.init so common arithmetic CSEs with `step`. + const final_idx = prevIndexND(b, final_grid_indices_buf[0..opts.grid.len], opts.grid); var sched_fin = Scheduler.init(b, b.subi(num_steps, c1_i32), opts.grid, num_steps, num_stages, opts.trace_scopes); - sched_fin.indices_storage[0] = final_idx; + for (0..opts.grid.len) |i| sched_fin.indices_storage[i] = final_idx[i]; sched_fin.recomputeDerivedIndices(b); for (brefs.items) |*br| sched_fin.finalize(b, br); for (brefs.items) |*br| if (br.is_trivial and br.buffer_type == .output) br.syncCopyOut(b); @@ -315,13 +322,12 @@ const BufferedRef = struct { fn appendSlotInits(self: *BufferedRef, b: *Builder, out: *std.ArrayList(*const mlir.Value)) void { const c0 = b.lift(@as(i32, 0)).inner; - const c1 = b.lift(@as(i32, 1)).inner; switch (self.buffer_type) { .input => { if (self.is_trivial) { out.appendAssumeCapacity(c0); } else { - out.appendAssumeCapacity(c1); // copy_in_slot: initialize_step advanced 0→1 + out.appendAssumeCapacity((self.copy_in_slot orelse b.lift(@as(i32, 0))).inner); out.appendAssumeCapacity(c0); } }, @@ -488,6 +494,7 @@ const BufferedRef = struct { fn sliceSrc(self: *const BufferedRef, b: *Builder, dma: DmaSlice) Value { const a = b.arena.allocator(); const bs = self.spec.block_shape.?; + const src = if (self.spec.source_reshape_shape) |shape| b.memRefReshape(self.src, shape) else self.src; var base = a.alloc(Value, bs.len) catch unreachable; var dyn = std.ArrayList(Value).initCapacity(a, bs.len) catch unreachable; var result_shape = a.alloc(i64, bs.len) catch unreachable; @@ -504,7 +511,12 @@ const BufferedRef = struct { }; } } - return b.memRefSlice(self.src, base, result_shape, dyn.items); + const sliced = b.memRefSlice(src, base, result_shape, dyn.items); + if (self.block_keep.len == bs.len) return sliced; + + const squeezed_shape = a.alloc(i64, self.block_compact.len) catch unreachable; + for (self.block_keep, 0..) |sd, ci| squeezed_shape[ci] = result_shape[sd]; + return b.memRefSqueeze(sliced, squeezed_shape); } /// Slices `alloca[slot, 0…]` (non-squeezed dims) then squeezes the slot dim. @@ -639,9 +651,12 @@ const Scheduler = struct { indices_storage: [4]Value = undefined, prev_storage: [4]Value = undefined, next_storage: [4]Value = undefined, + fetch_storage: [8][4]Value = undefined, + fetch_len: usize, indices_len: usize, fn init(b: *Builder, step: Value, grid: []const Value, num_steps: Value, num_stages: usize, trace_scopes: bool) Scheduler { + if (num_stages + 1 > 8) @panic("emitPipeline: buffer_count > 7 unsupported"); const c0 = b.lift(@as(i32, 0)); const c1 = b.lift(@as(i32, 1)); var s: Scheduler = .{ @@ -652,6 +667,7 @@ const Scheduler = struct { .trace_scopes = trace_scopes, .first_step = b.cmpi(.eq, step, c0), .last_step = b.cmpi(.eq, step, b.subi(num_steps, c1)), + .fetch_len = num_stages + 1, .indices_len = grid.len, }; for (0..grid.len) |i| s.indices_storage[i] = c0; @@ -668,11 +684,26 @@ const Scheduler = struct { fn nextIndices(self: *Scheduler) []const Value { return self.next_storage[0..self.indices_len]; } + fn fetchIndices(self: *Scheduler, lookahead: usize) []const Value { + std.debug.assert(lookahead < self.fetch_len); + return self.fetch_storage[lookahead][0..self.indices_len]; + } fn recomputeDerivedIndices(self: *Scheduler, b: *Builder) void { - const i = self.indices_storage[0]; - self.prev_storage[0] = prevIndex1D(b, i, self.num_steps); - self.next_storage[0] = nextIndex1D(b, i, self.num_steps); + const prev = prevIndexND(b, self.indices(), self.grid); + const next = nextIndexND(b, self.indices(), self.grid); + for (0..self.indices_len) |i| { + self.prev_storage[i] = prev[i]; + self.next_storage[i] = next[i]; + self.fetch_storage[0][i] = self.indices_storage[i]; + self.fetch_storage[1][i] = next[i]; + } + var fetch = next; + var lookahead: usize = 2; + while (lookahead < self.fetch_len) : (lookahead += 1) { + fetch = nextIndexND(b, fetch, self.grid); + for (0..self.indices_len) |i| self.fetch_storage[lookahead][i] = fetch[i]; + } } fn namedScope(self: *Scheduler, b: *Builder, name: []const u8) void { @@ -719,8 +750,7 @@ const Scheduler = struct { fn willChangeFetch(self: *Scheduler, b: *Builder, br: *const BufferedRef) Value { std.debug.assert(br.isBuffered() and !br.is_trivial); if (br.buffer_count < 2) return self.hasChanged(b, br); - std.debug.assert(br.buffer_count == 2); - return indicesDiffer(b, br, self.indices(), self.nextIndices()); + return indicesDiffer(b, br, self.fetchIndices(br.buffer_count - 2), self.fetchIndices(br.buffer_count - 1)); } fn advance(b: *Builder, slot: Value, pred: Value) Value { @@ -730,9 +760,24 @@ const Scheduler = struct { fn initializeStep(self: *Scheduler, b: *Builder, br: *BufferedRef, step: usize) void { if (br.buffer_type != .input or !br.isBuffered() or br.is_trivial) return; if (step + 1 >= br.buffer_count) return; - if (step != 0) @panic("emitPipeline: num_stages>2 (multi-step initialize) unsupported (TODO)"); - // static_zero_start=true: Python-int 0 grid indices enable folds in getDmaSlice. - br.copyIn(b, self.indices(), b.lift(@as(i32, 0)), true); + if (br.copy_in_slot == null) br.copy_in_slot = b.lift(@as(i32, 0)); + + const pred = if (step == 0) self.first_step else blk: { + const block_changed = indicesDiffer(b, br, self.fetchIndices(step), self.fetchIndices(step - 1)); + break :blk b.andi(self.first_step, block_changed); + }; + const fetch_indices = self.fetchIndices(step); + if (step == 0) { + // static_zero_start=true: Python-int 0 grid indices enable folds in getDmaSlice. + br.copyIn(b, fetch_indices, br.copy_in_slot.?, true); + } else { + var when = b.openIf(pred); + { + br.copyIn(b, fetch_indices, br.copy_in_slot.?, false); + when.yieldThen(.{}); + } + } + br.copy_in_slot = advance(b, br.copy_in_slot.?, pred); } fn copyIn(self: *Scheduler, b: *Builder, br: *BufferedRef) void { @@ -744,7 +789,7 @@ const Scheduler = struct { var when = b.openIf(pred); { self.namedScope(b, "ep_copy_in"); - br.copyIn(b, self.nextIndices(), br.copy_in_slot.?, false); + br.copyIn(b, self.fetchIndices(br.buffer_count - 1), br.copy_in_slot.?, false); self.endScope(b); when.yieldThen(.{}); } @@ -844,16 +889,54 @@ pub fn dmaSemRefTy(b: *Builder, n: ?usize) *const mlir.Type { return mlir.Type.memRef(tpu.dmaSemaphoreType(b.ctx), shape, null, MemorySpace.semaphore_mem.attribute(b.ctx)); } -fn nextIndex1D(b: *Builder, i: Value, n: Value) Value { - const inc = b.addi(i, b.lift(@as(i32, 1))); - const carry = b.cmpi(.eq, inc, n); - return b.select(carry, b.lift(@as(i32, 0)), inc); +fn gridSize(b: *Builder, grid: []const Value) Value { + var acc = grid[0]; + for (grid[1..]) |g| acc = b.muli(acc, g); + return acc; } -fn prevIndex1D(b: *Builder, i: Value, n: Value) Value { - const dec = b.subi(i, b.lift(@as(i32, 1))); - const borrow = b.cmpi(.eq, dec, b.lift(@as(i32, -1))); - return b.select(borrow, b.subi(n, b.lift(@as(i32, 1))), dec); +fn filterIndices(b: *Builder, indices: []const Value, grid: []const Value) [4]Value { + var out: [4]Value = undefined; + const c0 = b.lift(@as(i32, 0)); + for (indices, grid, 0..) |i, g, pos| { + out[pos] = if (g.asConstantInt() == @as(?i64, 1)) c0 else i; + } + return out; +} + +fn nextIndexND(b: *Builder, indices: []const Value, grid: []const Value) []const Value { + const a = b.arena.allocator(); + const out = a.alloc(Value, indices.len) catch @panic("nextIndexND OOM"); + const c0 = b.lift(@as(i32, 0)); + var carry: Value = b.cmpi(.eq, c0, c0); + var rev: usize = 0; + while (rev < indices.len) : (rev += 1) { + const pos = indices.len - 1 - rev; + const inc = b.select(carry, b.addi(indices[pos], b.lift(@as(i32, 1))), indices[pos]); + carry = b.cmpi(.eq, inc, grid[pos]); + out[pos] = b.select(carry, c0, inc); + } + const filtered = filterIndices(b, out, grid); + for (0..indices.len) |i| out[i] = filtered[i]; + return out; +} + +fn prevIndexND(b: *Builder, indices: []const Value, grid: []const Value) []const Value { + const a = b.arena.allocator(); + const out = a.alloc(Value, indices.len) catch @panic("prevIndexND OOM"); + const c0 = b.lift(@as(i32, 0)); + var borrow: Value = b.cmpi(.eq, c0, c0); + const cm1 = b.lift(@as(i32, -1)); + var rev: usize = 0; + while (rev < indices.len) : (rev += 1) { + const pos = indices.len - 1 - rev; + const dec = b.select(borrow, b.subi(indices[pos], b.lift(@as(i32, 1))), indices[pos]); + borrow = b.cmpi(.eq, dec, cm1); + out[pos] = b.select(borrow, b.subi(grid[pos], b.lift(@as(i32, 1))), dec); + } + const filtered = filterIndices(b, out, grid); + for (0..indices.len) |i| out[i] = filtered[i]; + return out; } /// `null` block_shape, or every dim is `.blocked(full_dim)`. diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 393d3bff..6daf1d04 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -150,6 +150,18 @@ zig_library( }), ) +zig_library( + name = "moe_mosaic_tpu_gmm_ep", + tags = ["manual"], + main = "moe/mosaic_tpu_kernels/gmm_ep.zig", + import_name = "zml/moe/mosaic_tpu_kernels/gmm_ep", + visibility = ["//visibility:public"], + deps = [ + ":zml", + "//kernels/mosaic_tpu:mosaic_tpu_builder", + ], +) + zig_test( name = "test", diff --git a/zml/moe/mosaic_tpu_kernels/gmm_ep.zig b/zml/moe/mosaic_tpu_kernels/gmm_ep.zig new file mode 100644 index 00000000..c2716f2c --- /dev/null +++ b/zml/moe/mosaic_tpu_kernels/gmm_ep.zig @@ -0,0 +1,450 @@ +const std = @import("std"); + +const zml = @import("zml"); +const mtt = @import("kernels/mosaic_tpu/builder"); +const pipeline = mtt.pipeline; +const mtt_kernel = zml.kernel.mosaic_tpu; + +fn cdiv(x: i64, y: i64) i64 { + return @divFloor(x + y - 1, y); +} + +fn alignTo(x: i64, y: i64) i64 { + return cdiv(x, y) * y; +} + +fn floorModI32(b: *mtt.Builder, x: mtt.Value, y: mtt.Value) mtt.Value { + const c0 = b.lift(@as(i32, 0)); + const r = b.remsi(x, y); + const r_nz = b.cmpi(.ne, r, c0); + const r_neg = b.cmpi(.slt, r, c0); + return b.select(b.andi(r_neg, r_nz), b.addi(r, y), r); +} + +fn floorDivPosI32(b: *mtt.Builder, x: mtt.Value, y: mtt.Value) mtt.Value { + const c0 = b.lift(@as(i32, 0)); + const c1 = b.lift(@as(i32, 1)); + const q = b.divsi(x, y); + const sign = b.subi( + b.extui(b.cmpi(.sgt, x, c0), .i32), + b.extui(b.cmpi(.slt, x, c0), .i32), + ); + const sign_not_pos = b.cmpi(.ne, sign, c1); + const rem_nz = b.cmpi(.ne, b.remsi(x, y), c0); + return b.select(b.andi(sign_not_pos, rem_nz), b.subi(q, c1), q); +} + +fn ceilDivPosI32(b: *mtt.Builder, x: mtt.Value, y: mtt.Value) mtt.Value { + return b.divsi(b.addi(x, b.subi(y, b.lift(@as(i32, 1)))), y); +} + +pub const Cfg = struct { + size_m: i64, + size_k: i64, + size_n: i64, + size_group: i64, + size_lhs_group: i64, + tile_m: i64, + tile_k: i64, + tile_n: i64, + lhs_dtype: mtt.DType = .bf16, + rhs_dtype: mtt.DType = .bf16, + out_dtype: mtt.DType = .bf16, + acc_dtype: mtt.DType = .f32, + size_lhs_sublane: i64 = 8, + + pub fn alignedN(self: Cfg) i64 { + return alignTo(self.size_n, 128); + } + + pub fn maxNumGm(self: Cfg) i64 { + return self.size_group + cdiv(self.size_m, self.tile_m) - 1; + } +}; + +fn validateConfig(cfg: Cfg) void { + std.debug.assert(cfg.size_group <= cfg.size_lhs_group); + std.debug.assert(cfg.size_m > 0); + std.debug.assert(cfg.size_k > 0); + std.debug.assert(cfg.size_n > 0); + std.debug.assert(cfg.tile_m > 0); + std.debug.assert(cfg.tile_k > 0); + std.debug.assert(cfg.tile_n > 0); + std.debug.assert(cfg.size_lhs_sublane > 0); + std.debug.assert(@mod(cfg.size_m, cfg.size_lhs_sublane) == 0); + std.debug.assert(@mod(cfg.size_k, cfg.tile_k) == 0); + std.debug.assert(@mod(cfg.alignedN(), cfg.tile_n) == 0); + std.debug.assert(cfg.acc_dtype == .f32); +} + +const Meta = struct { + group_id: mtt.Value, + m_offset: mtt.Value, +}; + +const PipeCtx = struct { + cfg: Cfg, + meta: Meta, + num_gm: mtt.Value, +}; + +fn winRef(b: *mtt.Builder, rw: pipeline.RefWindow, shape: []const i64) mtt.Value { + const buf = rw.buf.?; + const slot = rw.slot orelse return buf; + const a = b.arena.allocator(); + const rank = shape.len + 1; + const base = a.alloc(mtt.Value, rank) catch @panic("gmm_ep winRef OOM"); + const result_shape = a.alloc(i64, rank) catch @panic("gmm_ep winRef OOM"); + base[0] = slot; + result_shape[0] = 1; + for (shape, 0..) |d, i| { + base[i + 1] = b.lift(@as(i32, 0)); + result_shape[i + 1] = d; + } + const sliced = b.memRefSlice(buf, base, result_shape, &.{}); + return b.memRefSqueeze(sliced, shape); +} + +fn metaLoad(b: *mtt.Builder, ref: mtt.Value, idx: mtt.Value) mtt.Value { + return b.scalarLoad(ref, &.{b.toIndex(idx)}); +} + +fn lhsIndexMap(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopaque) []const pipeline.BlockIndex { + const ctx: *PipeCtx = @ptrCast(@alignCast(ctx_opaque.?)); + const cfg = ctx.cfg; + const gm_id = indices[1]; + const k_id = indices[2]; + + const m_start = metaLoad(b, ctx.meta.m_offset, gm_id); + const m_end = metaLoad(b, ctx.meta.m_offset, b.addi(gm_id, b.lift(@as(i32, 1)))); + const sublane = b.lift(@as(i32, @intCast(cfg.size_lhs_sublane))); + const row_start = floorDivPosI32(b, m_start, sublane); + const row_end = ceilDivPosI32(b, m_end, sublane); + const row_size = b.subi(row_end, row_start); + + const out = b.arena.allocator().alloc(pipeline.BlockIndex, 3) catch @panic("gmm_ep lhsIndexMap OOM"); + out[0] = .{ .slice = .{ .start = row_start, .size = row_size } }; + out[1] = .{ .scalar = b.lift(@as(i32, 0)) }; + out[2] = .{ .scalar = k_id }; + return out; +} + +fn rhsIndexMap(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopaque) []const pipeline.BlockIndex { + const ctx: *PipeCtx = @ptrCast(@alignCast(ctx_opaque.?)); + const group_id = metaLoad(b, ctx.meta.group_id, indices[1]); + const out = b.arena.allocator().alloc(pipeline.BlockIndex, 3) catch @panic("gmm_ep rhsIndexMap OOM"); + out[0] = .{ .scalar = group_id }; + out[1] = .{ .scalar = indices[2] }; + out[2] = .{ .scalar = indices[0] }; + return out; +} + +fn outIndexMap(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopaque) []const pipeline.BlockIndex { + const ctx: *PipeCtx = @ptrCast(@alignCast(ctx_opaque.?)); + const cfg = ctx.cfg; + const gm_id = indices[1]; + const n_id = indices[0]; + + const m_start = metaLoad(b, ctx.meta.m_offset, gm_id); + const m_end = metaLoad(b, ctx.meta.m_offset, b.addi(gm_id, b.lift(@as(i32, 1)))); + const sublane = b.lift(@as(i32, @intCast(cfg.size_lhs_sublane))); + const row_start = floorDivPosI32(b, m_start, sublane); + const capped_row_end = floorDivPosI32(b, m_end, sublane); + const last_row_end = ceilDivPosI32(b, m_end, sublane); + const row_end = b.select( + b.cmpi(.eq, gm_id, b.subi(ctx.num_gm, b.lift(@as(i32, 1)))), + last_row_end, + capped_row_end, + ); + const row_size = b.subi(row_end, row_start); + + const out = b.arena.allocator().alloc(pipeline.BlockIndex, 3) catch @panic("gmm_ep outIndexMap OOM"); + out[0] = .{ .slice = .{ .start = row_start, .size = row_size } }; + out[1] = .{ .scalar = b.lift(@as(i32, 0)) }; + out[2] = .{ .scalar = n_id }; + return out; +} + +fn indices1DTo3D(b: *mtt.Builder, indices: []const mtt.Value) []const mtt.Value { + const out = b.arena.allocator().alloc(mtt.Value, 3) catch @panic("gmm_ep indices1DTo3D OOM"); + out[0] = b.lift(@as(i32, 0)); + out[1] = indices[0]; + out[2] = b.lift(@as(i32, 0)); + return out; +} + +fn lhsIndexMap1D(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopaque) []const pipeline.BlockIndex { + return lhsIndexMap(b, indices1DTo3D(b, indices), ctx_opaque); +} + +fn rhsIndexMap1D(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopaque) []const pipeline.BlockIndex { + return rhsIndexMap(b, indices1DTo3D(b, indices), ctx_opaque); +} + +fn outIndexMap1D(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopaque) []const pipeline.BlockIndex { + return outIndexMap(b, indices1DTo3D(b, indices), ctx_opaque); +} + +fn fillMetadata(b: *mtt.Builder, cfg: Cfg, a: anytype) mtt.Value { + const c0 = b.lift(@as(i32, 0)); + const c1 = b.lift(@as(i32, 1)); + const group_offset = b.scalarLoad(a.group_offset, &.{b.cIndex(0)}); + const max_num_group = b.addi(group_offset, b.lift(@as(i32, @intCast(cfg.size_group)))); + b.scalarStore(a.metadata_m_offset, c0, &.{b.cIndex(0)}); + + var outer = b.openFor(c0, max_num_group, c1, .{ c0, c0 }); + { + b.traceStart(10, "outer_group_loop"); + const lhs_group_id = outer.iv; + const num_gm = outer.carried[0]; + const start_m_offset = outer.carried[1]; + const group_id = b.subi(lhs_group_id, group_offset); + const group_size = b.scalarLoad(a.group_sizes, &.{b.toIndex(lhs_group_id)}); + const end_m_offset = b.addi(start_m_offset, group_size); + const sublane = b.lift(@as(i32, @intCast(cfg.size_lhs_sublane))); + const local_offset = floorModI32(b, start_m_offset, sublane); + const aligned_group_size = b.addi(group_size, local_offset); + const tile_m = b.lift(@as(i32, @intCast(cfg.tile_m))); + var curr_num_gm = b.divsi(b.addi(aligned_group_size, b.subi(tile_m, c1)), tile_m); + const should_process = b.andi(b.cmpi(.sgt, group_size, c0), b.cmpi(.sge, group_id, c0)); + curr_num_gm = b.select(should_process, curr_num_gm, c0); + const next_num_gm = b.addi(num_gm, curr_num_gm); + + var inner = b.openFor(num_gm, next_num_gm, c1, .{start_m_offset}); + { + b.traceStart(10, "inner_tm_loop"); + const tm_id = inner.iv; + const curr_m_offset = inner.carried[0]; + const local = floorModI32(b, curr_m_offset, sublane); + const tm_room = b.subi(tile_m, local); + const remaining = b.subi(end_m_offset, curr_m_offset); + const tm_size = b.minsi(tm_room, remaining); + const next_m_offset = b.addi(curr_m_offset, tm_size); + b.scalarStore(a.metadata_group_id, group_id, &.{b.toIndex(tm_id)}); + b.scalarStore(a.metadata_m_offset, curr_m_offset, &.{b.toIndex(tm_id)}); + b.scalarStore(a.metadata_m_offset, next_m_offset, &.{b.toIndex(b.addi(tm_id, c1))}); + b.traceStop(); + inner.yield(.{next_m_offset}); + } + + b.traceStop(); + outer.yield(.{ next_num_gm, end_m_offset }); + } + return outer.results[0]; +} + +fn matmulByMxuColumns(b: *mtt.Builder, cfg: Cfg, lhs: mtt.Value, rhs: mtt.Value, cols: i64) mtt.Value { + if (cols <= 128) { + return b.matmulOpts(lhs, rhs, b.zeros(&.{ cfg.tile_m, cols }, cfg.acc_dtype), .{ + .dimension_numbers = b.dotDimensionNumbers(&.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}), + }); + } + + std.debug.assert(@mod(cols, 128) == 0); + const chunks = @divExact(cols, 128); + const parts = b.arena.allocator().alloc(mtt.Value, @intCast(chunks)) catch @panic("gmm_ep matmul parts OOM"); + var rev: i64 = 0; + while (rev < chunks) : (rev += 1) { + const chunk = chunks - 1 - rev; + const start_n = chunk * 128; + const rhs_chunk = b.vectorExtractStridedSlice(rhs, &.{ 0, start_n }, &.{ cfg.tile_k, 128 }, &.{ 1, 1 }); + const zero = b.zeros(&.{ cfg.tile_m, 128 }, cfg.acc_dtype); + parts[@intCast(chunk)] = b.matmulOpts(lhs, rhs_chunk, zero, .{ + .dimension_numbers = b.dotDimensionNumbers(&.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}), + }); + } + + const max_concat_inputs = 16; + if (chunks <= max_concat_inputs) return b.concatenate(parts, 1); + + const groups = cdiv(chunks, max_concat_inputs); + const group_parts = b.arena.allocator().alloc(mtt.Value, @intCast(groups)) catch @panic("gmm_ep matmul group parts OOM"); + var group: i64 = 0; + while (group < groups) : (group += 1) { + const start = group * max_concat_inputs; + const end = @min(start + max_concat_inputs, chunks); + const start_usize: usize = @intCast(start); + const end_usize: usize = @intCast(end); + const group_slice = parts[start_usize..end_usize]; + group_parts[@intCast(group)] = if (group_slice.len == 1) group_slice[0] else b.concatenate(group_slice, 1); + } + return b.concatenate(group_parts, 1); +} + +fn innerBody( + b: *mtt.Builder, + indices: []const mtt.Value, + refs: []const pipeline.RefWindow, + scratches: []const mtt.Value, + ctx_opaque: ?*anyopaque, +) mtt.FinishError!void { + const ctx: *PipeCtx = @ptrCast(@alignCast(ctx_opaque.?)); + const cfg = ctx.cfg; + const gm_id = indices[1]; + + const lhs_shape = &.{ cdiv(cfg.tile_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.tile_k }; + const out_shape = &.{ cdiv(cfg.tile_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.tile_n }; + + b.traceStart(10, "matmul_first_last"); + const lhs_ref = winRef(b, refs[0], lhs_shape); + const lhs_ref_2d = b.memRefReshape(lhs_ref, &.{ cfg.tile_m, cfg.tile_k }); + const lhs = b.refLoad(lhs_ref_2d); + const rhs_slot = b.toIndex(refs[1].slot.?); + const rhs = b.shapeCast( + b.vectorLoadShape(refs[1].buf.?, &.{ rhs_slot, b.cIndex(0), b.cIndex(0) }, &.{ 1, cfg.tile_k, cfg.tile_n }), + &.{ cfg.tile_k, cfg.tile_n }, + ); + var acc = matmulByMxuColumns(b, cfg, lhs, rhs, cfg.tile_n); + + const m_start = metaLoad(b, ctx.meta.m_offset, gm_id); + const m_end = metaLoad(b, ctx.meta.m_offset, b.addi(gm_id, b.lift(@as(i32, 1)))); + const sublane = b.lift(@as(i32, @intCast(cfg.size_lhs_sublane))); + const m_offset = b.subi(m_start, floorModI32(b, m_start, sublane)); + const m_start_local = b.subi(m_start, m_offset); + const m_end_local = b.subi(m_end, m_offset); + const row_iota = b.iota(&.{ cfg.tile_m, cfg.tile_n }, .i32, &.{0}); + const row_mask = b.andi( + b.cmpi(.sle, b.broadcastTo(m_start_local, &.{ cfg.tile_m, cfg.tile_n }), row_iota), + b.cmpi(.slt, row_iota, b.broadcastTo(m_end_local, &.{ cfg.tile_m, cfg.tile_n })), + ); + acc = b.select(row_mask, acc, b.zeros(&.{ cfg.tile_m, cfg.tile_n }, cfg.acc_dtype)); + + const out_slot_i32 = refs[2].slot.?; + const out_slot = b.toIndex(out_slot_i32); + b.vectorStoreAt( + refs[2].buf.?, + b.shapeCast(b.shapeCast(acc, out_shape).to(cfg.out_dtype), &.{ 1, cdiv(cfg.tile_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.tile_n }), + &.{ out_slot, b.cIndex(0), b.cIndex(0), b.cIndex(0) }, + ); + + const out_ref = winRef(b, refs[2], out_shape); + if (cfg.tile_m == cfg.size_lhs_sublane) { + const out_value = b.shapeCast(b.refLoad(out_ref), &.{ cfg.tile_m, cfg.tile_n }); + const partial_zero = b.zeros(&.{ cfg.tile_m, cfg.tile_n }, cfg.out_dtype); + const partial_prev = b.select( + b.cmpi(.eq, gm_id, b.lift(@as(i32, 0))), + partial_zero, + b.refLoad(scratches[0]), + ); + const out_accum = b.addf(out_value, partial_prev); + const out_ref_for_store = winRef(b, refs[2], out_shape); + b.refStore(out_ref_for_store, b.shapeCast(out_accum, out_shape)); + } else { + const first_row = b.shapeCast( + b.vectorLoadShape(out_ref, &.{ b.cIndex(0), b.cIndex(0), b.cIndex(0) }, &.{ 1, cfg.size_lhs_sublane, cfg.tile_n }), + &.{ cfg.size_lhs_sublane, cfg.tile_n }, + ); + const partial_zero = b.zeros(&.{ cfg.size_lhs_sublane, cfg.tile_n }, cfg.out_dtype); + const partial_prev = b.select( + b.cmpi(.eq, gm_id, b.lift(@as(i32, 0))), + partial_zero, + b.refLoad(scratches[0]), + ); + const out_accum = b.addf(first_row, partial_prev); + const out_ref_for_store = winRef(b, refs[2], out_shape); + b.vectorStoreAt( + out_ref_for_store, + b.shapeCast(out_accum, &.{ 1, cfg.size_lhs_sublane, cfg.tile_n }), + &.{ b.cIndex(0), b.cIndex(0), b.cIndex(0) }, + ); + } + + const last_row = floorDivPosI32(b, m_end_local, sublane); + const m_end_mod = floorModI32(b, m_end_local, sublane); + const partial_complete = b.cmpi(.eq, m_end_mod, b.lift(@as(i32, 0))); + const out_ref_for_partial = winRef(b, refs[2], out_shape); + const partial_shape = if (cfg.tile_m == cfg.size_lhs_sublane) + &.{ cfg.tile_m, cfg.tile_n } + else + &.{ cfg.size_lhs_sublane, cfg.tile_n }; + const partial_zero = b.zeros(partial_shape, cfg.out_dtype); + const partial_row = b.shapeCast( + b.vectorLoadShape(out_ref_for_partial, &.{ b.toIndex(last_row), b.cIndex(0), b.cIndex(0) }, &.{ 1, cfg.size_lhs_sublane, cfg.tile_n }), + partial_shape, + ); + b.refStore(scratches[0], b.select(partial_complete, partial_zero, partial_row)); + b.traceStop(); +} + +fn innerBody1D( + b: *mtt.Builder, + indices: []const mtt.Value, + refs: []const pipeline.RefWindow, + scratches: []const mtt.Value, + ctx_opaque: ?*anyopaque, +) mtt.FinishError!void { + try innerBody(b, indices1DTo3D(b, indices), refs, scratches, ctx_opaque); +} + +pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { + validateConfig(cfg); + + const a = try b.declareArgsOpts(.{ + .group_sizes = .{ .ref = .{ .shape = &.{cfg.size_lhs_group}, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, + .group_offset = .{ .ref = .{ .shape = &.{1}, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, + .lhs = .{ .ref = .{ .shape = &.{ cfg.size_m, cfg.size_k }, .dtype = cfg.lhs_dtype, .memory_space = .hbm } }, + .rhs = .{ .ref = .{ .shape = &.{ cfg.size_group, cfg.size_k, cfg.size_n }, .dtype = cfg.rhs_dtype, .memory_space = .hbm } }, + .out = .{ .ref = .{ .shape = &.{ cfg.size_m, cfg.alignedN() }, .dtype = cfg.out_dtype, .memory_space = .hbm, .role = .output } }, + .partial_out = .{ .ref = .{ .shape = &.{ cfg.size_lhs_sublane, cfg.tile_n }, .dtype = cfg.out_dtype, .role = .scratch } }, + .acc = .{ .ref = .{ .shape = &.{ cfg.tile_m, cfg.tile_n }, .dtype = cfg.acc_dtype, .role = .scratch } }, + .metadata_group_id = .{ .ref = .{ .shape = &.{cfg.maxNumGm()}, .dtype = .i32, .memory_space = .smem, .role = .scratch } }, + .metadata_m_offset = .{ .ref = .{ .shape = &.{cfg.maxNumGm() + 1}, .dtype = .i32, .memory_space = .smem, .role = .scratch } }, + }, &.{}, .{ + .dimension_semantics = &.{}, + .scalar_prefetch = 2, + .scratch_operands = 4, + }); + + const num_k = b.lift(@as(i32, @intCast(cdiv(cfg.size_k, cfg.tile_k)))); + const num_n = b.lift(@as(i32, @intCast(cdiv(cfg.alignedN(), cfg.tile_n)))); + const num_gm = fillMetadata(b, cfg, a); + + const ctx = b.arena.allocator().create(PipeCtx) catch @panic("gmm_ep PipeCtx OOM"); + ctx.* = .{ + .cfg = cfg, + .meta = .{ .group_id = a.metadata_group_id, .m_offset = a.metadata_m_offset }, + .num_gm = num_gm, + }; + + const lhs_spec = pipeline.PipeSpec{ + .full_shape = &.{ @divExact(cfg.size_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.size_k }, + .dtype = cfg.lhs_dtype, + .source_reshape_shape = &.{ @divExact(cfg.size_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.size_k }, + .block_shape = &.{ .{ .bounded_slice = cdiv(cfg.tile_m, cfg.size_lhs_sublane) }, .{ .blocked = cfg.size_lhs_sublane }, .{ .blocked = cfg.tile_k } }, + .index_map = if (cdiv(cfg.alignedN(), cfg.tile_n) == 1 and cdiv(cfg.size_k, cfg.tile_k) == 1 and cfg.size_m > cfg.tile_m) lhsIndexMap1D else lhsIndexMap, + .index_map_ctx = ctx, + .bounded_slice_tiling = 1, + }; + const rhs_spec = pipeline.PipeSpec{ + .full_shape = &.{ cfg.size_group, cfg.size_k, cfg.size_n }, + .dtype = cfg.rhs_dtype, + .block_shape = &.{ .squeezed, .{ .blocked = cfg.tile_k }, .{ .blocked = cfg.tile_n } }, + .index_map = if (cdiv(cfg.alignedN(), cfg.tile_n) == 1 and cdiv(cfg.size_k, cfg.tile_k) == 1 and cfg.size_m > cfg.tile_m) rhsIndexMap1D else rhsIndexMap, + .index_map_ctx = ctx, + .buffer_count = 3, + }; + const out_spec = pipeline.PipeSpec{ + .full_shape = &.{ @divExact(cfg.size_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.alignedN() }, + .dtype = cfg.out_dtype, + .source_reshape_shape = &.{ @divExact(cfg.size_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.alignedN() }, + .block_shape = &.{ .{ .bounded_slice = cdiv(cfg.tile_m, cfg.size_lhs_sublane) }, .{ .blocked = cfg.size_lhs_sublane }, .{ .blocked = cfg.tile_n } }, + .index_map = if (cdiv(cfg.alignedN(), cfg.tile_n) == 1 and cdiv(cfg.size_k, cfg.tile_k) == 1 and cfg.size_m > cfg.tile_m) outIndexMap1D else outIndexMap, + .index_map_ctx = ctx, + .bounded_slice_tiling = 1, + }; + const use_1d_pipeline = cdiv(cfg.alignedN(), cfg.tile_n) == 1 and cdiv(cfg.size_k, cfg.tile_k) == 1 and cfg.size_m > cfg.tile_m; + try pipeline.emitPipeline(b, &.{ a.lhs, a.rhs, a.out }, &.{ a.partial_out, a.acc, a.metadata_group_id, a.metadata_m_offset }, .{ + .body = if (use_1d_pipeline) innerBody1D else innerBody, + .body_ctx = ctx, + .grid = if (use_1d_pipeline) &.{num_gm} else &.{ num_n, num_gm, num_k }, + .in_specs = &.{ lhs_spec, rhs_spec }, + .out_specs = &.{out_spec}, + }); +} + +pub const TpuKernel = mtt_kernel.Kernel(Cfg, .{ + .name = "gmm_ep_kernel", + .inputs = &.{ "group_sizes", "group_offset", "lhs", "rhs" }, + .outputs = &.{"out"}, + .run = run, +}); From 39ffd27f6f7e5bb96532583f65bb13d1cff08671 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 4 Jun 2026 15:32:17 +0200 Subject: [PATCH 511/881] zml/module: expose autotune cache dir through an env var (#570) This opt-in flag allows XLA to save the autotuning resulting from previous runs in a folder, greatly speeding up the compilation. It's not meant to be used at all times, so it's undocumented and opt-in. --- zml/module.zig | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/zml/module.zig b/zml/module.zig index 6bdc2b39..51199028 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -663,6 +663,13 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: } } + switch (platform.target) { + .rocm, .cuda => if (std.c.getenv("ZML_AUTOTUNE_CACHE_DIR")) |path| { + try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_autotuner_cache_dir", std.mem.span(path), upb_arena); + }, + else => {}, + } + break :blk options; }; From c5bdcac63205c9ce27c3703edcb26508f1d81dca Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 4 Jun 2026 19:04:34 +0200 Subject: [PATCH 512/881] zml/sharding: fix max_rank for TPU (#572) Topology coords on TPU have rank 4 and not 3 --- zml/Sharding.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/Sharding.zig b/zml/Sharding.zig index 89c5b303..22f5e919 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -931,7 +931,7 @@ pub const PhysicalMesh = struct { const placements = try CoordsTopology.collect(allocator, platform_devices); defer allocator.free(placements); - const layout = try CoordsTopology.layout(placements, 3); + const layout = try CoordsTopology.layout(placements, 4); var rank = layout.rank; var axis_sizes = layout.axis_sizes; From 5d4e4db01c5c8f37727f4305d0295e79a92482c2 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 9 Jun 2026 16:10:34 +0200 Subject: [PATCH 513/881] zml/profiling/tracer: Add simpler infaillible span (#573) Renamed `scope` to `span` to match OpenTelemetry convention I'm more familiar with. Traceme doesn't seem to use `Scope` explicitly but talks more about a scope object AFAIK. Now `span` fails at compile time. if it cannot compute the appropriate buffer size for the name. In that case the caller should use `formatSpanName` which allocates. This makes `span` infaillible at runtime. --- zml/exe.zig | 12 +-- zml/io.zig | 4 +- zml/module.zig | 6 +- zml/profiling/tracer.zig | 166 ++++++++++++++++++++++++++++----------- 4 files changed, 130 insertions(+), 58 deletions(-) diff --git a/zml/exe.zig b/zml/exe.zig index f077b009..ba06e3d2 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -306,22 +306,22 @@ pub const Exe = struct { }; pub fn callOpts(self: *const Exe, io: std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { - var trace = tracer.scope("zml.exe.call", .{ + var span = tracer.span("zml.exe.call", .{ .wait = opts.wait, .arg_count = arguments.expected_shapes.len, .result_count = results_.expected_shapes.len, - }) catch unreachable; - defer trace.end(); + }); + defer span.end(); return self.internalCall(io, arguments, results_, opts); } pub fn call(self: *const Exe, arguments: Arguments, results_: *Results) void { - var trace = tracer.scope("zml.exe.call", .{ + var span = tracer.span("zml.exe.call", .{ .wait = false, .arg_count = arguments.expected_shapes.len, .result_count = results_.expected_shapes.len, - }) catch unreachable; - defer trace.end(); + }); + defer span.end(); return self.internalCall(null, arguments, results_, .{}); } }; diff --git a/zml/io.zig b/zml/io.zig index 8b61fbd9..b3c2c993 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -1071,10 +1071,10 @@ pub fn load( store: *const TensorStore, opts: LoadOpts, ) !Bufferized(ModelType) { - var trace = try tracer.scope("zml.io.load", .{ + var span = tracer.span("zml.io.load", .{ .tensor_count = meta.count(Tensor, model), }); - defer trace.end(); + defer span.end(); var bufferized = try mem.bufferize(allocator, ModelType, model); diff --git a/zml/module.zig b/zml/module.zig index 51199028..b1440eb4 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -209,11 +209,13 @@ pub fn compile( var st_io: std.Io.Threaded = .init_single_threaded; defer st_io.deinit(); - var trace = try tracer.scope("zml.module.compile", .{ + const span_name = try tracer.formatSpanName(allocator, "zml.module.compile", .{ .program_name = opts.program_name, .arg_count = args.len, }); - defer trace.end(); + defer allocator.free(span_name); + var span = tracer.Span.start(span_name); + defer span.end(); var compilation_context: CompilationContext = .init(allocator, st_io.io(), platform, opts); defer compilation_context.deinit(); diff --git a/zml/profiling/tracer.zig b/zml/profiling/tracer.zig index 4b93fd80..e3157d10 100644 --- a/zml/profiling/tracer.zig +++ b/zml/profiling/tracer.zig @@ -5,19 +5,22 @@ const c = @import("c"); const platforms = @import("platforms"); const zffi = @import("ffi"); -pub const Scope = struct { +pub const Span = struct { inner: ?*c.zml_traceme = null, - pub fn end(self: *Scope) void { + /// Name is copied underneath + pub fn start(name: []const u8) Span { + return .{ + .inner = c.zml_traceme_start(zffi.ZigSlice.from(name)), + }; + } + + pub fn end(self: *Span) void { if (self.inner) |inner| { c.zml_traceme_stop(inner); self.inner = null; } } - - pub fn deinit(self: *Scope) void { - self.end(); - } }; // TraceMe is always the base host tracing path. The C tracing bridge adds NVTX @@ -28,44 +31,55 @@ pub const supportsDeviceAnnotations = switch (builtin.os.tag) { .linux => platforms.target == .cuda or platforms.target == .rocm, else => false, }; -const inline_scope_name_buffer_size = 1024; pub fn enabled() bool { return c.zml_traceme_enabled(); } -pub fn scope(name: []const u8, metadata: anytype) !Scope { - const Metadata = @TypeOf(metadata); - comptime assertMetadataStruct(Metadata); +/// Creates a Span with encoded metadata. The metadata encoded size must be known at compile time. +/// If it's not your case, use `formatSpanName` to generate the span name before and use void for the metadata. +pub fn span(comptime name: []const u8, metadata: anytype) Span { + switch (@typeInfo(@TypeOf(metadata))) { + .void => { + return .start(name); + }, + .@"struct" => |info| { + if (info.fields.len == 0) { + return .start(name); + } - if (comptime metadataFieldCount(Metadata) == 0) { - return .{ - .inner = c.zml_traceme_start(zffi.ZigSlice.from(name)), - }; + var buffer: [computeEncodedNameLen(name, @TypeOf(metadata))]u8 = undefined; + var encoded: std.ArrayList(u8) = .initBuffer(&buffer); + + appendEncodedName(&encoded, .failing, name, info, metadata) catch unreachable; + return .start(encoded.items); + }, + else => @compileError("Unsupported metadata."), } - - var inline_buffer: [inline_scope_name_buffer_size]u8 = undefined; - var fixed_buffer_allocator: std.heap.FixedBufferAllocator = .init(&inline_buffer); - const fixed_allocator = fixed_buffer_allocator.allocator(); - - var encoded: std.ArrayList(u8) = .empty; - defer encoded.deinit(fixed_allocator); - - try appendEncodedName(&encoded, fixed_allocator, name, metadata); - return .{ - .inner = c.zml_traceme_start(zffi.ZigSlice.from(encoded.items)), - }; } -fn appendEncodedName(buffer: *std.ArrayList(u8), allocator: std.mem.Allocator, name: []const u8, metadata: anytype) !void { - const Metadata = @TypeOf(metadata); +/// Returns an allocator-owned encoded span name for metadata that cannot be +/// bounded by `span` at comptime. The caller owns the returned slice. +pub fn formatSpanName(allocator: std.mem.Allocator, name: []const u8, metadata: anytype) ![]const u8 { + switch (@typeInfo(@TypeOf(metadata))) { + .@"struct" => |info| { + var encoded: std.ArrayList(u8) = .empty; + errdefer encoded.deinit(allocator); + + try appendEncodedName(&encoded, allocator, name, info, metadata); + return try encoded.toOwnedSlice(allocator); + }, + else => @compileError("trace metadata must be a struct literal like .{ .step_num = 42 }"), + } +} + +fn appendEncodedName(buffer: *std.ArrayList(u8), allocator: std.mem.Allocator, name: []const u8, info: std.builtin.Type.Struct, metadata: anytype) !void { try buffer.appendSlice(allocator, name); - if (comptime metadataFieldCount(Metadata) == 0) return; var field_count: usize = 0; try buffer.append(allocator, '#'); - inline for (std.meta.fields(Metadata)) |field| { - if (shouldEncodeField(@TypeOf(@field(metadata, field.name)), @field(metadata, field.name), field.name)) { + inline for (info.fields) |field| { + if (shouldEncodeField(field, @field(metadata, field.name))) { if (field_count != 0) { try buffer.append(allocator, ','); } @@ -78,11 +92,13 @@ fn appendEncodedName(buffer: *std.ArrayList(u8), allocator: std.mem.Allocator, n try appendMetadataValue(buffer, allocator, @field(metadata, field.name), field.name); } } + + // We may not have written any fields because of optional values. if (field_count == 0) { buffer.items.len = name.len; - return; + } else { + try buffer.append(allocator, '#'); } - try buffer.append(allocator, '#'); } fn validateMetadataToken(token: []const u8) void { @@ -91,26 +107,59 @@ fn validateMetadataToken(token: []const u8) void { } } -fn assertMetadataStruct(comptime T: type) void { - switch (@typeInfo(T)) { - .@"struct" => {}, - else => @compileError("trace metadata must be a struct literal like .{ .step_num = 42 }"), - } -} - -fn metadataFieldCount(comptime T: type) usize { - assertMetadataStruct(T); - return std.meta.fields(T).len; -} - fn metadataKey(comptime field_name: []const u8) []const u8 { return if (comptime std.mem.eql(u8, field_name, "root")) "_r" else field_name; } -fn shouldEncodeField(comptime T: type, value: T, comptime field_name: []const u8) bool { - return switch (@typeInfo(T)) { +fn computeEncodedNameLen(comptime name: []const u8, comptime Metadata: type) usize { + switch (@typeInfo(Metadata)) { + .@"struct" => |info| { + if (info.fields.len == 0) return name.len; + + // name#[attrs]# + var max_len = name.len + 2; + inline for (info.fields) |field| { + const key = comptime metadataKey(field.name); + validateMetadataToken(key); + + // [key]=[value], + max_len += key.len + 2; + max_len += switch (@typeInfo(field.type)) { + .bool => 1, + .int => @max( + std.fmt.comptimePrint("{}", .{std.math.minInt(field.type)}).len, + std.fmt.comptimePrint("{}", .{std.math.maxInt(field.type)}).len, + ), + .@"enum" => |ty| b: { + var max_enum_len: usize = 0; + for (ty.fields) |enum_field| { + max_enum_len = @max(max_enum_len, enum_field.name.len); + } + break :b max_enum_len; + }, + .comptime_int => std.fmt.comptimePrint("{}", .{defaultMetadataValue(field).?}).len, + .comptime_float => std.fmt.comptimePrint("{d}", .{defaultMetadataValue(field).?}).len, + .enum_literal => @tagName(defaultMetadataValue(field).?).len, + else => @compileError("trace metadata field '" ++ field.name ++ "' is not statically bounded; use formatspanName"), + }; + } + return max_len; + }, + else => @compileError("trace metadata must be a struct literal like .{ .step_num = 42 }"), + } +} + +fn defaultMetadataValue(comptime field: std.builtin.Type.StructField) ?field.type { + return if (field.default_value_ptr) |default_opaque| + @as(*const field.type, @ptrCast(@alignCast(default_opaque))).* + else + null; +} + +fn shouldEncodeField(field: std.builtin.Type.StructField, value: anytype) bool { + return switch (@typeInfo(field.type)) { .optional => value != null, - .bool => if (comptime std.mem.eql(u8, field_name, "root")) value else true, + .bool => if (comptime std.mem.eql(u8, field.name, "root")) value else true, else => true, }; } @@ -163,3 +212,24 @@ fn appendMetadataValue(buffer: *std.ArrayList(u8), allocator: std.mem.Allocator, else => @compileError("unsupported trace metadata type for field '" ++ field_name ++ "'"), } } + +test "formatspanName encodes metadata" { + const formatted = try formatSpanName(std.testing.allocator, "test.span", .{ + .root = true, + .count = @as(u8, 42), + .label = "batch", + }); + defer std.testing.allocator.free(formatted); + + try std.testing.expectEqualStrings("test.span#_r=1,count=42,label=batch#", formatted); +} + +test "formatspanName omits empty metadata" { + const formatted = try formatSpanName(std.testing.allocator, "test.span", .{ + .root = false, + .optional = @as(?u8, null), + }); + defer std.testing.allocator.free(formatted); + + try std.testing.expectEqualStrings("test.span", formatted); +} From 1278805b8740c148733945a9bfdcf160d268db3e Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Fri, 12 Jun 2026 15:03:34 +0200 Subject: [PATCH 514/881] zml/pjrtx: alignCast the host buffer for host offloading (#575) --- zml/pjrtx.zig | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 038d5682..f492b120 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -76,15 +76,23 @@ pub const CustomCallBuffer = struct { }; } + // XLA: + // Align to 64-bytes, to mimic tsl::Allocator::kAllocatorAlignment. + // + // Preferred XLA:CPU alignment for buffers. XLA:CPU itself aligns intermediate + // buffers to this boundary (slices inside the preallocated temp buffer). + // https://github.com/openxla/xla/blob/fa2b7cfd909e8ffd343da1b7c5c341ace5e53aaa/xla/backends/cpu/alignment.h#L4 + pub const ALIGN = 64; + // Assumes as host-side buffer - pub fn asHostSlice(self: CustomCallBuffer) []u8 { - const bytes: [*]u8 = @ptrCast(@alignCast(self.ptr)); + pub fn asHostSlice(self: CustomCallBuffer) []align(ALIGN) u8 { + const bytes: [*]align(ALIGN) u8 = @ptrCast(@alignCast(self.ptr)); return bytes[0..self.shape.byteSize()]; } // Assumes as host-side buffer - pub fn asHostConstSlice(self: CustomCallBuffer) []const u8 { - const bytes: [*]const u8 = @ptrCast(@alignCast(self.ptr)); + pub fn asHostConstSlice(self: CustomCallBuffer) []align(ALIGN) const u8 { + const bytes: [*]align(ALIGN) const u8 = @ptrCast(@alignCast(self.ptr)); return bytes[0..self.shape.byteSize()]; } From be29637c072c6dc034f833a419b5842c17f745ea Mon Sep 17 00:00:00 2001 From: Louis Lechevalier Date: Tue, 16 Jun 2026 10:06:52 +0200 Subject: [PATCH 515/881] zml/moe: Mosaic TPU MoE backend. (#571) This PR add the TPU backend for MoE. It uses the GMM Expert parallelism kernel to perform the grouped gemms on gate_up and down weights (these weights must have a precise shape to run optimally). This backend is currently optimized for small batches. A specialized mosaic tpu kernel will be introduced to handle bigger ones. This implementation come with few fixes concerning the TPU sharding and the Qwen 3.5 moe model which uses the backend : - TPU sharding : PJRT reports TPU device coordinates in a max rank form. So it breaks folding. To avoid this we compute the Rank by not counting where coords =0. - Linear attention was reading padding token in the Qwen 3.5 model. So we introduce masking of them. n.b. : The kv head repetition to satisfy the sharding will be handled a more generic way in a next PR --------- Co-authored-by: Corentin Godeau --- examples/llm/models/common.zig | 21 +- examples/llm/models/qwen3_5_moe/inference.zig | 3 +- examples/llm/models/qwen3_5_moe/model.zig | 81 ++-- examples/llm/models/qwen3_5_moe/session.zig | 4 +- kernels/mosaic_tpu/pipeline.zig | 11 +- zml/BUILD.bazel | 2 + zml/moe/moe.zig | 130 ++++++- zml/moe/mosaic_tpu.zig | 368 ++++++++++++++++++ zml/moe/mosaic_tpu_kernels/gmm_ep.zig | 137 ++++++- zml/moe/triton.zig | 20 +- 10 files changed, 711 insertions(+), 66 deletions(-) create mode 100644 zml/moe/mosaic_tpu.zig diff --git a/examples/llm/models/common.zig b/examples/llm/models/common.zig index 7de783c2..dcc2ac77 100644 --- a/examples/llm/models/common.zig +++ b/examples/llm/models/common.zig @@ -47,10 +47,23 @@ pub const Shardings = struct { experts: zml.Sharding, pub fn init(platform: *zml.Platform) !Shardings { - return .{ - .model = try platform.registerSharding("model", .mesh(.{ .model = .high_bandwidth })), - .experts = try platform.registerSharding("experts", .mesh(.{ .experts = .high_bandwidth })), - }; + switch (platform.target) { + .tpu => { + var strategy_experts: zml.Sharding.Strategy = .parseBindings(.{ .experts = .link_x }); + strategy_experts.addFold(.link_x, &.{ .link_x, .link_y }); + var strategy_model: zml.Sharding.Strategy = .parseBindings(.{ .model = .link_x }); + strategy_model.addFold(.link_x, &.{ .link_x, .link_y }); + + return .{ + .model = try platform.registerShardingWithStrategy("model", .mesh(.{ .model = .high_bandwidth }), strategy_model), + .experts = try platform.registerShardingWithStrategy("experts", .mesh(.{ .experts = .high_bandwidth }), strategy_experts), + }; + }, + .cuda, .rocm, .oneapi, .neuron, .cpu => return .{ + .model = try platform.registerSharding("model", .mesh(.{ .model = .high_bandwidth })), + .experts = try platform.registerSharding("experts", .mesh(.{ .experts = .high_bandwidth })), + }, + } } pub fn all(self: Shardings) [2]zml.Sharding { diff --git a/examples/llm/models/qwen3_5_moe/inference.zig b/examples/llm/models/qwen3_5_moe/inference.zig index 2d475ec3..6431c4a2 100644 --- a/examples/llm/models/qwen3_5_moe/inference.zig +++ b/examples/llm/models/qwen3_5_moe/inference.zig @@ -36,7 +36,7 @@ pub const CompilationParameters = struct { .rng = .init(), .prefill_moe_metadata = initMoeMetadata(mdl, @intCast(seqlen), 1, moe_backend), .decode_moe_metadata = initMoeMetadata(mdl, 1, 1, moe_backend), - .moe_parameters = .init(.fromBackend(moe_backend, config.text_config.num_experts_per_tok, zml.moe.triton.Parameters.ActivationMode.silu)), + .moe_parameters = .init(.fromBackend(moe_backend, config.text_config.num_experts_per_tok, zml.moe.ActivationMode.silu)), .seqlen = seqlen, .shardings = shardings, .xla_dump_to = "/home/ubuntu/xla_dump", @@ -476,5 +476,6 @@ fn initMoeMetadata(qwen_model: model.Model, token_len: usize, batch_size: u32, b .w2_zero_bias_shape = w2_zero_bias_shape, }, }), + .mosaic_tpu => .init(.fromBackend(backend)), }; } diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index fc27eea7..9bc35b3f 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -49,31 +49,28 @@ pub const RopeParameters = struct { rope_theta: f32, }; -fn kvHeadsAreReplicated(axis_size: i64, model_partitions: i64) struct { bool, ?u32 } { - const is_replicated = axis_size > 0 and axis_size < model_partitions and @rem(model_partitions, axis_size) == 0; - const repeat_factor = if (is_replicated) @as(u32, @intCast(@divExact(model_partitions, axis_size))) else null; - return .{ is_replicated, repeat_factor }; +fn kvHeadRepeatFactor(axis_size: i64, model_partitions: i64) ?u32 { + if (axis_size <= 0 or axis_size >= model_partitions or @rem(model_partitions, axis_size) != 0) return null; + return @intCast(@divExact(model_partitions, axis_size)); } -fn partitionProjectedKv(tensor: zml.Tensor, replicate_kv_heads: bool, repeat_factor: ?u32) zml.Tensor { - return if (replicate_kv_heads) - tensor.repeat1d(.h, @as(u63, repeat_factor.?)).withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }) +fn partitionProjectedKv(tensor: zml.Tensor, repeat_factor: ?u32) zml.Tensor { + return if (repeat_factor) |factor| + tensor.stutter1d(tensor.axis(.h), @as(u63, factor)).withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }) else tensor.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); } -fn partitionCachedKv(tensor: zml.Tensor, replicate_kv_heads: bool, repeat_factor: ?u32) zml.Tensor { - return if (replicate_kv_heads) - tensor.rename(.{ .s = .k }).repeat1d(.h, @as(u63, repeat_factor.?)).withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }) +fn partitionCachedKv(tensor: zml.Tensor, repeat_factor: ?u32) zml.Tensor { + return if (repeat_factor) |factor| + tensor.rename(.{ .s = .k }).stutter1d(tensor.axis(.h), @as(u63, factor)).withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }) else tensor.rename(.{ .s = .k }).withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); } fn partitionKvCacheShape(kv_shape: zml.Shape, kv_heads: i64, model_partitions: i64) zml.Shape { - const is_replicated, const repeat_factor = kvHeadsAreReplicated(kv_heads, model_partitions); - - if (is_replicated) { - const repeated_h = kv_shape.dim(.h) * @as(i64, @intCast(repeat_factor.?)); + if (kvHeadRepeatFactor(kv_heads, model_partitions)) |repeat_factor| { + const repeated_h = kv_shape.dim(.h) * @as(i64, @intCast(repeat_factor)); return kv_shape .setDim(.h, repeated_h) .withPartitioning(.{ .h = .model }); @@ -434,7 +431,6 @@ pub const TransformerLayer = struct { moe_parameters: zml.moe.Parameters, ) struct { zml.Tensor, KvCache.GatedDeltaNetCache } { _ = config; - _ = token_index; const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); @@ -442,7 +438,7 @@ pub const TransformerLayer = struct { .linear_attn => |linear_attn| linear_attn, .self_attn => unreachable, }; - const attention_output, const updated_kv_cache = linear_attn.forward(normalized_x0, kv_cache); + const attention_output, const updated_kv_cache = linear_attn.forward(normalized_x0, kv_cache, token_index); const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); @@ -474,7 +470,7 @@ pub const TransformerLayer = struct { updated_kv_cache.self_attn = result[1]; }, .linear_attn => |*linear_attn| { - const result = linear_attn.forward(normalized_x0, kv_cache.cache.linear_attn); + const result = linear_attn.forward(normalized_x0, kv_cache.cache.linear_attn, token_index); attention_output = result[0]; updated_kv_cache.gated_delta_net = result[1]; }, @@ -580,20 +576,20 @@ pub const SelfAttn = struct { kv_cache: KvCache.SelfAttnCache, ) struct { zml.Tensor, KvCache.SelfAttnCache } { const model_partitions = zml.module.CompilationContext.current().partitioning.numPartitionsForLogicalAxis(self.q_proj.weight.shape(), .model) catch unreachable; - const replicate_kv_heads, const repeat_factor = kvHeadsAreReplicated(self.num_kv_heads, model_partitions); + const repeat_factor = kvHeadRepeatFactor(self.num_kv_heads, model_partitions); const x_qkv = x.withPartitioning(.{ .d = .replicated }); var q, var gate = self.projectQAndGate(x_qkv); var k, var v = self.projectKV(x_qkv); q = q.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); gate = gate.withPartitioning(.{ .s = .replicated, .d_out_proj = .model }); - k = partitionProjectedKv(k, replicate_kv_heads, repeat_factor); - v = partitionProjectedKv(v, replicate_kv_heads, repeat_factor); + k = partitionProjectedKv(k, repeat_factor); + v = partitionProjectedKv(v, repeat_factor); q = self.q_norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); k = self.k_norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); const dtype = q.dtype(); - const position_ids = zml.Tensor.arange(.{ .end = x.dim(.s) }, .i64) + const position_ids: zml.Tensor = zml.Tensor.arange(.{ .end = x.dim(.s) }, .i64) .withTags(.{.s}).insertAxes(.s, .{.b}).broad(zml.Shape.init(.{ .b = x.dim(.b), .s = x.dim(.s) }, .i64)) .add(token_index.convert(.i64).broad(zml.Shape.init(.{ .b = x.dim(.b), .s = x.dim(.s) }, .i64))); @@ -601,15 +597,15 @@ pub const SelfAttn = struct { q = self.rotary_embed.applyRope(q, cos, sin); k = self.rotary_embed.applyRope(k, cos, sin); q = q.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); - k = partitionProjectedKv(k, replicate_kv_heads, repeat_factor); - v = partitionProjectedKv(v, replicate_kv_heads, repeat_factor); + k = partitionProjectedKv(k, null); + v = partitionProjectedKv(v, null); const new_kv_cache = kv_cache.update(k, v, token_index.convert(.u32)); k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); q = q.rename(.{ .s = .q }).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }); - k = partitionCachedKv(k, replicate_kv_heads, repeat_factor); - v = partitionCachedKv(v, replicate_kv_heads, repeat_factor); + k = partitionCachedKv(k, repeat_factor); + v = partitionCachedKv(v, repeat_factor); const attn_output = zml.attention.attention.attention( q, @@ -766,7 +762,11 @@ pub const Moe = struct { ) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); const shared_gate = self.shared_expert_gate.forward(x).sigmoid().broad(x.shape()); - const shared = self.shared_expert.forward(x).rename(.{ .dout = .d }).mul(shared_gate); + const shared = self.shared_expert.forward(x).rename(.{ .dout = .d }).mul(shared_gate).withPartitioning(.{ + .b = .replicated, + .s = .replicated, + .d = .replicated, + }); return moe_output.add(shared); } @@ -968,7 +968,16 @@ pub const GatedDeltaNet = struct { return zml.Tensor.concatenate(&.{ padding, tail }, .s); } - pub fn forward(self: GatedDeltaNet, x: zml.Tensor, cache: KvCache.GatedDeltaNetCache) struct { zml.Tensor, KvCache.GatedDeltaNetCache } { + fn buildUpdatedConvStateFromPrefix(input: zml.Tensor, left_pad: i64, valid_len: zml.Tensor) zml.Tensor { + const start = valid_len.convert(.i64).addConstant(-left_pad).maximum(zml.Tensor.scalar(0, .i64)); + return input.dynamicSlice1d(input.axis(.s), .{ .start = start, .len = left_pad }); + } + + fn setPaddingToZero(input: zml.Tensor, valid_mask: zml.Tensor) zml.Tensor { + return valid_mask.select(input, zml.Tensor.zeroes(input.shape())); + } + + pub fn forward(self: GatedDeltaNet, x: zml.Tensor, cache: KvCache.GatedDeltaNetCache, token_index: zml.Tensor) struct { zml.Tensor, KvCache.GatedDeltaNetCache } { const key_dim = self.num_k_heads * self.head_k_dim; const value_dim = self.num_v_heads * self.head_v_dim; const conv_dim = 2 * key_dim + value_dim; @@ -1029,7 +1038,17 @@ pub const GatedDeltaNet = struct { const beta = b.sigmoid(); const aLog_type = self.aLog.dtype(); - const g = self.aLog.broad(a.shape()).exp().mul(softplus(a.convert(aLog_type).add(self.dt_bias.convert(aLog_type).broad(a.shape())))).negate(); + var g = self.aLog.broad(a.shape()).exp().mul(softplus(a.convert(aLog_type).add(self.dt_bias.convert(aLog_type).broad(a.shape())))).negate(); + var recurrent_value = value; + var recurrent_beta = beta; + if (!use_cached_state) { + const valid_mask: zml.Tensor = zml.Tensor.arange(.{ .end = x.dim(.s) }, .i64) + .withTags(.{.s}) + .cmp(.LT, token_index.convert(.i64)); + recurrent_value = setPaddingToZero(recurrent_value, valid_mask.broad(recurrent_value.shape())); + recurrent_beta = setPaddingToZero(recurrent_beta, valid_mask.broad(recurrent_beta.shape())); + g = setPaddingToZero(g, valid_mask.broad(g.shape())); + } const query_for_rule = if (self.qk_head_repetition == 1) query else query.stutter1d(@intCast(query.axis(.kh)), @intCast(self.qk_head_repetition)); const key_for_rule = if (self.qk_head_repetition == 1) key else key.stutter1d(@intCast(key.axis(.kh)), @intCast(self.qk_head_repetition)); @@ -1037,9 +1056,9 @@ pub const GatedDeltaNet = struct { const core_attn_out, const last_recurrent_state = recurrentGatedDeltaRule( query_for_rule, key_for_rule, - value, + recurrent_value, g, - beta, + recurrent_beta, if (use_cached_state) cache.recurrentState() else null, ); @@ -1054,7 +1073,7 @@ pub const GatedDeltaNet = struct { const output = self.out_proj.forward(core_attn_out_normed.merge(.{ .d = .{ .vh, .vhd } })) .rename(.{ .dout = .d }).withPartitioning(.{ .d = .replicated }); const updated_cache = cache.update( - buildUpdatedConvState(conv_input, left_pad), + if (use_cached_state) buildUpdatedConvState(conv_input, left_pad) else buildUpdatedConvStateFromPrefix(projected_qkv, left_pad, token_index), last_recurrent_state, ); return .{ output, updated_cache }; diff --git a/examples/llm/models/qwen3_5_moe/session.zig b/examples/llm/models/qwen3_5_moe/session.zig index ac7281f1..19e4d579 100644 --- a/examples/llm/models/qwen3_5_moe/session.zig +++ b/examples/llm/models/qwen3_5_moe/session.zig @@ -223,6 +223,8 @@ pub const Session = struct { var prefill_token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 0), .u32); defer prefill_token_index_buffer.deinit(); + var prefill_valid_len_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.len)), .u32); + defer prefill_valid_len_buffer.deinit(); var prefill_hidden_buffer = try zml.Buffer.uninitialized(self.io, self.platform, prefill_hidden_shape, replicated_sharding, .{}); defer prefill_hidden_buffer.deinit(); @@ -264,7 +266,7 @@ pub const Session = struct { const exe = self.compiled_model.prefill_linear_layer_exe orelse unreachable; self.linear_attn_layers_caches[linear_attn_cache_index].conv_state = self.kv_cache_buffers.gated_delta_net.conv_state; self.linear_attn_layers_caches[linear_attn_cache_index].recurrent_state = self.kv_cache_buffers.gated_delta_net.recurrent_state; - prefill_linear_layer_args.?.set(.{ layer_weights, prefill_hidden_buffer, prefill_token_index_buffer, self.linear_attn_layers_caches[linear_attn_cache_index], self.compiled_model.loaded_model.inner.config, self.prefill_moe_metadata_buffers }); + prefill_linear_layer_args.?.set(.{ layer_weights, prefill_hidden_buffer, prefill_valid_len_buffer, self.linear_attn_layers_caches[linear_attn_cache_index], self.compiled_model.loaded_model.inner.config, self.prefill_moe_metadata_buffers }); exe.call(prefill_linear_layer_args.?, &prefill_linear_layer_results.?); prefill_linear_layer_results.?.fill(.{ &prefill_hidden_buffer, &self.kv_cache_buffers.gated_delta_net }); linear_attn_cache_index += 1; diff --git a/kernels/mosaic_tpu/pipeline.zig b/kernels/mosaic_tpu/pipeline.zig index 235513a0..f93e4ffb 100644 --- a/kernels/mosaic_tpu/pipeline.zig +++ b/kernels/mosaic_tpu/pipeline.zig @@ -444,9 +444,16 @@ const BufferedRef = struct { .scalar => |v| v, .slice => @panic("emitPipeline: blocked dim must get a scalar index"), }; - if (@mod(self.spec.full_shape[d], n) != 0) @panic("emitPipeline: non-dividing blocked dim unsupported"); base[d] = if (n == 1) idx else idx.mul(@as(i32, @intCast(n))); - size[d] = null; + const dim_size = self.spec.full_shape[d]; + if (@mod(dim_size, n) == 0) { + size[d] = null; + } else { + const tiling: i64 = if (d == bs.len - 1) 128 else if (bs.len >= 2 and d == bs.len - 2) 8 else 1; + const clamped = @min(n, dim_size); + const sz_const = @divFloor(clamped + tiling - 1, tiling) * tiling; + size[d] = b.emit(tpu.assume_multiple(b.ctx, b.lift(@as(i32, @intCast(sz_const))).inner, @as(i32, @intCast(tiling)), b.loc())); + } }, .bounded_slice => { const sl = switch (bidx[d]) { diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 6daf1d04..132425be 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -82,6 +82,8 @@ zig_library( "attention/triton_attention.zig", "attention/triton_kernels/unified_attention.zig", "moe/moe.zig", + "moe/mosaic_tpu.zig", + "moe/mosaic_tpu_kernels/gmm_ep.zig", "moe/triton.zig", "moe/triton_kernels/triton_kernels.zig", "kernel.zig", diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index 909d93fd..7c1bbccd 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -2,12 +2,20 @@ const std = @import("std"); const zml = @import("../zml.zig"); const stdx = zml.stdx; pub const triton = @import("triton.zig"); +pub const mosaic_tpu = @import("mosaic_tpu.zig"); pub const triton_kernels = @import("triton_kernels/triton_kernels.zig"); +pub const ActivationMode = enum { + silu, + relu, + gelu, +}; + pub const Backend = enum { // Could select a more specific name like "triton_sm90_bf16" triton, + mosaic_tpu, pub fn auto(platform: *const zml.Platform, weights_dtype: zml.DataType) !Backend { return switch (platform.target) { @@ -26,6 +34,10 @@ pub const Backend = enum { } break :b error.UnsupportedComputeCapability; }, + .tpu => switch (weights_dtype) { + .bf16, .f16, .f32 => .mosaic_tpu, + else => error.UnsupportedDataType, + }, else => error.UnimplementedMoEBackend, }; } @@ -34,6 +46,7 @@ pub const Backend = enum { _ = allocator; return switch (backend) { .triton => {}, + .mosaic_tpu => {}, }; } @@ -41,19 +54,37 @@ pub const Backend = enum { _ = platform; return switch (backend) { .triton => {}, + .mosaic_tpu => {}, }; } }; pub const Parameters = union(Backend) { triton: triton.Parameters, + mosaic_tpu: mosaic_tpu.Parameters, pub const InitOptions = union(Backend) { triton: triton.Parameters.InitOptions, + mosaic_tpu: mosaic_tpu.Parameters.InitOptions, - pub fn fromBackend(backend: Backend, num_experts_per_tok: ?u32, activation: triton.Parameters.ActivationMode) InitOptions { + pub fn fromBackend(backend: Backend, num_experts_per_tok: ?u32, activation: ActivationMode) InitOptions { return switch (backend) { - .triton => .{ .triton = .{ .num_experts_per_tok = num_experts_per_tok.?, .activation = activation } }, + .triton => .{ .triton = .{ + .num_experts_per_tok = num_experts_per_tok.?, + .activation = switch (activation) { + .silu => .silu, + .relu => .relu, + .gelu => .gelu, + }, + } }, + .mosaic_tpu => .{ .mosaic_tpu = .{ + .num_experts_per_tok = num_experts_per_tok.?, + .activation = switch (activation) { + .silu => .silu, + .relu => .relu, + .gelu => .gelu, + }, + } }, }; } }; @@ -61,19 +92,23 @@ pub const Parameters = union(Backend) { pub fn init(opts: InitOptions) Parameters { return switch (opts) { .triton => |v| .{ .triton = triton.Parameters.init(v) }, + .mosaic_tpu => |v| .{ .mosaic_tpu = mosaic_tpu.Parameters.init(v) }, }; } }; pub const Metadata = union(Backend) { triton: triton.Metadata, + mosaic_tpu: mosaic_tpu.Metadata, pub const InitOptions = union(Backend) { triton: triton.Metadata.InitOptions, + mosaic_tpu: mosaic_tpu.Metadata.InitOptions, pub fn fromBackend(backend: Backend) InitOptions { return switch (backend) { .triton => .{ .triton = .{} }, + .mosaic_tpu => .{ .mosaic_tpu = .{} }, }; } }; @@ -81,18 +116,21 @@ pub const Metadata = union(Backend) { pub fn init(opts: InitOptions) Metadata { return switch (opts) { .triton => |v| .{ .triton = triton.Metadata.init(v) }, + .mosaic_tpu => |v| .{ .mosaic_tpu = mosaic_tpu.Metadata.init(v) }, }; } pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { return switch (self) { .triton => |metadata| .{ .triton = try metadata.initBuffer(io, platform) }, + .mosaic_tpu => |metadata| .{ .mosaic_tpu = try metadata.initBuffer(io, platform) }, }; } pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { switch (self.*) { .triton => |*metadata| triton.deinitBuffer(metadata), + .mosaic_tpu => |*metadata| mosaic_tpu.deinitBuffer(metadata), } } }; @@ -114,21 +152,17 @@ pub fn forwardMoe( .triton => b: { const triton_metadata = switch (metadata) { .triton => |v| v, + else => return error.InvalidMetadata, }; const expert_partition = weights_gate_up.shape().partition(.expert); if (expert_partition.eql(.init(.experts))) { - const expert_partitions = zml.module.CompilationContext.current().partitioning.numPartitionsForLogicalAxis( - weights_gate_up.shape(), - .experts, - ) catch unreachable; - - const global_num_experts = weights_down.shape().dim(.expert) * expert_partitions; + const global_num_experts = weights_down.dim(.expert); const partial_output = zml.ops.manualComputation( .{ input, topk_ids, topk_weights, weights_gate_up, weights_down }, - input.shape().withPartitioning(.{ .b = .replicated, .s = .replicated, .d = .replicated }), + input.shape(), .{ .activation = parameters.triton.activation, .global_num_experts = global_num_experts, @@ -138,7 +172,7 @@ pub fn forwardMoe( .bias_down = bias_down, }, (struct { - fn body(ctx: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output_shape: zml.Shape) zml.Tensor { + fn body(ctx: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, _: zml.Shape) zml.Tensor { const local_num_experts = sharded_inputs[3].dim(.expert); const partition_id = zml.ops.partitionId().convert(.i32); const expert_start = partition_id.scale(local_num_experts).convert(.i32); @@ -169,7 +203,6 @@ pub fn forwardMoe( .w2_bias = ctx.bias_down, }, ) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); - _ = output_shape; return local_output.reshape(sharded_inputs[0].shape().dims()).withTags(.{ .b, .s, .d }); } }).body, @@ -194,5 +227,80 @@ pub fn forwardMoe( }, ); }, + .mosaic_tpu => b: { + const tpu_metadata = switch (metadata) { + .mosaic_tpu => |v| v, + else => return error.InvalidMetadata, + }; + + const expert_partition = weights_gate_up.shape().partition(.expert); + + if (expert_partition.eql(.init(.experts))) { + const global_num_experts = weights_down.dim(.expert); + const partial_output = zml.ops.manualComputation( + .{ input, topk_ids, topk_weights, weights_gate_up, weights_down }, + input.shape(), + .{ + .activation = parameters.mosaic_tpu.activation, + .global_num_experts = global_num_experts, + .scales_gate_up = scales_gate_up, + .bias_gate_up = bias_gate_up, + .scales_down = scales_down, + .bias_down = bias_down, + }, + (struct { + fn body(ctx: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, _: zml.Shape) zml.Tensor { + const local_num_experts = sharded_inputs[3].dim(.expert); + const partition_id = zml.ops.partitionId().convert(.i32); + const expert_start = partition_id.scale(local_num_experts).convert(.i32); + const global_expert_ids = zml.Tensor.arange(.{ .end = ctx.global_num_experts }, .i32).withTags(.{.expert}); + + const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) + .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); + const expert_map = local_expert_mask.select( + global_expert_ids.sub(expert_start), + zml.Tensor.scalar(-1, .i32), + ); + const local_output = mosaic_tpu.fusedExpertsImpl( + sharded_inputs[0], + sharded_inputs[3], + sharded_inputs[4], + sharded_inputs[2], + sharded_inputs[1], + .{}, + .{ + .activation = ctx.activation, + .global_num_experts = ctx.global_num_experts, + .expert_map = expert_map, + .w1_scale = ctx.scales_gate_up, + .w2_scale = ctx.scales_down, + .w1_bias = ctx.bias_gate_up, + .w2_bias = ctx.bias_down, + }, + ) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); + return local_output.reshape(sharded_inputs[0].shape().dims()).withTags(.{ .b, .s, .d }); + } + }).body, + ); + break :b zml.ops.allReduce(partial_output, zml.Tensor.add); + } + + break :b try mosaic_tpu.fusedExpertsImpl( + input, + weights_gate_up, + weights_down, + topk_weights, + topk_ids, + tpu_metadata, + .{ + .activation = parameters.mosaic_tpu.activation, + .global_num_experts = weights_gate_up.dim(.expert), + .w1_scale = scales_gate_up, + .w2_scale = scales_down, + .w1_bias = bias_gate_up, + .w2_bias = bias_down, + }, + ); + }, }; } diff --git a/zml/moe/mosaic_tpu.zig b/zml/moe/mosaic_tpu.zig new file mode 100644 index 00000000..8023da7a --- /dev/null +++ b/zml/moe/mosaic_tpu.zig @@ -0,0 +1,368 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../zml.zig"); +const Tensor = zml.Tensor; + +pub const gmm_ep = @import("mosaic_tpu_kernels/gmm_ep.zig"); +const mtt = @import("kernels/mosaic_tpu/builder"); + +const log = std.log.scoped(.moe_mosaic_tpu); + +var gate_up_transpose: std.atomic.Value(bool) = .init(false); +var down_transpose: std.atomic.Value(bool) = .init(false); + +pub const ActivationMode = enum { + silu, + relu, + gelu, + quick_gelu_plus_one, +}; + +pub const Options = struct { + activation: ActivationMode = .silu, + global_num_experts: i64 = -1, + expert_map: ?Tensor = null, + w1_scale: ?Tensor = null, + w2_scale: ?Tensor = null, + w1_bias: ?Tensor = null, + w2_bias: ?Tensor = null, +}; + +pub const Parameters = struct { + num_experts_per_tok: u32, + activation: ActivationMode, + + pub const InitOptions = struct { + num_experts_per_tok: u32, + activation: ActivationMode = .silu, + }; + + pub fn init(opts: InitOptions) Parameters { + return .{ + .num_experts_per_tok = opts.num_experts_per_tok, + .activation = opts.activation, + }; + } +}; + +pub const Metadata = struct { + pub const InitOptions = struct {}; + + pub fn init(opts: InitOptions) Metadata { + _ = opts; + return .{}; + } + + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { + _ = self; + _ = io; + _ = platform; + return {}; + } +}; + +pub fn deinitBuffer(bufferized: *zml.Bufferized(Metadata)) void { + _ = bufferized; +} + +fn validateOptions(opts: Options) !void { + if (opts.expert_map != null and opts.global_num_experts == -1) return error.InvalidShape; + if (opts.w1_scale != null or opts.w2_scale != null) return error.UnsupportedQuantization; +} + +fn validateInputs(hidden: Tensor, gate_up: Tensor, down: Tensor, weights: Tensor, ids: Tensor) !void { + switch (hidden.dtype()) { + .bf16, .f16, .f32 => {}, + else => return error.UnsupportedType, + } + switch (gate_up.dtype()) { + .bf16, .f16, .f32 => {}, + else => return error.UnsupportedType, + } + switch (down.dtype()) { + .bf16, .f16, .f32 => {}, + else => return error.UnsupportedType, + } + switch (weights.dtype()) { + .bf16, .f16, .f32 => {}, + else => return error.UnsupportedType, + } + if (ids.dtype() != .i32) return error.UnsupportedType; + if (hidden.dim(.in) != gate_up.dim(.in)) return error.InvalidShape; + if (@rem(gate_up.dim(.out), 2) != 0) return error.InvalidShape; + if (down.dim(.mid) != @divFloor(gate_up.dim(.out), 2)) return error.InvalidShape; + if (ids.dim(.token) != hidden.dim(.token) or weights.dim(.token) != hidden.dim(.token)) return error.InvalidShape; + if (ids.dim(.topk) != weights.dim(.topk)) return error.InvalidShape; + if (gate_up.dim(.expert) != down.dim(.expert)) return error.InvalidShape; +} + +fn warningTranspose(flag: *std.atomic.Value(bool), projection_name: []const u8, expected_shape: []const u8) void { + if (!flag.swap(true, .monotonic)) { + log.warn("{s} MoE weights are transposed at runtime; weights must be loaded with the correct shape {s} to avoid the transpose at runtime.", .{ projection_name, expected_shape }); + } +} + +fn canonicalizeGateUp(w1: Tensor, hidden_size: i64) !Tensor { + if (w1.rank() != 3) return error.InvalidShape; + + if (w1.dim(1) == hidden_size) return w1.withTags(.{ .expert, .in, .out }); + if (w1.dim(2) == hidden_size) { + warningTranspose(&gate_up_transpose, "gate_up_proj", "[expert, in, out]"); + return w1.withTags(.{ .expert, .out, .in }).transpose(.{ .expert, .in, .out }); + } + return error.InvalidShape; +} + +fn canonicalizeDown(w2: Tensor, hidden_size: i64, gate_up_out: i64) !Tensor { + if (w2.rank() != 3) return error.InvalidShape; + + const mid_size = @divFloor(gate_up_out, 2); + if (w2.dim(1) == mid_size and w2.dim(2) == hidden_size) return w2.withTags(.{ .expert, .mid, .out }); + if (w2.dim(1) == hidden_size and w2.dim(2) == mid_size) { + warningTranspose(&down_transpose, "down_proj", "[expert, mid, out]"); + return w2.withTags(.{ .expert, .out, .mid }).transpose(.{ .expert, .mid, .out }); + } + return error.InvalidShape; +} + +fn histogramCounts(ids: Tensor, num_bins: i64) Tensor { + const zeros = Tensor.zeroes(.init(.{ .expert = num_bins }, .i32)); + const ones = Tensor.constant(.{ .i32 = 1 }).broad(ids.shape().withDtype(.i32)); + return zeros.scatterSlices( + .{ .expert = ids }, + ones, + .{ + .update_fn = Tensor.ScatterOpts.increment, + .indices_are_unique = false, + }, + ); +} + +fn alignSortedRowsByGroup(rows: Tensor, expert_ids_sorted: Tensor, group_sizes: Tensor, tile_m: i64) struct { + rows: Tensor, + group_sizes: Tensor, + positions: Tensor, +} { + const num_groups = group_sizes.dim(.expert); + const max_padded_rows = rows.dim(.token) + num_groups * (tile_m - 1); + + const group_ends = group_sizes.cumulativeSum(.expert); + const group_starts = Tensor.concatenate(&.{ + Tensor.zeroes(.init(.{ .expert = 1 }, .i32)), + group_ends.slice1d(.expert, .{ .end = num_groups - 1 }), + }, .expert); + + const padded_group_sizes = group_sizes.addConstant(tile_m - 1).divByConst(tile_m).mul(Tensor.scalar(tile_m, .i32)); + const padded_group_ends = padded_group_sizes.cumulativeSum(.expert); + const padded_group_starts = Tensor.concatenate(&.{ + Tensor.zeroes(.init(.{ .expert = 1 }, .i32)), + padded_group_ends.slice1d(.expert, .{ .end = num_groups - 1 }), + }, .expert); + + const logical_positions = Tensor.arange(.{ .end = rows.dim(.token) }, .i32).withTags(.{.token}); + const group_start_per_token = group_starts.gather(.{ .expert = expert_ids_sorted }, .{}); + const padded_group_start_per_token = padded_group_starts.gather(.{ .expert = expert_ids_sorted }, .{}); + const padded_positions = logical_positions.sub(group_start_per_token).add(padded_group_start_per_token).withTags(.{.token}); + + const padded = Tensor.zeroes(rows.shape().setDim(.token, max_padded_rows)) + .scatterSlices( + .{ .token = padded_positions }, + rows, + .{ .indices_are_unique = true, .update_fn = Tensor.ScatterOpts.override }, + ); + + return .{ + .rows = padded, + .group_sizes = padded_group_sizes, + .positions = padded_positions, + }; +} + +fn permuteSmallBatch(hidden: Tensor, sorted_route: Tensor, topk: i64) Tensor { + const num_tokens = hidden.dim(.token); + const num_routes = num_tokens * topk; + + const token_indices = Tensor.arange(.{ .end = num_tokens }, .i32) + .withTags(.{.token}) + .repeat1d(.token, @intCast(topk)) + .rename(.{ .token = .route }); + const token_indices_sorted = token_indices.gather(.{ .route = sorted_route }, .{}).rename(.{ .perm = .route }); + + const onehot_shape = zml.Shape.init(.{ .route = num_routes, .token_src = num_tokens }, .i32); + const token_indices_2d = token_indices_sorted.insertAxes(.last, .{.token_src}).broad(onehot_shape); + const source_tokens_2d = Tensor.arange(.{ .end = num_tokens }, .i32) + .withTags(.{.token_src}) + .insertAxes(0, .{.route}) + .broad(onehot_shape); + const onehot = token_indices_2d.cmp(.EQ, source_tokens_2d).convert(hidden.dtype()); + + return onehot.dot(hidden.rename(.{ .token = .token_src }), .token_src).rename(.{ .route = .token }); +} + +fn gmmDType(dtype: zml.DataType) mtt.DType { + return switch (zml.kernel.mosaic_tpu.from(dtype)) { + .bf16 => .bf16, + .f16 => .f16, + .f32 => .f32, + else => stdx.debug.panic("mosaic_tpu MoE expects bf16/f16/f32 tensors, got {}", .{dtype}), + }; +} + +pub fn callGmmEp( + lhs: Tensor, + rhs_k_n: Tensor, + group_sizes: Tensor, + out_dtype: zml.DataType, + fuse_act: gmm_ep.Cfg.FuseAct, +) Tensor { + const k = lhs.dim(1); + const n = rhs_k_n.dim(.out); + const out_n = switch (fuse_act) { + .none => n, + .silu, .gelu => @divExact(n, 2), + }; + const aligned_k = @divFloor(k + 127, 128) * 128; + const aligned_n = if (fuse_act == .none) @divFloor(out_n + 127, 128) * 128 else out_n; + const cfg: gmm_ep.Cfg = .{ + .size_m = lhs.dim(.token), + .size_k = k, + .size_n = n, + .size_group = rhs_k_n.dim(.expert), + .size_lhs_group = group_sizes.dim(.expert), + .tile_m = 128, + .tile_k = aligned_k, + .tile_n = if (fuse_act == .none) aligned_n else out_n, + .lhs_dtype = gmmDType(lhs.dtype()), + .rhs_dtype = gmmDType(rhs_k_n.dtype()), + .out_dtype = gmmDType(out_dtype), + .acc_dtype = .f32, + .size_lhs_sublane = 8, + .fuse_act = fuse_act, + }; + + const out = gmm_ep.TpuKernel.call( + .{ + .group_sizes = group_sizes.withTags(.{.expert}), + .group_offset = Tensor.zeroes(.init(.{1}, .i32)), + .lhs = lhs, + .rhs = rhs_k_n, + }, + .{ + .out = zml.Shape.init(.{ .token = lhs.dim(.token), .out = aligned_n }, out_dtype), + }, + .{ .cfg = cfg }, + ).out; + + return out.slice1d(.out, .{ .end = out_n }); +} + +fn applyActivation(x: Tensor, mode: ActivationMode) Tensor { + const mid = @divFloor(x.dim(.out), 2); + const gate = x.slice1d(.out, .{ .end = mid }); + const up = x.slice1d(.out, .{ .start = mid }); + + return switch (mode) { + .silu => gate.silu().mul(up), + .gelu => gate.gelu().mul(up), + .relu => x.relu().powByConst(2), + .quick_gelu_plus_one => blk: { + const gate_clamped = gate.minimum(Tensor.scalar(7, gate.dtype()).broad(gate.shape())); + const up_clamped = up.clamp( + Tensor.scalar(-7, up.dtype()).broad(up.shape()), + Tensor.scalar(7, up.dtype()).broad(up.shape()), + ); + break :blk gate_clamped.quickGelu().mul(up_clamped.addConstant(1)); + }, + }; +} + +pub fn fusedExpertsImpl( + hidden_states: Tensor, + w1: Tensor, + w2: Tensor, + topk_weights: Tensor, + topk_ids: Tensor, + metadata: Metadata, + opts: Options, +) !Tensor { + _ = metadata; + try validateOptions(opts); + + const b = hidden_states.dim(.b); + const s = hidden_states.dim(.s); + const hidden = hidden_states.reshape(.{ .token = b * s, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); + const gate_up = try canonicalizeGateUp(w1, hidden.dim(.in)); + const down = try canonicalizeDown(w2, hidden.dim(.in), gate_up.dim(.out)); + const weights = topk_weights.reshape(.{ .token = b * s, .topk = topk_weights.dim(.top_expert) }).withTags(.{ .token, .topk }); + const ids = topk_ids.reshape(.{ .token = b * s, .topk = topk_ids.dim(.top_expert) }).withTags(.{ .token, .topk }); + + const num_experts = if (opts.global_num_experts != -1) opts.global_num_experts else gate_up.dim(.expert); + if (opts.expert_map) |expert_map| { + if (expert_map.dtype() != .i32) return error.UnsupportedType; + if (expert_map.rank() != 1 or expert_map.dim(.expert) != num_experts) return error.InvalidShape; + } + + const flat_ids_global = ids.transpose(.{ .topk, .token }).flatten().withTags(.{.route}); + const flat_weights = weights.transpose(.{ .topk, .token }).flatten().withTags(.{.route}); + + const flat_ids_local, const local_route_mask = if (opts.expert_map) |expert_map| blk: { + const mapped = expert_map.gather(.{ .expert = flat_ids_global }, .{}).withTags(.{.route}); + const route_mask = mapped.cmp(.GE, Tensor.scalar(0, .i32).broad(mapped.shape())); + const safe_ids = route_mask.select(mapped, Tensor.zeroes(mapped.shape())); + break :blk .{ safe_ids, route_mask }; + } else .{ + flat_ids_global, + Tensor.scalar(true, .bool).broad(flat_ids_global.shape()), + }; + + const sorted_route = flat_ids_local.argsort(.route, .{ .descending = false }).rename(.{ .route = .perm }); + const hidden_sorted = if (opts.expert_map != null) + permuteSmallBatch(hidden, sorted_route, ids.dim(.topk)) + else blk: { + // If not sharded, we can save the sort + gather by just repeating the hidden states for each expert and gathering with the sorted route + const hidden_repeated = hidden.repeat1d(.token, @intCast(ids.dim(.topk))).rename(.{ .token = .route }); + break :blk hidden_repeated.gather(.{ .route = sorted_route }, .{}).rename(.{ .perm = .token }); + }; + const expert_ids_sorted = flat_ids_local.gather(.{ .route = sorted_route }, .{}).rename(.{ .perm = .token }); + const local_route_mask_sorted = local_route_mask.gather(.{ .route = sorted_route }, .{}).rename(.{ .perm = .token }); + const sorted_weights = flat_weights.gather(.{ .route = sorted_route }, .{}).rename(.{ .perm = .token }); + const router_weights_sorted = local_route_mask_sorted.select( + sorted_weights, + Tensor.zeroes(sorted_weights.shape()), + ); + + const group_sizes = histogramCounts(flat_ids_local, gate_up.dim(.expert)); + const tile_m: i64 = 128; + const aligned_hidden = alignSortedRowsByGroup(hidden_sorted, expert_ids_sorted, group_sizes, tile_m); + + var gate_up_out = callGmmEp(aligned_hidden.rows, gate_up, aligned_hidden.group_sizes, hidden.dtype(), .none) + .gather(.{ .token = aligned_hidden.positions.rename(.{ .token = .route }) }, .{}) + .rename(.{ .route = .token }); + + if (opts.w1_bias) |bias| { + const bias_per_token = bias.withTags(.{ .expert, .out }).gather(.{ .expert = expert_ids_sorted }, .{}); + gate_up_out = gate_up_out.add(bias_per_token); + } + + const activated = applyActivation(gate_up_out, opts.activation); + + const aligned_activated = alignSortedRowsByGroup(activated, expert_ids_sorted, group_sizes, tile_m); + var down_out = callGmmEp(aligned_activated.rows, down, aligned_activated.group_sizes, hidden.dtype(), .none) + .gather(.{ .token = aligned_activated.positions.rename(.{ .token = .route }) }, .{}) + .rename(.{ .route = .token }); + + if (opts.w2_bias) |bias| { + const bias_per_token = bias.withTags(.{ .expert, .out }).gather(.{ .expert = expert_ids_sorted }, .{}); + down_out = down_out.add(bias_per_token); + } + + const weighted = down_out.mul(router_weights_sorted.convert(down_out.dtype()).broad(down_out.shape())); + + const restore_indices = sorted_route.argsort(.perm, .{ .descending = false }).rename(.{ .perm = .restore }); + const unsorted = weighted.gather(.{ .token = restore_indices }, .{}).rename(.{ .restore = .route }); + const restored = unsorted.reshape(.{ .topk = ids.dim(.topk), .token = hidden.dim(.token), .out = down.dim(.out) }); + const combined = restored.sum(.topk).squeeze(.topk); + return combined.reshape(.{ .b = b, .s = s, .d = down.dim(.out) }); +} diff --git a/zml/moe/mosaic_tpu_kernels/gmm_ep.zig b/zml/moe/mosaic_tpu_kernels/gmm_ep.zig index c2716f2c..421bb7cc 100644 --- a/zml/moe/mosaic_tpu_kernels/gmm_ep.zig +++ b/zml/moe/mosaic_tpu_kernels/gmm_ep.zig @@ -1,6 +1,6 @@ const std = @import("std"); -const zml = @import("zml"); +const zml = @import("../../zml.zig"); const mtt = @import("kernels/mosaic_tpu/builder"); const pipeline = mtt.pipeline; const mtt_kernel = zml.kernel.mosaic_tpu; @@ -39,6 +39,12 @@ fn ceilDivPosI32(b: *mtt.Builder, x: mtt.Value, y: mtt.Value) mtt.Value { } pub const Cfg = struct { + pub const FuseAct = enum { + none, + silu, + gelu, + }; + size_m: i64, size_k: i64, size_n: i64, @@ -52,9 +58,27 @@ pub const Cfg = struct { out_dtype: mtt.DType = .bf16, acc_dtype: mtt.DType = .f32, size_lhs_sublane: i64 = 8, + fuse_act: FuseAct = .none, + + pub fn outSizeN(self: Cfg) i64 { + return switch (self.fuse_act) { + .none => self.size_n, + .silu, .gelu => @divExact(self.size_n, 2), + }; + } pub fn alignedN(self: Cfg) i64 { - return alignTo(self.size_n, 128); + return switch (self.fuse_act) { + .none => alignTo(self.outSizeN(), 128), + .silu, .gelu => self.outSizeN(), + }; + } + + pub fn rhsTileN(self: Cfg) i64 { + return switch (self.fuse_act) { + .none => self.tile_n, + .silu, .gelu => self.tile_n * 2, + }; } pub fn maxNumGm(self: Cfg) i64 { @@ -72,8 +96,15 @@ fn validateConfig(cfg: Cfg) void { std.debug.assert(cfg.tile_n > 0); std.debug.assert(cfg.size_lhs_sublane > 0); std.debug.assert(@mod(cfg.size_m, cfg.size_lhs_sublane) == 0); - std.debug.assert(@mod(cfg.size_k, cfg.tile_k) == 0); - std.debug.assert(@mod(cfg.alignedN(), cfg.tile_n) == 0); + std.debug.assert(cdiv(cfg.size_k, cfg.tile_k) == 1); + std.debug.assert(@mod(cfg.tile_k, 128) == 0); + std.debug.assert(@mod(cfg.rhsTileN(), 128) == 0); + if (cfg.fuse_act != .none) { + std.debug.assert(@mod(cfg.size_n, 2) == 0); + std.debug.assert(cdiv(cfg.alignedN(), cfg.tile_n) == 1); + } else { + std.debug.assert(@mod(cfg.alignedN(), cfg.tile_n) == 0); + } std.debug.assert(cfg.acc_dtype == .f32); } @@ -234,6 +265,48 @@ fn fillMetadata(b: *mtt.Builder, cfg: Cfg, a: anytype) mtt.Value { } fn matmulByMxuColumns(b: *mtt.Builder, cfg: Cfg, lhs: mtt.Value, rhs: mtt.Value, cols: i64) mtt.Value { + if (cfg.tile_k > cfg.size_k) { + std.debug.assert(cdiv(cfg.size_k, cfg.tile_k) == 1); + std.debug.assert(@mod(cols, 128) == 0); + + const chunks = @divExact(cols, 128); + const parts = b.arena.allocator().alloc(mtt.Value, @intCast(chunks)) catch @panic("gmm_ep matmul parts OOM"); + const tail_k = cfg.tile_k - cfg.size_k; + const lhs_valid = b.vectorExtractStridedSlice(lhs, &.{ 0, 0 }, &.{ cfg.tile_m, cfg.size_k }, &.{ 1, 1 }); + const lhs_tail = b.vectorExtractStridedSlice(lhs, &.{ 0, cfg.size_k }, &.{ cfg.tile_m, tail_k }, &.{ 1, 1 }); + + var rev: i64 = 0; + while (rev < chunks) : (rev += 1) { + const chunk = chunks - 1 - rev; + const start_n = chunk * 128; + const rhs_valid = b.vectorExtractStridedSlice(rhs, &.{ 0, start_n }, &.{ cfg.size_k, 128 }, &.{ 1, 1 }); + const rhs_tail = b.vectorExtractStridedSlice(rhs, &.{ cfg.size_k, start_n }, &.{ tail_k, 128 }, &.{ 1, 1 }); + const zero = b.zeros(&.{ cfg.tile_m, 128 }, cfg.acc_dtype); + const tail_acc = b.matmulOpts(lhs_tail, rhs_tail, zero, .{ + .dimension_numbers = b.dotDimensionNumbers(&.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}), + }); + parts[@intCast(chunk)] = b.matmulOpts(lhs_valid, rhs_valid, tail_acc, .{ + .dimension_numbers = b.dotDimensionNumbers(&.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}), + }); + } + + const max_concat_inputs = 16; + if (chunks <= max_concat_inputs) return b.concatenate(parts, 1); + + const groups = cdiv(chunks, max_concat_inputs); + const group_parts = b.arena.allocator().alloc(mtt.Value, @intCast(groups)) catch @panic("gmm_ep matmul group parts OOM"); + var group: i64 = 0; + while (group < groups) : (group += 1) { + const start = group * max_concat_inputs; + const end = @min(start + max_concat_inputs, chunks); + const start_usize: usize = @intCast(start); + const end_usize: usize = @intCast(end); + const group_slice = parts[start_usize..end_usize]; + group_parts[@intCast(group)] = if (group_slice.len == 1) group_slice[0] else b.concatenate(group_slice, 1); + } + return b.concatenate(group_parts, 1); + } + if (cols <= 128) { return b.matmulOpts(lhs, rhs, b.zeros(&.{ cfg.tile_m, cols }, cfg.acc_dtype), .{ .dimension_numbers = b.dotDimensionNumbers(&.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}), @@ -271,6 +344,44 @@ fn matmulByMxuColumns(b: *mtt.Builder, cfg: Cfg, lhs: mtt.Value, rhs: mtt.Value, return b.concatenate(group_parts, 1); } +fn geluApprox(b: *mtt.Builder, x: mtt.Value, shape: []const i64) mtt.Value { + const x2 = b.mulf(x, x); + const x3 = b.mulf(x2, x); + const cubic = b.mulf(x3, b.splat(@as(f32, 0.044715), shape, cfgAccDtype())); + const inner = b.mulf(b.addf(x, cubic), b.splat(@as(f32, 0.7978845608028654), shape, cfgAccDtype())); + return b.mulf( + b.mulf(x, b.splat(@as(f32, 0.5), shape, cfgAccDtype())), + b.addf(b.tanh(inner), b.splat(@as(f32, 1.0), shape, cfgAccDtype())), + ); +} + +fn cfgAccDtype() mtt.DType { + return .f32; +} + +fn silu(b: *mtt.Builder, x: mtt.Value, shape: []const i64) mtt.Value { + const one = b.splat(@as(f32, 1.0), shape, cfgAccDtype()); + const denom = b.addf(one, b.exp(b.negf(x))); + return b.divf(x, denom); +} + +fn applyFuseAct(b: *mtt.Builder, cfg: Cfg, acc: mtt.Value) mtt.Value { + return switch (cfg.fuse_act) { + .none => acc, + .silu, .gelu => { + const shape = &.{ cfg.tile_m, cfg.tile_n }; + const gate = b.vectorExtractStridedSlice(acc, &.{ 0, 0 }, shape, &.{ 1, 1 }); + const up = b.vectorExtractStridedSlice(acc, &.{ 0, cfg.outSizeN() }, shape, &.{ 1, 1 }); + const activated = switch (cfg.fuse_act) { + .silu => silu(b, gate, shape), + .gelu => geluApprox(b, gate, shape), + .none => unreachable, + }; + return b.mulf(activated, up); + }, + }; +} + fn innerBody( b: *mtt.Builder, indices: []const mtt.Value, @@ -284,17 +395,25 @@ fn innerBody( const lhs_shape = &.{ cdiv(cfg.tile_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.tile_k }; const out_shape = &.{ cdiv(cfg.tile_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.tile_n }; + const rhs_tile_n = cfg.rhsTileN(); b.traceStart(10, "matmul_first_last"); const lhs_ref = winRef(b, refs[0], lhs_shape); const lhs_ref_2d = b.memRefReshape(lhs_ref, &.{ cfg.tile_m, cfg.tile_k }); const lhs = b.refLoad(lhs_ref_2d); const rhs_slot = b.toIndex(refs[1].slot.?); - const rhs = b.shapeCast( - b.vectorLoadShape(refs[1].buf.?, &.{ rhs_slot, b.cIndex(0), b.cIndex(0) }, &.{ 1, cfg.tile_k, cfg.tile_n }), - &.{ cfg.tile_k, cfg.tile_n }, + var rhs = b.shapeCast( + b.vectorLoadShape(refs[1].buf.?, &.{ rhs_slot, b.cIndex(0), b.cIndex(0) }, &.{ 1, cfg.tile_k, rhs_tile_n }), + &.{ cfg.tile_k, rhs_tile_n }, ); - var acc = matmulByMxuColumns(b, cfg, lhs, rhs, cfg.tile_n); + const valid_k = @mod(cfg.size_k, cfg.tile_k); + if (valid_k != 0) { + const k_iota = b.iota(&.{ cfg.tile_k, rhs_tile_n }, .i32, &.{0}); + const valid = b.splat(@as(i32, @intCast(valid_k)), &.{ cfg.tile_k, rhs_tile_n }, .i32); + rhs = b.select(b.cmpi(.slt, k_iota, valid), rhs, b.zeros(&.{ cfg.tile_k, rhs_tile_n }, cfg.rhs_dtype)); + } + var acc = matmulByMxuColumns(b, cfg, lhs, rhs, rhs_tile_n); + acc = applyFuseAct(b, cfg, acc); const m_start = metaLoad(b, ctx.meta.m_offset, gm_id); const m_end = metaLoad(b, ctx.meta.m_offset, b.addi(gm_id, b.lift(@as(i32, 1)))); @@ -418,7 +537,7 @@ pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { const rhs_spec = pipeline.PipeSpec{ .full_shape = &.{ cfg.size_group, cfg.size_k, cfg.size_n }, .dtype = cfg.rhs_dtype, - .block_shape = &.{ .squeezed, .{ .blocked = cfg.tile_k }, .{ .blocked = cfg.tile_n } }, + .block_shape = &.{ .squeezed, .{ .blocked = cfg.tile_k }, .{ .blocked = cfg.rhsTileN() } }, .index_map = if (cdiv(cfg.alignedN(), cfg.tile_n) == 1 and cdiv(cfg.size_k, cfg.tile_k) == 1 and cfg.size_m > cfg.tile_m) rhsIndexMap1D else rhsIndexMap, .index_map_ctx = ctx, .buffer_count = 3, diff --git a/zml/moe/triton.zig b/zml/moe/triton.zig index 941c15dc..bd7569e5 100644 --- a/zml/moe/triton.zig +++ b/zml/moe/triton.zig @@ -59,6 +59,7 @@ pub const Parameters = struct { pub const ActivationMode = enum { silu, relu, + gelu, }; pub const InitOptions = struct { @@ -111,6 +112,17 @@ fn initZeroBiasBuffer(io: std.Io, platform: *const zml.Platform, sharding: zml.S return zml.Buffer.fromSlice(io, platform, zero_slice, sharding); } +fn applyActivation(x: Tensor, mode: Parameters.ActivationMode) Tensor { + const mid = @divFloor(x.dim(.out), 2); + const gate = x.slice1d(.out, .{ .end = mid }); + const up = x.slice1d(.out, .{ .start = mid }); + return switch (mode) { + .silu => gate.silu().mul(up), + .relu => x.relu().powByConst(2), + .gelu => gate.gelu().mul(up), + }; +} + // ============================================================================= // Top-level entry point // ============================================================================= @@ -212,13 +224,7 @@ pub fn fusedExpertsImpl( Shape.init(.{ .token = num_assignments, .out = gate_up.dim(.out) }, .bf16), ); - const activated = switch (opts.activation) { - .silu => b: { - const gate, const up = zml.nn.splitRealImg(first_out, .sequential); - break :b gate.silu().mul(up); - }, - .relu => first_out.relu().powByConst(2), - }; + const activated = applyActivation(first_out, options.activation); var activated_quant = activated; a_scale = opts.a2_scale orelse Tensor.scalar(1.0, .f32); From 1fead1c014fccc9ee37eca921e68a8f0b010fda2 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 16 Jun 2026 10:57:00 +0200 Subject: [PATCH 516/881] platforms/neuron: introduce NKI stack with SDPA kernel (#528) **Summary** This PR update to Neuron v2.30, introduces the Neuron NKI stack into ZML and wires it into the LLM attention path. The main goal is to make Neuron attention run through NKI kernels. It adds the compiler/runtime plumbing needed to embed NKI kernels as Neuron custom-native calls, then uses that path for SDPA-style prefill/decode attention. Paged Attention will come later. On inf2.8xlarge it runs : - 25 tok/s for Llama 3.1 8B BF16 - 63.5 tok/s for LFM2.5-1.2B-Instruct BF16 - 19 tok/s for Qwen3.5-9B BF16 --- .bazelrc | 4 + MODULE.bazel | 2 +- bin/zml-smi/platforms/neuron/nrt.zig | 3 +- docs/howtos/profiling.md | 54 +- examples/llm/main.zig | 2 +- examples/llm/models/llama/model.zig | 1 + examples/llm/models/llama/session.zig | 4 +- mlir/dialects/stablehlo/stablehlo.zig | 8 + platforms/neuron/BUILD.bazel | 122 ++- platforms/neuron/libneuronxla.zig | 274 ------- platforms/neuron/libpjrt_neuron.BUILD.bazel | 92 ++- platforms/neuron/libpjrt_neuron.zig | 133 ++-- platforms/neuron/neuron.bzl | 35 +- platforms/neuron/neuron.zig | 85 ++ platforms/neuron/neuronx-cc.zig | 109 --- platforms/neuron/neuronx_cc_shim.py | 16 + .../neuron/neuronxcc_compiler_launcher.zig | 9 + platforms/neuron/nki_compiler_launcher.zig | 12 + platforms/neuron/nki_kernel.zig | 160 ++++ platforms/neuron/nki_kernel_compiler.py | 179 +++++ platforms/neuron/packages.lock.json | 144 ++-- platforms/neuron/packages.yaml | 6 +- platforms/neuron/python_launcher.zig | 155 ++++ platforms/neuron/requirements.in | 5 +- platforms/neuron/requirements.lock.txt | 732 +++--------------- platforms/neuron/{topology => }/topology.zig | 155 ++-- platforms/neuron/topology/BUILD.bazel | 37 - tools/neuron/BUILD.bazel | 49 ++ tools/neuron/ingest.sh | 92 +++ tools/neuron/profile.sh | 33 + tools/neuron/server.sh | 14 + tools/neuron/summary.sh | 36 + zml/BUILD.bazel | 11 +- zml/Sharding.zig | 3 +- zml/attention/attention.zig | 16 +- zml/attention/nki/BUILD.bazel | 30 + zml/attention/nki/attention.py | 473 +++++++++++ zml/attention/nki/attention.zig | 99 +++ zml/attention/nki/attention_harness.py | 331 ++++++++ zml/buffer.zig | 15 +- zml/exe.zig | 3 + zml/ops.zig | 250 +++++- zml/tensor.zig | 75 +- 43 files changed, 2625 insertions(+), 1443 deletions(-) delete mode 100644 platforms/neuron/libneuronxla.zig delete mode 100644 platforms/neuron/neuronx-cc.zig create mode 100755 platforms/neuron/neuronx_cc_shim.py create mode 100644 platforms/neuron/neuronxcc_compiler_launcher.zig create mode 100644 platforms/neuron/nki_compiler_launcher.zig create mode 100644 platforms/neuron/nki_kernel.zig create mode 100755 platforms/neuron/nki_kernel_compiler.py create mode 100644 platforms/neuron/python_launcher.zig rename platforms/neuron/{topology => }/topology.zig (62%) delete mode 100644 platforms/neuron/topology/BUILD.bazel create mode 100644 tools/neuron/BUILD.bazel create mode 100755 tools/neuron/ingest.sh create mode 100755 tools/neuron/profile.sh create mode 100755 tools/neuron/server.sh create mode 100755 tools/neuron/summary.sh create mode 100644 zml/attention/nki/BUILD.bazel create mode 100644 zml/attention/nki/attention.py create mode 100644 zml/attention/nki/attention.zig create mode 100644 zml/attention/nki/attention_harness.py diff --git a/.bazelrc b/.bazelrc index b60a1782..3d6ddbdb 100644 --- a/.bazelrc +++ b/.bazelrc @@ -150,3 +150,7 @@ run:nsys --run_under=//tools:nsys run:rocprofv3 --@zml//platforms:rocm=true run:rocprofv3 --run_under=//tools:rocprofv3 + +run:neuron-profile --@zml//platforms:neuron=true +run:neuron-profile --@zml//platforms:cpu=false +run:neuron-profile --run_under='//tools/neuron:profile /tmp/zml-neuron-profile' diff --git a/MODULE.bazel b/MODULE.bazel index 265d98f2..7b99bb48 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -178,7 +178,7 @@ tpu = use_extension("//platforms/tpu:tpu.bzl", "tpu_packages") use_repo(tpu, "libpjrt_tpu") neuron = use_extension("//platforms/neuron:neuron.bzl", "neuron_packages") -use_repo(neuron, "aws-neuronx-collectives", "aws-neuronx-runtime-lib", "libgomp1", "libpjrt_neuron") +use_repo(neuron, "aws-neuronx-collectives", "aws-neuronx-runtime-lib", "aws-neuronx-tools", "libgcc_s1", "libgomp1", "libpjrt_neuron", "libstdcpp6") non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") use_repo(non_module_deps, "arocc", "cloud_accelerator_diagnostics", "com_google_sentencepiece", "flashattn_linux_amd64", "flashattn_linux_arm64", "iree", "libvaxis", "linenoise", "mnist", "mosaic_tpu", "org_swig_swig", "stb", "translate-c", "uucode", "xla", "zigimg") diff --git a/bin/zml-smi/platforms/neuron/nrt.zig b/bin/zml-smi/platforms/neuron/nrt.zig index 259300f0..8fe21a27 100644 --- a/bin/zml-smi/platforms/neuron/nrt.zig +++ b/bin/zml-smi/platforms/neuron/nrt.zig @@ -72,8 +72,9 @@ pub fn init(allocator: std.mem.Allocator, io: std.Io) !Nrt { var dev: ?*c.ndl_device_t = null; var t: c.struct_ndl_device_init_param = .{ .initialize_device = false, - .map_hbm = false, .num_dram_regions = 0, + .map_hbm = false, + .skip_copy_bufs = true, }; if (private_fns.ndl_open_device(device_idx, &t, &dev) == 0) { diff --git a/docs/howtos/profiling.md b/docs/howtos/profiling.md index d60fc63c..f98a7dc2 100644 --- a/docs/howtos/profiling.md +++ b/docs/howtos/profiling.md @@ -266,6 +266,55 @@ That means: - ZML still records local host `TraceMe` scopes - ROCTx ranges from `zml.tracer.scope(...)` stay visible as device annotations +## Neuron Profiling + +Use `--config=neuron-profile` with the command you already run. The wrapper +prints the profile directory when the program exits. + +```bash +NEURON_RT_VISIBLE_CORES=0,1 \ +bazel run --config=neuron-profile //examples/llm -- \ + --model=/path/to/model \ + --prompt="Explain mixture-of-experts routing in one paragraph." \ + --seqlen=128 \ + --backend=nki \ + --topk=1 +``` + +The default profile root is `/tmp/zml-neuron-profile`. Each run contains: + +```text +/tmp/zml-neuron-profile// + compile/ + execution/ +``` + +Start the Neuron Explorer server in one terminal: + +```bash +bazel run //tools/neuron:server +``` + +Open . When working through SSH, forward ports `3001` +and `3002`. + +With the server running, ingest the run from another terminal: + +```bash +bazel run //tools/neuron:ingest -- /tmp/zml-neuron-profile/ +``` + +Create a quick summary from one run when you want a file artifact: + +```bash +bazel run //tools/neuron:summary-json -- /tmp/zml-neuron-profile/ +bazel run //tools/neuron:summary-txt -- /tmp/zml-neuron-profile/ +bazel run //tools/neuron:summary-perfetto -- /tmp/zml-neuron-profile/ +``` + +Each summary command writes beside the run and prints the output path: +`summary.json`, `summary.txt`, or `summary.pftrace`. + ## macOS Instruments On macOS, `zml.tracer.scope(...)` emits `os_signpost` intervals. These show up @@ -314,8 +363,9 @@ which keeps large traces much more manageable than the old in-memory path. ## Troubleshooting -- If you run under `--config=nsys` or `--config=rocprofv3`, PJRT profiling is - skipped on purpose because the wrappers set `SKIP_PJRT_PROFILER=true`. +- If you run under `--config=nsys`, `--config=rocprofv3`, or + `--config=neuron-profile`, PJRT profiling is skipped on purpose because the + wrappers set `SKIP_PJRT_PROFILER=true`. - If you do not see device annotations, first check whether you are on a supported platform: - CUDA/Linux for NVTX diff --git a/examples/llm/main.zig b/examples/llm/main.zig index c1edf5b6..d9b7e4a9 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -32,7 +32,7 @@ const Args = struct { \\ --prompt= Prompt to use for generation (default: none) \\ --seqlen= Sequence length (default: 2048) \\ --topk= Top-k sampling cutoff (default: 4) - \\ --backend= Attention backend to use ([vanilla, attnd, cuda_fa2, cuda_fa3], default: auto-selection) + \\ --backend= Attention backend to use ([vanilla, attnd, nki, cuda_fa2, cuda_fa3], default: auto-selection) \\ --attnd-ip= Register and prefer the `attnd` backend at the provided `IP:PORT` \\ --profile Capture a PJRT profile for non-interactive runs and write a Perfetto trace \\ diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index 304cb26f..3c150883 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -541,6 +541,7 @@ const SelfAttn = struct { .vanilla => attention_metadata, .cuda_fa2 => attention_metadata, .cuda_fa3 => attention_metadata, + .nki => attention_metadata, }; const attn_output = zml.attention.attention.attention( diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig index 4d1ce119..e4d15018 100644 --- a/examples/llm/models/llama/session.zig +++ b/examples/llm/models/llama/session.zig @@ -62,7 +62,7 @@ pub const Session = struct { } }; break :b buffers; }, - .vanilla, .cuda_fa2, .cuda_fa3 => null, + .vanilla, .cuda_fa2, .cuda_fa3, .nki => null, }; errdefer if (decode_attention_metadata_buffers) |*buffers| zml.attention.attention.Metadata.deinitBuffer(buffers); @@ -167,7 +167,7 @@ pub const Session = struct { .layer_id = try zml.Buffer.scalar(self.io, self.platform, 0, .u16), .num_tokens = try zml.Buffer.scalar(self.io, self.platform, all_tokens.len, .u32), } }, - .vanilla, .cuda_fa2, .cuda_fa3 => self.attention_metadata_buffers, + .vanilla, .cuda_fa2, .cuda_fa3, .nki => self.attention_metadata_buffers, }; try self.compiled_model.prefill.run(.{ diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 487b40d8..1edcf4a7 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -862,6 +862,14 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result }); } +pub fn optimizationBarrier(ctx: *mlir.Context, inputs: []const *const mlir.Value, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.optimization_barrier", .{ + .operands = .{ .flat = inputs }, + .results = .{ .flat = result_types }, + .location = location, + }); +} + pub fn createBuffer(ctx: *mlir.Context, value: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return custom_call(ctx, &.{}, &.{value}, .{ .call_target_name = "CreateBuffer", diff --git a/platforms/neuron/BUILD.bazel b/platforms/neuron/BUILD.bazel index 1e815ff9..540426db 100644 --- a/platforms/neuron/BUILD.bazel +++ b/platforms/neuron/BUILD.bazel @@ -1,22 +1,32 @@ -load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_c_proto_library") load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_python//python/uv:lock.bzl", uv_lock = "lock") load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_shared_library") +load("@zml//bazel:cc_import.bzl", "cc_import") load("@zml//bazel:runfiles.bzl", "runfiles_to_default") +exports_files([ + "neuronx_cc_shim.py", + "nki_kernel_compiler.py", +]) + # A proxy PJRT Plugin that loads the Neuron PJRT Plugin # and returns the instance from nested GetPjrtApi. # -# Additionally, it provides a way to load implicit transitive dependencies -# of neuronx-cc (see add_needed of the patchelf target below). +# The proxy also initializes the Neuron runtime environment and exposes the +# sandboxed compiler binaries to libneuronxla 3.0. zig_shared_library( name = "libpjrt_neuron", + srcs = ["neuron.zig"], main = "libpjrt_neuron.zig", target_compatible_with = ["@platforms//os:linux"], visibility = ["@libpjrt_neuron//:__subpackages__"], zigopts = ["-fno-stack-check"], deps = [ - ":libpython", + ":libnrt_headers", + ":neuron_nrt", + ":zmlxneuron", + "//bazel", + "//pjrt", "//stdx", "@rules_zig//zig/runfiles", "@xla//xla/pjrt/c:pjrt_c_api_hdrs", @@ -25,12 +35,15 @@ zig_shared_library( zig_binary( name = "neuronx-cc", - target_compatible_with = ["@platforms//os:linux"], data = ["@neuron_py_deps//neuronx_cc"], linkopts = ["-Wl,-rpath,$$ORIGIN/../lib"], - main = "neuronx-cc.zig", + main = "neuronxcc_compiler_launcher.zig", tags = ["manual"], - deps = [":libpython"], + target_compatible_with = ["@platforms//os:linux"], + deps = [ + ":libpython", + ":python_launcher", + ], ) runfiles_to_default( @@ -40,10 +53,18 @@ runfiles_to_default( deps = [":neuronx-cc"], ) +runfiles_to_default( + name = "nki-cc_files", + target_compatible_with = ["@platforms//os:linux"], + visibility = ["@libpjrt_neuron//:__subpackages__"], + deps = [":nki-cc"], +) + cc_library( name = "libpython", - tags = ["manual"], hdrs = ["libpython.h"], + tags = ["manual"], + visibility = ["//platforms/neuron:__subpackages__"], deps = [ "@rules_python//python/cc:current_py_cc_headers", "@rules_python//python/cc:current_py_cc_libs", @@ -52,57 +73,77 @@ cc_library( cc_library( name = "empty", - tags = ["manual"], defines = ["ZML_RUNTIME_NEURON_DISABLED"], - visibility = ["//platforms/neuron/topology:__pkg__"], + tags = ["manual"], ) cc_library( name = "zmlxneuron", - tags = ["manual"], defines = ["ZML_RUNTIME_NEURON"], - visibility = ["//platforms/neuron/topology:__pkg__"], + tags = ["manual"], ) cc_library( name = "libnrt_headers", - tags = ["manual"], hdrs = ["nrt.h"], + tags = ["manual"], visibility = [ "//bin/zml-smi:__subpackages__", - "//platforms/neuron/topology:__pkg__", ], deps = ["@libpjrt_neuron//:libnrt_headers"], ) -upb_c_proto_library( - name = "xla_data_upb", - deps = ["@xla//xla:xla_data_proto"], +cc_import( + name = "neuron_ncfw", + shared_library = "@aws-neuronx-runtime-lib//:libncfw.patchelf", tags = ["manual"], ) -upb_c_proto_library( - name = "hlo_proto_upb", - deps = ["@xla//xla/service:hlo_proto"], +cc_import( + name = "neuron_nrt", + shared_library = "@aws-neuronx-runtime-lib//:libnrt.patchelf", tags = ["manual"], + deps = [":neuron_ncfw"], ) -zig_shared_library( - name = "libneuronxla", - main = "libneuronxla.zig", - shared_lib_name = "libneuronxla.so", - target_compatible_with = ["@platforms//os:linux"], - visibility = ["@libpjrt_neuron//:__subpackages__"], - zigopts = [ - "-fno-stack-check", - "-fPIC", +zig_library( + name = "python_launcher", + import_name = "platforms/neuron/python_launcher", + main = "python_launcher.zig", + tags = ["manual"], + visibility = ["//platforms/neuron:__subpackages__"], + deps = [":libpython"], +) + +zig_binary( + name = "nki-cc", + data = [ + "nki_kernel_compiler.py", + "@neuron_py_deps//neuronx_cc", + "@neuron_py_deps//nki", ], + linkopts = ["-Wl,-rpath,$$ORIGIN/../lib"], + main = "nki_compiler_launcher.zig", + tags = ["manual"], + target_compatible_with = ["@platforms//os:linux"], + visibility = ["//platforms/neuron:__pkg__"], deps = [ - ":hlo_proto_upb", ":libpython", - ":xla_data_upb", + ":python_launcher", + ], +) + +zig_library( + name = "nki_kernel", + import_name = "platforms/neuron/nki_kernel", + main = "nki_kernel.zig", + tags = ["manual"], + visibility = ["//zml:__pkg__"], + deps = [ + ":neuron", + "//bazel", "//stdx", - "//upb", + "@rules_zig//zig/runfiles", ], ) @@ -122,16 +163,16 @@ uv_lock( zig_library( name = "neuron", - tags = ["manual"], import_name = "platforms/neuron", main = "neuron.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ "//pjrt", ] + select({ "//platforms:neuron.enabled": [ ":libnrt_headers", - ":libpython", + ":neuron_nrt", ":zmlxneuron", "//bazel", "//stdx", @@ -141,3 +182,16 @@ zig_library( "//conditions:default": [":empty"], }), ) + +zig_library( + name = "topology", + data = [":libpjrt_neuron"], + import_name = "platforms/neuron/topology", + main = "topology.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":neuron", + ":zmlxneuron", + ], +) diff --git a/platforms/neuron/libneuronxla.zig b/platforms/neuron/libneuronxla.zig deleted file mode 100644 index 56a67ab0..00000000 --- a/platforms/neuron/libneuronxla.zig +++ /dev/null @@ -1,274 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const stdx = @import("stdx"); -const upb = @import("upb"); - -const log = std.log.scoped(.@"zml/platforms/neuron/libneuronxla"); - -pub fn makeTempDir(io: std.Io, buf: []u8, prefix: []const u8) ![]const u8 { - const tmp_dir = std.c.getenv("TMPDIR") orelse "/tmp"; - const ret = try std.fmt.bufPrint(buf, "{s}{s}{s}{d}", .{ - tmp_dir, - std.Io.Dir.path.sep_str_posix, - prefix, - std.Io.Clock.now(.real, io).toNanoseconds(), - }); - try std.Io.Dir.createDir(.cwd(), io, ret, .fromMode(0o700)); - return ret; -} - -const ModuleState = struct { - threaded: std.Io.Threaded, -}; - -var module_def: c.PyModuleDef = .{ - .m_base = std.mem.zeroes(c.PyModuleDef_Base), - .m_doc = "Zig bindings for libneuronxla", - .m_name = "libneuronxla", - .m_size = @sizeOf(ModuleState), - .m_methods = @constCast(&[_]c.PyMethodDef{ - .{ - .ml_name = "hook", - .ml_meth = @ptrCast(&hook), - .ml_flags = c.METH_NOARGS, - .ml_doc = "Return a greeting from Zig.", - }, - .{ - .ml_name = "neuronx_cc", - .ml_meth = @ptrCast(&neuronx_cc), - .ml_flags = c.METH_FASTCALL, - .ml_doc = "Return a greeting from Zig.", - }, - std.mem.zeroes(c.PyMethodDef), - }), - .m_slots = @constCast(&[_]c.PyModuleDef_Slot{ - .{ .slot = c.Py_mod_exec, .value = @ptrCast(@constCast(&module_exec)) }, - std.mem.zeroes(c.PyModuleDef_Slot), - }), - .m_traverse = null, - .m_clear = null, - .m_free = null, -}; - -fn module_exec(module: ?*c.PyObject) callconv(.c) c_int { - const state: *ModuleState = @ptrCast(@alignCast(c.PyModule_GetState(module))); - state.* = .{ - .threaded = .init(std.heap.c_allocator, .{}), - }; - return 0; -} - -fn hook(self: ?*c.PyObject, args: ?*c.PyObject) callconv(.c) ?*c.PyObject { - _ = self; - _ = args; - const none = c.Py_None(); - defer c.Py_IncRef(none); - return none; -} - -pub fn PyBytes_AsStringAndSize(object: *c.PyObject) []u8 { - var buf: [*c]u8 = undefined; - var len: c.Py_ssize_t = undefined; - _ = c.PyBytes_AsStringAndSize(object, &buf, &len); - return buf[0..@intCast(len)]; -} - -fn wrapNeffAsCustomCall(allocator: std.mem.Allocator, io: std.Io, hlo_code: []const u8, neff_file_path: []const u8) ![]const u8 { - var upb_alloc: upb.Allocator = .init(allocator); - const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); - - const hlo_module = try upb.parse(c.xla_HloModuleProto, upb_arena, hlo_code); - - const entry = blk: { - var size: usize = undefined; - const computations = c.xla_HloModuleProto_mutable_computations(hlo_module, &size)[0..size]; - for (computations) |comp| { - if (c.xla_HloComputationProto_id(comp) == c.xla_HloModuleProto_entry_computation_id(hlo_module)) { - break :blk comp; - } - } else return error.ComputationNotFound; - }; - - const entry_instructions = blk: { - var size: usize = undefined; - break :blk c.xla_HloComputationProto_instructions(entry, &size)[0..size]; - }; - c.xla_HloComputationProto_clear_instructions(entry); - - const fused_root = blk: { - for (entry_instructions) |instruction| { - if (c.xla_HloInstructionProto_id(instruction) == c.xla_HloComputationProto_root_id(entry)) { - break :blk try upb.shallowClone(c.xla_HloInstructionProto, upb_arena, instruction); - } - } else return error.ComputationNotFound; - }; - - c.xla_HloInstructionProto_set_opcode(fused_root, upb.stringView("custom-call")); - c.xla_HloInstructionProto_set_custom_call_target(fused_root, upb.stringView("AwsNeuronNeff")); - c.xla_HloInstructionProto_set_backend_config( - fused_root, - upb.stringView( - try stdx.Io.Dir.readFileAlloc(.cwd(), io, neff_file_path, allocator, .unlimited), - ), - ); - - const parameters_len = blk: { - var size: usize = undefined; - _ = c.xla_ProgramShapeProto_parameters( - c.xla_HloComputationProto_program_shape(entry), - &size, - ); - break :blk size; - }; - - { - var operand_ids: std.ArrayList(i64) = .initBuffer(c.xla_HloInstructionProto_resize_operand_ids(fused_root, parameters_len + 1, upb_arena)[0 .. parameters_len + 1]); - var new_instructions: std.ArrayList(*const c.xla_HloInstructionProto) = .initBuffer(@ptrCast(c.xla_HloComputationProto_resize_instructions(entry, parameters_len + 1, upb_arena)[0 .. parameters_len + 1])); - for (entry_instructions) |instruction| { - if (std.mem.eql(u8, upb.slice(c.xla_HloInstructionProto_opcode(instruction)) orelse continue, "parameter")) { - const id = c.xla_HloInstructionProto_id(instruction); - operand_ids.appendAssumeCapacity(id); - new_instructions.appendAssumeCapacity(instruction); - } - } - new_instructions.appendAssumeCapacity(fused_root); - } - - { - const fa = c.xla_HloInstructionProto_mutable_frontend_attributes(fused_root, upb_arena); - const map = c._xla_FrontendAttributes_map_mutable_upb_map(fa, upb_arena); - _ = c.upb_Map_Set( - map, - .{ .str_val = upb.stringView("valid_inputs") }, - .{ .str_val = blk: { - const valid_inputs_value = try allocator.alloc(u8, parameters_len * 2 - 1); - for (valid_inputs_value, 0..) |*char, i| { - char.* = if (i % 2 == 0) '1' else ','; - } - break :blk upb.stringView(valid_inputs_value); - } }, - upb_arena, - ); - } - - return try upb.serialize(hlo_module, upb_arena); -} - -fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) !?*c.PyObject { - const state: *ModuleState = @ptrCast(@alignCast(c.PyModule_GetState(self))); - const io = state.threaded.io(); - - var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); - defer arena.deinit(); - - const args = args_[0..@intCast(nargs_)]; - - const code = PyBytes_AsStringAndSize(args[0]); - const platform_version = PyBytes_AsStringAndSize(args[2]); - - const target = std.StaticStringMap([]const u8).initComptime(.{ - .{ "1.0", "inf1" }, - .{ "2.0", "trn1" }, - .{ "3.0", "trn2" }, - }).get(platform_version) orelse { - log.err("Unknown platform version: {s}\n", .{platform_version}); - return error.UnknownPlatformVersion; - }; - - var tmp_dir_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const tmp_dir = try makeTempDir(io, &tmp_dir_buf, "zml-neuronxcc-"); - defer std.Io.Dir.deleteTree(.cwd(), io, tmp_dir) catch |err| { - log.err("Error deleting temporary directory {s}: {}\n", .{ tmp_dir, err }); - }; - - const code_file = try std.Io.Dir.path.join(arena.allocator(), &.{ tmp_dir, "file.code" }); - try std.Io.Dir.writeFile(.cwd(), io, .{ - .data = code, - .sub_path = code_file, - .flags = .{ .truncate = true }, - }); - - const neff_file = try std.Io.Dir.path.join(arena.allocator(), &.{ tmp_dir, "file.neff" }); - - { - const gil_state = c.PyEval_SaveThread(); - defer c.PyEval_RestoreThread(gil_state); - - var neuronx_cc_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - var child = try std.process.spawn(io, .{ - .argv = &.{ - try stdx.Io.Dir.path.bufJoin(&neuronx_cc_buf, &.{ - stdx.process.selfSharedObjectDirPath(), - "..", - "bin", - "neuronx-cc", - }), - "compile", - "--framework=XLA", - "--target", - target, - "--verbose=35", - "--enable-internal-neff-wrapper", - "--output", - neff_file, - "--optlevel=1", - // generic is the default, but it fails on transformers, force it - "--model-type=transformer", - // disable it, we do our own - "--auto-cast=none", - "--enable-fast-loading-neuron-binaries", - code_file, - }, - .stdin = .ignore, - .stdout = .ignore, - .stderr = .ignore, - .cwd = .{ .path = tmp_dir }, - }); - - const term = try child.wait(io); - switch (term) { - .exited => |exit_code| { - if (exit_code != 0) { - log.err("neuronx-cc failed: {}", .{exit_code}); - return error.NeuronxCcFailed; - } - }, - else => { - log.err("neuronx-cc failed, unknown error", .{}); - return error.NeuronxCcFailedUnknown; - }, - } - } - - const neff_hlo_bytes = wrapNeffAsCustomCall(arena.allocator(), io, code, neff_file) catch |err| { - log.err("Error wrapping NEFF as custom call: {}\n", .{err}); - return err; - }; - - return c.PyTuple_Pack( - 2, - c.PyLong_FromLongLong(0), - c.PyBytes_FromStringAndSize(@ptrCast(neff_hlo_bytes), @intCast(neff_hlo_bytes.len)), - ); -} - -fn neuronx_cc(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) callconv(.c) ?*c.PyObject { - return neuronx_cc_(self, args_, nargs_) catch |err| { - log.err("Error in neuronx_cc: {}\n", .{err}); - - const none = c.Py_None(); - c.Py_IncRef(none); - const tuple = c.PyTuple_New(2) orelse { - c.Py_DecRef(none); - return null; - }; - _ = c.PyTuple_SetItem(tuple, 0, c.PyLong_FromLongLong(400).?); - _ = c.PyTuple_SetItem(tuple, 1, none); - return tuple; - }; -} - -pub export fn PyInit_libneuronxla() callconv(.c) ?*c.PyObject { - return c.PyModuleDef_Init(&module_def); -} diff --git a/platforms/neuron/libpjrt_neuron.BUILD.bazel b/platforms/neuron/libpjrt_neuron.BUILD.bazel index c95cb108..ff7bbabe 100644 --- a/platforms/neuron/libpjrt_neuron.BUILD.bazel +++ b/platforms/neuron/libpjrt_neuron.BUILD.bazel @@ -16,36 +16,84 @@ select_file( patchelf( name = "libneuronpjrt_so", - set_rpath = "$ORIGIN", src = ":libneuronpjrt_so_orig", + set_rpath = "$ORIGIN", soname = "libneuronpjrt.so", ) patchelf( name = "libpjrt_neuron_so", - set_rpath = "$ORIGIN", src = "@zml//platforms/neuron:libpjrt_neuron", + set_rpath = "$ORIGIN", soname = "libpjrt_neuron.so", ) +patchelf( + name = "neuron_explorer", + src = "@aws-neuronx-tools//:neuron-explorer", + add_needed = ["libpjrt_neuron.so"], + set_rpath = "$ORIGIN/../lib", +) + +# This is a compiler-enabled sandbox. Python stays packaged because AWS exposes +# neuronx-cc and NKI compilation through Python entrypoints; ZML wraps those as +# native ELF launchers in bin/ so the image does not need shell scripts or host +# Python. Keep AWS compiler/runtime payloads intact and only trim Python +# files that are not part of Neuron compilation. copy_to_directory( name = "sandbox", srcs = [ ":libneuronpjrt_so", + ":libneuronxla_files", ":libpjrt_neuron_so", - "@zml//platforms/neuron:neuronx-cc_files", - "@zml//platforms/neuron:libneuronxla", - "@rules_python//python:current_py_toolchain", - "@aws-neuronx-runtime-lib//:libnrt.patchelf", - "@aws-neuronx-runtime-lib//:libncfw.patchelf", - "@aws-neuronx-runtime-lib//:libnrtucode_extisa", + ":neuron_explorer", "@aws-neuronx-collectives//:libnccom", - "@zlib1g", + "@aws-neuronx-runtime-lib//:libncfw.patchelf", + "@aws-neuronx-runtime-lib//:libnrt.patchelf", + "@aws-neuronx-runtime-lib//:libnrtucode_extisa", + "@libgcc_s1", "@libgomp1", + "@libstdcpp6", + "@rules_python//python:current_py_toolchain", + "@zlib1g", + "@zml//platforms/neuron:neuronx-cc_files", + "@zml//platforms/neuron:neuronx_cc_shim.py", + "@zml//platforms/neuron:nki-cc_files", + "@zml//platforms/neuron:nki_kernel_compiler.py", + ], + add_directory_to_runfiles = False, + allow_overwrites = True, + exclude_srcs_patterns = [ + "**/__pycache__/**", + "**/*.pyc", + "**/tests/**", + "lib/libtcl*.so*", + "lib/libtk*.so*", + "lib/python*/idlelib/**", + "lib/python*/tkinter/**", + "lib/python*/turtledemo/**", + "lib/python*/ensurepip/**", + # "**/include/**", + "**/_solib_*/**", + ], + hardlink = "on", + include_external_repositories = ["**"], + include_srcs_patterns = [ + "platforms/neuron/neuronx-cc", + "platforms/neuron/neuronx_cc_shim.py", + "platforms/neuron/nki-cc", + "platforms/neuron/nki_kernel_compiler.py", + "neuron_explorer/**", + "**/lib**", + "lib/python*/**", + "site-packages/**", ], replace_prefixes = { "platforms/neuron/neuronx-cc": "bin/neuronx-cc", - "platforms/neuron/libneuronxla.so": "site-packages/libneuronxla.so", + "platforms/neuron/neuronx_cc_shim.py": "bin/python-shims/neuronx-cc", + "platforms/neuron/nki-cc": "bin/nki-cc", + "platforms/neuron/nki_kernel_compiler.py": "site-packages/nki_kernel_compiler.py", + "neuron_explorer": "bin", "libneuronpjrt_so": "lib", "libpjrt_neuron_so": "lib", "libnrt.patchelf": "lib", @@ -55,28 +103,15 @@ copy_to_directory( "usr/lib/x86_64-linux-gnu": "lib", "opt/neuron": "lib", }, - exclude_srcs_patterns = [ - "**/tests/**", - # "**/include/**", - "**/_solib_*/**", + visibility = [ + "@zml//tools/neuron:__pkg__", + "@zml//zml:__pkg__", + "@zml//zml/attention/nki:__pkg__", ], - include_srcs_patterns = [ - "platforms/neuron/neuronx-cc", - "**/lib**", - "lib/python*/**", - "site-packages/**", - ], - allow_overwrites = True, - add_directory_to_runfiles = False, - include_external_repositories = ["**"], - hardlink = "on", ) cc_library( name = "libnrt_headers", - deps = [ - "@aws-neuronx-runtime-lib//:libnrt_headers", - ], linkopts = [ # Defer function call resolution until the function is called # (lazy loading) rather than at load time. @@ -88,6 +123,9 @@ cc_library( "-Wl,-z,lazy", ], visibility = ["@zml//platforms/neuron:__subpackages__"], + deps = [ + "@aws-neuronx-runtime-lib//:libnrt_headers", + ], ) cc_library( diff --git a/platforms/neuron/libpjrt_neuron.zig b/platforms/neuron/libpjrt_neuron.zig index 1cfaafc9..968f1584 100644 --- a/platforms/neuron/libpjrt_neuron.zig +++ b/platforms/neuron/libpjrt_neuron.zig @@ -2,6 +2,7 @@ const std = @import("std"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); +const neuron = @import("neuron.zig"); const runfiles = @import("runfiles"); const stdx = @import("stdx"); @@ -38,11 +39,14 @@ pub export fn zmlxneuron_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaqu } extern fn setenv(name: [*:0]const u8, value: [*:0]const u8, overwrite: c_int) c_int; -fn setupNeuronEnv(io: std.Io) !void { - var buf: [256]u8 = undefined; + +fn setupNeuronEnv(io: std.Io, sandbox_path: []const u8) !void { + const scratchpad_page_size = "1024"; + + var root_comm_buf: [256]u8 = undefined; _ = setenv( "NEURON_RT_ROOT_COMM_ID", - try std.fmt.bufPrintZ(&buf, "127.0.0.1:{d}", .{try findFreeTcpPort(io)}), + try std.fmt.bufPrintZ(&root_comm_buf, "127.0.0.1:{d}", .{try findFreeTcpPort(io)}), 1, ); _ = setenv( @@ -53,86 +57,50 @@ fn setupNeuronEnv(io: std.Io) !void { }), 1, ); - _ = setenv( - "NEURON_RT_STOCHASTIC_ROUNDING_EN", - "1", - 1, + _ = setenv("NEURON_RT_LOG_LEVEL", "error", 0); + _ = setenv("NEURON_RT_DISABLE_EXECUTION_BARRIER", "1", 0); + _ = setenv("NEURON_RT_STOCHASTIC_ROUNDING_EN", "1", 1); + _ = setenv("NEURON_RT_ASYNC_EXEC_MAX_INFLIGHT_REQUESTS", "3", 0); + _ = setenv("NEURON_SCRATCHPAD_PAGE_SIZE", scratchpad_page_size, 0); + + var bin_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const bin_path = try stdx.Io.Dir.path.bufJoin(&bin_path_buf, &.{ sandbox_path, "bin" }); + + const old_path = if (std.c.getenv("PATH")) |path| std.mem.span(path) else ""; + var new_path_buf: [std.posix.PATH_MAX:0]u8 = undefined; + const new_path = if (old_path.len == 0) + try std.fmt.bufPrintZ(&new_path_buf, "{s}", .{bin_path}) + else + try std.fmt.bufPrintZ(&new_path_buf, "{s}:{s}", .{ bin_path, old_path }); + if (setenv("PATH", new_path.ptr, 1) != 0) return error.SetEnvFailed; + + if (std.c.getenv("NEURON_CC_FLAGS") != null) return; + + const log_level = if (std.c.getenv("NEURON_RT_LOG_LEVEL")) |level| std.mem.span(level) else "error"; + + var flags_buf: [512:0]u8 = undefined; + const instance = try neuron.instance(); + const target = @tagName(instance.compilerTarget()); + const flags = try std.fmt.bufPrintZ( + &flags_buf, + "--target={s} " ++ + "--optlevel=1 " ++ + "--model-type=transformer " ++ + "--enable-fast-loading-neuron-binaries " ++ + "--enable-fast-context-switch " ++ + "--hbm-scratchpad-page-size={s} " ++ + "--verbose {s} " ++ + "--logfile-verbose {s} " ++ + "--logfile=./log-neuron-cc.txt", + .{ + target, + scratchpad_page_size, + log_level, + log_level, + }, ); -} -fn pyStatusCheck(status: c.PyStatus) void { - if (c.PyStatus_Exception(status) != 0) { - if (c.PyStatus_IsExit(status) != 0) { - std.process.exit(@intCast(status.exitcode)); - } - c.Py_ExitStatusException(status); - } -} - -fn toPosixPathW(file_path: []const u8) error{ InvalidUtf8, NameTooLong, CodepointTooLarge }![std.posix.PATH_MAX - 1:0]c.wchar_t { - var view = std.unicode.Utf8View.init(file_path) catch return error.InvalidUtf8; - var it = view.iterator(); - - var path_with_null: [std.posix.PATH_MAX - 1:0]c.wchar_t = undefined; - var len: usize = 0; - while (it.nextCodepoint()) |cp| { - if (len + 1 >= path_with_null.len) return error.NameTooLong; - path_with_null[len] = std.math.cast(c.wchar_t, cp) orelse return error.CodepointTooLarge; - len += 1; - } - path_with_null[len] = 0; - return path_with_null; -} - -// redefine it because for some reason PyThreadState can't be sized -extern fn PyEval_SaveThread() *anyopaque; - -fn setupPythonEnv(sandbox_path: []const u8) !void { - const Static = struct { - var py_config: c.PyConfig = undefined; - }; - - { - var preconfig: c.PyPreConfig = undefined; - c.PyPreConfig_InitIsolatedConfig(&preconfig); - preconfig.utf8_mode = 1; - pyStatusCheck(c.Py_PreInitialize(&preconfig)); - } - - c.PyConfig_InitIsolatedConfig(&Static.py_config); - - Static.py_config.module_search_paths_set = 1; - Static.py_config.optimization_level = 2; - Static.py_config.write_bytecode = 0; - - { - var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const home = try std.fmt.bufPrintZ(&buf, "{f}{d}.{d}", .{ - std.Io.Dir.path.fmtJoin(&.{ - sandbox_path, - "lib", - "python", - }), - c.PY_MAJOR_VERSION, - c.PY_MINOR_VERSION, - }); - pyStatusCheck(c.PyConfig_SetBytesString(&Static.py_config, &Static.py_config.home, home)); - pyStatusCheck(c.PyWideStringList_Append(&Static.py_config.module_search_paths, &try toPosixPathW(home))); - } - - { - var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const site_packages = try stdx.Io.Dir.path.bufJoin(&buf, &.{ - sandbox_path, - "site-packages", - }); - pyStatusCheck(c.PyWideStringList_Append(&Static.py_config.module_search_paths, &try toPosixPathW(site_packages))); - } - - pyStatusCheck(c.Py_InitializeFromConfig(&Static.py_config)); - - // release the GIL - _ = PyEval_SaveThread(); + _ = setenv("NEURON_CC_FLAGS", flags.ptr, 0); } // Duplicates a PJRT Api object while being careful about struct size differences @@ -162,8 +130,7 @@ fn getPjrtApi() !*c.PJRT_Api { "..", }); - try setupNeuronEnv(io); - try setupPythonEnv(sandbox_path); + try setupNeuronEnv(io, sandbox_path); Static.inner = blk: { const GetPjrtApi_inner = GetPjrtApi_blk: { diff --git a/platforms/neuron/neuron.bzl b/platforms/neuron/neuron.bzl index 8814c246..76123ab0 100644 --- a/platforms/neuron/neuron.bzl +++ b/platforms/neuron/neuron.bzl @@ -13,8 +13,22 @@ package(default_visibility = ["//visibility:public"]) """ _UBUNTU_PACKAGES = { - "zlib1g": packages.filegroup(name = "zlib1g", srcs = ["lib/x86_64-linux-gnu/libz.so.1"]), - "libgomp1": packages.filegroup(name = "libgomp1", srcs = ["usr/lib/x86_64-linux-gnu/libgomp.so.1"]), + "libgcc_s1": { + "package": "libgcc-s1", + "build": packages.filegroup(name = "libgcc_s1", srcs = ["lib/x86_64-linux-gnu/libgcc_s.so.1"]), + }, + "libgomp1": { + "package": "libgomp1", + "build": packages.filegroup(name = "libgomp1", srcs = ["usr/lib/x86_64-linux-gnu/libgomp.so.1"]), + }, + "libstdcpp6": { + "package": "libstdc++6", + "build": packages.filegroup(name = "libstdcpp6", srcs = ["usr/lib/x86_64-linux-gnu/libstdc++.so.6"]), + }, + "zlib1g": { + "package": "zlib1g", + "build": packages.filegroup(name = "zlib1g", srcs = ["lib/x86_64-linux-gnu/libz.so.1"]), + }, } _NEURON_PACKAGES = { @@ -57,6 +71,12 @@ _NEURON_PACKAGES = { srcs = ["lib/libnccom.so.2"], ), ]), + "aws-neuronx-tools": "\n".join([ + packages.filegroup( + name = "neuron-explorer", + srcs = ["bin/neuron-explorer"], + ), + ]), } def _neuron_impl(mctx): @@ -69,13 +89,13 @@ def _neuron_impl(mctx): build_file = ":libpjrt_neuron.BUILD.bazel", ) - for pkg_name, build_file_content in _UBUNTU_PACKAGES.items(): - pkg = loaded_packages[pkg_name]["amd64"] + for repo_name, package in _UBUNTU_PACKAGES.items(): + pkg = loaded_packages[package["package"]]["amd64"] http_deb_archive( - name = pkg_name, + name = repo_name, urls = pkg["urls"], sha256 = pkg["sha256"], - build_file_content = _BUILD_FILE_PRELUDE + build_file_content, + build_file_content = _BUILD_FILE_PRELUDE + package["build"], ) for pkg_name, build_file_content in _NEURON_PACKAGES.items(): @@ -93,7 +113,10 @@ def _neuron_impl(mctx): root_module_direct_deps = [ "aws-neuronx-collectives", "aws-neuronx-runtime-lib", + "aws-neuronx-tools", + "libgcc_s1", "libgomp1", + "libstdcpp6", "libpjrt_neuron", ], root_module_direct_dev_deps = [], diff --git a/platforms/neuron/neuron.zig b/platforms/neuron/neuron.zig index e99fd991..a25e0cf0 100644 --- a/platforms/neuron/neuron.zig +++ b/platforms/neuron/neuron.zig @@ -9,6 +9,91 @@ const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/platforms/neuron"); +pub const CompilerTarget = enum { + trn1, + inf2, + trn1n, + trn2, + trn2n, + trn3, +}; + +pub const Family = enum(u32) { + unknown = 0, + inf1 = 1, + trn1 = 2, + trn1n = 3, + inf2 = 4, + trn2 = 5, + trn2n = 6, + inf2e = 7, + trn2p = 8, + trn2u = 9, + trn2e = 10, + trn2eu = 11, + trn2ac = 12, + trn2uac = 13, + trn3 = 14, + trn3pds98 = 15, + + pub fn compilerTarget(self: Family) CompilerTarget { + return switch (self) { + .unknown, .inf1 => unreachable, + .trn1 => .trn1, + .trn1n => .trn1n, + .inf2, .inf2e => .inf2, + .trn2, .trn2n, .trn2p, .trn2u, .trn2e, .trn2eu, .trn2ac, .trn2uac => .trn2, + .trn3, .trn3pds98 => .trn3, + }; + } +}; + +pub const Size = enum(u32) { + xl1 = 0, + xl2 = 1, + xl4 = 2, + xl6 = 3, + xl8 = 4, + xl24 = 5, + xl32 = 6, + xl48 = 7, + xl3 = 8, + unknown = 9, +}; + +pub const Instance = struct { + family: Family, + size: Size, + + pub fn compilerTarget(self: Instance) CompilerTarget { + return self.family.compilerTarget(); + } + + pub fn coresPerChip(self: Instance) !usize { + return switch (self.family) { + .inf1 => 4, + .trn1, .trn1n, .inf2, .inf2e => 2, + .trn2, .trn2n, .trn2p, .trn2u, .trn2e, .trn2eu, .trn2ac, .trn2uac, .trn3, .trn3pds98 => 8, + .unknown => unreachable, + }; + } +}; + +pub fn instance() !Instance { + var info = std.mem.zeroes(c.struct_nrt_instance_info); + info.family = @intFromEnum(Family.unknown); + info.size = @intFromEnum(Size.unknown); + + if (c.nrt_get_instance_info(&info, @sizeOf(c.struct_nrt_instance_info)) != c.NRT_SUCCESS) { + return error.Unavailable; + } + + return .{ + .family = @enumFromInt(info.family), + .size = @enumFromInt(info.size), + }; +} + pub fn isEnabled() bool { return @hasDecl(c, "ZML_RUNTIME_NEURON"); } diff --git a/platforms/neuron/neuronx-cc.zig b/platforms/neuron/neuronx-cc.zig deleted file mode 100644 index 1ac09e2d..00000000 --- a/platforms/neuron/neuronx-cc.zig +++ /dev/null @@ -1,109 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); - -fn pyStatusCheck(status: c.PyStatus) void { - if (c.PyStatus_Exception(status) != 0) { - if (c.PyStatus_IsExit(status) != 0) { - std.process.exit(@intCast(status.exitcode)); - } - c.Py_ExitStatusException(status); - } -} - -pub fn toPosixPathW(file_path: []const u8) error{ InvalidUtf8, NameTooLong, CodepointTooLarge }![std.posix.PATH_MAX - 1:0]c.wchar_t { - var view = std.unicode.Utf8View.init(file_path) catch return error.InvalidUtf8; - var it = view.iterator(); - - var path_with_null: [std.posix.PATH_MAX - 1:0]c.wchar_t = undefined; - var len: usize = 0; - while (it.nextCodepoint()) |cp| { - if (len + 1 >= path_with_null.len) return error.NameTooLong; - path_with_null[len] = std.math.cast(c.wchar_t, cp) orelse return error.CodepointTooLarge; - len += 1; - } - path_with_null[len] = 0; - return path_with_null; -} - -pub fn main(init: std.process.Init) !void { - const arena = init.arena; - const io = init.io; - - { - var preconfig: c.PyPreConfig = undefined; - c.PyPreConfig_InitIsolatedConfig(&preconfig); - preconfig.utf8_mode = 1; - pyStatusCheck(c.Py_PreInitialize(&preconfig)); - } - - var config: c.PyConfig = undefined; - c.PyConfig_InitIsolatedConfig(&config); - defer c.PyConfig_Clear(&config); - - config.module_search_paths_set = 1; - config.optimization_level = 2; - config.write_bytecode = 0; - - const args = try init.minimal.args.toSlice(arena.allocator()); - const cArgs = try arena.allocator().alloc([*:0]const u8, args.len); - for (args, 0..) |arg, i| { - cArgs[i] = (try arena.allocator().dupeZ(u8, arg)).ptr; - } - pyStatusCheck(c.PyConfig_SetBytesArgv(&config, @intCast(cArgs.len), @ptrCast(cArgs.ptr))); - - var self_exe_dir_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const self_exe_dir = blk: { - const n = try std.process.executableDirPath(io, &self_exe_dir_buf); - break :blk self_exe_dir_buf[0..n]; - }; - - { - var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const home = try std.fmt.bufPrintZ(&buf, "{f}{d}.{d}", .{ - std.Io.Dir.path.fmtJoin(&.{ - self_exe_dir, - "..", - "lib", - "python", - }), - c.PY_MAJOR_VERSION, - c.PY_MINOR_VERSION, - }); - pyStatusCheck(c.PyConfig_SetBytesString(&config, &config.home, home)); - pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(home))); - } - - { - var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const site_packages = try std.fmt.bufPrint(&buf, "{f}", .{ - std.Io.Dir.path.fmtJoin(&.{ - self_exe_dir, - "..", - "site-packages", - }), - }); - pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(site_packages))); - } - - pyStatusCheck(c.Py_InitializeFromConfig(&config)); - defer c.Py_Finalize(); - - const neuronxcc_main = blk: { - const module = c.PyImport_ImportModule("neuronxcc.driver.CommandDriver") orelse { - std.log.err("Failed to import neuronxcc.driver.CommandDriver", .{}); - c.PyErr_Print(); - return error.FailedToImportModule; - }; - defer c.Py_DecRef(module); - break :blk c.PyObject_GetAttrString(module, "main"); - }; - defer c.Py_DecRef(neuronxcc_main); - - const result = c.PyObject_CallNoArgs(neuronxcc_main); - if (result == null) { - c.PyErr_Print(); - std.process.exit(1); - } - defer c.Py_DecRef(result); -} diff --git a/platforms/neuron/neuronx_cc_shim.py b/platforms/neuron/neuronx_cc_shim.py new file mode 100755 index 00000000..deff0b5d --- /dev/null +++ b/platforms/neuron/neuronx_cc_shim.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +"""Python-compatible shim for harness NKI standalone compilation.""" + +import os +import sys + + +def main() -> None: + native = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "..", "neuronx-cc" + ) + os.execv(native, [native, *sys.argv[1:]]) + + +if __name__ == "__main__": + main() diff --git a/platforms/neuron/neuronxcc_compiler_launcher.zig b/platforms/neuron/neuronxcc_compiler_launcher.zig new file mode 100644 index 00000000..066923e2 --- /dev/null +++ b/platforms/neuron/neuronxcc_compiler_launcher.zig @@ -0,0 +1,9 @@ +const std = @import("std"); + +const python_launcher = @import("platforms/neuron/python_launcher"); + +pub fn main(init: std.process.Init) !void { + try python_launcher.runModuleEntrypoint(init, .{ + .module_name = "neuronxcc.driver.CommandDriver", + }); +} diff --git a/platforms/neuron/nki_compiler_launcher.zig b/platforms/neuron/nki_compiler_launcher.zig new file mode 100644 index 00000000..f8a6b884 --- /dev/null +++ b/platforms/neuron/nki_compiler_launcher.zig @@ -0,0 +1,12 @@ +const std = @import("std"); + +const python_launcher = @import("platforms/neuron/python_launcher"); + +pub fn main(init: std.process.Init) !void { + _ = std; + try python_launcher.runScriptMain(init, &.{ + "..", + "site-packages", + "nki_kernel_compiler.py", + }); +} diff --git a/platforms/neuron/nki_kernel.zig b/platforms/neuron/nki_kernel.zig new file mode 100644 index 00000000..7bbb60be --- /dev/null +++ b/platforms/neuron/nki_kernel.zig @@ -0,0 +1,160 @@ +const std = @import("std"); + +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); +const stdx = @import("stdx"); + +const neuron = @import("platforms/neuron"); + +const log = std.log.scoped(.@"zml/platforms/neuron/nki_kernel"); + +pub const TensorSignature = struct { + dtype: []const u8, + dims: []const i64, +}; + +pub const Kernel = struct { + name: []const u8, + entrypoint: []const u8, + source_path: []const u8, + compiler_target: []const u8, + inputs: []const TensorSignature, + outputs: []const TensorSignature, +}; + +const CompilerRequest = struct { + entrypoint: []const u8, + source: []const u8, + target: []const u8, + tool_bin_dir: []const u8, + neuronx_cc_args: []const []const u8, + inputs: []const TensorSignature, + outputs: []const TensorSignature, +}; + +const CompilerResponse = struct { + backend_config_b64: []const u8, +}; + +pub fn compilerTargetFromInstance() neuron.CompilerTarget { + const instance = neuron.instance() catch |err| { + std.debug.panic("failed to query Neuron instance for NKI compiler target: {}", .{err}); + }; + return instance.compilerTarget(); +} + +pub fn compileNkiKernel(allocator: std.mem.Allocator, io: std.Io, kernel: Kernel) ![]const u8 { + const compiler_flags = std.mem.span(std.c.getenv("NEURON_CC_FLAGS") orelse return error.MissingNeuronCompilerFlags); + var compiler_args: std.ArrayList([]const u8) = .empty; + defer compiler_args.deinit(allocator); + var compiler_flags_iter = std.mem.tokenizeScalar(u8, compiler_flags, ' '); + while (compiler_flags_iter.next()) |arg| { + try compiler_args.append(allocator, arg); + } + + const tmp_root_path = if (std.c.getenv("TMPDIR")) |tmpdir| std.mem.span(tmpdir) else "/tmp"; + var tmp_root = try std.Io.Dir.openDir(.cwd(), io, tmp_root_path, .{}); + defer tmp_root.close(io); + + var dir_name_buf: [128]u8 = undefined; + const dir_name = try std.fmt.bufPrint(&dir_name_buf, "zml-nki-{d}", .{std.Io.Timestamp.now(io, .real).nanoseconds}); + + var tmp_dir = try tmp_root.createDirPathOpen(io, dir_name, .{ .permissions = .fromMode(0o700) }); + defer tmp_dir.close(io); + + var cwd_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const cwd = try tmp_dir.realPath(io, &cwd_buf); + + const request_file = try tmp_dir.createFile(io, "kernel.request.json", .{ .read = true }); + defer request_file.close(io); + + const result_file = try tmp_dir.createFile(io, "kernel.result.json", .{ .read = true }); + defer result_file.close(io); + + var request_file_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const request_file_path = try request_file.realPath(io, &request_file_buf); + + var result_file_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const result_file_path = try result_file.realPath(io, &result_file_buf); + + const r = try bazel.runfiles(bazel_builtin.current_repository); + + var sandbox_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = try r.rlocation("libpjrt_neuron/sandbox", &sandbox_path_buf) orelse return error.FileNotFound; + + var source_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const source_path = try r.rlocation(kernel.source_path, &source_path_buf) orelse return error.FileNotFound; + const source = try std.Io.Dir.cwd().readFileAlloc(io, source_path, allocator, .unlimited); + defer allocator.free(source); + + var nki_cc_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const nki_cc_path = try stdx.Io.Dir.path.bufJoin(&nki_cc_buf, &.{ sandbox_path, "bin", "nki-cc" }); + + const bin_dirpath = std.fs.path.dirname(nki_cc_path) orelse return error.InvalidSandboxPath; + + const request: CompilerRequest = .{ + .entrypoint = kernel.entrypoint, + .source = source, + .target = kernel.compiler_target, + .tool_bin_dir = bin_dirpath, + .neuronx_cc_args = compiler_args.items, + .inputs = kernel.inputs, + .outputs = kernel.outputs, + }; + + const request_json = try std.fmt.allocPrint(allocator, "{f}", .{std.json.fmt(request, .{ .emit_null_optional_fields = false })}); + defer allocator.free(request_json); + + try request_file.writePositionalAll(io, request_json, 0); + + var spawn_threaded: std.Io.Threaded = .init(std.heap.c_allocator, .{}); + defer spawn_threaded.deinit(); + + const spawn_io = spawn_threaded.io(); + + var child = try std.process.spawn(spawn_io, .{ + .argv = &.{ + nki_cc_path, + "--request", + request_file_buf[0..request_file_path], + "--result", + result_file_buf[0..result_file_path], + }, + .stdin = .ignore, + .stdout = .inherit, + .stderr = .inherit, + .cwd = .{ .path = cwd_buf[0..cwd] }, + }); + + const term = try child.wait(spawn_io); + switch (term) { + .exited => |code| { + if (code != 0) { + log.err("nki-cc exited with code {}", .{code}); + return error.NkiCcFailed; + } + }, + .signal => |sig| { + log.err("nki-cc terminated by signal {}", .{sig}); + return error.NkiCcFailed; + }, + else => |status| { + log.err("nki-cc terminated unexpectedly: {}", .{status}); + return error.NkiCcFailed; + }, + } + + var result_reader = result_file.reader(io, &.{}); + const data = try result_reader.interface.readAlloc(allocator, try result_file.length(io)); + defer allocator.free(data); + + const parsed_response = try std.json.parseFromSlice(CompilerResponse, allocator, data, .{ + .ignore_unknown_fields = true, + }); + defer parsed_response.deinit(); + + const backend_config = try allocator.dupe(u8, parsed_response.value.backend_config_b64); + errdefer allocator.free(backend_config); + + return backend_config; +} diff --git a/platforms/neuron/nki_kernel_compiler.py b/platforms/neuron/nki_kernel_compiler.py new file mode 100755 index 00000000..207d0c26 --- /dev/null +++ b/platforms/neuron/nki_kernel_compiler.py @@ -0,0 +1,179 @@ +"""Compile inline ZML NKI kernels into Neuron custom-native backend configs. + +The `nki_kernel.zig` helper writes one JSON request per inline kernel. +This script asks NKI to produce BIR and returns the backend_config expected by +`AwsNeuronCustomNativeKernel`. +""" + +import argparse +import inspect +import json +import re +import sys +import tempfile +import types +from dataclasses import dataclass +from pathlib import Path + +import ml_dtypes +import numpy as np + + +@dataclass(frozen=True) +class TensorSpec: + dtype: str + shape: tuple[int, ...] + + +def dtype_from_name(name: str): + mapping = { + "bool": np.bool_, + "i8": np.int8, + "i16": np.int16, + "i32": np.int32, + "i64": np.int64, + "u8": np.uint8, + "u16": np.uint16, + "u32": np.uint32, + "u64": np.uint64, + "f16": np.float16, + "f32": np.float32, + "f64": np.float64, + } + + if name == "bf16": + return ml_dtypes.bfloat16 + + try: + return mapping[name] + except KeyError as exc: + raise ValueError(f"unsupported dtype: {name}") from exc + + +def parse_tensor_specs(raw_specs: list[dict]) -> tuple[TensorSpec, ...]: + return tuple( + TensorSpec( + dtype=spec["dtype"], + shape=tuple(spec.get("shape", spec["dims"])), + ) + for spec in raw_specs + ) + + +def load_kernel(source_path: Path, source: str, entrypoint: str): + module_name = re.sub(r"\W", "_", source_path.stem) or "kernel" + if module_name[0].isdigit(): + module_name = f"kernel_{module_name}" + + module = types.ModuleType(module_name) + module.__file__ = str(source_path) + module.__dict__["__builtins__"] = __builtins__ + sys.modules[module_name] = module + exec(compile(source, str(source_path), "exec"), module.__dict__) + + try: + return module.__dict__[entrypoint] + except KeyError as exc: + raise KeyError(f"entrypoint {entrypoint!r} not found in {source_path}") from exc + + +def make_inputs(inputs: tuple[TensorSpec, ...]) -> list[np.ndarray]: + """Create zero-filled specialization inputs matching the declared ZML signature.""" + arrays: list[np.ndarray] = [] + for input_spec in inputs: + arrays.append( + np.zeros( + shape=input_spec.shape, + dtype=dtype_from_name(input_spec.dtype), + ) + ) + return arrays + + +def compile_kernel( + source: str, + entrypoint: str, + target: str, + tool_bin_dir: str, + neuronx_cc_args: tuple[str, ...], + inputs: tuple[TensorSpec, ...], + outputs: tuple[TensorSpec, ...], +) -> str: + """Run the full embedded-kernel pipeline for one custom-call.""" + tmp_dir = Path(tempfile.mkdtemp(prefix="zml-nki-", dir=".")) + + source_path = tmp_dir / "kernel.py" + artifacts_dir = tmp_dir / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=False) + + source_path.write_text(source, encoding="utf-8") + + kernel = load_kernel(source_path, source, entrypoint) + + input_names = [ + parameter.name + for parameter in inspect.signature(kernel).parameters.values() + if parameter.kind + in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD) + ] + + if len(input_names) != len(inputs): + raise ValueError( + f"{entrypoint} expects {len(input_names)} inputs, ZML declared {len(inputs)}" + ) + + from nki.compiler.driver import compile_to_bir + from nki.compiler.frontend import ParserFrontend + from nki.compiler.ncc_driver import CompilationMode, CompileOptions + + compile_opts = CompileOptions( + target=target, + artifacts_dir=str(artifacts_dir), + output_path=str(artifacts_dir / "unused.neff"), + neuronx_cc_path=str(Path(tool_bin_dir) / "neuronx-cc"), + neuronx_cc_args=neuronx_cc_args, + mode=CompilationMode.INTEGRATION, + ).disable_backend_optimizations() + + nir = compile_to_bir( + kernel, + frontend=ParserFrontend(), + inputs=dict(zip(input_names, make_inputs(inputs), strict=True)), + compile_opts=compile_opts, + output_names=[f"output{i}" for i in range(len(outputs))], + ) + + return nir.build_config().backend_config_b64.decode("utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Compile a ZML NKI kernel to a custom-native backend config" + ) + parser.add_argument("--request", required=True) + parser.add_argument("--result", required=True) + args = parser.parse_args() + + request_path = Path(args.request) + result_path = Path(args.result) + + request = json.loads(request_path.read_text(encoding="utf-8")) + + backend_config_b64 = compile_kernel( + source=request["source"], + entrypoint=request["entrypoint"], + target=request["target"], + tool_bin_dir=request["tool_bin_dir"], + neuronx_cc_args=tuple(request["neuronx_cc_args"]), + inputs=parse_tensor_specs(request["inputs"]), + outputs=parse_tensor_specs(request["outputs"]), + ) + + result_path.write_text( + json.dumps({"backend_config_b64": backend_config_b64}), + encoding="utf-8", + ) + + +if __name__ == "__main__": + main() diff --git a/platforms/neuron/packages.lock.json b/platforms/neuron/packages.lock.json index 134d55e4..2f275bba 100755 --- a/platforms/neuron/packages.lock.json +++ b/platforms/neuron/packages.lock.json @@ -3,60 +3,24 @@ { "arch": "amd64", "dependencies": [ - { - "key": "debianutils_5.7-0.5_deb12u1_amd64", - "name": "debianutils", - "version": "5.7-0.5~deb12u1" - }, { "key": "libc6_2.36-9-p-deb12u10_amd64", "name": "libc6", "version": "2.36-9+deb12u10" }, - { - "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", - "name": "libgcc-s1", - "version": "12.2.0-14+deb12u1" - }, { "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", "name": "gcc-12-base", "version": "12.2.0-14+deb12u1" - }, - { - "key": "base-files_12.4-p-deb12u11_amd64", - "name": "base-files", - "version": "12.4+deb12u11" - }, - { - "key": "mawk_1.3.4.20200120-3.1_amd64", - "name": "mawk", - "version": "1.3.4.20200120-3.1" - }, - { - "key": "libtinfo6_6.4-4_amd64", - "name": "libtinfo6", - "version": "6.4-4" } ], - "key": "bash_5.2.15-2-p-b8_amd64", - "name": "bash", - "sha256": "23910515e6b8b4fb4a3b50549464ef2efdc149b568150e2aa45115b37b6e6578", + "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", + "name": "libgcc-s1", + "sha256": "3016e62cb4b7cd8038822870601f5ed131befe942774d0f745622cc77d8a88f7", "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/b/bash/bash_5.2.15-2+b8_amd64.deb" + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/g/gcc-12/libgcc-s1_12.2.0-14+deb12u1_amd64.deb" ], - "version": "5.2.15-2+b8" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "debianutils_5.7-0.5_deb12u1_amd64", - "name": "debianutils", - "sha256": "55f951359670eb3236c9e2ccd5fac9ccb3db734f5a22aff21589e7a30aee48c9", - "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/d/debianutils/debianutils_5.7-0.5~deb12u1_amd64.deb" - ], - "version": "5.7-0.5~deb12u1" + "version": "12.2.0-14+deb12u1" }, { "arch": "amd64", @@ -69,17 +33,6 @@ ], "version": "2.36-9+deb12u10" }, - { - "arch": "amd64", - "dependencies": [], - "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", - "name": "libgcc-s1", - "sha256": "3016e62cb4b7cd8038822870601f5ed131befe942774d0f745622cc77d8a88f7", - "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/g/gcc-12/libgcc-s1_12.2.0-14+deb12u1_amd64.deb" - ], - "version": "12.2.0-14+deb12u1" - }, { "arch": "amd64", "dependencies": [], @@ -91,39 +44,6 @@ ], "version": "12.2.0-14+deb12u1" }, - { - "arch": "amd64", - "dependencies": [], - "key": "base-files_12.4-p-deb12u11_amd64", - "name": "base-files", - "sha256": "c0b28aa96c88ca2e2f18c44911b6cd07de9335e0de7922fba778886a50061e64", - "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/b/base-files/base-files_12.4+deb12u11_amd64.deb" - ], - "version": "12.4+deb12u11" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "mawk_1.3.4.20200120-3.1_amd64", - "name": "mawk", - "sha256": "bcbc83f391854ea9d50ce2a4101aacf330de3b8b71d81a798faadba14a157f78", - "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/m/mawk/mawk_1.3.4.20200120-3.1_amd64.deb" - ], - "version": "1.3.4.20200120-3.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libtinfo6_6.4-4_amd64", - "name": "libtinfo6", - "sha256": "072d908f38f51090ca28ca5afa3b46b2957dc61fe35094c0b851426859a49a51", - "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/n/ncurses/libtinfo6_6.4-4_amd64.deb" - ], - "version": "6.4-4" - }, { "arch": "amd64", "dependencies": [ @@ -151,6 +71,33 @@ ], "version": "12.2.0-14+deb12u1" }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "libc6_2.36-9-p-deb12u10_amd64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + } + ], + "key": "libstdc-p--p-6_12.2.0-14-p-deb12u1_amd64", + "name": "libstdc++6", + "sha256": "5cd3171216d4ab0fc911cfe9c35509bf2dd8f47761c43b7f6a4296701551a24d", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/g/gcc-12/libstdc++6_12.2.0-14+deb12u1_amd64.deb" + ], + "version": "12.2.0-14+deb12u1" + }, { "arch": "amd64", "dependencies": [ @@ -181,13 +128,13 @@ { "arch": "amd64", "dependencies": [], - "key": "aws-neuronx-runtime-lib_2.29.40.0-f954cd7a5_amd64", + "key": "aws-neuronx-runtime-lib_2.32.31.0-0234f5ed2_amd64", "name": "aws-neuronx-runtime-lib", - "sha256": "60f5dc0c8731fdd48a9726493375dbb0f0d178d05d84f70716bcc36789fd9be9", + "sha256": "ed69354674b926490d917a6aeb7dea8b6300278a37958ee9a412e2cd89c41792", "urls": [ - "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-runtime-lib/aws-neuronx-runtime-lib_2.29.40.0-f954cd7a5_amd64.deb" + "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-runtime-lib/aws-neuronx-runtime-lib_2.32.31.0-0234f5ed2_amd64.deb" ], - "version": "2.29.40.0-f954cd7a5" + "version": "2.32.31.0-0234f5ed2" }, { "arch": "amd64", @@ -333,13 +280,13 @@ "version": "252.36-1~deb12u1" } ], - "key": "aws-neuronx-collectives_2.29.41.0-681fef5f5_amd64", + "key": "aws-neuronx-collectives_2.32.28.0-452cba8de_amd64", "name": "aws-neuronx-collectives", - "sha256": "c4bd37dfd6a18883ea87b9e7e665ee266f3e344536d0f4e9c9d6da4aa38e7417", + "sha256": "ec737c6647b6fbd38550e11f25e27d0f1a96dbb2cd28ae5bbf3f827faba5bdc7", "urls": [ - "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-collectives/aws-neuronx-collectives_2.29.41.0-681fef5f5_amd64.deb" + "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-collectives/aws-neuronx-collectives_2.32.28.0-452cba8de_amd64.deb" ], - "version": "2.29.41.0-681fef5f5" + "version": "2.32.28.0-452cba8de" }, { "arch": "amd64", @@ -615,6 +562,17 @@ "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/s/systemd/libudev1_252.36-1~deb12u1_amd64.deb" ], "version": "252.36-1~deb12u1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "aws-neuronx-tools_2.30.10.0-39feaa591_amd64", + "name": "aws-neuronx-tools", + "sha256": "b972e4db5e84a429aa237d1eb9347fcaa78c19e848049a3495efef734a0b240f", + "urls": [ + "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-tools/aws-neuronx-tools_2.30.10.0-39feaa591_amd64.deb" + ], + "version": "2.30.10.0-39feaa591" } ], "version": 1 diff --git a/platforms/neuron/packages.yaml b/platforms/neuron/packages.yaml index 6c3de28a..eb399f49 100644 --- a/platforms/neuron/packages.yaml +++ b/platforms/neuron/packages.yaml @@ -18,8 +18,10 @@ archs: - "amd64" packages: - - "bash" # needed to launch neuronx-cc + - "libgcc-s1" # implicit dep of libneuronpjrt/libstdc++ - "libgomp1" # implicit dep of neuronx-cc - - "zlib1g" # implicit dep of libpython + - "libstdc++6" # implicit dep of libneuronpjrt/libnccom + - "zlib1g" # implicit dep of the sandboxed Python runtime - "aws-neuronx-runtime-lib" - "aws-neuronx-collectives" + - "aws-neuronx-tools" diff --git a/platforms/neuron/python_launcher.zig b/platforms/neuron/python_launcher.zig new file mode 100644 index 00000000..26ca9766 --- /dev/null +++ b/platforms/neuron/python_launcher.zig @@ -0,0 +1,155 @@ +const std = @import("std"); + +const c = @import("c"); + +// Shared isolated-Python launcher used by Neuron helper binaries. The sandbox +// contains the Python stdlib, site-packages, and these tiny Zig executables; the +// helpers configure Python from their own location so they are hermetic. + +pub const ModuleEntrypoint = struct { + module_name: [:0]const u8, + attr_name: [:0]const u8 = "main", +}; + +// Run `module.attr()` inside the sandboxed Python interpreter. +pub fn runModuleEntrypoint(init: std.process.Init, entrypoint: ModuleEntrypoint) !void { + var config: c.PyConfig = undefined; + _ = try configureInterpreter(init, &config); + defer c.PyConfig_Clear(&config); + + pyStatusCheck(c.Py_InitializeFromConfig(&config)); + defer c.Py_Finalize(); + + const module = c.PyImport_ImportModule(entrypoint.module_name.ptr) orelse { + std.log.err("Failed to import {s}", .{entrypoint.module_name}); + c.PyErr_Print(); + return error.FailedToImportModule; + }; + defer c.Py_DecRef(module); + + const main_fn = c.PyObject_GetAttrString(module, entrypoint.attr_name.ptr) orelse { + std.log.err("Failed to resolve {s}.{s}", .{ entrypoint.module_name, entrypoint.attr_name }); + c.PyErr_Print(); + return error.FailedToResolveEntrypoint; + }; + defer c.Py_DecRef(main_fn); + + const result = c.PyObject_CallNoArgs(main_fn); + if (result == null) { + c.PyErr_Print(); + std.process.exit(1); + } + defer c.Py_DecRef(result); +} + +// Run a Python file relative to the helper binary and call its `main()`. +pub fn runScriptMain(init: std.process.Init, relative_script_path: []const []const u8) !void { + const arena = init.arena; + + var config: c.PyConfig = undefined; + const self_exe_dir = try configureInterpreter(init, &config); + defer c.PyConfig_Clear(&config); + + pyStatusCheck(c.Py_InitializeFromConfig(&config)); + defer c.Py_Finalize(); + + const script_path = try std.Io.Dir.path.join(arena.allocator(), relative_script_path); + const full_script_path = try std.Io.Dir.path.join(arena.allocator(), &.{ self_exe_dir, script_path }); + const bootstrap = try std.fmt.allocPrint(arena.allocator(), + \\import importlib.util + \\spec = importlib.util.spec_from_file_location("_zml_neuron_python_launcher", "{s}") + \\module = importlib.util.module_from_spec(spec) + \\assert spec.loader is not None + \\spec.loader.exec_module(module) + \\module.main() + , .{full_script_path}); + + if (c.PyRun_SimpleStringFlags(bootstrap.ptr, null) != 0) { + c.PyErr_Print(); + return error.CompilerExecutionFailed; + } +} + +fn pyStatusCheck(status: c.PyStatus) void { + if (c.PyStatus_Exception(status) != 0) { + if (c.PyStatus_IsExit(status) != 0) { + std.process.exit(@intCast(status.exitcode)); + } + c.Py_ExitStatusException(status); + } +} + +fn toPosixPathW(file_path: []const u8) error{ InvalidUtf8, NameTooLong, CodepointTooLarge }![std.posix.PATH_MAX - 1:0]c.wchar_t { + var view = std.unicode.Utf8View.init(file_path) catch return error.InvalidUtf8; + var it = view.iterator(); + + var path_with_null: [std.posix.PATH_MAX - 1:0]c.wchar_t = undefined; + var len: usize = 0; + while (it.nextCodepoint()) |cp| { + if (len + 1 >= path_with_null.len) return error.NameTooLong; + path_with_null[len] = std.math.cast(c.wchar_t, cp) orelse return error.CodepointTooLarge; + len += 1; + } + path_with_null[len] = 0; + return path_with_null; +} + +fn configureInterpreter(init: std.process.Init, config: *c.PyConfig) ![]const u8 { + const arena = init.arena; + const io = init.io; + + { + var preconfig: c.PyPreConfig = undefined; + c.PyPreConfig_InitIsolatedConfig(&preconfig); + preconfig.utf8_mode = 1; + pyStatusCheck(c.Py_PreInitialize(&preconfig)); + } + + c.PyConfig_InitIsolatedConfig(config); + config.module_search_paths_set = 1; + config.optimization_level = 2; + config.write_bytecode = 0; + + const args = try init.minimal.args.toSlice(arena.allocator()); + const c_args = try arena.allocator().alloc([*:0]const u8, args.len); + for (args, 0..) |arg, i| { + c_args[i] = (try arena.allocator().dupeZ(u8, arg)).ptr; + } + pyStatusCheck(c.PyConfig_SetBytesArgv(config, @intCast(c_args.len), @ptrCast(c_args.ptr))); + + var self_exe_dir_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const self_exe_dir = blk: { + const n = try std.process.executableDirPath(io, &self_exe_dir_buf); + break :blk self_exe_dir_buf[0..n]; + }; + + { + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const home = try std.fmt.bufPrintZ(&buf, "{f}{d}.{d}", .{ + std.Io.Dir.path.fmtJoin(&.{ + self_exe_dir, + "..", + "lib", + "python", + }), + c.PY_MAJOR_VERSION, + c.PY_MINOR_VERSION, + }); + pyStatusCheck(c.PyConfig_SetBytesString(config, &config.home, home)); + pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(home))); + } + + { + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const site_packages = try std.fmt.bufPrint(&buf, "{f}", .{ + std.Io.Dir.path.fmtJoin(&.{ + self_exe_dir, + "..", + "site-packages", + }), + }); + pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(site_packages))); + } + + return self_exe_dir; +} diff --git a/platforms/neuron/requirements.in b/platforms/neuron/requirements.in index f574686b..51174303 100644 --- a/platforms/neuron/requirements.in +++ b/platforms/neuron/requirements.in @@ -1,4 +1,5 @@ --find-links https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html --extra-index-url=https://pip.repos.neuron.amazonaws.com -libneuronxla>=2.2 -neuronx-cc>=2.22 +libneuronxla>=3.0 +neuronx-cc>=2.25 +nki>=0.4 diff --git a/platforms/neuron/requirements.lock.txt b/platforms/neuron/requirements.lock.txt index e18a16f4..0945aa7d 100644 --- a/platforms/neuron/requirements.lock.txt +++ b/platforms/neuron/requirements.lock.txt @@ -4,20 +4,20 @@ --extra-index-url https://pip.repos.neuron.amazonaws.com --find-links https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html -boto3==1.43.3 \ - --hash=sha256:7c7777862ffc898f05efa566032bbabfe226dbb810e35ec11125817f128bc5c5 \ - --hash=sha256:fb9fe51849ef2a78198d582756fc06f14f7de27f73e0fa90275d6aa4171eb4d0 +boto3==1.43.14 \ + --hash=sha256:574335744656cfed0b362a0a0467aaf2eb2bf15526edcd02d31d3c661f4b09e4 \ + --hash=sha256:5c0a994b3182061ee101812e721100717a4d664f9f4ceaf4a86b6d032ce9fc2d # via libneuronxla -botocore==1.43.3 \ - --hash=sha256:eac6da0fffccf87888ebf4d89f0b2378218a707efa748cd955b838995e944695 \ - --hash=sha256:ec0769eb0f7c5034856bb406a92698dbc02a3d4be0f78a384747106b161d8ea3 +botocore==1.43.14 \ + --hash=sha256:1f4a2a95ea78c10398e78431e98c1fe47adb54a7b10a32975144c1f541186658 \ + --hash=sha256:b9e500737e43d2f147c9d4e23b54360335e77d4c0ba90a318f51b65e06cb8516 # via # boto3 # libneuronxla # s3transfer -certifi==2026.4.22 \ - --hash=sha256:3cb2210c8f88ba2318d29b0388d1023c8492ff72ecdde4ebdaddbb13a31b1c4a \ - --hash=sha256:8d455352a37b71bf76a79caa83a3d6c25afee4a385d632127b6afb3963f1c580 +certifi==2026.5.20 \ + --hash=sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897 \ + --hash=sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d # via requests charset-normalizer==3.4.7 \ --hash=sha256:007d05ec7321d12a40227aae9e2bc6dca73f3cb21058999a1df9e193555a9dcc \ @@ -150,147 +150,13 @@ charset-normalizer==3.4.7 \ --hash=sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79 \ --hash=sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464 # via requests -contourpy==1.3.3 \ - --hash=sha256:023b44101dfe49d7d53932be418477dba359649246075c996866106da069af69 \ - --hash=sha256:07ce5ed73ecdc4a03ffe3e1b3e3c1166db35ae7584be76f65dbbe28a7791b0cc \ - --hash=sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880 \ - --hash=sha256:0bf67e0e3f482cb69779dd3061b534eb35ac9b17f163d851e2a547d56dba0a3a \ - --hash=sha256:0c1fc238306b35f246d61a1d416a627348b5cf0648648a031e14bb8705fcdfe8 \ - --hash=sha256:13b68d6a62db8eafaebb8039218921399baf6e47bf85006fd8529f2a08ef33fc \ - --hash=sha256:15ff10bfada4bf92ec8b31c62bf7c1834c244019b4a33095a68000d7075df470 \ - --hash=sha256:177fb367556747a686509d6fef71d221a4b198a3905fe824430e5ea0fda54eb5 \ - --hash=sha256:1cadd8b8969f060ba45ed7c1b714fe69185812ab43bd6b86a9123fe8f99c3263 \ - --hash=sha256:1fd43c3be4c8e5fd6e4f2baeae35ae18176cf2e5cced681cca908addf1cdd53b \ - --hash=sha256:22e9b1bd7a9b1d652cd77388465dc358dafcd2e217d35552424aa4f996f524f5 \ - --hash=sha256:23416f38bfd74d5d28ab8429cc4d63fa67d5068bd711a85edb1c3fb0c3e2f381 \ - --hash=sha256:283edd842a01e3dcd435b1c5116798d661378d83d36d337b8dde1d16a5fc9ba3 \ - --hash=sha256:2a2a8b627d5cc6b7c41a4beff6c5ad5eb848c88255fda4a8745f7e901b32d8e4 \ - --hash=sha256:2b7e9480ffe2b0cd2e787e4df64270e3a0440d9db8dc823312e2c940c167df7e \ - --hash=sha256:322ab1c99b008dad206d406bb61d014cf0174df491ae9d9d0fac6a6fda4f977f \ - --hash=sha256:33c82d0138c0a062380332c861387650c82e4cf1747aaa6938b9b6516762e772 \ - --hash=sha256:348ac1f5d4f1d66d3322420f01d42e43122f43616e0f194fc1c9f5d830c5b286 \ - --hash=sha256:3519428f6be58431c56581f1694ba8e50626f2dd550af225f82fb5f5814d2a42 \ - --hash=sha256:3c30273eb2a55024ff31ba7d052dde990d7d8e5450f4bbb6e913558b3d6c2301 \ - --hash=sha256:3d1a3799d62d45c18bafd41c5fa05120b96a28079f2393af559b843d1a966a77 \ - --hash=sha256:451e71b5a7d597379ef572de31eeb909a87246974d960049a9848c3bc6c41bf7 \ - --hash=sha256:459c1f020cd59fcfe6650180678a9993932d80d44ccde1fa1868977438f0b411 \ - --hash=sha256:4d00e655fcef08aba35ec9610536bfe90267d7ab5ba944f7032549c55a146da1 \ - --hash=sha256:4debd64f124ca62069f313a9cb86656ff087786016d76927ae2cf37846b006c9 \ - --hash=sha256:4feffb6537d64b84877da813a5c30f1422ea5739566abf0bd18065ac040e120a \ - --hash=sha256:50ed930df7289ff2a8d7afeb9603f8289e5704755c7e5c3bbd929c90c817164b \ - --hash=sha256:51e79c1f7470158e838808d4a996fa9bac72c498e93d8ebe5119bc1e6becb0db \ - --hash=sha256:556dba8fb6f5d8742f2923fe9457dbdd51e1049c4a43fd3986a0b14a1d815fc6 \ - --hash=sha256:598c3aaece21c503615fd59c92a3598b428b2f01bfb4b8ca9c4edeecc2438620 \ - --hash=sha256:5ed3657edf08512fc3fe81b510e35c2012fbd3081d2e26160f27ca28affec989 \ - --hash=sha256:626d60935cf668e70a5ce6ff184fd713e9683fb458898e4249b63be9e28286ea \ - --hash=sha256:644a6853d15b2512d67881586bd03f462c7ab755db95f16f14d7e238f2852c67 \ - --hash=sha256:655456777ff65c2c548b7c454af9c6f33f16c8884f11083244b5819cc214f1b5 \ - --hash=sha256:66c8a43a4f7b8df8b71ee1840e4211a3c8d93b214b213f590e18a1beca458f7d \ - --hash=sha256:6afc576f7b33cf00996e5c1102dc2a8f7cc89e39c0b55df93a0b78c1bd992b36 \ - --hash=sha256:6c3d53c796f8647d6deb1abe867daeb66dcc8a97e8455efa729516b997b8ed99 \ - --hash=sha256:709a48ef9a690e1343202916450bc48b9e51c049b089c7f79a267b46cffcdaa1 \ - --hash=sha256:70f9aad7de812d6541d29d2bbf8feb22ff7e1c299523db288004e3157ff4674e \ - --hash=sha256:8153b8bfc11e1e4d75bcb0bff1db232f9e10b274e0929de9d608027e0d34ff8b \ - --hash=sha256:87acf5963fc2b34825e5b6b048f40e3635dd547f590b04d2ab317c2619ef7ae8 \ - --hash=sha256:88df9880d507169449d434c293467418b9f6cbe82edd19284aa0409e7fdb933d \ - --hash=sha256:929ddf8c4c7f348e4c0a5a3a714b5c8542ffaa8c22954862a46ca1813b667ee7 \ - --hash=sha256:92d9abc807cf7d0e047b95ca5d957cf4792fcd04e920ca70d48add15c1a90ea7 \ - --hash=sha256:95b181891b4c71de4bb404c6621e7e2390745f887f2a026b2d99e92c17892339 \ - --hash=sha256:9e999574eddae35f1312c2b4b717b7885d4edd6cb46700e04f7f02db454e67c1 \ - --hash=sha256:a15459b0f4615b00bbd1e91f1b9e19b7e63aea7483d03d804186f278c0af2659 \ - --hash=sha256:a22738912262aa3e254e4f3cb079a95a67132fc5a063890e224393596902f5a4 \ - --hash=sha256:ab2fd90904c503739a75b7c8c5c01160130ba67944a7b77bbf36ef8054576e7f \ - --hash=sha256:ab3074b48c4e2cf1a960e6bbeb7f04566bf36b1861d5c9d4d8ac04b82e38ba20 \ - --hash=sha256:afe5a512f31ee6bd7d0dda52ec9864c984ca3d66664444f2d72e0dc4eb832e36 \ - --hash=sha256:b08a32ea2f8e42cf1d4be3169a98dd4be32bafe4f22b6c4cb4ba810fa9e5d2cb \ - --hash=sha256:b20c7c9a3bf701366556e1b1984ed2d0cedf999903c51311417cf5f591d8c78d \ - --hash=sha256:b2e8faa0ed68cb29af51edd8e24798bb661eac3bd9f65420c1887b6ca89987c8 \ - --hash=sha256:b7301b89040075c30e5768810bc96a8e8d78085b47d8be6e4c3f5a0b4ed478a0 \ - --hash=sha256:b7448cb5a725bb1e35ce88771b86fba35ef418952474492cf7c764059933ff8b \ - --hash=sha256:ca0fdcd73925568ca027e0b17ab07aad764be4706d0a925b89227e447d9737b7 \ - --hash=sha256:ca658cd1a680a5c9ea96dc61cdbae1e85c8f25849843aa799dfd3cb370ad4fbe \ - --hash=sha256:cbedb772ed74ff5be440fa8eee9bd49f64f6e3fc09436d9c7d8f1c287b121d77 \ - --hash=sha256:cd5dfcaeb10f7b7f9dc8941717c6c2ade08f587be2226222c12b25f0483ed497 \ - --hash=sha256:cf9022ef053f2694e31d630feaacb21ea24224be1c3ad0520b13d844274614fd \ - --hash=sha256:d002b6f00d73d69333dac9d0b8d5e84d9724ff9ef044fd63c5986e62b7c9e1b1 \ - --hash=sha256:d06bb1f751ba5d417047db62bca3c8fde202b8c11fb50742ab3ab962c81e8216 \ - --hash=sha256:d304906ecc71672e9c89e87c4675dc5c2645e1f4269a5063b99b0bb29f232d13 \ - --hash=sha256:e4e6b05a45525357e382909a4c1600444e2a45b4795163d3b22669285591c1ae \ - --hash=sha256:e74a9a0f5e3fff48fb5a7f2fd2b9b70a3fe014a67522f79b7cca4c0c7e43c9ae \ - --hash=sha256:ea37e7b45949df430fe649e5de8351c423430046a2af20b1c1961cae3afcda77 \ - --hash=sha256:f64836de09927cba6f79dcd00fdd7d5329f3fccc633468507079c829ca4db4e3 \ - --hash=sha256:fd6ec6be509c787f1caf6b247f0b1ca598bef13f4ddeaa126b7658215529ba0f \ - --hash=sha256:fd907ae12cd483cd83e414b12941c632a969171bf90fc937d0c9f268a31cafff \ - --hash=sha256:fd914713266421b7536de2bfa8181aa8c699432b6763a0ea64195ebe28bff6a9 \ - --hash=sha256:fde6c716d51c04b1c25d0b90364d0be954624a0ee9d60e23e850e8d48353d07a - # via matplotlib -cycler==0.12.1 \ - --hash=sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30 \ - --hash=sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c - # via matplotlib ec2-metadata==2.17.0 \ --hash=sha256:a696a899cb8ee1f0101eb9ec7bc2050e1ffc8574ee4cd748a52d9c748fb19d9e \ --hash=sha256:ad98a5afd8f4f49f688e4899dc505257da3237348761eceeb0fc7fc7d00cc90d # via neuronx-cc -einops==0.8.1 \ - --hash=sha256:919387eb55330f5757c6bea9165c5ff5cfe63a642682ea788a6d472576d81737 \ - --hash=sha256:de5d960a7a761225532e0f1959e5315ebeafc0cd43394732f103ca44b9837e84 - # via nki -fonttools==4.62.1 \ - --hash=sha256:0aa72c43a601cfa9273bb1ae0518f1acadc01ee181a6fc60cd758d7fdadffc04 \ - --hash=sha256:0b3ae47e8636156a9accff64c02c0924cbebad62854c4a6dbdc110cd5b4b341a \ - --hash=sha256:12859ff0b47dd20f110804c3e0d0970f7b832f561630cd879969011541a464a9 \ - --hash=sha256:149f7d84afca659d1a97e39a4778794a2f83bf344c5ee5134e09995086cc2392 \ - --hash=sha256:1596aeaddf7f78e21e68293c011316a25267b3effdaccaf4d59bc9159d681b82 \ - --hash=sha256:19177c8d96c7c36359266e571c5173bcee9157b59cfc8cb0153c5673dc5a3a7d \ - --hash=sha256:1c5c25671ce8805e0d080e2ffdeca7f1e86778c5cbfbeae86d7f866d8830517b \ - --hash=sha256:1eecc128c86c552fb963fe846ca4e011b1be053728f798185a1687502f6d398e \ - --hash=sha256:268abb1cb221e66c014acc234e872b7870d8b5d4657a83a8f4205094c32d2416 \ - --hash=sha256:2d850f66830a27b0d498ee05adb13a3781637b1826982cd7e2b3789ef0cc71ae \ - --hash=sha256:2e7abd2b1e11736f58c1de27819e1955a53267c21732e78243fa2fa2e5c1e069 \ - --hash=sha256:403d28ce06ebfc547fbcb0cb8b7f7cc2f7a2d3e1a67ba9a34b14632df9e080f9 \ - --hash=sha256:40975849bac44fb0b9253d77420c6d8b523ac4dcdcefeff6e4d706838a5b80f7 \ - --hash=sha256:486f32c8047ccd05652aba17e4a8819a3a9d78570eb8a0e3b4503142947880ed \ - --hash=sha256:49a445d2f544ce4a69338694cad575ba97b9a75fff02720da0882d1a73f12800 \ - --hash=sha256:59b372b4f0e113d3746b88985f1c796e7bf830dd54b28374cd85c2b8acd7583e \ - --hash=sha256:5a648bde915fba9da05ae98856987ca91ba832949a9e2888b48c47ef8b96c5a9 \ - --hash=sha256:5f37df1cac61d906e7b836abe356bc2f34c99d4477467755c216b72aa3dc748b \ - --hash=sha256:6706d1cb1d5e6251a97ad3c1b9347505c5615c112e66047abbef0f8545fa30d1 \ - --hash=sha256:68959f5fc58ed4599b44aad161c2837477d7f35f5f79402d97439974faebfebe \ - --hash=sha256:6acb4109f8bee00fec985c8c7afb02299e35e9c94b57287f3ea542f28bd0b0a7 \ - --hash=sha256:7487782e2113861f4ddcc07c3436450659e3caa5e470b27dc2177cade2d8e7fd \ - --hash=sha256:7aa21ff53e28a9c2157acbc44e5b401149d3c9178107130e82d74ceb500e5056 \ - --hash=sha256:7bca7a1c1faf235ffe25d4f2e555246b4750220b38de8261d94ebc5ce8a23c23 \ - --hash=sha256:8d337fdd49a79b0d51c4da87bc38169d21c3abbf0c1aa9367eff5c6656fb6dae \ - --hash=sha256:8f8fca95d3bb3208f59626a4b0ea6e526ee51f5a8ad5d91821c165903e8d9260 \ - --hash=sha256:90365821debbd7db678809c7491ca4acd1e0779b9624cdc6ddaf1f31992bf974 \ - --hash=sha256:92bb00a947e666169c99b43753c4305fc95a890a60ef3aeb2a6963e07902cc87 \ - --hash=sha256:93c316e0f5301b2adbe6a5f658634307c096fd5aae60a5b3412e4f3e1728ab24 \ - --hash=sha256:942b03094d7edbb99bdf1ae7e9090898cad7bf9030b3d21f33d7072dbcb51a53 \ - --hash=sha256:9c125ffa00c3d9003cdaaf7f2c79e6e535628093e14b5de1dccb08859b680936 \ - --hash=sha256:9dde91633f77fa576879a0c76b1d89de373cae751a98ddf0109d54e173b40f14 \ - --hash=sha256:9e7863e10b3de72376280b515d35b14f5eeed639d1aa7824f4cf06779ec65e42 \ - --hash=sha256:a24decd24d60744ee8b4679d38e88b8303d86772053afc29b19d23bb8207803c \ - --hash=sha256:a5d8825e1140f04e6c99bb7d37a9e31c172f3bc208afbe02175339e699c710e1 \ - --hash=sha256:aa69d10ed420d8121118e628ad47d86e4caa79ba37f968597b958f6cceab7eca \ - --hash=sha256:ad5cca75776cd453b1b035b530e943334957ae152a36a88a320e779d61fc980c \ - --hash=sha256:b4e0fcf265ad26e487c56cb12a42dffe7162de708762db951e1b3f755319507d \ - --hash=sha256:b820fcb92d4655513d8402d5b219f94481c4443d825b4372c75a2072aa4b357a \ - --hash=sha256:bd13b7999d59c5eb1c2b442eb2d0c427cb517a0b7a1f5798fc5c9e003f5ff782 \ - --hash=sha256:bdfe592802ef939a0e33106ea4a318eeb17822c7ee168c290273cbd5fabd746c \ - --hash=sha256:c05557a78f8fa514da0f869556eeda40887a8abc77c76ee3f74cf241778afd5a \ - --hash=sha256:c22b1014017111c401469e3acc5433e6acf6ebcc6aa9efb538a533c800971c79 \ - --hash=sha256:c9b9e288b4da2f64fd6180644221749de651703e8d0c16bd4b719533a3a7d6e3 \ - --hash=sha256:d241cdc4a67b5431c6d7f115fdf63335222414995e3a1df1a41e1182acd4bcc7 \ - --hash=sha256:e54c75fd6041f1122476776880f7c3c3295ffa31962dc6ebe2543c00dca58b5d \ - --hash=sha256:e8514f4924375f77084e81467e63238b095abda5107620f49421c368a6017ed2 \ - --hash=sha256:ee91628c08e76f77b533d65feb3fbe6d9dad699f95be51cf0d022db94089cdc4 \ - --hash=sha256:ef46db46c9447103b8f3ff91e8ba009d5fe181b1920a83757a5762551e32bb68 \ - --hash=sha256:fa1d16210b6b10a826d71bed68dd9ec24a9e218d5a5e2797f37c573e7ec215ca - # via matplotlib -idna==3.13 \ - --hash=sha256:585ea8fe5d69b9181ec1afba340451fba6ba764af97026f92a91d4eef164a242 \ - --hash=sha256:892ea0cde124a99ce773decba204c5552b69c3c67ffd5f232eb7696135bc8bb3 +idna==3.16 \ + --hash=sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5 \ + --hash=sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d # via requests islpy==2023.2.5 \ --hash=sha256:06446395028b88701fd84b685b1c82b9de493728906f96f576a4dca69d12dd1f \ @@ -340,226 +206,13 @@ jmespath==1.1.0 \ # via # boto3 # botocore -kiwisolver==1.5.0 \ - --hash=sha256:012b1eb16e28718fa782b5e61dc6f2da1f0792ca73bd05d54de6cb9561665fc9 \ - --hash=sha256:01808c6d15f4c3e8559595d6d1fe6411c68e4a3822b4b9972b44473b24f4e679 \ - --hash=sha256:0255a027391d52944eae1dbb5d4cc5903f57092f3674e8e544cdd2622826b3f0 \ - --hash=sha256:0b85aad90cea8ac6797a53b5d5f2e967334fa4d1149f031c4537569972596cb8 \ - --hash=sha256:0bf3acf1419fa93064a4c2189ac0b58e3be7872bf6ee6177b0d4c63dc4cea276 \ - --hash=sha256:0c50b89ffd3e1a911c69a1dd3de7173c0cd10b130f56222e57898683841e4f96 \ - --hash=sha256:0cbe94b69b819209a62cb27bdfa5dc2a8977d8de2f89dfd97ba4f53ed3af754e \ - --hash=sha256:0df54df7e686afa55e6f21fb86195224a6d9beb71d637e8d7920c95cf0f89aac \ - --hash=sha256:0e3aafb33aed7479377e5e9a82e9d4bf87063741fc99fc7ae48b0f16e32bdd6f \ - --hash=sha256:12e91c215a96e39f57989c8912ae761286ac5a9584d04030ceb3368a357f017a \ - --hash=sha256:1465387ac63576c3e125e5337a6892b9e99e0627d52317f3ca79e6930d889d15 \ - --hash=sha256:16b85d37c2cbb3253226d26e64663f755d88a03439a9c47df6246b35defbdfb7 \ - --hash=sha256:1b0feb50971481a2cc44d94e88bdb02cdd497618252ae226b8eb1201b957e368 \ - --hash=sha256:1d49a49ac4cbfb7c1375301cd1ec90169dfeae55ff84710d782260ce77a75a02 \ - --hash=sha256:1d9daea4ea6b9be74fe2f01f7fbade8d6ffab263e781274cffca0dba9be9eec9 \ - --hash=sha256:1dd9b0b119a350976a6d781e7278ec7aca0b201e1a9e2d23d9804afecb6ca681 \ - --hash=sha256:1f1489f769582498610e015a8ef2d36f28f505ab3096d0e16b4858a9ec214f57 \ - --hash=sha256:2517e24d7315eb51c10664cdb865195df38ab74456c677df67bb47f12d088a27 \ - --hash=sha256:295d9ffe712caa9f8a3081de8d32fc60191b4b51c76f02f951fd8407253528f4 \ - --hash=sha256:2a075bd7bd19c70cf67c8badfa36cf7c5d8de3c9ddb8420c51e10d9c50e94920 \ - --hash=sha256:32cc0a5365239a6ea0c6ed461e8838d053b57e397443c0ca894dcc8e388d4374 \ - --hash=sha256:332b4f0145c30b5f5ad9374881133e5aa64320428a57c2c2b61e9d891a51c2f3 \ - --hash=sha256:377815a8616074cabbf3f53354e1d040c35815a134e01d7614b7692e4bf8acfa \ - --hash=sha256:38f4a703656f493b0ad185211ccfca7f0386120f022066b018eb5296d8613e23 \ - --hash=sha256:3ac2360e93cb41be81121755c6462cff3beaa9967188c866e5fce5cf13170859 \ - --hash=sha256:3c4923e404d6bcd91b6779c009542e5647fef32e4a5d75e115e3bbac6f2335eb \ - --hash=sha256:3cdcb35dc9d807259c981a85531048ede628eabcffb3239adf3d17463518992d \ - --hash=sha256:41024ed50e44ab1a60d3fe0a9d15a4ccc9f5f2b1d814ff283c8d01134d5b81bc \ - --hash=sha256:413b820229730d358efd838ecbab79902fe97094565fdc80ddb6b0a18c18a581 \ - --hash=sha256:4432b835675f0ea7414aab3d37d119f7226d24869b7a829caeab49ebda407b0c \ - --hash=sha256:4db576bb8c3ef9365f8b40fe0f671644de6736ae2c27a2c62d7d8a1b4329f099 \ - --hash=sha256:4e7f886f47ab881692f278ae901039a234e4025a68e6dfab514263a0b1c4ae05 \ - --hash=sha256:4e9750bc21b886308024f8a54ccb9a2cc38ac9fa813bf4348434e3d54f337ff9 \ - --hash=sha256:5060731cc3ed12ca3a8b57acd4aeca5bbc2f49216dd0bec1650a1acd89486bcd \ - --hash=sha256:50847dca5d197fcbd389c805aa1a1cf32f25d2e7273dc47ab181a517666b68cc \ - --hash=sha256:5092eb5b1172947f57d6ea7d89b2f29650414e4293c47707eb499ec07a0ac796 \ - --hash=sha256:5124d1ea754509b09e53738ec185584cc609aae4a3b510aaf4ed6aa047ef9303 \ - --hash=sha256:51e8c4084897de9f05898c2c2a39af6318044ae969d46ff7a34ed3f96274adca \ - --hash=sha256:530a3fd64c87cffa844d4b6b9768774763d9caa299e9b75d8eca6a4423b31314 \ - --hash=sha256:56fa888f10d0f367155e76ce849fa1166fc9730d13bd2d65a2aa13b6f5424489 \ - --hash=sha256:58f812017cd2985c21fbffb4864d59174d4903dd66fa23815e74bbc7a0e2dd57 \ - --hash=sha256:59cd8683f575d96df5bb48f6add94afc055012c29e28124fcae2b63661b9efb1 \ - --hash=sha256:5ae8e62c147495b01a0f4765c878e9bfdf843412446a247e28df59936e99e797 \ - --hash=sha256:5b233ea3e165e43e35dba1d2b8ecc21cf070b45b65ae17dd2747d2713d942021 \ - --hash=sha256:6176c1811d9d5a04fa391c490cc44f451e240697a16977f11c6f722efb9041db \ - --hash=sha256:62f59da443c4f4849f73a51a193b1d9d258dcad0c41bc4d1b8fb2bcc04bfeb22 \ - --hash=sha256:6783e069732715ad0c3ce96dbf21dbc2235ab0593f2baf6338101f70371f4028 \ - --hash=sha256:6ab8ba9152203feec73758dad83af9a0bbe05001eb4639e547207c40cfb52083 \ - --hash=sha256:70d593af6a6ca332d1df73d519fddb5148edb15cd90d5f0155e3746a6d4fcc65 \ - --hash=sha256:72ec46b7eba5b395e0a7b63025490d3214c11013f4aacb4f5e8d6c3041829588 \ - --hash=sha256:7a32f72973f0f950c1920475d5c5ea3d971b81b6f0ec53b8d0a956cc965f22e0 \ - --hash=sha256:7a4aa69609f40fce3cbc3f87b2061f042eee32f94b8f11db707b66a26461591a \ - --hash=sha256:7c60d3c9b06fb23bd9c6139281ccbdc384297579ae037f08ae90c69f6845c0b1 \ - --hash=sha256:800ee55980c18545af444d93fdd60c56b580db5cc54867d8cbf8a1dc0829938c \ - --hash=sha256:80aa065ffd378ff784822a6d7c3212f2d5f5e9c3589614b5c228b311fd3063ac \ - --hash=sha256:86e0287879f75621ae85197b0877ed2f8b7aa57b511c7331dce2eb6f4de7d476 \ - --hash=sha256:893ff3a711d1b515ba9da14ee090519bad4610ed1962fbe298a434e8c5f8db53 \ - --hash=sha256:89fc958c702ee9a745e4700378f5d23fddbc46ff89e8fdbf5395c24d5c1452a3 \ - --hash=sha256:8c63c91f95173f9c2a67c7c526b2cea976828a0e7fced9cdcead2802dc10f8a4 \ - --hash=sha256:8df31fe574b8b3993cc61764f40941111b25c2d9fea13d3ce24a49907cd2d615 \ - --hash=sha256:8f9baf6f0a6e7571c45c8863010b45e837c3ee1c2c77fcd6ef423be91b21fedb \ - --hash=sha256:9027d773c4ff81487181a925945743413f6069634d0b122d0b37684ccf4f1e18 \ - --hash=sha256:9190426b7aa26c5229501fa297b8d0653cfd3f5a36f7990c264e157cbf886b3b \ - --hash=sha256:940dda65d5e764406b9fb92761cbf462e4e63f712ab60ed98f70552e496f3bf1 \ - --hash=sha256:94eff26096eb5395136634622515b234ecb6c9979824c1f5004c6e3c3c85ccd2 \ - --hash=sha256:9eed0f7edbb274413b6ee781cca50541c8c0facd3d6fd289779e494340a2b85c \ - --hash=sha256:ad4ae4ffd1ee9cd11357b4c66b612da9888f4f4daf2f36995eda64bd45370cac \ - --hash=sha256:b0f172dc8ffaccb8522d7c5d899de00133f2f1ca7b0a49b7da98e901de87bf2d \ - --hash=sha256:b2af221f268f5af85e776a73d62b0845fc8baf8ef0abfae79d29c77d0e776aaf \ - --hash=sha256:b7d335370ae48a780c6e6a6bbfa97342f563744c39c35562f3f367665f5c1de2 \ - --hash=sha256:b83af57bdddef03c01a9138034c6ff03181a3028d9a1003b301eb1a55e161a3f \ - --hash=sha256:bb5136fb5352d3f422df33f0c879a1b0c204004324150cc3b5e3c4f310c9049f \ - --hash=sha256:bc4d8e252f532ab46a1de9349e2d27b91fce46736a9eedaa37beaca66f574ed4 \ - --hash=sha256:bdd3e53429ff02aa319ba59dfe4ceeec345bf46cf180ec2cf6fd5b942e7975e9 \ - --hash=sha256:be12f931839a3bdfe28b584db0e640a65a8bcbc24560ae3fdb025a449b3d754e \ - --hash=sha256:be4a51a55833dc29ab5d7503e7bcb3b3af3402d266018137127450005cdfe737 \ - --hash=sha256:beb7f344487cdcb9e1efe4b7a29681b74d34c08f0043a327a74da852a6749e7b \ - --hash=sha256:bf4679a3d71012a7c2bf360e5cd878fbd5e4fcac0896b56393dec239d81529ed \ - --hash=sha256:c0e1403fd7c26d77c1f03e096dc58a5c726503fa0db0456678b8668f76f521e3 \ - --hash=sha256:c31c13da98624f957b0fb1b5bae5383b2333c2c3f6793d9825dd5ce79b525cb7 \ - --hash=sha256:c438f6ca858697c9ab67eb28246c92508af972e114cac34e57a6d4ba17a3ac08 \ - --hash=sha256:c8277104ded0a51e699c8c3aff63ce2c56d4ed5519a5f73e0fd7057f959a2b9e \ - --hash=sha256:c95cab08d1965db3d84a121f1c7ce7479bdd4072c9b3dafd8fecce48a2e6b902 \ - --hash=sha256:cc0b66c1eec9021353a4b4483afb12dfd50e3669ffbb9152d6842eb34c7e29fd \ - --hash=sha256:cdee07c4d7f6d72008d3f73b9bf027f4e11550224c7c50d8df1ae4a37c1402a6 \ - --hash=sha256:ce9bf03dad3b46408c08649c6fbd6ca28a9fce0eb32fdfffa6775a13103b5310 \ - --hash=sha256:cff8e5383db4989311f99e814feeb90c4723eb4edca425b9d5d9c3fefcdd9537 \ - --hash=sha256:d168fda2dbff7b9b5f38e693182d792a938c31db4dac3a80a4888de603c99554 \ - --hash=sha256:d1ffeb80b5676463d7a7d56acbe8e37a20ce725570e09549fe738e02ca6b7e1e \ - --hash=sha256:d36ca54cb4c6c4686f7cbb7b817f66f5911c12ddb519450bbe86707155028f87 \ - --hash=sha256:d4193f3d9dc3f6f79aaed0e5637f45d98850ebf01f7ca20e69457f3e8946b66a \ - --hash=sha256:d5cd5189fc2b6a538b75ae45433140c4823463918f7b1617c31e68b085c0022c \ - --hash=sha256:d618fd27420381a4f6044faa71f46d8bfd911bd077c555f7138ed88729bfbe79 \ - --hash=sha256:d76e2d8c75051d58177e762164d2e9ab92886534e3a12e795f103524f221dd8e \ - --hash=sha256:daae526907e262de627d8f70058a0f64acc9e2641c164c99c8f594b34a799a16 \ - --hash=sha256:db485b3847d182b908b483b2ed133c66d88d49cacf98fd278fadafe11b4478d1 \ - --hash=sha256:dd952e03bfbb096cfe2dd35cd9e00f269969b67536cb4370994afc20ff2d0875 \ - --hash=sha256:dda366d548e89a90d88a86c692377d18d8bd64b39c1fb2b92cb31370e2896bbd \ - --hash=sha256:e315e5ec90d88e140f57696ff85b484ff68bb311e36f2c414aa4286293e6dee0 \ - --hash=sha256:e4415a8db000bf49a6dd1c478bf70062eaacff0f462b92b0ba68791a905861f9 \ - --hash=sha256:e7a116ae737f0000343218c4edf5bd45893bfeaff0993c0b215d7124c9f77646 \ - --hash=sha256:e7c4c09a490dc4d4a7f8cbee56c606a320f9dc28cf92a7157a39d1ce7676a657 \ - --hash=sha256:ebae99ed6764f2b5771c522477b311be313e8841d2e0376db2b10922daebbba4 \ - --hash=sha256:ec4c85dc4b687c7f7f15f553ff26a98bfe8c58f5f7f0ac8905f0ba4c7be60232 \ - --hash=sha256:ed3a984b31da7481b103f68776f7128a89ef26ed40f4dc41a2223cda7fb24819 \ - --hash=sha256:f18c2d9782259a6dc132fdc7a63c168cbc74b35284b6d75c673958982a378384 \ - --hash=sha256:f1f9f4121ec58628c96baa3de1a55a4e3a333c5102c8e94b64e23bf7b2083309 \ - --hash=sha256:f42c23db5d1521218a3276bb08666dcb662896a0be7347cba864eca45ff64ede \ - --hash=sha256:f443b4825c50a51ee68585522ab4a1d1257fac65896f282b4c6763337ac9f5d2 \ - --hash=sha256:f6764a4ccab3078db14a632420930f6186058750df066b8ea2a7106df91d3203 \ - --hash=sha256:f7c7553b13f69c1b29a5bde08ddc6d9d0c8bfb84f9ed01c30db25944aeb852a7 \ - --hash=sha256:fa6248cd194edff41d7ea9425ced8ca3a6f838bfb295f6f1d6e6bb694a8518df \ - --hash=sha256:fa8eb9ecdb7efb0b226acec134e0d709e87a909fa4971a54c0c4f6e88635484c \ - --hash=sha256:fc20894c3d21194d8041a28b65622d5b86db786da6e3cfe73f0c762951a61167 \ - --hash=sha256:fc4d3f1fb9ca0ae9f97b095963bc6326f1dbfd3779d6679a1e016b9baaa153d3 \ - --hash=sha256:fd40bb9cd0891c4c3cb1ddf83f8bbfa15731a248fdc8162669405451e2724b09 \ - --hash=sha256:ff710414307fefa903e0d9bdf300972f892c23477829f49504e59834f4195398 - # via matplotlib -lets-plot==4.9.0 \ - --hash=sha256:103b09b9594448896bf31438b2f3866ed3d94a0e2f5df93047af27ae851c6951 \ - --hash=sha256:1563b52971670ec8fbf989980d39706ecf560072454001afac605cd77886c64a \ - --hash=sha256:1ba7aee59622fb444468c6cdb0c933504f36d884803da0a3b5e6d598f70cf1ad \ - --hash=sha256:1d64bdeb0c05146dfe0b5ebe26fc998f90224419beecf6354e2cb54207775ce0 \ - --hash=sha256:25873696e08e7e9df90aa42fe112e65112ca817da4341d04c54a74cdea74fd34 \ - --hash=sha256:2af3fee5f773d268e81daeaa346233e9bf6fee7aa9aa40dc01693bea986d8456 \ - --hash=sha256:318c0eae10218b7f8a153cac68f984ef4c0c313ef439b3db41fea9261097693b \ - --hash=sha256:3c2296a8dc550fa6f8906878e0fdb25bf25bb3acde22281b942528b441a52c1f \ - --hash=sha256:4310baa5cb8374e96a3c2b54152220603bc89851dfc57fc2c43b5d38fdb0832a \ - --hash=sha256:48359b96e5528c5cdbeed799e7e80a7dfa1b3af76cedb997abfd9f6f9ec0e506 \ - --hash=sha256:52b7569f91b8a44438fead2ec28b9907e1d23bdfeb4451589bc80cb3fba91384 \ - --hash=sha256:536bd8ea19b6f79c29b7998538b63bf602fe8e743e434f64c872c027cb9fb9f7 \ - --hash=sha256:5449e25a7a20c091ae4ee6f54d1711458cd2cc5e1963455c2293052249eda824 \ - --hash=sha256:5c9f2815ec9d3566145b98af486ed4ec0614e331238b3adbba10dd46c5bea472 \ - --hash=sha256:606f34d0401589c45a0cb7b1ab69823a8a10cd128d01d7ef8d464e343d128d18 \ - --hash=sha256:83269b428c6334bbcd5e873ea250ac6d2bb52aa763572eeba78988566786a5de \ - --hash=sha256:88f095a33f87bc64b87da9ed225dae6a0af6023e8a8c39e309e0bba40a8ca6f2 \ - --hash=sha256:9083b193748ae89bb4c118eeae944efface9777dd7b4d784af674106f8b5f2d8 \ - --hash=sha256:93a1ffea4736f49637c5d9ffeb18f69950343e15896333512c96c8f952f23656 \ - --hash=sha256:9793cc367b06fbe1664d6955bce0f7392369a8c318030d3a0cec419f00cc3b4a \ - --hash=sha256:a40036a03c98622cca9e316410feb8e29ddad5a941c3d63890f720821c8687c5 \ - --hash=sha256:a6cda1c544f91bef063b8dbea4876b21e8e6a1b709e89246976c7d422b1c3df2 \ - --hash=sha256:af591a4ee4af3cd2a3730d4e2ae4db74c0ceaa47da1c1ec2d33518431d86cd4b \ - --hash=sha256:b6d9666baab9810b54df30076c4918e6c28c17b98a54fea271b87b0aec3ec1ca \ - --hash=sha256:bc16a74a50395f9683ce3ea810e4fa154923e10406936f3756d6b32663124ae8 \ - --hash=sha256:be9766eeffd3690c80194faa7110e9f85235179da2e970ef8898791df1f97615 \ - --hash=sha256:ca34487455d7cb57f1a1d606419554bb5a8d3fa39692904d8b6e56342c67e9e1 \ - --hash=sha256:d26e09ea6fc8e5623d17ee4daa577ad06f70034badd13afad81e3fddb29d44ec \ - --hash=sha256:d2925f897963686df42e49a244e3b26e86f6acd5e1bddc6a0c24a6b70f9eb4ba \ - --hash=sha256:d420aa3b32b713bd6bb28a9474162d76a66ee4f3554d0bd0e0ef7685de0befe9 \ - --hash=sha256:dfd40b3aebfd54fd55e580223bfa8cdca91d66dfc3567c494c602c3f1918d960 \ - --hash=sha256:eb4442ae410d8b4be80432b182d0149d8efd049a00dc25b6c96b1ea9b9768a54 \ - --hash=sha256:ee643bb4adb8a3fe29126eab76c9661a4db87bf44ac0d8d01635fd2911465ca4 \ - --hash=sha256:ef61ba3b5f8613c48bd898a0fecfb40f1a76f15c6eee9030853a58bd4f7c8fd5 \ - --hash=sha256:f1dc14065a1ae08a2354971cc1704d2af236f8b5c69e2bade7cd3efa5aa5f1c8 - # via nki -libneuronxla==2.2.16408.0+50c26cbd \ - --hash=sha256:23b22819565826c27c89e660efc9e7745dfae1255d2ac72fe9fea1e34655b446 +libneuronxla==3.0.2891.0+e2a4b1f5 \ + --hash=sha256:b453709b37565902acc85bcfb4011d55e99381b81d831a5559847bba71f9cec3 # via -r platforms/neuron/requirements.in lockfile==0.12.2 \ --hash=sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799 \ --hash=sha256:6c3cb24f344923d30b2785d5ad75182c8ea7ac1b6171b08657258ec7429d50fa # via python-daemon -matplotlib==3.10.9 \ - --hash=sha256:09218df8a93712bd6ea133e83a153c755448cf7868316c531cffcc43f69d1cc9 \ - --hash=sha256:10cc5ce06d10231c36f40e875f3c7e8050362a4ee8f0ee5d29a6b3277d57bb42 \ - --hash=sha256:172db52c9e683f5d12eaf57f0f54834190e12581fe1cc2a19595a8f5acb4e77d \ - --hash=sha256:1872fb212a05b729e649754a72d5da61d03e0554d76e80303b6f83d1d2c0552b \ - --hash=sha256:1aa972116abb4c9d201bf245620b433726cb6856f3bef6a78f776a00f5c92d37 \ - --hash=sha256:1e7698ac9868428e84d2c967424803b2472ff7167d9d6590d4204ed775343c3b \ - --hash=sha256:2dc9477819ffd78ad12a20df1d9d6a6bd4fec6aaa9072681465fddca052f1456 \ - --hash=sha256:3225f4e1edcb8c86c884ddf79ebe20ecd0a67d30188f279897554ccd8fded4dc \ - --hash=sha256:336b9acc64d309063126edcdaca00db9373af3c476bb94388fe9c5a53ad13e6f \ - --hash=sha256:345f6f68ecc8da0ca56fad2ea08fde1a115eda530079eca185d50a7bc3e146c6 \ - --hash=sha256:34cf8167e023ad956c15f36302911d5406bd99a9862c1a8499ea6f7c0e015dc2 \ - --hash=sha256:3fc0364dfbe1d07f6d15c5ebd0c5bf89e126916e5a8667dd4a7a6e84c36653d4 \ - --hash=sha256:41cb28c2bd769aa3e98322c6ab09854cbcc52ab69d2759d681bba3e327b2b320 \ - --hash=sha256:42fb814efabe95c06c1994d8ab5a8385f43a249e23badd3ba931d4308e5bca20 \ - --hash=sha256:4e42042d54db34fda4e95a7bd3e5789c2a995d2dad3eb8850232ee534092fbbf \ - --hash=sha256:4edcfbd8565339aa62f1cd4012f7180926fdbe71850f7b0d3c379c175cd6b66c \ - --hash=sha256:51bf0ddbdc598e060d46c16b5590708f81a1624cefbaaf62f6a81bf9285b8c80 \ - --hash=sha256:56fc0bd271b00025c6edfdc7c2dcd247372c8e1544971d62e1dc7c17367e8bf9 \ - --hash=sha256:59476c6d29d612b8e9bb6ce8c5b631be6ba8f9e3a2421f22a02b192c7dd28716 \ - --hash=sha256:6640f75af2c6148293caa0a2b39dd806a492dd66c8a8b04035813e33d0fd2585 \ - --hash=sha256:68cfdcede415f7c8f5577b03303dd94526cdb6d11036cecdc205e08733b2d2bb \ - --hash=sha256:6b63d9c7c769b88ab81e10dc86e4e0607cf56817b9f9e6cf24b2a5f1693b8e38 \ - --hash=sha256:6be157fe17fc37cb95ac1d7374cf717ce9259616edec911a78d9d26dae8522d4 \ - --hash=sha256:6c63ebcd8b4b169eb2f5c200552ae6b8be8999a005b6b507ed76fb8d7d674fe2 \ - --hash=sha256:77210dce9cb8153dffc967efaae990543392563d5a376d4dd8539bebcb0ed217 \ - --hash=sha256:7a8d66a55def891c33147ba3ba9bfcabf0b526a43764c818acbb4525e5ed0838 \ - --hash=sha256:82368699727bfb7b0182e1aa13082e3c08e092fa1a25d3e1fd92405bff96f6d4 \ - --hash=sha256:82834c3c292d24d3a8aae77cd2d20019de69d692a34a970e4fdb8d33e2ea3dda \ - --hash=sha256:8e436d155fa8a3399dc62683f8f5d0e2e50d25d0144a73edd73f82eec8f4abfb \ - --hash=sha256:8f3bcac1ca5ed000a6f4337d47ba67dfddf37ed6a46c15fd7f014997f7bf865f \ - --hash=sha256:97e35e8d39ccc85859095e01a53847432ba9a53ddf7986f7a54a11b73d0e143f \ - --hash=sha256:985f2238880e2e69093f588f5fe2e46771747febf0649f3cf7f7b7480875317f \ - --hash=sha256:a49f1eadc84ca85fd72fa4e89e70e61bf86452df6f971af04b12c60761a0772c \ - --hash=sha256:a5a6104ed666402ba5106d7f36e0e0cdca4e8d7fa4d39708ca88019e2835a2eb \ - --hash=sha256:aba1615dabe83188e19d4f75a253c6a08423e04c1425e64039f800050a69de6b \ - --hash=sha256:ae20801130378b82d647ff5047c07316295b68dc054ca6b3c13519d0ea624285 \ - --hash=sha256:ae2f11957b27ce53497dd4d7b235c4d4f1faf383dfb39d0c5beb833bff883294 \ - --hash=sha256:b049278ddce116aaa1c1377ebf58adea909132dfce0281cf7e3a1ea9fc2e2c65 \ - --hash=sha256:b1b745c489cd1a77a0dc1120a05dc87af9798faebc913601feb8c73d89bf2d1e \ - --hash=sha256:b2b9516251cb89ff618d757daec0e2ed1bf21248013844a853d87ef85ab3081d \ - --hash=sha256:b580440f1ff81a0e34122051a3dfabb7e4b7f9e380629929bde0eff9af72165f \ - --hash=sha256:ba7b3b8ef09eab7df0e86e9ae086faa433efbfbdb46afcb3aa16aabf779469a8 \ - --hash=sha256:c27df8b3848f32a83d1767566595e43cfaa4460380974da06f4279a7ec143c39 \ - --hash=sha256:d091f9d758b34aaaaa6331d13574bf01891d903b3dec59bfff458ef7551de5d6 \ - --hash=sha256:d730e984eddf56974c3e72b6129c7ca462ac38dc624338f4b0b23eb23ecba00f \ - --hash=sha256:d75d11c949914165976c621b2324f9ef162af7ebf4b057ddf95dd1dba7e5edcf \ - --hash=sha256:d843374407c4017a6403b59c6c81606773d136f3259d5b6da3131bc814542cc2 \ - --hash=sha256:da4e09638420548f31c354032a6250e473c68e5a4e96899b4844cf39ddea23fe \ - --hash=sha256:de2445a0c6690d21b7eb6ce071cebad6d40a2e9bdf10d039074a96ba19797b99 \ - --hash=sha256:dfca0129678bd56379db26c52b5d77ed7de314c047492fbdc763aa7501710cfb \ - --hash=sha256:e9fae004b941b23ff2edcf1567a857ed77bafc8086ffa258190462328434faf8 \ - --hash=sha256:f0c3c28d9fbcc1fe7a03be236d73430cf6409c41fb2383a7ac52fe932b072cb1 \ - --hash=sha256:f4399f64b3e94cd500195490972ae1ee81170df1636fa15364d157d5bdd7b921 \ - --hash=sha256:f76e640a5268850bfda54b5131b1b1941cc685e42c5fa98ed9f2d64038308cba \ - --hash=sha256:fd66508e8c6877d98e586654b608a0456db8d7e8a546eb1e2600efd957302358 - # via nki ml-dtypes==0.5.4 \ --hash=sha256:0d2ffd05a2575b1519dc928c0b93c06339eb67173ff53acb00724502cda231cf \ --hash=sha256:11942cbf2cf92157db91e5022633c0d9474d4dfd813a909383bd23ce828a4b7d \ @@ -607,261 +260,107 @@ networkx==2.8.8 \ --hash=sha256:230d388117af870fce5647a3c52401fcf753e94720e6ea6b4197a5355648885e \ --hash=sha256:e435dfa75b1d7195c7b8378c3859f0445cd88c6b0375c181ed66823a9ceb7524 # via neuronx-cc -neuronx-cc==2.24.8799.0+6f62ff7c \ - --hash=sha256:d34bad1aea02a7eb033e374f7086b24b9f502da6b017c3cdd10eee6c9f872e76 +neuronx-cc==2.25.3371.0+f524f7f8 \ + --hash=sha256:c71cb7333d5320b6ea91e13e0168eec01be5e7ddfd5515bf62aa47088ee274c2 # via # -r platforms/neuron/requirements.in # libneuronxla -nki==0.3.0+23928721754.g18aa1271 \ - --hash=sha256:fe45d60d70c7977b4af61193c88cc91ce04ba4bd4b606cf047622bed9b5f2143 - # via neuronx-cc -numpy==2.4.4 \ - --hash=sha256:07077278157d02f65c43b1b26a3886bce886f95d20aabd11f87932750dfb14ed \ - --hash=sha256:08f2e31ed5e6f04b118e49821397f12767934cfdd12a1ce86a058f91e004ee50 \ - --hash=sha256:0aec54fd785890ecca25a6003fd9a5aed47ad607bbac5cd64f836ad8666f4959 \ - --hash=sha256:0d35aea54ad1d420c812bfa0385c71cd7cc5bcf7c65fed95fc2cd02fe8c79827 \ - --hash=sha256:0d4e437e295f18ec29bc79daf55e8a47a9113df44d66f702f02a293d93a2d6dd \ - --hash=sha256:0dfd3f9d3adbe2920b68b5cd3d51444e13a10792ec7154cd0a2f6e74d4ab3233 \ - --hash=sha256:1378871da56ca8943c2ba674530924bb8ca40cd228358a3b5f302ad60cf875fc \ - --hash=sha256:15716cfef24d3a9762e3acdf87e27f58dc823d1348f765bbea6bef8c639bfa1b \ - --hash=sha256:19710a9ca9992d7174e9c52f643d4272dcd1558c5f7af7f6f8190f633bd651a7 \ - --hash=sha256:23cbfd4c17357c81021f21540da84ee282b9c8fba38a03b7b9d09ba6b951421e \ - --hash=sha256:2483e4584a1cb3092da4470b38866634bafb223cbcd551ee047633fd2584599a \ - --hash=sha256:27a8d92cd10f1382a67d7cf4db7ce18341b66438bdd9f691d7b0e48d104c2a9d \ - --hash=sha256:28a650663f7314afc3e6ec620f44f333c386aad9f6fc472030865dc0ebb26ee3 \ - --hash=sha256:2aa0613a5177c264ff5921051a5719d20095ea586ca88cc802c5c218d1c67d3e \ - --hash=sha256:2c194dd721e54ecad9ad387c1d35e63dce5c4450c6dc7dd5611283dda239aabb \ - --hash=sha256:2d19e6e2095506d1736b7d80595e0f252d76b89f5e715c35e06e937679ea7d7a \ - --hash=sha256:2d390634c5182175533585cc89f3608a4682ccb173cc9bb940b2881c8d6f8fa0 \ - --hash=sha256:30caa73029a225b2d40d9fae193e008e24b2026b7ee1a867b7ee8d96ca1a448e \ - --hash=sha256:42c16925aa5a02362f986765f9ebabf20de75cdefdca827d14315c568dcab113 \ - --hash=sha256:45dbed2ab436a9e826e302fcdcbe9133f9b0006e5af7168afb8963a6520da103 \ - --hash=sha256:4636de7fd195197b7535f231b5de9e4b36d2c440b6e566d2e4e4746e6af0ca93 \ - --hash=sha256:4a19d9dba1a76618dd86b164d608566f393f8ec6ac7c44f0cc879011c45e65af \ - --hash=sha256:4bbc7f303d125971f60ec0aaad5e12c62d0d2c925f0ab1273debd0e4ba37aba5 \ - --hash=sha256:4d6d57903571f86180eb98f8f0c839fa9ebbfb031356d87f1361be91e433f5b7 \ - --hash=sha256:4e874c976154687c1f71715b034739b45c7711bec81db01914770373d125e392 \ - --hash=sha256:51fc224f7ca4d92656d5a5eb315f12eb5fe2c97a66249aa7b5f562528a3be38c \ - --hash=sha256:58c8b5929fcb8287cbd6f0a3fae19c6e03a5c48402ae792962ac465224a629a4 \ - --hash=sha256:5a285b3b96f951841799528cd1f4f01cd70e7e0204b4abebac9463eecfcf2a40 \ - --hash=sha256:5c70f1cc1c4efbe316a572e2d8b9b9cc44e89b95f79ca3331553fbb63716e2bf \ - --hash=sha256:62d6b0f03b694173f9fcb1fb317f7222fd0b0b103e784c6549f5e53a27718c44 \ - --hash=sha256:6a246d5914aa1c820c9443ddcee9c02bec3e203b0c080349533fae17727dfd1b \ - --hash=sha256:6aa3236c78803afbcb255045fbef97a9e25a1f6c9888357d205ddc42f4d6eba5 \ - --hash=sha256:6bbe4eb67390b0a0265a2c25458f6b90a409d5d069f1041e6aff1e27e3d9a79e \ - --hash=sha256:715d1c092715954784bc79e1174fc2a90093dc4dc84ea15eb14dad8abdcdeb74 \ - --hash=sha256:72944b19f2324114e9dc86a159787333b77874143efcf89a5167ef83cfee8af0 \ - --hash=sha256:81f4a14bee47aec54f883e0cad2d73986640c1590eb9bfaaba7ad17394481e6e \ - --hash=sha256:846300f379b5b12cc769334464656bc882e0735d27d9726568bc932fdc49d5ec \ - --hash=sha256:86b6f55f5a352b48d7fbfd2dbc3d5b780b2d79f4d3c121f33eb6efb22e9a2015 \ - --hash=sha256:874f200b2a981c647340f841730fc3a2b54c9d940566a3c4149099591e2c4c3d \ - --hash=sha256:8a87ec22c87be071b6bdbd27920b129b94f2fc964358ce38f3822635a3e2e03d \ - --hash=sha256:8b3b60bb7cba2c8c81837661c488637eee696f59a877788a396d33150c35d842 \ - --hash=sha256:8e3ed142f2728df44263aaf5fb1f5b0b99f4070c553a0d7f033be65338329150 \ - --hash=sha256:93e15038125dc1e5345d9b5b68aa7f996ec33b98118d18c6ca0d0b7d6198b7e8 \ - --hash=sha256:989824e9faf85f96ec9c7761cd8d29c531ad857bfa1daa930cba85baaecf1a9a \ - --hash=sha256:99d838547ace2c4aace6c4f76e879ddfe02bb58a80c1549928477862b7a6d6ed \ - --hash=sha256:9b2aec6af35c113b05695ebb5749a787acd63cafc83086a05771d1e1cd1e555f \ - --hash=sha256:9c585a1790d5436a5374bac930dad6ed244c046ed91b2b2a3634eb2971d21008 \ - --hash=sha256:a7164afb23be6e37ad90b2f10426149fd75aee07ca55653d2aa41e66c4ef697e \ - --hash=sha256:ac6b31e35612a26483e20750126d30d0941f949426974cace8e6b5c58a3657b0 \ - --hash=sha256:ad2e2ef14e0b04e544ea2fa0a36463f847f113d314aa02e5b402fdf910ef309e \ - --hash=sha256:b268594bccac7d7cf5844c7732e3f20c50921d94e36d7ec9b79e9857694b1b2f \ - --hash=sha256:b5f0362dc928a6ecd9db58868fca5e48485205e3855957bdedea308f8672ea4a \ - --hash=sha256:ba1f4fc670ed79f876f70082eff4f9583c15fb9a4b89d6188412de4d18ae2f40 \ - --hash=sha256:ba203255017337d39f89bdd58417f03c4426f12beed0440cfd933cb15f8669c7 \ - --hash=sha256:c901b15172510173f5cb310eae652908340f8dede90fff9e3bf6c0d8dfd92f83 \ - --hash=sha256:c9b39d38a9bd2ae1becd7eac1303d031c5c110ad31f2b319c6e7d98b135c934d \ - --hash=sha256:d2a8490669bfe99a233298348acc2d824d496dee0e66e31b66a6022c2ad74a5c \ - --hash=sha256:dddbbd259598d7240b18c9d87c56a9d2fb3b02fe266f49a7c101532e78c1d871 \ - --hash=sha256:df3775294accfdd75f32c74ae39fcba920c9a378a2fc18a12b6820aa8c1fb502 \ - --hash=sha256:e44319a2953c738205bf3354537979eaa3998ed673395b964c1176083dd46252 \ - --hash=sha256:e4a010c27ff6f210ff4c6ef34394cd61470d01014439b192ec22552ee867f2a8 \ - --hash=sha256:e823b8b6edc81e747526f70f71a9c0a07ac4e7ad13020aa736bb7c9d67196115 \ - --hash=sha256:e892aff75639bbef0d2a2cfd55535510df26ff92f63c92cd84ef8d4ba5a5557f \ - --hash=sha256:eea7ac5d2dce4189771cedb559c738a71512768210dc4e4753b107a2048b3d0e \ - --hash=sha256:ef4059d6e5152fa1a39f888e344c73fdc926e1b2dd58c771d67b0acfbf2aa67d \ - --hash=sha256:f169b9a863d34f5d11b8698ead99febeaa17a13ca044961aa8e2662a6c7766a0 \ - --hash=sha256:f2cf083b324a467e1ab358c105f6cad5ea950f50524668a80c486ff1db24e119 \ - --hash=sha256:f8474c4241bc18b750be2abea9d7a9ec84f46ef861dbacf86a4f6e043401f79e \ - --hash=sha256:f983334aea213c99992053ede6168500e5f086ce74fbc4acc3f2b00f5762e9db \ - --hash=sha256:f9e75681b59ddaa5e659898085ae0eaea229d054f2ac0c7e563a62205a700121 \ - --hash=sha256:fbc356aae7adf9e6336d336b9c8111d390a05df88f1805573ebb0807bd06fd1d \ - --hash=sha256:fcfe2045fd2e8f3cb0ce9d4ba6dba6333b8fa05bb8a4939c908cd43322d14c7e +nki==0.4.0+25940409122.gd30719f9 \ + --hash=sha256:09b520140e55790ba00ebe3e7d53da1e5f221ff3d6c1823414acfebe4b794879 + # via + # -r platforms/neuron/requirements.in + # neuronx-cc +numpy==2.4.6 \ + --hash=sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1 \ + --hash=sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4 \ + --hash=sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f \ + --hash=sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079 \ + --hash=sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096 \ + --hash=sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47 \ + --hash=sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66 \ + --hash=sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d \ + --hash=sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1 \ + --hash=sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e \ + --hash=sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147 \ + --hash=sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd \ + --hash=sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75 \ + --hash=sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063 \ + --hash=sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73 \ + --hash=sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab \ + --hash=sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4 \ + --hash=sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41 \ + --hash=sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402 \ + --hash=sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698 \ + --hash=sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7 \ + --hash=sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8 \ + --hash=sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b \ + --hash=sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8 \ + --hash=sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0 \ + --hash=sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662 \ + --hash=sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91 \ + --hash=sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0 \ + --hash=sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f \ + --hash=sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3 \ + --hash=sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f \ + --hash=sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67 \ + --hash=sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6 \ + --hash=sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997 \ + --hash=sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b \ + --hash=sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e \ + --hash=sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538 \ + --hash=sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627 \ + --hash=sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93 \ + --hash=sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02 \ + --hash=sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853 \ + --hash=sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c \ + --hash=sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43 \ + --hash=sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd \ + --hash=sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8 \ + --hash=sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089 \ + --hash=sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778 \ + --hash=sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1 \ + --hash=sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb \ + --hash=sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261 \ + --hash=sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb \ + --hash=sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a \ + --hash=sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8 \ + --hash=sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359 \ + --hash=sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5 \ + --hash=sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7 \ + --hash=sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751 \ + --hash=sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8 \ + --hash=sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605 \ + --hash=sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e \ + --hash=sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45 \ + --hash=sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2 \ + --hash=sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895 \ + --hash=sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe \ + --hash=sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb \ + --hash=sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a \ + --hash=sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577 \ + --hash=sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d \ + --hash=sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a \ + --hash=sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda \ + --hash=sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6 \ + --hash=sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20 # via - # contourpy - # matplotlib # ml-dtypes # neuronx-cc # nki - # pandas # scipy -packaging==26.2 \ - --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e \ - --hash=sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661 - # via matplotlib -palettable==3.3.3 \ - --hash=sha256:094dd7d9a5fc1cca4854773e5c1fc6a315b33bd5b3a8f47064928facaf0490a8 \ - --hash=sha256:74e9e7d7fe5a9be065e02397558ed1777b2df0b793a6f4ce1a5ee74f74fb0caa - # via lets-plot -pandas==3.0.2 \ - --hash=sha256:01f31a546acd5574ef77fe199bc90b55527c225c20ccda6601cf6b0fd5ed597c \ - --hash=sha256:0555c5882688a39317179ab4a0ed41d3ebc8812ab14c69364bbee8fb7a3f6288 \ - --hash=sha256:07a10f5c36512eead51bc578eb3354ad17578b22c013d89a796ab5eee90cd991 \ - --hash=sha256:08504503f7101300107ecdc8df73658e4347586db5cfdadabc1592e9d7e7a0fd \ - --hash=sha256:0f48afd9bb13300ffb5a3316973324c787054ba6665cda0da3fbd67f451995db \ - --hash=sha256:140f0cffb1fa2524e874dde5b477d9defe10780d8e9e220d259b2c0874c89d9d \ - --hash=sha256:232a70ebb568c0c4d2db4584f338c1577d81e3af63292208d615907b698a0f18 \ - --hash=sha256:32cc41f310ebd4a296d93515fcac312216adfedb1894e879303987b8f1e2b97d \ - --hash=sha256:339dda302bd8369dedeae979cb750e484d549b563c3f54f3922cb8ff4978c5eb \ - --hash=sha256:4544c7a54920de8eeacaa1466a6b7268ecfbc9bc64ab4dbb89c6bbe94d5e0660 \ - --hash=sha256:4d888a5c678a419a5bb41a2a93818e8ed9fd3172246555c0b37b7cc27027effd \ - --hash=sha256:5371b72c2d4d415d08765f32d689217a43227484e81b2305b52076e328f6f482 \ - --hash=sha256:57a07209bebcbcf768d2d13c9b78b852f9a15978dac41b9e6421a81ad4cdd276 \ - --hash=sha256:5880314e69e763d4c8b27937090de570f1fb8d027059a7ada3f7f8e98bdcb677 \ - --hash=sha256:5d3cfe227c725b1f3dff4278b43d8c784656a42a9325b63af6b1492a8232209e \ - --hash=sha256:5fdbfa05931071aba28b408e59226186b01eb5e92bea2ab78b65863ca3228d84 \ - --hash=sha256:60a80bb4feacbef5e1447a3f82c33209c8b7e07f28d805cfd1fb951e5cb443aa \ - --hash=sha256:61c2fd96d72b983a9891b2598f286befd4ad262161a609c92dc1652544b46b76 \ - --hash=sha256:63d141b56ef686f7f0d714cfb8de4e320475b86bf4b620aa0b7da89af8cbdbbb \ - --hash=sha256:6c4d8458b97a35717b62469a4ea0e85abd5ed8687277f5ccfc67f8a5126f8c53 \ - --hash=sha256:710246ba0616e86891b58ab95f2495143bb2bc83ab6b06747c74216f583a6ac9 \ - --hash=sha256:734be7551687c00fbd760dc0522ed974f82ad230d4a10f54bf51b80d44a08702 \ - --hash=sha256:7cadd7e9a44ec13b621aec60f9150e744cfc7a3dd32924a7e2f45edff31823b0 \ - --hash=sha256:81526c4afd31971f8b62671442a4b2b51e0aa9acc3819c9f0f12a28b6fcf85f1 \ - --hash=sha256:970762605cff1ca0d3f71ed4f3a769ea8f85fc8e6348f6e110b8fea7e6eb5a14 \ - --hash=sha256:a3096110bf9eac0070b7208465f2740e2d8a670d5cb6530b5bb884eca495fd39 \ - --hash=sha256:a4785e1d6547d8427c5208b748ae2efb64659a21bd82bf440d4262d02bfa02a4 \ - --hash=sha256:a727a73cbdba2f7458dc82449e2315899d5140b449015d822f515749a46cbbe0 \ - --hash=sha256:ae37e833ff4fed0ba352f6bdd8b73ba3ab3256a85e54edfd1ab51ae40cca0af8 \ - --hash=sha256:aff4e6f4d722e0652707d7bcb190c445fe58428500c6d16005b02401764b1b3d \ - --hash=sha256:b35d14bb5d8285d9494fe93815a9e9307c0876e10f1e8e89ac5b88f728ec8dcf \ - --hash=sha256:b444dc64c079e84df91baa8bf613d58405645461cabca929d9178f2cd392398d \ - --hash=sha256:b5329e26898896f06035241a626d7c335daa479b9bbc82be7c2742d048e41172 \ - --hash=sha256:b5918ba197c951dec132b0c5929a00c0bf05d5942f590d3c10a807f6e15a57d3 \ - --hash=sha256:b75c347eff42497452116ce05ef461822d97ce5b9ff8df6edacb8076092c855d \ - --hash=sha256:c3b723df9087a9a9a840e263ebd9f88b64a12075d1bf2ea401a5a42f254f084d \ - --hash=sha256:c934008c733b8bbea273ea308b73b3156f0181e5b72960790b09c18a2794fe1e \ - --hash=sha256:d1478075142e83a5571782ad007fb201ed074bdeac7ebcc8890c71442e96adf7 \ - --hash=sha256:d606a041c89c0a474a4702d532ab7e73a14fe35c8d427b972a625c8e46373668 \ - --hash=sha256:db0dbfd2a6cdf3770aa60464d50333d8f3d9165b2f2671bcc299b72de5a6677b \ - --hash=sha256:dbbd4aa20ca51e63b53bbde6a0fa4254b1aaabb74d2f542df7a7959feb1d760c \ - --hash=sha256:dbc20dea3b9e27d0e66d74c42b2d0c1bed9c2ffe92adea33633e3bedeb5ac235 \ - --hash=sha256:deeca1b5a931fdf0c2212c8a659ade6d3b1edc21f0914ce71ef24456ca7a6535 \ - --hash=sha256:ed72cb3f45190874eb579c64fa92d9df74e98fd63e2be7f62bce5ace0ade61df \ - --hash=sha256:ef8b27695c3d3dc78403c9a7d5e59a62d5464a7e1123b4e0042763f7104dc74f \ - --hash=sha256:f12b1a9e332c01e09510586f8ca9b108fd631fd656af82e452d7315ef6df5f9f \ - --hash=sha256:f4753e73e34c8d83221ba58f232433fca2748be8b18dbca02d242ed153945043 \ - --hash=sha256:f8d68083e49e16b84734eb1a4dcae4259a75c90fb6e2251ab9a00b61120c06ab - # via nki pgzip==0.4.0 \ --hash=sha256:41e44c03a9f1bbbd390dd6b6e37a11484113f8c159a049fa81cd8a9ddde82a21 \ --hash=sha256:91f6526f0c6e1f6c2d3522707777e5843fad7c791401745593fd01bd869b7a05 # via neuronx-cc -pillow==12.2.0 \ - --hash=sha256:00a2865911330191c0b818c59103b58a5e697cae67042366970a6b6f1b20b7f9 \ - --hash=sha256:01afa7cf67f74f09523699b4e88c73fb55c13346d212a59a2db1f86b0a63e8c5 \ - --hash=sha256:03e7e372d5240cc23e9f07deca4d775c0817bffc641b01e9c3af208dbd300987 \ - --hash=sha256:03f6fab9219220f041c74aeaa2939ff0062bd5c364ba9ce037197f4c6d498cd9 \ - --hash=sha256:042db20a421b9bafecc4b84a8b6e444686bd9d836c7fd24542db3e7df7baad9b \ - --hash=sha256:0538bd5e05efec03ae613fd89c4ce0368ecd2ba239cc25b9f9be7ed426b0af1f \ - --hash=sha256:0a34329707af4f73cf1782a36cd2289c0368880654a2c11f027bcee9052d35dd \ - --hash=sha256:0c838a5125cee37e68edec915651521191cef1e6aa336b855f495766e77a366e \ - --hash=sha256:144748b3af2d1b358d41286056d0003f47cb339b8c43a9ea42f5fea4d8c66b6e \ - --hash=sha256:1610dd6c61621ae1cf811bef44d77e149ce3f7b95afe66a4512f8c59f25d9ebe \ - --hash=sha256:1e1757442ed87f4912397c6d35a0db6a7b52592156014706f17658ff58bbf795 \ - --hash=sha256:22db17c68434de69d8ecfc2fe821569195c0c373b25cccb9cbdacf2c6e53c601 \ - --hash=sha256:25373b66e0dd5905ed63fa3cae13c82fbddf3079f2c8bf15c6fb6a35586324c1 \ - --hash=sha256:2bb4a8d594eacdfc59d9e5ad972aa8afdd48d584ffd5f13a937a664c3e7db0ed \ - --hash=sha256:2c727a6d53cb0018aadd8018c2b938376af27914a68a492f59dfcaca650d5eea \ - --hash=sha256:2d192a155bbcec180f8564f693e6fd9bccff5a7af9b32e2e4bf8c9c69dbad6b5 \ - --hash=sha256:2e589959f10d9824d39b350472b92f0ce3b443c0a3442ebf41c40cb8361c5b97 \ - --hash=sha256:2e5a76d03a6c6dcef67edabda7a52494afa4035021a79c8558e14af25313d453 \ - --hash=sha256:325ca0528c6788d2a6c3d40e3568639398137346c3d6e66bb61db96b96511c98 \ - --hash=sha256:34c0d99ecccea270c04882cb3b86e7b57296079c9a4aff88cb3b33563d95afaa \ - --hash=sha256:390ede346628ccc626e5730107cde16c42d3836b89662a115a921f28440e6a3b \ - --hash=sha256:394167b21da716608eac917c60aa9b969421b5dcbbe02ae7f013e7b85811c69d \ - --hash=sha256:3997232e10d2920a68d25191392e3a4487d8183039e1c74c2297f00ed1c50705 \ - --hash=sha256:3adc9215e8be0448ed6e814966ecf3d9952f0ea40eb14e89a102b87f450660d8 \ - --hash=sha256:3e080565d8d7c671db5802eedfb438e5565ffa40115216eabb8cd52d0ecce024 \ - --hash=sha256:4a6c9fa44005fa37a91ebfc95d081e8079757d2e904b27103f4f5fa6f0bf78c0 \ - --hash=sha256:4bfd07bc812fbd20395212969e41931001fd59eb55a60658b0e5710872e95286 \ - --hash=sha256:4e6c62e9d237e9b65fac06857d511e90d8461a32adcc1b9065ea0c0fa3a28150 \ - --hash=sha256:50d8520da2a6ce0af445fa6d648c4273c3eeefbc32d7ce049f22e8b5c3daecc2 \ - --hash=sha256:51c4167c34b0d8ba05b547a3bb23578d0ba17b80a5593f93bd8ecb123dd336a3 \ - --hash=sha256:56a3f9c60a13133a98ecff6197af34d7824de9b7b38c3654861a725c970c197b \ - --hash=sha256:56b25336f502b6ed02e889f4ece894a72612fe885889a6e8c4c80239ff6e5f5f \ - --hash=sha256:57850958fe9c751670e49b2cecf6294acc99e562531f4bd317fa5ddee2068463 \ - --hash=sha256:58f62cc0f00fd29e64b29f4fd923ffdb3859c9f9e6105bfc37ba1d08994e8940 \ - --hash=sha256:5c0a9f29ca8e79f09de89293f82fc9b0270bb4af1d58bc98f540cc4aedf03166 \ - --hash=sha256:5cdfebd752ec52bf5bb4e35d9c64b40826bc5b40a13df7c3cda20a2c03a0f5ed \ - --hash=sha256:5d04bfa02cc2d23b497d1e90a0f927070043f6cbf303e738300532379a4b4e0f \ - --hash=sha256:5d2fd0fa6b5d9d1de415060363433f28da8b1526c1c129020435e186794b3795 \ - --hash=sha256:62f5409336adb0663b7caa0da5c7d9e7bdbaae9ce761d34669420c2a801b2780 \ - --hash=sha256:632ff19b2778e43162304d50da0181ce24ac5bb8180122cbe1bf4673428328c7 \ - --hash=sha256:6562ace0d3fb5f20ed7290f1f929cae41b25ae29528f2af1722966a0a02e2aa1 \ - --hash=sha256:673aa32138f3e7531ccdbca7b3901dba9b70940a19ccecc6a37c77d5fdeb05b5 \ - --hash=sha256:6a6e67ea2e6feda684ed370f9a1c52e7a243631c025ba42149a2cc5934dec295 \ - --hash=sha256:6a9adfc6d24b10f89588096364cc726174118c62130c817c2837c60cf08a392b \ - --hash=sha256:6bb77b2dcb06b20f9f4b4a8454caa581cd4dd0643a08bacf821216a16d9c8354 \ - --hash=sha256:6e6b2a0c538fc200b38ff9eb6628228b77908c319a005815f2dde585a0664b60 \ - --hash=sha256:71cde9a1e1551df7d34a25462fc60325e8a11a82cc2e2f54578e5e9a1e153d65 \ - --hash=sha256:7371b48c4fa448d20d2714c9a1f775a81155050d383333e0a6c15b1123dda005 \ - --hash=sha256:766cef22385fa1091258ad7e6216792b156dc16d8d3fa607e7545b2b72061f1c \ - --hash=sha256:7b14cc0106cd9aecda615dd6903840a058b4700fcb817687d0ee4fc8b6e389be \ - --hash=sha256:7f84204dee22a783350679a0333981df803dac21a0190d706a50475e361c93f5 \ - --hash=sha256:8023abc91fba39036dbce14a7d6535632f99c0b857807cbbbf21ecc9f4717f06 \ - --hash=sha256:80b2da48193b2f33ed0c32c38140f9d3186583ce7d516526d462645fd98660ae \ - --hash=sha256:8297651f5b5679c19968abefd6bb84d95fe30ef712eb1b2d9b2d31ca61267f4c \ - --hash=sha256:88d387ff40b3ff7c274947ed3125dedf5262ec6919d83946753b5f3d7c67ea4c \ - --hash=sha256:88ddbc66737e277852913bd1e07c150cc7bb124539f94c4e2df5344494e0a612 \ - --hash=sha256:8bd7903a5f2a4545f6fd5935c90058b89d30045568985a71c79f5fd6edf9b91e \ - --hash=sha256:8be29e59487a79f173507c30ddf57e733a357f67881430449bb32614075a40ab \ - --hash=sha256:8c984051042858021a54926eb597d6ee3012393ce9c181814115df4c60b9a808 \ - --hash=sha256:8cbeb542b2ebc6fcdacabf8aca8c1a97c9b3ad3927d46b8723f9d4f033288a0f \ - --hash=sha256:8e9c4f5b3c546fa3458a29ab22646c1c6c787ea8f5ef51300e5a60300736905e \ - --hash=sha256:90e6f81de50ad6b534cab6e5aef77ff6e37722b2f5d908686f4a5c9eba17a909 \ - --hash=sha256:975385f4776fafde056abb318f612ef6285b10a1f12b8570f3647ad0d74b48ec \ - --hash=sha256:9a8a34cc89c67a65ea7437ce257cea81a9dad65b29805f3ecee8c8fe8ff25ffe \ - --hash=sha256:9aba9a17b623ef750a4d11b742cbafffeb48a869821252b30ee21b5e91392c50 \ - --hash=sha256:9f08483a632889536b8139663db60f6724bfcb443c96f1b18855860d7d5c0fd4 \ - --hash=sha256:a4e8f36e677d3336f35089648c8955c51c6d386a13cf6ee9c189c5f5bd713a9f \ - --hash=sha256:a52edc8bfff4429aaabdf4d9ee0daadbbf8562364f940937b941f87a4290f5ff \ - --hash=sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5 \ - --hash=sha256:aa88ccfe4e32d362816319ed727a004423aab09c5cea43c01a4b435643fa34eb \ - --hash=sha256:af73337013e0b3b46f175e79492d96845b16126ddf79c438d7ea7ff27783a414 \ - --hash=sha256:b1c1fbd8a5a1af3412a0810d060a78b5136ec0836c8a4ef9aa11807f2a22f4e1 \ - --hash=sha256:b85f66ae9eb53e860a873b858b789217ba505e5e405a24b85c0464822fe88032 \ - --hash=sha256:b86024e52a1b269467a802258c25521e6d742349d760728092e1bc2d135b4d76 \ - --hash=sha256:bd9c0c7a0c681a347b3194c500cb1e6ca9cab053ea4d82a5cf45b6b754560136 \ - --hash=sha256:bfa9c230d2fe991bed5318a5f119bd6780cda2915cca595393649fc118ab895e \ - --hash=sha256:d362d1878f00c142b7e1a16e6e5e780f02be8195123f164edf7eddd911eefe7c \ - --hash=sha256:d5d38f1411c0ed9f97bcb49b7bd59b6b7c314e0e27420e34d99d844b9ce3b6f3 \ - --hash=sha256:dac8d77255a37e81a2efcbd1fc05f1c15ee82200e6c240d7e127e25e365c39ea \ - --hash=sha256:dd025009355c926a84a612fecf58bb315a3f6814b17ead51a8e48d3823d9087f \ - --hash=sha256:deede7c263feb25dba4e82ea23058a235dcc2fe1f6021025dc71f2b618e26104 \ - --hash=sha256:e74473c875d78b8e9d5da2a70f7099549f9eb37ded4e2f6a463e60125bccd176 \ - --hash=sha256:ee3120ae9dff32f121610bb08e4313be87e03efeadfc6c0d18f89127e24d0c24 \ - --hash=sha256:eedf4b74eda2b5a4b2b2fb4c006d6295df3bf29e459e198c90ea48e130dc75c3 \ - --hash=sha256:efd8c21c98c5cc60653bcb311bef2ce0401642b7ce9d09e03a7da87c878289d4 \ - --hash=sha256:f1c943e96e85df3d3478f7b691f229887e143f81fedab9b20205349ab04d73ed \ - --hash=sha256:f278f034eb75b4e8a13a54a876cc4a5ab39173d2cdd93a638e1b467fc545ac43 \ - --hash=sha256:f3f40b3c5a968281fd507d519e444c35f0ff171237f4fdde090dd60699458421 \ - --hash=sha256:f490f9368b6fc026f021db16d7ec2fbf7d89e2edb42e8ec09d2c60505f5729c7 \ - --hash=sha256:fb043ee2f06b41473269765c2feae53fc2e2fbf96e5e22ca94fb5ad677856f06 \ - --hash=sha256:fc3d34d4a8fbec3e88a79b92e5465e0f9b842b628675850d860b8bd300b159f5 - # via - # lets-plot - # matplotlib -protobuf==7.34.1 \ - --hash=sha256:34b84ce27680df7cca9f231043ada0daa55d0c44a2ddfaa58ec1d0d89d8bf60a \ - --hash=sha256:403b093a6e28a960372b44e5eb081775c9b056e816a8029c61231743d63f881a \ - --hash=sha256:5185e0e948d07abe94bb76ec9b8416b604cfe5da6f871d67aad30cbf24c3110b \ - --hash=sha256:8ff40ce8cd688f7265326b38d5a1bed9bfdf5e6723d49961432f83e21d5713e4 \ - --hash=sha256:9ce42245e704cc5027be797c1db1eb93184d44d1cdd71811fb2d9b25ad541280 \ - --hash=sha256:bb3812cd53aefea2b028ef42bd780f5b96407247f20c6ef7c679807e9d188f11 \ - --hash=sha256:d8b2cc79c4d8f62b293ad9b11ec3aebce9af481fa73e64556969f7345ebf9fc7 \ - --hash=sha256:e97b55646e6ce5cbb0954a8c28cd39a5869b59090dfaa7df4598a7fba869468c +protobuf==7.35.0 \ + --hash=sha256:4c4617b83ade0e279d1d2bfe04025a1adb87f9ed657de038620dc0ff959357f6 \ + --hash=sha256:4cbf5cc286130e06a6c9bbefac442431173906dfcc979712183d4adcc01b37ee \ + --hash=sha256:66be6c513931c794fa92c080ffee41671390da3d79da219cf9c0c0907f035dda \ + --hash=sha256:6c0f98f10c8a05ea30f8993dfef2de093d27b490fdae78bb60c8343795d55011 \ + --hash=sha256:a2efd84605f41e559f1881b0912b44099d0a2ac9bf46b3474823f10fb393b0e6 \ + --hash=sha256:c13f325cf242bad135c350629eeb5d54b24228eb472fb3e2e9ebbd4c5dc20ca0 \ + --hash=sha256:f05bcadf9a2a6b8dda047007075135fb7d08c73d9177aabc067e1be46881a201 \ + --hash=sha256:fcbe42a4ac09d3ec9c987ddfcd956afd0b15f1ff613bd8371bde9405ffd5c8e5 # via neuronx-cc psutil==7.2.2 \ --hash=sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372 \ @@ -886,14 +385,6 @@ psutil==7.2.2 \ --hash=sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00 \ --hash=sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8 # via neuronx-cc -pyparsing==3.3.2 \ - --hash=sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d \ - --hash=sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc - # via matplotlib -pypng==0.20220715.0 \ - --hash=sha256:4a43e969b8f5aaafb2a415536c1a8ec7e341cd6a3f957fd5b5f32a4cfeed902c \ - --hash=sha256:739c433ba96f078315de54c0db975aee537cbc3e1d0ae4ed9aab0ca1e427e2c1 - # via lets-plot python-daemon==3.1.2 \ --hash=sha256:b906833cef63502994ad48e2eab213259ed9bb18d54fa8774dcba2ff7864cec6 \ --hash=sha256:f7b04335adc473de877f5117e26d5f1142f4c9f7cd765408f0877757be5afbf4 @@ -901,13 +392,10 @@ python-daemon==3.1.2 \ python-dateutil==2.9.0.post0 \ --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 - # via - # botocore - # matplotlib - # pandas -requests==2.33.1 \ - --hash=sha256:18817f8c57c6263968bc123d237e3b8b08ac046f5456bd1e307ee8f4250d3517 \ - --hash=sha256:4e6d1ef462f3626a1f0a0a9c42dd93c63bad33f9f1c1937509b8c5c8718ab56a + # via botocore +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed # via # ec2-metadata # requests-unixsocket @@ -994,9 +482,9 @@ typing-extensions==4.15.0 \ --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 # via neuronx-cc -urllib3==2.6.3 \ - --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ - --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 +urllib3==2.7.0 \ + --hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \ + --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 # via # botocore # requests diff --git a/platforms/neuron/topology/topology.zig b/platforms/neuron/topology.zig similarity index 62% rename from platforms/neuron/topology/topology.zig rename to platforms/neuron/topology.zig index fb1ba01c..201db15f 100644 --- a/platforms/neuron/topology/topology.zig +++ b/platforms/neuron/topology.zig @@ -1,11 +1,10 @@ const std = @import("std"); -const c = @import("c"); +const neuron = @import("platforms/neuron"); // References: -// - NRT exposes `nrt_instance_info.family` and `.size`, which map to the -// https://github.com/aws-neuron/aws-neuron-sdk/blob/master/src/libnrt/include/nrt/nrt.h -// Neuron instance families and sizes documented here: +// - `neuron.Instance` exposes the NRT instance family and size used below. +// - Neuron instance families and sizes are documented here: // https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/index.html // - Inf1: // https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/neuron-hardware/inf1-arch.html @@ -18,109 +17,8 @@ const c = @import("c"); // - Trn3: // https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/neuron-hardware/trn3-arch.html // -// The same instance-architecture pages are also the basis for: -// - `InterChipTopology`, including the 2D torus and ring-over-torus fabrics. -// - `Instance.coresPerChip()`, using documented NeuronCore counts per chip generation: -// Inferentia v1 = 4, Inferentia2 / Trainium v1 = 2, Trainium2 / Trainium3 = 8. -pub const Family = enum(u32) { - unknown = 0, - inf1 = 1, - trn1 = 2, - trn1n = 3, - inf2 = 4, - trn2 = 5, - trn2n = 6, - inf2e = 7, - trn2p = 8, - trn2u = 9, - trn2e = 10, - trn2eu = 11, - trn2ac = 12, - trn2uac = 13, - trn3 = 14, - _, -}; - -pub const Size = enum(u32) { - xl1 = 0, - xl2 = 1, - xl4 = 2, - xl6 = 3, - xl8 = 4, - xl24 = 5, - xl32 = 6, - xl48 = 7, - xl3 = 8, - unknown = 9, - _, -}; - -pub const Instance = struct { - family: Family, - size: Size, - - pub fn current() !Instance { - var info = std.mem.zeroes(c.struct_nrt_instance_info); - info.family = @intFromEnum(Family.unknown); - info.size = @intFromEnum(Size.unknown); - - if (c.nrt_get_instance_info(&info, @sizeOf(c.struct_nrt_instance_info)) != c.NRT_SUCCESS) { - return error.Unavailable; - } - - return .{ - .family = @enumFromInt(info.family), - .size = @enumFromInt(info.size), - }; - } - - pub fn coresPerChip(self: Instance) !usize { - return switch (self.family) { - .inf1 => 4, - .trn1, .trn1n, .inf2, .inf2e => 2, - .trn2, .trn2n, .trn2p, .trn2u, .trn2e, .trn2eu, .trn2ac, .trn2uac, .trn3 => 8, - else => error.Unavailable, - }; - } - - pub fn interChipTopology(self: Instance, chip_count: usize) InterChipTopology { - if (chip_count == 1) return .none; - - // References for chip-count and fabric-shape inference: - // - Inf1 instance sizes and 1/4/16-chip layouts - // - Inf2 instance sizes and 1/6/12-chip layouts - // - Trn1 / Trn1n 16-chip 2D torus - // - Trn2 16-chip torus and Trn2 UltraServer 64-chip ring-over-torus - // - Trn3 all-to-all / point-to-point scale-up - return switch (self.family) { - .inf1 => switch (chip_count) { - 4 => .{ .linear = 4 }, - else => .{ .ring = chip_count }, - }, - .inf2, .inf2e => switch (chip_count) { - 6 => .{ .torus2d = .{ .x = 3, .y = 2 } }, - 12 => .{ .torus2d = .{ .x = 4, .y = 3 } }, - else => switch (self.size) { - .xl24 => .{ .torus2d = .{ .x = 3, .y = 2 } }, - .xl48 => .{ .torus2d = .{ .x = 4, .y = 3 } }, - else => .{ .ring = chip_count }, - }, - }, - .trn1, .trn1n => switch (chip_count) { - 16 => .{ .torus2d = .{ .x = 4, .y = 4 } }, - else => .{ .ring = chip_count }, - }, - .trn2, .trn2n, .trn2p, .trn2u, .trn2e, .trn2eu, .trn2ac, .trn2uac => switch (chip_count) { - 16 => .{ .torus2d = .{ .x = 4, .y = 4 } }, - 64 => .{ .torus2d_ring = .{ .x = 4, .y = 4, .z = 4 } }, - else => .{ .ring = chip_count }, - }, - .trn3 => .{ .point_to_point = chip_count }, - else => .{ .ring = chip_count }, - }; - } -}; - +// The same instance-architecture pages are also the basis for the inter-chip +// topology inference below, including 2D torus and ring-over-torus fabrics. pub const InterChipTopology = union(enum) { none: void, linear: usize, @@ -165,7 +63,7 @@ pub const Placement = struct { }; pub const VisibleMesh = struct { - instance: Instance, + instance: neuron.Instance, chips: usize, cores_per_chip: usize, inter_chip_topology: InterChipTopology, @@ -207,7 +105,7 @@ pub const VisibleMesh = struct { pub fn visibleMeshFromNcIds(allocator: std.mem.Allocator, nc_ids: []const usize) !VisibleMesh { if (nc_ids.len == 0) return error.InvalidDeviceTopology; - const instance = try Instance.current(); + const instance = try neuron.instance(); const physical_cores_per_chip = try instance.coresPerChip(); const placements = try allocator.alloc(Placement, nc_ids.len); errdefer allocator.free(placements); @@ -232,7 +130,7 @@ pub fn visibleMeshFromNcIds(allocator: std.mem.Allocator, nc_ids: []const usize) const last_chip = placements[placements.len - 1].chip; const selected_chip_count = last_chip - first_chip + 1; const selected_cores_per_chip = try std.math.divExact(usize, placements.len, selected_chip_count); - const inter_chip_topology = instance.interChipTopology(selected_chip_count); + const inter_chip_topology = interChipTopology(instance, selected_chip_count); var res: VisibleMesh = .{ .instance = instance, @@ -269,6 +167,43 @@ pub fn visibleMeshFromNcIds(allocator: std.mem.Allocator, nc_ids: []const usize) return res; } +pub fn interChipTopology(instance: neuron.Instance, chip_count: usize) InterChipTopology { + if (chip_count == 1) return .none; + + // References for chip-count and fabric-shape inference: + // - Inf1 instance sizes and 1/4/16-chip layouts + // - Inf2 instance sizes and 1/6/12-chip layouts + // - Trn1 / Trn1n 16-chip 2D torus + // - Trn2 16-chip torus and Trn2 UltraServer 64-chip ring-over-torus + // - Trn3 all-to-all / point-to-point scale-up + return switch (instance.family) { + .inf1 => switch (chip_count) { + 4 => .{ .linear = 4 }, + else => .{ .ring = chip_count }, + }, + .inf2, .inf2e => switch (chip_count) { + 6 => .{ .torus2d = .{ .x = 3, .y = 2 } }, + 12 => .{ .torus2d = .{ .x = 4, .y = 3 } }, + else => switch (instance.size) { + .xl24 => .{ .torus2d = .{ .x = 3, .y = 2 } }, + .xl48 => .{ .torus2d = .{ .x = 4, .y = 3 } }, + else => .{ .ring = chip_count }, + }, + }, + .trn1, .trn1n => switch (chip_count) { + 16 => .{ .torus2d = .{ .x = 4, .y = 4 } }, + else => .{ .ring = chip_count }, + }, + .trn2, .trn2n, .trn2p, .trn2u, .trn2e, .trn2eu, .trn2ac, .trn2uac => switch (chip_count) { + 16 => .{ .torus2d = .{ .x = 4, .y = 4 } }, + 64 => .{ .torus2d_ring = .{ .x = 4, .y = 4, .z = 4 } }, + else => .{ .ring = chip_count }, + }, + .trn3, .trn3pds98 => .{ .point_to_point = chip_count }, + .unknown => .{ .ring = chip_count }, + }; +} + pub fn chipOf(nc_id: usize, physical_cores_per_chip: usize) usize { return @divFloor(nc_id, physical_cores_per_chip); } diff --git a/platforms/neuron/topology/BUILD.bazel b/platforms/neuron/topology/BUILD.bazel deleted file mode 100644 index 8c0885c4..00000000 --- a/platforms/neuron/topology/BUILD.bazel +++ /dev/null @@ -1,37 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") -load("@zml//bazel:cc_import.bzl", "cc_import") -load("@zml//bazel:runfiles.bzl", "runfiles_to_default") - -cc_import( - name = "neuron_ncfw", - tags = ["manual"], - shared_library = "@aws-neuronx-runtime-lib//:libncfw.patchelf", -) - -cc_import( - name = "neuron_nrt", - tags = ["manual"], - shared_library = "@aws-neuronx-runtime-lib//:libnrt.patchelf", - deps = [":neuron_ncfw"], -) - -runfiles_to_default( - name = "sandbox", - target_compatible_with = ["@platforms//os:linux"], - visibility = ["//visibility:public"], - deps = ["@libpjrt_neuron//:libpjrt_neuron"], -) - -zig_library( - name = "topology", - tags = ["manual"], - import_name = "platforms/neuron/topology", - main = "topology.zig", - visibility = ["//visibility:public"], - data = [":sandbox"], - deps = [ - ":neuron_nrt", - "//platforms/neuron:libnrt_headers", - "//platforms/neuron:zmlxneuron", - ], -) diff --git a/tools/neuron/BUILD.bazel b/tools/neuron/BUILD.bazel new file mode 100644 index 00000000..bc2e6a75 --- /dev/null +++ b/tools/neuron/BUILD.bazel @@ -0,0 +1,49 @@ +sh_binary( + name = "profile", + srcs = ["profile.sh"], + data = ["@libpjrt_neuron//:sandbox"], + visibility = ["//visibility:public"], +) + +sh_binary( + name = "server", + srcs = ["server.sh"], + args = [ + "/tmp/zml-neuron-explorer", + "3001", + ], + data = ["@libpjrt_neuron//:sandbox"], + visibility = ["//visibility:public"], +) + +sh_binary( + name = "ingest", + srcs = ["ingest.sh"], + args = [ + "/tmp/zml-neuron-explorer", + "/tmp/zml-neuron-profile", + ], + data = ["@libpjrt_neuron//:sandbox"], + visibility = ["//visibility:public"], +) + +sh_binary( + name = "summary-txt", + srcs = ["summary.sh"], + data = ["@libpjrt_neuron//:sandbox"], + visibility = ["//visibility:public"], +) + +sh_binary( + name = "summary-json", + srcs = ["summary.sh"], + data = ["@libpjrt_neuron//:sandbox"], + visibility = ["//visibility:public"], +) + +sh_binary( + name = "summary-perfetto", + srcs = ["summary.sh"], + data = ["@libpjrt_neuron//:sandbox"], + visibility = ["//visibility:public"], +) diff --git a/tools/neuron/ingest.sh b/tools/neuron/ingest.sh new file mode 100755 index 00000000..464fbe5a --- /dev/null +++ b/tools/neuron/ingest.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +set -euo pipefail +shopt -s nullglob + +neuron_explorer_bin="$0.runfiles/+neuron_packages+libpjrt_neuron/sandbox/bin/neuron-explorer" +data_path="$1" +default_profile_root="$2" +shift 2 + +mkdir -p "${data_path}" + +ingest_run() { + local run_dir="$1" + local execution_dir="${run_dir}/execution" + local run_id + run_id="$(basename "${run_dir}")" + + if [[ ! -d "${execution_dir}" ]]; then + echo "skipping ${run_dir}: missing execution directory" >&2 + return + fi + + for profile_session in "${execution_dir}"/*/*; do + [[ -f "${profile_session}/ntrace.pb" ]] || continue + + local system_dir + system_dir="$(mktemp -d)" + for file in ntrace.pb trace_info.pb cpu_util.pb host_mem.pb; do + [[ -f "${profile_session}/${file}" ]] && cp "${profile_session}/${file}" "${system_dir}/${file}" + done + + "${neuron_explorer_bin}" view \ + --ingest-only \ + --force \ + --data-path "${data_path}" \ + --display-name "${run_id}:system" \ + -d "${system_dir}" + + rm -rf "${system_dir}" + done + + for neff in "${execution_dir}"/*/*/*.neff; do + local neff_name + local program_name + local ntff_prefix + neff_name="$(basename "${neff}" .neff)" + program_name="$(grep -aEom1 'MODULE_[A-Za-z0-9_]+' "${neff}" | sed -E 's/^MODULE_//; s/_[0-9]+$//')" + if [[ -z "${program_name}" ]]; then + echo "failed to find program name in ${neff}" >&2 + exit 1 + fi + ntff_prefix="$(dirname "${neff}")/${neff_name}" + + if ! compgen -G "${ntff_prefix}_rank_*.ntff" >/dev/null && [[ ! -f "${ntff_prefix}.ntff" ]]; then + "${neuron_explorer_bin}" capture \ + -n "${neff}" \ + -s "${ntff_prefix}.ntff" \ + --num-exec=2 \ + --profile-nth-exec=2 \ + --io-from=neff + fi + + for ntff in "${ntff_prefix}"_rank_*.ntff "${ntff_prefix}".ntff; do + [[ -f "${ntff}" ]] || continue + + local core_suffix + local display_name + core_suffix="$(basename "${ntff}" .ntff | sed -E 's/^.*_rank_([0-9]+).*$/nc\1/; s/^.*_(vnc_[0-9]+)$/\1/')" + display_name="${run_id}:${program_name}_${core_suffix}" + + "${neuron_explorer_bin}" view \ + --ingest-only \ + --force \ + --data-path "${data_path}" \ + --display-name "${display_name}" \ + -n "${neff}" \ + -s "${ntff}" + done + done +} + +[[ $# -eq 0 ]] && set -- "${default_profile_root}" + +for path in "$@"; do + if [[ -d "${path}/execution" ]]; then + ingest_run "${path}" + else + for run_dir in "${path}"/*; do + [[ -d "${run_dir}" ]] && ingest_run "${run_dir}" + done + fi +done diff --git a/tools/neuron/profile.sh b/tools/neuron/profile.sh new file mode 100755 index 00000000..db6e10a1 --- /dev/null +++ b/tools/neuron/profile.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euo pipefail + +profile_root="$1" +shift + +run_dir="${profile_root}/$(date -u '+%Y%m%dT%H%M%SZ')" +execution_dir="${run_dir}/execution" +compile_dir="${run_dir}/compile" + +mkdir -p "${execution_dir}" "${compile_dir}" + +neuron_explorer_bin="$0.runfiles/+neuron_packages+libpjrt_neuron/sandbox/bin/neuron-explorer" + +xla_flags="--xla_dump_to=${compile_dir} --xla_dump_hlo_as_text --xla_dump_hlo_as_proto --xla_dump_hlo_pass_re=.*" + +set +e +env \ + NEURON_RT_INSPECT_ENABLE=1 \ + NEURON_RT_INSPECT_SYSTEM_PROFILE=1 \ + NEURON_RT_INSPECT_DEVICE_PROFILE=1 \ + NEURON_RT_INSPECT_OUTPUT_DIR="${execution_dir}" \ + SKIP_PJRT_PROFILER=true \ + XLA_IR_DEBUG=1 \ + XLA_HLO_DEBUG=1 \ + XLA_FLAGS="${xla_flags}" \ + "${neuron_explorer_bin}" inspect -o "${execution_dir}" "$@" +status=$? +set -e + +echo "Neuron profile dump path: ${run_dir}" >&2 + +exit "${status}" diff --git a/tools/neuron/server.sh b/tools/neuron/server.sh new file mode 100755 index 00000000..b0e57e41 --- /dev/null +++ b/tools/neuron/server.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +data_path="$1" +port="$2" +neuron_explorer_bin="$0.runfiles/+neuron_packages+libpjrt_neuron/sandbox/bin/neuron-explorer" + +mkdir -p "${data_path}" + +echo "Neuron Explorer server: http://127.0.0.1:${port}" + +exec "${neuron_explorer_bin}" view \ + --data-path "${data_path}" \ + --port "${port}" diff --git a/tools/neuron/summary.sh b/tools/neuron/summary.sh new file mode 100755 index 00000000..a4fe50cd --- /dev/null +++ b/tools/neuron/summary.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euo pipefail + +neuron_explorer_bin="$0.runfiles/+neuron_packages+libpjrt_neuron/sandbox/bin/neuron-explorer" + +run_dir="$1" +execution_dir="${run_dir}/execution" + +case "$(basename "$0")" in + summary-txt) + output_format="summary-text" + output_file="summary.txt" + ;; + summary-json) + output_format="summary-json" + output_file="summary.json" + ;; + summary-perfetto) + output_format="perfetto" + output_file="summary.pftrace" + ;; +esac + +if [[ "${output_format}" == "summary-text" || "${output_format}" == "summary-json" ]]; then + "${neuron_explorer_bin}" view \ + -d "${execution_dir}" \ + --output-format "${output_format}" \ + >"${run_dir}/${output_file}" +else + "${neuron_explorer_bin}" view \ + -d "${execution_dir}" \ + --output-format "${output_format}" \ + --output-file "${run_dir}/${output_file}" +fi + +echo "Neuron summary path: ${run_dir}/${output_file}" >&2 diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 132425be..598d1c30 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -70,13 +70,19 @@ zig_library( tags = ["manual"], data = [ "moe/triton_kernels/config.json", - ], + ] + select({ + "//platforms:neuron.enabled": [ + "//zml/attention/nki:attention_kernel", + ], + "//conditions:default": [], + }), srcs = [ "attention.zig", "attention/attention.zig", "attention/attnd.zig", "attention/flashattn.zig", "attention/mosaic_tpu_kernels/ragged_attention.zig", + "//zml/attention/nki:attention", "attention/paged_attention.zig", "attention/tpu_attention.zig", "attention/triton_attention.zig", @@ -141,7 +147,8 @@ zig_library( ":profiling_traceme_lib", ] + select({ "//platforms:neuron.enabled": [ - "//platforms/neuron/topology", + "//platforms/neuron:nki_kernel", + "//platforms/neuron:topology", ], "//conditions:default": [], }) + select({ diff --git a/zml/Sharding.zig b/zml/Sharding.zig index 22f5e919..88fd0e54 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -47,8 +47,7 @@ pub const Partitioner = union(enum) { pub fn fromTarget(target: Target) Partitioner { return switch (target) { - .cpu, .cuda, .rocm, .tpu, .oneapi => .shardy, - .neuron => .gspmd, + .cpu, .cuda, .rocm, .tpu, .oneapi, .neuron => .shardy, }; } }; diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index ca6345c6..570a9683 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -3,12 +3,14 @@ const std = @import("std"); const zml = @import("../zml.zig"); const attnd = @import("attnd.zig"); const flashattn = @import("flashattn.zig"); +const nki = @import("nki/attention.zig"); const Attention = @This(); pub const Backend = enum { vanilla, attnd, + nki, cuda_fa2, cuda_fa3, @@ -26,7 +28,8 @@ pub const Backend = enum { break :b .vanilla; }, - else => .vanilla, + .neuron => .nki, + .cpu, .rocm, .tpu, .oneapi => .vanilla, }; } }; @@ -34,12 +37,14 @@ pub const Backend = enum { pub const Parameters = union(Backend) { vanilla: void, attnd: attnd.Parameters, + nki: nki.Parameters, cuda_fa2: flashattn.fa2.Parameters, cuda_fa3: flashattn.fa3.Parameters, pub const InitOptions = union(Backend) { vanilla: void, attnd: void, + nki: nki.Parameters, cuda_fa2: flashattn.fa2.Parameters.InitOptions, cuda_fa3: flashattn.fa3.Parameters.InitOptions, @@ -47,6 +52,7 @@ pub const Parameters = union(Backend) { return switch (backend) { .vanilla => .{ .vanilla = {} }, .attnd => @panic("Must be initialized manually"), + .nki => .{ .nki = .init() }, .cuda_fa2 => .{ .cuda_fa2 = .{} }, .cuda_fa3 => .{ .cuda_fa3 = .{} }, }; @@ -57,6 +63,7 @@ pub const Parameters = union(Backend) { return switch (opts) { .vanilla => .{ .vanilla = {} }, .attnd => @panic("Must be initialized manually"), + .nki => |v| .{ .nki = v }, .cuda_fa2 => |v| .{ .cuda_fa2 = .init(v) }, .cuda_fa3 => |v| .{ .cuda_fa3 = .init(v) }, }; @@ -66,12 +73,14 @@ pub const Parameters = union(Backend) { pub const Metadata = union(Backend) { vanilla: void, attnd: attnd.Metadata, + nki: void, cuda_fa2: flashattn.fa2.Metadata, cuda_fa3: flashattn.fa3.Metadata, pub const InitOptions = union(Backend) { vanilla: void, attnd: void, + nki: void, cuda_fa2: flashattn.fa2.Metadata.InitOptions, cuda_fa3: flashattn.fa3.Metadata.InitOptions, @@ -79,6 +88,7 @@ pub const Metadata = union(Backend) { return switch (backend) { .vanilla => .{ .vanilla = {} }, .attnd => .{ .attnd = {} }, + .nki => .{ .nki = {} }, .cuda_fa2 => .{ .cuda_fa2 = .{ .seqlen = seqlen, .num_heads = num_heads } }, .cuda_fa3 => .{ .cuda_fa3 = .{ .seqlen = seqlen, .num_heads = num_heads } }, }; @@ -89,6 +99,7 @@ pub const Metadata = union(Backend) { return switch (opts) { .vanilla => .{ .vanilla = {} }, .attnd => @panic("Must be initialized manually"), + .nki => .{ .nki = {} }, .cuda_fa2 => |o| .{ .cuda_fa2 = flashattn.fa2.Metadata.init(o) }, .cuda_fa3 => |o| .{ .cuda_fa3 = flashattn.fa3.Metadata.init(o) }, }; @@ -98,6 +109,7 @@ pub const Metadata = union(Backend) { return switch (self) { .vanilla => .{ .vanilla = {} }, .attnd => |v| .{ .attnd = try v.initBuffer(io, platform, sharding) }, + .nki => .{ .nki = {} }, .cuda_fa2 => |v| .{ .cuda_fa2 = try v.initBuffer(io, platform, sharding) }, .cuda_fa3 => |v| .{ .cuda_fa3 = try v.initBuffer(io, platform, sharding) }, }; @@ -107,6 +119,7 @@ pub const Metadata = union(Backend) { switch (self.*) { .vanilla => {}, .attnd => |*v| attnd.Metadata.deinitBuffer(v), + .nki => {}, .cuda_fa2 => |*v| flashattn.fa2.Metadata.deinitBuffer(v), .cuda_fa3 => |*v| flashattn.fa3.Metadata.deinitBuffer(v), } @@ -127,6 +140,7 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T break :b attn_output; }, .attnd => attnd.causalAttention(q, k, v, token_index, metadata.attnd, parameters.attnd), + .nki => |params| nki.attention(q, k, v, token_index, params), .cuda_fa2 => flashattn.fa2.attention(q, k, v, token_index, metadata.cuda_fa2, parameters.cuda_fa2), .cuda_fa3 => flashattn.fa3.attention(q, k, v, token_index, metadata.cuda_fa3, parameters.cuda_fa3), }; diff --git a/zml/attention/nki/BUILD.bazel b/zml/attention/nki/BUILD.bazel new file mode 100644 index 00000000..12bbb568 --- /dev/null +++ b/zml/attention/nki/BUILD.bazel @@ -0,0 +1,30 @@ +load("@rules_python//python:py_binary.bzl", "py_binary") + +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "attention_kernel", + srcs = ["attention.py"], +) + +filegroup( + name = "attention", + srcs = ["attention.zig"], +) + +py_binary( + name = "attention_harness", + srcs = ["attention_harness.py"], + data = [ + ":attention.py", + "@libpjrt_neuron//:sandbox", + ], + visibility = ["//visibility:public"], + deps = [ + "@neuron_py_deps//ml_dtypes:pkg", + "@neuron_py_deps//neuronx_cc:pkg", + "@neuron_py_deps//nki:pkg", + "@neuron_py_deps//numpy:pkg", + "@rules_python//python/runfiles", + ], +) diff --git a/zml/attention/nki/attention.py b/zml/attention/nki/attention.py new file mode 100644 index 00000000..cf76d9ae --- /dev/null +++ b/zml/attention/nki/attention.py @@ -0,0 +1,473 @@ +import math + +import nki.isa as nisa +import nki.language as nl + +NEG_INF = -3.3895313892515355e38 + + +def decode(q, k, v, token_index): + q_len = q.shape[0] + num_q_heads = q.shape[1] + d_head = q.shape[2] + seq_len = k.shape[0] + num_kv_heads = k.shape[1] + heads_per_kv = num_q_heads // num_kv_heads + scale = 1.0 / (d_head**0.5) + + assert q_len == 1 + assert k.shape == v.shape + assert q.shape[2] == k.shape[2] + assert num_q_heads % num_kv_heads == 0 + + VALUE_TILE_TOKENS = nl.tile_size.pmax + assert seq_len % VALUE_TILE_TOKENS == 0, ( + "seq_len " + f"({seq_len}) must be a multiple of VALUE_TILE_TOKENS ({VALUE_TILE_TOKENS})" + ) + # Largest tensor-engine K tile that exactly partitions this static sequence. + BLOCK_K = math.gcd(seq_len, nl.tile_size.gemm_moving_fmax) + assert seq_len % BLOCK_K == 0, ( + f"seq_len ({seq_len}) must be a multiple of BLOCK_K ({BLOCK_K})" + ) + assert BLOCK_K % VALUE_TILE_TOKENS == 0, ( + "BLOCK_K " + f"({BLOCK_K}) must be a multiple of VALUE_TILE_TOKENS ({VALUE_TILE_TOKENS})" + ) + num_kv_blocks = seq_len // BLOCK_K + value_tiles_per_k_tile = BLOCK_K // VALUE_TILE_TOKENS + + out = nl.ndarray((q_len, num_q_heads, d_head), dtype=q.dtype, buffer=nl.shared_hbm) + token = nl.ndarray((1, 1), dtype=token_index.dtype, buffer=nl.sbuf) + nisa.dma_copy(dst=token, src=token_index[0:1, 0:1]) + + positions = nl.ndarray((q_len, BLOCK_K), dtype=token_index.dtype, buffer=nl.sbuf) + qk = nl.ndarray((heads_per_kv, BLOCK_K), dtype=nl.float32, buffer=nl.psum) + qk_sbuf = nl.ndarray((heads_per_kv, BLOCK_K), dtype=nl.float32, buffer=nl.sbuf) + masked_qk = nl.ndarray((heads_per_kv, BLOCK_K), dtype=nl.float32, buffer=nl.sbuf) + neg_tile = nl.full( + (heads_per_kv, BLOCK_K), NEG_INF, dtype=nl.float32, buffer=nl.sbuf + ) + tile_max = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + new_row_max = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + old_max_delta = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + old_max_scale = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + row_max = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + neg_row_max = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + exp_tile = nl.ndarray((heads_per_kv, BLOCK_K), dtype=nl.float32, buffer=nl.sbuf) + tile_sum = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + sum_row = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + scaled_sum_row = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + new_sum_row = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + inverse_sum_row = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + running_out = nl.ndarray((heads_per_kv, d_head), dtype=nl.float32, buffer=nl.sbuf) + scaled_running_out = nl.ndarray( + (heads_per_kv, d_head), dtype=nl.float32, buffer=nl.sbuf + ) + attn_out_psum = nl.ndarray((heads_per_kv, d_head), dtype=nl.float32, buffer=nl.psum) + attn_out_sbuf = nl.ndarray((heads_per_kv, d_head), dtype=q.dtype, buffer=nl.sbuf) + + prob_t_psum = nl.ndarray( + (VALUE_TILE_TOKENS, heads_per_kv), dtype=nl.float32, buffer=nl.psum + ) + prob_t = nl.ndarray( + (VALUE_TILE_TOKENS, heads_per_kv), dtype=v.dtype, buffer=nl.sbuf + ) + v_sbuf = nl.ndarray((VALUE_TILE_TOKENS, d_head), dtype=v.dtype, buffer=nl.sbuf) + + for kv_head in nl.affine_range(num_kv_heads): + q_head = kv_head * heads_per_kv + + q_sbuf = nl.ndarray((d_head, heads_per_kv), dtype=q.dtype, buffer=nl.sbuf) + nisa.dma_transpose( + dst=q_sbuf, + src=q[0, q_head : q_head + heads_per_kv, :], + ) + nisa.activation(dst=q_sbuf, op=nl.copy, data=q_sbuf, scale=scale) + + nisa.iota(positions, [[1, BLOCK_K]]) + + k_sbuf = nl.ndarray((d_head, BLOCK_K), dtype=k.dtype, buffer=nl.sbuf) + nisa.dma_transpose(dst=k_sbuf, src=k[nl.ds(0, BLOCK_K), kv_head, :]) + nisa.memset(dst=qk, value=0.0) + nisa.nc_matmul(dst=qk, stationary=q_sbuf, moving=k_sbuf) + nisa.tensor_copy(dst=qk_sbuf, src=qk) + + valid = nl.less_equal(positions, nl.broadcast_to(token, positions.shape)) + masked_qk = nl.where( + nl.broadcast_to(valid, masked_qk.shape), + qk_sbuf, + neg_tile, + dtype=nl.float32, + ) + nisa.tensor_reduce(dst=row_max, op=nl.maximum, data=masked_qk, axis=(1,)) + nisa.tensor_scalar( + dst=neg_row_max, + data=row_max, + op0=nl.multiply, + operand0=-1.0, + ) + nisa.activation( + dst=exp_tile, + op=nl.exp, + data=masked_qk, + bias=neg_row_max, + reduce_op=nl.add, + reduce_res=sum_row, + reduce_cmd=nisa.reduce_cmd.reset_reduce, + ) + + nisa.memset(dst=attn_out_psum, value=0.0) + for value_tile_idx in nl.affine_range(value_tiles_per_k_tile): + value_start = value_tile_idx * VALUE_TILE_TOKENS + nisa.nc_transpose( + dst=prob_t_psum, + data=exp_tile[:, nl.ds(value_start, VALUE_TILE_TOKENS)], + ) + nisa.tensor_copy(dst=prob_t, src=prob_t_psum) + + nisa.dma_copy( + dst=v_sbuf, + src=v[nl.ds(value_start, VALUE_TILE_TOKENS), kv_head, :], + ) + nisa.nc_matmul(dst=attn_out_psum, stationary=prob_t, moving=v_sbuf) + nisa.tensor_copy(dst=running_out, src=attn_out_psum) + + for k_tail_idx in nl.affine_range(num_kv_blocks - 1): + k_block_idx = k_tail_idx + 1 + k_start = k_block_idx * BLOCK_K + nisa.iota(positions, [[1, BLOCK_K]], offset=k_start) + + k_sbuf = nl.ndarray((d_head, BLOCK_K), dtype=k.dtype, buffer=nl.sbuf) + nisa.dma_transpose(dst=k_sbuf, src=k[nl.ds(k_start, BLOCK_K), kv_head, :]) + nisa.memset(dst=qk, value=0.0) + nisa.nc_matmul(dst=qk, stationary=q_sbuf, moving=k_sbuf) + nisa.tensor_copy(dst=qk_sbuf, src=qk) + + valid = nl.less_equal(positions, nl.broadcast_to(token, positions.shape)) + masked_qk = nl.where( + nl.broadcast_to(valid, masked_qk.shape), + qk_sbuf, + neg_tile, + dtype=nl.float32, + ) + nisa.tensor_reduce(dst=tile_max, op=nl.maximum, data=masked_qk, axis=(1,)) + nisa.tensor_tensor( + dst=new_row_max, + data1=row_max, + data2=tile_max, + op=nl.maximum, + ) + nisa.tensor_scalar( + dst=old_max_delta, + data=row_max, + op0=nl.subtract, + operand0=new_row_max, + ) + nisa.activation(dst=old_max_scale, op=nl.exp, data=old_max_delta) + nisa.tensor_scalar( + dst=neg_row_max, + data=new_row_max, + op0=nl.multiply, + operand0=-1.0, + ) + nisa.activation( + dst=exp_tile, + op=nl.exp, + data=masked_qk, + bias=neg_row_max, + reduce_op=nl.add, + reduce_res=tile_sum, + reduce_cmd=nisa.reduce_cmd.reset_reduce, + ) + nisa.tensor_scalar( + dst=scaled_sum_row, + data=sum_row, + op0=nl.multiply, + operand0=old_max_scale, + ) + nisa.tensor_tensor( + dst=new_sum_row, + data1=scaled_sum_row, + data2=tile_sum, + op=nl.add, + ) + nisa.tensor_scalar( + dst=scaled_running_out, + data=running_out, + op0=nl.multiply, + operand0=old_max_scale, + ) + + nisa.memset(dst=attn_out_psum, value=0.0) + for value_tile_idx in nl.affine_range(value_tiles_per_k_tile): + value_start = value_tile_idx * VALUE_TILE_TOKENS + nisa.nc_transpose( + dst=prob_t_psum, + data=exp_tile[:, nl.ds(value_start, VALUE_TILE_TOKENS)], + ) + nisa.tensor_copy(dst=prob_t, src=prob_t_psum) + + nisa.dma_copy( + dst=v_sbuf, + src=v[nl.ds(k_start + value_start, VALUE_TILE_TOKENS), kv_head, :], + ) + nisa.nc_matmul(dst=attn_out_psum, stationary=prob_t, moving=v_sbuf) + + nisa.tensor_tensor( + dst=running_out, + data1=scaled_running_out, + data2=attn_out_psum, + op=nl.add, + ) + nisa.tensor_copy(dst=row_max, src=new_row_max) + nisa.tensor_copy(dst=sum_row, src=new_sum_row) + + nisa.reciprocal(dst=inverse_sum_row, data=sum_row) + nisa.tensor_scalar( + dst=running_out, + data=running_out, + op0=nl.multiply, + operand0=inverse_sum_row, + ) + nisa.tensor_copy(dst=attn_out_sbuf, src=running_out) + + nisa.dma_copy( + dst=out[0, q_head : q_head + heads_per_kv, :], + src=attn_out_sbuf, + ) + + return out + + +def prefill(q, k, v): + q_len = q.shape[0] + num_q_heads = q.shape[1] + d_head = q.shape[2] + seq_len = k.shape[0] + num_kv_heads = k.shape[1] + heads_per_kv = num_q_heads // num_kv_heads + scale = 1.0 / (d_head**0.5) + + assert q.shape[2] == k.shape[2] + assert k.shape == v.shape + assert num_q_heads % num_kv_heads == 0 + assert q_len == seq_len + + VALUE_TILE_TOKENS = nl.tile_size.pmax + BLOCK_Q = nl.tile_size.pmax + # Largest tensor-engine K tile that exactly partitions this static sequence. + BLOCK_K = math.gcd(seq_len, nl.tile_size.gemm_moving_fmax) + + assert q_len % BLOCK_Q == 0, ( + f"q_len ({q_len}) must be a multiple of BLOCK_Q ({BLOCK_Q})" + ) + assert seq_len % BLOCK_K == 0, ( + f"seq_len ({seq_len}) must be a multiple of BLOCK_K ({BLOCK_K})" + ) + assert BLOCK_K % BLOCK_Q == 0, ( + f"BLOCK_K ({BLOCK_K}) must be a multiple of BLOCK_Q ({BLOCK_Q})" + ) + assert BLOCK_K % VALUE_TILE_TOKENS == 0, ( + "BLOCK_K " + f"({BLOCK_K}) must be a multiple of VALUE_TILE_TOKENS ({VALUE_TILE_TOKENS})" + ) + + num_q_blocks = q_len // BLOCK_Q + q_blocks_per_k_tile = BLOCK_K // BLOCK_Q + value_tiles_per_k_tile = BLOCK_K // VALUE_TILE_TOKENS + + out = nl.ndarray((q_len, num_q_heads, d_head), dtype=q.dtype, buffer=nl.shared_hbm) + + for kv_head in nl.affine_range(num_kv_heads): + for q_idx in nl.affine_range(heads_per_kv): + q_head = kv_head * heads_per_kv + q_idx + + for q_block_idx in nl.affine_range(num_q_blocks): + q_start = q_block_idx * BLOCK_Q + + q_sbuf = nl.ndarray((d_head, BLOCK_Q), dtype=q.dtype, buffer=nl.sbuf) + nisa.dma_transpose( + dst=q_sbuf, + src=q[nl.ds(q_start, BLOCK_Q), q_head, :], + ) + nisa.activation(dst=q_sbuf, op=nl.copy, data=q_sbuf, scale=scale) + + row_max = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) + sum_row = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) + running_out = nl.ndarray( + (BLOCK_Q, d_head), dtype=nl.float32, buffer=nl.sbuf + ) + + masked_qk = nl.ndarray( + (BLOCK_Q, BLOCK_K), dtype=nl.float32, buffer=nl.sbuf + ) + qk_sbuf = nl.ndarray( + (BLOCK_Q, BLOCK_K), dtype=nl.float32, buffer=nl.sbuf + ) + tile_max = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) + new_row_max = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) + old_max_delta = nl.ndarray( + (BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf + ) + old_max_scale = nl.ndarray( + (BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf + ) + neg_row_max = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) + exp_tile = nl.ndarray( + (BLOCK_Q, BLOCK_K), dtype=nl.float32, buffer=nl.sbuf + ) + tile_sum = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) + scaled_sum_row = nl.ndarray( + (BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf + ) + new_sum_row = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) + scaled_running_out = nl.ndarray( + (BLOCK_Q, d_head), dtype=nl.float32, buffer=nl.sbuf + ) + attn_out_psum = nl.ndarray( + (BLOCK_Q, d_head), dtype=nl.float32, buffer=nl.psum + ) + prob_t_psum = nl.ndarray( + (VALUE_TILE_TOKENS, BLOCK_Q), dtype=nl.float32, buffer=nl.psum + ) + prob_t = nl.ndarray( + (VALUE_TILE_TOKENS, BLOCK_Q), dtype=v.dtype, buffer=nl.sbuf + ) + qk_psum = nl.ndarray( + (BLOCK_Q, BLOCK_K), dtype=nl.float32, buffer=nl.psum + ) + v_sbuf = nl.ndarray( + (VALUE_TILE_TOKENS, d_head), dtype=v.dtype, buffer=nl.sbuf + ) + + nisa.memset(dst=row_max, value=NEG_INF) + nisa.memset(dst=sum_row, value=0.0) + nisa.memset(dst=running_out, value=0.0) + active_k_tiles = q_block_idx // q_blocks_per_k_tile + 1 + + for k_block_idx in nl.affine_range(active_k_tiles): + k_start = k_block_idx * BLOCK_K + k_sbuf = nl.ndarray( + (d_head, BLOCK_K), dtype=k.dtype, buffer=nl.sbuf + ) + nisa.dma_transpose( + dst=k_sbuf, + src=k[nl.ds(k_start, BLOCK_K), kv_head, :], + ) + nisa.memset(dst=qk_psum, value=0.0) + nisa.nc_matmul(dst=qk_psum, stationary=q_sbuf, moving=k_sbuf) + nisa.tensor_copy(dst=qk_sbuf, src=qk_psum) + nisa.affine_select( + dst=masked_qk, + pattern=[[-1, BLOCK_K]], + offset=q_start - k_start, + channel_multiplier=1, + on_true_tile=qk_sbuf, + on_false_value=NEG_INF, + cmp_op=nl.greater_equal, + ) + nisa.tensor_reduce( + dst=tile_max, + op=nl.maximum, + data=masked_qk, + axis=(1,), + ) + nisa.tensor_tensor( + dst=new_row_max, + data1=row_max, + data2=tile_max, + op=nl.maximum, + ) + nisa.tensor_scalar( + dst=old_max_delta, + data=row_max, + op0=nl.subtract, + operand0=new_row_max, + ) + nisa.activation(dst=old_max_scale, op=nl.exp, data=old_max_delta) + nisa.tensor_scalar( + dst=neg_row_max, + data=new_row_max, + op0=nl.multiply, + operand0=-1.0, + ) + nisa.activation( + dst=exp_tile, + op=nl.exp, + data=masked_qk, + bias=neg_row_max, + reduce_op=nl.add, + reduce_res=tile_sum, + reduce_cmd=nisa.reduce_cmd.reset_reduce, + ) + nisa.tensor_scalar( + dst=scaled_sum_row, + data=sum_row, + op0=nl.multiply, + operand0=old_max_scale, + ) + nisa.tensor_tensor( + dst=new_sum_row, + data1=scaled_sum_row, + data2=tile_sum, + op=nl.add, + ) + nisa.tensor_scalar( + dst=scaled_running_out, + data=running_out, + op0=nl.multiply, + operand0=old_max_scale, + ) + + nisa.memset(dst=attn_out_psum, value=0.0) + for value_tile_idx in nl.affine_range(value_tiles_per_k_tile): + value_start = value_tile_idx * VALUE_TILE_TOKENS + nisa.nc_transpose( + dst=prob_t_psum, + data=exp_tile[:, nl.ds(value_start, VALUE_TILE_TOKENS)], + ) + nisa.tensor_copy(dst=prob_t, src=prob_t_psum) + nisa.dma_copy( + dst=v_sbuf, + src=v[ + nl.ds(k_start + value_start, VALUE_TILE_TOKENS), + kv_head, + :, + ], + ) + nisa.nc_matmul( + dst=attn_out_psum, + stationary=prob_t, + moving=v_sbuf, + ) + + nisa.tensor_tensor( + dst=running_out, + data1=scaled_running_out, + data2=attn_out_psum, + op=nl.add, + ) + nisa.tensor_copy(dst=row_max, src=new_row_max) + nisa.tensor_copy(dst=sum_row, src=new_sum_row) + + inv_sum_row = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) + normalized_out = nl.ndarray( + (BLOCK_Q, d_head), dtype=nl.float32, buffer=nl.sbuf + ) + attn_out = nl.ndarray((BLOCK_Q, d_head), dtype=q.dtype, buffer=nl.sbuf) + nisa.reciprocal(dst=inv_sum_row, data=sum_row) + nisa.tensor_scalar( + dst=normalized_out, + data=running_out, + op0=nl.multiply, + operand0=inv_sum_row, + ) + nisa.tensor_copy(dst=attn_out, src=normalized_out) + + nisa.dma_copy( + dst=out[nl.ds(q_start, BLOCK_Q), q_head, :], + src=attn_out, + ) + + return out diff --git a/zml/attention/nki/attention.zig b/zml/attention/nki/attention.zig new file mode 100644 index 00000000..c08a5ac4 --- /dev/null +++ b/zml/attention/nki/attention.zig @@ -0,0 +1,99 @@ +const std = @import("std"); + +const platforms = @import("platforms"); + +const zml = @import("../../zml.zig"); +const stdx = zml.stdx; + +const KernelSpec = struct { + name: []const u8, + entrypoint: []const u8, + source_path: []const u8, +}; + +pub const Parameters = struct { + compiler_target: []const u8, + decode_kernel: KernelSpec, + prefill_kernel: KernelSpec, + + pub fn init() Parameters { + if (comptime platforms.isEnabled(.neuron)) { + const nki_kernel = @import("platforms/neuron/nki_kernel"); + const compiler_target = nki_kernel.compilerTargetFromInstance(); + return .{ + .compiler_target = @tagName(compiler_target), + .decode_kernel = switch (compiler_target) { + .trn1, .inf2, .trn1n, .trn2, .trn2n, .trn3 => .{ + .name = "nki_attention_trn1_decode", + .entrypoint = "decode", + .source_path = "zml/zml/attention/nki/attention.py", + }, + }, + .prefill_kernel = switch (compiler_target) { + .trn1, .inf2, .trn1n, .trn2, .trn2n, .trn3 => .{ + .name = "nki_attention_trn1_prefill", + .entrypoint = "prefill", + .source_path = "zml/zml/attention/nki/attention.py", + }, + }, + }; + } + + stdx.debug.panic("NKI attention requires the Neuron platform", .{}); + } +}; + +/// Attention for Neuron. Both prefill and decode lower to NKI custom calls. +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, parameters: Parameters) zml.Tensor { + const q_sharded = q.withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }); + const k_sharded = k.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); + const v_sharded = v.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); + + if (q.dim(.q) == 1) { + const token_index_2d = token_index.broad(zml.Shape.init(.{ .row = 1, .col = 1 }, token_index.dtype())) + .withPartitioning(.{ .row = .replicated, .col = .replicated }); + + return zml.ops.manualComputation( + .{ q_sharded, k_sharded, v_sharded, token_index_2d }, + q_sharded.shape(), + parameters, + (struct { + fn body(context: Parameters, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output: zml.Shape) zml.Tensor { + stdx.debug.assert(sharded_inputs.len == 4, "NKI decode attention expects 4 sharded inputs, got {}", .{sharded_inputs.len}); + const kernel = context.decode_kernel; + return zml.ops.neuronNki( + .{ sharded_inputs[0], sharded_inputs[1], sharded_inputs[2], sharded_inputs[3] }, + .{output}, + .{ + .name = kernel.name, + .entrypoint = kernel.entrypoint, + .source_path = kernel.source_path, + .compiler_target = context.compiler_target, + }, + )[0]; + } + }).body, + ).convert(q.dtype()); + } + + return zml.ops.manualComputation( + .{ q_sharded, k_sharded, v_sharded }, + q_sharded.shape(), + parameters, + (struct { + fn body(context: Parameters, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output: zml.Shape) zml.Tensor { + const kernel = context.prefill_kernel; + return zml.ops.neuronNki( + .{ sharded_inputs[0], sharded_inputs[1], sharded_inputs[2] }, + .{output}, + .{ + .name = kernel.name, + .entrypoint = kernel.entrypoint, + .source_path = kernel.source_path, + .compiler_target = context.compiler_target, + }, + )[0]; + } + }).body, + ).convert(q.dtype()); +} diff --git a/zml/attention/nki/attention_harness.py b/zml/attention/nki/attention_harness.py new file mode 100644 index 00000000..372a0fc0 --- /dev/null +++ b/zml/attention/nki/attention_harness.py @@ -0,0 +1,331 @@ +import argparse +import dataclasses +import importlib.util +import os +import sys +import time +from pathlib import Path + +import ml_dtypes +import nki +import numpy as np +from python.runfiles import Runfiles + + +DEFAULT_SEQ_LEN = 128 +ATOL = 1e-2 +RTOL = 1e-2 +MIN_CLOSE_FRACTION = 0.99 +INPUT_CASES = ("baseline", "signed") + + +@dataclasses.dataclass(frozen=True) +class ModelSpec: + name: str + num_heads: int + num_kv_heads: int + head_dim: int + + @property + def heads_per_kv(self): + return self.num_heads // self.num_kv_heads + + +MODEL_PRESETS = { + "llama-3.1-8b": ModelSpec( + name="llama-3.1-8b", + num_heads=32, + num_kv_heads=8, + head_dim=128, + ), +} + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Run the classic Neuron NKI attention correctness contract." + ) + parser.add_argument( + "--model", + choices=(*MODEL_PRESETS.keys(), "custom"), + default="llama-3.1-8b", + ) + parser.add_argument("--seq-len", type=int, default=DEFAULT_SEQ_LEN) + parser.add_argument("--dtype", choices=("bf16", "f16", "f32"), default="bf16") + parser.add_argument("--run", choices=("both", "prefill", "decode"), default="both") + parser.add_argument("--mode", choices=("verify", "benchmark"), default="verify") + parser.add_argument("--warmups", type=int, default=1) + parser.add_argument("--iterations", type=int, default=5) + parser.add_argument("--num-heads", type=int, default=None) + parser.add_argument("--num-kv-heads", type=int, default=None) + parser.add_argument("--head-dim", type=int, default=None) + return parser.parse_args() + + +def model_spec(args): + custom_values = (args.num_heads, args.num_kv_heads, args.head_dim) + if args.model != "custom": + if any(value is not None for value in custom_values): + raise ValueError("shape flags are only accepted with --model=custom") + return MODEL_PRESETS[args.model] + + if any(value is None for value in custom_values): + raise ValueError("--model=custom requires --num-heads, --num-kv-heads, and --head-dim") + spec = ModelSpec( + name="custom", + num_heads=args.num_heads, + num_kv_heads=args.num_kv_heads, + head_dim=args.head_dim, + ) + validate_spec(spec) + return spec + + +def validate_spec(spec): + if spec.num_heads <= 0 or spec.num_kv_heads <= 0 or spec.head_dim <= 0: + raise ValueError("model dimensions must be positive") + if spec.num_heads % spec.num_kv_heads != 0: + raise ValueError("num_heads must be divisible by num_kv_heads") + + +def validate_seq_len(seq_len): + if seq_len <= 0 or seq_len % 128 != 0: + raise ValueError("--seq-len must be a positive multiple of 128") + + +def validate_execution_args(args): + if args.mode == "verify": + return + if args.warmups < 0: + raise ValueError("--warmups must be non-negative") + if args.iterations <= 0: + raise ValueError("--iterations must be positive") + + +def decode_positions(seq_len): + return tuple(dict.fromkeys((0, 1, seq_len // 2, seq_len - 1))) + + +def configure_runtime_environment(mode): + if mode == "benchmark": + os.environ.pop("NEURON_RT_LOG_LEVEL", None) + + +def load_nki_attention_module(): + path = Path(__file__).with_name("attention.py") + spec = importlib.util.spec_from_file_location("zml_nki_attention", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"could not load {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def add_neuron_tools_to_path(): + runfiles = Runfiles.Create() + python_neuronx_cc = Path( + runfiles.Rlocation("+neuron_packages+libpjrt_neuron/sandbox/bin/python-shims/neuronx-cc") + ) + neuronx_cc = Path( + runfiles.Rlocation("+neuron_packages+libpjrt_neuron/sandbox/bin/neuronx-cc") + ) + os.environ["PATH"] = os.pathsep.join( + (str(python_neuronx_cc.parent), str(neuronx_cc.parent), os.environ["PATH"]) + ) + + +def numpy_dtype(name): + if name == "bf16": + return ml_dtypes.bfloat16 + if name == "f16": + return np.float16 + if name == "f32": + return np.float32 + raise ValueError(f"unsupported dtype: {name}") + + +def fill_input(shape, offset, input_case, dtype): + values = np.linspace(0.0, 1.0, num=int(np.prod(shape)), endpoint=False, dtype=np.float32) + if input_case == "signed": + data = offset + (values - 0.5) * 0.25 + else: + data = offset + values * 0.03 + return data.reshape(shape).astype(dtype) + + +def qkv_inputs(spec, seq_len, input_case, decode, dtype): + q_len = 1 if decode else seq_len + q = fill_input((q_len, spec.num_heads, spec.head_dim), 0.01, input_case, dtype) + k = fill_input((seq_len, spec.num_kv_heads, spec.head_dim), 0.02, input_case, dtype) + v = fill_input((seq_len, spec.num_kv_heads, spec.head_dim), 0.03, input_case, dtype) + return q, k, v + + +def reference_attention(q, k, v, token_index=None): + q_len, num_heads, head_dim = q.shape + seq_len, num_kv_heads, _ = k.shape + heads_per_kv = num_heads // num_kv_heads + scale = 1.0 / np.sqrt(head_dim) + q_f32 = q.astype(np.float32) + k_f32 = k.astype(np.float32) + v_f32 = v.astype(np.float32) + out = np.empty((q_len, num_heads, head_dim), dtype=np.float32) + + if token_index is None: + valid = np.arange(seq_len)[None, :] <= np.arange(q_len)[:, None] + else: + if q_len != 1: + raise ValueError("decode reference expects q_len=1") + valid = (np.arange(seq_len) <= token_index)[None, :] + + for kv_head in range(num_kv_heads): + keys = k_f32[:, kv_head, :] + values = v_f32[:, kv_head, :] + for q_idx in range(heads_per_kv): + q_head = kv_head * heads_per_kv + q_idx + scores = (q_f32[:, q_head, :] @ keys.T) * scale + scores = np.where(valid, scores, -np.inf) + scores -= np.max(scores, axis=1, keepdims=True) + probs = np.exp(scores) + probs /= np.sum(probs, axis=1, keepdims=True) + out[:, q_head, :] = probs @ values + + return out.astype(q.dtype) + + +def assert_attention_close(label, expected, actual): + expected_f32 = expected.astype(np.float32) + actual_f32 = actual.astype(np.float32) + finite = np.isfinite(actual_f32) + close = np.isclose(expected_f32, actual_f32, atol=ATOL, rtol=RTOL) + close_fraction = float(close.mean()) + max_abs = float(np.nanmax(np.abs(expected_f32 - actual_f32))) + print( + f"{label} finite={int(finite.sum())}/{actual_f32.size} " + f"close_fraction={close_fraction:.6f} max_abs={max_abs:.6g}" + ) + if not finite.all(): + raise AssertionError(f"{label}: output contains NaN or Inf") + if close_fraction < MIN_CLOSE_FRACTION: + raise AssertionError(f"{label}: close_fraction={close_fraction:.6f}") + + +def timed_run(kernel, inputs, mode, warmups, iterations): + if mode == "verify": + start = time.perf_counter() + output = kernel(*inputs) + elapsed = time.perf_counter() - start + return output, elapsed, 1 + + output = kernel(*inputs) + for _ in range(warmups): + output = kernel(*inputs) + + total = 0.0 + for _ in range(iterations): + start = time.perf_counter() + output = kernel(*inputs) + total += time.perf_counter() - start + return output, total, iterations + + +def print_timing(label, elapsed, iterations, output): + if iterations == 1: + print(f"{label} elapsed={elapsed * 1000.0:.3f}ms output={output.shape}") + return + + average = elapsed / iterations + print( + f"{label} total={elapsed * 1000.0:.3f}ms " + f"avg={average * 1000.0:.3f}ms iterations={iterations} output={output.shape}" + ) + + +def run_prefill(prefill_kernel, spec, seq_len, input_case, dtype, mode, warmups, iterations): + q, k, v = qkv_inputs(spec, seq_len, input_case, decode=False, dtype=dtype) + output, elapsed, timed_iterations = timed_run( + prefill_kernel, + (q, k, v), + mode, + warmups, + iterations, + ) + label = f"prefill input_case={input_case}" + if mode == "verify": + expected = reference_attention(q, k, v) + assert_attention_close(label, expected, output) + print_timing(label, elapsed, timed_iterations, output) + + +def run_decode(decode_kernel, spec, seq_len, input_case, token_index, dtype, mode, warmups, iterations): + q, k, v = qkv_inputs(spec, seq_len, input_case, decode=True, dtype=dtype) + token = np.array([[token_index]], dtype=np.uint32) + output, elapsed, timed_iterations = timed_run( + decode_kernel, + (q, k, v, token), + mode, + warmups, + iterations, + ) + label = f"decode input_case={input_case} token_index={token_index}" + if mode == "verify": + expected = reference_attention(q, k, v, token_index) + assert_attention_close(label, expected, output) + print_timing(label, elapsed, timed_iterations, output) + + +def main(): + args = parse_args() + spec = model_spec(args) + validate_spec(spec) + validate_seq_len(args.seq_len) + validate_execution_args(args) + positions = decode_positions(args.seq_len) + dtype = numpy_dtype(args.dtype) + + configure_runtime_environment(args.mode) + add_neuron_tools_to_path() + module = load_nki_attention_module() + prefill_kernel = nki.jit(module.prefill) + decode_kernel = nki.jit(module.decode) + + header = ( + "classic_attention_contract " + f"model={spec.name} seq_len={args.seq_len} num_heads={spec.num_heads} " + f"num_kv_heads={spec.num_kv_heads} heads_per_kv={spec.heads_per_kv} " + f"head_dim={spec.head_dim} dtype={args.dtype} run={args.run} " + f"mode={args.mode}" + ) + if args.mode == "benchmark": + header += f" warmups={args.warmups} iterations={args.iterations}" + print(f"{header} decode_positions={positions}") + for input_case in INPUT_CASES: + if args.run in ("both", "prefill"): + run_prefill( + prefill_kernel, + spec, + args.seq_len, + input_case, + dtype, + args.mode, + args.warmups, + args.iterations, + ) + if args.run in ("both", "decode"): + for token_index in positions: + run_decode( + decode_kernel, + spec, + args.seq_len, + input_case, + token_index, + dtype, + args.mode, + args.warmups, + args.iterations, + ) + + +if __name__ == "__main__": + main() diff --git a/zml/buffer.zig b/zml/buffer.zig index 499ee650..2f205fb5 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -186,25 +186,12 @@ pub const Buffer = struct { pub const UnitializedOptions = struct { memory: Memory.Kind = .default }; pub fn uninitialized( - io: std.Io, + _: std.Io, platform: *const Platform, sh: Shape, sharding: Sharding, opts: UnitializedOptions, ) !Buffer { - switch (platform.target) { - .neuron => { - const allocator = std.heap.smp_allocator; - - // TODO: use the strides trick to avoid allocating so much memory - const host = try allocator.alloc(u8, sh.byteSize()); - defer allocator.free(host); - - return try Buffer.from(io, platform, sh, sharding, host, .{ .wait = true, .memory = opts.memory }); - }, - .cpu, .cuda, .rocm, .tpu, .oneapi => {}, - } - var res: Buffer = .{ ._platform = platform, ._shape = sh, diff --git a/zml/exe.zig b/zml/exe.zig index ba06e3d2..2de92efc 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -287,6 +287,9 @@ pub const Exe = struct { .neuron => { for (events_slice.?) |e| { if (e) |ev| { + if (opts.wait) { + ev.await(self.platform.pjrt_api, io.?) catch unreachable; + } ev.deinit(self.platform.pjrt_api); } } diff --git a/zml/ops.zig b/zml/ops.zig index c5fac4e6..d3c7851e 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -3,6 +3,7 @@ const std = @import("std"); const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); +const platforms = @import("platforms"); const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; @@ -696,6 +697,108 @@ pub fn triton(inputs: anytype, outputs: anytype, opts: TritonOps) [outputs.len]T return outputs_; } +pub const NeuronNkiOps = struct { + name: []const u8, + entrypoint: []const u8, + source_path: []const u8, + compiler_target: []const u8, + has_side_effect: bool = false, + output_operand_aliases: []const dialects.stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, +}; + +/// Generate an MLIR call for a Neuron NKI kernel. +/// +/// This API is Neuron-only: the source is compiled to an +/// `AwsNeuronCustomNativeKernel` backend config while emitting the graph. +pub fn neuronNki(inputs: anytype, outputs: anytype, opts: NeuronNkiOps) [outputs.len]Tensor { + const ctx = CompilationContext.current(); + switch (ctx.platform.target) { + .neuron => {}, + .cpu, .cuda, .rocm, .tpu, .oneapi => { + stdx.debug.panic("neuronNki is only available on Neuron, got {s}", .{@tagName(ctx.platform.target)}); + }, + } + if (comptime !platforms.isEnabled(.neuron)) unreachable; + const nki_kernel = @import("platforms/neuron/nki_kernel"); + + const mlir_ctx = ctx.mlir_ctx; + var arena = std.heap.ArenaAllocator.init(ctx.allocator); + defer arena.deinit(); + + const allocator = arena.allocator(); + + var values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + values[i] = inputs[i].value(); + } + + var res_types: [outputs.len]*const mlir.Type = undefined; + inline for (outputs, 0..) |output, i| { + res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); + } + + var operands_layouts: [inputs.len][]const usize = undefined; + inline for (inputs, 0..) |input, i| { + operands_layouts[i] = allocator.dupe(usize, toUsize(constants.minorToMajor(input.rank())).constSlice()) catch unreachable; + } + + var results_layouts: [outputs.len][]const usize = undefined; + inline for (outputs, 0..) |output, i| { + results_layouts[i] = allocator.dupe(usize, toUsize(constants.minorToMajor(output.rank())).constSlice()) catch unreachable; + } + + var input_signatures: [inputs.len]nki_kernel.TensorSignature = undefined; + inline for (inputs, 0..) |input, i| { + input_signatures[i] = .{ + .dtype = @tagName(input.dtype()), + .dims = input.dims(), + }; + } + var output_signatures: [outputs.len]nki_kernel.TensorSignature = undefined; + inline for (outputs, 0..) |output, i| { + output_signatures[i] = .{ + .dtype = @tagName(output.dtype()), + .dims = output.dims(), + }; + } + + const compiled_backend_config = nki_kernel.compileNkiKernel(ctx.allocator, ctx.io, .{ + .name = opts.name, + .entrypoint = opts.entrypoint, + .source_path = opts.source_path, + .compiler_target = opts.compiler_target, + .inputs = &input_signatures, + .outputs = &output_signatures, + }) catch |err| { + stdx.debug.panic("failed to compile Neuron NKI kernel: {}", .{err}); + }; + defer ctx.allocator.free(compiled_backend_config); + + const op = dialects.stablehlo.custom_call( + mlir_ctx, + &values, + &res_types, + .{ + .call_target_name = "AwsNeuronCustomNativeKernel", + .has_side_effect = opts.has_side_effect, + .operand_layouts = &operands_layouts, + .result_layouts = &results_layouts, + .output_operand_aliases = opts.output_operand_aliases, + .additional_attributes = &.{ + .named(mlir_ctx, "backend_config", .string(mlir_ctx, compiled_backend_config)), + }, + }, + .unknown(mlir_ctx), + ).appendTo(ctx.currentScope().block); + + var outputs_: [outputs.len]Tensor = undefined; + inline for (outputs, 0..) |output, i| { + outputs_[i] = Tensor._result(output, op.result(i)); + } + + return outputs_; +} + test "triton" { const zml = @import("zml.zig"); const platform = zml.testing.env(); @@ -855,6 +958,9 @@ pub fn scatter( return .{self.dynamicUpdateSlice(index_tensors, update)}; } + var updates_values: [inputs.len]*const mlir.Value = undefined; + LoweringCompatibility.preserveScatterDropSemantics(indices_per_axis.slice(), updates, opts, &updates_values); + // TODO: ideally we should catch all possible scatter errors and provide nice error messages. var config = scatterConfig(self.shape(), update.shape(), indices_per_axis, indices_axes); const indices = scatterPrepareIndices(&config, self.shape(), update.shape(), &indices_per_axis, &indices_axes); @@ -866,9 +972,6 @@ pub fn scatter( input_values[i] = inputs[i].value(); } - var updates_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..updates.len) |i| updates_values[i] = updates[i].value(); - const op = dialects.stablehlo.scatter( mlir_ctx, &input_values, @@ -1100,15 +1203,19 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, stdx.debug.assert(a == idx_axes[i - 1] + 1, "gather expects 'idx_axes' to be sequential. But {any} aren't sequential in {f}", .{ idx_axes, self }); } } - var indices_shape = idx_per_axis[0].shape(); - for (idx_per_axis[1..]) |idx| { + var indices_per_axis, _ = Shape.parseStruct(Tensor, idx_per_axis); + + var indices_shape = indices_per_axis.get(0).shape(); + for (indices_per_axis.constSlice()[1..]) |idx| { if (idx.rank() > indices_shape.rank()) { indices_shape = idx.shape(); } } - for (idx_per_axis) |idx| { + for (indices_per_axis.slice()) |*idx| { stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "gather indices can't be broadcasted together {any}", .{idx_per_axis}); + idx.* = idx.broad(indices_shape); } + LoweringCompatibility.preserveGatherFillSemantics(indices_per_axis.slice()); var idx_batch_axes: Shape.DimsArray = .empty; @@ -1160,7 +1267,12 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, // Sometimes the backend recognize this pattern, but not always. // So let us handle that. if (indices_shape.count() == 1 and idx_axes.len == 1) { - return self.dynamicSlice1d(idx_axes[0], .{ .start = idx_per_axis[0].asScalar(), .len = 1 }).reshape(res_shape); + return self + .dynamicSlice1d(idx_axes[0], .{ .start = indices_per_axis.get(0).asScalar(), .len = 1 }) + // Keep downstream resharding after the slice. SPMD engines like Neuron otherwise + // may hoist an all-gather before the slice and materialize the full tensor. + .optimizationBarrier() + .reshape(res_shape); } var slice_dims: Shape.DimsArray = .empty; @@ -1173,7 +1285,7 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, } // TODO: try changing .last by other axis and see the perf impact. - const indices = Tensor.stack(idx_per_axis, .last, .coord); + const indices = Tensor.stack(indices_per_axis.constSlice(), .last, .coord); // scoped_log.debug("gather --> {} {any}", .{ res_shape, res_kind.constSlice() }); const gather_op = dialects.stablehlo.gather( mlir_ctx, @@ -1197,6 +1309,117 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, return Tensor._result(res_shape, gather_op.result(0)); } +/// Backend lowering compatibility shims live here so public tensor/model code +/// can keep expressing StableHLO semantics directly. +/// Each rewrite should +/// preserve the source operation's semantics while avoiding a backend compiler +/// or runtime path that currently mishandles that legal StableHLO. +/// Fill/drop indexed ops use max-value integer sentinels for inactive lanes; +/// those lanes must not issue indirect memory accesses after lowering. +pub const LoweringCompatibility = struct { + /// Keep scalar integer broadcasts as scalar broadcasts until StableHLO broadcast emission. + /// + /// Some backend compilers fold rank-0 integer scalars into full-rank dense + /// splats too early, which can perturb later indexed-update lowering. A + /// scalar optimization barrier preserves the source-level broadcast shape. + pub fn preserveIntegerScalarBroadcast(self: Tensor, output_shape: Shape) ?Tensor { + switch (CompilationContext.current().platform.target) { + .neuron => {}, + .cpu, .cuda, .rocm, .tpu, .oneapi => return null, + } + + if (self.rank() != 0 or output_shape.rank() == 0 or !self.dtype().isInteger()) return null; + return self.optimizationBarrier().broadcast(output_shape, &.{}); + } + + /// Preserve gather fill semantics before backend indirect-memory lowering. + /// + /// Frontends represent inactive gather rows by passing an out-of-bounds + /// integer sentinel together with fill semantics. StableHLO says those rows + /// should produce the fill value, so the sentinel lane is not a real memory + /// access. Some backend lowerings still route every lane through a gather/DGE + /// path before applying fill behavior, so sanitize inactive lanes early while + /// keeping the active rows unchanged. Related upstream Neuron report: + /// https://github.com/aws-neuron/aws-neuron-sdk/issues/1335. + pub fn preserveGatherFillSemantics(indices: []Tensor) void { + switch (CompilationContext.current().platform.target) { + .neuron => {}, + .cpu, .cuda, .rocm, .tpu, .oneapi => return, + } + + const active_lanes = activeLanesForFillDropIndices(indices) orelse return; + replaceInactiveIndirectIndices(indices, active_lanes); + } + + /// Preserve scatter drop semantics before backend indirect-memory lowering. + /// Same as above but for scatter drop semantics. + pub fn preserveScatterDropSemantics(indices: []Tensor, updates: anytype, opts: Tensor.ScatterOpts, update_values: *[updates.len]*const mlir.Value) void { + const active_lanes: ?Tensor = switch (CompilationContext.current().platform.target) { + .neuron => if (opts.update_fn == Tensor.ScatterOpts.increment) activeLanesForFillDropIndices(indices) else null, + .cpu, .cuda, .rocm, .tpu, .oneapi => null, + }; + + if (active_lanes) |active| replaceInactiveIndirectIndices(indices, active); + + inline for (updates, 0..) |update, i| { + const update_tensor = if (active_lanes) |active| blk: { + const mask = active.broad(update.shape().withDtype(.bool)); + const zero = Tensor.constant(update.dtype().zero()).broad(update.shape()); + break :blk mask.select(update, zero); + } else update; + update_values[i] = update_tensor.value(); + } + } + + /// Builds the lane mask used by fill/drop compatibility rewrites. + /// + /// For normal index widths, the max representable value is the inactive + /// sentinel. For 64-bit indices, the affected lowering path narrows to the + /// i32 indirect-index domain, so only values that remain valid after that + /// narrowing are considered active. When multiple index tensors form a + /// coordinate, a lane is active only if every component is active. + fn activeLanesForFillDropIndices(indices: []const Tensor) ?Tensor { + var active: ?Tensor = null; + for (indices) |idx| { + if (!idx.dtype().isInteger()) continue; + + const idx_active = switch (idx.dtype()) { + .i64 => blk: { + const zero = Tensor.scalar(@as(i64, 0), .i64).broad(idx.shape()); + const max_i32 = Tensor.scalar(@as(i64, std.math.maxInt(i32)), .i64).broad(idx.shape()); + break :blk idx.cmp(.GE, zero).logical(.AND, idx.cmp(.LE, max_i32)); + }, + .u64 => blk: { + const max_i32 = Tensor.scalar(@as(u64, std.math.maxInt(i32)), .u64).broad(idx.shape()); + break :blk idx.cmp(.LE, max_i32); + }, + inline .i2, .i4, .i8, .i16, .i32, .u2, .u4, .u8, .u16, .u32 => blk: { + const sentinel = Tensor.constant(idx.dtype().maxValue()).broad(idx.shape()); + break :blk idx.cmp(.NE, sentinel); + }, + .bool, .f4e2m1, .f8e3m4, .f8e4m3, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .f8e8m0, .bf16, .f16, .f32, .f64, .c64, .c128 => unreachable, + }; + active = if (active) |current| current.logical(.AND, idx_active) else idx_active; + } + return active; + } + + /// Replaces inactive sentinel indices with a safe index before indirect lowering. + /// + /// The replacement index is intentionally zero: it is always in-bounds for + /// non-empty operands and the corresponding scatter update is masked away by + /// the caller. This keeps the lowered indirect operation from touching the + /// sentinel address while preserving fill/drop semantics at the tensor level. + fn replaceInactiveIndirectIndices(indices: []Tensor, active_lanes: Tensor) void { + for (indices) |*idx| { + if (!idx.dtype().isInteger()) continue; + + const safe_index = Tensor.constant(idx.dtype().zero()).broad(idx.shape()); + idx.* = active_lanes.select(idx.*, safe_index); + } + } +}; + pub fn _collectAxes(T: type, bounded_array: stdx.BoundedArray(T, constants.MAX_RANK), value: T) stdx.BoundedArray(i64, constants.MAX_RANK) { var res: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; for (bounded_array.constSlice(), 0..) |v, ax| { @@ -1597,11 +1820,12 @@ fn manualComputationInternal( global_values[i] = shard_to_full.result(0); } - const barrier = mlir.Operation.make(ctx.mlir_ctx, "stablehlo.optimization_barrier", .{ - .operands = .{ .flat = global_values }, - .results = .{ .flat = global_types }, - .location = .unknown(ctx.mlir_ctx), - }).appendTo(ctx.currentScope().block); + const barrier = dialects.stablehlo.optimizationBarrier( + ctx.mlir_ctx, + global_values, + global_types, + .unknown(ctx.mlir_ctx), + ).appendTo(ctx.currentScope().block); const outputs_ = allocator.alloc(Tensor, outputs.len) catch unreachable; for (outputs, 0..) |output_shape, i| { diff --git a/zml/tensor.zig b/zml/tensor.zig index b8ffe93b..750a2c8f 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -188,7 +188,10 @@ pub const Tensor = struct { pub fn toMemory(self: Tensor, kind: Memory.Kind) Tensor { const ctx = CompilationContext.current(); - if (ctx.platform.target == .cpu) return self; + switch (ctx.platform.target) { + .cpu, .neuron, .oneapi => return self, + .cuda, .rocm, .tpu => {}, + } const frontend_attributes: *const mlir.Attribute = .dict(ctx.mlir_ctx, &.{ .named( @@ -220,7 +223,10 @@ pub const Tensor = struct { pub fn onMemory(self: Tensor, kind: Memory.Kind) Tensor { const ctx = CompilationContext.current(); - if (ctx.platform.target == .cpu) return self; + switch (ctx.platform.target) { + .cpu, .neuron, .oneapi => return self, + .cuda, .rocm, .tpu => {}, + } if (ctx.currentScope().id_to_argument.get(self.id) == null) { return self; @@ -2302,6 +2308,8 @@ pub const Tensor = struct { // Already the right shape if (std.mem.eql(i64, self.dims(), other.dims())) return self; + if (ops.LoweringCompatibility.preserveIntegerScalarBroadcast(self, other)) |rewritten| return rewritten; + // Non ambiguous broadcasting // TODO: broad is error prone because of this: // it will happily broadcast .{ .a = 10, .b = 1 } to .{ .b = 10, .a = 5 } @@ -2317,6 +2325,19 @@ pub const Tensor = struct { return self.broadcast(other, axes_.constSlice()); } + pub fn optimizationBarrier(self: Tensor) Tensor { + const ctx = CompilationContext.current(); + + const op = dialects.stablehlo.optimizationBarrier( + ctx.mlir_ctx, + &.{self.value()}, + &.{self.value().type_()}, + .unknown(ctx.mlir_ctx), + ).appendTo(ctx.currentScope().block); + + return _result(self.shape(), op.result(0)); + } + /// Reshapes the input Tensor with the given shape. pub fn reshape(self: Tensor, output_shape_: anytype) Tensor { const output_shape = self._shape.reshape(output_shape_); @@ -3221,9 +3242,53 @@ pub const Tensor = struct { }, else => stdx.debug.compileError(err_msg, .{}), }; - var result = self.sort(a, .{ .descending = opts.descending }); - result.values = result.values.slice1d(a, .{ .end = k }); - result.indices = result.indices.slice1d(a, .{ .end = k }); + const ctx = CompilationContext.current(); + var result: SortRes = switch (ctx.platform.target) { + // Work around https://github.com/aws-neuron/aws-neuron-sdk/issues/1339 until Neuron's sort+slice rewrite uses the slice size as k. + .neuron => blk: { + stdx.debug.assert(self.dtype().isFloat(), "Neuron Tensor.topK only supports float tensors, got {}", .{self.dtype()}); + stdx.debug.assert(k <= 16384, "Neuron AwsNeuronTopK supports k <= 16384, got {}", .{k}); + + const last_axis: u4 = self.rank() - 1; + const topk_input = if (a == last_axis) self else self.swapAxes(a, last_axis); + const call_input = if (opts.descending) topk_input else topk_input.negate(); + const values_shape = topk_input.shape().setDim(last_axis, @intCast(k)); + const indices_shape = values_shape.withDtype(.i32); + const result_types = [_]*const mlir.Type{ + mlirx.Type.rankedTensor(ctx.mlir_ctx, values_shape), + mlirx.Type.rankedTensor(ctx.mlir_ctx, indices_shape), + }; + + var backend_config_buf: [32]u8 = undefined; + const backend_config = std.fmt.bufPrint(&backend_config_buf, "{d}", .{k}) catch unreachable; + const op = dialects.stablehlo.custom_call( + ctx.mlir_ctx, + &.{call_input.value()}, + &result_types, + .{ + .call_target_name = "AwsNeuronTopK", + .has_side_effect = false, + .backend_config = .{ .original = backend_config }, + }, + .unknown(ctx.mlir_ctx), + ).appendTo(currentBlock()); + + var values = Tensor._result(values_shape, op.result(0)); + if (!opts.descending) values = values.negate(); + var indices = Tensor._result(indices_shape, op.result(1)); + if (a != last_axis) { + values = values.swapAxes(a, last_axis); + indices = indices.swapAxes(a, last_axis); + } + break :blk .{ .values = values, .indices = indices }; + }, + .cpu, .cuda, .rocm, .tpu, .oneapi => blk: { + var sorted = self.sort(a, .{ .descending = opts.descending }); + sorted.values = sorted.values.slice1d(a, .{ .end = k }); + sorted.indices = sorted.indices.slice1d(a, .{ .end = k }); + break :blk sorted; + }, + }; if (has_name) |new_name| { result.values._shape._tags.set(a, new_name.ptr); result.indices._shape._tags.set(a, new_name.ptr); From 85e71e405c795ac9b6b9a98ec4bbce3360917756 Mon Sep 17 00:00:00 2001 From: Reese Chong Date: Tue, 16 Jun 2026 17:58:35 +0200 Subject: [PATCH 517/881] zml/moe: fix - move allReduce into manual computation block (#577) Move the MoE allReduce inside the manual computation block. Previously, zml.moe.forwardMoe placed allReduce outside the manual computation block which caused XLA SPMD partitioning to throw the following error with certain models. "Cross-partition allreduce must be in (partial) manual partitioning mode." --- zml/moe/moe.zig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index 7c1bbccd..e24b1caa 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -160,7 +160,7 @@ pub fn forwardMoe( if (expert_partition.eql(.init(.experts))) { const global_num_experts = weights_down.dim(.expert); - const partial_output = zml.ops.manualComputation( + break :b zml.ops.manualComputation( .{ input, topk_ids, topk_weights, weights_gate_up, weights_down }, input.shape(), .{ @@ -203,11 +203,11 @@ pub fn forwardMoe( .w2_bias = ctx.bias_down, }, ) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); - return local_output.reshape(sharded_inputs[0].shape().dims()).withTags(.{ .b, .s, .d }); + const local_reshaped = local_output.reshape(sharded_inputs[0].shape().dims()).withTags(.{ .b, .s, .d }); + return zml.ops.allReduce(local_reshaped, zml.Tensor.add); } }).body, ); - break :b zml.ops.allReduce(partial_output, zml.Tensor.add); } break :b try triton.fusedExpertsImpl( From 700b57466b01e5ecc67c312f31a9cca6b608ba9f Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 18 Jun 2026 15:35:09 +0200 Subject: [PATCH 518/881] zml/io: expose DMA chunk windows to readers from DirectMemoryWriter (#576) DirectMemoryWriter used to expose the next shard-sized slice as its writable window. With HF/VFS this often produced tiny reads, even when adjacent tensor bytes could be streamed through a larger direct-load window. This PR changes direct loading to use an alias-window over the active primary shard DMA buffer. The public `std.Io.Writer` buffer still points at real DMA memory, but its visible length can coalesce across dispatch spans until a DMA chunk, shard buffer, or tensor boundary is reached. On commit, the first active primary segment stays zero-copy, while crossed spans and mirrors are scattered before any buffer is submitted or rotated. `DirectShardWriter` keeps ownership of DMA pool rotation, flip-flop events, and PJRT transfer state. `DirectMemoryWriter` only walks the precomputed dispatch spans with a small cursor. --- examples/io/main.zig | 2 +- zml/io.zig | 835 ++++++++++++++++++++++--------------------- 2 files changed, 425 insertions(+), 412 deletions(-) diff --git a/examples/io/main.zig b/examples/io/main.zig index 51032759..f2c180f6 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -209,7 +209,7 @@ pub fn main(init: std.process.Init) !void { _ = try zml.io.load(AllTensorsModel, &model, init.arena.allocator(), io, platform, &store, .{ .shardings = &.{sharded_sharding}, - .parallelism = 16, + .parallelism = 8, .dma_chunks = 16, .dma_chunk_size = 64 * zml.MiB, .progress = &progress, diff --git a/zml/io.zig b/zml/io.zig index b3c2c993..22c09f8d 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -295,12 +295,13 @@ pub const MemoryWriter = union(enum) { platform: *const Platform, pools: []mem.DynamicBufferPool, dma_allocators: []const mem.DmaAllocator, + dma_chunk_size: usize, shape: Shape, sharding: Sharding, buffer: *Buffer, ) !MemoryWriter { return switch (platform.target) { - .cuda, .oneapi => .{ .direct = try DirectMemoryWriter.init(allocator, io, platform, pools, dma_allocators, shape, sharding, buffer) }, + .cuda, .oneapi => .{ .direct = try DirectMemoryWriter.init(allocator, io, platform, pools, dma_allocators, dma_chunk_size, shape, sharding, buffer) }, .rocm, .tpu, .neuron, .cpu => .{ .buffered = try BufferedMemoryWriter.init(allocator, io, platform, shape, sharding, buffer) }, }; } @@ -408,6 +409,8 @@ const DirectShardWriter = struct { const pjrt_buffer = transfer_manager.retrieveBuffer(memory.platform.pjrt_api, 0) catch unreachable; + const buf = try pool.get(allocator, io); + return .{ .allocator = allocator, .io = io, @@ -417,8 +420,7 @@ const DirectShardWriter = struct { .pjrt_buffer = pjrt_buffer, .transfer_manager = transfer_manager, .interface = .{ - .buffer = try pool.get(allocator, io), - .end = 0, + .buffer = buf[0..@min(buf.len, shape.byteSize())], .vtable = &.{ .drain = drain, .flush = flush, @@ -432,52 +434,27 @@ const DirectShardWriter = struct { self.transfer_manager.deinit(self.memory.platform.pjrt_api); } - fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + fn drain(w: *std.Io.Writer, data: []const []const u8, _: usize) std.Io.Writer.Error!usize { const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); - var consumed: usize = 0; - const pushChunk = struct { - fn call(self_: *DirectShardWriter, chunk: []const u8, consumed_: *usize) std.Io.Writer.Error!void { - var remaining = chunk; - while (remaining.len > 0) { - if (self_.offset + self_.interface.end >= self_.total) { - return std.Io.Writer.Error.WriteFailed; - } + const chunk = data[0]; + if (chunk.len > self.interface.buffer.len) return std.Io.Writer.Error.WriteFailed; + if (chunk.len > self.total - (self.offset + self.interface.end)) return std.Io.Writer.Error.WriteFailed; - if (self_.interface.end == self_.interface.buffer.len) { - try self_.submitBuffered(); - } - - const writable_now = self_.interface.buffer.len - self_.interface.end; - const writable_to_total = self_.total - (self_.offset + self_.interface.end); - const to_copy = @min(remaining.len, @min(writable_now, writable_to_total)); - - @memcpy( - self_.interface.buffer[self_.interface.end .. self_.interface.end + to_copy], - remaining[0..to_copy], - ); - self_.interface.end += to_copy; - consumed_.* += to_copy; - remaining = remaining[to_copy..]; - - if (self_.interface.end == self_.interface.buffer.len or self_.offset + self_.interface.end == self_.total) { - try self_.submitBuffered(); - } - } - } - }.call; - - for (data) |chunk| { - try pushChunk(self, chunk, &consumed); - } - if (data.len > 0 and splat > 1) { - const last = data[data.len - 1]; - for (0..splat - 1) |_| { - try pushChunk(self, last, &consumed); - } + const needs_fresh_buffer = chunk.len > self.interface.buffer.len - self.interface.end; + if (needs_fresh_buffer) { + try self.interface.flush(); } - return consumed; + @memcpy(self.interface.buffer[self.interface.end..][0..chunk.len], chunk); + self.interface.end += chunk.len; + + const buffer_full = self.interface.end == self.interface.buffer.len; + if (buffer_full) { + try self.interface.flush(); + } + + return chunk.len; } fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { @@ -489,49 +466,33 @@ const DirectShardWriter = struct { const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); if (self.interface.buffer.len - self.interface.end >= capacity) return; if (preserve != 0) return std.Io.Writer.Error.WriteFailed; - - if (self.interface.end > 0) { - try self.submitBuffered(); - } - - if (self.interface.buffer.len - self.interface.end < capacity) { - return std.Io.Writer.Error.WriteFailed; - } + try self.interface.flush(); } fn flushBuffered(self: *DirectShardWriter) std.Io.Writer.Error!void { if (self.offset >= self.total) return; - if (self.interface.end == 0) return; - try self.submitBuffered(); - } - - fn submitBuffered(self: *DirectShardWriter) std.Io.Writer.Error!void { - if (self.offset >= self.total) return; const pjrt_api = self.memory.platform.pjrt_api; const current_buffer = self.interface.buffer; const buffered = self.interface.buffered(); - const sliced_buffer = self.interface.buffered()[0..@min(buffered.len, self.total - self.offset)]; - const written = sliced_buffer.len; - const is_last = (self.offset + sliced_buffer.len) >= self.total; - const transfer_event = self.transfer_manager.transferData( - pjrt_api, - 0, - sliced_buffer, - @intCast(self.offset), - is_last, - ) catch |err| { + const slice = buffered[0..@min(buffered.len, self.total - self.offset)]; + if (slice.len == 0) return; + const is_last = (self.offset + slice.len) >= self.total; + + const transfer_event = self.transfer_manager.transferData(pjrt_api, 0, slice, @intCast(self.offset), is_last) catch |err| { log.err("error when transferring data to device: {any}", .{err}); return std.Io.Writer.Error.WriteFailed; }; + const ctx = &self.events_contexts[@intCast(self.flip_flop)]; ctx.* = .{ .self = self, .buffer = current_buffer, .pjrt_event = transfer_event, }; + transfer_event.onReady(pjrt_api, EventContext, struct { fn call(err: ?*pjrt.Error, ctx_: *EventContext) void { ctx_.self.pool.put(ctx_.self.io, ctx_.buffer); @@ -573,193 +534,266 @@ const DirectShardWriter = struct { } } else { self.interface.end = 0; - self.interface.buffer = self.pool.get(self.allocator, self.io) catch |err| { + const buf = self.pool.get(self.allocator, self.io) catch |err| { log.err("unable to get a new buffer from the pool: {any}", .{err}); return std.Io.Writer.Error.WriteFailed; }; + self.interface.buffer = buf[0..@min(buf.len, self.total - (self.offset + slice.len))]; } self.flip_flop ^= 1; - self.offset += written; + self.offset += slice.len; } }; -pub const DirectMemoryWriter = struct { - const StreamPlanner = struct { - const AxisRange = struct { - start: i64, - size: i64, - }; +const ShardProgress = struct { + const scale: usize = 1024; - allocator: std.mem.Allocator, - shape: Shape, - sharding: Sharding, - raw_segments: std.ArrayList(RawSegment) = .empty, - segments: std.ArrayList(StreamSegment) = .empty, - mirrors: std.ArrayList(usize) = .empty, + node: std.Progress.Node, + label: [32]u8 = undefined, + completed: usize = 0, - fn init(allocator: std.mem.Allocator, shape: Shape, sharding: Sharding) StreamPlanner { - return .{ - .allocator = allocator, - .shape = shape, - .sharding = sharding, - }; - } + fn set(self: *ShardProgress, completed: usize) void { + self.completed = completed; + self.node.setCompletedItems(std.math.divCeil(usize, self.completed, scale) catch unreachable); + } +}; - fn deinit(self: *StreamPlanner) void { - self.raw_segments.deinit(self.allocator); - self.segments.deinit(self.allocator); - self.mirrors.deinit(self.allocator); - } - - fn build(self: *StreamPlanner) !StreamPlan { - try self.collectRawSegments(); - self.sortRawSegments(); - try self.compactSegments(); - - const owned_segments = try self.segments.toOwnedSlice(self.allocator); - errdefer self.allocator.free(owned_segments); - const owned_mirrors = try self.mirrors.toOwnedSlice(self.allocator); - errdefer self.allocator.free(owned_mirrors); - - return .{ - .segments = owned_segments, - .mirrors = owned_mirrors, - }; - } - - fn collectRawSegments(self: *StreamPlanner) !void { - const placement = try self.sharding.placement(self.shape); - for (self.sharding.devicesInCanonicalOrder(), 0..) |device, writer_index| { - try self.appendShardSegments(placement.slices(device.coords).constSlice(), writer_index); - } - } - - fn sortRawSegments(self: *StreamPlanner) void { - std.mem.sort(RawSegment, self.raw_segments.items, {}, struct { - fn lessThan(_: void, lhs: RawSegment, rhs: RawSegment) bool { - if (lhs.start != rhs.start) return lhs.start < rhs.start; - if (lhs.len != rhs.len) return lhs.len < rhs.len; - return lhs.writer_index < rhs.writer_index; - } - }.lessThan); - } - - fn compactSegments(self: *StreamPlanner) !void { - var i: usize = 0; - var cursor: usize = 0; - while (i < self.raw_segments.items.len) { - const first = self.raw_segments.items[i]; - if (first.start != cursor) return error.NonContiguousShardPlacement; - - const mirror_start = self.mirrors.items.len; - var j = i + 1; - while (j < self.raw_segments.items.len and - self.raw_segments.items[j].start == first.start and - self.raw_segments.items[j].len == first.len) : (j += 1) - { - try self.mirrors.append(self.allocator, self.raw_segments.items[j].writer_index); - } - - try self.segments.append(self.allocator, .{ - .len = first.len, - .primary_writer = first.writer_index, - .mirror_start = mirror_start, - .mirror_len = j - i - 1, - }); - cursor += first.len; - i = j; - } - - if (cursor != self.shape.byteSize()) return error.NonContiguousShardPlacement; - } - - fn appendRawSegment(self: *StreamPlanner, writer_index: usize, start: usize, len: usize) !void { - if (len == 0) return; - - if (self.raw_segments.items.len > 0) { - const last = &self.raw_segments.items[self.raw_segments.items.len - 1]; - if (last.writer_index == writer_index and last.start + last.len == start) { - last.len += len; - return; - } - } - - try self.raw_segments.append(self.allocator, .{ - .writer_index = writer_index, - .start = start, - .len = len, - }); - } - - fn appendShardSegmentsAtAxis( - self: *StreamPlanner, - slices: []const Placement.Slice1d, - writer_index: usize, - rank: usize, - axis: usize, - base_start: i64, - strides: []const i64, - ) !void { - const range: AxisRange = .{ .start = slices[axis].start, .size = slices[axis].size }; - if (range.size == 0) return; - - if (axis + 1 == rank) { - const segment_start: usize = @intCast(base_start + range.start * strides[axis]); - const segment_len: usize = @intCast(range.size * strides[axis]); - try self.appendRawSegment(writer_index, segment_start, segment_len); - return; - } - - var i: i64 = 0; - while (i < range.size) : (i += 1) { - const child_start = base_start + (range.start + i) * strides[axis]; - try self.appendShardSegmentsAtAxis(slices, writer_index, rank, axis + 1, child_start, strides); - } - } - - fn appendShardSegments(self: *StreamPlanner, slices: []const Placement.Slice1d, writer_index: usize) !void { - const rank = self.shape.rank(); - if (rank == 0) { - try self.appendRawSegment(writer_index, 0, self.shape.byteSize()); - return; - } - - const strides = self.shape.computeByteStrides(); - try self.appendShardSegmentsAtAxis(slices, writer_index, rank, 0, 0, strides.constSlice()); - } - }; - - const StreamSegment = struct { - len: usize, +// Dispatch planning bridges two different orders: +// +// 1. Placement traversal emits byte ranges per shard writer, in device order. +// That order is convenient for asking "what bytes belong to this shard?", +// but it does not match how a tensor file is read. +// +// 2. Readers stream tensor bytes in global row-major order. DirectMemoryWriter +// must therefore consume dispatch spans in increasing global byte offset. +// +// We first collect placement spans, sort them by global byte range, then fold +// identical ranges into one primary dispatch span plus mirror writers. Identical +// ranges represent replicated shard data: the reader provides those bytes once, +// the primary writer receives them zero-copy, and mirrors receive a copy. +const DispatchSpans = struct { + const DispatchSpan = struct { + start: usize, + end: usize, primary_writer: usize, - mirror_start: usize, - mirror_len: usize, + mirror_writer_start: usize, + mirror_writer_len: usize, }; - const RawSegment = struct { + const PlacementSpan = struct { writer_index: usize, start: usize, len: usize, + order: usize, }; - const StreamPlan = struct { - segments: []StreamSegment, - mirrors: []usize, - }; + spans: []DispatchSpan, + mirror_writers: []usize, - const ShardProgress = struct { - node: std.Progress.Node, - label: [32]u8 = undefined, - scale: usize = 1024, - }; + fn init(allocator: std.mem.Allocator, shape: Shape, sharding: Sharding) !DispatchSpans { + const placement = try sharding.placement(shape); + const ordered_devices = sharding.devicesInCanonicalOrder(); + var placement_span_count: usize = 0; + for (ordered_devices) |device| { + placement_span_count += placementSpanCount(shape, placement.slices(device.coords).constSlice()); + } + + var placement_spans: std.ArrayList(PlacementSpan) = try .initCapacity(allocator, placement_span_count); + defer placement_spans.deinit(allocator); + + const byte_strides = shape.computeByteStrides(); + + for (ordered_devices, 0..) |device, writer_index| { + appendShardPlacementSpans(&placement_spans, shape, placement.slices(device.coords).constSlice(), byte_strides.constSlice(), writer_index); + } + + std.debug.assert(placement_spans.items.len == placement_span_count); + + var spans: std.ArrayList(DispatchSpan) = try .initCapacity(allocator, placement_spans.items.len); + errdefer spans.deinit(allocator); + + var mirror_writers: std.ArrayList(usize) = try .initCapacity(allocator, placement_spans.items.len); + errdefer mirror_writers.deinit(allocator); + + try deduplicateByRange(allocator, placement_spans.items, shape.byteSize(), &spans, &mirror_writers); + + const spans_ = try spans.toOwnedSlice(allocator); + errdefer allocator.free(spans_); + + const mirror_writers_ = try mirror_writers.toOwnedSlice(allocator); + errdefer allocator.free(mirror_writers_); + + return .{ + .spans = spans_, + .mirror_writers = mirror_writers_, + }; + } + + fn deinit(self: DispatchSpans, allocator: std.mem.Allocator) void { + allocator.free(self.spans); + allocator.free(self.mirror_writers); + } + + fn deduplicateByRange( + allocator: std.mem.Allocator, + placement_spans: []PlacementSpan, + total_bytes: usize, + spans: *std.ArrayList(DispatchSpan), + mirror_writers: *std.ArrayList(usize), + ) !void { + const SortContext = struct { + fn lessThan(_: void, lhs: PlacementSpan, rhs: PlacementSpan) bool { + if (lhs.start != rhs.start) return lhs.start < rhs.start; + if (lhs.len != rhs.len) return lhs.len < rhs.len; + return lhs.order < rhs.order; + } + }; + + std.mem.sort(PlacementSpan, placement_spans, {}, SortContext.lessThan); + + var i: usize = 0; + var cursor: usize = 0; + while (i < placement_spans.len) { + const span = placement_spans[i]; + if (span.start != cursor) return error.NonContiguousShardPlacement; + + const mirror_writer_start = mirror_writers.items.len; + var j = i + 1; + while (j < placement_spans.len) : (j += 1) { + const mirror = placement_spans[j]; + if (mirror.start != span.start or mirror.len != span.len) break; + try mirror_writers.append(allocator, mirror.writer_index); + } + + try spans.append(allocator, .{ + .start = span.start, + .end = span.start + span.len, + .primary_writer = span.writer_index, + .mirror_writer_start = mirror_writer_start, + .mirror_writer_len = j - i - 1, + }); + cursor += span.len; + i = j; + } + + if (cursor != total_bytes) return error.NonContiguousShardPlacement; + } + + fn appendPlacementSpan(placement_spans: *std.ArrayList(PlacementSpan), writer_index: usize, start: usize, len: usize) void { + placement_spans.appendAssumeCapacity(.{ + .writer_index = writer_index, + .start = start, + .len = len, + .order = placement_spans.items.len, + }); + } + + fn appendShardPlacementSpans( + placement_spans: *std.ArrayList(PlacementSpan), + shape: Shape, + slices: []const Placement.Slice1d, + byte_strides: []const i64, + writer_index: usize, + ) void { + if (shape.rank() == 0) { + appendPlacementSpan(placement_spans, writer_index, 0, shape.byteSize()); + return; + } + + appendShardAxisPlacementSpans(placement_spans, slices, byte_strides, writer_index, 0, contiguousSliceAxis(shape, slices), 0); + } + + fn appendShardAxisPlacementSpans( + placement_spans: *std.ArrayList(PlacementSpan), + slices: []const Placement.Slice1d, + byte_strides: []const i64, + writer_index: usize, + axis: usize, + contiguous_axis: usize, + base_start: i64, + ) void { + const slice = slices[axis]; + if (slice.size == 0) return; + + if (axis == contiguous_axis) { + const span_start: usize = @intCast(base_start + slice.start * byte_strides[axis]); + const span_len: usize = @intCast(slice.size * byte_strides[axis]); + appendPlacementSpan(placement_spans, writer_index, span_start, span_len); + return; + } + + var i: i64 = 0; + while (i < slice.size) : (i += 1) { + const child_start = base_start + (slice.start + i) * byte_strides[axis]; + appendShardAxisPlacementSpans(placement_spans, slices, byte_strides, writer_index, axis + 1, contiguous_axis, child_start); + } + } + + fn placementSpanCount(shape: Shape, slices: []const Placement.Slice1d) usize { + if (shape.rank() == 0) return 1; + + const contiguous_axis = contiguousSliceAxis(shape, slices); + var count: usize = 1; + for (slices[0..contiguous_axis]) |slice| { + count *= @intCast(slice.size); + } + return count; + } + + fn contiguousSliceAxis(shape: Shape, slices: []const Placement.Slice1d) usize { + var axis = shape.rank() - 1; + while (axis > 0) { + const slice = slices[axis]; + if (slice.start != 0 or slice.size != shape.dim(axis)) break; + axis -= 1; + } + return axis; + } +}; + +// Direct load writer state machine. +// +// std.Io.Reader +// | +// v +// DirectMemoryWriter.interface.buffer +// | aliases active primary DirectShardWriter DMA buffer +// v +// commitWindow() +// |-- primary: advance DirectShardWriter.interface.end +// |-- mirrors: copy committed bytes with writeAll() +// |-- boundaries: flush full shard buffers / chunk fence +// v +// DirectShardWriter.flushBuffered() +// | +// v +// PJRT AsyncHostToDeviceTransferManager +// +// The public writer never owns a host staging buffer: it exposes a visible +// prefix of a real shard DMA buffer. The visible prefix may extend beyond the +// current dispatch span to coalesce reader requests, but `commitWindow` must +// scatter those bytes before submitting or rotating the active buffer. The +// first active-primary segment is zero-copy; crossed spans and mirrors treat +// the active buffer as scratch and copy into their own shard writers. +// +// `byte_cursor` is the global tensor position. Shard writer `interface.end` is +// local to the current shard DMA buffer. +pub const DirectMemoryWriter = struct { allocator: std.mem.Allocator, shard_writers: []DirectShardWriter, - stream_segments: []StreamSegment, - segment_mirrors: []usize, - segment_index: usize = 0, - segment_offset: usize = 0, - active_writer_index: ?usize = null, + // Global stream-order spans produced from placement; this is the routing table for committed bytes. + dispatch_spans: DispatchSpans, + // Current entry in `dispatch_spans.spans`; advances whenever `byte_cursor` reaches that span end. + span_index: usize = 0, + // Global tensor byte offset already scattered into shard writers. + byte_cursor: usize = 0, + // Shard writer whose DMA buffer is currently exposed as `interface.buffer`. + active_writer_index: usize, + // Start offset of new reader bytes inside the active shard writer buffer. + window_start: usize, + // Logical maximum public alias window before forcing a cross-shard flush fence. + dma_chunk_size: usize, shard_progress: ?[]ShardProgress = null, interface: std.Io.Writer, @@ -769,6 +803,7 @@ pub const DirectMemoryWriter = struct { platform: *const Platform, pools: []mem.DynamicBufferPool, dma_allocators: []const mem.DmaAllocator, + dma_chunk_size: usize, shape: Shape, sharding: Sharding, buffer: *Buffer, @@ -789,42 +824,32 @@ pub const DirectMemoryWriter = struct { const pool = &pools[device.id]; const shard_dma_allocator = dma_allocators[device.id].allocator(); - const dst_memory = platform.devices[device.id].memory(.default).?; + const pjrt_mem = platform.devices[device.id].memory(.default).?; - shard_writers[i] = try DirectShardWriter.init(shard_dma_allocator, io, dst_memory, pool, placement.shape); + shard_writers[i] = try .init(shard_dma_allocator, io, pjrt_mem, pool, placement.shape); pjrt_buffers.appendAssumeCapacity(shard_writers[i].pjrt_buffer); } - buffer.* = Buffer.fromPjrtBuffers(platform, shape, sharding, pjrt_buffers.constSlice()); + buffer.* = .fromPjrtBuffers(platform, shape, sharding, pjrt_buffers.constSlice()); - var planner = StreamPlanner.init(allocator, shape, sharding); - defer planner.deinit(); + const dispatch_spans: DispatchSpans = try .init(allocator, shape, sharding); + errdefer dispatch_spans.deinit(allocator); - const stream_plan = try planner.build(); - errdefer { - allocator.free(stream_plan.segments); - allocator.free(stream_plan.mirrors); - } - - const first_segment = stream_plan.segments[0]; - const initial_writer_index = first_segment.primary_writer; - const initial_writer = &shard_writers[initial_writer_index]; - const shard_remaining = initial_writer.total - (initial_writer.offset + initial_writer.interface.end); - const initial_visible_len: usize = @min( - initial_writer.interface.buffer.len, - initial_writer.interface.end + @min(first_segment.len, shard_remaining), - ); + const first_span = dispatch_spans.spans[0]; + const first_writer = &shard_writers[first_span.primary_writer]; + const first_window = @min(dma_chunk_size, first_writer.interface.buffer.len); return .{ .allocator = allocator, .shard_writers = shard_writers, - .stream_segments = stream_plan.segments, - .segment_mirrors = stream_plan.mirrors, - .active_writer_index = initial_writer_index, + .dispatch_spans = dispatch_spans, + .active_writer_index = first_span.primary_writer, + .window_start = first_writer.interface.end, + .dma_chunk_size = dma_chunk_size, .interface = .{ - .buffer = initial_writer.interface.buffer[0..initial_visible_len], - .end = initial_writer.interface.end, + .buffer = first_writer.interface.buffer[0..first_window], + .end = 0, .vtable = &.{ .drain = drain, .flush = flush, @@ -840,63 +865,62 @@ pub const DirectMemoryWriter = struct { writer.deinit(); } self.allocator.free(self.shard_writers); - self.allocator.free(self.stream_segments); - self.allocator.free(self.segment_mirrors); + self.dispatch_spans.deinit(self.allocator); } pub fn setProgress(self: *DirectMemoryWriter, progress: ?*std.Progress.Node) void { - if (self.shard_progress) |states| { + const parent = progress orelse { + const states = self.shard_progress orelse return; for (states) |*s| { s.node.end(); } self.allocator.free(states); self.shard_progress = null; - } + return; + }; - const parent = progress orelse return; + std.debug.assert(self.shard_progress == null); const states = self.allocator.alloc(ShardProgress, self.shard_writers.len) catch return; for (states, self.shard_writers, 0..) |*state, writer, i| { - const scale: usize = 1024; - const total_items = std.math.divCeil(usize, writer.total, scale) catch unreachable; + state.completed = 0; const label = std.fmt.bufPrint(&state.label, "shard[{d}]", .{i}) catch unreachable; - state.scale = scale; + const total_items = std.math.divCeil(usize, writer.total, ShardProgress.scale) catch unreachable; state.node = parent.start(label, total_items); state.node.setCompletedItems(0); } self.shard_progress = states; } - fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + fn drain(w: *std.Io.Writer, data: []const []const u8, _: usize) std.Io.Writer.Error!usize { const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); - var consumed: usize = 0; - for (data) |chunk| { - consumed += try self.writeChunk(chunk); - } - if (data.len > 0 and splat > 1) { - const last = data[data.len - 1]; - for (0..splat - 1) |_| { - consumed += try self.writeChunk(last); - } - } + const chunk = data[0]; + try self.commitWindow(); - return consumed; + const writable = self.interface.buffer.len - self.interface.end; + if (writable == 0) return std.Io.Writer.Error.WriteFailed; + + const n = @min(writable, chunk.len); + @memcpy(self.interface.buffer[self.interface.end..][0..n], chunk[0..n]); + self.interface.end += n; + try self.commitWindow(); + + return n; } fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); - try self.syncProxyIntoActive(); - self.refreshExposedWindow(); - - if (self.currentSegment() != null) return std.Io.Writer.Error.WriteFailed; + // Commit the active public window if one remains + if (self.span_index < self.dispatch_spans.spans.len) { + try self.commitWindow(); + } for (self.shard_writers, 0..) |*shard_writer, i| { - if (shard_writer.interface.end > 0) { - try shard_writer.flushBuffered(); + try shard_writer.interface.flush(); + if (self.shard_progress) |states| { + states[i].set(shard_writer.total); } - if (shard_writer.offset < shard_writer.total) return std.Io.Writer.Error.WriteFailed; - self.updateShardProgress(i); } self.interface = .failing; @@ -904,145 +928,130 @@ pub const DirectMemoryWriter = struct { fn rebase(w: *std.Io.Writer, preserve: usize, capacity: usize) std.Io.Writer.Error!void { const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); + + if (self.interface.buffer.len - self.interface.end >= capacity) return; if (preserve != 0) return std.Io.Writer.Error.WriteFailed; - try self.ensureWritable(capacity); + + try self.commitWindow(); + if (self.interface.buffer.len - self.interface.end < capacity) return std.Io.Writer.Error.WriteFailed; } - fn ensureWritable(self: *DirectMemoryWriter, capacity: usize) std.Io.Writer.Error!void { - if (capacity == 0) return; - while (self.interface.buffer.len - self.interface.end < capacity) { - try self.syncProxyIntoActive(); - self.refreshExposedWindow(); - - const segment = self.currentSegment() orelse { - self.interface = .failing; - return std.Io.Writer.Error.WriteFailed; - }; - - if (self.interface.buffer.len - self.interface.end >= capacity) return; - if (capacity > self.interface.buffer.len and self.interface.buffer.len != 0) { - return std.Io.Writer.Error.WriteFailed; - } - - const shard_writer = &self.shard_writers[segment.primary_writer]; - if (shard_writer.interface.end == 0) return std.Io.Writer.Error.WriteFailed; - try shard_writer.flushBuffered(); - - self.refreshExposedWindow(); - if (self.interface.buffer.len - self.interface.end >= capacity) return; - - if (self.currentSegmentRemaining() == 0) continue; - return std.Io.Writer.Error.WriteFailed; - } - } - - fn currentSegment(self: *const DirectMemoryWriter) ?StreamSegment { - if (self.segment_index >= self.stream_segments.len) return null; - return self.stream_segments[self.segment_index]; - } - - fn currentSegmentRemaining(self: *const DirectMemoryWriter) usize { - const segment = self.currentSegment() orelse return 0; - return segment.len -| self.segment_offset; - } - - fn writeChunk(self: *DirectMemoryWriter, chunk: []const u8) std.Io.Writer.Error!usize { - var consumed: usize = 0; - while (consumed < chunk.len) { - try self.ensureWritable(1); - - const segment_remaining = self.currentSegmentRemaining(); - const writable = self.interface.buffer.len - self.interface.end; - const to_write = @min(chunk.len - consumed, @min(segment_remaining, writable)); - if (to_write == 0) return std.Io.Writer.Error.WriteFailed; - - @memcpy( - self.interface.buffer[self.interface.end .. self.interface.end + to_write], - chunk[consumed..][0..to_write], - ); - self.interface.end += to_write; - try self.syncProxyIntoActive(); - consumed += to_write; - } - - return consumed; - } - - fn refreshExposedWindow(self: *DirectMemoryWriter) void { - self.skipCompletedSegments(); - if (self.currentSegment()) |segment| { - self.exposeShardWindow(segment.primary_writer, self.currentSegmentRemaining()); - } else { - self.hideExposedWindow(); - } - } - - fn skipCompletedSegments(self: *DirectMemoryWriter) void { - while (self.currentSegment()) |_| { - if (self.currentSegmentRemaining() != 0) return; - self.segment_index += 1; - self.segment_offset = 0; - } - } - - fn hideExposedWindow(self: *DirectMemoryWriter) void { - self.active_writer_index = null; - stdx.debug.assert(self.shard_writers.len > 0, "DirectMemoryWriter requires at least one shard writer", .{}); - self.interface.buffer = self.shard_writers[0].interface.buffer[0..0]; - self.interface.end = 0; - } - - fn syncProxyIntoActive(self: *DirectMemoryWriter) std.Io.Writer.Error!void { - const writer_index = self.active_writer_index orelse return; - const segment = self.currentSegment() orelse return; - if (segment.primary_writer != writer_index) return; - - const shard_writer = &self.shard_writers[writer_index]; - if (self.interface.end <= shard_writer.interface.end) { - self.interface.end = shard_writer.interface.end; + // `interface.buffer` is a clipped alias of the active primary shard DMA + // buffer. Reader writes advance only `DirectMemoryWriter.interface.end`; + // this function scatters that visible range through dispatch spans. + // + // The leading segment for `active_writer_index` is already in the right + // DMA buffer and is committed by advancing that shard writer. Later spans + // use the same bytes as scratch and are copied before any full buffers are + // submitted or the public alias is rotated. + fn commitWindow(self: *DirectMemoryWriter) std.Io.Writer.Error!void { + // All dispatch spans are consumed; no shard DMA buffer remains valid to + // expose as the public alias until flush marks the writer as failing. + if (self.span_index >= self.dispatch_spans.spans.len) { + self.interface.buffer = &.{}; + self.interface.end = 0; return; } - const old_end = shard_writer.interface.end; - const new_end = self.interface.end; - if (new_end > shard_writer.interface.buffer.len) return std.Io.Writer.Error.WriteFailed; - const chunk = shard_writer.interface.buffer[old_end..new_end]; - if (chunk.len > self.currentSegmentRemaining()) return std.Io.Writer.Error.WriteFailed; + // Only bytes after `window_start` are new reader writes. Bytes before + // it may already be pending in the active shard writer. + const source = self.interface.buffer[self.window_start..self.interface.end]; + if (source.len == 0) return; - shard_writer.interface.end = new_end; - self.segment_offset += chunk.len; - try self.mirrorChunk(segment, chunk); - self.updateShardProgress(writer_index); - } + // The visible alias window may cross many dispatch spans. Walk the + // global cursor and route each subrange to its primary and mirrors. + var consumed: usize = 0; + while (consumed < source.len) { + if (self.span_index >= self.dispatch_spans.spans.len) return std.Io.Writer.Error.WriteFailed; - fn exposeShardWindow(self: *DirectMemoryWriter, writer_index: usize, segment_remaining: usize) void { - const shard_writer = &self.shard_writers[writer_index]; - self.active_writer_index = writer_index; - const shard_remaining = shard_writer.total - (shard_writer.offset + shard_writer.interface.end); - const visible_remaining = @min(shard_remaining, segment_remaining); - const visible_len = @min(shard_writer.interface.buffer.len, shard_writer.interface.end + visible_remaining); - self.interface.buffer = shard_writer.interface.buffer[0..visible_len]; - self.interface.end = shard_writer.interface.end; - } + const span = self.dispatch_spans.spans[self.span_index]; + if (self.byte_cursor < span.start or self.byte_cursor >= span.end) return std.Io.Writer.Error.WriteFailed; - fn mirrorChunk(self: *DirectMemoryWriter, segment: StreamSegment, chunk: []const u8) std.Io.Writer.Error!void { - if (chunk.len == 0) return; - if (segment.mirror_len == 0) return; + const n = @min(source.len - consumed, span.end - self.byte_cursor); + const chunk = source[consumed..][0..n]; - for (self.segment_mirrors[segment.mirror_start .. segment.mirror_start + segment.mirror_len]) |writer_index| { - const shard_writer = &self.shard_writers[writer_index]; - const wrote = try shard_writer.interface.vtable.drain(&shard_writer.interface, &.{chunk}, 1); - if (wrote != chunk.len) return std.Io.Writer.Error.WriteFailed; - self.updateShardProgress(writer_index); + // The first span in the active alias is already in the primary DMA + // buffer. Later ranges use the active buffer as scratch and must be + // copied into their real destination writer. + if (span.primary_writer == self.active_writer_index and consumed == 0) { + self.shard_writers[span.primary_writer].interface.end += n; + } else { + const primary_writer = &self.shard_writers[span.primary_writer]; + if (span.primary_writer == self.active_writer_index) { + @memmove(primary_writer.interface.buffer[primary_writer.interface.end..][0..n], chunk); + primary_writer.interface.end += n; + } else { + try primary_writer.interface.writeAll(chunk); + } + } + if (self.shard_progress) |states| { + states[span.primary_writer].set(states[span.primary_writer].completed + n); + } + + const mirror_writer_end = span.mirror_writer_start + span.mirror_writer_len; + for (self.dispatch_spans.mirror_writers[span.mirror_writer_start..mirror_writer_end]) |mirror_writer_index| { + const mirror_writer = &self.shard_writers[mirror_writer_index]; + if (mirror_writer_index == self.active_writer_index) { + @memmove(mirror_writer.interface.buffer[mirror_writer.interface.end..][0..n], chunk); + mirror_writer.interface.end += n; + } else { + try mirror_writer.interface.writeAll(chunk); + } + if (self.shard_progress) |states| { + states[mirror_writer_index].set(states[mirror_writer_index].completed + n); + } + } + + self.byte_cursor += n; + consumed += n; + if (self.byte_cursor == span.end) { + self.span_index += 1; + } } - } - fn updateShardProgress(self: *DirectMemoryWriter, writer_index: usize) void { - const states = self.shard_progress orelse return; - const state = &states[writer_index]; - const writer = &self.shard_writers[writer_index]; - const completed_bytes = writer.offset + writer.interface.end; - state.node.setCompletedItems(completed_bytes / state.scale); + // All scratch copies are complete now, so full DMA buffers may be + // submitted without invalidating bytes still needed for scatter. + for (self.shard_writers) |*writer| { + if (writer.interface.end == writer.interface.buffer.len) { + try writer.interface.flush(); + } + } + + if (@mod(self.byte_cursor, self.dma_chunk_size) == 0) { + for (self.shard_writers) |*writer| { + try writer.interface.flush(); + } + } + + // The last committed window may finish the tensor exactly. Leave an + // empty public window so any further writes fail through std.Io. + if (self.span_index >= self.dispatch_spans.spans.len) { + self.interface.buffer = &.{}; + self.interface.end = 0; + return; + } + + // Rotate the public alias to the primary writer for the next stream + // span. `window_start` preserves that writer's already-buffered prefix + // so only newly advanced bytes are scattered on the next commit. + const next_span = self.dispatch_spans.spans[self.span_index]; + const next_writer = &self.shard_writers[next_span.primary_writer]; + self.active_writer_index = next_span.primary_writer; + self.window_start = next_writer.interface.end; + + // Publish the next public alias. Its backing memory is the next primary + // shard buffer, but the visible length is allowed to coalesce across + // upcoming spans until a buffer or tensor boundary. + const total = self.dispatch_spans.spans[self.dispatch_spans.spans.len - 1].end; + const buffer_remaining = next_writer.interface.buffer.len - self.window_start; + const chunk_offset = @mod(self.byte_cursor, self.dma_chunk_size); + const chunk_remaining = if (chunk_offset == 0) self.dma_chunk_size else self.dma_chunk_size - chunk_offset; + const tensor_remaining = total - self.byte_cursor; + const visible_remaining = @min(buffer_remaining, @min(chunk_remaining, tensor_remaining)); + if (visible_remaining == 0) return std.Io.Writer.Error.WriteFailed; + + self.interface.buffer = next_writer.interface.buffer[0 .. self.window_start + visible_remaining]; + self.interface.end = self.window_start; } }; @@ -1097,6 +1106,7 @@ pub fn load( const Ctx = struct { allocator: std.mem.Allocator, dma_allocators: []const mem.DmaAllocator, + dma_chunk_size: usize, pinned_buffer_pools: []mem.DynamicBufferPool, io: std.Io, platform: *const Platform, @@ -1113,6 +1123,7 @@ pub fn load( .store = store, .allocator = allocator, .dma_allocators = dma_allocators, + .dma_chunk_size = opts.dma_chunk_size, .pinned_buffer_pools = buffer_pools, .io = io, .shardings = opts.shardings, @@ -1150,6 +1161,7 @@ pub fn load( ctx_.platform, ctx_.pinned_buffer_pools, ctx_.dma_allocators, + ctx_.dma_chunk_size, shape, sharding, ctx_.buffers[i_], @@ -1310,6 +1322,7 @@ const DirectMemoryWriterDeviceTest = struct { platform, pools, dma_allocators, + pool_chunk_size, shape, sharding, &written_buffer, From ec776c53e09784e38f33347c8b80a0a0e0c8f5ef Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 18 Jun 2026 16:17:12 +0200 Subject: [PATCH 519/881] zml/platform: set defaults for ROCm PjRT create options (#579) ROCm was using only 0.75 of available memory. Align with CUDA, this permit us to have Gemma 4 26B A4B on 2xRTX 9700 for example. --- zml/platform.zig | 11 +++++------ zml/testing.zig | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/zml/platform.zig b/zml/platform.zig index d9928721..e61cb811 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -8,12 +8,12 @@ pub const Target = platforms.Platform; const stdx = @import("stdx"); const attention = @import("attention.zig"); +const constants = @import("constants.zig"); const Exe = @import("exe.zig").Exe; const pjrtx = @import("pjrtx.zig"); const profiler_ = @import("profiling/profiler.zig"); const Sharding = @import("Sharding.zig"); const zml = @import("zml.zig"); -const constants = @import("constants.zig"); const log = std.log.scoped(.zml); @@ -644,8 +644,7 @@ pub const CreateOptions = struct { // bump memory fraction from XLA defaults of 75% to 90%. // Even on a 8GB GPU it should leave enough space for the Cuda driver // https://github.com/openxla/xla/blob/3e87afa11a865cf91137522492918ad18bfe5b7c/xla/pjrt/plugin/xla_gpu/xla_gpu_allocator_config.h#L25-L60 - cuda: Cuda = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.90 } } }, - rocm: struct {} = .{}, + xla_gpu: XlaGpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.90 } } }, tpu: struct {} = .{}, neuron: struct {} = .{}, oneapi: struct {} = .{}, @@ -658,7 +657,7 @@ pub const CreateOptions = struct { } }; - pub const Cuda = struct { + pub const XlaGpu = struct { allocator: Allocator = .{ .bfc = .{} }, // TODO support all of https://github.com/openxla/xla/blob/3d31c48c719d331d432132b3e0c2c5ce52650675/xla/pjrt/c/pjrt_c_api_gpu_internal.cc#L76-L86 // visible_devices: []const i64 = &.{}, @@ -682,7 +681,7 @@ pub const CreateOptions = struct { }; }; - fn writeNamedValues(self: Cuda, values: *std.ArrayList(pjrt.NamedValue)) void { + fn writeNamedValues(self: XlaGpu, values: *std.ArrayList(pjrt.NamedValue)) void { switch (self.allocator) { .platform => { values.appendAssumeCapacity(.init(.string, "allocator", "platform")); @@ -710,7 +709,7 @@ pub const CreateOptions = struct { values.shrinkRetainingCapacity(0); switch (target) { .cpu => self.cpu.writeNamedValues(&values), - .cuda => self.cuda.writeNamedValues(&values), + .cuda, .rocm => self.xla_gpu.writeNamedValues(&values), inline else => |t| { stdx.debug.assertComptime(@hasField(CreateOptions, @tagName(t)), "zml.platform.CreateOptions doesn't list target {s}", .{@tagName(t)}); const options = @field(self, @tagName(t)); diff --git a/zml/testing.zig b/zml/testing.zig index 18dbea02..288a883b 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -15,7 +15,7 @@ pub fn env() *const Platform { if (!builtin.is_test) @compileError("Cannot use zml.testing.env outside of a test block"); if (_platform == null) { _platform = Platform.auto(std.heap.c_allocator, std.testing.io, .{ - .cuda = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.85 } } }, + .xla_gpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.85 } } }, }) catch unreachable; } From 56a6a4db642f84be393f32156bc824298f72036f Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Fri, 19 Jun 2026 11:38:58 +0200 Subject: [PATCH 520/881] zml/moe: enable triton moe kernels on ROCm (#580) --- zml/moe/moe.zig | 8 ++++++-- zml/moe/triton.zig | 10 +++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index e24b1caa..2b016866 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -1,9 +1,9 @@ const std = @import("std"); + const zml = @import("../zml.zig"); const stdx = zml.stdx; -pub const triton = @import("triton.zig"); pub const mosaic_tpu = @import("mosaic_tpu.zig"); - +pub const triton = @import("triton.zig"); pub const triton_kernels = @import("triton_kernels/triton_kernels.zig"); pub const ActivationMode = enum { @@ -34,6 +34,10 @@ pub const Backend = enum { } break :b error.UnsupportedComputeCapability; }, + .rocm => switch (weights_dtype) { + .bf16, .f16, .f32 => .triton, + else => error.UnsupportedDataType, + }, .tpu => switch (weights_dtype) { .bf16, .f16, .f32 => .mosaic_tpu, else => error.UnsupportedDataType, diff --git a/zml/moe/triton.zig b/zml/moe/triton.zig index bd7569e5..1f2771cc 100644 --- a/zml/moe/triton.zig +++ b/zml/moe/triton.zig @@ -1,20 +1,17 @@ const std = @import("std"); +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); const stdx = @import("stdx"); const zml = @import("../zml.zig"); const DataType = zml.DataType; const Tensor = zml.Tensor; const Shape = zml.Shape; -const bazel = @import("bazel"); -const bazel_builtin = @import("bazel_builtin"); - const tri = zml.kernel.triton; const DType = tri.DType; - -const kernels = @import("triton_kernels/triton_kernels.zig"); - const toDType = tri.from; +const kernels = @import("triton_kernels/triton_kernels.zig"); const log = std.log.scoped(.moe_triton); @@ -643,7 +640,6 @@ fn fp8ActivationGroupSize(x: Tensor) i64 { fn validateOptions(opts: Options) !void { if (opts.inplace) return error.Unimplemented; - if (opts.activation != .silu and opts.activation != .relu) return error.UnsupportedActivation; if (opts.apply_router_weight_on_input) return error.UnsupportedOption; if (opts.use_fp8_w8a8 or opts.use_int8_w8a8 or opts.use_int8_w8a16 or opts.use_int4_w4a16) return error.UnsupportedQuantization; if (opts.ocp_mx_scheme != null or opts.per_channel_quant) return error.UnsupportedOption; From 2cc6a541b34ff1636feb77cabe904b0f1f15fd11 Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Fri, 19 Jun 2026 14:15:41 +0100 Subject: [PATCH 521/881] zml/build: bump bazel to 8.7.0 (#584) --- .bazelversion | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bazelversion b/.bazelversion index f9c71a52..df5119ec 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -8.5.1 +8.7.0 From 4cd7139477749438d3d3ebc535d060a6c39d0077 Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Fri, 19 Jun 2026 17:30:57 +0100 Subject: [PATCH 522/881] zml/platform: add hermetic oneAPI 2026 runtime deps (#582) Fetch oneAPI runtime libraries from locked DEB packages and wire the oneAPI sandbox packaging to use those repositories. --- MODULE.bazel.lock | 12 +- README.md | 9 +- platforms/oneapi/libpjrt_oneapi.BUILD.bazel | 45 +++- platforms/oneapi/oneapi.bzl | 228 ++++++++++++++------ platforms/oneapi/oneapi.zig | 8 + platforms/oneapi/packages.lock.json | 147 +++++++++++++ platforms/oneapi/zmlxoneapi.zig | 13 +- zml/attention/paged_attention.zig | 17 ++ zml/tensor.zig | 8 +- 9 files changed, 394 insertions(+), 93 deletions(-) create mode 100644 platforms/oneapi/packages.lock.json diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index a296acfb..d2b58fce 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -387,7 +387,7 @@ "moduleExtensions": { "@@aspect_rules_py+//py:extensions.bzl%py_tools": { "general": { - "bzlTransitiveDigest": "GLVIk55uU2jb460gpL1FFbE/tBFqzMxRfta5QXYfjQo=", + "bzlTransitiveDigest": "pI39mvkEpRkqO9fPgwuZp2cFsNe/MqGRNr3gPBzTHaQ=", "usagesDigest": "9BiP3ASHlGl4doGhzXJwK17taNzv/q6x/IZ4Hb6LxNY=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -1076,7 +1076,7 @@ }, "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { "general": { - "bzlTransitiveDigest": "AEsQvKwdstptzt98AO1bJYfoWpPUn2upx53ra6o/Rc8=", + "bzlTransitiveDigest": "oA0OoCaohRv2x/x61/6mFy+2a9+GiuUKGAXLGcQy+hQ=", "usagesDigest": "QI2z8ZUR+mqtbwsf2fLqYdJAkPOHdOV+tF2yVAUgRzw=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -1140,7 +1140,7 @@ }, "@@rules_oci+//oci:extensions.bzl%oci": { "general": { - "bzlTransitiveDigest": "7oR14uSSRXhIDozspnn5ya0JWJlnqcG+JmT5yO1BsLw=", + "bzlTransitiveDigest": "Q7DvSdepUOK0DnFJtYmquDXHZCcR2DoLM9LbkrHSCKQ=", "usagesDigest": "JLbamplho20rviuebatMGqnQnaoj32ewCzLuRVusGoU=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -1434,7 +1434,7 @@ }, "@@rules_python+//python/uv:uv.bzl%uv": { "general": { - "bzlTransitiveDigest": "LM8+g04lV+afYP67YcE7wenTJMrNqvF5u/ZnOctwctY=", + "bzlTransitiveDigest": "IzGWfoGARM6X2a4AM7BN5gheLtDsFeP9Fra2qTod+mA=", "usagesDigest": "Ucfum/vsVnlkhtglTERDLTj+bBeG7dcpbV9h0LP9HOQ=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -1765,7 +1765,7 @@ }, "@@rules_zig+//zig:extensions.bzl%zig": { "general": { - "bzlTransitiveDigest": "u7Psh7szBG3+lbRmEeBBbWlZRHJWhWtxQo/wtRqvnTg=", + "bzlTransitiveDigest": "VGLOg2kVw+STO9R5OrQnLFbv7Vl5TPdtIebIs324IpU=", "usagesDigest": "4xBoe2eIScx2QI0jVieHWXdflezUUDIApwMqS+SWGbw=", "recordedFileInputs": { "@@//bazel/zig_index.json": "2e3b4b83f1add5d3ca12fd05797c04c38af21d86c7fc7c1a9b356560141a4797", @@ -2094,7 +2094,7 @@ }, "@@rules_zig+//zig/zls:extensions.bzl%zls": { "general": { - "bzlTransitiveDigest": "8ZwSSF/7dE4rb0W0cZqil2SfGF1bPK04LTv1/zjtkTM=", + "bzlTransitiveDigest": "ZuceGxE17jROQ+kQkaPqIcK7gMKGb56I97Aq1/Dubdc=", "usagesDigest": "b/ERjMTWazfa8VFWuTc0PTtQQzLDuloWSHlzXj6Zb5w=", "recordedFileInputs": { "@@rules_zig+//zig/zls/private/versions.json": "127803289dfd79b5255db5f74be1d9a2f790f03292d6147009af702d888aba69" diff --git a/README.md b/README.md index dd7bfe69..f35453ff 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ ZML is a production inference stack, purpose-built to decouple AI workloads from Any model, many hardwares, one codebase, peak performance. -Compiled directly to NVIDIA, AMD, TPU, Trainium for peak hardware performance on any accelerator. No rewriting. +Compiled directly to NVIDIA, AMD, INTEL, TPU, Trainium for peak hardware performance on any accelerator. No rewriting. It is built using the [Zig](https://ziglang.org) language, [MLIR](https://mlir.llvm.org), and [Bazel](https://bazel.build). @@ -95,6 +95,7 @@ Append one or more platform flags when compiling or running: - NVIDIA CUDA: `--@zml//platforms:cuda=true` - AMD RoCM: `--@zml//platforms:rocm=true` +- Intel OneAPI: `--@zml//platforms:oneapi=true` - Google TPU: `--@zml//platforms:tpu=true` - AWS Trainium / Inferentia 2: `--@zml//platforms:neuron=true` - Disable CPU compilation: `--@zml//platforms:cpu=false` @@ -111,6 +112,12 @@ Example on ROCm: bazel run //examples/llm --@zml//platforms:rocm=true -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="Write a haiku about Zig" ``` +Example on Intel OneAPI: + +```bash +bazel run //examples/llm --@zml//platforms:cpu=false --@zml//platforms:oneapi=true -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="Write a haiku about Zig" +``` + ## Run Tests ```bash diff --git a/platforms/oneapi/libpjrt_oneapi.BUILD.bazel b/platforms/oneapi/libpjrt_oneapi.BUILD.bazel index 1b28a04f..1dbd9489 100644 --- a/platforms/oneapi/libpjrt_oneapi.BUILD.bazel +++ b/platforms/oneapi/libpjrt_oneapi.BUILD.bazel @@ -1,9 +1,6 @@ load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@zml//bazel:patchelf.bzl", "patchelf") -ONEAPI_UMF_VERSION = "0.10.0" -ONEAPI_UR_VERSION = "0.11.10" - patchelf( name = "libpjrt_oneapi_so", src = "libpjrt_oneapi.so", @@ -18,25 +15,46 @@ patchelf( patchelf( name = "libsycl_so", - src = "@oneapi//:libsycl_so", + src = "@intel-oneapi-compiler-dpcpp-cpp-runtime-2026.0//:libsycl_so", set_rpath = "$ORIGIN", ) -ONEAPI_VERSION = "2025.1" -ONEAPI_BASE_LIB = "{}/lib".format(ONEAPI_VERSION) +patchelf( + name = "libccl_legacy_so", + src = "@intel-oneapi-ccl-2022.0//:libccl_legacy_so", + set_rpath = "$ORIGIN", +) + +ONEAPI_VERSION = "2026.0" +ONEAPI_CCL_LIB = "ccl/2022.0/lib" +ONEAPI_MPI_LIB = "mpi/2021.18/lib" +ONEAPI_MPI_LIBFABRIC_LIB = "mpi/2021.18/opt/mpi/libfabric/lib" +ONEAPI_TCM_VERSION = "1.5" +ONEAPI_UMF_VERSION = "1.1" ONEAPI_COMPILER_LIB = "compiler/{}/lib".format(ONEAPI_VERSION) ONEAPI_MKL_LIB = "mkl/{}/lib".format(ONEAPI_VERSION) +ONEAPI_TCM_LIB = "tcm/{}/lib".format(ONEAPI_TCM_VERSION) +ONEAPI_UMF_LIB = "umf/{}/lib".format(ONEAPI_UMF_VERSION) copy_to_directory( name = "sandbox", srcs = [ ":libpjrt_oneapi_so", ":libsycl_so", + "@intel-oneapi-ccl-2022.0//:ccl_runtime", + "@intel-oneapi-compiler-dpcpp-cpp-runtime-2026.0//:sycl_runtime", + "@intel-oneapi-compiler-shared-runtime-2026.0//:compiler_runtime", + "@intel-oneapi-mkl-core-2026.0//:mkl_core_runtime", + "@intel-oneapi-mkl-sycl-blas-2026.0//:mkl_sycl_blas", + "@intel-oneapi-mkl-sycl-dft-2026.0//:mkl_sycl_dft", + "@intel-oneapi-mkl-sycl-lapack-2026.0//:mkl_sycl_lapack", + "@intel-oneapi-mkl-sycl-rng-2026.0//:mkl_sycl_rng", + "@intel-oneapi-mkl-sycl-sparse-2026.0//:mkl_sycl_sparse", + "@intel-oneapi-mpi-2021.18//:mpi_runtime", + "@intel-oneapi-tcm-1.5//:hwloc", + "@intel-oneapi-umf-1.1//:umf", + "@zlib1g", "@zml//platforms/oneapi:zmlxoneapi", - "@oneapi//:base_runtime", - "@oneapi//:compiler_runtime", - "@oneapi//:mkl_runtime", - "@zero_loader//:all", ], add_directory_to_runfiles = False, allow_overwrites = True, @@ -45,9 +63,14 @@ copy_to_directory( replace_prefixes = { "libpjrt_oneapi_so": "lib", "libsycl_so": "lib", - ONEAPI_BASE_LIB: "lib", + ONEAPI_CCL_LIB: "lib", ONEAPI_COMPILER_LIB: "lib", + ONEAPI_MPI_LIB: "lib", + ONEAPI_MPI_LIBFABRIC_LIB: "lib", ONEAPI_MKL_LIB: "lib", + ONEAPI_TCM_LIB: "lib", + ONEAPI_UMF_LIB: "lib", + "lib/x86_64-linux-gnu": "lib", "platforms/oneapi/libzmlxoneapi": "lib/libzmlxoneapi", }, root_paths = ["."], diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index 9bee53f6..c646354b 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -1,107 +1,205 @@ load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") +load("//bazel:http_deb_archive.bzl", "http_deb_archive") -PJRT_ONEAPI_RELEASE = "manual-2026-05-21T17-13-00Z" -PJRT_ONEAPI_ARTIFACT_SHA256 = "24c518768b5f4994cde22603c7451ce6c53e9d236c3e574e65ca73cd32c6368a" +PJRT_ONEAPI_RELEASE = "manual-2026-06-18T20-14-00Z" +PJRT_ONEAPI_ARTIFACT_SHA256 = "aa030689405686c4bc933183198a90818c0212f2f23e05564b3f8455fe17003c" PJRT_ONEAPI_ARTIFACT_URL = "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-oneapi_linux-amd64.tar.gz".format( release = PJRT_ONEAPI_RELEASE, ) -ONEAPI_VERSION = "2025.1" -ONEAPI_BASE_TOOLKIT_VERSION = "2025.1.3.7" +ONEAPI_VERSION = "2026.0" +ONEAPI_TCM_VERSION = "1.5" +ONEAPI_UMF_VERSION = "1.1" -ONEAPI_BASE_LIB = "{}/lib".format(ONEAPI_VERSION) +ONEAPI_CCL_LIB = "ccl/2022.0/lib" ONEAPI_COMPILER_LIB = "compiler/{}/lib".format(ONEAPI_VERSION) +ONEAPI_MPI_LIB = "mpi/2021.18/lib" +ONEAPI_MPI_LIBFABRIC_LIB = "mpi/2021.18/opt/mpi/libfabric/lib" ONEAPI_MKL_LIB = "mkl/{}/lib".format(ONEAPI_VERSION) +ONEAPI_TCM_LIB = "tcm/{}/lib".format(ONEAPI_TCM_VERSION) +ONEAPI_UMF_LIB = "umf/{}/lib".format(ONEAPI_UMF_VERSION) -ONEAPI_RUNTIME_URL = "https://tensorflow-file-hosting.s3.us-east-1.amazonaws.com/intel-oneapi-base-toolkit-{version}.tar.gz".format( - version = ONEAPI_BASE_TOOLKIT_VERSION, -) -ONEAPI_RUNTIME_SHA256 = "2213104bd122336551aa144512e7ab99e4a84220e77980b5f346edc14ebd458a" -ONEAPI_RUNTIME_STRIP_PREFIX = "oneapi" - -ONEAPI_ZERO_LOADER_URL = "https://tensorflow-file-hosting.s3.us-east-1.amazonaws.com/ze_loader_libs.tar.gz" -ONEAPI_ZERO_LOADER_SHA256 = "71cbfd8ac59e1231f013e827ea8efe6cf5da36fad771da2e75e202423bd6b82e" - -_ONEAPI_BUILD_FILE_CONTENT = """\ +_BUILD_FILE_DEFAULT_VISIBILITY = """\ package(default_visibility = ["//visibility:public"]) +""" +_UBUNTU_STRIP_PREFIX = "./usr" +_ONEAPI_STRIP_PREFIX = "./opt/intel/oneapi" + +def _read_packages(mctx, labels): + ret = {} + for label in labels: + data = json.decode(mctx.read(Label(label))) + for pkg in data["packages"]: + ret.setdefault(pkg["name"], {})[pkg["arch"]] = pkg + return ret + +_UBUNTU_PACKAGES = { + "zlib1g": """ filegroup( - name = "base_runtime", - srcs = [ - "{ONEAPI_BASE_LIB}/libhwloc.so.15", - "{ONEAPI_BASE_LIB}/libumf.so.0", - "{ONEAPI_BASE_LIB}/libur_adapter_level_zero.so.0", - "{ONEAPI_BASE_LIB}/libur_adapter_opencl.so.0", - "{ONEAPI_BASE_LIB}/libur_loader.so.0", - ], -) + name = "zlib1g", + srcs = ["lib/x86_64-linux-gnu/libz.so.1"], +)""", +} +_ONEAPI_PACKAGES = { + "intel-oneapi-ccl-2022.0": """ +filegroup( + name = "ccl_runtime", + srcs = glob( + include = ["{ONEAPI_CCL_LIB}/**"], + exclude = ["{ONEAPI_CCL_LIB}/libccl_legacy.so"], + ), +)""".format(ONEAPI_CCL_LIB = ONEAPI_CCL_LIB), + + "intel-oneapi-mpi-2021.18": """ +filegroup( + name = "mpi_runtime", + srcs = glob([ + "{ONEAPI_MPI_LIB}/libmpi*.so*", + "{ONEAPI_MPI_LIB}/libmpicxx*.so*", + "{ONEAPI_MPI_LIB}/libmpifort*.so*", + "{ONEAPI_MPI_LIB}/mpi/libmpi_ze_hooks.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/libfabric.so*", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/*.so", + ]), +)""".format( + ONEAPI_MPI_LIB = ONEAPI_MPI_LIB, + ONEAPI_MPI_LIBFABRIC_LIB = ONEAPI_MPI_LIBFABRIC_LIB, + ), + + "intel-oneapi-tcm-1.5": """ +filegroup( + name = "hwloc", + srcs = ["{ONEAPI_TCM_LIB}/libhwloc.so.15"], +)""".format(ONEAPI_TCM_LIB = ONEAPI_TCM_LIB), + + "intel-oneapi-umf-1.1": """ +filegroup( + name = "umf", + srcs = ["{ONEAPI_UMF_LIB}/libumf.so.1"], +)""".format(ONEAPI_UMF_LIB = ONEAPI_UMF_LIB), + + "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.0": """ +filegroup( + name = "libsycl_so", + srcs = ["{ONEAPI_COMPILER_LIB}/libsycl.so.9"], +) +filegroup( + name = "sycl_runtime", + srcs = glob(["{ONEAPI_COMPILER_LIB}/*.spv"]) + [ + "{ONEAPI_COMPILER_LIB}/libur_adapter_level_zero.so.0", + "{ONEAPI_COMPILER_LIB}/libur_adapter_level_zero_v2.so.0", + "{ONEAPI_COMPILER_LIB}/libur_adapter_opencl.so.0", + "{ONEAPI_COMPILER_LIB}/libur_loader.so.0", + ], +)""".format(ONEAPI_COMPILER_LIB = ONEAPI_COMPILER_LIB), + + "intel-oneapi-compiler-shared-runtime-2026.0": """ filegroup( name = "compiler_runtime", - srcs = glob(["{ONEAPI_COMPILER_LIB}/*.spv"]) + [ - "{ONEAPI_COMPILER_LIB}/libOpenCL.so.1", + srcs = glob(["{ONEAPI_COMPILER_LIB}/libOpenCL.so*"]) + [ "{ONEAPI_COMPILER_LIB}/libimf.so", "{ONEAPI_COMPILER_LIB}/libintlc.so.5", "{ONEAPI_COMPILER_LIB}/libirc.so", "{ONEAPI_COMPILER_LIB}/libirng.so", "{ONEAPI_COMPILER_LIB}/libsvml.so", ], -) +)""".format(ONEAPI_COMPILER_LIB = ONEAPI_COMPILER_LIB), + "intel-oneapi-mkl-core-2026.0": """ filegroup( - name = "libsycl_so", - srcs = ["{ONEAPI_COMPILER_LIB}/libsycl.so.8"] -) - -filegroup( - name = "mkl_runtime", + name = "mkl_core_runtime", srcs = [ - "{ONEAPI_MKL_LIB}/libmkl_core.so.2", - "{ONEAPI_MKL_LIB}/libmkl_intel_ilp64.so.2", - "{ONEAPI_MKL_LIB}/libmkl_sequential.so.2", - "{ONEAPI_MKL_LIB}/libmkl_sycl_blas.so.5", + "{ONEAPI_MKL_LIB}/libmkl_core.so.3", + "{ONEAPI_MKL_LIB}/libmkl_intel_ilp64.so.3", + "{ONEAPI_MKL_LIB}/libmkl_sequential.so.3", ], -) -""".format( - ONEAPI_BASE_LIB = ONEAPI_BASE_LIB, - ONEAPI_COMPILER_LIB = ONEAPI_COMPILER_LIB, - ONEAPI_MKL_LIB = ONEAPI_MKL_LIB, -) +)""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), -_ZERO_LOADER_BUILD_FILE_CONTENT = """\ + "intel-oneapi-mkl-sycl-blas-2026.0": """ filegroup( - name = "all", - srcs = glob(["lib/**"]), - visibility = ["//visibility:public"], -) -""" + name = "mkl_sycl_blas", + srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_blas.so.6"], +)""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), + + "intel-oneapi-mkl-sycl-dft-2026.0": """ +filegroup( + name = "mkl_sycl_dft", + srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_dft.so.6"], +)""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), + + "intel-oneapi-mkl-sycl-lapack-2026.0": """ +filegroup( + name = "mkl_sycl_lapack", + srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_lapack.so.6"], +)""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), + + "intel-oneapi-mkl-sycl-rng-2026.0": """ +filegroup( + name = "mkl_sycl_rng", + srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_rng.so.6"], +)""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), + + "intel-oneapi-mkl-sycl-sparse-2026.0": """ +filegroup( + name = "mkl_sycl_sparse", + srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_sparse.so.6"], +)""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), +} + +_ROOT_MODULE_DIRECT_DEPS = [ + "intel-oneapi-ccl-2022.0", + "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.0", + "intel-oneapi-compiler-shared-runtime-2026.0", + "intel-oneapi-mkl-core-2026.0", + "intel-oneapi-mkl-sycl-blas-2026.0", + "intel-oneapi-mkl-sycl-dft-2026.0", + "intel-oneapi-mkl-sycl-lapack-2026.0", + "intel-oneapi-mkl-sycl-rng-2026.0", + "intel-oneapi-mkl-sycl-sparse-2026.0", + "intel-oneapi-mpi-2021.18", + "intel-oneapi-tcm-1.5", + "intel-oneapi-umf-1.1", + "libpjrt_oneapi", + "zlib1g", +] def _oneapi_impl(mctx): + loaded_packages = _read_packages(mctx, [ + "@zml//platforms/oneapi:packages.lock.json", + ]) + http_archive( name = "libpjrt_oneapi", build_file = "libpjrt_oneapi.BUILD.bazel", sha256 = PJRT_ONEAPI_ARTIFACT_SHA256, url = PJRT_ONEAPI_ARTIFACT_URL, ) - http_archive( - name = "oneapi", - build_file_content = _ONEAPI_BUILD_FILE_CONTENT, - sha256 = ONEAPI_RUNTIME_SHA256, - strip_prefix = ONEAPI_RUNTIME_STRIP_PREFIX, - url = ONEAPI_RUNTIME_URL, - ) - http_archive( - name = "zero_loader", - build_file_content = _ZERO_LOADER_BUILD_FILE_CONTENT, - sha256 = ONEAPI_ZERO_LOADER_SHA256, - url = ONEAPI_ZERO_LOADER_URL, - ) + + for pkg_name, build_file_content in _UBUNTU_PACKAGES.items(): + pkg = loaded_packages[pkg_name]["amd64"] + http_deb_archive( + name = pkg_name, + urls = pkg["urls"], + sha256 = pkg["sha256"], + strip_prefix = _UBUNTU_STRIP_PREFIX, + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, + ) + + for pkg_name, build_file_content in _ONEAPI_PACKAGES.items(): + pkg = loaded_packages[pkg_name]["amd64"] + http_deb_archive( + name = pkg_name, + urls = pkg["urls"], + sha256 = pkg["sha256"], + strip_prefix = _ONEAPI_STRIP_PREFIX, + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, + ) return mctx.extension_metadata( reproducible = True, - root_module_direct_deps = [ - "libpjrt_oneapi", - ], + root_module_direct_deps = _ROOT_MODULE_DIRECT_DEPS, root_module_direct_dev_deps = [], ) diff --git a/platforms/oneapi/oneapi.zig b/platforms/oneapi/oneapi.zig index c81b6356..d338bcac 100644 --- a/platforms/oneapi/oneapi.zig +++ b/platforms/oneapi/oneapi.zig @@ -27,6 +27,12 @@ fn hasOneApiDevices(io: std.Io) bool { return false; } +fn setupOneAPIEnv() void { + _ = c.setenv("CCL_LOG_LEVEL", c.getenv("CCL_LOG_LEVEL") orelse "error", 1); + _ = c.setenv("CCL_ATL_TRANSPORT", c.getenv("CCL_ATL_TRANSPORT") orelse "ofi", 1); + _ = c.setenv("CCL_TOPO_P2P_ACCESS", c.getenv("CCL_ATL_TRANSPORT") orelse "1", 1); +} + pub fn load(_: std.mem.Allocator, io: std.Io) !*const pjrt.Api { if (comptime !isEnabled()) { return error.Unavailable; @@ -48,6 +54,8 @@ pub fn load(_: std.mem.Allocator, io: std.Io) !*const pjrt.Api { return error.FileNotFound; }; + setupOneAPIEnv(); + return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_oneapi.so" }); diff --git a/platforms/oneapi/packages.lock.json b/platforms/oneapi/packages.lock.json new file mode 100644 index 00000000..68b56279 --- /dev/null +++ b/platforms/oneapi/packages.lock.json @@ -0,0 +1,147 @@ +{ + "packages": [ + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-ccl-2022.0_2022.0.0-49301_amd64", + "name": "intel-oneapi-ccl-2022.0", + "sha256": "1003034feabacc032a6054b4667960f7da82df378fb7a331956bfc8c106a4d60", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-ccl-2022.0-2022.0.0-49301_amd64.deb" + ], + "version": "2022.0.0-49301" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-mpi-2021.18_2021.18.0-745_amd64", + "name": "intel-oneapi-mpi-2021.18", + "sha256": "0eed61a43588fcaec64c7b09f6bd809c5227008e501d0e1231488fbd21eac51b", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mpi-2021.18-2021.18.0-745_amd64.deb" + ], + "version": "2021.18.0-745" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.0_2026.0.0-947_amd64", + "name": "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.0", + "sha256": "b7db83a2c8ad06ab5f080fb5ec7c5b004353157f92c0ddb9af3ece0422d9a249", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-compiler-dpcpp-cpp-runtime-2026.0-2026.0.0-947_amd64.deb" + ], + "version": "2026.0.0-947" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-compiler-shared-runtime-2026.0_2026.0.0-947_amd64", + "name": "intel-oneapi-compiler-shared-runtime-2026.0", + "sha256": "0357c84d73cee0af8d4c682bdb290908bf842885e8c24936f3e7e58159e9745e", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-compiler-shared-runtime-2026.0-2026.0.0-947_amd64.deb" + ], + "version": "2026.0.0-947" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-mkl-core-2026.0_2026.0.0-908_amd64", + "name": "intel-oneapi-mkl-core-2026.0", + "sha256": "3cc588084e9820af237c90dba4c1546e50f2d7e02a15d0072504ed4ccf764ec0", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-core-2026.0-2026.0.0-908_amd64.deb" + ], + "version": "2026.0.0-908" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-mkl-sycl-blas-2026.0_2026.0.0-908_amd64", + "name": "intel-oneapi-mkl-sycl-blas-2026.0", + "sha256": "b32078f1805ab126f78333276bb860f288d3a2785dc101b4b9df499cd35fa76d", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-blas-2026.0-2026.0.0-908_amd64.deb" + ], + "version": "2026.0.0-908" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-mkl-sycl-dft-2026.0_2026.0.0-908_amd64", + "name": "intel-oneapi-mkl-sycl-dft-2026.0", + "sha256": "e4168ed1fce35e19552125fdc27bce0820e7348eb42842f4373f05f7fc5f3635", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-dft-2026.0-2026.0.0-908_amd64.deb" + ], + "version": "2026.0.0-908" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-mkl-sycl-lapack-2026.0_2026.0.0-908_amd64", + "name": "intel-oneapi-mkl-sycl-lapack-2026.0", + "sha256": "5a31d222fa527c96bd7aa8bbc7819a11417cc7e6ba38803b55e224541aab68c3", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-lapack-2026.0-2026.0.0-908_amd64.deb" + ], + "version": "2026.0.0-908" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-mkl-sycl-rng-2026.0_2026.0.0-908_amd64", + "name": "intel-oneapi-mkl-sycl-rng-2026.0", + "sha256": "ae6910cd6226a9a380df846849288bf07c5eda0387896b90820cf904a29ee4a0", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-rng-2026.0-2026.0.0-908_amd64.deb" + ], + "version": "2026.0.0-908" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-mkl-sycl-sparse-2026.0_2026.0.0-908_amd64", + "name": "intel-oneapi-mkl-sycl-sparse-2026.0", + "sha256": "1552782efcad141280666f05db5983cfcec4e094a57672170786c7b5e1ff5f9c", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-sparse-2026.0-2026.0.0-908_amd64.deb" + ], + "version": "2026.0.0-908" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-tcm-1.5_1.5.0-489_amd64", + "name": "intel-oneapi-tcm-1.5", + "sha256": "7122ed7748d54342b24f0a2838a64f24b54e430ac4d06b2961a8eefee4ffb089", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-tcm-1.5-1.5.0-489_amd64.deb" + ], + "version": "1.5.0-489" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-umf-1.1_1.1.0-340_amd64", + "name": "intel-oneapi-umf-1.1", + "sha256": "8705a997042c76f67228b4135184754e1cd3f8f68b6629bcb261c1d5b958ce78", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-umf-1.1-1.1.0-340_amd64.deb" + ], + "version": "1.1.0-340" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "zlib1g_1-1.3.dfsg-3.1ubuntu2.1_amd64", + "name": "zlib1g", + "sha256": "7074b6a2f6367a10d280c00a1cb02e74277709180bab4f2491a2f355ab2d6c20", + "urls": [ + "https://archive.ubuntu.com/ubuntu/pool/main/z/zlib/zlib1g_1.3.dfsg-3.1ubuntu2.1_amd64.deb" + ], + "version": "1:1.3.dfsg-3.1ubuntu2.1" + } + ] +} diff --git a/platforms/oneapi/zmlxoneapi.zig b/platforms/oneapi/zmlxoneapi.zig index e5d94e10..baa57408 100644 --- a/platforms/oneapi/zmlxoneapi.zig +++ b/platforms/oneapi/zmlxoneapi.zig @@ -5,15 +5,16 @@ const stdx = @import("stdx"); pub export fn zmlxoneapi_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque { const replacements: std.StaticStringMap([:0]const u8) = .initComptime(.{ .{ "libOpenCL.so", "libOpenCL.so.1" }, - .{ "libumf.so", "libumf.so.0" }, + .{ "libumf.so", "libumf.so.1" }, .{ "libur_loader.so", "libur_loader.so.0" }, .{ "libur_adapter_level_zero.so", "libur_adapter_level_zero.so.0" }, + .{ "libur_adapter_level_zero_v2.so", "libur_adapter_level_zero_v2.so.0" }, .{ "libur_adapter_opencl.so", "libur_adapter_opencl.so.0" }, - .{ "libsycl.so", "libsycl.so.8" }, - .{ "libmkl_core.so", "libmkl_core.so.2" }, - .{ "libmkl_intel_ilp64.so", "libmkl_intel_ilp64.so.2" }, - .{ "libmkl_sequential.so", "libmkl_sequential.so.2" }, - .{ "libmkl_sycl_blas.so", "libmkl_sycl_blas.so.5" }, + .{ "libsycl.so", "libsycl.so.9" }, + .{ "libmkl_core.so", "libmkl_core.so.3" }, + .{ "libmkl_intel_ilp64.so", "libmkl_intel_ilp64.so.3" }, + .{ "libmkl_sequential.so", "libmkl_sequential.so.3" }, + .{ "libmkl_sycl_blas.so", "libmkl_sycl_blas.so.6" }, }); var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 603dbd1b..ff7f2ac7 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -19,6 +19,7 @@ pub const Backend = enum { return switch (platform.target) { .cuda => .triton, .rocm => .triton, + .oneapi => .triton, .tpu => .mosaic_tpu, else => stdx.debug.panic("Paged attention is not supported on {s} yet", .{@tagName(platform.target)}), }; @@ -234,3 +235,19 @@ test "Backend.auto selects mosaic_tpu on TPU" { try std.testing.expectEqual(Backend.mosaic_tpu, Backend.auto(&platform)); } + +test "Backend.auto selects triton on oneAPI" { + const platform: zml.Platform = .{ + .arena = undefined, + .target = .oneapi, + .pjrt_api = undefined, + .pjrt_client = undefined, + .devices = &.{}, + .memories = &.{}, + .physical_mesh = undefined, + .replicated_sharding = undefined, + .shardings = .empty, + }; + + try std.testing.expectEqual(Backend.triton, Backend.auto(&platform)); +} diff --git a/zml/tensor.zig b/zml/tensor.zig index 750a2c8f..0b34b371 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -189,8 +189,8 @@ pub const Tensor = struct { pub fn toMemory(self: Tensor, kind: Memory.Kind) Tensor { const ctx = CompilationContext.current(); switch (ctx.platform.target) { - .cpu, .neuron, .oneapi => return self, - .cuda, .rocm, .tpu => {}, + .cpu, .neuron => return self, + .cuda, .rocm, .tpu, .oneapi => {}, } const frontend_attributes: *const mlir.Attribute = .dict(ctx.mlir_ctx, &.{ @@ -224,8 +224,8 @@ pub const Tensor = struct { pub fn onMemory(self: Tensor, kind: Memory.Kind) Tensor { const ctx = CompilationContext.current(); switch (ctx.platform.target) { - .cpu, .neuron, .oneapi => return self, - .cuda, .rocm, .tpu => {}, + .cpu, .neuron => return self, + .cuda, .rocm, .tpu, .oneapi => {}, } if (ctx.currentScope().id_to_argument.get(self.id) == null) { From 984b60b75e0ccce5bd1b91235781f773c5be9474 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Sat, 20 Jun 2026 17:58:18 +0200 Subject: [PATCH 523/881] zml: various QoL improvements (#585) These are QoL improvements to problems I stumbled upon while working on #586 --- zml/mem.zig | 1 + zml/meta.zig | 1 + zml/module.zig | 4 +++- zml/safetensors.zig | 12 +++++++++--- zml/shape.zig | 4 ++-- 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/zml/mem.zig b/zml/mem.zig index 4b7b236f..c0a967a6 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -329,6 +329,7 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu const ModelBufferized = Bufferized(Model); if (ModelBufferized == Buffer) { + bufferized_._shards = .empty; return; } diff --git a/zml/meta.zig b/zml/meta.zig index 7c8a7c62..4f9402ee 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -509,6 +509,7 @@ pub fn visit(comptime callback: anytype, ctx: FnParam(callback, 0), v: anytype) .@"union" => switch (v.*) { inline else => |*v_field| if (can_error) try visit(callback, ctx, v_field) else visit(callback, ctx, v_field), }, + .pointer => if (can_error) try visit(callback, ctx, v.*) else visit(callback, ctx, v.*), else => stdx.debug.compileError("zml.meta.visit({any}) doesn't support visiting pointer type {any}", .{ @TypeOf(callback), ChildTypeV }), }, .slice => { diff --git a/zml/module.zig b/zml/module.zig index b1440eb4..22014be6 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -455,7 +455,9 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { const mlir_type = mlirx.Type.rankedTensor(ctx_.compilation_context.mlir_ctx, tensor.shape()); _ = ctx_.compilation_context.currentScope().block.addArgument(mlir_type, .unknown(ctx_.compilation_context.mlir_ctx)); - ctx_.compilation_context.currentScope().id_to_argument.put(ctx_.compilation_context.currentScope().arena.allocator(), tensor.id, ctx_.current_argument_id) catch unreachable; + const gop = ctx_.compilation_context.currentScope().id_to_argument.getOrPut(ctx_.compilation_context.currentScope().arena.allocator(), tensor.id) catch unreachable; + if (gop.found_existing) std.debug.panic("Tensor with id {} has already been used once as an argument", .{tensor.id}); + gop.value_ptr.* = ctx_.current_argument_id; ctx_.current_argument_id += 1; } }.cb, &context, &args); diff --git a/zml/safetensors.zig b/zml/safetensors.zig index 377e8124..2b36ce78 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -74,13 +74,15 @@ pub const TensorReader = struct { io: std.Io, interface: std.Io.Reader, + pub const InitOpts = struct { + alignment: ?std.mem.Alignment = null, + }; + pub fn init( io: std.Io, tensor: Tensor, buffer: []u8, - opts: struct { - alignment: ?std.mem.Alignment = null, - }, + opts: InitOpts, ) !TensorReader { const file = try std.Io.Dir.openFile(.cwd(), io, tensor.file_uri, .{ .mode = .read_only }); errdefer file.close(io); @@ -227,6 +229,10 @@ pub const Tensor = struct { return self.shape.byteSize(); } + pub fn reader(self: Tensor, io: std.Io, buffer: []u8, opts: TensorReader.InitOpts) !TensorReader { + return TensorReader.init(io, self, buffer, opts); + } + pub fn format(self: Tensor, writer: *std.Io.Writer) !void { try writer.print("Tensor(name={s} shape={f} size={d}, offset={d}, file_uri={s})", .{ self.name, diff --git a/zml/shape.zig b/zml/shape.zig index 8de83d95..467f8319 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -432,7 +432,7 @@ pub const Shape = struct { stdx.debug.compileError("axes expects an int-tuple or a tuple of enum literal, got {}", .{T}); } - fn axisFromInt(self: Shape, a: isize) u3 { + pub fn axisFromInt(self: Shape, a: isize) u3 { const rk: i8 = self.rank(); if (a < -rk or a > rk) { stdx.debug.panic("Tensor {f} doesn't have dimension: {d}", .{ self, a }); @@ -822,7 +822,7 @@ pub const Shape = struct { return res; } - stdx.debug.compileError("Expected a tuple of enum literals eg: .{ .a, .b, .c } got: {any}", .{@TypeOf(tagz)}); + stdx.debug.compileError("Expected a tuple of enum literals eg: .{{ .a, .b, .c }} got: {any}", .{@TypeOf(tagz)}); } test withTags { From 6ca94f34bd1b96d59729a79c0ebdd85ffcc180e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20Gauti=C3=A9?= <51361266+tgautie@users.noreply.github.com> Date: Mon, 22 Jun 2026 15:25:02 +0200 Subject: [PATCH 524/881] attention: support `is_causal` option in Triton and Mosaic paged-attn kernels (#587) We add suport for `is_causal` option in Triton and Mosaic paged-attention kernels, by adapting the mask-generation logic --- platforms/tpu/ragged_paged.zig | 20 +++++--- zml/attention/tpu_attention.zig | 7 ++- zml/attention/triton_attention.zig | 5 +- .../triton_kernels/unified_attention.zig | 51 ++++++++++++------- 4 files changed, 55 insertions(+), 28 deletions(-) diff --git a/platforms/tpu/ragged_paged.zig b/platforms/tpu/ragged_paged.zig index c8fdf9d5..2b1009a8 100644 --- a/platforms/tpu/ragged_paged.zig +++ b/platforms/tpu/ragged_paged.zig @@ -32,9 +32,12 @@ pub const Cfg = struct { k_scale: ?f32 = null, v_scale: ?f32 = null, - /// Sliding-window mask added on top of the causal mask: any `(row, col)` - /// with `row - sliding_window >= col` is masked out. Mirrors - /// `kernel.py:391-393`. + /// Whether to mask keys beyond the current query row. + is_causal: bool = true, + + /// Sliding-window mask added on top of the validity/causal mask: any + /// `(row, col)` with `row - sliding_window >= col` is masked out. Mirrors + /// `kernel.py:391-393` in causal mode. sliding_window: ?i64 = null, vmem_limit_bytes: ?i64 = null, @@ -395,11 +398,16 @@ fn flashAttention( k.broadcastTo(kv_len_start, &.{ flash_q, flash_k }), k.iota(&.{ flash_q, flash_k }, .i32, &.{1}), ); - var causal_mask = k.cmpi(.slt, row_ids, col_ids); + var score_mask = if (cfg.is_causal) + k.cmpi(.slt, row_ids, col_ids) + else + // `row_ids < row_ids` is an all-false mask with the same bool shape + // as the causal mask; sliding-window masking can still be OR-ed below. + k.cmpi(.slt, row_ids, row_ids); if (cfg.sliding_window) |sw| { const sw_splat = k.splat(@as(i32, @intCast(sw)), &.{ flash_q, flash_k }, .i32); const window_mask = k.cmpi(.sge, k.subi(row_ids, sw_splat), col_ids); - causal_mask = k.ori(causal_mask, window_mask); + score_mask = k.ori(score_mask, window_mask); } if (cfg.soft_cap) |sc| { const sc_splat = k.splat(sc, &.{ flash_q, flash_k }, .f32); @@ -407,7 +415,7 @@ fn flashAttention( } const qk_masked = k.addf( qk, - k.select(causal_mask, k.splat(cfg.mask_value, &.{ flash_q, flash_k }, .f32), k.zeros(&.{ flash_q, flash_k }, .f32)), + k.select(score_mask, k.splat(cfg.mask_value, &.{ flash_q, flash_k }, .f32), k.zeros(&.{ flash_q, flash_k }, .f32)), ); // jnp.max → maximumf (NaN-propagating). diff --git a/zml/attention/tpu_attention.zig b/zml/attention/tpu_attention.zig index 0ce40058..ea3b7eb6 100644 --- a/zml/attention/tpu_attention.zig +++ b/zml/attention/tpu_attention.zig @@ -240,13 +240,12 @@ pub const mosaic_tpu = struct { .q_dtype = cfgDtype(q_shape.dtype()), .kv_dtype = cfgDtype(kv_pages_shape.dtype()), .sm_scale = opts.scale orelse @floatCast(1.0 / @sqrt(logical_head_dim)), + .is_causal = opts.is_causal, .sliding_window = if (opts.sliding_window < 0) null else @intCast(opts.sliding_window), }; } pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, kv_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { - stdx.debug.assert(opts.is_causal, "mosaic_tpu ragged paged attention currently only supports causal attention", .{}); - const prepared = prepareInputs(parameters, q, kv_cache); const q_out = zml.ops.manualComputation( @@ -328,9 +327,13 @@ test "mosaic_tpu cfg uses kernel head dim and logical scale" { const cfg = mosaic_tpu.buildCfg(prepared_inputs, parameters, .{}); try std.testing.expectEqual(@as(i64, 128), cfg.head_dim); + try std.testing.expect(cfg.is_causal); try std.testing.expectApproxEqAbs( @as(f32, 1.0 / @sqrt(@as(f32, 64.0))), cfg.sm_scale, 0.000001, ); + + const non_causal_cfg = mosaic_tpu.buildCfg(prepared_inputs, parameters, .{ .is_causal = false }); + try std.testing.expect(!non_causal_cfg.is_causal); } diff --git a/zml/attention/triton_attention.zig b/zml/attention/triton_attention.zig index 2bfa6d4b..0acf6fea 100644 --- a/zml/attention/triton_attention.zig +++ b/zml/attention/triton_attention.zig @@ -293,7 +293,6 @@ pub const paged = struct { } pub fn pagedAttention2d(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, paged_attention_opts: PagedAttentionOptions) zml.Tensor { - _ = opts; const config = select2dConfig(paged_attention_opts); const kernel_config: kernels.KernelUnifiedAttention2dPtr.Config = .{ @@ -315,6 +314,7 @@ pub const paged = struct { .block_m = @intCast(config.block_m), .use_fp8 = false, .all_decode = paged_attention_opts.all_decode, + .is_causal = opts.is_causal, }; log.debug("pagedAttention2d config: {any}", .{kernel_config}); @@ -379,8 +379,6 @@ pub const paged = struct { } pub fn pagedAttention3d(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, paged_attention_opts: PagedAttentionOptions) zml.Tensor { - _ = opts; - const config = select3dConfig(paged_attention_opts); const head_size_padded: i64 = @intCast(std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim)); @@ -402,6 +400,7 @@ pub const paged = struct { .block_m = @intCast(config.attention.block_m), .num_segments_per_seq = @intCast(config.attention.num_segments_per_seq), .all_decode = paged_attention_opts.all_decode, + .is_causal = opts.is_causal, }; log.debug("pagedAttention3d attention config: {any}", .{attn_kernel_config}); diff --git a/zml/attention/triton_kernels/unified_attention.zig b/zml/attention/triton_kernels/unified_attention.zig index 1353e1d9..f59f44db 100644 --- a/zml/attention/triton_kernels/unified_attention.zig +++ b/zml/attention/triton_kernels/unified_attention.zig @@ -48,6 +48,7 @@ pub const KernelUnifiedAttention2dPtr = struct { fp8_min: f32 = FP8_E4M3_MIN, fp8_max: f32 = FP8_E4M3_MAX, all_decode: bool = false, + is_causal: bool = true, }; pub const Kernel = tri.Kernel(Config, .{ @@ -178,6 +179,7 @@ pub const KernelUnifiedAttention3dPtr = struct { block_m: i64, num_segments_per_seq: i64, all_decode: bool = false, + is_causal: bool = true, }; pub const Kernel = tri.Kernel(Config, .{ @@ -547,10 +549,14 @@ fn kernelUnifiedAttention2d( else qq_bias_ptr.splatTo(&.{BLOCK_M}); - // max_seq_prefix_len = context_len + q_block_local_idx*BLOCK_Q + (BLOCK_M-1)/NQ_PER_KV + 1 + // In causal mode, each query block only scans keys up to the last query + // represented in this block. In non-causal mode, scan the whole sequence + // so draft rows can attend to all rows written for the same sequence. const pad_term: i32 = @intCast(@divTrunc(BLOCK_M - 1, NUM_QUERIES_PER_KV) + 1); - const max_prefix_raw = context_len.add(q_local_bq).add(pad_term); - const max_seq_prefix_len = max_prefix_raw.minimum(seq_len); + const max_seq_prefix_len = if (config.is_causal) b: { + const max_prefix_raw = context_len.add(q_local_bq).add(pad_term); + break :b max_prefix_raw.minimum(seq_len); + } else seq_len; const num_tiles = cdivFn(max_seq_prefix_len, @as(i32, @intCast(TILE_SIZE))); @@ -559,12 +565,17 @@ fn kernelUnifiedAttention2d( var tile_end: Value = num_tiles; if (SLIDING_WINDOW > 0) { const qpos_lo = q_local_bq; - const qpos_hi_raw = qpos_lo.add(pad_term - 1); - const qpos_hi = qpos_hi_raw.minimum(cur_batch_query_len.sub(1)); - const first_allowed_key = context_len.add(qpos_lo).sub(@as(i32, @intCast(SLIDING_WINDOW - 1))); - const last_allowed_key = context_len.add(qpos_hi); - tile_start = first_allowed_key.div(@as(i32, @intCast(TILE_SIZE))).maximum(0); - tile_end = last_allowed_key.div(@as(i32, @intCast(TILE_SIZE))).add(1).minimum(num_tiles); + if (config.is_causal) { + const qpos_hi_raw = qpos_lo.add(pad_term - 1); + const qpos_hi = qpos_hi_raw.minimum(cur_batch_query_len.sub(1)); + const first_allowed_key = context_len.add(qpos_lo).sub(@as(i32, @intCast(SLIDING_WINDOW - 1))); + const last_allowed_key = context_len.add(qpos_hi); + tile_start = first_allowed_key.div(@as(i32, @intCast(TILE_SIZE))).maximum(0); + tile_end = last_allowed_key.div(@as(i32, @intCast(TILE_SIZE))).add(1).minimum(num_tiles); + } else { + const first_allowed_key = context_len.add(qpos_lo).sub(@as(i32, @intCast(SLIDING_WINDOW - 1))); + tile_start = first_allowed_key.div(@as(i32, @intCast(TILE_SIZE))).maximum(0); + } } const kv_cache_mod: ttir.CacheModifier = @@ -626,11 +637,13 @@ fn kernelUnifiedAttention2d( S = applySoftcap(k, S, softcap).mul(RCP_LN2); } - // seq_mask = seq_offset[None, :] < context_len + query_pos[:, None] + 1 const ql_2d_i32 = query_pos.expandDims(1); const so_2d = seq_offset.expandDims(0); - const rhs = ql_2d_i32.add(context_len).add(1); - const seq_mask = so_2d.lt(rhs); + const seq_mask = if (config.is_causal) b: { + // seq_offset[None, :] < context_len + query_pos[:, None] + 1 + const rhs = ql_2d_i32.add(context_len).add(1); + break :b so_2d.lt(rhs); + } else k.broadcastTo(tile_mask.expandDims(0), &.{ BLOCK_M, TILE_SIZE }); // S = tl.where(qmask_1 & qmask_0 & seq_mask, S, -inf) const qm0_2d = query_mask_0.expandDims(1); @@ -857,8 +870,10 @@ fn kernelUnifiedAttention3d( qq_bias_ptr.splatTo(&.{BLOCK_M}); const pad_term: i32 = @intCast(@divTrunc(BLOCK_M - 1, NUM_QUERIES_PER_KV) + 1); - const max_prefix_raw = context_len.add(q_local_bq).add(pad_term); - const max_seq_prefix_len = max_prefix_raw.minimum(seq_len); + const max_seq_prefix_len = if (config.is_causal) b: { + const max_prefix_raw = context_len.add(q_local_bq).add(pad_term); + break :b max_prefix_raw.minimum(seq_len); + } else seq_len; const num_tiles = cdivFn(max_seq_prefix_len, @as(i32, @intCast(TILE_SIZE))); const segm_tile_lo = segm_idx.mul(tiles_per_segment); @@ -913,11 +928,13 @@ fn kernelUnifiedAttention3d( }); const V = dequantKv(V_load, v_scale, isFp8(config.kv_dtype), config.q_dtype); - // seq_mask = seq_offset[None, :] < context_len + query_pos[:, None] + 1 const ql_2d_i32 = query_pos.expandDims(1); const so_2d = seq_offset.expandDims(0); - const rhs = context_len.add(ql_2d_i32).add(1); - const seq_mask = so_2d.lt(rhs); + const seq_mask = if (config.is_causal) b: { + // seq_offset[None, :] < context_len + query_pos[:, None] + 1 + const rhs = context_len.add(ql_2d_i32).add(1); + break :b so_2d.lt(rhs); + } else k.broadcastTo(tile_mask.expandDims(0), &.{ BLOCK_M, TILE_SIZE }); const acc_zero = k.zeros(&.{ BLOCK_M, TILE_SIZE }, .f32); const qk = k.dot(Q, K, acc_zero); From 7c3b60202d67b3aa8e9c76760d46a3a50fcf9592 Mon Sep 17 00:00:00 2001 From: Louis Lechevalier Date: Mon, 22 Jun 2026 18:52:35 +0200 Subject: [PATCH 525/881] zml/sharding : introduce shardableDim api (#583) This PR introduces shardableDim helper in Sharding to handle the case where : (dimension % number of sharding axes) != 0 Useful to stutter and shard correctly kv heads dim for example. --------- Co-authored-by: Hugo Mano --- examples/llm/models/qwen3_5/inference.zig | 3 +- examples/llm/models/qwen3_5/model.zig | 86 +++++++++--------- examples/llm/models/qwen3_5_moe/inference.zig | 3 +- examples/llm/models/qwen3_5_moe/model.zig | 90 +++++++++---------- zml/Sharding.zig | 36 +++++++- zml/moe/mosaic_tpu.zig | 12 ++- zml/moe/mosaic_tpu_kernels/gmm_ep.zig | 4 + zml/tensor.zig | 7 +- 8 files changed, 142 insertions(+), 99 deletions(-) diff --git a/examples/llm/models/qwen3_5/inference.zig b/examples/llm/models/qwen3_5/inference.zig index b856feee..1ad962fe 100644 --- a/examples/llm/models/qwen3_5/inference.zig +++ b/examples/llm/models/qwen3_5/inference.zig @@ -19,12 +19,11 @@ pub const CompilationParameters = struct { pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, shardings: common.Shardings) CompilationParameters { const dtype = mdl.text_model.embed_tokens.weight.dtype(); - const model_partitions = shardings.model.numPartitionsForLogicalAxis(.model); return .{ .prefill_tokens = .init(.{ .b = 1, .s = seqlen }, .u32), .decode_tokens = .init(.{ .b = 1, .s = 1 }, .u32), .token_index = .init(.{}, .u32), - .kv_cache = .init(config, 1, seqlen, dtype, .f32, model_partitions), + .kv_cache = .init(config, 1, seqlen, dtype, .f32, shardings.model), .rng = .init(), .seqlen = seqlen, .shardings = shardings, diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index f5b996a0..bc14ab72 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -44,18 +44,6 @@ pub const RopeParameters = struct { rope_theta: f32, }; -fn kvHeadsAreReplicated(axis_size: i64, model_partitions: i64) bool { - return axis_size > 0 and axis_size < model_partitions and @rem(model_partitions, axis_size) == 0; -} - -fn partitionKvCacheShape(kv_shape: zml.Shape, kv_heads: i64, model_partitions: i64) zml.Shape { - if (kvHeadsAreReplicated(kv_heads, model_partitions)) { - return kv_shape.withPartitioning(.{ .h = .replicated }); - } - - return kv_shape.withPartitioning(.{ .h = .model }); -} - pub const LoadedModel = struct { inner: Model, parsed_config: std.json.Parsed(Config), @@ -548,21 +536,27 @@ pub const SelfAttn = struct { return .{ k, v }; } - fn partitionProjectedKv(tensor: zml.Tensor, replicate_kv_heads: bool) zml.Tensor { - if (replicate_kv_heads) { - return tensor.withPartitioning(.{ .s = .replicated, .h = .replicated, .hd = .replicated }); - } - - return tensor.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); + fn partitionProjectedKv(kv: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { + return switch (kv_head_sharding) { + .sharded => |heads| blk: { + const sharded_kv = if (heads.factor != 1) kv.stutter1d(kv.axis(.h), heads.factor) else kv; + break :blk sharded_kv.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); + }, + .replicated => kv.withPartitioning(.{ .s = .replicated, .h = .replicated, .hd = .replicated }), + }; } - fn partitionCachedKv(tensor: zml.Tensor, replicate_kv_heads: bool) zml.Tensor { - const cache_tensor = tensor.rename(.{ .s = .k }); - if (replicate_kv_heads) { - return cache_tensor.withPartitioning(.{ .k = .replicated, .h = .replicated, .hd = .replicated }); - } - - return cache_tensor.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); + fn partitionCachedKv(tensor: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { + var cache_tensor = tensor.rename(.{ .s = .k }); + return switch (kv_head_sharding) { + .sharded => |heads| blk: { + if (heads.factor != 1) { + cache_tensor = cache_tensor.stutter1d(cache_tensor.axis(.h), heads.factor); + } + break :blk cache_tensor.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); + }, + .replicated => cache_tensor.withPartitioning(.{ .k = .replicated, .h = .replicated, .hd = .replicated }), + }; } pub fn forward( @@ -571,14 +565,18 @@ pub const SelfAttn = struct { token_index: zml.Tensor, kv_cache: KvCache.SelfAttnCache, ) struct { zml.Tensor, KvCache.SelfAttnCache } { - const model_partitions = zml.module.CompilationContext.current().partitioning.numPartitionsForLogicalAxis(self.q_proj.weight.shape(), .model) catch unreachable; - const replicate_kv_heads = kvHeadsAreReplicated(self.num_kv_heads, model_partitions); const x_qkv = x.withPartitioning(.{ .d = .replicated }); var q, var gate = self.projectQAndGate(x_qkv); var k, var v = self.projectKV(x_qkv); - k = partitionProjectedKv(k, replicate_kv_heads); - v = partitionProjectedKv(v, replicate_kv_heads); + const kv_head_sharding = zml.module.CompilationContext.current().partitioning.shardableDim( + k.shape().withPartitioning(.{ .h = .model }), + .h, + q.dim(.h), + ) catch unreachable; + + k = partitionProjectedKv(k, kv_head_sharding); + v = partitionProjectedKv(v, kv_head_sharding); q = self.q_norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); k = self.k_norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); @@ -590,15 +588,15 @@ pub const SelfAttn = struct { const cos, const sin = self.rotary_embed.getCosAndSin(position_ids, dtype); q = self.rotary_embed.applyRope(q, cos, sin); k = self.rotary_embed.applyRope(k, cos, sin); - k = partitionProjectedKv(k, replicate_kv_heads); - v = partitionProjectedKv(v, replicate_kv_heads); + k = partitionProjectedKv(k, kv_head_sharding); + v = partitionProjectedKv(v, kv_head_sharding); const new_kv_cache = kv_cache.update(k, v, token_index.convert(.u32)); k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); q = q.rename(.{ .s = .q }); - k = partitionCachedKv(k, replicate_kv_heads); - v = partitionCachedKv(v, replicate_kv_heads); + k = partitionCachedKv(k, kv_head_sharding); + v = partitionCachedKv(v, kv_head_sharding); const attn_output = zml.attention.attention.attention( q, @@ -710,7 +708,7 @@ pub const GatedDeltaNet = struct { num_k_heads: i64, num_v_heads: i64, - qk_head_repetition: i64, + qk_head_repetition: u32, head_k_dim: i64, head_v_dim: i64, conv_kernel_size: i64, @@ -720,8 +718,8 @@ pub const GatedDeltaNet = struct { } pub fn init(store: zml.io.TensorStore.View, config: Config) GatedDeltaNet { - const qk_head_repetition = - @divExact(config.text_config.linear_num_value_heads, config.text_config.linear_num_key_heads); + const qk_head_repetition: u32 = + @intCast(@divExact(config.text_config.linear_num_value_heads, config.text_config.linear_num_key_heads)); return .{ .in_proj_qkv = initProj(store.withPrefix("in_proj_qkv"), .{ .dout = .model, .d = .replicated }), .in_proj_z = initProj(store.withPrefix("in_proj_z"), .{ .dout = .model, .d = .replicated }), @@ -873,11 +871,11 @@ pub const GatedDeltaNet = struct { const query_for_rule = b: { if (self.qk_head_repetition == 1) break :b query; - break :b query.stutter1d(@intCast(query.axis(.kh)), @intCast(self.qk_head_repetition)); + break :b query.stutter1d(query.axis(.kh), self.qk_head_repetition); }; const key_for_rule = b: { if (self.qk_head_repetition == 1) break :b key; - break :b key.stutter1d(@intCast(key.axis(.kh)), @intCast(self.qk_head_repetition)); + break :b key.stutter1d(key.axis(.kh), self.qk_head_repetition); }; const core_attn_out, const last_recurrent_state = recurrentGatedDeltaRule( @@ -968,7 +966,7 @@ pub const KvCache = struct { pub const Buffers = zml.Bufferized(SelfAttnCache); - pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, model_partitions: i64) SelfAttnCache { + pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, model_sharding: zml.Sharding) SelfAttnCache { const num_self_attn_layers = countLayers(config.text_config.layer_types, .full_attention); const kv_shape = zml.Shape.init(.{ .b = batch_dim, @@ -977,7 +975,11 @@ pub const KvCache = struct { .h = config.text_config.num_key_value_heads, .hd = config.text_config.head_dim, }, dtype); - const sharded_kv_shape = partitionKvCacheShape(kv_shape, config.text_config.num_key_value_heads, model_partitions); + const kv_head_sharding = model_sharding.shardableDim(kv_shape.dim(.h), .model, config.text_config.num_attention_heads); + const sharded_kv_shape = switch (kv_head_sharding) { + .sharded => |heads| kv_shape.setDim(.h, heads.dim).withPartitioning(.{ .h = .model }), + .replicated => kv_shape.withPartitioning(.{ .h = .replicated }), + }; return .{ .k = .fromShape(sharded_kv_shape), .v = .fromShape(sharded_kv_shape), @@ -1169,11 +1171,11 @@ pub const KvCache = struct { max_seq_len: i64, cache_dtype: zml.DataType, recurrent_dtype: zml.DataType, - model_partitions: i64, + model_sharding: zml.Sharding, ) KvCache { return .{ .layer_types = config.text_config.layer_types, - .self_attn = .init(config, batch_dim, max_seq_len, cache_dtype, model_partitions), + .self_attn = .init(config, batch_dim, max_seq_len, cache_dtype, model_sharding), .gated_delta_net = .init(config, batch_dim, cache_dtype, recurrent_dtype), }; } diff --git a/examples/llm/models/qwen3_5_moe/inference.zig b/examples/llm/models/qwen3_5_moe/inference.zig index 6431c4a2..e8941892 100644 --- a/examples/llm/models/qwen3_5_moe/inference.zig +++ b/examples/llm/models/qwen3_5_moe/inference.zig @@ -30,9 +30,8 @@ pub const CompilationParameters = struct { pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, moe_backend: zml.moe.Backend, shardings: common.Shardings) CompilationParameters { const dtype = mdl.text_model.embed_tokens.weight.dtype(); - const model_partitions = shardings.model.numPartitionsForLogicalAxis(.model); return .{ - .kv_cache = .init(config, 1, seqlen, dtype, .f32, model_partitions), + .kv_cache = .init(config, 1, seqlen, dtype, .f32, shardings.model), .rng = .init(), .prefill_moe_metadata = initMoeMetadata(mdl, @intCast(seqlen), 1, moe_backend), .decode_moe_metadata = initMoeMetadata(mdl, 1, 1, moe_backend), diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 9bc35b3f..c222d8c1 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -1,8 +1,6 @@ const std = @import("std"); const zml = @import("zml"); -const CompilationContext = zml.module.CompilationContext; - const stdx = zml.stdx; const common = @import("../common.zig"); @@ -49,34 +47,27 @@ pub const RopeParameters = struct { rope_theta: f32, }; -fn kvHeadRepeatFactor(axis_size: i64, model_partitions: i64) ?u32 { - if (axis_size <= 0 or axis_size >= model_partitions or @rem(model_partitions, axis_size) != 0) return null; - return @intCast(@divExact(model_partitions, axis_size)); +fn partitionProjectedKv(kv: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { + return switch (kv_head_sharding) { + .sharded => |heads| blk: { + const sharded_kv = if (heads.factor != 1) kv.stutter1d(kv.axis(.h), heads.factor) else kv; + break :blk sharded_kv.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); + }, + .replicated => kv.withPartitioning(.{ .s = .replicated, .h = .replicated, .hd = .replicated }), + }; } -fn partitionProjectedKv(tensor: zml.Tensor, repeat_factor: ?u32) zml.Tensor { - return if (repeat_factor) |factor| - tensor.stutter1d(tensor.axis(.h), @as(u63, factor)).withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }) - else - tensor.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); -} - -fn partitionCachedKv(tensor: zml.Tensor, repeat_factor: ?u32) zml.Tensor { - return if (repeat_factor) |factor| - tensor.rename(.{ .s = .k }).stutter1d(tensor.axis(.h), @as(u63, factor)).withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }) - else - tensor.rename(.{ .s = .k }).withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); -} - -fn partitionKvCacheShape(kv_shape: zml.Shape, kv_heads: i64, model_partitions: i64) zml.Shape { - if (kvHeadRepeatFactor(kv_heads, model_partitions)) |repeat_factor| { - const repeated_h = kv_shape.dim(.h) * @as(i64, @intCast(repeat_factor)); - return kv_shape - .setDim(.h, repeated_h) - .withPartitioning(.{ .h = .model }); - } - - return kv_shape.withPartitioning(.{ .h = .model }); +fn partitionCachedKv(tensor: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { + var kv = tensor.rename(.{ .s = .k }); + return switch (kv_head_sharding) { + .sharded => |heads| blk: { + if (heads.factor != 1) { + kv = kv.stutter1d(kv.axis(.h), heads.factor); + } + break :blk kv.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); + }, + .replicated => kv.withPartitioning(.{ .k = .replicated, .h = .replicated, .hd = .replicated }), + }; } pub const LoadedModel = struct { @@ -575,16 +566,19 @@ pub const SelfAttn = struct { token_index: zml.Tensor, kv_cache: KvCache.SelfAttnCache, ) struct { zml.Tensor, KvCache.SelfAttnCache } { - const model_partitions = zml.module.CompilationContext.current().partitioning.numPartitionsForLogicalAxis(self.q_proj.weight.shape(), .model) catch unreachable; - const repeat_factor = kvHeadRepeatFactor(self.num_kv_heads, model_partitions); const x_qkv = x.withPartitioning(.{ .d = .replicated }); - var q, var gate = self.projectQAndGate(x_qkv); var k, var v = self.projectKV(x_qkv); + const kv_head_sharding = zml.module.CompilationContext.current().partitioning.shardableDim( + k.shape().withPartitioning(.{ .h = .model }), + .h, + q.dim(.h), + ) catch unreachable; + q = q.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); gate = gate.withPartitioning(.{ .s = .replicated, .d_out_proj = .model }); - k = partitionProjectedKv(k, repeat_factor); - v = partitionProjectedKv(v, repeat_factor); + k = partitionProjectedKv(k, kv_head_sharding); + v = partitionProjectedKv(v, kv_head_sharding); q = self.q_norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); k = self.k_norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); @@ -597,15 +591,15 @@ pub const SelfAttn = struct { q = self.rotary_embed.applyRope(q, cos, sin); k = self.rotary_embed.applyRope(k, cos, sin); q = q.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); - k = partitionProjectedKv(k, null); - v = partitionProjectedKv(v, null); + k = partitionProjectedKv(k, kv_head_sharding); + v = partitionProjectedKv(v, kv_head_sharding); const new_kv_cache = kv_cache.update(k, v, token_index.convert(.u32)); k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); q = q.rename(.{ .s = .q }).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }); - k = partitionCachedKv(k, repeat_factor); - v = partitionCachedKv(v, repeat_factor); + k = partitionCachedKv(k, kv_head_sharding); + v = partitionCachedKv(v, kv_head_sharding); const attn_output = zml.attention.attention.attention( q, @@ -871,7 +865,7 @@ pub const GatedDeltaNet = struct { num_k_heads: i64, num_v_heads: i64, - qk_head_repetition: i64, + qk_head_repetition: u32, head_k_dim: i64, head_v_dim: i64, conv_kernel_size: i64, @@ -881,8 +875,8 @@ pub const GatedDeltaNet = struct { } pub fn init(store: zml.io.TensorStore.View, config: Config) GatedDeltaNet { - const qk_head_repetition = - @divExact(config.text_config.linear_num_value_heads, config.text_config.linear_num_key_heads); + const qk_head_repetition: u32 = + @intCast(@divExact(config.text_config.linear_num_value_heads, config.text_config.linear_num_key_heads)); return .{ .in_proj_qkv = initProj(store.withPrefix("in_proj_qkv"), .{ .dout = .model, .d = .replicated }), .in_proj_qkv_scale = null, @@ -1050,8 +1044,8 @@ pub const GatedDeltaNet = struct { g = setPaddingToZero(g, valid_mask.broad(g.shape())); } - const query_for_rule = if (self.qk_head_repetition == 1) query else query.stutter1d(@intCast(query.axis(.kh)), @intCast(self.qk_head_repetition)); - const key_for_rule = if (self.qk_head_repetition == 1) key else key.stutter1d(@intCast(key.axis(.kh)), @intCast(self.qk_head_repetition)); + const query_for_rule = if (self.qk_head_repetition == 1) query else query.stutter1d(query.axis(.kh), self.qk_head_repetition); + const key_for_rule = if (self.qk_head_repetition == 1) key else key.stutter1d(key.axis(.kh), self.qk_head_repetition); const core_attn_out, const last_recurrent_state = recurrentGatedDeltaRule( query_for_rule, @@ -1135,7 +1129,7 @@ pub const KvCache = struct { v: zml.Tensor, layer_index: zml.Tensor, - pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, model_partitions: i64) SelfAttnCache { + pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, model_sharding: zml.Sharding) SelfAttnCache { const num_self_attn_layers = countLayers(config.text_config.layer_types, .full_attention); const kv_shape = zml.Shape.init(.{ .b = batch_dim, @@ -1144,7 +1138,11 @@ pub const KvCache = struct { .h = config.text_config.num_key_value_heads, .hd = config.text_config.head_dim, }, dtype); - const sharded_kv_shape = partitionKvCacheShape(kv_shape, config.text_config.num_key_value_heads, model_partitions); + const kv_head_sharding = model_sharding.shardableDim(kv_shape.dim(.h), .model, config.text_config.num_attention_heads); + const sharded_kv_shape = switch (kv_head_sharding) { + .sharded => |heads| kv_shape.setDim(.h, heads.dim).withPartitioning(.{ .h = .model }), + .replicated => kv_shape.withPartitioning(.{ .h = .replicated }), + }; return .{ .k = .fromShape(sharded_kv_shape), .v = .fromShape(sharded_kv_shape), @@ -1324,11 +1322,11 @@ pub const KvCache = struct { max_seq_len: i64, cache_dtype: zml.DataType, recurrent_dtype: zml.DataType, - model_partitions: i64, + model_sharding: zml.Sharding, ) KvCache { return .{ .layer_types = config.text_config.layer_types, - .self_attn = SelfAttnCache.init(config, batch_dim, max_seq_len, cache_dtype, model_partitions), + .self_attn = SelfAttnCache.init(config, batch_dim, max_seq_len, cache_dtype, model_sharding), .gated_delta_net = GatedDeltaNetCache.init(config, batch_dim, cache_dtype, recurrent_dtype), }; } diff --git a/zml/Sharding.zig b/zml/Sharding.zig index 88fd0e54..6017097d 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -1,5 +1,6 @@ //! Explain how to split a buffer across different devices. const std = @import("std"); +const builtin = @import("builtin"); const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); @@ -12,7 +13,6 @@ const PlatformDevice = @import("platform.zig").Device; const Shape = @import("shape.zig").Shape; const Slice = @import("slice.zig").Slice; const Target = @import("platform.zig").Target; -const builtin = @import("builtin"); const Sharding = @This(); @@ -117,6 +117,15 @@ pub const Partitioning = struct { return sharding.data.numPartitionsForLogicalAxis(logical_axis); } + pub fn shardableDim(self: Partitioning, shape: Shape, axis: anytype, must_divide: i64) !DimSharding { + const ax = shape.axis(axis); + const spec = shape.partition(ax); + if (spec != .axis) return .replicated; + + const sharding = try self.selectSharding(shape); + return sharding.shardableDim(shape.dim(ax), spec.axis, must_divide); + } + pub fn sdyPerValueShardingAttr(self: Partitioning, allocator: std.mem.Allocator, ctx: *mlir.Context, shapes: []const Shape) !*const mlir.Attribute { stdx.debug.assert(self.partitioner == .shardy, "sdyPerValueShardingAttr requires shardy partitioner", .{}); @@ -208,6 +217,31 @@ fn shapeHasAxisPartition(shape: Shape) bool { return false; } +pub const DimSharding = union(enum) { + sharded: struct { + dim: i64, + factor: u32, + }, + replicated, +}; + +pub fn shardableDim(sharding: Sharding, dim: i64, logical_axis: anytype, must_divide: i64) DimSharding { + const partitions = sharding.numPartitionsForLogicalAxis(logical_axis); + + if (@mod(dim, partitions) == 0) { + return if (@mod(must_divide, dim) == 0) .{ .sharded = .{ .dim = dim, .factor = 1 } } else .replicated; + } + + const gcd: i64 = @intCast(std.math.gcd(@as(u64, @intCast(dim)), @as(u64, @intCast(partitions)))); + const repeat_factor: u32 = @intCast(@divExact(partitions, gcd)); + const materialized_dim = std.math.mul(i64, dim, @as(i64, repeat_factor)) catch return .replicated; + if (@mod(must_divide, materialized_dim) == 0) { + return .{ .sharded = .{ .dim = materialized_dim, .factor = repeat_factor } }; + } else { + return .replicated; + } +} + /// Device as part of a PhysicalMesh. pub const Device = struct { /// Unique device identifier in the mesh diff --git a/zml/moe/mosaic_tpu.zig b/zml/moe/mosaic_tpu.zig index 8023da7a..428a5332 100644 --- a/zml/moe/mosaic_tpu.zig +++ b/zml/moe/mosaic_tpu.zig @@ -1,12 +1,11 @@ const std = @import("std"); +const mtt = @import("kernels/mosaic_tpu/builder"); const stdx = @import("stdx"); const zml = @import("../zml.zig"); const Tensor = zml.Tensor; - pub const gmm_ep = @import("mosaic_tpu_kernels/gmm_ep.zig"); -const mtt = @import("kernels/mosaic_tpu/builder"); const log = std.log.scoped(.moe_mosaic_tpu); @@ -225,6 +224,7 @@ pub fn callGmmEp( }; const aligned_k = @divFloor(k + 127, 128) * 128; const aligned_n = if (fuse_act == .none) @divFloor(out_n + 127, 128) * 128 else out_n; + const cfg: gmm_ep.Cfg = .{ .size_m = lhs.dim(.token), .size_k = k, @@ -252,7 +252,13 @@ pub fn callGmmEp( .{ .out = zml.Shape.init(.{ .token = lhs.dim(.token), .out = aligned_n }, out_dtype), }, - .{ .cfg = cfg }, + .{ + .cfg = cfg, + .extras = .{ + // todo leverage tpu api like this https://github.com/openxla/tokamax/blob/f65135e85acd7571531be91d4d3241e1929421f0/tokamax/_src/ops/ragged_dot/pallas_mosaic_tpu_v2_tgmm_kernel.py#L692 + .vmem_limit_bytes = 32 * 1024 * 1024, + }, + }, ).out; return out.slice1d(.out, .{ .end = out_n }); diff --git a/zml/moe/mosaic_tpu_kernels/gmm_ep.zig b/zml/moe/mosaic_tpu_kernels/gmm_ep.zig index 421bb7cc..2fca09f5 100644 --- a/zml/moe/mosaic_tpu_kernels/gmm_ep.zig +++ b/zml/moe/mosaic_tpu_kernels/gmm_ep.zig @@ -290,6 +290,8 @@ fn matmulByMxuColumns(b: *mtt.Builder, cfg: Cfg, lhs: mtt.Value, rhs: mtt.Value, }); } + if (chunks == 1) return parts[0]; + const max_concat_inputs = 16; if (chunks <= max_concat_inputs) return b.concatenate(parts, 1); @@ -327,6 +329,8 @@ fn matmulByMxuColumns(b: *mtt.Builder, cfg: Cfg, lhs: mtt.Value, rhs: mtt.Value, }); } + if (chunks == 1) return parts[0]; + const max_concat_inputs = 16; if (chunks <= max_concat_inputs) return b.concatenate(parts, 1); diff --git a/zml/tensor.zig b/zml/tensor.zig index 0b34b371..d62de5c6 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1979,17 +1979,18 @@ pub const Tensor = struct { /// /// * stutter1d([0, 1, 2, 3], -1, 2) = [0, 0, 1, 1, 2, 2, 3, 3] /// This is equivalent to repeat(ax+1) unless ax is the last axis. - pub fn stutter1d(self: Tensor, axis_: i8, n_rep: u63) Tensor { + pub fn stutter1d(self: Tensor, axis_: i8, n_rep: u32) Tensor { + stdx.debug.assert(n_rep > 0, "stutter1d expects a positive repeat count, got {}", .{n_rep}); const a = self.axis(axis_); const broadshape = self._shape.insert(a + 1, .{n_rep}); - const res_shape = self._shape.setDim(a, self.dim(a) * n_rep); + const res_shape = self._shape.setDim(a, self.dim(a) * @as(i64, n_rep)); const stutter_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a + 1); return self.broadcast(broadshape, stutter_dims.dims()).reshape(res_shape); } /// Repeats in line each value along the given axes. - pub fn stutter(self: Tensor, n_reps: []const u63) Tensor { + pub fn stutter(self: Tensor, n_reps: []const u32) Tensor { // TODO: this should support the tagged syntax: x.repeat(.{ .a = 3, .b = 2}); stdx.debug.assert(n_reps.len == self.rank(), "stutter expects tensor rank and 'n_reps' length to be equal, got {} and {}", .{ self.rank(), n_reps.len }); From 5131467252d28cb10a68a7d020faf0c1dedb8a75 Mon Sep 17 00:00:00 2001 From: elogir Date: Mon, 22 Jun 2026 19:06:48 +0200 Subject: [PATCH 526/881] zml/attention: tune triton kernel for oneAPI performance (#588) Speeds up paged-attention decode on the Intel/oneAPI backend --- zml/BUILD.bazel | 1 + zml/attention/triton_attention.zig | 163 +++++- .../unified_attention_oneapi.zig | 534 ++++++++++++++++++ 3 files changed, 695 insertions(+), 3 deletions(-) create mode 100644 zml/attention/triton_kernels/unified_attention_oneapi.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 598d1c30..353df1d5 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -87,6 +87,7 @@ zig_library( "attention/tpu_attention.zig", "attention/triton_attention.zig", "attention/triton_kernels/unified_attention.zig", + "attention/triton_kernels/unified_attention_oneapi.zig", "moe/moe.zig", "moe/mosaic_tpu.zig", "moe/mosaic_tpu_kernels/gmm_ep.zig", diff --git a/zml/attention/triton_attention.zig b/zml/attention/triton_attention.zig index 0acf6fea..e46b4407 100644 --- a/zml/attention/triton_attention.zig +++ b/zml/attention/triton_attention.zig @@ -6,16 +6,22 @@ const zml = @import("../zml.zig"); const AttentionOptions = @import("paged_attention.zig").AttentionOptions; const kernels = @import("triton_kernels/unified_attention.zig"); +const kernels_oneapi = @import("triton_kernels/unified_attention_oneapi.zig"); const tri = zml.kernel.triton; const log = std.log.scoped(.@"zml/attention/triton"); const toDType = tri.from; +fn isOneapiTarget() bool { + return zml.module.CompilationContext.current().platform.target == .oneapi; +} + fn use2dKernel(head_size: usize, sliding_window: usize, all_decode: bool, max_seqlen_q: usize, max_seqlen_k: usize, target_num_prgms: usize, num_2d_prgms: usize) bool { _ = head_size; - _ = all_decode; _ = max_seqlen_q; + // Intel decode spills the 2D whole-sequence kernel; force the 3D split-K path. + if (all_decode and isOneapiTarget()) return false; return sliding_window > 0 or max_seqlen_k <= 512 or num_2d_prgms > target_num_prgms; } @@ -82,7 +88,8 @@ pub const Config3D = struct { fn select3dConfig(options: paged.PagedAttentionOptions) Config3D { var reduce_num_warps: usize = 2; - const attn_warps: usize = 2; + // Intel decode needs more warps to spread the work and avoid register spill. + const attn_warps: usize = if (options.all_decode and isOneapiTarget()) 8 else 2; const tile_size = options.block_size; //const MAX_SEGMENTS: usize = @min(128, std.math.divCeil(usize, max_seqlen_k, tile_size)); @@ -241,7 +248,11 @@ pub const paged = struct { const num_heads: usize = @intCast(q_.dim(.hkv) * q_.dim(.hg)); const num_kv_heads: usize = @intCast(k_cache_.dim(.hkv)); const num_queries_per_kv: usize = num_heads / num_kv_heads; - const block_m: usize = if (num_queries_per_kv <= 16) 16 else std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv); + // Intel decode: pack exactly one GQA group per tile (block_q == 1) so the + // single decode query token doesn't carry masked-out fp32 acc lanes. + const block_m: usize = if (!ctx_.options.is_prefill and isOneapiTarget()) + num_queries_per_kv + else if (num_queries_per_kv <= 16) 16 else std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv); const block_q: usize = block_m / num_queries_per_kv; const num_tokens: usize = @intCast(q_.dim(.b)); const num_seqs: usize = @intCast(parameters_.block_table.dim(.b)); @@ -281,6 +292,8 @@ pub const paged = struct { ); const output = if (use_2d_kernel) pagedAttention2d(parameters_, q_, k_cache_, v_cache_, ctx_.opts, paged_attention_opts) + else if (isOneapiTarget()) + pagedAttention3dOneapi(parameters_, q_, k_cache_, v_cache_, ctx_.opts, paged_attention_opts) else pagedAttention3d(parameters_, q_, k_cache_, v_cache_, ctx_.opts, paged_attention_opts); @@ -500,4 +513,148 @@ pub const paged = struct { return output.output; } + + /// oneAPI/Intel specialization of pagedAttention3d: routes the attention pass to + /// the SIMD16-tuned kernel (compile-time KV strides, cached loads, same-page fast + /// path) while reusing the shared segment-reduce kernel. See unified_attention_oneapi.zig. + pub fn pagedAttention3dOneapi(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, paged_attention_opts: PagedAttentionOptions) zml.Tensor { + _ = opts; + + const config = select3dConfig(paged_attention_opts); + + const head_size_padded: i64 = @intCast(std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim)); + + const k_cache_shape_c = k_cache.shape(); + const k_strides_c = k_cache_shape_c.computeElementStrides().constSlice(); + const v_cache_shape_c = v_cache.shape(); + const v_strides_c = v_cache_shape_c.computeElementStrides().constSlice(); + + const attn_kernel_config: kernels_oneapi.KernelUnifiedAttention3dPtr.Config = .{ + .q_dtype = toDType(q.dtype()), + .kv_dtype = toDType(k_cache.dtype()), + .num_query_heads = @intCast(paged_attention_opts.num_heads), + .num_queries_per_kv = @intCast(paged_attention_opts.numQueriesPerKv()), + .block_size = @intCast(paged_attention_opts.block_size), + .tile_size = @intCast(config.attention.tile_size), + .head_size = @intCast(paged_attention_opts.head_dim), + .head_size_padded = head_size_padded, + .use_alibi_slopes = false, + .use_qq_bias = false, + .use_softcap = false, + .use_sinks = false, + .sliding_window = @intCast(paged_attention_opts.sliding_window), + .block_q = @intCast(config.attention.block_q), + .block_m = @intCast(config.attention.block_m), + .num_segments_per_seq = @intCast(config.attention.num_segments_per_seq), + .all_decode = paged_attention_opts.all_decode, + .stride_k_cache_0 = @intCast(k_strides_c[k_cache_shape_c.axis(.page)]), + .stride_k_cache_1 = @intCast(k_strides_c[k_cache_shape_c.axis(.k_chunk)]), + .stride_k_cache_2 = @intCast(k_strides_c[k_cache_shape_c.axis(.hkv)]), + .stride_v_cache_0 = @intCast(v_strides_c[v_cache_shape_c.axis(.page)]), + .stride_v_cache_1 = @intCast(v_strides_c[v_cache_shape_c.axis(.k_chunk)]), + .stride_v_cache_2 = @intCast(v_strides_c[v_cache_shape_c.axis(.hkv)]), + // Intel: keep KV loads cached (.none); the streaming .cg hint lowers to + // fully-uncached here and defeats the L2 prefetcher. + .kv_cache_modifier = .none, + }; + log.debug("pagedAttention3dOneapi attention config: {any}", .{attn_kernel_config}); + + const reduce_kernel_config: kernels.ReduceSegmentsPtr.Config = .{ + .o_dtype = toDType(q.dtype()), + .num_query_heads = @intCast(paged_attention_opts.num_heads), + .tile_size = @intCast(config.reduce.tile_size), + .head_size = @intCast(paged_attention_opts.head_dim), + .head_size_padded = head_size_padded, + .block_q = @intCast(config.reduce.block_q), + .num_segments_per_seq = @intCast(config.reduce.num_segments_per_seq), + .use_fp8 = false, + }; + log.debug("pagedAttention3d reduce config: {any}", .{reduce_kernel_config}); + + const dummy = zml.Tensor.constant(zml.DataType.i8.zero()); + const block_table_strides = parameters.block_table.shape().computeElementStrides().constSlice(); + const block_table_strides_ptr = zml.Tensor.constant(zml.DataType.i64.constant(block_table_strides[0])); + const q_shape = q.shape().mergeAxes(.{ .h = .{ .hkv, .hg } }); + const q_strides = q_shape.computeElementStrides().constSlice(); + const q_strides_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(q_strides[0])); + const q_strides_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(q_strides[1])); + const k_cache_shape = k_cache.shape(); + const k_strides = k_cache_shape.computeElementStrides().constSlice(); + const k_strides_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(k_strides[k_cache_shape.axis(.page)])); + const k_strides_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(k_strides[k_cache_shape.axis(.k_chunk)])); + const k_strides_2_ptr = zml.Tensor.constant(zml.DataType.i64.constant(k_strides[k_cache_shape.axis(.hkv)])); + const v_cache_shape = v_cache.shape(); + const v_strides = v_cache_shape.computeElementStrides().constSlice(); + const v_strides_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(v_strides[v_cache_shape.axis(.page)])); + const v_strides_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(v_strides[v_cache_shape.axis(.k_chunk)])); + const v_strides_2_ptr = zml.Tensor.constant(zml.DataType.i64.constant(v_strides[v_cache_shape.axis(.hkv)])); + const num_seqs_ptr = zml.Tensor.constant(zml.DataType.i32.constant(parameters.block_table.dim(0))); + const scale: f32 = paged_attention_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))); + const scale_ptr = zml.Tensor.constant(zml.DataType.f32.constant(scale)); + + const attn_grid: [3]i32 = .{ @intCast(config.attention.total_q_blocks), @intCast(paged_attention_opts.num_kv_heads), @intCast(config.attention.num_segments_per_seq) }; + const attn_output = kernels_oneapi.KernelUnifiedAttention3dPtr.Kernel.call( + .{ + .query_ptr = q, + .key_cache_ptr = k_cache, + .value_cache_ptr = v_cache, + .sink_ptr = dummy, + .block_tables_ptr = parameters.block_table, + .seq_lens_ptr = parameters.seq_lens, + .alibi_slopes_ptr = dummy, + .qq_bias_ptr = dummy, + .scale_ptr = scale_ptr, + .k_scale_ptr = dummy, + .v_scale_ptr = dummy, + .softcap_ptr = dummy, + .block_table_stride_ptr = block_table_strides_ptr, + .query_stride_0_ptr = q_strides_0_ptr, + .query_stride_1_ptr = q_strides_1_ptr, + .qq_bias_stride_0_ptr = dummy, + .stride_k_cache_0_ptr = k_strides_0_ptr, + .stride_k_cache_1_ptr = k_strides_1_ptr, + .stride_k_cache_2_ptr = k_strides_2_ptr, + .stride_v_cache_0_ptr = v_strides_0_ptr, + .stride_v_cache_1_ptr = v_strides_1_ptr, + .stride_v_cache_2_ptr = v_strides_2_ptr, + .query_start_len_ptr = parameters.query_start_len, + .num_seqs_ptr = num_seqs_ptr, + }, + .{ + .segm_output = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq, std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim) }, .f32), + .segm_max = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq }, .f32), + .segm_expsum = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq }, .f32), + }, + .{ + .cfg = attn_kernel_config, + .grid = attn_grid, + .num_stages = @intCast(config.attention.num_stages), + .num_warps = @intCast(config.attention.num_warps), + }, + ); + + const output = kernels.ReduceSegmentsPtr.Kernel.call( + .{ + .segm_output_ptr = attn_output.segm_output, + .segm_max_ptr = attn_output.segm_max, + .segm_expsum_ptr = attn_output.segm_expsum, + .seq_lens_ptr = parameters.seq_lens, + .num_seqs_ptr = num_seqs_ptr, + .out_scale_inv_ptr = dummy, + .output_stride_0_ptr = q_strides_0_ptr, + .output_stride_1_ptr = q_strides_1_ptr, + .block_table_stride_ptr = block_table_strides_ptr, + .query_start_len_ptr = parameters.query_start_len, + }, + .{ .output = q.shape() }, + .{ + .cfg = reduce_kernel_config, + .grid = .{ @intCast(paged_attention_opts.num_tokens), @intCast(paged_attention_opts.num_heads), 1 }, + .num_stages = @intCast(config.reduce.num_stages), + .num_warps = @intCast(config.reduce.num_warps), + }, + ); + + return output.output; + } }; diff --git a/zml/attention/triton_kernels/unified_attention_oneapi.zig b/zml/attention/triton_kernels/unified_attention_oneapi.zig new file mode 100644 index 00000000..66387db0 --- /dev/null +++ b/zml/attention/triton_kernels/unified_attention_oneapi.zig @@ -0,0 +1,534 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const dialects = @import("mlir/dialects"); +const ttir = dialects.ttir; + +const zml = @import("../../zml.zig"); +const tri = zml.kernel.triton; +const Builder = tri.Builder; +const Value = tri.Value; +const DType = tri.DType; + +const log = std.log.scoped(.@"zml/attention/triton"); + +// oneAPI/Intel (SIMD16) specialization of the 3D split-K paged-attention kernel. +// Duplicated from unified_attention.zig so the shared kernel stays backend-neutral; +// only the Intel decode path routes here (see triton_attention.zig pagedAttention3dOneapi). + +/// `log_2(e)` — pre-multiply with this to turn `exp(x)` into `exp2(x * RCP_LN2)`. +const RCP_LN2: f32 = 1.4426950408889634; + +fn isFp8(dt: DType) bool { + return dt == .f8e4m3fn or dt == .f8e5m2; +} + +fn cdivFn(x: Value, y: anytype) Value { + return switch (@typeInfo(@TypeOf(y))) { + .comptime_int, .comptime_float => x.add(y - 1).div(y), + else => x.add(y).sub(1).div(y), + }; +} + +fn applySoftcap(k: *Builder, s_val: Value, x: Value) Value { + const sdiv = s_val.div(x); + const p1 = k.exp2(sdiv); + const p2 = k.exp2(k.negf(sdiv)); + return x.mul(p1.sub(p2)).div(p1.add(p2)); +} + +fn findSeqIdx( + k: *Builder, + query_start_len_ptr: Value, + target_idx: Value, + num_seqs: Value, + block_q: i64, + use_q_block_mode: bool, +) Value { + const left_init = k.liftAs(0, .i32); + + var w = k.openWhile(.{ left_init, num_seqs }, .{ k.scalarTy(.i32), k.scalarTy(.i32) }); + { + const left = w.before_carried[0]; + const right = w.before_carried[1]; + w.yieldBefore(left.lt(right), .{ left, right }); + } + { + const left = w.after_carried[0]; + const right = w.after_carried[1]; + const mid = left.add(right).div(2); + const val = k.load(query_start_len_ptr.addPtr(mid)); + const mid_val = if (use_q_block_mode) + val.div(@as(i32, @intCast(block_q))).add(mid) + else + val; + + var i = k.openIfElse(mid_val.le(target_idx), .{ k.scalarTy(.i32), k.scalarTy(.i32) }); + { + i.yieldThen(.{ mid.add(1), right }); + } + { + i.yieldElse(.{ left, mid }); + } + w.yieldAfter(.{ i.results[0], i.results[1] }); + } + + return w.results[0].sub(1); +} + +fn dequantKv(loaded: Value, scale: Value, kv_is_fp8: bool, q_dtype: DType) Value { + if (!kv_is_fp8 or isFp8(q_dtype)) return loaded; + return loaded.to(.f32).mul(scale).to(q_dtype); +} + +pub const KernelUnifiedAttention3dPtr = struct { + pub const Config = struct { + q_dtype: DType, + kv_dtype: DType, + scale_dtype: DType = .f32, + + num_query_heads: i64, + num_queries_per_kv: i64, + + block_size: i64, + tile_size: i64, + head_size: i64, + head_size_padded: i64, + + use_alibi_slopes: bool, + use_qq_bias: bool, + use_softcap: bool, + use_sinks: bool, + sliding_window: i64, + + stride_k_cache_0: i64 = 0, + stride_k_cache_1: i64 = 0, + stride_k_cache_2: i64 = 0, + stride_k_cache_3: i64 = 1, + stride_v_cache_0: i64 = 0, + stride_v_cache_1: i64 = 0, + stride_v_cache_2: i64 = 0, + stride_v_cache_3: i64 = 1, + + block_q: i64, + block_m: i64, + num_segments_per_seq: i64, + all_decode: bool = false, + kv_cache_modifier: ttir.CacheModifier = .none, + }; + + pub const Kernel = tri.Kernel(Config, .{ + .name = "kernel_unified_attention_3d_ptr_oneapi", + .inputs = &.{ + "query_ptr", "key_cache_ptr", "value_cache_ptr", "sink_ptr", + "block_tables_ptr", "seq_lens_ptr", "alibi_slopes_ptr", "qq_bias_ptr", + "scale_ptr", "k_scale_ptr", "v_scale_ptr", "softcap_ptr", + "block_table_stride_ptr", "query_stride_0_ptr", "query_stride_1_ptr", "qq_bias_stride_0_ptr", + "stride_k_cache_0_ptr", "stride_k_cache_1_ptr", "stride_k_cache_2_ptr", "stride_v_cache_0_ptr", + "stride_v_cache_1_ptr", "stride_v_cache_2_ptr", "query_start_len_ptr", "num_seqs_ptr", + }, + .outputs = &.{ "segm_output", "segm_max", "segm_expsum" }, + .run = run, + }); + fn run(b: *Builder, cfg: Config) tri.FinishError!void { + const a = try b.declareArgs(.{ + .query_ptr = .{ .ptr = cfg.q_dtype }, + .key_cache_ptr = .{ .ptr = cfg.kv_dtype }, + .value_cache_ptr = .{ .ptr = cfg.kv_dtype }, + .sink_ptr = .{ .ptr = cfg.scale_dtype }, + .block_tables_ptr = .{ .ptr = .i32 }, + .seq_lens_ptr = .{ .ptr = .i32 }, + .alibi_slopes_ptr = .{ .ptr = cfg.scale_dtype }, + .qq_bias_ptr = .{ .ptr = cfg.scale_dtype }, + .scale_ptr = .{ .ptr = .f32 }, + .k_scale_ptr = .{ .ptr = .f32 }, + .v_scale_ptr = .{ .ptr = .f32 }, + .softcap_ptr = .{ .ptr = .f32 }, + .block_table_stride_ptr = .{ .ptr = .i64 }, + .query_stride_0_ptr = .{ .ptr = .i64 }, + .query_stride_1_ptr = .{ .ptr = .i64 }, + .qq_bias_stride_0_ptr = .{ .ptr = .i64 }, + .stride_k_cache_0_ptr = .{ .ptr = .i64 }, + .stride_k_cache_1_ptr = .{ .ptr = .i64 }, + .stride_k_cache_2_ptr = .{ .ptr = .i64 }, + .stride_v_cache_0_ptr = .{ .ptr = .i64 }, + .stride_v_cache_1_ptr = .{ .ptr = .i64 }, + .stride_v_cache_2_ptr = .{ .ptr = .i64 }, + .query_start_len_ptr = .{ .ptr = .i32 }, + .num_seqs_ptr = .{ .ptr = .i32 }, + .segm_output_ptr = .{ .ptr = .f32 }, + .segm_max_ptr = .{ .ptr = .f32 }, + .segm_expsum_ptr = .{ .ptr = .f32 }, + }); + + const scale = b.load(a.scale_ptr); + const k_scale = b.load(a.k_scale_ptr); + const v_scale = b.load(a.v_scale_ptr); + const softcap = b.load(a.softcap_ptr); + const block_table_stride = b.load(a.block_table_stride_ptr); + const query_stride_0 = b.load(a.query_stride_0_ptr); + const query_stride_1 = b.load(a.query_stride_1_ptr); + const qq_bias_stride_0 = b.load(a.qq_bias_stride_0_ptr); + const stride_k_cache_0 = b.liftAs(cfg.stride_k_cache_0, .i64); + const stride_k_cache_1 = b.liftAs(cfg.stride_k_cache_1, .i64); + const stride_k_cache_2 = b.liftAs(cfg.stride_k_cache_2, .i64); + const stride_v_cache_0 = b.liftAs(cfg.stride_v_cache_0, .i64); + const stride_v_cache_1 = b.liftAs(cfg.stride_v_cache_1, .i64); + const stride_v_cache_2 = b.liftAs(cfg.stride_v_cache_2, .i64); + const num_seqs = b.load(a.num_seqs_ptr); + + kernelUnifiedAttention3d( + b, + a.segm_output_ptr, + a.segm_max_ptr, + a.segm_expsum_ptr, + a.query_ptr, + a.key_cache_ptr, + a.value_cache_ptr, + a.sink_ptr, + a.block_tables_ptr, + a.seq_lens_ptr, + a.alibi_slopes_ptr, + a.qq_bias_ptr, + scale, + k_scale, + v_scale, + softcap, + block_table_stride, + query_stride_0, + query_stride_1, + qq_bias_stride_0, + stride_k_cache_0, + stride_k_cache_1, + stride_k_cache_2, + stride_v_cache_0, + stride_v_cache_1, + stride_v_cache_2, + a.query_start_len_ptr, + num_seqs, + cfg, + ); + } +}; + +fn kernelUnifiedAttention3d( + k: *Builder, + segm_output_ptr: Value, + segm_max_ptr: Value, + segm_expsum_ptr: Value, + query_ptr: Value, + key_cache_ptr: Value, + value_cache_ptr: Value, + sink_ptr: Value, + block_tables_ptr: Value, + seq_lens_ptr: Value, + alibi_slopes_ptr: Value, + qq_bias_ptr: Value, + scale: Value, + k_scale: Value, + v_scale: Value, + softcap: Value, + block_table_stride: Value, + query_stride_0: Value, + query_stride_1: Value, + qq_bias_stride_0: Value, + stride_k_cache_0: Value, + stride_k_cache_1: Value, + stride_k_cache_2: Value, + stride_v_cache_0: Value, + stride_v_cache_1: Value, + stride_v_cache_2: Value, + query_start_len_ptr: Value, + num_seqs: Value, + config: KernelUnifiedAttention3dPtr.Config, +) void { + const BLOCK_M: i64 = config.block_m; + const BLOCK_Q: i64 = config.block_q; + const BLOCK_SIZE: i64 = config.block_size; + const TILE_SIZE: i64 = config.tile_size; + const HEAD_SIZE: i64 = config.head_size; + const HEAD_SIZE_PADDED: i64 = config.head_size_padded; + const NUM_QUERIES_PER_KV: i64 = config.num_queries_per_kv; + const NUM_QUERY_HEADS: i64 = config.num_query_heads; + const NUM_SEGMENTS_PER_SEQ: i64 = config.num_segments_per_seq; + const SLIDING_WINDOW: i64 = config.sliding_window; + + const q_block_global_idx = k.programId(.x); + const kv_head_idx = k.programId(.y); + const segm_idx = k.programId(.z); + + const qk_scale = scale.mul(RCP_LN2); + + const seq_idx = findSeqIdx(k, query_start_len_ptr, q_block_global_idx, num_seqs, BLOCK_Q, true); + + const q_start_raw = k.load(query_start_len_ptr.addPtr(seq_idx)); + const q_block_start_idx = q_start_raw.div(@as(i32, @intCast(BLOCK_Q))).add(seq_idx); + const q_block_local_idx = q_block_global_idx.sub(q_block_start_idx); + + const cur_batch_in_all_start_index = q_start_raw; + // Match Python's left-to-right `query_start_len_ptr + seq_idx + 1`: two `tt.addptr`s. + const cur_batch_in_all_stop_index = k.load(query_start_len_ptr.addPtr(seq_idx).addPtr(1)); + const cur_batch_query_len = cur_batch_in_all_stop_index.sub(cur_batch_in_all_start_index); + + const q_out_of_range = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))).ge(cur_batch_query_len); + k.returnIf(q_out_of_range, .{}); + + const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); + const tiles_per_segment = cdivFn(seq_len, @as(i32, @intCast(NUM_SEGMENTS_PER_SEQ * TILE_SIZE))); + + const segm_lo = segm_idx.mul(tiles_per_segment).mul(@as(i32, @intCast(TILE_SIZE))); + const segm_out_of_range = segm_lo.ge(seq_len); + k.returnIf(segm_out_of_range, .{}); + + const offs_m = k.arange(0, BLOCK_M, .i32); + const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); + const offs_t = k.arange(0, TILE_SIZE, .i32); + + const q_local_bq = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))); + const query_pos = q_local_bq.add(offs_m.div(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); + + const query_offset_0 = cur_batch_in_all_start_index.add(query_pos); + const query_offset_1 = kv_head_idx.mul(@as(i32, @intCast(NUM_QUERIES_PER_KV))) + .add(offs_m.rem(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); + + // query_offset = qo_0[:, None]*qstride_0 + qo_1[:, None]*qstride_1 + offs_d[None, :] + // Inline `expandDims(offs_d, 0)` *inside* the chain so it's emitted after + // the qo0+qo1 addi + const qo0_2d = k.expandDims(query_offset_0, 1).mul(query_stride_0); + const qo1_2d = k.expandDims(query_offset_1, 1).mul(query_stride_1); + const query_offset = qo0_2d.add(qo1_2d).add(k.expandDims(offs_d, 0)); + + const dim_mask: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + offs_d.lt(@as(i32, @intCast(HEAD_SIZE))) + else + k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); + const query_mask_0 = query_pos.lt(cur_batch_query_len); + const query_mask_1 = query_offset_1.lt(@as(i32, @intCast(NUM_QUERY_HEADS))); + + const q_mask_pre = k.expandDims(query_mask_0, 1).bitAnd(k.expandDims(query_mask_1, 1)); + const q_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + q_mask_pre.bitAnd(k.expandDims(dim_mask, 0)) + else + q_mask_pre; + const q_ptrs = query_ptr.addPtr(query_offset); + const Q = k.loadOpts(q_ptrs, .{ + .mask = k.broadcastTo(q_mask_partial, &.{ BLOCK_M, HEAD_SIZE_PADDED }), + .other = k.zeros(&.{ BLOCK_M, HEAD_SIZE_PADDED }, config.q_dtype), + }); + + const block_table_offset = seq_idx.to(.i64).mul(block_table_stride); + + // M init: USE_SINKS + segm_idx==0 → loaded sinks; else -inf. + const segm_is_zero = segm_idx.eq(0); + const m_init: Value = if (config.use_sinks) mb: { + var mi = k.openIfElse(segm_is_zero, .{k.tensorTy(&.{BLOCK_M}, .f32)}); + { + const loaded = k.loadOpts(sink_ptr.addPtr(query_offset_1), .{ + .mask = query_mask_1, + .other = k.full(&.{BLOCK_M}, -std.math.inf(f32), config.scale_dtype), + }); + mi.yieldThen(.{loaded.to(.f32).mul(RCP_LN2)}); + } + { + mi.yieldElse(.{k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32)}); + } + break :mb mi.results[0]; + } else k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32); + + const l_init = k.full(&.{BLOCK_M}, 1.0, .f32); + const acc_init = k.zeros(&.{ BLOCK_M, HEAD_SIZE_PADDED }, .f32); + + const context_len = seq_len.sub(cur_batch_query_len); + + const alibi_slope: Value = if (config.use_alibi_slopes) + k.loadOpts(alibi_slopes_ptr.addPtr(query_offset_1), .{ + .mask = query_mask_1, + .other = k.full(&.{BLOCK_M}, 0.0, config.scale_dtype), + }) + else + k.zeros(&.{BLOCK_M}, config.scale_dtype); + + // See 2d-body note on qq_bias_row_ptrs: else branch needs an explicit splatTo + // (no sibling offset to auto-broadcast against). + const qq_bias_row_ptrs: Value = if (config.use_qq_bias) + qq_bias_ptr.addPtr(query_pos.to(.i64).mul(qq_bias_stride_0)) + else + qq_bias_ptr.splatTo(&.{BLOCK_M}); + + const pad_term: i32 = @intCast(@divTrunc(BLOCK_M - 1, NUM_QUERIES_PER_KV) + 1); + const max_prefix_raw = context_len.add(q_local_bq).add(pad_term); + const max_seq_prefix_len = max_prefix_raw.minimum(seq_len); + const num_tiles = cdivFn(max_seq_prefix_len, @as(i32, @intCast(TILE_SIZE))); + + const segm_tile_lo = segm_idx.mul(tiles_per_segment); + const segm_tile_hi = segm_idx.add(1).mul(tiles_per_segment).minimum(num_tiles); + + const kv_cache_mod: ttir.CacheModifier = config.kv_cache_modifier; + + var loop = k.openFor(segm_tile_lo, segm_tile_hi, 1, .{ m_init, l_init, acc_init }); + { + const j = loop.iv; + const M = loop.carried[0]; + const L = loop.carried[1]; + const acc = loop.carried[2]; + + const seq_offset = j.mul(@as(i32, @intCast(TILE_SIZE))).add(offs_t); + const tile_mask: Value = seq_offset.lt(max_seq_prefix_len); + + const same_block = TILE_SIZE == BLOCK_SIZE; + const in_block_offs: Value = if (same_block) offs_t else seq_offset.rem(@as(i32, @intCast(BLOCK_SIZE))); + const physical_block_idx: Value = if (same_block) + k.load(block_tables_ptr.addPtr(block_table_offset).addPtr(j)).to(.i64).splatTo(&.{TILE_SIZE}) + else + k.load( + block_tables_ptr.addPtr(block_table_offset).addPtr(seq_offset.div(@as(i32, @intCast(BLOCK_SIZE)))), + ).to(.i64); + + const v_offset = k.expandDims(physical_block_idx, 1).mul(stride_v_cache_0) + .add(kv_head_idx.to(.i64).mul(stride_v_cache_2)) + .add(k.expandDims(offs_d, 0).mul(config.stride_v_cache_3)) + .add(k.expandDims(in_block_offs, 1).mul(stride_v_cache_1)); + + const k_offset = k.expandDims(physical_block_idx, 0).mul(stride_k_cache_0) + .add(kv_head_idx.to(.i64).mul(stride_k_cache_2)) + .add(k.expandDims(offs_d, 1).mul(config.stride_k_cache_3)) + .add(k.expandDims(in_block_offs, 0).mul(stride_k_cache_1)); + + const block_io_loads = same_block and HEAD_SIZE_PADDED == HEAD_SIZE; + + const K_load = if (block_io_loads) + k.loadOpts(key_cache_ptr.addPtr(k_offset), .{ .cache_modifier = kv_cache_mod }) + else kb: { + const k_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + k.expandDims(dim_mask, 1).bitAnd(k.expandDims(tile_mask, 0)) + else + k.expandDims(tile_mask, 0); + break :kb k.loadOpts(key_cache_ptr.addPtr(k_offset), .{ + .mask = k.broadcastTo(k_mask_partial, &.{ HEAD_SIZE_PADDED, TILE_SIZE }), + .other = k.zeros(&.{ HEAD_SIZE_PADDED, TILE_SIZE }, config.kv_dtype), + .cache_modifier = kv_cache_mod, + }); + }; + const K = dequantKv(K_load, k_scale, isFp8(config.kv_dtype), config.q_dtype); + + const V_raw = if (block_io_loads) + k.loadOpts(value_cache_ptr.addPtr(v_offset), .{ .cache_modifier = kv_cache_mod }) + else vb: { + const v_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + k.expandDims(tile_mask, 1).bitAnd(k.expandDims(dim_mask, 0)) + else + k.expandDims(tile_mask, 1); + break :vb k.loadOpts(value_cache_ptr.addPtr(v_offset), .{ + .mask = k.broadcastTo(v_mask_partial, &.{ TILE_SIZE, HEAD_SIZE_PADDED }), + .other = k.zeros(&.{ TILE_SIZE, HEAD_SIZE_PADDED }, config.kv_dtype), + .cache_modifier = kv_cache_mod, + }); + }; + const V_load = if (block_io_loads) + k.where( + k.broadcastTo(k.expandDims(tile_mask, 1), &.{ TILE_SIZE, HEAD_SIZE_PADDED }), + V_raw, + k.zeros(&.{ TILE_SIZE, HEAD_SIZE_PADDED }, config.kv_dtype), + ) + else + V_raw; + const V = dequantKv(V_load, v_scale, isFp8(config.kv_dtype), config.q_dtype); + + // seq_mask = seq_offset[None, :] < context_len + query_pos[:, None] + 1 + const ql_2d_i32 = query_pos.expandDims(1); + const so_2d = seq_offset.expandDims(0); + const rhs = context_len.add(ql_2d_i32).add(1); + const seq_mask = so_2d.lt(rhs); + + const acc_zero = k.zeros(&.{ BLOCK_M, TILE_SIZE }, .f32); + const qk = k.dot(Q, K, acc_zero); + var S = qk_scale.mul(qk); + + if (config.use_softcap) { + S = applySoftcap(k, S, softcap).mul(RCP_LN2); + } + + const qm0_2d = query_mask_0.expandDims(1); + const qm1_2d = query_mask_1.expandDims(1); + const keep_mask = qm1_2d.bitAnd(qm0_2d).bitAnd(seq_mask); + S = k.where(keep_mask, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); + + if (SLIDING_WINDOW > 0) { + const diff = ql_2d_i32.add(context_len).sub(so_2d); + const in_win = diff.lt(@as(i32, @intCast(SLIDING_WINDOW))); + S = k.where(in_win, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); + } + + if (config.use_alibi_slopes) { + const alibi_2d = alibi_slope.expandDims(1); + const pos_diff = seq_offset.sub(context_len).to(config.scale_dtype).expandDims(0); + S = S.add(alibi_2d.mul(pos_diff).to(.f32).mul(RCP_LN2)); + } + + if (config.use_qq_bias) { + const key_rel_pos = seq_offset.sub(context_len); + const is_query_key = key_rel_pos.ge(0) + .bitAnd(key_rel_pos.to(.i64).lt(qq_bias_stride_0)); + const qq_ptrs = qq_bias_row_ptrs.expandDims(1) + .addPtr(key_rel_pos.to(.i64).expandDims(0)); + const qq_bias = k.loadOpts(qq_ptrs, .{ + .mask = is_query_key.expandDims(0), + .other = k.zeros(&.{ BLOCK_M, TILE_SIZE }, config.scale_dtype), + }); + S = S.add(qq_bias.to(.f32).mul(RCP_LN2)); + } + + var m_j = M.maximum(k.maxOpts(S, .{ .axis = 1 })); + m_j = k.where(m_j.gt(-std.math.inf(f32)), m_j, k.full(&.{BLOCK_M}, 0.0, .f32)); + + const mj_2d = m_j.expandDims(1); + const P = k.exp2(S.sub(mj_2d)); + const l_j = k.sumOpts(P, .{ .axis = 1 }); + const alpha = k.exp2(M.sub(m_j)); + + const alpha_2d = alpha.expandDims(1); + const acc_scaled = acc.mul(alpha_2d); + const new_L = L.mul(alpha).add(l_j); + + const P_cast = P.to(config.q_dtype); + const new_acc = k.dot(P_cast, V, acc_scaled); + + loop.yield(.{ m_j, new_L, new_acc }); + } + const M_final = loop.results[0]; + const L_final = loop.results[1]; + const acc_final = loop.results[2]; + + // segm_output_offset = qo_0[:, None].to(i64) * (NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) + // + qo_1[:, None] * (NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) + // + segm_idx * HEAD_SIZE_PADDED + offs_d[None, :] + const segm_out_head_stride_i32: i32 = @intCast(NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED); + const segm_out_token_stride_i64: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED; + + // Inline `segm_idx * HEAD_SIZE_PADDED` and `offs_d[None, :]` inside the chain + // so each term's ops emit in source order + const oo0_2d = k.expandDims(query_offset_0, 1).to(.i64).mul(segm_out_token_stride_i64); + const oo1_2d = k.expandDims(query_offset_1, 1).mul(segm_out_head_stride_i32); + const segm_output_offset = oo0_2d.add(oo1_2d) + .add(segm_idx.mul(@as(i32, @intCast(HEAD_SIZE_PADDED)))) + .add(k.expandDims(offs_d, 0)); + + const store_mask_2d = k.mask2d(query_mask_0, dim_mask, BLOCK_M, HEAD_SIZE_PADDED) + .bitAnd(query_mask_1.expandDims(1)); + k.storeOpts( + segm_output_ptr.addPtr(segm_output_offset), + acc_final, + .{ .mask = store_mask_2d }, + ); + + // segm_offset = qo_0.to(i64) * (NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ) + // + qo_1 * NUM_SEGMENTS_PER_SEQ + segm_idx + const segm_head_stride_i32: i32 = @intCast(NUM_SEGMENTS_PER_SEQ); + const segm_token_stride_i64: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ; + const segm_offset = query_offset_0.to(.i64).mul(segm_token_stride_i64) + .add(query_offset_1.mul(segm_head_stride_i32)) + .add(segm_idx); + const store_mask_1d = query_mask_0.bitAnd(query_mask_1); + k.storeOpts(segm_max_ptr.addPtr(segm_offset), M_final, .{ .mask = store_mask_1d }); + k.storeOpts(segm_expsum_ptr.addPtr(segm_offset), L_final, .{ .mask = store_mask_1d }); +} From a1acbd36d50b2604923b99257d920a54cf8713cd Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Tue, 23 Jun 2026 05:41:40 +0200 Subject: [PATCH 527/881] zml/onapi: set defaults for oneAPI PjRT create options (#592) same as https://github.com/zml/zml/pull/579 And update the pjrt plugin with the collective fix --- platforms/oneapi/oneapi.bzl | 4 ++-- zml/platform.zig | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index c646354b..59afdb8d 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -1,8 +1,8 @@ load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") -PJRT_ONEAPI_RELEASE = "manual-2026-06-18T20-14-00Z" -PJRT_ONEAPI_ARTIFACT_SHA256 = "aa030689405686c4bc933183198a90818c0212f2f23e05564b3f8455fe17003c" +PJRT_ONEAPI_RELEASE = "manual-2026-06-23T00-20-00Z" +PJRT_ONEAPI_ARTIFACT_SHA256 = "97e0892e7d3815118c897d4f4134004f8bca177332de8be2384f71f6f55e954f" PJRT_ONEAPI_ARTIFACT_URL = "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-oneapi_linux-amd64.tar.gz".format( release = PJRT_ONEAPI_RELEASE, ) diff --git a/zml/platform.zig b/zml/platform.zig index e61cb811..9199fc14 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -642,7 +642,7 @@ pub const CreateOptions = struct { cpu: Cpu = .{ .device_count = 4 }, // bump memory fraction from XLA defaults of 75% to 90%. - // Even on a 8GB GPU it should leave enough space for the Cuda driver + // Even on a 8GB GPU it should leave enough space for the platform driver/runtime. // https://github.com/openxla/xla/blob/3e87afa11a865cf91137522492918ad18bfe5b7c/xla/pjrt/plugin/xla_gpu/xla_gpu_allocator_config.h#L25-L60 xla_gpu: XlaGpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.90 } } }, tpu: struct {} = .{}, @@ -709,7 +709,7 @@ pub const CreateOptions = struct { values.shrinkRetainingCapacity(0); switch (target) { .cpu => self.cpu.writeNamedValues(&values), - .cuda, .rocm => self.xla_gpu.writeNamedValues(&values), + .cuda, .rocm, .oneapi => self.xla_gpu.writeNamedValues(&values), inline else => |t| { stdx.debug.assertComptime(@hasField(CreateOptions, @tagName(t)), "zml.platform.CreateOptions doesn't list target {s}", .{@tagName(t)}); const options = @field(self, @tagName(t)); From f586cbfc3ddf0c74b2b8c030da6f1be03dc28b33 Mon Sep 17 00:00:00 2001 From: elogir Date: Tue, 23 Jun 2026 09:16:21 +0200 Subject: [PATCH 528/881] third_party/iree: match all added tokens (#591) IREE's tokenizer only registered added tokens for pre-segmentation matching if they were flagged special or normalized. Tokens that are neither (e.g. was dropped) The fix is to match all added tokens --- .../iree/fix-added-token-matching.patch | 123 ++++++++++++++++++ third_party/iree/repo.bzl | 1 + 2 files changed, 124 insertions(+) create mode 100644 third_party/iree/fix-added-token-matching.patch diff --git a/third_party/iree/fix-added-token-matching.patch b/third_party/iree/fix-added-token-matching.patch new file mode 100644 index 00000000..2a418780 --- /dev/null +++ b/third_party/iree/fix-added-token-matching.patch @@ -0,0 +1,123 @@ +diff --git a/runtime/src/iree/tokenizer/format/huggingface/tokenizer_json.c b/runtime/src/iree/tokenizer/format/huggingface/tokenizer_json.c +index 47d2a04c4d..66f1474c81 100644 +--- a/runtime/src/iree/tokenizer/format/huggingface/tokenizer_json.c ++++ b/runtime/src/iree/tokenizer/format/huggingface/tokenizer_json.c +@@ -48,11 +48,12 @@ static const iree_string_view_t kTopLevelAllowedKeys[] = { + + // Builds special_tokens collections from added_tokens. + // +-// ALL added tokens with `normalized=true` are included (not just those with +-// `special=true`) because they must be matched BEFORE the segmenter runs. +-// Without this, ByteLevel segmentation transforms the input before these +-// tokens can match, breaking tokenizers like GPT-NeoX that have multi-space +-// tokens (e.g., " " at ID 50276). ++// ALL added tokens are included (regardless of their `special` flag) because ++// they must be matched BEFORE the segmenter runs, mirroring HuggingFace's ++// AddedVocabulary. Without this, ByteLevel segmentation transforms the input ++// before these tokens can match, breaking tokenizers like GPT-NeoX that have ++// multi-space tokens (e.g., " " at ID 50276) and dropping non-special added ++// tokens such as `` or ``. + // + // Tokens are split by their normalized flag: + // - out_special_tokens: normalized=false, matched in raw input before +@@ -70,22 +71,10 @@ static iree_status_t iree_tokenizer_huggingface_build_special_tokens( + iree_tokenizer_special_tokens_initialize(out_special_tokens); + iree_tokenizer_special_tokens_initialize(out_special_tokens_post_norm); + +- // Count tokens to process: special tokens OR tokens with normalized=true. +- // Non-special tokens with normalized=true must be matched before +- // segmentation. +- iree_host_size_t match_count = 0; +- for (iree_host_size_t i = 0; i < added_tokens->count; ++i) { +- const iree_tokenizer_huggingface_added_token_t* token = +- iree_tokenizer_huggingface_added_tokens_get(added_tokens, i); +- bool is_special = iree_any_bit_set( +- token->flags, IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_SPECIAL); +- bool is_normalized = iree_any_bit_set( +- token->flags, IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_NORMALIZED); +- // Include if special OR if normalized (needs pre-segmentation matching). +- if (is_special || is_normalized) { +- ++match_count; +- } +- } ++ // All added tokens are matched before segmentation, mirroring HuggingFace's ++ // AddedVocabulary which matches every added token regardless of its `special` ++ // flag. They are routed by their `normalized` flag below. ++ iree_host_size_t match_count = added_tokens->count; + + // Early exit if nothing to process. + if (match_count == 0) { +@@ -108,45 +97,38 @@ static iree_status_t iree_tokenizer_huggingface_build_special_tokens( + i < added_tokens->count && iree_status_is_ok(status); ++i) { + const iree_tokenizer_huggingface_added_token_t* token = + iree_tokenizer_huggingface_added_tokens_get(added_tokens, i); +- bool is_special = iree_any_bit_set( +- token->flags, IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_SPECIAL); + bool is_normalized = iree_any_bit_set( + token->flags, IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_NORMALIZED); + +- // Process if special OR if normalized (needs pre-segmentation matching). +- if (is_special || is_normalized) { +- iree_string_view_t content = +- iree_tokenizer_huggingface_added_token_content(added_tokens, token); +- +- // Convert HuggingFace flags to special_tokens flags. +- iree_tokenizer_special_token_flags_t special_flags = +- IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_NONE; +- if (iree_any_bit_set( +- token->flags, +- IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_LSTRIP)) { +- special_flags |= IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_LSTRIP; +- } +- if (iree_any_bit_set( +- token->flags, +- IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_RSTRIP)) { +- special_flags |= IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_RSTRIP; +- } +- if (iree_any_bit_set( +- token->flags, +- IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_SINGLE_WORD)) { +- special_flags |= IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_SINGLE_WORD; +- } ++ iree_string_view_t content = ++ iree_tokenizer_huggingface_added_token_content(added_tokens, token); + +- // Route to appropriate builder based on normalized flag. +- if (is_normalized) { +- // normalized=true: match after normalization but before segmentation. +- status = iree_tokenizer_special_tokens_builder_add( +- &builder_post_norm, content, token->id, special_flags); +- } else { +- // normalized=false (default): match before normalization. +- status = iree_tokenizer_special_tokens_builder_add( +- &builder_pre_norm, content, token->id, special_flags); +- } ++ // Convert HuggingFace flags to special_tokens flags. ++ iree_tokenizer_special_token_flags_t special_flags = ++ IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_NONE; ++ if (iree_any_bit_set(token->flags, ++ IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_LSTRIP)) { ++ special_flags |= IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_LSTRIP; ++ } ++ if (iree_any_bit_set(token->flags, ++ IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_RSTRIP)) { ++ special_flags |= IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_RSTRIP; ++ } ++ if (iree_any_bit_set( ++ token->flags, ++ IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_SINGLE_WORD)) { ++ special_flags |= IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_SINGLE_WORD; ++ } ++ ++ // Route to appropriate builder based on normalized flag. ++ if (is_normalized) { ++ // normalized=true: match after normalization but before segmentation. ++ status = iree_tokenizer_special_tokens_builder_add( ++ &builder_post_norm, content, token->id, special_flags); ++ } else { ++ // normalized=false (default): match before normalization. ++ status = iree_tokenizer_special_tokens_builder_add( ++ &builder_pre_norm, content, token->id, special_flags); + } + } + diff --git a/third_party/iree/repo.bzl b/third_party/iree/repo.bzl index fc84dee9..3b7f9b06 100644 --- a/third_party/iree/repo.bzl +++ b/third_party/iree/repo.bzl @@ -14,6 +14,7 @@ def repo(): patches = [ "//third_party/iree:tokenizer-only.patch", "//third_party/iree:fix-gemma4-encode.patch", + "//third_party/iree:fix-added-token-matching.patch", ], patch_args = ["-p1"], ) From d942ce014a021b56a0762b8316bdb9dda6f17476 Mon Sep 17 00:00:00 2001 From: Louis Lechevalier Date: Tue, 23 Jun 2026 17:51:37 +0200 Subject: [PATCH 529/881] platforms/tpu: bump plugin from v0.0.40 to v0.0.42 (#596) --- platforms/tpu/tpu.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platforms/tpu/tpu.bzl b/platforms/tpu/tpu.bzl index 2e7538dd..08d54ed7 100644 --- a/platforms/tpu/tpu.bzl +++ b/platforms/tpu/tpu.bzl @@ -4,9 +4,9 @@ def _tpu_impl(mctx): # https://storage.googleapis.com/jax-releases/libtpu_releases.html http_archive( name = "libpjrt_tpu", - url = "https://files.pythonhosted.org/packages/42/7f/cc3ad9e9b866c701e90e6d8d4e332557fefb1f3ad1bebd9914ff09778691/libtpu-0.0.40-cp314-cp314t-manylinux_2_31_x86_64.whl", + url = "https://storage.googleapis.com/libtpu-nightly-releases/wheels/libtpu/libtpu-0.0.42.dev20260613+nightly-cp314-cp314t-manylinux_2_31_x86_64.whl", type = "zip", - sha256 = "a1d01214bffe5a0910057014eaa17ed838eebc780e1538b4bd118684908ea120", + sha256 = "1d0bbb1608154bda6944902abd6f56f956638539d7503db4894927e21a41baed", build_file = "libpjrt_tpu.BUILD.bazel", ) return mctx.extension_metadata( From 33a3b37900321b281cbb225ebd877d3e43c5a4e5 Mon Sep 17 00:00:00 2001 From: elogir Date: Wed, 24 Jun 2026 09:54:11 +0200 Subject: [PATCH 530/881] platforms/metal: add Metal support (#593) Co-authored-by: Steeve Morin --- MODULE.bazel | 3 + examples/llm/models/common.zig | 2 +- examples/llm/models/llama/model.zig | 1 + examples/llm/models/llama/session.zig | 6 +- platforms/BUILD.bazel | 19 +++++- platforms/metal/BUILD.bazel | 42 ++++++++++++ platforms/metal/metal.bzl | 29 ++++++++ platforms/metal/metal.zig | 39 +++++++++++ platforms/platforms.zig | 4 ++ zml/BUILD.bazel | 1 + zml/Sharding.zig | 6 +- zml/attention.zig | 1 + zml/attention/attention.zig | 43 ++++++++++++ zml/attention/metal_attention.zig | 96 +++++++++++++++++++++++++++ zml/attention/paged_attention.zig | 21 ++++++ zml/exe.zig | 4 +- zml/io.zig | 2 +- zml/mem.zig | 2 +- zml/ops.zig | 8 +-- zml/platform.zig | 12 ++-- zml/tensor.zig | 8 +-- 21 files changed, 326 insertions(+), 23 deletions(-) create mode 100644 platforms/metal/BUILD.bazel create mode 100644 platforms/metal/metal.bzl create mode 100644 platforms/metal/metal.zig create mode 100644 zml/attention/metal_attention.zig diff --git a/MODULE.bazel b/MODULE.bazel index 7b99bb48..69f81b44 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -180,6 +180,9 @@ use_repo(tpu, "libpjrt_tpu") neuron = use_extension("//platforms/neuron:neuron.bzl", "neuron_packages") use_repo(neuron, "aws-neuronx-collectives", "aws-neuronx-runtime-lib", "aws-neuronx-tools", "libgcc_s1", "libgomp1", "libpjrt_neuron", "libstdcpp6") +metal = use_extension("//platforms/metal:metal.bzl", "metal_packages") +use_repo(metal, "libpjrt_metal") + non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") use_repo(non_module_deps, "arocc", "cloud_accelerator_diagnostics", "com_google_sentencepiece", "flashattn_linux_amd64", "flashattn_linux_arm64", "iree", "libvaxis", "linenoise", "mnist", "mosaic_tpu", "org_swig_swig", "stb", "translate-c", "uucode", "xla", "zigimg") diff --git a/examples/llm/models/common.zig b/examples/llm/models/common.zig index dcc2ac77..c5c6d332 100644 --- a/examples/llm/models/common.zig +++ b/examples/llm/models/common.zig @@ -59,7 +59,7 @@ pub const Shardings = struct { .experts = try platform.registerShardingWithStrategy("experts", .mesh(.{ .experts = .high_bandwidth }), strategy_experts), }; }, - .cuda, .rocm, .oneapi, .neuron, .cpu => return .{ + .cuda, .rocm, .oneapi, .neuron, .metal, .cpu => return .{ .model = try platform.registerSharding("model", .mesh(.{ .model = .high_bandwidth })), .experts = try platform.registerSharding("experts", .mesh(.{ .experts = .high_bandwidth })), }, diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index 3c150883..857c5171 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -542,6 +542,7 @@ const SelfAttn = struct { .cuda_fa2 => attention_metadata, .cuda_fa3 => attention_metadata, .nki => attention_metadata, + .metal_fa => attention_metadata, }; const attn_output = zml.attention.attention.attention( diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig index e4d15018..d336946f 100644 --- a/examples/llm/models/llama/session.zig +++ b/examples/llm/models/llama/session.zig @@ -62,7 +62,7 @@ pub const Session = struct { } }; break :b buffers; }, - .vanilla, .cuda_fa2, .cuda_fa3, .nki => null, + .vanilla, .cuda_fa2, .cuda_fa3, .nki, .metal_fa => null, }; errdefer if (decode_attention_metadata_buffers) |*buffers| zml.attention.attention.Metadata.deinitBuffer(buffers); @@ -161,12 +161,16 @@ pub const Session = struct { var prefill_tokens_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, prefill_tokens_slice, .replicated); defer prefill_tokens_buffer.deinit(); + var num_tokens_buffer: zml.Buffer = try .scalar(self.io, self.platform, all_tokens.len, .u32); + defer num_tokens_buffer.deinit(); + const attention_metadata_buffers: zml.Bufferized(zml.attention.attention.Metadata) = switch (self.compiled_model.params.prefill_attention_parameters) { .attnd => .{ .attnd = .{ .conversation_id = try zml.Buffer.scalar(self.io, self.platform, self.conversation_id, .u64), .layer_id = try zml.Buffer.scalar(self.io, self.platform, 0, .u16), .num_tokens = try zml.Buffer.scalar(self.io, self.platform, all_tokens.len, .u32), } }, + .metal_fa => .{ .metal_fa = .{ .num_tokens = num_tokens_buffer } }, .vanilla, .cuda_fa2, .cuda_fa3, .nki => self.attention_metadata_buffers, }; diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index ec0e462f..b5d8d526 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -8,6 +8,7 @@ PLATFORMS = { "tpu": False, "neuron": False, "oneapi": False, + "metal": False, } [ @@ -28,13 +29,29 @@ config_setting( visibility = ["//visibility:public"], ) +config_setting( + name = "metal.enabled", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:macos", + ], + flag_values = {":metal": "True"}, + visibility = ["//visibility:public"], +) + [ config_setting( name = "{}.enabled".format(runtime), flag_values = {":{}".format(runtime): "True"}, visibility = ["//visibility:public"], ) - for runtime in ["cpu", "cuda", "tpu", "neuron", "oneapi"] + for runtime in [ + "cpu", + "cuda", + "tpu", + "neuron", + "oneapi", + ] ] zig_library( diff --git a/platforms/metal/BUILD.bazel b/platforms/metal/BUILD.bazel new file mode 100644 index 00000000..a25996d1 --- /dev/null +++ b/platforms/metal/BUILD.bazel @@ -0,0 +1,42 @@ +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library") + +cc_library( + name = "empty", + defines = ["ZML_RUNTIME_METAL_DISABLED"], + tags = ["manual"], +) + +copy_to_directory( + name = "sandbox", + srcs = ["@libpjrt_metal"], + out = "sandbox/lib", + include_external_repositories = ["**"], +) + +cc_library( + name = "libpjrt_metal", + data = [":sandbox"], + defines = ["ZML_RUNTIME_METAL"], + tags = ["manual"], +) + +zig_library( + name = "metal", + import_name = "platforms/metal", + main = "metal.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//pjrt", + ] + select({ + "//platforms:metal.enabled": [ + ":libpjrt_metal", + "//bazel", + "//stdx", + "@rules_zig//zig/runfiles", + ], + "//conditions:default": [":empty"], + }), +) diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl new file mode 100644 index 00000000..1810bb95 --- /dev/null +++ b/platforms/metal/metal.bzl @@ -0,0 +1,29 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +_BUILD_FILE_CONTENT = """\ +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "libpjrt_metal", + srcs = ["libpjrt_c_api_gpu_plugin.dylib"], + visibility = ["@zml//platforms/metal:__subpackages__"], +) +""" + +def _metal_impl(mctx): + http_archive( + name = "libpjrt_metal", + build_file_content = _BUILD_FILE_CONTENT, + sha256 = "b2c7ce7c7a554e40cdcd48d27751f7dfcac0261def179e53ac07305032829421", + url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-06-23T00-20-00Z/pjrt-metal_macos-arm64.tar.zst", + ) + + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = ["libpjrt_metal"], + root_module_direct_dev_deps = [], + ) + +metal_packages = module_extension( + implementation = _metal_impl, +) diff --git a/platforms/metal/metal.zig b/platforms/metal/metal.zig new file mode 100644 index 00000000..014144a9 --- /dev/null +++ b/platforms/metal/metal.zig @@ -0,0 +1,39 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); +const c = @import("c"); +const pjrt = @import("pjrt"); +const stdx = @import("stdx"); + +const log = std.log.scoped(.@"zml/platforms/metal"); + +pub fn isEnabled() bool { + return @hasDecl(c, "ZML_RUNTIME_METAL"); +} + +pub fn load(_: std.mem.Allocator, io: std.Io) !*const pjrt.Api { + _ = io; + if (comptime !isEnabled()) { + return error.Unavailable; + } + + if (comptime builtin.os.tag != .macos) { + return error.Unavailable; + } + + const r = try bazel.runfiles(bazel_builtin.current_repository); + + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = try r.rlocation("zml/platforms/metal/sandbox/lib", &path_buf) orelse { + log.err("Failed to find sandbox path for Metal runtime", .{}); + return error.FileNotFound; + }; + + return blk: { + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "libpjrt_c_api_gpu_plugin.dylib" }); + break :blk pjrt.Api.loadFrom(path); + }; +} diff --git a/platforms/platforms.zig b/platforms/platforms.zig index 63a0a7b0..f618bd97 100644 --- a/platforms/platforms.zig +++ b/platforms/platforms.zig @@ -2,6 +2,7 @@ const std = @import("std"); const cpu = @import("platforms/cpu"); const cuda = @import("platforms/cuda"); +const metal = @import("platforms/metal"); const neuron = @import("platforms/neuron"); const pjrt = @import("pjrt"); const rocm = @import("platforms/rocm"); @@ -15,6 +16,7 @@ pub const Platform = enum { tpu, neuron, oneapi, + metal, }; pub fn load(allocator: std.mem.Allocator, io: std.Io, tag: Platform) !*const pjrt.Api { @@ -25,6 +27,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io, tag: Platform) !*const pjr .tpu => try tpu.load(allocator, io), .neuron => try neuron.load(allocator, io), .oneapi => try oneapi.load(allocator, io), + .metal => try metal.load(allocator, io), }; } @@ -36,5 +39,6 @@ pub fn isEnabled(tag: Platform) bool { .tpu => tpu.isEnabled(), .neuron => neuron.isEnabled(), .oneapi => oneapi.isEnabled(), + .metal => metal.isEnabled(), }; } diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 353df1d5..126da753 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -81,6 +81,7 @@ zig_library( "attention/attention.zig", "attention/attnd.zig", "attention/flashattn.zig", + "attention/metal_attention.zig", "attention/mosaic_tpu_kernels/ragged_attention.zig", "//zml/attention/nki:attention", "attention/paged_attention.zig", diff --git a/zml/Sharding.zig b/zml/Sharding.zig index 6017097d..42dc4d14 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -47,7 +47,7 @@ pub const Partitioner = union(enum) { pub fn fromTarget(target: Target) Partitioner { return switch (target) { - .cpu, .cuda, .rocm, .tpu, .oneapi, .neuron => .shardy, + .cpu, .cuda, .rocm, .tpu, .oneapi, .neuron, .metal => .shardy, }; } }; @@ -697,7 +697,7 @@ pub const PhysicalMesh = struct { .tpu => &.{ .link_x, .link_y, .link_z }, .neuron => &.{ .link, .link_x, .link_y, .link_z }, .cuda, .rocm => &.{.link}, - .cpu => &.{.bus}, + .cpu, .metal => &.{.bus}, .oneapi => &.{ .link, .bus }, }; } @@ -903,7 +903,7 @@ pub const PhysicalMesh = struct { .cuda, .rocm => gpu(allocator, platform_devices), .tpu => tpu(allocator, platform_devices), .neuron => return neuron(allocator, platform_devices), - .oneapi => cpu(allocator, platform_devices), + .oneapi, .metal => cpu(allocator, platform_devices), }; errdefer freeNode(allocator, root); diff --git a/zml/attention.zig b/zml/attention.zig index 74cd28e6..ca7b3da0 100644 --- a/zml/attention.zig +++ b/zml/attention.zig @@ -1,6 +1,7 @@ pub const attention = @import("attention/attention.zig"); pub const attnd = @import("attention/attnd.zig"); pub const flashattn = @import("attention/flashattn.zig"); +pub const metal = @import("attention/metal_attention.zig"); pub const paged_attention = @import("attention/paged_attention.zig"); pub const tpu = @import("attention/tpu_attention.zig"); pub const triton = @import("attention/triton_attention.zig"); diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index 570a9683..89ac817c 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -13,6 +13,7 @@ pub const Backend = enum { nki, cuda_fa2, cuda_fa3, + metal_fa, pub fn auto(platform: *const zml.Platform) Backend { return switch (platform.target) { @@ -29,6 +30,7 @@ pub const Backend = enum { break :b .vanilla; }, .neuron => .nki, + .metal => .metal_fa, .cpu, .rocm, .tpu, .oneapi => .vanilla, }; } @@ -40,6 +42,7 @@ pub const Parameters = union(Backend) { nki: nki.Parameters, cuda_fa2: flashattn.fa2.Parameters, cuda_fa3: flashattn.fa3.Parameters, + metal_fa: void, pub const InitOptions = union(Backend) { vanilla: void, @@ -47,6 +50,7 @@ pub const Parameters = union(Backend) { nki: nki.Parameters, cuda_fa2: flashattn.fa2.Parameters.InitOptions, cuda_fa3: flashattn.fa3.Parameters.InitOptions, + metal_fa: void, pub fn fromBackend(backend: Backend) InitOptions { return switch (backend) { @@ -55,6 +59,7 @@ pub const Parameters = union(Backend) { .nki => .{ .nki = .init() }, .cuda_fa2 => .{ .cuda_fa2 = .{} }, .cuda_fa3 => .{ .cuda_fa3 = .{} }, + .metal_fa => .{ .metal_fa = {} }, }; } }; @@ -66,16 +71,36 @@ pub const Parameters = union(Backend) { .nki => |v| .{ .nki = v }, .cuda_fa2 => |v| .{ .cuda_fa2 = .init(v) }, .cuda_fa3 => |v| .{ .cuda_fa3 = .init(v) }, + .metal_fa => .{ .metal_fa = {} }, }; } }; +pub const MetalFaMetadata = struct { + num_tokens: zml.Tensor, + + pub fn init() MetalFaMetadata { + return .{ .num_tokens = .fromShape(.scalar(.u32)) }; + } + + pub fn initBuffer(self: MetalFaMetadata, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(MetalFaMetadata) { + _ = self; + _ = sharding; + return .{ .num_tokens = try zml.Buffer.scalar(io, platform, 1, .u32) }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(MetalFaMetadata)) void { + self.num_tokens.deinit(); + } +}; + pub const Metadata = union(Backend) { vanilla: void, attnd: attnd.Metadata, nki: void, cuda_fa2: flashattn.fa2.Metadata, cuda_fa3: flashattn.fa3.Metadata, + metal_fa: MetalFaMetadata, pub const InitOptions = union(Backend) { vanilla: void, @@ -83,6 +108,7 @@ pub const Metadata = union(Backend) { nki: void, cuda_fa2: flashattn.fa2.Metadata.InitOptions, cuda_fa3: flashattn.fa3.Metadata.InitOptions, + metal_fa: void, pub fn fromBackend(backend: Backend, seqlen: i64, num_heads: i64) InitOptions { return switch (backend) { @@ -91,6 +117,7 @@ pub const Metadata = union(Backend) { .nki => .{ .nki = {} }, .cuda_fa2 => .{ .cuda_fa2 = .{ .seqlen = seqlen, .num_heads = num_heads } }, .cuda_fa3 => .{ .cuda_fa3 = .{ .seqlen = seqlen, .num_heads = num_heads } }, + .metal_fa => .{ .metal_fa = {} }, }; } }; @@ -102,6 +129,7 @@ pub const Metadata = union(Backend) { .nki => .{ .nki = {} }, .cuda_fa2 => |o| .{ .cuda_fa2 = flashattn.fa2.Metadata.init(o) }, .cuda_fa3 => |o| .{ .cuda_fa3 = flashattn.fa3.Metadata.init(o) }, + .metal_fa => .{ .metal_fa = .init() }, }; } @@ -112,6 +140,7 @@ pub const Metadata = union(Backend) { .nki => .{ .nki = {} }, .cuda_fa2 => |v| .{ .cuda_fa2 = try v.initBuffer(io, platform, sharding) }, .cuda_fa3 => |v| .{ .cuda_fa3 = try v.initBuffer(io, platform, sharding) }, + .metal_fa => |v| .{ .metal_fa = try v.initBuffer(io, platform, sharding) }, }; } @@ -122,6 +151,7 @@ pub const Metadata = union(Backend) { .nki => {}, .cuda_fa2 => |*v| flashattn.fa2.Metadata.deinitBuffer(v), .cuda_fa3 => |*v| flashattn.fa3.Metadata.deinitBuffer(v), + .metal_fa => |*v| MetalFaMetadata.deinitBuffer(v), } } }; @@ -143,5 +173,18 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T .nki => |params| nki.attention(q, k, v, token_index, params), .cuda_fa2 => flashattn.fa2.attention(q, k, v, token_index, metadata.cuda_fa2, parameters.cuda_fa2), .cuda_fa3 => flashattn.fa3.attention(q, k, v, token_index, metadata.cuda_fa3, parameters.cuda_fa3), + .metal_fa => b: { + const qc = q.transpose(.{ .h, .q, .hd }); + const kc = k.transpose(.{ .h, .k, .hd }); + const vc = v.transpose(.{ .h, .k, .hd }); + const tok_i32 = token_index.convert(.i32); + + const attn = if (q.dim(.q) > 1) + zml.ops.customCall("zml$flash_attn", .{ qc, kc, vc, tok_i32, metadata.metal_fa.num_tokens }, qc.shape(), .{}, .{ .has_side_effect = false }) + else + zml.ops.customCall("zml$flash_attn", .{ qc, kc, vc, tok_i32 }, qc.shape(), .{}, .{ .has_side_effect = false }); + + break :b attn.transpose(q.shape()); + }, }; } diff --git a/zml/attention/metal_attention.zig b/zml/attention/metal_attention.zig new file mode 100644 index 00000000..f5de61bf --- /dev/null +++ b/zml/attention/metal_attention.zig @@ -0,0 +1,96 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../zml.zig"); +const AttentionOptions = @import("paged_attention.zig").AttentionOptions; + +const log = std.log.scoped(.@"zml/attention/metal"); + +pub const paged = struct { + pub const Options = struct { + batch_size: usize, + max_num_pages: usize, + max_seqlen_q: usize, + is_prefill: bool, + + pub fn isPrefill(self: Options) bool { + return self.is_prefill; + } + + pub fn maxNumPages(self: Options) usize { + return self.max_num_pages; + } + }; + + pub const Parameters = struct { + block_table: zml.Tensor, + seq_lens: zml.Tensor, + query_start_len: zml.Tensor, + options_: Options, + + pub fn init(options_: Options) Parameters { + return .{ + .block_table = .init(.{ .b = options_.batch_size, .p = options_.max_num_pages }, .i32), + .seq_lens = .init(.{ .b = options_.batch_size }, .i32), + .query_start_len = .init(.{ .b = options_.batch_size + 1 }, .i32), + .options_ = options_, + }; + } + + pub fn allocationSize(self: Parameters) usize { + var allocation_size: usize = 0; + + allocation_size += self.block_table.byteSize(); + allocation_size += self.seq_lens.byteSize(); + allocation_size += self.query_start_len.byteSize(); + + return allocation_size; + } + + pub fn options(self: Parameters) Options { + return self.options_; + } + + pub fn onMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { + return .{ + .options_ = self.options_, + .block_table = self.block_table.onMemory(memory), + .seq_lens = self.seq_lens.onMemory(memory), + .query_start_len = self.query_start_len.onMemory(memory), + }; + } + + pub fn toMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { + return .{ + .options_ = self.options_, + .block_table = self.block_table.toMemory(memory), + .seq_lens = self.seq_lens.toMemory(memory), + .query_start_len = self.query_start_len.toMemory(memory), + }; + } + }; + + pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { + const num_kv_heads = q.dim(.hkv); + + const qh = q.merge(.{ .h = .{ .hkv, .hg } }).transpose(.{ .b, .h, .hd }); + const kc = k_cache.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const vc = v_cache.transpose(.{ .page, .k_chunk, .hkv, .hd }); + + const out = zml.ops.customCall( + "zml$paged_attn", + .{ qh, kc, vc, parameters.block_table, parameters.seq_lens, parameters.query_start_len }, + qh.shape(), + .{ + .is_causal = opts.is_causal, + .sliding_window = opts.sliding_window, + .scale = opts.scale, + }, + .{ .has_side_effect = false }, + ); + + // [total_q_tokens, num_heads, head_dim] -> {.b, .hkv, .hg, .hd} (q's order) + return out.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = .auto }).transpose(q.shape()); + } +}; diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index ff7f2ac7..b3045831 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -4,6 +4,7 @@ const stdx = @import("stdx"); const zml = @import("../zml.zig"); const flashattn = @import("flashattn.zig"); +const metal = @import("metal_attention.zig"); const tpu = @import("tpu_attention.zig"); const triton = @import("triton_attention.zig"); @@ -14,6 +15,7 @@ pub const Backend = enum { cuda_fa3, triton, mosaic_tpu, + metal, pub fn auto(platform: *const zml.Platform) Backend { return switch (platform.target) { @@ -21,6 +23,7 @@ pub const Backend = enum { .rocm => .triton, .oneapi => .triton, .tpu => .mosaic_tpu, + .metal => .metal, else => stdx.debug.panic("Paged attention is not supported on {s} yet", .{@tagName(platform.target)}), }; } @@ -31,6 +34,7 @@ pub const Options = union(Backend) { cuda_fa3: flashattn.paged_fa3.Options, triton: triton.paged.Options, mosaic_tpu: tpu.mosaic_tpu.Options, + metal: metal.paged.Options, const Args = struct { backend: Backend, @@ -109,6 +113,14 @@ pub const Options = union(Backend) { .is_prefill = args.is_prefill, }, }, + .metal => .{ + .metal = .{ + .batch_size = args.batch_size, + .max_num_pages = args.max_num_pages, + .max_seqlen_q = args.max_seqlen_q, + .is_prefill = args.is_prefill, + }, + }, .mosaic_tpu => .{ .mosaic_tpu = .{ .is_prefill = args.is_prefill, @@ -142,6 +154,7 @@ pub const Parameters = union(Backend) { cuda_fa3: flashattn.paged_fa3.Parameters, triton: triton.paged.Parameters, mosaic_tpu: tpu.mosaic_tpu.Parameters, + metal: metal.paged.Parameters, pub fn init(options_: Options) Parameters { return switch (options_) { @@ -149,6 +162,7 @@ pub const Parameters = union(Backend) { .cuda_fa3 => |cuda_fa3_options| .{ .cuda_fa3 = flashattn.paged_fa3.Parameters.init(cuda_fa3_options) }, .triton => |triton_options| .{ .triton = triton.paged.Parameters.init(triton_options) }, .mosaic_tpu => |mosaic_tpu_options| .{ .mosaic_tpu = tpu.mosaic_tpu.Parameters.init(mosaic_tpu_options) }, + .metal => |metal_options| .{ .metal = metal.paged.Parameters.init(metal_options) }, }; } @@ -158,6 +172,7 @@ pub const Parameters = union(Backend) { .cuda_fa3 => |v| .{ .cuda_fa3 = v.options() }, .triton => |v| .{ .triton = v.options() }, .mosaic_tpu => |v| .{ .mosaic_tpu = v.options() }, + .metal => |v| .{ .metal = v.options() }, }; } @@ -173,6 +188,7 @@ pub const Parameters = union(Backend) { .cuda_fa3 => |v| .{ .cuda_fa3 = v.onMemory(memory) }, .triton => |v| .{ .triton = v.onMemory(memory) }, .mosaic_tpu => |v| .{ .mosaic_tpu = v.onMemory(memory) }, + .metal => |v| .{ .metal = v.onMemory(memory) }, }; } @@ -182,6 +198,7 @@ pub const Parameters = union(Backend) { .cuda_fa3 => |v| .{ .cuda_fa3 = v.toMemory(memory) }, .triton => |v| .{ .triton = v.toMemory(memory) }, .mosaic_tpu => |v| .{ .mosaic_tpu = v.toMemory(memory) }, + .metal => |v| .{ .metal = v.toMemory(memory) }, }; } }; @@ -217,6 +234,10 @@ pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: z .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), }, .mosaic_tpu => |mosaic_tpu_parameters| tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, q, kv_cache.dense, opts), + .metal => |metal_parameters| switch (kv_cache) { + .split => |split| metal.paged.pagedAttention(metal_parameters, q, split.k, split.v, opts), + .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + }, }; } diff --git a/zml/exe.zig b/zml/exe.zig index 2de92efc..6bbd2e2b 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -266,7 +266,7 @@ pub const Exe = struct { const partition_events = events[0..@intCast(self.num_partitions)]; const events_slice: ?[]?*pjrt.Event = switch (self.platform.target) { .neuron => partition_events, - .cpu, .cuda, .rocm, .tpu, .oneapi => if (opts.wait) partition_events else null, + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal => if (opts.wait) partition_events else null, }; self.exe.execute(self.platform.pjrt_api, .{ @@ -294,7 +294,7 @@ pub const Exe = struct { } } }, - .cpu, .cuda, .rocm, .tpu, .oneapi => if (opts.wait) { + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal => if (opts.wait) { for (events_slice.?) |e| { if (e) |ev| { ev.await(self.platform.pjrt_api, io.?) catch unreachable; diff --git a/zml/io.zig b/zml/io.zig index 22c09f8d..18bcf1af 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -302,7 +302,7 @@ pub const MemoryWriter = union(enum) { ) !MemoryWriter { return switch (platform.target) { .cuda, .oneapi => .{ .direct = try DirectMemoryWriter.init(allocator, io, platform, pools, dma_allocators, dma_chunk_size, shape, sharding, buffer) }, - .rocm, .tpu, .neuron, .cpu => .{ .buffered = try BufferedMemoryWriter.init(allocator, io, platform, shape, sharding, buffer) }, + .rocm, .tpu, .neuron, .cpu, .metal => .{ .buffered = try BufferedMemoryWriter.init(allocator, io, platform, shape, sharding, buffer) }, }; } diff --git a/zml/mem.zig b/zml/mem.zig index c0a967a6..5048d0bc 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -23,7 +23,7 @@ pub const DmaAllocator = union(enum) { return switch (device.platform.target) { .cuda => .{ .dmam = .init(parent, device.platform) }, .oneapi, .tpu => .{ .uib = .init(device.memory(.host_pinned).?) }, - .rocm, .cpu, .neuron => .{ .passthrough = parent }, + .rocm, .cpu, .neuron, .metal => .{ .passthrough = parent }, }; } diff --git a/zml/ops.zig b/zml/ops.zig index d3c7851e..31bd71ee 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -714,7 +714,7 @@ pub fn neuronNki(inputs: anytype, outputs: anytype, opts: NeuronNkiOps) [outputs const ctx = CompilationContext.current(); switch (ctx.platform.target) { .neuron => {}, - .cpu, .cuda, .rocm, .tpu, .oneapi => { + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal => { stdx.debug.panic("neuronNki is only available on Neuron, got {s}", .{@tagName(ctx.platform.target)}); }, } @@ -1325,7 +1325,7 @@ pub const LoweringCompatibility = struct { pub fn preserveIntegerScalarBroadcast(self: Tensor, output_shape: Shape) ?Tensor { switch (CompilationContext.current().platform.target) { .neuron => {}, - .cpu, .cuda, .rocm, .tpu, .oneapi => return null, + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal => return null, } if (self.rank() != 0 or output_shape.rank() == 0 or !self.dtype().isInteger()) return null; @@ -1344,7 +1344,7 @@ pub const LoweringCompatibility = struct { pub fn preserveGatherFillSemantics(indices: []Tensor) void { switch (CompilationContext.current().platform.target) { .neuron => {}, - .cpu, .cuda, .rocm, .tpu, .oneapi => return, + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal => return, } const active_lanes = activeLanesForFillDropIndices(indices) orelse return; @@ -1356,7 +1356,7 @@ pub const LoweringCompatibility = struct { pub fn preserveScatterDropSemantics(indices: []Tensor, updates: anytype, opts: Tensor.ScatterOpts, update_values: *[updates.len]*const mlir.Value) void { const active_lanes: ?Tensor = switch (CompilationContext.current().platform.target) { .neuron => if (opts.update_fn == Tensor.ScatterOpts.increment) activeLanesForFillDropIndices(indices) else null, - .cpu, .cuda, .rocm, .tpu, .oneapi => null, + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal => null, }; if (active_lanes) |active| replaceInactiveIndirectIndices(indices, active); diff --git a/zml/platform.zig b/zml/platform.zig index 9199fc14..34ecb219 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -36,7 +36,7 @@ fn disableXlaLogs() void { fn validateDeviceCount(target: Target, num_devices: usize) !void { switch (target) { - .cpu, .cuda, .rocm, .tpu, .neuron, .oneapi => { + .cpu, .cuda, .rocm, .tpu, .neuron, .oneapi, .metal => { if (num_devices == 0) { log.err("Platform {} requires at least 1 device, got {}", .{ target, num_devices }); return error.ZeroVisibleDevices; @@ -98,7 +98,7 @@ pub const Memory = struct { }; return zml_kind == kind_; }, - .cpu, .neuron => return true, + .cpu, .neuron, .metal => return true, } } @@ -212,7 +212,7 @@ pub const Device = struct { fn platformDeviceSortId(target: Target, device: Device) usize { return switch (target) { .neuron => @intCast(device.localHardwareId()), - .cuda, .rocm, .tpu, .cpu, .oneapi => device.id(), + .cuda, .rocm, .tpu, .cpu, .oneapi, .metal => device.id(), }; } @@ -348,6 +348,7 @@ pub const Platform = struct { .rocm, .cuda, .oneapi, + .metal, .cpu, }; return for (ordered_targets) |target| { @@ -614,7 +615,7 @@ pub const Platform = struct { const default = platform.pjrt_client.defaultMemoryLayout(platform.pjrt_api, element_type, dims) catch @panic("Failed to get default memory layout"); return default.toMemoryLayout(); }, - .cuda, .rocm, .neuron, .oneapi, .cpu => .{ + .cuda, .rocm, .neuron, .oneapi, .cpu, .metal => .{ // If this is the default layout on the platform, there is no point calling PJRT .tiled = .{ .minor_to_major = constants.minorToMajor(@intCast(dims.len)), @@ -648,6 +649,7 @@ pub const CreateOptions = struct { tpu: struct {} = .{}, neuron: struct {} = .{}, oneapi: struct {} = .{}, + metal: struct {} = .{}, pub const Cpu = struct { device_count: u32, @@ -709,7 +711,7 @@ pub const CreateOptions = struct { values.shrinkRetainingCapacity(0); switch (target) { .cpu => self.cpu.writeNamedValues(&values), - .cuda, .rocm, .oneapi => self.xla_gpu.writeNamedValues(&values), + .cuda, .rocm, .oneapi, .metal => self.xla_gpu.writeNamedValues(&values), inline else => |t| { stdx.debug.assertComptime(@hasField(CreateOptions, @tagName(t)), "zml.platform.CreateOptions doesn't list target {s}", .{@tagName(t)}); const options = @field(self, @tagName(t)); diff --git a/zml/tensor.zig b/zml/tensor.zig index d62de5c6..af5e6c33 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -189,7 +189,7 @@ pub const Tensor = struct { pub fn toMemory(self: Tensor, kind: Memory.Kind) Tensor { const ctx = CompilationContext.current(); switch (ctx.platform.target) { - .cpu, .neuron => return self, + .cpu, .neuron, .metal => return self, .cuda, .rocm, .tpu, .oneapi => {}, } @@ -224,7 +224,7 @@ pub const Tensor = struct { pub fn onMemory(self: Tensor, kind: Memory.Kind) Tensor { const ctx = CompilationContext.current(); switch (ctx.platform.target) { - .cpu, .neuron => return self, + .cpu, .neuron, .metal => return self, .cuda, .rocm, .tpu, .oneapi => {}, } @@ -3283,7 +3283,7 @@ pub const Tensor = struct { } break :blk .{ .values = values, .indices = indices }; }, - .cpu, .cuda, .rocm, .tpu, .oneapi => blk: { + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal => blk: { var sorted = self.sort(a, .{ .descending = opts.descending }); sorted.values = sorted.values.slice1d(a, .{ .end = k }); sorted.indices = sorted.indices.slice1d(a, .{ .end = k }); @@ -4295,7 +4295,7 @@ pub const Tensor = struct { /// so it will slow down the program execution. pub fn print(input: Tensor, name: []const u8) void { switch (CompilationContext.current().platform.target) { - .cpu, .cuda, .rocm, .tpu => { + .cpu, .cuda, .rocm, .tpu, .metal => { ops.manualComputation(input, {}, .{ .name = name }, (struct { fn body(ctx_: anytype, _: std.mem.Allocator, sharded_input: Tensor, _: void) void { ops.customCall("zml$print", sharded_input, {}, .{ .name = ctx_.name }, .{ .has_side_effect = true }); From 1d71ee7bd39b33fa0cd5925cc51e3cc49ab5fc1b Mon Sep 17 00:00:00 2001 From: elogir Date: Wed, 24 Jun 2026 16:50:01 +0200 Subject: [PATCH 531/881] zml/safetensors: limit reads to 1GB (#597) On macOS, `pread` returns INT_MAX when the requested size is larger than 2GB --- zml/safetensors.zig | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/zml/safetensors.zig b/zml/safetensors.zig index 2b36ce78..94e8146d 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -128,13 +128,17 @@ pub const TensorReader = struct { self.file.close(self.io); } + // Limit single reads to 1GB to avoid issues on macOS where pread returns + // INT_MAX when the requested size is larger than 2GB + const max_single_read_bytes: u64 = 1 << 30; + fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { const self: *TensorReader = @fieldParentPtr("interface", r); if (self.remaining == 0) { return error.EndOfStream; } - const combined_limit = limit.min(.limited64(self.remaining)); + const combined_limit = limit.min(.limited64(self.remaining)).min(.limited64(max_single_read_bytes)); const n = try self.file_reader.interface.stream(w, combined_limit); self.remaining -= n; return n; @@ -158,7 +162,7 @@ pub const TensorReader = struct { return error.EndOfStream; } - const requested_bytes = limit.minInt64(self.remaining); + const requested_bytes = @min(limit.minInt64(self.remaining), max_single_read_bytes); if (self.padding_remaining > 0) { _ = try self.file_reader.interface.discard(.limited64(self.padding_remaining)); From d19949e25527d3a5425f4df6bcebbd6aafb0fc2a Mon Sep 17 00:00:00 2001 From: Olivier Boulant Date: Wed, 24 Jun 2026 17:45:39 +0200 Subject: [PATCH 532/881] zml/io/vfs(hf): introduce parallel read (#594) --------- Co-authored-by: Hugo Mano --- zml/io/vfs/hf.zig | 303 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 236 insertions(+), 67 deletions(-) diff --git a/zml/io/vfs/hf.zig b/zml/io/vfs/hf.zig index 0f38c98f..2423e638 100644 --- a/zml/io/vfs/hf.zig +++ b/zml/io/vfs/hf.zig @@ -6,6 +6,189 @@ const VFSBase = @import("base.zig").VFSBase; const log = std.log.scoped(.@"zml/io/vfs/hf"); +const ParallelRead = struct { + const Pool = struct { + pub const InitOpts = struct { + chunk_size: usize = 10 * 1024 * 1024, + num_workers: usize = 16, + queue_capacity: usize = 64, + }; + + group: std.Io.Group = .init, + job_queue: std.Io.Queue(Job), + queue_buf: []Job, + client: *std.http.Client, + chunk_size: usize, + + fn init(self: *Pool, allocator: std.mem.Allocator, io: std.Io, client: *std.http.Client, opts: InitOpts) !void { + stdx.debug.assert(opts.num_workers > 0, "Pool must have at least one worker", .{}); + stdx.debug.assert(opts.chunk_size > 0, "Pool must have a positive download chunk size", .{}); + stdx.debug.assert(opts.queue_capacity > 0, "Pool must have a positive queue capacity", .{}); + + const queue_buf = try allocator.alloc(Job, opts.queue_capacity); + errdefer allocator.free(queue_buf); + + self.* = .{ + .client = client, + .chunk_size = opts.chunk_size, + .job_queue = .init(queue_buf), + .queue_buf = queue_buf, + }; + errdefer self.group.cancel(io); + + for (0..opts.num_workers) |_| { + try self.group.concurrent(io, worker, .{ self, io }); + } + } + + fn deinit(self: *Pool, allocator: std.mem.Allocator, io: std.Io) void { + self.job_queue.close(io); + self.group.cancel(io); + + allocator.free(self.queue_buf); + } + + fn worker(pool: *Pool, io: std.Io) std.Io.Cancelable!void { + while (true) { + const job = pool.job_queue.getOne(io) catch break; + + if (!job.batch.failed()) { + if (job.perform(pool.client)) |err| job.batch.fail(err); + } + + job.batch.completeOne(io); + } + } + }; + + const Batch = struct { + uri: std.Uri, + url: []const u8, + authorization: std.http.Client.Request.Headers.Value, + pending: std.atomic.Value(u32), + err: std.atomic.Value(u16) = .init(0), + + fn failed(batch: *Batch) bool { + return batch.err.load(.acquire) != 0; + } + + fn fail(batch: *Batch, err: anyerror) void { + _ = batch.err.cmpxchgStrong(0, @intFromError(err), .release, .monotonic); + } + + fn firstError(batch: *Batch) ?anyerror { + const err = batch.err.load(.acquire); + return if (err == 0) null else @errorFromInt(err); + } + + fn completeOne(batch: *Batch, io: std.Io) void { + const previous = batch.pending.fetchSub(1, .release); + std.debug.assert(previous > 0); + if (previous == 1) io.futexWake(u32, &batch.pending.raw, 1); + } + + fn waitUncancelable(batch: *Batch, io: std.Io) void { + while (true) { + const pending = batch.pending.load(.acquire); + if (pending == 0) return; + io.futexWaitUncancelable(u32, &batch.pending.raw, pending); + } + } + }; + + pub const Job = struct { + data: []const []u8, + file_offset: u64, + chunk_offset: usize, + chunk_len: usize, + batch: *Batch, + + fn perform(job: Job, client: *std.http.Client) ?anyerror { + var range_buf: [64]u8 = undefined; + const range_header = std.fmt.bufPrint( + &range_buf, + "bytes={d}-{d}", + .{ job.file_offset, job.file_offset + @as(u64, @intCast(job.chunk_len - 1)) }, + ) catch unreachable; + + var req = client.request(.GET, job.batch.uri, .{ + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = job.batch.authorization, + }, + .extra_headers = &.{.{ .name = "Range", .value = range_header }}, + }) catch |err| return err; + defer req.deinit(); + + req.sendBodiless() catch |err| return err; + + var redirect_buffer: [8 * 1024]u8 = undefined; + var res = req.receiveHead(&redirect_buffer) catch |err| return err; + + if (res.head.status != .partial_content and res.head.status != .ok) { + log.err("Failed to perform read for {s}", .{job.batch.url}); + log.err("{s}", .{res.head.bytes}); + return error.RequestFailed; + } + + const content_range = blk: { + var it = res.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { + break :blk parseContentRange(header.value); + } + } + break :blk null; + }; + + const reader = res.reader(&.{}); + + if (content_range) |cr| { + if (cr.start > job.file_offset) return error.InvalidContentRange; + if (cr.start < job.file_offset) { + reader.discardAll(job.file_offset - cr.start) catch |err| return err; + } + } + + var remaining = job.chunk_len; + var skip = job.chunk_offset; + + for (job.data) |buf| { + if (skip >= buf.len) { + skip -= buf.len; + continue; + } + + const destination = buf[skip..][0..@min(remaining, buf.len - skip)]; + reader.readSliceAll(destination) catch |err| return err; + remaining -= destination.len; + if (remaining == 0) break; + skip = 0; + } + + return null; + } + }; + + const ContentRange = struct { + start: u64, + end: u64, + total: u64, + }; + + fn parseContentRange(value: []const u8) ?ContentRange { + const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; + const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; + const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; + + return .{ + .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, + .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, + .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, + }; + } +}; + pub const API = struct { const TREE_URL_TEMPLATE = "https://huggingface.co/api/models/{[repo]s}/{[model]s}/tree/{[rev]s}/{[path]s}?expand=false&recursive=true&limit=1000"; const LFS_FILE_URL_TEMPLATE = "https://huggingface.co/{[repo]s}/{[model]s}/resolve/{[rev]s}/{[path]s}"; @@ -50,6 +233,10 @@ const RepoKey = struct { }; pub const HF = struct { + pub const InitOpts = struct { + read_pool: ParallelRead.Pool.InitOpts = .{}, + }; + pub const Repo = struct { repo: []const u8, model: []const u8, @@ -102,14 +289,21 @@ pub const HF = struct { mutex: std.Io.Mutex = .init, client: *std.http.Client, authorization: std.http.Client.Request.Headers.Value, + read_pool: *ParallelRead.Pool, handles: stdx.SegmentedList(Handle, 0) = .{}, closed_handles: std.ArrayList(u32) = .empty, base: VFSBase, trees: std.StringHashMapUnmanaged(std.ArrayList(TreeNode)) = .{}, dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, - pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, hf_token: ?[]const u8) !HF { - return .{ + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, hf_token: ?[]const u8, opts: InitOpts) !HF { + const read_pool = try allocator.create(ParallelRead.Pool); + errdefer allocator.destroy(read_pool); + + try read_pool.init(allocator, inner, http_client, opts.read_pool); + errdefer read_pool.deinit(allocator, inner); + + var self: HF = .{ .allocator = allocator, .base = .init(inner), .client = http_client, @@ -120,7 +314,14 @@ pub const HF = struct { } else blk: { break :blk .default; }, + .read_pool = read_pool, }; + errdefer switch (self.authorization) { + .default, .omit => {}, + .override => |t| self.allocator.free(t), + }; + + return self; } pub fn auto(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, environ_map: *std.process.Environ.Map) !HF { @@ -145,10 +346,13 @@ pub const HF = struct { if (hf_token == null) log.warn("No Hugging Face authentication token found in environment or home config; proceeding without authentication.", .{}); - return init(allocator, inner, http_client, hf_token); + return init(allocator, inner, http_client, hf_token, .{}); } pub fn deinit(self: *HF) void { + self.read_pool.deinit(self.allocator, self.base.inner); + self.allocator.destroy(self.read_pool); + var idx: usize = 0; while (idx < self.handles.len) : (idx += 1) { const is_closed = for (self.closed_handles.items) |closed_idx| { @@ -698,79 +902,44 @@ pub const HF = struct { fn performRead(self: *HF, handle: *Handle, data: []const []u8, offset: u64) !usize { if (offset >= handle.size) return 0; - var range_buf: [64]u8 = undefined; - const range_header = blk: { - var total_bytes: u64 = 0; - for (data) |buf| { - total_bytes += @as(u64, buf.len); - } - const remaining = handle.size - offset; - const take = @min(remaining, total_bytes); - const end = offset + take - 1; - break :blk std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ offset, end }) catch unreachable; - }; + var read_size: usize = 0; + for (data) |buf| read_size += buf.len; + read_size = @intCast(@min(handle.size - offset, read_size)); + if (read_size == 0) return 0; + + const job_count = std.math.divCeil(usize, read_size, self.read_pool.chunk_size) catch unreachable; + + const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); + defer self.allocator.free(jobs); + const pending: u32 = std.math.cast(u32, job_count) orelse return error.SystemResources; const repo: Repo = try .parse(handle.uri); var url_buffer: [8 * 1024]u8 = undefined; - const url: []const u8 = try std.fmt.bufPrint(&url_buffer, API.LFS_FILE_URL_TEMPLATE, repo); - + const url = try std.fmt.bufPrint(&url_buffer, API.LFS_FILE_URL_TEMPLATE, repo); const uri: std.Uri = try .parse(url); - var req = try self.client.request(.GET, uri, .{ - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = self.authorization, - }, - .extra_headers = &.{.{ .name = "Range", .value = range_header }}, - }); - defer req.deinit(); - try req.sendBodiless(); - - var redirect_buffer: [8 * 1024]u8 = undefined; - var res = try req.receiveHead(&redirect_buffer); - - if (res.head.status != .partial_content and res.head.status != .ok) { - log.err("Failed to perform read for {s}", .{url}); - log.err("{s}", .{res.head.bytes}); - return error.RequestFailed; - } - - const content_range = blk: { - var it = res.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { - break :blk parseContentRange(header.value); - } - } - break :blk null; + var batch: ParallelRead.Batch = .{ + .uri = uri, + .url = url, + .authorization = self.authorization, + .pending = .init(pending), }; - const reader = res.reader(&.{}); - - if (content_range) |cr| { - if (cr.start < offset) { - try reader.discardAll(offset - cr.start); - } + for (jobs, 0..) |*job, i| { + const chunk_offset = i * self.read_pool.chunk_size; + job.* = .{ + .data = data, + .file_offset = offset + @as(u64, @intCast(chunk_offset)), + .chunk_offset = chunk_offset, + .chunk_len = @min(self.read_pool.chunk_size, read_size - chunk_offset), + .batch = &batch, + }; } - return try reader.readSliceShort(data[0]); - } + try self.read_pool.job_queue.putAll(self.base.inner, jobs); + batch.waitUncancelable(self.base.inner); + if (batch.firstError()) |err| return err; - const ContentRange = struct { - start: u64, - end: u64, - total: u64, - }; - - fn parseContentRange(value: []const u8) ?ContentRange { - const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; - const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; - const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; - - return .{ - .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, - .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, - .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, - }; + return read_size; } }; From f9ec63fd4e14c0ccb481f0b4cea656152b8e73fd Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Thu, 25 Jun 2026 14:17:46 +0200 Subject: [PATCH 533/881] workspace: Update to XLA @ 5e83525729fd21c4ad8d55073a3d047360d13f9c (#599) Updates to latest XLA which added support for bazel 8.x workspace and fixed a bunch of things that made bazel 9 impossible. 3 problematic changes with this bump: - PJRT headers were again incompatible with translate-c so a patch is made and PR has been done in XLA - PJRT introduced a proper type fo PJRT_Error so I used the InnerMixin pattern for those. - //tsl/platform/default package force evaluates Label("@cuda_nvcc") even for CPU builds and we don't have this repo. I commented for now until I find a solution on XLA side. Bumped a few deps necessary for this and additional llvm deps that are now evaluated as part of the llvm-project graph on XLA side (llvm_zlib and llvm_zstd). Next will be to investigate bazel 9 for real. --- MODULE.bazel | 11 +++---- MODULE.bazel.lock | 30 +++++++++++-------- pjrt/pjrt.zig | 16 +++++----- .../xla/cuda-root-path-local-defines.patch | 25 ++++++++++++++++ .../xla/pjrt-device-event-typedef.patch | 14 +++++++++ third_party/xla/repo.bzl | 7 ++++- third_party/xla/xla.bzl | 13 ++++---- 7 files changed, 84 insertions(+), 32 deletions(-) create mode 100644 third_party/xla/cuda-root-path-local-defines.patch create mode 100644 third_party/xla/pjrt-device-event-typedef.patch diff --git a/MODULE.bazel b/MODULE.bazel index 69f81b44..0c7ae4ec 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -11,7 +11,7 @@ bazel_dep(name = "bazel_skylib", version = "1.9.0") bazel_dep(name = "libbacktrace", version = "1.0.0-20250926-7939218") bazel_dep(name = "patchelf", version = "0.18.0.bcr.1") bazel_dep(name = "pcre2", version = "10.45") -bazel_dep(name = "platforms", version = "1.0.0") +bazel_dep(name = "platforms", version = "1.1.0") bazel_dep(name = "protobuf", version = "33.6", repo_name = "com_google_protobuf") single_version_override( module_name = "protobuf", @@ -20,7 +20,7 @@ single_version_override( bazel_dep(name = "pybind11_bazel", version = "3.0.1") bazel_dep(name = "re2", version = "2025-11-05", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_cc", version = "0.2.18") +bazel_dep(name = "rules_cc", version = "0.2.19") bazel_dep(name = "rules_distroless", version = "0.6.2") bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep(name = "rules_oci", version = "2.2.7") @@ -28,9 +28,10 @@ bazel_dep(name = "rules_proto", version = "7.1.0") bazel_dep(name = "rules_python", version = "1.9.0") bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "rules_zig", version = "0.15.1") - -bazel_dep(name = "tar.bzl", version = "0.9.0") -bazel_dep(name = "llvm", version = "0.8.3") +bazel_dep(name = "tar.bzl", version = "0.10.4") +bazel_dep(name = "llvm", version = "0.8.9") +bazel_dep(name = "zlib-ng", version = "2.3.3", repo_name = "llvm_zlib") +bazel_dep(name = "zstd", version = "1.5.7.bcr.1", repo_name = "llvm_zstd") bazel_dep(name = "with_cfg.bzl", version = "0.14.4") bazel_dep(name = "grpc", version = "1.74.0") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index d2b58fce..023f1366 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -56,7 +56,8 @@ "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", "https://bcr.bazel.build/modules/bazel_features/1.42.0/MODULE.bazel": "e8ca15cb2639c5f12183db6dcb678735555d0cdd739b32a0418b6532b5e565f8", "https://bcr.bazel.build/modules/bazel_features/1.43.0/MODULE.bazel": "defa2226f06ba20550d6548c3a2ea2a7929634437a52973869c20c225450eb91", - "https://bcr.bazel.build/modules/bazel_features/1.43.0/source.json": "1c4207dc858d6de0eecef30026793616bbf420c74aac27b6bad212534a730437", + "https://bcr.bazel.build/modules/bazel_features/1.47.0/MODULE.bazel": "e34df3cb35b1684cfa69923a61ae3803595babd3942cd306a488d51400886b30", + "https://bcr.bazel.build/modules/bazel_features/1.47.0/source.json": "4ba0b5138327f2d73352a51547a4e49a0a828ef400e046b15334d8905bf6b7ff", "https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b", "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", "https://bcr.bazel.build/modules/bazel_lib/3.0.0-beta.1/MODULE.bazel": "407729e232f611c3270005b016b437005daa7b1505826798ea584169a476e878", @@ -69,6 +70,7 @@ "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", "https://bcr.bazel.build/modules/bazel_skylib/1.2.1/MODULE.bazel": "f35baf9da0efe45fa3da1696ae906eea3d615ad41e2e3def4aeb4e8bc0ef9a7a", "https://bcr.bazel.build/modules/bazel_skylib/1.3.0/MODULE.bazel": "20228b92868bf5cfc41bda7afc8a8ba2a543201851de39d990ec957b513579c5", + "https://bcr.bazel.build/modules/bazel_skylib/1.4.0/MODULE.bazel": "2ab127ef8d56a739a99bb2ce00ec4c7d1ecc7977d4370c0ca6efd0d8f03d6d99", "https://bcr.bazel.build/modules/bazel_skylib/1.4.1/MODULE.bazel": "a0dcb779424be33100dcae821e9e27e4f2901d9dfd5333efe5ac6a8d7ab75e1d", "https://bcr.bazel.build/modules/bazel_skylib/1.4.2/MODULE.bazel": "3bd40978e7a1fac911d5989e6b09d8f64921865a45822d8b09e815eaa726a651", "https://bcr.bazel.build/modules/bazel_skylib/1.5.0/MODULE.bazel": "32880f5e2945ce6a03d1fbd588e9198c0a959bb42297b2cfaf1685b7bc32e138", @@ -152,8 +154,8 @@ "https://bcr.bazel.build/modules/libbacktrace/1.0.0-20250926-7939218/source.json": "83239989d09748f63dee4bdd430d261e217a012670e066e2238925a303a65d89", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", "https://bcr.bazel.build/modules/libpfm/4.11.0/source.json": "caaffb3ac2b59b8aac456917a4ecf3167d40478ee79f15ab7a877ec9273937c9", - "https://bcr.bazel.build/modules/llvm/0.8.3/MODULE.bazel": "dbc1bc13171f8f9980a75524883abef1c491613e8d8d805cd4ffa3767ba9708e", - "https://bcr.bazel.build/modules/llvm/0.8.3/source.json": "4b11874c26a9de53c03a25a517ebc36dee6b458c920fe9ed65e3ca279ff19775", + "https://bcr.bazel.build/modules/llvm/0.8.9/MODULE.bazel": "9e35ff5bcac996f9edc1b44b8f8baa58c7855e742c5608b07d925dcdbe642100", + "https://bcr.bazel.build/modules/llvm/0.8.9/source.json": "424b907b38875f1346b65946e5e482b202b842bd3a570abbe07047d2bc7d667b", "https://bcr.bazel.build/modules/mbedtls/3.6.0/MODULE.bazel": "8e380e4698107c5f8766264d4df92e36766248447858db28187151d884995a09", "https://bcr.bazel.build/modules/mbedtls/3.6.0/source.json": "1dbe7eb5258050afcc3806b9d43050f71c6f539ce0175535c670df606790b30c", "https://bcr.bazel.build/modules/nlohmann_json/3.11.3/MODULE.bazel": "87023db2f55fc3a9949c7b08dc711fae4d4be339a80a99d04453c4bb3998eefc", @@ -246,7 +248,8 @@ "https://bcr.bazel.build/modules/rules_cc/0.2.16/MODULE.bazel": "9242fa89f950c6ef7702801ab53922e99c69b02310c39fb6e62b2bd30df2a1d4", "https://bcr.bazel.build/modules/rules_cc/0.2.17/MODULE.bazel": "1849602c86cb60da8613d2de887f9566a6d354a6df6d7009f9d04a14402f9a84", "https://bcr.bazel.build/modules/rules_cc/0.2.18/MODULE.bazel": "4460ec36adc8f722a6a2a4ac9374cb91f2acebadaa93fc37966129afb3dece87", - "https://bcr.bazel.build/modules/rules_cc/0.2.18/source.json": "abad668ff2fd63ada1ac49bf386d37e27048b89a3465a6fd968bb832b00a09d3", + "https://bcr.bazel.build/modules/rules_cc/0.2.19/MODULE.bazel": "d5e0f05b63273281a16654eb6b1a8742a75ec153ac8b4f0419949d6e401e46f0", + "https://bcr.bazel.build/modules/rules_cc/0.2.19/source.json": "1ef48cdbd7aa6238015189b582d3d74ef0cbea3cb3e2cb259d782463f570c14a", "https://bcr.bazel.build/modules/rules_cc/0.2.4/MODULE.bazel": "1ff1223dfd24f3ecf8f028446d4a27608aa43c3f41e346d22838a4223980b8cc", "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", "https://bcr.bazel.build/modules/rules_distroless/0.6.2/MODULE.bazel": "eafe8b473c0c78038d9a6e1dbd9278b7bf83457ddc78207297483bfbe1012d56", @@ -361,7 +364,6 @@ "https://bcr.bazel.build/modules/tar.bzl/0.5.1/MODULE.bazel": "7c2eb3dcfc53b0f3d6f9acdfd911ca803eaf92aadf54f8ca6e4c1f3aee288351", "https://bcr.bazel.build/modules/tar.bzl/0.5.5/MODULE.bazel": "4bfab9bbc7a1966c2c5f7371f5848f5e2d27c465951b4435adc9aaf00ed681da", "https://bcr.bazel.build/modules/tar.bzl/0.6.0/MODULE.bazel": "a3584b4edcfafcabd9b0ef9819808f05b372957bbdff41601429d5fd0aac2e7c", - "https://bcr.bazel.build/modules/tar.bzl/0.9.0/MODULE.bazel": "452a22d7f02b1c9d7a22ab25edf20f46f3e1101f0f67dc4bfbf9a474ddf02445", "https://bcr.bazel.build/modules/upb/0.0.0-20211020-160625a/MODULE.bazel": "6cced416be2dc5b9c05efd5b997049ba795e5e4e6fafbe1624f4587767638928", "https://bcr.bazel.build/modules/upb/0.0.0-20230516-61a97ef/MODULE.bazel": "c0df5e35ad55e264160417fd0875932ee3c9dda63d9fccace35ac62f45e1b6f9", "https://bcr.bazel.build/modules/upb/0.0.0-20230907-e7430e6/MODULE.bazel": "3a7dedadf70346e678dc059dbe44d05cbf3ab17f1ce43a1c7a42edc7cbf93fd9", @@ -374,6 +376,8 @@ "https://bcr.bazel.build/modules/yq.bzl/0.1.1/MODULE.bazel": "9039681f9bcb8958ee2c87ffc74bdafba9f4369096a2b5634b88abc0eaefa072", "https://bcr.bazel.build/modules/yq.bzl/0.3.1/MODULE.bazel": "9bcb7151b3cd4681b89d350530eaf7b45e32a44dda94843b8932b0cb1cd4594a", "https://bcr.bazel.build/modules/yq.bzl/0.3.1/source.json": "f0b0f204a2a6b0e34b4c9541efe8c04f2ef1af65948daa784eccea738b21dbd2", + "https://bcr.bazel.build/modules/zlib-ng/2.3.3/MODULE.bazel": "0afa781a5f354b4fd811a5c9a086e111daf50e09d8e98ab1cf7c46eeacc33aac", + "https://bcr.bazel.build/modules/zlib-ng/2.3.3/source.json": "63314bf75a7683c75b7db365513e93dd1c13f865dacee676235310e8d92a7a4b", "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", "https://bcr.bazel.build/modules/zlib/1.2.13/MODULE.bazel": "aa6deb1b83c18ffecd940c4119aff9567cd0a671d7bba756741cb2ef043a29d5", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.1/MODULE.bazel": "6a9fe6e3fc865715a7be9823ce694ceb01e364c35f7a846bf0d2b34762bc066b", @@ -381,7 +385,9 @@ "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "eec517b5bbe5492629466e11dae908d043364302283de25581e3eb944326c4ca", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/source.json": "22bc55c47af97246cfc093d0acf683a7869377de362b5d1c552c2c2e16b7a806", "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198", - "https://bcr.bazel.build/modules/zlib/1.3/MODULE.bazel": "6a9c02f19a24dcedb05572b2381446e27c272cd383aed11d41d99da9e3167a72" + "https://bcr.bazel.build/modules/zlib/1.3/MODULE.bazel": "6a9c02f19a24dcedb05572b2381446e27c272cd383aed11d41d99da9e3167a72", + "https://bcr.bazel.build/modules/zstd/1.5.7.bcr.1/MODULE.bazel": "c5977176dd8555be7a9d598512ae0cae11831259c06f00a5e5e0037d5db4e3f5", + "https://bcr.bazel.build/modules/zstd/1.5.7.bcr.1/source.json": "aa95e0b5aac9d80195b9047d223beaf26b1948be55d49f0e803e180e9ccc6e75" }, "selectedYankedVersions": {}, "moduleExtensions": { @@ -455,7 +461,7 @@ }, "@@aspect_rules_py+//uv/private/tomltool:extension.bzl%tomltool": { "general": { - "bzlTransitiveDigest": "9BC6q6vdlG+lqF9L13wVxxakVkZBx8buKPFIndN6mHk=", + "bzlTransitiveDigest": "aqufRzOBX6GQKw+vjDEerVnbW8x1DyUJgtYc6AsJ3PQ=", "usagesDigest": "tQ76lH+Nn5OlAFLvBbnfvnCzNIiwGgY6tOumYZXH44w=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -537,7 +543,7 @@ }, "@@cel-spec+//:extensions.bzl%non_module_dependencies": { "general": { - "bzlTransitiveDigest": "VN89nCHLUuqo5KktrpNZqv8O+QKrrsGIqdoLPn91hhY=", + "bzlTransitiveDigest": "R7CVvqPh/Lj1IMR7Q95mJqs/QHMdtkq2v3pUA7iMInw=", "usagesDigest": "HFQJtQrL9nKaFZEjgwaHVMHALMW+cafu696xy7J4ueM=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -734,7 +740,7 @@ }, "@@envoy_api+//bazel:repositories.bzl%non_module_deps": { "general": { - "bzlTransitiveDigest": "JQHl7KCG4m+2+/1IddYzhdiav5qs2iUe+gkg5f+kUPA=", + "bzlTransitiveDigest": "lCuDUcaPC1MF1i415aUqG0ITUkf8b20feV/sPZCKZNg=", "usagesDigest": "OP2p9QPxt4WFY/lah8Iecqt6SOUwZHs89hYRyaV+rSg=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -1048,7 +1054,7 @@ }, "@@rules_apple+//apple:extensions.bzl%non_module_deps": { "general": { - "bzlTransitiveDigest": "fe3Htkk5VXHdsIrdLtxsT4oUgADmqgmxwx83bja4UqY=", + "bzlTransitiveDigest": "T8kvFvuOk8k1FnPb9/pjoArSxtviJhVIPLNo7oi3LWg=", "usagesDigest": "M3VqFpeTCo4qmrNKGZw0dxBHvTYDrfV3cscGzlSAhQ4=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -1360,7 +1366,7 @@ }, "@@rules_perl+//perl:extensions.bzl%perl_repositories": { "general": { - "bzlTransitiveDigest": "hVM1gWzR35NfpY87MWdNF1xPgjhXCgJv3lHedzj33TY=", + "bzlTransitiveDigest": "0vc/2RHDjju9Q1391ILlkclFi099x0+aF+aYoDr36HA=", "usagesDigest": "qSSNDdCNVxNhY36wMndEAFacdhR0ooLTmumfad0km9s=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -1606,7 +1612,7 @@ }, "@@rules_swift+//swift:extensions.bzl%non_module_deps": { "general": { - "bzlTransitiveDigest": "vwewHB1JdoOb38MyyeRQPIdGOkEKD52YNUYgGwmeOy8=", + "bzlTransitiveDigest": "nw2ZR52E52Bu14yqiZmfBMAg2hbF2GbIlmIxQNi7PiY=", "usagesDigest": "mhACFnrdMv9Wi0Mt67bxocJqviRkDSV+Ee5Mqdj5akA=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 81b5ccd3..0bcdbc5c 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -345,22 +345,24 @@ pub const ErrorCode = enum(c.PJRT_Error_Code) { }; pub const Error = opaque { + const inner = InnerMixin(c.PJRT_Error).inner; + pub fn deinit(self: *Error, api: *const Api) void { _ = api.call(.PJRT_Error_Destroy, .{ - .@"error" = @ptrCast(self), + .@"error" = self.inner(), }) catch unreachable; } pub fn getCode(self: *Error, api: *const Api) ErrorCode { const ret = api.call(.PJRT_Error_GetCode, .{ - .@"error" = @ptrCast(self), + .@"error" = self.inner(), }) catch unreachable; return @enumFromInt(ret.code); } pub fn getMessage(self: *Error, api: *const Api) []const u8 { const ret = api.call(.PJRT_Error_Message, .{ - .@"error" = @ptrCast(self), + .@"error" = self.inner(), }) catch unreachable; return ret.message[0..ret.message_size]; } @@ -492,7 +494,7 @@ pub const Client = opaque { .device_layout = @ptrCast(@constCast(&args.layout.toCStruct())), .host_buffer_semantics = @intFromEnum(args.host_buffer_semantics), .device = if (args.dst == .device) @ptrCast(@constCast(args.dst.device)) else null, - .memory = if (args.dst == .memory) @ptrCast(@constCast(args.dst.memory)) else null, + .memory = if (args.dst == .memory) args.dst.memory.inner() else null, }); return .{ @@ -578,7 +580,7 @@ pub const Client = opaque { .num_shape_specs = args.shape_specs.len, .device_layouts = if (args.device_layouts) |layouts| @ptrCast(@constCast(layouts)) else null, .num_device_layouts = if (args.device_layouts) |layouts| layouts.len else 0, - .memory = @ptrCast(@constCast(args.memory)), + .memory = args.memory.inner(), }); return @ptrCast(ret.transfer_manager.?); } @@ -659,7 +661,7 @@ pub const Client = opaque { .shape_element_type = @intFromEnum(args.element_type), .shape_layout = @ptrCast(&layout), .device = if (args.dst == .device) @ptrCast(@constCast(args.dst.device)) else null, - .memory = if (args.dst == .memory) @ptrCast(@constCast(args.dst.memory)) else null, + .memory = if (args.dst == .memory) args.dst.memory.inner() else null, }); return @ptrCast(ret.buffer.?); } @@ -1272,7 +1274,7 @@ pub const Buffer = opaque { pub fn copyToMemory(self: *const Buffer, api: *const Api, dst_memory: *const Memory) ApiError!*Buffer { const ret = try api.call(.PJRT_Buffer_CopyToMemory, .{ .buffer = self.inner(), - .dst_memory = @ptrCast(@constCast(dst_memory)), + .dst_memory = dst_memory.inner(), }); return @ptrCast(ret.dst_buffer); } diff --git a/third_party/xla/cuda-root-path-local-defines.patch b/third_party/xla/cuda-root-path-local-defines.patch new file mode 100644 index 00000000..c9ca6aa8 --- /dev/null +++ b/third_party/xla/cuda-root-path-local-defines.patch @@ -0,0 +1,25 @@ +diff --git a/xla/tsl/platform/default/BUILD b/xla/tsl/platform/default/BUILD +index f01bdfc67e..18a30d44a 100644 +--- a/xla/tsl/platform/default/BUILD ++++ b/xla/tsl/platform/default/BUILD +@@ -82,13 +82,13 @@ cc_library( + if_false = ["@cuda_nvcc//:nvvm"], + if_true = ["@cuda_nvvm//:nvvm"], + )), +- local_defines = [ +- "CUDA_NVCC_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_nvcc") + "\\\"", +- "CUDA_NVDISASM_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_nvdisasm") + "\\\"", +- "NVIDIA_NVSHMEM_REPO_NAME=\\\"" + get_canonical_repo_name("nvidia_nvshmem") + "\\\"", +- "CUDA_NVVM_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_nvvm") + "\\\"", +- "CUDA_CUDART_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_cudart") + "\\\"", +- ], ++ # local_defines = [ ++ # "CUDA_NVCC_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_nvcc") + "\\\"", ++ # "CUDA_NVDISASM_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_nvdisasm") + "\\\"", ++ # "NVIDIA_NVSHMEM_REPO_NAME=\\\"" + get_canonical_repo_name("nvidia_nvshmem") + "\\\"", ++ # "CUDA_NVVM_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_nvvm") + "\\\"", ++ # "CUDA_CUDART_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_cudart") + "\\\"", ++ # ], + tags = [ + "manual", + "no_oss", diff --git a/third_party/xla/pjrt-device-event-typedef.patch b/third_party/xla/pjrt-device-event-typedef.patch new file mode 100644 index 00000000..afacb555 --- /dev/null +++ b/third_party/xla/pjrt-device-event-typedef.patch @@ -0,0 +1,14 @@ +diff --git a/xla/pjrt/c/pjrt_c_api_device_event.h b/xla/pjrt/c/pjrt_c_api_device_event.h +index 5d33c05cf7..5f1dbbf6f1 100644 +--- a/xla/pjrt/c/pjrt_c_api_device_event.h ++++ b/xla/pjrt/c/pjrt_c_api_device_event.h +@@ -64,7 +64,7 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_DeviceEvent_FunctionTable, + + // A PJRT_DeviceEvent is a pair of pointers containing both type information + // and the actual opaque device event object. See: xla::PjRtDeviceEventRef. +-struct PJRT_DeviceEvent { ++typedef struct PJRT_DeviceEvent { + const struct PJRT_DeviceEvent_FunctionTable* vtable; + void* device_event; +-}; ++} PJRT_DeviceEvent; diff --git a/third_party/xla/repo.bzl b/third_party/xla/repo.bzl index d1af45ac..405f7bc9 100644 --- a/third_party/xla/repo.bzl +++ b/third_party/xla/repo.bzl @@ -4,5 +4,10 @@ def repo(): git_repository( name = "xla", remote = "https://github.com/openxla/xla.git", - commit = "47f005bb8150a13cb0217c2d7daf108bcdca34cc", + commit = "6d7284490a942e5be5a771ded9181f9c602fd692", + patches = [ + "//third_party/xla:pjrt-device-event-typedef.patch", + "//third_party/xla:cuda-root-path-local-defines.patch", + ], + patch_args = ["-p1"], ) diff --git a/third_party/xla/xla.bzl b/third_party/xla/xla.bzl index 84301e07..6634fa15 100644 --- a/third_party/xla/xla.bzl +++ b/third_party/xla/xla.bzl @@ -205,21 +205,20 @@ def _xla_impl(mctx): tf_http_archive( name = "com_github_grpc_grpc", - sha256 = "e2ace790a5f2d0f83259d1390a816a33b013ea34df2e86084d927e58daa4c5d9", - strip_prefix = "grpc-1.78.0", + sha256 = "41b695614b26652ff9e97ce50cfd4a6c7a3d45a9fe598d1454407746499bbf2c", + strip_prefix = "grpc-1.81.0", patch_file = ["//third_party/grpc:grpc.patch"], - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/refs/tags/v1.78.0.tar.gz"), + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/refs/tags/v1.81.0.tar.gz"), ) tf_vendored(name = "tsl", path = "third_party/tsl") tf_http_archive( name = "snappy", build_file = "//third_party:snappy.BUILD", - sha256 = "736aeb64d86566d2236ddffa2865ee5d7a82d26c9016b36218fcc27ea4f09f86", - strip_prefix = "snappy-1.2.1", - urls = tf_mirror_urls("https://github.com/google/snappy/archive/refs/tags/1.2.1.tar.gz"), + sha256 = "90f74bc1fbf78a6c56b3c4a082a05103b3a56bb17bca1a27e052ea11723292dc", + strip_prefix = "snappy-1.2.2", + urls = tf_mirror_urls("https://github.com/google/snappy/archive/refs/tags/1.2.2.tar.gz"), ) - _dummy_repos(mctx) return mctx.extension_metadata( From c023e26f2ca7568254f9b91ab37a110143682197 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 25 Jun 2026 14:32:44 +0200 Subject: [PATCH 534/881] zml/moe: enable moe for any compute capability (#600) --- zml/moe/moe.zig | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index 2b016866..e8715d46 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -19,22 +19,7 @@ pub const Backend = enum { pub fn auto(platform: *const zml.Platform, weights_dtype: zml.DataType) !Backend { return switch (platform.target) { - .cuda => b: { - const first_device = platform.pjrt_client.devices(platform.pjrt_api)[0]; - if (zml.platform.cuda.tryGetComputeCapabilities(platform, first_device)) |cc| { - if (std.mem.eql(u8, cc, "9.0") or - std.mem.eql(u8, cc, "10.0")) - { - break :b switch (weights_dtype) { - .bf16, .f16, .f8e4m3fn => .triton, - else => error.UnsupportedDataType, - }; - } - break :b error.UnsupportedComputeCapability; - } - break :b error.UnsupportedComputeCapability; - }, - .rocm => switch (weights_dtype) { + .cuda, .rocm => switch (weights_dtype) { .bf16, .f16, .f32 => .triton, else => error.UnsupportedDataType, }, From 33ced8fa078b3c7c8c709bd526ae17f22ed7238f Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Thu, 25 Jun 2026 15:57:35 +0200 Subject: [PATCH 535/881] zml/backend: add oneAPI as Triton backend selector for MoE (#603) --- zml/moe/moe.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index e8715d46..41527187 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -19,7 +19,7 @@ pub const Backend = enum { pub fn auto(platform: *const zml.Platform, weights_dtype: zml.DataType) !Backend { return switch (platform.target) { - .cuda, .rocm => switch (weights_dtype) { + .cuda, .rocm, .oneapi => switch (weights_dtype) { .bf16, .f16, .f32 => .triton, else => error.UnsupportedDataType, }, From aa90f92832046d39b0dcc453a1cb9c02813145a7 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Thu, 25 Jun 2026 19:37:51 +0200 Subject: [PATCH 536/881] workspace: bump to bazel 9.1.1 (#601) A few missing loads Remove all flipped flags or deperacted flags from .bazelrc Important: - Keep autoload rules_cc and rules_python because XLA still has transitive deps that are missing loads, PR in progress. --- .bazelrc | 24 +- .bazelversion | 2 +- MODULE.bazel.lock | 881 ++---------------- bazel/cc_import.bzl | 4 +- bin/zml-smi/BUILD.bazel | 1 + .../iree/futex-public-api-header.patch | 17 + third_party/iree/repo.bzl | 2 +- tools/BUILD.bazel | 1 + tools/neuron/BUILD.bazel | 2 + 9 files changed, 94 insertions(+), 840 deletions(-) create mode 100644 third_party/iree/futex-public-api-header.patch diff --git a/.bazelrc b/.bazelrc index 3d6ddbdb..2fa7d97f 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,15 +1,8 @@ # Faster hash function startup --digest_function=blake3 -# Disable WORKSPACE -common --enable_workspace=false - -# Don't need for now -common --lockfile_mode=update - # Needed by LLVM and/or XLA common --experimental_repo_remote_exec -common --experimental_remote_cache_eviction_retries=5 # Self explanatory common --enable_platform_specific_config @@ -27,19 +20,12 @@ common --cxxopt='-std=c++17' common --host_cxxopt='-std=c++17' common --enable_runfiles -# Ensure the local env doesn't leak into the build -common --incompatible_strict_action_env # Ensure rules_python doesn't rely on the system's Python for stage-1 bootstrap common --@rules_python//python/config_settings:bootstrap_impl=script # We don't to check the built files in the sandbox or repositories common --experimental_check_output_files=false -# Performance -common --legacy_external_runfiles=false - -common --experimental_output_directory_naming_scheme=diff_against_dynamic_baseline - # Mount the zig cache into the sandboxes build:macos --sandbox_add_mount_pair=/var/tmp build:linux --sandbox_add_mount_pair=/tmp @@ -47,17 +33,13 @@ build:linux --sandbox_add_mount_pair=/tmp # Ensure workers are sandboxed build --worker_sandboxing -# Allows symlinks with spaces -build --experimental_inprocess_symlink_creation - # Build in debug for now, by default common --compilation_mode=dbg # Various cc options -common --experimental_cc_static_library common --experimental_cc_shared_library common --incompatible_use_cc_configure_from_rules_cc -common --experimental_starlark_cc_import +common --incompatible_autoload_externally=+@rules_cc,+@rules_python,+@rules_shell,+@rules_java common --experimental_platform_in_output_dir @@ -67,9 +49,6 @@ build --experimental_inmemory_sandbox_stashes # clang by default common --compiler=clang -# Enable resolving proto compiler from toolchains -common --incompatible_enable_proto_toolchain_resolution - # Allow empty globs since some legacy rules common --incompatible_disallow_empty_glob=false @@ -137,7 +116,6 @@ common:remote --remote_download_toplevel common:remote --nobuild_runfile_links common:remote --remote_timeout=3600 common:remote --noexperimental_throttle_remote_action_building -common:remote --experimental_remote_execution_keepalive common:remote --remote_cache_compression common:remote --grpc_keepalive_time=30s common:remote --jobs=2000 diff --git a/.bazelversion b/.bazelversion index df5119ec..44931da2 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -8.7.0 +9.1.1 diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 023f1366..f7e89935 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1,5 +1,5 @@ { - "lockFileVersion": 24, + "lockFileVersion": 26, "registryFileHashes": { "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2", @@ -21,7 +21,10 @@ "https://bcr.bazel.build/modules/apple_support/1.11.1/MODULE.bazel": "1843d7cd8a58369a444fc6000e7304425fba600ff641592161d9f15b179fb896", "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", "https://bcr.bazel.build/modules/apple_support/1.17.1/MODULE.bazel": "655c922ab1209978a94ef6ca7d9d43e940cd97d9c172fb55f94d91ac53f8610b", + "https://bcr.bazel.build/modules/apple_support/1.21.0/MODULE.bazel": "ac1824ed5edf17dee2fdd4927ada30c9f8c3b520be1b5fd02a5da15bc10bff3e", + "https://bcr.bazel.build/modules/apple_support/1.21.1/MODULE.bazel": "5809fa3efab15d1f3c3c635af6974044bac8a4919c62238cce06acee8a8c11f1", "https://bcr.bazel.build/modules/apple_support/1.23.1/MODULE.bazel": "53763fed456a968cf919b3240427cf3a9d5481ec5466abc9d5dc51bc70087442", + "https://bcr.bazel.build/modules/apple_support/1.24.2/MODULE.bazel": "0e62471818affb9f0b26f128831d5c40b074d32e6dda5a0d3852847215a41ca4", "https://bcr.bazel.build/modules/apple_support/2.5.4/MODULE.bazel": "964e9268b37edb64e9a0fa525e5b5e535ba56bf63cadc68375f92e69fcb734f6", "https://bcr.bazel.build/modules/apple_support/2.5.4/source.json": "df85e46410ff71c4006a13dbdeeeee488feba563be78f3a9ed8497c1fb93fb5d", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", @@ -55,6 +58,7 @@ "https://bcr.bazel.build/modules/bazel_features/1.38.0/MODULE.bazel": "f9b8a9c890ebd216b4049fd12a31d3c2602e3403c7af636b04fbbd7453edc9c9", "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", "https://bcr.bazel.build/modules/bazel_features/1.42.0/MODULE.bazel": "e8ca15cb2639c5f12183db6dcb678735555d0cdd739b32a0418b6532b5e565f8", + "https://bcr.bazel.build/modules/bazel_features/1.42.1/MODULE.bazel": "275a59b5406ff18c01739860aa70ad7ccb3cfb474579411decca11c93b951080", "https://bcr.bazel.build/modules/bazel_features/1.43.0/MODULE.bazel": "defa2226f06ba20550d6548c3a2ea2a7929634437a52973869c20c225450eb91", "https://bcr.bazel.build/modules/bazel_features/1.47.0/MODULE.bazel": "e34df3cb35b1684cfa69923a61ae3803595babd3942cd306a488d51400886b30", "https://bcr.bazel.build/modules/bazel_features/1.47.0/source.json": "4ba0b5138327f2d73352a51547a4e49a0a828ef400e046b15334d8905bf6b7ff", @@ -89,8 +93,8 @@ "https://bcr.bazel.build/modules/boringssl/0.20241024.0/source.json": "d843092e682b84188c043ac742965d7f96e04c846c7e338187e03238674909a9", "https://bcr.bazel.build/modules/buildifier_prebuilt/8.5.1/MODULE.bazel": "77f2a1958d1d07376dd3ce3ae16540f2c1b01921c1fd21930827271260e75a66", "https://bcr.bazel.build/modules/buildifier_prebuilt/8.5.1/source.json": "ae9f3d9dc7bec033976cf47165a78788bebad2b5c272241063ae31bad8664fd4", - "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", - "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", + "https://bcr.bazel.build/modules/buildozer/8.5.1/MODULE.bazel": "a35d9561b3fc5b18797c330793e99e3b834a473d5fbd3d7d7634aafc9bdb6f8f", + "https://bcr.bazel.build/modules/buildozer/8.5.1/source.json": "e3386e6ff4529f2442800dee47ad28d3e6487f36a1f75ae39ae56c70f0cd2fbd", "https://bcr.bazel.build/modules/c-ares/1.15.0/MODULE.bazel": "ba0a78360fdc83f02f437a9e7df0532ad1fbaa59b722f6e715c11effebaa0166", "https://bcr.bazel.build/modules/c-ares/1.19.1/MODULE.bazel": "73bca21720772370ff91cc8e88bbbaf14897720c6473e87c1ddc0f848284c313", "https://bcr.bazel.build/modules/c-ares/1.19.1/source.json": "56bfa95b01e4e0012e90eaa9f1bab823c48c3af4454286b889b9cc74f5098da1", @@ -226,8 +230,9 @@ "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", "https://bcr.bazel.build/modules/rules_apple/3.16.0/MODULE.bazel": "0d1caf0b8375942ce98ea944be754a18874041e4e0459401d925577624d3a54a", - "https://bcr.bazel.build/modules/rules_apple/3.16.0/source.json": "d8b5fe461272018cc07cfafce11fe369c7525330804c37eec5a82f84cd475366", "https://bcr.bazel.build/modules/rules_apple/3.5.1/MODULE.bazel": "3d1bbf65ad3692003d36d8a29eff54d4e5c1c5f4bfb60f79e28646a924d9101c", + "https://bcr.bazel.build/modules/rules_apple/4.1.0/MODULE.bazel": "76e10fd4a48038d3fc7c5dc6e63b7063bbf5304a2e3bd42edda6ec660eebea68", + "https://bcr.bazel.build/modules/rules_apple/4.1.0/source.json": "8ee81e1708756f81b343a5eb2b2f0b953f1d25c4ab3d4a68dc02754872e80715", "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", "https://bcr.bazel.build/modules/rules_cc/0.0.10/MODULE.bazel": "ec1705118f7eaedd6e118508d3d26deba2a4e76476ada7e0e3965211be012002", "https://bcr.bazel.build/modules/rules_cc/0.0.13/MODULE.bazel": "0e8529ed7b323dad0775ff924d2ae5af7640b23553dfcd4d34344c7e7a867191", @@ -241,9 +246,11 @@ "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", "https://bcr.bazel.build/modules/rules_cc/0.1.1/MODULE.bazel": "2f0222a6f229f0bf44cd711dc13c858dad98c62d52bd51d8fc3a764a83125513", + "https://bcr.bazel.build/modules/rules_cc/0.1.2/MODULE.bazel": "557ddc3a96858ec0d465a87c0a931054d7dcfd6583af2c7ed3baf494407fd8d0", "https://bcr.bazel.build/modules/rules_cc/0.1.4/MODULE.bazel": "bb03a452a7527ac25a7518fb86a946ef63df860b9657d8323a0c50f8504fb0b9", "https://bcr.bazel.build/modules/rules_cc/0.1.5/MODULE.bazel": "88dfc9361e8b5ae1008ac38f7cdfd45ad738e4fa676a3ad67d19204f045a1fd8", "https://bcr.bazel.build/modules/rules_cc/0.2.0/MODULE.bazel": "b5c17f90458caae90d2ccd114c81970062946f49f355610ed89bebf954f5783c", + "https://bcr.bazel.build/modules/rules_cc/0.2.13/MODULE.bazel": "eecdd666eda6be16a8d9dc15e44b5c75133405e820f620a234acc4b1fdc5aa37", "https://bcr.bazel.build/modules/rules_cc/0.2.15/MODULE.bazel": "6a0a4a75a57aa6dc888300d848053a58c6b12a29f89d4304e1c41448514ec6e8", "https://bcr.bazel.build/modules/rules_cc/0.2.16/MODULE.bazel": "9242fa89f950c6ef7702801ab53922e99c69b02310c39fb6e62b2bd30df2a1d4", "https://bcr.bazel.build/modules/rules_cc/0.2.17/MODULE.bazel": "1849602c86cb60da8613d2de887f9566a6d354a6df6d7009f9d04a14402f9a84", @@ -282,12 +289,12 @@ "https://bcr.bazel.build/modules/rules_java/7.3.2/MODULE.bazel": "50dece891cfdf1741ea230d001aa9c14398062f2b7c066470accace78e412bc2", "https://bcr.bazel.build/modules/rules_java/7.4.0/MODULE.bazel": "a592852f8a3dd539e82ee6542013bf2cadfc4c6946be8941e189d224500a8934", "https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe", - "https://bcr.bazel.build/modules/rules_java/8.14.0/MODULE.bazel": "717717ed40cc69994596a45aec6ea78135ea434b8402fb91b009b9151dd65615", - "https://bcr.bazel.build/modules/rules_java/8.14.0/source.json": "8a88c4ca9e8759da53cddc88123880565c520503321e2566b4e33d0287a3d4bc", "https://bcr.bazel.build/modules/rules_java/8.5.1/MODULE.bazel": "d8a9e38cc5228881f7055a6079f6f7821a073df3744d441978e7a43e20226939", "https://bcr.bazel.build/modules/rules_java/8.6.0/MODULE.bazel": "9c064c434606d75a086f15ade5edb514308cccd1544c2b2a89bbac4310e41c71", "https://bcr.bazel.build/modules/rules_java/8.6.1/MODULE.bazel": "f4808e2ab5b0197f094cabce9f4b006a27766beb6a9975931da07099560ca9c2", "https://bcr.bazel.build/modules/rules_java/8.8.0/MODULE.bazel": "de589d0880911ac007abd521b9f0ddcd8b0dbd05c8553e6f8124a050b83acf7d", + "https://bcr.bazel.build/modules/rules_java/9.1.0/MODULE.bazel": "ee63f27e36a3fada80342869361182f120a9819c74320e8e65b1e04ba0cd7a9d", + "https://bcr.bazel.build/modules/rules_java/9.1.0/source.json": "da589573c1dee2c9ac4a568b301269a2e8191110ff0345c1a959fa7ea6c4dfd6", "https://bcr.bazel.build/modules/rules_jvm_external/5.2/MODULE.bazel": "d9351ba35217ad0de03816ef3ed63f89d411349353077348a45348b096615036", "https://bcr.bazel.build/modules/rules_jvm_external/5.3/MODULE.bazel": "bf93870767689637164657731849fb887ad086739bd5d360d90007a581d5527d", "https://bcr.bazel.build/modules/rules_jvm_external/6.0/MODULE.bazel": "37c93a5a78d32e895d52f86a8d0416176e915daabd029ccb5594db422e87c495", @@ -332,6 +339,7 @@ "https://bcr.bazel.build/modules/rules_python/0.40.0/MODULE.bazel": "9d1a3cd88ed7d8e39583d9ffe56ae8a244f67783ae89b60caafc9f5cf318ada7", "https://bcr.bazel.build/modules/rules_python/1.0.0/MODULE.bazel": "898a3d999c22caa585eb062b600f88654bf92efb204fa346fb55f6f8edffca43", "https://bcr.bazel.build/modules/rules_python/1.2.0/MODULE.bazel": "5aeeb48b2a6c19d668b48adf2b8a2b209a6310c230db0ce77450f148a89846e4", + "https://bcr.bazel.build/modules/rules_python/1.3.0/MODULE.bazel": "8361d57eafb67c09b75bf4bbe6be360e1b8f4f18118ab48037f2bd50aa2ccb13", "https://bcr.bazel.build/modules/rules_python/1.5.1/MODULE.bazel": "acfe65880942d44a69129d4c5c3122d57baaf3edf58ae5a6bd4edea114906bf5", "https://bcr.bazel.build/modules/rules_python/1.6.0/MODULE.bazel": "7e04ad8f8d5bea40451cf80b1bd8262552aa73f841415d20db96b7241bd027d8", "https://bcr.bazel.build/modules/rules_python/1.7.0/MODULE.bazel": "d01f995ecd137abf30238ad9ce97f8fc3ac57289c8b24bd0bf53324d937a14f8", @@ -345,7 +353,9 @@ "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", "https://bcr.bazel.build/modules/rules_swift/1.18.0/MODULE.bazel": "a6aba73625d0dc64c7b4a1e831549b6e375fbddb9d2dde9d80c9de6ec45b24c9", "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", - "https://bcr.bazel.build/modules/rules_swift/2.1.1/source.json": "40fc69dfaac64deddbb75bd99cdac55f4427d9ca0afbe408576a65428427a186", + "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", + "https://bcr.bazel.build/modules/rules_swift/3.1.2/MODULE.bazel": "72c8f5cf9d26427cee6c76c8e3853eb46ce6b0412a081b2b6db6e8ad56267400", + "https://bcr.bazel.build/modules/rules_swift/3.1.2/source.json": "e85761f3098a6faf40b8187695e3de6d97944e98abd0d8ce579cb2daf6319a66", "https://bcr.bazel.build/modules/rules_zig/0.15.1/MODULE.bazel": "f0b813d9191c8d201e0b3ecbe5c5220ddedb717a15009fc711a78b3e1db8710d", "https://bcr.bazel.build/modules/rules_zig/0.15.1/source.json": "0bd91d7459f504196dfdc90cd0fffbf1f0df2667c9cd5eb708feb3fd27578e6a", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", @@ -357,7 +367,8 @@ "https://bcr.bazel.build/modules/stardoc/0.7.2/MODULE.bazel": "fc152419aa2ea0f51c29583fab1e8c99ddefd5b3778421845606ee628629e0e5", "https://bcr.bazel.build/modules/stardoc/0.7.2/source.json": "58b029e5e901d6802967754adf0a9056747e8176f017cfe3607c0851f4d42216", "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "5e463fbfba7b1701d957555ed45097d7f984211330106ccd1352c6e0af0dcf91", - "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/source.json": "32bd87e5f4d7acc57c5b2ff7c325ae3061d5e242c0c4c214ae87e0f1c13e54cb", + "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.2/MODULE.bazel": "75aab2373a4bbe2a1260b9bf2a1ebbdbf872d3bd36f80bff058dccd82e89422f", + "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.2/source.json": "5fba48bbe0ba48761f9e9f75f92876cafb5d07c0ce059cc7a8027416de94a05b", "https://bcr.bazel.build/modules/tar.bzl/0.10.4/MODULE.bazel": "e8f9ff79199e8d9eaad7f1b0a77ad74b30bb82d794b87d8ca942bead5de83ae9", "https://bcr.bazel.build/modules/tar.bzl/0.10.4/source.json": "20143442376c03426f6135292ba02d825cb75308aa47e6bf42dd4cc5a435c2ff", "https://bcr.bazel.build/modules/tar.bzl/0.2.1/MODULE.bazel": "52d1c00a80a8cc67acbd01649e83d8dd6a9dc426a6c0b754a04fe8c219c76468", @@ -393,11 +404,13 @@ "moduleExtensions": { "@@aspect_rules_py+//py:extensions.bzl%py_tools": { "general": { - "bzlTransitiveDigest": "pI39mvkEpRkqO9fPgwuZp2cFsNe/MqGRNr3gPBzTHaQ=", + "bzlTransitiveDigest": "LaqXRQ7hUz2pKp1jB+4lExnjHXPJFKoz6+nOJqccLrQ=", "usagesDigest": "9BiP3ASHlGl4doGhzXJwK17taNzv/q6x/IZ4Hb6LxNY=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, + "recordedInputs": [ + "REPO_MAPPING:aspect_rules_py+,aspect_tools_telemetry_report aspect_tools_telemetry++telemetry+aspect_tools_telemetry_report", + "REPO_MAPPING:aspect_rules_py+,bazel_skylib bazel_skylib+", + "REPO_MAPPING:aspect_rules_py+,bazel_tools bazel_tools" + ], "generatedRepoSpecs": { "rules_py_tools.darwin_amd64": { "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", @@ -439,83 +452,17 @@ "downloaded_file_path": "pex-2.3.1-py2.py3-none-any.whl" } } - }, - "recordedRepoMappingEntries": [ - [ - "aspect_rules_py+", - "aspect_tools_telemetry_report", - "aspect_tools_telemetry++telemetry+aspect_tools_telemetry_report" - ], - [ - "aspect_rules_py+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "aspect_rules_py+", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@aspect_rules_py+//uv/private/tomltool:extension.bzl%tomltool": { - "general": { - "bzlTransitiveDigest": "aqufRzOBX6GQKw+vjDEerVnbW8x1DyUJgtYc6AsJ3PQ=", - "usagesDigest": "tQ76lH+Nn5OlAFLvBbnfvnCzNIiwGgY6tOumYZXH44w=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "toml2json_aarch64_osx_libsystem": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_darwin_arm64", - "sha256": "9adc44d976e8f5baf5a6d613ceb3db6e5f56b2e22e75ac56521fdce62f227d88", - "executable": true - } - }, - "toml2json_x86_64_osx_libsystem": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_darwin_amd64", - "sha256": "425bac4015394fd9840fabcd52ac60fc796a9c10655b36c57e36ffb7dc9f3dd4", - "executable": true - } - }, - "toml2json_aarch64_linux_gnu": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_linux_arm64", - "sha256": "be8376c8e3232a242eae0d187f741a475498a949b942361a9af6e95072ef5670", - "executable": true - } - }, - "toml2json_x86_64_linux_gnu": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_linux_amd64", - "sha256": "f3dd54fabf2d27d0c027b0421860e5d9d909080be1613f0ffd87057633b65e9a", - "executable": true - } - } - }, - "recordedRepoMappingEntries": [ - [ - "aspect_rules_py+", - "bazel_tools", - "bazel_tools" - ] - ] + } } }, "@@aspect_tools_telemetry+//:extension.bzl%telemetry": { "general": { "bzlTransitiveDigest": "MePriaXmQNSqUfE+YQvEtzBc2bU1mjITINIffFiZYgo=", "usagesDigest": "pyph4lSDn5CHIn4s2mtKJTkRaLV/o2zIFgnrpufr/8o=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, + "recordedInputs": [ + "REPO_MAPPING:aspect_tools_telemetry+,bazel_lib bazel_lib+", + "REPO_MAPPING:aspect_tools_telemetry+,bazel_skylib bazel_skylib+" + ], "generatedRepoSpecs": { "aspect_tools_telemetry_report": { "repoRuleId": "@@aspect_tools_telemetry+//:extension.bzl%tel_repository", @@ -526,280 +473,14 @@ } } } - }, - "recordedRepoMappingEntries": [ - [ - "aspect_tools_telemetry+", - "bazel_lib", - "bazel_lib+" - ], - [ - "aspect_tools_telemetry+", - "bazel_skylib", - "bazel_skylib+" - ] - ] - } - }, - "@@cel-spec+//:extensions.bzl%non_module_dependencies": { - "general": { - "bzlTransitiveDigest": "R7CVvqPh/Lj1IMR7Q95mJqs/QHMdtkq2v3pUA7iMInw=", - "usagesDigest": "HFQJtQrL9nKaFZEjgwaHVMHALMW+cafu696xy7J4ueM=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "com_google_googleapis": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "sha256": "bd8e735d881fb829751ecb1a77038dda4a8d274c45490cb9fcf004583ee10571", - "strip_prefix": "googleapis-07c27163ac591955d736f3057b1619ece66f5b99", - "urls": [ - "https://github.com/googleapis/googleapis/archive/07c27163ac591955d736f3057b1619ece66f5b99.tar.gz" - ] - } - } - }, - "recordedRepoMappingEntries": [ - [ - "cel-spec+", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@cel-spec+//:googleapis_ext.bzl%googleapis_ext": { - "general": { - "bzlTransitiveDigest": "65z4sd8o73L2a8X2Pd3yAjn73l9HBG45JVnVY2DUbzI=", - "usagesDigest": "Ek7VfZ+tuyRBx/1h5wcmtnW9EGpOb0dkXUwBluZbD8k=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "com_google_googleapis_imports": { - "repoRuleId": "@@cel-spec++non_module_dependencies+com_google_googleapis//:repository_rules.bzl%switched_rules", - "attributes": { - "rules": { - "proto_library_with_info": [ - "", - "" - ], - "moved_proto_library": [ - "", - "" - ], - "java_proto_library": [ - "", - "" - ], - "java_grpc_library": [ - "", - "" - ], - "java_gapic_library": [ - "", - "" - ], - "java_gapic_test": [ - "", - "" - ], - "java_gapic_assembly_gradle_pkg": [ - "", - "" - ], - "py_proto_library": [ - "", - "" - ], - "py_grpc_library": [ - "", - "" - ], - "py_gapic_library": [ - "", - "" - ], - "py_test": [ - "", - "" - ], - "py_gapic_assembly_pkg": [ - "", - "" - ], - "py_import": [ - "", - "" - ], - "go_proto_library": [ - "", - "" - ], - "go_library": [ - "", - "" - ], - "go_test": [ - "", - "" - ], - "go_gapic_library": [ - "", - "" - ], - "go_gapic_assembly_pkg": [ - "", - "" - ], - "cc_proto_library": [ - "native.cc_proto_library", - "" - ], - "cc_grpc_library": [ - "", - "" - ], - "cc_gapic_library": [ - "", - "" - ], - "php_proto_library": [ - "", - "php_proto_library" - ], - "php_grpc_library": [ - "", - "php_grpc_library" - ], - "php_gapic_library": [ - "", - "php_gapic_library" - ], - "php_gapic_assembly_pkg": [ - "", - "php_gapic_assembly_pkg" - ], - "nodejs_gapic_library": [ - "", - "typescript_gapic_library" - ], - "nodejs_gapic_assembly_pkg": [ - "", - "typescript_gapic_assembly_pkg" - ], - "ruby_proto_library": [ - "", - "" - ], - "ruby_grpc_library": [ - "", - "" - ], - "ruby_ads_gapic_library": [ - "", - "" - ], - "ruby_cloud_gapic_library": [ - "", - "" - ], - "ruby_gapic_assembly_pkg": [ - "", - "" - ], - "csharp_proto_library": [ - "", - "" - ], - "csharp_grpc_library": [ - "", - "" - ], - "csharp_gapic_library": [ - "", - "" - ], - "csharp_gapic_assembly_pkg": [ - "", - "" - ] - } - } - } - }, - "recordedRepoMappingEntries": [ - [ - "cel-spec+", - "com_google_googleapis", - "cel-spec++non_module_dependencies+com_google_googleapis" - ] - ] - } - }, - "@@envoy_api+//bazel:repositories.bzl%non_module_deps": { - "general": { - "bzlTransitiveDigest": "lCuDUcaPC1MF1i415aUqG0ITUkf8b20feV/sPZCKZNg=", - "usagesDigest": "OP2p9QPxt4WFY/lah8Iecqt6SOUwZHs89hYRyaV+rSg=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "prometheus_metrics_model": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/prometheus/client_model/archive/v0.6.1.tar.gz" - ], - "sha256": "b9b690bc35d80061f255faa7df7621eae39fe157179ccd78ff6409c3b004f05e", - "strip_prefix": "client_model-0.6.1", - "build_file_content": "\nload(\"@envoy_api//bazel:api_build_system.bzl\", \"api_cc_py_proto_library\")\nload(\"@io_bazel_rules_go//proto:def.bzl\", \"go_proto_library\")\n\napi_cc_py_proto_library(\n name = \"client_model\",\n srcs = [\n \"io/prometheus/client/metrics.proto\",\n ],\n visibility = [\"//visibility:public\"],\n)\n\ngo_proto_library(\n name = \"client_model_go_proto\",\n importpath = \"github.com/prometheus/client_model/go\",\n proto = \":client_model\",\n visibility = [\"//visibility:public\"],\n)\n" - } - }, - "com_github_bufbuild_buf": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/bufbuild/buf/releases/download/v1.49.0/buf-Linux-x86_64.tar.gz" - ], - "sha256": "ee8da9748249f7946d79191e36469ce7bc3b8ba80019bff1fa4289a44cbc23bf", - "strip_prefix": "buf", - "build_file_content": "\npackage(\n default_visibility = [\"//visibility:public\"],\n)\n\nfilegroup(\n name = \"buf\",\n srcs = [\n \"@com_github_bufbuild_buf//:bin/buf\",\n ],\n tags = [\"manual\"], # buf is downloaded as a linux binary; tagged manual to prevent build for non-linux users\n)\n" - } - }, - "envoy_toolshed": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/envoyproxy/toolshed/archive/bazel-v0.2.2.tar.gz" - ], - "sha256": "443fe177aba0cef8c17b7a48905c925c67b09005b10dd70ff12cd9f729a72d51", - "strip_prefix": "toolshed-bazel-v0.2.2/bazel" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "envoy_api+", - "bazel_tools", - "bazel_tools" - ], - [ - "envoy_api+", - "envoy_api", - "envoy_api+" - ] - ] + } } }, "@@googleapis+//:extensions.bzl%switched_rules": { "general": { "bzlTransitiveDigest": "JLpojYeGHLdgnwSUuQWOrtkL9UMygBUUEFJZh7ASlmA=", "usagesDigest": "66rLsrgz6YqDpU0pD+02mTGNxYxhb/of6Fh0ulHzjdE=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, + "recordedInputs": [], "generatedRepoSpecs": { "com_google_googleapis_imports": { "repoRuleId": "@@googleapis+//:repository_rules.bzl%switched_rules", @@ -956,137 +637,16 @@ } } } - }, - "recordedRepoMappingEntries": [] - } - }, - "@@rules_apple+//apple:apple.bzl%provisioning_profile_repository_extension": { - "general": { - "bzlTransitiveDigest": "4qt/sABmG3x78RRuugrSdt7saJl2/iSXQeuIJpyeVjY=", - "usagesDigest": "vsJl8Rw5NL+5Ag2wdUDoTeRF/5klkXO8545Iy7U1Q08=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "local_provisioning_profiles": { - "repoRuleId": "@@rules_apple+//apple/internal:local_provisioning_profiles.bzl%provisioning_profile_repository", - "attributes": {} - } - }, - "recordedRepoMappingEntries": [ - [ - "apple_support+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "bazel_tools", - "rules_cc", - "rules_cc+" - ], - [ - "rules_apple+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "rules_apple+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_apple+", - "build_bazel_apple_support", - "apple_support+" - ], - [ - "rules_apple+", - "build_bazel_rules_swift", - "rules_swift+" - ], - [ - "rules_cc+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_cc+", - "cc_compatibility_proxy", - "rules_cc++compatibility_proxy+cc_compatibility_proxy" - ], - [ - "rules_cc+", - "rules_cc", - "rules_cc+" - ], - [ - "rules_cc++compatibility_proxy+cc_compatibility_proxy", - "rules_cc", - "rules_cc+" - ], - [ - "rules_swift+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "rules_swift+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_swift+", - "build_bazel_apple_support", - "apple_support+" - ], - [ - "rules_swift+", - "build_bazel_rules_swift", - "rules_swift+" - ], - [ - "rules_swift+", - "build_bazel_rules_swift_local_config", - "rules_swift++non_module_deps+build_bazel_rules_swift_local_config" - ] - ] - } - }, - "@@rules_apple+//apple:extensions.bzl%non_module_deps": { - "general": { - "bzlTransitiveDigest": "T8kvFvuOk8k1FnPb9/pjoArSxtviJhVIPLNo7oi3LWg=", - "usagesDigest": "M3VqFpeTCo4qmrNKGZw0dxBHvTYDrfV3cscGzlSAhQ4=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "xctestrunner": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/google/xctestrunner/archive/b7698df3d435b6491b4b4c0f9fc7a63fbed5e3a6.tar.gz" - ], - "strip_prefix": "xctestrunner-b7698df3d435b6491b4b4c0f9fc7a63fbed5e3a6", - "sha256": "ae3a063c985a8633cb7eb566db21656f8db8eb9a0edb8c182312c7f0db53730d" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_apple+", - "bazel_tools", - "bazel_tools" - ] - ] + } } }, "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { "general": { - "bzlTransitiveDigest": "oA0OoCaohRv2x/x61/6mFy+2a9+GiuUKGAXLGcQy+hQ=", + "bzlTransitiveDigest": "76FMguAa6TZAMN5inJ4/C9SZ0VlZnKOlVnlRe9bgvd8=", "usagesDigest": "QI2z8ZUR+mqtbwsf2fLqYdJAkPOHdOV+tF2yVAUgRzw=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, + "recordedInputs": [ + "REPO_MAPPING:rules_kotlin+,bazel_tools bazel_tools" + ], "generatedRepoSpecs": { "com_github_jetbrains_kotlin_git": { "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:compiler.bzl%kotlin_compiler_git_repository", @@ -1134,23 +694,20 @@ ] } } - }, - "recordedRepoMappingEntries": [ - [ - "rules_kotlin+", - "bazel_tools", - "bazel_tools" - ] - ] + } } }, "@@rules_oci+//oci:extensions.bzl%oci": { "general": { - "bzlTransitiveDigest": "Q7DvSdepUOK0DnFJtYmquDXHZCcR2DoLM9LbkrHSCKQ=", + "bzlTransitiveDigest": "lRG1JBXWgJgDfEfl/34ZHgTFACz2I0bpfZWFtcoM+hc=", "usagesDigest": "JLbamplho20rviuebatMGqnQnaoj32ewCzLuRVusGoU=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, + "recordedInputs": [ + "REPO_MAPPING:aspect_bazel_lib+,bazel_tools bazel_tools", + "REPO_MAPPING:bazel_features+,bazel_tools bazel_tools", + "REPO_MAPPING:rules_oci+,aspect_bazel_lib aspect_bazel_lib+", + "REPO_MAPPING:rules_oci+,bazel_features bazel_features+", + "REPO_MAPPING:rules_oci+,bazel_skylib bazel_skylib+" + ], "generatedRepoSpecs": { "distroless_cc_debian12_linux_amd64": { "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_pull", @@ -1334,117 +891,17 @@ "explicitRootModuleDirectDevDeps": [], "useAllRepos": "NO", "reproducible": false - }, - "recordedRepoMappingEntries": [ - [ - "aspect_bazel_lib+", - "bazel_tools", - "bazel_tools" - ], - [ - "bazel_features+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_oci+", - "aspect_bazel_lib", - "aspect_bazel_lib+" - ], - [ - "rules_oci+", - "bazel_features", - "bazel_features+" - ], - [ - "rules_oci+", - "bazel_skylib", - "bazel_skylib+" - ] - ] - } - }, - "@@rules_perl+//perl:extensions.bzl%perl_repositories": { - "general": { - "bzlTransitiveDigest": "0vc/2RHDjju9Q1391ILlkclFi099x0+aF+aYoDr36HA=", - "usagesDigest": "qSSNDdCNVxNhY36wMndEAFacdhR0ooLTmumfad0km9s=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "perl_darwin_arm64": { - "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", - "attributes": { - "strip_prefix": "perl-darwin-arm64", - "sha256": "285769f3c50c339fb59a3987b216ae3c5c573b95babe6875a1ef56fb178433da", - "urls": [ - "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-darwin-arm64.tar.xz" - ] - } - }, - "perl_darwin_amd64": { - "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", - "attributes": { - "strip_prefix": "perl-darwin-amd64", - "sha256": "63bc5ee36f5394d71c50cca6cafdd333ee58f9eaa40bca63c85f9bd06f2c1fd6", - "urls": [ - "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-darwin-amd64.tar.xz" - ] - } - }, - "perl_linux_amd64": { - "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", - "attributes": { - "strip_prefix": "perl-linux-amd64", - "sha256": "3bdffa9d7a3f97c0207314637b260ba5115b1d0829f97e3e2e301191a4d4d076", - "urls": [ - "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-linux-amd64.tar.xz" - ] - } - }, - "perl_linux_arm64": { - "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", - "attributes": { - "strip_prefix": "perl-linux-arm64", - "sha256": "6fa4ece99e790ecbc2861f6ecb7b52694c01c2eeb215b4370f16a3b12d952117", - "urls": [ - "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-linux-arm64.tar.xz" - ] - } - }, - "perl_windows_x86_64": { - "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", - "attributes": { - "strip_prefix": "", - "sha256": "aeb973da474f14210d3e1a1f942dcf779e2ae7e71e4c535e6c53ebabe632cc98", - "urls": [ - "https://mirror.bazel.build/strawberryperl.com/download/5.32.1.1/strawberry-perl-5.32.1.1-64bit.zip", - "https://strawberryperl.com/download/5.32.1.1/strawberry-perl-5.32.1.1-64bit.zip" - ] - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_perl+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_perl+", - "rules_perl", - "rules_perl+" - ] - ] + } } }, "@@rules_python+//python/uv:uv.bzl%uv": { "general": { "bzlTransitiveDigest": "IzGWfoGARM6X2a4AM7BN5gheLtDsFeP9Fra2qTod+mA=", "usagesDigest": "Ucfum/vsVnlkhtglTERDLTj+bBeG7dcpbV9h0LP9HOQ=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, + "recordedInputs": [ + "REPO_MAPPING:rules_python+,bazel_tools bazel_tools", + "REPO_MAPPING:rules_python+,platforms platforms" + ], "generatedRepoSpecs": { "uv_0_9_30_aarch64_apple_darwin": { "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", @@ -1595,190 +1052,19 @@ "toolchain_target_settings": {} } } - }, - "recordedRepoMappingEntries": [ - [ - "rules_python+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_python+", - "platforms", - "platforms" - ] - ] - } - }, - "@@rules_swift+//swift:extensions.bzl%non_module_deps": { - "general": { - "bzlTransitiveDigest": "nw2ZR52E52Bu14yqiZmfBMAg2hbF2GbIlmIxQNi7PiY=", - "usagesDigest": "mhACFnrdMv9Wi0Mt67bxocJqviRkDSV+Ee5Mqdj5akA=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "com_github_apple_swift_protobuf": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-protobuf/archive/1.20.2.tar.gz" - ], - "sha256": "3fb50bd4d293337f202d917b6ada22f9548a0a0aed9d9a4d791e6fbd8a246ebb", - "strip_prefix": "swift-protobuf-1.20.2/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_protobuf/BUILD.overlay" - } - }, - "com_github_grpc_grpc_swift": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/grpc/grpc-swift/archive/1.16.0.tar.gz" - ], - "sha256": "58b60431d0064969f9679411264b82e40a217ae6bd34e17096d92cc4e47556a5", - "strip_prefix": "grpc-swift-1.16.0/", - "build_file": "@@rules_swift+//third_party:com_github_grpc_grpc_swift/BUILD.overlay" - } - }, - "com_github_apple_swift_docc_symbolkit": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-docc-symbolkit/archive/refs/tags/swift-5.10-RELEASE.tar.gz" - ], - "sha256": "de1d4b6940468ddb53b89df7aa1a81323b9712775b0e33e8254fa0f6f7469a97", - "strip_prefix": "swift-docc-symbolkit-swift-5.10-RELEASE", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_docc_symbolkit/BUILD.overlay" - } - }, - "com_github_apple_swift_nio": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio/archive/2.42.0.tar.gz" - ], - "sha256": "e3304bc3fb53aea74a3e54bd005ede11f6dc357117d9b1db642d03aea87194a0", - "strip_prefix": "swift-nio-2.42.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_http2": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-http2/archive/1.26.0.tar.gz" - ], - "sha256": "f0edfc9d6a7be1d587e5b403f2d04264bdfae59aac1d74f7d974a9022c6d2b25", - "strip_prefix": "swift-nio-http2-1.26.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_http2/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_transport_services": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-transport-services/archive/1.15.0.tar.gz" - ], - "sha256": "f3498dafa633751a52b9b7f741f7ac30c42bcbeb3b9edca6d447e0da8e693262", - "strip_prefix": "swift-nio-transport-services-1.15.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_transport_services/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_extras": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-extras/archive/1.4.0.tar.gz" - ], - "sha256": "4684b52951d9d9937bb3e8ccd6b5daedd777021ef2519ea2f18c4c922843b52b", - "strip_prefix": "swift-nio-extras-1.4.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_extras/BUILD.overlay" - } - }, - "com_github_apple_swift_log": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-log/archive/1.4.4.tar.gz" - ], - "sha256": "48fe66426c784c0c20031f15dc17faf9f4c9037c192bfac2f643f65cb2321ba0", - "strip_prefix": "swift-log-1.4.4/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_log/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_ssl": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-ssl/archive/2.23.0.tar.gz" - ], - "sha256": "4787c63f61dd04d99e498adc3d1a628193387e41efddf8de19b8db04544d016d", - "strip_prefix": "swift-nio-ssl-2.23.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_ssl/BUILD.overlay" - } - }, - "com_github_apple_swift_collections": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-collections/archive/1.0.4.tar.gz" - ], - "sha256": "d9e4c8a91c60fb9c92a04caccbb10ded42f4cb47b26a212bc6b39cc390a4b096", - "strip_prefix": "swift-collections-1.0.4/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_collections/BUILD.overlay" - } - }, - "com_github_apple_swift_atomics": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-atomics/archive/1.1.0.tar.gz" - ], - "sha256": "1bee7f469f7e8dc49f11cfa4da07182fbc79eab000ec2c17bfdce468c5d276fb", - "strip_prefix": "swift-atomics-1.1.0/", - "build_file": "@@rules_swift+//third_party:com_github_apple_swift_atomics/BUILD.overlay" - } - }, - "build_bazel_rules_swift_index_import": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "build_file": "@@rules_swift+//third_party:build_bazel_rules_swift_index_import/BUILD.overlay", - "canonical_id": "index-import-5.8", - "urls": [ - "https://github.com/MobileNativeFoundation/index-import/releases/download/5.8.0.1/index-import.tar.gz" - ], - "sha256": "28c1ffa39d99e74ed70623899b207b41f79214c498c603915aef55972a851a15" - } - }, - "build_bazel_rules_swift_local_config": { - "repoRuleId": "@@rules_swift+//swift/internal:swift_autoconfiguration.bzl%swift_autoconfiguration", - "attributes": {} - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_swift+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_swift+", - "build_bazel_rules_swift", - "rules_swift+" - ] - ] + } } }, "@@rules_zig+//zig:extensions.bzl%zig": { "general": { "bzlTransitiveDigest": "VGLOg2kVw+STO9R5OrQnLFbv7Vl5TPdtIebIs324IpU=", "usagesDigest": "4xBoe2eIScx2QI0jVieHWXdflezUUDIApwMqS+SWGbw=", - "recordedFileInputs": { - "@@//bazel/zig_index.json": "2e3b4b83f1add5d3ca12fd05797c04c38af21d86c7fc7c1a9b356560141a4797", - "@@rules_zig+//zig/private/versions.json": "3e4e370dee9c3f3fdc8bfc7a600e76743278fde40815eead5d0bcda3aec3e2bd" - }, - "recordedDirentsInputs": {}, - "envVariables": {}, + "recordedInputs": [ + "REPO_MAPPING:rules_zig+,bazel_skylib bazel_skylib+", + "REPO_MAPPING:rules_zig+,bazel_tools bazel_tools", + "FILE:@@//bazel/zig_index.json 2e3b4b83f1add5d3ca12fd05797c04c38af21d86c7fc7c1a9b356560141a4797", + "FILE:@@rules_zig+//zig/private/versions.json 3e4e370dee9c3f3fdc8bfc7a600e76743278fde40815eead5d0bcda3aec3e2bd" + ], "generatedRepoSpecs": { "zig_0.16.0_aarch64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", @@ -2083,30 +1369,19 @@ "target_settings": [] } } - }, - "recordedRepoMappingEntries": [ - [ - "rules_zig+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "rules_zig+", - "bazel_tools", - "bazel_tools" - ] - ] + } } }, "@@rules_zig+//zig/zls:extensions.bzl%zls": { "general": { "bzlTransitiveDigest": "ZuceGxE17jROQ+kQkaPqIcK7gMKGb56I97Aq1/Dubdc=", "usagesDigest": "b/ERjMTWazfa8VFWuTc0PTtQQzLDuloWSHlzXj6Zb5w=", - "recordedFileInputs": { - "@@rules_zig+//zig/zls/private/versions.json": "127803289dfd79b5255db5f74be1d9a2f790f03292d6147009af702d888aba69" - }, - "recordedDirentsInputs": {}, - "envVariables": {}, + "recordedInputs": [ + "REPO_MAPPING:,rules_zig rules_zig+", + "REPO_MAPPING:rules_zig+,bazel_skylib bazel_skylib+", + "REPO_MAPPING:rules_zig+,bazel_tools bazel_tools", + "FILE:@@rules_zig+//zig/zls/private/versions.json 127803289dfd79b5255db5f74be1d9a2f790f03292d6147009af702d888aba69" + ], "generatedRepoSpecs": { "zls_0.16.0_aarch64-linux": { "repoRuleId": "@@rules_zig+//zig/zls/private/repo:zls_repository.bzl%zls_repository", @@ -2219,33 +1494,14 @@ ] } } - }, - "recordedRepoMappingEntries": [ - [ - "", - "rules_zig", - "rules_zig+" - ], - [ - "rules_zig+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "rules_zig+", - "bazel_tools", - "bazel_tools" - ] - ] + } } }, "@@yq.bzl+//yq:extensions.bzl%yq": { "general": { "bzlTransitiveDigest": "iWlctm1cJYZLESuQEBygCQc6adq+oIRMb4Dmk0rYxkA=", "usagesDigest": "J3TR36exv3ePI9O2y+uqrzTDmT6s6ADkTsROB++nm6I=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, + "recordedInputs": [], "generatedRepoSpecs": { "yq_darwin_amd64": { "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", @@ -2309,8 +1565,7 @@ "user_repository_name": "yq" } } - }, - "recordedRepoMappingEntries": [] + } } } }, diff --git a/bazel/cc_import.bzl b/bazel/cc_import.bzl index 09d3f2e1..fc6ef727 100644 --- a/bazel/cc_import.bzl +++ b/bazel/cc_import.bzl @@ -1,4 +1,4 @@ -load("@bazel_tools//tools/build_defs/cc:cc_import.bzl", _cc_import = "cc_import") +load("@rules_cc//cc:defs.bzl", "cc_library", _cc_import = "cc_import") load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") load(":patchelf.bzl", "patchelf") @@ -71,7 +71,7 @@ def cc_import( interface_library = interface_library, visibility = visibility, ) - native.cc_library( + cc_library( name = name, data = data, deps = deps + [name + ".cc_import"], diff --git a/bin/zml-smi/BUILD.bazel b/bin/zml-smi/BUILD.bazel index 0c21893a..4032a85c 100644 --- a/bin/zml-smi/BUILD.bazel +++ b/bin/zml-smi/BUILD.bazel @@ -3,6 +3,7 @@ load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") load("@bazel_lib//tools/release:hashes.bzl", "hashes") load("@bazel_skylib//rules:run_binary.bzl", "run_binary") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load") +load("@rules_shell//shell:sh_binary.bzl", "sh_binary") load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") load("@tar.bzl//tar:mtree.bzl", "mtree_spec") load("@tar.bzl//tar:tar.bzl", "tar") diff --git a/third_party/iree/futex-public-api-header.patch b/third_party/iree/futex-public-api-header.patch new file mode 100644 index 00000000..7623f422 --- /dev/null +++ b/third_party/iree/futex-public-api-header.patch @@ -0,0 +1,17 @@ +diff --git a/runtime/src/iree/base/threading/futex.h b/runtime/src/iree/base/threading/futex.h +index 9b42d3cfd0..f2509b77e9 100644 +--- a/runtime/src/iree/base/threading/futex.h ++++ b/runtime/src/iree/base/threading/futex.h +@@ -30,11 +30,8 @@ + + #include + +-#include "iree/base/attributes.h" +-#include "iree/base/config.h" +-#include "iree/base/status.h" ++#include "iree/base/api.h" + #include "iree/base/target_platform.h" +-#include "iree/base/time.h" + + //===----------------------------------------------------------------------===// + // Platform detection diff --git a/third_party/iree/repo.bzl b/third_party/iree/repo.bzl index 3b7f9b06..e266c641 100644 --- a/third_party/iree/repo.bzl +++ b/third_party/iree/repo.bzl @@ -15,9 +15,9 @@ def repo(): "//third_party/iree:tokenizer-only.patch", "//third_party/iree:fix-gemma4-encode.patch", "//third_party/iree:fix-added-token-matching.patch", + "//third_party/iree:futex-public-api-header.patch", ], patch_args = ["-p1"], ) #new_local_repository(name = "flashattn", build_file="//:third_party/flashattn/BUILD.bazel", path="/home/corendos/flashattn/") - diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index f1483aa0..8e9b867c 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -1,5 +1,6 @@ load("@rules_cc//cc:cc_binary.bzl", "cc_binary") load("@rules_python//python:py_library.bzl", "py_library") +load("@rules_shell//shell:sh_binary.bzl", "sh_binary") py_library( name = "zml_utils", diff --git a/tools/neuron/BUILD.bazel b/tools/neuron/BUILD.bazel index bc2e6a75..f9e23117 100644 --- a/tools/neuron/BUILD.bazel +++ b/tools/neuron/BUILD.bazel @@ -1,3 +1,5 @@ +load("@rules_shell//shell:sh_binary.bzl", "sh_binary") + sh_binary( name = "profile", srcs = ["profile.sh"], From a4993854da7e25b48df486f3f87a8ae6f8e5e865 Mon Sep 17 00:00:00 2001 From: elogir Date: Fri, 26 Jun 2026 14:37:56 +0200 Subject: [PATCH 537/881] platforms/metal: sandbox Metal toolchain (#598) Fetch Apple's encrypted Metal toolchain and only exposes the minimal compile tools + runtime needed --- MODULE.bazel | 21 ++ bazel/http_aea_archive.bzl | 128 +++++++ platforms/metal/BUILD.bazel | 17 +- platforms/metal/libpjrt_metal.BUILD.bazel | 24 ++ platforms/metal/metal.bzl | 14 +- platforms/metal/metal.zig | 17 +- platforms/metal/metal_toolchain.BUILD.bazel | 382 ++++++++++++++++++++ 7 files changed, 585 insertions(+), 18 deletions(-) create mode 100644 bazel/http_aea_archive.bzl create mode 100644 platforms/metal/libpjrt_metal.BUILD.bazel create mode 100644 platforms/metal/metal_toolchain.BUILD.bazel diff --git a/MODULE.bazel b/MODULE.bazel index 0c7ae4ec..a5c4a775 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -181,6 +181,27 @@ use_repo(tpu, "libpjrt_tpu") neuron = use_extension("//platforms/neuron:neuron.bzl", "neuron_packages") use_repo(neuron, "aws-neuronx-collectives", "aws-neuronx-runtime-lib", "aws-neuronx-tools", "libgcc_s1", "libgomp1", "libpjrt_neuron", "libstdcpp6") +http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +http_archive( + name = "sevenzip_macos", + build_file_content = """exports_files(["7zz"])""", + sha256 = "26aa75bc262bb10bf0805617b95569c3035c2c590a99f7db55c7e9607b2685e0", + urls = ["https://www.7-zip.org/a/7z2501-mac.tar.xz"], +) + +http_aea_archive = use_repo_rule("//bazel:http_aea_archive.bzl", "http_aea_archive") +http_aea_archive( + name = "metal_toolchain", + archive_decryption_key = "v7VbaT39JmQPp5p4N3HoVPAKcZ5vyhY+nnjMRD79z5E=", + canonical_id = "com.apple.MobileAsset.MetalToolchain-17B54", + sha256 = "84f1cec3984878294777349e794f7bff0407e501a728bd600dc8605fa36abf6f", + urls = [ + "https://updates.cdn-apple.com/2025/mobileassets/022-20354/192ADD28-31EE-43C8-972D-E5BED5FB1845/com_apple_MobileAsset_MetalToolchain/637B0E26-E00A-488B-BA3B-CB50A8ECFD74.aar", + ], + build_file = "//platforms/metal:metal_toolchain.BUILD.bazel", + includes = ["Metal.xctoolchain/*"], +) + metal = use_extension("//platforms/metal:metal.bzl", "metal_packages") use_repo(metal, "libpjrt_metal") diff --git a/bazel/http_aea_archive.bzl b/bazel/http_aea_archive.bzl new file mode 100644 index 00000000..867bc504 --- /dev/null +++ b/bazel/http_aea_archive.bzl @@ -0,0 +1,128 @@ +"""Repository rule for macOS Apple Encrypted Archive.""" + +def _fail_result(step, result): + if result.return_code != 0: + fail("%s failed with exit code %d\nstdout:\n%s\nstderr:\n%s" % ( + step, + result.return_code, + result.stdout, + result.stderr, + )) + +def _execute(rctx, step, argv, quiet = True): + result = rctx.execute([str(arg) for arg in argv], quiet = quiet) + _fail_result(step, result) + return result + +def _require_macos(rctx): + os_name = rctx.os.name.lower() + if "mac" not in os_name and "darwin" not in os_name: + fail("http_aea_archive is macOS-only; current repository OS is %r" % rctx.os.name) + +def _single_restore_dmg(rctx): + restore_dir = rctx.path("AssetData/Restore") + if not restore_dir.exists: + fail("aa patch did not produce AssetData/Restore") + + dmgs = [] + entries = [] + for entry in restore_dir.readdir(): + entries.append(entry.basename) + if not entry.is_dir and entry.basename.endswith(".dmg"): + dmgs.append(entry) + + if len(dmgs) != 1: + fail("expected exactly one restore DMG in AssetData/Restore, got %d; entries: %s" % ( + len(dmgs), + ", ".join(sorted(entries)), + )) + + return dmgs[0] + +def _http_aea_archive_impl(rctx): + _require_macos(rctx) + + urls = [] + if rctx.attr.url: + urls.append(rctx.attr.url) + urls.extend(rctx.attr.urls) + if not urls: + fail("http_aea_archive requires url or urls") + + if not rctx.attr.sha256 and not rctx.attr.integrity: + fail("http_aea_archive requires sha256 or integrity for reproducible downloads") + if not rctx.attr.archive_decryption_key: + fail("archive_decryption_key is required") + + aar = rctx.path("asset.aar") + download_kwargs = { + "canonical_id": rctx.attr.canonical_id, + "integrity": rctx.attr.integrity, + "output": aar, + "sha256": rctx.attr.sha256, + "url": urls, + } + if rctx.attr.netrc: + download_kwargs["auth"] = rctx.use_netrc( + rctx.read_netrc(rctx.attr.netrc), + urls, + rctx.attr.auth_patterns, + ) + + rctx.download(**download_kwargs) + + _execute(rctx, "aa patch", [ + "/usr/bin/aa", + "patch", + "-i", + aar, + "-key-value", + "base64:%s" % rctx.attr.archive_decryption_key, + "-src", + "/var/empty", + "-dst", + rctx.path("."), + ]) + rctx.delete("asset.aar") + + dmg = _single_restore_dmg(rctx) + + extract_args = [ + rctx.path(rctx.attr._sevenzip), + "x", + "-y", + dmg, + ] + extract_args.extend(rctx.attr.includes) + _execute(rctx, "7z extract dmg", extract_args) + rctx.delete("AssetData") + + rctx.template("BUILD.bazel", rctx.attr.build_file) + + return None + +http_aea_archive = repository_rule( + implementation = _http_aea_archive_impl, + attrs = { + "url": attr.string(), + "urls": attr.string_list(), + "archive_decryption_key": attr.string(mandatory = True), + "sha256": attr.string(), + "integrity": attr.string(), + "canonical_id": attr.string(), + "netrc": attr.string(), + "auth_patterns": attr.string_dict(), + "build_file": attr.label( + allow_single_file = True, + ), + "includes": attr.string_list( + default = [], + doc = "Patterns to pass to 7z for selective extraction from the DMG. If empty, extracts everything.", + ), + "_sevenzip": attr.label( + allow_single_file = True, + default = Label("@sevenzip_macos//:7zz"), + ), + }, + doc = "Downloads and extracts a macOS AEA-wrapped AppleArchive asset", +) diff --git a/platforms/metal/BUILD.bazel b/platforms/metal/BUILD.bazel index a25996d1..5041fb8c 100644 --- a/platforms/metal/BUILD.bazel +++ b/platforms/metal/BUILD.bazel @@ -9,17 +9,26 @@ cc_library( ) copy_to_directory( - name = "sandbox", - srcs = ["@libpjrt_metal"], - out = "sandbox/lib", + name = "metal_toolchain", + srcs = [ + "@metal_toolchain//:metal_compile_tools", + "@metal_toolchain//:metal_compile_runtime", + ], + out = "metal_toolchain", + replace_prefixes = { + "Metal.xctoolchain/usr/metal/32023": ".", + }, include_external_repositories = ["**"], + exec_compatible_with = ["@platforms//os:macos"], + target_compatible_with = ["@platforms//os:macos"], ) cc_library( name = "libpjrt_metal", - data = [":sandbox"], + data = [":metal_toolchain"], defines = ["ZML_RUNTIME_METAL"], tags = ["manual"], + deps = ["@libpjrt_metal"], ) zig_library( diff --git a/platforms/metal/libpjrt_metal.BUILD.bazel b/platforms/metal/libpjrt_metal.BUILD.bazel new file mode 100644 index 00000000..9ffa0cc9 --- /dev/null +++ b/platforms/metal/libpjrt_metal.BUILD.bazel @@ -0,0 +1,24 @@ +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") + +copy_file( + name = "libpjrt_metal.dylib", + src = "libpjrt_c_api_gpu_plugin.dylib", + out = "lib/libpjrt_c_api_gpu_plugin.dylib", + allow_symlink = True, +) + +copy_to_directory( + name = "sandbox", + srcs = [ + ":libpjrt_metal.dylib", + ], + add_directory_to_runfiles = False, + include_external_repositories = ["**"], +) + +cc_library( + name = "libpjrt_metal", + data = [":sandbox"], + visibility = ["@zml//platforms/metal:__subpackages__"], +) diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 1810bb95..e26bf1f8 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -1,20 +1,10 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -_BUILD_FILE_CONTENT = """\ -package(default_visibility = ["//visibility:public"]) - -filegroup( - name = "libpjrt_metal", - srcs = ["libpjrt_c_api_gpu_plugin.dylib"], - visibility = ["@zml//platforms/metal:__subpackages__"], -) -""" - def _metal_impl(mctx): http_archive( name = "libpjrt_metal", - build_file_content = _BUILD_FILE_CONTENT, - sha256 = "b2c7ce7c7a554e40cdcd48d27751f7dfcac0261def179e53ac07305032829421", + build_file = "libpjrt_metal.BUILD.bazel", + sha256 = "d6dac3b0df3390f56e4f0fa41241e949ce32d6266b2f7ed2aa677be4c942acce", url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-06-23T00-20-00Z/pjrt-metal_macos-arm64.tar.zst", ) diff --git a/platforms/metal/metal.zig b/platforms/metal/metal.zig index 014144a9..e1fbd6c3 100644 --- a/platforms/metal/metal.zig +++ b/platforms/metal/metal.zig @@ -13,6 +13,15 @@ pub fn isEnabled() bool { return @hasDecl(c, "ZML_RUNTIME_METAL"); } +fn setMetalToolchainEnv(r: anytype) !void { + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const root = try r.rlocation("zml/platforms/metal/metal_toolchain", &buf) orelse + return error.ToolchainNotInRunfiles; + + var bindir_z: [std.Io.Dir.max_path_bytes]u8 = undefined; + _ = c.setenv("METAL_TOOLCHAIN", try stdx.Io.Dir.path.bufJoinZ(&bindir_z, &.{ root, "bin" }), 0); +} + pub fn load(_: std.mem.Allocator, io: std.Io) !*const pjrt.Api { _ = io; if (comptime !isEnabled()) { @@ -26,14 +35,18 @@ pub fn load(_: std.mem.Allocator, io: std.Io) !*const pjrt.Api { const r = try bazel.runfiles(bazel_builtin.current_repository); var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = try r.rlocation("zml/platforms/metal/sandbox/lib", &path_buf) orelse { + const sandbox_path = try r.rlocation("libpjrt_metal/sandbox", &path_buf) orelse { log.err("Failed to find sandbox path for Metal runtime", .{}); return error.FileNotFound; }; + setMetalToolchainEnv(r) catch |err| { + log.err("Bundled Metal toolchain not found in runfiles ({any})", .{err}); + }; + return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "libpjrt_c_api_gpu_plugin.dylib" }); + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_c_api_gpu_plugin.dylib" }); break :blk pjrt.Api.loadFrom(path); }; } diff --git a/platforms/metal/metal_toolchain.BUILD.bazel b/platforms/metal/metal_toolchain.BUILD.bazel new file mode 100644 index 00000000..ca70fff0 --- /dev/null +++ b/platforms/metal/metal_toolchain.BUILD.bazel @@ -0,0 +1,382 @@ +package(default_visibility = ["//visibility:public"]) + +# Static BUILD file for the 17B54 MetalToolchain asset. + +filegroup( + name = "toolchain_runtime", + srcs = glob(["Metal.xctoolchain/**"]), +) + +exports_files([ + "Metal.xctoolchain/ToolchainInfo.plist", + "Metal.xctoolchain/usr/bin/air-ar", + "Metal.xctoolchain/usr/bin/air-arch", + "Metal.xctoolchain/usr/bin/air-as", + "Metal.xctoolchain/usr/bin/air-binary-perf", + "Metal.xctoolchain/usr/bin/air-config", + "Metal.xctoolchain/usr/bin/air-dsymutil", + "Metal.xctoolchain/usr/bin/air-libtool", + "Metal.xctoolchain/usr/bin/air-link", + "Metal.xctoolchain/usr/bin/air-lipo", + "Metal.xctoolchain/usr/bin/air-lld", + "Metal.xctoolchain/usr/bin/air-nm", + "Metal.xctoolchain/usr/bin/air-nt", + "Metal.xctoolchain/usr/bin/air-objcopy", + "Metal.xctoolchain/usr/bin/air-objdump", + "Metal.xctoolchain/usr/bin/air-opt", + "Metal.xctoolchain/usr/bin/air-pack", + "Metal.xctoolchain/usr/bin/air-package", + "Metal.xctoolchain/usr/bin/air-ranlib", + "Metal.xctoolchain/usr/bin/air-readobj", + "Metal.xctoolchain/usr/bin/air-size", + "Metal.xctoolchain/usr/bin/air-strip", + "Metal.xctoolchain/usr/bin/air-tt", + "Metal.xctoolchain/usr/bin/air-vtool", + "Metal.xctoolchain/usr/bin/amdgpu-nt", + "Metal.xctoolchain/usr/bin/applegpu-nt", + "Metal.xctoolchain/usr/bin/intelgpu-nt", + "Metal.xctoolchain/usr/bin/metal", + "Metal.xctoolchain/usr/bin/metal-ar", + "Metal.xctoolchain/usr/bin/metal-arch", + "Metal.xctoolchain/usr/bin/metal-as", + "Metal.xctoolchain/usr/bin/metal-binary-perf", + "Metal.xctoolchain/usr/bin/metal-config", + "Metal.xctoolchain/usr/bin/metal-dsymutil", + "Metal.xctoolchain/usr/bin/metal-libtool", + "Metal.xctoolchain/usr/bin/metal-link", + "Metal.xctoolchain/usr/bin/metal-lipo", + "Metal.xctoolchain/usr/bin/metal-lld", + "Metal.xctoolchain/usr/bin/metal-nm", + "Metal.xctoolchain/usr/bin/metal-nt", + "Metal.xctoolchain/usr/bin/metal-objcopy", + "Metal.xctoolchain/usr/bin/metal-objdump", + "Metal.xctoolchain/usr/bin/metal-opt", + "Metal.xctoolchain/usr/bin/metal-pack", + "Metal.xctoolchain/usr/bin/metal-package", + "Metal.xctoolchain/usr/bin/metal-ranlib", + "Metal.xctoolchain/usr/bin/metal-readobj", + "Metal.xctoolchain/usr/bin/metal-size", + "Metal.xctoolchain/usr/bin/metal-source", + "Metal.xctoolchain/usr/bin/metal-strip", + "Metal.xctoolchain/usr/bin/metal-tt", + "Metal.xctoolchain/usr/bin/metal-vtool", + "Metal.xctoolchain/usr/bin/metallib", + "Metal.xctoolchain/usr/metal/32023/lib/AMDShared.bundle/Contents/PlugIns/libAMDIL902.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/AMDShared.bundle/Contents/PlugIns/libGCNASM.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/AMDShared.bundle/Contents/PlugIns/libSC.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/amd_13/AMDShared.bundle/Contents/PlugIns/libAMDIL902.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/amd_13/AMDShared.bundle/Contents/PlugIns/libGCNASM.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/amd_13/AMDShared.bundle/Contents/PlugIns/libSC.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/amd_13/libAMDNTPlugin.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_ios.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_iosmac.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_iossim.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_osx.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_tvos.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_tvossim.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_watchos.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_watchossim.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_xros.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_xrossim.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/libAMDNTPlugin.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/libMTLIGCCompilerPlugin.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/libMTLPasses.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/libapplegpu-nt.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/libapplegpu23-nt.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/libapplegpu24-nt.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/libapplegpuG9G12-nt.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/libigc.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/metalfe/libmetal.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/runtime.gpuencode.split", + "Metal.xctoolchain/usr/metal/32023/lib/runtime.igc64.split", + "Metal.xctoolchain/usr/metal/32023/lib/wrapper-nt.dylib", +]) + +alias(name = "toolchain_info", actual = "Metal.xctoolchain/ToolchainInfo.plist") + +alias( + name = "air-ar", + actual = "Metal.xctoolchain/usr/bin/air-ar", +) + +alias( + name = "air-arch", + actual = "Metal.xctoolchain/usr/bin/air-arch", +) + +alias( + name = "air-as", + actual = "Metal.xctoolchain/usr/bin/air-as", +) + +alias( + name = "air-binary-perf", + actual = "Metal.xctoolchain/usr/bin/air-binary-perf", +) + +alias( + name = "air-config", + actual = "Metal.xctoolchain/usr/bin/air-config", +) + +alias( + name = "air-dsymutil", + actual = "Metal.xctoolchain/usr/bin/air-dsymutil", +) + +alias( + name = "air-libtool", + actual = "Metal.xctoolchain/usr/bin/air-libtool", +) + +alias( + name = "air-link", + actual = "Metal.xctoolchain/usr/bin/air-link", +) + +alias( + name = "air-lipo", + actual = "Metal.xctoolchain/usr/bin/air-lipo", +) + +alias( + name = "air-lld", + actual = "Metal.xctoolchain/usr/bin/air-lld", +) + +alias( + name = "air-nm", + actual = "Metal.xctoolchain/usr/bin/air-nm", +) + +alias( + name = "air-nt", + actual = "Metal.xctoolchain/usr/bin/air-nt", +) + +alias( + name = "air-objcopy", + actual = "Metal.xctoolchain/usr/bin/air-objcopy", +) + +alias( + name = "air-objdump", + actual = "Metal.xctoolchain/usr/bin/air-objdump", +) + +alias( + name = "air-opt", + actual = "Metal.xctoolchain/usr/bin/air-opt", +) + +alias( + name = "air-pack", + actual = "Metal.xctoolchain/usr/bin/air-pack", +) + +alias( + name = "air-package", + actual = "Metal.xctoolchain/usr/bin/air-package", +) + +alias( + name = "air-ranlib", + actual = "Metal.xctoolchain/usr/bin/air-ranlib", +) + +alias( + name = "air-readobj", + actual = "Metal.xctoolchain/usr/bin/air-readobj", +) + +alias( + name = "air-size", + actual = "Metal.xctoolchain/usr/bin/air-size", +) + +alias( + name = "air-strip", + actual = "Metal.xctoolchain/usr/bin/air-strip", +) + +alias( + name = "air-tt", + actual = "Metal.xctoolchain/usr/bin/air-tt", +) + +alias( + name = "air-vtool", + actual = "Metal.xctoolchain/usr/bin/air-vtool", +) + +alias( + name = "amdgpu-nt", + actual = "Metal.xctoolchain/usr/bin/amdgpu-nt", +) + +alias( + name = "applegpu-nt", + actual = "Metal.xctoolchain/usr/bin/applegpu-nt", +) + +alias( + name = "intelgpu-nt", + actual = "Metal.xctoolchain/usr/bin/intelgpu-nt", +) + +alias( + name = "metal", + actual = "Metal.xctoolchain/usr/bin/metal", +) + +alias( + name = "metal-ar", + actual = "Metal.xctoolchain/usr/bin/metal-ar", +) + +alias( + name = "metal-arch", + actual = "Metal.xctoolchain/usr/bin/metal-arch", +) + +alias( + name = "metal-as", + actual = "Metal.xctoolchain/usr/bin/metal-as", +) + +alias( + name = "metal-binary-perf", + actual = "Metal.xctoolchain/usr/bin/metal-binary-perf", +) + +alias( + name = "metal-config", + actual = "Metal.xctoolchain/usr/bin/metal-config", +) + +alias( + name = "metal-dsymutil", + actual = "Metal.xctoolchain/usr/bin/metal-dsymutil", +) + +alias( + name = "metal-libtool", + actual = "Metal.xctoolchain/usr/bin/metal-libtool", +) + +alias( + name = "metal-link", + actual = "Metal.xctoolchain/usr/bin/metal-link", +) + +alias( + name = "metal-lipo", + actual = "Metal.xctoolchain/usr/bin/metal-lipo", +) + +alias( + name = "metal-lld", + actual = "Metal.xctoolchain/usr/bin/metal-lld", +) + +alias( + name = "metal-nm", + actual = "Metal.xctoolchain/usr/bin/metal-nm", +) + +alias( + name = "metal-nt", + actual = "Metal.xctoolchain/usr/bin/metal-nt", +) + +alias( + name = "metal-objcopy", + actual = "Metal.xctoolchain/usr/bin/metal-objcopy", +) + +alias( + name = "metal-objdump", + actual = "Metal.xctoolchain/usr/bin/metal-objdump", +) + +alias( + name = "metal-opt", + actual = "Metal.xctoolchain/usr/bin/metal-opt", +) + +alias( + name = "metal-pack", + actual = "Metal.xctoolchain/usr/bin/metal-pack", +) + +alias( + name = "metal-package", + actual = "Metal.xctoolchain/usr/bin/metal-package", +) + +alias( + name = "metal-ranlib", + actual = "Metal.xctoolchain/usr/bin/metal-ranlib", +) + +alias( + name = "metal-readobj", + actual = "Metal.xctoolchain/usr/bin/metal-readobj", +) + +alias( + name = "metal-size", + actual = "Metal.xctoolchain/usr/bin/metal-size", +) + +alias( + name = "metal-source", + actual = "Metal.xctoolchain/usr/bin/metal-source", +) + +alias( + name = "metal-strip", + actual = "Metal.xctoolchain/usr/bin/metal-strip", +) + +alias( + name = "metal-tt", + actual = "Metal.xctoolchain/usr/bin/metal-tt", +) + +alias( + name = "metal-vtool", + actual = "Metal.xctoolchain/usr/bin/metal-vtool", +) + +alias( + name = "metallib", + actual = "Metal.xctoolchain/usr/bin/metallib", +) + +filegroup( + name = "metal_headers", + srcs = glob(["Metal.xctoolchain/usr/metal/*/lib/clang/*/include/**/*.h"]), +) + +filegroup( + name = "tracepoint_runtime_metallibs", + srcs = glob(["Metal.xctoolchain/usr/metal/*/lib/clang/*/lib/darwin/*.metallib"]), +) + +filegroup( + name = "metal_compile_tools", + srcs = [ + "Metal.xctoolchain/usr/metal/32023/bin/metal", + "Metal.xctoolchain/usr/metal/32023/bin/air-opt", + "Metal.xctoolchain/usr/metal/32023/bin/air-as", + "Metal.xctoolchain/usr/metal/32023/bin/metallib", + ], +) + +filegroup( + name = "metal_compile_runtime", + srcs = glob([ + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/**", + ]), +) From c2cbcb82f110482f32ba155fbd5c993591b6d77b Mon Sep 17 00:00:00 2001 From: Louis Lechevalier Date: Fri, 26 Jun 2026 14:44:48 +0200 Subject: [PATCH 538/881] zml/attention: Optimize heuristic and parameters choices of triton attention (#605) Limits the number of segments for the 3d kernel. Effect : reduce the cost of the reduce kernel which follows. --- zml/attention/triton_attention.zig | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/zml/attention/triton_attention.zig b/zml/attention/triton_attention.zig index e46b4407..f904b3f0 100644 --- a/zml/attention/triton_attention.zig +++ b/zml/attention/triton_attention.zig @@ -17,12 +17,17 @@ fn isOneapiTarget() bool { return zml.module.CompilationContext.current().platform.target == .oneapi; } -fn use2dKernel(head_size: usize, sliding_window: usize, all_decode: bool, max_seqlen_q: usize, max_seqlen_k: usize, target_num_prgms: usize, num_2d_prgms: usize) bool { - _ = head_size; - _ = max_seqlen_q; +fn use2dKernel(all_decode: bool, batch_size: usize, num_kv_heads: usize) bool { // Intel decode spills the 2D whole-sequence kernel; force the 3D split-K path. if (all_decode and isOneapiTarget()) return false; - return sliding_window > 0 or max_seqlen_k <= 512 or num_2d_prgms > target_num_prgms; + // prefill uses 2D; decode uses 3D until the batch is large enough to + // provide at least 128 2D launch programs across KV heads. + if (all_decode) { + const seq_threshold_3d = @divFloor(128, num_kv_heads); + return batch_size > seq_threshold_3d; + } + + return true; } pub const Config2D = struct { @@ -96,6 +101,12 @@ fn select3dConfig(options: paged.PagedAttentionOptions) Config3D { var num_segments = std.math.divCeil(usize, options.target_num_prgms, options.num_2d_prgms) catch unreachable; num_segments = std.math.ceilPowerOfTwoAssert(usize, num_segments); num_segments = @min(num_segments, 128); + if (options.all_decode and !isOneapiTarget()) { + // Keep the number of segments small to then limit reduce cost + // Didn't change the computation of Intel decode at the momment + // Need to be tested + num_segments = @min(num_segments, 16); + } const min_segments: usize = if (tile_size <= 16) 16 else 8; num_segments = @max(num_segments, min_segments); if (num_segments == min_segments) { @@ -282,13 +293,9 @@ pub const paged = struct { }; const use_2d_kernel = use2dKernel( - paged_attention_opts.head_dim, - paged_attention_opts.sliding_window, paged_attention_opts.all_decode, - paged_attention_opts.max_seqlen_q, - paged_attention_opts.maxSeqLenK(), - paged_attention_opts.target_num_prgms, - paged_attention_opts.num_2d_prgms, + paged_attention_opts.batch_size, + paged_attention_opts.num_kv_heads, ); const output = if (use_2d_kernel) pagedAttention2d(parameters_, q_, k_cache_, v_cache_, ctx_.opts, paged_attention_opts) From 3099106f17ee1ee712fcd49216b72d600b422b61 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Fri, 26 Jun 2026 15:35:27 +0200 Subject: [PATCH 539/881] platforms: set linux amd64 legacy cpu (#607) part of highway XLA target selects which header to include based on legacy --cpu flags but the rest not... So even if we targeted linux amd64, the top part of the highway would be used correctly, but the extension specific part would fallback to --cpu darwin_arm64 and would not include hh_avx2.h resulting in compilation errors. The normal solution here is to use platform_mappings but since we pass `--force_pic=true` in platforms, this takes precedence and the mappings are ignored. The temporary solution while https://github.com/openxla/xla/pull/44823 is merged, is to pass `--cpu` in all platforms we use for cross compilation. --- platforms/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index b5d8d526..19900cd4 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -76,6 +76,7 @@ platform( ], flags = [ "--force_pic=true", # required for protobuf upb on Linux + "--cpu=x86_64", ], visibility = ["//visibility:public"], ) From 1dbe39d3e83d9bfd71b18decd7df3ec2f144e814 Mon Sep 17 00:00:00 2001 From: Olivier Boulant Date: Fri, 26 Jun 2026 19:24:31 +0200 Subject: [PATCH 540/881] zml/io/vfs: add parallel read to GCS and S3 (#604) # Summary This PR extends the support to chunked parallel read introduced initially for HF in the VFS, to GCS and S3. It: - Adds a shared `zml/io/vfs/parallel_read.zig` helper for chunked parallel HTTP range reads - Add configurable read pool options for S3/GCS initialization - Register the `gs://` VFS backend in `examples/llm`. # Tests ## S3 We used [MINIO](https://hub.docker.com/r/minio/minio) docker image from the `5090` GPU machine. From there, we mirror the `meta-llama/Llama-3.1-8B-Instruct` model through the S3 server. Then from the `4090` GPU machine, we run ```bash bazel run --@zml//platforms:cuda=true --@zml//platforms:cpu=false --config=release --config=native //examples/llm -- --model=s3://zml-test/Llama-3.1-8B-Instruct --prompt="hello, how are you ?" ``` It works both on 1 and 2 GPU. ## GCS In the same spirit, we put the `meta-llama/Llama-3.1-8B-Instruct` in a GCS bucket we own. Then, on the `4090`, we run ```bash bazel run --@zml//platforms:cuda=true --@zml//platforms:cpu=false --config=release --config=native //examples/llm -- --model=gs://zml-mirror2/models/meta-llama/Llama-3.1-8B-Instruct --prompt="hello, how are you ?" ``` It works both on 1 and 2 GPU. ### Performance As a reference, we run: ```bash gcloud storage cp gs://zml-mirror2/models/meta-llama/Llama-3.1-8B-Instruct/model-00001-of-00004.safetensors gs://zml-mirror2/models/meta-llama/Llama-3.1-8B-Instruct/model-00002-of-00004.safetensors gs://zml-mirror2/models/meta-llama/Llama-3.1-8B-Instruct/model-00003-of-00004.safetensors gs://zml-mirror2/models/meta-llama/Llama-3.1-8B-Instruct/model-00004-of-00004.safetensors /tmp/gcs-gcloud-llama-shards/ ``` It gives: ``` Completed files 4/4 | 15.0GiB/15.0GiB | 942.7MiB/s Average throughput: 918.8MiB/s ``` Using the VFS in various setups: * `dma_chunks = 32`, `dma_chunk_size = 128`, `parallelism = 16`, `chunk_size = 10 MiB`, `num_workers = 16`, `queue_capacity = 64` ```bash info (llama): Loaded weights [14.96GiB, 21.302s, 719.02MiB/s] ``` * `dma_chunks = 32`, `dma_chunk_size = 256`, `parallelism = 16`, `chunk_size = 10 MiB`, `num_workers = 16`, `queue_capacity = 64` ```bash info (llama): Loaded weights [14.96GiB, 18.227s, 840.32MiB/s] ``` * `dma_chunks = 32`, `dma_chunk_size = 256`, `parallelism = 16`, `chunk_size = 16 MiB`, `num_workers = 16`, `queue_capacity = 64` ```bash info (llama): Loaded weights [14.96GiB, 17.376s, 881.43MiB/s] ``` * `dma_chunks = 32`, `dma_chunk_size = 256`, `parallelism = 16`, `chunk_size = 16 MiB`, `num_workers = 32`, `queue_capacity = 64` ```bash info (llama): Loaded weights [14.96GiB, 17.28s, 886.36MiB/s] ``` * `dma_chunks = 32`, `dma_chunk_size = 512`, `parallelism = 16`, `chunk_size = 16 MiB`, `num_workers = 16`, `queue_capacity = 64` ```bash info (llama): Loaded weights [14.96GiB, 18.658s, 820.90MiB/s] ``` --- examples/llm/main.zig | 4 + zml/io/vfs/BUILD.bazel | 1 + zml/io/vfs/gcs.zig | 186 ++++++++++++++++++------------- zml/io/vfs/hf.zig | 141 ++--------------------- zml/io/vfs/parallel_read.zig | 151 +++++++++++++++++++++++++ zml/io/vfs/s3.zig | 209 +++++++++++++++++++++-------------- 6 files changed, 407 insertions(+), 285 deletions(-) create mode 100644 zml/io/vfs/parallel_read.zig diff --git a/examples/llm/main.zig b/examples/llm/main.zig index d9b7e4a9..c328eaa5 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -67,10 +67,14 @@ pub fn main(init: std.process.Init) !void { var s3_vfs: zml.io.VFS.S3 = try .auto(allocator, init.io, &http_client, init.environ_map); defer s3_vfs.deinit(); + var gcs_vfs: zml.io.VFS.GCS = try .auto(allocator, init.io, &http_client, init.environ_map); + defer gcs_vfs.deinit(); + var vfs: zml.io.VFS = try .init(allocator, init.io); defer vfs.deinit(); try vfs.register("file", vfs_file.io()); + try vfs.register("gs", gcs_vfs.io()); try vfs.register("hf", hf_vfs.io()); try vfs.register("s3", s3_vfs.io()); diff --git a/zml/io/vfs/BUILD.bazel b/zml/io/vfs/BUILD.bazel index 6d53b5d2..dfa53d2c 100644 --- a/zml/io/vfs/BUILD.bazel +++ b/zml/io/vfs/BUILD.bazel @@ -8,6 +8,7 @@ zig_library( "gcs.zig", "hf.zig", "http.zig", + "parallel_read.zig", "s3.zig", ], main = "index.zig", diff --git a/zml/io/vfs/gcs.zig b/zml/io/vfs/gcs.zig index 808a47c8..f67940ad 100644 --- a/zml/io/vfs/gcs.zig +++ b/zml/io/vfs/gcs.zig @@ -4,6 +4,7 @@ const builtin = @import("builtin"); const stdx = @import("stdx"); const VFSBase = @import("base.zig").VFSBase; +const parallel_read = @import("parallel_read.zig"); const log = std.log.scoped(.@"zml/io/vfs/gcs"); @@ -120,6 +121,73 @@ const Credentials = union(enum) { const ReadState = struct { index: usize, objects: [][]const u8 }; pub const GCS = struct { + const ParallelRead = struct { + const Pool = parallel_read.Pool(Job); + + const Batch = struct { + executor: parallel_read.BatchExecutor, + backend: *GCS, + uri: std.Uri, + path: []const u8, + authorization: std.http.Client.Request.Headers.Value, + }; + + const Job = struct { + data: []const []u8, + file_offset: u64, + chunk_offset: usize, + chunk_len: usize, + batch: *Batch, + + pub fn perform(job: Job, client: *std.http.Client) ?anyerror { + var range_buf: [64]u8 = undefined; + const range_header = std.fmt.bufPrint( + &range_buf, + "bytes={d}-{d}", + .{ job.file_offset, job.file_offset + @as(u64, @intCast(job.chunk_len - 1)) }, + ) catch unreachable; + + var req = client.request( + .GET, + job.batch.uri, + .{ + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = job.batch.authorization, + }, + .extra_headers = &.{.{ .name = "Range", .value = range_header }}, + }, + ) catch |err| return err; + defer req.deinit(); + + req.sendBodiless() catch |err| return err; + + var redirect_buffer: [8 * 1024]u8 = undefined; + var res = req.receiveHead(&redirect_buffer) catch |err| return err; + + if (res.head.status != .partial_content and res.head.status != .ok) { + log.err("Failed to read {s}: {s}", .{ job.batch.path, res.head.bytes }); + return error.RequestFailed; + } + + const content_range = blk: { + var it = res.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { + break :blk parallel_read.parseContentRange(header.value); + } + } + break :blk null; + }; + + const reader = res.reader(&.{}); + parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| return err; + + return null; + } + }; + }; + const Token = struct { header: []u8, expires_at: std.Io.Timestamp, @@ -164,32 +232,39 @@ pub const GCS = struct { client: *std.http.Client, config: Config, token: Token, + read_pool: *ParallelRead.Pool, handles: stdx.SegmentedList(Handle, 0) = .{}, closed_handles: std.ArrayList(u32) = .empty, dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, base: VFSBase, - pub const InitArgs = struct { + pub const InitOpts = struct { credentials: ?union(enum) { json: *std.Io.Reader, metadata_server: void, } = null, endpoint_url: []const u8 = "https://storage.googleapis.com", region: []const u8 = "auto", + read_pool: parallel_read.InitOpts = .{}, }; pub const InitError = error{ InvalidCredentialJson, - RequestFailed, Unexpected, - } || std.mem.Allocator.Error; + } || std.mem.Allocator.Error || std.Io.ConcurrentError; - pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, args: InitArgs) InitError!GCS { + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, opts: InitOpts) InitError!GCS { var arena: std.heap.ArenaAllocator = .init(allocator); errdefer arena.deinit(); + const read_pool = try allocator.create(ParallelRead.Pool); + errdefer allocator.destroy(read_pool); + + try read_pool.init(allocator, inner, http_client, opts.read_pool); + errdefer read_pool.deinit(allocator, inner); + const config: Config = .{ - .credentials = if (args.credentials) |creds| switch (creds) { + .credentials = if (opts.credentials) |creds| switch (creds) { .json => |reader| blk: { var json_reader: std.json.Reader = .init(allocator, reader); defer json_reader.deinit(); @@ -203,8 +278,8 @@ pub const GCS = struct { }, .metadata_server => .{ .metadata_server = {} }, } else null, - .endpoint_uri = std.Uri.parse(try arena.allocator().dupe(u8, args.endpoint_url)) catch return InitError.Unexpected, - .region = try arena.allocator().dupe(u8, args.region), + .endpoint_uri = std.Uri.parse(try arena.allocator().dupe(u8, opts.endpoint_url)) catch return InitError.Unexpected, + .region = try arena.allocator().dupe(u8, opts.region), }; const token: Token = .{ @@ -219,6 +294,7 @@ pub const GCS = struct { .client = http_client, .config = config, .token = token, + .read_pool = read_pool, }; } @@ -372,6 +448,7 @@ pub const GCS = struct { if (self.config.credentials == null) { return .omit; } + if (self.token.expired(self.base.inner)) { try self.refreshToken(); } @@ -379,6 +456,9 @@ pub const GCS = struct { } pub fn deinit(self: *GCS) void { + self.read_pool.deinit(self.allocator, self.base.inner); + self.allocator.destroy(self.read_pool); + var idx: usize = 0; while (idx < self.handles.len) : (idx += 1) { const is_closed = for (self.closed_handles.items) |closed_idx| { @@ -901,80 +981,38 @@ pub const GCS = struct { } fn performRead(self: *GCS, handle: *Handle, data: []const []u8, offset: u64) !usize { - if (offset >= handle.size) return 0; - - var range_buf: [64]u8 = undefined; - const range_header = blk: { - var total_bytes: u64 = 0; - for (data) |buf| { - total_bytes += @as(u64, buf.len); - } - const remaining = handle.size - offset; - const take = @min(remaining, total_bytes); - const end = offset + take - 1; - break :blk std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ offset, end }) catch unreachable; - }; + const read_size = parallel_read.readSize(handle.size, offset, data); + if (read_size == 0) return 0; const uri = self.gcsUri(handle.uri); - var req = try self.client.request( - .GET, - uri, - .{ - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = try self.getOrRefreshToken(), - }, - .extra_headers = &.{.{ .name = "Range", .value = range_header }}, - }, - ); - defer req.deinit(); + const job_count = std.math.divCeil(usize, read_size, self.read_pool.chunk_size) catch unreachable; + const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); + defer self.allocator.free(jobs); + const pending: u32 = @intCast(job_count); - try req.sendBodiless(); - - var redirect_buffer: [8 * 1024]u8 = undefined; - var res = try req.receiveHead(&redirect_buffer); - - if (res.head.status != .partial_content and res.head.status != .ok) { - log.err("Failed to read {s}: {s}", .{ handle.uri, res.head.bytes }); - return error.RequestFailed; - } - - const content_range = blk: { - var it = res.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { - break :blk parseContentRange(header.value); - } - } - break :blk null; + var batch: ParallelRead.Batch = .{ + .executor = .{ .pending = .init(pending) }, + .backend = self, + .uri = uri, + .path = handle.uri, + .authorization = try self.getOrRefreshToken(), }; - const reader = res.reader(&.{}); - - if (content_range) |cr| { - if (cr.start < offset) { - try reader.discardAll(offset - cr.start); - } + for (jobs, 0..) |*job, i| { + const chunk_offset = i * self.read_pool.chunk_size; + job.* = .{ + .data = data, + .file_offset = offset + @as(u64, @intCast(chunk_offset)), + .chunk_offset = chunk_offset, + .chunk_len = @min(self.read_pool.chunk_size, read_size - chunk_offset), + .batch = &batch, + }; } - return try reader.readSliceShort(data[0]); - } + try self.read_pool.job_queue.putAll(self.base.inner, jobs); + batch.executor.waitUncancelable(self.base.inner); + if (batch.executor.firstError()) |err| return err; - const ContentRange = struct { - start: u64, - end: u64, - total: u64, - }; - - fn parseContentRange(value: []const u8) ?ContentRange { - const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; - const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; - const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; - - return .{ - .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, - .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, - .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, - }; + return read_size; } }; diff --git a/zml/io/vfs/hf.zig b/zml/io/vfs/hf.zig index 2423e638..664fd55f 100644 --- a/zml/io/vfs/hf.zig +++ b/zml/io/vfs/hf.zig @@ -3,97 +3,18 @@ const std = @import("std"); const stdx = @import("stdx"); const VFSBase = @import("base.zig").VFSBase; +const parallel_read = @import("parallel_read.zig"); const log = std.log.scoped(.@"zml/io/vfs/hf"); const ParallelRead = struct { - const Pool = struct { - pub const InitOpts = struct { - chunk_size: usize = 10 * 1024 * 1024, - num_workers: usize = 16, - queue_capacity: usize = 64, - }; - - group: std.Io.Group = .init, - job_queue: std.Io.Queue(Job), - queue_buf: []Job, - client: *std.http.Client, - chunk_size: usize, - - fn init(self: *Pool, allocator: std.mem.Allocator, io: std.Io, client: *std.http.Client, opts: InitOpts) !void { - stdx.debug.assert(opts.num_workers > 0, "Pool must have at least one worker", .{}); - stdx.debug.assert(opts.chunk_size > 0, "Pool must have a positive download chunk size", .{}); - stdx.debug.assert(opts.queue_capacity > 0, "Pool must have a positive queue capacity", .{}); - - const queue_buf = try allocator.alloc(Job, opts.queue_capacity); - errdefer allocator.free(queue_buf); - - self.* = .{ - .client = client, - .chunk_size = opts.chunk_size, - .job_queue = .init(queue_buf), - .queue_buf = queue_buf, - }; - errdefer self.group.cancel(io); - - for (0..opts.num_workers) |_| { - try self.group.concurrent(io, worker, .{ self, io }); - } - } - - fn deinit(self: *Pool, allocator: std.mem.Allocator, io: std.Io) void { - self.job_queue.close(io); - self.group.cancel(io); - - allocator.free(self.queue_buf); - } - - fn worker(pool: *Pool, io: std.Io) std.Io.Cancelable!void { - while (true) { - const job = pool.job_queue.getOne(io) catch break; - - if (!job.batch.failed()) { - if (job.perform(pool.client)) |err| job.batch.fail(err); - } - - job.batch.completeOne(io); - } - } - }; + const Pool = parallel_read.Pool(Job); const Batch = struct { + executor: parallel_read.BatchExecutor, uri: std.Uri, url: []const u8, authorization: std.http.Client.Request.Headers.Value, - pending: std.atomic.Value(u32), - err: std.atomic.Value(u16) = .init(0), - - fn failed(batch: *Batch) bool { - return batch.err.load(.acquire) != 0; - } - - fn fail(batch: *Batch, err: anyerror) void { - _ = batch.err.cmpxchgStrong(0, @intFromError(err), .release, .monotonic); - } - - fn firstError(batch: *Batch) ?anyerror { - const err = batch.err.load(.acquire); - return if (err == 0) null else @errorFromInt(err); - } - - fn completeOne(batch: *Batch, io: std.Io) void { - const previous = batch.pending.fetchSub(1, .release); - std.debug.assert(previous > 0); - if (previous == 1) io.futexWake(u32, &batch.pending.raw, 1); - } - - fn waitUncancelable(batch: *Batch, io: std.Io) void { - while (true) { - const pending = batch.pending.load(.acquire); - if (pending == 0) return; - io.futexWaitUncancelable(u32, &batch.pending.raw, pending); - } - } }; pub const Job = struct { @@ -103,7 +24,7 @@ const ParallelRead = struct { chunk_len: usize, batch: *Batch, - fn perform(job: Job, client: *std.http.Client) ?anyerror { + pub fn perform(job: Job, client: *std.http.Client) ?anyerror { var range_buf: [64]u8 = undefined; const range_header = std.fmt.bufPrint( &range_buf, @@ -135,58 +56,18 @@ const ParallelRead = struct { var it = res.head.iterateHeaders(); while (it.next()) |header| { if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { - break :blk parseContentRange(header.value); + break :blk parallel_read.parseContentRange(header.value); } } break :blk null; }; const reader = res.reader(&.{}); - - if (content_range) |cr| { - if (cr.start > job.file_offset) return error.InvalidContentRange; - if (cr.start < job.file_offset) { - reader.discardAll(job.file_offset - cr.start) catch |err| return err; - } - } - - var remaining = job.chunk_len; - var skip = job.chunk_offset; - - for (job.data) |buf| { - if (skip >= buf.len) { - skip -= buf.len; - continue; - } - - const destination = buf[skip..][0..@min(remaining, buf.len - skip)]; - reader.readSliceAll(destination) catch |err| return err; - remaining -= destination.len; - if (remaining == 0) break; - skip = 0; - } + parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| return err; return null; } }; - - const ContentRange = struct { - start: u64, - end: u64, - total: u64, - }; - - fn parseContentRange(value: []const u8) ?ContentRange { - const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; - const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; - const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; - - return .{ - .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, - .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, - .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, - }; - } }; pub const API = struct { @@ -234,7 +115,7 @@ const RepoKey = struct { pub const HF = struct { pub const InitOpts = struct { - read_pool: ParallelRead.Pool.InitOpts = .{}, + read_pool: parallel_read.InitOpts = .{}, }; pub const Repo = struct { @@ -911,7 +792,7 @@ pub const HF = struct { const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); defer self.allocator.free(jobs); - const pending: u32 = std.math.cast(u32, job_count) orelse return error.SystemResources; + const pending: u32 = @intCast(job_count); const repo: Repo = try .parse(handle.uri); var url_buffer: [8 * 1024]u8 = undefined; @@ -919,10 +800,10 @@ pub const HF = struct { const uri: std.Uri = try .parse(url); var batch: ParallelRead.Batch = .{ + .executor = .{ .pending = .init(pending) }, .uri = uri, .url = url, .authorization = self.authorization, - .pending = .init(pending), }; for (jobs, 0..) |*job, i| { @@ -937,8 +818,8 @@ pub const HF = struct { } try self.read_pool.job_queue.putAll(self.base.inner, jobs); - batch.waitUncancelable(self.base.inner); - if (batch.firstError()) |err| return err; + batch.executor.waitUncancelable(self.base.inner); + if (batch.executor.firstError()) |err| return err; return read_size; } diff --git a/zml/io/vfs/parallel_read.zig b/zml/io/vfs/parallel_read.zig new file mode 100644 index 00000000..aa4102cb --- /dev/null +++ b/zml/io/vfs/parallel_read.zig @@ -0,0 +1,151 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +pub const InitOpts = struct { + chunk_size: usize = 10 * 1024 * 1024, + num_workers: usize = 16, + queue_capacity: usize = 64, +}; + +pub const BatchExecutor = struct { + pending: std.atomic.Value(u32), + err: std.atomic.Value(u16) = .init(0), + + pub fn failed(batch: *BatchExecutor) bool { + return batch.err.load(.acquire) != 0; + } + + pub fn fail(batch: *BatchExecutor, err: anyerror) void { + _ = batch.err.cmpxchgStrong(0, @intFromError(err), .release, .monotonic); + } + + pub fn firstError(batch: *BatchExecutor) ?anyerror { + const err = batch.err.load(.acquire); + return if (err == 0) null else @errorFromInt(err); + } + + pub fn completeOne(batch: *BatchExecutor, io: std.Io) void { + const previous = batch.pending.fetchSub(1, .release); + std.debug.assert(previous > 0); + if (previous == 1) io.futexWake(u32, &batch.pending.raw, 1); + } + + pub fn waitUncancelable(batch: *BatchExecutor, io: std.Io) void { + while (true) { + const pending = batch.pending.load(.acquire); + if (pending == 0) return; + io.futexWaitUncancelable(u32, &batch.pending.raw, pending); + } + } +}; + +pub fn Pool(comptime Job: type) type { + return struct { + const Self = @This(); + + group: std.Io.Group = .init, + job_queue: std.Io.Queue(Job), + queue_buf: []Job, + client: *std.http.Client, + chunk_size: usize, + + pub fn init(self: *Self, allocator: std.mem.Allocator, io: std.Io, client: *std.http.Client, opts: InitOpts) !void { + stdx.debug.assert(opts.num_workers > 0, "Pool must have at least one worker", .{}); + stdx.debug.assert(opts.chunk_size > 0, "Pool must have a positive download chunk size", .{}); + stdx.debug.assert(opts.queue_capacity > 0, "Pool must have a positive queue capacity", .{}); + + const queue_buf = try allocator.alloc(Job, opts.queue_capacity); + errdefer allocator.free(queue_buf); + + self.* = .{ + .client = client, + .chunk_size = opts.chunk_size, + .job_queue = .init(queue_buf), + .queue_buf = queue_buf, + }; + errdefer self.group.cancel(io); + + for (0..opts.num_workers) |_| { + try self.group.concurrent(io, worker, .{ self, io }); + } + } + + pub fn deinit(self: *Self, allocator: std.mem.Allocator, io: std.Io) void { + self.job_queue.close(io); + self.group.cancel(io); + + allocator.free(self.queue_buf); + } + + fn worker(pool: *Self, io: std.Io) std.Io.Cancelable!void { + while (true) { + const job = pool.job_queue.getOne(io) catch break; + + if (!job.batch.executor.failed()) { + if (job.perform(pool.client)) |err| job.batch.executor.fail(err); + } + + job.batch.executor.completeOne(io); + } + } + }; +} + +pub const ContentRange = struct { + start: u64, + end: u64, + total: u64, +}; + +pub fn parseContentRange(value: []const u8) ?ContentRange { + const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; + const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; + const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; + + return .{ + .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, + .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, + .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, + }; +} + +pub fn readSize(file_size: u64, offset: u64, data: []const []u8) usize { + if (offset >= file_size) return 0; + + var requested: usize = 0; + for (data) |buf| requested += buf.len; + return @intCast(@min(file_size - offset, requested)); +} + +pub fn readChunk( + reader: *std.Io.Reader, + content_range: ?ContentRange, + file_offset: u64, + data: []const []u8, + chunk_offset: usize, + chunk_len: usize, +) !void { + if (content_range) |cr| { + if (cr.start > file_offset) return error.InvalidContentRange; + if (cr.start < file_offset) { + try reader.discardAll(file_offset - cr.start); + } + } + + var remaining = chunk_len; + var skip = chunk_offset; + + for (data) |buf| { + if (skip >= buf.len) { + skip -= buf.len; + continue; + } + + const destination = buf[skip..][0..@min(remaining, buf.len - skip)]; + try reader.readSliceAll(destination); + remaining -= destination.len; + if (remaining == 0) break; + skip = 0; + } +} diff --git a/zml/io/vfs/s3.zig b/zml/io/vfs/s3.zig index b6bd2a52..303fa282 100644 --- a/zml/io/vfs/s3.zig +++ b/zml/io/vfs/s3.zig @@ -3,6 +3,7 @@ const std = @import("std"); const stdx = @import("stdx"); const VFSBase = @import("base.zig").VFSBase; +const parallel_read = @import("parallel_read.zig"); const log = std.log.scoped(.@"zml/io/vfs/s3"); @@ -140,6 +141,95 @@ pub const AwsSigV4 = struct { const ReadState = struct { index: usize, objects: [][]const u8 }; pub const S3 = struct { + const ParallelRead = struct { + const Pool = parallel_read.Pool(Job); + + const Batch = struct { + executor: parallel_read.BatchExecutor, + backend: *S3, + uri: std.Uri, + url: []const u8, + }; + + const Job = struct { + data: []const []u8, + file_offset: u64, + chunk_offset: usize, + chunk_len: usize, + batch: *Batch, + + pub fn perform(job: Job, client: *std.http.Client) ?anyerror { + var range_buf: [64]u8 = undefined; + const range_header = std.fmt.bufPrint( + &range_buf, + "bytes={d}-{d}", + .{ job.file_offset, job.file_offset + @as(u64, @intCast(job.chunk_len - 1)) }, + ) catch unreachable; + + var timestamp_buf: [16]u8 = undefined; + const timestamp = job.batch.backend.getTimestamp(×tamp_buf) catch |err| return err; + + const signer: AwsSigV4 = .{ + .access_key = job.batch.backend.config.access_key, + .secret_key = job.batch.backend.config.secret_key, + .region = job.batch.backend.config.region, + .service = job.batch.backend.config.auth_service, + }; + + var authorization_buffer: [512]u8 = undefined; + const authorization = signer.generateAuthHeader( + &authorization_buffer, + .GET, + job.batch.uri, + timestamp, + &.{.{ .name = "Range", .value = range_header }}, + ) catch |err| return err; + + var req = client.request(.GET, job.batch.uri, .{ + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = if (authorization) |auth| .{ .override = auth } else .omit, + }, + .extra_headers = &.{ + .{ .name = "x-amz-date", .value = timestamp }, + .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, + .{ .name = "Range", .value = range_header }, + }, + }) catch |err| return err; + defer req.deinit(); + + req.sendBodiless() catch |err| return err; + + var redirect_buffer: [8 * 1024]u8 = undefined; + var res = req.receiveHead(&redirect_buffer) catch |err| return err; + + if (res.head.status != .partial_content and res.head.status != .ok) { + log.err("Failed to read {s}: {s}", .{ job.batch.url, res.head.bytes }); + return error.RequestFailed; + } + + const content_range = blk: { + var it = res.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { + break :blk parallel_read.parseContentRange(header.value); + } + } + break :blk null; + }; + + const reader = res.reader(&.{}); + parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| return err; + + return null; + } + }; + }; + + pub const InitOpts = struct { + read_pool: parallel_read.InitOpts = .{}, + }; + pub const Config = struct { access_key: ?[]const u8 = null, secret_key: ?[]const u8 = null, @@ -174,6 +264,7 @@ pub const S3 = struct { mutex: std.Io.Mutex = .init, client: *std.http.Client, config: Config, + read_pool: *ParallelRead.Pool, handles: stdx.SegmentedList(Handle, 0) = .{}, closed_handles: std.ArrayList(u32) = .empty, dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, @@ -184,11 +275,19 @@ pub const S3 = struct { inner: std.Io, http_client: *std.http.Client, config: Config, + opts: InitOpts, ) !S3 { + const read_pool = try allocator.create(ParallelRead.Pool); + errdefer allocator.destroy(read_pool); + + try read_pool.init(allocator, inner, http_client, opts.read_pool); + errdefer read_pool.deinit(allocator, inner); + return .{ .allocator = allocator, .base = .init(inner), .client = http_client, + .read_pool = read_pool, .config = .{ .access_key = if (config.access_key) |k| try allocator.dupe(u8, k) else null, .secret_key = if (config.secret_key) |k| try allocator.dupe(u8, k) else null, @@ -241,10 +340,13 @@ pub const S3 = struct { .secret_key = secret_key, .endpoint_url = endpoint, .region = region, - }); + }, .{}); } pub fn deinit(self: *S3) void { + self.read_pool.deinit(self.allocator, self.base.inner); + self.allocator.destroy(self.read_pool); + var idx: usize = 0; while (idx < self.handles.len) : (idx += 1) { const is_closed = for (self.closed_handles.items) |closed_idx| { @@ -765,95 +867,40 @@ pub const S3 = struct { } fn performRead(self: *S3, handle: *Handle, data: []const []u8, offset: u64) !usize { - if (offset >= handle.size) return 0; + const read_size = parallel_read.readSize(handle.size, offset, data); + if (read_size == 0) return 0; var url_buf: [8 * 1024]u8 = undefined; const url = try self.s3Url(handle.uri, &url_buf); - const uri = std.Uri.parse(url) catch return error.BadPathName; - var range_buf: [64]u8 = undefined; - const range_header = blk: { - var total_bytes: u64 = 0; - for (data) |buf| { - total_bytes += @as(u64, buf.len); - } - const remaining = handle.size - offset; - const take = @min(remaining, total_bytes); - const end = offset + take - 1; - break :blk std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ offset, end }) catch unreachable; + const job_count = std.math.divCeil(usize, read_size, self.read_pool.chunk_size) catch unreachable; + const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); + defer self.allocator.free(jobs); + const pending: u32 = @intCast(job_count); + + var batch: ParallelRead.Batch = .{ + .executor = .{ .pending = .init(pending) }, + .backend = self, + .uri = uri, + .url = url, }; - var timestamp_buf: [16]u8 = undefined; - const timestamp = try self.getTimestamp(×tamp_buf); - - const signer: AwsSigV4 = .{ - .access_key = self.config.access_key, - .secret_key = self.config.secret_key, - .region = self.config.region, - .service = self.config.auth_service, - }; - - var authorization_buffer: [512]u8 = undefined; - const authorization = try signer.generateAuthHeader(&authorization_buffer, .GET, uri, timestamp, &.{}); - - var req = try self.client.request(.GET, uri, .{ .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = if (authorization) |auth| .{ .override = auth } else .omit, - }, .extra_headers = &.{ - .{ .name = "x-amz-date", .value = timestamp }, - .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, - .{ .name = "Range", .value = range_header }, - } }); - defer req.deinit(); - - try req.sendBodiless(); - - var redirect_buffer: [8 * 1024]u8 = undefined; - var res = try req.receiveHead(&redirect_buffer); - - if (res.head.status != .partial_content and res.head.status != .ok) { - log.err("Failed to read {s}: {s}", .{ handle.uri, res.head.bytes }); - return error.RequestFailed; + for (jobs, 0..) |*job, i| { + const chunk_offset = i * self.read_pool.chunk_size; + job.* = .{ + .data = data, + .file_offset = offset + @as(u64, @intCast(chunk_offset)), + .chunk_offset = chunk_offset, + .chunk_len = @min(self.read_pool.chunk_size, read_size - chunk_offset), + .batch = &batch, + }; } - const content_range = blk: { - var it = res.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { - break :blk parseContentRange(header.value); - } - } - break :blk null; - }; + try self.read_pool.job_queue.putAll(self.base.inner, jobs); + batch.executor.waitUncancelable(self.base.inner); + if (batch.executor.firstError()) |err| return err; - const reader = res.reader(&.{}); - - if (content_range) |cr| { - if (cr.start < offset) { - try reader.discardAll(offset - cr.start); - } - } - - return try reader.readSliceShort(data[0]); - } - - const ContentRange = struct { - start: u64, - end: u64, - total: u64, - }; - - fn parseContentRange(value: []const u8) ?ContentRange { - // Format: "bytes start-end/total" - const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; - const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; - const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; - - return .{ - .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, - .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, - .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, - }; + return read_size; } }; From 4792fba3c15ae44af87bd59554df309d6a49e025 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Sat, 27 Jun 2026 00:26:50 +0200 Subject: [PATCH 541/881] bazel: improve remote build resilience (#610) Enable lost-input rewinding and local fallback for the remote downloader. --- .bazelrc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.bazelrc b/.bazelrc index 2fa7d97f..8f1846fd 100644 --- a/.bazelrc +++ b/.bazelrc @@ -117,6 +117,10 @@ common:remote --nobuild_runfile_links common:remote --remote_timeout=3600 common:remote --noexperimental_throttle_remote_action_building common:remote --remote_cache_compression +# Re-runs actions instead of failing on transient lost remote/disk cache inputs. +common:remote --rewind_lost_inputs +# Falls back to local downloads if the remote asset downloader is unavailable. +common:remote --experimental_remote_downloader_local_fallback common:remote --grpc_keepalive_time=30s common:remote --jobs=2000 common:remote --extra_execution_platforms=@zml//:rbe_platform From 076417363a53078c0acf006b297997dcd8431eea Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Sat, 27 Jun 2026 00:51:55 +0200 Subject: [PATCH 542/881] workspace: add missing loads (#606) nits --- platforms/neuron/libpjrt_neuron.BUILD.bazel | 1 + platforms/neuron/neuron.bzl | 2 ++ platforms/rocm/libpjrt_rocm.BUILD.bazel | 21 +++++++++++---------- platforms/rocm/rocm.bzl | 2 ++ 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/platforms/neuron/libpjrt_neuron.BUILD.bazel b/platforms/neuron/libpjrt_neuron.BUILD.bazel index ff7bbabe..d8f43146 100644 --- a/platforms/neuron/libpjrt_neuron.BUILD.bazel +++ b/platforms/neuron/libpjrt_neuron.BUILD.bazel @@ -1,5 +1,6 @@ load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@bazel_skylib//rules:select_file.bzl", "select_file") +load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@zml//bazel:patchelf.bzl", "patchelf") load("@zml//bazel:runfiles.bzl", "runfiles_to_default") diff --git a/platforms/neuron/neuron.bzl b/platforms/neuron/neuron.bzl index 76123ab0..c6f772fa 100644 --- a/platforms/neuron/neuron.bzl +++ b/platforms/neuron/neuron.bzl @@ -10,6 +10,8 @@ STRIP_PREFIX = "./opt/aws/neuron" _BUILD_FILE_PRELUDE = """\ package(default_visibility = ["//visibility:public"]) + +load("@rules_cc//cc:cc_library.bzl", "cc_library") """ _UBUNTU_PACKAGES = { diff --git a/platforms/rocm/libpjrt_rocm.BUILD.bazel b/platforms/rocm/libpjrt_rocm.BUILD.bazel index efffb73e..6238e1a8 100644 --- a/platforms/rocm/libpjrt_rocm.BUILD.bazel +++ b/platforms/rocm/libpjrt_rocm.BUILD.bazel @@ -1,6 +1,7 @@ load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "string_list_flag") load("@bazel_skylib//rules:run_binary.bzl", "run_binary") +load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_binary") load("@zml//bazel:patchelf.bzl", "patchelf") @@ -51,20 +52,21 @@ run_binary( copy_to_directory( name = "sandbox", srcs = [ + ":amdgpu_ids", ":libpjrt_rocm_so", "@comgr//:amd_comgr", "@hip-runtime-amd//:amdhip", "@hip-runtime-amd//:hiprtc", - "@hipblaslt//:hipblaslt", + "@hipblaslt", "@hipblaslt//:rocroller", "@hipfft", "@hipsolver", "@hsa-amd-aqlprofile//:hsa-amd-aqlprofile", "@hsa-rocr//:hsa-runtime", - "@libdrm-amdgpu-amdgpu1", - ":amdgpu_ids", - "@libdrm2-amdgpu", "@libbz2-1.0", + "@libdrm-amdgpu-amdgpu1", + "@libdrm2-amdgpu", + "@libdw1", "@libelf1", "@liblzma5", "@libnuma1", @@ -72,27 +74,28 @@ copy_to_directory( "@libzstd1", "@miopen-hip//:MIOpen", "@rccl", - "@rocblas//:rocblas", + "@rocblas", "@rocblas//:runfiles", "@rocfft", "@rocm-core", "@rocm-device-libs//:runfiles", "@rocm-smi-lib//:rocm_smi", "@rocprofiler-register", - "@rocprofiler-sdk//:rocprofv3_tool", "@rocprofiler-sdk-rocpd//:rocprofiler-sdk-rocpd", "@rocprofiler-sdk-roctx//:rocprofiler-sdk-roctx", - "@rocsparse", + "@rocprofiler-sdk//:rocprofv3_tool", "@rocsolver", + "@rocsparse", "@roctracer", "@roctracer//:roctx", "@zlib1g", - "@libdw1", "@zml//platforms/rocm:zmlxrocm", ] + select({ ":_hipblaslt": ["@hipblaslt//:runfiles"], "//conditions:default": [], }), + add_directory_to_runfiles = True, + include_external_repositories = ["**"], replace_prefixes = { "MIOpen": "lib", "amdhip": "lib", @@ -123,8 +126,6 @@ copy_to_directory( "platforms/rocm": "lib", "usr/lib/x86_64-linux-gnu": "lib", }, - add_directory_to_runfiles = True, - include_external_repositories = ["**"], visibility = ["//visibility:public"], ) diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 040a1084..806603c6 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -5,6 +5,8 @@ load("//platforms:packages.bzl", "packages") _BUILD_FILE_DEFAULT_VISIBILITY = """\ package(default_visibility = ["//visibility:public"]) + +load("@rules_cc//cc:cc_library.bzl", "cc_library") """ _ROCM_STRIP_PREFIX = "./opt/rocm-7.2.2" From 6251953668928d55c5f25a9f2bed5d52c3dbd422 Mon Sep 17 00:00:00 2001 From: elogir Date: Mon, 29 Jun 2026 10:43:40 +0200 Subject: [PATCH 543/881] platforms/metal: add moe support (#613) --- examples/llm/models/qwen3_5_moe/inference.zig | 2 +- platforms/metal/metal.bzl | 2 +- zml/BUILD.bazel | 1 + zml/moe/metal.zig | 141 ++++++++++++++++++ zml/moe/moe.zig | 48 ++++++ 5 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 zml/moe/metal.zig diff --git a/examples/llm/models/qwen3_5_moe/inference.zig b/examples/llm/models/qwen3_5_moe/inference.zig index e8941892..516760d4 100644 --- a/examples/llm/models/qwen3_5_moe/inference.zig +++ b/examples/llm/models/qwen3_5_moe/inference.zig @@ -475,6 +475,6 @@ fn initMoeMetadata(qwen_model: model.Model, token_len: usize, batch_size: u32, b .w2_zero_bias_shape = w2_zero_bias_shape, }, }), - .mosaic_tpu => .init(.fromBackend(backend)), + .mosaic_tpu, .metal => .init(.fromBackend(backend)), }; } diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index e26bf1f8..7ecfcf4f 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,7 +4,7 @@ def _metal_impl(mctx): http_archive( name = "libpjrt_metal", build_file = "libpjrt_metal.BUILD.bazel", - sha256 = "d6dac3b0df3390f56e4f0fa41241e949ce32d6266b2f7ed2aa677be4c942acce", + sha256 = "a94aaf5719fe14b2e1fd4e614df9e916443b778ced453eebea419ee479f245aa", url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-06-23T00-20-00Z/pjrt-metal_macos-arm64.tar.zst", ) diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 126da753..da02522a 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -89,6 +89,7 @@ zig_library( "attention/triton_attention.zig", "attention/triton_kernels/unified_attention.zig", "attention/triton_kernels/unified_attention_oneapi.zig", + "moe/metal.zig", "moe/moe.zig", "moe/mosaic_tpu.zig", "moe/mosaic_tpu_kernels/gmm_ep.zig", diff --git a/zml/moe/metal.zig b/zml/moe/metal.zig new file mode 100644 index 00000000..8738904d --- /dev/null +++ b/zml/moe/metal.zig @@ -0,0 +1,141 @@ +const std = @import("std"); + +const zml = @import("../zml.zig"); +const Tensor = zml.Tensor; + +pub const ActivationMode = enum { + silu, + relu, + gelu, +}; + +pub const Options = struct { + activation: ActivationMode = .silu, + global_num_experts: i64 = -1, + expert_map: ?Tensor = null, + w1_scale: ?Tensor = null, + w2_scale: ?Tensor = null, + w1_bias: ?Tensor = null, + w2_bias: ?Tensor = null, +}; + +pub const Parameters = struct { + num_experts_per_tok: u32, + activation: ActivationMode, + + pub const InitOptions = struct { + num_experts_per_tok: u32, + activation: ActivationMode = .silu, + }; + + pub fn init(opts: InitOptions) Parameters { + return .{ + .num_experts_per_tok = opts.num_experts_per_tok, + .activation = opts.activation, + }; + } +}; + +pub const Metadata = struct { + pub const InitOptions = struct {}; + + pub fn init(opts: InitOptions) Metadata { + _ = opts; + return .{}; + } + + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { + _ = self; + _ = io; + _ = platform; + return {}; + } +}; + +pub fn deinitBuffer(bufferized: *zml.Bufferized(Metadata)) void { + _ = bufferized; +} + +fn validateOptions(opts: Options) !void { + if (opts.expert_map != null) return error.InvalidShape; + if (opts.w1_scale != null or opts.w2_scale != null) return error.UnsupportedQuantization; + if (opts.w1_bias != null or opts.w2_bias != null) return error.UnsupportedBias; +} + +fn validateInputs(hidden: Tensor, gate_up: Tensor, down: Tensor, weights: Tensor, ids: Tensor) !void { + for ([_]Tensor{ hidden, gate_up, down, weights }) |t| { + switch (t.dtype()) { + .bf16, .f16, .f32 => {}, + else => return error.UnsupportedType, + } + } + + if (ids.dtype() != .i32) return error.UnsupportedType; +} + +fn applyActivation(x: Tensor, mode: ActivationMode) Tensor { + const mid = @divFloor(x.dim(.out), 2); + const gate = x.slice1d(.out, .{ .end = mid }); + const up = x.slice1d(.out, .{ .start = mid }); + + const act = switch (mode) { + .silu => gate.silu().mul(up), + .gelu => gate.gelu().mul(up), + .relu => gate.relu().mul(up), + }; + + return act.rename(.{ .out = .mid }); +} + +pub fn fusedExpertsImpl( + hidden_states: Tensor, + gate_up: Tensor, + down: Tensor, + topk_weights: Tensor, + topk_ids: Tensor, + metadata: Metadata, + opts: Options, +) !Tensor { + _ = metadata; + try validateOptions(opts); + try validateInputs(hidden_states, gate_up, down, topk_weights, topk_ids); + + const b = hidden_states.dim(.b); + const s = hidden_states.dim(.s); + const d = hidden_states.dim(.d); + const num_tokens = b * s; + const top_expert = topk_ids.dim(.top_expert); + const num_routes = num_tokens * top_expert; + + const hidden = hidden_states.reshape(.{ .token = num_tokens, .d = d }).withTags(.{ .token, .d }); + const x_rows = hidden.insertAxes(.d, .{.top_expert}) + .broad(zml.Shape.init(.{ .token = num_tokens, .top_expert = top_expert, .d = d }, hidden.dtype())) + .merge(.{ .r = .{ .token, .top_expert } }); + const expert_ids = topk_ids.reshape(.{ .token = num_tokens, .top_expert = top_expert }).withTags(.{ .token, .top_expert }) + .merge(.{ .r = .{ .token, .top_expert } }).convert(.i32); + + const gate_up_out = zml.ops.customCall( + "__metal$moe_gemm", + .{ x_rows, gate_up, expert_ids }, + .{zml.Shape.init(.{ .r = num_routes, .out = gate_up.dim(.dout) }, x_rows.dtype())}, + .{}, + .{ .has_side_effect = false }, + ); + const activated = applyActivation(gate_up_out, opts.activation); + + const down_out = zml.ops.customCall( + "__metal$moe_gemm", + .{ activated, down, expert_ids }, + .{zml.Shape.init(.{ .r = num_routes, .d = down.dim(.d) }, activated.dtype())}, + .{}, + .{ .has_side_effect = false }, + ); + + const weights = topk_weights.reshape(.{ .token = num_tokens, .top_expert = top_expert }).withTags(.{ .token, .top_expert }) + .merge(.{ .r = .{ .token, .top_expert } }); + const weighted = down_out.mul(weights.convert(down_out.dtype()).broad(down_out.shape())); + const combined = weighted.splitAxis(.r, .{ .token = num_tokens, .top_expert = top_expert }) + .sum(.top_expert).squeeze(.top_expert); + + return combined.reshape(.{ .b = b, .s = s, .d = down.dim(.d) }); +} diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index 41527187..b5a943f0 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -2,6 +2,7 @@ const std = @import("std"); const zml = @import("../zml.zig"); const stdx = zml.stdx; +pub const metal = @import("metal.zig"); pub const mosaic_tpu = @import("mosaic_tpu.zig"); pub const triton = @import("triton.zig"); pub const triton_kernels = @import("triton_kernels/triton_kernels.zig"); @@ -16,6 +17,7 @@ pub const Backend = enum { // Could select a more specific name like "triton_sm90_bf16" triton, mosaic_tpu, + metal, pub fn auto(platform: *const zml.Platform, weights_dtype: zml.DataType) !Backend { return switch (platform.target) { @@ -27,6 +29,10 @@ pub const Backend = enum { .bf16, .f16, .f32 => .mosaic_tpu, else => error.UnsupportedDataType, }, + .metal => switch (weights_dtype) { + .bf16, .f16, .f32 => .metal, + else => error.UnsupportedDataType, + }, else => error.UnimplementedMoEBackend, }; } @@ -36,6 +42,7 @@ pub const Backend = enum { return switch (backend) { .triton => {}, .mosaic_tpu => {}, + .metal => {}, }; } @@ -44,6 +51,7 @@ pub const Backend = enum { return switch (backend) { .triton => {}, .mosaic_tpu => {}, + .metal => {}, }; } }; @@ -51,10 +59,12 @@ pub const Backend = enum { pub const Parameters = union(Backend) { triton: triton.Parameters, mosaic_tpu: mosaic_tpu.Parameters, + metal: metal.Parameters, pub const InitOptions = union(Backend) { triton: triton.Parameters.InitOptions, mosaic_tpu: mosaic_tpu.Parameters.InitOptions, + metal: metal.Parameters.InitOptions, pub fn fromBackend(backend: Backend, num_experts_per_tok: ?u32, activation: ActivationMode) InitOptions { return switch (backend) { @@ -74,6 +84,14 @@ pub const Parameters = union(Backend) { .gelu => .gelu, }, } }, + .metal => .{ .metal = .{ + .num_experts_per_tok = num_experts_per_tok.?, + .activation = switch (activation) { + .silu => .silu, + .relu => .relu, + .gelu => .gelu, + }, + } }, }; } }; @@ -82,6 +100,7 @@ pub const Parameters = union(Backend) { return switch (opts) { .triton => |v| .{ .triton = triton.Parameters.init(v) }, .mosaic_tpu => |v| .{ .mosaic_tpu = mosaic_tpu.Parameters.init(v) }, + .metal => |v| .{ .metal = metal.Parameters.init(v) }, }; } }; @@ -89,15 +108,18 @@ pub const Parameters = union(Backend) { pub const Metadata = union(Backend) { triton: triton.Metadata, mosaic_tpu: mosaic_tpu.Metadata, + metal: metal.Metadata, pub const InitOptions = union(Backend) { triton: triton.Metadata.InitOptions, mosaic_tpu: mosaic_tpu.Metadata.InitOptions, + metal: metal.Metadata.InitOptions, pub fn fromBackend(backend: Backend) InitOptions { return switch (backend) { .triton => .{ .triton = .{} }, .mosaic_tpu => .{ .mosaic_tpu = .{} }, + .metal => .{ .metal = .{} }, }; } }; @@ -106,6 +128,7 @@ pub const Metadata = union(Backend) { return switch (opts) { .triton => |v| .{ .triton = triton.Metadata.init(v) }, .mosaic_tpu => |v| .{ .mosaic_tpu = mosaic_tpu.Metadata.init(v) }, + .metal => |v| .{ .metal = metal.Metadata.init(v) }, }; } @@ -113,6 +136,7 @@ pub const Metadata = union(Backend) { return switch (self) { .triton => |metadata| .{ .triton = try metadata.initBuffer(io, platform) }, .mosaic_tpu => |metadata| .{ .mosaic_tpu = try metadata.initBuffer(io, platform) }, + .metal => |metadata| .{ .metal = try metadata.initBuffer(io, platform) }, }; } @@ -120,6 +144,7 @@ pub const Metadata = union(Backend) { switch (self.*) { .triton => |*metadata| triton.deinitBuffer(metadata), .mosaic_tpu => |*metadata| mosaic_tpu.deinitBuffer(metadata), + .metal => |*metadata| metal.deinitBuffer(metadata), } } }; @@ -291,5 +316,28 @@ pub fn forwardMoe( }, ); }, + .metal => b: { + const metal_metadata = switch (metadata) { + .metal => |v| v, + else => return error.InvalidMetadata, + }; + + break :b try metal.fusedExpertsImpl( + input, + weights_gate_up, + weights_down, + topk_weights, + topk_ids, + metal_metadata, + .{ + .activation = parameters.metal.activation, + .global_num_experts = weights_gate_up.dim(.expert), + .w1_scale = scales_gate_up, + .w2_scale = scales_down, + .w1_bias = bias_gate_up, + .w2_bias = bias_down, + }, + ); + }, }; } From 4c2393e5cf15316c01ce61777aad8bd4d405b7e7 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Mon, 29 Jun 2026 14:43:42 +0200 Subject: [PATCH 544/881] bazel: scope Zig cache sandbox mounts to Linux (#615) ## Summary - Limit Zig cache sandbox mount flags to Linux, where linux-sandbox and Bazel's Docker sandbox consume host path mounts. - Remove the macOS mount entry because sandbox-exec uses a symlinked execroot and ignores bind-mount settings. - Mark `/tmp` and `/var/tmp` writable alongside the Linux mount pairs. --- .bazelrc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.bazelrc b/.bazelrc index 8f1846fd..d5587eb3 100644 --- a/.bazelrc +++ b/.bazelrc @@ -26,9 +26,16 @@ common --@rules_python//python/config_settings:bootstrap_impl=script # We don't to check the built files in the sandbox or repositories common --experimental_check_output_files=false -# Mount the zig cache into the sandboxes -build:macos --sandbox_add_mount_pair=/var/tmp +# Expose Zig's shared cache directories to sandboxed actions when the +# selected sandbox implementation supports host path mounts. +# +# These flags are consumed by linux-sandbox, and by Bazel's Docker sandbox. +# They are ignored by processwrapper-sandbox and macOS sandbox-exec, which use +# a symlinked execroot rather than bind mounts. build:linux --sandbox_add_mount_pair=/tmp +build:linux --sandbox_writable_path=/tmp +build:linux --sandbox_add_mount_pair=/var/tmp +build:linux --sandbox_writable_path=/var/tmp # Ensure workers are sandboxed build --worker_sandboxing From d1e4ef9e29f3ffaa1515334956ea47f5897f235a Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Mon, 29 Jun 2026 14:54:07 +0200 Subject: [PATCH 545/881] zml/tensor: fix segmentation fault when using `split` Tensor (#614) Previously, split tensors where allocated using an arena that would be free at the end of the function scope. This commit uses the arena context instead. Signed-off-by: dmehala --- zml/tensor.zig | 49 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/zml/tensor.zig b/zml/tensor.zig index af5e6c33..e75208d8 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -3511,10 +3511,9 @@ pub const Tensor = struct { for (split_sizes) |n| split_sum += n; stdx.debug.assert(split_sum == d, "split expects sum of 'split_sizes' values and axis dimension to be equal, got {} and {}", .{ split_sum, d }); - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); - defer arena.deinit(); + const allocator = CompilationContext.current().arena.allocator(); - const res = arena.allocator().alloc(Tensor, split_sizes.len) catch @panic("OOM"); + const res = allocator.alloc(Tensor, split_sizes.len) catch @panic("OOM"); var start: i64 = 0; for (split_sizes, 0..) |n, i| { @@ -3524,6 +3523,50 @@ pub const Tensor = struct { return res; } + test split { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const Local = struct { + pub fn _split(x: Tensor, axis_: i8, split_sizes: []const i64) []Tensor { + return x.split(axis_, split_sizes); + } + }; + + inline for (.{ + .{ [3]f32{ 0, 1, 2 }, Shape.init(.{3}, .f32), 0, [3]i64{ 1, 1, 1 }, &.{ [1]f32{0}, [1]f32{1}, [1]f32{2} } }, + .{ [10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, Shape.init(.{10}, .f32), 0, [2]i64{ 1, 9 }, &.{ [1]f32{0}, [9]f32{ 1, 2, 3, 4, 5, 6, 7, 8, 9 } } }, + .{ [4][3]f32{ .{ 0, 1, 2 }, .{ 3, 4, 5 }, .{ 6, 7, 8 }, .{ 9, 10, 11 } }, Shape.init(.{ 4, 3 }, .f32), 0, [2]i64{ 1, 3 }, &.{ [1][3]f32{.{ 0, 1, 2 }}, [3][3]f32{ .{ 3, 4, 5 }, .{ 6, 7, 8 }, .{ 9, 10, 11 } } } }, + .{ [2][6]f32{ .{ 0, 1, 2, 3, 4, 5 }, .{ 6, 7, 8, 9, 10, 11 } }, Shape.init(.{ 2, 6 }, .f32), 1, [2]i64{ 2, 4 }, &.{ [2][2]f32{ .{ 0, 1 }, .{ 6, 7 } }, [2][4]f32{ .{ 2, 3, 4, 5 }, .{ 8, 9, 10, 11 } } } }, + }) |testcase| { + const x_data, const x_shape, const ax, const args, const expectation = testcase; + const x: Tensor = .fromShape(x_shape); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._split, .{ x, ax, &args }, platform, .{}); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&x_data)); + defer x_buffer.deinit(); + + var exe_args = try exe.args(std.testing.allocator); + defer exe_args.deinit(std.testing.allocator); + exe_args.set(.{x_buffer}); + + var results = try exe.results(std.testing.allocator); + defer results.deinit(std.testing.allocator); + exe.callOpts(std.testing.io, exe_args, &results, .{ .wait = true }); + + const res = try std.testing.allocator.alloc(zml.Buffer, expectation.len); + defer std.testing.allocator.free(res); + results.fill(res); + defer for (res) |*buffer| buffer.deinit(); + + inline for (expectation, 0..) |expected, i| { + try std.testing.expectEqual(expected, try res[i].getValue(@TypeOf(expected), std.testing.io)); + } + } + } + /// Slices the input Tensor along a specific axis, with a start offset known at runtime. /// Note: this doesn't support tagging, if you have tags, /// you should use `dynamicSlice` directly. From be684a6dce960ce789f27856aa2e2deb73ac4795 Mon Sep 17 00:00:00 2001 From: Reed Meyerson <31574681+reed-meyerson@users.noreply.github.com> Date: Mon, 29 Jun 2026 11:32:32 -0400 Subject: [PATCH 546/881] docs/tutorials: fix 'write_first_model' example code (#612) Fix in the `write_first_model.md` tutorial: Update main() code section in to match full example --- docs/tutorials/write_first_model.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/write_first_model.md b/docs/tutorials/write_first_model.md index f6174b36..f57a8f68 100644 --- a/docs/tutorials/write_first_model.md +++ b/docs/tutorials/write_first_model.md @@ -135,7 +135,7 @@ pub fn main(init: std.process.Init) !void { const io = init.io; var platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator); + defer platform.deinit(allocator, io); ... } From 89b0908ca55f0e280903ae67caf470c110344821 Mon Sep 17 00:00:00 2001 From: Reese Chong Date: Mon, 29 Jun 2026 17:37:13 +0200 Subject: [PATCH 547/881] zml/attention/flashattn: make fa2 window_size_left configurable (#595) Add a sliding_window field to the FA2 parameters and use it to set the window_size_left argument in zml.attention.attention(). --- zml/attention/flashattn.zig | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 723316c3..03234a64 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -215,6 +215,8 @@ pub const fa2 = struct { }); pub const Parameters = struct { + sliding_window: i32 = -1, + pub const InitOptions = struct {}; pub fn init(_: InitOptions) fa2.Parameters { @@ -263,7 +265,7 @@ pub const fa2 = struct { } }; - pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, _: Parameters) zml.Tensor { + pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { const ctx = CompilationContext.current(); stdx.debug.assert(q_.shape().hasTag(.b) == null or q_.dim(.b) == 1, "fa2.attention support for batch size != 1 is not supported yet.", .{}); @@ -291,7 +293,7 @@ pub const fa2 = struct { const num_heads_k = k_.dim(.h); const head_size = q_.dim(.hd); const ngroups = @divExact(num_heads, num_heads_k); - const seqlenq_ngroups_swapped = max_seqlen_q == 1 and num_heads > num_heads_k and @mod(head_size, 8) == 0; + const seqlenq_ngroups_swapped = max_seqlen_q == 1 and num_heads > num_heads_k and @mod(head_size, 8) == 0 and parameters.sliding_window < 0; if (seqlenq_ngroups_swapped) { q = q.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .tot, .ngroups, .h, .hd }).merge(.{ .tot = .{ .tot, .ngroups } }); } @@ -319,7 +321,7 @@ pub const fa2 = struct { break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); }, .is_causal = true, - .window_size_left = @as(i32, -1), + .window_size_left = parameters.sliding_window, .window_size_right = @as(i32, -1), .max_seqlen_q = max_seqlen_q, .max_seqlen_k = max_seqlen_k, From cb12e2ea71df08b0d695985cad61488082188585 Mon Sep 17 00:00:00 2001 From: Olivier Boulant Date: Tue, 30 Jun 2026 15:58:00 +0200 Subject: [PATCH 548/881] zml/io: update defaults for zml.io.load and read parallelism --- examples/io/main.zig | 2 +- examples/llm/models/lfm2/model.zig | 4 ++-- examples/llm/models/llama/model.zig | 4 ++-- examples/llm/models/qwen3_5/model.zig | 4 ++-- examples/llm/models/qwen3_5_moe/model.zig | 4 ++-- zml/io/vfs/parallel_read.zig | 6 +++--- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/examples/io/main.zig b/examples/io/main.zig index f2c180f6..d2c154ba 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -211,7 +211,7 @@ pub fn main(init: std.process.Init) !void { .shardings = &.{sharded_sharding}, .parallelism = 8, .dma_chunks = 16, - .dma_chunk_size = 64 * zml.MiB, + .dma_chunk_size = 256 * zml.MiB, .progress = &progress, .total_bytes = &total_bytes, }); diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index 8e4c0956..3d25c3cf 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -97,7 +97,7 @@ pub const LoadedModel = struct { const all_shardings = shardings.all(); return zml.io.load(Model, &self.inner, allocator, io, platform, store, .{ .dma_chunks = 32, - .dma_chunk_size = 128 * zml.MiB, + .dma_chunk_size = 256 * zml.MiB, .progress = progress, .shardings = &all_shardings, .parallelism = 16, @@ -189,7 +189,7 @@ pub const Model = struct { const all_shardings = shardings.all(); return zml.io.load(Model, self, allocator, io, platform, store, .{ .dma_chunks = 8, - .dma_chunk_size = 128 * zml.MiB, + .dma_chunk_size = 256 * zml.MiB, .progress = progress, .parallelism = 16, .total_bytes = &total_bytes, diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index 857c5171..ad0b1f8a 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -83,7 +83,7 @@ pub const LoadedModel = struct { return zml.io.load(Model, &self.inner, allocator, io, platform, store, .{ .dma_chunks = 32, - .dma_chunk_size = 128 * zml.MiB, + .dma_chunk_size = 256 * zml.MiB, .progress = progress, .shardings = &shardings.all(), .parallelism = 16, @@ -174,7 +174,7 @@ pub const Model = struct { return zml.io.load(Model, self, allocator, io, platform, store, .{ .dma_chunks = 32, - .dma_chunk_size = 128 * zml.MiB, + .dma_chunk_size = 256 * zml.MiB, .progress = progress, .shardings = shardings, .parallelism = 16, diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index bc14ab72..59fd2c8c 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -98,7 +98,7 @@ pub const LoadedModel = struct { const all_shardings = shardings.all(); return zml.io.load(Model, &self.inner, allocator, io, platform, store, .{ .dma_chunks = 32, - .dma_chunk_size = 128 * zml.MiB, + .dma_chunk_size = 256 * zml.MiB, .progress = progress, .shardings = &all_shardings, .parallelism = 16, @@ -191,7 +191,7 @@ pub const Model = struct { } return zml.io.load(Model, self, allocator, io, platform, store, .{ .dma_chunks = 32, - .dma_chunk_size = 128 * zml.MiB, + .dma_chunk_size = 256 * zml.MiB, .progress = progress, .shardings = shardings, .parallelism = 16, diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index c222d8c1..7b0d4a81 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -124,7 +124,7 @@ pub const LoadedModel = struct { const all_shardings = shardings.all(); var buffers = try zml.io.load(Model, &self.inner, allocator, io, platform, store, .{ .dma_chunks = 32, - .dma_chunk_size = 128 * zml.MiB, + .dma_chunk_size = 256 * zml.MiB, .progress = progress, .shardings = &all_shardings, .parallelism = 16, @@ -209,7 +209,7 @@ pub const Model = struct { } return zml.io.load(Model, self, allocator, io, platform, store, .{ .dma_chunks = 32, - .dma_chunk_size = 128 * zml.MiB, + .dma_chunk_size = 256 * zml.MiB, .progress = progress, .shardings = shardings, .parallelism = 16, diff --git a/zml/io/vfs/parallel_read.zig b/zml/io/vfs/parallel_read.zig index aa4102cb..d5d7855c 100644 --- a/zml/io/vfs/parallel_read.zig +++ b/zml/io/vfs/parallel_read.zig @@ -3,9 +3,9 @@ const std = @import("std"); const stdx = @import("stdx"); pub const InitOpts = struct { - chunk_size: usize = 10 * 1024 * 1024, - num_workers: usize = 16, - queue_capacity: usize = 64, + chunk_size: usize = 16 * 1024 * 1024, + num_workers: usize = 32, + queue_capacity: usize = 128, }; pub const BatchExecutor = struct { From 0e12e4c998ccb6afcfb770ab32618e2b144fffd1 Mon Sep 17 00:00:00 2001 From: Louis Lechevalier Date: Tue, 30 Jun 2026 17:31:14 +0200 Subject: [PATCH 549/881] platforms/cuda: update pjrt plugin (#617) related: https://github.com/zml/pjrt-artifacts/pull/91/changes From 130 to 200 tok/s for Gemma 4 26B A4B on GH200 --- platforms/cuda/cuda.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 4937d70d..fbfafb58 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -205,15 +205,15 @@ _UBUNTU_PACKAGES = { ], } -PJRT_CUDA_RELEASE = "manual-2026-04-30T17-13-00Z" +PJRT_CUDA_RELEASE = "manual-2026-06-30T00-16-30Z" _PJRT_CUDA_ASSETS = { "amd64": { - "sha256": "bab2fe379d8cfdd54f12cb537f44f72f8f023665cfe23524a06930f8d7d12010", + "sha256": "50c7219b111d0d74ad66c4748cb87691e7e146b7e21fb6ed66ae6af6d97589ca", "url": "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-cuda_linux-amd64.tar.gz", }, "arm64": { - "sha256": "cce6524a263fcbfa4f40bd39419d26517629f7f0c288707b3af81fe84ad93c45", + "sha256": "dcaec78fb141b52884f98e28cb12683d5074fb26aabe8cc56b208a6563fa1d9f", "url": "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-cuda_linux-arm64.tar.gz", }, } From d6e9b473914ab71449d8fd08effa5df8e5b56e2a Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 1 Jul 2026 16:20:48 +0200 Subject: [PATCH 550/881] zml/examples/llm: simplify dispatch code (#619) `@unionInit` ftw --- examples/llm/models.zig | 134 +++++++--------------------------------- 1 file changed, 24 insertions(+), 110 deletions(-) diff --git a/examples/llm/models.zig b/examples/llm/models.zig index 4f364f22..839c1706 100644 --- a/examples/llm/models.zig +++ b/examples/llm/models.zig @@ -41,10 +41,11 @@ pub const LoadedModel = union(ModelType) { log.info("Detected model type: {}", .{model_type}); return switch (model_type) { - .lfm2 => .{ .lfm2 = try lfm2.LoadedModel.init(allocator, io, repo, store, generation) }, - .llama => .{ .llama = try llama.LoadedModel.init(allocator, io, repo, store, generation) }, - .qwen3_5 => .{ .qwen3_5 = try qwen3_5.LoadedModel.init(allocator, io, repo, store, generation) }, - .qwen3_5_moe => .{ .qwen3_5_moe = try qwen3_5_moe.LoadedModel.init(allocator, io, repo, store, generation) }, + inline else => |t| @unionInit( + LoadedModel, + @tagName(t), + try .init(allocator, io, repo, store, generation), + ), }; } @@ -56,31 +57,17 @@ pub const LoadedModel = union(ModelType) { pub fn loadBuffers(self: *LoadedModel, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, shardings: Shardings) !Buffers { return switch (self.*) { - .lfm2 => |*m| .{ .lfm2 = try m.loadBuffers(allocator, io, platform, store, progress, shardings) }, - .llama => |*m| .{ .llama = try m.loadBuffers(allocator, io, platform, store, progress, shardings) }, - .qwen3_5 => |*m| .{ .qwen3_5 = try m.loadBuffers(allocator, io, platform, store, progress, shardings) }, - .qwen3_5_moe => |*m| .{ .qwen3_5_moe = try m.loadBuffers(allocator, io, platform, store, progress, shardings) }, + inline else => |*m, t| @unionInit( + Buffers, + @tagName(t), + try m.loadBuffers(allocator, io, platform, store, progress, shardings), + ), }; } pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { switch (self.*) { - .lfm2 => |*loaded_model| switch (buffers.*) { - .lfm2 => |*loaded_buffers| loaded_model.unloadBuffers(loaded_buffers, allocator), - else => unreachable, - }, - .llama => |*loaded_model| switch (buffers.*) { - .llama => |*loaded_buffers| loaded_model.unloadBuffers(loaded_buffers, allocator), - else => unreachable, - }, - .qwen3_5 => |*loaded_model| switch (buffers.*) { - .qwen3_5 => |*loaded_buffers| loaded_model.unloadBuffers(loaded_buffers, allocator), - else => unreachable, - }, - .qwen3_5_moe => |*loaded_model| switch (buffers.*) { - .qwen3_5_moe => |*loaded_buffers| loaded_model.unloadBuffers(loaded_buffers, allocator), - else => unreachable, - }, + inline else => |*loaded, t| loaded.unloadBuffers(&@field(buffers, @tagName(t)), allocator), } } @@ -95,47 +82,13 @@ pub const LoadedModel = union(ModelType) { progress: *std.Progress.Node, ) !CompiledModel { const inner: CompiledModel.Inner = switch (self.*) { - .lfm2 => |*m| .{ .lfm2 = try m.compile( - allocator, - io, - platform, - backend, - shardings, - seqlen, - progress, - ) }, - .llama => |*m| .{ .llama = try m.compile( - allocator, - io, - platform, - backend, - shardings, - seqlen, - progress, - ) }, - .qwen3_5 => |*m| .{ .qwen3_5 = try m.compile( - allocator, - io, - platform, - backend, - shardings, - seqlen, - progress, - ) }, - .qwen3_5_moe => |*m| .{ .qwen3_5_moe = try m.compile( - allocator, - io, - platform, - backend, - shardings, - seqlen, - progress, - ) }, - }; - return .{ - .inner = inner, - .seqlen = @intCast(seqlen), + inline else => |*m, t| @unionInit( + CompiledModel.Inner, + @tagName(t), + try m.compile(allocator, io, platform, backend, shardings, seqlen, progress), + ), }; + return .{ .inner = inner, .seqlen = @intCast(seqlen) }; } }; @@ -152,10 +105,7 @@ pub const CompiledModel = struct { pub fn deinit(self: *CompiledModel) void { switch (self.inner) { - .lfm2 => |*b| b.deinit(), - .llama => |*b| b.deinit(), - .qwen3_5 => |*b| b.deinit(), - .qwen3_5_moe => |*b| b.deinit(), + inline else => |*b| b.deinit(), } } @@ -168,48 +118,12 @@ pub const CompiledModel = struct { tokenizer: zml.tokenizer.Tokenizer, ) !Session { return switch (self.inner) { - .lfm2 => |*compiled| .{ - .inner = .{ .lfm2 = try lfm2.Session.init( - allocator, - io, - platform, - tokenizer, - compiled, - &model_buffers.lfm2, - ) }, - .seqlen = self.seqlen, - }, - .llama => |*compiled| .{ - .inner = .{ .llama = try llama.Session.init( - allocator, - io, - platform, - tokenizer, - compiled, - &model_buffers.llama, - ) }, - .seqlen = self.seqlen, - }, - .qwen3_5 => |*compiled| .{ - .inner = .{ .qwen3_5 = try qwen3_5.Session.init( - allocator, - io, - platform, - tokenizer, - compiled, - &model_buffers.qwen3_5, - ) }, - .seqlen = self.seqlen, - }, - .qwen3_5_moe => |*compiled| .{ - .inner = .{ .qwen3_5_moe = try qwen3_5_moe.Session.init( - allocator, - io, - platform, - tokenizer, - compiled, - &model_buffers.qwen3_5_moe, - ) }, + inline else => |*compiled, t| .{ + .inner = @unionInit( + Session.Inner, + @tagName(t), + try .init(allocator, io, platform, tokenizer, compiled, &@field(model_buffers, @tagName(t))), + ), .seqlen = self.seqlen, }, }; From e04ccb9ecb6c6e6d4f0665468645c571a7b223c9 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 1 Jul 2026 16:21:04 +0200 Subject: [PATCH 551/881] zml: introduce zml.Compiler (#618) Move the type out of `zml.compile` signature so that we can call `io.async` more easily. Sample code: ``` var layer_compile_future = io.async(zml.Compiler(TransformerLayer.forward).compile, .{ allocator, io, platform, layer_compile_opts, .{ model.model.layers[0], hidden, device_state.kv_cache.layers[0], }, }); ``` --- zml/module.zig | 15 +++++++++++++++ zml/zml.zig | 1 + 2 files changed, 16 insertions(+) diff --git a/zml/module.zig b/zml/module.zig index 22014be6..386db855 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -20,6 +20,7 @@ const Sharding = @import("Sharding.zig"); const Partitioning = Sharding.Partitioning; const Tensor = @import("tensor.zig").Tensor; +const zml_module = @This(); const log = std.log.scoped(.@"zml/module"); var mlir_global_init_mutex: std.Io.Mutex = .init; @@ -190,6 +191,20 @@ pub const CompilationContext = struct { } }; +pub fn Compiler(comptime func: anytype) type { + return struct { + pub fn compile( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + opts: CompilationOptions, + args: std.meta.ArgsTuple(@TypeOf(func)), + ) !Exe { + return zml_module.compile(allocator, io, func, args, platform, opts); + } + }; +} + pub fn compile( allocator: std.mem.Allocator, io: std.Io, diff --git a/zml/zml.zig b/zml/zml.zig index 7c216fcf..dd3dae36 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -30,6 +30,7 @@ pub const meta = @import("meta.zig"); pub const mlir = @import("mlirx.zig"); pub const module = @import("module.zig"); pub const CompilationOptions = module.CompilationOptions; +pub const Compiler = module.Compiler; pub const moe = @import("moe/moe.zig"); pub const nn = @import("nn.zig"); pub const ops = @import("ops.zig"); From e8eb3ca689729f7e4a94bc70fb2770c795e8cb18 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 1 Jul 2026 18:10:32 +0200 Subject: [PATCH 552/881] zml/moe/triton: reduce log level (#621) --- zml/moe/triton.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zml/moe/triton.zig b/zml/moe/triton.zig index 1f2771cc..2d00a2b5 100644 --- a/zml/moe/triton.zig +++ b/zml/moe/triton.zig @@ -167,7 +167,7 @@ pub fn fusedExpertsImpl( num_assignments + num_experts * (block_size_m - 1); const sorted_token_ids, const expert_ids_global, const num_tokens_post_padded = if (naive_block_assignment) blk: { - log.info("Using naive block assignment for MoE kernels. Num assignments: {d}, Num experts: {d}", .{ num_assignments, num_experts }); + log.debug("Using naive block assignment for MoE kernels. Num assignments: {d}, Num experts: {d}", .{ num_assignments, num_experts }); const naive_sorted_ids = Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)); const naive_expert_ids = ids.reshape(.{ .g = num_assignments }); const naive_num_tokens_post_padded = Tensor.constant(.{ .i32 = @as(i32, @intCast(max_num_tokens_padded)) }).reshape(.{1}); @@ -344,7 +344,7 @@ fn callFusedMoe( } fn alignBlockSize(topk_ids: Tensor, num_experts: i64, block_size_m: i64) struct { Tensor, Tensor, Tensor } { - log.info("Using triton kernels to sort and align tokens to experts with block size {d}", .{block_size_m}); + log.debug("Using triton kernels to sort and align tokens to experts with block size {d}", .{block_size_m}); const topk_ids_ = topk_ids.withTags(.{ .token, .topk }).convert(.i32); const num_tokens = topk_ids_.dim(.token); const topk = topk_ids_.dim(.topk); From 001dbbbde8d044247b60fcf72701a7f04b95f54f Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 2 Jul 2026 12:26:27 +0200 Subject: [PATCH 553/881] zml/tokenizer: Have matching tokenizer output (#623) The IREE tokenizer was not producing the same output as the HF tokenizer, potentially degrading the performance of the models. I put a clanker to the task (because the tokenizer is already full clanked). Outputs are now matching without any performance drop. I also fixed an edge case issue on the `Writer` of the tokenizer. --- third_party/iree/match-hf-tokenizer.patch | 1643 +++++++++++++++++++++ third_party/iree/repo.bzl | 5 +- zml/tokenizer/iree/tokenizer.zig | 6 +- 3 files changed, 1652 insertions(+), 2 deletions(-) create mode 100644 third_party/iree/match-hf-tokenizer.patch diff --git a/third_party/iree/match-hf-tokenizer.patch b/third_party/iree/match-hf-tokenizer.patch new file mode 100644 index 00000000..4cbab21a --- /dev/null +++ b/third_party/iree/match-hf-tokenizer.patch @@ -0,0 +1,1643 @@ +diff --git a/runtime/src/iree/tokenizer/model/bpe.c b/runtime/src/iree/tokenizer/model/bpe.c +index 309f1e261f..02fb055eee 100644 +--- a/runtime/src/iree/tokenizer/model/bpe.c ++++ b/runtime/src/iree/tokenizer/model/bpe.c +@@ -179,16 +179,17 @@ iree_status_t iree_tokenizer_bpe_model_allocate( + // Backtracking path capacities. + // Segments up to this size use the O(n) backtracking algorithm. + // Larger segments fall back to the O(n log L) window+heap path. +- // Capped at 4095 so the bitfield (ceil_div(4096, 64) = 64 words) fits in +- // a single uint64_t dirty mask for O(1) amortized per-segment init. ++ // Capped at 32768 to keep exact backtracking state bounded while covering ++ // large normalized prompts that still need HuggingFace-compatible ++ // suffix/prefix merge validation. + iree_host_size_t min_backtrack = 0; + if (!iree_host_size_checked_mul(16, model->max_token_length, + &min_backtrack)) { + // Overflow implies very large token length; clamp to maximum. +- min_backtrack = 4095; ++ min_backtrack = 32768; + } +- if (min_backtrack < 2048) min_backtrack = 2048; +- if (min_backtrack > 4095) min_backtrack = 4095; ++ if (min_backtrack < 32768) min_backtrack = 32768; ++ if (min_backtrack > 32768) min_backtrack = 32768; + model->max_backtrack_segment_bytes = min_backtrack; + // Stack holds at most one token per byte (worst case: all single-byte + // tokens). +@@ -311,6 +312,7 @@ static void iree_tokenizer_bpe_model_destroy( + iree_tokenizer_vocab_trie_free(model->trie); + iree_tokenizer_vocab_merge_hash_free(model->merge_hash); + iree_allocator_free(allocator, model->backtrack_tables.slab); ++ iree_allocator_free(allocator, model->backtrack_tables.alternate_suffix_slab); + iree_allocator_free(allocator, model); + + IREE_TRACE_ZONE_END(z0); +diff --git a/runtime/src/iree/tokenizer/model/bpe_backtrack.c b/runtime/src/iree/tokenizer/model/bpe_backtrack.c +index 7f2cab797c..02b272838f 100644 +--- a/runtime/src/iree/tokenizer/model/bpe_backtrack.c ++++ b/runtime/src/iree/tokenizer/model/bpe_backtrack.c +@@ -81,10 +81,9 @@ typedef struct iree_tokenizer_bpe_validation_frame_t { + static bool iree_tokenizer_bpe_is_valid_token_pair( + const iree_tokenizer_bpe_model_t* model, uint32_t token1, uint32_t token2, + uint32_t deferred_merge_rank) { +- const iree_tokenizer_vocab_t* vocab = model->vocab; +- const uint32_t* effective_rank = model->backtrack_tables.effective_rank; + const iree_tokenizer_bpe_split_entry_t* split_table = + model->backtrack_tables.split_table; ++ const iree_tokenizer_vocab_t* vocab = model->vocab; + + iree_tokenizer_bpe_validation_frame_t + stack[IREE_TOKENIZER_BPE_PAIR_VALIDATION_STACK_CAPACITY]; +@@ -111,7 +110,7 @@ static bool iree_tokenizer_bpe_is_valid_token_pair( + (int32_t)frame->token1, + (int32_t)frame->token2); + if (merge.result_id >= 0) { +- uint32_t combined_rank = effective_rank[(uint32_t)merge.result_id]; ++ uint32_t combined_rank = merge.rank + 1; + // A merge is invalid if it should have fired (rank < limit) UNLESS + // it was intentionally deferred for a better suffix merge. When + // suffix_blocked triggers, it reports the rank of the blocked token; +@@ -140,7 +139,8 @@ static bool iree_tokenizer_bpe_is_valid_token_pair( + // Need to split at least one token. Determine order based on rank. + // Higher-ranked tokens (formed later in BPE) are split first. + bool try_token1_first = +- effective_rank[frame->token1] > effective_rank[frame->token2]; ++ iree_tokenizer_bpe_token_formation_rank(model, frame->token1) > ++ iree_tokenizer_bpe_token_formation_rank(model, frame->token2); + + if (try_token1_first) { + frame->side = +@@ -172,16 +172,26 @@ static bool iree_tokenizer_bpe_is_valid_token_pair( + if (split.left_id != decompose_token) { + // Non-base token: split_table provides (left_id, right_id) directly. + // By construction, merge(left_id, right_id) == decompose_token. ++ uint32_t decomposition_rank = ++ iree_tokenizer_bpe_token_formation_rank(model, decompose_token); ++ iree_tokenizer_merge_hash_result_t split_merge = ++ iree_tokenizer_vocab_merge_hash_lookup( ++ model->merge_hash, (int32_t)split.left_id, ++ (int32_t)split.right_id); ++ if (split_merge.result_id == (int32_t)decompose_token) { ++ decomposition_rank = split_merge.rank + 1; ++ } ++ + uint32_t new_limit; + uint32_t new_token1, new_token2; + if (frame->side == 1) { + new_token1 = split.right_id; + new_token2 = frame->token2; +- new_limit = effective_rank[frame->token1]; ++ new_limit = decomposition_rank; + } else { + new_token1 = frame->token1; + new_token2 = split.left_id; +- new_limit = effective_rank[frame->token2] + 1; ++ new_limit = decomposition_rank + 1; + } + + if (stack_count < IREE_TOKENIZER_BPE_PAIR_VALIDATION_STACK_CAPACITY) { +@@ -243,16 +253,17 @@ static bool iree_tokenizer_bpe_is_valid_token_pair( + if (merge.result_id != (int32_t)decompose_token) continue; + + // Valid decomposition found. Compute new limit and push child frame. ++ uint32_t decomposition_rank = merge.rank + 1; + uint32_t new_limit; + uint32_t new_token1, new_token2; + if (frame->side == 1) { + new_token1 = (uint32_t)right_id; + new_token2 = frame->token2; +- new_limit = effective_rank[frame->token1]; ++ new_limit = decomposition_rank; + } else { + new_token1 = frame->token1; + new_token2 = (uint32_t)left_id; +- new_limit = effective_rank[frame->token2] + 1; ++ new_limit = decomposition_rank + 1; + } + + // Save resume position for backtracking. +@@ -381,160 +392,202 @@ static int32_t iree_tokenizer_bpe_find_first_token_at( + static int32_t iree_tokenizer_bpe_token_at_position( + const iree_tokenizer_bpe_model_t* model, const uint8_t* data, + iree_host_size_t size, iree_host_size_t* out_length) { +- int32_t token = iree_tokenizer_bpe_single_char_token(model, data[0]); +- if (token >= 0) { +- if (out_length) *out_length = 1; +- return token; ++ return iree_tokenizer_bpe_base_token_at_position(model, data, size, ++ out_length); ++} ++ ++static bool iree_tokenizer_bpe_is_suffix_blocked( ++ const iree_tokenizer_bpe_model_t* model, int32_t token, ++ const uint8_t* remaining_data, iree_host_size_t remaining_size, ++ const char* suffix_data, iree_host_size_t suffix_length, ++ uint32_t* out_blocking_rank); ++static bool iree_tokenizer_bpe_is_suffix_blocked_impl( ++ const iree_tokenizer_bpe_model_t* model, int32_t token, ++ const uint8_t* remaining_data, iree_host_size_t remaining_size, ++ const char* suffix_data, iree_host_size_t suffix_length, ++ uint32_t* out_blocking_rank, int depth); ++ ++// Recursive implementation for iree_tokenizer_bpe_is_token_consumed_rightward. ++// ++// |ready_rank_plus_one| tracks when the right compound is available. Base ++// tokens are available at rank 0; a compound built by an internal merge at ++// rank N is available starting at N+1, so an outer merge at rank R can use it ++// iff ready_rank_plus_one <= R. ++static bool iree_tokenizer_bpe_is_token_consumed_rightward_impl( ++ const iree_tokenizer_bpe_model_t* model, int32_t token, ++ const uint8_t* remaining_data, iree_host_size_t remaining_size, ++ iree_host_size_t token_end, uint32_t max_rank, int depth) { ++ if (token_end >= remaining_size) return false; ++ if (depth >= 8) return false; ++ ++ const iree_tokenizer_bpe_split_entry_t* split_table = ++ model->backtrack_tables.split_table; ++ const bool byte_level = ++ iree_all_bits_set(model->flags, IREE_TOKENIZER_BPE_FLAG_BYTE_LEVEL_INPUT); ++ ++ iree_tokenizer_trie_cursor_t cursor; ++ iree_tokenizer_trie_cursor_reset(&cursor, model->trie); ++ ++ for (iree_host_size_t i = token_end; i < remaining_size; ++i) { ++ if (!iree_tokenizer_bpe_trie_advance_byte(&cursor, remaining_data[i], ++ byte_level)) { ++ break; ++ } ++ ++ int32_t compound_token = iree_tokenizer_trie_cursor_token_id(&cursor); ++ if (compound_token < 0) continue; ++ ++ iree_host_size_t compound_end = i + 1; ++ iree_tokenizer_merge_hash_result_t merge = ++ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, token, ++ compound_token); ++ if (!iree_tokenizer_merge_hash_result_is_valid(merge) || ++ merge.rank >= max_rank) { ++ continue; ++ } ++ ++ uint32_t compound = (uint32_t)compound_token; ++ bool compound_is_base = split_table[compound].left_id == compound; ++ uint32_t compound_ready_rank = ++ iree_tokenizer_bpe_token_formation_rank(model, compound); ++ if (compound_ready_rank > merge.rank) continue; ++ if (!compound_is_base && ++ !iree_tokenizer_bpe_is_first_token_reachable(model, compound)) { ++ continue; ++ } ++ ++ if (!compound_is_base && compound_end < remaining_size) { ++ uint32_t rightmost_base = ++ iree_tokenizer_bpe_rightmost_base_token(model, compound); ++ uint32_t right_consumed = ++ iree_tokenizer_bpe_right_boundary_consumed_rank(model, compound); ++ if (right_consumed > 0 && ++ iree_tokenizer_bpe_is_token_consumed_rightward_impl( ++ model, (int32_t)rightmost_base, remaining_data, remaining_size, ++ compound_end, right_consumed - 1, depth + 1)) { ++ continue; ++ } ++ } ++ ++ uint32_t result_token = (uint32_t)merge.result_id; ++ if (compound_end < remaining_size && depth < 4) { ++ uint32_t blocking_rank = 0; ++ if (iree_tokenizer_bpe_is_suffix_blocked_impl( ++ model, (int32_t)result_token, remaining_data + compound_end, ++ remaining_size - compound_end, NULL, 0, &blocking_rank, ++ depth + 1)) { ++ continue; ++ } ++ } ++ ++ // merge(token, compound) would consume token if the compound is not itself ++ // consumed to the right before that merge rank. ++ if (!iree_tokenizer_bpe_is_token_consumed_rightward_impl( ++ model, compound_token, remaining_data, remaining_size, compound_end, ++ merge.rank, depth + 1)) { ++ return true; ++ } + } +- return iree_tokenizer_bpe_find_first_token_at(model, data, size, 8, +- out_length); ++ return false; + } + + // Checks whether a token at a given position is consumed by a rightward merge + // at rank lower than |max_rank|. This means merge(token, following) exists and + // fires before any merge at |max_rank| would. + // +-// To verify the rightward merge actually fires, also checks that the following +-// token isn't itself consumed by an even lower-rank merge. This handles chains +-// like: merge(c,h) would consume c, but merge(h,X) at lower rank consumes h +-// first — so merge(c,h) can't fire and c survives. ++// The following token may itself be a compound formed by lower-rank rightward ++// merges (for example t+ry after r+y has formed). This bounded search mirrors ++// the short compound lookahead used by suffix-blocking preemption. + static bool iree_tokenizer_bpe_is_token_consumed_rightward( + const iree_tokenizer_bpe_model_t* model, int32_t token, + const uint8_t* remaining_data, iree_host_size_t remaining_size, + iree_host_size_t token_end, uint32_t max_rank) { +- if (token_end >= remaining_size) return false; +- iree_host_size_t following_length = 1; +- int32_t following_token = iree_tokenizer_bpe_token_at_position( +- model, remaining_data + token_end, remaining_size - token_end, +- &following_length); +- if (following_token < 0) return false; +- +- iree_tokenizer_merge_hash_result_t merge = +- iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, token, +- following_token); +- if (!iree_tokenizer_merge_hash_result_is_valid(merge) || +- merge.rank >= max_rank) { +- return false; +- } +- +- // merge(token, following) would fire. Verify following is available: if +- // merge(following, next_after) fires at even lower rank, following is +- // consumed and merge(token, following) can't fire. +- iree_host_size_t next_after_end = token_end + following_length; +- if (next_after_end < remaining_size) { +- int32_t next_after_token = iree_tokenizer_bpe_token_at_position( +- model, remaining_data + next_after_end, remaining_size - next_after_end, +- NULL); +- if (next_after_token >= 0) { +- iree_tokenizer_merge_hash_result_t following_merge = +- iree_tokenizer_vocab_merge_hash_lookup( +- model->merge_hash, following_token, next_after_token); +- if (iree_tokenizer_merge_hash_result_is_valid(following_merge) && +- following_merge.rank < merge.rank) { +- return false; // Following consumed first, token survives. +- } +- } +- } +- return true; ++ return iree_tokenizer_bpe_is_token_consumed_rightward_impl( ++ model, token, remaining_data, remaining_size, token_end, max_rank, 0); + } + + // Checks if a suffix merge is preempted by lower-rank merges in the following + // input. Returns true if the suffix merge would NOT actually fire because the + // prefix token would be consumed first. + // +-// Two preemption paths: +-// Direct: merge(prefix, next) at rank < suffix_merge_rank, where the next +-// token is available (not consumed by a lower-rank rightward merge). +-// Compound: merge(prefix, compound) where compound is built by progressively +-// merging characters: next+third->c1, c1+fourth->c2, etc. This handles +-// cases where the direct merge can't fire but the prefix is still consumed +-// via a deeper compound (e.g., merge(l, ish) when i+s fires first). +-// +-// For non-byte-level tokenizers (e.g., SentencePiece), uses trie lookup to find +-// tokens when single-char lookup fails. This handles multi-byte base tokens +-// like metaspace (E2 96 81). ++// This delegates to the same bounded rightward-consumption search used to ++// validate prefix availability. That search handles direct and compound merges ++// while checking that each compound component is actually available at the rank ++// where it would be used. + static bool iree_tokenizer_bpe_is_suffix_merge_preempted( + const iree_tokenizer_bpe_model_t* model, int32_t prefix_token, + uint32_t suffix_merge_rank, const uint8_t* remaining_data, + iree_host_size_t remaining_size, iree_host_size_t prefix_end) { +- if (prefix_end >= remaining_size) return false; +- +- iree_host_size_t next_length = 1; +- int32_t next_token = iree_tokenizer_bpe_token_at_position( +- model, remaining_data + prefix_end, remaining_size - prefix_end, +- &next_length); +- if (next_token < 0) return false; +- +- // Direct preemption: merge(prefix, next) at rank < suffix_merge_rank. +- // Only valid if next is actually available (not consumed rightward). +- iree_tokenizer_merge_hash_result_t prefix_next = +- iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, prefix_token, +- next_token); +- if (iree_tokenizer_merge_hash_result_is_valid(prefix_next) && +- prefix_next.rank < suffix_merge_rank) { +- if (!iree_tokenizer_bpe_is_token_consumed_rightward( +- model, next_token, remaining_data, remaining_size, +- prefix_end + next_length, prefix_next.rank)) { ++ return iree_tokenizer_bpe_is_token_consumed_rightward( ++ model, prefix_token, remaining_data, remaining_size, prefix_end, ++ suffix_merge_rank); ++} ++ ++static bool iree_tokenizer_bpe_suffix_already_collected( ++ const uint32_t* suffixes, const uint32_t* suffix_consumed_at, ++ iree_host_size_t suffix_count, uint32_t suffix, uint32_t consumed_at) { ++ for (iree_host_size_t i = 0; i < suffix_count; ++i) { ++ if (suffixes[i] == suffix && suffix_consumed_at[i] == consumed_at) { + return true; + } + } ++ return false; ++} + +- // Compound preemption: build progressively longer compounds from the +- // characters following the prefix, checking merge(prefix, compound) at each +- // step. This catches cases where the direct merge can't fire but the prefix +- // merges with a multi-character compound that forms at lower rank. +- // +- // Example (Whisper "establish"): +- // prefix=l, next=i, merge(l,i) can't fire (i consumed by i+s at rank 15) +- // compound: i+s->is (rank 15), is+h->ish (rank 486) +- // merge(l, ish) at rank 1677 < suffix_merge_rank -> preempted +- iree_host_size_t compound_end = prefix_end + next_length; +- iree_host_size_t extension_length = 1; +- int32_t extension_token = -1; +- if (compound_end < remaining_size) { +- extension_token = iree_tokenizer_bpe_token_at_position( +- model, remaining_data + compound_end, remaining_size - compound_end, +- &extension_length); +- } +- if (extension_token < 0) return false; ++static void iree_tokenizer_bpe_collect_reachable_suffixes( ++ const iree_tokenizer_bpe_model_t* model, uint32_t token, ++ uint32_t* suffixes, uint32_t* suffix_consumed_at, ++ iree_host_size_t* suffix_count, iree_host_size_t suffix_capacity, ++ int depth) { ++ if (*suffix_count >= suffix_capacity || depth >= 16) return; + +- iree_tokenizer_merge_hash_result_t compound_merge = +- iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, next_token, +- extension_token); +- if (!iree_tokenizer_merge_hash_result_is_valid(compound_merge) || +- compound_merge.rank >= suffix_merge_rank) { +- return false; +- } ++ const iree_tokenizer_vocab_t* vocab = model->vocab; ++ const iree_tokenizer_bpe_split_entry_t* split_table = ++ model->backtrack_tables.split_table; + +- // Build the compound progressively, checking at each level. +- int32_t compound_token = compound_merge.result_id; +- compound_end += extension_length; +- for (int depth = 0; depth < 4; ++depth) { +- iree_tokenizer_merge_hash_result_t prefix_compound = +- iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, prefix_token, +- compound_token); +- if (iree_tokenizer_merge_hash_result_is_valid(prefix_compound) && +- prefix_compound.rank < suffix_merge_rank) { +- return true; ++ if (split_table[token].left_id == token) return; // Base token. ++ ++ iree_string_view_t token_text = ++ iree_tokenizer_vocab_token_text(vocab, (int32_t)token); ++ ++ for (iree_host_size_t split_pos = 1; ++ split_pos < token_text.size && *suffix_count < suffix_capacity; ++ ++split_pos) { ++ iree_string_view_t left_text = ++ iree_make_string_view(token_text.data, split_pos); ++ iree_string_view_t right_text = iree_make_string_view( ++ token_text.data + split_pos, token_text.size - split_pos); ++ ++ int32_t left_id = iree_tokenizer_vocab_lookup(vocab, left_text); ++ int32_t right_id = iree_tokenizer_vocab_lookup(vocab, right_text); ++ if (left_id < 0 || right_id < 0) continue; ++ ++ iree_tokenizer_merge_hash_result_t merge = ++ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, left_id, ++ right_id); ++ if (merge.result_id != (int32_t)token) continue; ++ ++ if (!iree_tokenizer_bpe_is_decomposition_reachable( ++ model, (uint32_t)left_id, (uint32_t)right_id, token)) { ++ continue; + } + +- // Extend compound by one more character. +- if (compound_end >= remaining_size) break; +- extension_token = iree_tokenizer_bpe_token_at_position( +- model, remaining_data + compound_end, remaining_size - compound_end, +- &extension_length); +- if (extension_token < 0) break; +- +- compound_merge = iree_tokenizer_vocab_merge_hash_lookup( +- model->merge_hash, compound_token, extension_token); +- if (!iree_tokenizer_merge_hash_result_is_valid(compound_merge) || +- compound_merge.rank >= suffix_merge_rank) { +- break; ++ uint32_t consumed_at = merge.rank + 1; ++ if (!iree_tokenizer_bpe_suffix_already_collected( ++ suffixes, suffix_consumed_at, *suffix_count, (uint32_t)right_id, ++ consumed_at)) { ++ suffixes[*suffix_count] = (uint32_t)right_id; ++ suffix_consumed_at[*suffix_count] = consumed_at; ++ ++*suffix_count; + } +- compound_token = compound_merge.result_id; +- compound_end += extension_length; ++ ++ // Walk deeper suffixes of the right child. This generalizes the old ++ // rightmost split_table walk to alternate reachable decompositions, e.g. ++ // recon can split as rec+on or re+con and both suffixes can matter. ++ iree_tokenizer_bpe_collect_reachable_suffixes( ++ model, (uint32_t)right_id, suffixes, suffix_consumed_at, suffix_count, ++ suffix_capacity, depth + 1); + } +- return false; + } + + // Checks if a candidate token is "suffix-blocked" - meaning a proper suffix of +@@ -566,21 +619,25 @@ static bool iree_tokenizer_bpe_is_suffix_merge_preempted( + // This allows pair validation to accept pairs whose merge rank >= + // blocking_rank, since those merges were intentionally deferred for better + // suffix merges. +-static bool iree_tokenizer_bpe_is_suffix_blocked( ++static bool iree_tokenizer_bpe_is_suffix_blocked_impl( + const iree_tokenizer_bpe_model_t* model, int32_t token, + const uint8_t* remaining_data, iree_host_size_t remaining_size, + const char* suffix_data, iree_host_size_t suffix_length, +- uint32_t* out_blocking_rank) { ++ uint32_t* out_blocking_rank, int depth) { + *out_blocking_rank = 0; ++ if (depth >= 8) return false; + if (remaining_size == 0 && suffix_length == 0) return false; + + const iree_tokenizer_bpe_split_entry_t* split_table = + model->backtrack_tables.split_table; +- const uint32_t* effective_rank = model->backtrack_tables.effective_rank; ++ const uint64_t* token_has_reachable_alternates = ++ model->backtrack_tables.token_has_reachable_alternates; ++ const iree_tokenizer_vocab_t* vocab = model->vocab; + const bool byte_level = + iree_all_bits_set(model->flags, IREE_TOKENIZER_BPE_FLAG_BYTE_LEVEL_INPUT); + +- uint32_t token_rank = effective_rank[(uint32_t)token]; ++ uint32_t token_rank = ++ iree_tokenizer_bpe_token_formation_rank(model, (uint32_t)token); + if (token_rank <= 1) return false; // Base token, no suffix to check. + + // Collect all rightmost suffixes with their consumption ranks. +@@ -601,29 +658,64 @@ static bool iree_tokenizer_bpe_is_suffix_blocked( + uint32_t suffix_consumed_at[32]; + iree_host_size_t suffix_count = 0; + ++ bool needs_alternate_suffixes = false; + for (uint32_t current = (uint32_t)token; +- suffix_count < 32 && split_table[current].left_id != current; ++ suffix_count < IREE_ARRAYSIZE(suffixes) && ++ split_table[current].left_id != current; + current = split_table[current].right_id) { +- // Check if this decomposition is actually reachable via BPE. If the +- // split_table decomposition is blocked by a boundary merge, skip this +- // suffix - it won't exist in the actual BPE token sequence. ++ if (iree_any_bit_set(token_has_reachable_alternates[current / 64], ++ 1ull << (current % 64))) { ++ needs_alternate_suffixes = true; ++ break; ++ } + uint32_t left = split_table[current].left_id; + uint32_t right = split_table[current].right_id; + bool reachable = iree_tokenizer_bpe_is_decomposition_reachable( + model, left, right, current); + if (!reachable) { +- // This decomposition is blocked. The actual BPE path uses a different +- // decomposition, so we can't rely on this suffix chain. Stop collecting +- // suffixes here - the actual suffixes depend on the alternate path which +- // we don't have easy access to. ++ needs_alternate_suffixes = true; + break; + } + suffixes[suffix_count] = right; +- suffix_consumed_at[suffix_count] = effective_rank[current]; ++ suffix_consumed_at[suffix_count] = ++ iree_tokenizer_bpe_token_formation_rank(model, current); + suffix_count++; ++ iree_tokenizer_bpe_collect_reachable_suffixes( ++ model, right, suffixes, suffix_consumed_at, &suffix_count, ++ IREE_ARRAYSIZE(suffixes), 0); ++ } ++ if (needs_alternate_suffixes) { ++ suffix_count = 0; ++ if (model->backtrack_tables.alternate_suffix_offsets) { ++ uint32_t token_id = (uint32_t)token; ++ uint32_t start = ++ model->backtrack_tables.alternate_suffix_offsets[token_id]; ++ uint32_t end = ++ model->backtrack_tables.alternate_suffix_offsets[token_id + 1]; ++ for (uint32_t i = start; ++ i < end && suffix_count < IREE_ARRAYSIZE(suffixes); ++i) { ++ suffixes[suffix_count] = ++ model->backtrack_tables.alternate_suffixes[i]; ++ suffix_consumed_at[suffix_count] = ++ model->backtrack_tables.alternate_suffix_consumed_at[i]; ++ ++suffix_count; ++ } ++ iree_host_size_t loaded_suffix_count = suffix_count; ++ for (iree_host_size_t i = 0; i < loaded_suffix_count; ++i) { ++ iree_tokenizer_bpe_collect_reachable_suffixes( ++ model, suffixes[i], suffixes, suffix_consumed_at, &suffix_count, ++ IREE_ARRAYSIZE(suffixes), 0); ++ } ++ } else { ++ iree_tokenizer_bpe_collect_reachable_suffixes( ++ model, (uint32_t)token, suffixes, suffix_consumed_at, &suffix_count, ++ IREE_ARRAYSIZE(suffixes), 0); ++ } ++ } else { ++ IREE_ASSERT(suffix_count < IREE_ARRAYSIZE(suffixes), ++ "suffix chain depth exceeds capacity"); + } +- IREE_ASSERT(suffix_count < IREE_ARRAYSIZE(suffixes), +- "suffix chain depth exceeds capacity"); ++ + if (suffix_count == 0) return false; + + // Walk the trie to enumerate all prefix tokens of the remaining input. +@@ -652,18 +744,33 @@ static bool iree_tokenizer_bpe_is_suffix_blocked( + + // Skip if merge rank is too high to matter. + uint32_t merge_effective_rank = merge.rank + 1; ++ uint32_t suffix_available_at = ++ iree_tokenizer_bpe_token_formation_rank(model, suffixes[s]); ++ uint32_t prefix_available_at = ++ iree_tokenizer_bpe_token_formation_rank(model, ++ (uint32_t)prefix_token); ++ if (merge_effective_rank < suffix_available_at) continue; ++ if (merge_effective_rank < prefix_available_at) continue; + if (merge_effective_rank >= token_rank) continue; + + // Skip if the suffix is consumed before this merge could fire. + if (merge_effective_rank >= suffix_consumed_at[s]) continue; + +- // Skip when suffix == prefix (repeating pattern case). +- // When the suffix token equals the prefix token, we have a repeating +- // pattern like consecutive metaspaces. In this case, any shorter token +- // would also face suffix blocking by the same pattern, creating an +- // infinite loop. The suffix blocking optimization doesn't help here +- // because there's no "better" tokenization to defer to. +- if ((int32_t)suffixes[s] == prefix_token) continue; ++ // If the prefix token would itself be split to enable a better suffix ++ // merge with following bytes, it is not available for this boundary ++ // merge. This handles chains such as ge|uffer|t where b+uffer would ++ // form "buffer" before ge+b, except "uffer" is blocked by er+t. ++ if (i + 1 < remaining_size && depth < 7 && ++ iree_tokenizer_bpe_token_formation_rank( ++ model, (uint32_t)prefix_token) > 1) { ++ uint32_t prefix_blocking_rank = 0; ++ if (iree_tokenizer_bpe_is_suffix_blocked_impl( ++ model, prefix_token, remaining_data + i + 1, ++ remaining_size - (i + 1), NULL, 0, &prefix_blocking_rank, ++ depth + 1)) { ++ continue; ++ } ++ } + + // Check if the prefix would be consumed by lower-rank merges first. + if (iree_tokenizer_bpe_is_suffix_merge_preempted( +@@ -678,7 +785,8 @@ static bool iree_tokenizer_bpe_is_suffix_blocked( + // the rightmost base of the prefix merges with the following + // character at a rank lower than the prefix's internal consumption + // of that base, the prefix cannot form here. +- if (effective_rank[(uint32_t)prefix_token] > 1 && ++ if (iree_tokenizer_bpe_token_formation_rank( ++ model, (uint32_t)prefix_token) > 1 && + i + 1 < remaining_size) { + uint32_t rightmost_base = iree_tokenizer_bpe_rightmost_base_token( + model, (uint32_t)prefix_token); +@@ -725,14 +833,18 @@ static bool iree_tokenizer_bpe_is_suffix_blocked( + + // Skip if merge rank is too high to matter. + uint32_t merge_effective_rank = merge.rank + 1; ++ uint32_t suffix_available_at = ++ iree_tokenizer_bpe_token_formation_rank(model, suffixes[s]); ++ uint32_t prefix_available_at = ++ iree_tokenizer_bpe_token_formation_rank(model, ++ (uint32_t)prefix_token); ++ if (merge_effective_rank < suffix_available_at) continue; ++ if (merge_effective_rank < prefix_available_at) continue; + if (merge_effective_rank >= token_rank) continue; + + // Skip if the token suffix is consumed before this merge could fire. + if (merge_effective_rank >= suffix_consumed_at[s]) continue; + +- // Skip when suffix == prefix (repeating pattern case). +- if ((int32_t)suffixes[s] == prefix_token) continue; +- + // Suffixed tokens consume all remaining input + suffix bytes, so no + // preemption check is needed. The prefix_token represents the complete + // remaining segment (remaining_data + partial suffix) and nothing +@@ -745,6 +857,16 @@ static bool iree_tokenizer_bpe_is_suffix_blocked( + return false; + } + ++static bool iree_tokenizer_bpe_is_suffix_blocked( ++ const iree_tokenizer_bpe_model_t* model, int32_t token, ++ const uint8_t* remaining_data, iree_host_size_t remaining_size, ++ const char* suffix_data, iree_host_size_t suffix_length, ++ uint32_t* out_blocking_rank) { ++ return iree_tokenizer_bpe_is_suffix_blocked_impl( ++ model, token, remaining_data, remaining_size, suffix_data, suffix_length, ++ out_blocking_rank, 0); ++} ++ + //===----------------------------------------------------------------------===// + // BPE Backtracking: Core Algorithm + //===----------------------------------------------------------------------===// +@@ -774,18 +896,11 @@ void iree_tokenizer_bpe_backtrack_encode( + iree_tokenizer_bpe_state_pair_cache(state, model); + const uint32_t pair_cache_mask = (uint32_t)(model->pair_cache_capacity - 1); + +- // Reset only the bitfield words dirtied by the previous segment's +- // backtracking. On the first call, dirty_mask is pre-set to all-ones by +- // state_initialize, so this performs the equivalent of a full init. +- // On subsequent calls with no backtracking, dirty_mask is 0 and this +- // loop body executes zero times. +- uint64_t dirty = state->backtrack.dirty_mask; +- while (dirty) { +- iree_host_size_t word_index = +- (iree_host_size_t)iree_math_count_trailing_zeros_u64(dirty); +- bitfield[word_index] = UINT64_MAX; +- dirty &= dirty - 1; // Clear lowest set bit. +- } ++ // Reset the reachability bitfield for this segment. The exact backtracking ++ // path supports more than 64 bitfield words, so a single dirty mask is not ++ // expressive enough. ++ memset(bitfield, 0xFF, ++ model->backtrack_bitfield_capacity * sizeof(uint64_t)); + state->backtrack.dirty_mask = 0; + + iree_host_size_t stack_count = 0; +@@ -984,7 +1099,8 @@ void iree_tokenizer_bpe_backtrack_encode( + // fires first, blocking the path to "hello" as a single token. + bool token_unreachable = + token_raw_length > 1 && +- model->backtrack_tables.effective_rank[(uint32_t)token] > 1 && ++ iree_tokenizer_bpe_token_formation_rank(model, (uint32_t)token) > ++ 1 && + !iree_any_bit_set( + model->backtrack_tables.token_reachable[(uint32_t)token / 64], + 1ull << ((uint32_t)token % 64)); +diff --git a/runtime/src/iree/tokenizer/model/bpe_encode.c b/runtime/src/iree/tokenizer/model/bpe_encode.c +index bf140ae97c..4fa090026f 100644 +--- a/runtime/src/iree/tokenizer/model/bpe_encode.c ++++ b/runtime/src/iree/tokenizer/model/bpe_encode.c +@@ -280,39 +280,29 @@ static iree_status_t iree_tokenizer_bpe_encode_segment( + byte_position < segment.size; ++byte_position) { + uint8_t input_byte = (uint8_t)segment.data[byte_position]; + +- // Look up single-byte token. For ByteLevel mode, this is the UTF-8 +- // encoding of the ByteLevel codepoint (e.g., space -> "Ġ"). +- int32_t token_id = model->byte_to_token[input_byte]; ++ // Seed with the shortest base vocabulary token. Some base symbols (for ++ // example SentencePiece's U+2581 metaspace) span multiple UTF-8 bytes, ++ // and byte-fallback vocabularies may also have tokens for each raw byte. ++ // The BPE heap must start from the base symbol, not the byte fallback. + iree_host_size_t token_byte_length = 1; ++ int32_t token_id = iree_tokenizer_bpe_base_token_at_position( ++ model, (const uint8_t*)segment.data + byte_position, ++ segment.size - byte_position, &token_byte_length); + if (token_id < 0) { +- // No direct single-byte token. Try a multi-byte trie match first: +- // some base vocabulary tokens (e.g., SentencePiece's ▁ = U+2581, +- // 3 bytes) span multiple raw bytes but are never produced by a merge. +- // The trie walk finds the longest such token; effective_rank filtering +- // inside backtrack_longest_match ensures we only accept tokens that +- // participate in BPE (not stray added tokens). +- iree_host_size_t trie_raw_length = 0; +- iree_tokenizer_bpe_backtrack_longest_match( +- model, (const uint8_t*)segment.data + byte_position, +- segment.size - byte_position, &token_id, &trie_raw_length); +- if (token_id >= 0) { +- token_byte_length = trie_raw_length; +- } else { +- // No trie match. Fall back to byte-fallback (<0xNN>) or UNK. +- // For FUSE_UNK, check the last token in the window (not yet emitted) +- // before falling back to last_emitted_token_id. +- int32_t previous_token = +- (state->window.count > 0) +- ? iree_tokenizer_bpe_window_at(state, model, +- state->window.count - 1) +- ->token_id +- : state->last_emitted_token_id; +- token_id = iree_tokenizer_bpe_handle_unknown_byte(model, input_byte, +- previous_token); +- if (token_id < 0) { +- // Byte was fused with previous UNK. Skip to next byte. +- continue; +- } ++ // No trie match. Fall back to byte-fallback (<0xNN>) or UNK. For ++ // FUSE_UNK, check the last token in the window (not yet emitted) ++ // before falling back to last_emitted_token_id. ++ int32_t previous_token = ++ (state->window.count > 0) ++ ? iree_tokenizer_bpe_window_at(state, model, ++ state->window.count - 1) ++ ->token_id ++ : state->last_emitted_token_id; ++ token_id = iree_tokenizer_bpe_handle_unknown_byte(model, input_byte, ++ previous_token); ++ if (token_id < 0) { ++ // Byte was fused with previous UNK. Skip to next byte. ++ continue; + } + } + +@@ -620,13 +610,14 @@ iree_host_size_t iree_tokenizer_bpe_state_reclaim( + window[index].end_byte -= (uint32_t)committed; + } + +- // Adjust heap entry byte positions. +- // After BYTE_LOOP processes all segment bytes, apply_pending_merges has +- // drained most entries. Any remaining entries reference current window tokens +- // (left_start_byte >= committed). Stale entries that would underflow are +- // discarded by lazy invalidation when popped. +- for (iree_host_size_t i = 0; i < bpe_state->heap.size; ++i) { +- bpe_state->heap.entries[i].left_start_byte -= (uint32_t)committed; ++ // Rebuild heap entries for the rebased window. The heap can contain stale ++ // entries for tokens that were already emitted; subtracting |committed| from ++ // those would underflow their start byte and leave noisy stale entries. The ++ // window is bounded, so rebuilding is cheap and keeps all held-back merges ++ // visible after reclaim. ++ iree_tokenizer_bpe_heap_reset(&bpe_state->heap); ++ for (iree_host_size_t i = 0; i + 1 < bpe_state->window.count; ++i) { ++ iree_tokenizer_bpe_maybe_add_merge(bpe_state, model, i); + } + + return committed; +diff --git a/runtime/src/iree/tokenizer/model/bpe_internal.h b/runtime/src/iree/tokenizer/model/bpe_internal.h +index 69546df3f0..4e9873e653 100644 +--- a/runtime/src/iree/tokenizer/model/bpe_internal.h ++++ b/runtime/src/iree/tokenizer/model/bpe_internal.h +@@ -45,50 +45,42 @@ typedef struct iree_tokenizer_bpe_split_entry_t { + uint32_t right_id; + } iree_tokenizer_bpe_split_entry_t; + +-// Backtracking stack entry: token ID and the byte position where it starts. +-// Bit-packed backtrack entry to maintain 8-byte size while supporting large +-// vocabs. start_byte is bounded by max_backtrack_segment_bytes (<=4095), so +-// we use 12 bits for it and 20 bits for deferred_merge_rank (supports 1M +-// tokens, covering all realistic vocabularies). +-#define IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_BITS 12 +-#define IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_MASK \ +- ((1u << IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_BITS) - 1) +-#define IREE_TOKENIZER_BPE_BACKTRACK_DEFERRED_RANK_BITS 20 ++// Backtracking stack entry: token ID, byte start, and deferred merge rank. ++// Keep deferred_merge_rank as a full 19-bit value (Gemma-sized vocabularies ++// need it) and store start_byte separately so exact backtracking can cover ++// larger normalized segments without falling back to the approximate heap path. ++#define IREE_TOKENIZER_BPE_BACKTRACK_DEFERRED_RANK_BITS 19 + #define IREE_TOKENIZER_BPE_BACKTRACK_DEFERRED_RANK_MAX \ + ((1u << IREE_TOKENIZER_BPE_BACKTRACK_DEFERRED_RANK_BITS) - 1) + + typedef struct iree_tokenizer_bpe_backtrack_entry_t { + int32_t token_id; +- // Packed field: lower 12 bits = start_byte, upper 20 bits = deferred_rank. ++ uint32_t start_byte; + // The deferred_merge_rank is the effective_rank that was active when this + // token was pushed. It tells pair validation at the next position which + // merges were intentionally deferred due to suffix blocking. +- uint32_t start_byte_and_deferred_rank; ++ uint32_t deferred_rank; + } iree_tokenizer_bpe_backtrack_entry_t; + + static inline uint32_t iree_tokenizer_bpe_backtrack_entry_start_byte( + const iree_tokenizer_bpe_backtrack_entry_t* entry) { +- return entry->start_byte_and_deferred_rank & +- IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_MASK; ++ return entry->start_byte; + } + + static inline uint32_t iree_tokenizer_bpe_backtrack_entry_deferred_rank( + const iree_tokenizer_bpe_backtrack_entry_t* entry) { +- return entry->start_byte_and_deferred_rank >> +- IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_BITS; ++ return entry->deferred_rank; + } + + static inline void iree_tokenizer_bpe_backtrack_entry_set( + iree_tokenizer_bpe_backtrack_entry_t* entry, int32_t token_id, + uint32_t start_byte, uint32_t deferred_rank) { + entry->token_id = token_id; +- // Cap deferred_rank at 20-bit max (supports vocabs up to 1M tokens). + if (deferred_rank > IREE_TOKENIZER_BPE_BACKTRACK_DEFERRED_RANK_MAX) { + deferred_rank = IREE_TOKENIZER_BPE_BACKTRACK_DEFERRED_RANK_MAX; + } +- entry->start_byte_and_deferred_rank = +- (start_byte & IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_MASK) | +- (deferred_rank << IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_BITS); ++ entry->start_byte = start_byte; ++ entry->deferred_rank = deferred_rank; + } + + //===----------------------------------------------------------------------===// +@@ -101,11 +93,19 @@ static inline void iree_tokenizer_bpe_backtrack_entry_set( + typedef struct iree_tokenizer_bpe_backtrack_tables_t { + // Single allocation for all tables below. + void* slab; ++ void* alternate_suffix_slab; + iree_tokenizer_bpe_split_entry_t* split_table; // Inverse merge table. + // Longest proper prefix token, or UINT32_MAX. + uint32_t* next_prefix_match; + uint32_t* effective_rank; // 0 = non-participant, >0 = BPE-reachable. + uint64_t* token_reachable; // Precomputed reachability bits. ++ // Tokens with more than one reachable decomposition. Suffix blocking must ++ // inspect alternate decompositions for these tokens instead of only walking ++ // split_table's right spine. ++ uint64_t* token_has_reachable_alternates; ++ uint32_t* alternate_suffix_offsets; // vocab_capacity + 1 entries. ++ uint32_t* alternate_suffixes; ++ uint32_t* alternate_suffix_consumed_at; + } iree_tokenizer_bpe_backtrack_tables_t; + + //===----------------------------------------------------------------------===// +@@ -118,10 +118,9 @@ typedef struct iree_tokenizer_bpe_backtrack_state_t { + iree_host_size_t stack_count; // Tokens in the stack. + // Next token to emit from stack. + iree_host_size_t emit_index; +- // Dirty mask for lazy bitfield reset. Each bit corresponds to a uint64_t +- // word in the backtrack bitfield that was modified (cleared) during the +- // previous segment's backtracking. On segment start, only these words are +- // reset to UINT64_MAX, giving O(backtracks) init instead of O(capacity). ++ // Retained for state layout compatibility with older code paths. The ++ // backtracking bitfield is reset as a whole at segment start because the ++ // exact path now supports more than 64 bitfield words. + uint64_t dirty_mask; + } iree_tokenizer_bpe_backtrack_state_t; + +@@ -210,6 +209,22 @@ typedef struct iree_tokenizer_bpe_model_t { + iree_host_size_t pair_cache_offset; + } iree_tokenizer_bpe_model_t; + ++static inline uint32_t iree_tokenizer_bpe_token_formation_rank( ++ const iree_tokenizer_bpe_model_t* model, uint32_t token) { ++ iree_tokenizer_bpe_split_entry_t split = ++ model->backtrack_tables.split_table[token]; ++ if (split.left_id == token) return 0; // Base token. ++ ++ iree_tokenizer_merge_hash_result_t merge = ++ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, ++ (int32_t)split.left_id, ++ (int32_t)split.right_id); ++ if (merge.result_id == (int32_t)token) { ++ return merge.rank + 1; ++ } ++ return model->backtrack_tables.effective_rank[token]; ++} ++ + //===----------------------------------------------------------------------===// + // Segment Context (State-Side) + //===----------------------------------------------------------------------===// +@@ -589,9 +604,9 @@ static inline bool iree_tokenizer_bpe_bitfield_is_set( + + static inline void iree_tokenizer_bpe_bitfield_clear( + uint64_t* bitfield, uint64_t* dirty_mask, iree_host_size_t position) { ++ (void)dirty_mask; + iree_host_size_t word_index = position / 64; + bitfield[word_index] &= ~((uint64_t)1 << (position % 64)); +- *dirty_mask |= (uint64_t)1 << word_index; + } + + //===----------------------------------------------------------------------===// +@@ -679,6 +694,73 @@ iree_tokenizer_bpe_backtrack_longest_match( + } + } + ++// Finds the shortest base vocabulary token matching raw input bytes at data[0]. ++// BPE starts from base symbols and applies merge rules by rank. For ++// SentencePiece-style tokenizers, some base symbols (for example U+2581 ++// metaspace) span multiple UTF-8 bytes and cannot be found through byte_to_token. ++// The window/heap path must seed those base symbols, not the longest reachable ++// merged token, otherwise rank-priority merges can be skipped. ++static inline int32_t iree_tokenizer_bpe_base_token_at_position( ++ const iree_tokenizer_bpe_model_t* model, const uint8_t* data, ++ iree_host_size_t size, iree_host_size_t* out_raw_length) { ++ if (size == 0) return -1; ++ ++ int32_t byte_token = model->byte_to_token[data[0]]; ++ ++ const bool byte_level = ++ iree_all_bits_set(model->flags, IREE_TOKENIZER_BPE_FLAG_BYTE_LEVEL_INPUT); ++ ++ iree_tokenizer_trie_cursor_t cursor; ++ iree_tokenizer_trie_cursor_reset(&cursor, model->trie); ++ ++ int32_t fallback_base_token = -1; ++ iree_host_size_t fallback_base_length = 0; ++ for (iree_host_size_t raw_consumed = 0; raw_consumed < size; ++ ++raw_consumed) { ++ if (!iree_tokenizer_bpe_trie_advance_byte(&cursor, data[raw_consumed], ++ byte_level)) { ++ break; ++ } ++ ++ int32_t token = iree_tokenizer_trie_cursor_token_id(&cursor); ++ if (token < 0) continue; ++ ++ const uint32_t token_id = (uint32_t)token; ++ if (token_id >= model->vocab_capacity) continue; ++ ++ iree_tokenizer_bpe_split_entry_t split = ++ model->backtrack_tables.split_table[token_id]; ++ const bool is_base_token = ++ split.left_id == token_id && split.right_id == token_id; ++ const bool is_matchable = ++ model->backtrack_tables.effective_rank[token_id] > 0; ++ if (is_base_token && is_matchable) { ++ iree_string_view_t token_text = ++ iree_tokenizer_vocab_token_text(model->vocab, token); ++ if (raw_consumed == 0 && data[0] >= 0x80 && token_text.size == 1 && ++ raw_consumed + 1 < size) { ++ fallback_base_token = token; ++ fallback_base_length = raw_consumed + 1; ++ continue; ++ } ++ if (out_raw_length) *out_raw_length = raw_consumed + 1; ++ return token; ++ } ++ } ++ ++ if (fallback_base_token >= 0) { ++ if (out_raw_length) *out_raw_length = fallback_base_length; ++ return fallback_base_token; ++ } ++ ++ if (byte_token >= 0) { ++ if (out_raw_length) *out_raw_length = 1; ++ return byte_token; ++ } ++ ++ return -1; ++} ++ + //===----------------------------------------------------------------------===// + // Table Construction (bpe_tables.c) + //===----------------------------------------------------------------------===// +diff --git a/runtime/src/iree/tokenizer/model/bpe_tables.c b/runtime/src/iree/tokenizer/model/bpe_tables.c +index c44021d3e1..480d7cc4e8 100644 +--- a/runtime/src/iree/tokenizer/model/bpe_tables.c ++++ b/runtime/src/iree/tokenizer/model/bpe_tables.c +@@ -13,6 +13,28 @@ + #include "iree/tokenizer/vocab/vocab_merge_hash.h" + #include "iree/tokenizer/vocab/vocab_trie.h" + ++static bool iree_tokenizer_bpe_is_repeated_initial_symbol( ++ iree_string_view_t text) { ++ if (text.size < 2) return false; ++ ++ bool all_newlines = true; ++ bool all_tabs = true; ++ for (iree_host_size_t i = 0; i < text.size; ++i) { ++ all_newlines &= text.data[i] == '\n'; ++ all_tabs &= text.data[i] == '\t'; ++ } ++ if (all_newlines || all_tabs) return true; ++ ++ static const char kMetaspace[] = "\xE2\x96\x81"; ++ if (text.size % (sizeof(kMetaspace) - 1) != 0) return false; ++ for (iree_host_size_t i = 0; i < text.size; i += sizeof(kMetaspace) - 1) { ++ if (memcmp(text.data + i, kMetaspace, sizeof(kMetaspace) - 1) != 0) { ++ return false; ++ } ++ } ++ return true; ++} ++ + //===----------------------------------------------------------------------===// + // BPE Backtracking: First Token Reachability + //===----------------------------------------------------------------------===// +@@ -56,7 +78,6 @@ uint32_t iree_tokenizer_bpe_right_boundary_consumed_rank( + const iree_tokenizer_bpe_model_t* model, uint32_t token) { + const iree_tokenizer_bpe_split_entry_t* split_table = + model->backtrack_tables.split_table; +- const uint32_t* effective_rank = model->backtrack_tables.effective_rank; + + // Walk down the right spine until we find where the rightmost base token + // is consumed (i.e., where the right child is a base token). +@@ -64,7 +85,7 @@ uint32_t iree_tokenizer_bpe_right_boundary_consumed_rank( + uint32_t right = split_table[token].right_id; + if (split_table[right].left_id == right) { + // Right child is a base token - it's consumed at this token's merge. +- return effective_rank[token]; ++ return iree_tokenizer_bpe_token_formation_rank(model, token); + } + // Recurse into right subtree. + token = right; +@@ -81,7 +102,6 @@ static uint32_t iree_tokenizer_bpe_left_boundary_consumed_rank( + const iree_tokenizer_bpe_model_t* model, uint32_t token) { + const iree_tokenizer_bpe_split_entry_t* split_table = + model->backtrack_tables.split_table; +- const uint32_t* effective_rank = model->backtrack_tables.effective_rank; + + // Walk down the left spine until we find where the leftmost base token + // is consumed (i.e., where the left child is a base token). +@@ -89,7 +109,7 @@ static uint32_t iree_tokenizer_bpe_left_boundary_consumed_rank( + uint32_t left = split_table[token].left_id; + if (split_table[left].left_id == left) { + // Left child is a base token - it's consumed at this token's merge. +- return effective_rank[token]; ++ return iree_tokenizer_bpe_token_formation_rank(model, token); + } + // Recurse into left subtree. + token = left; +@@ -121,6 +141,67 @@ bool iree_tokenizer_bpe_is_first_token_reachable( + 1ull << (token % 64)); + } + ++static bool iree_tokenizer_bpe_decomposition_has_suffix_prefix_block( ++ const iree_tokenizer_bpe_model_t* model, uint32_t left, uint32_t right, ++ uint32_t token, uint32_t token_rank) { ++ const iree_tokenizer_bpe_split_entry_t* split_table = ++ model->backtrack_tables.split_table; ++ ++ uint32_t suffixes[32]; ++ uint32_t suffix_consumed_at[32]; ++ iree_host_size_t suffix_count = 0; ++ for (uint32_t current = left; ++ suffix_count < IREE_ARRAYSIZE(suffixes) && ++ split_table[current].left_id != current; ++ current = split_table[current].right_id) { ++ suffixes[suffix_count] = split_table[current].right_id; ++ suffix_consumed_at[suffix_count] = ++ iree_tokenizer_bpe_token_formation_rank(model, current); ++ suffix_count++; ++ } ++ if (suffix_count == 0) return false; ++ ++ uint32_t prefixes[32]; ++ uint32_t prefix_consumed_at[32]; ++ iree_host_size_t prefix_count = 0; ++ prefixes[prefix_count] = right; ++ prefix_consumed_at[prefix_count] = UINT32_MAX; ++ ++prefix_count; ++ for (uint32_t current = right; ++ prefix_count < IREE_ARRAYSIZE(prefixes) && ++ split_table[current].left_id != current; ++ current = split_table[current].left_id) { ++ prefixes[prefix_count] = split_table[current].left_id; ++ prefix_consumed_at[prefix_count] = ++ iree_tokenizer_bpe_token_formation_rank(model, current); ++ ++prefix_count; ++ } ++ ++ for (iree_host_size_t p = 0; p < prefix_count; ++p) { ++ for (iree_host_size_t s = 0; s < suffix_count; ++s) { ++ iree_tokenizer_merge_hash_result_t merge = ++ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, ++ (int32_t)suffixes[s], ++ (int32_t)prefixes[p]); ++ if (!iree_tokenizer_merge_hash_result_is_valid(merge)) continue; ++ if ((uint32_t)merge.result_id == token) continue; ++ ++ uint32_t merge_effective_rank = merge.rank + 1; ++ uint32_t suffix_available_at = ++ iree_tokenizer_bpe_token_formation_rank(model, suffixes[s]); ++ uint32_t prefix_available_at = ++ iree_tokenizer_bpe_token_formation_rank(model, prefixes[p]); ++ if (merge_effective_rank < suffix_available_at) continue; ++ if (merge_effective_rank < prefix_available_at) continue; ++ if (merge_effective_rank >= token_rank) continue; ++ if (merge_effective_rank >= suffix_consumed_at[s]) continue; ++ if (merge_effective_rank > prefix_consumed_at[p]) continue; ++ return true; ++ } ++ } ++ return false; ++} ++ + // Checks if a specific decomposition (left + right) of a token is reachable. + // This is separate from is_first_token_reachable to allow checking alternative + // decompositions when a token has multiple merges producing it. +@@ -128,6 +209,22 @@ bool iree_tokenizer_bpe_is_decomposition_reachable( + const iree_tokenizer_bpe_model_t* model, uint32_t left, uint32_t right, + uint32_t token) { + const uint32_t* effective_rank = model->backtrack_tables.effective_rank; ++ uint32_t decomposition_rank = effective_rank[token]; ++ iree_tokenizer_merge_hash_result_t decomposition_merge = ++ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, (int32_t)left, ++ (int32_t)right); ++ if (decomposition_merge.result_id == (int32_t)token) { ++ decomposition_rank = decomposition_merge.rank + 1; ++ } ++ ++ uint32_t left_available_at = ++ iree_tokenizer_bpe_token_formation_rank(model, left); ++ uint32_t right_available_at = ++ iree_tokenizer_bpe_token_formation_rank(model, right); ++ if (decomposition_rank < left_available_at || ++ decomposition_rank < right_available_at) { ++ return false; ++ } + + // Check for blocking merge at the split boundary. + // +@@ -158,17 +255,18 @@ bool iree_tokenizer_bpe_is_decomposition_reachable( + if (boundary_merge.result_id >= 0 && + (uint32_t)boundary_merge.result_id != token) { + // There's a different merge at this boundary. It blocks only if it fires +- // before EITHER boundary token is consumed by internal merges. +- uint32_t boundary_rank = effective_rank[(uint32_t)boundary_merge.result_id]; ++ // before the left boundary token is consumed by an internal merge and ++ // before or tied with the right boundary token's internal merge. Ties are ++ // resolved left-to-right: an internal merge in the left token starts before ++ // this boundary and wins the tie, while an internal merge in the right ++ // token starts after this boundary and loses the tie. ++ uint32_t boundary_rank = boundary_merge.rank + 1; + uint32_t left_boundary_consumed = + iree_tokenizer_bpe_right_boundary_consumed_rank(model, left); + uint32_t right_boundary_consumed = + iree_tokenizer_bpe_left_boundary_consumed_rank(model, right); +- uint32_t min_boundary_consumed = +- left_boundary_consumed < right_boundary_consumed +- ? left_boundary_consumed +- : right_boundary_consumed; +- if (boundary_rank < min_boundary_consumed) { ++ if (boundary_rank < left_boundary_consumed && ++ boundary_rank <= right_boundary_consumed) { + return false; // Blocking merge fires before either boundary is consumed. + } + } +@@ -180,7 +278,74 @@ bool iree_tokenizer_bpe_is_decomposition_reachable( + iree_tokenizer_bpe_is_first_token_reachable(model, left); + bool right_reachable = + iree_tokenizer_bpe_is_first_token_reachable(model, right); +- return left_reachable && right_reachable; ++ if (!left_reachable || !right_reachable) return false; ++ ++ return !iree_tokenizer_bpe_decomposition_has_suffix_prefix_block( ++ model, left, right, token, decomposition_rank); ++} ++ ++static bool iree_tokenizer_bpe_table_suffix_already_collected( ++ const uint32_t* suffixes, const uint32_t* suffix_consumed_at, ++ iree_host_size_t suffix_count, uint32_t suffix, uint32_t consumed_at) { ++ for (iree_host_size_t i = 0; i < suffix_count; ++i) { ++ if (suffixes[i] == suffix && suffix_consumed_at[i] == consumed_at) { ++ return true; ++ } ++ } ++ return false; ++} ++ ++static void iree_tokenizer_bpe_table_collect_reachable_suffixes( ++ const iree_tokenizer_bpe_model_t* model, uint32_t token, ++ uint32_t* suffixes, uint32_t* suffix_consumed_at, ++ iree_host_size_t* suffix_count, iree_host_size_t suffix_capacity, ++ int depth) { ++ if (*suffix_count >= suffix_capacity || depth >= 16) return; ++ ++ const iree_tokenizer_vocab_t* vocab = model->vocab; ++ const iree_tokenizer_bpe_split_entry_t* split_table = ++ model->backtrack_tables.split_table; ++ ++ if (split_table[token].left_id == token) return; // Base token. ++ ++ iree_string_view_t token_text = ++ iree_tokenizer_vocab_token_text(vocab, (int32_t)token); ++ ++ for (iree_host_size_t split_pos = 1; ++ split_pos < token_text.size && *suffix_count < suffix_capacity; ++ ++split_pos) { ++ iree_string_view_t left_text = ++ iree_make_string_view(token_text.data, split_pos); ++ iree_string_view_t right_text = iree_make_string_view( ++ token_text.data + split_pos, token_text.size - split_pos); ++ ++ int32_t left_id = iree_tokenizer_vocab_lookup(vocab, left_text); ++ int32_t right_id = iree_tokenizer_vocab_lookup(vocab, right_text); ++ if (left_id < 0 || right_id < 0) continue; ++ ++ iree_tokenizer_merge_hash_result_t merge = ++ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, left_id, ++ right_id); ++ if (merge.result_id != (int32_t)token) continue; ++ ++ if (!iree_tokenizer_bpe_is_decomposition_reachable( ++ model, (uint32_t)left_id, (uint32_t)right_id, token)) { ++ continue; ++ } ++ ++ uint32_t consumed_at = merge.rank + 1; ++ if (!iree_tokenizer_bpe_table_suffix_already_collected( ++ suffixes, suffix_consumed_at, *suffix_count, (uint32_t)right_id, ++ consumed_at)) { ++ suffixes[*suffix_count] = (uint32_t)right_id; ++ suffix_consumed_at[*suffix_count] = consumed_at; ++ ++*suffix_count; ++ } ++ ++ iree_tokenizer_bpe_table_collect_reachable_suffixes( ++ model, (uint32_t)right_id, suffixes, suffix_consumed_at, suffix_count, ++ suffix_capacity, depth + 1); ++ } + } + + //===----------------------------------------------------------------------===// +@@ -213,6 +378,7 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( + iree_host_size_t next_prefix_offset = 0; + iree_host_size_t effective_rank_offset = 0; + iree_host_size_t token_reachable_offset = 0; ++ iree_host_size_t token_has_reachable_alternates_offset = 0; + IREE_RETURN_AND_END_ZONE_IF_ERROR( + z0, + IREE_STRUCT_LAYOUT( +@@ -222,7 +388,9 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( + IREE_STRUCT_FIELD(vocab_capacity, uint32_t, &next_prefix_offset), + IREE_STRUCT_FIELD(vocab_capacity, uint32_t, &effective_rank_offset), + IREE_STRUCT_FIELD(reachable_words, uint64_t, +- &token_reachable_offset))); ++ &token_reachable_offset), ++ IREE_STRUCT_FIELD(reachable_words, uint64_t, ++ &token_has_reachable_alternates_offset))); + + IREE_RETURN_AND_END_ZONE_IF_ERROR( + z0, iree_allocator_malloc(model->allocator, slab_size, +@@ -237,6 +405,8 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( + (uint32_t*)(slab + effective_rank_offset); + model->backtrack_tables.token_reachable = + (uint64_t*)(slab + token_reachable_offset); ++ model->backtrack_tables.token_has_reachable_alternates = ++ (uint64_t*)(slab + token_has_reachable_alternates_offset); + + // Initialize: base tokens map to (self, self), rank 0, no prefix. + for (iree_host_size_t i = 0; i < vocab_capacity; ++i) { +@@ -309,6 +479,30 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( + } + } + ++ // Gemma-style BPE vocabularies contain repeated initial-symbol tokens ++ // (metaspace, newline, and tab runs) whose merge entries are not a ++ // rank-ordered DAG: longer runs can appear before the shorter runs needed to ++ // build them. HuggingFace still matches these as direct vocabulary symbols. ++ // Treat them as base tokens so reachability and suffix blocking do not try ++ // to validate impossible synthetic merge trees. ++ for (iree_host_size_t token_id = 0; token_id < vocab_capacity; ++token_id) { ++ iree_tokenizer_token_attr_t attrs = ++ iree_tokenizer_vocab_token_attrs(vocab, (int32_t)token_id); ++ if (attrs & IREE_TOKENIZER_TOKEN_ATTR_SPECIAL) continue; ++ ++ iree_string_view_t text = ++ iree_tokenizer_vocab_token_text(vocab, (int32_t)token_id); ++ if (!iree_tokenizer_bpe_is_repeated_initial_symbol(text)) continue; ++ ++ model->backtrack_tables.split_table[token_id].left_id = ++ (uint32_t)token_id; ++ model->backtrack_tables.split_table[token_id].right_id = ++ (uint32_t)token_id; ++ if (model->backtrack_tables.effective_rank[token_id] == 0) { ++ model->backtrack_tables.effective_rank[token_id] = 1; ++ } ++ } ++ + // Mark multi-byte merge participants as matchable. + // + // Some base vocabulary tokens (e.g., SentencePiece's ▁ character, U+2581) +@@ -433,6 +627,8 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( + // and no blocking boundary merge. + memset(model->backtrack_tables.token_reachable, 0, + reachable_words * sizeof(uint64_t)); ++ memset(model->backtrack_tables.token_has_reachable_alternates, 0, ++ reachable_words * sizeof(uint64_t)); + + // Initialize: mark single-codepoint base tokens as reachable. + for (iree_host_size_t token_id = 0; token_id < vocab_capacity; ++token_id) { +@@ -448,7 +644,8 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( + codepoint_count++; + } + } +- if (codepoint_count <= 1) { ++ if (codepoint_count <= 1 || ++ iree_tokenizer_bpe_is_repeated_initial_symbol(token_text)) { + model->backtrack_tables.token_reachable[token_id / 64] |= + 1ull << (token_id % 64); + } +@@ -491,36 +688,10 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( + continue; + } + +- // Check for blocking merge at the boundary. +- uint32_t left_boundary = +- iree_tokenizer_bpe_rightmost_base_token(model, merge.left_id); +- uint32_t right_boundary = +- iree_tokenizer_bpe_leftmost_base_token(model, merge.right_id); +- +- iree_tokenizer_merge_hash_result_t boundary_merge = +- iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, +- (int32_t)left_boundary, +- (int32_t)right_boundary); +- if (boundary_merge.result_id >= 0 && +- (uint32_t)boundary_merge.result_id != (uint32_t)result_id) { +- // There's a different merge at this boundary. It blocks only if it +- // fires before EITHER boundary token is consumed by internal merges. +- uint32_t boundary_rank = +- model->backtrack_tables +- .effective_rank[(uint32_t)boundary_merge.result_id]; +- uint32_t left_boundary_consumed = +- iree_tokenizer_bpe_right_boundary_consumed_rank(model, +- merge.left_id); +- uint32_t right_boundary_consumed = +- iree_tokenizer_bpe_left_boundary_consumed_rank(model, +- merge.right_id); +- uint32_t min_boundary_consumed = +- left_boundary_consumed < right_boundary_consumed +- ? left_boundary_consumed +- : right_boundary_consumed; +- if (boundary_rank < min_boundary_consumed) { +- continue; // Blocking merge fires first. +- } ++ if (!iree_tokenizer_bpe_is_decomposition_reachable( ++ model, (uint32_t)merge.left_id, (uint32_t)merge.right_id, ++ (uint32_t)result_id)) { ++ continue; + } + + // This decomposition is reachable! +@@ -549,6 +720,175 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( + } + } + ++ // Mark tokens that have reachable decompositions besides the canonical ++ // split_table decomposition. Most tokens have exactly one reachable path and ++ // can use the cheap right-spine suffix walk during encode; ambiguous tokens ++ // need the slower all-splits suffix collection for HuggingFace parity. ++ for (iree_host_size_t rank = 0; rank < merge_count; ++rank) { ++ iree_tokenizer_merge_t merge = iree_tokenizer_vocab_merge(vocab, rank); ++ iree_tokenizer_merge_hash_result_t merge_result = ++ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, ++ merge.left_id, merge.right_id); ++ int32_t result_id = merge_result.result_id; ++ if (result_id < 0 || (iree_host_size_t)result_id >= vocab_capacity) { ++ continue; ++ } ++ ++ uint32_t result = (uint32_t)result_id; ++ if (!iree_any_bit_set(model->backtrack_tables.token_reachable[result / 64], ++ 1ull << (result % 64))) { ++ continue; ++ } ++ if (!iree_tokenizer_bpe_is_decomposition_reachable( ++ model, (uint32_t)merge.left_id, (uint32_t)merge.right_id, ++ result)) { ++ continue; ++ } ++ ++ iree_tokenizer_bpe_split_entry_t canonical = ++ model->backtrack_tables.split_table[result]; ++ if (canonical.left_id != (uint32_t)merge.left_id || ++ canonical.right_id != (uint32_t)merge.right_id) { ++ model->backtrack_tables.token_has_reachable_alternates[result / 64] |= ++ 1ull << (result % 64); ++ } ++ } ++ ++ // Propagate the ambiguity bit to any token whose right-spine suffixes pass ++ // through an ambiguous token. Those parent tokens also need the precomputed ++ // all-splits suffix list during suffix blocking. ++ for (iree_host_size_t token_id = 0; token_id < vocab_capacity; ++token_id) { ++ for (uint32_t current = (uint32_t)token_id; ++ model->backtrack_tables.split_table[current].left_id != current; ++ current = model->backtrack_tables.split_table[current].right_id) { ++ if (iree_any_bit_set( ++ model->backtrack_tables ++ .token_has_reachable_alternates[current / 64], ++ 1ull << (current % 64))) { ++ model->backtrack_tables ++ .token_has_reachable_alternates[token_id / 64] |= ++ 1ull << (token_id % 64); ++ break; ++ } ++ } ++ } ++ ++ for (iree_host_size_t token_id = 0; token_id < vocab_capacity; ++token_id) { ++ uint32_t cheap_suffixes[32]; ++ uint32_t cheap_consumed_at[32]; ++ iree_host_size_t cheap_count = 0; ++ for (uint32_t current = (uint32_t)token_id; ++ cheap_count < IREE_ARRAYSIZE(cheap_suffixes) && ++ model->backtrack_tables.split_table[current].left_id != current; ++ current = model->backtrack_tables.split_table[current].right_id) { ++ cheap_suffixes[cheap_count] = ++ model->backtrack_tables.split_table[current].right_id; ++ cheap_consumed_at[cheap_count] = ++ iree_tokenizer_bpe_token_formation_rank(model, current); ++ ++cheap_count; ++ } ++ ++ uint32_t all_suffixes[32]; ++ uint32_t all_consumed_at[32]; ++ iree_host_size_t all_count = 0; ++ iree_tokenizer_bpe_table_collect_reachable_suffixes( ++ model, (uint32_t)token_id, all_suffixes, all_consumed_at, &all_count, ++ IREE_ARRAYSIZE(all_suffixes), 0); ++ ++ bool differs = cheap_count != all_count; ++ for (iree_host_size_t i = 0; i < all_count && !differs; ++i) { ++ bool found = false; ++ for (iree_host_size_t j = 0; j < cheap_count; ++j) { ++ if (all_suffixes[i] == cheap_suffixes[j] && ++ all_consumed_at[i] == cheap_consumed_at[j]) { ++ found = true; ++ break; ++ } ++ } ++ differs = !found; ++ } ++ if (differs) { ++ model->backtrack_tables.token_has_reachable_alternates[token_id / 64] |= ++ 1ull << (token_id % 64); ++ } ++ } ++ ++ iree_host_size_t total_alternate_suffixes = 0; ++ for (iree_host_size_t token_id = 0; token_id < vocab_capacity; ++token_id) { ++ if (!iree_any_bit_set( ++ model->backtrack_tables ++ .token_has_reachable_alternates[token_id / 64], ++ 1ull << (token_id % 64))) { ++ continue; ++ } ++ uint32_t suffixes[32]; ++ uint32_t suffix_consumed_at[32]; ++ iree_host_size_t suffix_count = 0; ++ iree_tokenizer_bpe_table_collect_reachable_suffixes( ++ model, (uint32_t)token_id, suffixes, suffix_consumed_at, &suffix_count, ++ IREE_ARRAYSIZE(suffixes), 0); ++ total_alternate_suffixes += suffix_count; ++ } ++ ++ if (total_alternate_suffixes > 0) { ++ iree_host_size_t alt_slab_size = 0; ++ iree_host_size_t alt_offsets_offset = 0; ++ iree_host_size_t alt_suffixes_offset = 0; ++ iree_host_size_t alt_consumed_offset = 0; ++ IREE_RETURN_AND_END_ZONE_IF_ERROR( ++ z0, ++ IREE_STRUCT_LAYOUT( ++ /*base_size=*/0, &alt_slab_size, ++ IREE_STRUCT_FIELD(vocab_capacity + 1, uint32_t, ++ &alt_offsets_offset), ++ IREE_STRUCT_FIELD(total_alternate_suffixes, uint32_t, ++ &alt_suffixes_offset), ++ IREE_STRUCT_FIELD(total_alternate_suffixes, uint32_t, ++ &alt_consumed_offset))); ++ ++ IREE_RETURN_AND_END_ZONE_IF_ERROR( ++ z0, iree_allocator_malloc(model->allocator, alt_slab_size, ++ &model->backtrack_tables ++ .alternate_suffix_slab)); ++ ++ uint8_t* alt_slab = ++ (uint8_t*)model->backtrack_tables.alternate_suffix_slab; ++ model->backtrack_tables.alternate_suffix_offsets = ++ (uint32_t*)(alt_slab + alt_offsets_offset); ++ model->backtrack_tables.alternate_suffixes = ++ (uint32_t*)(alt_slab + alt_suffixes_offset); ++ model->backtrack_tables.alternate_suffix_consumed_at = ++ (uint32_t*)(alt_slab + alt_consumed_offset); ++ ++ iree_host_size_t write_index = 0; ++ for (iree_host_size_t token_id = 0; token_id < vocab_capacity; ++ ++token_id) { ++ model->backtrack_tables.alternate_suffix_offsets[token_id] = ++ (uint32_t)write_index; ++ if (!iree_any_bit_set( ++ model->backtrack_tables ++ .token_has_reachable_alternates[token_id / 64], ++ 1ull << (token_id % 64))) { ++ continue; ++ } ++ ++ uint32_t suffixes[32]; ++ uint32_t suffix_consumed_at[32]; ++ iree_host_size_t suffix_count = 0; ++ iree_tokenizer_bpe_table_collect_reachable_suffixes( ++ model, (uint32_t)token_id, suffixes, suffix_consumed_at, ++ &suffix_count, IREE_ARRAYSIZE(suffixes), 0); ++ for (iree_host_size_t i = 0; i < suffix_count; ++i) { ++ model->backtrack_tables.alternate_suffixes[write_index] = suffixes[i]; ++ model->backtrack_tables.alternate_suffix_consumed_at[write_index] = ++ suffix_consumed_at[i]; ++ ++write_index; ++ } ++ } ++ model->backtrack_tables.alternate_suffix_offsets[vocab_capacity] = ++ (uint32_t)write_index; ++ } ++ + IREE_TRACE_ZONE_END(z0); + return iree_ok_status(); + } +diff --git a/runtime/src/iree/tokenizer/model/bpe_test.cc b/runtime/src/iree/tokenizer/model/bpe_test.cc +index f2b82260d5..5bac3f0f44 100644 +--- a/runtime/src/iree/tokenizer/model/bpe_test.cc ++++ b/runtime/src/iree/tokenizer/model/bpe_test.cc +@@ -82,13 +82,13 @@ TEST_F(BPEModelTest, StateSizeIsReasonable) { + // tokens. For max_token_length=1, vocab_capacity=1: + // window: 2 * 12 = 24 + // heap: 3 * 8 = 24 +- // backtrack stack: 2048 * 8 = 16,384 +- // backtrack bitfield: 33 * 8 = 264 ++ // backtrack stack: 32768 * 12 = 393,216 ++ // backtrack bitfield: 513 * 8 = 4,104 + // pair cache: 4096 * 8 = 32,768 + // word cache: 0 (disabled, vocab < 256) + // struct: ~128 +- // total: ~49,592 +- EXPECT_LE(state_size, 65536u); ++ // total: ~430,240 ++ EXPECT_LE(state_size, 512 * 1024u); + } + + //===----------------------------------------------------------------------===// +@@ -2738,6 +2738,43 @@ TEST_F(BPEModelTest, MergePriorityWithPrecedingToken) { + /*expect_pending_after_encode=*/false); + } + ++// Gemma-style tokenizers use U+2581 metaspace as a multi-byte base token. Very ++// long segments bypass the backtracking path and use the window/heap encoder, ++// which must still seed base tokens before applying rank-priority merges. ++TEST_F(BPEModelTest, LongSegmentMergePriorityAfterMultiByteBaseToken) { ++ ScopedVocabBuilder builder; ++ ++ builder.AddToken(0, "▁"); ++ builder.AddToken(1, "z"); ++ builder.AddToken(2, "m"); ++ builder.AddToken(3, "l"); ++ builder.AddToken(4, "x"); ++ builder.AddToken(5, "▁z"); ++ builder.AddToken(6, "ml"); ++ builder.AddToken(7, "▁zm"); ++ builder.AddToken(8, "zm"); ++ ++ builder.AddMerge(0, 1); // Rank 0: ▁ + z -> ▁z ++ builder.AddMerge(2, 3); // Rank 1: m + l -> ml ++ builder.AddMerge(5, 2); // Rank 2: ▁z + m -> ▁zm ++ builder.AddMerge(0, 8); // Rank 3: ▁ + zm -> ▁zm (alternate path) ++ builder.AddMerge(1, 2); // Rank 4: z + m -> zm ++ ++ CreateModel(builder); ++ ++ std::string input = "▁zml"; ++ input.append(9000, 'x'); ++ auto tokens = EncodeAndFinalize(model(), input, ++ /*expect_pending_after_encode=*/false); ++ ++ ASSERT_EQ(tokens.size(), 9002u); ++ EXPECT_EQ(tokens[0], 5); // ▁z ++ EXPECT_EQ(tokens[1], 6); // ml ++ for (size_t i = 2; i < tokens.size(); ++i) { ++ EXPECT_EQ(tokens[i], 4); ++ } ++} ++ + // Tests ByteLevel transformation of multi-byte UTF-8 sequences. + // This is the Qwen unicode bug scenario where 'él' (UTF-8: c3 a9 6c) should + // transform to 'él' (UTF-8: c3 83 c2 a9 6c) and match as a single token. +diff --git a/runtime/src/iree/tokenizer/tokenizer.c b/runtime/src/iree/tokenizer/tokenizer.c +index 2299f0de7e..76011353f9 100644 +--- a/runtime/src/iree/tokenizer/tokenizer.c ++++ b/runtime/src/iree/tokenizer/tokenizer.c +@@ -1571,6 +1571,19 @@ static iree_status_t iree_tokenizer_encode_ring_as_segment( + state->transform_buffer.data, physical_write); + } + ++ if (is_partial) { ++ iree_host_size_t incomplete_tail = ++ iree_unicode_utf8_incomplete_tail_length( ++ (const char*)state->transform_buffer.data + physical_read, ++ data_length); ++ data_length -= incomplete_tail; ++ if (data_length == 0) { ++ *out_tokens_written = 0; ++ *out_segment_complete = false; ++ return iree_ok_status(); ++ } ++ } ++ + iree_tokenizer_segment_t segment = { + .start = physical_read, + .end = physical_read + data_length, +diff --git a/runtime/src/iree/tokenizer/tokenizer.h b/runtime/src/iree/tokenizer/tokenizer.h +index 098e6fef87..40db2bb459 100644 +--- a/runtime/src/iree/tokenizer/tokenizer.h ++++ b/runtime/src/iree/tokenizer/tokenizer.h +@@ -32,16 +32,17 @@ extern "C" { + // Maximum transform buffer allocation. + // Recommended values: + // 8KB: Minimal memory, suitable for most English text +-// 16KB: Good balance for mixed content and cache locality ++// 64KB: Keeps a 32KB logical ring so large metaspace-normalized prompts can ++// stay on the exact BPE path instead of the approximate streaming path. + // + // The transform buffer determines the maximum segment size that the segmenter + // pipeline processes. Larger buffers allow longer segments, but segments that +-// exceed L1 data cache (typically 32KB) cause DFA cache misses during +-// multi-stage pre-tokenization. 16KB (8KB logical capacity after ring buffer +-// halving) keeps the working set cache-friendly for segmenters with multiple +-// children (e.g., DeepSeek V3's 4-stage Sequence pre-tokenizer). ++// exceed L1 data cache (typically 32KB) can cause DFA cache misses during ++// multi-stage pre-tokenization. 64KB (32KB logical capacity after ring buffer ++// halving) favors HuggingFace-compatible exact BPE for long normalized ++// segments. + #ifndef IREE_TOKENIZER_TRANSFORM_BUFFER_MAX_SIZE +-#define IREE_TOKENIZER_TRANSFORM_BUFFER_MAX_SIZE (16 * 1024) ++#define IREE_TOKENIZER_TRANSFORM_BUFFER_MAX_SIZE (64 * 1024) + #endif + + // Normalizer expansion factor for transform buffer sizing. diff --git a/third_party/iree/repo.bzl b/third_party/iree/repo.bzl index e266c641..d51b195e 100644 --- a/third_party/iree/repo.bzl +++ b/third_party/iree/repo.bzl @@ -1,4 +1,5 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") +load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") def repo(): git_repository( @@ -16,8 +17,10 @@ def repo(): "//third_party/iree:fix-gemma4-encode.patch", "//third_party/iree:fix-added-token-matching.patch", "//third_party/iree:futex-public-api-header.patch", + "//third_party/iree:match-hf-tokenizer.patch", ], patch_args = ["-p1"], ) - #new_local_repository(name = "flashattn", build_file="//:third_party/flashattn/BUILD.bazel", path="/home/corendos/flashattn/") + # Use this if you want to use a local copy of IREE instead of the git repository. Make sure to update the path to point to your local IREE checkout. + #new_local_repository(name = "iree", build_file="//:third_party/iree/BUILD.bazel", path="/home/corendos/iree/") diff --git a/zml/tokenizer/iree/tokenizer.zig b/zml/tokenizer/iree/tokenizer.zig index 3526f189..ee64aca0 100644 --- a/zml/tokenizer/iree/tokenizer.zig +++ b/zml/tokenizer/iree/tokenizer.zig @@ -323,7 +323,11 @@ pub const Tokenizer = struct { }; out.advance(produced * @sizeOf(u32)); - break; + if (!c.iree_tokenizer_encode_state_has_pending(self.state)) break; + if (produced == 0) { + min_tokens *= 2; + std.debug.assert(min_tokens < 1 << 20); + } } } From 28e5d9ed7f6b80cc09e79565d41ffd32e3ba26cc Mon Sep 17 00:00:00 2001 From: Olivier Boulant Date: Thu, 2 Jul 2026 14:01:18 +0200 Subject: [PATCH 554/881] pjrt: fix misleading pjrt plugin load success log (#622) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the PJRT plugin “Loaded” log so it is only emitted after the dynamic library `.so` is opened and the PJRT API is initialized successfully. Previously, `Api.loadFrom` used a `defer` for the success log. Failed plugin load attempts could still print `Loaded: ...`. This was misleading during platform auto-detection, for example when probing TPU before falling back to CUDA. Example: ``` 2026-07-01 13:12:17 ███████╗███╗ ███╗██╗ ██╗ ██╗ ██╗ ███╗ ███╗██████╗ 2026-07-01 13:12:17 ╚══███╔╝████╗ ████║██║ ██╔╝ ██║ ██║ ████╗ ████║██╔══██╗ 2026-07-01 13:12:17 ███╔╝ ██╔████╔██║██║ ██╔╝ ██║ ██║ ██╔████╔██║██║ ██║ 2026-07-01 13:12:17 ███╔╝ ██║╚██╔╝██║██║ .ai ██╔╝ ██║ ██║ ██║╚██╔╝██║██║ ██║ 2026-07-01 13:12:17 ███████╗██║ ╚═╝ ██║███████╗ ██╔╝ ███████╗███████╗██║ ╚═╝ ██║██████╔╝ 2026-07-01 13:12:17 ╚══════╝╚═╝ ╚═╝╚══════╝ ╚═╝ ╚══════╝╚══════╝╚═╝ ╚═╝╚═════╝ 2026-07-01 13:12:17 Copyright (c) 2026 ZML 2026-07-01 13:12:17 warning(zml/io/vfs/s3): S3 initialized without full credentials (AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY). Access will be unauthenticated. Using default endpoint: https://s3.amazonaws.com (set AWS_ENDPOINT_URL or AWS_ENDPOINT_URL_S3). Using default region: us-east-1 (set AWS_REGION or AWS_DEFAULT_REGION). 2026-07-01 13:12:17 info(pjrt): Loading: libpjrt_tpu.so... 2026-07-01 13:12:17 info(llmd): Loading tokenizer 2026-07-01 13:12:17 error(pjrt): Unable to dlopen plugin: /llmd/llmd.runfiles/zml++tpu_packages+libpjrt_tpu/sandbox/lib/libpjrt_tpu.so 2026-07-01 13:12:17 info(pjrt): Loaded: libpjrt_tpu.so #################### HERE #################### 2026-07-01 13:12:17 info(llmd): Resolving model repo 2026-07-01 13:12:17 info(llmd): Loading model config 2026-07-01 13:12:17 info(llmd): Loaded model config [67.405ms] 2026-07-01 13:12:18 info(pjrt): Loading: libpjrt_cuda.so... 2026-07-01 13:12:18 warning(zml/platforms/cuda): Detected NVIDIA GPU that requires CUDA compatibility libraries. 2026-07-01 13:12:18 info(zml/platforms/cuda): Loaded CUDA compatibility libraries. 2026-07-01 13:12:18 info(pjrt): Loaded: libpjrt_cuda.so 2026-07-01 13:12:18 info(llmd): Loaded tokenizer [334.168ms] 2026-07-01 13:12:20 info(llmd): Devices: 2026-07-01 13:12:20 info(llmd): - NVIDIA RTX PRO 6000 Blackwell Server Edition (cuda:0) 2026-07-01 13:12:20 info: Using 31711 pages for the KvCache 2026-07-01 13:12:20 info: sizes: { 256 } 2026-07-01 13:12:27 info: Compiled all models [7.336s] 2026-07-01 13:12:50 info: Listening on 0.0.0.0:8000 2026-07-01 13:12:50 info(llama): Loaded weights [14.96GiB, 22.778s, 672.41MiB/s] ``` Moreover make `fromDynLib` return errors for missing `GetPjrtApi` and failed `PJRT_Plugin_Initialize` instead of panicking or using `catch unreachable`. So that `Api.loadFrom()` can catch it. After the change, the output is: ``` 2026-07-01 14:50:50 ███████╗███╗ ███╗██╗ ██╗ ██╗ ██╗ ███╗ ███╗██████╗ 2026-07-01 14:50:50 ╚══███╔╝████╗ ████║██║ ██╔╝ ██║ ██║ ████╗ ████║██╔══██╗ 2026-07-01 14:50:50 ███╔╝ ██╔████╔██║██║ ██╔╝ ██║ ██║ ██╔████╔██║██║ ██║ 2026-07-01 14:50:50 ███╔╝ ██║╚██╔╝██║██║ .ai ██╔╝ ██║ ██║ ██║╚██╔╝██║██║ ██║ 2026-07-01 14:50:50 ███████╗██║ ╚═╝ ██║███████╗ ██╔╝ ███████╗███████╗██║ ╚═╝ ██║██████╔╝ 2026-07-01 14:50:50 ╚══════╝╚═╝ ╚═╝╚══════╝ ╚═╝ ╚══════╝╚══════╝╚═╝ ╚═╝╚═════╝ 2026-07-01 14:50:50 Copyright (c) 2026 ZML 2026-07-01 14:50:51 warning(zml/io/vfs/s3): S3 initialized without full credentials (AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY). Access will be unauthenticated. Using default endpoint: https://s3.amazonaws.com (set AWS_ENDPOINT_URL or AWS_ENDPOINT_URL_S3). Using default region: us-east-1 (set AWS_REGION or AWS_DEFAULT_REGION). 2026-07-01 14:50:51 info(llmd): Resolving model repo 2026-07-01 14:50:51 info(llmd): Loading model config 2026-07-01 14:50:51 info(llmd): Loaded model config [101.024ms] 2026-07-01 14:50:51 info(llmd): Loading tokenizer 2026-07-01 14:50:51 info(pjrt): Loading: libpjrt_tpu.so... 2026-07-01 14:50:51 error(pjrt): Unable to dlopen plugin: /llmd/llmd.runfiles/zml++tpu_packages+libpjrt_tpu/sandbox/lib/libpjrt_tpu.so 2026-07-01 14:50:52 info(llmd): Loaded tokenizer [349.499ms] 2026-07-01 14:50:53 info(pjrt): Loading: libpjrt_cuda.so... 2026-07-01 14:50:53 warning(zml/platforms/cuda): Detected NVIDIA GPU that requires CUDA compatibility libraries. 2026-07-01 14:50:53 info(zml/platforms/cuda): Loaded CUDA compatibility libraries. 2026-07-01 14:51:11 info(pjrt): Loaded: libpjrt_cuda.so 2026-07-01 14:51:14 info: Using 31711 pages for the KvCache 2026-07-01 14:51:14 info: sizes: { 256 } 2026-07-01 14:51:14 info(llmd): Devices: 2026-07-01 14:51:14 info(llmd): - NVIDIA RTX PRO 6000 Blackwell Server Edition (cuda:0) 2026-07-01 14:51:22 info: Compiled all models [8.395s] 2026-07-01 14:51:46 info: Listening on 0.0.0.0:8000 2026-07-01 14:51:46 info(llama): Loaded weights [14.96GiB, 23.629s, 648.20MiB/s] ``` --- pjrt/pjrt.zig | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 0bcdbc5c..6c112490 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -99,7 +99,6 @@ pub const Api = struct { pub fn loadFrom(library: [:0]const u8) !*const Api { const basename = std.Io.Dir.path.basename(library); log.info("Loading: {s}...", .{basename}); - defer log.info("Loaded: {s}", .{basename}); var lib: std.DynLib = switch (builtin.os.tag) { .linux, .macos => blk: { @@ -124,16 +123,21 @@ pub const Api = struct { }, }; - return fromDynLib(&lib); + const api = fromDynLib(&lib) catch |err| { + log.err("Unable to load PJRT API from plugin: {s}: {}", .{ library, err }); + return err; + }; + log.info("Loaded: {s}", .{basename}); + return api; } pub fn fromDynLib(lib: *std.DynLib) !*const Api { const DynGetPjrtApi = lib.lookup(*const fn () callconv(.c) *const Api, "GetPjrtApi") orelse { - std.debug.panic("Unable to find GetPjrtApi symbol in library", .{}); + return error.MissingGetPjrtApi; }; const api = DynGetPjrtApi(); - _ = api.call(.PJRT_Plugin_Initialize, .{}) catch unreachable; + _ = try api.call(.PJRT_Plugin_Initialize, .{}); return api; } From 51db37245551594da5f277bd52d4ec78473f8aff Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 2 Jul 2026 15:46:02 +0200 Subject: [PATCH 555/881] third_party/iree: bump iree tokenizer to 4d4e97d00f099a21f38eeff26f82a6d9e3643a11 (#624) --- third_party/iree/repo.bzl | 4 +--- third_party/iree/tokenizer-only.patch | 6 ++++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/third_party/iree/repo.bzl b/third_party/iree/repo.bzl index d51b195e..d0e83ed5 100644 --- a/third_party/iree/repo.bzl +++ b/third_party/iree/repo.bzl @@ -5,7 +5,7 @@ def repo(): git_repository( name = "iree", remote = "https://github.com/iree-org/iree.git", - commit = "71af3a5e41a8e265330bc693194c708cf6df4724", + commit = "4d4e97d00f099a21f38eeff26f82a6d9e3643a11", sparse_checkout_patterns = [ "runtime/src/**", "build_tools/bazel/**", @@ -14,9 +14,7 @@ def repo(): ], patches = [ "//third_party/iree:tokenizer-only.patch", - "//third_party/iree:fix-gemma4-encode.patch", "//third_party/iree:fix-added-token-matching.patch", - "//third_party/iree:futex-public-api-header.patch", "//third_party/iree:match-hf-tokenizer.patch", ], patch_args = ["-p1"], diff --git a/third_party/iree/tokenizer-only.patch b/third_party/iree/tokenizer-only.patch index 26b5408b..c6be27da 100644 --- a/third_party/iree/tokenizer-only.patch +++ b/third_party/iree/tokenizer-only.patch @@ -1,8 +1,7 @@ diff --git a/build_tools/bazel/build_defs.oss.bzl b/build_tools/bazel/build_defs.oss.bzl -index 090c87f860..ed1ab94fd0 100644 --- a/build_tools/bazel/build_defs.oss.bzl +++ b/build_tools/bazel/build_defs.oss.bzl -@@ -4,23 +4,33 @@ +@@ -4,24 +4,35 @@ # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -11,12 +10,14 @@ index 090c87f860..ed1ab94fd0 100644 # All load statements must come first in Starlark. load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") +-load("@rules_shell//shell:sh_test.bzl", "sh_test") -load( - "//build_tools/bazel:iree_cc_fuzz.bzl", - _iree_cc_fuzz = "iree_cc_fuzz", - _iree_compiler_cc_fuzz = "iree_compiler_cc_fuzz", - _iree_runtime_cc_fuzz = "iree_runtime_cc_fuzz", -) ++# load("@rules_shell//shell:sh_test.bzl", "sh_test") +# load( +# "//build_tools/bazel:iree_cc_fuzz.bzl", +# _iree_cc_fuzz = "iree_cc_fuzz", @@ -40,6 +41,7 @@ index 090c87f860..ed1ab94fd0 100644 +gentbl_cc_library = generic_pass +gentbl_filegroup = generic_pass +td_library = generic_pass ++sh_test = generic_pass +iree_runtime_cc_fuzz = generic_pass +iree_compiler_cc_fuzz = generic_pass +iree_cc_fuzz = generic_pass From 73498264768f7ccc05ecbbbc919a98b8370ce45f Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 2 Jul 2026 17:02:51 +0200 Subject: [PATCH 556/881] zml/nn: improve `sampleTokensDynamic()` (#625) This PR improves the `zml.nn.sampleTokensDynamic()` function. Most notably: * Convert parameters to input data type * Support scalar/batched parameters * Correctly use the temperature parameter --- zml/nn.zig | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index e0a564d0..f99be197 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -1676,8 +1676,10 @@ fn fixupLogits(logits: Tensor, opts: DynamicSamplingStrategy) [2]Tensor { // After the topk, we don't have .voc indices, anymore, only topk. var x = full_topk.values.rename(.{ .voc = .topk }); // mask values above the dynamic top_k - x = Tensor.iota(x.shape(), .topk).cmp(.GE, opts.top_k).select(min_inf, x); - x = x.mul(opts.temperature); + const top_k = opts.top_k.broad(x.shape()); + x = Tensor.iota(x.shape(), .topk).cmp(.GE, top_k).select(min_inf, x); + const temperature = opts.temperature.convert(x.dtype()).broad(x.shape()); + x = x.div(temperature); // if there are high values in x, softmax can overflow and will create nans in full probs // this propagate to probs_sum and probs_max. @@ -1685,8 +1687,8 @@ fn fixupLogits(logits: Tensor, opts: DynamicSamplingStrategy) [2]Tensor { const probs_sum = probs.cumulativeSum(.topk); const probs_max = probs.slice1d(.topk, .{ .start = 0, .end = 1 }); - const top_p = opts.top_p.broad(x.shape()); - const min_p = probs_max.mul(opts.min_p).broad(x.shape()); + const top_p = opts.top_p.convert(x.dtype()).broad(x.shape()); + const min_p = opts.min_p.convert(x.dtype()).broad(probs_max.shape()).mul(probs_max).broad(x.shape()); // * if first candidate has very high prob, then probs_sum is always greater than top_p and candidate is full false // * if first candidate score is even bigger, the probs become Nan because of the softmax, @@ -1720,7 +1722,7 @@ test sampleTokensDynamic { // top_k == logits.len -> just sort the input .{ .{ .top_k = 4 }, [_]f32{ @log(4.0), @log(3.0), @log(2.0), @log(1.0) } }, .{ .{ .top_k = 2 }, [_]f32{ @log(4.0), @log(3.0), ___, ___ } }, - .{ .{ .top_k = 2, .temperature = 0.1 }, [_]f32{ @log(4.0) * 0.1, @log(3.0) * 0.1, ___, ___ } }, + .{ .{ .top_k = 2, .temperature = 0.1 }, [_]f32{ @log(4.0) / 0.1, @log(3.0) / 0.1, ___, ___ } }, // top_k == logits.len and small top_p -> make sure at least one is returned .{ .{ .top_k = 4, .top_p = 0.1 }, [_]f32{ @log(4.0), ___, ___, ___ } }, .{ .{ .top_k = 4, .top_p = 0.701 }, [_]f32{ @log(4.0), @log(3.0), ___, ___ } }, From 2057d68556af7e6b34d970de90250b472c7c06fe Mon Sep 17 00:00:00 2001 From: elogir Date: Fri, 3 Jul 2026 11:23:43 +0200 Subject: [PATCH 557/881] platforms/metal: bump plugin & remove metadata from struct (#620) - Bump Metal plugin version - Drop the attention metadata fields from the llama and lfm2 Sessions. Prefill and decode now each build their own metadata inline and free it. --- examples/llm/models/lfm2/session.zig | 19 +++++++--- examples/llm/models/llama/session.zig | 54 ++++++++++----------------- platforms/metal/metal.bzl | 4 +- zml/attention/attention.zig | 2 +- 4 files changed, 37 insertions(+), 42 deletions(-) diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig index d367cbe6..d78f5a8d 100644 --- a/examples/llm/models/lfm2/session.zig +++ b/examples/llm/models/lfm2/session.zig @@ -15,7 +15,6 @@ pub const Session = struct { config: *const model.Config, seqlen: u32, cache_buffers: zml.Bufferized(model.Cache), - attention_metadata_buffers: zml.Bufferized(attention.Metadata), rng_buf: zml.Bufferized(zml.Tensor.Rng), generated_token_slice: zml.Slice, think_start: ?u32, @@ -41,7 +40,6 @@ pub const Session = struct { .config = &compiled_model.loaded_model.parsed_config.value, .seqlen = compiled_model.params.seqlen, .cache_buffers = try compiled_model.params.cache.initBuffers(allocator, io, platform, compiled_model.params.shardings.model), - .attention_metadata_buffers = try compiled_model.params.attention_metadata.initBuffer(io, platform, compiled_model.params.shardings.model), .rng_buf = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed), .generated_token_slice = try .alloc(allocator, zml.Shape.init(.{ .batch = 1, .seq = 1 }, .u32)), .think_start = tokenizer.tokenId("") orelse unreachable, @@ -51,7 +49,6 @@ pub const Session = struct { pub fn deinit(self: *Session) void { model.Cache.unloadBuffers(&self.cache_buffers); - attention.Metadata.deinitBuffer(&self.attention_metadata_buffers); zml.Tensor.Rng.deinitBuffer(&self.rng_buf); self.generated_token_slice.free(self.allocator); } @@ -120,6 +117,13 @@ pub const Session = struct { var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice, .replicated); defer actual_seq_len_buf.deinit(); + const params = self.compiled_model.params; + var attention_metadata_buffers: zml.Bufferized(attention.Metadata) = switch (params.attention_metadata) { + .metal_fa => .{ .metal_fa = .{ .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32) } }, + else => try params.attention_metadata.initBuffer(self.io, self.platform, params.shardings.model), + }; + defer attention.Metadata.deinitBuffer(&attention_metadata_buffers); + try self.compiled_model.prefill.run(.{ .allocator = self.allocator, .io = self.io, @@ -130,7 +134,7 @@ pub const Session = struct { .actual_seq_len_buf = &actual_seq_len_buf, .rng_buf = &self.rng_buf, .cache_buffers = &self.cache_buffers, - .attention_metadata_buffers = self.attention_metadata_buffers, + .attention_metadata_buffers = attention_metadata_buffers, }); try tokens_buf.toSlice(self.io, tokens_slice); @@ -150,6 +154,11 @@ pub const Session = struct { const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); defer self.allocator.free(out_tokens_buffer); + + const params = self.compiled_model.params; + var attention_metadata_buffers = try params.attention_metadata.initBuffer(self.io, self.platform, params.shardings.model); + defer attention.Metadata.deinitBuffer(&attention_metadata_buffers); + generation: while (true) { const token_id = self.generated_token_slice.items(u32)[0]; @@ -182,7 +191,7 @@ pub const Session = struct { .actual_seq_len_buf = &actual_seq_len_buf, .rng_buf = &self.rng_buf, .cache_buffers = &self.cache_buffers, - .attention_metadata_buffers = self.attention_metadata_buffers, + .attention_metadata_buffers = attention_metadata_buffers, }); try current_token_buffer.toSlice(self.io, self.generated_token_slice); diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig index d336946f..3a55ea85 100644 --- a/examples/llm/models/llama/session.zig +++ b/examples/llm/models/llama/session.zig @@ -1,6 +1,7 @@ const std = @import("std"); const zml = @import("zml"); +const attention = zml.attention.attention; const inference = @import("inference.zig"); const model = @import("model.zig"); @@ -14,8 +15,6 @@ pub const Session = struct { decode_runner: inference.KernelExe.Runner, kv_cache_buffers: zml.Bufferized(model.KvCache), token_index_buffers: []zml.Buffer, - attention_metadata_buffers: zml.Bufferized(zml.attention.attention.Metadata), - decode_attention_metadata_buffers: ?zml.Bufferized(zml.attention.attention.Metadata), rng_buffers: zml.Bufferized(zml.Tensor.Rng), tokenizer: zml.tokenizer.Tokenizer, config: *const model.Config, @@ -48,24 +47,8 @@ pub const Session = struct { initialized_token_index_buffers = i + 1; } - var attention_metadata_buffers = try compiled_model.params.attention_metadata.initBuffer(io, platform, shardings.model); - errdefer zml.attention.attention.Metadata.deinitBuffer(&attention_metadata_buffers); - const conversation_id: u64 = @bitCast(std.Io.Clock.now(.real, io).toMicroseconds()); - var decode_attention_metadata_buffers: ?zml.Bufferized(zml.attention.attention.Metadata) = switch (compiled_model.params.decode_attention_parameters) { - .attnd => b: { - const buffers: zml.Bufferized(zml.attention.attention.Metadata) = .{ .attnd = .{ - .conversation_id = try zml.Buffer.scalar(io, platform, conversation_id, .u64), - .layer_id = try zml.Buffer.scalar(io, platform, 0, .u16), - .num_tokens = try zml.Buffer.scalar(io, platform, 1, .u32), - } }; - break :b buffers; - }, - .vanilla, .cuda_fa2, .cuda_fa3, .nki, .metal_fa => null, - }; - errdefer if (decode_attention_metadata_buffers) |*buffers| zml.attention.attention.Metadata.deinitBuffer(buffers); - const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); var rng_buffers = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); @@ -82,8 +65,6 @@ pub const Session = struct { .decode_runner = decode_runner, .kv_cache_buffers = kv_cache_buffers, .token_index_buffers = token_index_buffers, - .attention_metadata_buffers = attention_metadata_buffers, - .decode_attention_metadata_buffers = decode_attention_metadata_buffers, .rng_buffers = rng_buffers, .tokenizer = tokenizer, .config = &compiled_model.loaded_model.parsed_config.value, @@ -99,8 +80,6 @@ pub const Session = struct { token_index_buffer.deinit(); } self.allocator.free(self.token_index_buffers); - zml.attention.attention.Metadata.deinitBuffer(&self.attention_metadata_buffers); - if (self.decode_attention_metadata_buffers) |*buffers| zml.attention.attention.Metadata.deinitBuffer(buffers); zml.Tensor.Rng.deinitBuffer(&self.rng_buffers); } @@ -161,18 +140,17 @@ pub const Session = struct { var prefill_tokens_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, prefill_tokens_slice, .replicated); defer prefill_tokens_buffer.deinit(); - var num_tokens_buffer: zml.Buffer = try .scalar(self.io, self.platform, all_tokens.len, .u32); - defer num_tokens_buffer.deinit(); - - const attention_metadata_buffers: zml.Bufferized(zml.attention.attention.Metadata) = switch (self.compiled_model.params.prefill_attention_parameters) { + const params = self.compiled_model.params; + var attention_metadata_buffers: zml.Bufferized(attention.Metadata) = switch (params.attention_metadata) { .attnd => .{ .attnd = .{ - .conversation_id = try zml.Buffer.scalar(self.io, self.platform, self.conversation_id, .u64), - .layer_id = try zml.Buffer.scalar(self.io, self.platform, 0, .u16), - .num_tokens = try zml.Buffer.scalar(self.io, self.platform, all_tokens.len, .u32), + .conversation_id = try .scalar(self.io, self.platform, self.conversation_id, .u64), + .layer_id = try .scalar(self.io, self.platform, 0, .u16), + .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32), } }, - .metal_fa => .{ .metal_fa = .{ .num_tokens = num_tokens_buffer } }, - .vanilla, .cuda_fa2, .cuda_fa3, .nki => self.attention_metadata_buffers, + .metal_fa => .{ .metal_fa = .{ .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32) } }, + .vanilla, .cuda_fa2, .cuda_fa3, .nki => try params.attention_metadata.initBuffer(self.io, self.platform, params.shardings.model), }; + defer attention.Metadata.deinitBuffer(&attention_metadata_buffers); try self.compiled_model.prefill.run(.{ .allocator = self.allocator, @@ -201,6 +179,17 @@ pub const Session = struct { var current_token_buffer: zml.Buffer = try .fromBytes(self.io, self.platform, .init(.{ .s = 1 }, .u32), .replicated, @ptrCast(&last_token_id)); defer current_token_buffer.deinit(); + const params = self.compiled_model.params; + var attention_metadata_buffers: zml.Bufferized(attention.Metadata) = switch (params.attention_metadata) { + .attnd => .{ .attnd = .{ + .conversation_id = try .scalar(self.io, self.platform, self.conversation_id, .u64), + .layer_id = try .scalar(self.io, self.platform, 0, .u16), + .num_tokens = try .scalar(self.io, self.platform, 1, .u32), + } }, + .vanilla, .cuda_fa2, .cuda_fa3, .nki, .metal_fa => try params.attention_metadata.initBuffer(self.io, self.platform, params.shardings.model), + }; + defer attention.Metadata.deinitBuffer(&attention_metadata_buffers); + generation: while (true) { if (isEosToken(self.config, last_token_id)) break :generation; @@ -210,9 +199,6 @@ pub const Session = struct { try all_tokens.append(self.allocator, last_token_id); if (all_tokens.items.len >= self.seqlen) break :generation; - const attention_metadata_buffers: zml.Bufferized(zml.attention.attention.Metadata) = - self.decode_attention_metadata_buffers orelse self.attention_metadata_buffers; - try self.decode_runner.run(.{ .allocator = self.allocator, .io = self.io, diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 7ecfcf4f..525eac59 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libpjrt_metal", build_file = "libpjrt_metal.BUILD.bazel", - sha256 = "a94aaf5719fe14b2e1fd4e614df9e916443b778ced453eebea419ee479f245aa", - url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-06-23T00-20-00Z/pjrt-metal_macos-arm64.tar.zst", + sha256 = "f55356328d9d814c5a961b6c38741cff2f98b4043846a7ac3865e2a23549b767", + url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-06-30T00-16-30Z/pjrt-metal_macos-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index 89ac817c..8752763a 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -86,7 +86,7 @@ pub const MetalFaMetadata = struct { pub fn initBuffer(self: MetalFaMetadata, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(MetalFaMetadata) { _ = self; _ = sharding; - return .{ .num_tokens = try zml.Buffer.scalar(io, platform, 1, .u32) }; + return .{ .num_tokens = try zml.Buffer.scalar(io, platform, 0, .u32) }; } pub fn deinitBuffer(self: *zml.Bufferized(MetalFaMetadata)) void { From 833f42029d8c54a5d98e1038f58ad1a3c4b00a43 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Fri, 3 Jul 2026 14:22:23 +0200 Subject: [PATCH 558/881] platforms: bump cpu + cuda plugins and align xla (#626) Related PR: https://github.com/zml/monorepo/pull/304 --- MODULE.bazel | 46 +- MODULE.bazel.lock | 1565 +++++++++++++---- platforms/cpu/cpu.bzl | 12 +- platforms/cuda/cuda.bzl | 6 +- platforms/oneapi/oneapi.bzl | 4 +- third_party/abseil-cpp/bmi2intrin.patch | 13 + .../grpc/grpc++_unsecure-visibility.patch | 13 + third_party/grpc/protobuf-35-upb-load.patch | 14 + .../xla/pjrt-device-event-typedef.patch | 14 - third_party/xla/repo.bzl | 3 +- tools/xspace_to_perfetto/BUILD.bazel | 2 +- zml/BUILD.bazel | 2 +- zml/module.zig | 1 - 13 files changed, 1267 insertions(+), 428 deletions(-) create mode 100644 third_party/abseil-cpp/bmi2intrin.patch create mode 100644 third_party/grpc/grpc++_unsecure-visibility.patch create mode 100644 third_party/grpc/protobuf-35-upb-load.patch delete mode 100644 third_party/xla/pjrt-device-event-typedef.patch diff --git a/MODULE.bazel b/MODULE.bazel index a5c4a775..2649f117 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -2,7 +2,16 @@ module( name = "zml", ) -bazel_dep(name = "abseil-cpp", version = "20250814.1", repo_name = "com_google_absl") +bazel_dep(name = "abseil-cpp", version = "20260526.0", repo_name = "com_google_absl") +# Work around abseil-cpp#2071 until the BCR release includes the Clang BMI2 fix. +single_version_override( + module_name = "abseil-cpp", + patch_strip = 1, + patches = [ + "//:third_party/abseil-cpp/bmi2intrin.patch", + ], + version = "20260526.0", +) bazel_dep(name = "bazel_lib", version = "3.2.2") bazel_dep(name = "aspect_rules_py", version = "1.8.4") bazel_dep(name = "bazel_skylib", version = "1.9.0") @@ -12,11 +21,7 @@ bazel_dep(name = "libbacktrace", version = "1.0.0-20250926-7939218") bazel_dep(name = "patchelf", version = "0.18.0.bcr.1") bazel_dep(name = "pcre2", version = "10.45") bazel_dep(name = "platforms", version = "1.1.0") -bazel_dep(name = "protobuf", version = "33.6", repo_name = "com_google_protobuf") -single_version_override( - module_name = "protobuf", - version = "33.6", # important, until grpc-cpp is fixed -) +bazel_dep(name = "protobuf", version = "35.0", repo_name = "com_google_protobuf") bazel_dep(name = "pybind11_bazel", version = "3.0.1") bazel_dep(name = "re2", version = "2025-11-05", repo_name = "com_googlesource_code_re2") @@ -33,18 +38,24 @@ bazel_dep(name = "llvm", version = "0.8.9") bazel_dep(name = "zlib-ng", version = "2.3.3", repo_name = "llvm_zlib") bazel_dep(name = "zstd", version = "1.5.7.bcr.1", repo_name = "llvm_zstd") bazel_dep(name = "with_cfg.bzl", version = "0.14.4") -bazel_dep(name = "grpc", version = "1.74.0") +bazel_dep(name = "grpc", version = "1.80.0") +single_version_override( + module_name = "grpc", + patch_strip = 1, + patches = [ + "//:third_party/grpc/protobuf-35-upb-load.patch", + "//:third_party/grpc/grpc++_unsecure-visibility.patch", + ], + version = "1.80.0", +) +single_version_override( + module_name = "envoy_api", + version = "0.0.0-20250128-4de3c74", +) -# Added to support bazel 9+ since an old version is pulled by GRPC -# TODO: remove when GRPC is updated to v1.80+ +# Added to support bazel 9+ since an old version is pulled by GRPC. bazel_dep(name = "rules_foreign_cc", version = "0.15.1") -# Same as rules_foreign_cc -bazel_dep(name = "rules_go", version = "0.60.0") - -# Same as rules_foreign_cc -bazel_dep(name = "jsoncpp", version = "1.9.6.bcr.2", repo_name = "jsoncpp_git") - bazel_dep(name = "buildifier_prebuilt", version = "8.5.1", dev_dependency = True) # BSDTar toolchains @@ -281,8 +292,3 @@ use_repo( "distroless_cc_debian12", "distroless_cc_debian12_linux_amd64", ) - -single_version_override( - module_name = "grpc-java", - version = "1.74.0", -) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index f7e89935..10042d2a 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -2,60 +2,62 @@ "lockFileVersion": 26, "registryFileHashes": { "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", - "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2", - "https://bcr.bazel.build/modules/abseil-cpp/20220623.1/MODULE.bazel": "73ae41b6818d423a11fd79d95aedef1258f304448193d4db4ff90e5e7a0f076c", - "https://bcr.bazel.build/modules/abseil-cpp/20230125.1/MODULE.bazel": "89047429cb0207707b2dface14ba7f8df85273d484c2572755be4bab7ce9c3a0", - "https://bcr.bazel.build/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "1c8cec495288dccd14fdae6e3f95f772c1c91857047a098fad772034264cc8cb", - "https://bcr.bazel.build/modules/abseil-cpp/20230802.0/MODULE.bazel": "d253ae36a8bd9ee3c5955384096ccb6baf16a1b1e93e858370da0a3b94f77c16", - "https://bcr.bazel.build/modules/abseil-cpp/20230802.1/MODULE.bazel": "fa92e2eb41a04df73cdabeec37107316f7e5272650f81d6cc096418fe647b915", - "https://bcr.bazel.build/modules/abseil-cpp/20240116.0/MODULE.bazel": "98dc378d64c12a4e4741ad3362f87fb737ee6a0886b2d90c3cdbb4d93ea3e0bf", - "https://bcr.bazel.build/modules/abseil-cpp/20240116.1/MODULE.bazel": "37bcdb4440fbb61df6a1c296ae01b327f19e9bb521f9b8e26ec854b6f97309ed", - "https://bcr.bazel.build/modules/abseil-cpp/20240116.2/MODULE.bazel": "73939767a4686cd9a520d16af5ab440071ed75cec1a876bf2fcfaf1f71987a16", - "https://bcr.bazel.build/modules/abseil-cpp/20240722.0/MODULE.bazel": "88668a07647adbdc14cb3a7cd116fb23c9dda37a90a1681590b6c9d8339a5b84", - "https://bcr.bazel.build/modules/abseil-cpp/20250127.0/MODULE.bazel": "d1086e248cda6576862b4b3fe9ad76a214e08c189af5b42557a6e1888812c5d5", - "https://bcr.bazel.build/modules/abseil-cpp/20250127.1/MODULE.bazel": "c4a89e7ceb9bf1e25cf84a9f830ff6b817b72874088bf5141b314726e46a57c1", - "https://bcr.bazel.build/modules/abseil-cpp/20250512.0/MODULE.bazel": "c4d02dd22cd87458516655a45512060246ee2a4732f1fbe948a5bd9eb614e626", - "https://bcr.bazel.build/modules/abseil-cpp/20250512.1/MODULE.bazel": "d209fdb6f36ffaf61c509fcc81b19e81b411a999a934a032e10cd009a0226215", - "https://bcr.bazel.build/modules/abseil-cpp/20250814.1/MODULE.bazel": "51f2312901470cdab0dbdf3b88c40cd21c62a7ed58a3de45b365ddc5b11bcab2", - "https://bcr.bazel.build/modules/abseil-cpp/20250814.1/source.json": "cea3901d7e299da7320700abbaafe57a65d039f10d0d7ea601c4a66938ea4b0c", + "https://bcr.bazel.build/modules/abseil-cpp/20260526.0/MODULE.bazel": "b4a76a8d70ba08aa0253fd5e374ca5a5fa6b15aba985a7bbe4467c138954830f", + "https://bcr.bazel.build/modules/abseil-cpp/20260526.0/source.json": "a39a26fec1b7037bdd7ea6c25cc098b99ae80af10ff0b6ec8c4bfe0df694f451", + "https://bcr.bazel.build/modules/abseil-py/2.1.0/MODULE.bazel": "5ebe5bf853769c65707e5c28f216798f7a4b1042015e6a36e6d03094d94bec8a", + "https://bcr.bazel.build/modules/abseil-py/2.1.0/source.json": "0e8fc4f088ce07099c1cd6594c20c7ddbb48b4b3c0849b7d94ba94be88ff042b", "https://bcr.bazel.build/modules/apple_support/1.11.1/MODULE.bazel": "1843d7cd8a58369a444fc6000e7304425fba600ff641592161d9f15b179fb896", + "https://bcr.bazel.build/modules/apple_support/1.13.0/MODULE.bazel": "7c8cdea7e031b7f9f67f0b497adf6d2c6a2675e9304ca93a9af6ed84eef5a524", "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", "https://bcr.bazel.build/modules/apple_support/1.17.1/MODULE.bazel": "655c922ab1209978a94ef6ca7d9d43e940cd97d9c172fb55f94d91ac53f8610b", "https://bcr.bazel.build/modules/apple_support/1.21.0/MODULE.bazel": "ac1824ed5edf17dee2fdd4927ada30c9f8c3b520be1b5fd02a5da15bc10bff3e", "https://bcr.bazel.build/modules/apple_support/1.21.1/MODULE.bazel": "5809fa3efab15d1f3c3c635af6974044bac8a4919c62238cce06acee8a8c11f1", + "https://bcr.bazel.build/modules/apple_support/1.22.1/MODULE.bazel": "90bd1a660590f3ceffbdf524e37483094b29352d85317060b2327fff8f3f4458", "https://bcr.bazel.build/modules/apple_support/1.23.1/MODULE.bazel": "53763fed456a968cf919b3240427cf3a9d5481ec5466abc9d5dc51bc70087442", + "https://bcr.bazel.build/modules/apple_support/1.24.1/MODULE.bazel": "f46e8ddad60aef170ee92b2f3d00ef66c147ceafea68b6877cb45bd91737f5f8", "https://bcr.bazel.build/modules/apple_support/1.24.2/MODULE.bazel": "0e62471818affb9f0b26f128831d5c40b074d32e6dda5a0d3852847215a41ca4", + "https://bcr.bazel.build/modules/apple_support/2.3.0/MODULE.bazel": "d48f824ae8eeea5f837eb3038cef3615075d996d3e82eb9187192c2820605a81", "https://bcr.bazel.build/modules/apple_support/2.5.4/MODULE.bazel": "964e9268b37edb64e9a0fa525e5b5e535ba56bf63cadc68375f92e69fcb734f6", "https://bcr.bazel.build/modules/apple_support/2.5.4/source.json": "df85e46410ff71c4006a13dbdeeeee488feba563be78f3a9ed8497c1fb93fb5d", + "https://bcr.bazel.build/modules/aspect_bazel_lib/1.31.2/MODULE.bazel": "7bee702b4862612f29333590f4b658a5832d433d6f8e4395f090e8f4e85d442f", + "https://bcr.bazel.build/modules/aspect_bazel_lib/1.38.0/MODULE.bazel": "6307fec451ba9962c1c969eb516ebfe1e46528f7fa92e1c9ac8646bef4cdaa3f", + "https://bcr.bazel.build/modules/aspect_bazel_lib/1.40.3/MODULE.bazel": "668e6bcb4d957fc0e284316dba546b705c8d43c857f87119619ee83c4555b859", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.3/MODULE.bazel": "253d739ba126f62a5767d832765b12b59e9f8d2bc88cc1572f4a73e46eb298ca", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.22.5/MODULE.bazel": "004ba890363d05372a97248c37205ae64b6fa31047629cd2c0895a9d0c7779e8", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.22.5/source.json": "ac2c3213df8f985785f1d0aeb7f0f73d5324e6e67d593d9b9470fb74a25d4a9b", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "780d1a6522b28f5edb7ea09630748720721dfe27690d65a2d33aa7509de77e07", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "812d2dd42f65dca362152101fbec418029cc8fd34cbad1a2fde905383d705838", + "https://bcr.bazel.build/modules/aspect_rules_js/1.33.1/MODULE.bazel": "db3e7f16e471cf6827059d03af7c21859e7a0d2bc65429a3a11f005d46fc501b", + "https://bcr.bazel.build/modules/aspect_rules_js/1.39.0/MODULE.bazel": "aece421d479e3c31dc3e5f6d49a12acc2700457c03c556650ec7a0ff23fc0d95", + "https://bcr.bazel.build/modules/aspect_rules_lint/0.12.0/MODULE.bazel": "e767c5dbfeb254ec03275a7701b5cfde2c4d2873676804bc7cb27ddff3728fed", "https://bcr.bazel.build/modules/aspect_rules_py/1.8.4/MODULE.bazel": "4ab4a78e9d65cf15e755807f7ae504655e1771cefd7c3343ad34dbbb625b9564", "https://bcr.bazel.build/modules/aspect_rules_py/1.8.4/source.json": "7cd80ad811f0c2b3aad3c878506ae602fce2fe53b78487ecfbcc8bcde07d220e", "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.3/MODULE.bazel": "37c764292861c2f70314efa9846bb6dbb44fc0308903b3285da6528305450183", "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.3/source.json": "605086bbc197743a0d360f7ddc550a1d4dfa0441bc807236e17170f636153348", + "https://bcr.bazel.build/modules/bazel_features/0.1.0/MODULE.bazel": "47011d645b0f949f42ee67f2e8775188a9cf4a0a1528aa2fa4952f2fd00906fd", "https://bcr.bazel.build/modules/bazel_features/1.1.0/MODULE.bazel": "cfd42ff3b815a5f39554d97182657f8c4b9719568eb7fded2b9135f084bf760b", "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", "https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101", "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", + "https://bcr.bazel.build/modules/bazel_features/1.13.0/MODULE.bazel": "c14c33c7c3c730612bdbe14ebbb5e61936b6f11322ea95a6e91cd1ba962f94df", "https://bcr.bazel.build/modules/bazel_features/1.15.0/MODULE.bazel": "d38ff6e517149dc509406aca0db3ad1efdd890a85e049585b7234d04238e2a4d", "https://bcr.bazel.build/modules/bazel_features/1.17.0/MODULE.bazel": "039de32d21b816b47bd42c778e0454217e9c9caac4a3cf8e15c7231ee3ddee4d", "https://bcr.bazel.build/modules/bazel_features/1.18.0/MODULE.bazel": "1be0ae2557ab3a72a57aeb31b29be347bcdc5d2b1eb1e70f39e3851a7e97041a", "https://bcr.bazel.build/modules/bazel_features/1.19.0/MODULE.bazel": "59adcdf28230d220f0067b1f435b8537dd033bfff8db21335ef9217919c7fb58", "https://bcr.bazel.build/modules/bazel_features/1.20.0/MODULE.bazel": "8b85300b9c8594752e0721a37210e34879d23adc219ed9dc8f4104a4a1750920", "https://bcr.bazel.build/modules/bazel_features/1.21.0/MODULE.bazel": "675642261665d8eea09989aa3b8afb5c37627f1be178382c320d1b46afba5e3b", + "https://bcr.bazel.build/modules/bazel_features/1.23.0/MODULE.bazel": "fd1ac84bc4e97a5a0816b7fd7d4d4f6d837b0047cf4cbd81652d616af3a6591a", "https://bcr.bazel.build/modules/bazel_features/1.24.0/MODULE.bazel": "4796b4c25b47053e9bbffa792b3792d07e228ff66cd0405faef56a978708acd4", "https://bcr.bazel.build/modules/bazel_features/1.27.0/MODULE.bazel": "621eeee06c4458a9121d1f104efb80f39d34deff4984e778359c60eaf1a8cb65", "https://bcr.bazel.build/modules/bazel_features/1.28.0/MODULE.bazel": "4b4200e6cbf8fa335b2c3f43e1d6ef3e240319c33d43d60cc0fbd4b87ece299d", "https://bcr.bazel.build/modules/bazel_features/1.3.0/MODULE.bazel": "cdcafe83ec318cda34e02948e81d790aab8df7a929cec6f6969f13a489ccecd9", "https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87", + "https://bcr.bazel.build/modules/bazel_features/1.32.0/MODULE.bazel": "095d67022a58cb20f7e20e1aefecfa65257a222c18a938e2914fd257b5f1ccdc", "https://bcr.bazel.build/modules/bazel_features/1.33.0/MODULE.bazel": "8b8dc9d2a4c88609409c3191165bccec0e4cb044cd7a72ccbe826583303459f6", "https://bcr.bazel.build/modules/bazel_features/1.34.0/MODULE.bazel": "e8475ad7c8965542e0c7aac8af68eb48c4af904be3d614b6aa6274c092c2ea1e", - "https://bcr.bazel.build/modules/bazel_features/1.36.0/MODULE.bazel": "596cb62090b039caf1cad1d52a8bc35cf188ca9a4e279a828005e7ee49a1bec3", "https://bcr.bazel.build/modules/bazel_features/1.38.0/MODULE.bazel": "f9b8a9c890ebd216b4049fd12a31d3c2602e3403c7af636b04fbbd7453edc9c9", + "https://bcr.bazel.build/modules/bazel_features/1.39.0/MODULE.bazel": "28739425c1fc283c91931619749c832b555e60bcd1010b40d8441ce0a5cf726d", "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", "https://bcr.bazel.build/modules/bazel_features/1.42.0/MODULE.bazel": "e8ca15cb2639c5f12183db6dcb678735555d0cdd739b32a0418b6532b5e565f8", "https://bcr.bazel.build/modules/bazel_features/1.42.1/MODULE.bazel": "275a59b5406ff18c01739860aa70ad7ccb3cfb474579411decca11c93b951080", @@ -67,6 +69,7 @@ "https://bcr.bazel.build/modules/bazel_lib/3.0.0-beta.1/MODULE.bazel": "407729e232f611c3270005b016b437005daa7b1505826798ea584169a476e878", "https://bcr.bazel.build/modules/bazel_lib/3.0.0-rc.0/MODULE.bazel": "d6e00979a98ac14ada5e31c8794708b41434d461e7e7ca39b59b765e6d233b18", "https://bcr.bazel.build/modules/bazel_lib/3.0.0/MODULE.bazel": "22b70b80ac89ad3f3772526cd9feee2fa412c2b01933fea7ed13238a448d370d", + "https://bcr.bazel.build/modules/bazel_lib/3.1.0/MODULE.bazel": "6809765c14e3c766a9b9286c7b0ec56ed87a73326e48fe01749f0c0fdcfe3287", "https://bcr.bazel.build/modules/bazel_lib/3.2.2/MODULE.bazel": "e2c890c8a515d6bca9c66d47718aa9e44b458fde64ec7204b8030bf2d349058c", "https://bcr.bazel.build/modules/bazel_lib/3.2.2/source.json": "9e84e115c20e14652c5c21401ae85ff4daa8702e265b5c0b3bf89353f17aa212", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", @@ -79,13 +82,17 @@ "https://bcr.bazel.build/modules/bazel_skylib/1.4.2/MODULE.bazel": "3bd40978e7a1fac911d5989e6b09d8f64921865a45822d8b09e815eaa726a651", "https://bcr.bazel.build/modules/bazel_skylib/1.5.0/MODULE.bazel": "32880f5e2945ce6a03d1fbd588e9198c0a959bb42297b2cfaf1685b7bc32e138", "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/MODULE.bazel": "8fdee2dbaace6c252131c00e1de4b165dc65af02ea278476187765e1a617b917", + "https://bcr.bazel.build/modules/bazel_skylib/1.7.0/MODULE.bazel": "0db596f4563de7938de764cc8deeabec291f55e8ec15299718b93c4423e9796d", "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/MODULE.bazel": "3120d80c5861aa616222ec015332e5f8d3171e062e3e804a2a0253e1be26e59b", "https://bcr.bazel.build/modules/bazel_skylib/1.8.0/MODULE.bazel": "2fb3fb53675f6adfc1ca5bfbd5cfb655ae350fba4706d924a8ec7e3ba945671c", "https://bcr.bazel.build/modules/bazel_skylib/1.8.1/MODULE.bazel": "88ade7293becda963e0e3ea33e7d54d3425127e0a326e0d17da085a5f1f03ff6", "https://bcr.bazel.build/modules/bazel_skylib/1.8.2/MODULE.bazel": "69ad6927098316848b34a9142bcc975e018ba27f08c4ff403f50c1b6e646ca67", "https://bcr.bazel.build/modules/bazel_skylib/1.9.0/MODULE.bazel": "72997b29dfd95c3fa0d0c48322d05590418edef451f8db8db5509c57875fb4b7", "https://bcr.bazel.build/modules/bazel_skylib/1.9.0/source.json": "7ad77c1e8c1b84222d9b3f3cae016a76639435744c19330b0b37c0a3c9da7dc0", - "https://bcr.bazel.build/modules/boringssl/0.0.0-20211025-d4f1ab9/MODULE.bazel": "6ee6353f8b1a701fe2178e1d925034294971350b6d3ac37e67e5a7d463267834", + "https://bcr.bazel.build/modules/bazel_worker_api/0.0.8/MODULE.bazel": "396c1ef53835aafe3d42ce6619080531ee770648303731f16cfaa33fa056bf0c", + "https://bcr.bazel.build/modules/bazel_worker_api/0.0.8/source.json": "abaf8ac9d2ab2f47bda9af4c0c080ff7907378888e1f4bc62a0539dd13ba61e8", + "https://bcr.bazel.build/modules/bazel_worker_java/0.0.8/MODULE.bazel": "e76479eae70bd4e8f5f4c2dfc5d03ab971cfb18750246c7b3f3454c5c2ee6629", + "https://bcr.bazel.build/modules/bazel_worker_java/0.0.8/source.json": "9395c4679444bc47bf7e51a710366a4480aa371c6f6bed01868e2fabcf11acec", "https://bcr.bazel.build/modules/boringssl/0.0.0-20230215-5c22014/MODULE.bazel": "4b03dc0d04375fa0271174badcd202ed249870c8e895b26664fd7298abea7282", "https://bcr.bazel.build/modules/boringssl/0.0.0-20240530-2db0eb3/MODULE.bazel": "d0405b762c5e87cd445b7015f2b8da5400ef9a8dbca0bfefa6c1cea79d528a97", "https://bcr.bazel.build/modules/boringssl/0.20240913.0/MODULE.bazel": "fcaa7503a5213290831a91ed1eb538551cf11ac0bc3a6ad92d0fef92c5bd25fb", @@ -95,20 +102,18 @@ "https://bcr.bazel.build/modules/buildifier_prebuilt/8.5.1/source.json": "ae9f3d9dc7bec033976cf47165a78788bebad2b5c272241063ae31bad8664fd4", "https://bcr.bazel.build/modules/buildozer/8.5.1/MODULE.bazel": "a35d9561b3fc5b18797c330793e99e3b834a473d5fbd3d7d7634aafc9bdb6f8f", "https://bcr.bazel.build/modules/buildozer/8.5.1/source.json": "e3386e6ff4529f2442800dee47ad28d3e6487f36a1f75ae39ae56c70f0cd2fbd", - "https://bcr.bazel.build/modules/c-ares/1.15.0/MODULE.bazel": "ba0a78360fdc83f02f437a9e7df0532ad1fbaa59b722f6e715c11effebaa0166", "https://bcr.bazel.build/modules/c-ares/1.19.1/MODULE.bazel": "73bca21720772370ff91cc8e88bbbaf14897720c6473e87c1ddc0f848284c313", "https://bcr.bazel.build/modules/c-ares/1.19.1/source.json": "56bfa95b01e4e0012e90eaa9f1bab823c48c3af4454286b889b9cc74f5098da1", "https://bcr.bazel.build/modules/cel-spec/0.15.0/MODULE.bazel": "e1eed53d233acbdcf024b4b0bc1528116d92c29713251b5154078ab1348cb600", - "https://bcr.bazel.build/modules/cel-spec/0.15.0/source.json": "ab7dccdf21ea2261c0f809b5a5221a4d7f8b580309f285fdf1444baaca75d44a", + "https://bcr.bazel.build/modules/cel-spec/0.24.0/MODULE.bazel": "e310c7aff8490ed689ccafd32729b77a660b9547f5a5ba9b20e967011c324b36", + "https://bcr.bazel.build/modules/cel-spec/0.24.0/source.json": "522d08bc22524e07863276dd0f038f446a83166e91281dcfc07d5b8433c8d89e", "https://bcr.bazel.build/modules/civetweb/1.16/MODULE.bazel": "46a38f9daeb57392e3827fce7d40926be0c802bd23cdd6bfd3a96c804de42fae", "https://bcr.bazel.build/modules/civetweb/1.16/source.json": "ba8b9585adb8355cb51b999d57172fd05e7a762c56b8d4bac6db42c99de3beb7", - "https://bcr.bazel.build/modules/curl/8.4.0/MODULE.bazel": "0bc250aa1cb69590049383df7a9537c809591fcf876c620f5f097c58fdc9bc10", "https://bcr.bazel.build/modules/curl/8.7.1/MODULE.bazel": "088221c35a2939c555e6e47cb31a81c15f8b59f4daa8009b1e9271a502d33485", "https://bcr.bazel.build/modules/curl/8.8.0/MODULE.bazel": "7da3b3e79b0b4ee8f8c95d640bc6ad7b430ce66ef6e9c9d2bc29b3b5ef85f6fe", "https://bcr.bazel.build/modules/curl/8.8.0/source.json": "d7d138b6878cf38891692fee0649ace35357fd549b425614d571786f054374d4", "https://bcr.bazel.build/modules/cython/3.0.11-1/MODULE.bazel": "868b3f5c956c3657420d2302004c6bb92606bfa47e314bab7f2ba0630c7c966c", "https://bcr.bazel.build/modules/cython/3.0.11-1/source.json": "da318be900b8ca9c3d1018839d3bebc5a8e1645620d0848fa2c696d4ecf7c296", - "https://bcr.bazel.build/modules/envoy_api/0.0.0-20241214-918efc9/MODULE.bazel": "24e05f6f52f37be63a795192848555a2c8c855e7814dbc1ed419fb04a7005464", "https://bcr.bazel.build/modules/envoy_api/0.0.0-20250128-4de3c74/MODULE.bazel": "1fe72489212c530086e3ffb0e018b2bfef4663200ca03571570f9f006bef1d75", "https://bcr.bazel.build/modules/envoy_api/0.0.0-20250128-4de3c74/source.json": "028519164a2e24563f4b43d810fdedc702daed90e71e7042d45ba82ad807b46f", "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.1/MODULE.bazel": "cdf8cbe5ee750db04b78878c9633cc76e80dcf4416cbe982ac3a9222f80713c8", @@ -121,36 +126,43 @@ "https://bcr.bazel.build/modules/gazelle/0.34.0/MODULE.bazel": "abdd8ce4d70978933209db92e436deb3a8b737859e9354fb5fd11fb5c2004c8a", "https://bcr.bazel.build/modules/gazelle/0.36.0/MODULE.bazel": "e375d5d6e9a6ca59b0cb38b0540bc9a05b6aa926d322f2de268ad267a2ee74c0", "https://bcr.bazel.build/modules/gazelle/0.37.0/MODULE.bazel": "d1327ba0907d0275ed5103bfbbb13518f6c04955b402213319d0d6c0ce9839d4", - "https://bcr.bazel.build/modules/gazelle/0.37.0/source.json": "b3adc10e2394e7f63ea88fb1d622d4894bfe9ec6961c493ae9a887723ab16831", - "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb", + "https://bcr.bazel.build/modules/gazelle/0.39.1/MODULE.bazel": "1fa3fefad240e535066fd0e6950dfccd627d36dc699ee0034645e51dbde3980f", + "https://bcr.bazel.build/modules/gazelle/0.45.0/MODULE.bazel": "ecd19ebe9f8e024e1ccffb6d997cc893a974bcc581f1ae08f386bdd448b10687", + "https://bcr.bazel.build/modules/gazelle/0.47.0/MODULE.bazel": "b61bb007c4efad134aa30ee7f4a8e2a39b22aa5685f005edaa022fbd1de43ebc", + "https://bcr.bazel.build/modules/gazelle/0.47.0/source.json": "aeb2e5df14b7fb298625d75d08b9c65bdb0b56014c5eb89da9e5dd0572280ae6", "https://bcr.bazel.build/modules/google_benchmark/1.8.4/MODULE.bazel": "c6d54a11dcf64ee63545f42561eda3fd94c1b5f5ebe1357011de63ae33739d5e", - "https://bcr.bazel.build/modules/google_benchmark/1.8.5/MODULE.bazel": "9ba9b31b984022828a950e3300410977eda2e35df35584c6b0b2d0c2e52766b7", - "https://bcr.bazel.build/modules/google_benchmark/1.8.5/source.json": "2c9c685f9b496f125b9e3a9c696c549d1ed2f33b75830a2fb6ac94fab23c0398", + "https://bcr.bazel.build/modules/google_benchmark/1.8.4/source.json": "84590f7bc5a1fd99e1ef274ee16bb41c214f705e62847b42e705010dfa81fe53", + "https://bcr.bazel.build/modules/googleapis-cc/1.0.0/MODULE.bazel": "cf01757e7590c56140a4b81638ff2b3e7074769e6271720bbf738fcda25b6fc2", + "https://bcr.bazel.build/modules/googleapis-cc/1.0.0/source.json": "ab0e3a2ee9968a8848f59872fbbfa3e1f768597d71d2229e6caa319d357967c7", + "https://bcr.bazel.build/modules/googleapis-go/1.0.0/MODULE.bazel": "0a207a4c49da28c5cc1f7b3aeb23c2f7828c85c14aa8d9db0e30357a8d2250ed", + "https://bcr.bazel.build/modules/googleapis-go/1.0.0/source.json": "ef189be4e7853e1ebc6123fe20b71822bf9896bd1f8eed8f68505c4585f72a48", + "https://bcr.bazel.build/modules/googleapis-grpc-cc/1.0.0/MODULE.bazel": "3553358a9d8d96026c9e28d9fb6c268574950d0be7fa9b4c0aeaf3c37c73f2d3", + "https://bcr.bazel.build/modules/googleapis-grpc-cc/1.0.0/source.json": "fa7b79043b3c82bf74f1f2fa45af289e19b247375868d0752db2c114a1c7366c", + "https://bcr.bazel.build/modules/googleapis-java/1.0.0/MODULE.bazel": "d633989337d069b5a95e6101777319681d7a4af4677e36801f11839d6512095c", + "https://bcr.bazel.build/modules/googleapis-java/1.0.0/source.json": "ee59e2de37e4b531172870ac0296afa38f1ea004105ee21b2793c31a9d0ddccd", + "https://bcr.bazel.build/modules/googleapis-python/1.0.0/MODULE.bazel": "0ccd1614a914fb524b3ac267f9c97f9a5cd5412b027f0176b81a725882ec42ff", + "https://bcr.bazel.build/modules/googleapis-python/1.0.0/source.json": "24364f075ec5e6d5e0cfc5d651bd833b119e0cea1a45d51330c93461e3586e42", + "https://bcr.bazel.build/modules/googleapis-rules-registry/1.0.0/MODULE.bazel": "97c6a4d413b373d4cc97065da3de1b2166e22cbbb5f4cc9f05760bfa83619e24", + "https://bcr.bazel.build/modules/googleapis-rules-registry/1.0.0/source.json": "cf611c836a60e98e2e2ab2de8004f119e9f06878dcf4ea2d95a437b1b7a89fe9", "https://bcr.bazel.build/modules/googleapis/0.0.0-20240326-1c8d509c5/MODULE.bazel": "a4b7e46393c1cdcc5a00e6f85524467c48c565256b22b5fae20f84ab4a999a68", "https://bcr.bazel.build/modules/googleapis/0.0.0-20240819-fe8ba054a/MODULE.bazel": "117b7c7be7327ed5d6c482274533f2dbd78631313f607094d4625c28203cacdf", - "https://bcr.bazel.build/modules/googleapis/0.0.0-20240819-fe8ba054a/source.json": "b31fc7eb283a83f71d2e5bfc3d1c562d2994198fa1278409fbe8caec3afc1d3e", + "https://bcr.bazel.build/modules/googleapis/0.0.0-20241220-5e258e33.bcr.1/MODULE.bazel": "ee6c30f82ecd476e61f019fb1151aaab380ea419958ff274ef2f0efca7969f5c", + "https://bcr.bazel.build/modules/googleapis/0.0.0-20251003-2193a2bf/MODULE.bazel": "cc9e5ed294ed9ebf42cdbbdddd2df29048519e3797004df1e3f369f31ff4f2d4", + "https://bcr.bazel.build/modules/googleapis/0.0.0-20251003-2193a2bf/source.json": "21558a194c519e27262cca9cf031bb166a666a8b7fb89993b700c828f8dc0857", + "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6", - "https://bcr.bazel.build/modules/googletest/1.14.0/MODULE.bazel": "cfbcbf3e6eac06ef9d85900f64424708cc08687d1b527f0ef65aa7517af8118f", "https://bcr.bazel.build/modules/googletest/1.15.2/MODULE.bazel": "6de1edc1d26cafb0ea1a6ab3f4d4192d91a312fd2d360b63adaa213cd00b2108", - "https://bcr.bazel.build/modules/googletest/1.16.0/MODULE.bazel": "a175623c69e94fca4ca7acbc12031e637b0c489318cd4805606981d4d7adb34a", + "https://bcr.bazel.build/modules/googletest/1.17.0.bcr.2/MODULE.bazel": "827f54f492a3ce549c940106d73de332c2b30cebd0c20c0bc5d786aba7f116cb", + "https://bcr.bazel.build/modules/googletest/1.17.0.bcr.2/source.json": "3664514073a819992320ffbce5825e4238459df344d8b01748af2208f8d2e1eb", "https://bcr.bazel.build/modules/googletest/1.17.0/MODULE.bazel": "dbec758171594a705933a29fcf69293d2468c49ec1f2ebca65c36f504d72df46", - "https://bcr.bazel.build/modules/googletest/1.17.0/source.json": "38e4454b25fc30f15439c0378e57909ab1fd0a443158aa35aec685da727cd713", - "https://bcr.bazel.build/modules/grpc-java/1.74.0/MODULE.bazel": "6e6b8214d1b77244263e13cf62b71467db8284369ad60567cf6ffcc0650dc621", - "https://bcr.bazel.build/modules/grpc-java/1.74.0/source.json": "340f0cec843668dcdb6b7d1460c8f93937dd915e866c26d20ef1f30ffb4c3125", + "https://bcr.bazel.build/modules/grpc-java/1.62.2/MODULE.bazel": "99b8771e8c7cacb130170fed2a10c9e8fed26334a93e73b42d2953250885a158", + "https://bcr.bazel.build/modules/grpc-java/1.66.0/MODULE.bazel": "86ff26209fac846adb89db11f3714b3dc0090fb2fb81575673cc74880cda4e7e", "https://bcr.bazel.build/modules/grpc-proto/0.0.0-20240627-ec30f58/MODULE.bazel": "88de79051e668a04726e9ea94a481ec6f1692086735fd6f488ab908b3b909238", - "https://bcr.bazel.build/modules/grpc-proto/0.0.0-20240627-ec30f58/source.json": "5035d379c61042930244ab59e750106d893ec440add92ec0df6a0098ca7f131d", - "https://bcr.bazel.build/modules/grpc/1.41.0/MODULE.bazel": "5bcbfc2b274dabea628f0649dc50c90cf36543b1cfc31624832538644ad1aae8", - "https://bcr.bazel.build/modules/grpc/1.56.3.bcr.1/MODULE.bazel": "cd5b1eb276b806ec5ab85032921f24acc51735a69ace781be586880af20ab33f", - "https://bcr.bazel.build/modules/grpc/1.62.1/MODULE.bazel": "2998211594b8a79a6b459c4e797cfa19f0fb8b3be3149760ec7b8c99abfd426f", - "https://bcr.bazel.build/modules/grpc/1.63.1.bcr.1/MODULE.bazel": "d7b9fef03bd175e6825237b521b18a3c29f1ac15f8aa52c8a1a0f3bd8f33d54b", - "https://bcr.bazel.build/modules/grpc/1.66.0.bcr.2/MODULE.bazel": "0fa2b0fd028ce354febf0fe90f1ed8fecfbfc33118cddd95ac0418cc283333a0", - "https://bcr.bazel.build/modules/grpc/1.66.0.bcr.3/MODULE.bazel": "f6047e89faf488f5e3e65cb2594c6f5e86992abec7487163ff6b623526e543b0", - "https://bcr.bazel.build/modules/grpc/1.69.0/MODULE.bazel": "4e26e05c9e1ef291ccbc96aad8e457b1b8abedbc141623831629da2f8168eef6", - "https://bcr.bazel.build/modules/grpc/1.71.0/MODULE.bazel": "7fcab2c05530373f1a442c362b17740dd0c75b6a2a975eec8f5bf4c70a37928a", - "https://bcr.bazel.build/modules/grpc/1.74.0/MODULE.bazel": "83bca3739c1a3eba91f63f38b352e8681c9e58576f2def545e49a1e461b98a1f", - "https://bcr.bazel.build/modules/grpc/1.74.0/source.json": "87c009e2f3226fd43170ce4896a3f17c819bd4c7388ff1336158b12b636a0ed9", + "https://bcr.bazel.build/modules/grpc/1.80.0/MODULE.bazel": "35ef253a0a31055c8a2495640516224647545b51673b0ccb956b116ee92ef554", + "https://bcr.bazel.build/modules/grpc/1.80.0/source.json": "55bf4cdcf278a450185ff93bef843a0d7db0a83d0b9761d9c1dee91faa9cdf46", "https://bcr.bazel.build/modules/jq.bzl/0.1.0/MODULE.bazel": "2ce69b1af49952cd4121a9c3055faa679e748ce774c7f1fda9657f936cae902f", "https://bcr.bazel.build/modules/jq.bzl/0.1.0/source.json": "746bf13cac0860f091df5e4911d0c593971cd8796b5ad4e809b2f8e133eee3d5", + "https://bcr.bazel.build/modules/jsoncpp/1.9.5/MODULE.bazel": "31271aedc59e815656f5736f282bb7509a97c7ecb43e927ac1a37966e0578075", "https://bcr.bazel.build/modules/jsoncpp/1.9.6.bcr.2/MODULE.bazel": "64f508885f907ac2518039b3d0c5c1703a8f6137d9f5d635067ba93d33c2670a", "https://bcr.bazel.build/modules/jsoncpp/1.9.6.bcr.2/source.json": "13199fa0a267ca46814e9e6ab313a71890c8f1822e57376fdc23a2d2cd384051", "https://bcr.bazel.build/modules/jsoncpp/1.9.6/MODULE.bazel": "2f8d20d3b7d54143213c4dfc3d98225c42de7d666011528dc8fe91591e2e17b0", @@ -167,18 +179,13 @@ "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", "https://bcr.bazel.build/modules/opencensus-cpp/0.0.0-20230502-50eb5de.bcr.2/MODULE.bazel": "cc18734138dd18c912c6ce2a59186db28f85d8058c99c9f21b46ca3e0aba0ebe", "https://bcr.bazel.build/modules/opencensus-cpp/0.0.0-20230502-50eb5de.bcr.2/source.json": "7c135f9d42bb3b045669c3c6ab3bb3c208e00b46aca4422eea64c29811a5b240", - "https://bcr.bazel.build/modules/opencensus-cpp/0.0.0-20230502-50eb5de/MODULE.bazel": "02201d2921dadb4ec90c4980eca4b2a02904eddcf6fa02f3da7594fb7b0d821c", "https://bcr.bazel.build/modules/opencensus-proto/0.4.1.bcr.2/MODULE.bazel": "789706a714855f92c5c8cfcf1ef32bbb64dcd3b7c9066756ad7986ec59709d29", "https://bcr.bazel.build/modules/opencensus-proto/0.4.1.bcr.2/source.json": "aadf3f53e08b72376506b7c4ea3d167010c9efb160d7d6e1e304ed646bac1b36", "https://bcr.bazel.build/modules/opencensus-proto/0.4.1/MODULE.bazel": "4a2e8b4d0b544002502474d611a5a183aa282251e14f6a01afe841c0c1b10372", "https://bcr.bazel.build/modules/openssl/3.3.1.bcr.1/MODULE.bazel": "49c0c07e8fb87b480bccb842cfee1b32617f11dac590f732573c69058699a3d1", "https://bcr.bazel.build/modules/openssl/3.3.1.bcr.1/source.json": "0c0872e048bbea052a9c541fb47019481a19201ba5555a71d762ad591bf94e1f", - "https://bcr.bazel.build/modules/opentelemetry-cpp/1.14.2/MODULE.bazel": "089a5613c2a159c7dfde098dabfc61e966889c7d6a81a98422a84c51535ed17d", - "https://bcr.bazel.build/modules/opentelemetry-cpp/1.16.0/MODULE.bazel": "b7379a140f538cea3f749179a2d481ed81942cc6f7b05a6113723eb34ac3b3e7", "https://bcr.bazel.build/modules/opentelemetry-cpp/1.19.0/MODULE.bazel": "3455326c08b28415648a3d60d8e3c811847ebdbe64474f75b25878f25585aea1", "https://bcr.bazel.build/modules/opentelemetry-cpp/1.19.0/source.json": "4e48137e4c3ecb99401ff99876df8fa330598d7da051869bec643446e8a8ff95", - "https://bcr.bazel.build/modules/opentelemetry-proto/1.1.0/MODULE.bazel": "a49f406e99bf05ab43ed4f5b3322fbd33adfd484b6546948929d1316299b68bf", - "https://bcr.bazel.build/modules/opentelemetry-proto/1.3.1/MODULE.bazel": "0141a50e989576ee064c11ce8dd5ec89993525bd9f9a09c5618e4dacc8df9352", "https://bcr.bazel.build/modules/opentelemetry-proto/1.4.0.bcr.1/MODULE.bazel": "5ceaf25e11170d22eded4c8032728b4a3f273765fccda32f9e94f463755c4167", "https://bcr.bazel.build/modules/opentelemetry-proto/1.5.0/MODULE.bazel": "7543d91a53b98e7b5b37c5a0865b93bff12c1ee022b1e322cd236b968894b030", "https://bcr.bazel.build/modules/opentelemetry-proto/1.5.0/source.json": "046b721ce203e88cdaad44d7dd17a86b7200eab9388b663b234e72e13ff7b143", @@ -186,7 +193,8 @@ "https://bcr.bazel.build/modules/opentracing-cpp/1.6.0/source.json": "da1cb1add160f5e5074b7272e9db6fd8f1b3336c15032cd0a653af9d2f484aed", "https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92", "https://bcr.bazel.build/modules/package_metadata/0.0.3/MODULE.bazel": "77890552ecea9e284b5424c9de827a58099348763a4359e975c359a83d4faa83", - "https://bcr.bazel.build/modules/package_metadata/0.0.3/source.json": "742075a428ad12a3fa18a69014c2f57f01af910c6d9d18646c990200853e641a", + "https://bcr.bazel.build/modules/package_metadata/0.0.5/MODULE.bazel": "ef4f9439e3270fdd6b9fd4dbc3d2f29d13888e44c529a1b243f7a31dfbc2e8e4", + "https://bcr.bazel.build/modules/package_metadata/0.0.5/source.json": "2326db2f6592578177751c3e1f74786b79382cd6008834c9d01ec865b9126a85", "https://bcr.bazel.build/modules/patchelf/0.18.0.bcr.1/MODULE.bazel": "35320668481400503dea7c75313473b9ffd6dea4e7ba583b96a896199b0088d5", "https://bcr.bazel.build/modules/patchelf/0.18.0.bcr.1/source.json": "d877d271ef5d531407885322054c99c7f0617a9a85ff99c0a70f4ced7f13e13a", "https://bcr.bazel.build/modules/pcre2/10.45/MODULE.bazel": "9bdc838aa2e21be030d7181385e45a5c87b14575bb7ff35d10b5be3d56913bed", @@ -202,17 +210,37 @@ "https://bcr.bazel.build/modules/platforms/1.0.0/MODULE.bazel": "f05feb42b48f1b3c225e4ccf351f367be0371411a803198ec34a389fb22aa580", "https://bcr.bazel.build/modules/platforms/1.1.0/MODULE.bazel": "1c0c09f5bdcf4b3f924720d2478a3711cb39f4977019ca5988685e5b7e18b3d2", "https://bcr.bazel.build/modules/platforms/1.1.0/source.json": "fcf351c47596c939140ab0d333dfdd08ed1ea6ce33c2fe70c12493a301cf1344", - "https://bcr.bazel.build/modules/prometheus-cpp/1.2.4/MODULE.bazel": "0fbe5dcff66311947a3f6b86ebc6a6d9328e31a28413ca864debc4a043f371e5", "https://bcr.bazel.build/modules/prometheus-cpp/1.3.0.bcr.1/MODULE.bazel": "116ad46e97c1d2aeb020fe2899a342a7e703574ce7c0faf7e4810f938c974a9a", "https://bcr.bazel.build/modules/prometheus-cpp/1.3.0.bcr.1/source.json": "e813cce2d450708cfcb26e309c5172583a7440776edf354e83e6788c768e5cca", "https://bcr.bazel.build/modules/prometheus-cpp/1.3.0/MODULE.bazel": "ce82e086bbc0b60267e970f6a54b2ca6d0f22d3eb6633e00e2cc2899c700f3d8", - "https://bcr.bazel.build/modules/protobuf/33.6/MODULE.bazel": "af76890dbc467c56f592796b66f8f507013b34190f4b569a1b69318ea9a26c8b", - "https://bcr.bazel.build/modules/protobuf/33.6/source.json": "8921cf8549b5648ff3aaaee6c89160a23e4794528f45886b6743381127731f46", + "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7", + "https://bcr.bazel.build/modules/protobuf/23.1/MODULE.bazel": "88b393b3eb4101d18129e5db51847cd40a5517a53e81216144a8c32dfeeca52a", + "https://bcr.bazel.build/modules/protobuf/24.4/MODULE.bazel": "7bc7ce5f2abf36b3b7b7c8218d3acdebb9426aeb35c2257c96445756f970eb12", + "https://bcr.bazel.build/modules/protobuf/25.6/MODULE.bazel": "fc0ae073b47c7ede88b825ff79e64f1c058967c7a87a86cdf4abecd9e0516625", + "https://bcr.bazel.build/modules/protobuf/26.0/MODULE.bazel": "8402da964092af40097f4a205eec2a33fd4a7748dc43632b7d1629bfd9a2b856", + "https://bcr.bazel.build/modules/protobuf/27.0-rc2/MODULE.bazel": "b2b0dbafd57b6bec0ca9b251da02e628c357dab53a097570aa7d79d020f107cf", + "https://bcr.bazel.build/modules/protobuf/27.0/MODULE.bazel": "7873b60be88844a0a1d8f80b9d5d20cfbd8495a689b8763e76c6372998d3f64c", + "https://bcr.bazel.build/modules/protobuf/27.1/MODULE.bazel": "703a7b614728bb06647f965264967a8ef1c39e09e8f167b3ca0bb1fd80449c0d", + "https://bcr.bazel.build/modules/protobuf/27.2/MODULE.bazel": "32450b50673882e4c8c3d10a83f3bc82161b213ed2f80d17e38bece8f165c295", + "https://bcr.bazel.build/modules/protobuf/29.0-rc2.bcr.1/MODULE.bazel": "52f4126f63a2f0bbf36b99c2a87648f08467a4eaf92ba726bc7d6a500bbf770c", + "https://bcr.bazel.build/modules/protobuf/29.0-rc2/MODULE.bazel": "6241d35983510143049943fc0d57937937122baf1b287862f9dc8590fc4c37df", + "https://bcr.bazel.build/modules/protobuf/29.0-rc3/MODULE.bazel": "33c2dfa286578573afc55a7acaea3cada4122b9631007c594bf0729f41c8de92", + "https://bcr.bazel.build/modules/protobuf/29.0/MODULE.bazel": "319dc8bf4c679ff87e71b1ccfb5a6e90a6dbc4693501d471f48662ac46d04e4e", + "https://bcr.bazel.build/modules/protobuf/29.1/MODULE.bazel": "557c3457560ff49e122ed76c0bc3397a64af9574691cb8201b4e46d4ab2ecb95", + "https://bcr.bazel.build/modules/protobuf/29.3/MODULE.bazel": "77480eea5fb5541903e49683f24dc3e09f4a79e0eea247414887bb9fc0066e94", + "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0", + "https://bcr.bazel.build/modules/protobuf/3.19.2/MODULE.bazel": "532ffe5f2186b69fdde039efe6df13ba726ff338c6bc82275ad433013fa10573", + "https://bcr.bazel.build/modules/protobuf/3.19.6/MODULE.bazel": "9233edc5e1f2ee276a60de3eaa47ac4132302ef9643238f23128fea53ea12858", + "https://bcr.bazel.build/modules/protobuf/30.0/MODULE.bazel": "0e736de5d52ad7824113f47e65256a26ee74b689ba859c5447a0663e5a075409", + "https://bcr.bazel.build/modules/protobuf/31.1/MODULE.bazel": "379a389bb330b7b8c1cdf331cc90bf3e13de5614799b3b52cdb7c6f389f6b38e", + "https://bcr.bazel.build/modules/protobuf/32.1/MODULE.bazel": "89cd2866a9cb07fee9ff74c41ceace11554f32e0d849de4e23ac55515cfada4d", + "https://bcr.bazel.build/modules/protobuf/33.4/MODULE.bazel": "114775b816b38b6d0ca620450d6b02550c60ceedfdc8d9a229833b34a223dc42", + "https://bcr.bazel.build/modules/protobuf/35.0/MODULE.bazel": "ee97170c013d94c366bbb5fe8a97b0650477cbc00e5e0f6a2c297484bc34d81b", + "https://bcr.bazel.build/modules/protobuf/35.0/source.json": "2162e80adef862606ba85ae42e0df85eb519376fd6ab49caddee6fb776a1281e", "https://bcr.bazel.build/modules/protoc-gen-validate/1.0.4.bcr.2/MODULE.bazel": "c4bd2c850211ff5b7dadf9d2d0496c1c922fdedc303c775b01dfd3b3efc907ed", "https://bcr.bazel.build/modules/protoc-gen-validate/1.0.4/MODULE.bazel": "b8913c154b16177990f6126d2d2477d187f9ddc568e95ee3e2d50fc65d2c494a", "https://bcr.bazel.build/modules/protoc-gen-validate/1.2.1.bcr.1/MODULE.bazel": "4bf09676b62fa587ae07e073420a76ec8766dcce7545e5f8c68cfa8e484b5120", "https://bcr.bazel.build/modules/protoc-gen-validate/1.2.1.bcr.1/source.json": "c19071ebc4b53b5f1cfab9c66eefaf6e4179eb8a998970d07b1077687e777f29", - "https://bcr.bazel.build/modules/protoc-gen-validate/1.2.1/MODULE.bazel": "52b51f50533ec4fbd5d613cd093773f979ac2e035d954e02ca11de383f502505", "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e", "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/MODULE.bazel": "e6f4c20442eaa7c90d7190d8dc539d0ab422f95c65a57cc59562170c58ae3d34", "https://bcr.bazel.build/modules/pybind11_bazel/2.13.6/MODULE.bazel": "2d746fda559464b253b2b2e6073cb51643a2ac79009ca02100ebbc44b4548656", @@ -225,14 +253,17 @@ "https://bcr.bazel.build/modules/re2/2024-05-01/MODULE.bazel": "55a3f059538f381107824e7d00df5df6d061ba1fb80e874e4909c0f0549e8f3e", "https://bcr.bazel.build/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "b4963dda9b31080be1905ef085ecd7dd6cd47c05c79b9cdf83ade83ab2ab271a", "https://bcr.bazel.build/modules/re2/2024-07-02/MODULE.bazel": "0eadc4395959969297cbcf31a249ff457f2f1d456228c67719480205aa306daa", + "https://bcr.bazel.build/modules/re2/2025-08-12.bcr.1/MODULE.bazel": "e09b434b122bfb786a69179f9b325e35cb1856c3f56a7a81dd61609260ed46e1", "https://bcr.bazel.build/modules/re2/2025-11-05/MODULE.bazel": "d97e240364f132017500a3af816493af7efd4be40aab2bb403ed449cc96f70a3", "https://bcr.bazel.build/modules/re2/2025-11-05/source.json": "7df1394aabda1c9bc188a302f5d54b1c657924edd04ebc57d2be29dbd7efd141", "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", - "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", + "https://bcr.bazel.build/modules/rules_android/0.7.1/MODULE.bazel": "a806fc382a774252f228a40e3b11b9fcc6276f8778c7fb33e9f72937c6258363", + "https://bcr.bazel.build/modules/rules_android/0.7.1/source.json": "151440aed3f0f73a00d4ed5cec5d31f63a6fef9b95d8fab1eb1810150fa525f2", + "https://bcr.bazel.build/modules/rules_apple/3.13.0/MODULE.bazel": "b4559a2c6281ca3165275bb36c1f0ac74666632adc5bdb680e366de7ce845f43", "https://bcr.bazel.build/modules/rules_apple/3.16.0/MODULE.bazel": "0d1caf0b8375942ce98ea944be754a18874041e4e0459401d925577624d3a54a", - "https://bcr.bazel.build/modules/rules_apple/3.5.1/MODULE.bazel": "3d1bbf65ad3692003d36d8a29eff54d4e5c1c5f4bfb60f79e28646a924d9101c", "https://bcr.bazel.build/modules/rules_apple/4.1.0/MODULE.bazel": "76e10fd4a48038d3fc7c5dc6e63b7063bbf5304a2e3bd42edda6ec660eebea68", "https://bcr.bazel.build/modules/rules_apple/4.1.0/source.json": "8ee81e1708756f81b343a5eb2b2f0b953f1d25c4ab3d4a68dc02754872e80715", + "https://bcr.bazel.build/modules/rules_buf/0.1.1/MODULE.bazel": "6189aec18a4f7caff599ad41b851ab7645d4f1e114aa6431acf9b0666eb92162", "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", "https://bcr.bazel.build/modules/rules_cc/0.0.10/MODULE.bazel": "ec1705118f7eaedd6e118508d3d26deba2a4e76476ada7e0e3965211be012002", "https://bcr.bazel.build/modules/rules_cc/0.0.13/MODULE.bazel": "0e8529ed7b323dad0775ff924d2ae5af7640b23553dfcd4d34344c7e7a867191", @@ -242,15 +273,14 @@ "https://bcr.bazel.build/modules/rules_cc/0.0.17/MODULE.bazel": "2ae1d8f4238ec67d7185d8861cb0a2cdf4bc608697c331b95bf990e69b62e64a", "https://bcr.bazel.build/modules/rules_cc/0.0.2/MODULE.bazel": "6915987c90970493ab97393024c156ea8fb9f3bea953b2f3ec05c34f19b5695c", "https://bcr.bazel.build/modules/rules_cc/0.0.5/MODULE.bazel": "be41f87587998fe8890cd82ea4e848ed8eb799e053c224f78f3ff7fe1a1d9b74", - "https://bcr.bazel.build/modules/rules_cc/0.0.6/MODULE.bazel": "abf360251023dfe3efcef65ab9d56beefa8394d4176dd29529750e1c57eaa33f", "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", "https://bcr.bazel.build/modules/rules_cc/0.1.1/MODULE.bazel": "2f0222a6f229f0bf44cd711dc13c858dad98c62d52bd51d8fc3a764a83125513", "https://bcr.bazel.build/modules/rules_cc/0.1.2/MODULE.bazel": "557ddc3a96858ec0d465a87c0a931054d7dcfd6583af2c7ed3baf494407fd8d0", "https://bcr.bazel.build/modules/rules_cc/0.1.4/MODULE.bazel": "bb03a452a7527ac25a7518fb86a946ef63df860b9657d8323a0c50f8504fb0b9", "https://bcr.bazel.build/modules/rules_cc/0.1.5/MODULE.bazel": "88dfc9361e8b5ae1008ac38f7cdfd45ad738e4fa676a3ad67d19204f045a1fd8", - "https://bcr.bazel.build/modules/rules_cc/0.2.0/MODULE.bazel": "b5c17f90458caae90d2ccd114c81970062946f49f355610ed89bebf954f5783c", "https://bcr.bazel.build/modules/rules_cc/0.2.13/MODULE.bazel": "eecdd666eda6be16a8d9dc15e44b5c75133405e820f620a234acc4b1fdc5aa37", + "https://bcr.bazel.build/modules/rules_cc/0.2.14/MODULE.bazel": "353c99ed148887ee89c54a17d4100ae7e7e436593d104b668476019023b58df8", "https://bcr.bazel.build/modules/rules_cc/0.2.15/MODULE.bazel": "6a0a4a75a57aa6dc888300d848053a58c6b12a29f89d4304e1c41448514ec6e8", "https://bcr.bazel.build/modules/rules_cc/0.2.16/MODULE.bazel": "9242fa89f950c6ef7702801ab53922e99c69b02310c39fb6e62b2bd30df2a1d4", "https://bcr.bazel.build/modules/rules_cc/0.2.17/MODULE.bazel": "1849602c86cb60da8613d2de887f9566a6d354a6df6d7009f9d04a14402f9a84", @@ -265,6 +295,7 @@ "https://bcr.bazel.build/modules/rules_foreign_cc/0.15.1/MODULE.bazel": "c2c60d26c79fda484acb95cdbec46e89d6b28b4845cb277160ce1e0c8622bb88", "https://bcr.bazel.build/modules/rules_foreign_cc/0.15.1/source.json": "a161811a63ba8a859086da3b7ff3ad04f2e9c255d7727b41087103fc0eb22f55", "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", + "https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/MODULE.bazel": "40c97d1144356f52905566c55811f13b299453a14ac7769dfba2ac38192337a8", "https://bcr.bazel.build/modules/rules_go/0.33.0/MODULE.bazel": "a2b11b64cd24bf94f57454f53288a5dacfe6cb86453eee7761b7637728c1910c", "https://bcr.bazel.build/modules/rules_go/0.38.1/MODULE.bazel": "fb8e73dd3b6fc4ff9d260ceacd830114891d49904f5bda1c16bc147bcc254f71", "https://bcr.bazel.build/modules/rules_go/0.39.1/MODULE.bazel": "d34fb2a249403a5f4339c754f1e63dc9e5ad70b47c5e97faee1441fc6636cd61", @@ -273,42 +304,57 @@ "https://bcr.bazel.build/modules/rules_go/0.45.1/MODULE.bazel": "6d7884f0edf890024eba8ab31a621faa98714df0ec9d512389519f0edff0281a", "https://bcr.bazel.build/modules/rules_go/0.46.0/MODULE.bazel": "3477df8bdcc49e698b9d25f734c4f3a9f5931ff34ee48a2c662be168f5f2d3fd", "https://bcr.bazel.build/modules/rules_go/0.48.0/MODULE.bazel": "d00ebcae0908ee3f5e6d53f68677a303d6d59a77beef879598700049c3980a03", + "https://bcr.bazel.build/modules/rules_go/0.49.0/MODULE.bazel": "61cfc1ba17123356d1b12b6c50f6e0162b2cc7fd6f51753c12471e973a0f72a5", "https://bcr.bazel.build/modules/rules_go/0.50.1/MODULE.bazel": "b91a308dc5782bb0a8021ad4330c81fea5bda77f96b9e4c117b9b9c8f6665ee0", - "https://bcr.bazel.build/modules/rules_go/0.60.0/MODULE.bazel": "4a57ff2ffc2a3570e3c5646575c5a4b07287e91bcdac5d1f72383d51502b48cb", - "https://bcr.bazel.build/modules/rules_go/0.60.0/source.json": "1e21368c5e0c3013a110bd79a8fcff8ca46b5bcb2b561713a7273cbfcff7c464", + "https://bcr.bazel.build/modules/rules_go/0.53.0/MODULE.bazel": "a4ed760d3ac0dbc0d7b967631a9a3fd9100d28f7d9fcf214b4df87d4bfff5f9a", + "https://bcr.bazel.build/modules/rules_go/0.59.0/MODULE.bazel": "b7e43e7414a3139a7547d1b4909b29085fbe5182b6c58cbe1ed4c6272815aeae", + "https://bcr.bazel.build/modules/rules_go/0.59.0/source.json": "1df17bb7865cfc029492c30163cee891d0dd8658ea0d5bfdf252c4b6db5c1ef6", "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", - "https://bcr.bazel.build/modules/rules_java/5.1.0/MODULE.bazel": "324b6478b0343a3ce7a9add8586ad75d24076d6d43d2f622990b9c1cfd8a1b15", "https://bcr.bazel.build/modules/rules_java/5.3.5/MODULE.bazel": "a4ec4f2db570171e3e5eb753276ee4b389bae16b96207e9d3230895c99644b86", "https://bcr.bazel.build/modules/rules_java/5.5.0/MODULE.bazel": "486ad1aa15cdc881af632b4b1448b0136c76025a1fe1ad1b65c5899376b83a50", "https://bcr.bazel.build/modules/rules_java/6.0.0/MODULE.bazel": "8a43b7df601a7ec1af61d79345c17b31ea1fedc6711fd4abfd013ea612978e39", "https://bcr.bazel.build/modules/rules_java/6.3.0/MODULE.bazel": "a97c7678c19f236a956ad260d59c86e10a463badb7eb2eda787490f4c969b963", "https://bcr.bazel.build/modules/rules_java/6.4.0/MODULE.bazel": "e986a9fe25aeaa84ac17ca093ef13a4637f6107375f64667a15999f77db6c8f6", + "https://bcr.bazel.build/modules/rules_java/6.5.2/MODULE.bazel": "1d440d262d0e08453fa0c4d8f699ba81609ed0e9a9a0f02cd10b3e7942e61e31", + "https://bcr.bazel.build/modules/rules_java/7.1.0/MODULE.bazel": "30d9135a2b6561c761bd67bd4990da591e6bdc128790ce3e7afd6a3558b2fb64", "https://bcr.bazel.build/modules/rules_java/7.10.0/MODULE.bazel": "530c3beb3067e870561739f1144329a21c851ff771cd752a49e06e3dc9c2e71a", "https://bcr.bazel.build/modules/rules_java/7.12.2/MODULE.bazel": "579c505165ee757a4280ef83cda0150eea193eed3bef50b1004ba88b99da6de6", "https://bcr.bazel.build/modules/rules_java/7.2.0/MODULE.bazel": "06c0334c9be61e6cef2c8c84a7800cef502063269a5af25ceb100b192453d4ab", "https://bcr.bazel.build/modules/rules_java/7.3.2/MODULE.bazel": "50dece891cfdf1741ea230d001aa9c14398062f2b7c066470accace78e412bc2", "https://bcr.bazel.build/modules/rules_java/7.4.0/MODULE.bazel": "a592852f8a3dd539e82ee6542013bf2cadfc4c6946be8941e189d224500a8934", "https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe", + "https://bcr.bazel.build/modules/rules_java/7.6.5/MODULE.bazel": "481164be5e02e4cab6e77a36927683263be56b7e36fef918b458d7a8a1ebadb1", + "https://bcr.bazel.build/modules/rules_java/8.3.2/MODULE.bazel": "7336d5511ad5af0b8615fdc7477535a2e4e723a357b6713af439fe8cf0195017", "https://bcr.bazel.build/modules/rules_java/8.5.1/MODULE.bazel": "d8a9e38cc5228881f7055a6079f6f7821a073df3744d441978e7a43e20226939", "https://bcr.bazel.build/modules/rules_java/8.6.0/MODULE.bazel": "9c064c434606d75a086f15ade5edb514308cccd1544c2b2a89bbac4310e41c71", "https://bcr.bazel.build/modules/rules_java/8.6.1/MODULE.bazel": "f4808e2ab5b0197f094cabce9f4b006a27766beb6a9975931da07099560ca9c2", "https://bcr.bazel.build/modules/rules_java/8.8.0/MODULE.bazel": "de589d0880911ac007abd521b9f0ddcd8b0dbd05c8553e6f8124a050b83acf7d", + "https://bcr.bazel.build/modules/rules_java/8.9.0/MODULE.bazel": "e17c876cb53dcd817b7b7f0d2985b710610169729e8c371b2221cacdcd3dce4a", "https://bcr.bazel.build/modules/rules_java/9.1.0/MODULE.bazel": "ee63f27e36a3fada80342869361182f120a9819c74320e8e65b1e04ba0cd7a9d", - "https://bcr.bazel.build/modules/rules_java/9.1.0/source.json": "da589573c1dee2c9ac4a568b301269a2e8191110ff0345c1a959fa7ea6c4dfd6", + "https://bcr.bazel.build/modules/rules_java/9.3.0/MODULE.bazel": "f657c72d65ac449caae9abf2e68e66c0d36f9416848c4c4903d0b3234229e7f2", + "https://bcr.bazel.build/modules/rules_java/9.3.0/source.json": "59ae7e662c3c7042b88bbb42ad12483523e234c65ebe4c51611baa43e85cb248", + "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7", + "https://bcr.bazel.build/modules/rules_jvm_external/5.1/MODULE.bazel": "33f6f999e03183f7d088c9be518a63467dfd0be94a11d0055fe2d210f89aa909", "https://bcr.bazel.build/modules/rules_jvm_external/5.2/MODULE.bazel": "d9351ba35217ad0de03816ef3ed63f89d411349353077348a45348b096615036", "https://bcr.bazel.build/modules/rules_jvm_external/5.3/MODULE.bazel": "bf93870767689637164657731849fb887ad086739bd5d360d90007a581d5527d", "https://bcr.bazel.build/modules/rules_jvm_external/6.0/MODULE.bazel": "37c93a5a78d32e895d52f86a8d0416176e915daabd029ccb5594db422e87c495", "https://bcr.bazel.build/modules/rules_jvm_external/6.1/MODULE.bazel": "75b5fec090dbd46cf9b7d8ea08cf84a0472d92ba3585b476f44c326eda8059c4", + "https://bcr.bazel.build/modules/rules_jvm_external/6.2/MODULE.bazel": "36a6e52487a855f33cb960724eb56547fa87e2c98a0474c3acad94339d7f8e99", "https://bcr.bazel.build/modules/rules_jvm_external/6.3/MODULE.bazel": "c998e060b85f71e00de5ec552019347c8bca255062c990ac02d051bb80a38df0", "https://bcr.bazel.build/modules/rules_jvm_external/6.7/MODULE.bazel": "e717beabc4d091ecb2c803c2d341b88590e9116b8bf7947915eeb33aab4f96dd", - "https://bcr.bazel.build/modules/rules_jvm_external/6.7/source.json": "5426f412d0a7fc6b611643376c7e4a82dec991491b9ce5cb1cfdd25fe2e92be4", + "https://bcr.bazel.build/modules/rules_jvm_external/6.9/MODULE.bazel": "07c5db05527db7744a54fcffd653e1550d40e0540207a7f7e6d0a4de5bef8274", + "https://bcr.bazel.build/modules/rules_jvm_external/6.9/source.json": "b12970214f3cc144b26610caeb101fa622d910f1ab3d98f0bae1058edbd00bd4", "https://bcr.bazel.build/modules/rules_kotlin/1.9.0/MODULE.bazel": "ef85697305025e5a61f395d4eaede272a5393cee479ace6686dba707de804d59", + "https://bcr.bazel.build/modules/rules_kotlin/1.9.5/MODULE.bazel": "043a16a572f610558ec2030db3ff0c9938574e7dd9f58bded1bb07c0192ef025", "https://bcr.bazel.build/modules/rules_kotlin/1.9.6/MODULE.bazel": "d269a01a18ee74d0335450b10f62c9ed81f2321d7958a2934e44272fe82dcef3", - "https://bcr.bazel.build/modules/rules_kotlin/1.9.6/source.json": "2faa4794364282db7c06600b7e5e34867a564ae91bda7cae7c29c64e9466b7d5", + "https://bcr.bazel.build/modules/rules_kotlin/2.3.20/MODULE.bazel": "3443d53d275e14fecfebd0b491f01d06ea3883c04a1b3336e7ae9d5ec9066bef", + "https://bcr.bazel.build/modules/rules_kotlin/2.3.20/source.json": "5a5553cffea43f2c5156c8ad0de4a14ad95413ceb39cd4d08f50e2aea86927e8", "https://bcr.bazel.build/modules/rules_license/0.0.3/MODULE.bazel": "627e9ab0247f7d1e05736b59dbb1b6871373de5ad31c3011880b4133cafd4bd0", "https://bcr.bazel.build/modules/rules_license/0.0.7/MODULE.bazel": "088fbeb0b6a419005b89cf93fe62d9517c0a2b8bb56af3244af65ecfe37e7d5d", + "https://bcr.bazel.build/modules/rules_license/0.0.8/MODULE.bazel": "5669c6fe49b5134dbf534db681ad3d67a2d49cfc197e4a95f1ca2fd7f3aebe96", "https://bcr.bazel.build/modules/rules_license/1.0.0/MODULE.bazel": "a7fda60eefdf3d8c827262ba499957e4df06f659330bbe6cdbdb975b768bb65c", "https://bcr.bazel.build/modules/rules_license/1.0.0/source.json": "a52c89e54cc311196e478f8382df91c15f7a2bfdf4c6cd0e2675cc2ff0b56efb", + "https://bcr.bazel.build/modules/rules_nodejs/5.8.2/MODULE.bazel": "6bc03c8f37f69401b888023bf511cb6ee4781433b0cb56236b2e55a21e3a026a", "https://bcr.bazel.build/modules/rules_oci/2.2.7/MODULE.bazel": "f6150e4b224d459f7f6523ef65967464ca4efdd266c7fbf2f5a2a51011957e0c", "https://bcr.bazel.build/modules/rules_oci/2.2.7/source.json": "b099f02af330f47f19dc67fc9300ef6e1937a8c86882690db0e7a2fcea8c7f6b", "https://bcr.bazel.build/modules/rules_perl/0.2.4/MODULE.bazel": "5f5af7be4bf5fb88d91af7469518f0fd2161718aefc606188f7cd51f436ca938", @@ -324,40 +370,52 @@ "https://bcr.bazel.build/modules/rules_proto/7.0.2/MODULE.bazel": "bf81793bd6d2ad89a37a40693e56c61b0ee30f7a7fdbaf3eabbf5f39de47dea2", "https://bcr.bazel.build/modules/rules_proto/7.1.0/MODULE.bazel": "002d62d9108f75bb807cd56245d45648f38275cb3a99dcd45dfb864c5d74cb96", "https://bcr.bazel.build/modules/rules_proto/7.1.0/source.json": "39f89066c12c24097854e8f57ab8558929f9c8d474d34b2c00ac04630ad8940e", + "https://bcr.bazel.build/modules/rules_python/0.10.2/MODULE.bazel": "cc82bc96f2997baa545ab3ce73f196d040ffb8756fd2d66125a530031cd90e5f", "https://bcr.bazel.build/modules/rules_python/0.20.0/MODULE.bazel": "bfe14d17f20e3fe900b9588f526f52c967a6f281e47a1d6b988679bd15082286", "https://bcr.bazel.build/modules/rules_python/0.22.0/MODULE.bazel": "b8057bafa11a9e0f4b08fc3b7cd7bee0dcbccea209ac6fc9a3ff051cd03e19e9", "https://bcr.bazel.build/modules/rules_python/0.22.1/MODULE.bazel": "26114f0c0b5e93018c0c066d6673f1a2c3737c7e90af95eff30cfee38d0bbac7", "https://bcr.bazel.build/modules/rules_python/0.23.1/MODULE.bazel": "49ffccf0511cb8414de28321f5fcf2a31312b47c40cc21577144b7447f2bf300", "https://bcr.bazel.build/modules/rules_python/0.25.0/MODULE.bazel": "72f1506841c920a1afec76975b35312410eea3aa7b63267436bfb1dd91d2d382", - "https://bcr.bazel.build/modules/rules_python/0.29.0/MODULE.bazel": "2ac8cd70524b4b9ec49a0b8284c79e4cd86199296f82f6e0d5da3f783d660c82", + "https://bcr.bazel.build/modules/rules_python/0.28.0/MODULE.bazel": "cba2573d870babc976664a912539b320cbaa7114cd3e8f053c720171cde331ed", "https://bcr.bazel.build/modules/rules_python/0.31.0/MODULE.bazel": "93a43dc47ee570e6ec9f5779b2e64c1476a6ce921c48cc9a1678a91dd5f8fd58", - "https://bcr.bazel.build/modules/rules_python/0.32.2/MODULE.bazel": "01052470fc30b49de91fb8483d26bea6f664500cfad0b078d4605b03e3a83ed4", "https://bcr.bazel.build/modules/rules_python/0.33.2/MODULE.bazel": "3e036c4ad8d804a4dad897d333d8dce200d943df4827cb849840055be8d2e937", "https://bcr.bazel.build/modules/rules_python/0.34.0/MODULE.bazel": "1d623d026e075b78c9fde483a889cda7996f5da4f36dffb24c246ab30f06513a", + "https://bcr.bazel.build/modules/rules_python/0.35.0/MODULE.bazel": "c3657951764cdcdb5a7370d5e885fad5e8c1583320aad18d46f9f110d2c22755", "https://bcr.bazel.build/modules/rules_python/0.37.1/MODULE.bazel": "3faeb2d9fa0a81f8980643ee33f212308f4d93eea4b9ce6f36d0b742e71e9500", + "https://bcr.bazel.build/modules/rules_python/0.37.2/MODULE.bazel": "b5ffde91410745750b6c13be1c5dc4555ef5bc50562af4a89fd77807fdde626a", "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", "https://bcr.bazel.build/modules/rules_python/0.40.0/MODULE.bazel": "9d1a3cd88ed7d8e39583d9ffe56ae8a244f67783ae89b60caafc9f5cf318ada7", "https://bcr.bazel.build/modules/rules_python/1.0.0/MODULE.bazel": "898a3d999c22caa585eb062b600f88654bf92efb204fa346fb55f6f8edffca43", "https://bcr.bazel.build/modules/rules_python/1.2.0/MODULE.bazel": "5aeeb48b2a6c19d668b48adf2b8a2b209a6310c230db0ce77450f148a89846e4", "https://bcr.bazel.build/modules/rules_python/1.3.0/MODULE.bazel": "8361d57eafb67c09b75bf4bbe6be360e1b8f4f18118ab48037f2bd50aa2ccb13", + "https://bcr.bazel.build/modules/rules_python/1.4.1/MODULE.bazel": "8991ad45bdc25018301d6b7e1d3626afc3c8af8aaf4bc04f23d0b99c938b73a6", "https://bcr.bazel.build/modules/rules_python/1.5.1/MODULE.bazel": "acfe65880942d44a69129d4c5c3122d57baaf3edf58ae5a6bd4edea114906bf5", "https://bcr.bazel.build/modules/rules_python/1.6.0/MODULE.bazel": "7e04ad8f8d5bea40451cf80b1bd8262552aa73f841415d20db96b7241bd027d8", + "https://bcr.bazel.build/modules/rules_python/1.6.3/MODULE.bazel": "a7b80c42cb3de5ee2a5fa1abc119684593704fcd2fec83165ebe615dec76574f", "https://bcr.bazel.build/modules/rules_python/1.7.0/MODULE.bazel": "d01f995ecd137abf30238ad9ce97f8fc3ac57289c8b24bd0bf53324d937a14f8", "https://bcr.bazel.build/modules/rules_python/1.9.0/MODULE.bazel": "afc3a05f29f09f2d3ee95ad99a145250dab41a2b2d8d6f82cc91936b3213282c", "https://bcr.bazel.build/modules/rules_python/1.9.0/source.json": "3921ea0b65298d51aead5b9e4a82203d7be9b5918619b58b53f1c259f4e63169", + "https://bcr.bazel.build/modules/rules_robolectric/4.14.1.2/MODULE.bazel": "d44fec647d0aeb67b9f3b980cf68ba634976f3ae7ccd6c07d790b59b87a4f251", + "https://bcr.bazel.build/modules/rules_robolectric/4.14.1.2/source.json": "37c10335f2361c337c5c1f34ed36d2da70534c23088062b33a8bdaab68aa9dea", + "https://bcr.bazel.build/modules/rules_rust/0.51.0/MODULE.bazel": "2b6d1617ac8503bfdcc0e4520c20539d4bba3a691100bee01afe193ceb0310f9", + "https://bcr.bazel.build/modules/rules_rust/0.69.0/MODULE.bazel": "4326fec48f2fef0d514de46346f7f77e200c82936dd08b91c9ef039fbdad5c10", + "https://bcr.bazel.build/modules/rules_rust/0.69.0/source.json": "0d094307d690cc18b3ab003998697be8070a206f65592c5c8476999796f11c4b", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", + "https://bcr.bazel.build/modules/rules_shell/0.4.0/MODULE.bazel": "0f8f11bb3cd11755f0b48c1de0bbcf62b4b34421023aa41a2fc74ef68d9584f0", "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", "https://bcr.bazel.build/modules/rules_shell/0.6.1/MODULE.bazel": "72e76b0eea4e81611ef5452aa82b3da34caca0c8b7b5c0c9584338aa93bae26b", "https://bcr.bazel.build/modules/rules_shell/0.6.1/source.json": "20ec05cd5e592055e214b2da8ccb283c7f2a421ea0dc2acbf1aa792e11c03d0c", "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", - "https://bcr.bazel.build/modules/rules_swift/1.18.0/MODULE.bazel": "a6aba73625d0dc64c7b4a1e831549b6e375fbddb9d2dde9d80c9de6ec45b24c9", "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", + "https://bcr.bazel.build/modules/rules_swift/2.5.0/MODULE.bazel": "6d2fec72be0ad2e1fc2eab536cf9d3fb149037c6d2aa678078f3512c06a819e1", "https://bcr.bazel.build/modules/rules_swift/3.1.2/MODULE.bazel": "72c8f5cf9d26427cee6c76c8e3853eb46ce6b0412a081b2b6db6e8ad56267400", "https://bcr.bazel.build/modules/rules_swift/3.1.2/source.json": "e85761f3098a6faf40b8187695e3de6d97944e98abd0d8ce579cb2daf6319a66", "https://bcr.bazel.build/modules/rules_zig/0.15.1/MODULE.bazel": "f0b813d9191c8d201e0b3ecbe5c5220ddedb717a15009fc711a78b3e1db8710d", "https://bcr.bazel.build/modules/rules_zig/0.15.1/source.json": "0bd91d7459f504196dfdc90cd0fffbf1f0df2667c9cd5eb708feb3fd27578e6a", + "https://bcr.bazel.build/modules/stardoc/0.5.0/MODULE.bazel": "f9f1f46ba8d9c3362648eea571c6f9100680efc44913618811b58cc9c02cd678", + "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", "https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4", "https://bcr.bazel.build/modules/stardoc/0.5.6/MODULE.bazel": "c43dabc564990eeab55e25ed61c07a1aadafe9ece96a4efabb3f8bf9063b71ef", @@ -375,28 +433,26 @@ "https://bcr.bazel.build/modules/tar.bzl/0.5.1/MODULE.bazel": "7c2eb3dcfc53b0f3d6f9acdfd911ca803eaf92aadf54f8ca6e4c1f3aee288351", "https://bcr.bazel.build/modules/tar.bzl/0.5.5/MODULE.bazel": "4bfab9bbc7a1966c2c5f7371f5848f5e2d27c465951b4435adc9aaf00ed681da", "https://bcr.bazel.build/modules/tar.bzl/0.6.0/MODULE.bazel": "a3584b4edcfafcabd9b0ef9819808f05b372957bbdff41601429d5fd0aac2e7c", - "https://bcr.bazel.build/modules/upb/0.0.0-20211020-160625a/MODULE.bazel": "6cced416be2dc5b9c05efd5b997049ba795e5e4e6fafbe1624f4587767638928", + "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", "https://bcr.bazel.build/modules/upb/0.0.0-20230516-61a97ef/MODULE.bazel": "c0df5e35ad55e264160417fd0875932ee3c9dda63d9fccace35ac62f45e1b6f9", - "https://bcr.bazel.build/modules/upb/0.0.0-20230907-e7430e6/MODULE.bazel": "3a7dedadf70346e678dc059dbe44d05cbf3ab17f1ce43a1c7a42edc7cbf93fd9", "https://bcr.bazel.build/modules/with_cfg.bzl/0.12.0/MODULE.bazel": "b573395fe63aef4299ba095173e2f62ccfee5ad9bbf7acaa95dba73af9fc2b38", "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.1/MODULE.bazel": "aa0ef3f6c67dd35db7ac38c76785dc02f48b53f34e16b20487e70f51b5324d0a", "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.4/MODULE.bazel": "be2276f83878525d8f228b9ec777daee5eb7baba6100dad45081510d0cec35e3", "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.4/source.json": "63d661a0c144ca74fd8b0aa61e458c7627554c7d4a8e0a4d1ce00975437c776a", "https://bcr.bazel.build/modules/xds/0.0.0-20240423-555b57e/MODULE.bazel": "cea509976a77e34131411684ef05a1d6ad194dd71a8d5816643bc5b0af16dc0f", - "https://bcr.bazel.build/modules/xds/0.0.0-20240423-555b57e/source.json": "7227e1fcad55f3f3cab1a08691ecd753cb29cc6380a47bc650851be9f9ad6d20", + "https://bcr.bazel.build/modules/xds/0.0.0-20251210-ee656c7/MODULE.bazel": "03e1c50b16fc2403de5dd98ff8e2dc0b404865740434b8705c7c0853241f943d", + "https://bcr.bazel.build/modules/xds/0.0.0-20251210-ee656c7/source.json": "69b87f3bff38ba29399c012f3e3fc0f239ac4e5c1ae66ebd9481be6d8461da55", "https://bcr.bazel.build/modules/yq.bzl/0.1.1/MODULE.bazel": "9039681f9bcb8958ee2c87ffc74bdafba9f4369096a2b5634b88abc0eaefa072", "https://bcr.bazel.build/modules/yq.bzl/0.3.1/MODULE.bazel": "9bcb7151b3cd4681b89d350530eaf7b45e32a44dda94843b8932b0cb1cd4594a", "https://bcr.bazel.build/modules/yq.bzl/0.3.1/source.json": "f0b0f204a2a6b0e34b4c9541efe8c04f2ef1af65948daa784eccea738b21dbd2", "https://bcr.bazel.build/modules/zlib-ng/2.3.3/MODULE.bazel": "0afa781a5f354b4fd811a5c9a086e111daf50e09d8e98ab1cf7c46eeacc33aac", "https://bcr.bazel.build/modules/zlib-ng/2.3.3/source.json": "63314bf75a7683c75b7db365513e93dd1c13f865dacee676235310e8d92a7a4b", "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", - "https://bcr.bazel.build/modules/zlib/1.2.13/MODULE.bazel": "aa6deb1b83c18ffecd940c4119aff9567cd0a671d7bba756741cb2ef043a29d5", + "https://bcr.bazel.build/modules/zlib/1.2.12/MODULE.bazel": "3b1a8834ada2a883674be8cbd36ede1b6ec481477ada359cd2d3ddc562340b27", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.1/MODULE.bazel": "6a9fe6e3fc865715a7be9823ce694ceb01e364c35f7a846bf0d2b34762bc066b", - "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "af322bc08976524477c79d1e45e241b6efbeb918c497e8840b8ab116802dda79", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "eec517b5bbe5492629466e11dae908d043364302283de25581e3eb944326c4ca", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/source.json": "22bc55c47af97246cfc093d0acf683a7869377de362b5d1c552c2c2e16b7a806", "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198", - "https://bcr.bazel.build/modules/zlib/1.3/MODULE.bazel": "6a9c02f19a24dcedb05572b2381446e27c272cd383aed11d41d99da9e3167a72", "https://bcr.bazel.build/modules/zstd/1.5.7.bcr.1/MODULE.bazel": "c5977176dd8555be7a9d598512ae0cae11831259c06f00a5e5e0037d5db4e3f5", "https://bcr.bazel.build/modules/zstd/1.5.7.bcr.1/source.json": "aa95e0b5aac9d80195b9047d223beaf26b1948be55d49f0e803e180e9ccc6e75" }, @@ -455,6 +511,49 @@ } } }, + "@@aspect_rules_py+//uv/private/tomltool:extension.bzl%tomltool": { + "general": { + "bzlTransitiveDigest": "TwqHTUjLecbltZP/kGFRhI/12V79YPpuOflrAhnMaR8=", + "usagesDigest": "tQ76lH+Nn5OlAFLvBbnfvnCzNIiwGgY6tOumYZXH44w=", + "recordedInputs": [ + "REPO_MAPPING:aspect_rules_py+,bazel_tools bazel_tools" + ], + "generatedRepoSpecs": { + "toml2json_aarch64_osx_libsystem": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_darwin_arm64", + "sha256": "9adc44d976e8f5baf5a6d613ceb3db6e5f56b2e22e75ac56521fdce62f227d88", + "executable": true + } + }, + "toml2json_x86_64_osx_libsystem": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_darwin_amd64", + "sha256": "425bac4015394fd9840fabcd52ac60fc796a9c10655b36c57e36ffb7dc9f3dd4", + "executable": true + } + }, + "toml2json_aarch64_linux_gnu": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_linux_arm64", + "sha256": "be8376c8e3232a242eae0d187f741a475498a949b942361a9af6e95072ef5670", + "executable": true + } + }, + "toml2json_x86_64_linux_gnu": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_linux_amd64", + "sha256": "f3dd54fabf2d27d0c027b0421860e5d9d909080be1613f0ffd87057633b65e9a", + "executable": true + } + } + } + } + }, "@@aspect_tools_telemetry+//:extension.bzl%telemetry": { "general": { "bzlTransitiveDigest": "MePriaXmQNSqUfE+YQvEtzBc2bU1mjITINIffFiZYgo=", @@ -476,227 +575,139 @@ } } }, - "@@googleapis+//:extensions.bzl%switched_rules": { + "@@envoy_api+//bazel:repositories.bzl%non_module_deps": { "general": { - "bzlTransitiveDigest": "JLpojYeGHLdgnwSUuQWOrtkL9UMygBUUEFJZh7ASlmA=", - "usagesDigest": "66rLsrgz6YqDpU0pD+02mTGNxYxhb/of6Fh0ulHzjdE=", - "recordedInputs": [], + "bzlTransitiveDigest": "zeEpd78tNW+9P6M7a2DhAjl5PQl47oXTBJOoYY0WuNI=", + "usagesDigest": "OP2p9QPxt4WFY/lah8Iecqt6SOUwZHs89hYRyaV+rSg=", + "recordedInputs": [ + "REPO_MAPPING:envoy_api+,bazel_tools bazel_tools", + "REPO_MAPPING:envoy_api+,envoy_api envoy_api+" + ], "generatedRepoSpecs": { - "com_google_googleapis_imports": { - "repoRuleId": "@@googleapis+//:repository_rules.bzl%switched_rules", + "prometheus_metrics_model": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { - "rules": { - "proto_library_with_info": [ - "", - "" - ], - "moved_proto_library": [ - "", - "" - ], - "java_proto_library": [ - "", - "" - ], - "java_grpc_library": [ - "", - "" - ], - "java_gapic_library": [ - "", - "" - ], - "java_gapic_test": [ - "", - "" - ], - "java_gapic_assembly_gradle_pkg": [ - "", - "" - ], - "py_proto_library": [ - "", - "" - ], - "py_grpc_library": [ - "", - "" - ], - "py_gapic_library": [ - "", - "" - ], - "py_test": [ - "", - "" - ], - "py_gapic_assembly_pkg": [ - "", - "" - ], - "py_import": [ - "", - "" - ], - "go_proto_library": [ - "", - "" - ], - "go_grpc_library": [ - "", - "" - ], - "go_library": [ - "", - "" - ], - "go_test": [ - "", - "" - ], - "go_gapic_library": [ - "", - "" - ], - "go_gapic_assembly_pkg": [ - "", - "" - ], - "cc_proto_library": [ - "", - "" - ], - "cc_grpc_library": [ - "", - "" - ], - "cc_gapic_library": [ - "", - "" - ], - "php_proto_library": [ - "", - "php_proto_library" - ], - "php_grpc_library": [ - "", - "php_grpc_library" - ], - "php_gapic_library": [ - "", - "php_gapic_library" - ], - "php_gapic_assembly_pkg": [ - "", - "php_gapic_assembly_pkg" - ], - "nodejs_gapic_library": [ - "", - "typescript_gapic_library" - ], - "nodejs_gapic_assembly_pkg": [ - "", - "typescript_gapic_assembly_pkg" - ], - "ruby_proto_library": [ - "", - "" - ], - "ruby_grpc_library": [ - "", - "" - ], - "ruby_ads_gapic_library": [ - "", - "" - ], - "ruby_cloud_gapic_library": [ - "", - "" - ], - "ruby_gapic_assembly_pkg": [ - "", - "" - ], - "csharp_proto_library": [ - "", - "" - ], - "csharp_grpc_library": [ - "", - "" - ], - "csharp_gapic_library": [ - "", - "" - ], - "csharp_gapic_assembly_pkg": [ - "", - "" - ] - } + "urls": [ + "https://github.com/prometheus/client_model/archive/v0.6.1.tar.gz" + ], + "sha256": "b9b690bc35d80061f255faa7df7621eae39fe157179ccd78ff6409c3b004f05e", + "strip_prefix": "client_model-0.6.1", + "build_file_content": "\nload(\"@envoy_api//bazel:api_build_system.bzl\", \"api_cc_py_proto_library\")\nload(\"@io_bazel_rules_go//proto:def.bzl\", \"go_proto_library\")\n\napi_cc_py_proto_library(\n name = \"client_model\",\n srcs = [\n \"io/prometheus/client/metrics.proto\",\n ],\n visibility = [\"//visibility:public\"],\n)\n\ngo_proto_library(\n name = \"client_model_go_proto\",\n importpath = \"github.com/prometheus/client_model/go\",\n proto = \":client_model\",\n visibility = [\"//visibility:public\"],\n)\n" + } + }, + "com_github_bufbuild_buf": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/bufbuild/buf/releases/download/v1.49.0/buf-Linux-x86_64.tar.gz" + ], + "sha256": "ee8da9748249f7946d79191e36469ce7bc3b8ba80019bff1fa4289a44cbc23bf", + "strip_prefix": "buf", + "build_file_content": "\npackage(\n default_visibility = [\"//visibility:public\"],\n)\n\nfilegroup(\n name = \"buf\",\n srcs = [\n \"@com_github_bufbuild_buf//:bin/buf\",\n ],\n tags = [\"manual\"], # buf is downloaded as a linux binary; tagged manual to prevent build for non-linux users\n)\n" + } + }, + "envoy_toolshed": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/envoyproxy/toolshed/archive/bazel-v0.2.2.tar.gz" + ], + "sha256": "443fe177aba0cef8c17b7a48905c925c67b09005b10dd70ff12cd9f729a72d51", + "strip_prefix": "toolshed-bazel-v0.2.2/bazel" } } } } }, - "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { + "@@googleapis+//:extensions.bzl%switched_rules": { "general": { - "bzlTransitiveDigest": "76FMguAa6TZAMN5inJ4/C9SZ0VlZnKOlVnlRe9bgvd8=", - "usagesDigest": "QI2z8ZUR+mqtbwsf2fLqYdJAkPOHdOV+tF2yVAUgRzw=", + "bzlTransitiveDigest": "dGjdAR2OztzWMSyz8eMU1fObvEYPBQh2DdtbMY2PmNc=", + "usagesDigest": "eoktHDHIPNOPltVMlzzSHLOYyaf7GneTwkOWrpNh8PQ=", + "recordedInputs": [], + "generatedRepoSpecs": {} + } + }, + "@@grpc+//bazel:extensions.bzl%exec_properties": { + "general": { + "bzlTransitiveDigest": "tdXJDtOnvTse7uoYV1eSKHW4jguoFr52HSAV6fIOAeI=", + "usagesDigest": "hImqXzHtSPMkGXUwrV9REcdKmXLsLUdEto6iGSUMItA=", "recordedInputs": [ - "REPO_MAPPING:rules_kotlin+,bazel_tools bazel_tools" + "REPO_MAPPING:grpc+,bazel_toolchains grpc++http_archive+bazel_toolchains" ], "generatedRepoSpecs": { - "com_github_jetbrains_kotlin_git": { - "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:compiler.bzl%kotlin_compiler_git_repository", + "grpc_custom_exec_properties": { + "repoRuleId": "@@grpc++http_archive+bazel_toolchains//rules/exec_properties:exec_properties.bzl%_exec_property_sets_repository", "attributes": { - "urls": [ - "https://github.com/JetBrains/kotlin/releases/download/v1.9.23/kotlin-compiler-1.9.23.zip" - ], - "sha256": "93137d3aab9afa9b27cb06a824c2324195c6b6f6179d8a8653f440f5bd58be88" + "constants_bzl_content": "LARGE_MACHINE = {\"label:os\": \"ubuntu\", \"label:machine_size\": \"large\"}\n" } - }, - "com_github_jetbrains_kotlin": { - "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:compiler.bzl%kotlin_capabilities_repository", + } + } + } + }, + "@@protobuf+//python/dist:system_python.bzl%system_python_extension": { + "general": { + "bzlTransitiveDigest": "XIzUjPrPvfZw+ee/BZaUhkKywlpB0qAKyTi/+DyC/XI=", + "usagesDigest": "tCi55FyqtOJ2jXh9vcjrHCl4ov3kpWiwKl103nA9BOI=", + "recordedInputs": [], + "generatedRepoSpecs": { + "system_python": { + "repoRuleId": "@@protobuf+//python/dist:system_python.bzl%system_python", "attributes": { - "git_repository_name": "com_github_jetbrains_kotlin_git", - "compiler_version": "1.9.23" + "minimum_python_version": "3.9" } - }, - "com_github_google_ksp": { - "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:ksp.bzl%ksp_compiler_plugin_repository", - "attributes": { - "urls": [ - "https://github.com/google/ksp/releases/download/1.9.23-1.0.20/artifacts.zip" - ], - "sha256": "ee0618755913ef7fd6511288a232e8fad24838b9af6ea73972a76e81053c8c2d", - "strip_version": "1.9.23-1.0.20" - } - }, - "com_github_pinterest_ktlint": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "sha256": "01b2e0ef893383a50dbeb13970fe7fa3be36ca3e83259e01649945b09d736985", - "urls": [ - "https://github.com/pinterest/ktlint/releases/download/1.3.0/ktlint" - ], - "executable": true - } - }, - "rules_android": { + } + } + } + }, + "@@rules_android+//bzlmod_extensions:apksig.bzl%apksig_extension": { + "general": { + "bzlTransitiveDigest": "eYxUKP6jh5Ixolg9KdkZ/OBudos9hU2RBV+5XdqppZ4=", + "usagesDigest": "zr/niBQ/s2fHozWAsg4vI70wAxcuFjG+QtM15qGkq9o=", + "recordedInputs": [ + "REPO_MAPPING:rules_android+,bazel_tools bazel_tools" + ], + "generatedRepoSpecs": { + "apksig": { "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", "attributes": { - "sha256": "cd06d15dd8bb59926e4d65f9003bfc20f9da4b2519985c27e190cddc8b7a7806", - "strip_prefix": "rules_android-0.1.1", - "urls": [ - "https://github.com/bazelbuild/rules_android/archive/v0.1.1.zip" - ] + "url": "https://android.googlesource.com/platform/tools/apksig/+archive/24e3075e68ebe17c0b529bb24bfda819db5e2f3b.tar.gz", + "build_file": "@@rules_android+//bzlmod_extensions:apksig.BUILD" } } } } }, + "@@rules_android+//bzlmod_extensions:com_android_dex.bzl%com_android_dex_extension": { + "general": { + "bzlTransitiveDigest": "gy5TiFZm16I25Ieuw8gSwq8vLHlpWECIMp0gWY9YynI=", + "usagesDigest": "c1Y/KGGjUYCyd8zNIVTUh1bynVXRFz6xGKaSCBpQANM=", + "recordedInputs": [ + "REPO_MAPPING:rules_android+,bazel_tools bazel_tools" + ], + "generatedRepoSpecs": { + "com_android_dex": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://android.googlesource.com/platform/dalvik/+archive/5a81c499a569731e2395f7c8d13c0e0d4e17a2b6.tar.gz", + "build_file": "@@rules_android+//bzlmod_extensions:com_android_dex.BUILD" + } + } + } + } + }, + "@@rules_android+//rules/android_sdk_repository:rule.bzl%android_sdk_repository_extension": { + "general": { + "bzlTransitiveDigest": "aLDGLKolYFwLsBssOQfwOs+bo3Aof0yKkW0HQRPX3ms=", + "usagesDigest": "iEGI2aNDMkHt9LXCdViLNUUOslpiVj2DrevWWXZEFnU=", + "recordedInputs": [], + "generatedRepoSpecs": { + "androidsdk": { + "repoRuleId": "@@rules_android+//rules/android_sdk_repository:rule.bzl%_android_sdk_repository", + "attributes": {} + } + } + } + }, "@@rules_oci+//oci:extensions.bzl%oci": { "general": { "bzlTransitiveDigest": "lRG1JBXWgJgDfEfl/34ZHgTFACz2I0bpfZWFtcoM+hc=", @@ -894,6 +905,69 @@ } } }, + "@@rules_perl+//perl:extensions.bzl%perl_repositories": { + "general": { + "bzlTransitiveDigest": "kNKYG2Q4r+exUypJYjau+ehY90be7nvRR1ffN6loz7w=", + "usagesDigest": "qSSNDdCNVxNhY36wMndEAFacdhR0ooLTmumfad0km9s=", + "recordedInputs": [ + "REPO_MAPPING:rules_perl+,bazel_tools bazel_tools", + "REPO_MAPPING:rules_perl+,rules_perl rules_perl+" + ], + "generatedRepoSpecs": { + "perl_darwin_arm64": { + "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", + "attributes": { + "strip_prefix": "perl-darwin-arm64", + "sha256": "285769f3c50c339fb59a3987b216ae3c5c573b95babe6875a1ef56fb178433da", + "urls": [ + "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-darwin-arm64.tar.xz" + ] + } + }, + "perl_darwin_amd64": { + "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", + "attributes": { + "strip_prefix": "perl-darwin-amd64", + "sha256": "63bc5ee36f5394d71c50cca6cafdd333ee58f9eaa40bca63c85f9bd06f2c1fd6", + "urls": [ + "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-darwin-amd64.tar.xz" + ] + } + }, + "perl_linux_amd64": { + "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", + "attributes": { + "strip_prefix": "perl-linux-amd64", + "sha256": "3bdffa9d7a3f97c0207314637b260ba5115b1d0829f97e3e2e301191a4d4d076", + "urls": [ + "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-linux-amd64.tar.xz" + ] + } + }, + "perl_linux_arm64": { + "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", + "attributes": { + "strip_prefix": "perl-linux-arm64", + "sha256": "6fa4ece99e790ecbc2861f6ecb7b52694c01c2eeb215b4370f16a3b12d952117", + "urls": [ + "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-linux-arm64.tar.xz" + ] + } + }, + "perl_windows_x86_64": { + "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", + "attributes": { + "strip_prefix": "", + "sha256": "aeb973da474f14210d3e1a1f942dcf779e2ae7e71e4c535e6c53ebabe632cc98", + "urls": [ + "https://mirror.bazel.build/strawberryperl.com/download/5.32.1.1/strawberry-perl-5.32.1.1-64bit.zip", + "https://strawberryperl.com/download/5.32.1.1/strawberry-perl-5.32.1.1-64bit.zip" + ] + } + } + } + } + }, "@@rules_python+//python/uv:uv.bzl%uv": { "general": { "bzlTransitiveDigest": "IzGWfoGARM6X2a4AM7BN5gheLtDsFeP9Fra2qTod+mA=", @@ -1055,6 +1129,367 @@ } } }, + "@@rules_rust+//crate_universe:extension.bzl%crate": { + "general": { + "bzlTransitiveDigest": "sv4+a0gzluPfXdGS4VRD5W4GeawxvNnNht0NHBgIAPs=", + "usagesDigest": "EuFUqVKVHF263jHTWOHXs4tFACdRNVOhwpoytdk19bs=", + "recordedInputs": [ + "ENV:CARGO_BAZEL_DEBUG \\0", + "ENV:CARGO_BAZEL_GENERATOR_SHA256 \\0", + "ENV:CARGO_BAZEL_GENERATOR_URL \\0", + "ENV:CARGO_BAZEL_ISOLATED \\0", + "ENV:CARGO_BAZEL_REPIN \\0", + "ENV:CARGO_BAZEL_REPIN_ONLY \\0", + "ENV:CARGO_BAZEL_TIMEOUT \\0", + "ENV:REPIN \\0", + "REPO_MAPPING:bazel_features+,bazel_features_globals bazel_features++version_extension+bazel_features_globals", + "REPO_MAPPING:bazel_features+,bazel_features_version bazel_features++version_extension+bazel_features_version", + "REPO_MAPPING:rules_cc+,bazel_skylib bazel_skylib+", + "REPO_MAPPING:rules_cc+,bazel_tools bazel_tools", + "REPO_MAPPING:rules_cc+,cc_compatibility_proxy rules_cc++compatibility_proxy+cc_compatibility_proxy", + "REPO_MAPPING:rules_cc+,platforms platforms", + "REPO_MAPPING:rules_cc+,rules_cc rules_cc+", + "REPO_MAPPING:rules_cc++compatibility_proxy+cc_compatibility_proxy,rules_cc rules_cc+", + "REPO_MAPPING:rules_rust+,bazel_features bazel_features+", + "REPO_MAPPING:rules_rust+,bazel_skylib bazel_skylib+", + "REPO_MAPPING:rules_rust+,bazel_tools bazel_tools", + "REPO_MAPPING:rules_rust+,rules_cc rules_cc+", + "REPO_MAPPING:rules_rust+,rules_rust rules_rust+" + ], + "generatedRepoSpecs": { + "crates": { + "repoRuleId": "@@rules_rust+//crate_universe:extensions.bzl%_generate_repo", + "attributes": { + "contents": { + "BUILD.bazel": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files(\n [\n \"cargo-bazel.json\",\n \"crates.bzl\",\n \"defs.bzl\",\n ] + glob(\n allow_empty = True,\n include = [\"*.bazel\"],\n ),\n)\n\nfilegroup(\n name = \"srcs\",\n srcs = glob(\n allow_empty = True,\n include = [\n \"*.bazel\",\n \"*.bzl\",\n ],\n ),\n)\n\n# Workspace Member Dependencies\nalias(\n name = \"googletest-0.14.3\",\n actual = \"@crates__googletest-0.14.3//:googletest\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"googletest\",\n actual = \"@crates__googletest-0.14.3//:googletest\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"linkme-0.3.36\",\n actual = \"@crates__linkme-0.3.36//:linkme\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"linkme\",\n actual = \"@crates__linkme-0.3.36//:linkme\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"paste-1.0.15\",\n actual = \"@crates__paste-1.0.15//:paste\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"paste\",\n actual = \"@crates__paste-1.0.15//:paste\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"quote-1.0.46\",\n actual = \"@crates__quote-1.0.46//:quote\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"quote\",\n actual = \"@crates__quote-1.0.46//:quote\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"syn-2.0.118\",\n actual = \"@crates__syn-2.0.118//:syn\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"syn\",\n actual = \"@crates__syn-2.0.118//:syn\",\n tags = [\"manual\"],\n)\n", + "alias_rules.bzl": "\"\"\"Alias that transitions its target to `compilation_mode=opt`. Use `transition_alias=\"opt\"` to enable.\"\"\"\n\nload(\"@rules_cc//cc:defs.bzl\", \"CcInfo\")\nload(\"@rules_rust//rust:rust_common.bzl\", \"COMMON_PROVIDERS\")\n\ndef _transition_alias_impl(ctx):\n # `ctx.attr.actual` is a list of 1 item due to the transition\n providers = [ctx.attr.actual[0][provider] for provider in COMMON_PROVIDERS]\n if CcInfo in ctx.attr.actual[0]:\n providers.append(ctx.attr.actual[0][CcInfo])\n return providers\n\ndef _change_compilation_mode(compilation_mode):\n def _change_compilation_mode_impl(_settings, _attr):\n return {\n \"//command_line_option:compilation_mode\": compilation_mode,\n }\n\n return transition(\n implementation = _change_compilation_mode_impl,\n inputs = [],\n outputs = [\n \"//command_line_option:compilation_mode\",\n ],\n )\n\ndef _transition_alias_rule(compilation_mode):\n return rule(\n implementation = _transition_alias_impl,\n provides = COMMON_PROVIDERS,\n attrs = {\n \"actual\": attr.label(\n mandatory = True,\n doc = \"`rust_library()` target to transition to `compilation_mode=opt`.\",\n providers = COMMON_PROVIDERS,\n cfg = _change_compilation_mode(compilation_mode),\n ),\n \"_allowlist_function_transition\": attr.label(\n default = \"@bazel_tools//tools/allowlists/function_transition_allowlist\",\n ),\n },\n doc = \"Transitions a Rust library crate to the `compilation_mode=opt`.\",\n )\n\ntransition_alias_dbg = _transition_alias_rule(\"dbg\")\ntransition_alias_fastbuild = _transition_alias_rule(\"fastbuild\")\ntransition_alias_opt = _transition_alias_rule(\"opt\")\n", + "defs.bzl": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\"\"\"\n# `crates_repository` API\n\n- [aliases](#aliases)\n- [crate_deps](#crate_deps)\n- [all_crate_deps](#all_crate_deps)\n- [crate_repositories](#crate_repositories)\n\n\"\"\"\n\nload(\"@bazel_tools//tools/build_defs/repo:git.bzl\", \"new_git_repository\")\nload(\"@bazel_tools//tools/build_defs/repo:http.bzl\", \"http_archive\")\nload(\"@bazel_tools//tools/build_defs/repo:utils.bzl\", \"maybe\")\nload(\"@bazel_skylib//lib:selects.bzl\", \"selects\")\nload(\"@rules_rust//crate_universe/private:local_crate_mirror.bzl\", \"local_crate_mirror\")\n\n###############################################################################\n# MACROS API\n###############################################################################\n\n# An identifier that represent common dependencies (unconditional).\n_COMMON_CONDITION = \"\"\n\ndef _flatten_dependency_maps(all_dependency_maps):\n \"\"\"Flatten a list of dependency maps into one dictionary.\n\n Dependency maps have the following structure:\n\n ```python\n DEPENDENCIES_MAP = {\n # The first key in the map is a Bazel package\n # name of the workspace this file is defined in.\n \"workspace_member_package\": {\n\n # Not all dependencies are supported for all platforms.\n # the condition key is the condition required to be true\n # on the host platform.\n \"condition\": {\n\n # An alias to a crate target. # The label of the crate target the\n # Aliases are only crate names. # package name refers to.\n \"package_name\": \"@full//:label\",\n }\n }\n }\n ```\n\n Args:\n all_dependency_maps (list): A list of dicts as described above\n\n Returns:\n dict: A dictionary as described above\n \"\"\"\n dependencies = {}\n\n for workspace_deps_map in all_dependency_maps:\n for pkg_name, conditional_deps_map in workspace_deps_map.items():\n if pkg_name not in dependencies:\n non_frozen_map = dict()\n for key, values in conditional_deps_map.items():\n non_frozen_map.update({key: dict(values.items())})\n dependencies.setdefault(pkg_name, non_frozen_map)\n continue\n\n for condition, deps_map in conditional_deps_map.items():\n # If the condition has not been recorded, do so and continue\n if condition not in dependencies[pkg_name]:\n dependencies[pkg_name].setdefault(condition, dict(deps_map.items()))\n continue\n\n # Alert on any miss-matched dependencies\n inconsistent_entries = []\n for crate_name, crate_label in deps_map.items():\n existing = dependencies[pkg_name][condition].get(crate_name)\n if existing and existing != crate_label:\n inconsistent_entries.append((crate_name, existing, crate_label))\n dependencies[pkg_name][condition].update({crate_name: crate_label})\n\n return dependencies\n\ndef crate_deps(deps, package_name = None):\n \"\"\"Finds the fully qualified label of the requested crates for the package where this macro is called.\n\n Args:\n deps (list): The desired list of crate targets.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()`.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if not deps:\n return []\n\n if package_name == None:\n package_name = native.package_name()\n\n # Join both sets of dependencies\n dependencies = _flatten_dependency_maps([\n _NORMAL_DEPENDENCIES,\n _NORMAL_DEV_DEPENDENCIES,\n _PROC_MACRO_DEPENDENCIES,\n _PROC_MACRO_DEV_DEPENDENCIES,\n _BUILD_DEPENDENCIES,\n _BUILD_PROC_MACRO_DEPENDENCIES,\n ]).pop(package_name, {})\n\n # Combine all conditional packages so we can easily index over a flat list\n # TODO: Perhaps this should actually return select statements and maintain\n # the conditionals of the dependencies\n flat_deps = {}\n for deps_set in dependencies.values():\n for crate_name, crate_label in deps_set.items():\n flat_deps.update({crate_name: crate_label})\n\n missing_crates = []\n crate_targets = []\n for crate_target in deps:\n if crate_target not in flat_deps:\n missing_crates.append(crate_target)\n else:\n crate_targets.append(flat_deps[crate_target])\n\n if missing_crates:\n fail(\"Could not find crates `{}` among dependencies of `{}`. Available dependencies were `{}`\".format(\n missing_crates,\n package_name,\n dependencies,\n ))\n\n return crate_targets\n\ndef all_crate_deps(\n normal = False, \n normal_dev = False, \n proc_macro = False, \n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Finds the fully qualified label of all requested direct crate dependencies \\\n for the package where this macro is called.\n\n If no parameters are set, all normal dependencies are returned. Setting any one flag will\n otherwise impact the contents of the returned list.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list.\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_dependency_maps = []\n if normal:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n if normal_dev:\n all_dependency_maps.append(_NORMAL_DEV_DEPENDENCIES)\n if proc_macro:\n all_dependency_maps.append(_PROC_MACRO_DEPENDENCIES)\n if proc_macro_dev:\n all_dependency_maps.append(_PROC_MACRO_DEV_DEPENDENCIES)\n if build:\n all_dependency_maps.append(_BUILD_DEPENDENCIES)\n if build_proc_macro:\n all_dependency_maps.append(_BUILD_PROC_MACRO_DEPENDENCIES)\n\n # Default to always using normal dependencies\n if not all_dependency_maps:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n\n dependencies = _flatten_dependency_maps(all_dependency_maps).pop(package_name, None)\n\n if not dependencies:\n if dependencies == None:\n fail(\"Tried to get all_crate_deps for package \" + package_name + \" but that package had no Cargo.toml file\")\n else:\n return []\n\n crate_deps = list(dependencies.pop(_COMMON_CONDITION, {}).values())\n for condition, deps in dependencies.items():\n crate_deps += selects.with_or({\n tuple(_CONDITIONS[condition]): deps.values(),\n \"//conditions:default\": [],\n })\n\n return crate_deps\n\ndef aliases(\n normal = False,\n normal_dev = False,\n proc_macro = False,\n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Produces a map of Crate alias names to their original label\n\n If no dependency kinds are specified, `normal` and `proc_macro` are used by default.\n Setting any one flag will otherwise determine the contents of the returned dict.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n dict: The aliases of all associated packages\n \"\"\"\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_aliases_maps = []\n if normal:\n all_aliases_maps.append(_NORMAL_ALIASES)\n if normal_dev:\n all_aliases_maps.append(_NORMAL_DEV_ALIASES)\n if proc_macro:\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n if proc_macro_dev:\n all_aliases_maps.append(_PROC_MACRO_DEV_ALIASES)\n if build:\n all_aliases_maps.append(_BUILD_ALIASES)\n if build_proc_macro:\n all_aliases_maps.append(_BUILD_PROC_MACRO_ALIASES)\n\n # Default to always using normal aliases\n if not all_aliases_maps:\n all_aliases_maps.append(_NORMAL_ALIASES)\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n\n aliases = _flatten_dependency_maps(all_aliases_maps).pop(package_name, None)\n\n if not aliases:\n return dict()\n\n common_items = aliases.pop(_COMMON_CONDITION, {}).items()\n\n # If there are only common items in the dictionary, immediately return them\n if not len(aliases.keys()) == 1:\n return dict(common_items)\n\n # Build a single select statement where each conditional has accounted for the\n # common set of aliases.\n crate_aliases = {\"//conditions:default\": dict(common_items)}\n for condition, deps in aliases.items():\n condition_triples = _CONDITIONS[condition]\n for triple in condition_triples:\n if triple in crate_aliases:\n crate_aliases[triple].update(deps)\n else:\n crate_aliases.update({triple: dict(deps.items() + common_items)})\n\n return select(crate_aliases)\n\n###############################################################################\n# WORKSPACE MEMBER DEPS AND ALIASES\n###############################################################################\n\n_NORMAL_DEPENDENCIES = {\n \"\": {\n _COMMON_CONDITION: {\n \"googletest\": Label(\"@crates//:googletest-0.14.3\"),\n \"linkme\": Label(\"@crates//:linkme-0.3.36\"),\n \"quote\": Label(\"@crates//:quote-1.0.46\"),\n \"syn\": Label(\"@crates//:syn-2.0.118\"),\n },\n },\n}\n\n\n_NORMAL_ALIASES = {\n \"\": {\n _COMMON_CONDITION: {\n },\n },\n}\n\n\n_NORMAL_DEV_DEPENDENCIES = {\n \"\": {\n },\n}\n\n\n_NORMAL_DEV_ALIASES = {\n \"\": {\n },\n}\n\n\n_PROC_MACRO_DEPENDENCIES = {\n \"\": {\n _COMMON_CONDITION: {\n \"paste\": Label(\"@crates//:paste-1.0.15\"),\n },\n },\n}\n\n\n_PROC_MACRO_ALIASES = {\n \"\": {\n },\n}\n\n\n_PROC_MACRO_DEV_DEPENDENCIES = {\n \"\": {\n },\n}\n\n\n_PROC_MACRO_DEV_ALIASES = {\n \"\": {\n },\n}\n\n\n_BUILD_DEPENDENCIES = {\n \"\": {\n },\n}\n\n\n_BUILD_ALIASES = {\n \"\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_DEPENDENCIES = {\n \"\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_ALIASES = {\n \"\": {\n },\n}\n\n\n_CONDITIONS = {\n \"aarch64-apple-darwin\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\"],\n \"aarch64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\"],\n \"wasm32-unknown-unknown\": [\"@rules_rust//rust/platform:wasm32-unknown-unknown\"],\n \"wasm32-wasip1\": [\"@rules_rust//rust/platform:wasm32-wasip1\"],\n \"x86_64-pc-windows-msvc\": [\"@rules_rust//rust/platform:x86_64-pc-windows-msvc\"],\n \"x86_64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"x86_64-unknown-nixos-gnu\": [\"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\"],\n}\n\n###############################################################################\n\ndef crate_repositories():\n \"\"\"A macro for defining repositories for all generated crates.\n\n Returns:\n A list of repos visible to the module through the module extension.\n \"\"\"\n maybe(\n http_archive,\n name = \"crates__aho-corasick-1.1.4\",\n sha256 = \"ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/aho-corasick/1.1.4/download\"],\n strip_prefix = \"aho-corasick-1.1.4\",\n build_file = Label(\"@crates//crates:BUILD.aho-corasick-1.1.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__autocfg-1.5.1\",\n sha256 = \"f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/autocfg/1.5.1/download\"],\n strip_prefix = \"autocfg-1.5.1\",\n build_file = Label(\"@crates//crates:BUILD.autocfg-1.5.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__googletest-0.14.3\",\n sha256 = \"f6b5e2f2b556b7b90297a5a35c8267dd43a537923d2b329beefdba2b4ec19d94\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/googletest/0.14.3/download\"],\n strip_prefix = \"googletest-0.14.3\",\n build_file = Label(\"@crates//crates:BUILD.googletest-0.14.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__googletest_macro-0.14.3\",\n sha256 = \"2ae6abc96141edd26bf5aeec0f119c129c44de3ced09e5073711a02cb74725d0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/googletest_macro/0.14.3/download\"],\n strip_prefix = \"googletest_macro-0.14.3\",\n build_file = Label(\"@crates//crates:BUILD.googletest_macro-0.14.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__linkme-0.3.36\",\n sha256 = \"e83272d46373fb8decca684579ac3e7c8f3d71d4cc3aa693df8759e260ae41cf\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/linkme/0.3.36/download\"],\n strip_prefix = \"linkme-0.3.36\",\n build_file = Label(\"@crates//crates:BUILD.linkme-0.3.36.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__linkme-impl-0.3.36\",\n sha256 = \"32d59e20403c7d08fe62b4376edfe5c7fb2ef1e6b1465379686d0f21c8df444b\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/linkme-impl/0.3.36/download\"],\n strip_prefix = \"linkme-impl-0.3.36\",\n build_file = Label(\"@crates//crates:BUILD.linkme-impl-0.3.36.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__memchr-2.8.2\",\n sha256 = \"88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/memchr/2.8.2/download\"],\n strip_prefix = \"memchr-2.8.2\",\n build_file = Label(\"@crates//crates:BUILD.memchr-2.8.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__num-traits-0.2.19\",\n sha256 = \"071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/num-traits/0.2.19/download\"],\n strip_prefix = \"num-traits-0.2.19\",\n build_file = Label(\"@crates//crates:BUILD.num-traits-0.2.19.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__paste-1.0.15\",\n sha256 = \"57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/paste/1.0.15/download\"],\n strip_prefix = \"paste-1.0.15\",\n build_file = Label(\"@crates//crates:BUILD.paste-1.0.15.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__proc-macro2-1.0.106\",\n sha256 = \"8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/proc-macro2/1.0.106/download\"],\n strip_prefix = \"proc-macro2-1.0.106\",\n build_file = Label(\"@crates//crates:BUILD.proc-macro2-1.0.106.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__quote-1.0.46\",\n sha256 = \"dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/quote/1.0.46/download\"],\n strip_prefix = \"quote-1.0.46\",\n build_file = Label(\"@crates//crates:BUILD.quote-1.0.46.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-1.12.4\",\n sha256 = \"f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex/1.12.4/download\"],\n strip_prefix = \"regex-1.12.4\",\n build_file = Label(\"@crates//crates:BUILD.regex-1.12.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-automata-0.4.14\",\n sha256 = \"6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-automata/0.4.14/download\"],\n strip_prefix = \"regex-automata-0.4.14\",\n build_file = Label(\"@crates//crates:BUILD.regex-automata-0.4.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-syntax-0.8.11\",\n sha256 = \"d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-syntax/0.8.11/download\"],\n strip_prefix = \"regex-syntax-0.8.11\",\n build_file = Label(\"@crates//crates:BUILD.regex-syntax-0.8.11.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rustversion-1.0.22\",\n sha256 = \"b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rustversion/1.0.22/download\"],\n strip_prefix = \"rustversion-1.0.22\",\n build_file = Label(\"@crates//crates:BUILD.rustversion-1.0.22.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__syn-2.0.118\",\n sha256 = \"1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/syn/2.0.118/download\"],\n strip_prefix = \"syn-2.0.118\",\n build_file = Label(\"@crates//crates:BUILD.syn-2.0.118.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-ident-1.0.24\",\n sha256 = \"e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-ident/1.0.24/download\"],\n strip_prefix = \"unicode-ident-1.0.24\",\n build_file = Label(\"@crates//crates:BUILD.unicode-ident-1.0.24.bazel\"),\n )\n\n return [\n struct(repo=\"crates__googletest-0.14.3\", is_dev_dep = False),\n struct(repo=\"crates__linkme-0.3.36\", is_dev_dep = False),\n struct(repo=\"crates__paste-1.0.15\", is_dev_dep = False),\n struct(repo=\"crates__quote-1.0.46\", is_dev_dep = False),\n struct(repo=\"crates__syn-2.0.118\", is_dev_dep = False),\n ]\n" + } + } + }, + "crates__aho-corasick-1.1.4": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "remote_patch_strip": 1, + "sha256": "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/aho-corasick/1.1.4/download" + ], + "strip_prefix": "aho-corasick-1.1.4", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"aho_corasick\",\n deps = [\n \"@crates__memchr-2.8.2//:memchr\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"perf-literal\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=aho-corasick\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.1.4\",\n)\n" + } + }, + "crates__autocfg-1.5.1": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "remote_patch_strip": 1, + "sha256": "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/autocfg/1.5.1/download" + ], + "strip_prefix": "autocfg-1.5.1", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"autocfg\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=autocfg\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.5.1\",\n)\n" + } + }, + "crates__googletest-0.14.3": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "remote_patch_strip": 1, + "sha256": "f6b5e2f2b556b7b90297a5a35c8267dd43a537923d2b329beefdba2b4ec19d94", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/googletest/0.14.3/download" + ], + "strip_prefix": "googletest-0.14.3", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"googletest\",\n deps = [\n \"@crates__num-traits-0.2.19//:num_traits\",\n \"@crates__regex-1.12.4//:regex\",\n ],\n proc_macro_deps = [\n \"@crates__googletest_macro-0.14.3//:googletest_macro\",\n \"@crates__rustversion-1.0.22//:rustversion\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=googletest\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.14.3\",\n)\n" + } + }, + "crates__googletest_macro-0.14.3": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "remote_patch_strip": 1, + "sha256": "2ae6abc96141edd26bf5aeec0f119c129c44de3ced09e5073711a02cb74725d0", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/googletest_macro/0.14.3/download" + ], + "strip_prefix": "googletest_macro-0.14.3", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"googletest_macro\",\n deps = [\n \"@crates__proc-macro2-1.0.106//:proc_macro2\",\n \"@crates__quote-1.0.46//:quote\",\n \"@crates__syn-2.0.118//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=googletest_macro\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.14.3\",\n)\n" + } + }, + "crates__linkme-0.3.36": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "remote_patch_strip": 1, + "sha256": "e83272d46373fb8decca684579ac3e7c8f3d71d4cc3aa693df8759e260ae41cf", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/linkme/0.3.36/download" + ], + "strip_prefix": "linkme-0.3.36", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"linkme\",\n deps = [\n \"@crates__linkme-0.3.36//:build_script_build\",\n ],\n proc_macro_deps = [\n \"@crates__linkme-impl-0.3.36//:linkme_impl\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=linkme\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.36\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"linkme\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=linkme\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.3.36\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + } + }, + "crates__linkme-impl-0.3.36": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "remote_patch_strip": 1, + "sha256": "32d59e20403c7d08fe62b4376edfe5c7fb2ef1e6b1465379686d0f21c8df444b", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/linkme-impl/0.3.36/download" + ], + "strip_prefix": "linkme-impl-0.3.36", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"linkme_impl\",\n deps = [\n \"@crates__linkme-impl-0.3.36//:build_script_build\",\n \"@crates__proc-macro2-1.0.106//:proc_macro2\",\n \"@crates__quote-1.0.46//:quote\",\n \"@crates__syn-2.0.118//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=linkme-impl\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.36\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"linkme-impl\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=linkme-impl\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.3.36\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + } + }, + "crates__memchr-2.8.2": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "remote_patch_strip": 1, + "sha256": "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/memchr/2.8.2/download" + ], + "strip_prefix": "memchr-2.8.2", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"memchr\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=memchr\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.8.2\",\n)\n" + } + }, + "crates__num-traits-0.2.19": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "remote_patch_strip": 1, + "sha256": "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/num-traits/0.2.19/download" + ], + "strip_prefix": "num-traits-0.2.19", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"num_traits\",\n deps = [\n \"@crates__num-traits-0.2.19//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=num-traits\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.19\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n deps = [\n \"@crates__autocfg-1.5.1//:autocfg\",\n ],\n edition = \"2021\",\n pkg_name = \"num-traits\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=num-traits\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.2.19\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + } + }, + "crates__paste-1.0.15": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "remote_patch_strip": 1, + "sha256": "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/paste/1.0.15/download" + ], + "strip_prefix": "paste-1.0.15", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"paste\",\n deps = [\n \"@crates__paste-1.0.15//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=paste\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.15\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2018\",\n pkg_name = \"paste\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=paste\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.15\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + } + }, + "crates__proc-macro2-1.0.106": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "remote_patch_strip": 1, + "sha256": "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/proc-macro2/1.0.106/download" + ], + "strip_prefix": "proc-macro2-1.0.106", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"proc_macro2\",\n deps = [\n \"@crates__proc-macro2-1.0.106//:build_script_build\",\n \"@crates__unicode-ident-1.0.24//:unicode_ident\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"proc-macro\",\n ] + select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [\n \"default\", # aarch64-apple-darwin\n ],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [\n \"default\", # aarch64-unknown-linux-gnu\n ],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [\n \"default\", # x86_64-pc-windows-msvc\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [\n \"default\", # x86_64-unknown-linux-gnu\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [\n \"default\", # x86_64-unknown-nixos-gnu\n ],\n \"//conditions:default\": [],\n }),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=proc-macro2\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.106\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"proc-macro\",\n ] + select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [\n \"default\", # aarch64-apple-darwin\n ],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [\n \"default\", # aarch64-unknown-linux-gnu\n ],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [\n \"default\", # x86_64-pc-windows-msvc\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [\n \"default\", # x86_64-unknown-linux-gnu\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [\n \"default\", # x86_64-unknown-nixos-gnu\n ],\n \"//conditions:default\": [],\n }),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"proc-macro2\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=proc-macro2\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.106\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + } + }, + "crates__quote-1.0.46": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "remote_patch_strip": 1, + "sha256": "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/quote/1.0.46/download" + ], + "strip_prefix": "quote-1.0.46", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"quote\",\n deps = [\n \"@crates__proc-macro2-1.0.106//:proc_macro2\",\n \"@crates__quote-1.0.46//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"proc-macro\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=quote\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.46\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"proc-macro\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"quote\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=quote\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.46\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + } + }, + "crates__regex-1.12.4": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "remote_patch_strip": 1, + "sha256": "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/regex/1.12.4/download" + ], + "strip_prefix": "regex-1.12.4", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"regex\",\n deps = [\n \"@crates__aho-corasick-1.1.4//:aho_corasick\",\n \"@crates__memchr-2.8.2//:memchr\",\n \"@crates__regex-automata-0.4.14//:regex_automata\",\n \"@crates__regex-syntax-0.8.11//:regex_syntax\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"perf\",\n \"perf-backtrack\",\n \"perf-cache\",\n \"perf-dfa\",\n \"perf-inline\",\n \"perf-literal\",\n \"perf-onepass\",\n \"std\",\n \"unicode\",\n \"unicode-age\",\n \"unicode-bool\",\n \"unicode-case\",\n \"unicode-gencat\",\n \"unicode-perl\",\n \"unicode-script\",\n \"unicode-segment\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=regex\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.12.4\",\n)\n" + } + }, + "crates__regex-automata-0.4.14": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "remote_patch_strip": 1, + "sha256": "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/regex-automata/0.4.14/download" + ], + "strip_prefix": "regex-automata-0.4.14", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"regex_automata\",\n deps = [\n \"@crates__aho-corasick-1.1.4//:aho_corasick\",\n \"@crates__memchr-2.8.2//:memchr\",\n \"@crates__regex-syntax-0.8.11//:regex_syntax\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"dfa-onepass\",\n \"hybrid\",\n \"meta\",\n \"nfa-backtrack\",\n \"nfa-pikevm\",\n \"nfa-thompson\",\n \"perf-inline\",\n \"perf-literal\",\n \"perf-literal-multisubstring\",\n \"perf-literal-substring\",\n \"std\",\n \"syntax\",\n \"unicode\",\n \"unicode-age\",\n \"unicode-bool\",\n \"unicode-case\",\n \"unicode-gencat\",\n \"unicode-perl\",\n \"unicode-script\",\n \"unicode-segment\",\n \"unicode-word-boundary\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=regex-automata\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.4.14\",\n)\n" + } + }, + "crates__regex-syntax-0.8.11": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "remote_patch_strip": 1, + "sha256": "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/regex-syntax/0.8.11/download" + ], + "strip_prefix": "regex-syntax-0.8.11", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"regex_syntax\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n \"unicode\",\n \"unicode-age\",\n \"unicode-bool\",\n \"unicode-case\",\n \"unicode-gencat\",\n \"unicode-perl\",\n \"unicode-script\",\n \"unicode-segment\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=regex-syntax\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.11\",\n)\n" + } + }, + "crates__rustversion-1.0.22": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "remote_patch_strip": 1, + "sha256": "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/rustversion/1.0.22/download" + ], + "strip_prefix": "rustversion-1.0.22", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"rustversion\",\n deps = [\n \"@crates__rustversion-1.0.22//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rustversion\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.22\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build/build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2018\",\n pkg_name = \"rustversion\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rustversion\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.22\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + } + }, + "crates__syn-2.0.118": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "remote_patch_strip": 1, + "sha256": "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/syn/2.0.118/download" + ], + "strip_prefix": "syn-2.0.118", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"syn\",\n deps = [\n \"@crates__proc-macro2-1.0.106//:proc_macro2\",\n \"@crates__quote-1.0.46//:quote\",\n \"@crates__unicode-ident-1.0.24//:unicode_ident\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"clone-impls\",\n \"default\",\n \"derive\",\n \"parsing\",\n \"printing\",\n \"proc-macro\",\n ] + select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [\n \"extra-traits\", # aarch64-apple-darwin\n \"full\", # aarch64-apple-darwin\n ],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [\n \"extra-traits\", # aarch64-unknown-linux-gnu\n \"full\", # aarch64-unknown-linux-gnu\n ],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [\n \"extra-traits\", # x86_64-pc-windows-msvc\n \"full\", # x86_64-pc-windows-msvc\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [\n \"extra-traits\", # x86_64-unknown-linux-gnu\n \"full\", # x86_64-unknown-linux-gnu\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [\n \"extra-traits\", # x86_64-unknown-nixos-gnu\n \"full\", # x86_64-unknown-nixos-gnu\n ],\n \"//conditions:default\": [],\n }),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=syn\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.0.118\",\n)\n" + } + }, + "crates__unicode-ident-1.0.24": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "remote_patch_strip": 1, + "sha256": "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75", + "type": "tar.gz", + "urls": [ + "https://static.crates.io/crates/unicode-ident/1.0.24/download" + ], + "strip_prefix": "unicode-ident-1.0.24", + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"unicode_ident\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=unicode-ident\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.24\",\n)\n" + } + } + } + } + }, + "@@rules_rust+//crate_universe/private:internal_extensions.bzl%cu_nr": { + "general": { + "bzlTransitiveDigest": "R1Y9HkqS/IoqUbr25FqrM9MNwlYxdxjkDgGabT4WKKU=", + "usagesDigest": "tG3p3Nb5XxC7vWY/bcKdb//g0HoAxpxxH3F5/jBVlk4=", + "recordedInputs": [ + "REPO_MAPPING:bazel_features+,bazel_features_globals bazel_features++version_extension+bazel_features_globals", + "REPO_MAPPING:bazel_features+,bazel_features_version bazel_features++version_extension+bazel_features_version", + "REPO_MAPPING:rules_cc+,bazel_skylib bazel_skylib+", + "REPO_MAPPING:rules_cc+,bazel_tools bazel_tools", + "REPO_MAPPING:rules_cc+,cc_compatibility_proxy rules_cc++compatibility_proxy+cc_compatibility_proxy", + "REPO_MAPPING:rules_cc+,platforms platforms", + "REPO_MAPPING:rules_cc+,rules_cc rules_cc+", + "REPO_MAPPING:rules_cc++compatibility_proxy+cc_compatibility_proxy,rules_cc rules_cc+", + "REPO_MAPPING:rules_rust+,bazel_features bazel_features+", + "REPO_MAPPING:rules_rust+,bazel_skylib bazel_skylib+", + "REPO_MAPPING:rules_rust+,bazel_tools bazel_tools", + "REPO_MAPPING:rules_rust+,cui rules_rust++cu+cui", + "REPO_MAPPING:rules_rust+,rrc rules_rust++i2+rrc", + "REPO_MAPPING:rules_rust+,rules_cc rules_cc+", + "REPO_MAPPING:rules_rust+,rules_rust rules_rust+" + ], + "generatedRepoSpecs": { + "cargo_bazel_bootstrap": { + "repoRuleId": "@@rules_rust+//cargo/private:cargo_bootstrap.bzl%cargo_bootstrap_repository", + "attributes": { + "srcs": [ + "@@rules_rust+//crate_universe:src/api.rs", + "@@rules_rust+//crate_universe:src/api/lockfile.rs", + "@@rules_rust+//crate_universe:src/cli.rs", + "@@rules_rust+//crate_universe:src/cli/generate.rs", + "@@rules_rust+//crate_universe:src/cli/query.rs", + "@@rules_rust+//crate_universe:src/cli/render.rs", + "@@rules_rust+//crate_universe:src/cli/splice.rs", + "@@rules_rust+//crate_universe:src/cli/vendor.rs", + "@@rules_rust+//crate_universe:src/config.rs", + "@@rules_rust+//crate_universe:src/context.rs", + "@@rules_rust+//crate_universe:src/context/crate_context.rs", + "@@rules_rust+//crate_universe:src/context/platforms.rs", + "@@rules_rust+//crate_universe:src/lib.rs", + "@@rules_rust+//crate_universe:src/lockfile.rs", + "@@rules_rust+//crate_universe:src/main.rs", + "@@rules_rust+//crate_universe:src/metadata.rs", + "@@rules_rust+//crate_universe:src/metadata/cargo_bin.rs", + "@@rules_rust+//crate_universe:src/metadata/cargo_tree_resolver.rs", + "@@rules_rust+//crate_universe:src/metadata/cargo_tree_rustc_wrapper.bat", + "@@rules_rust+//crate_universe:src/metadata/cargo_tree_rustc_wrapper.sh", + "@@rules_rust+//crate_universe:src/metadata/dependency.rs", + "@@rules_rust+//crate_universe:src/metadata/metadata_annotation.rs", + "@@rules_rust+//crate_universe:src/rendering.rs", + "@@rules_rust+//crate_universe:src/rendering/template_engine.rs", + "@@rules_rust+//crate_universe:src/rendering/templates/module_bzl.j2", + "@@rules_rust+//crate_universe:src/rendering/templates/partials/header.j2", + "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/aliases_map.j2", + "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/deps_map.j2", + "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/repo_git.j2", + "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/repo_http.j2", + "@@rules_rust+//crate_universe:src/rendering/templates/vendor_module.j2", + "@@rules_rust+//crate_universe:src/rendering/verbatim/alias_rules.bzl", + "@@rules_rust+//crate_universe:src/select.rs", + "@@rules_rust+//crate_universe:src/splicing.rs", + "@@rules_rust+//crate_universe:src/splicing/cargo_config.rs", + "@@rules_rust+//crate_universe:src/splicing/crate_index_lookup.rs", + "@@rules_rust+//crate_universe:src/splicing/splicer.rs", + "@@rules_rust+//crate_universe:src/test.rs", + "@@rules_rust+//crate_universe:src/utils.rs", + "@@rules_rust+//crate_universe:src/utils/starlark.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/glob.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/label.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/select.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/select_dict.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/select_list.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/select_scalar.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/select_set.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/serialize.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/target_compatible_with.rs", + "@@rules_rust+//crate_universe:src/utils/symlink.rs", + "@@rules_rust+//crate_universe:src/utils/target_triple.rs" + ], + "binary": "cargo-bazel", + "cargo_lockfile": "@@rules_rust+//crate_universe:Cargo.lock", + "cargo_toml": "@@rules_rust+//crate_universe:Cargo.toml", + "version": "1.93.1", + "timeout": 900, + "rust_toolchain_cargo_template": "@rust_host_tools//:bin/{tool}", + "rust_toolchain_rustc_template": "@rust_host_tools//:bin/{tool}", + "compressed_windows_toolchain_names": false + } + } + }, + "moduleExtensionMetadata": { + "explicitRootModuleDirectDeps": [ + "cargo_bazel_bootstrap" + ], + "explicitRootModuleDirectDevDeps": [], + "useAllRepos": "NO", + "reproducible": false + } + } + }, "@@rules_zig+//zig:extensions.bzl%zig": { "general": { "bzlTransitiveDigest": "VGLOg2kVw+STO9R5OrQnLFbv7Vl5TPdtIebIs324IpU=", @@ -1571,114 +2006,6 @@ }, "facts": { "@@rules_go+//go:extensions.bzl%go_sdk": { - "1.19.1": { - "darwin_amd64": [ - "go1.19.1.darwin-amd64.tar.gz", - "b2828a2b05f0d2169afc74c11ed010775bf7cf0061822b275697b2f470495fb7" - ], - "darwin_arm64": [ - "go1.19.1.darwin-arm64.tar.gz", - "e46aecce83a9289be16ce4ba9b8478a5b89b8aa0230171d5c6adbc0c66640548" - ], - "freebsd_386": [ - "go1.19.1.freebsd-386.tar.gz", - "cfaca8c1d5784d2bc21e12d8893cfd2dc885a60db4c1a9a95e4ffc694d0925ce" - ], - "freebsd_amd64": [ - "go1.19.1.freebsd-amd64.tar.gz", - "db5b8f232e12c655cc6cde6af1adf4d27d842541807802d747c86161e89efa0a" - ], - "linux_386": [ - "go1.19.1.linux-386.tar.gz", - "9acc57342400c5b0c2da07b5b01b50da239dd4a7fad41a1fb56af8363ef4133f" - ], - "linux_amd64": [ - "go1.19.1.linux-amd64.tar.gz", - "acc512fbab4f716a8f97a8b3fbaa9ddd39606a28be6c2515ef7c6c6311acffde" - ], - "linux_arm64": [ - "go1.19.1.linux-arm64.tar.gz", - "49960821948b9c6b14041430890eccee58c76b52e2dbaafce971c3c38d43df9f" - ], - "linux_armv6l": [ - "go1.19.1.linux-armv6l.tar.gz", - "efe93f5671621ee84ce5e262e1e21acbc72acefbaba360f21778abd083d4ad16" - ], - "linux_ppc64le": [ - "go1.19.1.linux-ppc64le.tar.gz", - "4137984aa353de9c5ec1bd8fb3cd00a0624b75eafa3d4ec13d2f3f48261dba2e" - ], - "linux_s390x": [ - "go1.19.1.linux-s390x.tar.gz", - "ca1005cc80a3dd726536b4c6ea5fef0318939351ff273eff420bd66a377c74eb" - ], - "windows_386": [ - "go1.19.1.windows-386.zip", - "bc7043e7a9a8d34aacd06f8c2f70e166d1d148f6800814cff790c45b9ab31cee" - ], - "windows_amd64": [ - "go1.19.1.windows-amd64.zip", - "b33584c1d93b0e9c783de876b7aa99d3018bdeccd396aeb6d516a74e9d88d55f" - ], - "windows_arm64": [ - "go1.19.1.windows-arm64.zip", - "d8cf3f04762fa7d5d9c82dfa15b5adaae2404463af3bc8dcd7f89837512501fe" - ] - }, - "1.20.2": { - "darwin_amd64": [ - "go1.20.2.darwin-amd64.tar.gz", - "c93b8ced9517d07e1cd4c362c6e2d5242cb139e29b417a328fbf19aded08764c" - ], - "darwin_arm64": [ - "go1.20.2.darwin-arm64.tar.gz", - "7343c87f19e79c0063532e82e1c4d6f42175a32d99f7a4d15e658e88bf97f885" - ], - "freebsd_386": [ - "go1.20.2.freebsd-386.tar.gz", - "14f9be2004e042b3a64d0facb0c020756a9084a5c7333e33b0752b393b6016ea" - ], - "freebsd_amd64": [ - "go1.20.2.freebsd-amd64.tar.gz", - "b41b67b4f1b56797a7cecf6ee7f47fcf4f93960b2788a3683c07dd009d30b2a4" - ], - "linux_386": [ - "go1.20.2.linux-386.tar.gz", - "ee240ed33ae57504c41f04c12236aeaa17fbeb6ea9fcd096cd9dc7a89d10d4db" - ], - "linux_amd64": [ - "go1.20.2.linux-amd64.tar.gz", - "4eaea32f59cde4dc635fbc42161031d13e1c780b87097f4b4234cfce671f1768" - ], - "linux_arm64": [ - "go1.20.2.linux-arm64.tar.gz", - "78d632915bb75e9a6356a47a42625fd1a785c83a64a643fedd8f61e31b1b3bef" - ], - "linux_armv6l": [ - "go1.20.2.linux-armv6l.tar.gz", - "d79d56bafd6b52b8d8cbe3f8e967caaac5383a23d7a4fa9ac0e89778cd16a076" - ], - "linux_ppc64le": [ - "go1.20.2.linux-ppc64le.tar.gz", - "850564ddb760cb703db63bf20182dc4407abd2ff090a95fa66d6634d172fd095" - ], - "linux_s390x": [ - "go1.20.2.linux-s390x.tar.gz", - "8da24c5c4205fe8115f594237e5db7bcb1d23df67bc1fa9a999954b1976896e8" - ], - "windows_386": [ - "go1.20.2.windows-386.zip", - "31838b291117495bbb93683603e98d5118bfabd2eb318b4d07540bfd524bab86" - ], - "windows_amd64": [ - "go1.20.2.windows-amd64.zip", - "fe439f0e438f7555a7f5f7194ddb6f4a07b0de1fa414385d19f2aeb26d9f43db" - ], - "windows_arm64": [ - "go1.20.2.windows-arm64.zip", - "ac5010c8b8b22849228a8dea698d58b9c7be2195d30c6d778cce0f709858fa64" - ] - }, "1.21.1": { "aix_ppc64": [ "go1.21.1.aix-ppc64.tar.gz", @@ -1837,6 +2164,488 @@ "d7a2472999a80dbf6bbcb6783a2b4299916055473263117295022f18e7fad487" ] }, + "1.22.0": { + "aix_ppc64": [ + "go1.22.0.aix-ppc64.tar.gz", + "190e105fc4133a8b5bb1492f368fa89aa4b729270441120714be7ee82e871ebc" + ], + "darwin_amd64": [ + "go1.22.0.darwin-amd64.tar.gz", + "ebca81df938d2d1047cc992be6c6c759543cf309d401b86af38a6aed3d4090f4" + ], + "darwin_arm64": [ + "go1.22.0.darwin-arm64.tar.gz", + "bf8e388b09134164717cd52d3285a4ab3b68691b80515212da0e9f56f518fb1e" + ], + "dragonfly_amd64": [ + "go1.22.0.dragonfly-amd64.tar.gz", + "357ab446200effa26c73ccaf3e8551426428950bf2490859fb296a09e53228b1" + ], + "freebsd_386": [ + "go1.22.0.freebsd-386.tar.gz", + "b8065da37783e8b9e7086365a54d74537e832c92311b61101a66989ab2458d8e" + ], + "freebsd_amd64": [ + "go1.22.0.freebsd-amd64.tar.gz", + "50f421c7f217083ac94aab1e09400cb9c2fea7d337679ec11f1638a11460da30" + ], + "freebsd_arm64": [ + "go1.22.0.freebsd-arm64.tar.gz", + "e23385e5c640787fa02cd58f2301ea09e162c4d99f8ca9fa6d52766f428a933d" + ], + "freebsd_armv6l": [ + "go1.22.0.freebsd-arm.tar.gz", + "c9c8b305f90903536f4981bad9f029828c2483b3216ca1783777344fbe603f2d" + ], + "freebsd_riscv64": [ + "go1.22.0.freebsd-riscv64.tar.gz", + "c8f94d1de6024546194d58e7b9370dc7ea06176aad94a675b0062c25c40cb645" + ], + "illumos_amd64": [ + "go1.22.0.illumos-amd64.tar.gz", + "d6792f11ad6ee5fc42d2fe51e1f1683471aa2ee4f20e3ad1be22a4afdbd38e7f" + ], + "linux_386": [ + "go1.22.0.linux-386.tar.gz", + "1e209c4abde069067ac9afb341c8003db6a210f8173c77777f02d3a524313da3" + ], + "linux_amd64": [ + "go1.22.0.linux-amd64.tar.gz", + "f6c8a87aa03b92c4b0bf3d558e28ea03006eb29db78917daec5cfb6ec1046265" + ], + "linux_arm64": [ + "go1.22.0.linux-arm64.tar.gz", + "6a63fef0e050146f275bf02a0896badfe77c11b6f05499bb647e7bd613a45a10" + ], + "linux_armv6l": [ + "go1.22.0.linux-armv6l.tar.gz", + "0525f92f79df7ed5877147bce7b955f159f3962711b69faac66bc7121d36dcc4" + ], + "linux_loong64": [ + "go1.22.0.linux-loong64.tar.gz", + "b4b1d425cc113608452a32768469b6e34e538fd072bde9f508a75c8dbbdb843f" + ], + "linux_mips": [ + "go1.22.0.linux-mips.tar.gz", + "ec0f9194df750c78492c02e4c70ffe6f3864f2511e47c894415320db752719f8" + ], + "linux_mips64": [ + "go1.22.0.linux-mips64.tar.gz", + "47e938d215e4968ab42afb7307792e5e64184a717e8f176d0de7c411af96d63d" + ], + "linux_mips64le": [ + "go1.22.0.linux-mips64le.tar.gz", + "c162a1a516b8bc8839fc0c0283ad90f6b511e5aca3da4939ed4800f124b9e72c" + ], + "linux_mipsle": [ + "go1.22.0.linux-mipsle.tar.gz", + "6ce3e9a06e3a8ca0312dc1f85046b6914c19852eba5393c5cbf26acf698f8977" + ], + "linux_ppc64": [ + "go1.22.0.linux-ppc64.tar.gz", + "5ae23bf460731eb078f5700b01a50a303308b9f7174a2994892e28bf061f7c85" + ], + "linux_ppc64le": [ + "go1.22.0.linux-ppc64le.tar.gz", + "0e57f421df9449066f00155ce98a5be93744b3d81b00ee4c2c9b511be2a31d93" + ], + "linux_riscv64": [ + "go1.22.0.linux-riscv64.tar.gz", + "afe9cedcdbd6fdff27c57efd30aa5ce0f666f471fed5fa96cd4fb38d6b577086" + ], + "linux_s390x": [ + "go1.22.0.linux-s390x.tar.gz", + "2e546a3583ba7bd3988f8f476245698f6a93dfa9fe206a8ca8f85c1ceecb2446" + ], + "netbsd_386": [ + "go1.22.0.netbsd-386.tar.gz", + "9b7e2dbd43a169bff18bf533a6c2f462eebe04126ab01c138d0d669c046e1658" + ], + "netbsd_amd64": [ + "go1.22.0.netbsd-amd64.tar.gz", + "b11995c271d2256dfe85cf54882ca3655e18c49c4d7db0502bff9977767894e7" + ], + "netbsd_arm64": [ + "go1.22.0.netbsd-arm64.tar.gz", + "499918ccfedde78264d194244d251bf41d95cf606cc0adad13b36b8103cb974f" + ], + "netbsd_armv6l": [ + "go1.22.0.netbsd-arm.tar.gz", + "b57a3aa9c862300ec0ee8609ce5e0f430f132044f351677fd34711a504081872" + ], + "openbsd_386": [ + "go1.22.0.openbsd-386.tar.gz", + "379e0829104c18a139d78b33378e6bd3ab2e0792f6c43b4c38e7f20d8d77b99d" + ], + "openbsd_amd64": [ + "go1.22.0.openbsd-amd64.tar.gz", + "ceb0c97ffc3bfaf74e1df843cb8571d7fc3173a08432f0f42112495df6a31520" + ], + "openbsd_arm64": [ + "go1.22.0.openbsd-arm64.tar.gz", + "358801cab7122ab50d7d92727644f26d818e9e973403f09e85c2e935a625db75" + ], + "openbsd_armv6l": [ + "go1.22.0.openbsd-arm.tar.gz", + "8af5aea3df539bc95ed412c0a176fe84baced70ea1dd29f4aa82d0e9ce27fd9f" + ], + "plan9_386": [ + "go1.22.0.plan9-386.tar.gz", + "fa42c545c9025c45ca9af176dc13a0f4af0cc26bacff6fcb35bb4a170ac538e8" + ], + "plan9_amd64": [ + "go1.22.0.plan9-amd64.tar.gz", + "d8cf64f37a1dfd8e190c5a303c43ab2d49324868f098d88a3106072d137a5a0b" + ], + "plan9_armv6l": [ + "go1.22.0.plan9-arm.tar.gz", + "86fd6165f0cbb47ad551094f74b3e5a6c5e09de858d8b99de72d978d41be6e2a" + ], + "solaris_amd64": [ + "go1.22.0.solaris-amd64.tar.gz", + "a6c12651768d3a74f16104502b4b7bef513ea6b646d99990a28d934c261d1689" + ], + "windows_386": [ + "go1.22.0.windows-386.zip", + "553d44928509965cbda02a45b35ab01cf8b925534bc526a34e2d9dc7794b57e8" + ], + "windows_amd64": [ + "go1.22.0.windows-amd64.zip", + "78b3158fe3aa358e0b6c9f26ecd338f9a11441e88bc434ae2e9f0ca2b0cc4dd3" + ], + "windows_arm64": [ + "go1.22.0.windows-arm64.zip", + "31a61e41d06a3bb2189a303f5f3e777ca4b454eff439f0a67bc2b166330021f4" + ], + "windows_armv6l": [ + "go1.22.0.windows-arm.zip", + "495c7dfaea4e2bf48643662bb622e4ce6378d6d9840015238ad4b8792b99ddbf" + ] + }, + "1.22.4": { + "aix_ppc64": [ + "go1.22.4.aix-ppc64.tar.gz", + "b9647fa9fc83a0cc5d4f092a19eaeaecf45f063a5aa7d4962fde65aeb7ae6ce1" + ], + "darwin_amd64": [ + "go1.22.4.darwin-amd64.tar.gz", + "c95967f50aa4ace34af0c236cbdb49a9a3e80ee2ad09d85775cb4462a5c19ed3" + ], + "darwin_arm64": [ + "go1.22.4.darwin-arm64.tar.gz", + "242b78dc4c8f3d5435d28a0d2cec9b4c1aa999b601fb8aa59fb4e5a1364bf827" + ], + "dragonfly_amd64": [ + "go1.22.4.dragonfly-amd64.tar.gz", + "f2fbb51af4719d3616efb482d6ed2b96579b474156f85a7ddc6f126764feec4b" + ], + "freebsd_386": [ + "go1.22.4.freebsd-386.tar.gz", + "7c54884bb9f274884651d41e61d1bc12738863ad1497e97ea19ad0e9aa6bf7b5" + ], + "freebsd_amd64": [ + "go1.22.4.freebsd-amd64.tar.gz", + "88d44500e1701dd35797619774d6dd51bf60f45a8338b0a82ddc018e4e63fb78" + ], + "freebsd_arm64": [ + "go1.22.4.freebsd-arm64.tar.gz", + "726dc093cf020277be45debf03c3b02b43c2efb3e2a5d4fba8f52579d65327dc" + ], + "freebsd_armv6l": [ + "go1.22.4.freebsd-arm.tar.gz", + "3d9efe47db142a22679aba46b1772e3900b0d87ae13bd2b3bc80dbf2ac0b2cd6" + ], + "freebsd_riscv64": [ + "go1.22.4.freebsd-riscv64.tar.gz", + "5f6b67e5e32f1d6ccb2d4dcb44934a5e2e870a877ba7443d86ec43cfc28afa71" + ], + "illumos_amd64": [ + "go1.22.4.illumos-amd64.tar.gz", + "d56ecc2f85b6418a21ef83879594d0c42ab4f65391a676bb12254870e6690d63" + ], + "linux_386": [ + "go1.22.4.linux-386.tar.gz", + "47a2a8d249a91eb8605c33bceec63aedda0441a43eac47b4721e3975ff916cec" + ], + "linux_amd64": [ + "go1.22.4.linux-amd64.tar.gz", + "ba79d4526102575196273416239cca418a651e049c2b099f3159db85e7bade7d" + ], + "linux_arm64": [ + "go1.22.4.linux-arm64.tar.gz", + "a8e177c354d2e4a1b61020aca3562e27ea3e8f8247eca3170e3fa1e0c2f9e771" + ], + "linux_armv6l": [ + "go1.22.4.linux-armv6l.tar.gz", + "e2b143fbacbc9cbd448e9ef41ac3981f0488ce849af1cf37e2341d09670661de" + ], + "linux_loong64": [ + "go1.22.4.linux-loong64.tar.gz", + "e2ff9436e4b34bf6926b06d97916e26d67a909a2effec17967245900f0816f1d" + ], + "linux_mips": [ + "go1.22.4.linux-mips.tar.gz", + "73f0dcc60458c4770593b05a7bc01cc0d31fc98f948c0c2334812c7a1f2fc3f1" + ], + "linux_mips64": [ + "go1.22.4.linux-mips64.tar.gz", + "417af97fc2630a647052375768be4c38adcc5af946352ea5b28613ea81ca5d45" + ], + "linux_mips64le": [ + "go1.22.4.linux-mips64le.tar.gz", + "7486e2d7dd8c98eb44df815ace35a7fe7f30b7c02326e3741bd934077508139b" + ], + "linux_mipsle": [ + "go1.22.4.linux-mipsle.tar.gz", + "69479c8aad301e459a8365b40cad1074a0dbba5defb9291669f94809c4c4be6e" + ], + "linux_ppc64": [ + "go1.22.4.linux-ppc64.tar.gz", + "dd238847e65bc3e2745caca475a5db6522a2fcf85cf6c38fc36a06642b19efd7" + ], + "linux_ppc64le": [ + "go1.22.4.linux-ppc64le.tar.gz", + "a3e5834657ef92523f570f798fed42f1f87bc18222a16815ec76b84169649ec4" + ], + "linux_riscv64": [ + "go1.22.4.linux-riscv64.tar.gz", + "56a827ff7dc6245bcd7a1e9288dffaa1d8b0fd7468562264c1523daf3b4f1b4a" + ], + "linux_s390x": [ + "go1.22.4.linux-s390x.tar.gz", + "7590c3e278e2dc6040aae0a39da3ca1eb2e3921673a7304cc34d588c45889eec" + ], + "netbsd_386": [ + "go1.22.4.netbsd-386.tar.gz", + "ddd2eebe34471a2502de6c5dad04ab27c9fc80cbde7a9ad5b3c66ecec4504e1d" + ], + "netbsd_amd64": [ + "go1.22.4.netbsd-amd64.tar.gz", + "33af79f6f935f6fbacc5d23876450b3567b79348fc065beef8e64081127dd234" + ], + "netbsd_arm64": [ + "go1.22.4.netbsd-arm64.tar.gz", + "c9a2971dec9f6d320c6f2b049b2353c6d0a2d35e87b8a4b2d78a2f0d62545f8e" + ], + "netbsd_armv6l": [ + "go1.22.4.netbsd-arm.tar.gz", + "fa3550ebd5375a70b3bcd342b5a71f4bd271dcbbfaf4eabefa2144ab5d8924b6" + ], + "openbsd_386": [ + "go1.22.4.openbsd-386.tar.gz", + "d21af022331bfdc2b5b161d616c3a1a4573d33cf7a30416ee509a8f3641deb47" + ], + "openbsd_amd64": [ + "go1.22.4.openbsd-amd64.tar.gz", + "72c0094c43f7e5722ec49c2a3e9dfa7a1123ac43a5f3a63eecf3e3795d3ff0ae" + ], + "openbsd_arm64": [ + "go1.22.4.openbsd-arm64.tar.gz", + "a7ab8d4e0b02bf06ed144ba42c61c0e93ee00f2b433415dfd4ad4b6e79f31650" + ], + "openbsd_armv6l": [ + "go1.22.4.openbsd-arm.tar.gz", + "1096831ea3c5ea3ca57d14251d9eda3786889531eb40d7d6775dcaa324d4b065" + ], + "openbsd_ppc64": [ + "go1.22.4.openbsd-ppc64.tar.gz", + "9716327c8a628358798898dc5148c49dbbeb5196bf2cbf088e550721a6e4f60b" + ], + "plan9_386": [ + "go1.22.4.plan9-386.tar.gz", + "a8dd4503c95c32a502a616ab78870a19889c9325fe9bd31eb16dd69346e4bfa8" + ], + "plan9_amd64": [ + "go1.22.4.plan9-amd64.tar.gz", + "5423a25808d76fe5aca8607a2e5ac5673abf45446b168cb5e9d8519ee9fe39a1" + ], + "plan9_armv6l": [ + "go1.22.4.plan9-arm.tar.gz", + "6af939ad583f5c85c09c53728ab7d38c3cc2b39167562d6c18a07c5c6608b370" + ], + "solaris_amd64": [ + "go1.22.4.solaris-amd64.tar.gz", + "e8cabe69c03085725afdb32a6f9998191a3e55a747b270d835fd05000d56abba" + ], + "windows_386": [ + "go1.22.4.windows-386.zip", + "aca4e2c37278a10f1c70dd0df142f7d66b50334fcee48978d409202d308d6d25" + ], + "windows_amd64": [ + "go1.22.4.windows-amd64.zip", + "26321c4d945a0035d8a5bc4a1965b0df401ff8ceac66ce2daadabf9030419a98" + ], + "windows_arm64": [ + "go1.22.4.windows-arm64.zip", + "8a2daa9ea28cbdafddc6171aefed384f4e5b6e714fb52116fe9ed25a132f37ed" + ], + "windows_armv6l": [ + "go1.22.4.windows-arm.zip", + "5fcd0671a49cecf39b41021621ee1b6e7aa1370f37122b72e80d4fd4185833b6" + ] + }, + "1.24.6": { + "aix_ppc64": [ + "go1.24.6.aix-ppc64.tar.gz", + "5cafb2927af95969fc3d0884be1ad4df54f19103d86dda928d9f76494a5ae6a6" + ], + "darwin_amd64": [ + "go1.24.6.darwin-amd64.tar.gz", + "4a8d7a32052f223e71faab424a69430455b27b3fff5f4e651f9d97c3e51a8746" + ], + "darwin_arm64": [ + "go1.24.6.darwin-arm64.tar.gz", + "4e29202c49573b953be7cc3500e1f8d9e66ddd12faa8cf0939a4951411e09a2a" + ], + "dragonfly_amd64": [ + "go1.24.6.dragonfly-amd64.tar.gz", + "c6e21bf8347f7a1c653dd4136c8c27a858515e98501d2843023a4bb3f1f7fb63" + ], + "freebsd_386": [ + "go1.24.6.freebsd-386.tar.gz", + "9cd74ad74f3ad833e92529f2fd9b0d7d9ffaab46307eccadb0afcf9a1ba09553" + ], + "freebsd_amd64": [ + "go1.24.6.freebsd-amd64.tar.gz", + "4983e2b10ae1f754e4eb07e1e589691c7e1d0dc428a92c16bd0e2ba03cc23ed9" + ], + "freebsd_arm": [ + "go1.24.6.freebsd-arm.tar.gz", + "a8da621d8282a91ee17b257a46f2606391c019cc1a7d7be628638792ca8033ad" + ], + "freebsd_arm64": [ + "go1.24.6.freebsd-arm64.tar.gz", + "76a75ad5125217c268029c0ad9c7295cc7f6042fe9cba4bebf9a89f7f42ad8af" + ], + "freebsd_riscv64": [ + "go1.24.6.freebsd-riscv64.tar.gz", + "ac206417d8460662f26d46dc2ad0488b2f9e22039946069ba4b48a0cb646e8b0" + ], + "illumos_amd64": [ + "go1.24.6.illumos-amd64.tar.gz", + "c54199c46ec823c857c52335f859b493593433d71fe479b9e7f95d868a144ae2" + ], + "linux_386": [ + "go1.24.6.linux-386.tar.gz", + "bb5bf69d75e7edbc93339824753a1a4655a928451a2c5e13ff90959ad69e065b" + ], + "linux_amd64": [ + "go1.24.6.linux-amd64.tar.gz", + "bbca37cc395c974ffa4893ee35819ad23ebb27426df87af92e93a9ec66ef8712" + ], + "linux_arm64": [ + "go1.24.6.linux-arm64.tar.gz", + "124ea6033a8bf98aa9fbab53e58d134905262d45a022af3a90b73320f3c3afd5" + ], + "linux_armv6l": [ + "go1.24.6.linux-armv6l.tar.gz", + "7feb4d25f5e72f94fda81c99d4adb6630dfa2c35211e0819417d53af6e71809e" + ], + "linux_loong64": [ + "go1.24.6.linux-loong64.tar.gz", + "8424d9c52b254255d2720769366af73802747e4eaf339c6e86bb52e5a1809fcb" + ], + "linux_mips": [ + "go1.24.6.linux-mips.tar.gz", + "791964a6ce65f604a2099b7e61f71128cc7a3a4fcee858218d2ce99033fc2b09" + ], + "linux_mips64": [ + "go1.24.6.linux-mips64.tar.gz", + "2f5282fdf2eaca44515728d518caec888b0dc4948effecb8e7f2ddff1ff9aa9e" + ], + "linux_mips64le": [ + "go1.24.6.linux-mips64le.tar.gz", + "64ddc6b28907b4ab997f2d81c226be403511a59a6100536560af26ccfb10b4a4" + ], + "linux_mipsle": [ + "go1.24.6.linux-mipsle.tar.gz", + "5ca12d2d4939dea977100e76885f4e08cd3ef3b73823853527b58ecac4598fd3" + ], + "linux_ppc64": [ + "go1.24.6.linux-ppc64.tar.gz", + "9a4a61ffbba7a58b3ef97a6d3b2b394168f613f9ca6092016981a88c78a57f86" + ], + "linux_ppc64le": [ + "go1.24.6.linux-ppc64le.tar.gz", + "63fc9559a3d6dfd63aa902f714375b879bbc848466181c035c122489b9646e27" + ], + "linux_riscv64": [ + "go1.24.6.linux-riscv64.tar.gz", + "e92c19ff15a9004fe43e1f62d433555250ef9fe3fabee3dc29ebd4802c2b8021" + ], + "linux_s390x": [ + "go1.24.6.linux-s390x.tar.gz", + "4cde28d9ffb6eef86bf8dac6852a45db335009f67e60bee3e477dd0ba0ff9704" + ], + "netbsd_386": [ + "go1.24.6.netbsd-386.tar.gz", + "d238045676104d2aaad9da53c4ba3f489cf81b26d3315d2caad2b6e240664423" + ], + "netbsd_amd64": [ + "go1.24.6.netbsd-amd64.tar.gz", + "9f8a9245cf146f1ebdac6785f71646c3dd0edf07dcdb8e5a8df85ec6447e5308" + ], + "netbsd_arm": [ + "go1.24.6.netbsd-arm.tar.gz", + "2359c8af6b2d6979910060478a57fd216824682e0908ea1a4412ea8dc4a55db7" + ], + "netbsd_arm64": [ + "go1.24.6.netbsd-arm64.tar.gz", + "99101158c796997ec85023d395c133fa90c782bd5faa56f8c3824817d28b0ee8" + ], + "openbsd_386": [ + "go1.24.6.openbsd-386.tar.gz", + "b6193420489f24d5560b87ccb38caf52cecb308d892d99bf21d1ef0d2c99de04" + ], + "openbsd_amd64": [ + "go1.24.6.openbsd-amd64.tar.gz", + "8dc0d94afbc65be9cafebe543b88a841dcf3429e50f668ea714200ef9f4489bd" + ], + "openbsd_arm": [ + "go1.24.6.openbsd-arm.tar.gz", + "c29563d14d8e4b5fa89f130afac40abba7a1ee1e87fcc759a95f1e9b5831bf68" + ], + "openbsd_arm64": [ + "go1.24.6.openbsd-arm64.tar.gz", + "3d97bdc5e1997cccfaa398242c52e8a97d3b626d988a65248bbdbf2f1ab188bf" + ], + "openbsd_ppc64": [ + "go1.24.6.openbsd-ppc64.tar.gz", + "54cf517933db0bff58f3e527e7fa194f505d562c71b1fb0050d3365fcdda4914" + ], + "openbsd_riscv64": [ + "go1.24.6.openbsd-riscv64.tar.gz", + "f7bdd8f882ecbc01ff6c1782ce3e29298445dd99b3dd4299423257df307c7b0d" + ], + "plan9_386": [ + "go1.24.6.plan9-386.tar.gz", + "9bc0a801dc0d8d0705ac9493219b926b2e3c0105b8921c6c2fce69d14c4b9007" + ], + "plan9_amd64": [ + "go1.24.6.plan9-amd64.tar.gz", + "7133c28acc277d3c5e8317fd9454b3a0902138c384466f818644b504b8b89ffe" + ], + "plan9_arm": [ + "go1.24.6.plan9-arm.tar.gz", + "c4e2aa8aa4353f75fc083ca728b73917467cef88c6f5f917ca420578672c6d58" + ], + "solaris_amd64": [ + "go1.24.6.solaris-amd64.tar.gz", + "cfcb4f1f7e987c6dee893c4401546516cc271c1ac7d7fb37a32b25343dc7df32" + ], + "windows_386": [ + "go1.24.6.windows-386.zip", + "39b4d31b933d2f7c8913e70fb0fffef27252e441c386eac5c13632cbb28dfb6e" + ], + "windows_amd64": [ + "go1.24.6.windows-amd64.zip", + "4fbc8af2cfca9e5059019b5150a426eb78e1e57718bf08f0e52b1c942a2782bf" + ], + "windows_arm64": [ + "go1.24.6.windows-arm64.zip", + "45c41b237d00e92e4cf8adce11b4c5258048b47a92bfbb1f4ef3b928d6fcb0b2" + ] + }, "1.25.0": { "aix_ppc64": [ "go1.25.0.aix-ppc64.tar.gz", diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 99ce5d7c..7281d2d8 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -23,22 +23,22 @@ def _cpu_pjrt_plugin_impl(mctx): http_archive( name = "libpjrt_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "f71402ff75bb23afc49511d7b666a48371749e663d653da4fd70b95f6a5e77ab", - url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-04-30T17-13-00Z/pjrt-cpu_linux-amd64.tar.gz", + sha256 = "65e631db0f842845e7799d245a414b361a3c3e77bf4cc0547c20c71f28a9fd70", + url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-07-03T00-10-30Z/pjrt-cpu_linux-amd64.tar.gz", ) http_archive( name = "libpjrt_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "8ba86d6a100e89b9ab8912d3ca85e0d1ba1ffb36d7961f8b41cdc1a29d9c5df8", - url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-04-30T17-13-00Z/pjrt-cpu_darwin-amd64.tar.gz", + sha256 = "22130f752abdaaa0f3ff48e71b9191ce74b29cd3fdb0c4784f04e4c4a436f25e", + url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-07-03T00-10-30Z/pjrt-cpu_darwin-amd64.tar.gz", ) http_archive( name = "libpjrt_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "abc8403005cf48c2014fa8417dae3e92f25fc97810f840bb5368e9a5df487fdd", - url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-04-30T17-13-00Z/pjrt-cpu_darwin-arm64.tar.gz", + sha256 = "14c85504d801c75fa8d157ce951a2644d8a8d7983346b3ac281aa7f64abf8390", + url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-07-03T00-10-30Z/pjrt-cpu_darwin-arm64.tar.gz", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index fbfafb58..d75edc23 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -205,15 +205,15 @@ _UBUNTU_PACKAGES = { ], } -PJRT_CUDA_RELEASE = "manual-2026-06-30T00-16-30Z" +PJRT_CUDA_RELEASE = "manual-2026-07-03T00-10-30Z" _PJRT_CUDA_ASSETS = { "amd64": { - "sha256": "50c7219b111d0d74ad66c4748cb87691e7e146b7e21fb6ed66ae6af6d97589ca", + "sha256": "6380f724fe21b25dc9231f3ec468ae92e39d21f4bae9377bfa8ff01972521e0d", "url": "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-cuda_linux-amd64.tar.gz", }, "arm64": { - "sha256": "dcaec78fb141b52884f98e28cb12683d5074fb26aabe8cc56b208a6563fa1d9f", + "sha256": "98dc33f0740bc37f3a25611f5aad423cc46eb69679e4d45a22f7fc4dfa3efd8d", "url": "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-cuda_linux-arm64.tar.gz", }, } diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index 59afdb8d..22f9a91a 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -1,8 +1,8 @@ load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") -PJRT_ONEAPI_RELEASE = "manual-2026-06-23T00-20-00Z" -PJRT_ONEAPI_ARTIFACT_SHA256 = "97e0892e7d3815118c897d4f4134004f8bca177332de8be2384f71f6f55e954f" +PJRT_ONEAPI_RELEASE = "manual-2026-07-03T00-10-30Z" +PJRT_ONEAPI_ARTIFACT_SHA256 = "59d888d5cd15b496afd905000faba16ea2402e7ec53d3c4facf6e913cccd4113" PJRT_ONEAPI_ARTIFACT_URL = "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-oneapi_linux-amd64.tar.gz".format( release = PJRT_ONEAPI_RELEASE, ) diff --git a/third_party/abseil-cpp/bmi2intrin.patch b/third_party/abseil-cpp/bmi2intrin.patch new file mode 100644 index 00000000..8273731b --- /dev/null +++ b/third_party/abseil-cpp/bmi2intrin.patch @@ -0,0 +1,13 @@ +diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h +index 0d05ad7..85ed32d 100644 +--- a/absl/container/internal/raw_hash_set.h ++++ b/absl/container/internal/raw_hash_set.h +@@ -226,7 +226,7 @@ static_assert(std::numeric_limits::is_iec559, + #endif + + #ifdef __BMI2__ +-#include ++#include + #endif // __BMI2__ + + #if defined(__SSE2__) || defined(__aarch64__) || \ \ No newline at end of file diff --git a/third_party/grpc/grpc++_unsecure-visibility.patch b/third_party/grpc/grpc++_unsecure-visibility.patch new file mode 100644 index 00000000..3b4013ab --- /dev/null +++ b/third_party/grpc/grpc++_unsecure-visibility.patch @@ -0,0 +1,13 @@ +diff --git a/BUILD b/BUILD +index 0e4bda7e16..dbe1708421 100644 +--- a/BUILD ++++ b/BUILD +@@ -1236,7 +1236,7 @@ grpc_cc_library( + "avoid_dep", + "nofixdeps", + ], +- visibility = ["//bazel:grpc++_unsecure"], ++ visibility = ["//visibility:public"], + deps = [ + "channel_arg_names", + "global_callback_hook", diff --git a/third_party/grpc/protobuf-35-upb-load.patch b/third_party/grpc/protobuf-35-upb-load.patch new file mode 100644 index 00000000..7311b981 --- /dev/null +++ b/third_party/grpc/protobuf-35-upb-load.patch @@ -0,0 +1,14 @@ +diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl +index e0d47a29e1..e33f40d758 100644 +--- a/bazel/grpc_build_system.bzl ++++ b/bazel/grpc_build_system.bzl +@@ -31,7 +31,8 @@ load("@build_bazel_apple_support//rules:universal_binary.bzl", "universal_binary + load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test") + load("@build_bazel_rules_apple//apple/testing/default_runner:ios_test_runner.bzl", "ios_test_runner") + load("@com_google_protobuf//bazel:cc_proto_library.bzl", "cc_proto_library") +-load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_proto_library", "upb_proto_reflection_library") ++load("@com_google_protobuf//bazel:upb_c_proto_library.bzl", upb_proto_library = "upb_c_proto_library") ++load("@com_google_protobuf//bazel:upb_proto_reflection_library.bzl", "upb_proto_reflection_library") + load("@rules_cc//cc:cc_binary.bzl", "cc_binary") + load("@rules_cc//cc:cc_library.bzl", "cc_library") + load("@rules_cc//cc:cc_test.bzl", "cc_test") diff --git a/third_party/xla/pjrt-device-event-typedef.patch b/third_party/xla/pjrt-device-event-typedef.patch deleted file mode 100644 index afacb555..00000000 --- a/third_party/xla/pjrt-device-event-typedef.patch +++ /dev/null @@ -1,14 +0,0 @@ -diff --git a/xla/pjrt/c/pjrt_c_api_device_event.h b/xla/pjrt/c/pjrt_c_api_device_event.h -index 5d33c05cf7..5f1dbbf6f1 100644 ---- a/xla/pjrt/c/pjrt_c_api_device_event.h -+++ b/xla/pjrt/c/pjrt_c_api_device_event.h -@@ -64,7 +64,7 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_DeviceEvent_FunctionTable, - - // A PJRT_DeviceEvent is a pair of pointers containing both type information - // and the actual opaque device event object. See: xla::PjRtDeviceEventRef. --struct PJRT_DeviceEvent { -+typedef struct PJRT_DeviceEvent { - const struct PJRT_DeviceEvent_FunctionTable* vtable; - void* device_event; --}; -+} PJRT_DeviceEvent; diff --git a/third_party/xla/repo.bzl b/third_party/xla/repo.bzl index 405f7bc9..3e153172 100644 --- a/third_party/xla/repo.bzl +++ b/third_party/xla/repo.bzl @@ -4,9 +4,8 @@ def repo(): git_repository( name = "xla", remote = "https://github.com/openxla/xla.git", - commit = "6d7284490a942e5be5a771ded9181f9c602fd692", + commit = "41370d1124c74d7b93a207136a636d8c631cbed9", patches = [ - "//third_party/xla:pjrt-device-event-typedef.patch", "//third_party/xla:cuda-root-path-local-defines.patch", ], patch_args = ["-p1"], diff --git a/tools/xspace_to_perfetto/BUILD.bazel b/tools/xspace_to_perfetto/BUILD.bazel index 63658c51..5de164d6 100644 --- a/tools/xspace_to_perfetto/BUILD.bazel +++ b/tools/xspace_to_perfetto/BUILD.bazel @@ -1,4 +1,4 @@ -load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_c_proto_library") +load("@com_google_protobuf//bazel:upb_c_proto_library.bzl", "upb_c_proto_library") load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") upb_c_proto_library( diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index da02522a..78ce31cf 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -1,4 +1,4 @@ -load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_c_proto_library") +load("@com_google_protobuf//bazel:upb_c_proto_library.bzl", "upb_c_proto_library") load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") diff --git a/zml/module.zig b/zml/module.zig index 386db855..c78f1b77 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -644,7 +644,6 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: // NVIDIA recommends these settings // https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_latency_hiding_scheduler", true, upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_llvm_module_compilation_parallelism", true, upb_arena); }, .rocm => { // Use hipBLASLt only From f9183137a1025deae723e3e1a9d70a7792cf454b Mon Sep 17 00:00:00 2001 From: elogir Date: Fri, 3 Jul 2026 14:32:41 +0200 Subject: [PATCH 559/881] examples/llm: fix single-chip TPU sharding crash (#627) A 1-chip TPU mesh collapses to rank 1 (only `link_x`), so the unconditional `link_y` fold tripped InvalidPhysicalAxis. Only fold `link_y` when the physical mesh actually has it. --- examples/llm/models/common.zig | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/llm/models/common.zig b/examples/llm/models/common.zig index c5c6d332..2ec467b7 100644 --- a/examples/llm/models/common.zig +++ b/examples/llm/models/common.zig @@ -49,10 +49,14 @@ pub const Shardings = struct { pub fn init(platform: *zml.Platform) !Shardings { switch (platform.target) { .tpu => { + const has_link_y = platform.physical_mesh.hasAxis(.link_y); + var strategy_experts: zml.Sharding.Strategy = .parseBindings(.{ .experts = .link_x }); - strategy_experts.addFold(.link_x, &.{ .link_x, .link_y }); var strategy_model: zml.Sharding.Strategy = .parseBindings(.{ .model = .link_x }); - strategy_model.addFold(.link_x, &.{ .link_x, .link_y }); + if (has_link_y) { + strategy_experts.addFold(.link_x, &.{ .link_x, .link_y }); + strategy_model.addFold(.link_x, &.{ .link_x, .link_y }); + } return .{ .model = try platform.registerShardingWithStrategy("model", .mesh(.{ .model = .high_bandwidth }), strategy_model), From 73cbeca7cf08304e75528e636fd6cb0d266ecc78 Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Fri, 3 Jul 2026 17:02:23 +0200 Subject: [PATCH 560/881] platform/intel: update intel pjrt and limit the intel device to 2 (#628) --- platforms/oneapi/oneapi.bzl | 4 ++-- platforms/oneapi/oneapi.zig | 4 ++-- zml/platform.zig | 16 +++++++++++----- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index 22f9a91a..7de9fe7f 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -1,8 +1,8 @@ load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") -PJRT_ONEAPI_RELEASE = "manual-2026-07-03T00-10-30Z" -PJRT_ONEAPI_ARTIFACT_SHA256 = "59d888d5cd15b496afd905000faba16ea2402e7ec53d3c4facf6e913cccd4113" +PJRT_ONEAPI_RELEASE = "manual-2026-07-03T15-19-00Z" +PJRT_ONEAPI_ARTIFACT_SHA256 = "dece3736b9521485885770c82f335a5b13dbcf61f53cce7ac9f6c63226eb5293" PJRT_ONEAPI_ARTIFACT_URL = "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-oneapi_linux-amd64.tar.gz".format( release = PJRT_ONEAPI_RELEASE, ) diff --git a/platforms/oneapi/oneapi.zig b/platforms/oneapi/oneapi.zig index d338bcac..6bf172ad 100644 --- a/platforms/oneapi/oneapi.zig +++ b/platforms/oneapi/oneapi.zig @@ -29,8 +29,8 @@ fn hasOneApiDevices(io: std.Io) bool { fn setupOneAPIEnv() void { _ = c.setenv("CCL_LOG_LEVEL", c.getenv("CCL_LOG_LEVEL") orelse "error", 1); - _ = c.setenv("CCL_ATL_TRANSPORT", c.getenv("CCL_ATL_TRANSPORT") orelse "ofi", 1); - _ = c.setenv("CCL_TOPO_P2P_ACCESS", c.getenv("CCL_ATL_TRANSPORT") orelse "1", 1); + _ = c.setenv("CCL_ATL_TRANSPORT", "ofi", 1); + _ = c.setenv("FI_PROVIDER", "shm", 1); } pub fn load(_: std.mem.Allocator, io: std.Io) !*const pjrt.Api { diff --git a/zml/platform.zig b/zml/platform.zig index 34ecb219..5ef97718 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -35,17 +35,23 @@ fn disableXlaLogs() void { } fn validateDeviceCount(target: Target, num_devices: usize) !void { + if (num_devices == 0) { + log.err("The selected platform requires at least 1 device, got {}", .{num_devices}); + return error.MissingDevices; + } switch (target) { - .cpu, .cuda, .rocm, .tpu, .neuron, .oneapi, .metal => { - if (num_devices == 0) { - log.err("Platform {} requires at least 1 device, got {}", .{ target, num_devices }); - return error.ZeroVisibleDevices; - } + .cpu, .cuda, .rocm, .tpu, .neuron, .metal => { if (!std.math.isPowerOfTwo(num_devices)) { log.err("Platform {} requires a power-of-two device count, got {}", .{ target, num_devices }); return error.InvalidDeviceCount; } }, + .oneapi => { + if (num_devices > 2) { + log.err("Platform Intel is limited to a maximum of 2 devices, got {}", .{num_devices}); + return error.InvalidDeviceCount; + } + }, } } From 3b5530833cc62ae20a99430c885ab59c41df8ecd Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sat, 4 Jul 2026 15:40:33 -0400 Subject: [PATCH 561/881] workspace: replace deprecated new_git_repository (#629) ## Summary - replace the legacy `new_git_repository` alias with `git_repository` for `arocc`, `libvaxis`, `linenoise`, and `translate-c` ## Why `git_repository` is the canonical Bazel repository rule. `new_git_repository` doesn't exist at bazel HEAD --- third_party/arocc/repo.bzl | 4 ++-- third_party/libvaxis/repo.bzl | 4 ++-- third_party/linenoise/repo.bzl | 4 ++-- third_party/translate-c/repo.bzl | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/third_party/arocc/repo.bzl b/third_party/arocc/repo.bzl index 9f574c98..98a10e38 100644 --- a/third_party/arocc/repo.bzl +++ b/third_party/arocc/repo.bzl @@ -1,7 +1,7 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") def repo(): - new_git_repository( + git_repository( name = "arocc", remote = "https://github.com/Vexu/arocc.git", commit = "5f5a050569a95ecc40a426f0c3666ae7ef987ede", diff --git a/third_party/libvaxis/repo.bzl b/third_party/libvaxis/repo.bzl index 9396332b..8305d1c0 100644 --- a/third_party/libvaxis/repo.bzl +++ b/third_party/libvaxis/repo.bzl @@ -1,7 +1,7 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") def repo(): - new_git_repository( + git_repository( name = "libvaxis", remote = "https://github.com/rockorager/libvaxis.git", commit = "a3ae1d53feeeeaeb6218de3d38837559811acae4", diff --git a/third_party/linenoise/repo.bzl b/third_party/linenoise/repo.bzl index 7837ac67..1d7a1678 100644 --- a/third_party/linenoise/repo.bzl +++ b/third_party/linenoise/repo.bzl @@ -1,7 +1,7 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") def repo(): - new_git_repository( + git_repository( name = "linenoise", remote = "https://github.com/antirez/linenoise.git", commit = "dc83cc373ac2058030eb3cf5e404959a26fef112", diff --git a/third_party/translate-c/repo.bzl b/third_party/translate-c/repo.bzl index db21bd0d..11598b32 100644 --- a/third_party/translate-c/repo.bzl +++ b/third_party/translate-c/repo.bzl @@ -1,7 +1,7 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") def repo(): - new_git_repository( + git_repository( name = "translate-c", remote = "https://codeberg.org/ziglang/translate-c", commit = "46b5609b5ac4c0a896217d1d984f3ae50e4810b5", From 75b820acadcc3d35b6f4bcf0eee6a573ff6f700e Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Mon, 6 Jul 2026 17:12:45 +0200 Subject: [PATCH 562/881] zml/attention: pad block_m for oneAPI decode (#633) Use the standard minimum/power-of-two block_m sizing for paged attention instead of the raw GQA group size. This keeps oneAPI decode tile sizing valid for Triton IR generation. --- zml/attention/triton_attention.zig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zml/attention/triton_attention.zig b/zml/attention/triton_attention.zig index f904b3f0..9043724e 100644 --- a/zml/attention/triton_attention.zig +++ b/zml/attention/triton_attention.zig @@ -261,8 +261,9 @@ pub const paged = struct { const num_queries_per_kv: usize = num_heads / num_kv_heads; // Intel decode: pack exactly one GQA group per tile (block_q == 1) so the // single decode query token doesn't carry masked-out fp32 acc lanes. + // oneAPI decode keeps one GQA group per tile, padded to a power of two so tt.make_range emits legal Triton IR. const block_m: usize = if (!ctx_.options.is_prefill and isOneapiTarget()) - num_queries_per_kv + std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv) else if (num_queries_per_kv <= 16) 16 else std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv); const block_q: usize = block_m / num_queries_per_kv; const num_tokens: usize = @intCast(q_.dim(.b)); From 6f4e8b83654fb3a6328372e1ed6d7c5ef4841bbd Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 7 Jul 2026 17:42:36 +0200 Subject: [PATCH 563/881] io/vfs: file handle race (#634) --- zml/io/vfs.zig | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/zml/io/vfs.zig b/zml/io/vfs.zig index 45ff4e72..c87caf97 100644 --- a/zml/io/vfs.zig +++ b/zml/io/vfs.zig @@ -1,5 +1,6 @@ const std = @import("std"); +const stdx = @import("stdx"); const vfs = @import("vfs"); const log = std.log.scoped(.@"zml/io/vfs"); @@ -12,7 +13,6 @@ pub const VFS = struct { pub const GCS = vfs.GCS; const VFSBase = vfs.VFSBase; - const Handles = std.ArrayList(Handle); const ClosedHandles = std.ArrayList(u32); const CWD_HANDLE: u32 = 0; @@ -23,7 +23,7 @@ pub const VFS = struct { mutex: std.Io.Mutex = .init, backends: std.StringArrayHashMapUnmanaged(std.Io) = .empty, - handles: Handles = .empty, + handles: stdx.SegmentedList(Handle, 128) = .{}, closed_handles: ClosedHandles = .empty, base: VFSBase, @@ -31,7 +31,7 @@ pub const VFS = struct { pub fn init(allocator: std.mem.Allocator, base_io: std.Io) !VFS { const base = VFSBase.init(base_io); - var handles: Handles = .empty; + var handles: @FieldType(@This(), "handles") = .{}; try handles.append(allocator, .{ .handle = CWD_HANDLE, .backend_idx = null }); try handles.append(allocator, .{ .handle = std.posix.STDIN_FILENO, .backend_idx = null }); try handles.append(allocator, .{ .handle = std.posix.STDOUT_FILENO, .backend_idx = null }); @@ -98,9 +98,9 @@ pub const VFS = struct { defer self.mutex.unlock(self.base.inner); if (self.closed_handles.pop()) |idx| { - return .{ idx, &self.handles.items[idx] }; + return .{ idx, self.handles.at(idx) }; } - return .{ @intCast(self.handles.items.len), try self.handles.addOne(self.allocator) }; + return .{ @intCast(self.handles.len), try self.handles.addOne(self.allocator) }; } fn closeHandle(self: *VFS, idx: u32) !void { @@ -114,7 +114,7 @@ pub const VFS = struct { fn getFileHandle(self: *VFS, file: std.Io.File) struct { *Handle, std.Io } { self.mutex.lockUncancelable(self.base.inner); - const handle = &self.handles.items[@intCast(file.handle)]; + const handle = self.handles.at(@intCast(file.handle)); self.mutex.unlock(self.base.inner); return .{ handle, self.getBackend(handle.backend_idx) }; @@ -124,8 +124,8 @@ pub const VFS = struct { self.mutex.lockUncancelable(self.base.inner); defer self.mutex.unlock(self.base.inner); - if (std.meta.eql(dir, std.Io.Dir.cwd())) return &self.handles.items[CWD_HANDLE]; - return &self.handles.items[@intCast(dir.handle)]; + if (std.meta.eql(dir, std.Io.Dir.cwd())) return self.handles.at(CWD_HANDLE); + return self.handles.at(@intCast(dir.handle)); } fn getScheme(self: *VFS, backend_idx: ?usize) ?[]const u8 { From 8a89d73fb8ec491b614d5a51bb45be1b2cb365a7 Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Wed, 8 Jul 2026 09:33:36 +0200 Subject: [PATCH 564/881] build/tools xprof: refresh xprof-nightly lock (#637) Regenerate the xprof uv lockfile to move xprof-nightly off the removed 2.23.2a20260505 build and onto 2.23.3a20260707. Fixes Bazel fetch failures for //tools/xprof:xprof. --- tools/xprof/requirements.lock.txt | 1537 ++++++++++++++--------------- 1 file changed, 744 insertions(+), 793 deletions(-) diff --git a/tools/xprof/requirements.lock.txt b/tools/xprof/requirements.lock.txt index 7154ccd0..7ca0f132 100644 --- a/tools/xprof/requirements.lock.txt +++ b/tools/xprof/requirements.lock.txt @@ -1,130 +1,133 @@ # This file was autogenerated by uv via the following command: # bazel run //tools/xprof:requirements.update -aiohappyeyeballs==2.6.1 \ - --hash=sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558 \ - --hash=sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 +absl-py==2.5.0 \ + --hash=sha256:0c996f25c0490700fadabe6351630f6111534fa0ae252cc6d2014ea3b141135f \ + --hash=sha256:0f17b89f2a4eaaedc4f28c622998aa690564b3012a396a4ffad0821007fe03ba + # via xprof-nightly +aiohappyeyeballs==2.7.1 \ + --hash=sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d \ + --hash=sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472 # via aiohttp -aiohttp==3.13.5 \ - --hash=sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5 \ - --hash=sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b \ - --hash=sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9 \ - --hash=sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b \ - --hash=sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8 \ - --hash=sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a \ - --hash=sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2 \ - --hash=sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1 \ - --hash=sha256:147b4f501d0292077f29d5268c16bb7c864a1f054d7001c4c1812c0421ea1ed0 \ - --hash=sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037 \ - --hash=sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416 \ - --hash=sha256:178c7b5e62b454c2bc790786e6058c3cc968613b4419251b478c153a4aec32b1 \ - --hash=sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9 \ - --hash=sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a \ - --hash=sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36 \ - --hash=sha256:206b7b3ef96e4ce211754f0cd003feb28b7d81f0ad26b8d077a5d5161436067f \ - --hash=sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b \ - --hash=sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174 \ - --hash=sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b \ - --hash=sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8 \ - --hash=sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e \ - --hash=sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6 \ - --hash=sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c \ - --hash=sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe \ - --hash=sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9 \ - --hash=sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc \ - --hash=sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800 \ - --hash=sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286 \ - --hash=sha256:347542f0ea3f95b2a955ee6656461fa1c776e401ac50ebce055a6c38454a0adf \ - --hash=sha256:39380e12bd1f2fdab4285b6e055ad48efbaed5c836433b142ed4f5b9be71036a \ - --hash=sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc \ - --hash=sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9 \ - --hash=sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665 \ - --hash=sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832 \ - --hash=sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297 \ - --hash=sha256:4e704c52438f66fdd89588346183d898bb42167cf88f8b7ff1c0f9fc957c348f \ - --hash=sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73 \ - --hash=sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b \ - --hash=sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9 \ - --hash=sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090 \ - --hash=sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49 \ - --hash=sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d \ - --hash=sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46 \ - --hash=sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83 \ - --hash=sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796 \ - --hash=sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be \ - --hash=sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d \ - --hash=sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf \ - --hash=sha256:6f497a6876aa4b1a102b04996ce4c1170c7040d83faa9387dd921c16e30d5c83 \ - --hash=sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25 \ - --hash=sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06 \ - --hash=sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3 \ - --hash=sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6 \ - --hash=sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb \ - --hash=sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88 \ - --hash=sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9 \ - --hash=sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be \ - --hash=sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14 \ - --hash=sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a \ - --hash=sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c \ - --hash=sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3 \ - --hash=sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f \ - --hash=sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d \ - --hash=sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670 \ - --hash=sha256:912d4b6af530ddb1338a66229dac3a25ff11d4448be3ec3d6340583995f56031 \ - --hash=sha256:9277145d36a01653863899c665243871434694bcc3431922c3b35c978061bdb8 \ - --hash=sha256:95d14ca7abefde230f7639ec136ade282655431fd5db03c343b19dda72dd1643 \ - --hash=sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d \ - --hash=sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8 \ - --hash=sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8 \ - --hash=sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1 \ - --hash=sha256:9efcc0f11d850cefcafdd9275b9576ad3bfb539bed96807663b32ad99c4d4b88 \ - --hash=sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb \ - --hash=sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61 \ - --hash=sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4 \ - --hash=sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7 \ - --hash=sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9 \ - --hash=sha256:a8a4d3427e8de1312ddf309cc482186466c79895b3a139fed3259fc01dfa9a5b \ - --hash=sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500 \ - --hash=sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6 \ - --hash=sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2 \ - --hash=sha256:af545c2cffdb0967a96b6249e6f5f7b0d92cdfd267f9d5238d5b9ca63e8edb10 \ - --hash=sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1 \ - --hash=sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3 \ - --hash=sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e \ - --hash=sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a \ - --hash=sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5 \ - --hash=sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95 \ - --hash=sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074 \ - --hash=sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5 \ - --hash=sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b \ - --hash=sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772 \ - --hash=sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a \ - --hash=sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274 \ - --hash=sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94 \ - --hash=sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13 \ - --hash=sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac \ - --hash=sha256:cb979826071c0986a5f08333a36104153478ce6018c58cba7f9caddaf63d5d67 \ - --hash=sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76 \ - --hash=sha256:d147004fede1b12f6013a6dbb2a26a986a671a03c6ea740ddc76500e5f1c399f \ - --hash=sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8 \ - --hash=sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7 \ - --hash=sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8 \ - --hash=sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3 \ - --hash=sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be \ - --hash=sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b \ - --hash=sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c \ - --hash=sha256:e999f0c88a458c836d5fb521814e92ed2172c649200336a6df514987c1488258 \ - --hash=sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c \ - --hash=sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6 \ - --hash=sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56 \ - --hash=sha256:ee5e86776273de1795947d17bddd6bb19e0365fd2af4289c0d2c5454b6b1d36b \ - --hash=sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d \ - --hash=sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a \ - --hash=sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162 \ - --hash=sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1 \ - --hash=sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6 \ - --hash=sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5 \ - --hash=sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540 \ - --hash=sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254 +aiohttp==3.14.1 \ + --hash=sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5 \ + --hash=sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983 \ + --hash=sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521 \ + --hash=sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340 \ + --hash=sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d \ + --hash=sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a \ + --hash=sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4 \ + --hash=sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a \ + --hash=sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f \ + --hash=sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee \ + --hash=sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8 \ + --hash=sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb \ + --hash=sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397 \ + --hash=sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05 \ + --hash=sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8 \ + --hash=sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09 \ + --hash=sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2 \ + --hash=sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba \ + --hash=sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf \ + --hash=sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271 \ + --hash=sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5 \ + --hash=sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847 \ + --hash=sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264 \ + --hash=sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf \ + --hash=sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6 \ + --hash=sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df \ + --hash=sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035 \ + --hash=sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126 \ + --hash=sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6 \ + --hash=sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35 \ + --hash=sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4 \ + --hash=sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333 \ + --hash=sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203 \ + --hash=sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c \ + --hash=sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1 \ + --hash=sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251 \ + --hash=sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365 \ + --hash=sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b \ + --hash=sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621 \ + --hash=sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94 \ + --hash=sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da \ + --hash=sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491 \ + --hash=sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe \ + --hash=sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d \ + --hash=sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080 \ + --hash=sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42 \ + --hash=sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c \ + --hash=sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397 \ + --hash=sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9 \ + --hash=sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8 \ + --hash=sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345 \ + --hash=sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3 \ + --hash=sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602 \ + --hash=sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2 \ + --hash=sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966 \ + --hash=sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192 \ + --hash=sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95 \ + --hash=sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3 \ + --hash=sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b \ + --hash=sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444 \ + --hash=sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6 \ + --hash=sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573 \ + --hash=sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af \ + --hash=sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15 \ + --hash=sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe \ + --hash=sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2 \ + --hash=sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496 \ + --hash=sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876 \ + --hash=sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817 \ + --hash=sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448 \ + --hash=sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e \ + --hash=sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6 \ + --hash=sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd \ + --hash=sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f \ + --hash=sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe \ + --hash=sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c \ + --hash=sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca \ + --hash=sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c \ + --hash=sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa \ + --hash=sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc \ + --hash=sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0 \ + --hash=sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0 \ + --hash=sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2 \ + --hash=sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844 \ + --hash=sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719 \ + --hash=sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1 \ + --hash=sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3 \ + --hash=sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178 \ + --hash=sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3 \ + --hash=sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95 \ + --hash=sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730 \ + --hash=sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842 \ + --hash=sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd \ + --hash=sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d \ + --hash=sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96 \ + --hash=sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85 \ + --hash=sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1 \ + --hash=sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199 \ + --hash=sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a \ + --hash=sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588 \ + --hash=sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec \ + --hash=sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004 \ + --hash=sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480 \ + --hash=sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04 \ + --hash=sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8 \ + --hash=sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce \ + --hash=sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087 \ + --hash=sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505 \ + --hash=sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780 \ + --hash=sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4 \ + --hash=sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d \ + --hash=sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca \ + --hash=sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665 \ + --hash=sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296 \ + --hash=sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c \ + --hash=sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a \ + --hash=sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7 \ + --hash=sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451 \ + --hash=sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3 # via gcsfs aiosignal==1.4.0 \ --hash=sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e \ @@ -134,292 +137,271 @@ attrs==26.1.0 \ --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 \ --hash=sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32 # via aiohttp -certifi==2026.4.22 \ - --hash=sha256:3cb2210c8f88ba2318d29b0388d1023c8492ff72ecdde4ebdaddbb13a31b1c4a \ - --hash=sha256:8d455352a37b71bf76a79caa83a3d6c25afee4a385d632127b6afb3963f1c580 +certifi==2026.6.17 \ + --hash=sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432 \ + --hash=sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db # via requests -cffi==2.0.0 \ - --hash=sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb \ - --hash=sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b \ - --hash=sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f \ - --hash=sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9 \ - --hash=sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44 \ - --hash=sha256:0f6084a0ea23d05d20c3edcda20c3d006f9b6f3fefeac38f59262e10cef47ee2 \ - --hash=sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c \ - --hash=sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75 \ - --hash=sha256:1cd13c99ce269b3ed80b417dcd591415d3372bcac067009b6e0f59c7d4015e65 \ - --hash=sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e \ - --hash=sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a \ - --hash=sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e \ - --hash=sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25 \ - --hash=sha256:2081580ebb843f759b9f617314a24ed5738c51d2aee65d31e02f6f7a2b97707a \ - --hash=sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe \ - --hash=sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b \ - --hash=sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91 \ - --hash=sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592 \ - --hash=sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187 \ - --hash=sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c \ - --hash=sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1 \ - --hash=sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94 \ - --hash=sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba \ - --hash=sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb \ - --hash=sha256:3f4d46d8b35698056ec29bca21546e1551a205058ae1a181d871e278b0b28165 \ - --hash=sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529 \ - --hash=sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca \ - --hash=sha256:4647afc2f90d1ddd33441e5b0e85b16b12ddec4fca55f0d9671fef036ecca27c \ - --hash=sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6 \ - --hash=sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c \ - --hash=sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0 \ - --hash=sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743 \ - --hash=sha256:61d028e90346df14fedc3d1e5441df818d095f3b87d286825dfcbd6459b7ef63 \ - --hash=sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5 \ - --hash=sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5 \ - --hash=sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4 \ - --hash=sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d \ - --hash=sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b \ - --hash=sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93 \ - --hash=sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205 \ - --hash=sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27 \ - --hash=sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512 \ - --hash=sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d \ - --hash=sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c \ - --hash=sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037 \ - --hash=sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26 \ - --hash=sha256:89472c9762729b5ae1ad974b777416bfda4ac5642423fa93bd57a09204712322 \ - --hash=sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb \ - --hash=sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c \ - --hash=sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8 \ - --hash=sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4 \ - --hash=sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414 \ - --hash=sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9 \ - --hash=sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664 \ - --hash=sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9 \ - --hash=sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775 \ - --hash=sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739 \ - --hash=sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc \ - --hash=sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062 \ - --hash=sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe \ - --hash=sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9 \ - --hash=sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92 \ - --hash=sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5 \ - --hash=sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13 \ - --hash=sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d \ - --hash=sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26 \ - --hash=sha256:cb527a79772e5ef98fb1d700678fe031e353e765d1ca2d409c92263c6d43e09f \ - --hash=sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495 \ - --hash=sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b \ - --hash=sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6 \ - --hash=sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c \ - --hash=sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef \ - --hash=sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5 \ - --hash=sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18 \ - --hash=sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad \ - --hash=sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3 \ - --hash=sha256:de8dad4425a6ca6e4e5e297b27b5c824ecc7581910bf9aee86cb6835e6812aa7 \ - --hash=sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5 \ - --hash=sha256:e6e73b9e02893c764e7e8d5bb5ce277f1a009cd5243f8228f75f842bf937c534 \ - --hash=sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49 \ - --hash=sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2 \ - --hash=sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5 \ - --hash=sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453 \ - --hash=sha256:fe562eb1a64e67dd297ccc4f5addea2501664954f2692b69a76449ec7913ecbf +cffi==2.1.0 \ + --hash=sha256:02cb7ff33ded4f1532476731f89ede53e2e488a8e6205515a82144246ffa7dcc \ + --hash=sha256:03e9810d18c646077e501f661b682fbf5dee4676048527ca3cffe66faa9960dd \ + --hash=sha256:0520e1f4c35f44e209cbbb421b67eec42e6a157f59444dfb6058874ff3610e5d \ + --hash=sha256:0582a58f3051372229ca8e7f5f589f9e5632678208d8636fea3676711fdf7fe5 \ + --hash=sha256:0611e7ebf90573a535ebdc33ae9da222d037853983e13359f580fab781ca017f \ + --hash=sha256:0a42c688d19fca6e095a53c6a6e2295a5b050a8b289f109adab02a9e61a25de6 \ + --hash=sha256:0a96b74cda968eebbad56d973efe5098974f0a9fb323865bf99ea1fd24e3e64c \ + --hash=sha256:10537b1df4967ca26d21e5072d7d54188354483b91dc75058968d3f0cf13fbda \ + --hash=sha256:11b3fb55f4f8ad92274ed26705f65d8f91457de71f5380061eb6d125a768fecd \ + --hash=sha256:15faec4adfff450819f3aee0e2e02c812de6edb88203aa58807955db2003472a \ + --hash=sha256:164bff1657b2a74f0b6d54e11c9b375bc97b931f2ca9c43fcf875838da1570dd \ + --hash=sha256:1854b724d00f6654c742097d5387569021be12d3a0f770eae1df8f8acfcc6acd \ + --hash=sha256:19c54ac121cad98450b4896fa9a43ee0180d57bc4bc911a33db6cab1efab6cd3 \ + --hash=sha256:1b96bfe2c4bd825681b7d311ad6d9b7280a091f43e8f63da5729638083cd3bfb \ + --hash=sha256:1e9f50d192a3e525b15a75ab5114e442d83d657b7ec29182a991bc9a88fd3a66 \ + --hash=sha256:1ff3456eab0d889592d1936d6125bbfbc7ae4d3354a700f8bd80450a66445d4d \ + --hash=sha256:2282cd5e38aa8accd03e99d1256af8411c84cdbee6a89d841b563fdbd1f3e50f \ + --hash=sha256:276f20fffd7b396e12516ba8edf9509210ac248cbbc5acbc39cd512f9f59ebe6 \ + --hash=sha256:2b71d409cccee78310ab5dec549aed052aaea483346e282c7b02362596e01bb0 \ + --hash=sha256:2e9dabb9abcb7ad15938c7196ad5c1718a4e6d33cc79b4c0209bdb64c4a54a5c \ + --hash=sha256:30b65779d598c370374fefabf138d456fd6f3216bfa7bedfab1ba82025b0cd93 \ + --hash=sha256:33eb1ad83ebe8f313e0df035c406227d55a79456704a863fad9842136af5ad7d \ + --hash=sha256:35aaea0c7ee0e58a5cd8c2fd1a48fdf7ece0d2699b7ecdda08194e9ce5dd9b3d \ + --hash=sha256:3681e031db29958a7502f5c0c9d6bbc4c36cb20f7b104086fa642d1799631ff8 \ + --hash=sha256:379de10ce1ba048b1448599d1b37b24caee16309d1ac98d3982fc997f768700b \ + --hash=sha256:37f525a7e7e50c017fdebe58b787be310ad59357ae43a053943a6e1a6c526001 \ + --hash=sha256:3b926723c13eba9f81d2ef3820d63aeceec3b2d4639906047bf675cb8a7a500d \ + --hash=sha256:3d7f118b5adbfdfead90c25822690b02bc8074fba949bb7858bec4ebd55adb43 \ + --hash=sha256:46b1c8db8f6122420f32d02fffb924c2fe9bc772d228c7c711748fff56aabb2b \ + --hash=sha256:47ff3a8bfd8cb9da1af7524b965127095055654c177fcfc7578debcb015eecd0 \ + --hash=sha256:4d433a51f1870e43a13b6732f92aaf540ff77c2015097c78556f75a2d6c030e0 \ + --hash=sha256:4f26194e3d95e06501b942642855aed4f953d55e95d7d01b7c4483db3ecff458 \ + --hash=sha256:510aeeeac94811b138077451da1fb18b308a5feab47dd2b603af55804155e1c8 \ + --hash=sha256:5972433ad71a9e46516584ef60a0fda12d9dc459938d1539c3ddecf9bdc1368d \ + --hash=sha256:5ecbd0499275d57506d397eebe1981cee87b47fcd9ef5c22cab7ed7644a39a94 \ + --hash=sha256:6274dcb2d15cef48daa73ed1be5a40d501d74dccd0cd6db364776d12cb6ba022 \ + --hash=sha256:63960549e4f8dc41e31accb97b975abaecfc44c03e396c093a6436763c2ea7db \ + --hash=sha256:64c753a0f87a256020004f37a1c8c02c480e725f910f0b2a0f3f07debd1b2479 \ + --hash=sha256:6af371f3767faeffc6ac1ef57cdfd25844403e9d3f476c5537caee499de96376 \ + --hash=sha256:6ca4919c6e4f89aa99c42510b42cf54596892c00b3f9077f6bdd1505e24b9c8d \ + --hash=sha256:6d194185eabd279f1c05ebe3504265ddfc5ad2b58d0714f7db9f01da592e9eb6 \ + --hash=sha256:702c436735fbe99d59ada02a1f65cfc0d31c0ee8b7290912f8fbc5cd1e4b16c3 \ + --hash=sha256:716ff8ec22f20b4d988b12884086bcef0fc99737043e503f7a3935a6be99b1ea \ + --hash=sha256:762f99479dcb369f60ab9017ad4ab97a36a1dd7c1ee5a3b15db0f4b8659120cd \ + --hash=sha256:7762faa47e8ff7eb80bd261d9a7d8eea2d8baa69de5e95b70c1f338bbe712f02 \ + --hash=sha256:78474632761faa0fb96f30b1c928c84ebcf68713cbb80d15bab09dfe61640fde \ + --hash=sha256:799416bae98336e400981ff6e532d67d5c709cfb30afb79865a1315f94b0e224 \ + --hash=sha256:7d034dcffa09e9a46c93fa3a3be402096cb5354ac6e41ab8e5cc9cd8b642ad76 \ + --hash=sha256:7d28dff1db6764108bc30788d85d61c876beff416d9a49cb9dd7c5a9f34f5804 \ + --hash=sha256:7d3538f9c0e50670f4deb93dbb696576e60590369cae2faf7de681e597a8a1f1 \ + --hash=sha256:7d5980a3433d4b71a5e120f9dd551403d7824e31e2e67124fe2769c404c06913 \ + --hash=sha256:7ea6b3e2c4250ff1de21c630fe72d0f63eb95c2c32ffbf64a358cf4a8836d714 \ + --hash=sha256:86cf8755a791f72c85dc287128cc62d4f24d392e3f1e15837245623f4a33cccc \ + --hash=sha256:88023dfe18799507b73f1dbb0d14326a17465de1bc9c9c7655c22845e9ddc3a2 \ + --hash=sha256:89095c1968b4ba8285840e131bf2891b09ae137fe2146905acae0354fbce1b5e \ + --hash=sha256:8d35c139744adb3e727cd51b1a18324bbe44b8bd41bf8322bca4d41289f48eda \ + --hash=sha256:8e74a6135550c4748af665b1b1118b6aab33b1fc6a16f9aff630af107c3b4512 \ + --hash=sha256:8f9ec95b8a043d3dfbc74d9abc6f7baf524dd27a8dc160b0a32ff9cdab650c28 \ + --hash=sha256:90bec57cf82089383bd06a605b3eb8daebf7e5a668520beaf6e327a83a947699 \ + --hash=sha256:95f2954c2c9473d892eca6e0409f3568b37ab62a8eedb122461f73cc273476e3 \ + --hash=sha256:961be50688f7fba2fa65f63712d3b9b341a22311f5253460ce933f52f0de1c8c \ + --hash=sha256:98fff996e983a36d3aa2eca83af40c5821202e7e6f32d13ae94e3d2286f10cfe \ + --hash=sha256:9b8f0f26ca4e7513c534d351eca551947d053fac438f2a04ac96d882909b0d3a \ + --hash=sha256:9d72af0cf10a76a600a9690078fe31c63b9588c8e86bf9fd353f713c84b5db0f \ + --hash=sha256:9d8272c0e483b024e1b9ad029821470ed8ec65631dbd90217469da0e7cd89f1c \ + --hash=sha256:a016194dbe13d14ee9556e734b772d8d67b947092b268d757fd4290e3ba2dfc2 \ + --hash=sha256:a5781494d4d400a3f47f8f1da94b324f6e6b440a53387774002890a2a2f4b50f \ + --hash=sha256:a95b05f9baf29b91171b3a8bd2020b028835243e7b0ff6bb23e2a3c228518b1b \ + --hash=sha256:aa7a1b53a2a4452ada2d1b5dade9960b2522f1e61293a811a077439e39029565 \ + --hash=sha256:ac0f1a2d0cfa7eea3f2aaf006ab6e70e8feeb16b75d65b7e5939982ca2f11056 \ + --hash=sha256:af5e2915d41fe6c961694d7bfdc8562942638200f3ce2765dfb8b745cf997629 \ + --hash=sha256:b6422532152adf4e59b110cb2808cee7a033800952f5c036b4af047ee43199e7 \ + --hash=sha256:b65f590ef2a44640f9a05dbb548a429b4ade77913ce683ac8b1480777658a6c0 \ + --hash=sha256:ba00f661f8ba35d075c937174e27c2c421cec3942fd2e0ea3e66996757c0fdd9 \ + --hash=sha256:bccbbb5ee76a61f9d99b5bf3846a51d7fca4b6a732fe46f89295610edaf41853 \ + --hash=sha256:bf01d8c84cbea96b944c73b22182e6c7c432b3475632b8111dbfdc95ddad6e13 \ + --hash=sha256:bf5c6cf48238b0eb4c086978c492ad1cbc22373fc5b2d7353b3a598ce6db887a \ + --hash=sha256:c16914df9fb7f500e440e6875fa23ff5e0b31db01fa9c06af98d59a91f0dc2e4 \ + --hash=sha256:c351efb95e832a853a29361675f33a7ce53de1a109cd73fd47af0712213aa4ce \ + --hash=sha256:c4165821e131d6d4ca444347c2b694e2311bcfa3fe5a861cc72968f28867beac \ + --hash=sha256:c5f5df567f6eb216de69be06ce55c8b714090fae02b18a3b40da8163b8c5fa9c \ + --hash=sha256:c941bb58d5a6e1c3892d86e42927ed6c180302f07e6d395d08c416e594b98b46 \ + --hash=sha256:c97f080ea627e2863524c5af3836e2270b5f5dfff1f104392b959f8df0c5d384 \ + --hash=sha256:cb96698e3c7413d906ce83f8ffd245ec1bd94707541f299d0ce4d6b0193e982b \ + --hash=sha256:cbb7640ce37159548d2147b5b8c241f962143d4c71231431820783f4dc78f210 \ + --hash=sha256:cdf2448aab5f661c9315308ec8b93f4e8a1a67a3c733f8631067a2b67d5913dc \ + --hash=sha256:d2117334c3af3bdcb9a88522b844a2bdb5efdc4f71c6c822df55486ae1c3347a \ + --hash=sha256:d53d10f7da99ae46f7373b9150393e9c5eab9b224909982b43832668de4779f5 \ + --hash=sha256:d9fafc5aa2e2a39aaf7f8cc0c1f044a9b07fca12e558dca53a3cc5c654ad67a7 \ + --hash=sha256:db3eb7d46527159a878ec3460e9d40615bc25ba337d477db681aea6e4f05c5d2 \ + --hash=sha256:dbf7c7a88e2bac086f06d14577332760bdeecc42bdec8ac4077f6260557d9326 \ + --hash=sha256:df2b82571a1b30f58a87bf4e5a9e78d2b1eff6c6ce8fd3aa3757221f93f0863f \ + --hash=sha256:df92f2aba50eb4d96718b68ef76f2e57a57b54f2fa62333496d16c6d585a85ca \ + --hash=sha256:eb4e8997a49aa2c08a3e43c9045d224448b8941d88e7ac163c7d383e560cbf98 \ + --hash=sha256:efc1cdd798b1aaf39b4610bba7aad28c9bea9b910f25c784ccf9ec1fa719d1f9 \ + --hash=sha256:f146d154428a2523f9cc7936c02353c2459b8f6cf07d3cd1ee1c0a611109c5d5 \ + --hash=sha256:f5bce581e6b8c235e566a14768a943b172ada3ed73537bb0c0be1edee312d4e7 \ + --hash=sha256:f9912624a0c0b834b7520d7769b3644453aabc0a7e1c839da7359f050750e9bc \ + --hash=sha256:fb62edb5bb52cca65fab91a63afa7561607120d26090a7e8fda6fb9f064726da \ + --hash=sha256:ff067a8d8d880e7809e4ac88eb009bb848870115317b306666502ccad30b147f # via cryptography -charset-normalizer==3.4.7 \ - --hash=sha256:007d05ec7321d12a40227aae9e2bc6dca73f3cb21058999a1df9e193555a9dcc \ - --hash=sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c \ - --hash=sha256:07d9e39b01743c3717745f4c530a6349eadbfa043c7577eef86c502c15df2c67 \ - --hash=sha256:08e721811161356f97b4059a9ba7bafb23ea5ee2255402c42881c214e173c6b4 \ - --hash=sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0 \ - --hash=sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c \ - --hash=sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5 \ - --hash=sha256:12a6fff75f6bc66711b73a2f0addfc4c8c15a20e805146a02d147a318962c444 \ - --hash=sha256:12d8baf840cc7889b37c7c770f478adea7adce3dcb3944d02ec87508e2dcf153 \ - --hash=sha256:14265bfe1f09498b9d8ec91e9ec9fa52775edf90fcbde092b25f4a33d444fea9 \ - --hash=sha256:16d971e29578a5e97d7117866d15889a4a07befe0e87e703ed63cd90cb348c01 \ - --hash=sha256:177a0ba5f0211d488e295aaf82707237e331c24788d8d76c96c5a41594723217 \ - --hash=sha256:1a87ca9d5df6fe460483d9a5bbf2b18f620cbed41b432e2bddb686228282d10b \ - --hash=sha256:1c2a768fdd44ee4a9339a9b0b130049139b8ce3c01d2ce09f67f5a68048d477c \ - --hash=sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a \ - --hash=sha256:1dc8b0ea451d6e69735094606991f32867807881400f808a106ee1d963c46a83 \ - --hash=sha256:1efde3cae86c8c273f1eb3b287be7d8499420cf2fe7585c41d370d3e790054a5 \ - --hash=sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7 \ - --hash=sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb \ - --hash=sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c \ - --hash=sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1 \ - --hash=sha256:2cd4a60d0e2fb04537162c62bbbb4182f53541fe0ede35cdf270a1c1e723cc42 \ - --hash=sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab \ - --hash=sha256:2fe249cb4651fd12605b7288b24751d8bfd46d35f12a20b1ba33dea122e690df \ - --hash=sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e \ - --hash=sha256:320ade88cfb846b8cd6b4ddf5ee9e80ee0c1f52401f2456b84ae1ae6a1a5f207 \ - --hash=sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18 \ - --hash=sha256:36836d6ff945a00b88ba1e4572d721e60b5b8c98c155d465f56ad19d68f23734 \ - --hash=sha256:38c0109396c4cfc574d502df99742a45c72c08eff0a36158b6f04000043dbf38 \ - --hash=sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110 \ - --hash=sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18 \ - --hash=sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44 \ - --hash=sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d \ - --hash=sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48 \ - --hash=sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e \ - --hash=sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5 \ - --hash=sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d \ - --hash=sha256:4e5163c14bffd570ef2affbfdd77bba66383890797df43dc8b4cc7d6f500bf53 \ - --hash=sha256:511ef87c8aec0783e08ac18565a16d435372bc1ac25a91e6ac7f5ef2b0bff790 \ - --hash=sha256:532bc9bf33a68613fd7d65e4b1c71a6a38d7d42604ecf239c77392e9b4e8998c \ - --hash=sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b \ - --hash=sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116 \ - --hash=sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d \ - --hash=sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10 \ - --hash=sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6 \ - --hash=sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2 \ - --hash=sha256:6370e8686f662e6a3941ee48ed4742317cafbe5707e36406e9df792cdb535776 \ - --hash=sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a \ - --hash=sha256:65bcd23054beab4d166035cabbc868a09c1a49d1efe458fe8e4361215df40265 \ - --hash=sha256:66671f93accb62ed07da56613636f3641f1a12c13046ce91ffc923721f23c008 \ - --hash=sha256:6696b7688f54f5af4462118f0bfa7c1621eeb87154f77fa04b9295ce7a8f2943 \ - --hash=sha256:6785f414ae0f3c733c437e0f3929197934f526d19dfaa75e18fdb4f94c6fb374 \ - --hash=sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246 \ - --hash=sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e \ - --hash=sha256:6e0d51f618228538a3e8f46bd246f87a6cd030565e015803691603f55e12afb5 \ - --hash=sha256:6ed74185b2db44f41ef35fd1617c5888e59792da9bbc9190d6c7300617182616 \ - --hash=sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15 \ - --hash=sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41 \ - --hash=sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960 \ - --hash=sha256:750e02e074872a3fad7f233b47734166440af3cdea0add3e95163110816d6752 \ - --hash=sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e \ - --hash=sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72 \ - --hash=sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7 \ - --hash=sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8 \ - --hash=sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b \ - --hash=sha256:813c0e0132266c08eb87469a642cb30aaff57c5f426255419572aaeceeaa7bf4 \ - --hash=sha256:82b271f5137d07749f7bf32f70b17ab6eaabedd297e75dce75081a24f76eb545 \ - --hash=sha256:84c018e49c3bf790f9c2771c45e9313a08c2c2a6342b162cd650258b57817706 \ - --hash=sha256:8751d2787c9131302398b11e6c8068053dcb55d5a8964e114b6e196cf16cb366 \ - --hash=sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb \ - --hash=sha256:87fad7d9ba98c86bcb41b2dc8dbb326619be2562af1f8ff50776a39e55721c5a \ - --hash=sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e \ - --hash=sha256:8e385e4267ab76874ae30db04c627faaaf0b509e1ccc11a95b3fc3e83f855c00 \ - --hash=sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f \ - --hash=sha256:94e1885b270625a9a828c9793b4d52a64445299baa1fea5a173bf1d3dd9a1a5a \ - --hash=sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1 \ - --hash=sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66 \ - --hash=sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356 \ - --hash=sha256:a6c5863edfbe888d9eff9c8b8087354e27618d9da76425c119293f11712a6319 \ - --hash=sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4 \ - --hash=sha256:adb2597b428735679446b46c8badf467b4ca5f5056aae4d51a19f9570301b1ad \ - --hash=sha256:ae196f021b5e7c78e918242d217db021ed2a6ace2bc6ae94c0fc596221c7f58d \ - --hash=sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5 \ - --hash=sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7 \ - --hash=sha256:aef65cd602a6d0e0ff6f9930fcb1c8fec60dd2cfcb6facaf4bdb0e5873042db0 \ - --hash=sha256:af21eb4409a119e365397b2adbaca4c9ccab56543a65d5dbd9f920d6ac29f686 \ - --hash=sha256:b14b2d9dac08e28bb8046a1a0434b1750eb221c8f5b87a68f4fa11a6f97b5e34 \ - --hash=sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49 \ - --hash=sha256:bb8cc7534f51d9a017b93e3e85b260924f909601c3df002bcdb58ddb4dc41a5c \ - --hash=sha256:bc17a677b21b3502a21f66a8cc64f5bfad4df8a0b8434d661666f8ce90ac3af1 \ - --hash=sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e \ - --hash=sha256:bd9b23791fe793e4968dba0c447e12f78e425c59fc0e3b97f6450f4781f3ee60 \ - --hash=sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0 \ - --hash=sha256:c0f081d69a6e58272819b70288d3221a6ee64b98df852631c80f293514d3b274 \ - --hash=sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d \ - --hash=sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0 \ - --hash=sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae \ - --hash=sha256:c593052c465475e64bbfe5dbd81680f64a67fdc752c56d7a0ae205dc8aeefe0f \ - --hash=sha256:cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d \ - --hash=sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe \ - --hash=sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3 \ - --hash=sha256:cf29836da5119f3c8a8a70667b0ef5fdca3bb12f80fd06487cfa575b3909b393 \ - --hash=sha256:d4a48e5b3c2a489fae013b7589308a40146ee081f6f509e047e0e096084ceca1 \ - --hash=sha256:d560742f3c0d62afaccf9f41fe485ed69bd7661a241f86a3ef0f0fb8b1a397af \ - --hash=sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44 \ - --hash=sha256:d61f00a0869d77422d9b2aba989e2d24afa6ffd552af442e0e58de4f35ea6d00 \ - --hash=sha256:d635aab80466bc95771bb78d5370e74d36d1fe31467b6b29b8b57b2a3cd7d22c \ - --hash=sha256:dca4bbc466a95ba9c0234ef56d7dd9509f63da22274589ebd4ed7f1f4d4c54e3 \ - --hash=sha256:dd915403e231e6b1809fe9b6d9fc55cf8fb5e02765ac625d9cd623342a7905d7 \ - --hash=sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd \ - --hash=sha256:e060d01aec0a910bdccb8be71faf34e7799ce36950f8294c8bf612cba65a2c9e \ - --hash=sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b \ - --hash=sha256:e17b8d5d6a8c47c85e68ca8379def1303fd360c3e22093a807cd34a71cd082b8 \ - --hash=sha256:e5f4d355f0a2b1a31bc3edec6795b46324349c9cb25eed068049e4f472fb4259 \ - --hash=sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859 \ - --hash=sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46 \ - --hash=sha256:e80c8378d8f3d83cd3164da1ad2df9e37a666cdde7b1cb2298ed0b558064be30 \ - --hash=sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b \ - --hash=sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46 \ - --hash=sha256:ed065083d0898c9d5b4bbec7b026fd755ff7454e6e8b73a67f8c744b13986e24 \ - --hash=sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a \ - --hash=sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24 \ - --hash=sha256:f22dec1690b584cea26fade98b2435c132c1b5f68e39f5a0b7627cd7ae31f1dc \ - --hash=sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215 \ - --hash=sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063 \ - --hash=sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832 \ - --hash=sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6 \ - --hash=sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79 \ - --hash=sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464 +charset-normalizer==3.4.9 \ + --hash=sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380 \ + --hash=sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62 \ + --hash=sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c \ + --hash=sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226 \ + --hash=sha256:0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5 \ + --hash=sha256:0fa1aec2d32bcc03c8fa0f6f1712caad1adc38509f31142112e5c9daf5b9c833 \ + --hash=sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b \ + --hash=sha256:16d10d789dd9bcca1173c95af82c58433122564b7bc39385124be735a35cbe99 \ + --hash=sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501 \ + --hash=sha256:1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec \ + --hash=sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698 \ + --hash=sha256:231ddcbb35e2ff8973e1365db41fe0572662893b99a05deb183b68ad4c0c8bd4 \ + --hash=sha256:253a4a220747e8b5faf57ec320c4f5efb0cef05f647420bf267143ec15dba10a \ + --hash=sha256:280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3 \ + --hash=sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2 \ + --hash=sha256:304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a \ + --hash=sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e \ + --hash=sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4 \ + --hash=sha256:375b83ed0aecfce76c16d198fbc21f3b11b337d68662bea0a995046682a11419 \ + --hash=sha256:3c09a49d6cde137258beb3d551994a2927fd35ad5cf96aed573f61bbd67c5f84 \ + --hash=sha256:3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da \ + --hash=sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519 \ + --hash=sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe \ + --hash=sha256:432786d3561e69aeeae6c7e8648964ce0ad05736120135601f87ac26b9c83381 \ + --hash=sha256:43b9e366a31fdd1c87d0eb08f579b4a82b723ea54338f040d6b4e518a026ea29 \ + --hash=sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614 \ + --hash=sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0 \ + --hash=sha256:476743fe6dfe14a2da12e3ac79125dc84a3b2cf8094369a47a1529b0cd8549fe \ + --hash=sha256:4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29 \ + --hash=sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0 \ + --hash=sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917 \ + --hash=sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9 \ + --hash=sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32 \ + --hash=sha256:58150c9f9b9a552505912d182ccdf26f6396fb6094816ceebcbb20eecabaed94 \ + --hash=sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63 \ + --hash=sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd \ + --hash=sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198 \ + --hash=sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde \ + --hash=sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012 \ + --hash=sha256:6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1 \ + --hash=sha256:65a7ff3f705e57d392f7261b6d0550fe137c3019477431f1c355e0db0a7d3e15 \ + --hash=sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b \ + --hash=sha256:67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993 \ + --hash=sha256:68ce9f4d6b26d5ccbf7fd4459bf75f74a0a146677ebba80597df60cbdb20e6f4 \ + --hash=sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5 \ + --hash=sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8 \ + --hash=sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35 \ + --hash=sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642 \ + --hash=sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2 \ + --hash=sha256:79580094b00d1789d1f93ea55bc43cb2f611910c72235b7657f3482ddcc1b22d \ + --hash=sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9 \ + --hash=sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c \ + --hash=sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33 \ + --hash=sha256:871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db \ + --hash=sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf \ + --hash=sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9 \ + --hash=sha256:8c041122946b7ba21bb32c45b1aa57b1be35527690aeb3c5c234521085632eee \ + --hash=sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84 \ + --hash=sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44 \ + --hash=sha256:920079c3f7456fa213e0829ed2073aaa727fd39d889ead5b4f35d0de5460d04f \ + --hash=sha256:93d59d504b230e83c7a843251681959a0b6a9cd76f6e146ce1b8a80eb8739af9 \ + --hash=sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9 \ + --hash=sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177 \ + --hash=sha256:9bb41182d93ea91f60b4bc8fbf4c820c69ef8a12ab2d917f3f1834f1acad07e8 \ + --hash=sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b \ + --hash=sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39 \ + --hash=sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41 \ + --hash=sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0 \ + --hash=sha256:aa99adc8f081b475a12843953db36831eaf83ec33eb46a90629ca6a5de45a616 \ + --hash=sha256:ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d \ + --hash=sha256:ad41ba96094304aa090f5a30cb6e4fb3b3f1c264c523394b4c39bbacc4dc92ba \ + --hash=sha256:b5314963fce9b0b12743891de876e724997864ee22aa496f903f426c7e2fa5b2 \ + --hash=sha256:bcf74c1df76758a395bf0af608c04c82257523f55c9868b334f06270d0f2112b \ + --hash=sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9 \ + --hash=sha256:c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b \ + --hash=sha256:c1225416b463483160e4af85d5fc3a9690ccb53fd4b1865a6437825f5ede3209 \ + --hash=sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48 \ + --hash=sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046 \ + --hash=sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632 \ + --hash=sha256:cd6280cf040f233bd7d3407b743b4b4c74f70e8e1c4199cb112a62c941c0772a \ + --hash=sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2 \ + --hash=sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1 \ + --hash=sha256:ddf4af30b417d9fe16481e9b81c27ab2a7cde1ff7ba3e85653b02db7d145dc7b \ + --hash=sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990 \ + --hash=sha256:df7276909358e5635ae203673ab7e509ddd224225a8d6b0790bf13eb2bde1cc5 \ + --hash=sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b \ + --hash=sha256:e9701d0049d92c16703a42771b98d560b95248949f23f8cf7b4eddd201814fb9 \ + --hash=sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534 \ + --hash=sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81 \ + --hash=sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a \ + --hash=sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d \ + --hash=sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf \ + --hash=sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115 # via requests cheroot==11.1.2 \ --hash=sha256:0f6c0ba05c00fbc869fb46b1de4ec2384e1d85418ae963d3bc10ae83b688dbfa \ --hash=sha256:bfb70c49663f63b0440f2b54dbc6b0d1650e56dfe4e2641f59b2c6f727b44aca # via xprof-nightly -cryptography==48.0.0 \ - --hash=sha256:0890f502ddf7d9c6426129c3f49f5c0a39278ed7cd6322c8755ffca6ee675a13 \ - --hash=sha256:0c558d2cdffd8f4bbb30fc7134c74d2ca9a476f830bb053074498fbc86f41ed6 \ - --hash=sha256:16cd65b9330583e4619939b3a3843eec1e6e789744bb01e7c7e2e62e33c239c8 \ - --hash=sha256:18349bbc56f4743c8b12dc32e2bccb2cf83ee8b69a3bba74ef8ae857e26b3d25 \ - --hash=sha256:1e2d54c8be6152856a36f0882ab231e70f8ec7f14e93cf87db8a2ed056bf160c \ - --hash=sha256:22a5cb272895dce158b2cacdfdc3debd299019659f42947dbdac6f32d68fe832 \ - --hash=sha256:27241b1dc9962e056062a8eef1991d02c3a24569c95975bd2322a8a52c6e5e12 \ - --hash=sha256:2b4d59804e8408e2fea7d1fbaf218e5ec984325221db76e6a241a9abd6cdd95c \ - --hash=sha256:2eb992bbd4661238c5a397594c83f5b4dc2bc5b848c365c8f991b6780efcc5c7 \ - --hash=sha256:369a6348999f94bbd53435c894377b20ab95f25a9065c283570e70150d8abc3c \ - --hash=sha256:3cb07a3ed6431663cd321ea8a000a1314c74211f823e4177fefa2255e057d1ec \ - --hash=sha256:40ba1f85eaa6959837b1d51c9767e230e14612eea4ef110ee8854ada22da1bf5 \ - --hash=sha256:4defde8685ae324a9eb9d818717e93b4638ef67070ac9bc15b8ca85f63048355 \ - --hash=sha256:55b7718303bf06a5753dcdccf2f3945cf18ad7bffde41b61226e4db31ab89a9c \ - --hash=sha256:561215ea3879cb1cbbf272867e2efda62476f240fb58c64de6b393ae19246741 \ - --hash=sha256:58d00498e8933e4a194f3076aee1b4a97dfec1a6da444535755822fe5d8b0b86 \ - --hash=sha256:59baa2cb386c4f0b9905bd6eb4c2a79a69a128408fd31d32ca4d7102d4156321 \ - --hash=sha256:5a5ed8fde7a1d09376ca0b40e68cd59c69fe23b1f9768bd5824f54681626032a \ - --hash=sha256:5b012212e08b8dd5edc78ef54da83dd9892fd9105323b3993eff6bea65dc21d7 \ - --hash=sha256:5c3932f4436d1cccb036cb0eaef46e6e2db91035166f1ad6505c3c9d5a635920 \ - --hash=sha256:614d0949f4790582d2cc25553abd09dd723025f0c0e7c67376a1d77196743d6e \ - --hash=sha256:76341972e1eff8b4bea859f09c0d3e64b96ce931b084f9b9b7db8ef364c30eff \ - --hash=sha256:77a2ccbbe917f6710e05ba9adaa25fb5075620bf3ea6fb751997875aff4ae4bd \ - --hash=sha256:7995ef305d7165c3f11ae07f2517e5a4f1d5c18da1376a0a9ed496336b69e5f3 \ - --hash=sha256:7ce4bfae76319a532a2dc68f82cc32f5676ee792a983187dac07183690e5c66f \ - --hash=sha256:7e8eac43dfca5c4cccc6dad9a80504436fca53bb9bc3100a2386d730fbe6b602 \ - --hash=sha256:84cf79f0dc8b36ac5da873481716e87aef31fcfa0444f9e1d8b4b2cece142855 \ - --hash=sha256:8c7378637d7d88016fa6791c159f698b3d3eed28ebf844ac36b9dc04a14dae18 \ - --hash=sha256:8cd666227ef7af430aa5914a9910e0ddd703e75f039cef0825cd0da71b6b711a \ - --hash=sha256:906cbf0670286c6e0044156bc7d4af9cbb0ef6db9f73e52c3ec56ba6bdde5336 \ - --hash=sha256:9071196d81abc88b3516ac8cdfad32e2b66dd4a5393a8e68a961e9161ddc6239 \ - --hash=sha256:9249e3cd978541d665967ac2cb2787fd6a62bddf1e75b3e347a594d7dacf4f74 \ - --hash=sha256:984a20b0f62a26f48a3396c72e4bc34c66e356d356bf370053066b3b6d54634a \ - --hash=sha256:9be5aafa5736574f8f15f262adc81b2a9869e2cfe9014d52a44633905b40d52c \ - --hash=sha256:9c459db21422be75e2809370b829a87eb37f74cd785fc4aa9ea1e5f43b47cda4 \ - --hash=sha256:9ccdac7d40688ecb5a3b4a604b8a88c8002e3442d6c60aead1db2a89a041560c \ - --hash=sha256:a0e692c683f4df67815a2d258b324e66f4738bd7a96a218c826dce4f4bd05d8f \ - --hash=sha256:a5da777e32ffed6f85a7b2b3f7c5cbc88c146bfcd0a1d7baf5fcc6c52ee35dd4 \ - --hash=sha256:a64697c641c7b1b2178e573cbc31c7c6684cd56883a478d75143dbb7118036db \ - --hash=sha256:ad64688338ed4bc1a6618076ba75fd7194a5f1797ac60b47afe926285adb3166 \ - --hash=sha256:bd72e68b06bb1e96913f97dd4901119bc17f39d4586a5adf2d3e47bc2b9d58b5 \ - --hash=sha256:c17dfe85494deaeddc5ce251aebd1d60bbe6afc8b62071bb0b469431a000124f \ - --hash=sha256:c18684a7f0cc9a3cb60328f496b8e3372def7c5d2df39ac267878b05565aaaae \ - --hash=sha256:cc90c0b39b2e3c65ef52c804b72e3c58f8a04ab2a1871272798e5f9572c17d20 \ - --hash=sha256:db63bf618e5dea46c07de12e900fe1cdd2541e6dc9dbae772a70b7d4d4765f6a \ - --hash=sha256:ea8990436d914540a40ab24b6a77c0969695ed52f4a4874c5137ccf7045a7057 \ - --hash=sha256:ecde28a596bead48b0cfd2a1b4416c3d43074c2d785e3a398d7ec1fc4d0f7fbb \ - --hash=sha256:f5333311663ea94f75dd408665686aaf426563556bb5283554a3539177e03b8c \ - --hash=sha256:fdfef35d751d510fcef5252703621574364fec16418c4a1e5e1055248401054b - # via - # google-auth - # pyopenssl -decorator==5.2.1 \ - --hash=sha256:65f266143752f734b0a7cc83c46f4618af75b8c5911b00ccb61d0ac9b6da0360 \ - --hash=sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a +cryptography==49.0.0 \ + --hash=sha256:026ac7423e6fa66872d3bf889be5974507da3944f866f704fa200eadacd00001 \ + --hash=sha256:07cab27cc7b7e0fd28e5e26bb9eeedde5c135c868b46de4a27845abe94af6122 \ + --hash=sha256:084ef1af862eb07ec46d25f68689f2102a9fc0e05ce7b80f14f5fe51e4eef0f6 \ + --hash=sha256:0b82e28ee398a386f0807bba7884d30f25218855690f45115831bcce5d90822c \ + --hash=sha256:0e959b578856a3924bc0cbb710fc12c387b9412a951389f3ca61704a9e25f325 \ + --hash=sha256:0f21641cf4b30fca7aee061ced0ec7ad7b073518088b7c9969a297c0ae796c69 \ + --hash=sha256:196ecd6a36e4e9aa10270393bb98d8df88fccee0bf1e5128b91ae4eb4375896d \ + --hash=sha256:2400ef9c9e2299a25614eb1dea3db54a69b1349efd043bfac9c67630d136df36 \ + --hash=sha256:28d8b15e6275f12c8a207dc309dfa957903c927d08d0cc937ee3f63f200693cc \ + --hash=sha256:2afe9051da7ae7bd5905da5a949280c7d2bb75682e188f650a9d0f2756b834c6 \ + --hash=sha256:2eda353d8a27bcbcaa4cbed18994a74ab4d19a2ca897db188ea269ab9b71419b \ + --hash=sha256:32703d93296f5c1f4b53349ad3a250c2cae0fdecd3a3dd5d47e616d8d616af27 \ + --hash=sha256:33cd0565932807baddb67b96dbee92f2c374b5c89dee09fd74079aeb8c8dba61 \ + --hash=sha256:35b151772baff2c74cba7fa290ceaff4c3b11c0c881eb93eb5dbc05a7cfbba18 \ + --hash=sha256:36d1709f992593689b45bda411498d62c6e365f2ca00b84657d4dadd24de16db \ + --hash=sha256:42b0684e0e40cf26122427802486f6d93aea593612603a94fbf260c7eb1e9c1b \ + --hash=sha256:4ae387c9cb68ea569ca17e490d66d8142b81c3cc814bf179974b7d146e490bbb \ + --hash=sha256:53ecee2e23f7169b6117e99fc8a944e5e50f79e69758a83b52a00cb98ab2b2d2 \ + --hash=sha256:66ec79c3904820572d7e987abdf304281f141d37ad9a489b8e97066e7b9b6459 \ + --hash=sha256:67e1d20ad9ef3a563c59ef22e7a8a0b8210bd26604369ea4a30a7c66aefe504e \ + --hash=sha256:6f2debedf9ca60cf1d5bd466475638af5130f89965605cd818484d19987d3a21 \ + --hash=sha256:6fc361c34fb6aac015ce19435876635e5c6d21db31998b0920f675f131e043b8 \ + --hash=sha256:73a205dce83953d131a4aa1e0fd917a2fd1c5b1eef251e9d7152efefcbf5caf7 \ + --hash=sha256:7abcee80084cda3f7691f3eb1ce480d8df49cec637b429aa35986c1de71738aa \ + --hash=sha256:8c25ceb16df5b9435f3f6a9829204985b0e0cbee3b48aacd432c7d2c850b44d9 \ + --hash=sha256:966fe0e9c67490071f14c0d2b1cb2dfb3023c5ce39457343931415f08382f2db \ + --hash=sha256:9e82dcc8e56052715fb18b2429e3bca4823b1629136a2084fc45a9a5cecb9b64 \ + --hash=sha256:b20133d204d2bb56ba047642199603876c872026ca53e79c35b83772ab2cc505 \ + --hash=sha256:b39efa323140595abd3ecca8529d321ae50f55f3aa3ba9cc81ea56a6011953d5 \ + --hash=sha256:b47db11c2c3525083296069b98ac5221907455e989ae0c2e3008bde851921615 \ + --hash=sha256:b87e65d263b3e5d3bb92a57e2a6638e2f31110fa7aa890c7b2dbba42248d0a3f \ + --hash=sha256:b970c6da94d5bb18629db453d14f2a1300f6bf59b61e9b82377931ef95504866 \ + --hash=sha256:be9fcb48a55f023493482827d4f459bd263cc20efde64f204b97c123201850c6 \ + --hash=sha256:c2bc30226390d60ea19d9f82b19db005fe0452154a23c1c410c12ea801e43561 \ + --hash=sha256:c83782480a4a9da4d0feb51950131ba32e12e70813848b3343f6e18c28a66838 \ + --hash=sha256:cbc77da8c523d5abd028635ba850a6966fcee2c82e2bf65a41d1d8afe0f98be9 \ + --hash=sha256:ccac2bfebc306b862133e3bb71f3f6ee8bb525240089b2d952e4144b3a6d5da7 \ + --hash=sha256:d0527ce944105f257f605a827d6ebead966c752038b6e8656abb9c5edee6fc68 \ + --hash=sha256:d8ecde755e2e91bf773fc94e8c9d730cd7f2007004cb492263a794ec3899a1c8 \ + --hash=sha256:e3fb64c420688e5319ae25113a354015abbd8dffbfbc41781a1ea66fc7622ac3 \ + --hash=sha256:e5dfc1e64de5677cec922ffa8da89c546d0415bf6efdf081842e5d44c84e1f0e \ + --hash=sha256:ec5e529fb80935c94fe7b729f9972b50e351a0e6b50aa294fd5cabb109fcc29a \ + --hash=sha256:f37d847238971164fdbc68ade6f6574aecc9c0af714190e2083429ff68f4ce9d \ + --hash=sha256:f78ff2c9ed8dc2d036b0f4d640e22522213d047c1b14e61205a7e55c80a494d4 \ + --hash=sha256:f89660a348f4f78a92366240a61404e337586ef7f5909a2fef59ca88ef505493 \ + --hash=sha256:fc1e275c2f1d97b1a6450b8b0ea3ebfa6e087a611c2b26cb2404d48588abab7b + # via google-auth +decorator==5.3.1 \ + --hash=sha256:4cbcdd55a6efadb9dbea26b858f4fb3264567b52d69ca0d25b721b553f60ea82 \ + --hash=sha256:f47fe6fdbd2edd623ecfe36875d37aba411624e2670dd395dddae1358689bb3c # via gcsfs etils[epath]==1.14.0 \ --hash=sha256:8136e7f4c4173cd0af0ca5481c4475152f0b8686192951eefa60ee8711e1ede4 \ --hash=sha256:b5df7341f54dbe1405a4450b2741207b4a8c279780402b45f87202b94dfc52b4 # via xprof-nightly +fire==0.7.1 \ + --hash=sha256:3b208f05c736de98fb343310d090dcc4d8c78b2a89ea4f32b837c586270a9cbf \ + --hash=sha256:e43fd8a5033a9001e7e2973bab96070694b9f12f2e0ecf96d4683971b5ab1882 + # via xprof-nightly frozenlist==1.8.0 \ --hash=sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686 \ --hash=sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0 \ @@ -554,27 +536,27 @@ frozenlist==1.8.0 \ # via # aiohttp # aiosignal -fsspec==2026.4.0 \ - --hash=sha256:11ef7bb35dab8a394fde6e608221d5cf3e8499401c249bebaeaad760a1a8dec2 \ - --hash=sha256:301d8ac70ae90ef3ad05dcf94d6c3754a097f9b5fe4667d2787aa359ec7df7e4 +fsspec==2026.6.0 \ + --hash=sha256:02e0b71817df9b2169dc30a16832045764def1191b43dcff5bb85bdee212d2a1 \ + --hash=sha256:f5bac145310fe30e16e1471bd6840b2d990d609e872251d7e674241822abf01a # via # etils # gcsfs # xprof-nightly -gcsfs==2026.4.0 \ - --hash=sha256:d9e838834d8cce6cb623c6a6a5fad66a4d122dc5c609d4b1c1977b55f759dcc5 \ - --hash=sha256:f3d80dd1c98737798bc84472e6ff9c59873bd17a26e170d46ce90b3118db4390 +gcsfs==2026.7.0 \ + --hash=sha256:2df36ce1e9010e76dc4295774030495a97496838483c619e9f63bc0ab7bdc770 \ + --hash=sha256:8ce6c08ea64302d8fd4bc23a615ccd9e7752541cde1a88ec8362109bd79cde78 # via xprof-nightly -google-api-core[grpc]==2.30.3 \ - --hash=sha256:a85761ba72c444dad5d611c2220633480b2b6be2521eca69cca2dbb3ffd6bfe8 \ - --hash=sha256:e601a37f148585319b26db36e219df68c5d07b6382cff2d580e83404e44d641b +google-api-core[grpc]==2.31.0 \ + --hash=sha256:2be84ee0f584c48e6bde1b36766e23348b361fb7e55e56135fc76ce1c397f9c2 \ + --hash=sha256:ef79fb3784c71cbac89cbd03301ba0c8fb8ad2aa95d7f9204dd9628f7adf59ab # via # google-cloud-core # google-cloud-storage # google-cloud-storage-control -google-auth==2.50.0 \ - --hash=sha256:04382175e28b94f49694977f0a792688b59a668def1499e9d8de996dc9ce5b15 \ - --hash=sha256:f35eafb191195328e8ce10a7883970877e7aeb49c2bfaa54aa0e394316d353d0 +google-auth==2.55.2 \ + --hash=sha256:97ae7790ff740f2bc9db60eb864a7804f4ac19f5f02c38b3d942f2fea6e9b9ae \ + --hash=sha256:d715f265f2cafc6a5f1bf0dc19870d20e3119f6f6682785a250bce3d03d38a3b # via # gcsfs # google-api-core @@ -582,21 +564,23 @@ google-auth==2.50.0 \ # google-cloud-core # google-cloud-storage # google-cloud-storage-control -google-auth-oauthlib==1.3.1 \ - --hash=sha256:14c22c7b3dd3d06dbe44264144409039465effdd1eef94f7ce3710e486cc4bfa \ - --hash=sha256:1a139ef23f1318756805b0e95f655c238bffd29655329a2978218248da4ee7f8 +google-auth-oauthlib==1.4.0 \ + --hash=sha256:18b5e28880eb8eba9065c436becdc0ee8e4b59117a73a510679c82f70cd363d2 \ + --hash=sha256:251314f213a9ee46a5ae73988e84fd7cca8bb68e7ecf4bfd45940f9e7f51d070 # via gcsfs -google-cloud-core==2.5.1 \ - --hash=sha256:3dc94bdec9d05a31d9f355045ed0f369fbc0d8c665076c734f065d729800f811 \ - --hash=sha256:ea62cdf502c20e3e14be8a32c05ed02113d7bef454e40ff3fab6fe1ec9f1f4e7 +google-cloud-core==2.6.0 \ + --hash=sha256:6d63ac8e5eca6d9e4319d0a1e2265fadcd7f1049904378caecfa01cf52dd869e \ + --hash=sha256:e76149739f90fac1fc6757c09f47eaccb3145b54adbd7759b0f7c4b235f46c83 # via google-cloud-storage -google-cloud-storage==3.10.1 \ - --hash=sha256:97db9aa4460727982040edd2bd13ff3d5e2260b5331ad22895802da1fc2a5286 \ - --hash=sha256:a72f656759b7b99bda700f901adcb3425a828d4a29f911bc26b3ea79c5b1217f - # via gcsfs -google-cloud-storage-control==1.11.0 \ - --hash=sha256:98d2a3be8ab124ae09c33c1b8e4aadc68597c171c60c33e62666bcc74493f8ee \ - --hash=sha256:e97a0f3c99bc5ec5a2a770298ca9e8d5feeaa41bb55f2f2d7b36df2b817ee950 +google-cloud-storage==3.12.0 \ + --hash=sha256:03ae9847c6babb368f35f054126b8a08cbc0e3266efb990eb17b9926a45cf3be \ + --hash=sha256:3880773754ddf7c27567b04e2a4d193950b6b99429f37b9097d873686e95b09c + # via + # gcsfs + # xprof-nightly +google-cloud-storage-control==1.12.0 \ + --hash=sha256:20f7f1252fa5635d47e12f746aa69d719e31eb9405cbbd14cdfba317db27d421 \ + --hash=sha256:49090d03532c0c84c6246a5fd490c31fd4bbffb4c7771c0a6744ec23a194a5f6 # via gcsfs google-crc32c==1.8.0 \ --hash=sha256:014a7e68d623e9a4222d663931febc3033c5c7c9730785727de2a81f87d5bab8 \ @@ -635,13 +619,13 @@ google-crc32c==1.8.0 \ # via # google-cloud-storage # google-resumable-media -google-resumable-media==2.8.2 \ - --hash=sha256:82b6d8ccd11765268cdd2a2123f417ec806b8eef3000a9a38dfe3033da5fb220 \ - --hash=sha256:f3354a182ebd193ae3f42e3ef95e6c9b10f128320de23ac7637236713b1acd70 +google-resumable-media==2.10.0 \ + --hash=sha256:88152884bee37b2bf36a0ab81ad8c7fd12212c9803dd981d77c1b35b02d34e7c \ + --hash=sha256:e324bc9d0fdae4c52a08ae90456edc4e71ece858399e1217ac0eb3a51d6bc6ee # via google-cloud-storage -googleapis-common-protos[grpc]==1.74.0 \ - --hash=sha256:57971e4eeeba6aad1163c1f0fc88543f965bb49129b8bb55b2b7b26ecab084f1 \ - --hash=sha256:702216f78610bb510e3f12ac3cafd281b7ac45cc5d86e90ad87e4d301a3426b5 +googleapis-common-protos[grpc]==1.75.0 \ + --hash=sha256:53a062ff3c32552fbd62c11fe23768b78e4ddf0494d5e5fd97d3f4689c75fbbd \ + --hash=sha256:961ed60399c457ceb0ee8f285a84c870aabc9c6a832b9d37bb281b5bebde43ed # via # google-api-core # grpc-google-iam-v1 @@ -650,91 +634,81 @@ grpc-google-iam-v1==0.14.4 \ --hash=sha256:392b3796947ed6334e61171d9ab06bf7eb357f554e5fc7556ad7aab6d0e17038 \ --hash=sha256:412facc320fcbd94034b4df3d557662051d4d8adfa86e0ddb4dca70a3f739964 # via google-cloud-storage-control -grpcio==1.80.0 \ - --hash=sha256:00168469238b022500e486c1c33916acf2f2a9b2c022202cf8a1885d2e3073c1 \ - --hash=sha256:02e64bb0bb2da14d947a49e6f120a75e947250aebe65f9629b62bb1f5c14e6e9 \ - --hash=sha256:05d55e1798756282cddd52d56c896b3e7d673e3a8798c2f1cd05ba249a3bb4de \ - --hash=sha256:09e5e478b3d14afd23f12e49e8b44c8684ac3c5f08561c43a5b9691c54d136ab \ - --hash=sha256:0cb517eb1d0d0aaf1d87af7cc5b801d686557c1d88b2619f5e31fab3c2315921 \ - --hash=sha256:1b97cd29a8eda100b559b455331c487a80915b6ea6bd91cf3e89836c4ee8d957 \ - --hash=sha256:256507e2f524092f1473071a05e65a5b10d84b82e3ff24c5b571513cfaa61e2f \ - --hash=sha256:29aca15edd0688c22ba01d7cc01cb000d72b2033f4a3c72a81a19b56fd143257 \ - --hash=sha256:2bea16af2750fd0a899bf1abd9022244418b55d1f37da2202249ba4ba673838d \ - --hash=sha256:2dcc70e9f0ba987526e8e8603a610fb4f460e42899e74e7a518bf3c68fe1bf05 \ - --hash=sha256:2ed770b4c06984f3b47eb0517b1c69ad0b84ef3f40128f51448433be904634cd \ - --hash=sha256:31b9ac4ad1aa28ffee5503821fafd09e4da0a261ce1c1281c6c8da0423c83b6e \ - --hash=sha256:33eb763f18f006dc7fee1e69831d38d23f5eccd15b2e0f92a13ee1d9242e5e02 \ - --hash=sha256:367ce30ba67d05e0592470428f0ec1c31714cab9ef19b8f2e37be1f4c7d32fae \ - --hash=sha256:3b01e1f5464c583d2f567b2e46ff0d516ef979978f72091fd81f5ab7fa6e2e7f \ - --hash=sha256:3cb8130ba457d2aa09fa6b7c3ed6b6e4e6a2685fce63cb803d479576c4d80e21 \ - --hash=sha256:3d4147a97c8344d065d01bbf8b6acec2cf86fb0400d40696c8bdad34a64ffc0e \ - --hash=sha256:43168871f170d1e4ed16ae03d10cd21efa29f190e710a624cee7e5ae07da6f4f \ - --hash=sha256:448c884b668b868562b1bda833c5fce6272d26e1926ec46747cda05741d302c1 \ - --hash=sha256:4560cf0e86514595dbbd330cd65b7afad4b5c4b8c4905c041cfffa138d45e6fd \ - --hash=sha256:46c2390b59d67f84e882694d489f5b45707c657832d7934859ceb8c33f467069 \ - --hash=sha256:4e78c4ac0d97dc2e569b2f4bcbbb447491167cb358d1a389fc4af71ab6f70411 \ - --hash=sha256:4ed39fbdcf9b87370f6e8df4e39ca7b38b3e5e9d1b0013c7b6be9639d6578d14 \ - --hash=sha256:50a9871536d71c4fba24ee856abc03a87764570f0c457dd8db0b4018f379fed9 \ - --hash=sha256:51b4a7189b0bef2aa30adce3c78f09c83526cf3dddb24c6a96555e3b97340440 \ - --hash=sha256:52d143637e3872633fc7dd7c3c6a1c84e396b359f3a72e215f8bf69fd82084fc \ - --hash=sha256:5c07e82e822e1161354e32da2662f741a4944ea955f9f580ec8fb409dd6f6060 \ - --hash=sha256:627fb7312171cdc52828bd6fac8d7028ff2a64b89f1957b6f3416caa2218d141 \ - --hash=sha256:68e5851ac4b9afe07e7f84483803ad167852570d65326b34d54ca560bfa53fb6 \ - --hash=sha256:7b641fc3f1dc647bfd80bd713addc68f6d145956f64677e56d9ebafc0bd72388 \ - --hash=sha256:8502122a3cc1714038e39a0b071acb1207ca7844208d5ea0d091317555ee7106 \ - --hash=sha256:873ff5d17d68992ef6605330127425d2fc4e77e612fa3c3e0ed4e668685e3140 \ - --hash=sha256:886457a7768e408cdce226ad1ca67d2958917d306523a0e21e1a2fdaa75c9c9c \ - --hash=sha256:8ac393b58aa16991a2f1144ec578084d544038c12242da3a215966b512904d0f \ - --hash=sha256:8eb613f02d34721f1acf3626dfdb3545bd3c8505b0e52bf8b5710a28d02e8aa7 \ - --hash=sha256:92d787312e613754d4d8b9ca6d3297e69994a7912a32fa38c4c4e01c272974b0 \ - --hash=sha256:93b6f823810720912fd131f561f91f5fed0fda372b6b7028a2681b8194d5d294 \ - --hash=sha256:9a6284a5d907c37db53350645567c522be314bac859a64a7a5ca63b77bb7958f \ - --hash=sha256:9fe648599c0e37594c4809d81a9e77bd138cc82eb8baa71b6a86af65426723ff \ - --hash=sha256:a1dc80fe55685b4a543555e6eef975303b36c8db1023b1599b094b92aa77965f \ - --hash=sha256:a361c20ec1ccd3c3953d20fb6d7b4125093bdd10dff44c5e2bbb39e58917cedc \ - --hash=sha256:a72d84ad0514db063e21887fbacd1fd7acb4d494a564cae22227cd45c7fbf199 \ - --hash=sha256:aacdfb4ed3eb919ca997504d27e03d5dba403c85130b8ed450308590a738f7a4 \ - --hash=sha256:ba0915d51fd4ced2db5ff719f84e270afe0e2d4c45a7bdb1e8d036e4502928c2 \ - --hash=sha256:ba0db34f7e1d803a878284cd70e4c63cb6ae2510ba51937bf8f45ba997cefcf7 \ - --hash=sha256:bac1d573dfa84ce59a5547073e28fa7326d53352adda6912e362da0b917fcef4 \ - --hash=sha256:c51bf8ac4575af2e0678bccfb07e47321fc7acb5049b4482832c5c195e04e13a \ - --hash=sha256:c624cc9f1008361014378c9d776de7182b11fe8b2e5a81bc69f23a295f2a1ad0 \ - --hash=sha256:c71309cfce2f22be26aa4a847357c502db6c621f1a49825ae98aa0907595b193 \ - --hash=sha256:ce1794f4ea6cc3ca29463f42d665c32ba1b964b48958a66497917fe9069f26e6 \ - --hash=sha256:d334591df610ab94714048e0d5b4f3dd5ad1bee74dfec11eee344220077a79de \ - --hash=sha256:d8e11f167935b3eb089ac9038e1a063e6d7dbe995c0bb4a661e614583352e76f \ - --hash=sha256:dc053420fc75749c961e2a4c906398d7c15725d36ccc04ae6d16093167223b58 \ - --hash=sha256:deb10a1528473c11f72a0939eed36d83e847d7cbb63e8cc5611fb7a912d38614 \ - --hash=sha256:dfab85db094068ff42e2a3563f60ab3dddcc9d6488a35abf0132daec13209c8a \ - --hash=sha256:e172cf795a3ba5246d3529e4d34c53db70e888fa582a8ffebd2e6e48bc0cba50 \ - --hash=sha256:e9e408fc016dffd20661f0126c53d8a31c2821b5c13c5d67a0f5ed5de93319ad \ - --hash=sha256:ec0a592e926071b4abad50c1495cd0d0d513324b3ff5e7267067c33ba27506e4 \ - --hash=sha256:f14b618fc30de822681ee986cfdcc2d9327229dc4c98aed16896761cacd468b9 \ - --hash=sha256:f49eddcac43c3bf350c0385366a58f36bed8cc2c0ec35ef7b74b49e56552c0c2 \ - --hash=sha256:f7691a6788ad9196872f95716df5bc643ebba13c97140b7a5ee5c8e75d1dea81 +grpcio==1.81.1 \ + --hash=sha256:0490c30c261eded63f3f354979f9dc4502a9fb944cccb60cd9dc85f5a7349854 \ + --hash=sha256:0a37165cc80b1a368384b383e63a4c38116a10467ae44c904d2d7468c4470ec2 \ + --hash=sha256:12b7524c88d4026d3dcb7b0ebe16b6714f3b4af402ddd0f0639ab064a00c87c3 \ + --hash=sha256:15641444eca4a29358107b3dceb74c1c6305c55c822fd199b458aaea4068a7fb \ + --hash=sha256:1b22c80559854b789a01fd89e8929b3798a156c0829b5282a8939f33ad4115ad \ + --hash=sha256:1e123f9b37edb8375fd74130d1f69c944bbf0a7b06761ae7211154b8759e94d2 \ + --hash=sha256:24c8e57504c8f45b237e40b99262d181071e5099a07053695b75d97bb53053a0 \ + --hash=sha256:2c2e2ae6867c2966b8daccc836d54a13218e0007e9a490aeb81dd05be64d22d7 \ + --hash=sha256:30e825f6848d9f18bba350ed6c75c1b02a0b5184474a31db9a32b1fa66fd8c79 \ + --hash=sha256:3768a5ff1b2125e6f552e561b6b2dca0e64982d8949689b4df145cf8b98d7821 \ + --hash=sha256:3ad74f8bb1a18963914c5452d289422830b39459e8776ebbcd207be1fbfb1d94 \ + --hash=sha256:410482da976329fe5f4067270401b12cf2bd552ff8020f054ecfaddb5475f9d6 \ + --hash=sha256:428bec0161b48d8cf583c068591bc0016d0d9cfff52462b72b3884861ea768c5 \ + --hash=sha256:506f48f2f9c29b143fca3dad7b0d518c188b6c9648c75a2ae6e2d9f2c13a060b \ + --hash=sha256:58ad1131c300d3c9b933802b3cc4dc69d380822935ba50b28703156ea826fbf7 \ + --hash=sha256:592b5fee597faa91cce2dd294dd7d9a1c83d76c4dbf877e33ec1adb866b2fbed \ + --hash=sha256:61233fe8951e5c85dff81c2458b6528624760166946b5b47ea150a589168411f \ + --hash=sha256:62481553b1793a27e9b9c3cf9e5bd483ef045ca72462592074b46d42b0c4d9b9 \ + --hash=sha256:6282caffb41ec326d4cb67ca9cf53b739d1b2f975a2acb498c7418e9f7d9a416 \ + --hash=sha256:69ef28e54fc85397f91b8c19592b8ef3d81952080366914823bd8572a2958120 \ + --hash=sha256:6f9a0c9c1cc15c112d1c053064fd032b64917062292c3d70aea280e02ae10b77 \ + --hash=sha256:6fa10a767143a5e82e8eaab53918af0cd8909a57a27f8cb2288b80a613ac671b \ + --hash=sha256:766bc7c9a9c340342f4c864ccbda8e78111e4751f13b895812b9c148fb79e9d0 \ + --hash=sha256:78e29211f26da2fdd0e9c6d2b79f489476140cf7029b6a64808ade7ca4156a42 \ + --hash=sha256:819edbdcb42ab8598b494bcf0222684bbb7a3c772bd1b1f0be7e029a6063c28e \ + --hash=sha256:85b10a45b8993d195c4f3ff57025b8d1e11834909ee475c403bfa60cb4caefaf \ + --hash=sha256:88268ca418cacea64cecb0d1d600d3c6b3a8038fcba02e1e205178c5b1f47661 \ + --hash=sha256:8b39472beafc0bdcafc4c8c73ad082ebfdb449d566897a61e7acb4fa88089115 \ + --hash=sha256:8ea1936c26b99999b27479853039a7f34713f56c49375ad52b38535ec93a796c \ + --hash=sha256:98a07f9bf591e3a8919797bee1c53f026ba4acd587e5a4404c8e57c9ec36b2a5 \ + --hash=sha256:a185a04039df6cae8648bc8ab6d6fde7bf94f7188ecf7828e76ac52eef1e41d6 \ + --hash=sha256:a35009284d0d3d5c2c9601c164a911b8b4331608d98a9a66d47d97bb2f522b70 \ + --hash=sha256:a3acb384427816dd5d470f47e62137b87f74da694faa8a50147012cf40df276a \ + --hash=sha256:aa2ba7d2ad6df4d80127cea65e5b8d5e2c3adbf153ff4804452836328aca7c54 \ + --hash=sha256:b10e1ff4756ed27d5a29d7fc79cfce7ef1ff56ad20025b89bac7cf79e09abbbe \ + --hash=sha256:b137f4bf3ada9dc44d411478decc6ff09a79ed30b306cd2abaa98408c3588137 \ + --hash=sha256:b259a04a737cb3496be0901328eb8b7552ed8df4865d8c8f1cf1bffcfc0776a3 \ + --hash=sha256:b427c19380991a4eaab2f6144b64b99b412043314c6bf4ab544f97bb31ee4190 \ + --hash=sha256:bb693b1e3d9a2f3fd228e2110daf4b5aeedb36761ca1e4282f74725f6d89f611 \ + --hash=sha256:c261d74b1a945cf895a9d6eccd1685a8e837531beaab782da4d630a8d12deffb \ + --hash=sha256:c5bf2dc311127d91230cc79b92188c082634a06cf66c5234db49a43b910183b0 \ + --hash=sha256:ca1cc11d82677b9662082e5478b7528e2b7db7beaa6bdff42bd62789d81be399 \ + --hash=sha256:d4b2dddfc219f54f956ccd53cf76a1d338ffe68fc7f2849ec9c7feb9927ff692 \ + --hash=sha256:d71d30f2d92f67d944631c523713934fee37292469e182ebcd2c1dd8a64ce53f \ + --hash=sha256:d865db4a6318e1c1bea83292e0ed231090538fc4ca45425b0f0480eb338bbc6e \ + --hash=sha256:e2aa72e3ce1770317ef534f63d397b55e130725f5149bd36077c3b539019db27 \ + --hash=sha256:e3657301562ac3cb8018d30d0d3ebfa39932239f7b5703422057ef14b69949f5 \ + --hash=sha256:e64dd101d380a115cc5a0c7856788adb535f1a4e21fc543775602f8be95180ae \ + --hash=sha256:e8ca6a1fcdb2943c9cbc1804a1baf3acb6071d72a471591678ded84218006e14 \ + --hash=sha256:edb59506291b647a30884b1d51a599d605f40b20af4a7dc3d33786a47a31de60 \ + --hash=sha256:f9a0ebbe45c29b5e5866593c12b78bd9035f0f0f0d4bc8361680cd580d99db49 # via # google-api-core # google-cloud-storage-control # googleapis-common-protos # grpc-google-iam-v1 # grpcio-status -grpcio-status==1.62.3 \ - --hash=sha256:289bdd7b2459794a12cf95dc0cb727bd4a1742c37bd823f760236c937e53a485 \ - --hash=sha256:f9049b762ba8de6b1086789d8315846e094edac2c50beaf462338b301a8fd4b8 +grpcio-status==1.81.1 \ + --hash=sha256:08072fa9995f4a95c647fc6f4f85e2411573d00087bcabdf30f260114338f232 \ + --hash=sha256:9389a03e746017b10f0630c064289201458f3ce01f5d7ef4b0bebc1ef6cf82ad # via google-api-core gviz-api==1.10.0 \ --hash=sha256:846692dd8cc73224fc31b18e41589bd934e1cc05090c6576af4b4b26c2e71b90 \ --hash=sha256:a05055fed8c279f34f4b496eace7648c7fe9c1b06851e8a36e748541f1adbb05 # via xprof-nightly -idna==3.13 \ - --hash=sha256:585ea8fe5d69b9181ec1afba340451fba6ba764af97026f92a91d4eef164a242 \ - --hash=sha256:892ea0cde124a99ce773decba204c5552b69c3c67ffd5f232eb7696135bc8bb3 +idna==3.18 \ + --hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \ + --hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848 # via # requests # yarl -jaraco-functools==4.4.0 \ - --hash=sha256:9eec1e36f45c818d9bf307c8948eb03b2b56cd44087b3cdc989abca1f20b9176 \ - --hash=sha256:da21933b0417b89515562656547a77b4931f98176eb173644c0d35032a33d6bb +jaraco-functools==4.5.0 \ + --hash=sha256:3bb5665ea4a020cf78a7040e89154c77edadb3ca74f366479669c5999aa70b03 \ + --hash=sha256:79ce39246eddbde4b3a03b77ea5f0f7878dc669b166a66cf3fa8e266aa3fa2f4 # via cheroot markupsafe==3.0.3 \ --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f \ @@ -827,9 +801,9 @@ markupsafe==3.0.3 \ --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \ --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 # via werkzeug -more-itertools==11.0.2 \ - --hash=sha256:392a9e1e362cbc106a2457d37cabf9b36e5e12efd4ebff1654630e76597df804 \ - --hash=sha256:6e35b35f818b01f691643c6c611bc0902f2e92b46c18fffa77ae1e7c46e912e4 +more-itertools==11.1.0 \ + --hash=sha256:48e8f4d9e7e5878571ecf6f2b4e57634f93cd474cc8cfbd2376f2d11b396e30d \ + --hash=sha256:4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192 # via # cheroot # jaraco-functools @@ -987,147 +961,146 @@ oauthlib==3.3.1 \ --hash=sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9 \ --hash=sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1 # via requests-oauthlib -propcache==0.4.1 \ - --hash=sha256:0002004213ee1f36cfb3f9a42b5066100c44276b9b72b4e1504cddd3d692e86e \ - --hash=sha256:0013cb6f8dde4b2a2f66903b8ba740bdfe378c943c4377a200551ceb27f379e4 \ - --hash=sha256:005f08e6a0529984491e37d8dbc3dd86f84bd78a8ceb5fa9a021f4c48d4984be \ - --hash=sha256:031dce78b9dc099f4c29785d9cf5577a3faf9ebf74ecbd3c856a7b92768c3df3 \ - --hash=sha256:05674a162469f31358c30bcaa8883cb7829fa3110bf9c0991fe27d7896c42d85 \ - --hash=sha256:060b16ae65bc098da7f6d25bf359f1f31f688384858204fe5d652979e0015e5b \ - --hash=sha256:120c964da3fdc75e3731aa392527136d4ad35868cc556fd09bb6d09172d9a367 \ - --hash=sha256:15932ab57837c3368b024473a525e25d316d8353016e7cc0e5ba9eb343fbb1cf \ - --hash=sha256:17612831fda0138059cc5546f4d12a2aacfb9e47068c06af35c400ba58ba7393 \ - --hash=sha256:182b51b421f0501952d938dc0b0eb45246a5b5153c50d42b495ad5fb7517c888 \ - --hash=sha256:1cdb7988c4e5ac7f6d175a28a9aa0c94cb6f2ebe52756a3c0cda98d2809a9e37 \ - --hash=sha256:1eb2994229cc8ce7fe9b3db88f5465f5fd8651672840b2e426b88cdb1a30aac8 \ - --hash=sha256:1f0978529a418ebd1f49dad413a2b68af33f85d5c5ca5c6ca2a3bed375a7ac60 \ - --hash=sha256:204483131fb222bdaaeeea9f9e6c6ed0cac32731f75dfc1d4a567fc1926477c1 \ - --hash=sha256:296f4c8ed03ca7476813fe666c9ea97869a8d7aec972618671b33a38a5182ef4 \ - --hash=sha256:2ad890caa1d928c7c2965b48f3a3815c853180831d0e5503d35cf00c472f4717 \ - --hash=sha256:2b16ec437a8c8a965ecf95739448dd938b5c7f56e67ea009f4300d8df05f32b7 \ - --hash=sha256:2bb07ffd7eaad486576430c89f9b215f9e4be68c4866a96e97db9e97fead85dc \ - --hash=sha256:333ddb9031d2704a301ee3e506dc46b1fe5f294ec198ed6435ad5b6a085facfe \ - --hash=sha256:357f5bb5c377a82e105e44bd3d52ba22b616f7b9773714bff93573988ef0a5fb \ - --hash=sha256:35c3277624a080cc6ec6f847cbbbb5b49affa3598c4535a0a4682a697aaa5c75 \ - --hash=sha256:364426a62660f3f699949ac8c621aad6977be7126c5807ce48c0aeb8e7333ea6 \ - --hash=sha256:381914df18634f5494334d201e98245c0596067504b9372d8cf93f4bb23e025e \ - --hash=sha256:3d233076ccf9e450c8b3bc6720af226b898ef5d051a2d145f7d765e6e9f9bcff \ - --hash=sha256:3d902a36df4e5989763425a8ab9e98cd8ad5c52c823b34ee7ef307fd50582566 \ - --hash=sha256:3f7124c9d820ba5548d431afb4632301acf965db49e666aa21c305cbe8c6de12 \ - --hash=sha256:405aac25c6394ef275dee4c709be43745d36674b223ba4eb7144bf4d691b7367 \ - --hash=sha256:41a89040cb10bd345b3c1a873b2bf36413d48da1def52f268a055f7398514874 \ - --hash=sha256:43eedf29202c08550aac1d14e0ee619b0430aaef78f85864c1a892294fbc28cf \ - --hash=sha256:473c61b39e1460d386479b9b2f337da492042447c9b685f28be4f74d3529e566 \ - --hash=sha256:49a2dc67c154db2c1463013594c458881a069fcf98940e61a0569016a583020a \ - --hash=sha256:4b536b39c5199b96fc6245eb5fb796c497381d3942f169e44e8e392b29c9ebcc \ - --hash=sha256:4c3c70630930447f9ef1caac7728c8ad1c56bc5015338b20fed0d08ea2480b3a \ - --hash=sha256:4d3df5fa7e36b3225954fba85589da77a0fe6a53e3976de39caf04a0db4c36f1 \ - --hash=sha256:4d7af63f9f93fe593afbf104c21b3b15868efb2c21d07d8732c0c4287e66b6a6 \ - --hash=sha256:501d20b891688eb8e7aa903021f0b72d5a55db40ffaab27edefd1027caaafa61 \ - --hash=sha256:521a463429ef54143092c11a77e04056dd00636f72e8c45b70aaa3140d639726 \ - --hash=sha256:5558992a00dfd54ccbc64a32726a3357ec93825a418a401f5cc67df0ac5d9e49 \ - --hash=sha256:55c72fd6ea2da4c318e74ffdf93c4fe4e926051133657459131a95c846d16d44 \ - --hash=sha256:564d9f0d4d9509e1a870c920a89b2fec951b44bf5ba7d537a9e7c1ccec2c18af \ - --hash=sha256:580e97762b950f993ae618e167e7be9256b8353c2dcd8b99ec100eb50f5286aa \ - --hash=sha256:5a103c3eb905fcea0ab98be99c3a9a5ab2de60228aa5aceedc614c0281cf6153 \ - --hash=sha256:5c3310452e0d31390da9035c348633b43d7e7feb2e37be252be6da45abd1abcc \ - --hash=sha256:5d4e2366a9c7b837555cf02fb9be2e3167d333aff716332ef1b7c3a142ec40c5 \ - --hash=sha256:5fd37c406dd6dc85aa743e214cef35dc54bbdd1419baac4f6ae5e5b1a2976938 \ - --hash=sha256:60a8fda9644b7dfd5dece8c61d8a85e271cb958075bfc4e01083c148b61a7caf \ - --hash=sha256:66c1f011f45a3b33d7bcb22daed4b29c0c9e2224758b6be00686731e1b46f925 \ - --hash=sha256:671538c2262dadb5ba6395e26c1731e1d52534bfe9ae56d0b5573ce539266aa8 \ - --hash=sha256:678ae89ebc632c5c204c794f8dab2837c5f159aeb59e6ed0539500400577298c \ - --hash=sha256:67fad6162281e80e882fb3ec355398cf72864a54069d060321f6cd0ade95fe85 \ - --hash=sha256:6918ecbd897443087a3b7cd978d56546a812517dcaaca51b49526720571fa93e \ - --hash=sha256:6f6ff873ed40292cd4969ef5310179afd5db59fdf055897e282485043fc80ad0 \ - --hash=sha256:6f8b465489f927b0df505cbe26ffbeed4d6d8a2bbc61ce90eb074ff129ef0ab1 \ - --hash=sha256:71b749281b816793678ae7f3d0d84bd36e694953822eaad408d682efc5ca18e0 \ - --hash=sha256:74c1fb26515153e482e00177a1ad654721bf9207da8a494a0c05e797ad27b992 \ - --hash=sha256:7c2d1fa3201efaf55d730400d945b5b3ab6e672e100ba0f9a409d950ab25d7db \ - --hash=sha256:824e908bce90fb2743bd6b59db36eb4f45cd350a39637c9f73b1c1ea66f5b75f \ - --hash=sha256:8326e144341460402713f91df60ade3c999d601e7eb5ff8f6f7862d54de0610d \ - --hash=sha256:8873eb4460fd55333ea49b7d189749ecf6e55bf85080f11b1c4530ed3034cba1 \ - --hash=sha256:89eb3fa9524f7bec9de6e83cf3faed9d79bffa560672c118a96a171a6f55831e \ - --hash=sha256:8c9b3cbe4584636d72ff556d9036e0c9317fa27b3ac1f0f558e7e84d1c9c5900 \ - --hash=sha256:8e57061305815dfc910a3634dcf584f08168a8836e6999983569f51a8544cd89 \ - --hash=sha256:929d7cbe1f01bb7baffb33dc14eb5691c95831450a26354cd210a8155170c93a \ - --hash=sha256:92d1935ee1f8d7442da9c0c4fa7ac20d07e94064184811b685f5c4fada64553b \ - --hash=sha256:948dab269721ae9a87fd16c514a0a2c2a1bdb23a9a61b969b0f9d9ee2968546f \ - --hash=sha256:981333cb2f4c1896a12f4ab92a9cc8f09ea664e9b7dbdc4eff74627af3a11c0f \ - --hash=sha256:990f6b3e2a27d683cb7602ed6c86f15ee6b43b1194736f9baaeb93d0016633b1 \ - --hash=sha256:99d43339c83aaf4d32bda60928231848eee470c6bda8d02599cc4cebe872d183 \ - --hash=sha256:9a0bd56e5b100aef69bd8562b74b46254e7c8812918d3baa700c8a8009b0af66 \ - --hash=sha256:9a52009f2adffe195d0b605c25ec929d26b36ef986ba85244891dee3b294df21 \ - --hash=sha256:9d2b6caef873b4f09e26ea7e33d65f42b944837563a47a94719cc3544319a0db \ - --hash=sha256:9f302f4783709a78240ebc311b793f123328716a60911d667e0c036bc5dcbded \ - --hash=sha256:a0ee98db9c5f80785b266eb805016e36058ac72c51a064040f2bc43b61101cdb \ - --hash=sha256:a129e76735bc792794d5177069691c3217898b9f5cee2b2661471e52ffe13f19 \ - --hash=sha256:a78372c932c90ee474559c5ddfffd718238e8673c340dc21fe45c5b8b54559a0 \ - --hash=sha256:a9695397f85973bb40427dedddf70d8dc4a44b22f1650dd4af9eedf443d45165 \ - --hash=sha256:ab08df6c9a035bee56e31af99be621526bd237bea9f32def431c656b29e41778 \ - --hash=sha256:ab2943be7c652f09638800905ee1bab2c544e537edb57d527997a24c13dc1455 \ - --hash=sha256:ab4c29b49d560fe48b696cdcb127dd36e0bc2472548f3bf56cc5cb3da2b2984f \ - --hash=sha256:af223b406d6d000830c6f65f1e6431783fc3f713ba3e6cc8c024d5ee96170a4b \ - --hash=sha256:af2a6052aeb6cf17d3e46ee169099044fd8224cbaf75c76a2ef596e8163e2237 \ - --hash=sha256:bcc9aaa5d80322bc2fb24bb7accb4a30f81e90ab8d6ba187aec0744bc302ad81 \ - --hash=sha256:c07fda85708bc48578467e85099645167a955ba093be0a2dcba962195676e859 \ - --hash=sha256:c0d4b719b7da33599dfe3b22d3db1ef789210a0597bc650b7cee9c77c2be8c5c \ - --hash=sha256:c0ef0aaafc66fbd87842a3fe3902fd889825646bc21149eafe47be6072725835 \ - --hash=sha256:c2b5e7db5328427c57c8e8831abda175421b709672f6cfc3d630c3b7e2146393 \ - --hash=sha256:c30b53e7e6bda1d547cabb47c825f3843a0a1a42b0496087bb58d8fedf9f41b5 \ - --hash=sha256:c80ee5802e3fb9ea37938e7eecc307fb984837091d5fd262bb37238b1ae97641 \ - --hash=sha256:c9b822a577f560fbd9554812526831712c1436d2c046cedee4c3796d3543b144 \ - --hash=sha256:cae65ad55793da34db5f54e4029b89d3b9b9490d8abe1b4c7ab5d4b8ec7ebf74 \ - --hash=sha256:cb2d222e72399fcf5890d1d5cc1060857b9b236adff2792ff48ca2dfd46c81db \ - --hash=sha256:cbc3b6dfc728105b2a57c06791eb07a94229202ea75c59db644d7d496b698cac \ - --hash=sha256:cd547953428f7abb73c5ad82cbb32109566204260d98e41e5dfdc682eb7f8403 \ - --hash=sha256:cfc27c945f422e8b5071b6e93169679e4eb5bf73bbcbf1ba3ae3a83d2f78ebd9 \ - --hash=sha256:d472aeb4fbf9865e0c6d622d7f4d54a4e101a89715d8904282bb5f9a2f476c3f \ - --hash=sha256:d62cdfcfd89ccb8de04e0eda998535c406bf5e060ffd56be6c586cbcc05b3311 \ - --hash=sha256:d82ad62b19645419fe79dd63b3f9253e15b30e955c0170e5cebc350c1844e581 \ - --hash=sha256:d8f353eb14ee3441ee844ade4277d560cdd68288838673273b978e3d6d2c8f36 \ - --hash=sha256:daede9cd44e0f8bdd9e6cc9a607fc81feb80fae7a5fc6cecaff0e0bb32e42d00 \ - --hash=sha256:db65d2af507bbfbdcedb254a11149f894169d90488dd3e7190f7cdcb2d6cd57a \ - --hash=sha256:dee69d7015dc235f526fe80a9c90d65eb0039103fe565776250881731f06349f \ - --hash=sha256:e153e9cd40cc8945138822807139367f256f89c6810c2634a4f6902b52d3b4e2 \ - --hash=sha256:e35b88984e7fa64aacecea39236cee32dd9bd8c55f57ba8a75cf2399553f9bd7 \ - --hash=sha256:e53f3a38d3510c11953f3e6a33f205c6d1b001129f972805ca9b42fc308bc239 \ - --hash=sha256:e9b0d8d0845bbc4cfcdcbcdbf5086886bc8157aa963c31c777ceff7846c77757 \ - --hash=sha256:ec17c65562a827bba85e3872ead335f95405ea1674860d96483a02f5c698fa72 \ - --hash=sha256:ecef2343af4cc68e05131e45024ba34f6095821988a9d0a02aa7c73fcc448aa9 \ - --hash=sha256:ed5a841e8bb29a55fb8159ed526b26adc5bdd7e8bd7bf793ce647cb08656cdf4 \ - --hash=sha256:ee17f18d2498f2673e432faaa71698032b0127ebf23ae5974eeaf806c279df24 \ - --hash=sha256:f048da1b4f243fc44f205dfd320933a951b8d89e0afd4c7cacc762a8b9165207 \ - --hash=sha256:f10207adf04d08bec185bae14d9606a1444715bc99180f9331c9c02093e1959e \ - --hash=sha256:f1d2f90aeec838a52f1c1a32fe9a619fefd5e411721a9117fbf82aea638fe8a1 \ - --hash=sha256:f48107a8c637e80362555f37ecf49abe20370e557cc4ab374f04ec4423c97c3d \ - --hash=sha256:f7ee0e597f495cf415bcbd3da3caa3bd7e816b74d0d52b8145954c5e6fd3ff37 \ - --hash=sha256:f93243fdc5657247533273ac4f86ae106cc6445a0efacb9a1bfe982fcfefd90c \ - --hash=sha256:f95393b4d66bfae908c3ca8d169d5f79cd65636ae15b5e7a4f6e67af675adb0e \ - --hash=sha256:fc38cba02d1acba4e2869eef1a57a43dfbd3d49a59bf90dda7444ec2be6a5570 \ - --hash=sha256:fd0858c20f078a32cf55f7e81473d96dcf3b93fd2ccdb3d40fdf54b8573df3af \ - --hash=sha256:fd138803047fb4c062b1c1dd95462f5209456bfab55c734458f15d11da288f8f \ - --hash=sha256:fd2dbc472da1f772a4dae4fa24be938a6c544671a912e30529984dd80400cd88 \ - --hash=sha256:fd6f30fdcf9ae2a70abd34da54f18da086160e4d7d9251f81f3da0ff84fc5a48 \ - --hash=sha256:fe49d0a85038f36ba9e3ffafa1103e61170b28e95b16622e11be0a0ea07c6781 +propcache==0.5.2 \ + --hash=sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427 \ + --hash=sha256:04dc2390d9edbbaef7461f33322555976ffddf0b650a038649d026358714e6c5 \ + --hash=sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa \ + --hash=sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7 \ + --hash=sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a \ + --hash=sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0 \ + --hash=sha256:0fd59b5af35f74da48d905dcbad55449ba13be91823cb05a9bd590bbf5b61660 \ + --hash=sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94 \ + --hash=sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917 \ + --hash=sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42 \ + --hash=sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3 \ + --hash=sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa \ + --hash=sha256:1ca071adabaab6e9219924bbe00af821f1ee7de113a9eca1cdc292de3d120f4d \ + --hash=sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33 \ + --hash=sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a \ + --hash=sha256:2293949b855ce597f2826452d17c2d545fb5622379c4ea6fdf525e9b8e8a2511 \ + --hash=sha256:26a4dca084132874e639895c3135dfad5eb20bae209f62d1aeb31b03e601c3c0 \ + --hash=sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84 \ + --hash=sha256:29cbaac5ea0212663e6845e04b5e188d5a6ae6dd919810ac835bf1d3b42c3f4c \ + --hash=sha256:29f9309a2e42b0d273be006fdb4be2d6c39a47f6f57d8fb1cf9f81481df81b66 \ + --hash=sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821 \ + --hash=sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb \ + --hash=sha256:2f8ea531c794b9d6274acd4e8d2c2ebcac590a4361d27482edd3010b79f1325e \ + --hash=sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853 \ + --hash=sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56 \ + --hash=sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55 \ + --hash=sha256:3b199b9b2b3d6a7edf3183ba8a9a137a22b97f7df525feb5ae1eccf026d2a9c6 \ + --hash=sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704 \ + --hash=sha256:44e488ef40dbb452700b2b1f8188934121f6648f52c295055662d2191959ff82 \ + --hash=sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f \ + --hash=sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64 \ + --hash=sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999 \ + --hash=sha256:4621064bbf28fa77ff64dd5d94367c04684c67d3a5bf1dff25f0cd0d98a38f3b \ + --hash=sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb \ + --hash=sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d \ + --hash=sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4 \ + --hash=sha256:54adaa85a22078d1e306304a40984dc5be99d599bf3dc0a24dc98f7daeab89ab \ + --hash=sha256:552ffadf6ad409844bc5919c42a0a83d88314cedddaea0e41e80a8b8fffe881f \ + --hash=sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03 \ + --hash=sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5 \ + --hash=sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba \ + --hash=sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979 \ + --hash=sha256:5aaa2b923c1944ac8febd6609cb373540a5563e7cbcb0fd770f75dace2eb817b \ + --hash=sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144 \ + --hash=sha256:5fcb98e7598b1ee0addab320d90f65b530297a867dbfe9de52ea838077e16e3d \ + --hash=sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e \ + --hash=sha256:66ea454f095ddf5b6b14f56c064c0941c4788be11e18d2464cf643bf7203ff67 \ + --hash=sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117 \ + --hash=sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa \ + --hash=sha256:6bf3be92233808fcd338eba0fb4d0b59ec5772af4f4ecfcec450d1bfc0f8b5eb \ + --hash=sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96 \ + --hash=sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5 \ + --hash=sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476 \ + --hash=sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191 \ + --hash=sha256:74b70780220e2dd89175ca24b81b68b67c83db499ae611e7f2313cb329801c78 \ + --hash=sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078 \ + --hash=sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837 \ + --hash=sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a \ + --hash=sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba \ + --hash=sha256:8114f28879e0904748e831c3a7774261bd9e75f49be089f389a76f959dcd13fe \ + --hash=sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c \ + --hash=sha256:823581fd5cb08b12a48bfa11fe962a7916766b6170c17b028fbdf762b85eb9bf \ + --hash=sha256:85341b12b9d55bad0bded24cac341bb34289469e03a11f3f583ea1cc1db0326c \ + --hash=sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9 \ + --hash=sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8 \ + --hash=sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe \ + --hash=sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031 \ + --hash=sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913 \ + --hash=sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d \ + --hash=sha256:949c91d1a990cf3b2e8188dfcfb25005e0b834a06c63fa4ef9f360878ce21ecf \ + --hash=sha256:95f1e3f4760d404b13c9976c0229b2b49a3c8e2c62a9ce92efdd2b11ada75e3f \ + --hash=sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539 \ + --hash=sha256:a0e399a2eccb91ed18721f86aa85757727400b6865c89e88934781deb9c8498b \ + --hash=sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285 \ + --hash=sha256:a4840ab0ae0216d952f4b53dc6d0b992bfc2bedbfe360bdd9b548bc184c08959 \ + --hash=sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d \ + --hash=sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4 \ + --hash=sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f \ + --hash=sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836 \ + --hash=sha256:b05d643f944a8c3c4bd86d65ffd87bf3264b617f87791940302bc474d2ff5274 \ + --hash=sha256:b96db7141a592cbc968daf1feea83a118e6ab378af4abbc72b248c895414c22d \ + --hash=sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f \ + --hash=sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e \ + --hash=sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe \ + --hash=sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1 \ + --hash=sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a \ + --hash=sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39 \ + --hash=sha256:c6844ba6364fb12f403928a82cfd295ab103a2b315c77c747b2dbe4a41894ea7 \ + --hash=sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a \ + --hash=sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164 \ + --hash=sha256:cc1177027eda740fdb152706bd215a3f124e3eea15afc39f2cb9fe351b50619e \ + --hash=sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2 \ + --hash=sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0 \ + --hash=sha256:cd416c1de191973c52ff1a12a57446bfc7642797b282d7caf2162d7d1b8aa9a0 \ + --hash=sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335 \ + --hash=sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568 \ + --hash=sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4 \ + --hash=sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80 \ + --hash=sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2 \ + --hash=sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370 \ + --hash=sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4 \ + --hash=sha256:d5a81be28596d6559f6131ef33e10200de6e17643b3c74ce03f9eb103be6ae8b \ + --hash=sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42 \ + --hash=sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a \ + --hash=sha256:decfca4c79dd53ebab484b00cc4b6717d8c369f86e74aa4ca395a64ac651495e \ + --hash=sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757 \ + --hash=sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825 \ + --hash=sha256:e4294d04a94dcab1b3bccd8b66d962dcad411a1d19414b2a41d1445f1de32ad0 \ + --hash=sha256:e59bc9e66329185b93dab73f210f1a37f81cb40f321501db8017c9aea15dba27 \ + --hash=sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf \ + --hash=sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f \ + --hash=sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d \ + --hash=sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366 \ + --hash=sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc \ + --hash=sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c \ + --hash=sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7 \ + --hash=sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702 \ + --hash=sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098 \ + --hash=sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751 \ + --hash=sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e \ + --hash=sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6 # via # aiohttp # yarl -proto-plus==1.27.2 \ - --hash=sha256:6432f75893d3b9e70b9c412f1d2f03f65b11fb164b793d14ae2ca01821d22718 \ - --hash=sha256:b2adde53adadf75737c44d3dcb0104fde65250dfc83ad59168b4aa3e574b6a24 +proto-plus==1.28.0 \ + --hash=sha256:38e5696342835b08fc116f30a25665b29531cda9d5d5643e9b81fc312385abd9 \ + --hash=sha256:a630604310899e73c59ec302e5765c058d412b2f090b9c79c8822589f14955b8 # via # google-api-core # google-cloud-storage-control -protobuf==7.34.1 \ - --hash=sha256:34b84ce27680df7cca9f231043ada0daa55d0c44a2ddfaa58ec1d0d89d8bf60a \ - --hash=sha256:403b093a6e28a960372b44e5eb081775c9b056e816a8029c61231743d63f881a \ - --hash=sha256:5185e0e948d07abe94bb76ec9b8416b604cfe5da6f871d67aad30cbf24c3110b \ - --hash=sha256:8ff40ce8cd688f7265326b38d5a1bed9bfdf5e6723d49961432f83e21d5713e4 \ - --hash=sha256:9ce42245e704cc5027be797c1db1eb93184d44d1cdd71811fb2d9b25ad541280 \ - --hash=sha256:bb3812cd53aefea2b028ef42bd780f5b96407247f20c6ef7c679807e9d188f11 \ - --hash=sha256:d8b2cc79c4d8f62b293ad9b11ec3aebce9af481fa73e64556969f7345ebf9fc7 \ - --hash=sha256:e97b55646e6ce5cbb0954a8c28cd39a5869b59090dfaa7df4598a7fba869468c +protobuf==7.35.1 \ + --hash=sha256:11d6b0ec246892d85215b0a13ca6e0233cf5284b68f0ac02646427f4ff88a799 \ + --hash=sha256:230a75ddfc2de4806e56696ce9640c1cdfdb6543b7cfce98d42a4c0a0e7bdb87 \ + --hash=sha256:24f857477359a85c0c235261b8ba905fd51b2562f4a64ca1df5473f29850cbf6 \ + --hash=sha256:353652e4efd0bca5b5fc2656abf8307ef351f0cf938c9eba09f0e09c20a25c30 \ + --hash=sha256:4bc97768d8fe4ad6743c8a19403e314511ed9f6d13205b687e52421c023ac1b9 \ + --hash=sha256:74758715c53d7158fb76caf4f0cfdacc5329a4b1bb994f865d6cf302d413a1c4 \ + --hash=sha256:b73f9489a4b8b1c9cb1f8ed951c736392592edb24b9d6819f36d2e10b171d5b4 \ + --hash=sha256:ce115a26fe0c39a2c29973d914d327e516a6455464489fe3cd1e51a1b354f81a # via # google-api-core # google-cloud-storage-control @@ -1148,13 +1121,9 @@ pycparser==3.0 \ --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 # via cffi -pyopenssl==26.2.0 \ - --hash=sha256:4f9d971bc5298b8bc1fab282803da04bf000c755d4ad9d99b52de2569ca19a70 \ - --hash=sha256:8c6fcecd1183a7fc897548dfe388b0cdb7f37e018200d8409cf33959dbe35387 - # via xprof-nightly -requests==2.33.1 \ - --hash=sha256:18817f8c57c6263968bc123d237e3b8b08ac046f5456bd1e307ee8f4250d3517 \ - --hash=sha256:4e6d1ef462f3626a1f0a0a9c42dd93c63bad33f9f1c1937509b8c5c8718ab56a +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed # via # gcsfs # google-api-core @@ -1164,9 +1133,9 @@ requests-oauthlib==2.0.0 \ --hash=sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36 \ --hash=sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9 # via google-auth-oauthlib -setuptools==82.0.1 \ - --hash=sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9 \ - --hash=sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb +setuptools==83.0.0 \ + --hash=sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef \ + --hash=sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3 # via xprof-nightly six==1.17.0 \ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ @@ -1174,159 +1143,141 @@ six==1.17.0 \ # via # gviz-api # xprof-nightly -typing-extensions==4.15.0 \ - --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ - --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 +termcolor==3.3.0 \ + --hash=sha256:348871ca648ec6a9a983a13ab626c0acce02f515b9e1983332b17af7979521c5 \ + --hash=sha256:cf642efadaf0a8ebbbf4bc7a31cec2f9b5f21a9f726f4ccbb08192c9c26f43a5 + # via fire +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 # via + # aiohttp # aiosignal # etils # grpcio - # pyopenssl -urllib3==2.6.3 \ - --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ - --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 - # via requests +urllib3==2.7.0 \ + --hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \ + --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 + # via + # requests + # xprof-nightly werkzeug==3.1.8 \ --hash=sha256:63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50 \ --hash=sha256:9bad61a4268dac112f1c5cd4630a56ede601b6ed420300677a869083d70a4c44 # via xprof-nightly -xprof-nightly==2.23.2a20260505 \ - --hash=sha256:1cadc9aeb0063924da030774b9c491804bfb06458fc6622ab473adb785bb35bc \ - --hash=sha256:9619bac6f10b71afc64909d0641b732257b1ebc2bc9f9e80fdf5e15f53388baa \ - --hash=sha256:b2fafa447b221f58c2a848fb136ed00a48239331b5e9c68aeabb9d535ba8125d \ - --hash=sha256:e804d2f135711975b8b6d7f730fb97d1680e17e6b2a225a7ce7775795e835e77 +xprof-nightly==2.23.3a20260707 \ + --hash=sha256:006db5a8c1d6f31b21d80456d098e5ac6156c90cb3d88910d5a60d71f993ae02 \ + --hash=sha256:4d531ac724f16f3bbe318220ccf2fed19d01fddfa1a49568334d9797f9df7b03 \ + --hash=sha256:aba882245c392cd997ced825b7a320d9cced99e9b289452fd65ddfde745ec44c \ + --hash=sha256:da0078ef75c6504a3733ab0f72d5f8846d781e512bbf7c36dabedb7433f2708d # via -r tools/xprof/requirements.in -yarl==1.23.0 \ - --hash=sha256:03214408cfa590df47728b84c679ae4ef00be2428e11630277be0727eba2d7cc \ - --hash=sha256:041b1a4cefacf65840b4e295c6985f334ba83c30607441ae3cf206a0eed1a2e4 \ - --hash=sha256:0793e2bd0cf14234983bbb371591e6bea9e876ddf6896cdcc93450996b0b5c85 \ - --hash=sha256:0e1fdaa14ef51366d7757b45bde294e95f6c8c049194e793eedb8387c86d5993 \ - --hash=sha256:0e40111274f340d32ebcc0a5668d54d2b552a6cca84c9475859d364b380e3222 \ - --hash=sha256:115136c4a426f9da976187d238e84139ff6b51a20839aa6e3720cd1026d768de \ - --hash=sha256:13a563739ae600a631c36ce096615fe307f131344588b0bc0daec108cdb47b25 \ - --hash=sha256:16c6994ac35c3e74fb0ae93323bf8b9c2a9088d55946109489667c510a7d010e \ - --hash=sha256:170e26584b060879e29fac213e4228ef063f39128723807a312e5c7fec28eff2 \ - --hash=sha256:17235362f580149742739cc3828b80e24029d08cbb9c4bda0242c7b5bc610a8e \ - --hash=sha256:1932b6b8bba8d0160a9d1078aae5838a66039e8832d41d2992daa9a3a08f7860 \ - --hash=sha256:1b6b572edd95b4fa8df75de10b04bc81acc87c1c7d16bcdd2035b09d30acc957 \ - --hash=sha256:1c3a3598a832590c5a3ce56ab5576361b5688c12cb1d39429cf5dba30b510760 \ - --hash=sha256:1c57676bdedc94cd3bc37724cf6f8cd2779f02f6aba48de45feca073e714fe52 \ - --hash=sha256:1dc702e42d0684f42d6519c8d581e49c96cefaaab16691f03566d30658ee8788 \ - --hash=sha256:21d1b7305a71a15b4794b5ff22e8eef96ff4a6d7f9657155e5aa419444b28912 \ - --hash=sha256:23f371bd662cf44a7630d4d113101eafc0cfa7518a2760d20760b26021454719 \ - --hash=sha256:2569b67d616eab450d262ca7cb9f9e19d2f718c70a8b88712859359d0ab17035 \ - --hash=sha256:263cd4f47159c09b8b685890af949195b51d1aa82ba451c5847ca9bc6413c220 \ - --hash=sha256:2803ed8b21ca47a43da80a6fd1ed3019d30061f7061daa35ac54f63933409412 \ - --hash=sha256:2a6940a074fb3c48356ed0158a3ca5699c955ee4185b4d7d619be3c327143e05 \ - --hash=sha256:2e27c8841126e017dd2a054a95771569e6070b9ee1b133366d8b31beb5018a41 \ - --hash=sha256:31c9921eb8bd12633b41ad27686bbb0b1a2a9b8452bfdf221e34f311e9942ed4 \ - --hash=sha256:34b6cf500e61c90f305094911f9acc9c86da1a05a7a3f5be9f68817043f486e4 \ - --hash=sha256:3650dc2480f94f7116c364096bc84b1d602f44224ef7d5c7208425915c0475dd \ - --hash=sha256:389871e65468400d6283c0308e791a640b5ab5c83bcee02a2f51295f95e09748 \ - --hash=sha256:39004f0ad156da43e86aa71f44e033de68a44e5a31fc53507b36dd253970054a \ - --hash=sha256:394906945aa8b19fc14a61cf69743a868bb8c465efe85eee687109cc540b98f4 \ - --hash=sha256:3ceb13c5c858d01321b5d9bb65e4cf37a92169ea470b70fec6f236b2c9dd7e34 \ - --hash=sha256:411225bae281f114067578891bc75534cfb3d92a3b4dfef7a6ca78ba354e6069 \ - --hash=sha256:44bb7bef4ea409384e3f8bc36c063d77ea1b8d4a5b2706956c0d6695f07dcc25 \ - --hash=sha256:4503053d296bc6e4cbd1fad61cf3b6e33b939886c4f249ba7c78b602214fabe2 \ - --hash=sha256:4764a6a7588561a9aef92f65bda2c4fb58fe7c675c0883862e6df97559de0bfb \ - --hash=sha256:4966242ec68afc74c122f8459abd597afd7d8a60dc93d695c1334c5fd25f762f \ - --hash=sha256:4a42e651629dafb64fd5b0286a3580613702b5809ad3f24934ea87595804f2c5 \ - --hash=sha256:4a59ba56f340334766f3a4442e0efd0af895fae9e2b204741ef885c446b3a1a8 \ - --hash=sha256:4c41e021bc6d7affb3364dc1e1e5fa9582b470f283748784bd6ea0558f87f42c \ - --hash=sha256:5023346c4ee7992febc0068e7593de5fa2bf611848c08404b35ebbb76b1b0512 \ - --hash=sha256:50f9d8d531dfb767c565f348f33dd5139a6c43f5cbdf3f67da40d54241df93f6 \ - --hash=sha256:51430653db848d258336cfa0244427b17d12db63d42603a55f0d4546f50f25b5 \ - --hash=sha256:531ef597132086b6cf96faa7c6c1dcd0361dd5f1694e5cc30375907b9b7d3ea9 \ - --hash=sha256:53ad387048f6f09a8969631e4de3f1bf70c50e93545d64af4f751b2498755072 \ - --hash=sha256:53b1ea6ca88ebd4420379c330aea57e258408dd0df9af0992e5de2078dc9f5d5 \ - --hash=sha256:575aa4405a656e61a540f4a80eaa5260f2a38fff7bfdc4b5f611840d76e9e277 \ - --hash=sha256:578110dd426f0d209d1509244e6d4a3f1a3e9077655d98c5f22583d63252a08a \ - --hash=sha256:5ec2f42d41ccbd5df0270d7df31618a8ee267bfa50997f5d720ddba86c4a83a6 \ - --hash=sha256:5ee586fb17ff8f90c91cf73c6108a434b02d69925f44f5f8e0d7f2f260607eae \ - --hash=sha256:5f10fd85e4b75967468af655228fbfd212bdf66db1c0d135065ce288982eda26 \ - --hash=sha256:609d3614d78d74ebe35f54953c5bbd2ac647a7ddb9c30a5d877580f5e86b22f2 \ - --hash=sha256:62694e275c93d54f7ccedcfef57d42761b2aad5234b6be1f3e3026cae4001cd4 \ - --hash=sha256:63e92247f383c85ab00dd0091e8c3fa331a96e865459f5ee80353c70a4a42d70 \ - --hash=sha256:682bae25f0a0dd23a056739f23a134db9f52a63e2afd6bfb37ddc76292bbd723 \ - --hash=sha256:6b41389c19b07c760c7e427a3462e8ab83c4bb087d127f0e854c706ce1b9215c \ - --hash=sha256:6e87a6e8735b44816e7db0b2fbc9686932df473c826b0d9743148432e10bb9b9 \ - --hash=sha256:6f0fd84de0c957b2d280143522c4f91a73aada1923caee763e24a2b3fda9f8a5 \ - --hash=sha256:70efd20be968c76ece7baa8dafe04c5be06abc57f754d6f36f3741f7aa7a208e \ - --hash=sha256:71d006bee8397a4a89f469b8deb22469fe7508132d3c17fa6ed871e79832691c \ - --hash=sha256:73309162a6a571d4cbd3b6a1dcc703c7311843ae0d1578df6f09be4e98df38d4 \ - --hash=sha256:75e3026ab649bf48f9a10c0134512638725b521340293f202a69b567518d94e0 \ - --hash=sha256:76855800ac56f878847a09ce6dba727c93ca2d89c9e9d63002d26b916810b0a2 \ - --hash=sha256:7c6b9461a2a8b47c65eef63bb1c76a4f1c119618ffa99ea79bc5bb1e46c5821b \ - --hash=sha256:803a3c3ce4acc62eaf01eaca1208dcf0783025ef27572c3336502b9c232005e7 \ - --hash=sha256:80e6d33a3d42a7549b409f199857b4fb54e2103fc44fb87605b6663b7a7ff750 \ - --hash=sha256:8419ebd326430d1cbb7efb5292330a2cf39114e82df5cc3d83c9a0d5ebeaf2f2 \ - --hash=sha256:85610b4f27f69984932a7abbe52703688de3724d9f72bceb1cca667deff27474 \ - --hash=sha256:85e9beda1f591bc73e77ea1c51965c68e98dafd0fec72cdd745f77d727466716 \ - --hash=sha256:877b0738624280e34c55680d6054a307aa94f7d52fa0e3034a9cc6e790871da7 \ - --hash=sha256:88f9fb0116fbfcefcab70f85cf4b74a2b6ce5d199c41345296f49d974ddb4123 \ - --hash=sha256:8c4fe09e0780c6c3bf2b7d4af02ee2394439d11a523bbcf095cf4747c2932007 \ - --hash=sha256:93a784271881035ab4406a172edb0faecb6e7d00f4b53dc2f55919d6c9688595 \ - --hash=sha256:94f8575fbdf81749008d980c17796097e645574a3b8c28ee313931068dad14fe \ - --hash=sha256:95451e6ce06c3e104556d73b559f5da6c34a069b6b62946d3ad66afcd51642ea \ - --hash=sha256:99c8a9ed30f4164bc4c14b37a90208836cbf50d4ce2a57c71d0f52c7fb4f7598 \ - --hash=sha256:9a18d6f9359e45722c064c97464ec883eb0e0366d33eda61cb19a244bf222679 \ - --hash=sha256:9cbf44c5cb4a7633d078788e1b56387e3d3cf2b8139a3be38040b22d6c3221c8 \ - --hash=sha256:9ee33b875f0b390564c1fb7bc528abf18c8ee6073b201c6ae8524aca778e2d83 \ - --hash=sha256:a0e317df055958a0c1e79e5d2aa5a5eaa4a6d05a20d4b0c9c3f48918139c9fc6 \ - --hash=sha256:a2df6afe50dea8ae15fa34c9f824a3ee958d785fd5d089063d960bae1daa0a3f \ - --hash=sha256:a31de1613658308efdb21ada98cbc86a97c181aa050ba22a808120bb5be3ab94 \ - --hash=sha256:a3d2bff8f37f8d0f96c7ec554d16945050d54462d6e95414babaa18bfafc7f51 \ - --hash=sha256:a41bcf68efd19073376eb8cf948b8d9be0af26256403e512bb18f3966f1f9120 \ - --hash=sha256:a82836cab5f197a0514235aaf7ffccdc886ccdaa2324bc0aafdd4ae898103039 \ - --hash=sha256:a8d00f29b42f534cc8aa3931cfe773b13b23e561e10d2b26f27a8d309b0e82a1 \ - --hash=sha256:aafe5dcfda86c8af00386d7781d4c2181b5011b7be3f2add5e99899ea925df05 \ - --hash=sha256:ab5f043cb8a2d71c981c09c510da013bc79fd661f5c60139f00dd3c3cc4f2ffb \ - --hash=sha256:ac09d42f48f80c9ee1635b2fcaa819496a44502737660d3c0f2ade7526d29144 \ - --hash=sha256:aecfed0b41aa72b7881712c65cf764e39ce2ec352324f5e0837c7048d9e6daaa \ - --hash=sha256:b2c6b50c7b0464165472b56b42d4c76a7b864597007d9c085e8b63e185cf4a7a \ - --hash=sha256:b35d13d549077713e4414f927cdc388d62e543987c572baee613bf82f11a4b99 \ - --hash=sha256:b39cb32a6582750b6cc77bfb3c49c0f8760dc18dc96ec9fb55fbb0f04e08b928 \ - --hash=sha256:b5405bb8f0e783a988172993cfc627e4d9d00432d6bbac65a923041edacf997d \ - --hash=sha256:baaf55442359053c7d62f6f8413a62adba3205119bcb6f49594894d8be47e5e3 \ - --hash=sha256:bd654fad46d8d9e823afbb4f87c79160b5a374ed1ff5bde24e542e6ba8f41434 \ - --hash=sha256:be61f6fff406ca40e3b1d84716fde398fc08bc63dd96d15f3a14230a0973ed86 \ - --hash=sha256:bf49a3ae946a87083ef3a34c8f677ae4243f5b824bfc4c69672e72b3d6719d46 \ - --hash=sha256:c4a80f77dc1acaaa61f0934176fccca7096d9b1ff08c8ba9cddf5ae034a24319 \ - --hash=sha256:c75eb09e8d55bceb4367e83496ff8ef2bc7ea6960efb38e978e8073ea59ecb67 \ - --hash=sha256:c7f8dc16c498ff06497c015642333219871effba93e4a2e8604a06264aca5c5c \ - --hash=sha256:c8aa34a5c864db1087d911a0b902d60d203ea3607d91f615acd3f3108ac32169 \ - --hash=sha256:cbb0fef01f0c6b38cb0f39b1f78fc90b807e0e3c86a7ff3ce74ad77ce5c7880c \ - --hash=sha256:cde9a2ecd91668bcb7f077c4966d8ceddb60af01b52e6e3e2680e4cf00ad1a59 \ - --hash=sha256:cff6d44cb13d39db2663a22b22305d10855efa0fa8015ddeacc40bc59b9d8107 \ - --hash=sha256:d1009abedb49ae95b136a8904a3f71b342f849ffeced2d3747bf29caeda218c4 \ - --hash=sha256:d38c1e8231722c4ce40d7593f28d92b5fc72f3e9774fe73d7e800ec32299f63a \ - --hash=sha256:d53834e23c015ee83a99377db6e5e37d8484f333edb03bd15b4bc312cc7254fb \ - --hash=sha256:d7504f2b476d21653e4d143f44a175f7f751cd41233525312696c76aa3dbb23f \ - --hash=sha256:dbf507e9ef5688bada447a24d68b4b58dd389ba93b7afc065a2ba892bea54769 \ - --hash=sha256:dc52310451fc7c629e13c4e061cbe2dd01684d91f2f8ee2821b083c58bd72432 \ - --hash=sha256:dd00607bffbf30250fe108065f07453ec124dbf223420f57f5e749b04295e090 \ - --hash=sha256:dda608c88cf709b1d406bdfcd84d8d63cff7c9e577a403c6108ce8ce9dcc8764 \ - --hash=sha256:debe9c4f41c32990771be5c22b56f810659f9ddf3d63f67abfdcaa2c6c9c5c1d \ - --hash=sha256:e09fd068c2e169a7070d83d3bde728a4d48de0549f975290be3c108c02e499b4 \ - --hash=sha256:e0fd068364a6759bc794459f0a735ab151d11304346332489c7972bacbe9e72b \ - --hash=sha256:e4c53f8347cd4200f0d70a48ad059cabaf24f5adc6ba08622a23423bc7efa10d \ - --hash=sha256:e5723c01a56c5028c807c701aa66722916d2747ad737a046853f6c46f4875543 \ - --hash=sha256:e7b0460976dc75cb87ad9cc1f9899a4b97751e7d4e77ab840fc9b6d377b8fd24 \ - --hash=sha256:e9d9a4d06d3481eab79803beb4d9bd6f6a8e781ec078ac70d7ef2dcc29d1bea5 \ - --hash=sha256:ead11956716a940c1abc816b7df3fa2b84d06eaed8832ca32f5c5e058c65506b \ - --hash=sha256:ed5f69ce7be7902e5c70ea19eb72d20abf7d725ab5d49777d696e32d4fc1811d \ - --hash=sha256:f2af5c81a1f124609d5f33507082fc3f739959d4719b56877ab1ee7e7b3d602b \ - --hash=sha256:f40e782d49630ad384db66d4d8b73ff4f1b8955dc12e26b09a3e3af064b3b9d6 \ - --hash=sha256:f514f6474e04179d3d33175ed3f3e31434d3130d42ec153540d5b157deefd735 \ - --hash=sha256:f69f57305656a4852f2a7203efc661d8c042e6cc67f7acd97d8667fb448a426e \ - --hash=sha256:fb1e8b8d66c278b21d13b0a7ca22c41dd757a7c209c6b12c313e445c31dd3b28 \ - --hash=sha256:fb4948814a2a98e3912505f09c9e7493b1506226afb1f881825368d6fb776ee3 \ - --hash=sha256:fda207c815b253e34f7e1909840fd14299567b1c0eb4908f8c2ce01a41265401 \ - --hash=sha256:fe8f8f5e70e6dbdfca9882cd9deaac058729bcf323cf7a58660901e55c9c94f6 \ - --hash=sha256:fffc45637bcd6538de8b85f51e3df3223e4ad89bccbfca0481c08c7fc8b7ed7d +yarl==1.24.2 \ + --hash=sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b \ + --hash=sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30 \ + --hash=sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc \ + --hash=sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f \ + --hash=sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae \ + --hash=sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8 \ + --hash=sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75 \ + --hash=sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a \ + --hash=sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c \ + --hash=sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461 \ + --hash=sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44 \ + --hash=sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b \ + --hash=sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727 \ + --hash=sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9 \ + --hash=sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd \ + --hash=sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67 \ + --hash=sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420 \ + --hash=sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db \ + --hash=sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50 \ + --hash=sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b \ + --hash=sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50 \ + --hash=sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9 \ + --hash=sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1 \ + --hash=sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488 \ + --hash=sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2 \ + --hash=sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f \ + --hash=sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d \ + --hash=sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003 \ + --hash=sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536 \ + --hash=sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a \ + --hash=sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a \ + --hash=sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa \ + --hash=sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f \ + --hash=sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e \ + --hash=sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035 \ + --hash=sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12 \ + --hash=sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe \ + --hash=sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4 \ + --hash=sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294 \ + --hash=sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7 \ + --hash=sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761 \ + --hash=sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643 \ + --hash=sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413 \ + --hash=sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57 \ + --hash=sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36 \ + --hash=sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14 \ + --hash=sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd \ + --hash=sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5 \ + --hash=sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656 \ + --hash=sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad \ + --hash=sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c \ + --hash=sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0 \ + --hash=sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992 \ + --hash=sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342 \ + --hash=sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1 \ + --hash=sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf \ + --hash=sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024 \ + --hash=sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986 \ + --hash=sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb \ + --hash=sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d \ + --hash=sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543 \ + --hash=sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d \ + --hash=sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed \ + --hash=sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617 \ + --hash=sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996 \ + --hash=sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8 \ + --hash=sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2 \ + --hash=sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3 \ + --hash=sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535 \ + --hash=sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630 \ + --hash=sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215 \ + --hash=sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592 \ + --hash=sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf \ + --hash=sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b \ + --hash=sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac \ + --hash=sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0 \ + --hash=sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92 \ + --hash=sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122 \ + --hash=sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1 \ + --hash=sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8 \ + --hash=sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576 \ + --hash=sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8 \ + --hash=sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712 \ + --hash=sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1 \ + --hash=sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2 \ + --hash=sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b \ + --hash=sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a \ + --hash=sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53 \ + --hash=sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1 \ + --hash=sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d \ + --hash=sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208 \ + --hash=sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0 \ + --hash=sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c \ + --hash=sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607 \ + --hash=sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c \ + --hash=sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8 \ + --hash=sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39 \ + --hash=sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f \ + --hash=sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8 \ + --hash=sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90 \ + --hash=sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45 \ + --hash=sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2 \ + --hash=sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056 \ + --hash=sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14 # via aiohttp -zipp==3.23.1 \ - --hash=sha256:0b3596c50a5c700c9cb40ba8d86d9f2cc4807e9bedb06bcdf7fac85633e444dc \ - --hash=sha256:32120e378d32cd9714ad503c1d024619063ec28aad2248dc6672ad13edfa5110 +zipp==4.1.0 \ + --hash=sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f \ + --hash=sha256:4cb57381f544315db7688e976e922a2b18cdb513d21cc194eb42232ba2a3e602 # via etils From 5cacbe606e2e4c4d15db9b61aab0f2cfbcc70b99 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 8 Jul 2026 12:02:28 +0200 Subject: [PATCH 565/881] platforms/intel: full hermetic sandboxing (#638) Needed sandboxing for SDK less and distroless OCI image --------- Co-authored-by: Kevin Barre --- platforms/oneapi/libpjrt_oneapi.BUILD.bazel | 52 ++++++ platforms/oneapi/oneapi.bzl | 181 +++++++++++++++++--- platforms/oneapi/oneapi.zig | 20 ++- platforms/oneapi/packages.lock.json | 99 +++++++++++ platforms/oneapi/zmlxoneapi.zig | 1 + 5 files changed, 322 insertions(+), 31 deletions(-) diff --git a/platforms/oneapi/libpjrt_oneapi.BUILD.bazel b/platforms/oneapi/libpjrt_oneapi.BUILD.bazel index 1dbd9489..22edb2d6 100644 --- a/platforms/oneapi/libpjrt_oneapi.BUILD.bazel +++ b/platforms/oneapi/libpjrt_oneapi.BUILD.bazel @@ -25,22 +25,63 @@ patchelf( set_rpath = "$ORIGIN", ) +patchelf( + name = "libze_intel_gpu_so", + src = "@libze-intel-gpu1//:libze_intel_gpu_so", + set_rpath = "$ORIGIN", + soname = "libze_intel_gpu.so.1", +) + +patchelf( + name = "libigc_so_2", + src = "@libigc2//:libigc_so_2", + set_rpath = "$ORIGIN", + soname = "libigc.so.2", +) + +patchelf( + name = "libiga64_so_2", + src = "@libigc2//:libiga64_so_2", + set_rpath = "$ORIGIN", + soname = "libiga64.so.2", +) + +patchelf( + name = "libigdfcl_so_2", + src = "@libigdfcl2//:libigdfcl_so_2", + set_rpath = "$ORIGIN", + soname = "libigdfcl.so.2", +) + ONEAPI_VERSION = "2026.0" + ONEAPI_CCL_LIB = "ccl/2022.0/lib" + ONEAPI_MPI_LIB = "mpi/2021.18/lib" + ONEAPI_MPI_LIBFABRIC_LIB = "mpi/2021.18/opt/mpi/libfabric/lib" + ONEAPI_TCM_VERSION = "1.5" + ONEAPI_UMF_VERSION = "1.1" + ONEAPI_COMPILER_LIB = "compiler/{}/lib".format(ONEAPI_VERSION) + ONEAPI_MKL_LIB = "mkl/{}/lib".format(ONEAPI_VERSION) + ONEAPI_TCM_LIB = "tcm/{}/lib".format(ONEAPI_TCM_VERSION) + ONEAPI_UMF_LIB = "umf/{}/lib".format(ONEAPI_UMF_VERSION) copy_to_directory( name = "sandbox", srcs = [ + ":libiga64_so_2", + ":libigdfcl_so_2", + ":libigc_so_2", ":libpjrt_oneapi_so", ":libsycl_so", + ":libze_intel_gpu_so", "@intel-oneapi-ccl-2022.0//:ccl_runtime", "@intel-oneapi-compiler-dpcpp-cpp-runtime-2026.0//:sycl_runtime", "@intel-oneapi-compiler-shared-runtime-2026.0//:compiler_runtime", @@ -53,6 +94,13 @@ copy_to_directory( "@intel-oneapi-mpi-2021.18//:mpi_runtime", "@intel-oneapi-tcm-1.5//:hwloc", "@intel-oneapi-umf-1.1//:umf", + "@libigdfcl2//:libopencl_clang", + "@libigdgmm12", + "@libnl-3-200//:libnl_3", + "@libnl-genl-3-200//:libnl_genl_3", + "@libnl-route-3-200//:libnl_route_3", + "@libnuma1", + "@libze1", "@zlib1g", "@zml//platforms/oneapi:zmlxoneapi", ], @@ -61,6 +109,9 @@ copy_to_directory( hardlink = "auto", include_external_repositories = ["**"], replace_prefixes = { + "libiga64_so_2": "lib", + "libigdfcl_so_2": "lib", + "libigc_so_2": "lib", "libpjrt_oneapi_so": "lib", "libsycl_so": "lib", ONEAPI_CCL_LIB: "lib", @@ -71,6 +122,7 @@ copy_to_directory( ONEAPI_TCM_LIB: "lib", ONEAPI_UMF_LIB: "lib", "lib/x86_64-linux-gnu": "lib", + "libze_intel_gpu_so": "lib", "platforms/oneapi/libzmlxoneapi": "lib/libzmlxoneapi", }, root_paths = ["."], diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index 7de9fe7f..dbc1eae7 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -34,7 +34,85 @@ def _read_packages(mctx, labels): ret.setdefault(pkg["name"], {})[pkg["arch"]] = pkg return ret +_ROOT_PACKAGES = { + "libnl-3-200": """ +filegroup( + name = "libnl_3", + srcs = ["lib/x86_64-linux-gnu/libnl-3.so.200"], +)""", + "libnl-genl-3-200": """ +filegroup( + name = "libnl_genl_3", + srcs = ["lib/x86_64-linux-gnu/libnl-genl-3.so.200"], +)""", +} + _UBUNTU_PACKAGES = { + "libnl-route-3-200": """ +filegroup( + name = "libnl_route_3", + srcs = ["lib/x86_64-linux-gnu/libnl-route-3.so.200"], +)""", + "libnuma1": """ +filegroup( + name = "libnuma1", + srcs = ["lib/x86_64-linux-gnu/libnuma.so.1"], +)""", + "libigdgmm12": """ +filegroup( + name = "libigdgmm12", + srcs = ["lib/x86_64-linux-gnu/libigdgmm.so.12"], +)""", + "libigc2": """ +filegroup( + name = "libigc_so_2", + srcs = ["lib/x86_64-linux-gnu/libigc.so.2.36.3+0"], +) +filegroup( + name = "libiga64_so_2", + srcs = ["lib/x86_64-linux-gnu/libiga64.so.2.36.3+0"], +) +filegroup( + name = "libigc2", + srcs = [ + ":libiga64_so_2", + ":libigc_so_2", + ], +)""", + "libigdfcl2": """ +filegroup( + name = "libigdfcl_so_2", + srcs = ["lib/x86_64-linux-gnu/libigdfcl.so.2.36.3+0"], +) +filegroup( + name = "libopencl_clang", + srcs = ["lib/x86_64-linux-gnu/libopencl-clang.so.16"], +) +filegroup( + name = "libigdfcl2", + srcs = [ + ":libigdfcl_so_2", + ":libopencl_clang", + ], +)""", + "libze-intel-gpu1": """ +filegroup( + name = "libze_intel_gpu_so", + srcs = ["lib/x86_64-linux-gnu/libze_intel_gpu.so.1"], +) +filegroup( + name = "libze_intel_gpu", + srcs = ["lib/x86_64-linux-gnu/libze_intel_gpu.so.1"], +)""", + "libze1": """ +filegroup( + name = "libze1", + srcs = [ + "lib/x86_64-linux-gnu/libze_loader.so.1", + "lib/x86_64-linux-gnu/libze_tracing_layer.so.1", + "lib/x86_64-linux-gnu/libze_validation_layer.so.1", + ], +)""", "zlib1g": """ filegroup( name = "zlib1g", @@ -46,40 +124,81 @@ _ONEAPI_PACKAGES = { "intel-oneapi-ccl-2022.0": """ filegroup( name = "ccl_runtime", - srcs = glob( - include = ["{ONEAPI_CCL_LIB}/**"], - exclude = ["{ONEAPI_CCL_LIB}/libccl_legacy.so"], - ), + srcs = [ + "{ONEAPI_CCL_LIB}/ccl/cpu/lib/libccl.so", + "{ONEAPI_CCL_LIB}/ccl/cpu/lib/libccl.so.1", + "{ONEAPI_CCL_LIB}/ccl/cpu/lib/libccl.so.1.0", + "{ONEAPI_CCL_LIB}/ccl/cpu/lib/libccl_openmp.so", + "{ONEAPI_CCL_LIB}/ccl/cpu/lib/libccl_openmp.so.0", + "{ONEAPI_CCL_LIB}/ccl/cpu/lib/libccl_openmp.so.0.1", + "{ONEAPI_CCL_LIB}/ccl/kernels/kernels.spv", + "{ONEAPI_CCL_LIB}/libccl.so", + "{ONEAPI_CCL_LIB}/libccl.so.1", + "{ONEAPI_CCL_LIB}/libccl.so.1.0", + "{ONEAPI_CCL_LIB}/libccl.so.2", + "{ONEAPI_CCL_LIB}/libccl.so.2.0", + "{ONEAPI_CCL_LIB}/libccl_legacy_cpu.so", + "{ONEAPI_CCL_LIB}/libccl_openmp.so", + "{ONEAPI_CCL_LIB}/libccl_openmp.so.0", + "{ONEAPI_CCL_LIB}/libccl_openmp.so.0.1", + ], )""".format(ONEAPI_CCL_LIB = ONEAPI_CCL_LIB), - "intel-oneapi-mpi-2021.18": """ filegroup( name = "mpi_runtime", - srcs = glob([ - "{ONEAPI_MPI_LIB}/libmpi*.so*", - "{ONEAPI_MPI_LIB}/libmpicxx*.so*", - "{ONEAPI_MPI_LIB}/libmpifort*.so*", + srcs = [ + "{ONEAPI_MPI_LIB}/libmpi.so", + "{ONEAPI_MPI_LIB}/libmpi.so.12", + "{ONEAPI_MPI_LIB}/libmpi.so.12.0", + "{ONEAPI_MPI_LIB}/libmpi.so.12.0.0", + "{ONEAPI_MPI_LIB}/libmpi_abi.so", + "{ONEAPI_MPI_LIB}/libmpi_abi.so.1", + "{ONEAPI_MPI_LIB}/libmpi_abi.so.1.0", + "{ONEAPI_MPI_LIB}/libmpi_ilp64.so", + "{ONEAPI_MPI_LIB}/libmpi_ilp64.so.4", + "{ONEAPI_MPI_LIB}/libmpi_ilp64.so.4.1", + "{ONEAPI_MPI_LIB}/libmpi_shm_heap_proxy.so", + "{ONEAPI_MPI_LIB}/libmpicxx.so", + "{ONEAPI_MPI_LIB}/libmpicxx.so.12", + "{ONEAPI_MPI_LIB}/libmpicxx.so.12.0", + "{ONEAPI_MPI_LIB}/libmpicxx.so.12.0.0", + "{ONEAPI_MPI_LIB}/libmpifort.so", + "{ONEAPI_MPI_LIB}/libmpifort.so.12", + "{ONEAPI_MPI_LIB}/libmpifort.so.12.0", + "{ONEAPI_MPI_LIB}/libmpifort.so.12.0.0", + "{ONEAPI_MPI_LIB}/libmpifort_abi.so", + "{ONEAPI_MPI_LIB}/libmpifort_abi.so.1", + "{ONEAPI_MPI_LIB}/libmpifort_abi.so.1.0", + "{ONEAPI_MPI_LIB}/libmpijava.so", + "{ONEAPI_MPI_LIB}/libmpijava.so.1", + "{ONEAPI_MPI_LIB}/libmpijava.so.1.0", "{ONEAPI_MPI_LIB}/mpi/libmpi_ze_hooks.so", - "{ONEAPI_MPI_LIBFABRIC_LIB}/libfabric.so*", - "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/*.so", - ]), + "{ONEAPI_MPI_LIBFABRIC_LIB}/libfabric.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/libfabric.so.1", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libefa-fi.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libmlx-fi.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libpsm3-fi.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libpsmx2-fi.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/librxm-fi.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libshm-fi.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libtcp-fi.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libverbs-1.1-fi.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libverbs-1.12-fi.so", + ], )""".format( ONEAPI_MPI_LIB = ONEAPI_MPI_LIB, ONEAPI_MPI_LIBFABRIC_LIB = ONEAPI_MPI_LIBFABRIC_LIB, ), - "intel-oneapi-tcm-1.5": """ filegroup( name = "hwloc", srcs = ["{ONEAPI_TCM_LIB}/libhwloc.so.15"], )""".format(ONEAPI_TCM_LIB = ONEAPI_TCM_LIB), - "intel-oneapi-umf-1.1": """ filegroup( name = "umf", srcs = ["{ONEAPI_UMF_LIB}/libumf.so.1"], )""".format(ONEAPI_UMF_LIB = ONEAPI_UMF_LIB), - "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.0": """ filegroup( name = "libsycl_so", @@ -87,18 +206,20 @@ filegroup( ) filegroup( name = "sycl_runtime", - srcs = glob(["{ONEAPI_COMPILER_LIB}/*.spv"]) + [ + srcs = [ "{ONEAPI_COMPILER_LIB}/libur_adapter_level_zero.so.0", "{ONEAPI_COMPILER_LIB}/libur_adapter_level_zero_v2.so.0", "{ONEAPI_COMPILER_LIB}/libur_adapter_opencl.so.0", "{ONEAPI_COMPILER_LIB}/libur_loader.so.0", ], )""".format(ONEAPI_COMPILER_LIB = ONEAPI_COMPILER_LIB), - "intel-oneapi-compiler-shared-runtime-2026.0": """ filegroup( name = "compiler_runtime", - srcs = glob(["{ONEAPI_COMPILER_LIB}/libOpenCL.so*"]) + [ + srcs = [ + "{ONEAPI_COMPILER_LIB}/libOpenCL.so", + "{ONEAPI_COMPILER_LIB}/libOpenCL.so.1", + "{ONEAPI_COMPILER_LIB}/libOpenCL.so.1.0.0", "{ONEAPI_COMPILER_LIB}/libimf.so", "{ONEAPI_COMPILER_LIB}/libintlc.so.5", "{ONEAPI_COMPILER_LIB}/libirc.so", @@ -106,7 +227,6 @@ filegroup( "{ONEAPI_COMPILER_LIB}/libsvml.so", ], )""".format(ONEAPI_COMPILER_LIB = ONEAPI_COMPILER_LIB), - "intel-oneapi-mkl-core-2026.0": """ filegroup( name = "mkl_core_runtime", @@ -116,31 +236,26 @@ filegroup( "{ONEAPI_MKL_LIB}/libmkl_sequential.so.3", ], )""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), - "intel-oneapi-mkl-sycl-blas-2026.0": """ filegroup( name = "mkl_sycl_blas", srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_blas.so.6"], )""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), - "intel-oneapi-mkl-sycl-dft-2026.0": """ filegroup( name = "mkl_sycl_dft", srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_dft.so.6"], )""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), - "intel-oneapi-mkl-sycl-lapack-2026.0": """ filegroup( name = "mkl_sycl_lapack", srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_lapack.so.6"], )""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), - "intel-oneapi-mkl-sycl-rng-2026.0": """ filegroup( name = "mkl_sycl_rng", srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_rng.so.6"], )""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), - "intel-oneapi-mkl-sycl-sparse-2026.0": """ filegroup( name = "mkl_sycl_sparse", @@ -161,6 +276,15 @@ _ROOT_MODULE_DIRECT_DEPS = [ "intel-oneapi-mpi-2021.18", "intel-oneapi-tcm-1.5", "intel-oneapi-umf-1.1", + "libnl-3-200", + "libnl-genl-3-200", + "libnl-route-3-200", + "libnuma1", + "libigdgmm12", + "libigc2", + "libigdfcl2", + "libze-intel-gpu1", + "libze1", "libpjrt_oneapi", "zlib1g", ] @@ -177,6 +301,15 @@ def _oneapi_impl(mctx): url = PJRT_ONEAPI_ARTIFACT_URL, ) + for pkg_name, build_file_content in _ROOT_PACKAGES.items(): + pkg = loaded_packages[pkg_name]["amd64"] + http_deb_archive( + name = pkg_name, + urls = pkg["urls"], + sha256 = pkg["sha256"], + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, + ) + for pkg_name, build_file_content in _UBUNTU_PACKAGES.items(): pkg = loaded_packages[pkg_name]["amd64"] http_deb_archive( diff --git a/platforms/oneapi/oneapi.zig b/platforms/oneapi/oneapi.zig index 6bf172ad..f1175b69 100644 --- a/platforms/oneapi/oneapi.zig +++ b/platforms/oneapi/oneapi.zig @@ -13,24 +13,27 @@ pub fn isEnabled() bool { return @hasDecl(c, "ZML_RUNTIME_ONEAPI"); } -fn hasOneApiDevices(io: std.Io) bool { - var dir = std.Io.Dir.openDirAbsolute(io, "/dev/dri", .{ .iterate = true }) catch return false; +fn countOneApiDevices(io: std.Io) usize { + var dir = std.Io.Dir.openDirAbsolute(io, "/dev/dri", .{ .iterate = true }) catch return 0; defer dir.close(io); + var count: usize = 0; var it = dir.iterate(); while (it.next(io) catch null) |entry| { if (std.mem.startsWith(u8, entry.name, "renderD")) { - return true; + count += 1; } } - return false; + return count; } -fn setupOneAPIEnv() void { +fn setupOneAPIEnv(driver_path: [:0]const u8, device_count: usize) void { _ = c.setenv("CCL_LOG_LEVEL", c.getenv("CCL_LOG_LEVEL") orelse "error", 1); _ = c.setenv("CCL_ATL_TRANSPORT", "ofi", 1); _ = c.setenv("FI_PROVIDER", "shm", 1); + _ = c.setenv("ONEAPI_DEVICE_SELECTOR", if (device_count > 1) "level_zero:0,1" else "level_zero:0", 0); + _ = c.setenv("ZE_ENABLE_ALT_DRIVERS", driver_path, 0); } pub fn load(_: std.mem.Allocator, io: std.Io) !*const pjrt.Api { @@ -42,7 +45,8 @@ pub fn load(_: std.mem.Allocator, io: std.Io) !*const pjrt.Api { return error.Unavailable; } - if (!hasOneApiDevices(io)) { + const device_count = countOneApiDevices(io); + if (device_count == 0) { return error.Unavailable; } @@ -54,7 +58,9 @@ pub fn load(_: std.mem.Allocator, io: std.Io) !*const pjrt.Api { return error.FileNotFound; }; - setupOneAPIEnv(); + var driver_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const driver_path = try stdx.Io.Dir.path.bufJoinZ(&driver_path_buf, &.{ sandbox_path, "lib", "libze_intel_gpu.so.1" }); + setupOneAPIEnv(driver_path, device_count); return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; diff --git a/platforms/oneapi/packages.lock.json b/platforms/oneapi/packages.lock.json index 68b56279..3fb3fce3 100644 --- a/platforms/oneapi/packages.lock.json +++ b/platforms/oneapi/packages.lock.json @@ -132,6 +132,105 @@ ], "version": "1.1.0-340" }, + { + "arch": "amd64", + "dependencies": [], + "key": "libnl-3-200_3.5.0-0.1_amd64", + "name": "libnl-3-200", + "sha256": "4cd689b817d3e6453aa8f6d91d77d35db36dfc4507ab11e780adaabdc4e0bc35", + "urls": [ + "https://archive.ubuntu.com/ubuntu/pool/main/libn/libnl3/libnl-3-200_3.5.0-0.1_amd64.deb" + ], + "version": "3.5.0-0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libnl-genl-3-200_3.5.0-0.1_amd64", + "name": "libnl-genl-3-200", + "sha256": "3f73295eee7b9ef208f7272fdb604415778156e4597a25d600563967c15958ea", + "urls": [ + "https://archive.ubuntu.com/ubuntu/pool/main/libn/libnl3/libnl-genl-3-200_3.5.0-0.1_amd64.deb" + ], + "version": "3.5.0-0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libnl-route-3-200_3.5.0-0.1_amd64", + "name": "libnl-route-3-200", + "sha256": "781291056f30ff840fa99c7c23731836355852889c87f2d530a910e5b4e33abe", + "urls": [ + "https://archive.ubuntu.com/ubuntu/pool/main/libn/libnl3/libnl-route-3-200_3.5.0-0.1_amd64.deb" + ], + "version": "3.5.0-0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "sha256": "0721c89001fbbd1ada23e89da5d60e762763c1a7b3dc814a2e9a518480a8043d", + "urls": [ + "https://archive.ubuntu.com/ubuntu/pool/main/n/numactl/libnuma1_2.0.14-3ubuntu2_amd64.deb" + ], + "version": "2.0.14-3ubuntu2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libigdgmm12_22.10.0-1~24.04~ppa1_amd64", + "name": "libigdgmm12", + "sha256": "63fe9e124c3394193226c6cb8398dcabdc9306b5a10f63668118f7c799f5d792", + "urls": [ + "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-gmmlib/libigdgmm12_22.10.0-1~24.04~ppa1_amd64.deb" + ], + "version": "22.10.0-1~24.04~ppa1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libigc2_2.36.3-2~24.04_amd64", + "name": "libigc2", + "sha256": "c11dd6e083d1010c50bbc33cabf5b614e64df53b61d349955e97efdababb85bf", + "urls": [ + "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-graphics-compiler/libigc2_2.36.3-2~24.04_amd64.deb" + ], + "version": "2.36.3-2~24.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libigdfcl2_2.36.3-2~24.04_amd64", + "name": "libigdfcl2", + "sha256": "019c4d2802b03d5e1ddad888543e6b7b6003a46a14ccf2e16a2320e8923ad007", + "urls": [ + "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-graphics-compiler/libigdfcl2_2.36.3-2~24.04_amd64.deb" + ], + "version": "2.36.3-2~24.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libze-intel-gpu1_26.22.38646.6-1~24.04~ppa1_amd64", + "name": "libze-intel-gpu1", + "sha256": "5d0903aae7ac4f94f43ae7d282a394bc988a0da12f813d92533dea1650eb91ff", + "urls": [ + "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-compute-runtime/libze-intel-gpu1_26.22.38646.6-1~24.04~ppa1_amd64.deb" + ], + "version": "26.22.38646.6-1~24.04~ppa1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libze1_1.28.6-1~24.04~ppa1_amd64", + "name": "libze1", + "sha256": "ca4aec5982d46f92875659e74e459372f24c9d8add2810556c1c7e71531e1ec0", + "urls": [ + "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/l/level-zero-loader/libze1_1.28.6-1~24.04~ppa1_amd64.deb" + ], + "version": "1.28.6-1~24.04~ppa1" + }, { "arch": "amd64", "dependencies": [], diff --git a/platforms/oneapi/zmlxoneapi.zig b/platforms/oneapi/zmlxoneapi.zig index baa57408..6be2505a 100644 --- a/platforms/oneapi/zmlxoneapi.zig +++ b/platforms/oneapi/zmlxoneapi.zig @@ -5,6 +5,7 @@ const stdx = @import("stdx"); pub export fn zmlxoneapi_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque { const replacements: std.StaticStringMap([:0]const u8) = .initComptime(.{ .{ "libOpenCL.so", "libOpenCL.so.1" }, + .{ "libze_loader.so", "libze_loader.so.1" }, .{ "libumf.so", "libumf.so.1" }, .{ "libur_loader.so", "libur_loader.so.0" }, .{ "libur_adapter_level_zero.so", "libur_adapter_level_zero.so.0" }, From 19a15ca9662a11e1233d2c69a1f6d7f43fa0bcd1 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Thu, 9 Jul 2026 14:02:57 +0200 Subject: [PATCH 566/881] examples/llm: shiny ZML logo (#636) Co-authored-by: Tristan Gautie --- examples/llm/main.zig | 27 ++---- zml/BUILD.bazel | 1 + zml/logo.zig | 211 ++++++++++++++++++++++++++++++++++++++++++ zml/zml.zig | 1 + 4 files changed, 223 insertions(+), 17 deletions(-) create mode 100644 zml/logo.zig diff --git a/examples/llm/main.zig b/examples/llm/main.zig index c328eaa5..e15baea9 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -144,7 +144,7 @@ pub fn main(init: std.process.Init) !void { // // We're ready to go! Let's start a chat... // - try printZmlLogo(io); + try printZmlLogo(io, init.environ_map); const interactive = args.prompt == null; const prompt = if (args.prompt) |prompt| b: { @@ -194,21 +194,14 @@ fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, prog return try .fromBytes(allocator, bytes); } -pub fn printZmlLogo(io: std.Io) !void { - const LOGO = - \\ - \\ - \\ ███████╗███╗ ███╗██╗ - \\ ╚══███╔╝████╗ ████║██║ - \\ ███╔╝ ██╔████╔██║██║ - \\ ███╔╝ ██║╚██╔╝██║██║ .ai - \\ ███████╗██║ ╚═╝ ██║███████╗ - \\ ╚══════╝╚═╝ ╚═╝╚══════╝ - \\ - \\ - \\ - ; - var writer = std.Io.File.stdout().writerStreaming(io, &.{}); - try writer.interface.writeAll(LOGO); +pub fn printZmlLogo(io: std.Io, environ_map: *const std.process.Environ.Map) !void { + const stdout = std.Io.File.stdout(); + const logo_options: zml.logo.Options = .{ + .color_support = zml.logo.detectColorSupport(io, stdout, environ_map), + }; + var writer = stdout.writerStreaming(io, &.{}); + try writer.interface.writeAll("\n\n"); + try zml.logo.writeZmlArt(&writer.interface, logo_options); + try writer.interface.writeAll("\n\n\n"); try writer.interface.flush(); } diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 78ce31cf..a319c6a4 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -102,6 +102,7 @@ zig_library( "exe.zig", "floats.zig", "io.zig", + "logo.zig", "mem.zig", "meta.zig", "mlirx.zig", diff --git a/zml/logo.zig b/zml/logo.zig new file mode 100644 index 00000000..235bc43d --- /dev/null +++ b/zml/logo.zig @@ -0,0 +1,211 @@ +const std = @import("std"); + +pub const Color = enum { + reset, + cyan, + sky, + blue, + violet, + pink, + shine_cyan, + shine_sky, + shine_blue, + shine_violet, + shine_pink, + + pub fn escape(self: Color, support: ColorSupport) []const u8 { + return switch (support) { + .true_color => self.trueColor(), + ._256 => self.xterm256(), + else => "", + }; + } + + fn trueColor(self: Color) []const u8 { + return switch (self) { + .reset => "\x1b[0m", + .cyan => "\x1b[38;2;0;255;255m", + .sky => "\x1b[38;2;0;221;255m", + .blue => "\x1b[38;2;135;143;255m", + .violet => "\x1b[38;2;215;90;219m", + .pink => "\x1b[38;2;242;48;174m", + .shine_cyan => "\x1b[1;38;2;140;255;255m", + .shine_sky => "\x1b[1;38;2;128;244;255m", + .shine_blue => "\x1b[1;38;2;205;210;255m", + .shine_violet => "\x1b[1;38;2;255;170;255m", + .shine_pink => "\x1b[1;38;2;255;150;220m", + }; + } + + fn xterm256(self: Color) []const u8 { + return switch (self) { + .reset => "\x1b[0m", + .cyan => "\x1b[38;5;51m", + .sky => "\x1b[38;5;45m", + .blue => "\x1b[38;5;105m", + .violet => "\x1b[38;5;170m", + .pink => "\x1b[38;5;205m", + .shine_cyan => "\x1b[1;38;5;123m", + .shine_sky => "\x1b[1;38;5;123m", + .shine_blue => "\x1b[1;38;5;189m", + .shine_violet => "\x1b[1;38;5;219m", + .shine_pink => "\x1b[1;38;5;212m", + }; + } +}; + +pub const Row = struct { + text: []const u8, + color: Color = .reset, + shine: Color = .reset, +}; + +pub const Block = struct { + rows: []const Row, +}; + +pub const ColorSupport = enum { + no_color, + ansi, + _256, + true_color, +}; + +pub const Options = struct { + color_support: ColorSupport, + is_shiny: *const fn ([]const u8) bool = isShineGlyph, +}; + +pub const zml_art_blocks = [_]Block{ + .{ .rows = &.{ + .{ .text = " ", .color = .cyan, .shine = .shine_cyan }, + .{ .text = " ", .color = .cyan, .shine = .shine_cyan }, + .{ .text = " ███████╗", .color = .cyan, .shine = .shine_cyan }, + .{ .text = " ╚══███╔╝", .color = .sky, .shine = .shine_sky }, + .{ .text = " ███╔╝ ", .color = .blue, .shine = .shine_blue }, + .{ .text = " ███╔╝ ", .color = .violet, .shine = .shine_violet }, + .{ .text = " ███████╗", .color = .pink, .shine = .shine_pink }, + .{ .text = " ╚══════╝", .color = .pink, .shine = .shine_pink }, + } }, + .{ .rows = &.{ + .{ .text = " ", .color = .cyan, .shine = .shine_cyan }, + .{ .text = "███╗ ███╗", .color = .cyan, .shine = .shine_cyan }, + .{ .text = "████╗ ████║", .color = .sky, .shine = .shine_sky }, + .{ .text = "██╔████╔██║", .color = .blue, .shine = .shine_blue }, + .{ .text = "██║╚██╔╝██║", .color = .violet, .shine = .shine_violet }, + .{ .text = "██║ ╚═╝ ██║", .color = .pink, .shine = .shine_pink }, + .{ .text = "╚═╝ ╚═╝", .color = .pink, .shine = .shine_pink }, + .{ .text = " ", .color = .pink, .shine = .shine_pink }, + } }, + .{ .rows = &.{ + .{ .text = "██╗ ", .color = .cyan, .shine = .shine_cyan }, + .{ .text = "██║ ", .color = .sky, .shine = .shine_sky }, + .{ .text = "██║ ", .color = .blue, .shine = .shine_blue }, + .{ .text = "██║ .ai", .color = .violet, .shine = .shine_violet }, + .{ .text = "███████╗", .color = .pink, .shine = .shine_pink }, + .{ .text = "╚══════╝", .color = .pink, .shine = .shine_pink }, + .{ .text = " ", .color = .pink, .shine = .shine_pink }, + .{ .text = " ", .color = .pink, .shine = .shine_pink }, + } }, +}; + +pub fn detectColorSupport(io: std.Io, file: std.Io.File, env: *const std.process.Environ.Map) ColorSupport { + var no_color: bool = false; + var clicolor_force: bool = false; + no_color = if (env.get("NO_COLOR")) |v| v.len != 0 else false; + clicolor_force = if (env.get("CLICOLOR_FORCE")) |v| v.len != 0 else false; + + const mode = std.Io.Terminal.Mode.detect(io, file, no_color, clicolor_force) catch return .no_color; + if (mode != .escape_codes) return .no_color; + + if (env.get("COLORTERM")) |value| { + if (std.ascii.eqlIgnoreCase(value, "truecolor") or + std.ascii.eqlIgnoreCase(value, "24bit")) + { + return .true_color; + } + } + + if (env.get("TERM")) |value| { + if (std.mem.endsWith(u8, value, "-direct")) return .true_color; + if (std.mem.indexOf(u8, value, "truecolor") != null) return .true_color; + + if (std.mem.indexOf(u8, value, "256color") != null) return ._256; + } + + return .ansi; +} + +pub fn writeZmlArt(writer: *std.Io.Writer, options: Options) !void { + try write(writer, zml_art_blocks[0..], options); +} + +pub fn write(writer: *std.Io.Writer, blocks: []const Block, options: Options) !void { + if (blocks.len == 0) return; + + const row_count = blocks[0].rows.len; + for (blocks[1..]) |block| { + if (block.rows.len != row_count) return error.InvalidLogoBlocks; + } + + for (0..row_count) |row_index| { + const last_block = findLastVisibleBlockIndex(blocks, row_index) orelse { + try writer.writeAll("\n"); + continue; + }; + + for (blocks[0 .. last_block + 1], 0..) |block, block_index| { + const row = block.rows[row_index]; + try writeRow(writer, row, options, block_index == last_block); + } + try writer.writeAll(Color.reset.escape(options.color_support)); + try writer.writeAll("\n"); + } +} + +pub fn writeRow(writer: *std.Io.Writer, row: Row, options: Options, rtrim_whitespace: bool) !void { + const text = if (rtrim_whitespace) trimRightWhitespace(row.text) else row.text; + if (text.len == 0) return; + + const color = row.color.escape(options.color_support); + const shine = row.shine.escape(options.color_support); + + var view: std.unicode.Utf8View = .initUnchecked(text); + var iter = view.iterator(); + var is_shiny = false; + + try writer.writeAll(color); + while (iter.nextCodepointSlice()) |codepoint| { + const should_shine = options.is_shiny(codepoint); + if (should_shine != is_shiny) { + try writer.writeAll(if (should_shine) shine else color); + is_shiny = should_shine; + } + try writer.writeAll(codepoint); + } +} + +fn findLastVisibleBlockIndex(blocks: []const Block, row_index: usize) ?usize { + var index = blocks.len; + while (index > 0) { + index -= 1; + if (trimRightWhitespace(blocks[index].rows[row_index].text).len != 0) return index; + } + return null; +} + +fn trimRightWhitespace(text: []const u8) []const u8 { + return std.mem.trimEnd(u8, text, " \t\r\n"); +} + +pub fn isShineGlyph(codepoint: []const u8) bool { + return std.mem.eql(u8, codepoint, "╔") or + std.mem.eql(u8, codepoint, "╗") or + std.mem.eql(u8, codepoint, "╚") or + std.mem.eql(u8, codepoint, "╝") or + std.mem.eql(u8, codepoint, "═") or + std.mem.eql(u8, codepoint, "║") or + std.mem.eql(u8, codepoint, ".") or + std.mem.eql(u8, codepoint, "a") or + std.mem.eql(u8, codepoint, "i"); +} diff --git a/zml/zml.zig b/zml/zml.zig index dd3dae36..11a45717 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -24,6 +24,7 @@ pub const Exe = exe.Exe; pub const floats = @import("floats.zig"); pub const io = @import("io.zig"); pub const kernel = @import("kernel.zig"); +pub const logo = @import("logo.zig"); pub const mem = @import("mem.zig"); pub const Bufferized = mem.Bufferized; pub const meta = @import("meta.zig"); From dcad988918223f9ffcac7914bb9c951977f7e50c Mon Sep 17 00:00:00 2001 From: Adrien Date: Thu, 9 Jul 2026 16:40:20 +0200 Subject: [PATCH 567/881] zml/vfs: treat scheme:// paths as absolute in the router (#635) In the VFS router, a scheme-qualified path like `hf://owner/model/file` is meant to be absolute: it should resolve from the scheme's root. But `lookupDir` only treated it as absolute when the base directory was `cwd`. When such a path is opened *relative to an already-open directory* of the same backend, the router kept the directory's path and glued the new path after it, duplicating it. So `hf://owner/model` + `hf://owner/model/file` became `owner/model/owner/model/file`, and the lookup failed with `FileNotFound`. Fix: on `lookupDir`, any `scheme://` sub-path resolves from the scheme's root regardless of the base directory, the same way an absolute POSIX path already does. About four lines. --- zml/io/vfs.zig | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/zml/io/vfs.zig b/zml/io/vfs.zig index c87caf97..93d3981a 100644 --- a/zml/io/vfs.zig +++ b/zml/io/vfs.zig @@ -137,6 +137,20 @@ pub const VFS = struct { } fn lookupDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?usize, std.Io.Dir, std.Io } { + // A scheme-qualified path (e.g. "hf://owner/model/file") is absolute: it + // is resolved from the scheme's root regardless of `dir`. Without this, + // opening such a path relative to an already-open dir of the same + // backend double-prefixes the dir's path. + if (sub_path) |sp| { + if (std.mem.indexOf(u8, sp, "://") != null) { + const uri = std.Uri.parse(sp) catch return error.VFSNotRegistered; + const backend_idx: usize = for (self.backends.entries.items(.key), 0..) |s, idx| { + if (std.mem.eql(u8, uri.scheme, s)) break idx; + } else return error.VFSNotRegistered; + return .{ backend_idx, std.Io.Dir.cwd(), self.getBackend(backend_idx) }; + } + } + if (std.meta.eql(dir, std.Io.Dir.cwd())) { if (sub_path == null) return .{ null, dir, self.base.inner }; if (std.fs.path.isAbsolutePosix(sub_path.?)) return .{ null, dir, self.base.inner }; From 0b4db7f1086ac3743d683323e811a2bb3b21b2cf Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Fri, 10 Jul 2026 11:55:38 +0200 Subject: [PATCH 568/881] zml/io/vfs: Retry on retryable errors (#639) --- zml/io/vfs/gcs.zig | 98 ++++++++++++++++++++++++++---- zml/io/vfs/hf.zig | 114 ++++++++++++++++++++++++++++++----- zml/io/vfs/parallel_read.zig | 78 +++++++++++++++++++++--- zml/io/vfs/s3.zig | 105 +++++++++++++++++++++++++++----- 4 files changed, 344 insertions(+), 51 deletions(-) diff --git a/zml/io/vfs/gcs.zig b/zml/io/vfs/gcs.zig index f67940ad..186c55c1 100644 --- a/zml/io/vfs/gcs.zig +++ b/zml/io/vfs/gcs.zig @@ -3,8 +3,8 @@ const builtin = @import("builtin"); const stdx = @import("stdx"); -const VFSBase = @import("base.zig").VFSBase; const parallel_read = @import("parallel_read.zig"); +const VFSBase = @import("base.zig").VFSBase; const log = std.log.scoped(.@"zml/io/vfs/gcs"); @@ -125,7 +125,7 @@ pub const GCS = struct { const Pool = parallel_read.Pool(Job); const Batch = struct { - executor: parallel_read.BatchExecutor, + state: parallel_read.BatchState, backend: *GCS, uri: std.Uri, path: []const u8, @@ -139,7 +139,7 @@ pub const GCS = struct { chunk_len: usize, batch: *Batch, - pub fn perform(job: Job, client: *std.http.Client) ?anyerror { + pub fn perform(job: Job, client: *std.http.Client) anyerror!parallel_read.Status { var range_buf: [64]u8 = undefined; const range_header = std.fmt.bufPrint( &range_buf, @@ -157,17 +157,76 @@ pub const GCS = struct { }, .extra_headers = &.{.{ .name = "Range", .value = range_header }}, }, - ) catch |err| return err; + ) catch |err| { + switch (err) { + // transient connect failures + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.Timeout, + error.NameServerFailure, + => { + log.warn("Failed to connect: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to connect: {}", .{err}); + return err; + }, + } + }; defer req.deinit(); - req.sendBodiless() catch |err| return err; + req.sendBodiless() catch |err| switch (err) { + error.WriteFailed => { + log.warn("Failed to send headers: {}", .{err}); + return .retry(); + }, + }; var redirect_buffer: [8 * 1024]u8 = undefined; - var res = req.receiveHead(&redirect_buffer) catch |err| return err; + var res = req.receiveHead(&redirect_buffer) catch |err| { + switch (err) { + // stale keep-alive / peer closed while waiting for response head + error.HttpConnectionClosing, + error.HttpRequestTruncated, + + // transport read/write failure; retry on a fresh connection + error.ReadFailed, + error.WriteFailed, + + // transient connect failures + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.Timeout, + error.NameServerFailure, + => { + log.warn("Failed to receive headers: {}", .{err}); + return .retry(); + }, + + else => { + log.err("Failed to receive headers: {}", .{err}); + return err; + }, + } + }; if (res.head.status != .partial_content and res.head.status != .ok) { - log.err("Failed to read {s}: {s}", .{ job.batch.path, res.head.bytes }); - return error.RequestFailed; + const status: parallel_read.Status = switch (res.head.status) { + .request_timeout, .too_many_requests => .retry(), + else => if (res.head.status.class() == .server_error) .retry() else { + log.err("Failed to read {s}: {s}", .{ job.batch.path, res.head.bytes }); + return error.RequestFailed; + }, + }; + log.warn("Failed to read {s}: {s}", .{ job.batch.path, res.head.bytes }); + return status; } const content_range = blk: { @@ -181,9 +240,22 @@ pub const GCS = struct { }; const reader = res.reader(&.{}); - parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| return err; + parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| { + switch (err) { + error.EndOfStream, + error.ReadFailed, + => { + log.warn("Failed to read from response: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to read from response: {}", .{err}); + return err; + }, + } + }; - return null; + return .success; } }; }; @@ -991,7 +1063,7 @@ pub const GCS = struct { const pending: u32 = @intCast(job_count); var batch: ParallelRead.Batch = .{ - .executor = .{ .pending = .init(pending) }, + .state = .{ .pending = .init(pending) }, .backend = self, .uri = uri, .path = handle.uri, @@ -1010,8 +1082,8 @@ pub const GCS = struct { } try self.read_pool.job_queue.putAll(self.base.inner, jobs); - batch.executor.waitUncancelable(self.base.inner); - if (batch.executor.firstError()) |err| return err; + batch.state.waitUncancelable(self.base.inner); + if (batch.state.anyError()) |err| return err; return read_size; } diff --git a/zml/io/vfs/hf.zig b/zml/io/vfs/hf.zig index 664fd55f..6d1fd325 100644 --- a/zml/io/vfs/hf.zig +++ b/zml/io/vfs/hf.zig @@ -2,8 +2,8 @@ const std = @import("std"); const stdx = @import("stdx"); -const VFSBase = @import("base.zig").VFSBase; const parallel_read = @import("parallel_read.zig"); +const VFSBase = @import("base.zig").VFSBase; const log = std.log.scoped(.@"zml/io/vfs/hf"); @@ -11,7 +11,7 @@ const ParallelRead = struct { const Pool = parallel_read.Pool(Job); const Batch = struct { - executor: parallel_read.BatchExecutor, + state: parallel_read.BatchState, uri: std.Uri, url: []const u8, authorization: std.http.Client.Request.Headers.Value, @@ -24,7 +24,7 @@ const ParallelRead = struct { chunk_len: usize, batch: *Batch, - pub fn perform(job: Job, client: *std.http.Client) ?anyerror { + pub fn perform(job: Job, client: *std.http.Client) anyerror!parallel_read.Status { var range_buf: [64]u8 = undefined; const range_header = std.fmt.bufPrint( &range_buf, @@ -38,18 +38,91 @@ const ParallelRead = struct { .authorization = job.batch.authorization, }, .extra_headers = &.{.{ .name = "Range", .value = range_header }}, - }) catch |err| return err; + }) catch |err| { + switch (err) { + // transient connect failures + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.Timeout, + error.NameServerFailure, + => { + log.warn("Failed to connect: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to connect: {}", .{err}); + return err; + }, + } + }; defer req.deinit(); - req.sendBodiless() catch |err| return err; + req.sendBodiless() catch |err| switch (err) { + error.WriteFailed => { + log.warn("Failed to send headers: {}", .{err}); + return .retry(); + }, + }; var redirect_buffer: [8 * 1024]u8 = undefined; - var res = req.receiveHead(&redirect_buffer) catch |err| return err; + var res = req.receiveHead(&redirect_buffer) catch |err| { + switch (err) { + // stale keep-alive / peer closed while waiting for response head + error.HttpConnectionClosing, + error.HttpRequestTruncated, + + // transport read/write failure; retry on a fresh connection + error.ReadFailed, + error.WriteFailed, + + // transient connect failures + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.Timeout, + error.NameServerFailure, + => { + log.warn("Failed to receive headers: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to receive headers: {}", .{err}); + return err; + }, + } + }; if (res.head.status != .partial_content and res.head.status != .ok) { - log.err("Failed to perform read for {s}", .{job.batch.url}); - log.err("{s}", .{res.head.bytes}); - return error.RequestFailed; + const status: parallel_read.Status = switch (res.head.status) { + .request_timeout => .retry(), + .too_many_requests => b: { + var it = res.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "RateLimit")) { + var parts = std.mem.splitScalar(u8, header.value, ';'); + while (parts.next()) |part| { + const p = std.mem.trim(u8, part, " \t"); + if (std.mem.startsWith(u8, p, "t=")) { + const seconds = std.fmt.parseInt(u32, p[2..], 10) catch continue; + break :b .{ .retry_after = .fromSeconds(seconds) }; + } + } + } + } + break :b .retry(); + }, + else => if (res.head.status.class() == .server_error) .retry() else { + log.err("Failed to perform read for {s}\n{s}", .{ job.batch.url, res.head.bytes }); + return error.RequestFailed; + }, + }; + log.warn("Failed to perform read for {s}\n{s}", .{ job.batch.url, res.head.bytes }); + return status; } const content_range = blk: { @@ -63,9 +136,22 @@ const ParallelRead = struct { }; const reader = res.reader(&.{}); - parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| return err; + parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| { + switch (err) { + error.EndOfStream, + error.ReadFailed, + => { + log.warn("Failed to read from response: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to read from response: {}", .{err}); + return err; + }, + } + }; - return null; + return .success; } }; }; @@ -800,7 +886,7 @@ pub const HF = struct { const uri: std.Uri = try .parse(url); var batch: ParallelRead.Batch = .{ - .executor = .{ .pending = .init(pending) }, + .state = .{ .pending = .init(pending) }, .uri = uri, .url = url, .authorization = self.authorization, @@ -818,8 +904,8 @@ pub const HF = struct { } try self.read_pool.job_queue.putAll(self.base.inner, jobs); - batch.executor.waitUncancelable(self.base.inner); - if (batch.executor.firstError()) |err| return err; + batch.state.waitUncancelable(self.base.inner); + if (batch.state.anyError()) |err| return err; return read_size; } diff --git a/zml/io/vfs/parallel_read.zig b/zml/io/vfs/parallel_read.zig index d5d7855c..2a08d407 100644 --- a/zml/io/vfs/parallel_read.zig +++ b/zml/io/vfs/parallel_read.zig @@ -6,32 +6,44 @@ pub const InitOpts = struct { chunk_size: usize = 16 * 1024 * 1024, num_workers: usize = 32, queue_capacity: usize = 128, + max_retries: usize = 5, + retry_initial_delay: std.Io.Duration = .fromMilliseconds(500), + retry_max_delay: std.Io.Duration = .fromSeconds(30), }; -pub const BatchExecutor = struct { +pub const Status = union(enum) { + success, + retry_after: ?std.Io.Duration, + + pub fn retry() Status { + return .{ .retry_after = null }; + } +}; + +pub const BatchState = struct { pending: std.atomic.Value(u32), err: std.atomic.Value(u16) = .init(0), - pub fn failed(batch: *BatchExecutor) bool { + pub fn failed(batch: *BatchState) bool { return batch.err.load(.acquire) != 0; } - pub fn fail(batch: *BatchExecutor, err: anyerror) void { + pub fn fail(batch: *BatchState, err: anyerror) void { _ = batch.err.cmpxchgStrong(0, @intFromError(err), .release, .monotonic); } - pub fn firstError(batch: *BatchExecutor) ?anyerror { + pub fn anyError(batch: *BatchState) ?anyerror { const err = batch.err.load(.acquire); return if (err == 0) null else @errorFromInt(err); } - pub fn completeOne(batch: *BatchExecutor, io: std.Io) void { + pub fn completeOne(batch: *BatchState, io: std.Io) void { const previous = batch.pending.fetchSub(1, .release); std.debug.assert(previous > 0); if (previous == 1) io.futexWake(u32, &batch.pending.raw, 1); } - pub fn waitUncancelable(batch: *BatchExecutor, io: std.Io) void { + pub fn waitUncancelable(batch: *BatchState, io: std.Io) void { while (true) { const pending = batch.pending.load(.acquire); if (pending == 0) return; @@ -48,12 +60,19 @@ pub fn Pool(comptime Job: type) type { job_queue: std.Io.Queue(Job), queue_buf: []Job, client: *std.http.Client, + prng: std.Random.DefaultPrng, chunk_size: usize, + max_retries: usize, + retry_initial_delay: std.Io.Duration, + retry_max_delay: std.Io.Duration, pub fn init(self: *Self, allocator: std.mem.Allocator, io: std.Io, client: *std.http.Client, opts: InitOpts) !void { stdx.debug.assert(opts.num_workers > 0, "Pool must have at least one worker", .{}); stdx.debug.assert(opts.chunk_size > 0, "Pool must have a positive download chunk size", .{}); stdx.debug.assert(opts.queue_capacity > 0, "Pool must have a positive queue capacity", .{}); + stdx.debug.assert(opts.retry_initial_delay.nanoseconds >= 0, "Pool retry initial delay must not be negative", .{}); + stdx.debug.assert(opts.retry_max_delay.nanoseconds >= 0, "Pool retry max delay must not be negative", .{}); + stdx.debug.assert(opts.retry_max_delay.nanoseconds >= opts.retry_initial_delay.nanoseconds, "Pool retry max delay must be at least the initial delay", .{}); const queue_buf = try allocator.alloc(Job, opts.queue_capacity); errdefer allocator.free(queue_buf); @@ -61,6 +80,14 @@ pub fn Pool(comptime Job: type) type { self.* = .{ .client = client, .chunk_size = opts.chunk_size, + .max_retries = opts.max_retries, + .retry_initial_delay = opts.retry_initial_delay, + .retry_max_delay = opts.retry_max_delay, + .prng = .init(blk: { + var seed: u64 = undefined; + io.random(@ptrCast(&seed)); + break :blk seed; + }), .job_queue = .init(queue_buf), .queue_buf = queue_buf, }; @@ -82,13 +109,46 @@ pub fn Pool(comptime Job: type) type { while (true) { const job = pool.job_queue.getOne(io) catch break; - if (!job.batch.executor.failed()) { - if (job.perform(pool.client)) |err| job.batch.executor.fail(err); + var attempt: usize = 0; + while (true) { + if (job.batch.state.failed()) break; + const status = job.perform(pool.client) catch |err| { + job.batch.state.fail(err); + break; + }; + + switch (status) { + .success => break, + .retry_after => |duration| { + if (attempt >= pool.max_retries) { + job.batch.state.fail(error.RetriesExhausted); + break; + } + + io.sleep(duration orelse pool.backoffDuration(attempt), .awake) catch { + job.batch.state.fail(error.RetriesExhausted); + break; + }; + + attempt += 1; + }, + } + continue; } - job.batch.executor.completeOne(io); + job.batch.state.completeOne(io); } } + + fn backoffDuration(pool: *Self, attempt: usize) std.Io.Duration { + // FullJitter algorithm + const max_delay_ns: i96 = @min( + pool.retry_max_delay.toNanoseconds(), + pool.retry_initial_delay.toNanoseconds() *| (@as(i96, 1) << @as(u7, @intCast(@min(attempt, std.math.maxInt(u7))))), + ); + const delay_ns = pool.prng.random().intRangeAtMost(i96, 0, max_delay_ns); + return .fromNanoseconds(delay_ns); + } }; } diff --git a/zml/io/vfs/s3.zig b/zml/io/vfs/s3.zig index 303fa282..58d71cdf 100644 --- a/zml/io/vfs/s3.zig +++ b/zml/io/vfs/s3.zig @@ -2,8 +2,8 @@ const std = @import("std"); const stdx = @import("stdx"); -const VFSBase = @import("base.zig").VFSBase; const parallel_read = @import("parallel_read.zig"); +const VFSBase = @import("base.zig").VFSBase; const log = std.log.scoped(.@"zml/io/vfs/s3"); @@ -145,7 +145,7 @@ pub const S3 = struct { const Pool = parallel_read.Pool(Job); const Batch = struct { - executor: parallel_read.BatchExecutor, + state: parallel_read.BatchState, backend: *S3, uri: std.Uri, url: []const u8, @@ -158,7 +158,7 @@ pub const S3 = struct { chunk_len: usize, batch: *Batch, - pub fn perform(job: Job, client: *std.http.Client) ?anyerror { + pub fn perform(job: Job, client: *std.http.Client) anyerror!parallel_read.Status { var range_buf: [64]u8 = undefined; const range_header = std.fmt.bufPrint( &range_buf, @@ -167,7 +167,7 @@ pub const S3 = struct { ) catch unreachable; var timestamp_buf: [16]u8 = undefined; - const timestamp = job.batch.backend.getTimestamp(×tamp_buf) catch |err| return err; + const timestamp = job.batch.backend.getTimestamp(×tamp_buf) catch unreachable; const signer: AwsSigV4 = .{ .access_key = job.batch.backend.config.access_key, @@ -183,7 +183,10 @@ pub const S3 = struct { job.batch.uri, timestamp, &.{.{ .name = "Range", .value = range_header }}, - ) catch |err| return err; + ) catch |err| { + log.err("Failed to generate auth header: {}", .{err}); + return err; + }; var req = client.request(.GET, job.batch.uri, .{ .headers = .{ @@ -195,17 +198,76 @@ pub const S3 = struct { .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, .{ .name = "Range", .value = range_header }, }, - }) catch |err| return err; + }) catch |err| { + switch (err) { + // transient connect failures + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.Timeout, + error.NameServerFailure, + => { + log.warn("Failed to connect: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to connect: {}", .{err}); + return err; + }, + } + }; defer req.deinit(); - req.sendBodiless() catch |err| return err; + req.sendBodiless() catch |err| switch (err) { + error.WriteFailed => { + log.warn("Failed to send headers: {}", .{err}); + return .retry(); + }, + }; var redirect_buffer: [8 * 1024]u8 = undefined; - var res = req.receiveHead(&redirect_buffer) catch |err| return err; + var res = req.receiveHead(&redirect_buffer) catch |err| { + switch (err) { + // stale keep-alive / peer closed while waiting for response head + error.HttpConnectionClosing, + error.HttpRequestTruncated, + + // transport read/write failure; retry on a fresh connection + error.ReadFailed, + error.WriteFailed, + + // transient connect failures + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.Timeout, + error.NameServerFailure, + => { + log.warn("Failed to receive headers: {}", .{err}); + return .retry(); + }, + + else => { + log.err("Failed to receive headers: {}", .{err}); + return err; + }, + } + }; if (res.head.status != .partial_content and res.head.status != .ok) { - log.err("Failed to read {s}: {s}", .{ job.batch.url, res.head.bytes }); - return error.RequestFailed; + const status: parallel_read.Status = switch (res.head.status) { + .request_timeout, .too_many_requests => .retry(), + else => if (res.head.status.class() == .server_error) .retry() else { + log.err("Failed to read {s}: {s}", .{ job.batch.url, res.head.bytes }); + return error.RequestFailed; + }, + }; + log.warn("Failed to read {s}: {s}", .{ job.batch.url, res.head.bytes }); + return status; } const content_range = blk: { @@ -219,9 +281,22 @@ pub const S3 = struct { }; const reader = res.reader(&.{}); - parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| return err; + parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| { + switch (err) { + error.EndOfStream, + error.ReadFailed, + => { + log.warn("Failed to read from response: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to read from response: {}", .{err}); + return err; + }, + } + }; - return null; + return .success; } }; }; @@ -880,7 +955,7 @@ pub const S3 = struct { const pending: u32 = @intCast(job_count); var batch: ParallelRead.Batch = .{ - .executor = .{ .pending = .init(pending) }, + .state = .{ .pending = .init(pending) }, .backend = self, .uri = uri, .url = url, @@ -898,8 +973,8 @@ pub const S3 = struct { } try self.read_pool.job_queue.putAll(self.base.inner, jobs); - batch.executor.waitUncancelable(self.base.inner); - if (batch.executor.firstError()) |err| return err; + batch.state.waitUncancelable(self.base.inner); + if (batch.state.anyError()) |err| return err; return read_size; } From a1b595043b027591a69b1cf83e0a8df1c20d2d06 Mon Sep 17 00:00:00 2001 From: Ben Younes Date: Fri, 10 Jul 2026 12:13:29 +0100 Subject: [PATCH 569/881] zml/tensor: fix IRFFT output shape (#642) ## Summary Fix IRFFT shape inference so it uses the requested FFT output length and validates that the complex input length matches the corresponding RFFT output length. Fixes #411. ## Why RFFT changes the innermost real dimension `n` into `n / 2 + 1` complex bins. IRFFT receives those complex bins plus the requested real FFT length, so its output shape should restore that requested length instead of deriving a smaller value from the complex input dimension. ## Test verification (RED -> GREEN) Command used for all runs: ```bash npx -y @bazel/bazelisk@1.25.0 test //zml:test --test_output=errors ``` RED, with the new regression test before the fix: ```text 102/139 tensor.Tensor.test.Tensor.fft IRFFT restores RFFT last dimension...thread 1701155 panic: exact division produced remainder INFO: Build completed, 1 test FAILED, 5355 total actions //zml:test FAILED in 24.9s Executed 1 out of 1 test: 1 fails locally. ``` GREEN, with the fix applied: ```text INFO: Build completed successfully, 6 total actions //zml:test PASSED in 8.1s Executed 1 out of 1 test: 1 test passes. ``` Revert RED, with the production fix reverted while keeping the test: ```text 102/139 tensor.Tensor.test.Tensor.fft IRFFT restores RFFT last dimension...thread 1711292 panic: fft(.{ .kind = .IRFFT, .length = { 10 } }) expects tensor last dimensions to match given lengths, got 1 and 1 INFO: Build completed, 1 test FAILED, 6 total actions //zml:test FAILED in 22.9s Executed 1 out of 1 test: 1 fails locally. ``` ## Additional checks ```text zig fmt --check zml/tensor.zig fmt_status=0 ``` --- zml/tensor.zig | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/zml/tensor.zig b/zml/tensor.zig index e75208d8..18c5b910 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -559,18 +559,21 @@ pub const Tensor = struct { .f32 => .c64, else => .c128, }; - const shape_ = self._shape.setDim(-1, @divExact(self.dim(-1), 2) + 1); + const shape_ = self._shape.setDim(-1, rfftOutputLen(self.dim(-1))); break :blk shape_.withDtype(dt); }, .IRFFT => blk: { stdx.debug.assert(self.dtype().isComplex(), "fft({any}) expects tensor type to be complex, got {}", .{ opts, self.dtype() }); - stdx.debug.assert(std.mem.eql(i64, self.dims()[self.rank() - opts.length.len ..], opts.length), "fft({any}) expects tensor last dimensions to match given lengths, got {} and {}", .{ opts, self.dims()[self.rank() - opts.length.len ..].len, opts.length.len }); + const fft_dims = self.dims()[self.rank() - opts.length.len ..]; + const fft_last_dim_index = opts.length.len - 1; + stdx.debug.assert(std.mem.eql(i64, fft_dims[0..fft_last_dim_index], opts.length[0..fft_last_dim_index]), "fft({any}) expects non-innermost tensor FFT dimensions to match given lengths, got {any} and {any}", .{ opts, fft_dims[0..fft_last_dim_index], opts.length[0..fft_last_dim_index] }); + stdx.debug.assert(self.dim(-1) == rfftOutputLen(opts.length[fft_last_dim_index]), "fft({any}) expects IRFFT innermost dimension to match the RFFT output length, got {} for FFT length {}", .{ opts, self.dim(-1), opts.length[fft_last_dim_index] }); const dt: DataType = switch (self.dtype()) { .c64 => .f32, else => .f64, }; - const shape_ = self._shape.setDim(-1, @divExact(self.dim(-1) - 1, 2)); + const shape_ = self._shape.setDim(-1, opts.length[fft_last_dim_index]); break :blk shape_.withDtype(dt); }, }; @@ -579,6 +582,38 @@ pub const Tensor = struct { return _result(sh, op.result(0)); } + fn rfftOutputLen(input_len: i64) i64 { + const rfft_len_divisor = 2; + const rfft_edge_bin_count = 1; + return @divTrunc(input_len, rfft_len_divisor) + rfft_edge_bin_count; + } + + test "Tensor.fft IRFFT restores RFFT last dimension" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const rfft_input_len: i64 = 10; + + const Local = struct { + pub fn _fwd(x: Tensor) Tensor { + const freq = x.fft(.{ .kind = .RFFT, .length = &.{rfft_input_len} }); + std.debug.assert(freq.dim(-1) == rfftOutputLen(rfft_input_len)); + const time = freq.fft(.{ .kind = .IRFFT, .length = &.{rfft_input_len} }); + std.debug.assert(time.dim(-1) == x.dim(-1)); + return time; + } + }; + + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Local._fwd, + .{Tensor.init(.{rfft_input_len}, .f32)}, + platform, + .{}, + ); + defer exe.deinit(); + } + pub const Rng = struct { _state: Tensor, algorithm: dialects.stablehlo.RngAlgorithm.Type = .DEFAULT, From 87a211d3ea770fdfd8f903116ba07f94aa08f724 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Fri, 10 Jul 2026 13:21:30 +0200 Subject: [PATCH 570/881] zml/vfs/hf: Increased chunk size for HF (#646) With only 16MiB chunks, I'm hitting the authenticated user rate limits when loading Qwen3.6-27B sharded over two devices. --- zml/io/vfs/gcs.zig | 9 ++++++++- zml/io/vfs/hf.zig | 10 +++++++++- zml/io/vfs/parallel_read.zig | 12 ++++++------ zml/io/vfs/s3.zig | 9 ++++++++- 4 files changed, 31 insertions(+), 9 deletions(-) diff --git a/zml/io/vfs/gcs.zig b/zml/io/vfs/gcs.zig index 186c55c1..65fd07f8 100644 --- a/zml/io/vfs/gcs.zig +++ b/zml/io/vfs/gcs.zig @@ -317,7 +317,14 @@ pub const GCS = struct { } = null, endpoint_url: []const u8 = "https://storage.googleapis.com", region: []const u8 = "auto", - read_pool: parallel_read.InitOpts = .{}, + read_pool: parallel_read.InitOpts = .{ + .chunk_size = 16 << 20, + .num_workers = 32, + .queue_capacity = 128, + .max_retries = 5, + .retry_initial_delay = .fromMilliseconds(500), + .retry_max_delay = .fromSeconds(30), + }, }; pub const InitError = error{ diff --git a/zml/io/vfs/hf.zig b/zml/io/vfs/hf.zig index 6d1fd325..c3415e4c 100644 --- a/zml/io/vfs/hf.zig +++ b/zml/io/vfs/hf.zig @@ -201,7 +201,15 @@ const RepoKey = struct { pub const HF = struct { pub const InitOpts = struct { - read_pool: parallel_read.InitOpts = .{}, + read_pool: parallel_read.InitOpts = .{ + // Chunk size needs to be big enough to avoid hitting the rate limits of HF. + .chunk_size = 32 << 20, + .num_workers = 32, + .queue_capacity = 128, + .max_retries = 5, + .retry_initial_delay = .fromMilliseconds(500), + .retry_max_delay = .fromSeconds(30), + }, }; pub const Repo = struct { diff --git a/zml/io/vfs/parallel_read.zig b/zml/io/vfs/parallel_read.zig index 2a08d407..70c9d89d 100644 --- a/zml/io/vfs/parallel_read.zig +++ b/zml/io/vfs/parallel_read.zig @@ -3,12 +3,12 @@ const std = @import("std"); const stdx = @import("stdx"); pub const InitOpts = struct { - chunk_size: usize = 16 * 1024 * 1024, - num_workers: usize = 32, - queue_capacity: usize = 128, - max_retries: usize = 5, - retry_initial_delay: std.Io.Duration = .fromMilliseconds(500), - retry_max_delay: std.Io.Duration = .fromSeconds(30), + chunk_size: usize, + num_workers: usize, + queue_capacity: usize, + max_retries: usize, + retry_initial_delay: std.Io.Duration, + retry_max_delay: std.Io.Duration, }; pub const Status = union(enum) { diff --git a/zml/io/vfs/s3.zig b/zml/io/vfs/s3.zig index 58d71cdf..3936dfe3 100644 --- a/zml/io/vfs/s3.zig +++ b/zml/io/vfs/s3.zig @@ -302,7 +302,14 @@ pub const S3 = struct { }; pub const InitOpts = struct { - read_pool: parallel_read.InitOpts = .{}, + read_pool: parallel_read.InitOpts = .{ + .chunk_size = 16 << 20, + .num_workers = 32, + .queue_capacity = 128, + .max_retries = 5, + .retry_initial_delay = .fromMilliseconds(500), + .retry_max_delay = .fromSeconds(30), + }, }; pub const Config = struct { From 6970e7923e53b9e69cef0ec3c7280bfa7b1baf0c Mon Sep 17 00:00:00 2001 From: Pierre Chapuis Date: Fri, 10 Jul 2026 16:52:05 +0200 Subject: [PATCH 571/881] bin/zml-smi: fix memory & compute over-reporting for oneapi (#645) When a DRM fd is duplicated or inherited through fork(), the same oneAPI DRM client can appear multiple times under /proc//fdinfo. Summing each fdinfo entry double-counts GPU memory and engine time. Parse drm-client-id and skip entries whose (device_idx, client_id) pair has already been seen during the collection pass. Include the device in the key because drm-client-id may be scoped per DRM device. Add a focused test covering duplicate clients on the same device and distinct clients across devices or client IDs. --------- Co-authored-by: Kevin Barre --- bin/zml-smi/platforms/oneapi/monitor.zig | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/bin/zml-smi/platforms/oneapi/monitor.zig b/bin/zml-smi/platforms/oneapi/monitor.zig index 80a7a336..503ae0df 100644 --- a/bin/zml-smi/platforms/oneapi/monitor.zig +++ b/bin/zml-smi/platforms/oneapi/monitor.zig @@ -243,6 +243,9 @@ pub fn collectProcessUsage(allocator: std.mem.Allocator, io: std.Io, devices: [] var result: ProcessUsage = .{}; errdefer result.deinit(allocator); + var seen_device_clients: std.AutoHashMapUnmanaged(struct { u16, u64 }, void) = .{}; + defer seen_device_clients.deinit(allocator); + var proc_dir = try std.Io.Dir.openDirAbsolute(io, "/proc", .{ .iterate = true }); defer proc_dir.close(io); @@ -263,6 +266,13 @@ pub fn collectProcessUsage(allocator: std.mem.Allocator, io: std.Io, devices: [] const parsed = parseFdinfoFile(io, file_path, now_ns) catch continue; const bus_device_identifier = parsed.bus_device_identifier orelse continue; const dev_idx = matchDevice(devices, bus_device_identifier) orelse continue; + + // Skip fd if DRM client has been seen for this device. + // Summing after dup() / fork() over-reports memory and compute. + if (parsed.drm_client_id) |cid| { + if ((try seen_device_clients.getOrPut(allocator, .{ dev_idx, cid })).found_existing) continue; + } + const key = processKey(pid, dev_idx); const gop = try result.getOrPut(allocator, key); if (!gop.found_existing) { @@ -505,6 +515,7 @@ const FdinfoSample = struct { const ParsedFdinfo = struct { bus_device_identifier: ?[bus_device_identifier_len]u8 = null, + drm_client_id: ?u64 = null, sample: FdinfoSample = .{}, resident_vram_kib: u64 = 0, total_vram_kib: u64 = 0, @@ -555,6 +566,11 @@ inline fn parseFdinfoLine(parsed: *ParsedFdinfo, line: []const u8, timestamp_ns: return; } + if (std.mem.startsWith(u8, line, "drm-client-id:")) { + parsed.drm_client_id = firstInt(line["drm-client-id:".len..]); + return; + } + if (std.mem.startsWith(u8, line, "vram mem:")) { parsed.total_vram_kib += memoryKiB(line["vram mem:".len..]); return; @@ -715,6 +731,7 @@ inline fn containsAsciiIgnoreCase(haystack: []const u8, needle: []const u8) bool test "oneAPI fdinfo parser" { var parsed: ParsedFdinfo = .{}; parseFdinfoLine(&parsed, "drm-pdev:\t0000:03:00.0", 10); + parseFdinfoLine(&parsed, "drm-client-id:\t42", 10); parseFdinfoLine(&parsed, "drm-total-memory:\t2048 KiB", 10); parseFdinfoLine(&parsed, "drm-engine-render:\t100 ns", 10); parseFdinfoLine(&parsed, "drm-engine-compute:\t50 ns", 10); @@ -722,6 +739,7 @@ test "oneAPI fdinfo parser" { try std.testing.expect(parsed.bus_device_identifier != null); try std.testing.expectEqualSlices(u8, "0000:03:00.0", &parsed.bus_device_identifier.?); + try std.testing.expectEqual(@as(?u64, 42), parsed.drm_client_id); try std.testing.expectEqual(@as(?u64, 2048), parsed.sample.mem_kib); try std.testing.expectEqual(@as(u64, 150), parsed.sample.engine.?.engine_ns); try std.testing.expectEqual(@as(?EngineSample, null), parsed.sample.encoder); @@ -785,6 +803,16 @@ test "oneAPI fdinfo parser falls back to total vram" { try std.testing.expectEqual(@as(?u64, 2048), parsed.sample.mem_kib); } +test "oneAPI DRM clients dedup per device" { + var seen_device_clients: std.AutoHashMapUnmanaged(struct { u16, u64 }, void) = .{}; + defer seen_device_clients.deinit(std.testing.allocator); + + try std.testing.expectEqual(false, (try seen_device_clients.getOrPut(std.testing.allocator, .{ 0, 42 })).found_existing); + try std.testing.expectEqual(true, (try seen_device_clients.getOrPut(std.testing.allocator, .{ 0, 42 })).found_existing); + try std.testing.expectEqual(false, (try seen_device_clients.getOrPut(std.testing.allocator, .{ 1, 42 })).found_existing); + try std.testing.expectEqual(false, (try seen_device_clients.getOrPut(std.testing.allocator, .{ 0, 43 })).found_existing); +} + test "oneAPI process utilization delta" { try std.testing.expectEqual(@as(?u16, 50), processUtil(.{ .engine_ns = 100, .timestamp_ns = 1000 }, .{ .engine_ns = 600, .timestamp_ns = 2000 })); try std.testing.expectEqual(@as(?u16, 1), processUtil(.{ .engine_ns = 100, .timestamp_ns = 1000 }, .{ .engine_ns = 101, .timestamp_ns = 2000 })); From ed2bd190e8dd1e47bda2819015f443874b68243a Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Fri, 10 Jul 2026 18:35:37 +0200 Subject: [PATCH 572/881] bin/zml-smi: reuse shared shiny logo art (#647) Render the zml-smi text logo fallback from the shared ZML logo data, including the shiny per-glyph colors. Add a lightweight //zml:logo target so the TUI can depend on the logo art without pulling in //zml. Verified with: - bazel test //bin/zml-smi/tui:test - bazel build //bin/zml-smi:zml-smi --- bin/zml-smi/tui/BUILD.bazel | 1 + bin/zml-smi/tui/pages/overview.zig | 10 ++-- bin/zml-smi/tui/widgets/logo.zig | 74 +++++++++++++++++------------- zml/BUILD.bazel | 8 ++++ 4 files changed, 56 insertions(+), 37 deletions(-) diff --git a/bin/zml-smi/tui/BUILD.bazel b/bin/zml-smi/tui/BUILD.bazel index e0eb6420..29f4824c 100644 --- a/bin/zml-smi/tui/BUILD.bazel +++ b/bin/zml-smi/tui/BUILD.bazel @@ -45,6 +45,7 @@ zig_library( "//bazel/stamp:zig", "//bin/zml-smi/info", "//bin/zml-smi/utils:double_buffer", + "//zml:logo", "@libvaxis//:vaxis", ] + select({ "@platforms//os:macos": ["//bin/zml-smi/platforms/macos"], diff --git a/bin/zml-smi/tui/pages/overview.zig b/bin/zml-smi/tui/pages/overview.zig index 6605d2ff..e91253bd 100644 --- a/bin/zml-smi/tui/pages/overview.zig +++ b/bin/zml-smi/tui/pages/overview.zig @@ -48,12 +48,11 @@ pub fn deinit(self: *Overview, allocator: std.mem.Allocator) void { } fn drawNarrowBanner(self: *const Overview, ctx: vxfw.DrawContext, content_w: u16) !vxfw.Surface { - const logo: Logo = .{ .image = image_cache.global.get("logo"), .compact = true }; + const logo: Logo = .{ .image = image_cache.global.get("logo") }; const info_lines: InfoLines = .{ .state = self.state }; - const logo_h: u16 = if (logo.image != null) Logo.compact_image_height else Logo.compact_height; const children = [2]vxfw.Widget{ - try compose.sized(ctx.arena, try compose.center(ctx.arena, ui.widget(&logo)), .{ .width = content_w, .height = logo_h }), + try compose.sized(ctx.arena, try compose.center(ctx.arena, ui.widget(&logo)), .{ .width = content_w, .height = Logo.logo_height }), ui.widget(&info_lines), }; const layout: ColumnLayout = .{ .children = &children, .gap = 1 }; @@ -61,13 +60,12 @@ fn drawNarrowBanner(self: *const Overview, ctx: vxfw.DrawContext, content_w: u16 } fn drawWideBanner(self: *const Overview, ctx: vxfw.DrawContext, content_w: u16) !vxfw.Surface { - const logo: Logo = .{ .image = image_cache.global.get("logo"), .compact = false }; + const logo: Logo = .{ .image = image_cache.global.get("logo") }; const info_lines: InfoLines = .{ .state = self.state }; - const logo_h: u16 = if (logo.image != null) Logo.image_height else Logo.logo_height; const logo_box_w = Logo.logo_width + 6; // +4 centering + 2 absorbed page margin const info_max_w = @min(content_w -| logo_box_w, max_info_width); - const banner_h = @max(logo_h, InfoLines.entry_count); + const banner_h = @max(Logo.logo_height, InfoLines.entry_count); const flex_items = [2]vxfw.FlexItem{ .{ .widget = try compose.sized(ctx.arena, try compose.center(ctx.arena, ui.widget(&logo)), .{ .width = logo_box_w, .height = banner_h }), .flex = 0 }, diff --git a/bin/zml-smi/tui/widgets/logo.zig b/bin/zml-smi/tui/widgets/logo.zig index b530c890..e6970485 100644 --- a/bin/zml-smi/tui/widgets/logo.zig +++ b/bin/zml-smi/tui/widgets/logo.zig @@ -1,51 +1,63 @@ const std = @import("std"); const vaxis = @import("vaxis"); const vxfw = vaxis.vxfw; -const theme = @import("../theme.zig"); const ui = @import("../lib/ui.zig"); const compose = @import("../lib/compose.zig"); const Image = @import("../lib/image.zig"); +const zml_logo = @import("zml/logo"); const Logo = @This(); -style: vaxis.Cell.Style = theme.header_style, +style: vaxis.Cell.Style = .{}, image: ?vaxis.Image = null, -compact: bool = false, -const ascii_lines = [_][]const u8{ - " ", - "█████╗███╗ ███╗██╗ ", - "╚═██╔╝████╗ ████║██║ ", - " ██╔╝ ██╔████╔██║██║ ", - "█████╗██║ ╚═╝ ██║████╗", - "╚════╝╚═╝ ╚═╝╚═══╝", -}; - -pub const logo_width: u16 = 22; -pub const logo_height: u16 = ascii_lines.len; -pub const compact_height: u16 = ascii_lines.len - 1; -pub const image_height: u16 = 11; -pub const compact_image_height: u16 = 7; +pub const logo_width: u16 = 28; +pub const logo_height: u16 = @intCast(zml_logo.zml_art_blocks[0].rows.len); pub fn draw(self: *const Logo, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { if (self.image) |img| { - const h = if (self.compact) compact_image_height else image_height; - const img_w: Image = .{ .image = img, .rows = h }; + const img_w: Image = .{ .image = img, .rows = logo_height }; return img_w.draw(ctx); } - const lines: []const []const u8 = if (self.compact) ascii_lines[1..] else &ascii_lines; - const h: u16 = if (self.compact) compact_height else logo_height; - const logo_color: vaxis.Cell.Color = .{ .rgb = .{ 140, 180, 255 } }; - var sb = compose.surfaceBuilder(ctx.arena); - for (lines, 0..) |line, row| { - const text: vxfw.Text = .{ - .text = line, - .style = .{ .bold = self.style.bold, .fg = logo_color }, - .softwrap = false, - }; - try sb.add(@intCast(row), 0, try text.draw(ui.maxSize(ctx, logo_width, 1))); + for (0..logo_height, 0..) |logo_row, row| { + try sb.add(@intCast(row), 0, try self.drawLogoRow(ctx, logo_row)); } - return sb.finish(.{ .width = logo_width, .height = h }, ui.widget(self)); + return sb.finish(.{ .width = logo_width, .height = logo_height }, ui.widget(self)); +} + +fn drawLogoRow(self: *const Logo, ctx: vxfw.DrawContext, row_index: usize) std.mem.Allocator.Error!vxfw.Surface { + var segments: std.ArrayList(vaxis.Cell.Segment) = .empty; + for (zml_logo.zml_art_blocks[0..]) |block| { + const row = block.rows[row_index]; + if (row.text.len == 0) continue; + + var view: std.unicode.Utf8View = .initUnchecked(row.text); + var iter = view.iterator(); + while (iter.nextCodepointSlice()) |codepoint| { + const shiny = zml_logo.isShineGlyph(codepoint); + try segments.append(ctx.arena, .{ + .text = codepoint, + .style = .{ + .fg = switch (if (shiny) row.shine else row.color) { + .reset => .default, + .cyan => .{ .rgb = .{ 0, 255, 255 } }, + .sky => .{ .rgb = .{ 0, 221, 255 } }, + .blue => .{ .rgb = .{ 135, 143, 255 } }, + .violet => .{ .rgb = .{ 215, 90, 219 } }, + .pink => .{ .rgb = .{ 242, 48, 174 } }, + .shine_cyan => .{ .rgb = .{ 140, 255, 255 } }, + .shine_sky => .{ .rgb = .{ 128, 244, 255 } }, + .shine_blue => .{ .rgb = .{ 205, 210, 255 } }, + .shine_violet => .{ .rgb = .{ 255, 170, 255 } }, + .shine_pink => .{ .rgb = .{ 255, 150, 220 } }, + }, + .bg = self.style.bg, + .bold = shiny, + }, + }); + } + } + return ui.drawRichLine(ctx, segments.items, logo_width); } diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index a319c6a4..5fa7a4c5 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -65,6 +65,14 @@ cc_library( }), ) +zig_library( + name = "logo", + srcs = ["logo.zig"], + import_name = "zml/logo", + main = "logo.zig", + visibility = ["//visibility:public"], +) + zig_library( name = "zml", tags = ["manual"], From 715047e025d37b16bda3d2ac2e920fdf4887f719 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 13 Jul 2026 14:52:35 +0200 Subject: [PATCH 573/881] workspace: add LLMD issue template --- .github/ISSUE_TEMPLATE/llmd--bug-report.md | 34 ++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/llmd--bug-report.md diff --git a/.github/ISSUE_TEMPLATE/llmd--bug-report.md b/.github/ISSUE_TEMPLATE/llmd--bug-report.md new file mode 100644 index 00000000..b9629993 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/llmd--bug-report.md @@ -0,0 +1,34 @@ +--- +name: 'LLMD: Bug report' +about: If you hit a bug in LLMD, let us know +title: "[LLMD]" +labels: LLMD +assignees: '' +type: Bug + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Configuration:** + - Model: [e.g. `google/gemma-4-31B-it`] + - OS: [e.g. `Ubuntu 24.04`] + - Platform: [`cuda` / `rocm` / `oneapi` / `metal` / `tpu`] + - Accelerator: [e.g. `AMD MI300X`] + - Driver version: [e.g. `595.71.05`] + - Version: [e.g. 0.1.2 or docker image `sha256@`] + - Command line: [e.g. `docker run ...`] + +**Additional context** +Add any other context about the problem here. From ece52d46bfac7be4b1182aee7ddc559fefed58e1 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 15 Jul 2026 13:53:54 +0200 Subject: [PATCH 574/881] zml: allow PJRT compilation to fail (#654) We used to panic when PJRT compilation failed. I now think it's better to return an error, since we introduced the WIP Metal plugin that can fail on legitimate programs. This will make it possible to run the ZML test suite on the Metal plugin and seeing all failing tests. ``` bazel run --@zml//platforms:metal=true --@zml//platforms:cpu=false //zml:test 33/138 nn.decltest.sampleTokens...FAIL (Internal) ... 34/138 nn.decltest.sampleTokensDynamic...FAIL (Internal) ... 102/138 tensor.Tensor.decltest.convert...FAIL (Internal) ... 115/138 tensor.Tensor.decltest.argsort...FAIL (Internal) ``` --- zml/module.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/module.zig b/zml/module.zig index c78f1b77..3c309173 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -266,7 +266,7 @@ pub fn compile( var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); - const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), st_io.io(), platform, compilation_context.module, compilation_context.partitioning, opts) catch unreachable; + const loaded_executable = try compileModuleToPjrtExecutable(arena.allocator(), st_io.io(), platform, compilation_context.module, compilation_context.partitioning, opts); log.debug("\n******** ZML generated MLIR ********\n{f}", .{compilation_context.module.operation()}); const exe = try Exe.init( From dbf3834b29bbb6d72d30054bf3001c451f628cdb Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 15 Jul 2026 14:17:22 +0200 Subject: [PATCH 575/881] zml: improve Buffer sharding related error messages (#655) It's easy to break `Buffer.from(x_tensor.shape(), data, sharding)` by creating `x_tensor` in the model code with a specific sharding and you pass a different one to `from`. I think this should be a crash (like when you mix-up allocators in init/deinit) and it should have a dedicated error message to guide the developers. --- zml/Sharding.zig | 8 ++-- zml/buffer.zig | 107 ++++++++++++++++++++++++++++++++++++++++------- zml/testing.zig | 21 ++++++++-- 3 files changed, 114 insertions(+), 22 deletions(-) diff --git a/zml/Sharding.zig b/zml/Sharding.zig index 42dc4d14..cb52e542 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -18,6 +18,8 @@ const Sharding = @This(); data: *const Data, +pub const Error = error{ MissingLogicalBinding, IncompatibleSharding }; + pub const MAX_MESH_RANK = 4; var _replicated: [11]u8 align(@alignOf(Data)) = "_replicated".*; @@ -1767,12 +1769,12 @@ pub const Strategy = struct { }; /// For a given shape, compute the shape of the slice each shard will receive. -pub fn shardedShape(sharding: Sharding, shape: Shape) !Shape { +pub fn shardedShape(sharding: Sharding, shape: Shape) Error!Shape { const pl = try sharding.placement(shape); return pl.shape; } -pub fn placement(sharding: Sharding, shape: Shape) !Placement { +pub fn placement(sharding: Sharding, shape: Shape) Error!Placement { return .init(sharding, shape); } @@ -1792,7 +1794,7 @@ pub const Placement = struct { axis_plans: stdx.BoundedArray(AxisSplit, Shape.MAX_RANK), - pub fn init(sharding: Sharding, shape: Shape) !Placement { + pub fn init(sharding: Sharding, shape: Shape) Error!Placement { var pl: Placement = .{ .sharding = sharding, .shape = shape, // modified below diff --git a/zml/buffer.zig b/zml/buffer.zig index 2f205fb5..4e9ca6d7 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -18,6 +18,10 @@ const testing = @import("testing.zig"); const log = std.log.scoped(.zml); +test { + std.testing.refAllDecls(Buffer); +} + /// Buffer is a multi-dimension array, whose memory is allocated on an accelerator. /// /// * contains a handle that the ZML runtime can use to convert into a physical address, but there is no guarantee this address is visible from the CPU. @@ -118,7 +122,7 @@ pub const Buffer = struct { const slice = Slice.init(sh, data_); const buffer_type = pjrtx.bufferTypeFromDtype(sh.dtype()); - const placement = try res._sharding.placement(sh); + const placement = placementOrPanic(res._sharding, sh); const shard_dims: []const i64 = placement.shape.dims(); const layout = platform.defaultMemoryLayout(shard_dims, sh.dtype()); @@ -204,7 +208,7 @@ pub const Buffer = struct { stdx.debug.assert(platform.devices[0].memory(opts.memory) != null, "Device doesn't have {} memory", .{opts.memory}); const element_type = pjrtx.bufferTypeFromDtype(sh.dtype()); - const placement = try res._sharding.placement(sh); + const placement = placementOrPanic(res._sharding, sh); const shard_dims: []const i64 = placement.shape.dims(); const layout = platform.defaultMemoryLayout(shard_dims, sh.dtype()); @@ -253,7 +257,7 @@ pub const Buffer = struct { pub fn toSlice(self: Buffer, io: std.Io, slice: Slice) !void { stdx.debug.assert(self._shape.eql(slice.shape), "Buffer shape {f} doesn't match destination slice {f}", .{ self._shape, slice.shape }); - const placement = try self._sharding.placement(self._shape); + const placement = placementOrPanic(self._sharding, self._shape); for (self._sharding.devicesInCanonicalOrder(), 0..) |device, shard_index| { // TODO: handle replicated information, we shouldn't iterate over all the devices unless needed const sub_slice = placement.shardSlice(device.coords, slice); @@ -275,36 +279,107 @@ pub const Buffer = struct { const slice = try Slice.alloc(allocator, self.shape()); errdefer slice.free(allocator); - const placement = try self._sharding.placement(self._shape); + const placement = placementOrPanic(self._sharding, self._shape); + + var shard_slice = try Slice.alloc(allocator, placement.shape); + defer shard_slice.free(allocator); + for (self._sharding.devicesInCanonicalOrder(), 0..) |device, shard_index| { const sub_slice = placement.shardSlice(device.coords, slice); - - var shard_slice = try Slice.alloc(allocator, sub_slice.shape); - defer shard_slice.free(allocator); - const maybe_event = try self._shards.get(shard_index).toHostBuffer(self._platform.pjrt_api, shard_slice.data()); if (maybe_event) |event| { try event.await(self._platform.pjrt_api, io); } + // TODO: why is this using Slice.copy while `toSlice` errors out ? + // TODO: why is this writing in a copy rather than directly in place. sub_slice.copy(shard_slice.constData()); } return slice; } + /// The memory used by this Buffer across all devices + /// ie: `num_devices * shard_byte_size` + /// `shard_byte_size` can be up to `self.shape().byteSize()` when the buffer is fully replicated. pub fn byteSize(self: Buffer) usize { - var byte_size: usize = 0; - - for (self._sharding.devicesInCanonicalOrder()) |device| { - const placement = self._sharding.placement(self._shape, device) catch unreachable; - byte_size += placement.shape.byteSize(); - } - - return byte_size; + const placement = placementOrPanic(self._sharding, self._shape); + return placement.shape.byteSize() * self._sharding.devicesInCanonicalOrder().len; } pub fn opaqueDevicePtr(self: Buffer, device_id: usize) *anyopaque { return self._shards.get(device_id).opaqueDeviceMemoryDataPointer(self._platform.pjrt_api) catch unreachable; } }; + +test "device round-trip" { + const zml = @import("zml.zig"); + const io = std.testing.io; + const allocator = std.testing.allocator; + const platform = zml.testing.env(); + + const x: [8][8]u32 = .{ + .{ 0, 1, 2, 3, 4, 5, 6, 7 }, + .{ 8, 9, 10, 11, 12, 13, 14, 15 }, + .{ 16, 17, 18, 19, 20, 21, 22, 23 }, + .{ 24, 25, 26, 27, 28, 29, 30, 31 }, + .{ 32, 33, 34, 35, 36, 37, 38, 39 }, + .{ 40, 41, 42, 43, 44, 45, 46, 47 }, + .{ 48, 49, 50, 51, 52, 53, 54, 55 }, + .{ 56, 57, 58, 59, 60, 61, 62, 63 }, + }; + + const x_h: zml.Slice = .init(.withPartitioning( + .init(.{ .b = 8, .d = 8 }, .u32), + .{ .b = .model }, + ), std.mem.asBytes(&x)); + // no free: x_h is stack allocated + const model_sharding: zml.Sharding = platform.shardings.get("model").?; + const x_d: zml.Buffer = try .fromSlice(io, platform, x_h, model_sharding); + try std.testing.expectEqual(platform.devices.len, x_d.numShards()); + + { + const x_h_reborn: zml.Slice = try x_d.toSliceAlloc(allocator, io); + defer x_h_reborn.free(allocator); + + errdefer std.log.err(" - reference: {d}\n- actual: {d}", .{ x_h, x_h_reborn }); + try zml.testing.expectClose(io, x_h, x_h_reborn, .exact_match); + } + + { + var x_2: @TypeOf(x) = undefined; + const x_h_reborn: zml.Slice = .init(x_h.shape, std.mem.asBytes(&x_2)); + // no free: x_h_reborn is stack allocated + try x_d.toSlice(io, x_h_reborn); + + errdefer std.log.err(" - reference: {d}\n- actual: {d}", .{ x_h, x_h_reborn }); + try zml.testing.expectClose(io, x_h, x_h_reborn, .exact_match); + } +} + +fn placementOrPanic(sharding: Sharding, shape: Shape) Sharding.Placement { + return sharding.placement(shape) catch |err| { + @branchHint(.cold); + switch (err) { + error.MissingLogicalBinding => { + log.err( + \\Failed to shard Buffer of shape {f}, with sharding: + \\{f} + \\ + \\The Buffer is probably inheriting a partitionned shape from a Tensor, + \\So Buffer creation must pass a Sharding, that maps the logical sharding of the Tensor to the physical mesh. + , .{ shape, sharding }); + @panic("Buffer shape and sharding should be consistent"); + }, + error.IncompatibleSharding => { + log.err( + \\Failed to shard Buffer of shape {f}, with sharding: + \\{f} + \\ + \\The Buffer dimension isn't properly divisible by the number of devices along the sharded axis. + , .{ shape, sharding }); + @panic("Buffer shape should be divisible by the number of devices along the sharded axis."); + }, + } + }; +} diff --git a/zml/testing.zig b/zml/testing.zig index 288a883b..968328af 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -14,9 +14,17 @@ var _platform: ?*const Platform = null; pub fn env() *const Platform { if (!builtin.is_test) @compileError("Cannot use zml.testing.env outside of a test block"); if (_platform == null) { - _platform = Platform.auto(std.heap.c_allocator, std.testing.io, .{ - .xla_gpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.85 } } }, - }) catch unreachable; + const platform = Platform.auto( + std.heap.c_allocator, + std.testing.io, + .{ + .xla_gpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.85 } } }, + }, + ) catch @panic("Pjrt not available"); + + // Register shardings required by flash attention. + _ = platform.registerSharding("model", .mesh(.{ .model = .high_bandwidth })) catch unreachable; + _platform = platform; } return _platform.?; @@ -118,6 +126,13 @@ pub const CompareOpts = struct { relative_tolerance: f32 = 1e-2, epsilon_relative: f32 = 1e-6, minimum_close_fraction: f32 = 0.999, + + pub const exact_match: CompareOpts = .{ + .absolute_tolerance = 0, + .relative_tolerance = 0, + .epsilon_relative = 0, + .minimum_close_fraction = 1.00, + }; }; pub const CompareReport = struct { From 16ea2e2718178e68d5c7981575eea88369360411 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 15 Jul 2026 14:27:50 +0200 Subject: [PATCH 576/881] zml/tensor: remove `concatenate` and `stack` limits (#656) We can now rely on the compilation context to allocate when needed. --- zml/tensor.zig | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/zml/tensor.zig b/zml/tensor.zig index 18c5b910..653a3fcf 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1893,8 +1893,7 @@ pub const Tensor = struct { /// Concatenates the input Tensors along the given axis. pub fn concatenate(tensors: []const Tensor, axis_: anytype) Tensor { if (tensors.len == 1) return tensors[0]; - stdx.debug.assert(tensors.len <= 32, "concatenate only supports up to 32 tensors, got {}", .{tensors.len}); - var buffer: [32]*const mlir.Value = undefined; + var buffer = CompilationContext.current().arena.allocator().alloc(*const mlir.Value, tensors.len) catch @panic("OOM"); std.debug.assert(tensors.len <= buffer.len); std.debug.assert(tensors.len > 0); const a = tensors[0].axis(axis_); @@ -1907,7 +1906,7 @@ pub const Tensor = struct { } const res_shape = tensors[0]._shape.set(a, concatenated_dim); - const op = dialects.stablehlo.concatenate(mlirCtx(), buffer[0..tensors.len], a, .unknown(mlirCtx())).appendTo(currentBlock()); + const op = dialects.stablehlo.concatenate(mlirCtx(), buffer, a, .unknown(mlirCtx())).appendTo(currentBlock()); // log.debug("concatenate({}, {}, {d}) -> {d}", .{ tensors[0], tensors[1], a, res_shape }); return _result(res_shape, op.result(0)); } @@ -1918,9 +1917,6 @@ pub const Tensor = struct { /// - Tensor.stack(&.{x, y, z}, 1, .layers) -> .{ .a, .layers, .b, .c } /// - Tensor.stack(&.{x, y, z}, .last, .layers) -> .{ .a, .b, .c, .layers } pub fn stack(tensors: []const Tensor, axis_: anytype, tag: anytype) Tensor { - // Note: we could ask the compilation context for some memory instead of stack allocating - stdx.debug.assert(tensors.len <= 32, "stack only supports up to 32 tensors, got {}", .{tensors.len}); - const shape0 = tensors[0]._shape; const res_shape = shape0.insertTag(axis_, 1, tag); @@ -1928,7 +1924,7 @@ pub const Tensor = struct { stdx.debug.assert(shape0.eqlWithTags(tensor._shape), "stack expects tensor shapes to match, got {f} and {f}", .{ shape0, tensor._shape }); } - var reshaped: [32]Tensor = undefined; + var reshaped = CompilationContext.current().arena.allocator().alloc(Tensor, tensors.len) catch @panic("OOM"); for (tensors, 0..) |tensor, i| { reshaped[i] = tensor.reshape(res_shape); } @@ -1940,7 +1936,7 @@ pub const Tensor = struct { else shape0.axis(axis_); - return Tensor.concatenate(reshaped[0..tensors.len], ax); + return Tensor.concatenate(reshaped, ax); } /// Repeats a Tensor several times along the given axis. From 24165146b9e791635f87e6c3b18bbffb8d873cfc Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Wed, 15 Jul 2026 16:21:56 +0200 Subject: [PATCH 577/881] platform/intel: bump oneapi=2026.1 and rules_distroless=0.8.0 (#644) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Bump intel oneapi and rules_distroless The Debian package set is now also exposed separately through the rules_distroless APT extension: Now use `rules_distroless` to install the oneAPI packages in a distroless container image. So the platforms/oneapi/packages.lock.json file is now generated from the platforms/oneapi/packages.yaml file, and the packages.yaml file is now the source of truth for the oneAPI packages. The MODULE.bazel contain rules_distroless patch `third_party/rules_distroless/accept-nospace-version-constraints.patch` because the rules_distroless APT extension does not support version constraints with spaces, e.g. `>= 2023.2.0` is not supported, but `>=2023.2.0` is supported. A PR has been submitted to rules_distroless to fix this issue, but until it is merged and released, the patch is needed in zml https://github.com/bazel-contrib/rules_distroless/pull/229. `_ROOT_MODULE_DIRECT_DEPS` does not need anymore to list the individual oneAPI and Ubuntu package repositories. `_ROOT_PACKAGES` is now in `_UBUNTU_PACKAGES` because all the package share the same `_UBUNTU_STRIP_PREFIX` ## Tests ``` bazel run @apt_oneapi//:lock ``` ``` ONEAPI_DEVICE_SELECTOR=level_zero:0 \ bazel run \ --config=release \ --@zml//platforms:cpu=false \ --@zml//platforms:oneapi=true \ //examples/llm -- \ --model=/var/models/meta-llama/Llama-3.1-8B-Instruct \ --topk=4 \ --prompt="Tell me a story about evariste galois" ``` __-> decode 26.248s · 34.9tok/s__ ``` ONEAPI_DEVICE_SELECTOR=level_zero:0,1 \ bazel run \ --config=release \ --@zml//platforms:cpu=false \ --@zml//platforms:oneapi=true \ //examples/llm -- \ --model=/var/models/meta-llama/Llama-3.1-8B-Instruct \ --topk=4 \ --prompt="Tell me a story about evariste galois" ``` __-> decode 13.017s · 56.6tok/s__ --- MODULE.bazel | 21 ++- MODULE.bazel.lock | 6 +- platforms/oneapi/libpjrt_oneapi.BUILD.bazel | 35 ++-- platforms/oneapi/oneapi.bzl | 66 ++------ platforms/oneapi/packages.lock.json | 157 +++++++++--------- platforms/oneapi/packages.yaml | 45 +++++ .../accept-nospace-version-constraints.patch | 28 ++++ 7 files changed, 205 insertions(+), 153 deletions(-) create mode 100644 platforms/oneapi/packages.yaml create mode 100644 third_party/rules_distroless/accept-nospace-version-constraints.patch diff --git a/MODULE.bazel b/MODULE.bazel index 2649f117..1f7bd1f9 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -26,7 +26,18 @@ bazel_dep(name = "protobuf", version = "35.0", repo_name = "com_google_protobuf" bazel_dep(name = "pybind11_bazel", version = "3.0.1") bazel_dep(name = "re2", version = "2025-11-05", repo_name = "com_googlesource_code_re2") bazel_dep(name = "rules_cc", version = "0.2.19") -bazel_dep(name = "rules_distroless", version = "0.6.2") +bazel_dep(name = "rules_distroless", version = "0.8.0") +# Once the rules_distroless can parse unspaced apt version constraints +# Update the rules_distroless version and remove the patch +# https://github.com/bazel-contrib/rules_distroless/pull/229 +single_version_override( + module_name = "rules_distroless", + patch_strip = 1, + patches = [ + "//:third_party/rules_distroless/accept-nospace-version-constraints.patch", + ], + version = "0.8.0", +) bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep(name = "rules_oci", version = "2.2.7") bazel_dep(name = "rules_proto", version = "7.1.0") @@ -265,6 +276,14 @@ apt.install( ) use_repo(apt, "apt_rocm") +apt.install( + name = "apt_oneapi", + lock = "//platforms/oneapi:packages.lock.json", + manifest = "//platforms/oneapi:packages.yaml", + resolve_transitive = False, +) +use_repo(apt, "apt_oneapi") + apt.install( name = "apt_neuron", lock = "//platforms/neuron:packages.lock.json", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 10042d2a..92d5b624 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -289,8 +289,8 @@ "https://bcr.bazel.build/modules/rules_cc/0.2.19/source.json": "1ef48cdbd7aa6238015189b582d3d74ef0cbea3cb3e2cb259d782463f570c14a", "https://bcr.bazel.build/modules/rules_cc/0.2.4/MODULE.bazel": "1ff1223dfd24f3ecf8f028446d4a27608aa43c3f41e346d22838a4223980b8cc", "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", - "https://bcr.bazel.build/modules/rules_distroless/0.6.2/MODULE.bazel": "eafe8b473c0c78038d9a6e1dbd9278b7bf83457ddc78207297483bfbe1012d56", - "https://bcr.bazel.build/modules/rules_distroless/0.6.2/source.json": "830d7a25ce382e571e58791fbed3b47e6da3237805624bab8997ae59d763bdc2", + "https://bcr.bazel.build/modules/rules_distroless/0.8.0/MODULE.bazel": "a3e473943b685190e20668f5149f62a116fdcbeba66c7b9778f71d2aed4533c1", + "https://bcr.bazel.build/modules/rules_distroless/0.8.0/source.json": "a6931e20d97a13675693adb8b6cd3a0e6eb5218db43dfc83b75566ddc4158efc", "https://bcr.bazel.build/modules/rules_foreign_cc/0.10.1/MODULE.bazel": "b9527010e5fef060af92b6724edb3691970a5b1f76f74b21d39f7d433641be60", "https://bcr.bazel.build/modules/rules_foreign_cc/0.15.1/MODULE.bazel": "c2c60d26c79fda484acb95cdbec46e89d6b28b4845cb277160ce1e0c8622bb88", "https://bcr.bazel.build/modules/rules_foreign_cc/0.15.1/source.json": "a161811a63ba8a859086da3b7ff3ad04f2e9c255d7727b41087103fc0eb22f55", @@ -1935,7 +1935,7 @@ "@@yq.bzl+//yq:extensions.bzl%yq": { "general": { "bzlTransitiveDigest": "iWlctm1cJYZLESuQEBygCQc6adq+oIRMb4Dmk0rYxkA=", - "usagesDigest": "J3TR36exv3ePI9O2y+uqrzTDmT6s6ADkTsROB++nm6I=", + "usagesDigest": "1CP5kLHwnlFZ3obmnV0eEOp+80JltkcSimeCHXe8/+E=", "recordedInputs": [], "generatedRepoSpecs": { "yq_darwin_amd64": { diff --git a/platforms/oneapi/libpjrt_oneapi.BUILD.bazel b/platforms/oneapi/libpjrt_oneapi.BUILD.bazel index 22edb2d6..e1e5875f 100644 --- a/platforms/oneapi/libpjrt_oneapi.BUILD.bazel +++ b/platforms/oneapi/libpjrt_oneapi.BUILD.bazel @@ -15,13 +15,13 @@ patchelf( patchelf( name = "libsycl_so", - src = "@intel-oneapi-compiler-dpcpp-cpp-runtime-2026.0//:libsycl_so", + src = "@intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1//:libsycl_so", set_rpath = "$ORIGIN", ) patchelf( name = "libccl_legacy_so", - src = "@intel-oneapi-ccl-2022.0//:libccl_legacy_so", + src = "@intel-oneapi-ccl-2022.1//:libccl_legacy_so", set_rpath = "$ORIGIN", ) @@ -53,24 +53,15 @@ patchelf( soname = "libigdfcl.so.2", ) -ONEAPI_VERSION = "2026.0" - -ONEAPI_CCL_LIB = "ccl/2022.0/lib" - +ONEAPI_VERSION = "2026.1" +ONEAPI_CCL_LIB = "ccl/2022.1/lib" ONEAPI_MPI_LIB = "mpi/2021.18/lib" - ONEAPI_MPI_LIBFABRIC_LIB = "mpi/2021.18/opt/mpi/libfabric/lib" - ONEAPI_TCM_VERSION = "1.5" - ONEAPI_UMF_VERSION = "1.1" - ONEAPI_COMPILER_LIB = "compiler/{}/lib".format(ONEAPI_VERSION) - ONEAPI_MKL_LIB = "mkl/{}/lib".format(ONEAPI_VERSION) - ONEAPI_TCM_LIB = "tcm/{}/lib".format(ONEAPI_TCM_VERSION) - ONEAPI_UMF_LIB = "umf/{}/lib".format(ONEAPI_UMF_VERSION) copy_to_directory( @@ -82,15 +73,15 @@ copy_to_directory( ":libpjrt_oneapi_so", ":libsycl_so", ":libze_intel_gpu_so", - "@intel-oneapi-ccl-2022.0//:ccl_runtime", - "@intel-oneapi-compiler-dpcpp-cpp-runtime-2026.0//:sycl_runtime", - "@intel-oneapi-compiler-shared-runtime-2026.0//:compiler_runtime", - "@intel-oneapi-mkl-core-2026.0//:mkl_core_runtime", - "@intel-oneapi-mkl-sycl-blas-2026.0//:mkl_sycl_blas", - "@intel-oneapi-mkl-sycl-dft-2026.0//:mkl_sycl_dft", - "@intel-oneapi-mkl-sycl-lapack-2026.0//:mkl_sycl_lapack", - "@intel-oneapi-mkl-sycl-rng-2026.0//:mkl_sycl_rng", - "@intel-oneapi-mkl-sycl-sparse-2026.0//:mkl_sycl_sparse", + "@intel-oneapi-ccl-2022.1//:ccl_runtime", + "@intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1//:sycl_runtime", + "@intel-oneapi-compiler-shared-runtime-2026.1//:compiler_runtime", + "@intel-oneapi-mkl-core-2026.1//:mkl_core_runtime", + "@intel-oneapi-mkl-sycl-blas-2026.1//:mkl_sycl_blas", + "@intel-oneapi-mkl-sycl-dft-2026.1//:mkl_sycl_dft", + "@intel-oneapi-mkl-sycl-lapack-2026.1//:mkl_sycl_lapack", + "@intel-oneapi-mkl-sycl-rng-2026.1//:mkl_sycl_rng", + "@intel-oneapi-mkl-sycl-sparse-2026.1//:mkl_sycl_sparse", "@intel-oneapi-mpi-2021.18//:mpi_runtime", "@intel-oneapi-tcm-1.5//:hwloc", "@intel-oneapi-umf-1.1//:umf", diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index dbc1eae7..22561d9e 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -7,11 +7,11 @@ PJRT_ONEAPI_ARTIFACT_URL = "https://github.com/zml/pjrt-artifacts/releases/downl release = PJRT_ONEAPI_RELEASE, ) -ONEAPI_VERSION = "2026.0" +ONEAPI_VERSION = "2026.1" ONEAPI_TCM_VERSION = "1.5" ONEAPI_UMF_VERSION = "1.1" -ONEAPI_CCL_LIB = "ccl/2022.0/lib" +ONEAPI_CCL_LIB = "ccl/2022.1/lib" ONEAPI_COMPILER_LIB = "compiler/{}/lib".format(ONEAPI_VERSION) ONEAPI_MPI_LIB = "mpi/2021.18/lib" ONEAPI_MPI_LIBFABRIC_LIB = "mpi/2021.18/opt/mpi/libfabric/lib" @@ -34,7 +34,7 @@ def _read_packages(mctx, labels): ret.setdefault(pkg["name"], {})[pkg["arch"]] = pkg return ret -_ROOT_PACKAGES = { +_UBUNTU_PACKAGES = { "libnl-3-200": """ filegroup( name = "libnl_3", @@ -45,9 +45,6 @@ filegroup( name = "libnl_genl_3", srcs = ["lib/x86_64-linux-gnu/libnl-genl-3.so.200"], )""", -} - -_UBUNTU_PACKAGES = { "libnl-route-3-200": """ filegroup( name = "libnl_route_3", @@ -121,7 +118,11 @@ filegroup( } _ONEAPI_PACKAGES = { - "intel-oneapi-ccl-2022.0": """ + "intel-oneapi-ccl-2022.1": """ +filegroup( + name = "libccl_legacy_so", + srcs = ["{ONEAPI_CCL_LIB}/libccl_legacy.so"], +) filegroup( name = "ccl_runtime", srcs = [ @@ -199,7 +200,7 @@ filegroup( name = "umf", srcs = ["{ONEAPI_UMF_LIB}/libumf.so.1"], )""".format(ONEAPI_UMF_LIB = ONEAPI_UMF_LIB), - "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.0": """ + "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1": """ filegroup( name = "libsycl_so", srcs = ["{ONEAPI_COMPILER_LIB}/libsycl.so.9"], @@ -213,7 +214,7 @@ filegroup( "{ONEAPI_COMPILER_LIB}/libur_loader.so.0", ], )""".format(ONEAPI_COMPILER_LIB = ONEAPI_COMPILER_LIB), - "intel-oneapi-compiler-shared-runtime-2026.0": """ + "intel-oneapi-compiler-shared-runtime-2026.1": """ filegroup( name = "compiler_runtime", srcs = [ @@ -227,7 +228,7 @@ filegroup( "{ONEAPI_COMPILER_LIB}/libsvml.so", ], )""".format(ONEAPI_COMPILER_LIB = ONEAPI_COMPILER_LIB), - "intel-oneapi-mkl-core-2026.0": """ + "intel-oneapi-mkl-core-2026.1": """ filegroup( name = "mkl_core_runtime", srcs = [ @@ -236,58 +237,34 @@ filegroup( "{ONEAPI_MKL_LIB}/libmkl_sequential.so.3", ], )""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), - "intel-oneapi-mkl-sycl-blas-2026.0": """ + "intel-oneapi-mkl-sycl-blas-2026.1": """ filegroup( name = "mkl_sycl_blas", srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_blas.so.6"], )""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), - "intel-oneapi-mkl-sycl-dft-2026.0": """ + "intel-oneapi-mkl-sycl-dft-2026.1": """ filegroup( name = "mkl_sycl_dft", srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_dft.so.6"], )""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), - "intel-oneapi-mkl-sycl-lapack-2026.0": """ + "intel-oneapi-mkl-sycl-lapack-2026.1": """ filegroup( name = "mkl_sycl_lapack", srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_lapack.so.6"], )""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), - "intel-oneapi-mkl-sycl-rng-2026.0": """ + "intel-oneapi-mkl-sycl-rng-2026.1": """ filegroup( name = "mkl_sycl_rng", srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_rng.so.6"], )""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), - "intel-oneapi-mkl-sycl-sparse-2026.0": """ + "intel-oneapi-mkl-sycl-sparse-2026.1": """ filegroup( name = "mkl_sycl_sparse", srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_sparse.so.6"], )""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), } -_ROOT_MODULE_DIRECT_DEPS = [ - "intel-oneapi-ccl-2022.0", - "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.0", - "intel-oneapi-compiler-shared-runtime-2026.0", - "intel-oneapi-mkl-core-2026.0", - "intel-oneapi-mkl-sycl-blas-2026.0", - "intel-oneapi-mkl-sycl-dft-2026.0", - "intel-oneapi-mkl-sycl-lapack-2026.0", - "intel-oneapi-mkl-sycl-rng-2026.0", - "intel-oneapi-mkl-sycl-sparse-2026.0", - "intel-oneapi-mpi-2021.18", - "intel-oneapi-tcm-1.5", - "intel-oneapi-umf-1.1", - "libnl-3-200", - "libnl-genl-3-200", - "libnl-route-3-200", - "libnuma1", - "libigdgmm12", - "libigc2", - "libigdfcl2", - "libze-intel-gpu1", - "libze1", - "libpjrt_oneapi", - "zlib1g", -] +_ROOT_MODULE_DIRECT_DEPS = ["libpjrt_oneapi"] def _oneapi_impl(mctx): loaded_packages = _read_packages(mctx, [ @@ -301,15 +278,6 @@ def _oneapi_impl(mctx): url = PJRT_ONEAPI_ARTIFACT_URL, ) - for pkg_name, build_file_content in _ROOT_PACKAGES.items(): - pkg = loaded_packages[pkg_name]["amd64"] - http_deb_archive( - name = pkg_name, - urls = pkg["urls"], - sha256 = pkg["sha256"], - build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, - ) - for pkg_name, build_file_content in _UBUNTU_PACKAGES.items(): pkg = loaded_packages[pkg_name]["amd64"] http_deb_archive( diff --git a/platforms/oneapi/packages.lock.json b/platforms/oneapi/packages.lock.json index 3fb3fce3..da9fd965 100644 --- a/platforms/oneapi/packages.lock.json +++ b/platforms/oneapi/packages.lock.json @@ -3,112 +3,112 @@ { "arch": "amd64", "dependencies": [], - "key": "intel-oneapi-ccl-2022.0_2022.0.0-49301_amd64", - "name": "intel-oneapi-ccl-2022.0", - "sha256": "1003034feabacc032a6054b4667960f7da82df378fb7a331956bfc8c106a4d60", + "key": "intel-oneapi-ccl-2022.1_2022.1.0-142_amd64", + "name": "intel-oneapi-ccl-2022.1", + "sha256": "3569a089879b6b7358c5afb496ae782754715c8ace5137e00797e6b111fb4cc5", "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-ccl-2022.0-2022.0.0-49301_amd64.deb" + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-ccl-2022.1-2022.1.0-142_amd64.deb" ], - "version": "2022.0.0-49301" + "version": "2022.1.0-142" }, { "arch": "amd64", "dependencies": [], - "key": "intel-oneapi-mpi-2021.18_2021.18.0-745_amd64", + "key": "intel-oneapi-mpi-2021.18_2021.18.1-9_amd64", "name": "intel-oneapi-mpi-2021.18", - "sha256": "0eed61a43588fcaec64c7b09f6bd809c5227008e501d0e1231488fbd21eac51b", + "sha256": "f6f76f39755329bbac890deeb72fb105df064dc5dcb67cde1396e4d6ef0f6706", "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mpi-2021.18-2021.18.0-745_amd64.deb" + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mpi-2021.18-2021.18.1-9_amd64.deb" ], - "version": "2021.18.0-745" + "version": "2021.18.1-9" }, { "arch": "amd64", "dependencies": [], - "key": "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.0_2026.0.0-947_amd64", - "name": "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.0", - "sha256": "b7db83a2c8ad06ab5f080fb5ec7c5b004353157f92c0ddb9af3ece0422d9a249", + "key": "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1_2026.1.0-235_amd64", + "name": "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1", + "sha256": "f557b04243b6bb2087646b9d9eee94026bf1abc80b188dcfa3e3b689e292d95e", "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-compiler-dpcpp-cpp-runtime-2026.0-2026.0.0-947_amd64.deb" + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1-2026.1.0-235_amd64.deb" ], - "version": "2026.0.0-947" + "version": "2026.1.0-235" }, { "arch": "amd64", "dependencies": [], - "key": "intel-oneapi-compiler-shared-runtime-2026.0_2026.0.0-947_amd64", - "name": "intel-oneapi-compiler-shared-runtime-2026.0", - "sha256": "0357c84d73cee0af8d4c682bdb290908bf842885e8c24936f3e7e58159e9745e", + "key": "intel-oneapi-compiler-shared-runtime-2026.1_2026.1.0-235_amd64", + "name": "intel-oneapi-compiler-shared-runtime-2026.1", + "sha256": "b5a077f78f192f93bba608f5497d7c138f77ec211a9d86356696a0ca68d3bf5d", "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-compiler-shared-runtime-2026.0-2026.0.0-947_amd64.deb" + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-compiler-shared-runtime-2026.1-2026.1.0-235_amd64.deb" ], - "version": "2026.0.0-947" + "version": "2026.1.0-235" }, { "arch": "amd64", "dependencies": [], - "key": "intel-oneapi-mkl-core-2026.0_2026.0.0-908_amd64", - "name": "intel-oneapi-mkl-core-2026.0", - "sha256": "3cc588084e9820af237c90dba4c1546e50f2d7e02a15d0072504ed4ccf764ec0", + "key": "intel-oneapi-mkl-core-2026.1_2026.1.0-236_amd64", + "name": "intel-oneapi-mkl-core-2026.1", + "sha256": "3b230c02fc0ac44f8f71062c20cd883b37c6c1e00940dc0a7f13b02d9c47bd93", "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-core-2026.0-2026.0.0-908_amd64.deb" + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-core-2026.1-2026.1.0-236_amd64.deb" ], - "version": "2026.0.0-908" + "version": "2026.1.0-236" }, { "arch": "amd64", "dependencies": [], - "key": "intel-oneapi-mkl-sycl-blas-2026.0_2026.0.0-908_amd64", - "name": "intel-oneapi-mkl-sycl-blas-2026.0", - "sha256": "b32078f1805ab126f78333276bb860f288d3a2785dc101b4b9df499cd35fa76d", + "key": "intel-oneapi-mkl-sycl-blas-2026.1_2026.1.0-236_amd64", + "name": "intel-oneapi-mkl-sycl-blas-2026.1", + "sha256": "9668d51a7cf68ae4b8944a8cdaa24f6d4ef72604c048f68fd8603b2b792bd783", "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-blas-2026.0-2026.0.0-908_amd64.deb" + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-blas-2026.1-2026.1.0-236_amd64.deb" ], - "version": "2026.0.0-908" + "version": "2026.1.0-236" }, { "arch": "amd64", "dependencies": [], - "key": "intel-oneapi-mkl-sycl-dft-2026.0_2026.0.0-908_amd64", - "name": "intel-oneapi-mkl-sycl-dft-2026.0", - "sha256": "e4168ed1fce35e19552125fdc27bce0820e7348eb42842f4373f05f7fc5f3635", + "key": "intel-oneapi-mkl-sycl-dft-2026.1_2026.1.0-236_amd64", + "name": "intel-oneapi-mkl-sycl-dft-2026.1", + "sha256": "31862dd325a31ccacc1e5e9c93140f4c8bd8a0353e0b3658e2b87e329e58d373", "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-dft-2026.0-2026.0.0-908_amd64.deb" + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-dft-2026.1-2026.1.0-236_amd64.deb" ], - "version": "2026.0.0-908" + "version": "2026.1.0-236" }, { "arch": "amd64", "dependencies": [], - "key": "intel-oneapi-mkl-sycl-lapack-2026.0_2026.0.0-908_amd64", - "name": "intel-oneapi-mkl-sycl-lapack-2026.0", - "sha256": "5a31d222fa527c96bd7aa8bbc7819a11417cc7e6ba38803b55e224541aab68c3", + "key": "intel-oneapi-mkl-sycl-lapack-2026.1_2026.1.0-236_amd64", + "name": "intel-oneapi-mkl-sycl-lapack-2026.1", + "sha256": "f701832f5e49b29731c15a27a094122bebe6a1fbd90ddf14c431b7923b8fd921", "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-lapack-2026.0-2026.0.0-908_amd64.deb" + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-lapack-2026.1-2026.1.0-236_amd64.deb" ], - "version": "2026.0.0-908" + "version": "2026.1.0-236" }, { "arch": "amd64", "dependencies": [], - "key": "intel-oneapi-mkl-sycl-rng-2026.0_2026.0.0-908_amd64", - "name": "intel-oneapi-mkl-sycl-rng-2026.0", - "sha256": "ae6910cd6226a9a380df846849288bf07c5eda0387896b90820cf904a29ee4a0", + "key": "intel-oneapi-mkl-sycl-rng-2026.1_2026.1.0-236_amd64", + "name": "intel-oneapi-mkl-sycl-rng-2026.1", + "sha256": "b0748e152f2042dedd1d21c784fb2a8b551c573b7e7ed0361ada2423deb275bf", "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-rng-2026.0-2026.0.0-908_amd64.deb" + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-rng-2026.1-2026.1.0-236_amd64.deb" ], - "version": "2026.0.0-908" + "version": "2026.1.0-236" }, { "arch": "amd64", "dependencies": [], - "key": "intel-oneapi-mkl-sycl-sparse-2026.0_2026.0.0-908_amd64", - "name": "intel-oneapi-mkl-sycl-sparse-2026.0", - "sha256": "1552782efcad141280666f05db5983cfcec4e094a57672170786c7b5e1ff5f9c", + "key": "intel-oneapi-mkl-sycl-sparse-2026.1_2026.1.0-236_amd64", + "name": "intel-oneapi-mkl-sycl-sparse-2026.1", + "sha256": "6a499dc1c5f049ad522a5288151ff0c93f535b8e8ab34edcd773785338e07d45", "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-sparse-2026.0-2026.0.0-908_amd64.deb" + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-sparse-2026.1-2026.1.0-236_amd64.deb" ], - "version": "2026.0.0-908" + "version": "2026.1.0-236" }, { "arch": "amd64", @@ -135,99 +135,99 @@ { "arch": "amd64", "dependencies": [], - "key": "libnl-3-200_3.5.0-0.1_amd64", + "key": "libnl-3-200_3.7.0-0.3build1.1_amd64", "name": "libnl-3-200", - "sha256": "4cd689b817d3e6453aa8f6d91d77d35db36dfc4507ab11e780adaabdc4e0bc35", + "sha256": "15b3f61db753dc3db6f62f4259f8b822299343e500886776486612a13a2ed247", "urls": [ - "https://archive.ubuntu.com/ubuntu/pool/main/libn/libnl3/libnl-3-200_3.5.0-0.1_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20260715T000000Z/pool/main/libn/libnl3/libnl-3-200_3.7.0-0.3build1.1_amd64.deb" ], - "version": "3.5.0-0.1" + "version": "3.7.0-0.3build1.1" }, { "arch": "amd64", "dependencies": [], - "key": "libnl-genl-3-200_3.5.0-0.1_amd64", + "key": "libnl-genl-3-200_3.7.0-0.3build1.1_amd64", "name": "libnl-genl-3-200", - "sha256": "3f73295eee7b9ef208f7272fdb604415778156e4597a25d600563967c15958ea", + "sha256": "9ca0cb63f295d00e6346b9924278c07cfc61311cbd288b805b06e25aa08c9fd8", "urls": [ - "https://archive.ubuntu.com/ubuntu/pool/main/libn/libnl3/libnl-genl-3-200_3.5.0-0.1_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20260715T000000Z/pool/main/libn/libnl3/libnl-genl-3-200_3.7.0-0.3build1.1_amd64.deb" ], - "version": "3.5.0-0.1" + "version": "3.7.0-0.3build1.1" }, { "arch": "amd64", "dependencies": [], - "key": "libnl-route-3-200_3.5.0-0.1_amd64", + "key": "libnl-route-3-200_3.7.0-0.3build1.1_amd64", "name": "libnl-route-3-200", - "sha256": "781291056f30ff840fa99c7c23731836355852889c87f2d530a910e5b4e33abe", + "sha256": "c9dbd29cb7fdcf1a0587726cbc5432a7d3e81242108dfcfa2dd6716c44c74d4d", "urls": [ - "https://archive.ubuntu.com/ubuntu/pool/main/libn/libnl3/libnl-route-3-200_3.5.0-0.1_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20260715T000000Z/pool/main/libn/libnl3/libnl-route-3-200_3.7.0-0.3build1.1_amd64.deb" ], - "version": "3.5.0-0.1" + "version": "3.7.0-0.3build1.1" }, { "arch": "amd64", "dependencies": [], - "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "key": "libnuma1_2.0.18-1ubuntu0.24.04.1_amd64", "name": "libnuma1", - "sha256": "0721c89001fbbd1ada23e89da5d60e762763c1a7b3dc814a2e9a518480a8043d", + "sha256": "f333b8edf6f0b705c19f2a67008194df66f7aab0fa2350dc98510569a033ff29", "urls": [ - "https://archive.ubuntu.com/ubuntu/pool/main/n/numactl/libnuma1_2.0.14-3ubuntu2_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20260715T000000Z/pool/main/n/numactl/libnuma1_2.0.18-1ubuntu0.24.04.1_amd64.deb" ], - "version": "2.0.14-3ubuntu2" + "version": "2.0.18-1ubuntu0.24.04.1" }, { "arch": "amd64", "dependencies": [], - "key": "libigdgmm12_22.10.0-1~24.04~ppa1_amd64", + "key": "libigdgmm12_22.10.0-1_24.04_ppa1_amd64", "name": "libigdgmm12", "sha256": "63fe9e124c3394193226c6cb8398dcabdc9306b5a10f63668118f7c799f5d792", "urls": [ - "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-gmmlib/libigdgmm12_22.10.0-1~24.04~ppa1_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260715T000000Z/pool/main/i/intel-gmmlib/libigdgmm12_22.10.0-1~24.04~ppa1_amd64.deb" ], "version": "22.10.0-1~24.04~ppa1" }, { "arch": "amd64", "dependencies": [], - "key": "libigc2_2.36.3-2~24.04_amd64", + "key": "libigc2_2.36.3-2_24.04_amd64", "name": "libigc2", "sha256": "c11dd6e083d1010c50bbc33cabf5b614e64df53b61d349955e97efdababb85bf", "urls": [ - "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-graphics-compiler/libigc2_2.36.3-2~24.04_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260715T000000Z/pool/main/i/intel-graphics-compiler/libigc2_2.36.3-2~24.04_amd64.deb" ], "version": "2.36.3-2~24.04" }, { "arch": "amd64", "dependencies": [], - "key": "libigdfcl2_2.36.3-2~24.04_amd64", + "key": "libigdfcl2_2.36.3-2_24.04_amd64", "name": "libigdfcl2", "sha256": "019c4d2802b03d5e1ddad888543e6b7b6003a46a14ccf2e16a2320e8923ad007", "urls": [ - "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-graphics-compiler/libigdfcl2_2.36.3-2~24.04_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260715T000000Z/pool/main/i/intel-graphics-compiler/libigdfcl2_2.36.3-2~24.04_amd64.deb" ], "version": "2.36.3-2~24.04" }, { "arch": "amd64", "dependencies": [], - "key": "libze-intel-gpu1_26.22.38646.6-1~24.04~ppa1_amd64", + "key": "libze-intel-gpu1_26.22.38646.6-1_24.04_ppa1_amd64", "name": "libze-intel-gpu1", "sha256": "5d0903aae7ac4f94f43ae7d282a394bc988a0da12f813d92533dea1650eb91ff", "urls": [ - "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-compute-runtime/libze-intel-gpu1_26.22.38646.6-1~24.04~ppa1_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260715T000000Z/pool/main/i/intel-compute-runtime/libze-intel-gpu1_26.22.38646.6-1~24.04~ppa1_amd64.deb" ], "version": "26.22.38646.6-1~24.04~ppa1" }, { "arch": "amd64", "dependencies": [], - "key": "libze1_1.28.6-1~24.04~ppa1_amd64", + "key": "libze1_1.28.6-1_24.04_ppa1_amd64", "name": "libze1", "sha256": "ca4aec5982d46f92875659e74e459372f24c9d8add2810556c1c7e71531e1ec0", "urls": [ - "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/l/level-zero-loader/libze1_1.28.6-1~24.04~ppa1_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260715T000000Z/pool/main/l/level-zero-loader/libze1_1.28.6-1~24.04~ppa1_amd64.deb" ], "version": "1.28.6-1~24.04~ppa1" }, @@ -238,9 +238,10 @@ "name": "zlib1g", "sha256": "7074b6a2f6367a10d280c00a1cb02e74277709180bab4f2491a2f355ab2d6c20", "urls": [ - "https://archive.ubuntu.com/ubuntu/pool/main/z/zlib/zlib1g_1.3.dfsg-3.1ubuntu2.1_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20260715T000000Z/pool/main/z/zlib/zlib1g_1.3.dfsg-3.1ubuntu2.1_amd64.deb" ], "version": "1:1.3.dfsg-3.1ubuntu2.1" } - ] -} + ], + "version": 1 +} \ No newline at end of file diff --git a/platforms/oneapi/packages.yaml b/platforms/oneapi/packages.yaml new file mode 100644 index 00000000..febdda06 --- /dev/null +++ b/platforms/oneapi/packages.yaml @@ -0,0 +1,45 @@ +# +# bazel run @apt_oneapi//:lock +# +version: 1 + +sources: + - channel: all main + url: https://apt.repos.intel.com/oneapi + - channel: noble main + url: https://snapshot.ubuntu.com/ubuntu/20260715T000000Z + - channel: noble-security main + url: https://snapshot.ubuntu.com/ubuntu/20260715T000000Z + - channel: noble-updates main + url: https://snapshot.ubuntu.com/ubuntu/20260715T000000Z + - channel: noble main + url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260715T000000Z + +archs: + - "amd64" + +packages: + # Do not use "intel-oneapi-toolkit" package + # It will install an opaque deb file that will at preinst download the sdk toolkit + - "intel-oneapi-ccl-2022.1" + - "intel-oneapi-mpi-2021.18" + - "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1" + - "intel-oneapi-compiler-shared-runtime-2026.1" + - "intel-oneapi-mkl-core-2026.1" + - "intel-oneapi-mkl-sycl-blas-2026.1" + - "intel-oneapi-mkl-sycl-dft-2026.1" + - "intel-oneapi-mkl-sycl-lapack-2026.1" + - "intel-oneapi-mkl-sycl-rng-2026.1" + - "intel-oneapi-mkl-sycl-sparse-2026.1" + - "intel-oneapi-tcm-1.5" + - "intel-oneapi-umf-1.1" + - "libnl-3-200" + - "libnl-genl-3-200" + - "libnl-route-3-200" + - "libnuma1" + - "libigdgmm12" + - "libigc2" + - "libigdfcl2" + - "libze-intel-gpu1" + - "libze1" + - "zlib1g" diff --git a/third_party/rules_distroless/accept-nospace-version-constraints.patch b/third_party/rules_distroless/accept-nospace-version-constraints.patch new file mode 100644 index 00000000..62973749 --- /dev/null +++ b/third_party/rules_distroless/accept-nospace-version-constraints.patch @@ -0,0 +1,28 @@ +diff --git a/apt/private/version_constraint.bzl b/apt/private/version_constraint.bzl +index 531fa01..b3a3d30 100644 +--- a/apt/private/version_constraint.bzl ++++ b/apt/private/version_constraint.bzl +@@ -2,11 +2,19 @@ + + load(":version.bzl", version_lib = "version") + ++_VERSION_CONSTRAINT_OPERATORS = ["<<", ">>", "<=", ">=", "="] ++ + def _parse_version_constraint(rawv): +- vconst_i = rawv.find(" ") +- if vconst_i == -1: +- fail('invalid version string %s expected a version constraint ">=", "=", ">=", "<<", ">>"' % rawv) +- return (rawv[:vconst_i], rawv[vconst_i + 1:]) ++ rawv = rawv.strip() ++ for op in _VERSION_CONSTRAINT_OPERATORS: ++ if rawv.startswith(op): ++ version = rawv[len(op):].strip() ++ if version: ++ return (op, version) ++ ++ break ++ ++ fail('invalid version string %s expected a version constraint ">=", "=", "<=", "<<", ">>"' % rawv) + + def _parse_dep(raw): + raw = raw.strip() # remove leading & trailing whitespace From 2e1a07e7f17cb822c815ff9e2b39d9284be1443b Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 15 Jul 2026 16:43:55 +0200 Subject: [PATCH 578/881] stdx/json: fix potential invalid free (#631) When we parsed from a std.json.Value we weren't correctly duping the strings, potentially leading to invalid free. --- stdx/json.zig | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/stdx/json.zig b/stdx/json.zig index a8bff835..dd85eb95 100644 --- a/stdx/json.zig +++ b/stdx/json.zig @@ -45,8 +45,14 @@ pub fn Union(comptime T: type) type { switch (field.type) { bool => if (source == .bool) return .{ .value = @unionInit(T, field.name, source.bool) }, []const u8 => switch (source) { - .string => |v| return .{ .value = @unionInit(T, field.name, v) }, - .number_string => |v| return .{ .value = @unionInit(T, field.name, v) }, + .string => |v| { + const dupe = try allocator.dupe(u8, v); + return .{ .value = @unionInit(T, field.name, dupe) }; + }, + .number_string => |v| { + const dupe = try allocator.dupe(u8, v); + return .{ .value = @unionInit(T, field.name, dupe) }; + }, else => {}, }, else => switch (@typeInfo(field.type)) { @@ -239,6 +245,32 @@ test "union" { try std.testing.expectError(error.UnknownField, Union(Answer).jsonParseFromValue(allocator, null_value, .{})); } +test "strings in union can be freed" { + const Answer = union(enum) { + string: []const u8, + number: u32, + }; + const Struct = struct { + answer1: Union(Answer), + answer2: Union(Answer), + }; + const struct_json = + \\{"answer1":"I'm 29","answer2": 1e29} + ; + var parsed_value = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, struct_json, .{ .parse_numbers = false }); + defer parsed_value.deinit(); + + const parsed = try std.json.parseFromValueLeaky(Struct, std.testing.allocator, parsed_value.value, .{}); + + try std.testing.expect(parsed.answer1.value == .string); + try std.testing.expectEqualStrings("I'm 29", parsed.answer1.value.string); + std.testing.allocator.free(parsed.answer1.value.string); + + try std.testing.expect(parsed.answer2.value == .string); + try std.testing.expectEqualStrings("1e29", parsed.answer2.value.string); + std.testing.allocator.free(parsed.answer2.value.string); +} + test "tagged union" { const Entry = union(enum) { faq: struct { From 2e05c383e7e624613a6d053145333f051e4e0b6f Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 16 Jul 2026 10:11:08 +0200 Subject: [PATCH 579/881] zml: cleanup attention code (#653) --- zml/attention/attention.zig | 55 ++++++++++--------------------- zml/attention/flashattn.zig | 12 +++---- zml/attention/metal_attention.zig | 32 ++++++++++++++++++ zml/attention/tpu_attention.zig | 24 +++++--------- zml/nn.zig | 9 ----- 5 files changed, 63 insertions(+), 69 deletions(-) diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index 8752763a..e904bc3d 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -3,6 +3,7 @@ const std = @import("std"); const zml = @import("../zml.zig"); const attnd = @import("attnd.zig"); const flashattn = @import("flashattn.zig"); +const metal = @import("metal_attention.zig"); const nki = @import("nki/attention.zig"); const Attention = @This(); @@ -76,31 +77,13 @@ pub const Parameters = union(Backend) { } }; -pub const MetalFaMetadata = struct { - num_tokens: zml.Tensor, - - pub fn init() MetalFaMetadata { - return .{ .num_tokens = .fromShape(.scalar(.u32)) }; - } - - pub fn initBuffer(self: MetalFaMetadata, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(MetalFaMetadata) { - _ = self; - _ = sharding; - return .{ .num_tokens = try zml.Buffer.scalar(io, platform, 0, .u32) }; - } - - pub fn deinitBuffer(self: *zml.Bufferized(MetalFaMetadata)) void { - self.num_tokens.deinit(); - } -}; - pub const Metadata = union(Backend) { vanilla: void, attnd: attnd.Metadata, nki: void, cuda_fa2: flashattn.fa2.Metadata, cuda_fa3: flashattn.fa3.Metadata, - metal_fa: MetalFaMetadata, + metal_fa: metal.Metadata, pub const InitOptions = union(Backend) { vanilla: void, @@ -136,11 +119,8 @@ pub const Metadata = union(Backend) { pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(Metadata) { return switch (self) { .vanilla => .{ .vanilla = {} }, - .attnd => |v| .{ .attnd = try v.initBuffer(io, platform, sharding) }, .nki => .{ .nki = {} }, - .cuda_fa2 => |v| .{ .cuda_fa2 = try v.initBuffer(io, platform, sharding) }, - .cuda_fa3 => |v| .{ .cuda_fa3 = try v.initBuffer(io, platform, sharding) }, - .metal_fa => |v| .{ .metal_fa = try v.initBuffer(io, platform, sharding) }, + inline else => |v, tag| @unionInit(zml.Bufferized(Metadata), @tagName(tag), try v.initBuffer(io, platform, sharding)), }; } @@ -151,11 +131,22 @@ pub const Metadata = union(Backend) { .nki => {}, .cuda_fa2 => |*v| flashattn.fa2.Metadata.deinitBuffer(v), .cuda_fa3 => |*v| flashattn.fa3.Metadata.deinitBuffer(v), - .metal_fa => |*v| MetalFaMetadata.deinitBuffer(v), + .metal_fa => |*v| metal.Metadata.deinitBuffer(v), } } }; +/// Causal attention as used in transformers. +/// +/// **Shapes**: +/// - q, result: .{ .q, .h, .hd } +/// - k, v: .{ .k, .h, .hd } +/// +/// Where: +/// - .h is the number of head +/// - .q is the number of queries +/// - .k is the number of keys +/// - .hd is the head dimension pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { return switch (parameters) { .vanilla => b: { @@ -166,25 +157,13 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T // Note: in Pytorch it would be very inefficient to generate the full attn_mask, // then slice into it, but XLA is able to optimize this correctly. attn_mask = attn_mask.gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, attn_mask.dtype()), token_index.reshape(.{ .coord = 1 }), .{}); - const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask, .allow_cudnn = true }); + const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask }); break :b attn_output; }, .attnd => attnd.causalAttention(q, k, v, token_index, metadata.attnd, parameters.attnd), .nki => |params| nki.attention(q, k, v, token_index, params), .cuda_fa2 => flashattn.fa2.attention(q, k, v, token_index, metadata.cuda_fa2, parameters.cuda_fa2), .cuda_fa3 => flashattn.fa3.attention(q, k, v, token_index, metadata.cuda_fa3, parameters.cuda_fa3), - .metal_fa => b: { - const qc = q.transpose(.{ .h, .q, .hd }); - const kc = k.transpose(.{ .h, .k, .hd }); - const vc = v.transpose(.{ .h, .k, .hd }); - const tok_i32 = token_index.convert(.i32); - - const attn = if (q.dim(.q) > 1) - zml.ops.customCall("zml$flash_attn", .{ qc, kc, vc, tok_i32, metadata.metal_fa.num_tokens }, qc.shape(), .{}, .{ .has_side_effect = false }) - else - zml.ops.customCall("zml$flash_attn", .{ qc, kc, vc, tok_i32 }, qc.shape(), .{}, .{ .has_side_effect = false }); - - break :b attn.transpose(q.shape()); - }, + .metal_fa => metal.attention(q, k, v, token_index, metadata.metal_fa), }; } diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 03234a64..dbc35f66 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -988,7 +988,7 @@ pub const paged_fa2 = struct { .hg = num_head_groups, .b = q.dim(.b), }, .f32)).withPartitioning(.{ .hkv = .model }); - const dummy_cu_seqlens_k = zml.Tensor.constant(zml.DataType.i32.zero()).broad(cu_seqlens_q.shape()); + const dummy_cu_seqlens_k: zml.Tensor = .zeroes(cu_seqlens_q.shape()); const batch_dim = q.dim(.b); var q2 = q; @@ -1068,7 +1068,7 @@ pub const paged_fa2 = struct { .hg = num_head_groups, .b = q.dim(.b), }, .f32)).withPartitioning(.{ .hkv = .model }); - const dummy_cu_seqlens_k_prefill = zml.Tensor.constant(zml.DataType.i32.zero()).broad(cu_seqlens_q_prefill.shape()); + const dummy_cu_seqlens_k_prefill: zml.Tensor = .zeroes(cu_seqlens_q_prefill.shape()); var q2 = q; q2 = q2.transpose(.{ .b, .hkv, .hg, .hd }).merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); @@ -1129,7 +1129,7 @@ pub const paged_fa2 = struct { .hg = num_head_groups, .b = batch_dim_decode, }, .f32)).withPartitioning(.{ .hkv = .model }); - const dummy_cu_seqlens_k_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(cu_seqlens_q_decode.shape()); + const dummy_cu_seqlens_k_decode = zml.Tensor.zeroes(cu_seqlens_q_decode.shape()); var q_decode = q.dynamicSlice1d(0, .{ .start = mixed_parameters.metadata.decode_offset, .len = batch_dim_decode }); if (seqlenq_ngroups_swapped) { @@ -1673,7 +1673,7 @@ pub const paged_fa3 = struct { .hkv = num_kv_heads, .hg = num_head_groups, }, .f32)).withPartitioning(.{ .hkv = .model }); - const scheduler_metadata = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = batch_size + 1 }, .i32)).withPartitioning(.{ .b = .replicated }); + const scheduler_metadata = zml.Tensor.zeroes(.init(.{ .b = batch_size + 1 }, .i32)).withPartitioning(.{ .b = .replicated }); var q2 = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); @@ -1741,7 +1741,7 @@ pub const paged_fa3 = struct { .hkv = num_kv_heads, .hg = num_head_groups, }, .f32)).withPartitioning(.{ .hkv = .model }); - const scheduler_metadata_prefill = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = batch_size_prefill + 1 }, .i32)).withPartitioning(.{ .b = .replicated }); + const scheduler_metadata_prefill = zml.Tensor.zeroes(.init(.{ .b = batch_size_prefill + 1 }, .i32)).withPartitioning(.{ .b = .replicated }); var q2 = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); @@ -1797,7 +1797,7 @@ pub const paged_fa3 = struct { .hkv = num_kv_heads, .hg = num_head_groups, }, .f32)); - const scheduler_metadata_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = batch_size_decode + 1 }, .i32)).withPartitioning(.{ .b = .replicated }); + const scheduler_metadata_decode = zml.Tensor.zeroes(.init(.{ .b = batch_size_decode + 1 }, .i32)).withPartitioning(.{ .b = .replicated }); var q_decode = q.dynamicSlice1d(0, .{ .start = mixed_parameters.metadata.decode_offset, .len = batch_size_decode }).withPartitioning(.{ .hkv = .model }); q_decode = q_decode.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); diff --git a/zml/attention/metal_attention.zig b/zml/attention/metal_attention.zig index f5de61bf..17f9de62 100644 --- a/zml/attention/metal_attention.zig +++ b/zml/attention/metal_attention.zig @@ -7,6 +7,38 @@ const AttentionOptions = @import("paged_attention.zig").AttentionOptions; const log = std.log.scoped(.@"zml/attention/metal"); +pub const Metadata = struct { + num_tokens: zml.Tensor, + + pub fn init() Metadata { + return .{ .num_tokens = .fromShape(.scalar(.u32)) }; + } + + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(Metadata) { + _ = self; + _ = sharding; + return .{ .num_tokens = try zml.Buffer.scalar(io, platform, 0, .u32) }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { + self.num_tokens.deinit(); + } +}; + +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata) zml.Tensor { + const qc = q.transpose(.{ .h, .q, .hd }); + const kc = k.transpose(.{ .h, .k, .hd }); + const vc = v.transpose(.{ .h, .k, .hd }); + const tok_i32 = token_index.convert(.i32); + + const attn = if (q.dim(.q) > 1) + zml.ops.customCall("zml$flash_attn", .{ qc, kc, vc, tok_i32, metadata.num_tokens }, qc.shape(), .{}, .{ .has_side_effect = false }) + else + zml.ops.customCall("zml$flash_attn", .{ qc, kc, vc, tok_i32 }, qc.shape(), .{}, .{ .has_side_effect = false }); + + return attn.transpose(q.shape()); +} + pub const paged = struct { pub const Options = struct { batch_size: usize, diff --git a/zml/attention/tpu_attention.zig b/zml/attention/tpu_attention.zig index ea3b7eb6..7ce5c93c 100644 --- a/zml/attention/tpu_attention.zig +++ b/zml/attention/tpu_attention.zig @@ -98,12 +98,6 @@ pub const mosaic_tpu = struct { } }; - fn kernelFrontendAttrs(comp_ctx: *CompilationContext) *const mlir.Attribute { - return .dict(comp_ctx.mlir_ctx, &.{ - mlir.NamedAttribute.named(comp_ctx.mlir_ctx, "kernel_metadata", .string(comp_ctx.mlir_ctx, "{}")), - }); - } - fn raggedPagedKernelCall( q: zml.Tensor, kv_pages: zml.Tensor, @@ -113,20 +107,13 @@ pub const mosaic_tpu = struct { num_seqs: zml.Tensor, cfg: ragged_paged.Cfg, ) zml.Tensor { - const seq_buf_idx = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{2}, .i32)); - const comp_ctx = CompilationContext.current(); - - const additional_attrs = [_]mlir.NamedAttribute{ - .named(comp_ctx.mlir_ctx, "kernel_name", .string(comp_ctx.mlir_ctx, "ragged_paged_attention_kernel")), - .named(comp_ctx.mlir_ctx, "mhlo.frontend_attributes", kernelFrontendAttrs(comp_ctx)), - }; - + const mlir_ctx = CompilationContext.current().mlir_ctx; const out = ragged_attention.Kernel.call( .{ .kv_lens = seq_lens, .page_indices = block_table, .cu_q_lens = query_start_len, - .seq_buf_idx = seq_buf_idx, + .seq_buf_idx = .zeroes(.init(.{2}, .i32)), .num_seqs = num_seqs, .q = q, .kv_pages = kv_pages, @@ -136,7 +123,12 @@ pub const mosaic_tpu = struct { .cfg = cfg, .extras = .{ .vmem_limit_bytes = cfg.vmem_limit_bytes, - .additional_attributes = &additional_attrs, + .additional_attributes = &.{ + .named(mlir_ctx, "kernel_name", .string(mlir_ctx, "ragged_paged_attention_kernel")), + .named(mlir_ctx, "mhlo.frontend_attributes", .dict(mlir_ctx, &.{ + .named(mlir_ctx, "kernel_metadata", .string(mlir_ctx, "{}")), + })), + }, }, }, ); diff --git a/zml/nn.zig b/zml/nn.zig index f99be197..00636385 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -1171,9 +1171,6 @@ pub const SdpaOpts = struct { attn_mask: ?Tensor = null, scale: ?Tensor = null, softmax_bias: ?Tensor = null, - allow_cudnn: bool = true, - // TODO: put a callback instead of all this field, - // so that }; /// Scaled dot product attention. @@ -1199,11 +1196,6 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { stdx.debug.assert(k.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "k is missing tags {{.h, .k, .hd}}", err_args); stdx.debug.assert(v.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "v is missing tags {{.h, .k, .hd}}", err_args); - // TODO(Corentin): Re-enable that - //if (opts.allow_cudnn and cuda.canUseCudnnSdpa(q.shape()) and opts.softmax_bias == null) { - // return cuda.sdpa(q, k, v, opts); - //} - // Handle different numbers of head by splitting q heads. // This is a bit error prone in the sense that it depends of the layout of q heads. // This is the Llama convention though. @@ -1218,7 +1210,6 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { k = k.mul(head_scaling.convert(k.dtype())); var attn_weights = q.dot(k, .hd); - // log.debug("attn_weights : {f}, attn_mask : {?f}", .{ attn_weights, attn_mask }); if (attn_mask) |mask| attn_weights = attn_weights.add(mask.broad(attn_weights.shape())); attn_weights = attn_weights.convert(.f32); attn_weights = if (opts.softmax_bias) |softmax_bias| attn: { From 1c65836a442afc2b2e2f576a79d3b6c3e5bf469f Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 16 Jul 2026 17:44:50 +0200 Subject: [PATCH 580/881] pjrt: better error message on dlopen failure (#657) I hit an issue with the plugin refusing to load on macos. There was no information. After adding this I saw that the the file was apparently not existing in the bazel sandbox, so I could reduce the bug to a Bazel issue. --- pjrt/pjrt.zig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 6c112490..196413d0 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -111,20 +111,20 @@ pub const Api = struct { break :blk .{ .inner = .{ .handle = std.c.dlopen(library, rtld) orelse { - log.err("Unable to dlopen plugin: {s}", .{library}); + log.err("Unable to dlopen plugin {s}\n{s}", .{ library, std.c.dlerror() orelse "" }); return error.FileNotFound; }, }, }; }, else => std.DynLib.open(library) catch |err| { - log.err("Unable to dlopen plugin: {s}", .{library}); + log.err("Unable to dlopen plugin {s}: {}", .{ library, err }); return err; }, }; const api = fromDynLib(&lib) catch |err| { - log.err("Unable to load PJRT API from plugin: {s}: {}", .{ library, err }); + log.err("Unable to load PJRT API from plugin {s}: {}", .{ library, err }); return err; }; log.info("Loaded: {s}", .{basename}); From f29ed52ea8fb060f3893bf0714c4742c63365cee Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Fri, 17 Jul 2026 11:18:53 +0200 Subject: [PATCH 581/881] platforms/oneapi: 4 GPUs fixes (#661) Co-authored-by: Kevin Barre --- platforms/oneapi/oneapi.bzl | 4 ++-- platforms/oneapi/oneapi.zig | 22 ++++++++++++---------- zml/module.zig | 8 +++++++- zml/platform.zig | 8 +------- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index 22561d9e..be022b15 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -1,8 +1,8 @@ load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") -PJRT_ONEAPI_RELEASE = "manual-2026-07-03T15-19-00Z" -PJRT_ONEAPI_ARTIFACT_SHA256 = "dece3736b9521485885770c82f335a5b13dbcf61f53cce7ac9f6c63226eb5293" +PJRT_ONEAPI_RELEASE = "manual-2026-07-17T10-00-00Z" +PJRT_ONEAPI_ARTIFACT_SHA256 = "7b2883157ae108c09003c9d63a8a5d638adfbf9c511517b766c4cdbf4276b1bd" PJRT_ONEAPI_ARTIFACT_URL = "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-oneapi_linux-amd64.tar.gz".format( release = PJRT_ONEAPI_RELEASE, ) diff --git a/platforms/oneapi/oneapi.zig b/platforms/oneapi/oneapi.zig index f1175b69..0e411fbf 100644 --- a/platforms/oneapi/oneapi.zig +++ b/platforms/oneapi/oneapi.zig @@ -13,26 +13,29 @@ pub fn isEnabled() bool { return @hasDecl(c, "ZML_RUNTIME_ONEAPI"); } -fn countOneApiDevices(io: std.Io) usize { - var dir = std.Io.Dir.openDirAbsolute(io, "/dev/dri", .{ .iterate = true }) catch return 0; +fn hasOneApiDevice(io: std.Io) bool { + var dir = std.Io.Dir.openDirAbsolute(io, "/dev/dri", .{ .iterate = true }) catch return false; defer dir.close(io); - var count: usize = 0; var it = dir.iterate(); while (it.next(io) catch null) |entry| { if (std.mem.startsWith(u8, entry.name, "renderD")) { - count += 1; + return true; } } - return count; + return false; } -fn setupOneAPIEnv(driver_path: [:0]const u8, device_count: usize) void { +fn setupOneAPIEnv(driver_path: [:0]const u8) void { _ = c.setenv("CCL_LOG_LEVEL", c.getenv("CCL_LOG_LEVEL") orelse "error", 1); _ = c.setenv("CCL_ATL_TRANSPORT", "ofi", 1); _ = c.setenv("FI_PROVIDER", "shm", 1); - _ = c.setenv("ONEAPI_DEVICE_SELECTOR", if (device_count > 1) "level_zero:0,1" else "level_zero:0", 0); + // collective-permute through oneccl recv/send often relies on sycl d2d memcpy underneath, + // that doesn't work because of peer residency likely this: + // https://github.com/intel/compute-runtime/issues/953 + _ = c.setenv("XLA_ONECCL_COLLECTIVE_PERMUTE_BYPASS_SYCL_P2P", "1", 1); + _ = c.setenv("ONEAPI_DEVICE_SELECTOR", "level_zero:*", 0); _ = c.setenv("ZE_ENABLE_ALT_DRIVERS", driver_path, 0); } @@ -45,8 +48,7 @@ pub fn load(_: std.mem.Allocator, io: std.Io) !*const pjrt.Api { return error.Unavailable; } - const device_count = countOneApiDevices(io); - if (device_count == 0) { + if (!hasOneApiDevice(io)) { return error.Unavailable; } @@ -60,7 +62,7 @@ pub fn load(_: std.mem.Allocator, io: std.Io) !*const pjrt.Api { var driver_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const driver_path = try stdx.Io.Dir.path.bufJoinZ(&driver_path_buf, &.{ sandbox_path, "lib", "libze_intel_gpu.so.1" }); - setupOneAPIEnv(driver_path, device_count); + setupOneAPIEnv(driver_path); return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; diff --git a/zml/module.zig b/zml/module.zig index 3c309173..e0d58ae9 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -660,9 +660,15 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "CUBLAS,CUBLASLT,CUSTOM_CALL,CUDNN,DYNAMIC_SLICE_FUSION", upb_arena); }, .oneapi => { + // More efficient for the allgather/broadcast implementation of the collective permute. + try setXlaOverrideFlag(overrides_map, "xla_gpu_collective_permute_connected_components", true, upb_arena); try setXlaOverrideFlag(overrides_map, "xla_gpu_autotune_level", 0, upb_arena); try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "", upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_cublaslt", false, upb_arena); + + // Not supported by OneAPI + try setXlaOverrideFlag(overrides_map, "xla_disable_hlo_passes", "scan-rewriter", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_ragged_dot_grouped_gemm", false, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_cub_radix_sort", false, upb_arena); }, else => {}, } diff --git a/zml/platform.zig b/zml/platform.zig index 5ef97718..4a4e2fb6 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -40,18 +40,12 @@ fn validateDeviceCount(target: Target, num_devices: usize) !void { return error.MissingDevices; } switch (target) { - .cpu, .cuda, .rocm, .tpu, .neuron, .metal => { + .cpu, .cuda, .rocm, .tpu, .neuron, .metal, .oneapi => { if (!std.math.isPowerOfTwo(num_devices)) { log.err("Platform {} requires a power-of-two device count, got {}", .{ target, num_devices }); return error.InvalidDeviceCount; } }, - .oneapi => { - if (num_devices > 2) { - log.err("Platform Intel is limited to a maximum of 2 devices, got {}", .{num_devices}); - return error.InvalidDeviceCount; - } - }, } } From 787d8cafa3dd35fec51365527a37d9450921b86c Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 17 Jul 2026 12:21:19 +0200 Subject: [PATCH 582/881] zml: handle passthroug in zml.nn.rope (#662) alternative to #660 --------- Co-authored-by: Corentin Godeau --- examples/llm/models/lfm2/model.zig | 2 +- examples/llm/models/llama/model.zig | 2 +- examples/llm/models/qwen3_5/model.zig | 2 +- examples/llm/models/qwen3_5_moe/model.zig | 2 +- zml/nn.zig | 317 +++++++++++++++------- zml/tensor.zig | 6 +- 6 files changed, 222 insertions(+), 109 deletions(-) diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index 3d25c3cf..91da6aab 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -481,7 +481,7 @@ pub const Attention = struct { .k_layernorm = RmsNorm.init(store.withPrefix("k_layernorm"), config.norm_eps, .hd), .head_dim = head_dim, .num_key_value_groups = num_key_value_groups, - .rope_opts = .{ .layout = .sequential, .scaling = .{ .default = .{ .rope_theta = config.rope_theta } } }, + .rope_opts = .{ .layout = .real_im_pass, .scaling = .{ .default = .{ .rope_theta = config.rope_theta } } }, }; } diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index ad0b1f8a..0874598a 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -467,7 +467,7 @@ const SelfAttn = struct { .num_heads = @intCast(config.num_attention_heads), .num_kv_heads = @intCast(config.num_key_value_heads), .rope_opts = .{ - .layout = if (config.hf_rope_impl) .sequential else .interleaved, + .layout = if (config.hf_rope_impl) .real_im_pass else .interleaved, .scaling = rope_scaling, }, }; diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 59fd2c8c..5726c24d 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -625,7 +625,7 @@ pub const TextRotaryEmbedding = struct { pub fn init(rotary_dim: i64, theta: f32, mrope_section: [3]i64) TextRotaryEmbedding { return .{ .rope_opts = .{ - .layout = .sequential, + .layout = .real_im_pass, .scaling = .{ .default = .{ .rope_theta = theta } }, }, .rotary_dim = rotary_dim, diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 7b0d4a81..8d356da3 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -784,7 +784,7 @@ pub const TextRotaryEmbedding = struct { pub fn init(rotary_dim: i64, theta: f32, mrope_section: [3]i64) TextRotaryEmbedding { return .{ .rope_opts = .{ - .layout = .sequential, + .layout = .real_im_pass, .scaling = .{ .default = .{ .rope_theta = theta } }, }, .rotary_dim = rotary_dim, diff --git a/zml/nn.zig b/zml/nn.zig index 00636385..f893a2cf 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -141,14 +141,18 @@ test normalizeL2 { } pub const RopeOpts = struct { - layout: Layout = .sequential, + layout: Layout = .real_im_pass, scaling: Scaling = .{ .default = .{} }, - /// There are two layouts corresponding to how to split `x` in real/imag parts. - /// * Interleaved means that the real/imag of each scalar is contiguous. - /// * Sequential means that you first read all real values then all imag values. - /// Typically HF models use sequential, while GGUF use interleaved. - pub const Layout = enum { interleaved, sequential }; + /// There are 3 layouts corresponding to how to split `x` in real/imag/passthrough parts. + /// The hard part is that HF models don't specify the layout they use. + /// But HF models don't use the `interleaved` format, + /// and most models don't have a Rope passthrough, so real_im_pass and real_pass_im_pass are the same. + /// + /// * real_im_pass: [x_re0, x_re1, ..., x_im0, x_im1, ..., p_re0, p_re1, ..., p_im0, p_im1, ...] + /// * real_pass_im_pass: [x_re0, x_re1, ..., p_re0, p_re1, ..., x_im0, x_im1, ..., p_im0, p_im1, ...] + /// * interleaved: : [x_re0, x_im0, x_re1, x_im1, ..., p_re0, p_im0, p_re1, p_im1, ...] + pub const Layout = enum { real_im_pass, real_pass_im_pass, interleaved }; /// There are several ways to init the scaling aka "inv_freq" pub const Scaling = union(enum) { @@ -182,6 +186,7 @@ pub const RopeOpts = struct { rope_theta: f32 = 10000, llama_4_scaling_beta: ?f32 = null, attention_factor: ?f32 = null, + partial_rotary_factor: f32 = 1.0, }; pub const Linear = struct { @@ -190,7 +195,7 @@ pub const RopeOpts = struct { }; pub const Proportional = struct { - partial_rotary_factor: f32, + partial_rotary_factor: f32 = 1.0, rope_theta: f32 = 10000, }; @@ -265,6 +270,15 @@ pub const RopeOpts = struct { inline else => |s| s.rope_theta, }; } + + pub fn partialRotaryDim(self: *const Scaling, head_dim: i64) u32 { + return switch (self.*) { + inline .yarn, .proportional => |s| @intFromFloat(@floor( + @as(f32, @floatFromInt(head_dim)) * std.math.clamp(s.partial_rotary_factor, 0.0, 1.0), + )), + .default, .llama3, .linear => @intCast(head_dim), + }; + } }; }; @@ -278,7 +292,8 @@ pub const RopeOpts = struct { /// - pos_idx: optional tensor which indicates which positions are needed. /// When not set `rope` return all positions from 0 to x.dim(.s) which is the max seq len. pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { - stdx.debug.assert(@mod(x.dim(.hd), 2) == 0, "rope expects a even head dim (.hd), got {f}", .{x}); + const head_dim = x.dim(.hd); + stdx.debug.assert(@mod(head_dim, 2) == 0, "rope expects a even head dim (.hd), got {f}", .{x}); const idx = if (pos_idx) |idx| blk: { stdx.debug.assert(x.shape().hasTags(.{.hd}), "rope expects x argument to have .hd axes got: rope(x={f}, idx={f})", .{ x, idx }); @@ -287,11 +302,15 @@ pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { stdx.debug.assert(x.shape().hasTags(.{ .s, .hd }), "rope expects x argument to have both .s and .hd axes got: rope(x={f})", .{x}); break :blk Tensor.arange(.{ .end = x.dim(.s) }, .f32).withTags(.{.s}); }; - const x_real, const x_imag = splitRealImg(x, opts.layout); + + const rotary_dim: u32 = opts.scaling.partialRotaryDim(head_dim); + stdx.debug.assert(rotary_dim > 0 and @mod(rotary_dim, 2) == 0, "partial rope expects a even head dim (.hd), got {d}", .{rotary_dim}); + + const x_real, const x_imag, const x_pass = zml.nn.splitRealImgPass(x, opts.layout, rotary_dim); + const inv_freq = invFreq(head_dim, opts).withTags(.{.hd}); // compute sin and cos in f32 before downcasting to x type. - const inv_freq = invFreq(x.dim(.hd), opts).withTags(.{.hd}); - const inv_freq_pos = Tensor.outer(idx.convert(.f32), inv_freq); + const inv_freq_pos = zml.Tensor.outer(idx.convert(.f32), inv_freq); const scaling = opts.scaling.attentionScaling(); const cos = inv_freq_pos.cos().scale(scaling).convert(x.dtype()).broad(x_real.shape()); const sin = inv_freq_pos.sin().scale(scaling).convert(x.dtype()).broad(x_real.shape()); @@ -300,15 +319,14 @@ pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { const y_real = x_real.mul(cos).sub(x_imag.mul(sin)); const y_imag = x_real.mul(sin).add(x_imag.mul(cos)); - // flatten last dimensions - return mergeRealImg(y_real, y_imag, opts.layout); + return zml.nn.mergeRealImgPass(y_real, y_imag, x_pass, opts.layout); } pub fn splitRealImg(x: Tensor, layout: RopeOpts.Layout) [2]Tensor { const n = x.dim(-1); return switch (layout) { - .sequential => .{ + .real_im_pass, .real_pass_im_pass => .{ x.slice1d(-1, .{ .end = @divExact(n, 2) }), x.slice1d(-1, .{ .start = @divExact(n, 2), .end = n }), }, @@ -319,29 +337,84 @@ pub fn splitRealImg(x: Tensor, layout: RopeOpts.Layout) [2]Tensor { }; } +const Pass = union(RopeOpts.Layout) { + real_im_pass: Tensor, + real_pass_im_pass: [2]Tensor, + interleaved: Tensor, +}; + +pub fn splitRealImgPass(x: Tensor, layout: RopeOpts.Layout, rotary_dim: u32) struct { Tensor, Tensor, ?Pass } { + const ax = x.axis(.hd); + const hd = x.dim(ax); + if (rotary_dim == hd) { + const real, const img = splitRealImg(x, layout); + return .{ real, img, null }; + } + + const half = @divExact(x.dim(ax), 2); + const half_rotary = @divExact(rotary_dim, 2); + return switch (layout) { + .real_im_pass => .{ + x.slice1d(ax, .{ .end = half_rotary }), + x.slice1d(ax, .{ .start = half_rotary, .end = rotary_dim }), + .{ .real_im_pass = x.slice1d(ax, .{ .start = rotary_dim }) }, + }, + .real_pass_im_pass => .{ + x.slice1d(ax, .{ .end = half_rotary }), + x.slice1d(ax, .{ .start = half, .end = half + half_rotary }), + .{ .real_pass_im_pass = .{ + x.slice1d(ax, .{ .start = half_rotary, .end = half }), + x.slice1d(ax, .{ .start = half + half_rotary }), + } }, + }, + .interleaved => .{ + x.slice1d(ax, .{ .start = 0, .end = rotary_dim, .step = 2 }), + x.slice1d(ax, .{ .start = 1, .end = rotary_dim, .step = 2 }), + .{ .interleaved = x.slice1d(ax, .{ .start = rotary_dim }) }, + }, + }; +} + pub fn mergeRealImg(x_real: Tensor, x_imag: Tensor, layout: RopeOpts.Layout) Tensor { return switch (layout) { - .sequential => Tensor.concatenate(&.{ x_real, x_imag }, -1), - .interleaved => Tensor.concatenate(&.{ - x_real.appendAxes(.{.interleaved_real_img}), - x_imag.appendAxes(.{.interleaved_real_img}), - }, -1).reshape(x_imag.shape().setDim(-1, -1)), + .real_im_pass, .real_pass_im_pass => .concatenate(&.{ x_real, x_imag }, -1), + .interleaved => Tensor.stack(&.{ x_real, x_imag }, .last, .interleaved_real_img) + .merge(.{ .hd = .{ .hd, .interleaved_real_img } }), + }; +} + +pub fn mergeRealImgPass(x_real: Tensor, x_imag: Tensor, x_pass: ?Pass, layout: RopeOpts.Layout) Tensor { + if (x_pass == null) return mergeRealImg(x_real, x_imag, layout); + + return switch (x_pass.?) { + .real_im_pass => |pass| .concatenate(&.{ x_real, x_imag, pass }, .hd), + .real_pass_im_pass => |pass| .concatenate(&.{ x_real, pass[0], x_imag, pass[1] }, .hd), + .interleaved => |pass| .concatenate( + &.{ + Tensor.stack(&.{ x_real, x_imag }, .last, .interleaved_real_img) + .merge(.{ .hd = .{ .hd, .interleaved_real_img } }), + pass, + }, + .hd, + ), }; } /// {exp( - n * ln(10_000) / N ) | n in [0..N] } pub fn invFreq(N: i64, opts: RopeOpts) Tensor { const allocator = zml.module.CompilationContext.current().allocator; - const N_half: usize = @intCast(@divExact(N, 2)); - const inv_freq = allocator.alloc(f32, N_half) catch @panic("OOM"); + const N_half: u32 = @intCast(@divExact(N, 2)); + const num_freqs: u32 = opts.scaling.partialRotaryDim(N_half); + + const inv_freq = allocator.alloc(f32, num_freqs) catch @panic("OOM"); defer allocator.free(inv_freq); - _invFreq(opts, inv_freq); - return zml.Tensor.constantTensor(zml.Shape.init(.{@divExact(N, 2)}, .f32), std.mem.sliceAsBytes(inv_freq)); + _invFreq(opts, inv_freq, N_half); + return .constantTensor(.init(.{num_freqs}, .f32), @ptrCast(inv_freq)); } -fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { - const N = inv_freq.len; +fn _invFreq(opts: RopeOpts, inv_freq: []f32, half_head_dim: u32) void { + const N: u32 = @intCast(inv_freq.len); // Default frequencies for (0.., inv_freq) |n, *f| { f.* = @exp(-@log(opts.scaling.getRopeTheta()) * stdx.math.divFloat(f32, n, N)); @@ -349,16 +422,16 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { switch (opts.scaling) { .default => {}, + .proportional => { + // proportional uses the original number of dimensions, + // instead of the actual number of rotated dims. + for (0.., inv_freq[0..]) |n, *f| { + f.* = @exp(-@log(opts.scaling.getRopeTheta()) * stdx.math.divFloat(f32, n, half_head_dim)); + } + }, .linear => |l| { for (inv_freq) |*f| f.* /= l.factor; }, - .proportional => |p| { - const rotary_fraction = std.math.clamp(p.partial_rotary_factor, 0.0, 1.0); - const rotary_frequencies: usize = @intFromFloat(@floor(@as(f32, @floatFromInt(N)) * rotary_fraction)); - for (inv_freq[rotary_frequencies..]) |*f| { - f.* = 0; - } - }, .llama3 => |s| { // https://arxiv.org/pdf/2309.16039 // After Llama2 they observed that the rope frequencies where too sharp and hurting long distance attention. @@ -383,7 +456,7 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { } }, .yarn => |s| { - const N_f: f64 = @floatFromInt(inv_freq.len); + const N_f: f64 = @floatFromInt(N); const M: f64 = @floatFromInt(s.original_max_position_embeddings); const f_high = s.beta_fast * (2 * std.math.pi) / M; const f_low = s.beta_slow * (2 * std.math.pi) / M; @@ -427,7 +500,7 @@ test "invFreq Llama3" { const llama_freq = [_]f32{ 1.000000000000e+00, 6.636012792587e-01, 4.403666257858e-01, 2.922278344631e-01, 1.939227581024e-01, 1.286873817444e-01, 8.539710193872e-02, 5.666961893439e-02, 3.760603070259e-02, 2.495540864766e-02, 1.656044088304e-02, 1.098952908069e-02, 7.292665075511e-03, 4.839421249926e-03, 3.211446106434e-03, 1.290548010729e-03, 4.295567050576e-04, 9.708286233945e-05, 1.946163865796e-05, 1.291476746701e-05, 8.570255886298e-06, 5.687232260243e-06, 3.774054448513e-06, 2.504467147446e-06, 1.661967417022e-06, 1.102883629756e-06, 7.318749339902e-07, 4.856731266045e-07, 3.222932889457e-07, 2.138742303259e-07, 1.419272024350e-07, 9.418306490261e-08 }; var inv_freq: @TypeOf(llama_freq) = undefined; - _invFreq(llama_conf, &inv_freq); + _invFreq(llama_conf, &inv_freq, 32); for (llama_freq, inv_freq, 0..) |expected, actual, i| { errdefer log.err("Mismatch at position {d}.\nExpected: {any}\nActual: {any}", .{ i, llama_freq, inv_freq }); try std.testing.expectApproxEqRel(expected, actual, 1e-5); @@ -448,7 +521,7 @@ test "invFreq Yarn" { const yarn_freq = [_]f32{ 1.000000000000e+00, 6.890442967415e-01, 4.747820496559e-01, 3.271458745003e-01, 2.254180014133e-01, 1.553229838610e-01, 1.070244237781e-01, 7.374456524849e-02, 5.081327259541e-02, 3.162075206637e-02, 1.945096626878e-02, 1.179219130427e-02, 7.015713956207e-03, 4.069554619491e-03, 2.277272054926e-03, 1.206130953506e-03, 5.809474969283e-04, 2.279478358105e-04, 3.830881178146e-05, 2.639646845637e-05, 1.818833698053e-05, 1.253256959899e-05, 8.635495760245e-06, 5.950239483354e-06, 4.099978468730e-06, 2.825066758305e-06, 1.946596285052e-06, 1.341290953860e-06, 9.242089618056e-07, 6.368209142238e-07, 4.387978549403e-07, 3.023511396805e-07 }; var inv_freq: @TypeOf(yarn_freq) = undefined; - _invFreq(yarn_conf, &inv_freq); + _invFreq(yarn_conf, &inv_freq, 32); for (yarn_freq, inv_freq, 0..) |expected, actual, i| { errdefer log.err("Mismatch at position {d}.\nExpected: {d}\nActual: {d}", .{ i, stdx.fmt.slice(&yarn_freq), stdx.fmt.slice(&inv_freq) }); try std.testing.expectApproxEqRel(expected, actual, 1e-5); @@ -496,72 +569,12 @@ test "RopeOpts.Scaling parses default rope type" { } } -test "invFreq Proportional" { - const proportional_conf: RopeOpts = .{ - .scaling = .{ .proportional = .{ - .partial_rotary_factor = 0.25, - .rope_theta = 1_000_000, - } }, - }; - const default_conf: RopeOpts = .{ - .scaling = .{ .default = .{ - .rope_theta = 1_000_000, - } }, - }; - - var inv_freq: [16]f32 = undefined; - var default_inv_freq: [16]f32 = undefined; - _invFreq(proportional_conf, &inv_freq); - _invFreq(default_conf, &default_inv_freq); - - for (0..4) |i| { - try std.testing.expectApproxEqRel(default_inv_freq[i], inv_freq[i], 1e-6); - } - for (4..inv_freq.len) |i| { - try std.testing.expectEqual(@as(f32, 0.0), inv_freq[i]); - } -} - -test "invFreq Proportional clamps partial rotary factor" { - const default_conf: RopeOpts = .{ - .scaling = .{ .default = .{ - .rope_theta = 10_000, - } }, - }; - const negative_conf: RopeOpts = .{ - .scaling = .{ .proportional = .{ - .partial_rotary_factor = -0.5, - .rope_theta = 10_000, - } }, - }; - const overfull_conf: RopeOpts = .{ - .scaling = .{ .proportional = .{ - .partial_rotary_factor = 1.5, - .rope_theta = 10_000, - } }, - }; - - var default_inv_freq: [8]f32 = undefined; - var negative_inv_freq: [8]f32 = undefined; - var overfull_inv_freq: [8]f32 = undefined; - _invFreq(default_conf, &default_inv_freq); - _invFreq(negative_conf, &negative_inv_freq); - _invFreq(overfull_conf, &overfull_inv_freq); - - for (negative_inv_freq) |f| { - try std.testing.expectEqual(@as(f32, 0.0), f); - } - for (default_inv_freq, overfull_inv_freq) |expected, actual| { - try std.testing.expectApproxEqRel(expected, actual, 1e-6); - } -} - test "real/img" { const platform = zml.testing.env(); const Fns = struct { fn testSplitMergeIsId(layout: RopeOpts.Layout) Tensor { - const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ 5, 4 }); + const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ .n = 5, .hd = 4 }); const real, const imag = splitRealImg(x, layout); const y = mergeRealImg(real, imag, layout); const real2, const imag2 = splitRealImg(y, layout); @@ -571,8 +584,8 @@ test "real/img" { } fn testSplitSeqVoid(_: void) Tensor { - const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ 5, 4 }); - const real, const imag = splitRealImg(x, .sequential); + const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ .n = 5, .hd = 4 }); + const real, const imag = splitRealImg(x, .real_im_pass); const x_real = Tensor.concatenate(&.{ Tensor.arange(.{ .start = 0, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), Tensor.arange(.{ .start = 1, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), @@ -589,7 +602,7 @@ test "real/img" { fn testSplitSeq() Tensor { const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ 5, 4 }); - const real, const imag = splitRealImg(x, .sequential); + const real, const imag = splitRealImg(x, .real_im_pass); const x_real = Tensor.concatenate(&.{ Tensor.arange(.{ .start = 0, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), Tensor.arange(.{ .start = 1, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), @@ -624,7 +637,7 @@ test "real/img" { try std.testing.expectEqual(20, try d_interleaved.getValue(i32, std.testing.io)); } { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.sequential}, platform, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.real_im_pass}, platform, .{}); defer exe.deinit(); var d_sequential = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitMergeIsId, {}); @@ -670,7 +683,7 @@ test rope { var input = x; { // Convert input to the requested format - const real, const imag = splitRealImg(input, .sequential); + const real, const imag = splitRealImg(input, .real_im_pass); input = mergeRealImg(real, imag, opts.layout); } var res = rope(input, null, opts).squeeze(0); @@ -678,7 +691,7 @@ test rope { { // Convert back to sequential const real, const imag = splitRealImg(res, opts.layout); - res = mergeRealImg(real, imag, .sequential); + res = mergeRealImg(real, imag, .real_im_pass); } return res; } @@ -690,7 +703,7 @@ test rope { var exe_interleaved = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .interleaved } }, platform, .{}); defer exe_interleaved.deinit(); - var exe_sequential = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .sequential } }, platform, .{}); + var exe_sequential = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .real_im_pass } }, platform, .{}); defer exe_sequential.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]f32{ 1.0, 0.1, -1.0, -0.5 } ** 5)); @@ -704,6 +717,104 @@ test rope { try zml.testing.expectClose(std.testing.io, res1, res2, .{}); } +test "rope: Proportional" { + const io = std.testing.io; + const allocator = std.testing.allocator; + const platform = zml.testing.env(); + + const x: zml.Tensor = .init(.{ .s = 5, .hd = 16 }, .f32); + var exe = try zml.module.compile( + allocator, + io, + rope, + .{ + x, + null, + .{ + .layout = .real_pass_im_pass, + .scaling = .{ .proportional = .{ .partial_rotary_factor = 0.25 } }, + }, + }, + platform, + .{}, + ); + defer exe.deinit(); + + const x_h: [5][16]f32 = @splat([_]f32{ 0.754, -0.146, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }); + var x_buffer: zml.Buffer = try .fromBytes(io, platform, x.shape(), .replicated, @ptrCast(&x_h)); + defer x_buffer.deinit(); + + const expected_h: [5][16]f32 = .{ + .{ 0.754, -0.146, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + .{ 0.667, -0.449, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, 0.468, 0.904, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + .{ -0.0328, -0.708, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, 0.814, 0.719, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + .{ -0.703, -0.897, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, 0.412, 0.464, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + .{ -0.727, -0.997, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, -0.369, 0.162, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + }; + const expected: zml.Slice = .init(x.shape(), std.mem.sliceAsBytes(&expected_h)); + + var res = try zml.testing.autoCall(allocator, io, &exe, rope, .{ x_buffer, null }); + defer res.deinit(); + const res_h = try res.toSliceAlloc(allocator, io); + defer res_h.free(allocator); + + errdefer std.log.warn("{d:20}", .{res_h}); + try zml.testing.expectClose(io, expected, res, .{}); +} + +test "rope: Yarn with partial_rotary_factor" { + const io = std.testing.io; + const allocator = std.testing.allocator; + const platform = zml.testing.env(); + + const x: zml.Tensor = .init(.{ .s = 5, .hd = 16 }, .f32); + var exe = try zml.module.compile( + allocator, + io, + rope, + .{ + x, + null, + RopeOpts{ + .layout = .real_im_pass, + .scaling = .{ .yarn = .{ + .factor = 32.0, + .beta_fast = 32.0, + .beta_slow = 1.0, + .original_max_position_embeddings = 4096, + .truncate = true, + .rope_theta = 150_000, + .partial_rotary_factor = 0.25, + } }, + }, + }, + platform, + .{}, + ); + defer exe.deinit(); + + const x_h: [5][16]f32 = @splat([_]f32{ 0.754, -0.146, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }); + var x_buffer: zml.Buffer = try .fromBytes(io, platform, x.shape(), .replicated, @ptrCast(&x_h)); + defer x_buffer.deinit(); + + const expected_h: [5][16]f32 = .{ + .{ 1.02, -0.197, 0.00597, -1.29, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + .{ 0.544, -0.195, 0.858, -1.29, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + .{ -0.428, -0.193, 0.921, -1.29, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + .{ -1.01, -0.191, 0.137, -1.29, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + .{ -0.659, -0.19, -0.772, -1.29, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + }; + const expected: zml.Slice = .init(x.shape(), std.mem.sliceAsBytes(&expected_h)); + + var res = try zml.testing.autoCall(allocator, io, &exe, rope, .{ x_buffer, null }); + defer res.deinit(); + const res_h = try res.toSliceAlloc(allocator, io); + defer res_h.free(allocator); + + errdefer std.log.warn("{d:20}", .{res_h}); + try zml.testing.expectClose(io, expected, res, .{}); +} + pub const UpsampleMode = enum { nearest, // TODO: Linear, diff --git a/zml/tensor.zig b/zml/tensor.zig index 653a3fcf..a378171e 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1771,7 +1771,8 @@ pub const Tensor = struct { const start = if (s.start < 0) d + s.start else s.start; const end = if (s.end == to_the_end) d else if (s.end < 0) d + s.end else s.end; const res: Slice = .{ .start = start, .end = end, .step = s.step, .singleton = s.singleton }; - stdx.debug.assert(start < end, "Slice {f} is invalid for axis of dimension {d} (resolved to {f}", .{ s, d, res }); + stdx.debug.assert(start < end, "Slice {f} is invalid for axis of dimension {d} (resolved to {f})", .{ s, d, res }); + stdx.debug.assert(end <= d, "Slice {f} is invalid for axis of dimension {d} (resolved to {f})", .{ s, d, res }); return res; } @@ -1792,7 +1793,7 @@ pub const Tensor = struct { /// Slices the input Tensor over the given axis using the given parameters. pub fn slice1d(self: Tensor, axis_: anytype, s: Slice) Tensor { - var slices = [_]Slice{.{}} ** constants.MAX_RANK; + var slices: [constants.MAX_RANK]Slice = @splat(.{}); slices[self.axis(axis_)] = s; return self.slice(slices[0..self.rank()]); } @@ -1806,6 +1807,7 @@ pub const Tensor = struct { for (slices, 0..) |s, a| { stdx.debug.assert(s.step > 0, "slice expects 'step' to be positive, got {} at index {}", .{ s.step, a }); + stdx.debug.assert(s.step > 0, "slice expects 'step' to be positive, got {} at index {}", .{ s.step, a }); const args: Slice = s.absolute(self.dim(a)); start_indices[a] = args.start; From d6489fbb841c28644c19a19f8b0af444265b68a7 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 17 Jul 2026 14:10:41 +0200 Subject: [PATCH 583/881] zml/attention: implement batching in cu_fa2 backend (#641) --- examples/llm/main.zig | 6 +- examples/llm/models.zig | 2 +- examples/llm/models/common.zig | 2 +- examples/llm/models/lfm2/inference.zig | 12 +- examples/llm/models/lfm2/model.zig | 19 +- examples/llm/models/lfm2/session.zig | 7 +- examples/llm/models/lfm2_tests.zig | 15 +- examples/llm/models/llama/inference.zig | 21 +- examples/llm/models/llama/model.zig | 24 +- examples/llm/models/llama/session.zig | 9 +- examples/llm/models/qwen3_5/model.zig | 8 +- examples/llm/models/qwen3_5_moe/model.zig | 8 +- zml/BUILD.bazel | 1 - zml/attention.zig | 327 +++++++++++++++++++++- zml/attention/attention.zig | 169 ----------- zml/attention/flashattn.zig | 94 ++++--- zml/constants.zig | 3 + zml/ops.zig | 33 ++- zml/platform.zig | 15 +- zml/shape.zig | 44 ++- zml/slice.zig | 11 + zml/tensor.zig | 76 +++-- 22 files changed, 569 insertions(+), 337 deletions(-) delete mode 100644 zml/attention/attention.zig diff --git a/examples/llm/main.zig b/examples/llm/main.zig index e15baea9..620421ea 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -18,7 +18,7 @@ const Args = struct { prompt: ?[]const u8 = null, seqlen: u32 = 2048, topk: u32 = 4, - backend: ?zml.attention.attention.Backend = null, + backend: ?zml.attention.Backend = null, attnd_ip: ?[]const u8 = null, profile: bool = false, @@ -93,8 +93,8 @@ pub fn main(init: std.process.Init) !void { .destination = try .parseLiteral(attnd_ip), .mtu = 9000, }); - break :b zml.attention.attention.Backend.attnd; - } else zml.attention.attention.Backend.auto(platform); + break :b zml.attention.Backend.attnd; + } else zml.attention.Backend.auto(platform); defer if (args.attnd_ip) |_| zml.attention.attnd.deinit(); log.info("Selected backend: {}", .{backend}); diff --git a/examples/llm/models.zig b/examples/llm/models.zig index 839c1706..8a5df1be 100644 --- a/examples/llm/models.zig +++ b/examples/llm/models.zig @@ -76,7 +76,7 @@ pub const LoadedModel = union(ModelType) { allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, - backend: zml.attention.attention.Backend, + backend: zml.attention.Backend, shardings: Shardings, seqlen: usize, progress: *std.Progress.Node, diff --git a/examples/llm/models/common.zig b/examples/llm/models/common.zig index 2ec467b7..debeff00 100644 --- a/examples/llm/models/common.zig +++ b/examples/llm/models/common.zig @@ -4,7 +4,7 @@ const zml = @import("zml"); pub const SessionOptions = struct { seqlen: u32, - backend: zml.attention.attention.Backend, + backend: zml.attention.Backend, }; pub const GenerationOptions = struct { diff --git a/examples/llm/models/lfm2/inference.zig b/examples/llm/models/lfm2/inference.zig index 899c9100..045c60e3 100644 --- a/examples/llm/models/lfm2/inference.zig +++ b/examples/llm/models/lfm2/inference.zig @@ -2,25 +2,23 @@ const std = @import("std"); const zml = @import("zml"); const stdx = zml.stdx; -const attention = zml.attention.attention; const common = @import("../common.zig"); +const Phase = common.Phase; const model = @import("model.zig"); const log = std.log.scoped(.lfm); -const Phase = common.Phase; - pub const CompilationParameters = struct { hidden_dim: usize, batch_dim: usize, rng: zml.Tensor.Rng, cache: model.Cache, - attention_metadata: attention.Metadata, - attention_parameters: attention.Parameters, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, seqlen: u32, shardings: common.Shardings, - pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: attention.Backend, shardings: common.Shardings) CompilationParameters { + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: zml.attention.Backend, shardings: common.Shardings) CompilationParameters { stdx.debug.assert(seqlen >= config.conv_L_cache, "seqlen ({}) must be at least conv_L_cache ({})", .{ seqlen, config.conv_L_cache }); const cache: model.Cache = .{ .kv = .init(.init(.{ @@ -63,7 +61,7 @@ pub const Args = struct { actual_seq_len_buf: *zml.Buffer, rng_buf: *zml.Bufferized(zml.Tensor.Rng), cache_buffers: *zml.Bufferized(model.Cache), - attention_metadata_buffers: zml.Bufferized(attention.Metadata), + attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata), }; pub const CompiledModel = struct { diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index 91da6aab..e31d9a38 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -2,7 +2,6 @@ const std = @import("std"); const zml = @import("zml"); const stdx = zml.stdx; -const attention = zml.attention.attention; const common = @import("../common.zig"); const inference = @import("inference.zig"); @@ -115,7 +114,7 @@ pub const LoadedModel = struct { allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, - backend: zml.attention.attention.Backend, + backend: zml.attention.Backend, shardings: common.Shardings, seqlen: usize, progress: *std.Progress.Node, @@ -213,8 +212,8 @@ pub const Model = struct { actual_seq_len: zml.Tensor, rng: zml.Tensor.Rng, cache_: Cache, - attention_metadata: zml.attention.attention.Metadata, - attention_parameters: zml.attention.attention.Parameters, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, conv_parameters: ConvParameters, ) struct { zml.Tensor, Cache, zml.Tensor.Rng } { stdx.debug.assert(tokens.shape().hasTags(.{ .batch, .seq }), "Tokens should have tags {{.batch, .seq}}, got {f}", .{tokens.shape()}); @@ -330,8 +329,8 @@ pub const DecoderLayer = struct { cache_: Cache, conv_cache_index_: zml.Tensor, kv_cache_index_: zml.Tensor, - attention_metadata: zml.attention.attention.Metadata, - attention_parameters: zml.attention.attention.Parameters, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, conv_parameters: ConvParameters, ) struct { zml.Tensor, Cache, zml.Tensor, zml.Tensor } { var cache = cache_; @@ -491,8 +490,8 @@ pub const Attention = struct { tokens_position_offset: zml.Tensor, kv_cache: KvCache, cache_index: zml.Tensor, - attention_metadata: zml.attention.attention.Metadata, - attention_parameters: zml.attention.attention.Parameters, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, ) struct { zml.Tensor, KvCache } { const x_qkv = x.withPartitioning(.{ .d = .replicated }); @@ -521,7 +520,7 @@ pub const Attention = struct { stdx.debug.assert(q.dim(.batch) == 1, "LFM attention currently expects batch size 1 for flash attention backend, got {}", .{q.dim(.batch)}); const attn = switch (attention_parameters) { - .vanilla => zml.attention.attention.attention( + .vanilla => zml.attention.attention( q, k, v, @@ -529,7 +528,7 @@ pub const Attention = struct { attention_metadata, attention_parameters, ).merge(.{ .d = .{ .h, .hd } }), - else => zml.attention.attention.attention( + else => zml.attention.attention( q.squeeze(.batch), k.squeeze(.batch), v.squeeze(.batch), diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig index d78f5a8d..d841ff6f 100644 --- a/examples/llm/models/lfm2/session.zig +++ b/examples/llm/models/lfm2/session.zig @@ -1,7 +1,6 @@ const std = @import("std"); const zml = @import("zml"); -const attention = zml.attention.attention; const inference = @import("inference.zig"); const model = @import("model.zig"); @@ -118,11 +117,11 @@ pub const Session = struct { defer actual_seq_len_buf.deinit(); const params = self.compiled_model.params; - var attention_metadata_buffers: zml.Bufferized(attention.Metadata) = switch (params.attention_metadata) { + var attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = switch (params.attention_metadata) { .metal_fa => .{ .metal_fa = .{ .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32) } }, else => try params.attention_metadata.initBuffer(self.io, self.platform, params.shardings.model), }; - defer attention.Metadata.deinitBuffer(&attention_metadata_buffers); + defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); try self.compiled_model.prefill.run(.{ .allocator = self.allocator, @@ -157,7 +156,7 @@ pub const Session = struct { const params = self.compiled_model.params; var attention_metadata_buffers = try params.attention_metadata.initBuffer(self.io, self.platform, params.shardings.model); - defer attention.Metadata.deinitBuffer(&attention_metadata_buffers); + defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); generation: while (true) { const token_id = self.generated_token_slice.items(u32)[0]; diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig index 8e60e9d5..040a1a68 100644 --- a/examples/llm/models/lfm2_tests.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -1,7 +1,6 @@ const std = @import("std"); const zml = @import("zml"); -const attention = zml.attention.attention; const common = @import("common.zig"); const lfm2 = @import("lfm2.zig"); @@ -14,7 +13,7 @@ pub const std_options: std.Options = .{ const Args = struct { model: []const u8, activations: []const u8, - backend: ?attention.Backend = null, + backend: ?zml.attention.Backend = null, pub const help = \\Use lfm2_tests --model= --activations= [options] @@ -52,7 +51,7 @@ pub fn main(init: std.process.Init) !void { var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress, shardings); defer repo_model.unloadBuffers(&model_buffers, allocator); - const backend = args.backend orelse attention.Backend.auto(platform); + const backend = args.backend orelse zml.attention.Backend.auto(platform); const params = lfm2.CompilationParameters.init( repo_model.inner, repo_model.parsed_config.value, @@ -73,8 +72,8 @@ pub fn run( config: lfm2.Config, mdl: lfm2.Model, model_buffers: *lfm2.Buffers, - attention_metadata: attention.Metadata, - attention_parameters: attention.Parameters, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, ) !void { var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, activations_path); defer registry.deinit(); @@ -132,8 +131,8 @@ const TestContext = struct { io: std.Io, platform: *zml.Platform, activations_store: *zml.io.TensorStore, - attention_metadata: attention.Metadata, - attention_parameters: attention.Parameters, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, sharding: zml.Sharding, fn testLayerPrint(self: *TestContext, comptime name_fmt: []const u8, name_args: anytype, tagz: anytype, layer: anytype, layer_buffers: anytype, opts: zml.testing.CompareOpts) !void { @@ -279,7 +278,7 @@ const TestContext = struct { const kv_cache: zml.Bufferized(model.KvCache) = .{ .k = key_cache_buffer, .v = value_cache_buffer }; var attention_metadata_buffers = try self.attention_metadata.initBuffer(self.io, self.platform, self.sharding); - defer attention.Metadata.deinitBuffer(&attention_metadata_buffers); + defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); var cache_index_buf: zml.Buffer = try .scalar(self.io, self.platform, cache_ix, .u32); defer cache_index_buf.deinit(); diff --git a/examples/llm/models/llama/inference.zig b/examples/llm/models/llama/inference.zig index c2d13701..80c3c0ac 100644 --- a/examples/llm/models/llama/inference.zig +++ b/examples/llm/models/llama/inference.zig @@ -1,7 +1,6 @@ const std = @import("std"); const zml = @import("zml"); -const attention = zml.attention.attention; const common = @import("../common.zig"); const model = @import("model.zig"); @@ -15,13 +14,13 @@ pub const CompilationParameters = struct { token_index: zml.Tensor, kv_cache: model.KvCache, rng: zml.Tensor.Rng, - attention_metadata: attention.Metadata, - prefill_attention_parameters: attention.Parameters, - decode_attention_parameters: attention.Parameters, + attention_metadata: zml.attention.Metadata, + prefill_attention_parameters: zml.attention.Parameters, + decode_attention_parameters: zml.attention.Parameters, seqlen: usize, shardings: common.Shardings, - pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: attention.Backend, shardings: common.Shardings) CompilationParameters { + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: zml.attention.Backend, shardings: common.Shardings) CompilationParameters { const head_dim = config.head_dim orelse @divExact(config.hidden_size, config.num_attention_heads); return .{ @@ -76,7 +75,7 @@ pub const Args = struct { token_index_buf: *zml.Buffer, kv_cache_buffers: *zml.Bufferized(model.KvCache), rng_buffers: *zml.Bufferized(zml.Tensor.Rng), - attention_metadata_buffers: *const zml.Bufferized(attention.Metadata), + attention_metadata_buffers: *const zml.Bufferized(zml.attention.Metadata), }; pub const CompiledModel = struct { @@ -264,7 +263,7 @@ pub const KernelExe = struct { llama_model: model.Model, parameters: CompilationOptions, seqlen: usize, - attention_parameters: attention.Parameters, + attention_parameters: zml.attention.Parameters, phase: Phase, progress: *std.Progress.Node, ) !KernelExe { @@ -315,7 +314,7 @@ const ComposedKernelExe = struct { llama_model: model.Model, parameters: CompilationOptions, seqlen: usize, - attention_parameters: attention.Parameters, + attention_parameters: zml.attention.Parameters, phase: Phase, progress: *std.Progress.Node, ) !ComposedKernelExe { @@ -501,7 +500,7 @@ const ComposedKernelExe = struct { llama_model: model.Model, parameters: CompilationOptions, seqlen: usize, - attention_parameters: attention.Parameters, + attention_parameters: zml.attention.Parameters, phase: Phase, progress: *std.Progress.Node, ) !zml.Exe { @@ -514,8 +513,8 @@ const ComposedKernelExe = struct { token_index: zml.Tensor, kv_cache: model.KvCache, kv_cache_index: zml.Tensor, - attention_metadata: attention.Metadata, - attention_parameters_: attention.Parameters, + attention_metadata: zml.attention.Metadata, + attention_parameters_: zml.attention.Parameters, ) struct { zml.Tensor, model.KvCache } { const new_hidden, const new_kv_cache, _ = self.layer.forward( hidden, diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index 0874598a..c63fbb98 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -101,7 +101,7 @@ pub const LoadedModel = struct { allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, - backend: zml.attention.attention.Backend, + backend: zml.attention.Backend, shardings: common.Shardings, seqlen: usize, progress: *std.Progress.Node, @@ -202,8 +202,8 @@ pub const Model = struct { token_index: zml.Tensor, kv_cache: KvCache, rng: zml.Tensor.Rng, - attention_metadata: zml.attention.attention.Metadata, - attention_parameters: zml.attention.attention.Parameters, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { const tokens = tokens_.withPartialTags(.{.s}); @@ -268,8 +268,8 @@ const Llama = struct { tokens: zml.Tensor, token_index: zml.Tensor, kv_cache: KvCache, - attention_metadata: zml.attention.attention.Metadata, - attention_parameters: zml.attention.attention.Parameters, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, ) struct { zml.Tensor, KvCache } { const embeds = self.embed_tokens.forward(tokens).withPartialTags(.{.d}); var hidden = embeds; @@ -354,8 +354,8 @@ pub const TransformerLayer = struct { token_index: zml.Tensor, kv_cache: KvCache, kv_cache_index: zml.Tensor, - attention_metadata: zml.attention.attention.Metadata, - attention_parameters: zml.attention.attention.Parameters, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, ) struct { zml.Tensor, KvCache, zml.Tensor } { // Self Attention //log.debug("TransformerLayer({f}) -> {f}", .{ x0, self.input_layernorm.forward(x0) }); @@ -487,7 +487,7 @@ const SelfAttn = struct { if (self.k_norm) |*k_norm| RmsNorm.unloadBuffers(k_norm); } - /// Self zml.attention.attention. + /// Self zml.attention. /// - If token_index is set, x is assumed to be the representation of one new token, /// and kv_cache will be read for the previous tokens. /// - If token_index is not set, x is assumed to be the representation of all tokens @@ -500,8 +500,8 @@ const SelfAttn = struct { token_index: zml.Tensor, kv_cache: KvCache, kv_cache_index: zml.Tensor, - attention_metadata: zml.attention.attention.Metadata, - attention_parameters: zml.attention.attention.Parameters, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, ) struct { zml.Tensor, KvCache } { const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; @@ -532,7 +532,7 @@ const SelfAttn = struct { k = new_kv_cache.keysAt(kv_cache_index).convert(dtype); v = new_kv_cache.valuesAt(kv_cache_index).convert(dtype); - const layer_attention_metadata: zml.attention.attention.Metadata = switch (attention_parameters) { + const layer_attention_metadata: zml.attention.Metadata = switch (attention_parameters) { .attnd => .{ .attnd = .{ .layer_id = kv_cache_index.convert(.u16), .conversation_id = attention_metadata.attnd.conversation_id, @@ -545,7 +545,7 @@ const SelfAttn = struct { .metal_fa => attention_metadata, }; - const attn_output = zml.attention.attention.attention( + const attn_output = zml.attention.attention( q, k, v, diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig index 3a55ea85..9cc69c01 100644 --- a/examples/llm/models/llama/session.zig +++ b/examples/llm/models/llama/session.zig @@ -1,7 +1,6 @@ const std = @import("std"); const zml = @import("zml"); -const attention = zml.attention.attention; const inference = @import("inference.zig"); const model = @import("model.zig"); @@ -141,7 +140,7 @@ pub const Session = struct { defer prefill_tokens_buffer.deinit(); const params = self.compiled_model.params; - var attention_metadata_buffers: zml.Bufferized(attention.Metadata) = switch (params.attention_metadata) { + var attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = switch (params.attention_metadata) { .attnd => .{ .attnd = .{ .conversation_id = try .scalar(self.io, self.platform, self.conversation_id, .u64), .layer_id = try .scalar(self.io, self.platform, 0, .u16), @@ -150,7 +149,7 @@ pub const Session = struct { .metal_fa => .{ .metal_fa = .{ .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32) } }, .vanilla, .cuda_fa2, .cuda_fa3, .nki => try params.attention_metadata.initBuffer(self.io, self.platform, params.shardings.model), }; - defer attention.Metadata.deinitBuffer(&attention_metadata_buffers); + defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); try self.compiled_model.prefill.run(.{ .allocator = self.allocator, @@ -180,7 +179,7 @@ pub const Session = struct { defer current_token_buffer.deinit(); const params = self.compiled_model.params; - var attention_metadata_buffers: zml.Bufferized(attention.Metadata) = switch (params.attention_metadata) { + var attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = switch (params.attention_metadata) { .attnd => .{ .attnd = .{ .conversation_id = try .scalar(self.io, self.platform, self.conversation_id, .u64), .layer_id = try .scalar(self.io, self.platform, 0, .u16), @@ -188,7 +187,7 @@ pub const Session = struct { } }, .vanilla, .cuda_fa2, .cuda_fa3, .nki, .metal_fa => try params.attention_metadata.initBuffer(self.io, self.platform, params.shardings.model), }; - defer attention.Metadata.deinitBuffer(&attention_metadata_buffers); + defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); generation: while (true) { if (isEosToken(self.config, last_token_id)) break :generation; diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 5726c24d..4191965a 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -117,7 +117,7 @@ pub const LoadedModel = struct { allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, - backend: zml.attention.attention.Backend, + backend: zml.attention.Backend, shardings: common.Shardings, seqlen: usize, progress: *std.Progress.Node, @@ -598,13 +598,13 @@ pub const SelfAttn = struct { k = partitionCachedKv(k, kv_head_sharding); v = partitionCachedKv(v, kv_head_sharding); - const attn_output = zml.attention.attention.attention( + const attn_output = zml.attention.attention( q, k, v, token_index, - zml.attention.attention.Metadata.init(.fromBackend(.vanilla, x.dim(.s), self.num_heads)), - zml.attention.attention.Parameters.init(.fromBackend(.vanilla)), + zml.attention.Metadata.init(.fromBackend(.vanilla, x.dim(.s), self.num_heads)), + zml.attention.Parameters.init(.fromBackend(.vanilla)), ).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); const gated_output = attn_output.mul(gate.sigmoid()); diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 8d356da3..f06b6586 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -144,7 +144,7 @@ pub const LoadedModel = struct { allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, - backend: zml.attention.attention.Backend, + backend: zml.attention.Backend, shardings: common.Shardings, seqlen: usize, progress: *std.Progress.Node, @@ -601,13 +601,13 @@ pub const SelfAttn = struct { k = partitionCachedKv(k, kv_head_sharding); v = partitionCachedKv(v, kv_head_sharding); - const attn_output = zml.attention.attention.attention( + const attn_output = zml.attention.attention( q, k, v, token_index, - zml.attention.attention.Metadata.init(.fromBackend(.vanilla, x.dim(.s), self.num_heads)), - zml.attention.attention.Parameters.init(.fromBackend(.vanilla)), + zml.attention.Metadata.init(.fromBackend(.vanilla, x.dim(.s), self.num_heads)), + zml.attention.Parameters.init(.fromBackend(.vanilla)), ).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); const gated_output = attn_output.mul(gate.sigmoid()); diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 5fa7a4c5..1d1df5c0 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -86,7 +86,6 @@ zig_library( }), srcs = [ "attention.zig", - "attention/attention.zig", "attention/attnd.zig", "attention/flashattn.zig", "attention/metal_attention.zig", diff --git a/zml/attention.zig b/zml/attention.zig index ca7b3da0..a6962f47 100644 --- a/zml/attention.zig +++ b/zml/attention.zig @@ -1,13 +1,336 @@ -pub const attention = @import("attention/attention.zig"); +const std = @import("std"); + pub const attnd = @import("attention/attnd.zig"); pub const flashattn = @import("attention/flashattn.zig"); pub const metal = @import("attention/metal_attention.zig"); +pub const nki = @import("attention/nki/attention.zig"); pub const paged_attention = @import("attention/paged_attention.zig"); pub const tpu = @import("attention/tpu_attention.zig"); pub const triton = @import("attention/triton_attention.zig"); pub const triton_kernels = @import("attention/triton_kernels/unified_attention.zig"); +const zml = @import("zml.zig"); test { - const std = @import("std"); std.testing.refAllDecls(@This()); } + +pub const Backend = enum { + vanilla, + attnd, + nki, + cuda_fa2, + cuda_fa3, + metal_fa, + + pub fn auto(platform: *const zml.Platform) Backend { + return switch (platform.target) { + .cuda => b: { + const first_device = platform.pjrt_client.devices(platform.pjrt_api)[0]; + + if (zml.platform.cuda.tryGetComputeCapabilities(platform, first_device)) |cc| { + break :b if (std.mem.eql(u8, cc, "9.0")) + .cuda_fa3 + else + .cuda_fa2; + } + + break :b .vanilla; + }, + .neuron => .nki, + .metal => .metal_fa, + .cpu, .rocm, .tpu, .oneapi => .vanilla, + }; + } + + pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { + return switch (backend) { + .vanilla => true, + .attnd => true, // attnd runs over network + .nki => platform.target == .neuron, + .metal_fa => platform.target == .metal, + .cuda_fa2 => platform.target == .cuda, + .cuda_fa3 => { + if (platform.target != .cuda) return false; + const first_device = platform.pjrt_client.devices(platform.pjrt_api)[0]; + const cc = zml.platform.cuda.tryGetComputeCapabilities(platform, first_device) orelse return false; + return std.mem.eql(u8, cc, "9.0"); + }, + }; + } +}; + +pub const Parameters = union(Backend) { + vanilla: void, + attnd: attnd.Parameters, + nki: nki.Parameters, + cuda_fa2: flashattn.fa2.Parameters, + cuda_fa3: flashattn.fa3.Parameters, + metal_fa: void, + + pub const InitOptions = union(Backend) { + vanilla: void, + attnd: void, + nki: nki.Parameters, + cuda_fa2: flashattn.fa2.Parameters.InitOptions, + cuda_fa3: flashattn.fa3.Parameters.InitOptions, + metal_fa: void, + + pub fn fromBackend(backend: Backend) InitOptions { + return switch (backend) { + .vanilla => .{ .vanilla = {} }, + .attnd => @panic("Must be initialized manually"), + .nki => .{ .nki = .init() }, + .cuda_fa2 => .{ .cuda_fa2 = .{} }, + .cuda_fa3 => .{ .cuda_fa3 = .{} }, + .metal_fa => .{ .metal_fa = {} }, + }; + } + }; + + pub fn init(opts: InitOptions) Parameters { + return switch (opts) { + .vanilla => .{ .vanilla = {} }, + .attnd => @panic("Must be initialized manually"), + .nki => |v| .{ .nki = v }, + .cuda_fa2 => |v| .{ .cuda_fa2 = .init(v) }, + .cuda_fa3 => |v| .{ .cuda_fa3 = .init(v) }, + .metal_fa => .{ .metal_fa = {} }, + }; + } +}; + +pub const Metadata = union(Backend) { + vanilla: void, + attnd: attnd.Metadata, + nki: void, + cuda_fa2: flashattn.fa2.Metadata, + cuda_fa3: flashattn.fa3.Metadata, + metal_fa: metal.Metadata, + + pub const InitOptions = union(Backend) { + vanilla: void, + attnd: void, + nki: void, + cuda_fa2: flashattn.fa2.Metadata.InitOptions, + cuda_fa3: flashattn.fa3.Metadata.InitOptions, + metal_fa: void, + + pub fn fromBackend(backend: Backend, seqlen: i64, num_heads: i64) InitOptions { + return switch (backend) { + .vanilla => .{ .vanilla = {} }, + .attnd => .{ .attnd = {} }, + .nki => .{ .nki = {} }, + .cuda_fa2 => .{ .cuda_fa2 = .{ .seqlen = seqlen, .num_heads = num_heads } }, + .cuda_fa3 => .{ .cuda_fa3 = .{ .seqlen = seqlen, .num_heads = num_heads } }, + .metal_fa => .{ .metal_fa = {} }, + }; + } + }; + + pub fn init(opts: InitOptions) Metadata { + return switch (opts) { + .vanilla => .{ .vanilla = {} }, + .attnd => @panic("Must be initialized manually"), + .nki => .{ .nki = {} }, + .cuda_fa2 => |o| .{ .cuda_fa2 = flashattn.fa2.Metadata.init(o) }, + .cuda_fa3 => |o| .{ .cuda_fa3 = flashattn.fa3.Metadata.init(o) }, + .metal_fa => .{ .metal_fa = .init() }, + }; + } + + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(Metadata) { + return switch (self) { + .vanilla => .{ .vanilla = {} }, + .nki => .{ .nki = {} }, + inline else => |v, tag| @unionInit(zml.Bufferized(Metadata), @tagName(tag), try v.initBuffer(io, platform, sharding)), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { + switch (self.*) { + .vanilla => {}, + .attnd => |*v| attnd.Metadata.deinitBuffer(v), + .nki => {}, + .cuda_fa2 => |*v| flashattn.fa2.Metadata.deinitBuffer(v), + .cuda_fa3 => |*v| flashattn.fa3.Metadata.deinitBuffer(v), + .metal_fa => |*v| metal.Metadata.deinitBuffer(v), + } + } +}; + +/// Causal attention as used in transformers. +/// +/// **Shapes**: +/// - q, result: .{ .q, .h, .hd } +/// - k, v: .{ .k, .h, .hd } +/// +/// Where: +/// - .h is the number of head +/// - .q is the number of queries +/// - .k is the number of keys +/// - .hd is the head dimension +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { + return switch (parameters) { + .vanilla => b: { + // Generate the attention mask. + const seq_len = k.dim(.k); + var attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, q.dtype(), null); + + // Note: in Pytorch it would be very inefficient to generate the full attn_mask, + // then slice into it, but XLA is able to optimize this correctly. + attn_mask = attn_mask.gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, attn_mask.dtype()), token_index.appendAxes(.{.coord}), .{}); + const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask }); + break :b attn_output; + }, + .attnd => attnd.causalAttention(q, k, v, token_index, metadata.attnd, parameters.attnd), + .nki => |params| nki.attention(q, k, v, token_index, params), + .cuda_fa2 => flashattn.fa2.attention(q, k, v, token_index, metadata.cuda_fa2, parameters.cuda_fa2), + .cuda_fa3 => flashattn.fa3.attention(q, k, v, token_index, metadata.cuda_fa3, parameters.cuda_fa3), + .metal_fa => metal.attention(q, k, v, token_index, metadata.metal_fa), + }; +} + +test "attention: q=1,qh=64,kh=8" { + try testAttention( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{63}, + ); + + try testAttention( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{36}, + ); +} + +test "attention: b=4,q=1,qh=64,kh=8" { + // Full attention + try testAttention( + .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{ 63, 63, 63, 63 }, + ); + + // Partial attention + try testAttention( + .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{ 61, 57, 23, 63 }, + ); +} + +test "attention: q=1,qh=8,kh=8" { + try testAttention( + .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{62}, + ); + try testAttention( + .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{63}, + ); +} + +test "attention: q=8,qh=64,kh=8" { + try testAttention( + .init(.{ .q = 8, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{56}, + ); +} + +pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32) !void { + const io = std.testing.io; + const allocator = std.testing.allocator; + + const platform = zml.testing.env(); + + var arena_state: std.heap.ArenaAllocator = .init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + const token_index_shape: zml.Shape = if (q_shape.hasTag(.b)) |_| b: { + std.debug.assert(token_index_h.len == q_shape.dim(.b)); + break :b .init(.{ .b = q_shape.dim(.b) }, .u32); + } else .init(.{}, .u32); + + const max_k_idx: i64 = k_shape.dim(.k) - 1; + const max_q_idx: i64 = q_shape.dim(.q) - 1; + for (token_index_h) |index| { + // Check for out of bound reads + std.debug.assert(index + max_q_idx <= max_k_idx); + } + + const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor } = .{ + .q = .fromShape(q_shape), + .k = .fromShape(k_shape), + .v = .fromShape(k_shape), + .token_index = .fromShape(token_index_shape), + }; + + const rng_q = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.q.shape(), .{ .mean = 0, .stddev = 1 } }, .{}); + defer rng_q.deinit(); + const rng_k = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.k.shape(), .{ .mean = 0, .stddev = 1 } }, .{}); + defer rng_k.deinit(); + + const q = try zml.testing.autoCall(allocator, io, &rng_q, zml.Tensor.Rng.normal, {}); + const k = try zml.testing.autoCall(allocator, io, &rng_k, zml.Tensor.Rng.normal, {}); + const v = try zml.testing.autoCall(allocator, io, &rng_k, zml.Tensor.Rng.normal, {}); + const token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, .replicated, @ptrCast(token_index_h)); + + const shardings = platform.shardings.values(); + const vanilla_exe = try platform.compileFn(allocator, io, attention, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, .vanilla, .vanilla }, .{ + .program_name = "attention_vanilla", + .shardings = shardings, + }); + defer vanilla_exe.deinit(); + + const vanilla_d = try zml.testing.autoCall(allocator, io, &vanilla_exe, attention, .{ q, k, v, token_index, .vanilla }); + try vanilla_d.await(io); + const vanilla_h: zml.Slice = try vanilla_d.toSliceAlloc(allocator, io); + defer vanilla_h.free(allocator); + + for (std.enums.values(Backend)) |backend| { + switch (backend) { + .attnd, .vanilla => continue, + else => if (!backend.isAvailable(platform)) continue, + } + + const metadata: Metadata = .init(.fromBackend(backend, tensors.k.dim(.k), tensors.q.dim(.h))); + const parameters: Parameters = .init(.fromBackend(backend)); + const exe = try platform.compileFn( + allocator, + io, + attention, + .{ tensors.q, tensors.k, tensors.v, tensors.token_index, metadata, parameters }, + .{ + .program_name = try std.fmt.allocPrint(arena, "attention_{t}", .{backend}), + .shardings = shardings, + }, + ); + defer exe.deinit(); + + var metadata_d = try metadata.initBuffer(io, platform, platform.shardings.get("model").?); + defer Metadata.deinitBuffer(&metadata_d); + + var output_d = try zml.testing.autoCall(allocator, io, &exe, attention, .{ q, k, v, token_index, metadata_d }); + defer output_d.deinit(); + try output_d.await(io); + const output_h = try output_d.toSliceAlloc(allocator, io); + defer output_h.free(allocator); + + errdefer std.log.err( + \\ Attention test failed, {0t} output doesn't match reference. + \\ - reference: {1d} + \\ - {0t}: {2d} + , .{ backend, vanilla_h, output_h }); + try zml.testing.expectClose(io, vanilla_h, output_h, .{ + .absolute_tolerance = 5e-3, + .relative_tolerance = 1e-2, + .epsilon_relative = 1e-3, + .minimum_close_fraction = 0.99, + }); + } +} diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig deleted file mode 100644 index e904bc3d..00000000 --- a/zml/attention/attention.zig +++ /dev/null @@ -1,169 +0,0 @@ -const std = @import("std"); - -const zml = @import("../zml.zig"); -const attnd = @import("attnd.zig"); -const flashattn = @import("flashattn.zig"); -const metal = @import("metal_attention.zig"); -const nki = @import("nki/attention.zig"); - -const Attention = @This(); - -pub const Backend = enum { - vanilla, - attnd, - nki, - cuda_fa2, - cuda_fa3, - metal_fa, - - pub fn auto(platform: *const zml.Platform) Backend { - return switch (platform.target) { - .cuda => b: { - const first_device = platform.pjrt_client.devices(platform.pjrt_api)[0]; - - if (zml.platform.cuda.tryGetComputeCapabilities(platform, first_device)) |cc| { - break :b if (std.mem.eql(u8, cc, "9.0")) - .cuda_fa3 - else - .cuda_fa2; - } - - break :b .vanilla; - }, - .neuron => .nki, - .metal => .metal_fa, - .cpu, .rocm, .tpu, .oneapi => .vanilla, - }; - } -}; - -pub const Parameters = union(Backend) { - vanilla: void, - attnd: attnd.Parameters, - nki: nki.Parameters, - cuda_fa2: flashattn.fa2.Parameters, - cuda_fa3: flashattn.fa3.Parameters, - metal_fa: void, - - pub const InitOptions = union(Backend) { - vanilla: void, - attnd: void, - nki: nki.Parameters, - cuda_fa2: flashattn.fa2.Parameters.InitOptions, - cuda_fa3: flashattn.fa3.Parameters.InitOptions, - metal_fa: void, - - pub fn fromBackend(backend: Backend) InitOptions { - return switch (backend) { - .vanilla => .{ .vanilla = {} }, - .attnd => @panic("Must be initialized manually"), - .nki => .{ .nki = .init() }, - .cuda_fa2 => .{ .cuda_fa2 = .{} }, - .cuda_fa3 => .{ .cuda_fa3 = .{} }, - .metal_fa => .{ .metal_fa = {} }, - }; - } - }; - - pub fn init(opts: InitOptions) Parameters { - return switch (opts) { - .vanilla => .{ .vanilla = {} }, - .attnd => @panic("Must be initialized manually"), - .nki => |v| .{ .nki = v }, - .cuda_fa2 => |v| .{ .cuda_fa2 = .init(v) }, - .cuda_fa3 => |v| .{ .cuda_fa3 = .init(v) }, - .metal_fa => .{ .metal_fa = {} }, - }; - } -}; - -pub const Metadata = union(Backend) { - vanilla: void, - attnd: attnd.Metadata, - nki: void, - cuda_fa2: flashattn.fa2.Metadata, - cuda_fa3: flashattn.fa3.Metadata, - metal_fa: metal.Metadata, - - pub const InitOptions = union(Backend) { - vanilla: void, - attnd: void, - nki: void, - cuda_fa2: flashattn.fa2.Metadata.InitOptions, - cuda_fa3: flashattn.fa3.Metadata.InitOptions, - metal_fa: void, - - pub fn fromBackend(backend: Backend, seqlen: i64, num_heads: i64) InitOptions { - return switch (backend) { - .vanilla => .{ .vanilla = {} }, - .attnd => .{ .attnd = {} }, - .nki => .{ .nki = {} }, - .cuda_fa2 => .{ .cuda_fa2 = .{ .seqlen = seqlen, .num_heads = num_heads } }, - .cuda_fa3 => .{ .cuda_fa3 = .{ .seqlen = seqlen, .num_heads = num_heads } }, - .metal_fa => .{ .metal_fa = {} }, - }; - } - }; - - pub fn init(opts: InitOptions) Metadata { - return switch (opts) { - .vanilla => .{ .vanilla = {} }, - .attnd => @panic("Must be initialized manually"), - .nki => .{ .nki = {} }, - .cuda_fa2 => |o| .{ .cuda_fa2 = flashattn.fa2.Metadata.init(o) }, - .cuda_fa3 => |o| .{ .cuda_fa3 = flashattn.fa3.Metadata.init(o) }, - .metal_fa => .{ .metal_fa = .init() }, - }; - } - - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(Metadata) { - return switch (self) { - .vanilla => .{ .vanilla = {} }, - .nki => .{ .nki = {} }, - inline else => |v, tag| @unionInit(zml.Bufferized(Metadata), @tagName(tag), try v.initBuffer(io, platform, sharding)), - }; - } - - pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { - switch (self.*) { - .vanilla => {}, - .attnd => |*v| attnd.Metadata.deinitBuffer(v), - .nki => {}, - .cuda_fa2 => |*v| flashattn.fa2.Metadata.deinitBuffer(v), - .cuda_fa3 => |*v| flashattn.fa3.Metadata.deinitBuffer(v), - .metal_fa => |*v| metal.Metadata.deinitBuffer(v), - } - } -}; - -/// Causal attention as used in transformers. -/// -/// **Shapes**: -/// - q, result: .{ .q, .h, .hd } -/// - k, v: .{ .k, .h, .hd } -/// -/// Where: -/// - .h is the number of head -/// - .q is the number of queries -/// - .k is the number of keys -/// - .hd is the head dimension -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { - return switch (parameters) { - .vanilla => b: { - // Generate the attention mask. - const seq_len = k.dim(.k); - var attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, q.dtype(), null); - - // Note: in Pytorch it would be very inefficient to generate the full attn_mask, - // then slice into it, but XLA is able to optimize this correctly. - attn_mask = attn_mask.gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, attn_mask.dtype()), token_index.reshape(.{ .coord = 1 }), .{}); - const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask }); - break :b attn_output; - }, - .attnd => attnd.causalAttention(q, k, v, token_index, metadata.attnd, parameters.attnd), - .nki => |params| nki.attention(q, k, v, token_index, params), - .cuda_fa2 => flashattn.fa2.attention(q, k, v, token_index, metadata.cuda_fa2, parameters.cuda_fa2), - .cuda_fa3 => flashattn.fa3.attention(q, k, v, token_index, metadata.cuda_fa3, parameters.cuda_fa3), - .metal_fa => metal.attention(q, k, v, token_index, metadata.metal_fa), - }; -} diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index dbc35f66..e2be7006 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -33,9 +33,10 @@ fn flashattnDataTypeFromZmlDataType(dtype: zml.DataType) flashattn.DataType { .f32 => .f32, .f16 => .f16, .bf16 => .bf16, + .u32 => .i32, .i32 => .i32, .i8 => .i8, - else => unreachable, + else => std.debug.panic("Unsupported dtype for attention backend cu_fa2: {t}", .{dtype}), }; } @@ -149,6 +150,7 @@ pub const fa2 = struct { v: zml.Tensor, cu_seqlens_q: zml.Tensor, cu_seqlens_k: zml.Tensor, + seqused_k: zml.Tensor, softmax_lse: zml.Tensor, softmax_lse_accum: zml.Tensor, out_accum: zml.Tensor, @@ -194,7 +196,7 @@ pub const fa2 = struct { &toFlashattnTensor(output.o), &toFlashattnTensor(input.cu_seqlens_q), &toFlashattnTensor(input.cu_seqlens_k), - null, + &toFlashattnTensor(input.seqused_k), null, &toFlashattnTensor(input.softmax_lse), null, @@ -265,49 +267,63 @@ pub const fa2 = struct { } }; - pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { + pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index_: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { const ctx = CompilationContext.current(); - stdx.debug.assert(q_.shape().hasTag(.b) == null or q_.dim(.b) == 1, "fa2.attention support for batch size != 1 is not supported yet.", .{}); - const seqused_k = token_index.addConstant(q_.dim(.q)).reshape(.{1}); - // TODO(Corendos): replace with cumsum - const cu_seqlens_k = b: { - const zero = zml.Tensor.constant(token_index.dtype().zero()).reshape(.{1}); - break :b zml.Tensor.concatenate(&.{ zero, seqused_k }, 0).convert(.i32); - }; + var bs: i64 = 1; + var q = q_; + var k = k_; + var v = v_; + var token_index = token_index_; + + // Follow cu_fa2 convention + // We are a bit more restrictive because we assume each sequence has the same number of queries and keys. + // https://deepwiki.com/Dao-AILab/flash-attention/3.1-flashattention-2-python-interface#flash_attn_varlen_func + if (q.shape().hasTag(.b)) |_| { + bs = q.dim(.b); + q = q.merge(.{ .tot = .{ .b, .q } }); + k = k.merge(.{ .tot = .{ .b, .k } }); + v = v.merge(.{ .tot = .{ .b, .k } }); + } else { + q = q.rename(.{ .q = .tot }); + k = k.rename(.{ .k = .tot }); + v = v.rename(.{ .k = .tot }); + token_index = token_index.broad(.init(.{ .b = 1 }, .i32)); + } + const max_seqlen_q: i32 = @intCast(q_.dim(.q)); const max_seqlen_k: i32 = @intCast(k_.dim(.k)); - var q, const k, const v = if (q_.shape().hasTag(.b) != null) b: { - break :b [_]zml.Tensor{ - q_.merge(.{ .tot = .{ .b, .q } }), k_.merge(.{ .tot = .{ .b, .k } }), v_.merge(.{ .tot = .{ .b, .k } }), - }; - } else b: { - break :b [_]zml.Tensor{ q_.rename(.{ .q = .tot }), k_.rename(.{ .k = .tot }), v_.rename(.{ .k = .tot }) }; - }; - // TODO(Corendos): replace with cumsum - const cu_seqlens_q = zml.Tensor.constantTensor(zml.Shape.init(.{2}, .i32), std.mem.sliceAsBytes(&[2]i32{ 0, max_seqlen_q })) - .withPartitioning(.{ ._0 = .replicated }); + + // We still have a rectangle layout, q and k haven't been compacted. + // So each sequence have the same number of queries, keys + const cu_seqlens_q: zml.Tensor = .arange(.{ .end = max_seqlen_q * (bs + 1), .step = max_seqlen_q }, .i32); + const cu_seqlens_k: zml.Tensor = .arange(.{ .end = max_seqlen_k * (bs + 1), .step = max_seqlen_k }, .i32); + // But here we correctly pass the number of used k per sequence + const seqused_k = token_index.addConstant(max_seqlen_q).convert(.i32); const original_tot = q.dim(.tot); - const num_heads = q_.dim(.h); - const num_heads_k = k_.dim(.h); - const head_size = q_.dim(.hd); - const ngroups = @divExact(num_heads, num_heads_k); - const seqlenq_ngroups_swapped = max_seqlen_q == 1 and num_heads > num_heads_k and @mod(head_size, 8) == 0 and parameters.sliding_window < 0; + const num_heads: i32 = @intCast(q_.dim(.h)); + const num_k_heads = k_.dim(.h); + const head_dim = q.dim(.hd); + const ngroups = @divExact(num_heads, num_k_heads); + const seqlenq_ngroups_swapped = max_seqlen_q == 1 and num_heads > num_k_heads and @mod(head_dim, 8) == 0 and parameters.sliding_window < 0; if (seqlenq_ngroups_swapped) { - q = q.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .tot, .ngroups, .h, .hd }).merge(.{ .tot = .{ .tot, .ngroups } }); + q = q.splitAxis(.h, .{ .h = num_k_heads, .ngroups = ngroups }) + .transpose(.{ .tot, .ngroups, .h, .hd }) + .merge(.{ .tot = .{ .tot, .ngroups } }); } const q_sharded = q.withPartitioning(.{ .h = .model }); - const model_partitions = ctx.partitioning.numPartitionsForLogicalAxis(q_sharded.shape(), .model) catch unreachable; + const model_partitions: i32 = @intCast(ctx.partitioning.numPartitionsForLogicalAxis(q_sharded.shape(), .model) catch std.debug.panic("cu_fa2 attention backend requires a .model sharding", .{})); const output = fa2_mha_varlen_fwd.call( .{ .q = q_sharded, - .k = k, - .v = v, + .k = k.withPartitioning(.{ .h = .model }), + .v = v.withPartitioning(.{ .h = .model }), .cu_seqlens_q = cu_seqlens_q, - .cu_seqlens_k = cu_seqlens_k.withTags(.{.i}).withPartitioning(.{ .i = .replicated }), + .cu_seqlens_k = cu_seqlens_k, + .seqused_k = seqused_k, .softmax_lse = metadata.softmax_lse.withPartitioning(.{ .h = .model }), .softmax_lse_accum = metadata.softmax_lse_accum.withPartitioning(.{ .h = .model }), .out_accum = metadata.out_accum.withPartitioning(.{ .h = .model }), @@ -316,25 +332,27 @@ pub const fa2 = struct { .o = q_sharded.shape(), }, .{ - .softmax_scale = b: { - const head_dim = q.shape().dim(2); - break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); - }, + .softmax_scale = 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))), .is_causal = true, .window_size_left = parameters.sliding_window, - .window_size_right = @as(i32, -1), + .window_size_right = -1, .max_seqlen_q = max_seqlen_q, .max_seqlen_k = max_seqlen_k, - .num_heads = @as(i32, @intCast(@divExact(num_heads, model_partitions))), + .num_heads = @divExact(num_heads, model_partitions), }, ); var o = output.o; if (seqlenq_ngroups_swapped) { - o = o.splitAxis(.tot, .{ .tot = original_tot, .ngroups = ngroups }).transpose(.{ .tot, .h, .ngroups, .hd }).merge(.{ .h = .{ .h, .ngroups } }); + o = o.splitAxis(.tot, .{ .tot = original_tot, .ngroups = ngroups }) + .transpose(.{ .tot, .h, .ngroups, .hd }) + .merge(.{ .h = .{ .h, .ngroups } }); } - return o.splitAxis(.tot, .{ .b = 1, .q = q_.dim(.q) }).squeeze(.b); + return if (q_.shape().hasTag(.b)) |_| + o.splitAxis(.tot, .{ .b = bs, .q = q_.dim(.q) }) + else + o.rename(.{ .tot = .q }); } }; diff --git a/zml/constants.zig b/zml/constants.zig index ad8ff445..39586fa4 100644 --- a/zml/constants.zig +++ b/zml/constants.zig @@ -13,3 +13,6 @@ pub const MINOR_TO_MAJOR: [MAX_RANK]i64 = b: { pub fn minorToMajor(rank: u8) []const i64 { return MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - rank ..]; } + +pub const ones_i64: [MAX_RANK]i64 = @splat(1); +pub const zeroes_i64: [MAX_RANK]i64 = @splat(0); diff --git a/zml/ops.zig b/zml/ops.zig index 31bd71ee..a695719d 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -307,26 +307,35 @@ pub const ReduceWindowOpts = struct { padding: []const [2]i64, }; -pub fn reduceWindow(inputs: anytype, inits: anytype, opts: ReduceWindowOpts, comptime func: anytype, context: anytype) stdx.meta.FnReturn(func) { +pub fn ReduceWindowFn(N: comptime_int) type { + return @Fn( + &@as([N]type, @splat(ReduceArgs)), + &@as([N]std.builtin.Type.Fn.Param.Attributes, @splat(.{})), + [N]Tensor, + .{}, + ); +} + +pub fn reduceWindow(N: comptime_int, inputs: [N]Tensor, inits: [N]Tensor, opts: ReduceWindowOpts, func: *const ReduceWindowFn(N)) [N]Tensor { var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); defer arena.deinit(); const mlir_ctx = CompilationContext.current().mlir_ctx; const reduce_block, var result = b: { - const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inits.len); - var args: ArgsType = undefined; - var block_types: [2 * inits.len]*const mlir.Type = undefined; + const Args = @Tuple(&@as([N]type, @splat(ReduceArgs))); + var args: Args = undefined; + var block_types: [2 * N]*const mlir.Type = undefined; - inline for (0..inits.len) |i| { + inline for (0..N) |i| { args[i].left = .fromShape(inits[i].shape()); args[i].right = .fromShape(inits[i].shape()); block_types[i] = mlirx.Type.rankedTensor(mlir_ctx, args[i].left.shape()); - block_types[i + inits.len] = mlirx.Type.rankedTensor(mlir_ctx, args[i].right.shape()); + block_types[i + N] = mlirx.Type.rankedTensor(mlir_ctx, args[i].right.shape()); } - const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); + const block_locs: [2 * N]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); const reduce_block = mlir.Block.init(&block_types, &block_locs); errdefer reduce_block.deinit(); @@ -334,15 +343,15 @@ pub fn reduceWindow(inputs: anytype, inits: anytype, opts: ReduceWindowOpts, com defer CompilationContext.current().popBlock(); const scope = CompilationContext.current().currentScope(); - inline for (0..inits.len) |i| { + inline for (0..N) |i| { scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; - scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inits.len) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + N) catch unreachable; } - var result = @call(.auto, func, args ++ context); + var result = @call(.auto, func, args); - var result_values: [inits.len]*const mlir.Value = undefined; - inline for (0..inits.len) |i| { + var result_values: [N]*const mlir.Value = undefined; + inline for (0..N) |i| { result_values[i] = result[i].value(); } diff --git a/zml/platform.zig b/zml/platform.zig index 4a4e2fb6..9e172e06 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -566,8 +566,17 @@ pub const Platform = struct { /// Create a Sharding based on the given logical mesh and the default strategy. /// Memory is owned by the platform, making it safe to copy around. - pub fn registerSharding(platform: *Platform, name: []const u8, logical: Sharding.LogicalMesh) !Sharding { - return platform.registerShardingWithStrategy(name, logical, .suggest(logical, &platform.physical_mesh)); + pub fn registerSharding(platform: *Platform, name: []const u8, logical: Sharding.LogicalMesh) error{OutOfMemory}!Sharding { + return platform.registerShardingWithStrategy( + name, + logical, + .suggest(logical, &platform.physical_mesh), + ) catch |err| switch (err) { + error.InvalidPhysicalMesh, error.InvalidStrategy, error.InvalidPhysicalAxis => { + std.debug.panic("ZML failed to create a valid sharding for logical mesh: {f}\nand physical_mesh: {f}\nPlease report this bug.", .{ logical, platform.physical_mesh }); + }, + error.OutOfMemory => |e| return e, + }; } /// Create a Sharding based on the given logical mesh and a strategy. @@ -820,7 +829,7 @@ fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { const slice: zml.Slice = .init(shape, host_visible_data[0..shape.byteSize()]); const name = call_frame.attrs.getByName(.string, "name").?.slice(); - log.info("{s} {f} [device={d}]: {d}", .{ name, slice.shape, device_ordinal, slice }); + std.debug.print("{s} {f} [device={d}]: {d}\n", .{ name, slice.shape, device_ordinal, slice }); return null; } diff --git a/zml/shape.zig b/zml/shape.zig index 467f8319..40e1fe07 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -573,12 +573,13 @@ pub const Shape = struct { var new_shape: Shape = .{ ._dtype = self.dtype() }; new_shape._dims, new_shape._tags = parseDimensions(new_shape_); new_shape._partitioning.appendNTimes(.unknown, new_shape._dims.len) catch @panic("Rank too large"); - new_shape.inferMissingAxis(self.count()); - stdx.debug.assert(self.count() == new_shape.count(), "Can't reshape {any} to {any}", .{ self.dims(), new_shape.dims() }); + new_shape.inferMissingAxis(self.count()) catch |err| { + std.debug.panic("Can't reshape {any} to {any}: {t}", .{ self.dims(), new_shape.dims(), err }); + }; return new_shape; } - fn inferMissingAxis(self: *Shape, n_: usize) void { + fn inferMissingAxis(self: *Shape, n_: usize) error{ UnexpectedRemainder, TooBig }!void { stdx.debug.assert(std.mem.count(i64, self.dims(), &.{-1}) < 2, "Cannot infer multiple dimensions when reshaping to: {f}", .{self.*}); const inferred_ax = std.mem.indexOfScalar(i64, self.dims(), -1) orelse return; @@ -590,10 +591,10 @@ pub const Shape = struct { } } const n: i64 = @intCast(n_); + if (tmp_count > n) return error.TooBig; // Abort, `reshape` will panic with more context. - if (@mod(n, tmp_count) != 0) { - return; - } + if (@mod(n, tmp_count) != 0) return error.UnexpectedRemainder; + self._dims.set(inferred_ax, @divExact(n, tmp_count)); } @@ -611,6 +612,11 @@ pub const Shape = struct { const res = x.reshape(.{-1}); try testing.expectEqualSlices(i64, &.{30}, res.dims()); } + { + var y: Shape = .init(.{ 2, 5, -1 }, .f32); + try testing.expectError(error.UnexpectedRemainder, inferMissingAxis(&y, 15)); + try testing.expectError(error.TooBig, inferMissingAxis(&y, 8)); + } } pub fn setDim(self: Shape, ax: anytype, d: i64) Shape { @@ -1329,20 +1335,21 @@ pub const Shape = struct { new_shape._partitioning.insert(ax, .unknown) catch unreachable; } - new_shape.inferMissingAxis(self.count()); + new_shape.inferMissingAxis(self.count()) catch |err| { + std.debug.panic("Can't split {any} along axis {d} into {any}: {t}", .{ self.dims(), ax, new_shape.dims(), err }); + }; + return new_shape; } test splitAxis { - try testing.expect( - Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).eql( - Shape.init(.{ .a = 10, .b = 3 }, .f32).splitAxis(.a, .{ .a1 = 5, .a2 = 2 }), - ), + try expectEqualShapes( + .init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32), + .splitAxis(.init(.{ .a = 10, .b = 3 }, .f32), .a, .{ .a1 = 5, .a2 = 2 }), ); - try testing.expect( - Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).eql( - Shape.init(.{ .a = 10, .b = 3 }, .f32).splitAxis(.a, .{ .a1 = .auto, .a2 = 2 }), - ), + try expectEqualShapes( + .init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32), + .splitAxis(.init(.{ .a = 10, .b = 3 }, .f32), .a, .{ .a1 = .auto, .a2 = 2 }), ); } @@ -1742,3 +1749,10 @@ test "MultiDimIterator / Shape with dimension of size 1" { } try std.testing.expectEqual(6, count); } + +fn expectEqualShapes(expected: Shape, actual: Shape) error{TestExpectedEqual}!void { + if (expected.eqlWithTags(actual)) return; + + std.debug.print("Expected {f}, got {f}", .{ expected, actual }); + return error.TestExpectedEqual; +} diff --git a/zml/slice.zig b/zml/slice.zig index 1fb363e5..a447a2a9 100644 --- a/zml/slice.zig +++ b/zml/slice.zig @@ -38,6 +38,17 @@ pub const Slice = struct { }; } + pub fn initConst(shape: Shape, bytes: []const u8) Slice { + stdx.debug.assert(bytes.len == shape.byteSize(), "Expected \"bytes\" to have the same length as shape.byteSize() ({}), got {}", .{ shape.byteSize(), bytes.len }); + return .{ + .bytes = bytes, + .mutable = false, + .shape = shape, + .offset_bytes = 0, + .byte_strides = shape.computeByteStrides(), + }; + } + pub fn alloc(allocator: std.mem.Allocator, shape_: Shape) !Slice { const size = shape_.byteSize(); const bytes: []u8 = switch (shape_.dtype().alignOf()) { diff --git a/zml/tensor.zig b/zml/tensor.zig index a378171e..be2314ef 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1586,30 +1586,28 @@ pub const Tensor = struct { const rk = self.rank(); const a = self.axis(axis_); - const ones = [_]i64{1} ** constants.MAX_RANK; - var window_dimensions = ones; + var window_dimensions = constants.ones_i64; window_dimensions[a] = self.dim(a); - var padding = [_][2]i64{.{ 0, 0 }} ** constants.MAX_RANK; + var padding: [constants.MAX_RANK][2]i64 = @splat(.{ 0, 0 }); padding[a] = .{ self.dim(a) - 1, 0 }; const result = ops.reduceWindow( + 1, .{self}, - .{Tensor.scalar(0, self.dtype())}, + .{.scalar(0, self.dtype())}, .{ - .base_dilations = ones[0..rk], - .window_dilations = ones[0..rk], - .window_strides = ones[0..rk], + .base_dilations = constants.ones_i64[0..rk], + .window_dilations = constants.ones_i64[0..rk], + .window_strides = constants.ones_i64[0..rk], .window_dimensions = window_dimensions[0..rk], .padding = padding[0..rk], }, struct { - fn add(values: ops.ReduceArgs) struct { Tensor } { - return .{values.left.add(values.right)}; + fn add(values: ops.ReduceArgs) [1]Tensor { + return .{.add(values.left, values.right)}; } }.add, - .{}, ); - return result[0]; } @@ -1639,7 +1637,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, @ptrCast(&[2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } })); defer x_buffer.deinit(); var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._cumsum, .{x_buffer}); @@ -2130,7 +2128,7 @@ pub const Tensor = struct { end: i64, step: i64 = 1, }; - /// + /// Returns a Tensor containing evenly spaced values within a given interval. pub fn arange(args: ArangeArgs, dt: DataType) Tensor { stdx.debug.assert(args.start <= args.end, "arange expects 'args.start' to be less than 'args.end', got {} and {}", .{ args.start, args.end }); @@ -2157,6 +2155,30 @@ pub const Tensor = struct { return res; } + test arange { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.arange, + .{ .{ .start = 2, .end = 26, .step = 4 }, .u32 }, + platform, + .{}, + ); + defer exe.deinit(); + + const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.arange, {}); + + try zml.testing.expectClose( + std.testing.io, + zml.Slice.initConst(.init(.{6}, .u32), @ptrCast(&[_]u32{ 2, 6, 10, 14, 18, 22 })), + output, + .exact_match, + ); + } + /// Returns a Tensor containing values in increasing order starting from 0 along the given axis. /// /// The output dtype will be `.i32`, unless the given axis has a too big dimension, in that case we use `.i64`. @@ -3066,7 +3088,7 @@ pub const Tensor = struct { values: Tensor, indices: Tensor, - fn cmp(values: ops.ReduceArgs, indices: ops.ReduceArgs) struct { Tensor, Tensor } { + fn cmp(values: ops.ReduceArgs, indices: ops.ReduceArgs) [2]Tensor { const left_gt_right = values.left.cmp(.GT, values.right); const is_nan = values.left.cmp(.NE, values.left); const left_gt_or_nan = left_gt_right.logical(.OR, is_nan); @@ -3365,9 +3387,10 @@ pub const Tensor = struct { var padding = [_][2]i64{.{ 0, 0 }} ** constants.MAX_RANK; padding[a] = opts.padding; - const result = ops.reduceWindow( + const values, const indices = ops.reduceWindow( + 2, .{ self, iota(self._shape, a) }, - .{ Tensor.constant(self.dtype().minValue()), Tensor.scalar(0, .i32) }, + .{ .constant(self.dtype().minValue()), .scalar(0, .i32) }, .{ .window_dimensions = window_dimensions[0..self.rank()], .window_strides = window_strides[0..self.rank()], @@ -3375,10 +3398,9 @@ pub const Tensor = struct { .window_dilations = window_dilations[0..self.rank()], .padding = padding[0..self.rank()], }, - MaxPoolRes.cmp, - .{}, + &MaxPoolRes.cmp, ); - return .{ .values = result[0], .indices = result[1] }; + return .{ .values = values, .indices = indices }; } /// Computes the 2d maxPool operation on the input Tensor. @@ -3405,9 +3427,10 @@ pub const Tensor = struct { padding[a - 1] = opts.padding[0]; padding[a] = opts.padding[1]; - const result = ops.reduceWindow( + const values, const indices = ops.reduceWindow( + 2, .{ self, iota(self._shape, a) }, - .{ Tensor.constant(self.dtype().minValue()), Tensor.scalar(0, .i32) }, + .{ .constant(self.dtype().minValue()), .scalar(0, .i32) }, .{ .window_dimensions = window_dimensions[0..self.rank()], .window_strides = window_strides[0..self.rank()], @@ -3415,10 +3438,9 @@ pub const Tensor = struct { .window_dilations = window_dilations[0..self.rank()], .padding = padding[0..self.rank()], }, - MaxPoolRes.cmp, - .{}, + &MaxPoolRes.cmp, ); - return .{ .values = result[0], .indices = result[1] }; + return .{ .values = values, .indices = indices }; } /// Chunk a given tensor into exactly n parts of equal shape. @@ -4407,7 +4429,7 @@ pub const Tensor = struct { fn initPoolArg(rank: usize, data: []const i64) [constants.MAX_RANK]i64 { // TODO use shape - var result = [_]i64{1} ** constants.MAX_RANK; + var result: [constants.MAX_RANK]i64 = @splat(1); const start = rank - data.len; @memcpy(result[start .. start + data.len], data); return result; @@ -4415,7 +4437,7 @@ fn initPoolArg(rank: usize, data: []const i64) [constants.MAX_RANK]i64 { fn getPoolResDims(dt: DataType, in_dims: []const i64, base_dilations: @Vector(constants.MAX_RANK, i64), padding: []const i64, window_dimensions: @Vector(constants.MAX_RANK, i64), window_dilations: @Vector(constants.MAX_RANK, i64), window_strides: @Vector(constants.MAX_RANK, i64)) Shape { // TODO use shape - var input_dims = [_]i64{1} ** constants.MAX_RANK; + var input_dims: [constants.MAX_RANK]i64 = @splat(1); @memcpy(input_dims[0..in_dims.len], in_dims); const input_dims_: @Vector(constants.MAX_RANK, i64) = input_dims; @@ -4438,7 +4460,7 @@ fn getPoolResDims(dt: DataType, in_dims: []const i64, base_dilations: @Vector(co } fn getComparisonType(ctx: *mlir.Context, dtype: DataType) *const dialects.stablehlo.CompareType { - return dialects.stablehlo.CompareType.init(ctx, switch (dtype.class()) { + return .init(ctx, switch (dtype.class()) { .bool => .UNSIGNED, .integer => if (dtype.isSignedInt()) .SIGNED else .UNSIGNED, .float => .FLOAT, From 577d6452bf674dfb75343ea7ff4b40fe3a52f7cb Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 17 Jul 2026 18:41:18 +0200 Subject: [PATCH 584/881] zml/io: rework `zml.io.load` (#586) The old `zml.io.load` was plagued with problems. This PR is an attempt at improving the state of things. One major problem of `zml.io.load` was that it was not very composable. It was either using it to load all the model or doing everything by hand. Another problem was that it didn't offer a way to apply transformations at load time to the tensors stored in a safetensors file. For example, it happens that for MoE models, instead of the experts weights to be tightly packed in a nice 3D tensor, the weight file contain one tensor for each expert. This is nice until your implementation required a 3D tensor as input. Using the old `zml.io.load` API this resulted in very convoluted way to do concatenation at load time. With this new API, we offer convenience so that it can be handled nicely and as fast as possible. The main change is the introduction of the `Loader` struct. It's a struct that you instantiate once and reuse whenever you want to load tensors. It relies on `Binding`, which are a way to store which tensor are required and how they need to be combined (if it's a complex binding involving more than one source tensor). One advantage of this `Loader` is that it's composable. Loading a struct using `Loader.auto(Struct)` or loading each individual field of `Struct` has essentially the same behavior. This solved the major problem of the old solution. For simplicity I kept the old model loading code almost identical, as this new solution is a drop-in replacement. # Update After some discussions with @gwenzek I've changed my mind. Instead of having a declarative API, we'll provide helper to do transformation easily. The main changes that this PR now bring are: * Introduce `Loader` struct * `Loader` exposes `load` which is the composable replacement of `zml.io.load` * `Loader` exposes `loadExecute` that allows you to execute an executable with the buffers loaded from the sources associated with a tensor. This allows for example to concatenate tensors. We'll continue to work on it, this is just a preliminary work that allows us to do transformation at load time in LLMD. --- examples/io/main.zig | 22 +- examples/llm/models/lfm2/model.zig | 62 +-- examples/llm/models/llama/model.zig | 58 +-- examples/llm/models/qwen3_5/model.zig | 56 +-- examples/llm/models/qwen3_5_moe/model.zig | 58 +-- examples/mnist/mnist.zig | 15 +- zml/io.zig | 490 ++++++++++++++-------- 7 files changed, 457 insertions(+), 304 deletions(-) diff --git a/examples/io/main.zig b/examples/io/main.zig index d2c154ba..d536b750 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -200,21 +200,27 @@ pub fn main(init: std.process.Init) !void { defer progress.end(); const now: std.Io.Timestamp = .now(io, .awake); - var total_bytes: usize = 0; + + var buffers = try zml.mem.bufferize(allocator, AllTensorsModel, &model); defer { - const took = now.untilNow(io, .awake); - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + for (buffers.tensors) |*b| b.deinit(); + allocator.free(buffers.tensors); } - _ = try zml.io.load(AllTensorsModel, &model, init.arena.allocator(), io, platform, &store, .{ - .shardings = &.{sharded_sharding}, + var loader: zml.io.Loader = try .init(allocator, platform, .{ .parallelism = 8, .dma_chunks = 16, .dma_chunk_size = 256 * zml.MiB, - .progress = &progress, - .total_bytes = &total_bytes, }); + defer loader.deinit(); + + loader.load(io, AllTensorsModel, &model, &buffers, &store, &.{sharded_sharding}, .{ .progress = &progress }); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); }, } } diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index e31d9a38..bcc25fdc 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -81,27 +81,29 @@ pub const LoadedModel = struct { progress: *std.Progress.Node, shardings: common.Shardings, ) !Buffers { + progress.increaseEstimatedTotalItems(store.view().count()); const now: std.Io.Timestamp = .now(io, .awake); - var total_bytes: usize = 0; - defer { - const took = now.untilNow(io, .awake); - const bytes_per_sec: u64 = @intFromFloat( - @as(f64, @floatFromInt(total_bytes)) / - (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s), - ); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - } + var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); + errdefer self.unloadBuffers(&buffers, allocator); - const all_shardings = shardings.all(); - return zml.io.load(Model, &self.inner, allocator, io, platform, store, .{ + var loader: zml.io.Loader = try .init(allocator, platform, .{ .dma_chunks = 32, .dma_chunk_size = 256 * zml.MiB, - .progress = progress, - .shardings = &all_shardings, .parallelism = 16, - .total_bytes = &total_bytes, }); + defer loader.deinit(); + + const all_shardings = shardings.all(); + loader.load(io, Model, &self.inner, &buffers, store, &all_shardings, .{ .progress = progress }); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; } pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { @@ -175,25 +177,27 @@ pub const Model = struct { ) !zml.Bufferized(Model) { progress.increaseEstimatedTotalItems(store.view().count()); const now: std.Io.Timestamp = .now(io, .awake); - var total_bytes: usize = 0; - defer { - const took = now.untilNow(io, .awake); - const took_ns: usize = @max(1, @as(usize, @intCast(took.toNanoseconds()))); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ - total_bytes, - took, - total_bytes * std.time.ns_per_s / took_ns, - }); - } - const all_shardings = shardings.all(); - return zml.io.load(Model, self, allocator, io, platform, store, .{ + + var buffers = try zml.mem.bufferize(allocator, Model, self); + errdefer Model.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ .dma_chunks = 8, .dma_chunk_size = 256 * zml.MiB, - .progress = progress, .parallelism = 16, - .total_bytes = &total_bytes, - .shardings = &all_shardings, }); + defer loader.deinit(); + + const all_shardings = shardings.all(); + loader.load(io, Model, self, &buffers, store, &all_shardings); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; } pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index c63fbb98..d7194c2b 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -6,6 +6,8 @@ const stdx = zml.stdx; const common = @import("../common.zig"); const inference = @import("inference.zig"); +const log = std.log.scoped(.llama); + pub const Config = struct { bos_token_id: u32, eos_token_id: stdx.json.Union(union(enum) { @@ -71,24 +73,27 @@ pub const LoadedModel = struct { ) !Buffers { progress.increaseEstimatedTotalItems(store.view().count()); const now: std.Io.Timestamp = .now(io, .awake); - var total_bytes: usize = 0; - defer { - const took = now.untilNow(io, .awake); - const bytes_per_sec: u64 = @intFromFloat( - @as(f64, @floatFromInt(total_bytes)) / - (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s), - ); - std.log.scoped(.llama).info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - } - return zml.io.load(Model, &self.inner, allocator, io, platform, store, .{ + var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); + errdefer self.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ .dma_chunks = 32, .dma_chunk_size = 256 * zml.MiB, - .progress = progress, - .shardings = &shardings.all(), .parallelism = 16, - .total_bytes = &total_bytes, }); + defer loader.deinit(); + + const all_shardings = shardings.all(); + loader.load(io, Model, &self.inner, &buffers, store, &all_shardings, .{ .progress = progress }); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; } pub fn unloadBuffers(_: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { @@ -161,25 +166,26 @@ pub const Model = struct { ) !zml.Bufferized(Model) { progress.increaseEstimatedTotalItems(store.view().count()); const now: std.Io.Timestamp = .now(io, .awake); - var total_bytes: usize = 0; - defer { - const took = now.untilNow(io, .awake); - const bytes_per_sec: u64 = @intFromFloat( - @as(f64, @floatFromInt(total_bytes)) / - (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s), - ); - std.log.scoped(.llama).info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - } + var buffers = try zml.mem.bufferize(allocator, Model, self); + errdefer Model.unloadBuffers(&buffers, allocator); - return zml.io.load(Model, self, allocator, io, platform, store, .{ + var loader: zml.io.Loader = try .init(allocator, platform, .{ .dma_chunks = 32, .dma_chunk_size = 256 * zml.MiB, - .progress = progress, - .shardings = shardings, .parallelism = 16, - .total_bytes = &total_bytes, }); + defer loader.deinit(); + + loader.load(io, Model, self, &buffers, store, shardings); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; } pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 4191965a..d7f33270 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -85,25 +85,27 @@ pub const LoadedModel = struct { ) !Buffers { progress.increaseEstimatedTotalItems(store.view().count()); const now: std.Io.Timestamp = .now(io, .awake); - var total_bytes: usize = 0; - defer { - const took = now.untilNow(io, .awake); - const bytes_per_sec: u64 = @intFromFloat( - @as(f64, @floatFromInt(total_bytes)) / - (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s), - ); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - } - const all_shardings = shardings.all(); - return zml.io.load(Model, &self.inner, allocator, io, platform, store, .{ + var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); + errdefer self.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ .dma_chunks = 32, .dma_chunk_size = 256 * zml.MiB, - .progress = progress, - .shardings = &all_shardings, .parallelism = 16, - .total_bytes = &total_bytes, }); + defer loader.deinit(); + + const all_shardings = shardings.all(); + loader.load(io, Model, &self.inner, &buffers, store, &all_shardings, .{ .progress = progress }); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; } pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { @@ -183,20 +185,26 @@ pub const Model = struct { ) !zml.Bufferized(Model) { progress.increaseEstimatedTotalItems(store.view().count()); const now: std.Io.Timestamp = .now(io, .awake); - var total_bytes: usize = 0; - defer { - const took = now.untilNow(io, .awake); - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - } - return zml.io.load(Model, self, allocator, io, platform, store, .{ + + var buffers = try zml.mem.bufferize(allocator, Model, self); + errdefer Model.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ .dma_chunks = 32, .dma_chunk_size = 256 * zml.MiB, - .progress = progress, - .shardings = shardings, .parallelism = 16, - .total_bytes = &total_bytes, }); + defer loader.deinit(); + + loader.load(io, Model, self, &buffers, store, shardings); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; } pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index f06b6586..3e34a098 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -6,7 +6,7 @@ const stdx = zml.stdx; const common = @import("../common.zig"); const inference = @import("inference.zig"); -const log = std.log.scoped(.qwen3_5); +const log = std.log.scoped(.qwen3_5_moe); pub const Config = struct { text_config: TextConfig, @@ -111,26 +111,26 @@ pub const LoadedModel = struct { ) !Buffers { progress.increaseEstimatedTotalItems(store.view().count()); const now: std.Io.Timestamp = .now(io, .awake); - var total_bytes: usize = 0; - defer { - const took = now.untilNow(io, .awake); - const bytes_per_sec: u64 = @intFromFloat( - @as(f64, @floatFromInt(total_bytes)) / - (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s), - ); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - } - const all_shardings = shardings.all(); - var buffers = try zml.io.load(Model, &self.inner, allocator, io, platform, store, .{ + var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); + errdefer self.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ .dma_chunks = 32, .dma_chunk_size = 256 * zml.MiB, - .progress = progress, - .shardings = &all_shardings, .parallelism = 16, - .total_bytes = &total_bytes, }); - errdefer TextModel.unloadBuffers(&buffers.text_model, allocator); + defer loader.deinit(); + + const all_shardings = shardings.all(); + loader.load(io, Model, &self.inner, &buffers, store, &all_shardings, .{ .progress = progress }); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + return buffers; } @@ -201,20 +201,26 @@ pub const Model = struct { ) !zml.Bufferized(Model) { progress.increaseEstimatedTotalItems(store.view().count()); const now: std.Io.Timestamp = .now(io, .awake); - var total_bytes: usize = 0; - defer { - const took = now.untilNow(io, .awake); - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - } - return zml.io.load(Model, self, allocator, io, platform, store, .{ + + var buffers = try zml.mem.bufferize(allocator, Model, self); + errdefer Model.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ .dma_chunks = 32, .dma_chunk_size = 256 * zml.MiB, - .progress = progress, - .shardings = shardings, .parallelism = 16, - .total_bytes = &total_bytes, }); + defer loader.deinit(); + + loader.load(io, Model, self, &buffers, store, shardings); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; } pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 718a2dd0..65de147b 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -43,11 +43,16 @@ const Mnist = struct { platform: *const zml.Platform, store: *const zml.io.TensorStore, ) !zml.Bufferized(Mnist) { - return zml.io.load(Mnist, self, allocator, io, platform, store, .{ - .parallelism = 1, - .dma_chunks = 1, - .dma_chunk_size = 16 * 1024 * 1024, - }); + var buffers = try zml.mem.bufferize(allocator, Mnist, self); + errdefer unloadBuffers(&buffers); + + var loader: zml.io.Loader = try .init(allocator, platform, .default); + errdefer loader.deinit(); + + loader.load(io, Mnist, self, &buffers, store, &.{}, .{}); + try loader.await(io); + + return buffers; } pub fn unloadBuffers(self: *zml.Bufferized(Mnist)) void { diff --git a/zml/io.zig b/zml/io.zig index 18bcf1af..81bd1ab9 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -4,6 +4,7 @@ const pjrt = @import("pjrt"); const stdx = @import("stdx"); pub const VFS = @import("io").VFS; +const Exe = @import("exe.zig").Exe; const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("zml.zig").Bufferized; const CreateOptions = @import("platform.zig").CreateOptions; @@ -24,31 +25,33 @@ const log = std.log.scoped(.@"zml/io"); pub const TensorStore = struct { registry: *safetensors.TensorRegistry, - id_map: std.AutoHashMapUnmanaged(usize, *safetensors.Tensor), + id_to_sources: std.AutoHashMapUnmanaged(usize, []*safetensors.Tensor), allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, pub fn fromRegistry(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry) TensorStore { + const arena: std.heap.ArenaAllocator = .init(allocator); return .{ .registry = registry, - .id_map = .empty, + .id_to_sources = .empty, .allocator = allocator, + .arena = arena, }; } pub fn deinit(self: *TensorStore) void { - self.id_map.deinit(self.allocator); + self.id_to_sources.deinit(self.allocator); + self.arena.deinit(); } - fn bindIdToKey(self: *TensorStore, key: []const u8, id: usize) !void { - const tensor_desc_ptr = self.registry.tensors.getPtr(key).?; - - const gop = try self.id_map.getOrPut(self.allocator, id); + fn putSourcesNoClobber(self: *TensorStore, id: usize, sources: []*safetensors.Tensor) std.mem.Allocator.Error!void { + const gop = try self.id_to_sources.getOrPut(self.allocator, id); if (gop.found_existing) { - stdx.debug.panic("Key {s} already has an associated tensor (id: {})", .{ key, gop.key_ptr.* }); + stdx.debug.panic("Id {} already has associated sources", .{id}); } - errdefer self.id_map.removeByPtr(gop.key_ptr); + errdefer self.id_to_sources.removeByPtr(gop.key_ptr); - gop.value_ptr.* = tensor_desc_ptr; + gop.value_ptr.* = sources; } fn getPtrFromKey(self: *const TensorStore, key: []const u8) ?*safetensors.Tensor { @@ -57,8 +60,9 @@ pub const TensorStore = struct { } fn getPtrFromId(self: *const TensorStore, id: usize) ?*safetensors.Tensor { - const tensor_desc_ptr = self.id_map.get(id) orelse return null; - return tensor_desc_ptr; + const sources = self.id_to_sources.get(id) orelse return null; + stdx.debug.assert(sources.len == 1, "Expect tensor with id {} to have only one source, got {}", .{ id, sources.len }); + return sources[0]; } pub fn getReader(self: *const TensorStore, key: []const u8, io: std.Io, buffer: []u8) !safetensors.TensorReader { @@ -66,9 +70,19 @@ pub const TensorStore = struct { } pub fn getReaderById(self: *const TensorStore, id: usize, io: std.Io, buffer: []u8) !safetensors.TensorReader { - const tensor_desc = self.id_map.get(id) orelse return error.NotFound; + const sources = self.id_to_sources.get(id) orelse return error.NotFound; + stdx.debug.assert(sources.len == 1, "Expect tensor with id {} to have only one source, got {}", .{ id, sources.len }); - return safetensors.TensorReader.init(io, tensor_desc.*, buffer, .{}); + return sources[0].reader(io, buffer, .{}); + } + + pub fn getSourcesById(self: *const TensorStore, id: usize) ?[]*safetensors.Tensor { + return self.id_to_sources.get(id); + } + + pub fn getShape(self: *const TensorStore, key: []const u8) ?Shape { + const entry_ptr = self.getPtrFromKey(key) orelse return null; + return entry_ptr.shape; } pub fn view(self: *TensorStore) View { @@ -101,7 +115,7 @@ pub const TensorStore = struct { pub fn withPrefix(self: *const View, prefix_: []const u8) View { var buffer: [256]u8 = undefined; - const new_prefix = std.fmt.bufPrint(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }) catch unreachable; + const new_prefix = makeKey(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }); return .{ .store = self.store, @@ -112,7 +126,7 @@ pub const TensorStore = struct { pub fn withLayer(self: *const View, index: usize) View { var buffer: [256]u8 = undefined; - const new_prefix = std.fmt.bufPrint(&buffer, "{s}{d}.", .{ self.prefix() orelse "", index }) catch unreachable; + const new_prefix = makeKey(&buffer, "{s}{d}.", .{ self.prefix() orelse "", index }); return .{ .store = self.store, @@ -127,7 +141,7 @@ pub const TensorStore = struct { pub fn hasKey(self: *const View, subkey: []const u8) bool { var buffer: [256]u8 = undefined; - const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); return for (self.store.registry.tensors.keys()) |k| { if (std.mem.startsWith(u8, k, key)) break true; } else false; @@ -135,33 +149,19 @@ pub const TensorStore = struct { pub fn maybeCreateTensor(self: View, subkey: []const u8, tagz: anytype, partitioning: anytype) ?Tensor { var buffer: [256]u8 = undefined; - const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); + const source = self.store.getPtrFromKey(key) orelse return null; - const ptr = self.store.getPtrFromKey(key) orelse return null; - if (@TypeOf(tagz) != @TypeOf(null)) { - switch (@typeInfo(@TypeOf(tagz))) { - .optional => if (tagz) |t| { - ptr.shape = ptr.shape.withTags(t); - }, - else => ptr.shape = ptr.shape.withTags(tagz), - } - } + const sources = self.store.arena.allocator().alloc(*safetensors.Tensor, 1) catch |e| std.debug.panic("Not handling {} errors", .{e}); + errdefer self.store.arena.allocator().free(sources); + sources[0] = source; - if (@TypeOf(partitioning) == @TypeOf(null)) { - @compileError("TensorStore.View.createTensor partitioning cannot be null; pass .replicated or an explicit partitioning"); - } + var shape = source.shape; + shape = applyTags(shape, tagz); + shape = applyPartitioning(shape, partitioning); - switch (@typeInfo(@TypeOf(partitioning))) { - .optional => @compileError("TensorStore.View.createTensor partitioning cannot be optional; pass .replicated or an explicit partitioning"), - .enum_literal => switch (partitioning) { - .replicated => ptr.shape = ptr.shape.withReplicatedPartitioning(), - else => @compileError("Only .replicated is supported as a standalone partitioning enum literal"), - }, - else => ptr.shape = ptr.shape.withPartitioning(partitioning), - } - - const tensor: Tensor = .fromShape(ptr.shape); - self.store.bindIdToKey(key, tensor.id) catch unreachable; + const tensor: Tensor = .fromShape(shape); + self.store.putSourcesNoClobber(tensor.id, sources) catch |e| std.debug.panic("Not handling {} errors", .{e}); return tensor; } @@ -170,11 +170,64 @@ pub const TensorStore = struct { return self.maybeCreateTensor(subkey, tagz, partitioning).?; } + fn applyTags(shape_: Shape, tagz: anytype) Shape { + var shape = shape_; + if (@TypeOf(tagz) != @TypeOf(null)) { + switch (@typeInfo(@TypeOf(tagz))) { + .optional => if (tagz) |t| { + shape = shape.withTags(t); + }, + else => shape = shape.withTags(tagz), + } + } + return shape; + } + + fn applyPartitioning(shape_: Shape, partitioning: anytype) Shape { + var shape = shape_; + + if (@TypeOf(partitioning) == @TypeOf(null)) { + @compileError("TensorStore.View.createTensor partitioning cannot be null; pass .replicated or an explicit partitioning"); + } + + switch (@typeInfo(@TypeOf(partitioning))) { + .optional => @compileError("TensorStore.View.createTensor partitioning cannot be optional; pass .replicated or an explicit partitioning"), + .enum_literal => switch (partitioning) { + .replicated => shape = shape.withReplicatedPartitioning(), + else => @compileError("Only .replicated is supported as a standalone partitioning enum literal"), + }, + else => shape = shape.withPartitioning(partitioning), + } + + return shape; + } + + pub fn maybeCreateBinding(self: View, sources: []const []const u8, shape: Shape) ?Tensor { + const arena = self.store.arena.allocator(); + + var tensor_list = std.ArrayList(*safetensors.Tensor).initCapacity(arena, sources.len) catch |e| std.debug.panic("Not handling {} errors", .{e}); + defer tensor_list.deinit(arena); + + var buffer: [256]u8 = undefined; + for (sources) |subkey| { + const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); + const tensor = self.store.getPtrFromKey(key) orelse return null; + tensor_list.appendAssumeCapacity(tensor); + } + + const tensors = tensor_list.toOwnedSlice(arena) catch unreachable; + errdefer arena.free(tensors); + + const tensor: Tensor = .fromShape(shape); + self.store.putSourcesNoClobber(tensor.id, tensors) catch |e| std.debug.panic("Not handling {} errors", .{e}); + + return tensor; + } + pub fn getShape(self: View, subkey: []const u8) ?Shape { var buffer: [256]u8 = undefined; - const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; - const entry_ptr = self.store.getPtrFromKey(key) orelse return null; - return entry_ptr.shape; + const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); + return self.store.getShape(key); } pub fn getShapeOpts(self: View, subkey: []const u8, opts: struct { no_prefix: bool = false }) ?Shape { @@ -182,15 +235,14 @@ pub const TensorStore = struct { const key = if (opts.no_prefix) subkey else b: { - break :b std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + break :b makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); }; - const entry_ptr = self.store.getPtrFromKey(key) orelse return null; - return entry_ptr.shape; + return self.store.getShape(key); } pub fn getReader(self: View, subkey: []const u8, io: std.Io, buffer: []u8) !safetensors.TensorReader { var key_buffer: [256]u8 = undefined; - const key = std.fmt.bufPrint(&key_buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + const key = makeKey(&key_buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); return self.store.getReader(key, io, buffer); } @@ -206,9 +258,213 @@ pub const TensorStore = struct { } return count_; } + + fn makeKey(buffer: []u8, comptime fmt: []const u8, args: anytype) []const u8 { + const key = std.fmt.bufPrint(buffer, fmt, args) catch + std.debug.panic("Expected key to be less than {} characters", .{buffer.len}); + return key; + } }; }; +pub const Loader = struct { + allocator: std.mem.Allocator, + platform: *const Platform, + dma_allocators: []const mem.DmaAllocator, + dma_chunk_size: usize, + pinned_buffer_pools: []mem.DynamicBufferPool, + group: stdx.Io.LimitedGroup, + bytes_loaded: std.atomic.Value(usize) = .init(0), + + pub const Opts = struct { + pub const default: Opts = .{ + .parallelism = 1, + .dma_chunks = 2, + .dma_chunk_size = 4096, + }; + parallelism: usize, + dma_chunks: usize, + dma_chunk_size: usize, + }; + + pub fn init(allocator: std.mem.Allocator, platform: *const Platform, opts: Opts) !Loader { + const pool_count = platform.devices.len; + const dma_allocators = try allocator.alloc(mem.DmaAllocator, pool_count); + errdefer allocator.free(dma_allocators); + for (platform.devices, 0..) |*device, i| { + dma_allocators[i] = .init(allocator, device); + } + + const buffer_pools = try allocator.alloc(mem.DynamicBufferPool, pool_count); + errdefer allocator.free(buffer_pools); + for (buffer_pools) |*pool_| { + pool_.* = .init(opts.dma_chunks, opts.dma_chunk_size); + } + errdefer for (buffer_pools, 0..) |*pool_, i| { + pool_.deinit(dma_allocators[i].allocator()); + }; + + return .{ + .allocator = allocator, + .platform = platform, + .dma_allocators = dma_allocators, + .dma_chunk_size = opts.dma_chunk_size, + .pinned_buffer_pools = buffer_pools, + .group = .init(opts.parallelism), + }; + } + + pub fn deinit(self: Loader) void { + for (self.pinned_buffer_pools, 0..) |*pool, i| pool.deinit(self.dma_allocators[i].allocator()); + self.allocator.free(self.pinned_buffer_pools); + self.allocator.free(self.dma_allocators); + } + + pub fn await(self: *Loader, io: std.Io) std.Io.Cancelable!void { + return self.group.await(io); + } + + pub const LoadOpts = struct { + progress: ?*std.Progress.Node = null, + }; + + pub fn load(self: *Loader, io: std.Io, comptime T: type, model: *const T, buffers: *Bufferized(T), store: *const TensorStore, shardings: []const Sharding, opts: LoadOpts) void { + const tensor_count = meta.count(Tensor, model); + + var arena: std.heap.ArenaAllocator = .init(self.allocator); + defer arena.deinit(); + + const flattened_buffers = arena.allocator().alloc(*Buffer, tensor_count) catch @panic("Errors can't be handled in `loadInner`"); + meta.forEachVisit(buffers, *Buffer, struct { + fn call(i: usize, buffer: *Buffer, flattened_buffers_: []*Buffer) void { + flattened_buffers_[i] = buffer; + } + }.call, .{flattened_buffers}); + + const Ctx = struct { + self: *Loader, + io: std.Io, + store: *const TensorStore, + shardings: []const Sharding, + buffers: []*Buffer, + opts: LoadOpts, + }; + + var ctx: Ctx = .{ + .self = self, + .io = io, + .store = store, + .shardings = shardings, + .buffers = flattened_buffers, + .opts = opts, + }; + + meta.forEachVisit(model, *const Tensor, struct { + fn call(i: usize, tensor: *const Tensor, ctx_: *Ctx) void { + ctx_.self.group.async(ctx_.io, defaultCallback, .{ ctx_.self, ctx_.io, tensor, ctx_.buffers[i], ctx_.store, ctx_.shardings, ctx_.opts }); + } + }.call, .{&ctx}); + } + + fn defaultCallback(self: *Loader, io: std.Io, tensor: *const Tensor, buffer: *Buffer, store: *const TensorStore, shardings: []const Sharding, opts: LoadOpts) void { + const sources = store.getSourcesById(tensor.id) orelse { + std.log.warn("Failed to get sources for tensor with id: {}", .{tensor.id}); + return; + }; + + if (sources.len != 1) { + std.debug.panic("Expected loaded tensor to have only 1 source, got {}", .{sources.len}); + } + + self.loadSingleInner(io, sources[0], buffer, shardings, opts) catch |e| { + log.err("Errors are not handled in `defaultCallback`, got {}", .{e}); + unreachable; + }; + } + + fn loadSingle(self: *Loader, io: std.Io, source: *safetensors.Tensor, buffer: *Buffer, loaded: *bool, shardings: []const Sharding, opts: LoadOpts) void { + self.loadSingleInner(io, source, buffer, shardings, opts) catch |e| { + log.err("Failed to load tensor {s}: {}", .{ source.name, e }); + loaded.* = false; + }; + loaded.* = true; + } + + fn loadSingleInner(self: *Loader, io: std.Io, source: *safetensors.Tensor, buffer: *Buffer, shardings: []const Sharding, opts: LoadOpts) !void { + var reader = try source.reader(io, &.{}, .{}); + defer reader.deinit(); + + const shape = reader.tensor.shape; + const sharding = Sharding.pickSharding(shardings, shape, .explicit_axis_binding) orelse blk: { + log.debug("No sharding strategy found for tensor {s} with shape {f}, using replicated sharding", .{ reader.tensor.name, shape }); + break :blk self.platform.replicated_sharding; + }; + + var writer = try MemoryWriter.init( + self.allocator, + io, + self.platform, + self.pinned_buffer_pools, + self.dma_allocators, + self.dma_chunk_size, + shape, + sharding, + buffer, + ); + defer writer.deinit(self.allocator); + + const scale = 1024; + + if (opts.progress) |progress| { + var node = progress.start(reader.tensor.name, reader.tensor.shape.byteSize() / scale); + defer node.end(); + writer.setProgress(&node); + defer writer.setProgress(null); + var progress_writer: ProgressWriter = .init(writer.interface(), &node, .{ .scale = scale }); + const total = try reader.interface.streamRemaining(&progress_writer.interface); + try progress_writer.interface.flush(); + _ = self.bytes_loaded.fetchAdd(total, .monotonic); + } else { + const total = try reader.interface.streamRemaining(writer.interface()); + try writer.interface().flush(); + _ = self.bytes_loaded.fetchAdd(total, .monotonic); + } + } + + pub fn loadExecute( + self: *Loader, + arena: std.mem.Allocator, + io: std.Io, + tensor: Tensor, + buffer: *Buffer, + store: *const TensorStore, + shardings: []const Sharding, + exe: *const Exe, + ) !void { + const sources = store.getSourcesById(tensor.id) orelse return error.NotFound; + const buffers = try arena.alloc(Buffer, sources.len); + const loaded = try arena.alloc(bool, sources.len); + @memset(loaded, false); + defer for (buffers, loaded) |*b, l| if (l) b.deinit(); + + for (sources, 0..) |source, i| { + self.loadSingle(io, source, &buffers[i], &loaded[i], shardings, .{}); + } + + if (std.mem.findScalar(bool, loaded, false)) |_| { + return error.LoadFailed; + } + + var args = try exe.args(arena); + var results = try exe.results(arena); + + args.set(.{buffers}); + exe.call(args, &results); + + buffer.* = results.get(Buffer); + } +}; + pub const ProgressWriter = struct { inner: *std.Io.Writer, progress: *std.Progress.Node, @@ -1055,144 +1311,6 @@ pub const DirectMemoryWriter = struct { } }; -pub const LoadOpts = struct { - pub const auto: LoadOpts = .{ - .parallelism = 1, - .shardings = &.{}, - .dma_chunks = 2, - .dma_chunk_size = 4096, - }; - - parallelism: usize, - shardings: []const Sharding = &.{}, - progress: ?*std.Progress.Node = null, - dma_chunks: usize, - dma_chunk_size: usize, - total_bytes: ?*usize = null, -}; - -pub fn load( - comptime ModelType: type, - model: *const ModelType, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - store: *const TensorStore, - opts: LoadOpts, -) !Bufferized(ModelType) { - var span = tracer.span("zml.io.load", .{ - .tensor_count = meta.count(Tensor, model), - }); - defer span.end(); - - var bufferized = try mem.bufferize(allocator, ModelType, model); - - const pool_count = platform.devices.len; - const dma_allocators = try allocator.alloc(mem.DmaAllocator, pool_count); - defer allocator.free(dma_allocators); - for (platform.devices, 0..) |*device, i| { - dma_allocators[i] = .init(allocator, device); - } - - const buffer_pools = try allocator.alloc(mem.DynamicBufferPool, pool_count); - defer allocator.free(buffer_pools); - for (buffer_pools) |*pool_| { - pool_.* = .init(opts.dma_chunks, opts.dma_chunk_size); - } - defer for (buffer_pools, 0..) |*pool_, i| { - pool_.deinit(dma_allocators[i].allocator()); - }; - - const Ctx = struct { - allocator: std.mem.Allocator, - dma_allocators: []const mem.DmaAllocator, - dma_chunk_size: usize, - pinned_buffer_pools: []mem.DynamicBufferPool, - io: std.Io, - platform: *const Platform, - buffers: []*Buffer, - shardings: []const Sharding, - store: *const TensorStore, - group: stdx.Io.LimitedGroup, - total: std.atomic.Value(usize) = .init(0), - progress: ?*std.Progress.Node, - }; - var walk_ctx: Ctx = .{ - .platform = platform, - .buffers = try allocator.alloc(*Buffer, meta.count(Tensor, model)), - .store = store, - .allocator = allocator, - .dma_allocators = dma_allocators, - .dma_chunk_size = opts.dma_chunk_size, - .pinned_buffer_pools = buffer_pools, - .io = io, - .shardings = opts.shardings, - .progress = opts.progress, - .group = .init(opts.parallelism), - }; - defer allocator.free(walk_ctx.buffers); - - defer if (opts.total_bytes) |total_bytes_ptr| { - total_bytes_ptr.* = walk_ctx.total.load(.monotonic); - }; - - meta.forEachVisit(&bufferized, *Buffer, struct { - fn call(i: usize, buffer: *Buffer, ctx: *Ctx) void { - ctx.buffers[i] = buffer; - } - }.call, .{&walk_ctx}); - - meta.forEachVisit(model, *const Tensor, struct { - fn call(i: usize, tensor: *const Tensor, ctx: *Ctx) void { - ctx.group.concurrent(ctx.io, struct { - fn call(i_: usize, tensor_: *const Tensor, ctx_: *Ctx) !void { - var reader = ctx_.store.getReaderById(tensor_.id, ctx_.io, &.{}) catch unreachable; - defer reader.deinit(); - - const shape = reader.tensor.shape; - const sharding = Sharding.pickSharding(ctx_.shardings, shape, .explicit_axis_binding) orelse blk: { - log.debug("No sharding strategy found for tensor {s} with shape {f}, using replicated sharding", .{ reader.tensor.name, shape }); - break :blk ctx_.platform.replicated_sharding; - }; - - var writer = MemoryWriter.init( - ctx_.allocator, - ctx_.io, - ctx_.platform, - ctx_.pinned_buffer_pools, - ctx_.dma_allocators, - ctx_.dma_chunk_size, - shape, - sharding, - ctx_.buffers[i_], - ) catch unreachable; - defer writer.deinit(ctx_.allocator); - - const scale = 1024; - - if (ctx_.progress) |progress| { - var node = progress.start(reader.tensor.name, reader.tensor.shape.byteSize() / scale); - defer node.end(); - writer.setProgress(&node); - defer writer.setProgress(null); - var progress_writer: ProgressWriter = .init(writer.interface(), &node, .{ .scale = scale }); - const total = reader.interface.streamRemaining(&progress_writer.interface) catch unreachable; - progress_writer.interface.flush() catch unreachable; - _ = ctx_.total.fetchAdd(total, .monotonic); - } else { - const total = reader.interface.streamRemaining(writer.interface()) catch unreachable; - writer.interface().flush() catch unreachable; - _ = ctx_.total.fetchAdd(total, .monotonic); - } - } - }.call, .{ i, tensor, ctx }) catch unreachable; - } - }.call, .{&walk_ctx}); - walk_ctx.group.await(io) catch unreachable; - - return bufferized; -} - fn buildMesh2x2( allocator: std.mem.Allocator, target: @import("platform.zig").Target, From 9c38bad5be34bdc11ba717f8bdf91af263d60391 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Fri, 17 Jul 2026 22:23:57 +0200 Subject: [PATCH 585/881] platforms/oneapi: update pjrt artifact with collectives fixes (#666) --- platforms/oneapi/oneapi.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index be022b15..66c113ff 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -1,8 +1,8 @@ load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") -PJRT_ONEAPI_RELEASE = "manual-2026-07-17T10-00-00Z" -PJRT_ONEAPI_ARTIFACT_SHA256 = "7b2883157ae108c09003c9d63a8a5d638adfbf9c511517b766c4cdbf4276b1bd" +PJRT_ONEAPI_RELEASE = "manual-2026-07-17T21-30-00Z" +PJRT_ONEAPI_ARTIFACT_SHA256 = "8e28d1ff00e2193a40982294d6b055a8102f49eb2cb46f32a6076f9bb541e8a2" PJRT_ONEAPI_ARTIFACT_URL = "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-oneapi_linux-amd64.tar.gz".format( release = PJRT_ONEAPI_RELEASE, ) From f4ffbbc0179f544cb6a66f5eafd55906a411e60d Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 17 Jul 2026 22:27:55 +0200 Subject: [PATCH 586/881] zml/io: use correct shape when loading tensor (#665) Small mistake happened while cleaning #586, I factorized code that shouldn't have been factorized blindly. This PR fixes a bug where tensor loaded using `Loader.load` were not correctly sharded. --- zml/io.zig | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/zml/io.zig b/zml/io.zig index 81bd1ab9..cd8a254a 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -376,25 +376,24 @@ pub const Loader = struct { std.debug.panic("Expected loaded tensor to have only 1 source, got {}", .{sources.len}); } - self.loadSingleInner(io, sources[0], buffer, shardings, opts) catch |e| { + self.loadSingleInner(io, sources[0], tensor.shape(), buffer, shardings, opts) catch |e| { log.err("Errors are not handled in `defaultCallback`, got {}", .{e}); unreachable; }; } - fn loadSingle(self: *Loader, io: std.Io, source: *safetensors.Tensor, buffer: *Buffer, loaded: *bool, shardings: []const Sharding, opts: LoadOpts) void { - self.loadSingleInner(io, source, buffer, shardings, opts) catch |e| { + fn loadSingle(self: *Loader, io: std.Io, source: *safetensors.Tensor, shape: Shape, buffer: *Buffer, loaded: *bool, shardings: []const Sharding, opts: LoadOpts) void { + self.loadSingleInner(io, source, shape, buffer, shardings, opts) catch |e| { log.err("Failed to load tensor {s}: {}", .{ source.name, e }); loaded.* = false; }; loaded.* = true; } - fn loadSingleInner(self: *Loader, io: std.Io, source: *safetensors.Tensor, buffer: *Buffer, shardings: []const Sharding, opts: LoadOpts) !void { + fn loadSingleInner(self: *Loader, io: std.Io, source: *safetensors.Tensor, shape: Shape, buffer: *Buffer, shardings: []const Sharding, opts: LoadOpts) !void { var reader = try source.reader(io, &.{}, .{}); defer reader.deinit(); - const shape = reader.tensor.shape; const sharding = Sharding.pickSharding(shardings, shape, .explicit_axis_binding) orelse blk: { log.debug("No sharding strategy found for tensor {s} with shape {f}, using replicated sharding", .{ reader.tensor.name, shape }); break :blk self.platform.replicated_sharding; @@ -448,7 +447,7 @@ pub const Loader = struct { defer for (buffers, loaded) |*b, l| if (l) b.deinit(); for (sources, 0..) |source, i| { - self.loadSingle(io, source, &buffers[i], &loaded[i], shardings, .{}); + self.loadSingle(io, source, source.shape, &buffers[i], &loaded[i], shardings, .{}); } if (std.mem.findScalar(bool, loaded, false)) |_| { From 9ecd60f4d496628b76e05ccc4b93b553ab5d19d1 Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Sun, 19 Jul 2026 11:12:00 +0200 Subject: [PATCH 587/881] nvim: merge configurations table instead of overwriting (#611) Update the local neovim configuration to override only the fields that needs to change (such as `cmd` for LSP) instead of replacing the user's entire configuration. The main difference is `vim.lsp.config("zls", { ... })` which merges the provided table into the existing configuration, while `vim.lsp.config["zls"] = { ... }` overwrite the LSP configuration for ZLS. I also fixed the same issue for conform. Signed-off-by: dmehala --- .nvim.lua | 104 +++++++++++++++++++++++++++--------------------------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/.nvim.lua b/.nvim.lua index 56c43b69..65995b81 100644 --- a/.nvim.lua +++ b/.nvim.lua @@ -4,63 +4,63 @@ local cwd = vim.fn.getcwd() -- Load the local config file if it exists local f, err = loadfile(".nvim.local.lua") if not err then - f() + local ok, err_f = pcall(f) + if not ok then + vim.print(err_f) + end else - vim.lsp.config["zls"] = { - cmd = { cwd .. "/tools/zls.sh" }, - } - vim.lsp.enable("zls") + vim.lsp.config("zls", { + cmd = { cwd .. "/tools/zls.sh" }, + }) + vim.lsp.enable("zls") - function runFixAll() - local params = vim.lsp.util.make_range_params() - params.context = { - only = { "source.fixAll" }, - diagnostics = {}, - } + function runFixAll() + local params = vim.lsp.util.make_range_params() + params.context = { + only = { "source.fixAll" }, + diagnostics = {}, + } - -- Send synchronous request (timeout 2 seconds) - local responses = vim.lsp.buf_request_sync(0, "textDocument/codeAction", params, 1000) - if not responses then return end + -- Send synchronous request (timeout 2 seconds) + local responses = vim.lsp.buf_request_sync(0, "textDocument/codeAction", params, 1000) + if not responses then + return + end - for _, resp in pairs(responses) do - for _, action in ipairs(resp.result or {}) do - -- If action has edits, apply them - if action.edit then - vim.lsp.util.apply_workspace_edit(action.edit, "utf-16") - end - -- If the action has a command, execute it - if action.command then - vim.lsp.buf.execute_command(action.command) - end - end - end - end + for _, resp in pairs(responses) do + for _, action in ipairs(resp.result or {}) do + -- If action has edits, apply them + if action.edit then + vim.lsp.util.apply_workspace_edit(action.edit, "utf-16") + end + -- If the action has a command, execute it + if action.command then + vim.lsp.buf.execute_command(action.command) + end + end + end + end - vim.api.nvim_create_autocmd('BufWritePre',{ - pattern = {"*.zig", "*.zon"}, - callback = runFixAll, - }) + vim.api.nvim_create_autocmd("BufWritePre", { + pattern = { "*.zig", "*.zon" }, + callback = runFixAll, + }) - local ok, mod = pcall(require, "conform") - if ok then - mod.setup({ - formatters_by_ft = { - bzl = { "buildifier" }, - }, - formatters = { - buildifier = { - command = "tools/buildifier.sh", - } - }, - }) + local ok, conform = pcall(require, "conform") - vim.api.nvim_create_autocmd("FileType", { - pattern = "bzl", - callback = function() - vim.keymap.set('n', 'f', function() - mod.format() - end, { remap = false, silent = true }) - end - }) - end + if ok then + conform.formatters_by_ft.bzl = { "buildifier" } + conform.formatters.buildifier = { + command = cwd .. "/tools/buildifier.sh", + } + + vim.api.nvim_create_autocmd("FileType", { + pattern = "bzl", + callback = function() + vim.keymap.set("n", "f", function() + mod.format() + end, { remap = false, silent = true }) + end, + }) + end end From d57d851b63f39c115876ef726bc9728b36f4878f Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 20 Jul 2026 13:31:54 +0200 Subject: [PATCH 588/881] zml: ground work to support Poolside's Laguna models (#669) This PR contains two main changes: * A patch for the tokenizer to support the tokenizer config file * A temporary workaround to speed-up loading of the model when executing arbitrary executable at load time --- .../iree/laguna-tokenizer-compat.patch | 78 +++++++++++++++++++ third_party/iree/repo.bzl | 1 + zml/io.zig | 3 +- zml/nn.zig | 21 +++++ zml/tokenizer/iree/tokenizer.zig | 66 ++++++++++++++++ 5 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 third_party/iree/laguna-tokenizer-compat.patch diff --git a/third_party/iree/laguna-tokenizer-compat.patch b/third_party/iree/laguna-tokenizer-compat.patch new file mode 100644 index 00000000..c978c591 --- /dev/null +++ b/third_party/iree/laguna-tokenizer-compat.patch @@ -0,0 +1,78 @@ +diff --git a/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c b/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c +--- a/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c ++++ b/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c +@@ -217,0 +218,10 @@ ++ // Some code-oriented tokenizers use this lookahead split only to attach ++ // single-newline runs to the next segment. The IREE regex compiler does not ++ // support lookahead; the following ByteLevel/GPT split still preserves the ++ // input bytes, so treat this grouping hint as a no-op instead of rejecting ++ // the tokenizer at load time. ++ if (iree_string_view_equal( ++ pattern, IREE_SV("(?:\\r?\\n)+(?!\\r?\\n)")) && ++ behavior == IREE_TOKENIZER_UTIL_REGEX_SPLIT_MERGED_WITH_NEXT && !invert) { ++ return iree_ok_status(); ++ } +diff --git a/runtime/src/iree/tokenizer/format/huggingface/model_json.c b/runtime/src/iree/tokenizer/format/huggingface/model_json.c +--- a/runtime/src/iree/tokenizer/format/huggingface/model_json.c ++++ b/runtime/src/iree/tokenizer/format/huggingface/model_json.c +@@ -444,0 +445,46 @@ ++static char iree_tokenizer_ascii_upper(char c) { ++ return c >= 'a' && c <= 'z' ? (char)(c - ('a' - 'A')) : c; ++} ++ ++static bool iree_tokenizer_string_view_contains_ascii_case( ++ iree_string_view_t value, iree_string_view_t needle) { ++ if (needle.size == 0 || needle.size > value.size) return false; ++ for (iree_host_size_t i = 0; i <= value.size - needle.size; ++i) { ++ bool match = true; ++ for (iree_host_size_t j = 0; j < needle.size; ++j) { ++ if (iree_tokenizer_ascii_upper(value.data[i + j]) != ++ iree_tokenizer_ascii_upper(needle.data[j])) { ++ match = false; ++ break; ++ } ++ } ++ if (match) return true; ++ } ++ return false; ++} ++ ++static bool iree_tokenizer_huggingface_find_added_unk_token( ++ const iree_tokenizer_huggingface_added_tokens_t* added_tokens, ++ iree_tokenizer_token_id_t* out_token_id) { ++ *out_token_id = IREE_TOKENIZER_TOKEN_ID_INVALID; ++ if (!added_tokens) return false; ++ for (iree_host_size_t i = 0; i < added_tokens->count; ++i) { ++ const iree_tokenizer_huggingface_added_token_t* token = ++ iree_tokenizer_huggingface_added_tokens_get(added_tokens, i); ++ if (!iree_any_bit_set( ++ token->flags, ++ IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_SPECIAL)) { ++ continue; ++ } ++ iree_string_view_t content = ++ iree_tokenizer_huggingface_added_token_content(added_tokens, token); ++ if (!iree_tokenizer_string_view_contains_ascii_case(content, ++ IREE_SV("unk"))) { ++ continue; ++ } ++ *out_token_id = token->id; ++ return true; ++ } ++ return false; ++} ++ +@@ -584,3 +630,10 @@ +- status = iree_make_status(IREE_STATUS_NOT_FOUND, +- "unk_token '%.*s' not found in vocabulary", +- (int)unk_length, unk_buffer); ++ // Some tokenizer.json files keep a stale default `unk_token` in the BPE ++ // model section while declaring the real unknown token in added_tokens ++ // (for example, custom bracketed forms such as `〈|UNK|〉`). Prefer that ++ // explicit special added token before failing the whole tokenizer load. ++ if (!iree_tokenizer_huggingface_find_added_unk_token(added_tokens, ++ &unk_token_id)) { ++ status = iree_make_status(IREE_STATUS_NOT_FOUND, ++ "unk_token '%.*s' not found in vocabulary", ++ (int)unk_length, unk_buffer); ++ } diff --git a/third_party/iree/repo.bzl b/third_party/iree/repo.bzl index d0e83ed5..860118ac 100644 --- a/third_party/iree/repo.bzl +++ b/third_party/iree/repo.bzl @@ -16,6 +16,7 @@ def repo(): "//third_party/iree:tokenizer-only.patch", "//third_party/iree:fix-added-token-matching.patch", "//third_party/iree:match-hf-tokenizer.patch", + "//third_party/iree:laguna-tokenizer-compat.patch", ], patch_args = ["-p1"], ) diff --git a/zml/io.zig b/zml/io.zig index cd8a254a..fb640920 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -447,8 +447,9 @@ pub const Loader = struct { defer for (buffers, loaded) |*b, l| if (l) b.deinit(); for (sources, 0..) |source, i| { - self.loadSingle(io, source, source.shape, &buffers[i], &loaded[i], shardings, .{}); + self.group.async(io, loadSingle, .{ self, io, source, source.shape, &buffers[i], &loaded[i], shardings, .{} }); } + try self.group.await(io); if (std.mem.findScalar(bool, loaded, false)) |_| { return error.LoadFailed; diff --git a/zml/nn.zig b/zml/nn.zig index f893a2cf..295028c6 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -550,6 +550,27 @@ test "RopeOpts.Scaling parses proportional" { } } +test "RopeOpts.Scaling parses yarn partial rotary factor" { + const json = + \\{ + \\ "rope_type": "yarn", + \\ "factor": 32.0, + \\ "original_max_position_embeddings": 4096, + \\ "partial_rotary_factor": 0.5 + \\} + ; + var value = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, json, .{}); + defer value.deinit(); + + const scaling = try RopeOpts.Scaling.jsonParseFromValue(std.testing.allocator, value.value, .{}); + switch (scaling) { + .yarn => |y| { + try std.testing.expectApproxEqRel(0.5, y.partial_rotary_factor, 1e-6); + }, + else => try std.testing.expect(false), + } +} + test "RopeOpts.Scaling parses default rope type" { const json = \\{ diff --git a/zml/tokenizer/iree/tokenizer.zig b/zml/tokenizer/iree/tokenizer.zig index ee64aca0..179ca9bd 100644 --- a/zml/tokenizer/iree/tokenizer.zig +++ b/zml/tokenizer/iree/tokenizer.zig @@ -67,6 +67,11 @@ inline fn statusCode(status: c.iree_status_t) u32 { fn checkOk(status: c.iree_status_t) !void { if (status == null) return; const code = statusCode(status); + var buffer: [1024]u8 = undefined; + var buffer_length: usize = 0; + if (c.iree_status_format(status, buffer.len, &buffer, &buffer_length)) { + log.err("IREE tokenizer error: {s}", .{buffer[0..@min(buffer_length, buffer.len - 1)]}); + } _ = c.iree_status_ignore(status); return switch (code) { c.IREE_STATUS_CANCELLED => Error.Cancelled, @@ -532,6 +537,67 @@ test "huggingface json encode/decode" { try std.testing.expectEqualStrings("hello world", decoded.written()); } +test "loads bpe tokenizer with newline lookahead split and added unk" { + const allocator = std.testing.allocator; + const json = + \\{ + \\ "model": { + \\ "type": "BPE", + \\ "unk_token": "[UNK]", + \\ "vocab": { + \\ "<|UNK|>": 0, + \\ "h": 1, + \\ "e": 2, + \\ "l": 3, + \\ "o": 4, + \\ "he": 5, + \\ "ll": 6, + \\ "lo": 7, + \\ "hel": 8, + \\ "hello": 9 + \\ }, + \\ "merges": ["h e", "l l", "l o", "he l", "hel lo"] + \\ }, + \\ "added_tokens": [ + \\ { + \\ "id": 0, + \\ "content": "<|UNK|>", + \\ "single_word": false, + \\ "lstrip": false, + \\ "rstrip": false, + \\ "normalized": false, + \\ "special": true + \\ } + \\ ], + \\ "pre_tokenizer": { + \\ "type": "Sequence", + \\ "pretokenizers": [ + \\ { + \\ "type": "Split", + \\ "pattern": {"Regex": "(?:\\r?\\n)+(?!\\r?\\n)"}, + \\ "behavior": "MergedWithNext", + \\ "invert": false + \\ }, + \\ {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": true} + \\ ] + \\ }, + \\ "decoder": {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true} + \\} + ; + + var tokenizer = try Tokenizer.fromBytes(allocator, json); + defer tokenizer.deinit(); + + try std.testing.expectEqual(@as(u32, 0), tokenizer.tokenId("<|UNK|>").?); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const ids = try encoder.encodeAlloc(allocator, "hello"); + defer allocator.free(ids); + try std.testing.expectEqualSlices(u32, &.{ 5, 6, 4 }, ids); +} + test "writer" { const allocator = std.testing.allocator; const json = From a35fac997e1c8ac90d817704111541d543869ec5 Mon Sep 17 00:00:00 2001 From: Louis Lechevalier Date: Mon, 20 Jul 2026 20:57:19 +0200 Subject: [PATCH 589/881] platforms/cuda: Bump CUDA to 13.3.0 (#672) Bump CUDA to 13.3.0 and CUDNN to 9.22. --- platforms/cuda/cuda.bzl | 45 ++++++++++++------------- platforms/cuda/libpjrt_cuda.BUILD.bazel | 4 --- 2 files changed, 22 insertions(+), 27 deletions(-) diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index d75edc23..0046e650 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -6,17 +6,17 @@ load("//platforms:packages.bzl", "packages") ARCHS = ["linux-x86_64", "linux-sbsa"] CUDA_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cuda/redist/" -CUDA_VERSION = "13.1.1" -CUDA_VARIANT = "cuda13.1" -CUDA_REDIST_JSON_SHA256 = "97cf605ccc4751825b1865f4af571c9b50dd29ffd13e9a38b296a9ecb1f0d422" +CUDA_VERSION = "13.3.0" +CUDA_VARIANT = "cuda13.3" +CUDA_REDIST_JSON_SHA256 = "507eddaab1360336bc0fe17b77552e0b7dfe1e74da888671c3a2f5fad7775db1" CUDNN_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cudnn/redist/" -CUDNN_VERSION = "9.19.1" -CUDNN_REDIST_JSON_SHA256 = "ee7bd6872b8611017bfc9ac99a4a71932652d1851b5917aa2c66bf29a12f8fd4" +CUDNN_VERSION = "9.22.0" +CUDNN_REDIST_JSON_SHA256 = "3dbb9002d52112ef69aa09187f523ef1ff07f8baf3892ee01e540af639d8f55f" NVSHMEM_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/nvshmem/redist/" -NVSHMEM_VERSION = "3.5.19" -NVSHMEM_REDIST_JSON_SHA256 = "6dced4193eb728542504b346cfb768da6e3de2abca0cded95fda3a69729994d2" +NVSHMEM_VERSION = "3.6.5" +NVSHMEM_REDIST_JSON_SHA256 = "afbf1ad5c4174c25a66cadd5b925d6814e679e8f836ef1fee1442afb6cb7fdd3" _BUILD_FILE_DEFAULT_VISIBILITY = """\ package(default_visibility = ["//visibility:public"]) @@ -85,10 +85,12 @@ CUDA_PACKAGES = { """filegroup( name = "cuda_compat", srcs = [ - "compat/libnvidia-gpucomp.so.590.48.01", - "compat/libnvidia-tileiras.so.590.48.01", + "compat/libnvidia-gpucomp.so.610.43.02", + "compat/libnvidia-tileiras.so.610.43.02", ] + {} + select({{ - "@llvm//platforms/config:linux_x86_64": ["compat/libnvidia-pkcs11-openssl3.so.590.48.01"], + "@llvm//platforms/config:linux_x86_64": [ + "compat/libnvidia-pkcs11-openssl3.so.610.43.02", + ], "@llvm//platforms/config:linux_aarch64": [], }}), )""".format(repr([ @@ -128,10 +130,6 @@ CUDA_PACKAGES = { "bin/nvlink", ], ), - packages.cc_import( - name = "nvptxcompiler", - static_library = "lib/libnvptxcompiler_static.a", - ), ], "libnvvm": [ packages.filegroup( @@ -147,7 +145,7 @@ CUDA_PACKAGES = { name = "cuda_nvrtc", srcs = [ "lib/libnvrtc.so.13", - "lib/libnvrtc-builtins.so.13.1", + "lib/libnvrtc-builtins.so.13.3", ], ), ], @@ -174,6 +172,7 @@ CUDNN_PACKAGES = { "lib/libcudnn_graph.so.9", "lib/libcudnn_engines_precompiled.so.9", "lib/libcudnn_engines_runtime_compiled.so.9", + "lib/libcudnn_engines_tensor_ir.so.9", "lib/libcudnn_heuristic.so.9", ], ), @@ -187,7 +186,7 @@ NVSHMEM_PACKAGES = { srcs = [ "lib/libnvshmem_host.so.3", "lib/nvshmem_bootstrap_uid.so.3", - "lib/nvshmem_transport_ibrc.so.4", + "lib/nvshmem_transport_ibrc.so.5", ], ), ], @@ -205,27 +204,27 @@ _UBUNTU_PACKAGES = { ], } -PJRT_CUDA_RELEASE = "manual-2026-07-03T00-10-30Z" +PJRT_CUDA_RELEASE = "manual-2026-07-20T19-00-00Z" _PJRT_CUDA_ASSETS = { "amd64": { - "sha256": "6380f724fe21b25dc9231f3ec468ae92e39d21f4bae9377bfa8ff01972521e0d", + "sha256": "30e09906a92cfe6b9e04013a1edd6372423188365ade9fb45aba4fa3759a862f", "url": "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-cuda_linux-amd64.tar.gz", }, "arm64": { - "sha256": "98dc33f0740bc37f3a25611f5aad423cc46eb69679e4d45a22f7fc4dfa3efd8d", + "sha256": "12a2176302cb9b3f8157be230e5d27ece9ee85ce0cd1381a8f243963e937ee51", "url": "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-cuda_linux-arm64.tar.gz", }, } _NCCL_ASSETS = { "amd64": { - "url": "https://pypi.nvidia.com/nvidia-nccl-cu13/nvidia_nccl_cu13-2.29.3-py3-none-manylinux_2_18_x86_64.whl", - "sha256": "2a321629f49490e4e0122ecb578a4b4a6f89e72740dd988e04dfa4758fab7fc3", + "url": "https://pypi.nvidia.com/nvidia-nccl-cu13/nvidia_nccl_cu13-2.30.4-py3-none-manylinux_2_18_x86_64.whl", + "sha256": "534dbf3058cadb625f08ab0d17f1dffad3b961a2bfa360d66633fcf21be53f57", }, "arm64": { - "url": "https://pypi.nvidia.com/nvidia-nccl-cu13/nvidia_nccl_cu13-2.29.3-py3-none-manylinux_2_18_aarch64.whl", - "sha256": "eab9f5c565ab3326906f1d1b5be5773a174c2a1b47002faed76f9e957392f713", + "url": "https://pypi.nvidia.com/nvidia-nccl-cu13/nvidia_nccl_cu13-2.30.4-py3-none-manylinux_2_18_aarch64.whl", + "sha256": "e99308a3a89fba78918d50886e81072a6c8b0b4199feb02c3903e63713a6525a", }, } diff --git a/platforms/cuda/libpjrt_cuda.BUILD.bazel b/platforms/cuda/libpjrt_cuda.BUILD.bazel index bf4e545a..bb22645a 100644 --- a/platforms/cuda/libpjrt_cuda.BUILD.bazel +++ b/platforms/cuda/libpjrt_cuda.BUILD.bazel @@ -12,10 +12,6 @@ patchelf( rename_dynamic_symbols = { "dlopen": "zmlxcuda_dlopen", }, - replace_needed = { - "nvshmem_transport_ibrc.so.3": "nvshmem_transport_ibrc.so.4", - "libnvrtc-builtins.so.13.0": "libnvrtc-builtins.so.13.1", - }, set_rpath = "$ORIGIN", ) From 62a4d3c6517650989ce0778282a814fa7bebeffe Mon Sep 17 00:00:00 2001 From: Louis Lechevalier Date: Mon, 20 Jul 2026 20:57:36 +0200 Subject: [PATCH 590/881] zml/platform: set async dispatch to true when platform is cuda (#673) This PR introduce a new option gpu_async_dispatch that set to true by default the legacy pjrt API option: `use_tfrt_gpu_client`. This option in xla creates an other thread per device to dispatch the thunk and command buffers asynchronously on the gpu. It strongly reduces the cpu overhead when the model is sharded across several devices. --- zml/platform.zig | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/zml/platform.zig b/zml/platform.zig index 9e172e06..77d4cae9 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -670,6 +670,9 @@ pub const CreateOptions = struct { pub const XlaGpu = struct { allocator: Allocator = .{ .bfc = .{} }, + /// The PJRT C API still exposes this under legacy + /// `use_tfrt_gpu_client` name. + gpu_async_dispatch: bool = true, // TODO support all of https://github.com/openxla/xla/blob/3d31c48c719d331d432132b3e0c2c5ce52650675/xla/pjrt/c/pjrt_c_api_gpu_internal.cc#L76-L86 // visible_devices: []const i64 = &.{}, // node_id @@ -692,7 +695,7 @@ pub const CreateOptions = struct { }; }; - fn writeNamedValues(self: XlaGpu, values: *std.ArrayList(pjrt.NamedValue)) void { + fn writeNamedValues(self: XlaGpu, target: Target, values: *std.ArrayList(pjrt.NamedValue)) void { switch (self.allocator) { .platform => { values.appendAssumeCapacity(.init(.string, "allocator", "platform")); @@ -712,6 +715,10 @@ pub const CreateOptions = struct { } }, } + switch (target) { + .cuda => values.appendAssumeCapacity(.init(.bool, "use_tfrt_gpu_client", self.gpu_async_dispatch)), + else => {}, + } } }; @@ -720,7 +727,7 @@ pub const CreateOptions = struct { values.shrinkRetainingCapacity(0); switch (target) { .cpu => self.cpu.writeNamedValues(&values), - .cuda, .rocm, .oneapi, .metal => self.xla_gpu.writeNamedValues(&values), + .cuda, .rocm, .oneapi, .metal => self.xla_gpu.writeNamedValues(target, &values), inline else => |t| { stdx.debug.assertComptime(@hasField(CreateOptions, @tagName(t)), "zml.platform.CreateOptions doesn't list target {s}", .{@tagName(t)}); const options = @field(self, @tagName(t)); From 031b2a56a8d3ad210c12997e0a1bfb66badbbf3e Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 21 Jul 2026 09:13:57 +0200 Subject: [PATCH 591/881] zml/io: add some feedback to `loadExecute` (#670) It's a subotimal solution as we print a generic message but it makes the progress behave as intended. --- zml/io.zig | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/zml/io.zig b/zml/io.zig index fb640920..c70828a2 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -439,6 +439,7 @@ pub const Loader = struct { store: *const TensorStore, shardings: []const Sharding, exe: *const Exe, + opts: LoadOpts, ) !void { const sources = store.getSourcesById(tensor.id) orelse return error.NotFound; const buffers = try arena.alloc(Buffer, sources.len); @@ -446,6 +447,17 @@ pub const Loader = struct { @memset(loaded, false); defer for (buffers, loaded) |*b, l| if (l) b.deinit(); + var node = if (opts.progress) |progress| b: { + var writer = std.Io.Writer.Allocating.init(arena); + try writer.writer.writeAll("Running executable on "); + for (sources, 0..) |source, i| { + try writer.writer.print("{s}{s}", .{ if (i != 0) ", " else "", source.name }); + } + + break :b progress.start(writer.written(), 1); + } else null; + defer if (node) |*n| n.end(); + for (sources, 0..) |source, i| { self.group.async(io, loadSingle, .{ self, io, source, source.shape, &buffers[i], &loaded[i], shardings, .{} }); } From 3e589c97720e338627da515295901bd4decdef09 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 21 Jul 2026 16:18:28 +0200 Subject: [PATCH 592/881] platforms/rocm: bump to rocm-jaxlib-v0.10.2 with TheRock 7.14 (#671) Linked PR: https://github.com/zml/pjrt-artifacts/pull/96 --- MODULE.bazel | 20 +- bin/zml-smi/BUILD.bazel | 7 +- bin/zml-smi/platforms/amdsmi/BUILD.bazel | 2 +- platforms/rocm/BUILD.bazel | 13 +- platforms/rocm/libpjrt_rocm.BUILD.bazel | 106 +- platforms/rocm/packages.lock.json | 10096 +++++++++++---------- platforms/rocm/packages.yaml | 47 +- platforms/rocm/rocm.bzl | 440 +- platforms/rocm/zmlxrocm.zig | 45 +- tools/BUILD.bazel | 2 +- tools/rocprofv3.sh | 12 +- zml/BUILD.bazel | 2 +- zml/module.zig | 5 - 13 files changed, 5652 insertions(+), 5145 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 1f7bd1f9..5d4cb3a9 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -192,7 +192,25 @@ cuda = use_extension("//platforms/cuda:cuda.bzl", "cuda_packages") use_repo(cuda, "cuda_nvml_dev_linux_sbsa", "cuda_nvml_dev_linux_x86_64", "cuda_nvtx_linux_sbsa", "cuda_nvtx_linux_x86_64", "libpjrt_cuda_linux_amd64", "libpjrt_cuda_linux_arm64", "zlib1g_linux_amd64", "zlib1g_linux_arm64") rocm = use_extension("//platforms/rocm:rocm.bzl", "rocm_packages") -use_repo(rocm, "amd-smi-lib", "hipblaslt", "libdrm-amdgpu-amdgpu1", "libdrm2-amdgpu", "libpjrt_rocm", "rocblas", "rocprofiler-sdk", "rocprofiler-sdk-roctx") +use_repo( + rocm, + "amdrocm-amdsmi", + "amdrocm-base", + "amdrocm-blas", + "amdrocm-dnn", + "amdrocm-fft", + "amdrocm-llvm", + "amdrocm-math-common", + "amdrocm-profiler-base", + "amdrocm-rccl", + "amdrocm-runtime", + "amdrocm-solver", + "amdrocm-sparse", + "amdrocm-sysdeps", + "libatomic1", + "libdrm-common", + "libpjrt_rocm", +) oneapi = use_extension("//platforms/oneapi:oneapi.bzl", "oneapi_packages") use_repo(oneapi, "libpjrt_oneapi") diff --git a/bin/zml-smi/BUILD.bazel b/bin/zml-smi/BUILD.bazel index 4032a85c..ed22f18b 100644 --- a/bin/zml-smi/BUILD.bazel +++ b/bin/zml-smi/BUILD.bazel @@ -46,18 +46,15 @@ copy_to_directory( ":libncfw.stripped", ":libnrt.stripped", "//platforms/rocm:zmlxrocm", - "@amd-smi-lib//:libamd_smi", - "@libdrm-amdgpu-amdgpu1", - "@libdrm2-amdgpu//:libdrm2-amdgpu", + "@amdrocm-amdsmi//:libamd_smi", + "@amdrocm-sysdeps//:system_libs", "@libpjrt_rocm//:amdgpu_ids", ], "@llvm//platforms/config:linux_aarch64": [], }), include_external_repositories = ["**"], replace_prefixes = { - "opt/amdgpu/lib/x86_64-linux-gnu": "lib", "amdgpu.ids": "share/libdrm/amdgpu.ids", - "libdrm-amdgpu-amdgpu1": "lib", "platforms/rocm": "lib", "libnrt.so.1": "lib/libnrt.so.1", "libncfw.so.2": "lib/libncfw.so.2", diff --git a/bin/zml-smi/platforms/amdsmi/BUILD.bazel b/bin/zml-smi/platforms/amdsmi/BUILD.bazel index 1543e391..e3ac355d 100644 --- a/bin/zml-smi/platforms/amdsmi/BUILD.bazel +++ b/bin/zml-smi/platforms/amdsmi/BUILD.bazel @@ -16,6 +16,6 @@ zig_library( "//bin/zml-smi/utils:dynlib", "//bin/zml-smi/utils:sandbox", "//stdx", - "@amd-smi-lib//:amdsmi", + "@amdrocm-amdsmi//:amdsmi", ], ) diff --git a/platforms/rocm/BUILD.bazel b/platforms/rocm/BUILD.bazel index ce2a28c8..34064208 100644 --- a/platforms/rocm/BUILD.bazel +++ b/platforms/rocm/BUILD.bazel @@ -10,6 +10,7 @@ zig_shared_library( name = "zmlxrocm", main = "zmlxrocm.zig", shared_lib_name = "libzmlxrocm.so.0", + target_compatible_with = ["@platforms//os:linux"], visibility = [ "//bin/zml-smi:__pkg__", "@libpjrt_rocm//:__subpackages__", @@ -18,32 +19,26 @@ zig_shared_library( # to avoid requiring on the _zig_probe_stack symbol. zigopts = ["-fno-stack-check"], deps = ["//stdx"], - target_compatible_with = ["@platforms//os:linux"], -) - -alias( - name = "hipblaslt", - actual = "@libpjrt_rocm//:hipblaslt", ) cc_library( name = "empty", - tags = ["manual"], defines = ["ZML_RUNTIME_ROCM_DISABLED"], + tags = ["manual"], ) cc_library( name = "libpjrt_rocm", - tags = ["manual"], defines = ["ZML_RUNTIME_ROCM"], + tags = ["manual"], deps = ["@libpjrt_rocm"], ) zig_library( name = "rocm", - tags = ["manual"], import_name = "platforms/rocm", main = "rocm.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ "//pjrt", diff --git a/platforms/rocm/libpjrt_rocm.BUILD.bazel b/platforms/rocm/libpjrt_rocm.BUILD.bazel index 6238e1a8..504e71b1 100644 --- a/platforms/rocm/libpjrt_rocm.BUILD.bazel +++ b/platforms/rocm/libpjrt_rocm.BUILD.bazel @@ -1,21 +1,9 @@ load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") -load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "string_list_flag") load("@bazel_skylib//rules:run_binary.bzl", "run_binary") load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_binary") load("@zml//bazel:patchelf.bzl", "patchelf") -bool_flag( - name = "hipblaslt", - build_setting_default = True, - visibility = ["//visibility:public"], -) - -config_setting( - name = "_hipblaslt", - flag_values = {":hipblaslt": "True"}, -) - patchelf( name = "libpjrt_rocm_so", src = "libpjrt_rocm.so", @@ -25,7 +13,7 @@ patchelf( rename_dynamic_symbols = { "dlopen": "zmlxrocm_dlopen", }, - set_rpath = "$ORIGIN", + set_rpath = "$ORIGIN:$ORIGIN/rocm_sysdeps/lib", ) zig_binary( @@ -36,12 +24,12 @@ zig_binary( run_binary( name = "amdgpu_ids", srcs = [ - "@libdrm-amdgpu-common//:amdgpu_ids", + "@libdrm-common//:amdgpu_ids", "@libdrm_mesa_amdgpu_ids//file:amdgpu.ids", ], outs = ["amdgpu.ids"], args = [ - "$(location @libdrm-amdgpu-common//:amdgpu_ids)", + "$(location @libdrm-common//:amdgpu_ids)", "$(location @libdrm_mesa_amdgpu_ids//file:amdgpu.ids)", "$(location amdgpu.ids)", ], @@ -54,73 +42,71 @@ copy_to_directory( srcs = [ ":amdgpu_ids", ":libpjrt_rocm_so", - "@comgr//:amd_comgr", - "@hip-runtime-amd//:amdhip", - "@hip-runtime-amd//:hiprtc", - "@hipblaslt", - "@hipblaslt//:rocroller", - "@hipfft", - "@hipsolver", - "@hsa-amd-aqlprofile//:hsa-amd-aqlprofile", - "@hsa-rocr//:hsa-runtime", - "@libbz2-1.0", - "@libdrm-amdgpu-amdgpu1", - "@libdrm2-amdgpu", - "@libdw1", - "@libelf1", - "@liblzma5", - "@libnuma1", - "@libtinfo6", - "@libzstd1", - "@miopen-hip//:MIOpen", - "@rccl", - "@rocblas", - "@rocblas//:runfiles", - "@rocfft", - "@rocm-core", - "@rocm-device-libs//:runfiles", - "@rocm-smi-lib//:rocm_smi", - "@rocprofiler-register", - "@rocprofiler-sdk-rocpd//:rocprofiler-sdk-rocpd", - "@rocprofiler-sdk-roctx//:rocprofiler-sdk-roctx", - "@rocprofiler-sdk//:rocprofv3_tool", - "@rocsolver", - "@rocsparse", - "@roctracer", - "@roctracer//:roctx", - "@zlib1g", + "@amdrocm-amdsmi//:libamd_smi", + "@amdrocm-base//:rocm-core", + "@amdrocm-base//:rocm_smi", + "@amdrocm-base//:rocprofiler-register", + "@amdrocm-blas//:hipblaslt", + "@amdrocm-blas//:hipblaslt_runfiles", + "@amdrocm-blas//:rocblas", + "@amdrocm-blas//:rocblas_runfiles", + "@amdrocm-dnn//:MIOpen", + "@amdrocm-fft//:hipfft", + "@amdrocm-fft//:rocfft", + "@amdrocm-llvm//:llvm_libs", + "@amdrocm-llvm//:rocm_device_libs_runfiles", + "@amdrocm-math-common//:host_math_libs", + "@amdrocm-profiler-base//:hsa-amd-aqlprofile", + "@amdrocm-profiler-base//:rocprofiler-sdk", + "@amdrocm-profiler-base//:rocprofv3_tool", + "@amdrocm-profiler-base//:roctracer", + "@amdrocm-profiler-base//:roctx", + "@amdrocm-rccl//:rccl", + "@amdrocm-runtime//:amd_comgr", + "@amdrocm-runtime//:amdhip", + "@amdrocm-runtime//:hiprtc", + "@amdrocm-runtime//:hsa-runtime", + "@amdrocm-solver//:hipsolver", + "@amdrocm-solver//:rocsolver", + "@amdrocm-sparse//:rocsparse", + "@amdrocm-sysdeps//:system_libs", + "@libatomic1", "@zml//platforms/rocm:zmlxrocm", - ] + select({ - ":_hipblaslt": ["@hipblaslt//:runfiles"], - "//conditions:default": [], - }), + ], add_directory_to_runfiles = True, include_external_repositories = ["**"], replace_prefixes = { "MIOpen": "lib", + "amdhip_so": "lib", "amdhip": "lib", "amd_comgr": "lib", + "hipdnn_backend": "lib", + "hipblas": "lib", "hiprtc_so": "lib", + "hipfft": "lib", + "hiprtc": "lib", "hipsolver": "lib", + "hipsparse": "lib", + "hipsparselt": "lib", "hsa-amd-aqlprofile": "lib", "hsa-runtime": "lib", "hipblaslt": "lib", - "lib/x86_64-linux-gnu": "lib", - "libdrm-amdgpu-amdgpu1": "lib", - "libelf1": "lib", + "lib/host-math/lib": "lib", + "lib/llvm/lib": "lib", + "lib/llvm/amdgcn": "amdgcn", "libpjrt_rocm_so": "lib", - "opt/amdgpu/lib/x86_64-linux-gnu": "lib", - "opt/amdgpu/share": "share", "amdgpu.ids": "share/libdrm/amdgpu.ids", "rccl": "lib", "rocblas": "lib", + "rocfft": "lib", "rocm-core": "lib", "rocprofiler-register": "lib", "rocprofiler-sdk_so": "lib", - "rocprofiler-sdk-roctx": "lib", + "rocprofiler-sdk": "lib", "rocm_smi": "lib", "rocsolver": "lib", "rocroller": "lib", + "rocsparse": "lib", "roctracer": "lib", "roctx": "lib", "platforms/rocm": "lib", diff --git a/platforms/rocm/packages.lock.json b/platforms/rocm/packages.lock.json index e69e33f2..a5e839b6 100644 --- a/platforms/rocm/packages.lock.json +++ b/platforms/rocm/packages.lock.json @@ -24,13 +24,13 @@ "version": "12.3.0-1ubuntu1~22.04" } ], - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "sha256": "3bfeaf4259eadbb7faa09feee86cd6cad172cd95907d7465afd0eb5aebb5433f", + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "sha256": "56ec991983c7c89a2c15bd5134534531d722ec5b51a60679f7ceaa1a09b37a09", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/b/bzip2/libbz2-1.0_1.0.8-5build1_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libatomic1_12.3.0-1ubuntu1~22.04_amd64.deb" ], - "version": "1.0.8-5build1" + "version": "12.3.0-1ubuntu1~22.04" }, { "arch": "amd64", @@ -76,119 +76,24 @@ ], "version": "12.3.0-1ubuntu1~22.04" }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - } - ], - "key": "libdw1_0.186-1ubuntu0.1_amd64", - "name": "libdw1", - "sha256": "487120e8e4468844a1bd024c3684e97dbe831fbaa04b975a337e8482370c4e17", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/e/elfutils/libdw1_0.186-1ubuntu0.1_amd64.deb" - ], - "version": "0.186-1ubuntu0.1" - }, { "arch": "amd64", "dependencies": [], - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "sha256": "9dc17e51a1be2d9ed63b7b84ef0e4e29c5abe6f1bc62cb03e7181483cce8a2f2", + "key": "libdrm-common_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-common", + "sha256": "35a306712d8b15b30c42ecd73ec087813eb01c0b3125dc8f7ca2b5134e133522", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/z/zlib/zlib1g_1.2.11.dfsg-2ubuntu9.2_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libd/libdrm/libdrm-common_2.4.113-2~ubuntu0.22.04.1_all.deb" ], - "version": "1:1.2.11.dfsg-2ubuntu9.2" + "version": "2.4.113-2~ubuntu0.22.04.1" }, { "arch": "amd64", "dependencies": [ { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - } - ], - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "sha256": "8f1c46e7d3f5102a5e4fdca7c949728a343ba71c2a7c124118df2c13d4c444f7", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/x/xz-utils/liblzma5_5.2.5-2ubuntu1_amd64.deb" - ], - "version": "5.2.5-2ubuntu1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "sha256": "2cc781614b045b450a1e073ee3b0902f393ee58ede1d2ea4526ad805fcccf538", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/e/elfutils/libelf1_0.186-1ubuntu0.1_amd64.deb" - ], - "version": "0.186-1ubuntu0.1" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "rocm-core_7.2.2.70202-86_22.04_amd64", - "name": "rocm-core", - "version": "7.2.2.70202-86~22.04" + "key": "amdrocm-sysdeps7.14_7.14.0-3_amd64", + "name": "amdrocm-sysdeps7.14", + "version": "7.14.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -409,86 +314,26 @@ "key": "python3-minimal_3.10.6-1_22.04.1_amd64", "name": "python3-minimal", "version": "3.10.6-1~22.04.1" - }, - { - "key": "python3-setuptools_59.6.0-1.2ubuntu0.22.04.3_amd64", - "name": "python3-setuptools", - "version": "59.6.0-1.2ubuntu0.22.04.3" - }, - { - "key": "python3-distutils_3.10.8-1_22.04_amd64", - "name": "python3-distutils", - "version": "3.10.8-1~22.04" - }, - { - "key": "python3-lib2to3_3.10.8-1_22.04_amd64", - "name": "python3-lib2to3", - "version": "3.10.8-1~22.04" - }, - { - "key": "python3-pkg-resources_59.6.0-1.2ubuntu0.22.04.3_amd64", - "name": "python3-pkg-resources", - "version": "59.6.0-1.2ubuntu0.22.04.3" - }, - { - "key": "sudo_1.9.9-1ubuntu2.5_amd64", - "name": "sudo", - "version": "1.9.9-1ubuntu2.5" - }, - { - "key": "lsb-base_11.1.0ubuntu4_amd64", - "name": "lsb-base", - "version": "11.1.0ubuntu4" - }, - { - "key": "libpam-modules_1.4.0-11ubuntu2.6_amd64", - "name": "libpam-modules", - "version": "1.4.0-11ubuntu2.6" - }, - { - "key": "libpam-modules-bin_1.4.0-11ubuntu2.6_amd64", - "name": "libpam-modules-bin", - "version": "1.4.0-11ubuntu2.6" - }, - { - "key": "libpam0g_1.4.0-11ubuntu2.6_amd64", - "name": "libpam0g", - "version": "1.4.0-11ubuntu2.6" - }, - { - "key": "libaudit1_1-3.0.7-1build1_amd64", - "name": "libaudit1", - "version": "1:3.0.7-1build1" - }, - { - "key": "libcap-ng0_0.7.9-2.2build3_amd64", - "name": "libcap-ng0", - "version": "0.7.9-2.2build3" - }, - { - "key": "libaudit-common_1-3.0.7-1build1_amd64", - "name": "libaudit-common", - "version": "1:3.0.7-1build1" } ], - "key": "amd-smi-lib_26.2.2.70202-86_22.04_amd64", - "name": "amd-smi-lib", - "sha256": "406bcedc70b24b30e71037d5f625125cb15c4033f073fabcb1718d8261395e85", + "key": "amdrocm-amdsmi7.14_7.14.0-3_amd64", + "name": "amdrocm-amdsmi7.14", + "sha256": "34260f01ab7f6234a9d39b9cb4dbdc5634f0469d58caa0f604042ce8838f02b6", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/a/amd-smi-lib/amd-smi-lib_26.2.2.70202-86~22.04_amd64.deb" + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-amdsmi7.14_7.14.0-3_amd64.deb" ], - "version": "26.2.2.70202-86~22.04" + "version": "7.14.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "rocm-core_7.2.2.70202-86_22.04_amd64", - "name": "rocm-core", - "sha256": "03fce430bfb5769fda2c9dcf0f23b6a20c1cf6fd7bf091b28a9a1b978716042e", + "key": "amdrocm-sysdeps7.14_7.14.0-3_amd64", + "name": "amdrocm-sysdeps7.14", + "sha256": "c28c1863f3f5006fb60976e5dde098fa8ae8fe8e4b65f02116586d68fc261a00", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocm-core/rocm-core_7.2.2.70202-86~22.04_amd64.deb" + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sysdeps7.14_7.14.0-3_amd64.deb" ], - "version": "7.2.2.70202-86~22.04" + "version": "7.14.0-3" }, { "arch": "amd64", @@ -677,6 +522,17 @@ ], "version": "2.3.1-1" }, + { + "arch": "amd64", + "dependencies": [], + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "sha256": "9dc17e51a1be2d9ed63b7b84ef0e4e29c5abe6f1bc62cb03e7181483cce8a2f2", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/z/zlib/zlib1g_1.2.11.dfsg-2ubuntu9.2_amd64.deb" + ], + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, { "arch": "amd64", "dependencies": [], @@ -688,6 +544,28 @@ ], "version": "1.4.8+dfsg-3build1" }, + { + "arch": "amd64", + "dependencies": [], + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "sha256": "8f1c46e7d3f5102a5e4fdca7c949728a343ba71c2a7c124118df2c13d4c444f7", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/x/xz-utils/liblzma5_5.2.5-2ubuntu1_amd64.deb" + ], + "version": "5.2.5-2ubuntu1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "sha256": "3bfeaf4259eadbb7faa09feee86cd6cad172cd95907d7465afd0eb5aebb5433f", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/b/bzip2/libbz2-1.0_1.0.8-5build1_amd64.deb" + ], + "version": "1.0.8-5build1" + }, { "arch": "amd64", "dependencies": [], @@ -897,715 +775,23 @@ ], "version": "3.10.6-1~22.04.1" }, - { - "arch": "amd64", - "dependencies": [], - "key": "python3-setuptools_59.6.0-1.2ubuntu0.22.04.3_amd64", - "name": "python3-setuptools", - "sha256": "553fd6b076e22c35123f29b4b5ff08e762164b39c72569f05af7265571a85c13", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/s/setuptools/python3-setuptools_59.6.0-1.2ubuntu0.22.04.3_all.deb" - ], - "version": "59.6.0-1.2ubuntu0.22.04.3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "python3-distutils_3.10.8-1_22.04_amd64", - "name": "python3-distutils", - "sha256": "b2bada10bef81fa7dc4bde61a1a462f37ca5027805e7c96e58685e9034c6094f", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3-stdlib-extensions/python3-distutils_3.10.8-1~22.04_all.deb" - ], - "version": "3.10.8-1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "python3-lib2to3_3.10.8-1_22.04_amd64", - "name": "python3-lib2to3", - "sha256": "7bd212e61e8bb7fabf4768629e5dc97e6f8a7f98cc9335b5a3ce8c11cb633248", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3-stdlib-extensions/python3-lib2to3_3.10.8-1~22.04_all.deb" - ], - "version": "3.10.8-1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "python3-pkg-resources_59.6.0-1.2ubuntu0.22.04.3_amd64", - "name": "python3-pkg-resources", - "sha256": "e838b1d16912f11c8c955d99ba5b7f378025ebb29b3ea72096cc7e9ec4d5e8f8", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/s/setuptools/python3-pkg-resources_59.6.0-1.2ubuntu0.22.04.3_all.deb" - ], - "version": "59.6.0-1.2ubuntu0.22.04.3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "sudo_1.9.9-1ubuntu2.5_amd64", - "name": "sudo", - "sha256": "479c1e3247887315cc69dcec0b2a58504c77be3f45448de3a0b0e44b0f4ca775", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/s/sudo/sudo_1.9.9-1ubuntu2.5_amd64.deb" - ], - "version": "1.9.9-1ubuntu2.5" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "lsb-base_11.1.0ubuntu4_amd64", - "name": "lsb-base", - "sha256": "23ac3e99b0c2cd0179a7f40fa5a24248345e16c515f222164dad2a638b4f04cf", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/l/lsb/lsb-base_11.1.0ubuntu4_all.deb" - ], - "version": "11.1.0ubuntu4" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libpam-modules_1.4.0-11ubuntu2.6_amd64", - "name": "libpam-modules", - "sha256": "6f03b50258a365d69a037fdd9c29a6b976fa3953c84bafacfc19385cd8fe4b68", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pam/libpam-modules_1.4.0-11ubuntu2.6_amd64.deb" - ], - "version": "1.4.0-11ubuntu2.6" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libpam-modules-bin_1.4.0-11ubuntu2.6_amd64", - "name": "libpam-modules-bin", - "sha256": "3b85776823736451ec59708181ebdcdde0429ef9bdea76ffeba8f2428146331c", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pam/libpam-modules-bin_1.4.0-11ubuntu2.6_amd64.deb" - ], - "version": "1.4.0-11ubuntu2.6" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libpam0g_1.4.0-11ubuntu2.6_amd64", - "name": "libpam0g", - "sha256": "9c0e509cdf68364212910805ed673f481b7762109eb6499b68d7f9bf599b0c4a", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pam/libpam0g_1.4.0-11ubuntu2.6_amd64.deb" - ], - "version": "1.4.0-11ubuntu2.6" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libaudit1_1-3.0.7-1build1_amd64", - "name": "libaudit1", - "sha256": "36710fbc49150a13f14f1f9c2c8288ee840cf9425f8a609c22fefa6b64eb51c1", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/a/audit/libaudit1_3.0.7-1build1_amd64.deb" - ], - "version": "1:3.0.7-1build1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libcap-ng0_0.7.9-2.2build3_amd64", - "name": "libcap-ng0", - "sha256": "124d0c8748a841f279e996298ef8aac69a249e294792f4b5e16dc00496f1d3ac", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libc/libcap-ng/libcap-ng0_0.7.9-2.2build3_amd64.deb" - ], - "version": "0.7.9-2.2build3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libaudit-common_1-3.0.7-1build1_amd64", - "name": "libaudit-common", - "sha256": "86697036d05c956b5b42339bf61ecf3743f7c6fd4d59a67ab16add9a2cd7802a", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/a/audit/libaudit-common_3.0.7-1build1_all.deb" - ], - "version": "1:3.0.7-1build1" - }, { "arch": "amd64", "dependencies": [ { - "key": "amdgpu-core_1-7.2.70201-2303469.22.04_amd64", - "name": "amdgpu-core", - "version": "1:7.2.70201-2303469.22.04" - } - ], - "key": "libdrm-amdgpu-common_1.0.0.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-common", - "sha256": "288719d661cc1fbba776ec08d10053d745d53a35afe8cbfcf232828aa5a27091", - "urls": [ - "https://repo.radeon.com/graphics/7.2.1/ubuntu/pool/main/libd/libdrm-amdgpu-common/libdrm-amdgpu-common_1.0.0.70201-2303469.22.04_all.deb" - ], - "version": "1.0.0.70201-2303469.22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdgpu-core_1-7.2.70201-2303469.22.04_amd64", - "name": "amdgpu-core", - "sha256": "54af80759a376028bb442344cdcf39565063438d3708a40f125f02ccb2d8813e", - "urls": [ - "https://repo.radeon.com/graphics/7.2.1/ubuntu/pool/main/a/amdgpu-core/amdgpu-core_7.2.70201-2303469.22.04_all.deb" - ], - "version": "1:7.2.70201-2303469.22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "rocprofiler-register_0.6.0.70202-86_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.6.0.70202-86~22.04" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "rocm-core_7.2.2.70202-86_22.04_amd64", - "name": "rocm-core", - "version": "7.2.2.70202-86~22.04" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" + "key": "amdrocm-runtime7.14_7.14.0-3_amd64", + "name": "amdrocm-runtime7.14", + "version": "7.14.0-3" }, { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.125.70201-2303469.22.04" + "key": "amdrocm-llvm7.14_7.14.0-3_amd64", + "name": "amdrocm-llvm7.14", + "version": "7.14.0-3" }, { - "key": "libdrm-amdgpu-common_1.0.0.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-common", - "version": "1.0.0.70201-2303469.22.04" - }, - { - "key": "amdgpu-core_1-7.2.70201-2303469.22.04_amd64", - "name": "amdgpu-core", - "version": "1:7.2.70201-2303469.22.04" - }, - { - "key": "libdrm2-amdgpu_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm2-amdgpu", - "version": "1:2.4.125.70201-2303469.22.04" - } - ], - "key": "hsa-rocr_1.18.0.70202-86_22.04_amd64", - "name": "hsa-rocr", - "sha256": "a9e6ca15a56e91f38bc20325afdd5cbc271815f20b6f1c51088e914917338a04", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/h/hsa-rocr/hsa-rocr_1.18.0.70202-86~22.04_amd64.deb" - ], - "version": "1.18.0.70202-86~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "rocm-core_7.2.2.70202-86_22.04_amd64", - "name": "rocm-core", - "version": "7.2.2.70202-86~22.04" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - } - ], - "key": "rocprofiler-register_0.6.0.70202-86_22.04_amd64", - "name": "rocprofiler-register", - "sha256": "1d6cefddc97dbda37e46aedff955ca9c27352ad26cf50934f82a8130a5b0b897", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocprofiler-register/rocprofiler-register_0.6.0.70202-86~22.04_amd64.deb" - ], - "version": "0.6.0.70202-86~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "sha256": "0721c89001fbbd1ada23e89da5d60e762763c1a7b3dc814a2e9a518480a8043d", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/numactl/libnuma1_2.0.14-3ubuntu2_amd64.deb" - ], - "version": "2.0.14-3ubuntu2" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "sha256": "f8b455610e6603c2966ffed5e7530b2b2c1a6742fba99aab08f0930b4f638f58", - "urls": [ - "https://repo.radeon.com/graphics/7.2.1/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-amdgpu1_2.4.125.70201-2303469.22.04_amd64.deb" - ], - "version": "1:2.4.125.70201-2303469.22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libdrm2-amdgpu_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm2-amdgpu", - "sha256": "320439f513cada1bc308482a94eb07b7af04cd90a8163ad627983279c0be5ef3", - "urls": [ - "https://repo.radeon.com/graphics/7.2.1/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm2-amdgpu_2.4.125.70201-2303469.22.04_amd64.deb" - ], - "version": "1:2.4.125.70201-2303469.22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "rocm-core_7.2.2.70202-86_22.04_amd64", - "name": "rocm-core", - "version": "7.2.2.70202-86~22.04" + "key": "amdrocm-sysdeps7.14_7.14.0-3_amd64", + "name": "amdrocm-sysdeps7.14", + "version": "7.14.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -1628,275 +814,18 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "hsa-rocr_1.18.0.70202-86_22.04_amd64", - "name": "hsa-rocr", - "version": "1.18.0.70202-86~22.04" - }, - { - "key": "rocprofiler-register_0.6.0.70202-86_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.6.0.70202-86~22.04" - }, - { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.125.70201-2303469.22.04" - }, - { - "key": "libdrm-amdgpu-common_1.0.0.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-common", - "version": "1.0.0.70201-2303469.22.04" - }, - { - "key": "amdgpu-core_1-7.2.70201-2303469.22.04_amd64", - "name": "amdgpu-core", - "version": "1:7.2.70201-2303469.22.04" - }, - { - "key": "libdrm2-amdgpu_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm2-amdgpu", - "version": "1:2.4.125.70201-2303469.22.04" - } - ], - "key": "hsa-amd-aqlprofile_1.0.0.70202-86_22.04_amd64", - "name": "hsa-amd-aqlprofile", - "sha256": "55b92d285fab53a312d7d2ff132fd1e6216e2d46f2f3b47758299fcb0505a586", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/h/hsa-amd-aqlprofile/hsa-amd-aqlprofile_1.0.0.70202-86~22.04_amd64.deb" - ], - "version": "1.0.0.70202-86~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", "version": "12.3.0-1ubuntu1~22.04" }, { @@ -1905,219 +834,44 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "rocm-core_7.2.2.70202-86_22.04_amd64", - "name": "rocm-core", - "version": "7.2.2.70202-86~22.04" + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" }, { "key": "libc6-dev_2.35-0ubuntu3.10_amd64", @@ -2134,6 +888,111 @@ "name": "libtirpc-dev", "version": "1.3.2-2ubuntu0.1" }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, { "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", "name": "rpcsvc-proto", @@ -2155,51 +1014,561 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" } ], - "key": "comgr_3.0.0.70202-86_22.04_amd64", - "name": "comgr", - "sha256": "e55ca10c183f8debdd595d8b3e4a8fd5df8e648ac8bc0edb1281b28795534eea", + "key": "amdrocm-base7.14_7.14.0-3_amd64", + "name": "amdrocm-base7.14", + "sha256": "23a5667f15b45fcc1b06197eed8800d8d6757e7fd7f8806557eb7e9d80715358", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/c/comgr/comgr_3.0.0.70202-86~22.04_amd64.deb" + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-base7.14_7.14.0-3_amd64.deb" ], - "version": "3.0.0.70202-86~22.04" + "version": "7.14.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "sha256": "145736b5bec52cd4fc1b2c09517759383b95255f9fd304ace1de7d830295bb33", + "key": "amdrocm-runtime7.14_7.14.0-3_amd64", + "name": "amdrocm-runtime7.14", + "sha256": "19d6547f447dcfab9890d3608c5aa9b35647ab293bfb5ffbc829402e6fcc7233", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libtinfo-dev_6.3-2ubuntu0.1_amd64.deb" + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-runtime7.14_7.14.0-3_amd64.deb" ], - "version": "6.3-2ubuntu0.1" + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "amdrocm-runtime7.14_7.14.0-3_amd64", + "name": "amdrocm-runtime7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sysdeps7.14_7.14.0-3_amd64", + "name": "amdrocm-sysdeps7.14", + "version": "7.14.0-3" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "amdrocm-base7.14_7.14.0-3_amd64", + "name": "amdrocm-base7.14", + "version": "7.14.0-3" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + } + ], + "key": "amdrocm-llvm7.14_7.14.0-3_amd64", + "name": "amdrocm-llvm7.14", + "sha256": "e1f1e278f87e35d6d52b2fceb01e083a1ef6e2ed808eb30701f6e7cf0a332d98", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-llvm7.14_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "sha256": "1de196ffdd885b5953a44cde13e68b77dc7ceb9c9921482dc84e358bd1ad8b02", + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "sha256": "06faa64479f1987e4f870ff9c1ccfdf3c0c441bd3886136d6e24c11316c9f3d6", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libncurses-dev_6.3-2ubuntu0.1_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-11/libgcc-11-dev_11.4.0-1ubuntu1~22.04_amd64.deb" ], - "version": "6.3-2ubuntu0.1" + "version": "11.4.0-1ubuntu1~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "sha256": "6382f5c1d7d0bf39cbb94e208ece2bdf556cb3a8dfe278748542c7b8b57a2e6d", + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "sha256": "6bf564c2bb191aefcf96758502a7ac24b2af4d644c87b3678e4b1107c5494b13", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/ncurses-bin_6.3-2ubuntu0.1_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libquadmath0_12.3.0-1ubuntu1~22.04_amd64.deb" ], - "version": "6.3-2ubuntu0.1" + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "sha256": "becb1726d3b8a45cfd336e502fc29d7c43e024689857c670ecf75a5362732c63", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libubsan1_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "sha256": "7b14e81317fce5e62d6cc2454d975d3ec4eac90710a1a536d63c9ee84a50b6f2", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-11/libtsan0_11.4.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "sha256": "5448bba72109dbf2df6ee30463b5f06c074138006cd1d5730fb5189b93e6d563", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-11/gcc-11-base_11.4.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "sha256": "606e212c3cbc8fb622313fdb32f78a024b4b5b9fe54b1cc5a8f731297023b899", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/liblsan0_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "sha256": "0580e3209722dd0e6d9e57c4149a0f8f6073f975cf8b5ede26ac54289c2fc99a", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-11/libasan6_11.4.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "sha256": "ac3c96ed50a42b4f57c410576bdf816710684e3e32e34f704a7295014565b724", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libitm1_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "sha256": "072a7ee2dc00bc92a816e11ae62ad5a497c86fc1f0473c0a095f29e13003da72", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libgomp1_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "sha256": "5ab9c245d4c65a3986d567c54e2503251c92bd369c9f42e48a2c4b1f1835dcf5", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-11/libstdc++-11-dev_11.4.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "11.4.0-1ubuntu1~22.04" }, { "arch": "amd64", @@ -2278,29 +1647,43 @@ ], "version": "2.35-0ubuntu3.10" }, - { - "arch": "amd64", - "dependencies": [], - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "sha256": "4a828e6cb2a721be6f4ed7b34256d3fc9aab1495b2b4923fd579d4dc589496fd", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libncurses6_6.3-2ubuntu0.1_amd64.deb" - ], - "version": "6.3-2ubuntu0.1" - }, { "arch": "amd64", "dependencies": [ { - "key": "rocprofiler-sdk-rocpd_1.1.0-86_22.04_amd64", - "name": "rocprofiler-sdk-rocpd", - "version": "1.1.0-86~22.04" + "key": "amdrocm-blas7.14-gfx950_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx950", + "version": "7.14.0-3" }, { - "key": "rocm-core_7.2.2.70202-86_22.04_amd64", - "name": "rocm-core", - "version": "7.2.2.70202-86~22.04" + "key": "amdrocm-solver7.14-gfx950_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx950", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx950_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx950", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse-host7.14_7.14.0-3_amd64", + "name": "amdrocm-sparse-host7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-runtime7.14_7.14.0-3_amd64", + "name": "amdrocm-runtime7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-llvm7.14_7.14.0-3_amd64", + "name": "amdrocm-llvm7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sysdeps7.14_7.14.0-3_amd64", + "name": "amdrocm-sysdeps7.14", + "version": "7.14.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -2323,159 +1706,19 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" }, { "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", @@ -2483,368 +1726,44 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "rocprofiler-sdk-roctx_1.1.0-86_22.04_amd64", - "name": "rocprofiler-sdk-roctx", - "version": "1.1.0-86~22.04" - }, - { - "key": "rocprofiler-register_0.6.0.70202-86_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.6.0.70202-86~22.04" - } - ], - "key": "rocprofiler-sdk_1.1.0-86_22.04_amd64", - "name": "rocprofiler-sdk", - "sha256": "42c2b64ab72e027fd8e9b5c0892413e1cc476f3593a7eaeb414df985896ce032", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocprofiler-sdk/rocprofiler-sdk_1.1.0-86~22.04_amd64.deb" - ], - "version": "1.1.0-86~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "rocprofiler-sdk-rocpd_1.1.0-86_22.04_amd64", - "name": "rocprofiler-sdk-rocpd", - "sha256": "09573c79fc2cc11e590c9dfae2143d0a9021ef3e44cf50220beb09130ca82061", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocprofiler-sdk-rocpd/rocprofiler-sdk-rocpd_1.1.0-86~22.04_amd64.deb" - ], - "version": "1.1.0-86~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "rocprofiler-sdk-roctx_1.1.0-86_22.04_amd64", - "name": "rocprofiler-sdk-roctx", - "sha256": "0a23a509af5ee30a5a05ff8832236be48271e2228fdbd00c362062a8baef2891", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocprofiler-sdk-roctx/rocprofiler-sdk-roctx_1.1.0-86~22.04_amd64.deb" - ], - "version": "1.1.0-86~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "rocm-core_7.2.2.70202-86_22.04_amd64", - "name": "rocm-core", - "version": "7.2.2.70202-86~22.04" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "rocrand_4.2.0.70202-86_22.04_amd64", - "name": "rocrand", - "version": "4.2.0.70202-86~22.04" - }, - { - "key": "hipblaslt_1.2.2.70202-86_22.04_amd64", - "name": "hipblaslt", - "version": "1.2.2.70202-86~22.04" - }, - { - "key": "roctracer_4.1.70202.70202-86_22.04_amd64", - "name": "roctracer", - "version": "4.1.70202.70202-86~22.04" - }, - { - "key": "rocblas_5.2.0.70202-86_22.04_amd64", - "name": "rocblas", - "version": "5.2.0.70202-86~22.04" - }, - { - "key": "hip-runtime-amd_7.2.53211.70202-86_22.04_amd64", - "name": "hip-runtime-amd", - "version": "7.2.53211.70202-86~22.04" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "rocprofiler-register_0.6.0.70202-86_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.6.0.70202-86~22.04" - }, - { - "key": "comgr_3.0.0.70202-86_22.04_amd64", - "name": "comgr", - "version": "3.0.0.70202-86~22.04" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" }, { "key": "libc6-dev_2.35-0ubuntu3.10_amd64", @@ -2861,6 +1780,111 @@ "name": "libtirpc-dev", "version": "1.3.2-2ubuntu0.1" }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, { "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", "name": "rpcsvc-proto", @@ -2882,97 +1906,540 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", + "key": "amdrocm-base7.14_7.14.0-3_amd64", + "name": "amdrocm-base7.14", + "version": "7.14.0-3" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70202-86_22.04_amd64", - "name": "rocminfo", - "version": "1.0.0.70202-86~22.04" + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" }, { - "key": "pciutils_1-3.7.0-6_amd64", - "name": "pciutils", - "version": "1:3.7.0-6" + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" }, { - "key": "libpci3_1-3.7.0-6_amd64", - "name": "libpci3", - "version": "1:3.7.0-6" + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" }, { - "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", - "name": "pci.ids", - "version": "0.0~2022.01.22-1ubuntu0.1" + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" }, { - "key": "libudev1_249.11-0ubuntu3.16_amd64", - "name": "libudev1", - "version": "249.11-0ubuntu3.16" + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" }, { - "key": "libkmod2_29-1ubuntu1_amd64", - "name": "libkmod2", - "version": "29-1ubuntu1" + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" }, { - "key": "hsa-rocr_1.18.0.70202-86_22.04_amd64", - "name": "hsa-rocr", - "version": "1.18.0.70202-86~22.04" + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" }, { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" }, { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.125.70201-2303469.22.04" + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" }, { - "key": "libdrm-amdgpu-common_1.0.0.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-common", - "version": "1.0.0.70201-2303469.22.04" + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" }, { - "key": "amdgpu-core_1-7.2.70201-2303469.22.04_amd64", - "name": "amdgpu-core", - "version": "1:7.2.70201-2303469.22.04" + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" }, { - "key": "libdrm2-amdgpu_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm2-amdgpu", - "version": "1:2.4.125.70201-2303469.22.04" + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "amdrocm-profiler-base7.14_7.14.0-3_amd64", + "name": "amdrocm-profiler-base7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver-host7.14_7.14.0-3_amd64", + "name": "amdrocm-solver-host7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-math-common7.14_7.14.0-3_amd64", + "name": "amdrocm-math-common7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas-host7.14_7.14.0-3_amd64", + "name": "amdrocm-blas-host7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx942_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx942", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx942_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx942", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx942_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx942", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx90a_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx90a", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx90a_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx90a", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx90a_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx90a", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx908_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx908", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx908_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx908", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx908_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx908", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1250_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1250", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1250_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1250", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1250_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1250", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1201_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1201", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1201_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1201", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1201_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1201", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1200_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1200", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1200_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1200", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1200_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1200", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1153_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1153", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1153_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1153", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1153_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1153", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1152_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1152", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1152_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1152", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1152_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1152", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1151_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1151", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1151_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1151", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1151_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1151", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1150_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1150", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1150_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1150", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1150_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1150", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1103_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1103", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1103_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1103", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1103_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1103", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1102_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1102", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1102_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1102", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1102_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1102", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1101_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1101", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1101_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1101", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1101_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1101", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1100_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1100", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1100_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1100", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1100_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1100", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1036_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1036", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1036_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1036", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1036_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1036", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1035_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1035", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1035_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1035", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1035_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1035", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1034_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1034", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1034_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1034", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1034_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1034", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1033_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1033", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1033_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1033", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1033_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1033", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1032_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1032", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1032_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1032", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1032_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1032", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1031_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1031", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1031_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1031", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1031_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1031", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1030_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1030", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1030_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1030", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1030_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1030", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1012_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1012", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1012_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1012", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1012_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1012", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1011_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1011", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1011_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1011", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1011_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1011", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1010_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1010", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1010_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1010", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1010_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1010", + "version": "7.14.0-3" } ], - "key": "miopen-hip_3.5.1.70202-86_22.04_amd64", - "name": "miopen-hip", - "sha256": "d8b43108d7328d5681b9e5ad847ed9f3ac2b744312e46f7b11246c0cc887407b", + "key": "amdrocm-blas7.14_7.14.0-3_amd64", + "name": "amdrocm-blas7.14", + "sha256": "6185f9456503100afbb0ca00ad0ed0a9db8d1b181cb17006602c6265535f0699", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/m/miopen-hip/miopen-hip_3.5.1.70202-86~22.04_amd64.deb" + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14_7.14.0-3_amd64.deb" ], - "version": "3.5.1.70202-86~22.04" + "version": "7.14.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "rocrand_4.2.0.70202-86_22.04_amd64", - "name": "rocrand", - "sha256": "2add5082b1322fb59e3d383bdd086676fd91f4437864b93b1ece7cc3c1380e56", + "key": "amdrocm-blas7.14-gfx950_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx950", + "sha256": "37555d5dc4cc2ed7bf67029d8fe13d6ec96002ef186461ec6616e651b000fa45", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocrand/rocrand_4.2.0.70202-86~22.04_amd64.deb" + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx950_7.14.0-3_amd64.deb" ], - "version": "4.2.0.70202-86~22.04" + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver7.14-gfx950_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx950", + "sha256": "94af3096b972d69ad8b183f2c4c3d8ef85817e968481ccee4608e18c69fc4104", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx950_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse7.14-gfx950_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx950", + "sha256": "3704bff11ff1832de409d677cee1687330dfab49893dd8ef674c0b8a0035656c", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx950_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse-host7.14_7.14.0-3_amd64", + "name": "amdrocm-sparse-host7.14", + "sha256": "8f00e94a7c845926e2638429c19fb50181a3feaeff603ddceaacaa974ab75d5d", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse-host7.14_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.2.70202-86_22.04_amd64", - "name": "rocm-core", - "version": "7.2.2.70202-86~22.04" + "key": "amdrocm-sysdeps7.14_7.14.0-3_amd64", + "name": "amdrocm-sysdeps7.14", + "version": "7.14.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -2995,159 +2462,29 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" + "key": "amdrocm-runtime7.14_7.14.0-3_amd64", + "name": "amdrocm-runtime7.14", + "version": "7.14.0-3" }, { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" + "key": "amdrocm-llvm7.14_7.14.0-3_amd64", + "name": "amdrocm-llvm7.14", + "version": "7.14.0-3" }, { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" }, { "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", @@ -3155,558 +2492,44 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "roctracer_4.1.70202.70202-86_22.04_amd64", - "name": "roctracer", - "version": "4.1.70202.70202-86~22.04" - } - ], - "key": "hipblaslt_1.2.2.70202-86_22.04_amd64", - "name": "hipblaslt", - "sha256": "c3f5086d42d040b4dc10cfa45743047e9d89c7abbe169955caf978c82d85a930", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/h/hipblaslt/hipblaslt_1.2.2.70202-86~22.04_amd64.deb" - ], - "version": "1.2.2.70202-86~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "rocm-core_7.2.2.70202-86_22.04_amd64", - "name": "rocm-core", - "version": "7.2.2.70202-86~22.04" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - } - ], - "key": "roctracer_4.1.70202.70202-86_22.04_amd64", - "name": "roctracer", - "sha256": "b5f436accb35050035e5b59e669d991b3808389781278be33ca110611aff4d7e", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/roctracer/roctracer_4.1.70202.70202-86~22.04_amd64.deb" - ], - "version": "4.1.70202.70202-86~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "rocm-core_7.2.2.70202-86_22.04_amd64", - "name": "rocm-core", - "version": "7.2.2.70202-86~22.04" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "hip-runtime-amd_7.2.53211.70202-86_22.04_amd64", - "name": "hip-runtime-amd", - "version": "7.2.53211.70202-86~22.04" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "rocprofiler-register_0.6.0.70202-86_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.6.0.70202-86~22.04" - }, - { - "key": "comgr_3.0.0.70202-86_22.04_amd64", - "name": "comgr", - "version": "3.0.0.70202-86~22.04" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" }, { "key": "libc6-dev_2.35-0ubuntu3.10_amd64", @@ -3723,6 +2546,111 @@ "name": "libtirpc-dev", "version": "1.3.2-2ubuntu0.1" }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, { "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", "name": "rpcsvc-proto", @@ -3744,131 +2672,9 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "rocminfo_1.0.0.70202-86_22.04_amd64", - "name": "rocminfo", - "version": "1.0.0.70202-86~22.04" - }, - { - "key": "pciutils_1-3.7.0-6_amd64", - "name": "pciutils", - "version": "1:3.7.0-6" - }, - { - "key": "libpci3_1-3.7.0-6_amd64", - "name": "libpci3", - "version": "1:3.7.0-6" - }, - { - "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", - "name": "pci.ids", - "version": "0.0~2022.01.22-1ubuntu0.1" - }, - { - "key": "libudev1_249.11-0ubuntu3.16_amd64", - "name": "libudev1", - "version": "249.11-0ubuntu3.16" - }, - { - "key": "libkmod2_29-1ubuntu1_amd64", - "name": "libkmod2", - "version": "29-1ubuntu1" - }, - { - "key": "hsa-rocr_1.18.0.70202-86_22.04_amd64", - "name": "hsa-rocr", - "version": "1.18.0.70202-86~22.04" - }, - { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" - }, - { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.125.70201-2303469.22.04" - }, - { - "key": "libdrm-amdgpu-common_1.0.0.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-common", - "version": "1.0.0.70201-2303469.22.04" - }, - { - "key": "amdgpu-core_1-7.2.70201-2303469.22.04_amd64", - "name": "amdgpu-core", - "version": "1:7.2.70201-2303469.22.04" - }, - { - "key": "libdrm2-amdgpu_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm2-amdgpu", - "version": "1:2.4.125.70201-2303469.22.04" - }, - { - "key": "hipblaslt_1.2.2.70202-86_22.04_amd64", - "name": "hipblaslt", - "version": "1.2.2.70202-86~22.04" - }, - { - "key": "roctracer_4.1.70202.70202-86_22.04_amd64", - "name": "roctracer", - "version": "4.1.70202.70202-86~22.04" - } - ], - "key": "rocblas_5.2.0.70202-86_22.04_amd64", - "name": "rocblas", - "sha256": "8c390b22297321d06453db51d8aa6de8329b47d952d96e91da934c8ffc413a30", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocblas/rocblas_5.2.0.70202-86~22.04_amd64.deb" - ], - "version": "5.2.0.70202-86~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "rocprofiler-register_0.6.0.70202-86_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.6.0.70202-86~22.04" - }, - { - "key": "rocm-core_7.2.2.70202-86_22.04_amd64", - "name": "rocm-core", - "version": "7.2.2.70202-86~22.04" + "key": "amdrocm-base7.14_7.14.0-3_amd64", + "name": "amdrocm-base7.14", + "version": "7.14.0-3" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -3890,6 +2696,1080 @@ "name": "libuuid1", "version": "2.37.2-4ubuntu3.4" }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + } + ], + "key": "amdrocm-profiler-base7.14_7.14.0-3_amd64", + "name": "amdrocm-profiler-base7.14", + "sha256": "507a46e794e7cc31a69ab022d1ce79e845c7e166808c1decfd669988016a22f1", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-profiler-base7.14_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver-host7.14_7.14.0-3_amd64", + "name": "amdrocm-solver-host7.14", + "sha256": "073ebe38a298201071c5dddea99f17578d8ce9aa0be7a9e8f96571f0cffc8a98", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver-host7.14_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-math-common7.14_7.14.0-3_amd64", + "name": "amdrocm-math-common7.14", + "sha256": "e1f0d89d538af693070c8238f28c5bc1c75eb853cd8b3dc25307beb044f8c78e", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-math-common7.14_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas-host7.14_7.14.0-3_amd64", + "name": "amdrocm-blas-host7.14", + "sha256": "d882b1402f11b5a512712ba582a52a73d47411392151b089e707082f918a0323", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas-host7.14_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas7.14-gfx942_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx942", + "sha256": "5f77db1f8df1275d3b97154d1680ef5387cc023e05f5ed782e957442524b1846", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx942_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver7.14-gfx942_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx942", + "sha256": "84718ac1480b4b9cd37d83702b4c711531665c950f96bc585e9374dc8546bb0a", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx942_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse7.14-gfx942_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx942", + "sha256": "a7a93ee9e98dcfc3b2b6a49b695e453482d3c72d55d180185265195a4a175ae0", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx942_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas7.14-gfx90a_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx90a", + "sha256": "2378006fc21ebc592bff0cca01ef1974ed1c18f5e1daa4059fa67eb032d7a3ec", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx90a_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver7.14-gfx90a_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx90a", + "sha256": "4bfb918d48762449d298b8c9b2cc9f81a7edbc047137139c26009c4a4fe66d59", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx90a_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse7.14-gfx90a_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx90a", + "sha256": "8c4e3ff68571f355376704fe0510edb0e70eb99a862f0bafe3fec873894231b0", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx90a_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas7.14-gfx908_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx908", + "sha256": "5d904096524027c02b6e2a91aba901a2527b24c233045f60410130e705ad8d27", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx908_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver7.14-gfx908_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx908", + "sha256": "0db8deab37561a8c3e8dc01071f93cc4d90464070ad1e81b7742b5ddac670a3e", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx908_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse7.14-gfx908_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx908", + "sha256": "849a815de62271f3f3fe775a11e3d5e9c270e136b02ae403adedb05b06f5b64a", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx908_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas7.14-gfx1250_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1250", + "sha256": "3549c185d9fd4cd2304b31eb1a923e61fe5f3ece58f65eff9cce814da6cc383b", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1250_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver7.14-gfx1250_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1250", + "sha256": "e3d3977896ebfa70842aab355f0e104401d7c5a0b6a4e6f69d02d6f38ac025de", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1250_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse7.14-gfx1250_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1250", + "sha256": "2073a0bf77e2dbef917286cc1f0b21012d4bd5494efacaa090bb31e447afbe01", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1250_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas7.14-gfx1201_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1201", + "sha256": "29af1bea27f6ce2e7be0fa4361e073364b67468e8dc7873b8782dfb00f777729", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1201_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver7.14-gfx1201_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1201", + "sha256": "c914feb757a5af0b015bee9f8604b5ecfaa034c23747c2ffa77ce567fc0c505c", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1201_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse7.14-gfx1201_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1201", + "sha256": "3a00cdceb9a4c88b2e8cd6d76c8ec3fe92dae8f3c5c2af51704efdaea7e6836a", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1201_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas7.14-gfx1200_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1200", + "sha256": "b8d908829bf0cbaa2699a52e86653d44a6312deee4f028bff2ef52da28d62e01", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1200_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver7.14-gfx1200_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1200", + "sha256": "12b347316dbedc95953d60509d1244711a6685b856615a767fbb05e34c82110c", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1200_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse7.14-gfx1200_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1200", + "sha256": "d341d18d28f201de15f5d2f42645e56e43ca0d84fe0f00ec08d4e41da8cad08e", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1200_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas7.14-gfx1153_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1153", + "sha256": "e66495b2a965237a9646af29971c25c7ab0626613963e88e2aaf3657f7ce09d8", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1153_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver7.14-gfx1153_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1153", + "sha256": "0acd58606717214c209178100a89d4943f4c061134c47e7dc88bace3a770c3f1", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1153_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse7.14-gfx1153_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1153", + "sha256": "87854d7d6ffa380d30b2dcb8f8737e39ad226fc184e9ceaa851ca84d973ea74e", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1153_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas7.14-gfx1152_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1152", + "sha256": "4bac4db8173aa54471bd32d22de071032c3a1ee45e9a6c76c5599e3cd21ba650", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1152_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver7.14-gfx1152_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1152", + "sha256": "7ebe93fdadf24b4a40f6f400919522084b5396aee017a3555c7b877362bd51bb", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1152_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse7.14-gfx1152_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1152", + "sha256": "a1485536d93d03ffc720c7b642a7cdd68fa7fbcc33e852deef79aadad4179653", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1152_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas7.14-gfx1151_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1151", + "sha256": "06918d9e3684158fd3bf7857403e3f3e8cd24d8a68ccd71018ea1e59e5c2199e", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1151_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver7.14-gfx1151_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1151", + "sha256": "0b6894acba6d0e63826271a664150a54386c25336cb30b27b4d6a08f8db4056b", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1151_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse7.14-gfx1151_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1151", + "sha256": "961e7e5903c0647b0f8df08b5f0e0ca334135f715c1bd02e0d95b10a5394540b", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1151_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas7.14-gfx1150_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1150", + "sha256": "62acba597d258961a03d1227ef6d1f8a79961a929ef5f19941fbdd38470b9f5c", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1150_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver7.14-gfx1150_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1150", + "sha256": "8fb4e92db83c9f75b7912843dfc2c30b2b9077dc6174a14c6b54ad1ca853a480", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1150_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse7.14-gfx1150_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1150", + "sha256": "2a1342a284990cdaf192b7907f1e5dbb7d8abc7b032dc7dd1bfd743ea30efae2", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1150_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas7.14-gfx1103_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1103", + "sha256": "69a45e92bc0d6062100518e2228ed7f30477ed74154c5d951c7c784fd699987d", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1103_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver7.14-gfx1103_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1103", + "sha256": "091c9bb0939d77b895c5957207a103086a5cf249360aa6709509b84413c5ad9c", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1103_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse7.14-gfx1103_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1103", + "sha256": "1536402b703fe143fdbbfa092276bdced04095bf993f8f4cf1ca2fbb30616f0b", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1103_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas7.14-gfx1102_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1102", + "sha256": "28f90078cc3afe13febfe69ba6b9dfe28a42781b06d26ecfbc3eb71a4b08c4ee", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1102_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver7.14-gfx1102_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1102", + "sha256": "ff6147c9adc3d721495cf9525f50789559990734a4343e69fdb4f40bd8c3dd99", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1102_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse7.14-gfx1102_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1102", + "sha256": "13b2c9c27738af4cdccdb9401fb3169cfde8590b3db784f3725c5168e2bf7f7c", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1102_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas7.14-gfx1101_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1101", + "sha256": "7be4aa68d39786bb64cb326621f4f7c9ec1e31df85e5a1300b2899df93f601b0", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1101_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver7.14-gfx1101_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1101", + "sha256": "41cbbfc85150022e4f692324daddad46cad908e692da2e969146d44c5c101de2", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1101_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse7.14-gfx1101_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1101", + "sha256": "e5f3ba34cbfcb41d6343536d4d06c454e5832c9ebe00290fb4ba3d66aba8ce44", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1101_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas7.14-gfx1100_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1100", + "sha256": "073e99c4fca56c93dfc54576a22ed7d6622dc968e0d0eb480c4c4dc8b51b1374", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1100_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver7.14-gfx1100_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1100", + "sha256": "5e3a1bca9c408f07d8eb9d6147005c99a635104ad3ec29bffabc46cd1198bd6c", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1100_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse7.14-gfx1100_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1100", + "sha256": "988f2f212f215bfc675884dacfcc81aa077e62a97ece927ce6432d5fbb6b616d", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1100_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas7.14-gfx1036_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1036", + "sha256": "127db0014b130e582b42c2d3cda0f66f3130130c719030eb254dfdecace825a3", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1036_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver7.14-gfx1036_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1036", + "sha256": "03a5a620a762acf72fd5130c1d63bb966ac9a913277bba93377e1da4c7678261", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1036_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse7.14-gfx1036_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1036", + "sha256": "47a9a9b9bf978d93183fa2091bd9b1c42659fa901479c294580d3f3611de4830", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1036_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas7.14-gfx1035_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1035", + "sha256": "2e1d9d05d10445bbf14fc5282321878e90ebb2e11cd513675adf5c1b8779c591", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1035_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver7.14-gfx1035_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1035", + "sha256": "0e775a542340bcf2fa90364b990f7897719604cccc02fc75c223995cb848820b", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1035_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse7.14-gfx1035_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1035", + "sha256": "5b64cd6d61a066516d25027f743855d15c72aa1e05845095bb1074f342292cc8", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1035_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas7.14-gfx1034_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1034", + "sha256": "eedca192ef7c17d0800ced37f613bb6ec73b7ec3e963a99c265f8d6ead4dda34", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1034_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver7.14-gfx1034_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1034", + "sha256": "1a65bae315a2371625a046fc3f4bc146f80064cd2badd59d4e1c9bd24f06f355", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1034_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse7.14-gfx1034_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1034", + "sha256": "c4e218ac85700a4f91fc0e9cbb69d4455b307269932383f76722d097cb6d5ae3", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1034_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas7.14-gfx1033_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1033", + "sha256": "768d266b8b89954e2b1c1b025d13273d1056f80d8ffafb3ba4e7731b56b6b260", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1033_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver7.14-gfx1033_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1033", + "sha256": "2025c0fe847ac052d474c73bf1e638c9f9c4aab6c8ae44ff434a2c59454ab397", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1033_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse7.14-gfx1033_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1033", + "sha256": "8a3fe04c8b88892710f9c0c15d1928b7bea8a63aad743a98085d9373ff46a4c6", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1033_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas7.14-gfx1032_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1032", + "sha256": "bd1eb0fd3b38ad2cc369ff3ae834902d28f53f6499e31f7031139cdfbfee8af5", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1032_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver7.14-gfx1032_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1032", + "sha256": "783ca6c75ee5c64d61038a91ac0b0ca677247afa591809e681698f498e8ecf0f", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1032_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse7.14-gfx1032_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1032", + "sha256": "43fadd0f6d36ffa653e72e41869c50c56efd579402419ec4476aeba3ab072cd4", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1032_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas7.14-gfx1031_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1031", + "sha256": "22cd13492b455071f866165df4f94b262dcd8919c2810d9ed614916c40102ef6", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1031_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver7.14-gfx1031_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1031", + "sha256": "6aa0219cce80ce25558fc4a0147a05afcfd26bbbe4d511313ce71f483874f748", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1031_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse7.14-gfx1031_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1031", + "sha256": "38319e83a0ac3591da259f059bf7c5407e119533dce6d9ed04748c5453bf516a", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1031_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas7.14-gfx1030_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1030", + "sha256": "1cf5b27e6dd02cfaee80ba0e9d1a691a169d4753a3ca7b3a71c7ce4fec821fdc", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1030_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver7.14-gfx1030_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1030", + "sha256": "4f083b45ebf3c0994e7f2bcc3a25425c90057da27e6cf0cda9b7136ea12a8b3a", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1030_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse7.14-gfx1030_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1030", + "sha256": "d16d68234b195ebc9c2cae77c288fc0c3e6ba2b8f9c92468acd7259e767c0c41", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1030_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas7.14-gfx1012_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1012", + "sha256": "4193b310cfb51a101e032e1bf7ab6dbd3dc7c3c8df959d512ddaf01a8ea758c4", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1012_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver7.14-gfx1012_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1012", + "sha256": "78e023559a533ddcffb4b313da935e01eeaffd0bc388ab5937c3adde92340c6a", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1012_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse7.14-gfx1012_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1012", + "sha256": "707d42d3b433e4c11105e0e9a32fc7b9254eb5f390a9476721ff72d318835925", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1012_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas7.14-gfx1011_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1011", + "sha256": "159e9129e7ef839e188a415c97f6b88280bf06020a5507c699e9dfd768ecdfc9", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1011_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver7.14-gfx1011_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1011", + "sha256": "e5d7ab7881ebe5d764548846e765822ef92d1f06c501206076d1cd75fa3f98a8", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1011_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse7.14-gfx1011_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1011", + "sha256": "4f398be0c30b507ef2cb9b42130c764a68a71b1f0caf9127754314c8c6ca953c", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1011_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas7.14-gfx1010_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1010", + "sha256": "2027bf09cad4242f7fc59df5bdc5fa165a62b61b4e2d14a7ab8ec61c62f2a845", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1010_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver7.14-gfx1010_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1010", + "sha256": "14d752fef6f49fc4de090a8400f9c392ccbe0bfbb068a55de34af022e5d2e6f3", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1010_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse7.14-gfx1010_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1010", + "sha256": "7bc93af4c56e7661c1ddda1d23bd9f3dbf2f99b9411814ea66d41dd566c29dd7", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1010_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + } + ], + "key": "amdrocm-blas-dev7.14_7.14.0-3_amd64", + "name": "amdrocm-blas-dev7.14", + "sha256": "faee0d272c0d16e63b019e8664bff61815a9d8395b9793520af2c59f89957058", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas-dev7.14_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "amdrocm-dnn7.14-gfx950_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx950", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx950_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx950", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx950_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx950", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx950_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx950", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse-host7.14_7.14.0-3_amd64", + "name": "amdrocm-sparse-host7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-runtime7.14_7.14.0-3_amd64", + "name": "amdrocm-runtime7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-llvm7.14_7.14.0-3_amd64", + "name": "amdrocm-llvm7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sysdeps7.14_7.14.0-3_amd64", + "name": "amdrocm-sysdeps7.14", + "version": "7.14.0-3" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, { "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", "name": "libtirpc3", @@ -3990,116 +3870,11 @@ "name": "libcom-err2", "version": "1.46.5-2ubuntu1.2" }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, { "key": "libnsl2_1.3.0-2build2_amd64", "name": "libnsl2", "version": "1.3.0-2build2" }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "comgr_3.0.0.70202-86_22.04_amd64", - "name": "comgr", - "version": "3.0.0.70202-86~22.04" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" - }, { "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", "name": "rpcsvc-proto", @@ -4121,177 +3896,9 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "rocminfo_1.0.0.70202-86_22.04_amd64", - "name": "rocminfo", - "version": "1.0.0.70202-86~22.04" - }, - { - "key": "pciutils_1-3.7.0-6_amd64", - "name": "pciutils", - "version": "1:3.7.0-6" - }, - { - "key": "libpci3_1-3.7.0-6_amd64", - "name": "libpci3", - "version": "1:3.7.0-6" - }, - { - "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", - "name": "pci.ids", - "version": "0.0~2022.01.22-1ubuntu0.1" - }, - { - "key": "libudev1_249.11-0ubuntu3.16_amd64", - "name": "libudev1", - "version": "249.11-0ubuntu3.16" - }, - { - "key": "libkmod2_29-1ubuntu1_amd64", - "name": "libkmod2", - "version": "29-1ubuntu1" - }, - { - "key": "hsa-rocr_1.18.0.70202-86_22.04_amd64", - "name": "hsa-rocr", - "version": "1.18.0.70202-86~22.04" - }, - { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" - }, - { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.125.70201-2303469.22.04" - }, - { - "key": "libdrm-amdgpu-common_1.0.0.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-common", - "version": "1.0.0.70201-2303469.22.04" - }, - { - "key": "amdgpu-core_1-7.2.70201-2303469.22.04_amd64", - "name": "amdgpu-core", - "version": "1:7.2.70201-2303469.22.04" - }, - { - "key": "libdrm2-amdgpu_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm2-amdgpu", - "version": "1:2.4.125.70201-2303469.22.04" - } - ], - "key": "hip-runtime-amd_7.2.53211.70202-86_22.04_amd64", - "name": "hip-runtime-amd", - "sha256": "4c70edc23440b7e7c99b5a9985538adc725d59ce7ffc97b2191db1d598c0708b", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/h/hip-runtime-amd/hip-runtime-amd_7.2.53211.70202-86~22.04_amd64.deb" - ], - "version": "7.2.53211.70202-86~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "rocminfo_1.0.0.70202-86_22.04_amd64", - "name": "rocminfo", - "sha256": "2373c0f0891e115914abfd4fa66358bfcae44236c1bae8c613ccd38c7da40c20", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocminfo/rocminfo_1.0.0.70202-86~22.04_amd64.deb" - ], - "version": "1.0.0.70202-86~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "pciutils_1-3.7.0-6_amd64", - "name": "pciutils", - "sha256": "830f2957662723860d3ebac949b22867df00d9a4c5ab1c2bf98e10674dc1c756", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pciutils/pciutils_3.7.0-6_amd64.deb" - ], - "version": "1:3.7.0-6" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libpci3_1-3.7.0-6_amd64", - "name": "libpci3", - "sha256": "624bea7b5a9073a5bc2dfa5d956571d6127c20400fcfcfca1e47b7448ba5d351", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pciutils/libpci3_3.7.0-6_amd64.deb" - ], - "version": "1:3.7.0-6" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", - "name": "pci.ids", - "sha256": "026609ed71363cd6ed13ace2dac0c9813d94be3874c778539a8a64d6d2cc5566", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pci.ids/pci.ids_0.0~2022.01.22-1ubuntu0.1_all.deb" - ], - "version": "0.0~2022.01.22-1ubuntu0.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libudev1_249.11-0ubuntu3.16_amd64", - "name": "libudev1", - "sha256": "451d4eaa84dbeabc045611937a78489710ad2c2baf277a63aacda7ee5226e3ea", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/s/systemd/libudev1_249.11-0ubuntu3.16_amd64.deb" - ], - "version": "249.11-0ubuntu3.16" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libkmod2_29-1ubuntu1_amd64", - "name": "libkmod2", - "sha256": "f355cebb66405a84cc88bb313440abd39f97cf45d3f468ea9ec7ba34e11b6c8a", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/k/kmod/libkmod2_29-1ubuntu1_amd64.deb" - ], - "version": "29-1ubuntu1" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "rocm-core_7.2.2.70202-86_22.04_amd64", - "name": "rocm-core", - "version": "7.2.2.70202-86~22.04" + "key": "amdrocm-base7.14_7.14.0-3_amd64", + "name": "amdrocm-base7.14", + "version": "7.14.0-3" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -4313,6 +3920,658 @@ "name": "libuuid1", "version": "2.37.2-4ubuntu3.4" }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "amdrocm-profiler-base7.14_7.14.0-3_amd64", + "name": "amdrocm-profiler-base7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver-host7.14_7.14.0-3_amd64", + "name": "amdrocm-solver-host7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-math-common7.14_7.14.0-3_amd64", + "name": "amdrocm-math-common7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas-host7.14_7.14.0-3_amd64", + "name": "amdrocm-blas-host7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-dnn-host7.14_7.14.0-3_amd64", + "name": "amdrocm-dnn-host7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-dnn7.14-gfx942_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx942", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx942_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx942", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx942_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx942", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx942_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx942", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-dnn7.14-gfx90a_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx90a", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx90a_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx90a", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx90a_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx90a", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx90a_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx90a", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-dnn7.14-gfx908_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx908", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx908_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx908", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx908_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx908", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx908_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx908", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-dnn7.14-gfx1201_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx1201", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1201_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1201", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1201_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1201", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1201_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1201", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-dnn7.14-gfx1200_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx1200", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1200_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1200", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1200_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1200", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1200_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1200", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-dnn7.14-gfx1153_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx1153", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1153_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1153", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1153_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1153", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1153_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1153", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-dnn7.14-gfx1152_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx1152", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1152_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1152", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1152_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1152", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1152_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1152", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-dnn7.14-gfx1151_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx1151", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1151_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1151", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1151_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1151", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1151_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1151", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-dnn7.14-gfx1150_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx1150", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1150_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1150", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1150_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1150", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1150_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1150", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-dnn7.14-gfx1102_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx1102", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1102_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1102", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1102_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1102", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1102_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1102", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-dnn7.14-gfx1101_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx1101", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1101_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1101", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1101_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1101", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1101_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1101", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-dnn7.14-gfx1100_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx1100", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1100_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1100", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1100_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1100", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1100_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1100", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-dnn7.14-gfx1030_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx1030", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1030_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1030", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1030_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1030", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1030_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1030", + "version": "7.14.0-3" + } + ], + "key": "amdrocm-dnn7.14_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14", + "sha256": "c4bb3af3ca298c2ac846e76380f16f66ccce584781098ed142039755358b4d7e", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn7.14-gfx950_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx950", + "sha256": "b8addad1da1bbbc0764622166c50f89e79a525b58ce0a0ae7ec0e1e4d819ba3c", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx950_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn-host7.14_7.14.0-3_amd64", + "name": "amdrocm-dnn-host7.14", + "sha256": "19341008efe6153a7c3ab4cc7d4159098f44fb57aa35b014b8a23bcdf438773d", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn-host7.14_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn7.14-gfx942_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx942", + "sha256": "53cb410c17bda3ec250b1942c45915f008b1b8bf6d1ffe27881dc3fd817ce190", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx942_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn7.14-gfx90a_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx90a", + "sha256": "364d71a34fab72e903a98e29f0c7ad1e0b8a59785b4200df8bec9fb072d56996", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx90a_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn7.14-gfx908_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx908", + "sha256": "f512008bae762a5e607872d1df43d42d7583c4e1d26aadebd75a852e6833a97c", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx908_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn7.14-gfx1201_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx1201", + "sha256": "d3d3f74e63493aa66e2e3ed3ba25aa334befe70d1cb8df20c06fc01a02e205af", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1201_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn7.14-gfx1200_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx1200", + "sha256": "0f4a6bf4785f434186eafc32303631e96fbf5879a9898b8ac192394ea80be9ba", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1200_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn7.14-gfx1153_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx1153", + "sha256": "e34111409adf768e49e1074cdcfd9befdad4f1f15baabe09bc83bb62719d9ec8", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1153_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn7.14-gfx1152_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx1152", + "sha256": "ebfe98d8eda69e6e96728247fc05a945409c88f8d782fbb4dbc39626c2de617a", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1152_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn7.14-gfx1151_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx1151", + "sha256": "a0bbb2c8bafbd44583854ed0cbd76078adcc60d8ed0b5405af99107dc6bca3b6", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1151_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn7.14-gfx1150_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx1150", + "sha256": "19fd6d95d7fd72bedc6469d03768bb8e80d4f03d20ba115bbfe1e3436493d813", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1150_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn7.14-gfx1102_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx1102", + "sha256": "fa8a0f778d4a1cac0e5a35beda20eab50e5d09968be4d046d70e010f850613ae", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1102_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn7.14-gfx1101_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx1101", + "sha256": "cabe934788d36f37fb063433819a9a98c9d4b081398344fbaae0debe3a6c01ba", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1101_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn7.14-gfx1100_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx1100", + "sha256": "0fb351aa31b7f3d217ad9da70689f7581531d6e1cb4171fbea53b67dfc76619a", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1100_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn7.14-gfx1030_7.14.0-3_amd64", + "name": "amdrocm-dnn7.14-gfx1030", + "sha256": "634771d432604d5514a460babf13fb3270edef6225b0d56961d582d783bdb2a2", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1030_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "amdrocm-fft7.14-gfx950_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx950", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-fft-host7.14_7.14.0-3_amd64", + "name": "amdrocm-fft-host7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sysdeps7.14_7.14.0-3_amd64", + "name": "amdrocm-sysdeps7.14", + "version": "7.14.0-3" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "amdrocm-runtime7.14_7.14.0-3_amd64", + "name": "amdrocm-runtime7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-llvm7.14_7.14.0-3_amd64", + "name": "amdrocm-llvm7.14", + "version": "7.14.0-3" + }, + { + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, { "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", "name": "libtirpc3", @@ -4413,136 +4672,11 @@ "name": "libcom-err2", "version": "1.46.5-2ubuntu1.2" }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, { "key": "libnsl2_1.3.0-2build2_amd64", "name": "libnsl2", "version": "1.3.0-2build2" }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "rocm-smi-lib_7.8.0.70202-86_22.04_amd64", - "name": "rocm-smi-lib", - "version": "7.8.0.70202-86~22.04" - }, - { - "key": "hip-runtime-amd_7.2.53211.70202-86_22.04_amd64", - "name": "hip-runtime-amd", - "version": "7.2.53211.70202-86~22.04" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "rocprofiler-register_0.6.0.70202-86_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.6.0.70202-86~22.04" - }, - { - "key": "comgr_3.0.0.70202-86_22.04_amd64", - "name": "comgr", - "version": "3.0.0.70202-86~22.04" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" - }, { "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", "name": "rpcsvc-proto", @@ -4564,97 +4698,537 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", + "key": "amdrocm-base7.14_7.14.0-3_amd64", + "name": "amdrocm-base7.14", + "version": "7.14.0-3" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70202-86_22.04_amd64", - "name": "rocminfo", - "version": "1.0.0.70202-86~22.04" + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" }, { - "key": "pciutils_1-3.7.0-6_amd64", - "name": "pciutils", - "version": "1:3.7.0-6" + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" }, { - "key": "libpci3_1-3.7.0-6_amd64", - "name": "libpci3", - "version": "1:3.7.0-6" + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" }, { - "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", - "name": "pci.ids", - "version": "0.0~2022.01.22-1ubuntu0.1" + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" }, { - "key": "libudev1_249.11-0ubuntu3.16_amd64", - "name": "libudev1", - "version": "249.11-0ubuntu3.16" + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" }, { - "key": "libkmod2_29-1ubuntu1_amd64", - "name": "libkmod2", - "version": "29-1ubuntu1" + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" }, { - "key": "hsa-rocr_1.18.0.70202-86_22.04_amd64", - "name": "hsa-rocr", - "version": "1.18.0.70202-86~22.04" + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" }, { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" }, { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.125.70201-2303469.22.04" + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" }, { - "key": "libdrm-amdgpu-common_1.0.0.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-common", - "version": "1.0.0.70201-2303469.22.04" + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" }, { - "key": "amdgpu-core_1-7.2.70201-2303469.22.04_amd64", - "name": "amdgpu-core", - "version": "1:7.2.70201-2303469.22.04" + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" }, { - "key": "libdrm2-amdgpu_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm2-amdgpu", - "version": "1:2.4.125.70201-2303469.22.04" + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "amdrocm-fft7.14-gfx942_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx942", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-fft7.14-gfx90a_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx90a", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-fft7.14-gfx908_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx908", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-fft7.14-gfx1250_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1250", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-fft7.14-gfx1201_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1201", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-fft7.14-gfx1200_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1200", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-fft7.14-gfx1153_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1153", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-fft7.14-gfx1152_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1152", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-fft7.14-gfx1151_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1151", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-fft7.14-gfx1150_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1150", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-fft7.14-gfx1103_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1103", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-fft7.14-gfx1102_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1102", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-fft7.14-gfx1101_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1101", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-fft7.14-gfx1100_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1100", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-fft7.14-gfx1036_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1036", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-fft7.14-gfx1035_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1035", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-fft7.14-gfx1034_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1034", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-fft7.14-gfx1033_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1033", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-fft7.14-gfx1032_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1032", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-fft7.14-gfx1031_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1031", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-fft7.14-gfx1030_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1030", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-fft7.14-gfx1012_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1012", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-fft7.14-gfx1011_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1011", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-fft7.14-gfx1010_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1010", + "version": "7.14.0-3" } ], - "key": "rccl_2.27.7.70202-86_22.04_amd64", - "name": "rccl", - "sha256": "942e98104888cc56bbe136d70b771b8dceba241ebfb1f460aebe5d01dd425ea9", + "key": "amdrocm-fft7.14_7.14.0-3_amd64", + "name": "amdrocm-fft7.14", + "sha256": "9dd7c4e05ded2098254acf778d2d9e9a9274cd1bb80d86f00cde373797faf54a", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rccl/rccl_2.27.7.70202-86~22.04_amd64.deb" + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14_7.14.0-3_amd64.deb" ], - "version": "2.27.7.70202-86~22.04" + "version": "7.14.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "rocm-smi-lib_7.8.0.70202-86_22.04_amd64", - "name": "rocm-smi-lib", - "sha256": "ee8c1d211b1db5d8340abfbe0d4cae99866a365f3f04f84445b97ba5394b00e2", + "key": "amdrocm-fft7.14-gfx950_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx950", + "sha256": "f098768c3a9dab1b2870e4ef0f4d38e6e40b3afd70d918e3ba02bbd58c99495b", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocm-smi-lib/rocm-smi-lib_7.8.0.70202-86~22.04_amd64.deb" + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx950_7.14.0-3_amd64.deb" ], - "version": "7.8.0.70202-86~22.04" + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft-host7.14_7.14.0-3_amd64", + "name": "amdrocm-fft-host7.14", + "sha256": "69fd1ad23c2b45af82595ccb779390e7edf88f34d4e029ed05f53a4f3040f7c5", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft-host7.14_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft7.14-gfx942_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx942", + "sha256": "0bd770f086eab2607a6503ac5fb5d140d5978ba5876a679f69fd46f120b893f0", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx942_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft7.14-gfx90a_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx90a", + "sha256": "bb3fefd71fe9ebdc0926dadb680206674247d22f8c981cb779dfe2b915b35f28", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx90a_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft7.14-gfx908_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx908", + "sha256": "3e9e6f25d32aa37e26fcea8c3768ee139be7c21d08b350cea7fbd7a067304b31", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx908_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft7.14-gfx1250_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1250", + "sha256": "192be4897e6cc0ff63ca41b869f722d6271088e5eab862742ffe4c5cdba161b4", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1250_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft7.14-gfx1201_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1201", + "sha256": "803e967a36ed13de102f9a682027df3b86e82ecef5dc01a90ee2afd24396ed04", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1201_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft7.14-gfx1200_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1200", + "sha256": "c68b37108c9e28a35159565785a53d97f6e65e1bab50da6a3b44e625ef5f8404", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1200_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft7.14-gfx1153_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1153", + "sha256": "c18c459194e38022b130dd3b36d08858fa6016a0266f0f38b869654beca3b8be", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1153_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft7.14-gfx1152_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1152", + "sha256": "6f0603072a41775facda20c7e8bed77fef65d0fc5c29cf30bb2610bc801fa789", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1152_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft7.14-gfx1151_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1151", + "sha256": "838583cc6141c1e03e737cce5a3dd7ec769b017353c0599ebd2dde8f1ed0e42d", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1151_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft7.14-gfx1150_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1150", + "sha256": "fb6f1145c9cb44d41114b83600974b853982589ac4d0a819e3e16ad0bbc77784", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1150_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft7.14-gfx1103_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1103", + "sha256": "7f8c2ea4a9f49ebc55604d8e27df6fa10874c3767831c71cb31d3b98b26416dc", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1103_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft7.14-gfx1102_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1102", + "sha256": "80142da4ae1481339589db8c07f85b0618af98d73351eb3dc6561f1c073cdd22", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1102_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft7.14-gfx1101_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1101", + "sha256": "183085a543f9b6c3a5af9299f1b142d5edc7c0b54de081e35ebe7ee87b178a0a", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1101_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft7.14-gfx1100_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1100", + "sha256": "323dbfea5dd39d8240aa42a05d9b8d58bed81014bf7ee469b4b363e33e692901", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1100_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft7.14-gfx1036_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1036", + "sha256": "610f90101bb496f6bf4f902b51852d823fdbc55941e2cfb489e167d2e7e40623", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1036_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft7.14-gfx1035_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1035", + "sha256": "0151828a3521b62c6a160e931e407b6e44ca50ec595dc03efa7d430542c6567c", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1035_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft7.14-gfx1034_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1034", + "sha256": "0d38e34fb0a839131f1e4556be30ec3eb36cb3d36676bedb44176785a3f1bc45", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1034_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft7.14-gfx1033_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1033", + "sha256": "4d6c786de4b29fbe49400559942344e1cb2ddb394cdbbe216f8ae2404c7ed95f", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1033_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft7.14-gfx1032_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1032", + "sha256": "f41f876967f28e36a8b105778d44dc327bd8f5b6d55b7d3d1af6c2572dc067fe", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1032_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft7.14-gfx1031_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1031", + "sha256": "09860c022cf6f27045b0ffb05c80dac7bd1bfeb059c51347104c6f34e273bed6", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1031_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft7.14-gfx1030_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1030", + "sha256": "eae84444e0efce3b635f8ad4e2ae281768c2bd7571e20763a996e90b23feabcd", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1030_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft7.14-gfx1012_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1012", + "sha256": "8cee80ba00d14c89517a42bfba30bd951282beea1def52c8a0fff67320c95daf", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1012_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft7.14-gfx1011_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1011", + "sha256": "a28c56a34213c95112a947c646ed7ceeab3ef243fa535fa681859301c6def686", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1011_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft7.14-gfx1010_7.14.0-3_amd64", + "name": "amdrocm-fft7.14-gfx1010", + "sha256": "cd38bdbe5e0fb0d20dbc25561426bab914c58cb5e8e4ea143321b22f2e26cb44", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1010_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.2.70202-86_22.04_amd64", - "name": "rocm-core", - "version": "7.2.2.70202-86~22.04" + "key": "amdrocm-rccl7.14-gfx950_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx950", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-rccl-host7.14_7.14.0-3_amd64", + "name": "amdrocm-rccl-host7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-profiler-base7.14_7.14.0-3_amd64", + "name": "amdrocm-profiler-base7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sysdeps7.14_7.14.0-3_amd64", + "name": "amdrocm-sysdeps7.14", + "version": "7.14.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -4677,24 +5251,89 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" + "key": "amdrocm-runtime7.14_7.14.0-3_amd64", + "name": "amdrocm-runtime7.14", + "version": "7.14.0-3" }, { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" + "key": "amdrocm-llvm7.14_7.14.0-3_amd64", + "name": "amdrocm-llvm7.14", + "version": "7.14.0-3" }, { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" }, { "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", @@ -4796,6 +5435,56 @@ "name": "libcom-err2", "version": "1.46.5-2ubuntu1.2" }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "amdrocm-base7.14_7.14.0-3_amd64", + "name": "amdrocm-base7.14", + "version": "7.14.0-3" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, { "key": "libtinfo6_6.3-2ubuntu0.1_amd64", "name": "libtinfo6", @@ -4816,11 +5505,6 @@ "name": "readline-common", "version": "8.1.2-1" }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, { "key": "libncursesw6_6.3-2ubuntu0.1_amd64", "name": "libncursesw6", @@ -4831,11 +5515,6 @@ "name": "libmpdec3", "version": "2.5.1-2build2" }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, { "key": "libffi8_3.4.2-4_amd64", "name": "libffi8", @@ -4875,33 +5554,385 @@ "key": "python3-minimal_3.10.6-1_22.04.1_amd64", "name": "python3-minimal", "version": "3.10.6-1~22.04.1" + }, + { + "key": "amdrocm-amdsmi7.14_7.14.0-3_amd64", + "name": "amdrocm-amdsmi7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-rccl7.14-gfx942_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx942", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-rccl7.14-gfx90a_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx90a", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-rccl7.14-gfx908_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx908", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-rccl7.14-gfx1250_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1250", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-rccl7.14-gfx1201_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1201", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-rccl7.14-gfx1200_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1200", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-rccl7.14-gfx1151_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1151", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-rccl7.14-gfx1102_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1102", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-rccl7.14-gfx1101_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1101", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-rccl7.14-gfx1100_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1100", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-rccl7.14-gfx1036_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1036", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-rccl7.14-gfx1035_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1035", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-rccl7.14-gfx1034_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1034", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-rccl7.14-gfx1033_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1033", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-rccl7.14-gfx1032_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1032", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-rccl7.14-gfx1031_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1031", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-rccl7.14-gfx1030_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1030", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-rccl7.14-gfx1012_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1012", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-rccl7.14-gfx1011_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1011", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-rccl7.14-gfx1010_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1010", + "version": "7.14.0-3" } ], - "key": "rocm-device-libs_1.0.0.70202-86_22.04_amd64", - "name": "rocm-device-libs", - "sha256": "ceecef256d804a47a2d57184920cd9a1cbd6190937332998f2200d8b18a5fb27", + "key": "amdrocm-rccl7.14_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14", + "sha256": "f872534a2a7e87f2b6d33f82aa3f6a7f7f1ba7c126129ef0a10efb19c8c996b0", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocm-device-libs/rocm-device-libs_1.0.0.70202-86~22.04_amd64.deb" + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14_7.14.0-3_amd64.deb" ], - "version": "1.0.0.70202-86~22.04" + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl7.14-gfx950_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx950", + "sha256": "5e8f399ffe83df15c2d92f7c992d15b274dc93c68b9ff7a2558d5e7b032a6fcb", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx950_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl-host7.14_7.14.0-3_amd64", + "name": "amdrocm-rccl-host7.14", + "sha256": "43f64145318b9fc02c2a76e43f2bc529fc4ea768f24a3b24a7a11c98693c8214", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl-host7.14_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl7.14-gfx942_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx942", + "sha256": "4d5feafba0fd901d2416cc8c7f4970c4172f332880c334ab2cfb225aeb32c5d7", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx942_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl7.14-gfx90a_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx90a", + "sha256": "2a00389b4d7f01830f464a8f55fe9bf24a2a7238c1109fcf46d2d92a83716d6a", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx90a_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl7.14-gfx908_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx908", + "sha256": "85262a9f0bcf3356748a970fa78ae7c4747b5a235636607dd09a4dc37b4b2940", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx908_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl7.14-gfx1250_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1250", + "sha256": "4a73a6de53f819b066397e1bbcb734fe7f7c0c515eaa57046f286c4efd55e05d", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1250_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl7.14-gfx1201_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1201", + "sha256": "a699ddb683d8a0c640f021e36da256843030281fee0fe538756386113bf61699", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1201_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl7.14-gfx1200_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1200", + "sha256": "27a35cf5734dc2602c8f68692bfb7434b76eac8134ab54bfb04a5d6fd61ecf13", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1200_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl7.14-gfx1151_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1151", + "sha256": "bdfb7c11cf24d9d94dc5aac2e516ac463046622dea4659e26fbc09f1c1d0459c", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1151_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl7.14-gfx1102_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1102", + "sha256": "9fb0ff90c45be824c48b0ad8e91c11ec6300034a584119c7dbf263e2ebda7f3f", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1102_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl7.14-gfx1101_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1101", + "sha256": "ecabf24eb13f23702fc4b9412fbcc8fe778d98c19a55d6e3e932d7541f8cfd8f", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1101_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl7.14-gfx1100_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1100", + "sha256": "a2be1b2d47dd46dd389d282dedc2b1b78a94657bc02699e2bbd6ddd25de2d8c8", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1100_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl7.14-gfx1036_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1036", + "sha256": "3265c4d45b42aabc98775015eedb3cc0f14396fa413f24f754224bc391c2a38a", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1036_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl7.14-gfx1035_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1035", + "sha256": "77f2f1d81c52a7168c9cd85f145987077565a289377814ada074d69b5f2427f9", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1035_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl7.14-gfx1034_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1034", + "sha256": "630acf3eae8ec24edf029012afa060e8242bfb059b4affa4c60188679cf2db64", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1034_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl7.14-gfx1033_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1033", + "sha256": "56f154f9e9fd2ddaa07f447e6487bd83cca5478aa136686f73c3e1785a3299c9", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1033_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl7.14-gfx1032_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1032", + "sha256": "76e368957e1990585754d6d08c17fdb0d60e344382029cb52808862e30b81570", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1032_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl7.14-gfx1031_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1031", + "sha256": "bcdce1101be547acb05026941fb44656bfd023e8b357b58e4cd2a5e24c4fc851", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1031_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl7.14-gfx1030_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1030", + "sha256": "ed957d5a57e00950c46878bfec5dd7a92fcf547840132b0071eb5facd1bea5d9", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1030_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl7.14-gfx1012_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1012", + "sha256": "8fa4e40c0a0c5c32b0f71fbf8343fd465940c39f1843c93fe4501abcbd2b3dcf", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1012_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl7.14-gfx1011_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1011", + "sha256": "93ceea63db909a8f6ab2435d057b7bc7dbf22575c4b4eb26da38f268e193d18e", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1011_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl7.14-gfx1010_7.14.0-3_amd64", + "name": "amdrocm-rccl7.14-gfx1010", + "sha256": "b67d6fade488b33cce5a69d66d01940bc0f61632fab5624d21c95b7f824702ef", + "urls": [ + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1010_7.14.0-3_amd64.deb" + ], + "version": "7.14.0-3" }, { "arch": "amd64", "dependencies": [ { - "key": "hipcc_1.1.1.70202-86_22.04_amd64", - "name": "hipcc", - "version": "1.1.1.70202-86~22.04" + "key": "amdrocm-llvm-dev7.14_7.14.0-3_amd64", + "name": "amdrocm-llvm-dev7.14", + "version": "7.14.0-3" }, { - "key": "rocm-llvm_22.0.0.26084.70202-86_22.04_amd64", - "name": "rocm-llvm", - "version": "22.0.0.26084.70202-86~22.04" + "key": "amdrocm-llvm7.14_7.14.0-3_amd64", + "name": "amdrocm-llvm7.14", + "version": "7.14.0-3" }, { - "key": "rocm-core_7.2.2.70202-86_22.04_amd64", - "name": "rocm-core", - "version": "7.2.2.70202-86~22.04" + "key": "amdrocm-runtime7.14_7.14.0-3_amd64", + "name": "amdrocm-runtime7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-base7.14_7.14.0-3_amd64", + "name": "amdrocm-base7.14", + "version": "7.14.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -5123,6 +6154,11 @@ "name": "python3-minimal", "version": "3.10.6-1~22.04.1" }, + { + "key": "amdrocm-sysdeps7.14_7.14.0-3_amd64", + "name": "amdrocm-sysdeps7.14", + "version": "7.14.0-3" + }, { "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", "name": "libgcc-11-dev", @@ -5212,362 +6248,54 @@ "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", "name": "libc-dev-bin", "version": "2.35-0ubuntu3.10" - }, - { - "key": "hsa-rocr-dev_1.18.0.70202-86_22.04_amd64", - "name": "hsa-rocr-dev", - "version": "1.18.0.70202-86~22.04" - }, - { - "key": "hsa-rocr_1.18.0.70202-86_22.04_amd64", - "name": "hsa-rocr", - "version": "1.18.0.70202-86~22.04" - }, - { - "key": "rocprofiler-register_0.6.0.70202-86_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.6.0.70202-86~22.04" - }, - { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.125.70201-2303469.22.04" - }, - { - "key": "libdrm-amdgpu-common_1.0.0.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-common", - "version": "1.0.0.70201-2303469.22.04" - }, - { - "key": "amdgpu-core_1-7.2.70201-2303469.22.04_amd64", - "name": "amdgpu-core", - "version": "1:7.2.70201-2303469.22.04" - }, - { - "key": "libdrm2-amdgpu_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm2-amdgpu", - "version": "1:2.4.125.70201-2303469.22.04" - }, - { - "key": "libdrm-amdgpu-dev_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-dev", - "version": "1:2.4.125.70201-2303469.22.04" - }, - { - "key": "libdrm-amdgpu-radeon1_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-radeon1", - "version": "1:2.4.125.70201-2303469.22.04" - }, - { - "key": "hip-runtime-amd_7.2.53211.70202-86_22.04_amd64", - "name": "hip-runtime-amd", - "version": "7.2.53211.70202-86~22.04" - }, - { - "key": "comgr_3.0.0.70202-86_22.04_amd64", - "name": "comgr", - "version": "3.0.0.70202-86~22.04" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "rocminfo_1.0.0.70202-86_22.04_amd64", - "name": "rocminfo", - "version": "1.0.0.70202-86~22.04" - }, - { - "key": "pciutils_1-3.7.0-6_amd64", - "name": "pciutils", - "version": "1:3.7.0-6" - }, - { - "key": "libpci3_1-3.7.0-6_amd64", - "name": "libpci3", - "version": "1:3.7.0-6" - }, - { - "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", - "name": "pci.ids", - "version": "0.0~2022.01.22-1ubuntu0.1" - }, - { - "key": "libudev1_249.11-0ubuntu3.16_amd64", - "name": "libudev1", - "version": "249.11-0ubuntu3.16" - }, - { - "key": "libkmod2_29-1ubuntu1_amd64", - "name": "libkmod2", - "version": "29-1ubuntu1" - }, - { - "key": "file_1-5.41-3ubuntu0.1_amd64", - "name": "file", - "version": "1:5.41-3ubuntu0.1" - }, - { - "key": "libmagic1_1-5.41-3ubuntu0.1_amd64", - "name": "libmagic1", - "version": "1:5.41-3ubuntu0.1" - }, - { - "key": "libmagic-mgc_1-5.41-3ubuntu0.1_amd64", - "name": "libmagic-mgc", - "version": "1:5.41-3ubuntu0.1" } ], - "key": "hip-dev_7.2.53211.70202-86_22.04_amd64", - "name": "hip-dev", - "sha256": "97fbde73590f02e5253d8287006269cbb8885d3b0d1366071bc96d3f51f598e5", + "key": "amdrocm-runtime-dev7.14_7.14.0-3_amd64", + "name": "amdrocm-runtime-dev7.14", + "sha256": "443661803de9a556c6cdba9adbf8fd9a1bfd8dadbea8b31ea1e0a9236e97f5c9", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/h/hip-dev/hip-dev_7.2.53211.70202-86~22.04_amd64.deb" + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-runtime-dev7.14_7.14.0-3_amd64.deb" ], - "version": "7.2.53211.70202-86~22.04" + "version": "7.14.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "hipcc_1.1.1.70202-86_22.04_amd64", - "name": "hipcc", - "sha256": "2b2a0e83fa1f4d8929e12c532bef83914b56c68c651c34bb96c77deff8a9133b", + "key": "amdrocm-llvm-dev7.14_7.14.0-3_amd64", + "name": "amdrocm-llvm-dev7.14", + "sha256": "2a99eefb2648d2fa02a6b40ca7b5e7a8426e2f4db86d3fb0525dfefd2eebc404", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/h/hipcc/hipcc_1.1.1.70202-86~22.04_amd64.deb" + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-llvm-dev7.14_7.14.0-3_amd64.deb" ], - "version": "1.1.1.70202-86~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "rocm-llvm_22.0.0.26084.70202-86_22.04_amd64", - "name": "rocm-llvm", - "sha256": "506d8b818edea66827fc2d36f1f2d883245df6ae72afdbfd51a6ef4e3d50b067", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocm-llvm/rocm-llvm_22.0.0.26084.70202-86~22.04_amd64.deb" - ], - "version": "22.0.0.26084.70202-86~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libgcc-11-dev", - "sha256": "06faa64479f1987e4f870ff9c1ccfdf3c0c441bd3886136d6e24c11316c9f3d6", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-11/libgcc-11-dev_11.4.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", - "name": "libquadmath0", - "sha256": "6bf564c2bb191aefcf96758502a7ac24b2af4d644c87b3678e4b1107c5494b13", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libquadmath0_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libubsan1", - "sha256": "becb1726d3b8a45cfd336e502fc29d7c43e024689857c670ecf75a5362732c63", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libubsan1_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", - "name": "libtsan0", - "sha256": "7b14e81317fce5e62d6cc2454d975d3ec4eac90710a1a536d63c9ee84a50b6f2", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-11/libtsan0_11.4.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", - "name": "gcc-11-base", - "sha256": "5448bba72109dbf2df6ee30463b5f06c074138006cd1d5730fb5189b93e6d563", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-11/gcc-11-base_11.4.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", - "name": "liblsan0", - "sha256": "606e212c3cbc8fb622313fdb32f78a024b4b5b9fe54b1cc5a8f731297023b899", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/liblsan0_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", - "name": "libasan6", - "sha256": "0580e3209722dd0e6d9e57c4149a0f8f6073f975cf8b5ede26ac54289c2fc99a", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-11/libasan6_11.4.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libatomic1", - "sha256": "56ec991983c7c89a2c15bd5134534531d722ec5b51a60679f7ceaa1a09b37a09", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libatomic1_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libitm1", - "sha256": "ac3c96ed50a42b4f57c410576bdf816710684e3e32e34f704a7295014565b724", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libitm1_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgomp1", - "sha256": "072a7ee2dc00bc92a816e11ae62ad5a497c86fc1f0473c0a095f29e13003da72", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libgomp1_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libstdc++-11-dev", - "sha256": "5ab9c245d4c65a3986d567c54e2503251c92bd369c9f42e48a2c4b1f1835dcf5", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-11/libstdc++-11-dev_11.4.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "hsa-rocr-dev_1.18.0.70202-86_22.04_amd64", - "name": "hsa-rocr-dev", - "sha256": "50cffa604bad1b459d2c06252c1aca80134cab6b31df4d595e078c51c77de079", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/h/hsa-rocr-dev/hsa-rocr-dev_1.18.0.70202-86~22.04_amd64.deb" - ], - "version": "1.18.0.70202-86~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libdrm-amdgpu-dev_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-dev", - "sha256": "23b402f6a8915f401e26a3986429b2a64207477e5eec99293021e608ed812ea3", - "urls": [ - "https://repo.radeon.com/graphics/7.2.1/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-dev_2.4.125.70201-2303469.22.04_amd64.deb" - ], - "version": "1:2.4.125.70201-2303469.22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libdrm-amdgpu-radeon1_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-radeon1", - "sha256": "4b735739687e0796c3d967397ffead2d81499acab208f7e1709342b237657ef3", - "urls": [ - "https://repo.radeon.com/graphics/7.2.1/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-radeon1_2.4.125.70201-2303469.22.04_amd64.deb" - ], - "version": "1:2.4.125.70201-2303469.22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "file_1-5.41-3ubuntu0.1_amd64", - "name": "file", - "sha256": "b63b48e8b32c6f0fce5ad37d5ec3269cbbdc8feed13170aae1f416a43ebe898b", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/f/file/file_5.41-3ubuntu0.1_amd64.deb" - ], - "version": "1:5.41-3ubuntu0.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libmagic1_1-5.41-3ubuntu0.1_amd64", - "name": "libmagic1", - "sha256": "344d0cc65fea4175ca1ab222c20b5b95fb1357acafea6b016add9988760829d0", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/f/file/libmagic1_5.41-3ubuntu0.1_amd64.deb" - ], - "version": "1:5.41-3ubuntu0.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libmagic-mgc_1-5.41-3ubuntu0.1_amd64", - "name": "libmagic-mgc", - "sha256": "23fcefb8b9ee73eb74e127fc230330918a8f81e127fa54015f191cfd34726b3c", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/f/file/libmagic-mgc_5.41-3ubuntu0.1_amd64.deb" - ], - "version": "1:5.41-3ubuntu0.1" + "version": "7.14.0-3" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.2.70202-86_22.04_amd64", - "name": "rocm-core", - "version": "7.2.2.70202-86~22.04" + "key": "amdrocm-solver7.14-gfx950_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx950", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx950_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx950", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas-host7.14_7.14.0-3_amd64", + "name": "amdrocm-blas-host7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-profiler-base7.14_7.14.0-3_amd64", + "name": "amdrocm-profiler-base7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sysdeps7.14_7.14.0-3_amd64", + "name": "amdrocm-sysdeps7.14", + "version": "7.14.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -5590,24 +6318,89 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" + "key": "amdrocm-runtime7.14_7.14.0-3_amd64", + "name": "amdrocm-runtime7.14", + "version": "7.14.0-3" }, { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" + "key": "amdrocm-llvm7.14_7.14.0-3_amd64", + "name": "amdrocm-llvm7.14", + "version": "7.14.0-3" }, { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" }, { "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", @@ -5709,141 +6502,11 @@ "name": "libcom-err2", "version": "1.46.5-2ubuntu1.2" }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, { "key": "libnsl2_1.3.0-2build2_amd64", "name": "libnsl2", "version": "1.3.0-2build2" }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "rocblas_5.2.0.70202-86_22.04_amd64", - "name": "rocblas", - "version": "5.2.0.70202-86~22.04" - }, - { - "key": "hip-runtime-amd_7.2.53211.70202-86_22.04_amd64", - "name": "hip-runtime-amd", - "version": "7.2.53211.70202-86~22.04" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "rocprofiler-register_0.6.0.70202-86_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.6.0.70202-86~22.04" - }, - { - "key": "comgr_3.0.0.70202-86_22.04_amd64", - "name": "comgr", - "version": "3.0.0.70202-86~22.04" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" - }, { "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", "name": "rpcsvc-proto", @@ -5865,96 +6528,516 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", + "key": "amdrocm-base7.14_7.14.0-3_amd64", + "name": "amdrocm-base7.14", + "version": "7.14.0-3" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70202-86_22.04_amd64", - "name": "rocminfo", - "version": "1.0.0.70202-86~22.04" + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" }, { - "key": "pciutils_1-3.7.0-6_amd64", - "name": "pciutils", - "version": "1:3.7.0-6" + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" }, { - "key": "libpci3_1-3.7.0-6_amd64", - "name": "libpci3", - "version": "1:3.7.0-6" + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" }, { - "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", - "name": "pci.ids", - "version": "0.0~2022.01.22-1ubuntu0.1" + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" }, { - "key": "libudev1_249.11-0ubuntu3.16_amd64", - "name": "libudev1", - "version": "249.11-0ubuntu3.16" + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" }, { - "key": "libkmod2_29-1ubuntu1_amd64", - "name": "libkmod2", - "version": "29-1ubuntu1" + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" }, { - "key": "hsa-rocr_1.18.0.70202-86_22.04_amd64", - "name": "hsa-rocr", - "version": "1.18.0.70202-86~22.04" + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" }, { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" }, { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.125.70201-2303469.22.04" + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" }, { - "key": "libdrm-amdgpu-common_1.0.0.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-common", - "version": "1.0.0.70201-2303469.22.04" + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" }, { - "key": "amdgpu-core_1-7.2.70201-2303469.22.04_amd64", - "name": "amdgpu-core", - "version": "1:7.2.70201-2303469.22.04" + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" }, { - "key": "libdrm2-amdgpu_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm2-amdgpu", - "version": "1:2.4.125.70201-2303469.22.04" + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" }, { - "key": "hipblaslt_1.2.2.70202-86_22.04_amd64", - "name": "hipblaslt", - "version": "1.2.2.70202-86~22.04" + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" }, { - "key": "roctracer_4.1.70202.70202-86_22.04_amd64", - "name": "roctracer", - "version": "4.1.70202.70202-86~22.04" + "key": "amdrocm-sparse7.14-gfx950_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx950", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse-host7.14_7.14.0-3_amd64", + "name": "amdrocm-sparse-host7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver-host7.14_7.14.0-3_amd64", + "name": "amdrocm-solver-host7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-math-common7.14_7.14.0-3_amd64", + "name": "amdrocm-math-common7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx942_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx942", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx942_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx942", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx942_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx942", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx90a_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx90a", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx90a_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx90a", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx90a_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx90a", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx908_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx908", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx908_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx908", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx908_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx908", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1250_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1250", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1250_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1250", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1250_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1250", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1201_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1201", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1201_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1201", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1201_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1201", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1200_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1200", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1200_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1200", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1200_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1200", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1153_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1153", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1153_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1153", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1153_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1153", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1152_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1152", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1152_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1152", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1152_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1152", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1151_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1151", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1151_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1151", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1151_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1151", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1150_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1150", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1150_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1150", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1150_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1150", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1103_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1103", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1103_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1103", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1103_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1103", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1102_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1102", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1102_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1102", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1102_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1102", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1101_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1101", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1101_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1101", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1101_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1101", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1100_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1100", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1100_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1100", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1100_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1100", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1036_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1036", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1036_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1036", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1036_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1036", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1035_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1035", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1035_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1035", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1035_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1035", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1034_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1034", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1034_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1034", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1034_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1034", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1033_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1033", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1033_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1033", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1033_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1033", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1032_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1032", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1032_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1032", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1032_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1032", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1031_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1031", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1031_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1031", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1031_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1031", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1030_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1030", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1030_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1030", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1030_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1030", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1012_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1012", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1012_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1012", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1012_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1012", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1011_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1011", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1011_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1011", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1011_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1011", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1010_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1010", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1010_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1010", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1010_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1010", + "version": "7.14.0-3" } ], - "key": "rocsolver_3.32.0.70202-86_22.04_amd64", - "name": "rocsolver", - "sha256": "71d201126423ea88e9d3274fdde5754334ed81189db0f653069ac47aa66236ec", + "key": "amdrocm-solver7.14_7.14.0-3_amd64", + "name": "amdrocm-solver7.14", + "sha256": "a871ff1d8745d277275b6f989892a854cd1567fcac3c750f27b017404750162a", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocsolver/rocsolver_3.32.0.70202-86~22.04_amd64.deb" + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14_7.14.0-3_amd64.deb" ], - "version": "3.32.0.70202-86~22.04" + "version": "7.14.0-3" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.2.70202-86_22.04_amd64", - "name": "rocm-core", - "version": "7.2.2.70202-86~22.04" + "key": "amdrocm-sparse7.14-gfx950_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx950", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx950_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx950", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx950_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx950", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver-host7.14_7.14.0-3_amd64", + "name": "amdrocm-solver-host7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-math-common7.14_7.14.0-3_amd64", + "name": "amdrocm-math-common7.14", + "version": "7.14.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -5977,24 +7060,104 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" + "key": "amdrocm-blas-host7.14_7.14.0-3_amd64", + "name": "amdrocm-blas-host7.14", + "version": "7.14.0-3" }, { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" + "key": "amdrocm-profiler-base7.14_7.14.0-3_amd64", + "name": "amdrocm-profiler-base7.14", + "version": "7.14.0-3" }, { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" + "key": "amdrocm-sysdeps7.14_7.14.0-3_amd64", + "name": "amdrocm-sysdeps7.14", + "version": "7.14.0-3" }, { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" + "key": "amdrocm-runtime7.14_7.14.0-3_amd64", + "name": "amdrocm-runtime7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-llvm7.14_7.14.0-3_amd64", + "name": "amdrocm-llvm7.14", + "version": "7.14.0-3" + }, + { + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" }, { "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", @@ -6096,146 +7259,11 @@ "name": "libcom-err2", "version": "1.46.5-2ubuntu1.2" }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, { "key": "libnsl2_1.3.0-2build2_amd64", "name": "libnsl2", "version": "1.3.0-2build2" }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "rocsolver_3.32.0.70202-86_22.04_amd64", - "name": "rocsolver", - "version": "3.32.0.70202-86~22.04" - }, - { - "key": "rocblas_5.2.0.70202-86_22.04_amd64", - "name": "rocblas", - "version": "5.2.0.70202-86~22.04" - }, - { - "key": "hip-runtime-amd_7.2.53211.70202-86_22.04_amd64", - "name": "hip-runtime-amd", - "version": "7.2.53211.70202-86~22.04" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "rocprofiler-register_0.6.0.70202-86_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.6.0.70202-86~22.04" - }, - { - "key": "comgr_3.0.0.70202-86_22.04_amd64", - "name": "comgr", - "version": "3.0.0.70202-86~22.04" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" - }, { "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", "name": "rpcsvc-proto", @@ -6257,116 +7285,9 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "rocminfo_1.0.0.70202-86_22.04_amd64", - "name": "rocminfo", - "version": "1.0.0.70202-86~22.04" - }, - { - "key": "pciutils_1-3.7.0-6_amd64", - "name": "pciutils", - "version": "1:3.7.0-6" - }, - { - "key": "libpci3_1-3.7.0-6_amd64", - "name": "libpci3", - "version": "1:3.7.0-6" - }, - { - "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", - "name": "pci.ids", - "version": "0.0~2022.01.22-1ubuntu0.1" - }, - { - "key": "libudev1_249.11-0ubuntu3.16_amd64", - "name": "libudev1", - "version": "249.11-0ubuntu3.16" - }, - { - "key": "libkmod2_29-1ubuntu1_amd64", - "name": "libkmod2", - "version": "29-1ubuntu1" - }, - { - "key": "hsa-rocr_1.18.0.70202-86_22.04_amd64", - "name": "hsa-rocr", - "version": "1.18.0.70202-86~22.04" - }, - { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" - }, - { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.125.70201-2303469.22.04" - }, - { - "key": "libdrm-amdgpu-common_1.0.0.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-common", - "version": "1.0.0.70201-2303469.22.04" - }, - { - "key": "amdgpu-core_1-7.2.70201-2303469.22.04_amd64", - "name": "amdgpu-core", - "version": "1:7.2.70201-2303469.22.04" - }, - { - "key": "libdrm2-amdgpu_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm2-amdgpu", - "version": "1:2.4.125.70201-2303469.22.04" - }, - { - "key": "hipblaslt_1.2.2.70202-86_22.04_amd64", - "name": "hipblaslt", - "version": "1.2.2.70202-86~22.04" - }, - { - "key": "roctracer_4.1.70202.70202-86_22.04_amd64", - "name": "roctracer", - "version": "4.1.70202.70202-86~22.04" - } - ], - "key": "hipsolver_3.2.0.70202-86_22.04_amd64", - "name": "hipsolver", - "sha256": "ba2e5679edda0c87c698edd5d9749d9c9fca10be5e6c8f2b9c65320aeb2fea4b", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/h/hipsolver/hipsolver_3.2.0.70202-86~22.04_amd64.deb" - ], - "version": "3.2.0.70202-86~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "rocm-core_7.2.2.70202-86_22.04_amd64", - "name": "rocm-core", - "version": "7.2.2.70202-86~22.04" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" + "key": "amdrocm-base7.14_7.14.0-3_amd64", + "name": "amdrocm-base7.14", + "version": "7.14.0-3" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -6388,106 +7309,6 @@ "name": "libuuid1", "version": "2.37.2-4ubuntu3.4" }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, { "key": "libtinfo6_6.3-2ubuntu0.1_amd64", "name": "libtinfo6", @@ -6508,11 +7329,6 @@ "name": "readline-common", "version": "8.1.2-1" }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, { "key": "libncursesw6_6.3-2ubuntu0.1_amd64", "name": "libncursesw6", @@ -6523,11 +7339,6 @@ "name": "libmpdec3", "version": "2.5.1-2build2" }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, { "key": "libffi8_3.4.2-4_amd64", "name": "libffi8", @@ -6569,679 +7380,378 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "rocfft_1.0.36.70202-86_22.04_amd64", - "name": "rocfft", - "version": "1.0.36.70202-86~22.04" + "key": "amdrocm-sparse-host7.14_7.14.0-3_amd64", + "name": "amdrocm-sparse-host7.14", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx942_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx942", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx942_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx942", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx942_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx942", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx90a_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx90a", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx90a_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx90a", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx90a_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx90a", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx908_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx908", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx908_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx908", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx908_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx908", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1250_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1250", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1250_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1250", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1250_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1250", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1201_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1201", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1201_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1201", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1201_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1201", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1200_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1200", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1200_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1200", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1200_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1200", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1153_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1153", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1153_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1153", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1153_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1153", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1152_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1152", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1152_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1152", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1152_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1152", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1151_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1151", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1151_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1151", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1151_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1151", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1150_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1150", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1150_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1150", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1150_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1150", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1103_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1103", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1103_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1103", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1103_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1103", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1102_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1102", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1102_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1102", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1102_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1102", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1101_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1101", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1101_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1101", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1101_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1101", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1100_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1100", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1100_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1100", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1100_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1100", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1036_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1036", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1036_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1036", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1036_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1036", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1035_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1035", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1035_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1035", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1035_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1035", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1034_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1034", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1034_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1034", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1034_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1034", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1033_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1033", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1033_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1033", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1033_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1033", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1032_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1032", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1032_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1032", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1032_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1032", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1031_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1031", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1031_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1031", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1031_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1031", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1030_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1030", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1030_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1030", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1030_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1030", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1012_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1012", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1012_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1012", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1012_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1012", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1011_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1011", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1011_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1011", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1011_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1011", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-sparse7.14-gfx1010_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14-gfx1010", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-blas7.14-gfx1010_7.14.0-3_amd64", + "name": "amdrocm-blas7.14-gfx1010", + "version": "7.14.0-3" + }, + { + "key": "amdrocm-solver7.14-gfx1010_7.14.0-3_amd64", + "name": "amdrocm-solver7.14-gfx1010", + "version": "7.14.0-3" } ], - "key": "hipfft_1.0.22.70202-86_22.04_amd64", - "name": "hipfft", - "sha256": "ac124bc6158c7df85b6facc4debdcc03a164f140d057d80845536c9712ad05d8", + "key": "amdrocm-sparse7.14_7.14.0-3_amd64", + "name": "amdrocm-sparse7.14", + "sha256": "674509967d9d055791bb05f1a1afb8c835680142921ce2665db9dcd873d20a46", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/h/hipfft/hipfft_1.0.22.70202-86~22.04_amd64.deb" + "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14_7.14.0-3_amd64.deb" ], - "version": "1.0.22.70202-86~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "rocfft_1.0.36.70202-86_22.04_amd64", - "name": "rocfft", - "sha256": "185e2b237d9e688f25d23309a25909823be1e1021163589c2c975c48c5dc7fe8", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocfft/rocfft_1.0.36.70202-86~22.04_amd64.deb" - ], - "version": "1.0.36.70202-86~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "hipblaslt_1.2.2.70202-86_22.04_amd64", - "name": "hipblaslt", - "version": "1.2.2.70202-86~22.04" - }, - { - "key": "rocm-core_7.2.2.70202-86_22.04_amd64", - "name": "rocm-core", - "version": "7.2.2.70202-86~22.04" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "roctracer_4.1.70202.70202-86_22.04_amd64", - "name": "roctracer", - "version": "4.1.70202.70202-86~22.04" - }, - { - "key": "hipblas-common-dev_1.4.0.70202-86_22.04_amd64", - "name": "hipblas-common-dev", - "version": "1.4.0.70202-86~22.04" - } - ], - "key": "hipblaslt-dev_1.2.2.70202-86_22.04_amd64", - "name": "hipblaslt-dev", - "sha256": "6d9e236ccbbddca308a5939196bd61547002e67de93318b1add927e5680efc2e", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/h/hipblaslt-dev/hipblaslt-dev_1.2.2.70202-86~22.04_amd64.deb" - ], - "version": "1.2.2.70202-86~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "hipblas-common-dev_1.4.0.70202-86_22.04_amd64", - "name": "hipblas-common-dev", - "sha256": "92b5ab03085a869a9e19945fc9b0fed5686824c082ecf0c22058e6441823e3e3", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/h/hipblas-common-dev/hipblas-common-dev_1.4.0.70202-86~22.04_amd64.deb" - ], - "version": "1.4.0.70202-86~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "rocm-core_7.2.2.70202-86_22.04_amd64", - "name": "rocm-core", - "version": "7.2.2.70202-86~22.04" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "rocblas_5.2.0.70202-86_22.04_amd64", - "name": "rocblas", - "version": "5.2.0.70202-86~22.04" - }, - { - "key": "hip-runtime-amd_7.2.53211.70202-86_22.04_amd64", - "name": "hip-runtime-amd", - "version": "7.2.53211.70202-86~22.04" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "rocprofiler-register_0.6.0.70202-86_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.6.0.70202-86~22.04" - }, - { - "key": "comgr_3.0.0.70202-86_22.04_amd64", - "name": "comgr", - "version": "3.0.0.70202-86~22.04" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "rocminfo_1.0.0.70202-86_22.04_amd64", - "name": "rocminfo", - "version": "1.0.0.70202-86~22.04" - }, - { - "key": "pciutils_1-3.7.0-6_amd64", - "name": "pciutils", - "version": "1:3.7.0-6" - }, - { - "key": "libpci3_1-3.7.0-6_amd64", - "name": "libpci3", - "version": "1:3.7.0-6" - }, - { - "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", - "name": "pci.ids", - "version": "0.0~2022.01.22-1ubuntu0.1" - }, - { - "key": "libudev1_249.11-0ubuntu3.16_amd64", - "name": "libudev1", - "version": "249.11-0ubuntu3.16" - }, - { - "key": "libkmod2_29-1ubuntu1_amd64", - "name": "libkmod2", - "version": "29-1ubuntu1" - }, - { - "key": "hsa-rocr_1.18.0.70202-86_22.04_amd64", - "name": "hsa-rocr", - "version": "1.18.0.70202-86~22.04" - }, - { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" - }, - { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.125.70201-2303469.22.04" - }, - { - "key": "libdrm-amdgpu-common_1.0.0.70201-2303469.22.04_amd64", - "name": "libdrm-amdgpu-common", - "version": "1.0.0.70201-2303469.22.04" - }, - { - "key": "amdgpu-core_1-7.2.70201-2303469.22.04_amd64", - "name": "amdgpu-core", - "version": "1:7.2.70201-2303469.22.04" - }, - { - "key": "libdrm2-amdgpu_1-2.4.125.70201-2303469.22.04_amd64", - "name": "libdrm2-amdgpu", - "version": "1:2.4.125.70201-2303469.22.04" - }, - { - "key": "hipblaslt_1.2.2.70202-86_22.04_amd64", - "name": "hipblaslt", - "version": "1.2.2.70202-86~22.04" - }, - { - "key": "roctracer_4.1.70202.70202-86_22.04_amd64", - "name": "roctracer", - "version": "4.1.70202.70202-86~22.04" - } - ], - "key": "rocsparse_4.2.0.70202-86_22.04_amd64", - "name": "rocsparse", - "sha256": "02ef09cbbf0b4dc801ae097617c3743a725f10dfbef5bd17b6a40589c533f3fc", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2.2/pool/main/r/rocsparse/rocsparse_4.2.0.70202-86~22.04_amd64.deb" - ], - "version": "4.2.0.70202-86~22.04" + "version": "7.14.0-3" } ], "version": 1 diff --git a/platforms/rocm/packages.yaml b/platforms/rocm/packages.yaml index 686963e3..4c8f22a7 100644 --- a/platforms/rocm/packages.yaml +++ b/platforms/rocm/packages.yaml @@ -4,10 +4,8 @@ version: 1 sources: - - channel: jammy main - url: https://repo.radeon.com/rocm/apt/7.2.2 - - channel: jammy main - url: https://repo.radeon.com/graphics/7.2.1/ubuntu + - channel: stable main + url: https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204 - channel: jammy main url: https://snapshot.ubuntu.com/ubuntu/20250711T030400Z - channel: jammy-security main @@ -18,31 +16,18 @@ sources: archs: - "amd64" -# readelf -d libpjrt_rocm.so | grep NEEDED -# find $(bazel info bazel-bin)/external/+rocm_packages+libpjrt_rocm/sandbox/ -type f -name '*.so*' -exec readelf -d {} + 2>/dev/null | grep NEEDED | sed -E 's/.*Shared library: \[(.*)\]/\1/' | sort -u packages: - - "libbz2-1.0" - - "libdw1" - - "liblzma5" - # - "rocm-smi-lib" - - "amd-smi-lib" - - "libdrm-amdgpu-common" - # Do not use hsa-rocr from amdgpu repo because it is overriden by hsa-runtime-rocr4wsl-amdgpu - - "hsa-rocr (<< 20)" - - "hsa-amd-aqlprofile" - - "comgr" - - "rocprofiler-register" - - "rocprofiler-sdk" - - "miopen-hip" - - "rccl" - - "rocm-device-libs" - - "hip-dev" - - "rocblas" - - "rocsolver" - - "hipsolver" - - "hipfft" - - "roctracer" - - "hipblaslt" - - "hipblaslt-dev" - - "hip-runtime-amd" - - "rocsparse" + - "libatomic1" + - "libdrm-common" + - "amdrocm-amdsmi7.14" + - "amdrocm-base7.14" + - "amdrocm-blas7.14" + - "amdrocm-blas-dev7.14" + - "amdrocm-dnn7.14" + - "amdrocm-fft7.14" + - "amdrocm-llvm7.14" + - "amdrocm-profiler-base7.14" + - "amdrocm-rccl7.14" + - "amdrocm-runtime-dev7.14" + - "amdrocm-solver7.14" + - "amdrocm-sparse7.14" diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 806603c6..04abe15c 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -9,7 +9,21 @@ package(default_visibility = ["//visibility:public"]) load("@rules_cc//cc:cc_library.bzl", "cc_library") """ -_ROCM_STRIP_PREFIX = "./opt/rocm-7.2.2" +_ROCM_VERSION = "7.14" +_ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION +_PJRT_ROCM_URL = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-07-20T15-30-00Z/pjrt-rocm_linux-amd64.tar.gz" +_PJRT_ROCM_SHA256 = "6fd0515beb299550e298996f6919db09aec79859feee7362443bf2ebff900d0f" + +def _rocm_package_name(name): + return name + _ROCM_VERSION + +def _rocm_repo_name(package_name): + return package_name.replace(_ROCM_VERSION, "") + +_UBUNTU_PACKAGES = { + "libatomic1": packages.filegroup(name = "libatomic1", srcs = ["usr/lib/x86_64-linux-gnu/libatomic.so.1"]), + "libdrm-common": packages.filegroup(name = "amdgpu_ids", srcs = ["usr/share/libdrm/amdgpu.ids"]), +} def _rocm_dlopen_patchelf(name, src): return "\n".join([ @@ -18,228 +32,203 @@ def _rocm_dlopen_patchelf(name, src): name = name, src = src, add_needed = ["libzmlxrocm.so.0"], - set_rpath = "$ORIGIN", + set_rpath = "$ORIGIN:$ORIGIN/rocm_sysdeps/lib", rename_dynamic_symbols = { "dlopen": "zmlxrocm_dlopen", + "pthread_attr_setstacksize": "zmlxrocm_pthread_attr_setstacksize", }, ), ]) -_UBUNTU_PACKAGES = { - "libbz2-1.0": packages.filegroup(name = "libbz2-1.0", srcs = ["lib/x86_64-linux-gnu/libbz2.so.1.0"]), - "libdrm2-amdgpu": packages.filegroup(name = "libdrm2-amdgpu", srcs = ["opt/amdgpu/lib/x86_64-linux-gnu/libdrm.so.2"]), - "libelf1": "\n".join([ - packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), - packages.patchelf( - name = "libelf1", - src = "usr/lib/x86_64-linux-gnu/libelf.so.1", - set_rpath = "$ORIGIN", - ), - ]), - "libdrm-amdgpu-common": packages.filegroup(name = "amdgpu_ids", srcs = ["opt/amdgpu/share/libdrm/amdgpu.ids"]), - "libnuma1": packages.filegroup(name = "libnuma1", srcs = ["usr/lib/x86_64-linux-gnu/libnuma.so.1"]), - "liblzma5": packages.filegroup(name = "liblzma5", srcs = ["lib/x86_64-linux-gnu/liblzma.so.5"]), - "libzstd1": packages.filegroup(name = "libzstd1", srcs = ["usr/lib/x86_64-linux-gnu/libzstd.so.1"]), - "libdrm-amdgpu-amdgpu1": "\n".join([ - packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), - packages.patchelf( - name = "libdrm-amdgpu-amdgpu1", - src = "opt/amdgpu/lib/x86_64-linux-gnu/libdrm_amdgpu.so.1", - add_needed = ["libzmlxrocm.so.0"], - set_rpath = "$ORIGIN", - rename_dynamic_symbols = { - "fopen64": "zmlxrocm_fopen64", - }, - ), - packages.filegroup(name = "libdrm_amdgpu", srcs = ["opt/amdgpu/lib/x86_64-linux-gnu/libdrm_amdgpu.so.1"]), - ]), - "libtinfo6": packages.filegroup(name = "libtinfo6", srcs = ["lib/x86_64-linux-gnu/libtinfo.so.6"]), - "zlib1g": packages.filegroup(name = "zlib1g", srcs = ["lib/x86_64-linux-gnu/libz.so.1"]), - "libdw1": packages.filegroup(name = "libdw1", srcs = ["usr/lib/x86_64-linux-gnu/libdw.so.1"]), -} +def _glob_filegroup(name, srcs): + return """filegroup( + name = "{name}", + srcs = glob({srcs}), +)""".format(name = name, srcs = repr(srcs)) -_ROCM_PACKAGES = { - "rocm-core": _rocm_dlopen_patchelf( - name = "rocm-core", - src = "lib/librocm-core.so.1", - ), - "rocm-smi-lib": _rocm_dlopen_patchelf( - name = "rocm_smi", - src = "lib/librocm_smi64.so.1", - ), - "amd-smi-lib": "\n".join([ - packages.cc_library(name = "amdsmi", hdrs = ["include/amd_smi/amdsmi.h"], includes = ["include/amd_smi"]), - packages.filegroup(name = "libamd_smi", srcs = ["lib/libamd_smi.so.26"]), - ]), - "rocprofiler-sdk": "\n".join([ - packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), - packages.patchelf( - name = "rocprofiler-sdk_so", - src = "lib/librocprofiler-sdk.so.1", - add_needed = ["libzmlxrocm.so.0"], - set_rpath = "$ORIGIN", - rename_dynamic_symbols = { - "dlopen": "zmlxrocm_dlopen", - }, - ), - packages.filegroup(name = "rocprofiler-sdk-attach", srcs = ["lib/librocprofiler-sdk-attach.so.1"]), - packages.filegroup(name = "rocprofiler-sdk", srcs = [ - ":rocprofiler-sdk_so", - ":rocprofiler-sdk-attach", +def _family_runfile_package_names(loaded_packages, family): + prefix = "amdrocm-{}{}-gfx".format(family, _ROCM_VERSION) + return sorted([ + pkg_name + for pkg_name in loaded_packages.keys() + if pkg_name.startswith(prefix) + ]) + +def _family_runfile_labels(loaded_packages, family): + return [ + "@{}//:runfiles".format(_rocm_repo_name(pkg_name)) + for pkg_name in _family_runfile_package_names(loaded_packages, family) + ] + +def _rocm_runfile_build_files(loaded_packages): + rocm_packages = {} + + for pkg_name in _family_runfile_package_names(loaded_packages, "blas"): + rocm_packages[pkg_name] = _glob_filegroup("runfiles", [ + ".kpack/**", + "lib/hipblaslt/library/**", + "lib/rocblas/library/**", + ]) + + for pkg_name in _family_runfile_package_names(loaded_packages, "dnn"): + rocm_packages[pkg_name] = _glob_filegroup("runfiles", [ + "lib/libMIOpenCKGroupedConv_*.so", + "share/miopen/db/**", + ]) + + for pkg_name in _family_runfile_package_names(loaded_packages, "fft"): + rocm_packages[pkg_name] = _glob_filegroup("runfiles", [".kpack/**"]) + + for pkg_name in _family_runfile_package_names(loaded_packages, "rccl"): + rocm_packages[pkg_name] = _glob_filegroup("runfiles", [".kpack/**"]) + + return rocm_packages + +def _rocm_base_build_files(loaded_packages): + return { + _rocm_package_name("amdrocm-amdsmi"): "\n\n".join([ + packages.cc_library(name = "amdsmi", hdrs = ["include/amd_smi/amdsmi.h"], includes = ["include/amd_smi"]), + packages.filegroup(name = "libamd_smi", srcs = ["lib/libamd_smi.so.26"]), ]), - """filegroup( - name = "rocprofv3_tool", - srcs = glob([ - "bin/**", - "lib/**", + _rocm_package_name("amdrocm-base"): "\n\n".join([ + _rocm_dlopen_patchelf(name = "rocm-core", src = "lib/librocm-core.so.1"), + _rocm_dlopen_patchelf(name = "rocm_smi", src = "lib/librocm_smi64.so.1"), + _rocm_dlopen_patchelf(name = "rocprofiler-register", src = "lib/librocprofiler-register.so.0"), + ]), + _rocm_package_name("amdrocm-blas-dev"): _glob_filegroup("headers", ["include/**"]), + _rocm_package_name("amdrocm-blas-host"): "\n\n".join([ + _rocm_dlopen_patchelf(name = "hipblas", src = "lib/libhipblas.so.3"), + packages.filegroup(name = "hipblaslt", srcs = ["lib/libhipblaslt.so.1"]), + _rocm_dlopen_patchelf(name = "hipsparselt", src = "lib/libhipsparselt.so.0"), + packages.filegroup(name = "rocblas", srcs = ["lib/librocblas.so.5"]), + _rocm_dlopen_patchelf(name = "rocroller", src = "lib/librocroller.so.1"), + _glob_filegroup("hipblaslt_support", [ + "lib/hipdnn_plugins/engines/libhipblaslt_plugin.so", + "share/hipblaslt/**", + ]), + ]), + _rocm_package_name("amdrocm-blas"): "\n\n".join([ + packages.filegroup(name = "hipblaslt", srcs = [ + "@amdrocm-blas-host//:hipblaslt", + "@amdrocm-blas-host//:rocroller", + ]), + packages.filegroup(name = "hipblaslt_runfiles", srcs = _family_runfile_labels(loaded_packages, "blas") + [ + "@amdrocm-blas-host//:hipblaslt_support", + ]), + packages.filegroup(name = "rocblas", srcs = [ + "@amdrocm-blas-host//:hipblas", + "@amdrocm-blas-host//:hipsparselt", + "@amdrocm-blas-host//:rocblas", + ]), + packages.filegroup(name = "rocblas_runfiles", srcs = _family_runfile_labels(loaded_packages, "blas")), + ]), + _rocm_package_name("amdrocm-dnn-host"): "\n\n".join([ + _rocm_dlopen_patchelf(name = "MIOpen", src = "lib/libMIOpen.so.1"), + _rocm_dlopen_patchelf(name = "hipdnn_backend", src = "lib/libhipdnn_backend.so"), + _glob_filegroup("runfiles", [ + "lib/hipdnn_plugins/**", + "share/miopen/**", + ]), + ]), + _rocm_package_name("amdrocm-dnn"): packages.filegroup(name = "MIOpen", srcs = [ + "@amdrocm-dnn-host//:MIOpen", + "@amdrocm-dnn-host//:hipdnn_backend", + "@amdrocm-dnn-host//:runfiles", + ] + _family_runfile_labels(loaded_packages, "dnn")), + _rocm_package_name("amdrocm-fft-host"): "\n\n".join([ + _rocm_dlopen_patchelf(name = "hipfft", src = "lib/libhipfft.so.0"), + _rocm_dlopen_patchelf(name = "hipfftw", src = "lib/libhipfftw.so.0"), + packages.filegroup(name = "rocfft", srcs = ["lib/librocfft.so.0"]), + ]), + _rocm_package_name("amdrocm-fft"): "\n\n".join([ + packages.filegroup(name = "hipfft", srcs = [ + "@amdrocm-fft-host//:hipfft", + "@amdrocm-fft-host//:hipfftw", + ]), + packages.filegroup(name = "rocfft", srcs = [ + "@amdrocm-fft-host//:rocfft", + ] + _family_runfile_labels(loaded_packages, "fft")), + ]), + _rocm_package_name("amdrocm-llvm"): "\n\n".join([ + packages.filegroup(name = "llvm_libs", srcs = [ + "lib/llvm/lib/libLLVM.so.23.0git", + "lib/llvm/lib/libclang-cpp.so.23.0git", + ]), + _glob_filegroup("rocm_device_libs_runfiles", ["lib/llvm/amdgcn/**"]), + ]), + _rocm_package_name("amdrocm-math-common"): _glob_filegroup("host_math_libs", ["lib/host-math/lib/*.so*"]), + _rocm_package_name("amdrocm-profiler-base"): "\n\n".join([ + _rocm_dlopen_patchelf(name = "hsa-amd-aqlprofile", src = "lib/libhsa-amd-aqlprofile64.so.1"), + _rocm_dlopen_patchelf(name = "rocprofiler-sdk_so", src = "lib/librocprofiler-sdk.so.1"), + packages.filegroup(name = "rocprofiler-sdk-attach", srcs = ["lib/librocprofiler-sdk-attach.so.1"]), + packages.filegroup(name = "rocprofiler-sdk-rocattach", srcs = ["lib/librocprofiler-sdk-rocattach.so.1"]), + _rocm_dlopen_patchelf(name = "roctracer", src = "lib/libroctracer64.so.4"), + _rocm_dlopen_patchelf(name = "roctx", src = "lib/libroctx64.so.4"), + packages.cc_library_hdrs_glob( + name = "roctx_headers", + hdrs_glob = ["include/rocprofiler-sdk-roctx/**/*.h"], + includes = ["include"], + ), + packages.filegroup(name = "rocprofiler-sdk", srcs = [ + ":rocprofiler-sdk_so", + ":rocprofiler-sdk-attach", + ":rocprofiler-sdk-rocattach", + ]), + _glob_filegroup("rocprofv3_tool", [ + "bin/rocprof*", + "bin/rocpd*", "libexec/rocprofiler-sdk/**", "share/rocprofiler-sdk/**", ]), - )""", - ]), - "rocprofiler-sdk-rocpd": packages.filegroup(name = "rocprofiler-sdk-rocpd", srcs = ["lib/librocprofiler-sdk-rocpd.so.1"]), - "rocprofiler-sdk-roctx": "\n".join([ - packages.cc_library_hdrs_glob( - name = "headers", - hdrs_glob = ["include/rocprofiler-sdk-roctx/**/*.h"], - includes = ["include"], - ), - packages.filegroup( - name = "rocprofiler-sdk-roctx", - srcs = ["lib/librocprofiler-sdk-roctx.so.1"], - ), - ]), - "hsa-rocr": _rocm_dlopen_patchelf( - name = "hsa-runtime", - src = "lib/libhsa-runtime64.so.1", - ), - "hsa-amd-aqlprofile": _rocm_dlopen_patchelf( - name = "hsa-amd-aqlprofile", - src = "lib/libhsa-amd-aqlprofile64.so.1", - ), - "comgr": _rocm_dlopen_patchelf( - name = "amd_comgr", - src = "lib/libamd_comgr.so.3", - ), - "rocprofiler-register": _rocm_dlopen_patchelf( - name = "rocprofiler-register", - src = "lib/librocprofiler-register.so.0", - ), - "miopen-hip": "\n".join([ - packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), - packages.patchelf( - name = "MIOpen", - src = "lib/libMIOpen.so.1", - add_needed = ["libzmlxrocm.so.0"], - set_rpath = "$ORIGIN", - rename_dynamic_symbols = { - "dlopen": "zmlxrocm_dlopen", - }, - ), - """filegroup(name = "runfiles", srcs = glob(["share/miopen/**"]))""", - ]), - "rccl": _rocm_dlopen_patchelf( - name = "rccl", - src = "lib/librccl.so.1", - ), - "rocm-device-libs": """filegroup(name = "runfiles", srcs = glob(["amdgcn/**"]))""", - "hip-dev": """filegroup(name = "runfiles", srcs = glob(["share/**"]))""", - "rocblas": "\n".join([ - packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), - packages.patchelf( - name = "rocblas", - src = "lib/librocblas.so.5", - add_needed = ["libzmlxrocm.so.0"], - rename_dynamic_symbols = { - "dlopen": "zmlxrocm_dlopen", - }, - ), - """filegroup( - name = "runfiles", - srcs = glob(["lib/rocblas/library/**"]), - ) - """, - ]), - "rocfft": packages.filegroup(name = "rocfft", srcs = ["lib/librocfft.so.0"]), - "rocsolver": _rocm_dlopen_patchelf( - name = "rocsolver", - src = "lib/librocsolver.so.0", - ), - "roctracer": "\n".join([ - packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), - packages.patchelf( - name = "roctracer", - src = "lib/libroctracer64.so.4", - add_needed = ["libzmlxrocm.so.0"], - set_rpath = "$ORIGIN", - rename_dynamic_symbols = { - "dlopen": "zmlxrocm_dlopen", - }, - ), - packages.patchelf( - name = "roctx", - src = "lib/libroctx64.so.4", - add_needed = ["libzmlxrocm.so.0"], - set_rpath = "$ORIGIN", - rename_dynamic_symbols = { - "dlopen": "zmlxrocm_dlopen", - }, - ), - ]), - "hipblaslt": "\n".join([ - packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), - packages.patchelf( - name = "hipblaslt", - src = "lib/libhipblaslt.so.1", - add_needed = ["libzmlxrocm.so.0"], - rename_dynamic_symbols = { - "dlopen": "zmlxrocm_dlopen", - }, - ), - packages.patchelf( - name = "rocroller", - src = "lib/librocroller.so.1", - set_rpath = "$ORIGIN", - add_needed = ["libzmlxrocm.so.0"], - rename_dynamic_symbols = { - "dlopen": "zmlxrocm_dlopen", - }, - ), - """filegroup( - name = "runfiles", - srcs = glob(["lib/hipblaslt/library/**"]), - ) - """, - ]), - "hipfft": packages.filegroup(name = "hipfft", srcs = ["lib/libhipfft.so.0"]), - "hip-runtime-amd": "\n".join([ - packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), - packages.patchelf( - name = "amdhip", - src = "lib/libamdhip64.so.7", - add_needed = ["libzmlxrocm.so.0"], - set_rpath = "$ORIGIN", - rename_dynamic_symbols = { - "dlopen": "zmlxrocm_dlopen", - }, - ), - packages.patchelf( - name = "hiprtc_so", - src = "lib/libhiprtc.so.7", - add_needed = ["libzmlxrocm.so.0"], - set_rpath = "$ORIGIN", - rename_dynamic_symbols = { - "dlopen": "zmlxrocm_dlopen", - }, - ), - packages.filegroup(name = "hiprtc", srcs = [":hiprtc_so", "lib/libhiprtc-builtins.so.7"]), - ]), - "hipsolver": _rocm_dlopen_patchelf( - name = "hipsolver", - src = "lib/libhipsolver.so.1", - ), - "rocsparse": packages.filegroup(name = "rocsparse", srcs = ["lib/librocsparse.so.1"]), -} + ]), + _rocm_package_name("amdrocm-rccl-host"): packages.filegroup(name = "rccl", srcs = ["lib/librccl.so.1"]), + _rocm_package_name("amdrocm-rccl"): packages.filegroup(name = "rccl", srcs = [ + "@amdrocm-rccl-host//:rccl", + ] + _family_runfile_labels(loaded_packages, "rccl")), + _rocm_package_name("amdrocm-runtime"): "\n\n".join([ + _rocm_dlopen_patchelf(name = "amd_comgr", src = "lib/libamd_comgr.so.3"), + _rocm_dlopen_patchelf(name = "amdhip_so", src = "lib/libamdhip64.so.7"), + _rocm_dlopen_patchelf(name = "hiprtc_so", src = "lib/libhiprtc.so.7"), + _rocm_dlopen_patchelf(name = "hsa-runtime", src = "lib/libhsa-runtime64.so.1"), + packages.filegroup(name = "amdhip", srcs = [ + ":amdhip_so", + "lib/librocm_kpack.so.0", + ]), + packages.filegroup(name = "hiprtc", srcs = [ + ":hiprtc_so", + "lib/libhiprtc-builtins.so.7", + ]), + ]), + _rocm_package_name("amdrocm-runtime-dev"): _glob_filegroup("headers", ["include/**"]), + _rocm_package_name("amdrocm-solver-host"): "\n\n".join([ + _rocm_dlopen_patchelf(name = "hipsolver", src = "lib/libhipsolver.so.1"), + packages.filegroup(name = "rocsolver", srcs = ["lib/librocsolver.so.0"]), + ]), + _rocm_package_name("amdrocm-solver"): "\n\n".join([ + packages.filegroup(name = "hipsolver", srcs = ["@amdrocm-solver-host//:hipsolver"]), + packages.filegroup(name = "rocsolver", srcs = [ + "@amdrocm-solver-host//:rocsolver", + "@amdrocm-math-common//:host_math_libs", + ]), + ]), + _rocm_package_name("amdrocm-sparse-host"): "\n\n".join([ + _rocm_dlopen_patchelf(name = "hipsparse", src = "lib/libhipsparse.so.4"), + packages.filegroup(name = "rocsparse", srcs = ["lib/librocsparse.so.1"]), + ]), + _rocm_package_name("amdrocm-sparse"): packages.filegroup(name = "rocsparse", srcs = [ + "@amdrocm-sparse-host//:hipsparse", + "@amdrocm-sparse-host//:rocsparse", + ]), + _rocm_package_name("amdrocm-sysdeps"): _glob_filegroup("system_libs", [ + "lib/rocm_sysdeps/lib/*.so*", + "lib/rocm_sysdeps/share/**", + ]), + } + +def _rocm_build_files(loaded_packages): + rocm_packages = _rocm_base_build_files(loaded_packages) + rocm_packages.update(_rocm_runfile_build_files(loaded_packages)) + return rocm_packages def _rocm_impl(mctx): loaded_packages = packages.read(mctx, [ @@ -255,10 +244,10 @@ def _rocm_impl(mctx): build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, ) - for pkg_name, build_file_content in _ROCM_PACKAGES.items(): + for pkg_name, build_file_content in _rocm_build_files(loaded_packages).items(): pkg = loaded_packages[pkg_name]["amd64"] http_deb_archive( - name = pkg_name, + name = _rocm_repo_name(pkg_name), urls = pkg["urls"], sha256 = pkg["sha256"], strip_prefix = _ROCM_STRIP_PREFIX, @@ -275,13 +264,30 @@ def _rocm_impl(mctx): http_archive( name = "libpjrt_rocm", build_file = "libpjrt_rocm.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-04-30T17-13-00Z/pjrt-rocm_linux-amd64.tar.gz", - sha256 = "e543e48f902656598234b5ffc05196dde84a87106be21ada126c7323572f792b", + url = _PJRT_ROCM_URL, + sha256 = _PJRT_ROCM_SHA256, ) return mctx.extension_metadata( reproducible = True, - root_module_direct_deps = ["amd-smi-lib", "libpjrt_rocm", "libdrm2-amdgpu", "libdrm-amdgpu-amdgpu1", "hipblaslt", "rocblas", "rocprofiler-sdk", "rocprofiler-sdk-roctx"], + root_module_direct_deps = [ + "amdrocm-amdsmi", + "amdrocm-base", + "amdrocm-blas", + "amdrocm-dnn", + "amdrocm-fft", + "amdrocm-llvm", + "amdrocm-math-common", + "amdrocm-profiler-base", + "amdrocm-rccl", + "amdrocm-runtime", + "amdrocm-solver", + "amdrocm-sparse", + "amdrocm-sysdeps", + "libatomic1", + "libdrm-common", + "libpjrt_rocm", + ], root_module_direct_dev_deps = [], ) diff --git a/platforms/rocm/zmlxrocm.zig b/platforms/rocm/zmlxrocm.zig index e3024de6..069c08bb 100644 --- a/platforms/rocm/zmlxrocm.zig +++ b/platforms/rocm/zmlxrocm.zig @@ -4,22 +4,11 @@ const stdx = @import("stdx"); pub export fn zmlxrocm_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque { const replacements: std.StaticStringMap([:0]const u8) = .initComptime(.{ - .{ "librocm-core.so", "librocm-core.so.1" }, - .{ "librocm_smi64.so", "librocm_smi64.so.1" }, - .{ "libhsa-runtime64.so", "libhsa-runtime64.so.1" }, - .{ "libhsa-amd-aqlprofile64.so", "libhsa-amd-aqlprofile64.so.1" }, - .{ "libamd_comgr.so", "libamd_comgr.so.3" }, - .{ "librocprofiler-register.so", "librocprofiler-register.so.0" }, - .{ "libMIOpen.so", "libMIOpen.so.1" }, - .{ "libMIOpen.so.1", "libMIOpen.so.1" }, - .{ "librccl.so", "librccl.so.1" }, - .{ "librocblas.so.5", "librocblas.so.5" }, - .{ "librocblas.so", "librocblas.so.5" }, - .{ "libroctracer64.so", "libroctracer64.so.4" }, - .{ "libroctx64.so", "libroctx64.so.4" }, - .{ "libhipblaslt.so", "libhipblaslt.so.1" }, .{ "libamdhip64.so", "libamdhip64.so.7" }, - .{ "libhiprtc.so", "libhiprtc.so.7" }, + .{ "libhsa-amd-aqlprofile64.so", "libhsa-amd-aqlprofile64.so.1" }, + .{ "librocprofiler-sdk-attach.so", "librocprofiler-sdk-attach.so.1" }, + .{ "librocprofiler-sdk.so", "librocprofiler-sdk.so.1" }, + .{ "libroctx64.so", "libroctx64.so.4" }, }); var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; @@ -50,3 +39,29 @@ pub export fn zmlxrocm_fopen64(pathname: [*c]const u8, mode: [*c]const u8) ?*std return std.c.fopen64(new_pathname, mode); } + +extern fn pthread_attr_setstacksize(attr: ?*anyopaque, stacksize: usize) c_int; + +// ROCclr asks for small helper-thread stacks: CQ_THREAD_STACK_SIZE is 256 KiB, +// then ROCclr adds the pthread guard size before calling pthread_attr_setstacksize. +// https://github.com/ROCm/clr/blob/1736f3eba9134a5ee4b56475acf15665f065ab05/rocclr/utils/flags.hpp#L25 +// https://github.com/ROCm/clr/blob/1736f3eba9134a5ee4b56475acf15665f065ab05/rocclr/os/os_posix.cpp#L378-L385 +// glibc 2.28 accepts stack sizes >= PTHREAD_STACK_MIN at attribute-set time, +// but pthread_create can still reject stacks that do not leave room for the +// guard, static TLS, and MINIMAL_REST_STACK: +// https://github.com/bminor/glibc/blob/glibc-2.28/nptl/pthread_attr_setstacksize.c#L35-L38 +// https://github.com/bminor/glibc/blob/glibc-2.28/nptl/allocatestack.c#L532-L544 +// The 1 MiB floor below is a local compatibility margin for that glibc path; +// it is intentionally larger than ROCclr's request and not a ROCm ABI value. +pub export fn zmlxrocm_pthread_attr_setstacksize(attr: ?*anyopaque, stacksize: usize) c_int { + const min_stack_size = 1024 * 1024; + const adjusted_stacksize = blk: { + if (stacksize < min_stack_size) { + break :blk min_stack_size; + } else { + break :blk stacksize; + } + }; + + return pthread_attr_setstacksize(attr, adjusted_stacksize); +} diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 8e9b867c..066f76bd 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -17,8 +17,8 @@ sh_binary( sh_binary( name = "rocprofv3", data = [ + "@amdrocm-profiler-base//:rocprofv3_tool", "@libpjrt_rocm//:sandbox", - "@rocprofiler-sdk//:rocprofv3_tool", ], srcs = ["rocprofv3.sh"], visibility = ["//visibility:public"], diff --git a/tools/rocprofv3.sh b/tools/rocprofv3.sh index f698d893..1b9c025b 100755 --- a/tools/rocprofv3.sh +++ b/tools/rocprofv3.sh @@ -15,18 +15,18 @@ else exit 1 fi -rocprofv3_bin="${runfiles_root}/+rocm_packages+rocprofiler-sdk/bin/rocprofv3" -if [[ ! -x "${rocprofv3_bin}" ]]; then - echo "sandboxed rocprofv3 not found at ${rocprofv3_bin}" >&2 - exit 1 -fi - rocm_root="${runfiles_root}/+rocm_packages+libpjrt_rocm/sandbox" if [[ ! -d "${rocm_root}" ]]; then echo "sandboxed ROCm runtime root not found at ${rocm_root}" >&2 exit 1 fi +rocprofv3_bin="${rocm_root}/bin/rocprofv3" +if [[ ! -x "${rocprofv3_bin}" ]]; then + echo "sandboxed rocprofv3 not found at ${rocprofv3_bin}" >&2 + exit 1 +fi + declare -a rocprof_args if [[ -n "${ZML_ROCPROFV3_ARGS:-}" ]]; then rocprof_args=(${ZML_ROCPROFV3_ARGS}) diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 1d1df5c0..a843515b 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -59,7 +59,7 @@ cc_library( "//conditions:default": [], }) + select({ "//platforms:rocm.enabled": [ - "@rocprofiler-sdk-roctx//:headers", + "@amdrocm-profiler-base//:roctx_headers", ], "//conditions:default": [], }), diff --git a/zml/module.zig b/zml/module.zig index e0d58ae9..bd4614e1 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -646,11 +646,6 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_latency_hiding_scheduler", true, upb_arena); }, .rocm => { - // Use hipBLASLt only - try setXlaOverrideFlag(overrides_map, "xla_gpu_autotune_level", 0, upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_triton_gemm", false, upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_cublaslt", true, upb_arena); - // Use lld from libllvm instead of invoking the ld.lld binary. // This saves us from having to sandbox it. try setXlaOverrideFlag(overrides_map, "xla_gpu_use_inprocess_lld", true, upb_arena); From 252113095c91582dc32ab4bc09f23ce8f3b402c8 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 22 Jul 2026 15:30:46 +0200 Subject: [PATCH 593/881] zml/testing: Do not pre-allocate memory for the zml.testing.env() (#679) Pre-allocating 85% of memory for simple tests is overkill, we'll just let the BFC grow as needed. That should hopefully limit the problem of having concurrent CI runs. --- zml/testing.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/testing.zig b/zml/testing.zig index 968328af..978dbaac 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -18,7 +18,7 @@ pub fn env() *const Platform { std.heap.c_allocator, std.testing.io, .{ - .xla_gpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.85 } } }, + .xla_gpu = .{ .allocator = .{ .bfc = .{ .preallocate = false, .memory_fraction = 0.85 } } }, }, ) catch @panic("Pjrt not available"); From d1d313cf67b4751973f852f223e4c6b299308081 Mon Sep 17 00:00:00 2001 From: Olivier Boulant Date: Wed, 22 Jul 2026 15:59:11 +0200 Subject: [PATCH 594/881] mosaic_tpu: support custom buffered-ref transfer strategies (#676) ## Context Translate https://github.com/vllm-project/tpu-inference/tree/main/tpu_inference/kernels/gdn/v3 kernels. ## Summary - add `BufferedRefStrategy` callbacks for overriding `copyIn`, `waitIn`, `copyOut`, and `waitOut` on individual `PipeSpec`s - add the `input_output` buffer role, with separate receive/send semaphores and four loop-carried slot counters: copy-in, wait-in, copy-out, and wait-out (input-only refs use `copy_in_slot` and `wait_in_slot` ; output-only refs use `copy_out_slot` and `wait_out_slot`, `input_output` refs use all four) - provide custom transfer callbacks with the references, buffer slots, and semaphores needed to emit their DMA operations (`sourceRef`, `windowRef`, `copyInSlot`, `waitInSlot`, `receiveSemaphoreAt`, `copyOutSlot`, `waitOutSlot`, `sendSemaphoreAt`) - fall back to the existing transfer implementation for callbacks that are not overridden ## Validation - all 7 `fused_conv1d_gdn_batched` sweeps produce matching Python and Zig MLIR, covering tile size, state count, head counts, Q/K-to-V head ratio, distinct head dimensions, and convolution kernel size - all 19 existing `decoding_gdn-bt_8` sweeps continue to produce matching Python and Zig MLIR --- kernels/mosaic_tpu/pipeline.zig | 257 ++++++++++++++++++++++++++++---- 1 file changed, 227 insertions(+), 30 deletions(-) diff --git a/kernels/mosaic_tpu/pipeline.zig b/kernels/mosaic_tpu/pipeline.zig index f93e4ffb..9d0a051d 100644 --- a/kernels/mosaic_tpu/pipeline.zig +++ b/kernels/mosaic_tpu/pipeline.zig @@ -52,6 +52,16 @@ pub const RefWindow = struct { buf: ?Value = null, slot: ?Value = null }; /// The pipeline body — `pallas_call`'s inner kernel. pub const BodyFn = *const fn (b: *Builder, grid_indices: []const Value, refs: []const RefWindow, scratches: []const Value, ctx: ?*anyopaque) FinishError!void; +// Optional overrides for a buffered reference's DMA operations. +// Unspecified operations retain the standard pipeline behavior. +pub const BufferedRefStrategy = struct { + ctx: ?*anyopaque = null, + copy_in: ?*const fn (br: *BufferedRef, b: *Builder, indices: []const Value, slot_cumulative: Value, static_zero_start: bool, ctx: ?*anyopaque) void = null, + wait_in: ?*const fn (br: *BufferedRef, b: *Builder, indices: []const Value, ctx: ?*anyopaque) void = null, + copy_out: ?*const fn (br: *BufferedRef, b: *Builder, indices: []const Value, ctx: ?*anyopaque) void = null, + wait_out: ?*const fn (br: *BufferedRef, b: *Builder, indices: []const Value, ctx: ?*anyopaque) void = null, +}; + /// `pl.BlockSpec` for one pipeline operand. `null` slot ↔ `None` spec. pub const PipeSpec = struct { full_shape: []const i64, @@ -71,6 +81,12 @@ pub const PipeSpec = struct { bounded_slice_tiling: i32 = 1, /// Pallas's `disable_bounds_checks`: skip OOB clamp, emit only `tpu.assume_multiple`. bounded_slice_skip_clamp: bool = false, + + /// Optional custom DMA behavior for this operand. + transfer_strategy: ?BufferedRefStrategy = null, + /// Override the role infered from whether this spec appears + /// in `in_specs` or `out_specs` + buffer_type: ?BufferType = null, }; pub const PipelineOpts = struct { @@ -125,7 +141,7 @@ pub fn emitPipeline( { var when = b.openIf(b.cmpi(.sgt, num_steps, c0_i32)); { - for (brefs.items) |*br| if (br.is_trivial and br.buffer_type == .input) br.syncCopyIn(b); + for (brefs.items) |*br| if (br.is_trivial and (br.buffer_type == .input or br.buffer_type == .input_output)) br.syncCopyIn(b); const num_stages = blk: { var m: usize = 2; @@ -145,7 +161,7 @@ pub fn emitPipeline( } // iter_args: each bref's slots in spec order (see appendSlotInits), then grid indices. - var inits: std.ArrayList(*const mlir.Value) = try .initCapacity(a, brefs.items.len * 2 + opts.grid.len); + var inits: std.ArrayList(*const mlir.Value) = try .initCapacity(a, brefs.items.len * 4 + opts.grid.len); for (brefs.items) |*br| br.appendSlotInits(b, &inits); for (0..opts.grid.len) |_| inits.appendAssumeCapacity(c0_i32.inner); const n_slots = inits.items.len - opts.grid.len; @@ -173,9 +189,14 @@ pub fn emitPipeline( for (0..opts.grid.len) |i| sched.indices_storage[i] = grid_indices_buf[i]; sched.recomputeDerivedIndices(b); - for (brefs.items) |*br| if (br.buffer_type == .input) sched.copyIn(b, br); - for (brefs.items) |*br| if (br.buffer_type == .input) sched.waitIn(b, br); - + for (brefs.items) |*br| { + if (br.buffer_type == .input or br.buffer_type == .input_output) + sched.copyIn(b, br); + } + for (brefs.items) |*br| { + if (br.buffer_type == .input or br.buffer_type == .input_output) + sched.waitIn(b, br); + } // Emit slot `rem` ops BEFORE the body; body re-emits slice/squeeze per use. var current_refs = a.alloc(RefWindow, refs.len) catch unreachable; for (current_refs) |*r| r.* = .{}; @@ -183,7 +204,10 @@ pub fn emitPipeline( if (br.is_trivial or !br.isBuffered()) { current_refs[br.spec_index] = .{ .buf = br.window_ref }; } else { - const active = if (br.buffer_type == .output) br.copy_out_slot.? else br.wait_in_slot.?; + const active = switch (br.buffer_type) { + .input => br.wait_in_slot.?, + .output, .input_output => br.copy_out_slot.?, + }; current_refs[br.spec_index] = .{ .buf = br.window_ref, .slot = b.remui(active, b.lift(@as(i32, @intCast(br.buffer_count)))) }; } } @@ -192,12 +216,20 @@ pub fn emitPipeline( try opts.body(b, sched.indices_storage[0..opts.grid.len], current_refs, scratches, opts.body_ctx); if (opts.trace_scopes) b.traceStop(); - for (brefs.items) |*br| if (br.buffer_type == .output) sched.copyOut(b, br); - for (brefs.items) |*br| if (br.buffer_type == .output) sched.waitOut(b, br); + for (brefs.items) |*br| { + if (br.buffer_type == .output or br.buffer_type == .input_output) + sched.copyOut(b, br); + } + for (brefs.items) |*br| { + if (br.buffer_type == .output or br.buffer_type == .input_output) + sched.waitOut(b, br); + } - for (brefs.items) |*br| if (br.buffer_type == .input) sched.advanceSlots(b, br); - - var yields: std.ArrayList(*const mlir.Value) = try .initCapacity(a, brefs.items.len * 2 + opts.grid.len); + for (brefs.items) |*br| { + if (br.buffer_type == .input or br.buffer_type == .input_output) + sched.advanceSlots(b, br); + } + var yields: std.ArrayList(*const mlir.Value) = try .initCapacity(a, brefs.items.len * 4 + opts.grid.len); for (brefs.items) |*br| br.appendSlotYields(b, &yields); const next_indices = nextIndexND(b, sched.indices(), opts.grid); for (next_indices) |idx| yields.appendAssumeCapacity(idx.inner); @@ -219,7 +251,7 @@ pub fn emitPipeline( for (0..opts.grid.len) |i| sched_fin.indices_storage[i] = final_idx[i]; sched_fin.recomputeDerivedIndices(b); for (brefs.items) |*br| sched_fin.finalize(b, br); - for (brefs.items) |*br| if (br.is_trivial and br.buffer_type == .output) br.syncCopyOut(b); + for (brefs.items) |*br| if (br.is_trivial and (br.buffer_type == .output or br.buffer_type == .input_output)) br.syncCopyOut(b); when.yieldThen(.{}); } @@ -230,10 +262,10 @@ pub fn emitPipeline( _ = tpu.region(b.ctx, region_block, &.{}, b.loc()).appendTo(b.currentBlock()); } -const BufferType = enum { input, output }; +pub const BufferType = enum { input, output, input_output }; /// VMEM double-buffer + DMA semaphores for one pipeline operand. -const BufferedRef = struct { +pub const BufferedRef = struct { spec: PipeSpec, buffer_type: BufferType, buffer_count: usize, @@ -260,11 +292,12 @@ const BufferedRef = struct { fn create(b: *Builder, spec: PipeSpec, bt: BufferType, src: Value, spec_index: usize) BufferedRef { const a = b.arena.allocator(); + const buffer_type = spec.buffer_type orelse bt; if (spec.memory_space != .vmem) @panic("emitPipeline: only VMEM operands supported"); const trivial = specHasTrivialWindowing(spec); var bc: usize = if (spec.buffer_count) |c| c else 2; if (spec.buffer_count == null and trivial) bc = 1; - if (bt == .output and bc > 2) @panic("emitPipeline: output buffer_count > 2 unsupported"); + if (buffer_type != .input and bc > 2) @panic("emitPipeline: output and input_output buffer_count > 2 unsupported"); const block_len = if (spec.block_shape) |bs| bs.len else 0; var keep = std.ArrayList(usize).initCapacity(a, block_len) catch unreachable; @@ -284,7 +317,7 @@ const BufferedRef = struct { } return .{ .spec = spec, - .buffer_type = bt, + .buffer_type = buffer_type, .buffer_count = bc, .is_trivial = trivial, .spec_index = spec_index, @@ -315,8 +348,8 @@ const BufferedRef = struct { } if (!self.is_trivial) { const sem_arr_ty = dmaSemRefTy(b, self.buffer_count); - const sem = semAllocOp(b, sem_arr_ty); - if (self.buffer_type == .input) self.sem_recvs = sem else self.sem_sends = sem; + if (self.buffer_type == .input or self.buffer_type == .input_output) self.sem_recvs = semAllocOp(b, sem_arr_ty); + if (self.buffer_type == .output or self.buffer_type == .input_output) self.sem_sends = semAllocOp(b, sem_arr_ty); } } @@ -335,6 +368,14 @@ const BufferedRef = struct { out.appendAssumeCapacity(c0); out.appendAssumeCapacity(c0); }, + .input_output => { + out.appendAssumeCapacity( + (self.copy_in_slot orelse b.lift(@as(i32, 0))).inner, + ); + out.appendAssumeCapacity(c0); + out.appendAssumeCapacity(c0); + out.appendAssumeCapacity(c0); + }, } } @@ -342,6 +383,7 @@ const BufferedRef = struct { return switch (self.buffer_type) { .input => if (self.is_trivial) 1 else 2, .output => 2, + .input_output => 4, }; } @@ -367,6 +409,12 @@ const BufferedRef = struct { self.copy_out_slot = arg(b, body, &k); self.wait_out_slot = arg(b, body, &k); }, + .input_output => { + self.copy_in_slot = arg(b, body, &k); + self.wait_in_slot = arg(b, body, &k); + self.copy_out_slot = arg(b, body, &k); + self.wait_out_slot = arg(b, body, &k); + }, } return k; } @@ -386,6 +434,12 @@ const BufferedRef = struct { out.appendAssumeCapacity(self.copy_out_slot.?.inner); out.appendAssumeCapacity(self.wait_out_slot.?.inner); }, + .input_output => { + out.appendAssumeCapacity(self.copy_in_slot.?.inner); + out.appendAssumeCapacity(self.wait_in_slot.?.inner); + out.appendAssumeCapacity(self.copy_out_slot.?.inner); + out.appendAssumeCapacity(self.wait_out_slot.?.inner); + }, } } @@ -411,6 +465,12 @@ const BufferedRef = struct { self.copy_out_slot = take(b, for_op, &k); self.wait_out_slot = take(b, for_op, &k); }, + .input_output => { + self.copy_in_slot = take(b, for_op, &k); + self.wait_in_slot = take(b, for_op, &k); + self.copy_out_slot = take(b, for_op, &k); + self.wait_out_slot = take(b, for_op, &k); + }, } return k; } @@ -606,7 +666,7 @@ const BufferedRef = struct { // Op order mirrors pipeline.py: copy_in/copy_out/wait_out take slot rem // before get_dma_slice; wait_in does it after. - fn copyIn(self: *BufferedRef, b: *Builder, indices: []const Value, slot_cumulative: Value, static_zero_start: bool) void { + fn defaultCopyIn(self: *BufferedRef, b: *Builder, indices: []const Value, slot_cumulative: Value, static_zero_start: bool) void { const slot = b.remui(slot_cumulative, b.lift(@as(i32, @intCast(self.buffer_count)))); const dma = self.getDmaSlice(b, indices, static_zero_start); const src_sl = self.sliceSrc(b, dma); @@ -615,7 +675,7 @@ const BufferedRef = struct { b.enqueueDma(src_sl, dst_sl, sem, .{}); } - fn waitIn(self: *BufferedRef, b: *Builder, indices: []const Value) void { + fn defaultWaitIn(self: *BufferedRef, b: *Builder, indices: []const Value) void { const dma = self.getDmaSlice(b, indices, false); const slot = b.remui(self.wait_in_slot.?, b.lift(@as(i32, @intCast(self.buffer_count)))); const src_sl = self.sliceSrc(b, dma); @@ -624,7 +684,7 @@ const BufferedRef = struct { b.waitDma2(sem, src_sl, dst_sl, .{}); } - fn copyOut(self: *BufferedRef, b: *Builder, indices: []const Value) void { + fn defaultCopyOut(self: *BufferedRef, b: *Builder, indices: []const Value) void { if (self.buffer_count == 1) @panic("emitPipeline: single-buffered output sync_copy_out unsupported (TODO)"); const slot = b.remui(self.copy_out_slot.?, b.lift(@as(i32, @intCast(self.buffer_count)))); const dma = self.getDmaSlice(b, indices, false); @@ -635,7 +695,7 @@ const BufferedRef = struct { b.enqueueDma(src_sl, dst_sl, sem, .{}); } - fn waitOut(self: *BufferedRef, b: *Builder, indices: []const Value) void { + fn defaultWaitOut(self: *BufferedRef, b: *Builder, indices: []const Value) void { if (self.buffer_count <= 1) return; const slot = b.remui(self.wait_out_slot.?, b.lift(@as(i32, @intCast(self.buffer_count)))); const dma = self.getDmaSlice(b, indices, false); @@ -644,6 +704,87 @@ const BufferedRef = struct { const sem = semAt(b, self.sem_sends.?, slot); b.waitDma2(sem, src_sl, dst_sl, .{}); } + + fn copyIn(self: *BufferedRef, b: *Builder, indices: []const Value, slot_cumulative: Value, static_zero_start: bool) void { + if (self.is_trivial) return; + + if (self.spec.transfer_strategy) |strategy| { + if (strategy.copy_in) |customCopyIn| { + customCopyIn(self, b, indices, slot_cumulative, static_zero_start, strategy.ctx); + return; + } + } + + self.defaultCopyIn(b, indices, slot_cumulative, static_zero_start); + } + + fn waitIn(self: *BufferedRef, b: *Builder, indices: []const Value) void { + if (self.spec.transfer_strategy) |strategy| { + if (strategy.wait_in) |customWaitIn| { + customWaitIn(self, b, indices, strategy.ctx); + return; + } + } + self.defaultWaitIn(b, indices); + } + + fn copyOut(self: *BufferedRef, b: *Builder, indices: []const Value) void { + if (self.spec.transfer_strategy) |strategy| { + if (strategy.copy_out) |customCopyOut| { + customCopyOut(self, b, indices, strategy.ctx); + return; + } + } + self.defaultCopyOut(b, indices); + } + + fn waitOut(self: *BufferedRef, b: *Builder, indices: []const Value) void { + if (self.spec.transfer_strategy) |strategy| { + if (strategy.wait_out) |customWaitOut| { + customWaitOut(self, b, indices, strategy.ctx); + return; + } + } + self.defaultWaitOut(b, indices); + } + + pub fn sourceRef(self: *const BufferedRef) Value { + return self.src; + } + + pub fn windowRef(self: *const BufferedRef) Value { + return self.window_ref; + } + + pub fn copyInSlot(self: *const BufferedRef, b: *Builder, slot_cumulative: Value) Value { + return b.remui(slot_cumulative, b.lift(@as(i32, @intCast(self.buffer_count)))); + } + + pub fn waitInSlot(self: *const BufferedRef, b: *Builder) Value { + return b.remui(self.wait_in_slot.?, b.lift(@as(i32, @intCast(self.buffer_count)))); + } + + pub fn receiveSemaphoreAt(self: *const BufferedRef, b: *Builder, slot: Value) Value { + return semAt(b, self.sem_recvs.?, slot); + } + + pub fn copyOutSlot(self: *const BufferedRef, b: *Builder) Value { + const buffer_count: i32 = @intCast(self.buffer_count); + return b.remui(self.copy_out_slot.?, b.lift(buffer_count)); + } + + pub fn waitOutSlot(self: *const BufferedRef, b: *Builder) Value { + const buffer_count: i32 = @intCast(self.buffer_count); + return b.remui(self.wait_out_slot.?, b.lift(buffer_count)); + } + + pub fn sendSemaphoreAt( + self: *const BufferedRef, + b: *Builder, + slot: Value, + ) Value { + return semAt(b, self.sem_sends.?, slot); + } }; /// Prologue/loop/epilogue copy+wait schedule. @@ -765,7 +906,7 @@ const Scheduler = struct { } fn initializeStep(self: *Scheduler, b: *Builder, br: *BufferedRef, step: usize) void { - if (br.buffer_type != .input or !br.isBuffered() or br.is_trivial) return; + if ((br.buffer_type != .input and br.buffer_type != .input_output) or !br.isBuffered() or br.is_trivial) return; if (step + 1 >= br.buffer_count) return; if (br.copy_in_slot == null) br.copy_in_slot = b.lift(@as(i32, 0)); @@ -791,7 +932,7 @@ const Scheduler = struct { if (br.is_trivial) return; var pred = b.andi(self.willChangeFetch(b, br), self.notOutOfFetch(b, br)); if (br.isBuffered() and br.buffer_count < 2) pred = b.ori(pred, self.first_step); - if (br.buffer_type != .input) return; + if (br.buffer_type != .input and br.buffer_type != .input_output) return; { var when = b.openIf(pred); { @@ -811,7 +952,7 @@ const Scheduler = struct { var when = b.openIf(pred); { self.namedScope(b, "ep_wait_in"); - if (br.buffer_type == .input) br.waitIn(b, self.indices()); + if (br.buffer_type == .input or br.buffer_type == .input_output) br.waitIn(b, self.indices()); self.endScope(b); when.yieldThen(.{}); } @@ -825,7 +966,7 @@ const Scheduler = struct { var when = b.openIf(pred); { self.namedScope(b, "ep_copy_out"); - if (br.buffer_type == .output) br.copyOut(b, self.indices()); + if (br.buffer_type == .output or br.buffer_type == .input_output) br.copyOut(b, self.indices()); self.endScope(b); when.yieldThen(.{}); } @@ -841,7 +982,7 @@ const Scheduler = struct { { self.namedScope(b, "ep_wait_out"); // copy_out was issued the previous iteration. - if (br.buffer_type == .output) br.waitOut(b, self.prevIndices()); + if (br.buffer_type == .output or br.buffer_type == .input_output) br.waitOut(b, self.prevIndices()); self.endScope(b); when.yieldThen(.{}); } @@ -850,7 +991,7 @@ const Scheduler = struct { } fn finalize(self: *Scheduler, b: *Builder, br: *BufferedRef) void { - if (br.is_trivial or br.buffer_type != .output) return; + if (br.is_trivial or (br.buffer_type != .output and br.buffer_type != .input_output)) return; var when = b.openIf(self.last_step); { self.namedScope(b, "ep_finalize"); @@ -861,9 +1002,13 @@ const Scheduler = struct { } fn advanceSlots(self: *Scheduler, b: *Builder, br: *BufferedRef) void { - if (br.buffer_type != .input) return; + if (br.buffer_type != .input and br.buffer_type != .input_output) return; if (br.is_trivial) { - br.slot = advance(b, br.slot.?, self.last_step); + switch (br.buffer_type) { + .input => br.slot = advance(b, br.slot.?, self.last_step), + .input_output => br.wait_in_slot = advance(b, br.wait_in_slot.?, self.last_step), + .output => unreachable, + } } else { const pred = b.ori(self.willChangeCurrent(b, br), self.last_step); br.wait_in_slot = advance(b, br.wait_in_slot.?, pred); @@ -960,3 +1105,55 @@ fn specHasTrivialWindowing(spec: PipeSpec) bool { test { std.testing.refAllDecls(@This()); } + +test "trivial input-output ref is copied in and out" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + inline for (builder.dialects_needed) |dialect| registry.registerDialect(dialect); + mlir.registerFuncExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + + var b = try Builder.init(std.testing.allocator, ctx, "trivial_input_output", &.{ + .{ + .name = "state", + .kind = .{ .ref = .{ + .shape = &.{ 8, 128 }, + .dtype = .bf16, + .memory_space = .hbm, + } }, + }, + }, &.{}); + defer b.deinit(); + + const noOpBody = struct { + fn call( + _: *Builder, + _: []const Value, + _: []const RefWindow, + _: []const Value, + _: ?*anyopaque, + ) FinishError!void {} + }.call; + + try emitPipeline(&b, &.{b.arg(0)}, &.{}, .{ + .body = noOpBody, + .grid = &.{b.lift(@as(i32, 1))}, + .in_specs = &.{.{ + .full_shape = &.{ 8, 128 }, + .dtype = .bf16, + .buffer_type = .input_output, + }}, + .out_specs = &.{}, + .trace_scopes = false, + }); + + const ir = try b.finish(&.{}); + defer std.testing.allocator.free(ir); + + // One enqueue/wait pair for copy_in, one for copy_out. + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "tpu.enqueue_dma")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "tpu.wait_dma")); +} From c6415035db27f6836163ead69c5f73a9e8cc50f0 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Thu, 23 Jul 2026 10:54:22 +0200 Subject: [PATCH 595/881] tools/xprof: Fix xprof after update (#664) --- tools/xprof/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/xprof/main.py b/tools/xprof/main.py index 77da9b46..468c5603 100644 --- a/tools/xprof/main.py +++ b/tools/xprof/main.py @@ -1,6 +1,6 @@ import sys -from xprof.server import main as xprof_main +from xprof.cli.xprof_cli import main as xprof_main def _argv() -> list[str]: From a0d0af2a8ddba348209cca9580455890b95cbe8a Mon Sep 17 00:00:00 2001 From: Olivier Boulant Date: Thu, 23 Jul 2026 11:17:28 +0200 Subject: [PATCH 596/881] flashattn: harmonize zml.Tensor.zeroes() calls (#659) Following https://github.com/zml/zml/pull/653 --- zml/attention/flashattn.zig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index e2be7006..3d11ed24 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -1691,7 +1691,7 @@ pub const paged_fa3 = struct { .hkv = num_kv_heads, .hg = num_head_groups, }, .f32)).withPartitioning(.{ .hkv = .model }); - const scheduler_metadata = zml.Tensor.zeroes(.init(.{ .b = batch_size + 1 }, .i32)).withPartitioning(.{ .b = .replicated }); + const scheduler_metadata = zml.Tensor.zeroes(.init(.{ .b = batch_size + 1 }, .i32)); var q2 = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); @@ -1759,7 +1759,7 @@ pub const paged_fa3 = struct { .hkv = num_kv_heads, .hg = num_head_groups, }, .f32)).withPartitioning(.{ .hkv = .model }); - const scheduler_metadata_prefill = zml.Tensor.zeroes(.init(.{ .b = batch_size_prefill + 1 }, .i32)).withPartitioning(.{ .b = .replicated }); + const scheduler_metadata_prefill = zml.Tensor.zeroes(.init(.{ .b = batch_size_prefill + 1 }, .i32)); var q2 = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); @@ -1815,7 +1815,7 @@ pub const paged_fa3 = struct { .hkv = num_kv_heads, .hg = num_head_groups, }, .f32)); - const scheduler_metadata_decode = zml.Tensor.zeroes(.init(.{ .b = batch_size_decode + 1 }, .i32)).withPartitioning(.{ .b = .replicated }); + const scheduler_metadata_decode = zml.Tensor.zeroes(.init(.{ .b = batch_size_decode + 1 }, .i32)); var q_decode = q.dynamicSlice1d(0, .{ .start = mixed_parameters.metadata.decode_offset, .len = batch_size_decode }).withPartitioning(.{ .hkv = .model }); q_decode = q_decode.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); From f4282a47aec72edbef1fafe5dc92e6c95c854caf Mon Sep 17 00:00:00 2001 From: Louis Lechevalier Date: Tue, 28 Jul 2026 15:57:04 +0200 Subject: [PATCH 597/881] platforms/cuda: Update libpjrt cuda (#684) Update of libpjrt cuda after rebase and patches update --- platforms/cuda/cuda.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 0046e650..58d2df70 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -204,15 +204,15 @@ _UBUNTU_PACKAGES = { ], } -PJRT_CUDA_RELEASE = "manual-2026-07-20T19-00-00Z" +PJRT_CUDA_RELEASE = "manual-2026-07-28T12-29-00Z" _PJRT_CUDA_ASSETS = { "amd64": { - "sha256": "30e09906a92cfe6b9e04013a1edd6372423188365ade9fb45aba4fa3759a862f", + "sha256": "a167dcb3af6a367dad51063c9e2a214da0895d375cea3a0ab48c574856360bd1", "url": "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-cuda_linux-amd64.tar.gz", }, "arm64": { - "sha256": "12a2176302cb9b3f8157be230e5d27ece9ee85ce0cd1381a8f243963e937ee51", + "sha256": "c06beb6ff2eee0f265065c0c5dab2a5611cc3e220045be78f16887fd60c9749a", "url": "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-cuda_linux-arm64.tar.gz", }, } From bf7e277f069ccab5c013495913e3059702ef4034 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 28 Jul 2026 18:34:52 +0200 Subject: [PATCH 598/881] zml: simplify memory allocation during compilation (#682) --- zml/module.zig | 19 ++++++++++++++++++- zml/tensor.zig | 40 ++++++++++++++++------------------------ 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/zml/module.zig b/zml/module.zig index bd4614e1..391ec6fc 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -12,6 +12,7 @@ const Exe = @import("exe.zig").Exe; const Memory = @import("platform.zig").Memory; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); +const ops = @import("ops.zig"); const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const tracer = @import("profiling/tracer.zig"); @@ -149,6 +150,10 @@ pub const CompilationContext = struct { } pub fn deinit(self: *CompilationContext) void { + if (_current == self) _current = null; + for (self.scopes.slice()) |*scope| { + scope.deinit(); + } self.mlir_pass_manager.deinit(); self.module.deinit(); self.mlir_ctx.deinit(); @@ -189,6 +194,15 @@ pub const CompilationContext = struct { self.channel_id += 1; return self.channel_id; } + + pub fn abortOOM(self: *CompilationContext) noreturn { + _ = self; + @panic("OOM"); + } + + pub fn alloc(self: *CompilationContext, T: type, n: usize) []T { + return self.arena.allocator().alloc(T, n) catch self.abortOOM(); + } }; pub fn Compiler(comptime func: anytype) type { @@ -235,7 +249,10 @@ pub fn compile( var compilation_context: CompilationContext = .init(allocator, st_io.io(), platform, opts); defer compilation_context.deinit(); - const result = emitMlir(&compilation_context, func, args) catch unreachable; + const result = emitMlir(&compilation_context, func, args) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => unreachable, + }; defer result.output_info.deinit(compilation_context.allocator); defer result.input_info.deinit(compilation_context.allocator); diff --git a/zml/tensor.zig b/zml/tensor.zig index be2314ef..07c887d0 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1893,7 +1893,8 @@ pub const Tensor = struct { /// Concatenates the input Tensors along the given axis. pub fn concatenate(tensors: []const Tensor, axis_: anytype) Tensor { if (tensors.len == 1) return tensors[0]; - var buffer = CompilationContext.current().arena.allocator().alloc(*const mlir.Value, tensors.len) catch @panic("OOM"); + const ctx = CompilationContext.current(); + var buffer = ctx.alloc(*const mlir.Value, tensors.len); std.debug.assert(tensors.len <= buffer.len); std.debug.assert(tensors.len > 0); const a = tensors[0].axis(axis_); @@ -1924,7 +1925,8 @@ pub const Tensor = struct { stdx.debug.assert(shape0.eqlWithTags(tensor._shape), "stack expects tensor shapes to match, got {f} and {f}", .{ shape0, tensor._shape }); } - var reshaped = CompilationContext.current().arena.allocator().alloc(Tensor, tensors.len) catch @panic("OOM"); + const ctx = CompilationContext.current(); + var reshaped = ctx.alloc(Tensor, tensors.len); for (tensors, 0..) |tensor, i| { reshaped[i] = tensor.reshape(res_shape); } @@ -3493,29 +3495,21 @@ pub const Tensor = struct { pub fn chunkAllowTrailing( self: Tensor, axis_: i64, - n_chunks: comptime_int, - ) ![]Tensor { + n_chunks: usize, + ) []Tensor { const a = self.axis(axis_); - const d = self.dim(a); - const chunk_size: i64 = @divFloor(d, n_chunks); + const d: i64 = self.dim(a); + const chunk_size: i64 = @divFloor(d, @as(i64, @intCast(n_chunks))); const tail_chunk_size: i64 = @rem(d, chunk_size); - const allocator = CompilationContext.current().arena.allocator(); + const len: usize = if (tail_chunk_size == 0) n_chunks else n_chunks + 1; + const chunks = CompilationContext.current().alloc(Tensor, len); - var chunks = std.ArrayList(Tensor).initCapacity(allocator, n_chunks + 1) catch @panic("OOM"); - defer chunks.deinit(allocator); - - for (0..n_chunks) |i| { + for (0.., chunks) |i, *chunk| { const start: i64 = @as(i64, @intCast(i)) * chunk_size; - chunks.appendAssumeCapacity( - self.slice1d(a, .{ .start = start, .end = start + chunk_size }), - ); + chunk.* = self.slice1d(a, .{ .start = start, .end = @min(start + chunk_size, d) }); } - if (tail_chunk_size != 0) { - const start: i64 = n_chunks * chunk_size; - chunks.appendAssumeCapacity(self.slice1d(a, .{ .start = start })); - } - return chunks.toOwnedSlice(allocator); + return chunks; } test chunkAllowTrailing { @@ -3540,8 +3534,8 @@ pub const Tensor = struct { .{ .{ 12, 2 }, 0, 3, .{ 4, 2 }, .{} }, }) |testcase| { const x_shape, const ax, const n_chunks, const res, const trailing = testcase; - const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); - const chunks = try x.chunkAllowTrailing(x.axis(ax), n_chunks); + const x = Tensor.zeroes(.init(x_shape, .f16)); + const chunks = x.chunkAllowTrailing(x.axis(ax), n_chunks); const res_shape = Shape.init(res, .f16); for (chunks[0..n_chunks]) |chk| { @@ -3566,9 +3560,7 @@ pub const Tensor = struct { for (split_sizes) |n| split_sum += n; stdx.debug.assert(split_sum == d, "split expects sum of 'split_sizes' values and axis dimension to be equal, got {} and {}", .{ split_sum, d }); - const allocator = CompilationContext.current().arena.allocator(); - - const res = allocator.alloc(Tensor, split_sizes.len) catch @panic("OOM"); + const res = CompilationContext.current().alloc(Tensor, split_sizes.len); var start: i64 = 0; for (split_sizes, 0..) |n, i| { From 1affa657ae109b82e956ef723a1fd1545f8e9677 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 29 Jul 2026 17:34:53 +0200 Subject: [PATCH 599/881] zml/Slice: correctly handle strides (#685) While working on paged attention I realize that `zml.testing.expectClose` wasn't properly handling strided Slice. To fix that I'm adding a `ContiguousItemsIterator` that loops over the contiguous part of the slice. Only used in test now, so no perf considerations. --- zml/slice.zig | 115 ++++++++++++++++++++++++++++++++++++++++++++++-- zml/testing.zig | 59 +++++++++++++++++++++---- 2 files changed, 163 insertions(+), 11 deletions(-) diff --git a/zml/slice.zig b/zml/slice.zig index a447a2a9..32aa07c3 100644 --- a/zml/slice.zig +++ b/zml/slice.zig @@ -135,11 +135,15 @@ pub const Slice = struct { pub fn data(slice: Slice) []u8 { std.debug.assert(slice.mutable); - return @constCast(slice.bytes[slice.offset_bytes..]); + std.debug.assert(slice.isContiguous()); + const end = slice.offset_bytes + slice.shape.byteSize(); + return @constCast(slice.bytes[slice.offset_bytes..end]); } pub fn constData(slice: Slice) []const u8 { - return slice.bytes[slice.offset_bytes..]; + std.debug.assert(slice.isContiguous()); + const end = slice.offset_bytes + slice.shape.byteSize(); + return slice.bytes[slice.offset_bytes..end]; } pub fn items(slice: Slice, comptime T: type) []T { @@ -150,6 +154,87 @@ pub const Slice = struct { return @ptrCast(@alignCast(slice.constData())); } + pub fn contiguousItemsIterator(slice: *const Slice, comptime T: type) ContiguousItemsIterator(T) { + std.debug.assert(slice.dtype() == DataType.fromZigType(T)); + return .init(slice); + } + + fn ContiguousItemsIterator(comptime T: type) type { + return struct { + slice: *const Slice, + contiguous_axis_start: u4, + contiguous_item_count: usize, + outer_index: usize, + outer_count: usize, + outer_coords: [constants.MAX_RANK]usize, + next_offset_bytes: usize, + + const Iterator = @This(); + + fn init(slice: *const Slice) Iterator { + const rank = slice.shape.rank(); + const total_count = slice.shape.count(); + var contiguous_axis_start: u4 = rank; + var contiguous_item_count: usize = 1; + var expected_stride: i64 = @intCast(@sizeOf(T)); + + var axis = rank; + while (axis > 0) { + axis -= 1; + const dim = slice.shape.dim(axis); + const stride = slice.byte_strides.get(axis); + if (dim != 1 and stride != expected_stride) break; + contiguous_axis_start = axis; + contiguous_item_count *= @intCast(dim); + expected_stride *= dim; + } + + if (total_count == 0) { + contiguous_item_count = 0; + } + + return .{ + .slice = slice, + .contiguous_axis_start = contiguous_axis_start, + .contiguous_item_count = contiguous_item_count, + .outer_index = 0, + .outer_count = if (contiguous_item_count == 0) 0 else total_count / contiguous_item_count, + .outer_coords = @splat(0), + .next_offset_bytes = slice.offset_bytes, + }; + } + + pub fn next(self: *Iterator) ?[]const T { + if (self.outer_index >= self.outer_count) return null; + + const start = self.next_offset_bytes; + self.outer_index += 1; + if (self.outer_index < self.outer_count) { + self.advanceOuterCoords(); + } + + const byte_len = self.contiguous_item_count * @sizeOf(T); + return @ptrCast(@alignCast(self.slice.bytes[start..][0..byte_len])); + } + + fn advanceOuterCoords(self: *Iterator) void { + var axis = self.contiguous_axis_start; + while (axis > 0) { + axis -= 1; + + self.outer_coords[axis] += 1; + self.next_offset_bytes = @intCast(@as(i64, @intCast(self.next_offset_bytes)) + self.slice.byte_strides.get(axis)); + + const dim: usize = @intCast(self.slice.shape.dim(axis)); + if (self.outer_coords[axis] < dim) return; + + self.outer_coords[axis] = 0; + self.next_offset_bytes = @intCast(@as(i64, @intCast(self.next_offset_bytes)) - @as(i64, @intCast(dim)) * self.slice.byte_strides.get(axis)); + } + } + }; + } + pub fn format( slice: @This(), writer: *std.Io.Writer, @@ -194,7 +279,10 @@ pub const Slice = struct { const stride = @divExact(slice.byte_strides.get(0), @as(i64, @intCast(@sizeOf(T)))); const needed_len: usize = if (n == 0) 0 else @intCast(@abs((n - 1) * stride) + 1); - const values = slice.constItems(T)[0..needed_len]; + // The formatter consumes this physical tail using `stride`, so this + // rank-1 path can read raw items without requiring contiguity. + const raw_values: []const T = @ptrCast(@alignCast(slice.bytes[slice.offset_bytes..])); + const values = raw_values[0..needed_len]; switch (comptime dt.class()) { .float => try stdx.fmt.formatFloatSlice(values, options, stride, writer), @@ -274,6 +362,27 @@ pub const Slice = struct { } }; +test "slice expectClose compares dense and strided slices" { + const testing = @import("testing.zig"); + + const dense_data: [2][2]f32 = .{ + .{ 1, 2 }, + .{ 5, 6 }, + }; + const strided_data: [2][4]f32 = .{ + .{ 1, 2, 99, 98 }, + .{ 5, 6, 97, 96 }, + }; + + const dense = Slice.init(.init(.{ 2, 2 }, .f32), std.mem.asBytes(&dense_data)); + const storage = Slice.init(.init(.{ 2, 4 }, .f32), std.mem.asBytes(&strided_data)); + const strided = storage.subSlice(1, 0, 2); + + try std.testing.expect(!strided.isContiguous()); + try testing.expectClose(std.testing.io, dense, strided, .exact_match); + try testing.expectClose(std.testing.io, strided, dense, .exact_match); +} + test "slice pretty print rank 3" { const data: [4][4][4]i32 = .{ .{ diff --git a/zml/testing.zig b/zml/testing.zig index 978dbaac..2ef8af78 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -87,7 +87,6 @@ pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, opts: CompareOpt .f8e8m0, => |t| { const L = t.toZigType(); - const left_data = left.constItems(L); switch (right.dtype()) { inline .bf16, .f16, @@ -100,22 +99,19 @@ pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, opts: CompareOpt .f8e5m2fnuz, => |rt| { const R = rt.toZigType(); - const right_data = right.constItems(R); - const compare_report = try compareSlices(allocator, L, R, left_data, right_data, opts); - const ok = !compare_report.nan_or_inf and compare_report.close_fraction >= opts.minimum_close_fraction; - if (ok) { - return; - } else { + if (try compareIterators(false, allocator, L, R, left, right, opts)) |compare_report| { try w.print("{f}\n", .{compare_report}); return error.TestUnexpectedResult; } + return; }, else => unreachable, } }, inline .bool, .u2, .u4, .u8, .u16, .u32, .u64, .i2, .i4, .i8, .i16, .i32, .i64 => |t| { const T = t.toZigType(); - return std.testing.expectEqualSlices(T, left.constItems(T), right.constItems(T)); + _ = try compareIterators(true, allocator, T, T, left, right, opts); + return; }, .c64, .c128 => @panic("TODO: support comparison of complex"), } @@ -176,6 +172,53 @@ pub const CompareReport = struct { } }; +fn compareReportOk(report: CompareReport, opts: CompareOpts) bool { + return !report.nan_or_inf and report.close_fraction >= opts.minimum_close_fraction; +} + +fn compareIterators(comptime mem_eql: bool, allocator: std.mem.Allocator, comptime L: type, comptime R: type, left: Slice, right: Slice, opts: CompareOpts) !?CompareReport { + var left_iter = (&left).contiguousItemsIterator(L); + var right_iter = (&right).contiguousItemsIterator(R); + var left_chunk: []const L = &.{}; + var right_chunk: []const R = &.{}; + + while (true) { + if (left_chunk.len == 0) { + left_chunk = left_iter.next() orelse &.{}; + } + if (right_chunk.len == 0) { + right_chunk = right_iter.next() orelse &.{}; + } + + if (left_chunk.len == 0 or right_chunk.len == 0) { + std.debug.assert(left_chunk.len == right_chunk.len); + return null; + } + + const n = @min(left_chunk.len, right_chunk.len); + if (comptime mem_eql) { + if (!std.mem.eql(L, left_chunk[0..n], right_chunk[0..n])) { + return error.TestUnexpectedResult; + } + } else { + const compare_report = try compareSlices( + allocator, + L, + R, + left_chunk[0..n], + right_chunk[0..n], + opts, + ); + if (!compareReportOk(compare_report, opts)) { + return compare_report; + } + } + + left_chunk = left_chunk[n..]; + right_chunk = right_chunk[n..]; + } +} + pub fn compareSlices(allocator: std.mem.Allocator, comptime L: type, comptime R: type, left: []const L, right: []const R, opts: CompareOpts) !CompareReport { var nan_or_inf: bool = false; var max_absolute_error: f32 = 0; From fdb6d5cd23c326e0493c71d594659b65646ba54d Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 29 Jul 2026 18:01:59 +0200 Subject: [PATCH 600/881] platforms/rocm: Update package.lock.json (#687) --- platforms/rocm/packages.lock.json | 314 +++++++++++++++--------------- 1 file changed, 157 insertions(+), 157 deletions(-) diff --git a/platforms/rocm/packages.lock.json b/platforms/rocm/packages.lock.json index a5e839b6..c15018c7 100644 --- a/platforms/rocm/packages.lock.json +++ b/platforms/rocm/packages.lock.json @@ -318,7 +318,7 @@ ], "key": "amdrocm-amdsmi7.14_7.14.0-3_amd64", "name": "amdrocm-amdsmi7.14", - "sha256": "34260f01ab7f6234a9d39b9cb4dbdc5634f0469d58caa0f604042ce8838f02b6", + "sha256": "f0004a48ebec4e8e2c059cc196058e870eeb8ab38daa3b58c24e350f7b42b20c", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-amdsmi7.14_7.14.0-3_amd64.deb" ], @@ -329,7 +329,7 @@ "dependencies": [], "key": "amdrocm-sysdeps7.14_7.14.0-3_amd64", "name": "amdrocm-sysdeps7.14", - "sha256": "c28c1863f3f5006fb60976e5dde098fa8ae8fe8e4b65f02116586d68fc261a00", + "sha256": "6e6ab8b0560f9157ceb2afefe481eddb93789041cbb53137018571d749205a8b", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sysdeps7.14_7.14.0-3_amd64.deb" ], @@ -1106,7 +1106,7 @@ ], "key": "amdrocm-base7.14_7.14.0-3_amd64", "name": "amdrocm-base7.14", - "sha256": "23a5667f15b45fcc1b06197eed8800d8d6757e7fd7f8806557eb7e9d80715358", + "sha256": "4221475e72a3481ae746f7562d6db2094133bb5b9c394fd7f9f1820a5f83695b", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-base7.14_7.14.0-3_amd64.deb" ], @@ -1117,7 +1117,7 @@ "dependencies": [], "key": "amdrocm-runtime7.14_7.14.0-3_amd64", "name": "amdrocm-runtime7.14", - "sha256": "19d6547f447dcfab9890d3608c5aa9b35647ab293bfb5ffbc829402e6fcc7233", + "sha256": "3b940320e839d68e05360ef70e6c4d3222cd135a4b02d68ece2267366330748c", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-runtime7.14_7.14.0-3_amd64.deb" ], @@ -1454,7 +1454,7 @@ ], "key": "amdrocm-llvm7.14_7.14.0-3_amd64", "name": "amdrocm-llvm7.14", - "sha256": "e1f1e278f87e35d6d52b2fceb01e083a1ef6e2ed808eb30701f6e7cf0a332d98", + "sha256": "46f336f876528d0c2c98503491cb1e1e176986caaf2ea281f84c56c1a8822747", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-llvm7.14_7.14.0-3_amd64.deb" ], @@ -2383,7 +2383,7 @@ ], "key": "amdrocm-blas7.14_7.14.0-3_amd64", "name": "amdrocm-blas7.14", - "sha256": "6185f9456503100afbb0ca00ad0ed0a9db8d1b181cb17006602c6265535f0699", + "sha256": "438d5d0d33dcc638b2daefd3e3358b9c5a523dd61188ea23631651dcbb277855", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14_7.14.0-3_amd64.deb" ], @@ -2394,7 +2394,7 @@ "dependencies": [], "key": "amdrocm-blas7.14-gfx950_7.14.0-3_amd64", "name": "amdrocm-blas7.14-gfx950", - "sha256": "37555d5dc4cc2ed7bf67029d8fe13d6ec96002ef186461ec6616e651b000fa45", + "sha256": "fb78b8ae9d4242466168c34c80a4a5aa3d909117c5f13f74dafd24d8f31906ed", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx950_7.14.0-3_amd64.deb" ], @@ -2405,7 +2405,7 @@ "dependencies": [], "key": "amdrocm-solver7.14-gfx950_7.14.0-3_amd64", "name": "amdrocm-solver7.14-gfx950", - "sha256": "94af3096b972d69ad8b183f2c4c3d8ef85817e968481ccee4608e18c69fc4104", + "sha256": "fd7a7ed315f374d130fb2c020dbd0d1894b148b8f83baa8e9f9d15b6dcf81eac", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx950_7.14.0-3_amd64.deb" ], @@ -2416,7 +2416,7 @@ "dependencies": [], "key": "amdrocm-sparse7.14-gfx950_7.14.0-3_amd64", "name": "amdrocm-sparse7.14-gfx950", - "sha256": "3704bff11ff1832de409d677cee1687330dfab49893dd8ef674c0b8a0035656c", + "sha256": "1d317132ee84eb02e0d30bfad61d90238cae74c7d2bd420606864c70c4644d0d", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx950_7.14.0-3_amd64.deb" ], @@ -2427,7 +2427,7 @@ "dependencies": [], "key": "amdrocm-sparse-host7.14_7.14.0-3_amd64", "name": "amdrocm-sparse-host7.14", - "sha256": "8f00e94a7c845926e2638429c19fb50181a3feaeff603ddceaacaa974ab75d5d", + "sha256": "44fcdf4dde5b78c5dcc36fe59f6dcf37cb2b6cbcab76a257c8a55e9ac4a246b3", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse-host7.14_7.14.0-3_amd64.deb" ], @@ -2769,7 +2769,7 @@ ], "key": "amdrocm-profiler-base7.14_7.14.0-3_amd64", "name": "amdrocm-profiler-base7.14", - "sha256": "507a46e794e7cc31a69ab022d1ce79e845c7e166808c1decfd669988016a22f1", + "sha256": "ae0657637788e4e22f8a8a56b7465751650465efda0f56eb7e1498f24950ac48", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-profiler-base7.14_7.14.0-3_amd64.deb" ], @@ -2780,7 +2780,7 @@ "dependencies": [], "key": "amdrocm-solver-host7.14_7.14.0-3_amd64", "name": "amdrocm-solver-host7.14", - "sha256": "073ebe38a298201071c5dddea99f17578d8ce9aa0be7a9e8f96571f0cffc8a98", + "sha256": "a90bdb7806459ec274446dac2dafe5355290211068f4f719348ab578e55589a7", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver-host7.14_7.14.0-3_amd64.deb" ], @@ -2791,7 +2791,7 @@ "dependencies": [], "key": "amdrocm-math-common7.14_7.14.0-3_amd64", "name": "amdrocm-math-common7.14", - "sha256": "e1f0d89d538af693070c8238f28c5bc1c75eb853cd8b3dc25307beb044f8c78e", + "sha256": "fc6c0a3388a0e662c26924668404ac65f4293cc2e1274b69da5aa73837685817", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-math-common7.14_7.14.0-3_amd64.deb" ], @@ -2802,7 +2802,7 @@ "dependencies": [], "key": "amdrocm-blas-host7.14_7.14.0-3_amd64", "name": "amdrocm-blas-host7.14", - "sha256": "d882b1402f11b5a512712ba582a52a73d47411392151b089e707082f918a0323", + "sha256": "6ae820948d8aa9cbcdfcbadcd1bb0b8e2601b08e329463ef88259b354ae77118", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas-host7.14_7.14.0-3_amd64.deb" ], @@ -2813,7 +2813,7 @@ "dependencies": [], "key": "amdrocm-blas7.14-gfx942_7.14.0-3_amd64", "name": "amdrocm-blas7.14-gfx942", - "sha256": "5f77db1f8df1275d3b97154d1680ef5387cc023e05f5ed782e957442524b1846", + "sha256": "43e62de97c42bc40e78e5a833ca181662233f68af77df3412afbb348a7f07841", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx942_7.14.0-3_amd64.deb" ], @@ -2824,7 +2824,7 @@ "dependencies": [], "key": "amdrocm-solver7.14-gfx942_7.14.0-3_amd64", "name": "amdrocm-solver7.14-gfx942", - "sha256": "84718ac1480b4b9cd37d83702b4c711531665c950f96bc585e9374dc8546bb0a", + "sha256": "a186901689b546b9c79189431baee991602528dded1fdea268d70a33b356e8c0", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx942_7.14.0-3_amd64.deb" ], @@ -2835,7 +2835,7 @@ "dependencies": [], "key": "amdrocm-sparse7.14-gfx942_7.14.0-3_amd64", "name": "amdrocm-sparse7.14-gfx942", - "sha256": "a7a93ee9e98dcfc3b2b6a49b695e453482d3c72d55d180185265195a4a175ae0", + "sha256": "64375a7d1be88879e697fd079f69f2d5b57f9c034d61934ef1a5d8f3948e299a", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx942_7.14.0-3_amd64.deb" ], @@ -2846,7 +2846,7 @@ "dependencies": [], "key": "amdrocm-blas7.14-gfx90a_7.14.0-3_amd64", "name": "amdrocm-blas7.14-gfx90a", - "sha256": "2378006fc21ebc592bff0cca01ef1974ed1c18f5e1daa4059fa67eb032d7a3ec", + "sha256": "ed695f335ae112920846d3d8d6ccf9eebd0546dff13bbb30bddf7bcbe7acc079", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx90a_7.14.0-3_amd64.deb" ], @@ -2857,7 +2857,7 @@ "dependencies": [], "key": "amdrocm-solver7.14-gfx90a_7.14.0-3_amd64", "name": "amdrocm-solver7.14-gfx90a", - "sha256": "4bfb918d48762449d298b8c9b2cc9f81a7edbc047137139c26009c4a4fe66d59", + "sha256": "eedd257207dc3a57ce1121577e287dc81132a71b46f1cfa23a8ab83c82d7cf7a", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx90a_7.14.0-3_amd64.deb" ], @@ -2868,7 +2868,7 @@ "dependencies": [], "key": "amdrocm-sparse7.14-gfx90a_7.14.0-3_amd64", "name": "amdrocm-sparse7.14-gfx90a", - "sha256": "8c4e3ff68571f355376704fe0510edb0e70eb99a862f0bafe3fec873894231b0", + "sha256": "cde6903dd58348623343433a6696556cfce427ded9bcf9f94259b29c1f5bae9a", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx90a_7.14.0-3_amd64.deb" ], @@ -2879,7 +2879,7 @@ "dependencies": [], "key": "amdrocm-blas7.14-gfx908_7.14.0-3_amd64", "name": "amdrocm-blas7.14-gfx908", - "sha256": "5d904096524027c02b6e2a91aba901a2527b24c233045f60410130e705ad8d27", + "sha256": "721b7c73c2e8041ae07059c7efad152db6d4be7e1692c78ffae8321ed0f84bdc", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx908_7.14.0-3_amd64.deb" ], @@ -2890,7 +2890,7 @@ "dependencies": [], "key": "amdrocm-solver7.14-gfx908_7.14.0-3_amd64", "name": "amdrocm-solver7.14-gfx908", - "sha256": "0db8deab37561a8c3e8dc01071f93cc4d90464070ad1e81b7742b5ddac670a3e", + "sha256": "30414b9c9e5aef3d59b66d168c7a94c2ab284f52a9250f7df08f5651abf392e5", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx908_7.14.0-3_amd64.deb" ], @@ -2901,7 +2901,7 @@ "dependencies": [], "key": "amdrocm-sparse7.14-gfx908_7.14.0-3_amd64", "name": "amdrocm-sparse7.14-gfx908", - "sha256": "849a815de62271f3f3fe775a11e3d5e9c270e136b02ae403adedb05b06f5b64a", + "sha256": "58611375a235e7e720c759db37d32dafa20aa1519ce711e6e39a790a9e5ab90c", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx908_7.14.0-3_amd64.deb" ], @@ -2912,7 +2912,7 @@ "dependencies": [], "key": "amdrocm-blas7.14-gfx1250_7.14.0-3_amd64", "name": "amdrocm-blas7.14-gfx1250", - "sha256": "3549c185d9fd4cd2304b31eb1a923e61fe5f3ece58f65eff9cce814da6cc383b", + "sha256": "931ab849205b4739d4e2494e77af9ca8e9a4e5b8a24a7bffd4e7ff69b2ae8fed", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1250_7.14.0-3_amd64.deb" ], @@ -2923,7 +2923,7 @@ "dependencies": [], "key": "amdrocm-solver7.14-gfx1250_7.14.0-3_amd64", "name": "amdrocm-solver7.14-gfx1250", - "sha256": "e3d3977896ebfa70842aab355f0e104401d7c5a0b6a4e6f69d02d6f38ac025de", + "sha256": "d87abf95815e7f6aa183cca6ad9fba9fb9a67b7d93e7e2a8eab1177021a809f9", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1250_7.14.0-3_amd64.deb" ], @@ -2934,7 +2934,7 @@ "dependencies": [], "key": "amdrocm-sparse7.14-gfx1250_7.14.0-3_amd64", "name": "amdrocm-sparse7.14-gfx1250", - "sha256": "2073a0bf77e2dbef917286cc1f0b21012d4bd5494efacaa090bb31e447afbe01", + "sha256": "0de2d194fed57eddf6bd70ce418f58bf88a5b66b1cea9e7af96c3fe7bec9d1e2", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1250_7.14.0-3_amd64.deb" ], @@ -2945,7 +2945,7 @@ "dependencies": [], "key": "amdrocm-blas7.14-gfx1201_7.14.0-3_amd64", "name": "amdrocm-blas7.14-gfx1201", - "sha256": "29af1bea27f6ce2e7be0fa4361e073364b67468e8dc7873b8782dfb00f777729", + "sha256": "df33bae5bf26e5dff8828355a25a25bfdcb076296aeb491ddc1deb9edb151227", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1201_7.14.0-3_amd64.deb" ], @@ -2956,7 +2956,7 @@ "dependencies": [], "key": "amdrocm-solver7.14-gfx1201_7.14.0-3_amd64", "name": "amdrocm-solver7.14-gfx1201", - "sha256": "c914feb757a5af0b015bee9f8604b5ecfaa034c23747c2ffa77ce567fc0c505c", + "sha256": "8f9f8bc4689edb0227152e2bb99654f209705fb99633f49e69d1bf38ff44009d", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1201_7.14.0-3_amd64.deb" ], @@ -2967,7 +2967,7 @@ "dependencies": [], "key": "amdrocm-sparse7.14-gfx1201_7.14.0-3_amd64", "name": "amdrocm-sparse7.14-gfx1201", - "sha256": "3a00cdceb9a4c88b2e8cd6d76c8ec3fe92dae8f3c5c2af51704efdaea7e6836a", + "sha256": "81f2492452b4bb1bf7f82632ed386df27a90d6feafbeb65640694577aca27c8e", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1201_7.14.0-3_amd64.deb" ], @@ -2978,7 +2978,7 @@ "dependencies": [], "key": "amdrocm-blas7.14-gfx1200_7.14.0-3_amd64", "name": "amdrocm-blas7.14-gfx1200", - "sha256": "b8d908829bf0cbaa2699a52e86653d44a6312deee4f028bff2ef52da28d62e01", + "sha256": "87358ca164a6519483c55d478fe3ac67ceeed35b4881d7ec4c52b646b1dff715", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1200_7.14.0-3_amd64.deb" ], @@ -2989,7 +2989,7 @@ "dependencies": [], "key": "amdrocm-solver7.14-gfx1200_7.14.0-3_amd64", "name": "amdrocm-solver7.14-gfx1200", - "sha256": "12b347316dbedc95953d60509d1244711a6685b856615a767fbb05e34c82110c", + "sha256": "291c3bb0c3ca7a61820ee626626eecae9bad2b42a54aeedab2c3a5ccba359090", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1200_7.14.0-3_amd64.deb" ], @@ -3000,7 +3000,7 @@ "dependencies": [], "key": "amdrocm-sparse7.14-gfx1200_7.14.0-3_amd64", "name": "amdrocm-sparse7.14-gfx1200", - "sha256": "d341d18d28f201de15f5d2f42645e56e43ca0d84fe0f00ec08d4e41da8cad08e", + "sha256": "b0324406210d54b925ab5c0df383e4ee80776a792019f4fa9ae0fa16cecc91c3", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1200_7.14.0-3_amd64.deb" ], @@ -3011,7 +3011,7 @@ "dependencies": [], "key": "amdrocm-blas7.14-gfx1153_7.14.0-3_amd64", "name": "amdrocm-blas7.14-gfx1153", - "sha256": "e66495b2a965237a9646af29971c25c7ab0626613963e88e2aaf3657f7ce09d8", + "sha256": "153c4ac2cead0fa5f91c26a6a0471a7572082f38452e3b36d8baf49541579dad", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1153_7.14.0-3_amd64.deb" ], @@ -3022,7 +3022,7 @@ "dependencies": [], "key": "amdrocm-solver7.14-gfx1153_7.14.0-3_amd64", "name": "amdrocm-solver7.14-gfx1153", - "sha256": "0acd58606717214c209178100a89d4943f4c061134c47e7dc88bace3a770c3f1", + "sha256": "9ee83c070b1eec3dcf2e2ecf5ab31af311f2e6fc22e82f7c0e0811ce90e732f2", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1153_7.14.0-3_amd64.deb" ], @@ -3033,7 +3033,7 @@ "dependencies": [], "key": "amdrocm-sparse7.14-gfx1153_7.14.0-3_amd64", "name": "amdrocm-sparse7.14-gfx1153", - "sha256": "87854d7d6ffa380d30b2dcb8f8737e39ad226fc184e9ceaa851ca84d973ea74e", + "sha256": "2595f52211b7ab173be376d14fa0867e9f88baf00ffa0a882c52b42362786148", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1153_7.14.0-3_amd64.deb" ], @@ -3044,7 +3044,7 @@ "dependencies": [], "key": "amdrocm-blas7.14-gfx1152_7.14.0-3_amd64", "name": "amdrocm-blas7.14-gfx1152", - "sha256": "4bac4db8173aa54471bd32d22de071032c3a1ee45e9a6c76c5599e3cd21ba650", + "sha256": "b168022b0d9bb346999076aab759db346454bc6e1544e384365f87bdcd46f4d2", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1152_7.14.0-3_amd64.deb" ], @@ -3055,7 +3055,7 @@ "dependencies": [], "key": "amdrocm-solver7.14-gfx1152_7.14.0-3_amd64", "name": "amdrocm-solver7.14-gfx1152", - "sha256": "7ebe93fdadf24b4a40f6f400919522084b5396aee017a3555c7b877362bd51bb", + "sha256": "f83f9da40f6c5aa6a4a8be037fa14c315aadeb57627020cc655bea952c100359", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1152_7.14.0-3_amd64.deb" ], @@ -3066,7 +3066,7 @@ "dependencies": [], "key": "amdrocm-sparse7.14-gfx1152_7.14.0-3_amd64", "name": "amdrocm-sparse7.14-gfx1152", - "sha256": "a1485536d93d03ffc720c7b642a7cdd68fa7fbcc33e852deef79aadad4179653", + "sha256": "5d2d78888d289f0fdc11b70c3d8e5f39489410d0147a56deecc371b272c20584", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1152_7.14.0-3_amd64.deb" ], @@ -3077,7 +3077,7 @@ "dependencies": [], "key": "amdrocm-blas7.14-gfx1151_7.14.0-3_amd64", "name": "amdrocm-blas7.14-gfx1151", - "sha256": "06918d9e3684158fd3bf7857403e3f3e8cd24d8a68ccd71018ea1e59e5c2199e", + "sha256": "0e3ef5efb688114b036d09ceea520c7d63e6b4e76d3e06551c3894bf3236b26a", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1151_7.14.0-3_amd64.deb" ], @@ -3088,7 +3088,7 @@ "dependencies": [], "key": "amdrocm-solver7.14-gfx1151_7.14.0-3_amd64", "name": "amdrocm-solver7.14-gfx1151", - "sha256": "0b6894acba6d0e63826271a664150a54386c25336cb30b27b4d6a08f8db4056b", + "sha256": "736ab9ee57018110fd6299eeaf092edcc67d9fbf38ea671f1843241174bdf3db", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1151_7.14.0-3_amd64.deb" ], @@ -3099,7 +3099,7 @@ "dependencies": [], "key": "amdrocm-sparse7.14-gfx1151_7.14.0-3_amd64", "name": "amdrocm-sparse7.14-gfx1151", - "sha256": "961e7e5903c0647b0f8df08b5f0e0ca334135f715c1bd02e0d95b10a5394540b", + "sha256": "735c32e3b322ed32476dd343e4c46bffa7d9ac48c98a53acb538c9f251e6ec75", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1151_7.14.0-3_amd64.deb" ], @@ -3110,7 +3110,7 @@ "dependencies": [], "key": "amdrocm-blas7.14-gfx1150_7.14.0-3_amd64", "name": "amdrocm-blas7.14-gfx1150", - "sha256": "62acba597d258961a03d1227ef6d1f8a79961a929ef5f19941fbdd38470b9f5c", + "sha256": "1c820ee86dfd2aaea8eecdf2dbdc1a9e162f2acd3e8102c4a49459a315ac76de", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1150_7.14.0-3_amd64.deb" ], @@ -3121,7 +3121,7 @@ "dependencies": [], "key": "amdrocm-solver7.14-gfx1150_7.14.0-3_amd64", "name": "amdrocm-solver7.14-gfx1150", - "sha256": "8fb4e92db83c9f75b7912843dfc2c30b2b9077dc6174a14c6b54ad1ca853a480", + "sha256": "32930f5dfa59fccbbfa23782d9778e19a7fb3ead1cd2a715d8531a0236502ccf", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1150_7.14.0-3_amd64.deb" ], @@ -3132,7 +3132,7 @@ "dependencies": [], "key": "amdrocm-sparse7.14-gfx1150_7.14.0-3_amd64", "name": "amdrocm-sparse7.14-gfx1150", - "sha256": "2a1342a284990cdaf192b7907f1e5dbb7d8abc7b032dc7dd1bfd743ea30efae2", + "sha256": "88c3952d7e798691d455701b4c06895e7b8a99cfff62b6a053bfeeefd47f9505", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1150_7.14.0-3_amd64.deb" ], @@ -3143,7 +3143,7 @@ "dependencies": [], "key": "amdrocm-blas7.14-gfx1103_7.14.0-3_amd64", "name": "amdrocm-blas7.14-gfx1103", - "sha256": "69a45e92bc0d6062100518e2228ed7f30477ed74154c5d951c7c784fd699987d", + "sha256": "3ea904b6c1aaece72759c30497342e75edf884af93f6fa5b8d1f6010c47009aa", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1103_7.14.0-3_amd64.deb" ], @@ -3154,7 +3154,7 @@ "dependencies": [], "key": "amdrocm-solver7.14-gfx1103_7.14.0-3_amd64", "name": "amdrocm-solver7.14-gfx1103", - "sha256": "091c9bb0939d77b895c5957207a103086a5cf249360aa6709509b84413c5ad9c", + "sha256": "7ac5d2d630efb88fe3e3550f07fd564e78d9f6406481eb21e01ddaeaa04ff9d2", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1103_7.14.0-3_amd64.deb" ], @@ -3165,7 +3165,7 @@ "dependencies": [], "key": "amdrocm-sparse7.14-gfx1103_7.14.0-3_amd64", "name": "amdrocm-sparse7.14-gfx1103", - "sha256": "1536402b703fe143fdbbfa092276bdced04095bf993f8f4cf1ca2fbb30616f0b", + "sha256": "e34e89ab81b6711c7c766d2c8dcc5575a33f6a641cfaafd77f6391f07a6fe5e3", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1103_7.14.0-3_amd64.deb" ], @@ -3176,7 +3176,7 @@ "dependencies": [], "key": "amdrocm-blas7.14-gfx1102_7.14.0-3_amd64", "name": "amdrocm-blas7.14-gfx1102", - "sha256": "28f90078cc3afe13febfe69ba6b9dfe28a42781b06d26ecfbc3eb71a4b08c4ee", + "sha256": "b670c3bcff65f8a79da828685da147d2131451351ba6ffadb604f15e456208bd", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1102_7.14.0-3_amd64.deb" ], @@ -3187,7 +3187,7 @@ "dependencies": [], "key": "amdrocm-solver7.14-gfx1102_7.14.0-3_amd64", "name": "amdrocm-solver7.14-gfx1102", - "sha256": "ff6147c9adc3d721495cf9525f50789559990734a4343e69fdb4f40bd8c3dd99", + "sha256": "add2034c3d41bf6767b7bffa307ef6d204d2807f400dec343f02f3a61d2628e2", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1102_7.14.0-3_amd64.deb" ], @@ -3198,7 +3198,7 @@ "dependencies": [], "key": "amdrocm-sparse7.14-gfx1102_7.14.0-3_amd64", "name": "amdrocm-sparse7.14-gfx1102", - "sha256": "13b2c9c27738af4cdccdb9401fb3169cfde8590b3db784f3725c5168e2bf7f7c", + "sha256": "00b364d767b504194a070126b02fe6b614018043366cf9a1e199038cf4074f57", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1102_7.14.0-3_amd64.deb" ], @@ -3209,7 +3209,7 @@ "dependencies": [], "key": "amdrocm-blas7.14-gfx1101_7.14.0-3_amd64", "name": "amdrocm-blas7.14-gfx1101", - "sha256": "7be4aa68d39786bb64cb326621f4f7c9ec1e31df85e5a1300b2899df93f601b0", + "sha256": "fdfb768ffa00776bbc96f077ef387ef6a757ebb9b35b9e3454bee410ebd3617b", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1101_7.14.0-3_amd64.deb" ], @@ -3220,7 +3220,7 @@ "dependencies": [], "key": "amdrocm-solver7.14-gfx1101_7.14.0-3_amd64", "name": "amdrocm-solver7.14-gfx1101", - "sha256": "41cbbfc85150022e4f692324daddad46cad908e692da2e969146d44c5c101de2", + "sha256": "169fa743505585b7c96e025da8878d956e4268657aec302bfb07e95f891950be", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1101_7.14.0-3_amd64.deb" ], @@ -3231,7 +3231,7 @@ "dependencies": [], "key": "amdrocm-sparse7.14-gfx1101_7.14.0-3_amd64", "name": "amdrocm-sparse7.14-gfx1101", - "sha256": "e5f3ba34cbfcb41d6343536d4d06c454e5832c9ebe00290fb4ba3d66aba8ce44", + "sha256": "aeecc9ce33b28948ffbc33362784a9bfb9fe74275939191d378df1fccc1dc080", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1101_7.14.0-3_amd64.deb" ], @@ -3242,7 +3242,7 @@ "dependencies": [], "key": "amdrocm-blas7.14-gfx1100_7.14.0-3_amd64", "name": "amdrocm-blas7.14-gfx1100", - "sha256": "073e99c4fca56c93dfc54576a22ed7d6622dc968e0d0eb480c4c4dc8b51b1374", + "sha256": "0203356336f2d90d4e60070b12d71abafa0b24432c43ebe6dbbed3ed4af3f283", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1100_7.14.0-3_amd64.deb" ], @@ -3253,7 +3253,7 @@ "dependencies": [], "key": "amdrocm-solver7.14-gfx1100_7.14.0-3_amd64", "name": "amdrocm-solver7.14-gfx1100", - "sha256": "5e3a1bca9c408f07d8eb9d6147005c99a635104ad3ec29bffabc46cd1198bd6c", + "sha256": "e0a57a6519a7c0e0be684d07409c11127ef1408aeedcc5e0efc8fdd5996e66aa", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1100_7.14.0-3_amd64.deb" ], @@ -3264,7 +3264,7 @@ "dependencies": [], "key": "amdrocm-sparse7.14-gfx1100_7.14.0-3_amd64", "name": "amdrocm-sparse7.14-gfx1100", - "sha256": "988f2f212f215bfc675884dacfcc81aa077e62a97ece927ce6432d5fbb6b616d", + "sha256": "692f7ef41e2b6eae1b8bd2fa77435f0783967a0ee34353718ea0179059a22f5c", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1100_7.14.0-3_amd64.deb" ], @@ -3275,7 +3275,7 @@ "dependencies": [], "key": "amdrocm-blas7.14-gfx1036_7.14.0-3_amd64", "name": "amdrocm-blas7.14-gfx1036", - "sha256": "127db0014b130e582b42c2d3cda0f66f3130130c719030eb254dfdecace825a3", + "sha256": "cb5b86af54a4d10a08ee9775c329e087ef4d98b041706b80363a8713e2348e42", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1036_7.14.0-3_amd64.deb" ], @@ -3286,7 +3286,7 @@ "dependencies": [], "key": "amdrocm-solver7.14-gfx1036_7.14.0-3_amd64", "name": "amdrocm-solver7.14-gfx1036", - "sha256": "03a5a620a762acf72fd5130c1d63bb966ac9a913277bba93377e1da4c7678261", + "sha256": "e702804b6edcde357548e9e72b7dd6625d945a377379193a1db8c072258f837b", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1036_7.14.0-3_amd64.deb" ], @@ -3297,7 +3297,7 @@ "dependencies": [], "key": "amdrocm-sparse7.14-gfx1036_7.14.0-3_amd64", "name": "amdrocm-sparse7.14-gfx1036", - "sha256": "47a9a9b9bf978d93183fa2091bd9b1c42659fa901479c294580d3f3611de4830", + "sha256": "eb343fb36c7d1fd2dd46f2388bf8a116339cbca1e6305e3fa68ec6db06614a88", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1036_7.14.0-3_amd64.deb" ], @@ -3308,7 +3308,7 @@ "dependencies": [], "key": "amdrocm-blas7.14-gfx1035_7.14.0-3_amd64", "name": "amdrocm-blas7.14-gfx1035", - "sha256": "2e1d9d05d10445bbf14fc5282321878e90ebb2e11cd513675adf5c1b8779c591", + "sha256": "d1da329e9d6bdf46f095de4d728f0206b01f82767536e6aaa57000e4e5fc3c17", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1035_7.14.0-3_amd64.deb" ], @@ -3319,7 +3319,7 @@ "dependencies": [], "key": "amdrocm-solver7.14-gfx1035_7.14.0-3_amd64", "name": "amdrocm-solver7.14-gfx1035", - "sha256": "0e775a542340bcf2fa90364b990f7897719604cccc02fc75c223995cb848820b", + "sha256": "afcba030a7b8fd22dd44e89dffd816a18247f976e9152bcd2151ff814346180a", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1035_7.14.0-3_amd64.deb" ], @@ -3330,7 +3330,7 @@ "dependencies": [], "key": "amdrocm-sparse7.14-gfx1035_7.14.0-3_amd64", "name": "amdrocm-sparse7.14-gfx1035", - "sha256": "5b64cd6d61a066516d25027f743855d15c72aa1e05845095bb1074f342292cc8", + "sha256": "34d0581c35b5cb042ce3bd039ea70d9b4ca7b3ddb02efb895423833ac4d34d26", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1035_7.14.0-3_amd64.deb" ], @@ -3341,7 +3341,7 @@ "dependencies": [], "key": "amdrocm-blas7.14-gfx1034_7.14.0-3_amd64", "name": "amdrocm-blas7.14-gfx1034", - "sha256": "eedca192ef7c17d0800ced37f613bb6ec73b7ec3e963a99c265f8d6ead4dda34", + "sha256": "ee69a306ee6b0e4420fb0d153d20a9bc2c03b1962d6ec7f331320e32ea704fd0", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1034_7.14.0-3_amd64.deb" ], @@ -3352,7 +3352,7 @@ "dependencies": [], "key": "amdrocm-solver7.14-gfx1034_7.14.0-3_amd64", "name": "amdrocm-solver7.14-gfx1034", - "sha256": "1a65bae315a2371625a046fc3f4bc146f80064cd2badd59d4e1c9bd24f06f355", + "sha256": "aef66e78e7ea247f6f5849d9cc7f395f9db42ef5c8e33b8dab31c1490203d3a9", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1034_7.14.0-3_amd64.deb" ], @@ -3363,7 +3363,7 @@ "dependencies": [], "key": "amdrocm-sparse7.14-gfx1034_7.14.0-3_amd64", "name": "amdrocm-sparse7.14-gfx1034", - "sha256": "c4e218ac85700a4f91fc0e9cbb69d4455b307269932383f76722d097cb6d5ae3", + "sha256": "31f93def4df1d42ba2c8f650701929db005c83fc3c237b72ee99b6f58ff4cd5e", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1034_7.14.0-3_amd64.deb" ], @@ -3374,7 +3374,7 @@ "dependencies": [], "key": "amdrocm-blas7.14-gfx1033_7.14.0-3_amd64", "name": "amdrocm-blas7.14-gfx1033", - "sha256": "768d266b8b89954e2b1c1b025d13273d1056f80d8ffafb3ba4e7731b56b6b260", + "sha256": "57510da3b868d360a3c931d60e263bdaf9347626fa8f6814f0b123b2f0358208", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1033_7.14.0-3_amd64.deb" ], @@ -3385,7 +3385,7 @@ "dependencies": [], "key": "amdrocm-solver7.14-gfx1033_7.14.0-3_amd64", "name": "amdrocm-solver7.14-gfx1033", - "sha256": "2025c0fe847ac052d474c73bf1e638c9f9c4aab6c8ae44ff434a2c59454ab397", + "sha256": "66db48a235042d92146745eec1f14370a6d31844c21a8d43ccd607ffcbc2d02d", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1033_7.14.0-3_amd64.deb" ], @@ -3396,7 +3396,7 @@ "dependencies": [], "key": "amdrocm-sparse7.14-gfx1033_7.14.0-3_amd64", "name": "amdrocm-sparse7.14-gfx1033", - "sha256": "8a3fe04c8b88892710f9c0c15d1928b7bea8a63aad743a98085d9373ff46a4c6", + "sha256": "893bcd9b43d56835b9a3a04ea10b0d4fdfb0a2dd7f8c7be74030a9e281a5bb42", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1033_7.14.0-3_amd64.deb" ], @@ -3407,7 +3407,7 @@ "dependencies": [], "key": "amdrocm-blas7.14-gfx1032_7.14.0-3_amd64", "name": "amdrocm-blas7.14-gfx1032", - "sha256": "bd1eb0fd3b38ad2cc369ff3ae834902d28f53f6499e31f7031139cdfbfee8af5", + "sha256": "0775f39717812228d56cad734a931d81d0222dc086f2dda0cc3055864f62c057", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1032_7.14.0-3_amd64.deb" ], @@ -3418,7 +3418,7 @@ "dependencies": [], "key": "amdrocm-solver7.14-gfx1032_7.14.0-3_amd64", "name": "amdrocm-solver7.14-gfx1032", - "sha256": "783ca6c75ee5c64d61038a91ac0b0ca677247afa591809e681698f498e8ecf0f", + "sha256": "c7016ccff068f1defaef11f43390c87ce9ad7b5a44daf70ca6fc6c64f7fabf05", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1032_7.14.0-3_amd64.deb" ], @@ -3429,7 +3429,7 @@ "dependencies": [], "key": "amdrocm-sparse7.14-gfx1032_7.14.0-3_amd64", "name": "amdrocm-sparse7.14-gfx1032", - "sha256": "43fadd0f6d36ffa653e72e41869c50c56efd579402419ec4476aeba3ab072cd4", + "sha256": "19f4afaaee5ab6edbf3d7f5307a2c075a397dbc81bcaca9813cdcea038138821", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1032_7.14.0-3_amd64.deb" ], @@ -3440,7 +3440,7 @@ "dependencies": [], "key": "amdrocm-blas7.14-gfx1031_7.14.0-3_amd64", "name": "amdrocm-blas7.14-gfx1031", - "sha256": "22cd13492b455071f866165df4f94b262dcd8919c2810d9ed614916c40102ef6", + "sha256": "b2a9cb9b3cb84b1c856ee44c4d9951c077b6d6a5338bd058c64bba5cd3e7e18b", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1031_7.14.0-3_amd64.deb" ], @@ -3451,7 +3451,7 @@ "dependencies": [], "key": "amdrocm-solver7.14-gfx1031_7.14.0-3_amd64", "name": "amdrocm-solver7.14-gfx1031", - "sha256": "6aa0219cce80ce25558fc4a0147a05afcfd26bbbe4d511313ce71f483874f748", + "sha256": "ad636af6a4a3cb422cd2983c0ded18f926a51952dd6fabe7c8aedd6205001d20", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1031_7.14.0-3_amd64.deb" ], @@ -3462,7 +3462,7 @@ "dependencies": [], "key": "amdrocm-sparse7.14-gfx1031_7.14.0-3_amd64", "name": "amdrocm-sparse7.14-gfx1031", - "sha256": "38319e83a0ac3591da259f059bf7c5407e119533dce6d9ed04748c5453bf516a", + "sha256": "d8cd61af88b60309d76b584638e6c3b4974c67709db095bd6dc165e5975cf449", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1031_7.14.0-3_amd64.deb" ], @@ -3473,7 +3473,7 @@ "dependencies": [], "key": "amdrocm-blas7.14-gfx1030_7.14.0-3_amd64", "name": "amdrocm-blas7.14-gfx1030", - "sha256": "1cf5b27e6dd02cfaee80ba0e9d1a691a169d4753a3ca7b3a71c7ce4fec821fdc", + "sha256": "8aaa8907924c4396fb4e602911691ca69867b99d7ca9e8a39128d7a6027a7392", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1030_7.14.0-3_amd64.deb" ], @@ -3484,7 +3484,7 @@ "dependencies": [], "key": "amdrocm-solver7.14-gfx1030_7.14.0-3_amd64", "name": "amdrocm-solver7.14-gfx1030", - "sha256": "4f083b45ebf3c0994e7f2bcc3a25425c90057da27e6cf0cda9b7136ea12a8b3a", + "sha256": "09452341928fca391c3be8bfca14bafe9b84a2277a22a7a3abf315124ceb387e", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1030_7.14.0-3_amd64.deb" ], @@ -3495,7 +3495,7 @@ "dependencies": [], "key": "amdrocm-sparse7.14-gfx1030_7.14.0-3_amd64", "name": "amdrocm-sparse7.14-gfx1030", - "sha256": "d16d68234b195ebc9c2cae77c288fc0c3e6ba2b8f9c92468acd7259e767c0c41", + "sha256": "2d7e48205f5a19290d8d7908cda57123a35e72f452ad6aaa1330a34cf477ebbc", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1030_7.14.0-3_amd64.deb" ], @@ -3506,7 +3506,7 @@ "dependencies": [], "key": "amdrocm-blas7.14-gfx1012_7.14.0-3_amd64", "name": "amdrocm-blas7.14-gfx1012", - "sha256": "4193b310cfb51a101e032e1bf7ab6dbd3dc7c3c8df959d512ddaf01a8ea758c4", + "sha256": "f04222a1088e0080d0a8e4758982e7bf8bb5c0042acf1493e3d4c4489b719594", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1012_7.14.0-3_amd64.deb" ], @@ -3517,7 +3517,7 @@ "dependencies": [], "key": "amdrocm-solver7.14-gfx1012_7.14.0-3_amd64", "name": "amdrocm-solver7.14-gfx1012", - "sha256": "78e023559a533ddcffb4b313da935e01eeaffd0bc388ab5937c3adde92340c6a", + "sha256": "d322dce522cd973bf0b9cd19166d3c4c67d01f9cf33fca571a8303cdbcbc2ae9", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1012_7.14.0-3_amd64.deb" ], @@ -3528,7 +3528,7 @@ "dependencies": [], "key": "amdrocm-sparse7.14-gfx1012_7.14.0-3_amd64", "name": "amdrocm-sparse7.14-gfx1012", - "sha256": "707d42d3b433e4c11105e0e9a32fc7b9254eb5f390a9476721ff72d318835925", + "sha256": "1343cbeed7851ee5f1e38723f31bed8af9eb6543d1a80ddf8fd3b87846f9d26f", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1012_7.14.0-3_amd64.deb" ], @@ -3539,7 +3539,7 @@ "dependencies": [], "key": "amdrocm-blas7.14-gfx1011_7.14.0-3_amd64", "name": "amdrocm-blas7.14-gfx1011", - "sha256": "159e9129e7ef839e188a415c97f6b88280bf06020a5507c699e9dfd768ecdfc9", + "sha256": "2079a6378938c2e07f11e20135849773b2679ddfc3da29eed35dab2dffbbf99a", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1011_7.14.0-3_amd64.deb" ], @@ -3550,7 +3550,7 @@ "dependencies": [], "key": "amdrocm-solver7.14-gfx1011_7.14.0-3_amd64", "name": "amdrocm-solver7.14-gfx1011", - "sha256": "e5d7ab7881ebe5d764548846e765822ef92d1f06c501206076d1cd75fa3f98a8", + "sha256": "88c4190d84c013a822f937e2322208c4e66a999b35ff82613351114018d12ed8", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1011_7.14.0-3_amd64.deb" ], @@ -3561,7 +3561,7 @@ "dependencies": [], "key": "amdrocm-sparse7.14-gfx1011_7.14.0-3_amd64", "name": "amdrocm-sparse7.14-gfx1011", - "sha256": "4f398be0c30b507ef2cb9b42130c764a68a71b1f0caf9127754314c8c6ca953c", + "sha256": "a62ee0afb6d26bf672039805b2439b8765e1fddfeaceaa8f03661a1a01bfeeea", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1011_7.14.0-3_amd64.deb" ], @@ -3572,7 +3572,7 @@ "dependencies": [], "key": "amdrocm-blas7.14-gfx1010_7.14.0-3_amd64", "name": "amdrocm-blas7.14-gfx1010", - "sha256": "2027bf09cad4242f7fc59df5bdc5fa165a62b61b4e2d14a7ab8ec61c62f2a845", + "sha256": "a1a79437e677a5905f56f4aa4492b228c23e8ffbd95016191b6054e9c3a14372", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1010_7.14.0-3_amd64.deb" ], @@ -3583,7 +3583,7 @@ "dependencies": [], "key": "amdrocm-solver7.14-gfx1010_7.14.0-3_amd64", "name": "amdrocm-solver7.14-gfx1010", - "sha256": "14d752fef6f49fc4de090a8400f9c392ccbe0bfbb068a55de34af022e5d2e6f3", + "sha256": "cc05c8b169ed7b38d49121f9aa9be9a5a4373e967eed26e24162fcd208fa44f4", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1010_7.14.0-3_amd64.deb" ], @@ -3594,7 +3594,7 @@ "dependencies": [], "key": "amdrocm-sparse7.14-gfx1010_7.14.0-3_amd64", "name": "amdrocm-sparse7.14-gfx1010", - "sha256": "7bc93af4c56e7661c1ddda1d23bd9f3dbf2f99b9411814ea66d41dd566c29dd7", + "sha256": "9ab745331d7344f0bdd43a906a43e0e5df74d5b7a1af910bfbe03ece7e6e40bf", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1010_7.14.0-3_amd64.deb" ], @@ -3626,7 +3626,7 @@ ], "key": "amdrocm-blas-dev7.14_7.14.0-3_amd64", "name": "amdrocm-blas-dev7.14", - "sha256": "faee0d272c0d16e63b019e8664bff61815a9d8395b9793520af2c59f89957058", + "sha256": "37c4dae51923ad0110937c183ce0bbf463b84191c6b86f6a8cf668eeea82d0f0", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas-dev7.14_7.14.0-3_amd64.deb" ], @@ -4278,7 +4278,7 @@ ], "key": "amdrocm-dnn7.14_7.14.0-3_amd64", "name": "amdrocm-dnn7.14", - "sha256": "c4bb3af3ca298c2ac846e76380f16f66ccce584781098ed142039755358b4d7e", + "sha256": "59654345f316a40ad389d984e6a14a22bd00a8f9818343be18cbee7a4bcf663b", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14_7.14.0-3_amd64.deb" ], @@ -4289,7 +4289,7 @@ "dependencies": [], "key": "amdrocm-dnn7.14-gfx950_7.14.0-3_amd64", "name": "amdrocm-dnn7.14-gfx950", - "sha256": "b8addad1da1bbbc0764622166c50f89e79a525b58ce0a0ae7ec0e1e4d819ba3c", + "sha256": "798e334ae094930cb64027cc5a262692095aeb4d94adebf705ebc34ea04d7793", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx950_7.14.0-3_amd64.deb" ], @@ -4300,7 +4300,7 @@ "dependencies": [], "key": "amdrocm-dnn-host7.14_7.14.0-3_amd64", "name": "amdrocm-dnn-host7.14", - "sha256": "19341008efe6153a7c3ab4cc7d4159098f44fb57aa35b014b8a23bcdf438773d", + "sha256": "60b5881d4de4965e8c943505bdf3de644b4f654f95d82d4929ce3f81bd543296", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn-host7.14_7.14.0-3_amd64.deb" ], @@ -4311,7 +4311,7 @@ "dependencies": [], "key": "amdrocm-dnn7.14-gfx942_7.14.0-3_amd64", "name": "amdrocm-dnn7.14-gfx942", - "sha256": "53cb410c17bda3ec250b1942c45915f008b1b8bf6d1ffe27881dc3fd817ce190", + "sha256": "6dc1468b464c6bf6c34df884e2a5bdc4a313832e1c80152aa77b5c686a9cda36", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx942_7.14.0-3_amd64.deb" ], @@ -4322,7 +4322,7 @@ "dependencies": [], "key": "amdrocm-dnn7.14-gfx90a_7.14.0-3_amd64", "name": "amdrocm-dnn7.14-gfx90a", - "sha256": "364d71a34fab72e903a98e29f0c7ad1e0b8a59785b4200df8bec9fb072d56996", + "sha256": "1eb5accc0333896f9d4fb2903ffb7545b61c1c3618de8ee6d4a6676c4139b9c8", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx90a_7.14.0-3_amd64.deb" ], @@ -4333,7 +4333,7 @@ "dependencies": [], "key": "amdrocm-dnn7.14-gfx908_7.14.0-3_amd64", "name": "amdrocm-dnn7.14-gfx908", - "sha256": "f512008bae762a5e607872d1df43d42d7583c4e1d26aadebd75a852e6833a97c", + "sha256": "5996e133669830de9b1f6bc25b0426c8e844e9b4f181e057396821b76fcf2c51", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx908_7.14.0-3_amd64.deb" ], @@ -4344,7 +4344,7 @@ "dependencies": [], "key": "amdrocm-dnn7.14-gfx1201_7.14.0-3_amd64", "name": "amdrocm-dnn7.14-gfx1201", - "sha256": "d3d3f74e63493aa66e2e3ed3ba25aa334befe70d1cb8df20c06fc01a02e205af", + "sha256": "bfb6db6d3b202d2267180e7bd9ea408da0fdc7d6ce302b753cb52273222ccd58", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1201_7.14.0-3_amd64.deb" ], @@ -4355,7 +4355,7 @@ "dependencies": [], "key": "amdrocm-dnn7.14-gfx1200_7.14.0-3_amd64", "name": "amdrocm-dnn7.14-gfx1200", - "sha256": "0f4a6bf4785f434186eafc32303631e96fbf5879a9898b8ac192394ea80be9ba", + "sha256": "6d7ee796b578339d5446827d7a78acab2bb3c0638b859cde4c10386306afb38d", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1200_7.14.0-3_amd64.deb" ], @@ -4366,7 +4366,7 @@ "dependencies": [], "key": "amdrocm-dnn7.14-gfx1153_7.14.0-3_amd64", "name": "amdrocm-dnn7.14-gfx1153", - "sha256": "e34111409adf768e49e1074cdcfd9befdad4f1f15baabe09bc83bb62719d9ec8", + "sha256": "a0ac3315e9d8d2198030135110ee69b686cc06d489dd9e012e3d0c7c15da029a", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1153_7.14.0-3_amd64.deb" ], @@ -4377,7 +4377,7 @@ "dependencies": [], "key": "amdrocm-dnn7.14-gfx1152_7.14.0-3_amd64", "name": "amdrocm-dnn7.14-gfx1152", - "sha256": "ebfe98d8eda69e6e96728247fc05a945409c88f8d782fbb4dbc39626c2de617a", + "sha256": "1ce94ef4c1b6bbede2bc3baea52505522ad3608f4f56756c37c2773cc15d548f", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1152_7.14.0-3_amd64.deb" ], @@ -4388,7 +4388,7 @@ "dependencies": [], "key": "amdrocm-dnn7.14-gfx1151_7.14.0-3_amd64", "name": "amdrocm-dnn7.14-gfx1151", - "sha256": "a0bbb2c8bafbd44583854ed0cbd76078adcc60d8ed0b5405af99107dc6bca3b6", + "sha256": "b09507f24042c4b4c5bb7cc1f23598de9399f40d46c5db9d5e2eba720e980ec6", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1151_7.14.0-3_amd64.deb" ], @@ -4399,7 +4399,7 @@ "dependencies": [], "key": "amdrocm-dnn7.14-gfx1150_7.14.0-3_amd64", "name": "amdrocm-dnn7.14-gfx1150", - "sha256": "19fd6d95d7fd72bedc6469d03768bb8e80d4f03d20ba115bbfe1e3436493d813", + "sha256": "389e5a3c4d12b62084fbc2caaed91650c3ad90b15d531ec7a56d7264dc73a008", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1150_7.14.0-3_amd64.deb" ], @@ -4410,7 +4410,7 @@ "dependencies": [], "key": "amdrocm-dnn7.14-gfx1102_7.14.0-3_amd64", "name": "amdrocm-dnn7.14-gfx1102", - "sha256": "fa8a0f778d4a1cac0e5a35beda20eab50e5d09968be4d046d70e010f850613ae", + "sha256": "3e184a8ba6e6afeaca1ae8c5370bfe7b6fe90099d57746489098067523d16b47", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1102_7.14.0-3_amd64.deb" ], @@ -4421,7 +4421,7 @@ "dependencies": [], "key": "amdrocm-dnn7.14-gfx1101_7.14.0-3_amd64", "name": "amdrocm-dnn7.14-gfx1101", - "sha256": "cabe934788d36f37fb063433819a9a98c9d4b081398344fbaae0debe3a6c01ba", + "sha256": "b2c292835fb4c8250df22fd441bd677d4c195b93e1d1f77c013cbf5a82cd5675", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1101_7.14.0-3_amd64.deb" ], @@ -4432,7 +4432,7 @@ "dependencies": [], "key": "amdrocm-dnn7.14-gfx1100_7.14.0-3_amd64", "name": "amdrocm-dnn7.14-gfx1100", - "sha256": "0fb351aa31b7f3d217ad9da70689f7581531d6e1cb4171fbea53b67dfc76619a", + "sha256": "d8d0d13bfe0a0962081531acdcdbc523edca2dd663600b2764738ef8aac8a42f", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1100_7.14.0-3_amd64.deb" ], @@ -4443,7 +4443,7 @@ "dependencies": [], "key": "amdrocm-dnn7.14-gfx1030_7.14.0-3_amd64", "name": "amdrocm-dnn7.14-gfx1030", - "sha256": "634771d432604d5514a460babf13fb3270edef6225b0d56961d582d783bdb2a2", + "sha256": "ccf950df526b22358463fb4bfb8a315bc0df63ab64917bc258bc42a4c40b113f", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1030_7.14.0-3_amd64.deb" ], @@ -4915,7 +4915,7 @@ ], "key": "amdrocm-fft7.14_7.14.0-3_amd64", "name": "amdrocm-fft7.14", - "sha256": "9dd7c4e05ded2098254acf778d2d9e9a9274cd1bb80d86f00cde373797faf54a", + "sha256": "3e0f943fa7f864b57f0e4654d6263c5e8bcc43bf3c62add2fbadaa20520d7a21", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14_7.14.0-3_amd64.deb" ], @@ -4926,7 +4926,7 @@ "dependencies": [], "key": "amdrocm-fft7.14-gfx950_7.14.0-3_amd64", "name": "amdrocm-fft7.14-gfx950", - "sha256": "f098768c3a9dab1b2870e4ef0f4d38e6e40b3afd70d918e3ba02bbd58c99495b", + "sha256": "40682957167935c086ae8f7f30022e778a612fc6cf04e246c3b480109ccf7b1a", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx950_7.14.0-3_amd64.deb" ], @@ -4937,7 +4937,7 @@ "dependencies": [], "key": "amdrocm-fft-host7.14_7.14.0-3_amd64", "name": "amdrocm-fft-host7.14", - "sha256": "69fd1ad23c2b45af82595ccb779390e7edf88f34d4e029ed05f53a4f3040f7c5", + "sha256": "fb7b39883e51e25fbe9bd46aa4b76b089cd0edfea97be80e8631a85c6c70934d", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft-host7.14_7.14.0-3_amd64.deb" ], @@ -4948,7 +4948,7 @@ "dependencies": [], "key": "amdrocm-fft7.14-gfx942_7.14.0-3_amd64", "name": "amdrocm-fft7.14-gfx942", - "sha256": "0bd770f086eab2607a6503ac5fb5d140d5978ba5876a679f69fd46f120b893f0", + "sha256": "b141660c1db7ba6882fb35d5ffd4666d5c878a97fe078b80b721813d9249d2d3", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx942_7.14.0-3_amd64.deb" ], @@ -4959,7 +4959,7 @@ "dependencies": [], "key": "amdrocm-fft7.14-gfx90a_7.14.0-3_amd64", "name": "amdrocm-fft7.14-gfx90a", - "sha256": "bb3fefd71fe9ebdc0926dadb680206674247d22f8c981cb779dfe2b915b35f28", + "sha256": "7786d24549a0d698a7505d534b6d020d037e49ff481d937d83fcb1ae4d669c67", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx90a_7.14.0-3_amd64.deb" ], @@ -4970,7 +4970,7 @@ "dependencies": [], "key": "amdrocm-fft7.14-gfx908_7.14.0-3_amd64", "name": "amdrocm-fft7.14-gfx908", - "sha256": "3e9e6f25d32aa37e26fcea8c3768ee139be7c21d08b350cea7fbd7a067304b31", + "sha256": "bc5999397f7dab4a159d0dfe5272a7db73c7f3647353a79073f8f6cf335473ba", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx908_7.14.0-3_amd64.deb" ], @@ -4981,7 +4981,7 @@ "dependencies": [], "key": "amdrocm-fft7.14-gfx1250_7.14.0-3_amd64", "name": "amdrocm-fft7.14-gfx1250", - "sha256": "192be4897e6cc0ff63ca41b869f722d6271088e5eab862742ffe4c5cdba161b4", + "sha256": "a30b041906961c7ce8b90f2d0e03da94986d06e5374f59671e3323bf2ab38d4b", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1250_7.14.0-3_amd64.deb" ], @@ -4992,7 +4992,7 @@ "dependencies": [], "key": "amdrocm-fft7.14-gfx1201_7.14.0-3_amd64", "name": "amdrocm-fft7.14-gfx1201", - "sha256": "803e967a36ed13de102f9a682027df3b86e82ecef5dc01a90ee2afd24396ed04", + "sha256": "ab8018b742f52356e84c9662f2d620a39f802e757c79d34498847f52c78be1e2", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1201_7.14.0-3_amd64.deb" ], @@ -5003,7 +5003,7 @@ "dependencies": [], "key": "amdrocm-fft7.14-gfx1200_7.14.0-3_amd64", "name": "amdrocm-fft7.14-gfx1200", - "sha256": "c68b37108c9e28a35159565785a53d97f6e65e1bab50da6a3b44e625ef5f8404", + "sha256": "a43aa5568864e61f669abe0e96a4acfe02c6ada7b0adf7362bc0f288af85d7be", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1200_7.14.0-3_amd64.deb" ], @@ -5014,7 +5014,7 @@ "dependencies": [], "key": "amdrocm-fft7.14-gfx1153_7.14.0-3_amd64", "name": "amdrocm-fft7.14-gfx1153", - "sha256": "c18c459194e38022b130dd3b36d08858fa6016a0266f0f38b869654beca3b8be", + "sha256": "75e65dca76d758b06a8804bb6dd0cda494f5a09bcc336efb1c324ccd99d0e003", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1153_7.14.0-3_amd64.deb" ], @@ -5025,7 +5025,7 @@ "dependencies": [], "key": "amdrocm-fft7.14-gfx1152_7.14.0-3_amd64", "name": "amdrocm-fft7.14-gfx1152", - "sha256": "6f0603072a41775facda20c7e8bed77fef65d0fc5c29cf30bb2610bc801fa789", + "sha256": "dcde969b559140d8eba42459e4a500b372767f4c9951d6986841d2be9c7d23fd", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1152_7.14.0-3_amd64.deb" ], @@ -5036,7 +5036,7 @@ "dependencies": [], "key": "amdrocm-fft7.14-gfx1151_7.14.0-3_amd64", "name": "amdrocm-fft7.14-gfx1151", - "sha256": "838583cc6141c1e03e737cce5a3dd7ec769b017353c0599ebd2dde8f1ed0e42d", + "sha256": "152fea84cdae050455d94380e91b4f890b774f87970460cd66e0badc22262e7c", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1151_7.14.0-3_amd64.deb" ], @@ -5047,7 +5047,7 @@ "dependencies": [], "key": "amdrocm-fft7.14-gfx1150_7.14.0-3_amd64", "name": "amdrocm-fft7.14-gfx1150", - "sha256": "fb6f1145c9cb44d41114b83600974b853982589ac4d0a819e3e16ad0bbc77784", + "sha256": "55721427c464921300572ef3620c237eef35c209aa407b2b953a661a2e8a3c3b", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1150_7.14.0-3_amd64.deb" ], @@ -5058,7 +5058,7 @@ "dependencies": [], "key": "amdrocm-fft7.14-gfx1103_7.14.0-3_amd64", "name": "amdrocm-fft7.14-gfx1103", - "sha256": "7f8c2ea4a9f49ebc55604d8e27df6fa10874c3767831c71cb31d3b98b26416dc", + "sha256": "d2355cabc2e1b4ee0c13b1141c9131f3c95e796d3871b063a2cb71b892f701e4", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1103_7.14.0-3_amd64.deb" ], @@ -5069,7 +5069,7 @@ "dependencies": [], "key": "amdrocm-fft7.14-gfx1102_7.14.0-3_amd64", "name": "amdrocm-fft7.14-gfx1102", - "sha256": "80142da4ae1481339589db8c07f85b0618af98d73351eb3dc6561f1c073cdd22", + "sha256": "64b3d713c0eff13dc9142acea7f8bda2635cd730ab45edcb05b1a562b52cfc48", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1102_7.14.0-3_amd64.deb" ], @@ -5080,7 +5080,7 @@ "dependencies": [], "key": "amdrocm-fft7.14-gfx1101_7.14.0-3_amd64", "name": "amdrocm-fft7.14-gfx1101", - "sha256": "183085a543f9b6c3a5af9299f1b142d5edc7c0b54de081e35ebe7ee87b178a0a", + "sha256": "df5c62070a5a79ed90bf120dc88f43668620e2ad9d774fbab183f027a65804db", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1101_7.14.0-3_amd64.deb" ], @@ -5091,7 +5091,7 @@ "dependencies": [], "key": "amdrocm-fft7.14-gfx1100_7.14.0-3_amd64", "name": "amdrocm-fft7.14-gfx1100", - "sha256": "323dbfea5dd39d8240aa42a05d9b8d58bed81014bf7ee469b4b363e33e692901", + "sha256": "e391f137ad41548f0dc16e13e8605a52f3b5a1e9f2da772bb08c07fb8ed38ae1", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1100_7.14.0-3_amd64.deb" ], @@ -5102,7 +5102,7 @@ "dependencies": [], "key": "amdrocm-fft7.14-gfx1036_7.14.0-3_amd64", "name": "amdrocm-fft7.14-gfx1036", - "sha256": "610f90101bb496f6bf4f902b51852d823fdbc55941e2cfb489e167d2e7e40623", + "sha256": "0efd59d401afaa6c25df4ba18649053f91a3f9d7f8ca5baf023a063882fdaffb", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1036_7.14.0-3_amd64.deb" ], @@ -5113,7 +5113,7 @@ "dependencies": [], "key": "amdrocm-fft7.14-gfx1035_7.14.0-3_amd64", "name": "amdrocm-fft7.14-gfx1035", - "sha256": "0151828a3521b62c6a160e931e407b6e44ca50ec595dc03efa7d430542c6567c", + "sha256": "3ad1e464c73d22ae971ed647337f9854cd13540bcdb2aefdf542551371d9ba01", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1035_7.14.0-3_amd64.deb" ], @@ -5124,7 +5124,7 @@ "dependencies": [], "key": "amdrocm-fft7.14-gfx1034_7.14.0-3_amd64", "name": "amdrocm-fft7.14-gfx1034", - "sha256": "0d38e34fb0a839131f1e4556be30ec3eb36cb3d36676bedb44176785a3f1bc45", + "sha256": "733de05e2e6b4100f4fe693fa6508ceca84c57e576a177e348f0b27bef7325e3", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1034_7.14.0-3_amd64.deb" ], @@ -5135,7 +5135,7 @@ "dependencies": [], "key": "amdrocm-fft7.14-gfx1033_7.14.0-3_amd64", "name": "amdrocm-fft7.14-gfx1033", - "sha256": "4d6c786de4b29fbe49400559942344e1cb2ddb394cdbbe216f8ae2404c7ed95f", + "sha256": "60a9886a6735887fc0ce51267928d4990e227e2a6c517cce90714f200c91559a", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1033_7.14.0-3_amd64.deb" ], @@ -5146,7 +5146,7 @@ "dependencies": [], "key": "amdrocm-fft7.14-gfx1032_7.14.0-3_amd64", "name": "amdrocm-fft7.14-gfx1032", - "sha256": "f41f876967f28e36a8b105778d44dc327bd8f5b6d55b7d3d1af6c2572dc067fe", + "sha256": "4b2e74a1c35d5f0b3ef83165e6e3ee94030a759980dcec39275ffccf0e79cbc9", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1032_7.14.0-3_amd64.deb" ], @@ -5157,7 +5157,7 @@ "dependencies": [], "key": "amdrocm-fft7.14-gfx1031_7.14.0-3_amd64", "name": "amdrocm-fft7.14-gfx1031", - "sha256": "09860c022cf6f27045b0ffb05c80dac7bd1bfeb059c51347104c6f34e273bed6", + "sha256": "5922a31921a36d726e4ef2e51bc78348712c4bd7e4e375031216bbf28ffbf657", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1031_7.14.0-3_amd64.deb" ], @@ -5168,7 +5168,7 @@ "dependencies": [], "key": "amdrocm-fft7.14-gfx1030_7.14.0-3_amd64", "name": "amdrocm-fft7.14-gfx1030", - "sha256": "eae84444e0efce3b635f8ad4e2ae281768c2bd7571e20763a996e90b23feabcd", + "sha256": "cdb09413498d8ccd143f600dd0f4263d9f8c0ac16d4c2a083cf97a4a438ef7f6", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1030_7.14.0-3_amd64.deb" ], @@ -5179,7 +5179,7 @@ "dependencies": [], "key": "amdrocm-fft7.14-gfx1012_7.14.0-3_amd64", "name": "amdrocm-fft7.14-gfx1012", - "sha256": "8cee80ba00d14c89517a42bfba30bd951282beea1def52c8a0fff67320c95daf", + "sha256": "ad47313cacbe4f40288dc4ab7ae91d33217c202873e7ebc70c0120e46e619a12", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1012_7.14.0-3_amd64.deb" ], @@ -5190,7 +5190,7 @@ "dependencies": [], "key": "amdrocm-fft7.14-gfx1011_7.14.0-3_amd64", "name": "amdrocm-fft7.14-gfx1011", - "sha256": "a28c56a34213c95112a947c646ed7ceeab3ef243fa535fa681859301c6def686", + "sha256": "27aba1514f9186f58e0a7ab2da97cb6387c4aaa8ec17b54468eff9681110764e", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1011_7.14.0-3_amd64.deb" ], @@ -5201,7 +5201,7 @@ "dependencies": [], "key": "amdrocm-fft7.14-gfx1010_7.14.0-3_amd64", "name": "amdrocm-fft7.14-gfx1010", - "sha256": "cd38bdbe5e0fb0d20dbc25561426bab914c58cb5e8e4ea143321b22f2e26cb44", + "sha256": "e3a018100c31f7dcbcf099a6443814c1a0e2e8377672d2cab72bc3c2d075ab59", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1010_7.14.0-3_amd64.deb" ], @@ -5663,7 +5663,7 @@ ], "key": "amdrocm-rccl7.14_7.14.0-3_amd64", "name": "amdrocm-rccl7.14", - "sha256": "f872534a2a7e87f2b6d33f82aa3f6a7f7f1ba7c126129ef0a10efb19c8c996b0", + "sha256": "ab7b72c172252d259bebefb2c325034db4b0e46ecde0191ce50f447c8beeac41", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14_7.14.0-3_amd64.deb" ], @@ -5674,7 +5674,7 @@ "dependencies": [], "key": "amdrocm-rccl7.14-gfx950_7.14.0-3_amd64", "name": "amdrocm-rccl7.14-gfx950", - "sha256": "5e8f399ffe83df15c2d92f7c992d15b274dc93c68b9ff7a2558d5e7b032a6fcb", + "sha256": "aa736e0e6b1e32104e80794f3d555828f01c9b28c90c4ca2d4fac0647a8a4565", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx950_7.14.0-3_amd64.deb" ], @@ -5685,7 +5685,7 @@ "dependencies": [], "key": "amdrocm-rccl-host7.14_7.14.0-3_amd64", "name": "amdrocm-rccl-host7.14", - "sha256": "43f64145318b9fc02c2a76e43f2bc529fc4ea768f24a3b24a7a11c98693c8214", + "sha256": "0d418c1f2a5f15d329e26458a39a395d8236b10cc16397922e1df61e699b90d6", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl-host7.14_7.14.0-3_amd64.deb" ], @@ -5696,7 +5696,7 @@ "dependencies": [], "key": "amdrocm-rccl7.14-gfx942_7.14.0-3_amd64", "name": "amdrocm-rccl7.14-gfx942", - "sha256": "4d5feafba0fd901d2416cc8c7f4970c4172f332880c334ab2cfb225aeb32c5d7", + "sha256": "00e33c9f72e853a033641d7d46c0bbf94a7b546eac586fafbf86dab19ce96db4", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx942_7.14.0-3_amd64.deb" ], @@ -5707,7 +5707,7 @@ "dependencies": [], "key": "amdrocm-rccl7.14-gfx90a_7.14.0-3_amd64", "name": "amdrocm-rccl7.14-gfx90a", - "sha256": "2a00389b4d7f01830f464a8f55fe9bf24a2a7238c1109fcf46d2d92a83716d6a", + "sha256": "54d7d090219946a7631596543582eaa9ad6ffd35a4f56ee86a2ba774628b522e", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx90a_7.14.0-3_amd64.deb" ], @@ -5718,7 +5718,7 @@ "dependencies": [], "key": "amdrocm-rccl7.14-gfx908_7.14.0-3_amd64", "name": "amdrocm-rccl7.14-gfx908", - "sha256": "85262a9f0bcf3356748a970fa78ae7c4747b5a235636607dd09a4dc37b4b2940", + "sha256": "282008f6ea550e218c93def030e79e890932f9c8d1886ba993071a7896b8ccc9", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx908_7.14.0-3_amd64.deb" ], @@ -5729,7 +5729,7 @@ "dependencies": [], "key": "amdrocm-rccl7.14-gfx1250_7.14.0-3_amd64", "name": "amdrocm-rccl7.14-gfx1250", - "sha256": "4a73a6de53f819b066397e1bbcb734fe7f7c0c515eaa57046f286c4efd55e05d", + "sha256": "8013142575c5fad8ed411911e1574cfe165857eded83be2a220478e64dbe8e98", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1250_7.14.0-3_amd64.deb" ], @@ -5740,7 +5740,7 @@ "dependencies": [], "key": "amdrocm-rccl7.14-gfx1201_7.14.0-3_amd64", "name": "amdrocm-rccl7.14-gfx1201", - "sha256": "a699ddb683d8a0c640f021e36da256843030281fee0fe538756386113bf61699", + "sha256": "771dc4beedc633de7588849d7c6c2bded9924bb72b5b1ff7e1bdc7fb999beb27", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1201_7.14.0-3_amd64.deb" ], @@ -5751,7 +5751,7 @@ "dependencies": [], "key": "amdrocm-rccl7.14-gfx1200_7.14.0-3_amd64", "name": "amdrocm-rccl7.14-gfx1200", - "sha256": "27a35cf5734dc2602c8f68692bfb7434b76eac8134ab54bfb04a5d6fd61ecf13", + "sha256": "2a4d1d6f12da39ae3bc57d5cdab56e9dc6d28ba335fe532c46bcd6fe9588b1ba", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1200_7.14.0-3_amd64.deb" ], @@ -5762,7 +5762,7 @@ "dependencies": [], "key": "amdrocm-rccl7.14-gfx1151_7.14.0-3_amd64", "name": "amdrocm-rccl7.14-gfx1151", - "sha256": "bdfb7c11cf24d9d94dc5aac2e516ac463046622dea4659e26fbc09f1c1d0459c", + "sha256": "2cdf93160d63d3e329399de716b9261b9c9b6f776b09ffbe7bab4746f7d86ed3", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1151_7.14.0-3_amd64.deb" ], @@ -5773,7 +5773,7 @@ "dependencies": [], "key": "amdrocm-rccl7.14-gfx1102_7.14.0-3_amd64", "name": "amdrocm-rccl7.14-gfx1102", - "sha256": "9fb0ff90c45be824c48b0ad8e91c11ec6300034a584119c7dbf263e2ebda7f3f", + "sha256": "d8782810acb321f94c42a6b0c6334d307917ca2acbbfaeaa0244a563ec6165f5", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1102_7.14.0-3_amd64.deb" ], @@ -5784,7 +5784,7 @@ "dependencies": [], "key": "amdrocm-rccl7.14-gfx1101_7.14.0-3_amd64", "name": "amdrocm-rccl7.14-gfx1101", - "sha256": "ecabf24eb13f23702fc4b9412fbcc8fe778d98c19a55d6e3e932d7541f8cfd8f", + "sha256": "3c1ebeb61a1e8921c983d9432973ce83d9f919b82e3d8b6dfe8999662c605b6e", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1101_7.14.0-3_amd64.deb" ], @@ -5795,7 +5795,7 @@ "dependencies": [], "key": "amdrocm-rccl7.14-gfx1100_7.14.0-3_amd64", "name": "amdrocm-rccl7.14-gfx1100", - "sha256": "a2be1b2d47dd46dd389d282dedc2b1b78a94657bc02699e2bbd6ddd25de2d8c8", + "sha256": "ea6da253d619f8e56ea573b980133b3bbd4dfa70e6dd58b339326d00b749da6f", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1100_7.14.0-3_amd64.deb" ], @@ -5806,7 +5806,7 @@ "dependencies": [], "key": "amdrocm-rccl7.14-gfx1036_7.14.0-3_amd64", "name": "amdrocm-rccl7.14-gfx1036", - "sha256": "3265c4d45b42aabc98775015eedb3cc0f14396fa413f24f754224bc391c2a38a", + "sha256": "0811c8f1212247711abd50c61b818446926226ea17901ee05ad5edf07ad87916", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1036_7.14.0-3_amd64.deb" ], @@ -5817,7 +5817,7 @@ "dependencies": [], "key": "amdrocm-rccl7.14-gfx1035_7.14.0-3_amd64", "name": "amdrocm-rccl7.14-gfx1035", - "sha256": "77f2f1d81c52a7168c9cd85f145987077565a289377814ada074d69b5f2427f9", + "sha256": "f4a7924a8032d4cc271541df68ef679e22b6c0ac4643e1d77d7ea742bed09752", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1035_7.14.0-3_amd64.deb" ], @@ -5828,7 +5828,7 @@ "dependencies": [], "key": "amdrocm-rccl7.14-gfx1034_7.14.0-3_amd64", "name": "amdrocm-rccl7.14-gfx1034", - "sha256": "630acf3eae8ec24edf029012afa060e8242bfb059b4affa4c60188679cf2db64", + "sha256": "8f38d0016c32e18bb254ef97ce597c10a203f471788fa960b946cd10d277397c", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1034_7.14.0-3_amd64.deb" ], @@ -5839,7 +5839,7 @@ "dependencies": [], "key": "amdrocm-rccl7.14-gfx1033_7.14.0-3_amd64", "name": "amdrocm-rccl7.14-gfx1033", - "sha256": "56f154f9e9fd2ddaa07f447e6487bd83cca5478aa136686f73c3e1785a3299c9", + "sha256": "d0a8a971fba23dac720b87afc2d6a8c2f4d5df3c92ac6071708bb9a0216d6d86", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1033_7.14.0-3_amd64.deb" ], @@ -5850,7 +5850,7 @@ "dependencies": [], "key": "amdrocm-rccl7.14-gfx1032_7.14.0-3_amd64", "name": "amdrocm-rccl7.14-gfx1032", - "sha256": "76e368957e1990585754d6d08c17fdb0d60e344382029cb52808862e30b81570", + "sha256": "d5f46671549deb3682ca6f15049d4ec7cd500b19364c6e40d8d455a62927718e", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1032_7.14.0-3_amd64.deb" ], @@ -5861,7 +5861,7 @@ "dependencies": [], "key": "amdrocm-rccl7.14-gfx1031_7.14.0-3_amd64", "name": "amdrocm-rccl7.14-gfx1031", - "sha256": "bcdce1101be547acb05026941fb44656bfd023e8b357b58e4cd2a5e24c4fc851", + "sha256": "6139cadb284ff1b6d9267fd2780196265c1ff7ee9909d534e7f797fdaf1b01ac", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1031_7.14.0-3_amd64.deb" ], @@ -5872,7 +5872,7 @@ "dependencies": [], "key": "amdrocm-rccl7.14-gfx1030_7.14.0-3_amd64", "name": "amdrocm-rccl7.14-gfx1030", - "sha256": "ed957d5a57e00950c46878bfec5dd7a92fcf547840132b0071eb5facd1bea5d9", + "sha256": "e0257ac0b24a1ea20d78272546cb05fed57342f58d331e128565e658155393a0", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1030_7.14.0-3_amd64.deb" ], @@ -5883,7 +5883,7 @@ "dependencies": [], "key": "amdrocm-rccl7.14-gfx1012_7.14.0-3_amd64", "name": "amdrocm-rccl7.14-gfx1012", - "sha256": "8fa4e40c0a0c5c32b0f71fbf8343fd465940c39f1843c93fe4501abcbd2b3dcf", + "sha256": "060f6cfde9f945271f4dc7bdad87425ff16ef0edd55d172e04950a4a9f112100", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1012_7.14.0-3_amd64.deb" ], @@ -5894,7 +5894,7 @@ "dependencies": [], "key": "amdrocm-rccl7.14-gfx1011_7.14.0-3_amd64", "name": "amdrocm-rccl7.14-gfx1011", - "sha256": "93ceea63db909a8f6ab2435d057b7bc7dbf22575c4b4eb26da38f268e193d18e", + "sha256": "bb13e8800079678a4e73871fd34501dbd75c62e4a468b0ef47a21185a19d69bf", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1011_7.14.0-3_amd64.deb" ], @@ -5905,7 +5905,7 @@ "dependencies": [], "key": "amdrocm-rccl7.14-gfx1010_7.14.0-3_amd64", "name": "amdrocm-rccl7.14-gfx1010", - "sha256": "b67d6fade488b33cce5a69d66d01940bc0f61632fab5624d21c95b7f824702ef", + "sha256": "8d93d6320a5b74e6933cdd523ce2994c5a668fb47c789533fbec3cef3fb6baff", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1010_7.14.0-3_amd64.deb" ], @@ -6252,7 +6252,7 @@ ], "key": "amdrocm-runtime-dev7.14_7.14.0-3_amd64", "name": "amdrocm-runtime-dev7.14", - "sha256": "443661803de9a556c6cdba9adbf8fd9a1bfd8dadbea8b31ea1e0a9236e97f5c9", + "sha256": "2b3f822939a87291ccf19670c3a0a20472df6aa4ae3042472988afa7de552e8d", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-runtime-dev7.14_7.14.0-3_amd64.deb" ], @@ -6263,7 +6263,7 @@ "dependencies": [], "key": "amdrocm-llvm-dev7.14_7.14.0-3_amd64", "name": "amdrocm-llvm-dev7.14", - "sha256": "2a99eefb2648d2fa02a6b40ca7b5e7a8426e2f4db86d3fb0525dfefd2eebc404", + "sha256": "d5319c83a26687ece983b03da6d68ca6b3dcf3f20313c5c93efd7fd9df9b7de9", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-llvm-dev7.14_7.14.0-3_amd64.deb" ], @@ -7005,7 +7005,7 @@ ], "key": "amdrocm-solver7.14_7.14.0-3_amd64", "name": "amdrocm-solver7.14", - "sha256": "a871ff1d8745d277275b6f989892a854cd1567fcac3c750f27b017404750162a", + "sha256": "d2440e2d0c746724ef52a61f1f5301bf08eec77078ddf4b732a596502ca7e8f9", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14_7.14.0-3_amd64.deb" ], @@ -7747,7 +7747,7 @@ ], "key": "amdrocm-sparse7.14_7.14.0-3_amd64", "name": "amdrocm-sparse7.14", - "sha256": "674509967d9d055791bb05f1a1afb8c835680142921ce2665db9dcd873d20a46", + "sha256": "e2ebd0788d98eb6cef4362dcecf43066f4bef3a07ce98f41e0b7549b4fd93f68", "urls": [ "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14_7.14.0-3_amd64.deb" ], From bdbfeb19ec62c62efd505875f49aa08973c8bd10 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 30 Jul 2026 11:29:42 +0200 Subject: [PATCH 601/881] zml.ops: refactor while loop API (#686) removes `anytype` from `zml.ops.while`, helping with type checking and simplifying a bit the logic. A `While` struct needs * a State struct definition that captures all the loop state that will change on each iteration * a `cond(While, State) Tensor` function returning a scalar boolean tensor * a `body(While, State) State` function that preserves the state shapes. example: ``` const While = struct { // While struct can contain tensor and non-tensor fields. limit: i64, step: Tensor, pub const State = struct { i: zml.Tensor, sum: zml.Tensor, }; pub fn cond(self: @This(), state: State) Tensor { return state.i.cmp(.LT, Tensor.scalar(self.limit, .i64)); } pub fn body(self: @This(), state: State) State { return .{ .i = state.i.add(self.step), .sum = state.sum.add(self.step), }; } fn forward(i: Tensor, sum: Tensor) State { return zml.ops.@"while"( @This(), .{ .limit = 10, .step = .scalar(1, .i64) }, .{ .i = i, .sum = sum }, ); } }; ``` The code is a bit more verbose for simple example, but for GatedDeltaNet it's equivalent, and I have also used it for paged attention. --- stdx/fmt.zig | 8 ++- zml/meta.zig | 25 +++++++ zml/nn.zig | 189 +++++++++++++++++++------------------------------ zml/ops.zig | 186 ++++++++++++++++++++++++++---------------------- zml/tensor.zig | 4 ++ 5 files changed, 208 insertions(+), 204 deletions(-) diff --git a/stdx/fmt.zig b/stdx/fmt.zig index 16dd843a..b176a099 100644 --- a/stdx/fmt.zig +++ b/stdx/fmt.zig @@ -63,7 +63,13 @@ pub fn formatBool(value: bool, spec: std.fmt.Number, writer: *std.Io.Writer) !vo pub fn formatAny(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { var buf: [48]u8 = undefined; const T = @TypeOf(value); - const fmt = if (@hasDecl(T, "formatNumber")) "{d}" else "{f}"; + const fmt = switch (@typeInfo(T)) { + .@"struct", .@"union", .@"enum", .@"opaque" => if (@hasDecl(T, "formatNumber")) + "{d}" + else + "{f}", + else => "{f}", + }; const s = std.fmt.bufPrint(&buf, fmt, .{value}) catch blk: { buf[45..].* = "...".*; diff --git a/zml/meta.zig b/zml/meta.zig index 4f9402ee..dcc5c546 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -748,6 +748,31 @@ pub fn collectBuf(func: anytype, func_ctx: _CollectCtx(func), obj: anytype, out: std.debug.assert(context.idx == context.out.len); } +/// Finds all X in the given object, and write their pointers into an arraylist. +pub fn collectPtrs( + T: type, + allocator: std.mem.Allocator, + obj: anytype, +) std.mem.Allocator.Error![]*const T { + const CollectPtrCtx = struct { + allocator: std.mem.Allocator, + out: std.ArrayList(*const T) = .empty, + oom: bool = false, + + fn cb(ctx: *@This(), val: *const T) void { + if (ctx.oom) return; + ctx.out.append(ctx.allocator, val) catch { + ctx.oom = true; + }; + } + }; + var context = CollectPtrCtx{ .allocator = allocator }; + visit(CollectPtrCtx.cb, &context, obj); + if (context.oom) return error.OutOfMemory; + + return context.out.toOwnedSlice(allocator); +} + fn _CollectCtx(func: anytype) type { const params = @typeInfo(@TypeOf(func)).@"fn".params; if (params.len == 1) return void; diff --git a/zml/nn.zig b/zml/nn.zig index 295028c6..949063b9 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -1356,83 +1356,95 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { } pub const GatedDeltaNet = struct { + /// Query tensor .{ .s, .h, .k }. + queries: Tensor, + /// Key tensor .{ .s, .h, .k }. + keys: Tensor, + /// Value tensor .{ .s, .h, .v }. + values: Tensor, + /// Forget gate .{ .s, .h }. + alphas: Tensor, + /// Delta gate .{ .s, .h }. + betas: Tensor, + pub const State = struct { /// Per-head recurrent state with shape .{ .h, .v, .k }. s: Tensor, + /// Sequence of outputs with shape .{ .s, .h, .v }. + outputs: Tensor, + + /// Current step, scalar + step: Tensor, }; - pub const StepInputs = struct { - /// Query tensor with shape .{ .h, .k }. - q: Tensor, - /// Key tensor with shape .{ .h, .k }. - k: Tensor, - /// Value tensor with shape .{ .h, .v }. - v: Tensor, - /// Forget gate with shape .{ .h }. - alpha: Tensor, - /// Delta gate with shape .{ .h }. - beta: Tensor, - }; - - pub const StepOutput = struct { - state: State, - output: Tensor, + pub const Input = struct { + s: Tensor, }; pub const Output = struct { + /// Per-head recurrent state with shape .{ .h, .v, .k }. + state: Input, /// Sequence of outputs with shape .{ .s, .h, .v }. outputs: Tensor, - state: State, }; + pub fn cond(gdn: GatedDeltaNet, state: State) Tensor { + return state.step.cmp(.LT, .scalar(gdn.queries.dim(.s), .i32)); + } + + /// Single-step recurrent update for Gated Delta Net. + pub fn body(gdn: GatedDeltaNet, state: State) State { + // q, k: .{ .h, .k } + const q = sliceStep(gdn.queries, state.step); + const k = sliceStep(gdn.keys, state.step); + // v: .{ .h, .v } + const v = sliceStep(gdn.values, state.step); + // alpha, beta: .{ .h } + const alpha = sliceStep(gdn.alphas, state.step); + const beta = sliceStep(gdn.betas, state.step); + + const hvk = state.s.shape(); + const hv = hvk.drop(.k); + + const v_hat = state.s.dot(k, .k).mul(alpha.broad(hv)); + const delta = v.sub(v_hat).mul(beta.broad(hv)); + const delta_k = delta.broad(hvk).mul(k.broad(hvk)); + const s_new = state.s.mul(alpha.broad(hvk)).add(delta_k); + const y = s_new.dot(q, .k); + + return .{ + .s = s_new, + .outputs = state.outputs.dynamicUpdateSlice(.{ .s = state.step }, y), + .step = state.step.addConstant(1), + }; + } + fn sliceStep(input: Tensor, step_: Tensor) Tensor { return input.dynamicSlice(.{ .s = Tensor.DynSlice{ .start = step_, .len = 1 } }).squeeze(.s); } - fn validateStep(state: State, inputs: StepInputs) void { + fn validateInitialState(gdn: GatedDeltaNet, state: State) void { const err_template = "GatedDeltaNet.step(state: {f}, q: {f}, k: {f}, v: {f}, alpha: {f}, beta: {f}) is invalid ! "; - const err_args = .{ state.s, inputs.q, inputs.k, inputs.v, inputs.alpha, inputs.beta }; + const err_args = .{ state.s, gdn.queries, gdn.keys, gdn.values, gdn.alphas, gdn.betas }; stdx.debug.assert(state.s.shape().hasTags(.{ .h, .v, .k }), err_template ++ "state.s is missing tags {{.h, .v, .k}}", err_args); - stdx.debug.assert(inputs.q.shape().hasTags(.{ .h, .k }), err_template ++ "q is missing tags {{.h, .k}}", err_args); - stdx.debug.assert(inputs.k.shape().hasTags(.{ .h, .k }), err_template ++ "k is missing tags {{.h, .k}}", err_args); - stdx.debug.assert(inputs.v.shape().hasTags(.{ .h, .v }), err_template ++ "v is missing tags {{.h, .v}}", err_args); - stdx.debug.assert(inputs.alpha.shape().hasTags(.{.h}), err_template ++ "alpha is missing tag {{.h}}", err_args); - stdx.debug.assert(inputs.beta.shape().hasTags(.{.h}), err_template ++ "beta is missing tag {{.h}}", err_args); + stdx.debug.assert(gdn.queries.shape().hasTags(.{ .s, .h, .k }), err_template ++ "q is missing tags {{.h, .k}}", err_args); + stdx.debug.assert(gdn.keys.shape().hasTags(.{ .s, .h, .k }), err_template ++ "k is missing tags {{.h, .k}}", err_args); + stdx.debug.assert(gdn.values.shape().hasTags(.{ .s, .h, .v }), err_template ++ "v is missing tags {{.h, .v}}", err_args); + stdx.debug.assert(gdn.alphas.shape().hasTags(.{ .s, .h }), err_template ++ "alphas is missing tag {{.h}}", err_args); + stdx.debug.assert(gdn.betas.shape().hasTags(.{ .s, .h }), err_template ++ "betas is missing tag {{.h}}", err_args); - _ = collectDims(.{.h}, &.{ state.s, inputs.q, inputs.k, inputs.v, inputs.alpha, inputs.beta }, .strict) catch { + _ = collectDims(.{.h}, &.{ state.s, gdn.queries, gdn.keys, gdn.values, gdn.alphas, gdn.betas }, .strict) catch { stdx.debug.panic(err_template ++ "head dimensions are inconsistent.", err_args); }; - _ = collectDims(.{.k}, &.{ state.s, inputs.q, inputs.k }, .strict) catch { + _ = collectDims(.{.k}, &.{ state.s, gdn.queries, gdn.keys }, .strict) catch { stdx.debug.panic(err_template ++ "key dimensions are inconsistent.", err_args); }; - _ = collectDims(.{.v}, &.{ state.s, inputs.v }, .strict) catch { + _ = collectDims(.{.v}, &.{ state.s, gdn.values }, .strict) catch { stdx.debug.panic(err_template ++ "value dimensions are inconsistent.", err_args); }; } - /// Single-step recurrent update for Gated Delta Net. - /// - /// Shapes: - /// - `state.s`: .{ .h, .v, .k } - /// - `inputs.q`, `inputs.k`: .{ .h, .k } - /// - `inputs.v`: .{ .h, .v } - /// - `inputs.alpha`, `inputs.beta`: .{ .h } - pub fn step(state: State, inputs: StepInputs) StepOutput { - validateStep(state, inputs); - - const v_hat = state.s.dot(inputs.k, .k).mul(inputs.alpha.insertAxes(.last, .{.v})); - const delta = inputs.v.sub(v_hat).mul(inputs.beta.insertAxes(.last, .{.v})); - const delta_k = delta.insertAxes(.last, .{.k}).broad(state.s.shape()).mul(inputs.k.insertAxes(.k, .{.v}).broad(state.s.shape())); - const s_new = state.s.mul(inputs.alpha.insertAxes(.last, .{ .v, .k })).add(delta_k); - const y = s_new.dot(inputs.q, .k); - - return .{ - .state = .{ .s = s_new }, - .output = y, - }; - } - /// Processes a sequence with Gated Delta Net recurrence using `stablehlo.while`. /// /// Shapes: @@ -1446,83 +1458,26 @@ pub const GatedDeltaNet = struct { values: Tensor, alphas: Tensor, betas: Tensor, - initial_state: State, + initial_state: Input, ) Output { - const err_template = "GatedDeltaNet.forward(queries: {f}, keys: {f}, values: {f}, alphas: {f}, betas: {f}, state: {f}) is invalid ! "; - const err_args = .{ queries, keys, values, alphas, betas, initial_state.s }; - - stdx.debug.assert(queries.shape().hasTags(.{ .s, .h, .k }), err_template ++ "queries is missing tags {{.s, .h, .k}}", err_args); - stdx.debug.assert(keys.shape().hasTags(.{ .s, .h, .k }), err_template ++ "keys is missing tags {{.s, .h, .k}}", err_args); - stdx.debug.assert(values.shape().hasTags(.{ .s, .h, .v }), err_template ++ "values is missing tags {{.s, .h, .v}}", err_args); - stdx.debug.assert(alphas.shape().hasTags(.{ .s, .h }), err_template ++ "alphas is missing tags {{.s, .h}}", err_args); - stdx.debug.assert(betas.shape().hasTags(.{ .s, .h }), err_template ++ "betas is missing tags {{.s, .h}}", err_args); - stdx.debug.assert(initial_state.s.shape().hasTags(.{ .h, .v, .k }), err_template ++ "initial_state.s is missing tags {{.h, .v, .k}}", err_args); - - _ = collectDims(.{ .s, .h }, &.{ queries, keys, values, alphas, betas }, .strict) catch { - stdx.debug.panic(err_template ++ "sequence/head dimensions are inconsistent.", err_args); - }; - _ = collectDims(.{.k}, &.{ queries, keys, initial_state.s }, .strict) catch { - stdx.debug.panic(err_template ++ "key dimensions are inconsistent.", err_args); - }; - _ = collectDims(.{.v}, &.{ values, initial_state.s }, .strict) catch { - stdx.debug.panic(err_template ++ "value dimensions are inconsistent.", err_args); - }; - - const WhileContext = struct { - queries: Tensor, - keys: Tensor, - values: Tensor, - alphas: Tensor, - betas: Tensor, - seq_len: Tensor, - }; - const while_context: WhileContext = .{ + const gdn: GatedDeltaNet = .{ .queries = queries, .keys = keys, .values = values, .alphas = alphas, .betas = betas, - .seq_len = Tensor.scalar(queries.dim(.s), .i32), }; - - const Local = struct { - fn cond(step_: Tensor, _: Tensor, _: Tensor, ctx: WhileContext) Tensor { - return step_.cmp(.LT, ctx.seq_len); - } - - fn body(step_: Tensor, s_prev: Tensor, outputs_prev: Tensor, ctx: WhileContext) [3]Tensor { - const step_out = step( - .{ .s = s_prev }, - .{ - .q = sliceStep(ctx.queries, step_), - .k = sliceStep(ctx.keys, step_), - .v = sliceStep(ctx.values, step_), - .alpha = sliceStep(ctx.alphas, step_), - .beta = sliceStep(ctx.betas, step_), - }, - ); - const outputs_new = outputs_prev.dynamicUpdateSlice(.{ .s = step_ }, step_out.output); - - return .{ - step_.add(Tensor.scalar(1, .i32)), - step_out.state.s, - outputs_new, - }; - } + const state: GatedDeltaNet.State = .{ + .step = .scalar(0, .i32), + .s = initial_state.s, + .outputs = .zeroes(values.shape()), }; - - const step0 = Tensor.scalar(0, .i32); - const outputs0 = Tensor.zeroes(values.shape()); - const loop_state = ops.@"while"( - .{ step0, initial_state.s, outputs0 }, - Local.cond, - Local.body, - .{while_context}, - ); + validateInitialState(gdn, state); + const final_state = ops.@"while"(GatedDeltaNet, gdn, state); return .{ - .outputs = loop_state[2], - .state = .{ .s = loop_state[1] }, + .outputs = final_state.outputs, + .state = .{ .s = final_state.s }, }; } }; @@ -1541,7 +1496,7 @@ test "gated delta net" { std.testing.allocator, std.testing.io, GatedDeltaNet.forward, - .{ queries, keys, values, alphas, betas, GatedDeltaNet.State{ .s = initial_s } }, + .{ queries, keys, values, alphas, betas, .{ .s = initial_s } }, platform, .{}, ); diff --git a/zml/ops.zig b/zml/ops.zig index a695719d..c77d4573 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const builtin = @import("builtin"); const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); @@ -460,123 +461,126 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytyp return result; } -pub fn @"while"(operands: anytype, comptime cond: anytype, comptime body: anytype, context: anytype) stdx.meta.FnReturn(body) { - stdx.debug.assertComptime(stdx.meta.isTupleOf(@TypeOf(operands), Tensor), "zml.ops.while expects a tuple of Tensor operands, got {}", .{@TypeOf(operands)}); - stdx.debug.assertComptime(stdx.meta.isTuple(@TypeOf(context)), "zml.ops.while expects tuple context like .{{ ... }}, got {}", .{@TypeOf(context)}); +/// Implement a while loop. +/// The given While struct contains all the parameters that are fixed through the while. +/// It must have: +/// * a State struct definition that captures all the loop state that will change on each iteration +/// * a `cond(While, State) Tensor` function returning a scalar boolean tensor +/// * a `body(While, State) State` function that preserves the state shapes. +pub fn @"while"( + While: type, + context: While, + initial_state: While.State, +) While.State { + const comp = CompilationContext.current(); - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + var arena = std.heap.ArenaAllocator.init(comp.allocator); defer arena.deinit(); + const allocator = arena.allocator(); - const mlir_ctx = CompilationContext.current().mlir_ctx; + const mlir_ctx = comp.mlir_ctx; const location = mlir.Location.unknown(mlir_ctx); - var captured_context: @TypeOf(context) = undefined; + // Force to materialize tensor.value() before we push a new scope. + var captured_context: While = undefined; meta.mapAlloc(struct { fn capture(_: void, tensor: Tensor) Tensor { return Tensor._result(tensor.shape(), tensor.value()); } - }.capture, arena.allocator(), {}, context, &captured_context) catch unreachable; + }.capture, allocator, {}, context, &captured_context) catch unreachable; - var block_types: [operands.len]*const mlir.Type = undefined; - var block_locs: [operands.len]*const mlir.Location = @splat(location); - var operand_values: [operands.len]*const mlir.Value = undefined; - var operand_shapes: [operands.len]Shape = undefined; + const flat_operands: []*const Tensor = meta.collectPtrs(Tensor, allocator, &initial_state) catch @panic("OOM"); - inline for (0..operands.len) |i| { - operand_shapes[i] = operands[i].shape(); - block_types[i] = mlirx.Type.rankedTensor(mlir_ctx, operand_shapes[i]); - operand_values[i] = operands[i].value(); + const OperandInfo = struct { + type: *const mlir.Type, + location: *const mlir.Location, + value: *const mlir.Value, + }; + + var operands_info = std.MultiArrayList(OperandInfo).initCapacity(allocator, flat_operands.len) catch @panic("OOM"); + + for (flat_operands) |input| { + operands_info.appendAssumeCapacity(.{ + .type = mlirx.Type.rankedTensor(mlir_ctx, input._shape), + .location = location, + .value = input.value(), + }); } const cond_block = b: { - var cond_args: @TypeOf(operands) = undefined; - inline for (0..operands.len) |i| { - cond_args[i] = .fromShape(operand_shapes[i]); - } - - const block = mlir.Block.init(&block_types, &block_locs); + const block = mlir.Block.init(operands_info.items(.type), operands_info.items(.location)); errdefer block.deinit(); - CompilationContext.current().pushBlock(block); - defer CompilationContext.current().popBlock(); + comp.pushBlock(block); + defer comp.popBlock(); - const scope = CompilationContext.current().currentScope(); - inline for (0..operands.len) |i| { - scope.id_to_argument.put(scope.arena.allocator(), cond_args[i].id, i) catch unreachable; + // Interpret initial_state as the block argument + const scope = comp.currentScope(); + scope.id_to_argument.ensureUnusedCapacity(scope.arena.allocator(), flat_operands.len) catch @panic("OOM"); + for (0.., flat_operands) |i, input| { + scope.id_to_argument.putAssumeCapacity(input.id, i); } - const cond_result = @call(.auto, cond, cond_args ++ captured_context); - stdx.debug.assert(meta.count(Tensor, &cond_result) == 1, "zml.ops.while expects cond to return exactly one Tensor, got {}", .{@TypeOf(cond_result)}); + const cond: Tensor = captured_context.cond(initial_state); + stdx.debug.assert(cond.rank() == 0 and cond.dtype() == .bool, "zml.ops.while expects cond to return a scalar bool Tensor, got {f}", .{cond}); - const cond_tensor = meta.first(Tensor, cond_result); - stdx.debug.assert(cond_tensor.rank() == 0 and cond_tensor.dtype() == .bool, "zml.ops.while expects cond to return a scalar bool Tensor, got {f}", .{cond_tensor}); - - _ = dialects.stablehlo.return_(mlir_ctx, cond_tensor.value(), location).appendTo(block); + _ = dialects.stablehlo.return_(mlir_ctx, cond.value(), location).appendTo(block); break :b block; }; const body_block, var result = b: { - var body_args: @TypeOf(operands) = undefined; - inline for (0..operands.len) |i| { - body_args[i] = .fromShape(operand_shapes[i]); - } - - const block = mlir.Block.init(&block_types, &block_locs); + const block = mlir.Block.init(operands_info.items(.type), operands_info.items(.location)); errdefer block.deinit(); - CompilationContext.current().pushBlock(block); - defer CompilationContext.current().popBlock(); + comp.pushBlock(block); + defer comp.popBlock(); - const scope = CompilationContext.current().currentScope(); - inline for (0..operands.len) |i| { - scope.id_to_argument.put(scope.arena.allocator(), body_args[i].id, i) catch unreachable; + // Interpret operands as the block argument + const scope = comp.currentScope(); + scope.id_to_argument.ensureUnusedCapacity(scope.arena.allocator(), flat_operands.len) catch @panic("OOM"); + for (0.., flat_operands) |i, input| { + scope.id_to_argument.putAssumeCapacity(input.id, i); } - const result = @call(.auto, body, body_args ++ captured_context); - stdx.debug.assert(meta.count(Tensor, &result) == operands.len, "zml.ops.while expects body to return {d} Tensor values, got {}", .{ operands.len, @TypeOf(result) }); + const result: While.State = captured_context.body(initial_state); - var result_tensors: [operands.len]Tensor = undefined; - meta.collectBuf(struct { - fn cb(tensor: Tensor) Tensor { - return tensor; + if (builtin.mode == .Debug) { + const result_tensors = meta.collectPtrs(Tensor, allocator, &result) catch @panic("OOM"); + stdx.debug.assert(flat_operands.len == result_tensors.len, "zml.ops.while({s}) expects body to return {d} Tensor values, got {d} values in ", .{ @typeName(While), flat_operands.len, result_tensors.len }); + for (0.., flat_operands, result_tensors) |i, in, out| { + stdx.debug.assert(in.shape().eql(out.shape()), "zml.ops.while({s}) expects body to preserve loop state shape, got {f} for operand {d} but expected {f}", .{ @typeName(While), in, i, out }); } - }.cb, {}, &result, &result_tensors); - - var result_values: [operands.len]*const mlir.Value = undefined; - inline for (0..operands.len) |i| { - stdx.debug.assert(result_tensors[i].shape().eql(operand_shapes[i]), "zml.ops.while expects body to preserve loop state shape, got {f} for operand {d} but expected {f}", .{ result_tensors[i], i, operands[i] }); - result_values[i] = result_tensors[i].value(); } - _ = dialects.stablehlo.returns(mlir_ctx, &result_values, location).appendTo(block); + const result_values = meta.collectAlloc(Tensor.value, {}, allocator, &result) catch @panic("OOM"); + _ = dialects.stablehlo.returns(mlir_ctx, result_values, location).appendTo(block); break :b .{ block, result }; }; const op = dialects.stablehlo.while_( mlir_ctx, - &operand_values, - &block_types, + operands_info.items(.value), + operands_info.items(.type), cond_block, body_block, location, - ).appendTo(CompilationContext.current().currentScope().block); + ).appendTo(comp.currentScope().block); - const n_operands = operands.len; const AssignResultCtx = struct { op_: *mlir.Operation, - shapes: [n_operands]Shape, + og_tensors: []*const Tensor, idx: usize = 0, + + fn assign(ctx: *@This(), tensor: *Tensor) void { + tensor.* = Tensor._result(ctx.og_tensors[ctx.idx].shape(), ctx.op_.result(ctx.idx)); + ctx.idx += 1; + } }; var assign_result_ctx: AssignResultCtx = .{ .op_ = op, - .shapes = operand_shapes, + .og_tensors = flat_operands, }; - meta.visit(struct { - fn cb(ctx: *AssignResultCtx, tensor: *Tensor) void { - tensor.* = Tensor._result(ctx.shapes[ctx.idx], ctx.op_.result(ctx.idx)); - ctx.idx += 1; - } - }.cb, &assign_result_ctx, &result); + meta.visit(AssignResultCtx.assign, &assign_result_ctx, &result); return result; } @@ -587,32 +591,42 @@ test @"while" { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const initial_i: zml.Tensor = .init(.{}, .i64); - const initial_sum: zml.Tensor = .init(.{}, .i64); + const While = struct { + // While struct can contain tensor and non-tensor fields. + limit: i64, + step: Tensor, - const Local = struct { - fn cond(i: Tensor, sum: Tensor) Tensor { - _ = sum; - return i.cmp(.LT, Tensor.scalar(10, .i64)); + pub const State = struct { + i: zml.Tensor, + sum: zml.Tensor, + }; + + pub fn cond(self: @This(), state: State) Tensor { + return state.i.cmp(.LT, Tensor.scalar(self.limit, .i64)); } - fn body(i: Tensor, sum: Tensor) [2]Tensor { - const one = Tensor.scalar(1, .i64); + pub fn body(self: @This(), state: State) State { return .{ - i.add(one), - sum.add(one), + .i = state.i.add(self.step), + .sum = state.sum.add(self.step), }; } - fn forward(i: Tensor, sum: Tensor) [2]Tensor { - return zml.ops.@"while"(.{ i, sum }, cond, body, .{}); + fn forward(i: Tensor, sum: Tensor) State { + return zml.ops.@"while"( + @This(), + .{ .limit = 10, .step = .scalar(1, .i64) }, + .{ .i = i, .sum = sum }, + ); } }; + const initial_i: zml.Tensor = .init(.{}, .i64); + const initial_sum: zml.Tensor = .init(.{}, .i64); var exe = try zml.module.compile( std.testing.allocator, std.testing.io, - Local.forward, + While.forward, .{ initial_i, initial_sum }, platform, .{}, @@ -624,13 +638,13 @@ test @"while" { var sum_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_sum.shape(), .replicated, std.mem.sliceAsBytes(&[1]i64{0})); defer sum_buffer.deinit(); - var results = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local.forward, .{ i_buffer, sum_buffer }); - defer results[0].deinit(); - defer results[1].deinit(); + var results = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, While.forward, .{ i_buffer, sum_buffer }); + defer results.i.deinit(); + defer results.sum.deinit(); - var cpu_i = try results[0].toSliceAlloc(std.testing.allocator, std.testing.io); + var cpu_i = try results.i.toSliceAlloc(std.testing.allocator, std.testing.io); defer cpu_i.free(std.testing.allocator); - var cpu_sum = try results[1].toSliceAlloc(std.testing.allocator, std.testing.io); + var cpu_sum = try results.sum.toSliceAlloc(std.testing.allocator, std.testing.io); defer cpu_sum.free(std.testing.allocator); try std.testing.expectEqual(@as(i64, 10), cpu_i.items(i64)[0]); diff --git a/zml/tensor.zig b/zml/tensor.zig index 07c887d0..6a2c1cc9 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -4158,6 +4158,10 @@ pub const Tensor = struct { return _result(on_true._shape, op.result(0)); } + pub fn mask(x: Tensor, pred: Tensor, mask_value: anytype) Tensor { + return pred.broad(x.shape()).select(x, .scalar(mask_value, x.dtype())); + } + /// Returns a Tensor containing the element-wise not logical operation of the input Tensor. pub fn not(self: Tensor) Tensor { const op = dialects.stablehlo.not(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); From 09a7a9130ace54a3e2961978a6fd239e8a395d1a Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 30 Jul 2026 13:21:00 +0200 Subject: [PATCH 602/881] zml/attention: test paged attention (#681) Add unit test for paged attention, enforcing all our backends produce the same outputs. To simplify memory management of many small buffer provided in a struct (like paged attention metadata buffers), I introduce `zml.Buffer.deinitAll` backed by `zml.meta.visitFlatStruct`. Contrary to og `zml.meta.visit` the `flatStruct` variant emit a compile error if the input has a slice/ptr inside it. The idea is that since it enforces the struct to be a trivial, it means the struct won't require complex `deinit` and `zml.Buffer.deinitAll` is in fact everything you need to `deinit`. This avoid previous footgun of `zml.io.unloadAll`. This led to more improvement with `zml.Tensor.toMemoryAll` and `zml.Tensor.onMemoryAll` that also leverage the new `zml.meta.visitFlatStruct` --- zml/attention/flashattn.zig | 140 ------------ zml/attention/metal_attention.zig | 18 -- zml/attention/paged_attention.zig | 331 +++++++++++++++++++++++++---- zml/attention/tpu_attention.zig | 18 -- zml/attention/triton_attention.zig | 211 ++++++++---------- zml/buffer.zig | 9 + zml/meta.zig | 211 ++++++++++++++---- zml/module.zig | 4 + zml/tensor.zig | 54 ++++- 9 files changed, 603 insertions(+), 393 deletions(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 3d11ed24..3370ffa0 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -578,20 +578,6 @@ pub const paged_fa2 = struct { .mixed => |v| .{ .mixed = v.options }, }; } - - pub fn onMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { - return switch (self) { - .mixed => |s| .{ .mixed = s.onMemory(memory) }, - .decode => |s| .{ .decode = s.onMemory(memory) }, - }; - } - - pub fn toMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { - return switch (self) { - .mixed => |s| .{ .mixed = s.toMemory(memory) }, - .decode => |s| .{ .decode = s.toMemory(memory) }, - }; - } }; pub const DecodeOptions = struct { @@ -653,24 +639,6 @@ pub const paged_fa2 = struct { return allocation_size; } - - pub fn onMemory(self: DecodeParameters, memory: zml.platform.Memory.Kind) DecodeParameters { - return .{ - .block_table = self.block_table.onMemory(memory), - .cu_seqlens_q = self.cu_seqlens_q.onMemory(memory), - .seqused_k = self.seqused_k.onMemory(memory), - .options = self.options, - }; - } - - pub fn toMemory(self: DecodeParameters, memory: zml.platform.Memory.Kind) DecodeParameters { - return .{ - .block_table = self.block_table.toMemory(memory), - .cu_seqlens_q = self.cu_seqlens_q.toMemory(memory), - .seqused_k = self.seqused_k.toMemory(memory), - .options = self.options, - }; - } }; pub const MixedOptions = struct { @@ -774,32 +742,6 @@ pub const paged_fa2 = struct { return allocation_size; } - - pub fn onMemory(self: MixedParameters, memory: zml.platform.Memory.Kind) MixedParameters { - return .{ - .block_table_prefill = self.block_table_prefill.onMemory(memory), - .cu_seqlens_q_prefill = self.cu_seqlens_q_prefill.onMemory(memory), - .seqused_k_prefill = self.seqused_k_prefill.onMemory(memory), - .block_table_decode = self.block_table_decode.onMemory(memory), - .cu_seqlens_q_decode = self.cu_seqlens_q_decode.onMemory(memory), - .seqused_k_decode = self.seqused_k_decode.onMemory(memory), - .metadata = self.metadata.onMemory(memory), - .options = self.options, - }; - } - - pub fn toMemory(self: MixedParameters, memory: zml.platform.Memory.Kind) MixedParameters { - return .{ - .block_table_prefill = self.block_table_prefill.toMemory(memory), - .cu_seqlens_q_prefill = self.cu_seqlens_q_prefill.toMemory(memory), - .seqused_k_prefill = self.seqused_k_prefill.toMemory(memory), - .block_table_decode = self.block_table_decode.toMemory(memory), - .cu_seqlens_q_decode = self.cu_seqlens_q_decode.toMemory(memory), - .seqused_k_decode = self.seqused_k_decode.toMemory(memory), - .metadata = self.metadata.toMemory(memory), - .options = self.options, - }; - } }; pub const MixedMetadata = struct { @@ -817,18 +759,6 @@ pub const paged_fa2 = struct { allocation_size += self.decode_offset.byteSize(); return allocation_size; } - - pub fn onMemory(self: MixedMetadata, memory: zml.platform.Memory.Kind) MixedMetadata { - return .{ - .decode_offset = self.decode_offset.onMemory(memory), - }; - } - - pub fn toMemory(self: MixedMetadata, memory: zml.platform.Memory.Kind) MixedMetadata { - return .{ - .decode_offset = self.decode_offset.toMemory(memory), - }; - } }; pub const Prefill = struct { @@ -1254,20 +1184,6 @@ pub const paged_fa3 = struct { .mixed => |v| .{ .mixed = v.options }, }; } - - pub fn onMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { - return switch (self) { - .mixed => |s| .{ .mixed = s.onMemory(memory) }, - .decode => |s| .{ .decode = s.onMemory(memory) }, - }; - } - - pub fn toMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { - return switch (self) { - .mixed => |s| .{ .mixed = s.toMemory(memory) }, - .decode => |s| .{ .decode = s.toMemory(memory) }, - }; - } }; pub const DecodeOptions = struct { @@ -1329,24 +1245,6 @@ pub const paged_fa3 = struct { return allocation_size; } - - pub fn onMemory(self: DecodeParameters, memory: zml.platform.Memory.Kind) DecodeParameters { - return .{ - .block_table = self.block_table.onMemory(memory), - .cu_seqlens_q = self.cu_seqlens_q.onMemory(memory), - .seqused_k = self.seqused_k.onMemory(memory), - .options = self.options, - }; - } - - pub fn toMemory(self: DecodeParameters, memory: zml.platform.Memory.Kind) DecodeParameters { - return .{ - .block_table = self.block_table.toMemory(memory), - .cu_seqlens_q = self.cu_seqlens_q.toMemory(memory), - .seqused_k = self.seqused_k.toMemory(memory), - .options = self.options, - }; - } }; pub const MixedOptions = struct { @@ -1447,32 +1345,6 @@ pub const paged_fa3 = struct { return allocation_size; } - - pub fn onMemory(self: MixedParameters, memory: zml.platform.Memory.Kind) MixedParameters { - return .{ - .block_table_prefill = self.block_table_prefill.onMemory(memory), - .cu_seqlens_q_prefill = self.cu_seqlens_q_prefill.onMemory(memory), - .seqused_k_prefill = self.seqused_k_prefill.onMemory(memory), - .block_table_decode = self.block_table_decode.onMemory(memory), - .cu_seqlens_q_decode = self.cu_seqlens_q_decode.onMemory(memory), - .seqused_k_decode = self.seqused_k_decode.onMemory(memory), - .metadata = self.metadata.onMemory(memory), - .options = self.options, - }; - } - - pub fn toMemory(self: MixedParameters, memory: zml.platform.Memory.Kind) MixedParameters { - return .{ - .block_table_prefill = self.block_table_prefill.toMemory(memory), - .cu_seqlens_q_prefill = self.cu_seqlens_q_prefill.toMemory(memory), - .seqused_k_prefill = self.seqused_k_prefill.toMemory(memory), - .block_table_decode = self.block_table_decode.toMemory(memory), - .cu_seqlens_q_decode = self.cu_seqlens_q_decode.toMemory(memory), - .seqused_k_decode = self.seqused_k_decode.toMemory(memory), - .metadata = self.metadata.toMemory(memory), - .options = self.options, - }; - } }; pub const MixedMetadata = struct { @@ -1490,18 +1362,6 @@ pub const paged_fa3 = struct { allocation_size += self.decode_offset.byteSize(); return allocation_size; } - - pub fn onMemory(self: MixedMetadata, memory: zml.platform.Memory.Kind) MixedMetadata { - return .{ - .decode_offset = self.decode_offset.onMemory(memory), - }; - } - - pub fn toMemory(self: MixedMetadata, memory: zml.platform.Memory.Kind) MixedMetadata { - return .{ - .decode_offset = self.decode_offset.toMemory(memory), - }; - } }; pub const Prefill = struct { diff --git a/zml/attention/metal_attention.zig b/zml/attention/metal_attention.zig index 17f9de62..667d8130 100644 --- a/zml/attention/metal_attention.zig +++ b/zml/attention/metal_attention.zig @@ -83,24 +83,6 @@ pub const paged = struct { pub fn options(self: Parameters) Options { return self.options_; } - - pub fn onMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { - return .{ - .options_ = self.options_, - .block_table = self.block_table.onMemory(memory), - .seq_lens = self.seq_lens.onMemory(memory), - .query_start_len = self.query_start_len.onMemory(memory), - }; - } - - pub fn toMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { - return .{ - .options_ = self.options_, - .block_table = self.block_table.toMemory(memory), - .seq_lens = self.seq_lens.toMemory(memory), - .query_start_len = self.query_start_len.toMemory(memory), - }; - } }; pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index b3045831..898a5a26 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -10,12 +10,19 @@ const triton = @import("triton_attention.zig"); const PagedAttention = @This(); +test { + std.testing.refAllDecls(Backend); + std.testing.refAllDecls(Options); + std.testing.refAllDecls(Parameters); +} + pub const Backend = enum { cuda_fa2, cuda_fa3, triton, mosaic_tpu, metal, + // vanilla, pub fn auto(platform: *const zml.Platform) Backend { return switch (platform.target) { @@ -24,7 +31,24 @@ pub const Backend = enum { .oneapi => .triton, .tpu => .mosaic_tpu, .metal => .metal, - else => stdx.debug.panic("Paged attention is not supported on {s} yet", .{@tagName(platform.target)}), + // .cpu => .vanilla + .cpu, .neuron => stdx.debug.panic("Paged attention is not supported on {s} yet", .{@tagName(platform.target)}), + }; + } + + pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { + return switch (backend) { + // .vanilla => true, + .triton => platform.target != .cpu, + .metal => platform.target == .metal, + .mosaic_tpu => platform.target == .tpu, + .cuda_fa2 => platform.target == .cuda, + .cuda_fa3 => { + if (platform.target != .cuda) return false; + const first_device = platform.pjrt_client.devices(platform.pjrt_api)[0]; + const cc = zml.platform.cuda.tryGetComputeCapabilities(platform, first_device) orelse return false; + return std.mem.eql(u8, cc, "9.0"); + }, }; } }; @@ -40,6 +64,8 @@ pub const Options = union(Backend) { backend: Backend, is_prefill: bool, batch_size: u32, + batch_size_prefill: ?u32 = null, + batch_size_decode: ?u32 = null, seq_len: u32, max_num_pages: u32, max_token_count: u32, @@ -54,8 +80,8 @@ pub const Options = union(Backend) { .cuda_fa2 => if (args.is_prefill) .{ .cuda_fa2 = .{ .mixed = .{ - .batch_size_decode = args.batch_size, - .batch_size_prefill = args.batch_size, + .batch_size_decode = args.batch_size_decode orelse args.batch_size, + .batch_size_prefill = args.batch_size_prefill orelse args.batch_size, .max_num_pages = args.max_num_pages, .max_seqlen_k = args.seq_len, .max_seqlen_q = args.max_seqlen_q, @@ -81,8 +107,8 @@ pub const Options = union(Backend) { .cuda_fa3 => if (args.is_prefill) .{ .cuda_fa3 = .{ .mixed = .{ - .batch_size_decode = args.batch_size, - .batch_size_prefill = args.batch_size, + .batch_size_decode = args.batch_size_decode orelse args.batch_size, + .batch_size_prefill = args.batch_size_prefill orelse args.batch_size, .max_num_pages = args.max_num_pages, .max_seqlen_k = args.seq_len, .max_seqlen_q = args.max_seqlen_q, @@ -105,22 +131,6 @@ pub const Options = union(Backend) { }, }, }, - .triton => .{ - .triton = .{ - .batch_size = args.batch_size, - .max_num_pages = args.max_num_pages, - .max_seqlen_q = args.max_seqlen_q, - .is_prefill = args.is_prefill, - }, - }, - .metal => .{ - .metal = .{ - .batch_size = args.batch_size, - .max_num_pages = args.max_num_pages, - .max_seqlen_q = args.max_seqlen_q, - .is_prefill = args.is_prefill, - }, - }, .mosaic_tpu => .{ .mosaic_tpu = .{ .is_prefill = args.is_prefill, @@ -133,6 +143,12 @@ pub const Options = union(Backend) { .head_dim = args.head_dim, }, }, + inline .triton, .metal => |t| @unionInit(Options, @tagName(t), .{ + .batch_size = args.batch_size, + .max_num_pages = args.max_num_pages, + .max_seqlen_q = args.max_seqlen_q, + .is_prefill = args.is_prefill, + }), }; } @@ -158,21 +174,13 @@ pub const Parameters = union(Backend) { pub fn init(options_: Options) Parameters { return switch (options_) { - .cuda_fa2 => |cuda_fa2_options| .{ .cuda_fa2 = flashattn.paged_fa2.Parameters.init(cuda_fa2_options) }, - .cuda_fa3 => |cuda_fa3_options| .{ .cuda_fa3 = flashattn.paged_fa3.Parameters.init(cuda_fa3_options) }, - .triton => |triton_options| .{ .triton = triton.paged.Parameters.init(triton_options) }, - .mosaic_tpu => |mosaic_tpu_options| .{ .mosaic_tpu = tpu.mosaic_tpu.Parameters.init(mosaic_tpu_options) }, - .metal => |metal_options| .{ .metal = metal.paged.Parameters.init(metal_options) }, + inline else => |opts, backend| @unionInit(Parameters, @tagName(backend), .init(opts)), }; } pub fn options(self: Parameters) Options { return switch (self) { - .cuda_fa2 => |v| .{ .cuda_fa2 = v.options() }, - .cuda_fa3 => |v| .{ .cuda_fa3 = v.options() }, - .triton => |v| .{ .triton = v.options() }, - .mosaic_tpu => |v| .{ .mosaic_tpu = v.options() }, - .metal => |v| .{ .metal = v.options() }, + inline else => |v, backend| @unionInit(Options, @tagName(backend), v.options()), }; } @@ -183,23 +191,12 @@ pub const Parameters = union(Backend) { } pub fn onMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { - return switch (self) { - .cuda_fa2 => |v| .{ .cuda_fa2 = v.onMemory(memory) }, - .cuda_fa3 => |v| .{ .cuda_fa3 = v.onMemory(memory) }, - .triton => |v| .{ .triton = v.onMemory(memory) }, - .mosaic_tpu => |v| .{ .mosaic_tpu = v.onMemory(memory) }, - .metal => |v| .{ .metal = v.onMemory(memory) }, - }; + zml.Tensor.onMemoryAll(&self, memory); + return self; } pub fn toMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { - return switch (self) { - .cuda_fa2 => |v| .{ .cuda_fa2 = v.toMemory(memory) }, - .cuda_fa3 => |v| .{ .cuda_fa3 = v.toMemory(memory) }, - .triton => |v| .{ .triton = v.toMemory(memory) }, - .mosaic_tpu => |v| .{ .mosaic_tpu = v.toMemory(memory) }, - .metal => |v| .{ .metal = v.toMemory(memory) }, - }; + return zml.Tensor.toMemoryAll(self, memory); } }; @@ -215,6 +212,44 @@ pub const KvCache = union(enum) { v: zml.Tensor, }, dense: zml.Tensor, + + pub fn update( + self: KvCache, + new_k: zml.Tensor, + new_v: zml.Tensor, + slot_mapping: zml.Tensor, + chunk_size: u32, + backend: Backend, + ) KvCache { + const page_index, const offset = getPageAndOffsetFromSlotMapping(slot_mapping, chunk_size); + + var kv: KvCache = switch (self) { + .split => |split| switch (backend) { + .cuda_fa2, .cuda_fa3, .triton, .mosaic_tpu, .metal => .{ .split = .{ + .k = split.k.scatterSlices( + .{ .page = page_index, .k_chunk = offset }, + new_k, + .{ .update_fn = zml.Tensor.ScatterOpts.override, .indices_are_unique = false, .indices_are_sorted = false }, + ), + .v = split.v.scatterSlices( + .{ .page = page_index, .k_chunk = offset }, + new_v, + .{ .update_fn = zml.Tensor.ScatterOpts.override, .indices_are_unique = false, .indices_are_sorted = false }, + ), + } }, + }, + .dense => @panic("TODO"), + }; + kv.split.k = kv.split.k.reuseBuffer(self.split.k); + kv.split.v = kv.split.v.reuseBuffer(self.split.v); + return kv; + } + + pub fn getPageAndOffsetFromSlotMapping(slot_mapping: zml.Tensor, page_size: u32) [2]zml.Tensor { + const page_index = slot_mapping.divByConst(page_size); + const offset = slot_mapping.remainder(.scalar(page_size, slot_mapping.dtype())); + return .{ page_index, offset }; + } }; pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache, opts: AttentionOptions) zml.Tensor { @@ -272,3 +307,209 @@ test "Backend.auto selects triton on oneAPI" { try std.testing.expectEqual(Backend.triton, Backend.auto(&platform)); } + +test pagedAttention { + const platform = zml.testing.env(); + const io = std.testing.io; + const allocator = std.testing.allocator; + var arena_state: std.heap.ArenaAllocator = .init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + const num_pages = 64; + const page_size = 16; + const max_num_pages = 16; + const prefill_token_count = 32; + const num_prefill = 1; + const num_decode = 6; + const active_batch_size = num_prefill + num_decode; + const batch_size = active_batch_size + 1; + const query_token_count = prefill_token_count + num_decode; + const dt: zml.DataType = .bf16; + const partition = .{ .hkv = .model }; + const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache } = .{ + .q = .withPartitioning(.init(.{ .b = query_token_count, .hkv = 2, .hg = 4, .hd = 32 }, dt), partition), + .k = .withPartitioning(.init(.{ .b = query_token_count, .hkv = 2, .hd = 32 }, dt), partition), + .v = .withPartitioning(.init(.{ .b = query_token_count, .hkv = 2, .hd = 32 }, dt), partition), + .kv_cache = .{ + .split = .{ + .k = .withPartitioning(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 2, .hd = 32 }, dt), partition), + .v = .withPartitioning(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 2, .hd = 32 }, dt), partition), + }, + }, + }; + + const shardings: []const zml.Sharding = &.{ platform.replicated_sharding, platform.shardings.get("model").? }; + const rng_q = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.q.shape(), .{} }, .{ .shardings = shardings }); + defer rng_q.deinit(); + const rng_k = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.k.shape(), .{} }, .{ .shardings = shardings }); + defer rng_k.deinit(); + const rng_kv_cache = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.kv_cache.split.k.shape(), .{} }, .{ .shardings = shardings }); + defer rng_kv_cache.deinit(); + + const triton_options_args: Options.Args = .{ + .backend = .triton, + .is_prefill = true, + .batch_size = batch_size, + .batch_size_prefill = num_prefill, + .batch_size_decode = num_decode, + .seq_len = max_num_pages * page_size, + .max_num_pages = max_num_pages, + .max_token_count = 16 * page_size, + .num_heads = @intCast(tensors.q.dim(.hg) * tensors.q.dim(.hkv)), + .num_kv_heads = @intCast(tensors.q.dim(.hkv)), + .head_dim = @intCast(tensors.q.dim(.hd)), + .max_seqlen_q = 16 * 2, + }; + const triton_parameters: Parameters = .init(.fromBackend(triton_options_args)); + const attn_opts: AttentionOptions = .{ .is_causal = true }; + var q = try zml.testing.autoCall(allocator, io, &rng_q, zml.Tensor.Rng.normal, {}); + defer q.deinit(); + var new_k = try zml.testing.autoCall(allocator, io, &rng_k, zml.Tensor.Rng.normal, {}); + defer new_k.deinit(); + var new_v = try zml.testing.autoCall(allocator, io, &rng_k, zml.Tensor.Rng.normal, {}); + defer new_v.deinit(); + + var kv_cache_d: zml.Bufferized(KvCache) = .{ .split = .{ + .k = try zml.testing.autoCall(allocator, io, &rng_kv_cache, zml.Tensor.Rng.normal, {}), + .v = try zml.testing.autoCall(allocator, io, &rng_kv_cache, zml.Tensor.Rng.normal, {}), + } }; + defer zml.Buffer.deinitAll(KvCache, &kv_cache_d); + + const block_table: [batch_size][max_num_pages]i32 = .{ + // prefilling pages 9-10 + .{ 0, 1, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, + // generation + .{ 0, 1, 2, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, + .{ 0, 1, 2, 3, 4, 6, 7, 8, -1, -1, -1, -1, -1, -1, -1, -1 }, + .{ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63 }, + .{ 0, 1, 2, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, + .{ 0, 1, 2, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, + .{ 0, 1, 2, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, + .{ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, + }; + + const seq_lens: [batch_size]i32 = .{ + 4 * page_size, + 4 * page_size + 3, + 7 * page_size + 11, + 15 * page_size + 9, + 4 * page_size + 3, + 4 * page_size + 3, + 4 * page_size + 3, + 0, + }; + + const query_start_len: [batch_size + 1]i32 = .{ 0, 32, 33, 34, 35, 36, 37, 38, 38 }; + var triton_parameters_d: zml.Bufferized(Parameters) = .{ .triton = .{ + .block_table = try .fromBytes(io, platform, triton_parameters.triton.block_table.shape(), .replicated, @ptrCast(&block_table)), + .seq_lens = try .fromBytes(io, platform, triton_parameters.triton.seq_lens.shape(), .replicated, @ptrCast(&seq_lens)), + .query_start_len = try .fromBytes(io, platform, triton_parameters.triton.query_start_len.shape(), .replicated, @ptrCast(&query_start_len)), + } }; + defer zml.Buffer.deinitAll(Parameters, &triton_parameters_d); + + var results_per_backend: std.enums.EnumArray(Backend, ?zml.Slice) = .initFill(null); + defer { + for (std.enums.values(Backend)) |backend| { + if (results_per_backend.get(backend)) |slice| slice.free(allocator); + } + } + + for (std.enums.values(Backend)) |backend| { + if (!backend.isAvailable(platform)) { + std.log.warn("paged_attention backend {t} not available", .{backend}); + continue; + } + std.log.warn("Testing paged_attention with backend {t}", .{backend}); + + var backend_options_args = triton_options_args; + backend_options_args.backend = backend; + const parameters: Parameters = .init(.fromBackend(backend_options_args)); + + const exe = try platform.compileFn( + allocator, + io, + pagedAttention, + .{ parameters, tensors.q, tensors.k, tensors.v, tensors.kv_cache, attn_opts }, + .{ .program_name = try std.fmt.allocPrint(arena, "paged_attention_{t}", .{backend}), .shardings = shardings }, + ); + defer exe.deinit(); + + var parameters_d: zml.Bufferized(Parameters) = switch (parameters) { + // No materializer implemented for cuda fa3 + .cuda_fa3 => continue, + .cuda_fa2 => |params| cuda_fa2: { + var block_table_prefill: [num_prefill][max_num_pages]i32 = undefined; + @memcpy(&block_table_prefill, block_table[0..num_prefill]); + var block_table_decode: [num_decode][max_num_pages]i32 = undefined; + @memcpy(&block_table_decode, block_table[num_prefill .. num_prefill + num_decode]); + + var cu_seqlens_q_prefill: [num_prefill + 1]i32 = undefined; + @memcpy(&cu_seqlens_q_prefill, query_start_len[0 .. num_prefill + 1]); + var cu_seqlens_q_decode: [num_decode + 1]i32 = undefined; + for (&cu_seqlens_q_decode, query_start_len[num_prefill .. num_prefill + num_decode + 1]) |*decode_len, query_start| { + decode_len.* = query_start - prefill_token_count; + } + + var seqused_k_prefill: [num_prefill]i32 = undefined; + @memcpy(&seqused_k_prefill, seq_lens[0..num_prefill]); + var seqused_k_decode: [num_decode]i32 = undefined; + @memcpy(&seqused_k_decode, seq_lens[num_prefill .. num_prefill + num_decode]); + + break :cuda_fa2 .{ .cuda_fa2 = .{ .mixed = .{ + .block_table_prefill = try .fromBytes(io, platform, params.mixed.block_table_prefill.shape(), .replicated, @ptrCast(&block_table_prefill)), + .cu_seqlens_q_prefill = try .fromBytes(io, platform, params.mixed.cu_seqlens_q_prefill.shape(), .replicated, @ptrCast(&cu_seqlens_q_prefill)), + .seqused_k_prefill = try .fromBytes(io, platform, params.mixed.seqused_k_prefill.shape(), .replicated, @ptrCast(&seqused_k_prefill)), + + .block_table_decode = try .fromBytes(io, platform, params.mixed.block_table_decode.shape(), .replicated, @ptrCast(&block_table_decode)), + .cu_seqlens_q_decode = try .fromBytes(io, platform, params.mixed.cu_seqlens_q_decode.shape(), .replicated, @ptrCast(&cu_seqlens_q_decode)), + .seqused_k_decode = try .fromBytes(io, platform, params.mixed.seqused_k_decode.shape(), .replicated, @ptrCast(&seqused_k_decode)), + + .metadata = .{ .decode_offset = try .scalar(io, platform, prefill_token_count, .i32) }, + } } }; + }, + .triton => triton_parameters_d, + inline .metal, .mosaic_tpu => |_, t| @unionInit(zml.Bufferized(Parameters), @tagName(t), .{ + .block_table = triton_parameters_d.triton.block_table, + .seq_lens = triton_parameters_d.triton.seq_lens, + .query_start_len = triton_parameters_d.triton.query_start_len, + }), + }; + // cu_fa2 creates new buffers while other reuse triton buffers. + defer if (backend == .cuda_fa2) zml.Buffer.deinitAll(Parameters, ¶meters_d); + + var output_d = try zml.testing.autoCall(allocator, io, &exe, pagedAttention, .{ parameters_d, q, new_k, new_v, kv_cache_d }); + defer output_d.deinit(); + results_per_backend.set(backend, try output_d.toSliceAlloc(allocator, io)); + } + + const reference_backend: Backend = .triton; + const reference = results_per_backend.get(reference_backend) orelse return error.SkipZigTest; + + var num_failed: u32 = 0; + for (std.enums.values(Backend)) |backend| { + if (backend == reference_backend) continue; + const output_h = results_per_backend.get(backend) orelse continue; + + const tolerance: zml.testing.CompareOpts = .{ + .absolute_tolerance = 1e-2, + .relative_tolerance = 1e-2, + .epsilon_relative = 1e-6, + }; + + zml.testing.expectClose(io, reference, output_h, tolerance) catch |err| switch (err) { + error.TestUnexpectedResult => { + num_failed += 1; + std.log.err("test pagedAttention failed on backend {0t}\n--> reference ({1t}): {2d:32.3}\n--> pagedAttention({0t}): {3d:32.3}", .{ + backend, + reference_backend, + reference.subSlice(1, 0, 1).subSlice(2, 0, 1), + output_h.subSlice(1, 0, 1).subSlice(2, 0, 1), + }); + }, + else => |e| return e, + }; + } + + if (num_failed > 0) return error.TestUnexpectedResult; +} diff --git a/zml/attention/tpu_attention.zig b/zml/attention/tpu_attention.zig index 7ce5c93c..b0771ea9 100644 --- a/zml/attention/tpu_attention.zig +++ b/zml/attention/tpu_attention.zig @@ -78,24 +78,6 @@ pub const mosaic_tpu = struct { allocation_size += self.query_start_len.byteSize(); return allocation_size; } - - pub fn onMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { - return .{ - .opts = self.opts, - .block_table = self.block_table.onMemory(memory), - .seq_lens = self.seq_lens.onMemory(memory), - .query_start_len = self.query_start_len.onMemory(memory), - }; - } - - pub fn toMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { - return .{ - .opts = self.opts, - .block_table = self.block_table.toMemory(memory), - .seq_lens = self.seq_lens.toMemory(memory), - .query_start_len = self.query_start_len.toMemory(memory), - }; - } }; fn raggedPagedKernelCall( diff --git a/zml/attention/triton_attention.zig b/zml/attention/triton_attention.zig index 9043724e..81e80a51 100644 --- a/zml/attention/triton_attention.zig +++ b/zml/attention/triton_attention.zig @@ -3,16 +3,13 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../zml.zig"); +const triton = zml.kernel.triton; const AttentionOptions = @import("paged_attention.zig").AttentionOptions; - const kernels = @import("triton_kernels/unified_attention.zig"); const kernels_oneapi = @import("triton_kernels/unified_attention_oneapi.zig"); -const tri = zml.kernel.triton; const log = std.log.scoped(.@"zml/attention/triton"); -const toDType = tri.from; - fn isOneapiTarget() bool { return zml.module.CompilationContext.current().platform.target == .oneapi; } @@ -184,24 +181,6 @@ pub const paged = struct { pub fn options(self: Parameters) Options { return self.options_; } - - pub fn onMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { - return .{ - .options_ = self.options_, - .block_table = self.block_table.onMemory(memory), - .seq_lens = self.seq_lens.onMemory(memory), - .query_start_len = self.query_start_len.onMemory(memory), - }; - } - - pub fn toMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { - return .{ - .options_ = self.options_, - .block_table = self.block_table.toMemory(memory), - .seq_lens = self.seq_lens.toMemory(memory), - .query_start_len = self.query_start_len.toMemory(memory), - }; - } }; pub const PagedAttentionOptions = struct { @@ -317,9 +296,9 @@ pub const paged = struct { const config = select2dConfig(paged_attention_opts); const kernel_config: kernels.KernelUnifiedAttention2dPtr.Config = .{ - .q_dtype = toDType(q.dtype()), - .kv_dtype = toDType(k_cache.dtype()), - .o_dtype = toDType(q.dtype()), + .q_dtype = triton.from(q.dtype()), + .kv_dtype = triton.from(k_cache.dtype()), + .o_dtype = triton.from(q.dtype()), .num_query_heads = @intCast(paged_attention_opts.num_heads), .num_queries_per_kv = @intCast(paged_attention_opts.numQueriesPerKv()), .block_size = @intCast(paged_attention_opts.block_size), @@ -339,24 +318,16 @@ pub const paged = struct { }; log.debug("pagedAttention2d config: {any}", .{kernel_config}); - const dummy = zml.Tensor.constant(zml.DataType.i8.zero()); + const dummy: zml.Tensor = .scalar(0, .i8); const block_table_strides = parameters.block_table.shape().computeElementStrides().constSlice(); - const block_table_strides_ptr = zml.Tensor.constant(zml.DataType.i64.constant(block_table_strides[0])); + const q_shape = q.shape().mergeAxes(.{ .h = .{ .hkv, .hg } }); const q_strides = q_shape.computeElementStrides().constSlice(); - const q_strides_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(q_strides[0])); - const q_strides_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(q_strides[1])); const k_strides = k_cache.shape().computeElementStrides().constSlice(); - const k_strides_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(k_strides[0])); - const k_strides_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(k_strides[1])); - const k_strides_2_ptr = zml.Tensor.constant(zml.DataType.i64.constant(k_strides[2])); const v_strides = v_cache.shape().computeElementStrides().constSlice(); - const v_strides_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(v_strides[0])); - const v_strides_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(v_strides[1])); - const v_strides_2_ptr = zml.Tensor.constant(zml.DataType.i64.constant(v_strides[2])); - const num_seqs_ptr = zml.Tensor.constant(zml.DataType.i32.constant(parameters.block_table.dim(0))); + const scale: f32 = paged_attention_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))); - const scale_ptr = zml.Tensor.constant(zml.DataType.f32.constant(scale)); + const num_seqs = parameters.block_table.dim(0); const output = kernels.KernelUnifiedAttention2dPtr.Kernel.call( .{ @@ -368,25 +339,25 @@ pub const paged = struct { .seq_lens_ptr = parameters.seq_lens, .alibi_slopes_ptr = dummy, .qq_bias_ptr = dummy, - .scale_ptr = scale_ptr, + .scale_ptr = .scalar(scale, .f32), .k_scale_ptr = dummy, .v_scale_ptr = dummy, .out_scale_ptr = dummy, .softcap_ptr = dummy, - .block_table_stride_ptr = block_table_strides_ptr, - .query_stride_0_ptr = q_strides_0_ptr, - .query_stride_1_ptr = q_strides_1_ptr, - .output_stride_0_ptr = q_strides_0_ptr, - .output_stride_1_ptr = q_strides_1_ptr, + .block_table_stride_ptr = .scalar(block_table_strides[0], .i64), + .query_stride_0_ptr = .scalar(q_strides[0], .i64), + .query_stride_1_ptr = .scalar(q_strides[1], .i64), + .output_stride_0_ptr = .scalar(q_strides[0], .i64), + .output_stride_1_ptr = .scalar(q_strides[1], .i64), .qq_bias_stride_0_ptr = dummy, - .stride_k_cache_0_ptr = k_strides_0_ptr, - .stride_k_cache_1_ptr = k_strides_1_ptr, - .stride_k_cache_2_ptr = k_strides_2_ptr, - .stride_v_cache_0_ptr = v_strides_0_ptr, - .stride_v_cache_1_ptr = v_strides_1_ptr, - .stride_v_cache_2_ptr = v_strides_2_ptr, + .stride_k_cache_0_ptr = .scalar(k_strides[0], .i64), + .stride_k_cache_1_ptr = .scalar(k_strides[1], .i64), + .stride_k_cache_2_ptr = .scalar(k_strides[2], .i64), + .stride_v_cache_0_ptr = .scalar(v_strides[0], .i64), + .stride_v_cache_1_ptr = .scalar(v_strides[1], .i64), + .stride_v_cache_2_ptr = .scalar(v_strides[2], .i64), .query_start_len_ptr = parameters.query_start_len, - .num_seqs_ptr = num_seqs_ptr, + .num_seqs_ptr = .scalar(num_seqs, .i32), }, .{ .output = q.shape() }, .{ @@ -404,8 +375,8 @@ pub const paged = struct { const head_size_padded: i64 = @intCast(std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim)); const attn_kernel_config: kernels.KernelUnifiedAttention3dPtr.Config = .{ - .q_dtype = toDType(q.dtype()), - .kv_dtype = toDType(k_cache.dtype()), + .q_dtype = triton.from(q.dtype()), + .kv_dtype = triton.from(k_cache.dtype()), .num_query_heads = @intCast(paged_attention_opts.num_heads), .num_queries_per_kv = @intCast(paged_attention_opts.numQueriesPerKv()), .block_size = @intCast(paged_attention_opts.block_size), @@ -426,7 +397,7 @@ pub const paged = struct { log.debug("pagedAttention3d attention config: {any}", .{attn_kernel_config}); const reduce_kernel_config: kernels.ReduceSegmentsPtr.Config = .{ - .o_dtype = toDType(q.dtype()), + .o_dtype = triton.from(q.dtype()), .num_query_heads = @intCast(paged_attention_opts.num_heads), .tile_size = @intCast(config.reduce.tile_size), .head_size = @intCast(paged_attention_opts.head_dim), @@ -437,26 +408,22 @@ pub const paged = struct { }; log.debug("pagedAttention3d reduce config: {any}", .{reduce_kernel_config}); - const dummy = zml.Tensor.constant(zml.DataType.i8.zero()); + const dummy: zml.Tensor = .scalar(0, .i8); const block_table_strides = parameters.block_table.shape().computeElementStrides().constSlice(); - const block_table_strides_ptr = zml.Tensor.constant(zml.DataType.i64.constant(block_table_strides[0])); + const q_shape = q.shape().mergeAxes(.{ .h = .{ .hkv, .hg } }); const q_strides = q_shape.computeElementStrides().constSlice(); - const q_strides_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(q_strides[0])); - const q_strides_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(q_strides[1])); const k_strides = k_cache.shape().computeElementStrides().constSlice(); - const k_strides_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(k_strides[0])); - const k_strides_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(k_strides[1])); - const k_strides_2_ptr = zml.Tensor.constant(zml.DataType.i64.constant(k_strides[2])); const v_strides = v_cache.shape().computeElementStrides().constSlice(); - const v_strides_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(v_strides[0])); - const v_strides_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(v_strides[1])); - const v_strides_2_ptr = zml.Tensor.constant(zml.DataType.i64.constant(v_strides[2])); - const num_seqs_ptr = zml.Tensor.constant(zml.DataType.i32.constant(parameters.block_table.dim(0))); - const scale: f32 = paged_attention_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))); - const scale_ptr = zml.Tensor.constant(zml.DataType.f32.constant(scale)); - const attn_grid: [3]i32 = .{ @intCast(config.attention.total_q_blocks), @intCast(paged_attention_opts.num_kv_heads), @intCast(config.attention.num_segments_per_seq) }; + const scale: f32 = paged_attention_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))); + const num_seqs = parameters.block_table.dim(0); + + const attn_grid: [3]i32 = .{ + @intCast(config.attention.total_q_blocks), + @intCast(paged_attention_opts.num_kv_heads), + @intCast(config.attention.num_segments_per_seq), + }; const attn_output = kernels.KernelUnifiedAttention3dPtr.Kernel.call( .{ .query_ptr = q, @@ -467,22 +434,22 @@ pub const paged = struct { .seq_lens_ptr = parameters.seq_lens, .alibi_slopes_ptr = dummy, .qq_bias_ptr = dummy, - .scale_ptr = scale_ptr, + .scale_ptr = .scalar(scale, .f32), .k_scale_ptr = dummy, .v_scale_ptr = dummy, .softcap_ptr = dummy, - .block_table_stride_ptr = block_table_strides_ptr, - .query_stride_0_ptr = q_strides_0_ptr, - .query_stride_1_ptr = q_strides_1_ptr, + .block_table_stride_ptr = .scalar(block_table_strides[0], .i64), + .query_stride_0_ptr = .scalar(q_strides[0], .i64), + .query_stride_1_ptr = .scalar(q_strides[1], .i64), .qq_bias_stride_0_ptr = dummy, - .stride_k_cache_0_ptr = k_strides_0_ptr, - .stride_k_cache_1_ptr = k_strides_1_ptr, - .stride_k_cache_2_ptr = k_strides_2_ptr, - .stride_v_cache_0_ptr = v_strides_0_ptr, - .stride_v_cache_1_ptr = v_strides_1_ptr, - .stride_v_cache_2_ptr = v_strides_2_ptr, + .stride_k_cache_0_ptr = .scalar(k_strides[0], .i64), + .stride_k_cache_1_ptr = .scalar(k_strides[1], .i64), + .stride_k_cache_2_ptr = .scalar(k_strides[2], .i64), + .stride_v_cache_0_ptr = .scalar(v_strides[0], .i64), + .stride_v_cache_1_ptr = .scalar(v_strides[1], .i64), + .stride_v_cache_2_ptr = .scalar(v_strides[2], .i64), .query_start_len_ptr = parameters.query_start_len, - .num_seqs_ptr = num_seqs_ptr, + .num_seqs_ptr = .scalar(num_seqs, .i32), }, .{ .segm_output = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq, std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim) }, .f32), @@ -503,17 +470,21 @@ pub const paged = struct { .segm_max_ptr = attn_output.segm_max, .segm_expsum_ptr = attn_output.segm_expsum, .seq_lens_ptr = parameters.seq_lens, - .num_seqs_ptr = num_seqs_ptr, + .num_seqs_ptr = .scalar(num_seqs, .i32), .out_scale_inv_ptr = dummy, - .output_stride_0_ptr = q_strides_0_ptr, - .output_stride_1_ptr = q_strides_1_ptr, - .block_table_stride_ptr = block_table_strides_ptr, + .output_stride_0_ptr = .scalar(q_strides[0], .i64), + .output_stride_1_ptr = .scalar(q_strides[1], .i64), + .block_table_stride_ptr = .scalar(block_table_strides[0], .i64), .query_start_len_ptr = parameters.query_start_len, }, .{ .output = q.shape() }, .{ .cfg = reduce_kernel_config, - .grid = .{ @intCast(paged_attention_opts.num_tokens), @intCast(paged_attention_opts.num_heads), 1 }, + .grid = .{ + @intCast(paged_attention_opts.num_tokens), + @intCast(paged_attention_opts.num_heads), + 1, + }, .num_stages = @intCast(config.reduce.num_stages), .num_warps = @intCast(config.reduce.num_warps), }, @@ -532,14 +503,12 @@ pub const paged = struct { const head_size_padded: i64 = @intCast(std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim)); - const k_cache_shape_c = k_cache.shape(); - const k_strides_c = k_cache_shape_c.computeElementStrides().constSlice(); - const v_cache_shape_c = v_cache.shape(); - const v_strides_c = v_cache_shape_c.computeElementStrides().constSlice(); + const k_strides = k_cache.shape().computeElementStrides().constSlice(); + const v_strides = v_cache.shape().computeElementStrides().constSlice(); const attn_kernel_config: kernels_oneapi.KernelUnifiedAttention3dPtr.Config = .{ - .q_dtype = toDType(q.dtype()), - .kv_dtype = toDType(k_cache.dtype()), + .q_dtype = triton.from(q.dtype()), + .kv_dtype = triton.from(k_cache.dtype()), .num_query_heads = @intCast(paged_attention_opts.num_heads), .num_queries_per_kv = @intCast(paged_attention_opts.numQueriesPerKv()), .block_size = @intCast(paged_attention_opts.block_size), @@ -555,12 +524,12 @@ pub const paged = struct { .block_m = @intCast(config.attention.block_m), .num_segments_per_seq = @intCast(config.attention.num_segments_per_seq), .all_decode = paged_attention_opts.all_decode, - .stride_k_cache_0 = @intCast(k_strides_c[k_cache_shape_c.axis(.page)]), - .stride_k_cache_1 = @intCast(k_strides_c[k_cache_shape_c.axis(.k_chunk)]), - .stride_k_cache_2 = @intCast(k_strides_c[k_cache_shape_c.axis(.hkv)]), - .stride_v_cache_0 = @intCast(v_strides_c[v_cache_shape_c.axis(.page)]), - .stride_v_cache_1 = @intCast(v_strides_c[v_cache_shape_c.axis(.k_chunk)]), - .stride_v_cache_2 = @intCast(v_strides_c[v_cache_shape_c.axis(.hkv)]), + .stride_k_cache_0 = k_strides[k_cache.axis(.page)], + .stride_k_cache_1 = k_strides[k_cache.axis(.k_chunk)], + .stride_k_cache_2 = k_strides[k_cache.axis(.hkv)], + .stride_v_cache_0 = v_strides[v_cache.axis(.page)], + .stride_v_cache_1 = v_strides[v_cache.axis(.k_chunk)], + .stride_v_cache_2 = v_strides[v_cache.axis(.hkv)], // Intel: keep KV loads cached (.none); the streaming .cg hint lowers to // fully-uncached here and defeats the L2 prefetcher. .kv_cache_modifier = .none, @@ -568,7 +537,7 @@ pub const paged = struct { log.debug("pagedAttention3dOneapi attention config: {any}", .{attn_kernel_config}); const reduce_kernel_config: kernels.ReduceSegmentsPtr.Config = .{ - .o_dtype = toDType(q.dtype()), + .o_dtype = triton.from(q.dtype()), .num_query_heads = @intCast(paged_attention_opts.num_heads), .tile_size = @intCast(config.reduce.tile_size), .head_size = @intCast(paged_attention_opts.head_dim), @@ -579,26 +548,14 @@ pub const paged = struct { }; log.debug("pagedAttention3d reduce config: {any}", .{reduce_kernel_config}); - const dummy = zml.Tensor.constant(zml.DataType.i8.zero()); + const dummy: zml.Tensor = .scalar(0, .i8); const block_table_strides = parameters.block_table.shape().computeElementStrides().constSlice(); - const block_table_strides_ptr = zml.Tensor.constant(zml.DataType.i64.constant(block_table_strides[0])); + const q_shape = q.shape().mergeAxes(.{ .h = .{ .hkv, .hg } }); const q_strides = q_shape.computeElementStrides().constSlice(); - const q_strides_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(q_strides[0])); - const q_strides_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(q_strides[1])); - const k_cache_shape = k_cache.shape(); - const k_strides = k_cache_shape.computeElementStrides().constSlice(); - const k_strides_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(k_strides[k_cache_shape.axis(.page)])); - const k_strides_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(k_strides[k_cache_shape.axis(.k_chunk)])); - const k_strides_2_ptr = zml.Tensor.constant(zml.DataType.i64.constant(k_strides[k_cache_shape.axis(.hkv)])); - const v_cache_shape = v_cache.shape(); - const v_strides = v_cache_shape.computeElementStrides().constSlice(); - const v_strides_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(v_strides[v_cache_shape.axis(.page)])); - const v_strides_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(v_strides[v_cache_shape.axis(.k_chunk)])); - const v_strides_2_ptr = zml.Tensor.constant(zml.DataType.i64.constant(v_strides[v_cache_shape.axis(.hkv)])); - const num_seqs_ptr = zml.Tensor.constant(zml.DataType.i32.constant(parameters.block_table.dim(0))); + const scale: f32 = paged_attention_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))); - const scale_ptr = zml.Tensor.constant(zml.DataType.f32.constant(scale)); + const num_seqs = parameters.block_table.dim(0); const attn_grid: [3]i32 = .{ @intCast(config.attention.total_q_blocks), @intCast(paged_attention_opts.num_kv_heads), @intCast(config.attention.num_segments_per_seq) }; const attn_output = kernels_oneapi.KernelUnifiedAttention3dPtr.Kernel.call( @@ -611,22 +568,22 @@ pub const paged = struct { .seq_lens_ptr = parameters.seq_lens, .alibi_slopes_ptr = dummy, .qq_bias_ptr = dummy, - .scale_ptr = scale_ptr, + .scale_ptr = .scalar(scale, .f32), .k_scale_ptr = dummy, .v_scale_ptr = dummy, .softcap_ptr = dummy, - .block_table_stride_ptr = block_table_strides_ptr, - .query_stride_0_ptr = q_strides_0_ptr, - .query_stride_1_ptr = q_strides_1_ptr, + .block_table_stride_ptr = .scalar(block_table_strides[0], .i64), + .query_stride_0_ptr = .scalar(q_strides[0], .i64), + .query_stride_1_ptr = .scalar(q_strides[1], .i64), .qq_bias_stride_0_ptr = dummy, - .stride_k_cache_0_ptr = k_strides_0_ptr, - .stride_k_cache_1_ptr = k_strides_1_ptr, - .stride_k_cache_2_ptr = k_strides_2_ptr, - .stride_v_cache_0_ptr = v_strides_0_ptr, - .stride_v_cache_1_ptr = v_strides_1_ptr, - .stride_v_cache_2_ptr = v_strides_2_ptr, + .stride_k_cache_0_ptr = .scalar(k_strides[k_cache.axis(.page)], .i64), + .stride_k_cache_1_ptr = .scalar(k_strides[k_cache.axis(.k_chunk)], .i64), + .stride_k_cache_2_ptr = .scalar(k_strides[k_cache.axis(.hkv)], .i64), + .stride_v_cache_0_ptr = .scalar(v_strides[v_cache.axis(.page)], .i64), + .stride_v_cache_1_ptr = .scalar(v_strides[v_cache.axis(.k_chunk)], .i64), + .stride_v_cache_2_ptr = .scalar(v_strides[v_cache.axis(.hkv)], .i64), .query_start_len_ptr = parameters.query_start_len, - .num_seqs_ptr = num_seqs_ptr, + .num_seqs_ptr = .scalar(num_seqs, .i32), }, .{ .segm_output = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq, std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim) }, .f32), @@ -647,11 +604,11 @@ pub const paged = struct { .segm_max_ptr = attn_output.segm_max, .segm_expsum_ptr = attn_output.segm_expsum, .seq_lens_ptr = parameters.seq_lens, - .num_seqs_ptr = num_seqs_ptr, + .num_seqs_ptr = .scalar(num_seqs, .i32), .out_scale_inv_ptr = dummy, - .output_stride_0_ptr = q_strides_0_ptr, - .output_stride_1_ptr = q_strides_1_ptr, - .block_table_stride_ptr = block_table_strides_ptr, + .output_stride_0_ptr = .scalar(q_strides[0], .i64), + .output_stride_1_ptr = .scalar(q_strides[1], .i64), + .block_table_stride_ptr = .scalar(block_table_strides[0], .i64), .query_start_len_ptr = parameters.query_start_len, }, .{ .output = q.shape() }, diff --git a/zml/buffer.zig b/zml/buffer.zig index 4e9ca6d7..991fe32a 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -8,6 +8,7 @@ const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const mem = @import("mem.zig"); const Memory = @import("platform.zig").Memory; +const meta = @import("meta.zig"); const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; @@ -76,6 +77,14 @@ pub const Buffer = struct { } } + pub fn deinitAll(T: type, buffers: *mem.Bufferized(T)) void { + meta.visitFlatStruct(struct { + fn deinit(_: void, x: *Buffer) void { + x.deinit(); + } + }.deinit, {}, buffers); + } + /// This Buffer shape. pub fn shape(self: Buffer) Shape { return self._shape; diff --git a/zml/meta.zig b/zml/meta.zig index dcc5c546..3090b609 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -445,12 +445,12 @@ pub fn VisitReturn(comptime cb: anytype) type { } const Visitor = struct { - pub const VisitorType = enum { + pub const Action = enum { callback, recurse, }; - pub fn determineAction(comptime callback: anytype, comptime PtrTypeOfV: type) VisitorType { + pub fn determineAction(comptime callback: anytype, comptime PtrTypeOfV: type) Action { const ptr_info = switch (@typeInfo(PtrTypeOfV)) { .pointer => |info| info, else => stdx.debug.compileError("zml.meta.visit({any}) is expecting a pointer/slice input, but received: {any}", .{ @TypeOf(callback), PtrTypeOfV }), @@ -478,10 +478,8 @@ pub fn visit(comptime callback: anytype, ctx: FnParam(callback, 0), v: anytype) const can_error = stdx.meta.FnReturnErrorSet(callback) != null; - switch (action) { - .callback => { - return if (can_error) try callback(ctx, v) else callback(ctx, v); - }, + return switch (action) { + .callback => callback(ctx, v), .recurse => { const TargetType, const mutating_cb = switch (@typeInfo(FnParam(callback, 1))) { .pointer => |info| .{ info.child, !info.is_const }, @@ -504,12 +502,14 @@ pub fn visit(comptime callback: anytype, ctx: FnParam(callback, 0), v: anytype) if (can_error) try visit(callback, ctx, &@field(v, field.name)) else visit(callback, ctx, &@field(v, field.name)); } }, - .array => inline for (v) |*elem| if (can_error) try visit(callback, ctx, elem) else visit(callback, ctx, elem), - .optional => if (v.* != null) if (can_error) try visit(callback, ctx, &v.*.?) else visit(callback, ctx, &v.*.?), - .@"union" => switch (v.*) { - inline else => |*v_field| if (can_error) try visit(callback, ctx, v_field) else visit(callback, ctx, v_field), + .array => for (v[0..]) |*elem| { + if (can_error) try visit(callback, ctx, elem) else visit(callback, ctx, elem); }, - .pointer => if (can_error) try visit(callback, ctx, v.*) else visit(callback, ctx, v.*), + .optional => return if (v.*) |*opt| visit(callback, ctx, opt), + .@"union" => return switch (v.*) { + inline else => |*v_field| visit(callback, ctx, v_field), + }, + .pointer => return visit(callback, ctx, v.*), else => stdx.debug.compileError("zml.meta.visit({any}) doesn't support visiting pointer type {any}", .{ @TypeOf(callback), ChildTypeV }), }, .slice => { @@ -520,7 +520,7 @@ pub fn visit(comptime callback: anytype, ctx: FnParam(callback, 0), v: anytype) .many, .c => stdx.debug.compileError("zml.meta.visit({any}) doesn't support [*] style pointers got {any}", .{ @TypeOf(callback), ChildTypeV }), } }, - } + }; } test visit { @@ -529,20 +529,26 @@ test visit { const NestedAttr = struct { nested: Attr }; const NestedAttrOptional = struct { nested: ?Attr }; const SimpleStruct = struct { prop: Attr }; - const MultipleTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: ?Attr }; + const MultipleTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: ?Attr, prop4: []Attr }; const NestedTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: NestedAttr, prop4: NestedAttrOptional, prop5: stdx.BoundedArray(Attr, 8) }; - const LocalContext = struct { result: usize }; + const LocalContext = struct { + result: usize, + + fn cb(ctx: *@This(), attr: *const Attr) void { + ctx.result += attr.data; + } + + fn mutCb(ctx: *@This(), attr: *Attr) void { + ctx.result += attr.data; + } + }; { var context: LocalContext = .{ .result = 0 }; const container: SimpleStruct = .{ .prop = .{ .data = 1 } }; - visit((struct { - fn cb(ctx: *LocalContext, attr: *const Attr) void { - ctx.result += attr.data; - } - }).cb, &context, &container); + visit(LocalContext.cb, &context, &container); try std.testing.expectEqual(1, context.result); } @@ -550,37 +556,28 @@ test visit { var context: LocalContext = .{ .result = 0 }; var container: SimpleStruct = .{ .prop = .{ .data = 1 } }; - visit((struct { - fn cb(ctx: *LocalContext, attr: *Attr) void { - ctx.result += attr.data; - } - }).cb, &context, &container); + visit(LocalContext.mutCb, &context, &container); try std.testing.expectEqual(1, context.result); } { var context: LocalContext = .{ .result = 0 }; - var container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = null }; + var more_props: [3]Attr = .{ .{ .data = 4 }, .{ .data = 5 }, .{ .data = 6 } }; - visit((struct { - fn cb(ctx: *LocalContext, attr: *Attr) void { - ctx.result += attr.data; - } - }).cb, &context, &container); + var container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = null, .prop4 = &more_props }; - try std.testing.expectEqual(1, context.result); + visit(LocalContext.mutCb, &context, &container); + + try std.testing.expectEqual(1 + 4 + 5 + 6, context.result); } { var context: LocalContext = .{ .result = 0 }; - const container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = .{ .data = 2 } }; + var more_props: [3]Attr = .{ .{ .data = 4 }, .{ .data = 5 }, .{ .data = 6 } }; + const container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = .{ .data = 2 }, .prop4 = &more_props }; - visit((struct { - fn cb(ctx: *LocalContext, attr: *const Attr) void { - ctx.result += attr.data; - } - }).cb, &context, &container); + visit(LocalContext.cb, &context, &container); - try std.testing.expectEqual(3, context.result); + try std.testing.expectEqual(1 + 2 + 4 + 5 + 6, context.result); } { var context: LocalContext = .{ .result = 0 }; @@ -596,16 +593,144 @@ test visit { .prop5 = prop5, // 4 will be counted twice. }; - visit((struct { - fn cb(ctx: *LocalContext, attr: *const Attr) void { - ctx.result += attr.data; - } - }).cb, &context, &container); + visit(LocalContext.cb, &context, &container); try std.testing.expectEqual(14, context.result); } } +/// Similar to `visit` but doesn't follow pointers or slices. +/// This guarantees that the given type is a flat struct. +/// +/// see: `zml.meta.visit` +pub fn visitFlatStruct(comptime callback: anytype, ctx: FnParam(callback, 0), v: anytype) VisitReturn(callback) { + const action = comptime Visitor.determineAction(callback, @TypeOf(v)); + + const TargetType = switch (@typeInfo(FnParam(callback, 1))) { + .pointer => |info| info.child, + else => stdx.debug.compileError("zml.meta.visitStruct({any}) is expecting a callback with a pointer as second argument but found {any}", .{ @TypeOf(callback), FnParam(callback, 1) }), + }; + + const PtrTypeOfV = @TypeOf(v); + const ptr_info_v = @typeInfo(PtrTypeOfV).pointer; + const ChildTypeV = ptr_info_v.child; + + const can_error = stdx.meta.FnReturnErrorSet(callback) != null; + const err_msg = "zml.meta.visitStruct only handles flat struct, found pointer in {}"; + const err_args = .{ChildTypeV}; + + switch (action) { + .callback => return callback(ctx, v), + + .recurse => { + if (comptime !Contains(ChildTypeV, TargetType)) return; + + // bounded array support (like visit) + if (@typeInfo(ChildTypeV) == .@"struct" and @hasDecl(ChildTypeV, "constSlice") and @hasDecl(ChildTypeV, "slice")) { + return for (v.slice()) |*elem| { + if (can_error) + try visitFlatStruct(callback, ctx, elem) + else + visitFlatStruct(callback, ctx, elem); + }; + } + + return switch (ptr_info_v.size) { + .one => switch (@typeInfo(ChildTypeV)) { + .@"struct" => |s| inline for (s.fields) |field| { + if (field.is_comptime or comptime !Contains(field.type, TargetType)) continue; + stdx.debug.assertComptime(@typeInfo(field.type) != .pointer, err_msg, err_args); + + if (can_error) try visitFlatStruct(callback, ctx, &@field(v, field.name)) else visitFlatStruct(callback, ctx, &@field(v, field.name)); + }, + .array => for (v[0..]) |*elem| { + if (can_error) try visitFlatStruct(callback, ctx, elem) else visitFlatStruct(callback, ctx, elem); + }, + .optional => if (v.*) |*opt| visitFlatStruct(callback, ctx, opt), + .@"union" => switch (v.*) { + inline else => |*v_field| visitFlatStruct(callback, ctx, v_field), + }, + else => stdx.debug.compileError(err_msg, err_args), + }, + else => stdx.debug.compileError(err_msg, err_args), + }; + }, + } +} + +test visitFlatStruct { + const Attr = struct { data: usize }; + const OtherAttr = struct { other: []const u8 }; + const NestedAttr = struct { nested: Attr }; + const NestedAttrOptional = struct { nested: ?Attr }; + const SimpleStruct = struct { prop: Attr }; + // Note: no slice anymore, no stdx.BoundedArray + const MultipleTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: ?Attr }; + const NestedTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: NestedAttr, prop4: NestedAttrOptional, prop5: stdx.BoundedArray(Attr, 8) }; + + const LocalContext = struct { + result: usize, + + fn cb(ctx: *@This(), attr: *const Attr) void { + ctx.result += attr.data; + } + + fn mutCb(ctx: *@This(), attr: *Attr) void { + ctx.result += attr.data; + } + }; + + { + var context: LocalContext = .{ .result = 0 }; + const container: SimpleStruct = .{ .prop = .{ .data = 1 } }; + + visitFlatStruct(LocalContext.cb, &context, &container); + + try std.testing.expectEqual(1, context.result); + } + { + var context: LocalContext = .{ .result = 0 }; + var container: SimpleStruct = .{ .prop = .{ .data = 1 } }; + + visitFlatStruct(LocalContext.mutCb, &context, &container); + + try std.testing.expectEqual(1, context.result); + } + { + var context: LocalContext = .{ .result = 0 }; + var container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = null }; + + visitFlatStruct(LocalContext.mutCb, &context, &container); + + try std.testing.expectEqual(1, context.result); + } + { + var context: LocalContext = .{ .result = 0 }; + const container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = .{ .data = 2 } }; + + visitFlatStruct(LocalContext.cb, &context, &container); + + try std.testing.expectEqual(1 + 2, context.result); + } + { + var context: LocalContext = .{ .result = 0 }; + const prop5: stdx.BoundedArray(Attr, 8) = .{ + .buffer = @splat(.{ .data = 4 }), + .len = 2, + }; + const container: NestedTypesStruct = .{ + .prop1 = .{ .data = 1 }, + .prop2 = .{ .other = "hello" }, + .prop3 = .{ .nested = .{ .data = 2 } }, + .prop4 = .{ .nested = .{ .data = 3 } }, + .prop5 = prop5, // 4 will be counted twice. + }; + + visitFlatStruct(LocalContext.cb, &context, &container); + + try std.testing.expectEqual(1 + 2 + 3 + 4 + 4, context.result); + } +} pub fn count(T: type, value: anytype) u32 { var counter: u32 = 0; visit(struct { diff --git a/zml/module.zig b/zml/module.zig index 391ec6fc..7ce89c05 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -174,6 +174,10 @@ pub const CompilationContext = struct { return _current.?; } + pub fn currentOrNull() ?*CompilationContext { + return _current; + } + pub fn currentScope(self: *CompilationContext) *Scope { return &self.scopes.slice()[self.scopes.len - 1]; } diff --git a/zml/tensor.zig b/zml/tensor.zig index 6a2c1cc9..5d22e934 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -148,10 +148,22 @@ pub const Tensor = struct { } pub fn withPartitioning(self: Tensor, axes_: anytype) Tensor { - const ctx = CompilationContext.current(); const partitioned_shape = self._shape.withPartitioning(axes_); - const attr = ctx.partitioning.tensorShardingAttr(ctx.allocator, ctx.mlir_ctx, partitioned_shape, null) catch unreachable; + const ctx = CompilationContext.currentOrNull() orelse { + var res = self; + res._shape = partitioned_shape; + return res; + }; + + const attr = ctx.partitioning.tensorShardingAttr(ctx.allocator, ctx.mlir_ctx, partitioned_shape, null) catch |err| switch (err) { + error.NoSuitableSharding => std.debug.panic( + "{f}.withPartitioning({f}) failed to resolve because it's using unknown sharding. Pass more shardings to `zml.compile`. Known shardings: {f}", + .{ self, partitioned_shape, stdx.fmt.slice(ctx.partitioning.shardings) }, + ), + error.OutOfMemory, error.WriteFailed => @panic("OOM"), + error.MissingDeviceInTile => @panic("TODO"), + }; const op_result = switch (ctx.partitioning.partitioner) { .shardy => blk: { @@ -186,6 +198,7 @@ pub const Tensor = struct { return _result(partitioned_shape, op_result); } + /// Copy the given tensor to the specified memory. pub fn toMemory(self: Tensor, kind: Memory.Kind) Tensor { const ctx = CompilationContext.current(); switch (ctx.platform.target) { @@ -221,6 +234,26 @@ pub const Tensor = struct { return res; } + /// Copy all the given tensor to the specified memory. + /// The input struct is copied on the stack, so it must be a simple flat struct without pointers. + pub fn toMemoryAll(flat_tensors: anytype, kind: Memory.Kind) @TypeOf(flat_tensors) { + const ctx = CompilationContext.current(); + switch (ctx.platform.target) { + .cpu, .neuron, .metal => return flat_tensors, + .cuda, .rocm, .tpu, .oneapi => {}, + } + + var copy = flat_tensors; + meta.visitFlatStruct(struct { + fn onMemory(k: Memory.Kind, x: *Tensor) void { + x.* = x.toMemory(k); + } + }.onMemory, kind, ©); + return copy; + } + + /// Mark the given input tensor as being physically located on a specific memory. + /// Has no effect if the input tensor is not an executable input. pub fn onMemory(self: Tensor, kind: Memory.Kind) Tensor { const ctx = CompilationContext.current(); switch (ctx.platform.target) { @@ -237,6 +270,23 @@ pub const Tensor = struct { return self; } + /// Mark all the given input tensors as being physically located on a specific memory + /// see `zml.Tensor.onMemory` + pub fn onMemoryAll(tensors: anytype, kind: Memory.Kind) void { + const ctx = CompilationContext.current(); + switch (ctx.platform.target) { + // Only one memory kind on those platform + .cpu, .neuron, .metal => return, + .cuda, .rocm, .tpu, .oneapi => {}, + } + + meta.visit(struct { + fn onMemory(k: Memory.Kind, x: *const Tensor) void { + _ = x.onMemory(k); + } + }.onMemory, kind, &tensors); + } + /// Returns a Tensor with new tag names. pub fn rename(self: Tensor, renames: anytype) Tensor { var res = self; From 2b9320fe721dfbbc0b3b03551d297e5433a6ec5d Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 30 Jul 2026 14:21:36 +0200 Subject: [PATCH 603/881] //zml/io/vfs: promote to //vfs (#680) Simplify vfs architecture: merge `//zml/io` and `//zml/io/vfs`. They move to top level `@zml//vfs` The PR also include two bug fix: - S3: when no content length we were trying to read exactly 1M, while we should read up to 1M - GCP: restore the commented code for parsing bucket info ## Test plan: ``` bazel run --config=debug //examples/io:playground -- ls gs://gcp-public-data-landsat/ bazel run --config=debug //examples/io:playground -- tree s3://noaa-goes19/ABI-Flood-Day-Shapefiles/2025/08 ``` --- examples/io/main.zig | 9 +- {zml/io/vfs => vfs}/BUILD.bazel | 2 +- {zml/io/vfs => vfs}/base.zig | 0 {zml/io/vfs => vfs}/file.zig | 2 +- {zml/io/vfs => vfs}/gcs.zig | 293 ++++++++++-------- {zml/io/vfs => vfs}/hf.zig | 2 +- {zml/io/vfs => vfs}/http.zig | 2 +- {zml/io/vfs => vfs}/parallel_read.zig | 0 {zml/io/vfs => vfs}/s3.zig | 21 +- vfs/vfs.zig | 409 +++++++++++++++++++++++++ zml/BUILD.bazel | 2 +- zml/io.zig | 4 +- zml/io/BUILD.bazel | 15 - zml/io/io.zig | 1 - zml/io/vfs.zig | 413 -------------------------- zml/io/vfs/index.zig | 10 - 16 files changed, 601 insertions(+), 584 deletions(-) rename {zml/io/vfs => vfs}/BUILD.bazel (93%) rename {zml/io/vfs => vfs}/base.zig (100%) rename {zml/io/vfs => vfs}/file.zig (99%) rename {zml/io/vfs => vfs}/gcs.zig (83%) rename {zml/io/vfs => vfs}/hf.zig (99%) rename {zml/io/vfs => vfs}/http.zig (99%) rename {zml/io/vfs => vfs}/parallel_read.zig (100%) rename {zml/io/vfs => vfs}/s3.zig (98%) create mode 100644 vfs/vfs.zig delete mode 100644 zml/io/BUILD.bazel delete mode 100644 zml/io/io.zig delete mode 100644 zml/io/vfs.zig delete mode 100644 zml/io/vfs/index.zig diff --git a/examples/io/main.zig b/examples/io/main.zig index d536b750..b512392c 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -9,15 +9,16 @@ pub const std_options: std.Options = .{ .log_level = .info, }; +const Command = enum { cat, tree, ls, cp, stat, realpath, safetensors, load }; + // -- ls hf://openai/gpt-oss-20b@6cee5e8 // -- ls hf://Qwen/Qwen3-235B-A22B-Instruct-2507 // -- ls hf://meta-llama/Llama-3.1-8B-Instruct@0e9e39f +// -- tree s3://noaa-goes19/ABI-Flood-Day-Shapefiles/2025/08 // -- cat https://iprs.fly.dev pub fn main(init: std.process.Init) !void { const allocator = init.gpa; - const Command = enum { cat, ls, cp, stat, realpath, safetensors, load }; - var it = init.minimal.args.iterate(); _ = it.next(); // skip program name const command: Command = std.meta.stringToEnum(Command, it.next() orelse return error.MissingCommand) orelse return error.CommandInvalid; @@ -72,7 +73,7 @@ pub fn main(init: std.process.Init) !void { try stdout_writer.interface.print("Wrote {B:.2} to stdout from {s}\n", .{ read, path }); }, - .ls => { + .ls, .tree => { var dir = try std.Io.Dir.openDir(.cwd(), io, path, .{ .iterate = true }); defer dir.close(io); @@ -80,7 +81,7 @@ pub fn main(init: std.process.Init) !void { try stdout_writer.interface.print("{s} - {B:.2}\n", .{ path, dir_stat.size }); var counts: TreeCounts = .{}; - try printTree(io, &stdout_writer.interface, dir, "", 10, &counts); + try printTree(io, &stdout_writer.interface, dir, "", if (command == .tree) 10 else 1, &counts); try stdout_writer.interface.print("\n{d} directories, {d} files\n", .{ counts.dirs, counts.files }); }, .cp => { diff --git a/zml/io/vfs/BUILD.bazel b/vfs/BUILD.bazel similarity index 93% rename from zml/io/vfs/BUILD.bazel rename to vfs/BUILD.bazel index dfa53d2c..0ebc39e1 100644 --- a/zml/io/vfs/BUILD.bazel +++ b/vfs/BUILD.bazel @@ -2,6 +2,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library") zig_library( name = "vfs", + main = "vfs.zig", srcs = [ "base.zig", "file.zig", @@ -11,7 +12,6 @@ zig_library( "parallel_read.zig", "s3.zig", ], - main = "index.zig", tags = ["manual"], visibility = ["//visibility:public"], deps = [ diff --git a/zml/io/vfs/base.zig b/vfs/base.zig similarity index 100% rename from zml/io/vfs/base.zig rename to vfs/base.zig diff --git a/zml/io/vfs/file.zig b/vfs/file.zig similarity index 99% rename from zml/io/vfs/file.zig rename to vfs/file.zig index 497c6b83..627f79d9 100644 --- a/zml/io/vfs/file.zig +++ b/vfs/file.zig @@ -5,7 +5,7 @@ const stdx = @import("stdx"); const VFSBase = @import("base.zig").VFSBase; -const log = std.log.scoped(.@"zml/io/vfs/file"); +const log = std.log.scoped(.@"zml/vfs/file"); pub const DirectIoError = error{ UnsupportedPlatform, diff --git a/zml/io/vfs/gcs.zig b/vfs/gcs.zig similarity index 83% rename from zml/io/vfs/gcs.zig rename to vfs/gcs.zig index 65fd07f8..3824aa15 100644 --- a/zml/io/vfs/gcs.zig +++ b/vfs/gcs.zig @@ -6,7 +6,7 @@ const stdx = @import("stdx"); const parallel_read = @import("parallel_read.zig"); const VFSBase = @import("base.zig").VFSBase; -const log = std.log.scoped(.@"zml/io/vfs/gcs"); +const log = std.log.scoped(.@"zml/vfs/gcs"); const EndpointUrl = "https://storage.googleapis.com"; const MetadataUrl = "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"; @@ -730,59 +730,54 @@ pub const GCS = struct { fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); - _ = reader; - _ = entries; - _ = self; - return 0; + if (reader.state == .finished) return 0; - // if (reader.state == .finished) return 0; + if (reader.state == .reset) { + if (self.dir_read_states.fetchRemove(reader)) |kv| { + for (kv.value.objects) |obj| self.allocator.free(obj); + self.allocator.free(kv.value.objects); + } - // if (reader.state == .reset) { - // if (self.dir_read_states.fetchRemove(reader)) |kv| { - // for (kv.value.objects) |obj| self.allocator.free(obj); - // self.allocator.free(kv.value.objects); - // } + const handle = self.getDirHandle(reader.dir); + const objects = self.listObjects(handle.uri) catch return std.Io.Dir.Reader.Error.Unexpected; - // const handle = self.getDirHandle(reader.dir); - // const objects = self.listObjects(handle.uri) catch return std.Io.Dir.Reader.Error.Unexpected; + self.dir_read_states.put(self.allocator, reader, .{ + .index = 0, + .objects = objects, + }) catch return std.Io.Dir.Reader.Error.Unexpected; - // self.dir_read_states.put(self.allocator, reader, .{ - // .index = 0, - // .objects = objects, - // }) catch return std.Io.Dir.Reader.Error.Unexpected; + reader.state = if (objects.len > 0) .reading else .finished; - // reader.state = if (objects.len > 0) .reading else .finished; + if (objects.len == 0) return 0; + } - // if (objects.len == 0) return 0; - // } + const state = self.dir_read_states.getPtr(reader) orelse return std.Io.Dir.Reader.Error.Unexpected; - // const state = self.dir_read_states.getPtr(reader) orelse return std.Io.Dir.Reader.Error.Unexpected; + var count: usize = 0; + while (count < entries.len and state.index < state.objects.len) { + const obj_key = state.objects[state.index]; + const kind: std.Io.File.Kind = if (std.mem.endsWith(u8, obj_key, "/")) .directory else .file; - // var count: usize = 0; - // while (count < entries.len and state.index < state.objects.len) { - // const obj_key = state.objects[state.index]; - // const kind: std.Io.File.Kind = if (std.mem.endsWith(u8, obj_key, "/")) .directory else .file; + const name = if (std.mem.lastIndexOfScalar(u8, std.mem.trimEnd(u8, obj_key, "/"), '/')) |idx| + obj_key[idx + 1 ..] + else + obj_key; - // const name = if (std.mem.lastIndexOfScalar(u8, std.mem.trimEnd(u8, obj_key, "/"), '/')) |idx| - // obj_key[idx + 1 ..] - // else - // obj_key; + entries[count] = .{ + .name = std.mem.trimEnd(u8, name, "/"), + .kind = kind, + .inode = state.index, + }; + count += 1; + state.index += 1; + } - // entries[count] = .{ - // .name = std.mem.trimEnd(u8, name, "/"), - // .kind = kind, - // .inode = state.index, - // }; - // count += 1; - // state.index += 1; - // } + if (state.index >= state.objects.len) { + reader.state = .finished; + } - // if (state.index >= state.objects.len) { - // reader.state = .finished; - // } - - // return count; + return count; } fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { @@ -884,16 +879,15 @@ pub const GCS = struct { }); } - // fn pathComponents(self: *GCS, path_: []const u8) struct { []const u8, []const u8, []const u8 } { - // const endpoint = std.mem.trimEnd(u8, self.config.endpoint_url, "/"); - // const path = std.mem.trim(u8, path_, "/"); + fn pathComponents(_: *GCS, path_: []const u8) struct { []const u8, []const u8 } { + const path = std.mem.trim(u8, path_, "/"); - // if (std.mem.findScalar(u8, path, '/')) |idx| { - // return .{ endpoint, path[0..idx], if (idx + 1 < path.len) path[idx + 1 ..] else "" }; - // } else { - // return .{ endpoint, path, "" }; - // } - // } + if (std.mem.indexOfScalar(u8, path, '/')) |idx| { + return .{ path[0..idx], if (idx + 1 < path.len) path[idx + 1 ..] else "" }; + } else { + return .{ path, "" }; + } + } fn gcsUri(self: *GCS, path: []const u8) std.Uri { var uri = self.config.endpoint_uri; @@ -932,99 +926,105 @@ pub const GCS = struct { // }; // } - // fn listObjects(self: *GCS, prefix: []const u8) ![][]const u8 { - // const endpoint, const bucket, const key_prefix = self.pathComponents(prefix); + fn listObjectsBody(self: *GCS, bucket: []const u8, key_prefix: []const u8) ![]u8 { + var query_buf: [4096]u8 = undefined; + var query_writer = std.Io.Writer.fixed(&query_buf); - // var query_buf: [4096]u8 = undefined; - // var query_writer = std.Io.Writer.fixed(&query_buf); + try query_writer.writeAll("delimiter="); + try std.Uri.Component.percentEncode(&query_writer, "/", gcsEncodeIsValid); + try query_writer.writeAll("&max-keys=1000"); - // try query_writer.writeAll("delimiter="); - // try std.Uri.Component.percentEncode(&query_writer, "/", gcsEncodeIsValid); + if (key_prefix.len > 0) { + try query_writer.writeAll("&prefix="); + try std.Uri.Component.percentEncode(&query_writer, key_prefix, gcsEncodeIsValid); + if (!std.mem.endsWith(u8, key_prefix, "/")) { + try std.Uri.Component.percentEncode(&query_writer, "/", gcsEncodeIsValid); + } + } - // if (key_prefix.len > 0) { - // try query_writer.writeAll("&prefix="); - // try std.Uri.Component.percentEncode(&query_writer, key_prefix, gcsEncodeIsValid); - // try std.Uri.Component.percentEncode(&query_writer, "/", gcsEncodeIsValid); - // } + var path_buf: [256]u8 = undefined; + var uri = self.config.endpoint_uri; + uri.path = .{ .percent_encoded = try std.fmt.bufPrint(&path_buf, "/{s}", .{bucket}) }; + uri.query = .{ .percent_encoded = query_writer.buffered() }; - // const endpoint_uri = try std.Uri.parse(endpoint); + var req = try self.client.request(.GET, uri, .{ + .redirect_behavior = .not_allowed, + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = try self.getOrRefreshToken(), + }, + .extra_headers = if (self.quotaProjectId()) |project| + &.{.{ .name = "x-goog-user-project", .value = project }} + else + &.{}, + }); + defer req.deinit(); - // var path_buf: [256]u8 = undefined; + try req.sendBodiless(); - // const uri: std.Uri = .{ - // .scheme = endpoint_uri.scheme, - // .host = endpoint_uri.host, - // .port = endpoint_uri.port, - // .path = .{ .percent_encoded = try std.fmt.bufPrint(&path_buf, "/{s}", .{bucket}) }, - // .query = .{ .percent_encoded = query_writer.buffered() }, - // .fragment = null, - // }; + var redirect_buffer: [2 * 1024]u8 = undefined; + var res = try req.receiveHead(&redirect_buffer); - // var timestamp_buf: [16]u8 = undefined; - // const timestamp = try self.getTimestamp(×tamp_buf); + if (res.head.status != .ok) { + log.err("Failed to list object {f}", .{uri}); + log.err("{s}", .{res.head.bytes}); + return error.RequestFailed; + } - // var authorization_buffer: [1024]u8 = undefined; - // const auth = try self.authHeaders(.GET, uri, timestamp, &authorization_buffer); + return if (res.head.content_length) |content_len| + try res.reader(&.{}).readAlloc(self.allocator, content_len) + else + try res.reader(&.{}).allocRemaining(self.allocator, .limited(1024 * 1024)); + } - // var extra_headers_buf: [3]std.http.Header = undefined; - // for (0..auth.extra_len) |i| { - // extra_headers_buf[i] = auth.extra_headers[i].?; - // } + fn listObjects(self: *GCS, prefix: []const u8) ![][]const u8 { + const bucket, const key_prefix = self.pathComponents(prefix); + const body = try self.listObjectsBody(bucket, key_prefix); + defer self.allocator.free(body); - // var req = try self.client.request(.GET, uri, .{ - // .redirect_behavior = .not_allowed, - // .headers = .{ - // .accept_encoding = .{ .override = "identity" }, - // .authorization = if (self.getOrRefreshToken()) |hdr| .{ .override = hdr } else .omit, - // }, - // .extra_headers = if (self.config.quota_project_id) |project| - // &.{.{ .name = "x-goog-user-project", .value = project }} - // else - // &.{}, - // }); - // defer req.deinit(); + return try self.parseListObjectsResponse(body, key_prefix); + } - // try req.sendBodiless(); + fn gcsEncodeIsValid(c: u8) bool { + return switch (c) { + 'A'...'Z', 'a'...'z', '0'...'9', '-', '.', '_', '~' => true, + else => false, + }; + } - // var redirect_buffer: [2 * 1024]u8 = undefined; - // var res = try req.receiveHead(&redirect_buffer); + fn quotaProjectId(self: *const GCS) ?[]const u8 { + const credentials = self.config.credentials orelse return null; + return switch (credentials) { + .authorized_user => |authorized_user| authorized_user.quota_project_id, + .service_account => |service_account| service_account.quota_project_id, + .metadata_server => null, + }; + } - // if (res.head.status != .ok) { - // log.err("Failed to list object {f}", .{uri}); - // log.err("{s}", .{res.head.bytes}); - // return error.RequestFailed; - // } + fn parseListObjectsResponse(self: *GCS, xml: []const u8, prefix: []const u8) ![][]const u8 { + var results: std.ArrayListUnmanaged([]const u8) = .empty; + errdefer { + for (results.items) |item| self.allocator.free(item); + results.deinit(self.allocator); + } - // const body = try res.reader(&.{}).readAlloc(self.allocator, res.head.content_length orelse 1024 * 1024); - // defer self.allocator.free(body); + inline for (.{ "Key", "Prefix" }) |tag| { + var pos: usize = 0; + while (std.mem.indexOfPos(u8, xml, pos, "<" ++ tag ++ ">")) |start| { + const v_start = start + tag.len + 2; + const end = std.mem.indexOfPos(u8, xml, v_start, "") orelse break; + const val = xml[v_start..end]; - // return try self.parseListObjectsResponse(body, key_prefix); - // } + const is_self = std.mem.eql(u8, std.mem.trimEnd(u8, val, "/"), std.mem.trimEnd(u8, prefix, "/")); + if (val.len > 0 and !is_self) { + try results.append(self.allocator, try self.allocator.dupe(u8, val)); + } + pos = end + tag.len + 3; + } + } - // fn parseListObjectsResponse(self: *GCS, xml: []const u8, prefix: []const u8) ![][]const u8 { - // var results = std.ArrayListUnmanaged([]const u8){}; - // errdefer { - // for (results.items) |item| self.allocator.free(item); - // results.deinit(self.allocator); - // } - - // inline for (.{ "Key", "Prefix" }) |tag| { - // var pos: usize = 0; - // while (std.mem.indexOfPos(u8, xml, pos, "<" ++ tag ++ ">")) |start| { - // const v_start = start + tag.len + 2; - // const end = std.mem.indexOfPos(u8, xml, v_start, "") orelse break; - // const val = xml[v_start..end]; - - // const is_self = std.mem.eql(u8, std.mem.trimEnd(u8, val, "/"), std.mem.trimEnd(u8, prefix, "/")); - // if (val.len > 0 and !is_self) { - // try results.append(self.allocator, try self.allocator.dupe(u8, val)); - // } - // pos = end + tag.len + 3; - // } - // } - - // return try results.toOwnedSlice(self.allocator); - // } + return try results.toOwnedSlice(self.allocator); + } fn fetchSize(self: *GCS, dir: std.Io.Dir, sub_path: []const u8) !u64 { var path_buffer: [8 * 1024]u8 = undefined; @@ -1095,3 +1095,40 @@ pub const GCS = struct { return read_size; } }; + +test "GCS parses XML bucket listing objects and common prefixes" { + const xml = + "" ++ + "" ++ + "bucket" ++ + "LC08/01/001/002/" ++ + "LC08/01/001/002/scene_$folder$6" ++ + "LC08/01/001/002/scene/" ++ + "LC08/01/001/002/" ++ + ""; + + var backend: GCS = undefined; + backend.allocator = std.testing.allocator; + + const objects = try backend.parseListObjectsResponse(xml, "LC08/01/001/002/"); + defer { + for (objects) |object| std.testing.allocator.free(object); + std.testing.allocator.free(objects); + } + + try std.testing.expectEqual(@as(usize, 2), objects.len); + try std.testing.expectEqualStrings("LC08/01/001/002/scene_$folder$", objects[0]); + try std.testing.expectEqualStrings("LC08/01/001/002/scene/", objects[1]); +} + +test "GCS splits bucket and object prefix" { + var backend: GCS = undefined; + + const root_bucket, const root_prefix = backend.pathComponents("gcp-public-data-landsat/"); + try std.testing.expectEqualStrings("gcp-public-data-landsat", root_bucket); + try std.testing.expectEqualStrings("", root_prefix); + + const bucket, const prefix = backend.pathComponents("gcp-public-data-landsat/LC08/01"); + try std.testing.expectEqualStrings("gcp-public-data-landsat", bucket); + try std.testing.expectEqualStrings("LC08/01", prefix); +} diff --git a/zml/io/vfs/hf.zig b/vfs/hf.zig similarity index 99% rename from zml/io/vfs/hf.zig rename to vfs/hf.zig index c3415e4c..2a1e9249 100644 --- a/zml/io/vfs/hf.zig +++ b/vfs/hf.zig @@ -5,7 +5,7 @@ const stdx = @import("stdx"); const parallel_read = @import("parallel_read.zig"); const VFSBase = @import("base.zig").VFSBase; -const log = std.log.scoped(.@"zml/io/vfs/hf"); +const log = std.log.scoped(.@"zml/vfs/hf"); const ParallelRead = struct { const Pool = parallel_read.Pool(Job); diff --git a/zml/io/vfs/http.zig b/vfs/http.zig similarity index 99% rename from zml/io/vfs/http.zig rename to vfs/http.zig index 2661a51b..ed8c5ee4 100644 --- a/zml/io/vfs/http.zig +++ b/vfs/http.zig @@ -4,7 +4,7 @@ const stdx = @import("stdx"); const VFSBase = @import("base.zig").VFSBase; -const log = std.log.scoped(.@"zml/io/vfs/http"); +const log = std.log.scoped(.@"zml/vfs/http"); pub const HTTP = struct { const Handle = struct { diff --git a/zml/io/vfs/parallel_read.zig b/vfs/parallel_read.zig similarity index 100% rename from zml/io/vfs/parallel_read.zig rename to vfs/parallel_read.zig diff --git a/zml/io/vfs/s3.zig b/vfs/s3.zig similarity index 98% rename from zml/io/vfs/s3.zig rename to vfs/s3.zig index 3936dfe3..51adcc7c 100644 --- a/zml/io/vfs/s3.zig +++ b/vfs/s3.zig @@ -5,7 +5,7 @@ const stdx = @import("stdx"); const parallel_read = @import("parallel_read.zig"); const VFSBase = @import("base.zig").VFSBase; -const log = std.log.scoped(.@"zml/io/vfs/s3"); +const log = std.log.scoped(.@"zml/vfs/s3"); pub const AwsSigV4 = struct { access_key: ?[]const u8, @@ -791,9 +791,7 @@ pub const S3 = struct { return try std.fmt.bufPrint(buf, "{s}/{s}/{s}", .{ endpoint, bucket, key }); } - fn listObjects(self: *S3, prefix: []const u8) ![][]const u8 { - const endpoint, const bucket, const key_prefix = self.pathComponents(prefix); - + fn listObjectsBody(self: *S3, endpoint: []const u8, bucket: []const u8, key_prefix: []const u8) ![]u8 { var query_buf: [4096]u8 = undefined; var query_writer = std.Io.Writer.fixed(&query_buf); @@ -804,7 +802,9 @@ pub const S3 = struct { if (key_prefix.len > 0) { try query_writer.writeAll("&prefix="); try std.Uri.Component.percentEncode(&query_writer, key_prefix, s3EncodeIsValid); - try std.Uri.Component.percentEncode(&query_writer, "/", s3EncodeIsValid); + if (!std.mem.endsWith(u8, key_prefix, "/")) { + try std.Uri.Component.percentEncode(&query_writer, "/", s3EncodeIsValid); + } } const endpoint_uri = try std.Uri.parse(endpoint); @@ -857,7 +857,16 @@ pub const S3 = struct { return error.RequestFailed; } - const body = try res.reader(&.{}).readAlloc(self.allocator, res.head.content_length orelse 1024 * 1024); + return if (res.head.content_length) |content_len| + try res.reader(&.{}).readAlloc(self.allocator, content_len) + else + // When we don't have a content length, put a reasonable limit + try res.reader(&.{}).allocRemaining(self.allocator, .limited(1024 * 1024)); + } + + fn listObjects(self: *S3, prefix: []const u8) ![][]const u8 { + const endpoint, const bucket, const key_prefix = self.pathComponents(prefix); + const body = try self.listObjectsBody(endpoint, bucket, key_prefix); defer self.allocator.free(body); return try self.parseListObjectsResponse(body, key_prefix); diff --git a/vfs/vfs.zig b/vfs/vfs.zig new file mode 100644 index 00000000..b5e5c580 --- /dev/null +++ b/vfs/vfs.zig @@ -0,0 +1,409 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +pub const File = @import("file.zig").File; +pub const GCS = @import("gcs.zig").GCS; +pub const HF = @import("hf.zig").HF; +pub const HTTP = @import("http.zig").HTTP; +pub const S3 = @import("s3.zig").S3; +pub const VFSBase = @import("base.zig").VFSBase; + +const log = std.log.scoped(.@"zml/vfs"); + +const CWD_HANDLE: u32 = 0; + +const VFS = @This(); +const Handle = struct { handle: u32, backend_idx: ?usize, flags: std.Io.File.Flags = .{ .nonblocking = false } }; + +allocator: std.mem.Allocator, +mutex: std.Io.Mutex = .init, + +backends: std.StringArrayHashMapUnmanaged(std.Io) = .empty, +handles: stdx.SegmentedList(Handle, 128) = .{}, +closed_handles: std.ArrayList(u32) = .empty, + +base: VFSBase, + +pub fn init(allocator: std.mem.Allocator, base_io: std.Io) !VFS { + const base = VFSBase.init(base_io); + + var handles: @FieldType(@This(), "handles") = .{}; + try handles.append(allocator, .{ .handle = CWD_HANDLE, .backend_idx = null }); + try handles.append(allocator, .{ .handle = std.posix.STDIN_FILENO, .backend_idx = null }); + try handles.append(allocator, .{ .handle = std.posix.STDOUT_FILENO, .backend_idx = null }); + try handles.append(allocator, .{ .handle = std.posix.STDERR_FILENO, .backend_idx = null }); + + return .{ + .allocator = allocator, + .handles = handles, + .base = base, + }; +} + +pub fn deinit(self: *VFS) void { + self.handles.deinit(self.allocator); + self.closed_handles.deinit(self.allocator); + self.backends.deinit(self.allocator); +} + +pub fn register(self: *VFS, scheme: []const u8, backend: std.Io) std.mem.Allocator.Error!void { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + try self.backends.put(self.allocator, scheme, backend); +} + +pub fn unregister(self: *VFS, scheme: []const u8) bool { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + return self.backends.remove(scheme); +} + +pub fn io(self: *VFS) std.Io { + return .{ + .userdata = &self.base, + .vtable = &comptime VFSBase.vtable(.{ + .operate = operate, + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirCreateFile = dirCreateFile, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRead = dirRead, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileWritePositional = fileWritePositional, + .fileWriteFileStreaming = fileWriteFileStreaming, + .fileWriteFilePositional = fileWriteFilePositional, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileRealPath = fileRealPath, + }), + }; +} + +fn openHandle(self: *VFS) !struct { u32, *Handle } { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + if (self.closed_handles.pop()) |idx| { + return .{ idx, self.handles.at(idx) }; + } + return .{ @intCast(self.handles.len), try self.handles.addOne(self.allocator) }; +} + +fn closeHandle(self: *VFS, idx: u32) !void { + if (idx == CWD_HANDLE) return; + + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + try self.closed_handles.append(self.allocator, idx); +} + +fn getFileHandle(self: *VFS, file: std.Io.File) struct { *Handle, std.Io } { + self.mutex.lockUncancelable(self.base.inner); + const handle = self.handles.at(@intCast(file.handle)); + self.mutex.unlock(self.base.inner); + + return .{ handle, self.getBackend(handle.backend_idx) }; +} + +fn getDirHandle(self: *VFS, dir: std.Io.Dir) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + if (std.meta.eql(dir, std.Io.Dir.cwd())) return self.handles.at(CWD_HANDLE); + return self.handles.at(@intCast(dir.handle)); +} + +fn getScheme(self: *VFS, backend_idx: ?usize) ?[]const u8 { + if (backend_idx) |idx| return self.backends.entries.items(.key)[idx] else return null; +} + +fn getBackend(self: *VFS, backend_idx: ?usize) std.Io { + if (backend_idx) |idx| return self.backends.entries.items(.value)[idx] else return self.base.inner; +} + +fn lookupDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?usize, std.Io.Dir, std.Io } { + // A scheme-qualified path (e.g. "hf://owner/model/file") is absolute: it + // is resolved from the scheme's root regardless of `dir`. Without this, + // opening such a path relative to an already-open dir of the same + // backend double-prefixes the dir's path. + if (sub_path) |sp| { + if (std.mem.indexOf(u8, sp, "://") != null) { + const uri = std.Uri.parse(sp) catch return error.VFSNotRegistered; + const backend_idx: usize = for (self.backends.entries.items(.key), 0..) |s, idx| { + if (std.mem.eql(u8, uri.scheme, s)) break idx; + } else return error.VFSNotRegistered; + return .{ backend_idx, std.Io.Dir.cwd(), self.getBackend(backend_idx) }; + } + } + + if (std.meta.eql(dir, std.Io.Dir.cwd())) { + if (sub_path == null) return .{ null, dir, self.base.inner }; + if (std.fs.path.isAbsolutePosix(sub_path.?)) return .{ null, dir, self.base.inner }; + + const uri = std.Uri.parse(sub_path.?) catch null; + if (uri) |u| { + const backend_idx: ?usize = for (self.backends.entries.items(.key), 0..) |s, idx| { + if (std.mem.eql(u8, u.scheme, s)) break idx; + } else null; + if (backend_idx == null) return error.VFSNotRegistered; + return .{ backend_idx, std.Io.Dir.cwd(), self.getBackend(backend_idx) }; + } else { + return .{ null, std.Io.Dir.cwd(), self.base.inner }; + } + } else { + const handle = self.getDirHandle(dir); + if (handle.backend_idx) |backend_idx| { + return .{ backend_idx, .{ .handle = @intCast(handle.handle) }, self.getBackend(backend_idx) }; + } else { + return .{ null, .{ .handle = @intCast(handle.handle) }, self.base.inner }; + } + } +} + +fn stripScheme(path: []const u8) []const u8 { + const uri = std.Uri.parse(path) catch return path; + return path[uri.scheme.len + 3 ..]; +} + +fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + switch (operation) { + .file_read_streaming => |o| { + const handle, const backend = self.getFileHandle(o.file); + return backend.vtable.operate(backend.userdata, .{ .file_read_streaming = .{ + .file = .{ .handle = @intCast(handle.handle), .flags = handle.flags }, + .data = o.data, + } }); + }, + .device_io_control, .file_write_streaming, .net_receive => { + return self.base.inner.vtable.operate(self.base.inner.userdata, operation); + }, + } +} + +fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for opening dir '{s}' : {any}", .{ sub_path, err }); + return std.Io.Dir.OpenError.Unexpected; + }; + const fs_dir = try backend.vtable.dirOpenDir(backend.userdata, dir_, stripScheme(sub_path), options); + const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; + handle.* = .{ + .handle = @intCast(fs_dir.handle), + .backend_idx = backend_idx, + }; + return .{ .handle = @intCast(idx) }; +} + +fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + _, const dir_, const backend = self.lookupDir(dir, null) catch |err| { + log.err("Failed to lookup backend for dir stat : {any}", .{err}); + return std.Io.Dir.StatError.Unexpected; + }; + return backend.vtable.dirStat(backend.userdata, dir_); +} + +pub fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + _, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for dir stat file '{s}' : {any}", .{ sub_path, err }); + return std.Io.Dir.StatFileError.Unexpected; + }; + return backend.vtable.dirStatFile(backend.userdata, dir_, stripScheme(sub_path), options); +} + +pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + _, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for dir access '{s}' : {any}", .{ sub_path, err }); + return std.Io.Dir.AccessError.Unexpected; + }; + return backend.vtable.dirAccess(backend.userdata, dir_, stripScheme(sub_path), options); +} + +pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.CreateFlags) std.Io.File.OpenError!std.Io.File { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for dir create file '{s}' : {any}", .{ sub_path, err }); + return std.Io.File.OpenError.Unexpected; + }; + + const file = try backend.vtable.dirCreateFile(backend.userdata, dir_, stripScheme(sub_path), flags); + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; + handle.* = .{ + .handle = @intCast(file.handle), + .backend_idx = backend_idx, + }; + return .{ .handle = @intCast(idx), .flags = .{ .nonblocking = false } }; +} + +fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for opening file '{s}' : {any}", .{ sub_path, err }); + return std.Io.File.OpenError.Unexpected; + }; + const file = try backend.vtable.dirOpenFile(backend.userdata, dir_, stripScheme(sub_path), flags); + const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; + handle.* = .{ + .handle = @intCast(file.handle), + .backend_idx = backend_idx, + }; + return .{ .handle = @intCast(idx), .flags = handle.flags }; +} + +fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + for (dirs) |dir| { + _, const dir_, const backend = self.lookupDir(dir, null) catch |err| { + log.err("Failed to lookup backend for closing dir : {any}", .{err}); + continue; + }; + backend.vtable.dirClose(backend.userdata, &.{dir_}); + self.closeHandle(@intCast(dir.handle)) catch unreachable; + } +} + +fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + _, const dir_, const backend = self.lookupDir(reader.dir, null) catch |err| { + log.err("Failed to lookup backend for dir real path : {any}", .{err}); + return std.Io.Dir.Reader.Error.Unexpected; + }; + + const original_dir = reader.dir; + reader.dir = dir_; + defer reader.dir = original_dir; + + return backend.vtable.dirRead(backend.userdata, reader, entries); +} + +fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const backend_idx, const dir_, const backend = self.lookupDir(dir, null) catch |err| { + log.err("Failed to lookup backend for dir real path : {any}", .{err}); + return std.Io.Dir.RealPathError.Unexpected; + }; + + if (self.getScheme(backend_idx)) |s| { + const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); + const path_len = try backend.vtable.dirRealPath(backend.userdata, dir_, out_buffer[prefix.len..]); + return prefix.len + path_len; + } else { + return try backend.vtable.dirRealPath(backend.userdata, dir_, out_buffer); + } +} + +fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const backend_idx, const dir_, const backend = self.lookupDir(dir, path_name) catch |err| { + log.err("Failed to lookup backend for dir real path file '{s}' : {any}", .{ path_name, err }); + return std.Io.Dir.RealPathFileError.Unexpected; + }; + + if (self.getScheme(backend_idx)) |s| { + const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); + const path_len = try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer[prefix.len..]); + return prefix.len + path_len; + } else { + return try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer); + } +} + +fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileStat(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }); +} + +fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileLength(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }); +} + +fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + for (files) |file| { + const handle, const backend = self.getFileHandle(file); + backend.vtable.fileClose(backend.userdata, &.{.{ .handle = @intCast(handle.handle), .flags = handle.flags }}); + self.closeHandle(@intCast(file.handle)) catch unreachable; + } +} + +fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) std.Io.File.WritePositionalError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileWritePositional(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, header, data, splat, offset); +} + +fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + + const dst_handle, const dst_backend = self.getFileHandle(file); + const src_handle, _ = self.getFileHandle(reader.file); + + const original_src_handle = reader.file; + reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; + defer reader.file = original_src_handle; + + return dst_backend.vtable.fileWriteFileStreaming(dst_backend.userdata, .{ .handle = @intCast(dst_handle.handle), .flags = dst_handle.flags }, header, reader, limit); +} + +fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + + const dst_handle, const dst_backend = self.getFileHandle(file); + const src_handle, _ = self.getFileHandle(reader.file); + + const original_src_handle = reader.file; + reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; + defer reader.file = original_src_handle; + + return dst_backend.vtable.fileWriteFilePositional(dst_backend.userdata, .{ .handle = @intCast(dst_handle.handle), .flags = dst_handle.flags }, header, reader, limit, offset); +} + +fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileReadPositional(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, data, offset); +} + +fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileSeekBy(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, relative_offset); +} + +fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileSeekTo(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, absolute_offset); +} + +fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + + if (self.getScheme(handle.backend_idx)) |s| { + const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); + const path_len = try backend.vtable.fileRealPath(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, out_buffer[prefix.len..]); + return prefix.len + path_len; + } else { + return try backend.vtable.fileRealPath(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, out_buffer); + } +} diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index a843515b..14049cf9 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -152,7 +152,7 @@ zig_library( "//stdx", "//tools/xspace_to_perfetto:xspace_to_perfetto_lib", "//upb", - "//zml/io", + "//vfs", "//zml/tokenizer", "@rules_zig//zig/runfiles", ":profiling_traceme_lib", diff --git a/zml/io.zig b/zml/io.zig index c70828a2..248e469d 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -2,12 +2,12 @@ const std = @import("std"); const pjrt = @import("pjrt"); const stdx = @import("stdx"); -pub const VFS = @import("io").VFS; +pub const VFS = @import("vfs"); -const Exe = @import("exe.zig").Exe; const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("zml.zig").Bufferized; const CreateOptions = @import("platform.zig").CreateOptions; +const Exe = @import("exe.zig").Exe; const mem = @import("mem.zig"); const Memory = @import("platform.zig").Memory; const meta = @import("meta.zig"); diff --git a/zml/io/BUILD.bazel b/zml/io/BUILD.bazel deleted file mode 100644 index e12eaa01..00000000 --- a/zml/io/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "io", - tags = ["manual"], - srcs = [ - "vfs.zig", - ], - main = "io.zig", - visibility = ["//visibility:public"], - deps = [ - "//stdx", - "//zml/io/vfs", - ], -) diff --git a/zml/io/io.zig b/zml/io/io.zig deleted file mode 100644 index 78c59e21..00000000 --- a/zml/io/io.zig +++ /dev/null @@ -1 +0,0 @@ -pub const VFS = @import("vfs.zig").VFS; diff --git a/zml/io/vfs.zig b/zml/io/vfs.zig deleted file mode 100644 index 93d3981a..00000000 --- a/zml/io/vfs.zig +++ /dev/null @@ -1,413 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); -const vfs = @import("vfs"); - -const log = std.log.scoped(.@"zml/io/vfs"); - -pub const VFS = struct { - pub const File = vfs.File; - pub const HTTP = vfs.HTTP; - pub const HF = vfs.HF; - pub const S3 = vfs.S3; - pub const GCS = vfs.GCS; - const VFSBase = vfs.VFSBase; - - const ClosedHandles = std.ArrayList(u32); - - const CWD_HANDLE: u32 = 0; - - const Handle = struct { handle: u32, backend_idx: ?usize, flags: std.Io.File.Flags = .{ .nonblocking = false } }; - - allocator: std.mem.Allocator, - mutex: std.Io.Mutex = .init, - - backends: std.StringArrayHashMapUnmanaged(std.Io) = .empty, - handles: stdx.SegmentedList(Handle, 128) = .{}, - closed_handles: ClosedHandles = .empty, - - base: VFSBase, - - pub fn init(allocator: std.mem.Allocator, base_io: std.Io) !VFS { - const base = VFSBase.init(base_io); - - var handles: @FieldType(@This(), "handles") = .{}; - try handles.append(allocator, .{ .handle = CWD_HANDLE, .backend_idx = null }); - try handles.append(allocator, .{ .handle = std.posix.STDIN_FILENO, .backend_idx = null }); - try handles.append(allocator, .{ .handle = std.posix.STDOUT_FILENO, .backend_idx = null }); - try handles.append(allocator, .{ .handle = std.posix.STDERR_FILENO, .backend_idx = null }); - - return .{ - .allocator = allocator, - .handles = handles, - .base = base, - }; - } - - pub fn deinit(self: *VFS) void { - self.handles.deinit(self.allocator); - self.closed_handles.deinit(self.allocator); - self.backends.deinit(self.allocator); - } - - pub fn register(self: *VFS, scheme: []const u8, backend: std.Io) std.mem.Allocator.Error!void { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - try self.backends.put(self.allocator, scheme, backend); - } - - pub fn unregister(self: *VFS, scheme: []const u8) bool { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - return self.backends.remove(scheme); - } - - pub fn io(self: *VFS) std.Io { - return .{ - .userdata = &self.base, - .vtable = &comptime VFSBase.vtable(.{ - .operate = operate, - .dirOpenDir = dirOpenDir, - .dirStat = dirStat, - .dirStatFile = dirStatFile, - .dirAccess = dirAccess, - .dirCreateFile = dirCreateFile, - .dirOpenFile = dirOpenFile, - .dirClose = dirClose, - .dirRead = dirRead, - .dirRealPath = dirRealPath, - .dirRealPathFile = dirRealPathFile, - .fileStat = fileStat, - .fileLength = fileLength, - .fileClose = fileClose, - .fileWritePositional = fileWritePositional, - .fileWriteFileStreaming = fileWriteFileStreaming, - .fileWriteFilePositional = fileWriteFilePositional, - .fileReadPositional = fileReadPositional, - .fileSeekBy = fileSeekBy, - .fileSeekTo = fileSeekTo, - .fileRealPath = fileRealPath, - }), - }; - } - - fn openHandle(self: *VFS) !struct { u32, *Handle } { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - if (self.closed_handles.pop()) |idx| { - return .{ idx, self.handles.at(idx) }; - } - return .{ @intCast(self.handles.len), try self.handles.addOne(self.allocator) }; - } - - fn closeHandle(self: *VFS, idx: u32) !void { - if (idx == CWD_HANDLE) return; - - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - try self.closed_handles.append(self.allocator, idx); - } - - fn getFileHandle(self: *VFS, file: std.Io.File) struct { *Handle, std.Io } { - self.mutex.lockUncancelable(self.base.inner); - const handle = self.handles.at(@intCast(file.handle)); - self.mutex.unlock(self.base.inner); - - return .{ handle, self.getBackend(handle.backend_idx) }; - } - - fn getDirHandle(self: *VFS, dir: std.Io.Dir) *Handle { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - if (std.meta.eql(dir, std.Io.Dir.cwd())) return self.handles.at(CWD_HANDLE); - return self.handles.at(@intCast(dir.handle)); - } - - fn getScheme(self: *VFS, backend_idx: ?usize) ?[]const u8 { - if (backend_idx) |idx| return self.backends.entries.items(.key)[idx] else return null; - } - - fn getBackend(self: *VFS, backend_idx: ?usize) std.Io { - if (backend_idx) |idx| return self.backends.entries.items(.value)[idx] else return self.base.inner; - } - - fn lookupDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?usize, std.Io.Dir, std.Io } { - // A scheme-qualified path (e.g. "hf://owner/model/file") is absolute: it - // is resolved from the scheme's root regardless of `dir`. Without this, - // opening such a path relative to an already-open dir of the same - // backend double-prefixes the dir's path. - if (sub_path) |sp| { - if (std.mem.indexOf(u8, sp, "://") != null) { - const uri = std.Uri.parse(sp) catch return error.VFSNotRegistered; - const backend_idx: usize = for (self.backends.entries.items(.key), 0..) |s, idx| { - if (std.mem.eql(u8, uri.scheme, s)) break idx; - } else return error.VFSNotRegistered; - return .{ backend_idx, std.Io.Dir.cwd(), self.getBackend(backend_idx) }; - } - } - - if (std.meta.eql(dir, std.Io.Dir.cwd())) { - if (sub_path == null) return .{ null, dir, self.base.inner }; - if (std.fs.path.isAbsolutePosix(sub_path.?)) return .{ null, dir, self.base.inner }; - - const uri = std.Uri.parse(sub_path.?) catch null; - if (uri) |u| { - const backend_idx: ?usize = for (self.backends.entries.items(.key), 0..) |s, idx| { - if (std.mem.eql(u8, u.scheme, s)) break idx; - } else null; - if (backend_idx == null) return error.VFSNotRegistered; - return .{ backend_idx, std.Io.Dir.cwd(), self.getBackend(backend_idx) }; - } else { - return .{ null, std.Io.Dir.cwd(), self.base.inner }; - } - } else { - const handle = self.getDirHandle(dir); - if (handle.backend_idx) |backend_idx| { - return .{ backend_idx, .{ .handle = @intCast(handle.handle) }, self.getBackend(backend_idx) }; - } else { - return .{ null, .{ .handle = @intCast(handle.handle) }, self.base.inner }; - } - } - } - - fn stripScheme(path: []const u8) []const u8 { - const uri = std.Uri.parse(path) catch return path; - return path[uri.scheme.len + 3 ..]; - } - - fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - switch (operation) { - .file_read_streaming => |o| { - const handle, const backend = self.getFileHandle(o.file); - return backend.vtable.operate(backend.userdata, .{ .file_read_streaming = .{ - .file = .{ .handle = @intCast(handle.handle), .flags = handle.flags }, - .data = o.data, - } }); - }, - .device_io_control, .file_write_streaming, .net_receive => { - return self.base.inner.vtable.operate(self.base.inner.userdata, operation); - }, - } - } - - fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { - log.err("Failed to lookup backend for opening dir '{s}' : {any}", .{ sub_path, err }); - return std.Io.Dir.OpenError.Unexpected; - }; - const fs_dir = try backend.vtable.dirOpenDir(backend.userdata, dir_, stripScheme(sub_path), options); - const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; - handle.* = .{ - .handle = @intCast(fs_dir.handle), - .backend_idx = backend_idx, - }; - return .{ .handle = @intCast(idx) }; - } - - fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - _, const dir_, const backend = self.lookupDir(dir, null) catch |err| { - log.err("Failed to lookup backend for dir stat : {any}", .{err}); - return std.Io.Dir.StatError.Unexpected; - }; - return backend.vtable.dirStat(backend.userdata, dir_); - } - - pub fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - _, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { - log.err("Failed to lookup backend for dir stat file '{s}' : {any}", .{ sub_path, err }); - return std.Io.Dir.StatFileError.Unexpected; - }; - return backend.vtable.dirStatFile(backend.userdata, dir_, stripScheme(sub_path), options); - } - - pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - _, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { - log.err("Failed to lookup backend for dir access '{s}' : {any}", .{ sub_path, err }); - return std.Io.Dir.AccessError.Unexpected; - }; - return backend.vtable.dirAccess(backend.userdata, dir_, stripScheme(sub_path), options); - } - - pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.CreateFlags) std.Io.File.OpenError!std.Io.File { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { - log.err("Failed to lookup backend for dir create file '{s}' : {any}", .{ sub_path, err }); - return std.Io.File.OpenError.Unexpected; - }; - - const file = try backend.vtable.dirCreateFile(backend.userdata, dir_, stripScheme(sub_path), flags); - const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; - handle.* = .{ - .handle = @intCast(file.handle), - .backend_idx = backend_idx, - }; - return .{ .handle = @intCast(idx), .flags = .{ .nonblocking = false } }; - } - - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { - log.err("Failed to lookup backend for opening file '{s}' : {any}", .{ sub_path, err }); - return std.Io.File.OpenError.Unexpected; - }; - const file = try backend.vtable.dirOpenFile(backend.userdata, dir_, stripScheme(sub_path), flags); - const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; - handle.* = .{ - .handle = @intCast(file.handle), - .backend_idx = backend_idx, - }; - return .{ .handle = @intCast(idx), .flags = handle.flags }; - } - - fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - for (dirs) |dir| { - _, const dir_, const backend = self.lookupDir(dir, null) catch |err| { - log.err("Failed to lookup backend for closing dir : {any}", .{err}); - continue; - }; - backend.vtable.dirClose(backend.userdata, &.{dir_}); - self.closeHandle(@intCast(dir.handle)) catch unreachable; - } - } - - fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - _, const dir_, const backend = self.lookupDir(reader.dir, null) catch |err| { - log.err("Failed to lookup backend for dir real path : {any}", .{err}); - return std.Io.Dir.Reader.Error.Unexpected; - }; - - const original_dir = reader.dir; - reader.dir = dir_; - defer reader.dir = original_dir; - - return backend.vtable.dirRead(backend.userdata, reader, entries); - } - - fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const backend_idx, const dir_, const backend = self.lookupDir(dir, null) catch |err| { - log.err("Failed to lookup backend for dir real path : {any}", .{err}); - return std.Io.Dir.RealPathError.Unexpected; - }; - - if (self.getScheme(backend_idx)) |s| { - const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); - const path_len = try backend.vtable.dirRealPath(backend.userdata, dir_, out_buffer[prefix.len..]); - return prefix.len + path_len; - } else { - return try backend.vtable.dirRealPath(backend.userdata, dir_, out_buffer); - } - } - - fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const backend_idx, const dir_, const backend = self.lookupDir(dir, path_name) catch |err| { - log.err("Failed to lookup backend for dir real path file '{s}' : {any}", .{ path_name, err }); - return std.Io.Dir.RealPathFileError.Unexpected; - }; - - if (self.getScheme(backend_idx)) |s| { - const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); - const path_len = try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer[prefix.len..]); - return prefix.len + path_len; - } else { - return try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer); - } - } - - fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle, const backend = self.getFileHandle(file); - return backend.vtable.fileStat(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }); - } - - fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle, const backend = self.getFileHandle(file); - return backend.vtable.fileLength(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }); - } - - fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - for (files) |file| { - const handle, const backend = self.getFileHandle(file); - backend.vtable.fileClose(backend.userdata, &.{.{ .handle = @intCast(handle.handle), .flags = handle.flags }}); - self.closeHandle(@intCast(file.handle)) catch unreachable; - } - } - - fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) std.Io.File.WritePositionalError!usize { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle, const backend = self.getFileHandle(file); - return backend.vtable.fileWritePositional(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, header, data, splat, offset); - } - - fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - - const dst_handle, const dst_backend = self.getFileHandle(file); - const src_handle, _ = self.getFileHandle(reader.file); - - const original_src_handle = reader.file; - reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; - defer reader.file = original_src_handle; - - return dst_backend.vtable.fileWriteFileStreaming(dst_backend.userdata, .{ .handle = @intCast(dst_handle.handle), .flags = dst_handle.flags }, header, reader, limit); - } - - fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - - const dst_handle, const dst_backend = self.getFileHandle(file); - const src_handle, _ = self.getFileHandle(reader.file); - - const original_src_handle = reader.file; - reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; - defer reader.file = original_src_handle; - - return dst_backend.vtable.fileWriteFilePositional(dst_backend.userdata, .{ .handle = @intCast(dst_handle.handle), .flags = dst_handle.flags }, header, reader, limit, offset); - } - - fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle, const backend = self.getFileHandle(file); - return backend.vtable.fileReadPositional(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, data, offset); - } - - fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle, const backend = self.getFileHandle(file); - return backend.vtable.fileSeekBy(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, relative_offset); - } - - fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle, const backend = self.getFileHandle(file); - return backend.vtable.fileSeekTo(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, absolute_offset); - } - - fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle, const backend = self.getFileHandle(file); - - if (self.getScheme(handle.backend_idx)) |s| { - const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); - const path_len = try backend.vtable.fileRealPath(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, out_buffer[prefix.len..]); - return prefix.len + path_len; - } else { - return try backend.vtable.fileRealPath(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, out_buffer); - } - } -}; diff --git a/zml/io/vfs/index.zig b/zml/io/vfs/index.zig deleted file mode 100644 index 9e6ca3c1..00000000 --- a/zml/io/vfs/index.zig +++ /dev/null @@ -1,10 +0,0 @@ -const log = std.log.scoped(.@"zml/io/vfs/vfs"); - -const std = @import("std"); - -pub const File = @import("file.zig").File; -pub const GCS = @import("gcs.zig").GCS; -pub const HF = @import("hf.zig").HF; -pub const HTTP = @import("http.zig").HTTP; -pub const S3 = @import("s3.zig").S3; -pub const VFSBase = @import("base.zig").VFSBase; From 648c5fd2ee1718067d876c97ee91552a0444d13e Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 31 Jul 2026 10:57:02 +0200 Subject: [PATCH 604/881] zml/attention: stablehlo paged attention (#688) Implement paged_attention in pure stablehlo. introduces `zml.ops.if` and `zml.ops.if2` The first one is similar to `zml.ops.while` and requires a full struct with two functions. it's a bit heavy to use, but it follows the stablehlo API, and if one of the branch has side effects it will correctly evaluate the side effect only in that branch. `zml.ops.if2` is a simpler version that doesn't require to create intermediary struct, but as a consequence, if you have side effect they will always evaluate, and it relies on the compiler to correctly avoid duplicated compute by moving relevant compute inside the branches. (open to feedback about if/if2) --- mlir/mlir.zig | 4 +- zml/attention/paged_attention.zig | 288 ++++++++++++++++++++++++++++-- zml/floats.zig | 13 +- zml/nn.zig | 4 +- zml/ops.zig | 215 ++++++++++++++++++++++ zml/platform.zig | 2 +- zml/slice.zig | 4 +- zml/tensor.zig | 11 +- 8 files changed, 507 insertions(+), 34 deletions(-) diff --git a/mlir/mlir.zig b/mlir/mlir.zig index a6efea3e..02559eb7 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -1294,11 +1294,11 @@ pub const Operation = opaque { state.addAttributes(attrs); } if (args.blocks) |blocks| { - const body_region = Region.init(); for (blocks) |block_| { + const body_region = Region.init(); body_region.appendOwnedBlock(block_); + state.addOwnedRegions(&.{body_region}); } - state.addOwnedRegions(&.{body_region}); } const new_op = try Operation.init(&state); errdefer new_op.deinit(); diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 898a5a26..156dd556 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -22,7 +22,7 @@ pub const Backend = enum { triton, mosaic_tpu, metal, - // vanilla, + stablehlo, pub fn auto(platform: *const zml.Platform) Backend { return switch (platform.target) { @@ -31,14 +31,14 @@ pub const Backend = enum { .oneapi => .triton, .tpu => .mosaic_tpu, .metal => .metal, - // .cpu => .vanilla - .cpu, .neuron => stdx.debug.panic("Paged attention is not supported on {s} yet", .{@tagName(platform.target)}), + .cpu => .stablehlo, + .neuron => stdx.debug.panic("Paged attention is not supported on {s} yet", .{@tagName(platform.target)}), }; } pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { return switch (backend) { - // .vanilla => true, + .stablehlo => true, .triton => platform.target != .cpu, .metal => platform.target == .metal, .mosaic_tpu => platform.target == .tpu, @@ -59,6 +59,7 @@ pub const Options = union(Backend) { triton: triton.paged.Options, mosaic_tpu: tpu.mosaic_tpu.Options, metal: metal.paged.Options, + stablehlo: triton.paged.Options, const Args = struct { backend: Backend, @@ -143,7 +144,7 @@ pub const Options = union(Backend) { .head_dim = args.head_dim, }, }, - inline .triton, .metal => |t| @unionInit(Options, @tagName(t), .{ + inline .triton, .metal, .stablehlo => |t| @unionInit(Options, @tagName(t), .{ .batch_size = args.batch_size, .max_num_pages = args.max_num_pages, .max_seqlen_q = args.max_seqlen_q, @@ -171,6 +172,7 @@ pub const Parameters = union(Backend) { triton: triton.paged.Parameters, mosaic_tpu: tpu.mosaic_tpu.Parameters, metal: metal.paged.Parameters, + stablehlo: triton.paged.Parameters, pub fn init(options_: Options) Parameters { return switch (options_) { @@ -225,7 +227,7 @@ pub const KvCache = union(enum) { var kv: KvCache = switch (self) { .split => |split| switch (backend) { - .cuda_fa2, .cuda_fa3, .triton, .mosaic_tpu, .metal => .{ .split = .{ + .cuda_fa2, .cuda_fa3, .triton, .mosaic_tpu, .metal, .stablehlo => .{ .split = .{ .k = split.k.scatterSlices( .{ .page = page_index, .k_chunk = offset }, new_k, @@ -250,6 +252,19 @@ pub const KvCache = union(enum) { const offset = slot_mapping.remainder(.scalar(page_size, slot_mapping.dtype())); return .{ page_index, offset }; } + + pub fn getPage( + self: KvCache, + page_index: zml.Tensor, + ) [2]zml.Tensor { + return switch (self) { + .split => |split| .{ + split.k.dynamicSlice1d(split.k.axis(.page), .{ .start = page_index, .len = 1 }).squeeze(.page), + split.v.dynamicSlice1d(split.k.axis(.page), .{ .start = page_index, .len = 1 }).squeeze(.page), + }, + .dense => @panic("TODO"), + }; + } }; pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache, opts: AttentionOptions) zml.Tensor { @@ -273,6 +288,7 @@ pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: z .split => |split| metal.paged.pagedAttention(metal_parameters, q, split.k, split.v, opts), .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), }, + .stablehlo => |params| stablehlo_pagedAttention(params, q, kv_cache, opts), }; } @@ -328,13 +344,13 @@ test pagedAttention { const dt: zml.DataType = .bf16; const partition = .{ .hkv = .model }; const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache } = .{ - .q = .withPartitioning(.init(.{ .b = query_token_count, .hkv = 2, .hg = 4, .hd = 32 }, dt), partition), - .k = .withPartitioning(.init(.{ .b = query_token_count, .hkv = 2, .hd = 32 }, dt), partition), - .v = .withPartitioning(.init(.{ .b = query_token_count, .hkv = 2, .hd = 32 }, dt), partition), + .q = .withPartitioning(.init(.{ .b = query_token_count, .hkv = 4, .hg = 4, .hd = 32 }, dt), partition), + .k = .withPartitioning(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt), partition), + .v = .withPartitioning(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt), partition), .kv_cache = .{ .split = .{ - .k = .withPartitioning(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 2, .hd = 32 }, dt), partition), - .v = .withPartitioning(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 2, .hd = 32 }, dt), partition), + .k = .withPartitioning(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt), partition), + .v = .withPartitioning(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt), partition), }, }, }; @@ -469,7 +485,7 @@ test pagedAttention { } } }; }, .triton => triton_parameters_d, - inline .metal, .mosaic_tpu => |_, t| @unionInit(zml.Bufferized(Parameters), @tagName(t), .{ + inline .metal, .mosaic_tpu, .stablehlo => |_, t| @unionInit(zml.Bufferized(Parameters), @tagName(t), .{ .block_table = triton_parameters_d.triton.block_table, .seq_lens = triton_parameters_d.triton.seq_lens, .query_start_len = triton_parameters_d.triton.query_start_len, @@ -503,8 +519,8 @@ test pagedAttention { std.log.err("test pagedAttention failed on backend {0t}\n--> reference ({1t}): {2d:32.3}\n--> pagedAttention({0t}): {3d:32.3}", .{ backend, reference_backend, - reference.subSlice(1, 0, 1).subSlice(2, 0, 1), - output_h.subSlice(1, 0, 1).subSlice(2, 0, 1), + reference.subSlice(1, 0, 1).squeeze(1).subSlice(1, 0, 1).squeeze(1), + output_h.subSlice(1, 0, 1).squeeze(1).subSlice(1, 0, 1).squeeze(1), }); }, else => |e| return e, @@ -513,3 +529,247 @@ test pagedAttention { if (num_failed > 0) return error.TestUnexpectedResult; } + +fn stablehlo_pagedAttention( + parameters: triton.paged.Parameters, + q: zml.Tensor, + kv_cache: KvCache, + opts: AttentionOptions, +) zml.Tensor { + const page_size = kv_cache.split.k.dim(.k_chunk); + + const final_state = zml.ops.@"while"( + AttentionLoop, + .{ .q = q, .kv_cache = kv_cache, .parameters = parameters, .opts = opts }, + .{ + // i32 to match triton conventions + .slot_id = .scalar(0, .i32), + .seq_id = .scalar(0, .i32), + .q_page_idx = .scalar(0, .i32), + .k_page_idx = .scalar(0, .i32), + // Note: we do all partial softmax in f32 + .partial_softmax_prefill = .zeroes(q.shape().setDim(.b, page_size).withDtype(.f32)), + .partial_softmax_decode = .zeroes(q.shape().setDim(.b, 1).withDtype(.f32)), + .out = .zeroes(q.shape()), + }, + ); + + return final_state.out; +} + +const AttentionLoop = struct { + q: zml.Tensor, + kv_cache: KvCache, + parameters: triton.paged.Parameters, + opts: AttentionOptions, + + const While = @This(); + + pub const State = struct { + slot_id: zml.Tensor, + seq_id: zml.Tensor, + q_page_idx: zml.Tensor, + k_page_idx: zml.Tensor, + partial_softmax_prefill: PartialSoftmax, + partial_softmax_decode: PartialSoftmax, + out: zml.Tensor, + }; + + pub fn cond(self: While, state: State) zml.Tensor { + const has_more_sequences = state.seq_id.cmp(.LT, .scalar(self.parameters.seq_lens.count(), .i32)); + const query_start = self.parameters.query_start_len.dynamicSlice1d(0, .{ .start = state.seq_id, .len = 1 }); + const query_end = self.parameters.query_start_len.dynamicSlice1d(0, .{ .start = state.seq_id.addConstant(1), .len = 1 }); + const has_queries = query_start.cmp(.LT, query_end).asScalar(); + return has_more_sequences.logical(.AND, has_queries); + } + + pub fn body(self: While, state_: State) State { + const query_start = self.parameters.query_start_len.dynamicSlice1d(0, .{ .start = state_.seq_id, .len = 1 }); + const query_end = self.parameters.query_start_len.dynamicSlice1d(0, .{ .start = state_.seq_id.addConstant(1), .len = 1 }); + const seq_num_queries_: zml.Tensor = .asScalar(.sub(query_end, query_start)); + const page_size_: u32 = @intCast(self.kv_cache.split.k.dim(.k_chunk)); + + return zml.ops.@"if"( + struct { + const IfPrefill = @This(); + while_body: While, + state: State, + seq_num_queries: zml.Tensor, + page_size: u32, + + /// Prefill version: num_slots == page_size, we multiply a full page of queries with a page of keys. + pub fn onTrue(if_ctx: IfPrefill) State { + const while_body = if_ctx.while_body; + const state = if_ctx.state; + const page_size = if_ctx.page_size; + const num_slots: u32 = page_size; + + // seq_lens includes the current queries; subtracting their count gives the context length. + const seq_len = while_body.parameters.seq_lens.dynamicSlice1d(0, .{ .start = state.seq_id, .len = 1 }).asScalar(); + const q_offset = seq_len.sub(if_ctx.seq_num_queries).add(state.q_page_idx.scale(page_size)); + const next_k_offset = state.k_page_idx.addConstant(1).scale(page_size); + + // The current K page is processed in this iteration. Continue if the next page contains + // any key visible to the current query chunk, whose exclusive end is q_offset + num_slots. + const query_has_more_keys: zml.Tensor = .cmp(next_k_offset, .LT, q_offset.addConstant(num_slots)); + const sequence_has_more_queries: zml.Tensor = .cmp(state.q_page_idx.addConstant(1).scale(num_slots), .LT, if_ctx.seq_num_queries); + + const partial_softmax = while_body.attentionOnePage(state, q_offset, num_slots, page_size); + return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax); + } + + /// Decode version: num_slots == 1, we multiply one query with a page of keys. + pub fn onFalse(if_ctx: IfPrefill) State { + const while_body = if_ctx.while_body; + const state = if_ctx.state; + const page_size = if_ctx.page_size; + const num_slots: u32 = 1; + + // A decode query is the final token in seq_len, so its zero-based offset is seq_len - 1. + const seq_len = while_body.parameters.seq_lens.dynamicSlice1d(0, .{ .start = state.seq_id, .len = 1 }).asScalar(); + const q_offset = seq_len.sub(if_ctx.seq_num_queries); + const next_k_offset = state.k_page_idx.addConstant(1).scale(page_size); + + const query_has_more_keys: zml.Tensor = .cmp(next_k_offset, .LT, q_offset.addConstant(num_slots)); + const sequence_has_more_queries: zml.Tensor = .scalar(false, .bool); + + const partial_softmax = while_body.attentionOnePage(state, q_offset, num_slots, page_size); + return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax); + } + }, + .{ + .while_body = self, + .state = state_, + .seq_num_queries = seq_num_queries_, + .page_size = page_size_, + }, + .cmp(seq_num_queries_, .GT, .scalar(1, .i32)), + ); + } + + pub fn attentionOnePage(self: While, state: State, q_offset: zml.Tensor, q_chunk: u32, page_size: u32) PartialSoftmax { + const k_offset = state.k_page_idx.scale(page_size); + const active_q = self.q.dynamicSlice1d(self.q.axis(.b), .{ .start = state.slot_id, .len = q_chunk }); + + const active_k_page_id = self.parameters.block_table.dynamicSlice(.{ + .b = zml.Tensor.DynSlice{ .start = state.seq_id, .len = 1 }, + .p = zml.Tensor.DynSlice{ .start = state.k_page_idx, .len = 1 }, + }).asScalar(); + const active_k, const active_v = self.kv_cache.getPage(active_k_page_id); + + const dtype = self.q.dtype(); + const attn_mask: zml.Tensor = attn_mask: { + const mask_shape: zml.Shape = .init(.{ .b = q_chunk, .k_chunk = page_size }, dtype); + + const q_idx = zml.Tensor.iota(mask_shape, .b).add(q_offset); + const k_idx = zml.Tensor.iota(mask_shape, .k_chunk).add(k_offset); + + const mask = zml.Tensor.cmp(k_idx, .LE, q_idx); + const minus_inf = zml.Tensor.constant(dtype.minValue()); + break :attn_mask .select(mask, .scalar(0, dtype), minus_inf); + }; + + const scale: f32 = self.opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(active_q.dim(.hd))))); + var attn_weights = active_q.dot(active_k, .hd).scale(scale); + attn_weights = attn_weights.add(attn_mask.broad(attn_weights.shape())); + + const og_softmax = if (q_chunk == 1) state.partial_softmax_decode else state.partial_softmax_prefill; + // Note: we compute q * k using q and k dtype, but the partial softmax is in f32. + const precise_dt = og_softmax.values.dtype(); + var partial = partialSoftmax(attn_weights.convert(precise_dt), .k_chunk); + partial.values = partial.values.dot(active_v.convert(precise_dt), .k_chunk).transpose(og_softmax.values.shape()); + partial.max_value = partial.max_value.transpose(og_softmax.max_value.shape()); + partial.exp_sum = partial.exp_sum.transpose(og_softmax.exp_sum.shape()); + + return og_softmax.merge(partial); + } + + pub fn updateState(state: State, num_slots: u32, query_has_more_keys: zml.Tensor, sequence_has_more_queries: zml.Tensor, softmax: PartialSoftmax) State { + return zml.ops.if2( + query_has_more_keys, + // If the query has more pages: update partial softmax with attention on current page, delay updating output. + State{ + .slot_id = state.slot_id, + .seq_id = state.seq_id, + .q_page_idx = state.q_page_idx, + .k_page_idx = state.k_page_idx.addConstant(1), + .partial_softmax_prefill = if (num_slots == 1) state.partial_softmax_prefill else softmax, + .partial_softmax_decode = if (num_slots == 1) softmax else state.partial_softmax_decode, + .out = state.out, + }, + // Query has no more pages, we commit softmax to output and increment slot + State{ + .k_page_idx = .scalar(0, .i32), + .partial_softmax_prefill = if (num_slots == 1) state.partial_softmax_prefill else .reset(softmax), + .partial_softmax_decode = if (num_slots == 1) .reset(softmax) else state.partial_softmax_decode, + .out = state.out.dynamicUpdateSlice(.{ .b = state.slot_id }, softmax.finalize().convert(state.out.dtype())), + .slot_id = state.slot_id.addConstant(num_slots), + // If the sequence has more query chunks: increment q_page_idx, otherwise start over at 0. + .q_page_idx = .select(sequence_has_more_queries, state.q_page_idx.addConstant(1), .scalar(0, .i32)), + // If the sequence has more query chunks: keep seq_id, otherwise increment seq_id. + .seq_id = .select(sequence_has_more_queries, state.seq_id, state.seq_id.addConstant(1)), + }, + ); + } +}; + +pub const PartialSoftmax = struct { + values: zml.Tensor, + exp_sum: zml.Tensor, + max_value: zml.Tensor, + + pub fn zeroes(q_shape: zml.Shape) PartialSoftmax { + return .{ + .values = .zeroes(q_shape), + .exp_sum = .zeroes(q_shape.drop(.hd)), + .max_value = .broad(.constant(q_shape.dtype().minValue()), q_shape.drop(.hd)), + }; + } + + pub fn reset(self: PartialSoftmax) PartialSoftmax { + return .zeroes(self.values.shape()); + } + + pub fn merge(self: PartialSoftmax, other: PartialSoftmax) PartialSoftmax { + // Rescale self and other using the new global_max. + const global_max = self.max_value.maximum(other.max_value.convert(self.max_value.dtype())); + const new_self = self.rescale(global_max); + const new_other = other.rescale(global_max); + + // Now that self and other are using the same scale, we can just add them: + return .{ + .max_value = global_max, + .values = new_self.values.add(new_other.values), + .exp_sum = new_self.exp_sum.add(new_other.exp_sum), + }; + } + + /// Update max_value and rescale attn and exp_sum accordingly. + pub fn rescale(self: PartialSoftmax, max_value: zml.Tensor) PartialSoftmax { + const max_diff_exp: zml.Tensor = .exp(self.max_value.sub(max_value)); + const sum_dtype = self.exp_sum.dtype(); + return .{ + .max_value = max_value, + .values = self.values.mul(max_diff_exp.broad(self.values.shape())), + .exp_sum = self.exp_sum.mul(max_diff_exp.convert(sum_dtype)), + }; + } + + /// Divides the intermediary results by the exp_sum to get the proper attention values. + pub fn finalize(self: PartialSoftmax) zml.Tensor { + return self.values.div(self.exp_sum.broad(self.values.shape()).convert(self.values.dtype())); + } +}; + +/// Compute softmax over a chunk. +/// Returns intermediary results to allow aggregating later. +pub fn partialSoftmax(self: zml.Tensor, axis: anytype) PartialSoftmax { + const a = self.axis(axis); + const max_val = self.max(a).squeeze(a).maximum(.scalar(-1e16, self.dtype())); + const out = self.sub(max_val.broad(self.shape())).exp(); + return .{ + .values = out, + .exp_sum = out.convert(.f32).sum(a).squeeze(a), + .max_value = max_val, + }; +} diff --git a/zml/floats.zig b/zml/floats.zig index 538722cd..3a0efb00 100644 --- a/zml/floats.zig +++ b/zml/floats.zig @@ -29,16 +29,15 @@ fn FloatConversionHelpers(Float: type) type { const vf32: Float32 = @bitCast(f); const exponent: i16 = @as(i16, vf32.exponent) - exp_off; - const overflow = exponent > std.math.maxInt(@FieldType(Float, "exponent")); + const overflow = std.math.isInf(f) or exponent > std.math.maxInt(@FieldType(Float, "exponent")); if (overflow) { @branchHint(.unlikely); return if (@hasDecl(Float, "inf")) if (vf32.sign == 0) Float.inf else Float.minus_inf else if (@hasDecl(Float, "nan")) Float.nan - else b: { - break :b if (vf32.sign == 0) Float.max else Float.min; - }; + else + std.debug.panic("{} doesn't have infinite representations", .{Float}); } return if (exponent <= 0) @@ -271,10 +270,8 @@ test "Float8E4" { }) |Float8T| { try testCustomFloat(Float8T, test_case_e4); try std.testing.expectEqual(0.0, Float8T.fromF32(1.0 / 2048.0).toF32()); - if (@hasDecl(Float8T, "inf")) { - try std.testing.expectEqual(Float8T.inf, Float8T.fromF32(128.0)); - try std.testing.expectEqual(Float8T.inf.neg(), Float8T.fromF32(-128.0)); - } + + if (@hasDecl(Float8T, "inf")) @compileError("Float8E4 don't have ±∞ representations"); } } diff --git a/zml/nn.zig b/zml/nn.zig index 949063b9..c87f25b4 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -1289,9 +1289,7 @@ pub fn causalAttnMask( } if (dtype.isFloat()) { - const zeros = Tensor.constant(dtype.zero()).broad(mask.shape()); - const minus_inf = Tensor.constant(dtype.minValue()).broad(mask.shape()); - mask = Tensor.select(mask, zeros, minus_inf); + mask = Tensor.select(mask, .scalar(0, dtype), .scalar(-std.math.inf(f32), dtype)); } else { mask = mask.convert(dtype); } diff --git a/zml/ops.zig b/zml/ops.zig index c77d4573..eadd1772 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -651,6 +651,221 @@ test @"while" { try std.testing.expectEqual(@as(i64, 9), cpu_sum.items(i64)[0]); } +/// Implement if/else branches. +/// The given If struct contains all the parameters that can be referenced in the if/else branches. +/// +/// It must have: +/// * an `onTrue(If) T` function returning some tensors +/// * an `onFalse(If) T` function returning similar shaped tensors +/// +/// If some expression in `onTrue` or `onFalse` has side effects (like printing), +/// stablehlo guarantees they will only be evaluated in its branch. +pub fn @"if"( + If: type, + if_captures: If, + pred: Tensor, +) stdx.meta.FnReturn(If.onTrue) { + stdx.debug.assertComptime( + stdx.meta.FnReturn(If.onTrue) == stdx.meta.FnReturn(If.onFalse), + "zml.ops.if expects same return types for onTrue and onFalse, got {s} and {s}", + .{ @typeName(stdx.meta.FnReturn(If.onTrue)), @typeName(stdx.meta.FnReturn(If.onFalse)) }, + ); + + stdx.debug.assert(pred.dtype() == .bool and pred.count() == 1, "zml.ops.if expects the condition to have exactly one element of dtype .bool, got {f}", .{pred}); + + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const allocator = arena.allocator(); + var blkctx: If = undefined; + + // Explicitly capture values from the parent block. + meta.mapAlloc(struct { + fn capture(_: void, tensor: Tensor) Tensor { + return Tensor._result(tensor.shape(), tensor.value()); + } + }.capture, arena.allocator(), {}, if_captures, &blkctx) catch unreachable; + + const mlir_ctx = CompilationContext.current().mlir_ctx; + const loc: *const mlir.Location = .unknown(mlir_ctx); + + const true_branch, const true_branch_block = b: { + const block = mlir.Block.init(&.{}, &.{}); + errdefer block.deinit(); + + CompilationContext.current().pushBlock(block); + defer CompilationContext.current().popBlock(); + + const result = blkctx.onTrue(); + const result_values = meta.collectAlloc(Tensor.value, {}, allocator, &result) catch @panic("OOM"); + defer allocator.free(result_values); + _ = dialects.stablehlo.returns(mlir_ctx, result_values, loc).appendTo(block); + break :b .{ result, block }; + }; + + const false_branch, const false_branch_block = b: { + const block = mlir.Block.init(&.{}, &.{}); + errdefer block.deinit(); + + CompilationContext.current().pushBlock(block); + defer CompilationContext.current().popBlock(); + + const result = blkctx.onFalse(); + const result_values = meta.collectAlloc(Tensor.value, {}, allocator, &result) catch @panic("OOM"); + defer allocator.free(result_values); + _ = dialects.stablehlo.returns(mlir_ctx, result_values, loc).appendTo(block); + break :b .{ result, block }; + }; + // TODO check that false branch values matches right branch + _ = false_branch; + + const op = mlir.Operation.make(mlir_ctx, "stablehlo.if", .{ + .operands = .{ .flat = &.{pred.asScalar().value()} }, + .result_type_inference = true, + .blocks = &.{ true_branch_block, false_branch_block }, + // We can't verify right away, cause the weights captured by the if haven't been added yet. + .verify = false, + .location = loc, + }); + _ = op.appendTo(CompilationContext.current().currentScope().block); + + return fromMlirOperationWithTags(op, true_branch); +} + +test "if" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const allocator = std.testing.allocator; + + const IfMod = struct { + a: Tensor, + b: Tensor, + + pub fn _fwd(pred: Tensor, a: Tensor, b: Tensor) Tensor { + return @"if"(@This(), .{ .a = a, .b = b }, pred.convert(.bool)); + } + + pub fn onTrue(ctx: @This()) Tensor { + return ctx.a.dotGeneral(ctx.b, &.{.{ 1, 0 }}, &.{}); + } + + pub fn onFalse(ctx: @This()) Tensor { + return ctx.b.dotGeneral(ctx.a, &.{.{ 1, 0 }}, &.{}); + } + }; + + { + const pred: Tensor = .init(.{}, .i32); + const a: Tensor = .init(.{ 4, 4 }, .f32); + const b: Tensor = .init(.{ 4, 4 }, .f32); + const mod = try platform.compileFn(allocator, std.testing.io, IfMod._fwd, .{ pred, a, b }, .{}); + defer mod.deinit(); + } +} + +/// Simpler variant of `zml.ops.if` that assumes code-motion is supported by the backend. +/// The two branches are evaluated before the condition, then the condition chose which branches to keep. +pub fn if2( + pred: Tensor, + on_true: anytype, + on_false: @TypeOf(on_true), +) @TypeOf(on_true) { + stdx.debug.assert(pred.dtype() == .bool and pred.count() == 1, "zml.ops.if expects the condition to have exactly one element of dtype .bool, got {f}", .{pred}); + + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const allocator = arena.allocator(); + + const mlir_ctx = CompilationContext.current().mlir_ctx; + const loc: *const mlir.Location = .unknown(mlir_ctx); + + const true_values = meta.collectAlloc(Tensor.value, {}, allocator, &on_true) catch @panic("OOM"); + defer allocator.free(true_values); + const true_branch_block = b: { + const block = mlir.Block.init(&.{}, &.{}); + errdefer block.deinit(); + + CompilationContext.current().pushBlock(block); + defer CompilationContext.current().popBlock(); + _ = dialects.stablehlo.returns(mlir_ctx, true_values, loc).appendTo(block); + break :b block; + }; + + const false_values = meta.collectAlloc(Tensor.value, {}, allocator, &on_false) catch @panic("OOM"); + defer allocator.free(false_values); + const false_branch_block = b: { + const block = mlir.Block.init(&.{}, &.{}); + errdefer block.deinit(); + + CompilationContext.current().pushBlock(block); + defer CompilationContext.current().popBlock(); + + _ = dialects.stablehlo.returns(mlir_ctx, false_values, loc).appendTo(block); + break :b block; + }; + + const op = mlir.Operation.make(mlir_ctx, "stablehlo.if", .{ + .operands = .{ .flat = &.{pred.asScalar().value()} }, + .result_type_inference = true, + .blocks = &.{ true_branch_block, false_branch_block }, + .location = loc, + .verify = false, + }); + _ = op.appendTo(CompilationContext.current().currentScope().block); + + return fromMlirOperationWithTags(op, on_true); +} + +test if2 { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const allocator = std.testing.allocator; + + const IfMod = struct { + pub fn _fwd(pred: Tensor, a: Tensor, b: Tensor) Tensor { + return if2( + pred.convert(.bool), + a.dotGeneral(b, &.{.{ 1, 0 }}, &.{}), + b.dotGeneral(a, &.{.{ 1, 0 }}, &.{}), + ); + } + }; + + { + const pred: Tensor = .init(.{}, .i32); + const a: Tensor = .init(.{ 4, 4 }, .f32); + const b: Tensor = .init(.{ 4, 4 }, .f32); + const mod = try platform.compileFn(allocator, std.testing.io, IfMod._fwd, .{ pred, a, b }, .{}); + defer mod.deinit(); + } +} + +/// Create a Tensor struct similar to base, keeping base tags, +/// but using mlir value and dims from the mlir operation. +fn fromMlirOperationWithTags(op: *const mlir.Operation, base: anytype) @TypeOf(base) { + const LocalContext = struct { + index: usize, + op: *const mlir.Operation, + }; + var context = LocalContext{ .index = 0, .op = op }; + var res = base; + meta.visit((struct { + fn cb(inner_ctx: *LocalContext, tensor: *Tensor) void { + var new = Tensor.fromMlirValue(inner_ctx.op.result(inner_ctx.index)); + stdx.debug.internalAssert(new.rank() == tensor.rank(), "expected operand result to have rank {} but got {f}", .{ tensor.rank(), new }); + // copy tags and sharding info over + // some ops can change dims eg reduceWindow, so we trust mlir here. + new._shape._tags = tensor._shape._tags; + new._shape._partitioning = tensor._shape._partitioning; + tensor.* = new; + inner_ctx.index += 1; + } + }).cb, &context, &res); + std.debug.assert(context.index == op.numResults()); + return res; +} + pub const TritonOps = struct { debug: bool = false, name: [:0]const u8, diff --git a/zml/platform.zig b/zml/platform.zig index 77d4cae9..6d9f597b 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -836,7 +836,7 @@ fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { const slice: zml.Slice = .init(shape, host_visible_data[0..shape.byteSize()]); const name = call_frame.attrs.getByName(.string, "name").?.slice(); - std.debug.print("{s} {f} [device={d}]: {d}\n", .{ name, slice.shape, device_ordinal, slice }); + std.debug.print("{s} [device={d}]: {d}\n", .{ name, device_ordinal, slice }); return null; } diff --git a/zml/slice.zig b/zml/slice.zig index 32aa07c3..d642c620 100644 --- a/zml/slice.zig +++ b/zml/slice.zig @@ -125,7 +125,7 @@ pub const Slice = struct { return res; } - fn dropAxis(slice: Slice, axis_: anytype) Slice { + pub fn squeeze(slice: Slice, axis_: anytype) Slice { const axis = slice.shape.axis(axis_); var res = slice; res.shape = slice.shape.drop(axis); @@ -354,7 +354,7 @@ pub const Slice = struct { const outer_span_bytes = total_bytes / outer_len; for (0..outer_len) |i| { - const destination_outer = destination.subSlice(0, @intCast(i), 1).dropAxis(0); + const destination_outer = destination.subSlice(0, @intCast(i), 1).squeeze(0); const source_outer_start = i * outer_span_bytes; const source_outer_end = source_outer_start + outer_span_bytes; destination_outer.copy(source[source_outer_start..source_outer_end]); diff --git a/zml/tensor.zig b/zml/tensor.zig index 5d22e934..275ff142 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -4438,11 +4438,14 @@ pub const Tensor = struct { /// Only for debug purpose, it inserts device to host synchronization /// so it will slow down the program execution. pub fn print(input: Tensor, name: []const u8) void { - switch (CompilationContext.current().platform.target) { + const ctx = CompilationContext.current(); + const full_name = std.fmt.allocPrint(ctx.arena.allocator(), "{s}: {f}", .{ name, input.shape() }) catch @panic("OOM"); + defer ctx.arena.allocator().free(full_name); + switch (ctx.platform.target) { .cpu, .cuda, .rocm, .tpu, .metal => { - ops.manualComputation(input, {}, .{ .name = name }, (struct { - fn body(ctx_: anytype, _: std.mem.Allocator, sharded_input: Tensor, _: void) void { - ops.customCall("zml$print", sharded_input, {}, .{ .name = ctx_.name }, .{ .has_side_effect = true }); + ops.manualComputation(input, {}, full_name, (struct { + fn body(_full_name: []const u8, _: std.mem.Allocator, sharded_input: Tensor, _: void) void { + ops.customCall("zml$print", sharded_input, {}, .{ .name = _full_name }, .{ .has_side_effect = true }); } }).body); }, From 14a0a8f507dd0e62d31db92fbb5349fa8950cc31 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Fri, 31 Jul 2026 11:02:07 +0200 Subject: [PATCH 605/881] zml: start of quantization support (#690) Add MLIR composites, `scaledDot`, and a Linear path for NVFP4 weights (unpack, optional activation quant on Blackwell, global scales). Skip fused multi-source tensors in the generic loader so pack/`loadExecute` can fill them (may be used later for fused QKV etc). --- mlir/dialects/stablehlo/stablehlo.zig | 28 ++++ mlir/dialects/vector.zig | 18 +- mlir/mlir.zig | 7 + platforms/cuda/cuda.bzl | 10 +- platforms/cuda/libpjrt_cuda.BUILD.bazel | 36 ++-- zml/io.zig | 3 +- zml/module.zig | 11 ++ zml/nn.zig | 209 +++++++++++++++++++++++- zml/ops.zig | 93 +++++++++++ zml/tensor.zig | 43 ++++- 10 files changed, 418 insertions(+), 40 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 1edcf4a7..ad4b5dbb 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -107,6 +107,34 @@ pub fn clamp(ctx: *mlir.Context, min: *const mlir.Value, value: *const mlir.Valu }); } +pub const CompositeOpts = struct { + name: []const u8, + decomposition: []const u8, + composite_attributes: []const mlir.NamedAttribute = &.{}, + version: i32 = 0, +}; + +pub fn composite( + ctx: *mlir.Context, + inputs: []const *const mlir.Value, + result_types: []const *const mlir.Type, + opts: CompositeOpts, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.composite", .{ + .operands = .{ .flat = inputs }, + .results = .{ .flat = result_types }, + .attributes = &.{ + .named(ctx, "name", .string(ctx, opts.name)), + .named(ctx, "decomposition", .flatSymbolRef(ctx, opts.decomposition)), + .named(ctx, "composite_attributes", .dict(ctx, opts.composite_attributes)), + .named(ctx, "version", .int(ctx, .i32, opts.version)), + }, + .verify = false, + .location = location, + }); +} + pub const DotPrecision = union(enum) { fast, high, diff --git a/mlir/dialects/vector.zig b/mlir/dialects/vector.zig index d9bc904e..13cb6351 100644 --- a/mlir/dialects/vector.zig +++ b/mlir/dialects/vector.zig @@ -140,14 +140,6 @@ pub const StridedSliceArgs = struct { strides: []const i64, }; -/// extract/insert_strided_slice need offsets/sizes/strides as an ArrayAttr of IntegerAttr, not a DenseArrayAttr -fn intArrayAttribute(ctx: *mlir.Context, comptime T: type, values: []const T) *const mlir.Attribute { - const it: mlir.IntegerTypes = comptime @field(mlir.IntegerTypes, @typeName(T)); - var buf: stdx.BoundedArray(*const mlir.Attribute, mlir.ShapedType.MAX_RANK) = .empty; - for (values) |v| buf.appendAssumeCapacity(.int(ctx, it, v)); - return .array(ctx, buf.constSlice()); -} - pub fn extract_strided_slice( ctx: *mlir.Context, src: *const mlir.Value, @@ -159,9 +151,9 @@ pub fn extract_strided_slice( .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "offsets", intArrayAttribute(ctx, i64, args.offsets)), - .named(ctx, "sizes", intArrayAttribute(ctx, i64, args.sizes)), - .named(ctx, "strides", intArrayAttribute(ctx, i64, args.strides)), + .named(ctx, "offsets", .intArray(ctx, i64, args.offsets)), + .named(ctx, "sizes", .intArray(ctx, i64, args.sizes)), + .named(ctx, "strides", .intArray(ctx, i64, args.strides)), }, .location = location, }); @@ -179,8 +171,8 @@ pub fn insert_strided_slice( .operands = .{ .flat = &.{ src, dest } }, .results = .{ .flat = &.{dest.type_()} }, .attributes = &.{ - .named(ctx, "offsets", intArrayAttribute(ctx, i64, offsets)), - .named(ctx, "strides", intArrayAttribute(ctx, i64, strides)), + .named(ctx, "offsets", .intArray(ctx, i64, offsets)), + .named(ctx, "strides", .intArray(ctx, i64, strides)), }, .location = location, }); diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 02559eb7..16d490f0 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -611,6 +611,13 @@ pub const Attribute = opaque { return @ptrCast(ArrayAttribute.init(ctx, attrs)); } + pub fn intArray(ctx: *Context, comptime T: type, values: []const T) *const Attribute { + const it: IntegerTypes = comptime @field(IntegerTypes, @typeName(T)); + var buf: stdx.BoundedArray(*const Attribute, ShapedType.MAX_RANK) = .empty; + for (values) |v| buf.appendAssumeCapacity(.int(ctx, it, v)); + return .array(ctx, buf.constSlice()); + } + pub fn flatSymbolRef(ctx: *Context, symbol: []const u8) *const Attribute { return @ptrCast(FlatSymbolRefAttribute.init(ctx, symbol)); } diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 58d2df70..9297ba0d 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -131,12 +131,21 @@ CUDA_PACKAGES = { ], ), ], + "cuda_tileiras": [ + packages.filegroup( + name = "cuda_tileiras", + srcs = ["bin/tileiras"], + ), + ], "libnvvm": [ packages.filegroup( name = "libnvvm", srcs = [ "nvvm/bin/cicc", "nvvm/libdevice/libdevice.10.bc", + "nvvm/lib64/libnvvm.so", + "nvvm/lib64/libnvvm.so.4", + "nvvm/lib64/libnvvm.so.4.0.0", ], ), ], @@ -243,7 +252,6 @@ def _read_redist_json(mctx, url, sha256): ) return json.decode(mctx.read(fname)) - def _cuda_impl(mctx): loaded_packages = packages.read(mctx, [ "@zml//platforms/cuda:packages.lock.json", diff --git a/platforms/cuda/libpjrt_cuda.BUILD.bazel b/platforms/cuda/libpjrt_cuda.BUILD.bazel index bb22645a..84057fb5 100644 --- a/platforms/cuda/libpjrt_cuda.BUILD.bazel +++ b/platforms/cuda/libpjrt_cuda.BUILD.bazel @@ -1,6 +1,6 @@ load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") -load("@zml//bazel:patchelf.bzl", "patchelf") load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@zml//bazel:patchelf.bzl", "patchelf") load("@zml//platforms/cuda:cuda.bzl", "CUDA_COMPAT_FILES") patchelf( @@ -23,42 +23,46 @@ copy_to_directory( "@zml//platforms/cuda:zmlxcuda", ] + select({ "@llvm//platforms/config:linux_x86_64": [ + "@cuda_compat_linux_x86_64//:cuda_compat", "@cuda_cudart_linux_x86_64//:cuda_cudart", "@cuda_cupti_linux_x86_64//:cuda_cupti", - "@cuda_compat_linux_x86_64//:cuda_compat", "@cuda_nvcc_linux_x86_64//:cuda_nvcc", - "@libnvvm_linux_x86_64//:libnvvm", "@cuda_nvrtc_linux_x86_64//:cuda_nvrtc", "@cuda_nvtx_linux_x86_64//:cuda_nvtx", + "@cuda_tileiras_linux_x86_64//:cuda_tileiras", "@cudnn_linux_x86_64//:cudnn", - "@libnvshmem_linux_x86_64//:libnvshmem", "@libcublas_linux_x86_64//:libcublas", "@libcufft_linux_x86_64//:libcufft", "@libcusolver_linux_x86_64//:libcusolver", "@libcusparse_linux_x86_64//:libcusparse", "@libnvjitlink_linux_x86_64//:libnvjitlink", + "@libnvshmem_linux_x86_64//:libnvshmem", + "@libnvvm_linux_x86_64//:libnvvm", "@nccl_linux_amd64//:nccl", "@zlib1g_linux_amd64//:zlib1g", ], "@llvm//platforms/config:linux_aarch64": [ + "@cuda_compat_linux_sbsa//:cuda_compat", "@cuda_cudart_linux_sbsa//:cuda_cudart", "@cuda_cupti_linux_sbsa//:cuda_cupti", - "@cuda_compat_linux_sbsa//:cuda_compat", "@cuda_nvcc_linux_sbsa//:cuda_nvcc", - "@libnvvm_linux_sbsa//:libnvvm", "@cuda_nvrtc_linux_sbsa//:cuda_nvrtc", "@cuda_nvtx_linux_sbsa//:cuda_nvtx", + "@cuda_tileiras_linux_sbsa//:cuda_tileiras", "@cudnn_linux_sbsa//:cudnn", - "@libnvshmem_linux_sbsa//:libnvshmem", "@libcublas_linux_sbsa//:libcublas", "@libcufft_linux_sbsa//:libcufft", "@libcusolver_linux_sbsa//:libcusolver", "@libcusparse_linux_sbsa//:libcusparse", "@libnvjitlink_linux_sbsa//:libnvjitlink", + "@libnvshmem_linux_sbsa//:libnvshmem", + "@libnvvm_linux_sbsa//:libnvvm", "@nccl_linux_arm64//:nccl", "@zlib1g_linux_arm64//:zlib1g", ], }), + add_directory_to_runfiles = False, + include_external_repositories = ["**"], replace_prefixes = { "compat": "lib/compat", "nvidia/nccl/lib": "lib", @@ -72,21 +76,11 @@ copy_to_directory( "{}.patchelf".format(file): "lib/compat" for file in CUDA_COMPAT_FILES }, - add_directory_to_runfiles = False, - include_external_repositories = ["**"], ) cc_library( name = "libpjrt_cuda", data = [":sandbox"], - deps = select({ - "@llvm//platforms/config:linux_x86_64": [ - "@cuda_cudart_linux_x86_64//:cuda", - ], - "@llvm//platforms/config:linux_aarch64": [ - "@cuda_cudart_linux_sbsa//:cuda", - ], - }), linkopts = [ # Defer function call resolution until the function is called # (lazy loading) rather than at load time. @@ -98,4 +92,12 @@ cc_library( "-Wl,-z,lazy", ], visibility = ["@zml//platforms/cuda:__subpackages__"], + deps = select({ + "@llvm//platforms/config:linux_x86_64": [ + "@cuda_cudart_linux_x86_64//:cuda", + ], + "@llvm//platforms/config:linux_aarch64": [ + "@cuda_cudart_linux_sbsa//:cuda", + ], + }), ) diff --git a/zml/io.zig b/zml/io.zig index 248e469d..0e14d60b 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -373,7 +373,8 @@ pub const Loader = struct { }; if (sources.len != 1) { - std.debug.panic("Expected loaded tensor to have only 1 source, got {}", .{sources.len}); + log.debug("Skipping fused tensor with {} sources; expected to be loaded via loadExecute", .{sources.len}); + return; } self.loadSingleInner(io, sources[0], tensor.shape(), buffer, shardings, opts) catch |e| { diff --git a/zml/module.zig b/zml/module.zig index 7ce89c05..15537404 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -101,6 +101,8 @@ pub const CompilationContext = struct { channel_id: i64 = 0, + composite_id: i64 = 0, + threadlocal var _current: ?*CompilationContext = null; pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: CompilationOptions) CompilationContext { @@ -199,6 +201,11 @@ pub const CompilationContext = struct { return self.channel_id; } + pub fn nextCompositeId(self: *CompilationContext) i64 { + self.composite_id += 1; + return self.composite_id; + } + pub fn abortOOM(self: *CompilationContext) noreturn { _ = self; @panic("OOM"); @@ -207,6 +214,10 @@ pub const CompilationContext = struct { pub fn alloc(self: *CompilationContext, T: type, n: usize) []T { return self.arena.allocator().alloc(T, n) catch self.abortOOM(); } + + pub fn allocPrint(self: *CompilationContext, comptime fmt: []const u8, args: anytype) []u8 { + return std.fmt.allocPrint(self.arena.allocator(), fmt, args) catch self.abortOOM(); + } }; pub fn Compiler(comptime func: anytype) type { diff --git a/zml/nn.zig b/zml/nn.zig index c87f25b4..3577b792 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -1,8 +1,10 @@ //! Common layer definition and functions for Neural Networks (NN) const std = @import("std"); +const mlir = @import("mlir"); const stdx = @import("stdx"); +const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); const ops = @import("ops.zig"); @@ -17,6 +19,9 @@ pub const Linear = struct { weight: Tensor, bias: ?Tensor = null, tag: Shape.Tag, + scales: ?Tensor = null, + global_scale: ?Tensor = null, + input_global_scale: ?Tensor = null, pub fn init(weight: Tensor, bias: ?Tensor, tag: anytype) Linear { return .{ @@ -26,13 +31,213 @@ pub const Linear = struct { }; } - pub fn forward(self: Linear, x: Tensor) Tensor { - var y = x.dot(self.weight, self.tag); + pub fn unloadBuffers(self: *zml.Bufferized(Linear)) void { + zml.Buffer.deinitAll(Linear, self); + } + pub fn forward(self: Linear, x: Tensor) Tensor { + const y = self.forwardWeight(x); return if (self.bias) |bias| y.add(bias.broad(y.shape())) else y; } + + fn forwardWeight(self: Linear, x: Tensor) Tensor { + const scales = self.scales orelse return x.dot(self.weight, self.tag); + + const weight = if (self.weight.dtype() == .u8) + unpackNvfp4(self.weight.withTags(.{ .dout, .kw }), self.tag) + else + self.weight; + + const igs: ?Tensor = if (self.input_global_scale) |g| g.convert(.f32).asScalar() else null; + const wgs: ?Tensor = if (self.global_scale) |g| g.convert(.f32).asScalar() else null; + + const platform = zml.module.CompilationContext.current().platform; + if (weight.dtype() == .f4e2m1 and supportsNvfp4ActivationQuant(platform)) { + const q = quantizeNvfp4(x.convert(.bf16), igs, self.tag); + const acc = scaledDot(q.values, weight, q.scales, scales, self.tag); + return applyGlobalScale(acc, igs, wgs).convert(x.dtype()); + } + + const acc = scaledDot(x.convert(.bf16), weight, null, scales, self.tag); + return applyGlobalScale(acc, null, wgs).convert(x.dtype()); + } }; +pub fn unpackNvfp4(w: Tensor, k_tag: anytype) Tensor { + stdx.debug.assert(w.dtype() == .u8, "unpackNvfp4 expects packed u8 weights, got {}", .{w.dtype()}); + return w.bitCast(.f4e2m1) + .merge(.{ .kb = .{ .kw, .bitcast } }) + .renameTag(.kb, Shape.toTag(k_tag)); +} + +pub fn quantizeNvfp4(x: Tensor, input_global_scale: ?Tensor, axis: anytype) struct { + values: Tensor, + scales: Tensor, +} { + const block_size = 16; + const value_max = 6.0; + const scale_min_normal = 0x1p-6; + const scale_max = DataType.f8e4m3fn.maxValue().as(f32); + + stdx.debug.assert(x.shape().hasTag(axis) != null, "quantizeNvfp4 expects x to have {any} tag, got {f}", .{ axis, x.shape() }); + stdx.debug.assert(@rem(x.dim(axis), block_size) == 0, "quantizeNvfp4 expects {any} to be a multiple of {}, got {f}", .{ axis, block_size, x.shape() }); + + const dt = x.dtype(); + const scaled = if (input_global_scale) |igs| + x.mul(igs.convert(dt).broad(x.shape())) + else + x; + const grouped = scaled.splitAxis(axis, .{ .sc = -1, .blk = block_size }); + const amax = grouped.abs().max(.blk); + + const scales = amax.scale(1.0 / value_max) + .clamp(.scalar(scale_min_normal, dt), .scalar(scale_max, dt)) + .convert(.f8e4m3fn); + + const divisor = scales.convert(dt) + .maximum(.scalar(scale_min_normal, dt)) + .broad(grouped.shape()); + + const values = grouped.div(divisor) + .convert(.f4e2m1) + .reshape(x.shape().withDtype(.f4e2m1)); + + return .{ + .values = values, + .scales = scales.squeeze(.blk), + }; +} + +/// Scaled matrix multiply (`xla.scaled_dot`): `acc = (lhs * lhs_scale) @ (rhs * rhs_scale)`. +/// +/// - **NVFP4**: values `.f4e2m1`, scales `.f8e4m3fn`, block 16 (weight-only bf16 lhs ok) +/// - **MXFP4**: values `.f4e2m1`, scales `.f8e8m0fnu`, block 32 +/// - **MXFP8**: values `.f8e4m3fn` / `.f8e5m2`, scales `.f8e8m0fnu`, block 32 +/// - TODO: INT4/8 and FP8 with block 128 and per tensor +/// +/// Backends: +/// 1. TileIR if CUDA sm>=10 and same lhs/rhs dtype +/// 2. Triton otherwise +/// 3. Unsupported combos fall back to dequant + Dot +/// +/// CPU has no specialized path. +pub fn scaledDot( + lhs: Tensor, + rhs: Tensor, + lhs_scale: ?Tensor, + rhs_scale: Tensor, + args: anytype, +) Tensor { + const dot_axes = lhs.dotAxes(rhs, args); + + const Axes = stdx.BoundedArray(i64, constants.MAX_RANK); + + const result_dtype: DataType = switch (lhs.dtype()) { + .f4e2m1, .f8e4m3, .f8e4m3fn, .f8e5m2, .f8e4m3b11fnuz, .f8e4m3fnuz, .f8e5m2fnuz => .bf16, + else => lhs.dtype(), + }; + var res_shape: Shape = .{ ._dtype = result_dtype }; + var lhs_batching_axes: Axes = .empty; + var rhs_batching_axes: Axes = .empty; + for (dot_axes.batching.constSlice()) |b_axes| { + const l, const r = b_axes; + stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "scaledDot expects batching dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); + var t = lhs._shape.tag(l); + if (t == Shape.TagUnknown) t = rhs._shape.tag(r); + res_shape = res_shape.appendDim(lhs._shape.dim(l), t); + lhs_batching_axes.appendAssumeCapacity(lhs._shape.axis(l)); + rhs_batching_axes.appendAssumeCapacity(rhs._shape.axis(r)); + } + + var lhs_contracting_axes: Axes = .empty; + var rhs_contracting_axes: Axes = .empty; + for (dot_axes.contracting.constSlice()) |c_axes| { + const l, const r = c_axes; + stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "scaledDot expects contracting dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); + lhs_contracting_axes.appendAssumeCapacity(lhs._shape.axis(l)); + rhs_contracting_axes.appendAssumeCapacity(rhs._shape.axis(r)); + } + + for (0..lhs.rank()) |l| { + if (std.mem.indexOfScalar(i64, lhs_contracting_axes.constSlice(), @intCast(l))) |_| { + continue; + } + if (std.mem.indexOfScalar(i64, lhs_batching_axes.constSlice(), @intCast(l))) |_| { + continue; + } + res_shape = res_shape.appendDim(lhs._shape.dim(l), lhs._shape.tag(l)); + } + for (0..rhs.rank()) |r| { + if (std.mem.indexOfScalar(i64, rhs_contracting_axes.constSlice(), @intCast(r))) |_| { + continue; + } + if (std.mem.indexOfScalar(i64, rhs_batching_axes.constSlice(), @intCast(r))) |_| { + continue; + } + res_shape = res_shape.appendDim(rhs._shape.dim(r), rhs._shape.tag(r)); + } + + const lhs_scale_operand = lhs_scale orelse blk: { + var lhs_scale_shape = lhs.shape().withDtype(.bf16); + for (0..lhs.rank()) |i| { + lhs_scale_shape = lhs_scale_shape.setDim(i, 1); + } + break :blk Tensor.constantTensor(lhs_scale_shape, DataType.bf16.one().asBytes()); + }; + + const mlir_ctx = zml.module.CompilationContext.current().mlir_ctx; + const dnums = mlir.Attribute.array(mlir_ctx, &.{ + .array(mlir_ctx, &.{ + .intArray(mlir_ctx, i64, lhs_contracting_axes.constSlice()), + .intArray(mlir_ctx, i64, rhs_contracting_axes.constSlice()), + }), + .array(mlir_ctx, &.{ + .intArray(mlir_ctx, i64, lhs_batching_axes.constSlice()), + .intArray(mlir_ctx, i64, rhs_batching_axes.constSlice()), + }), + }); + + const operands: []const Tensor = &.{ lhs, rhs, lhs_scale_operand, rhs_scale }; + + const outs = ops.composite("xla.scaled_dot", operands, &.{res_shape}, scaledDotReference, res_shape, .{ + .composite_attributes = &.{.named(mlir_ctx, "dimension_numbers", dnums)}, + }); + + return outs[0]; +} + +fn scaledDotReference(in: []const Tensor, out_shape: Shape) Tensor { + return ops.customCall( + "zml$scaled_dot_unmatched", + .{ in[0], in[1], in[2], in[3] }, + out_shape, + {}, + .{ .has_side_effect = false }, + ); +} + +fn supportsNvfp4ActivationQuant(platform: *const zml.Platform) bool { + if (platform.target != .cuda) { + return false; + } + + const device = platform.pjrt_client.devices(platform.pjrt_api)[0]; + const capability = zml.platform.cuda.tryGetComputeCapabilities(platform, device) orelse return false; + const major = std.fmt.parseInt(u8, std.mem.sliceTo(capability, '.'), 10) catch return false; + + return major >= 10; +} + +fn applyGlobalScale(acc: Tensor, igs: ?Tensor, wgs: ?Tensor) Tensor { + const combined: ?Tensor = if (igs) |i| + (if (wgs) |w| i.mul(w) else i) + else + wgs; + const alpha = combined orelse return acc; + const f32_acc = acc.convert(.f32); + return f32_acc.div(alpha.broad(f32_acc.shape())).convert(acc.dtype()); +} + pub const TokenEmbedding = struct { weight: Tensor, diff --git a/zml/ops.zig b/zml/ops.zig index eadd1772..1acc93e2 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1743,6 +1743,99 @@ pub fn customCallOutputOperandAliases( return &output_operand_aliases; } +pub const CompositeOpts = struct { + version: i32 = 0, + composite_attributes: []const mlir.NamedAttribute = &.{}, +}; + +pub fn composite( + name: [:0]const u8, + inputs: []const Tensor, + outputs: []const Shape, + comptime decomposition: anytype, + context: anytype, + opts: CompositeOpts, +) []Tensor { + const ctx = CompilationContext.current(); + const mlir_ctx = ctx.mlir_ctx; + + const decomp_name = ctx.allocPrint("{s}.impl_{d}", .{ name, ctx.nextCompositeId() }); + + { + const block_types = ctx.alloc(*const mlir.Type, inputs.len); + const block_locs = ctx.alloc(*const mlir.Location, inputs.len); + + for (inputs, 0..) |t, i| { + block_types[i] = mlirx.Type.rankedTensor(mlir_ctx, t.shape()); + block_locs[i] = mlir.Location.unknown(mlir_ctx); + } + + const block = mlir.Block.init(block_types, block_locs); + + ctx.pushBlock(block); + { + const arg_tensors = ctx.alloc(Tensor, inputs.len); + for (inputs, 0..) |t, i| { + arg_tensors[i] = Tensor._result(t.shape(), block.argument(i)); + } + + var result = @call(.auto, decomposition, .{ arg_tensors, context }); + const rtensors = ctx.alloc(Tensor, outputs.len); + meta.collectBuf((struct { + pub fn func(t: Tensor) Tensor { + return t; + } + }).func, {}, &result, rtensors); + + const rvals = ctx.alloc(*const mlir.Value, outputs.len); + for (rtensors, 0..) |t, i| { + rvals[i] = t.value(); + } + + _ = dialects.func.returns(mlir_ctx, rvals, .unknown(mlir_ctx)).appendTo(block); + } + ctx.popBlock(); + + _ = dialects.func.func(mlir_ctx, .{ + .name = decomp_name, + .block = block, + .location = .unknown(mlir_ctx), + .visibility = .private, + .verify = false, + }).appendTo(ctx.module.body()); + } + + const operand_values = ctx.alloc(*const mlir.Value, inputs.len); + for (inputs, 0..) |t, i| { + operand_values[i] = t.value(); + } + + const result_types = ctx.alloc(*const mlir.Type, outputs.len); + for (outputs, 0..) |s, i| { + result_types[i] = mlirx.Type.rankedTensor(mlir_ctx, s); + } + + const op = dialects.stablehlo.composite( + mlir_ctx, + operand_values, + result_types, + .{ + .name = name, + .decomposition = decomp_name, + .composite_attributes = opts.composite_attributes, + .version = opts.version, + }, + .unknown(mlir_ctx), + ).appendTo(ctx.currentScope().block); + + const out_tensors = ctx.alloc(Tensor, outputs.len); + for (outputs, 0..) |s, i| { + out_tensors[i] = Tensor._result(s, op.result(i)); + } + + return out_tensors; +} + pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, metadata: anytype, opts: CustomCallOptions) CustomCallResultTypeFromOutputSpec(@TypeOf(outputs)) { // Transform generic inputs to flat slice. const inputs_: []const Tensor = switch (@typeInfo(@TypeOf(inputs))) { diff --git a/zml/tensor.zig b/zml/tensor.zig index 275ff142..5895c30b 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1283,21 +1283,52 @@ pub const Tensor = struct { /// Axes with the same tag on both sides, and which aren't contracting, /// are considered "batching axes". pub fn dot(lhs: Tensor, rhs: Tensor, args: anytype) Tensor { - stdx.debug.assert(lhs.shape().hasTag(args) != null, "Expected lhs to have {any} tag, got {f}", .{ args, lhs.shape() }); - stdx.debug.assert(rhs.shape().hasTag(args) != null, "Expected rhs to have {any} tag, got {f}", .{ args, rhs.shape() }); + const dot_axes = lhs.dotAxes(rhs, args); + return lhs.dotGeneral(rhs, dot_axes.contracting.constSlice(), dot_axes.batching.constSlice()); + } - const lhs_contracting_dim: i8 = @intCast(lhs.shape().hasTag(args).?); - const rhs_contracting_dim: i8 = @intCast(rhs.shape().hasTag(args).?); + pub const DotAxes = struct { + contracting: stdx.BoundedArray([2]i8, constants.MAX_RANK), + batching: stdx.BoundedArray([2]i8, constants.MAX_RANK), + }; + + /// Returns contracting and batching axis pairs inferred from one or more tags. + pub fn dotAxes(lhs: Tensor, rhs: Tensor, tags: anytype) DotAxes { + const T = @TypeOf(tags); + if (comptime T == Shape.Tag or T == @EnumLiteral()) { + return lhs.dotAxes(rhs, .{tags}); + } + + const lhs_contracting_axes, _ = lhs.shape().parseAxes(tags); + const rhs_contracting_axes, _ = rhs.shape().parseAxes(tags); + var contracting_axes: stdx.BoundedArray([2]i8, constants.MAX_RANK) = .empty; + for (lhs_contracting_axes.constSlice(), rhs_contracting_axes.constSlice()) |l, r| { + contracting_axes.appendAssumeCapacity(.{ @intCast(l), @intCast(r) }); + } var batching_axes: stdx.BoundedArray([2]i8, constants.MAX_RANK) = .empty; for (0..lhs.rank()) |lhs_tag_index| { + if (std.mem.indexOfScalar(u3, lhs_contracting_axes.constSlice(), @intCast(lhs_tag_index)) != null) { + continue; + } + const lhs_tag = lhs.shape().tag(lhs_tag_index); - if (lhs_tag == Shape.toTag(args)) continue; if (rhs.shape().hasTag(lhs_tag)) |rhs_tag_index| { batching_axes.appendAssumeCapacity(.{ @intCast(lhs_tag_index), @intCast(rhs_tag_index) }); } } - return lhs.dotGeneral(rhs, &.{.{ lhs_contracting_dim, rhs_contracting_dim }}, batching_axes.slice()); + + return .{ .contracting = contracting_axes, .batching = batching_axes }; + } + + test dotAxes { + const lhs: Tensor = .init(.{ .a = 20, .b = 21, .c = 22 }, .f32); + const rhs: Tensor = .init(.{ .c = 22, .b = 21, .d = 23, .a = 20 }, .f32); + + const dot_axes = lhs.dotAxes(rhs, .{ .c, .a }); + + try std.testing.expectEqualSlices([2]i8, &.{ .{ 2, 0 }, .{ 0, 3 } }, dot_axes.contracting.constSlice()); + try std.testing.expectEqualSlices([2]i8, &.{.{ 1, 1 }}, dot_axes.batching.constSlice()); } test dot { From 57eabe72deb54879518bd4b25517f2ca6491adac Mon Sep 17 00:00:00 2001 From: Olivier Boulant Date: Fri, 31 Jul 2026 14:52:11 +0200 Subject: [PATCH 606/881] kernels/mosaic_tpu: add log1p builder operation (#691) Add `Builder.log1p` to Mosaic TPU kernel DSL The MLIR math dialect already exposes `math.log1p`. This change makes it available through the Mosaic TPU builder alongside the already existing operations. --> This avoids kernels constructind `math.log1p` operations manually and makes the operation reusable for future DSL kernel translations. A companion DSL-harness changes replaces existing manual `math.log1p` construction with `Builder.log1p`. --- kernels/mosaic_tpu/builder.zig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kernels/mosaic_tpu/builder.zig b/kernels/mosaic_tpu/builder.zig index 2f6bd2ab..1801cd12 100644 --- a/kernels/mosaic_tpu/builder.zig +++ b/kernels/mosaic_tpu/builder.zig @@ -1380,6 +1380,9 @@ pub const Builder = struct { pub fn log(self: *Builder, x: Value) Value { return self.emit(math.log(self.ctx, x.inner, self.loc())); } + pub fn log1p(self: *Builder, x: Value) Value { + return self.emit(math.log1p(self.ctx, x.inner, self.loc())); + } pub fn log2(self: *Builder, x: Value) Value { return self.emit(math.log2(self.ctx, x.inner, self.loc())); } From 4976dfbeb3babe14c51cd48c33dfa027eaaacc8f Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 31 Jul 2026 14:59:58 +0200 Subject: [PATCH 607/881] bazel: use github.com instead of codeberg.org as reference (#695) --- third_party/translate-c/repo.bzl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/third_party/translate-c/repo.bzl b/third_party/translate-c/repo.bzl index 11598b32..8315b2af 100644 --- a/third_party/translate-c/repo.bzl +++ b/third_party/translate-c/repo.bzl @@ -3,7 +3,8 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") def repo(): git_repository( name = "translate-c", - remote = "https://codeberg.org/ziglang/translate-c", + # Use ZML Github mirror of translate-c because Codeber is too unstable + remote = "https://github.com/zml/translate-c", commit = "46b5609b5ac4c0a896217d1d984f3ae50e4810b5", build_file = Label("//third_party/translate-c:translate-c.bazel"), ) From 830c8facffb25f514ce76c9ee81d40751b25bdcf Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Fri, 31 Jul 2026 19:22:03 +0200 Subject: [PATCH 608/881] zml/nn: treat Linear global scales as dequant multipliers (#698) Make `Linear.global_scale` / `input_global_scale` match checkpoint conventions --- zml/nn.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index 3577b792..b99ed02f 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -84,7 +84,7 @@ pub fn quantizeNvfp4(x: Tensor, input_global_scale: ?Tensor, axis: anytype) stru const dt = x.dtype(); const scaled = if (input_global_scale) |igs| - x.mul(igs.convert(dt).broad(x.shape())) + x.div(igs.convert(dt).broad(x.shape())) else x; const grouped = scaled.splitAxis(axis, .{ .sc = -1, .blk = block_size }); @@ -235,7 +235,7 @@ fn applyGlobalScale(acc: Tensor, igs: ?Tensor, wgs: ?Tensor) Tensor { wgs; const alpha = combined orelse return acc; const f32_acc = acc.convert(.f32); - return f32_acc.div(alpha.broad(f32_acc.shape())).convert(acc.dtype()); + return f32_acc.mul(alpha.broad(f32_acc.shape())).convert(acc.dtype()); } pub const TokenEmbedding = struct { From 039fc9516a134649c239bbdf25685fef0bcf7168 Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Fri, 31 Jul 2026 20:08:08 +0200 Subject: [PATCH 609/881] zml/cuda : update PJRT CUDA asset SHA256 checksums for amd64 and arm64 (#700) --- platforms/cuda/cuda.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 9297ba0d..c5b01869 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -213,15 +213,15 @@ _UBUNTU_PACKAGES = { ], } -PJRT_CUDA_RELEASE = "manual-2026-07-28T12-29-00Z" +PJRT_CUDA_RELEASE = "manual-2026-07-31T19-22-00Z" _PJRT_CUDA_ASSETS = { "amd64": { - "sha256": "a167dcb3af6a367dad51063c9e2a214da0895d375cea3a0ab48c574856360bd1", + "sha256": "c3fe395ed8b0493975e4afeba454e0cbecec7eac540488522fd4523103e353f3", "url": "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-cuda_linux-amd64.tar.gz", }, "arm64": { - "sha256": "c06beb6ff2eee0f265065c0c5dab2a5611cc3e220045be78f16887fd60c9749a", + "sha256": "3c23c31dbfb3a97ca1bf8558d1e8347431fa30c192236d97de7bd6539d354ca5", "url": "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-cuda_linux-arm64.tar.gz", }, } From 154b212caac4155c4edc208b66f14ee861be0660 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sun, 2 Aug 2026 20:35:09 -0400 Subject: [PATCH 610/881] build: disable default Xcode discovery (#630) --- .bazelrc | 1 + BUILD.bazel | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index d5587eb3..81972716 100644 --- a/.bazelrc +++ b/.bazelrc @@ -77,6 +77,7 @@ common --workspace_status_command=./tools/workspace_status.sh common --@rules_cc//cc/toolchains/args/archiver_flags:use_libtool_on_macos=false common --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 common --repo_env=BAZEL_NO_APPLE_CPP_TOOLCHAIN=1 +common --xcode_version_config=//:disable_xcode # Required for protobuf upb on Linux common:linux --copt=-fPIC diff --git a/BUILD.bazel b/BUILD.bazel index e2ffd349..19061490 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,5 +1,12 @@ -load("@rules_zig//zig/zls:zls_completion.bzl", "zls_completion") +load("@@apple_support+//xcode:xcode_config.bzl", "xcode_config") load("@rules_zig//zig/translate-c:toolchain.bzl", "translate_c_toolchain") +load("@rules_zig//zig/zls:zls_completion.bzl", "zls_completion") + +# Dummy xcode_config so we don't need to build xcode_locator in the repository rule. +xcode_config( + name = "disable_xcode", + visibility = ["//visibility:public"], +) alias( name = "translate-c", From a37f903c3fa01599587254b404506fcc9fedd395 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sun, 2 Aug 2026 21:56:42 -0400 Subject: [PATCH 611/881] [build] bump rules_swift to 3.6.1 (#640) This prevent eager toolchain load that resulted in 10ish seconds of `swiftc` calls --- MODULE.bazel | 2 ++ MODULE.bazel.lock | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 5d4cb3a9..b2cfce40 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -43,6 +43,8 @@ bazel_dep(name = "rules_oci", version = "2.2.7") bazel_dep(name = "rules_proto", version = "7.1.0") bazel_dep(name = "rules_python", version = "1.9.0") bazel_dep(name = "rules_shell", version = "0.6.1") +# repo_name = None prevents rules_swift from eagerly auto-registering toolchains. +bazel_dep(name = "rules_swift", version = "3.6.1", repo_name = None) bazel_dep(name = "rules_zig", version = "0.15.1") bazel_dep(name = "tar.bzl", version = "0.10.4") bazel_dep(name = "llvm", version = "0.8.9") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 92d5b624..841260f0 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -175,7 +175,8 @@ "https://bcr.bazel.build/modules/mbedtls/3.6.0/MODULE.bazel": "8e380e4698107c5f8766264d4df92e36766248447858db28187151d884995a09", "https://bcr.bazel.build/modules/mbedtls/3.6.0/source.json": "1dbe7eb5258050afcc3806b9d43050f71c6f539ce0175535c670df606790b30c", "https://bcr.bazel.build/modules/nlohmann_json/3.11.3/MODULE.bazel": "87023db2f55fc3a9949c7b08dc711fae4d4be339a80a99d04453c4bb3998eefc", - "https://bcr.bazel.build/modules/nlohmann_json/3.11.3/source.json": "296c63a90c6813e53b3812d24245711981fc7e563d98fe15625f55181494488a", + "https://bcr.bazel.build/modules/nlohmann_json/3.12.0.bcr.1/MODULE.bazel": "a1c8bb07b5b91d971727c635f449d05623ac9608f6fe4f5f04254ea12f08e349", + "https://bcr.bazel.build/modules/nlohmann_json/3.12.0.bcr.1/source.json": "93f82a5ae985eb935c539bfee95e04767187818189241ac956f3ccadbdb8fb02", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", "https://bcr.bazel.build/modules/opencensus-cpp/0.0.0-20230502-50eb5de.bcr.2/MODULE.bazel": "cc18734138dd18c912c6ce2a59186db28f85d8058c99c9f21b46ca3e0aba0ebe", "https://bcr.bazel.build/modules/opencensus-cpp/0.0.0-20230502-50eb5de.bcr.2/source.json": "7c135f9d42bb3b045669c3c6ab3bb3c208e00b46aca4422eea64c29811a5b240", @@ -235,6 +236,7 @@ "https://bcr.bazel.build/modules/protobuf/31.1/MODULE.bazel": "379a389bb330b7b8c1cdf331cc90bf3e13de5614799b3b52cdb7c6f389f6b38e", "https://bcr.bazel.build/modules/protobuf/32.1/MODULE.bazel": "89cd2866a9cb07fee9ff74c41ceace11554f32e0d849de4e23ac55515cfada4d", "https://bcr.bazel.build/modules/protobuf/33.4/MODULE.bazel": "114775b816b38b6d0ca620450d6b02550c60ceedfdc8d9a229833b34a223dc42", + "https://bcr.bazel.build/modules/protobuf/34.0.bcr.1/MODULE.bazel": "74e541b0ba877813da786a11707d4e394433c157841d5111a36be0d44b907931", "https://bcr.bazel.build/modules/protobuf/35.0/MODULE.bazel": "ee97170c013d94c366bbb5fe8a97b0650477cbc00e5e0f6a2c297484bc34d81b", "https://bcr.bazel.build/modules/protobuf/35.0/source.json": "2162e80adef862606ba85ae42e0df85eb519376fd6ab49caddee6fb776a1281e", "https://bcr.bazel.build/modules/protoc-gen-validate/1.0.4.bcr.2/MODULE.bazel": "c4bd2c850211ff5b7dadf9d2d0496c1c922fdedc303c775b01dfd3b3efc907ed", @@ -411,7 +413,8 @@ "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", "https://bcr.bazel.build/modules/rules_swift/2.5.0/MODULE.bazel": "6d2fec72be0ad2e1fc2eab536cf9d3fb149037c6d2aa678078f3512c06a819e1", "https://bcr.bazel.build/modules/rules_swift/3.1.2/MODULE.bazel": "72c8f5cf9d26427cee6c76c8e3853eb46ce6b0412a081b2b6db6e8ad56267400", - "https://bcr.bazel.build/modules/rules_swift/3.1.2/source.json": "e85761f3098a6faf40b8187695e3de6d97944e98abd0d8ce579cb2daf6319a66", + "https://bcr.bazel.build/modules/rules_swift/3.6.1/MODULE.bazel": "5a68bb05b6b8044f8a7bd7184d56150189e7eca05522ce525517c234fed50ed8", + "https://bcr.bazel.build/modules/rules_swift/3.6.1/source.json": "9af2e521ffdb41961b6b9ac05293bd6519ee2fb52849a2be658a20b0221cb8db", "https://bcr.bazel.build/modules/rules_zig/0.15.1/MODULE.bazel": "f0b813d9191c8d201e0b3ecbe5c5220ddedb717a15009fc711a78b3e1db8710d", "https://bcr.bazel.build/modules/rules_zig/0.15.1/source.json": "0bd91d7459f504196dfdc90cd0fffbf1f0df2667c9cd5eb708feb3fd27578e6a", "https://bcr.bazel.build/modules/stardoc/0.5.0/MODULE.bazel": "f9f1f46ba8d9c3362648eea571c6f9100680efc44913618811b58cc9c02cd678", From 067602b241227c4d7743ac1ca829e5d030fe2d3e Mon Sep 17 00:00:00 2001 From: Louis Lechevalier Date: Mon, 17 Aug 2026 09:41:28 +0200 Subject: [PATCH 612/881] zml/shape: Effective byte size on device (#699) This PR introduces deviceByteSize which computes the byte size of a shape on device i.e. with packed elements when they are less than 8 bits. --- zml/shape.zig | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/zml/shape.zig b/zml/shape.zig index 40e1fe07..a2c7e9c9 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -498,11 +498,26 @@ pub const Shape = struct { return @intCast(res); } - /// Total size in bytes needed to represent this shape. + /// Total size in bytes needed to represent this shape on host pub fn byteSize(self: Shape) usize { return self.dtype().sizeOf() * self.count(); } + /// Total size in bytes needed to represent this shape on device + /// https://github.com/openxla/xla/blob/0f7240cad3fa14b52a809ac1f4a351a9edfc541e/xla/primitive_util.h#L807 + pub fn deviceByteSize(self: Shape) usize { + const bits_per_element: usize = switch (self.dtype()) { + .bool => 8, + else => self.dtype().bitSizeOf(), + }; + + return std.math.divCeil( + usize, + self.count() * bits_per_element, + 8, + ) catch unreachable; + } + /// Compares the two shapes described, ignoring tagging. pub fn eql(self: Shape, other: Shape) bool { return std.mem.eql(i64, self.dims(), other.dims()) and self.dtype() == other.dtype(); From 49a12235bb4be0a674f0937cc60cc0265a4f65a2 Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Mon, 17 Aug 2026 15:16:33 +0200 Subject: [PATCH 613/881] zml/pjrt: propagate ready-event failures and validate opaque pointers (#703) Fix two PJRT error-handling cases that could allow execution to continue after a backend failure. - Use the same error conversion and destruction path for ready-event and callback errors. - Reject a successful `PJRT_Buffer_OpaqueDeviceMemoryDataPointer` result when `device_memory_ptr` is null. --- pjrt/pjrt.zig | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 196413d0..6f498971 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -74,6 +74,13 @@ pub const ApiError = error{ Unauthenticated, }; +inline fn interpretPjrtError(api: *const Api, pjrt_error: *Error, context: []const u8) ApiError { + defer pjrt_error.deinit(api); + const err_code = pjrt_error.getCode(api).toApiError(); + log.err("[{s}] {t}: {s}", .{ context, err_code, pjrt_error.getMessage(api) }); + return err_code; +} + fn InnerMixin(comptime innerT: type) type { return struct { fn inner(self: anytype) *innerT { @@ -169,8 +176,7 @@ pub const Api = struct { } if (result) |pjrt_c_error| { const pjrt_error: *Error = @ptrCast(pjrt_c_error); - log.err("[{s}] {s}", .{ @tagName(method), pjrt_error.getMessage(self) }); - return pjrt_error.getCode(self).toApiError(); + return interpretPjrtError(self, pjrt_error, @tagName(method)); } } @@ -1262,7 +1268,10 @@ pub const Buffer = opaque { const ret = try api.call(.PJRT_Buffer_OpaqueDeviceMemoryDataPointer, .{ .buffer = self.inner(), }); - return ret.device_memory_ptr.?; + return ret.device_memory_ptr orelse { + log.err("[PJRT_Buffer_OpaqueDeviceMemoryDataPointer] plugin returned success with null device_memory_ptr", .{}); + return error.Internal; + }; } pub fn copyRawToHost(self: *const Buffer, api: *const Api, dst: []u8, offset: i64) ApiError!?*Event { @@ -1346,12 +1355,7 @@ pub const Event = opaque { }.call, &ctx); ctx.event.waitUncancelable(io); - if (ctx.err) |e| { - defer e.deinit(api); - const err_code = e.getCode(api).toApiError(); - log.err("{t} {s}", .{ err_code, e.getMessage(api) }); - return err_code; - } + if (ctx.err) |err| return interpretPjrtError(api, err, "PJRT_Event_OnReady"); } pub fn awaitRaw(self: *const Event, api: *const Api) ApiError!void { From d539f6bdb8ad4e090950359d3f6e74c0deeba0b1 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Mon, 17 Aug 2026 19:11:16 +0200 Subject: [PATCH 614/881] zml/io: wait for loadExecute call before deiniting input buffers (#705) Sometimes there is a race where the input buffers get deinited at the end of loadExecute before the executable call finishes, which can corrupt data. Waiting on the call closes that race, but it also prevents concurrency, so we may need another way to keep the buffers alive without blocking. --- zml/io.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/io.zig b/zml/io.zig index 0e14d60b..48d7a390 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -472,7 +472,7 @@ pub const Loader = struct { var results = try exe.results(arena); args.set(.{buffers}); - exe.call(args, &results); + exe.callOpts(io, args, &results, .{ .wait = true }); buffer.* = results.get(Buffer); } From bb860a050b486a042c87efc3a32eaa3baca404aa Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Tue, 18 Aug 2026 09:56:05 +0200 Subject: [PATCH 615/881] platforms/oneapi: bundle oneDNN and TBB with PJRT update PJRT_ONEAPI_RELEASE (#706) platforms/oneapi: bundle oneDNN and TBB with PJRT Add oneDNN 2026.0 and TBB 2023.1 to the hermetic oneAPI packages. the pjrj plugin need it for the mistral models type with the support of the FP8 scaled matmul with libdnn. platforms/oneapi: update PJRT_ONEAPI_RELEASE and SHA256 for new artifact version --- platforms/oneapi/libpjrt_oneapi.BUILD.bazel | 21 +++++++++++++++++- platforms/oneapi/oneapi.bzl | 20 +++++++++++++++-- platforms/oneapi/packages.lock.json | 24 ++++++++++++++++++++- platforms/oneapi/packages.yaml | 2 ++ 4 files changed, 63 insertions(+), 4 deletions(-) diff --git a/platforms/oneapi/libpjrt_oneapi.BUILD.bazel b/platforms/oneapi/libpjrt_oneapi.BUILD.bazel index e1e5875f..ea7f51e9 100644 --- a/platforms/oneapi/libpjrt_oneapi.BUILD.bazel +++ b/platforms/oneapi/libpjrt_oneapi.BUILD.bazel @@ -13,6 +13,20 @@ patchelf( set_rpath = "$ORIGIN", ) +patchelf( + name = "libdnnl_so_3", + src = "@intel-oneapi-dnnl-2026.0//:libdnnl_so_3", + set_rpath = "$ORIGIN", + soname = "libdnnl.so.3", +) + +patchelf( + name = "libtbb_so_12", + src = "@intel-oneapi-tbb-2023.1//:libtbb_so_12", + set_rpath = "$ORIGIN", + soname = "libtbb.so.12", +) + patchelf( name = "libsycl_so", src = "@intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1//:libsycl_so", @@ -59,6 +73,7 @@ ONEAPI_MPI_LIB = "mpi/2021.18/lib" ONEAPI_MPI_LIBFABRIC_LIB = "mpi/2021.18/opt/mpi/libfabric/lib" ONEAPI_TCM_VERSION = "1.5" ONEAPI_UMF_VERSION = "1.1" + ONEAPI_COMPILER_LIB = "compiler/{}/lib".format(ONEAPI_VERSION) ONEAPI_MKL_LIB = "mkl/{}/lib".format(ONEAPI_VERSION) ONEAPI_TCM_LIB = "tcm/{}/lib".format(ONEAPI_TCM_VERSION) @@ -67,11 +82,13 @@ ONEAPI_UMF_LIB = "umf/{}/lib".format(ONEAPI_UMF_VERSION) copy_to_directory( name = "sandbox", srcs = [ + ":libdnnl_so_3", ":libiga64_so_2", - ":libigdfcl_so_2", ":libigc_so_2", + ":libigdfcl_so_2", ":libpjrt_oneapi_so", ":libsycl_so", + ":libtbb_so_12", ":libze_intel_gpu_so", "@intel-oneapi-ccl-2022.1//:ccl_runtime", "@intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1//:sycl_runtime", @@ -103,8 +120,10 @@ copy_to_directory( "libiga64_so_2": "lib", "libigdfcl_so_2": "lib", "libigc_so_2": "lib", + "libdnnl_so_3": "lib", "libpjrt_oneapi_so": "lib", "libsycl_so": "lib", + "libtbb_so_12": "lib", ONEAPI_CCL_LIB: "lib", ONEAPI_COMPILER_LIB: "lib", ONEAPI_MPI_LIB: "lib", diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index 66c113ff..fa3b121e 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -1,21 +1,27 @@ load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") -PJRT_ONEAPI_RELEASE = "manual-2026-07-17T21-30-00Z" -PJRT_ONEAPI_ARTIFACT_SHA256 = "8e28d1ff00e2193a40982294d6b055a8102f49eb2cb46f32a6076f9bb541e8a2" +PJRT_ONEAPI_RELEASE = "manual-2026-08-17T19-00-00Z" +PJRT_ONEAPI_ARTIFACT_SHA256 = "af68f460376245013c6d42b661c24e15685dea2aa9ea9f8267d1878ef0c363a1" PJRT_ONEAPI_ARTIFACT_URL = "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-oneapi_linux-amd64.tar.gz".format( release = PJRT_ONEAPI_RELEASE, ) ONEAPI_VERSION = "2026.1" +# www.intel.com/content/www/us/en/developer/tools/oneapi/onednn-download.html +ONEAPI_DNNL_VERSION = "2026.0" +# https://github.com/uxlfoundation/onetbb +ONEAPI_TBB_VERSION = "2023.1" ONEAPI_TCM_VERSION = "1.5" ONEAPI_UMF_VERSION = "1.1" ONEAPI_CCL_LIB = "ccl/2022.1/lib" ONEAPI_COMPILER_LIB = "compiler/{}/lib".format(ONEAPI_VERSION) +ONEAPI_DNNL_LIB = "dnnl/{}/lib".format(ONEAPI_DNNL_VERSION) ONEAPI_MPI_LIB = "mpi/2021.18/lib" ONEAPI_MPI_LIBFABRIC_LIB = "mpi/2021.18/opt/mpi/libfabric/lib" ONEAPI_MKL_LIB = "mkl/{}/lib".format(ONEAPI_VERSION) +ONEAPI_TBB_LIB = "tbb/{}/lib".format(ONEAPI_TBB_VERSION) ONEAPI_TCM_LIB = "tcm/{}/lib".format(ONEAPI_TCM_VERSION) ONEAPI_UMF_LIB = "umf/{}/lib".format(ONEAPI_UMF_VERSION) @@ -190,6 +196,16 @@ filegroup( ONEAPI_MPI_LIB = ONEAPI_MPI_LIB, ONEAPI_MPI_LIBFABRIC_LIB = ONEAPI_MPI_LIBFABRIC_LIB, ), + "intel-oneapi-dnnl-2026.0": """ +filegroup( + name = "libdnnl_so_3", + srcs = ["{ONEAPI_DNNL_LIB}/libdnnl.so.3.11"], +)""".format(ONEAPI_DNNL_LIB = ONEAPI_DNNL_LIB), + "intel-oneapi-tbb-2023.1": """ +filegroup( + name = "libtbb_so_12", + srcs = ["{ONEAPI_TBB_LIB}/libtbb.so.12.19"], +)""".format(ONEAPI_TBB_LIB = ONEAPI_TBB_LIB), "intel-oneapi-tcm-1.5": """ filegroup( name = "hwloc", diff --git a/platforms/oneapi/packages.lock.json b/platforms/oneapi/packages.lock.json index da9fd965..26a096ee 100644 --- a/platforms/oneapi/packages.lock.json +++ b/platforms/oneapi/packages.lock.json @@ -44,6 +44,17 @@ ], "version": "2026.1.0-235" }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-dnnl-2026.0_2026.0.2-40_amd64", + "name": "intel-oneapi-dnnl-2026.0", + "sha256": "4b0f7e23415a695f204f8080328977225759534158d813c2225403f8851433e8", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-dnnl-2026.0-2026.0.2-40_amd64.deb" + ], + "version": "2026.0.2-40" + }, { "arch": "amd64", "dependencies": [], @@ -110,6 +121,17 @@ ], "version": "2026.1.0-236" }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-tbb-2023.1_2023.1.0-151_amd64", + "name": "intel-oneapi-tbb-2023.1", + "sha256": "f6e4ae469436a8c5ee2250203268344164c9ff455a2207219c75c463aab99a8b", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-tbb-2023.1-2023.1.0-151_amd64.deb" + ], + "version": "2023.1.0-151" + }, { "arch": "amd64", "dependencies": [], @@ -244,4 +266,4 @@ } ], "version": 1 -} \ No newline at end of file +} diff --git a/platforms/oneapi/packages.yaml b/platforms/oneapi/packages.yaml index febdda06..6abf6e47 100644 --- a/platforms/oneapi/packages.yaml +++ b/platforms/oneapi/packages.yaml @@ -25,12 +25,14 @@ packages: - "intel-oneapi-mpi-2021.18" - "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1" - "intel-oneapi-compiler-shared-runtime-2026.1" + - "intel-oneapi-dnnl-2026.0" - "intel-oneapi-mkl-core-2026.1" - "intel-oneapi-mkl-sycl-blas-2026.1" - "intel-oneapi-mkl-sycl-dft-2026.1" - "intel-oneapi-mkl-sycl-lapack-2026.1" - "intel-oneapi-mkl-sycl-rng-2026.1" - "intel-oneapi-mkl-sycl-sparse-2026.1" + - "intel-oneapi-tbb-2023.1" - "intel-oneapi-tcm-1.5" - "intel-oneapi-umf-1.1" - "libnl-3-200" From c58d81a7b3b2be54d418774c63bd05521e1af337 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 18 Aug 2026 16:36:38 +0200 Subject: [PATCH 616/881] zml/exe: TypedExe (#697) in LLMD and //examples/llm, we compile multiple executable, re-use the args & results. We do all of this manually, in addition we don't have any type-safety on the arguments/outputs which is a bit too easy to misuse IMHO. So adding two abstractions: - Runner: Packs the args, exe and results together to avoid calling `args.set`, `exe.run` and `results.fill` with similar arguments. Today it owns the Exe. It doensn't strictly needs it, but it's simpler and works for us for now. - TypedExe: Type-safe version of an Exe, expecting a single input & ouptut to have keyed arguments to have fully unambiguous arguments & results mapping. I've updated //example/llm with this new pattern. Also adding an explicit `zml.module.CompileError`. --- examples/llm/chat.zig | 10 +- examples/llm/main.zig | 4 +- examples/llm/models.zig | 2 +- examples/llm/models/lfm2/inference.zig | 525 +++++------ examples/llm/models/lfm2/model.zig | 147 ++-- examples/llm/models/lfm2/session.zig | 53 +- examples/llm/models/lfm2_tests.zig | 5 +- examples/llm/models/llama/inference.zig | 711 +++++---------- examples/llm/models/llama/model.zig | 102 ++- examples/llm/models/llama/session.zig | 47 +- examples/llm/models/qwen3_5/inference.zig | 821 +++++------------- examples/llm/models/qwen3_5/model.zig | 124 ++- examples/llm/models/qwen3_5/session.zig | 69 +- examples/llm/models/qwen3_5_moe/inference.zig | 625 ++++++------- examples/llm/models/qwen3_5_moe/model.zig | 146 ++-- examples/llm/models/qwen3_5_moe/session.zig | 392 ++------- zml/exe.zig | 252 ++++++ zml/module.zig | 12 +- zml/zml.zig | 1 + 19 files changed, 1757 insertions(+), 2291 deletions(-) diff --git a/examples/llm/chat.zig b/examples/llm/chat.zig index a976c796..c02efd56 100644 --- a/examples/llm/chat.zig +++ b/examples/llm/chat.zig @@ -34,24 +34,26 @@ pub const Chat = struct { io: std.Io, platform: *const zml.Platform, tokenizer: zml.tokenizer.Tokenizer, - compiled_model: *const models.CompiledModel, + compiled_model: *models.CompiledModel, model_buffers: *models.Buffers, ) !Chat { - var session = try compiled_model.newSession( + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, compiled_model.seqlen); + errdefer tokens.deinit(allocator); + + const session = try compiled_model.newSession( allocator, io, platform, model_buffers, tokenizer, ); - errdefer session.deinit(); return .{ .allocator = allocator, .io = io, .platform = platform, .session = session, - .tokens = try .initCapacity(allocator, session.maxTokens()), + .tokens = tokens, }; } diff --git a/examples/llm/main.zig b/examples/llm/main.zig index 620421ea..6a924354 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -147,8 +147,8 @@ pub fn main(init: std.process.Init) !void { try printZmlLogo(io, init.environ_map); const interactive = args.prompt == null; - const prompt = if (args.prompt) |prompt| b: { - break :b try allocator.dupe(u8, prompt); + const prompt = if (args.prompt) |prompt_arg| b: { + break :b try allocator.dupe(u8, prompt_arg); } else b: { chat.initHistory(); const line = c.linenoise(chat.prompt_prefix) orelse return; diff --git a/examples/llm/models.zig b/examples/llm/models.zig index 8a5df1be..4195abec 100644 --- a/examples/llm/models.zig +++ b/examples/llm/models.zig @@ -110,7 +110,7 @@ pub const CompiledModel = struct { } pub fn newSession( - self: *const CompiledModel, + self: *CompiledModel, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, diff --git a/examples/llm/models/lfm2/inference.zig b/examples/llm/models/lfm2/inference.zig index 045c60e3..43913563 100644 --- a/examples/llm/models/lfm2/inference.zig +++ b/examples/llm/models/lfm2/inference.zig @@ -8,6 +8,7 @@ const Phase = common.Phase; const model = @import("model.zig"); const log = std.log.scoped(.lfm); + pub const CompilationParameters = struct { hidden_dim: usize, batch_dim: usize, @@ -52,10 +53,7 @@ pub const CompilationParameters = struct { pub const CompilationOptions = CompilationParameters; pub const Args = struct { - allocator: std.mem.Allocator, io: std.Io, - platform: *const zml.Platform, - model_buffers: *model.Buffers, tokens_buf: *zml.Buffer, tokens_pos_buf: *zml.Buffer, actual_seq_len_buf: *zml.Buffer, @@ -79,10 +77,13 @@ pub const CompiledModel = struct { opts: CompilationParameters, progress: *std.Progress.Node, ) !CompiledModel { + const prefill = try compileKernel(allocator, io, platform, mdl, opts, opts.seqlen, .prefill, progress); + errdefer prefill.deinit(); + const decode = try compileKernel(allocator, io, platform, mdl, opts, 1, .decode, progress); return .{ .loaded_model = loaded_model, - .prefill = try KernelExe.init(allocator, io, platform, mdl, opts, opts.seqlen, .prefill, progress), - .decode = try KernelExe.init(allocator, io, platform, mdl, opts, 1, .decode, progress), + .prefill = prefill, + .decode = decode, .params = opts, }; } @@ -95,323 +96,227 @@ pub const CompiledModel = struct { pub const Inference = CompiledModel; +pub const EmbedExe = zml.TypedExe(model.TokenEmbedding.forward); +pub const LayerExe = zml.TypedExe(model.DecoderLayer.forward); +pub const SampleExe = zml.TypedExe(model.LmHead.forward); + pub const KernelExe = struct { - composed: ComposedKernelExe, + embed: EmbedExe, + conv: LayerExe, + self_attn: LayerExe, + sample: SampleExe, - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - mdl: model.Model, - opts: CompilationOptions, - seqlen: u32, - phase: Phase, - progress: *std.Progress.Node, - ) !KernelExe { - return .{ - .composed = try ComposedKernelExe.init(allocator, io, platform, mdl, opts, seqlen, phase, progress), - }; - } - - pub fn deinit(self: KernelExe) void { - self.composed.deinit(); - } - - pub fn run(self: *const KernelExe, args: Args) !void { - try self.composed.run(args); + pub fn deinit(self: *const KernelExe) void { + self.embed.deinit(); + self.conv.deinit(); + self.self_attn.deinit(); + self.sample.deinit(); } }; -pub const ComposedKernelExe = struct { - embed_tokens: zml.Exe, - conv_layer: zml.Exe, - attn_layer: zml.Exe, - lm_head: zml.Exe, +pub const KernelRunner = struct { + embed: EmbedExe.Runner(.{.embedding}), + layers: []LayerExe.Runner(.{.layer}), + sample: SampleExe.Runner(.{ .lm_head, .embed_tokens }), - fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - mdl: model.Model, - opts: CompilationOptions, - seqlen: u32, - phase: Phase, - progress: *std.Progress.Node, - ) !ComposedKernelExe { - const embed_tokens = try ComposedKernelExe.compileEmbedTokens(allocator, io, platform, mdl.embed_tokens, opts, seqlen, phase, progress); - errdefer embed_tokens.deinit(); + pub fn init(allocator: std.mem.Allocator, exe: *const KernelExe, buffers: *const model.Buffers) !KernelRunner { + var embed = try EmbedExe.Runner(.{.embedding}).init(&exe.embed, allocator, .{ .embedding = buffers.embed_tokens }); + errdefer embed.deinit(allocator); - const conv_layer = try ComposedKernelExe.compileConvLayer(allocator, io, platform, mdl, opts, seqlen, phase, progress); - errdefer conv_layer.deinit(); - - const attn_layer = try ComposedKernelExe.compileAttnLayer(allocator, io, platform, mdl, opts, seqlen, phase, progress); - errdefer attn_layer.deinit(); - - const lm_head = try ComposedKernelExe.compileLmHead(allocator, io, platform, mdl, opts, seqlen, phase, progress); - errdefer lm_head.deinit(); - - return .{ - .embed_tokens = embed_tokens, - .conv_layer = conv_layer, - .attn_layer = attn_layer, - .lm_head = lm_head, - }; - } - - fn deinit(self: ComposedKernelExe) void { - self.embed_tokens.deinit(); - self.conv_layer.deinit(); - self.attn_layer.deinit(); - self.lm_head.deinit(); - } - - pub fn run(self: *const ComposedKernelExe, args: Args) !void { - var hidden_buf: zml.Buffer = b: { - var exe_args = try self.embed_tokens.args(args.allocator); - defer exe_args.deinit(args.allocator); - - var results = try self.embed_tokens.results(args.allocator); - defer results.deinit(args.allocator); - - exe_args.set(.{ args.model_buffers.embed_tokens, args.tokens_buf }); - self.embed_tokens.call(exe_args, &results); - break :b results.get(zml.Buffer); - }; - defer hidden_buf.deinit(); - - var conv_cache_index_buf: zml.Buffer = try .scalar(args.io, args.platform, 0, .u32); - defer conv_cache_index_buf.deinit(); - var kv_cache_index_buf: zml.Buffer = try .scalar(args.io, args.platform, 0, .u32); - defer kv_cache_index_buf.deinit(); - - for (args.model_buffers.layers) |layer_bufs| { - const exe = switch (layer_bufs.operator) { - .conv => &self.conv_layer, - .self_attn => &self.attn_layer, + const layers = try allocator.alloc(LayerExe.Runner(.{.layer}), buffers.layers.len); + errdefer allocator.free(layers); + var initialized_layers: usize = 0; + errdefer for (layers[0..initialized_layers]) |*layer| layer.deinit(allocator); + for (layers, buffers.layers) |*layer, layer_buffers| { + const layer_exe: *const LayerExe = switch (layer_buffers.operator) { + .conv => &exe.conv, + .self_attn => &exe.self_attn, }; - - var exe_args = try exe.args(args.allocator); - defer exe_args.deinit(args.allocator); - var results = try exe.results(args.allocator); - defer results.deinit(args.allocator); - - exe_args.set(.{ - layer_bufs, - &hidden_buf, - args.tokens_pos_buf, - args.actual_seq_len_buf, - args.cache_buffers, - &conv_cache_index_buf, - &kv_cache_index_buf, - args.attention_metadata_buffers, - }); - ComposedKernelExe.runLayer(exe, &exe_args, &results, args.cache_buffers, &hidden_buf, &conv_cache_index_buf, &kv_cache_index_buf); + layer.* = try LayerExe.Runner(.{.layer}).init(layer_exe, allocator, .{ .layer = layer_buffers }); + initialized_layers += 1; } - var exe_args = try self.lm_head.args(args.allocator); - defer exe_args.deinit(args.allocator); - var results = try self.lm_head.results(args.allocator); - defer results.deinit(args.allocator); - - exe_args.set(.{ args.model_buffers.lm_head, hidden_buf, args.model_buffers.embed_tokens, args.tokens_buf, args.rng_buf }); - self.runLmHead(&exe_args, &results, args.tokens_buf, args.rng_buf); - } - - fn runLayer( - exe: *const zml.Exe, - exe_args: *zml.exe.Exe.Arguments, - results: *zml.exe.Exe.Results, - cache_buffers: *zml.Bufferized(model.Cache), - hidden_buf: *zml.Buffer, - conv_cache_index_buf: *zml.Buffer, - kv_cache_index_buf: *zml.Buffer, - ) void { - exe.call(exe_args.*, results); - - var new_hidden, var new_cache, var new_conv_cache_index, var new_kv_cache_index = results.get(struct { - zml.Buffer, - zml.Bufferized(model.Cache), - zml.Buffer, - zml.Buffer, + var sample = try SampleExe.Runner(.{ .lm_head, .embed_tokens }).init(&exe.sample, allocator, .{ + .lm_head = buffers.lm_head, + .embed_tokens = buffers.embed_tokens, }); - ComposedKernelExe.replaceBuffer(hidden_buf, &new_hidden); - ComposedKernelExe.replaceCacheBuffers(cache_buffers, &new_cache); - ComposedKernelExe.replaceBuffer(conv_cache_index_buf, &new_conv_cache_index); - ComposedKernelExe.replaceBuffer(kv_cache_index_buf, &new_kv_cache_index); + errdefer sample.deinit(allocator); + + return .{ .embed = embed, .layers = layers, .sample = sample }; } - fn runLmHead( - self: *const ComposedKernelExe, - exe_args: *zml.exe.Exe.Arguments, - results: *zml.exe.Exe.Results, - tokens_buf: *zml.Buffer, - rng_buf: *zml.Bufferized(zml.Tensor.Rng), - ) void { - self.lm_head.call(exe_args.*, results); - - var new_tokens, var new_rng = results.get(struct { - zml.Buffer, - zml.Bufferized(zml.Tensor.Rng), - }); - ComposedKernelExe.replaceBuffer(tokens_buf, &new_tokens); - ComposedKernelExe.replaceBuffer(&rng_buf._state, &new_rng._state); - } - - fn replaceCacheBuffers(dst: *zml.Bufferized(model.Cache), src: *zml.Bufferized(model.Cache)) void { - ComposedKernelExe.replaceBuffer(&dst.conv.state, &src.conv.state); - ComposedKernelExe.replaceBuffer(&dst.kv.k, &src.kv.k); - ComposedKernelExe.replaceBuffer(&dst.kv.v, &src.kv.v); - } - - fn replaceBuffer(dst: *zml.Buffer, src: *zml.Buffer) void { - if (!ComposedKernelExe.sameBufferHandle(dst.*, src.*)) { - dst.deinit(); - } - dst.* = src.*; - } - - fn sameBufferHandle(a: zml.Buffer, b: zml.Buffer) bool { - if (a._shards.len != b._shards.len) return false; - for (a._shards.constSlice(), b._shards.constSlice()) |a_shard, b_shard| { - if (a_shard != b_shard) return false; - } - return true; - } - - fn compileEmbedTokens( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - embed_tokens: model.TokenEmbedding, - opts: CompilationOptions, - seqlen: u32, - phase: Phase, - progress: *std.Progress.Node, - ) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start(phase.startMessage("embed_tokens"), 1); - defer node.end(); - - const from: std.Io.Timestamp = .now(io, .awake); - defer phase.logCompileDone(log, "embed_tokens", io, from); - - const tokens: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32); - - return platform.compile(allocator, io, embed_tokens, .forward, .{tokens}, .{ - .shardings = &opts.shardings.all(), - .program_name = phase.programName("lfm2", "embed_tokens"), - }); - } - - fn compileConvLayer( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - mdl: model.Model, - opts: CompilationOptions, - seqlen: u32, - phase: Phase, - progress: *std.Progress.Node, - ) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start(phase.startMessage("conv layer"), 1); - defer node.end(); - - const from: std.Io.Timestamp = .now(io, .awake); - defer phase.logCompileDone(log, "conv layer", io, from); - - const conv_layer = for (mdl.layers) |layer| { - if (layer.operator == .conv) break layer; - } else unreachable; - - const hidden: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()); - const token_position_offset: zml.Tensor = .init(.{ .batch = opts.batch_dim }, .u32); - const actual_seq_len: zml.Tensor = .init(.{}, .u32); - const conv_cache_index: zml.Tensor = .init(.{}, .u32); - const kv_cache_index: zml.Tensor = .init(.{}, .u32); - - return platform.compile(allocator, io, conv_layer, .forward, .{ - hidden, - token_position_offset, - actual_seq_len, - opts.cache, - conv_cache_index, - kv_cache_index, - opts.attention_metadata, - opts.attention_parameters, - model.ConvParameters{ .is_prefill = phase.isPrefill() }, - }, .{ - .shardings = &opts.shardings.all(), - .program_name = phase.programName("lfm2", "conv_layer"), - }); - } - - fn compileAttnLayer( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - mdl: model.Model, - opts: CompilationOptions, - seqlen: u32, - phase: Phase, - progress: *std.Progress.Node, - ) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start(phase.startMessage("attn layer"), 1); - defer node.end(); - - const from: std.Io.Timestamp = .now(io, .awake); - defer phase.logCompileDone(log, "attn layer", io, from); - - const attn_layer = for (mdl.layers) |layer| { - if (layer.operator == .self_attn) break layer; - } else unreachable; - - const hidden: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()); - const token_position_offset: zml.Tensor = .init(.{ .batch = opts.batch_dim }, .u32); - const actual_seq_len: zml.Tensor = .init(.{}, .u32); - const conv_cache_index: zml.Tensor = .init(.{}, .u32); - const kv_cache_index: zml.Tensor = .init(.{}, .u32); - - return platform.compile(allocator, io, attn_layer, .forward, .{ - hidden, - token_position_offset, - actual_seq_len, - opts.cache, - conv_cache_index, - kv_cache_index, - opts.attention_metadata, - opts.attention_parameters, - model.ConvParameters{ .is_prefill = phase.isPrefill() }, - }, .{ - .shardings = &opts.shardings.all(), - .program_name = phase.programName("lfm2", "attn_layer"), - }); - } - - fn compileLmHead( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - mdl: model.Model, - opts: CompilationOptions, - seqlen: u32, - phase: Phase, - progress: *std.Progress.Node, - ) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start(phase.startMessage("lm_head"), 1); - defer node.end(); - - const from: std.Io.Timestamp = .now(io, .awake); - defer phase.logCompileDone(log, "lm_head", io, from); - - const hidden: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()); - const tokens: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32); - - return platform.compile(allocator, io, mdl.lm_head, .forward, .{ hidden, mdl.embed_tokens, tokens, opts.rng }, .{ - .shardings = &opts.shardings.all(), - .program_name = phase.programName("lfm2", "lm_head"), - }); + pub fn deinit(self: *KernelRunner, allocator: std.mem.Allocator) void { + self.embed.deinit(allocator); + for (self.layers) |*layer| layer.deinit(allocator); + allocator.free(self.layers); + self.sample.deinit(allocator); } }; + +pub fn run( + runner: *KernelRunner, + args: Args, + conv_cache_index_buffer: *zml.Buffer, + kv_cache_index_buffer: *zml.Buffer, +) void { + var hidden_buffer: zml.Buffer = undefined; + runner.embed.run(args.io, .{ + .inputs = .{ + .tokens = args.tokens_buf.*, + }, + .outputs = .{ .hidden = &hidden_buffer }, + }); + defer hidden_buffer.deinit(); + + for (runner.layers) |*layer| { + layer.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .tokens_position_offset = args.tokens_pos_buf.*, + .actual_seq_len = args.actual_seq_len_buf.*, + .cache = args.cache_buffers.*, + .conv_cache_index = conv_cache_index_buffer.*, + .kv_cache_index = kv_cache_index_buffer.*, + .attention_metadata = args.attention_metadata_buffers, + }, + .outputs = .{ + .hidden = &hidden_buffer, + .cache = args.cache_buffers, + .conv_cache_index = conv_cache_index_buffer, + .kv_cache_index = kv_cache_index_buffer, + }, + }); + } + + runner.sample.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .tokens = args.tokens_buf.*, + .rng = args.rng_buf.*, + }, + .outputs = .{ + .tokens = args.tokens_buf, + .rng = args.rng_buf, + }, + }); +} + +fn compileKernel( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + mdl: model.Model, + opts: CompilationOptions, + seqlen: u32, + phase: Phase, + progress: *std.Progress.Node, +) !KernelExe { + const embed = try compileEmbed(allocator, io, platform, mdl.embed_tokens, opts, seqlen, phase, progress); + errdefer embed.deinit(); + const conv_layer = try compileLayer(allocator, io, platform, mdl, opts, seqlen, .conv, phase, progress); + errdefer conv_layer.deinit(); + const attn_layer = try compileLayer(allocator, io, platform, mdl, opts, seqlen, .full_attention, phase, progress); + errdefer attn_layer.deinit(); + const sample = try compileSample(allocator, io, platform, mdl, opts, seqlen, phase, progress); + errdefer sample.deinit(); + return .{ .embed = embed, .conv = conv_layer, .self_attn = attn_layer, .sample = sample }; +} + +fn compileEmbed( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + embed_tokens: model.TokenEmbedding, + opts: CompilationOptions, + seqlen: u32, + phase: Phase, + progress: *std.Progress.Node, +) !EmbedExe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("embed_tokens"), 1); + defer node.end(); + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "embed_tokens", io, from); + + return EmbedExe.compile(allocator, io, platform, .{ + .shardings = &opts.shardings.all(), + .program_name = phase.programName("lfm2", "embed_tokens"), + }, .{.{ + .embedding = embed_tokens, + .tokens = zml.Tensor.init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32), + }}); +} + +fn compileLayer( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + mdl: model.Model, + opts: CompilationOptions, + seqlen: u32, + comptime kind: model.OperatorKind, + phase: Phase, + progress: *std.Progress.Node, +) !LayerExe { + const label = switch (kind) { + .conv => "conv layer", + .full_attention => "attn layer", + }; + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage(label), 1); + defer node.end(); + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, label, io, from); + + const layer = for (mdl.layers) |candidate| { + const candidate_kind: model.OperatorKind = switch (candidate.operator) { + .conv => .conv, + .self_attn => .full_attention, + }; + if (candidate_kind == kind) break candidate; + } else unreachable; + + return LayerExe.compile(allocator, io, platform, .{ + .shardings = &opts.shardings.all(), + .program_name = phase.programName("lfm2", if (kind == .conv) "conv_layer" else "attn_layer"), + }, .{.{ + .layer = layer, + .hidden = zml.Tensor.init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()), + .tokens_position_offset = zml.Tensor.init(.{ .batch = opts.batch_dim }, .u32), + .actual_seq_len = zml.Tensor.init(.{}, .u32), + .cache = opts.cache, + .conv_cache_index = zml.Tensor.init(.{}, .u32), + .kv_cache_index = zml.Tensor.init(.{}, .u32), + .attention_metadata = opts.attention_metadata, + .attention_parameters = opts.attention_parameters, + .conv_parameters = .{ .is_prefill = phase.isPrefill() }, + }}); +} + +fn compileSample( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + mdl: model.Model, + opts: CompilationOptions, + seqlen: u32, + phase: Phase, + progress: *std.Progress.Node, +) !SampleExe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("lm_head"), 1); + defer node.end(); + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "lm_head", io, from); + + return SampleExe.compile(allocator, io, platform, .{ + .shardings = &opts.shardings.all(), + .program_name = phase.programName("lfm2", "lm_head"), + }, .{.{ + .lm_head = mdl.lm_head, + .embed_tokens = mdl.embed_tokens, + .hidden = zml.Tensor.init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()), + .tokens = zml.Tensor.init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32), + .rng = opts.rng, + }}); +} diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index bcc25fdc..5df4628b 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -222,7 +222,10 @@ pub const Model = struct { ) struct { zml.Tensor, Cache, zml.Tensor.Rng } { stdx.debug.assert(tokens.shape().hasTags(.{ .batch, .seq }), "Tokens should have tags {{.batch, .seq}}, got {f}", .{tokens.shape()}); - const embeds = self.embed_tokens.forward(tokens); + const embeds = TokenEmbedding.forward(.{ + .embedding = self.embed_tokens, + .tokens = tokens, + }).hidden; var hidden = embeds; var cache = cache_; @@ -230,20 +233,31 @@ pub const Model = struct { var conv_cache_index = zml.Tensor.scalar(@as(u32, 0), .u32); var kv_cache_index = zml.Tensor.scalar(@as(u32, 0), .u32); for (self.layers) |layer| { - hidden, cache, conv_cache_index, kv_cache_index = layer.forward( - hidden, - tokens_position_offset, - actual_seq_len, - cache, - conv_cache_index, - kv_cache_index, - attention_metadata, - attention_parameters, - conv_parameters, - ); + const result = DecoderLayer.forward(.{ + .layer = layer, + .hidden = hidden, + .tokens_position_offset = tokens_position_offset, + .actual_seq_len = actual_seq_len, + .cache = cache, + .conv_cache_index = conv_cache_index, + .kv_cache_index = kv_cache_index, + .attention_metadata = attention_metadata, + .attention_parameters = attention_parameters, + .conv_parameters = conv_parameters, + }); + hidden = result.hidden; + cache = result.cache; + conv_cache_index = result.conv_cache_index; + kv_cache_index = result.kv_cache_index; } - const new_tokens, const new_rng = self.lm_head.forward(hidden, self.embed_tokens, tokens, rng); - return .{ new_tokens, cache.reuseBuffer(cache_), new_rng }; + const result = LmHead.forward(.{ + .lm_head = self.lm_head, + .embed_tokens = self.embed_tokens, + .hidden = hidden, + .tokens = tokens, + .rng = rng, + }); + return .{ result.tokens, cache.reuseBuffer(cache_), result.rng }; } pub fn deinit(self: Model, allocator: std.mem.Allocator) void { @@ -254,14 +268,23 @@ pub const Model = struct { pub const TokenEmbedding = struct { weight: zml.Tensor, + pub const Input = struct { + embedding: TokenEmbedding, + tokens: zml.Tensor, + }; + + pub const Output = struct { + hidden: zml.Tensor, + }; + pub fn init(store: zml.io.TensorStore.View) TokenEmbedding { return .{ .weight = store.createTensor("weight", .{ .voc, .d }, .{ .voc = .replicated, .d = .model }) }; } - pub fn forward(self: TokenEmbedding, tokens: zml.Tensor) zml.Tensor { - stdx.debug.assert(tokens.dtype().isInteger(), "TokenEmbedding expects an integer input, received: {f}", .{tokens}); - stdx.debug.assert(self.weight.rank() == 2, "TokenEmbedding expects it's weight zml.Tensor to be a 2D matrix, got {f}", .{self.weight}); - return self.weight.gather(.{ .voc = tokens }, .{}); + pub fn forward(input: Input) Output { + stdx.debug.assert(input.tokens.dtype().isInteger(), "TokenEmbedding expects an integer input, received: {f}", .{input.tokens}); + stdx.debug.assert(input.embedding.weight.rank() == 2, "TokenEmbedding expects it's weight zml.Tensor to be a 2D matrix, got {f}", .{input.embedding.weight}); + return .{ .hidden = input.embedding.weight.gather(.{ .voc = input.tokens }, .{}) }; } pub fn unembed(self: TokenEmbedding, embeds: zml.Tensor) zml.Tensor { @@ -274,6 +297,19 @@ pub const LmHead = struct { embedding_norm: RmsNorm, sampling_strategy: zml.nn.SamplingStrategy, + pub const Input = struct { + lm_head: LmHead, + embed_tokens: TokenEmbedding, + hidden: zml.Tensor, + tokens: zml.Tensor, + rng: zml.Tensor.Rng, + }; + + pub const Output = struct { + tokens: zml.Tensor, + rng: zml.Tensor.Rng, + }; + pub fn init(store: zml.io.TensorStore.View, config: Config, sampling_strategy: zml.nn.SamplingStrategy) LmHead { return .{ .embedding_norm = RmsNorm.init(store.withPrefix("embedding_norm"), config.norm_eps, .d), @@ -281,10 +317,11 @@ pub const LmHead = struct { }; } - pub fn forward(self: LmHead, hidden: zml.Tensor, embed_tokens: TokenEmbedding, tokens: zml.Tensor, rng: zml.Tensor.Rng) struct { zml.Tensor, zml.Tensor.Rng } { - const logits = embed_tokens.unembed(self.embedding_norm.forward(hidden)); - const new_tokens, const new_rng = zml.nn.sampleTokens(logits, self.sampling_strategy, rng); - return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), new_rng }; + pub fn forward(input: Input) Output { + const self = input.lm_head; + const logits = input.embed_tokens.unembed(self.embedding_norm.forward(input.hidden)); + const new_tokens, const new_rng = zml.nn.sampleTokens(logits, self.sampling_strategy, input.rng); + return .{ .tokens = new_tokens.convert(input.tokens.dtype()).reuseBuffer(input.tokens), .rng = new_rng }; } pub fn unloadBuffers(self: *zml.Bufferized(LmHead)) void { @@ -301,6 +338,26 @@ pub const DecoderLayer = struct { ffn_norm: RmsNorm, feed_forward: Mlp, + pub const Input = struct { + layer: DecoderLayer, + hidden: zml.Tensor, + tokens_position_offset: zml.Tensor, + actual_seq_len: zml.Tensor, + cache: Cache, + conv_cache_index: zml.Tensor, + kv_cache_index: zml.Tensor, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, + conv_parameters: ConvParameters, + }; + + pub const Output = struct { + hidden: zml.Tensor, + cache: Cache, + conv_cache_index: zml.Tensor, + kv_cache_index: zml.Tensor, + }; + pub fn parseOperatorKind(layer_type: []const u8) OperatorKind { return std.meta.stringToEnum(OperatorKind, layer_type) orelse { stdx.debug.assert(false, "Unsupported layer type {s}", .{layer_type}); @@ -325,30 +382,20 @@ pub const DecoderLayer = struct { }; } - pub fn forward( - self: DecoderLayer, - input: zml.Tensor, - tokens_position_offset: zml.Tensor, - actual_seq_len: zml.Tensor, - cache_: Cache, - conv_cache_index_: zml.Tensor, - kv_cache_index_: zml.Tensor, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, - conv_parameters: ConvParameters, - ) struct { zml.Tensor, Cache, zml.Tensor, zml.Tensor } { - var cache = cache_; - var conv_cache_index = conv_cache_index_; - var kv_cache_index = kv_cache_index_; + pub fn forward(input: Input) Output { + const self = input.layer; + var cache = input.cache; + var conv_cache_index = input.conv_cache_index; + var kv_cache_index = input.kv_cache_index; const residual = switch (self.operator) { .conv => |operator| b: { const residual, const updated_conv_cache = operator.forward( - self.operator_norm.forward(input), - tokens_position_offset, - actual_seq_len, + self.operator_norm.forward(input.hidden), + input.tokens_position_offset, + input.actual_seq_len, cache.conv, conv_cache_index, - conv_parameters, + input.conv_parameters, ); cache.conv = updated_conv_cache; conv_cache_index = conv_cache_index.add(zml.Tensor.scalar(@as(u32, 1), .u32)); @@ -356,12 +403,12 @@ pub const DecoderLayer = struct { }, .self_attn => |operator| b: { const residual, const updated_kv_cache = operator.forward( - self.operator_norm.forward(input), - tokens_position_offset, + self.operator_norm.forward(input.hidden), + input.tokens_position_offset, cache.kv, kv_cache_index, - attention_metadata, - attention_parameters, + input.attention_metadata, + input.attention_parameters, ); cache.kv = updated_kv_cache; kv_cache_index = kv_cache_index.add(zml.Tensor.scalar(@as(u32, 1), .u32)); @@ -369,13 +416,13 @@ pub const DecoderLayer = struct { }, }; - const x = input.add(residual); + const x = input.hidden.add(residual); return .{ - x.add(self.feed_forward.forward(self.ffn_norm.forward(x))).reuseBuffer(input), - cache.reuseBuffer(cache_), - conv_cache_index.reuseBuffer(conv_cache_index_), - kv_cache_index.reuseBuffer(kv_cache_index_), + .hidden = x.add(self.feed_forward.forward(self.ffn_norm.forward(x))).reuseBuffer(input.hidden), + .cache = cache.reuseBuffer(input.cache), + .conv_cache_index = conv_cache_index.reuseBuffer(input.conv_cache_index), + .kv_cache_index = kv_cache_index.reuseBuffer(input.kv_cache_index), }; } diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig index d841ff6f..67287861 100644 --- a/examples/llm/models/lfm2/session.zig +++ b/examples/llm/models/lfm2/session.zig @@ -9,8 +9,9 @@ pub const Session = struct { allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, - model_buffers: *model.Buffers, - compiled_model: *const inference.CompiledModel, + compiled_model: *inference.CompiledModel, + prefill: inference.KernelRunner, + decode: inference.KernelRunner, config: *const model.Config, seqlen: u32, cache_buffers: zml.Bufferized(model.Cache), @@ -25,28 +26,42 @@ pub const Session = struct { io: std.Io, platform: *const zml.Platform, tokenizer: zml.tokenizer.Tokenizer, - compiled_model: *const inference.CompiledModel, + compiled_model: *inference.CompiledModel, model_buffers: *model.Buffers, ) !Session { const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); + var cache_buffers = try compiled_model.params.cache.initBuffers(allocator, io, platform, compiled_model.params.shardings.model); + errdefer model.Cache.unloadBuffers(&cache_buffers); + var rng_buf = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed); + errdefer zml.Tensor.Rng.deinitBuffer(&rng_buf); + const generated_token_slice = try zml.Slice.alloc(allocator, zml.Shape.init(.{ .batch = 1, .seq = 1 }, .u32)); + errdefer generated_token_slice.free(allocator); + + var prefill = try inference.KernelRunner.init(allocator, &compiled_model.prefill, model_buffers); + errdefer prefill.deinit(allocator); + const decode = try inference.KernelRunner.init(allocator, &compiled_model.decode, model_buffers); + return .{ .allocator = allocator, .io = io, .platform = platform, - .model_buffers = model_buffers, .compiled_model = compiled_model, + .prefill = prefill, + .decode = decode, .tokenizer = tokenizer, .config = &compiled_model.loaded_model.parsed_config.value, .seqlen = compiled_model.params.seqlen, - .cache_buffers = try compiled_model.params.cache.initBuffers(allocator, io, platform, compiled_model.params.shardings.model), - .rng_buf = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed), - .generated_token_slice = try .alloc(allocator, zml.Shape.init(.{ .batch = 1, .seq = 1 }, .u32)), + .cache_buffers = cache_buffers, + .rng_buf = rng_buf, + .generated_token_slice = generated_token_slice, .think_start = tokenizer.tokenId("") orelse unreachable, .think_end = tokenizer.tokenId("") orelse unreachable, }; } pub fn deinit(self: *Session) void { + self.prefill.deinit(self.allocator); + self.decode.deinit(self.allocator); model.Cache.unloadBuffers(&self.cache_buffers); zml.Tensor.Rng.deinitBuffer(&self.rng_buf); self.generated_token_slice.free(self.allocator); @@ -123,18 +138,20 @@ pub const Session = struct { }; defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); - try self.compiled_model.prefill.run(.{ - .allocator = self.allocator, + var conv_cache_index_buffer: zml.Buffer = try .scalar(self.io, self.platform, 0, .u32); + defer conv_cache_index_buffer.deinit(); + var kv_cache_index_buffer: zml.Buffer = try .scalar(self.io, self.platform, 0, .u32); + defer kv_cache_index_buffer.deinit(); + + inference.run(&self.prefill, .{ .io = self.io, - .platform = self.platform, - .model_buffers = self.model_buffers, .tokens_buf = &tokens_buf, .tokens_pos_buf = &tokens_pos_buf, .actual_seq_len_buf = &actual_seq_len_buf, .rng_buf = &self.rng_buf, .cache_buffers = &self.cache_buffers, .attention_metadata_buffers = attention_metadata_buffers, - }); + }, &conv_cache_index_buffer, &kv_cache_index_buffer); try tokens_buf.toSlice(self.io, tokens_slice); self.generated_token_slice.items(u32)[0] = tokens_slice.items(u32)[all_tokens.len - 1]; @@ -180,18 +197,20 @@ pub const Session = struct { var token_pos_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice, .replicated); defer token_pos_buffer.deinit(); - try self.compiled_model.decode.run(.{ - .allocator = self.allocator, + var conv_cache_index_buffer: zml.Buffer = try .scalar(self.io, self.platform, 0, .u32); + defer conv_cache_index_buffer.deinit(); + var kv_cache_index_buffer: zml.Buffer = try .scalar(self.io, self.platform, 0, .u32); + defer kv_cache_index_buffer.deinit(); + + inference.run(&self.decode, .{ .io = self.io, - .platform = self.platform, - .model_buffers = self.model_buffers, .tokens_buf = ¤t_token_buffer, .tokens_pos_buf = &token_pos_buffer, .actual_seq_len_buf = &actual_seq_len_buf, .rng_buf = &self.rng_buf, .cache_buffers = &self.cache_buffers, .attention_metadata_buffers = attention_metadata_buffers, - }); + }, &conv_cache_index_buffer, &kv_cache_index_buffer); try current_token_buffer.toSlice(self.io, self.generated_token_slice); } diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig index 040a1a68..aae2e608 100644 --- a/examples/llm/models/lfm2_tests.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -155,7 +155,10 @@ const TestContext = struct { var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key, self.sharding); defer out_buffer_expected.deinit(); - const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor }, .{ .shardings = &.{self.sharding} }); + const exe = if (comptime @TypeOf(layer) == model.TokenEmbedding) + try self.platform.compileFn(self.allocator, self.io, model.TokenEmbedding.forward, .{.{ .embedding = layer, .tokens = in_tensor }}, .{ .shardings = &.{self.sharding} }) + else + try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor }, .{ .shardings = &.{self.sharding} }); defer exe.deinit(); var args = try exe.args(self.allocator); diff --git a/examples/llm/models/llama/inference.zig b/examples/llm/models/llama/inference.zig index 80c3c0ac..7795de78 100644 --- a/examples/llm/models/llama/inference.zig +++ b/examples/llm/models/llama/inference.zig @@ -67,10 +67,7 @@ pub const CompilationParameters = struct { pub const CompilationOptions = CompilationParameters; pub const Args = struct { - allocator: std.mem.Allocator, io: std.Io, - platform: *const zml.Platform, - model_buffers: *model.Buffers, tokens_buf: *zml.Buffer, token_index_buf: *zml.Buffer, kv_cache_buffers: *zml.Bufferized(model.KvCache), @@ -93,10 +90,14 @@ pub const CompiledModel = struct { parameters: CompilationParameters, progress: *std.Progress.Node, ) !CompiledModel { + const prefill = try compileKernel(allocator, io, platform, llama_model, parameters, @intCast(parameters.prefill_tokens.dim(.s)), parameters.prefill_attention_parameters, .prefill, progress); + errdefer prefill.deinit(); + const decode = try compileKernel(allocator, io, platform, llama_model, parameters, @intCast(parameters.decode_tokens.dim(.s)), parameters.decode_attention_parameters, .decode, progress); + return .{ .loaded_model = loaded_model, - .prefill = try KernelExe.init(allocator, io, platform, llama_model, parameters, @intCast(parameters.prefill_tokens.dim(.s)), parameters.prefill_attention_parameters, .prefill, progress), - .decode = try KernelExe.init(allocator, io, platform, llama_model, parameters, @intCast(parameters.decode_tokens.dim(.s)), parameters.decode_attention_parameters, .decode, progress), + .prefill = prefill, + .decode = decode, .params = parameters, }; } @@ -109,489 +110,227 @@ pub const CompiledModel = struct { pub const Inference = CompiledModel; +pub const EmbedExe = zml.TypedExe(model.EmbedTokens.forward); +pub const LayerExe = zml.TypedExe(model.TransformerLayer.forward); +pub const SampleExe = zml.TypedExe(model.LmHead.forward); + pub const KernelExe = struct { - composed: ComposedKernelExe, + embed: EmbedExe, + layer: LayerExe, + sample: SampleExe, - pub const Runner = struct { - exe: *const ComposedKernelExe, - embed_args: zml.exe.Exe.Arguments, - embed_results: zml.exe.Exe.Results, - layers: Layers, - lm_head_args: zml.exe.Exe.Arguments, - lm_head_results: zml.exe.Exe.Results, - - const Layers = struct { - args: []zml.exe.Exe.Arguments, - results: []zml.exe.Exe.Results, - kv_cache_indices: []zml.Buffer, - - fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - exe: *const ComposedKernelExe, - model_buffers: *model.Buffers, - ) !Layers { - const args = try allocator.alloc(zml.exe.Exe.Arguments, model_buffers.model.layers.len); - errdefer allocator.free(args); - - const results = try allocator.alloc(zml.exe.Exe.Results, model_buffers.model.layers.len); - errdefer allocator.free(results); - - const kv_cache_indices = try allocator.alloc(zml.Buffer, model_buffers.model.layers.len); - errdefer allocator.free(kv_cache_indices); - - var initialized_args: usize = 0; - errdefer { - for (args[0..initialized_args]) |*exe_args| { - exe_args.deinit(allocator); - } - } - - var initialized_results: usize = 0; - errdefer { - for (results[0..initialized_results]) |*exe_results| { - exe_results.deinit(allocator); - } - } - - var initialized_kv_cache_indices: usize = 0; - errdefer { - for (kv_cache_indices[0..initialized_kv_cache_indices]) |*kv_cache_index| { - kv_cache_index.deinit(); - } - } - - for (model_buffers.model.layers, 0..) |layer_bufs, i| { - args[i] = try exe.layer.args(allocator); - initialized_args = i + 1; - args[i].bake(layer_bufs); - - results[i] = try exe.layer.results(allocator); - initialized_results = i + 1; - - kv_cache_indices[i] = try zml.Buffer.scalar(io, platform, i, .u32); - initialized_kv_cache_indices = i + 1; - } - - return .{ .args = args, .results = results, .kv_cache_indices = kv_cache_indices }; - } - - fn deinit(self: *Layers, allocator: std.mem.Allocator) void { - for (self.args) |*exe_args| { - exe_args.deinit(allocator); - } - allocator.free(self.args); - - for (self.results) |*exe_results| { - exe_results.deinit(allocator); - } - allocator.free(self.results); - - for (self.kv_cache_indices) |*kv_cache_index| { - kv_cache_index.deinit(); - } - allocator.free(self.kv_cache_indices); - } - }; - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - exe: *const ComposedKernelExe, - model_buffers: *model.Buffers, - ) !Runner { - var embed_args = try exe.embed_tokens.args(allocator); - errdefer embed_args.deinit(allocator); - - embed_args.bake(ComposedKernelExe.embedTokensBuffers(model_buffers)); - - var embed_results = try exe.embed_tokens.results(allocator); - errdefer embed_results.deinit(allocator); - - var layers = try Layers.init(allocator, io, platform, exe, model_buffers); - errdefer layers.deinit(allocator); - - var lm_head_args = try exe.lm_head.args(allocator); - errdefer lm_head_args.deinit(allocator); - - lm_head_args.bake(ComposedKernelExe.lmHeadBuffers(model_buffers)); - - var lm_head_results = try exe.lm_head.results(allocator); - errdefer lm_head_results.deinit(allocator); - - return .{ - .exe = exe, - .embed_args = embed_args, - .embed_results = embed_results, - .layers = layers, - .lm_head_args = lm_head_args, - .lm_head_results = lm_head_results, - }; - } - - pub fn deinit(self: *Runner, allocator: std.mem.Allocator) void { - self.embed_args.deinit(allocator); - self.embed_results.deinit(allocator); - self.layers.deinit(allocator); - self.lm_head_args.deinit(allocator); - self.lm_head_results.deinit(allocator); - } - - pub fn run(self: *Runner, args: Args) !void { - var hidden_buf: zml.Buffer = b: { - self.embed_args.set(.{args.tokens_buf}); - self.exe.embed_tokens.call(self.embed_args, &self.embed_results); - - break :b self.embed_results.get(zml.Buffer); - }; - defer hidden_buf.deinit(); - - for (self.layers.args, self.layers.results, self.layers.kv_cache_indices) |*exe_args, *results, *kv_cache_index_buf| { - self.exe.runLayer(exe_args, results, args, &hidden_buf, kv_cache_index_buf); - } - - self.exe.runLmHead(&self.lm_head_args, &self.lm_head_results, args, &hidden_buf); - } - }; - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - llama_model: model.Model, - parameters: CompilationOptions, - seqlen: usize, - attention_parameters: zml.attention.Parameters, - phase: Phase, - progress: *std.Progress.Node, - ) !KernelExe { - return .{ - .composed = try ComposedKernelExe.init(allocator, io, platform, llama_model, parameters, seqlen, attention_parameters, phase, progress), - }; - } - - pub fn deinit(self: KernelExe) void { - self.composed.deinit(); - } - - pub fn run(self: *const KernelExe, args: Args) !void { - try self.composed.run(args); - } - - pub fn initRunner( - self: *const KernelExe, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - model_buffers: *model.Buffers, - ) !Runner { - return .init(allocator, io, platform, &self.composed, model_buffers); - } -}; - -const ComposedKernelExe = struct { - embed_tokens: zml.Exe, - layer: zml.Exe, - lm_head: zml.Exe, - - const EmbedTokens = struct { - embed_tokens: zml.nn.TokenEmbedding, - - pub fn forward(self: EmbedTokens, tokens_: zml.Tensor) zml.Tensor { - const tokens = tokens_.withPartialTags(.{.s}); - return self.embed_tokens.forward(tokens) - .withPartialTags(.{.d}) - .withPartitioning(.{ .d = .replicated }); - } - }; - - fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - llama_model: model.Model, - parameters: CompilationOptions, - seqlen: usize, - attention_parameters: zml.attention.Parameters, - phase: Phase, - progress: *std.Progress.Node, - ) !ComposedKernelExe { - const embed_tokens = try ComposedKernelExe.compileEmbedTokens(allocator, io, platform, llama_model.model.embed_tokens, parameters, seqlen, phase, progress); - errdefer embed_tokens.deinit(); - - const layer = try ComposedKernelExe.compileLayer(allocator, io, platform, llama_model, parameters, seqlen, attention_parameters, phase, progress); - errdefer layer.deinit(); - - const lm_head = try ComposedKernelExe.compileLmHead(allocator, io, platform, llama_model, parameters, seqlen, phase, progress); - errdefer lm_head.deinit(); - - return .{ - .embed_tokens = embed_tokens, - .layer = layer, - .lm_head = lm_head, - }; - } - - fn deinit(self: ComposedKernelExe) void { - self.embed_tokens.deinit(); + pub fn deinit(self: *const KernelExe) void { + self.embed.deinit(); self.layer.deinit(); - self.lm_head.deinit(); - } - - fn run(self: *const ComposedKernelExe, args: Args) !void { - var hidden_buf: zml.Buffer = b: { - var exe_args = try self.embed_tokens.args(args.allocator); - defer exe_args.deinit(args.allocator); - - var results = try self.embed_tokens.results(args.allocator); - defer results.deinit(args.allocator); - - exe_args.bake(ComposedKernelExe.embedTokensBuffers(args.model_buffers)); - exe_args.set(.{args.tokens_buf}); - - self.embed_tokens.call(exe_args, &results); - - break :b results.get(zml.Buffer); - }; - defer hidden_buf.deinit(); - - for (args.model_buffers.model.layers, 0..) |layer_bufs, i| { - var exe_args = try self.layer.args(args.allocator); - defer exe_args.deinit(args.allocator); - - var results = try self.layer.results(args.allocator); - defer results.deinit(args.allocator); - - var kv_cache_index_buf: zml.Buffer = try .scalar(args.io, args.platform, i, .u32); - defer kv_cache_index_buf.deinit(); - - exe_args.bake(layer_bufs); - - self.runLayer(&exe_args, &results, args, &hidden_buf, &kv_cache_index_buf); - } - - { - var exe_args = try self.lm_head.args(args.allocator); - defer exe_args.deinit(args.allocator); - - var results = try self.lm_head.results(args.allocator); - defer results.deinit(args.allocator); - - exe_args.bake(ComposedKernelExe.lmHeadBuffers(args.model_buffers)); - - self.runLmHead(&exe_args, &results, args, &hidden_buf); - } - } - - fn runLayer( - self: *const ComposedKernelExe, - exe_args: *zml.exe.Exe.Arguments, - results: *zml.exe.Exe.Results, - args: Args, - hidden_buf: *zml.Buffer, - kv_cache_index_buf: *zml.Buffer, - ) void { - exe_args.set(.{ - hidden_buf, - args.token_index_buf, - args.kv_cache_buffers, - kv_cache_index_buf, - args.attention_metadata_buffers, - }); - - self.layer.call(exe_args.*, results); - - var new_hidden, var new_kv_cache = results.get(struct { - zml.Buffer, - zml.Bufferized(model.KvCache), - }); - - ComposedKernelExe.replaceBuffer(hidden_buf, &new_hidden); - ComposedKernelExe.replaceKvCacheBuffers(args.kv_cache_buffers, &new_kv_cache); - } - - fn runLmHead( - self: *const ComposedKernelExe, - exe_args: *zml.exe.Exe.Arguments, - results: *zml.exe.Exe.Results, - args: Args, - hidden_buf: *zml.Buffer, - ) void { - exe_args.set(.{ hidden_buf, args.tokens_buf, args.rng_buffers }); - self.lm_head.call(exe_args.*, results); - - var new_tokens, var new_rng = results.get(struct { - zml.Buffer, - zml.Bufferized(zml.Tensor.Rng), - }); - - ComposedKernelExe.replaceBuffer(args.tokens_buf, &new_tokens); - ComposedKernelExe.replaceBuffer(&args.rng_buffers._state, &new_rng._state); - } - - fn embedTokensBuffers(model_buffers: *const model.Buffers) zml.Bufferized(EmbedTokens) { - return .{ - .embed_tokens = model_buffers.model.embed_tokens, - }; - } - - fn lmHeadBuffers(model_buffers: *const model.Buffers) zml.Bufferized(model.LmHead) { - return .{ - .lm_head = model_buffers.lm_head, - .embed_tokens = model_buffers.model.embed_tokens, - .norm = model_buffers.model.norm, - }; - } - - fn replaceKvCacheBuffers(dst: *zml.Bufferized(model.KvCache), src: *zml.Bufferized(model.KvCache)) void { - ComposedKernelExe.replaceBuffer(&dst.k, &src.k); - ComposedKernelExe.replaceBuffer(&dst.v, &src.v); - } - - fn replaceBuffer(dst: *zml.Buffer, src: *zml.Buffer) void { - if (!ComposedKernelExe.sameBufferHandle(dst.*, src.*)) { - dst.deinit(); - } - - dst.* = src.*; - } - - fn sameBufferHandle(a: zml.Buffer, b: zml.Buffer) bool { - if (a._shards.len != b._shards.len) return false; - - for (a._shards.constSlice(), b._shards.constSlice()) |a_shard, b_shard| { - if (a_shard != b_shard) return false; - } - - return true; - } - - fn compileEmbedTokens( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - embed_tokens: zml.nn.TokenEmbedding, - parameters: CompilationOptions, - seqlen: usize, - phase: Phase, - progress: *std.Progress.Node, - ) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start(phase.startMessage("embed_tokens"), 1); - defer node.end(); - - const from: std.Io.Timestamp = .now(io, .awake); - defer phase.logCompileDone(log, "embed_tokens", io, from); - - const tokens: zml.Tensor = .init(.{ .s = seqlen }, .u32); - - return platform.compile(allocator, io, EmbedTokens{ .embed_tokens = embed_tokens }, .forward, .{tokens}, .{ - .shardings = ¶meters.shardings.all(), - .program_name = phase.programName("llama", "embed_tokens"), - }); - } - - fn compileLayer( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - llama_model: model.Model, - parameters: CompilationOptions, - seqlen: usize, - attention_parameters: zml.attention.Parameters, - phase: Phase, - progress: *std.Progress.Node, - ) !zml.Exe { - const Layer = struct { - layer: model.TransformerLayer, - - pub fn forward( - self: @This(), - hidden: zml.Tensor, - token_index: zml.Tensor, - kv_cache: model.KvCache, - kv_cache_index: zml.Tensor, - attention_metadata: zml.attention.Metadata, - attention_parameters_: zml.attention.Parameters, - ) struct { zml.Tensor, model.KvCache } { - const new_hidden, const new_kv_cache, _ = self.layer.forward( - hidden, - token_index, - kv_cache, - kv_cache_index, - attention_metadata, - attention_parameters_, - ); - - return .{ new_hidden, new_kv_cache }; - } - }; - - progress.increaseEstimatedTotalItems(1); - - var node = progress.start(phase.startMessage("transformer layer"), 1); - defer node.end(); - - const from: std.Io.Timestamp = .now(io, .awake); - defer phase.logCompileDone(log, "transformer layer", io, from); - - const hidden: zml.Tensor = .fromShape(zml.Shape.init( - .{ .s = seqlen, .d = llama_model.config.hidden_size }, - llama_model.model.embed_tokens.weight.dtype(), - ).withPartitioning(.{ .d = .replicated })); - - const kv_cache_index: zml.Tensor = .init(.{}, .u32); - - return platform.compile( - allocator, - io, - Layer{ .layer = llama_model.model.layers[0] }, - .forward, - .{ - hidden, - parameters.token_index, - parameters.kv_cache, - kv_cache_index, - parameters.attention_metadata, - attention_parameters, - }, - .{ - .shardings = ¶meters.shardings.all(), - .program_name = phase.programName("llama", "layer"), - }, - ); - } - - fn compileLmHead( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - llama_model: model.Model, - parameters: CompilationOptions, - seqlen: usize, - phase: Phase, - progress: *std.Progress.Node, - ) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - - var node = progress.start(phase.startMessage("lm_head"), 1); - defer node.end(); - - const from: std.Io.Timestamp = .now(io, .awake); - defer phase.logCompileDone(log, "lm_head", io, from); - - const hidden: zml.Tensor = .fromShape(zml.Shape.init( - .{ .s = seqlen, .d = llama_model.config.hidden_size }, - llama_model.model.embed_tokens.weight.dtype(), - ).withPartitioning(.{ .d = .replicated })); - - const tokens: zml.Tensor = .init(.{ .s = seqlen }, .u32); - - return platform.compile(allocator, io, model.LmHead.init(llama_model), .forward, .{ hidden, tokens, parameters.rng }, .{ - .shardings = ¶meters.shardings.all(), - .program_name = phase.programName("llama", "lm_head"), - }); + self.sample.deinit(); } }; + +pub const KernelRunner = struct { + embed: EmbedExe.Runner(.{.embedding}), + layers: []LayerExe.Runner(.{.layer}), + sample: SampleExe.Runner(.{.lm_head}), + + pub fn init(allocator: std.mem.Allocator, exe: *const KernelExe, buffers: *const model.Buffers) !KernelRunner { + var embed = try EmbedExe.Runner(.{.embedding}).init(&exe.embed, allocator, .{ + .embedding = .{ .embed_tokens = buffers.model.embed_tokens }, + }); + errdefer embed.deinit(allocator); + + const layers = try allocator.alloc(LayerExe.Runner(.{.layer}), buffers.model.layers.len); + errdefer allocator.free(layers); + var initialized_layers: usize = 0; + errdefer for (layers[0..initialized_layers]) |*layer| layer.deinit(allocator); + for (layers, buffers.model.layers) |*layer, layer_buffers| { + layer.* = try LayerExe.Runner(.{.layer}).init(&exe.layer, allocator, .{ .layer = layer_buffers }); + initialized_layers += 1; + } + + var sample = try SampleExe.Runner(.{.lm_head}).init(&exe.sample, allocator, .{ + .lm_head = .{ + .lm_head = buffers.lm_head, + .embed_tokens = buffers.model.embed_tokens, + .norm = buffers.model.norm, + }, + }); + errdefer sample.deinit(allocator); + + return .{ .embed = embed, .layers = layers, .sample = sample }; + } + + pub fn deinit(self: *KernelRunner, allocator: std.mem.Allocator) void { + self.embed.deinit(allocator); + for (self.layers) |*layer| layer.deinit(allocator); + allocator.free(self.layers); + self.sample.deinit(allocator); + } +}; + +pub fn run(runner: *KernelRunner, args: Args, kv_cache_index_buffers: []const zml.Buffer) void { + var hidden_buffer: zml.Buffer = undefined; + runner.embed.run(args.io, .{ + .inputs = .{ + .tokens = args.tokens_buf.*, + }, + .outputs = .{ .hidden = &hidden_buffer }, + }); + defer hidden_buffer.deinit(); + + for (runner.layers, kv_cache_index_buffers) |*layer, kv_cache_index_buffer| { + layer.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .token_index = args.token_index_buf.*, + .kv_cache = args.kv_cache_buffers.*, + .kv_cache_index = kv_cache_index_buffer, + .attention_metadata = args.attention_metadata_buffers.*, + }, + .outputs = .{ + .hidden = &hidden_buffer, + .kv_cache = args.kv_cache_buffers, + }, + }); + } + + runner.sample.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .tokens = args.tokens_buf.*, + .rng = args.rng_buffers.*, + }, + .outputs = .{ + .tokens = args.tokens_buf, + .rng = args.rng_buffers, + }, + }); +} + +fn compileKernel( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + llama_model: model.Model, + parameters: CompilationOptions, + seqlen: usize, + attention_parameters: zml.attention.Parameters, + phase: Phase, + progress: *std.Progress.Node, +) !KernelExe { + const embed = try compileEmbed(allocator, io, platform, llama_model.model.embed_tokens, parameters, seqlen, phase, progress); + errdefer embed.deinit(); + const layer = try compileLayer(allocator, io, platform, llama_model, parameters, seqlen, attention_parameters, phase, progress); + errdefer layer.deinit(); + const sample = try compileSample(allocator, io, platform, llama_model, parameters, seqlen, phase, progress); + errdefer sample.deinit(); + return .{ .embed = embed, .layer = layer, .sample = sample }; +} + +fn compileEmbed( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + embed_tokens: zml.nn.TokenEmbedding, + parameters: CompilationOptions, + seqlen: usize, + phase: Phase, + progress: *std.Progress.Node, +) !EmbedExe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("embed_tokens"), 1); + defer node.end(); + + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "embed_tokens", io, from); + + const tokens: zml.Tensor = .init(.{ .s = seqlen }, .u32); + + return EmbedExe.compile(allocator, io, platform, .{ + .shardings = ¶meters.shardings.all(), + .program_name = phase.programName("llama", "embed_tokens"), + }, .{.{ + .embedding = .{ .embed_tokens = embed_tokens }, + .tokens = tokens, + }}); +} + +fn compileLayer( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + llama_model: model.Model, + parameters: CompilationOptions, + seqlen: usize, + attention_parameters: zml.attention.Parameters, + phase: Phase, + progress: *std.Progress.Node, +) !LayerExe { + progress.increaseEstimatedTotalItems(1); + + var node = progress.start(phase.startMessage("transformer layer"), 1); + defer node.end(); + + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "transformer layer", io, from); + + const hidden: zml.Tensor = .fromShape(zml.Shape.init( + .{ .s = seqlen, .d = llama_model.config.hidden_size }, + llama_model.model.embed_tokens.weight.dtype(), + ).withPartitioning(.{ .d = .replicated })); + + const kv_cache_index: zml.Tensor = .init(.{}, .u32); + + return LayerExe.compile( + allocator, + io, + platform, + .{ + .shardings = ¶meters.shardings.all(), + .program_name = phase.programName("llama", "layer"), + }, + .{.{ + .layer = llama_model.model.layers[0], + .hidden = hidden, + .token_index = parameters.token_index, + .kv_cache = parameters.kv_cache, + .kv_cache_index = kv_cache_index, + .attention_metadata = parameters.attention_metadata, + .attention_parameters = attention_parameters, + }}, + ); +} + +fn compileSample( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + llama_model: model.Model, + parameters: CompilationOptions, + seqlen: usize, + phase: Phase, + progress: *std.Progress.Node, +) !SampleExe { + progress.increaseEstimatedTotalItems(1); + + var node = progress.start(phase.startMessage("lm_head"), 1); + defer node.end(); + + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "lm_head", io, from); + + const hidden: zml.Tensor = .fromShape(zml.Shape.init( + .{ .s = seqlen, .d = llama_model.config.hidden_size }, + llama_model.model.embed_tokens.weight.dtype(), + ).withPartitioning(.{ .d = .replicated })); + + const tokens: zml.Tensor = .init(.{ .s = seqlen }, .u32); + + return SampleExe.compile(allocator, io, platform, .{ + .shardings = ¶meters.shardings.all(), + .program_name = phase.programName("llama", "lm_head"), + }, .{.{ + .lm_head = model.LmHead.init(llama_model), + .hidden = hidden, + .tokens = tokens, + .rng = parameters.rng, + }}); +} diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index d7194c2b..e41de508 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -221,9 +221,14 @@ pub const Model = struct { attention_parameters, ); - const new_tokens, const new_rng = self.lmHead().forward(out, tokens, rng); + const sample = LmHead.forward(.{ + .lm_head = self.lmHead(), + .hidden = out, + .tokens = tokens, + .rng = rng, + }); - return .{ new_tokens, updated_kv_cache, new_rng }; + return .{ sample.tokens, updated_kv_cache, sample.rng }; } }; @@ -283,20 +288,44 @@ const Llama = struct { var kv_cache_index = zml.Tensor.scalar(0, .u32); for (self.layers) |layer| { - hidden, updated_kv_cache, kv_cache_index = layer.forward( - hidden, - token_index, - updated_kv_cache, - kv_cache_index, - attention_metadata, - attention_parameters, - ); + const result = TransformerLayer.forward(.{ + .layer = layer, + .hidden = hidden, + .token_index = token_index, + .kv_cache = updated_kv_cache, + .kv_cache_index = kv_cache_index, + .attention_metadata = attention_metadata, + .attention_parameters = attention_parameters, + }); + hidden = result.hidden; + updated_kv_cache = result.kv_cache; + kv_cache_index = kv_cache_index.add(zml.Tensor.scalar(@as(u32, 1), .u32)); } return .{ self.norm.forward(hidden), updated_kv_cache.reuseBuffer(kv_cache) }; } }; +pub const EmbedTokens = struct { + embed_tokens: zml.nn.TokenEmbedding, + + pub const Input = struct { + embedding: EmbedTokens, + tokens: zml.Tensor, + }; + + pub const Output = struct { + hidden: zml.Tensor, + }; + + pub fn forward(input: Input) Output { + const tokens = input.tokens.withPartialTags(.{.s}); + return .{ .hidden = input.embedding.embed_tokens.forward(tokens) + .withPartialTags(.{.d}) + .withPartitioning(.{ .d = .replicated }) }; + } +}; + pub const LmHead = struct { lm_head: ?zml.nn.Linear, embed_tokens: zml.nn.TokenEmbedding, @@ -312,9 +341,22 @@ pub const LmHead = struct { }; } - pub fn forward(self: LmHead, hidden_: zml.Tensor, tokens_: zml.Tensor, rng: zml.Tensor.Rng) struct { zml.Tensor, zml.Tensor.Rng } { - const tokens = tokens_.withPartialTags(.{.s}); - const hidden = self.norm.forward(hidden_.withPartialTags(.{ .s, .d })); + pub const Input = struct { + lm_head: LmHead, + hidden: zml.Tensor, + tokens: zml.Tensor, + rng: zml.Tensor.Rng, + }; + + pub const Output = struct { + tokens: zml.Tensor, + rng: zml.Tensor.Rng, + }; + + pub fn forward(input: Input) Output { + const self = input.lm_head; + const tokens = input.tokens.withPartialTags(.{.s}); + const hidden = self.norm.forward(input.hidden.withPartialTags(.{ .s, .d })); var logits = blk: { if (self.lm_head) |lm_head| { @@ -327,8 +369,8 @@ pub const LmHead = struct { if (logits.shape().hasTag(.voc) == null) logits = logits.rename(.{ .d = .voc }); - const next_tokens, const new_rng = zml.nn.sampleTokens(logits, self.gen_opts, rng); - return .{ next_tokens.convert(tokens.dtype()).reuseBuffer(tokens), new_rng }; + const next_tokens, const new_rng = zml.nn.sampleTokens(logits, self.gen_opts, input.rng); + return .{ .tokens = next_tokens.convert(tokens.dtype()).reuseBuffer(tokens), .rng = new_rng }; } }; @@ -354,15 +396,24 @@ pub const TransformerLayer = struct { Mlp.unloadBuffers(&self.mlp); } - pub fn forward( - self: TransformerLayer, - x0: zml.Tensor, + pub const Input = struct { + layer: TransformerLayer, + hidden: zml.Tensor, token_index: zml.Tensor, kv_cache: KvCache, kv_cache_index: zml.Tensor, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, - ) struct { zml.Tensor, KvCache, zml.Tensor } { + }; + + pub const Output = struct { + hidden: zml.Tensor, + kv_cache: KvCache, + }; + + pub fn forward(input: Input) Output { + const self = input.layer; + const x0 = input.hidden; // Self Attention //log.debug("TransformerLayer({f}) -> {f}", .{ x0, self.input_layernorm.forward(x0) }); stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {f}", .{x0}); @@ -372,13 +423,12 @@ pub const TransformerLayer = struct { const x0_normalized = self.input_layernorm.forward(x0_replicated); const delta0, const updated_kv_cache = self.self_attn.forward( x0_normalized, - token_index, - kv_cache, - kv_cache_index, - attention_metadata, - attention_parameters, + input.token_index, + input.kv_cache, + input.kv_cache_index, + input.attention_metadata, + input.attention_parameters, ); - const updated_kv_cache_index = kv_cache_index.add(zml.Tensor.scalar(@as(u32, 1), .u32)); // Fully Connected const x1 = x0_replicated.add(delta0).withPartitioning(.{ .d = .replicated }); @@ -389,7 +439,7 @@ pub const TransformerLayer = struct { .add(x1) .withPartitioning(.{ .d = .replicated }); - return .{ x2.reuseBuffer(x0), updated_kv_cache, updated_kv_cache_index.reuseBuffer(kv_cache_index) }; + return .{ .hidden = x2.reuseBuffer(x0), .kv_cache = updated_kv_cache }; } }; diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig index 9cc69c01..d86a8728 100644 --- a/examples/llm/models/llama/session.zig +++ b/examples/llm/models/llama/session.zig @@ -9,11 +9,12 @@ pub const Session = struct { allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, - model_buffers: *model.Buffers, - compiled_model: *const inference.CompiledModel, - decode_runner: inference.KernelExe.Runner, + compiled_model: *inference.CompiledModel, + prefill: inference.KernelRunner, + decode: inference.KernelRunner, kv_cache_buffers: zml.Bufferized(model.KvCache), token_index_buffers: []zml.Buffer, + kv_cache_index_buffers: []zml.Buffer, rng_buffers: zml.Bufferized(zml.Tensor.Rng), tokenizer: zml.tokenizer.Tokenizer, config: *const model.Config, @@ -26,7 +27,7 @@ pub const Session = struct { io: std.Io, platform: *const zml.Platform, tokenizer: zml.tokenizer.Tokenizer, - compiled_model: *const inference.CompiledModel, + compiled_model: *inference.CompiledModel, model_buffers: *model.Buffers, ) !Session { const shardings = &compiled_model.params.shardings; @@ -52,18 +53,29 @@ pub const Session = struct { var rng_buffers = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); - var decode_runner = try compiled_model.decode.initRunner(allocator, io, platform, model_buffers); - errdefer decode_runner.deinit(allocator); + const kv_cache_index_buffers = try allocator.alloc(zml.Buffer, model_buffers.model.layers.len); + errdefer allocator.free(kv_cache_index_buffers); + var initialized_kv_cache_index_buffers: usize = 0; + errdefer for (kv_cache_index_buffers[0..initialized_kv_cache_index_buffers]) |*buffer| buffer.deinit(); + for (kv_cache_index_buffers, 0..) |*buffer, i| { + buffer.* = try .scalar(io, platform, i, .u32); + initialized_kv_cache_index_buffers = i + 1; + } + + var prefill = try inference.KernelRunner.init(allocator, &compiled_model.prefill, model_buffers); + errdefer prefill.deinit(allocator); + const decode = try inference.KernelRunner.init(allocator, &compiled_model.decode, model_buffers); return .{ .allocator = allocator, .io = io, .platform = platform, - .model_buffers = model_buffers, .compiled_model = compiled_model, - .decode_runner = decode_runner, + .prefill = prefill, + .decode = decode, .kv_cache_buffers = kv_cache_buffers, .token_index_buffers = token_index_buffers, + .kv_cache_index_buffers = kv_cache_index_buffers, .rng_buffers = rng_buffers, .tokenizer = tokenizer, .config = &compiled_model.loaded_model.parsed_config.value, @@ -73,12 +85,15 @@ pub const Session = struct { } pub fn deinit(self: *Session) void { - self.decode_runner.deinit(self.allocator); + self.prefill.deinit(self.allocator); + self.decode.deinit(self.allocator); model.KvCache.deinitBuffer(&self.kv_cache_buffers); for (self.token_index_buffers) |*token_index_buffer| { token_index_buffer.deinit(); } self.allocator.free(self.token_index_buffers); + for (self.kv_cache_index_buffers) |*buffer| buffer.deinit(); + self.allocator.free(self.kv_cache_index_buffers); zml.Tensor.Rng.deinitBuffer(&self.rng_buffers); } @@ -151,17 +166,14 @@ pub const Session = struct { }; defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); - try self.compiled_model.prefill.run(.{ - .allocator = self.allocator, + inference.run(&self.prefill, .{ .io = self.io, - .platform = self.platform, - .model_buffers = self.model_buffers, .tokens_buf = &prefill_tokens_buffer, .token_index_buf = &self.token_index_buffers[0], .kv_cache_buffers = &self.kv_cache_buffers, .rng_buffers = &self.rng_buffers, .attention_metadata_buffers = &attention_metadata_buffers, - }); + }, self.kv_cache_index_buffers); try prefill_tokens_buffer.toSlice(self.io, prefill_tokens_slice); self.last_generated_token = prefill_tokens_slice.items(u32)[all_tokens.len - 1]; @@ -198,17 +210,14 @@ pub const Session = struct { try all_tokens.append(self.allocator, last_token_id); if (all_tokens.items.len >= self.seqlen) break :generation; - try self.decode_runner.run(.{ - .allocator = self.allocator, + inference.run(&self.decode, .{ .io = self.io, - .platform = self.platform, - .model_buffers = self.model_buffers, .tokens_buf = ¤t_token_buffer, .token_index_buf = &self.token_index_buffers[all_tokens.items.len], .kv_cache_buffers = &self.kv_cache_buffers, .rng_buffers = &self.rng_buffers, .attention_metadata_buffers = &attention_metadata_buffers, - }); + }, self.kv_cache_index_buffers); last_token_id = try current_token_buffer.getValue(u32, self.io); } diff --git a/examples/llm/models/qwen3_5/inference.zig b/examples/llm/models/qwen3_5/inference.zig index 1ad962fe..4a2a9331 100644 --- a/examples/llm/models/qwen3_5/inference.zig +++ b/examples/llm/models/qwen3_5/inference.zig @@ -34,10 +34,7 @@ pub const CompilationParameters = struct { pub const CompilationOptions = CompilationParameters; pub const Args = struct { - allocator: std.mem.Allocator, io: std.Io, - platform: *const zml.Platform, - model_buffers: *model.Buffers, tokens_buf: *zml.Buffer, token_index_buf: *zml.Buffer, kv_cache_buffers: *zml.Bufferized(model.KvCache), @@ -59,28 +56,13 @@ pub const CompiledModel = struct { parameters: CompilationParameters, progress: *std.Progress.Node, ) !CompiledModel { + const prefill = try compileKernel(allocator, io, platform, qwen_model, parameters, @intCast(parameters.prefill_tokens.dim(.s)), .prefill, progress); + errdefer prefill.deinit(); + const decode = try compileKernel(allocator, io, platform, qwen_model, parameters, @intCast(parameters.decode_tokens.dim(.s)), .decode, progress); return .{ .loaded_model = loaded_model, - .prefill = try .init( - allocator, - io, - platform, - qwen_model, - parameters, - @intCast(parameters.prefill_tokens.dim(.s)), - .prefill, - progress, - ), - .decode = try .init( - allocator, - io, - platform, - qwen_model, - parameters, - @intCast(parameters.decode_tokens.dim(.s)), - .decode, - progress, - ), + .prefill = prefill, + .decode = decode, .params = parameters, }; } @@ -93,628 +75,243 @@ pub const CompiledModel = struct { pub const Inference = CompiledModel; +pub const EmbedExe = zml.TypedExe(model.EmbedTokens.forward); +pub const FullAttentionExe = zml.TypedExe(model.TransformerLayer.forwardSelfAttn); +pub const LinearAttentionExe = zml.TypedExe(model.TransformerLayer.forwardLinearAttn); +pub const SampleExe = zml.TypedExe(model.Sampler.sampleTokens); + pub const KernelExe = struct { - composed: ComposedKernelExe, + embed: EmbedExe, + full_attention: FullAttentionExe, + linear_attention: LinearAttentionExe, + sample: SampleExe, - pub const Runner = struct { - exe: *const ComposedKernelExe, - embed_args: zml.exe.Exe.Arguments, - embed_results: zml.exe.Exe.Results, - layers: Layers, - sampler_args: zml.exe.Exe.Arguments, - sampler_results: zml.exe.Exe.Results, - - const Layers = struct { - args: []zml.exe.Exe.Arguments, - results: []zml.exe.Exe.Results, - layer_indices: []zml.Buffer, - layer_types: []const model.LayerType, - - fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - exe: *const ComposedKernelExe, - model_buffers: *model.Buffers, - ) !Layers { - const args = try allocator.alloc(zml.exe.Exe.Arguments, model_buffers.text_model.layers.len); - errdefer allocator.free(args); - - const results = try allocator.alloc(zml.exe.Exe.Results, model_buffers.text_model.layers.len); - errdefer allocator.free(results); - - const layer_indices = try allocator.alloc(zml.Buffer, model_buffers.text_model.layers.len); - errdefer allocator.free(layer_indices); - - var initialized_args: usize = 0; - errdefer { - for (args[0..initialized_args]) |*exe_args| { - exe_args.deinit(allocator); - } - } - - var initialized_results: usize = 0; - errdefer { - for (results[0..initialized_results]) |*exe_results| { - exe_results.deinit(allocator); - } - } - - var initialized_layer_indices: usize = 0; - errdefer { - for (layer_indices[0..initialized_layer_indices]) |*layer_index| { - layer_index.deinit(); - } - } - - var self_attn_layer_index: usize = 0; - var linear_attn_layer_index: usize = 0; - for (model_buffers.text_model.layers, exe.layer_types, 0..) |layer_bufs, layer_type, i| { - args[i] = try switch (layer_type) { - .full_attention => (exe.full_attention_layer orelse unreachable).args(allocator), - .linear_attention => (exe.linear_attention_layer orelse unreachable).args(allocator), - }; - initialized_args = i + 1; - args[i].bake(layer_bufs); - - results[i] = try switch (layer_type) { - .full_attention => (exe.full_attention_layer orelse unreachable).results(allocator), - .linear_attention => (exe.linear_attention_layer orelse unreachable).results(allocator), - }; - initialized_results = i + 1; - - layer_indices[i] = try switch (layer_type) { - .full_attention => b: { - defer self_attn_layer_index += 1; - break :b zml.Buffer.scalar(io, platform, @as(u32, @intCast(self_attn_layer_index)), .u32); - }, - .linear_attention => b: { - defer linear_attn_layer_index += 1; - break :b zml.Buffer.scalar(io, platform, @as(u32, @intCast(linear_attn_layer_index)), .u32); - }, - }; - initialized_layer_indices = i + 1; - } - - return .{ - .args = args, - .results = results, - .layer_indices = layer_indices, - .layer_types = exe.layer_types, - }; - } - - fn deinit(self: *Layers, allocator: std.mem.Allocator) void { - for (self.args) |*exe_args| { - exe_args.deinit(allocator); - } - allocator.free(self.args); - - for (self.results) |*exe_results| { - exe_results.deinit(allocator); - } - allocator.free(self.results); - - for (self.layer_indices) |*layer_index| { - layer_index.deinit(); - } - allocator.free(self.layer_indices); - } - }; - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - exe: *const ComposedKernelExe, - model_buffers: *model.Buffers, - ) !Runner { - var embed_args = try exe.embed_tokens.args(allocator); - errdefer embed_args.deinit(allocator); - embed_args.bake(ComposedKernelExe.embedTokensBuffers(model_buffers)); - - var embed_results = try exe.embed_tokens.results(allocator); - errdefer embed_results.deinit(allocator); - - var layers = try Layers.init(allocator, io, platform, exe, model_buffers); - errdefer layers.deinit(allocator); - - var sampler_args = try exe.sampler.args(allocator); - errdefer sampler_args.deinit(allocator); - sampler_args.bake(ComposedKernelExe.samplerBuffers(model_buffers)); - - var sampler_results = try exe.sampler.results(allocator); - errdefer sampler_results.deinit(allocator); - - return .{ - .exe = exe, - .embed_args = embed_args, - .embed_results = embed_results, - .layers = layers, - .sampler_args = sampler_args, - .sampler_results = sampler_results, - }; - } - - pub fn deinit(self: *Runner, allocator: std.mem.Allocator) void { - self.embed_args.deinit(allocator); - self.embed_results.deinit(allocator); - self.layers.deinit(allocator); - self.sampler_args.deinit(allocator); - self.sampler_results.deinit(allocator); - } - - pub fn run(self: *Runner, args: Args) !void { - var hidden_buf: zml.Buffer = b: { - self.embed_args.set(.{args.tokens_buf}); - self.exe.embed_tokens.call(self.embed_args, &self.embed_results); - - break :b self.embed_results.get(zml.Buffer); - }; - defer hidden_buf.deinit(); - - for ( - self.layers.args, - self.layers.results, - self.layers.layer_indices, - self.layers.layer_types, - ) |*exe_args, *results, *layer_index_buf, layer_type| { - self.exe.runLayer(exe_args, results, layer_type, args, &hidden_buf, layer_index_buf); - } - - self.exe.runSampler(&self.sampler_args, &self.sampler_results, args, &hidden_buf); - } - }; - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - qwen_model: model.Model, - parameters: CompilationOptions, - seqlen: usize, - phase: Phase, - progress: *std.Progress.Node, - ) !KernelExe { - return .{ - .composed = try .init(allocator, io, platform, qwen_model, parameters, seqlen, phase, progress), - }; - } - - pub fn deinit(self: KernelExe) void { - self.composed.deinit(); - } - - pub fn run(self: *const KernelExe, args: Args) !void { - try self.composed.run(args); - } - - pub fn initRunner( - self: *const KernelExe, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - model_buffers: *model.Buffers, - ) !Runner { - return .init(allocator, io, platform, &self.composed, model_buffers); + pub fn deinit(self: *const KernelExe) void { + self.embed.deinit(); + self.full_attention.deinit(); + self.linear_attention.deinit(); + self.sample.deinit(); } }; -const ComposedKernelExe = struct { - embed_tokens: zml.Exe, - full_attention_layer: ?zml.Exe, - linear_attention_layer: ?zml.Exe, - sampler: zml.Exe, - layer_types: []const model.LayerType, - phase: Phase, +pub const LayerRunner = union(enum) { + full_attention: FullAttentionExe.Runner(.{.layer}), + linear_attention: LinearAttentionExe.Runner(.{.layer}), - const EmbedTokens = struct { - embed_tokens: zml.nn.TokenEmbedding, - - pub fn forward(self: EmbedTokens, tokens_: zml.Tensor) zml.Tensor { - const tokens = tokens_.withPartialTags(.{.s}); - return self.embed_tokens.forward(tokens) - .withPartialTags(.{.d}) - .withPartitioning(.{ .d = .replicated }); - } - }; - - fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - qwen_model: model.Model, - parameters: CompilationOptions, - seqlen: usize, - phase: Phase, - progress: *std.Progress.Node, - ) !ComposedKernelExe { - const embed_tokens = try ComposedKernelExe.compileEmbedTokens(allocator, io, platform, qwen_model.text_model.embed_tokens, parameters, seqlen, phase, progress); - errdefer embed_tokens.deinit(); - - const full_attention_layer: ?zml.Exe = b: { - const index = ComposedKernelExe.findFirstLayerIndex(qwen_model.config.text_config.layer_types, .full_attention) orelse break :b null; - break :b try ComposedKernelExe.compileFullAttentionLayer(allocator, io, platform, qwen_model, parameters, seqlen, index, phase, progress); - }; - errdefer if (full_attention_layer) |exe| exe.deinit(); - - const linear_attention_layer: ?zml.Exe = b: { - const index = ComposedKernelExe.findFirstLayerIndex(qwen_model.config.text_config.layer_types, .linear_attention) orelse break :b null; - break :b try ComposedKernelExe.compileLinearAttentionLayer(allocator, io, platform, qwen_model, parameters, seqlen, index, phase, progress); - }; - errdefer if (linear_attention_layer) |exe| exe.deinit(); - - const sampler = try ComposedKernelExe.compileSampler(allocator, io, platform, qwen_model, parameters, seqlen, phase, progress); - errdefer sampler.deinit(); - - return .{ - .embed_tokens = embed_tokens, - .full_attention_layer = full_attention_layer, - .linear_attention_layer = linear_attention_layer, - .sampler = sampler, - .layer_types = qwen_model.config.text_config.layer_types, - .phase = phase, - }; - } - - fn deinit(self: ComposedKernelExe) void { - self.embed_tokens.deinit(); - if (self.full_attention_layer) |exe| exe.deinit(); - if (self.linear_attention_layer) |exe| exe.deinit(); - self.sampler.deinit(); - } - - fn run(self: *const ComposedKernelExe, args: Args) !void { - var hidden_buf: zml.Buffer = b: { - var exe_args = try self.embed_tokens.args(args.allocator); - defer exe_args.deinit(args.allocator); - - var results = try self.embed_tokens.results(args.allocator); - defer results.deinit(args.allocator); - - exe_args.bake(ComposedKernelExe.embedTokensBuffers(args.model_buffers)); - exe_args.set(.{args.tokens_buf}); - - self.embed_tokens.call(exe_args, &results); - - break :b results.get(zml.Buffer); - }; - defer hidden_buf.deinit(); - - var self_attn_layer_index: usize = 0; - var linear_attn_layer_index: usize = 0; - for (args.model_buffers.text_model.layers, self.layer_types) |layer_bufs, layer_type| { - var exe_args = try switch (layer_type) { - .full_attention => (self.full_attention_layer orelse unreachable).args(args.allocator), - .linear_attention => (self.linear_attention_layer orelse unreachable).args(args.allocator), - }; - defer exe_args.deinit(args.allocator); - exe_args.bake(layer_bufs); - - var results = try switch (layer_type) { - .full_attention => (self.full_attention_layer orelse unreachable).results(args.allocator), - .linear_attention => (self.linear_attention_layer orelse unreachable).results(args.allocator), - }; - defer results.deinit(args.allocator); - - var layer_index_buf: zml.Buffer = try switch (layer_type) { - .full_attention => b: { - defer self_attn_layer_index += 1; - break :b zml.Buffer.scalar(args.io, args.platform, @as(u32, @intCast(self_attn_layer_index)), .u32); - }, - .linear_attention => b: { - defer linear_attn_layer_index += 1; - break :b zml.Buffer.scalar(args.io, args.platform, @as(u32, @intCast(linear_attn_layer_index)), .u32); - }, - }; - defer layer_index_buf.deinit(); - - self.runLayer(&exe_args, &results, layer_type, args, &hidden_buf, &layer_index_buf); - } - - { - var exe_args = try self.sampler.args(args.allocator); - defer exe_args.deinit(args.allocator); - - var results = try self.sampler.results(args.allocator); - defer results.deinit(args.allocator); - - exe_args.bake(ComposedKernelExe.samplerBuffers(args.model_buffers)); - - self.runSampler(&exe_args, &results, args, &hidden_buf); + pub fn deinit(self: *LayerRunner, allocator: std.mem.Allocator) void { + switch (self.*) { + inline else => |*runner| runner.deinit(allocator), } } +}; - fn runLayer( - self: *const ComposedKernelExe, - exe_args: *zml.exe.Exe.Arguments, - results: *zml.exe.Exe.Results, - layer_type: model.LayerType, - args: Args, - hidden_buf: *zml.Buffer, - layer_index_buf: *zml.Buffer, - ) void { - switch (layer_type) { - .full_attention => { - const layer_cache: zml.Bufferized(model.KvCache.SelfAttnCache) = .{ +pub const KernelRunner = struct { + embed: EmbedExe.Runner(.{.embedding}), + layers: []LayerRunner, + sample: SampleExe.Runner(.{.sampler}), + + pub fn init(allocator: std.mem.Allocator, exe: *const KernelExe, buffers: *const model.Buffers) !KernelRunner { + var embed = try EmbedExe.Runner(.{.embedding}).init(&exe.embed, allocator, .{ + .embedding = .{ .embed_tokens = buffers.text_model.embed_tokens }, + }); + errdefer embed.deinit(allocator); + + const layers = try allocator.alloc(LayerRunner, buffers.text_model.layers.len); + errdefer allocator.free(layers); + var initialized_layers: usize = 0; + errdefer for (layers[0..initialized_layers]) |*layer| layer.deinit(allocator); + for (layers, buffers.text_model.layers) |*layer, layer_buffers| { + layer.* = switch (layer_buffers.attn) { + .full_attention => .{ .full_attention = try FullAttentionExe.Runner(.{.layer}).init( + &exe.full_attention, + allocator, + .{ .layer = layer_buffers }, + ) }, + .linear_attention => .{ .linear_attention = try LinearAttentionExe.Runner(.{.layer}).init( + &exe.linear_attention, + allocator, + .{ .layer = layer_buffers }, + ) }, + }; + initialized_layers += 1; + } + + var sample = try SampleExe.Runner(.{.sampler}).init(&exe.sample, allocator, .{ + .sampler = .{ + .norm = buffers.text_model.norm, + .lm_head = buffers.lm_head, + }, + }); + errdefer sample.deinit(allocator); + + return .{ .embed = embed, .layers = layers, .sample = sample }; + } + + pub fn deinit(self: *KernelRunner, allocator: std.mem.Allocator) void { + self.embed.deinit(allocator); + for (self.layers) |*layer| layer.deinit(allocator); + allocator.free(self.layers); + self.sample.deinit(allocator); + } +}; + +pub fn run(runner: *KernelRunner, args: Args, layer_index_buffers: []const zml.Buffer) void { + var hidden_buffer: zml.Buffer = undefined; + runner.embed.run(args.io, .{ + .inputs = .{ + .tokens = args.tokens_buf.*, + }, + .outputs = .{ .hidden = &hidden_buffer }, + }); + defer hidden_buffer.deinit(); + + for (runner.layers, layer_index_buffers) |*layer_runner, layer_index_buffer| { + switch (layer_runner.*) { + .full_attention => |*layer| { + var layer_cache: zml.Bufferized(model.KvCache.SelfAttnCache) = .{ .k = args.kv_cache_buffers.self_attn.k, .v = args.kv_cache_buffers.self_attn.v, - .layer_index = layer_index_buf.*, + .layer_index = layer_index_buffer, }; - exe_args.set(.{ hidden_buf, args.token_index_buf, layer_cache }); - (self.full_attention_layer orelse unreachable).call(exe_args.*, results); - - var new_hidden, var new_cache = results.get(struct { - zml.Buffer, - zml.Bufferized(model.KvCache.SelfAttnCache), + layer.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .token_index = args.token_index_buf.*, + .cache = layer_cache, + }, + .outputs = .{ .hidden = &hidden_buffer, .cache = &layer_cache }, }); - ComposedKernelExe.replaceBuffer(hidden_buf, &new_hidden); - ComposedKernelExe.replaceBuffer(&args.kv_cache_buffers.self_attn.k, &new_cache.k); - ComposedKernelExe.replaceBuffer(&args.kv_cache_buffers.self_attn.v, &new_cache.v); - ComposedKernelExe.releaseBuffer(layer_index_buf.*, &new_cache.layer_index); + args.kv_cache_buffers.self_attn.k = layer_cache.k; + args.kv_cache_buffers.self_attn.v = layer_cache.v; }, - .linear_attention => { - const layer_cache: zml.Bufferized(model.KvCache.GatedDeltaNetCache) = .{ + .linear_attention => |*layer| { + var layer_cache: zml.Bufferized(model.KvCache.GatedDeltaNetCache) = .{ .conv_state = args.kv_cache_buffers.gated_delta_net.conv_state, .recurrent_state = args.kv_cache_buffers.gated_delta_net.recurrent_state, - .layer_index = layer_index_buf.*, + .layer_index = layer_index_buffer, }; - exe_args.set(.{ hidden_buf, args.token_index_buf, layer_cache }); - (self.linear_attention_layer orelse unreachable).call(exe_args.*, results); - - var new_hidden, var new_cache = results.get(struct { - zml.Buffer, - zml.Bufferized(model.KvCache.GatedDeltaNetCache), + layer.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .token_index = args.token_index_buf.*, + .cache = layer_cache, + }, + .outputs = .{ .hidden = &hidden_buffer, .cache = &layer_cache }, }); - ComposedKernelExe.replaceBuffer(hidden_buf, &new_hidden); - ComposedKernelExe.replaceBuffer(&args.kv_cache_buffers.gated_delta_net.conv_state, &new_cache.conv_state); - ComposedKernelExe.replaceBuffer(&args.kv_cache_buffers.gated_delta_net.recurrent_state, &new_cache.recurrent_state); - ComposedKernelExe.releaseBuffer(layer_index_buf.*, &new_cache.layer_index); + args.kv_cache_buffers.gated_delta_net.conv_state = layer_cache.conv_state; + args.kv_cache_buffers.gated_delta_net.recurrent_state = layer_cache.recurrent_state; }, } } - fn runSampler( - self: *const ComposedKernelExe, - exe_args: *zml.exe.Exe.Arguments, - results: *zml.exe.Exe.Results, - args: Args, - hidden_buf: *zml.Buffer, - ) void { - switch (self.phase) { - .prefill => self.runPrefillSampler(exe_args, results, args, hidden_buf), - .decode => self.runDecodeSampler(exe_args, results, args, hidden_buf), - } - } + runner.sample.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .rng = args.rng_buffers.*, + .token_index = args.token_index_buf.*, + }, + .outputs = .{ + .tokens = args.tokens_buf, + .rng = args.rng_buffers, + .token_index = args.token_index_buf, + }, + }); +} - fn runPrefillSampler( - self: *const ComposedKernelExe, - exe_args: *zml.exe.Exe.Arguments, - results: *zml.exe.Exe.Results, - args: Args, - hidden_buf: *zml.Buffer, - ) void { - exe_args.set(.{ hidden_buf, args.rng_buffers }); - self.sampler.call(exe_args.*, results); +fn compileKernel( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + qwen_model: model.Model, + parameters: CompilationOptions, + seqlen: usize, + phase: Phase, + progress: *std.Progress.Node, +) !KernelExe { + const full_index = findFirstLayerIndex(qwen_model.config.text_config.layer_types, .full_attention) orelse return error.MissingFullAttentionLayer; + const linear_index = findFirstLayerIndex(qwen_model.config.text_config.layer_types, .linear_attention) orelse return error.MissingLinearAttentionLayer; - var new_tokens, var new_rng = results.get(struct { - zml.Buffer, - zml.Bufferized(zml.Tensor.Rng), - }); + const embed = try compileEmbed(allocator, io, platform, qwen_model, parameters, seqlen, phase, progress); + errdefer embed.deinit(); + const full_attention = try compileFullAttention(allocator, io, platform, qwen_model, parameters, seqlen, full_index, phase, progress); + errdefer full_attention.deinit(); + const linear_attention = try compileLinearAttention(allocator, io, platform, qwen_model, parameters, seqlen, linear_index, phase, progress); + errdefer linear_attention.deinit(); + const sample = try compileSample(allocator, io, platform, qwen_model, parameters, seqlen, phase, progress); + errdefer sample.deinit(); + return .{ .embed = embed, .full_attention = full_attention, .linear_attention = linear_attention, .sample = sample }; +} - ComposedKernelExe.replaceBuffer(args.tokens_buf, &new_tokens); - ComposedKernelExe.replaceBuffer(&args.rng_buffers._state, &new_rng._state); - } +fn compileEmbed(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationOptions, seqlen: usize, phase: Phase, progress: *std.Progress.Node) !EmbedExe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("embed tokens"), 1); + defer node.end(); + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "embed tokens", io, from); + return EmbedExe.compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("qwen3_5", "embed_tokens") }, .{.{ + .embedding = .{ .embed_tokens = mdl.text_model.embed_tokens }, + .tokens = zml.Tensor.init(.{ .b = 1, .s = seqlen }, .u32), + }}); +} - fn runDecodeSampler( - self: *const ComposedKernelExe, - exe_args: *zml.exe.Exe.Arguments, - results: *zml.exe.Exe.Results, - args: Args, - hidden_buf: *zml.Buffer, - ) void { - exe_args.set(.{ hidden_buf, args.rng_buffers, args.token_index_buf }); - self.sampler.call(exe_args.*, results); - - var new_tokens, var new_rng, var new_token_index = results.get(struct { - zml.Buffer, - zml.Bufferized(zml.Tensor.Rng), - zml.Buffer, - }); - - ComposedKernelExe.replaceBuffer(args.tokens_buf, &new_tokens); - ComposedKernelExe.replaceBuffer(&args.rng_buffers._state, &new_rng._state); - ComposedKernelExe.replaceBuffer(args.token_index_buf, &new_token_index); - } - - fn embedTokensBuffers(model_buffers: *const model.Buffers) zml.Bufferized(EmbedTokens) { - return .{ - .embed_tokens = model_buffers.text_model.embed_tokens, - }; - } - - fn samplerBuffers(model_buffers: *const model.Buffers) zml.Bufferized(model.Sampler) { - return .{ - .norm = model_buffers.text_model.norm, - .lm_head = model_buffers.lm_head, - }; - } - - fn replaceBuffer(dst: *zml.Buffer, src: *zml.Buffer) void { - if (!ComposedKernelExe.sameBufferHandle(dst.*, src.*)) { - dst.deinit(); - } - - dst.* = src.*; - } - - fn releaseBuffer(expected: zml.Buffer, actual: *zml.Buffer) void { - if (!ComposedKernelExe.sameBufferHandle(expected, actual.*)) { - actual.deinit(); - } - } - - fn sameBufferHandle(a: zml.Buffer, b: zml.Buffer) bool { - if (a._shards.len != b._shards.len) return false; - - for (a._shards.constSlice(), b._shards.constSlice()) |a_shard, b_shard| { - if (a_shard != b_shard) return false; - } - - return true; - } - - fn compileEmbedTokens( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - embed_tokens: zml.nn.TokenEmbedding, - parameters: CompilationOptions, - seqlen: usize, - phase: Phase, - progress: *std.Progress.Node, - ) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start(phase.startMessage("embed tokens"), 1); - defer node.end(); - - const from: std.Io.Timestamp = .now(io, .awake); - defer phase.logCompileDone(log, "embed tokens", io, from); - - const tokens: zml.Tensor = .init(.{ .b = 1, .s = seqlen }, .u32); - - return platform.compile(allocator, io, EmbedTokens{ .embed_tokens = embed_tokens }, .forward, .{tokens}, .{ - .shardings = ¶meters.shardings.all(), - .program_name = phase.programName("qwen3_5", "embed_tokens"), - }); - } - - fn compileFullAttentionLayer( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - qwen_model: model.Model, - parameters: CompilationOptions, - seqlen: usize, - layer_index: usize, - phase: Phase, - progress: *std.Progress.Node, - ) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start(phase.startMessage("full attention layer"), 1); - defer node.end(); - - const from: std.Io.Timestamp = .now(io, .awake); - defer phase.logCompileDone(log, "full attention layer", io, from); - - const hidden_tensor = ComposedKernelExe.hidden(qwen_model, seqlen); - const self_attn_cache: model.KvCache.SelfAttnCache = .{ +fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationOptions, seqlen: usize, layer_index: usize, phase: Phase, progress: *std.Progress.Node) !FullAttentionExe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("full attention layer"), 1); + defer node.end(); + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "full attention layer", io, from); + return FullAttentionExe.compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("qwen3_5", "full_attention_layer") }, .{.{ + .layer = mdl.text_model.layers[layer_index], + .hidden = hiddenTensor(mdl, seqlen), + .token_index = parameters.token_index, + .cache = .{ .k = parameters.kv_cache.self_attn.k, .v = parameters.kv_cache.self_attn.v, .layer_index = zml.Tensor.init(.{}, .u32), - }; + }, + }}); +} - return platform.compile( - allocator, - io, - qwen_model.text_model.layers[layer_index], - .forwardSelfAttn, - .{ hidden_tensor, parameters.token_index, self_attn_cache }, - .{ - .shardings = ¶meters.shardings.all(), - .program_name = phase.programName("qwen3_5", "full_attention_layer"), - }, - ); - } - - fn compileLinearAttentionLayer( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - qwen_model: model.Model, - parameters: CompilationOptions, - seqlen: usize, - layer_index: usize, - phase: Phase, - progress: *std.Progress.Node, - ) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start(phase.startMessage("linear attention layer"), 1); - defer node.end(); - - const from: std.Io.Timestamp = .now(io, .awake); - defer phase.logCompileDone(log, "linear attention layer", io, from); - - const hidden_tensor = ComposedKernelExe.hidden(qwen_model, seqlen); - const linear_attn_cache: model.KvCache.GatedDeltaNetCache = .{ +fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationOptions, seqlen: usize, layer_index: usize, phase: Phase, progress: *std.Progress.Node) !LinearAttentionExe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("linear attention layer"), 1); + defer node.end(); + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "linear attention layer", io, from); + return LinearAttentionExe.compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("qwen3_5", "linear_attention_layer") }, .{.{ + .layer = mdl.text_model.layers[layer_index], + .hidden = hiddenTensor(mdl, seqlen), + .token_index = parameters.token_index, + .cache = .{ .conv_state = parameters.kv_cache.gated_delta_net.conv_state, .recurrent_state = parameters.kv_cache.gated_delta_net.recurrent_state, .layer_index = zml.Tensor.init(.{}, .u32), - }; + }, + }}); +} - return platform.compile( - allocator, - io, - qwen_model.text_model.layers[layer_index], - .forwardLinearAttn, - .{ hidden_tensor, parameters.token_index, linear_attn_cache }, - .{ - .shardings = ¶meters.shardings.all(), - .program_name = phase.programName("qwen3_5", "linear_attention_layer"), - }, - ); - } +fn compileSample(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationOptions, seqlen: usize, phase: Phase, progress: *std.Progress.Node) !SampleExe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("sampler"), 1); + defer node.end(); + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "sampler", io, from); + return SampleExe.compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("qwen3_5", "sampler") }, .{.{ + .sampler = mdl.sampler(), + .hidden = hiddenTensor(mdl, seqlen), + .rng = parameters.rng, + .token_index = parameters.token_index, + }}); +} - fn compileSampler( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - qwen_model: model.Model, - parameters: CompilationOptions, - seqlen: usize, - phase: Phase, - progress: *std.Progress.Node, - ) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start(phase.startMessage("sampler"), 1); - defer node.end(); +fn hiddenTensor(mdl: model.Model, seqlen: usize) zml.Tensor { + return .fromShape(zml.Shape.init( + .{ .b = 1, .s = seqlen, .d = mdl.config.text_config.hidden_size }, + mdl.text_model.embed_tokens.weight.dtype(), + ).withPartitioning(.{ .b = .replicated, .s = .replicated, .d = .replicated })); +} - const from: std.Io.Timestamp = .now(io, .awake); - defer phase.logCompileDone(log, "sampler", io, from); - - const token_index: ?zml.Tensor = switch (phase) { - .prefill => null, - .decode => parameters.token_index, - }; - - return platform.compile( - allocator, - io, - qwen_model.sampler(), - .sampleTokens, - .{ ComposedKernelExe.hidden(qwen_model, seqlen), parameters.rng, token_index }, - .{ - .shardings = ¶meters.shardings.all(), - .program_name = phase.programName("qwen3_5", "sampler"), - }, - ); - } - - fn hidden(qwen_model: model.Model, seqlen: usize) zml.Tensor { - return .fromShape(zml.Shape.init( - .{ .b = 1, .s = seqlen, .d = qwen_model.config.text_config.hidden_size }, - qwen_model.text_model.embed_tokens.weight.dtype(), - ).withPartitioning(.{ - .b = .replicated, - .s = .replicated, - .d = .replicated, - })); - } - - fn findFirstLayerIndex(layer_types: []const model.LayerType, target: model.LayerType) ?usize { - for (layer_types, 0..) |layer_type, index| { - if (layer_type == target) return index; - } - return null; - } -}; +fn findFirstLayerIndex(layer_types: []const model.LayerType, target: model.LayerType) ?usize { + for (layer_types, 0..) |layer_type, index| if (layer_type == target) return index; + return null; +} diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index d7f33270..20df599c 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -220,8 +220,13 @@ pub const Model = struct { ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { const tokens = tokens_.withPartialTags(.{.s}); const text_model_output, const updated_kv_cache = self.text_model.forward(tokens, token_index, kv_cache); - const new_tokens, const new_rng, _ = self.sampler().sampleTokens(text_model_output, rng, null); - return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; + const result = Sampler.sampleTokens(.{ + .sampler = self.sampler(), + .hidden = text_model_output, + .rng = rng, + .token_index = token_index, + }); + return .{ result.tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, result.rng }; } pub fn sampler(self: Model) Sampler { @@ -238,19 +243,49 @@ pub const Sampler = struct { lm_head: zml.nn.Linear, gen_options: Model.GenOptions, - pub fn sampleTokens( - self: Sampler, - out: zml.Tensor, + pub const Input = struct { + sampler: Sampler, + hidden: zml.Tensor, rng: zml.Tensor.Rng, - token_index: ?zml.Tensor, - ) struct { zml.Tensor, zml.Tensor.Rng, ?zml.Tensor } { - const x = self.norm.forward(out); + token_index: zml.Tensor, + }; + + pub const Output = struct { + tokens: zml.Tensor, + rng: zml.Tensor.Rng, + token_index: zml.Tensor, + }; + + pub fn sampleTokens(input: Input) Output { + const self = input.sampler; + const x = self.norm.forward(input.hidden); const logits = self.lm_head.forward(x.withPartialTags(.{.d})).rename(.{ .dout = .voc }); - const next_tokens, const new_rng = zml.nn.sampleTokens(logits, self.gen_options.sampling_strategy, rng); - if (token_index) |token_idx| { - return .{ next_tokens.convert(.u32), new_rng, token_idx.addConstant(1) }; - } - return .{ next_tokens.convert(.u32), new_rng, null }; + const next_tokens, const new_rng = zml.nn.sampleTokens(logits, self.gen_options.sampling_strategy, input.rng); + return .{ + .tokens = next_tokens.convert(.u32), + .rng = new_rng, + .token_index = input.token_index.addConstant(1), + }; + } +}; + +pub const EmbedTokens = struct { + embed_tokens: zml.nn.TokenEmbedding, + + pub const Input = struct { + embedding: EmbedTokens, + tokens: zml.Tensor, + }; + + pub const Output = struct { + hidden: zml.Tensor, + }; + + pub fn forward(input: Input) Output { + const tokens = input.tokens.withPartialTags(.{.s}); + return .{ .hidden = input.embedding.embed_tokens.forward(tokens) + .withPartialTags(.{.d}) + .withPartitioning(.{ .d = .replicated }) }; } }; @@ -297,7 +332,10 @@ pub const TextModel = struct { token_index: zml.Tensor, kv_cache: KvCache, ) struct { zml.Tensor, KvCache } { - var hidden_states = self.embed_tokens.weight.gather(.{ .voc = tokens }, .{}); + var hidden_states = EmbedTokens.forward(.{ + .embedding = .{ .embed_tokens = self.embed_tokens }, + .tokens = tokens, + }).hidden; var updated_kv_cache = kv_cache; for (self.layers, 0..) |layer, i| { @@ -319,6 +357,30 @@ pub const TransformerLayer = struct { mlp: Mlp, post_attention_layernorm: RmsNorm, + pub const SelfAttnInput = struct { + layer: TransformerLayer, + hidden: zml.Tensor, + token_index: zml.Tensor, + cache: KvCache.SelfAttnCache, + }; + + pub const SelfAttnOutput = struct { + hidden: zml.Tensor, + cache: KvCache.SelfAttnCache, + }; + + pub const LinearAttnInput = struct { + layer: TransformerLayer, + hidden: zml.Tensor, + token_index: zml.Tensor, + cache: KvCache.GatedDeltaNetCache, + }; + + pub const LinearAttnOutput = struct { + hidden: zml.Tensor, + cache: KvCache.GatedDeltaNetCache, + }; + pub fn init(store: zml.io.TensorStore.View, config: Config, layer_index: usize) !TransformerLayer { const is_full_attention = config.text_config.layer_types[layer_index] == .full_attention; const attn: Attn = switch (is_full_attention) { @@ -383,12 +445,9 @@ pub const TransformerLayer = struct { return .{ mlp_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), updated_kv_cache }; } - pub fn forwardSelfAttn( - self: TransformerLayer, - x0: zml.Tensor, - token_index: zml.Tensor, - kv_cache: KvCache.SelfAttnCache, - ) struct { zml.Tensor, KvCache.SelfAttnCache } { + pub fn forwardSelfAttn(input: SelfAttnInput) SelfAttnOutput { + const self = input.layer; + const x0 = input.hidden; const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); @@ -396,22 +455,22 @@ pub const TransformerLayer = struct { .full_attention => |self_attn| self_attn, .linear_attention => unreachable, }; - const attention_output, const updated_kv_cache = self_attn.forward(normalized_x0, token_index, kv_cache); + const attention_output, const updated_kv_cache = self_attn.forward(normalized_x0, input.token_index, input.cache); const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.{ .d = .replicated }); - return .{ mlp_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), updated_kv_cache }; + return .{ + .hidden = mlp_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), + .cache = updated_kv_cache, + }; } - pub fn forwardLinearAttn( - self: TransformerLayer, - x0: zml.Tensor, - token_index: zml.Tensor, - kv_cache: KvCache.GatedDeltaNetCache, - ) struct { zml.Tensor, KvCache.GatedDeltaNetCache } { - _ = token_index; + pub fn forwardLinearAttn(input: LinearAttnInput) LinearAttnOutput { + const self = input.layer; + const x0 = input.hidden; + _ = input.token_index; const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); @@ -419,13 +478,16 @@ pub const TransformerLayer = struct { .linear_attention => |linear_attn| linear_attn, .full_attention => unreachable, }; - const attention_output, const updated_kv_cache = linear_attn.forward(normalized_x0, kv_cache); + const attention_output, const updated_kv_cache = linear_attn.forward(normalized_x0, input.cache); const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.{ .d = .replicated }); - return .{ mlp_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), updated_kv_cache }; + return .{ + .hidden = mlp_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), + .cache = updated_kv_cache, + }; } }; diff --git a/examples/llm/models/qwen3_5/session.zig b/examples/llm/models/qwen3_5/session.zig index d593e319..33ceeac0 100644 --- a/examples/llm/models/qwen3_5/session.zig +++ b/examples/llm/models/qwen3_5/session.zig @@ -9,9 +9,10 @@ pub const Session = struct { allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, - model_buffers: *model.Buffers, - compiled_model: *const inference.CompiledModel, - decode_runner: inference.KernelExe.Runner, + compiled_model: *inference.CompiledModel, + prefill: inference.KernelRunner, + decode: inference.KernelRunner, + layer_index_buffers: []zml.Buffer, kv_cache_buffers: zml.Bufferized(model.KvCache), rng_buffers: zml.Bufferized(zml.Tensor.Rng), tokenizer: zml.tokenizer.Tokenizer, @@ -27,7 +28,7 @@ pub const Session = struct { io: std.Io, platform: *const zml.Platform, tokenizer: zml.tokenizer.Tokenizer, - compiled_model: *const inference.CompiledModel, + compiled_model: *inference.CompiledModel, model_buffers: *model.Buffers, ) !Session { var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform, compiled_model.params.shardings.model); @@ -37,25 +38,46 @@ pub const Session = struct { var rng_buffers = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); - var decode_runner = try compiled_model.decode.initRunner( - allocator, - io, - platform, - model_buffers, - ); - errdefer decode_runner.deinit(allocator); + const layer_types = compiled_model.loaded_model.inner.config.text_config.layer_types; + const layer_index_buffers = try allocator.alloc(zml.Buffer, layer_types.len); + errdefer allocator.free(layer_index_buffers); + var initialized_layer_index_buffers: usize = 0; + errdefer for (layer_index_buffers[0..initialized_layer_index_buffers]) |*buffer| buffer.deinit(); + var full_attention_index: u32 = 0; + var linear_attention_index: u32 = 0; + for (layer_index_buffers, layer_types) |*buffer, layer_type| { + buffer.* = try switch (layer_type) { + .full_attention => b: { + defer full_attention_index += 1; + break :b zml.Buffer.scalar(io, platform, full_attention_index, .u32); + }, + .linear_attention => b: { + defer linear_attention_index += 1; + break :b zml.Buffer.scalar(io, platform, linear_attention_index, .u32); + }, + }; + initialized_layer_index_buffers += 1; + } + + const generated_token_slice = try zml.Slice.alloc(allocator, zml.Shape.init(.{ .b = 1, .s = 1 }, .u32)); + errdefer generated_token_slice.free(allocator); + + var prefill = try inference.KernelRunner.init(allocator, &compiled_model.prefill, model_buffers); + errdefer prefill.deinit(allocator); + const decode = try inference.KernelRunner.init(allocator, &compiled_model.decode, model_buffers); return .{ .allocator = allocator, .io = io, .platform = platform, - .model_buffers = model_buffers, .compiled_model = compiled_model, - .decode_runner = decode_runner, + .prefill = prefill, + .decode = decode, + .layer_index_buffers = layer_index_buffers, .kv_cache_buffers = kv_cache_buffers, .rng_buffers = rng_buffers, .tokenizer = tokenizer, - .generated_token_slice = try .alloc(allocator, zml.Shape.init(.{ .b = 1, .s = 1 }, .u32)), + .generated_token_slice = generated_token_slice, .seqlen = compiled_model.params.seqlen, .eos_token_id = compiled_model.loaded_model.inner.special_tokens.end_of_text_token_id, .special_tokens = compiled_model.loaded_model.inner.special_tokens, @@ -65,7 +87,10 @@ pub const Session = struct { } pub fn deinit(self: *Session) void { - self.decode_runner.deinit(self.allocator); + self.prefill.deinit(self.allocator); + self.decode.deinit(self.allocator); + for (self.layer_index_buffers) |*buffer| buffer.deinit(); + self.allocator.free(self.layer_index_buffers); model.KvCache.deinitBuffer(&self.kv_cache_buffers); zml.Tensor.Rng.deinitBuffer(&self.rng_buffers); self.generated_token_slice.free(self.allocator); @@ -94,16 +119,13 @@ pub const Session = struct { var prefill_token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 0), .u32); defer prefill_token_index_buffer.deinit(); - try self.compiled_model.prefill.run(.{ - .allocator = self.allocator, + inference.run(&self.prefill, .{ .io = self.io, - .platform = self.platform, - .model_buffers = self.model_buffers, .tokens_buf = &prefill_tokens_buffer, .token_index_buf = &prefill_token_index_buffer, .kv_cache_buffers = &self.kv_cache_buffers, .rng_buffers = &self.rng_buffers, - }); + }, self.layer_index_buffers); try prefill_tokens_buffer.toSlice(self.io, prefill_tokens_slice); const generated_token = prefill_tokens_slice.items(u32)[all_tokens.len - 1]; @@ -141,16 +163,13 @@ pub const Session = struct { try all_tokens.append(self.allocator, token_id); if (all_tokens.items.len >= self.seqlen) break :generation; - try self.decode_runner.run(.{ - .allocator = self.allocator, + inference.run(&self.decode, .{ .io = self.io, - .platform = self.platform, - .model_buffers = self.model_buffers, .tokens_buf = ¤t_token_buffer, .token_index_buf = &token_index_buffer, .kv_cache_buffers = &self.kv_cache_buffers, .rng_buffers = &self.rng_buffers, - }); + }, self.layer_index_buffers); try current_token_buffer.toSlice(self.io, self.generated_token_slice); } diff --git a/examples/llm/models/qwen3_5_moe/inference.zig b/examples/llm/models/qwen3_5_moe/inference.zig index 516760d4..ed95d29a 100644 --- a/examples/llm/models/qwen3_5_moe/inference.zig +++ b/examples/llm/models/qwen3_5_moe/inference.zig @@ -7,17 +7,6 @@ const model = @import("model.zig"); const log = std.log.scoped(.qwen3_5_moe); -const CompileModelResult = struct { - prefill_embedding_exe: zml.Exe, - decode_embedding_exe: zml.Exe, - prefill_full_layer_exe: ?zml.Exe, - decode_full_layer_exe: ?zml.Exe, - prefill_linear_layer_exe: ?zml.Exe, - decode_linear_layer_exe: ?zml.Exe, - prefill_sampling_exe: zml.Exe, - decode_sampling_exe: zml.Exe, -}; - pub const CompilationParameters = struct { kv_cache: model.KvCache, rng: zml.Tensor.Rng, @@ -45,16 +34,26 @@ pub const CompilationParameters = struct { pub const CompilationOptions = CompilationParameters; +pub const LayerIndexBuffer = union(enum) { + self_attn: zml.Buffer, + linear_attn: zml.Buffer, +}; + +pub const RunArgs = struct { + io: std.Io, + tokens_buffer: *zml.Buffer, + full_attention_token_index_buffer: *zml.Buffer, + linear_attention_token_index_buffer: *zml.Buffer, + kv_cache_buffers: *zml.Bufferized(model.KvCache), + moe_metadata_buffers: zml.Bufferized(zml.moe.Metadata), + rng_buffers: *zml.Bufferized(zml.Tensor.Rng), + layer_index_buffers: []const LayerIndexBuffer, +}; + pub const CompiledModel = struct { loaded_model: *const model.LoadedModel, - prefill_embedding_exe: zml.Exe, - decode_embedding_exe: zml.Exe, - prefill_full_layer_exe: ?zml.Exe, - decode_full_layer_exe: ?zml.Exe, - prefill_linear_layer_exe: ?zml.Exe, - decode_linear_layer_exe: ?zml.Exe, - prefill_sampling_exe: zml.Exe, - decode_sampling_exe: zml.Exe, + prefill: KernelExe, + decode: KernelExe, params: CompilationParameters, pub fn init( @@ -66,362 +65,302 @@ pub const CompiledModel = struct { parameters: CompilationParameters, progress: *std.Progress.Node, ) !CompiledModel { - const compile_result = try compileModel(allocator, io, platform, qwen_model, parameters, progress); + const prefill = try compileKernel(allocator, io, platform, qwen_model, parameters, parameters.seqlen, parameters.prefill_moe_metadata, "prefill", progress); + errdefer prefill.deinit(); + const decode = try compileKernel(allocator, io, platform, qwen_model, parameters, 1, parameters.decode_moe_metadata, "decode", progress); return .{ .loaded_model = loaded_model, - .prefill_embedding_exe = compile_result.prefill_embedding_exe, - .decode_embedding_exe = compile_result.decode_embedding_exe, - .prefill_full_layer_exe = compile_result.prefill_full_layer_exe, - .decode_full_layer_exe = compile_result.decode_full_layer_exe, - .prefill_linear_layer_exe = compile_result.prefill_linear_layer_exe, - .decode_linear_layer_exe = compile_result.decode_linear_layer_exe, - .prefill_sampling_exe = compile_result.prefill_sampling_exe, - .decode_sampling_exe = compile_result.decode_sampling_exe, + .prefill = prefill, + .decode = decode, .params = parameters, }; } pub fn deinit(self: *CompiledModel) void { - self.prefill_embedding_exe.deinit(); - self.decode_embedding_exe.deinit(); - if (self.prefill_full_layer_exe) |exe| exe.deinit(); - if (self.decode_full_layer_exe) |exe| exe.deinit(); - if (self.prefill_linear_layer_exe) |exe| exe.deinit(); - if (self.decode_linear_layer_exe) |exe| exe.deinit(); - self.prefill_sampling_exe.deinit(); - self.decode_sampling_exe.deinit(); + self.prefill.deinit(); + self.decode.deinit(); } }; pub const Inference = CompiledModel; -fn compileSelfAttnLayerExe( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - mdl: model.TransformerLayer, - hidden: zml.Tensor, - token_index: zml.Tensor, - cache: model.KvCache.SelfAttnCache, - config: model.Config, - moe_metadata: zml.moe.Metadata, - moe_parameters: zml.moe.Parameters, - shardings: []const zml.Sharding, - xla_dump_to: ?[]const u8, - progress: *std.Progress.Node, - label: []const u8, -) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - const compiling_label = try std.fmt.allocPrint(allocator, "Compiling {s}...", .{label}); - defer allocator.free(compiling_label); - var node = progress.start(compiling_label, 1); - defer node.end(); +pub const EmbedExe = zml.TypedExe(model.EmbedTokens.forward); +pub const FullAttentionExe = zml.TypedExe(model.TransformerLayer.forwardSelfAttn); +pub const LinearAttentionExe = zml.TypedExe(model.TransformerLayer.forwardLinearAttn); +pub const SampleExe = zml.TypedExe(model.Sampler.sampleTokens); - const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Compiled {s} [{f}]", .{ label, now.untilNow(io, .awake) }); +pub const KernelExe = struct { + embed: EmbedExe, + full_attention: FullAttentionExe, + linear_attention: LinearAttentionExe, + sample: SampleExe, - return platform.compile(allocator, io, mdl, .forwardSelfAttn, .{ hidden, token_index, cache, config, moe_metadata, moe_parameters }, .{ - .shardings = shardings, - .xla_dump_to = xla_dump_to, + pub fn deinit(self: *const KernelExe) void { + self.embed.deinit(); + self.full_attention.deinit(); + self.linear_attention.deinit(); + self.sample.deinit(); + } +}; + +pub const LayerRunner = union(enum) { + full_attention: FullAttentionExe.Runner(.{.layer}), + linear_attention: LinearAttentionExe.Runner(.{.layer}), + + pub fn deinit(self: *LayerRunner, allocator: std.mem.Allocator) void { + switch (self.*) { + inline else => |*runner| runner.deinit(allocator), + } + } +}; + +pub const KernelRunner = struct { + embed: EmbedExe.Runner(.{.embedding}), + layers: []LayerRunner, + sample: SampleExe.Runner(.{.sampler}), + + pub fn init(allocator: std.mem.Allocator, exe: *const KernelExe, buffers: *const model.Buffers) !KernelRunner { + var embed = try EmbedExe.Runner(.{.embedding}).init(&exe.embed, allocator, .{ + .embedding = buffers.text_model.embed_tokens, + }); + errdefer embed.deinit(allocator); + + const layers = try allocator.alloc(LayerRunner, buffers.text_model.layers.len); + errdefer allocator.free(layers); + var initialized_layers: usize = 0; + errdefer for (layers[0..initialized_layers]) |*layer| layer.deinit(allocator); + for (layers, buffers.text_model.layers) |*layer, layer_buffers| { + layer.* = switch (layer_buffers.attn) { + .self_attn => .{ .full_attention = try FullAttentionExe.Runner(.{.layer}).init( + &exe.full_attention, + allocator, + .{ .layer = layer_buffers }, + ) }, + .linear_attn => .{ .linear_attention = try LinearAttentionExe.Runner(.{.layer}).init( + &exe.linear_attention, + allocator, + .{ .layer = layer_buffers }, + ) }, + }; + initialized_layers += 1; + } + + var sample = try SampleExe.Runner(.{.sampler}).init(&exe.sample, allocator, .{ + .sampler = .{ + .norm = buffers.text_model.norm, + .lm_head = buffers.text_model.lm_head, + }, + }); + errdefer sample.deinit(allocator); + + return .{ .embed = embed, .layers = layers, .sample = sample }; + } + + pub fn deinit(self: *KernelRunner, allocator: std.mem.Allocator) void { + self.embed.deinit(allocator); + for (self.layers) |*layer| layer.deinit(allocator); + allocator.free(self.layers); + self.sample.deinit(allocator); + } +}; + +pub fn run(runner: *KernelRunner, args: RunArgs) void { + var hidden_buffer: zml.Buffer = undefined; + runner.embed.run(args.io, .{ + .inputs = .{ + .tokens = args.tokens_buffer.*, + }, + .outputs = .{ .hidden = &hidden_buffer }, + }); + defer hidden_buffer.deinit(); + + for (runner.layers, args.layer_index_buffers) |*layer_runner, layer_index_buffer| { + switch (layer_runner.*) { + .full_attention => |*layer| { + const index_buffer = switch (layer_index_buffer) { + .self_attn => |buffer| buffer, + .linear_attn => unreachable, + }; + var layer_cache: zml.Bufferized(model.KvCache.SelfAttnCache) = .{ + .k = args.kv_cache_buffers.self_attn.k, + .v = args.kv_cache_buffers.self_attn.v, + .layer_index = index_buffer, + }; + layer.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .token_index = args.full_attention_token_index_buffer.*, + .cache = layer_cache, + .moe_metadata = args.moe_metadata_buffers, + }, + .outputs = .{ .hidden = &hidden_buffer, .cache = &layer_cache }, + }); + args.kv_cache_buffers.self_attn.k = layer_cache.k; + args.kv_cache_buffers.self_attn.v = layer_cache.v; + }, + .linear_attention => |*layer| { + const index_buffer = switch (layer_index_buffer) { + .linear_attn => |buffer| buffer, + .self_attn => unreachable, + }; + var layer_cache: zml.Bufferized(model.KvCache.GatedDeltaNetCache) = .{ + .conv_state = args.kv_cache_buffers.gated_delta_net.conv_state, + .recurrent_state = args.kv_cache_buffers.gated_delta_net.recurrent_state, + .layer_index = index_buffer, + }; + layer.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .token_index = args.linear_attention_token_index_buffer.*, + .cache = layer_cache, + .moe_metadata = args.moe_metadata_buffers, + }, + .outputs = .{ .hidden = &hidden_buffer, .cache = &layer_cache }, + }); + args.kv_cache_buffers.gated_delta_net.conv_state = layer_cache.conv_state; + args.kv_cache_buffers.gated_delta_net.recurrent_state = layer_cache.recurrent_state; + }, + } + } + + runner.sample.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .rng = args.rng_buffers.*, + .token_index = args.full_attention_token_index_buffer.*, + }, + .outputs = .{ + .tokens = args.tokens_buffer, + .rng = args.rng_buffers, + .token_index = args.full_attention_token_index_buffer, + }, }); } -fn compileLinearAttnLayerExe( +fn compileKernel( allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, - mdl: model.TransformerLayer, - hidden: zml.Tensor, - token_index: zml.Tensor, - cache: model.KvCache.GatedDeltaNetCache, - config: model.Config, + mdl: model.Model, + parameters: CompilationParameters, + seqlen: usize, moe_metadata: zml.moe.Metadata, - moe_parameters: zml.moe.Parameters, - shardings: []const zml.Sharding, - xla_dump_to: ?[]const u8, + phase: []const u8, progress: *std.Progress.Node, - label: []const u8, -) !zml.Exe { +) !KernelExe { + const full_index = findFirstLayerIndex(mdl.config.text_config.layer_types, .full_attention) orelse return error.MissingFullAttentionLayer; + const linear_index = findFirstLayerIndex(mdl.config.text_config.layer_types, .linear_attention) orelse return error.MissingLinearAttentionLayer; + + var embed_future = try io.concurrent(compileEmbed, .{ allocator, io, platform, mdl, parameters, seqlen, phase, progress }); + errdefer if (embed_future.cancel(io)) |exe| exe.deinit() else |_| {}; + var full_attention_future = try io.concurrent(compileFullAttention, .{ allocator, io, platform, mdl, parameters, seqlen, full_index, moe_metadata, phase, progress }); + errdefer if (full_attention_future.cancel(io)) |exe| exe.deinit() else |_| {}; + var linear_attention_future = try io.concurrent(compileLinearAttention, .{ allocator, io, platform, mdl, parameters, seqlen, linear_index, moe_metadata, phase, progress }); + errdefer if (linear_attention_future.cancel(io)) |exe| exe.deinit() else |_| {}; + var sample_future = try io.concurrent(compileSample, .{ allocator, io, platform, mdl, parameters, seqlen, phase, progress }); + errdefer if (sample_future.cancel(io)) |exe| exe.deinit() else |_| {}; + + const embed = try embed_future.await(io); + errdefer embed.deinit(); + const full_attention = try full_attention_future.await(io); + errdefer full_attention.deinit(); + const linear_attention = try linear_attention_future.await(io); + errdefer linear_attention.deinit(); + const sample = try sample_future.await(io); + errdefer sample.deinit(); + + return .{ + .embed = embed, + .full_attention = full_attention, + .linear_attention = linear_attention, + .sample = sample, + }; +} + +fn compileEmbed(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, phase: []const u8, progress: *std.Progress.Node) !EmbedExe { + return compileExe(allocator, io, platform, model.EmbedTokens.forward, .{.{ + .embedding = mdl.text_model.embed_tokens, + .tokens = zml.Tensor.init(.{ .b = 1, .s = seqlen }, .u32), + }}, parameters, progress, phase, "embedding"); +} + +fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, layer_index: usize, moe_metadata: zml.moe.Metadata, phase: []const u8, progress: *std.Progress.Node) !FullAttentionExe { + return compileExe(allocator, io, platform, model.TransformerLayer.forwardSelfAttn, .{.{ + .layer = mdl.text_model.layers[layer_index], + .hidden = hiddenTensor(mdl, seqlen), + .token_index = zml.Tensor.init(.{}, .u32), + .cache = .{ + .k = parameters.kv_cache.self_attn.k, + .v = parameters.kv_cache.self_attn.v, + .layer_index = zml.Tensor.init(.{}, .u32), + }, + .config = mdl.config, + .moe_metadata = moe_metadata, + .moe_parameters = parameters.moe_parameters, + }}, parameters, progress, phase, "full-attention layer"); +} + +fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, layer_index: usize, moe_metadata: zml.moe.Metadata, phase: []const u8, progress: *std.Progress.Node) !LinearAttentionExe { + return compileExe(allocator, io, platform, model.TransformerLayer.forwardLinearAttn, .{.{ + .layer = mdl.text_model.layers[layer_index], + .hidden = hiddenTensor(mdl, seqlen), + .token_index = zml.Tensor.init(.{}, .u32), + .cache = .{ + .conv_state = parameters.kv_cache.gated_delta_net.conv_state, + .recurrent_state = parameters.kv_cache.gated_delta_net.recurrent_state, + .layer_index = zml.Tensor.init(.{}, .u32), + }, + .config = mdl.config, + .moe_metadata = moe_metadata, + .moe_parameters = parameters.moe_parameters, + }}, parameters, progress, phase, "linear-attention layer"); +} + +fn compileSample(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, phase: []const u8, progress: *std.Progress.Node) !SampleExe { + return compileExe(allocator, io, platform, model.Sampler.sampleTokens, .{.{ + .sampler = mdl.text_model.sampler(), + .hidden = hiddenTensor(mdl, seqlen), + .rng = parameters.rng, + .token_index = zml.Tensor.init(.{}, .u32), + }}, parameters, progress, phase, "sampling"); +} + +fn compileExe( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + comptime function: anytype, + args: std.meta.ArgsTuple(@TypeOf(function)), + parameters: CompilationParameters, + progress: *std.Progress.Node, + phase: []const u8, + component: []const u8, +) !zml.TypedExe(function) { progress.increaseEstimatedTotalItems(1); - const compiling_label = try std.fmt.allocPrint(allocator, "Compiling {s}...", .{label}); - defer allocator.free(compiling_label); - var node = progress.start(compiling_label, 1); + const label = try std.fmt.allocPrint(allocator, "Compiling {s} {s}...", .{ phase, component }); + defer allocator.free(label); + var node = progress.start(label, 1); defer node.end(); - const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Compiled {s} [{f}]", .{ label, now.untilNow(io, .awake) }); + defer log.info("Compiled {s} {s} [{f}]", .{ phase, component, now.untilNow(io, .awake) }); + return zml.TypedExe(function).compile(allocator, io, platform, .{ + .shardings = ¶meters.shardings.all(), + .xla_dump_to = parameters.xla_dump_to, + }, args); +} - return platform.compile(allocator, io, mdl, .forwardLinearAttn, .{ hidden, token_index, cache, config, moe_metadata, moe_parameters }, .{ - .shardings = shardings, - .xla_dump_to = xla_dump_to, - }); +fn hiddenTensor(mdl: model.Model, seqlen: usize) zml.Tensor { + return .fromShape(zml.Shape.init( + .{ .b = 1, .s = seqlen, .d = mdl.config.text_config.hidden_size }, + mdl.text_model.embed_tokens.weight.dtype(), + ).withPartitioning(.{ .b = .replicated, .s = .replicated, .d = .replicated })); } fn findFirstLayerIndex(layer_types: []const model.LayerType, target: model.LayerType) ?usize { - for (layer_types, 0..) |layer_type, index| { - if (layer_type == target) return index; - } + for (layer_types, 0..) |layer_type, index| if (layer_type == target) return index; return null; } -fn compileModel( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - qwen35_model: model.Model, - parameters: CompilationParameters, - progress: *std.Progress.Node, -) !CompileModelResult { - const now: std.Io.Timestamp = .now(io, .awake); - const all_shardings = parameters.shardings.all(); - const prefill_len: usize = @intCast(parameters.seqlen); - defer log.info("Compiled model [{f}]", .{now.untilNow(io, .awake)}); - log.info("Compiling model for platform {any} with prefill length {d}...", .{ platform.target, prefill_len }); - - const prefill_tokens = zml.Tensor.init(.{ .b = 1, .s = prefill_len }, .u32); - const decode_tokens = zml.Tensor.init(.{ .b = 1, .s = 1 }, .u32); - const hidden_dtype = qwen35_model.text_model.embed_tokens.weight.dtype(); - const prefill_hidden: zml.Tensor = .fromShape(zml.Shape.init( - .{ .b = 1, .s = prefill_len, .d = qwen35_model.config.text_config.hidden_size }, - hidden_dtype, - ).withPartitioning(.{ - .b = .replicated, - .s = .replicated, - .d = .replicated, - })); - const decode_hidden: zml.Tensor = .fromShape(zml.Shape.init( - .{ .b = 1, .s = 1, .d = qwen35_model.config.text_config.hidden_size }, - hidden_dtype, - ).withPartitioning(.{ - .b = .replicated, - .s = .replicated, - .d = .replicated, - })); - const token_index = zml.Tensor.init(.{}, .u32); - const self_attn_cache: model.KvCache.SelfAttnCache = .{ - .k = parameters.kv_cache.self_attn.k, - .v = parameters.kv_cache.self_attn.v, - .layer_index = zml.Tensor.init(.{}, .u32), - }; - const linear_attn_cache: model.KvCache.GatedDeltaNetCache = .{ - .conv_state = parameters.kv_cache.gated_delta_net.conv_state, - .recurrent_state = parameters.kv_cache.gated_delta_net.recurrent_state, - .layer_index = zml.Tensor.init(.{}, .u32), - }; - - var prefill_embedding_future = try io.concurrent(struct { - fn call( - allocator_: std.mem.Allocator, - io_: std.Io, - platform_: *const zml.Platform, - model_: zml.nn.TokenEmbedding, - prefill_tokens_: zml.Tensor, - shardings_: []const zml.Sharding, - progress_: *std.Progress.Node, - ) !zml.Exe { - progress_.increaseEstimatedTotalItems(1); - var node_ = progress_.start("Compiling prefill embedding...", 1); - defer node_.end(); - const now_: std.Io.Timestamp = .now(io_, .awake); - defer log.info("Compiled prefill embedding [{f}]", .{now_.untilNow(io_, .awake)}); - return platform_.compile(allocator_, io_, model_, .forward, .{prefill_tokens_}, .{ .shardings = shardings_ }); - } - }.call, .{ allocator, io, platform, qwen35_model.text_model.embed_tokens, prefill_tokens, &all_shardings, progress }); - errdefer if (prefill_embedding_future.cancel(io)) |v| { - v.deinit(); - } else |_| {}; - - var decode_embedding_future = try io.concurrent(struct { - fn call( - allocator_: std.mem.Allocator, - io_: std.Io, - platform_: *const zml.Platform, - model_: zml.nn.TokenEmbedding, - decode_tokens_: zml.Tensor, - shardings_: []const zml.Sharding, - progress_: *std.Progress.Node, - ) !zml.Exe { - progress_.increaseEstimatedTotalItems(1); - var node_ = progress_.start("Compiling decode embedding...", 1); - defer node_.end(); - const now_: std.Io.Timestamp = .now(io_, .awake); - defer log.info("Compiled decode embedding [{f}]", .{now_.untilNow(io_, .awake)}); - return platform_.compile(allocator_, io_, model_, .forward, .{decode_tokens_}, .{ .shardings = shardings_ }); - } - }.call, .{ allocator, io, platform, qwen35_model.text_model.embed_tokens, decode_tokens, &all_shardings, progress }); - errdefer if (decode_embedding_future.cancel(io)) |v| { - v.deinit(); - } else |_| {}; - - const prefill_embedding_exe = try prefill_embedding_future.await(io); - const decode_embedding_exe = try decode_embedding_future.await(io); - - const full_layer_index = findFirstLayerIndex(qwen35_model.config.text_config.layer_types, .full_attention) orelse return error.MissingFullAttentionLayer; - const linear_layer_index = findFirstLayerIndex(qwen35_model.config.text_config.layer_types, .linear_attention) orelse return error.MissingLinearAttentionLayer; - const full_layer_model = qwen35_model.text_model.layers[full_layer_index]; - const linear_layer_model = qwen35_model.text_model.layers[linear_layer_index]; - - var prefill_full_layer_future = try io.concurrent(struct { - fn call( - allocator_: std.mem.Allocator, - io_: std.Io, - platform_: *const zml.Platform, - layer_model_: model.TransformerLayer, - hidden_: zml.Tensor, - token_index_: zml.Tensor, - cache_: model.KvCache.SelfAttnCache, - config_: model.Config, - moe_metadata_: zml.moe.Metadata, - moe_parameters_: zml.moe.Parameters, - shardings_: []const zml.Sharding, - xla_dump_to_: ?[]const u8, - progress_: *std.Progress.Node, - ) !zml.Exe { - return compileSelfAttnLayerExe(allocator_, io_, platform_, layer_model_, hidden_, token_index_, cache_, config_, moe_metadata_, moe_parameters_, shardings_, xla_dump_to_, progress_, "prefill full-attention layer..."); - } - }.call, .{ allocator, io, platform, full_layer_model, prefill_hidden, token_index, self_attn_cache, qwen35_model.config, parameters.prefill_moe_metadata, parameters.moe_parameters, &all_shardings, parameters.xla_dump_to, progress }); - errdefer if (prefill_full_layer_future.cancel(io)) |v| v.deinit() else |_| {}; - - var decode_full_layer_future = try io.concurrent(struct { - fn call( - allocator_: std.mem.Allocator, - io_: std.Io, - platform_: *const zml.Platform, - layer_model_: model.TransformerLayer, - hidden_: zml.Tensor, - token_index_: zml.Tensor, - cache_: model.KvCache.SelfAttnCache, - config_: model.Config, - moe_metadata_: zml.moe.Metadata, - moe_parameters_: zml.moe.Parameters, - shardings_: []const zml.Sharding, - xla_dump_to_: ?[]const u8, - progress_: *std.Progress.Node, - ) !zml.Exe { - return compileSelfAttnLayerExe(allocator_, io_, platform_, layer_model_, hidden_, token_index_, cache_, config_, moe_metadata_, moe_parameters_, shardings_, xla_dump_to_, progress_, "decode full-attention layer..."); - } - }.call, .{ allocator, io, platform, full_layer_model, decode_hidden, token_index, self_attn_cache, qwen35_model.config, parameters.decode_moe_metadata, parameters.moe_parameters, &all_shardings, parameters.xla_dump_to, progress }); - errdefer if (decode_full_layer_future.cancel(io)) |v| v.deinit() else |_| {}; - - var prefill_full_layer_exe: ?zml.Exe = null; - errdefer if (prefill_full_layer_exe) |exe| exe.deinit(); - var decode_full_layer_exe: ?zml.Exe = null; - errdefer if (decode_full_layer_exe) |exe| exe.deinit(); - prefill_full_layer_exe = try prefill_full_layer_future.await(io); - decode_full_layer_exe = try decode_full_layer_future.await(io); - - var prefill_linear_layer_future = try io.concurrent(struct { - fn call( - allocator_: std.mem.Allocator, - io_: std.Io, - platform_: *const zml.Platform, - layer_model_: model.TransformerLayer, - hidden_: zml.Tensor, - token_index_: zml.Tensor, - cache_: model.KvCache.GatedDeltaNetCache, - config_: model.Config, - moe_metadata_: zml.moe.Metadata, - moe_parameters_: zml.moe.Parameters, - shardings_: []const zml.Sharding, - xla_dump_to_: ?[]const u8, - progress_: *std.Progress.Node, - ) !zml.Exe { - return compileLinearAttnLayerExe(allocator_, io_, platform_, layer_model_, hidden_, token_index_, cache_, config_, moe_metadata_, moe_parameters_, shardings_, xla_dump_to_, progress_, "prefill linear-attention layer..."); - } - }.call, .{ allocator, io, platform, linear_layer_model, prefill_hidden, token_index, linear_attn_cache, qwen35_model.config, parameters.prefill_moe_metadata, parameters.moe_parameters, &all_shardings, parameters.xla_dump_to, progress }); - errdefer if (prefill_linear_layer_future.cancel(io)) |v| v.deinit() else |_| {}; - - var decode_linear_layer_future = try io.concurrent(struct { - fn call( - allocator_: std.mem.Allocator, - io_: std.Io, - platform_: *const zml.Platform, - layer_model_: model.TransformerLayer, - hidden_: zml.Tensor, - token_index_: zml.Tensor, - cache_: model.KvCache.GatedDeltaNetCache, - config_: model.Config, - moe_metadata_: zml.moe.Metadata, - moe_parameters_: zml.moe.Parameters, - shardings_: []const zml.Sharding, - xla_dump_to_: ?[]const u8, - progress_: *std.Progress.Node, - ) !zml.Exe { - return compileLinearAttnLayerExe(allocator_, io_, platform_, layer_model_, hidden_, token_index_, cache_, config_, moe_metadata_, moe_parameters_, shardings_, xla_dump_to_, progress_, "decode linear-attention layer..."); - } - }.call, .{ allocator, io, platform, linear_layer_model, decode_hidden, token_index, linear_attn_cache, qwen35_model.config, parameters.decode_moe_metadata, parameters.moe_parameters, &all_shardings, parameters.xla_dump_to, progress }); - errdefer if (decode_linear_layer_future.cancel(io)) |v| v.deinit() else |_| {}; - - var prefill_linear_layer_exe: ?zml.Exe = null; - errdefer if (prefill_linear_layer_exe) |exe| exe.deinit(); - var decode_linear_layer_exe: ?zml.Exe = null; - errdefer if (decode_linear_layer_exe) |exe| exe.deinit(); - - prefill_linear_layer_exe = try prefill_linear_layer_future.await(io); - decode_linear_layer_exe = try decode_linear_layer_future.await(io); - - var prefill_sampling_future = try io.concurrent(struct { - fn call( - allocator_: std.mem.Allocator, - io_: std.Io, - platform_: *const zml.Platform, - sampler_: model.Sampler, - prefill_hidden_: zml.Tensor, - rng_: zml.Tensor.Rng, - shardings_: []const zml.Sharding, - progress_: *std.Progress.Node, - ) !zml.Exe { - progress_.increaseEstimatedTotalItems(1); - var node_ = progress_.start("Compiling prefill sampling...", 1); - defer node_.end(); - const now_: std.Io.Timestamp = .now(io_, .awake); - defer log.info("Compiled prefill sampling [{f}]", .{now_.untilNow(io_, .awake)}); - return platform_.compile(allocator_, io_, sampler_, .sampleTokens, .{ prefill_hidden_, rng_, null }, .{ .shardings = shardings_ }); - } - }.call, .{ allocator, io, platform, qwen35_model.text_model.sampler(), prefill_hidden, parameters.rng, &all_shardings, progress }); - errdefer if (prefill_sampling_future.cancel(io)) |v| { - v.deinit(); - } else |_| {}; - const prefill_sampling_exe = try prefill_sampling_future.await(io); - - var decode_sampling_future = try io.concurrent(struct { - fn call( - allocator_: std.mem.Allocator, - io_: std.Io, - platform_: *const zml.Platform, - sampler_: model.Sampler, - decode_hidden_: zml.Tensor, - rng_: zml.Tensor.Rng, - token_index_: zml.Tensor, - shardings_: []const zml.Sharding, - progress_: *std.Progress.Node, - ) !zml.Exe { - progress_.increaseEstimatedTotalItems(1); - var node_ = progress_.start("Compiling decode sampling...", 1); - defer node_.end(); - const now_: std.Io.Timestamp = .now(io_, .awake); - defer log.info("Compiled decode sampling [{f}]", .{now_.untilNow(io_, .awake)}); - return platform_.compile(allocator_, io_, sampler_, .sampleTokens, .{ decode_hidden_, rng_, token_index_ }, .{ .shardings = shardings_ }); - } - }.call, .{ allocator, io, platform, qwen35_model.text_model.sampler(), decode_hidden, parameters.rng, token_index, &all_shardings, progress }); - errdefer if (decode_sampling_future.cancel(io)) |v| { - v.deinit(); - } else |_| {}; - - const decode_sampling_exe = try decode_sampling_future.await(io); - - return .{ - .prefill_embedding_exe = prefill_embedding_exe, - .decode_embedding_exe = decode_embedding_exe, - .prefill_full_layer_exe = prefill_full_layer_exe, - .decode_full_layer_exe = decode_full_layer_exe, - .prefill_linear_layer_exe = prefill_linear_layer_exe, - .decode_linear_layer_exe = decode_linear_layer_exe, - .prefill_sampling_exe = prefill_sampling_exe, - .decode_sampling_exe = decode_sampling_exe, - }; -} - fn initMoeMetadata(qwen_model: model.Model, token_len: usize, batch_size: u32, backend: zml.moe.Backend) zml.moe.Metadata { if (qwen_model.config.text_config.num_experts_per_tok == null) { return .init(.fromBackend(backend)); diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 3e34a098..cc28fa6d 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -249,19 +249,44 @@ pub const Sampler = struct { lm_head: zml.nn.Linear, gen_options: Model.GenOptions, - pub fn sampleTokens( - self: Sampler, - out: zml.Tensor, + pub const Input = struct { + sampler: Sampler, + hidden: zml.Tensor, rng: zml.Tensor.Rng, - token_index: ?zml.Tensor, - ) struct { zml.Tensor, zml.Tensor.Rng, ?zml.Tensor } { - const x = self.norm.forward(out); + token_index: zml.Tensor, + }; + + pub const Output = struct { + tokens: zml.Tensor, + rng: zml.Tensor.Rng, + token_index: zml.Tensor, + }; + + pub fn sampleTokens(input: Input) Output { + const self = input.sampler; + const x = self.norm.forward(input.hidden); const logits = self.lm_head.forward(x.withPartialTags(.{.d})).rename(.{ .dout = .voc }); - const next_tokens, const new_rng = zml.nn.sampleTokens(logits, self.gen_options.sampling_strategy, rng); - if (token_index) |token_idx| { - return .{ next_tokens.convert(.u32), new_rng, token_idx.addConstant(1) }; - } - return .{ next_tokens.convert(.u32), new_rng, null }; + const next_tokens, const new_rng = zml.nn.sampleTokens(logits, self.gen_options.sampling_strategy, input.rng); + return .{ + .tokens = next_tokens.convert(.u32), + .rng = new_rng, + .token_index = input.token_index.addConstant(1), + }; + } +}; + +pub const EmbedTokens = struct { + pub const Input = struct { + embedding: zml.nn.TokenEmbedding, + tokens: zml.Tensor, + }; + + pub const Output = struct { + hidden: zml.Tensor, + }; + + pub fn forward(input: Input) Output { + return .{ .hidden = input.embedding.forward(input.tokens) }; } }; @@ -332,24 +357,23 @@ pub const TextModel = struct { moe_metadata: zml.moe.Metadata, moe_parameters: zml.moe.Parameters, ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { - var hidden_states = self.embed_tokens.weight.gather(.{ .voc = tokens }, .{}); + var hidden_states = EmbedTokens.forward(.{ + .embedding = self.embed_tokens, + .tokens = tokens, + }).hidden; var updated_kv_cache = kv_cache; for (self.layers, 0..) |layer, i| { hidden_states, updated_kv_cache = layer.forward(hidden_states, token_index, updated_kv_cache.atLayer(i), config, moe_metadata, moe_parameters); } - const new_tokens, const new_rng = self.sampleTokens(hidden_states, rng); - return .{ new_tokens, updated_kv_cache.reuseBuffer(kv_cache), new_rng }; - } - - pub fn sampleTokens( - self: TextModel, - out: zml.Tensor, - rng: zml.Tensor.Rng, - token_index: ?zml.Tensor, - ) struct { zml.Tensor, zml.Tensor.Rng, ?zml.Tensor } { - return self.sampler().sampleTokens(out, rng, token_index); + const result = Sampler.sampleTokens(.{ + .sampler = self.sampler(), + .hidden = hidden_states, + .rng = rng, + .token_index = token_index, + }); + return .{ result.tokens, updated_kv_cache.reuseBuffer(kv_cache), result.rng }; } }; @@ -364,6 +388,36 @@ pub const TransformerLayer = struct { moe: Moe, post_attention_layernorm: RmsNorm, + pub const SelfAttnInput = struct { + layer: TransformerLayer, + hidden: zml.Tensor, + token_index: zml.Tensor, + cache: KvCache.SelfAttnCache, + config: Config, + moe_metadata: zml.moe.Metadata, + moe_parameters: zml.moe.Parameters, + }; + + pub const SelfAttnOutput = struct { + hidden: zml.Tensor, + cache: KvCache.SelfAttnCache, + }; + + pub const LinearAttnInput = struct { + layer: TransformerLayer, + hidden: zml.Tensor, + token_index: zml.Tensor, + cache: KvCache.GatedDeltaNetCache, + config: Config, + moe_metadata: zml.moe.Metadata, + moe_parameters: zml.moe.Parameters, + }; + + pub const LinearAttnOutput = struct { + hidden: zml.Tensor, + cache: KvCache.GatedDeltaNetCache, + }; + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config, layer_index: usize) !TransformerLayer { const is_full_attention = config.text_config.layer_types[layer_index] == .full_attention; return .{ @@ -391,16 +445,10 @@ pub const TransformerLayer = struct { RmsNorm.unloadBuffers(&self.post_attention_layernorm); } - pub fn forwardSelfAttn( - self: TransformerLayer, - x0: zml.Tensor, - token_index: zml.Tensor, - kv_cache: KvCache.SelfAttnCache, - config: Config, - moe_metadata: zml.moe.Metadata, - moe_parameters: zml.moe.Parameters, - ) struct { zml.Tensor, KvCache.SelfAttnCache } { - _ = config; + pub fn forwardSelfAttn(input: SelfAttnInput) SelfAttnOutput { + const self = input.layer; + const x0 = input.hidden; + _ = input.config; const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); @@ -408,26 +456,23 @@ pub const TransformerLayer = struct { .self_attn => |self_attn| self_attn, .linear_attn => unreachable, }; - const attention_output, const updated_kv_cache = self_attn.forward(normalized_x0, token_index, kv_cache); + const attention_output, const updated_kv_cache = self_attn.forward(normalized_x0, input.token_index, input.cache); const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); - const moe_output = self.moe.forward(normalized_hidden, moe_metadata, moe_parameters); + const moe_output = self.moe.forward(normalized_hidden, input.moe_metadata, input.moe_parameters); - return .{ moe_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), updated_kv_cache }; + return .{ + .hidden = moe_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), + .cache = updated_kv_cache, + }; } - pub fn forwardLinearAttn( - self: TransformerLayer, - x0: zml.Tensor, - token_index: zml.Tensor, - kv_cache: KvCache.GatedDeltaNetCache, - config: Config, - moe_metadata: zml.moe.Metadata, - moe_parameters: zml.moe.Parameters, - ) struct { zml.Tensor, KvCache.GatedDeltaNetCache } { - _ = config; + pub fn forwardLinearAttn(input: LinearAttnInput) LinearAttnOutput { + const self = input.layer; + const x0 = input.hidden; + _ = input.config; const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); @@ -435,14 +480,17 @@ pub const TransformerLayer = struct { .linear_attn => |linear_attn| linear_attn, .self_attn => unreachable, }; - const attention_output, const updated_kv_cache = linear_attn.forward(normalized_x0, kv_cache, token_index); + const attention_output, const updated_kv_cache = linear_attn.forward(normalized_x0, input.cache, input.token_index); const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); - const moe_output = self.moe.forward(normalized_hidden, moe_metadata, moe_parameters); + const moe_output = self.moe.forward(normalized_hidden, input.moe_metadata, input.moe_parameters); - return .{ moe_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), updated_kv_cache }; + return .{ + .hidden = moe_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), + .cache = updated_kv_cache, + }; } pub fn forward( diff --git a/examples/llm/models/qwen3_5_moe/session.zig b/examples/llm/models/qwen3_5_moe/session.zig index 19e4d579..04ab1838 100644 --- a/examples/llm/models/qwen3_5_moe/session.zig +++ b/examples/llm/models/qwen3_5_moe/session.zig @@ -5,26 +5,18 @@ const zml = @import("zml"); const inference = @import("inference.zig"); const model = @import("model.zig"); -const LayerIndexBuffer = union(enum) { - self_attn: zml.Buffer, - linear_attn: zml.Buffer, -}; - pub const Session = struct { allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, - model_buffers: *model.Buffers, - compiled_model: *const inference.CompiledModel, + compiled_model: *inference.CompiledModel, + prefill: inference.KernelRunner, + decode: inference.KernelRunner, kv_cache_buffers: zml.Bufferized(model.KvCache), prefill_moe_metadata_buffers: zml.Bufferized(zml.moe.Metadata), decode_moe_metadata_buffers: zml.Bufferized(zml.moe.Metadata), rng_buffers: zml.Bufferized(zml.Tensor.Rng), - layer_index_buffers: []LayerIndexBuffer, - decode_token_index_buffer: zml.Buffer, - self_attn_layers_caches: []zml.Bufferized(model.KvCache.SelfAttnCache), - linear_attn_layers_caches: []zml.Bufferized(model.KvCache.GatedDeltaNetCache), - step_token_slice: zml.Slice, + layer_index_buffers: []inference.LayerIndexBuffer, generated_token_slice: zml.Slice, tokenizer: zml.tokenizer.Tokenizer, seqlen: u32, @@ -38,7 +30,7 @@ pub const Session = struct { io: std.Io, platform: *const zml.Platform, tokenizer: zml.tokenizer.Tokenizer, - compiled_model: *const inference.CompiledModel, + compiled_model: *inference.CompiledModel, model_buffers: *model.Buffers, ) !Session { const shardings = compiled_model.params.shardings; @@ -47,7 +39,6 @@ pub const Session = struct { var prefill_moe_metadata_buffers = try compiled_model.params.prefill_moe_metadata.initBuffer(io, platform); errdefer zml.moe.Metadata.deinitBuffer(&prefill_moe_metadata_buffers); - var decode_moe_metadata_buffers = try compiled_model.params.decode_moe_metadata.initBuffer(io, platform); errdefer zml.moe.Metadata.deinitBuffer(&decode_moe_metadata_buffers); @@ -56,90 +47,52 @@ pub const Session = struct { errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); const layer_types = compiled_model.loaded_model.inner.config.text_config.layer_types; - var layer_index_buffers = try allocator.alloc(LayerIndexBuffer, compiled_model.loaded_model.inner.text_model.layers.len); - errdefer { - for (layer_index_buffers) |*layer_index_buffer| { - switch (layer_index_buffer.*) { - .self_attn => |*buffer| buffer.deinit(), - .linear_attn => |*buffer| buffer.deinit(), - } + const layer_index_buffers = try allocator.alloc(inference.LayerIndexBuffer, layer_types.len); + errdefer allocator.free(layer_index_buffers); + var initialized_layer_index_buffers: usize = 0; + errdefer for (layer_index_buffers[0..initialized_layer_index_buffers]) |*layer_index_buffer| { + switch (layer_index_buffer.*) { + .self_attn => |*buffer| buffer.deinit(), + .linear_attn => |*buffer| buffer.deinit(), } - allocator.free(layer_index_buffers); + }; + + var self_attn_layer_index: u32 = 0; + var linear_attn_layer_index: u32 = 0; + for (layer_types, layer_index_buffers) |layer_type, *layer_index_buffer| { + layer_index_buffer.* = switch (layer_type) { + .full_attention => b: { + defer self_attn_layer_index += 1; + break :b .{ .self_attn = try .scalar(io, platform, self_attn_layer_index, .u32) }; + }, + .linear_attention => b: { + defer linear_attn_layer_index += 1; + break :b .{ .linear_attn = try .scalar(io, platform, linear_attn_layer_index, .u32) }; + }, + }; + initialized_layer_index_buffers += 1; } - var self_attn_layer_index: usize = 0; - var linear_attn_layer_index: usize = 0; - for (layer_types, 0..) |layer_type, layer_index| { - switch (layer_type) { - .full_attention => { - const layer_index_buffer = try zml.Buffer.scalar(io, platform, @as(u32, @intCast(self_attn_layer_index)), .u32); - layer_index_buffers[layer_index] = .{ .self_attn = layer_index_buffer }; - self_attn_layer_index += 1; - }, - .linear_attention => { - const layer_index_buffer = try zml.Buffer.scalar(io, platform, @as(u32, @intCast(linear_attn_layer_index)), .u32); - layer_index_buffers[layer_index] = .{ .linear_attn = layer_index_buffer }; - linear_attn_layer_index += 1; - }, - } - } + const generated_token_slice = try zml.Slice.alloc(allocator, zml.Shape.init(.{ .b = 1, .s = 1 }, .u32)); + errdefer generated_token_slice.free(allocator); - var decode_token_index_buffer = try zml.Buffer.scalar(io, platform, @as(u32, 0), .u32); - errdefer decode_token_index_buffer.deinit(); - - var self_attn_layers_caches = try allocator.alloc(zml.Bufferized(model.KvCache.SelfAttnCache), self_attn_layer_index); - errdefer allocator.free(self_attn_layers_caches); - - var linear_attn_layers_caches = try allocator.alloc(zml.Bufferized(model.KvCache.GatedDeltaNetCache), linear_attn_layer_index); - errdefer allocator.free(linear_attn_layers_caches); - - var self_attn_cache_index: usize = 0; - var linear_attn_cache_index: usize = 0; - for (layer_types, 0..) |layer_type, layer_index| { - switch (layer_type) { - .full_attention => { - const layer_index_buffer = switch (layer_index_buffers[layer_index]) { - .self_attn => |buffer| buffer, - .linear_attn => unreachable, - }; - self_attn_layers_caches[self_attn_cache_index] = .{ - .k = kv_cache_buffers.self_attn.k, - .v = kv_cache_buffers.self_attn.v, - .layer_index = layer_index_buffer, - }; - self_attn_cache_index += 1; - }, - .linear_attention => { - const layer_index_buffer = switch (layer_index_buffers[layer_index]) { - .linear_attn => |buffer| buffer, - .self_attn => unreachable, - }; - linear_attn_layers_caches[linear_attn_cache_index] = .{ - .conv_state = kv_cache_buffers.gated_delta_net.conv_state, - .recurrent_state = kv_cache_buffers.gated_delta_net.recurrent_state, - .layer_index = layer_index_buffer, - }; - linear_attn_cache_index += 1; - }, - } - } + var prefill = try inference.KernelRunner.init(allocator, &compiled_model.prefill, model_buffers); + errdefer prefill.deinit(allocator); + const decode = try inference.KernelRunner.init(allocator, &compiled_model.decode, model_buffers); return .{ .allocator = allocator, .io = io, .platform = platform, - .model_buffers = model_buffers, .compiled_model = compiled_model, + .prefill = prefill, + .decode = decode, .kv_cache_buffers = kv_cache_buffers, .prefill_moe_metadata_buffers = prefill_moe_metadata_buffers, .decode_moe_metadata_buffers = decode_moe_metadata_buffers, .rng_buffers = rng_buffers, .layer_index_buffers = layer_index_buffers, - .decode_token_index_buffer = decode_token_index_buffer, - .self_attn_layers_caches = self_attn_layers_caches, - .linear_attn_layers_caches = linear_attn_layers_caches, - .step_token_slice = try .alloc(allocator, zml.Shape.init(.{ .b = 1, .s = 1 }, .u32)), - .generated_token_slice = try .alloc(allocator, zml.Shape.init(.{ .b = 1, .s = 1 }, .u32)), + .generated_token_slice = generated_token_slice, .tokenizer = tokenizer, .seqlen = compiled_model.params.seqlen, .eos_token_id = compiled_model.loaded_model.inner.special_tokens.end_of_text_token_id, @@ -150,11 +103,12 @@ pub const Session = struct { } pub fn deinit(self: *Session) void { + self.prefill.deinit(self.allocator); + self.decode.deinit(self.allocator); model.KvCache.deinitBuffer(&self.kv_cache_buffers); zml.moe.Metadata.deinitBuffer(&self.prefill_moe_metadata_buffers); zml.moe.Metadata.deinitBuffer(&self.decode_moe_metadata_buffers); zml.Tensor.Rng.deinitBuffer(&self.rng_buffers); - self.decode_token_index_buffer.deinit(); for (self.layer_index_buffers) |*layer_index_buffer| { switch (layer_index_buffer.*) { .self_attn => |*buffer| buffer.deinit(), @@ -162,9 +116,6 @@ pub const Session = struct { } } self.allocator.free(self.layer_index_buffers); - self.allocator.free(self.self_attn_layers_caches); - self.allocator.free(self.linear_attn_layers_caches); - self.step_token_slice.free(self.allocator); self.generated_token_slice.free(self.allocator); } @@ -176,247 +127,68 @@ pub const Session = struct { return tokenizeChatPrompt(allocator, self.tokenizer, prompt, self.special_tokens, false); } - fn storeSelfAttnLayerCache( - self: *Session, - dense_layer_index: usize, - layer_index: usize, - layer_cache: zml.Bufferized(model.KvCache.SelfAttnCache), - ) void { - self.kv_cache_buffers.self_attn.k = layer_cache.k; - self.kv_cache_buffers.self_attn.v = layer_cache.v; - self.self_attn_layers_caches[dense_layer_index] = layer_cache; - self.layer_index_buffers[layer_index] = .{ .self_attn = layer_cache.layer_index }; - } - - fn storeLinearAttnLayerCache( - self: *Session, - dense_layer_index: usize, - layer_index: usize, - layer_cache: zml.Bufferized(model.KvCache.GatedDeltaNetCache), - ) void { - self.kv_cache_buffers.gated_delta_net.conv_state = layer_cache.conv_state; - self.kv_cache_buffers.gated_delta_net.recurrent_state = layer_cache.recurrent_state; - self.linear_attn_layers_caches[dense_layer_index] = layer_cache; - self.layer_index_buffers[layer_index] = .{ .linear_attn = layer_cache.layer_index }; - } - pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { - const hidden_size = self.compiled_model.loaded_model.inner.config.text_config.hidden_size; - const model_dtype = self.compiled_model.loaded_model.inner.text_model.embed_tokens.weight.dtype(); + const tokens_slice = try zml.Slice.alloc(self.allocator, .init(.{ .b = 1, .s = self.seqlen }, .u32)); + defer tokens_slice.free(self.allocator); + @memset(tokens_slice.items(u32), 0); + @memcpy(tokens_slice.items(u32)[0..all_tokens.len], all_tokens); - const prefill_tokens_shape = zml.Shape.init(.{ .b = 1, .s = self.seqlen }, .u32); - const prefill_hidden_shape = zml.Shape.init(.{ .b = 1, .s = self.seqlen, .d = hidden_size }, model_dtype).withPartitioning(.{ - .b = .replicated, - .s = .replicated, - .d = .replicated, + var tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, tokens_slice, .replicated); + defer tokens_buffer.deinit(); + var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 0), .u32); + defer token_index_buffer.deinit(); + var valid_len_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.len)), .u32); + defer valid_len_buffer.deinit(); + + inference.run(&self.prefill, .{ + .io = self.io, + .tokens_buffer = &tokens_buffer, + .full_attention_token_index_buffer = &token_index_buffer, + .linear_attention_token_index_buffer = &valid_len_buffer, + .kv_cache_buffers = &self.kv_cache_buffers, + .moe_metadata_buffers = self.prefill_moe_metadata_buffers, + .rng_buffers = &self.rng_buffers, + .layer_index_buffers = self.layer_index_buffers, }); - const prefill_tokens_slice = try zml.Slice.alloc(self.allocator, prefill_tokens_shape); - defer prefill_tokens_slice.free(self.allocator); - @memset(prefill_tokens_slice.items(u32), 0); - @memcpy(prefill_tokens_slice.items(u32)[0..all_tokens.len], all_tokens); - - const replicated_sharding: zml.Sharding = .replicated; - - var prefill_tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, prefill_tokens_slice, replicated_sharding); - defer prefill_tokens_buffer.deinit(); - - var prefill_token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 0), .u32); - defer prefill_token_index_buffer.deinit(); - var prefill_valid_len_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.len)), .u32); - defer prefill_valid_len_buffer.deinit(); - - var prefill_hidden_buffer = try zml.Buffer.uninitialized(self.io, self.platform, prefill_hidden_shape, replicated_sharding, .{}); - defer prefill_hidden_buffer.deinit(); - - var embedding_prefill_args = try self.compiled_model.prefill_embedding_exe.args(self.allocator); - defer embedding_prefill_args.deinit(self.allocator); - var embedding_prefill_results = try self.compiled_model.prefill_embedding_exe.results(self.allocator); - defer embedding_prefill_results.deinit(self.allocator); - - embedding_prefill_args.set(.{ self.model_buffers.text_model.embed_tokens, prefill_tokens_buffer }); - self.compiled_model.prefill_embedding_exe.call(embedding_prefill_args, &embedding_prefill_results); - embedding_prefill_results.fill(.{&prefill_hidden_buffer}); - - var prefill_full_layer_args: ?zml.Exe.Arguments = if (self.compiled_model.prefill_full_layer_exe) |exe| try exe.args(self.allocator) else null; - defer if (prefill_full_layer_args) |*args| args.deinit(self.allocator); - var prefill_full_layer_results: ?zml.Exe.Results = if (self.compiled_model.prefill_full_layer_exe) |exe| try exe.results(self.allocator) else null; - defer if (prefill_full_layer_results) |*results| results.deinit(self.allocator); - - var prefill_linear_layer_args: ?zml.Exe.Arguments = if (self.compiled_model.prefill_linear_layer_exe) |exe| try exe.args(self.allocator) else null; - defer if (prefill_linear_layer_args) |*args| args.deinit(self.allocator); - var prefill_linear_layer_results: ?zml.Exe.Results = if (self.compiled_model.prefill_linear_layer_exe) |exe| try exe.results(self.allocator) else null; - defer if (prefill_linear_layer_results) |*results| results.deinit(self.allocator); - - var self_attn_cache_index: usize = 0; - var linear_attn_cache_index: usize = 0; - - for (self.model_buffers.text_model.layers, 0..) |layer_weights, i| { - switch (self.layer_index_buffers[i]) { - .self_attn => { - const exe = self.compiled_model.prefill_full_layer_exe orelse unreachable; - self.self_attn_layers_caches[self_attn_cache_index].k = self.kv_cache_buffers.self_attn.k; - self.self_attn_layers_caches[self_attn_cache_index].v = self.kv_cache_buffers.self_attn.v; - prefill_full_layer_args.?.set(.{ layer_weights, prefill_hidden_buffer, prefill_token_index_buffer, self.self_attn_layers_caches[self_attn_cache_index], self.compiled_model.loaded_model.inner.config, self.prefill_moe_metadata_buffers }); - exe.call(prefill_full_layer_args.?, &prefill_full_layer_results.?); - prefill_full_layer_results.?.fill(.{ &prefill_hidden_buffer, &self.kv_cache_buffers.self_attn }); - self_attn_cache_index += 1; - }, - .linear_attn => { - const exe = self.compiled_model.prefill_linear_layer_exe orelse unreachable; - self.linear_attn_layers_caches[linear_attn_cache_index].conv_state = self.kv_cache_buffers.gated_delta_net.conv_state; - self.linear_attn_layers_caches[linear_attn_cache_index].recurrent_state = self.kv_cache_buffers.gated_delta_net.recurrent_state; - prefill_linear_layer_args.?.set(.{ layer_weights, prefill_hidden_buffer, prefill_valid_len_buffer, self.linear_attn_layers_caches[linear_attn_cache_index], self.compiled_model.loaded_model.inner.config, self.prefill_moe_metadata_buffers }); - exe.call(prefill_linear_layer_args.?, &prefill_linear_layer_results.?); - prefill_linear_layer_results.?.fill(.{ &prefill_hidden_buffer, &self.kv_cache_buffers.gated_delta_net }); - linear_attn_cache_index += 1; - }, - } - } - - var sampling_prefill_args = try self.compiled_model.prefill_sampling_exe.args(self.allocator); - defer sampling_prefill_args.deinit(self.allocator); - var sampling_prefill_results = try self.compiled_model.prefill_sampling_exe.results(self.allocator); - defer sampling_prefill_results.deinit(self.allocator); - - sampling_prefill_args.set(.{ .{ - .norm = self.model_buffers.text_model.norm, - .lm_head = self.model_buffers.text_model.lm_head, - .gen_options = self.compiled_model.loaded_model.inner.text_model.gen_options, - }, prefill_hidden_buffer, self.rng_buffers }); - self.compiled_model.prefill_sampling_exe.call(sampling_prefill_args, &sampling_prefill_results); - sampling_prefill_results.fill(.{ &prefill_tokens_buffer, &self.rng_buffers }); - - try prefill_tokens_buffer.toSlice(self.io, prefill_tokens_slice); - const generated_token = prefill_tokens_slice.items(u32)[all_tokens.len - 1]; - self.generated_token_slice.items(u32)[0] = generated_token; + try tokens_buffer.toSlice(self.io, tokens_slice); + self.generated_token_slice.items(u32)[0] = tokens_slice.items(u32)[all_tokens.len - 1]; } pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { var decoder = try self.tokenizer.decoder(); defer decoder.deinit(); - - const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); + const out_tokens_buffer = try self.allocator.alloc(u8, 1024); defer self.allocator.free(out_tokens_buffer); - const hidden_size = self.compiled_model.loaded_model.inner.config.text_config.hidden_size; - const model_dtype = self.compiled_model.loaded_model.inner.text_model.embed_tokens.weight.dtype(); - - const decode_hidden_shape = zml.Shape.init(.{ .b = 1, .s = 1, .d = hidden_size }, model_dtype).withPartitioning(.{ - .b = .replicated, - .s = .replicated, - .d = .replicated, - }); - const replicated_sharding: zml.Sharding = .replicated; - - var embedding_decode_args = try self.compiled_model.decode_embedding_exe.args(self.allocator); - defer embedding_decode_args.deinit(self.allocator); - var embedding_decode_results = try self.compiled_model.decode_embedding_exe.results(self.allocator); - defer embedding_decode_results.deinit(self.allocator); - - var decode_full_layer_args: ?zml.Exe.Arguments = if (self.compiled_model.decode_full_layer_exe) |exe| try exe.args(self.allocator) else null; - defer if (decode_full_layer_args) |*args| args.deinit(self.allocator); - var decode_full_layer_results: ?zml.Exe.Results = if (self.compiled_model.decode_full_layer_exe) |exe| try exe.results(self.allocator) else null; - defer if (decode_full_layer_results) |*results| results.deinit(self.allocator); - - var decode_linear_layer_args: ?zml.Exe.Arguments = if (self.compiled_model.decode_linear_layer_exe) |exe| try exe.args(self.allocator) else null; - defer if (decode_linear_layer_args) |*args| args.deinit(self.allocator); - var decode_linear_layer_results: ?zml.Exe.Results = if (self.compiled_model.decode_linear_layer_exe) |exe| try exe.results(self.allocator) else null; - defer if (decode_linear_layer_results) |*results| results.deinit(self.allocator); - - var sampling_decode_args = try self.compiled_model.decode_sampling_exe.args(self.allocator); - defer sampling_decode_args.deinit(self.allocator); - var sampling_decode_results = try self.compiled_model.decode_sampling_exe.results(self.allocator); - defer sampling_decode_results.deinit(self.allocator); - - var decode_hidden_buffer = try zml.Buffer.uninitialized(self.io, self.platform, decode_hidden_shape, replicated_sharding, .{}); - defer decode_hidden_buffer.deinit(); - - var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice, replicated_sharding); + var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice, .replicated); defer current_token_buffer.deinit(); - var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.items.len)), .u32); defer token_index_buffer.deinit(); - var self_attn_layer_index: usize = 0; - var linear_attn_layer_index: usize = 0; - generation: while (true) { const token_id = self.generated_token_slice.items(u32)[0]; if (token_id == self.eos_token_id) break :generation; const token = try decoder.feedOne(token_id, out_tokens_buffer); - if (self.think_start) |think_start| if (token_id == think_start) { - try stdout.writeAll("\x1b[2m"); - }; + if (self.think_start) |think_start| if (token_id == think_start) try stdout.writeAll("\x1b[2m"); try stdout.writeAll(token); - if (self.think_end) |think_end| if (token_id == think_end) { - try stdout.writeAll("\x1b[0m"); - }; + if (self.think_end) |think_end| if (token_id == think_end) try stdout.writeAll("\x1b[0m"); try stdout.flush(); try all_tokens.append(self.allocator, token_id); if (all_tokens.items.len >= self.seqlen) break :generation; - self_attn_layer_index = 0; - linear_attn_layer_index = 0; - - embedding_decode_args.set(.{ self.model_buffers.text_model.embed_tokens, current_token_buffer }); - self.compiled_model.decode_embedding_exe.call(embedding_decode_args, &embedding_decode_results); - embedding_decode_results.fill(.{&decode_hidden_buffer}); - - for (self.model_buffers.text_model.layers, 0..) |layer_weights, layer_index| { - switch (self.layer_index_buffers[layer_index]) { - .self_attn => { - const exe = self.compiled_model.decode_full_layer_exe orelse unreachable; - self.self_attn_layers_caches[self_attn_layer_index].k = self.kv_cache_buffers.self_attn.k; - self.self_attn_layers_caches[self_attn_layer_index].v = self.kv_cache_buffers.self_attn.v; - - decode_full_layer_args.?.set(.{ - layer_weights, - decode_hidden_buffer, - token_index_buffer, - self.self_attn_layers_caches[self_attn_layer_index], - self.compiled_model.loaded_model.inner.config, - self.decode_moe_metadata_buffers, - }); - - exe.call(decode_full_layer_args.?, &decode_full_layer_results.?); - decode_full_layer_results.?.fill(.{ &decode_hidden_buffer, &self.kv_cache_buffers.self_attn }); - - self_attn_layer_index += 1; - }, - .linear_attn => { - const exe = self.compiled_model.decode_linear_layer_exe orelse unreachable; - - self.linear_attn_layers_caches[linear_attn_layer_index].conv_state = self.kv_cache_buffers.gated_delta_net.conv_state; - self.linear_attn_layers_caches[linear_attn_layer_index].recurrent_state = self.kv_cache_buffers.gated_delta_net.recurrent_state; - - decode_linear_layer_args.?.set(.{ - layer_weights, - decode_hidden_buffer, - token_index_buffer, - self.linear_attn_layers_caches[linear_attn_layer_index], - self.compiled_model.loaded_model.inner.config, - self.decode_moe_metadata_buffers, - }); - - exe.call(decode_linear_layer_args.?, &decode_linear_layer_results.?); - decode_linear_layer_results.?.fill(.{ &decode_hidden_buffer, &self.kv_cache_buffers.gated_delta_net }); - - linear_attn_layer_index += 1; - }, - } - } - - sampling_decode_args.set(.{ .{ - .norm = self.model_buffers.text_model.norm, - .lm_head = self.model_buffers.text_model.lm_head, - .gen_options = self.compiled_model.loaded_model.inner.text_model.gen_options, - }, decode_hidden_buffer, self.rng_buffers, token_index_buffer }); - self.compiled_model.decode_sampling_exe.call(sampling_decode_args, &sampling_decode_results); - sampling_decode_results.fill(.{ ¤t_token_buffer, &self.rng_buffers, &token_index_buffer }); - + inference.run(&self.decode, .{ + .io = self.io, + .tokens_buffer = ¤t_token_buffer, + .full_attention_token_index_buffer = &token_index_buffer, + .linear_attention_token_index_buffer = &token_index_buffer, + .kv_cache_buffers = &self.kv_cache_buffers, + .moe_metadata_buffers = self.decode_moe_metadata_buffers, + .rng_buffers = &self.rng_buffers, + .layer_index_buffers = self.layer_index_buffers, + }); try current_token_buffer.toSlice(self.io, self.generated_token_slice); } @@ -436,31 +208,25 @@ fn tokenizeChatPrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tok if (!is_first_turn) { try tokens.append(allocator, im_end); const newline = try encoder.encodeAlloc(allocator, "\n"); + defer allocator.free(newline); try tokens.appendSlice(allocator, newline); - allocator.free(newline); } try tokens.append(allocator, im_start); const user = try encoder.encodeAlloc(allocator, "user\n"); + defer allocator.free(user); try tokens.appendSlice(allocator, user); - allocator.free(user); const prompt_encoded = try encoder.encodeAlloc(allocator, prompt); + defer allocator.free(prompt_encoded); try tokens.appendSlice(allocator, prompt_encoded); - allocator.free(prompt_encoded); try tokens.append(allocator, im_end); const newline = try encoder.encodeAlloc(allocator, "\n"); + defer allocator.free(newline); try tokens.appendSlice(allocator, newline); - allocator.free(newline); try tokens.append(allocator, im_start); const assistant = try encoder.encodeAlloc(allocator, "assistant\n"); + defer allocator.free(assistant); try tokens.appendSlice(allocator, assistant); - allocator.free(assistant); return tokens.toOwnedSlice(allocator); } - -fn encodeSingleToken(encoder: *zml.tokenizer.Tokenizer.Encoder, text: []const u8) !u32 { - const encoded = try encoder.encode(text); - if (encoded.len != 1) return error.InvalidTokenizerEncoding; - return encoded[0]; -} diff --git a/zml/exe.zig b/zml/exe.zig index 6bbd2e2b..02633f44 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -4,11 +4,14 @@ const pjrt = @import("pjrt"); const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; +const mem = @import("mem.zig"); const meta = @import("meta.zig"); +const module = @import("module.zig"); const Platform = @import("platform.zig").Platform; const tracer = @import("profiling/tracer.zig"); const Shape = @import("shape.zig").Shape; const Sharding = @import("Sharding.zig"); +const Tensor = @import("tensor.zig").Tensor; pub const Exe = struct { platform: *const Platform, @@ -259,6 +262,42 @@ pub const Exe = struct { } }; + /// An executable together with reusable argument and result storage. + /// A runner owns all three values and must not be used concurrently. + pub const Runner = struct { + exe: Exe, + args: Arguments, + results: Results, + + /// Takes ownership of the executable and allocates reusable argument and result storage. + pub fn init(exe: Exe, allocator: std.mem.Allocator) !Runner { + errdefer exe.deinit(); + var arguments = try exe.args(allocator); + errdefer arguments.deinit(allocator); + var results_ = try exe.results(allocator); + errdefer results_.deinit(allocator); + return .{ .exe = exe, .args = arguments, .results = results_ }; + } + + pub fn deinit(self: *Runner, allocator: std.mem.Allocator) void { + self.results.deinit(allocator); + self.args.deinit(allocator); + self.exe.deinit(); + } + + pub fn run(self: *Runner, input_values: anytype, output_values: anytype) void { + self.args.set(input_values); + self.exe.call(self.args, &self.results); + self.results.fill(output_values); + } + + pub fn runOpts(self: *Runner, io: std.Io, input_values: anytype, output_values: anytype, opts: CallOpts) void { + self.args.set(input_values); + self.exe.callOpts(io, self.args, &self.results, opts); + self.results.fill(output_values); + } + }; + pub fn internalCall(self: *const Exe, io: ?std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { stdx.debug.assert(opts.wait == false or io != null, "io should not be null when waiting for execution completion", .{}); var events = [_]?*pjrt.Event{null} ** Platform.MAX_NUM_DEVICES; @@ -328,3 +367,216 @@ pub const Exe = struct { return self.internalCall(null, arguments, results_, .{}); } }; + +/// A typed executable whose input and output buffer structures are derived from +/// the function used to compile it. +pub fn TypedExe(comptime function_: anytype) type { + const function_info = switch (@typeInfo(@TypeOf(function_))) { + .@"fn" => |info| info, + else => @compileError("TypedExe expects a function, got " ++ @typeName(@TypeOf(function_))), + }; + if (function_info.is_var_args or function_info.params.len != 1) { + @compileError("TypedExe function must accept exactly one input struct"); + } + + const FunctionInput = typedFunctionInput(function_); + const FunctionOutput = typedFunctionOutput(function_); + + return struct { + raw: Exe, + + pub const Input = mem.Bufferized(FunctionInput); + pub const Output = typedOutputDestinations(mem.Bufferized(FunctionOutput)); + + const Self = @This(); + + pub fn init(raw: Exe) Self { + return .{ .raw = raw }; + } + + pub fn compile( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + opts: module.CompilationOptions, + args: std.meta.ArgsTuple(@TypeOf(function_)), + ) module.CompileError!Self { + return .{ .raw = try module.Compiler(function_).compile(allocator, io, platform, opts, args) }; + } + + pub fn deinit(self: *const Self) void { + self.raw.deinit(); + } + + /// An executable together with reusable argument and result storage. + /// A runner owns all three values and must not be used concurrently. + pub fn Runner(comptime baked_fields: anytype) type { + const count = countBackedFields(Input, baked_fields); + + return struct { + exe: Exe, + args: Exe.Arguments, + results: Exe.Results, + + const RunnerSelf = @This(); + + pub const BakedInput = structFieldRange(Input, 0, count); + pub const NonBakedInput = structFieldRange(Input, count, @typeInfo(Input).@"struct".fields.len); + + pub fn init(exe: *const Self, allocator: std.mem.Allocator, baked: BakedInput) !RunnerSelf { + var arguments = try exe.raw.args(allocator); + errdefer arguments.deinit(allocator); + var results = try exe.raw.results(allocator); + errdefer results.deinit(allocator); + arguments.bake(baked); + return .{ .exe = exe.raw, .args = arguments, .results = results }; + } + + pub fn deinit(self: *RunnerSelf, allocator: std.mem.Allocator) void { + self.results.deinit(allocator); + self.args.deinit(allocator); + } + + pub fn run(self: *RunnerSelf, io: std.Io, call: struct { + inputs: NonBakedInput, + outputs: Output, + opts: Exe.CallOpts = .{}, + }) void { + self.args.set(call.inputs); + self.exe.callOpts(io, self.args, &self.results, call.opts); + self.results.fill(call.outputs); + } + }; + } + }; +} + +fn countBackedFields(comptime Input: type, comptime baked_fields: anytype) usize { + const baked_info = switch (@typeInfo(@TypeOf(baked_fields))) { + .@"struct" => |info| info, + else => @compileError("TypedExe baked fields must be a tuple of enum literals"), + }; + if (!baked_info.is_tuple) { + @compileError("TypedExe baked fields must be a tuple of enum literals"); + } + + const input_fields = @typeInfo(Input).@"struct".fields; + if (baked_info.fields.len > input_fields.len) { + @compileError("TypedExe baked fields must be a prefix of its bufferized input fields"); + } + + inline for (baked_info.fields, 0..) |tuple_field, i| { + const baked_field = @field(baked_fields, tuple_field.name); + if (@TypeOf(baked_field) != @EnumLiteral()) { + @compileError("TypedExe baked fields must be enum literals"); + } + if (!std.mem.eql(u8, @tagName(baked_field), input_fields[i].name)) { + @compileError("TypedExe baked fields must be an ordered prefix of its bufferized input fields"); + } + } + return baked_info.fields.len; +} + +fn structFieldRange(comptime Struct: type, comptime start: usize, comptime end: usize) type { + const fields = @typeInfo(Struct).@"struct".fields[start..end]; + var field_names: [fields.len][]const u8 = undefined; + var field_types: [fields.len]type = undefined; + var field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; + for (&field_names, &field_types, &field_attrs, fields) |*name, *T, *attrs, field| { + name.* = field.name; + T.* = field.type; + attrs.* = .{ + .@"comptime" = field.is_comptime, + .@"align" = field.alignment, + .default_value_ptr = field.default_value_ptr, + }; + } + return @Struct(.auto, null, &field_names, &field_types, &field_attrs); +} + +fn typedFunctionInput(comptime function_: anytype) type { + const Input = @typeInfo(@TypeOf(function_)).@"fn".params[0].type orelse + @compileError("TypedExe function must have a concrete input type"); + validateTypedExeStruct("input", Input); + return Input; +} + +fn typedFunctionOutput(comptime function_: anytype) type { + const Output = @typeInfo(@TypeOf(function_)).@"fn".return_type orelse @compileError("TypedExe function must return an output struct"); + validateTypedExeStruct("output", Output); + return Output; +} + +fn validateTypedExeStruct(comptime role: []const u8, comptime T: type) void { + const info = switch (@typeInfo(T)) { + .@"struct" => |info| info, + else => @compileError("TypedExe " ++ role ++ " must be a struct, got " ++ @typeName(T)), + }; + if (info.is_tuple) { + @compileError("TypedExe " ++ role ++ " must use named fields"); + } +} + +fn typedOutputDestinations(comptime BufferizedOutput: type) type { + const output_info = @typeInfo(BufferizedOutput).@"struct"; + const fields = output_info.fields; + var field_names: [fields.len][]const u8 = undefined; + var field_types: [fields.len]type = undefined; + var field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; + for (&field_names, &field_types, &field_attrs, fields) |*name, *T, *attrs, field| { + name.* = field.name; + T.* = *field.type; + attrs.* = .{ .@"align" = @alignOf(*field.type) }; + } + return @Struct(.auto, null, &field_names, &field_types, &field_attrs); +} + +test "TypedExe derives baked and runtime calls from a function" { + const Inputs = struct { + weights: struct { + tensor: Tensor, + scale: f32, + }, + bias: Tensor, + tokens: Tensor, + token_count: usize, + }; + const Outputs = struct { + hidden: Tensor, + cache: struct { + key: Tensor, + length: usize, + }, + metadata: usize, + }; + const Functions = struct { + fn forward(inputs: Inputs) Outputs { + _ = inputs; + return undefined; + } + }; + + const Model = TypedExe(Functions.forward); + try std.testing.expect(@hasField(Model.Input, "weights")); + try std.testing.expect(@hasField(Model.Input, "bias")); + try std.testing.expect(@hasField(Model.Input, "tokens")); + try std.testing.expect(!@hasField(Model.Input, "token_count")); + try std.testing.expect(!@hasField(@FieldType(Model.Input, "weights"), "scale")); + + const Baked = Model.Runner(.{ .weights, .bias }).BakedInput; + try std.testing.expect(@hasField(Baked, "weights")); + try std.testing.expect(@hasField(Baked, "bias")); + try std.testing.expect(!@hasField(Baked, "tokens")); + + const Runtime = Model.Runner(.{ .weights, .bias }).NonBakedInput; + try std.testing.expect(!@hasField(Runtime, "weights")); + try std.testing.expect(!@hasField(Runtime, "bias")); + try std.testing.expectEqual(Buffer, @FieldType(Runtime, "tokens")); + + try std.testing.expectEqual(*Buffer, @FieldType(Model.Output, "hidden")); + try std.testing.expectEqual( + *mem.Bufferized(@FieldType(Outputs, "cache")), + @FieldType(Model.Output, "cache"), + ); + try std.testing.expect(!@hasField(Model.Output, "metadata")); +} diff --git a/zml/module.zig b/zml/module.zig index 15537404..d4610f27 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -57,6 +57,14 @@ pub const CompilationOptions = struct { xla_dump_emitter_re: ?[]const u8 = null, }; +/// Errors surfaced while lowering and compiling a ZML program. +pub const CompileError = std.mem.Allocator.Error || + std.Io.Writer.Error || + mlir.Error || + upb.SerializeError || + pjrtx.Client.CompileError || + error{MissingDeviceInTile}; + const AttributeList = stdx.BoundedArray(mlir.NamedAttribute, 3); pub const CompilationContext = struct { @@ -228,7 +236,7 @@ pub fn Compiler(comptime func: anytype) type { platform: *const Platform, opts: CompilationOptions, args: std.meta.ArgsTuple(@TypeOf(func)), - ) !Exe { + ) CompileError!Exe { return zml_module.compile(allocator, io, func, args, platform, opts); } }; @@ -241,7 +249,7 @@ pub fn compile( args: std.meta.ArgsTuple(@TypeOf(func)), platform: *const Platform, opts: CompilationOptions, -) !Exe { +) CompileError!Exe { // TODO: Here we have somewhat of a requirement // Emitting MLIR requires to have the compilation context available at all times using `CompilationContext.current()`. // If in the future, we inject an Io that is not thread-based, we might have some surprises. diff --git a/zml/zml.zig b/zml/zml.zig index 11a45717..6ce6a2a3 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -21,6 +21,7 @@ pub const Data = dtype.Data; pub const DataType = dtype.DataType; pub const exe = @import("exe.zig"); pub const Exe = exe.Exe; +pub const TypedExe = exe.TypedExe; pub const floats = @import("floats.zig"); pub const io = @import("io.zig"); pub const kernel = @import("kernel.zig"); From 5eff1ed501bbcff50b001382710e70267d590cec Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Wed, 19 Aug 2026 15:43:42 +0200 Subject: [PATCH 617/881] zml/moe: NVFP4 and block-FP8 Metal custom calls (#709) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Emits `__metal$moe_gemm` / `$f4` / `$f8` so a MoE layer can run bf16, NVFP4 or DeepSeek block-FP8 (`f8e4m3fn` weights + bf16 128×128 scales) `forwardMoe` takes two new optional arguments, `w1_global_scale` and `w2_global_scale`. The per-expert global (`weight_scale_2`) is applied in-graph after the GEMM. --- examples/llm/models/qwen3_5_moe/model.zig | 2 + zml/moe/metal.zig | 140 +++++++++++++++++----- zml/moe/moe.zig | 6 +- 3 files changed, 117 insertions(+), 31 deletions(-) diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index cc28fa6d..09ff549e 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -805,6 +805,8 @@ pub const Moe = struct { self.down_proj, null, null, + null, + null, moe_metadata, moe_parameters, ) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); diff --git a/zml/moe/metal.zig b/zml/moe/metal.zig index 8738904d..9f669434 100644 --- a/zml/moe/metal.zig +++ b/zml/moe/metal.zig @@ -15,6 +15,8 @@ pub const Options = struct { expert_map: ?Tensor = null, w1_scale: ?Tensor = null, w2_scale: ?Tensor = null, + w1_global_scale: ?Tensor = null, + w2_global_scale: ?Tensor = null, w1_bias: ?Tensor = null, w2_bias: ?Tensor = null, }; @@ -56,21 +58,22 @@ pub fn deinitBuffer(bufferized: *zml.Bufferized(Metadata)) void { _ = bufferized; } -fn validateOptions(opts: Options) !void { - if (opts.expert_map != null) return error.InvalidShape; - if (opts.w1_scale != null or opts.w2_scale != null) return error.UnsupportedQuantization; - if (opts.w1_bias != null or opts.w2_bias != null) return error.UnsupportedBias; +const QuantMode = enum { none, fp8, nvfp4 }; + +fn quantMode(gate_up: Tensor, down: Tensor) !QuantMode { + if (gate_up.dtype() != down.dtype()) return error.UnsupportedType; + return switch (gate_up.dtype()) { + .bf16, .f16, .f32 => .none, + .f8e4m3fn => .fp8, + .u8, .f4e2m1 => .nvfp4, + else => error.UnsupportedType, + }; } -fn validateInputs(hidden: Tensor, gate_up: Tensor, down: Tensor, weights: Tensor, ids: Tensor) !void { - for ([_]Tensor{ hidden, gate_up, down, weights }) |t| { - switch (t.dtype()) { - .bf16, .f16, .f32 => {}, - else => return error.UnsupportedType, - } - } - - if (ids.dtype() != .i32) return error.UnsupportedType; +fn requireScalePair(opts: Options, dtype: zml.DataType) !void { + const s1 = opts.w1_scale orelse return error.UnsupportedQuantization; + const s2 = opts.w2_scale orelse return error.UnsupportedQuantization; + if (s1.dtype() != dtype or s2.dtype() != dtype) return error.UnsupportedType; } fn applyActivation(x: Tensor, mode: ActivationMode) Tensor { @@ -87,6 +90,46 @@ fn applyActivation(x: Tensor, mode: ActivationMode) Tensor { return act.rename(.{ .out = .mid }); } +fn moeGemm( + x_rows: Tensor, + w: Tensor, + scale: ?Tensor, + expert_ids: Tensor, + out_shape: zml.Shape, + mode: QuantMode, +) Tensor { + const side: zml.ops.CustomCallOptions = .{ .has_side_effect = false }; + return switch (mode) { + .none => zml.ops.customCall("__metal$moe_gemm", .{ x_rows, w, expert_ids }, .{out_shape}, .{}, side), + .fp8 => zml.ops.customCall("__metal$moe_gemm$f8", .{ x_rows, w, scale.?, expert_ids }, .{out_shape}, .{}, side), + .nvfp4 => zml.ops.customCall("__metal$moe_gemm$f4", .{ x_rows, w, scale.?, expert_ids }, .{out_shape}, .{}, side), + }; +} + +fn applyGateUpGlobalScale(output: Tensor, global_scale: ?Tensor, expert_ids: Tensor) !Tensor { + const scale = global_scale orelse return output; + if (scale.dtype() != .f32 or scale.rank() != 2 or scale.dim(.proj) != 2) { + return error.InvalidShape; + } + + const mid = @divExact(output.dim(.out), 2); + const selected = scale.gather(.{ .expert = expert_ids }, .{}); + const expanded = selected.appendAxes(.{.mid}) + .broad(zml.Shape.init(.{ .r = output.dim(.r), .proj = 2, .mid = mid }, .f32)) + .merge(.{ .out = .{ .proj, .mid } }); + return output.convert(.f32).mul(expanded).convert(output.dtype()); +} + +fn applyDownGlobalScale(output: Tensor, global_scale: ?Tensor, expert_ids: Tensor) !Tensor { + const scale = global_scale orelse return output; + if (scale.dtype() != .f32 or scale.rank() != 1) return error.InvalidShape; + + const selected = scale.gather(.{ .expert = expert_ids }, .{}) + .appendAxes(.{.d}) + .broad(output.shape().withDtype(.f32)); + return output.convert(.f32).mul(selected).convert(output.dtype()); +} + pub fn fusedExpertsImpl( hidden_states: Tensor, gate_up: Tensor, @@ -97,8 +140,35 @@ pub fn fusedExpertsImpl( opts: Options, ) !Tensor { _ = metadata; - try validateOptions(opts); - try validateInputs(hidden_states, gate_up, down, topk_weights, topk_ids); + if (opts.expert_map != null) return error.InvalidShape; + if (opts.w1_bias != null or opts.w2_bias != null) return error.UnsupportedBias; + + const mode = try quantMode(gate_up, down); + switch (mode) { + .none => { + if (opts.w1_scale != null or opts.w2_scale != null) return error.UnsupportedQuantization; + if (opts.w1_global_scale != null or opts.w2_global_scale != null) return error.UnsupportedQuantization; + switch (hidden_states.dtype()) { + .bf16, .f16, .f32 => {}, + else => return error.UnsupportedType, + } + }, + .fp8 => { + try requireScalePair(opts, .bf16); + if (opts.w1_global_scale != null or opts.w2_global_scale != null) return error.UnsupportedQuantization; + if (hidden_states.dtype() != .bf16) return error.UnsupportedType; + }, + .nvfp4 => { + try requireScalePair(opts, .f8e4m3fn); + if (hidden_states.dtype() != .bf16) return error.UnsupportedType; + }, + } + + switch (topk_weights.dtype()) { + .bf16, .f16, .f32 => {}, + else => return error.UnsupportedType, + } + if (topk_ids.dtype() != .i32) return error.UnsupportedType; const b = hidden_states.dim(.b); const s = hidden_states.dim(.s); @@ -107,29 +177,39 @@ pub fn fusedExpertsImpl( const top_expert = topk_ids.dim(.top_expert); const num_routes = num_tokens * top_expert; + const act_dtype: zml.DataType = switch (mode) { + .none => hidden_states.dtype(), + .fp8, .nvfp4 => .bf16, + }; + const hidden = hidden_states.reshape(.{ .token = num_tokens, .d = d }).withTags(.{ .token, .d }); const x_rows = hidden.insertAxes(.d, .{.top_expert}) - .broad(zml.Shape.init(.{ .token = num_tokens, .top_expert = top_expert, .d = d }, hidden.dtype())) + .broad(zml.Shape.init(.{ .token = num_tokens, .top_expert = top_expert, .d = d }, act_dtype)) .merge(.{ .r = .{ .token, .top_expert } }); const expert_ids = topk_ids.reshape(.{ .token = num_tokens, .top_expert = top_expert }).withTags(.{ .token, .top_expert }) .merge(.{ .r = .{ .token, .top_expert } }).convert(.i32); - const gate_up_out = zml.ops.customCall( - "__metal$moe_gemm", - .{ x_rows, gate_up, expert_ids }, - .{zml.Shape.init(.{ .r = num_routes, .out = gate_up.dim(.dout) }, x_rows.dtype())}, - .{}, - .{ .has_side_effect = false }, - ); + const gate_up_w = if (gate_up.dtype() == .u8) zml.nn.unpackNvfp4(gate_up, .d) else gate_up; + const down_w = if (down.dtype() == .u8) zml.nn.unpackNvfp4(down, .dout) else down; + + const gate_up_out = try applyGateUpGlobalScale(moeGemm( + x_rows, + gate_up_w, + opts.w1_scale, + expert_ids, + zml.Shape.init(.{ .r = num_routes, .out = gate_up_w.dim(.dout) }, act_dtype), + mode, + ), opts.w1_global_scale, expert_ids); const activated = applyActivation(gate_up_out, opts.activation); - const down_out = zml.ops.customCall( - "__metal$moe_gemm", - .{ activated, down, expert_ids }, - .{zml.Shape.init(.{ .r = num_routes, .d = down.dim(.d) }, activated.dtype())}, - .{}, - .{ .has_side_effect = false }, - ); + const down_out = try applyDownGlobalScale(moeGemm( + activated, + down_w, + opts.w2_scale, + expert_ids, + zml.Shape.init(.{ .r = num_routes, .d = down_w.dim(.d) }, act_dtype), + mode, + ), opts.w2_global_scale, expert_ids); const weights = topk_weights.reshape(.{ .token = num_tokens, .top_expert = top_expert }).withTags(.{ .token, .top_expert }) .merge(.{ .r = .{ .token, .top_expert } }); diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index b5a943f0..9ad56b4d 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -30,7 +30,7 @@ pub const Backend = enum { else => error.UnsupportedDataType, }, .metal => switch (weights_dtype) { - .bf16, .f16, .f32 => .metal, + .bf16, .f16, .f32, .f4e2m1, .u8, .f8e4m3fn => .metal, else => error.UnsupportedDataType, }, else => error.UnimplementedMoEBackend, @@ -159,6 +159,8 @@ pub fn forwardMoe( weights_down: zml.Tensor, scales_down: ?zml.Tensor, bias_down: ?zml.Tensor, + w1_global_scale: ?zml.Tensor, + w2_global_scale: ?zml.Tensor, metadata: Metadata, parameters: Parameters, ) !zml.Tensor { @@ -334,6 +336,8 @@ pub fn forwardMoe( .global_num_experts = weights_gate_up.dim(.expert), .w1_scale = scales_gate_up, .w2_scale = scales_down, + .w1_global_scale = w1_global_scale, + .w2_global_scale = w2_global_scale, .w1_bias = bias_gate_up, .w2_bias = bias_down, }, From 19f562560dd9c2a989058f7cd868e789a78b16b0 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Wed, 19 Aug 2026 16:25:50 +0200 Subject: [PATCH 618/881] zml/nn: classify quantization schemes and serve them on Linear (#710) `QuantScheme` classifies a checkpoint's scheme from the weight and scale shapes rather than from key names, and `Linear.forward` serves it: | scheme | stored as | |---|---| | NVFP4 | `f4e2m1`, or two values per byte in `u8` | | FP8 per-channel | `f8e4m3fn` + `[N, 1]` bf16 | | FP8 block-128 | `f8e4m3fn` + `[N/128, K/128]` bf16 | --- zml/exe.zig | 9 +- zml/io.zig | 18 +++- zml/nn.zig | 252 ++++++++++++++++++++++++++++++++++++++------ zml/safetensors.zig | 1 + 4 files changed, 241 insertions(+), 39 deletions(-) diff --git a/zml/exe.zig b/zml/exe.zig index 02633f44..46938ea0 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -327,7 +327,9 @@ pub const Exe = struct { for (events_slice.?) |e| { if (e) |ev| { if (opts.wait) { - ev.await(self.platform.pjrt_api, io.?) catch unreachable; + ev.await(self.platform.pjrt_api, io.?) catch |err| { + std.debug.panic("PJRT execution failed with: {}", .{err}); + }; } ev.deinit(self.platform.pjrt_api); } @@ -336,7 +338,10 @@ pub const Exe = struct { .cpu, .cuda, .rocm, .tpu, .oneapi, .metal => if (opts.wait) { for (events_slice.?) |e| { if (e) |ev| { - ev.await(self.platform.pjrt_api, io.?) catch unreachable; + ev.await(self.platform.pjrt_api, io.?) catch |err| { + std.debug.panic("PJRT execution failed with: {}", .{err}); + }; + ev.deinit(self.platform.pjrt_api); } } }, diff --git a/zml/io.zig b/zml/io.zig index 48d7a390..e4f079ad 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -59,6 +59,15 @@ pub const TensorStore = struct { return tensor_desc_ptr; } + fn dupeSource(self: *TensorStore, key: []const u8) ?*safetensors.Tensor { + const entry = self.getPtrFromKey(key) orelse return null; + + const copy = self.arena.allocator().create(safetensors.Tensor) catch @panic("OOM"); + copy.* = entry.*; + + return copy; + } + fn getPtrFromId(self: *const TensorStore, id: usize) ?*safetensors.Tensor { const sources = self.id_to_sources.get(id) orelse return null; stdx.debug.assert(sources.len == 1, "Expect tensor with id {} to have only one source, got {}", .{ id, sources.len }); @@ -135,7 +144,7 @@ pub const TensorStore = struct { }; } - fn prefix(self: *const View) ?[]const u8 { + pub fn prefix(self: *const View) ?[]const u8 { return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; } @@ -150,7 +159,7 @@ pub const TensorStore = struct { pub fn maybeCreateTensor(self: View, subkey: []const u8, tagz: anytype, partitioning: anytype) ?Tensor { var buffer: [256]u8 = undefined; const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); - const source = self.store.getPtrFromKey(key) orelse return null; + const source = self.store.dupeSource(key) orelse return null; const sources = self.store.arena.allocator().alloc(*safetensors.Tensor, 1) catch |e| std.debug.panic("Not handling {} errors", .{e}); errdefer self.store.arena.allocator().free(sources); @@ -167,7 +176,8 @@ pub const TensorStore = struct { } pub fn createTensor(self: View, subkey: []const u8, tagz: anytype, partitioning: anytype) Tensor { - return self.maybeCreateTensor(subkey, tagz, partitioning).?; + return self.maybeCreateTensor(subkey, tagz, partitioning) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); } fn applyTags(shape_: Shape, tagz: anytype) Shape { @@ -211,7 +221,7 @@ pub const TensorStore = struct { var buffer: [256]u8 = undefined; for (sources) |subkey| { const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); - const tensor = self.store.getPtrFromKey(key) orelse return null; + const tensor = self.store.dupeSource(key) orelse return null; tensor_list.appendAssumeCapacity(tensor); } diff --git a/zml/nn.zig b/zml/nn.zig index b99ed02f..e9f7ee9c 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -19,9 +19,32 @@ pub const Linear = struct { weight: Tensor, bias: ?Tensor = null, tag: Shape.Tag, - scales: ?Tensor = null, - global_scale: ?Tensor = null, - input_global_scale: ?Tensor = null, + quant: ?Quant = null, + + pub const Quant = struct { + scheme: QuantScheme, // decided when the checkpoint is read + scales: Tensor, + weight_scale: ?TensorScale = null, + input_scale: ?TensorScale = null, + }; + + /// A per-tensor scale and the polarity its producer wrote it in + pub const TensorScale = struct { + value: Tensor, + direction: Direction, + + pub const Direction = enum { multiplier, divisor }; + + pub fn asMultiplier(self: TensorScale) Tensor { + const s = self.value.convert(.f32); + const flat = if (s.shape().count() == 1) s.asScalar() else s; // essentially normalization + + return switch (self.direction) { + .multiplier => flat, + .divisor => Tensor.scalar(1.0, .f32).broad(flat.shape()).div(flat), + }; + } + }; pub fn init(weight: Tensor, bias: ?Tensor, tag: anytype) Linear { return .{ @@ -41,40 +64,185 @@ pub const Linear = struct { } fn forwardWeight(self: Linear, x: Tensor) Tensor { - const scales = self.scales orelse return x.dot(self.weight, self.tag); + const q = self.quant orelse return x.dot(self.weight, self.tag); - const weight = if (self.weight.dtype() == .u8) - unpackNvfp4(self.weight.withTags(.{ .dout, .kw }), self.tag) - else - self.weight; + const wgs: ?Tensor = if (q.weight_scale) |s| s.asMultiplier() else null; + const igs: ?Tensor = if (q.input_scale) |s| s.asMultiplier() else null; - const igs: ?Tensor = if (self.input_global_scale) |g| g.convert(.f32).asScalar() else null; - const wgs: ?Tensor = if (self.global_scale) |g| g.convert(.f32).asScalar() else null; + const weight = if (isPackedNvfp4(q.scheme, self.weight.dtype())) unpackNvfp4(self.weight, self.tag) else self.weight; + + var lhs = x.convert(.bf16); + var lhs_scale: ?Tensor = null; + var undo: ?Tensor = null; const platform = zml.module.CompilationContext.current().platform; - if (weight.dtype() == .f4e2m1 and supportsNvfp4ActivationQuant(platform)) { - const q = quantizeNvfp4(x.convert(.bf16), igs, self.tag); - const acc = scaledDot(q.values, weight, q.scales, scales, self.tag); - return applyGlobalScale(acc, igs, wgs).convert(x.dtype()); + if (q.scheme.activationQuant()) |aq| { + if (aq.supportedOn(platform)) { + const quantized = aq.apply(lhs, igs, self.tag); + lhs = quantized.values; + lhs_scale = quantized.scales; + undo = igs; + } } - const acc = scaledDot(x.convert(.bf16), weight, null, scales, self.tag); - return applyGlobalScale(acc, null, wgs).convert(x.dtype()); + const acc = scaledDot(lhs, weight, lhs_scale, q.scales, self.tag); + return applyGlobalScale(acc, undo, wgs).convert(x.dtype()); } }; +pub const nvfp4_block_size = 16; + +pub const QuantScheme = enum { + /// f4e2m1 values (`u8`-packed or native), f8e4m3fn scale per 16 contracted values. + /// Emitted by llm-compressor and by NVIDIA's ModelOpt. + nvfp4, + /// f8e4m3fn values, one bf16 scale per output channel, constant along the contraction. + /// Emitted by llm-compressor, including for the layers an NVFP4 recipe leaves in FP8. + fp8_per_channel, + /// f8e4m3fn values, one bf16 scale per 128x128 tile. The DeepSeek-style FP8 that model + /// vendors publish themselves, under `weight_scale_inv`. + fp8_block128, + /// f8e4m3fn values, one scale for the whole tensor. Spelled `[1, 1]` rather than as a + /// scalar: XLA's composite rewriter requires the scale to have the operand's rank. + fp8_per_tensor, + + pub fn accepts(self: QuantScheme, weight: Shape, scale: Shape) bool { + if (weight.rank() != 2) return false; + + const n = weight.dim(0); + const k = if (isPackedNvfp4(self, weight.dtype())) 2 * weight.dim(1) else weight.dim(1); + + return switch (self) { + .nvfp4 => (weight.dtype() == .u8 or weight.dtype() == .f4e2m1) and + scale.dtype() == .f8e4m3fn and scale.rank() == 2 and + scale.dim(0) == n and scale.dim(1) * nvfp4_block_size == k, + .fp8_per_tensor => weight.dtype() == .f8e4m3fn and scale.count() == 1, + .fp8_per_channel => weight.dtype() == .f8e4m3fn and scale.dtype() == .bf16 and + scale.count() > 1 and scale.rank() == 2 and + scale.dim(0) == n and scale.dim(1) == 1, + .fp8_block128 => weight.dtype() == .f8e4m3fn and scale.dtype() == .bf16 and + scale.count() > 1 and scale.rank() == 2 and + @rem(n, 128) == 0 and @rem(k, 128) == 0 and + scale.dim(0) == @divExact(n, 128) and scale.dim(1) == @divExact(k, 128), + }; + } + + pub fn classify(weight: Shape, scale: Shape) ?QuantScheme { + for (std.enums.values(QuantScheme)) |scheme| { + if (scheme.accepts(weight, scale)) return scheme; + } + + return null; + } + + pub fn activationQuant(self: QuantScheme) ?ActivationQuant { + return switch (self) { + .nvfp4 => .nvfp4, + .fp8_per_channel, .fp8_block128, .fp8_per_tensor => null, + }; + } +}; + +pub const ActivationQuant = enum { + nvfp4, + + fn supportedOn(self: ActivationQuant, platform: *const zml.Platform) bool { + return switch (self) { + .nvfp4 => supportsNvfp4ActivationQuant(platform), + }; + } + + fn apply(self: ActivationQuant, x: Tensor, global_scale: ?Tensor, axis: Shape.Tag) QuantizedActivations { + return switch (self) { + .nvfp4 => quantizeNvfp4(x, global_scale, axis), + }; + } +}; + +pub fn isPackedNvfp4(scheme: ?QuantScheme, weight_dtype: DataType) bool { + return scheme == .nvfp4 and weight_dtype == .u8; +} + +// Note: This test will evolve as we support more (for example, MXFP4/8 will be added) +test "QuantScheme.classify" { + const expect = std.testing.expectEqual; + + // unsloth/Qwen3.6-27B-NVFP4: compressed-tensors, carrying both of its schemes under + // `weight_scale` -- NVFP4 on the MLPs of layers 0-55, FP8 per-channel everywhere else. + const nvfp4_packed: Shape = .init(.{ .dout = 17408, .kw = 2560 }, .u8); + try expect(@as(?QuantScheme, .nvfp4), QuantScheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 320 }, .f8e4m3fn))); + try expect(@as(?QuantScheme, .fp8_per_channel), QuantScheme.classify( + .init(.{ .dout = 17408, .d = 5120 }, .f8e4m3fn), + .init(.{ .dout = 17408, .sc = 1 }, .bf16), + )); + + // nvidia/Gemma-4-31B-IT-NVFP4: ModelOpt, packed values under the plain `weight` name. + try expect(@as(?QuantScheme, .nvfp4), QuantScheme.classify( + .init(.{ .dout = 21504, .kw = 2688 }, .u8), + .init(.{ .dout = 21504, .sc = 336 }, .f8e4m3fn), + )); + + // Native (unpacked) f4e2m1, with K no longer doubled. + try expect(@as(?QuantScheme, .nvfp4), QuantScheme.classify( + .init(.{ .dout = 17408, .d = 5120 }, .f4e2m1), + .init(.{ .dout = 17408, .sc = 320 }, .f8e4m3fn), + )); + + // Qwen/Qwen3.6-27B-FP8: block 128, spelled `weight_scale_inv`. + try expect(@as(?QuantScheme, .fp8_block128), QuantScheme.classify( + .init(.{ .dout = 17408, .d = 5120 }, .f8e4m3fn), + .init(.{ .dout = 136, .sc = 40 }, .bf16), + )); + try expect(@as(?QuantScheme, .fp8_block128), QuantScheme.classify( + .init(.{ .dout = 5120, .d = 6144 }, .f8e4m3fn), + .init(.{ .dout = 40, .sc = 48 }, .bf16), + )); + + // Mistral's per-tensor FP8: one scale for the whole tensor, rank 0 or [1]. + try expect(@as(?QuantScheme, .fp8_per_tensor), QuantScheme.classify(.init(.{ .dout = 4096, .d = 4096 }, .f8e4m3fn), .init(.{}, .f32))); + try expect(@as(?QuantScheme, .fp8_per_tensor), QuantScheme.classify(.init(.{ .dout = 4096, .d = 4096 }, .f8e4m3fn), .init(.{ .g = 1 }, .f32))); + + // Rejected: everything no backend here can express. + const fp8: Shape = .init(.{ .dout = 10240, .d = 5120 }, .f8e4m3fn); + // Group 32 (MXFP4) rather than 16. + try expect(@as(?QuantScheme, null), QuantScheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 160 }, .f8e4m3fn))); + // e8m0 scales: valid MX, nothing here fuses or expands it. + try expect(@as(?QuantScheme, null), QuantScheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 320 }, .f8e8m0))); + try expect(@as(?QuantScheme, null), QuantScheme.classify(fp8, .init(.{ .dout = 10240, .sc = 160 }, .f8e8m0))); + // Per-channel, but f32 rather than bf16. + try expect(@as(?QuantScheme, null), QuantScheme.classify(fp8, .init(.{ .dout = 10240, .sc = 1 }, .f32))); + // A block grid that is neither per-channel nor 128x128. + try expect(@as(?QuantScheme, null), QuantScheme.classify(fp8, .init(.{ .dout = 160, .sc = 80 }, .bf16))); + // Stacked MoE weights are rank 3 and go through the MoE path, not this one. + try expect(@as(?QuantScheme, null), QuantScheme.classify( + .init(.{ .expert = 128, .dout = 1408, .kw = 1408 }, .u8), + .init(.{ .expert = 128, .dout = 1408, .sc = 176 }, .f8e4m3fn), + )); + + // A one-tile block grid is indistinguishable from a per-tensor scale by shape alone; the + // `count() > 1` guard on the grids is what makes per-tensor win. + try expect(@as(?QuantScheme, .fp8_per_tensor), QuantScheme.classify( + .init(.{ .dout = 128, .d = 128 }, .f8e4m3fn), + .init(.{ .dout = 1, .sc = 1 }, .bf16), + )); +} + +/// Unpacks two f4e2m1 values per byte. `w` must be tagged with `.kw` on the packed axis, +/// which is merged with the unpacked pair and renamed to `k_tag`. pub fn unpackNvfp4(w: Tensor, k_tag: anytype) Tensor { stdx.debug.assert(w.dtype() == .u8, "unpackNvfp4 expects packed u8 weights, got {}", .{w.dtype()}); - return w.bitCast(.f4e2m1) + return w.bitCast(.f4e2m1) // bitcast inserts a tag (it respects shlo), but maybe we should simplify it .merge(.{ .kb = .{ .kw, .bitcast } }) .renameTag(.kb, Shape.toTag(k_tag)); } -pub fn quantizeNvfp4(x: Tensor, input_global_scale: ?Tensor, axis: anytype) struct { +pub const QuantizedActivations = struct { values: Tensor, scales: Tensor, -} { - const block_size = 16; +}; + +pub fn quantizeNvfp4(x: Tensor, input_global_scale: ?Tensor, axis: anytype) QuantizedActivations { + const block_size = nvfp4_block_size; const value_max = 6.0; const scale_min_normal = 0x1p-6; const scale_max = DataType.f8e4m3fn.maxValue().as(f32); @@ -178,13 +346,14 @@ pub fn scaledDot( } const lhs_scale_operand = lhs_scale orelse blk: { - var lhs_scale_shape = lhs.shape().withDtype(.bf16); - for (0..lhs.rank()) |i| { - lhs_scale_shape = lhs_scale_shape.setDim(i, 1); - } - break :blk Tensor.constantTensor(lhs_scale_shape, DataType.bf16.one().asBytes()); + break :blk Tensor.constantTensor(onesGrid(lhs.shape(), .bf16), DataType.bf16.one().asBytes()); }; + const rhs_scale_operand = if (rhs_scale.rank() != rhs.rank() and rhs_scale.shape().count() == 1) + rhs_scale.reshape(onesGrid(rhs.shape(), rhs_scale.dtype())) + else + rhs_scale; + const mlir_ctx = zml.module.CompilationContext.current().mlir_ctx; const dnums = mlir.Attribute.array(mlir_ctx, &.{ .array(mlir_ctx, &.{ @@ -197,7 +366,7 @@ pub fn scaledDot( }), }); - const operands: []const Tensor = &.{ lhs, rhs, lhs_scale_operand, rhs_scale }; + const operands: []const Tensor = &.{ lhs, rhs, lhs_scale_operand, rhs_scale_operand }; const outs = ops.composite("xla.scaled_dot", operands, &.{res_shape}, scaledDotReference, res_shape, .{ .composite_attributes = &.{.named(mlir_ctx, "dimension_numbers", dnums)}, @@ -206,6 +375,16 @@ pub fn scaledDot( return outs[0]; } +/// `shape` with every dimension collapsed to 1 +fn onesGrid(shape: Shape, dt: DataType) Shape { + var res = shape.withDtype(dt); + for (0..res.rank()) |i| { + res = res.setDim(i, 1); + } + + return res; +} + fn scaledDotReference(in: []const Tensor, out_shape: Shape) Tensor { return ops.customCall( "zml$scaled_dot_unmatched", @@ -229,13 +408,20 @@ fn supportsNvfp4ActivationQuant(platform: *const zml.Platform) bool { } fn applyGlobalScale(acc: Tensor, igs: ?Tensor, wgs: ?Tensor) Tensor { - const combined: ?Tensor = if (igs) |i| - (if (wgs) |w| i.mul(w) else i) - else - wgs; - const alpha = combined orelse return acc; - const f32_acc = acc.convert(.f32); - return f32_acc.mul(alpha.broad(f32_acc.shape())).convert(acc.dtype()); + if (igs == null and wgs == null) { + return acc; + } + + var res = acc.convert(.f32); + if (wgs) |w| { + res = res.mul(w.convert(.f32).broad(res.shape())); + } + + if (igs) |i| { + res = res.mul(i.convert(.f32).broad(res.shape())); + } + + return res.convert(acc.dtype()); } pub const TokenEmbedding = struct { diff --git a/zml/safetensors.zig b/zml/safetensors.zig index 94e8146d..9672a60f 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -757,6 +757,7 @@ fn stringToDtype(safetensor_type: []const u8) !DataType { .{ "F16", .f16 }, .{ "BF16", .bf16 }, .{ "F8_E4M3", .f8e4m3fn }, + .{ "F4_E2M1", .f4e2m1 }, .{ "I64", .i64 }, .{ "I32", .i32 }, .{ "I16", .i16 }, From 12211f376550615411de1ada7422db6a48975484 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Thu, 20 Aug 2026 10:56:32 +0200 Subject: [PATCH 619/881] zml/exe: borrowed Exe in Runner (#708) It's a bit of a mess to transfer ownership of the Exe to the runner as that needs careful handling of defers. Fundamentally there is also no particular reason why the Runner should own the Exe and it was not consistent across zml.Exe and zml.TypedExe. Now it's always borrowed. Renaming `zml.TypedExe` to `zml.FnExe` because it's shorter. I've removed all the aliases `type LayerExe = zml.TypedExe(...)` it's useless with ZLS, it's a lot easier to have `zml.FnExe()` --- examples/llm/main.zig | 6 +- examples/llm/models.zig | 5 +- examples/llm/models/lfm2/inference.zig | 40 +++++------ examples/llm/models/llama/inference.zig | 36 +++++----- examples/llm/models/qwen3_5/inference.zig | 45 ++++++------- examples/llm/models/qwen3_5_moe/inference.zig | 41 +++++------- zml/exe.zig | 67 +++++++++++-------- zml/zml.zig | 2 +- 8 files changed, 119 insertions(+), 123 deletions(-) diff --git a/examples/llm/main.zig b/examples/llm/main.zig index 6a924354..2684dbec 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -132,7 +132,9 @@ pub fn main(init: std.process.Init) !void { var tokenizer = try loadTokenizer(allocator, io, repo, &progress); defer tokenizer.deinit(); - var compiled_model = try models.LoadedModel.compile(&model, allocator, io, platform, backend, shardings, args.seqlen, &progress); + var compiled_model = try allocator.create(models.CompiledModel); + defer allocator.destroy(compiled_model); + compiled_model.* = try models.LoadedModel.compile(&model, allocator, io, platform, backend, shardings, args.seqlen, &progress); defer compiled_model.deinit(); // Load buffers after the model compilation to be sure to give enough room to the autotune. @@ -163,7 +165,7 @@ pub fn main(init: std.process.Init) !void { io, platform, tokenizer, - &compiled_model, + compiled_model, &model_buffers, ); defer llm_chat.deinit(); diff --git a/examples/llm/models.zig b/examples/llm/models.zig index 4195abec..b88a855c 100644 --- a/examples/llm/models.zig +++ b/examples/llm/models.zig @@ -88,7 +88,10 @@ pub const LoadedModel = union(ModelType) { try m.compile(allocator, io, platform, backend, shardings, seqlen, progress), ), }; - return .{ .inner = inner, .seqlen = @intCast(seqlen) }; + return .{ + .inner = inner, + .seqlen = @intCast(seqlen), + }; } }; diff --git a/examples/llm/models/lfm2/inference.zig b/examples/llm/models/lfm2/inference.zig index 43913563..a39aba0f 100644 --- a/examples/llm/models/lfm2/inference.zig +++ b/examples/llm/models/lfm2/inference.zig @@ -96,15 +96,11 @@ pub const CompiledModel = struct { pub const Inference = CompiledModel; -pub const EmbedExe = zml.TypedExe(model.TokenEmbedding.forward); -pub const LayerExe = zml.TypedExe(model.DecoderLayer.forward); -pub const SampleExe = zml.TypedExe(model.LmHead.forward); - pub const KernelExe = struct { - embed: EmbedExe, - conv: LayerExe, - self_attn: LayerExe, - sample: SampleExe, + embed: zml.FnExe(model.TokenEmbedding.forward), + conv: zml.FnExe(model.DecoderLayer.forward), + self_attn: zml.FnExe(model.DecoderLayer.forward), + sample: zml.FnExe(model.LmHead.forward), pub fn deinit(self: *const KernelExe) void { self.embed.deinit(); @@ -115,28 +111,28 @@ pub const KernelExe = struct { }; pub const KernelRunner = struct { - embed: EmbedExe.Runner(.{.embedding}), - layers: []LayerExe.Runner(.{.layer}), - sample: SampleExe.Runner(.{ .lm_head, .embed_tokens }), + embed: zml.FnExe(model.TokenEmbedding.forward).Runner(.{.embedding}), + layers: []zml.FnExe(model.DecoderLayer.forward).Runner(.{.layer}), + sample: zml.FnExe(model.LmHead.forward).Runner(.{ .lm_head, .embed_tokens }), pub fn init(allocator: std.mem.Allocator, exe: *const KernelExe, buffers: *const model.Buffers) !KernelRunner { - var embed = try EmbedExe.Runner(.{.embedding}).init(&exe.embed, allocator, .{ .embedding = buffers.embed_tokens }); + var embed = try zml.FnExe(model.TokenEmbedding.forward).Runner(.{.embedding}).init(&exe.embed, allocator, .{ .embedding = buffers.embed_tokens }); errdefer embed.deinit(allocator); - const layers = try allocator.alloc(LayerExe.Runner(.{.layer}), buffers.layers.len); + const layers = try allocator.alloc(zml.FnExe(model.DecoderLayer.forward).Runner(.{.layer}), buffers.layers.len); errdefer allocator.free(layers); var initialized_layers: usize = 0; errdefer for (layers[0..initialized_layers]) |*layer| layer.deinit(allocator); for (layers, buffers.layers) |*layer, layer_buffers| { - const layer_exe: *const LayerExe = switch (layer_buffers.operator) { + const layer_exe: *const zml.FnExe(model.DecoderLayer.forward) = switch (layer_buffers.operator) { .conv => &exe.conv, .self_attn => &exe.self_attn, }; - layer.* = try LayerExe.Runner(.{.layer}).init(layer_exe, allocator, .{ .layer = layer_buffers }); + layer.* = try zml.FnExe(model.DecoderLayer.forward).Runner(.{.layer}).init(layer_exe, allocator, .{ .layer = layer_buffers }); initialized_layers += 1; } - var sample = try SampleExe.Runner(.{ .lm_head, .embed_tokens }).init(&exe.sample, allocator, .{ + var sample = try zml.FnExe(model.LmHead.forward).Runner(.{ .lm_head, .embed_tokens }).init(&exe.sample, allocator, .{ .lm_head = buffers.lm_head, .embed_tokens = buffers.embed_tokens, }); @@ -231,14 +227,14 @@ fn compileEmbed( seqlen: u32, phase: Phase, progress: *std.Progress.Node, -) !EmbedExe { +) !zml.FnExe(model.TokenEmbedding.forward) { progress.increaseEstimatedTotalItems(1); var node = progress.start(phase.startMessage("embed_tokens"), 1); defer node.end(); const from: std.Io.Timestamp = .now(io, .awake); defer phase.logCompileDone(log, "embed_tokens", io, from); - return EmbedExe.compile(allocator, io, platform, .{ + return zml.FnExe(model.TokenEmbedding.forward).compile(allocator, io, platform, .{ .shardings = &opts.shardings.all(), .program_name = phase.programName("lfm2", "embed_tokens"), }, .{.{ @@ -257,7 +253,7 @@ fn compileLayer( comptime kind: model.OperatorKind, phase: Phase, progress: *std.Progress.Node, -) !LayerExe { +) !zml.FnExe(model.DecoderLayer.forward) { const label = switch (kind) { .conv => "conv layer", .full_attention => "attn layer", @@ -276,7 +272,7 @@ fn compileLayer( if (candidate_kind == kind) break candidate; } else unreachable; - return LayerExe.compile(allocator, io, platform, .{ + return zml.FnExe(model.DecoderLayer.forward).compile(allocator, io, platform, .{ .shardings = &opts.shardings.all(), .program_name = phase.programName("lfm2", if (kind == .conv) "conv_layer" else "attn_layer"), }, .{.{ @@ -302,14 +298,14 @@ fn compileSample( seqlen: u32, phase: Phase, progress: *std.Progress.Node, -) !SampleExe { +) !zml.FnExe(model.LmHead.forward) { progress.increaseEstimatedTotalItems(1); var node = progress.start(phase.startMessage("lm_head"), 1); defer node.end(); const from: std.Io.Timestamp = .now(io, .awake); defer phase.logCompileDone(log, "lm_head", io, from); - return SampleExe.compile(allocator, io, platform, .{ + return zml.FnExe(model.LmHead.forward).compile(allocator, io, platform, .{ .shardings = &opts.shardings.all(), .program_name = phase.programName("lfm2", "lm_head"), }, .{.{ diff --git a/examples/llm/models/llama/inference.zig b/examples/llm/models/llama/inference.zig index 7795de78..53a655ac 100644 --- a/examples/llm/models/llama/inference.zig +++ b/examples/llm/models/llama/inference.zig @@ -110,14 +110,10 @@ pub const CompiledModel = struct { pub const Inference = CompiledModel; -pub const EmbedExe = zml.TypedExe(model.EmbedTokens.forward); -pub const LayerExe = zml.TypedExe(model.TransformerLayer.forward); -pub const SampleExe = zml.TypedExe(model.LmHead.forward); - pub const KernelExe = struct { - embed: EmbedExe, - layer: LayerExe, - sample: SampleExe, + embed: zml.FnExe(model.EmbedTokens.forward), + layer: zml.FnExe(model.TransformerLayer.forward), + sample: zml.FnExe(model.LmHead.forward), pub fn deinit(self: *const KernelExe) void { self.embed.deinit(); @@ -127,26 +123,26 @@ pub const KernelExe = struct { }; pub const KernelRunner = struct { - embed: EmbedExe.Runner(.{.embedding}), - layers: []LayerExe.Runner(.{.layer}), - sample: SampleExe.Runner(.{.lm_head}), + embed: zml.FnExe(model.EmbedTokens.forward).Runner(.{.embedding}), + layers: []zml.FnExe(model.TransformerLayer.forward).Runner(.{.layer}), + sample: zml.FnExe(model.LmHead.forward).Runner(.{.lm_head}), pub fn init(allocator: std.mem.Allocator, exe: *const KernelExe, buffers: *const model.Buffers) !KernelRunner { - var embed = try EmbedExe.Runner(.{.embedding}).init(&exe.embed, allocator, .{ + var embed = try zml.FnExe(model.EmbedTokens.forward).Runner(.{.embedding}).init(&exe.embed, allocator, .{ .embedding = .{ .embed_tokens = buffers.model.embed_tokens }, }); errdefer embed.deinit(allocator); - const layers = try allocator.alloc(LayerExe.Runner(.{.layer}), buffers.model.layers.len); + const layers = try allocator.alloc(zml.FnExe(model.TransformerLayer.forward).Runner(.{.layer}), buffers.model.layers.len); errdefer allocator.free(layers); var initialized_layers: usize = 0; errdefer for (layers[0..initialized_layers]) |*layer| layer.deinit(allocator); for (layers, buffers.model.layers) |*layer, layer_buffers| { - layer.* = try LayerExe.Runner(.{.layer}).init(&exe.layer, allocator, .{ .layer = layer_buffers }); + layer.* = try zml.FnExe(model.TransformerLayer.forward).Runner(.{.layer}).init(&exe.layer, allocator, .{ .layer = layer_buffers }); initialized_layers += 1; } - var sample = try SampleExe.Runner(.{.lm_head}).init(&exe.sample, allocator, .{ + var sample = try zml.FnExe(model.LmHead.forward).Runner(.{.lm_head}).init(&exe.sample, allocator, .{ .lm_head = .{ .lm_head = buffers.lm_head, .embed_tokens = buffers.model.embed_tokens, @@ -234,7 +230,7 @@ fn compileEmbed( seqlen: usize, phase: Phase, progress: *std.Progress.Node, -) !EmbedExe { +) !zml.FnExe(model.EmbedTokens.forward) { progress.increaseEstimatedTotalItems(1); var node = progress.start(phase.startMessage("embed_tokens"), 1); defer node.end(); @@ -244,7 +240,7 @@ fn compileEmbed( const tokens: zml.Tensor = .init(.{ .s = seqlen }, .u32); - return EmbedExe.compile(allocator, io, platform, .{ + return zml.FnExe(model.EmbedTokens.forward).compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("llama", "embed_tokens"), }, .{.{ @@ -263,7 +259,7 @@ fn compileLayer( attention_parameters: zml.attention.Parameters, phase: Phase, progress: *std.Progress.Node, -) !LayerExe { +) !zml.FnExe(model.TransformerLayer.forward) { progress.increaseEstimatedTotalItems(1); var node = progress.start(phase.startMessage("transformer layer"), 1); @@ -279,7 +275,7 @@ fn compileLayer( const kv_cache_index: zml.Tensor = .init(.{}, .u32); - return LayerExe.compile( + return zml.FnExe(model.TransformerLayer.forward).compile( allocator, io, platform, @@ -308,7 +304,7 @@ fn compileSample( seqlen: usize, phase: Phase, progress: *std.Progress.Node, -) !SampleExe { +) !zml.FnExe(model.LmHead.forward) { progress.increaseEstimatedTotalItems(1); var node = progress.start(phase.startMessage("lm_head"), 1); @@ -324,7 +320,7 @@ fn compileSample( const tokens: zml.Tensor = .init(.{ .s = seqlen }, .u32); - return SampleExe.compile(allocator, io, platform, .{ + return zml.FnExe(model.LmHead.forward).compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("llama", "lm_head"), }, .{.{ diff --git a/examples/llm/models/qwen3_5/inference.zig b/examples/llm/models/qwen3_5/inference.zig index 4a2a9331..2375081a 100644 --- a/examples/llm/models/qwen3_5/inference.zig +++ b/examples/llm/models/qwen3_5/inference.zig @@ -75,16 +75,11 @@ pub const CompiledModel = struct { pub const Inference = CompiledModel; -pub const EmbedExe = zml.TypedExe(model.EmbedTokens.forward); -pub const FullAttentionExe = zml.TypedExe(model.TransformerLayer.forwardSelfAttn); -pub const LinearAttentionExe = zml.TypedExe(model.TransformerLayer.forwardLinearAttn); -pub const SampleExe = zml.TypedExe(model.Sampler.sampleTokens); - pub const KernelExe = struct { - embed: EmbedExe, - full_attention: FullAttentionExe, - linear_attention: LinearAttentionExe, - sample: SampleExe, + embed: zml.FnExe(model.EmbedTokens.forward), + full_attention: zml.FnExe(model.TransformerLayer.forwardSelfAttn), + linear_attention: zml.FnExe(model.TransformerLayer.forwardLinearAttn), + sample: zml.FnExe(model.Sampler.sampleTokens), pub fn deinit(self: *const KernelExe) void { self.embed.deinit(); @@ -95,8 +90,8 @@ pub const KernelExe = struct { }; pub const LayerRunner = union(enum) { - full_attention: FullAttentionExe.Runner(.{.layer}), - linear_attention: LinearAttentionExe.Runner(.{.layer}), + full_attention: zml.FnExe(model.TransformerLayer.forwardSelfAttn).Runner(.{.layer}), + linear_attention: zml.FnExe(model.TransformerLayer.forwardLinearAttn).Runner(.{.layer}), pub fn deinit(self: *LayerRunner, allocator: std.mem.Allocator) void { switch (self.*) { @@ -106,12 +101,12 @@ pub const LayerRunner = union(enum) { }; pub const KernelRunner = struct { - embed: EmbedExe.Runner(.{.embedding}), + embed: zml.FnExe(model.EmbedTokens.forward).Runner(.{.embedding}), layers: []LayerRunner, - sample: SampleExe.Runner(.{.sampler}), + sample: zml.FnExe(model.Sampler.sampleTokens).Runner(.{.sampler}), pub fn init(allocator: std.mem.Allocator, exe: *const KernelExe, buffers: *const model.Buffers) !KernelRunner { - var embed = try EmbedExe.Runner(.{.embedding}).init(&exe.embed, allocator, .{ + var embed = try zml.FnExe(model.EmbedTokens.forward).Runner(.{.embedding}).init(&exe.embed, allocator, .{ .embedding = .{ .embed_tokens = buffers.text_model.embed_tokens }, }); errdefer embed.deinit(allocator); @@ -122,12 +117,12 @@ pub const KernelRunner = struct { errdefer for (layers[0..initialized_layers]) |*layer| layer.deinit(allocator); for (layers, buffers.text_model.layers) |*layer, layer_buffers| { layer.* = switch (layer_buffers.attn) { - .full_attention => .{ .full_attention = try FullAttentionExe.Runner(.{.layer}).init( + .full_attention => .{ .full_attention = try zml.FnExe(model.TransformerLayer.forwardSelfAttn).Runner(.{.layer}).init( &exe.full_attention, allocator, .{ .layer = layer_buffers }, ) }, - .linear_attention => .{ .linear_attention = try LinearAttentionExe.Runner(.{.layer}).init( + .linear_attention => .{ .linear_attention = try zml.FnExe(model.TransformerLayer.forwardLinearAttn).Runner(.{.layer}).init( &exe.linear_attention, allocator, .{ .layer = layer_buffers }, @@ -136,7 +131,7 @@ pub const KernelRunner = struct { initialized_layers += 1; } - var sample = try SampleExe.Runner(.{.sampler}).init(&exe.sample, allocator, .{ + var sample = try zml.FnExe(model.Sampler.sampleTokens).Runner(.{.sampler}).init(&exe.sample, allocator, .{ .sampler = .{ .norm = buffers.text_model.norm, .lm_head = buffers.lm_head, @@ -242,25 +237,25 @@ fn compileKernel( return .{ .embed = embed, .full_attention = full_attention, .linear_attention = linear_attention, .sample = sample }; } -fn compileEmbed(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationOptions, seqlen: usize, phase: Phase, progress: *std.Progress.Node) !EmbedExe { +fn compileEmbed(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationOptions, seqlen: usize, phase: Phase, progress: *std.Progress.Node) !zml.FnExe(model.EmbedTokens.forward) { progress.increaseEstimatedTotalItems(1); var node = progress.start(phase.startMessage("embed tokens"), 1); defer node.end(); const from: std.Io.Timestamp = .now(io, .awake); defer phase.logCompileDone(log, "embed tokens", io, from); - return EmbedExe.compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("qwen3_5", "embed_tokens") }, .{.{ + return zml.FnExe(model.EmbedTokens.forward).compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("qwen3_5", "embed_tokens") }, .{.{ .embedding = .{ .embed_tokens = mdl.text_model.embed_tokens }, .tokens = zml.Tensor.init(.{ .b = 1, .s = seqlen }, .u32), }}); } -fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationOptions, seqlen: usize, layer_index: usize, phase: Phase, progress: *std.Progress.Node) !FullAttentionExe { +fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationOptions, seqlen: usize, layer_index: usize, phase: Phase, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardSelfAttn) { progress.increaseEstimatedTotalItems(1); var node = progress.start(phase.startMessage("full attention layer"), 1); defer node.end(); const from: std.Io.Timestamp = .now(io, .awake); defer phase.logCompileDone(log, "full attention layer", io, from); - return FullAttentionExe.compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("qwen3_5", "full_attention_layer") }, .{.{ + return zml.FnExe(model.TransformerLayer.forwardSelfAttn).compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("qwen3_5", "full_attention_layer") }, .{.{ .layer = mdl.text_model.layers[layer_index], .hidden = hiddenTensor(mdl, seqlen), .token_index = parameters.token_index, @@ -272,13 +267,13 @@ fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *con }}); } -fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationOptions, seqlen: usize, layer_index: usize, phase: Phase, progress: *std.Progress.Node) !LinearAttentionExe { +fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationOptions, seqlen: usize, layer_index: usize, phase: Phase, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardLinearAttn) { progress.increaseEstimatedTotalItems(1); var node = progress.start(phase.startMessage("linear attention layer"), 1); defer node.end(); const from: std.Io.Timestamp = .now(io, .awake); defer phase.logCompileDone(log, "linear attention layer", io, from); - return LinearAttentionExe.compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("qwen3_5", "linear_attention_layer") }, .{.{ + return zml.FnExe(model.TransformerLayer.forwardLinearAttn).compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("qwen3_5", "linear_attention_layer") }, .{.{ .layer = mdl.text_model.layers[layer_index], .hidden = hiddenTensor(mdl, seqlen), .token_index = parameters.token_index, @@ -290,13 +285,13 @@ fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *c }}); } -fn compileSample(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationOptions, seqlen: usize, phase: Phase, progress: *std.Progress.Node) !SampleExe { +fn compileSample(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationOptions, seqlen: usize, phase: Phase, progress: *std.Progress.Node) !zml.FnExe(model.Sampler.sampleTokens) { progress.increaseEstimatedTotalItems(1); var node = progress.start(phase.startMessage("sampler"), 1); defer node.end(); const from: std.Io.Timestamp = .now(io, .awake); defer phase.logCompileDone(log, "sampler", io, from); - return SampleExe.compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("qwen3_5", "sampler") }, .{.{ + return zml.FnExe(model.Sampler.sampleTokens).compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("qwen3_5", "sampler") }, .{.{ .sampler = mdl.sampler(), .hidden = hiddenTensor(mdl, seqlen), .rng = parameters.rng, diff --git a/examples/llm/models/qwen3_5_moe/inference.zig b/examples/llm/models/qwen3_5_moe/inference.zig index ed95d29a..5b568d23 100644 --- a/examples/llm/models/qwen3_5_moe/inference.zig +++ b/examples/llm/models/qwen3_5_moe/inference.zig @@ -84,16 +84,11 @@ pub const CompiledModel = struct { pub const Inference = CompiledModel; -pub const EmbedExe = zml.TypedExe(model.EmbedTokens.forward); -pub const FullAttentionExe = zml.TypedExe(model.TransformerLayer.forwardSelfAttn); -pub const LinearAttentionExe = zml.TypedExe(model.TransformerLayer.forwardLinearAttn); -pub const SampleExe = zml.TypedExe(model.Sampler.sampleTokens); - pub const KernelExe = struct { - embed: EmbedExe, - full_attention: FullAttentionExe, - linear_attention: LinearAttentionExe, - sample: SampleExe, + embed: zml.FnExe(model.EmbedTokens.forward), + full_attention: zml.FnExe(model.TransformerLayer.forwardSelfAttn), + linear_attention: zml.FnExe(model.TransformerLayer.forwardLinearAttn), + sample: zml.FnExe(model.Sampler.sampleTokens), pub fn deinit(self: *const KernelExe) void { self.embed.deinit(); @@ -104,8 +99,8 @@ pub const KernelExe = struct { }; pub const LayerRunner = union(enum) { - full_attention: FullAttentionExe.Runner(.{.layer}), - linear_attention: LinearAttentionExe.Runner(.{.layer}), + full_attention: zml.FnExe(model.TransformerLayer.forwardSelfAttn).Runner(.{.layer}), + linear_attention: zml.FnExe(model.TransformerLayer.forwardLinearAttn).Runner(.{.layer}), pub fn deinit(self: *LayerRunner, allocator: std.mem.Allocator) void { switch (self.*) { @@ -115,12 +110,12 @@ pub const LayerRunner = union(enum) { }; pub const KernelRunner = struct { - embed: EmbedExe.Runner(.{.embedding}), + embed: zml.FnExe(model.EmbedTokens.forward).Runner(.{.embedding}), layers: []LayerRunner, - sample: SampleExe.Runner(.{.sampler}), + sample: zml.FnExe(model.Sampler.sampleTokens).Runner(.{.sampler}), pub fn init(allocator: std.mem.Allocator, exe: *const KernelExe, buffers: *const model.Buffers) !KernelRunner { - var embed = try EmbedExe.Runner(.{.embedding}).init(&exe.embed, allocator, .{ + var embed = try zml.FnExe(model.EmbedTokens.forward).Runner(.{.embedding}).init(&exe.embed, allocator, .{ .embedding = buffers.text_model.embed_tokens, }); errdefer embed.deinit(allocator); @@ -131,12 +126,12 @@ pub const KernelRunner = struct { errdefer for (layers[0..initialized_layers]) |*layer| layer.deinit(allocator); for (layers, buffers.text_model.layers) |*layer, layer_buffers| { layer.* = switch (layer_buffers.attn) { - .self_attn => .{ .full_attention = try FullAttentionExe.Runner(.{.layer}).init( + .self_attn => .{ .full_attention = try zml.FnExe(model.TransformerLayer.forwardSelfAttn).Runner(.{.layer}).init( &exe.full_attention, allocator, .{ .layer = layer_buffers }, ) }, - .linear_attn => .{ .linear_attention = try LinearAttentionExe.Runner(.{.layer}).init( + .linear_attn => .{ .linear_attention = try zml.FnExe(model.TransformerLayer.forwardLinearAttn).Runner(.{.layer}).init( &exe.linear_attention, allocator, .{ .layer = layer_buffers }, @@ -145,7 +140,7 @@ pub const KernelRunner = struct { initialized_layers += 1; } - var sample = try SampleExe.Runner(.{.sampler}).init(&exe.sample, allocator, .{ + var sample = try zml.FnExe(model.Sampler.sampleTokens).Runner(.{.sampler}).init(&exe.sample, allocator, .{ .sampler = .{ .norm = buffers.text_model.norm, .lm_head = buffers.text_model.lm_head, @@ -277,14 +272,14 @@ fn compileKernel( }; } -fn compileEmbed(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, phase: []const u8, progress: *std.Progress.Node) !EmbedExe { +fn compileEmbed(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.EmbedTokens.forward) { return compileExe(allocator, io, platform, model.EmbedTokens.forward, .{.{ .embedding = mdl.text_model.embed_tokens, .tokens = zml.Tensor.init(.{ .b = 1, .s = seqlen }, .u32), }}, parameters, progress, phase, "embedding"); } -fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, layer_index: usize, moe_metadata: zml.moe.Metadata, phase: []const u8, progress: *std.Progress.Node) !FullAttentionExe { +fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, layer_index: usize, moe_metadata: zml.moe.Metadata, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardSelfAttn) { return compileExe(allocator, io, platform, model.TransformerLayer.forwardSelfAttn, .{.{ .layer = mdl.text_model.layers[layer_index], .hidden = hiddenTensor(mdl, seqlen), @@ -300,7 +295,7 @@ fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *con }}, parameters, progress, phase, "full-attention layer"); } -fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, layer_index: usize, moe_metadata: zml.moe.Metadata, phase: []const u8, progress: *std.Progress.Node) !LinearAttentionExe { +fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, layer_index: usize, moe_metadata: zml.moe.Metadata, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardLinearAttn) { return compileExe(allocator, io, platform, model.TransformerLayer.forwardLinearAttn, .{.{ .layer = mdl.text_model.layers[layer_index], .hidden = hiddenTensor(mdl, seqlen), @@ -316,7 +311,7 @@ fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *c }}, parameters, progress, phase, "linear-attention layer"); } -fn compileSample(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, phase: []const u8, progress: *std.Progress.Node) !SampleExe { +fn compileSample(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.Sampler.sampleTokens) { return compileExe(allocator, io, platform, model.Sampler.sampleTokens, .{.{ .sampler = mdl.text_model.sampler(), .hidden = hiddenTensor(mdl, seqlen), @@ -335,7 +330,7 @@ fn compileExe( progress: *std.Progress.Node, phase: []const u8, component: []const u8, -) !zml.TypedExe(function) { +) !zml.FnExe(function) { progress.increaseEstimatedTotalItems(1); const label = try std.fmt.allocPrint(allocator, "Compiling {s} {s}...", .{ phase, component }); defer allocator.free(label); @@ -343,7 +338,7 @@ fn compileExe( defer node.end(); const now: std.Io.Timestamp = .now(io, .awake); defer log.info("Compiled {s} {s} [{f}]", .{ phase, component, now.untilNow(io, .awake) }); - return zml.TypedExe(function).compile(allocator, io, platform, .{ + return zml.FnExe(function).compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .xla_dump_to = parameters.xla_dump_to, }, args); diff --git a/zml/exe.zig b/zml/exe.zig index 46938ea0..49a64b04 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -262,27 +262,32 @@ pub const Exe = struct { } }; - /// An executable together with reusable argument and result storage. - /// A runner owns all three values and must not be used concurrently. + pub fn runner(self: *const Exe, allocator: std.mem.Allocator) !Runner { + return .init(self, allocator); + } + + /// Reusable argument and result storage for a borrowed executable. + /// The executable must outlive the runner, and the runner must not be used concurrently. pub const Runner = struct { - exe: Exe, + exe: *const Exe, args: Arguments, results: Results, - /// Takes ownership of the executable and allocates reusable argument and result storage. - pub fn init(exe: Exe, allocator: std.mem.Allocator) !Runner { - errdefer exe.deinit(); + pub fn init(exe: *const Exe, allocator: std.mem.Allocator) !Runner { var arguments = try exe.args(allocator); errdefer arguments.deinit(allocator); var results_ = try exe.results(allocator); errdefer results_.deinit(allocator); - return .{ .exe = exe, .args = arguments, .results = results_ }; + return .{ + .exe = exe, + .args = arguments, + .results = results_, + }; } pub fn deinit(self: *Runner, allocator: std.mem.Allocator) void { self.results.deinit(allocator); self.args.deinit(allocator); - self.exe.deinit(); } pub fn run(self: *Runner, input_values: anytype, output_values: anytype) void { @@ -375,13 +380,13 @@ pub const Exe = struct { /// A typed executable whose input and output buffer structures are derived from /// the function used to compile it. -pub fn TypedExe(comptime function_: anytype) type { +pub fn FnExe(comptime function_: anytype) type { const function_info = switch (@typeInfo(@TypeOf(function_))) { .@"fn" => |info| info, - else => @compileError("TypedExe expects a function, got " ++ @typeName(@TypeOf(function_))), + else => @compileError("FnExe expects a function, got " ++ @typeName(@TypeOf(function_))), }; if (function_info.is_var_args or function_info.params.len != 1) { - @compileError("TypedExe function must accept exactly one input struct"); + @compileError("FnExe function must accept exactly one input struct"); } const FunctionInput = typedFunctionInput(function_); @@ -413,13 +418,17 @@ pub fn TypedExe(comptime function_: anytype) type { self.raw.deinit(); } - /// An executable together with reusable argument and result storage. - /// A runner owns all three values and must not be used concurrently. + pub fn runner(self: *const Self, allocator: std.mem.Allocator) !Runner(.{}) { + return .init(self, allocator, .{}); + } + + /// Reusable argument and result storage for a borrowed executable. + /// The executable must outlive the runner, and the runner must not be used concurrently. pub fn Runner(comptime baked_fields: anytype) type { const count = countBackedFields(Input, baked_fields); return struct { - exe: Exe, + exe: *const Exe, args: Exe.Arguments, results: Exe.Results, @@ -434,7 +443,7 @@ pub fn TypedExe(comptime function_: anytype) type { var results = try exe.raw.results(allocator); errdefer results.deinit(allocator); arguments.bake(baked); - return .{ .exe = exe.raw, .args = arguments, .results = results }; + return .{ .exe = &exe.raw, .args = arguments, .results = results }; } pub fn deinit(self: *RunnerSelf, allocator: std.mem.Allocator) void { @@ -459,24 +468,24 @@ pub fn TypedExe(comptime function_: anytype) type { fn countBackedFields(comptime Input: type, comptime baked_fields: anytype) usize { const baked_info = switch (@typeInfo(@TypeOf(baked_fields))) { .@"struct" => |info| info, - else => @compileError("TypedExe baked fields must be a tuple of enum literals"), + else => @compileError("FnExe baked fields must be a tuple of enum literals"), }; if (!baked_info.is_tuple) { - @compileError("TypedExe baked fields must be a tuple of enum literals"); + @compileError("FnExe baked fields must be a tuple of enum literals"); } const input_fields = @typeInfo(Input).@"struct".fields; if (baked_info.fields.len > input_fields.len) { - @compileError("TypedExe baked fields must be a prefix of its bufferized input fields"); + @compileError("FnExe baked fields must be a prefix of its bufferized input fields"); } inline for (baked_info.fields, 0..) |tuple_field, i| { const baked_field = @field(baked_fields, tuple_field.name); if (@TypeOf(baked_field) != @EnumLiteral()) { - @compileError("TypedExe baked fields must be enum literals"); + @compileError("FnExe baked fields must be enum literals"); } if (!std.mem.eql(u8, @tagName(baked_field), input_fields[i].name)) { - @compileError("TypedExe baked fields must be an ordered prefix of its bufferized input fields"); + @compileError("FnExe baked fields must be an ordered prefix of its bufferized input fields"); } } return baked_info.fields.len; @@ -501,24 +510,24 @@ fn structFieldRange(comptime Struct: type, comptime start: usize, comptime end: fn typedFunctionInput(comptime function_: anytype) type { const Input = @typeInfo(@TypeOf(function_)).@"fn".params[0].type orelse - @compileError("TypedExe function must have a concrete input type"); - validateTypedExeStruct("input", Input); + @compileError("FnExe function must have a concrete input type"); + validateFnExeStruct("input", Input); return Input; } fn typedFunctionOutput(comptime function_: anytype) type { - const Output = @typeInfo(@TypeOf(function_)).@"fn".return_type orelse @compileError("TypedExe function must return an output struct"); - validateTypedExeStruct("output", Output); + const Output = @typeInfo(@TypeOf(function_)).@"fn".return_type orelse @compileError("FnExe function must return an output struct"); + validateFnExeStruct("output", Output); return Output; } -fn validateTypedExeStruct(comptime role: []const u8, comptime T: type) void { +fn validateFnExeStruct(comptime role: []const u8, comptime T: type) void { const info = switch (@typeInfo(T)) { .@"struct" => |info| info, - else => @compileError("TypedExe " ++ role ++ " must be a struct, got " ++ @typeName(T)), + else => @compileError("FnExe " ++ role ++ " must be a struct, got " ++ @typeName(T)), }; if (info.is_tuple) { - @compileError("TypedExe " ++ role ++ " must use named fields"); + @compileError("FnExe " ++ role ++ " must use named fields"); } } @@ -536,7 +545,7 @@ fn typedOutputDestinations(comptime BufferizedOutput: type) type { return @Struct(.auto, null, &field_names, &field_types, &field_attrs); } -test "TypedExe derives baked and runtime calls from a function" { +test "FnExe derives baked and runtime calls from a function" { const Inputs = struct { weights: struct { tensor: Tensor, @@ -561,7 +570,7 @@ test "TypedExe derives baked and runtime calls from a function" { } }; - const Model = TypedExe(Functions.forward); + const Model = FnExe(Functions.forward); try std.testing.expect(@hasField(Model.Input, "weights")); try std.testing.expect(@hasField(Model.Input, "bias")); try std.testing.expect(@hasField(Model.Input, "tokens")); diff --git a/zml/zml.zig b/zml/zml.zig index 6ce6a2a3..c88fa6ef 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -21,7 +21,7 @@ pub const Data = dtype.Data; pub const DataType = dtype.DataType; pub const exe = @import("exe.zig"); pub const Exe = exe.Exe; -pub const TypedExe = exe.TypedExe; +pub const FnExe = exe.FnExe; pub const floats = @import("floats.zig"); pub const io = @import("io.zig"); pub const kernel = @import("kernel.zig"); From 533afb89150fb18735a44a1a2bb7c67f534cd85f Mon Sep 17 00:00:00 2001 From: Louis Lechevalier Date: Thu, 20 Aug 2026 11:51:32 +0200 Subject: [PATCH 620/881] zml/moe: flashinfer nvfp4 x nvfp4 moe backend (#707) This PR add the support of nvfp4 x nvfp4 moe backend on sm120 / sm100 (natively supported nvfp4 hardware). It : - retrieves the released artifact cutlass moe flashinfer : https://github.com/zml/flashinfer/releases/tag/cutlass-moe-v0.1.0 - loads the .so - implements the nvfp4 x nvfp4 backend (added to existing triton and mosaic tpu). --- MODULE.bazel | 22 +- examples/llm/models/qwen3_5_moe/inference.zig | 2 +- pjrt/ffi.zig | 1 + .../cuda/flashinfer_cutlass_moe/BUILD.bazel | 48 ++ .../flashinfer_cutlass_moe.zig | 151 ++++ .../flashinfer_cutlass_moe/BUILD.bazel | 4 + .../flashinfer_cutlass_moe.BUILD.bazel | 33 + third_party/flashinfer_cutlass_moe/repo.bzl | 38 + third_party/non_module_deps.bzl | 2 + zml/BUILD.bazel | 2 + zml/attention/paged_attention.zig | 2 + zml/moe/cutlass_flashinfer.zig | 725 ++++++++++++++++++ zml/moe/moe.zig | 236 +++++- zml/platform.zig | 56 ++ 14 files changed, 1312 insertions(+), 10 deletions(-) create mode 100644 platforms/cuda/flashinfer_cutlass_moe/BUILD.bazel create mode 100644 platforms/cuda/flashinfer_cutlass_moe/flashinfer_cutlass_moe.zig create mode 100644 third_party/flashinfer_cutlass_moe/BUILD.bazel create mode 100644 third_party/flashinfer_cutlass_moe/flashinfer_cutlass_moe.BUILD.bazel create mode 100644 third_party/flashinfer_cutlass_moe/repo.bzl create mode 100644 zml/moe/cutlass_flashinfer.zig diff --git a/MODULE.bazel b/MODULE.bazel index b2cfce40..21e88df8 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -248,7 +248,27 @@ metal = use_extension("//platforms/metal:metal.bzl", "metal_packages") use_repo(metal, "libpjrt_metal") non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") -use_repo(non_module_deps, "arocc", "cloud_accelerator_diagnostics", "com_google_sentencepiece", "flashattn_linux_amd64", "flashattn_linux_arm64", "iree", "libvaxis", "linenoise", "mnist", "mosaic_tpu", "org_swig_swig", "stb", "translate-c", "uucode", "xla", "zigimg") +use_repo( + non_module_deps, + "arocc", + "cloud_accelerator_diagnostics", + "com_google_sentencepiece", + "flashattn_linux_amd64", + "flashattn_linux_arm64", + "flashinfer_cutlass_moe_linux_amd64", + "flashinfer_cutlass_moe_linux_arm64", + "iree", + "libvaxis", + "linenoise", + "mnist", + "mosaic_tpu", + "org_swig_swig", + "stb", + "translate-c", + "uucode", + "xla", + "zigimg", +) xla = use_extension("//third_party/xla:xla.bzl", "xla") use_repo( diff --git a/examples/llm/models/qwen3_5_moe/inference.zig b/examples/llm/models/qwen3_5_moe/inference.zig index 5b568d23..464f6511 100644 --- a/examples/llm/models/qwen3_5_moe/inference.zig +++ b/examples/llm/models/qwen3_5_moe/inference.zig @@ -409,6 +409,6 @@ fn initMoeMetadata(qwen_model: model.Model, token_len: usize, batch_size: u32, b .w2_zero_bias_shape = w2_zero_bias_shape, }, }), - .mosaic_tpu, .metal => .init(.fromBackend(backend)), + .flashinfer_cutlass, .mosaic_tpu, .metal => .init(.fromBackend(backend)), }; } diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index 83fad4d2..0c160321 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -252,6 +252,7 @@ pub const DataType = enum(c.XLA_FFI_DataType) { f8e4m3b11fnuz = c.XLA_FFI_DataType_F8E4M3B11FNUZ, f8e5m2fnuz = c.XLA_FFI_DataType_F8E5M2FNUZ, f8e4m3fnuz = c.XLA_FFI_DataType_F8E4M3FNUZ, + f4e2m1 = c.XLA_FFI_DataType_F4E2M1FN, }; pub const Buffer = extern struct { diff --git a/platforms/cuda/flashinfer_cutlass_moe/BUILD.bazel b/platforms/cuda/flashinfer_cutlass_moe/BUILD.bazel new file mode 100644 index 00000000..8a61e394 --- /dev/null +++ b/platforms/cuda/flashinfer_cutlass_moe/BUILD.bazel @@ -0,0 +1,48 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +_DEPS = [ + "//bazel", + "@rules_zig//zig/runfiles", +] + select({ + "@llvm//platforms/config:linux_x86_64": [ + "@flashinfer_cutlass_moe_linux_amd64//:c", + ], + "@llvm//platforms/config:linux_aarch64": [ + "@flashinfer_cutlass_moe_linux_arm64//:c", + ], + "//conditions:default": [], +}) + +_DATA = select({ + "@llvm//platforms/config:linux_x86_64": [ + "@flashinfer_cutlass_moe_linux_amd64//:libraries", + ], + "@llvm//platforms/config:linux_aarch64": [ + "@flashinfer_cutlass_moe_linux_arm64//:libraries", + ], + "//conditions:default": [], +}) + +zig_library( + name = "flashinfer_cutlass_moe", + data = _DATA, + import_name = "platforms/cuda/flashinfer_cutlass_moe", + main = "flashinfer_cutlass_moe.zig", + tags = ["manual"], + target_compatible_with = [ + "@platforms//os:linux", + ], + visibility = ["//visibility:public"], + deps = _DEPS, +) + +zig_test( + name = "test", + data = _DATA, + main = "flashinfer_cutlass_moe.zig", + tags = ["manual"], + target_compatible_with = [ + "@platforms//os:linux", + ], + deps = _DEPS, +) diff --git a/platforms/cuda/flashinfer_cutlass_moe/flashinfer_cutlass_moe.zig b/platforms/cuda/flashinfer_cutlass_moe/flashinfer_cutlass_moe.zig new file mode 100644 index 00000000..a8d01ea8 --- /dev/null +++ b/platforms/cuda/flashinfer_cutlass_moe/flashinfer_cutlass_moe.zig @@ -0,0 +1,151 @@ +const std = @import("std"); + +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); +const builtin = @import("builtin"); +pub const c = @import("c"); + +pub const Status = c.zml_fi_cutlass_moe_status; +pub const DType = c.zml_fi_cutlass_moe_dtype; +pub const Activation = c.zml_fi_cutlass_moe_activation; +pub const RunnerOptions = c.zml_fi_cutlass_moe_runner_options; +pub const Context = c.zml_fi_cutlass_moe_context; +pub const Io = c.zml_fi_cutlass_moe_io; +pub const Workspace = c.zml_fi_cutlass_moe_workspace; +pub const WorkspaceRequirements = c.zml_fi_cutlass_moe_workspace_requirements; +pub const Runner = c.zml_fi_cutlass_moe_runner; + +pub const abiVersionExpected = c.ZML_FI_CUTLASS_MOE_ABI_VERSION; + +const AbiVersionFn = @TypeOf(&c.zml_fi_cutlass_moe_abi_version); +const CompiledSmFn = @TypeOf(&c.zml_fi_cutlass_moe_compiled_sm); +const LastErrorFn = @TypeOf(&c.zml_fi_cutlass_moe_last_error); +const DeviceIsSupportedFn = @TypeOf(&c.zml_fi_cutlass_moe_device_is_supported); +const RunnerCreateFn = @TypeOf(&c.zml_fi_cutlass_moe_runner_create); +const RunnerDestroyFn = @TypeOf(&c.zml_fi_cutlass_moe_runner_destroy); +const GetTacticCountsFn = @TypeOf(&c.zml_fi_cutlass_moe_get_tactic_counts); +const GetTacticOccupancyFn = @TypeOf(&c.zml_fi_cutlass_moe_get_tactic_occupancy); +const GetWorkspaceRequirementsFn = @TypeOf(&c.zml_fi_cutlass_moe_get_workspace_requirements); +const RunFn = @TypeOf(&c.zml_fi_cutlass_moe_run); + +pub const Api = struct { + library: std.DynLib, + abiVersion: AbiVersionFn, + compiledSm: CompiledSmFn, + lastError: LastErrorFn, + deviceIsSupported: DeviceIsSupportedFn, + runnerCreate: RunnerCreateFn, + runnerDestroy: RunnerDestroyFn, + getTacticCounts: GetTacticCountsFn, + getTacticOccupancy: GetTacticOccupancyFn, + getWorkspaceRequirements: GetWorkspaceRequirementsFn, + run: RunFn, +}; + +var loaded_api: ?Api = null; + +fn libraryRunfile(sm: u16, buffer: []u8) ![]const u8 { + if (comptime builtin.os.tag != .linux) return error.UnsupportedPlatform; + const cpu_arch = comptime switch (builtin.cpu.arch) { + .aarch64 => "arm64", + .x86_64 => "amd64", + else => return error.UnsupportedPlatform, + }; + return try std.fmt.bufPrint( + buffer, + "flashinfer_cutlass_moe_linux_{s}/lib/libflashinfer_cutlass_moe_sm{d}.so", + .{ cpu_arch, sm }, + ); +} + +fn loadApi(path: []const u8) !Api { + var library = std.DynLib.open(path) catch |err| { + std.log.err("Failed to open FlashInfer CUTLASS MoE library {s}: {any}", .{ path, err }); + return err; + }; + errdefer library.close(); + + const api: Api = .{ + .library = library, + .abiVersion = library.lookup(AbiVersionFn, "zml_fi_cutlass_moe_abi_version") orelse + return error.SymbolNotFound, + .compiledSm = library.lookup(CompiledSmFn, "zml_fi_cutlass_moe_compiled_sm") orelse + return error.SymbolNotFound, + .lastError = library.lookup(LastErrorFn, "zml_fi_cutlass_moe_last_error") orelse + return error.SymbolNotFound, + .deviceIsSupported = library.lookup(DeviceIsSupportedFn, "zml_fi_cutlass_moe_device_is_supported") orelse + return error.SymbolNotFound, + .runnerCreate = library.lookup(RunnerCreateFn, "zml_fi_cutlass_moe_runner_create") orelse + return error.SymbolNotFound, + .runnerDestroy = library.lookup(RunnerDestroyFn, "zml_fi_cutlass_moe_runner_destroy") orelse + return error.SymbolNotFound, + .getTacticCounts = library.lookup(GetTacticCountsFn, "zml_fi_cutlass_moe_get_tactic_counts") orelse + return error.SymbolNotFound, + .getTacticOccupancy = library.lookup(GetTacticOccupancyFn, "zml_fi_cutlass_moe_get_tactic_occupancy") orelse + return error.SymbolNotFound, + .getWorkspaceRequirements = library.lookup( + GetWorkspaceRequirementsFn, + "zml_fi_cutlass_moe_get_workspace_requirements", + ) orelse return error.SymbolNotFound, + .run = library.lookup(RunFn, "zml_fi_cutlass_moe_run") orelse + return error.SymbolNotFound, + }; + if (api.abiVersion() != abiVersionExpected) return error.AbiVersionMismatch; + return api; +} + +pub fn load(allocator: std.mem.Allocator, io: std.Io, sm: u16) !void { + _ = allocator; + if (loaded_api) |*api| { + if (api.compiledSm() != sm) return error.ArchitectureAlreadyLoaded; + return; + } + + const runfiles = try bazel.runfiles(bazel_builtin.current_repository); + var runfile_name_buffer: [160]u8 = undefined; + const runfile_name = try libraryRunfile(sm, &runfile_name_buffer); + var runfile_path_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; + const runfile_path = (try runfiles.rlocation(runfile_name, &runfile_path_buffer)) orelse + return error.NotFound; + + // Preserve the producer library's $ORIGIN-relative dependency lookup when + // the development repository exposes it through Bazel symlinks. + var canonical_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; + const canonical_length = if (std.fs.path.isAbsolute(runfile_path)) + try std.Io.Dir.realPathFileAbsolute(io, runfile_path, &canonical_buffer) + else + try std.Io.Dir.cwd().realPathFile(io, runfile_path, &canonical_buffer); + var api = try loadApi(canonical_buffer[0..canonical_length]); + errdefer api.library.close(); + if (api.compiledSm() != sm) return error.WrongArchitectureLibrary; + loaded_api = api; +} + +pub fn apiForDevice(device: i32) !*Api { + const api = if (loaded_api) |*value| value else return error.BackendNotLoaded; + var supported: u8 = 0; + const status = api.deviceIsSupported(device, &supported); + if (status != c.ZML_FI_CUTLASS_MOE_STATUS_SUCCESS) { + if (api.lastError()) |message| { + std.log.err("FlashInfer CUTLASS MoE device probe failed: {s}", .{std.mem.span(message)}); + } + return error.DeviceProbeFailed; + } + if (supported != 0) return api; + return error.UnsupportedArchitecture; +} + +test "select host and GPU architecture-specific FlashInfer CUTLASS MoE library" { + var buffer: [160]u8 = undefined; + const path = try libraryRunfile(120, &buffer); + const expected = switch (builtin.cpu.arch) { + .aarch64 => "flashinfer_cutlass_moe_linux_arm64/lib/libflashinfer_cutlass_moe_sm120.so", + .x86_64 => "flashinfer_cutlass_moe_linux_amd64/lib/libflashinfer_cutlass_moe_sm120.so", + else => return, + }; + try std.testing.expectEqualStrings(expected, path); + + try load(std.testing.allocator, std.testing.io, 120); + const api = if (loaded_api) |*value| value else return error.NotLoaded; + try std.testing.expectEqual(@as(i32, 120), api.compiledSm()); +} diff --git a/third_party/flashinfer_cutlass_moe/BUILD.bazel b/third_party/flashinfer_cutlass_moe/BUILD.bazel new file mode 100644 index 00000000..b2ddeede --- /dev/null +++ b/third_party/flashinfer_cutlass_moe/BUILD.bazel @@ -0,0 +1,4 @@ +exports_files([ + "flashinfer_cutlass_moe.BUILD.bazel", + "repo.bzl", +]) diff --git a/third_party/flashinfer_cutlass_moe/flashinfer_cutlass_moe.BUILD.bazel b/third_party/flashinfer_cutlass_moe/flashinfer_cutlass_moe.BUILD.bazel new file mode 100644 index 00000000..b51d3392 --- /dev/null +++ b/third_party/flashinfer_cutlass_moe/flashinfer_cutlass_moe.BUILD.bazel @@ -0,0 +1,33 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "lib_sm90", + srcs = ["lib/libflashinfer_cutlass_moe_sm90.so"], +) + +filegroup( + name = "lib_sm100", + srcs = ["lib/libflashinfer_cutlass_moe_sm100.so"], +) + +filegroup( + name = "lib_sm120", + srcs = ["lib/libflashinfer_cutlass_moe_sm120.so"], +) + +filegroup( + name = "libraries", + srcs = [ + ":lib_sm100", + ":lib_sm120", + ":lib_sm90", + ], +) + +cc_library( + name = "c", + hdrs = ["include/flashinfer_cutlass_moe.h"], + includes = ["include"], +) diff --git a/third_party/flashinfer_cutlass_moe/repo.bzl b/third_party/flashinfer_cutlass_moe/repo.bzl new file mode 100644 index 00000000..60b2b70f --- /dev/null +++ b/third_party/flashinfer_cutlass_moe/repo.bzl @@ -0,0 +1,38 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +_RELEASE = "cutlass-moe-v0.1.0" + +_ASSETS = { + "amd64": { + "sha256": "d40348ad771e2bc34b3dd59602ed5f3ef47b58da06ddad582a1fd993f872fb40", + "url": "https://github.com/zml/flashinfer/releases/download/{release}/flashinfer-cutlass-moe_linux-amd64.tar.gz", + }, + "arm64": { + "sha256": "4278b3900d789e2ad6c207f7e698728a0b68ab200a0fc815625dfc25a2a6e1a8", + "url": "https://github.com/zml/flashinfer/releases/download/{release}/flashinfer-cutlass-moe_linux-arm64.tar.gz", + }, +} + +def _archive(name, arch): + asset = _ASSETS[arch] + + http_archive( + name = name, + urls = [ + asset["url"].format(release = _RELEASE), + ], + sha256 = asset["sha256"], + build_file = Label( + "//third_party/flashinfer_cutlass_moe:flashinfer_cutlass_moe.BUILD.bazel", + ), + ) + +def repo(): + _archive( + name = "flashinfer_cutlass_moe_linux_amd64", + arch = "amd64", + ) + _archive( + name = "flashinfer_cutlass_moe_linux_arm64", + arch = "arm64", + ) diff --git a/third_party/non_module_deps.bzl b/third_party/non_module_deps.bzl index 5cdbd542..de6bdc6e 100644 --- a/third_party/non_module_deps.bzl +++ b/third_party/non_module_deps.bzl @@ -2,6 +2,7 @@ load("//third_party/arocc:repo.bzl", arocc = "repo") load("//third_party/cloud_accelerator_diagnostics:repo.bzl", cloud_accelerator_diagnostics = "repo") load("//third_party/com_google_sentencepiece:repo.bzl", com_google_sentencepiece = "repo") load("//third_party/flashattn:repo.bzl", flashattn = "repo") +load("//third_party/flashinfer_cutlass_moe:repo.bzl", flashinfer_cutlass_moe = "repo") load("//third_party/iree:repo.bzl", iree = "repo") load("//third_party/libvaxis:repo.bzl", libvaxis = "repo") load("//third_party/linenoise:repo.bzl", linenoise = "repo") @@ -23,6 +24,7 @@ def _non_module_deps_impl(mctx): arocc() translate_c() flashattn() + flashinfer_cutlass_moe() linenoise() stb() zigimg() diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 14049cf9..b2c7c83e 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -96,6 +96,7 @@ zig_library( "attention/triton_attention.zig", "attention/triton_kernels/unified_attention.zig", "attention/triton_kernels/unified_attention_oneapi.zig", + "moe/cutlass_flashinfer.zig", "moe/metal.zig", "moe/moe.zig", "moe/mosaic_tpu.zig", @@ -165,6 +166,7 @@ zig_library( }) + select({ "//platforms:cuda.enabled": [ "//platforms/cuda/flashattn", + "//platforms/cuda/flashinfer_cutlass_moe", ], "//conditions:default": [], }), diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 156dd556..55d2b0ca 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -298,6 +298,7 @@ test "Backend.auto selects mosaic_tpu on TPU" { .target = .tpu, .pjrt_api = undefined, .pjrt_client = undefined, + .state = zml.platform.State.init(.tpu), .devices = &.{}, .memories = &.{}, .physical_mesh = undefined, @@ -314,6 +315,7 @@ test "Backend.auto selects triton on oneAPI" { .target = .oneapi, .pjrt_api = undefined, .pjrt_client = undefined, + .state = zml.platform.State.init(.oneapi), .devices = &.{}, .memories = &.{}, .physical_mesh = undefined, diff --git a/zml/moe/cutlass_flashinfer.zig b/zml/moe/cutlass_flashinfer.zig new file mode 100644 index 00000000..a85f572a --- /dev/null +++ b/zml/moe/cutlass_flashinfer.zig @@ -0,0 +1,725 @@ +const std = @import("std"); + +const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); +const platforms = @import("platforms"); +const zml = @import("../zml.zig"); + +const log = std.log.scoped(.moe_cutlass_flashinfer); + +pub const auto_tactic: i32 = -1; + +pub const Options = struct { + /// Device used at graph-construction time to select the architecture + /// library and size the XLA-owned scratch buffer. + workspace_query_device: i32 = 0, + activation: Activation = .swiglu, + enable_pdl: bool = false, + gemm1_tactic: i32 = auto_tactic, + /// GEMM2 uses FlashInfer's absolute tactic index. Query tacticCounts() to + /// obtain the first valid GEMM2 index. + gemm2_tactic: i32 = auto_tactic, +}; + +pub const Activation = enum { + swiglu, + geglu, + geglu_tanh, + swiglu_step, + relu2, +}; + +pub const Parameters = struct { + num_experts_per_tok: u32, + activation: ActivationMode, + workspace_query_device: i32 = 0, + enable_pdl: bool = false, + gemm1_tactic: i32 = auto_tactic, + gemm2_tactic: i32 = auto_tactic, + + pub const ActivationMode = enum { + silu, + relu, + gelu, + }; + + pub const InitOptions = struct { + num_experts_per_tok: u32, + activation: ActivationMode, + }; + + pub fn init(opts: InitOptions) Parameters { + return .{ + .num_experts_per_tok = opts.num_experts_per_tok, + .activation = opts.activation, + }; + } + + pub fn runnerOptions(self: Parameters) !Options { + return .{ + .workspace_query_device = self.workspace_query_device, + .activation = switch (self.activation) { + .silu => .swiglu, + // Tensor.gelu() is ZML's tanh GELU approximation. + .gelu => .geglu_tanh, + .relu => .relu2, + }, + .enable_pdl = self.enable_pdl, + .gemm1_tactic = self.gemm1_tactic, + .gemm2_tactic = self.gemm2_tactic, + }; + } +}; + +pub const Nvfp4Scales = struct { + /// Dynamic activation quantization multiplier. + fc1_act_global: zml.Tensor, + /// Interleaved E4M3 scales with shape returned by fc1BlockScaleShape(). + fc1_weight_block: zml.Tensor, + /// Final GEMM1 output alpha, one value per expert. + fc1_global: zml.Tensor, + /// Dynamic activation quantization multiplier. + fc2_act_global: zml.Tensor, + /// Interleaved E4M3 scales with shape returned by fc2BlockScaleShape(). + fc2_weight_block: zml.Tensor, + /// Final GEMM2 output alpha, one value per expert. + fc2_global: zml.Tensor, +}; + +pub const Metadata = struct { + variant: Variant = .bf16xbf16, + nvfp4_scales: ?Nvfp4Scales = null, + + pub const InitOptions = struct { + variant: Variant = .bf16xbf16, + nvfp4_scales: ?Nvfp4Scales = null, + }; + + pub fn init(opts: InitOptions) Metadata { + return .{ + .variant = opts.variant, + .nvfp4_scales = opts.nvfp4_scales, + }; + } + + pub fn initBuffer( + self: Metadata, + _: std.Io, + _: *const zml.Platform, + ) !zml.Bufferized(Metadata) { + if (self.nvfp4_scales != null) return error.Nvfp4ScaleBuffersRequired; + return .{ .nvfp4_scales = null }; + } +}; + +/// NVFP4 scale buffers are borrowed from the model and are not owned by Metadata. +pub fn deinitBuffer(_: *zml.Bufferized(Metadata)) void {} + +const Input = struct { + hidden_states: zml.Tensor, + fc1_weights: zml.Tensor, + fc2_weights: zml.Tensor, + topk_weights: zml.Tensor, + topk_ids: zml.Tensor, + fc1_act_global: zml.Tensor, + fc1_weight_block: zml.Tensor, + fc1_global: zml.Tensor, + fc2_act_global: zml.Tensor, + fc2_weight_block: zml.Tensor, + fc2_global: zml.Tensor, +}; + +const Bf16Input = struct { + hidden_states: zml.Tensor, + fc1_weights: zml.Tensor, + fc2_weights: zml.Tensor, + topk_weights: zml.Tensor, + topk_ids: zml.Tensor, +}; + +const Output = struct { + output: zml.Shape, + workspace: zml.Shape, +}; + +const Attributes = struct { + runners: u64 = 0, + num_tokens: i64, + hidden_size: i64, + intermediate_size: i64, + num_experts: i32, + top_k: i32, + activation: i32, + enable_pdl: bool, + fc1_act_per_expert: bool, + fc2_act_per_expert: bool, + gemm1_tactic: i32, + gemm2_tactic: i32, +}; + +const DeviceRunner = struct { + api: *fi_cutlass_moe.Api, + runner: *fi_cutlass_moe.Runner, +}; + +pub const Variant = enum { + bf16xbf16, + nvfp4xnvfp4, +}; + +const max_num_devices = zml.platform.Platform.MAX_NUM_DEVICES; + +pub const Runners = struct { + runners: [std.meta.fields(Variant).len][max_num_devices]?DeviceRunner = + @splat(@splat(null)), + + pub fn deinit(self: *Runners) void { + for (&self.runners) |*variant_runners| { + for (variant_runners) |device_runner| { + if (device_runner) |runner| { + _ = runner.api.runnerDestroy(runner.runner); + } + } + } + } + + fn ensureRunner(self: *Runners, device: i32, variant: Variant) !DeviceRunner { + if (device < 0 or device >= max_num_devices) return error.UnsupportedDevice; + + const index: usize = @intCast(device); + const variant_index: usize = @intFromEnum(variant); + if (self.runners[variant_index][index]) |runner| return runner; + + const api = try fi_cutlass_moe.apiForDevice(device); + const options = runnerOptions(device, variant); + var runner: ?*fi_cutlass_moe.Runner = null; + try checkStatus(api, api.runnerCreate(&options, &runner)); + const result: DeviceRunner = .{ + .api = api, + .runner = runner orelse return error.RunnerInitializationFailed, + }; + self.runners[variant_index][index] = result; + return result; + } +}; + +fn checkStatus(api: *const fi_cutlass_moe.Api, status: fi_cutlass_moe.Status) !void { + if (status == fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_SUCCESS) return; + if (api.lastError()) |message| { + log.err("FlashInfer CUTLASS MoE failed (status {d}): {s}", .{ + status, + std.mem.span(message), + }); + } else { + log.err("FlashInfer CUTLASS MoE failed with status {d}", .{status}); + } + return switch (status) { + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_INVALID_ARGUMENT => error.InvalidArgument, + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_UNSUPPORTED => error.UnsupportedArchitecture, + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_CUDA_ERROR => error.Cuda, + else => error.FlashinferCutlassMoe, + }; +} + +fn runnerOptions(device: i32, variant: Variant) fi_cutlass_moe.RunnerOptions { + var options = std.mem.zeroes(fi_cutlass_moe.RunnerOptions); + options.struct_size = @sizeOf(fi_cutlass_moe.RunnerOptions); + options.activation_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; + options.weight_dtype = switch (variant) { + .bf16xbf16 => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16, + .nvfp4xnvfp4 => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_PACKED_FP4, + }; + options.output_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; + options.device = device; + options.use_fused_finalize = 1; + return options; +} + +fn runnersFromAttributes(attributes: Attributes) !*Runners { + if (attributes.runners == 0) return error.RunnersNotLoaded; + return @ptrFromInt(attributes.runners); +} + +fn currentRunners() !*Runners { + const platform = zml.module.CompilationContext.current().platform; + const runners = platform.state.cuda.fi_cutlass_moe_runners; + return runners orelse error.RunnersNotLoaded; +} + +fn cutlassActivation(activation: Activation) i32 { + return switch (activation) { + .swiglu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SWIGLU, + .geglu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GEGLU, + .geglu_tanh => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GEGLU_TANH, + .swiglu_step => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SWIGLU_STEP, + .relu2 => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_RELU2, + }; +} + +fn makeContext(attributes: Attributes) fi_cutlass_moe.Context { + var context = std.mem.zeroes(fi_cutlass_moe.Context); + context.struct_size = @sizeOf(fi_cutlass_moe.Context); + context.num_tokens = attributes.num_tokens; + context.hidden_size = attributes.hidden_size; + context.intermediate_size = attributes.intermediate_size; + context.num_experts = attributes.num_experts; + context.num_experts_on_rank = attributes.num_experts; + context.top_k = attributes.top_k; + context.tp_size = 1; + context.ep_size = 1; + context.activation = @intCast(attributes.activation); + context.enable_pdl = @intFromBool(attributes.enable_pdl); + context.swizzled_input_sf = 1; + return context; +} + +fn ffiCallNvfp4( + call_frame: *zml.pjrt.ffi.CallFrame, + input: zml.pjrtx.TensorToCustomCallBuffer(Input), + output: zml.pjrtx.ShapeToCustomCallBuffer(Output), + attributes: Attributes, +) !?*zml.pjrt.ffi.Error { + const device = try call_frame.ctx.getDeviceOrdinal(call_frame.api); + const runners = try runnersFromAttributes(attributes); + const deviceRunner = try runners.ensureRunner(device, .nvfp4xnvfp4); + const context = makeContext(attributes); + + var io = std.mem.zeroes(fi_cutlass_moe.Io); + io.struct_size = @sizeOf(fi_cutlass_moe.Io); + io.input = input.hidden_states.ptr; + io.token_selected_experts = @ptrCast(@alignCast(input.topk_ids.ptr)); + io.token_final_scales = @ptrCast(@alignCast(input.topk_weights.ptr)); + io.fc1_expert_weights = input.fc1_weights.ptr; + io.fc2_expert_weights = input.fc2_weights.ptr; + io.quant_scales[0] = input.fc1_act_global.ptr; + io.quant_scales[1] = input.fc1_weight_block.ptr; + io.quant_scales[2] = input.fc1_global.ptr; + io.quant_scales[3] = input.fc2_act_global.ptr; + io.quant_scales[4] = input.fc2_weight_block.ptr; + io.quant_scales[5] = input.fc2_global.ptr; + io.quant_scale_count = 6; + io.quant_scale_per_expert_mask = + (@as(u32, @intFromBool(attributes.fc1_act_per_expert)) << 0) | + (@as(u32, @intFromBool(attributes.fc2_act_per_expert)) << 3); + io.output = output.output.ptr; + + var workspace = std.mem.zeroes(fi_cutlass_moe.Workspace); + workspace.struct_size = @sizeOf(fi_cutlass_moe.Workspace); + workspace.data = output.workspace.ptr; + workspace.data_bytes = output.workspace.shape.byteSize(); + + try checkStatus( + deviceRunner.api, + deviceRunner.api.run( + deviceRunner.runner, + &context, + &io, + &workspace, + @ptrCast(call_frame.api.stream(call_frame.ctx)), + attributes.gemm1_tactic, + attributes.gemm2_tactic, + ), + ); + return null; +} + +const routedNvfp4Call = zml.ops.CustomCall(Input, Output, Attributes, ffiCallNvfp4, .{ + .name = "flashinfer_cutlass_nvfp4_routed_moe", + // Expert sharding is owned by forwardMoe outer manual computation. + .sharding_aware = false, + .has_side_effect = false, +}); + +fn ffiCallBf16( + call_frame: *zml.pjrt.ffi.CallFrame, + input: zml.pjrtx.TensorToCustomCallBuffer(Bf16Input), + output: zml.pjrtx.ShapeToCustomCallBuffer(Output), + attributes: Attributes, +) !?*zml.pjrt.ffi.Error { + const device = try call_frame.ctx.getDeviceOrdinal(call_frame.api); + const runners = try runnersFromAttributes(attributes); + const deviceRunner = try runners.ensureRunner(device, .bf16xbf16); + const context = makeContext(attributes); + + var io = std.mem.zeroes(fi_cutlass_moe.Io); + io.struct_size = @sizeOf(fi_cutlass_moe.Io); + io.input = input.hidden_states.ptr; + io.token_selected_experts = @ptrCast(@alignCast(input.topk_ids.ptr)); + io.token_final_scales = @ptrCast(@alignCast(input.topk_weights.ptr)); + io.fc1_expert_weights = input.fc1_weights.ptr; + io.fc2_expert_weights = input.fc2_weights.ptr; + io.output = output.output.ptr; + + var workspace = std.mem.zeroes(fi_cutlass_moe.Workspace); + workspace.struct_size = @sizeOf(fi_cutlass_moe.Workspace); + workspace.data = output.workspace.ptr; + workspace.data_bytes = output.workspace.shape.byteSize(); + + try checkStatus( + deviceRunner.api, + deviceRunner.api.run( + deviceRunner.runner, + &context, + &io, + &workspace, + @ptrCast(call_frame.api.stream(call_frame.ctx)), + attributes.gemm1_tactic, + attributes.gemm2_tactic, + ), + ); + return null; +} + +const routedBf16Call = zml.ops.CustomCall(Bf16Input, Output, Attributes, ffiCallBf16, .{ + .name = "flashinfer_cutlass_bf16_routed_moe", + .sharding_aware = false, + .has_side_effect = false, +}); + +fn computeCapability(platform: *const zml.Platform) !u16 { + if (platform.target != .cuda) return error.UnsupportedPlatform; + const devices = platform.pjrt_client.devices(platform.pjrt_api); + if (devices.len == 0) return error.NoDevices; + + const compute_capability = zml.platform.cuda.tryGetComputeCapabilities(platform, devices[0]) orelse + return error.UnknownComputeCapability; + + if (std.mem.eql(u8, compute_capability, "9.0")) return 90; + if (std.mem.eql(u8, compute_capability, "10.0")) return 100; + if (std.mem.eql(u8, compute_capability, "12.0")) return 120; + return error.UnsupportedArchitecture; +} + +pub fn load( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, +) !void { + if (comptime platforms.isEnabled(.cuda)) { + var cuda_state = &platform.state.cuda; + if (cuda_state.fi_cutlass_moe_runners != null) return; + try fi_cutlass_moe.load(allocator, io, try computeCapability(platform)); + const runners = try allocator.create(Runners); + runners.* = .{}; + cuda_state.fi_cutlass_moe_runners = runners; + return; + } + return error.UnsupportedPlatform; +} + +pub fn register(platform: *const zml.Platform) !void { + if (comptime platforms.isEnabled(.cuda)) { + try routedNvfp4Call.register(platform); + try routedBf16Call.register(platform); + return; + } + return error.UnsupportedPlatform; +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.state.cuda.fi_cutlass_moe_runners == null) return false; + _ = computeCapability(platform) catch return false; + return true; +} + +pub fn isNvfp4Supported(platform: *const zml.Platform) bool { + if (platform.state.cuda.fi_cutlass_moe_runners == null) return false; + const compute_capability = computeCapability(platform) catch return false; + return compute_capability == 100 or compute_capability == 120; +} + +pub fn tacticCounts( + platform: *const zml.Platform, + device: i32, + variant: Variant, +) !struct { gemm1: i32, gemm2: i32 } { + const runners = platform.state.cuda.fi_cutlass_moe_runners orelse return error.RunnersNotLoaded; + const deviceRunner = try runners.ensureRunner(device, variant); + var gemm1: i32 = 0; + var gemm2: i32 = 0; + try checkStatus( + deviceRunner.api, + deviceRunner.api.getTacticCounts(deviceRunner.runner, &gemm1, &gemm2), + ); + return .{ .gemm1 = gemm1, .gemm2 = gemm2 }; +} + +fn roundUp(value: i64, alignment: i64) i64 { + return @divTrunc(value + alignment - 1, alignment) * alignment; +} + +pub fn fc1BlockScaleShape( + num_experts: i64, + hidden_size: i64, + intermediate_size: i64, + activation: Activation, +) zml.Shape { + const rows = switch (activation) { + .swiglu, .geglu, .geglu_tanh, .swiglu_step => 2 * intermediate_size, + .relu2 => intermediate_size, + }; + return .init( + .{ num_experts, roundUp(rows, 128), roundUp(@divExact(hidden_size, 16), 4) }, + .f8e4m3fn, + ); +} + +pub fn fc2BlockScaleShape( + num_experts: i64, + hidden_size: i64, + intermediate_size: i64, +) zml.Shape { + return .init( + .{ num_experts, roundUp(hidden_size, 128), roundUp(@divExact(intermediate_size, 16), 4) }, + .f8e4m3fn, + ); +} + +fn isGlobalOrPerExpertScale(tensor: zml.Tensor, num_experts: i64) bool { + return tensor.dtype() == .f32 and + (tensor.rank() == 0 or (tensor.rank() == 1 and tensor.dim(0) == num_experts)); +} + +fn validateInputs( + hidden_states: zml.Tensor, + fc1_weights: zml.Tensor, + fc2_weights: zml.Tensor, + topk_weights: zml.Tensor, + topk_ids: zml.Tensor, + scales: Nvfp4Scales, + options: Options, +) !Attributes { + if (hidden_states.dtype() != .bf16 or + fc1_weights.dtype() != .f4e2m1 or + fc2_weights.dtype() != .f4e2m1 or + topk_weights.dtype() != .f32 or + topk_ids.dtype() != .i32 or + scales.fc1_weight_block.dtype() != .f8e4m3fn or + scales.fc2_weight_block.dtype() != .f8e4m3fn or + scales.fc1_global.dtype() != .f32 or + scales.fc2_global.dtype() != .f32) + { + return error.UnsupportedType; + } + if (hidden_states.rank() != 3 or + fc1_weights.rank() != 3 or + fc2_weights.rank() != 3 or + topk_weights.rank() != 3 or + topk_ids.rank() != 3) + { + return error.InvalidShape; + } + + const batch = hidden_states.dim(0); + const sequence = hidden_states.dim(1); + const hidden_size = hidden_states.dim(2); + const num_experts = fc1_weights.dim(0); + const fc1_rows = fc1_weights.dim(1); + const intermediate_size = switch (options.activation) { + .swiglu, .geglu, .geglu_tanh, .swiglu_step => @divExact(fc1_rows, 2), + .relu2 => fc1_rows, + }; + const top_k = topk_ids.dim(2); + + if (batch <= 0 or sequence <= 0 or hidden_size <= 0 or + num_experts <= 0 or intermediate_size <= 0 or + @mod(hidden_size, 16) != 0 or @mod(intermediate_size, 16) != 0 or + top_k <= 0 or top_k > num_experts) + { + return error.InvalidShape; + } + if (fc1_weights.dim(2) != hidden_size or + fc2_weights.dim(0) != num_experts or + fc2_weights.dim(1) != hidden_size or + fc2_weights.dim(2) != intermediate_size or + topk_weights.dim(0) != batch or + topk_weights.dim(1) != sequence or + topk_weights.dim(2) != top_k or + topk_ids.dim(0) != batch or + topk_ids.dim(1) != sequence or + !scales.fc1_weight_block.shape().eql(fc1BlockScaleShape( + num_experts, + hidden_size, + intermediate_size, + options.activation, + )) or + !scales.fc2_weight_block.shape().eql(fc2BlockScaleShape( + num_experts, + hidden_size, + intermediate_size, + )) or + !scales.fc1_global.shape().eql(.init(.{num_experts}, .f32)) or + !scales.fc2_global.shape().eql(.init(.{num_experts}, .f32)) or + !isGlobalOrPerExpertScale(scales.fc1_act_global, num_experts) or + !isGlobalOrPerExpertScale(scales.fc2_act_global, num_experts)) + { + return error.InvalidShape; + } + + return .{ + .num_tokens = batch * sequence, + .hidden_size = hidden_size, + .intermediate_size = intermediate_size, + .num_experts = @intCast(num_experts), + .top_k = @intCast(top_k), + .activation = cutlassActivation(options.activation), + .enable_pdl = options.enable_pdl, + .fc1_act_per_expert = scales.fc1_act_global.rank() == 1, + .fc2_act_per_expert = scales.fc2_act_global.rank() == 1, + .gemm1_tactic = options.gemm1_tactic, + .gemm2_tactic = options.gemm2_tactic, + }; +} + +/// Routed NVFP4 CUTLASS MoE. Weights are logical E2M1 tensors in [expert, output, input] order. +/// Block scales must use FlashInfer nvfp4_block_scale_interleave layout +/// BF16 activations are dynamically quantized to NVFP4 by the fused runner before each expert GEMM. +pub fn fusedExpertsNvfp4( + hidden_states: zml.Tensor, + fc1_weights: zml.Tensor, + fc2_weights: zml.Tensor, + topk_weights: zml.Tensor, + topk_ids: zml.Tensor, + scales: Nvfp4Scales, + options: Options, +) !zml.Tensor { + const runners = try currentRunners(); + var attributes = try validateInputs( + hidden_states, + fc1_weights, + fc2_weights, + topk_weights, + topk_ids, + scales, + options, + ); + attributes.runners = @intFromPtr(runners); + const deviceRunner = try runners.ensureRunner(options.workspace_query_device, .nvfp4xnvfp4); + const context = makeContext(attributes); + var requirements = std.mem.zeroes(fi_cutlass_moe.WorkspaceRequirements); + requirements.struct_size = @sizeOf(fi_cutlass_moe.WorkspaceRequirements); + try checkStatus( + deviceRunner.api, + deviceRunner.api.getWorkspaceRequirements( + deviceRunner.runner, + &context, + &requirements, + ), + ); + + const result = routedNvfp4Call.call( + .{ + .hidden_states = hidden_states, + .fc1_weights = fc1_weights, + .fc2_weights = fc2_weights, + .topk_weights = topk_weights, + .topk_ids = topk_ids, + .fc1_act_global = scales.fc1_act_global, + .fc1_weight_block = scales.fc1_weight_block, + .fc1_global = scales.fc1_global, + .fc2_act_global = scales.fc2_act_global, + .fc2_weight_block = scales.fc2_weight_block, + .fc2_global = scales.fc2_global, + }, + .{ + .output = hidden_states.shape(), + .workspace = .init(.{@as(i64, @intCast(requirements.total_bytes))}, .u8), + }, + attributes, + ); + return result.output; +} + +/// Routed BF16 x BF16 CUTLASS MoE [expert, output, input] weights. +/// Non-quantized FlashInfer path used on Hopper and Blackwell +pub fn fusedExpertsBf16( + hidden_states: zml.Tensor, + fc1_weights: zml.Tensor, + fc2_weights: zml.Tensor, + topk_weights: zml.Tensor, + topk_ids: zml.Tensor, + options: Options, +) !zml.Tensor { + if (hidden_states.dtype() != .bf16 or + fc1_weights.dtype() != .bf16 or + fc2_weights.dtype() != .bf16 or + topk_weights.dtype() != .f32 or + topk_ids.dtype() != .i32 or + hidden_states.rank() != 3 or + fc1_weights.rank() != 3 or + fc2_weights.rank() != 3 or + topk_weights.rank() != 3 or + topk_ids.rank() != 3) + { + return error.InvalidInput; + } + + const batch = hidden_states.dim(0); + const sequence = hidden_states.dim(1); + const hiddenSize = hidden_states.dim(2); + const numExperts = fc1_weights.dim(0); + const fc1Rows = fc1_weights.dim(1); + const intermediateSize = switch (options.activation) { + .swiglu, .geglu, .geglu_tanh, .swiglu_step => @divExact(fc1Rows, 2), + .relu2 => fc1Rows, + }; + const topK = topk_ids.dim(2); + if (batch <= 0 or sequence <= 0 or hiddenSize <= 0 or + numExperts <= 0 or intermediateSize <= 0 or + topK <= 0 or topK > numExperts or + fc1_weights.dim(2) != hiddenSize or + fc2_weights.dim(0) != numExperts or + fc2_weights.dim(1) != hiddenSize or + fc2_weights.dim(2) != intermediateSize or + topk_weights.dim(0) != batch or + topk_weights.dim(1) != sequence or + topk_weights.dim(2) != topK or + topk_ids.dim(0) != batch or + topk_ids.dim(1) != sequence) + { + return error.InvalidShape; + } + + const runners = try currentRunners(); + const attributes: Attributes = .{ + .runners = @intFromPtr(runners), + .num_tokens = batch * sequence, + .hidden_size = hiddenSize, + .intermediate_size = intermediateSize, + .num_experts = @intCast(numExperts), + .top_k = @intCast(topK), + .activation = cutlassActivation(options.activation), + .enable_pdl = options.enable_pdl, + .fc1_act_per_expert = false, + .fc2_act_per_expert = false, + .gemm1_tactic = options.gemm1_tactic, + .gemm2_tactic = options.gemm2_tactic, + }; + const deviceRunner = try runners.ensureRunner(options.workspace_query_device, .bf16xbf16); + const context = makeContext(attributes); + var requirements = std.mem.zeroes(fi_cutlass_moe.WorkspaceRequirements); + requirements.struct_size = @sizeOf(fi_cutlass_moe.WorkspaceRequirements); + try checkStatus( + deviceRunner.api, + deviceRunner.api.getWorkspaceRequirements( + deviceRunner.runner, + &context, + &requirements, + ), + ); + + const result = routedBf16Call.call( + .{ + .hidden_states = hidden_states, + .fc1_weights = fc1_weights, + .fc2_weights = fc2_weights, + .topk_weights = topk_weights, + .topk_ids = topk_ids, + }, + .{ + .output = hidden_states.shape(), + .workspace = .init(.{@as(i64, @intCast(requirements.total_bytes))}, .u8), + }, + attributes, + ); + return result.output; +} diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index 9ad56b4d..559568b1 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -1,7 +1,9 @@ const std = @import("std"); +const platforms = @import("platforms"); const zml = @import("../zml.zig"); const stdx = zml.stdx; +pub const cutlass_flashinfer = @import("cutlass_flashinfer.zig"); pub const metal = @import("metal.zig"); pub const mosaic_tpu = @import("mosaic_tpu.zig"); pub const triton = @import("triton.zig"); @@ -14,14 +16,26 @@ pub const ActivationMode = enum { }; pub const Backend = enum { - // Could select a more specific name like "triton_sm90_bf16" + flashinfer_cutlass, triton, mosaic_tpu, metal, pub fn auto(platform: *const zml.Platform, weights_dtype: zml.DataType) !Backend { return switch (platform.target) { - .cuda, .rocm, .oneapi => switch (weights_dtype) { + .cuda => switch (weights_dtype) { + .bf16 => if (cutlass_flashinfer.isAvailable(platform)) + .flashinfer_cutlass + else + .triton, + .f4e2m1 => if (cutlass_flashinfer.isNvfp4Supported(platform)) + .flashinfer_cutlass + else + return error.UnsupportedDataType, + .f16, .f32 => .triton, + else => error.UnsupportedDataType, + }, + .rocm, .oneapi => switch (weights_dtype) { .bf16, .f16, .f32 => .triton, else => error.UnsupportedDataType, }, @@ -37,18 +51,21 @@ pub const Backend = enum { }; } - pub fn load(backend: Backend, allocator: std.mem.Allocator) !void { - _ = allocator; + pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { return switch (backend) { - .triton => {}, - .mosaic_tpu => {}, - .metal => {}, + .flashinfer_cutlass => cutlass_flashinfer.isAvailable(platform), + .triton => switch (platform.target) { + .cuda, .rocm, .oneapi => true, + else => false, + }, + .mosaic_tpu => platform.target == .tpu, + .metal => platform.target == .metal, }; } pub fn register(backend: Backend, platform: *zml.Platform) !void { - _ = platform; return switch (backend) { + .flashinfer_cutlass => cutlass_flashinfer.register(platform), .triton => {}, .mosaic_tpu => {}, .metal => {}, @@ -57,17 +74,27 @@ pub const Backend = enum { }; pub const Parameters = union(Backend) { + flashinfer_cutlass: cutlass_flashinfer.Parameters, triton: triton.Parameters, mosaic_tpu: mosaic_tpu.Parameters, metal: metal.Parameters, pub const InitOptions = union(Backend) { + flashinfer_cutlass: cutlass_flashinfer.Parameters.InitOptions, triton: triton.Parameters.InitOptions, mosaic_tpu: mosaic_tpu.Parameters.InitOptions, metal: metal.Parameters.InitOptions, pub fn fromBackend(backend: Backend, num_experts_per_tok: ?u32, activation: ActivationMode) InitOptions { return switch (backend) { + .flashinfer_cutlass => .{ .flashinfer_cutlass = .{ + .num_experts_per_tok = num_experts_per_tok.?, + .activation = switch (activation) { + .silu => .silu, + .relu => .relu, + .gelu => .gelu, + }, + } }, .triton => .{ .triton = .{ .num_experts_per_tok = num_experts_per_tok.?, .activation = switch (activation) { @@ -98,6 +125,7 @@ pub const Parameters = union(Backend) { pub fn init(opts: InitOptions) Parameters { return switch (opts) { + .flashinfer_cutlass => |v| .{ .flashinfer_cutlass = cutlass_flashinfer.Parameters.init(v) }, .triton => |v| .{ .triton = triton.Parameters.init(v) }, .mosaic_tpu => |v| .{ .mosaic_tpu = mosaic_tpu.Parameters.init(v) }, .metal => |v| .{ .metal = metal.Parameters.init(v) }, @@ -106,17 +134,20 @@ pub const Parameters = union(Backend) { }; pub const Metadata = union(Backend) { + flashinfer_cutlass: cutlass_flashinfer.Metadata, triton: triton.Metadata, mosaic_tpu: mosaic_tpu.Metadata, metal: metal.Metadata, pub const InitOptions = union(Backend) { + flashinfer_cutlass: cutlass_flashinfer.Metadata.InitOptions, triton: triton.Metadata.InitOptions, mosaic_tpu: mosaic_tpu.Metadata.InitOptions, metal: metal.Metadata.InitOptions, pub fn fromBackend(backend: Backend) InitOptions { return switch (backend) { + .flashinfer_cutlass => .{ .flashinfer_cutlass = .{} }, .triton => .{ .triton = .{} }, .mosaic_tpu => .{ .mosaic_tpu = .{} }, .metal => .{ .metal = .{} }, @@ -126,6 +157,7 @@ pub const Metadata = union(Backend) { pub fn init(opts: InitOptions) Metadata { return switch (opts) { + .flashinfer_cutlass => |v| .{ .flashinfer_cutlass = cutlass_flashinfer.Metadata.init(v) }, .triton => |v| .{ .triton = triton.Metadata.init(v) }, .mosaic_tpu => |v| .{ .mosaic_tpu = mosaic_tpu.Metadata.init(v) }, .metal => |v| .{ .metal = metal.Metadata.init(v) }, @@ -134,6 +166,7 @@ pub const Metadata = union(Backend) { pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { return switch (self) { + .flashinfer_cutlass => |metadata| .{ .flashinfer_cutlass = try metadata.initBuffer(io, platform) }, .triton => |metadata| .{ .triton = try metadata.initBuffer(io, platform) }, .mosaic_tpu => |metadata| .{ .mosaic_tpu = try metadata.initBuffer(io, platform) }, .metal => |metadata| .{ .metal = try metadata.initBuffer(io, platform) }, @@ -142,6 +175,7 @@ pub const Metadata = union(Backend) { pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { switch (self.*) { + .flashinfer_cutlass => |*metadata| cutlass_flashinfer.deinitBuffer(metadata), .triton => |*metadata| triton.deinitBuffer(metadata), .mosaic_tpu => |*metadata| mosaic_tpu.deinitBuffer(metadata), .metal => |*metadata| metal.deinitBuffer(metadata), @@ -165,6 +199,192 @@ pub fn forwardMoe( parameters: Parameters, ) !zml.Tensor { return switch (parameters) { + .flashinfer_cutlass => b: { + if (comptime !platforms.isEnabled(.cuda)) { + return error.UnsupportedPlatform; + } + const flashinfer_metadata = switch (metadata) { + .flashinfer_cutlass => |v| v, + else => return error.InvalidMetadata, + }; + if (scales_gate_up != null or scales_down != null) { + return error.UnsupportedQuantization; + } + if (bias_gate_up != null or bias_down != null) { + return error.UnsupportedBias; + } + + const runner_options = try parameters.flashinfer_cutlass.runnerOptions(); + const expert_partition = weights_gate_up.shape().partition(.expert); + + if (flashinfer_metadata.variant == .nvfp4xnvfp4) { + const nvfp4 = flashinfer_metadata.nvfp4_scales orelse + return error.MissingNvfp4Scales; + if (expert_partition.eql(.init(.experts))) { + break :b zml.ops.manualComputation( + .{ + input, + topk_ids, + topk_weights, + weights_gate_up, + weights_down, + nvfp4.fc1_act_global, + nvfp4.fc1_weight_block, + nvfp4.fc1_global, + nvfp4.fc2_act_global, + nvfp4.fc2_weight_block, + nvfp4.fc2_global, + }, + input.shape(), + .{ + .activation = runner_options.activation, + .enable_pdl = runner_options.enable_pdl, + .gemm1_tactic = runner_options.gemm1_tactic, + .gemm2_tactic = runner_options.gemm2_tactic, + .workspace_query_device = runner_options.workspace_query_device, + }, + (struct { + fn body( + ctx: anytype, + _: std.mem.Allocator, + sharded_inputs: []const zml.Tensor, + _: zml.Shape, + ) zml.Tensor { + const local_num_experts = sharded_inputs[3].dim(.expert); + const partition_id = zml.ops.partitionId().convert(.i32); + const expert_start = partition_id.scale(local_num_experts).convert(.i32); + const expert_end = expert_start.addConstant(local_num_experts); + + const local_route_mask = sharded_inputs[1] + .cmp(.GE, expert_start) + .logical(.AND, sharded_inputs[1].cmp(.LT, expert_end)); + const local_topk_ids = local_route_mask.select( + sharded_inputs[1].sub(expert_start), + zml.Tensor.scalar(0, .i32), + ); + const local_topk_weights = local_route_mask.select( + sharded_inputs[2], + zml.Tensor.scalar(0, sharded_inputs[2].dtype()), + ); + + const local_output = cutlass_flashinfer.fusedExpertsNvfp4( + sharded_inputs[0], + sharded_inputs[3], + sharded_inputs[4], + local_topk_weights, + local_topk_ids, + .{ + .fc1_act_global = sharded_inputs[5], + .fc1_weight_block = sharded_inputs[6], + .fc1_global = sharded_inputs[7], + .fc2_act_global = sharded_inputs[8], + .fc2_weight_block = sharded_inputs[9], + .fc2_global = sharded_inputs[10], + }, + .{ + .workspace_query_device = ctx.workspace_query_device, + .activation = ctx.activation, + .enable_pdl = ctx.enable_pdl, + .gemm1_tactic = ctx.gemm1_tactic, + .gemm2_tactic = ctx.gemm2_tactic, + }, + ) catch |err| stdx.debug.panic( + "FlashInfer CUTLASS NVFP4 MoE backend failed: {}", + .{err}, + ); + const local_reshaped = local_output + .reshape(sharded_inputs[0].shape().dims()) + .withTags(.{ .b, .s, .d }); + return zml.ops.allReduce(local_reshaped, zml.Tensor.add); + } + }).body, + ); + } + + break :b try cutlass_flashinfer.fusedExpertsNvfp4( + input, + weights_gate_up, + weights_down, + topk_weights, + topk_ids, + nvfp4, + runner_options, + ); + } + if (flashinfer_metadata.nvfp4_scales != null) { + return error.UnexpectedNvfp4Scales; + } + + if (expert_partition.eql(.init(.experts))) { + break :b zml.ops.manualComputation( + .{ input, topk_ids, topk_weights, weights_gate_up, weights_down }, + input.shape(), + .{ + .activation = runner_options.activation, + .enable_pdl = runner_options.enable_pdl, + .gemm1_tactic = runner_options.gemm1_tactic, + .gemm2_tactic = runner_options.gemm2_tactic, + .workspace_query_device = runner_options.workspace_query_device, + }, + (struct { + fn body( + ctx: anytype, + _: std.mem.Allocator, + sharded_inputs: []const zml.Tensor, + _: zml.Shape, + ) zml.Tensor { + const local_num_experts = sharded_inputs[3].dim(.expert); + const partition_id = zml.ops.partitionId().convert(.i32); + const expert_start = partition_id.scale(local_num_experts).convert(.i32); + const expert_end = expert_start.addConstant(local_num_experts); + + const local_route_mask = sharded_inputs[1] + .cmp(.GE, expert_start) + .logical(.AND, sharded_inputs[1].cmp(.LT, expert_end)); + const local_topk_ids = local_route_mask.select( + sharded_inputs[1].sub(expert_start), + zml.Tensor.scalar(0, .i32), + ); + const local_topk_weights = local_route_mask.select( + sharded_inputs[2], + zml.Tensor.scalar(0, sharded_inputs[2].dtype()), + ); + + const local_output = cutlass_flashinfer.fusedExpertsBf16( + sharded_inputs[0], + sharded_inputs[3], + sharded_inputs[4], + local_topk_weights, + local_topk_ids, + .{ + .workspace_query_device = ctx.workspace_query_device, + .activation = ctx.activation, + .enable_pdl = ctx.enable_pdl, + .gemm1_tactic = ctx.gemm1_tactic, + .gemm2_tactic = ctx.gemm2_tactic, + }, + ) catch |err| stdx.debug.panic( + "FlashInfer CUTLASS MoE backend failed: {}", + .{err}, + ); + const local_reshaped = local_output + .reshape(sharded_inputs[0].shape().dims()) + .withTags(.{ .b, .s, .d }); + return zml.ops.allReduce(local_reshaped, zml.Tensor.add); + } + }).body, + ); + } + + break :b try cutlass_flashinfer.fusedExpertsBf16( + input, + weights_gate_up, + weights_down, + topk_weights, + topk_ids, + runner_options, + ); + }, .triton => b: { const triton_metadata = switch (metadata) { .triton => |v| v, diff --git a/zml/platform.zig b/zml/platform.zig index 6d9f597b..0818dec5 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -234,11 +234,53 @@ fn sortDevicesById(target: Target, devices: []Device) void { } } +// State union tagged on target platform to handle related resources +pub const State = union(Target) { + cpu: void, + cuda: CudaState, + rocm: void, + tpu: void, + neuron: void, + oneapi: void, + metal: void, + + pub const CudaState = struct { + fi_cutlass_moe_runners: ?*zml.moe.cutlass_flashinfer.Runners = null, + + fn deinit(self: *CudaState) void { + if (self.fi_cutlass_moe_runners) |runners| { + runners.deinit(); + self.fi_cutlass_moe_runners = null; + } + } + }; + + pub fn init(target: Target) State { + return switch (target) { + .cpu => .{ .cpu = {} }, + .cuda => .{ .cuda = .{} }, + .rocm => .{ .rocm = {} }, + .tpu => .{ .tpu = {} }, + .neuron => .{ .neuron = {} }, + .oneapi => .{ .oneapi = {} }, + .metal => .{ .metal = {} }, + }; + } + + pub fn deinit(self: *State) void { + switch (self.*) { + .cuda => |*cuda_state| cuda_state.deinit(), + else => {}, + } + } +}; + pub const Platform = struct { arena: std.heap.ArenaAllocator, target: Target, pjrt_api: *const pjrt.Api, pjrt_client: *pjrt.Client, + state: State, devices: []const Device, memories: []const Memory, physical_mesh: zml.Sharding.PhysicalMesh, @@ -276,6 +318,7 @@ pub const Platform = struct { .target = target, .pjrt_api = api, .pjrt_client = pjrt_client, + .state = State.init(target), .shardings = .empty, // set below .devices = undefined, @@ -324,6 +367,16 @@ pub const Platform = struct { zml.attention.flashattn.register(platform) catch { log.warn("Failed to register flashattn custom call", .{}); }; + if (zml.moe.cutlass_flashinfer.load(arena, io, platform)) { + zml.moe.cutlass_flashinfer.register(platform) catch |err| { + log.warn( + "Failed to register FlashInfer CUTLASS MoE custom calls: {}", + .{err}, + ); + }; + } else |err| { + log.warn("Failed to load FlashInfer CUTLASS MoE: {}", .{err}); + } }, else => {}, } @@ -475,6 +528,9 @@ pub const Platform = struct { pub fn deinit(self: *Platform, allocator: std.mem.Allocator, io: std.Io) void { _ = io; _ = allocator; + if (comptime platforms.isEnabled(.cuda)) { + self.state.deinit(); + } self.physical_mesh.deinit(self.arena.allocator()); self.pjrt_client.deinit(self.pjrt_api); self.arena.deinit(); From f223acccb75455a65ccacfc2a729bad41638d810 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Thu, 20 Aug 2026 13:13:31 +0200 Subject: [PATCH 621/881] zml/nn: accept an f32 per-channel FP8 scale (#711) --- zml/nn.zig | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index e9f7ee9c..e3f37347 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -96,7 +96,7 @@ pub const QuantScheme = enum { /// f4e2m1 values (`u8`-packed or native), f8e4m3fn scale per 16 contracted values. /// Emitted by llm-compressor and by NVIDIA's ModelOpt. nvfp4, - /// f8e4m3fn values, one bf16 scale per output channel, constant along the contraction. + /// f8e4m3fn values, one bf16 or f32 scale per output channel, constant along the contraction. /// Emitted by llm-compressor, including for the layers an NVFP4 recipe leaves in FP8. fp8_per_channel, /// f8e4m3fn values, one bf16 scale per 128x128 tile. The DeepSeek-style FP8 that model @@ -117,7 +117,8 @@ pub const QuantScheme = enum { scale.dtype() == .f8e4m3fn and scale.rank() == 2 and scale.dim(0) == n and scale.dim(1) * nvfp4_block_size == k, .fp8_per_tensor => weight.dtype() == .f8e4m3fn and scale.count() == 1, - .fp8_per_channel => weight.dtype() == .f8e4m3fn and scale.dtype() == .bf16 and + .fp8_per_channel => weight.dtype() == .f8e4m3fn and + (scale.dtype() == .bf16 or scale.dtype() == .f32) and scale.count() > 1 and scale.rank() == 2 and scale.dim(0) == n and scale.dim(1) == 1, .fp8_block128 => weight.dtype() == .f8e4m3fn and scale.dtype() == .bf16 and @@ -209,8 +210,9 @@ test "QuantScheme.classify" { // e8m0 scales: valid MX, nothing here fuses or expands it. try expect(@as(?QuantScheme, null), QuantScheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 320 }, .f8e8m0))); try expect(@as(?QuantScheme, null), QuantScheme.classify(fp8, .init(.{ .dout = 10240, .sc = 160 }, .f8e8m0))); - // Per-channel, but f32 rather than bf16. - try expect(@as(?QuantScheme, null), QuantScheme.classify(fp8, .init(.{ .dout = 10240, .sc = 1 }, .f32))); + // Per-channel in f32: what fusing per-tensor projections produces. The Metal + // per-channel kernels have f32 entries, so this is a real scheme, not a decline. + try expect(@as(?QuantScheme, .fp8_per_channel), QuantScheme.classify(fp8, .init(.{ .dout = 10240, .sc = 1 }, .f32))); // A block grid that is neither per-channel nor 128x128. try expect(@as(?QuantScheme, null), QuantScheme.classify(fp8, .init(.{ .dout = 160, .sc = 80 }, .bf16))); // Stacked MoE weights are rank 3 and go through the MoE path, not this one. From b132a650107b657b0d5c30684cd3a0cf48441add Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Mon, 24 Aug 2026 12:58:05 +0200 Subject: [PATCH 622/881] zml: support F8_E8M0 tensors (#715) Adds safetensors parsing and MLIR type support for F8_E8M0 tensors --- zml/mlirx.zig | 3 ++- zml/safetensors.zig | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/zml/mlirx.zig b/zml/mlirx.zig index a6e86522..5b5cb142 100644 --- a/zml/mlirx.zig +++ b/zml/mlirx.zig @@ -23,7 +23,7 @@ pub const Type = struct { .f8e4m3b11fnuz => .float(ctx, .f8e4m3b11fnuz), .f8e5m2 => .float(ctx, .f8e5m2), .f8e5m2fnuz => .float(ctx, .f8e5m2fnuz), - .f8e8m0 => @panic("Unsupported type"), + .f8e8m0 => .float(ctx, .f8e8m0fnu), .bf16 => .float(ctx, .bf16), .f16 => .float(ctx, .f16), .f32 => .float(ctx, .f32), @@ -51,6 +51,7 @@ pub const Type = struct { const mapping = .{ .{ .bool, mlir.Type.int(ctx, .i1) }, + .{ .f8e8m0, mlir.Type.float(ctx, .f8e8m0fnu) }, .{ .f8e4m3b11fnuz, mlir.Type.float(ctx, .f8e4m3b11fnuz) }, .{ .f8e4m3fn, mlir.Type.float(ctx, .f8e4m3fn) }, .{ .f8e4m3fnuz, mlir.Type.float(ctx, .f8e4m3fnuz) }, diff --git a/zml/safetensors.zig b/zml/safetensors.zig index 9672a60f..35f6acd1 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -756,6 +756,7 @@ fn stringToDtype(safetensor_type: []const u8) !DataType { .{ "F32", .f32 }, .{ "F16", .f16 }, .{ "BF16", .bf16 }, + .{ "F8_E8M0", .f8e8m0 }, .{ "F8_E4M3", .f8e4m3fn }, .{ "F4_E2M1", .f4e2m1 }, .{ "I64", .i64 }, From 60bd1ee2335331735eae19514e106ea139bfff6c Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Mon, 24 Aug 2026 13:40:08 +0200 Subject: [PATCH 623/881] zml/attention: add sparse MLA (#714) --- zml/BUILD.bazel | 66 ++-- zml/attention/paged_attention.zig | 277 ++++++++++++- zml/attention/triton_attention.zig | 203 ++++++++++ .../triton_kernels/unified_sparse_mla.zig | 363 ++++++++++++++++++ 4 files changed, 858 insertions(+), 51 deletions(-) create mode 100644 zml/attention/triton_kernels/unified_sparse_mla.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index b2c7c83e..4267f641 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -4,31 +4,30 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") upb_c_proto_library( name = "xla_data_upb", - deps = ["@xla//xla:xla_data_proto"], tags = ["manual"], + deps = ["@xla//xla:xla_data_proto"], ) upb_c_proto_library( name = "xla_compile_options_upb", - deps = ["@xla//xla/pjrt/proto:compile_options_proto"], tags = ["manual"], + deps = ["@xla//xla/pjrt/proto:compile_options_proto"], ) upb_c_proto_library( name = "profiler_options", - deps = ["@tsl//tsl/profiler/protobuf:profiler_options_proto"], tags = ["manual"], + deps = ["@tsl//tsl/profiler/protobuf:profiler_options_proto"], ) cc_library( name = "posix", - tags = ["manual"], hdrs = ["posix.h"], + tags = ["manual"], ) cc_library( name = "profiling_traceme_lib", - tags = ["manual"], srcs = ["profiling/traceme_lib.cc"], hdrs = ["profiling/traceme.h"], defines = select({ @@ -42,6 +41,7 @@ cc_library( "//platforms:rocm.enabled": ["-ldl"], "//conditions:default": [], }), + tags = ["manual"], visibility = ["//visibility:public"], deps = [ "//ffi:cc", @@ -75,27 +75,31 @@ zig_library( zig_library( name = "zml", - tags = ["manual"], - data = [ - "moe/triton_kernels/config.json", - ] + select({ - "//platforms:neuron.enabled": [ - "//zml/attention/nki:attention_kernel", - ], - "//conditions:default": [], - }), srcs = [ + "Sharding.zig", "attention.zig", "attention/attnd.zig", "attention/flashattn.zig", "attention/metal_attention.zig", "attention/mosaic_tpu_kernels/ragged_attention.zig", - "//zml/attention/nki:attention", "attention/paged_attention.zig", "attention/tpu_attention.zig", "attention/triton_attention.zig", "attention/triton_kernels/unified_attention.zig", "attention/triton_kernels/unified_attention_oneapi.zig", + "attention/triton_kernels/unified_sparse_mla.zig", + "buffer.zig", + "constants.zig", + "dtype.zig", + "exe.zig", + "floats.zig", + "io.zig", + "kernel.zig", + "logo.zig", + "mem.zig", + "meta.zig", + "mlirx.zig", + "module.zig", "moe/cutlass_flashinfer.zig", "moe/metal.zig", "moe/moe.zig", @@ -103,20 +107,7 @@ zig_library( "moe/mosaic_tpu_kernels/gmm_ep.zig", "moe/triton.zig", "moe/triton_kernels/triton_kernels.zig", - "kernel.zig", - "buffer.zig", - "constants.zig", - "dtype.zig", - "exe.zig", - "floats.zig", - "io.zig", - "logo.zig", - "mem.zig", - "meta.zig", - "mlirx.zig", - "module.zig", "nn.zig", - "Sharding.zig", "ops.zig", "pjrtx.zig", "platform.zig", @@ -128,8 +119,18 @@ zig_library( "tensor.zig", "testing.zig", "zml.zig", + "//zml/attention/nki:attention", ], + data = [ + "moe/triton_kernels/config.json", + ] + select({ + "//platforms:neuron.enabled": [ + "//zml/attention/nki:attention_kernel", + ], + "//conditions:default": [], + }), main = "zml.zig", + tags = ["manual"], visibility = ["//visibility:public"], zigopts = [ "-lc", @@ -137,11 +138,12 @@ zig_library( ], deps = [ ":posix", + ":profiler_options", + ":profiling_traceme_lib", ":xla_compile_options_upb", ":xla_data_upb", "//bazel", "//ffi:zig", - ":profiler_options", "//kernels/mosaic_tpu:mosaic_tpu_builder", "//kernels/triton:triton_builder", "//mlir", @@ -156,7 +158,6 @@ zig_library( "//vfs", "//zml/tokenizer", "@rules_zig//zig/runfiles", - ":profiling_traceme_lib", ] + select({ "//platforms:neuron.enabled": [ "//platforms/neuron:nki_kernel", @@ -174,9 +175,9 @@ zig_library( zig_library( name = "moe_mosaic_tpu_gmm_ep", - tags = ["manual"], - main = "moe/mosaic_tpu_kernels/gmm_ep.zig", import_name = "zml/moe/mosaic_tpu_kernels/gmm_ep", + main = "moe/mosaic_tpu_kernels/gmm_ep.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ ":zml", @@ -184,7 +185,6 @@ zig_library( ], ) - zig_test( name = "test", deps = ["//zml"], diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 55d2b0ca..10e468c9 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -213,7 +213,9 @@ pub const KvCache = union(enum) { k: zml.Tensor, v: zml.Tensor, }, + // TODO: rename `dense` to `fused`. dense: zml.Tensor, + latent: zml.Tensor, pub fn update( self: KvCache, @@ -225,25 +227,33 @@ pub const KvCache = union(enum) { ) KvCache { const page_index, const offset = getPageAndOffsetFromSlotMapping(slot_mapping, chunk_size); - var kv: KvCache = switch (self) { + const kv: KvCache = switch (self) { .split => |split| switch (backend) { - .cuda_fa2, .cuda_fa3, .triton, .mosaic_tpu, .metal, .stablehlo => .{ .split = .{ - .k = split.k.scatterSlices( - .{ .page = page_index, .k_chunk = offset }, - new_k, - .{ .update_fn = zml.Tensor.ScatterOpts.override, .indices_are_unique = false, .indices_are_sorted = false }, - ), - .v = split.v.scatterSlices( - .{ .page = page_index, .k_chunk = offset }, - new_v, - .{ .update_fn = zml.Tensor.ScatterOpts.override, .indices_are_unique = false, .indices_are_sorted = false }, - ), - } }, + .cuda_fa2, .cuda_fa3, .triton, .mosaic_tpu, .metal, .stablehlo => .{ + .split = .{ + .k = split.k.scatterSlices( + .{ .page = page_index, .k_chunk = offset }, + new_k, + .{ .update_fn = zml.Tensor.ScatterOpts.override, .indices_are_unique = false, .indices_are_sorted = false }, + ).reuseBuffer(self.split.k), + .v = split.v.scatterSlices( + .{ .page = page_index, .k_chunk = offset }, + new_v, + .{ .update_fn = zml.Tensor.ScatterOpts.override, .indices_are_unique = false, .indices_are_sorted = false }, + ).reuseBuffer(self.split.v), + }, + }, }, .dense => @panic("TODO"), + .latent => |latent_kv| .{ + .latent = latent_kv.scatterSlices(.{ + .{ .page = page_index, .k_chunk = offset }, + new_k, + .{ .update_fn = zml.Tensor.ScatterOpts.override, .indices_are_unique = false, .indices_are_sorted = false }, + }).reuseBuffer(self.latent), + }, }; - kv.split.k = kv.split.k.reuseBuffer(self.split.k); - kv.split.v = kv.split.v.reuseBuffer(self.split.v); + return kv; } @@ -262,7 +272,7 @@ pub const KvCache = union(enum) { split.k.dynamicSlice1d(split.k.axis(.page), .{ .start = page_index, .len = 1 }).squeeze(.page), split.v.dynamicSlice1d(split.k.axis(.page), .{ .start = page_index, .len = 1 }).squeeze(.page), }, - .dense => @panic("TODO"), + .dense, .latent => @panic("TODO"), }; } }; @@ -274,21 +284,32 @@ pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: z .cuda_fa2 => |cuda_fa2_parameters| switch (kv_cache) { .split => |split| flashattn.paged_fa2.pagedAttention(cuda_fa2_parameters, q, split.k, split.v, opts), .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + .latent => std.debug.panic("latent KV pages are only supported with Multi-Latent Attention", .{}), }, .cuda_fa3 => |cuda_fa3_parameters| switch (kv_cache) { .split => |split| flashattn.paged_fa3.pagedAttention(cuda_fa3_parameters, q, split.k, split.v, opts), .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + .latent => std.debug.panic("latent KV pages are only supported with Multi-Latent Attention", .{}), }, .triton => |triton_parameters| switch (kv_cache) { .split => |split| triton.paged.pagedAttention(triton_parameters, q, split.k, split.v, opts), .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + .latent => std.debug.panic("latent KV pages are only supported with Multi-Latent Attention", .{}), + }, + .mosaic_tpu => |mosaic_tpu_parameters| switch (kv_cache) { + .split => std.debug.panic("split KV pages is not supported with the mosaic_tpu backend", .{}), + .latent => std.debug.panic("latent KV pages are only supported with Multi-Latent Attention", .{}), + .dense => tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, q, kv_cache.dense, opts), }, - .mosaic_tpu => |mosaic_tpu_parameters| tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, q, kv_cache.dense, opts), .metal => |metal_parameters| switch (kv_cache) { .split => |split| metal.paged.pagedAttention(metal_parameters, q, split.k, split.v, opts), .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + .latent => std.debug.panic("latent KV pages are only supported with Multi-Latent Attention", .{}), + }, + .stablehlo => |params| switch (kv_cache) { + .latent => std.debug.panic("latent KV pages are only supported with Multi-Latent Attention", .{}), + else => stablehlo_pagedAttention(params, q, kv_cache, opts), }, - .stablehlo => |params| stablehlo_pagedAttention(params, q, kv_cache, opts), }; } @@ -775,3 +796,223 @@ pub fn partialSoftmax(self: zml.Tensor, axis: anytype) PartialSoftmax { .max_value = max_val, }; } + +pub const Mla = struct { + pub const Options = struct { + rope_rank: i64, + scale: ?f32 = null, + }; + + fn stablehlo_pagedSparseAttention(q: zml.Tensor, kv: zml.Tensor, sink: ?zml.Tensor, topk: zml.Tensor, opts: Mla.Options) zml.Tensor { + stdx.debug.assert(kv.dim(.hkv) == 1, "StableHLO MLA expects one latent KV head, got {}", .{kv.dim(.hkv)}); + const kv_flat = kv.squeeze(.hkv).reshape(.{ + .kv = kv.dim(.page) * kv.dim(.k_chunk), + .hd = kv.dim(.hd), + }); + const valid_topk = topk.cmp(.GE, zml.Tensor.zeroes(topk.shape())); + const safe_topk = zml.Tensor.select(valid_topk, topk, zml.Tensor.zeroes(topk.shape())); + const mask = valid_topk.insertAxes(.topk, .{.h}); + const selected_kv = kv_flat.gather(.{ .kv = safe_topk.rename(.{ .q = .b }) }, .{}).rename(.{ .b = .q, .topk = .kv }).convert(.f32); + + const dims = zml.nn.collectDims(.{ .h, .q, .kv, .hd }, &.{ q, kv_flat }, .strict) catch { + stdx.debug.panic("Inputs have incompatible shapes (q: {f}, kv: {f}).", .{ q, kv_flat }); + }; + + const sqrt_head_dim = opts.scale orelse 1.0 / std.math.sqrt(@as(f32, @floatFromInt(dims.hd))); + const q_32 = q.convert(.f32); + var scores = q_32.dot(selected_kv, .hd).scale(sqrt_head_dim); + scores = zml.Tensor.select(mask.broad(scores.shape()), scores, zml.Tensor.constant(scores.dtype().minValue())); + + const sink_shape = q.shape().set(.hd, 1); + const attn_sink = sink orelse stdx.debug.panic("ragged MLA attention requires an attention sink", .{}); + const sink_ = attn_sink.insertAxes(0, .{.q}).insertAxes(.last, .{.hd}).broad(sink_shape); + const scores_sink = zml.Tensor.concatenate(&.{ scores, sink_.convert(scores.dtype()) }, .kv); + + const attn_weights = scores_sink.softmax(.kv); + const attn_weights_non_sink = attn_weights.slice(&.{ + .{}, + .{}, + .{ .end = topk.dim(.topk) }, + }); + return attn_weights_non_sink.dot(selected_kv, .kv).convert(q.dtype()); + } + + pub fn pagedSparseAttention(parameters: Parameters, q: zml.Tensor, kv_cache: KvCache, sink: ?zml.Tensor, topk: zml.Tensor, tokens_pos: zml.Tensor, opts: Mla.Options) zml.Tensor { + const latent_kv = switch (kv_cache) { + .latent => |latent_kv| latent_kv, + else => std.debug.panic("Sparse Multi-Latent Attention support only latent KV pages, got: {}", .{std.meta.activeTag(kv_cache)}), + }; + + stdx.debug.assert(q.shape().hasTags(.{ .q, .h, .hd }), "expected q to have tags .q, .h, .hd after flattening, got {f}", .{q.shape()}); + stdx.debug.assert(q.dim(.hd) > opts.rope_rank, "expected q head dim ({}) to include a rope tail of {}", .{ q.dim(.hd), opts.rope_rank }); + stdx.debug.assert(latent_kv.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "expected paged latent KV cache to have tags .page, .k_chunk, .hkv, .hd, got {f}", .{latent_kv.shape()}); + stdx.debug.assert(latent_kv.dim(.hd) == q.dim(.hd), "expected q and kv cache head dims to match, got q={} kv={}", .{ q.dim(.hd), latent_kv.dim(.hd) }); + + return switch (parameters) { + .triton => |triton_parameters| triton.paged.pagedSparseMla(triton_parameters, q, latent_kv, sink, topk, tokens_pos, opts), + .stablehlo => |stablehlo_parameters| stablehlo_pagedSparseAttention( + q, + latent_kv, + sink, + triton.paged.topkToPhysical(stablehlo_parameters, topk, tokens_pos, latent_kv.dim(.k_chunk)), + opts, + ), + else => @panic("NOPE"), + }; + } +}; + +test "use mla kernel" { + const platform = zml.testing.env(); + if (!Backend.triton.isAvailable(platform)) return error.SkipZigTest; + + const parameters = Parameters.init(.fromBackend(.{ + .backend = .triton, + .is_prefill = true, + .batch_size = 1, + .seq_len = 32, + .max_num_pages = 2, + .max_token_count = 1, + .num_heads = 16, + .num_kv_heads = 1, + .head_dim = 128, + .max_seqlen_q = 1, + })); + const q_shape = zml.Shape.init(.{ .q = 1, .h = 16, .hd = 128 }, .f32); + const kv_shape = zml.Shape.init(.{ .page = 2, .k_chunk = 16, .hkv = 1, .hd = 128 }, .f32); + const sink_shape = zml.Shape.init(.{ .h = 16 }, .f32); + const topk_shape = zml.Shape.init(.{ .q = 1, .topk = 2 }, .i32); + const tokens_pos_shape = zml.Shape.init(.{ .q = 1 }, .i32); + + var q_data: [1][16][128]f32 = undefined; + @memset(std.mem.sliceAsBytes(&q_data), 0); + var kv_data: [2][16][1][128]f32 = undefined; + @memset(std.mem.sliceAsBytes(&kv_data), 0); + for (&kv_data[1][0][0]) |*value| value.* = 10; + for (&kv_data[1][1][0]) |*value| value.* = 11; + var sink_data: [16]f32 = undefined; + for (&sink_data) |*value| value.* = -std.math.inf(f32); + const topk_data: [1][2]i32 = .{.{ 0, 1 }}; + const tokens_pos_data: [1]i32 = .{31}; + const block_table: [1][2]i32 = .{.{ 1, 0 }}; + const seq_lens: [1]i32 = .{32}; + const query_start_len: [2]i32 = .{ 0, 1 }; + + const q = zml.Tensor.init(q_shape, .f32); + const kv: KvCache = .{ .latent = zml.Tensor.init(kv_shape, .f32) }; + const sink = zml.Tensor.init(sink_shape, .f32); + const topk = zml.Tensor.init(topk_shape, .i32); + const tokens_pos = zml.Tensor.init(.{ .q = 1 }, .i32); + + var parameters_d: zml.Bufferized(Parameters) = .{ .triton = .{ + .block_table = try .fromBytes(std.testing.io, platform, parameters.triton.block_table.shape(), .replicated, std.mem.sliceAsBytes(&block_table)), + .seq_lens = try .fromBytes(std.testing.io, platform, parameters.triton.seq_lens.shape(), .replicated, std.mem.sliceAsBytes(&seq_lens)), + .query_start_len = try .fromBytes(std.testing.io, platform, parameters.triton.query_start_len.shape(), .replicated, std.mem.sliceAsBytes(&query_start_len)), + } }; + defer zml.Buffer.deinitAll(Parameters, ¶meters_d); + var q_d = try zml.Buffer.fromBytes(std.testing.io, platform, q_shape, .replicated, std.mem.sliceAsBytes(&q_data)); + defer q_d.deinit(); + var kv_d: zml.Bufferized(KvCache) = .{ .latent = try .fromBytes(std.testing.io, platform, kv_shape, .replicated, std.mem.sliceAsBytes(&kv_data)) }; + defer zml.Buffer.deinitAll(KvCache, &kv_d); + var sink_d = try zml.Buffer.fromBytes(std.testing.io, platform, sink_shape, .replicated, std.mem.sliceAsBytes(&sink_data)); + defer sink_d.deinit(); + var topk_d = try zml.Buffer.fromBytes(std.testing.io, platform, topk_shape, .replicated, std.mem.sliceAsBytes(&topk_data)); + defer topk_d.deinit(); + var tokens_pos_d = try zml.Buffer.fromBytes(std.testing.io, platform, tokens_pos_shape, .replicated, std.mem.sliceAsBytes(&tokens_pos_data)); + defer tokens_pos_d.deinit(); + + const exe = try platform.compileFn( + std.testing.allocator, + std.testing.io, + Mla.pagedSparseAttention, + .{ parameters, q, kv, sink, topk, tokens_pos, .{ .rope_rank = 64 } }, + .{}, + ); + defer exe.deinit(); + + var output_d = try zml.testing.autoCall( + std.testing.allocator, + std.testing.io, + &exe, + Mla.pagedSparseAttention, + .{ parameters_d, q_d, kv_d, sink_d, topk_d, tokens_pos_d }, + ); + defer zml.Buffer.deinitAll(zml.Tensor, &output_d); +} + +test "execute stablehlo mla kernel" { + const platform = zml.testing.env(); + const parameters = Parameters.init(.fromBackend(.{ + .backend = .stablehlo, + .is_prefill = true, + .batch_size = 1, + .seq_len = 32, + .max_num_pages = 2, + .max_token_count = 1, + .num_heads = 16, + .num_kv_heads = 1, + .head_dim = 128, + .max_seqlen_q = 1, + })); + + const q_shape = zml.Shape.init(.{ .q = 1, .h = 16, .hd = 128 }, .f32); + const kv_shape = zml.Shape.init(.{ .page = 2, .k_chunk = 16, .hkv = 1, .hd = 128 }, .f32); + const sink_shape = zml.Shape.init(.{ .h = 16 }, .f32); + const topk_shape = zml.Shape.init(.{ .q = 1, .topk = 2 }, .i32); + const tokens_pos_shape = zml.Shape.init(.{ .q = 1 }, .i32); + + var q_data: [1][16][128]f32 = undefined; + @memset(std.mem.sliceAsBytes(&q_data), 0); + var kv_data: [2][16][1][128]f32 = undefined; + @memset(std.mem.sliceAsBytes(&kv_data), 0); + for (&kv_data[1][0][0]) |*value| value.* = 10; + for (&kv_data[1][1][0]) |*value| value.* = 11; + var sink_data: [16]f32 = undefined; + for (&sink_data) |*value| value.* = -std.math.inf(f32); + const topk_data: [1][2]i32 = .{.{ 0, 1 }}; + const tokens_pos_data: [1]i32 = .{31}; + const block_table: [1][2]i32 = .{.{ 1, 0 }}; + const seq_lens: [1]i32 = .{32}; + const query_start_len: [2]i32 = .{ 0, 1 }; + + const q = zml.Tensor.init(q_shape, .f32); + const kv: KvCache = .{ .latent = zml.Tensor.init(kv_shape, .f32) }; + const sink = zml.Tensor.init(sink_shape, .f32); + const topk = zml.Tensor.init(topk_shape, .i32); + const tokens_pos = zml.Tensor.init(tokens_pos_shape, .i32); + + var parameters_d: zml.Bufferized(Parameters) = .{ .stablehlo = .{ + .block_table = try .fromBytes(std.testing.io, platform, parameters.stablehlo.block_table.shape(), .replicated, std.mem.sliceAsBytes(&block_table)), + .seq_lens = try .fromBytes(std.testing.io, platform, parameters.stablehlo.seq_lens.shape(), .replicated, std.mem.sliceAsBytes(&seq_lens)), + .query_start_len = try .fromBytes(std.testing.io, platform, parameters.stablehlo.query_start_len.shape(), .replicated, std.mem.sliceAsBytes(&query_start_len)), + } }; + defer zml.Buffer.deinitAll(Parameters, ¶meters_d); + var q_d = try zml.Buffer.fromBytes(std.testing.io, platform, q_shape, .replicated, std.mem.sliceAsBytes(&q_data)); + defer q_d.deinit(); + var kv_d: zml.Bufferized(KvCache) = .{ .latent = try .fromBytes(std.testing.io, platform, kv_shape, .replicated, std.mem.sliceAsBytes(&kv_data)) }; + defer zml.Buffer.deinitAll(KvCache, &kv_d); + var sink_d = try zml.Buffer.fromBytes(std.testing.io, platform, sink_shape, .replicated, std.mem.sliceAsBytes(&sink_data)); + defer sink_d.deinit(); + var topk_d = try zml.Buffer.fromBytes(std.testing.io, platform, topk_shape, .replicated, std.mem.sliceAsBytes(&topk_data)); + defer topk_d.deinit(); + var tokens_pos_d = try zml.Buffer.fromBytes(std.testing.io, platform, tokens_pos_shape, .replicated, std.mem.sliceAsBytes(&tokens_pos_data)); + defer tokens_pos_d.deinit(); + + const exe = try platform.compileFn( + std.testing.allocator, + std.testing.io, + Mla.pagedSparseAttention, + .{ parameters, q, kv, sink, topk, tokens_pos, .{ .rope_rank = 64 } }, + .{}, + ); + defer exe.deinit(); + + var output_d = try zml.testing.autoCall( + std.testing.allocator, + std.testing.io, + &exe, + Mla.pagedSparseAttention, + .{ parameters_d, q_d, kv_d, sink_d, topk_d, tokens_pos_d }, + ); + defer zml.Buffer.deinitAll(zml.Tensor, &output_d); +} diff --git a/zml/attention/triton_attention.zig b/zml/attention/triton_attention.zig index 81e80a51..64b39bb7 100644 --- a/zml/attention/triton_attention.zig +++ b/zml/attention/triton_attention.zig @@ -5,8 +5,10 @@ const stdx = @import("stdx"); const zml = @import("../zml.zig"); const triton = zml.kernel.triton; const AttentionOptions = @import("paged_attention.zig").AttentionOptions; +const MlaOptions = @import("paged_attention.zig").Mla.Options; const kernels = @import("triton_kernels/unified_attention.zig"); const kernels_oneapi = @import("triton_kernels/unified_attention_oneapi.zig"); +const mla_kernels = @import("triton_kernels/unified_sparse_mla.zig"); const log = std.log.scoped(.@"zml/attention/triton"); @@ -622,4 +624,205 @@ pub const paged = struct { return output.output; } + + fn pagedSparseMla2d(q: zml.Tensor, kv_cache: zml.Tensor, sink: ?zml.Tensor, topk: zml.Tensor, parameters: Parameters, paged_opts: PagedSparseMlaOptions) zml.Tensor { + const rope_rank: i64 = @intCast(paged_opts.rope_rank); + const nope_rank: i64 = @intCast(paged_opts.head_dim - paged_opts.rope_rank); + const kv_lora_rank: i64 = @intCast(std.math.ceilPowerOfTwoAssert(usize, @intCast(nope_rank))); + + const out_shape = q.shape(); + + const q_strides = q.shape().computeElementStrides().constSlice(); + const out_strides = out_shape.computeElementStrides().constSlice(); + const kv_strides = kv_cache.shape().computeElementStrides().constSlice(); + + const sink_: zml.Tensor = sink orelse .scalar(0, .i8); + const sm_scale: f32 = paged_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(paged_opts.head_dim)))); + + const config: Config2D = .{ + .block_m = paged_opts.block_m, + .block_q = 1, + .total_q_blocks = paged_opts.total_q_blocks, + .num_warps = 4, + .num_stages = 2, + .tile_size = @min(@as(usize, @intCast(topk.dim(.topk))), 16), + }; + + const attn_grid: [3]i32 = .{ @intCast(config.total_q_blocks * @divExact(paged_opts.num_heads, paged_opts.block_m)), 1, 1 }; + + const kernel_config: mla_kernels.Kernel.Config = .{ + .q_dtype = triton.from(q.dtype()), + .kv_dtype = triton.from(kv_cache.dtype()), + .sink_dtype = triton.from(sink_.dtype()), + .o_dtype = triton.from(q.dtype()), + .num_query_heads = @intCast(paged_opts.num_heads), + .num_queries_per_kv = @intCast(paged_opts.num_heads), + .block_size = @intCast(paged_opts.block_size), + .block_m = @intCast(paged_opts.block_m), + .topk_count = topk.dim(.topk), + .rope_rank = rope_rank, + .qk_lora_rank = nope_rank, + .kv_lora_rank = kv_lora_rank, + .rope_offset = nope_rank, + .value_rank = @intCast(paged_opts.head_dim), + .tile_size = @intCast(config.tile_size), + .use_attn_sink = if (sink) |_| true else false, + .all_decode = !parameters.options_.is_prefill, + }; + log.debug("pagedSparseMla2d config: {any}", .{kernel_config}); + + const out = mla_kernels.Kernel.call(.{ + .query_ptr = q, + .key_cache_ptr = kv_cache, + .value_cache_ptr = kv_cache, + .attn_sink_ptr = sink_, + .block_tables_ptr = parameters.block_table, + .topk_indices_ptr = topk, + .seq_lens_ptr = parameters.seq_lens, + .scale_ptr = zml.Tensor.constant(zml.DataType.f32.constant(sm_scale)), + .block_table_stride_ptr = zml.Tensor.constant(zml.DataType.i64.constant(parameters.block_table.dim(.p))), + .query_stride_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(q_strides[0])), + .query_stride_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(q_strides[1])), + .output_stride_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(out_strides[0])), + .output_stride_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(out_strides[1])), + .stride_k_cache_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(kv_strides[kv_cache.shape().axis(.page)])), + .stride_k_cache_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(kv_strides[kv_cache.shape().axis(.k_chunk)])), + .stride_k_cache_2_ptr = zml.Tensor.constant(zml.DataType.i64.constant(kv_strides[kv_cache.shape().axis(.hkv)])), + .stride_v_cache_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(kv_strides[kv_cache.shape().axis(.page)])), + .stride_v_cache_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(kv_strides[kv_cache.shape().axis(.k_chunk)])), + .stride_v_cache_2_ptr = zml.Tensor.constant(zml.DataType.i64.constant(kv_strides[kv_cache.shape().axis(.hkv)])), + .query_start_len_ptr = parameters.query_start_len, + .num_seqs_ptr = zml.Tensor.constant(.{ .i32 = @as(i32, @intCast(parameters.block_table.dim(.b))) }).reshape(.{1}), + }, .{ + .output = out_shape, + }, .{ + .cfg = kernel_config, + .grid = attn_grid, + .num_warps = @intCast(config.num_warps), + .num_stages = @intCast(config.num_stages), + }); + + return out.output.reshape(q.shape()); + } + + fn tokenToSequence(query_start_len: zml.Tensor, query_count: i64) zml.Tensor { + const sequence_count = query_start_len.dim(.b) - 1; + const starts = query_start_len.slice1d(.b, .{ .end = sequence_count }).rename(.{ .b = .seq }); + const ends = query_start_len.slice1d(.b, .{ .start = 1 }).rename(.{ .b = .seq }); + const query_x_sequence_shape = zml.Shape.init(.{ .q = query_count, .seq = sequence_count }, .i32); + const query = zml.Tensor.iota(query_x_sequence_shape, .q).convert(.i32); + const in_range = query.cmp(.GE, starts.broad(query_x_sequence_shape)).convert(.i32) + .mul(query.cmp(.LT, ends.broad(query_x_sequence_shape)).convert(.i32)) + .cmp(.NE, zml.Tensor.zeroes(query_x_sequence_shape)); + const sequence = zml.Tensor.iota(query_x_sequence_shape, .seq).convert(.i32); + return zml.Tensor.select(in_range, sequence, zml.Tensor.zeroes(query_x_sequence_shape)).sum(.seq).squeeze(.seq); + } + + pub fn topkToPhysical(parameters: anytype, topk: zml.Tensor, tokens_pos: zml.Tensor, block_size: i64) zml.Tensor { + const topk_i32 = topk.convert(.i32); + const topk_shape = topk_i32.shape(); + + stdx.debug.assert(topk_shape.hasTags(.{ .q, .topk }), "paged MLA topk must have .q and .topk axes, got {f}", .{topk_shape}); + stdx.debug.assert(tokens_pos.shape().hasTags(.{.q}), "paged MLA token positions must have a .q axis, got {f}", .{tokens_pos.shape()}); + + const query_to_sequence = tokenToSequence(parameters.query_start_len, topk_i32.dim(.q)); + const sequence_ends = parameters.query_start_len.slice1d(.b, .{ .start = 1 }).rename(.{ .b = .seq }); + const last_query = sequence_ends.gather(.{ .seq = query_to_sequence }, .{}).sub(.scalar(1, .i32)); + const last_token_pos = tokens_pos + .gather(.{ .q = last_query.rename(.{ .q = .lookup }) }, .{}) + .rename(.{ .lookup = .q }) + .convert(.i32); + const seq_lens = parameters.seq_lens.rename(.{ .b = .seq }).gather(.{ .seq = query_to_sequence }, .{}); + const first_visible_token = last_token_pos.addConstant(1).sub(seq_lens); + const relative_topk = topk_i32.sub(first_visible_token.broad(topk_shape)); + + const block_size_scalar = zml.Tensor.scalar(@as(i32, @intCast(block_size)), .i32).broad(topk_shape); + const zero = zml.Tensor.zeroes(topk_shape); + const valid_nonnegative = relative_topk.cmp(.GE, zero); + const valid_in_sequence = relative_topk.cmp(.LT, seq_lens.broad(topk_shape)); + const valid_topk = valid_nonnegative.logical(.AND, valid_in_sequence); + + const safe_topk = zml.Tensor.select(valid_topk, relative_topk, zero); + const logical_block = safe_topk.div(block_size_scalar); + const slot = safe_topk.remainder(block_size_scalar); + + const sequence = query_to_sequence.broad(topk_shape); + const block_table = parameters.block_table.rename(.{ .b = .seq }); + const physical_block = block_table.gather(.{ .seq = sequence, .p = logical_block }, .{}); + + const physical_topk = physical_block.mul(block_size_scalar).add(slot); + + return zml.Tensor.select(valid_topk, physical_topk, zml.Tensor.scalar(-1, .i32).broad(topk_shape)); + } + + pub const PagedSparseMlaOptions = struct { + head_dim: usize, + num_heads: usize, + block_size: usize, + block_m: usize, + rope_rank: usize, + scale: ?f32, + total_q_blocks: usize, + }; + + pub fn pagedSparseMla(parameters: Parameters, q: zml.Tensor, kv_cache: zml.Tensor, sink: ?zml.Tensor, topk: zml.Tensor, tokens_pos: zml.Tensor, opts: MlaOptions) zml.Tensor { + return zml.ops.manualComputation( + .{ + q, + kv_cache, + sink, + topk, + tokens_pos, + parameters.block_table, + parameters.seq_lens, + parameters.query_start_len, + }, + q.shape(), + .{ + .opts = opts, + .options = parameters.options_, + }, + (struct { + fn body(ctx_: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, _: zml.Shape) zml.Tensor { + const q_ = sharded_inputs[0]; + const kv_cache_ = sharded_inputs[1]; + + const block_size = kv_cache_.dim(.k_chunk); + + const parameters_: Parameters = .{ + .block_table = sharded_inputs[5], + .seq_lens = sharded_inputs[6], + .query_start_len = sharded_inputs[7], + .options_ = ctx_.options, + }; + + const topk_final = topkToPhysical(parameters_, sharded_inputs[3], sharded_inputs[4], block_size); + stdx.debug.assert(topk_final.dim(.q) == q_.dim(.q), "expected topk q dim ({}) to match q dim ({})", .{ topk_final.dim(.q), q_.dim(.q) }); + + const num_heads: usize = @intCast(q_.dim(.h)); + const block_m: usize = if (num_heads <= 16) 16 else std.math.ceilPowerOfTwoAssert(usize, num_heads); + stdx.debug.assert(@mod(num_heads, block_m) == 0, "expected q heads ({}) to be divisible by block_m ({})", .{ num_heads, block_m }); + + const paged_opts: PagedSparseMlaOptions = .{ + .head_dim = @intCast(q_.dim(.hd)), + .num_heads = num_heads, + .block_size = @intCast(kv_cache_.dim(.k_chunk)), + .block_m = block_m, + .rope_rank = @intCast(ctx_.opts.rope_rank), + .scale = ctx_.opts.scale, + .total_q_blocks = @intCast(q_.dim(.q)), + }; + + return pagedSparseMla2d( + q_, + kv_cache_, + sharded_inputs[2], + topk_final, + parameters_, + paged_opts, + ); + } + }).body, + ); + } }; diff --git a/zml/attention/triton_kernels/unified_sparse_mla.zig b/zml/attention/triton_kernels/unified_sparse_mla.zig new file mode 100644 index 00000000..4028e8b3 --- /dev/null +++ b/zml/attention/triton_kernels/unified_sparse_mla.zig @@ -0,0 +1,363 @@ +const std = @import("std"); + +const zml = @import("../../zml.zig"); +const ops = zml.ops; +const tri = zml.kernel.triton; +const Builder = tri.Builder; +const DType = tri.DType; +const Tensor = zml.Tensor; +const Shape = zml.Shape; +const Value = tri.Value; + +const Cfg = struct { + q_dtype: DType = .bf16, + kv_dtype: DType = .bf16, + sink_dtype: DType = .f32, + o_dtype: DType = .bf16, + num_query_heads: i64 = 32, + num_queries_per_kv: i64 = 32, + block_size: i64 = 16, + stride_k_cache_3: i64 = 1, + stride_v_cache_3: i64 = 1, + topk_count: i64 = 32, + block_m: i64 = 16, + rope_rank: i64 = 64, + qk_lora_rank: i64 = 512, + kv_lora_rank: i64 = 512, + rope_offset: i64 = 512, + value_rank: i64 = 512, + tile_size: i64 = 16, + use_attn_sink: bool = false, + all_decode: bool = false, +}; + +pub const Kernel = tri.Kernel(Cfg, .{ + .name = "_kernel_unified_attention_sparse_mla_2d_ptr", + .inputs = &.{ + "query_ptr", + "key_cache_ptr", + "value_cache_ptr", + "attn_sink_ptr", + "block_tables_ptr", + "topk_indices_ptr", + "seq_lens_ptr", + "scale_ptr", + "block_table_stride_ptr", + "query_stride_0_ptr", + "query_stride_1_ptr", + "output_stride_0_ptr", + "output_stride_1_ptr", + "stride_k_cache_0_ptr", + "stride_k_cache_1_ptr", + "stride_k_cache_2_ptr", + "stride_v_cache_0_ptr", + "stride_v_cache_1_ptr", + "stride_v_cache_2_ptr", + "query_start_len_ptr", + "num_seqs_ptr", + }, + .outputs = &.{"output"}, + .run = run, +}); + +fn run(b: *tri.Builder, cfg: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ + .query_ptr = .{ .ptr = cfg.q_dtype }, + .key_cache_ptr = .{ .ptr = cfg.kv_dtype }, + .value_cache_ptr = .{ .ptr = cfg.kv_dtype }, + .attn_sink_ptr = .{ .ptr = cfg.sink_dtype }, + .block_tables_ptr = .{ .ptr = .i32 }, + .topk_indices_ptr = .{ .ptr = .i32 }, + .seq_lens_ptr = .{ .ptr = .i32 }, + .scale_ptr = .{ .ptr = .f32 }, + .block_table_stride_ptr = .{ .ptr = .i64 }, + .query_stride_0_ptr = .{ .ptr = .i64 }, + .query_stride_1_ptr = .{ .ptr = .i64 }, + .output_stride_0_ptr = .{ .ptr = .i64 }, + .output_stride_1_ptr = .{ .ptr = .i64 }, + .stride_k_cache_0_ptr = .{ .ptr = .i64 }, + .stride_k_cache_1_ptr = .{ .ptr = .i64 }, + .stride_k_cache_2_ptr = .{ .ptr = .i64 }, + .stride_v_cache_0_ptr = .{ .ptr = .i64 }, + .stride_v_cache_1_ptr = .{ .ptr = .i64 }, + .stride_v_cache_2_ptr = .{ .ptr = .i64 }, + .query_start_len_ptr = .{ .ptr = .i32 }, + .num_seqs_ptr = .{ .ptr = .i32 }, + .output_ptr = .{ .ptr = cfg.o_dtype }, + }); + + const scale = b.load(a.scale_ptr); + const query_stride_0 = b.load(a.query_stride_0_ptr); + const query_stride_1 = b.load(a.query_stride_1_ptr); + const output_stride_0 = b.load(a.output_stride_0_ptr); + const output_stride_1 = b.load(a.output_stride_1_ptr); + const stride_k_cache_0 = b.load(a.stride_k_cache_0_ptr); + const stride_k_cache_1 = b.load(a.stride_k_cache_1_ptr); + const stride_v_cache_0 = b.load(a.stride_v_cache_0_ptr); + const stride_v_cache_1 = b.load(a.stride_v_cache_1_ptr); + const num_seqs = b.load(a.num_seqs_ptr); + + kernelUnifiedAttentionSparseMla2d( + b, + a.output_ptr, + a.query_ptr, + a.key_cache_ptr, + a.value_cache_ptr, + a.attn_sink_ptr, + a.topk_indices_ptr, + scale, + query_stride_0, + query_stride_1, + output_stride_0, + output_stride_1, + stride_k_cache_0, + stride_k_cache_1, + stride_v_cache_0, + stride_v_cache_1, + a.query_start_len_ptr, + num_seqs, + cfg, + ); +} + +fn findSeqIdx( + k: *Builder, + query_start_len_ptr: Value, + target_idx: Value, + num_seqs: Value, + block_q: i64, + use_q_block_mode: bool, +) Value { + const left_init = k.liftAs(0, .i32); + + var w = k.openWhile(.{ left_init, num_seqs }, .{ k.scalarTy(.i32), k.scalarTy(.i32) }); + { + const left = w.before_carried[0]; + const right = w.before_carried[1]; + w.yieldBefore(left.lt(right), .{ left, right }); + } + { + const left = w.after_carried[0]; + const right = w.after_carried[1]; + const mid = left.add(right).div(2); + const val = k.load(query_start_len_ptr.addPtr(mid)); + const mid_val = if (use_q_block_mode) + val.div(@as(i32, @intCast(block_q))).add(mid) + else + val; + + var i = k.openIfElse(mid_val.le(target_idx), .{ k.scalarTy(.i32), k.scalarTy(.i32) }); + { + i.yieldThen(.{ mid.add(1), right }); + } + { + i.yieldElse(.{ left, mid }); + } + w.yieldAfter(.{ i.results[0], i.results[1] }); + } + + return w.results[0].sub(1); +} + +fn kernelUnifiedAttentionSparseMla2d( + k: *Builder, + output_ptr: Value, + query_ptr: Value, + key_cache_ptr: Value, + value_cache_ptr: Value, + attn_sink_ptr: Value, + topk_indices_ptr: Value, + scale: Value, + query_stride_0: Value, + query_stride_1: Value, + output_stride_0: Value, + output_stride_1: Value, + stride_k_cache_0: Value, + stride_k_cache_1: Value, + stride_v_cache_0: Value, + stride_v_cache_1: Value, + query_start_len_ptr: Value, + num_seqs: Value, + config: Cfg, +) void { + const BLOCK_Q: i64 = 1; + const BLOCK_M: i64 = config.block_m; + const BLOCK_SIZE: i64 = config.block_size; + const ROPE_RANK: i64 = config.rope_rank; + const QK_LORA_RANK: i64 = config.qk_lora_rank; + const KV_LORA_RANK: i64 = config.kv_lora_rank; + const ROPE_OFFSET: i64 = config.rope_offset; + const VALUE_RANK: i64 = config.value_rank; + const TILE_SIZE: i64 = config.tile_size; + const NUM_QUERIES_PER_KV: i64 = config.num_queries_per_kv; + const NUM_QUERY_HEADS: i64 = config.num_query_heads; + const NUM_HEAD_BLOCKS: i64 = @divTrunc(NUM_QUERY_HEADS, BLOCK_M); + const NUM_TILES: i64 = @divTrunc(config.topk_count + TILE_SIZE - 1, TILE_SIZE); + + const q_block_global_idx = k.programId(.x); + const q_ind = q_block_global_idx.div(@as(i32, @intCast(NUM_HEAD_BLOCKS))); + const head_ind = q_block_global_idx.rem(@as(i32, @intCast(NUM_HEAD_BLOCKS))); + const seq_idx = findSeqIdx(k, query_start_len_ptr, q_ind, num_seqs, BLOCK_Q, false); + const q_block_start_idx = k.load(query_start_len_ptr.addPtr(seq_idx)); + + const q_block_local_idx = q_ind.sub(q_block_start_idx); + const cur_batch_in_all_start_index = k.load(query_start_len_ptr.addPtr(seq_idx)); + const cur_batch_in_all_stop_index = k.load(query_start_len_ptr.addPtr(seq_idx).addPtr(1)); + const cur_batch_query_len = cur_batch_in_all_stop_index.sub(cur_batch_in_all_start_index); + + k.returnIf(q_block_local_idx.ge(cur_batch_query_len), .{}); + + const offs_m = k.arange(0, BLOCK_M, .i32).add(head_ind.mul(@as(i32, @intCast(BLOCK_M)))); + const offs_lora = k.arange(0, KV_LORA_RANK, .i32); + const offs_rope = k.arange(ROPE_OFFSET, ROPE_OFFSET + ROPE_RANK, .i32); + const offs_value = k.arange(0, VALUE_RANK, .i32); + const offs_t = k.arange(0, TILE_SIZE, .i32); + + const query_pos = q_block_local_idx.add(offs_m.div(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); + + const query_offset_0 = cur_batch_in_all_start_index.add(query_pos); + const query_offset_1 = offs_m.rem(@as(i32, @intCast(NUM_QUERIES_PER_KV))); + + const query_mask_0 = query_pos.lt(cur_batch_query_len); + const query_mask_1 = query_offset_1.lt(@as(i32, @intCast(NUM_QUERY_HEADS))); + + const qo0_2d = query_offset_0.expandDims(1).mul(query_stride_0); + const qo1_2d = query_offset_1.expandDims(1).mul(query_stride_1); + const q_rope_offset = qo0_2d.add(qo1_2d).add(offs_rope.expandDims(0)); + const q_mask = query_mask_0.expandDims(1).bitAnd(query_mask_1.expandDims(1)); + const Q_rope = k.loadOpts(query_ptr.addPtr(q_rope_offset), .{ + .mask = q_mask, + .other = k.zeros(&.{ BLOCK_M, ROPE_RANK }, config.q_dtype), + .cache_modifier = if (config.all_decode or BLOCK_M >= NUM_QUERY_HEADS) .cg else .none, + }); + + const q_lora_offset = qo0_2d.add(qo1_2d).add(offs_lora.expandDims(0)); + const q_lora_mask = q_mask.bitAnd(offs_lora.lt(@as(i32, @intCast(QK_LORA_RANK))).expandDims(0)); + const Q_lora = k.loadOpts(query_ptr.addPtr(q_lora_offset), .{ + .mask = q_lora_mask, + .other = k.zeros(&.{ BLOCK_M, KV_LORA_RANK }, config.q_dtype), + .cache_modifier = if (config.all_decode or BLOCK_M >= NUM_QUERY_HEADS) .cg else .none, + }); + + const m_init = k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32); + const l_init = k.full(&.{BLOCK_M}, 0.0, .f32); + const acc_init = k.zeros(&.{ BLOCK_M, VALUE_RANK }, .f32); + + var loop = k.openFor(0, NUM_TILES, 1, .{ m_init, l_init, acc_init }); + { + const t = loop.iv; + const M = loop.carried[0]; + const L = loop.carried[1]; + const acc = loop.carried[2]; + + const tile_start = t.mul(@as(i32, @intCast(TILE_SIZE))); + const tile_offsets = tile_start.add(offs_t); + var valid_t = tile_offsets.lt(@as(i32, @intCast(config.topk_count))); + + const topk_row_ptr = topk_indices_ptr.addPtr(q_ind.mul(@as(i32, @intCast(config.topk_count)))); + const topk_pos = k.loadOpts(topk_row_ptr.addPtr(tile_start).addPtr(offs_t), .{ + .mask = valid_t, + .other = k.zeros(&.{TILE_SIZE}, .i32), + }); + valid_t = valid_t.bitAnd(topk_pos.ne(-1)); + + const physical_block_idx = topk_pos.div(@as(i32, @intCast(BLOCK_SIZE))); + const slot = topk_pos.rem(@as(i32, @intCast(BLOCK_SIZE))); + + var S = k.zeros(&.{ BLOCK_M, TILE_SIZE }, .f32); + + const physical_block_idx_t = physical_block_idx.expandDims(0); + const key_block_offsets = physical_block_idx_t.to(.i64).mul(stride_k_cache_0); + const key_block_ptrs = key_cache_ptr.addPtr(key_block_offsets); + const k_rope_dim_offsets = offs_rope.expandDims(1).mul(@as(i32, @intCast(config.stride_k_cache_3))); + const k_rope_dim_ptrs = k.broadcastTo(key_block_ptrs, &.{ ROPE_RANK, TILE_SIZE }) + .addPtr(k.broadcastTo(k_rope_dim_offsets, &.{ ROPE_RANK, TILE_SIZE })); + const slot_t = slot.expandDims(0); + const key_slot_offsets = slot_t.to(.i64).mul(stride_k_cache_1); + const K_rope = k.loadOpts(k_rope_dim_ptrs.addPtr(k.broadcastTo(key_slot_offsets, &.{ ROPE_RANK, TILE_SIZE })), .{ + .mask = valid_t.expandDims(0), + .other = k.zeros(&.{ ROPE_RANK, TILE_SIZE }, config.kv_dtype), + .cache_modifier = if (config.all_decode) .cg else .none, + }); + S = S.add(scale.mul(k.dot(Q_rope, K_rope, k.zeros(&.{ BLOCK_M, TILE_SIZE }, .f32)))); + + const k_lora_dim_offsets = offs_lora.expandDims(1).mul(@as(i32, @intCast(config.stride_k_cache_3))); + const k_lora_dim_ptrs = k.broadcastTo(key_block_ptrs, &.{ KV_LORA_RANK, TILE_SIZE }) + .addPtr(k.broadcastTo(k_lora_dim_offsets, &.{ KV_LORA_RANK, TILE_SIZE })); + const k_lora_mask = valid_t.expandDims(0).bitAnd(offs_lora.lt(@as(i32, @intCast(QK_LORA_RANK))).expandDims(1)); + const K_lora = k.loadOpts(k_lora_dim_ptrs.addPtr(k.broadcastTo(key_slot_offsets, &.{ KV_LORA_RANK, TILE_SIZE })), .{ + .mask = k_lora_mask, + .other = k.zeros(&.{ KV_LORA_RANK, TILE_SIZE }, config.kv_dtype), + .cache_modifier = if (config.all_decode) .cg else .none, + }); + + S = S.add(scale.mul(k.dot(Q_lora, K_lora, k.zeros(&.{ BLOCK_M, TILE_SIZE }, .f32)))); + + const keep_mask = query_mask_1.expandDims(1) + .bitAnd(query_mask_0.expandDims(1)) + .bitAnd(valid_t.expandDims(0)); + S = k.where(keep_mask, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); + + var m_j = M.maximum(k.maxOpts(S, .{ .axis = 1 })); + m_j = k.where(m_j.gt(-std.math.inf(f32)), m_j, k.full(&.{BLOCK_M}, 0.0, .f32)); + const P = k.exp(S.sub(m_j.expandDims(1))); + const l_j = k.sumOpts(P, .{ .axis = 1 }); + const alpha = k.exp(M.sub(m_j)); + + const acc_scaled = acc.mul(alpha.expandDims(1)); + const new_L = L.mul(alpha).add(l_j); + + const physical_block_idx_v = physical_block_idx.expandDims(1).to(.i64); + const v_block_offsets = physical_block_idx_v.mul(stride_v_cache_0); + const v_base_ptr = k.splat(value_cache_ptr, &.{ TILE_SIZE, 1 }); + const slot_v = slot.expandDims(1).to(.i64); + const v_slot_offsets = slot_v.mul(stride_v_cache_1); + const v_base_offsets = v_block_offsets.add(v_slot_offsets); + const v_base_ptrs = v_base_ptr.addPtr(v_base_offsets); + const v_lora_dim_offsets = offs_value.expandDims(0).mul(@as(i32, @intCast(config.stride_v_cache_3))); + const V_lora = k.loadOpts(k.broadcastTo(v_base_ptrs, &.{ TILE_SIZE, VALUE_RANK }).addPtr(k.broadcastTo(v_lora_dim_offsets, &.{ TILE_SIZE, VALUE_RANK })), .{ + .mask = valid_t.expandDims(1), + .other = k.zeros(&.{ TILE_SIZE, VALUE_RANK }, config.kv_dtype), + .cache_modifier = if (config.all_decode) .cg else .none, + }); + + const new_acc = k.dot(P.to(config.kv_dtype), V_lora, acc_scaled); + + loop.yield(.{ m_j, new_L, new_acc }); + } + + const M = loop.results[0]; + var L = loop.results[1]; + var acc = loop.results[2]; + + if (config.use_attn_sink) { + const sink_mask = query_mask_0.bitAnd(query_mask_1); + const sink_logits = k.loadOpts(attn_sink_ptr.addPtr(query_offset_1), .{ + .mask = sink_mask, + .other = k.zeros(&.{BLOCK_M}, config.sink_dtype), + }).to(.f32); + const sink_score = k.where( + sink_mask, + sink_logits, + k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32), + ); + const n_m = M.maximum(sink_score); + const alpha = k.exp(M.sub(n_m)); + const sink_p = k.exp(sink_score.sub(n_m)); + + acc = acc.mul(alpha.expandDims(1)); + L = L.mul(alpha).add(sink_p); + } + + const one_over_L = k.full(&.{ BLOCK_M, 1 }, 1.0, .f32).div(L.expandDims(1)); + acc = acc.mul(k.broadcastTo(one_over_L, &.{ BLOCK_M, VALUE_RANK })); + + const output_offs_lora = query_offset_0.expandDims(1).mul(output_stride_0) + .add(query_offset_1.expandDims(1).mul(output_stride_1)) + .add(offs_value.expandDims(0)); + k.storeOpts( + output_ptr.addPtr(output_offs_lora), + acc.to(config.o_dtype), + .{ .mask = q_mask }, + ); +} From c8cae615c5f6fb262a9152de741af625b15fad41 Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Mon, 24 Aug 2026 13:54:54 +0200 Subject: [PATCH 624/881] zml/tensor: add constant remainder and substraction helpers (#716) --- zml/tensor.zig | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/zml/tensor.zig b/zml/tensor.zig index 5895c30b..148b6654 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1152,11 +1152,21 @@ pub const Tensor = struct { return binaryOp("remainder", dialects.stablehlo.remainder)(self, other); } + /// Returns a Tensor containing the element-wise substraction of the input Tensor with a constant. + pub fn remainderConst(self: Tensor, b: anytype) Tensor { + return self.remainder(.scalar(b, self.dtype())); + } + /// Returns a Tensor containing the element-wise addition of the input Tensor with a constant. pub fn addConstant(self: Tensor, b: anytype) Tensor { return self.add(.scalar(b, self.dtype())); } + /// Returns a Tensor containing the element-wise substraction of the input Tensor with a constant. + pub fn subConstant(self: Tensor, b: anytype) Tensor { + return self.sub(.scalar(b, self.dtype())); + } + /// Returns a Tensor containing the element-wise division of the input Tensor by a constant. pub fn divByConst(self: Tensor, b: anytype) Tensor { return self.div(.scalar(b, self.dtype())); From 007be52c54cf126eee36c1a3f0500563d57d2fd2 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Mon, 24 Aug 2026 16:17:41 +0200 Subject: [PATCH 625/881] zml/nn: load MXFP8 and MXFP4 checkpoints (#719) --- zml/mlirx.zig | 1 + zml/moe/metal.zig | 4 +- zml/nn.zig | 99 ++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 89 insertions(+), 15 deletions(-) diff --git a/zml/mlirx.zig b/zml/mlirx.zig index 5b5cb142..a2f52123 100644 --- a/zml/mlirx.zig +++ b/zml/mlirx.zig @@ -59,6 +59,7 @@ pub const Type = struct { .{ .f8e5m2, mlir.Type.float(ctx, .f8e5m2) }, .{ .f8e5m2fnuz, mlir.Type.float(ctx, .f8e5m2fnuz) }, .{ .f4e2m1, mlir.Type.float(ctx, .f4e2m1fn) }, + .{ .f8e8m0, mlir.Type.float(ctx, .f8e8m0fnu) }, .{ .bf16, mlir.Type.float(ctx, .bf16) }, .{ .f16, mlir.Type.float(ctx, .f16) }, .{ .f32, mlir.Type.float(ctx, .f32) }, diff --git a/zml/moe/metal.zig b/zml/moe/metal.zig index 9f669434..08d58a21 100644 --- a/zml/moe/metal.zig +++ b/zml/moe/metal.zig @@ -189,8 +189,8 @@ pub fn fusedExpertsImpl( const expert_ids = topk_ids.reshape(.{ .token = num_tokens, .top_expert = top_expert }).withTags(.{ .token, .top_expert }) .merge(.{ .r = .{ .token, .top_expert } }).convert(.i32); - const gate_up_w = if (gate_up.dtype() == .u8) zml.nn.unpackNvfp4(gate_up, .d) else gate_up; - const down_w = if (down.dtype() == .u8) zml.nn.unpackNvfp4(down, .dout) else down; + const gate_up_w = if (gate_up.dtype() == .u8) zml.nn.unpackFp4(gate_up, .d) else gate_up; + const down_w = if (down.dtype() == .u8) zml.nn.unpackFp4(down, .dout) else down; const gate_up_out = try applyGateUpGlobalScale(moeGemm( x_rows, diff --git a/zml/nn.zig b/zml/nn.zig index e3f37347..b2c7da54 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -69,7 +69,11 @@ pub const Linear = struct { const wgs: ?Tensor = if (q.weight_scale) |s| s.asMultiplier() else null; const igs: ?Tensor = if (q.input_scale) |s| s.asMultiplier() else null; - const weight = if (isPackedNvfp4(q.scheme, self.weight.dtype())) unpackNvfp4(self.weight, self.tag) else self.weight; + const weight = if (isPackedFp4(q.scheme, self.weight.dtype())) unpackFp4(self.weight, self.tag) else self.weight; + const scales = if (q.scheme.isMx() and q.scales.dtype() == .u8) + q.scales.bitCast(.f8e8m0) + else + q.scales; var lhs = x.convert(.bf16); var lhs_scale: ?Tensor = null; @@ -85,17 +89,23 @@ pub const Linear = struct { } } - const acc = scaledDot(lhs, weight, lhs_scale, q.scales, self.tag); + const acc = scaledDot(lhs, weight, lhs_scale, scales, self.tag); return applyGlobalScale(acc, undo, wgs).convert(x.dtype()); } }; pub const nvfp4_block_size = 16; +pub const mx_block_size = 32; pub const QuantScheme = enum { /// f4e2m1 values (`u8`-packed or native), f8e4m3fn scale per 16 contracted values. /// Emitted by llm-compressor and by NVIDIA's ModelOpt. nvfp4, + /// f8e4m3fn values, one e8m0 (power-of-two) scale per 32 contracted values + mxfp8, + /// f4e2m1 values (`u8`-packed or native), e8m0 scale per 32 contracted values. + // GPT-OSS like + mxfp4, /// f8e4m3fn values, one bf16 or f32 scale per output channel, constant along the contraction. /// Emitted by llm-compressor, including for the layers an NVFP4 recipe leaves in FP8. fp8_per_channel, @@ -110,12 +120,17 @@ pub const QuantScheme = enum { if (weight.rank() != 2) return false; const n = weight.dim(0); - const k = if (isPackedNvfp4(self, weight.dtype())) 2 * weight.dim(1) else weight.dim(1); + const k = if (isPackedFp4(self, weight.dtype())) 2 * weight.dim(1) else weight.dim(1); return switch (self) { .nvfp4 => (weight.dtype() == .u8 or weight.dtype() == .f4e2m1) and scale.dtype() == .f8e4m3fn and scale.rank() == 2 and scale.dim(0) == n and scale.dim(1) * nvfp4_block_size == k, + .mxfp8 => weight.dtype() == .f8e4m3fn and isMxScale(scale) and + scale.dim(0) == n and scale.dim(1) * mx_block_size == k, + .mxfp4 => (weight.dtype() == .u8 or weight.dtype() == .f4e2m1) and + isMxScale(scale) and + scale.dim(0) == n and scale.dim(1) * mx_block_size == k, .fp8_per_tensor => weight.dtype() == .f8e4m3fn and scale.count() == 1, .fp8_per_channel => weight.dtype() == .f8e4m3fn and (scale.dtype() == .bf16 or scale.dtype() == .f32) and @@ -139,9 +154,13 @@ pub const QuantScheme = enum { pub fn activationQuant(self: QuantScheme) ?ActivationQuant { return switch (self) { .nvfp4 => .nvfp4, - .fp8_per_channel, .fp8_block128, .fp8_per_tensor => null, + .mxfp8, .mxfp4, .fp8_per_channel, .fp8_block128, .fp8_per_tensor => null, }; } + + pub fn isMx(self: QuantScheme) bool { + return self == .mxfp8 or self == .mxfp4; + } }; pub const ActivationQuant = enum { @@ -160,11 +179,16 @@ pub const ActivationQuant = enum { } }; -pub fn isPackedNvfp4(scheme: ?QuantScheme, weight_dtype: DataType) bool { - return scheme == .nvfp4 and weight_dtype == .u8; +pub fn isPackedFp4(scheme: ?QuantScheme, weight_dtype: DataType) bool { + return (scheme == .nvfp4 or scheme == .mxfp4) and weight_dtype == .u8; } -// Note: This test will evolve as we support more (for example, MXFP4/8 will be added) +fn isMxScale(scale: Shape) bool { + return scale.rank() == 2 and (scale.dtype() == .f8e8m0 or scale.dtype() == .u8); +} + +// Note: this test will evolve as we support more (INT4/8 and FP8 block-128 as a +// scaled-dot are still open; see the scaledDot doc comment). test "QuantScheme.classify" { const expect = std.testing.expectEqual; @@ -203,13 +227,53 @@ test "QuantScheme.classify" { try expect(@as(?QuantScheme, .fp8_per_tensor), QuantScheme.classify(.init(.{ .dout = 4096, .d = 4096 }, .f8e4m3fn), .init(.{}, .f32))); try expect(@as(?QuantScheme, .fp8_per_tensor), QuantScheme.classify(.init(.{ .dout = 4096, .d = 4096 }, .f8e4m3fn), .init(.{ .g = 1 }, .f32))); + // RadixArk/Muse-Glimmer-NVFP4: ModelOpt's MIXED_PRECISION recipe, whose MXFP8 half + // lands on down_proj and lm_head. safetensors has no e8m0 dtype, so the scale arrives + // as u8 under `weight_scale_inv` -- the shape below is verbatim from the checkpoint. + try expect(@as(?QuantScheme, .mxfp8), QuantScheme.classify( + .init(.{ .dout = 6656, .d = 19968 }, .f8e4m3fn), + .init(.{ .dout = 6656, .sc = 624 }, .u8), + )); + try expect(@as(?QuantScheme, .mxfp8), QuantScheme.classify( + .init(.{ .dout = 202048, .d = 6656 }, .f8e4m3fn), + .init(.{ .dout = 202048, .sc = 208 }, .u8), + )); + // The same tensor from a store that types the scale natively. + try expect(@as(?QuantScheme, .mxfp8), QuantScheme.classify( + .init(.{ .dout = 6656, .d = 19968 }, .f8e4m3fn), + .init(.{ .dout = 6656, .sc = 624 }, .f8e8m0), + )); + // MXFP8 rejects any group but 32: 19968 / 16 = 1248, / 64 = 312. + try expect(@as(?QuantScheme, null), QuantScheme.classify( + .init(.{ .dout = 6656, .d = 19968 }, .f8e4m3fn), + .init(.{ .dout = 6656, .sc = 1248 }, .u8), + )); + try expect(@as(?QuantScheme, null), QuantScheme.classify( + .init(.{ .dout = 6656, .d = 19968 }, .f8e4m3fn), + .init(.{ .dout = 6656, .sc = 312 }, .u8), + )); + + // MXFP4: the same u8 packing as NVFP4 (two nibbles per byte, K halved on disk), with + // an e8m0 scale per 32 -- so K = 2 * 2560 = 5120 and 5120 / 32 = 160. + try expect(@as(?QuantScheme, .mxfp4), QuantScheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 160 }, .u8))); + try expect(@as(?QuantScheme, .mxfp4), QuantScheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 160 }, .f8e8m0))); + // Native (unpacked) f4e2m1, K no longer halved. + try expect(@as(?QuantScheme, .mxfp4), QuantScheme.classify( + .init(.{ .dout = 17408, .d = 5120 }, .f4e2m1), + .init(.{ .dout = 17408, .sc = 160 }, .f8e8m0), + )); + // An e8m0 scale never turns a non-fp4/fp8 weight into MX. + try expect(@as(?QuantScheme, null), QuantScheme.classify( + .init(.{ .dout = 17408, .d = 5120 }, .bf16), + .init(.{ .dout = 17408, .sc = 160 }, .f8e8m0), + )); + // Rejected: everything no backend here can express. const fp8: Shape = .init(.{ .dout = 10240, .d = 5120 }, .f8e4m3fn); - // Group 32 (MXFP4) rather than 16. + // Group 32 with an e4m3 scale: neither NVFP4 (wrong group) nor MX (wrong scale type). try expect(@as(?QuantScheme, null), QuantScheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 160 }, .f8e4m3fn))); - // e8m0 scales: valid MX, nothing here fuses or expands it. + // Group 16 with an e8m0 scale: the mirror image, and equally unclaimed. try expect(@as(?QuantScheme, null), QuantScheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 320 }, .f8e8m0))); - try expect(@as(?QuantScheme, null), QuantScheme.classify(fp8, .init(.{ .dout = 10240, .sc = 160 }, .f8e8m0))); // Per-channel in f32: what fusing per-tensor projections produces. The Metal // per-channel kernels have f32 entries, so this is a real scheme, not a decline. try expect(@as(?QuantScheme, .fp8_per_channel), QuantScheme.classify(fp8, .init(.{ .dout = 10240, .sc = 1 }, .f32))); @@ -221,6 +285,14 @@ test "QuantScheme.classify" { .init(.{ .expert = 128, .dout = 1408, .sc = 176 }, .f8e4m3fn), )); + // `fp8_per_tensor` never checks the scale dtype, so a [1, 1] u8 scale on a [1, 32] + // weight satisfies it as well as `mxfp8`. Declaration order is what decides, and + // `mxfp8` is declared first + try expect(@as(?QuantScheme, .mxfp8), QuantScheme.classify( + .init(.{ .dout = 1, .d = 32 }, .f8e4m3fn), + .init(.{ .dout = 1, .sc = 1 }, .u8), + )); + // A one-tile block grid is indistinguishable from a per-tensor scale by shape alone; the // `count() > 1` guard on the grids is what makes per-tensor win. try expect(@as(?QuantScheme, .fp8_per_tensor), QuantScheme.classify( @@ -230,9 +302,10 @@ test "QuantScheme.classify" { } /// Unpacks two f4e2m1 values per byte. `w` must be tagged with `.kw` on the packed axis, -/// which is merged with the unpacked pair and renamed to `k_tag`. -pub fn unpackNvfp4(w: Tensor, k_tag: anytype) Tensor { - stdx.debug.assert(w.dtype() == .u8, "unpackNvfp4 expects packed u8 weights, got {}", .{w.dtype()}); +/// which is merged with the unpacked pair and renamed to `k_tag`. Group-size agnostic: +/// NVFP4 (16) and MXFP4 (32) share this packing. +pub fn unpackFp4(w: Tensor, k_tag: anytype) Tensor { + stdx.debug.assert(w.dtype() == .u8, "unpackFp4 expects packed u8 weights, got {}", .{w.dtype()}); return w.bitCast(.f4e2m1) // bitcast inserts a tag (it respects shlo), but maybe we should simplify it .merge(.{ .kb = .{ .kw, .bitcast } }) .renameTag(.kb, Shape.toTag(k_tag)); From baa6bb43547c974d9442e66caed93b8e977a480d Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Mon, 24 Aug 2026 15:03:59 +0000 Subject: [PATCH 626/881] zml/attention/mla: Add 3d kernel (#717) On 8 GPUs, we ended up only using one SM per GPU without 3d attention. --- zml/attention/paged_attention.zig | 11 +- zml/attention/triton_attention.zig | 272 +++++++++++++++--- .../triton_kernels/unified_sparse_mla.zig | 259 ++++++++++++++++- 3 files changed, 493 insertions(+), 49 deletions(-) diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 10e468c9..ada58259 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -801,6 +801,8 @@ pub const Mla = struct { pub const Options = struct { rope_rank: i64, scale: ?f32 = null, + /// null selects automatically; 1 forces the 2D kernel; other values must be powers of two up to 16. + num_kv_splits: ?u8 = null, }; fn stablehlo_pagedSparseAttention(q: zml.Tensor, kv: zml.Tensor, sink: ?zml.Tensor, topk: zml.Tensor, opts: Mla.Options) zml.Tensor { @@ -881,7 +883,7 @@ test "use mla kernel" { const q_shape = zml.Shape.init(.{ .q = 1, .h = 16, .hd = 128 }, .f32); const kv_shape = zml.Shape.init(.{ .page = 2, .k_chunk = 16, .hkv = 1, .hd = 128 }, .f32); const sink_shape = zml.Shape.init(.{ .h = 16 }, .f32); - const topk_shape = zml.Shape.init(.{ .q = 1, .topk = 2 }, .i32); + const topk_shape = zml.Shape.init(.{ .q = 1, .topk = 32 }, .i32); const tokens_pos_shape = zml.Shape.init(.{ .q = 1 }, .i32); var q_data: [1][16][128]f32 = undefined; @@ -892,7 +894,10 @@ test "use mla kernel" { for (&kv_data[1][1][0]) |*value| value.* = 11; var sink_data: [16]f32 = undefined; for (&sink_data) |*value| value.* = -std.math.inf(f32); - const topk_data: [1][2]i32 = .{.{ 0, 1 }}; + var topk_data: [1][32]i32 = undefined; + @memset(&topk_data[0], -1); + topk_data[0][0] = 0; + topk_data[0][1] = 1; const tokens_pos_data: [1]i32 = .{31}; const block_table: [1][2]i32 = .{.{ 1, 0 }}; const seq_lens: [1]i32 = .{32}; @@ -925,7 +930,7 @@ test "use mla kernel" { std.testing.allocator, std.testing.io, Mla.pagedSparseAttention, - .{ parameters, q, kv, sink, topk, tokens_pos, .{ .rope_rank = 64 } }, + .{ parameters, q, kv, sink, topk, tokens_pos, .{ .rope_rank = 64, .num_kv_splits = 2 } }, .{}, ); defer exe.deinit(); diff --git a/zml/attention/triton_attention.zig b/zml/attention/triton_attention.zig index 64b39bb7..faf6d961 100644 --- a/zml/attention/triton_attention.zig +++ b/zml/attention/triton_attention.zig @@ -38,6 +38,76 @@ pub const Config2D = struct { total_q_blocks: usize, }; +const SparseMlaLaunchConfig = struct { + block_m: usize, + tile_size: usize, + num_tiles: usize, + num_splits: usize, + direct_programs: usize, +}; + +fn selectSparseMlaLaunchConfig( + query_count: usize, + num_heads: usize, + topk_count: usize, + cu_count_: usize, + requested_splits: ?u8, +) SparseMlaLaunchConfig { + stdx.debug.assert(query_count > 0, "sparse MLA requires at least one query", .{}); + stdx.debug.assert(num_heads > 0, "sparse MLA requires at least one query head", .{}); + stdx.debug.assert(topk_count > 0, "sparse MLA requires at least one top-k entry", .{}); + + const block_m = @min(num_heads, 16); + stdx.debug.assert(@mod(num_heads, block_m) == 0, "expected q heads ({}) to be divisible by block_m ({})", .{ num_heads, block_m }); + + const topk_padded = std.math.ceilPowerOfTwoAssert(usize, topk_count); + const tile_size = @min(topk_padded, 16); + const num_tiles = std.math.divCeil(usize, topk_count, tile_size) catch unreachable; + const direct_programs = query_count * @divExact(num_heads, block_m); + const cu_count = @max(cu_count_, 1); + + var max_splits: usize = 1; + while (max_splits * 2 <= @min(num_tiles, 16)) max_splits *= 2; + + if (requested_splits) |requested| { + const num_splits: usize = requested; + stdx.debug.assert(std.math.isPowerOfTwo(num_splits), "MLA num_kv_splits ({}) must be a power of two", .{num_splits}); + stdx.debug.assert(num_splits <= 16, "MLA num_kv_splits ({}) must not exceed 16", .{num_splits}); + stdx.debug.assert(num_splits <= num_tiles, "MLA num_kv_splits ({}) must not exceed sparse tile count ({})", .{ num_splits, num_tiles }); + return .{ + .block_m = block_m, + .tile_size = tile_size, + .num_tiles = num_tiles, + .num_splits = num_splits, + .direct_programs = direct_programs, + }; + } + + var best_splits: usize = 1; + var best_cost: usize = std.math.maxInt(usize); + const candidates = [_]usize{ 1, 2, 4, 8, 16 }; + for (candidates) |num_splits| { + if (num_splits > max_splits) break; + const programs = direct_programs * num_splits; + const rounds = std.math.divCeil(usize, programs, cu_count) catch unreachable; + const tiles_per_program = std.math.divCeil(usize, num_tiles, num_splits) catch unreachable; + const cost = rounds * tiles_per_program; + // Candidates are ordered by split count, so ties retain 2D or the lower-overhead 3D launch. + if (cost < best_cost) { + best_cost = cost; + best_splits = num_splits; + } + } + + return .{ + .block_m = block_m, + .tile_size = tile_size, + .num_tiles = num_tiles, + .num_splits = best_splits, + .direct_programs = direct_programs, + }; +} + fn select2dConfig(options: paged.PagedAttentionOptions) Config2D { const max_num_stages_2d: usize = if (options.head_dim <= 128) 4 else 2; @@ -135,8 +205,10 @@ fn select3dConfig(options: paged.PagedAttentionOptions) Config3D { fn getCuCount() usize { const platform = zml.module.CompilationContext.current().platform; - - return @intCast(platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count").?.int64); + if (platform.devices.len == 0) return 1; + const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return 1; + if (attribute.int64 <= 0) return 1; + return @intCast(attribute.int64); } pub const paged = struct { @@ -625,7 +697,7 @@ pub const paged = struct { return output.output; } - fn pagedSparseMla2d(q: zml.Tensor, kv_cache: zml.Tensor, sink: ?zml.Tensor, topk: zml.Tensor, parameters: Parameters, paged_opts: PagedSparseMlaOptions) zml.Tensor { + fn pagedSparseMlaKernel(q: zml.Tensor, kv_cache: zml.Tensor, sink: ?zml.Tensor, topk: zml.Tensor, parameters: Parameters, paged_opts: PagedSparseMlaOptions) zml.Tensor { const rope_rank: i64 = @intCast(paged_opts.rope_rank); const nope_rank: i64 = @intCast(paged_opts.head_dim - paged_opts.rope_rank); const kv_lora_rank: i64 = @intCast(std.math.ceilPowerOfTwoAssert(usize, @intCast(nope_rank))); @@ -637,20 +709,18 @@ pub const paged = struct { const kv_strides = kv_cache.shape().computeElementStrides().constSlice(); const sink_: zml.Tensor = sink orelse .scalar(0, .i8); + const use_sink = sink != null; const sm_scale: f32 = paged_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(paged_opts.head_dim)))); - const config: Config2D = .{ - .block_m = paged_opts.block_m, - .block_q = 1, - .total_q_blocks = paged_opts.total_q_blocks, - .num_warps = 4, - .num_stages = 2, - .tile_size = @min(@as(usize, @intCast(topk.dim(.topk))), 16), - }; + const launch = selectSparseMlaLaunchConfig( + paged_opts.total_q_blocks, + paged_opts.num_heads, + @intCast(topk.dim(.topk)), + getCuCount(), + paged_opts.num_kv_splits, + ); - const attn_grid: [3]i32 = .{ @intCast(config.total_q_blocks * @divExact(paged_opts.num_heads, paged_opts.block_m)), 1, 1 }; - - const kernel_config: mla_kernels.Kernel.Config = .{ + const kernel_config: mla_kernels.Config = .{ .q_dtype = triton.from(q.dtype()), .kv_dtype = triton.from(kv_cache.dtype()), .sink_dtype = triton.from(sink_.dtype()), @@ -658,20 +728,21 @@ pub const paged = struct { .num_query_heads = @intCast(paged_opts.num_heads), .num_queries_per_kv = @intCast(paged_opts.num_heads), .block_size = @intCast(paged_opts.block_size), - .block_m = @intCast(paged_opts.block_m), + .block_m = @intCast(launch.block_m), .topk_count = topk.dim(.topk), .rope_rank = rope_rank, .qk_lora_rank = nope_rank, .kv_lora_rank = kv_lora_rank, .rope_offset = nope_rank, .value_rank = @intCast(paged_opts.head_dim), - .tile_size = @intCast(config.tile_size), - .use_attn_sink = if (sink) |_| true else false, + .tile_size = @intCast(launch.tile_size), + .num_splits = @intCast(launch.num_splits), + .use_attn_sink = use_sink, .all_decode = !parameters.options_.is_prefill, }; - log.debug("pagedSparseMla2d config: {any}", .{kernel_config}); + log.debug("pagedSparseMla launch: {any}, kernel: {any}", .{ launch, kernel_config }); - const out = mla_kernels.Kernel.call(.{ + const kernel_inputs: mla_kernels.Kernel2D.Inputs = .{ .query_ptr = q, .key_cache_ptr = kv_cache, .value_cache_ptr = kv_cache, @@ -693,14 +764,82 @@ pub const paged = struct { .stride_v_cache_2_ptr = zml.Tensor.constant(zml.DataType.i64.constant(kv_strides[kv_cache.shape().axis(.hkv)])), .query_start_len_ptr = parameters.query_start_len, .num_seqs_ptr = zml.Tensor.constant(.{ .i32 = @as(i32, @intCast(parameters.block_table.dim(.b))) }).reshape(.{1}), - }, .{ - .output = out_shape, - }, .{ - .cfg = kernel_config, - .grid = attn_grid, - .num_warps = @intCast(config.num_warps), - .num_stages = @intCast(config.num_stages), - }); + }; + + if (launch.num_splits == 1) { + const out = mla_kernels.Kernel2D.call( + kernel_inputs, + .{ .output = out_shape }, + .{ + .cfg = kernel_config, + .grid = .{ @intCast(launch.direct_programs), 1, 1 }, + .num_warps = 4, + .num_stages = 2, + }, + ); + return out.output.reshape(q.shape()); + } + + const num_splits: i64 = @intCast(launch.num_splits); + const kernel_3d_inputs: mla_kernels.Kernel3D.Inputs = .{ + .query_ptr = kernel_inputs.query_ptr, + .key_cache_ptr = kernel_inputs.key_cache_ptr, + .value_cache_ptr = kernel_inputs.value_cache_ptr, + .attn_sink_ptr = kernel_inputs.attn_sink_ptr, + .block_tables_ptr = kernel_inputs.block_tables_ptr, + .topk_indices_ptr = kernel_inputs.topk_indices_ptr, + .seq_lens_ptr = kernel_inputs.seq_lens_ptr, + .scale_ptr = kernel_inputs.scale_ptr, + .block_table_stride_ptr = kernel_inputs.block_table_stride_ptr, + .query_stride_0_ptr = kernel_inputs.query_stride_0_ptr, + .query_stride_1_ptr = kernel_inputs.query_stride_1_ptr, + .output_stride_0_ptr = kernel_inputs.output_stride_0_ptr, + .output_stride_1_ptr = kernel_inputs.output_stride_1_ptr, + .stride_k_cache_0_ptr = kernel_inputs.stride_k_cache_0_ptr, + .stride_k_cache_1_ptr = kernel_inputs.stride_k_cache_1_ptr, + .stride_k_cache_2_ptr = kernel_inputs.stride_k_cache_2_ptr, + .stride_v_cache_0_ptr = kernel_inputs.stride_v_cache_0_ptr, + .stride_v_cache_1_ptr = kernel_inputs.stride_v_cache_1_ptr, + .stride_v_cache_2_ptr = kernel_inputs.stride_v_cache_2_ptr, + .query_start_len_ptr = kernel_inputs.query_start_len_ptr, + .num_seqs_ptr = kernel_inputs.num_seqs_ptr, + }; + const partials = mla_kernels.Kernel3D.call( + kernel_3d_inputs, + .{ + .partial_output = zml.Shape.init(.{ q.dim(.q), @as(i64, @intCast(paged_opts.num_heads)), num_splits, @as(i64, @intCast(paged_opts.head_dim)) }, .f32), + .partial_lse = zml.Shape.init(.{ q.dim(.q), @as(i64, @intCast(paged_opts.num_heads)), num_splits }, .f32), + }, + .{ + .cfg = kernel_config, + .grid = .{ @intCast(launch.direct_programs), @intCast(launch.num_splits), 1 }, + .num_warps = 4, + .num_stages = 2, + }, + ); + const out = mla_kernels.Reduce3D.call( + .{ + .partial_output_ptr = partials.partial_output, + .partial_lse_ptr = partials.partial_lse, + .attn_sink_ptr = sink_, + .output_stride_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(out_strides[0])), + .output_stride_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(out_strides[1])), + }, + .{ .output = out_shape }, + .{ + .cfg = .{ + .sink_dtype = triton.from(sink_.dtype()), + .o_dtype = triton.from(q.dtype()), + .num_query_heads = @intCast(paged_opts.num_heads), + .value_rank = @intCast(paged_opts.head_dim), + .num_splits = num_splits, + .use_attn_sink = use_sink, + }, + .grid = .{ @intCast(q.dim(.q)), @intCast(paged_opts.num_heads), 1 }, + .num_warps = 1, + .num_stages = 1, + }, + ); return out.output.reshape(q.shape()); } @@ -759,10 +898,10 @@ pub const paged = struct { head_dim: usize, num_heads: usize, block_size: usize, - block_m: usize, rope_rank: usize, scale: ?f32, total_q_blocks: usize, + num_kv_splits: ?u8, }; pub fn pagedSparseMla(parameters: Parameters, q: zml.Tensor, kv_cache: zml.Tensor, sink: ?zml.Tensor, topk: zml.Tensor, tokens_pos: zml.Tensor, opts: MlaOptions) zml.Tensor { @@ -800,20 +939,17 @@ pub const paged = struct { stdx.debug.assert(topk_final.dim(.q) == q_.dim(.q), "expected topk q dim ({}) to match q dim ({})", .{ topk_final.dim(.q), q_.dim(.q) }); const num_heads: usize = @intCast(q_.dim(.h)); - const block_m: usize = if (num_heads <= 16) 16 else std.math.ceilPowerOfTwoAssert(usize, num_heads); - stdx.debug.assert(@mod(num_heads, block_m) == 0, "expected q heads ({}) to be divisible by block_m ({})", .{ num_heads, block_m }); - const paged_opts: PagedSparseMlaOptions = .{ .head_dim = @intCast(q_.dim(.hd)), .num_heads = num_heads, .block_size = @intCast(kv_cache_.dim(.k_chunk)), - .block_m = block_m, .rope_rank = @intCast(ctx_.opts.rope_rank), .scale = ctx_.opts.scale, .total_q_blocks = @intCast(q_.dim(.q)), + .num_kv_splits = ctx_.opts.num_kv_splits, }; - return pagedSparseMla2d( + return pagedSparseMlaKernel( q_, kv_cache_, sharded_inputs[2], @@ -826,3 +962,73 @@ pub const paged = struct { ); } }; + +test "sparse MLA launch selection balances occupancy and split overhead" { + const saturated = selectSparseMlaLaunchConfig(120, 16, 512, 120, null); + try std.testing.expectEqual(@as(usize, 1), saturated.num_splits); + + const under_occupied = selectSparseMlaLaunchConfig(1, 16, 512, 120, null); + try std.testing.expectEqual(@as(usize, 16), under_occupied.num_splits); + + const tile_limited = selectSparseMlaLaunchConfig(1, 16, 32, 120, null); + try std.testing.expectEqual(@as(usize, 2), tile_limited.num_splits); + + const missing_cu_count = selectSparseMlaLaunchConfig(1, 16, 512, 0, null); + try std.testing.expectEqual(@as(usize, 1), missing_cu_count.num_splits); +} + +test "sparse MLA launch selection honors valid explicit splits" { + const forced = selectSparseMlaLaunchConfig(1, 16, 64, 120, 4); + try std.testing.expectEqual(@as(usize, 4), forced.num_splits); + + const one_tile = selectSparseMlaLaunchConfig(1, 16, 5, 120, null); + try std.testing.expectEqual(@as(usize, 8), one_tile.tile_size); + try std.testing.expectEqual(@as(usize, 1), one_tile.num_splits); +} + +test "sparse MLA emits 2D and 3D Triton kernels" { + const platform = zml.testing.env(); + var compilation = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{}); + defer compilation.deinit(); + compilation.activate(); + defer compilation.deactivate(); + + const block = @import("mlir").Block.init(&.{}, &.{}); + compilation.pushBlock(block); + defer compilation.popBlock(); + + const q = zml.Tensor.zeroes(zml.Shape.init(.{ .q = 1, .h = 16, .hd = 128 }, .bf16)); + const kv = zml.Tensor.zeroes(zml.Shape.init(.{ .page = 32, .k_chunk = 1, .hkv = 1, .hd = 128 }, .bf16)); + const sink = zml.Tensor.zeroes(zml.Shape.init(.{ .h = 16 }, .bf16)); + const topk = zml.Tensor.zeroes(zml.Shape.init(.{ .q = 1, .topk = 32 }, .i32)); + const parameters: paged.Parameters = .{ + .block_table = zml.Tensor.zeroes(zml.Shape.init(.{ .b = 1, .p = 1 }, .i32)), + .seq_lens = zml.Tensor.zeroes(zml.Shape.init(.{ .b = 1 }, .i32)), + .query_start_len = zml.Tensor.zeroes(zml.Shape.init(.{ .b = 2 }, .i32)), + .options_ = .{ + .batch_size = 1, + .max_num_pages = 1, + .max_seqlen_q = 1, + .is_prefill = false, + }, + }; + const two_d_opts: paged.PagedSparseMlaOptions = .{ + .head_dim = 128, + .num_heads = 16, + .block_size = 1, + .rope_rank = 64, + .scale = null, + .total_q_blocks = 1, + .num_kv_splits = 1, + }; + + const two_d = paged.pagedSparseMlaKernel(q, kv, sink, topk, parameters, two_d_opts); + try std.testing.expect(two_d.shape().eql(q.shape())); + try std.testing.expect(two_d.value().owner().verify()); + + var three_d_opts = two_d_opts; + three_d_opts.num_kv_splits = 2; + const three_d = paged.pagedSparseMlaKernel(q, kv, sink, topk, parameters, three_d_opts); + try std.testing.expect(three_d.shape().eql(q.shape())); + try std.testing.expect(three_d.value().owner().verify()); +} diff --git a/zml/attention/triton_kernels/unified_sparse_mla.zig b/zml/attention/triton_kernels/unified_sparse_mla.zig index 4028e8b3..2a3a3b43 100644 --- a/zml/attention/triton_kernels/unified_sparse_mla.zig +++ b/zml/attention/triton_kernels/unified_sparse_mla.zig @@ -9,7 +9,7 @@ const Tensor = zml.Tensor; const Shape = zml.Shape; const Value = tri.Value; -const Cfg = struct { +pub const Config = struct { q_dtype: DType = .bf16, kv_dtype: DType = .bf16, sink_dtype: DType = .f32, @@ -27,11 +27,12 @@ const Cfg = struct { rope_offset: i64 = 512, value_rank: i64 = 512, tile_size: i64 = 16, + num_splits: i64 = 1, use_attn_sink: bool = false, all_decode: bool = false, }; -pub const Kernel = tri.Kernel(Cfg, .{ +pub const Kernel2D = tri.Kernel(Config, .{ .name = "_kernel_unified_attention_sparse_mla_2d_ptr", .inputs = &.{ "query_ptr", @@ -57,10 +58,10 @@ pub const Kernel = tri.Kernel(Cfg, .{ "num_seqs_ptr", }, .outputs = &.{"output"}, - .run = run, + .run = run2D, }); -fn run(b: *tri.Builder, cfg: Cfg) tri.FinishError!void { +fn run2D(b: *tri.Builder, cfg: Config) tri.FinishError!void { const a = try b.declareArgs(.{ .query_ptr = .{ .ptr = cfg.q_dtype }, .key_cache_ptr = .{ .ptr = cfg.kv_dtype }, @@ -97,9 +98,10 @@ fn run(b: *tri.Builder, cfg: Cfg) tri.FinishError!void { const stride_v_cache_1 = b.load(a.stride_v_cache_1_ptr); const num_seqs = b.load(a.num_seqs_ptr); - kernelUnifiedAttentionSparseMla2d( + kernelUnifiedAttentionSparseMla( b, a.output_ptr, + null, a.query_ptr, a.key_cache_ptr, a.value_cache_ptr, @@ -117,6 +119,99 @@ fn run(b: *tri.Builder, cfg: Cfg) tri.FinishError!void { a.query_start_len_ptr, num_seqs, cfg, + false, + ); +} + +pub const Kernel3D = tri.Kernel(Config, .{ + .name = "_kernel_unified_attention_sparse_mla_3d_ptr", + .inputs = &.{ + "query_ptr", + "key_cache_ptr", + "value_cache_ptr", + "attn_sink_ptr", + "block_tables_ptr", + "topk_indices_ptr", + "seq_lens_ptr", + "scale_ptr", + "block_table_stride_ptr", + "query_stride_0_ptr", + "query_stride_1_ptr", + "output_stride_0_ptr", + "output_stride_1_ptr", + "stride_k_cache_0_ptr", + "stride_k_cache_1_ptr", + "stride_k_cache_2_ptr", + "stride_v_cache_0_ptr", + "stride_v_cache_1_ptr", + "stride_v_cache_2_ptr", + "query_start_len_ptr", + "num_seqs_ptr", + }, + .outputs = &.{ "partial_output", "partial_lse" }, + .run = run3D, +}); + +fn run3D(b: *tri.Builder, cfg: Config) tri.FinishError!void { + const a = try b.declareArgs(.{ + .query_ptr = .{ .ptr = cfg.q_dtype }, + .key_cache_ptr = .{ .ptr = cfg.kv_dtype }, + .value_cache_ptr = .{ .ptr = cfg.kv_dtype }, + .attn_sink_ptr = .{ .ptr = cfg.sink_dtype }, + .block_tables_ptr = .{ .ptr = .i32 }, + .topk_indices_ptr = .{ .ptr = .i32 }, + .seq_lens_ptr = .{ .ptr = .i32 }, + .scale_ptr = .{ .ptr = .f32 }, + .block_table_stride_ptr = .{ .ptr = .i64 }, + .query_stride_0_ptr = .{ .ptr = .i64 }, + .query_stride_1_ptr = .{ .ptr = .i64 }, + .output_stride_0_ptr = .{ .ptr = .i64 }, + .output_stride_1_ptr = .{ .ptr = .i64 }, + .stride_k_cache_0_ptr = .{ .ptr = .i64 }, + .stride_k_cache_1_ptr = .{ .ptr = .i64 }, + .stride_k_cache_2_ptr = .{ .ptr = .i64 }, + .stride_v_cache_0_ptr = .{ .ptr = .i64 }, + .stride_v_cache_1_ptr = .{ .ptr = .i64 }, + .stride_v_cache_2_ptr = .{ .ptr = .i64 }, + .query_start_len_ptr = .{ .ptr = .i32 }, + .num_seqs_ptr = .{ .ptr = .i32 }, + .partial_output_ptr = .{ .ptr = .f32 }, + .partial_lse_ptr = .{ .ptr = .f32 }, + }); + + const scale = b.load(a.scale_ptr); + const query_stride_0 = b.load(a.query_stride_0_ptr); + const query_stride_1 = b.load(a.query_stride_1_ptr); + const output_stride_0 = b.load(a.output_stride_0_ptr); + const output_stride_1 = b.load(a.output_stride_1_ptr); + const stride_k_cache_0 = b.load(a.stride_k_cache_0_ptr); + const stride_k_cache_1 = b.load(a.stride_k_cache_1_ptr); + const stride_v_cache_0 = b.load(a.stride_v_cache_0_ptr); + const stride_v_cache_1 = b.load(a.stride_v_cache_1_ptr); + const num_seqs = b.load(a.num_seqs_ptr); + + kernelUnifiedAttentionSparseMla( + b, + a.partial_output_ptr, + a.partial_lse_ptr, + a.query_ptr, + a.key_cache_ptr, + a.value_cache_ptr, + a.attn_sink_ptr, + a.topk_indices_ptr, + scale, + query_stride_0, + query_stride_1, + output_stride_0, + output_stride_1, + stride_k_cache_0, + stride_k_cache_1, + stride_v_cache_0, + stride_v_cache_1, + a.query_start_len_ptr, + num_seqs, + cfg, + true, ); } @@ -159,9 +254,10 @@ fn findSeqIdx( return w.results[0].sub(1); } -fn kernelUnifiedAttentionSparseMla2d( +fn kernelUnifiedAttentionSparseMla( k: *Builder, output_ptr: Value, + partial_lse_ptr: ?Value, query_ptr: Value, key_cache_ptr: Value, value_cache_ptr: Value, @@ -178,7 +274,8 @@ fn kernelUnifiedAttentionSparseMla2d( stride_v_cache_1: Value, query_start_len_ptr: Value, num_seqs: Value, - config: Cfg, + config: Config, + comptime three_d: bool, ) void { const BLOCK_Q: i64 = 1; const BLOCK_M: i64 = config.block_m; @@ -193,8 +290,13 @@ fn kernelUnifiedAttentionSparseMla2d( const NUM_QUERY_HEADS: i64 = config.num_query_heads; const NUM_HEAD_BLOCKS: i64 = @divTrunc(NUM_QUERY_HEADS, BLOCK_M); const NUM_TILES: i64 = @divTrunc(config.topk_count + TILE_SIZE - 1, TILE_SIZE); + const NUM_SPLITS: i64 = if (three_d) config.num_splits else 1; + const TILES_PER_SPLIT: i64 = @divTrunc(NUM_TILES + NUM_SPLITS - 1, NUM_SPLITS); const q_block_global_idx = k.programId(.x); + const split_idx = if (three_d) k.programId(.y) else k.liftAs(0, .i32); + const split_tile_start = split_idx.mul(@as(i32, @intCast(NUM_TILES))).div(@as(i32, @intCast(NUM_SPLITS))); + const split_tile_end = split_idx.add(1).mul(@as(i32, @intCast(NUM_TILES))).div(@as(i32, @intCast(NUM_SPLITS))); const q_ind = q_block_global_idx.div(@as(i32, @intCast(NUM_HEAD_BLOCKS))); const head_ind = q_block_global_idx.rem(@as(i32, @intCast(NUM_HEAD_BLOCKS))); const seq_idx = findSeqIdx(k, query_start_len_ptr, q_ind, num_seqs, BLOCK_Q, false); @@ -243,16 +345,18 @@ fn kernelUnifiedAttentionSparseMla2d( const l_init = k.full(&.{BLOCK_M}, 0.0, .f32); const acc_init = k.zeros(&.{ BLOCK_M, VALUE_RANK }, .f32); - var loop = k.openFor(0, NUM_TILES, 1, .{ m_init, l_init, acc_init }); + var loop = k.openFor(0, TILES_PER_SPLIT, 1, .{ m_init, l_init, acc_init }); { const t = loop.iv; const M = loop.carried[0]; const L = loop.carried[1]; const acc = loop.carried[2]; - const tile_start = t.mul(@as(i32, @intCast(TILE_SIZE))); + const tile_idx = split_tile_start.add(t); + const tile_start = tile_idx.mul(@as(i32, @intCast(TILE_SIZE))); const tile_offsets = tile_start.add(offs_t); - var valid_t = tile_offsets.lt(@as(i32, @intCast(config.topk_count))); + var valid_t = tile_offsets.lt(@as(i32, @intCast(config.topk_count))) + .bitAnd(tile_idx.lt(split_tile_end)); const topk_row_ptr = topk_indices_ptr.addPtr(q_ind.mul(@as(i32, @intCast(config.topk_count)))); const topk_pos = k.loadOpts(topk_row_ptr.addPtr(tile_start).addPtr(offs_t), .{ @@ -330,7 +434,7 @@ fn kernelUnifiedAttentionSparseMla2d( var L = loop.results[1]; var acc = loop.results[2]; - if (config.use_attn_sink) { + if (!three_d and config.use_attn_sink) { const sink_mask = query_mask_0.bitAnd(query_mask_1); const sink_logits = k.loadOpts(attn_sink_ptr.addPtr(query_offset_1), .{ .mask = sink_mask, @@ -349,8 +453,47 @@ fn kernelUnifiedAttentionSparseMla2d( L = L.mul(alpha).add(sink_p); } - const one_over_L = k.full(&.{ BLOCK_M, 1 }, 1.0, .f32).div(L.expandDims(1)); - acc = acc.mul(k.broadcastTo(one_over_L, &.{ BLOCK_M, VALUE_RANK })); + const has_value = L.gt(0.0); + const safe_l = k.where(has_value, L, k.full(&.{BLOCK_M}, 1.0, .f32)); + const one_over_l = k.full(&.{ BLOCK_M, 1 }, 1.0, .f32).div(safe_l.expandDims(1)); + acc = acc.mul(k.broadcastTo(one_over_l, &.{ BLOCK_M, VALUE_RANK })); + acc = k.where( + has_value.expandDims(1), + acc, + k.zeros(&.{ BLOCK_M, VALUE_RANK }, .f32), + ); + + if (three_d) { + // Compiler-managed workspace layouts are [query, head, split, value] + // and [query, head, split], both in f32. + const partial_out_token_stride: i64 = NUM_QUERY_HEADS * NUM_SPLITS * VALUE_RANK; + const partial_out_head_stride: i64 = NUM_SPLITS * VALUE_RANK; + const partial_out_offset = q_ind.to(.i64).mul(partial_out_token_stride) + .add(offs_m.expandDims(1).to(.i64).mul(partial_out_head_stride)) + .add(split_idx.to(.i64).mul(VALUE_RANK)) + .add(offs_value.expandDims(0).to(.i64)); + k.storeOpts( + output_ptr.addPtr(partial_out_offset), + acc, + .{ .mask = q_mask }, + ); + + const partial_lse_token_stride: i64 = NUM_QUERY_HEADS * NUM_SPLITS; + const partial_lse_offset = q_ind.to(.i64).mul(partial_lse_token_stride) + .add(offs_m.to(.i64).mul(NUM_SPLITS)) + .add(split_idx.to(.i64)); + const partial_lse = k.where( + has_value, + M.add(k.log(safe_l)), + k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32), + ); + k.storeOpts( + partial_lse_ptr.?.addPtr(partial_lse_offset), + partial_lse, + .{ .mask = query_mask_0.bitAnd(query_mask_1) }, + ); + return; + } const output_offs_lora = query_offset_0.expandDims(1).mul(output_stride_0) .add(query_offset_1.expandDims(1).mul(output_stride_1)) @@ -361,3 +504,93 @@ fn kernelUnifiedAttentionSparseMla2d( .{ .mask = q_mask }, ); } + +const Reduce3DCfg = struct { + sink_dtype: DType = .f32, + o_dtype: DType = .bf16, + num_query_heads: i64 = 32, + value_rank: i64 = 512, + num_splits: i64 = 1, + use_attn_sink: bool = false, +}; + +pub const Reduce3D = tri.Kernel(Reduce3DCfg, .{ + .name = "_kernel_reduce_sparse_mla_3d_ptr", + .inputs = &.{ + "partial_output_ptr", + "partial_lse_ptr", + "attn_sink_ptr", + "output_stride_0_ptr", + "output_stride_1_ptr", + }, + .outputs = &.{"output"}, + .run = runReduce3D, +}); + +fn runReduce3D(b: *tri.Builder, cfg: Reduce3DCfg) tri.FinishError!void { + const a = try b.declareArgs(.{ + .partial_output_ptr = .{ .ptr = .f32 }, + .partial_lse_ptr = .{ .ptr = .f32 }, + .attn_sink_ptr = .{ .ptr = cfg.sink_dtype }, + .output_stride_0_ptr = .{ .ptr = .i64 }, + .output_stride_1_ptr = .{ .ptr = .i64 }, + .output_ptr = .{ .ptr = cfg.o_dtype }, + }); + + const output_stride_0 = b.load(a.output_stride_0_ptr); + const output_stride_1 = b.load(a.output_stride_1_ptr); + const query_idx = b.programId(.x); + const head_idx = b.programId(.y); + const offs_s = b.arange(0, cfg.num_splits, .i32); + const offs_d = b.arange(0, cfg.value_rank, .i32); + + const partial_lse_token_stride: i64 = cfg.num_query_heads * cfg.num_splits; + const partial_lse_offset = query_idx.to(.i64).mul(partial_lse_token_stride) + .add(head_idx.to(.i64).mul(cfg.num_splits)) + .add(offs_s.to(.i64)); + const partial_lse = b.load(a.partial_lse_ptr.addPtr(partial_lse_offset)); + + const sink_score = if (cfg.use_attn_sink) + b.load(a.attn_sink_ptr.addPtr(head_idx)).to(.f32) + else + b.liftAs(-std.math.inf(f32), .f32); + const overall_max = b.max(partial_lse).maximum(sink_score); + const has_value = overall_max.gt(-std.math.inf(f32)); + const safe_max = b.where(has_value, overall_max, b.liftAs(0.0, .f32)); + const split_weights = b.exp(partial_lse.sub(safe_max)); + const sink_weight = if (cfg.use_attn_sink) + b.exp(sink_score.sub(safe_max)) + else + b.liftAs(0.0, .f32); + const denominator = b.sumOpts(split_weights, .{ .axis = 0 }).add(sink_weight); + + const partial_out_token_stride: i64 = cfg.num_query_heads * cfg.num_splits * cfg.value_rank; + const partial_out_head_stride: i64 = cfg.num_splits * cfg.value_rank; + const partial_out_base = query_idx.to(.i64).mul(partial_out_token_stride) + .add(head_idx.to(.i64).mul(partial_out_head_stride)); + const partial_out_offset = b.broadcastTo( + partial_out_base.add(offs_s.expandDims(1).to(.i64).mul(cfg.value_rank)), + &.{ cfg.num_splits, cfg.value_rank }, + ).add(b.broadcastTo(offs_d.expandDims(0).to(.i64), &.{ cfg.num_splits, cfg.value_rank })); + const partial_output = b.load(a.partial_output_ptr.addPtr(partial_out_offset)); + const weighted_output = partial_output.mul(split_weights.expandDims(1)); + const numerator = b.sumOpts(weighted_output, .{ .axis = 0 }); + const safe_denominator = b.where( + denominator.gt(0.0), + denominator, + b.liftAs(1.0, .f32), + ); + const output = b.where( + has_value, + numerator.div(safe_denominator), + b.zeros(&.{cfg.value_rank}, .f32), + ); + + const output_offset = query_idx.to(.i64).mul(output_stride_0) + .add(head_idx.to(.i64).mul(output_stride_1)) + .add(offs_d.to(.i64)); + b.store( + a.output_ptr.addPtr(output_offset), + output.to(cfg.o_dtype), + ); +} From 1b6a3cb3951c0a9b0a75525d3d9d7257e3f6ab0a Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 25 Aug 2026 16:47:49 +0200 Subject: [PATCH 627/881] zml/testing: apply close fraction across strided slices (#721) correctly accumulates error statistics across the full Buffer instead of checking each sub - slice independently. --- zml/slice.zig | 26 ++++ zml/testing.zig | 350 ++++++++++++++++++++++++++++-------------------- 2 files changed, 230 insertions(+), 146 deletions(-) diff --git a/zml/slice.zig b/zml/slice.zig index d642c620..238314ef 100644 --- a/zml/slice.zig +++ b/zml/slice.zig @@ -383,6 +383,32 @@ test "slice expectClose compares dense and strided slices" { try testing.expectClose(std.testing.io, strided, dense, .exact_match); } +test "slice expectClose applies close fraction across strided slices" { + const testing = @import("testing.zig"); + + const dense_data: [2][2]f32 = .{ + .{ 1, 2 }, + .{ 5, 6 }, + }; + const strided_data: [2][4]f32 = .{ + .{ 1, 3, 99, 98 }, + .{ 5, 6, 97, 96 }, + }; + + const dense = Slice.init(.init(.{ 2, 2 }, .f32), std.mem.asBytes(&dense_data)); + const storage = Slice.init(.init(.{ 2, 4 }, .f32), std.mem.asBytes(&strided_data)); + const strided = storage.subSlice(1, 0, 2); + const opts: testing.CompareOpts = .{ + .absolute_tolerance = 0, + .relative_tolerance = 0, + .minimum_close_fraction = 0.75, + }; + + try std.testing.expect(!strided.isContiguous()); + try testing.expectClose(std.testing.io, dense, strided, opts); + try testing.expectClose(std.testing.io, strided, dense, opts); +} + test "slice pretty print rank 3" { const data: [4][4][4]i32 = .{ .{ diff --git a/zml/testing.zig b/zml/testing.zig index 2ef8af78..8483a9be 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -99,8 +99,9 @@ pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, opts: CompareOpt .f8e5m2fnuz, => |rt| { const R = rt.toZigType(); - if (try compareIterators(false, allocator, L, R, left, right, opts)) |compare_report| { - try w.print("{f}\n", .{compare_report}); + const report: CompareOpts.Report = try .compareSlices(true, allocator, L, R, left, right, opts); + if (!report.ok(opts)) { + try w.print("{f}\n", .{report}); return error.TestUnexpectedResult; } return; @@ -110,7 +111,11 @@ pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, opts: CompareOpt }, inline .bool, .u2, .u4, .u8, .u16, .u32, .u64, .i2, .i4, .i8, .i16, .i32, .i64 => |t| { const T = t.toZigType(); - _ = try compareIterators(true, allocator, T, T, left, right, opts); + const report: CompareOpts.Report = try .compareSlices(false, allocator, T, T, left, right, opts); + if (!report.ok(opts)) { + try w.print("{f}\n", .{report}); + return error.TestUnexpectedResult; + } return; }, .c64, .c128 => @panic("TODO: support comparison of complex"), @@ -129,160 +134,213 @@ pub const CompareOpts = struct { .epsilon_relative = 0, .minimum_close_fraction = 1.00, }; -}; -pub const CompareReport = struct { - nan_or_inf: bool, - max_absolute_error: f32, - mean_absolute_error: f32, - rmse: f32, + pub const Report = struct { + nan_or_inf: bool, + max_absolute_error: f32, + mean_absolute_error: f32, + rmse: f32, - close_fraction: f32, + close_fraction: f32, - p50_absolute_error: f32, - p90_absolute_error: f32, - p99_absolute_error: f32, - p999_absolute_error: f32, + p50_absolute_error: f32, + p90_absolute_error: f32, + p99_absolute_error: f32, + p999_absolute_error: f32, - pub fn format( - self: @This(), - writer: *std.Io.Writer, - ) std.Io.Writer.Error!void { - try writer.print( - \\ nan_or_inf: {} - \\ max_absolute_error: {d} - \\ mean_absolute_error: {d} - \\ rmse: {d} - \\ close_fraction: {d} - \\ p50_absolute_error: {d} - \\ p90_absolute_error: {d} - \\ p99_absolute_error: {d} - \\ p999_absolute_error: {d} - , .{ - self.nan_or_inf, - self.max_absolute_error, - self.mean_absolute_error, - self.rmse, - self.close_fraction, - self.p50_absolute_error, - self.p90_absolute_error, - self.p99_absolute_error, - self.p999_absolute_error, - }); - } -}; - -fn compareReportOk(report: CompareReport, opts: CompareOpts) bool { - return !report.nan_or_inf and report.close_fraction >= opts.minimum_close_fraction; -} - -fn compareIterators(comptime mem_eql: bool, allocator: std.mem.Allocator, comptime L: type, comptime R: type, left: Slice, right: Slice, opts: CompareOpts) !?CompareReport { - var left_iter = (&left).contiguousItemsIterator(L); - var right_iter = (&right).contiguousItemsIterator(R); - var left_chunk: []const L = &.{}; - var right_chunk: []const R = &.{}; - - while (true) { - if (left_chunk.len == 0) { - left_chunk = left_iter.next() orelse &.{}; - } - if (right_chunk.len == 0) { - right_chunk = right_iter.next() orelse &.{}; + pub fn ok(report: Report, opts: CompareOpts) bool { + return !report.nan_or_inf and report.close_fraction >= opts.minimum_close_fraction; } - if (left_chunk.len == 0 or right_chunk.len == 0) { - std.debug.assert(left_chunk.len == right_chunk.len); - return null; - } + fn compareSlices(comptime float_cmp: bool, allocator: std.mem.Allocator, comptime L: type, comptime R: type, left: Slice, right: Slice, opts: CompareOpts) !CompareOpts.Report { + var left_iter = (&left).contiguousItemsIterator(L); + var right_iter = (&right).contiguousItemsIterator(R); + var left_chunk: []const L = &.{}; + var right_chunk: []const R = &.{}; + var acc: Accumulator = if (comptime float_cmp) try .init(allocator, left.shape.count()) else .initNoQuantiles(left.shape.count()); + defer acc.deinit(allocator); - const n = @min(left_chunk.len, right_chunk.len); - if (comptime mem_eql) { - if (!std.mem.eql(L, left_chunk[0..n], right_chunk[0..n])) { - return error.TestUnexpectedResult; - } - } else { - const compare_report = try compareSlices( - allocator, - L, - R, - left_chunk[0..n], - right_chunk[0..n], - opts, - ); - if (!compareReportOk(compare_report, opts)) { - return compare_report; + while (true) { + if (left_chunk.len == 0) { + left_chunk = left_iter.next() orelse &.{}; + } + if (right_chunk.len == 0) { + right_chunk = right_iter.next() orelse &.{}; + } + + if (left_chunk.len == 0 or right_chunk.len == 0) { + std.debug.assert(left_chunk.len == right_chunk.len); + return acc.report(); + } + + const n = @min(left_chunk.len, right_chunk.len); + if (comptime float_cmp) { + acc.cmpFloatSlices(L, R, left_chunk[0..n], right_chunk[0..n], opts); + } else { + acc.cmpIntSlices(L, left_chunk[0..n], right_chunk[0..n]); + } + + left_chunk = left_chunk[n..]; + right_chunk = right_chunk[n..]; } } - left_chunk = left_chunk[n..]; - right_chunk = right_chunk[n..]; - } -} - -pub fn compareSlices(allocator: std.mem.Allocator, comptime L: type, comptime R: type, left: []const L, right: []const R, opts: CompareOpts) !CompareReport { - var nan_or_inf: bool = false; - var max_absolute_error: f32 = 0; - var sum_absolute_error: f64 = 0; - var sum_squared_error: f64 = 0; - var count_close: usize = 0; - var absolute_errors = try allocator.alloc(f32, left.len); - defer allocator.free(absolute_errors); - var relative_errors = try allocator.alloc(f32, left.len); - defer allocator.free(relative_errors); - - for (left, right, 0..) |l, r, i| { - const l_f32 = zml.floats.floatCast(f32, l); - const r_f32 = zml.floats.floatCast(f32, r); - if (!std.math.isFinite(l_f32) or !std.math.isFinite(r_f32)) { - nan_or_inf = true; - continue; + pub fn format( + self: Report, + writer: *std.Io.Writer, + ) std.Io.Writer.Error!void { + try writer.print( + \\ nan_or_inf: {} + \\ max_absolute_error: {d} + \\ mean_absolute_error: {d} + \\ rmse: {d} + \\ close_fraction: {d} + \\ p50_absolute_error: {d} + \\ p90_absolute_error: {d} + \\ p99_absolute_error: {d} + \\ p999_absolute_error: {d} + , .{ + self.nan_or_inf, + self.max_absolute_error, + self.mean_absolute_error, + self.rmse, + self.close_fraction, + self.p50_absolute_error, + self.p90_absolute_error, + self.p99_absolute_error, + self.p999_absolute_error, + }); } - const absolute_error = @abs(l_f32 - r_f32); - max_absolute_error = @max(max_absolute_error, absolute_error); - sum_absolute_error += @as(f64, @floatCast(absolute_error)); - sum_squared_error += @as(f64, @floatCast(absolute_error)) * @as(f64, @floatCast(absolute_error)); - - const scale = @max(@abs(l_f32), @abs(r_f32)); - const tolerance = opts.absolute_tolerance + opts.relative_tolerance * scale; - if (absolute_error <= tolerance) { - count_close += 1; - } - - absolute_errors[i] = absolute_error; - const denom = @max(opts.epsilon_relative, scale); - relative_errors[i] = absolute_error / denom; - } - - std.sort.heap(f32, absolute_errors, {}, std.sort.asc(f32)); - std.sort.heap(f32, relative_errors, {}, std.sort.asc(f32)); - - const q = struct { - pub fn q(values: []const f32, frac: f32) f32 { - if (values.len == 0) return 0; - const idx: usize = @intFromFloat(std.math.round(@as(f32, @floatFromInt(values.len - 1)) * frac)); - return values[idx]; - } - }.q; - - const mean_absolute_error: f32 = @floatCast(sum_absolute_error / @as(f64, @floatFromInt(left.len))); - const rmse: f32 = @floatCast(std.math.sqrt(sum_squared_error / @as(f64, @floatFromInt(left.len)))); - const close_fraction = @as(f32, @floatFromInt(count_close)) / @as(f32, @floatFromInt(left.len)); - - const report: CompareReport = .{ - .nan_or_inf = nan_or_inf, - .max_absolute_error = max_absolute_error, - .mean_absolute_error = mean_absolute_error, - .rmse = rmse, - .close_fraction = close_fraction, - .p50_absolute_error = q(absolute_errors, 0.5), - .p90_absolute_error = q(absolute_errors, 0.9), - .p99_absolute_error = q(absolute_errors, 0.99), - .p999_absolute_error = q(absolute_errors, 0.999), }; - // TODO: Move - //const ok = !report.nan_or_inf and report.close_fraction >= opts.minimum_close_fraction; - return report; + const Accumulator = struct { + absolute_errors: []f32, + relative_errors: []f32, + expected_total_count: usize, + processed_count: usize = 0, + nan_or_inf: bool = false, + max_absolute_error: f32 = 0, + sum_absolute_error: f64 = 0, + sum_squared_error: f64 = 0, + count_close: usize = 0, + + fn init(allocator: std.mem.Allocator, count: usize) !Accumulator { + const absolute_errors = try allocator.alloc(f32, count); + errdefer allocator.free(absolute_errors); + const relative_errors = try allocator.alloc(f32, count); + + @memset(absolute_errors, 0); + @memset(relative_errors, 0); + return .{ + .expected_total_count = count, + .absolute_errors = absolute_errors, + .relative_errors = relative_errors, + }; + } + + fn initNoQuantiles(count: usize) Accumulator { + return .{ + .expected_total_count = count, + .absolute_errors = &.{}, + .relative_errors = &.{}, + }; + } + + fn deinit(self: *Accumulator, allocator: std.mem.Allocator) void { + allocator.free(self.absolute_errors); + allocator.free(self.relative_errors); + } + + fn cmpFloatSlices(self: *Accumulator, comptime L: type, comptime R: type, left: []const L, right: []const R, opts: CompareOpts) void { + std.debug.assert(left.len == right.len); + std.debug.assert(self.processed_count + left.len <= self.absolute_errors.len); + + const start = self.processed_count; + for (left, right, 0..) |l, r, i| { + const l_f32 = zml.floats.floatCast(f32, l); + const r_f32 = zml.floats.floatCast(f32, r); + if (!std.math.isFinite(l_f32) or !std.math.isFinite(r_f32)) { + self.nan_or_inf = true; + continue; + } + const absolute_error: f32 = @abs(l_f32 - r_f32); + self.max_absolute_error = @max(self.max_absolute_error, absolute_error); + self.sum_absolute_error += absolute_error; + self.sum_squared_error += absolute_error * absolute_error; + + const scale = @max(@abs(l_f32), @abs(r_f32)); + const tolerance = opts.absolute_tolerance + opts.relative_tolerance * scale; + if (absolute_error <= tolerance) { + self.count_close += 1; + } + + self.absolute_errors[start + i] = absolute_error; + const denom = @max(opts.epsilon_relative, scale); + self.relative_errors[start + i] = absolute_error / denom; + } + self.processed_count += left.len; + } + + fn cmpIntSlices(self: *Accumulator, comptime T: type, left: []const T, right: []const T) void { + std.debug.assert(left.len == right.len); + + for (left, right) |l, r| { + self.count_close += if (l == r) 1 else 0; + } + self.processed_count += left.len; + } + + fn report(self: *Accumulator) CompareOpts.Report { + std.debug.assert(self.processed_count == self.expected_total_count); + if (self.processed_count == 0) { + return .{ + .nan_or_inf = false, + .max_absolute_error = 0, + .mean_absolute_error = std.math.nan(f32), + .rmse = std.math.nan(f32), + .close_fraction = std.math.nan(f32), + .p50_absolute_error = 0, + .p90_absolute_error = 0, + .p99_absolute_error = 0, + .p999_absolute_error = 0, + }; + } + + std.sort.heap(f32, self.absolute_errors, {}, std.sort.asc(f32)); + std.sort.heap(f32, self.relative_errors, {}, std.sort.asc(f32)); + + const q = struct { + fn q(values: []const f32, frac: f32) f32 { + if (values.len == 0) return 0; + const idx: usize = @intFromFloat(std.math.round(@as(f32, @floatFromInt(values.len - 1)) * frac)); + return values[idx]; + } + }.q; + + const mean_absolute_error = stdx.math.divFloat(f64, self.sum_absolute_error, self.processed_count); + const rmse = std.math.sqrt(stdx.math.divFloat(f64, self.sum_squared_error, self.processed_count)); + const close_fraction = stdx.math.divFloat(f64, self.count_close, self.processed_count); + + return .{ + .nan_or_inf = self.nan_or_inf, + .max_absolute_error = self.max_absolute_error, + .mean_absolute_error = @floatCast(mean_absolute_error), + .rmse = @floatCast(rmse), + .close_fraction = @floatCast(close_fraction), + .p50_absolute_error = q(self.absolute_errors, 0.5), + .p90_absolute_error = q(self.absolute_errors, 0.9), + .p99_absolute_error = q(self.absolute_errors, 0.99), + .p999_absolute_error = q(self.absolute_errors, 0.999), + }; + } + }; +}; + +pub fn compareFloats(allocator: std.mem.Allocator, comptime L: type, comptime R: type, left: []const L, right: []const R, opts: CompareOpts) !CompareOpts.Report { + var acc: CompareOpts.Accumulator = try .init(allocator, left.len); + defer acc.deinit(allocator); + acc.cmpFloatSlices(true, L, R, left, right, opts); + return acc.report(); } pub fn expectEqualShapes(expected: zml.Shape, actual: zml.Shape) error{TestExpectedEqual}!void { @@ -418,7 +476,7 @@ pub fn testLayer( var failed: bool = false; var reader_buffer: [4096]u8 = undefined; for (0..output_count) |i| { - const expected_slice = b: { + const expected_slice = s: { var buffer: [256]u8 = undefined; const subkey = try std.fmt.bufPrint(&buffer, "{d}", .{i}); var reader = try store_output.getReader(subkey, io, &reader_buffer); @@ -427,7 +485,7 @@ pub fn testLayer( const expected_slice: zml.Slice = try .alloc(allocator, shape); errdefer expected_slice.free(allocator); try reader.interface.readSliceAll(expected_slice.data()); - break :b expected_slice; + break :s expected_slice; }; defer expected_slice.free(allocator); From c0dcc7cf7f8564340c704dfe180590fb7f5e3937 Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Wed, 26 Aug 2026 10:34:47 +0200 Subject: [PATCH 628/881] zml/nn: detect deepseek-like MXFP4 quantization (#726) --- zml/nn.zig | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index b2c7da54..2a112927 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -94,8 +94,8 @@ pub const Linear = struct { } }; -pub const nvfp4_block_size = 16; -pub const mx_block_size = 32; +const nvfp4_block_size = 16; +const mx_block_size = 32; pub const QuantScheme = enum { /// f4e2m1 values (`u8`-packed or native), f8e4m3fn scale per 16 contracted values. @@ -128,7 +128,7 @@ pub const QuantScheme = enum { scale.dim(0) == n and scale.dim(1) * nvfp4_block_size == k, .mxfp8 => weight.dtype() == .f8e4m3fn and isMxScale(scale) and scale.dim(0) == n and scale.dim(1) * mx_block_size == k, - .mxfp4 => (weight.dtype() == .u8 or weight.dtype() == .f4e2m1) and + .mxfp4 => (weight.dtype() == .u8 or weight.dtype() == .i8 or weight.dtype() == .f4e2m1) and isMxScale(scale) and scale.dim(0) == n and scale.dim(1) * mx_block_size == k, .fp8_per_tensor => weight.dtype() == .f8e4m3fn and scale.count() == 1, @@ -180,7 +180,7 @@ pub const ActivationQuant = enum { }; pub fn isPackedFp4(scheme: ?QuantScheme, weight_dtype: DataType) bool { - return (scheme == .nvfp4 or scheme == .mxfp4) and weight_dtype == .u8; + return (scheme == .nvfp4 or scheme == .mxfp4) and (weight_dtype == .u8 or weight_dtype == .i8); } fn isMxScale(scale: Shape) bool { @@ -257,6 +257,11 @@ test "QuantScheme.classify" { // an e8m0 scale per 32 -- so K = 2 * 2560 = 5120 and 5120 / 32 = 160. try expect(@as(?QuantScheme, .mxfp4), QuantScheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 160 }, .u8))); try expect(@as(?QuantScheme, .mxfp4), QuantScheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 160 }, .f8e8m0))); + // DeepSeek V4 stores the same packed FP4 bits in signed bytes. + try expect(@as(?QuantScheme, .mxfp4), QuantScheme.classify( + .init(.{ .dout = 2048, .kw = 2048 }, .i8), + .init(.{ .dout = 2048, .sc = 128 }, .f8e8m0), + )); // Native (unpacked) f4e2m1, K no longer halved. try expect(@as(?QuantScheme, .mxfp4), QuantScheme.classify( .init(.{ .dout = 17408, .d = 5120 }, .f4e2m1), @@ -305,7 +310,7 @@ test "QuantScheme.classify" { /// which is merged with the unpacked pair and renamed to `k_tag`. Group-size agnostic: /// NVFP4 (16) and MXFP4 (32) share this packing. pub fn unpackFp4(w: Tensor, k_tag: anytype) Tensor { - stdx.debug.assert(w.dtype() == .u8, "unpackFp4 expects packed u8 weights, got {}", .{w.dtype()}); + stdx.debug.assert(w.dtype() == .u8 or w.dtype() == .i8, "unpackFp4 expects packed 8-bit weights, got {}", .{w.dtype()}); return w.bitCast(.f4e2m1) // bitcast inserts a tag (it respects shlo), but maybe we should simplify it .merge(.{ .kb = .{ .kw, .bitcast } }) .renameTag(.kb, Shape.toTag(k_tag)); From a452efe972da39ad099fdf80784e0ed5982f82c0 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 26 Aug 2026 10:49:19 +0200 Subject: [PATCH 629/881] zig 0.17: replace ** by splat (#728) update in prevision of Zig 0.17 --- bin/zml-smi/platforms/neuron/process.zig | 2 +- bin/zml-smi/tui/data.zig | 2 +- zml/Sharding.zig | 4 ++-- zml/exe.zig | 2 +- zml/nn.zig | 3 ++- zml/ops.zig | 12 ++++++++---- zml/shape.zig | 4 ++-- zml/tensor.zig | 20 ++++++++++---------- zml/tokenizer/homemade.zig | 2 +- 9 files changed, 28 insertions(+), 23 deletions(-) diff --git a/bin/zml-smi/platforms/neuron/process.zig b/bin/zml-smi/platforms/neuron/process.zig index 70bab67b..2dbf363c 100644 --- a/bin/zml-smi/platforms/neuron/process.zig +++ b/bin/zml-smi/platforms/neuron/process.zig @@ -42,7 +42,7 @@ fn pollOnce(allocator: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer const apps = if (apps_result.ptr) |ptr| ptr[0..apps_result.count] else continue; - var total_us_per_core: [c.MAX_NC_PER_DEVICE]u64 = .{0} ** c.MAX_NC_PER_DEVICE; + var total_us_per_core: [c.MAX_NC_PER_DEVICE]u64 = @splat(0); for (apps) |*app| { if (app.pid <= 0) { diff --git a/bin/zml-smi/tui/data.zig b/bin/zml-smi/tui/data.zig index 43bb4d26..bc6171b7 100644 --- a/bin/zml-smi/tui/data.zig +++ b/bin/zml-smi/tui/data.zig @@ -24,7 +24,7 @@ pub fn RingBuffer(comptime T: type, comptime capacity: usize) type { return struct { const Self = @This(); - buf: [capacity]T = [_]T{0} ** capacity, + buf: [capacity]T = @splat(0), head: usize = 0, len: usize = 0, diff --git a/zml/Sharding.zig b/zml/Sharding.zig index cb52e542..58684561 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -791,7 +791,7 @@ pub const PhysicalMesh = struct { if (coord_placement.coords.len != rank) return error.InvalidDeviceCoordsRank; } - var axis_sizes = [_]usize{1} ** MAX_MESH_RANK; + var axis_sizes: [MAX_MESH_RANK]usize = @splat(1); for (0..rank) |ax_i| { var max_coord: usize = 0; for (placements) |coord_placement| { @@ -1025,7 +1025,7 @@ pub const PhysicalMesh = struct { }); } - var coords_buf: [MAX_MESH_RANK]usize = [_]usize{0} ** MAX_MESH_RANK; + var coords_buf: [MAX_MESH_RANK]usize = @splat(0); var next_device: usize = 0; const Node = struct { diff --git a/zml/exe.zig b/zml/exe.zig index 49a64b04..dfb85dbf 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -305,7 +305,7 @@ pub const Exe = struct { pub fn internalCall(self: *const Exe, io: ?std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { stdx.debug.assert(opts.wait == false or io != null, "io should not be null when waiting for execution completion", .{}); - var events = [_]?*pjrt.Event{null} ** Platform.MAX_NUM_DEVICES; + var events: [Platform.MAX_NUM_DEVICES]?*pjrt.Event = @splat(null); const partition_events = events[0..@intCast(self.num_partitions)]; const events_slice: ?[]?*pjrt.Event = switch (self.platform.target) { diff --git a/zml/nn.zig b/zml/nn.zig index 2a112927..1449a1be 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -1198,7 +1198,8 @@ test rope { var exe_sequential = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .real_im_pass } }, platform, .{}); defer exe_sequential.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]f32{ 1.0, 0.1, -1.0, -0.5 } ** 5)); + const x_values: [5][4]f32 = @splat(.{ 1.0, 0.1, -1.0, -0.5 }); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&x_values)); defer x_buffer.deinit(); var res1 = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_interleaved, Local._fwd, .{x_buffer}); diff --git a/zml/ops.zig b/zml/ops.zig index 1acc93e2..f88a8f94 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -69,7 +69,8 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@T if (num_devices <= 1) return inputs; const reducer_block = b: { - var args: std.meta.Tuple(&[1]type{ReduceArgs} ** input_tensors.len) = undefined; + const ArgsTypes: [input_tensors.len]type = @splat(ReduceArgs); + var args: std.meta.Tuple(&ArgsTypes) = undefined; var block_types: [2 * input_tensors.len]*const mlir.Type = undefined; inline for (0..input_tensors.len) |i| { @@ -213,7 +214,8 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func const mlir_ctx = CompilationContext.current().mlir_ctx; const reduce_block, var result = b: { - const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inits.len); + const ArgsTypes: [inits.len]type = @splat(ReduceArgs); + const ArgsType = std.meta.Tuple(&ArgsTypes); var args: ArgsType = undefined; var block_types: [2 * inits.len]*const mlir.Type = undefined; @@ -405,7 +407,8 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytyp const mlir_ctx = CompilationContext.current().mlir_ctx; const sort_block = b: { - const ArgsType = std.meta.Tuple(&[1]type{SortArgs} ** inputs.len); + const ArgsTypes: [inputs.len]type = @splat(SortArgs); + const ArgsType = std.meta.Tuple(&ArgsTypes); var args: ArgsType = undefined; var block_types: [2 * inputs.len]*const mlir.Type = undefined; @@ -1126,7 +1129,8 @@ pub fn scatter( const mlir_ctx = CompilationContext.current().mlir_ctx; const update_block, var result = b: { - const ArgsType = std.meta.Tuple(&[1]type{ScatterArgs} ** inputs.len); + const ArgsTypes: [inputs.len]type = @splat(ScatterArgs); + const ArgsType = std.meta.Tuple(&ArgsTypes); var args: ArgsType = undefined; var block_types: [2 * inputs.len]*const mlir.Type = undefined; diff --git a/zml/shape.zig b/zml/shape.zig index a2c7e9c9..75895fec 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -153,7 +153,7 @@ pub const Shape = struct { pub const AxesArray = stdx.BoundedArray(u3, constants.MAX_RANK); pub const PartitionArray = stdx.BoundedArray(PartitionSpec, constants.MAX_RANK); - const UnknownTags: TagsArray = .{ .len = 0, .buffer = [_]Tag{TagUnknown} ** constants.MAX_RANK }; + const UnknownTags: TagsArray = .{ .len = 0, .buffer = @splat(TagUnknown) }; _dtype: DataType, _dims: DimsArray = .empty, @@ -1636,7 +1636,7 @@ pub const Shape = struct { pub fn iterator(self: Shape) MultiDimIterator { return .{ .shape = self, - .current_coords = .{0} ** Shape.MAX_RANK, + .current_coords = @splat(0), .flat_index = 0, }; } diff --git a/zml/tensor.zig b/zml/tensor.zig index 148b6654..32c929d4 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1973,7 +1973,7 @@ pub const Tensor = struct { pub fn choose(self: Tensor, offsets: anytype) Tensor { const off, const tags = Shape.parseDimensions(offsets); - var slices = [_]Slice{.{}} ** constants.MAX_RANK; + var slices: [constants.MAX_RANK]Slice = @splat(.{}); for (off.constSlice(), tags.constSlice()) |o, t| { const ax = self.axis(t); slices[ax] = .single(o); @@ -2512,7 +2512,7 @@ pub const Tensor = struct { pub fn pad(self: Tensor, padding_value: anytype, paddings: anytype) Tensor { const _paddings = self.shape().parseAxesOptions(Pad, paddings, .{}); - const ZEROS = [_]i64{0} ** constants.MAX_RANK; + const ZEROS: [constants.MAX_RANK]i64 = @splat(0); var low = ZEROS; var high = ZEROS; var interior = ZEROS; @@ -2550,7 +2550,7 @@ pub const Tensor = struct { self.axis(axis_); var res_shape = self._shape; - const ones = [_]i64{1} ** constants.MAX_RANK; + const ones: [constants.MAX_RANK]i64 = @splat(1); res_shape._dims.insertSlice(ax, ones[0..tags_.len]) catch unreachable; res_shape._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; @@ -3466,7 +3466,7 @@ pub const Tensor = struct { // TODO: support maxPool on non last axis const a = self.axis(-1); - const ones = [_]i64{1} ** constants.MAX_RANK; + const ones: [constants.MAX_RANK]i64 = @splat(1); var window_dimensions = ones; window_dimensions[a] = opts.window_dimensions; var window_strides = window_dimensions; @@ -3477,7 +3477,7 @@ pub const Tensor = struct { var window_dilations = ones; window_dilations[a] = opts.window_dilations; - var padding = [_][2]i64{.{ 0, 0 }} ** constants.MAX_RANK; + var padding: [constants.MAX_RANK][2]i64 = @splat(.{ 0, 0 }); padding[a] = opts.padding; const values, const indices = ops.reduceWindow( @@ -3516,7 +3516,7 @@ pub const Tensor = struct { const base_dilation = initPoolArg(self.rank(), &opts.base_dilations); const window_dilations = initPoolArg(self.rank(), &opts.window_dilations); - var padding = [_][2]i64{.{ 0, 0 }} ** constants.MAX_RANK; + var padding: [constants.MAX_RANK][2]i64 = @splat(.{ 0, 0 }); padding[a - 1] = opts.padding[0]; padding[a] = opts.padding[1]; @@ -3714,7 +3714,7 @@ pub const Tensor = struct { const a = self.axis(axis_); const new_shape = self._shape.set(a, slice_.len); - var start_indices = [_]*const mlir.Value{constant(slice_.start.dtype().zero()).value()} ** constants.MAX_RANK; + var start_indices: [constants.MAX_RANK]*const mlir.Value = @splat(constant(slice_.start.dtype().zero()).value()); start_indices[a] = slice_.start.value(); const op = dialects.stablehlo.dynamic_slice( @@ -3755,7 +3755,7 @@ pub const Tensor = struct { const idx_dtype = if (slices.len > 0) slices.get(0).start.dtype() else .i32; const zero = Tensor.scalar(0, idx_dtype).value(); - var offset_values = [_]*const mlir.Value{zero} ** constants.MAX_RANK; + var offset_values: [constants.MAX_RANK]*const mlir.Value = @splat(zero); var res_shape = self._shape; for (slices.constSlice(), 0..) |slice_, i| { const offset = slice_.start; @@ -3812,7 +3812,7 @@ pub const Tensor = struct { /// Note this is the untagged api, if you have tags, you should use dynamicUpdateSlice directly. pub fn dynamicUpdateSlice1d(self: Tensor, update: Tensor, axis_: i64, offset: Tensor) Tensor { const placeholder = Tensor.scalar(0, .i32); - var start_indices = [_]Tensor{placeholder} ** constants.MAX_RANK; + var start_indices: [constants.MAX_RANK]Tensor = @splat(placeholder); start_indices[self.axis(axis_)] = offset; return self.dynamicUpdateSlice(start_indices[0..self.rank()], update); } @@ -3887,7 +3887,7 @@ pub const Tensor = struct { } else { // If an axis isn't specified, update the full slice. // This is only allowed when using tagged sliced. - offset_values = .{zero} ** constants.MAX_RANK; + offset_values = @splat(zero); for (offset.constSlice(), offset_tags.constSlice()) |start, t| { const a = self._shape.hasTag(t) orelse stdx.debug.panic("dynamicUpdateSlice expects input tensor to have tags used in 'offset_' but {s} is missing (input shape is {f})", .{ t, self._shape }); offset_values[a] = start.value(); diff --git a/zml/tokenizer/homemade.zig b/zml/tokenizer/homemade.zig index 079529b3..e2b142f7 100644 --- a/zml/tokenizer/homemade.zig +++ b/zml/tokenizer/homemade.zig @@ -839,7 +839,7 @@ pub const Normalizer = struct { const normalizer = try Normalizer.fromHfJson(config.object); const expected = Normalizer{ - ._whitespace = .{ .buffer = [_]u8{ 0xe2, 0x96, 0x81 } ++ [_]u8{0} ** 5, .len = 3 }, + ._whitespace = .{ .buffer = [_]u8{ 0xe2, 0x96, 0x81 } ++ @as([5]u8, @splat(0)), .len = 3 }, .flags = .{ .remove_extra_whitespaces = false, .add_dummy_prefix = true, From 98f20d0ec35eeb52c3f57bb5d15187363167186d Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 26 Aug 2026 10:50:14 +0200 Subject: [PATCH 630/881] zml.Slice: improve formatting (#723) --- zml/slice.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/slice.zig b/zml/slice.zig index 238314ef..ea3b8c14 100644 --- a/zml/slice.zig +++ b/zml/slice.zig @@ -239,7 +239,7 @@ pub const Slice = struct { slice: @This(), writer: *std.Io.Writer, ) std.Io.Writer.Error!void { - return writer.print("{any}", .{slice}); + return writer.print("Slice({f})@0x{x} [mut={}, off={d}, strides={any}]", .{ slice.shape, @intFromPtr(slice.bytes.ptr), slice.mutable, slice.offset_bytes, slice.byte_strides.slice() }); } pub fn formatNumber(slice: Slice, writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { From 3fe735f4915738ab6e979655f46734ba2763bf37 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 26 Aug 2026 16:54:04 +0200 Subject: [PATCH 631/881] zml: mlir known types (#722) cache base MLIR type corresponding to zml.DataType --- zml/mlirx.zig | 46 ---------------------------------------------- zml/module.zig | 23 +++++++++++++++++++++++ zml/tensor.zig | 3 ++- 3 files changed, 25 insertions(+), 47 deletions(-) diff --git a/zml/mlirx.zig b/zml/mlirx.zig index a2f52123..bc92eee0 100644 --- a/zml/mlirx.zig +++ b/zml/mlirx.zig @@ -44,50 +44,4 @@ pub const Type = struct { .c128 => .complex(ctx, .c128), }; } - - // TODO(Corentin): Maybe remove that as it now requires a mlir context. - // It's weird to have to provide a mlir context to get a zml.DataType from a mlir.Type. - pub fn toDType(ctx: *mlir.Context, mlir_type: *const mlir.Type) dtype.DataType { - const mapping = .{ - .{ .bool, mlir.Type.int(ctx, .i1) }, - - .{ .f8e8m0, mlir.Type.float(ctx, .f8e8m0fnu) }, - .{ .f8e4m3b11fnuz, mlir.Type.float(ctx, .f8e4m3b11fnuz) }, - .{ .f8e4m3fn, mlir.Type.float(ctx, .f8e4m3fn) }, - .{ .f8e4m3fnuz, mlir.Type.float(ctx, .f8e4m3fnuz) }, - .{ .f8e3m4, mlir.Type.float(ctx, .f8e3m4) }, - .{ .f8e5m2, mlir.Type.float(ctx, .f8e5m2) }, - .{ .f8e5m2fnuz, mlir.Type.float(ctx, .f8e5m2fnuz) }, - .{ .f4e2m1, mlir.Type.float(ctx, .f4e2m1fn) }, - .{ .f8e8m0, mlir.Type.float(ctx, .f8e8m0fnu) }, - .{ .bf16, mlir.Type.float(ctx, .bf16) }, - .{ .f16, mlir.Type.float(ctx, .f16) }, - .{ .f32, mlir.Type.float(ctx, .f32) }, - .{ .f64, mlir.Type.float(ctx, .f64) }, - - .{ .i4, mlir.Type.int(ctx, .i4) }, - .{ .i8, mlir.Type.int(ctx, .i8) }, - .{ .i16, mlir.Type.int(ctx, .i16) }, - .{ .i32, mlir.Type.int(ctx, .i32) }, - .{ .i64, mlir.Type.int(ctx, .i64) }, - - .{ .u4, mlir.Type.int(ctx, .u4) }, - .{ .u8, mlir.Type.int(ctx, .u8) }, - .{ .u16, mlir.Type.int(ctx, .u16) }, - .{ .u32, mlir.Type.int(ctx, .u32) }, - .{ .u64, mlir.Type.int(ctx, .u64) }, - - .{ .c64, mlir.Type.complex(ctx, .c64) }, - .{ .c128, mlir.Type.complex(ctx, .c128) }, - }; - - inline for (mapping) |entry| { - const dt, const mlirT = entry; - if (mlirT.eql(mlir_type)) { - return dt; - } - } - - std.debug.panic("Could not convert mlir.Type to DataType: {f}", .{mlir_type}); - } }; diff --git a/zml/module.zig b/zml/module.zig index d4610f27..14c451a5 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -8,6 +8,7 @@ const stdx = @import("stdx"); const upb = @import("upb"); const Buffer = @import("buffer.zig").Buffer; +const DataType = @import("dtype.zig").DataType; const Exe = @import("exe.zig").Exe; const Memory = @import("platform.zig").Memory; const meta = @import("meta.zig"); @@ -104,6 +105,8 @@ pub const CompilationContext = struct { platform: *const Platform, partitioning: Sharding.Partitioning, + mlir_known_types: std.enums.EnumArray(DataType, *const mlir.Type), + scopes: stdx.BoundedArray(Scope, 16) = .empty, manual_computation_depth: usize = 0, @@ -135,6 +138,13 @@ pub const CompilationContext = struct { } } + var mlir_known_types: std.enums.EnumArray(DataType, *const mlir.Type) = .initUndefined(); + { + for (0.., &mlir_known_types.values) |i, *mlir_type| { + mlir_type.* = mlirx.Type.fromDType(mlir_ctx, @enumFromInt(i)); + } + } + // Ensure replicated sharding is always included as a fallback option. var shardings = std.ArrayList(Sharding).initCapacity(arena.allocator(), opts.shardings.len + 1) catch @panic("OOM"); var needs_replicated: bool = true; @@ -153,6 +163,7 @@ pub const CompilationContext = struct { .mlir_registry = mlir_registry, .mlir_ctx = mlir_ctx, .mlir_pass_manager = pass_manager, + .mlir_known_types = mlir_known_types, .module = module, .platform = platform, .partitioning = partitioning, @@ -214,6 +225,18 @@ pub const CompilationContext = struct { return self.composite_id; } + pub fn mlirType(self: *const CompilationContext, dt: DataType) *const mlir.Type { + return self.mlir_known_types.get(dt); + } + + pub fn dtype(self: *const CompilationContext, mlir_type: *const mlir.Type) DataType { + @setRuntimeSafety(false); + for (0.., &self.mlir_known_types.values) |i, known_type| { + if (known_type == mlir_type) return @enumFromInt(i); + } + std.debug.panic("Can't convert unknown mlir type to dtype: {f}", .{mlir_type}); + } + pub fn abortOOM(self: *CompilationContext) noreturn { _ = self; @panic("OOM"); diff --git a/zml/tensor.zig b/zml/tensor.zig index 32c929d4..72269269 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -90,12 +90,13 @@ pub const Tensor = struct { /// /// The shape is derived from the type of the mlir.Value. pub fn fromMlirValue(val: *const mlir.Value) Tensor { + const ctx = CompilationContext.current(); const ranked_tensor = val.type_().isA(mlir.RankedTensorType).?; const n = ranked_tensor.rank(); stdx.debug.assert(n <= constants.MAX_RANK, "Can't represent MLIR tensor of rank {}, max supported rank is {}.", .{ n, constants.MAX_RANK }); - var sh: Shape = .{ ._dtype = mlirx.Type.toDType(mlirCtx(), ranked_tensor.elementType()) }; + var sh: Shape = .{ ._dtype = ctx.dtype(ranked_tensor.elementType()) }; for (0..n) |i| { sh._dims.appendAssumeCapacity(ranked_tensor.dimension(i)); } From 163a38767395d91c207bc8cbc6e2fc6d4221ef0a Mon Sep 17 00:00:00 2001 From: nzinci <43704979+nzinci@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:54:27 +0300 Subject: [PATCH 632/881] zml/attention: fix latent KV cache update (#727) Latent `KvCache.update` currently passes one tuple to `scatterSlices`, so the branch fails to compile when instantiated. Pass the three arguments directly and add a latent-cache compile regression test. Tests: - `bazel test --jobs=2 //zml:test` - `bazel build --jobs=2 //zml` --- zml/attention/paged_attention.zig | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index ada58259..7cb6b712 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -14,6 +14,7 @@ test { std.testing.refAllDecls(Backend); std.testing.refAllDecls(Options); std.testing.refAllDecls(Parameters); + std.testing.refAllDecls(KvCache); } pub const Backend = enum { @@ -246,11 +247,11 @@ pub const KvCache = union(enum) { }, .dense => @panic("TODO"), .latent => |latent_kv| .{ - .latent = latent_kv.scatterSlices(.{ + .latent = latent_kv.scatterSlices( .{ .page = page_index, .k_chunk = offset }, new_k, .{ .update_fn = zml.Tensor.ScatterOpts.override, .indices_are_unique = false, .indices_are_sorted = false }, - }).reuseBuffer(self.latent), + ).reuseBuffer(self.latent), }, }; From a05d7f920acaac72ca04ffc5ad6132216cabb13a Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Thu, 27 Aug 2026 11:13:09 +0200 Subject: [PATCH 633/881] mlir/func: support up to 64 results (#731) --- mlir/dialects/func.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlir/dialects/func.zig b/mlir/dialects/func.zig index aee70d49..93407bd7 100644 --- a/mlir/dialects/func.zig +++ b/mlir/dialects/func.zig @@ -24,7 +24,7 @@ const FuncOpArgs = struct { pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { var args_buffer: stdx.BoundedArray(*const mlir.Type, 1024) = .empty; - var results_buffer: stdx.BoundedArray(*const mlir.Type, 32) = .empty; + var results_buffer: stdx.BoundedArray(*const mlir.Type, 64) = .empty; var attr_tuples_buffer: stdx.BoundedArray(mlir.NamedAttribute, 16) = .empty; attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "sym_name", .string(ctx, args.name))); From 8cb020ca437249a4262e32c3651c83f93ae59240 Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Thu, 27 Aug 2026 13:58:21 +0200 Subject: [PATCH 634/881] zml/moe: introduce A16WMXFP4 kernel (#730) Changes: - add `Options` struct to the MoE API. - `num_experts_per_tok` field is now mandatory. - updated qwen 3.5 moe to use the new api. --- examples/llm/models/qwen3_5_moe/inference.zig | 8 +- examples/llm/models/qwen3_5_moe/model.zig | 7 +- zml/BUILD.bazel | 1 + zml/moe/moe.zig | 146 ++-- zml/moe/triton.zig | 685 ++++++++++++++++-- zml/moe/triton_kernels/a16w4_kernel.zig | 189 +++++ 6 files changed, 922 insertions(+), 114 deletions(-) create mode 100644 zml/moe/triton_kernels/a16w4_kernel.zig diff --git a/examples/llm/models/qwen3_5_moe/inference.zig b/examples/llm/models/qwen3_5_moe/inference.zig index 464f6511..3a3b9da2 100644 --- a/examples/llm/models/qwen3_5_moe/inference.zig +++ b/examples/llm/models/qwen3_5_moe/inference.zig @@ -357,17 +357,13 @@ fn findFirstLayerIndex(layer_types: []const model.LayerType, target: model.Layer } fn initMoeMetadata(qwen_model: model.Model, token_len: usize, batch_size: u32, backend: zml.moe.Backend) zml.moe.Metadata { - if (qwen_model.config.text_config.num_experts_per_tok == null) { - return .init(.fromBackend(backend)); - } - var w1_zero_bias_shape: ?zml.Shape = null; var w2_zero_bias_shape: ?zml.Shape = null; var first_out_shape: ?zml.Shape = null; var second_out_shape: ?zml.Shape = null; - const num_experts_per_tok = qwen_model.config.text_config.num_experts_per_tok.?; - const num_experts = qwen_model.config.text_config.num_experts.?; + const num_experts_per_tok = qwen_model.config.text_config.num_experts_per_tok; + const num_experts = qwen_model.config.text_config.num_experts; for (qwen_model.text_model.layers) |layer| { const gate_up_shape = zml.Shape.init(.{ diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 09ff549e..8f32e124 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -31,8 +31,8 @@ pub const TextConfig = struct { linear_num_value_heads: i64, linear_value_head_dim: i64, // MoE - num_experts: ?i64 = null, - num_experts_per_tok: ?u32 = null, + num_experts: i64, + num_experts_per_tok: u32, }; // Each layer uses either: full attention (SelfAttn) or linear attention (GatedDeltaNet). @@ -783,7 +783,7 @@ pub const Moe = struct { ), .gate_up_proj = gate_up_proj_tensor, .down_proj = down_proj_tensor, - .router = Router.init(store.withPrefix("gate"), config.text_config.num_experts_per_tok.?), + .router = Router.init(store.withPrefix("gate"), config.text_config.num_experts_per_tok), }; } @@ -807,6 +807,7 @@ pub const Moe = struct { null, null, null, + .{}, moe_metadata, moe_parameters, ) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 4267f641..b8f5432e 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -106,6 +106,7 @@ zig_library( "moe/mosaic_tpu.zig", "moe/mosaic_tpu_kernels/gmm_ep.zig", "moe/triton.zig", + "moe/triton_kernels/a16w4_kernel.zig", "moe/triton_kernels/triton_kernels.zig", "nn.zig", "ops.zig", diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index 559568b1..6f3e684c 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -32,7 +32,7 @@ pub const Backend = enum { .flashinfer_cutlass else return error.UnsupportedDataType, - .f16, .f32 => .triton, + .f8e8m0, .f16, .f32 => .triton, else => error.UnsupportedDataType, }, .rocm, .oneapi => switch (weights_dtype) { @@ -85,10 +85,10 @@ pub const Parameters = union(Backend) { mosaic_tpu: mosaic_tpu.Parameters.InitOptions, metal: metal.Parameters.InitOptions, - pub fn fromBackend(backend: Backend, num_experts_per_tok: ?u32, activation: ActivationMode) InitOptions { + pub fn fromBackend(backend: Backend, num_experts_per_tok: u32, activation: ActivationMode) InitOptions { return switch (backend) { .flashinfer_cutlass => .{ .flashinfer_cutlass = .{ - .num_experts_per_tok = num_experts_per_tok.?, + .num_experts_per_tok = num_experts_per_tok, .activation = switch (activation) { .silu => .silu, .relu => .relu, @@ -96,7 +96,7 @@ pub const Parameters = union(Backend) { }, } }, .triton => .{ .triton = .{ - .num_experts_per_tok = num_experts_per_tok.?, + .num_experts_per_tok = num_experts_per_tok, .activation = switch (activation) { .silu => .silu, .relu => .relu, @@ -104,7 +104,7 @@ pub const Parameters = union(Backend) { }, } }, .mosaic_tpu => .{ .mosaic_tpu = .{ - .num_experts_per_tok = num_experts_per_tok.?, + .num_experts_per_tok = num_experts_per_tok, .activation = switch (activation) { .silu => .silu, .relu => .relu, @@ -112,7 +112,7 @@ pub const Parameters = union(Backend) { }, } }, .metal => .{ .metal = .{ - .num_experts_per_tok = num_experts_per_tok.?, + .num_experts_per_tok = num_experts_per_tok, .activation = switch (activation) { .silu => .silu, .relu => .relu, @@ -183,6 +183,11 @@ pub const Metadata = union(Backend) { } }; +pub const Options = struct { + activation_threshold: ?f32 = null, + quant_scheme: ?zml.nn.QuantScheme = null, +}; + pub fn forwardMoe( input: zml.Tensor, topk_ids: zml.Tensor, @@ -195,6 +200,7 @@ pub fn forwardMoe( bias_down: ?zml.Tensor, w1_global_scale: ?zml.Tensor, w2_global_scale: ?zml.Tensor, + opts: Options, metadata: Metadata, parameters: Parameters, ) !zml.Tensor { @@ -391,76 +397,88 @@ pub fn forwardMoe( else => return error.InvalidMetadata, }; + const global_num_experts = weights_gate_up.dim(.expert); const expert_partition = weights_gate_up.shape().partition(.expert); - if (expert_partition.eql(.init(.experts))) { - const global_num_experts = weights_down.dim(.expert); - - break :b zml.ops.manualComputation( - .{ input, topk_ids, topk_weights, weights_gate_up, weights_down }, - input.shape(), + if (!expert_partition.eql(.init(.experts))) { + break :b try triton.fusedExpertsImpl( + input, + weights_gate_up, + weights_down, + topk_weights, + topk_ids, + triton_metadata, .{ .activation = parameters.triton.activation, .global_num_experts = global_num_experts, - .scales_gate_up = scales_gate_up, - .bias_gate_up = bias_gate_up, - .scales_down = scales_down, - .bias_down = bias_down, + .w1_scale = scales_gate_up, + .w2_scale = scales_down, + .w1_bias = bias_gate_up, + .w2_bias = bias_down, + .quant_scheme = opts.quant_scheme, + .activation_threshold = opts.activation_threshold, }, - (struct { - fn body(ctx: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, _: zml.Shape) zml.Tensor { - const local_num_experts = sharded_inputs[3].dim(.expert); - const partition_id = zml.ops.partitionId().convert(.i32); - const expert_start = partition_id.scale(local_num_experts).convert(.i32); - // List of global expert ids - const global_expert_ids = zml.Tensor.arange(.{ .end = ctx.global_num_experts }, .i32).withTags(.{.expert}); - - // Mapping of local experts to global expert ids, -1 if the global expert is not present in the local partition - const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) - .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); - const expert_map = local_expert_mask.select( - global_expert_ids.sub(expert_start), - zml.Tensor.scalar(-1, .i32), - ); - const local_output = triton.fusedExpertsImpl( - sharded_inputs[0], - sharded_inputs[3], - sharded_inputs[4], - sharded_inputs[2], - sharded_inputs[1], - .{}, - .{ - .activation = ctx.activation, - .global_num_experts = ctx.global_num_experts, - .expert_map = expert_map, - .w1_scale = ctx.scales_gate_up, - .w2_scale = ctx.scales_down, - .w1_bias = ctx.bias_gate_up, - .w2_bias = ctx.bias_down, - }, - ) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); - const local_reshaped = local_output.reshape(sharded_inputs[0].shape().dims()).withTags(.{ .b, .s, .d }); - return zml.ops.allReduce(local_reshaped, zml.Tensor.add); - } - }).body, ); } - break :b try triton.fusedExpertsImpl( - input, - weights_gate_up, - weights_down, - topk_weights, - topk_ids, - triton_metadata, + break :b zml.ops.manualComputation( + .{ + input, + topk_ids, + topk_weights, + weights_gate_up, + weights_down, + scales_gate_up, + scales_down, + }, + input.shape(), .{ .activation = parameters.triton.activation, - .global_num_experts = weights_gate_up.dim(.expert), - .w1_scale = scales_gate_up, - .w2_scale = scales_down, - .w1_bias = bias_gate_up, - .w2_bias = bias_down, + .global_num_experts = global_num_experts, + .bias_gate_up = bias_gate_up, + .bias_down = bias_down, + .quant_scheme = opts.quant_scheme, + .activation_threshold = opts.activation_threshold, }, + (struct { + fn body(ctx: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, _: zml.Shape) zml.Tensor { + const local_num_experts = sharded_inputs[3].dim(.expert); + const partition_id = zml.ops.partitionId().convert(.i32); + const expert_start = partition_id.scale(local_num_experts).convert(.i32); + // List of global expert ids + const global_expert_ids = zml.Tensor.arange(.{ .end = ctx.global_num_experts }, .i32).withTags(.{.expert}); + + // Mapping of local experts to global expert ids, -1 if the global expert is not present in the local partition + const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) + .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); + const expert_map = local_expert_mask.select( + global_expert_ids.sub(expert_start), + zml.Tensor.scalar(-1, .i32), + ); + + const local_output = triton.fusedExpertsImpl( + sharded_inputs[0], + sharded_inputs[3], + sharded_inputs[4], + sharded_inputs[2], + sharded_inputs[1], + .{}, + .{ + .activation = ctx.activation, + .global_num_experts = ctx.global_num_experts, + .expert_map = expert_map, + .w1_scale = sharded_inputs[5], + .w2_scale = sharded_inputs[6], + .w1_bias = ctx.bias_gate_up, + .w2_bias = ctx.bias_down, + .quant_scheme = ctx.quant_scheme, + .activation_threshold = ctx.activation_threshold, + }, + ) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); + const local_reshaped = local_output.reshape(sharded_inputs[0].shape().dims()).withTags(.{ .b, .s, .d }); + return zml.ops.allReduce(local_reshaped, zml.Tensor.add); + } + }).body, ); }, .mosaic_tpu => b: { diff --git a/zml/moe/triton.zig b/zml/moe/triton.zig index 2d00a2b5..699e3cce 100644 --- a/zml/moe/triton.zig +++ b/zml/moe/triton.zig @@ -12,6 +12,7 @@ const tri = zml.kernel.triton; const DType = tri.DType; const toDType = tri.from; const kernels = @import("triton_kernels/triton_kernels.zig"); +const a16w4_kernel = @import("triton_kernels/a16w4_kernel.zig"); const log = std.log.scoped(.moe_triton); @@ -23,12 +24,7 @@ pub const Options = struct { inplace: bool = false, activation: Parameters.ActivationMode = .silu, apply_router_weight_on_input: bool = false, - use_fp8_w8a8: bool = false, - use_int8_w8a8: bool = false, - use_int8_w8a16: bool = false, - use_int4_w4a16: bool = false, - ocp_mx_scheme: ?[]const u8 = null, - per_channel_quant: bool = false, + quant_scheme: ?zml.nn.QuantScheme = null, global_num_experts: i64 = -1, expert_map: ?Tensor = null, w1_scale: ?Tensor = null, @@ -47,6 +43,7 @@ pub const Options = struct { num_warps: i64 = 8, num_stages: i64 = 4, dynamic_launch_by_num_tokens: bool = true, + activation_threshold: ?f32 = null, }; pub const Parameters = struct { @@ -124,6 +121,10 @@ fn applyActivation(x: Tensor, mode: Parameters.ActivationMode) Tensor { // Top-level entry point // ============================================================================= +fn isMxFp8(quant_scheme: ?zml.nn.QuantScheme) bool { + return quant_scheme != null and quant_scheme.? == .mxfp8; +} + pub fn fusedExpertsImpl( hidden_states: Tensor, w1: Tensor, @@ -134,6 +135,36 @@ pub fn fusedExpertsImpl( opts: Options, ) !Tensor { try validateOptions(opts); + + if (opts.quant_scheme != null and opts.quant_scheme.? == .mxfp4) { + const local_topk_ids, const local_topk_weights = if (opts.expert_map) |expert_map| blk: { + const local_num_experts = w1.dim(.expert); + const mapped_ids = expert_map + .gather(.{ .expert = topk_ids.convert(.i32) }, .{}) + .withTags(topk_ids.shape().tags()); + const in_range = mapped_ids.cmp(.GE, Tensor.scalar(0, .i32)) + .logical(.AND, mapped_ids.cmp(.LT, Tensor.scalar(local_num_experts, .i32))); + + break :blk .{ + in_range.select(mapped_ids, Tensor.scalar(local_num_experts, .i32)), + in_range.select(topk_weights, Tensor.scalar(0, topk_weights.dtype())), + }; + } else .{ topk_ids, topk_weights }; + + return fusedExpertsImpl_fp4( + hidden_states, + local_topk_ids, + local_topk_weights, + w1, + opts.w1_scale.?, + opts.w1_bias, + w2, + opts.w2_scale.?, + opts.w2_bias, + opts.activation_threshold, + ); + } + const options = applyJsonTokenConfig(opts, hidden_states.dim(0)) catch |err| fallback: { log.warn("Failed to load MoE launch config from JSON ({}), falling back to built-in token heuristic", .{err}); break :fallback applyDefaultTokenConfig(opts, hidden_states.dim(0), w1.dim(0)); @@ -155,29 +186,12 @@ pub fn fusedExpertsImpl( if (expert_map.dtype() != .i32) return error.UnsupportedType; if (expert_map.rank() != 1 or expert_map.dim(.expert) != num_experts) return error.InvalidShape; } - const num_assignments = hidden.dim(.token) * ids.dim(.topk); - const sparsity_factor: i64 = 4; - const naive_block_assignment = num_assignments * sparsity_factor <= num_experts; - - const max_num_tokens_padded = if (naive_block_assignment) - num_assignments * block_size_m - else if (num_assignments < num_experts) - num_assignments * block_size_m - else - num_assignments + num_experts * (block_size_m - 1); - - const sorted_token_ids, const expert_ids_global, const num_tokens_post_padded = if (naive_block_assignment) blk: { - log.debug("Using naive block assignment for MoE kernels. Num assignments: {d}, Num experts: {d}", .{ num_assignments, num_experts }); - const naive_sorted_ids = Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)); - const naive_expert_ids = ids.reshape(.{ .g = num_assignments }); - const naive_num_tokens_post_padded = Tensor.constant(.{ .i32 = @as(i32, @intCast(max_num_tokens_padded)) }).reshape(.{1}); - break :blk .{ naive_sorted_ids, naive_expert_ids, naive_num_tokens_post_padded }; - } else alignBlockSize(ids, num_experts, block_size_m); + const routing = prepareRouting(ids, num_experts, block_size_m); const expert_ids = if (opts.expert_map) |expert_map| - expert_map.gather(.{ .expert = expert_ids_global }, .{}).withTags(.{.g}) + expert_map.gather(.{ .expert = routing.expert_ids }, .{}).withTags(.{.g}) else - expert_ids_global; + routing.expert_ids; var hidden_quant = hidden; var a_scale = opts.a1_scale orelse Tensor.scalar(1.0, .f32); @@ -190,7 +204,7 @@ pub fn fusedExpertsImpl( hidden_quant, gate_up, options, - naive_block_assignment, + routing.naive_block_assignment, ids.dim(.topk), false, false, @@ -211,14 +225,14 @@ pub fn fusedExpertsImpl( a_scale, b_scale_1, weights, - sorted_token_ids, + routing.sorted_token_ids, expert_ids, - num_tokens_post_padded, + routing.num_tokens_post_padded, first_cfg, options, - max_num_tokens_padded, - num_assignments, - Shape.init(.{ .token = num_assignments, .out = gate_up.dim(.out) }, .bf16), + routing.max_num_tokens_padded, + routing.num_assignments, + Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), ); const activated = applyActivation(first_out, options.activation); @@ -233,7 +247,7 @@ pub fn fusedExpertsImpl( activated_quant, down, options, - naive_block_assignment, + routing.naive_block_assignment, 1, true, false, @@ -254,13 +268,13 @@ pub fn fusedExpertsImpl( a_scale, b_scale_2, weights, - sorted_token_ids, + routing.sorted_token_ids, expert_ids, - num_tokens_post_padded, + routing.num_tokens_post_padded, second_cfg, options, - max_num_tokens_padded, - num_assignments, + routing.max_num_tokens_padded, + routing.num_assignments, Shape.init(.{ .token = b * s, .topk = ids.dim(.topk), .out = down.dim(.out) }, .bf16), ); @@ -343,6 +357,46 @@ fn callFusedMoe( ).c; } +const Routing = struct { + sorted_token_ids: Tensor, + expert_ids: Tensor, + num_tokens_post_padded: Tensor, + max_num_tokens_padded: i64, + num_assignments: i64, + naive_block_assignment: bool, +}; + +fn prepareRouting(topk_ids: Tensor, num_experts: i64, block_size_m: i64) Routing { + const ids = topk_ids.withTags(.{ .token, .topk }).convert(.i32); + const num_assignments = ids.dim(.token) * ids.dim(.topk); + const sparsity_factor: i64 = 4; + const naive_block_assignment = num_assignments * sparsity_factor <= num_experts; + const max_num_tokens_padded = if (naive_block_assignment) + num_assignments * block_size_m + else if (num_assignments < num_experts) + num_assignments * block_size_m + else + num_assignments + num_experts * (block_size_m - 1); + + const sorted_token_ids, const expert_ids, const num_tokens_post_padded = if (naive_block_assignment) blk: { + log.debug("Using naive block assignment for MoE kernels. Num assignments: {d}, Num experts: {d}", .{ num_assignments, num_experts }); + break :blk .{ + Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)), + ids.reshape(.{ .g = num_assignments }), + Tensor.constant(.{ .i32 = @as(i32, @intCast(max_num_tokens_padded)) }).reshape(.{1}), + }; + } else alignBlockSize(ids, num_experts, block_size_m); + + return .{ + .sorted_token_ids = sorted_token_ids, + .expert_ids = expert_ids, + .num_tokens_post_padded = num_tokens_post_padded, + .max_num_tokens_padded = max_num_tokens_padded, + .num_assignments = num_assignments, + .naive_block_assignment = naive_block_assignment, + }; +} + fn alignBlockSize(topk_ids: Tensor, num_experts: i64, block_size_m: i64) struct { Tensor, Tensor, Tensor } { log.debug("Using triton kernels to sort and align tokens to experts with block size {d}", .{block_size_m}); const topk_ids_ = topk_ids.withTags(.{ .token, .topk }).convert(.i32); @@ -490,7 +544,7 @@ fn makeFusedMoeConfig( has_bias: bool, output_dtype: DataType, ) kernels.FusedMoe.Cfg { - var use_fp8 = opts.use_fp8_w8a8; + var use_fp8 = isMxFp8(opts.quant_scheme); if (b.dtype() == .f8e4m3fn) use_fp8 = true; return .{ .a_dtype = toDType(a.dtype()), @@ -544,7 +598,7 @@ fn applyDefaultTokenConfig(opts: Options, num_tokens: i64, num_experts: i64) Opt } out.block_size_n = if (num_tokens <= 64) 64 else 128; - out.block_size_k = if (opts.use_fp8_w8a8 or num_tokens <= 64) 128 else 64; + out.block_size_k = if (isMxFp8(opts.quant_scheme) or num_tokens <= 64) 128 else 64; const tokens_per_expert = @divFloor(num_tokens, @max(num_experts, 1)); out.group_size_m = if (tokens_per_expert > 128) 16 else 1; @@ -641,8 +695,7 @@ fn fp8ActivationGroupSize(x: Tensor) i64 { fn validateOptions(opts: Options) !void { if (opts.inplace) return error.Unimplemented; if (opts.apply_router_weight_on_input) return error.UnsupportedOption; - if (opts.use_fp8_w8a8 or opts.use_int8_w8a8 or opts.use_int8_w8a16 or opts.use_int4_w4a16) return error.UnsupportedQuantization; - if (opts.ocp_mx_scheme != null or opts.per_channel_quant) return error.UnsupportedOption; + if (opts.quant_scheme != null and opts.quant_scheme.? != .mxfp4) return error.UnsupportedQuantization; if (opts.expert_map != null and opts.global_num_experts == -1) return error.InvalidShape; if (opts.w1_zp != null or opts.w2_zp != null) return error.UnsupportedOption; if (opts.a1_scale != null or opts.a2_scale != null or opts.block_shape != null) return error.UnsupportedOption; @@ -662,3 +715,553 @@ fn validateInputs(hidden: Tensor, gate_up: Tensor, down: Tensor, weights: Tensor if (ids.dim(.topk) != weights.dim(.topk)) return error.InvalidShape; if (gate_up.dim(.expert) != down.dim(.expert)) return error.InvalidShape; } + +// ===== +// A16W4 +// ===== +pub fn fusedExpertsImpl_fp4( + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + weights_gate_up: zml.Tensor, + scales_gate_up: zml.Tensor, + bias_gate_up: ?zml.Tensor, + weights_down: zml.Tensor, + scales_down: zml.Tensor, + bias_down: ?zml.Tensor, + activation_limit: ?f32, +) !zml.Tensor { + const x = input.reshape(.{ + .token = @divExact(@as(i64, @intCast(input.count())), input.dim(.d)), + .d = input.dim(.d), + }); + const num_tokens = x.dim(.token); + const num_routes: i64 = @intCast(topk_ids.count()); + stdx.debug.assert(@mod(num_routes, num_tokens) == 0, "expected {} routing ids to be divisible by {} tokens", .{ num_routes, num_tokens }); + stdx.debug.assert(topk_weights.count() == topk_ids.count(), "expected matching routing id and weight counts, got {} and {}", .{ topk_ids.count(), topk_weights.count() }); + const topk = @divExact(num_routes, num_tokens); + const flat_topk_ids = topk_ids.reshape(.{ .token = num_tokens, .topk = topk }); + const flat_topk_weights = topk_weights.reshape(.{ .token = num_tokens, .topk = topk }); + const kernel_cfg = getBestConfig( + @intCast(num_tokens), + @intCast(topk), + @intCast(weights_gate_up.dim(.expert)), + ); + const num_experts = weights_gate_up.dim(.expert); + const aligned_routing = prepareRouting(flat_topk_ids, num_experts, @intCast(kernel_cfg.block_m)); + const routing = prepareFp4Routing( + aligned_routing, + flat_topk_ids, + flat_topk_weights, + num_tokens, + num_experts, + @intCast(kernel_cfg.block_m), + ); + + const hidden_shape: zml.Shape = .init(.{ + .route = routing.num_rows, + .dout = @divExact(weights_gate_up.dim(.dout), 2), + }, .bf16); + + const hidden = try runGemm( + x, + weights_gate_up, + scales_gate_up, + .{ + .routing = routing, + .weight_contract_tag = zml.Shape.toTag(.d), + .weight_output_tag = zml.Shape.toTag(.dout), + .output_shape = hidden_shape, + .gather = routing.sorted_route_ids, + .gammas = routing.sorted_weights, + .bias = bias_gate_up, + .apply_swiglu = true, + .activation_limit = activation_limit, + .block_m = kernel_cfg.block_m, + .block_n = kernel_cfg.block_n, + .block_k = kernel_cfg.block_k, + .group_m = kernel_cfg.group_m, + .num_warps = kernel_cfg.num_warps, + .num_stages = kernel_cfg.num_stages, + }, + ); + + const routed_shape: zml.Shape = .init(.{ + .route = routing.num_rows, + .d = weights_down.dim(.d), + }, .bf16); + + const routed = try runGemm( + hidden, + weights_down, + scales_down, + .{ + .routing = routing, + .weight_contract_tag = zml.Shape.toTag(.dout), + .weight_output_tag = zml.Shape.toTag(.d), + .output_shape = routed_shape, + .bias = bias_down, + .apply_swiglu = false, + .activation_limit = 1.0, + .block_m = kernel_cfg.block_m, + .block_n = kernel_cfg.block_n, + .block_k = kernel_cfg.block_k, + .group_m = kernel_cfg.group_m, + .num_warps = kernel_cfg.num_warps, + .num_stages = kernel_cfg.num_stages, + }, + ); + + const active_routed = routing.active_routes.broad(routed.shape().withDtype(.bool)).select( + routed, + zml.Tensor.zeroes(routed.shape()), + ); + const token_ids = routing.sorted_route_ids.divByConst(routing.topk).withTags(.{.route}); + const output_flat_shape: zml.Shape = .init(.{ .token = routing.num_tokens, .d = input.dim(.d) }, .f32); + const output_flat = zml.Tensor.zeroes(output_flat_shape).scatterSlices( + .{ .token = token_ids }, + active_routed.convert(.f32), + .{}, + ); + + return output_flat.reshape(input.shape().withDtype(.f32)).convert(input.dtype()); +} + +const KernelConf = struct { + block_m: u32, + block_n: u32, + block_k: u32, + group_m: u32, + num_warps: u32, + num_stages: u32, +}; + +const kernel_config_token_buckets = [_]u32{ + 1, 2, 4, 8, 16, 24, 32, 48, 64, + 96, 128, 256, 512, 1024, 1536, 2048, 3072, 4096, +}; + +fn configForTokenBucket(num_tokens: u32) KernelConf { + return switch (num_tokens) { + 1 => .{ + .block_m = 16, + .block_n = 32, + .block_k = 64, + .group_m = 1, + .num_warps = 4, + .num_stages = 4, + }, + 2 => .{ + .block_m = 16, + .block_n = 32, + .block_k = 64, + .group_m = 1, + .num_warps = 4, + .num_stages = 4, + }, + 4 => .{ + .block_m = 16, + .block_n = 32, + .block_k = 64, + .group_m = 1, + .num_warps = 4, + .num_stages = 3, + }, + 8 => .{ + .block_m = 16, + .block_n = 128, + .block_k = 128, + .group_m = 1, + .num_warps = 8, + .num_stages = 3, + }, + 16 => .{ + .block_m = 16, + .block_n = 64, + .block_k = 64, + .group_m = 64, + .num_warps = 4, + .num_stages = 5, + }, + 24 => .{ + .block_m = 16, + .block_n = 64, + .block_k = 128, + .group_m = 1, + .num_warps = 8, + .num_stages = 2, + }, + 32 => .{ + .block_m = 16, + .block_n = 32, + .block_k = 128, + .group_m = 1, + .num_warps = 4, + .num_stages = 2, + }, + 48 => .{ + .block_m = 16, + .block_n = 32, + .block_k = 128, + .group_m = 64, + .num_warps = 4, + .num_stages = 2, + }, + 64 => .{ + .block_m = 16, + .block_n = 64, + .block_k = 128, + .group_m = 1, + .num_warps = 4, + .num_stages = 2, + }, + 96 => .{ + .block_m = 16, + .block_n = 128, + .block_k = 128, + .group_m = 1, + .num_warps = 8, + .num_stages = 3, + }, + 128 => .{ + .block_m = 16, + .block_n = 256, + .block_k = 128, + .group_m = 1, + .num_warps = 8, + .num_stages = 2, + }, + 256 => .{ + .block_m = 16, + .block_n = 256, + .block_k = 128, + .group_m = 1, + .num_warps = 8, + .num_stages = 2, + }, + 512 => .{ + .block_m = 32, + .block_n = 128, + .block_k = 128, + .group_m = 1, + .num_warps = 8, + .num_stages = 3, + }, + 1024 => .{ + .block_m = 64, + .block_n = 128, + .block_k = 64, + .group_m = 1, + .num_warps = 4, + .num_stages = 3, + }, + 1536 => .{ + .block_m = 64, + .block_n = 128, + .block_k = 64, + .group_m = 1, + .num_warps = 4, + .num_stages = 3, + }, + 2048 => .{ + .block_m = 128, + .block_n = 128, + .block_k = 64, + .group_m = 16, + .num_warps = 8, + .num_stages = 3, + }, + 3072 => .{ + .block_m = 128, + .block_n = 256, + .block_k = 64, + .group_m = 1, + .num_warps = 8, + .num_stages = 4, + }, + 4096 => .{ + .block_m = 128, + .block_n = 256, + .block_k = 64, + .group_m = 16, + .num_warps = 8, + .num_stages = 4, + }, + else => unreachable, + }; +} + +fn getBestConfig(num_tokens: u32, topk: u32, num_experts: u32) KernelConf { + const num_routes = std.math.mul(u32, num_tokens, topk) catch std.math.maxInt(u32); + var config = getBestTokenBucketConfig(num_routes); + + if (num_tokens <= 32 and num_routes <= 256 and num_experts <= 64) { + config.block_m = 16; + config.block_n = 256; + config.block_k = 128; + config.group_m = 1; + config.num_warps = 4; + config.num_stages = 2; + } else if (num_tokens <= 64 and num_routes <= 512 and num_experts <= 64) { + config.block_m = 16; + config.block_n = 128; + config.block_k = 128; + config.group_m = 1; + config.num_warps = 4; + config.num_stages = 2; + } + + return config; +} + +fn getBestTokenBucketConfig(num_tokens: u32) KernelConf { + var best_num_tokens = kernel_config_token_buckets[0]; + var best_distance = tokenDistance(num_tokens, best_num_tokens); + + for (kernel_config_token_buckets[1..]) |candidate| { + const distance = tokenDistance(num_tokens, candidate); + if (distance < best_distance or (distance == best_distance and candidate < best_num_tokens)) { + best_num_tokens = candidate; + best_distance = distance; + } + } + + return configForTokenBucket(best_num_tokens); +} + +fn tokenDistance(a: u32, b: u32) u32 { + return if (a >= b) a - b else b - a; +} + +const Fp4Routing = struct { + num_tokens: i64, + num_rows: i64, + topk: i64, + gather_divisor: i64, + grid_m: i64, + sorted_route_ids: zml.Tensor, + sorted_weights: zml.Tensor, + active_routes: zml.Tensor, + tile_experts: zml.Tensor, + tile_starts: zml.Tensor, + tile_ends: zml.Tensor, +}; + +fn prepareFp4Routing( + aligned: Routing, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + num_tokens: i64, + num_experts: i64, + block_m: i64, +) Fp4Routing { + const topk = topk_ids.dim(.topk); + const num_routes = aligned.num_assignments; + const num_rows = if (aligned.naive_block_assignment) + num_routes + else + aligned.max_num_tokens_padded; + const grid_m = if (aligned.naive_block_assignment) + num_routes + else + std.math.divCeil(i64, num_rows, block_m) catch unreachable; + + const sorted_route_candidates = if (aligned.naive_block_assignment) + zml.Tensor.arange(.{ .end = num_routes }, .i32).withTags(.{.route}) + else + aligned.sorted_token_ids.withTags(.{.route}); + const route_index_valid = sorted_route_candidates.cmp(.GE, zml.Tensor.scalar(0, .i32)) + .logical(.AND, sorted_route_candidates.cmp(.LT, zml.Tensor.scalar(num_routes, .i32))); + const sorted_route_ids = route_index_valid.select( + sorted_route_candidates, + zml.Tensor.zeroes(sorted_route_candidates.shape()), + ); + const gather_indices = sorted_route_ids.rename(.{ .route = .sorted_route }); + + const flat_expert_ids = topk_ids.flatten().withTags(.{.route}).convert(.i32); + const sorted_expert_ids = flat_expert_ids + .gather(.{ .route = gather_indices }, .{}) + .rename(.{ .sorted_route = .route }); + const active_routes = route_index_valid + .logical(.AND, sorted_expert_ids.cmp(.GE, zml.Tensor.scalar(0, .i32))) + .logical(.AND, sorted_expert_ids.cmp(.LT, zml.Tensor.scalar(num_experts, .i32))); + + const flat_weights = topk_weights.flatten().withTags(.{.route}); + const gathered_weights = flat_weights + .gather(.{ .route = gather_indices }, .{}) + .rename(.{ .sorted_route = .route }) + .convert(.f32); + const sorted_weights = active_routes.select(gathered_weights, zml.Tensor.zeroes(gathered_weights.shape())); + + const raw_tile_experts = aligned.expert_ids.withTags(.{.tile}).convert(.i32); + const valid_tile_experts = raw_tile_experts.cmp(.GE, zml.Tensor.scalar(0, .i32)) + .logical(.AND, raw_tile_experts.cmp(.LT, zml.Tensor.scalar(num_experts, .i32))); + const tile_experts = valid_tile_experts.select( + raw_tile_experts, + zml.Tensor.zeroes(raw_tile_experts.shape()), + ); + const tile_starts = if (aligned.naive_block_assignment) + zml.Tensor.arange(.{ .end = grid_m }, .i64).withTags(.{.tile}) + else + zml.Tensor.arange(.{ .end = grid_m }, .i64).withTags(.{.tile}).scale(block_m); + const tile_ends = if (aligned.naive_block_assignment) + valid_tile_experts.select(tile_starts.addConstant(1), tile_starts) + else blk: { + const num_tokens_post_padded = aligned.num_tokens_post_padded + .withTags(.{.tile}) + .convert(.i64) + .broad(tile_starts.shape()); + const active_tiles = valid_tile_experts.logical(.AND, tile_starts.cmp(.LT, num_tokens_post_padded)); + break :blk active_tiles.select( + tile_starts.addConstant(block_m).minimum(num_tokens_post_padded), + tile_starts, + ); + }; + + return .{ + .num_tokens = num_tokens, + .num_rows = num_rows, + .topk = topk, + .gather_divisor = topk, + .grid_m = grid_m, + .sorted_route_ids = sorted_route_ids, + .sorted_weights = sorted_weights, + .active_routes = active_routes, + .tile_experts = tile_experts, + .tile_starts = tile_starts, + .tile_ends = tile_ends, + }; +} + +const GemmOpts = struct { + routing: Fp4Routing, + weight_contract_tag: zml.Shape.Tag, + weight_output_tag: zml.Shape.Tag, + output_shape: zml.Shape, + gather: ?zml.Tensor = null, + gammas: ?zml.Tensor = null, + bias: ?zml.Tensor = null, + apply_swiglu: bool = false, + activation_limit: ?f32 = null, + block_m: u32, + block_n: u32, + block_k: u32, + group_m: u32, + num_warps: u32, + num_stages: u32, +}; +fn runGemm( + input: zml.Tensor, + weights: zml.Tensor, + scales: zml.Tensor, + opts: GemmOpts, +) !zml.Tensor { + const input_matrix = input.withTags(.{ .row, .k }); + const contract_k = input_matrix.dim(.k); + const packed_k = weights.dim(opts.weight_contract_tag); + const scale_k = scales.dim(opts.weight_contract_tag); + const n = weights.dim(opts.weight_output_tag); + + stdx.debug.assert(packed_k * 2 == contract_k, "expected packed int4 weight K {} to match activation K {}", .{ packed_k, contract_k }); + stdx.debug.assert(scale_k * 32 == contract_k, "expected MX scale K {} to match activation K {}", .{ scale_k, contract_k }); + const activation_reduction_n: i64 = if (opts.apply_swiglu) 2 else 1; + stdx.debug.assert(@mod(n, activation_reduction_n) == 0, "invalid GEMM output width {}", .{n}); + stdx.debug.assert(opts.output_shape.dim(-1) == @divExact(n, activation_reduction_n), "output shape {f} does not match GEMM N {}", .{ opts.output_shape, n }); + stdx.debug.assert(opts.bias == null, "MXFP4 Triton MoE GEMM bias is not wired yet", .{}); + + const block_m: i32 = @intCast(opts.block_m); + const block_n: i32 = @intCast(opts.block_n); + const block_k: i32 = @intCast(opts.block_k); + // TODO: update the kernel to support uneven K. + if (@mod(contract_k, block_k) != 0) return error.InvalidShape; + const grid_n = std.math.divCeil(i64, n, block_n) catch unreachable; + const has_gammas = opts.gammas != null; + const gathered_input = if (opts.gather) |gather| blk: { + const token_ids = gather.divByConst(opts.routing.gather_divisor).withTags(.{.route}); + break :blk input_matrix.gather(.{ .row = token_ids }, .{}).rename(.{ .route = .row }); + } else input_matrix; + const raw_output_shape = if (opts.apply_swiglu) + opts.output_shape.set(-1, n) + else + opts.output_shape; + + const cfg: a16w4_kernel.Cfg = .{ + .a_dtype = zml.kernel.triton.from(gathered_input.dtype()), + .wp_dtype = packedByteDtype(weights.dtype()), + .ws_dtype = packedByteDtype(scales.dtype()), + .c_dtype = zml.kernel.triton.from(raw_output_shape.dtype()), + .BLOCK_M = block_m, + .BLOCK_N = block_n, + .BLOCK_K = block_k, + .SPLIT_K = 1, + .GROUP_M = @intCast(opts.group_m), + .num_warps = @intCast(opts.num_warps), + .num_stages = @intCast(opts.num_stages), + }; + + var y = a16w4_kernel.Kernel.call( + .{ + .a_ptr = gathered_input, + .wp_ptr = weights, + .ws_ptr = scales, + .tile_expert_ptr = opts.routing.tile_experts, + .tile_mstart_ptr = opts.routing.tile_starts, + .tile_mend_ptr = opts.routing.tile_ends, + .NUM_M_TILES_ptr = scalarI64(opts.routing.grid_m), + .N_ptr = scalarI64(n), + .K_ptr = scalarI64(contract_k), + .stride_am_ptr = scalarI64(contract_k), + .stride_ak_ptr = scalarI64(1), + .stride_we_ptr = scalarI64(n * packed_k), + .stride_wk_ptr = scalarI64(1), + .stride_wn_ptr = scalarI64(packed_k), + .stride_se_ptr = scalarI64(n * scale_k), + .stride_sk_ptr = scalarI64(1), + .stride_sn_ptr = scalarI64(scale_k), + .stride_cm_ptr = scalarI64(raw_output_shape.dim(-1)), + .stride_cn_ptr = scalarI64(1), + }, + .{ .c = raw_output_shape }, + .{ + .cfg = cfg, + .grid = .{ @intCast(opts.routing.grid_m * grid_n), 1, 1 }, + .num_warps = @intCast(opts.num_warps), + .num_stages = @intCast(opts.num_stages), + }, + ).c; + + if (opts.apply_swiglu) { + y = applySwiGlu(y.convert(.f32), opts.activation_limit).convert(opts.output_shape.dtype()); + } + + if (has_gammas) { + const gammas = opts.gammas.?.convert(.f32).appendAxes(.{.dout}).broad(opts.output_shape.withDtype(.f32)); + y = y.convert(.f32).mul(gammas).convert(opts.output_shape.dtype()); + } + + return y; +} + +fn applySwiGlu(input: zml.Tensor, activation_limit: ?f32) zml.Tensor { + var gate = input.slice1d(.dout, .{ .start = 0, .step = 2 }); + var up = input.slice1d(.dout, .{ .start = 1, .step = 2 }); + + if (activation_limit) |limit| { + const threshold = zml.Tensor.scalar(limit, .f32); + gate = gate.minimum(threshold); + up = up.clamp(threshold.negate(), threshold); + } + + return gate.silu().mul(up); +} + +fn packedByteDtype(dt: zml.DataType) zml.kernel.triton.DType { + return switch (dt) { + .i8, .u8, .f4e2m1, .f8e8m0 => .i8, + else => zml.kernel.triton.from(dt), + }; +} + +fn scalarI64(v: i64) zml.Tensor { + return zml.Tensor.constant(.{ .i64 = v }).reshape(.{1}); +} diff --git a/zml/moe/triton_kernels/a16w4_kernel.zig b/zml/moe/triton_kernels/a16w4_kernel.zig new file mode 100644 index 00000000..dd5f016d --- /dev/null +++ b/zml/moe/triton_kernels/a16w4_kernel.zig @@ -0,0 +1,189 @@ +const std = @import("std"); + +const zml = @import("../../zml.zig"); +const tri = zml.kernel.triton; + +pub const Cfg = struct { + a_dtype: tri.DType = .bf16, + wp_dtype: tri.DType = .i8, + ws_dtype: tri.DType = .i8, + c_dtype: tri.DType = .bf16, + + BLOCK_M: i32 = 16, + BLOCK_N: i32 = 64, + BLOCK_K: i32 = 64, + SPLIT_K: i32 = 1, + GROUP_M: i32 = 1, + num_warps: i32 = 4, + num_stages: i32 = 2, +}; + +pub const Kernel = tri.Kernel(Cfg, .{ + .name = "_gemm_a16mxfp4", + .inputs = &.{ + "a_ptr", + "wp_ptr", + "ws_ptr", + "tile_expert_ptr", + "tile_mstart_ptr", + "tile_mend_ptr", + "NUM_M_TILES_ptr", + "N_ptr", + "K_ptr", + "stride_am_ptr", + "stride_ak_ptr", + "stride_we_ptr", + "stride_wk_ptr", + "stride_wn_ptr", + "stride_se_ptr", + "stride_sk_ptr", + "stride_sn_ptr", + "stride_cm_ptr", + "stride_cn_ptr", + }, + .outputs = &.{"c"}, + .run = run, +}); + +fn run(b: *tri.Builder, cfg: Cfg) tri.FinishError!void { + std.debug.assert(@mod(cfg.BLOCK_K, 32) == 0); + + const a = try b.declareArgs(.{ + .a_ptr = .{ .ptr = cfg.a_dtype }, + .wp_ptr = .{ .ptr = cfg.wp_dtype }, + .ws_ptr = .{ .ptr = cfg.ws_dtype }, + .tile_expert_ptr = .{ .ptr = .i32 }, + .tile_mstart_ptr = .{ .ptr = .i64 }, + .tile_mend_ptr = .{ .ptr = .i64 }, + .NUM_M_TILES_ptr = .{ .ptr = .i64 }, + .N_ptr = .{ .ptr = .i64 }, + .K_ptr = .{ .ptr = .i64 }, + .stride_am_ptr = .{ .ptr = .i64 }, + .stride_ak_ptr = .{ .ptr = .i64 }, + .stride_we_ptr = .{ .ptr = .i64 }, + .stride_wk_ptr = .{ .ptr = .i64 }, + .stride_wn_ptr = .{ .ptr = .i64 }, + .stride_se_ptr = .{ .ptr = .i64 }, + .stride_sk_ptr = .{ .ptr = .i64 }, + .stride_sn_ptr = .{ .ptr = .i64 }, + .stride_cm_ptr = .{ .ptr = .i64 }, + .stride_cn_ptr = .{ .ptr = .i64 }, + .c_ptr = .{ .ptr = cfg.c_dtype }, + }); + + const NUM_M_TILES = b.load(a.NUM_M_TILES_ptr); + const N_val = b.load(a.N_ptr); + const K_val = b.load(a.K_ptr); + const stride_am = b.load(a.stride_am_ptr); + const stride_ak = b.load(a.stride_ak_ptr); + const stride_we = b.load(a.stride_we_ptr); + const stride_wk = b.load(a.stride_wk_ptr); + const stride_wn = b.load(a.stride_wn_ptr); + const stride_se = b.load(a.stride_se_ptr); + const stride_sk = b.load(a.stride_sk_ptr); + const stride_sn = b.load(a.stride_sn_ptr); + const stride_cm = b.load(a.stride_cm_ptr); + const stride_cn = b.load(a.stride_cn_ptr); + + // L2-friendly swizzle: group GROUP_M consecutive m-tiles (usually the same + // expert) and sweep n within them, so each weight column tile is reused. + const pid = b.programId(.x); + const pid_k = b.programId(.y); + const num_pid_n = N_val.cdiv(cfg.BLOCK_N); + const width = b.liftAs(cfg.GROUP_M, .i64).mul(num_pid_n); + const pid_i64 = pid.to(.i64); + const group_id = pid_i64.div(width); + const group_size = NUM_M_TILES.sub(group_id.mul(cfg.GROUP_M)).minimum(cfg.GROUP_M); + const pid_m = group_id.mul(cfg.GROUP_M).add(pid_i64.rem(group_size)); + const pid_n = pid_i64.rem(width).div(group_size); + + const expert = b.load(a.tile_expert_ptr.addPtr(pid_m)); + const m_start = b.load(a.tile_mstart_ptr.addPtr(pid_m)); + const m_end = b.load(a.tile_mend_ptr.addPtr(pid_m)); + + const offs_m = m_start.add(b.arange(0, cfg.BLOCK_M, .i32)); + const offs_n = pid_n.mul(cfg.BLOCK_N).add(b.arange(0, cfg.BLOCK_N, .i32)); + const m_mask = offs_m.lt(m_end); + const n_mask = offs_n.lt(N_val); + + const BLOCK_K_PACKED: i32 = @divExact(cfg.BLOCK_K, 2); + const BLOCK_K_SCALE: i32 = @divExact(cfg.BLOCK_K, 32); + const offs_kh = b.arange(0, BLOCK_K_PACKED, .i32); // packed K rows within a tile + const offs_kg = b.arange(0, BLOCK_K_SCALE, .i32); // scale groups within a tile + + const wp_base = a.wp_ptr.addPtr(expert.mul(stride_we)); + const ws_base = a.ws_ptr.addPtr(expert.mul(stride_se)); + + const acc = b.zeros(&.{ cfg.BLOCK_M, cfg.BLOCK_N }, .f32); + const num_k_tiles = K_val.div(cfg.BLOCK_K); + const pid_k_i64 = pid_k.to(.i64); + var loop = b.openFor(pid_k_i64, num_k_tiles, cfg.SPLIT_K, .{acc}); + { + const t = loop.iv; + const acc_loop = loop.carried[0]; + + const k0 = t.mul(cfg.BLOCK_K); + const offs_ak = k0.add(b.arange(0, cfg.BLOCK_K, .i32)); + const a_mask = m_mask.expandDims(1); + const a_rows = offs_m.expandDims(1); + const a_row_offsets = a_rows.mul(stride_am); + const a_ptrs_col = a.a_ptr.addPtr(a_row_offsets); + const a_cols = offs_ak.expandDims(0); + const a_col_offsets = a_cols.mul(stride_ak); + const a_ptrs_base = b.broadcastTo(a_ptrs_col, &.{ cfg.BLOCK_M, cfg.BLOCK_K }); + const a_col_offsets_full = b.broadcastTo(a_col_offsets, &.{ cfg.BLOCK_M, cfg.BLOCK_K }); + const a_ptrs = a_ptrs_base.addPtr(a_col_offsets_full); + const a_mask_full = b.broadcastTo(a_mask, &.{ cfg.BLOCK_M, cfg.BLOCK_K }); + const a_tile = b.loadOpts(a_ptrs, .{ + .mask = a_mask_full, + .other = b.zeros(&.{ cfg.BLOCK_M, cfg.BLOCK_K }, cfg.a_dtype), + }); // (BM, BK) bf16 + + const pj = k0.div(2).add(offs_kh); + const w_mask = n_mask.expandDims(0); + const w_rows = pj.expandDims(1); + const w_row_offsets = w_rows.mul(stride_wk); + const w_ptrs_col = wp_base.addPtr(w_row_offsets); + const w_cols = offs_n.expandDims(0); + const w_col_offsets = w_cols.mul(stride_wn); + const w_ptrs_base = b.broadcastTo(w_ptrs_col, &.{ BLOCK_K_PACKED, cfg.BLOCK_N }); + const w_col_offsets_full = b.broadcastTo(w_col_offsets, &.{ BLOCK_K_PACKED, cfg.BLOCK_N }); + const w_ptrs = w_ptrs_base.addPtr(w_col_offsets_full); + const w_mask_full = b.broadcastTo(w_mask, &.{ BLOCK_K_PACKED, cfg.BLOCK_N }); + const w = b.loadOpts(w_ptrs, .{ + .mask = w_mask_full, + .other = b.zeros(&.{ BLOCK_K_PACKED, cfg.BLOCK_N }, cfg.wp_dtype), + }); // (BK//2, BN) uint8 + + const kg = k0.div(32).add(offs_kg); + // rhs_scale must be laid out [N, K//32] for dot_scaled + const sc_mask = n_mask.expandDims(1); + const sc_rows = offs_n.expandDims(1); + const sc_row_offsets = sc_rows.mul(stride_sn); + const sc_ptrs_col = ws_base.addPtr(sc_row_offsets); + const sc_cols = kg.expandDims(0); + const sc_col_offsets = sc_cols.mul(stride_sk); + const sc_ptrs_base = b.broadcastTo(sc_ptrs_col, &.{ cfg.BLOCK_N, BLOCK_K_SCALE }); + const sc_col_offsets_full = b.broadcastTo(sc_col_offsets, &.{ cfg.BLOCK_N, BLOCK_K_SCALE }); + const sc_ptrs = sc_ptrs_base.addPtr(sc_col_offsets_full); + const sc_mask_full = b.broadcastTo(sc_mask, &.{ cfg.BLOCK_N, BLOCK_K_SCALE }); + const sc = b.loadOpts(sc_ptrs, .{ + .mask = sc_mask_full, + .other = b.full(&.{ cfg.BLOCK_N, BLOCK_K_SCALE }, 127, cfg.ws_dtype), + }); // (BN, BK//32) e8m0 + const new_acc = b.dotScaledOpts(a_tile, w, acc_loop, null, sc, .bf16, .e2m1, .{ .fast_math = true }); + + loop.yield(.{new_acc}); + } + + const c_rows = offs_m.expandDims(1); + const c_row_offsets = c_rows.mul(stride_cm); + const c_ptrs_col = a.c_ptr.addPtr(c_row_offsets); + const c_cols = offs_n.expandDims(0); + const c_col_offsets = c_cols.mul(stride_cn); + const c_ptrs_base = b.broadcastTo(c_ptrs_col, &.{ cfg.BLOCK_M, cfg.BLOCK_N }); + const c_col_offsets_full = b.broadcastTo(c_col_offsets, &.{ cfg.BLOCK_M, cfg.BLOCK_N }); + const c_ptrs = c_ptrs_base.addPtr(c_col_offsets_full); + const c_mask = m_mask.expandDims(1).bitAnd(n_mask.expandDims(0)); + b.storeOpts(c_ptrs, loop.results[0].to(.bf16), .{ .mask = c_mask }); +} From 082c48b21165ed5e8c86703d1eb6618bf71c322a Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 27 Aug 2026 16:06:36 +0200 Subject: [PATCH 635/881] zml: refactor slice API (#729) ## Overview * merge `dynamicSlice` and `slice` into `slices` * merge `dynamicSlice1d` and `slice1d` into `slice` * drop `choose`, `choose1d` * handle singleton for dynamic slices too * typed API so that it's easier to specify arguments ## Migration guide: ```diff // new .slice accepts all old .slice1d inputs - const x1 = x.slice1d(-1, .{ .start = 0, .end = half_dim }); + const x1 = x.slice(-1, .{ .start = 0, .end = half_dim }); - k.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = cache_index, .len = 1 } }).squeeze(.layer); + k.slice(.layer, .dynSingle(cache_index)); - k.dynamicSlice(.{ - .layer = zml.Tensor.DynSlice{ .start = cache_index, .len = 1 }, - }).squeeze(.layer).choose1d(.k, kv_head); + k.slice(.{ .layer, .k }, &.{ .dynSingle(cache_index), .single(kv_head) }); - o.choose1d(ax, 5), + o.slice(ax, .single(5)) ``` Now the `Tensor.Slice` struct is bigger and now as a Tensor in it, which may break user code if your model was storing a `Slice` inside it, because now `zml.Bufferize(Model)` will contain one more `?Buffer`. You can either pass an extra `null` to the `zml.Exe` or refactor to remove `Slice` out of `Model`. I haven't seen this happen so far. The idea is that `Slice` will mostly be a stack only struct, only created to call `slice` --- examples/llm/models/lfm2/model.zig | 10 +- examples/llm/models/llama/model.zig | 4 +- examples/llm/models/qwen3_5/model.zig | 26 +- examples/llm/models/qwen3_5_moe/model.zig | 28 +- zml/attention/attnd.zig | 4 +- zml/attention/flashattn.zig | 4 +- zml/attention/paged_attention.zig | 32 +-- zml/attention/tpu_attention.zig | 6 +- zml/attention/triton_attention.zig | 6 +- zml/moe/metal.zig | 4 +- zml/moe/mosaic_tpu.zig | 10 +- zml/moe/triton.zig | 4 +- zml/nn.zig | 36 +-- zml/ops.zig | 2 +- zml/tensor.zig | 317 +++++++++++----------- 15 files changed, 247 insertions(+), 246 deletions(-) diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index 5df4628b..efb39fcc 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -471,7 +471,7 @@ pub const ShortConv = struct { const sh = tokens_position_offset.shape().insert(.last, .{ .seq = self.config.conv_L_cache }); break :lkp zml.Tensor.iota(sh, .seq).convert(.u32).add(left_pad.convert(.u32).broad(sh)).broad(sh); }; - const scatter_data = Bx.dynamicSlice(.{ .seq = @as(zml.Tensor.DynSlice, .{ .start = start.convert(.u32), .len = @intCast(self.config.conv_L_cache) }) }); + const scatter_data = Bx.slice(.seq, .dyn(start, self.config.conv_L_cache)); cache.state = cache.state.scatterSlices(.{ .layer = cache_index, .seq = cache_seq_indices }, scatter_data, .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(cache.state); const padding = self.config.conv_L_cache - 1; const conv_out_ = zml.Tensor.conv1d(Bx, self.kernel, .{ @@ -487,12 +487,12 @@ pub const ShortConv = struct { .output_spatial_dimensions = Bx.axis(.seq), .feature_group_count = Bx.dim(.d), }); - break :b conv_out_.slice1d(.seq, .{ .end = Bx.dim(.seq) }); + break :b conv_out_.slice(.seq, .{ .end = Bx.dim(.seq) }); } else b: { cache.state = cache.state.rollRight1d(.seq, -1).scatterSlices(.{ .layer = cache_index, .seq = tokens_position_offset.convert(.u32).clamp(.scalar(@as(u32, 0), .u32), .scalar(self.config.conv_L_cache - 1, .u32)) }, Bx, .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(cache.state); const kernel = self.kernel.squeeze(.in).rename(.{ .out = .d, .kernel_size = .seq }); const sh = kernel.shape().insert(.last, .{ .batch = tokens_position_offset.dim(.batch) }); - break :b cache.state.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = cache_index, .len = 1 } }).squeeze(.layer).mul(kernel.broad(sh).transpose(.{ .batch, .seq, .d })).sum(.seq); + break :b cache.state.slice(.layer, .dynSingle(cache_index)).mul(kernel.broad(sh).transpose(.{ .batch, .seq, .d })).sum(.seq); }; const y = C.mul(conv_out); @@ -669,11 +669,11 @@ pub const KvCache = struct { } pub fn keys(self: KvCache, cache_index: zml.Tensor) zml.Tensor { - return self.k.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = cache_index, .len = 1 } }).squeeze(.layer); + return self.k.slice(.layer, .dynSingle(cache_index)); } pub fn values(self: KvCache, cache_index: zml.Tensor) zml.Tensor { - return self.v.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = cache_index, .len = 1 } }).squeeze(.layer); + return self.v.slice(.layer, .dynSingle(cache_index)); } pub fn update(self: KvCache, new_k: zml.Tensor, new_v: zml.Tensor, token_position: zml.Tensor, cache_index: zml.Tensor) KvCache { diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index e41de508..7152952d 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -646,11 +646,11 @@ pub const KvCache = struct { } pub fn keysAt(kv: KvCache, layer_index: zml.Tensor) zml.Tensor { - return kv.k.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = layer_index, .len = 1 } }).squeeze(.layer); + return kv.k.slice(.layer, .dynSingle(layer_index)); } pub fn valuesAt(kv: KvCache, layer_index: zml.Tensor) zml.Tensor { - return kv.v.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = layer_index, .len = 1 } }).squeeze(.layer); + return kv.v.slice(.layer, .dynSingle(layer_index)); } pub fn updateAt(kv: KvCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: zml.Tensor, layer_index: zml.Tensor) KvCache { diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 20df599c..7ddafe9c 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -747,14 +747,14 @@ pub const TextRotaryEmbedding = struct { fn rotateHalf(x: zml.Tensor) zml.Tensor { const half_dim = @divExact(x.dim(-1), 2); - const x1 = x.slice1d(-1, .{ .start = 0, .end = half_dim }); - const x2 = x.slice1d(-1, .{ .start = half_dim, .end = x.dim(-1) }); + const x1 = x.slice(-1, .{ .start = 0, .end = half_dim }); + const x2 = x.slice(-1, .{ .start = half_dim, .end = x.dim(-1) }); return zml.Tensor.concatenate(&.{ x2.negate(), x1 }, -1); } pub fn applyRope(self: TextRotaryEmbedding, x: zml.Tensor, cos: zml.Tensor, sin: zml.Tensor) zml.Tensor { - const x_rot = x.slice1d(-1, .{ .start = 0, .end = self.rotary_dim }); - const x_pass = x.slice1d(-1, .{ .start = self.rotary_dim, .end = x.dim(-1) }); + const x_rot = x.slice(-1, .{ .start = 0, .end = self.rotary_dim }); + const x_pass = x.slice(-1, .{ .start = self.rotary_dim, .end = x.dim(-1) }); const cos_x = cos.insertAxes(.hd, .{.h}).broad(x_rot.shape()); const sin_x = sin.insertAxes(.hd, .{.h}).broad(x_rot.shape()); @@ -871,7 +871,7 @@ pub const GatedDeltaNet = struct { fn buildUpdatedConvState(input: zml.Tensor, left_pad: i64) zml.Tensor { const copy_len = @min(input.dim(.s), left_pad); - const tail = input.slice1d(.s, .{ .start = input.dim(.s) - copy_len, .end = input.dim(.s) }); + const tail = input.slice(.s, .{ .start = input.dim(.s) - copy_len, .end = input.dim(.s) }); if (copy_len == left_pad) return tail; const padding_shape = zml.Shape.init(.{ .b = input.dim(.b), .s = left_pad - copy_len, .mix = input.dim(.mix) }, input.dtype()); @@ -916,7 +916,7 @@ pub const GatedDeltaNet = struct { .silu(); if (use_cached_state) { - mixed_qkv = mixed_qkv.slice1d(.s, .{ .start = mixed_qkv.dim(.s) - 1, .end = mixed_qkv.dim(.s) }); + mixed_qkv = mixed_qkv.slice(.s, .{ .start = mixed_qkv.dim(.s) - 1, .end = mixed_qkv.dim(.s) }); } mixed_qkv = mixed_qkv.withPartitioning(.{ .s = .replicated, .mix = .model }); @@ -926,13 +926,13 @@ pub const GatedDeltaNet = struct { const a = self.in_proj_a.forward(x_in).rename(.{ .dout = .vh }); const query = mixed_qkv - .slice1d(.mix, .{ .start = 0, .end = key_dim }) + .slice(.mix, .{ .start = 0, .end = key_dim }) .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }); const key = mixed_qkv - .slice1d(.mix, .{ .start = key_dim, .end = 2 * key_dim }) + .slice(.mix, .{ .start = key_dim, .end = 2 * key_dim }) .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }); const value = mixed_qkv - .slice1d(.mix, .{ .start = 2 * key_dim, .end = 2 * key_dim + value_dim }) + .slice(.mix, .{ .start = 2 * key_dim, .end = 2 * key_dim + value_dim }) .splitAxis(.mix, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }); const beta = b.sigmoid(); @@ -1072,11 +1072,11 @@ pub const KvCache = struct { } pub fn keys(kv: SelfAttnCache) zml.Tensor { - return kv.k.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = kv.layer_index, .len = 1 } }).squeeze(.layer); + return kv.k.slice(.layer, .dynSingle(kv.layer_index)); } pub fn values(kv: SelfAttnCache) zml.Tensor { - return kv.v.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = kv.layer_index, .len = 1 } }).squeeze(.layer); + return kv.v.slice(.layer, .dynSingle(kv.layer_index)); } pub fn update(kv: SelfAttnCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: ?zml.Tensor) SelfAttnCache { @@ -1179,11 +1179,11 @@ pub const KvCache = struct { } pub fn convState(self: GatedDeltaNetCache) zml.Tensor { - return self.conv_state.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + return self.conv_state.slice(.layer, .dynSingle(self.layer_index)); } pub fn recurrentState(self: GatedDeltaNetCache) zml.Tensor { - return self.recurrent_state.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + return self.recurrent_state.slice(.layer, .dynSingle(self.layer_index)); } pub fn update(self: GatedDeltaNetCache, new_conv_state: ?zml.Tensor, new_recurrent_state: ?zml.Tensor) GatedDeltaNetCache { diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 8f32e124..98d6ae25 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -888,14 +888,14 @@ pub const TextRotaryEmbedding = struct { fn rotateHalf(x: zml.Tensor) zml.Tensor { const half_dim = @divExact(x.dim(-1), 2); - const x1 = x.slice1d(-1, .{ .start = 0, .end = half_dim }); - const x2 = x.slice1d(-1, .{ .start = half_dim, .end = x.dim(-1) }); + const x1 = x.slice(-1, .{ .start = 0, .end = half_dim }); + const x2 = x.slice(-1, .{ .start = half_dim, .end = x.dim(-1) }); return zml.Tensor.concatenate(&.{ x2.negate(), x1 }, -1); } pub fn applyRope(self: TextRotaryEmbedding, x: zml.Tensor, cos: zml.Tensor, sin: zml.Tensor) zml.Tensor { - const x_rot = x.slice1d(-1, .{ .start = 0, .end = self.rotary_dim }); - const x_pass = x.slice1d(-1, .{ .start = self.rotary_dim, .end = x.dim(-1) }); + const x_rot = x.slice(-1, .{ .start = 0, .end = self.rotary_dim }); + const x_pass = x.slice(-1, .{ .start = self.rotary_dim, .end = x.dim(-1) }); const cos_x = cos.insertAxes(.hd, .{.h}).broad(x_rot.shape()); const sin_x = sin.insertAxes(.hd, .{.h}).broad(x_rot.shape()); @@ -1011,7 +1011,7 @@ pub const GatedDeltaNet = struct { fn buildUpdatedConvState(input: zml.Tensor, left_pad: i64) zml.Tensor { const copy_len = @min(input.dim(.s), left_pad); - const tail = input.slice1d(.s, .{ .start = input.dim(.s) - copy_len, .end = input.dim(.s) }); + const tail = input.slice(.s, .{ .start = input.dim(.s) - copy_len, .end = input.dim(.s) }); if (copy_len == left_pad) return tail; const padding_shape = zml.Shape.init(.{ .b = input.dim(.b), .s = left_pad - copy_len, .mix = input.dim(.mix) }, input.dtype()); @@ -1021,7 +1021,7 @@ pub const GatedDeltaNet = struct { fn buildUpdatedConvStateFromPrefix(input: zml.Tensor, left_pad: i64, valid_len: zml.Tensor) zml.Tensor { const start = valid_len.convert(.i64).addConstant(-left_pad).maximum(zml.Tensor.scalar(0, .i64)); - return input.dynamicSlice1d(input.axis(.s), .{ .start = start, .len = left_pad }); + return input.slice(.s, .dyn(start, left_pad)); } fn setPaddingToZero(input: zml.Tensor, valid_mask: zml.Tensor) zml.Tensor { @@ -1064,7 +1064,7 @@ pub const GatedDeltaNet = struct { .silu(); if (use_cached_state) { - mixed_qkv = mixed_qkv.slice1d(.s, .{ .start = mixed_qkv.dim(.s) - 1, .end = mixed_qkv.dim(.s) }); + mixed_qkv = mixed_qkv.slice(.s, .{ .start = mixed_qkv.dim(.s) - 1, .end = mixed_qkv.dim(.s) }); } mixed_qkv = mixed_qkv.withPartitioning(.{ .s = .replicated, .mix = .model }); @@ -1075,15 +1075,15 @@ pub const GatedDeltaNet = struct { const a = self.in_proj_a.forward(x_in).rename(.{ .dout = .vh }).withPartitioning(.{ .s = .replicated, .vh = .model }); const query = mixed_qkv - .slice1d(.mix, .{ .start = 0, .end = key_dim }) + .slice(.mix, .{ .start = 0, .end = key_dim }) .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }) .withPartitioning(.{ .s = .replicated, .kh = .model, .khd = .replicated }); const key = mixed_qkv - .slice1d(.mix, .{ .start = key_dim, .end = 2 * key_dim }) + .slice(.mix, .{ .start = key_dim, .end = 2 * key_dim }) .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }) .withPartitioning(.{ .s = .replicated, .kh = .model, .khd = .replicated }); const value = mixed_qkv - .slice1d(.mix, .{ .start = 2 * key_dim, .end = 2 * key_dim + value_dim }) + .slice(.mix, .{ .start = 2 * key_dim, .end = 2 * key_dim + value_dim }) .splitAxis(.mix, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }) .withPartitioning(.{ .s = .replicated, .vh = .model, .vhd = .replicated }); @@ -1222,11 +1222,11 @@ pub const KvCache = struct { } pub fn keys(self: SelfAttnCache) zml.Tensor { - return self.k.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + return self.k.slice(.layer, .dynSingle(self.layer_index)); } pub fn values(self: SelfAttnCache) zml.Tensor { - return self.v.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + return self.v.slice(.layer, .dynSingle(self.layer_index)); } pub fn update(self: SelfAttnCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: ?zml.Tensor) SelfAttnCache { @@ -1323,11 +1323,11 @@ pub const KvCache = struct { } pub fn convState(self: GatedDeltaNetCache) zml.Tensor { - return self.conv_state.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + return self.conv_state.slice(.layer, .dynSingle(self.layer_index)); } pub fn recurrentState(self: GatedDeltaNetCache) zml.Tensor { - return self.recurrent_state.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + return self.recurrent_state.slice(.layer, .dynSingle(self.layer_index)); } pub fn update(self: GatedDeltaNetCache, new_conv_state: ?zml.Tensor, new_recurrent_state: ?zml.Tensor) GatedDeltaNetCache { diff --git a/zml/attention/attnd.zig b/zml/attention/attnd.zig index ddf9c04f..2e191868 100644 --- a/zml/attention/attnd.zig +++ b/zml/attention/attnd.zig @@ -147,8 +147,8 @@ pub fn causalAttention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_offset .{ k, v } else .{ - k.dynamicSlice(.{ .k = zml.Tensor.DynSlice{ .start = token_offset, .len = 1 } }), - v.dynamicSlice(.{ .k = zml.Tensor.DynSlice{ .start = token_offset, .len = 1 } }), + k.slice(.k, .dynSingle(token_offset)), + v.slice(.k, .dynSingle(token_offset)), }; // Concatenate q/k/v and metadata into a single byte buffer on the GPU. diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 3370ffa0..977cc462 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -1078,7 +1078,7 @@ pub const paged_fa2 = struct { .b = batch_dim_decode, }, .f32)).withPartitioning(.{ .hkv = .model }); const dummy_cu_seqlens_k_decode = zml.Tensor.zeroes(cu_seqlens_q_decode.shape()); - var q_decode = q.dynamicSlice1d(0, .{ .start = mixed_parameters.metadata.decode_offset, .len = batch_dim_decode }); + var q_decode = q.slice(0, .dyn(mixed_parameters.metadata.decode_offset, batch_dim_decode)); if (seqlenq_ngroups_swapped) { q_decode = q_decode.transpose(.{ .b, .hg, .hkv, .hd }).merge(.{ .b = .{ .b, .hg } }).withPartitioning(.{ .hkv = .model }); @@ -1676,7 +1676,7 @@ pub const paged_fa3 = struct { .hg = num_head_groups, }, .f32)); const scheduler_metadata_decode = zml.Tensor.zeroes(.init(.{ .b = batch_size_decode + 1 }, .i32)); - var q_decode = q.dynamicSlice1d(0, .{ .start = mixed_parameters.metadata.decode_offset, .len = batch_size_decode }).withPartitioning(.{ .hkv = .model }); + var q_decode = q.slice(0, .dyn(mixed_parameters.metadata.decode_offset, batch_size_decode)).withPartitioning(.{ .hkv = .model }); q_decode = q_decode.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 7cb6b712..39d2864d 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -270,8 +270,8 @@ pub const KvCache = union(enum) { ) [2]zml.Tensor { return switch (self) { .split => |split| .{ - split.k.dynamicSlice1d(split.k.axis(.page), .{ .start = page_index, .len = 1 }).squeeze(.page), - split.v.dynamicSlice1d(split.k.axis(.page), .{ .start = page_index, .len = 1 }).squeeze(.page), + split.k.slice(split.k.axis(.page), .dynSingle(page_index)), + split.v.slice(split.k.axis(.page), .dynSingle(page_index)), }, .dense, .latent => @panic("TODO"), }; @@ -601,15 +601,15 @@ const AttentionLoop = struct { pub fn cond(self: While, state: State) zml.Tensor { const has_more_sequences = state.seq_id.cmp(.LT, .scalar(self.parameters.seq_lens.count(), .i32)); - const query_start = self.parameters.query_start_len.dynamicSlice1d(0, .{ .start = state.seq_id, .len = 1 }); - const query_end = self.parameters.query_start_len.dynamicSlice1d(0, .{ .start = state.seq_id.addConstant(1), .len = 1 }); + const query_start = self.parameters.query_start_len.slice(0, .dynSingle(state.seq_id)); + const query_end = self.parameters.query_start_len.slice(0, .dynSingle(state.seq_id.addConstant(1))); const has_queries = query_start.cmp(.LT, query_end).asScalar(); return has_more_sequences.logical(.AND, has_queries); } pub fn body(self: While, state_: State) State { - const query_start = self.parameters.query_start_len.dynamicSlice1d(0, .{ .start = state_.seq_id, .len = 1 }); - const query_end = self.parameters.query_start_len.dynamicSlice1d(0, .{ .start = state_.seq_id.addConstant(1), .len = 1 }); + const query_start = self.parameters.query_start_len.slice(0, .dynSingle(state_.seq_id)); + const query_end = self.parameters.query_start_len.slice(0, .dynSingle(state_.seq_id.addConstant(1))); const seq_num_queries_: zml.Tensor = .asScalar(.sub(query_end, query_start)); const page_size_: u32 = @intCast(self.kv_cache.split.k.dim(.k_chunk)); @@ -629,7 +629,7 @@ const AttentionLoop = struct { const num_slots: u32 = page_size; // seq_lens includes the current queries; subtracting their count gives the context length. - const seq_len = while_body.parameters.seq_lens.dynamicSlice1d(0, .{ .start = state.seq_id, .len = 1 }).asScalar(); + const seq_len = while_body.parameters.seq_lens.slice(0, .dynSingle(state.seq_id)); const q_offset = seq_len.sub(if_ctx.seq_num_queries).add(state.q_page_idx.scale(page_size)); const next_k_offset = state.k_page_idx.addConstant(1).scale(page_size); @@ -650,7 +650,7 @@ const AttentionLoop = struct { const num_slots: u32 = 1; // A decode query is the final token in seq_len, so its zero-based offset is seq_len - 1. - const seq_len = while_body.parameters.seq_lens.dynamicSlice1d(0, .{ .start = state.seq_id, .len = 1 }).asScalar(); + const seq_len = while_body.parameters.seq_lens.slice(0, .dynSingle(state.seq_id)).asScalar(); const q_offset = seq_len.sub(if_ctx.seq_num_queries); const next_k_offset = state.k_page_idx.addConstant(1).scale(page_size); @@ -673,12 +673,12 @@ const AttentionLoop = struct { pub fn attentionOnePage(self: While, state: State, q_offset: zml.Tensor, q_chunk: u32, page_size: u32) PartialSoftmax { const k_offset = state.k_page_idx.scale(page_size); - const active_q = self.q.dynamicSlice1d(self.q.axis(.b), .{ .start = state.slot_id, .len = q_chunk }); + const active_q = self.q.slice(self.q.axis(.b), .dyn(state.slot_id, q_chunk)); - const active_k_page_id = self.parameters.block_table.dynamicSlice(.{ - .b = zml.Tensor.DynSlice{ .start = state.seq_id, .len = 1 }, - .p = zml.Tensor.DynSlice{ .start = state.k_page_idx, .len = 1 }, - }).asScalar(); + const active_k_page_id = self.parameters.block_table.slices( + .{ .b, .p }, + &.{ .dynSingle(state.seq_id), .dynSingle(state.k_page_idx) }, + ); const active_k, const active_v = self.kv_cache.getPage(active_k_page_id); const dtype = self.q.dtype(); @@ -832,11 +832,7 @@ pub const Mla = struct { const scores_sink = zml.Tensor.concatenate(&.{ scores, sink_.convert(scores.dtype()) }, .kv); const attn_weights = scores_sink.softmax(.kv); - const attn_weights_non_sink = attn_weights.slice(&.{ - .{}, - .{}, - .{ .end = topk.dim(.topk) }, - }); + const attn_weights_non_sink = attn_weights.slice(-1, .{ .end = topk.dim(.topk) }); return attn_weights_non_sink.dot(selected_kv, .kv).convert(q.dtype()); } diff --git a/zml/attention/tpu_attention.zig b/zml/attention/tpu_attention.zig index b0771ea9..0538f9b1 100644 --- a/zml/attention/tpu_attention.zig +++ b/zml/attention/tpu_attention.zig @@ -118,8 +118,8 @@ pub const mosaic_tpu = struct { } fn activeSequenceCount(query_start_len: zml.Tensor) zml.Tensor { - const start = query_start_len.slice1d(.b, .{ .end = query_start_len.dim(.b) - 1 }); - const end = query_start_len.slice1d(.b, .{ .start = 1 }); + const start = query_start_len.slice(.b, .{ .end = query_start_len.dim(.b) - 1 }); + const end = query_start_len.slice(.b, .{ .start = 1 }); const query_lens = end.sub(start); return query_lens .cmp(.GT, .zeroes(query_lens.shape())) @@ -144,7 +144,7 @@ pub const mosaic_tpu = struct { "mosaic_tpu ragged paged attention cannot restore output from head_dim {} to {}", .{ restored.dim(.hd), q_template.dim(.hd) }, ); - return restored.slice1d(.hd, .{ .end = q_template.dim(.hd) }); + return restored.slice(.hd, .{ .end = q_template.dim(.hd) }); } inline fn alignQueryHeadDimForKernel(q: zml.Tensor, target_head_dim: i64) zml.Tensor { diff --git a/zml/attention/triton_attention.zig b/zml/attention/triton_attention.zig index faf6d961..86d42732 100644 --- a/zml/attention/triton_attention.zig +++ b/zml/attention/triton_attention.zig @@ -846,8 +846,8 @@ pub const paged = struct { fn tokenToSequence(query_start_len: zml.Tensor, query_count: i64) zml.Tensor { const sequence_count = query_start_len.dim(.b) - 1; - const starts = query_start_len.slice1d(.b, .{ .end = sequence_count }).rename(.{ .b = .seq }); - const ends = query_start_len.slice1d(.b, .{ .start = 1 }).rename(.{ .b = .seq }); + const starts = query_start_len.slice(.b, .{ .end = sequence_count }).rename(.{ .b = .seq }); + const ends = query_start_len.slice(.b, .{ .start = 1 }).rename(.{ .b = .seq }); const query_x_sequence_shape = zml.Shape.init(.{ .q = query_count, .seq = sequence_count }, .i32); const query = zml.Tensor.iota(query_x_sequence_shape, .q).convert(.i32); const in_range = query.cmp(.GE, starts.broad(query_x_sequence_shape)).convert(.i32) @@ -865,7 +865,7 @@ pub const paged = struct { stdx.debug.assert(tokens_pos.shape().hasTags(.{.q}), "paged MLA token positions must have a .q axis, got {f}", .{tokens_pos.shape()}); const query_to_sequence = tokenToSequence(parameters.query_start_len, topk_i32.dim(.q)); - const sequence_ends = parameters.query_start_len.slice1d(.b, .{ .start = 1 }).rename(.{ .b = .seq }); + const sequence_ends = parameters.query_start_len.slice(.b, .{ .start = 1 }).rename(.{ .b = .seq }); const last_query = sequence_ends.gather(.{ .seq = query_to_sequence }, .{}).sub(.scalar(1, .i32)); const last_token_pos = tokens_pos .gather(.{ .q = last_query.rename(.{ .q = .lookup }) }, .{}) diff --git a/zml/moe/metal.zig b/zml/moe/metal.zig index 08d58a21..607adba5 100644 --- a/zml/moe/metal.zig +++ b/zml/moe/metal.zig @@ -78,8 +78,8 @@ fn requireScalePair(opts: Options, dtype: zml.DataType) !void { fn applyActivation(x: Tensor, mode: ActivationMode) Tensor { const mid = @divFloor(x.dim(.out), 2); - const gate = x.slice1d(.out, .{ .end = mid }); - const up = x.slice1d(.out, .{ .start = mid }); + const gate = x.slice(.out, .{ .end = mid }); + const up = x.slice(.out, .{ .start = mid }); const act = switch (mode) { .silu => gate.silu().mul(up), diff --git a/zml/moe/mosaic_tpu.zig b/zml/moe/mosaic_tpu.zig index 428a5332..7353c4dd 100644 --- a/zml/moe/mosaic_tpu.zig +++ b/zml/moe/mosaic_tpu.zig @@ -150,14 +150,14 @@ fn alignSortedRowsByGroup(rows: Tensor, expert_ids_sorted: Tensor, group_sizes: const group_ends = group_sizes.cumulativeSum(.expert); const group_starts = Tensor.concatenate(&.{ Tensor.zeroes(.init(.{ .expert = 1 }, .i32)), - group_ends.slice1d(.expert, .{ .end = num_groups - 1 }), + group_ends.slice(.expert, .{ .end = num_groups - 1 }), }, .expert); const padded_group_sizes = group_sizes.addConstant(tile_m - 1).divByConst(tile_m).mul(Tensor.scalar(tile_m, .i32)); const padded_group_ends = padded_group_sizes.cumulativeSum(.expert); const padded_group_starts = Tensor.concatenate(&.{ Tensor.zeroes(.init(.{ .expert = 1 }, .i32)), - padded_group_ends.slice1d(.expert, .{ .end = num_groups - 1 }), + padded_group_ends.slice(.expert, .{ .end = num_groups - 1 }), }, .expert); const logical_positions = Tensor.arange(.{ .end = rows.dim(.token) }, .i32).withTags(.{.token}); @@ -261,13 +261,13 @@ pub fn callGmmEp( }, ).out; - return out.slice1d(.out, .{ .end = out_n }); + return out.slice(.out, .{ .end = out_n }); } fn applyActivation(x: Tensor, mode: ActivationMode) Tensor { const mid = @divFloor(x.dim(.out), 2); - const gate = x.slice1d(.out, .{ .end = mid }); - const up = x.slice1d(.out, .{ .start = mid }); + const gate = x.slice(.out, .{ .end = mid }); + const up = x.slice(.out, .{ .start = mid }); return switch (mode) { .silu => gate.silu().mul(up), diff --git a/zml/moe/triton.zig b/zml/moe/triton.zig index 699e3cce..e4adcbcf 100644 --- a/zml/moe/triton.zig +++ b/zml/moe/triton.zig @@ -108,8 +108,8 @@ fn initZeroBiasBuffer(io: std.Io, platform: *const zml.Platform, sharding: zml.S fn applyActivation(x: Tensor, mode: Parameters.ActivationMode) Tensor { const mid = @divFloor(x.dim(.out), 2); - const gate = x.slice1d(.out, .{ .end = mid }); - const up = x.slice1d(.out, .{ .start = mid }); + const gate = x.slice(.out, .{ .end = mid }); + const up = x.slice(.out, .{ .start = mid }); return switch (mode) { .silu => gate.silu().mul(up), .relu => x.relu().powByConst(2), diff --git a/zml/nn.zig b/zml/nn.zig index 1449a1be..9ba61223 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -798,12 +798,12 @@ pub fn splitRealImg(x: Tensor, layout: RopeOpts.Layout) [2]Tensor { return switch (layout) { .real_im_pass, .real_pass_im_pass => .{ - x.slice1d(-1, .{ .end = @divExact(n, 2) }), - x.slice1d(-1, .{ .start = @divExact(n, 2), .end = n }), + x.slice(-1, .{ .end = @divExact(n, 2) }), + x.slice(-1, .{ .start = @divExact(n, 2), .end = n }), }, .interleaved => .{ - x.slice1d(-1, .{ .start = 0, .step = 2 }), - x.slice1d(-1, .{ .start = 1, .step = 2 }), + x.slice(-1, .{ .start = 0, .step = 2 }), + x.slice(-1, .{ .start = 1, .step = 2 }), }, }; } @@ -826,22 +826,22 @@ pub fn splitRealImgPass(x: Tensor, layout: RopeOpts.Layout, rotary_dim: u32) str const half_rotary = @divExact(rotary_dim, 2); return switch (layout) { .real_im_pass => .{ - x.slice1d(ax, .{ .end = half_rotary }), - x.slice1d(ax, .{ .start = half_rotary, .end = rotary_dim }), - .{ .real_im_pass = x.slice1d(ax, .{ .start = rotary_dim }) }, + x.slice(ax, .{ .end = half_rotary }), + x.slice(ax, .{ .start = half_rotary, .end = rotary_dim }), + .{ .real_im_pass = x.slice(ax, .{ .start = rotary_dim }) }, }, .real_pass_im_pass => .{ - x.slice1d(ax, .{ .end = half_rotary }), - x.slice1d(ax, .{ .start = half, .end = half + half_rotary }), + x.slice(ax, .{ .end = half_rotary }), + x.slice(ax, .{ .start = half, .end = half + half_rotary }), .{ .real_pass_im_pass = .{ - x.slice1d(ax, .{ .start = half_rotary, .end = half }), - x.slice1d(ax, .{ .start = half + half_rotary }), + x.slice(ax, .{ .start = half_rotary, .end = half }), + x.slice(ax, .{ .start = half + half_rotary }), } }, }, .interleaved => .{ - x.slice1d(ax, .{ .start = 0, .end = rotary_dim, .step = 2 }), - x.slice1d(ax, .{ .start = 1, .end = rotary_dim, .step = 2 }), - .{ .interleaved = x.slice1d(ax, .{ .start = rotary_dim }) }, + x.slice(ax, .{ .start = 0, .end = rotary_dim, .step = 2 }), + x.slice(ax, .{ .start = 1, .end = rotary_dim, .step = 2 }), + .{ .interleaved = x.slice(ax, .{ .start = rotary_dim }) }, }, }; } @@ -1584,7 +1584,7 @@ pub fn resizeBilinear(image: Tensor, resized_axes: anytype, opt: ResizeOpts) Ten for (new_size.constSlice(), tags_.constSlice()) |d, t| { const ax = image.shape().axis(t); const child_opt: ResizeOpts = .{ - .original_len = if (opt.original_len) |o| o.choose1d(0, ax) else null, + .original_len = if (opt.original_len) |o| o.slice(0, .single(ax)) else null, }; out = resizeLinear1d(out, ax, d, child_opt); } @@ -1651,7 +1651,7 @@ pub fn resizeBicubic(image: Tensor, resized_axes: anytype, opt: ResizeOpts) Tens for (new_size.constSlice(), tags_.constSlice()) |d, t| { const ax = image.shape().axis(t); const child_opt: ResizeOpts = .{ - .original_len = if (opt.original_len) |o| o.choose1d(0, ax) else null, + .original_len = if (opt.original_len) |o| o.slice(0, .single(ax)) else null, }; out = resizeCubic1d(out, ax, d, child_opt); } @@ -1890,7 +1890,7 @@ pub const GatedDeltaNet = struct { } fn sliceStep(input: Tensor, step_: Tensor) Tensor { - return input.dynamicSlice(.{ .s = Tensor.DynSlice{ .start = step_, .len = 1 } }).squeeze(.s); + return input.slice(.s, .dynSingle(step_)); } fn validateInitialState(gdn: GatedDeltaNet, state: State) void { @@ -2233,7 +2233,7 @@ fn fixupLogits(logits: Tensor, opts: DynamicSamplingStrategy) [2]Tensor { // this propagate to probs_sum and probs_max. const probs = x.softmax(.topk); const probs_sum = probs.cumulativeSum(.topk); - const probs_max = probs.slice1d(.topk, .{ .start = 0, .end = 1 }); + const probs_max = probs.slice(.topk, .{ .start = 0, .end = 1 }); const top_p = opts.top_p.convert(x.dtype()).broad(x.shape()); const min_p = opts.min_p.convert(x.dtype()).broad(probs_max.shape()).mul(probs_max).broad(x.shape()); diff --git a/zml/ops.zig b/zml/ops.zig index f88a8f94..1b872228 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1510,7 +1510,7 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, // So let us handle that. if (indices_shape.count() == 1 and idx_axes.len == 1) { return self - .dynamicSlice1d(idx_axes[0], .{ .start = indices_per_axis.get(0).asScalar(), .len = 1 }) + .slice(idx_axes[0], .dyn(indices_per_axis.get(0).asScalar(), 1)) // Keep downstream resharding after the slice. SPMD engines like Neuron otherwise // may hoist an all-gather before the slice and materialize the full tensor. .optimizationBarrier() diff --git a/zml/tensor.zig b/zml/tensor.zig index 72269269..e274738d 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -30,6 +30,8 @@ pub const Tensor = struct { _shape: Shape, _value: ?*const mlir.Value = null, + const ResolvedAxis = u3; + pub fn init(shape_like: anytype, dt: DataType) Tensor { return .fromShape(.init(shape_like, dt)); } @@ -121,14 +123,14 @@ pub const Tensor = struct { /// Returns the index of axis 'axis_'. /// /// 'axis_' can be an integer or a tag. - pub fn axis(self: Tensor, axis_: anytype) u3 { + pub fn axis(self: Tensor, axis_: anytype) ResolvedAxis { return self._shape.axis(axis_); } /// Returns the indices of each of the given axes. /// /// 'axis_' can be an integer or a tag. - pub fn axes(self: Tensor, axes_: anytype) stdx.BoundedArray(u3, constants.MAX_RANK) { + pub fn axes(self: Tensor, axes_: anytype) stdx.BoundedArray(ResolvedAxis, constants.MAX_RANK) { return self._shape.axes(axes_); } @@ -1847,19 +1849,43 @@ pub const Tensor = struct { pub const Slice = struct { start: i64 = 0, + dyn_start: ?Tensor = null, + len: i64 = to_the_end, end: i64 = to_the_end, step: i32 = 1, singleton: bool = false, - const full = .{ .start = 0, .end = to_the_end, .step = 1 }; + /// Select the full slice + pub const full: Slice = .{ .start = 0, .end = to_the_end, .step = 1 }; + /// Select a single element and drop the corresponding axis (to keep the axis use .{ .start = off, .len =1 }) pub fn single(offset: i64) Slice { return .{ .start = offset, .end = offset + 1, .singleton = true }; } + /// Select a single element and drop the corresponding axis (to keep the axis use .{ .start = off, .len =1 }) + pub fn dyn(start: Tensor, len: i64) Slice { + stdx.debug.assert(start.rank() == 0, "Slice.dyn expects scalar input, got {f}", .{start}); + return .{ .dyn_start = start, .len = len }; + } + + pub fn dynSingle(start: Tensor) Slice { + stdx.debug.assert(start.rank() == 0, "Slice.dynSingle expects scalar input, got {f}", .{start}); + return .{ .dyn_start = start, .len = 1, .singleton = true }; + } + pub fn absolute(s: Slice, d: i64) Slice { const start = if (s.start < 0) d + s.start else s.start; - const end = if (s.end == to_the_end) d else if (s.end < 0) d + s.end else s.end; + const end = if (s.singleton) + start + 1 + else if (s.end == to_the_end) + d + else if (s.end < 0) + d + s.end + else if (s.len != to_the_end) + start + s.len + else + s.end; const res: Slice = .{ .start = start, .end = end, .step = s.step, .singleton = s.singleton }; stdx.debug.assert(start < end, "Slice {f} is invalid for axis of dimension {d} (resolved to {f})", .{ s, d, res }); stdx.debug.assert(end <= d, "Slice {f} is invalid for axis of dimension {d} (resolved to {f})", .{ s, d, res }); @@ -1868,36 +1894,98 @@ pub const Tensor = struct { const to_the_end = std.math.maxInt(i64); - pub fn format(self: Slice, writer: *std.Io.Writer) !void { - if (self.singleton) { - try writer.print("[{d}]", .{self.start}); - } else if (self.end == to_the_end and self.step == 1) { - try writer.print("[{d}..]", .{self.start}); - } else if (self.step == 1) { - try writer.print("[{d}..{d}]", .{ self.start, self.end }); + pub fn format(s: Slice, writer: *std.Io.Writer) !void { + const end = if (s.len != to_the_end) s.start +| s.len else s.end; + if (s.dyn_start) |_| { + if (s.singleton) + try writer.writeAll("[x]") + else + try writer.print("[x..x+{}]", .{s.len}); + } else if (s.singleton) { + try writer.print("[{d}]", .{s.start}); + } else if (end == to_the_end and s.step == 1) { + try writer.print("[{d}..]", .{s.start}); + } else if (s.step == 1) { + try writer.print("[{d}..{d}]", .{ s.start, end }); } else { - try writer.print("[{d}..{d}:{d}]", .{ self.start, self.end, self.step }); + try writer.print("[{d}..{d}:{d}]", .{ s.start, end, s.step }); } } }; - /// Slices the input Tensor over the given axis using the given parameters. - pub fn slice1d(self: Tensor, axis_: anytype, s: Slice) Tensor { - var slices: [constants.MAX_RANK]Slice = @splat(.{}); - slices[self.axis(axis_)] = s; - return self.slice(slices[0..self.rank()]); + /// Slice a Tensor across a specific axes. + /// + /// Due to the nature of stablehlo, the length of the slices need to be known when compiling the IR. + /// Examples: + /// ``` + /// Tensor(.{.a=20,.b=30,.c=40 }).slice(.a, .{ .start = 5, .len = 11}); + /// Tensor(.{.a=20,.b=30,.c=40 }).slice(0, .single(5)); + /// ``` + pub fn slice(self: Tensor, axis_: anytype, slice_: Slice) Tensor { + return self.slices(@as([]const ResolvedAxis, &.{self.axis(axis_)}), &.{slice_}); } - /// Slices the input Tensor using the given parameters. - pub fn slice(self: Tensor, slices: []const Slice) Tensor { - var start_indices: [constants.MAX_RANK]i64 = undefined; - var strides: [constants.MAX_RANK]i64 = undefined; - var limit_indices: [constants.MAX_RANK]i64 = undefined; + /// Slices a Tensor across many axes. + /// + /// ``` + /// Tensor(.{.a=20,.b=30,.c=40 }).slices(.{ .a, .b }, &.{ .start = 5, .end = 11}, .dyn(x) }); + /// Tensor(.{.a=20,.b=30,.c=40 }).slices(.{ + /// .a = .{ .start = a_off, .len = 11 }, + /// .b = .{ .start = b_off, .len = 12 }, + /// }); + /// Tensor(.{ 20,30,40}).slices(.{ 0, 2 }, .{.dyn(a_off, .len = 20), .{ .start = 8, .len = 12 }}); + /// ``` + pub fn slices(self: Tensor, axes_: anytype, slices_: []const Slice) Tensor { + // Note: with typed slices, the callsites are less akwards. + if (@TypeOf(axes_) != []const ResolvedAxis) { + const resolved_axes: []const ResolvedAxis = self._shape.parseAxes(axes_)[0].slice(); + return self.slices(resolved_axes, slices_); + } + + const rk = self.rank(); + any_dyn: { + for (slices_) |s| { + if (s.dyn_start) |_| break :any_dyn; + } + return self.staticSlices(axes_, slices_); + } + + const zero = scalar(0, .i64).value(); + var start_indices: [constants.MAX_RANK]*const mlir.Value = @splat(zero); + var new_shape: Shape = self._shape; + var to_remove: Shape.AxesArray = .empty; + + for (axes_, slices_) |a, s| { + stdx.debug.assert(s.step == 1, "Tensor.slices doesn't support mixing strided and dynamic slices, got {f}", .{stdx.fmt.slice(slices_)}); + new_shape._dims.buffer[a] = s.len; + start_indices[a] = if (s.dyn_start) |dyn| + dyn.convert(.i64).value() + else + scalar(s.start, .i64).value(); + if (s.singleton) to_remove.appendAssumeCapacity(@intCast(a)); + } + + const op = dialects.stablehlo.dynamic_slice( + mlirCtx(), + self.value(), + new_shape.dims(), + start_indices[0..rk], + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + const res = _result(new_shape, op.result(0)); + return res.reshape(new_shape.removeMany(to_remove.constSlice())); + } + + fn staticSlices(self: Tensor, axes_: []const ResolvedAxis, slices_: []const Slice) Tensor { + var start_indices: [constants.MAX_RANK]i64 = @splat(0); + var strides: [constants.MAX_RANK]i64 = @splat(1); + var limit_indices: [constants.MAX_RANK]i64 = self._shape._dims.buffer; var res_shape: Shape = self._shape; - for (slices, 0..) |s, a| { - stdx.debug.assert(s.step > 0, "slice expects 'step' to be positive, got {} at index {}", .{ s.step, a }); - stdx.debug.assert(s.step > 0, "slice expects 'step' to be positive, got {} at index {}", .{ s.step, a }); + for (axes_, slices_) |a, s| { + stdx.debug.assert(s.step > 0, "slice expects 'step' to be positive, got {} on axis {}", .{ s.step, a }); + stdx.debug.assert(s.step > 0, "slice expects 'step' to be positive, got {} on axis {}", .{ s.step, a }); const args: Slice = s.absolute(self.dim(a)); start_indices[a] = args.start; @@ -1919,8 +2007,8 @@ pub const Tensor = struct { var res = _result(res_shape, slice_op.result(0)); var to_remove: Shape.AxesArray = .empty; - for (slices, 0..) |s, a| { - if (s.singleton) to_remove.appendAssumeCapacity(@intCast(a)); + for (axes_, slices_) |a, s| { + if (s.singleton) to_remove.appendAssumeCapacity(a); } return res.reshape(res_shape.removeMany(to_remove.constSlice())); } @@ -1934,10 +2022,11 @@ pub const Tensor = struct { var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); defer x_buffer.deinit(); - // Wrap slice1d to hide the anytype in the signature. + // Wrap slice to hide the anytype in the signature. + // TODO(wzk): stdx.meta.specifyFn (waiting 0.17 for SoA meta programming) const Local = struct { - pub fn _slice1dAxis(input: Tensor, ax: i8, slice_: Tensor.Slice) Tensor { - return input.slice1d(ax, slice_); + fn _slice1dAxis(input: Tensor, ax: i8, slice_: Tensor.Slice) Tensor { + return input.slice(ax, slice_); } }; @@ -1957,7 +2046,40 @@ pub const Tensor = struct { ); defer exe.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._slice1dAxis, .{x_buffer}); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._slice1dAxis, .{ x_buffer, .{} }); + defer res.deinit(); + + try std.testing.expectEqual(expectation, try res.getValue(@TypeOf(expectation), std.testing.io)); + } + } + + test "dynamic slices" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const Local = struct { + fn dynSliceResolved(x: Tensor, ax: ResolvedAxis, dyn_slice: Slice) Tensor { + return x.slice(ax, dyn_slice); + } + }; + + inline for (.{ + .{ [10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, Shape.init(.{10}, .f32), [2]f32{ 4, 5 }, 4, 0 }, + .{ [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }, Shape.init(.{ 2, 5 }, .f32), [4]f32{ 3, 4, 8, 9 }, 3, 1 }, + }) |testcase| { + const x_data, const x_shape, const expectation, const z_value: i32, const ax = testcase; + const x: Tensor = .fromShape(x_shape); + const z: Tensor = .init(.{}, .i32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.dynSliceResolved, .{ x, ax, .dyn(z, 2) }, platform, .{}); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&x_data)); + defer x_buffer.deinit(); + var z_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, z.shape(), .replicated, std.mem.asBytes(&z_value)); + defer z_buffer.deinit(); + + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local.dynSliceResolved, .{ x_buffer, .{ .dyn_start = z_buffer } }); defer res.deinit(); try std.testing.expectEqual(expectation, try res.getValue(@TypeOf(expectation), std.testing.io)); @@ -1968,20 +2090,6 @@ pub const Tensor = struct { return if (idx < 0) self.dim(axis_) + idx else idx; } - pub fn choose1d(self: Tensor, axis_: anytype, i: i64) Tensor { - return self.slice1d(axis_, .single(i)); - } - - pub fn choose(self: Tensor, offsets: anytype) Tensor { - const off, const tags = Shape.parseDimensions(offsets); - var slices: [constants.MAX_RANK]Slice = @splat(.{}); - for (off.constSlice(), tags.constSlice()) |o, t| { - const ax = self.axis(t); - slices[ax] = .single(o); - } - return self.slice(slices[0..self.rank()]); - } - /// Concatenates the input Tensors along the given axis. pub fn concatenate(tensors: []const Tensor, axis_: anytype) Tensor { if (tensors.len == 1) return tensors[0]; @@ -3004,7 +3112,7 @@ pub const Tensor = struct { pub fn _scatterCB(self: Tensor, coords: Tensor, updates: Tensor) Tensor { return self.scatterSlices( - .{ .c = coords.choose1d(.coord, 0), .b = coords.choose1d(.coord, 1) }, + .{ .c = coords.slice(.coord, .single(0)), .b = coords.slice(.coord, .single(1)) }, updates, .{ .update_fn = ScatterOpts.increment }, ); @@ -3434,8 +3542,8 @@ pub const Tensor = struct { }, .cpu, .cuda, .rocm, .tpu, .oneapi, .metal => blk: { var sorted = self.sort(a, .{ .descending = opts.descending }); - sorted.values = sorted.values.slice1d(a, .{ .end = k }); - sorted.indices = sorted.indices.slice1d(a, .{ .end = k }); + sorted.values = sorted.values.slice(a, .{ .end = k }); + sorted.indices = sorted.indices.slice(a, .{ .end = k }); break :blk sorted; }, }; @@ -3546,7 +3654,7 @@ pub const Tensor = struct { var chunks: [n_chunks]Tensor = undefined; for (0..n_chunks) |i| { const start: i64 = @as(i64, @intCast(i)) * chunk_size; - chunks[i] = self.slice1d(a, .{ .start = start, .end = start + chunk_size }); + chunks[i] = self.slice(a, .{ .start = start, .end = start + chunk_size }); } return chunks; } @@ -3599,7 +3707,7 @@ pub const Tensor = struct { for (0.., chunks) |i, *chunk| { const start: i64 = @as(i64, @intCast(i)) * chunk_size; - chunk.* = self.slice1d(a, .{ .start = start, .end = @min(start + chunk_size, d) }); + chunk.* = self.slice(a, .{ .start = start, .end = @min(start + chunk_size, d) }); } return chunks; } @@ -3656,7 +3764,7 @@ pub const Tensor = struct { var start: i64 = 0; for (split_sizes, 0..) |n, i| { - res[i] = self.slice1d(a, .{ .start = start, .end = start + n }); + res[i] = self.slice(a, .{ .start = start, .end = start + n }); start += n; } return res; @@ -3706,109 +3814,6 @@ pub const Tensor = struct { } } - /// Slices the input Tensor along a specific axis, with a start offset known at runtime. - /// Note: this doesn't support tagging, if you have tags, - /// you should use `dynamicSlice` directly. - pub fn dynamicSlice1d(self: Tensor, axis_: i8, slice_: DynSlice) Tensor { - stdx.debug.assert(slice_.start.rank() == 0, "dynamicSlice1d expects 'slice_.start' tensor rank to be a scalar, got {f}", .{slice_.start}); - - const a = self.axis(axis_); - const new_shape = self._shape.set(a, slice_.len); - - var start_indices: [constants.MAX_RANK]*const mlir.Value = @splat(constant(slice_.start.dtype().zero()).value()); - start_indices[a] = slice_.start.value(); - - const op = dialects.stablehlo.dynamic_slice( - mlirCtx(), - self.value(), - new_shape.dims(), - start_indices[0..self.rank()], - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - - return _result(new_shape, op.result(0)); - } - - pub const DynSlice = struct { start: Tensor, len: i64 }; - - /// Slices a Tensor across many axes, with runtime known offsets. - /// - /// Due to the nature of stablehlo, the length of the slices need to be known when compiling the IR. - /// When using the tagged API it is allowed to not specify some axes. - /// But with the non-tagged API all slices need to be specified. - /// Examples: - /// ``` - /// Tensor(.{.a=20,.b=30,.c=40 }).dynamicSlice(.{ .a = .{ .start = a_off, .len = 11}); - /// Tensor(.{.a=20,.b=30,.c=40 }).dynamicSlice(.{ - /// .a = .{ .start = a_off, .len = 11 }, - /// .b = .{ .start = b_off, .len = 12 }, - /// }); - /// Tensor(.{ 20,30,40}).dynamicSlice(.{.{ .start = scalar(0, .i32), .len = 20 }, .{ .start = b_off, .len = 12 }, .{ .start = scalar(0, .i32), .len = 40 }}); - /// ``` - pub fn dynamicSlice(self: Tensor, slices_: anytype) Tensor { - // TODO: the untagged api is a bit verbose. Should I allow: `Tensor(.{ 20,30,40}).dynamicSlice(.{.{}, .{ .start = b_off, .len = 12 }, .{}});` ?? - // - const slices, const slices_tags = Shape.parseStruct(DynSlice, slices_); - - // TODO use slices and slices_tags for the format. - // Currently this prints: "dynSlice(struct{q: struct{start: tensor.Tensor, comptime len: comptime_int = 1}}{ .q = struct{start: tensor.Tensor, comptime len: comptime_int = 1}{ .start = Tensor({1,10}, dtype=.i64), .len = 1 } })" - // which is kinda ugly. - - const idx_dtype = if (slices.len > 0) slices.get(0).start.dtype() else .i32; - const zero = Tensor.scalar(0, idx_dtype).value(); - var offset_values: [constants.MAX_RANK]*const mlir.Value = @splat(zero); - var res_shape = self._shape; - for (slices.constSlice(), 0..) |slice_, i| { - const offset = slice_.start; - const len = slice_.len; - if (slices_tags.len == 0) { - stdx.debug.assert(self.rank() == slices.len, "dynamicSlice expects tensor rank and 'slices_' length to be equal, got {d} and {d}", .{ self.rank(), slices.len }); - - offset_values[i] = offset.value(); - res_shape._dims.set(i, len); - - stdx.debug.assert(len <= self.dim(i), "dynamicSlice expects slices 'len' to be less than or equal to their corresponding dimension in input tensor, got {d} and {d} for index {d}", .{ len, self.dim(i), i }); - } else { - const t = slices_tags.get(i); - const a = res_shape.hasTag(t) orelse stdx.debug.panic("dynamicSlice expects input tensor to have tags used in 'slices_' but {s} is missing (input shape is {f})", .{ t, self._shape }); - - stdx.debug.assert(len <= self.dim(a), "dynamicSlice expects slices 'len' to be less than their corresponding dimension in input tensor, got {d} and {d} for axis {s}", .{ len, self.dim(a), t }); - - offset_values[a] = offset.value(); - res_shape._dims.set(a, len); - } - } - const op = dialects.stablehlo.dynamic_slice(mlirCtx(), self.value(), res_shape.dims(), offset_values[0..self.rank()], .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(res_shape, op.result(0)); - } - - test dynamicSlice { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - inline for (.{ - .{ [10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, Shape.init(.{10}, .f32), [2]f32{ 4, 5 }, 4, 0 }, - .{ [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }, Shape.init(.{ 2, 5 }, .f32), [4]f32{ 3, 4, 8, 9 }, 3, 1 }, - }) |testcase| { - const x_data, const x_shape, const expectation, const z_value: i32, const ax = testcase; - const x: Tensor = .fromShape(x_shape); - const z: Tensor = .init(.{}, .i32); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.dynamicSlice1d, .{ x, ax, .{ .len = 2, .start = z } }, platform, .{}); - defer exe.deinit(); - - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&x_data)); - defer x_buffer.deinit(); - var z_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, z.shape(), .replicated, std.mem.asBytes(&z_value)); - defer z_buffer.deinit(); - - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.dynamicSlice1d, .{ x_buffer, .{ .start = z_buffer } }); - defer res.deinit(); - - try std.testing.expectEqual(expectation, try res.getValue(@TypeOf(expectation), std.testing.io)); - } - } - /// Updates a slice of the input Tensor along a specific axis using the given 'update' Tensor, with a start offset known at runtime. /// Note this is the untagged api, if you have tags, you should use dynamicUpdateSlice directly. pub fn dynamicUpdateSlice1d(self: Tensor, update: Tensor, axis_: i64, offset: Tensor) Tensor { @@ -4389,8 +4394,8 @@ pub const Tensor = struct { const k = @mod(shift, n); // normalize shift to [0, n) if (k == 0) return self; return Tensor.concatenate(&.{ - self.slice1d(a, .{ .start = n - k, .end = n }), - self.slice1d(a, .{ .start = 0, .end = n - k }), + self.slice(a, .{ .start = n - k, .end = n }), + self.slice(a, .{ .start = 0, .end = n - k }), }, a); } From 805cf8cd74437d037e6d7e0230d297b2526b35ac Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 27 Aug 2026 17:32:20 +0200 Subject: [PATCH 636/881] zml/moe: fixup triton.zig (#734) fix after conflict between #729 and #730 --- zml/moe/moe.zig | 4 ++++ zml/moe/triton.zig | 10 +++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index 6f3e684c..5c0c3883 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -9,6 +9,10 @@ pub const mosaic_tpu = @import("mosaic_tpu.zig"); pub const triton = @import("triton.zig"); pub const triton_kernels = @import("triton_kernels/triton_kernels.zig"); +test { + std.testing.refAllDecls(@This()); +} + pub const ActivationMode = enum { silu, relu, diff --git a/zml/moe/triton.zig b/zml/moe/triton.zig index e4adcbcf..9da3177d 100644 --- a/zml/moe/triton.zig +++ b/zml/moe/triton.zig @@ -11,11 +11,15 @@ const Shape = zml.Shape; const tri = zml.kernel.triton; const DType = tri.DType; const toDType = tri.from; -const kernels = @import("triton_kernels/triton_kernels.zig"); const a16w4_kernel = @import("triton_kernels/a16w4_kernel.zig"); +const kernels = @import("triton_kernels/triton_kernels.zig"); const log = std.log.scoped(.moe_triton); +test { + std.testing.refAllDecls(@This()); +} + // ============================================================================= // Public API // ============================================================================= @@ -1243,8 +1247,8 @@ fn runGemm( } fn applySwiGlu(input: zml.Tensor, activation_limit: ?f32) zml.Tensor { - var gate = input.slice1d(.dout, .{ .start = 0, .step = 2 }); - var up = input.slice1d(.dout, .{ .start = 1, .step = 2 }); + var gate = input.slice(.dout, .{ .start = 0, .step = 2 }); + var up = input.slice(.dout, .{ .start = 1, .step = 2 }); if (activation_limit) |limit| { const threshold = zml.Tensor.scalar(limit, .f32); From cd477b858a920732463a824dfdb43f4b613e635b Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 27 Aug 2026 18:31:50 +0200 Subject: [PATCH 637/881] zml: refactor `module.zig` into `Compiler.zig` (#736) Migration instructions: zml.module.CompilationContext -> zml.Compiler zml.module.CompileError -> zml.Compiler.Error zml.module.CompilationOptions -> zml.Compiler.Options zml.module.compile -> zml.Platform.compiler or zml.Compiler.compile Breaking changes: none, I've kept a small compatibility layer at the top level, with upgrade instructions. We can remove it in a second time. --- docs/learn/concepts.md | 2 +- examples/llm/models/qwen3_5/model.zig | 2 +- examples/llm/models/qwen3_5_moe/model.zig | 2 +- examples/sharding/main.zig | 2 +- kernels/mosaic_tpu/README.md | 2 +- kernels/mosaic_tpu/builder.zig | 2 +- kernels/triton/README.md | 8 +- zml/BUILD.bazel | 2 +- zml/{module.zig => Compiler.zig} | 479 +++++++++--------- zml/attention/attnd.zig | 2 +- zml/attention/flashattn.zig | 5 +- zml/attention/tpu_attention.zig | 4 +- zml/attention/triton_attention.zig | 6 +- .../triton_kernels/unified_attention.zig | 2 +- zml/exe.zig | 10 +- zml/kernel.zig | 8 +- zml/moe/cutlass_flashinfer.zig | 2 +- zml/moe/triton.zig | 2 +- zml/nn.zig | 56 +- zml/ops.zig | 116 ++--- zml/platform.zig | 8 +- zml/tensor.zig | 58 +-- zml/zml.zig | 23 +- 23 files changed, 404 insertions(+), 399 deletions(-) rename zml/{module.zig => Compiler.zig} (64%) diff --git a/docs/learn/concepts.md b/docs/learn/concepts.md index 0b9a6f35..1a3cea05 100644 --- a/docs/learn/concepts.md +++ b/docs/learn/concepts.md @@ -138,7 +138,7 @@ corresponding types. 5. Load some user inputs (custom struct), encode them into arrays of numbers (`zml.Slice`), and copy them to the accelerator (`zml.Buffer`). -6. Call the executable on the user inputs. `module.call` accepts `zml.Buffer` +6. Call the executable on the user inputs. `exe.call` accepts `zml.Buffer` arguments and returns `zml.Buffer` 7. Return the model output (`zml.Buffer`) to the host (`zml.Slice`), diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 7ddafe9c..90f1ced4 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -639,7 +639,7 @@ pub const SelfAttn = struct { var q, var gate = self.projectQAndGate(x_qkv); var k, var v = self.projectKV(x_qkv); - const kv_head_sharding = zml.module.CompilationContext.current().partitioning.shardableDim( + const kv_head_sharding = zml.Compiler.current().partitioning.shardableDim( k.shape().withPartitioning(.{ .h = .model }), .h, q.dim(.h), diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 98d6ae25..6b7b9bc7 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -623,7 +623,7 @@ pub const SelfAttn = struct { const x_qkv = x.withPartitioning(.{ .d = .replicated }); var q, var gate = self.projectQAndGate(x_qkv); var k, var v = self.projectKV(x_qkv); - const kv_head_sharding = zml.module.CompilationContext.current().partitioning.shardableDim( + const kv_head_sharding = zml.Compiler.current().partitioning.shardableDim( k.shape().withPartitioning(.{ .h = .model }), .h, q.dim(.h), diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index cc028539..4e919c7f 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -7,7 +7,7 @@ const log = std.log.scoped(.sharding); pub const std_options: std.Options = .{ .log_level = .info, .log_scope_levels = &.{ - .{ .scope = .@"zml/module", .level = .debug }, + .{ .scope = .@"zml/compiler", .level = .debug }, }, }; diff --git a/kernels/mosaic_tpu/README.md b/kernels/mosaic_tpu/README.md index 8ebf3d06..2b0d7004 100644 --- a/kernels/mosaic_tpu/README.md +++ b/kernels/mosaic_tpu/README.md @@ -20,7 +20,7 @@ Two layers define the surface: for the final IR string. Pass `Builder.open` / `Builder.init` a `zml.kernel.mosaic_tpu.newContext()` (a throwaway context with the dialects the DSL emits), `defer ctx.deinit()` once you have the string — not a - `CompilationContext`'s `mlir_ctx`. (`Builder.buildOpts(...)` is a convenience + `Compilation`'s `mlir_ctx`. (`Builder.buildOpts(...)` is a convenience that bundles `open + declareArgs` into one call and returns a heap-allocated `*Built(Spec)`; reach for it in escape-hatch code that drives the lifecycle manually.) - **High layer** — `zml.kernel.mosaic_tpu.Kernel(Config, spec)`: the diff --git a/kernels/mosaic_tpu/builder.zig b/kernels/mosaic_tpu/builder.zig index 1801cd12..157648df 100644 --- a/kernels/mosaic_tpu/builder.zig +++ b/kernels/mosaic_tpu/builder.zig @@ -2209,7 +2209,7 @@ pub const Builder = struct { defer pm.deinit(); // `canonicalize` folds `x * 1`, `x + 0`, etc.; `cse` then // dedupes identical constants and subexpressions. Same - // pipeline `zml/module.zig` runs on every compiled module. + // pipeline `zml/Compiler.zig` runs on every compiled module. const opm = pm.asOpPassManager(); inline for (.{ "canonicalize", "cse", "canonicalize" }) |pass| { try opm.addPipeline(pass); diff --git a/kernels/triton/README.md b/kernels/triton/README.md index d36fae08..8860107d 100644 --- a/kernels/triton/README.md +++ b/kernels/triton/README.md @@ -13,7 +13,7 @@ Two layers define the surface: `Builder.open` / `Builder.init` take an `*mlir.Context` — pass them a `zml.kernel.triton.newContext()` (a throwaway context with the dialects the DSL emits), `defer ctx.deinit()` once you have the IR string. Not a - `CompilationContext`'s `mlir_ctx`: that one carries only `func`/`stablehlo`/`sdy` + `zml.Compiler`'s `mlir_ctx`: that one carries only `func`/`stablehlo`/`sdy` (a registered `llvm` dialect makes XLA:TPU choke). - **High layer** — `zml.kernel.triton.Kernel(Config, spec)`: the declarative kernel form. Bundles a config type, a typed spec literal (name + named @@ -195,8 +195,8 @@ pub fn forward(self: Model, x: zml.Tensor, y: zml.Tensor) zml.Tensor { ``` `call` is a thin wrapper over `Kernel.emit(...)` + `zml.ops.triton(...)`. -It must be called from inside a `CompilationContext` (i.e., during -`zml.module.compile(...)`) — `emit` builds the TTIR string in its own +It must be called from inside a `Compilation` (i.e., during +`zml.Compiler.compile(...)`) — `emit` builds the TTIR string in its own throwaway context; `zml.ops.triton(...)` drops the `stablehlo.custom_call` into the model's context. For offline tools, `Kernel.emit(allocator, cfg)` gives you just the TTIR string. @@ -1010,7 +1010,7 @@ builder and call `b.declareArgs(spec)` mid-stream — that's exactly what `zml.kernel.triton.Kernel(...)` does internally. Pass these constructors a `zml.kernel.triton.newContext()`, not a -`CompilationContext`'s `mlir_ctx`; `defer ctx.deinit()` once you have the IR +`Compilation`'s `mlir_ctx`; `defer ctx.deinit()` once you have the IR string. `Kernel.emit` does this for you, which is why it takes no context. Prefer `zml.kernel.triton.Kernel(Config, spec)` for the common case. diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index b8f5432e..5d8304cb 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -99,7 +99,7 @@ zig_library( "mem.zig", "meta.zig", "mlirx.zig", - "module.zig", + "Compiler.zig", "moe/cutlass_flashinfer.zig", "moe/metal.zig", "moe/moe.zig", diff --git a/zml/module.zig b/zml/Compiler.zig similarity index 64% rename from zml/module.zig rename to zml/Compiler.zig index 14c451a5..e973f75c 100644 --- a/zml/module.zig +++ b/zml/Compiler.zig @@ -22,9 +22,29 @@ const Sharding = @import("Sharding.zig"); const Partitioning = Sharding.Partitioning; const Tensor = @import("tensor.zig").Tensor; -const zml_module = @This(); -const log = std.log.scoped(.@"zml/module"); +const Compiler = @This(); +const log = std.log.scoped(.@"zml/compiler"); +allocator: std.mem.Allocator, +io: std.Io, +arena: std.heap.ArenaAllocator, + +mlir_registry: *mlir.DialectRegistry, +mlir_ctx: *mlir.Context, +mlir_pass_manager: *mlir.PassManager, +module: *mlir.Module, +platform: *const Platform, +partitioning: Sharding.Partitioning, + +mlir_known_types: std.enums.EnumArray(DataType, *const mlir.Type), + +scopes: stdx.BoundedArray(Scope, 16) = .empty, +manual_computation_depth: usize = 0, + +channel_id: i64 = 0, +composite_id: i64 = 0, + +threadlocal var _current: ?*Compiler = null; var mlir_global_init_mutex: std.Io.Mutex = .init; var mlir_global_registry: ?*mlir.DialectRegistry = null; @@ -46,7 +66,8 @@ fn mlirRegistry(io: std.Io) *mlir.DialectRegistry { return mlir_global_registry.?; } -pub const CompilationOptions = struct { + +pub const Options = struct { shardings: []const Sharding = &.{}, // If null, will be initialized from the target partitioner: ?Sharding.Partitioner = null, @@ -59,7 +80,7 @@ pub const CompilationOptions = struct { }; /// Errors surfaced while lowering and compiling a ZML program. -pub const CompileError = std.mem.Allocator.Error || +pub const Error = std.mem.Allocator.Error || std.Io.Writer.Error || mlir.Error || upb.SerializeError || @@ -68,199 +89,175 @@ pub const CompileError = std.mem.Allocator.Error || const AttributeList = stdx.BoundedArray(mlir.NamedAttribute, 3); -pub const CompilationContext = struct { - pub const Scope = struct { - block: *mlir.Block, - id_to_argument: std.AutoArrayHashMapUnmanaged(usize, usize), - id_to_donation: std.AutoArrayHashMapUnmanaged(usize, usize), - id_to_output_memory_kind: std.AutoArrayHashMapUnmanaged(usize, Memory.Kind), - id_to_input_memory_kind: std.AutoArrayHashMapUnmanaged(usize, Memory.Kind), - arena: std.heap.ArenaAllocator, - - pub fn initFromBlock(allocator: std.mem.Allocator, block: *mlir.Block) Scope { - const arena: std.heap.ArenaAllocator = .init(allocator); - return .{ - .block = block, - .id_to_argument = .empty, - .id_to_donation = .empty, - .id_to_output_memory_kind = .empty, - .id_to_input_memory_kind = .empty, - .arena = arena, - }; - } - - pub fn deinit(self: *Scope) void { - self.arena.deinit(); - } - }; - - allocator: std.mem.Allocator, - io: std.Io, +pub const Scope = struct { + block: *mlir.Block, + id_to_argument: std.AutoArrayHashMapUnmanaged(usize, usize), + id_to_donation: std.AutoArrayHashMapUnmanaged(usize, usize), + id_to_output_memory_kind: std.AutoArrayHashMapUnmanaged(usize, Memory.Kind), + id_to_input_memory_kind: std.AutoArrayHashMapUnmanaged(usize, Memory.Kind), arena: std.heap.ArenaAllocator, - mlir_registry: *mlir.DialectRegistry, - mlir_ctx: *mlir.Context, - mlir_pass_manager: *mlir.PassManager, - module: *mlir.Module, - platform: *const Platform, - partitioning: Sharding.Partitioning, - - mlir_known_types: std.enums.EnumArray(DataType, *const mlir.Type), - - scopes: stdx.BoundedArray(Scope, 16) = .empty, - manual_computation_depth: usize = 0, - - channel_id: i64 = 0, - - composite_id: i64 = 0, - - threadlocal var _current: ?*CompilationContext = null; - - pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: CompilationOptions) CompilationContext { - var arena = std.heap.ArenaAllocator.init(allocator); - const mlir_registry = mlirRegistry(io); - var mlir_ctx = mlir.Context.init(.{ .registry = mlir_registry, .threading = false }) catch unreachable; - mlir_ctx.loadAllAvailableDialects(); - - const module = mlir.Module.init(.unknown(mlir_ctx)); - module.operation().setAttributeByName("sym_name", .string(mlir_ctx, opts.program_name)); - - const pass_manager = mlir.PassManager.init(mlir_ctx); - { - var opm = pass_manager.asOpPassManager(); - const passes: []const []const u8 = &.{ - "canonicalize", - "cse", - "canonicalize", - }; - for (passes) |pass| { - opm.addPipeline(pass) catch unreachable; - } - } - - var mlir_known_types: std.enums.EnumArray(DataType, *const mlir.Type) = .initUndefined(); - { - for (0.., &mlir_known_types.values) |i, *mlir_type| { - mlir_type.* = mlirx.Type.fromDType(mlir_ctx, @enumFromInt(i)); - } - } - - // Ensure replicated sharding is always included as a fallback option. - var shardings = std.ArrayList(Sharding).initCapacity(arena.allocator(), opts.shardings.len + 1) catch @panic("OOM"); - var needs_replicated: bool = true; - for (opts.shardings) |sharding| { - if (sharding.data == platform.replicated_sharding.data) needs_replicated = false; - shardings.appendAssumeCapacity(sharding.resolve(platform)); - } - if (needs_replicated) shardings.appendAssumeCapacity(platform.replicated_sharding); - - const partitioning = Sharding.Partitioning.init(opts.partitioner orelse .fromTarget(platform.target), shardings.items) catch @panic("OOM"); - + pub fn initFromBlock(allocator: std.mem.Allocator, block: *mlir.Block) Scope { + const arena: std.heap.ArenaAllocator = .init(allocator); return .{ - .allocator = allocator, - .io = io, + .block = block, + .id_to_argument = .empty, + .id_to_donation = .empty, + .id_to_output_memory_kind = .empty, + .id_to_input_memory_kind = .empty, .arena = arena, - .mlir_registry = mlir_registry, - .mlir_ctx = mlir_ctx, - .mlir_pass_manager = pass_manager, - .mlir_known_types = mlir_known_types, - .module = module, - .platform = platform, - .partitioning = partitioning, }; } - pub fn deinit(self: *CompilationContext) void { - if (_current == self) _current = null; - for (self.scopes.slice()) |*scope| { - scope.deinit(); - } - self.mlir_pass_manager.deinit(); - self.module.deinit(); - self.mlir_ctx.deinit(); + pub fn deinit(self: *Scope) void { self.arena.deinit(); } - - pub fn activate(self: *CompilationContext) void { - std.debug.assert(_current == null); - _current = self; - } - - pub fn deactivate(self: *CompilationContext) void { - _ = self; - _current = null; - } - - pub fn current() *CompilationContext { - return _current.?; - } - - pub fn currentOrNull() ?*CompilationContext { - return _current; - } - - pub fn currentScope(self: *CompilationContext) *Scope { - return &self.scopes.slice()[self.scopes.len - 1]; - } - - pub fn pushBlock(self: *CompilationContext, block: *mlir.Block) void { - const scope = Scope.initFromBlock(self.allocator, block); - self.scopes.appendAssumeCapacity(scope); - } - - pub fn popBlock(self: *CompilationContext) void { - var maybe_popped_scope = self.scopes.pop(); - if (maybe_popped_scope) |*popped| { - popped.deinit(); - } - } - - pub fn nextChannelId(self: *CompilationContext) i64 { - self.channel_id += 1; - return self.channel_id; - } - - pub fn nextCompositeId(self: *CompilationContext) i64 { - self.composite_id += 1; - return self.composite_id; - } - - pub fn mlirType(self: *const CompilationContext, dt: DataType) *const mlir.Type { - return self.mlir_known_types.get(dt); - } - - pub fn dtype(self: *const CompilationContext, mlir_type: *const mlir.Type) DataType { - @setRuntimeSafety(false); - for (0.., &self.mlir_known_types.values) |i, known_type| { - if (known_type == mlir_type) return @enumFromInt(i); - } - std.debug.panic("Can't convert unknown mlir type to dtype: {f}", .{mlir_type}); - } - - pub fn abortOOM(self: *CompilationContext) noreturn { - _ = self; - @panic("OOM"); - } - - pub fn alloc(self: *CompilationContext, T: type, n: usize) []T { - return self.arena.allocator().alloc(T, n) catch self.abortOOM(); - } - - pub fn allocPrint(self: *CompilationContext, comptime fmt: []const u8, args: anytype) []u8 { - return std.fmt.allocPrint(self.arena.allocator(), fmt, args) catch self.abortOOM(); - } }; -pub fn Compiler(comptime func: anytype) type { +pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: Options) Compiler { + var arena = std.heap.ArenaAllocator.init(allocator); + const mlir_registry = mlirRegistry(io); + var mlir_ctx = mlir.Context.init(.{ .registry = mlir_registry, .threading = false }) catch unreachable; + mlir_ctx.loadAllAvailableDialects(); + + const module = mlir.Module.init(.unknown(mlir_ctx)); + module.operation().setAttributeByName("sym_name", .string(mlir_ctx, opts.program_name)); + + const pass_manager = mlir.PassManager.init(mlir_ctx); + { + var opm = pass_manager.asOpPassManager(); + const passes: []const []const u8 = &.{ + "canonicalize", + "cse", + "canonicalize", + }; + for (passes) |pass| { + opm.addPipeline(pass) catch unreachable; + } + } + + var mlir_known_types: std.enums.EnumArray(DataType, *const mlir.Type) = .initUndefined(); + { + for (0.., &mlir_known_types.values) |i, *mlir_type| { + mlir_type.* = mlirx.Type.fromDType(mlir_ctx, @enumFromInt(i)); + } + } + + // Ensure replicated sharding is always included as a fallback option. + var shardings = std.ArrayList(Sharding).initCapacity(arena.allocator(), opts.shardings.len + 1) catch @panic("OOM"); + var needs_replicated: bool = true; + for (opts.shardings) |sharding| { + if (sharding.data == platform.replicated_sharding.data) needs_replicated = false; + shardings.appendAssumeCapacity(sharding.resolve(platform)); + } + if (needs_replicated) shardings.appendAssumeCapacity(platform.replicated_sharding); + + const partitioning = Sharding.Partitioning.init(opts.partitioner orelse .fromTarget(platform.target), shardings.items) catch @panic("OOM"); + + return .{ + .allocator = allocator, + .io = io, + .arena = arena, + .mlir_registry = mlir_registry, + .mlir_ctx = mlir_ctx, + .mlir_pass_manager = pass_manager, + .mlir_known_types = mlir_known_types, + .module = module, + .platform = platform, + .partitioning = partitioning, + }; +} + +pub fn deinit(self: *Compiler) void { + if (_current == self) _current = null; + for (self.scopes.slice()) |*scope| { + scope.deinit(); + } + self.mlir_pass_manager.deinit(); + self.module.deinit(); + self.mlir_ctx.deinit(); + self.arena.deinit(); +} + +pub fn activate(self: *Compiler) void { + std.debug.assert(_current == null); + _current = self; +} + +pub fn deactivate(self: *Compiler) void { + _ = self; + _current = null; +} + +pub fn current() *Compiler { + return _current.?; +} + +pub fn currentOrNull() ?*Compiler { + return _current; +} + +pub fn currentScope(self: *Compiler) *Scope { + return &self.scopes.slice()[self.scopes.len - 1]; +} + +pub fn pushBlock(self: *Compiler, block: *mlir.Block) void { + const scope = Scope.initFromBlock(self.allocator, block); + self.scopes.appendAssumeCapacity(scope); +} + +pub fn popBlock(self: *Compiler) void { + var maybe_popped_scope = self.scopes.pop(); + if (maybe_popped_scope) |*popped| { + popped.deinit(); + } +} + +pub fn nextChannelId(self: *Compiler) i64 { + self.channel_id += 1; + return self.channel_id; +} + +pub fn nextCompositeId(self: *Compiler) i64 { + self.composite_id += 1; + return self.composite_id; +} + +pub fn mlirType(self: *const Compiler, dt: DataType) *const mlir.Type { + return self.mlir_known_types.get(dt); +} + +pub fn dtype(self: *const Compiler, mlir_type: *const mlir.Type) DataType { + @setRuntimeSafety(false); + for (0.., &self.mlir_known_types.values) |i, known_type| { + if (known_type == mlir_type) return @enumFromInt(i); + } + std.debug.panic("Can't convert unknown mlir type to dtype: {f}", .{mlir_type}); +} + +pub fn abortOOM(self: *Compiler) noreturn { + _ = self; + @panic("OOM"); +} + +pub fn alloc(self: *Compiler, T: type, n: usize) []T { + return self.arena.allocator().alloc(T, n) catch self.abortOOM(); +} + +pub fn allocPrint(self: *Compiler, comptime fmt: []const u8, args: anytype) []u8 { + return std.fmt.allocPrint(self.arena.allocator(), fmt, args) catch self.abortOOM(); +} + +pub fn Typed(comptime func: anytype) type { return struct { pub fn compile( allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, - opts: CompilationOptions, + opts: Options, args: std.meta.ArgsTuple(@TypeOf(func)), - ) CompileError!Exe { - return zml_module.compile(allocator, io, func, args, platform, opts); + ) Error!Exe { + return Compiler.compile(allocator, io, platform, func, args, opts); } }; } @@ -268,13 +265,13 @@ pub fn Compiler(comptime func: anytype) type { pub fn compile( allocator: std.mem.Allocator, io: std.Io, + platform: *const Platform, comptime func: anytype, args: std.meta.ArgsTuple(@TypeOf(func)), - platform: *const Platform, - opts: CompilationOptions, -) CompileError!Exe { + opts: Options, +) Error!Exe { // TODO: Here we have somewhat of a requirement - // Emitting MLIR requires to have the compilation context available at all times using `CompilationContext.current()`. + // Emitting MLIR requires to have the compiler context available at all times using `Compiler.current()`. // If in the future, we inject an Io that is not thread-based, we might have some surprises. // // I think the correct implementation would be to dispatch `emitMlir` to a thread pool, then wait for the result @@ -292,36 +289,36 @@ pub fn compile( var span = tracer.Span.start(span_name); defer span.end(); - var compilation_context: CompilationContext = .init(allocator, st_io.io(), platform, opts); - defer compilation_context.deinit(); + var compiler: Compiler = .init(allocator, st_io.io(), platform, opts); + defer compiler.deinit(); - const result = emitMlir(&compilation_context, func, args) catch |err| switch (err) { + const result = emitMlir(&compiler, func, args) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => unreachable, }; - defer result.output_info.deinit(compilation_context.allocator); - defer result.input_info.deinit(compilation_context.allocator); + defer result.output_info.deinit(compiler.allocator); + defer result.input_info.deinit(compiler.allocator); - try addPartitionerOperations(&compilation_context); + try addPartitionerOperations(&compiler); - _ = result.func.appendTo(compilation_context.module.body()); + _ = result.func.appendTo(compiler.module.body()); - const num_partitions = compilation_context.partitioning.numPartitions(); - const num_replicas = compilation_context.partitioning.numReplicas(); - const num_devices = compilation_context.partitioning.numDevices(); + const num_partitions = compiler.partitioning.numPartitions(); + const num_replicas = compiler.partitioning.numReplicas(); + const num_devices = compiler.partitioning.numDevices(); - compilation_context.module.operation().setAttributeByName( + compiler.module.operation().setAttributeByName( "mhlo.num_partitions", - .int(compilation_context.mlir_ctx, .i32, num_partitions), + .int(compiler.mlir_ctx, .i32, num_partitions), ); - compilation_context.module.operation().setAttributeByName( + compiler.module.operation().setAttributeByName( "mhlo.num_replicas", - .int(compilation_context.mlir_ctx, .i32, num_replicas), + .int(compiler.mlir_ctx, .i32, num_replicas), ); - compilation_context.mlir_pass_manager.runOnOp(compilation_context.module.operation()) catch |err| switch (err) { + compiler.mlir_pass_manager.runOnOp(compiler.module.operation()) catch |err| switch (err) { error.MlirUnexpected => { - std.log.err("Failed to canonicalize invalid mlir: \n {f} \n ", .{compilation_context.module.operation()}); + std.log.err("Failed to canonicalize invalid mlir: \n {f} \n ", .{compiler.module.operation()}); @panic("ZML generated invalid mlir. Please open a bug report"); }, }; @@ -329,8 +326,8 @@ pub fn compile( var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); - const loaded_executable = try compileModuleToPjrtExecutable(arena.allocator(), st_io.io(), platform, compilation_context.module, compilation_context.partitioning, opts); - log.debug("\n******** ZML generated MLIR ********\n{f}", .{compilation_context.module.operation()}); + const loaded_executable = try compileModuleToPjrtExecutable(arena.allocator(), st_io.io(), platform, compiler.module, compiler.partitioning, opts); + log.debug("\n******** ZML generated MLIR ********\n{f}", .{compiler.module.operation()}); const exe = try Exe.init( allocator, @@ -348,7 +345,7 @@ pub fn compile( return exe; } -fn addPartitionerOperations(ctx: *CompilationContext) !void { +fn addPartitionerOperations(ctx: *Compiler) !void { const allocator = ctx.arena.allocator(); const mlir_ctx = ctx.mlir_ctx; const module = ctx.module; @@ -509,31 +506,31 @@ fn finalizeAttributeList(allocator_: std.mem.Allocator, mlir_ctx: *mlir.Context, return res; } -fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, args: std.meta.ArgsTuple(@TypeOf(func))) !EmitMlirResult { - var arena = std.heap.ArenaAllocator.init(compilation_context.allocator); +fn emitMlir(compiler: *Compiler, comptime func: anytype, args: std.meta.ArgsTuple(@TypeOf(func))) !EmitMlirResult { + var arena = std.heap.ArenaAllocator.init(compiler.allocator); defer arena.deinit(); - const module = mlir.Module.init(.unknown(compilation_context.mlir_ctx)); + const module = mlir.Module.init(.unknown(compiler.mlir_ctx)); errdefer module.deinit(); const block = mlir.Block.init(&.{}, &.{}); errdefer block.deinit(); - compilation_context.pushBlock(block); - defer compilation_context.popBlock(); + compiler.pushBlock(block); + defer compiler.popBlock(); const LocalContext = struct { - compilation_context: *CompilationContext, + compiler: *Compiler, current_argument_id: usize = 0, }; var context: LocalContext = .{ - .compilation_context = compilation_context, + .compiler = compiler, }; meta.visit(struct { fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { - const mlir_type = mlirx.Type.rankedTensor(ctx_.compilation_context.mlir_ctx, tensor.shape()); - _ = ctx_.compilation_context.currentScope().block.addArgument(mlir_type, .unknown(ctx_.compilation_context.mlir_ctx)); - const gop = ctx_.compilation_context.currentScope().id_to_argument.getOrPut(ctx_.compilation_context.currentScope().arena.allocator(), tensor.id) catch unreachable; + const mlir_type = mlirx.Type.rankedTensor(ctx_.compiler.mlir_ctx, tensor.shape()); + _ = ctx_.compiler.currentScope().block.addArgument(mlir_type, .unknown(ctx_.compiler.mlir_ctx)); + const gop = ctx_.compiler.currentScope().id_to_argument.getOrPut(ctx_.compiler.currentScope().arena.allocator(), tensor.id) catch unreachable; if (gop.found_existing) std.debug.panic("Tensor with id {} has already been used once as an argument", .{tensor.id}); gop.value_ptr.* = ctx_.current_argument_id; ctx_.current_argument_id += 1; @@ -541,21 +538,21 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar }.cb, &context, &args); const output_info, const input_info = b: { - compilation_context.activate(); - defer compilation_context.deactivate(); + compiler.activate(); + defer compiler.deactivate(); const result = @call(.auto, func, args); - const input_info = try collectInputInfo(compilation_context.allocator, compilation_context.partitioning, &args); - errdefer input_info.deinit(compilation_context.allocator); + const input_info = try collectInputInfo(compiler.allocator, compiler.partitioning, &args); + errdefer input_info.deinit(compiler.allocator); - const output_info = try collectOutputInfo(compilation_context.allocator, compilation_context.partitioning, &result); - errdefer output_info.deinit(compilation_context.allocator); + const output_info = try collectOutputInfo(compiler.allocator, compiler.partitioning, &result); + errdefer output_info.deinit(compiler.allocator); break :b .{ output_info, input_info }; }; - errdefer input_info.deinit(compilation_context.allocator); - errdefer output_info.deinit(compilation_context.allocator); + errdefer input_info.deinit(compiler.allocator); + errdefer output_info.deinit(compiler.allocator); const input_attributes = try arena.allocator().alloc(AttributeList, input_info.shapes.len); @memset(input_attributes, .empty); @@ -564,59 +561,59 @@ fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, ar @memset(output_attributes, .empty); for (output_info.donations, 0..) |donation, index| if (donation) |argument_index| { - input_attributes[argument_index].appendAssumeCapacity(.named(compilation_context.mlir_ctx, "tf.aliasing_output", .int(compilation_context.mlir_ctx, .i32, index))); + input_attributes[argument_index].appendAssumeCapacity(.named(compiler.mlir_ctx, "tf.aliasing_output", .int(compiler.mlir_ctx, .i32, index))); }; for (output_info.output_memory_kinds, 0..) |output_memory_kind, index| { if (output_memory_kind == .device) continue; output_attributes[index].appendAssumeCapacity(.named( - compilation_context.mlir_ctx, + compiler.mlir_ctx, "mhlo.memory_kind", .string( - compilation_context.mlir_ctx, - compilation_context.platform.memoryKind(output_memory_kind), + compiler.mlir_ctx, + compiler.platform.memoryKind(output_memory_kind), ), )); } - _ = dialects.func.returns(compilation_context.mlir_ctx, output_info.values, .unknown(compilation_context.mlir_ctx)).appendTo(compilation_context.currentScope().block); + _ = dialects.func.returns(compiler.mlir_ctx, output_info.values, .unknown(compiler.mlir_ctx)).appendTo(compiler.currentScope().block); for (input_info.shapes, input_info.shardings, input_info.memory_kinds, 0..) |shape, sharding, maybe_memory_kind, i| { - const attr = try compilation_context.partitioning.tensorShardingAttr(compilation_context.arena.allocator(), compilation_context.mlir_ctx, shape, sharding); - const name = switch (compilation_context.partitioning.partitioner) { + const attr = try compiler.partitioning.tensorShardingAttr(compiler.arena.allocator(), compiler.mlir_ctx, shape, sharding); + const name = switch (compiler.partitioning.partitioner) { .gspmd => "mhlo.sharding", .shardy => "sdy.sharding", }; - input_attributes[i].appendAssumeCapacity(.named(compilation_context.mlir_ctx, name, attr)); + input_attributes[i].appendAssumeCapacity(.named(compiler.mlir_ctx, name, attr)); if (maybe_memory_kind) |memory_kind| { if (memory_kind == .device) continue; input_attributes[i].appendAssumeCapacity(.named( - compilation_context.mlir_ctx, + compiler.mlir_ctx, "mhlo.memory_kind", .string( - compilation_context.mlir_ctx, - compilation_context.platform.memoryKind(memory_kind), + compiler.mlir_ctx, + compiler.platform.memoryKind(memory_kind), ), )); } } for (output_info.shapes, output_info.shardings, 0..) |shape, sharding, i| { - const attr = try compilation_context.partitioning.tensorShardingAttr(compilation_context.arena.allocator(), compilation_context.mlir_ctx, shape, sharding); - const name = switch (compilation_context.partitioning.partitioner) { + const attr = try compiler.partitioning.tensorShardingAttr(compiler.arena.allocator(), compiler.mlir_ctx, shape, sharding); + const name = switch (compiler.partitioning.partitioner) { .gspmd => "mhlo.sharding", .shardy => "sdy.sharding", }; - output_attributes[i].appendAssumeCapacity(.named(compilation_context.mlir_ctx, name, attr)); + output_attributes[i].appendAssumeCapacity(.named(compiler.mlir_ctx, name, attr)); } - const mlir_func = dialects.func.func(compilation_context.mlir_ctx, .{ + const mlir_func = dialects.func.func(compiler.mlir_ctx, .{ .name = "main", - .block = compilation_context.currentScope().block, - .location = .unknown(compilation_context.mlir_ctx), - .args_attributes = try finalizeAttributeList(arena.allocator(), compilation_context.mlir_ctx, input_attributes), - .results_attributes = try finalizeAttributeList(arena.allocator(), compilation_context.mlir_ctx, output_attributes), + .block = compiler.currentScope().block, + .location = .unknown(compiler.mlir_ctx), + .args_attributes = try finalizeAttributeList(arena.allocator(), compiler.mlir_ctx, input_attributes), + .results_attributes = try finalizeAttributeList(arena.allocator(), compiler.mlir_ctx, output_attributes), .verify = false, }); @@ -649,7 +646,7 @@ fn setXlaOverrideFlag(map: *c.upb_Map, flag: []const u8, value: anytype, upb_are } } -fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: *const Platform, module: *const mlir.Module, partitioning: Partitioning, opts: CompilationOptions) !*pjrt.LoadedExecutable { +fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: *const Platform, module: *const mlir.Module, partitioning: Partitioning, opts: Options) !*pjrt.LoadedExecutable { var upb_alloc: upb.Allocator = .init(arena); const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); defer c.upb_Arena_Free(upb_arena); diff --git a/zml/attention/attnd.zig b/zml/attention/attnd.zig index 2e191868..b1c39cd1 100644 --- a/zml/attention/attnd.zig +++ b/zml/attention/attnd.zig @@ -140,7 +140,7 @@ const Context = struct { }; pub fn causalAttention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_offset: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { - const ctx = zml.module.CompilationContext.current(); + const ctx = zml.Compiler.current(); const num_partitions = ctx.partitioning.numPartitionsForLogicalAxis(q.shape(), .model) catch unreachable; const actual_k, const actual_v = if (parameters.is_prefill) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 977cc462..7581a85a 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -4,7 +4,6 @@ const flashattn = @import("platforms/cuda/flashattn"); const platforms = @import("platforms"); const stdx = @import("stdx"); -const CompilationContext = @import("../module.zig").CompilationContext; const zml = @import("../zml.zig"); const ffi = zml.pjrt.ffi; const AttentionOptions = @import("paged_attention.zig").AttentionOptions; @@ -268,7 +267,7 @@ pub const fa2 = struct { }; pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index_: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { - const ctx = CompilationContext.current(); + const ctx = zml.Compiler.current(); var bs: i64 = 1; var q = q_; @@ -900,7 +899,7 @@ pub const paged_fa2 = struct { stdx.debug.assert(q.shape().hasTags(.{ .b, .hg, .hkv, .hd }), "Expected q to have tags .b, .h, .hd", .{}); stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); - const ctx = CompilationContext.current(); + const ctx = zml.Compiler.current(); const num_head_groups = q.dim(.hg); const num_kv_heads = q.dim(.hkv); diff --git a/zml/attention/tpu_attention.zig b/zml/attention/tpu_attention.zig index 0538f9b1..e7511b3f 100644 --- a/zml/attention/tpu_attention.zig +++ b/zml/attention/tpu_attention.zig @@ -4,7 +4,7 @@ const mlir = @import("mlir"); const ragged_paged = @import("platforms/tpu/ragged_paged"); const stdx = @import("stdx"); -const CompilationContext = @import("../module.zig").CompilationContext; +const Compiler = @import("../Compiler.zig"); const zml = @import("../zml.zig"); const AttentionOptions = @import("paged_attention.zig").AttentionOptions; const ragged_attention = @import("mosaic_tpu_kernels/ragged_attention.zig"); @@ -89,7 +89,7 @@ pub const mosaic_tpu = struct { num_seqs: zml.Tensor, cfg: ragged_paged.Cfg, ) zml.Tensor { - const mlir_ctx = CompilationContext.current().mlir_ctx; + const mlir_ctx = Compiler.current().mlir_ctx; const out = ragged_attention.Kernel.call( .{ .kv_lens = seq_lens, diff --git a/zml/attention/triton_attention.zig b/zml/attention/triton_attention.zig index 86d42732..7a7b798c 100644 --- a/zml/attention/triton_attention.zig +++ b/zml/attention/triton_attention.zig @@ -13,7 +13,7 @@ const mla_kernels = @import("triton_kernels/unified_sparse_mla.zig"); const log = std.log.scoped(.@"zml/attention/triton"); fn isOneapiTarget() bool { - return zml.module.CompilationContext.current().platform.target == .oneapi; + return zml.Compiler.current().platform.target == .oneapi; } fn use2dKernel(all_decode: bool, batch_size: usize, num_kv_heads: usize) bool { @@ -204,7 +204,7 @@ fn select3dConfig(options: paged.PagedAttentionOptions) Config3D { } fn getCuCount() usize { - const platform = zml.module.CompilationContext.current().platform; + const platform = zml.Compiler.current().platform; if (platform.devices.len == 0) return 1; const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return 1; if (attribute.int64 <= 0) return 1; @@ -988,7 +988,7 @@ test "sparse MLA launch selection honors valid explicit splits" { test "sparse MLA emits 2D and 3D Triton kernels" { const platform = zml.testing.env(); - var compilation = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{}); + var compilation = zml.Compiler.init(std.testing.allocator, std.testing.io, platform, .{}); defer compilation.deinit(); compilation.activate(); defer compilation.deactivate(); diff --git a/zml/attention/triton_kernels/unified_attention.zig b/zml/attention/triton_kernels/unified_attention.zig index f59f44db..a94fae73 100644 --- a/zml/attention/triton_kernels/unified_attention.zig +++ b/zml/attention/triton_kernels/unified_attention.zig @@ -349,7 +349,7 @@ pub const ReduceSegmentsPtr = struct { // ============================================================================ fn ctx() *mlir.Context { - return zml.module.CompilationContext.current().mlir_ctx; + return zml.Compiler.current().mlir_ctx; } fn isFp8(dt: DType) bool { diff --git a/zml/exe.zig b/zml/exe.zig index dfb85dbf..9efe7674 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -4,9 +4,9 @@ const pjrt = @import("pjrt"); const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; +const Compiler = @import("Compiler.zig"); const mem = @import("mem.zig"); const meta = @import("meta.zig"); -const module = @import("module.zig"); const Platform = @import("platform.zig").Platform; const tracer = @import("profiling/tracer.zig"); const Shape = @import("shape.zig").Shape; @@ -319,7 +319,7 @@ pub const Exe = struct { .results = results_.flat_buffers.buffers, .events = events_slice, // this allows to tell a specific buffer shouldn't be donated, - // even if it has been marked as "can be donated" during compilation. + // even if it has been marked as "can be donated" during compiler. // TODO: expose it ? .non_donatable_input_indices = &.{}, .context = self.context, @@ -408,10 +408,10 @@ pub fn FnExe(comptime function_: anytype) type { allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, - opts: module.CompilationOptions, + opts: Compiler.Options, args: std.meta.ArgsTuple(@TypeOf(function_)), - ) module.CompileError!Self { - return .{ .raw = try module.Compiler(function_).compile(allocator, io, platform, opts, args) }; + ) Compiler.Error!Self { + return .{ .raw = try Compiler.Typed(function_).compile(allocator, io, platform, opts, args) }; } pub fn deinit(self: *const Self) void { diff --git a/zml/kernel.zig b/zml/kernel.zig index 9f8a98b9..1c4f629e 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -7,7 +7,7 @@ const mosaic_tpu_builder = @import("kernels/mosaic_tpu/builder"); const tpu_dialect = @import("mlir/dialects/mosaic_tpu"); const triton_builder = @import("kernels/triton/builder"); -const CompilationContext = @import("module.zig").CompilationContext; +const Compiler = @import("Compiler.zig"); const DataType = @import("dtype.zig").DataType; const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); @@ -126,7 +126,7 @@ pub const triton = struct { } pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = CompilationContext.current(); + const cur = Compiler.current(); const ttir = emit(cur.allocator, opts.cfg) catch |err| std.debug.panic("zml.kernel.triton.Kernel({s}).call: emit failed: {}", .{ name, err }); @@ -232,7 +232,7 @@ pub const mosaic_tpu = struct { } pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = CompilationContext.current(); + const cur = Compiler.current(); const ir = emit(cur.allocator, opts.cfg) catch |err| std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: emit failed: {}", .{ name, err }); @@ -365,7 +365,7 @@ pub const mosaic_tpu = struct { }; fn callTpuCustomCall(args: TpuCustomCallArgs) *mlir.Operation { - const cur = CompilationContext.current(); + const cur = Compiler.current(); var all_inputs: stdx.BoundedArray(*const mlir.Value, dialects.stablehlo.CustomCallOpts.MAX_OPERANDS) = .empty; for (args.dynamic_grid_bounds) |t| all_inputs.appendAssumeCapacity(t.value()); diff --git a/zml/moe/cutlass_flashinfer.zig b/zml/moe/cutlass_flashinfer.zig index a85f572a..47f21dab 100644 --- a/zml/moe/cutlass_flashinfer.zig +++ b/zml/moe/cutlass_flashinfer.zig @@ -240,7 +240,7 @@ fn runnersFromAttributes(attributes: Attributes) !*Runners { } fn currentRunners() !*Runners { - const platform = zml.module.CompilationContext.current().platform; + const platform = zml.Compiler.current().platform; const runners = platform.state.cuda.fi_cutlass_moe_runners; return runners orelse error.RunnersNotLoaded; } diff --git a/zml/moe/triton.zig b/zml/moe/triton.zig index 9da3177d..a399bdb3 100644 --- a/zml/moe/triton.zig +++ b/zml/moe/triton.zig @@ -647,7 +647,7 @@ fn applyJsonTokenConfig(opts: Options, num_tokens: i64) !Options { var out = opts; if (!opts.dynamic_launch_by_num_tokens) return out; - const compilation_context = zml.module.CompilationContext.current(); + const compilation_context = zml.Compiler.current(); const io = compilation_context.io; const allocator = compilation_context.allocator; diff --git a/zml/nn.zig b/zml/nn.zig index 9ba61223..e83f9b8c 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -79,7 +79,7 @@ pub const Linear = struct { var lhs_scale: ?Tensor = null; var undo: ?Tensor = null; - const platform = zml.module.CompilationContext.current().platform; + const platform = zml.Compiler.current().platform; if (q.scheme.activationQuant()) |aq| { if (aq.supportedOn(platform)) { const quantized = aq.apply(lhs, igs, self.tag); @@ -434,7 +434,7 @@ pub fn scaledDot( else rhs_scale; - const mlir_ctx = zml.module.CompilationContext.current().mlir_ctx; + const mlir_ctx = zml.Compiler.current().mlir_ctx; const dnums = mlir.Attribute.array(mlir_ctx, &.{ .array(mlir_ctx, &.{ .intArray(mlir_ctx, i64, lhs_contracting_axes.constSlice()), @@ -598,7 +598,7 @@ test normalizeL2 { const input: zml.Tensor = .init(.{ 2, 2 }, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, normalizeL2, .{ input, 1e-12 }, platform, .{}); + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, normalizeL2, .{ input, 1e-12 }, .{}); defer exe.deinit(); var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&[_]f32{ -0.9686, -1.0058, -1.7808, 0.6698 })); @@ -873,7 +873,7 @@ pub fn mergeRealImgPass(x_real: Tensor, x_imag: Tensor, x_pass: ?Pass, layout: R /// {exp( - n * ln(10_000) / N ) | n in [0..N] } pub fn invFreq(N: i64, opts: RopeOpts) Tensor { - const allocator = zml.module.CompilationContext.current().allocator; + const allocator = zml.Compiler.current().allocator; const N_half: u32 = @intCast(@divExact(N, 2)); const num_freqs: u32 = opts.scaling.partialRotaryDim(N_half); @@ -1121,7 +1121,7 @@ test "real/img" { } }; { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.interleaved}, platform, .{}); + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.interleaved}, .{}); defer exe.deinit(); var d_interleaved = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitMergeIsId, {}); @@ -1129,7 +1129,7 @@ test "real/img" { try std.testing.expectEqual(20, try d_interleaved.getValue(i32, std.testing.io)); } { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.real_im_pass}, platform, .{}); + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.real_im_pass}, .{}); defer exe.deinit(); var d_sequential = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitMergeIsId, {}); @@ -1139,7 +1139,7 @@ test "real/img" { // test the function that accepts 1 void argument { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitSeqVoid, .{{}}, platform, .{}); + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Fns.testSplitSeqVoid, .{{}}, .{}); defer exe.deinit(); var d_split_seq_void = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitSeqVoid, {}); @@ -1149,7 +1149,7 @@ test "real/img" { // test the function that takes NO arguments { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitSeq, .{}, platform, .{}); + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Fns.testSplitSeq, .{}, .{}); defer exe.deinit(); var d_split_seq = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitSeq, {}); @@ -1158,7 +1158,7 @@ test "real/img" { } { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitInterleaved, .{}, platform, .{}); + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Fns.testSplitInterleaved, .{}, .{}); defer exe.deinit(); var d_split_seq = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitInterleaved, {}); @@ -1192,10 +1192,10 @@ test rope { // x is made such as the interleaved and sequential reps are the same. // So the two implementations should give the same results. const x: zml.Tensor = .init(.{ .b = 1, .s = 5, .hd = 4 }, .f32); - var exe_interleaved = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .interleaved } }, platform, .{}); + var exe_interleaved = try platform.compileFn(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .interleaved } }, .{}); defer exe_interleaved.deinit(); - var exe_sequential = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .real_im_pass } }, platform, .{}); + var exe_sequential = try platform.compileFn(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .real_im_pass } }, .{}); defer exe_sequential.deinit(); const x_values: [5][4]f32 = @splat(.{ 1.0, 0.1, -1.0, -0.5 }); @@ -1216,7 +1216,7 @@ test "rope: Proportional" { const platform = zml.testing.env(); const x: zml.Tensor = .init(.{ .s = 5, .hd = 16 }, .f32); - var exe = try zml.module.compile( + var exe = try platform.compileFn( allocator, io, rope, @@ -1228,7 +1228,6 @@ test "rope: Proportional" { .scaling = .{ .proportional = .{ .partial_rotary_factor = 0.25 } }, }, }, - platform, .{}, ); defer exe.deinit(); @@ -1261,7 +1260,7 @@ test "rope: Yarn with partial_rotary_factor" { const platform = zml.testing.env(); const x: zml.Tensor = .init(.{ .s = 5, .hd = 16 }, .f32); - var exe = try zml.module.compile( + var exe = try platform.compileFn( allocator, io, rope, @@ -1281,7 +1280,6 @@ test "rope: Yarn with partial_rotary_factor" { } }, }, }, - platform, .{}, ); defer exe.deinit(); @@ -1370,12 +1368,11 @@ test nearest { // 3D Tensor (basic) { const input_3d_basic: zml.Tensor = .init(.{ 1, 1, 2 }, .i32); - var exe = try zml.module.compile( + var exe = try platform.compileFn( std.testing.allocator, std.testing.io, upsample, .{ input_3d_basic, .{ .scale_factor = &.{3}, .mode = .nearest } }, - platform, .{}, ); defer exe.deinit(); @@ -1400,12 +1397,11 @@ test nearest { // 3D Tensor (advanced) { const input_3d_advanced: zml.Tensor = .init(.{ 2, 3, 4 }, .i32); - var exe = try zml.module.compile( + var exe = try platform.compileFn( std.testing.allocator, std.testing.io, upsample, .{ input_3d_advanced, .{ .scale_factor = &.{2}, .mode = .nearest } }, - platform, .{}, ); defer exe.deinit(); @@ -1438,12 +1434,11 @@ test nearest { // 4D Tensor (basic) { const input_4d_basic: zml.Tensor = .init(.{ 1, 1, 2, 2 }, .i32); - var exe = try zml.module.compile( + var exe = try platform.compileFn( std.testing.allocator, std.testing.io, upsample, .{ input_4d_basic, .{ .scale_factor = &.{ 3, 3 }, .mode = .nearest } }, - platform, .{}, ); defer exe.deinit(); @@ -1468,12 +1463,11 @@ test nearest { // 4D Tensor (advanced) { const input_4d_advanced: zml.Tensor = .init(.{ 2, 2, 2, 2 }, .i32); - var exe = try zml.module.compile( + var exe = try platform.compileFn( std.testing.allocator, std.testing.io, upsample, .{ input_4d_advanced, .{ .scale_factor = &.{ 2, 2 }, .mode = .nearest } }, - platform, .{}, ); defer exe.deinit(); @@ -1526,12 +1520,11 @@ test nearest { // 5D Tensor (basic) { const input_5d: zml.Tensor = .init(.{ 1, 1, 1, 2, 2 }, .i32); - var exe = try zml.module.compile( + var exe = try platform.compileFn( std.testing.allocator, std.testing.io, upsample, .{ input_5d, .{ .scale_factor = &.{2}, .mode = .nearest } }, - platform, .{}, ); defer exe.deinit(); @@ -1595,7 +1588,7 @@ test resizeBilinear { const platform = zml.testing.env(); // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{}); + var comp = zml.Compiler.init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -1662,7 +1655,7 @@ test resizeBicubic { const platform = zml.testing.env(); // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{}); + var comp = zml.Compiler.init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -1962,12 +1955,11 @@ test "gated delta net" { const betas: zml.Tensor = .init(.{ .s = 2, .h = 2 }, .f32); const initial_s: zml.Tensor = .init(.{ .h = 2, .v = 2, .k = 2 }, .f32); - var exe = try zml.module.compile( + var exe = try platform.compileFn( std.testing.allocator, std.testing.io, GatedDeltaNet.forward, .{ queries, keys, values, alphas, betas, .{ .s = initial_s } }, - platform, .{}, ); defer exe.deinit(); @@ -2117,7 +2109,7 @@ test sampleTokens { const rng: zml.Tensor.Rng = .init(); const activations: zml.Tensor = .init(.{ .voc = 4 }, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, sampleTokens, .{ activations, .{ .topk = 4, .temperature = 2.0 }, rng }, platform, .{}); + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, sampleTokens, .{ activations, .{ .topk = 4, .temperature = 2.0 }, rng }, .{}); defer exe.deinit(); var rng_buffer = try zml.Tensor.Rng.initBuffer(std.testing.io, platform, .replicated, 0xdeadbeef); @@ -2259,7 +2251,7 @@ test sampleTokensDynamic { const logits: zml.Tensor = .init(.{ .voc = logits_data.len }, .f32); const dynamic_sampling_strategy = DynamicSamplingStrategy.init(.f32, 0); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, fixupLogits, .{ logits, dynamic_sampling_strategy }, platform, .{}); + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, fixupLogits, .{ logits, dynamic_sampling_strategy }, .{}); defer exe.deinit(); var logits_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits.shape(), .replicated, std.mem.sliceAsBytes(&logits_data)); @@ -2296,7 +2288,7 @@ test sampleTokensDynamic { const logits_bf16: zml.Tensor = .init(.{ .voc = logits_data.len }, .bf16); const dynamic_sampling_strategy_bf16 = DynamicSamplingStrategy.init(.bf16, 0); - var exe_bf16 = try zml.module.compile(std.testing.allocator, std.testing.io, fixupLogits, .{ logits_bf16, dynamic_sampling_strategy_bf16 }, platform, .{}); + var exe_bf16 = try platform.compileFn(std.testing.allocator, std.testing.io, fixupLogits, .{ logits_bf16, dynamic_sampling_strategy_bf16 }, .{}); defer exe_bf16.deinit(); const boost = bf16.inf; diff --git a/zml/ops.zig b/zml/ops.zig index 1b872228..b950de14 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -9,7 +9,7 @@ const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("mem.zig").Bufferized; -const CompilationContext = @import("module.zig").CompilationContext; +const Compiler = @import("Compiler.zig"); const constants = @import("constants.zig"); const CustomCallBuffer = @import("pjrtx.zig").CustomCallBuffer; const DataType = @import("dtype.zig").DataType; @@ -22,7 +22,7 @@ const Tensor = @import("tensor.zig").Tensor; pub const TensorToCustomCallBuffer = @import("pjrtx.zig").TensorToCustomCallBuffer; pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@TypeOf(inputs)) { - const ctx = CompilationContext.current(); + const ctx = Compiler.current(); const mlir_ctx = ctx.mlir_ctx; const InputsT = @TypeOf(inputs); @@ -194,7 +194,7 @@ fn AllReduceReturnType(comptime InputsT: type) type { } pub fn partitionId() Tensor { - const ctx = CompilationContext.current(); + const ctx = Compiler.current(); const op = mlir.Operation.make(ctx.mlir_ctx, "stablehlo.partition_id", .{ .results = .{ .flat = &.{mlirx.Type.rankedTensor(ctx.mlir_ctx, Shape.scalar(.u32))} }, .location = .unknown(ctx.mlir_ctx), @@ -208,10 +208,10 @@ pub const ReduceArgs = struct { }; pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func: anytype, context: anytype) stdx.meta.FnReturn(func) { - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + var arena = std.heap.ArenaAllocator.init(Compiler.current().allocator); defer arena.deinit(); - const mlir_ctx = CompilationContext.current().mlir_ctx; + const mlir_ctx = Compiler.current().mlir_ctx; const reduce_block, var result = b: { const ArgsTypes: [inits.len]type = @splat(ReduceArgs); @@ -231,10 +231,10 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func const reduce_block = mlir.Block.init(&block_types, &block_locs); errdefer reduce_block.deinit(); - CompilationContext.current().pushBlock(reduce_block); - defer CompilationContext.current().popBlock(); + Compiler.current().pushBlock(reduce_block); + defer Compiler.current().popBlock(); - const scope = CompilationContext.current().currentScope(); + const scope = Compiler.current().currentScope(); inline for (0..inits.len) |i| { scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inits.len) catch unreachable; @@ -267,7 +267,7 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func }, .verify = true, .location = .unknown(mlir_ctx), - }).appendTo(CompilationContext.current().currentScope().block); + }).appendTo(Compiler.current().currentScope().block); // `stablehlo.reduce` drops axes. We want to avoid that to propagate tags. // So we need to broadcast the output of `stablehlo.reduce` to the input shapes. @@ -293,7 +293,7 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func broadcasting_axes.slice()[0 .. reduced_shape.rank() - axes_.len], mlirx.Type.rankedTensor(mlir_ctx, reduced_shape), .unknown(mlir_ctx), - ).appendTo(CompilationContext.current().currentScope().block); + ).appendTo(Compiler.current().currentScope().block); result[i] = Tensor._result(reduced_shape, broad_op.result(0)); } @@ -320,10 +320,10 @@ pub fn ReduceWindowFn(N: comptime_int) type { } pub fn reduceWindow(N: comptime_int, inputs: [N]Tensor, inits: [N]Tensor, opts: ReduceWindowOpts, func: *const ReduceWindowFn(N)) [N]Tensor { - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + var arena = std.heap.ArenaAllocator.init(Compiler.current().allocator); defer arena.deinit(); - const mlir_ctx = CompilationContext.current().mlir_ctx; + const mlir_ctx = Compiler.current().mlir_ctx; const reduce_block, var result = b: { const Args = @Tuple(&@as([N]type, @splat(ReduceArgs))); @@ -342,10 +342,10 @@ pub fn reduceWindow(N: comptime_int, inputs: [N]Tensor, inits: [N]Tensor, opts: const reduce_block = mlir.Block.init(&block_types, &block_locs); errdefer reduce_block.deinit(); - CompilationContext.current().pushBlock(reduce_block); - defer CompilationContext.current().popBlock(); + Compiler.current().pushBlock(reduce_block); + defer Compiler.current().popBlock(); - const scope = CompilationContext.current().currentScope(); + const scope = Compiler.current().currentScope(); inline for (0..N) |i| { scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + N) catch unreachable; @@ -386,7 +386,7 @@ pub fn reduceWindow(N: comptime_int, inputs: [N]Tensor, inits: [N]Tensor, opts: }, .verify = true, .location = .unknown(mlir_ctx), - }).appendTo(CompilationContext.current().currentScope().block); + }).appendTo(Compiler.current().currentScope().block); inline for (0..result.len) |i| { result[i] = Tensor.fromMlirValue(reduce_op.result(i)).withTags(inputs[i].shape()); @@ -401,10 +401,10 @@ pub const SortArgs = struct { }; pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytype, is_stable: bool) [inputs.len]Tensor { - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + var arena = std.heap.ArenaAllocator.init(Compiler.current().allocator); defer arena.deinit(); - const mlir_ctx = CompilationContext.current().mlir_ctx; + const mlir_ctx = Compiler.current().mlir_ctx; const sort_block = b: { const ArgsTypes: [inputs.len]type = @splat(SortArgs); @@ -424,10 +424,10 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytyp const sort_block = mlir.Block.init(&block_types, &block_locs); errdefer sort_block.deinit(); - CompilationContext.current().pushBlock(sort_block); - defer CompilationContext.current().popBlock(); + Compiler.current().pushBlock(sort_block); + defer Compiler.current().popBlock(); - const scope = CompilationContext.current().currentScope(); + const scope = Compiler.current().currentScope(); inline for (0..inputs.len) |i| { scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, 2 * i) catch unreachable; scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, 2 * i + 1) catch unreachable; @@ -454,7 +454,7 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytyp }, .verify = true, .location = .unknown(mlir_ctx), - }).appendTo(CompilationContext.current().currentScope().block); + }).appendTo(Compiler.current().currentScope().block); var result: [inputs.len]Tensor = undefined; inline for (0..inputs.len) |i| { @@ -475,7 +475,7 @@ pub fn @"while"( context: While, initial_state: While.State, ) While.State { - const comp = CompilationContext.current(); + const comp = Compiler.current(); var arena = std.heap.ArenaAllocator.init(comp.allocator); defer arena.deinit(); @@ -676,7 +676,7 @@ pub fn @"if"( stdx.debug.assert(pred.dtype() == .bool and pred.count() == 1, "zml.ops.if expects the condition to have exactly one element of dtype .bool, got {f}", .{pred}); - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + var arena = std.heap.ArenaAllocator.init(Compiler.current().allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -689,15 +689,15 @@ pub fn @"if"( } }.capture, arena.allocator(), {}, if_captures, &blkctx) catch unreachable; - const mlir_ctx = CompilationContext.current().mlir_ctx; + const mlir_ctx = Compiler.current().mlir_ctx; const loc: *const mlir.Location = .unknown(mlir_ctx); const true_branch, const true_branch_block = b: { const block = mlir.Block.init(&.{}, &.{}); errdefer block.deinit(); - CompilationContext.current().pushBlock(block); - defer CompilationContext.current().popBlock(); + Compiler.current().pushBlock(block); + defer Compiler.current().popBlock(); const result = blkctx.onTrue(); const result_values = meta.collectAlloc(Tensor.value, {}, allocator, &result) catch @panic("OOM"); @@ -710,8 +710,8 @@ pub fn @"if"( const block = mlir.Block.init(&.{}, &.{}); errdefer block.deinit(); - CompilationContext.current().pushBlock(block); - defer CompilationContext.current().popBlock(); + Compiler.current().pushBlock(block); + defer Compiler.current().popBlock(); const result = blkctx.onFalse(); const result_values = meta.collectAlloc(Tensor.value, {}, allocator, &result) catch @panic("OOM"); @@ -730,7 +730,7 @@ pub fn @"if"( .verify = false, .location = loc, }); - _ = op.appendTo(CompilationContext.current().currentScope().block); + _ = op.appendTo(Compiler.current().currentScope().block); return fromMlirOperationWithTags(op, true_branch); } @@ -775,12 +775,12 @@ pub fn if2( ) @TypeOf(on_true) { stdx.debug.assert(pred.dtype() == .bool and pred.count() == 1, "zml.ops.if expects the condition to have exactly one element of dtype .bool, got {f}", .{pred}); - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + var arena = std.heap.ArenaAllocator.init(Compiler.current().allocator); defer arena.deinit(); const allocator = arena.allocator(); - const mlir_ctx = CompilationContext.current().mlir_ctx; + const mlir_ctx = Compiler.current().mlir_ctx; const loc: *const mlir.Location = .unknown(mlir_ctx); const true_values = meta.collectAlloc(Tensor.value, {}, allocator, &on_true) catch @panic("OOM"); @@ -789,8 +789,8 @@ pub fn if2( const block = mlir.Block.init(&.{}, &.{}); errdefer block.deinit(); - CompilationContext.current().pushBlock(block); - defer CompilationContext.current().popBlock(); + Compiler.current().pushBlock(block); + defer Compiler.current().popBlock(); _ = dialects.stablehlo.returns(mlir_ctx, true_values, loc).appendTo(block); break :b block; }; @@ -801,8 +801,8 @@ pub fn if2( const block = mlir.Block.init(&.{}, &.{}); errdefer block.deinit(); - CompilationContext.current().pushBlock(block); - defer CompilationContext.current().popBlock(); + Compiler.current().pushBlock(block); + defer Compiler.current().popBlock(); _ = dialects.stablehlo.returns(mlir_ctx, false_values, loc).appendTo(block); break :b block; @@ -815,7 +815,7 @@ pub fn if2( .location = loc, .verify = false, }); - _ = op.appendTo(CompilationContext.current().currentScope().block); + _ = op.appendTo(Compiler.current().currentScope().block); return fromMlirOperationWithTags(op, on_true); } @@ -881,8 +881,8 @@ pub const TritonOps = struct { /// Generate an MLIR call to the given member function with the given tensors. pub fn triton(inputs: anytype, outputs: anytype, opts: TritonOps) [outputs.len]Tensor { - const mlir_ctx = CompilationContext.current().mlir_ctx; - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + const mlir_ctx = Compiler.current().mlir_ctx; + var arena = std.heap.ArenaAllocator.init(Compiler.current().allocator); defer arena.deinit(); var values: [inputs.len]*const mlir.Value = undefined; @@ -928,7 +928,7 @@ pub fn triton(inputs: anytype, outputs: anytype, opts: TritonOps) [outputs.len]T .output_operand_aliases = opts.output_operand_aliases, }, .unknown(mlir_ctx), - ).appendTo(CompilationContext.current().currentScope().block); + ).appendTo(Compiler.current().currentScope().block); var outputs_: [outputs.len]Tensor = undefined; inline for (outputs, 0..) |output, i| { @@ -952,7 +952,7 @@ pub const NeuronNkiOps = struct { /// This API is Neuron-only: the source is compiled to an /// `AwsNeuronCustomNativeKernel` backend config while emitting the graph. pub fn neuronNki(inputs: anytype, outputs: anytype, opts: NeuronNkiOps) [outputs.len]Tensor { - const ctx = CompilationContext.current(); + const ctx = Compiler.current(); switch (ctx.platform.target) { .neuron => {}, .cpu, .cuda, .rocm, .tpu, .oneapi, .metal => { @@ -1123,10 +1123,10 @@ pub fn scatter( context: anytype, opts: Tensor.ScatterOpts, ) stdx.meta.FnReturn(func) { - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + var arena = std.heap.ArenaAllocator.init(Compiler.current().allocator); defer arena.deinit(); - const mlir_ctx = CompilationContext.current().mlir_ctx; + const mlir_ctx = Compiler.current().mlir_ctx; const update_block, var result = b: { const ArgsTypes: [inputs.len]type = @splat(ScatterArgs); @@ -1146,10 +1146,10 @@ pub fn scatter( const update_block = mlir.Block.init(&block_types, &block_locs); errdefer update_block.deinit(); - CompilationContext.current().pushBlock(update_block); - defer CompilationContext.current().popBlock(); + Compiler.current().pushBlock(update_block); + defer Compiler.current().popBlock(); - const scope = CompilationContext.current().currentScope(); + const scope = Compiler.current().currentScope(); inline for (0..inputs.len) |i| { scope.id_to_argument.put(scope.arena.allocator(), args[i].input.id, i) catch unreachable; scope.id_to_argument.put(scope.arena.allocator(), args[i].update.id, i + inputs.len) catch unreachable; @@ -1231,7 +1231,7 @@ pub fn scatter( .unique_indices = opts.indices_are_unique, }, .unknown(mlir_ctx), - ).appendTo(CompilationContext.current().currentScope().block); + ).appendTo(Compiler.current().currentScope().block); inline for (0..result.len) |i| { result[i] = Tensor._result(inputs[i].shape(), op.result(i)); @@ -1343,7 +1343,7 @@ test scatterConfig { const zml = @import("zml.zig"); const platform = zml.testing.env(); - var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{}); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -1437,7 +1437,7 @@ pub const GatherAxisKind = enum { batching, offset, collapsed, indices }; pub const GatherOpts = struct { indices_are_sorted: bool = false }; pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, opts: GatherOpts) Tensor { - const mlir_ctx = CompilationContext.current().mlir_ctx; + const mlir_ctx = Compiler.current().mlir_ctx; stdx.debug.assert(idx_axes.len > 0, "gather expects 1 or more axes to operate one, received none. Example: `x.gather(.a, indices, .{{}})`", .{}); for (idx_axes, 0..) |a, i| { @@ -1544,7 +1544,7 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, .indices_are_sorted = opts.indices_are_sorted, }, .unknown(mlir_ctx), - ).appendTo(CompilationContext.current().currentScope().block); + ).appendTo(Compiler.current().currentScope().block); const mlir_shape = Tensor.fromMlirValue(gather_op.result(0)).shape(); stdx.debug.assert(mlir_shape.eql(res_shape), "gather expects that batching indices appear in the same order in 'self' and 'indices', got: self={f}, indices={f}. You should transpose one or the other.", .{ self, indices }); @@ -1565,7 +1565,7 @@ pub const LoweringCompatibility = struct { /// splats too early, which can perturb later indexed-update lowering. A /// scalar optimization barrier preserves the source-level broadcast shape. pub fn preserveIntegerScalarBroadcast(self: Tensor, output_shape: Shape) ?Tensor { - switch (CompilationContext.current().platform.target) { + switch (Compiler.current().platform.target) { .neuron => {}, .cpu, .cuda, .rocm, .tpu, .oneapi, .metal => return null, } @@ -1584,7 +1584,7 @@ pub const LoweringCompatibility = struct { /// keeping the active rows unchanged. Related upstream Neuron report: /// https://github.com/aws-neuron/aws-neuron-sdk/issues/1335. pub fn preserveGatherFillSemantics(indices: []Tensor) void { - switch (CompilationContext.current().platform.target) { + switch (Compiler.current().platform.target) { .neuron => {}, .cpu, .cuda, .rocm, .tpu, .oneapi, .metal => return, } @@ -1596,7 +1596,7 @@ pub const LoweringCompatibility = struct { /// Preserve scatter drop semantics before backend indirect-memory lowering. /// Same as above but for scatter drop semantics. pub fn preserveScatterDropSemantics(indices: []Tensor, updates: anytype, opts: Tensor.ScatterOpts, update_values: *[updates.len]*const mlir.Value) void { - const active_lanes: ?Tensor = switch (CompilationContext.current().platform.target) { + const active_lanes: ?Tensor = switch (Compiler.current().platform.target) { .neuron => if (opts.update_fn == Tensor.ScatterOpts.increment) activeLanesForFillDropIndices(indices) else null, .cpu, .cuda, .rocm, .tpu, .oneapi, .metal => null, }; @@ -1704,7 +1704,7 @@ pub const CustomCallOptions = struct { compute_on_host: bool = false, }; -fn customCallAdditionalAttributes(ctx: *CompilationContext, opts: CustomCallOptions) []const mlir.NamedAttribute { +fn customCallAdditionalAttributes(ctx: *Compiler, opts: CustomCallOptions) []const mlir.NamedAttribute { if (!opts.compute_on_host or ctx.platform.target == .cpu) return &.{}; const frontend_attributes = mlir.Attribute.dict(ctx.mlir_ctx, &.{ @@ -1760,7 +1760,7 @@ pub fn composite( context: anytype, opts: CompositeOpts, ) []Tensor { - const ctx = CompilationContext.current(); + const ctx = Compiler.current(); const mlir_ctx = ctx.mlir_ctx; const decomp_name = ctx.allocPrint("{s}.impl_{d}", .{ name, ctx.nextCompositeId() }); @@ -1989,7 +1989,7 @@ fn manualComputationInternal( const BodyInputsT = stdx.meta.FnParam(body_fn, 2); const BodyOutputShapesT = stdx.meta.FnParam(body_fn, 3); - const ctx = CompilationContext.current(); + const ctx = Compiler.current(); const allocator = ctx.arena.allocator(); const input_shapes = allocator.alloc(Shape, inputs.len) catch unreachable; @@ -2464,7 +2464,7 @@ pub fn typedCustomCall( const Output = @TypeOf(output); const Attributes = @TypeOf(attributes); - const ctx = CompilationContext.current(); + const ctx = Compiler.current(); const allocator = ctx.arena.allocator(); stdx.debug.assert(!opts.has_side_effect or ctx.manual_computation_depth > 0, "side-effect customCall '{s}' must be emitted inside manualComputation", .{target_name}); @@ -2622,7 +2622,7 @@ test customCall { const zml = @import("zml.zig"); const platform = zml.testing.env(); - var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{}); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); diff --git a/zml/platform.zig b/zml/platform.zig index 0818dec5..5acd696b 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -545,7 +545,7 @@ pub const Platform = struct { args: stdx.meta.Tail( std.meta.ArgsTuple(@TypeOf(@field(@TypeOf(model_), @tagName(func)))), ), - opts: zml.module.CompilationOptions, + opts: zml.Compiler.Options, ) !Exe { return self.compileFn( allocator, @@ -563,7 +563,7 @@ pub const Platform = struct { comptime func: anytype, model: stdx.meta.Head(std.meta.ArgsTuple(@TypeOf(func))), args: stdx.meta.Tail(std.meta.ArgsTuple(@TypeOf(func))), - opts: zml.module.CompilationOptions, + opts: zml.Compiler.Options, ) !Exe { return self.compileFn(allocator, io, func, .{model} ++ args, opts); } @@ -574,9 +574,9 @@ pub const Platform = struct { io: std.Io, comptime func: anytype, args: std.meta.ArgsTuple(@TypeOf(func)), - opts: zml.module.CompilationOptions, + opts: zml.Compiler.Options, ) !Exe { - return zml.module.compile(allocator, io, func, args, self, opts); + return zml.Compiler.compile(allocator, io, self, func, args, opts); } pub fn format(self: *const Platform, writer: *std.Io.Writer) std.Io.Writer.Error!void { diff --git a/zml/tensor.zig b/zml/tensor.zig index e274738d..ca693a7f 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -5,7 +5,7 @@ const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); const stdx = @import("stdx"); -const CompilationContext = @import("module.zig").CompilationContext; +const Compiler = @import("Compiler.zig"); const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const mem = @import("mem.zig"); @@ -92,7 +92,7 @@ pub const Tensor = struct { /// /// The shape is derived from the type of the mlir.Value. pub fn fromMlirValue(val: *const mlir.Value) Tensor { - const ctx = CompilationContext.current(); + const ctx = Compiler.current(); const ranked_tensor = val.type_().isA(mlir.RankedTensorType).?; const n = ranked_tensor.rank(); @@ -153,7 +153,7 @@ pub const Tensor = struct { pub fn withPartitioning(self: Tensor, axes_: anytype) Tensor { const partitioned_shape = self._shape.withPartitioning(axes_); - const ctx = CompilationContext.currentOrNull() orelse { + const ctx = Compiler.currentOrNull() orelse { var res = self; res._shape = partitioned_shape; return res; @@ -203,7 +203,7 @@ pub const Tensor = struct { /// Copy the given tensor to the specified memory. pub fn toMemory(self: Tensor, kind: Memory.Kind) Tensor { - const ctx = CompilationContext.current(); + const ctx = Compiler.current(); switch (ctx.platform.target) { .cpu, .neuron, .metal => return self, .cuda, .rocm, .tpu, .oneapi => {}, @@ -240,7 +240,7 @@ pub const Tensor = struct { /// Copy all the given tensor to the specified memory. /// The input struct is copied on the stack, so it must be a simple flat struct without pointers. pub fn toMemoryAll(flat_tensors: anytype, kind: Memory.Kind) @TypeOf(flat_tensors) { - const ctx = CompilationContext.current(); + const ctx = Compiler.current(); switch (ctx.platform.target) { .cpu, .neuron, .metal => return flat_tensors, .cuda, .rocm, .tpu, .oneapi => {}, @@ -258,7 +258,7 @@ pub const Tensor = struct { /// Mark the given input tensor as being physically located on a specific memory. /// Has no effect if the input tensor is not an executable input. pub fn onMemory(self: Tensor, kind: Memory.Kind) Tensor { - const ctx = CompilationContext.current(); + const ctx = Compiler.current(); switch (ctx.platform.target) { .cpu, .neuron, .metal => return self, .cuda, .rocm, .tpu, .oneapi => {}, @@ -276,7 +276,7 @@ pub const Tensor = struct { /// Mark all the given input tensors as being physically located on a specific memory /// see `zml.Tensor.onMemory` pub fn onMemoryAll(tensors: anytype, kind: Memory.Kind) void { - const ctx = CompilationContext.current(); + const ctx = Compiler.current(); switch (ctx.platform.target) { // Only one memory kind on those platform .cpu, .neuron, .metal => return, @@ -311,10 +311,10 @@ pub const Tensor = struct { /// Returns the mlir.Value associated with the Tensor. /// - /// This will fail if used outside of a compilation context. + /// This will fail if used outside of a Compiler context. pub fn value(self: Tensor) *const mlir.Value { - if (CompilationContext.current().currentScope().id_to_argument.get(self.id)) |argument_index| { - return CompilationContext.current().currentScope().block.argument(argument_index); + if (Compiler.current().currentScope().id_to_argument.get(self.id)) |argument_index| { + return Compiler.current().currentScope().block.argument(argument_index); } else if (self._value) |v| { return v; } else @panic("Something went really wrong, tensor is not an argument nor has an mlir.Value"); @@ -328,7 +328,7 @@ pub const Tensor = struct { /// is not allowed to reuse the donated input buffer after the call. /// For `reuseBuffer` to be effective, it needs to propagate all the way through the output. pub fn reuseBuffer(self: Tensor, origin: Tensor) Tensor { - const compilation_context = CompilationContext.current(); + const compilation_context = Compiler.current(); const scope = compilation_context.currentScope(); if (scope.id_to_argument.get(origin.id)) |argument_index| { const gop = scope.id_to_donation.getOrPut(scope.arena.allocator(), self.id) catch unreachable; @@ -2093,7 +2093,7 @@ pub const Tensor = struct { /// Concatenates the input Tensors along the given axis. pub fn concatenate(tensors: []const Tensor, axis_: anytype) Tensor { if (tensors.len == 1) return tensors[0]; - const ctx = CompilationContext.current(); + const ctx = Compiler.current(); var buffer = ctx.alloc(*const mlir.Value, tensors.len); std.debug.assert(tensors.len <= buffer.len); std.debug.assert(tensors.len > 0); @@ -2125,7 +2125,7 @@ pub const Tensor = struct { stdx.debug.assert(shape0.eqlWithTags(tensor._shape), "stack expects tensor shapes to match, got {f} and {f}", .{ shape0, tensor._shape }); } - const ctx = CompilationContext.current(); + const ctx = Compiler.current(); var reshaped = ctx.alloc(Tensor, tensors.len); for (tensors, 0..) |tensor, i| { reshaped[i] = tensor.reshape(res_shape); @@ -2476,7 +2476,7 @@ pub const Tensor = struct { } pub fn uninitialized(sh: Shape) Tensor { - const ctx = CompilationContext.current(); + const ctx = Compiler.current(); const buffer_type = mlir.Type.memRef( mlirx.Type.fromDType(ctx.mlir_ctx, sh.dtype()), sh.dims(), @@ -2584,7 +2584,7 @@ pub const Tensor = struct { } pub fn optimizationBarrier(self: Tensor) Tensor { - const ctx = CompilationContext.current(); + const ctx = Compiler.current(); const op = dialects.stablehlo.optimizationBarrier( ctx.mlir_ctx, @@ -2748,7 +2748,7 @@ pub const Tensor = struct { { // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{}); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -2900,7 +2900,7 @@ pub const Tensor = struct { { // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{}); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -3125,7 +3125,7 @@ pub const Tensor = struct { { // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{}); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -3500,7 +3500,7 @@ pub const Tensor = struct { }, else => stdx.debug.compileError(err_msg, .{}), }; - const ctx = CompilationContext.current(); + const ctx = Compiler.current(); var result: SortRes = switch (ctx.platform.target) { // Work around https://github.com/aws-neuron/aws-neuron-sdk/issues/1339 until Neuron's sort+slice rewrite uses the slice size as k. .neuron => blk: { @@ -3664,7 +3664,7 @@ pub const Tensor = struct { const platform = zml.testing.env(); // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{}); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -3703,7 +3703,7 @@ pub const Tensor = struct { const tail_chunk_size: i64 = @rem(d, chunk_size); const len: usize = if (tail_chunk_size == 0) n_chunks else n_chunks + 1; - const chunks = CompilationContext.current().alloc(Tensor, len); + const chunks = Compiler.current().alloc(Tensor, len); for (0.., chunks) |i, *chunk| { const start: i64 = @as(i64, @intCast(i)) * chunk_size; @@ -3717,7 +3717,7 @@ pub const Tensor = struct { const platform = zml.testing.env(); // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, std.testing.io, platform, .{}); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -3760,7 +3760,7 @@ pub const Tensor = struct { for (split_sizes) |n| split_sum += n; stdx.debug.assert(split_sum == d, "split expects sum of 'split_sizes' values and axis dimension to be equal, got {} and {}", .{ split_sum, d }); - const res = CompilationContext.current().alloc(Tensor, split_sizes.len); + const res = Compiler.current().alloc(Tensor, split_sizes.len); var start: i64 = 0; for (split_sizes, 0..) |n, i| { @@ -4485,7 +4485,7 @@ pub const Tensor = struct { /// Only for debug purpose, it inserts device to host synchronization /// so it will slow down the program execution. pub fn print(input: Tensor, name: []const u8) void { - const ctx = CompilationContext.current(); + const ctx = Compiler.current(); const full_name = std.fmt.allocPrint(ctx.arena.allocator(), "{s}: {f}", .{ name, input.shape() }) catch @panic("OOM"); defer ctx.arena.allocator().free(full_name); switch (ctx.platform.target) { @@ -4501,25 +4501,25 @@ pub const Tensor = struct { } fn mlirCtx() *mlir.Context { - return CompilationContext.current().mlir_ctx; + return Compiler.current().mlir_ctx; } fn currentBlock() *mlir.Block { - return CompilationContext.current().currentScope().block; + return Compiler.current().currentScope().block; } /// Returns the donation data of the tensor. pub fn donation(self: Tensor) ?usize { - return CompilationContext.current().currentScope().id_to_donation.get(self.id); + return Compiler.current().currentScope().id_to_donation.get(self.id); } /// Returns the output memory kind of the tensor. pub fn outputMemoryKind(self: Tensor) Memory.Kind { - return CompilationContext.current().currentScope().id_to_output_memory_kind.get(self.id) orelse .device; + return Compiler.current().currentScope().id_to_output_memory_kind.get(self.id) orelse .device; } pub fn inputMemoryKind(self: Tensor) ?Memory.Kind { - return CompilationContext.current().currentScope().id_to_input_memory_kind.get(self.id); + return Compiler.current().currentScope().id_to_input_memory_kind.get(self.id); } }; diff --git a/zml/zml.zig b/zml/zml.zig index c88fa6ef..d72fb786 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -15,6 +15,7 @@ pub const tokenizer = @import("zml/tokenizer"); pub const attention = @import("attention.zig"); pub const Buffer = @import("buffer.zig").Buffer; +pub const Compiler = @import("Compiler.zig"); pub const constants = @import("constants.zig"); pub const dtype = @import("dtype.zig"); pub const Data = dtype.Data; @@ -30,9 +31,6 @@ pub const mem = @import("mem.zig"); pub const Bufferized = mem.Bufferized; pub const meta = @import("meta.zig"); pub const mlir = @import("mlirx.zig"); -pub const module = @import("module.zig"); -pub const CompilationOptions = module.CompilationOptions; -pub const Compiler = module.Compiler; pub const moe = @import("moe/moe.zig"); pub const nn = @import("nn.zig"); pub const ops = @import("ops.zig"); @@ -52,6 +50,25 @@ pub const tensor = @import("tensor.zig"); pub const Tensor = tensor.Tensor; pub const testing = @import("testing.zig"); +/// Deprecated, use `zml.Compiler` instead +pub const module = struct { + pub const CompilationContext = Compiler; + pub const CompileError = Compiler.Error; + pub const CompilationOptions = Compiler.Options; + + /// Deprecated, use `platform.compileFn` or `zml.Compiler.compile` + pub fn compile( + allocator: std.mem.Allocator, + io_: std.Io, + comptime func: anytype, + args: std.meta.ArgsTuple(@TypeOf(func)), + platform_: *const Platform, + opts: Compiler.Options, + ) Compiler.Error!Exe { + return Compiler.compile(allocator, io_, platform_, func, args, opts); + } +}; + test "zml" { std.testing.refAllDecls(@This()); } From 450abb613174975770a865b20547abcde8b825e5 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Fri, 28 Aug 2026 15:38:11 +0200 Subject: [PATCH 638/881] zml/attention: optimize sparse MLA value accumulation (#737) GLM 5.2 represents each MLA query/key head as 512 latent dimensions followed by 64 RoPE dimensions. All 576 dimensions participate in the attention score, but the value is only the 512-dimensional latent prefix. Previously, sparse MLA accumulated a 576-dimensional context and sliced off the RoPE suffix afterward. Return the 512-dimensional value context directly instead. This also reduces the padded value accumulator from 1024 to 512 elements. Make the value width explicit so the same implementation also supports models such as DeepSeek V4, where the RoPE dimensions are included in the value. Keys and values share the same latent cache. Remove the redundant value pointer and strides, reuse the loaded key tile for value accumulation, and remove paging arguments that are no longer needed after converting top-k indices to physical cache positions. Add a grouped partial-reduction path for the high split counts that improve occupancy on MI300, and tune the shape-based launch selection more aggressively for GB300. --- zml/attention/paged_attention.zig | 128 ++++-- zml/attention/triton_attention.zig | 363 +++++++++------- .../triton_kernels/unified_sparse_mla.zig | 411 ++++++++---------- 3 files changed, 483 insertions(+), 419 deletions(-) diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 39d2864d..d154db52 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -801,8 +801,9 @@ pub fn partialSoftmax(self: zml.Tensor, axis: anytype) PartialSoftmax { pub const Mla = struct { pub const Options = struct { rope_rank: i64, + value_rank: i64, scale: ?f32 = null, - /// null selects automatically; 1 forces the 2D kernel; other values must be powers of two up to 16. + /// null selects automatically; 1 forces the 2D kernel; other values must be powers of two up to 128. num_kv_splits: ?u8 = null, }; @@ -833,9 +834,12 @@ pub const Mla = struct { const attn_weights = scores_sink.softmax(.kv); const attn_weights_non_sink = attn_weights.slice(-1, .{ .end = topk.dim(.topk) }); - return attn_weights_non_sink.dot(selected_kv, .kv).convert(q.dtype()); + const selected_values = selected_kv.slice(.hd, .{ .end = opts.value_rank }); + return attn_weights_non_sink.dot(selected_values, .kv).convert(q.dtype()); } + /// Computes sparse MLA scores over the complete cached key and returns + /// `opts.value_rank` output dimensions per head. pub fn pagedSparseAttention(parameters: Parameters, q: zml.Tensor, kv_cache: KvCache, sink: ?zml.Tensor, topk: zml.Tensor, tokens_pos: zml.Tensor, opts: Mla.Options) zml.Tensor { const latent_kv = switch (kv_cache) { .latent => |latent_kv| latent_kv, @@ -844,6 +848,8 @@ pub const Mla = struct { stdx.debug.assert(q.shape().hasTags(.{ .q, .h, .hd }), "expected q to have tags .q, .h, .hd after flattening, got {f}", .{q.shape()}); stdx.debug.assert(q.dim(.hd) > opts.rope_rank, "expected q head dim ({}) to include a rope tail of {}", .{ q.dim(.hd), opts.rope_rank }); + stdx.debug.assert(opts.value_rank > 0, "expected MLA value rank to be positive, got {}", .{opts.value_rank}); + stdx.debug.assert(opts.value_rank == q.dim(.hd) or opts.value_rank + opts.rope_rank == q.dim(.hd), "expected MLA value rank ({}) to cover either the complete qk head ({}) or its non-RoPE prefix ({})", .{ opts.value_rank, q.dim(.hd), q.dim(.hd) - opts.rope_rank }); stdx.debug.assert(latent_kv.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "expected paged latent KV cache to have tags .page, .k_chunk, .hkv, .hd, got {f}", .{latent_kv.shape()}); stdx.debug.assert(latent_kv.dim(.hd) == q.dim(.hd), "expected q and kv cache head dims to match, got q={} kv={}", .{ q.dim(.hd), latent_kv.dim(.hd) }); @@ -861,7 +867,7 @@ pub const Mla = struct { } }; -test "use mla kernel" { +test "Triton sparse MLA value ranks and padded queries" { const platform = zml.testing.env(); if (!Backend.triton.isAvailable(platform)) return error.SkipZigTest; @@ -871,40 +877,45 @@ test "use mla kernel" { .batch_size = 1, .seq_len = 32, .max_num_pages = 2, - .max_token_count = 1, + .max_token_count = 2, .num_heads = 16, .num_kv_heads = 1, .head_dim = 128, - .max_seqlen_q = 1, + .max_seqlen_q = 2, })); - const q_shape = zml.Shape.init(.{ .q = 1, .h = 16, .hd = 128 }, .f32); + const q_shape = zml.Shape.init(.{ .q = 2, .h = 16, .hd = 128 }, .f32); const kv_shape = zml.Shape.init(.{ .page = 2, .k_chunk = 16, .hkv = 1, .hd = 128 }, .f32); const sink_shape = zml.Shape.init(.{ .h = 16 }, .f32); - const topk_shape = zml.Shape.init(.{ .q = 1, .topk = 32 }, .i32); - const tokens_pos_shape = zml.Shape.init(.{ .q = 1 }, .i32); + const topk_shape = zml.Shape.init(.{ .q = 2, .topk = 32 }, .i32); + const tokens_pos_shape = zml.Shape.init(.{ .q = 2 }, .i32); - var q_data: [1][16][128]f32 = undefined; + var q_data: [2][16][128]f32 = undefined; @memset(std.mem.sliceAsBytes(&q_data), 0); + for (&q_data[0]) |*head| head[64] = 1; var kv_data: [2][16][1][128]f32 = undefined; @memset(std.mem.sliceAsBytes(&kv_data), 0); - for (&kv_data[1][0][0]) |*value| value.* = 10; - for (&kv_data[1][1][0]) |*value| value.* = 11; + @memset(kv_data[1][0][0][0..64], 10); + @memset(kv_data[1][1][0][0..64], 20); + kv_data[1][0][0][64] = 0; + kv_data[1][1][0][64] = 2; var sink_data: [16]f32 = undefined; for (&sink_data) |*value| value.* = -std.math.inf(f32); - var topk_data: [1][32]i32 = undefined; + var topk_data: [2][32]i32 = undefined; @memset(&topk_data[0], -1); + @memset(&topk_data[1], -1); topk_data[0][0] = 0; topk_data[0][1] = 1; - const tokens_pos_data: [1]i32 = .{31}; + const tokens_pos_data: [2]i32 = .{ 31, 31 }; const block_table: [1][2]i32 = .{.{ 1, 0 }}; const seq_lens: [1]i32 = .{32}; + // Only the first of the two statically allocated query rows is active. const query_start_len: [2]i32 = .{ 0, 1 }; const q = zml.Tensor.init(q_shape, .f32); const kv: KvCache = .{ .latent = zml.Tensor.init(kv_shape, .f32) }; const sink = zml.Tensor.init(sink_shape, .f32); const topk = zml.Tensor.init(topk_shape, .i32); - const tokens_pos = zml.Tensor.init(.{ .q = 1 }, .i32); + const tokens_pos = zml.Tensor.init(tokens_pos_shape, .i32); var parameters_d: zml.Bufferized(Parameters) = .{ .triton = .{ .block_table = try .fromBytes(std.testing.io, platform, parameters.triton.block_table.shape(), .replicated, std.mem.sliceAsBytes(&block_table)), @@ -923,23 +934,59 @@ test "use mla kernel" { var tokens_pos_d = try zml.Buffer.fromBytes(std.testing.io, platform, tokens_pos_shape, .replicated, std.mem.sliceAsBytes(&tokens_pos_data)); defer tokens_pos_d.deinit(); - const exe = try platform.compileFn( - std.testing.allocator, - std.testing.io, - Mla.pagedSparseAttention, - .{ parameters, q, kv, sink, topk, tokens_pos, .{ .rope_rank = 64, .num_kv_splits = 2 } }, - .{}, - ); - defer exe.deinit(); + const TestCase = struct { + num_kv_splits: u8, + value_rank: i64, + }; + inline for ([_]TestCase{ + .{ .num_kv_splits = 2, .value_rank = 64 }, + .{ .num_kv_splits = 1, .value_rank = 128 }, + }) |case| { + var exe = try platform.compileFn( + std.testing.allocator, + std.testing.io, + Mla.pagedSparseAttention, + .{ parameters, q, kv, sink, topk, tokens_pos, .{ + .rope_rank = 64, + .value_rank = case.value_rank, + .scale = 1, + .num_kv_splits = case.num_kv_splits, + } }, + .{}, + ); + defer exe.deinit(); - var output_d = try zml.testing.autoCall( - std.testing.allocator, - std.testing.io, - &exe, - Mla.pagedSparseAttention, - .{ parameters_d, q_d, kv_d, sink_d, topk_d, tokens_pos_d }, - ); - defer zml.Buffer.deinitAll(zml.Tensor, &output_d); + var output_d = try zml.testing.autoCall( + std.testing.allocator, + std.testing.io, + &exe, + Mla.pagedSparseAttention, + .{ parameters_d, q_d, kv_d, sink_d, topk_d, tokens_pos_d }, + ); + defer output_d.deinit(); + try std.testing.expect(output_d.shape().eql(q_shape.set(.hd, case.value_rank))); + + var output = try output_d.toSliceAlloc(std.testing.allocator, std.testing.io); + defer output.free(std.testing.allocator); + const active_output = output.subSlice(output.shape.axis(.q), 0, 1); + + const expected_shape = q_shape.set(.q, 1).set(.hd, case.value_rank); + const expected_data = try std.testing.allocator.alloc(f32, expected_shape.count()); + defer std.testing.allocator.free(expected_data); + const exp_two = @exp(@as(f32, 2)); + const second_weight = exp_two / (1 + exp_two); + for (expected_data, 0..) |*value, i| { + const dim = i % @as(usize, @intCast(case.value_rank)); + value.* = if (dim < 64) + 10 + 10 * second_weight + else if (dim == 64) + 2 * second_weight + else + 0; + } + const expected = zml.Slice.init(expected_shape, std.mem.sliceAsBytes(expected_data)); + try zml.testing.expectClose(std.testing.io, expected, active_output, .{ .absolute_tolerance = 0.001, .relative_tolerance = 0.001 }); + } } test "execute stablehlo mla kernel" { @@ -965,10 +1012,13 @@ test "execute stablehlo mla kernel" { var q_data: [1][16][128]f32 = undefined; @memset(std.mem.sliceAsBytes(&q_data), 0); + for (&q_data[0]) |*head| head[64] = 1; var kv_data: [2][16][1][128]f32 = undefined; @memset(std.mem.sliceAsBytes(&kv_data), 0); - for (&kv_data[1][0][0]) |*value| value.* = 10; - for (&kv_data[1][1][0]) |*value| value.* = 11; + @memset(kv_data[1][0][0][0..64], 10); + @memset(kv_data[1][1][0][0..64], 20); + kv_data[1][0][0][64] = 0; + kv_data[1][1][0][64] = 2; var sink_data: [16]f32 = undefined; for (&sink_data) |*value| value.* = -std.math.inf(f32); const topk_data: [1][2]i32 = .{.{ 0, 1 }}; @@ -1004,7 +1054,11 @@ test "execute stablehlo mla kernel" { std.testing.allocator, std.testing.io, Mla.pagedSparseAttention, - .{ parameters, q, kv, sink, topk, tokens_pos, .{ .rope_rank = 64 } }, + .{ parameters, q, kv, sink, topk, tokens_pos, .{ + .rope_rank = 64, + .value_rank = 64, + .scale = 1, + } }, .{}, ); defer exe.deinit(); @@ -1017,4 +1071,12 @@ test "execute stablehlo mla kernel" { .{ parameters_d, q_d, kv_d, sink_d, topk_d, tokens_pos_d }, ); defer zml.Buffer.deinitAll(zml.Tensor, &output_d); + + const output_shape = q_shape.set(.hd, 64); + var expected_data: [1][16][64]f32 = undefined; + const exp_two = @exp(@as(f32, 2)); + const expected_value = (10 + 20 * exp_two) / (1 + exp_two); + for (&expected_data[0]) |*head| @memset(head, expected_value); + const expected = zml.Slice.init(output_shape, std.mem.sliceAsBytes(&expected_data)); + try zml.testing.expectClose(std.testing.io, expected, output_d, .{ .absolute_tolerance = 0.001, .relative_tolerance = 0.001 }); } diff --git a/zml/attention/triton_attention.zig b/zml/attention/triton_attention.zig index 7a7b798c..62c08ec3 100644 --- a/zml/attention/triton_attention.zig +++ b/zml/attention/triton_attention.zig @@ -38,74 +38,98 @@ pub const Config2D = struct { total_q_blocks: usize, }; -const SparseMlaLaunchConfig = struct { +const SparseMlaConfig = struct { block_m: usize, tile_size: usize, - num_tiles: usize, num_splits: usize, direct_programs: usize, + main_num_warps: usize = 4, + main_num_stages: usize = 2, + grouped_reduce_threshold: usize = 32, + splits_per_group: usize = 16, + grouped_reduce_num_warps: usize = 1, + parallel_reduce_min_splits: usize = std.math.maxInt(usize), + parallel_reduce_num_warps: usize = 1, }; -fn selectSparseMlaLaunchConfig( - query_count: usize, - num_heads: usize, - topk_count: usize, - cu_count_: usize, - requested_splits: ?u8, -) SparseMlaLaunchConfig { - stdx.debug.assert(query_count > 0, "sparse MLA requires at least one query", .{}); - stdx.debug.assert(num_heads > 0, "sparse MLA requires at least one query head", .{}); +fn isCudaComputeCapability(expected: []const u8) bool { + const platform = zml.module.CompilationContext.current().platform; + if (platform.target != .cuda) return false; + const devices = platform.pjrt_client.devices(platform.pjrt_api); + if (devices.len == 0) return false; + const actual = zml.platform.cuda.tryGetComputeCapabilities(platform, devices[0]) orelse return false; + return std.mem.eql(u8, actual, expected); +} + +fn sparseMlaConfig(paged_opts: paged.PagedSparseMlaOptions, topk_count: usize, cu_count_: usize) SparseMlaConfig { + stdx.debug.assert(paged_opts.total_q_blocks > 0, "sparse MLA requires at least one query", .{}); + stdx.debug.assert(paged_opts.num_heads > 0, "sparse MLA requires at least one query head", .{}); stdx.debug.assert(topk_count > 0, "sparse MLA requires at least one top-k entry", .{}); - const block_m = @min(num_heads, 16); - stdx.debug.assert(@mod(num_heads, block_m) == 0, "expected q heads ({}) to be divisible by block_m ({})", .{ num_heads, block_m }); + const padded_heads = std.math.ceilPowerOfTwoAssert(usize, paged_opts.num_heads); + const padded_topk = std.math.ceilPowerOfTwoAssert(usize, topk_count); + var config: SparseMlaConfig = .{ + .block_m = @min(padded_heads, 16), + .tile_size = @min(padded_topk, 16), + .num_splits = 1, + .direct_programs = undefined, + }; + const is_sm103 = isCudaComputeCapability("10.3"); - const topk_padded = std.math.ceilPowerOfTwoAssert(usize, topk_count); - const tile_size = @min(topk_padded, 16); - const num_tiles = std.math.divCeil(usize, topk_count, tile_size) catch unreachable; - const direct_programs = query_count * @divExact(num_heads, block_m); - const cu_count = @max(cu_count_, 1); - - var max_splits: usize = 1; - while (max_splits * 2 <= @min(num_tiles, 16)) max_splits *= 2; - - if (requested_splits) |requested| { - const num_splits: usize = requested; - stdx.debug.assert(std.math.isPowerOfTwo(num_splits), "MLA num_kv_splits ({}) must be a power of two", .{num_splits}); - stdx.debug.assert(num_splits <= 16, "MLA num_kv_splits ({}) must not exceed 16", .{num_splits}); - stdx.debug.assert(num_splits <= num_tiles, "MLA num_kv_splits ({}) must not exceed sparse tile count ({})", .{ num_splits, num_tiles }); - return .{ - .block_m = block_m, - .tile_size = tile_size, - .num_tiles = num_tiles, - .num_splits = num_splits, - .direct_programs = direct_programs, - }; + // GB300 (sm_103): a single wide query benefits from more head blocks and + // wider sparse tiles. The split selection below still derives each layer's + // split count from query/head/top-k shapes (16 for DSV4 CSA/HCA and 4 for + // its 128-entry full-attention layers). + if (is_sm103 and + paged_opts.all_decode and + paged_opts.total_q_blocks == 1 and + paged_opts.value_rank >= 512) + { + config.block_m = @min(padded_heads, 8); + config.tile_size = @min(padded_topk, 32); + config.parallel_reduce_min_splits = 16; + config.parallel_reduce_num_warps = 2; + } + // Full 256-query prefill chunks already expose enough query parallelism. + // Retain BLOCK_M=16 and the generic split model, but process wider sparse + // tiles. The predicate uses the post-sharding flattened query shape. + if (is_sm103 and + !paged_opts.all_decode and + paged_opts.total_q_blocks >= 256 and + paged_opts.value_rank >= 512) + { + config.tile_size = @min(padded_topk, 32); } - var best_splits: usize = 1; - var best_cost: usize = std.math.maxInt(usize); - const candidates = [_]usize{ 1, 2, 4, 8, 16 }; - for (candidates) |num_splits| { - if (num_splits > max_splits) break; - const programs = direct_programs * num_splits; - const rounds = std.math.divCeil(usize, programs, cu_count) catch unreachable; - const tiles_per_program = std.math.divCeil(usize, num_tiles, num_splits) catch unreachable; - const cost = rounds * tiles_per_program; - // Candidates are ordered by split count, so ties retain 2D or the lower-overhead 3D launch. - if (cost < best_cost) { - best_cost = cost; - best_splits = num_splits; + const num_tiles = std.math.divCeil(usize, topk_count, config.tile_size) catch unreachable; + const head_blocks = std.math.divCeil(usize, paged_opts.num_heads, config.block_m) catch unreachable; + config.direct_programs = paged_opts.total_q_blocks * head_blocks; + const cu_count = @max(cu_count_, 1); + + if (paged_opts.num_kv_splits) |requested| { + const num_splits: usize = requested; + stdx.debug.assert(std.math.isPowerOfTwo(num_splits), "MLA num_kv_splits ({}) must be a power of two", .{num_splits}); + stdx.debug.assert(num_splits <= 128, "MLA num_kv_splits ({}) must not exceed 128", .{num_splits}); + stdx.debug.assert(num_splits <= num_tiles, "MLA num_kv_splits ({}) must not exceed sparse tile count ({})", .{ num_splits, num_tiles }); + config.num_splits = num_splits; + } else { + var best_cost: usize = std.math.maxInt(usize); + const candidates = [_]usize{ 1, 2, 4, 8, 16, 32, 64, 128 }; + for (candidates) |num_splits| { + if (num_splits > num_tiles) break; + const programs = config.direct_programs * num_splits; + const rounds = std.math.divCeil(usize, programs, cu_count) catch unreachable; + const tiles_per_program = std.math.divCeil(usize, num_tiles, num_splits) catch unreachable; + const cost = rounds * tiles_per_program; + // Candidates are ordered by split count, so ties retain 2D or the lower-overhead 3D launch. + if (cost < best_cost) { + best_cost = cost; + config.num_splits = num_splits; + } } } - return .{ - .block_m = block_m, - .tile_size = tile_size, - .num_tiles = num_tiles, - .num_splits = best_splits, - .direct_programs = direct_programs, - }; + return config; } fn select2dConfig(options: paged.PagedAttentionOptions) Config2D { @@ -697,12 +721,14 @@ pub const paged = struct { return output.output; } - fn pagedSparseMlaKernel(q: zml.Tensor, kv_cache: zml.Tensor, sink: ?zml.Tensor, topk: zml.Tensor, parameters: Parameters, paged_opts: PagedSparseMlaOptions) zml.Tensor { - const rope_rank: i64 = @intCast(paged_opts.rope_rank); - const nope_rank: i64 = @intCast(paged_opts.head_dim - paged_opts.rope_rank); - const kv_lora_rank: i64 = @intCast(std.math.ceilPowerOfTwoAssert(usize, @intCast(nope_rank))); + fn pagedSparseMlaKernel(q: zml.Tensor, kv_cache: zml.Tensor, sink: ?zml.Tensor, topk: zml.Tensor, active_query_count: zml.Tensor, paged_opts: PagedSparseMlaOptions) zml.Tensor { + const value_rank: i64 = @intCast(paged_opts.value_rank); + const rope_rank: i64 = if (paged_opts.value_rank == paged_opts.qk_rank) 0 else @intCast(paged_opts.rope_rank); + stdx.debug.assert(std.math.isPowerOfTwo(paged_opts.value_rank), "sparse MLA value rank ({}) must be a power of two", .{paged_opts.value_rank}); + stdx.debug.assert(rope_rank == 0 or std.math.isPowerOfTwo(@as(usize, @intCast(rope_rank))), "sparse MLA separate RoPE rank ({}) must be a power of two", .{rope_rank}); + stdx.debug.assert(value_rank + rope_rank == paged_opts.qk_rank, "sparse MLA expects either a full-width value or adjacent value/RoPE regions, got qk={} value={} rope={}", .{ paged_opts.qk_rank, value_rank, rope_rank }); - const out_shape = q.shape(); + const out_shape = q.shape().set(.hd, value_rank); const q_strides = q.shape().computeElementStrides().constSlice(); const out_strides = out_shape.computeElementStrides().constSlice(); @@ -710,15 +736,9 @@ pub const paged = struct { const sink_: zml.Tensor = sink orelse .scalar(0, .i8); const use_sink = sink != null; - const sm_scale: f32 = paged_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(paged_opts.head_dim)))); + const sm_scale: f32 = paged_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(paged_opts.qk_rank)))); - const launch = selectSparseMlaLaunchConfig( - paged_opts.total_q_blocks, - paged_opts.num_heads, - @intCast(topk.dim(.topk)), - getCuCount(), - paged_opts.num_kv_splits, - ); + const config = sparseMlaConfig(paged_opts, @intCast(topk.dim(.topk)), getCuCount()); const kernel_config: mla_kernels.Config = .{ .q_dtype = triton.from(q.dtype()), @@ -726,102 +746,115 @@ pub const paged = struct { .sink_dtype = triton.from(sink_.dtype()), .o_dtype = triton.from(q.dtype()), .num_query_heads = @intCast(paged_opts.num_heads), - .num_queries_per_kv = @intCast(paged_opts.num_heads), .block_size = @intCast(paged_opts.block_size), - .block_m = @intCast(launch.block_m), + .block_m = @intCast(config.block_m), .topk_count = topk.dim(.topk), .rope_rank = rope_rank, - .qk_lora_rank = nope_rank, - .kv_lora_rank = kv_lora_rank, - .rope_offset = nope_rank, - .value_rank = @intCast(paged_opts.head_dim), - .tile_size = @intCast(launch.tile_size), - .num_splits = @intCast(launch.num_splits), + .value_rank = value_rank, + .rope_offset = value_rank, + .tile_size = @intCast(config.tile_size), + .num_splits = @intCast(config.num_splits), .use_attn_sink = use_sink, - .all_decode = !parameters.options_.is_prefill, + .all_decode = paged_opts.all_decode, }; - log.debug("pagedSparseMla launch: {any}, kernel: {any}", .{ launch, kernel_config }); + log.debug("pagedSparseMla config: {any}, kernel: {any}", .{ config, kernel_config }); const kernel_inputs: mla_kernels.Kernel2D.Inputs = .{ .query_ptr = q, - .key_cache_ptr = kv_cache, - .value_cache_ptr = kv_cache, + .kv_cache_ptr = kv_cache, .attn_sink_ptr = sink_, - .block_tables_ptr = parameters.block_table, .topk_indices_ptr = topk, - .seq_lens_ptr = parameters.seq_lens, + .active_query_count_ptr = active_query_count, .scale_ptr = zml.Tensor.constant(zml.DataType.f32.constant(sm_scale)), - .block_table_stride_ptr = zml.Tensor.constant(zml.DataType.i64.constant(parameters.block_table.dim(.p))), .query_stride_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(q_strides[0])), .query_stride_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(q_strides[1])), .output_stride_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(out_strides[0])), .output_stride_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(out_strides[1])), - .stride_k_cache_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(kv_strides[kv_cache.shape().axis(.page)])), - .stride_k_cache_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(kv_strides[kv_cache.shape().axis(.k_chunk)])), - .stride_k_cache_2_ptr = zml.Tensor.constant(zml.DataType.i64.constant(kv_strides[kv_cache.shape().axis(.hkv)])), - .stride_v_cache_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(kv_strides[kv_cache.shape().axis(.page)])), - .stride_v_cache_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(kv_strides[kv_cache.shape().axis(.k_chunk)])), - .stride_v_cache_2_ptr = zml.Tensor.constant(zml.DataType.i64.constant(kv_strides[kv_cache.shape().axis(.hkv)])), - .query_start_len_ptr = parameters.query_start_len, - .num_seqs_ptr = zml.Tensor.constant(.{ .i32 = @as(i32, @intCast(parameters.block_table.dim(.b))) }).reshape(.{1}), + .stride_cache_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(kv_strides[kv_cache.shape().axis(.page)])), + .stride_cache_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(kv_strides[kv_cache.shape().axis(.k_chunk)])), }; - if (launch.num_splits == 1) { + if (config.num_splits == 1) { const out = mla_kernels.Kernel2D.call( kernel_inputs, .{ .output = out_shape }, .{ .cfg = kernel_config, - .grid = .{ @intCast(launch.direct_programs), 1, 1 }, - .num_warps = 4, - .num_stages = 2, + .grid = .{ @intCast(config.direct_programs), 1, 1 }, + .num_warps = @intCast(config.main_num_warps), + .num_stages = @intCast(config.main_num_stages), }, ); - return out.output.reshape(q.shape()); + return out.output.reshape(out_shape); } - const num_splits: i64 = @intCast(launch.num_splits); + const num_splits: i64 = @intCast(config.num_splits); const kernel_3d_inputs: mla_kernels.Kernel3D.Inputs = .{ .query_ptr = kernel_inputs.query_ptr, - .key_cache_ptr = kernel_inputs.key_cache_ptr, - .value_cache_ptr = kernel_inputs.value_cache_ptr, + .kv_cache_ptr = kernel_inputs.kv_cache_ptr, .attn_sink_ptr = kernel_inputs.attn_sink_ptr, - .block_tables_ptr = kernel_inputs.block_tables_ptr, .topk_indices_ptr = kernel_inputs.topk_indices_ptr, - .seq_lens_ptr = kernel_inputs.seq_lens_ptr, + .active_query_count_ptr = kernel_inputs.active_query_count_ptr, .scale_ptr = kernel_inputs.scale_ptr, - .block_table_stride_ptr = kernel_inputs.block_table_stride_ptr, .query_stride_0_ptr = kernel_inputs.query_stride_0_ptr, .query_stride_1_ptr = kernel_inputs.query_stride_1_ptr, .output_stride_0_ptr = kernel_inputs.output_stride_0_ptr, .output_stride_1_ptr = kernel_inputs.output_stride_1_ptr, - .stride_k_cache_0_ptr = kernel_inputs.stride_k_cache_0_ptr, - .stride_k_cache_1_ptr = kernel_inputs.stride_k_cache_1_ptr, - .stride_k_cache_2_ptr = kernel_inputs.stride_k_cache_2_ptr, - .stride_v_cache_0_ptr = kernel_inputs.stride_v_cache_0_ptr, - .stride_v_cache_1_ptr = kernel_inputs.stride_v_cache_1_ptr, - .stride_v_cache_2_ptr = kernel_inputs.stride_v_cache_2_ptr, - .query_start_len_ptr = kernel_inputs.query_start_len_ptr, - .num_seqs_ptr = kernel_inputs.num_seqs_ptr, + .stride_cache_0_ptr = kernel_inputs.stride_cache_0_ptr, + .stride_cache_1_ptr = kernel_inputs.stride_cache_1_ptr, }; const partials = mla_kernels.Kernel3D.call( kernel_3d_inputs, .{ - .partial_output = zml.Shape.init(.{ q.dim(.q), @as(i64, @intCast(paged_opts.num_heads)), num_splits, @as(i64, @intCast(paged_opts.head_dim)) }, .f32), + .partial_output = zml.Shape.init(.{ q.dim(.q), @as(i64, @intCast(paged_opts.num_heads)), num_splits, value_rank }, .f32), .partial_lse = zml.Shape.init(.{ q.dim(.q), @as(i64, @intCast(paged_opts.num_heads)), num_splits }, .f32), }, .{ .cfg = kernel_config, - .grid = .{ @intCast(launch.direct_programs), @intCast(launch.num_splits), 1 }, - .num_warps = 4, - .num_stages = 2, + .grid = .{ @intCast(config.direct_programs), @intCast(config.num_splits), 1 }, + .num_warps = @intCast(config.main_num_warps), + .num_stages = @intCast(config.main_num_stages), }, ); + var reduce_partial_output = partials.partial_output; + var reduce_partial_lse = partials.partial_lse; + var reduce_num_splits = num_splits; + if (num_splits > config.grouped_reduce_threshold) { + const grouped_splits: i64 = @divExact(num_splits, @as(i64, @intCast(config.splits_per_group))); + const grouped = mla_kernels.Reduce3DPartials.call( + .{ + .input = partials.partial_output, + .input_lse = partials.partial_lse, + .active_query_count_ptr = active_query_count, + }, + .{ + .partial_output = zml.Shape.init(.{ q.dim(.q), @as(i64, @intCast(paged_opts.num_heads)), grouped_splits, value_rank }, .f32), + .partial_lse = zml.Shape.init(.{ q.dim(.q), @as(i64, @intCast(paged_opts.num_heads)), grouped_splits }, .f32), + }, + .{ + .cfg = .{ + .num_query_heads = @intCast(paged_opts.num_heads), + .value_rank = value_rank, + .num_input_splits = num_splits, + .num_output_splits = grouped_splits, + .all_decode = paged_opts.all_decode, + }, + .grid = .{ @intCast(q.dim(.q)), @intCast(paged_opts.num_heads * @as(usize, @intCast(grouped_splits))), 1 }, + .num_warps = @intCast(config.grouped_reduce_num_warps), + .num_stages = 1, + }, + ); + reduce_partial_output = grouped.partial_output; + reduce_partial_lse = grouped.partial_lse; + reduce_num_splits = grouped_splits; + } + const out = mla_kernels.Reduce3D.call( .{ - .partial_output_ptr = partials.partial_output, - .partial_lse_ptr = partials.partial_lse, + .partial_output_ptr = reduce_partial_output, + .partial_lse_ptr = reduce_partial_lse, .attn_sink_ptr = sink_, + .active_query_count_ptr = active_query_count, .output_stride_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(out_strides[0])), .output_stride_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(out_strides[1])), }, @@ -831,17 +864,21 @@ pub const paged = struct { .sink_dtype = triton.from(sink_.dtype()), .o_dtype = triton.from(q.dtype()), .num_query_heads = @intCast(paged_opts.num_heads), - .value_rank = @intCast(paged_opts.head_dim), - .num_splits = num_splits, + .value_rank = value_rank, + .num_splits = reduce_num_splits, .use_attn_sink = use_sink, + .all_decode = paged_opts.all_decode, }, .grid = .{ @intCast(q.dim(.q)), @intCast(paged_opts.num_heads), 1 }, - .num_warps = 1, + .num_warps = @intCast(if (reduce_num_splits >= config.parallel_reduce_min_splits) + config.parallel_reduce_num_warps + else + 1), .num_stages = 1, }, ); - return out.output.reshape(q.shape()); + return out.output.reshape(out_shape); } fn tokenToSequence(query_start_len: zml.Tensor, query_count: i64) zml.Tensor { @@ -895,16 +932,19 @@ pub const paged = struct { } pub const PagedSparseMlaOptions = struct { - head_dim: usize, + qk_rank: usize, + value_rank: usize, num_heads: usize, block_size: usize, rope_rank: usize, scale: ?f32, total_q_blocks: usize, num_kv_splits: ?u8, + all_decode: bool, }; pub fn pagedSparseMla(parameters: Parameters, q: zml.Tensor, kv_cache: zml.Tensor, sink: ?zml.Tensor, topk: zml.Tensor, tokens_pos: zml.Tensor, opts: MlaOptions) zml.Tensor { + const output_shape = q.shape().set(.hd, opts.value_rank); return zml.ops.manualComputation( .{ q, @@ -916,7 +956,7 @@ pub const paged = struct { parameters.seq_lens, parameters.query_start_len, }, - q.shape(), + output_shape, .{ .opts = opts, .options = parameters.options_, @@ -936,17 +976,22 @@ pub const paged = struct { }; const topk_final = topkToPhysical(parameters_, sharded_inputs[3], sharded_inputs[4], block_size); + const active_query_count = sharded_inputs[7] + .slice(.b, .{ .start = sharded_inputs[7].dim(.b) - 1 }) + .squeeze(.b); stdx.debug.assert(topk_final.dim(.q) == q_.dim(.q), "expected topk q dim ({}) to match q dim ({})", .{ topk_final.dim(.q), q_.dim(.q) }); const num_heads: usize = @intCast(q_.dim(.h)); const paged_opts: PagedSparseMlaOptions = .{ - .head_dim = @intCast(q_.dim(.hd)), + .qk_rank = @intCast(q_.dim(.hd)), + .value_rank = @intCast(ctx_.opts.value_rank), .num_heads = num_heads, .block_size = @intCast(kv_cache_.dim(.k_chunk)), .rope_rank = @intCast(ctx_.opts.rope_rank), .scale = ctx_.opts.scale, .total_q_blocks = @intCast(q_.dim(.q)), .num_kv_splits = ctx_.opts.num_kv_splits, + .all_decode = !ctx_.options.is_prefill, }; return pagedSparseMlaKernel( @@ -954,7 +999,7 @@ pub const paged = struct { kv_cache_, sharded_inputs[2], topk_final, - parameters_, + active_query_count, paged_opts, ); } @@ -963,29 +1008,6 @@ pub const paged = struct { } }; -test "sparse MLA launch selection balances occupancy and split overhead" { - const saturated = selectSparseMlaLaunchConfig(120, 16, 512, 120, null); - try std.testing.expectEqual(@as(usize, 1), saturated.num_splits); - - const under_occupied = selectSparseMlaLaunchConfig(1, 16, 512, 120, null); - try std.testing.expectEqual(@as(usize, 16), under_occupied.num_splits); - - const tile_limited = selectSparseMlaLaunchConfig(1, 16, 32, 120, null); - try std.testing.expectEqual(@as(usize, 2), tile_limited.num_splits); - - const missing_cu_count = selectSparseMlaLaunchConfig(1, 16, 512, 0, null); - try std.testing.expectEqual(@as(usize, 1), missing_cu_count.num_splits); -} - -test "sparse MLA launch selection honors valid explicit splits" { - const forced = selectSparseMlaLaunchConfig(1, 16, 64, 120, 4); - try std.testing.expectEqual(@as(usize, 4), forced.num_splits); - - const one_tile = selectSparseMlaLaunchConfig(1, 16, 5, 120, null); - try std.testing.expectEqual(@as(usize, 8), one_tile.tile_size); - try std.testing.expectEqual(@as(usize, 1), one_tile.num_splits); -} - test "sparse MLA emits 2D and 3D Triton kernels" { const platform = zml.testing.env(); var compilation = zml.Compiler.init(std.testing.allocator, std.testing.io, platform, .{}); @@ -997,38 +1019,47 @@ test "sparse MLA emits 2D and 3D Triton kernels" { compilation.pushBlock(block); defer compilation.popBlock(); - const q = zml.Tensor.zeroes(zml.Shape.init(.{ .q = 1, .h = 16, .hd = 128 }, .bf16)); - const kv = zml.Tensor.zeroes(zml.Shape.init(.{ .page = 32, .k_chunk = 1, .hkv = 1, .hd = 128 }, .bf16)); - const sink = zml.Tensor.zeroes(zml.Shape.init(.{ .h = 16 }, .bf16)); + const q = zml.Tensor.zeroes(zml.Shape.init(.{ .q = 1, .h = 6, .hd = 576 }, .bf16)); + const kv = zml.Tensor.zeroes(zml.Shape.init(.{ .page = 32, .k_chunk = 1, .hkv = 1, .hd = 576 }, .bf16)); + const sink = zml.Tensor.zeroes(zml.Shape.init(.{ .h = 6 }, .bf16)); const topk = zml.Tensor.zeroes(zml.Shape.init(.{ .q = 1, .topk = 32 }, .i32)); - const parameters: paged.Parameters = .{ - .block_table = zml.Tensor.zeroes(zml.Shape.init(.{ .b = 1, .p = 1 }, .i32)), - .seq_lens = zml.Tensor.zeroes(zml.Shape.init(.{ .b = 1 }, .i32)), - .query_start_len = zml.Tensor.zeroes(zml.Shape.init(.{ .b = 2 }, .i32)), - .options_ = .{ - .batch_size = 1, - .max_num_pages = 1, - .max_seqlen_q = 1, - .is_prefill = false, - }, - }; + const active_query_count = zml.Tensor.scalar(1, .i32); const two_d_opts: paged.PagedSparseMlaOptions = .{ - .head_dim = 128, - .num_heads = 16, + .qk_rank = 576, + .value_rank = 512, + .num_heads = 6, .block_size = 1, .rope_rank = 64, .scale = null, .total_q_blocks = 1, .num_kv_splits = 1, + .all_decode = true, }; + const output_shape = q.shape().set(.hd, 512); - const two_d = paged.pagedSparseMlaKernel(q, kv, sink, topk, parameters, two_d_opts); - try std.testing.expect(two_d.shape().eql(q.shape())); + const two_d = paged.pagedSparseMlaKernel(q, kv, sink, topk, active_query_count, two_d_opts); + try std.testing.expect(two_d.shape().eql(output_shape)); try std.testing.expect(two_d.value().owner().verify()); var three_d_opts = two_d_opts; three_d_opts.num_kv_splits = 2; - const three_d = paged.pagedSparseMlaKernel(q, kv, sink, topk, parameters, three_d_opts); - try std.testing.expect(three_d.shape().eql(q.shape())); + const three_d = paged.pagedSparseMlaKernel(q, kv, sink, topk, active_query_count, three_d_opts); + try std.testing.expect(three_d.shape().eql(output_shape)); try std.testing.expect(three_d.value().owner().verify()); + + const dsv4_q = zml.Tensor.zeroes(zml.Shape.init(.{ .q = 1, .h = 6, .hd = 512 }, .bf16)); + const dsv4_kv = zml.Tensor.zeroes(zml.Shape.init(.{ .page = 32, .k_chunk = 1, .hkv = 1, .hd = 512 }, .bf16)); + var dsv4_opts = two_d_opts; + dsv4_opts.qk_rank = 512; + dsv4_opts.value_rank = 512; + const dsv4_output_shape = dsv4_q.shape(); + + const dsv4_two_d = paged.pagedSparseMlaKernel(dsv4_q, dsv4_kv, sink, topk, active_query_count, dsv4_opts); + try std.testing.expect(dsv4_two_d.shape().eql(dsv4_output_shape)); + try std.testing.expect(dsv4_two_d.value().owner().verify()); + + dsv4_opts.num_kv_splits = 2; + const dsv4_three_d = paged.pagedSparseMlaKernel(dsv4_q, dsv4_kv, sink, topk, active_query_count, dsv4_opts); + try std.testing.expect(dsv4_three_d.shape().eql(dsv4_output_shape)); + try std.testing.expect(dsv4_three_d.value().owner().verify()); } diff --git a/zml/attention/triton_kernels/unified_sparse_mla.zig b/zml/attention/triton_kernels/unified_sparse_mla.zig index 2a3a3b43..07ee1b78 100644 --- a/zml/attention/triton_kernels/unified_sparse_mla.zig +++ b/zml/attention/triton_kernels/unified_sparse_mla.zig @@ -15,17 +15,13 @@ pub const Config = struct { sink_dtype: DType = .f32, o_dtype: DType = .bf16, num_query_heads: i64 = 32, - num_queries_per_kv: i64 = 32, block_size: i64 = 16, - stride_k_cache_3: i64 = 1, - stride_v_cache_3: i64 = 1, + stride_cache_dim: i64 = 1, topk_count: i64 = 32, block_m: i64 = 16, rope_rank: i64 = 64, - qk_lora_rank: i64 = 512, - kv_lora_rank: i64 = 512, - rope_offset: i64 = 512, value_rank: i64 = 512, + rope_offset: i64 = 512, tile_size: i64 = 16, num_splits: i64 = 1, use_attn_sink: bool = false, @@ -36,26 +32,17 @@ pub const Kernel2D = tri.Kernel(Config, .{ .name = "_kernel_unified_attention_sparse_mla_2d_ptr", .inputs = &.{ "query_ptr", - "key_cache_ptr", - "value_cache_ptr", + "kv_cache_ptr", "attn_sink_ptr", - "block_tables_ptr", "topk_indices_ptr", - "seq_lens_ptr", + "active_query_count_ptr", "scale_ptr", - "block_table_stride_ptr", "query_stride_0_ptr", "query_stride_1_ptr", "output_stride_0_ptr", "output_stride_1_ptr", - "stride_k_cache_0_ptr", - "stride_k_cache_1_ptr", - "stride_k_cache_2_ptr", - "stride_v_cache_0_ptr", - "stride_v_cache_1_ptr", - "stride_v_cache_2_ptr", - "query_start_len_ptr", - "num_seqs_ptr", + "stride_cache_0_ptr", + "stride_cache_1_ptr", }, .outputs = &.{"output"}, .run = run2D, @@ -64,26 +51,17 @@ pub const Kernel2D = tri.Kernel(Config, .{ fn run2D(b: *tri.Builder, cfg: Config) tri.FinishError!void { const a = try b.declareArgs(.{ .query_ptr = .{ .ptr = cfg.q_dtype }, - .key_cache_ptr = .{ .ptr = cfg.kv_dtype }, - .value_cache_ptr = .{ .ptr = cfg.kv_dtype }, + .kv_cache_ptr = .{ .ptr = cfg.kv_dtype }, .attn_sink_ptr = .{ .ptr = cfg.sink_dtype }, - .block_tables_ptr = .{ .ptr = .i32 }, .topk_indices_ptr = .{ .ptr = .i32 }, - .seq_lens_ptr = .{ .ptr = .i32 }, + .active_query_count_ptr = .{ .ptr = .i32 }, .scale_ptr = .{ .ptr = .f32 }, - .block_table_stride_ptr = .{ .ptr = .i64 }, .query_stride_0_ptr = .{ .ptr = .i64 }, .query_stride_1_ptr = .{ .ptr = .i64 }, .output_stride_0_ptr = .{ .ptr = .i64 }, .output_stride_1_ptr = .{ .ptr = .i64 }, - .stride_k_cache_0_ptr = .{ .ptr = .i64 }, - .stride_k_cache_1_ptr = .{ .ptr = .i64 }, - .stride_k_cache_2_ptr = .{ .ptr = .i64 }, - .stride_v_cache_0_ptr = .{ .ptr = .i64 }, - .stride_v_cache_1_ptr = .{ .ptr = .i64 }, - .stride_v_cache_2_ptr = .{ .ptr = .i64 }, - .query_start_len_ptr = .{ .ptr = .i32 }, - .num_seqs_ptr = .{ .ptr = .i32 }, + .stride_cache_0_ptr = .{ .ptr = .i64 }, + .stride_cache_1_ptr = .{ .ptr = .i64 }, .output_ptr = .{ .ptr = cfg.o_dtype }, }); @@ -92,32 +70,25 @@ fn run2D(b: *tri.Builder, cfg: Config) tri.FinishError!void { const query_stride_1 = b.load(a.query_stride_1_ptr); const output_stride_0 = b.load(a.output_stride_0_ptr); const output_stride_1 = b.load(a.output_stride_1_ptr); - const stride_k_cache_0 = b.load(a.stride_k_cache_0_ptr); - const stride_k_cache_1 = b.load(a.stride_k_cache_1_ptr); - const stride_v_cache_0 = b.load(a.stride_v_cache_0_ptr); - const stride_v_cache_1 = b.load(a.stride_v_cache_1_ptr); - const num_seqs = b.load(a.num_seqs_ptr); + const stride_cache_0 = b.load(a.stride_cache_0_ptr); + const stride_cache_1 = b.load(a.stride_cache_1_ptr); kernelUnifiedAttentionSparseMla( b, a.output_ptr, null, a.query_ptr, - a.key_cache_ptr, - a.value_cache_ptr, + a.kv_cache_ptr, a.attn_sink_ptr, a.topk_indices_ptr, + a.active_query_count_ptr, scale, query_stride_0, query_stride_1, output_stride_0, output_stride_1, - stride_k_cache_0, - stride_k_cache_1, - stride_v_cache_0, - stride_v_cache_1, - a.query_start_len_ptr, - num_seqs, + stride_cache_0, + stride_cache_1, cfg, false, ); @@ -127,26 +98,17 @@ pub const Kernel3D = tri.Kernel(Config, .{ .name = "_kernel_unified_attention_sparse_mla_3d_ptr", .inputs = &.{ "query_ptr", - "key_cache_ptr", - "value_cache_ptr", + "kv_cache_ptr", "attn_sink_ptr", - "block_tables_ptr", "topk_indices_ptr", - "seq_lens_ptr", + "active_query_count_ptr", "scale_ptr", - "block_table_stride_ptr", "query_stride_0_ptr", "query_stride_1_ptr", "output_stride_0_ptr", "output_stride_1_ptr", - "stride_k_cache_0_ptr", - "stride_k_cache_1_ptr", - "stride_k_cache_2_ptr", - "stride_v_cache_0_ptr", - "stride_v_cache_1_ptr", - "stride_v_cache_2_ptr", - "query_start_len_ptr", - "num_seqs_ptr", + "stride_cache_0_ptr", + "stride_cache_1_ptr", }, .outputs = &.{ "partial_output", "partial_lse" }, .run = run3D, @@ -155,26 +117,17 @@ pub const Kernel3D = tri.Kernel(Config, .{ fn run3D(b: *tri.Builder, cfg: Config) tri.FinishError!void { const a = try b.declareArgs(.{ .query_ptr = .{ .ptr = cfg.q_dtype }, - .key_cache_ptr = .{ .ptr = cfg.kv_dtype }, - .value_cache_ptr = .{ .ptr = cfg.kv_dtype }, + .kv_cache_ptr = .{ .ptr = cfg.kv_dtype }, .attn_sink_ptr = .{ .ptr = cfg.sink_dtype }, - .block_tables_ptr = .{ .ptr = .i32 }, .topk_indices_ptr = .{ .ptr = .i32 }, - .seq_lens_ptr = .{ .ptr = .i32 }, + .active_query_count_ptr = .{ .ptr = .i32 }, .scale_ptr = .{ .ptr = .f32 }, - .block_table_stride_ptr = .{ .ptr = .i64 }, .query_stride_0_ptr = .{ .ptr = .i64 }, .query_stride_1_ptr = .{ .ptr = .i64 }, .output_stride_0_ptr = .{ .ptr = .i64 }, .output_stride_1_ptr = .{ .ptr = .i64 }, - .stride_k_cache_0_ptr = .{ .ptr = .i64 }, - .stride_k_cache_1_ptr = .{ .ptr = .i64 }, - .stride_k_cache_2_ptr = .{ .ptr = .i64 }, - .stride_v_cache_0_ptr = .{ .ptr = .i64 }, - .stride_v_cache_1_ptr = .{ .ptr = .i64 }, - .stride_v_cache_2_ptr = .{ .ptr = .i64 }, - .query_start_len_ptr = .{ .ptr = .i32 }, - .num_seqs_ptr = .{ .ptr = .i32 }, + .stride_cache_0_ptr = .{ .ptr = .i64 }, + .stride_cache_1_ptr = .{ .ptr = .i64 }, .partial_output_ptr = .{ .ptr = .f32 }, .partial_lse_ptr = .{ .ptr = .f32 }, }); @@ -184,111 +137,57 @@ fn run3D(b: *tri.Builder, cfg: Config) tri.FinishError!void { const query_stride_1 = b.load(a.query_stride_1_ptr); const output_stride_0 = b.load(a.output_stride_0_ptr); const output_stride_1 = b.load(a.output_stride_1_ptr); - const stride_k_cache_0 = b.load(a.stride_k_cache_0_ptr); - const stride_k_cache_1 = b.load(a.stride_k_cache_1_ptr); - const stride_v_cache_0 = b.load(a.stride_v_cache_0_ptr); - const stride_v_cache_1 = b.load(a.stride_v_cache_1_ptr); - const num_seqs = b.load(a.num_seqs_ptr); + const stride_cache_0 = b.load(a.stride_cache_0_ptr); + const stride_cache_1 = b.load(a.stride_cache_1_ptr); kernelUnifiedAttentionSparseMla( b, a.partial_output_ptr, a.partial_lse_ptr, a.query_ptr, - a.key_cache_ptr, - a.value_cache_ptr, + a.kv_cache_ptr, a.attn_sink_ptr, a.topk_indices_ptr, + a.active_query_count_ptr, scale, query_stride_0, query_stride_1, output_stride_0, output_stride_1, - stride_k_cache_0, - stride_k_cache_1, - stride_v_cache_0, - stride_v_cache_1, - a.query_start_len_ptr, - num_seqs, + stride_cache_0, + stride_cache_1, cfg, true, ); } -fn findSeqIdx( - k: *Builder, - query_start_len_ptr: Value, - target_idx: Value, - num_seqs: Value, - block_q: i64, - use_q_block_mode: bool, -) Value { - const left_init = k.liftAs(0, .i32); - - var w = k.openWhile(.{ left_init, num_seqs }, .{ k.scalarTy(.i32), k.scalarTy(.i32) }); - { - const left = w.before_carried[0]; - const right = w.before_carried[1]; - w.yieldBefore(left.lt(right), .{ left, right }); - } - { - const left = w.after_carried[0]; - const right = w.after_carried[1]; - const mid = left.add(right).div(2); - const val = k.load(query_start_len_ptr.addPtr(mid)); - const mid_val = if (use_q_block_mode) - val.div(@as(i32, @intCast(block_q))).add(mid) - else - val; - - var i = k.openIfElse(mid_val.le(target_idx), .{ k.scalarTy(.i32), k.scalarTy(.i32) }); - { - i.yieldThen(.{ mid.add(1), right }); - } - { - i.yieldElse(.{ left, mid }); - } - w.yieldAfter(.{ i.results[0], i.results[1] }); - } - - return w.results[0].sub(1); -} - fn kernelUnifiedAttentionSparseMla( k: *Builder, output_ptr: Value, partial_lse_ptr: ?Value, query_ptr: Value, - key_cache_ptr: Value, - value_cache_ptr: Value, + kv_cache_ptr: Value, attn_sink_ptr: Value, topk_indices_ptr: Value, + active_query_count_ptr: Value, scale: Value, query_stride_0: Value, query_stride_1: Value, output_stride_0: Value, output_stride_1: Value, - stride_k_cache_0: Value, - stride_k_cache_1: Value, - stride_v_cache_0: Value, - stride_v_cache_1: Value, - query_start_len_ptr: Value, - num_seqs: Value, + stride_cache_0: Value, + stride_cache_1: Value, config: Config, comptime three_d: bool, ) void { - const BLOCK_Q: i64 = 1; const BLOCK_M: i64 = config.block_m; const BLOCK_SIZE: i64 = config.block_size; const ROPE_RANK: i64 = config.rope_rank; - const QK_LORA_RANK: i64 = config.qk_lora_rank; - const KV_LORA_RANK: i64 = config.kv_lora_rank; - const ROPE_OFFSET: i64 = config.rope_offset; const VALUE_RANK: i64 = config.value_rank; + const ROPE_OFFSET: i64 = config.rope_offset; const TILE_SIZE: i64 = config.tile_size; - const NUM_QUERIES_PER_KV: i64 = config.num_queries_per_kv; const NUM_QUERY_HEADS: i64 = config.num_query_heads; - const NUM_HEAD_BLOCKS: i64 = @divTrunc(NUM_QUERY_HEADS, BLOCK_M); + const NUM_HEAD_BLOCKS: i64 = @divTrunc(NUM_QUERY_HEADS + BLOCK_M - 1, BLOCK_M); const NUM_TILES: i64 = @divTrunc(config.topk_count + TILE_SIZE - 1, TILE_SIZE); const NUM_SPLITS: i64 = if (three_d) config.num_splits else 1; const TILES_PER_SPLIT: i64 = @divTrunc(NUM_TILES + NUM_SPLITS - 1, NUM_SPLITS); @@ -299,45 +198,38 @@ fn kernelUnifiedAttentionSparseMla( const split_tile_end = split_idx.add(1).mul(@as(i32, @intCast(NUM_TILES))).div(@as(i32, @intCast(NUM_SPLITS))); const q_ind = q_block_global_idx.div(@as(i32, @intCast(NUM_HEAD_BLOCKS))); const head_ind = q_block_global_idx.rem(@as(i32, @intCast(NUM_HEAD_BLOCKS))); - const seq_idx = findSeqIdx(k, query_start_len_ptr, q_ind, num_seqs, BLOCK_Q, false); - const q_block_start_idx = k.load(query_start_len_ptr.addPtr(seq_idx)); + if (!config.all_decode) { + const active_query_count = k.load(active_query_count_ptr); + k.returnIf(q_ind.ge(active_query_count), .{}); + } - const q_block_local_idx = q_ind.sub(q_block_start_idx); - const cur_batch_in_all_start_index = k.load(query_start_len_ptr.addPtr(seq_idx)); - const cur_batch_in_all_stop_index = k.load(query_start_len_ptr.addPtr(seq_idx).addPtr(1)); - const cur_batch_query_len = cur_batch_in_all_stop_index.sub(cur_batch_in_all_start_index); - - k.returnIf(q_block_local_idx.ge(cur_batch_query_len), .{}); - - const offs_m = k.arange(0, BLOCK_M, .i32).add(head_ind.mul(@as(i32, @intCast(BLOCK_M)))); - const offs_lora = k.arange(0, KV_LORA_RANK, .i32); - const offs_rope = k.arange(ROPE_OFFSET, ROPE_OFFSET + ROPE_RANK, .i32); + const offs_h = k.arange(0, BLOCK_M, .i32).add(head_ind.mul(@as(i32, @intCast(BLOCK_M)))); const offs_value = k.arange(0, VALUE_RANK, .i32); + const offs_rope: ?Value = if (ROPE_RANK > 0) + k.arange(ROPE_OFFSET, ROPE_OFFSET + ROPE_RANK, .i32) + else + null; const offs_t = k.arange(0, TILE_SIZE, .i32); - const query_pos = q_block_local_idx.add(offs_m.div(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); - - const query_offset_0 = cur_batch_in_all_start_index.add(query_pos); - const query_offset_1 = offs_m.rem(@as(i32, @intCast(NUM_QUERIES_PER_KV))); - - const query_mask_0 = query_pos.lt(cur_batch_query_len); - const query_mask_1 = query_offset_1.lt(@as(i32, @intCast(NUM_QUERY_HEADS))); + const query_offset_0 = k.splat(q_ind, &.{BLOCK_M}); + const query_offset_1 = offs_h; + const head_mask = offs_h.lt(@as(i32, @intCast(NUM_QUERY_HEADS))); const qo0_2d = query_offset_0.expandDims(1).mul(query_stride_0); const qo1_2d = query_offset_1.expandDims(1).mul(query_stride_1); - const q_rope_offset = qo0_2d.add(qo1_2d).add(offs_rope.expandDims(0)); - const q_mask = query_mask_0.expandDims(1).bitAnd(query_mask_1.expandDims(1)); - const Q_rope = k.loadOpts(query_ptr.addPtr(q_rope_offset), .{ - .mask = q_mask, - .other = k.zeros(&.{ BLOCK_M, ROPE_RANK }, config.q_dtype), - .cache_modifier = if (config.all_decode or BLOCK_M >= NUM_QUERY_HEADS) .cg else .none, - }); + const Q_rope: ?Value = if (ROPE_RANK > 0) blk: { + const q_rope_offset = qo0_2d.add(qo1_2d).add(offs_rope.?.expandDims(0)); + break :blk k.loadOpts(query_ptr.addPtr(q_rope_offset), .{ + .mask = head_mask.expandDims(1), + .other = k.zeros(&.{ BLOCK_M, ROPE_RANK }, config.q_dtype), + .cache_modifier = if (config.all_decode or BLOCK_M >= NUM_QUERY_HEADS) .cg else .none, + }); + } else null; - const q_lora_offset = qo0_2d.add(qo1_2d).add(offs_lora.expandDims(0)); - const q_lora_mask = q_mask.bitAnd(offs_lora.lt(@as(i32, @intCast(QK_LORA_RANK))).expandDims(0)); - const Q_lora = k.loadOpts(query_ptr.addPtr(q_lora_offset), .{ - .mask = q_lora_mask, - .other = k.zeros(&.{ BLOCK_M, KV_LORA_RANK }, config.q_dtype), + const q_value_offset = qo0_2d.add(qo1_2d).add(offs_value.expandDims(0)); + const Q_value = k.loadOpts(query_ptr.addPtr(q_value_offset), .{ + .mask = head_mask.expandDims(1), + .other = k.zeros(&.{ BLOCK_M, VALUE_RANK }, config.q_dtype), .cache_modifier = if (config.all_decode or BLOCK_M >= NUM_QUERY_HEADS) .cg else .none, }); @@ -363,7 +255,7 @@ fn kernelUnifiedAttentionSparseMla( .mask = valid_t, .other = k.zeros(&.{TILE_SIZE}, .i32), }); - valid_t = valid_t.bitAnd(topk_pos.ne(-1)); + valid_t = valid_t.bitAnd(topk_pos.ge(0)); const physical_block_idx = topk_pos.div(@as(i32, @intCast(BLOCK_SIZE))); const slot = topk_pos.rem(@as(i32, @intCast(BLOCK_SIZE))); @@ -371,34 +263,34 @@ fn kernelUnifiedAttentionSparseMla( var S = k.zeros(&.{ BLOCK_M, TILE_SIZE }, .f32); const physical_block_idx_t = physical_block_idx.expandDims(0); - const key_block_offsets = physical_block_idx_t.to(.i64).mul(stride_k_cache_0); - const key_block_ptrs = key_cache_ptr.addPtr(key_block_offsets); - const k_rope_dim_offsets = offs_rope.expandDims(1).mul(@as(i32, @intCast(config.stride_k_cache_3))); - const k_rope_dim_ptrs = k.broadcastTo(key_block_ptrs, &.{ ROPE_RANK, TILE_SIZE }) - .addPtr(k.broadcastTo(k_rope_dim_offsets, &.{ ROPE_RANK, TILE_SIZE })); + const cache_block_offsets = physical_block_idx_t.to(.i64).mul(stride_cache_0); + const cache_block_ptrs = kv_cache_ptr.addPtr(cache_block_offsets); const slot_t = slot.expandDims(0); - const key_slot_offsets = slot_t.to(.i64).mul(stride_k_cache_1); - const K_rope = k.loadOpts(k_rope_dim_ptrs.addPtr(k.broadcastTo(key_slot_offsets, &.{ ROPE_RANK, TILE_SIZE })), .{ + const cache_slot_offsets = slot_t.to(.i64).mul(stride_cache_1); + if (ROPE_RANK > 0) { + const k_rope_dim_offsets = offs_rope.?.expandDims(1).mul(@as(i32, @intCast(config.stride_cache_dim))); + const k_rope_dim_ptrs = k.broadcastTo(cache_block_ptrs, &.{ ROPE_RANK, TILE_SIZE }) + .addPtr(k.broadcastTo(k_rope_dim_offsets, &.{ ROPE_RANK, TILE_SIZE })); + const K_rope = k.loadOpts(k_rope_dim_ptrs.addPtr(k.broadcastTo(cache_slot_offsets, &.{ ROPE_RANK, TILE_SIZE })), .{ + .mask = valid_t.expandDims(0), + .other = k.zeros(&.{ ROPE_RANK, TILE_SIZE }, config.kv_dtype), + .cache_modifier = if (config.all_decode) .cg else .none, + }); + S = S.add(scale.mul(k.dot(Q_rope.?, K_rope, k.zeros(&.{ BLOCK_M, TILE_SIZE }, .f32)))); + } + + const k_value_dim_offsets = offs_value.expandDims(1).mul(@as(i32, @intCast(config.stride_cache_dim))); + const k_value_dim_ptrs = k.broadcastTo(cache_block_ptrs, &.{ VALUE_RANK, TILE_SIZE }) + .addPtr(k.broadcastTo(k_value_dim_offsets, &.{ VALUE_RANK, TILE_SIZE })); + const K_value = k.loadOpts(k_value_dim_ptrs.addPtr(k.broadcastTo(cache_slot_offsets, &.{ VALUE_RANK, TILE_SIZE })), .{ .mask = valid_t.expandDims(0), - .other = k.zeros(&.{ ROPE_RANK, TILE_SIZE }, config.kv_dtype), - .cache_modifier = if (config.all_decode) .cg else .none, - }); - S = S.add(scale.mul(k.dot(Q_rope, K_rope, k.zeros(&.{ BLOCK_M, TILE_SIZE }, .f32)))); - - const k_lora_dim_offsets = offs_lora.expandDims(1).mul(@as(i32, @intCast(config.stride_k_cache_3))); - const k_lora_dim_ptrs = k.broadcastTo(key_block_ptrs, &.{ KV_LORA_RANK, TILE_SIZE }) - .addPtr(k.broadcastTo(k_lora_dim_offsets, &.{ KV_LORA_RANK, TILE_SIZE })); - const k_lora_mask = valid_t.expandDims(0).bitAnd(offs_lora.lt(@as(i32, @intCast(QK_LORA_RANK))).expandDims(1)); - const K_lora = k.loadOpts(k_lora_dim_ptrs.addPtr(k.broadcastTo(key_slot_offsets, &.{ KV_LORA_RANK, TILE_SIZE })), .{ - .mask = k_lora_mask, - .other = k.zeros(&.{ KV_LORA_RANK, TILE_SIZE }, config.kv_dtype), + .other = k.zeros(&.{ VALUE_RANK, TILE_SIZE }, config.kv_dtype), .cache_modifier = if (config.all_decode) .cg else .none, }); - S = S.add(scale.mul(k.dot(Q_lora, K_lora, k.zeros(&.{ BLOCK_M, TILE_SIZE }, .f32)))); + S = S.add(scale.mul(k.dot(Q_value, K_value, k.zeros(&.{ BLOCK_M, TILE_SIZE }, .f32)))); - const keep_mask = query_mask_1.expandDims(1) - .bitAnd(query_mask_0.expandDims(1)) + const keep_mask = head_mask.expandDims(1) .bitAnd(valid_t.expandDims(0)); S = k.where(keep_mask, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); @@ -411,21 +303,8 @@ fn kernelUnifiedAttentionSparseMla( const acc_scaled = acc.mul(alpha.expandDims(1)); const new_L = L.mul(alpha).add(l_j); - const physical_block_idx_v = physical_block_idx.expandDims(1).to(.i64); - const v_block_offsets = physical_block_idx_v.mul(stride_v_cache_0); - const v_base_ptr = k.splat(value_cache_ptr, &.{ TILE_SIZE, 1 }); - const slot_v = slot.expandDims(1).to(.i64); - const v_slot_offsets = slot_v.mul(stride_v_cache_1); - const v_base_offsets = v_block_offsets.add(v_slot_offsets); - const v_base_ptrs = v_base_ptr.addPtr(v_base_offsets); - const v_lora_dim_offsets = offs_value.expandDims(0).mul(@as(i32, @intCast(config.stride_v_cache_3))); - const V_lora = k.loadOpts(k.broadcastTo(v_base_ptrs, &.{ TILE_SIZE, VALUE_RANK }).addPtr(k.broadcastTo(v_lora_dim_offsets, &.{ TILE_SIZE, VALUE_RANK })), .{ - .mask = valid_t.expandDims(1), - .other = k.zeros(&.{ TILE_SIZE, VALUE_RANK }, config.kv_dtype), - .cache_modifier = if (config.all_decode) .cg else .none, - }); - - const new_acc = k.dot(P.to(config.kv_dtype), V_lora, acc_scaled); + const V = k.trans(K_value, &.{ 1, 0 }); + const new_acc = k.dot(P.to(config.kv_dtype), V, acc_scaled); loop.yield(.{ m_j, new_L, new_acc }); } @@ -435,13 +314,12 @@ fn kernelUnifiedAttentionSparseMla( var acc = loop.results[2]; if (!three_d and config.use_attn_sink) { - const sink_mask = query_mask_0.bitAnd(query_mask_1); const sink_logits = k.loadOpts(attn_sink_ptr.addPtr(query_offset_1), .{ - .mask = sink_mask, + .mask = head_mask, .other = k.zeros(&.{BLOCK_M}, config.sink_dtype), }).to(.f32); const sink_score = k.where( - sink_mask, + head_mask, sink_logits, k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32), ); @@ -469,18 +347,18 @@ fn kernelUnifiedAttentionSparseMla( const partial_out_token_stride: i64 = NUM_QUERY_HEADS * NUM_SPLITS * VALUE_RANK; const partial_out_head_stride: i64 = NUM_SPLITS * VALUE_RANK; const partial_out_offset = q_ind.to(.i64).mul(partial_out_token_stride) - .add(offs_m.expandDims(1).to(.i64).mul(partial_out_head_stride)) + .add(offs_h.expandDims(1).to(.i64).mul(partial_out_head_stride)) .add(split_idx.to(.i64).mul(VALUE_RANK)) .add(offs_value.expandDims(0).to(.i64)); k.storeOpts( output_ptr.addPtr(partial_out_offset), acc, - .{ .mask = q_mask }, + .{ .mask = head_mask.expandDims(1) }, ); const partial_lse_token_stride: i64 = NUM_QUERY_HEADS * NUM_SPLITS; const partial_lse_offset = q_ind.to(.i64).mul(partial_lse_token_stride) - .add(offs_m.to(.i64).mul(NUM_SPLITS)) + .add(offs_h.to(.i64).mul(NUM_SPLITS)) .add(split_idx.to(.i64)); const partial_lse = k.where( has_value, @@ -490,21 +368,110 @@ fn kernelUnifiedAttentionSparseMla( k.storeOpts( partial_lse_ptr.?.addPtr(partial_lse_offset), partial_lse, - .{ .mask = query_mask_0.bitAnd(query_mask_1) }, + .{ .mask = head_mask }, ); return; } - const output_offs_lora = query_offset_0.expandDims(1).mul(output_stride_0) + const output_offsets = query_offset_0.expandDims(1).mul(output_stride_0) .add(query_offset_1.expandDims(1).mul(output_stride_1)) .add(offs_value.expandDims(0)); k.storeOpts( - output_ptr.addPtr(output_offs_lora), + output_ptr.addPtr(output_offsets), acc.to(config.o_dtype), - .{ .mask = q_mask }, + .{ .mask = head_mask.expandDims(1) }, ); } +const Reduce3DPartialsCfg = struct { + num_query_heads: i64, + value_rank: i64, + num_input_splits: i64, + num_output_splits: i64, + all_decode: bool, +}; + +/// Reduce a large split dimension in fixed-size groups before the final +/// attention-sink reduction. Keeping the local reduction width small avoids +/// register spilling when sparse decode uses enough splits to fill a MI300X. +pub const Reduce3DPartials = tri.Kernel(Reduce3DPartialsCfg, .{ + .name = "_kernel_reduce_sparse_mla_3d_partials_ptr", + .inputs = &.{ "input", "input_lse", "active_query_count_ptr" }, + .outputs = &.{ "partial_output", "partial_lse" }, + .run = runReduce3DPartials, +}); + +fn runReduce3DPartials(b: *tri.Builder, cfg: Reduce3DPartialsCfg) tri.FinishError!void { + const a = try b.declareArgs(.{ + .input_ptr = .{ .ptr = .f32 }, + .input_lse_ptr = .{ .ptr = .f32 }, + .active_query_count_ptr = .{ .ptr = .i32 }, + .partial_output_ptr = .{ .ptr = .f32 }, + .partial_lse_ptr = .{ .ptr = .f32 }, + }); + + const query_idx = b.programId(.x); + if (!cfg.all_decode) { + const active_query_count = b.load(a.active_query_count_ptr); + b.returnIf(query_idx.ge(active_query_count), .{}); + } + const head_group_idx = b.programId(.y); + const head_idx = head_group_idx.div(@as(i32, @intCast(cfg.num_output_splits))); + const group_idx = head_group_idx.rem(@as(i32, @intCast(cfg.num_output_splits))); + const splits_per_group = @divExact(cfg.num_input_splits, cfg.num_output_splits); + const offs_s = b.arange(0, splits_per_group, .i32); + const offs_d = b.arange(0, cfg.value_rank, .i32); + const group_start = group_idx.to(.i64).mul(splits_per_group); + + const input_lse_token_stride = cfg.num_query_heads * cfg.num_input_splits; + const input_lse_offset = query_idx.to(.i64).mul(input_lse_token_stride) + .add(head_idx.to(.i64).mul(cfg.num_input_splits)) + .add(group_start) + .add(offs_s.to(.i64)); + const input_lse = b.load(a.input_lse_ptr.addPtr(input_lse_offset)); + const overall_max = b.max(input_lse); + const has_value = overall_max.gt(-std.math.inf(f32)); + const safe_max = b.where(has_value, overall_max, b.liftAs(0.0, .f32)); + const split_weights = b.exp(input_lse.sub(safe_max)); + const denominator = b.sumOpts(split_weights, .{ .axis = 0 }); + + const input_output_token_stride = cfg.num_query_heads * cfg.num_input_splits * cfg.value_rank; + const input_output_head_stride = cfg.num_input_splits * cfg.value_rank; + const input_output_base = query_idx.to(.i64).mul(input_output_token_stride) + .add(head_idx.to(.i64).mul(input_output_head_stride)) + .add(group_start.mul(cfg.value_rank)); + const input_output_offset = b.broadcastTo( + input_output_base.add(offs_s.expandDims(1).to(.i64).mul(cfg.value_rank)), + &.{ splits_per_group, cfg.value_rank }, + ).add(b.broadcastTo(offs_d.expandDims(0).to(.i64), &.{ splits_per_group, cfg.value_rank })); + const input_output = b.load(a.input_ptr.addPtr(input_output_offset)); + const numerator = b.sumOpts(input_output.mul(split_weights.expandDims(1)), .{ .axis = 0 }); + const safe_denominator = b.where(denominator.gt(0.0), denominator, b.liftAs(1.0, .f32)); + const output = b.where( + has_value, + numerator.div(safe_denominator), + b.zeros(&.{cfg.value_rank}, .f32), + ); + + const output_token_stride = cfg.num_query_heads * cfg.num_output_splits * cfg.value_rank; + const output_head_stride = cfg.num_output_splits * cfg.value_rank; + const output_offset = query_idx.to(.i64).mul(output_token_stride) + .add(head_idx.to(.i64).mul(output_head_stride)) + .add(group_idx.to(.i64).mul(cfg.value_rank)) + .add(offs_d.to(.i64)); + b.store(a.partial_output_ptr.addPtr(output_offset), output); + + const output_lse_offset = query_idx.to(.i64).mul(cfg.num_query_heads * cfg.num_output_splits) + .add(head_idx.to(.i64).mul(cfg.num_output_splits)) + .add(group_idx.to(.i64)); + const output_lse = b.where( + has_value, + safe_max.add(b.log(safe_denominator)), + b.liftAs(-std.math.inf(f32), .f32), + ); + b.store(a.partial_lse_ptr.addPtr(output_lse_offset), output_lse); +} + const Reduce3DCfg = struct { sink_dtype: DType = .f32, o_dtype: DType = .bf16, @@ -512,6 +479,7 @@ const Reduce3DCfg = struct { value_rank: i64 = 512, num_splits: i64 = 1, use_attn_sink: bool = false, + all_decode: bool = false, }; pub const Reduce3D = tri.Kernel(Reduce3DCfg, .{ @@ -520,6 +488,7 @@ pub const Reduce3D = tri.Kernel(Reduce3DCfg, .{ "partial_output_ptr", "partial_lse_ptr", "attn_sink_ptr", + "active_query_count_ptr", "output_stride_0_ptr", "output_stride_1_ptr", }, @@ -532,6 +501,7 @@ fn runReduce3D(b: *tri.Builder, cfg: Reduce3DCfg) tri.FinishError!void { .partial_output_ptr = .{ .ptr = .f32 }, .partial_lse_ptr = .{ .ptr = .f32 }, .attn_sink_ptr = .{ .ptr = cfg.sink_dtype }, + .active_query_count_ptr = .{ .ptr = .i32 }, .output_stride_0_ptr = .{ .ptr = .i64 }, .output_stride_1_ptr = .{ .ptr = .i64 }, .output_ptr = .{ .ptr = cfg.o_dtype }, @@ -541,6 +511,10 @@ fn runReduce3D(b: *tri.Builder, cfg: Reduce3DCfg) tri.FinishError!void { const output_stride_1 = b.load(a.output_stride_1_ptr); const query_idx = b.programId(.x); const head_idx = b.programId(.y); + if (!cfg.all_decode) { + const active_query_count = b.load(a.active_query_count_ptr); + b.returnIf(query_idx.ge(active_query_count), .{}); + } const offs_s = b.arange(0, cfg.num_splits, .i32); const offs_d = b.arange(0, cfg.value_rank, .i32); @@ -589,8 +563,5 @@ fn runReduce3D(b: *tri.Builder, cfg: Reduce3DCfg) tri.FinishError!void { const output_offset = query_idx.to(.i64).mul(output_stride_0) .add(head_idx.to(.i64).mul(output_stride_1)) .add(offs_d.to(.i64)); - b.store( - a.output_ptr.addPtr(output_offset), - output.to(cfg.o_dtype), - ); + b.store(a.output_ptr.addPtr(output_offset), output.to(cfg.o_dtype)); } From 54c0f16d227ed1f4dbf04b0325949d6166459020 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 28 Aug 2026 18:18:46 +0200 Subject: [PATCH 639/881] zml/attention/triton_attention: use specific block_m on GB300 (#742) Investigation has been done with the help of a clanker. The problem might manifest on other blackwell hardware but for the time being, simply gate to GB300. @elogir If you can add some details --- zml/attention/triton_attention.zig | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/zml/attention/triton_attention.zig b/zml/attention/triton_attention.zig index 62c08ec3..e7878297 100644 --- a/zml/attention/triton_attention.zig +++ b/zml/attention/triton_attention.zig @@ -132,9 +132,30 @@ fn sparseMlaConfig(paged_opts: paged.PagedSparseMlaOptions, topk_count: usize, c return config; } +/// Largest `BLOCK_M * HEAD_SIZE_PADDED` the 2D kernel survives. That product is +/// the shape of both the query tile and the f32 accumulator carried through the +/// tile loop; above it the kernel faults with CUDA_ERROR_MISALIGNED_ADDRESS. +/// +/// Measured on a GB300 with Gemma-4-31B, whose global-attention layers are +/// head_dim 512 (its sliding ones are 256): +/// +/// 64x256, 32x512, 16x512 ok 64x512, 128x512 fault +/// +/// The bound is on this product alone. Two things that do *not* move it, so do +/// not reach for them if a new head size faults: TILE_SIZE, which at 64 pushes +/// total shared memory to ~160 KB and still passes at BLOCK_M 32 while 64x512 +/// faults at 98 KB; and num_warps, where 8 halves the per-thread register share +/// and still faults -- which is also why the obvious "the accumulator outgrows +/// the register file" story cannot be the whole explanation. The underlying +/// Triton codegen bug is not diagnosed; this only keeps us out of it. +const max_query_tile_elements: usize = 16 * 1024; + fn select2dConfig(options: paged.PagedAttentionOptions) Config2D { const max_num_stages_2d: usize = if (options.head_dim <= 128) 4 else 2; + // Until we test on other platforms, gate the fix to GB300 + const is_gb300 = isCudaComputeCapability("10.3"); + var num_stages_2d: usize, var num_warps: usize, var tile_size: usize = if (!options.all_decode) .{ 1, 2, 64 } else .{ 3, 2, options.block_size }; var block_m = options.block_m; @@ -146,6 +167,14 @@ fn select2dConfig(options: paged.PagedAttentionOptions) Config2D { } else { block_m = 128; } + + if (is_gb300) { + // Halving BLOCK_M for every doubling of the head keeps that product + // constant: 128 at head_dim 128, 64 at 256, 32 at 512. The first two + // are the values this branch has always picked. + const head_dim_padded = std.math.ceilPowerOfTwoAssert(usize, options.head_dim); + block_m = @min(128, max_query_tile_elements / head_dim_padded); + } num_stages_2d = 1; num_warps = 4; } From fdcdade631a182adcf2a08111fec2eaf6260889e Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Fri, 28 Aug 2026 19:38:39 +0200 Subject: [PATCH 640/881] zml/moe: fix manualComputation scales usage (#743) --- zml/moe/moe.zig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index 5c0c3883..752b0953 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -432,8 +432,6 @@ pub fn forwardMoe( topk_weights, weights_gate_up, weights_down, - scales_gate_up, - scales_down, }, input.shape(), .{ @@ -443,6 +441,8 @@ pub fn forwardMoe( .bias_down = bias_down, .quant_scheme = opts.quant_scheme, .activation_threshold = opts.activation_threshold, + .scales_gate_up = scales_gate_up, + .scales_down = scales_down, }, (struct { fn body(ctx: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, _: zml.Shape) zml.Tensor { @@ -471,8 +471,8 @@ pub fn forwardMoe( .activation = ctx.activation, .global_num_experts = ctx.global_num_experts, .expert_map = expert_map, - .w1_scale = sharded_inputs[5], - .w2_scale = sharded_inputs[6], + .w1_scale = ctx.scales_gate_up, + .w2_scale = ctx.scales_down, .w1_bias = ctx.bias_gate_up, .w2_bias = ctx.bias_down, .quant_scheme = ctx.quant_scheme, From fcb0d2a9c163eb28a6ccc13490f0797d614da850 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 31 Aug 2026 13:19:52 +0200 Subject: [PATCH 641/881] zml.Compiler: cleanup (#738) * use multiArrayList in `emitMlirFunc` to store information about input/output tensors. * merge OutputInfo and InputInfo: The code is collecting same info for input/output, some code was merge. `emitMlirFunc` is now clearly split between the function specific wrapper, and the part that works only with slices of tensor info. * zml.Sharding: clarify error sources, added more error messages. Most `zml.Sharding` function can fail because they resolve the sharding on the fly. Changed some APIs to explicitly take a resolved sharding, leaving only OOM errors to handle. I've put a TODO for you @hugomano about GSPMD sharding that can still fails pretty late when generating the attribute. I feel this should be checked when constructing the Partitionning. * add Tensor.Id instead of old `usize` this allows accidental confusion or operation on type id. * improve Compiler.Scope API: `pushBlock` now returns the `scope` this remove the old pattern of `pushBlock(); const scope = currentScope();` --- zml/Compiler.zig | 416 +++++++++++++---------------- zml/Sharding.zig | 149 ++++++----- zml/attention/triton_attention.zig | 4 +- zml/io.zig | 6 +- zml/nn.zig | 8 +- zml/ops.zig | 294 ++++++++++---------- zml/tensor.zig | 82 +++--- 7 files changed, 446 insertions(+), 513 deletions(-) diff --git a/zml/Compiler.zig b/zml/Compiler.zig index e973f75c..08547fcf 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -87,30 +87,36 @@ pub const Error = std.mem.Allocator.Error || pjrtx.Client.CompileError || error{MissingDeviceInTile}; -const AttributeList = stdx.BoundedArray(mlir.NamedAttribute, 3); - pub const Scope = struct { + compiler: *Compiler, block: *mlir.Block, - id_to_argument: std.AutoArrayHashMapUnmanaged(usize, usize), - id_to_donation: std.AutoArrayHashMapUnmanaged(usize, usize), - id_to_output_memory_kind: std.AutoArrayHashMapUnmanaged(usize, Memory.Kind), - id_to_input_memory_kind: std.AutoArrayHashMapUnmanaged(usize, Memory.Kind), + id_to_argument: std.AutoArrayHashMapUnmanaged(Tensor.Id, *const mlir.Value), + id_to_donation: std.AutoArrayHashMapUnmanaged(Tensor.Id, usize), + id_to_memory: std.AutoArrayHashMapUnmanaged(Tensor.Id, Memory.Kind), arena: std.heap.ArenaAllocator, - pub fn initFromBlock(allocator: std.mem.Allocator, block: *mlir.Block) Scope { - const arena: std.heap.ArenaAllocator = .init(allocator); + pub fn initFromBlock(compiler: *Compiler, block: *mlir.Block) Scope { + const arena: std.heap.ArenaAllocator = .init(compiler.allocator); return .{ + .compiler = compiler, .block = block, .id_to_argument = .empty, .id_to_donation = .empty, - .id_to_output_memory_kind = .empty, - .id_to_input_memory_kind = .empty, + .id_to_memory = .empty, .arena = arena, }; } - pub fn deinit(self: *Scope) void { - self.arena.deinit(); + pub fn pop(self: *Scope) void { + const compiler = self.compiler; + std.debug.assert(self == compiler.currentScope()); + var arena = self.arena; + _ = compiler.scopes.pop(); + arena.deinit(); + } + + pub fn registerTensorAsBlockArgument(scope: *Scope, id: Tensor.Id, arg_id: usize) void { + scope.id_to_argument.put(scope.arena.allocator(), id, scope.block.argument(arg_id)) catch @panic("OOM"); } }; @@ -170,9 +176,7 @@ pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, pub fn deinit(self: *Compiler) void { if (_current == self) _current = null; - for (self.scopes.slice()) |*scope| { - scope.deinit(); - } + std.debug.assert(self.scopes.len == 0); self.mlir_pass_manager.deinit(); self.module.deinit(); self.mlir_ctx.deinit(); @@ -201,16 +205,10 @@ pub fn currentScope(self: *Compiler) *Scope { return &self.scopes.slice()[self.scopes.len - 1]; } -pub fn pushBlock(self: *Compiler, block: *mlir.Block) void { - const scope = Scope.initFromBlock(self.allocator, block); +pub fn pushBlock(self: *Compiler, block: *mlir.Block) *Scope { + const scope = Scope.initFromBlock(self, block); self.scopes.appendAssumeCapacity(scope); -} - -pub fn popBlock(self: *Compiler) void { - var maybe_popped_scope = self.scopes.pop(); - if (maybe_popped_scope) |*popped| { - popped.deinit(); - } + return self.currentScope(); } pub fn nextChannelId(self: *Compiler) i64 { @@ -292,7 +290,7 @@ pub fn compile( var compiler: Compiler = .init(allocator, st_io.io(), platform, opts); defer compiler.deinit(); - const result = emitMlir(&compiler, func, args) catch |err| switch (err) { + var result = emitMlir(&compiler, func, args) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => unreachable, }; @@ -335,10 +333,11 @@ pub fn compile( loaded_executable, @intCast(num_devices), num_partitions, - result.input_info.shapes, - result.output_info.shapes, - result.input_info.shardings, - result.output_info.shardings, + // This will get copied into exe + result.input_info.items(.shape), + result.output_info.items(.shape), + result.input_info.items(.sharding), + result.output_info.items(.sharding), ); errdefer exe.deinit(); @@ -376,244 +375,193 @@ fn addPartitionerOperations(ctx: *Compiler) !void { } } -pub const OutputInfo = struct { - shapes: []Shape, - shardings: []Sharding, - values: []*const mlir.Value, - donations: []?usize, - output_memory_kinds: []Memory.Kind, - - pub fn deinit(self: OutputInfo, allocator: std.mem.Allocator) void { - allocator.free(self.shapes); - allocator.free(self.shardings); - allocator.free(self.values); - allocator.free(self.donations); - allocator.free(self.output_memory_kinds); - } -}; - -fn collectOutputInfo(allocator: std.mem.Allocator, partitioning: Sharding.Partitioning, v: anytype) !OutputInfo { - const LocalContext = struct { - shape_list: *std.array_list.Managed(Shape), - sharding_list: *std.array_list.Managed(Sharding), - value_list: *std.array_list.Managed(*const mlir.Value), - donation_list: *std.array_list.Managed(?usize), - output_memory_kind_list: *std.array_list.Managed(Memory.Kind), - partitioning: Sharding.Partitioning, - }; - - var shape_list = std.array_list.Managed(Shape).init(allocator); - errdefer shape_list.deinit(); - var sharding_list = std.array_list.Managed(Sharding).init(allocator); - errdefer sharding_list.deinit(); - var value_list = std.array_list.Managed(*const mlir.Value).init(allocator); - errdefer value_list.deinit(); - var donation_list = std.array_list.Managed(?usize).init(allocator); - errdefer donation_list.deinit(); - var output_memory_kind_list = std.array_list.Managed(Memory.Kind).init(allocator); - errdefer output_memory_kind_list.deinit(); - - var context: LocalContext = .{ - .shape_list = &shape_list, - .sharding_list = &sharding_list, - .value_list = &value_list, - .donation_list = &donation_list, - .output_memory_kind_list = &output_memory_kind_list, - .partitioning = partitioning, - }; - - try meta.visit(struct { - fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { - try ctx_.shape_list.append(tensor.shape()); - try ctx_.sharding_list.append(try ctx_.partitioning.selectSharding(tensor.shape())); - try ctx_.value_list.append(tensor.value()); - try ctx_.donation_list.append(tensor.donation()); - try ctx_.output_memory_kind_list.append(tensor.outputMemoryKind()); - } - }.cb, &context, v); - - return .{ - .shapes = try shape_list.toOwnedSlice(), - .shardings = try sharding_list.toOwnedSlice(), - .values = try value_list.toOwnedSlice(), - .donations = try donation_list.toOwnedSlice(), - .output_memory_kinds = try output_memory_kind_list.toOwnedSlice(), - }; -} - -pub const InputInfo = struct { - shapes: []Shape, - shardings: []Sharding, - memory_kinds: []?Memory.Kind, - - pub fn deinit(self: InputInfo, allocator: std.mem.Allocator) void { - allocator.free(self.shapes); - allocator.free(self.shardings); - allocator.free(self.memory_kinds); - } -}; - -fn collectInputInfo(allocator: std.mem.Allocator, partitioning: Sharding.Partitioning, v: anytype) !InputInfo { - const LocalContext = struct { - shape_list: *std.array_list.Managed(Shape), - sharding_list: *std.array_list.Managed(Sharding), - memory_kind_list: *std.array_list.Managed(?Memory.Kind), - partitioning: Sharding.Partitioning, - }; - - var shape_list = std.array_list.Managed(Shape).init(allocator); - errdefer shape_list.deinit(); - - var sharding_list = std.array_list.Managed(Sharding).init(allocator); - errdefer sharding_list.deinit(); - - var memory_kind_list = std.array_list.Managed(?Memory.Kind).init(allocator); - errdefer memory_kind_list.deinit(); - - var context: LocalContext = .{ - .shape_list = &shape_list, - .sharding_list = &sharding_list, - .memory_kind_list = &memory_kind_list, - .partitioning = partitioning, - }; - - try meta.visit(struct { - fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { - try ctx_.shape_list.append(tensor.shape()); - try ctx_.sharding_list.append(try ctx_.partitioning.selectSharding(tensor.shape())); - try ctx_.memory_kind_list.append(tensor.inputMemoryKind()); - } - }.cb, &context, v); - - return .{ - .shapes = try shape_list.toOwnedSlice(), - .shardings = try sharding_list.toOwnedSlice(), - .memory_kinds = try memory_kind_list.toOwnedSlice(), - }; -} - const EmitMlirResult = struct { func: *mlir.Operation, - input_info: InputInfo, - output_info: OutputInfo, + input_info: std.MultiArrayList(TensorInfo), + output_info: std.MultiArrayList(TensorInfo), }; -fn finalizeAttributeList(allocator_: std.mem.Allocator, mlir_ctx: *mlir.Context, attributes: []AttributeList) ![]*const mlir.Attribute { - const res = try allocator_.alloc(*const mlir.Attribute, attributes.len); - for (res, attributes) |*r, attr| { - r.* = .dict(mlir_ctx, attr.constSlice()); +pub const TensorInfo = struct { + id: Tensor.Id, + shape: Shape, + sharding: Sharding, + value: *const mlir.Value, + + // Only used for input tensors, stores which output tensor ends up with their buffer + aliasing_output: ?u32 = null, + + pub fn attributes(info: TensorInfo, arena: std.mem.Allocator, compiler: *const Compiler, scope: *const Scope) error{OutOfMemory}!AttributeList { + var attrs: AttributeList = .empty; + + const mlir_ctx = compiler.mlir_ctx; + const sharding_attr = try compiler.partitioning.tensorShardingAttr(arena, mlir_ctx, info.shape, info.sharding); + const name = switch (compiler.partitioning.partitioner) { + .gspmd => "mhlo.sharding", + .shardy => "sdy.sharding", + }; + attrs.appendAssumeCapacity(.named(mlir_ctx, name, sharding_attr)); + + const memory = scope.id_to_memory.get(info.id) orelse .device; + if (memory != .device) { + attrs.appendAssumeCapacity(.named( + mlir_ctx, + "mhlo.memory_kind", + .string(mlir_ctx, compiler.platform.memoryKind(memory)), + )); + } + + if (info.aliasing_output) |output_index| { + attrs.appendAssumeCapacity( + .named(mlir_ctx, "tf.aliasing_output", .int(mlir_ctx, .i32, output_index)), + ); + } + + return attrs; } - return res; -} +}; + +const AttributeList = stdx.BoundedArray(mlir.NamedAttribute, 3); fn emitMlir(compiler: *Compiler, comptime func: anytype, args: std.meta.ArgsTuple(@TypeOf(func))) !EmitMlirResult { - var arena = std.heap.ArenaAllocator.init(compiler.allocator); - defer arena.deinit(); - const module = mlir.Module.init(.unknown(compiler.mlir_ctx)); errdefer module.deinit(); const block = mlir.Block.init(&.{}, &.{}); errdefer block.deinit(); - compiler.pushBlock(block); - defer compiler.popBlock(); + const fn_scope: *Scope = compiler.pushBlock(block); + defer fn_scope.pop(); - const LocalContext = struct { - compiler: *Compiler, - current_argument_id: usize = 0, - }; - var context: LocalContext = .{ - .compiler = compiler, - }; - meta.visit(struct { - fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { - const mlir_type = mlirx.Type.rankedTensor(ctx_.compiler.mlir_ctx, tensor.shape()); - _ = ctx_.compiler.currentScope().block.addArgument(mlir_type, .unknown(ctx_.compiler.mlir_ctx)); - const gop = ctx_.compiler.currentScope().id_to_argument.getOrPut(ctx_.compiler.currentScope().arena.allocator(), tensor.id) catch unreachable; - if (gop.found_existing) std.debug.panic("Tensor with id {} has already been used once as an argument", .{tensor.id}); - gop.value_ptr.* = ctx_.current_argument_id; - ctx_.current_argument_id += 1; - } - }.cb, &context, &args); + var input_info = try createBlockArguments(compiler, fn_scope, &args); + errdefer input_info.deinit(compiler.allocator); - const output_info, const input_info = b: { + var result = result: { compiler.activate(); defer compiler.deactivate(); - const result = @call(.auto, func, args); - - const input_info = try collectInputInfo(compiler.allocator, compiler.partitioning, &args); - errdefer input_info.deinit(compiler.allocator); - - const output_info = try collectOutputInfo(compiler.allocator, compiler.partitioning, &result); - errdefer output_info.deinit(compiler.allocator); - - break :b .{ output_info, input_info }; + break :result @call(.auto, func, args); }; - errdefer input_info.deinit(compiler.allocator); + + var output_info = try collectOutputInfo(compiler, fn_scope, &result); errdefer output_info.deinit(compiler.allocator); - const input_attributes = try arena.allocator().alloc(AttributeList, input_info.shapes.len); - @memset(input_attributes, .empty); + // Finalize in a separate function that doesn't depend on `func` type. + return try finalizeMlirFunc(compiler, fn_scope, input_info, output_info); +} - const output_attributes = try arena.allocator().alloc(AttributeList, output_info.shapes.len); - @memset(output_attributes, .empty); +fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{OutOfMemory}!std.MultiArrayList(TensorInfo) { + const CreateBlockArgumentsCb = struct { + compiler: *Compiler, + scope: *Scope, - for (output_info.donations, 0..) |donation, index| if (donation) |argument_index| { - input_attributes[argument_index].appendAssumeCapacity(.named(compiler.mlir_ctx, "tf.aliasing_output", .int(compiler.mlir_ctx, .i32, index))); - }; - for (output_info.output_memory_kinds, 0..) |output_memory_kind, index| { - if (output_memory_kind == .device) continue; - output_attributes[index].appendAssumeCapacity(.named( - compiler.mlir_ctx, - "mhlo.memory_kind", - .string( - compiler.mlir_ctx, - compiler.platform.memoryKind(output_memory_kind), - ), - )); - } - _ = dialects.func.returns(compiler.mlir_ctx, output_info.values, .unknown(compiler.mlir_ctx)).appendTo(compiler.currentScope().block); + current_argument_id: u32 = 0, + infos: std.MultiArrayList(TensorInfo) = .empty, - for (input_info.shapes, input_info.shardings, input_info.memory_kinds, 0..) |shape, sharding, maybe_memory_kind, i| { - const attr = try compiler.partitioning.tensorShardingAttr(compiler.arena.allocator(), compiler.mlir_ctx, shape, sharding); - const name = switch (compiler.partitioning.partitioner) { - .gspmd => "mhlo.sharding", - .shardy => "sdy.sharding", - }; + fn cb(ctx: *@This(), tensor: *const Tensor) error{OutOfMemory}!void { + const mlir_type = mlirx.Type.rankedTensor(ctx.compiler.mlir_ctx, tensor._shape); - input_attributes[i].appendAssumeCapacity(.named(compiler.mlir_ctx, name, attr)); + const value = ctx.scope.block.addArgument(mlir_type, .unknown(ctx.compiler.mlir_ctx)); + const gop = ctx.scope.id_to_argument.getOrPutValue(ctx.scope.arena.allocator(), tensor.id, value) catch @panic("OOM"); + if (gop.found_existing) std.debug.panic("Tensor with id {} has already been used once as an argument", .{tensor.id}); - if (maybe_memory_kind) |memory_kind| { - if (memory_kind == .device) continue; - input_attributes[i].appendAssumeCapacity(.named( - compiler.mlir_ctx, - "mhlo.memory_kind", - .string( - compiler.mlir_ctx, - compiler.platform.memoryKind(memory_kind), + // Associate each input argument with their own buffer + ctx.scope.id_to_donation.putNoClobber(ctx.scope.arena.allocator(), tensor.id, ctx.current_argument_id) catch @panic("OOM"); + + defer ctx.current_argument_id += 1; + + const input_sharding = ctx.compiler.partitioning.selectSharding(tensor._shape) catch |err| switch (err) { + error.NoSuitableSharding => std.debug.panic( + "Failed to resolve sharding for input {f}({d}) because it's using unknown sharding. Pass more shardings to `platform.compile`. Known shardings: {f}", + .{ tensor._shape, ctx.current_argument_id, stdx.fmt.slice(ctx.compiler.partitioning.shardings) }, ), - )); + }; + try ctx.infos.append(ctx.compiler.allocator, .{ + .id = tensor.id, + .shape = tensor._shape, + .sharding = input_sharding, + .value = value, + }); + } + }; + + var context: CreateBlockArgumentsCb = .{ .compiler = compiler, .scope = scope }; + errdefer context.infos.deinit(compiler.allocator); + + try meta.visit(CreateBlockArgumentsCb.cb, &context, v); + + return context.infos; +} + +fn collectOutputInfo(compiler: *Compiler, scope: *Scope, v: anytype) error{OutOfMemory}!std.MultiArrayList(TensorInfo) { + const CollectOutputInfoCb = struct { + compiler: *Compiler, + scope: *Scope, + + infos: std.MultiArrayList(TensorInfo) = .empty, + + fn cb(ctx: *@This(), tensor: *const Tensor) !void { + const value = ctx.scope.id_to_argument.get(tensor.id) orelse + tensor._value orelse + @panic("no value found for output tensor"); + + try ctx.infos.append(ctx.compiler.allocator, .{ + .id = tensor.id, + .shape = tensor._shape, + // Note: the panic should have been triggered during emitMlir or collectInputInfo + .sharding = ctx.compiler.partitioning.selectSharding(tensor._shape) catch @panic("failed to resolve output sharding"), + .value = value, + }); + } + }; + + var context: CollectOutputInfoCb = .{ .compiler = compiler, .scope = scope }; + errdefer context.infos.deinit(compiler.allocator); + + try meta.visit(CollectOutputInfoCb.cb, &context, v); + return context.infos; +} + +fn finalizeMlirFunc(compiler: *Compiler, fn_scope: *Scope, input_info: std.MultiArrayList(TensorInfo), output_info: std.MultiArrayList(TensorInfo)) error{OutOfMemory}!EmitMlirResult { + var arena_state = std.heap.ArenaAllocator.init(compiler.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + const mlir_ctx = compiler.mlir_ctx; + + const fn_return = dialects.func.returns(mlir_ctx, output_info.items(.value), .unknown(mlir_ctx)); + _ = fn_return.appendTo(fn_scope.block); + + // Resolve donations + for (output_info.items(.id), 0..) |output_id, output_index| { + if (fn_scope.id_to_donation.get(output_id)) |donated_input| { + const aliasing_output: *?u32 = &input_info.items(.aliasing_output)[donated_input]; + if (aliasing_output.*) |previous_aliased_output| { + std.debug.panic("Input {d} buffer {} was reused twice with `reuseBuffer` for output {} and output {}. Expected `reuseBuffer` to be called at most once", .{ donated_input, input_info.items(.shape)[donated_input], previous_aliased_output, output_index }); + } + + aliasing_output.* = @intCast(output_index); } } - for (output_info.shapes, output_info.shardings, 0..) |shape, sharding, i| { - const attr = try compiler.partitioning.tensorShardingAttr(compiler.arena.allocator(), compiler.mlir_ctx, shape, sharding); - const name = switch (compiler.partitioning.partitioner) { - .gspmd => "mhlo.sharding", - .shardy => "sdy.sharding", - }; - - output_attributes[i].appendAssumeCapacity(.named(compiler.mlir_ctx, name, attr)); + // Input sharding/memory/aliasing attributes + const input_attributes = try arena.alloc(*const mlir.Attribute, input_info.len); + for (0.., input_attributes) |i, *input_attrs| { + const attrs_list = try input_info.get(i).attributes(arena, compiler, fn_scope); + input_attrs.* = .dict(mlir_ctx, attrs_list.constSlice()); } - const mlir_func = dialects.func.func(compiler.mlir_ctx, .{ + // Output sharding/memory attributes + const output_attributes = try arena.alloc(*const mlir.Attribute, output_info.len); + for (0.., output_attributes) |i, *output_attrs| { + const attrs_list = try output_info.get(i).attributes(arena, compiler, fn_scope); + output_attrs.* = .dict(mlir_ctx, attrs_list.constSlice()); + } + + const mlir_func = dialects.func.func(mlir_ctx, .{ .name = "main", - .block = compiler.currentScope().block, - .location = .unknown(compiler.mlir_ctx), - .args_attributes = try finalizeAttributeList(arena.allocator(), compiler.mlir_ctx, input_attributes), - .results_attributes = try finalizeAttributeList(arena.allocator(), compiler.mlir_ctx, output_attributes), + .block = fn_scope.block, + .location = .unknown(mlir_ctx), + .args_attributes = input_attributes, + .results_attributes = output_attributes, .verify = false, }); diff --git a/zml/Sharding.zig b/zml/Sharding.zig index 58684561..b510a9c4 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -99,11 +99,21 @@ pub const Partitioning = struct { }; } - pub fn tensorShardingAttr(self: Partitioning, allocator: std.mem.Allocator, ctx: *mlir.Context, shape: Shape, sharding: ?Sharding) !*const mlir.Attribute { - const selected_sharding = sharding orelse try self.selectSharding(shape); + pub fn tensorShardingAttr( + self: Partitioning, + allocator: std.mem.Allocator, + ctx: *mlir.Context, + shape: Shape, + sharding: Sharding, + ) error{OutOfMemory}!*const mlir.Attribute { return switch (self.partitioner) { - .shardy => (try selected_sharding.data.sdyShardingAttrForShape(allocator, ctx, shape)).asAttr(), - .gspmd => try selected_sharding.data.gspmdShardingAttrForShape(allocator, ctx, shape), + .shardy => (try sharding.data.sdyShardingAttrForShape(allocator, ctx, shape)).asAttr(), + .gspmd => sharding.data.gspmdShardingAttrForShape(allocator, ctx, shape) catch |err| switch (err) { + error.WriteFailed => error.OutOfMemory, // We're writing to memory + error.OutOfMemory => error.OutOfMemory, + // TODO(hugomano): clarify what can trigger this and consider moving the check to the Sharding creation + error.MissingDeviceInTile => @panic("MissingDeviceInTile"), + }, }; } @@ -128,68 +138,7 @@ pub const Partitioning = struct { return sharding.shardableDim(shape.dim(ax), spec.axis, must_divide); } - pub fn sdyPerValueShardingAttr(self: Partitioning, allocator: std.mem.Allocator, ctx: *mlir.Context, shapes: []const Shape) !*const mlir.Attribute { - stdx.debug.assert(self.partitioner == .shardy, "sdyPerValueShardingAttr requires shardy partitioner", .{}); - - const shardings = try allocator.alloc(*const dialects.shardy.TensorShardingAttribute, shapes.len); - for (shapes, 0..) |shape, i| { - const sharding = try self.selectSharding(shape); - shardings[i] = try sharding.data.sdyShardingAttrForShape(allocator, ctx, shape); - } - return dialects.shardy.TensorShardingPerValueAttribute.init(ctx, shardings).asAttr(); - } - - pub fn sdyManualAxesAttr(self: Partitioning, allocator: std.mem.Allocator, ctx: *mlir.Context, in_shapes: []const Shape, out_shapes: []const Shape) !*const mlir.Attribute { - stdx.debug.assert(self.partitioner == .shardy, "sdyManualAxesAttr requires shardy partitioner", .{}); - - var axis_names = std.ArrayList([]const u8).empty; - defer axis_names.deinit(allocator); - - const Collect = struct { - fn appendUnique(list: *std.ArrayList([]const u8), allocator_: std.mem.Allocator, axis_name: []const u8) void { - for (list.items) |existing| { - if (std.mem.eql(u8, existing, axis_name)) return; - } - list.append(allocator_, axis_name) catch unreachable; - } - }; - - for (in_shapes) |shape| { - const sharding = try self.selectSharding(shape); - const attr = try sharding.data.sdyShardingAttrForShape(allocator, ctx, shape); - for (0..attr.numReplicatedAxes()) |i| { - Collect.appendUnique(&axis_names, allocator, attr.replicatedAxis(i).name()); - } - for (0..attr.numDimensions()) |i| { - const dim = attr.dimension(i); - for (0..dim.numAxes()) |j| { - Collect.appendUnique(&axis_names, allocator, dim.axis(j).name()); - } - } - } - for (out_shapes) |shape| { - const sharding = try self.selectSharding(shape); - const attr = try sharding.data.sdyShardingAttrForShape(allocator, ctx, shape); - for (0..attr.numReplicatedAxes()) |i| { - Collect.appendUnique(&axis_names, allocator, attr.replicatedAxis(i).name()); - } - for (0..attr.numDimensions()) |i| { - const dim = attr.dimension(i); - for (0..dim.numAxes()) |j| { - Collect.appendUnique(&axis_names, allocator, dim.axis(j).name()); - } - } - } - - const axes = try allocator.alloc(*const mlir.StringAttribute, axis_names.items.len); - for (axis_names.items, 0..) |axis_name, i| { - axes[i] = mlir.StringAttribute.init(ctx, axis_name); - } - - return dialects.shardy.ManualAxesAttribute.init(ctx, axes).asAttr(); - } - - pub fn selectSharding(self: Partitioning, shape: Shape) !Sharding { + pub fn selectSharding(self: Partitioning, shape: Shape) error{NoSuitableSharding}!Sharding { return pickSharding(self.shardings, shape, .any_covering) orelse error.NoSuitableSharding; } @@ -244,6 +193,72 @@ pub fn shardableDim(sharding: Sharding, dim: i64, logical_axis: anytype, must_di } } +pub fn sdyPerValueShardingAttr( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + shapes: []const Shape, + shardings: []const Sharding, +) error{OutOfMemory}!*const mlir.Attribute { + const sharding_attrs = try allocator.alloc(*const dialects.shardy.TensorShardingAttribute, shapes.len); + for (sharding_attrs, shapes, shardings) |*attr, shape, sharding| { + attr.* = try sharding.data.sdyShardingAttrForShape(allocator, ctx, shape); + } + return dialects.shardy.TensorShardingPerValueAttribute.init(ctx, sharding_attrs).asAttr(); +} + +pub fn sdyManualAxesAttr( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + in_shapes: []const Shape, + in_shardings: []const Sharding, + out_shapes: []const Shape, + out_shardings: []const Sharding, +) error{OutOfMemory}!*const mlir.Attribute { + var axis_names = std.ArrayList([]const u8).empty; + defer axis_names.deinit(allocator); + + const Collect = struct { + fn appendUnique(list: *std.ArrayList([]const u8), allocator_: std.mem.Allocator, axis_name: []const u8) void { + for (list.items) |existing| { + if (std.mem.eql(u8, existing, axis_name)) return; + } + list.append(allocator_, axis_name) catch unreachable; + } + }; + + for (in_shapes, in_shardings) |shape, sharding| { + const attr = try sharding.data.sdyShardingAttrForShape(allocator, ctx, shape); + for (0..attr.numReplicatedAxes()) |i| { + Collect.appendUnique(&axis_names, allocator, attr.replicatedAxis(i).name()); + } + for (0..attr.numDimensions()) |i| { + const dim = attr.dimension(i); + for (0..dim.numAxes()) |j| { + Collect.appendUnique(&axis_names, allocator, dim.axis(j).name()); + } + } + } + for (out_shapes, out_shardings) |shape, sharding| { + const attr = try sharding.data.sdyShardingAttrForShape(allocator, ctx, shape); + for (0..attr.numReplicatedAxes()) |i| { + Collect.appendUnique(&axis_names, allocator, attr.replicatedAxis(i).name()); + } + for (0..attr.numDimensions()) |i| { + const dim = attr.dimension(i); + for (0..dim.numAxes()) |j| { + Collect.appendUnique(&axis_names, allocator, dim.axis(j).name()); + } + } + } + + const axes = try allocator.alloc(*const mlir.StringAttribute, axis_names.items.len); + for (axis_names.items, 0..) |axis_name, i| { + axes[i] = mlir.StringAttribute.init(ctx, axis_name); + } + + return dialects.shardy.ManualAxesAttribute.init(ctx, axes).asAttr(); +} + /// Device as part of a PhysicalMesh. pub const Device = struct { /// Unique device identifier in the mesh @@ -1460,7 +1475,7 @@ pub const Data = struct { parent_allocator: std.mem.Allocator, ctx: *mlir.Context, shape: Shape, - ) !*const dialects.shardy.TensorShardingAttribute { + ) error{OutOfMemory}!*const dialects.shardy.TensorShardingAttribute { var arena = try stdx.arenaWithCapacity(parent_allocator, 1024); defer arena.deinit(); const allocator = arena.allocator(); diff --git a/zml/attention/triton_attention.zig b/zml/attention/triton_attention.zig index e7878297..1f80d792 100644 --- a/zml/attention/triton_attention.zig +++ b/zml/attention/triton_attention.zig @@ -1045,8 +1045,8 @@ test "sparse MLA emits 2D and 3D Triton kernels" { defer compilation.deactivate(); const block = @import("mlir").Block.init(&.{}, &.{}); - compilation.pushBlock(block); - defer compilation.popBlock(); + const scope = compilation.pushBlock(block); + defer scope.pop(); const q = zml.Tensor.zeroes(zml.Shape.init(.{ .q = 1, .h = 6, .hd = 576 }, .bf16)); const kv = zml.Tensor.zeroes(zml.Shape.init(.{ .page = 32, .k_chunk = 1, .hkv = 1, .hd = 576 }, .bf16)); diff --git a/zml/io.zig b/zml/io.zig index e4f079ad..3b9af053 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -25,7 +25,7 @@ const log = std.log.scoped(.@"zml/io"); pub const TensorStore = struct { registry: *safetensors.TensorRegistry, - id_to_sources: std.AutoHashMapUnmanaged(usize, []*safetensors.Tensor), + id_to_sources: std.AutoHashMapUnmanaged(Tensor.Id, []*safetensors.Tensor), allocator: std.mem.Allocator, arena: std.heap.ArenaAllocator, @@ -44,7 +44,7 @@ pub const TensorStore = struct { self.arena.deinit(); } - fn putSourcesNoClobber(self: *TensorStore, id: usize, sources: []*safetensors.Tensor) std.mem.Allocator.Error!void { + fn putSourcesNoClobber(self: *TensorStore, id: Tensor.Id, sources: []*safetensors.Tensor) std.mem.Allocator.Error!void { const gop = try self.id_to_sources.getOrPut(self.allocator, id); if (gop.found_existing) { stdx.debug.panic("Id {} already has associated sources", .{id}); @@ -85,7 +85,7 @@ pub const TensorStore = struct { return sources[0].reader(io, buffer, .{}); } - pub fn getSourcesById(self: *const TensorStore, id: usize) ?[]*safetensors.Tensor { + pub fn getSourcesById(self: *const TensorStore, id: Tensor.Id) ?[]*safetensors.Tensor { return self.id_to_sources.get(id); } diff --git a/zml/nn.zig b/zml/nn.zig index e83f9b8c..4d34de8c 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -1594,8 +1594,8 @@ test resizeBilinear { defer comp.deactivate(); const block = @import("mlir").Block.init(&.{}, &.{}); - comp.pushBlock(block); - defer comp.popBlock(); + const scope = comp.pushBlock(block); + defer scope.pop(); inline for (.{ .{ .{ .a = 10, .b = 10 }, .{ .a = 20 }, .{ .a = 20, .b = 10 } }, @@ -1661,8 +1661,8 @@ test resizeBicubic { defer comp.deactivate(); const block = @import("mlir").Block.init(&.{}, &.{}); - comp.pushBlock(block); - defer comp.popBlock(); + const scope = comp.pushBlock(block); + defer scope.pop(); inline for (.{ .{ .{ .a = 10, .b = 10 }, .{ .a = 20 }, .{ .a = 20, .b = 10 } }, diff --git a/zml/ops.zig b/zml/ops.zig index b950de14..ca291f75 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -17,9 +17,10 @@ const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; -pub const ShapeToCustomCallBuffer = @import("pjrtx.zig").ShapeToCustomCallBuffer; +const ShapeToCustomCallBuffer = @import("pjrtx.zig").ShapeToCustomCallBuffer; +const Sharding = @import("Sharding.zig"); const Tensor = @import("tensor.zig").Tensor; -pub const TensorToCustomCallBuffer = @import("pjrtx.zig").TensorToCustomCallBuffer; +const TensorToCustomCallBuffer = @import("pjrtx.zig").TensorToCustomCallBuffer; pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@TypeOf(inputs)) { const ctx = Compiler.current(); @@ -86,13 +87,12 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@T const block = mlir.Block.init(&block_types, &block_locs); errdefer block.deinit(); - ctx.pushBlock(block); - defer ctx.popBlock(); + const reducer_scope = ctx.pushBlock(block); + defer reducer_scope.pop(); - const scope = ctx.currentScope(); inline for (0..input_tensors.len) |i| { - scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; - scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + input_tensors.len) catch unreachable; + reducer_scope.registerTensorAsBlockArgument(args[i].left.id, i); + reducer_scope.registerTensorAsBlockArgument(args[i].right.id, i + input_tensors.len); } var reduced_values: [input_tensors.len]*const mlir.Value = undefined; @@ -102,14 +102,10 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@T } else { var reduced = @call(.auto, func, args); var reduced_tensors: [input_tensors.len]Tensor = undefined; - meta.collectBuf((struct { - pub fn cb(t: Tensor) Tensor { - return t; - } - }).cb, {}, &reduced, &reduced_tensors); + meta.collectBuf(stdx.meta.identity, {}, &reduced, &reduced_tensors); - inline for (0..input_tensors.len) |i| { - reduced_values[i] = reduced_tensors[i].value(); + for (reduced_values, reduced_tensors) |*vi, ti| { + vi.* = ti.value(); } } @@ -208,11 +204,11 @@ pub const ReduceArgs = struct { }; pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func: anytype, context: anytype) stdx.meta.FnReturn(func) { - var arena = std.heap.ArenaAllocator.init(Compiler.current().allocator); - defer arena.deinit(); - - const mlir_ctx = Compiler.current().mlir_ctx; + const compiler = Compiler.current(); + const caller_scope = compiler.currentScope(); + const mlir_ctx = compiler.mlir_ctx; + // Note: inputs and inits are infered to be tuple of ReduceArgs const reduce_block, var result = b: { const ArgsTypes: [inits.len]type = @splat(ReduceArgs); const ArgsType = std.meta.Tuple(&ArgsTypes); @@ -231,13 +227,12 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func const reduce_block = mlir.Block.init(&block_types, &block_locs); errdefer reduce_block.deinit(); - Compiler.current().pushBlock(reduce_block); - defer Compiler.current().popBlock(); + const reduce_scope = compiler.pushBlock(reduce_block); + defer reduce_scope.pop(); - const scope = Compiler.current().currentScope(); inline for (0..inits.len) |i| { - scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; - scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inits.len) catch unreachable; + reduce_scope.registerTensorAsBlockArgument(args[i].left.id, i); + reduce_scope.registerTensorAsBlockArgument(args[i].right.id, i + inits.len); } var result = @call(.auto, func, args ++ context); @@ -247,13 +242,12 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func result_values[i] = result[i].value(); } - _ = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)).appendTo(reduce_block); + const block_result = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)); + _ = block_result.appendTo(reduce_block); break :b .{ reduce_block, result }; }; var input_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| { - input_values[i] = inputs[i].value(); - } + inline for (0..inputs.len) |i| input_values[i] = inputs[i].value(); var init_values: [inputs.len]*const mlir.Value = undefined; inline for (0..inits.len) |i| init_values[i] = inits[i].value(); @@ -267,7 +261,7 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func }, .verify = true, .location = .unknown(mlir_ctx), - }).appendTo(Compiler.current().currentScope().block); + }).appendTo(caller_scope.block); // `stablehlo.reduce` drops axes. We want to avoid that to propagate tags. // So we need to broadcast the output of `stablehlo.reduce` to the input shapes. @@ -293,7 +287,7 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func broadcasting_axes.slice()[0 .. reduced_shape.rank() - axes_.len], mlirx.Type.rankedTensor(mlir_ctx, reduced_shape), .unknown(mlir_ctx), - ).appendTo(Compiler.current().currentScope().block); + ).appendTo(caller_scope.block); result[i] = Tensor._result(reduced_shape, broad_op.result(0)); } @@ -320,10 +314,8 @@ pub fn ReduceWindowFn(N: comptime_int) type { } pub fn reduceWindow(N: comptime_int, inputs: [N]Tensor, inits: [N]Tensor, opts: ReduceWindowOpts, func: *const ReduceWindowFn(N)) [N]Tensor { - var arena = std.heap.ArenaAllocator.init(Compiler.current().allocator); - defer arena.deinit(); - - const mlir_ctx = Compiler.current().mlir_ctx; + const compiler = Compiler.current(); + const mlir_ctx = compiler.mlir_ctx; const reduce_block, var result = b: { const Args = @Tuple(&@as([N]type, @splat(ReduceArgs))); @@ -342,13 +334,12 @@ pub fn reduceWindow(N: comptime_int, inputs: [N]Tensor, inits: [N]Tensor, opts: const reduce_block = mlir.Block.init(&block_types, &block_locs); errdefer reduce_block.deinit(); - Compiler.current().pushBlock(reduce_block); - defer Compiler.current().popBlock(); + const reduce_scope = compiler.pushBlock(reduce_block); + defer reduce_scope.pop(); - const scope = Compiler.current().currentScope(); inline for (0..N) |i| { - scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; - scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + N) catch unreachable; + reduce_scope.registerTensorAsBlockArgument(args[i].left.id, i); + reduce_scope.registerTensorAsBlockArgument(args[i].right.id, i + N); } var result = @call(.auto, func, args); @@ -386,7 +377,7 @@ pub fn reduceWindow(N: comptime_int, inputs: [N]Tensor, inits: [N]Tensor, opts: }, .verify = true, .location = .unknown(mlir_ctx), - }).appendTo(Compiler.current().currentScope().block); + }).appendTo(compiler.currentScope().block); inline for (0..result.len) |i| { result[i] = Tensor.fromMlirValue(reduce_op.result(i)).withTags(inputs[i].shape()); @@ -401,10 +392,8 @@ pub const SortArgs = struct { }; pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytype, is_stable: bool) [inputs.len]Tensor { - var arena = std.heap.ArenaAllocator.init(Compiler.current().allocator); - defer arena.deinit(); - - const mlir_ctx = Compiler.current().mlir_ctx; + const compiler = Compiler.current(); + const mlir_ctx = compiler.mlir_ctx; const sort_block = b: { const ArgsTypes: [inputs.len]type = @splat(SortArgs); @@ -424,13 +413,12 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytyp const sort_block = mlir.Block.init(&block_types, &block_locs); errdefer sort_block.deinit(); - Compiler.current().pushBlock(sort_block); - defer Compiler.current().popBlock(); + const scope = compiler.pushBlock(sort_block); + defer scope.pop(); - const scope = Compiler.current().currentScope(); inline for (0..inputs.len) |i| { - scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, 2 * i) catch unreachable; - scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, 2 * i + 1) catch unreachable; + scope.registerTensorAsBlockArgument(args[i].left.id, 2 * i); + scope.registerTensorAsBlockArgument(args[i].right.id, 2 * i + 1); } var result = @call(.auto, func, args ++ context); @@ -454,7 +442,7 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytyp }, .verify = true, .location = .unknown(mlir_ctx), - }).appendTo(Compiler.current().currentScope().block); + }).appendTo(compiler.currentScope().block); var result: [inputs.len]Tensor = undefined; inline for (0..inputs.len) |i| { @@ -514,14 +502,12 @@ pub fn @"while"( const block = mlir.Block.init(operands_info.items(.type), operands_info.items(.location)); errdefer block.deinit(); - comp.pushBlock(block); - defer comp.popBlock(); + const scope = comp.pushBlock(block); + defer scope.pop(); // Interpret initial_state as the block argument - const scope = comp.currentScope(); - scope.id_to_argument.ensureUnusedCapacity(scope.arena.allocator(), flat_operands.len) catch @panic("OOM"); for (0.., flat_operands) |i, input| { - scope.id_to_argument.putAssumeCapacity(input.id, i); + scope.registerTensorAsBlockArgument(input.id, i); } const cond: Tensor = captured_context.cond(initial_state); @@ -535,14 +521,13 @@ pub fn @"while"( const block = mlir.Block.init(operands_info.items(.type), operands_info.items(.location)); errdefer block.deinit(); - comp.pushBlock(block); - defer comp.popBlock(); + const scope = comp.pushBlock(block); + defer scope.pop(); // Interpret operands as the block argument - const scope = comp.currentScope(); scope.id_to_argument.ensureUnusedCapacity(scope.arena.allocator(), flat_operands.len) catch @panic("OOM"); for (0.., flat_operands) |i, input| { - scope.id_to_argument.putAssumeCapacity(input.id, i); + scope.registerTensorAsBlockArgument(input.id, i); } const result: While.State = captured_context.body(initial_state); @@ -696,8 +681,8 @@ pub fn @"if"( const block = mlir.Block.init(&.{}, &.{}); errdefer block.deinit(); - Compiler.current().pushBlock(block); - defer Compiler.current().popBlock(); + const scope = Compiler.current().pushBlock(block); + defer scope.pop(); const result = blkctx.onTrue(); const result_values = meta.collectAlloc(Tensor.value, {}, allocator, &result) catch @panic("OOM"); @@ -710,8 +695,8 @@ pub fn @"if"( const block = mlir.Block.init(&.{}, &.{}); errdefer block.deinit(); - Compiler.current().pushBlock(block); - defer Compiler.current().popBlock(); + const scope = Compiler.current().pushBlock(block); + defer scope.pop(); const result = blkctx.onFalse(); const result_values = meta.collectAlloc(Tensor.value, {}, allocator, &result) catch @panic("OOM"); @@ -789,8 +774,8 @@ pub fn if2( const block = mlir.Block.init(&.{}, &.{}); errdefer block.deinit(); - Compiler.current().pushBlock(block); - defer Compiler.current().popBlock(); + const scope = Compiler.current().pushBlock(block); + defer scope.pop(); _ = dialects.stablehlo.returns(mlir_ctx, true_values, loc).appendTo(block); break :b block; }; @@ -801,8 +786,8 @@ pub fn if2( const block = mlir.Block.init(&.{}, &.{}); errdefer block.deinit(); - Compiler.current().pushBlock(block); - defer Compiler.current().popBlock(); + const scope = Compiler.current().pushBlock(block); + defer scope.pop(); _ = dialects.stablehlo.returns(mlir_ctx, false_values, loc).appendTo(block); break :b block; @@ -1146,13 +1131,12 @@ pub fn scatter( const update_block = mlir.Block.init(&block_types, &block_locs); errdefer update_block.deinit(); - Compiler.current().pushBlock(update_block); - defer Compiler.current().popBlock(); + const scope = Compiler.current().pushBlock(update_block); + defer scope.pop(); - const scope = Compiler.current().currentScope(); inline for (0..inputs.len) |i| { - scope.id_to_argument.put(scope.arena.allocator(), args[i].input.id, i) catch unreachable; - scope.id_to_argument.put(scope.arena.allocator(), args[i].update.id, i + inputs.len) catch unreachable; + scope.registerTensorAsBlockArgument(args[i].input.id, i); + scope.registerTensorAsBlockArgument(args[i].update.id, i + inputs.len); } var result = @call(.auto, func, args ++ context); @@ -1349,8 +1333,8 @@ test scatterConfig { defer comp.deactivate(); const block = mlir.Block.init(&.{}, &.{}); - comp.pushBlock(block); - defer comp.popBlock(); + const scope = comp.pushBlock(block); + defer scope.pop(); const Local = struct { pub fn _idx(idx_shape: anytype) Tensor { @@ -1776,7 +1760,7 @@ pub fn composite( const block = mlir.Block.init(block_types, block_locs); - ctx.pushBlock(block); + const scope = ctx.pushBlock(block); { const arg_tensors = ctx.alloc(Tensor, inputs.len); for (inputs, 0..) |t, i| { @@ -1798,7 +1782,7 @@ pub fn composite( _ = dialects.func.returns(mlir_ctx, rvals, .unknown(mlir_ctx)).appendTo(block); } - ctx.popBlock(); + scope.pop(); _ = dialects.func.func(mlir_ctx, .{ .name = decomp_name, @@ -1975,8 +1959,11 @@ pub fn manualComputation( else => @compileError("Unsupported manualComputation output type: " ++ @typeName(@TypeOf(outputs))), }; + const sharded_outputs: []const Tensor = manualComputationInternal(inputs_, output_shapes, body_context, body_fn) catch |err| switch (err) { + error.OutOfMemory => @panic("OOM"), + }; const ReturnT = manualComputationReturnType(body_fn); - return manualComputationSliceToReturn(ReturnT, manualComputationInternal(inputs_, output_shapes, body_context, body_fn)); + return manualComputationSliceToReturn(ReturnT, sharded_outputs); } fn manualComputationInternal( @@ -1984,53 +1971,71 @@ fn manualComputationInternal( outputs: []const Shape, body_context: anytype, comptime body_fn: anytype, -) []Tensor { +) error{OutOfMemory}![]Tensor { const BodyReturnT = manualComputationReturnType(body_fn); const BodyInputsT = stdx.meta.FnParam(body_fn, 2); const BodyOutputShapesT = stdx.meta.FnParam(body_fn, 3); const ctx = Compiler.current(); - const allocator = ctx.arena.allocator(); + const scope = ctx.currentScope(); - const input_shapes = allocator.alloc(Shape, inputs.len) catch unreachable; - const input_values = allocator.alloc(*const mlir.Value, inputs.len) catch unreachable; + var arena_state: std.heap.ArenaAllocator = .init(std.heap.page_allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + const input_shapes = try arena.alloc(Shape, inputs.len); + const input_values = try arena.alloc(*const mlir.Value, inputs.len); for (inputs, 0..) |input, i| { input_shapes[i] = input.shape(); input_values[i] = input.value(); } + const local_input_shapes = try arena.alloc(Shape, inputs.len); + const local_output_shapes = try arena.alloc(Shape, outputs.len); + const input_shardings = try arena.alloc(Sharding, inputs.len); + const output_shardings = try arena.alloc(Sharding, outputs.len); + + for (input_shapes, 0..) |shape, i| { + const sharding = ctx.partitioning.selectSharding(shape) catch |err| switch (err) { + error.NoSuitableSharding => std.debug.panic( + "failed to shard manualComputation input {f}({d}) because it's using unknown sharding. Pass more shardings to `.compile`. Known shardings: {f}", + .{ shape, i, stdx.fmt.slice(ctx.partitioning.shardings) }, + ), + }; + input_shardings[i] = sharding; + local_input_shapes[i] = sharding.shardedShape(shape) catch std.debug.panic("can't shard {f} for {f}", .{ shape, sharding }); + } + for (outputs, 0..) |shape, i| { + const sharding = ctx.partitioning.selectSharding(shape) catch |err| switch (err) { + error.NoSuitableSharding => std.debug.panic( + "failed to shard manualComputation output {f}({d}) because it's using unknown sharding. Pass more shardings to `.compile`. Known shardings: {f}", + .{ shape, i, stdx.fmt.slice(ctx.partitioning.shardings) }, + ), + }; + output_shardings[i] = sharding; + local_output_shapes[i] = sharding.shardedShape(shape) catch std.debug.panic("can't shard {f} for {f}", .{ shape, sharding }); + } return switch (ctx.partitioning.partitioner) { - .shardy => blk: { - const local_input_shapes = allocator.alloc(Shape, inputs.len) catch unreachable; - const local_output_shapes = allocator.alloc(Shape, outputs.len) catch unreachable; + .shardy => { + const in_shardings_attr = try Sharding.sdyPerValueShardingAttr(arena, ctx.mlir_ctx, input_shapes, input_shardings); + const out_shardings_attr = try Sharding.sdyPerValueShardingAttr(arena, ctx.mlir_ctx, outputs, output_shardings); + const manual_axes_attr = try Sharding.sdyManualAxesAttr(arena, ctx.mlir_ctx, input_shapes, input_shardings, outputs, output_shardings); - for (input_shapes, 0..) |input_shape, i| { - local_input_shapes[i] = ctx.partitioning.localShapeForShape(input_shape) catch unreachable; - } - for (outputs, 0..) |output_shape, i| { - local_output_shapes[i] = ctx.partitioning.localShapeForShape(output_shape) catch unreachable; - } - - const in_shardings_attr = ctx.partitioning.sdyPerValueShardingAttr(allocator, ctx.mlir_ctx, input_shapes) catch unreachable; - const out_shardings_attr = ctx.partitioning.sdyPerValueShardingAttr(allocator, ctx.mlir_ctx, outputs) catch unreachable; - const manual_axes_attr = ctx.partitioning.sdyManualAxesAttr(allocator, ctx.mlir_ctx, input_shapes, outputs) catch unreachable; - - const block_types = allocator.alloc(*const mlir.Type, inputs.len) catch unreachable; - const block_locs = allocator.alloc(*const mlir.Location, inputs.len) catch unreachable; + const block_types = try arena.alloc(*const mlir.Type, inputs.len); for (local_input_shapes, 0..) |input_shape, i| { block_types[i] = mlirx.Type.rankedTensor(ctx.mlir_ctx, input_shape); - block_locs[i] = mlir.Location.unknown(ctx.mlir_ctx); } + const block_locs = try arena.alloc(*const mlir.Location, inputs.len); + @memset(block_locs, mlir.Location.unknown(ctx.mlir_ctx)); - const parent_block = ctx.currentScope().block; const manual_block = mlir.Block.init(block_types, block_locs); errdefer manual_block.deinit(); - ctx.pushBlock(manual_block); - defer ctx.popBlock(); + const manual_scope = ctx.pushBlock(manual_block); + defer manual_scope.pop(); - const local_inputs = allocator.alloc(Tensor, inputs.len) catch unreachable; + const local_inputs = try arena.alloc(Tensor, inputs.len); for (0..inputs.len) |i| { local_inputs[i] = Tensor._result(local_input_shapes[i], manual_block.argument(i)); } @@ -2040,11 +2045,11 @@ fn manualComputationInternal( const body_inputs = manualComputationInputsArg(BodyInputsT, local_inputs); const body_output_shapes = manualComputationOutputShapesArg(BodyOutputShapesT, local_output_shapes); - const body_result = @call(.auto, body_fn, .{ body_context, allocator, body_inputs, body_output_shapes }); - const local_outputs = manualComputationBodyToSlice(BodyReturnT, allocator, body_result); + const body_result = @call(.auto, body_fn, .{ body_context, arena, body_inputs, body_output_shapes }); + const local_outputs = manualComputationBodyToSlice(BodyReturnT, arena, body_result); stdx.debug.assert(local_outputs.len == outputs.len, "manualComputation body returned {} values, expected {}", .{ local_outputs.len, outputs.len }); - const local_output_values = allocator.alloc(*const mlir.Value, outputs.len) catch unreachable; + const local_output_values = try arena.alloc(*const mlir.Value, outputs.len); for (0..outputs.len) |i| { stdx.debug.assert(local_outputs[i].shape().eql(local_output_shapes[i]), "manualComputation body returned shape {f}, expected {f}", .{ local_outputs[i].shape(), local_output_shapes[i] }); local_output_values[i] = local_outputs[i].value(); @@ -2056,7 +2061,7 @@ fn manualComputationInternal( .location = .unknown(ctx.mlir_ctx), }).appendTo(manual_block); - const global_result_types = allocator.alloc(*const mlir.Type, outputs.len) catch unreachable; + const global_result_types = try arena.alloc(*const mlir.Type, outputs.len); for (outputs, 0..) |output_shape, i| { global_result_types[i] = mlirx.Type.rankedTensor(ctx.mlir_ctx, output_shape); } @@ -2072,29 +2077,17 @@ fn manualComputationInternal( }, .verify = true, .location = .unknown(ctx.mlir_ctx), - }).appendTo(parent_block); + }).appendTo(scope.block); - const outputs_ = allocator.alloc(Tensor, outputs.len) catch unreachable; + // Use the compiler allocator to return memory to the parent + const sharded_outputs = ctx.alloc(Tensor, outputs.len); for (outputs, 0..) |output, i| { - outputs_[i] = Tensor._result(output, op.result(i)); + sharded_outputs[i] = Tensor._result(output, op.result(i)); } - break :blk outputs_; + return sharded_outputs; }, - .gspmd => blk: { - const manual_sharding = "{manual}"; - const output_shardings = allocator.alloc(*const mlir.Attribute, outputs.len) catch unreachable; - const local_input_shapes = allocator.alloc(Shape, inputs.len) catch unreachable; - const local_output_shapes = allocator.alloc(Shape, outputs.len) catch unreachable; - - for (input_shapes, 0..) |input_shape, i| { - local_input_shapes[i] = ctx.partitioning.localShapeForShape(input_shape) catch unreachable; - } - for (outputs, 0..) |output_shape, i| { - local_output_shapes[i] = ctx.partitioning.localShapeForShape(output_shape) catch unreachable; - output_shardings[i] = ctx.partitioning.tensorShardingAttr(allocator, ctx.mlir_ctx, output_shape, null) catch unreachable; - } - - const local_input_values = allocator.alloc(*const mlir.Value, inputs.len) catch unreachable; + .gspmd => { + const local_input_values = try arena.alloc(*const mlir.Value, inputs.len); for (0..inputs.len) |i| { const local_type = mlirx.Type.rankedTensor(ctx.mlir_ctx, local_input_shapes[i]); const full_to_shard = dialects.stablehlo.custom_call( @@ -2106,15 +2099,15 @@ fn manualComputationInternal( .has_side_effect = false, .backend_config = .{ .original = "" }, .additional_attributes = &.{ - .named(ctx.mlir_ctx, "mhlo.sharding", .string(ctx.mlir_ctx, manual_sharding)), + .named(ctx.mlir_ctx, "mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")), }, }, .unknown(ctx.mlir_ctx), - ).appendTo(ctx.currentScope().block); + ).appendTo(scope.block); local_input_values[i] = full_to_shard.result(0); } - const local_inputs = allocator.alloc(Tensor, inputs.len) catch unreachable; + const local_inputs = try arena.alloc(Tensor, inputs.len); for (0..inputs.len) |i| { local_inputs[i] = Tensor._result(local_input_shapes[i], local_input_values[i]); } @@ -2123,20 +2116,21 @@ fn manualComputationInternal( defer ctx.manual_computation_depth -= 1; const body_inputs = manualComputationInputsArg(BodyInputsT, local_inputs); const body_output_shapes = manualComputationOutputShapesArg(BodyOutputShapesT, local_output_shapes); - const body_result = @call(.auto, body_fn, .{ body_context, allocator, body_inputs, body_output_shapes }); - const local_outputs = manualComputationBodyToSlice(BodyReturnT, allocator, body_result); + const body_result = @call(.auto, body_fn, .{ body_context, arena, body_inputs, body_output_shapes }); + const local_outputs = manualComputationBodyToSlice(BodyReturnT, arena, body_result); stdx.debug.assert(local_outputs.len == outputs.len, "manualComputation body returned {} values, expected {}", .{ local_outputs.len, outputs.len }); for (0..outputs.len) |i| { stdx.debug.assert(local_outputs[i].shape().eql(local_output_shapes[i]), "manualComputation body returned shape {f}, expected {f}", .{ local_outputs[i].shape(), local_output_shapes[i] }); } - if (outputs.len == 0) { - break :blk allocator.alloc(Tensor, 0) catch unreachable; - } + // Skip optimization barrier for custom call that don't return values + if (outputs.len == 0) return &.{}; + + const global_values = try arena.alloc(*const mlir.Value, outputs.len); + const global_types = try arena.alloc(*const mlir.Type, outputs.len); + for (outputs, output_shardings, 0..) |output_shape, output_sharding, i| { + const gspmd_attr = try ctx.partitioning.tensorShardingAttr(arena, ctx.mlir_ctx, output_shape, output_sharding); - const global_values = allocator.alloc(*const mlir.Value, outputs.len) catch unreachable; - const global_types = allocator.alloc(*const mlir.Type, outputs.len) catch unreachable; - for (outputs, 0..) |output_shape, i| { global_types[i] = mlirx.Type.rankedTensor(ctx.mlir_ctx, output_shape); const shard_to_full = dialects.stablehlo.custom_call( ctx.mlir_ctx, @@ -2147,11 +2141,11 @@ fn manualComputationInternal( .has_side_effect = false, .backend_config = .{ .original = "" }, .additional_attributes = &.{ - .named(ctx.mlir_ctx, "mhlo.sharding", output_shardings[i]), + .named(ctx.mlir_ctx, "mhlo.sharding", gspmd_attr), }, }, .unknown(ctx.mlir_ctx), - ).appendTo(ctx.currentScope().block); + ).appendTo(scope.block); global_values[i] = shard_to_full.result(0); } @@ -2160,13 +2154,13 @@ fn manualComputationInternal( global_values, global_types, .unknown(ctx.mlir_ctx), - ).appendTo(ctx.currentScope().block); + ).appendTo(scope.block); - const outputs_ = allocator.alloc(Tensor, outputs.len) catch unreachable; + const sharded_outputs = ctx.alloc(Tensor, outputs.len); for (outputs, 0..) |output_shape, i| { - outputs_[i] = Tensor._result(output_shape, barrier.result(i)); + sharded_outputs[i] = Tensor._result(output_shape, barrier.result(i)); } - break :blk outputs_; + return sharded_outputs; }, }; } @@ -2245,7 +2239,7 @@ fn manualComputationOutputShapesArg(comptime OutputShapesT: type, local_output_s }; } -fn manualComputationSliceToReturn(comptime ReturnT: type, outputs: []Tensor) ReturnT { +fn manualComputationSliceToReturn(comptime ReturnT: type, outputs: []const Tensor) ReturnT { if (ReturnT == void) { stdx.debug.assert(outputs.len == 0, "manualComputation expected no outputs, got {}", .{outputs.len}); return; @@ -2433,17 +2427,11 @@ pub fn shardingAwareTypedCustomCall( output_shapes[i] = @field(output, field.name); } - const output_tensors = manualComputationInternal(&input_tensors, &output_shapes, attributes, (struct { + const output_tensors: []const Tensor = manualComputationInternal(&input_tensors, &output_shapes, attributes, (struct { fn body(attributes_: Attributes, _: std.mem.Allocator, sharded_input_tensors: []const Tensor, sharded_output_shapes: []const Shape) []const Tensor { - return typedCustomCall( - target_name, - opts, - sharded_input_tensors, - sharded_output_shapes, - attributes_, - ); + return typedCustomCall(target_name, opts, sharded_input_tensors, sharded_output_shapes, attributes_); } - }).body); + }).body) catch @panic("OOM"); // Convert the slice back to a struct var out: ShapeToTensor(Output) = undefined; @@ -2628,8 +2616,8 @@ test customCall { defer comp.deactivate(); const block = mlir.Block.init(&.{}, &.{}); - comp.pushBlock(block); - defer comp.popBlock(); + const scope = comp.pushBlock(block); + defer scope.pop(); const shape = zml.Shape.init(.{128}, .bf16).withPartitioning(.{ ._0 = .x }); const input = Tensor.constant(zml.DataType.bf16.constant(0)).broad(shape); diff --git a/zml/tensor.zig b/zml/tensor.zig index ca693a7f..22d14138 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -23,9 +23,10 @@ test { } pub const Tensor = struct { - var current_id: std.atomic.Value(usize) = .{ .raw = 1 }; + pub const Id = enum(u64) { _ }; + var current_id: std.atomic.Value(u64) = .init(1); - id: usize, + id: Tensor.Id, auto_broadcast: bool = false, _shape: Shape, _value: ?*const mlir.Value = null, @@ -36,8 +37,12 @@ pub const Tensor = struct { return .fromShape(.init(shape_like, dt)); } + fn nextTensorId() Id { + return @enumFromInt(Tensor.current_id.fetchAdd(1, .seq_cst)); + } + pub fn fromShape(shape_: Shape) Tensor { - return .{ .id = Tensor.current_id.fetchAdd(1, .seq_cst), ._shape = shape_ }; + return .{ .id = nextTensorId(), ._shape = shape_ }; } pub fn format(self: Tensor, writer: *std.Io.Writer) !void { @@ -73,11 +78,7 @@ pub const Tensor = struct { /// /// Creates a tensor from a Shape and an mlir.Value. pub fn _result(sh: Shape, val: *const mlir.Value) Tensor { - const res: Tensor = .{ - ._shape = sh, - ._value = val, - .id = Tensor.current_id.fetchAdd(1, .seq_cst), - }; + const res: Tensor = .{ ._shape = sh, ._value = val, .id = nextTensorId() }; if (builtin.mode == .Debug) { // Check that the MLIR value actually have the same shape. @@ -105,7 +106,7 @@ pub const Tensor = struct { sh._tags.appendNTimes(Shape.TagUnknown, n) catch unreachable; sh._partitioning.appendNTimes(.unknown, n) catch unreachable; - return .{ ._shape = sh, ._value = val, .id = Tensor.current_id.fetchAdd(1, .seq_cst) }; + return .{ ._shape = sh, ._value = val, .id = nextTensorId() }; } /// Returns the dimension of axis 'axis_'. @@ -159,14 +160,13 @@ pub const Tensor = struct { return res; }; - const attr = ctx.partitioning.tensorShardingAttr(ctx.allocator, ctx.mlir_ctx, partitioned_shape, null) catch |err| switch (err) { + const sharding = ctx.partitioning.selectSharding(partitioned_shape) catch |err| switch (err) { error.NoSuitableSharding => std.debug.panic( "{f}.withPartitioning({f}) failed to resolve because it's using unknown sharding. Pass more shardings to `zml.compile`. Known shardings: {f}", .{ self, partitioned_shape, stdx.fmt.slice(ctx.partitioning.shardings) }, ), - error.OutOfMemory, error.WriteFailed => @panic("OOM"), - error.MissingDeviceInTile => @panic("TODO"), }; + const attr = ctx.partitioning.tensorShardingAttr(ctx.allocator, ctx.mlir_ctx, partitioned_shape, sharding) catch @panic("OOM"); const op_result = switch (ctx.partitioning.partitioner) { .shardy => blk: { @@ -233,7 +233,7 @@ pub const Tensor = struct { ).appendTo(currentBlock()); const res = _result(self._shape, op.result(0)); - ctx.currentScope().id_to_output_memory_kind.put(ctx.currentScope().arena.allocator(), res.id, kind) catch unreachable; + ctx.currentScope().id_to_memory.putNoClobber(ctx.currentScope().arena.allocator(), res.id, kind) catch @panic("OOM"); return res; } @@ -248,10 +248,10 @@ pub const Tensor = struct { var copy = flat_tensors; meta.visitFlatStruct(struct { - fn onMemory(k: Memory.Kind, x: *Tensor) void { + fn toMemory(k: Memory.Kind, x: *Tensor) void { x.* = x.toMemory(k); } - }.onMemory, kind, ©); + }.toMemory, kind, ©); return copy; } @@ -268,7 +268,7 @@ pub const Tensor = struct { return self; } - ctx.currentScope().id_to_input_memory_kind.put(ctx.currentScope().arena.allocator(), self.id, kind) catch unreachable; + ctx.currentScope().id_to_memory.put(ctx.currentScope().arena.allocator(), self.id, kind) catch unreachable; return self; } @@ -313,11 +313,10 @@ pub const Tensor = struct { /// /// This will fail if used outside of a Compiler context. pub fn value(self: Tensor) *const mlir.Value { - if (Compiler.current().currentScope().id_to_argument.get(self.id)) |argument_index| { - return Compiler.current().currentScope().block.argument(argument_index); - } else if (self._value) |v| { - return v; - } else @panic("Something went really wrong, tensor is not an argument nor has an mlir.Value"); + const scope = Compiler.current().currentScope(); + return scope.id_to_argument.get(self.id) orelse + self._value orelse + @panic("Something went really wrong, tensor is not an argument nor has an mlir.Value"); } /// Tell PJRT compiler that memory should be reuse between the two tensors. @@ -330,10 +329,7 @@ pub const Tensor = struct { pub fn reuseBuffer(self: Tensor, origin: Tensor) Tensor { const compilation_context = Compiler.current(); const scope = compilation_context.currentScope(); - if (scope.id_to_argument.get(origin.id)) |argument_index| { - const gop = scope.id_to_donation.getOrPut(scope.arena.allocator(), self.id) catch unreachable; - gop.value_ptr.* = argument_index; - } else if (scope.id_to_donation.get(origin.id)) |origin_donation| { + if (scope.id_to_donation.get(origin.id)) |origin_donation| { const gop = scope.id_to_donation.getOrPut(scope.arena.allocator(), self.id) catch unreachable; gop.value_ptr.* = origin_donation; } @@ -350,7 +346,7 @@ pub const Tensor = struct { const right = Tensor.init(.{ 2, 6 }, .f32); const Local = struct { - pub fn memcopy(x: Tensor, y: Tensor) Tensor { + fn memcopy(x: Tensor, y: Tensor) Tensor { return x.addConstant(1).reuseBuffer(y); } }; @@ -2754,8 +2750,8 @@ pub const Tensor = struct { defer comp.deactivate(); const block = mlir.Block.init(&.{}, &.{}); - comp.pushBlock(block); - defer comp.popBlock(); + const scope = comp.pushBlock(block); + defer scope.pop(); inline for (.{ .{ .{ .a = 10 }, .{ .a = idx(.{}) }, .{} }, @@ -2906,8 +2902,8 @@ pub const Tensor = struct { defer comp.deactivate(); const block = mlir.Block.init(&.{}, &.{}); - comp.pushBlock(block); - defer comp.popBlock(); + const scope = comp.pushBlock(block); + defer scope.pop(); inline for (.{ .{ .{ .a = 10 }, .{}, .{ ._ = 0 }, .{ .a = 10 } }, @@ -3132,8 +3128,8 @@ pub const Tensor = struct { const block = mlir.Block.init(&.{}, &.{}); defer block.deinit(); - comp.pushBlock(block); - defer comp.popBlock(); + const scope = comp.pushBlock(block); + defer scope.pop(); const idx = Local._idx; @@ -3671,8 +3667,8 @@ pub const Tensor = struct { const block = mlir.Block.init(&.{}, &.{}); defer block.deinit(); - comp.pushBlock(block); - defer comp.popBlock(); + const scope = comp.pushBlock(block); + defer scope.pop(); inline for (.{ .{ .{ .a = 12 }, .a, 3, .{ .a = 4 } }, @@ -3724,8 +3720,8 @@ pub const Tensor = struct { const block = mlir.Block.init(&.{}, &.{}); defer block.deinit(); - comp.pushBlock(block); - defer comp.popBlock(); + const scope = comp.pushBlock(block); + defer scope.pop(); inline for (.{ .{ .{ .a = 10 }, .a, 3, .{ .a = 3 }, .{ .a = 1 } }, @@ -4507,20 +4503,6 @@ pub const Tensor = struct { fn currentBlock() *mlir.Block { return Compiler.current().currentScope().block; } - - /// Returns the donation data of the tensor. - pub fn donation(self: Tensor) ?usize { - return Compiler.current().currentScope().id_to_donation.get(self.id); - } - - /// Returns the output memory kind of the tensor. - pub fn outputMemoryKind(self: Tensor) Memory.Kind { - return Compiler.current().currentScope().id_to_output_memory_kind.get(self.id) orelse .device; - } - - pub fn inputMemoryKind(self: Tensor) ?Memory.Kind { - return Compiler.current().currentScope().id_to_input_memory_kind.get(self.id); - } }; fn initPoolArg(rank: usize, data: []const i64) [constants.MAX_RANK]i64 { From a46e4cac0688e8e617e0f5d615ecf30aa8ae2dfb Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 31 Aug 2026 13:21:52 +0200 Subject: [PATCH 642/881] zml: handle packed fp4 on CPU side (#724) The XLA API assumes that all sub-bytes data is given to it unpacked. But when you read safetensors files the data is packed. My proposition is the following: when we have a zml.fp4 tensor/buffer we say to pjrt it's a u8 buffer instead. Since PJRT infers the input/output shape from stablehlo I had to introduce some bitcast at the edge of the compilation: - fp4 input are declared as u8, but I insert bitcast operand before calling the zig code, so the model code only see the fp4 tensors. - when the model code returns a fp4 output, I bitcast to u8 before exiting the final compilation Breaking changes: none Currently fp4 can't cross the pjrt boundary, so all fp4 code already manually lie about the data being u8, and it will keep working. In the future this code will be able to be "upgraded" by explicitly tagging the input as fp4 and remove the "unpacking" code from their model code. Note: the tracking XLA issue seems pretty cold https://github.com/openxla/xla/issues/16795 --- examples/benchmark/main.zig | 7 +- stdx/fmt.zig | 27 ++++-- zml/Compiler.zig | 76 ++++++++++++++-- zml/buffer.zig | 6 +- zml/dtype.zig | 18 +++- zml/floats.zig | 19 ++++ zml/shape.zig | 33 +++---- zml/slice.zig | 19 +++- zml/tensor.zig | 167 ++++++++++++++++++++++++------------ zml/testing.zig | 6 +- 10 files changed, 284 insertions(+), 94 deletions(-) diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index d3ba9eee..c1e4086e 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -102,18 +102,19 @@ fn createRandomBuffer(allocator: std.mem.Allocator, io: std.Io, platform: *const switch (shape.dtype()) { inline else => |v| { - const ZigType = v.toZigType(); + const ZigType = v.toPackedZigType(); switch (comptime v.class()) { .bool => unreachable, .integer => { - for (slice.items(ZigType)) |*e| e.* = random.int(ZigType); + for (slice.items(ZigType)) |*e| e.* = @bitCast(random.int(@Int(.unsigned, @bitSizeOf(ZigType)))); }, .float => { const value = random.float(f32); for (slice.items(ZigType)) |*e| e.* = switch (ZigType) { f64, f32 => value, f16 => @floatCast(value), - inline else => |T| if (@hasDecl(T, "fromF32")) T.fromF32(value) else unreachable, + zml.floats.Float4E2M1.Packed => .fromF32(value, -value), + inline else => |T| if (@hasDecl(T, "fromF32")) .fromF32(value) else unreachable, }; }, .complex => unreachable, diff --git a/stdx/fmt.zig b/stdx/fmt.zig index b176a099..47ec59aa 100644 --- a/stdx/fmt.zig +++ b/stdx/fmt.zig @@ -37,15 +37,32 @@ pub fn formatFloat(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) .float => value, else => @compileError("formatFloat expects a float, got: " ++ @typeName(@TypeOf(value))), }; - return writer.printFloat(x, spec); + switch (@typeInfo(@TypeOf(x))) { + .float => try writer.printFloat(x, spec), + // Handle F4.toF32() being a [2]f32 + .array => inline for (0.., x) |i, xi| { + try writer.printFloat(xi, spec); + if (i < x.len - 1) try writer.writeByte(','); + }, + else => @compileError("formatFloat expects a float, got: " ++ @typeName(@TypeOf(value))), + } } pub fn formatInt(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { - switch (@typeInfo(@TypeOf(value))) { - .int => {}, + return switch (@typeInfo(@TypeOf(value))) { + .int => try writer.printInt(value, spec.mode.base().?, spec.case, .{ .alignment = spec.alignment, .fill = spec.fill }), + .vector => { + const vector = @typeInfo(@TypeOf(value)).vector; + const array: [vector.len]vector.child = value; + for (&array, 0..) |elem, i| { + try writer.printInt(elem, spec.mode.base().?, spec.case, .{ .alignment = spec.alignment, .fill = spec.fill }); + if (i < array.len - 1) { + try writer.writeAll(", "); + } + } + }, else => @compileError("formatInt expects an int, got: " ++ @typeName(@TypeOf(value))), - } - return writer.printInt(value, spec.mode.base().?, spec.case, .{ .alignment = spec.alignment, .fill = spec.fill }); + }; } pub fn formatComplex(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 08547fcf..641fd93d 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -458,9 +458,17 @@ fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{Ou infos: std.MultiArrayList(TensorInfo) = .empty, fn cb(ctx: *@This(), tensor: *const Tensor) error{OutOfMemory}!void { - const mlir_type = mlirx.Type.rankedTensor(ctx.compiler.mlir_ctx, tensor._shape); + // Declare the program argument as a packed u8 + const og_shape = tensor._shape; + const packed_shape = og_shape.packedShape(); + const mlir_type = mlirx.Type.rankedTensor(ctx.compiler.mlir_ctx, packed_shape); + var value = ctx.scope.block.addArgument(mlir_type, .unknown(ctx.compiler.mlir_ctx)); - const value = ctx.scope.block.addArgument(mlir_type, .unknown(ctx.compiler.mlir_ctx)); + // But immediately create the MLIR value corresponding to the fp4 data. + // That way the user code only sees a proper fp4 tensor. + if (tensor.dtype().bitSizeOf() < 8) { + value = unpack(ctx.compiler, ctx.scope, og_shape, value); + } const gop = ctx.scope.id_to_argument.getOrPutValue(ctx.scope.arena.allocator(), tensor.id, value) catch @panic("OOM"); if (gop.found_existing) std.debug.panic("Tensor with id {} has already been used once as an argument", .{tensor.id}); @@ -469,15 +477,15 @@ fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{Ou defer ctx.current_argument_id += 1; - const input_sharding = ctx.compiler.partitioning.selectSharding(tensor._shape) catch |err| switch (err) { + const input_sharding = ctx.compiler.partitioning.selectSharding(packed_shape) catch |err| switch (err) { error.NoSuitableSharding => std.debug.panic( "Failed to resolve sharding for input {f}({d}) because it's using unknown sharding. Pass more shardings to `platform.compile`. Known shardings: {f}", - .{ tensor._shape, ctx.current_argument_id, stdx.fmt.slice(ctx.compiler.partitioning.shardings) }, + .{ packed_shape, ctx.current_argument_id, stdx.fmt.slice(ctx.compiler.partitioning.shardings) }, ), }; try ctx.infos.append(ctx.compiler.allocator, .{ .id = tensor.id, - .shape = tensor._shape, + .shape = og_shape, .sharding = input_sharding, .value = value, }); @@ -500,15 +508,21 @@ fn collectOutputInfo(compiler: *Compiler, scope: *Scope, v: anytype) error{OutOf infos: std.MultiArrayList(TensorInfo) = .empty, fn cb(ctx: *@This(), tensor: *const Tensor) !void { - const value = ctx.scope.id_to_argument.get(tensor.id) orelse + const og_shape = tensor.shape(); + const packed_shape = og_shape.packedShape(); + var value = ctx.scope.id_to_argument.get(tensor.id) orelse tensor._value orelse @panic("no value found for output tensor"); + if (tensor.dtype().bitSizeOf() < 8) { + value = repack(ctx.compiler, ctx.scope, og_shape, value); + } + try ctx.infos.append(ctx.compiler.allocator, .{ .id = tensor.id, - .shape = tensor._shape, - // Note: the panic should have been triggered during emitMlir or collectInputInfo - .sharding = ctx.compiler.partitioning.selectSharding(tensor._shape) catch @panic("failed to resolve output sharding"), + .shape = og_shape, + // Note: the panic should have been triggered during createBlockArguments or emitMlir + .sharding = ctx.compiler.partitioning.selectSharding(packed_shape) catch @panic("failed to resolve output sharding"), .value = value, }); } @@ -572,6 +586,50 @@ fn finalizeMlirFunc(compiler: *Compiler, fn_scope: *Scope, input_info: std.Multi }; } +fn unpack(compiler: *Compiler, scope: *Scope, og_shape: Shape, tensor_value: *const mlir.Value) *const mlir.Value { + const true_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, og_shape); + const elem_per_bytes = @divExact(8, og_shape.dtype().bitSizeOf()); + const intermediary_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, og_shape.splitAxis(-1, .{ -1, elem_per_bytes })); + + const bit_cast_op = dialects.stablehlo.bitcast_convert( + compiler.mlir_ctx, + tensor_value, + intermediary_type, + .unknown(compiler.mlir_ctx), + ).appendTo(scope.block); + + const reshape_op = dialects.stablehlo.reshape( + compiler.mlir_ctx, + bit_cast_op.result(0), + true_type, + .unknown(compiler.mlir_ctx), + ).appendTo(scope.block); + + return reshape_op.result(0); +} + +fn repack(compiler: *Compiler, scope: *Scope, og_shape: Shape, value: *const mlir.Value) *const mlir.Value { + const packed_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, og_shape.packedShape()); + const elem_per_bytes = @divExact(8, og_shape.dtype().bitSizeOf()); + const intermediary_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, og_shape.splitAxis(-1, .{ -1, elem_per_bytes })); + + const reshape_op = dialects.stablehlo.reshape( + compiler.mlir_ctx, + value, + intermediary_type, + .unknown(compiler.mlir_ctx), + ).appendTo(scope.block); + + const bit_cast_op = dialects.stablehlo.bitcast_convert( + compiler.mlir_ctx, + reshape_op.result(0), + packed_type, + .unknown(compiler.mlir_ctx), + ).appendTo(scope.block); + + return bit_cast_op.result(0); +} + fn setXlaOverrideFlag(map: *c.upb_Map, flag: []const u8, value: anytype, upb_arena: *c.upb_Arena) !void { const result = c.upb_Map_Set( map, diff --git a/zml/buffer.zig b/zml/buffer.zig index 991fe32a..d8c31b4e 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -112,14 +112,16 @@ pub const Buffer = struct { pub fn from( io: std.Io, platform: *const Platform, - sh: Shape, + shape_: Shape, sharding: Sharding, data_: []const u8, opts: FromOptions, ) !Buffer { + // Use the PJRT shape for everything + const sh = shape_.packedShape(); var res: Buffer = .{ ._platform = platform, - ._shape = sh, + ._shape = shape_, ._sharding = sharding.resolve(platform), ._shards = .empty, }; diff --git a/zml/dtype.zig b/zml/dtype.zig index 8ec73c3d..c56fdd5b 100644 --- a/zml/dtype.zig +++ b/zml/dtype.zig @@ -7,8 +7,8 @@ const C128 = std.math.Complex(f64); pub const DataType = enum(u8) { bool, - // Note: the support of the float8 is a bit spotty, f8e4m3b11fnuz seems to be the most supported one on Cuda. f4e2m1, + // Note: the support of the float8 is a bit spotty, f8e4m3b11fnuz seems to be the most supported one on Cuda. f8e3m4, f8e4m3, f8e4m3b11fnuz, @@ -136,9 +136,22 @@ pub const DataType = enum(u8) { return @FieldType(Value, @tagName(dtype)); } + pub fn toPackedZigType(comptime dtype: DataType) type { + return switch (dtype) { + .bool => bool, + .i2, .i4, .u2, .u4 => |dt| @Vector(8 / dt.bitSizeOf(), @FieldType(Value, @tagName(dtype))), + .f4e2m1 => floats.Float4E2M1.Packed, + else => { + const T = @FieldType(Value, @tagName(dtype)); + if (@bitSizeOf(T) < 8) @compileLog("Forgot type !", T); + return T; + }, + }; + } + pub fn isSignedInt(dtype: DataType) bool { return switch (dtype) { - .i4, .i8, .i16, .i32, .i64 => true, + .i2, .i4, .i8, .i16, .i32, .i64 => true, else => false, }; } @@ -151,6 +164,7 @@ pub const DataType = enum(u8) { pub fn bitSizeOf(self: DataType) u16 { return switch (self) { + .bool => 8, // In PJRT/stablehlo bool always occupy a full byte inline else => |tag| @bitSizeOf(tag.toZigType()), }; } diff --git a/zml/floats.zig b/zml/floats.zig index 3a0efb00..7bf08824 100644 --- a/zml/floats.zig +++ b/zml/floats.zig @@ -502,6 +502,25 @@ pub const Float4E2M1 = packed struct(u4) { } try std.testing.expectEqualSlices(u4, &.{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, @ptrCast(&from_f32_res)); } + + pub const Packed = packed struct(@Int(.unsigned, @bitSizeOf(Float4E2M1) * 2)) { + x: Float4E2M1, + y: Float4E2M1, + + pub fn fromF32(x: f32, y: f32) Packed { + return .{ .x = .fromF32(x), .y = .fromF32(y) }; + } + + pub fn toF32(xy: Packed) [2]f32 { + return .{ xy.x.toF32(), xy.y.toF32() }; + } + + pub fn formatNumber(xy: Packed, w: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + try xy.x.formatNumber(w, n); + w.writeByte(','); + try xy.y.formatNumber(w, n); + } + }; }; pub fn floatCast(T: type, x: anytype) T { diff --git a/zml/shape.zig b/zml/shape.zig index 75895fec..8afae990 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -500,22 +500,7 @@ pub const Shape = struct { /// Total size in bytes needed to represent this shape on host pub fn byteSize(self: Shape) usize { - return self.dtype().sizeOf() * self.count(); - } - - /// Total size in bytes needed to represent this shape on device - /// https://github.com/openxla/xla/blob/0f7240cad3fa14b52a809ac1f4a351a9edfc541e/xla/primitive_util.h#L807 - pub fn deviceByteSize(self: Shape) usize { - const bits_per_element: usize = switch (self.dtype()) { - .bool => 8, - else => self.dtype().bitSizeOf(), - }; - - return std.math.divCeil( - usize, - self.count() * bits_per_element, - 8, - ) catch unreachable; + return std.math.divCeil(usize, self.count() * self.dtype().bitSizeOf(), 8) catch unreachable; } /// Compares the two shapes described, ignoring tagging. @@ -1640,6 +1625,22 @@ pub const Shape = struct { .flat_index = 0, }; } + + /// Generate the packed shape visible to PJRT + pub fn packedShape(sh: Shape) Shape { + const bit_size = sh.dtype().bitSizeOf(); + if (bit_size >= 8) return sh; + + const items_per_byte = @divExact(8, bit_size); + var res: Shape = sh.setDim(-1, @divExact(sh.dim(-1), items_per_byte)); + res._dtype = .u8; + return res; + } + + test packedShape { + const x: Shape = .init(.{ 4, 8 }, .u2); + try expectEqualShapes(.init(.{ 4, 2 }, .u8), x.packedShape()); + } }; /// Iterates over all coordinates of a shape, yielding the multi-dimensional diff --git a/zml/slice.zig b/zml/slice.zig index ea3b8c14..f4e0d2ea 100644 --- a/zml/slice.zig +++ b/zml/slice.zig @@ -147,6 +147,7 @@ pub const Slice = struct { } pub fn items(slice: Slice, comptime T: type) []T { + stdx.debug.assertComptime(@bitSizeOf(T) >= 8, "zml.Slice stores packed sub-bytes type so you need to pass a packed type here like @Vector({}, {}). Got: {}", .{ @divFloor(8, @bitSizeOf(T)), T, T }); return @ptrCast(@alignCast(slice.data())); } @@ -273,14 +274,17 @@ pub const Slice = struct { try writer.splatByteAll(' ', indent_level); switch (slice.dtype()) { inline else => |dt| { - const T = dt.toZigType(); + const T = dt.toPackedZigType(); const n = slice.shape.dim(0); + const elem_per_bytes: comptime_int = @max(1, 8 / comptime dt.bitSizeOf()); + const stride = @divExact(slice.byte_strides.get(0), @as(i64, @sizeOf(T))); - const stride = @divExact(slice.byte_strides.get(0), @as(i64, @intCast(@sizeOf(T)))); + // Fetch n next elements, accounting for strides and packing + const needed_len: usize = @as(usize, if (n == 0) 0 else @intCast(@abs((n - 1) * stride) + 1)) / elem_per_bytes; - const needed_len: usize = if (n == 0) 0 else @intCast(@abs((n - 1) * stride) + 1); // The formatter consumes this physical tail using `stride`, so this // rank-1 path can read raw items without requiring contiguity. + std.log.warn("Printing {} elems from offset {} out of {} total elems @ {}", .{ n, slice.offset_bytes, needed_len, stride }); const raw_values: []const T = @ptrCast(@alignCast(slice.bytes[slice.offset_bytes..])); const values = raw_values[0..needed_len]; @@ -512,3 +516,12 @@ test "slice pretty print ellipsis" { ; try std.testing.expectFmt(expected, "{d}", .{slice}); } + +test "slice pretty print f4" { + const x_f4_packed = [_]floats.Float4E2M1.Packed{ .fromF32(0, 0.5), .fromF32(1, 1.5), .fromF32(2, 3), .fromF32(4, 6) }; + const slice = Slice.initConst(.init(.{8}, .f4e2m1), @ptrCast(&x_f4_packed)); + + const expected = "{0,0.5,1,1.5,2,3,4,6}"; + + try std.testing.expectFmt(expected, "{d}", .{slice}); +} diff --git a/zml/tensor.zig b/zml/tensor.zig index 22d14138..f26afad3 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -83,7 +83,9 @@ pub const Tensor = struct { if (builtin.mode == .Debug) { // Check that the MLIR value actually have the same shape. const other = fromMlirValue(val); - stdx.debug.internalAssert(sh.eql(other._shape), "Created a {f} from Mlir value but expected {f}", .{ other._shape, res._shape }); + const eql_dtype = sh.dtype() == other.dtype(); + const eql_dims = std.mem.eql(i64, sh.dims(), other.dims()); + stdx.debug.internalAssert(eql_dims and eql_dtype, "Created a {f} from Mlir value but expected {f}", .{ other._shape, res._shape }); } return res; @@ -1208,71 +1210,130 @@ pub const Tensor = struct { return self; } - const res_type = mlir.Type.rankedTensor(self.shape().dims(), mlirx.Type.fromDType(mlirCtx(), to)); + const res_type = mlirx.Type.rankedTensor(mlirCtx(), self.shape().withDtype(to)); const op = dialects.stablehlo.convert(mlirCtx(), self.value(), res_type, .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape.withDtype(to), op.result(0)); } - test convert { + test "convert f32 -> f4e2m1" { const floats = @import("floats.zig"); const zml = @import("zml.zig"); const platform = zml.testing.env(); - - // f4e2m1 - { - const x = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; - var x_f4: [x.len]floats.Float4E2M1 = undefined; - for (&x_f4, &x) |*xi_f4, xi| xi_f4.* = .fromF32(xi); - - const x_d: Tensor = .init(.{x.len}, .f32); - const exe = try zml.module.compile( - std.testing.allocator, - std.testing.io, - Tensor.convert, - .{ x_d, .f4e2m1 }, - platform, - .{}, - ); - defer exe.deinit(); - - var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), .replicated, std.mem.sliceAsBytes(&x)); - defer x_d_buffer.deinit(); - - var x_f4_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); - const x_f4_xla = try x_f4_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); - defer x_f4_xla.free(std.testing.allocator); - - errdefer std.log.warn("convert(.f4e2m1) failed !\ninput f32:\n{e}\nzml.floats computed:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x), x_f4, x_f4_xla }); - try std.testing.expectEqualDeep(&x_f4, x_f4_xla.items(floats.Float4E2M1)); + const x_f32 = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; + var x_f4: [x_f32.len / 2]floats.Float4E2M1.Packed = undefined; + for (&x_f4, 0..) |*f4_pack, i| { + f4_pack.* = .fromF32(x_f32[2 * i], x_f32[2 * i + 1]); } - // f8e3m4 - { - const x = [_]f32{ 1.1 / 4.0, 1.1 / 8.0, 1.1 / 16.0, 1.1 / 32.0, 1.1 / 64.0, 1.1 / 128.0 }; - var x_f8e3: [x.len]floats.Float8E3M4 = undefined; - for (&x_f8e3, &x) |*xi_f8e3, xi| xi_f8e3.* = .fromF32(xi); + const x_d: Tensor = .init(.{x_f32.len}, .f32); + const exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.convert, + .{ x_d, .f4e2m1 }, + platform, + .{}, + ); + defer exe.deinit(); - const x_d: Tensor = .init(.{x.len}, .f32); - const exe = try zml.module.compile( - std.testing.allocator, - std.testing.io, - Tensor.convert, - .{ x_d, .f8e3m4 }, - platform, - .{}, - ); - defer exe.deinit(); + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), .replicated, @ptrCast(&x_f32)); + defer x_d_buffer.deinit(); - var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), .replicated, std.mem.sliceAsBytes(&x)); - defer x_d_buffer.deinit(); + var x_f4_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); + const x_f4_xla = try x_f4_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); + defer x_f4_xla.free(std.testing.allocator); + const x_f4_items = x_f4_xla.items(floats.Float4E2M1.Packed); - var x_f8e3_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); - const x_f8e3_xla = try x_f8e3_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); - defer x_f8e3_xla.free(std.testing.allocator); + errdefer std.log.warn("convert(f32 -> f4e2m1) failed !\ninput f32:\n{d}\nzml.floats computed:\n{d}\nxla computed:\n{d}", .{ stdx.fmt.slice(&x_f32), stdx.fmt.slice(&x_f4), x_f4_xla }); + try std.testing.expectEqualSlices(floats.Float4E2M1.Packed, &x_f4, x_f4_items); + } - errdefer std.log.warn("convert(.f8e3m4) failed !\ninput f32:\n{e}\nzml.floats computed:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x), x_f8e3, x_f8e3_xla }); - try std.testing.expectEqualDeep(&x_f8e3, x_f8e3_xla.items(floats.Float8E3M4)); - } + test "convert f32 -> f8e3m4" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const x = [_]f32{ 1.1 / 4.0, 1.1 / 8.0, 1.1 / 16.0, 1.1 / 32.0, 1.1 / 64.0, 1.1 / 128.0 }; + var x_f8e3: [x.len]zml.floats.Float8E3M4 = undefined; + for (&x_f8e3, &x) |*xi_f8e3, xi| xi_f8e3.* = .fromF32(xi); + + const x_d: Tensor = .init(.{x.len}, .f32); + const exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.convert, + .{ x_d, .f8e3m4 }, + platform, + .{}, + ); + defer exe.deinit(); + + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), .replicated, std.mem.sliceAsBytes(&x)); + defer x_d_buffer.deinit(); + + var x_f8e3_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); + const x_f8e3_xla = try x_f8e3_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); + defer x_f8e3_xla.free(std.testing.allocator); + + errdefer std.log.warn("convert(.f8e3m4) failed !\ninput f32:\n{d}\nzml.floats computed:\n{d}\nxla computed:\n{d}", .{ stdx.fmt.slice(&x), stdx.fmt.slice(&x_f8e3), x_f8e3_xla }); + try std.testing.expectEqualSlices(u8, @ptrCast(&x_f8e3), x_f8e3_xla.constData()); + } + + test "convert u2 -> u8" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const x_u2: [2]@Vector(4, u2) = .{ .{ 0, 1, 2, 3 }, .{ 3, 2, 1, 0 } }; + std.debug.assert(@sizeOf(@TypeOf(x_u2)) == 2); + const x_u8: [8]u8 = .{ 0, 1, 2, 3, 3, 2, 1, 0 }; + + const x_u2_t: Tensor = .init(.{8}, .u2); + const exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.convert, + .{ x_u2_t, .u8 }, + platform, + .{}, + ); + defer exe.deinit(); + + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_u2_t.shape(), .replicated, @ptrCast(&x_u2)); + defer x_d_buffer.deinit(); + + var x_u8_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); + defer x_u8_xla_d.deinit(); + const x_u8_xla = try x_u8_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); + defer x_u8_xla.free(std.testing.allocator); + + try std.testing.expectEqualSlices(u8, &x_u8, x_u8_xla.constData()); + } + + test "convert f4e2m1 -> f32" { + const floats = @import("floats.zig"); + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const x_f4_packed = [_]floats.Float4E2M1.Packed{ .fromF32(0, 0.5), .fromF32(1, 1.5), .fromF32(2, 3), .fromF32(4, 6) }; + + const x_t: Tensor = .init(.{x_f4_packed.len * 2}, .f4e2m1); + const exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.convert, + .{ x_t, .f32 }, + platform, + .{}, + ); + defer exe.deinit(); + + var x_f4_d: zml.Buffer = try .fromBytes(std.testing.io, platform, x_t.shape(), .replicated, @ptrCast(&x_f4_packed)); + defer x_f4_d.deinit(); + + var x_f32_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_f4_d}); + const x_f32_h = try x_f32_d.toSliceAlloc(std.testing.allocator, std.testing.io); + defer x_f32_h.free(std.testing.allocator); + + const x_f32_expected = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6 }; + errdefer std.log.warn("convert(f4e2m1x2 -> f32) failed !\ninput f4e2m1x2:\n{e}\nexpected:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x_f4_packed), x_f32_expected, x_f32_h }); + try std.testing.expectEqualSlices(f32, &x_f32_expected, x_f32_h.items(f32)); } /// Returns a Tensor containing the element-wise rounding operation of the input Tensor. diff --git a/zml/testing.zig b/zml/testing.zig index 8483a9be..dd8cd2a8 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -285,7 +285,11 @@ pub const CompareOpts = struct { std.debug.assert(left.len == right.len); for (left, right) |l, r| { - self.count_close += if (l == r) 1 else 0; + switch (@typeInfo(T)) { + .int, .float => self.count_close += if (l == r) 1 else 0, + .vector => |vec| self.count_close += @popCount(@as(@Int(.unsigned, vec.len), @bitCast(l == r))), + else => @panic("cmpIntSlices expect int or vector of int"), + } } self.processed_count += left.len; } From 6e171a5ef986da5c6d67239cd9b8328b625a6de6 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 31 Aug 2026 17:01:49 +0200 Subject: [PATCH 643/881] zml: remove implicit donation [bugfix] (#751) in #738 I accidentally introduced implicit donation `fn foo(x: Tensor) Tensor { return x; }` would not make a copy of the input tensor but return the same pointers. I'm not sure an implicit copies is better than an implicit aliasing, but fixing since this is an un-planned breaking change. Also fixing a compiler error for `.items(bool)` after #724. Note: this is an argument for snapshot testing: the CI should check if a given PR changes the emitted IR for a given model. --- zml/Compiler.zig | 19 ++++++++--- zml/buffer.zig | 5 +-- zml/slice.zig | 14 ++++++-- zml/tensor.zig | 85 ++++++++++++++++++++++++++++++++++++------------ 4 files changed, 93 insertions(+), 30 deletions(-) diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 641fd93d..fa662bd0 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -87,11 +87,16 @@ pub const Error = std.mem.Allocator.Error || pjrtx.Client.CompileError || error{MissingDeviceInTile}; +pub const Donation = union(enum) { + implicit: u32, + explicit: u32, +}; + pub const Scope = struct { compiler: *Compiler, block: *mlir.Block, id_to_argument: std.AutoArrayHashMapUnmanaged(Tensor.Id, *const mlir.Value), - id_to_donation: std.AutoArrayHashMapUnmanaged(Tensor.Id, usize), + id_to_donation: std.AutoArrayHashMapUnmanaged(Tensor.Id, Donation), id_to_memory: std.AutoArrayHashMapUnmanaged(Tensor.Id, Memory.Kind), arena: std.heap.ArenaAllocator, @@ -473,7 +478,7 @@ fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{Ou if (gop.found_existing) std.debug.panic("Tensor with id {} has already been used once as an argument", .{tensor.id}); // Associate each input argument with their own buffer - ctx.scope.id_to_donation.putNoClobber(ctx.scope.arena.allocator(), tensor.id, ctx.current_argument_id) catch @panic("OOM"); + ctx.scope.id_to_donation.putNoClobber(ctx.scope.arena.allocator(), tensor.id, .{ .implicit = ctx.current_argument_id }) catch @panic("OOM"); defer ctx.current_argument_id += 1; @@ -545,13 +550,17 @@ fn finalizeMlirFunc(compiler: *Compiler, fn_scope: *Scope, input_info: std.Multi _ = fn_return.appendTo(fn_scope.block); // Resolve donations - for (output_info.items(.id), 0..) |output_id, output_index| { - if (fn_scope.id_to_donation.get(output_id)) |donated_input| { + for (0.., output_info.items(.id)) |output_index, output_id| { + if (fn_scope.id_to_donation.get(output_id)) |donation| { + const donated_input = switch (donation) { + // don't emit implicit donation since they modify the way the function is called + .implicit => continue, + .explicit => |input| input, + }; const aliasing_output: *?u32 = &input_info.items(.aliasing_output)[donated_input]; if (aliasing_output.*) |previous_aliased_output| { std.debug.panic("Input {d} buffer {} was reused twice with `reuseBuffer` for output {} and output {}. Expected `reuseBuffer` to be called at most once", .{ donated_input, input_info.items(.shape)[donated_input], previous_aliased_output, output_index }); } - aliasing_output.* = @intCast(output_index); } } diff --git a/zml/buffer.zig b/zml/buffer.zig index d8c31b4e..25c319af 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -203,13 +203,14 @@ pub const Buffer = struct { pub fn uninitialized( _: std.Io, platform: *const Platform, - sh: Shape, + shape_: Shape, sharding: Sharding, opts: UnitializedOptions, ) !Buffer { + const sh = shape_.packedShape(); var res: Buffer = .{ ._platform = platform, - ._shape = sh, + ._shape = shape_, ._sharding = sharding.resolve(platform), ._shards = .empty, }; diff --git a/zml/slice.zig b/zml/slice.zig index f4e0d2ea..e7d239e5 100644 --- a/zml/slice.zig +++ b/zml/slice.zig @@ -147,11 +147,12 @@ pub const Slice = struct { } pub fn items(slice: Slice, comptime T: type) []T { - stdx.debug.assertComptime(@bitSizeOf(T) >= 8, "zml.Slice stores packed sub-bytes type so you need to pass a packed type here like @Vector({}, {}). Got: {}", .{ @divFloor(8, @bitSizeOf(T)), T, T }); + stdx.debug.assertComptime(T == bool or @bitSizeOf(T) >= 8, "zml.Slice stores packed sub-bytes type so you need to pass a packed type here like @Vector({}, {}). Got: {}", .{ @divFloor(8, @bitSizeOf(T)), T, T }); return @ptrCast(@alignCast(slice.data())); } pub fn constItems(slice: Slice, comptime T: type) []const T { + stdx.debug.assertComptime(T == bool or @bitSizeOf(T) >= 8, "zml.Slice stores packed sub-bytes type so you need to pass a packed type here like @Vector({}, {}). Got: {}", .{ @divFloor(8, @bitSizeOf(T)), T, T }); return @ptrCast(@alignCast(slice.constData())); } @@ -256,7 +257,8 @@ pub const Slice = struct { // Special case input tensor is a scalar return switch (slice.dtype()) { inline else => |dt| { - const val: dt.toZigType() = slice.constItems(dt.toZigType())[0]; + const single_ptr: *const dt.toZigType() = @ptrCast(@alignCast(slice.bytes.ptr)); + const val = single_ptr.*; return switch (comptime dt.class()) { // Since we have custom floats, we need to explicitly convert to float32 ourselves. .float => stdx.fmt.formatFloat(floats.floatCast(f32, val), options, writer), @@ -498,6 +500,14 @@ test "slice pretty print rank 0" { try std.testing.expectFmt(expected, "{d}", .{slice}); } +test "slice pretty print bool" { + const data: [2]bool = .{ true, false }; + const slice = Slice.init(.init(.{2}, .bool), std.mem.asBytes(&data)); + const expected = "{1,0}"; + try std.testing.expectEqualSlices(bool, &data, slice.constItems(bool)); + try std.testing.expectFmt(expected, "{d}", .{slice}); +} + test "slice pretty print ellipsis" { const data: [9][1]i32 = .{ .{0}, .{1}, .{2}, .{3}, .{4}, .{5}, .{6}, .{7}, .{8} }; const slice = Slice.init(.init(.{ 9, 1 }, .i32), std.mem.asBytes(&data)); diff --git a/zml/tensor.zig b/zml/tensor.zig index f26afad3..6c37dbf9 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -239,6 +239,35 @@ pub const Tensor = struct { return res; } + test toMemory { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const io = std.testing.io; + + const inputs: [8]f32 = .{ -3.0, -2, -1, 1, 2, 3, 5, -5 }; + const x_t = Tensor.init(.{8}, .f32); + + const Local = struct { + fn memcpyH2D(x: Tensor) Tensor { + return x.onMemory(.host_pinned).toMemory(.device); + } + }; + + const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.memcpyH2D, .{x_t}, platform, .{}); + defer exe.deinit(); + + var x_h = try zml.Buffer.fromBytesOpts(io, platform, x_t.shape(), .replicated, @ptrCast(&inputs), .{ .memory = .host_pinned }); + defer x_h.deinit(); + + const x_h_ptr: [*]f32 = @ptrCast(@alignCast(x_h.opaqueDevicePtr(0))); + try std.testing.expectEqualSlices(f32, &inputs, x_h_ptr[0..8]); + + var x_d = try zml.testing.autoCall(std.testing.allocator, io, &exe, Local.memcpyH2D, .{x_h}); + defer x_d.deinit(); + + try zml.testing.expectClose(std.testing.io, x_h, x_d, .exact_match); + } + /// Copy all the given tensor to the specified memory. /// The input struct is copied on the stack, so it must be a simple flat struct without pointers. pub fn toMemoryAll(flat_tensors: anytype, kind: Memory.Kind) @TypeOf(flat_tensors) { @@ -331,9 +360,13 @@ pub const Tensor = struct { pub fn reuseBuffer(self: Tensor, origin: Tensor) Tensor { const compilation_context = Compiler.current(); const scope = compilation_context.currentScope(); - if (scope.id_to_donation.get(origin.id)) |origin_donation| { - const gop = scope.id_to_donation.getOrPut(scope.arena.allocator(), self.id) catch unreachable; - gop.value_ptr.* = origin_donation; + if (scope.id_to_donation.get(origin.id)) |og_donation| { + // reuseBuffer is making the donation explicit + const donation: Compiler.Donation = switch (og_donation) { + .implicit => |input| .{ .explicit = input }, + .explicit => og_donation, + }; + scope.id_to_donation.put(scope.arena.allocator(), self.id, donation) catch @panic("OOM"); } return self; } @@ -343,39 +376,49 @@ pub const Tensor = struct { const platform = zml.testing.env(); const io = std.testing.io; - const inputs: [2][6]f32 = .{ .{ -3.0, -2, -1, 1, 2, 3 }, .{ 1, 2, 3, 4, 5, -5 } }; - const left = Tensor.init(.{ 2, 6 }, .f32); - const right = Tensor.init(.{ 2, 6 }, .f32); + const inputs: [6]@Vector(2, i4) = .{ .{ -3.0, -2 }, .{ -1, 1 }, .{ 2, 3 }, .{ 1, 2 }, .{ 3, 4 }, .{ 5, -5 } }; + const x_t = Tensor.init(.{ 6, 2 }, .i4); + const y_t = Tensor.init(.{ 6, 2 }, .i4); + const z_t = Tensor.init(.{ 6, 2 }, .i4); const Local = struct { - fn memcopy(x: Tensor, y: Tensor) Tensor { - return x.addConstant(1).reuseBuffer(y); + fn memcopy(x: Tensor, y: Tensor, z: Tensor) [2]Tensor { + // the first output reuses the given y, but the second ouput does NOT reuse z. + return .{ x.addConstant(1).reuseBuffer(y), z }; } }; - const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.memcopy, .{ left, right }, platform, .{}); + const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.memcopy, .{ x_t, y_t, z_t }, platform, .{}); defer exe.deinit(); - var left_d = try zml.Buffer.fromBytes(io, platform, left.shape(), .replicated, @ptrCast(&inputs)); - defer left_d.deinit(); + var x_d = try zml.Buffer.fromBytes(io, platform, x_t.shape(), .replicated, @ptrCast(&inputs)); + defer x_d.deinit(); - var right_d = try zml.Buffer.uninitialized(io, platform, left.shape(), .replicated, .{}); - defer right_d.deinit(); + var y_d = try zml.Buffer.uninitialized(io, platform, y_t.shape(), .replicated, .{}); + defer y_d.deinit(); - var right_memory: [Platform.MAX_NUM_DEVICES]*anyopaque = undefined; - for (0..right_d.numShards()) |dev| { - right_memory[dev] = right_d.opaqueDevicePtr(dev); + var z_d = try zml.Buffer.uninitialized(io, platform, z_t.shape(), .replicated, .{}); + defer z_d.deinit(); + + var y_memory: [Platform.MAX_NUM_DEVICES]*anyopaque = undefined; + for (0..y_d.numShards()) |dev| { + y_memory[dev] = y_d.opaqueDevicePtr(dev); } - const output_d = try zml.testing.autoCall(std.testing.allocator, io, &exe, Local.memcopy, .{ left_d, right_d }); + const y2_d, const z2_d = try zml.testing.autoCall(std.testing.allocator, io, &exe, Local.memcopy, .{ x_d, y_d, z_d }); var output_memory: [Platform.MAX_NUM_DEVICES]*anyopaque = undefined; - for (0..output_d.numShards()) |dev| { - output_memory[dev] = output_d.opaqueDevicePtr(dev); + for (0..y2_d.numShards()) |dev| { + output_memory[dev] = y2_d.opaqueDevicePtr(dev); } - // Check that right_d and output_d point to the same addresses on the devices. - try std.testing.expectEqualSlices(*anyopaque, right_memory[0..right_d.numShards()], output_memory[0..output_d.numShards()]); + // Check that y_d and y2_d point to the same addresses on the devices. + try std.testing.expectEqualSlices(*anyopaque, y_memory[0..y_d.numShards()], output_memory[0..y2_d.numShards()]); + + // Check that z_d and z2_d DO NOT POINT to the same addresses + for (0..z2_d.numShards()) |dev| { + try std.testing.expect(z2_d.opaqueDevicePtr(dev) != z_d.opaqueDevicePtr(dev)); + } } /// Returns a Tensor containing the absolute value of each element of the input Tensor. From 325cf2185ee53d714923ed0ba7922c5a9d9c2f95 Mon Sep 17 00:00:00 2001 From: nzinci <43704979+nzinci@users.noreply.github.com> Date: Tue, 1 Sep 2026 12:15:08 +0300 Subject: [PATCH 644/881] zml/tensor: fix triangularSolve batched RHS contract (#753) `Tensor.triangularSolve` currently rejects batched inputs and derives the result shape from the coefficient matrix. Allow tensors with rank 2 or greater and preserve the right-hand side's shape in the result. This was found while working on a separate Qwen3.5 GDN optimization. Tests: - `bazel test //zml:test` - `bazel build //zml` --- zml/tensor.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zml/tensor.zig b/zml/tensor.zig index 6c37dbf9..68ca695a 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -560,10 +560,10 @@ pub const Tensor = struct { /// Solves the system of linear equations formed by the input tensors. pub fn triangularSolve(self: Tensor, other: Tensor, opts: dialects.stablehlo.TriangularSolveOpts) Tensor { stdx.debug.assert(self.dtype() == other.dtype(), "triangularSolve expects tensors to be of the same type, got {} and {}", .{ self.dtype(), other.dtype() }); - stdx.debug.assert(self.rank() <= 2 and self.rank() == other.rank(), "triangularSolve expects tensors to have the same rank and be <= 2, got {} and {}", .{ self.rank(), other.rank() }); + stdx.debug.assert(self.rank() >= 2 and self.rank() == other.rank(), "triangularSolve expects tensors to have the same rank >= 2, got {} and {}", .{ self.rank(), other.rank() }); const op = dialects.stablehlo.triangular_solve(mlirCtx(), self.value(), other.value(), opts, .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape, op.result(0)); + return _result(other._shape, op.result(0)); } /// Returns a Tensor containing the element-wise rounding towards the nearest integer, breaking ties away from zero, of the input Tensor. From de134385c54828808e7c7301c1f3c85c07bfc76b Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 27 Aug 2026 22:59:34 +0200 Subject: [PATCH 645/881] zml/vfs: handle invalid GCS json we do not support "external_account", but for now at least do not crash when we can't parse the JSON file. --- vfs/gcs.zig | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/vfs/gcs.zig b/vfs/gcs.zig index 3824aa15..649e9958 100644 --- a/vfs/gcs.zig +++ b/vfs/gcs.zig @@ -111,7 +111,8 @@ const Credentials = union(enum) { ), }; } - unreachable; + log.warn("Unknown credentials type: {s}", .{obj_type.string}); + return std.json.ParseFromValueError.InvalidEnumTag; }, else => std.json.ParseFromValueError.UnexpectedToken, }; @@ -380,20 +381,23 @@ pub const GCS = struct { pub fn auto(allocator: std.mem.Allocator, inner_io: std.Io, http_client: *std.http.Client, environ_map: *std.process.Environ.Map) !GCS { var jsonBuffer: [1024]u8 = undefined; - if (environ_map.get("GOOGLE_APPLICATION_CREDENTIALS")) |json_path| { - var f = try std.Io.Dir.openFile(.cwd(), inner_io, json_path, .{}); - defer f.close(inner_io); - var reader = f.reader(inner_io, &jsonBuffer); - return try .init(allocator, inner_io, http_client, .{ .credentials = .{ .json = &reader.interface } }); - } + const credentials_file: ?std.Io.File = if (environ_map.get("GOOGLE_APPLICATION_CREDENTIALS")) |json_path| blk: { + break :blk std.Io.Dir.openFile(.cwd(), inner_io, json_path, .{}) catch |err| { + log.err("Unable to open GCS credentials file {s}, skipping: {t}", .{ json_path, err }); + break :blk null; + }; + } else if (applicationDefaultCredentials(inner_io, environ_map)) |f| + f + else + null; + defer if (credentials_file) |f| f.close(inner_io); - if (applicationDefaultCredentials(inner_io, environ_map)) |f| { - defer f.close(inner_io); + if (credentials_file) |f| blk: { var reader = f.reader(inner_io, &jsonBuffer); return GCS.init(allocator, inner_io, http_client, .{ .credentials = .{ .json = &reader.interface } }) catch |err| switch (err) { InitError.InvalidCredentialJson => { - log.warn("Invalid GCS credential JSON", .{}); - return .init(allocator, inner_io, http_client, .{}); + log.err("Invalid GCS credentials JSON", .{}); + break :blk; }, else => return err, }; From 6e5829a061673c4c568892923f3c68dd6070ef3b Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 1 Sep 2026 14:17:48 +0200 Subject: [PATCH 646/881] zml: Use zml.nn.Linear for forwardMoe and rename quantization structs (#725) `forwardMoe` is effectively consuming a `gate_up` and `down` linear MLP, it "just" only applies parts of it. I've renamed: ``` Quant -> Quantization QuantSchem -> Quantization.Scheme TensorScale -> Quantization.GlobalScale Direction { multiplier, divisor } -> Operation { multiply, divide } ``` and moved it outside of zml.nn.Linear This makes it easier in LLMD to just load zml.nn.Linear --- examples/llm/models/llama/model.zig | 23 +- examples/llm/models/qwen3_5/model.zig | 32 +-- examples/llm/models/qwen3_5_moe/model.zig | 66 +---- zml/BUILD.bazel | 1 + zml/moe/metal.zig | 11 +- zml/moe/moe.zig | 137 +++++----- zml/moe/triton.zig | 4 +- zml/nn.zig | 311 +--------------------- zml/quantization.zig | 282 ++++++++++++++++++++ zml/zml.zig | 1 + 10 files changed, 397 insertions(+), 471 deletions(-) create mode 100644 zml/quantization.zig diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index 7152952d..06cff075 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -97,7 +97,7 @@ pub const LoadedModel = struct { } pub fn unloadBuffers(_: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { - if (buffers.lm_head) |*lm_head| lm_head.weight.deinit(); + if (buffers.lm_head) |*lm_head| zml.nn.Linear.unloadBuffers(lm_head); Llama.unloadBuffers(&buffers.model, allocator); } @@ -189,7 +189,7 @@ pub const Model = struct { } pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { - if (self.lm_head) |*lm_head| lm_head.weight.deinit(); + if (self.lm_head) |*lm_head| zml.nn.Linear.unloadBuffers(lm_head); Llama.unloadBuffers(&self.model, allocator); } @@ -480,12 +480,7 @@ const Mlp = struct { } pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { - self.up_proj.weight.deinit(); - if (self.up_proj.bias) |*bias| bias.deinit(); - self.gate_proj.weight.deinit(); - if (self.gate_proj.bias) |*bias| bias.deinit(); - self.down_proj.weight.deinit(); - if (self.down_proj.bias) |*bias| bias.deinit(); + zml.Buffer.deinitAll(Mlp, self); } pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { @@ -530,17 +525,7 @@ const SelfAttn = struct { } pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { - self.q_proj.weight.deinit(); - if (self.q_proj.bias) |*bias| bias.deinit(); - self.k_proj.weight.deinit(); - if (self.k_proj.bias) |*bias| bias.deinit(); - self.v_proj.weight.deinit(); - if (self.v_proj.bias) |*bias| bias.deinit(); - self.o_proj.weight.deinit(); - if (self.o_proj.bias) |*bias| bias.deinit(); - - if (self.q_norm) |*q_norm| RmsNorm.unloadBuffers(q_norm); - if (self.k_norm) |*k_norm| RmsNorm.unloadBuffers(k_norm); + zml.Buffer.deinitAll(SelfAttn, self); } /// Self zml.attention. diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 90f1ced4..41601fad 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -111,7 +111,7 @@ pub const LoadedModel = struct { pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { _ = self; TextModel.unloadBuffers(&buffers.text_model, allocator); - buffers.lm_head.weight.deinit(); + zml.nn.Linear.unloadBuffers(&buffers.lm_head); } pub fn compile( @@ -209,7 +209,7 @@ pub const Model = struct { pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { TextModel.unloadBuffers(&self.text_model, allocator); - self.lm_head.weight.deinit(); + zml.nn.Linear.unloadBuffers(&self.lm_head); } pub fn forward( self: Model, @@ -517,12 +517,7 @@ pub const Mlp = struct { } pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { - self.up_proj.weight.deinit(); - if (self.up_proj.bias) |*bias| bias.deinit(); - self.gate_proj.weight.deinit(); - if (self.gate_proj.bias) |*bias| bias.deinit(); - self.down_proj.weight.deinit(); - if (self.down_proj.bias) |*bias| bias.deinit(); + zml.Buffer.deinitAll(Mlp, self); } pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { @@ -577,16 +572,7 @@ pub const SelfAttn = struct { } pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { - self.q_proj.weight.deinit(); - if (self.q_proj.bias) |*bias| bias.deinit(); - self.k_proj.weight.deinit(); - if (self.k_proj.bias) |*bias| bias.deinit(); - self.v_proj.weight.deinit(); - if (self.v_proj.bias) |*bias| bias.deinit(); - self.o_proj.weight.deinit(); - if (self.o_proj.bias) |*bias| bias.deinit(); - RmsNorm.unloadBuffers(&self.q_norm); - RmsNorm.unloadBuffers(&self.k_norm); + zml.Buffer.deinitAll(SelfAttn, self); } fn projectQAndGate(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { @@ -810,15 +796,7 @@ pub const GatedDeltaNet = struct { } pub fn unloadBuffers(self: *zml.Bufferized(GatedDeltaNet)) void { - self.in_proj_qkv.weight.deinit(); - self.in_proj_z.weight.deinit(); - self.in_proj_b.weight.deinit(); - self.in_proj_a.weight.deinit(); - self.out_proj.weight.deinit(); - self.conv1d_weight.deinit(); - self.dt_bias.deinit(); - self.aLog.deinit(); - RmsNormGated.unloadBuffers(&self.norm); + zml.Buffer.deinitAll(GatedDeltaNet, self); } fn recurrentGatedDeltaRule( diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 6b7b9bc7..e54a6f14 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -150,7 +150,7 @@ pub const LoadedModel = struct { progress: *std.Progress.Node, ) !inference.CompiledModel { _ = backend; - const moe_dtype = self.inner.text_model.layers[0].moe.gate_up_proj.dtype(); + const moe_dtype = self.inner.text_model.layers[0].moe.gate_up_proj.weight.dtype(); log.info("Moe dtype : {}", .{moe_dtype}); const moe_backend = try zml.moe.Backend.auto(platform, moe_dtype); const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), moe_backend, shardings); @@ -336,7 +336,7 @@ pub const TextModel = struct { } allocator.free(self.layers); RmsNorm.unloadBuffers(&self.norm); - self.lm_head.weight.deinit(); + zml.nn.Linear.unloadBuffers(&self.lm_head); } pub fn sampler(self: TextModel) Sampler { @@ -581,20 +581,7 @@ pub const SelfAttn = struct { } pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { - self.q_proj.weight.deinit(); - if (self.q_proj.bias) |*bias| bias.deinit(); - if (self.q_proj_scale) |*scale| scale.deinit(); - self.k_proj.weight.deinit(); - if (self.k_proj.bias) |*bias| bias.deinit(); - if (self.k_proj_scale) |*scale| scale.deinit(); - self.v_proj.weight.deinit(); - if (self.v_proj.bias) |*bias| bias.deinit(); - if (self.v_proj_scale) |*scale| scale.deinit(); - self.o_proj.weight.deinit(); - if (self.o_proj.bias) |*bias| bias.deinit(); - if (self.o_proj_scale) |*scale| scale.deinit(); - RmsNorm.unloadBuffers(&self.q_norm); - RmsNorm.unloadBuffers(&self.k_norm); + zml.Buffer.deinitAll(SelfAttn, self); } fn projectQAndGate(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { @@ -703,15 +690,7 @@ pub const Mlp = struct { } pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { - self.up_proj.weight.deinit(); - if (self.up_proj.bias) |*bias| bias.deinit(); - if (self.up_proj_scale) |*scale| scale.deinit(); - self.gate_proj.weight.deinit(); - if (self.gate_proj.bias) |*bias| bias.deinit(); - if (self.gate_proj_scale) |*scale| scale.deinit(); - self.down_proj.weight.deinit(); - if (self.down_proj.bias) |*bias| bias.deinit(); - if (self.down_proj_scale) |*scale| scale.deinit(); + zml.Buffer.deinitAll(Mlp, self); } pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { @@ -740,8 +719,7 @@ const Router = struct { } pub fn unloadBuffers(self: *zml.Bufferized(Router)) void { - self.router.weight.deinit(); - if (self.router.bias) |*bias| bias.deinit(); + zml.Buffer.deinitAll(Router, self); } pub fn forward(self: Router, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { @@ -756,8 +734,8 @@ const Router = struct { pub const Moe = struct { shared_expert: Mlp, shared_expert_gate: zml.nn.Linear, - gate_up_proj: zml.Tensor, - down_proj: zml.Tensor, + gate_up_proj: zml.nn.Linear, + down_proj: zml.nn.Linear, router: Router, pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config) !Moe { @@ -781,8 +759,8 @@ pub const Moe = struct { store.withPrefix("shared_expert_gate").maybeCreateTensor("bias", .{.dout}, .{ .dout = .replicated }), .d, ), - .gate_up_proj = gate_up_proj_tensor, - .down_proj = down_proj_tensor, + .gate_up_proj = .init(gate_up_proj_tensor, null, .d), + .down_proj = .init(down_proj_tensor, null, .dout), .router = Router.init(store.withPrefix("gate"), config.text_config.num_experts_per_tok), }; } @@ -800,13 +778,7 @@ pub const Moe = struct { topk_ids, routing_scores, self.gate_up_proj, - null, - null, self.down_proj, - null, - null, - null, - null, .{}, moe_metadata, moe_parameters, @@ -824,12 +796,7 @@ pub const Moe = struct { pub fn unloadBuffers(self: *zml.Bufferized(Moe), allocator: std.mem.Allocator) void { _ = allocator; - Mlp.unloadBuffers(&self.shared_expert); - self.shared_expert_gate.weight.deinit(); - if (self.shared_expert_gate.bias) |*bias| bias.deinit(); - self.gate_up_proj.deinit(); - self.down_proj.deinit(); - Router.unloadBuffers(&self.router); + zml.Buffer.deinitAll(Moe, self); } }; @@ -957,18 +924,7 @@ pub const GatedDeltaNet = struct { } pub fn unloadBuffers(self: *zml.Bufferized(GatedDeltaNet)) void { - self.in_proj_qkv.weight.deinit(); - if (self.in_proj_qkv_scale) |*scale| scale.deinit(); - self.in_proj_z.weight.deinit(); - if (self.in_proj_z_scale) |*scale| scale.deinit(); - self.in_proj_b.weight.deinit(); - self.in_proj_a.weight.deinit(); - self.out_proj.weight.deinit(); - if (self.out_proj_scale) |*scale| scale.deinit(); - self.conv1d_weight.deinit(); - self.dt_bias.deinit(); - self.aLog.deinit(); - RmsNormGated.unloadBuffers(&self.norm); + zml.Buffer.deinitAll(GatedDeltaNet, self); } fn recurrentGatedDeltaRule(query: zml.Tensor, key: zml.Tensor, value: zml.Tensor, g: zml.Tensor, beta: zml.Tensor, initial_state: ?zml.Tensor) struct { zml.Tensor, zml.Tensor } { diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 5d8304cb..9f49801d 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -114,6 +114,7 @@ zig_library( "platform.zig", "profiling/profiler.zig", "profiling/tracer.zig", + "quantization.zig", "safetensors.zig", "shape.zig", "slice.zig", diff --git a/zml/moe/metal.zig b/zml/moe/metal.zig index 607adba5..eae31e9e 100644 --- a/zml/moe/metal.zig +++ b/zml/moe/metal.zig @@ -189,25 +189,22 @@ pub fn fusedExpertsImpl( const expert_ids = topk_ids.reshape(.{ .token = num_tokens, .top_expert = top_expert }).withTags(.{ .token, .top_expert }) .merge(.{ .r = .{ .token, .top_expert } }).convert(.i32); - const gate_up_w = if (gate_up.dtype() == .u8) zml.nn.unpackFp4(gate_up, .d) else gate_up; - const down_w = if (down.dtype() == .u8) zml.nn.unpackFp4(down, .dout) else down; - const gate_up_out = try applyGateUpGlobalScale(moeGemm( x_rows, - gate_up_w, + gate_up, opts.w1_scale, expert_ids, - zml.Shape.init(.{ .r = num_routes, .out = gate_up_w.dim(.dout) }, act_dtype), + zml.Shape.init(.{ .r = num_routes, .out = gate_up.dim(.dout) }, act_dtype), mode, ), opts.w1_global_scale, expert_ids); const activated = applyActivation(gate_up_out, opts.activation); const down_out = try applyDownGlobalScale(moeGemm( activated, - down_w, + down, opts.w2_scale, expert_ids, - zml.Shape.init(.{ .r = num_routes, .d = down_w.dim(.d) }, act_dtype), + zml.Shape.init(.{ .r = num_routes, .d = down.dim(.d) }, act_dtype), mode, ), opts.w2_global_scale, expert_ids); diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index 752b0953..e34560d2 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -1,6 +1,7 @@ const std = @import("std"); const platforms = @import("platforms"); + const zml = @import("../zml.zig"); const stdx = zml.stdx; pub const cutlass_flashinfer = @import("cutlass_flashinfer.zig"); @@ -189,25 +190,28 @@ pub const Metadata = union(Backend) { pub const Options = struct { activation_threshold: ?f32 = null, - quant_scheme: ?zml.nn.QuantScheme = null, }; pub fn forwardMoe( input: zml.Tensor, topk_ids: zml.Tensor, topk_weights: zml.Tensor, - weights_gate_up: zml.Tensor, - scales_gate_up: ?zml.Tensor, - bias_gate_up: ?zml.Tensor, - weights_down: zml.Tensor, - scales_down: ?zml.Tensor, - bias_down: ?zml.Tensor, - w1_global_scale: ?zml.Tensor, - w2_global_scale: ?zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, opts: Options, metadata: Metadata, parameters: Parameters, ) !zml.Tensor { + const gate_up_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; + const down_scheme: ?zml.Quantization.Scheme = if (down.quantization) |q| q.scheme else null; + if (gate_up_scheme != down_scheme) return error.UnsupportedQuantization; + + const gate_up_scales: ?zml.Tensor = if (gate_up.quantization) |q| q.scales else null; + const down_scales: ?zml.Tensor = if (down.quantization) |q| q.scales else null; + const gate_up_global_scale: ?zml.Tensor = if (gate_up.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; + const down_global_scale: ?zml.Tensor = if (down.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; + const quant_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; + return switch (parameters) { .flashinfer_cutlass => b: { if (comptime !platforms.isEnabled(.cuda)) { @@ -217,17 +221,16 @@ pub fn forwardMoe( .flashinfer_cutlass => |v| v, else => return error.InvalidMetadata, }; - if (scales_gate_up != null or scales_down != null) { - return error.UnsupportedQuantization; - } - if (bias_gate_up != null or bias_down != null) { + if (gate_up.bias != null or down.bias != null) { return error.UnsupportedBias; } const runner_options = try parameters.flashinfer_cutlass.runnerOptions(); - const expert_partition = weights_gate_up.shape().partition(.expert); + const expert_partition = gate_up.weight.shape().partition(.expert); if (flashinfer_metadata.variant == .nvfp4xnvfp4) { + const gate_up_weight_unpacked = unpackedWeight(gate_up); + const down_weight_unpacked = unpackedWeight(down); const nvfp4 = flashinfer_metadata.nvfp4_scales orelse return error.MissingNvfp4Scales; if (expert_partition.eql(.init(.experts))) { @@ -236,8 +239,8 @@ pub fn forwardMoe( input, topk_ids, topk_weights, - weights_gate_up, - weights_down, + gate_up_weight_unpacked, + down_weight_unpacked, nvfp4.fc1_act_global, nvfp4.fc1_weight_block, nvfp4.fc1_global, @@ -313,8 +316,8 @@ pub fn forwardMoe( break :b try cutlass_flashinfer.fusedExpertsNvfp4( input, - weights_gate_up, - weights_down, + gate_up_weight_unpacked, + down_weight_unpacked, topk_weights, topk_ids, nvfp4, @@ -327,7 +330,7 @@ pub fn forwardMoe( if (expert_partition.eql(.init(.experts))) { break :b zml.ops.manualComputation( - .{ input, topk_ids, topk_weights, weights_gate_up, weights_down }, + .{ input, topk_ids, topk_weights, gate_up.weight, down.weight }, input.shape(), .{ .activation = runner_options.activation, @@ -388,8 +391,8 @@ pub fn forwardMoe( break :b try cutlass_flashinfer.fusedExpertsBf16( input, - weights_gate_up, - weights_down, + gate_up.weight, + down.weight, topk_weights, topk_ids, runner_options, @@ -401,25 +404,25 @@ pub fn forwardMoe( else => return error.InvalidMetadata, }; - const global_num_experts = weights_gate_up.dim(.expert); - const expert_partition = weights_gate_up.shape().partition(.expert); + const global_num_experts = gate_up.weight.dim(.expert); + const expert_partition = gate_up.weight.shape().partition(.expert); if (!expert_partition.eql(.init(.experts))) { break :b try triton.fusedExpertsImpl( input, - weights_gate_up, - weights_down, + gate_up.weight, + down.weight, topk_weights, topk_ids, triton_metadata, .{ .activation = parameters.triton.activation, .global_num_experts = global_num_experts, - .w1_scale = scales_gate_up, - .w2_scale = scales_down, - .w1_bias = bias_gate_up, - .w2_bias = bias_down, - .quant_scheme = opts.quant_scheme, + .w1_scale = gate_up_scales, + .w2_scale = down_scales, + .w1_bias = gate_up.bias, + .w2_bias = down.bias, + .quant_scheme = quant_scheme, .activation_threshold = opts.activation_threshold, }, ); @@ -430,19 +433,19 @@ pub fn forwardMoe( input, topk_ids, topk_weights, - weights_gate_up, - weights_down, + gate_up.weight, + down.weight, }, input.shape(), .{ .activation = parameters.triton.activation, .global_num_experts = global_num_experts, - .bias_gate_up = bias_gate_up, - .bias_down = bias_down, - .quant_scheme = opts.quant_scheme, + .bias_gate_up = gate_up.bias, + .bias_down = down.bias, + .quant_scheme = quant_scheme, .activation_threshold = opts.activation_threshold, - .scales_gate_up = scales_gate_up, - .scales_down = scales_down, + .scales_gate_up = gate_up_scales, + .scales_down = down_scales, }, (struct { fn body(ctx: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, _: zml.Shape) zml.Tensor { @@ -491,20 +494,20 @@ pub fn forwardMoe( else => return error.InvalidMetadata, }; - const expert_partition = weights_gate_up.shape().partition(.expert); + const expert_partition = gate_up.weight.shape().partition(.expert); if (expert_partition.eql(.init(.experts))) { - const global_num_experts = weights_down.dim(.expert); + const global_num_experts = down.weight.dim(.expert); const partial_output = zml.ops.manualComputation( - .{ input, topk_ids, topk_weights, weights_gate_up, weights_down }, + .{ input, topk_ids, topk_weights, gate_up.weight, down.weight }, input.shape(), .{ .activation = parameters.mosaic_tpu.activation, .global_num_experts = global_num_experts, - .scales_gate_up = scales_gate_up, - .bias_gate_up = bias_gate_up, - .scales_down = scales_down, - .bias_down = bias_down, + .gate_up_scales = gate_up_scales, + .bias_gate_up = gate_up.bias, + .down_scales = down_scales, + .bias_down = down.bias, }, (struct { fn body(ctx: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, _: zml.Shape) zml.Tensor { @@ -530,8 +533,8 @@ pub fn forwardMoe( .activation = ctx.activation, .global_num_experts = ctx.global_num_experts, .expert_map = expert_map, - .w1_scale = ctx.scales_gate_up, - .w2_scale = ctx.scales_down, + .w1_scale = ctx.gate_up_scales, + .w2_scale = ctx.down_scales, .w1_bias = ctx.bias_gate_up, .w2_bias = ctx.bias_down, }, @@ -545,22 +548,24 @@ pub fn forwardMoe( break :b try mosaic_tpu.fusedExpertsImpl( input, - weights_gate_up, - weights_down, + gate_up.weight, + down.weight, topk_weights, topk_ids, tpu_metadata, .{ .activation = parameters.mosaic_tpu.activation, - .global_num_experts = weights_gate_up.dim(.expert), - .w1_scale = scales_gate_up, - .w2_scale = scales_down, - .w1_bias = bias_gate_up, - .w2_bias = bias_down, + .global_num_experts = gate_up.weight.dim(.expert), + .w1_scale = gate_up_scales, + .w2_scale = down_scales, + .w1_bias = gate_up.bias, + .w2_bias = down.bias, }, ); }, .metal => b: { + const gate_up_weight_unpacked = unpackedWeight(gate_up); + const down_weight_unpacked = unpackedWeight(down); const metal_metadata = switch (metadata) { .metal => |v| v, else => return error.InvalidMetadata, @@ -568,22 +573,30 @@ pub fn forwardMoe( break :b try metal.fusedExpertsImpl( input, - weights_gate_up, - weights_down, + gate_up_weight_unpacked, + down_weight_unpacked, topk_weights, topk_ids, metal_metadata, .{ .activation = parameters.metal.activation, - .global_num_experts = weights_gate_up.dim(.expert), - .w1_scale = scales_gate_up, - .w2_scale = scales_down, - .w1_global_scale = w1_global_scale, - .w2_global_scale = w2_global_scale, - .w1_bias = bias_gate_up, - .w2_bias = bias_down, + .global_num_experts = gate_up_weight_unpacked.dim(.expert), + .w1_scale = gate_up_scales, + .w2_scale = down_scales, + .w1_global_scale = gate_up_global_scale, + .w2_global_scale = down_global_scale, + .w1_bias = gate_up.bias, + .w2_bias = down.bias, }, ); }, }; } + +fn unpackedWeight(linear: zml.nn.Linear) zml.Tensor { + const quantization = linear.quantization orelse return linear.weight; + return if (zml.nn.isPackedFp4(quantization.scheme, linear.weight.dtype())) + zml.nn.unpackFp4(linear.weight, linear.tag) + else + linear.weight; +} diff --git a/zml/moe/triton.zig b/zml/moe/triton.zig index a399bdb3..28c92be0 100644 --- a/zml/moe/triton.zig +++ b/zml/moe/triton.zig @@ -28,7 +28,7 @@ pub const Options = struct { inplace: bool = false, activation: Parameters.ActivationMode = .silu, apply_router_weight_on_input: bool = false, - quant_scheme: ?zml.nn.QuantScheme = null, + quant_scheme: ?zml.Quantization.Scheme = null, global_num_experts: i64 = -1, expert_map: ?Tensor = null, w1_scale: ?Tensor = null, @@ -125,7 +125,7 @@ fn applyActivation(x: Tensor, mode: Parameters.ActivationMode) Tensor { // Top-level entry point // ============================================================================= -fn isMxFp8(quant_scheme: ?zml.nn.QuantScheme) bool { +fn isMxFp8(quant_scheme: ?zml.Quantization.Scheme) bool { return quant_scheme != null and quant_scheme.? == .mxfp8; } diff --git a/zml/nn.zig b/zml/nn.zig index 4d34de8c..9e328127 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -8,6 +8,8 @@ const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); const ops = @import("ops.zig"); +const quantization = @import("quantization.zig"); +pub const Quantization = quantization.Quantization; const Shape = @import("shape.zig").Shape; const Slice = @import("slice.zig").Slice; const Tensor = @import("tensor.zig").Tensor; @@ -19,32 +21,7 @@ pub const Linear = struct { weight: Tensor, bias: ?Tensor = null, tag: Shape.Tag, - quant: ?Quant = null, - - pub const Quant = struct { - scheme: QuantScheme, // decided when the checkpoint is read - scales: Tensor, - weight_scale: ?TensorScale = null, - input_scale: ?TensorScale = null, - }; - - /// A per-tensor scale and the polarity its producer wrote it in - pub const TensorScale = struct { - value: Tensor, - direction: Direction, - - pub const Direction = enum { multiplier, divisor }; - - pub fn asMultiplier(self: TensorScale) Tensor { - const s = self.value.convert(.f32); - const flat = if (s.shape().count() == 1) s.asScalar() else s; // essentially normalization - - return switch (self.direction) { - .multiplier => flat, - .divisor => Tensor.scalar(1.0, .f32).broad(flat.shape()).div(flat), - }; - } - }; + quantization: ?Quantization = null, pub fn init(weight: Tensor, bias: ?Tensor, tag: anytype) Linear { return .{ @@ -64,10 +41,9 @@ pub const Linear = struct { } fn forwardWeight(self: Linear, x: Tensor) Tensor { - const q = self.quant orelse return x.dot(self.weight, self.tag); + const q = self.quantization orelse return x.dot(self.weight, self.tag); - const wgs: ?Tensor = if (q.weight_scale) |s| s.asMultiplier() else null; - const igs: ?Tensor = if (q.input_scale) |s| s.asMultiplier() else null; + const weight_global_scale: ?Tensor = if (q.global_scale) |s| s.asMultiplier() else null; const weight = if (isPackedFp4(q.scheme, self.weight.dtype())) unpackFp4(self.weight, self.tag) else self.weight; const scales = if (q.scheme.isMx() and q.scales.dtype() == .u8) @@ -77,235 +53,24 @@ pub const Linear = struct { var lhs = x.convert(.bf16); var lhs_scale: ?Tensor = null; - var undo: ?Tensor = null; + var undo_input_scale: ?Tensor = null; const platform = zml.Compiler.current().platform; - if (q.scheme.activationQuant()) |aq| { - if (aq.supportedOn(platform)) { - const quantized = aq.apply(lhs, igs, self.tag); - lhs = quantized.values; - lhs_scale = quantized.scales; - undo = igs; - } + if (quantization.quantizeInput(q, lhs, self.tag, platform)) |quantized_input| { + lhs = quantized_input.values; + lhs_scale = quantized_input.scales; + undo_input_scale = quantized_input.global_scale; } const acc = scaledDot(lhs, weight, lhs_scale, scales, self.tag); - return applyGlobalScale(acc, undo, wgs).convert(x.dtype()); + return applyGlobalScale(acc, undo_input_scale, weight_global_scale).convert(x.dtype()); } }; -const nvfp4_block_size = 16; -const mx_block_size = 32; - -pub const QuantScheme = enum { - /// f4e2m1 values (`u8`-packed or native), f8e4m3fn scale per 16 contracted values. - /// Emitted by llm-compressor and by NVIDIA's ModelOpt. - nvfp4, - /// f8e4m3fn values, one e8m0 (power-of-two) scale per 32 contracted values - mxfp8, - /// f4e2m1 values (`u8`-packed or native), e8m0 scale per 32 contracted values. - // GPT-OSS like - mxfp4, - /// f8e4m3fn values, one bf16 or f32 scale per output channel, constant along the contraction. - /// Emitted by llm-compressor, including for the layers an NVFP4 recipe leaves in FP8. - fp8_per_channel, - /// f8e4m3fn values, one bf16 scale per 128x128 tile. The DeepSeek-style FP8 that model - /// vendors publish themselves, under `weight_scale_inv`. - fp8_block128, - /// f8e4m3fn values, one scale for the whole tensor. Spelled `[1, 1]` rather than as a - /// scalar: XLA's composite rewriter requires the scale to have the operand's rank. - fp8_per_tensor, - - pub fn accepts(self: QuantScheme, weight: Shape, scale: Shape) bool { - if (weight.rank() != 2) return false; - - const n = weight.dim(0); - const k = if (isPackedFp4(self, weight.dtype())) 2 * weight.dim(1) else weight.dim(1); - - return switch (self) { - .nvfp4 => (weight.dtype() == .u8 or weight.dtype() == .f4e2m1) and - scale.dtype() == .f8e4m3fn and scale.rank() == 2 and - scale.dim(0) == n and scale.dim(1) * nvfp4_block_size == k, - .mxfp8 => weight.dtype() == .f8e4m3fn and isMxScale(scale) and - scale.dim(0) == n and scale.dim(1) * mx_block_size == k, - .mxfp4 => (weight.dtype() == .u8 or weight.dtype() == .i8 or weight.dtype() == .f4e2m1) and - isMxScale(scale) and - scale.dim(0) == n and scale.dim(1) * mx_block_size == k, - .fp8_per_tensor => weight.dtype() == .f8e4m3fn and scale.count() == 1, - .fp8_per_channel => weight.dtype() == .f8e4m3fn and - (scale.dtype() == .bf16 or scale.dtype() == .f32) and - scale.count() > 1 and scale.rank() == 2 and - scale.dim(0) == n and scale.dim(1) == 1, - .fp8_block128 => weight.dtype() == .f8e4m3fn and scale.dtype() == .bf16 and - scale.count() > 1 and scale.rank() == 2 and - @rem(n, 128) == 0 and @rem(k, 128) == 0 and - scale.dim(0) == @divExact(n, 128) and scale.dim(1) == @divExact(k, 128), - }; - } - - pub fn classify(weight: Shape, scale: Shape) ?QuantScheme { - for (std.enums.values(QuantScheme)) |scheme| { - if (scheme.accepts(weight, scale)) return scheme; - } - - return null; - } - - pub fn activationQuant(self: QuantScheme) ?ActivationQuant { - return switch (self) { - .nvfp4 => .nvfp4, - .mxfp8, .mxfp4, .fp8_per_channel, .fp8_block128, .fp8_per_tensor => null, - }; - } - - pub fn isMx(self: QuantScheme) bool { - return self == .mxfp8 or self == .mxfp4; - } -}; - -pub const ActivationQuant = enum { - nvfp4, - - fn supportedOn(self: ActivationQuant, platform: *const zml.Platform) bool { - return switch (self) { - .nvfp4 => supportsNvfp4ActivationQuant(platform), - }; - } - - fn apply(self: ActivationQuant, x: Tensor, global_scale: ?Tensor, axis: Shape.Tag) QuantizedActivations { - return switch (self) { - .nvfp4 => quantizeNvfp4(x, global_scale, axis), - }; - } -}; - -pub fn isPackedFp4(scheme: ?QuantScheme, weight_dtype: DataType) bool { +pub fn isPackedFp4(scheme: ?Quantization.Scheme, weight_dtype: DataType) bool { return (scheme == .nvfp4 or scheme == .mxfp4) and (weight_dtype == .u8 or weight_dtype == .i8); } -fn isMxScale(scale: Shape) bool { - return scale.rank() == 2 and (scale.dtype() == .f8e8m0 or scale.dtype() == .u8); -} - -// Note: this test will evolve as we support more (INT4/8 and FP8 block-128 as a -// scaled-dot are still open; see the scaledDot doc comment). -test "QuantScheme.classify" { - const expect = std.testing.expectEqual; - - // unsloth/Qwen3.6-27B-NVFP4: compressed-tensors, carrying both of its schemes under - // `weight_scale` -- NVFP4 on the MLPs of layers 0-55, FP8 per-channel everywhere else. - const nvfp4_packed: Shape = .init(.{ .dout = 17408, .kw = 2560 }, .u8); - try expect(@as(?QuantScheme, .nvfp4), QuantScheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 320 }, .f8e4m3fn))); - try expect(@as(?QuantScheme, .fp8_per_channel), QuantScheme.classify( - .init(.{ .dout = 17408, .d = 5120 }, .f8e4m3fn), - .init(.{ .dout = 17408, .sc = 1 }, .bf16), - )); - - // nvidia/Gemma-4-31B-IT-NVFP4: ModelOpt, packed values under the plain `weight` name. - try expect(@as(?QuantScheme, .nvfp4), QuantScheme.classify( - .init(.{ .dout = 21504, .kw = 2688 }, .u8), - .init(.{ .dout = 21504, .sc = 336 }, .f8e4m3fn), - )); - - // Native (unpacked) f4e2m1, with K no longer doubled. - try expect(@as(?QuantScheme, .nvfp4), QuantScheme.classify( - .init(.{ .dout = 17408, .d = 5120 }, .f4e2m1), - .init(.{ .dout = 17408, .sc = 320 }, .f8e4m3fn), - )); - - // Qwen/Qwen3.6-27B-FP8: block 128, spelled `weight_scale_inv`. - try expect(@as(?QuantScheme, .fp8_block128), QuantScheme.classify( - .init(.{ .dout = 17408, .d = 5120 }, .f8e4m3fn), - .init(.{ .dout = 136, .sc = 40 }, .bf16), - )); - try expect(@as(?QuantScheme, .fp8_block128), QuantScheme.classify( - .init(.{ .dout = 5120, .d = 6144 }, .f8e4m3fn), - .init(.{ .dout = 40, .sc = 48 }, .bf16), - )); - - // Mistral's per-tensor FP8: one scale for the whole tensor, rank 0 or [1]. - try expect(@as(?QuantScheme, .fp8_per_tensor), QuantScheme.classify(.init(.{ .dout = 4096, .d = 4096 }, .f8e4m3fn), .init(.{}, .f32))); - try expect(@as(?QuantScheme, .fp8_per_tensor), QuantScheme.classify(.init(.{ .dout = 4096, .d = 4096 }, .f8e4m3fn), .init(.{ .g = 1 }, .f32))); - - // RadixArk/Muse-Glimmer-NVFP4: ModelOpt's MIXED_PRECISION recipe, whose MXFP8 half - // lands on down_proj and lm_head. safetensors has no e8m0 dtype, so the scale arrives - // as u8 under `weight_scale_inv` -- the shape below is verbatim from the checkpoint. - try expect(@as(?QuantScheme, .mxfp8), QuantScheme.classify( - .init(.{ .dout = 6656, .d = 19968 }, .f8e4m3fn), - .init(.{ .dout = 6656, .sc = 624 }, .u8), - )); - try expect(@as(?QuantScheme, .mxfp8), QuantScheme.classify( - .init(.{ .dout = 202048, .d = 6656 }, .f8e4m3fn), - .init(.{ .dout = 202048, .sc = 208 }, .u8), - )); - // The same tensor from a store that types the scale natively. - try expect(@as(?QuantScheme, .mxfp8), QuantScheme.classify( - .init(.{ .dout = 6656, .d = 19968 }, .f8e4m3fn), - .init(.{ .dout = 6656, .sc = 624 }, .f8e8m0), - )); - // MXFP8 rejects any group but 32: 19968 / 16 = 1248, / 64 = 312. - try expect(@as(?QuantScheme, null), QuantScheme.classify( - .init(.{ .dout = 6656, .d = 19968 }, .f8e4m3fn), - .init(.{ .dout = 6656, .sc = 1248 }, .u8), - )); - try expect(@as(?QuantScheme, null), QuantScheme.classify( - .init(.{ .dout = 6656, .d = 19968 }, .f8e4m3fn), - .init(.{ .dout = 6656, .sc = 312 }, .u8), - )); - - // MXFP4: the same u8 packing as NVFP4 (two nibbles per byte, K halved on disk), with - // an e8m0 scale per 32 -- so K = 2 * 2560 = 5120 and 5120 / 32 = 160. - try expect(@as(?QuantScheme, .mxfp4), QuantScheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 160 }, .u8))); - try expect(@as(?QuantScheme, .mxfp4), QuantScheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 160 }, .f8e8m0))); - // DeepSeek V4 stores the same packed FP4 bits in signed bytes. - try expect(@as(?QuantScheme, .mxfp4), QuantScheme.classify( - .init(.{ .dout = 2048, .kw = 2048 }, .i8), - .init(.{ .dout = 2048, .sc = 128 }, .f8e8m0), - )); - // Native (unpacked) f4e2m1, K no longer halved. - try expect(@as(?QuantScheme, .mxfp4), QuantScheme.classify( - .init(.{ .dout = 17408, .d = 5120 }, .f4e2m1), - .init(.{ .dout = 17408, .sc = 160 }, .f8e8m0), - )); - // An e8m0 scale never turns a non-fp4/fp8 weight into MX. - try expect(@as(?QuantScheme, null), QuantScheme.classify( - .init(.{ .dout = 17408, .d = 5120 }, .bf16), - .init(.{ .dout = 17408, .sc = 160 }, .f8e8m0), - )); - - // Rejected: everything no backend here can express. - const fp8: Shape = .init(.{ .dout = 10240, .d = 5120 }, .f8e4m3fn); - // Group 32 with an e4m3 scale: neither NVFP4 (wrong group) nor MX (wrong scale type). - try expect(@as(?QuantScheme, null), QuantScheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 160 }, .f8e4m3fn))); - // Group 16 with an e8m0 scale: the mirror image, and equally unclaimed. - try expect(@as(?QuantScheme, null), QuantScheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 320 }, .f8e8m0))); - // Per-channel in f32: what fusing per-tensor projections produces. The Metal - // per-channel kernels have f32 entries, so this is a real scheme, not a decline. - try expect(@as(?QuantScheme, .fp8_per_channel), QuantScheme.classify(fp8, .init(.{ .dout = 10240, .sc = 1 }, .f32))); - // A block grid that is neither per-channel nor 128x128. - try expect(@as(?QuantScheme, null), QuantScheme.classify(fp8, .init(.{ .dout = 160, .sc = 80 }, .bf16))); - // Stacked MoE weights are rank 3 and go through the MoE path, not this one. - try expect(@as(?QuantScheme, null), QuantScheme.classify( - .init(.{ .expert = 128, .dout = 1408, .kw = 1408 }, .u8), - .init(.{ .expert = 128, .dout = 1408, .sc = 176 }, .f8e4m3fn), - )); - - // `fp8_per_tensor` never checks the scale dtype, so a [1, 1] u8 scale on a [1, 32] - // weight satisfies it as well as `mxfp8`. Declaration order is what decides, and - // `mxfp8` is declared first - try expect(@as(?QuantScheme, .mxfp8), QuantScheme.classify( - .init(.{ .dout = 1, .d = 32 }, .f8e4m3fn), - .init(.{ .dout = 1, .sc = 1 }, .u8), - )); - - // A one-tile block grid is indistinguishable from a per-tensor scale by shape alone; the - // `count() > 1` guard on the grids is what makes per-tensor win. - try expect(@as(?QuantScheme, .fp8_per_tensor), QuantScheme.classify( - .init(.{ .dout = 128, .d = 128 }, .f8e4m3fn), - .init(.{ .dout = 1, .sc = 1 }, .bf16), - )); -} - /// Unpacks two f4e2m1 values per byte. `w` must be tagged with `.kw` on the packed axis, /// which is merged with the unpacked pair and renamed to `k_tag`. Group-size agnostic: /// NVFP4 (16) and MXFP4 (32) share this packing. @@ -316,46 +81,6 @@ pub fn unpackFp4(w: Tensor, k_tag: anytype) Tensor { .renameTag(.kb, Shape.toTag(k_tag)); } -pub const QuantizedActivations = struct { - values: Tensor, - scales: Tensor, -}; - -pub fn quantizeNvfp4(x: Tensor, input_global_scale: ?Tensor, axis: anytype) QuantizedActivations { - const block_size = nvfp4_block_size; - const value_max = 6.0; - const scale_min_normal = 0x1p-6; - const scale_max = DataType.f8e4m3fn.maxValue().as(f32); - - stdx.debug.assert(x.shape().hasTag(axis) != null, "quantizeNvfp4 expects x to have {any} tag, got {f}", .{ axis, x.shape() }); - stdx.debug.assert(@rem(x.dim(axis), block_size) == 0, "quantizeNvfp4 expects {any} to be a multiple of {}, got {f}", .{ axis, block_size, x.shape() }); - - const dt = x.dtype(); - const scaled = if (input_global_scale) |igs| - x.div(igs.convert(dt).broad(x.shape())) - else - x; - const grouped = scaled.splitAxis(axis, .{ .sc = -1, .blk = block_size }); - const amax = grouped.abs().max(.blk); - - const scales = amax.scale(1.0 / value_max) - .clamp(.scalar(scale_min_normal, dt), .scalar(scale_max, dt)) - .convert(.f8e4m3fn); - - const divisor = scales.convert(dt) - .maximum(.scalar(scale_min_normal, dt)) - .broad(grouped.shape()); - - const values = grouped.div(divisor) - .convert(.f4e2m1) - .reshape(x.shape().withDtype(.f4e2m1)); - - return .{ - .values = values, - .scales = scales.squeeze(.blk), - }; -} - /// Scaled matrix multiply (`xla.scaled_dot`): `acc = (lhs * lhs_scale) @ (rhs * rhs_scale)`. /// /// - **NVFP4**: values `.f4e2m1`, scales `.f8e4m3fn`, block 16 (weight-only bf16 lhs ok) @@ -475,18 +200,6 @@ fn scaledDotReference(in: []const Tensor, out_shape: Shape) Tensor { ); } -fn supportsNvfp4ActivationQuant(platform: *const zml.Platform) bool { - if (platform.target != .cuda) { - return false; - } - - const device = platform.pjrt_client.devices(platform.pjrt_api)[0]; - const capability = zml.platform.cuda.tryGetComputeCapabilities(platform, device) orelse return false; - const major = std.fmt.parseInt(u8, std.mem.sliceTo(capability, '.'), 10) catch return false; - - return major >= 10; -} - fn applyGlobalScale(acc: Tensor, igs: ?Tensor, wgs: ?Tensor) Tensor { if (igs == null and wgs == null) { return acc; diff --git a/zml/quantization.zig b/zml/quantization.zig new file mode 100644 index 00000000..f57f60b7 --- /dev/null +++ b/zml/quantization.zig @@ -0,0 +1,282 @@ +//! Quantization metadata and scheme classification. +const std = @import("std"); +const stdx = @import("stdx"); + +const DataType = @import("dtype.zig").DataType; +const platform_mod = @import("platform.zig"); +const Platform = platform_mod.Platform; +const Shape = @import("shape.zig").Shape; +const Tensor = @import("tensor.zig").Tensor; + +pub const nvfp4_block_size = 16; +pub const mx_block_size = 32; + +pub const Quantization = struct { + scheme: Scheme, // decided when the checkpoint is read + scales: Tensor, + global_scale: ?GlobalScale = null, + input_scale: ?GlobalScale = null, + + /// A per-tensor scale and the polarity its producer wrote it in + pub const GlobalScale = struct { + value: Tensor, + operation: Operation, + + pub const Operation = enum { multiply, divide }; + + pub fn asMultiplier(self: GlobalScale) Tensor { + const s = self.value.convert(.f32); + const flat = if (s.shape().count() == 1) s.asScalar() else s; // essentially normalization + + return switch (self.operation) { + .multiply => flat, + .divide => Tensor.scalar(1.0, .f32).broad(flat.shape()).div(flat), + }; + } + }; + + pub const Scheme = enum { + /// f4e2m1 values (`u8`-packed or native), f8e4m3fn scale per 16 contracted values. + /// Emitted by llm-compressor and by NVIDIA's ModelOpt. + nvfp4, + /// f8e4m3fn values, one e8m0 (power-of-two) scale per 32 contracted values + mxfp8, + /// f4e2m1 values (`u8`-packed or native), e8m0 scale per 32 contracted values. + // GPT-OSS like + mxfp4, + /// f8e4m3fn values, one bf16 or f32 scale per output channel, constant along the contraction. + /// Emitted by llm-compressor, including for the layers an NVFP4 recipe leaves in FP8. + fp8_per_channel, + /// f8e4m3fn values, one bf16 scale per 128x128 tile. The DeepSeek-style FP8 that model + /// vendors publish themselves, under `weight_scale_inv`. + fp8_block128, + /// f8e4m3fn values, one scale for the whole tensor. Spelled `[1, 1]` rather than as a + /// scalar: XLA's composite rewriter requires the scale to have the operand's rank. + fp8_per_tensor, + + pub fn accepts(self: Scheme, weight: Shape, scale: Shape) bool { + if (weight.rank() != 2) return false; + + const n = weight.dim(0); + const k = if (isPackedFp4(self, weight.dtype())) 2 * weight.dim(1) else weight.dim(1); + + return switch (self) { + .nvfp4 => (weight.dtype() == .u8 or weight.dtype() == .f4e2m1) and + scale.dtype() == .f8e4m3fn and scale.rank() == 2 and + scale.dim(0) == n and scale.dim(1) * nvfp4_block_size == k, + .mxfp8 => weight.dtype() == .f8e4m3fn and isMxScale(scale) and + scale.dim(0) == n and scale.dim(1) * mx_block_size == k, + .mxfp4 => (weight.dtype() == .u8 or weight.dtype() == .i8 or weight.dtype() == .f4e2m1) and + isMxScale(scale) and + scale.dim(0) == n and scale.dim(1) * mx_block_size == k, + .fp8_per_tensor => weight.dtype() == .f8e4m3fn and scale.count() == 1, + .fp8_per_channel => weight.dtype() == .f8e4m3fn and + (scale.dtype() == .bf16 or scale.dtype() == .f32) and + scale.count() > 1 and scale.rank() == 2 and + scale.dim(0) == n and scale.dim(1) == 1, + .fp8_block128 => weight.dtype() == .f8e4m3fn and scale.dtype() == .bf16 and + scale.count() > 1 and scale.rank() == 2 and + @rem(n, 128) == 0 and @rem(k, 128) == 0 and + scale.dim(0) == @divExact(n, 128) and scale.dim(1) == @divExact(k, 128), + }; + } + + pub fn classify(weight: Shape, scale: Shape) ?Scheme { + for (std.enums.values(Scheme)) |scheme| { + if (scheme.accepts(weight, scale)) return scheme; + } + + return null; + } + + pub fn isMx(self: Scheme) bool { + return self == .mxfp8 or self == .mxfp4; + } + }; +}; + +pub const QuantizedInput = struct { + values: Tensor, + scales: Tensor, + global_scale: ?Tensor, +}; + +pub fn quantizeInput(quantization: Quantization, input: Tensor, axis: Shape.Tag, platform: *const Platform) ?QuantizedInput { + const global_scale: ?Tensor = if (quantization.input_scale) |scale| scale.asMultiplier() else null; + return switch (quantization.scheme) { + .nvfp4 => if (supportsNvfp4InputQuantization(platform)) quantizeNvfp4(input, global_scale, axis) else null, + .mxfp8, .mxfp4, .fp8_per_channel, .fp8_block128, .fp8_per_tensor => null, + }; +} + +pub fn quantizeNvfp4(x: Tensor, input_global_scale: ?Tensor, axis: anytype) QuantizedInput { + const value_max = 6.0; + const scale_min_normal = 0x1p-6; + const scale_max = DataType.f8e4m3fn.maxValue().as(f32); + + stdx.debug.assert(x.shape().hasTag(axis) != null, "quantizeNvfp4 expects x to have {any} tag, got {f}", .{ axis, x.shape() }); + stdx.debug.assert(@rem(x.dim(axis), nvfp4_block_size) == 0, "quantizeNvfp4 expects {any} to be a multiple of {}, got {f}", .{ axis, nvfp4_block_size, x.shape() }); + + const dt = x.dtype(); + const scaled = if (input_global_scale) |igs| + x.div(igs.convert(dt).broad(x.shape())) + else + x; + const grouped = scaled.splitAxis(axis, .{ .sc = -1, .blk = nvfp4_block_size }); + const amax = grouped.abs().max(.blk); + + const scales = amax.scale(1.0 / value_max) + .clamp(.scalar(scale_min_normal, dt), .scalar(scale_max, dt)) + .convert(.f8e4m3fn); + + const divisor = scales.convert(dt) + .maximum(.scalar(scale_min_normal, dt)) + .broad(grouped.shape()); + + const values = grouped.div(divisor) + .convert(.f4e2m1) + .reshape(x.shape().withDtype(.f4e2m1)); + + return .{ + .values = values, + .scales = scales.squeeze(.blk), + .global_scale = input_global_scale, + }; +} + +fn supportsNvfp4InputQuantization(platform: *const Platform) bool { + if (platform.target != .cuda) return false; + + const device = platform.pjrt_client.devices(platform.pjrt_api)[0]; + const capability = platform_mod.cuda.tryGetComputeCapabilities(platform, device) orelse return false; + const major = std.fmt.parseInt(u8, std.mem.sliceTo(capability, '.'), 10) catch return false; + + return major >= 10; +} + +fn isPackedFp4(scheme: ?Quantization.Scheme, weight_dtype: DataType) bool { + return (scheme == .nvfp4 or scheme == .mxfp4) and (weight_dtype == .u8 or weight_dtype == .i8); +} + +fn isMxScale(scale: Shape) bool { + return scale.rank() == 2 and (scale.dtype() == .f8e8m0 or scale.dtype() == .u8); +} + +// Note: this test will evolve as we support more (INT4/8 and FP8 block-128 as a +// scaled-dot are still open; see the scaledDot doc comment). +test "Quantization.Scheme.classify" { + const expect = std.testing.expectEqual; + + // unsloth/Qwen3.6-27B-NVFP4: compressed-tensors, carrying both of its schemes under + // `weight_scale` -- NVFP4 on the MLPs of layers 0-55, FP8 per-channel everywhere else. + const nvfp4_packed: Shape = .init(.{ .dout = 17408, .kw = 2560 }, .u8); + try expect(@as(?Quantization.Scheme, .nvfp4), Quantization.Scheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 320 }, .f8e4m3fn))); + try expect(@as(?Quantization.Scheme, .fp8_per_channel), Quantization.Scheme.classify( + .init(.{ .dout = 17408, .d = 5120 }, .f8e4m3fn), + .init(.{ .dout = 17408, .sc = 1 }, .bf16), + )); + + // nvidia/Gemma-4-31B-IT-NVFP4: ModelOpt, packed values under the plain `weight` name. + try expect(@as(?Quantization.Scheme, .nvfp4), Quantization.Scheme.classify( + .init(.{ .dout = 21504, .kw = 2688 }, .u8), + .init(.{ .dout = 21504, .sc = 336 }, .f8e4m3fn), + )); + + // Native (unpacked) f4e2m1, with K no longer doubled. + try expect(@as(?Quantization.Scheme, .nvfp4), Quantization.Scheme.classify( + .init(.{ .dout = 17408, .d = 5120 }, .f4e2m1), + .init(.{ .dout = 17408, .sc = 320 }, .f8e4m3fn), + )); + + // Qwen/Qwen3.6-27B-FP8: block 128, spelled `weight_scale_inv`. + try expect(@as(?Quantization.Scheme, .fp8_block128), Quantization.Scheme.classify( + .init(.{ .dout = 17408, .d = 5120 }, .f8e4m3fn), + .init(.{ .dout = 136, .sc = 40 }, .bf16), + )); + try expect(@as(?Quantization.Scheme, .fp8_block128), Quantization.Scheme.classify( + .init(.{ .dout = 5120, .d = 6144 }, .f8e4m3fn), + .init(.{ .dout = 40, .sc = 48 }, .bf16), + )); + + // Mistral's per-tensor FP8: one scale for the whole tensor, rank 0 or [1]. + try expect(@as(?Quantization.Scheme, .fp8_per_tensor), Quantization.Scheme.classify(.init(.{ .dout = 4096, .d = 4096 }, .f8e4m3fn), .init(.{}, .f32))); + try expect(@as(?Quantization.Scheme, .fp8_per_tensor), Quantization.Scheme.classify(.init(.{ .dout = 4096, .d = 4096 }, .f8e4m3fn), .init(.{ .g = 1 }, .f32))); + + // RadixArk/Muse-Glimmer-NVFP4: ModelOpt's MIXED_PRECISION recipe, whose MXFP8 half + // lands on down_proj and lm_head. safetensors has no e8m0 dtype, so the scale arrives + // as u8 under `weight_scale_inv` -- the shape below is verbatim from the checkpoint. + try expect(@as(?Quantization.Scheme, .mxfp8), Quantization.Scheme.classify( + .init(.{ .dout = 6656, .d = 19968 }, .f8e4m3fn), + .init(.{ .dout = 6656, .sc = 624 }, .u8), + )); + try expect(@as(?Quantization.Scheme, .mxfp8), Quantization.Scheme.classify( + .init(.{ .dout = 202048, .d = 6656 }, .f8e4m3fn), + .init(.{ .dout = 202048, .sc = 208 }, .u8), + )); + // The same tensor from a store that types the scale natively. + try expect(@as(?Quantization.Scheme, .mxfp8), Quantization.Scheme.classify( + .init(.{ .dout = 6656, .d = 19968 }, .f8e4m3fn), + .init(.{ .dout = 6656, .sc = 624 }, .f8e8m0), + )); + // MXFP8 rejects any group but 32: 19968 / 16 = 1248, / 64 = 312. + try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify( + .init(.{ .dout = 6656, .d = 19968 }, .f8e4m3fn), + .init(.{ .dout = 6656, .sc = 1248 }, .u8), + )); + try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify( + .init(.{ .dout = 6656, .d = 19968 }, .f8e4m3fn), + .init(.{ .dout = 6656, .sc = 312 }, .u8), + )); + + // MXFP4: the same u8 packing as NVFP4 (two nibbles per byte, K halved on disk), with + // an e8m0 scale per 32 -- so K = 2 * 2560 = 5120 and 5120 / 32 = 160. + try expect(@as(?Quantization.Scheme, .mxfp4), Quantization.Scheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 160 }, .u8))); + try expect(@as(?Quantization.Scheme, .mxfp4), Quantization.Scheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 160 }, .f8e8m0))); + // DeepSeek V4 stores the same packed FP4 bits in signed bytes. + try expect(@as(?Quantization.Scheme, .mxfp4), Quantization.Scheme.classify( + .init(.{ .dout = 2048, .kw = 2048 }, .i8), + .init(.{ .dout = 2048, .sc = 128 }, .f8e8m0), + )); + // Native (unpacked) f4e2m1, K no longer halved. + try expect(@as(?Quantization.Scheme, .mxfp4), Quantization.Scheme.classify( + .init(.{ .dout = 17408, .d = 5120 }, .f4e2m1), + .init(.{ .dout = 17408, .sc = 160 }, .f8e8m0), + )); + // An e8m0 scale never turns a non-fp4/fp8 weight into MX. + try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify( + .init(.{ .dout = 17408, .d = 5120 }, .bf16), + .init(.{ .dout = 17408, .sc = 160 }, .f8e8m0), + )); + + // Rejected: everything no backend here can express. + const fp8: Shape = .init(.{ .dout = 10240, .d = 5120 }, .f8e4m3fn); + // Group 32 with an e4m3 scale: neither NVFP4 (wrong group) nor MX (wrong scale type). + try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 160 }, .f8e4m3fn))); + // Group 16 with an e8m0 scale: the mirror image, and equally unclaimed. + try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 320 }, .f8e8m0))); + // Per-channel in f32: what fusing per-tensor projections produces. The Metal + // per-channel kernels have f32 entries, so this is a real scheme, not a decline. + try expect(@as(?Quantization.Scheme, .fp8_per_channel), Quantization.Scheme.classify(fp8, .init(.{ .dout = 10240, .sc = 1 }, .f32))); + // A block grid that is neither per-channel nor 128x128. + try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify(fp8, .init(.{ .dout = 160, .sc = 80 }, .bf16))); + // Stacked MoE weights are rank 3 and go through the MoE path, not this one. + try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify( + .init(.{ .expert = 128, .dout = 1408, .kw = 1408 }, .u8), + .init(.{ .expert = 128, .dout = 1408, .sc = 176 }, .f8e4m3fn), + )); + + // `fp8_per_tensor` never checks the scale dtype, so a [1, 1] u8 scale on a [1, 32] + // weight satisfies it as well as `mxfp8`. Declaration order is what decides, and + // `mxfp8` is declared first + try expect(@as(?Quantization.Scheme, .mxfp8), Quantization.Scheme.classify( + .init(.{ .dout = 1, .d = 32 }, .f8e4m3fn), + .init(.{ .dout = 1, .sc = 1 }, .u8), + )); + + // A one-tile block grid is indistinguishable from a per-tensor scale by shape alone; the + // `count() > 1` guard on the grids is what makes per-tensor win. + try expect(@as(?Quantization.Scheme, .fp8_per_tensor), Quantization.Scheme.classify( + .init(.{ .dout = 128, .d = 128 }, .f8e4m3fn), + .init(.{ .dout = 1, .sc = 1 }, .bf16), + )); +} diff --git a/zml/zml.zig b/zml/zml.zig index d72fb786..9461f6a3 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -33,6 +33,7 @@ pub const meta = @import("meta.zig"); pub const mlir = @import("mlirx.zig"); pub const moe = @import("moe/moe.zig"); pub const nn = @import("nn.zig"); +pub const Quantization = @import("quantization.zig").Quantization; pub const ops = @import("ops.zig"); pub const pjrtx = @import("pjrtx.zig"); pub const platform = @import("platform.zig"); From e1e983c8029e1224130d04f9d134d8a25a7fed4a Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Tue, 1 Sep 2026 16:11:57 +0200 Subject: [PATCH 647/881] zml/moe: support Flashinfer cutlass MoE for sm 10.3 (#740) Update flashinfer cutlass moe to retrieve sm103 compiled .so. Unpack forwardMoe abi by removing the Nvfp4Scales type. --------- Co-authored-by: Louis Lechevalier Co-authored-by: Corentin Godeau --- .../flashinfer_cutlass_moe.zig | 34 +++++--- .../flashinfer_cutlass_moe.BUILD.bazel | 6 ++ third_party/flashinfer_cutlass_moe/repo.bzl | 6 +- zml/moe/cutlass_flashinfer.zig | 86 +++++++++---------- zml/moe/moe.zig | 47 +++++----- 5 files changed, 98 insertions(+), 81 deletions(-) diff --git a/platforms/cuda/flashinfer_cutlass_moe/flashinfer_cutlass_moe.zig b/platforms/cuda/flashinfer_cutlass_moe/flashinfer_cutlass_moe.zig index a8d01ea8..0b4360cf 100644 --- a/platforms/cuda/flashinfer_cutlass_moe/flashinfer_cutlass_moe.zig +++ b/platforms/cuda/flashinfer_cutlass_moe/flashinfer_cutlass_moe.zig @@ -136,16 +136,30 @@ pub fn apiForDevice(device: i32) !*Api { } test "select host and GPU architecture-specific FlashInfer CUTLASS MoE library" { - var buffer: [160]u8 = undefined; - const path = try libraryRunfile(120, &buffer); - const expected = switch (builtin.cpu.arch) { - .aarch64 => "flashinfer_cutlass_moe_linux_arm64/lib/libflashinfer_cutlass_moe_sm120.so", - .x86_64 => "flashinfer_cutlass_moe_linux_amd64/lib/libflashinfer_cutlass_moe_sm120.so", - else => return, + const test_case = &.{ + .{ + .compat_version = 120, + .expected = switch (builtin.cpu.arch) { + .aarch64 => "flashinfer_cutlass_moe_linux_arm64/lib/libflashinfer_cutlass_moe_sm120.so", + .x86_64 => "flashinfer_cutlass_moe_linux_amd64/lib/libflashinfer_cutlass_moe_sm120.so", + }, + }, + .{ + .compat_version = 103, + .expected = switch (builtin.cpu.arch) { + .aarch64 => "flashinfer_cutlass_moe_linux_arm64/lib/libflashinfer_cutlass_moe_sm103.so", + .x86_64 => "flashinfer_cutlass_moe_linux_amd64/lib/libflashinfer_cutlass_moe_sm103.so", + }, + }, }; - try std.testing.expectEqualStrings(expected, path); - try load(std.testing.allocator, std.testing.io, 120); - const api = if (loaded_api) |*value| value else return error.NotLoaded; - try std.testing.expectEqual(@as(i32, 120), api.compiledSm()); + inline for (test_case) |t| { + var buffer: [160]u8 = undefined; + const path = try libraryRunfile(t.compat_version, &buffer); + try std.testing.expectEqualStrings(t.expected, path); + + try load(std.testing.allocator, std.testing.io, 120); + const api = if (loaded_api) |*value| value else return error.NotLoaded; + try std.testing.expectEqual(@as(i32, 120), api.compiledSm()); + } } diff --git a/third_party/flashinfer_cutlass_moe/flashinfer_cutlass_moe.BUILD.bazel b/third_party/flashinfer_cutlass_moe/flashinfer_cutlass_moe.BUILD.bazel index b51d3392..037c2b10 100644 --- a/third_party/flashinfer_cutlass_moe/flashinfer_cutlass_moe.BUILD.bazel +++ b/third_party/flashinfer_cutlass_moe/flashinfer_cutlass_moe.BUILD.bazel @@ -12,6 +12,11 @@ filegroup( srcs = ["lib/libflashinfer_cutlass_moe_sm100.so"], ) +filegroup( + name = "lib_sm103", + srcs = ["lib/libflashinfer_cutlass_moe_sm103.so"], +) + filegroup( name = "lib_sm120", srcs = ["lib/libflashinfer_cutlass_moe_sm120.so"], @@ -21,6 +26,7 @@ filegroup( name = "libraries", srcs = [ ":lib_sm100", + ":lib_sm103", ":lib_sm120", ":lib_sm90", ], diff --git a/third_party/flashinfer_cutlass_moe/repo.bzl b/third_party/flashinfer_cutlass_moe/repo.bzl index 60b2b70f..2f897c2d 100644 --- a/third_party/flashinfer_cutlass_moe/repo.bzl +++ b/third_party/flashinfer_cutlass_moe/repo.bzl @@ -1,14 +1,14 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -_RELEASE = "cutlass-moe-v0.1.0" +_RELEASE = "cutlass-moe-v0.2.0" _ASSETS = { "amd64": { - "sha256": "d40348ad771e2bc34b3dd59602ed5f3ef47b58da06ddad582a1fd993f872fb40", + "sha256": "5c3a29da0fd91e2281d046eb2e467a7a40aeb24a0be8ba0487006356d45fecc4", "url": "https://github.com/zml/flashinfer/releases/download/{release}/flashinfer-cutlass-moe_linux-amd64.tar.gz", }, "arm64": { - "sha256": "4278b3900d789e2ad6c207f7e698728a0b68ab200a0fc815625dfc25a2a6e1a8", + "sha256": "60b6fdbd193e11c24589add1eb206f59a3900cde4760e53719d6b2add492343e", "url": "https://github.com/zml/flashinfer/releases/download/{release}/flashinfer-cutlass-moe_linux-arm64.tar.gz", }, } diff --git a/zml/moe/cutlass_flashinfer.zig b/zml/moe/cutlass_flashinfer.zig index 47f21dab..2efdbd45 100644 --- a/zml/moe/cutlass_flashinfer.zig +++ b/zml/moe/cutlass_flashinfer.zig @@ -70,44 +70,25 @@ pub const Parameters = struct { } }; -pub const Nvfp4Scales = struct { - /// Dynamic activation quantization multiplier. - fc1_act_global: zml.Tensor, - /// Interleaved E4M3 scales with shape returned by fc1BlockScaleShape(). - fc1_weight_block: zml.Tensor, - /// Final GEMM1 output alpha, one value per expert. - fc1_global: zml.Tensor, - /// Dynamic activation quantization multiplier. - fc2_act_global: zml.Tensor, - /// Interleaved E4M3 scales with shape returned by fc2BlockScaleShape(). - fc2_weight_block: zml.Tensor, - /// Final GEMM2 output alpha, one value per expert. - fc2_global: zml.Tensor, -}; - pub const Metadata = struct { variant: Variant = .bf16xbf16, - nvfp4_scales: ?Nvfp4Scales = null, pub const InitOptions = struct { variant: Variant = .bf16xbf16, - nvfp4_scales: ?Nvfp4Scales = null, }; pub fn init(opts: InitOptions) Metadata { return .{ .variant = opts.variant, - .nvfp4_scales = opts.nvfp4_scales, }; } pub fn initBuffer( - self: Metadata, + _: Metadata, _: std.Io, _: *const zml.Platform, ) !zml.Bufferized(Metadata) { - if (self.nvfp4_scales != null) return error.Nvfp4ScaleBuffersRequired; - return .{ .nvfp4_scales = null }; + return {}; } }; @@ -385,6 +366,7 @@ fn computeCapability(platform: *const zml.Platform) !u16 { if (std.mem.eql(u8, compute_capability, "9.0")) return 90; if (std.mem.eql(u8, compute_capability, "10.0")) return 100; + if (std.mem.eql(u8, compute_capability, "10.3")) return 103; if (std.mem.eql(u8, compute_capability, "12.0")) return 120; return error.UnsupportedArchitecture; } @@ -424,7 +406,10 @@ pub fn isAvailable(platform: *const zml.Platform) bool { pub fn isNvfp4Supported(platform: *const zml.Platform) bool { if (platform.state.cuda.fi_cutlass_moe_runners == null) return false; const compute_capability = computeCapability(platform) catch return false; - return compute_capability == 100 or compute_capability == 120; + return switch (compute_capability) { + 100, 103, 120 => true, + else => false, + }; } pub fn tacticCounts( @@ -485,7 +470,12 @@ fn validateInputs( fc2_weights: zml.Tensor, topk_weights: zml.Tensor, topk_ids: zml.Tensor, - scales: Nvfp4Scales, + fc1_act_global: zml.Tensor, + fc1_weight_block: zml.Tensor, + fc1_global: zml.Tensor, + fc2_act_global: zml.Tensor, + fc2_weight_block: zml.Tensor, + fc2_global: zml.Tensor, options: Options, ) !Attributes { if (hidden_states.dtype() != .bf16 or @@ -493,10 +483,10 @@ fn validateInputs( fc2_weights.dtype() != .f4e2m1 or topk_weights.dtype() != .f32 or topk_ids.dtype() != .i32 or - scales.fc1_weight_block.dtype() != .f8e4m3fn or - scales.fc2_weight_block.dtype() != .f8e4m3fn or - scales.fc1_global.dtype() != .f32 or - scales.fc2_global.dtype() != .f32) + fc1_weight_block.dtype() != .f8e4m3fn or + fc2_weight_block.dtype() != .f8e4m3fn or + fc1_global.dtype() != .f32 or + fc2_global.dtype() != .f32) { return error.UnsupportedType; } @@ -536,21 +526,21 @@ fn validateInputs( topk_weights.dim(2) != top_k or topk_ids.dim(0) != batch or topk_ids.dim(1) != sequence or - !scales.fc1_weight_block.shape().eql(fc1BlockScaleShape( + !fc1_weight_block.shape().eql(fc1BlockScaleShape( num_experts, hidden_size, intermediate_size, options.activation, )) or - !scales.fc2_weight_block.shape().eql(fc2BlockScaleShape( + !fc2_weight_block.shape().eql(fc2BlockScaleShape( num_experts, hidden_size, intermediate_size, )) or - !scales.fc1_global.shape().eql(.init(.{num_experts}, .f32)) or - !scales.fc2_global.shape().eql(.init(.{num_experts}, .f32)) or - !isGlobalOrPerExpertScale(scales.fc1_act_global, num_experts) or - !isGlobalOrPerExpertScale(scales.fc2_act_global, num_experts)) + !fc1_global.shape().eql(.init(.{num_experts}, .f32)) or + !fc2_global.shape().eql(.init(.{num_experts}, .f32)) or + !isGlobalOrPerExpertScale(fc1_act_global, num_experts) or + !isGlobalOrPerExpertScale(fc2_act_global, num_experts)) { return error.InvalidShape; } @@ -563,8 +553,8 @@ fn validateInputs( .top_k = @intCast(top_k), .activation = cutlassActivation(options.activation), .enable_pdl = options.enable_pdl, - .fc1_act_per_expert = scales.fc1_act_global.rank() == 1, - .fc2_act_per_expert = scales.fc2_act_global.rank() == 1, + .fc1_act_per_expert = fc1_act_global.rank() == 1, + .fc2_act_per_expert = fc2_act_global.rank() == 1, .gemm1_tactic = options.gemm1_tactic, .gemm2_tactic = options.gemm2_tactic, }; @@ -579,7 +569,12 @@ pub fn fusedExpertsNvfp4( fc2_weights: zml.Tensor, topk_weights: zml.Tensor, topk_ids: zml.Tensor, - scales: Nvfp4Scales, + fc1_act_global: zml.Tensor, + fc1_weight_block: zml.Tensor, + fc1_global: zml.Tensor, + fc2_act_global: zml.Tensor, + fc2_weight_block: zml.Tensor, + fc2_global: zml.Tensor, options: Options, ) !zml.Tensor { const runners = try currentRunners(); @@ -589,7 +584,12 @@ pub fn fusedExpertsNvfp4( fc2_weights, topk_weights, topk_ids, - scales, + fc1_act_global, + fc1_weight_block, + fc1_global, + fc2_act_global, + fc2_weight_block, + fc2_global, options, ); attributes.runners = @intFromPtr(runners); @@ -613,12 +613,12 @@ pub fn fusedExpertsNvfp4( .fc2_weights = fc2_weights, .topk_weights = topk_weights, .topk_ids = topk_ids, - .fc1_act_global = scales.fc1_act_global, - .fc1_weight_block = scales.fc1_weight_block, - .fc1_global = scales.fc1_global, - .fc2_act_global = scales.fc2_act_global, - .fc2_weight_block = scales.fc2_weight_block, - .fc2_global = scales.fc2_global, + .fc1_act_global = fc1_act_global, + .fc1_weight_block = fc1_weight_block, + .fc1_global = fc1_global, + .fc2_act_global = fc2_act_global, + .fc2_weight_block = fc2_weight_block, + .fc2_global = fc2_global, }, .{ .output = hidden_states.shape(), diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index e34560d2..e34996c1 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -217,10 +217,6 @@ pub fn forwardMoe( if (comptime !platforms.isEnabled(.cuda)) { return error.UnsupportedPlatform; } - const flashinfer_metadata = switch (metadata) { - .flashinfer_cutlass => |v| v, - else => return error.InvalidMetadata, - }; if (gate_up.bias != null or down.bias != null) { return error.UnsupportedBias; } @@ -228,11 +224,12 @@ pub fn forwardMoe( const runner_options = try parameters.flashinfer_cutlass.runnerOptions(); const expert_partition = gate_up.weight.shape().partition(.expert); - if (flashinfer_metadata.variant == .nvfp4xnvfp4) { + if (quant_scheme != null and quant_scheme == .nvfp4) { const gate_up_weight_unpacked = unpackedWeight(gate_up); const down_weight_unpacked = unpackedWeight(down); - const nvfp4 = flashinfer_metadata.nvfp4_scales orelse - return error.MissingNvfp4Scales; + + // TODO(Corentin): Do error checking on nvfp4 + // Also, maybe pass `zml.nn.Linear` directly if (expert_partition.eql(.init(.experts))) { break :b zml.ops.manualComputation( .{ @@ -241,12 +238,12 @@ pub fn forwardMoe( topk_weights, gate_up_weight_unpacked, down_weight_unpacked, - nvfp4.fc1_act_global, - nvfp4.fc1_weight_block, - nvfp4.fc1_global, - nvfp4.fc2_act_global, - nvfp4.fc2_weight_block, - nvfp4.fc2_global, + gate_up.quantization.?.input_scale.?.asMultiplier(), + gate_up.quantization.?.scales, + gate_up.quantization.?.global_scale.?.asMultiplier(), + down.quantization.?.input_scale.?.asMultiplier(), + down.quantization.?.scales, + down.quantization.?.global_scale.?.asMultiplier(), }, input.shape(), .{ @@ -286,14 +283,12 @@ pub fn forwardMoe( sharded_inputs[4], local_topk_weights, local_topk_ids, - .{ - .fc1_act_global = sharded_inputs[5], - .fc1_weight_block = sharded_inputs[6], - .fc1_global = sharded_inputs[7], - .fc2_act_global = sharded_inputs[8], - .fc2_weight_block = sharded_inputs[9], - .fc2_global = sharded_inputs[10], - }, + sharded_inputs[5], + sharded_inputs[6], + sharded_inputs[7], + sharded_inputs[8], + sharded_inputs[9], + sharded_inputs[10], .{ .workspace_query_device = ctx.workspace_query_device, .activation = ctx.activation, @@ -320,13 +315,15 @@ pub fn forwardMoe( down_weight_unpacked, topk_weights, topk_ids, - nvfp4, + gate_up.quantization.?.input_scale.?.asMultiplier(), + gate_up.quantization.?.scales, + gate_up.quantization.?.global_scale.?.asMultiplier(), + down.quantization.?.input_scale.?.asMultiplier(), + down.quantization.?.scales, + down.quantization.?.global_scale.?.asMultiplier(), runner_options, ); } - if (flashinfer_metadata.nvfp4_scales != null) { - return error.UnexpectedNvfp4Scales; - } if (expert_partition.eql(.init(.experts))) { break :b zml.ops.manualComputation( From 5d0686de0294d8e06b9703dc55af48e2821e5ae5 Mon Sep 17 00:00:00 2001 From: nzinci <43704979+nzinci@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:10:18 +0300 Subject: [PATCH 648/881] zml/safetensors: close model discovery handles (#758) Model discovery leaves its safetensors entrypoint and repository handles open after building the tensor registry, including when parsing fails. Close handles owned by `fromRepo` and `fromPath`, and close repositories at the existing `fromRepo` call sites. --- examples/llm/main.zig | 1 + examples/llm/models/lfm2_tests.zig | 1 + examples/llm/models/llama_tests.zig | 1 + zml/safetensors.zig | 16 +++++++++------- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/examples/llm/main.zig b/examples/llm/main.zig index 2684dbec..4407aadb 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -103,6 +103,7 @@ pub fn main(init: std.process.Init) !void { // log.info("Resolving model repository..", .{}); const repo = try zml.safetensors.resolveModelRepo(io, args.model); + defer repo.close(io); log.info("Initializing model..", .{}); var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig index aae2e608..1347403a 100644 --- a/examples/llm/models/lfm2_tests.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -37,6 +37,7 @@ pub fn main(init: std.process.Init) !void { defer platform.deinit(allocator, io); const repo = try zml.safetensors.resolveModelRepo(io, args.model); + defer repo.close(io); var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); defer registry.deinit(); var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); diff --git a/examples/llm/models/llama_tests.zig b/examples/llm/models/llama_tests.zig index 08d86ec8..5b5de5f7 100644 --- a/examples/llm/models/llama_tests.zig +++ b/examples/llm/models/llama_tests.zig @@ -35,6 +35,7 @@ pub fn main(init: std.process.Init) !void { defer platform.deinit(allocator, io); const repo = try zml.safetensors.resolveModelRepo(io, args.model); + defer repo.close(io); var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); defer registry.deinit(); var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); diff --git a/zml/safetensors.zig b/zml/safetensors.zig index 35f6acd1..6a35a795 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -285,6 +285,7 @@ pub const TensorRegistry = struct { repo: std.Io.Dir, ) !TensorRegistry { const entrypoint = try resolveModelEntrypoint(io, repo); + defer entrypoint.close(io); return try fetchRegistry(allocator, io, repo, entrypoint); } @@ -294,15 +295,16 @@ pub const TensorRegistry = struct { path: []const u8, ) !TensorRegistry { var repo = try resolveModelRepo(io, path); + defer repo.close(io); - if (std.mem.endsWith(u8, path, ".safetensors.index.json") or + const entrypoint = if (std.mem.endsWith(u8, path, ".safetensors.index.json") or std.mem.endsWith(u8, path, ".safetensors")) - { - return try fetchRegistry(allocator, io, repo, try repo.openFile(io, path, .{ .mode = .read_only })); - } else { - const entrypoint = try resolveModelEntrypoint(io, repo); - return try fetchRegistry(allocator, io, repo, entrypoint); - } + try repo.openFile(io, path, .{ .mode = .read_only }) + else + try resolveModelEntrypoint(io, repo); + defer entrypoint.close(io); + + return try fetchRegistry(allocator, io, repo, entrypoint); } pub fn deinit(self: *TensorRegistry) void { From 3178484b27429251905497fbc0ea9700331e86f6 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Wed, 2 Sep 2026 12:51:09 +0200 Subject: [PATCH 649/881] zml/io: record computed bindings in the store (#749) A binding created through `maybeCreateBinding` is computed from its sources by an executable and must never be streamed from the checkpoint. `Loader.defaultCallback` inferred that from `sources.len != 1`, which a single-source binding fails resulting to the raw bytes being loaded and producing wrong results --- examples/io/main.zig | 2 +- examples/llm/models/lfm2/model.zig | 2 +- examples/llm/models/llama/model.zig | 2 +- examples/llm/models/qwen3_5/model.zig | 2 +- examples/llm/models/qwen3_5_moe/model.zig | 2 +- examples/mnist/mnist.zig | 2 +- zml/io.zig | 106 ++++++++++++++-------- 7 files changed, 73 insertions(+), 45 deletions(-) diff --git a/examples/io/main.zig b/examples/io/main.zig index b512392c..5675010d 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -215,7 +215,7 @@ pub fn main(init: std.process.Init) !void { }); defer loader.deinit(); - loader.load(io, AllTensorsModel, &model, &buffers, &store, &.{sharded_sharding}, .{ .progress = &progress }); + try loader.load(io, AllTensorsModel, &model, &buffers, &store, &.{sharded_sharding}, .{ .progress = &progress }); try loader.await(io); const took = now.untilNow(io, .awake); diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index efb39fcc..53f1071c 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -95,7 +95,7 @@ pub const LoadedModel = struct { defer loader.deinit(); const all_shardings = shardings.all(); - loader.load(io, Model, &self.inner, &buffers, store, &all_shardings, .{ .progress = progress }); + try loader.load(io, Model, &self.inner, &buffers, store, &all_shardings, .{ .progress = progress }); try loader.await(io); const took = now.untilNow(io, .awake); diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index 06cff075..0b51b8b7 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -85,7 +85,7 @@ pub const LoadedModel = struct { defer loader.deinit(); const all_shardings = shardings.all(); - loader.load(io, Model, &self.inner, &buffers, store, &all_shardings, .{ .progress = progress }); + try loader.load(io, Model, &self.inner, &buffers, store, &all_shardings, .{ .progress = progress }); try loader.await(io); const took = now.untilNow(io, .awake); diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 41601fad..8c0d40f0 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -97,7 +97,7 @@ pub const LoadedModel = struct { defer loader.deinit(); const all_shardings = shardings.all(); - loader.load(io, Model, &self.inner, &buffers, store, &all_shardings, .{ .progress = progress }); + try loader.load(io, Model, &self.inner, &buffers, store, &all_shardings, .{ .progress = progress }); try loader.await(io); const took = now.untilNow(io, .awake); diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index e54a6f14..5ea280a3 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -123,7 +123,7 @@ pub const LoadedModel = struct { defer loader.deinit(); const all_shardings = shardings.all(); - loader.load(io, Model, &self.inner, &buffers, store, &all_shardings, .{ .progress = progress }); + try loader.load(io, Model, &self.inner, &buffers, store, &all_shardings, .{ .progress = progress }); try loader.await(io); const took = now.untilNow(io, .awake); diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 65de147b..63529e1c 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -49,7 +49,7 @@ const Mnist = struct { var loader: zml.io.Loader = try .init(allocator, platform, .default); errdefer loader.deinit(); - loader.load(io, Mnist, self, &buffers, store, &.{}, .{}); + try loader.load(io, Mnist, self, &buffers, store, &.{}, .{}); try loader.await(io); return buffers; diff --git a/zml/io.zig b/zml/io.zig index 3b9af053..6cbf136e 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -24,8 +24,13 @@ const Tensor = @import("tensor.zig").Tensor; const log = std.log.scoped(.@"zml/io"); pub const TensorStore = struct { + pub const Binding = struct { + tensors: []*safetensors.Tensor, + transformed: bool, + }; + registry: *safetensors.TensorRegistry, - id_to_sources: std.AutoHashMapUnmanaged(Tensor.Id, []*safetensors.Tensor), + id_to_sources: std.AutoHashMapUnmanaged(Tensor.Id, Binding), allocator: std.mem.Allocator, arena: std.heap.ArenaAllocator, @@ -44,7 +49,7 @@ pub const TensorStore = struct { self.arena.deinit(); } - fn putSourcesNoClobber(self: *TensorStore, id: Tensor.Id, sources: []*safetensors.Tensor) std.mem.Allocator.Error!void { + fn putSourcesNoClobber(self: *TensorStore, id: Tensor.Id, sources: Binding) std.mem.Allocator.Error!void { const gop = try self.id_to_sources.getOrPut(self.allocator, id); if (gop.found_existing) { stdx.debug.panic("Id {} already has associated sources", .{id}); @@ -70,8 +75,8 @@ pub const TensorStore = struct { fn getPtrFromId(self: *const TensorStore, id: usize) ?*safetensors.Tensor { const sources = self.id_to_sources.get(id) orelse return null; - stdx.debug.assert(sources.len == 1, "Expect tensor with id {} to have only one source, got {}", .{ id, sources.len }); - return sources[0]; + stdx.debug.assert(sources.tensors.len == 1, "Expect tensor with id {} to have only one source, got {}", .{ id, sources.tensors.len }); + return sources.tensors[0]; } pub fn getReader(self: *const TensorStore, key: []const u8, io: std.Io, buffer: []u8) !safetensors.TensorReader { @@ -80,12 +85,12 @@ pub const TensorStore = struct { pub fn getReaderById(self: *const TensorStore, id: usize, io: std.Io, buffer: []u8) !safetensors.TensorReader { const sources = self.id_to_sources.get(id) orelse return error.NotFound; - stdx.debug.assert(sources.len == 1, "Expect tensor with id {} to have only one source, got {}", .{ id, sources.len }); + stdx.debug.assert(sources.tensors.len == 1, "Expect tensor with id {} to have only one source, got {}", .{ id, sources.tensors.len }); - return sources[0].reader(io, buffer, .{}); + return sources.tensors[0].reader(io, buffer, .{}); } - pub fn getSourcesById(self: *const TensorStore, id: Tensor.Id) ?[]*safetensors.Tensor { + pub fn getSourcesById(self: *const TensorStore, id: Tensor.Id) ?Binding { return self.id_to_sources.get(id); } @@ -170,7 +175,7 @@ pub const TensorStore = struct { shape = applyPartitioning(shape, partitioning); const tensor: Tensor = .fromShape(shape); - self.store.putSourcesNoClobber(tensor.id, sources) catch |e| std.debug.panic("Not handling {} errors", .{e}); + self.store.putSourcesNoClobber(tensor.id, .{ .tensors = sources, .transformed = false }) catch |e| std.debug.panic("Not handling {} errors", .{e}); return tensor; } @@ -229,7 +234,7 @@ pub const TensorStore = struct { errdefer arena.free(tensors); const tensor: Tensor = .fromShape(shape); - self.store.putSourcesNoClobber(tensor.id, tensors) catch |e| std.debug.panic("Not handling {} errors", .{e}); + self.store.putSourcesNoClobber(tensor.id, .{ .tensors = tensors, .transformed = true }) catch |e| std.debug.panic("Not handling {} errors", .{e}); return tensor; } @@ -285,6 +290,7 @@ pub const Loader = struct { pinned_buffer_pools: []mem.DynamicBufferPool, group: stdx.Io.LimitedGroup, bytes_loaded: std.atomic.Value(usize) = .init(0), + delivered: std.AutoHashMapUnmanaged(Tensor.Id, void) = .empty, pub const Opts = struct { pub const default: Opts = .{ @@ -324,7 +330,8 @@ pub const Loader = struct { }; } - pub fn deinit(self: Loader) void { + pub fn deinit(self: *Loader) void { + self.delivered.deinit(self.allocator); for (self.pinned_buffer_pools, 0..) |*pool, i| pool.deinit(self.dma_allocators[i].allocator()); self.allocator.free(self.pinned_buffer_pools); self.allocator.free(self.dma_allocators); @@ -338,7 +345,9 @@ pub const Loader = struct { progress: ?*std.Progress.Node = null, }; - pub fn load(self: *Loader, io: std.Io, comptime T: type, model: *const T, buffers: *Bufferized(T), store: *const TensorStore, shardings: []const Sharding, opts: LoadOpts) void { + pub const LoadError = error{TransformedTensorNotDelivered}; + + pub fn load(self: *Loader, io: std.Io, comptime T: type, model: *const T, buffers: *Bufferized(T), store: *const TensorStore, shardings: []const Sharding, opts: LoadOpts) LoadError!void { const tensor_count = meta.count(Tensor, model); var arena: std.heap.ArenaAllocator = .init(self.allocator); @@ -351,29 +360,50 @@ pub const Loader = struct { } }.call, .{flattened_buffers}); - const Ctx = struct { - self: *Loader, - io: std.Io, - store: *const TensorStore, - shardings: []const Sharding, - buffers: []*Buffer, - opts: LoadOpts, - }; - - var ctx: Ctx = .{ - .self = self, - .io = io, - .store = store, - .shardings = shardings, - .buffers = flattened_buffers, - .opts = opts, - }; - + const flattened_tensors = arena.allocator().alloc(*const Tensor, tensor_count) catch @panic("Errors can't be handled in `loadInner`"); meta.forEachVisit(model, *const Tensor, struct { - fn call(i: usize, tensor: *const Tensor, ctx_: *Ctx) void { - ctx_.self.group.async(ctx_.io, defaultCallback, .{ ctx_.self, ctx_.io, tensor, ctx_.buffers[i], ctx_.store, ctx_.shardings, ctx_.opts }); + fn call(i: usize, tensor: *const Tensor, flattened_tensors_: []*const Tensor) void { + flattened_tensors_[i] = tensor; } - }.call, .{&ctx}); + }.call, .{flattened_tensors}); + + // Refuse before scheduling anything, so a failure leaves no task in flight. + var missing: usize = 0; + for (flattened_tensors) |tensor| { + const sources = store.getSourcesById(tensor.id) orelse continue; + if (!sources.transformed or self.delivered.contains(tensor.id)) { + continue; + } + + missing += 1; + logNotDelivered(arena.allocator(), tensor, sources.tensors); + } + + if (missing != 0) { + return error.TransformedTensorNotDelivered; + } + + for (flattened_tensors, flattened_buffers) |tensor, buffer| { + if (store.getSourcesById(tensor.id)) |sources| { + if (sources.transformed) continue; + } + + self.group.async(io, defaultCallback, .{ self, io, tensor, buffer, store, shardings, opts }); + } + } + + fn logNotDelivered(arena: std.mem.Allocator, tensor: *const Tensor, sources: []const *safetensors.Tensor) void { + const max_names = 8; + var names: std.Io.Writer.Allocating = .init(arena); + for (sources[0..@min(sources.len, max_names)], 0..) |source, i| { + names.writer.print("{s}{s}", .{ if (i != 0) ", " else "", source.name }) catch break; + } + + if (sources.len > max_names) { + names.writer.print(" and {} more", .{sources.len - max_names}) catch {}; + } + + log.err("Transformed tensor {} {f} was not delivered by loadExecute before load; sources: {s}", .{ tensor.id, tensor.shape(), names.written() }); } fn defaultCallback(self: *Loader, io: std.Io, tensor: *const Tensor, buffer: *Buffer, store: *const TensorStore, shardings: []const Sharding, opts: LoadOpts) void { @@ -381,13 +411,9 @@ pub const Loader = struct { std.log.warn("Failed to get sources for tensor with id: {}", .{tensor.id}); return; }; + stdx.debug.assert(!sources.transformed and sources.tensors.len == 1, "Tensor {} is transformed or has {} sources; `load` only streams single-source tensors", .{ tensor.id, sources.tensors.len }); - if (sources.len != 1) { - log.debug("Skipping fused tensor with {} sources; expected to be loaded via loadExecute", .{sources.len}); - return; - } - - self.loadSingleInner(io, sources[0], tensor.shape(), buffer, shardings, opts) catch |e| { + self.loadSingleInner(io, sources.tensors[0], tensor.shape(), buffer, shardings, opts) catch |e| { log.err("Errors are not handled in `defaultCallback`, got {}", .{e}); unreachable; }; @@ -397,6 +423,7 @@ pub const Loader = struct { self.loadSingleInner(io, source, shape, buffer, shardings, opts) catch |e| { log.err("Failed to load tensor {s}: {}", .{ source.name, e }); loaded.* = false; + return; }; loaded.* = true; } @@ -452,7 +479,7 @@ pub const Loader = struct { exe: *const Exe, opts: LoadOpts, ) !void { - const sources = store.getSourcesById(tensor.id) orelse return error.NotFound; + const sources = (store.getSourcesById(tensor.id) orelse return error.NotFound).tensors; const buffers = try arena.alloc(Buffer, sources.len); const loaded = try arena.alloc(bool, sources.len); @memset(loaded, false); @@ -485,6 +512,7 @@ pub const Loader = struct { exe.callOpts(io, args, &results, .{ .wait = true }); buffer.* = results.get(Buffer); + try self.delivered.put(self.allocator, tensor.id, {}); } }; From de5850a99c627c57ff7777e43fbb1b40ea36745f Mon Sep 17 00:00:00 2001 From: Olivier Boulant Date: Wed, 2 Sep 2026 13:51:29 +0200 Subject: [PATCH 650/881] ci: prevent RunOn runner stealing (#759) Give each concurrent RunsOn job a unique routing label using the workflow run ID, run attempt, and job name. This prevents runners provisioned for one matrix job from being assigned to another compatible job. - [Runner stealing](https://runs-on.com/docs/maintenance/troubleshooting/#runner-stealing) --- Replace the deprecated image-level `disk: large` setting with `volume=80gb` on each RunsOn job. Storage belongs to the runner configuration rather than the custom image definition. - [RunsOn storage labels](https://runs-on.com/docs/runners/labels/#disk) - [Custom runners and images](https://runs-on.com/docs/runners/custom/) --- .github/runs-on.yml | 1 - .github/workflows/ci.yaml | 24 +++++++++++------------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/.github/runs-on.yml b/.github/runs-on.yml index 09d88bf4..384121f7 100644 --- a/.github/runs-on.yml +++ b/.github/runs-on.yml @@ -3,7 +3,6 @@ images: platform: "linux" arch: "amd64" ami: "ami-04a92520784b93e73" - disk: large preinstall: | #!/bin/bash wget https://github.com/jbdalido/bazel-remote/releases/download/test/bazel-remote -O /usr/local/bin/bzlcache diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index faa25340..43964175 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -23,33 +23,33 @@ jobs: self_hosted: false run_tests: true runner_labels: - - runs-on + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-cpu - runner=16cpu-linux-x64 - family=c8+c7 - image=ubuntu24-amd64 - - run-id=${{ github.run_id }} + - volume=80gb - extras=s3-cache - target: neuron bazel_opts: "--@zml//platforms:neuron=true --@zml//platforms:cpu=false" self_hosted: false run_tests: false runner_labels: - - runs-on + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-neuron - runner=16cpu-linux-x64 - family=c8+c7 - image=ubuntu24-amd64 - - run-id=${{ github.run_id }} + - volume=80gb - extras=s3-cache - target: tpu bazel_opts: "--@zml//platforms:tpu=true --@zml//platforms:cpu=false" self_hosted: false run_tests: false runner_labels: - - runs-on + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-tpu - runner=16cpu-linux-x64 - family=c8+c7 - image=ubuntu24-amd64 - - run-id=${{ github.run_id }} + - volume=80gb - extras=s3-cache - target: cuda bazel_opts: "--@zml//platforms:cuda=true --@zml//platforms:cpu=false" @@ -98,13 +98,11 @@ jobs: lint: runs-on: - [ - "runs-on", - "runner=4cpu-linux-x64", - "image=ubuntu24-amd64", - "run-id=${{ github.run_id }}", - "extras=s3-cache", - ] + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-lint + - runner=4cpu-linux-x64 + - image=ubuntu24-amd64 + - volume=80gb + - extras=s3-cache steps: - uses: runs-on/action@v2.1.0 - name: Checkout From 67ac9c7feb74aaebdcbb130296890aad0327efcb Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 2 Sep 2026 16:37:56 +0200 Subject: [PATCH 651/881] zml/moe/moe: handle optional scales (#761) While waiting for #757, this fixes triton moe for mxfp4 --- zml/moe/moe.zig | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index e34996c1..787946a7 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -425,14 +425,12 @@ pub fn forwardMoe( ); } + const manual_inputs: []const zml.Tensor = if (quant_scheme == .mxfp4) + &.{ input, topk_ids, topk_weights, gate_up.weight, down.weight, gate_up_scales.?, down_scales.? } + else + &.{ input, topk_ids, topk_weights, gate_up.weight, down.weight }; break :b zml.ops.manualComputation( - .{ - input, - topk_ids, - topk_weights, - gate_up.weight, - down.weight, - }, + manual_inputs, input.shape(), .{ .activation = parameters.triton.activation, @@ -441,8 +439,6 @@ pub fn forwardMoe( .bias_down = down.bias, .quant_scheme = quant_scheme, .activation_threshold = opts.activation_threshold, - .scales_gate_up = gate_up_scales, - .scales_down = down_scales, }, (struct { fn body(ctx: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, _: zml.Shape) zml.Tensor { @@ -471,8 +467,8 @@ pub fn forwardMoe( .activation = ctx.activation, .global_num_experts = ctx.global_num_experts, .expert_map = expert_map, - .w1_scale = ctx.scales_gate_up, - .w2_scale = ctx.scales_down, + .w1_scale = if (ctx.quant_scheme == .mxfp4) sharded_inputs[5] else null, + .w2_scale = if (ctx.quant_scheme == .mxfp4) sharded_inputs[6] else null, .w1_bias = ctx.bias_gate_up, .w2_bias = ctx.bias_down, .quant_scheme = ctx.quant_scheme, From e97abbaad43e092f58fe312670613a5bca2665d7 Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Thu, 3 Sep 2026 11:02:59 +0200 Subject: [PATCH 652/881] zml/ops: refactor `manualComputation` (#757) Simplify `manualComputation` by merging `inputs` and `body_context` into a single `input` argument. Internally, collect all and shard all tensors (including `?Tensor`) from `input` before invoking the body function. --- examples/sharding/main.zig | 12 +- zml/attention/flashattn.zig | 272 +++++++++++++++---------- zml/attention/nki/attention.zig | 46 +++-- zml/attention/tpu_attention.zig | 50 ++--- zml/attention/triton_attention.zig | 157 +++++++------- zml/moe/moe.zig | 316 +++++++++++++++++------------ zml/ops.zig | 246 ++++++++++++++-------- zml/tensor.zig | 11 +- 8 files changed, 667 insertions(+), 443 deletions(-) diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index 4e919c7f..37b8c68b 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -39,15 +39,17 @@ const DemoModel = struct { const gate = y.scale(0.01).sigmoid(); return zml.ops.manualComputation( - .{ y, gate }, - y.shape(), - {}, (struct { - fn body(_: void, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output_sharded: zml.Shape) zml.Tensor { - const local_result = sharded_inputs[0].relu().mul(sharded_inputs[1]); + y: zml.Tensor, + gate: zml.Tensor, + + fn body(ctx: @This(), output_sharded: zml.Shape) zml.Tensor { + const local_result = ctx.y.relu().mul(ctx.gate); return local_result.reshape(output_sharded); } }).body, + .{ .y = y, .gate = gate }, + y.shape(), ); } }; diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 7581a85a..0c71b32d 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -947,20 +947,34 @@ pub const paged_fa2 = struct { const output_shape = q2.shape(); var o = zml.ops.manualComputation( + (struct { + inputs: struct { zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor }, + metadata: struct { + is_causal: bool, + max_seqlen_k: usize, + num_heads: i64, + window_size_left: i32, + softmax_scale: ?f32, + }, + opts: zml.ops.CustomCallOptions, + + fn body(self: @This(), output: zml.Shape) zml.Tensor { + return zml.ops.customCall(Decode.custom_call_name, self.inputs, output, self.metadata, self.opts); + } + }).body, .{ - q2, - k_cache, - v_cache, - cu_seqlens_q, - dummy_cu_seqlens_k, - seqused_k, - block_table, - softmax_lse, - softmax_lse_accum, - out_accum, - }, - output_shape, - .{ + .inputs = .{ + q2, + k_cache, + v_cache, + cu_seqlens_q, + dummy_cu_seqlens_k, + seqused_k, + block_table, + softmax_lse, + softmax_lse_accum, + out_accum, + }, .metadata = .{ .is_causal = opts.is_causal, .max_seqlen_k = decode_parameters.options.max_seqlen_k, @@ -972,11 +986,7 @@ pub const paged_fa2 = struct { .has_side_effect = false, }, }, - (struct { - fn body(ctx_: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output: zml.Shape) zml.Tensor { - return zml.ops.customCall(Decode.custom_call_name, sharded_inputs, output, ctx_.metadata, ctx_.opts); - } - }).body, + output_shape, ); if (seqlenq_ngroups_swapped) { @@ -1022,20 +1032,35 @@ pub const paged_fa2 = struct { const output_shape = q2.shape(); var o = zml.ops.manualComputation( + (struct { + inputs: struct { zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor }, + metadata: struct { + is_causal: bool, + max_seqlen_k: usize, + max_seqlen_q: usize, + num_heads: i64, + window_size_left: i32, + softmax_scale: ?f32, + }, + opts: zml.ops.CustomCallOptions, + + fn body(self: @This(), output: zml.Shape) zml.Tensor { + return zml.ops.customCall(Prefill.custom_call_name, self.inputs, output, self.metadata, self.opts); + } + }).body, .{ - q2, - k_cache, - v_cache, - cu_seqlens_q_prefill, - dummy_cu_seqlens_k_prefill, - seqused_k_prefill, - block_table_prefill, - softmax_lse_prefill, - softmax_lse_accum_prefill, - out_accum_prefill, - }, - output_shape, - .{ + .inputs = .{ + q2, + k_cache, + v_cache, + cu_seqlens_q_prefill, + dummy_cu_seqlens_k_prefill, + seqused_k_prefill, + block_table_prefill, + softmax_lse_prefill, + softmax_lse_accum_prefill, + out_accum_prefill, + }, .metadata = .{ .is_causal = opts.is_causal, .max_seqlen_k = mixed_parameters.options.max_seqlen_k, @@ -1048,11 +1073,7 @@ pub const paged_fa2 = struct { .has_side_effect = false, }, }, - (struct { - fn body(ctx_: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output: zml.Shape) zml.Tensor { - return zml.ops.customCall(Prefill.custom_call_name, sharded_inputs, output, ctx_.metadata, ctx_.opts); - } - }).body, + output_shape, ); o = o.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = num_head_groups }); @@ -1087,20 +1108,34 @@ pub const paged_fa2 = struct { const output_shape_decode = q_decode.shape(); var o_decode = zml.ops.manualComputation( + (struct { + inputs: struct { zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor }, + metadata: struct { + is_causal: bool, + max_seqlen_k: usize, + num_heads: i64, + window_size_left: i32, + softmax_scale: ?f32, + }, + opts: zml.ops.CustomCallOptions, + + fn body(self: @This(), output: zml.Shape) zml.Tensor { + return zml.ops.customCall(Decode.custom_call_name, self.inputs, output, self.metadata, self.opts); + } + }).body, .{ - q_decode, - k_cache, - v_cache, - cu_seqlens_q_decode, - dummy_cu_seqlens_k_decode, - seqused_k_decode, - block_table_decode, - softmax_lse_decode, - softmax_lse_accum_decode, - out_accum_decode, - }, - output_shape_decode, - .{ + .inputs = .{ + q_decode, + k_cache, + v_cache, + cu_seqlens_q_decode, + dummy_cu_seqlens_k_decode, + seqused_k_decode, + block_table_decode, + softmax_lse_decode, + softmax_lse_accum_decode, + out_accum_decode, + }, .metadata = .{ .is_causal = opts.is_causal, .max_seqlen_k = mixed_parameters.options.max_seqlen_k, @@ -1112,11 +1147,7 @@ pub const paged_fa2 = struct { .has_side_effect = false, }, }, - (struct { - fn body(ctx_: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output: zml.Shape) zml.Tensor { - return zml.ops.customCall(Decode.custom_call_name, sharded_inputs, output, ctx_.metadata, ctx_.opts); - } - }).body, + output_shape_decode, ); if (seqlenq_ngroups_swapped) { @@ -1556,20 +1587,32 @@ pub const paged_fa3 = struct { const output_shape = q2.shape(); var o = zml.ops.manualComputation( + (struct { + inputs: struct { zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor }, + metadata: struct { + is_causal: bool, + max_seqlen_k: usize, + window_size_left: i32, + }, + opts: zml.ops.CustomCallOptions, + + fn body(self: @This(), output: zml.Shape) zml.Tensor { + return zml.ops.customCall(Decode.custom_call_name, self.inputs, output, self.metadata, self.opts); + } + }).body, .{ - q2, - k_cache, - v_cache, - cu_seqlens_q, - seqused_k, - block_table, - softmax_lse, - softmax_lse_accum, - out_accum, - scheduler_metadata, - }, - output_shape, - .{ + .inputs = .{ + q2, + k_cache, + v_cache, + cu_seqlens_q, + seqused_k, + block_table, + softmax_lse, + softmax_lse_accum, + out_accum, + scheduler_metadata, + }, .metadata = .{ .is_causal = opts.is_causal, .max_seqlen_k = decode_parameters.options.max_seqlen_k, @@ -1579,11 +1622,7 @@ pub const paged_fa3 = struct { .has_side_effect = false, }, }, - (struct { - fn body(ctx_: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output: zml.Shape) zml.Tensor { - return zml.ops.customCall(Decode.custom_call_name, sharded_inputs, output, ctx_.metadata, ctx_.opts); - } - }).body, + output_shape, ); o = o.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = num_head_groups }); @@ -1624,20 +1663,33 @@ pub const paged_fa3 = struct { const output_shape = q2.shape(); var o = zml.ops.manualComputation( + (struct { + inputs: struct { zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor }, + metadata: struct { + is_causal: bool, + max_seqlen_k: usize, + max_seqlen_q: usize, + window_size_left: i32, + }, + opts: zml.ops.CustomCallOptions, + + fn body(self: @This(), output: zml.Shape) zml.Tensor { + return zml.ops.customCall(Prefill.custom_call_name, self.inputs, output, self.metadata, self.opts); + } + }).body, .{ - q2, - k_cache, - v_cache, - cu_seqlens_q_prefill, - seqused_k_prefill, - block_table_prefill, - softmax_lse_prefill, - softmax_lse_accum_prefill, - out_accum_prefill, - scheduler_metadata_prefill, - }, - output_shape, - .{ + .inputs = .{ + q2, + k_cache, + v_cache, + cu_seqlens_q_prefill, + seqused_k_prefill, + block_table_prefill, + softmax_lse_prefill, + softmax_lse_accum_prefill, + out_accum_prefill, + scheduler_metadata_prefill, + }, .metadata = .{ .is_causal = opts.is_causal, .max_seqlen_k = mixed_parameters.options.max_seqlen_k, @@ -1646,11 +1698,7 @@ pub const paged_fa3 = struct { }, .opts = zml.ops.CustomCallOptions{ .has_side_effect = false }, }, - (struct { - fn body(ctx_: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output: zml.Shape) zml.Tensor { - return zml.ops.customCall(Prefill.custom_call_name, sharded_inputs, output, ctx_.metadata, ctx_.opts); - } - }).body, + output_shape, ); o = o.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = num_head_groups }); @@ -1681,20 +1729,32 @@ pub const paged_fa3 = struct { const decode_output_shape = q_decode.shape(); var o_decode = zml.ops.manualComputation( + (struct { + inputs: struct { zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor }, + metadata: struct { + is_causal: bool, + max_seqlen_k: usize, + window_size_left: i32, + }, + opts: zml.ops.CustomCallOptions, + + fn body(self: @This(), output: zml.Shape) zml.Tensor { + return zml.ops.customCall(Decode.custom_call_name, self.inputs, output, self.metadata, self.opts); + } + }).body, .{ - q_decode, - k_cache, - v_cache, - cu_seqlens_q_decode, - seqused_k_decode, - block_table_decode, - softmax_lse_decode, - softmax_lse_accum_decode, - out_accum_decode, - scheduler_metadata_decode, - }, - decode_output_shape, - .{ + .inputs = .{ + q_decode, + k_cache, + v_cache, + cu_seqlens_q_decode, + seqused_k_decode, + block_table_decode, + softmax_lse_decode, + softmax_lse_accum_decode, + out_accum_decode, + scheduler_metadata_decode, + }, .metadata = .{ .is_causal = opts.is_causal, .max_seqlen_k = mixed_parameters.options.max_seqlen_k, @@ -1702,11 +1762,7 @@ pub const paged_fa3 = struct { }, .opts = zml.ops.CustomCallOptions{ .has_side_effect = false }, }, - (struct { - fn body(ctx_: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output: zml.Shape) zml.Tensor { - return zml.ops.customCall(Decode.custom_call_name, sharded_inputs, output, ctx_.metadata, ctx_.opts); - } - }).body, + decode_output_shape, ); o_decode = o_decode.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = num_head_groups }); diff --git a/zml/attention/nki/attention.zig b/zml/attention/nki/attention.zig index c08a5ac4..6553b463 100644 --- a/zml/attention/nki/attention.zig +++ b/zml/attention/nki/attention.zig @@ -1,5 +1,3 @@ -const std = @import("std"); - const platforms = @import("platforms"); const zml = @import("../../zml.zig"); @@ -54,46 +52,60 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T .withPartitioning(.{ .row = .replicated, .col = .replicated }); return zml.ops.manualComputation( - .{ q_sharded, k_sharded, v_sharded, token_index_2d }, - q_sharded.shape(), - parameters, (struct { - fn body(context: Parameters, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output: zml.Shape) zml.Tensor { - stdx.debug.assert(sharded_inputs.len == 4, "NKI decode attention expects 4 sharded inputs, got {}", .{sharded_inputs.len}); - const kernel = context.decode_kernel; + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + token_index: zml.Tensor, + parameters: Parameters, + + fn body(self: @This(), output: zml.Shape) zml.Tensor { + const kernel = self.parameters.decode_kernel; return zml.ops.neuronNki( - .{ sharded_inputs[0], sharded_inputs[1], sharded_inputs[2], sharded_inputs[3] }, + .{ self.q, self.k, self.v, self.token_index }, .{output}, .{ .name = kernel.name, .entrypoint = kernel.entrypoint, .source_path = kernel.source_path, - .compiler_target = context.compiler_target, + .compiler_target = self.parameters.compiler_target, }, )[0]; } }).body, + .{ + .q = q_sharded, + .k = k_sharded, + .v = v_sharded, + .token_index = token_index_2d, + .parameters = parameters, + }, + q_sharded.shape(), ).convert(q.dtype()); } return zml.ops.manualComputation( - .{ q_sharded, k_sharded, v_sharded }, - q_sharded.shape(), - parameters, (struct { - fn body(context: Parameters, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output: zml.Shape) zml.Tensor { - const kernel = context.prefill_kernel; + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + parameters: Parameters, + + fn body(self: @This(), output: zml.Shape) zml.Tensor { + const kernel = self.parameters.prefill_kernel; return zml.ops.neuronNki( - .{ sharded_inputs[0], sharded_inputs[1], sharded_inputs[2] }, + .{ self.q, self.k, self.v }, .{output}, .{ .name = kernel.name, .entrypoint = kernel.entrypoint, .source_path = kernel.source_path, - .compiler_target = context.compiler_target, + .compiler_target = self.parameters.compiler_target, }, )[0]; } }).body, + .{ .q = q_sharded, .k = k_sharded, .v = v_sharded, .parameters = parameters }, + q_sharded.shape(), ).convert(q.dtype()); } diff --git a/zml/attention/tpu_attention.zig b/zml/attention/tpu_attention.zig index e7511b3f..537bb1b1 100644 --- a/zml/attention/tpu_attention.zig +++ b/zml/attention/tpu_attention.zig @@ -223,34 +223,27 @@ pub const mosaic_tpu = struct { const prepared = prepareInputs(parameters, q, kv_cache); const q_out = zml.ops.manualComputation( - .{ - prepared.q, - prepared.kv_pages, - prepared.seq_lens, - prepared.block_table, - prepared.query_start_len, - prepared.num_seqs, - }, - prepared.q.shape(), - .{ - .opts = opts, - .parameters = parameters, - }, (struct { - fn body(body_context: anytype, allocator: std.mem.Allocator, sharded_inputs: []const zml.Tensor, output: zml.Shape) zml.Tensor { - _ = allocator; - stdx.debug.assert(sharded_inputs.len == 6, "mosaic_tpu ragged paged manualComputation expects 6 inputs, got {}", .{sharded_inputs.len}); + q: zml.Tensor, + kv_pages: zml.Tensor, + seq_lens: zml.Tensor, + block_table: zml.Tensor, + query_start_len: zml.Tensor, + num_seqs: zml.Tensor, + opts: AttentionOptions, + parameters: Parameters, + fn body(self: @This(), output: zml.Shape) zml.Tensor { const prepared_inputs: PreparedInputs = .{ - .q = sharded_inputs[0], - .kv_pages = sharded_inputs[1], - .seq_lens = sharded_inputs[2], - .block_table = sharded_inputs[3], - .query_start_len = sharded_inputs[4], - .num_seqs = sharded_inputs[5], + .q = self.q, + .kv_pages = self.kv_pages, + .seq_lens = self.seq_lens, + .block_table = self.block_table, + .query_start_len = self.query_start_len, + .num_seqs = self.num_seqs, }; - const cfg = buildCfg(prepared_inputs, body_context.parameters, body_context.opts); + const cfg = buildCfg(prepared_inputs, self.parameters, self.opts); const q_out = raggedPagedKernelCall( prepared_inputs.q, prepared_inputs.kv_pages, @@ -264,6 +257,17 @@ pub const mosaic_tpu = struct { return q_out; } }).body, + .{ + .q = prepared.q, + .kv_pages = prepared.kv_pages, + .seq_lens = prepared.seq_lens, + .block_table = prepared.block_table, + .query_start_len = prepared.query_start_len, + .num_seqs = prepared.num_seqs, + .opts = opts, + .parameters = parameters, + }, + prepared.q.shape(), ); const restored = restoreQueryHeads(q, q_out); diff --git a/zml/attention/triton_attention.zig b/zml/attention/triton_attention.zig index 1f80d792..34afefd5 100644 --- a/zml/attention/triton_attention.zig +++ b/zml/attention/triton_attention.zig @@ -341,38 +341,36 @@ pub const paged = struct { pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { const output = zml.ops.manualComputation( - .{ - q, - k_cache, - v_cache, - parameters.block_table, - parameters.seq_lens, - parameters.query_start_len, - }, - q.shape(), - .{ - .opts = opts, - .options = parameters.options_, - }, (struct { - fn body(ctx_: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, _: zml.Shape) zml.Tensor { - const q_ = sharded_inputs[0]; - const k_cache_ = sharded_inputs[1]; - const v_cache_ = sharded_inputs[2]; - const parameters_: Parameters = .{ .block_table = sharded_inputs[3], .seq_lens = sharded_inputs[4], .query_start_len = sharded_inputs[5], .options_ = ctx_.options }; + q: zml.Tensor, + k_cache: zml.Tensor, + v_cache: zml.Tensor, + block_table: zml.Tensor, + seq_lens: zml.Tensor, + query_start_len: zml.Tensor, + opts: AttentionOptions, + options: Options, + + fn body(self: @This(), _: zml.Shape) zml.Tensor { + const parameters_: Parameters = .{ + .block_table = self.block_table, + .seq_lens = self.seq_lens, + .query_start_len = self.query_start_len, + .options_ = self.options, + }; const cu_count = getCuCount(); - const num_heads: usize = @intCast(q_.dim(.hkv) * q_.dim(.hg)); - const num_kv_heads: usize = @intCast(k_cache_.dim(.hkv)); + const num_heads: usize = @intCast(self.q.dim(.hkv) * self.q.dim(.hg)); + const num_kv_heads: usize = @intCast(self.k_cache.dim(.hkv)); const num_queries_per_kv: usize = num_heads / num_kv_heads; // Intel decode: pack exactly one GQA group per tile (block_q == 1) so the // single decode query token doesn't carry masked-out fp32 acc lanes. // oneAPI decode keeps one GQA group per tile, padded to a power of two so tt.make_range emits legal Triton IR. - const block_m: usize = if (!ctx_.options.is_prefill and isOneapiTarget()) + const block_m: usize = if (!self.options.is_prefill and isOneapiTarget()) std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv) else if (num_queries_per_kv <= 16) 16 else std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv); const block_q: usize = block_m / num_queries_per_kv; - const num_tokens: usize = @intCast(q_.dim(.b)); + const num_tokens: usize = @intCast(self.q.dim(.b)); const num_seqs: usize = @intCast(parameters_.block_table.dim(.b)); const total_q_blocks: usize = num_tokens / block_q + num_seqs; const target_num_prgms: usize = cu_count * 4; @@ -380,23 +378,23 @@ pub const paged = struct { const paged_attention_opts: PagedAttentionOptions = .{ .cu_count = getCuCount(), - .all_decode = !ctx_.options.is_prefill, + .all_decode = !self.options.is_prefill, .num_tokens = num_tokens, .num_heads = num_heads, .num_kv_heads = num_kv_heads, - .head_dim = @intCast(q_.dim(.hd)), + .head_dim = @intCast(self.q.dim(.hd)), .batch_size = @intCast(parameters_.block_table.dim(.b)), - .block_size = @intCast(k_cache_.dim(.k_chunk)), - .num_blocks = @intCast(k_cache_.dim(.page)), + .block_size = @intCast(self.k_cache.dim(.k_chunk)), + .num_blocks = @intCast(self.k_cache.dim(.page)), .max_num_block_per_seq = @intCast(parameters_.block_table.dim(.p)), - .sliding_window = if (ctx_.opts.sliding_window < 0) 0 else @intCast(ctx_.opts.sliding_window), + .sliding_window = if (self.opts.sliding_window < 0) 0 else @intCast(self.opts.sliding_window), .block_m = block_m, .block_q = block_q, .total_q_blocks = total_q_blocks, .target_num_prgms = target_num_prgms, .num_2d_prgms = num_2d_prgms, - .max_seqlen_q = ctx_.options.max_seqlen_q, - .scale = ctx_.opts.scale, + .max_seqlen_q = self.options.max_seqlen_q, + .scale = self.opts.scale, }; const use_2d_kernel = use2dKernel( @@ -405,15 +403,26 @@ pub const paged = struct { paged_attention_opts.num_kv_heads, ); const output = if (use_2d_kernel) - pagedAttention2d(parameters_, q_, k_cache_, v_cache_, ctx_.opts, paged_attention_opts) + pagedAttention2d(parameters_, self.q, self.k_cache, self.v_cache, self.opts, paged_attention_opts) else if (isOneapiTarget()) - pagedAttention3dOneapi(parameters_, q_, k_cache_, v_cache_, ctx_.opts, paged_attention_opts) + pagedAttention3dOneapi(parameters_, self.q, self.k_cache, self.v_cache, self.opts, paged_attention_opts) else - pagedAttention3d(parameters_, q_, k_cache_, v_cache_, ctx_.opts, paged_attention_opts); + pagedAttention3d(parameters_, self.q, self.k_cache, self.v_cache, self.opts, paged_attention_opts); return output; } }).body, + .{ + .q = q, + .k_cache = k_cache, + .v_cache = v_cache, + .block_table = parameters.block_table, + .seq_lens = parameters.seq_lens, + .query_start_len = parameters.query_start_len, + .opts = opts, + .options = parameters.options_, + }, + q.shape(), ); return output; @@ -975,64 +984,70 @@ pub const paged = struct { pub fn pagedSparseMla(parameters: Parameters, q: zml.Tensor, kv_cache: zml.Tensor, sink: ?zml.Tensor, topk: zml.Tensor, tokens_pos: zml.Tensor, opts: MlaOptions) zml.Tensor { const output_shape = q.shape().set(.hd, opts.value_rank); return zml.ops.manualComputation( - .{ - q, - kv_cache, - sink, - topk, - tokens_pos, - parameters.block_table, - parameters.seq_lens, - parameters.query_start_len, - }, - output_shape, - .{ - .opts = opts, - .options = parameters.options_, - }, (struct { - fn body(ctx_: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, _: zml.Shape) zml.Tensor { - const q_ = sharded_inputs[0]; - const kv_cache_ = sharded_inputs[1]; + q: zml.Tensor, + kv_cache: zml.Tensor, + sink: ?zml.Tensor, + topk: zml.Tensor, + tokens_pos: zml.Tensor, + block_table: zml.Tensor, + seq_lens: zml.Tensor, + query_start_len: zml.Tensor, + opts: MlaOptions, + options: Options, - const block_size = kv_cache_.dim(.k_chunk); + fn body(self: @This(), _: zml.Shape) zml.Tensor { + const block_size = self.kv_cache.dim(.k_chunk); const parameters_: Parameters = .{ - .block_table = sharded_inputs[5], - .seq_lens = sharded_inputs[6], - .query_start_len = sharded_inputs[7], - .options_ = ctx_.options, + .block_table = self.block_table, + .seq_lens = self.seq_lens, + .query_start_len = self.query_start_len, + .options_ = self.options, }; - const topk_final = topkToPhysical(parameters_, sharded_inputs[3], sharded_inputs[4], block_size); - const active_query_count = sharded_inputs[7] - .slice(.b, .{ .start = sharded_inputs[7].dim(.b) - 1 }) + const topk_final = topkToPhysical(parameters_, self.topk, self.tokens_pos, block_size); + const active_query_count = self.query_start_len + .slice(.b, .{ .start = self.query_start_len.dim(.b) - 1 }) .squeeze(.b); - stdx.debug.assert(topk_final.dim(.q) == q_.dim(.q), "expected topk q dim ({}) to match q dim ({})", .{ topk_final.dim(.q), q_.dim(.q) }); + stdx.debug.assert(topk_final.dim(.q) == self.q.dim(.q), "expected topk q dim ({}) to match q dim ({})", .{ topk_final.dim(.q), self.q.dim(.q) }); - const num_heads: usize = @intCast(q_.dim(.h)); + const num_heads: usize = @intCast(self.q.dim(.h)); const paged_opts: PagedSparseMlaOptions = .{ - .qk_rank = @intCast(q_.dim(.hd)), - .value_rank = @intCast(ctx_.opts.value_rank), + .qk_rank = @intCast(self.q.dim(.hd)), + .value_rank = @intCast(self.opts.value_rank), .num_heads = num_heads, - .block_size = @intCast(kv_cache_.dim(.k_chunk)), - .rope_rank = @intCast(ctx_.opts.rope_rank), - .scale = ctx_.opts.scale, - .total_q_blocks = @intCast(q_.dim(.q)), - .num_kv_splits = ctx_.opts.num_kv_splits, - .all_decode = !ctx_.options.is_prefill, + .block_size = @intCast(self.kv_cache.dim(.k_chunk)), + .rope_rank = @intCast(self.opts.rope_rank), + .scale = self.opts.scale, + .total_q_blocks = @intCast(self.q.dim(.q)), + .num_kv_splits = self.opts.num_kv_splits, + .all_decode = !self.options.is_prefill, }; return pagedSparseMlaKernel( - q_, - kv_cache_, - sharded_inputs[2], + self.q, + self.kv_cache, + self.sink, topk_final, active_query_count, paged_opts, ); } }).body, + .{ + .q = q, + .kv_cache = kv_cache, + .sink = sink, + .topk = topk, + .tokens_pos = tokens_pos, + .block_table = parameters.block_table, + .seq_lens = parameters.seq_lens, + .query_start_len = parameters.query_start_len, + .opts = opts, + .options = parameters.options_, + }, + output_shape, ); } }; diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index 787946a7..46a0b785 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -232,80 +232,93 @@ pub fn forwardMoe( // Also, maybe pass `zml.nn.Linear` directly if (expert_partition.eql(.init(.experts))) { break :b zml.ops.manualComputation( - .{ - input, - topk_ids, - topk_weights, - gate_up_weight_unpacked, - down_weight_unpacked, - gate_up.quantization.?.input_scale.?.asMultiplier(), - gate_up.quantization.?.scales, - gate_up.quantization.?.global_scale.?.asMultiplier(), - down.quantization.?.input_scale.?.asMultiplier(), - down.quantization.?.scales, - down.quantization.?.global_scale.?.asMultiplier(), - }, - input.shape(), - .{ - .activation = runner_options.activation, - .enable_pdl = runner_options.enable_pdl, - .gemm1_tactic = runner_options.gemm1_tactic, - .gemm2_tactic = runner_options.gemm2_tactic, - .workspace_query_device = runner_options.workspace_query_device, - }, (struct { + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + gate_up_weight_unpacked: zml.Tensor, + down_weight_unpacked: zml.Tensor, + gate_up_input_scale: zml.Tensor, + gate_up_scales: zml.Tensor, + gate_up_global_scale: zml.Tensor, + down_input_scale: zml.Tensor, + down_scales: zml.Tensor, + down_global_scale: zml.Tensor, + activation: cutlass_flashinfer.Activation, + enable_pdl: bool, + gemm1_tactic: i32, + gemm2_tactic: i32, + workspace_query_device: i32, + fn body( - ctx: anytype, - _: std.mem.Allocator, - sharded_inputs: []const zml.Tensor, + self: @This(), _: zml.Shape, ) zml.Tensor { - const local_num_experts = sharded_inputs[3].dim(.expert); + const local_num_experts = self.gate_up_weight_unpacked.dim(.expert); const partition_id = zml.ops.partitionId().convert(.i32); const expert_start = partition_id.scale(local_num_experts).convert(.i32); const expert_end = expert_start.addConstant(local_num_experts); - const local_route_mask = sharded_inputs[1] + const local_route_mask = self.topk_ids .cmp(.GE, expert_start) - .logical(.AND, sharded_inputs[1].cmp(.LT, expert_end)); + .logical(.AND, self.topk_ids.cmp(.LT, expert_end)); const local_topk_ids = local_route_mask.select( - sharded_inputs[1].sub(expert_start), + self.topk_ids.sub(expert_start), zml.Tensor.scalar(0, .i32), ); const local_topk_weights = local_route_mask.select( - sharded_inputs[2], - zml.Tensor.scalar(0, sharded_inputs[2].dtype()), + self.topk_weights, + zml.Tensor.scalar(0, self.topk_weights.dtype()), ); const local_output = cutlass_flashinfer.fusedExpertsNvfp4( - sharded_inputs[0], - sharded_inputs[3], - sharded_inputs[4], + self.input, + self.gate_up_weight_unpacked, + self.down_weight_unpacked, local_topk_weights, local_topk_ids, - sharded_inputs[5], - sharded_inputs[6], - sharded_inputs[7], - sharded_inputs[8], - sharded_inputs[9], - sharded_inputs[10], + self.gate_up_input_scale, + self.gate_up_scales, + self.gate_up_global_scale, + self.down_input_scale, + self.down_scales, + self.down_global_scale, .{ - .workspace_query_device = ctx.workspace_query_device, - .activation = ctx.activation, - .enable_pdl = ctx.enable_pdl, - .gemm1_tactic = ctx.gemm1_tactic, - .gemm2_tactic = ctx.gemm2_tactic, + .workspace_query_device = self.workspace_query_device, + .activation = self.activation, + .enable_pdl = self.enable_pdl, + .gemm1_tactic = self.gemm1_tactic, + .gemm2_tactic = self.gemm2_tactic, }, ) catch |err| stdx.debug.panic( "FlashInfer CUTLASS NVFP4 MoE backend failed: {}", .{err}, ); const local_reshaped = local_output - .reshape(sharded_inputs[0].shape().dims()) + .reshape(self.input.shape().dims()) .withTags(.{ .b, .s, .d }); return zml.ops.allReduce(local_reshaped, zml.Tensor.add); } }).body, + .{ + .input = input, + .topk_ids = topk_ids, + .topk_weights = topk_weights, + .gate_up_weight_unpacked = gate_up_weight_unpacked, + .down_weight_unpacked = down_weight_unpacked, + .gate_up_input_scale = gate_up.quantization.?.input_scale.?.asMultiplier(), + .gate_up_scales = gate_up.quantization.?.scales, + .gate_up_global_scale = gate_up.quantization.?.global_scale.?.asMultiplier(), + .down_input_scale = down.quantization.?.input_scale.?.asMultiplier(), + .down_scales = down.quantization.?.scales, + .down_global_scale = down.quantization.?.global_scale.?.asMultiplier(), + .activation = runner_options.activation, + .enable_pdl = runner_options.enable_pdl, + .gemm1_tactic = runner_options.gemm1_tactic, + .gemm2_tactic = runner_options.gemm2_tactic, + .workspace_query_device = runner_options.workspace_query_device, + }, + input.shape(), ); } @@ -327,62 +340,75 @@ pub fn forwardMoe( if (expert_partition.eql(.init(.experts))) { break :b zml.ops.manualComputation( - .{ input, topk_ids, topk_weights, gate_up.weight, down.weight }, - input.shape(), - .{ - .activation = runner_options.activation, - .enable_pdl = runner_options.enable_pdl, - .gemm1_tactic = runner_options.gemm1_tactic, - .gemm2_tactic = runner_options.gemm2_tactic, - .workspace_query_device = runner_options.workspace_query_device, - }, (struct { + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + weights_gate_up: zml.Tensor, + weights_down: zml.Tensor, + activation: cutlass_flashinfer.Activation, + enable_pdl: bool, + gemm1_tactic: i32, + gemm2_tactic: i32, + workspace_query_device: i32, + fn body( - ctx: anytype, - _: std.mem.Allocator, - sharded_inputs: []const zml.Tensor, + self: @This(), _: zml.Shape, ) zml.Tensor { - const local_num_experts = sharded_inputs[3].dim(.expert); + const local_num_experts = self.weights_gate_up.dim(.expert); const partition_id = zml.ops.partitionId().convert(.i32); const expert_start = partition_id.scale(local_num_experts).convert(.i32); const expert_end = expert_start.addConstant(local_num_experts); - const local_route_mask = sharded_inputs[1] + const local_route_mask = self.topk_ids .cmp(.GE, expert_start) - .logical(.AND, sharded_inputs[1].cmp(.LT, expert_end)); + .logical(.AND, self.topk_ids.cmp(.LT, expert_end)); const local_topk_ids = local_route_mask.select( - sharded_inputs[1].sub(expert_start), + self.topk_ids.sub(expert_start), zml.Tensor.scalar(0, .i32), ); const local_topk_weights = local_route_mask.select( - sharded_inputs[2], - zml.Tensor.scalar(0, sharded_inputs[2].dtype()), + self.topk_weights, + zml.Tensor.scalar(0, self.topk_weights.dtype()), ); const local_output = cutlass_flashinfer.fusedExpertsBf16( - sharded_inputs[0], - sharded_inputs[3], - sharded_inputs[4], + self.input, + self.weights_gate_up, + self.weights_down, local_topk_weights, local_topk_ids, .{ - .workspace_query_device = ctx.workspace_query_device, - .activation = ctx.activation, - .enable_pdl = ctx.enable_pdl, - .gemm1_tactic = ctx.gemm1_tactic, - .gemm2_tactic = ctx.gemm2_tactic, + .workspace_query_device = self.workspace_query_device, + .activation = self.activation, + .enable_pdl = self.enable_pdl, + .gemm1_tactic = self.gemm1_tactic, + .gemm2_tactic = self.gemm2_tactic, }, ) catch |err| stdx.debug.panic( "FlashInfer CUTLASS MoE backend failed: {}", .{err}, ); const local_reshaped = local_output - .reshape(sharded_inputs[0].shape().dims()) + .reshape(self.input.shape().dims()) .withTags(.{ .b, .s, .d }); return zml.ops.allReduce(local_reshaped, zml.Tensor.add); } }).body, + .{ + .input = input, + .topk_ids = topk_ids, + .topk_weights = topk_weights, + .weights_gate_up = gate_up.weight, + .weights_down = down.weight, + .activation = runner_options.activation, + .enable_pdl = runner_options.enable_pdl, + .gemm1_tactic = runner_options.gemm1_tactic, + .gemm2_tactic = runner_options.gemm2_tactic, + .workspace_query_device = runner_options.workspace_query_device, + }, + input.shape(), ); } @@ -425,28 +451,28 @@ pub fn forwardMoe( ); } - const manual_inputs: []const zml.Tensor = if (quant_scheme == .mxfp4) - &.{ input, topk_ids, topk_weights, gate_up.weight, down.weight, gate_up_scales.?, down_scales.? } - else - &.{ input, topk_ids, topk_weights, gate_up.weight, down.weight }; break :b zml.ops.manualComputation( - manual_inputs, - input.shape(), - .{ - .activation = parameters.triton.activation, - .global_num_experts = global_num_experts, - .bias_gate_up = gate_up.bias, - .bias_down = down.bias, - .quant_scheme = quant_scheme, - .activation_threshold = opts.activation_threshold, - }, (struct { - fn body(ctx: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, _: zml.Shape) zml.Tensor { - const local_num_experts = sharded_inputs[3].dim(.expert); + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + weights_gate_up: zml.Tensor, + weights_down: zml.Tensor, + activation: triton.Parameters.ActivationMode, + global_num_experts: i64, + bias_gate_up: ?zml.Tensor, + bias_down: ?zml.Tensor, + quant_scheme: ?zml.Quantization.Scheme, + activation_threshold: ?f32, + scales_gate_up: ?zml.Tensor, + scales_down: ?zml.Tensor, + + fn call(self: @This(), _: zml.Shape) zml.Tensor { + const local_num_experts = self.weights_gate_up.dim(.expert); const partition_id = zml.ops.partitionId().convert(.i32); const expert_start = partition_id.scale(local_num_experts).convert(.i32); // List of global expert ids - const global_expert_ids = zml.Tensor.arange(.{ .end = ctx.global_num_experts }, .i32).withTags(.{.expert}); + const global_expert_ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); // Mapping of local experts to global expert ids, -1 if the global expert is not present in the local partition const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) @@ -457,28 +483,44 @@ pub fn forwardMoe( ); const local_output = triton.fusedExpertsImpl( - sharded_inputs[0], - sharded_inputs[3], - sharded_inputs[4], - sharded_inputs[2], - sharded_inputs[1], + self.input, + self.weights_gate_up, + self.weights_down, + self.topk_weights, + self.topk_ids, .{}, .{ - .activation = ctx.activation, - .global_num_experts = ctx.global_num_experts, + .activation = self.activation, + .global_num_experts = self.global_num_experts, .expert_map = expert_map, - .w1_scale = if (ctx.quant_scheme == .mxfp4) sharded_inputs[5] else null, - .w2_scale = if (ctx.quant_scheme == .mxfp4) sharded_inputs[6] else null, - .w1_bias = ctx.bias_gate_up, - .w2_bias = ctx.bias_down, - .quant_scheme = ctx.quant_scheme, - .activation_threshold = ctx.activation_threshold, + .w1_scale = self.scales_gate_up, + .w2_scale = self.scales_down, + .w1_bias = self.bias_gate_up, + .w2_bias = self.bias_down, + .quant_scheme = self.quant_scheme, + .activation_threshold = self.activation_threshold, }, ) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); - const local_reshaped = local_output.reshape(sharded_inputs[0].shape().dims()).withTags(.{ .b, .s, .d }); + const local_reshaped = local_output.reshape(self.input.shape().dims()).withTags(.{ .b, .s, .d }); return zml.ops.allReduce(local_reshaped, zml.Tensor.add); } - }).body, + }).call, + .{ + .input = input, + .topk_ids = topk_ids, + .topk_weights = topk_weights, + .weights_gate_up = gate_up.weight, + .weights_down = down.weight, + .activation = parameters.triton.activation, + .global_num_experts = global_num_experts, + .bias_gate_up = gate_up.bias, + .bias_down = down.bias, + .quant_scheme = quant_scheme, + .activation_threshold = opts.activation_threshold, + .scales_gate_up = gate_up_scales, + .scales_down = down_scales, + }, + input.shape(), ); }, .mosaic_tpu => b: { @@ -492,22 +534,24 @@ pub fn forwardMoe( if (expert_partition.eql(.init(.experts))) { const global_num_experts = down.weight.dim(.expert); const partial_output = zml.ops.manualComputation( - .{ input, topk_ids, topk_weights, gate_up.weight, down.weight }, - input.shape(), - .{ - .activation = parameters.mosaic_tpu.activation, - .global_num_experts = global_num_experts, - .gate_up_scales = gate_up_scales, - .bias_gate_up = gate_up.bias, - .down_scales = down_scales, - .bias_down = down.bias, - }, (struct { - fn body(ctx: anytype, _: std.mem.Allocator, sharded_inputs: []const zml.Tensor, _: zml.Shape) zml.Tensor { - const local_num_experts = sharded_inputs[3].dim(.expert); + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + weights_gate_up: zml.Tensor, + weights_down: zml.Tensor, + activation: mosaic_tpu.ActivationMode, + global_num_experts: i64, + gate_up_scales: ?zml.Tensor, + bias_gate_up: ?zml.Tensor, + down_scales: ?zml.Tensor, + bias_down: ?zml.Tensor, + + fn body(self: @This(), _: zml.Shape) zml.Tensor { + const local_num_experts = self.weights_gate_up.dim(.expert); const partition_id = zml.ops.partitionId().convert(.i32); const expert_start = partition_id.scale(local_num_experts).convert(.i32); - const global_expert_ids = zml.Tensor.arange(.{ .end = ctx.global_num_experts }, .i32).withTags(.{.expert}); + const global_expert_ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); @@ -516,25 +560,39 @@ pub fn forwardMoe( zml.Tensor.scalar(-1, .i32), ); const local_output = mosaic_tpu.fusedExpertsImpl( - sharded_inputs[0], - sharded_inputs[3], - sharded_inputs[4], - sharded_inputs[2], - sharded_inputs[1], + self.input, + self.weights_gate_up, + self.weights_down, + self.topk_weights, + self.topk_ids, .{}, .{ - .activation = ctx.activation, - .global_num_experts = ctx.global_num_experts, + .activation = self.activation, + .global_num_experts = self.global_num_experts, .expert_map = expert_map, - .w1_scale = ctx.gate_up_scales, - .w2_scale = ctx.down_scales, - .w1_bias = ctx.bias_gate_up, - .w2_bias = ctx.bias_down, + .w1_scale = self.gate_up_scales, + .w2_scale = self.down_scales, + .w1_bias = self.bias_gate_up, + .w2_bias = self.bias_down, }, ) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); - return local_output.reshape(sharded_inputs[0].shape().dims()).withTags(.{ .b, .s, .d }); + return local_output.reshape(self.input.shape().dims()).withTags(.{ .b, .s, .d }); } }).body, + .{ + .input = input, + .topk_ids = topk_ids, + .topk_weights = topk_weights, + .weights_gate_up = gate_up.weight, + .weights_down = down.weight, + .activation = parameters.mosaic_tpu.activation, + .global_num_experts = global_num_experts, + .gate_up_scales = gate_up_scales, + .bias_gate_up = gate_up.bias, + .down_scales = down_scales, + .bias_down = down.bias, + }, + input.shape(), ); break :b zml.ops.allReduce(partial_output, zml.Tensor.add); } diff --git a/zml/ops.zig b/zml/ops.zig index ca291f75..2dccdb25 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1909,33 +1909,10 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, } pub fn manualComputation( - inputs: anytype, - outputs: anytype, - body_context: anytype, comptime body_fn: anytype, + inputs: stdx.meta.FnParam(body_fn, 0), + outputs: anytype, ) manualComputationReturnType(body_fn) { - const inputs_: []const Tensor = switch (@typeInfo(@TypeOf(inputs))) { - .@"struct" => |struct_info| b: { - if (@TypeOf(inputs) == Tensor) { - break :b &[1]Tensor{inputs}; - } - if (!struct_info.is_tuple) @compileError("Expected tuple inputs"); - var inputs_flat: [struct_info.fields.len]Tensor = undefined; - meta.collectBuf((struct { - pub fn func(t: Tensor) Tensor { - return t; - } - }).func, {}, &inputs, &inputs_flat); - break :b &inputs_flat; - }, - .array => &inputs, - .pointer => |pointer_info| b: { - if (pointer_info.size != .slice) @compileError("Expected input slice"); - break :b inputs; - }, - else => @compileError("Unsupported manualComputation input type: " ++ @typeName(@TypeOf(inputs))), - }; - const output_shapes: []const Shape = switch (@typeInfo(@TypeOf(outputs))) { .void => &.{}, .@"struct" => |struct_info| b: { @@ -1959,22 +1936,39 @@ pub fn manualComputation( else => @compileError("Unsupported manualComputation output type: " ++ @typeName(@TypeOf(outputs))), }; - const sharded_outputs: []const Tensor = manualComputationInternal(inputs_, output_shapes, body_context, body_fn) catch |err| switch (err) { + const sharded_outputs: []const Tensor = manualComputationInternal(inputs, output_shapes, body_fn) catch |err| switch (err) { error.OutOfMemory => @panic("OOM"), }; const ReturnT = manualComputationReturnType(body_fn); return manualComputationSliceToReturn(ReturnT, sharded_outputs); } +fn manualComputationLocalizeInputs(allocator: std.mem.Allocator, inputs: anytype, local_tensors: []const Tensor) !@TypeOf(inputs) { + const Context = struct { + local_tensors: []const Tensor, + input_idx: usize = 0, + + fn localize(self: *@This(), _: Tensor) Tensor { + stdx.debug.assert(self.input_idx < self.local_tensors.len, "manualComputation ran out of localized inputs", .{}); + defer self.input_idx += 1; + return self.local_tensors[self.input_idx]; + } + }; + + var context: Context = .{ .local_tensors = local_tensors }; + var local_inputs: @TypeOf(inputs) = undefined; + try meta.mapAlloc(Context.localize, allocator, &context, inputs, &local_inputs); + stdx.debug.assert(context.input_idx == local_tensors.len, "manualComputation localized {} inputs, expected {}", .{ context.input_idx, local_tensors.len }); + return local_inputs; +} + fn manualComputationInternal( - inputs: []const Tensor, + inputs: anytype, outputs: []const Shape, - body_context: anytype, comptime body_fn: anytype, ) error{OutOfMemory}![]Tensor { const BodyReturnT = manualComputationReturnType(body_fn); - const BodyInputsT = stdx.meta.FnParam(body_fn, 2); - const BodyOutputShapesT = stdx.meta.FnParam(body_fn, 3); + const BodyOutputShapesT = stdx.meta.FnParam(body_fn, 1); const ctx = Compiler.current(); const scope = ctx.currentScope(); @@ -1983,16 +1977,12 @@ fn manualComputationInternal( defer arena_state.deinit(); const arena = arena_state.allocator(); - const input_shapes = try arena.alloc(Shape, inputs.len); - const input_values = try arena.alloc(*const mlir.Value, inputs.len); + const input_shapes = try meta.collectAlloc(Tensor.shape, {}, arena, &inputs); + const input_values = try meta.collectAlloc(Tensor.value, {}, arena, &inputs); - for (inputs, 0..) |input, i| { - input_shapes[i] = input.shape(); - input_values[i] = input.value(); - } - const local_input_shapes = try arena.alloc(Shape, inputs.len); + const local_input_shapes = try arena.alloc(Shape, input_shapes.len); const local_output_shapes = try arena.alloc(Shape, outputs.len); - const input_shardings = try arena.alloc(Sharding, inputs.len); + const input_shardings = try arena.alloc(Sharding, input_shapes.len); const output_shardings = try arena.alloc(Sharding, outputs.len); for (input_shapes, 0..) |shape, i| { @@ -2022,11 +2012,11 @@ fn manualComputationInternal( const out_shardings_attr = try Sharding.sdyPerValueShardingAttr(arena, ctx.mlir_ctx, outputs, output_shardings); const manual_axes_attr = try Sharding.sdyManualAxesAttr(arena, ctx.mlir_ctx, input_shapes, input_shardings, outputs, output_shardings); - const block_types = try arena.alloc(*const mlir.Type, inputs.len); + const block_types = try arena.alloc(*const mlir.Type, input_shapes.len); for (local_input_shapes, 0..) |input_shape, i| { block_types[i] = mlirx.Type.rankedTensor(ctx.mlir_ctx, input_shape); } - const block_locs = try arena.alloc(*const mlir.Location, inputs.len); + const block_locs = try arena.alloc(*const mlir.Location, input_shapes.len); @memset(block_locs, mlir.Location.unknown(ctx.mlir_ctx)); const manual_block = mlir.Block.init(block_types, block_locs); @@ -2035,17 +2025,17 @@ fn manualComputationInternal( const manual_scope = ctx.pushBlock(manual_block); defer manual_scope.pop(); - const local_inputs = try arena.alloc(Tensor, inputs.len); - for (0..inputs.len) |i| { - local_inputs[i] = Tensor._result(local_input_shapes[i], manual_block.argument(i)); + const local_input_tensors = try arena.alloc(Tensor, input_shapes.len); + for (0..input_shapes.len) |i| { + local_input_tensors[i] = Tensor._result(local_input_shapes[i], manual_block.argument(i)); } ctx.manual_computation_depth += 1; defer ctx.manual_computation_depth -= 1; - const body_inputs = manualComputationInputsArg(BodyInputsT, local_inputs); + const local_inputs = try manualComputationLocalizeInputs(arena, inputs, local_input_tensors); const body_output_shapes = manualComputationOutputShapesArg(BodyOutputShapesT, local_output_shapes); - const body_result = @call(.auto, body_fn, .{ body_context, arena, body_inputs, body_output_shapes }); + const body_result = @call(.auto, body_fn, .{ local_inputs, body_output_shapes }); const local_outputs = manualComputationBodyToSlice(BodyReturnT, arena, body_result); stdx.debug.assert(local_outputs.len == outputs.len, "manualComputation body returned {} values, expected {}", .{ local_outputs.len, outputs.len }); @@ -2087,8 +2077,8 @@ fn manualComputationInternal( return sharded_outputs; }, .gspmd => { - const local_input_values = try arena.alloc(*const mlir.Value, inputs.len); - for (0..inputs.len) |i| { + const local_input_values = try arena.alloc(*const mlir.Value, input_shapes.len); + for (0..input_shapes.len) |i| { const local_type = mlirx.Type.rankedTensor(ctx.mlir_ctx, local_input_shapes[i]); const full_to_shard = dialects.stablehlo.custom_call( ctx.mlir_ctx, @@ -2107,16 +2097,16 @@ fn manualComputationInternal( local_input_values[i] = full_to_shard.result(0); } - const local_inputs = try arena.alloc(Tensor, inputs.len); - for (0..inputs.len) |i| { - local_inputs[i] = Tensor._result(local_input_shapes[i], local_input_values[i]); + const local_input_tensors = try arena.alloc(Tensor, input_shapes.len); + for (0..input_shapes.len) |i| { + local_input_tensors[i] = Tensor._result(local_input_shapes[i], local_input_values[i]); } ctx.manual_computation_depth += 1; defer ctx.manual_computation_depth -= 1; - const body_inputs = manualComputationInputsArg(BodyInputsT, local_inputs); + const local_inputs = try manualComputationLocalizeInputs(arena, inputs, local_input_tensors); const body_output_shapes = manualComputationOutputShapesArg(BodyOutputShapesT, local_output_shapes); - const body_result = @call(.auto, body_fn, .{ body_context, arena, body_inputs, body_output_shapes }); + const body_result = @call(.auto, body_fn, .{ local_inputs, body_output_shapes }); const local_outputs = manualComputationBodyToSlice(BodyReturnT, arena, body_result); stdx.debug.assert(local_outputs.len == outputs.len, "manualComputation body returned {} values, expected {}", .{ local_outputs.len, outputs.len }); for (0..outputs.len) |i| { @@ -2165,6 +2155,112 @@ fn manualComputationInternal( }; } +test "manualComputation handler API" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); + defer comp.deinit(); + comp.activate(); + defer comp.deactivate(); + + const block = mlir.Block.init(&.{}, &.{}); + const scope = comp.pushBlock(block); + defer scope.pop(); + + const shape = Shape.init(.{8}, .f32); + const lhs = Tensor.constant(DataType.f32.constant(1)).broad(shape); + const rhs = Tensor.constant(DataType.f32.constant(2)).broad(shape); + + const Configured = struct { + lhs: Tensor, + bias: ?Tensor, + missing_bias: ?Tensor, + enabled: bool, + rhs: Tensor, + + pub fn compute(self: @This(), output_shape: Shape) Tensor { + stdx.debug.assert(self.enabled, "manualComputation did not preserve handler configuration", .{}); + stdx.debug.assert(self.missing_bias == null, "manualComputation did not preserve a null optional tensor", .{}); + return self.lhs.add(self.rhs).add(self.bias.?).reshape(output_shape); + } + }; + const configured = manualComputation( + Configured.compute, + .{ + .lhs = lhs, + .bias = rhs, + .missing_bias = null, + .enabled = true, + .rhs = rhs, + }, + shape, + ); + try zml.testing.expectEqualShapes(shape, configured.shape()); + + const passthrough = manualComputation( + (struct { + input: Tensor, + + pub fn call(self: @This(), output_shape: Shape) Tensor { + return self.input.reshape(output_shape); + } + }).call, + .{ .input = configured }, + shape, + ); + try zml.testing.expectEqualShapes(shape, passthrough.shape()); + + const NestedInputs = struct { + pair: [2]Tensor, + optional: ?struct { tensor: Tensor }, + }; + const slice_inputs = [_]Tensor{ lhs, rhs }; + const original_values = [_]usize{ + @intFromPtr(lhs.value()), + @intFromPtr(rhs.value()), + @intFromPtr(lhs.value()), + @intFromPtr(lhs.value()), + @intFromPtr(rhs.value()), + }; + + const NestedStruct = struct { + nested_inputs: NestedInputs, + slice_inputs: []const Tensor, + original_values: [5]usize, + + pub fn compute(self: @This(), output_shape: Shape) Tensor { + const localized_inputs = [_]Tensor{ + self.nested_inputs.pair[0], + self.nested_inputs.pair[1], + self.nested_inputs.optional.?.tensor, + self.slice_inputs[0], + self.slice_inputs[1], + }; + for (localized_inputs, self.original_values) |input, original_value| { + stdx.debug.assert(@intFromPtr(input.value()) != original_value, "manualComputation did not localize a nested input", .{}); + } + + var result = localized_inputs[0]; + for (localized_inputs[1..]) |input| result = result.add(input); + return result.reshape(output_shape); + } + }; + const nested = manualComputation( + NestedStruct.compute, + .{ + .nested_inputs = .{ + .pair = .{ lhs, rhs }, + .optional = .{ .tensor = lhs }, + }, + .slice_inputs = &slice_inputs, + .original_values = original_values, + }, + shape, + ); + try zml.testing.expectEqualShapes(shape, nested.shape()); +} + fn manualComputationReturnType(comptime body_fn: anytype) type { const ReturnT = stdx.meta.FnReturn(body_fn); if (ReturnT == void or ReturnT == Tensor) return ReturnT; @@ -2191,30 +2287,6 @@ fn manualComputationBodyToSlice(comptime ReturnT: type, allocator: std.mem.Alloc return result; } -fn manualComputationInputsArg(comptime InputsT: type, local_inputs: []const Tensor) InputsT { - if (InputsT == void) { - stdx.debug.assert(local_inputs.len == 0, "manualComputation body expects no inputs, got {}", .{local_inputs.len}); - return; - } - if (InputsT == Tensor) { - stdx.debug.assert(local_inputs.len == 1, "manualComputation body expects one input, got {}", .{local_inputs.len}); - return local_inputs[0]; - } - - return switch (@typeInfo(InputsT)) { - .pointer => |pointer_info| b: { - if (pointer_info.size != .slice or pointer_info.child != Tensor) { - @compileError("manualComputation body inputs must be void, Tensor, []Tensor, or []const Tensor; got " ++ @typeName(InputsT)); - } - break :b if (pointer_info.is_const) - local_inputs - else - @constCast(local_inputs); - }, - else => @compileError("manualComputation body inputs must be void, Tensor, []Tensor, or []const Tensor; got " ++ @typeName(InputsT)), - }; -} - fn manualComputationOutputShapesArg(comptime OutputShapesT: type, local_output_shapes: []const Shape) OutputShapesT { if (OutputShapesT == void) { stdx.debug.assert(local_output_shapes.len == 0, "manualComputation body expects no output shapes, got {}", .{local_output_shapes.len}); @@ -2416,22 +2488,24 @@ pub fn shardingAwareTypedCustomCall( const Output = @TypeOf(output); const Attributes = @TypeOf(attributes); - // Convert to slices so that manualComputationInternal can inspect the input/output. - var input_tensors: [@typeInfo(Input).@"struct".fields.len]Tensor = undefined; - inline for (@typeInfo(Input).@"struct".fields, 0..) |field, i| { - input_tensors[i] = @field(input, field.name); - } - var output_shapes: [@typeInfo(Output).@"struct".fields.len]Shape = undefined; inline for (@typeInfo(Output).@"struct".fields, 0..) |field, i| { output_shapes[i] = @field(output, field.name); } - const output_tensors: []const Tensor = manualComputationInternal(&input_tensors, &output_shapes, attributes, (struct { - fn body(attributes_: Attributes, _: std.mem.Allocator, sharded_input_tensors: []const Tensor, sharded_output_shapes: []const Shape) []const Tensor { - return typedCustomCall(target_name, opts, sharded_input_tensors, sharded_output_shapes, attributes_); + const Handler = struct { + input: Input, + attributes: Attributes, + + fn body(self: @This(), sharded_output_shapes: []const Shape) []const Tensor { + return typedCustomCall(target_name, opts, self.input, sharded_output_shapes, self.attributes); } - }).body) catch @panic("OOM"); + }; + const output_tensors = manualComputation( + Handler.body, + .{ .input = input, .attributes = attributes }, + @as([]const Shape, &output_shapes), + ); // Convert the slice back to a struct var out: ShapeToTensor(Output) = undefined; diff --git a/zml/tensor.zig b/zml/tensor.zig index 68ca695a..cca909ff 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -4590,11 +4590,14 @@ pub const Tensor = struct { defer ctx.arena.allocator().free(full_name); switch (ctx.platform.target) { .cpu, .cuda, .rocm, .tpu, .metal => { - ops.manualComputation(input, {}, full_name, (struct { - fn body(_full_name: []const u8, _: std.mem.Allocator, sharded_input: Tensor, _: void) void { - ops.customCall("zml$print", sharded_input, {}, .{ .name = _full_name }, .{ .has_side_effect = true }); + ops.manualComputation((struct { + input: Tensor, + name: []const u8, + + fn body(body_ctx: @This(), _: void) void { + ops.customCall("zml$print", body_ctx.input, {}, .{ .name = body_ctx.name }, .{ .has_side_effect = true }); } - }).body); + }).body, .{ .input = input, .name = full_name }, {}); }, .oneapi, .neuron => {}, } From 1b1cfe89abfa6d5408e4fd97ab4c06c82d1ea1d8 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Thu, 3 Sep 2026 17:15:45 +0200 Subject: [PATCH 653/881] zml/nn: unpack FP4 along an explicit axis (#767) Make unpackFp4 accept the packed source axis explicitly instead of requiring the hardcoded .kw tag. Updates the Linear and MoE callers and adds regression coverage for expanding and renaming the packed axis. --- zml/moe/moe.zig | 2 +- zml/nn.zig | 30 +++++++++++++++++++++++------- zml/quantization.zig | 8 ++++---- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index 46a0b785..d3b8c567 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -647,7 +647,7 @@ pub fn forwardMoe( fn unpackedWeight(linear: zml.nn.Linear) zml.Tensor { const quantization = linear.quantization orelse return linear.weight; return if (zml.nn.isPackedFp4(quantization.scheme, linear.weight.dtype())) - zml.nn.unpackFp4(linear.weight, linear.tag) + zml.nn.unpackFp4(linear.weight, linear.tag, linear.tag) else linear.weight; } diff --git a/zml/nn.zig b/zml/nn.zig index 9e328127..367fac30 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -45,7 +45,7 @@ pub const Linear = struct { const weight_global_scale: ?Tensor = if (q.global_scale) |s| s.asMultiplier() else null; - const weight = if (isPackedFp4(q.scheme, self.weight.dtype())) unpackFp4(self.weight, self.tag) else self.weight; + const weight = if (isPackedFp4(q.scheme, self.weight.dtype())) unpackFp4(self.weight, self.tag, self.tag) else self.weight; const scales = if (q.scheme.isMx() and q.scales.dtype() == .u8) q.scales.bitCast(.f8e8m0) else @@ -71,14 +71,30 @@ pub fn isPackedFp4(scheme: ?Quantization.Scheme, weight_dtype: DataType) bool { return (scheme == .nvfp4 or scheme == .mxfp4) and (weight_dtype == .u8 or weight_dtype == .i8); } -/// Unpacks two f4e2m1 values per byte. `w` must be tagged with `.kw` on the packed axis, -/// which is merged with the unpacked pair and renamed to `k_tag`. Group-size agnostic: -/// NVFP4 (16) and MXFP4 (32) share this packing. -pub fn unpackFp4(w: Tensor, k_tag: anytype) Tensor { +/// Unpacks two f4e2m1 values per byte along `packed_tag` and names the expanded +/// axis `k_tag`. Group-size agnostic: NVFP4 (16) and MXFP4 (32) share this packing. +pub fn unpackFp4(w: Tensor, packed_tag: anytype, contracting_tag: anytype) Tensor { stdx.debug.assert(w.dtype() == .u8 or w.dtype() == .i8, "unpackFp4 expects packed 8-bit weights, got {}", .{w.dtype()}); + const source_tag = Shape.toTag(packed_tag); + const result_tag = Shape.toTag(contracting_tag); return w.bitCast(.f4e2m1) // bitcast inserts a tag (it respects shlo), but maybe we should simplify it - .merge(.{ .kb = .{ .kw, .bitcast } }) - .renameTag(.kb, Shape.toTag(k_tag)); + .merge(.{ .kb = .{ source_tag, .bitcast } }) + .renameTag(.kb, result_tag); +} + +test "unpackFp4 expands the requested axis" { + const platform = zml.testing.env(); + const packed_weight: Tensor = .init(.{ .out = 2, .stored = 4 }, .u8); + const Local = struct { + fn call(w: Tensor) Tensor { + return unpackFp4(w, .stored, .contracted); + } + }; + + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Local.call, .{packed_weight}, .{}); + defer exe.deinit(); + + try zml.testing.expectEqualShapes(.init(.{ .out = 2, .contracted = 8 }, .f4e2m1), exe.output_shapes[0]); } /// Scaled matrix multiply (`xla.scaled_dot`): `acc = (lhs * lhs_scale) @ (rhs * rhs_scale)`. diff --git a/zml/quantization.zig b/zml/quantization.zig index f57f60b7..de1e324d 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -169,7 +169,7 @@ test "Quantization.Scheme.classify" { // unsloth/Qwen3.6-27B-NVFP4: compressed-tensors, carrying both of its schemes under // `weight_scale` -- NVFP4 on the MLPs of layers 0-55, FP8 per-channel everywhere else. - const nvfp4_packed: Shape = .init(.{ .dout = 17408, .kw = 2560 }, .u8); + const nvfp4_packed: Shape = .init(.{ .dout = 17408, .d = 2560 }, .u8); try expect(@as(?Quantization.Scheme, .nvfp4), Quantization.Scheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 320 }, .f8e4m3fn))); try expect(@as(?Quantization.Scheme, .fp8_per_channel), Quantization.Scheme.classify( .init(.{ .dout = 17408, .d = 5120 }, .f8e4m3fn), @@ -178,7 +178,7 @@ test "Quantization.Scheme.classify" { // nvidia/Gemma-4-31B-IT-NVFP4: ModelOpt, packed values under the plain `weight` name. try expect(@as(?Quantization.Scheme, .nvfp4), Quantization.Scheme.classify( - .init(.{ .dout = 21504, .kw = 2688 }, .u8), + .init(.{ .dout = 21504, .d = 2688 }, .u8), .init(.{ .dout = 21504, .sc = 336 }, .f8e4m3fn), )); @@ -234,7 +234,7 @@ test "Quantization.Scheme.classify" { try expect(@as(?Quantization.Scheme, .mxfp4), Quantization.Scheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 160 }, .f8e8m0))); // DeepSeek V4 stores the same packed FP4 bits in signed bytes. try expect(@as(?Quantization.Scheme, .mxfp4), Quantization.Scheme.classify( - .init(.{ .dout = 2048, .kw = 2048 }, .i8), + .init(.{ .dout = 2048, .d = 2048 }, .i8), .init(.{ .dout = 2048, .sc = 128 }, .f8e8m0), )); // Native (unpacked) f4e2m1, K no longer halved. @@ -261,7 +261,7 @@ test "Quantization.Scheme.classify" { try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify(fp8, .init(.{ .dout = 160, .sc = 80 }, .bf16))); // Stacked MoE weights are rank 3 and go through the MoE path, not this one. try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify( - .init(.{ .expert = 128, .dout = 1408, .kw = 1408 }, .u8), + .init(.{ .expert = 128, .dout = 1408, .d = 1408 }, .u8), .init(.{ .expert = 128, .dout = 1408, .sc = 176 }, .f8e4m3fn), )); From 54bf33a02f5ee6220da2ab6de10afdd0a3acde99 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Thu, 3 Sep 2026 17:17:09 +0200 Subject: [PATCH 654/881] zml/Compiler: Provide accurate device memory size to XLA (#764) By default, XLA assumes 80% of the memory to be available during compilation. It wasn't an issue as long as we had one executable per layer, because most of the weights weren't parts of the inputs, but for a full forward executable we exceed that 80% threshold in llmd with the cache being sized to use almost everything up to the BFC limit. So XLA ends up generating an executable the least possible temporary buffers even though it has more than that. --- pjrt/pjrt.zig | 2 +- zml/Compiler.zig | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 6f498971..5f71050e 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -77,7 +77,7 @@ pub const ApiError = error{ inline fn interpretPjrtError(api: *const Api, pjrt_error: *Error, context: []const u8) ApiError { defer pjrt_error.deinit(api); const err_code = pjrt_error.getCode(api).toApiError(); - log.err("[{s}] {t}: {s}", .{ context, err_code, pjrt_error.getMessage(api) }); + log.warn("[{s}] {t}: {s}", .{ context, err_code, pjrt_error.getMessage(api) }); return err_code; } diff --git a/zml/Compiler.zig b/zml/Compiler.zig index fa662bd0..adf85695 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -686,6 +686,23 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: c.xla_ExecutableBuildOptionsProto_set_use_spmd_partitioning(exec_build_options, true); c.xla_ExecutableBuildOptionsProto_set_use_shardy_partitioner(exec_build_options, use_shardy_partitioner); + // Tell XLA how much device memory PJRT actually made available. + // Without this, the GPU scheduler falls back to 80% of physical + // memory. Programs whose donated inputs and outputs exceed that + // artificial limit get a zero-byte temporary-memory budget and + // can trigger excessive rematerialization. + var device_memory_size: ?u64 = null; + for (platform.devices) |device| { + const bytes_limit = device.memoryStats().bytes_limit orelse { + device_memory_size = null; + break; + }; + device_memory_size = @min(device_memory_size orelse bytes_limit, bytes_limit); + } + if (device_memory_size) |bytes_limit| { + c.xla_ExecutableBuildOptionsProto_set_device_memory_size(exec_build_options, @intCast(bytes_limit)); + } + c.xla_ExecutableBuildOptionsProto_set_device_assignment(exec_build_options, device_assignment_blk: { const device_assignment_proto = try upb.new(c.xla_DeviceAssignmentProto, upb_arena); From 0b41272a0854d15b341f660c445c6323bb19f8b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20Gauti=C3=A9?= <51361266+tgautie@users.noreply.github.com> Date: Thu, 3 Sep 2026 18:26:40 +0200 Subject: [PATCH 655/881] attention/flashattn: fix sliding window off-by-one error (#763) We fix an off-by-one error in Flash-Attn: `window_size_left` is inclusive of the current position, so it is minus 1 compared to the actual `sliding_window` size. Ref: https://github.com/Dao-AILab/flash-attention#how-to-use-flashattention --- zml/attention/flashattn.zig | 24 +++- zml/attention/paged_attention.zig | 230 +++++++++++++++++------------- zml/slice.zig | 1 - 3 files changed, 146 insertions(+), 109 deletions(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 0c71b32d..142bdbb3 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -900,6 +900,7 @@ pub const paged_fa2 = struct { stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); const ctx = zml.Compiler.current(); + const window_size_left = windowSizeLeft(opts.sliding_window); const num_head_groups = q.dim(.hg); const num_kv_heads = q.dim(.hkv); @@ -979,7 +980,7 @@ pub const paged_fa2 = struct { .is_causal = opts.is_causal, .max_seqlen_k = decode_parameters.options.max_seqlen_k, .num_heads = num_heads_per_shard, - .window_size_left = opts.sliding_window, + .window_size_left = window_size_left, .softmax_scale = opts.scale, }, .opts = zml.ops.CustomCallOptions{ @@ -1066,7 +1067,7 @@ pub const paged_fa2 = struct { .max_seqlen_k = mixed_parameters.options.max_seqlen_k, .max_seqlen_q = mixed_parameters.options.max_seqlen_q, .num_heads = num_heads_per_shard, - .window_size_left = opts.sliding_window, + .window_size_left = window_size_left, .softmax_scale = opts.scale, }, .opts = zml.ops.CustomCallOptions{ @@ -1140,7 +1141,7 @@ pub const paged_fa2 = struct { .is_causal = opts.is_causal, .max_seqlen_k = mixed_parameters.options.max_seqlen_k, .num_heads = num_heads_per_shard, - .window_size_left = opts.sliding_window, + .window_size_left = window_size_left, .softmax_scale = opts.scale, }, .opts = zml.ops.CustomCallOptions{ @@ -1551,6 +1552,7 @@ pub const paged_fa3 = struct { stdx.debug.assert(q.shape().hasTags(.{ .b, .hg, .hkv, .hd }), "Expected q to have tags .b, .h, .hd", .{}); stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); + const window_size_left = windowSizeLeft(opts.sliding_window); const num_head_groups = q.dim(.hg); const num_kv_heads = q.dim(.hkv); @@ -1616,7 +1618,7 @@ pub const paged_fa3 = struct { .metadata = .{ .is_causal = opts.is_causal, .max_seqlen_k = decode_parameters.options.max_seqlen_k, - .window_size_left = opts.sliding_window, + .window_size_left = window_size_left, }, .opts = zml.ops.CustomCallOptions{ .has_side_effect = false, @@ -1694,7 +1696,7 @@ pub const paged_fa3 = struct { .is_causal = opts.is_causal, .max_seqlen_k = mixed_parameters.options.max_seqlen_k, .max_seqlen_q = mixed_parameters.options.max_seqlen_q, - .window_size_left = opts.sliding_window, + .window_size_left = window_size_left, }, .opts = zml.ops.CustomCallOptions{ .has_side_effect = false }, }, @@ -1758,7 +1760,7 @@ pub const paged_fa3 = struct { .metadata = .{ .is_causal = opts.is_causal, .max_seqlen_k = mixed_parameters.options.max_seqlen_k, - .window_size_left = opts.sliding_window, + .window_size_left = window_size_left, }, .opts = zml.ops.CustomCallOptions{ .has_side_effect = false }, }, @@ -1774,3 +1776,13 @@ pub const paged_fa3 = struct { return o; } }; + +fn windowSizeLeft(sliding_window: i32) i32 { + return if (sliding_window > 0) sliding_window - 1 else sliding_window; +} + +test "FlashAttention sliding window uses an inclusive offset" { + try std.testing.expectEqual(@as(i32, 2047), windowSizeLeft(2048)); + try std.testing.expectEqual(@as(i32, 0), windowSizeLeft(1)); + try std.testing.expectEqual(@as(i32, -1), windowSizeLeft(-1)); +} diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index d154db52..8d3a8981 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -402,7 +402,6 @@ test pagedAttention { .max_seqlen_q = 16 * 2, }; const triton_parameters: Parameters = .init(.fromBackend(triton_options_args)); - const attn_opts: AttentionOptions = .{ .is_causal = true }; var q = try zml.testing.autoCall(allocator, io, &rng_q, zml.Tensor.Rng.normal, {}); defer q.deinit(); var new_k = try zml.testing.autoCall(allocator, io, &rng_k, zml.Tensor.Rng.normal, {}); @@ -448,110 +447,137 @@ test pagedAttention { } }; defer zml.Buffer.deinitAll(Parameters, &triton_parameters_d); - var results_per_backend: std.enums.EnumArray(Backend, ?zml.Slice) = .initFill(null); - defer { + const all_backends = std.enums.values(Backend); + // stablehlo ignores these options, cuda_fa3 has no materializer in this fixture, and + // mosaic_tpu requires a dense KV cache instead of the split cache used by this test. + const option_sensitive_backends = [_]Backend{ .cuda_fa2, .triton, .metal }; + const test_cases = [_]struct { + name: []const u8, + attention_options: AttentionOptions, + backends: []const Backend, + }{ + .{ + .name = "unbounded", + .attention_options = .{ .is_causal = true }, + .backends = all_backends, + }, + .{ + .name = "non_causal", + .attention_options = .{ .is_causal = false }, + .backends = &option_sensitive_backends, + }, + .{ + .name = "sliding_window", + .attention_options = .{ .is_causal = true, .sliding_window = page_size }, + .backends = &option_sensitive_backends, + }, + }; + + for (test_cases) |test_case| { + var results_per_backend: std.enums.EnumArray(Backend, ?zml.Slice) = .initFill(null); + defer { + for (std.enums.values(Backend)) |backend| { + if (results_per_backend.get(backend)) |slice| slice.free(allocator); + } + } + + for (test_case.backends) |backend| { + if (!backend.isAvailable(platform)) { + std.log.warn("paged_attention backend {t} not available", .{backend}); + continue; + } + std.log.warn("Testing paged_attention {s} with backend {t}", .{ test_case.name, backend }); + + var backend_options_args = triton_options_args; + backend_options_args.backend = backend; + const parameters: Parameters = .init(.fromBackend(backend_options_args)); + + const exe = try platform.compileFn( + allocator, + io, + pagedAttention, + .{ parameters, tensors.q, tensors.k, tensors.v, tensors.kv_cache, test_case.attention_options }, + .{ .program_name = try std.fmt.allocPrint(arena, "paged_attention_{s}_{t}", .{ test_case.name, backend }), .shardings = shardings }, + ); + defer exe.deinit(); + + var parameters_d: zml.Bufferized(Parameters) = switch (parameters) { + // No materializer implemented for cuda fa3 + .cuda_fa3 => continue, + .cuda_fa2 => |params| cuda_fa2: { + var block_table_prefill: [num_prefill][max_num_pages]i32 = undefined; + @memcpy(&block_table_prefill, block_table[0..num_prefill]); + var block_table_decode: [num_decode][max_num_pages]i32 = undefined; + @memcpy(&block_table_decode, block_table[num_prefill .. num_prefill + num_decode]); + + var cu_seqlens_q_prefill: [num_prefill + 1]i32 = undefined; + @memcpy(&cu_seqlens_q_prefill, query_start_len[0 .. num_prefill + 1]); + var cu_seqlens_q_decode: [num_decode + 1]i32 = undefined; + for (&cu_seqlens_q_decode, query_start_len[num_prefill .. num_prefill + num_decode + 1]) |*decode_len, query_start| { + decode_len.* = query_start - prefill_token_count; + } + + var seqused_k_prefill: [num_prefill]i32 = undefined; + @memcpy(&seqused_k_prefill, seq_lens[0..num_prefill]); + var seqused_k_decode: [num_decode]i32 = undefined; + @memcpy(&seqused_k_decode, seq_lens[num_prefill .. num_prefill + num_decode]); + + break :cuda_fa2 .{ .cuda_fa2 = .{ .mixed = .{ + .block_table_prefill = try .fromBytes(io, platform, params.mixed.block_table_prefill.shape(), .replicated, @ptrCast(&block_table_prefill)), + .cu_seqlens_q_prefill = try .fromBytes(io, platform, params.mixed.cu_seqlens_q_prefill.shape(), .replicated, @ptrCast(&cu_seqlens_q_prefill)), + .seqused_k_prefill = try .fromBytes(io, platform, params.mixed.seqused_k_prefill.shape(), .replicated, @ptrCast(&seqused_k_prefill)), + + .block_table_decode = try .fromBytes(io, platform, params.mixed.block_table_decode.shape(), .replicated, @ptrCast(&block_table_decode)), + .cu_seqlens_q_decode = try .fromBytes(io, platform, params.mixed.cu_seqlens_q_decode.shape(), .replicated, @ptrCast(&cu_seqlens_q_decode)), + .seqused_k_decode = try .fromBytes(io, platform, params.mixed.seqused_k_decode.shape(), .replicated, @ptrCast(&seqused_k_decode)), + + .metadata = .{ .decode_offset = try .scalar(io, platform, prefill_token_count, .i32) }, + } } }; + }, + .triton => triton_parameters_d, + inline .metal, .mosaic_tpu, .stablehlo => |_, t| @unionInit(zml.Bufferized(Parameters), @tagName(t), .{ + .block_table = triton_parameters_d.triton.block_table, + .seq_lens = triton_parameters_d.triton.seq_lens, + .query_start_len = triton_parameters_d.triton.query_start_len, + }), + }; + // cu_fa2 creates new buffers while other reuse triton buffers. + defer if (backend == .cuda_fa2) zml.Buffer.deinitAll(Parameters, ¶meters_d); + + var output_d = try zml.testing.autoCall(allocator, io, &exe, pagedAttention, .{ parameters_d, q, new_k, new_v, kv_cache_d }); + defer output_d.deinit(); + results_per_backend.set(backend, try output_d.toSliceAlloc(allocator, io)); + } + + const reference_backend: Backend = .triton; + const reference = results_per_backend.get(reference_backend) orelse return error.SkipZigTest; + + var num_failed: u32 = 0; for (std.enums.values(Backend)) |backend| { - if (results_per_backend.get(backend)) |slice| slice.free(allocator); + if (backend == reference_backend) continue; + const output_h = results_per_backend.get(backend) orelse continue; + + const tolerance: zml.testing.CompareOpts = .{ + .absolute_tolerance = 1e-2, + .relative_tolerance = 1e-2, + .epsilon_relative = 1e-6, + }; + + zml.testing.expectClose(io, reference, output_h, tolerance) catch |err| switch (err) { + error.TestUnexpectedResult => { + num_failed += 1; + std.log.err("test pagedAttention {s} failed on backend {t} against reference {t}", .{ + test_case.name, + backend, + reference_backend, + }); + }, + else => |e| return e, + }; } + + if (num_failed > 0) return error.TestUnexpectedResult; } - - for (std.enums.values(Backend)) |backend| { - if (!backend.isAvailable(platform)) { - std.log.warn("paged_attention backend {t} not available", .{backend}); - continue; - } - std.log.warn("Testing paged_attention with backend {t}", .{backend}); - - var backend_options_args = triton_options_args; - backend_options_args.backend = backend; - const parameters: Parameters = .init(.fromBackend(backend_options_args)); - - const exe = try platform.compileFn( - allocator, - io, - pagedAttention, - .{ parameters, tensors.q, tensors.k, tensors.v, tensors.kv_cache, attn_opts }, - .{ .program_name = try std.fmt.allocPrint(arena, "paged_attention_{t}", .{backend}), .shardings = shardings }, - ); - defer exe.deinit(); - - var parameters_d: zml.Bufferized(Parameters) = switch (parameters) { - // No materializer implemented for cuda fa3 - .cuda_fa3 => continue, - .cuda_fa2 => |params| cuda_fa2: { - var block_table_prefill: [num_prefill][max_num_pages]i32 = undefined; - @memcpy(&block_table_prefill, block_table[0..num_prefill]); - var block_table_decode: [num_decode][max_num_pages]i32 = undefined; - @memcpy(&block_table_decode, block_table[num_prefill .. num_prefill + num_decode]); - - var cu_seqlens_q_prefill: [num_prefill + 1]i32 = undefined; - @memcpy(&cu_seqlens_q_prefill, query_start_len[0 .. num_prefill + 1]); - var cu_seqlens_q_decode: [num_decode + 1]i32 = undefined; - for (&cu_seqlens_q_decode, query_start_len[num_prefill .. num_prefill + num_decode + 1]) |*decode_len, query_start| { - decode_len.* = query_start - prefill_token_count; - } - - var seqused_k_prefill: [num_prefill]i32 = undefined; - @memcpy(&seqused_k_prefill, seq_lens[0..num_prefill]); - var seqused_k_decode: [num_decode]i32 = undefined; - @memcpy(&seqused_k_decode, seq_lens[num_prefill .. num_prefill + num_decode]); - - break :cuda_fa2 .{ .cuda_fa2 = .{ .mixed = .{ - .block_table_prefill = try .fromBytes(io, platform, params.mixed.block_table_prefill.shape(), .replicated, @ptrCast(&block_table_prefill)), - .cu_seqlens_q_prefill = try .fromBytes(io, platform, params.mixed.cu_seqlens_q_prefill.shape(), .replicated, @ptrCast(&cu_seqlens_q_prefill)), - .seqused_k_prefill = try .fromBytes(io, platform, params.mixed.seqused_k_prefill.shape(), .replicated, @ptrCast(&seqused_k_prefill)), - - .block_table_decode = try .fromBytes(io, platform, params.mixed.block_table_decode.shape(), .replicated, @ptrCast(&block_table_decode)), - .cu_seqlens_q_decode = try .fromBytes(io, platform, params.mixed.cu_seqlens_q_decode.shape(), .replicated, @ptrCast(&cu_seqlens_q_decode)), - .seqused_k_decode = try .fromBytes(io, platform, params.mixed.seqused_k_decode.shape(), .replicated, @ptrCast(&seqused_k_decode)), - - .metadata = .{ .decode_offset = try .scalar(io, platform, prefill_token_count, .i32) }, - } } }; - }, - .triton => triton_parameters_d, - inline .metal, .mosaic_tpu, .stablehlo => |_, t| @unionInit(zml.Bufferized(Parameters), @tagName(t), .{ - .block_table = triton_parameters_d.triton.block_table, - .seq_lens = triton_parameters_d.triton.seq_lens, - .query_start_len = triton_parameters_d.triton.query_start_len, - }), - }; - // cu_fa2 creates new buffers while other reuse triton buffers. - defer if (backend == .cuda_fa2) zml.Buffer.deinitAll(Parameters, ¶meters_d); - - var output_d = try zml.testing.autoCall(allocator, io, &exe, pagedAttention, .{ parameters_d, q, new_k, new_v, kv_cache_d }); - defer output_d.deinit(); - results_per_backend.set(backend, try output_d.toSliceAlloc(allocator, io)); - } - - const reference_backend: Backend = .triton; - const reference = results_per_backend.get(reference_backend) orelse return error.SkipZigTest; - - var num_failed: u32 = 0; - for (std.enums.values(Backend)) |backend| { - if (backend == reference_backend) continue; - const output_h = results_per_backend.get(backend) orelse continue; - - const tolerance: zml.testing.CompareOpts = .{ - .absolute_tolerance = 1e-2, - .relative_tolerance = 1e-2, - .epsilon_relative = 1e-6, - }; - - zml.testing.expectClose(io, reference, output_h, tolerance) catch |err| switch (err) { - error.TestUnexpectedResult => { - num_failed += 1; - std.log.err("test pagedAttention failed on backend {0t}\n--> reference ({1t}): {2d:32.3}\n--> pagedAttention({0t}): {3d:32.3}", .{ - backend, - reference_backend, - reference.subSlice(1, 0, 1).squeeze(1).subSlice(1, 0, 1).squeeze(1), - output_h.subSlice(1, 0, 1).squeeze(1).subSlice(1, 0, 1).squeeze(1), - }); - }, - else => |e| return e, - }; - } - - if (num_failed > 0) return error.TestUnexpectedResult; } fn stablehlo_pagedAttention( diff --git a/zml/slice.zig b/zml/slice.zig index e7d239e5..f096608b 100644 --- a/zml/slice.zig +++ b/zml/slice.zig @@ -286,7 +286,6 @@ pub const Slice = struct { // The formatter consumes this physical tail using `stride`, so this // rank-1 path can read raw items without requiring contiguity. - std.log.warn("Printing {} elems from offset {} out of {} total elems @ {}", .{ n, slice.offset_bytes, needed_len, stride }); const raw_values: []const T = @ptrCast(@alignCast(slice.bytes[slice.offset_bytes..])); const values = raw_values[0..needed_len]; From 59dfe4216b4a7a55b528f9bd3a38b521ad4237c1 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 4 Sep 2026 10:24:26 +0200 Subject: [PATCH 656/881] zml.Buffer.freeDeviceBufferMemoryButKeepHostMetadataMemory (#760) --- platforms/platforms.zig | 49 ++++++++++++++++++++++------------------- zml/buffer.zig | 14 ++++++++++++ zml/exe.zig | 2 +- zml/io.zig | 43 ++++++++++++++++++++++++++---------- zml/mem.zig | 8 ++++++- zml/platform.zig | 15 ++++++------- 6 files changed, 86 insertions(+), 45 deletions(-) diff --git a/platforms/platforms.zig b/platforms/platforms.zig index f618bd97..38093ce7 100644 --- a/platforms/platforms.zig +++ b/platforms/platforms.zig @@ -9,6 +9,8 @@ const rocm = @import("platforms/rocm"); const oneapi = @import("platforms/oneapi"); const tpu = @import("platforms/tpu"); +const platforms = @This(); + pub const Platform = enum { cpu, cuda, @@ -17,28 +19,29 @@ pub const Platform = enum { neuron, oneapi, metal, + + pub fn selectFirstAcceleratorEnabled() error{Unavailable}!Platform { + inline for (@typeInfo(Platform).@"enum".fields) |field| { + const target: Platform = @enumFromInt(field.value); + if (comptime (target != .cpu and target.isEnabled())) return target; + } + + if (!comptime Platform.cpu.isEnabled()) @compileError("No platform was enabled, use --@zml//platforms:cuda=true to eg enable Cuda"); + return .cpu; + } + + pub fn isEnabled(target: Platform) bool { + return switch (target) { + inline else => |tag| @field(platforms, @tagName(tag)).isEnabled(), + }; + } + + pub fn load(target: Platform, allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { + return switch (target) { + inline else => |tag| @field(platforms, @tagName(tag)).load(allocator, io), + }; + } }; -pub fn load(allocator: std.mem.Allocator, io: std.Io, tag: Platform) !*const pjrt.Api { - return switch (tag) { - .cpu => try cpu.load(allocator, io), - .cuda => try cuda.load(allocator, io), - .rocm => try rocm.load(allocator, io), - .tpu => try tpu.load(allocator, io), - .neuron => try neuron.load(allocator, io), - .oneapi => try oneapi.load(allocator, io), - .metal => try metal.load(allocator, io), - }; -} - -pub fn isEnabled(tag: Platform) bool { - return switch (tag) { - .cpu => cpu.isEnabled(), - .cuda => cuda.isEnabled(), - .rocm => rocm.isEnabled(), - .tpu => tpu.isEnabled(), - .neuron => neuron.isEnabled(), - .oneapi => oneapi.isEnabled(), - .metal => metal.isEnabled(), - }; -} +pub const load = Platform.load; +pub const isEnabled = Platform.isEnabled; diff --git a/zml/buffer.zig b/zml/buffer.zig index 25c319af..f1c2ee3f 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -75,8 +75,10 @@ pub const Buffer = struct { for (self._shards.constSlice()) |buffer| { buffer.deinit(self._platform.pjrt_api); } + self._shards = .empty; } + /// Given a flat struct (static size, no slices) containing `zml.Buffer`, `deinit` each one of them. pub fn deinitAll(T: type, buffers: *mem.Bufferized(T)) void { meta.visitFlatStruct(struct { fn deinit(_: void, x: *Buffer) void { @@ -85,6 +87,17 @@ pub const Buffer = struct { }.deinit, {}, buffers); } + /// Given an arbitrary struct `deinit` all `zml.Buffer` containing. + /// If the struct contains slices of `zml.Buffer` the memory of the slices will NOT be freed, + /// This only impacts device memory. + pub fn freeDeviceMemoryButKeepHostMetadataMemory(T: type, buffers: *mem.Bufferized(T)) void { + meta.visit(struct { + fn deinit(_: void, x: *Buffer) void { + x.deinit(); + } + }.deinit, {}, buffers); + } + /// This Buffer shape. pub fn shape(self: Buffer) Shape { return self._shape; @@ -207,6 +220,7 @@ pub const Buffer = struct { sharding: Sharding, opts: UnitializedOptions, ) !Buffer { + std.log.warn("uninitialized {f}", .{shape_}); const sh = shape_.packedShape(); var res: Buffer = .{ ._platform = platform, diff --git a/zml/exe.zig b/zml/exe.zig index 9efe7674..890756b3 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -114,7 +114,7 @@ pub const Exe = struct { baked_count: usize = 0, shardings: []const Sharding, - pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, shardings: []const Sharding, num_devices: usize) !Arguments { + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, shardings: []const Sharding, num_devices: usize) error{OutOfMemory}!Arguments { const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); errdefer flat_buffers.deinit(allocator); diff --git a/zml/io.zig b/zml/io.zig index 6cbf136e..ee020b7e 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -303,7 +303,7 @@ pub const Loader = struct { dma_chunk_size: usize, }; - pub fn init(allocator: std.mem.Allocator, platform: *const Platform, opts: Opts) !Loader { + pub fn init(allocator: std.mem.Allocator, platform: *const Platform, opts: Opts) error{OutOfMemory}!Loader { const pool_count = platform.devices.len; const dma_allocators = try allocator.alloc(mem.DmaAllocator, pool_count); errdefer allocator.free(dma_allocators); @@ -406,6 +406,10 @@ pub const Loader = struct { log.err("Transformed tensor {} {f} was not delivered by loadExecute before load; sources: {s}", .{ tensor.id, tensor.shape(), names.written() }); } + pub fn loadBuffer(loader: *Loader, io: std.Io, buffer: *Buffer, tensor: Tensor, store: *const TensorStore, shardings: []const Sharding, opts: LoadOpts) void { + loader.group.async(io, defaultCallback, .{ loader, io, &tensor, buffer, store, shardings, opts }); + } + fn defaultCallback(self: *Loader, io: std.Io, tensor: *const Tensor, buffer: *Buffer, store: *const TensorStore, shardings: []const Sharding, opts: LoadOpts) void { const sources = store.getSourcesById(tensor.id) orelse { std.log.warn("Failed to get sources for tensor with id: {}", .{tensor.id}); @@ -428,7 +432,15 @@ pub const Loader = struct { loaded.* = true; } - fn loadSingleInner(self: *Loader, io: std.Io, source: *safetensors.Tensor, shape: Shape, buffer: *Buffer, shardings: []const Sharding, opts: LoadOpts) !void { + fn loadSingleInner( + self: *Loader, + io: std.Io, + source: *safetensors.Tensor, + shape: Shape, + buffer: *Buffer, + shardings: []const Sharding, + opts: LoadOpts, + ) !void { var reader = try source.reader(io, &.{}, .{}); defer reader.deinit(); @@ -478,28 +490,31 @@ pub const Loader = struct { shardings: []const Sharding, exe: *const Exe, opts: LoadOpts, - ) !void { + ) error{ NotFound, OutOfMemory, LoadFailed, Canceled }!void { const sources = (store.getSourcesById(tensor.id) orelse return error.NotFound).tensors; const buffers = try arena.alloc(Buffer, sources.len); const loaded = try arena.alloc(bool, sources.len); @memset(loaded, false); defer for (buffers, loaded) |*b, l| if (l) b.deinit(); - var node = if (opts.progress) |progress| b: { + var node = if (opts.progress) |progress| p: { var writer = std.Io.Writer.Allocating.init(arena); - try writer.writer.writeAll("Running executable on "); - for (sources, 0..) |source, i| { - try writer.writer.print("{s}{s}", .{ if (i != 0) ", " else "", source.name }); + print: { + writer.writer.writeAll("Running executable on ") catch break :print; + for (sources, 0..) |source, i| { + writer.writer.print("{s}{s}", .{ if (i != 0) ", " else "", source.name }) catch break :print; + } } - - break :b progress.start(writer.written(), 1); + break :p progress.start(writer.written(), 1); } else null; defer if (node) |*n| n.end(); for (sources, 0..) |source, i| { self.group.async(io, loadSingle, .{ self, io, source, source.shape, &buffers[i], &loaded[i], shardings, .{} }); } - try self.group.await(io); + self.group.await(io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + }; if (std.mem.findScalar(bool, loaded, false)) |_| { return error.LoadFailed; @@ -608,8 +623,12 @@ pub const MemoryWriter = union(enum) { buffer: *Buffer, ) !MemoryWriter { return switch (platform.target) { - .cuda, .oneapi => .{ .direct = try DirectMemoryWriter.init(allocator, io, platform, pools, dma_allocators, dma_chunk_size, shape, sharding, buffer) }, - .rocm, .tpu, .neuron, .cpu, .metal => .{ .buffered = try BufferedMemoryWriter.init(allocator, io, platform, shape, sharding, buffer) }, + .cuda, .oneapi => .{ + .direct = try .init(allocator, io, platform, pools, dma_allocators, dma_chunk_size, shape, sharding, buffer), + }, + .rocm, .tpu, .neuron, .cpu, .metal => .{ + .buffered = try .init(allocator, io, platform, shape, sharding, buffer), + }, }; } diff --git a/zml/mem.zig b/zml/mem.zig index 5048d0bc..a0f853a0 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -329,7 +329,13 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu const ModelBufferized = Bufferized(Model); if (ModelBufferized == Buffer) { - bufferized_._shards = .empty; + bufferized_.* = .{ + // I'm not sure I like that. I'd rather set all fields but _shards than leaving most of the m undefined. + ._shape = model._shape, + ._shards = .empty, + ._sharding = undefined, + ._platform = undefined, + }; return; } diff --git a/zml/platform.zig b/zml/platform.zig index 5acd696b..23432742 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -3,8 +3,7 @@ const builtin = @import("builtin"); const c = @import("c"); const pjrt = @import("pjrt"); -const platforms = @import("platforms"); -pub const Target = platforms.Platform; +pub const Target = @import("platforms").Platform; const stdx = @import("stdx"); const attention = @import("attention.zig"); @@ -51,10 +50,10 @@ fn validateDeviceCount(target: Target, num_devices: usize) !void { fn loadOrGetApi(allocator: std.mem.Allocator, io: std.Io, target: Target) !*const pjrt.Api { return switch (target) { - inline else => |tag| api_map.get(tag) orelse b: { + inline else => |tgt| api_map.get(tgt) orelse b: { disableXlaLogs(); - const api = try platforms.load(allocator, io, tag); - api_map.set(tag, api); + const api = try tgt.load(allocator, io); + api_map.set(tgt, api); break :b api; }, }; @@ -287,7 +286,7 @@ pub const Platform = struct { replicated_sharding: zml.Sharding, shardings: std.StringArrayHashMapUnmanaged(zml.Sharding), - pub const MAX_NUM_DEVICES: u16 = if (platforms.isEnabled(.tpu)) 64 else 32; + pub const MAX_NUM_DEVICES: u16 = if (Target.tpu.isEnabled()) 64 else 32; pub fn init(allocator: std.mem.Allocator, io: std.Io, target: Target, options: CreateOptions) !*Platform { const api = try loadOrGetApi(allocator, io, target); @@ -528,7 +527,7 @@ pub const Platform = struct { pub fn deinit(self: *Platform, allocator: std.mem.Allocator, io: std.Io) void { _ = io; _ = allocator; - if (comptime platforms.isEnabled(.cuda)) { + if (comptime Target.cuda.isEnabled()) { self.state.deinit(); } self.physical_mesh.deinit(self.arena.allocator()); @@ -675,7 +674,7 @@ pub const Platform = struct { // but given it's compiled out (except for TPU), I'm not gonna care for now. return switch (platform.target) { .tpu => { - if (comptime !platforms.isEnabled(.tpu)) unreachable; + if (comptime !Target.tpu.isEnabled()) unreachable; const element_type = pjrtx.bufferTypeFromDtype(dtype); const default = platform.pjrt_client.defaultMemoryLayout(platform.pjrt_api, element_type, dims) catch @panic("Failed to get default memory layout"); return default.toMemoryLayout(); From 6b4b790a00046192c3fb45ef6cca8f8e37a89e86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20Gauti=C3=A9?= <51361266+tgautie@users.noreply.github.com> Date: Fri, 4 Sep 2026 12:45:57 +0200 Subject: [PATCH 657/881] nn/gated_delta_net: bound GDN recurrence (#769) Fixing an issue raised in https://github.com/zml/zml/pull/754, to bound the GDN recurrence in the Qwen3.5 example model to the number of active tokens. We also make qwen3.5 and qwen3.5-moe slightly more consistent. --- examples/llm/models/qwen3_5/inference.zig | 5 +- examples/llm/models/qwen3_5/model.zig | 30 +++++++--- examples/llm/models/qwen3_5/session.zig | 6 ++ examples/llm/models/qwen3_5_moe/inference.zig | 14 ++--- examples/llm/models/qwen3_5_moe/model.zig | 59 ++++++++++--------- examples/llm/models/qwen3_5_moe/session.zig | 14 +++-- zml/nn.zig | 54 ++++++++++++++++- 7 files changed, 126 insertions(+), 56 deletions(-) diff --git a/examples/llm/models/qwen3_5/inference.zig b/examples/llm/models/qwen3_5/inference.zig index 2375081a..83825352 100644 --- a/examples/llm/models/qwen3_5/inference.zig +++ b/examples/llm/models/qwen3_5/inference.zig @@ -37,6 +37,7 @@ pub const Args = struct { io: std.Io, tokens_buf: *zml.Buffer, token_index_buf: *zml.Buffer, + active_length_buf: *zml.Buffer, kv_cache_buffers: *zml.Bufferized(model.KvCache), rng_buffers: *zml.Bufferized(zml.Tensor.Rng), }; @@ -188,7 +189,7 @@ pub fn run(runner: *KernelRunner, args: Args, layer_index_buffers: []const zml.B layer.run(args.io, .{ .inputs = .{ .hidden = hidden_buffer, - .token_index = args.token_index_buf.*, + .active_length = args.active_length_buf.*, .cache = layer_cache, }, .outputs = .{ .hidden = &hidden_buffer, .cache = &layer_cache }, @@ -276,7 +277,7 @@ fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *c return zml.FnExe(model.TransformerLayer.forwardLinearAttn).compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("qwen3_5", "linear_attention_layer") }, .{.{ .layer = mdl.text_model.layers[layer_index], .hidden = hiddenTensor(mdl, seqlen), - .token_index = parameters.token_index, + .active_length = zml.Tensor.init(.{}, .u32), .cache = .{ .conv_state = parameters.kv_cache.gated_delta_net.conv_state, .recurrent_state = parameters.kv_cache.gated_delta_net.recurrent_state, diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 8c0d40f0..22fb4d92 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -215,11 +215,12 @@ pub const Model = struct { self: Model, tokens_: zml.Tensor, token_index: zml.Tensor, + active_length: zml.Tensor, kv_cache: KvCache, rng: zml.Tensor.Rng, ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { const tokens = tokens_.withPartialTags(.{.s}); - const text_model_output, const updated_kv_cache = self.text_model.forward(tokens, token_index, kv_cache); + const text_model_output, const updated_kv_cache = self.text_model.forward(tokens, token_index, active_length, kv_cache); const result = Sampler.sampleTokens(.{ .sampler = self.sampler(), .hidden = text_model_output, @@ -330,6 +331,7 @@ pub const TextModel = struct { self: TextModel, tokens: zml.Tensor, token_index: zml.Tensor, + active_length: zml.Tensor, kv_cache: KvCache, ) struct { zml.Tensor, KvCache } { var hidden_states = EmbedTokens.forward(.{ @@ -339,7 +341,7 @@ pub const TextModel = struct { var updated_kv_cache = kv_cache; for (self.layers, 0..) |layer, i| { - hidden_states, updated_kv_cache = layer.forward(hidden_states, token_index, updated_kv_cache.atLayer(i)); + hidden_states, updated_kv_cache = layer.forward(hidden_states, token_index, active_length, updated_kv_cache.atLayer(i)); } return .{ hidden_states, updated_kv_cache.reuseBuffer(kv_cache) }; @@ -372,7 +374,7 @@ pub const TransformerLayer = struct { pub const LinearAttnInput = struct { layer: TransformerLayer, hidden: zml.Tensor, - token_index: zml.Tensor, + active_length: zml.Tensor, cache: KvCache.GatedDeltaNetCache, }; @@ -409,6 +411,7 @@ pub const TransformerLayer = struct { self: TransformerLayer, x0: zml.Tensor, token_index: zml.Tensor, + active_length: zml.Tensor, kv_cache: KvCache.LayerView, ) struct { zml.Tensor, KvCache } { const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); @@ -431,7 +434,7 @@ pub const TransformerLayer = struct { .linear_attn => |cache| cache, .self_attn => unreachable, }; - const result = linear_attn.forward(normalized_x0, cache); + const result = linear_attn.forward(normalized_x0, cache, active_length); attention_output = result[0]; updated_kv_cache.gated_delta_net = result[1].reuseBuffer(updated_kv_cache.gated_delta_net); }, @@ -470,7 +473,6 @@ pub const TransformerLayer = struct { pub fn forwardLinearAttn(input: LinearAttnInput) LinearAttnOutput { const self = input.layer; const x0 = input.hidden; - _ = input.token_index; const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); @@ -478,7 +480,7 @@ pub const TransformerLayer = struct { .linear_attention => |linear_attn| linear_attn, .full_attention => unreachable, }; - const attention_output, const updated_kv_cache = linear_attn.forward(normalized_x0, input.cache); + const attention_output, const updated_kv_cache = linear_attn.forward(normalized_x0, input.cache, input.active_length); const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); @@ -806,6 +808,7 @@ pub const GatedDeltaNet = struct { g: zml.Tensor, beta: zml.Tensor, initial_state: ?zml.Tensor, + active_length: zml.Tensor, ) struct { zml.Tensor, zml.Tensor } { const scale: f32 = 1.0 / @sqrt(@as(f32, @floatFromInt(query.dim(.khd)))); const query_norm = zml.nn.normalizeL2(query.rename(.{ .kh = .vh }), 1e-6); @@ -839,6 +842,7 @@ pub const GatedDeltaNet = struct { alpha_f32, beta_f32, .{ .s = initial_recurrent_state }, + active_length, ); return .{ @@ -857,7 +861,13 @@ pub const GatedDeltaNet = struct { return zml.Tensor.concatenate(&.{ padding, tail }, .s); } - pub fn forward(self: GatedDeltaNet, x: zml.Tensor, cache: KvCache.GatedDeltaNetCache) struct { zml.Tensor, KvCache.GatedDeltaNetCache } { + fn buildUpdatedConvStateFromPrefix(input: zml.Tensor, left_pad: i64, active_length: zml.Tensor) zml.Tensor { + const padding = zml.Tensor.zeroes(input.shape().setDim(.s, left_pad)); + const padded = zml.Tensor.concatenate(&.{ padding, input }, .s); + return padded.slice(.s, .dyn(active_length.convert(.i64), left_pad)); + } + + pub fn forward(self: GatedDeltaNet, x: zml.Tensor, cache: KvCache.GatedDeltaNetCache, active_length: zml.Tensor) struct { zml.Tensor, KvCache.GatedDeltaNetCache } { const key_dim = self.num_k_heads * self.head_k_dim; const value_dim = self.num_v_heads * self.head_v_dim; const conv_dim = 2 * key_dim + value_dim; @@ -936,6 +946,7 @@ pub const GatedDeltaNet = struct { if (use_cached_state) break :b cache.recurrentState(); break :b null; }, + active_length, ); const core_attn_out_normed = self.norm @@ -950,7 +961,7 @@ pub const GatedDeltaNet = struct { .rename(.{ .dout = .d }) .withPartitioning(.{ .d = .replicated }); const updated_cache = cache.update( - buildUpdatedConvState(conv_input, left_pad), + if (use_cached_state) buildUpdatedConvState(conv_input, left_pad) else buildUpdatedConvStateFromPrefix(projected_qkv, left_pad, active_length), last_recurrent_state, ); return .{ output, updated_cache }; @@ -993,9 +1004,10 @@ pub const RmsNormGated = struct { pub fn forward(self: RmsNormGated, x: zml.Tensor, gate: zml.Tensor) zml.Tensor { const x_f32 = x.convert(.f32); const gate_f32 = gate.convert(.f32); + const weight_f32 = self.weight.convert(.f32); const normalized = zml.nn.rmsNorm(x_f32, .d, self.eps); - const output = normalized.mul(self.weight.broad(x.shape())); + const output = normalized.mul(weight_f32.broad(x.shape())); const gated_output = output.mul(gate_f32.silu()); return gated_output.convert(x.dtype()); diff --git a/examples/llm/models/qwen3_5/session.zig b/examples/llm/models/qwen3_5/session.zig index 33ceeac0..a49940cc 100644 --- a/examples/llm/models/qwen3_5/session.zig +++ b/examples/llm/models/qwen3_5/session.zig @@ -118,11 +118,14 @@ pub const Session = struct { var prefill_token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 0), .u32); defer prefill_token_index_buffer.deinit(); + var active_length_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.len)), .u32); + defer active_length_buffer.deinit(); inference.run(&self.prefill, .{ .io = self.io, .tokens_buf = &prefill_tokens_buffer, .token_index_buf = &prefill_token_index_buffer, + .active_length_buf = &active_length_buffer, .kv_cache_buffers = &self.kv_cache_buffers, .rng_buffers = &self.rng_buffers, }, self.layer_index_buffers); @@ -145,6 +148,8 @@ pub const Session = struct { var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.items.len)), .u32); defer token_index_buffer.deinit(); + var active_length_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 1), .u32); + defer active_length_buffer.deinit(); generation: while (true) { const token_id = self.generated_token_slice.items(u32)[0]; @@ -167,6 +172,7 @@ pub const Session = struct { .io = self.io, .tokens_buf = ¤t_token_buffer, .token_index_buf = &token_index_buffer, + .active_length_buf = &active_length_buffer, .kv_cache_buffers = &self.kv_cache_buffers, .rng_buffers = &self.rng_buffers, }, self.layer_index_buffers); diff --git a/examples/llm/models/qwen3_5_moe/inference.zig b/examples/llm/models/qwen3_5_moe/inference.zig index 3a3b9da2..a9ff1de0 100644 --- a/examples/llm/models/qwen3_5_moe/inference.zig +++ b/examples/llm/models/qwen3_5_moe/inference.zig @@ -42,8 +42,8 @@ pub const LayerIndexBuffer = union(enum) { pub const RunArgs = struct { io: std.Io, tokens_buffer: *zml.Buffer, - full_attention_token_index_buffer: *zml.Buffer, - linear_attention_token_index_buffer: *zml.Buffer, + token_index_buffer: *zml.Buffer, + active_length_buffer: *zml.Buffer, kv_cache_buffers: *zml.Bufferized(model.KvCache), moe_metadata_buffers: zml.Bufferized(zml.moe.Metadata), rng_buffers: *zml.Bufferized(zml.Tensor.Rng), @@ -184,7 +184,7 @@ pub fn run(runner: *KernelRunner, args: RunArgs) void { layer.run(args.io, .{ .inputs = .{ .hidden = hidden_buffer, - .token_index = args.full_attention_token_index_buffer.*, + .token_index = args.token_index_buffer.*, .cache = layer_cache, .moe_metadata = args.moe_metadata_buffers, }, @@ -206,7 +206,7 @@ pub fn run(runner: *KernelRunner, args: RunArgs) void { layer.run(args.io, .{ .inputs = .{ .hidden = hidden_buffer, - .token_index = args.linear_attention_token_index_buffer.*, + .active_length = args.active_length_buffer.*, .cache = layer_cache, .moe_metadata = args.moe_metadata_buffers, }, @@ -222,12 +222,12 @@ pub fn run(runner: *KernelRunner, args: RunArgs) void { .inputs = .{ .hidden = hidden_buffer, .rng = args.rng_buffers.*, - .token_index = args.full_attention_token_index_buffer.*, + .token_index = args.token_index_buffer.*, }, .outputs = .{ .tokens = args.tokens_buffer, .rng = args.rng_buffers, - .token_index = args.full_attention_token_index_buffer, + .token_index = args.token_index_buffer, }, }); } @@ -299,7 +299,7 @@ fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *c return compileExe(allocator, io, platform, model.TransformerLayer.forwardLinearAttn, .{.{ .layer = mdl.text_model.layers[layer_index], .hidden = hiddenTensor(mdl, seqlen), - .token_index = zml.Tensor.init(.{}, .u32), + .active_length = zml.Tensor.init(.{}, .u32), .cache = .{ .conv_state = parameters.kv_cache.gated_delta_net.conv_state, .recurrent_state = parameters.kv_cache.gated_delta_net.recurrent_state, diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 5ea280a3..45c830f4 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -230,13 +230,14 @@ pub const Model = struct { self: Model, tokens_: zml.Tensor, token_index: zml.Tensor, + active_length: zml.Tensor, kv_cache: KvCache, rng: zml.Tensor.Rng, moe_metadata: zml.moe.Metadata, moe_parameters: zml.moe.Parameters, ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { const tokens = tokens_.withPartialTags(.{.s}); - const new_tokens, const updated_kv_cache, const new_rng = self.text_model.forward(tokens, token_index, kv_cache, self.config, rng, moe_metadata, moe_parameters); + const new_tokens, const updated_kv_cache, const new_rng = self.text_model.forward(tokens, token_index, active_length, kv_cache, self.config, rng, moe_metadata, moe_parameters); return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; } }; @@ -351,6 +352,7 @@ pub const TextModel = struct { self: TextModel, tokens: zml.Tensor, token_index: zml.Tensor, + active_length: zml.Tensor, kv_cache: KvCache, config: Config, rng: zml.Tensor.Rng, @@ -364,7 +366,7 @@ pub const TextModel = struct { var updated_kv_cache = kv_cache; for (self.layers, 0..) |layer, i| { - hidden_states, updated_kv_cache = layer.forward(hidden_states, token_index, updated_kv_cache.atLayer(i), config, moe_metadata, moe_parameters); + hidden_states, updated_kv_cache = layer.forward(hidden_states, token_index, active_length, updated_kv_cache.atLayer(i), config, moe_metadata, moe_parameters); } const result = Sampler.sampleTokens(.{ @@ -406,7 +408,7 @@ pub const TransformerLayer = struct { pub const LinearAttnInput = struct { layer: TransformerLayer, hidden: zml.Tensor, - token_index: zml.Tensor, + active_length: zml.Tensor, cache: KvCache.GatedDeltaNetCache, config: Config, moe_metadata: zml.moe.Metadata, @@ -480,7 +482,7 @@ pub const TransformerLayer = struct { .linear_attn => |linear_attn| linear_attn, .self_attn => unreachable, }; - const attention_output, const updated_kv_cache = linear_attn.forward(normalized_x0, input.cache, input.token_index); + const attention_output, const updated_kv_cache = linear_attn.forward(normalized_x0, input.cache, input.active_length); const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); @@ -497,6 +499,7 @@ pub const TransformerLayer = struct { self: TransformerLayer, x0: zml.Tensor, token_index: zml.Tensor, + active_length: zml.Tensor, kv_cache: KvCache.LayerView, config: Config, moe_metadata: zml.moe.Metadata, @@ -515,7 +518,7 @@ pub const TransformerLayer = struct { updated_kv_cache.self_attn = result[1]; }, .linear_attn => |*linear_attn| { - const result = linear_attn.forward(normalized_x0, kv_cache.cache.linear_attn, token_index); + const result = linear_attn.forward(normalized_x0, kv_cache.cache.linear_attn, active_length); attention_output = result[0]; updated_kv_cache.gated_delta_net = result[1]; }, @@ -927,7 +930,15 @@ pub const GatedDeltaNet = struct { zml.Buffer.deinitAll(GatedDeltaNet, self); } - fn recurrentGatedDeltaRule(query: zml.Tensor, key: zml.Tensor, value: zml.Tensor, g: zml.Tensor, beta: zml.Tensor, initial_state: ?zml.Tensor) struct { zml.Tensor, zml.Tensor } { + fn recurrentGatedDeltaRule( + query: zml.Tensor, + key: zml.Tensor, + value: zml.Tensor, + g: zml.Tensor, + beta: zml.Tensor, + initial_state: ?zml.Tensor, + active_length: zml.Tensor, + ) struct { zml.Tensor, zml.Tensor } { const scale: f32 = 1.0 / @sqrt(@as(f32, @floatFromInt(query.dim(.khd)))); const query_norm = zml.nn.normalizeL2(query.rename(.{ .kh = .vh }), 1e-6); const key_norm = zml.nn.normalizeL2(key.rename(.{ .kh = .vh }), 1e-6); @@ -957,6 +968,7 @@ pub const GatedDeltaNet = struct { alpha_f32, beta_f32, .{ .s = initial_recurrent_state }, + active_length, ); return .{ @@ -975,16 +987,13 @@ pub const GatedDeltaNet = struct { return zml.Tensor.concatenate(&.{ padding, tail }, .s); } - fn buildUpdatedConvStateFromPrefix(input: zml.Tensor, left_pad: i64, valid_len: zml.Tensor) zml.Tensor { - const start = valid_len.convert(.i64).addConstant(-left_pad).maximum(zml.Tensor.scalar(0, .i64)); - return input.slice(.s, .dyn(start, left_pad)); + fn buildUpdatedConvStateFromPrefix(input: zml.Tensor, left_pad: i64, active_length: zml.Tensor) zml.Tensor { + const padding = zml.Tensor.zeroes(input.shape().setDim(.s, left_pad)); + const padded = zml.Tensor.concatenate(&.{ padding, input }, .s); + return padded.slice(.s, .dyn(active_length.convert(.i64), left_pad)); } - fn setPaddingToZero(input: zml.Tensor, valid_mask: zml.Tensor) zml.Tensor { - return valid_mask.select(input, zml.Tensor.zeroes(input.shape())); - } - - pub fn forward(self: GatedDeltaNet, x: zml.Tensor, cache: KvCache.GatedDeltaNetCache, token_index: zml.Tensor) struct { zml.Tensor, KvCache.GatedDeltaNetCache } { + pub fn forward(self: GatedDeltaNet, x: zml.Tensor, cache: KvCache.GatedDeltaNetCache, active_length: zml.Tensor) struct { zml.Tensor, KvCache.GatedDeltaNetCache } { const key_dim = self.num_k_heads * self.head_k_dim; const value_dim = self.num_v_heads * self.head_v_dim; const conv_dim = 2 * key_dim + value_dim; @@ -1045,17 +1054,7 @@ pub const GatedDeltaNet = struct { const beta = b.sigmoid(); const aLog_type = self.aLog.dtype(); - var g = self.aLog.broad(a.shape()).exp().mul(softplus(a.convert(aLog_type).add(self.dt_bias.convert(aLog_type).broad(a.shape())))).negate(); - var recurrent_value = value; - var recurrent_beta = beta; - if (!use_cached_state) { - const valid_mask: zml.Tensor = zml.Tensor.arange(.{ .end = x.dim(.s) }, .i64) - .withTags(.{.s}) - .cmp(.LT, token_index.convert(.i64)); - recurrent_value = setPaddingToZero(recurrent_value, valid_mask.broad(recurrent_value.shape())); - recurrent_beta = setPaddingToZero(recurrent_beta, valid_mask.broad(recurrent_beta.shape())); - g = setPaddingToZero(g, valid_mask.broad(g.shape())); - } + const g = self.aLog.broad(a.shape()).exp().mul(softplus(a.convert(aLog_type).add(self.dt_bias.convert(aLog_type).broad(a.shape())))).negate(); const query_for_rule = if (self.qk_head_repetition == 1) query else query.stutter1d(query.axis(.kh), self.qk_head_repetition); const key_for_rule = if (self.qk_head_repetition == 1) key else key.stutter1d(key.axis(.kh), self.qk_head_repetition); @@ -1063,10 +1062,11 @@ pub const GatedDeltaNet = struct { const core_attn_out, const last_recurrent_state = recurrentGatedDeltaRule( query_for_rule, key_for_rule, - recurrent_value, + value, g, - recurrent_beta, + beta, if (use_cached_state) cache.recurrentState() else null, + active_length, ); const core_attn_out_normed = self.norm @@ -1080,7 +1080,7 @@ pub const GatedDeltaNet = struct { const output = self.out_proj.forward(core_attn_out_normed.merge(.{ .d = .{ .vh, .vhd } })) .rename(.{ .dout = .d }).withPartitioning(.{ .d = .replicated }); const updated_cache = cache.update( - if (use_cached_state) buildUpdatedConvState(conv_input, left_pad) else buildUpdatedConvStateFromPrefix(projected_qkv, left_pad, token_index), + if (use_cached_state) buildUpdatedConvState(conv_input, left_pad) else buildUpdatedConvStateFromPrefix(projected_qkv, left_pad, active_length), last_recurrent_state, ); return .{ output, updated_cache }; @@ -1123,9 +1123,10 @@ pub const RmsNormGated = struct { pub fn forward(self: RmsNormGated, x: zml.Tensor, gate: zml.Tensor) zml.Tensor { const x_f32 = x.convert(.f32); const gate_f32 = gate.convert(.f32); + const weight_f32 = self.weight.convert(.f32); const normalized = zml.nn.rmsNorm(x_f32, .d, self.eps); - const output = normalized.mul(self.weight.broad(x.shape())); + const output = normalized.mul(weight_f32.broad(x.shape())); const gated_output = output.mul(gate_f32.silu()); return gated_output.convert(x.dtype()); diff --git a/examples/llm/models/qwen3_5_moe/session.zig b/examples/llm/models/qwen3_5_moe/session.zig index 04ab1838..4e038531 100644 --- a/examples/llm/models/qwen3_5_moe/session.zig +++ b/examples/llm/models/qwen3_5_moe/session.zig @@ -137,14 +137,14 @@ pub const Session = struct { defer tokens_buffer.deinit(); var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 0), .u32); defer token_index_buffer.deinit(); - var valid_len_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.len)), .u32); - defer valid_len_buffer.deinit(); + var active_length_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.len)), .u32); + defer active_length_buffer.deinit(); inference.run(&self.prefill, .{ .io = self.io, .tokens_buffer = &tokens_buffer, - .full_attention_token_index_buffer = &token_index_buffer, - .linear_attention_token_index_buffer = &valid_len_buffer, + .token_index_buffer = &token_index_buffer, + .active_length_buffer = &active_length_buffer, .kv_cache_buffers = &self.kv_cache_buffers, .moe_metadata_buffers = self.prefill_moe_metadata_buffers, .rng_buffers = &self.rng_buffers, @@ -165,6 +165,8 @@ pub const Session = struct { defer current_token_buffer.deinit(); var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.items.len)), .u32); defer token_index_buffer.deinit(); + var active_length_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 1), .u32); + defer active_length_buffer.deinit(); generation: while (true) { const token_id = self.generated_token_slice.items(u32)[0]; @@ -182,8 +184,8 @@ pub const Session = struct { inference.run(&self.decode, .{ .io = self.io, .tokens_buffer = ¤t_token_buffer, - .full_attention_token_index_buffer = &token_index_buffer, - .linear_attention_token_index_buffer = &token_index_buffer, + .token_index_buffer = &token_index_buffer, + .active_length_buffer = &active_length_buffer, .kv_cache_buffers = &self.kv_cache_buffers, .moe_metadata_buffers = self.decode_moe_metadata_buffers, .rng_buffers = &self.rng_buffers, diff --git a/zml/nn.zig b/zml/nn.zig index 367fac30..00344a92 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -1558,6 +1558,8 @@ pub const GatedDeltaNet = struct { alphas: Tensor, /// Delta gate .{ .s, .h }. betas: Tensor, + /// Number of sequence positions to process, clamped to the input length. + active_length: Tensor, pub const State = struct { /// Per-head recurrent state with shape .{ .h, .v, .k }. @@ -1581,7 +1583,11 @@ pub const GatedDeltaNet = struct { }; pub fn cond(gdn: GatedDeltaNet, state: State) Tensor { - return state.step.cmp(.LT, .scalar(gdn.queries.dim(.s), .i32)); + const active_length = gdn.active_length + .convert(.i32) + .maximum(.scalar(0, .i32)) + .minimum(.scalar(gdn.queries.dim(.s), .i32)); + return state.step.cmp(.LT, active_length); } /// Single-step recurrent update for Gated Delta Net. @@ -1625,6 +1631,7 @@ pub const GatedDeltaNet = struct { stdx.debug.assert(gdn.values.shape().hasTags(.{ .s, .h, .v }), err_template ++ "v is missing tags {{.h, .v}}", err_args); stdx.debug.assert(gdn.alphas.shape().hasTags(.{ .s, .h }), err_template ++ "alphas is missing tag {{.h}}", err_args); stdx.debug.assert(gdn.betas.shape().hasTags(.{ .s, .h }), err_template ++ "betas is missing tag {{.h}}", err_args); + stdx.debug.assert(gdn.active_length.rank() == 0 and gdn.active_length.dtype().isInteger(), err_template ++ "active_length must be an integer scalar", err_args); _ = collectDims(.{.h}, &.{ state.s, gdn.queries, gdn.keys, gdn.values, gdn.alphas, gdn.betas }, .strict) catch { stdx.debug.panic(err_template ++ "head dimensions are inconsistent.", err_args); @@ -1644,6 +1651,7 @@ pub const GatedDeltaNet = struct { /// - `values`, `outputs`: .{ .s, .h, .v } /// - `alphas`, `betas`: .{ .s, .h } /// - `initial_state.s`: .{ .h, .v, .k } + /// - `active_length`: integer scalar pub fn forward( queries: Tensor, keys: Tensor, @@ -1651,6 +1659,7 @@ pub const GatedDeltaNet = struct { alphas: Tensor, betas: Tensor, initial_state: Input, + active_length: Tensor, ) Output { const gdn: GatedDeltaNet = .{ .queries = queries, @@ -1658,6 +1667,7 @@ pub const GatedDeltaNet = struct { .values = values, .alphas = alphas, .betas = betas, + .active_length = active_length, }; const state: GatedDeltaNet.State = .{ .step = .scalar(0, .i32), @@ -1683,12 +1693,13 @@ test "gated delta net" { const alphas: zml.Tensor = .init(.{ .s = 2, .h = 2 }, .f32); const betas: zml.Tensor = .init(.{ .s = 2, .h = 2 }, .f32); const initial_s: zml.Tensor = .init(.{ .h = 2, .v = 2, .k = 2 }, .f32); + const active_length: zml.Tensor = .init(.{}, .u32); var exe = try platform.compileFn( std.testing.allocator, std.testing.io, GatedDeltaNet.forward, - .{ queries, keys, values, alphas, betas, .{ .s = initial_s } }, + .{ queries, keys, values, alphas, betas, .{ .s = initial_s }, active_length }, .{}, ); defer exe.deinit(); @@ -1759,13 +1770,15 @@ test "gated delta net" { }), ); defer initial_s_buffer.deinit(); + var oversized_length_buffer = try zml.Buffer.scalar(std.testing.io, platform, @as(u32, 3), .u32); + defer oversized_length_buffer.deinit(); var result = try zml.testing.autoCall( std.testing.allocator, std.testing.io, &exe, GatedDeltaNet.forward, - .{ queries_buffer, keys_buffer, values_buffer, alphas_buffer, betas_buffer, .{ .s = initial_s_buffer } }, + .{ queries_buffer, keys_buffer, values_buffer, alphas_buffer, betas_buffer, .{ .s = initial_s_buffer }, oversized_length_buffer }, ); defer result.outputs.deinit(); defer result.state.s.deinit(); @@ -1793,6 +1806,41 @@ test "gated delta net" { .absolute_tolerance = 1e-4, .relative_tolerance = 1e-4, }); + + var prefix_length_buffer = try zml.Buffer.scalar(std.testing.io, platform, @as(u32, 1), .u32); + defer prefix_length_buffer.deinit(); + var prefix_result = try zml.testing.autoCall( + std.testing.allocator, + std.testing.io, + &exe, + GatedDeltaNet.forward, + .{ queries_buffer, keys_buffer, values_buffer, alphas_buffer, betas_buffer, .{ .s = initial_s_buffer }, prefix_length_buffer }, + ); + defer prefix_result.outputs.deinit(); + defer prefix_result.state.s.deinit(); + + const expected_prefix_outputs: zml.Slice = .init( + zml.Shape.init(.{ 2, 2, 2 }, .f32), + std.mem.sliceAsBytes(&[2][2][2]f32{ + .{ .{ 3.0, 0.0 }, .{ 1.125, 2.0 } }, + .{ .{ 0.0, 0.0 }, .{ 0.0, 0.0 } }, + }), + ); + const expected_prefix_final_s: zml.Slice = .init( + zml.Shape.init(.{ 2, 2, 2 }, .f32), + std.mem.sliceAsBytes(&[2][2][2]f32{ + .{ .{ 3.0, 5.0 }, .{ 0.0, 0.5 } }, + .{ .{ 0.5, 1.125 }, .{ 0.25, 2.0 } }, + }), + ); + try zml.testing.expectClose(std.testing.io, expected_prefix_outputs, prefix_result.outputs, .{ + .absolute_tolerance = 1e-4, + .relative_tolerance = 1e-4, + }); + try zml.testing.expectClose(std.testing.io, expected_prefix_final_s, prefix_result.state.s, .{ + .absolute_tolerance = 1e-4, + .relative_tolerance = 1e-4, + }); } /// Options controlling generation. The default values correspond to greedy decoding. From 567434be798db31ad888c586293eecedff10b526 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Fri, 4 Sep 2026 19:13:27 +0200 Subject: [PATCH 658/881] zml/nn: Linear.quantizationScheme (#772) --- zml/nn.zig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zml/nn.zig b/zml/nn.zig index 00344a92..8a300897 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -35,6 +35,10 @@ pub const Linear = struct { zml.Buffer.deinitAll(Linear, self); } + pub fn quantizationScheme(self: Linear) ?Quantization.Scheme { + return if (self.quantization) |q| q.scheme else null; + } + pub fn forward(self: Linear, x: Tensor) Tensor { const y = self.forwardWeight(x); return if (self.bias) |bias| y.add(bias.broad(y.shape())) else y; From 053577d82d702225499d4717b25af3c584608cce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20Gauti=C3=A9?= <51361266+tgautie@users.noreply.github.com> Date: Mon, 7 Sep 2026 09:37:39 +0200 Subject: [PATCH 659/881] zml/buffer: switch uninitialized buffer log to debug level (#771) --- zml/buffer.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index f1c2ee3f..a620efd7 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -220,7 +220,7 @@ pub const Buffer = struct { sharding: Sharding, opts: UnitializedOptions, ) !Buffer { - std.log.warn("uninitialized {f}", .{shape_}); + std.log.debug("uninitialized {f}", .{shape_}); const sh = shape_.packedShape(); var res: Buffer = .{ ._platform = platform, From 1fd10b6528855196f17e0a5a8e4f180457745960 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Mon, 7 Sep 2026 17:02:57 +0200 Subject: [PATCH 660/881] platforms/metal: update plugin (#773) improve gated delta net and rebase on latest XLA --- platforms/metal/metal.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 525eac59..f1af18c0 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libpjrt_metal", build_file = "libpjrt_metal.BUILD.bazel", - sha256 = "f55356328d9d814c5a961b6c38741cff2f98b4043846a7ac3865e2a23549b767", - url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-06-30T00-16-30Z/pjrt-metal_macos-arm64.tar.zst", + sha256 = "45667d5726844c093b319568d1696694a022fb9d58bba7321822c2544f1417ce", + url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-09-07T15-23-00Z/pjrt-metal_macos-arm64.tar.zst", ) return mctx.extension_metadata( From 7f5c678a58432b1f4bc7a6549e0fbc50ab7e3f11 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 8 Sep 2026 13:52:52 +0200 Subject: [PATCH 661/881] kernels: Add support for e4m3fnuz dtype (#780) --- kernels/triton/builder.zig | 12 +++++++----- kernels/triton/dtype.zig | 6 ++++-- zml/kernel.zig | 1 + 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/kernels/triton/builder.zig b/kernels/triton/builder.zig index bcae6421..a1cbf5b1 100644 --- a/kernels/triton/builder.zig +++ b/kernels/triton/builder.zig @@ -774,7 +774,7 @@ pub const Builder = struct { .i32 => r, .i64 => self.extsi(r, dtype), .i16, .i8 => self.trunci(r, dtype), - .f16, .bf16, .f32, .f64, .f8e4m3fn, .f8e5m2 => self.sitofp(r, dtype), + .f16, .bf16, .f32, .f64, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2 => self.sitofp(r, dtype), else => @panic("Builder.arange: unsupported dtype"), }; } @@ -1303,8 +1303,10 @@ pub const Builder = struct { } /// Numeric cast with options. Dispatch: - /// - fp↔fp narrow: truncf, except fp8 or custom rounding → tt.fp_to_fp. - /// - fp↔fp wide: extf. i1→fp: uitofp. int↔int: extsi/trunci/bitcast by width. + /// - fp↔fp with fp8: tt.fp_to_fp, with rounding only on downcasts. + /// - other fp↔fp narrow: truncf, except custom rounding → tt.fp_to_fp. + /// - other fp↔fp wide: extf. + /// - i1→fp: uitofp. int↔int: extsi/trunci/bitcast by width. pub fn castOpts(self: *Builder, src: Value, dtype: DType, opts: CastOpts) Value { if (opts.bitcast) return self.bitcast(src, dtype); const cur_elem = src.elemType(); @@ -1317,8 +1319,8 @@ pub const Builder = struct { const cur_bw = dtypeBitwidth(cur_dtype); const tgt_bw = dtypeBitwidth(dtype); if (tgt_bw > cur_bw) return self.extf(src, dtype); - const fp8_involved = (cur_dtype == .f8e4m3fn or cur_dtype == .f8e5m2 or - dtype == .f8e4m3fn or dtype == .f8e5m2); + const fp8_involved = (cur_dtype == .f8e4m3fn or cur_dtype == .f8e4m3fnuz or cur_dtype == .f8e5m2 or + dtype == .f8e4m3fn or dtype == .f8e4m3fnuz or dtype == .f8e5m2); const custom_rounding = opts.fp_downcast_rounding != null and opts.fp_downcast_rounding.? != .rtne; if (fp8_involved or custom_rounding) { return self.fpToFpOpts(src, dtype, .{ .rounding = opts.fp_downcast_rounding orelse .rtne }); diff --git a/kernels/triton/dtype.zig b/kernels/triton/dtype.zig index 3ac696b8..c8c1d623 100644 --- a/kernels/triton/dtype.zig +++ b/kernels/triton/dtype.zig @@ -12,6 +12,7 @@ pub const DType = enum { f32, f64, f8e4m3fn, + f8e4m3fnuz, f8e5m2, pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { @@ -26,6 +27,7 @@ pub const DType = enum { .f32 => .float(ctx, .f32), .f64 => .float(ctx, .f64), .f8e4m3fn => .float(ctx, .f8e4m3fn), + .f8e4m3fnuz => .float(ctx, .f8e4m3fnuz), .f8e5m2 => .float(ctx, .f8e5m2), }; } @@ -33,7 +35,7 @@ pub const DType = enum { pub fn isFloatDtype(dt: DType) bool { return switch (dt) { - .f16, .bf16, .f32, .f64, .f8e4m3fn, .f8e5m2 => true, + .f16, .bf16, .f32, .f64, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2 => true, else => false, }; } @@ -41,7 +43,7 @@ pub fn isFloatDtype(dt: DType) bool { pub fn dtypeBitwidth(dt: DType) usize { return switch (dt) { .i1 => 1, - .i8, .f8e4m3fn, .f8e5m2 => 8, + .i8, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2 => 8, .i16, .f16, .bf16 => 16, .i32, .f32 => 32, .i64, .f64 => 64, diff --git a/zml/kernel.zig b/zml/kernel.zig index 1c4f629e..b6791b8b 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -79,6 +79,7 @@ pub const triton = struct { .f32 => .f32, .f64 => .f64, .f8e4m3fn => .f8e4m3fn, + .f8e4m3fnuz => .f8e4m3fnuz, .f8e5m2 => .f8e5m2, else => std.debug.panic("zml.kernel.triton.from: dtype {s} has no Triton equivalent", .{@tagName(dt)}), }; From 12b299186a489f97d239d40efdb2de255566567b Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Tue, 8 Sep 2026 14:34:30 +0200 Subject: [PATCH 662/881] platforms/cuda: bump the plugin and cuDNN to 9.24.0 (#774) --- platforms/cuda/cuda.bzl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index c5b01869..2b248e3c 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -11,8 +11,8 @@ CUDA_VARIANT = "cuda13.3" CUDA_REDIST_JSON_SHA256 = "507eddaab1360336bc0fe17b77552e0b7dfe1e74da888671c3a2f5fad7775db1" CUDNN_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cudnn/redist/" -CUDNN_VERSION = "9.22.0" -CUDNN_REDIST_JSON_SHA256 = "3dbb9002d52112ef69aa09187f523ef1ff07f8baf3892ee01e540af639d8f55f" +CUDNN_VERSION = "9.24.0" +CUDNN_REDIST_JSON_SHA256 = "89a0626f1925d9f4ceb8d6bc0b2de160b30e92afaae5471e6cfbb50234ae5488" NVSHMEM_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/nvshmem/redist/" NVSHMEM_VERSION = "3.6.5" @@ -213,15 +213,15 @@ _UBUNTU_PACKAGES = { ], } -PJRT_CUDA_RELEASE = "manual-2026-07-31T19-22-00Z" +PJRT_CUDA_RELEASE = "manual-2026-09-08T13-23-00Z" _PJRT_CUDA_ASSETS = { "amd64": { - "sha256": "c3fe395ed8b0493975e4afeba454e0cbecec7eac540488522fd4523103e353f3", + "sha256": "e86ee894daeeeb83c95347bcda89df4317185d5e9ef9ff9290afa471f57a135a", "url": "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-cuda_linux-amd64.tar.gz", }, "arm64": { - "sha256": "3c23c31dbfb3a97ca1bf8558d1e8347431fa30c192236d97de7bd6539d354ca5", + "sha256": "60aae4b73193d980a04d478f85038f4c6069f2dd9559a6549a8f14c420e0d857", "url": "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-cuda_linux-arm64.tar.gz", }, } From 520f8d63c1d0124b07d5ee81ace2d388fc6bd023 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Tue, 8 Sep 2026 14:39:08 +0200 Subject: [PATCH 663/881] zml/Compiler: set the fork's flag defaults per target (#775) The fork keeps upstream's flag defaults, so zml turns on what each target actually wants: - the Tile IR scaled-dot backend, subchannel dequantize fusion and Triton multi-output fusion on CUDA - Metal AIR fast math off, since it lets the AIR compiler assume no NaNs and is_finite(NaN) comes back true. --- zml/Compiler.zig | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/zml/Compiler.zig b/zml/Compiler.zig index adf85695..d8e455ce 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -736,6 +736,9 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: // NVIDIA recommends these settings // https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_latency_hiding_scheduler", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_scaled_dot_with_tile_ir", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_enable_subchannel_dequantisation_fusion", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_unsupported_enable_triton_multi_output_fusion", true, upb_arena); }, .rocm => { // Use lld from libllvm instead of invoking the ld.lld binary. @@ -746,6 +749,9 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: // This is what AMD recommendeds in the meantime. try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "CUBLAS,CUBLASLT,CUSTOM_CALL,CUDNN,DYNAMIC_SLICE_FUSION", upb_arena); }, + .metal => { + try setXlaOverrideFlag(overrides_map, "xla_gpu_metal_fast_math", false, upb_arena); + }, .oneapi => { // More efficient for the allgather/broadcast implementation of the collective permute. try setXlaOverrideFlag(overrides_map, "xla_gpu_collective_permute_connected_components", true, upb_arena); From 9c92ff62039497432cf41edf86318bc3be427bc1 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Tue, 8 Sep 2026 14:42:32 +0200 Subject: [PATCH 664/881] zml/platform: one ComputeCapability for every CUDA gate (#777) Every CUDA gate used to parse the compute capability string itself. They now share `platform.cuda.computeCapability`, which returns null off CUDA and offers `.is` / `.atLeast` / `.sm`. Also folds the NVFP4 scale clamp into its reduce to remove one launch per projection, and is equivalent. --- zml/attention.zig | 19 +++------------- zml/attention/paged_attention.zig | 7 +----- zml/attention/triton_attention.zig | 12 ++++------ zml/moe/cutlass_flashinfer.zig | 17 +++++--------- zml/platform.zig | 36 +++++++++++++++++++++++++----- zml/quantization.zig | 16 +++++-------- 6 files changed, 50 insertions(+), 57 deletions(-) diff --git a/zml/attention.zig b/zml/attention.zig index a6962f47..1b492810 100644 --- a/zml/attention.zig +++ b/zml/attention.zig @@ -25,16 +25,8 @@ pub const Backend = enum { pub fn auto(platform: *const zml.Platform) Backend { return switch (platform.target) { .cuda => b: { - const first_device = platform.pjrt_client.devices(platform.pjrt_api)[0]; - - if (zml.platform.cuda.tryGetComputeCapabilities(platform, first_device)) |cc| { - break :b if (std.mem.eql(u8, cc, "9.0")) - .cuda_fa3 - else - .cuda_fa2; - } - - break :b .vanilla; + const cc = zml.platform.cuda.computeCapability(platform) orelse break :b .vanilla; + break :b if (cc.eql(.{ .major = 9, .minor = 0 })) .cuda_fa3 else .cuda_fa2; }, .neuron => .nki, .metal => .metal_fa, @@ -49,12 +41,7 @@ pub const Backend = enum { .nki => platform.target == .neuron, .metal_fa => platform.target == .metal, .cuda_fa2 => platform.target == .cuda, - .cuda_fa3 => { - if (platform.target != .cuda) return false; - const first_device = platform.pjrt_client.devices(platform.pjrt_api)[0]; - const cc = zml.platform.cuda.tryGetComputeCapabilities(platform, first_device) orelse return false; - return std.mem.eql(u8, cc, "9.0"); - }, + .cuda_fa3 => if (zml.platform.cuda.computeCapability(platform)) |cc| cc.eql(.{ .major = 9, .minor = 0 }) else false, }; } }; diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 8d3a8981..c2f73895 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -44,12 +44,7 @@ pub const Backend = enum { .metal => platform.target == .metal, .mosaic_tpu => platform.target == .tpu, .cuda_fa2 => platform.target == .cuda, - .cuda_fa3 => { - if (platform.target != .cuda) return false; - const first_device = platform.pjrt_client.devices(platform.pjrt_api)[0]; - const cc = zml.platform.cuda.tryGetComputeCapabilities(platform, first_device) orelse return false; - return std.mem.eql(u8, cc, "9.0"); - }, + .cuda_fa3 => if (zml.platform.cuda.computeCapability(platform)) |cc| cc.eql(.{ .major = 9, .minor = 0 }) else false, }; } }; diff --git a/zml/attention/triton_attention.zig b/zml/attention/triton_attention.zig index 34afefd5..360cc87c 100644 --- a/zml/attention/triton_attention.zig +++ b/zml/attention/triton_attention.zig @@ -52,13 +52,9 @@ const SparseMlaConfig = struct { parallel_reduce_num_warps: usize = 1, }; -fn isCudaComputeCapability(expected: []const u8) bool { +fn isCudaComputeCapability(expected: zml.platform.cuda.ComputeCapability) bool { const platform = zml.module.CompilationContext.current().platform; - if (platform.target != .cuda) return false; - const devices = platform.pjrt_client.devices(platform.pjrt_api); - if (devices.len == 0) return false; - const actual = zml.platform.cuda.tryGetComputeCapabilities(platform, devices[0]) orelse return false; - return std.mem.eql(u8, actual, expected); + return if (zml.platform.cuda.computeCapability(platform)) |cc| cc.eql(expected) else false; } fn sparseMlaConfig(paged_opts: paged.PagedSparseMlaOptions, topk_count: usize, cu_count_: usize) SparseMlaConfig { @@ -74,7 +70,7 @@ fn sparseMlaConfig(paged_opts: paged.PagedSparseMlaOptions, topk_count: usize, c .num_splits = 1, .direct_programs = undefined, }; - const is_sm103 = isCudaComputeCapability("10.3"); + const is_sm103 = isCudaComputeCapability(.{ .major = 10, .minor = 3 }); // GB300 (sm_103): a single wide query benefits from more head blocks and // wider sparse tiles. The split selection below still derives each layer's @@ -154,7 +150,7 @@ fn select2dConfig(options: paged.PagedAttentionOptions) Config2D { const max_num_stages_2d: usize = if (options.head_dim <= 128) 4 else 2; // Until we test on other platforms, gate the fix to GB300 - const is_gb300 = isCudaComputeCapability("10.3"); + const is_gb300 = isCudaComputeCapability(.{ .major = 10, .minor = 3 }); var num_stages_2d: usize, var num_warps: usize, var tile_size: usize = if (!options.all_decode) .{ 1, 2, 64 } else .{ 3, 2, options.block_size }; diff --git a/zml/moe/cutlass_flashinfer.zig b/zml/moe/cutlass_flashinfer.zig index 2efdbd45..81dfaffa 100644 --- a/zml/moe/cutlass_flashinfer.zig +++ b/zml/moe/cutlass_flashinfer.zig @@ -357,18 +357,11 @@ const routedBf16Call = zml.ops.CustomCall(Bf16Input, Output, Attributes, ffiCall }); fn computeCapability(platform: *const zml.Platform) !u16 { - if (platform.target != .cuda) return error.UnsupportedPlatform; - const devices = platform.pjrt_client.devices(platform.pjrt_api); - if (devices.len == 0) return error.NoDevices; - - const compute_capability = zml.platform.cuda.tryGetComputeCapabilities(platform, devices[0]) orelse - return error.UnknownComputeCapability; - - if (std.mem.eql(u8, compute_capability, "9.0")) return 90; - if (std.mem.eql(u8, compute_capability, "10.0")) return 100; - if (std.mem.eql(u8, compute_capability, "10.3")) return 103; - if (std.mem.eql(u8, compute_capability, "12.0")) return 120; - return error.UnsupportedArchitecture; + const cc = zml.platform.cuda.computeCapability(platform) orelse return error.UnsupportedPlatform; + return switch (cc.sm()) { + 90, 100, 103, 120 => |sm| sm, + else => error.UnsupportedArchitecture, + }; } pub fn load( diff --git a/zml/platform.zig b/zml/platform.zig index 23432742..7106aee0 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -795,14 +795,40 @@ pub const CreateOptions = struct { // TODO(Corendos): Consider moving that in its own file if its size increase too much. pub const cuda = struct { - pub fn tryGetComputeCapabilities(platform: *const zml.Platform, device: *const pjrt.Device) ?[]const u8 { - stdx.debug.assert(platform.target == .cuda, "tryGetComputeCapabilities expects .cuda platform, got {}", .{platform.target}); - const description = device.getDescription(platform.pjrt_api); + pub const ComputeCapability = struct { + major: u8, + minor: u8, - const attributes = description.attributes(platform.pjrt_api); + pub fn parse(text: []const u8) ?ComputeCapability { + var parts = std.mem.splitScalar(u8, text, '.'); + return .{ + .major = std.fmt.parseInt(u8, parts.first(), 10) catch return null, + .minor = std.fmt.parseInt(u8, parts.next() orelse "0", 10) catch return null, + }; + } + + pub fn eql(self: ComputeCapability, other: ComputeCapability) bool { + return self.major == other.major and self.minor == other.minor; + } + + pub fn atLeast(self: ComputeCapability, other: ComputeCapability) bool { + return self.major > other.major or (self.major == other.major and self.minor >= other.minor); + } + + pub fn sm(self: ComputeCapability) u16 { + return @as(u16, self.major) * 10 + self.minor; + } + }; + + pub fn computeCapability(platform: *const zml.Platform) ?ComputeCapability { + if (platform.target != .cuda) return null; + const devices = platform.pjrt_client.devices(platform.pjrt_api); + if (devices.len == 0) return null; + + const attributes = devices[0].getDescription(platform.pjrt_api).attributes(platform.pjrt_api); return for (attributes) |attr| { if (std.mem.eql(u8, attr.name(), "compute_capability")) { - break attr.value().string; + break ComputeCapability.parse(attr.value().string); } } else null; } diff --git a/zml/quantization.zig b/zml/quantization.zig index de1e324d..ad25339b 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -123,11 +123,13 @@ pub fn quantizeNvfp4(x: Tensor, input_global_scale: ?Tensor, axis: anytype) Quan else x; const grouped = scaled.splitAxis(axis, .{ .sc = -1, .blk = nvfp4_block_size }); - const amax = grouped.abs().max(.blk); - const scales = amax.scale(1.0 / value_max) + const amax = grouped.abs() + .scale(1.0 / value_max) .clamp(.scalar(scale_min_normal, dt), .scalar(scale_max, dt)) - .convert(.f8e4m3fn); + .max(.blk); + + const scales = amax.convert(.f8e4m3fn); const divisor = scales.convert(dt) .maximum(.scalar(scale_min_normal, dt)) @@ -145,13 +147,7 @@ pub fn quantizeNvfp4(x: Tensor, input_global_scale: ?Tensor, axis: anytype) Quan } fn supportsNvfp4InputQuantization(platform: *const Platform) bool { - if (platform.target != .cuda) return false; - - const device = platform.pjrt_client.devices(platform.pjrt_api)[0]; - const capability = platform_mod.cuda.tryGetComputeCapabilities(platform, device) orelse return false; - const major = std.fmt.parseInt(u8, std.mem.sliceTo(capability, '.'), 10) catch return false; - - return major >= 10; + return if (platform_mod.cuda.computeCapability(platform)) |cc| cc.atLeast(.{ .major = 10, .minor = 0 }) else false; } fn isPackedFp4(scheme: ?Quantization.Scheme, weight_dtype: DataType) bool { From 77e41879d31aba5d2442bc1e272c4635751801ad Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Tue, 8 Sep 2026 14:57:40 +0200 Subject: [PATCH 665/881] zml/kernel: add a CUDA Tile IR (cuda_tile) Zig DSL (#779) A third kernel language beside Triton and Mosaic TPU. It emits NVIDIA's cuda_tile dialect and reaches the GPU through XLA's `__gpu$xla.gpu.cuda_tile` custom call. - `third_party/cuda_tile`: cuda-tile v13.3.3 - `mlir/dialects/cuda_tile`: every type and enum through the dialect's C API, one constructor per op (all 100, in Ops.td order), and a small C shim for the TileView interface the C API does not expose. - `kernels/cuda_tile`: views, tokens, for/if/loop, reduce/scan (single and variadic), atomics, MMA, every elementwise op. - `zml.kernel.cuda_tile.Kernel` and `ops.cudaTile`: the custom call, with a grid and no num_warps/num_stages (those are optimization_hints in the IR). --- MODULE.bazel | 1 + kernels/cuda_tile/BUILD.bazel | 24 + kernels/cuda_tile/README.md | 132 ++ kernels/cuda_tile/builder.zig | 2197 +++++++++++++++++++++ kernels/cuda_tile/control_flow.zig | 155 ++ kernels/cuda_tile/dtype.zig | 103 + mlir/dialects/BUILD.bazel | 1 + mlir/dialects/cuda_tile/BUILD.bazel | 34 + mlir/dialects/cuda_tile/cuda_tile.zig | 1696 ++++++++++++++++ mlir/dialects/cuda_tile/cuda_tile_capi.cc | 17 + mlir/dialects/cuda_tile/cuda_tile_capi.h | 24 + mlir/dialects/dialects.zig | 1 + third_party/cuda_tile/BUILD.bazel | 0 third_party/cuda_tile/cuda_tile.bazel | 415 ++++ third_party/cuda_tile/repo.bzl | 10 + third_party/non_module_deps.bzl | 2 + zml/BUILD.bazel | 2 + zml/kernel.zig | 132 ++ zml/ops.zig | 254 +++ 19 files changed, 5200 insertions(+) create mode 100644 kernels/cuda_tile/BUILD.bazel create mode 100644 kernels/cuda_tile/README.md create mode 100644 kernels/cuda_tile/builder.zig create mode 100644 kernels/cuda_tile/control_flow.zig create mode 100644 kernels/cuda_tile/dtype.zig create mode 100644 mlir/dialects/cuda_tile/BUILD.bazel create mode 100644 mlir/dialects/cuda_tile/cuda_tile.zig create mode 100644 mlir/dialects/cuda_tile/cuda_tile_capi.cc create mode 100644 mlir/dialects/cuda_tile/cuda_tile_capi.h create mode 100644 third_party/cuda_tile/BUILD.bazel create mode 100644 third_party/cuda_tile/cuda_tile.bazel create mode 100644 third_party/cuda_tile/repo.bzl diff --git a/MODULE.bazel b/MODULE.bazel index 21e88df8..974cd6d8 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -253,6 +253,7 @@ use_repo( "arocc", "cloud_accelerator_diagnostics", "com_google_sentencepiece", + "cuda_tile", "flashattn_linux_amd64", "flashattn_linux_arm64", "flashinfer_cutlass_moe_linux_amd64", diff --git a/kernels/cuda_tile/BUILD.bazel b/kernels/cuda_tile/BUILD.bazel new file mode 100644 index 00000000..03208310 --- /dev/null +++ b/kernels/cuda_tile/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "cuda_tile_builder", + tags = ["manual"], + srcs = [ + "control_flow.zig", + "dtype.zig", + ], + import_name = "kernels/cuda_tile/builder", + main = "builder.zig", + visibility = ["//visibility:public"], + deps = [ + "//kernels/common", + "//mlir", + "//mlir/dialects/cuda_tile", + "//stdx", + ], +) + +zig_test( + name = "test", + deps = [":cuda_tile_builder"], +) diff --git a/kernels/cuda_tile/README.md b/kernels/cuda_tile/README.md new file mode 100644 index 00000000..aac90f2f --- /dev/null +++ b/kernels/cuda_tile/README.md @@ -0,0 +1,132 @@ +# CUDA Tile IR kernels in Zig + +`kernels/cuda_tile` is a Zig DSL that emits NVIDIA's `cuda_tile` MLIR dialect +(CUDA Tile IR). It is the third instance of the `kernels/triton` / +`kernels/mosaic_tpu` pattern: a `Builder` that owns an MLIR module and appends +ops to a block stack, a `Value` with fluent arithmetic, and a +`zml.kernel.cuda_tile.Kernel` wrapper that turns a `run` function into a +`stablehlo.custom_call` XLA compiles with `tileiras`. + +The dialect binding lives in `mlir/dialects/cuda_tile`: one function per op +(all 100 of them, in `Ops.td` order), every type through the dialect's C API, +every enum attribute through its C getter. + +## A kernel + +```zig +const zml = @import("zml"); +const cut = zml.kernel.cuda_tile; + +const Cfg = struct { n: i64 }; + +pub const AddOne = cut.Kernel(Cfg, .{ + .name = "add_one", + .inputs = &.{"x"}, + .outputs = &.{"out"}, + .run = run, +}); + +fn run(b: *cut.Builder, cfg: Cfg) cut.FinishError!void { + // Every parameter is a rank-0 tile>; views are built inside. + const a = try b.declareArgsOpts(.{ + .x = .{ .ptr = .f32 }, + .out = .{ .ptr = .f32 }, + }, .{ .hints = &.{.{ .arch = .sm_120, .num_worker_warps_per_cta = 4 }} }); + + // pointer -> tensor_view -> partition_view; the view pads the ragged edge. + const vx = b.partitionView(b.tensorView(a.x, &.{cfg.n}, &.{1}), &.{128}, .{}); + const vo = b.partitionView(b.tensorView(a.out, &.{cfg.n}, &.{1}), &.{128}, .{}); + + const bid = b.tileBlockId(); + _ = b.store(b.load(vx, &.{bid.x}).add(1.0), vo, &.{bid.x}); +} + +// In a model: +const out = AddOne.call(.{ .x = x }, .{ .out = x.shape() }, .{ + .cfg = .{ .n = 1024 }, + .grid = .{ 8, 1, 1 }, +}).out; +``` + +## What is different from the Triton DSL + +* **Rank 0 is the scalar.** Every value is a `tile<...>`; `entry` parameters + must be rank 0, so buffers arrive as `tile>`. +* **Memory is a view stack.** `tensorView` → `partitionView` (or + `stridedView`, `gatherScatterView`) → tile-space `load`/`store`. A partial + edge tile reads the view's `padding` and its store is masked; there is no + mask tile. The pointer-tile arm (`loadPtr`/`storePtr` with `offset`) exists + for gathers but cannot use TMA. +* **Every load and store returns a token.** The dialect has no barrier or + fence; `loadOpts`/`storeOpts` take and return tokens, `load`/`store` drop + them. +* **Float ops carry a rounding mode.** `addf` defaults to `nearest_even`; + `addfOpts` sets it. Integer ops carry none. +* **Loop bodies end with `continue`, `if` bodies with `yield`, and `loop` + exits with `break`** — `ForScope`, `IfScope`, `LoopScope` do this for you. +* **No `num_warps`/`num_stages`.** Warp and CTA tuning is + `optimization_hints` on the entry, per architecture (`Opts.hints`), and + `allow_tma`/`latency` on view loads and stores (`LoadOpts.hints`). +* **Tile dims are powers of two**, at most 2^24 elements; `tileTy` and + `partitionView` check this in Zig. +* **Some conversions are one-way.** `ftof` to `f8e8m0fnu` (the MX scale + type) rounds toward zero by default (`CastOpts.rounding = .positive_inf` + is the other legal choice); an integer reaches it through `f32`, which + `cast`/`to` do for you. `ftoi` always truncates toward zero. `tanh` takes + only `approx`/`full`. `Value.cdiv` is integer-only. +* **`i4` is a tile element type, not a pointee.** Declare a 4-bit buffer as + `.i8` and `unpack` it in the kernel. +* **`atomicRedView` is relaxed-only** on an unpadded view and takes no mask; + use the pointer atomics (`atomicRmw`, `atomicCas`) for masked updates. +* **Reductions over several tiles at once** (argmax, an online softmax's + (max, sum)) go through `reduceMulti`/`scanMulti`. +* **A scalar argument is a pointer**: `.{ .n = .{ .ptr = .i32 } }` then + `b.loadPtr(a.n)` gives the rank-0 tile; XLA passes device addresses only. + +## The custom call + +`ops.cudaTile` emits `__gpu$xla.gpu.cuda_tile` with a printed dictionary: +`name`, `kernel_type = "cuda_tile"`, `ir` (the textual module), `grid_x/y/z`, +optional `ir_version`. XLA parses the module, renames the entry, checks the +argument count against the operands + result leaves, scrubs locations, +serializes bytecode, runs `tileiras`, and launches over `grid` tile blocks with +a `(1,1,1)` block and no shared memory — cuda-tile's launch ABI. Every operand +and result is one raw device pointer in the default layout, operands first; +XLA rejects an entry whose parameters are not all `tile>`. + +To see what XLA received, run with `--xla_dump_to=`: it writes +`.cuda_tile.mlir` (post-rename) and `.tilebc` (the exact bytes +handed to `tileiras`). + +## Finding what exists + +There is one function per op and no hidden API, so grep is the index: + +```sh +# every op of the dialect, in Ops.td order (115 constructors) +grep -n '^pub fn ' mlir/dialects/cuda_tile/cuda_tile.zig + +# the types, enums and attribute helpers they take +grep -n '^pub const ' mlir/dialects/cuda_tile/cuda_tile.zig + +# the DSL: Value's fluent methods (add, mul, lt, to, sum, reshape, ...) +awk '/^pub const Value = struct/,/^};/' kernels/cuda_tile/builder.zig | grep 'pub fn ' + +# the DSL: everything on Builder (views, load/store, math, casts, reductions, +# atomics, control flow) -- the `*Opts` variant of a call takes the options +awk '/^pub const Builder = struct/,0' kernels/cuda_tile/builder.zig | grep ' pub fn ' + +# what each option struct holds, and its defaults +grep -n '^pub const [A-Za-z]*Opts = struct' -A 8 kernels/cuda_tile/builder.zig + +# for / if / loop scopes and their terminators +grep -n 'pub fn ' kernels/cuda_tile/control_flow.zig + +# element types +grep -n 'pub const DType = enum' -A 30 kernels/cuda_tile/dtype.zig +``` + +Every op also has a test: `grep -n '^test "' kernels/cuda_tile/builder.zig` +lists them, and each one prints IR that round-trips through the verifier, so +they double as usage examples. A worked, runnable example (saxpy, softmax, +gemm, in-place relu) lives in `examples/cuda_tile`. diff --git a/kernels/cuda_tile/builder.zig b/kernels/cuda_tile/builder.zig new file mode 100644 index 00000000..d73742a1 --- /dev/null +++ b/kernels/cuda_tile/builder.zig @@ -0,0 +1,2197 @@ +const std = @import("std"); + +const ct = @import("mlir/dialects/cuda_tile"); +const dsl = @import("kernels/common"); +const tupleArity = dsl.tupleArity; +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +const cf = @import("control_flow.zig"); +const dtypes = @import("dtype.zig"); +pub const DType = dtypes.DType; +const isFloatDtype = dtypes.isFloatDtype; +const dtypeBitwidth = dtypes.dtypeBitwidth; +const intBitwidth = dtypes.intBitwidth; + +pub const RoundingMode = ct.RoundingMode; +pub const Signedness = ct.Signedness; +pub const IntegerOverflow = ct.IntegerOverflow; +pub const ComparisonPredicate = ct.ComparisonPredicate; +pub const ComparisonOrdering = ct.ComparisonOrdering; +pub const AtomicRMWMode = ct.AtomicRMWMode; +pub const MemoryScope = ct.MemoryScope; +pub const MemoryOrdering = ct.MemoryOrdering; +pub const PaddingValue = ct.PaddingValue; +pub const Arch = ct.Arch; +pub const EntryHint = ct.EntryHint; +pub const LoadStoreHint = ct.LoadStoreHint; +pub const PtrLoadStoreHint = ct.PtrLoadStoreHint; + +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(Builder); + std.testing.refAllDecls(Value); +} + +pub const dialects_needed = [_][]const u8{"cuda_tile"}; + +pub const FinishError = error{InvalidMlir} || std.mem.Allocator.Error || std.Io.Writer.Error; + +pub const Value = struct { + inner: *const mlir.Value, + kernel: ?*Builder = null, + + pub const Shape = stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK); + + pub fn type_(self: Value) *const mlir.Type { + return self.inner.type_(); + } + + fn kern(self: Value) *Builder { + return self.kernel orelse @panic("Value has no owning kernel; use Builder.* helpers instead"); + } + + pub fn tile(self: Value) *const ct.TileType { + return self.type_().isA(ct.TileType) orelse std.debug.panic("Value {f} is not a tile", .{self.type_()}); + } + + pub fn isTile(self: Value) bool { + return self.type_().isA(ct.TileType) != null; + } + + pub fn elemType(self: Value) *const mlir.Type { + return self.tile().elementType(); + } + + pub fn rank(self: Value) usize { + return self.tile().rank(); + } + + pub fn isScalar(self: Value) bool { + return self.rank() == 0; + } + + pub fn dim(self: Value, i: usize) i64 { + return self.tile().dimension(i); + } + + pub fn shape(self: Value) Value.Shape { + var out: Value.Shape = .empty; + const t = self.tile(); + for (0..t.rank()) |i| out.appendAssumeCapacity(t.dimension(i)); + return out; + } + + pub fn isFloatElem(self: Value) bool { + const et = self.elemType(); + inline for (std.meta.fields(mlir.FloatTypes)) |f| { + if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, f.name))) != null) return true; + } + return false; + } + + pub fn isIntElem(self: Value) bool { + return self.elemType().isA(mlir.IntegerType) != null; + } + + pub fn isPtrElem(self: Value) bool { + return self.elemType().isA(ct.PointerType) != null; + } + + pub fn add(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.addf(l, r) else k.addi(l, r); + } + + pub fn sub(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.subf(l, r) else k.subi(l, r); + } + + pub fn mul(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.mulf(l, r) else k.muli(l, r); + } + + pub fn div(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.divf(l, r) else k.divi(l, r, .signed); + } + + pub fn rem(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.remf(l, r) else k.remi(l, r, .signed); + } + + pub fn cdiv(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + if (l.isFloatElem()) @panic("Value.cdiv is integer-only; use div (and ceil) on floats"); + return k.diviOpts(l, r, .{ .signedness = .signed, .rounding = .positive_inf }); + } + + pub fn neg(self: Value) Value { + const k = self.kern(); + return if (self.isFloatElem()) k.negf(self) else k.negi(self); + } + + pub fn abs(self: Value) Value { + const k = self.kern(); + return if (self.isFloatElem()) k.absf(self) else k.absi(self); + } + + pub fn bitAnd(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.andi(l, r); + } + + pub fn bitOr(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.ori(l, r); + } + + pub fn bitXor(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.xori(l, r); + } + + pub fn shl(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.shli(l, r); + } + + pub fn shr(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.shri(l, r, .signed); + } + + pub fn minimum(self: Value, rhs: anytype) Value { + return self.kern().minimum(self, rhs); + } + + pub fn maximum(self: Value, rhs: anytype) Value { + return self.kern().maximum(self, rhs); + } + + fn compare(self: Value, rhs: anytype, pred: ComparisonPredicate) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.cmpf(pred, l, r) else k.cmpi(pred, l, r, .signed); + } + + pub fn lt(self: Value, rhs: anytype) Value { + return self.compare(rhs, .less_than); + } + pub fn le(self: Value, rhs: anytype) Value { + return self.compare(rhs, .less_than_or_equal); + } + pub fn gt(self: Value, rhs: anytype) Value { + return self.compare(rhs, .greater_than); + } + pub fn ge(self: Value, rhs: anytype) Value { + return self.compare(rhs, .greater_than_or_equal); + } + pub fn eq(self: Value, rhs: anytype) Value { + return self.compare(rhs, .equal); + } + pub fn ne(self: Value, rhs: anytype) Value { + return self.compare(rhs, .not_equal); + } + + pub fn to(self: Value, dtype: DType) Value { + return self.kern().cast(self, dtype); + } + + pub fn reshape(self: Value, shape_: []const i64) Value { + return self.kern().reshape(self, shape_); + } + + pub fn broadcastTo(self: Value, shape_: []const i64) Value { + return self.kern().broadcastTo(self, shape_); + } + + pub fn expandDims(self: Value, axis: usize) Value { + return self.kern().expandDims(self, axis); + } + + pub fn permute(self: Value, order: []const i32) Value { + return self.kern().permute(self, order); + } + + pub fn sum(self: Value, axis: usize) Value { + return self.kern().sum(self, axis); + } + + pub fn max(self: Value, axis: usize) Value { + return self.kern().max(self, axis); + } + + pub fn min(self: Value, axis: usize) Value { + return self.kern().min(self, axis); + } + + pub fn cumsum(self: Value, axis: usize) Value { + return self.kern().cumsum(self, axis); + } + + pub fn offset(self: Value, off: anytype) Value { + const k = self.kern(); + const o: Value = if (@TypeOf(off) == Value) off else k.lift(off); + const p, const o2 = k.coerceShapes(self, o); + return k.offset(p, o2); + } +}; + +pub const Loaded = struct { + tile: Value, + token: Value, +}; + +pub const BlockId = struct { x: Value, y: Value, z: Value }; + +pub const ArgSpec = struct { + name: []const u8, + kind: Kind, + + /// XLA passes one raw device pointer per custom-call argument, so every + /// parameter is a rank-0 `tile>` and views are built inside the + /// kernel. A scalar reaches the kernel through a pointer (`loadPtr` on a + /// rank-0 pointer tile) or as a comptime constant in `cfg`. + pub const Kind = union(enum) { + ptr: DType, + ptr_opts: PtrOpts, + }; + + /// `div_by` becomes a `cuda_tile.assume #cuda_tile.div_by` at the top + /// of the entry, and `arg(i)` returns the refined value. XLA guarantees + /// only 16 bytes on the pointer it passes. + pub const PtrOpts = struct { + dtype: DType, + div_by: ?u32 = 16, + }; +}; + +pub const Opts = struct { + hints: []const EntryHint = &.{}, +}; + +pub const FloatOpts = struct { + rounding: RoundingMode = .nearest_even, + flush_to_zero: bool = false, +}; + +pub const MinMaxOpts = struct { + propagate_nan: bool = false, + flush_to_zero: bool = false, +}; + +pub const IntOpts = struct { + overflow: IntegerOverflow = .none, +}; + +pub const DivIOpts = struct { + signedness: Signedness = .signed, + rounding: ?RoundingMode = null, +}; + +/// `rounding` null picks the target's legal default: `nearest_even`, except +/// `f8e8m0fnu`, which the dialect only lets you reach with `zero` or +/// `positive_inf`. +pub const CastOpts = struct { + rounding: ?RoundingMode = null, + signedness: Signedness = .signed, +}; + +/// `ftoi` admits exactly one rounding mode (`nearest_int_to_zero`), so only +/// the signedness is a choice. +pub const FToIOpts = struct { + signedness: Signedness = .signed, +}; + +pub const MmaOpts = struct { + /// 13.3 only. + fast_acc: bool = false, +}; + +pub const PartitionOpts = struct { + padding: ?PaddingValue = .zero, + dim_map: []const i32 = &.{}, +}; + +pub const LoadOpts = struct { + ordering: MemoryOrdering = .weak, + scope: ?MemoryScope = null, + token: ?Value = null, + hints: []const LoadStoreHint = &.{}, +}; + +pub const StoreOpts = LoadOpts; + +/// The pointer arm takes `latency` hints only; `allow_tma` is a view-op hint. +pub const LoadPtrOpts = struct { + mask: ?Value = null, + padding: ?Value = null, + ordering: MemoryOrdering = .weak, + scope: ?MemoryScope = null, + token: ?Value = null, + hints: []const PtrLoadStoreHint = &.{}, +}; + +pub const StorePtrOpts = struct { + mask: ?Value = null, + ordering: MemoryOrdering = .weak, + scope: ?MemoryScope = null, + token: ?Value = null, + hints: []const PtrLoadStoreHint = &.{}, +}; + +pub const AtomicOpts = struct { + ordering: MemoryOrdering = .acq_rel, + scope: MemoryScope = .device, + mask: ?Value = null, + token: ?Value = null, +}; + +/// `atomic_red_view_tko` is relaxed-only, scoped to the block or device, and +/// has no mask operand — the view's padding decides what is touched. +pub const RedViewOpts = struct { + scope: MemoryScope = .device, + token: ?Value = null, +}; + +pub const AllocaOpts = struct { + alignment: i64 = 16, + global: bool = false, +}; + +pub const GlobalOpts = struct { + alignment: ?i64 = null, + constant: bool = false, + visibility: ?ct.SymbolVisibility = null, +}; + +pub const ScanOpts = struct { + reverse: bool = false, +}; + +pub fn ForScope(comptime N: usize) type { + return cf.ForScope(Builder, Value, N); +} + +pub const IfOnlyScope = cf.IfOnlyScope(Builder, Value); + +pub fn IfScope(comptime N: usize) type { + return cf.IfScope(Builder, Value, N); +} + +pub fn LoopScope(comptime N: usize) type { + return cf.LoopScope(Builder, Value, N); +} + +/// A custom reduction: `combine(builder, element, accumulator, ctx)` runs on +/// rank-0 tiles of `src`'s element type; `identity` is the element attribute +/// the dialect requires (see `Builder.identityAttr`). +pub fn ReduceArgs(comptime CtxT: type) type { + return struct { + src: Value, + axis: usize, + identity: *const mlir.Attribute, + combine: *const fn (*Builder, Value, Value, CtxT) Value, + }; +} + +pub fn ScanArgs(comptime CtxT: type) type { + return struct { + src: Value, + axis: usize, + reverse: bool = false, + identity: *const mlir.Attribute, + combine: *const fn (*Builder, Value, Value, CtxT) Value, + }; +} + +/// A reduction over several same-shaped tiles at once (argmax, an online +/// softmax's (max, sum)): `combine(builder, elements, accumulators, ctx)` +/// gets one rank-0 tile per source in each slice and returns the new +/// accumulators, one per source; `identities` is one element attribute per +/// source. +pub fn ReduceMultiArgs(comptime CtxT: type) type { + return struct { + srcs: []const Value, + axis: usize, + identities: []const *const mlir.Attribute, + combine: *const fn (*Builder, []const Value, []const Value, CtxT) []const Value, + }; +} + +pub fn ScanMultiArgs(comptime CtxT: type) type { + return struct { + srcs: []const Value, + axis: usize, + reverse: bool = false, + identities: []const *const mlir.Attribute, + combine: *const fn (*Builder, []const Value, []const Value, CtxT) []const Value, + }; +} + +pub const Builder = struct { + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + ctx: *mlir.Context, + module: *mlir.Module, + name: []const u8, + func_op: ?*mlir.Operation, + entry_block: ?*mlir.Block, + block_stack: std.ArrayList(*mlir.Block), + ct_module: ?*mlir.Operation = null, + ct_body: ?*mlir.Block = null, + refined_args: []?Value = &.{}, + + pub fn init( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + args: []const ArgSpec, + ) !Builder { + return initOpts(allocator, ctx, name, args, .{}); + } + + pub fn initOpts( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + args: []const ArgSpec, + opts: Opts, + ) !Builder { + var b = try open(allocator, ctx, name); + errdefer b.deinit(); + try b.declareArgsLowOpts(args, opts); + return b; + } + + pub fn open( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + ) !Builder { + const unknown_loc: *const mlir.Location = .unknown(ctx); + const module: *mlir.Module = .init(unknown_loc); + errdefer module.deinit(); + + var arena: std.heap.ArenaAllocator = .init(allocator); + errdefer arena.deinit(); + + return .{ + .allocator = allocator, + .arena = arena, + .ctx = ctx, + .module = module, + .name = name, + .func_op = null, + .entry_block = null, + .block_stack = .empty, + }; + } + + pub fn declareArgs(self: *Builder, spec: anytype) !dsl.NamedArgs(@TypeOf(spec), Value) { + return self.declareArgsOpts(spec, .{}); + } + + pub fn declareArgsOpts(self: *Builder, spec: anytype, opts: Opts) !dsl.NamedArgs(@TypeOf(spec), Value) { + const Spec = @TypeOf(spec); + const fields = @typeInfo(Spec).@"struct".fields; + + var arg_specs: [fields.len]ArgSpec = undefined; + inline for (fields, 0..) |f, i| { + const raw = @field(spec, f.name); + const kind: ArgSpec.Kind = if (@TypeOf(raw) == ArgSpec.Kind) raw else blk: { + const variant = @typeInfo(@TypeOf(raw)).@"struct".fields[0].name; + const tag = @field(std.meta.Tag(ArgSpec.Kind), variant); + const inner = @field(raw, variant); + break :blk switch (tag) { + .ptr => .{ .ptr = inner }, + .ptr_opts => .{ .ptr_opts = .{ + .dtype = inner.dtype, + .div_by = if (@hasField(@TypeOf(inner), "div_by")) inner.div_by else 16, + } }, + }; + }; + arg_specs[i] = .{ .name = f.name, .kind = kind }; + } + + try self.declareArgsLowOpts(&arg_specs, opts); + + var named: dsl.NamedArgs(Spec, Value) = undefined; + inline for (fields, 0..) |f, i| { + @field(named, f.name) = self.arg(i); + } + return named; + } + + fn declareArgsLowOpts(self: *Builder, args: []const ArgSpec, opts: Opts) !void { + std.debug.assert(self.entry_block == null); + const ctx = self.ctx; + const unknown_loc: *const mlir.Location = .unknown(ctx); + const scratch = self.arena.allocator(); + + const arg_types = try scratch.alloc(*const mlir.Type, args.len); + const arg_locs = try scratch.alloc(*const mlir.Location, args.len); + for (arg_types, arg_locs, args) |*ty, *a_loc, a| { + ty.* = switch (a.kind) { + .ptr => |dt| self.ptrTy(dt), + .ptr_opts => |p| self.ptrTy(p.dtype), + }; + a_loc.* = unknown_loc; + } + + const entry = mlir.Block.init(arg_types, arg_locs); + const ct_body = mlir.Block.init(&.{}, &.{}); + const ct_module = ct.module(ctx, self.name, ct_body, unknown_loc); + _ = ct_module.appendTo(self.module.body()); + + const func_op = ct.entry(ctx, .{ + .name = self.name, + .block = entry, + .optimization_hints = if (opts.hints.len > 0) ct.optimizationHints(ctx, opts.hints) else null, + .location = unknown_loc, + }); + _ = func_op.appendTo(ct_body); + + self.func_op = func_op; + self.entry_block = entry; + self.ct_module = ct_module; + self.ct_body = ct_body; + + self.refined_args = try scratch.alloc(?Value, args.len); + for (self.refined_args, args, 0..) |*refined, a, i| { + refined.* = switch (a.kind) { + .ptr => self.assumeDivBy(self.rawArg(i), 16), + .ptr_opts => |p| if (p.div_by) |n| self.assumeDivBy(self.rawArg(i), n) else null, + }; + } + } + + pub fn deinit(self: *Builder) void { + self.module.deinit(); + self.arena.deinit(); + } + + fn rawArg(self: *Builder, i: usize) Value { + const eb = self.entry_block orelse @panic("Builder.arg called before declareArgs"); + return .{ .inner = eb.argument(i), .kernel = self }; + } + + /// Parameter `i`, refined by its `assume` when one was declared. The + /// owner is re-stamped: `initOpts` fills `refined_args` through a Builder + /// that is then moved out of its frame. + pub fn arg(self: *Builder, i: usize) Value { + if (i < self.refined_args.len) { + if (self.refined_args[i]) |v| return .{ .inner = v.inner, .kernel = self }; + } + return self.rawArg(i); + } + + pub fn pushBlock(self: *Builder, b: *mlir.Block) void { + self.block_stack.append(self.arena.allocator(), b) catch @panic("Builder.pushBlock OOM"); + } + + pub fn popBlock(self: *Builder) void { + _ = self.block_stack.pop(); + } + + pub fn currentBlock(self: *Builder) *mlir.Block { + if (self.block_stack.items.len > 0) { + return self.block_stack.items[self.block_stack.items.len - 1]; + } + return self.entry_block orelse @panic("Builder has no current block — call declareArgs first"); + } + + fn innerSlice(self: *Builder, values: []const Value) []const *const mlir.Value { + const out = self.arena.allocator().alloc(*const mlir.Value, values.len) catch @panic("Builder.innerSlice OOM"); + for (values, 0..) |v, i| out[i] = v.inner; + return out; + } + + fn innerOpt(v: ?Value) ?*const mlir.Value { + return if (v) |x| x.inner else null; + } + + pub fn emit(self: *Builder, op: *mlir.Operation) Value { + _ = op.appendTo(self.currentBlock()); + return .{ .inner = op.result(0), .kernel = self }; + } + + pub fn emitMulti(self: *Builder, op: *mlir.Operation, n: usize) []Value { + _ = op.appendTo(self.currentBlock()); + const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.emitMulti OOM"); + for (0..n) |i| out[i] = .{ .inner = op.result(i), .kernel = self }; + return out; + } + + pub fn yield(self: *Builder, values: anytype) []const Value { + const T = @TypeOf(values); + const info = @typeInfo(T); + if (info != .@"struct" or !info.@"struct".is_tuple) + @compileError("Builder.yield expects a tuple literal like `.{ v1, v2 }`"); + const n = info.@"struct".fields.len; + const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.yield OOM"); + inline for (info.@"struct".fields, 0..) |f, i| { + if (f.type != Value) + @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(f.type)); + out[i] = @field(values, f.name); + } + return out; + } + + pub fn loc(self: *const Builder) *const mlir.Location { + return .unknown(self.ctx); + } + + /// `tile`; `&.{}` for a scalar. + pub fn tileTy(self: *const Builder, shape: []const i64, dtype: DType) *const mlir.Type { + if (!dtypes.tileLegal(shape)) { + std.debug.panic("Builder.tileTy: {any} is not a legal tile shape (power-of-two dims, at most 2^24 elements)", .{shape}); + } + return ct.tileType(self.ctx, shape, dtype.toMlir(self.ctx)); + } + + pub fn scalarTy(self: *const Builder, dtype: DType) *const mlir.Type { + return self.tileTy(&.{}, dtype); + } + + /// `tile>` — the type of a buffer parameter. + pub fn ptrTy(self: *const Builder, dtype: DType) *const mlir.Type { + return self.ptrTileTy(&.{}, dtype); + } + + /// `tile>`. `i4` is a tile element type but not a pointee + /// (Types.td: CudaTile_NumberType excludes Int4), so a 4-bit buffer is + /// declared as `.i8` and `unpack`ed in the kernel. + pub fn ptrTileTy(self: *const Builder, shape: []const i64, dtype: DType) *const mlir.Type { + if (dtype == .i4) @panic("Builder: i4 is a tile element type only, never a pointee; declare the buffer as .i8 and unpack in-kernel"); + return ct.tileType(self.ctx, shape, ct.pointerType(self.ctx, dtype.toMlir(self.ctx))); + } + + pub fn tokenTy(self: *const Builder) *const mlir.Type { + return ct.tokenType(self.ctx); + } + + fn tileLike(self: *const Builder, v: Value, elem: *const mlir.Type) *const mlir.Type { + return ct.tileType(self.ctx, v.shape().constSlice(), elem); + } + + fn mlirElemToDType(self: *const Builder, elem: *const mlir.Type) DType { + return dtypes.mlirElemToDType(self.ctx, elem); + } + + fn pointee(v: Value) *const mlir.Type { + const p = v.elemType().isA(ct.PointerType) orelse std.debug.panic("expected a pointer tile, got {f}", .{v.type_()}); + return p.pointee(); + } + + fn elemAttr(self: *Builder, dtype: DType, value: anytype) *const mlir.Attribute { + const elem = dtype.toMlir(self.ctx); + const T = @TypeOf(value); + if (isFloatDtype(dtype)) { + const v: f64 = switch (@typeInfo(T)) { + .comptime_int, .int => @floatFromInt(value), + .comptime_float, .float => @floatCast(value), + else => @compileError("Builder: unsupported scalar " ++ @typeName(T)), + }; + return ct.floatElem(self.ctx, elem, v); + } + const v: i64 = switch (@typeInfo(T)) { + .comptime_int => blk: { + if (value >= std.math.minInt(i64) and value <= std.math.maxInt(i64)) + break :blk @intCast(value); + if (value >= 0 and value <= std.math.maxInt(u64)) + break :blk @bitCast(@as(u64, value)); + @compileError("Builder: integer literal out of 64-bit range"); + }, + .int => |info| if (info.signedness == .signed) @intCast(value) else @bitCast(@as(u64, @intCast(value))), + .bool => @intFromBool(value), + .comptime_float, .float => @intFromFloat(value), + else => @compileError("Builder: unsupported scalar " ++ @typeName(T)), + }; + return ct.intElem(elem, v); + } + + fn splat(self: *Builder, tile_ty: *const mlir.Type, dtype: DType, value: anytype) Value { + return self.emit(ct.constant(self.ctx, ct.splatAttr(tile_ty, self.elemAttr(dtype, value)), tile_ty, self.loc())); + } + + pub fn cst(self: *Builder, dtype: DType, value: anytype) Value { + return self.splat(self.scalarTy(dtype), dtype, value); + } + + pub fn full(self: *Builder, shape: []const i64, value: anytype, dtype: DType) Value { + return self.splat(self.tileTy(shape, dtype), dtype, value); + } + + pub fn zeros(self: *Builder, shape: []const i64, dtype: DType) Value { + return self.full(shape, 0, dtype); + } + + pub fn ones(self: *Builder, shape: []const i64, dtype: DType) Value { + return self.full(shape, 1, dtype); + } + + /// A dense constant from a slice of Zig scalars in row-major order. MLIR + /// stores every element in whole bytes except `i1`, which it bit-packs; + /// the slice must hold exactly that many bytes (use `full` for a splat). + pub fn dense(self: *Builder, shape: []const i64, dtype: DType, values: anytype) Value { + checkDenseBytes("Builder.dense", shape, dtype, values); + const ty = self.tileTy(shape, dtype); + return self.emit(ct.constant(self.ctx, ct.denseAttr(ty, values), ty, self.loc())); + } + + fn checkDenseBytes(comptime who: []const u8, shape: []const i64, dtype: DType, values: anytype) void { + var elems: usize = 1; + for (shape) |d| elems *= @intCast(d); + const bits = dtypeBitwidth(dtype); + const expected: usize = if (bits == 1) (elems + 7) / 8 else ((bits + 7) / 8) * elems; + const got = std.mem.sliceAsBytes(values).len; + if (got != expected) { + std.debug.panic("{s}: {d} bytes given for {any} x {s}, {d} expected", .{ who, got, shape, @tagName(dtype), expected }); + } + } + + pub fn iota(self: *Builder, n: i64, dtype: DType) Value { + return self.emit(ct.iota(self.ctx, self.tileTy(&.{n}, dtype), self.loc())); + } + + /// The element attribute `reduce`/`scan` need as an identity. + pub fn identityAttr(self: *Builder, dtype: DType, value: anytype) *const mlir.Attribute { + return self.elemAttr(dtype, value); + } + + /// Lift a Zig scalar to a rank-0 constant. Dtype follows the source type; + /// unsigned ints are bit-cast. Use `liftAs` for an explicit dtype. + pub fn lift(self: *Builder, value: anytype) Value { + const T = @TypeOf(value); + if (T == Value) return value; + return switch (@typeInfo(T)) { + .comptime_int => blk: { + if (value >= std.math.minInt(i32) and value <= std.math.maxInt(i32)) + break :blk self.cst(.i32, value); + break :blk self.cst(.i64, value); + }, + .int => |info| if (info.bits > 32) self.cst(.i64, value) else self.cst(.i32, value), + .bool => self.cst(.i1, value), + .comptime_float => self.cst(.f32, value), + .float => |info| switch (info.bits) { + 16 => self.cst(.f16, value), + 32 => self.cst(.f32, value), + 64 => self.cst(.f64, value), + else => @compileError("Builder.lift: unsupported float bitwidth"), + }, + else => @compileError("Builder.lift: unsupported type " ++ @typeName(T)), + }; + } + + pub fn liftAs(self: *Builder, value: anytype, dtype: DType) Value { + if (@TypeOf(value) == Value) return value; + return self.cst(dtype, value); + } + + fn liftMatching(self: *Builder, value: anytype, ref_elem: *const mlir.Type) Value { + const T = @TypeOf(value); + if (T == Value) return value; + return switch (@typeInfo(T)) { + .comptime_int, .comptime_float => self.liftAs(value, self.mlirElemToDType(ref_elem)), + else => self.lift(value), + }; + } + + fn constMatching(self: *Builder, value: anytype, ref: *const mlir.Type) Value { + const elem = if (ref.isA(ct.TileType)) |t| t.elementType() else ref; + return self.liftAs(value, self.mlirElemToDType(elem)); + } + + pub fn broadcastLike(self: *Builder, value: anytype, ref: Value) Value { + const v = self.liftMatching(value, ref.elemType()); + return self.expandTo(v, ref.shape().constSlice()); + } + + fn expandTo(self: *Builder, v: Value, target: []const i64) Value { + const cur = v.shape(); + if (std.mem.eql(i64, cur.constSlice(), target)) return v; + var aligned = v; + if (cur.len != target.len) { + std.debug.assert(cur.len <= target.len); + var padded: Value.Shape = .empty; + for (0..target.len - cur.len) |_| padded.appendAssumeCapacity(1); + padded.appendSliceAssumeCapacity(cur.constSlice()); + aligned = self.reshape(v, padded.constSlice()); + } + if (std.mem.eql(i64, aligned.shape().constSlice(), target)) return aligned; + return self.broadcastTo(aligned, target); + } + + fn coerceShapes(self: *Builder, a: Value, b: Value) struct { Value, Value } { + const a_sh = a.shape(); + const b_sh = b.shape(); + if (std.mem.eql(i64, a_sh.constSlice(), b_sh.constSlice())) return .{ a, b }; + const n = @max(a_sh.len, b_sh.len); + var target: Value.Shape = .empty; + for (0..n) |i| { + const ai: i64 = if (i + a_sh.len >= n) a_sh.get(i + a_sh.len - n) else 1; + const bi: i64 = if (i + b_sh.len >= n) b_sh.get(i + b_sh.len - n) else 1; + if (ai == bi or bi == 1) { + target.appendAssumeCapacity(ai); + } else if (ai == 1) { + target.appendAssumeCapacity(bi); + } else { + std.debug.panic("Builder.coerce: incompatible shapes {any} vs {any} at axis {d}", .{ a_sh.constSlice(), b_sh.constSlice(), i }); + } + } + return .{ self.expandTo(a, target.constSlice()), self.expandTo(b, target.constSlice()) }; + } + + pub fn coerce(self: *Builder, a: anytype, b: anytype) struct { Value, Value } { + const a_ref: ?*const mlir.Type = if (@TypeOf(b) == Value) b.elemType() else null; + const b_ref: ?*const mlir.Type = if (@TypeOf(a) == Value) a.elemType() else null; + var av = if (a_ref) |t| self.liftMatching(a, t) else self.lift(a); + var bv = if (b_ref) |t| self.liftMatching(b, t) else self.lift(b); + + if (av.isIntElem() and bv.isIntElem()) { + const a_dt = self.mlirElemToDType(av.elemType()); + const b_dt = self.mlirElemToDType(bv.elemType()); + const aw = intBitwidth(a_dt); + const bw = intBitwidth(b_dt); + if (aw < bw) av = av.to(b_dt); + if (bw < aw) bv = bv.to(a_dt); + } + + return self.coerceShapes(av, bv); + } + + pub fn tileBlockId(self: *Builder) BlockId { + const r = self.emitMulti(ct.get_tile_block_id(self.ctx, self.loc()), 3); + return .{ .x = r[0], .y = r[1], .z = r[2] }; + } + + pub fn numTileBlocks(self: *Builder) BlockId { + const r = self.emitMulti(ct.get_num_tile_blocks(self.ctx, self.loc()), 3); + return .{ .x = r[0], .y = r[1], .z = r[2] }; + } + + pub fn assumeDivBy(self: *Builder, v: Value, n: u64) Value { + return self.emit(ct.assume(self.ctx, v.inner, ct.divBy(self.ctx, n, null, null), self.loc())); + } + + pub fn assumeDivByEvery(self: *Builder, v: Value, n: u64, every: i64, along: i64) Value { + return self.emit(ct.assume(self.ctx, v.inner, ct.divBy(self.ctx, n, every, along), self.loc())); + } + + pub fn assumeSameElements(self: *Builder, v: Value, values: []const i64) Value { + return self.emit(ct.assume(self.ctx, v.inner, ct.sameElements(self.ctx, values), self.loc())); + } + + pub fn assumeBounded(self: *Builder, v: Value, lb: ?i64, ub: ?i64) Value { + return self.emit(ct.assume(self.ctx, v.inner, ct.bounded(self.ctx, lb, ub), self.loc())); + } + + pub fn tensorView(self: *Builder, base: Value, shape: []const i64, strides: []const i64) Value { + const ty = ct.tensorViewType(self.ctx, pointee(base), shape, strides); + return self.emit(ct.make_tensor_view(self.ctx, base.inner, &.{}, &.{}, ty, self.loc())); + } + + pub const Dim = union(enum) { + static: i64, + dynamic: Value, + }; + + pub fn tensorViewDyn(self: *Builder, base: Value, shape: []const Dim, strides: []const Dim) Value { + const scratch = self.arena.allocator(); + var static_shape: Value.Shape = .empty; + var static_strides: Value.Shape = .empty; + var dyn_shape = std.ArrayList(*const mlir.Value).empty; + var dyn_strides = std.ArrayList(*const mlir.Value).empty; + for (shape) |d| switch (d) { + .static => |s| static_shape.appendAssumeCapacity(s), + .dynamic => |v| { + static_shape.appendAssumeCapacity(ct.TensorViewType.dynamic()); + dyn_shape.append(scratch, v.inner) catch @panic("OOM"); + }, + }; + for (strides) |d| switch (d) { + .static => |s| static_strides.appendAssumeCapacity(s), + .dynamic => |v| { + static_strides.appendAssumeCapacity(ct.TensorViewType.dynamic()); + dyn_strides.append(scratch, v.inner) catch @panic("OOM"); + }, + }; + const ty = ct.tensorViewType(self.ctx, pointee(base), static_shape.constSlice(), static_strides.constSlice()); + return self.emit(ct.make_tensor_view(self.ctx, base.inner, dyn_shape.items, dyn_strides.items, ty, self.loc())); + } + + fn i32Slice(self: *Builder, values: []const i64) []const i32 { + const out = self.arena.allocator().alloc(i32, values.len) catch @panic("OOM"); + for (values, 0..) |v, i| out[i] = @intCast(v); + return out; + } + + /// Tiles `tv` into `tile`-shaped pieces addressed by tile-space indices. + /// `padding` fills the ragged edge on load; stores past the edge are + /// masked. That is why this DSL has no mask tile. + pub fn partitionView(self: *Builder, tv: Value, tile: []const i64, opts: PartitionOpts) Value { + if (!dtypes.tileLegal(tile)) std.debug.panic("Builder.partitionView: {any} is not a legal tile shape", .{tile}); + const ty: *const mlir.Type = @ptrCast(ct.PartitionViewType.get(self.ctx, self.i32Slice(tile), tv.type_(), opts.dim_map, opts.padding)); + return self.emit(ct.make_partition_view(self.ctx, tv.inner, ty, self.loc())); + } + + /// 13.3: like `partitionView`, stepping by `traversal_strides` elements + /// per index instead of a whole tile. + pub fn stridedView(self: *Builder, tv: Value, tile: []const i64, traversal_strides: []const i64, opts: PartitionOpts) Value { + if (!dtypes.tileLegal(tile)) std.debug.panic("Builder.stridedView: {any} is not a legal tile shape", .{tile}); + const ty: *const mlir.Type = @ptrCast(ct.StridedViewType.get(self.ctx, self.i32Slice(tile), self.i32Slice(traversal_strides), tv.type_(), opts.dim_map, opts.padding)); + return self.emit(ct.make_strided_view(self.ctx, tv.inner, ty, self.loc())); + } + + /// 13.3: a view whose `sparse_dim` is indexed by a rank-1 tile of + /// `tile[sparse_dim]` indices (every other index stays rank 0). + pub fn gatherScatterView(self: *Builder, tv: Value, tile: []const i64, sparse_dim: u32, padding: ?PaddingValue) Value { + if (!dtypes.tileLegal(tile)) std.debug.panic("Builder.gatherScatterView: {any} is not a legal tile shape", .{tile}); + const ty = ct.gatherScatterViewType(self.ctx, tile, tv.type_(), sparse_dim, padding); + return self.emit(ct.make_gather_scatter_view(self.ctx, tv.inner, ty, self.loc())); + } + + fn i32Scalars(self: *Builder, n: usize) []const *const mlir.Type { + const out = self.arena.allocator().alloc(*const mlir.Type, n) catch @panic("OOM"); + for (out) |*t| t.* = self.scalarTy(.i32); + return out; + } + + pub fn indexSpaceShape(self: *Builder, view: Value) []Value { + const n = ct.indexRankOfView(view.type_()); + return self.emitMulti(ct.get_index_space_shape(self.ctx, view.inner, self.i32Scalars(n), self.loc()), n); + } + + pub fn tensorShape(self: *Builder, tv: Value) []Value { + const t = tv.type_().isA(ct.TensorViewType) orelse std.debug.panic("expected a tensor_view, got {f}", .{tv.type_()}); + const n = t.rank(); + return self.emitMulti(ct.get_tensor_shape(self.ctx, tv.inner, self.i32Scalars(n), self.loc()), n); + } + + fn memOpts(self: *Builder, ordering: MemoryOrdering, scope: ?MemoryScope, token: ?Value, hints: anytype) ct.MemOpts { + return .{ + .ordering = ordering, + .scope = scope, + .token = innerOpt(token), + .optimization_hints = if (hints.len > 0) ct.optimizationHints(self.ctx, hints) else null, + }; + } + + pub fn load(self: *Builder, view: Value, index: []const Value) Value { + return self.loadOpts(view, index, .{}).tile; + } + + pub fn loadOpts(self: *Builder, view: Value, index: []const Value, opts: LoadOpts) Loaded { + const expected = ct.indexRankOfView(view.type_()); + if (index.len != expected) std.debug.panic("Builder.load: view takes {d} indices, got {d}", .{ expected, index.len }); + const r = self.emitMulti(ct.load_view_tko(self.ctx, view.inner, self.innerSlice(index), self.memOpts(opts.ordering, opts.scope, opts.token, opts.hints), self.loc()), 2); + return .{ .tile = r[0], .token = r[1] }; + } + + pub fn store(self: *Builder, tile: Value, view: Value, index: []const Value) Value { + return self.storeOpts(tile, view, index, .{}); + } + + pub fn storeOpts(self: *Builder, tile: Value, view: Value, index: []const Value, opts: StoreOpts) Value { + const expected = ct.indexRankOfView(view.type_()); + if (index.len != expected) std.debug.panic("Builder.store: view takes {d} indices, got {d}", .{ expected, index.len }); + return self.emit(ct.store_view_tko(self.ctx, tile.inner, view.inner, self.innerSlice(index), self.memOpts(opts.ordering, opts.scope, opts.token, opts.hints), self.loc())); + } + + /// Gather through a tile of pointers, dropping the token. This arm cannot + /// use TMA; prefer views for dense access. + pub fn loadPtr(self: *Builder, ptrs: Value) Value { + return self.loadPtrOpts(ptrs, .{}).tile; + } + + pub fn loadPtrOpts(self: *Builder, ptrs: Value, opts: LoadPtrOpts) Loaded { + const result_ty = self.tileLike(ptrs, pointee(ptrs)); + const mask: ?Value = if (opts.mask) |m| self.expandTo(m, ptrs.shape().constSlice()) else null; + const r = self.emitMulti(ct.load_ptr_tko(self.ctx, ptrs.inner, result_ty, .{ + .mask = innerOpt(mask), + .padding = innerOpt(opts.padding), + .mem = self.memOpts(opts.ordering, opts.scope, opts.token, opts.hints), + }, self.loc()), 2); + return .{ .tile = r[0], .token = r[1] }; + } + + pub fn storePtr(self: *Builder, ptrs: Value, value: Value) Value { + return self.storePtrOpts(ptrs, value, .{}); + } + + pub fn storePtrOpts(self: *Builder, ptrs: Value, value: Value, opts: StorePtrOpts) Value { + const mask: ?Value = if (opts.mask) |m| self.expandTo(m, ptrs.shape().constSlice()) else null; + return self.emit(ct.store_ptr_tko(self.ctx, ptrs.inner, value.inner, .{ + .mask = innerOpt(mask), + .mem = self.memOpts(opts.ordering, opts.scope, opts.token, opts.hints), + }, self.loc())); + } + + pub fn makeToken(self: *Builder) Value { + return self.emit(ct.make_token(self.ctx, self.loc())); + } + + pub fn joinTokens(self: *Builder, tokens: []const Value) Value { + return self.emit(ct.join_tokens(self.ctx, self.innerSlice(tokens), self.loc())); + } + + pub fn print(self: *Builder, str: []const u8, args: []const Value) Value { + return self.printOpts(str, args, null); + } + + pub fn printOpts(self: *Builder, str: []const u8, args: []const Value, token: ?Value) Value { + return self.emit(ct.print_tko(self.ctx, str, self.innerSlice(args), innerOpt(token), self.loc())); + } + + fn atomicOpts(opts: AtomicOpts) ct.AtomicOpts { + return .{ .ordering = opts.ordering, .scope = opts.scope, .mask = innerOpt(opts.mask), .token = innerOpt(opts.token) }; + } + + pub fn atomicCas(self: *Builder, ptrs: Value, cmp: Value, val: Value, opts: AtomicOpts) Loaded { + const r = self.emitMulti(ct.atomic_cas_tko(self.ctx, ptrs.inner, cmp.inner, val.inner, atomicOpts(opts), self.loc()), 2); + return .{ .tile = r[0], .token = r[1] }; + } + + pub fn atomicRmw(self: *Builder, mode: AtomicRMWMode, ptrs: Value, operand: Value, opts: AtomicOpts) Loaded { + const r = self.emitMulti(ct.atomic_rmw_tko(self.ctx, ptrs.inner, mode, operand.inner, atomicOpts(opts), self.loc()), 2); + return .{ .tile = r[0], .token = r[1] }; + } + + /// 13.3: an atomic reduction into an unpadded tiled view; returns the + /// token. `xchg` is not a reduction and the dialect rejects it. + pub fn atomicRedView(self: *Builder, mode: AtomicRMWMode, view: Value, index: []const Value, value: Value, opts: RedViewOpts) Value { + return self.emit(ct.atomic_red_view_tko(self.ctx, view.inner, self.innerSlice(index), mode, value.inner, .{ + .scope = opts.scope, + .token = innerOpt(opts.token), + }, self.loc())); + } + + /// 13.3: `num_elem` elements of `dtype`, as a rank-0 pointer tile. + pub fn alloca(self: *Builder, dtype: DType, num_elem: i64, opts: AllocaOpts) Value { + return self.emit(ct.alloca(self.ctx, num_elem, opts.alignment, opts.global, self.ptrTy(dtype), self.loc())); + } + + pub fn global(self: *Builder, name: []const u8, shape: []const i64, dtype: DType, value: anytype, opts: GlobalOpts) void { + const ty = self.tileTy(shape, dtype); + const attr = ct.splatAttr(ty, self.elemAttr(dtype, value)); + self.appendGlobal(ct.global(self.ctx, name, attr, .{ .alignment = opts.alignment, .constant = opts.constant, .visibility = opts.visibility }, self.loc())); + } + + /// A module-level dense buffer from a slice of Zig scalars (see `dense` + /// for the byte-count rule). + pub fn globalDense(self: *Builder, name: []const u8, shape: []const i64, dtype: DType, values: anytype, opts: GlobalOpts) void { + checkDenseBytes("Builder.globalDense", shape, dtype, values); + const ty = self.tileTy(shape, dtype); + self.appendGlobal(ct.global(self.ctx, name, ct.denseAttr(ty, values), .{ .alignment = opts.alignment, .constant = opts.constant, .visibility = opts.visibility }, self.loc())); + } + + fn appendGlobal(self: *Builder, op: *mlir.Operation) void { + const body = self.ct_body orelse @panic("Builder.global called before declareArgs"); + const first = body.firstOperation() orelse { + body.appendOwnedOperation(op); + return; + }; + body.insertOwnedOperationBefore(first, op); + } + + pub fn getGlobal(self: *Builder, name: []const u8, dtype: DType) Value { + return self.emit(ct.get_global(self.ctx, name, self.ptrTy(dtype), self.loc())); + } + + pub fn addf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.addfOpts(lhs, rhs, .{}); + } + pub fn addfOpts(self: *Builder, lhs: Value, rhs: Value, opts: FloatOpts) Value { + return self.emit(ct.addf(self.ctx, lhs.inner, rhs.inner, opts.rounding, opts.flush_to_zero, self.loc())); + } + pub fn subf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.subfOpts(lhs, rhs, .{}); + } + pub fn subfOpts(self: *Builder, lhs: Value, rhs: Value, opts: FloatOpts) Value { + return self.emit(ct.subf(self.ctx, lhs.inner, rhs.inner, opts.rounding, opts.flush_to_zero, self.loc())); + } + pub fn mulf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.mulfOpts(lhs, rhs, .{}); + } + pub fn mulfOpts(self: *Builder, lhs: Value, rhs: Value, opts: FloatOpts) Value { + return self.emit(ct.mulf(self.ctx, lhs.inner, rhs.inner, opts.rounding, opts.flush_to_zero, self.loc())); + } + pub fn divf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.divfOpts(lhs, rhs, .{}); + } + pub fn divfOpts(self: *Builder, lhs: Value, rhs: Value, opts: FloatOpts) Value { + return self.emit(ct.divf(self.ctx, lhs.inner, rhs.inner, opts.rounding, opts.flush_to_zero, self.loc())); + } + pub fn remf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(ct.remf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn negf(self: *Builder, x: Value) Value { + return self.emit(ct.negf(self.ctx, x.inner, self.loc())); + } + pub fn absf(self: *Builder, x: Value) Value { + return self.emit(ct.absf(self.ctx, x.inner, self.loc())); + } + pub fn fma(self: *Builder, a: Value, b: Value, acc: Value) Value { + return self.fmaOpts(a, b, acc, .{}); + } + pub fn fmaOpts(self: *Builder, a: Value, b: Value, acc: Value, opts: FloatOpts) Value { + return self.emit(ct.fma(self.ctx, a.inner, b.inner, acc.inner, opts.rounding, opts.flush_to_zero, self.loc())); + } + pub fn maxf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.maxfOpts(lhs, rhs, .{}); + } + pub fn maxfOpts(self: *Builder, lhs: Value, rhs: Value, opts: MinMaxOpts) Value { + return self.emit(ct.maxf(self.ctx, lhs.inner, rhs.inner, opts.propagate_nan, opts.flush_to_zero, self.loc())); + } + pub fn minf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.minfOpts(lhs, rhs, .{}); + } + pub fn minfOpts(self: *Builder, lhs: Value, rhs: Value, opts: MinMaxOpts) Value { + return self.emit(ct.minf(self.ctx, lhs.inner, rhs.inner, opts.propagate_nan, opts.flush_to_zero, self.loc())); + } + pub fn pow(self: *Builder, x: Value, exponent: Value) Value { + return self.emit(ct.pow(self.ctx, x.inner, exponent.inner, self.loc())); + } + pub fn atan2(self: *Builder, x: Value, y: Value) Value { + return self.emit(ct.atan2(self.ctx, x.inner, y.inner, self.loc())); + } + pub fn exp(self: *Builder, x: Value) Value { + return self.emit(ct.exp(self.ctx, x.inner, null, self.loc())); + } + /// 13.3: `exp` with an explicit rounding mode. + pub fn expRound(self: *Builder, x: Value, rounding: RoundingMode) Value { + return self.emit(ct.exp(self.ctx, x.inner, rounding, self.loc())); + } + pub fn exp2(self: *Builder, x: Value) Value { + return self.exp2Opts(x, false); + } + pub fn exp2Opts(self: *Builder, x: Value, flush_to_zero: bool) Value { + return self.emit(ct.exp2(self.ctx, x.inner, flush_to_zero, self.loc())); + } + pub fn log(self: *Builder, x: Value) Value { + return self.emit(ct.log(self.ctx, x.inner, self.loc())); + } + pub fn log2(self: *Builder, x: Value) Value { + return self.emit(ct.log2(self.ctx, x.inner, self.loc())); + } + pub fn sqrt(self: *Builder, x: Value) Value { + return self.sqrtOpts(x, .{}); + } + pub fn sqrtOpts(self: *Builder, x: Value, opts: FloatOpts) Value { + return self.emit(ct.sqrt(self.ctx, x.inner, opts.rounding, opts.flush_to_zero, self.loc())); + } + pub fn rsqrt(self: *Builder, x: Value) Value { + return self.rsqrtOpts(x, false); + } + pub fn rsqrtOpts(self: *Builder, x: Value, flush_to_zero: bool) Value { + return self.emit(ct.rsqrt(self.ctx, x.inner, flush_to_zero, self.loc())); + } + pub fn sin(self: *Builder, x: Value) Value { + return self.emit(ct.sin(self.ctx, x.inner, self.loc())); + } + pub fn cos(self: *Builder, x: Value) Value { + return self.emit(ct.cos(self.ctx, x.inner, self.loc())); + } + pub fn tan(self: *Builder, x: Value) Value { + return self.emit(ct.tan(self.ctx, x.inner, self.loc())); + } + pub fn sinh(self: *Builder, x: Value) Value { + return self.emit(ct.sinh(self.ctx, x.inner, self.loc())); + } + pub fn cosh(self: *Builder, x: Value) Value { + return self.emit(ct.cosh(self.ctx, x.inner, self.loc())); + } + pub fn tanh(self: *Builder, x: Value) Value { + return self.emit(ct.tanh(self.ctx, x.inner, null, self.loc())); + } + /// 13.2: `tanh` with an explicit rounding mode. + pub fn tanhRound(self: *Builder, x: Value, rounding: RoundingMode) Value { + return self.emit(ct.tanh(self.ctx, x.inner, rounding, self.loc())); + } + pub fn ceil(self: *Builder, x: Value) Value { + return self.emit(ct.ceil(self.ctx, x.inner, self.loc())); + } + pub fn floor(self: *Builder, x: Value) Value { + return self.emit(ct.floor(self.ctx, x.inner, self.loc())); + } + + pub fn cmpf(self: *Builder, predicate: ComparisonPredicate, lhs: Value, rhs: Value) Value { + return self.cmpfOpts(predicate, lhs, rhs, .ordered); + } + pub fn cmpfOpts(self: *Builder, predicate: ComparisonPredicate, lhs: Value, rhs: Value, ordering: ComparisonOrdering) Value { + return self.emit(ct.cmpf(self.ctx, predicate, ordering, lhs.inner, rhs.inner, self.loc())); + } + + pub fn mmaf(self: *Builder, a: Value, b: Value, acc: Value) Value { + return self.mmafOpts(a, b, acc, .{}); + } + pub fn mmafOpts(self: *Builder, a: Value, b: Value, acc: Value, opts: MmaOpts) Value { + return self.emit(ct.mmaf(self.ctx, a.inner, b.inner, acc.inner, opts.fast_acc, self.loc())); + } + /// 13.3: block-scaled MMA. Scales are plain tiles in logical layout — + /// `a_scale` is `[M, K/V]`, `b_scale` is `[K/V, N]` — never pre-swizzled. + pub fn mmafScaled(self: *Builder, a: Value, b: Value, acc: Value, a_scale: Value, b_scale: Value) Value { + return self.emit(ct.mmaf_scaled(self.ctx, a.inner, b.inner, acc.inner, a_scale.inner, b_scale.inner, self.loc())); + } + pub fn mmai(self: *Builder, a: Value, b: Value, acc: Value, a_signedness: Signedness, b_signedness: Signedness) Value { + return self.emit(ct.mmai(self.ctx, a.inner, b.inner, acc.inner, a_signedness, b_signedness, self.loc())); + } + + pub fn addi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.addiOpts(lhs, rhs, .{}); + } + pub fn addiOpts(self: *Builder, lhs: Value, rhs: Value, opts: IntOpts) Value { + return self.emit(ct.addi(self.ctx, lhs.inner, rhs.inner, opts.overflow, self.loc())); + } + pub fn subi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.subiOpts(lhs, rhs, .{}); + } + pub fn subiOpts(self: *Builder, lhs: Value, rhs: Value, opts: IntOpts) Value { + return self.emit(ct.subi(self.ctx, lhs.inner, rhs.inner, opts.overflow, self.loc())); + } + pub fn muli(self: *Builder, lhs: Value, rhs: Value) Value { + return self.muliOpts(lhs, rhs, .{}); + } + pub fn muliOpts(self: *Builder, lhs: Value, rhs: Value, opts: IntOpts) Value { + return self.emit(ct.muli(self.ctx, lhs.inner, rhs.inner, opts.overflow, self.loc())); + } + pub fn mulhii(self: *Builder, x: Value, y: Value) Value { + return self.emit(ct.mulhii(self.ctx, x.inner, y.inner, self.loc())); + } + pub fn divi(self: *Builder, lhs: Value, rhs: Value, signedness: Signedness) Value { + return self.diviOpts(lhs, rhs, .{ .signedness = signedness }); + } + pub fn diviOpts(self: *Builder, lhs: Value, rhs: Value, opts: DivIOpts) Value { + return self.emit(ct.divi(self.ctx, lhs.inner, rhs.inner, opts.signedness, opts.rounding, self.loc())); + } + pub fn remi(self: *Builder, lhs: Value, rhs: Value, signedness: Signedness) Value { + return self.emit(ct.remi(self.ctx, lhs.inner, rhs.inner, signedness, self.loc())); + } + pub fn maxi(self: *Builder, lhs: Value, rhs: Value, signedness: Signedness) Value { + return self.emit(ct.maxi(self.ctx, lhs.inner, rhs.inner, signedness, self.loc())); + } + pub fn mini(self: *Builder, lhs: Value, rhs: Value, signedness: Signedness) Value { + return self.emit(ct.mini(self.ctx, lhs.inner, rhs.inner, signedness, self.loc())); + } + pub fn negi(self: *Builder, x: Value) Value { + return self.emit(ct.negi(self.ctx, x.inner, .none, self.loc())); + } + pub fn absi(self: *Builder, x: Value) Value { + return self.emit(ct.absi(self.ctx, x.inner, self.loc())); + } + pub fn shli(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(ct.shli(self.ctx, lhs.inner, rhs.inner, .none, self.loc())); + } + pub fn shri(self: *Builder, lhs: Value, rhs: Value, signedness: Signedness) Value { + return self.emit(ct.shri(self.ctx, lhs.inner, rhs.inner, signedness, self.loc())); + } + pub fn andi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(ct.andi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn ori(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(ct.ori(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn xori(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(ct.xori(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn cmpi(self: *Builder, predicate: ComparisonPredicate, lhs: Value, rhs: Value, signedness: Signedness) Value { + return self.emit(ct.cmpi(self.ctx, predicate, signedness, lhs.inner, rhs.inner, self.loc())); + } + + pub fn maximum(self: *Builder, a: anytype, b: anytype) Value { + const l, const r = self.coerce(a, b); + return if (l.isFloatElem()) self.maxf(l, r) else self.maxi(l, r, .signed); + } + + pub fn minimum(self: *Builder, a: anytype, b: anytype) Value { + const l, const r = self.coerce(a, b); + return if (l.isFloatElem()) self.minf(l, r) else self.mini(l, r, .signed); + } + + pub fn select(self: *Builder, cond: Value, t: Value, f: Value) Value { + return self.emit(ct.select(self.ctx, cond.inner, t.inner, f.inner, self.loc())); + } + + pub fn where(self: *Builder, cond: Value, x: anytype, y: anytype) Value { + var xv, var yv = self.coerce(x, y); + var cv = cond; + cv, xv = self.coerceShapes(cv, xv); + xv, yv = self.coerceShapes(xv, yv); + cv, _ = self.coerceShapes(cv, xv); + return self.select(cv, xv, yv); + } + + pub fn bitcast(self: *Builder, src: Value, dtype: DType) Value { + return self.emit(ct.bitcast(self.ctx, src.inner, self.tileLike(src, dtype.toMlir(self.ctx)), self.loc())); + } + pub fn exti(self: *Builder, src: Value, dtype: DType, signedness: Signedness) Value { + return self.emit(ct.exti(self.ctx, src.inner, signedness, self.tileLike(src, dtype.toMlir(self.ctx)), self.loc())); + } + pub fn trunci(self: *Builder, src: Value, dtype: DType) Value { + return self.emit(ct.trunci(self.ctx, src.inner, self.tileLike(src, dtype.toMlir(self.ctx)), .none, self.loc())); + } + pub fn ftof(self: *Builder, src: Value, dtype: DType) Value { + return self.ftofOpts(src, dtype, .{}); + } + pub fn ftofOpts(self: *Builder, src: Value, dtype: DType, opts: CastOpts) Value { + const rounding: RoundingMode = opts.rounding orelse (if (dtype == .f8e8m0fnu) .zero else .nearest_even); + return self.emit(ct.ftof(self.ctx, src.inner, self.tileLike(src, dtype.toMlir(self.ctx)), rounding, self.loc())); + } + /// Float -> integer, truncating toward zero (the only rounding `ftoi` + /// admits). + pub fn ftoi(self: *Builder, src: Value, dtype: DType) Value { + return self.ftoiOpts(src, dtype, .{}); + } + pub fn ftoiOpts(self: *Builder, src: Value, dtype: DType, opts: FToIOpts) Value { + return self.emit(ct.ftoi(self.ctx, src.inner, opts.signedness, .nearest_int_to_zero, self.tileLike(src, dtype.toMlir(self.ctx)), self.loc())); + } + /// Integer -> float. `f8e8m0fnu` is not a legal target; go through `f32`. + pub fn itof(self: *Builder, src: Value, dtype: DType) Value { + return self.itofOpts(src, dtype, .{}); + } + pub fn itofOpts(self: *Builder, src: Value, dtype: DType, opts: CastOpts) Value { + if (dtype == .f8e8m0fnu) @panic("Builder.itof: the dialect refuses integer -> f8e8m0fnu; convert to .f32 first (Builder.cast does)"); + return self.emit(ct.itof(self.ctx, src.inner, opts.signedness, opts.rounding orelse .nearest_even, self.tileLike(src, dtype.toMlir(self.ctx)), self.loc())); + } + pub fn intToPtr(self: *Builder, src: Value, pointee_dtype: DType) Value { + return self.emit(ct.int_to_ptr(self.ctx, src.inner, self.ptrTileTy(src.shape().constSlice(), pointee_dtype), self.loc())); + } + pub fn ptrToInt(self: *Builder, src: Value) Value { + return self.emit(ct.ptr_to_int(self.ctx, src.inner, self.tileLike(src, DType.i64.toMlir(self.ctx)), self.loc())); + } + pub fn ptrToPtr(self: *Builder, src: Value, pointee_dtype: DType) Value { + return self.emit(ct.ptr_to_ptr(self.ctx, src.inner, self.ptrTileTy(src.shape().constSlice(), pointee_dtype), self.loc())); + } + /// 13.3: a sub-byte tile packed into `tile`. + pub fn pack(self: *Builder, src: Value, shape: []const i64) Value { + return self.emit(ct.pack(self.ctx, src.inner, self.tileTy(shape, .i8), self.loc())); + } + /// 13.3: an i8 tile unpacked into `tile` of a sub-byte `dtype`. + pub fn unpack(self: *Builder, src: Value, shape: []const i64, dtype: DType) Value { + return self.emit(ct.unpack(self.ctx, src.inner, self.tileTy(shape, dtype), self.loc())); + } + + /// Numeric cast with auto-dispatch: ftof / ftoi / itof / exti / trunci / + /// bitcast by element kind and width. Integers are signed; i1 is not. + pub fn cast(self: *Builder, src: Value, dtype: DType) Value { + const cur_elem = src.elemType(); + const tgt_elem = dtype.toMlir(self.ctx); + if (cur_elem.eql(tgt_elem)) return src; + const cur_dtype = self.mlirElemToDType(cur_elem); + const cur_is_float = src.isFloatElem(); + const tgt_is_float = isFloatDtype(dtype); + if (cur_is_float and tgt_is_float) return self.ftof(src, dtype); + if (cur_is_float) return self.ftoi(src, dtype); + if (tgt_is_float) { + const signedness: Signedness = if (cur_dtype == .i1) .unsigned else .signed; + if (dtype == .f8e8m0fnu) return self.ftof(self.itofOpts(src, .f32, .{ .signedness = signedness }), dtype); + return self.itofOpts(src, dtype, .{ .signedness = signedness }); + } + const cur_bw = dtypeBitwidth(cur_dtype); + const tgt_bw = dtypeBitwidth(dtype); + if (tgt_bw > cur_bw) return self.exti(src, dtype, if (cur_dtype == .i1) .unsigned else .signed); + if (tgt_bw < cur_bw) return self.trunci(src, dtype); + return self.bitcast(src, dtype); + } + + pub fn reshape(self: *Builder, src: Value, shape: []const i64) Value { + return self.emit(ct.reshape(self.ctx, src.inner, self.tileTyOf(shape, src.elemType()), self.loc())); + } + + fn tileTyOf(self: *const Builder, shape: []const i64, elem: *const mlir.Type) *const mlir.Type { + if (!dtypes.tileLegal(shape)) std.debug.panic("Builder: {any} is not a legal tile shape", .{shape}); + return ct.tileType(self.ctx, shape, elem); + } + + pub fn broadcastTo(self: *Builder, src: Value, shape: []const i64) Value { + return self.emit(ct.broadcast(self.ctx, src.inner, self.tileTyOf(shape, src.elemType()), self.loc())); + } + + pub fn expandDims(self: *Builder, src: Value, axis: usize) Value { + const in_shape = src.shape(); + var out: Value.Shape = .empty; + for (0..in_shape.len + 1) |i| { + if (i == axis) out.appendAssumeCapacity(1) else out.appendAssumeCapacity(in_shape.get(if (i < axis) i else i - 1)); + } + return self.reshape(src, out.constSlice()); + } + + pub fn cat(self: *Builder, lhs: Value, rhs: Value, dim: usize) Value { + var out = lhs.shape(); + out.set(dim, out.get(dim) + rhs.dim(dim)); + return self.emit(ct.cat(self.ctx, lhs.inner, rhs.inner, @intCast(dim), self.tileTyOf(out.constSlice(), lhs.elemType()), self.loc())); + } + + pub fn extract(self: *Builder, src: Value, indices: []const Value, shape: []const i64) Value { + return self.emit(ct.extract(self.ctx, src.inner, self.innerSlice(indices), self.tileTyOf(shape, src.elemType()), self.loc())); + } + + pub fn permute(self: *Builder, src: Value, order: []const i32) Value { + const src_shape = src.shape(); + std.debug.assert(order.len == src_shape.len); + var out: Value.Shape = .empty; + for (order) |i| out.appendAssumeCapacity(src_shape.get(@intCast(i))); + return self.emit(ct.permute(self.ctx, src.inner, order, self.tileTyOf(out.constSlice(), src.elemType()), self.loc())); + } + + pub fn transpose(self: *Builder, src: Value) Value { + std.debug.assert(src.rank() == 2); + return self.permute(src, &.{ 1, 0 }); + } + + pub fn offset(self: *Builder, ptrs: Value, off: Value) Value { + return self.emit(ct.offset(self.ctx, ptrs.inner, off.inner, self.loc())); + } + + fn reducedShape(src: Value, axis: usize) Value.Shape { + if (axis >= src.rank()) std.debug.panic("Builder.reduce: axis {d} out of range for rank {d}", .{ axis, src.rank() }); + var out: Value.Shape = .empty; + const s = src.shape(); + for (0..s.len) |i| if (i != axis) out.appendAssumeCapacity(s.get(i)); + return out; + } + + fn checkAxis(src: Value, axis: usize) void { + if (axis >= src.rank()) std.debug.panic("Builder.scan: axis {d} out of range for rank {d}", .{ axis, src.rank() }); + } + + fn combineBody(self: *Builder, ctx: anytype, elem: *const mlir.Type, combine: anytype) *mlir.Block { + const scalar = ct.tileType(self.ctx, &.{}, elem); + const body = mlir.Block.init(&.{ scalar, scalar }, &.{ self.loc(), self.loc() }); + self.pushBlock(body); + const element: Value = .{ .inner = body.argument(0), .kernel = self }; + const acc: Value = .{ .inner = body.argument(1), .kernel = self }; + const combined: Value = combine(self, element, acc, ctx); + _ = ct.yield(self.ctx, &.{combined.inner}, self.loc()).appendTo(body); + self.popBlock(); + return body; + } + + pub fn reduce(self: *Builder, ctx: anytype, args: ReduceArgs(@TypeOf(ctx))) Value { + const result_ty = self.tileTyOf(reducedShape(args.src, args.axis).constSlice(), args.src.elemType()); + const body = self.combineBody(ctx, args.src.elemType(), args.combine); + return self.emit(ct.reduce(self.ctx, &.{args.src.inner}, @intCast(args.axis), &.{args.identity}, body, &.{result_ty}, self.loc())); + } + + pub fn scan(self: *Builder, ctx: anytype, args: ScanArgs(@TypeOf(ctx))) Value { + checkAxis(args.src, args.axis); + const body = self.combineBody(ctx, args.src.elemType(), args.combine); + return self.emit(ct.scan(self.ctx, &.{args.src.inner}, @intCast(args.axis), args.reverse, &.{args.identity}, body, &.{args.src.type_()}, self.loc())); + } + + /// The `2N`-argument body of a variadic `reduce`/`scan`: arguments come in + /// (element, accumulator) pairs, one pair per source. + fn combineBodyMulti(self: *Builder, ctx: anytype, srcs: []const Value, combine: anytype) *mlir.Block { + const scratch = self.arena.allocator(); + const n = srcs.len; + const types = scratch.alloc(*const mlir.Type, 2 * n) catch @panic("OOM"); + const locs = scratch.alloc(*const mlir.Location, 2 * n) catch @panic("OOM"); + for (srcs, 0..) |s, i| { + const scalar = ct.tileType(self.ctx, &.{}, s.elemType()); + types[2 * i] = scalar; + types[2 * i + 1] = scalar; + locs[2 * i] = self.loc(); + locs[2 * i + 1] = self.loc(); + } + const body = mlir.Block.init(types, locs); + self.pushBlock(body); + const elements = scratch.alloc(Value, n) catch @panic("OOM"); + const accs = scratch.alloc(Value, n) catch @panic("OOM"); + for (0..n) |i| { + elements[i] = .{ .inner = body.argument(2 * i), .kernel = self }; + accs[i] = .{ .inner = body.argument(2 * i + 1), .kernel = self }; + } + const out: []const Value = combine(self, elements, accs, ctx); + if (out.len != n) std.debug.panic("Builder.reduceMulti: combine returned {d} values for {d} sources", .{ out.len, n }); + _ = ct.yield(self.ctx, self.innerSlice(out), self.loc()).appendTo(body); + self.popBlock(); + return body; + } + + pub fn reduceMulti(self: *Builder, ctx: anytype, args: ReduceMultiArgs(@TypeOf(ctx))) []Value { + std.debug.assert(args.srcs.len == args.identities.len); + const result_types = self.arena.allocator().alloc(*const mlir.Type, args.srcs.len) catch @panic("OOM"); + for (args.srcs, 0..) |s, i| result_types[i] = self.tileTyOf(reducedShape(s, args.axis).constSlice(), s.elemType()); + const body = self.combineBodyMulti(ctx, args.srcs, args.combine); + return self.emitMulti(ct.reduce(self.ctx, self.innerSlice(args.srcs), @intCast(args.axis), args.identities, body, result_types, self.loc()), args.srcs.len); + } + + pub fn scanMulti(self: *Builder, ctx: anytype, args: ScanMultiArgs(@TypeOf(ctx))) []Value { + std.debug.assert(args.srcs.len == args.identities.len); + const result_types = self.arena.allocator().alloc(*const mlir.Type, args.srcs.len) catch @panic("OOM"); + for (args.srcs, 0..) |s, i| { + checkAxis(s, args.axis); + result_types[i] = s.type_(); + } + const body = self.combineBodyMulti(ctx, args.srcs, args.combine); + return self.emitMulti(ct.scan(self.ctx, self.innerSlice(args.srcs), @intCast(args.axis), args.reverse, args.identities, body, result_types, self.loc()), args.srcs.len); + } + + const ReduceKind = enum { sum, max, min, prod }; + + fn combineKind(k: *Builder, element: Value, acc: Value, kind: ReduceKind) Value { + const is_float = element.isFloatElem(); + return switch (kind) { + .sum => if (is_float) k.addf(element, acc) else k.addi(element, acc), + .prod => if (is_float) k.mulf(element, acc) else k.muli(element, acc), + .max => if (is_float) k.maxf(element, acc) else k.maxi(element, acc, .signed), + .min => if (is_float) k.minf(element, acc) else k.mini(element, acc, .signed), + }; + } + + fn kindIdentity(self: *Builder, src: Value, kind: ReduceKind) *const mlir.Attribute { + const dt = self.mlirElemToDType(src.elemType()); + if (isFloatDtype(dt)) { + return self.identityAttr(dt, switch (kind) { + .sum => @as(f64, 0), + .prod => @as(f64, 1), + .max => -std.math.inf(f64), + .min => std.math.inf(f64), + }); + } + const bits: u32 = intBitwidth(dt); + const min_int: i64 = if (bits == 64) std.math.minInt(i64) else -(@as(i64, 1) << @intCast(bits - 1)); + const max_int: i64 = if (bits == 64) std.math.maxInt(i64) else (@as(i64, 1) << @intCast(bits - 1)) - 1; + return self.identityAttr(dt, switch (kind) { + .sum => @as(i64, 0), + .prod => @as(i64, 1), + .max => min_int, + .min => max_int, + }); + } + + fn reduceKind(self: *Builder, src: Value, axis: usize, comptime kind: ReduceKind) Value { + const combine = struct { + fn c(k: *Builder, element: Value, acc: Value, _: void) Value { + return combineKind(k, element, acc, kind); + } + }.c; + return self.reduce({}, .{ .src = src, .axis = axis, .identity = self.kindIdentity(src, kind), .combine = combine }); + } + + pub fn sum(self: *Builder, src: Value, axis: usize) Value { + return self.reduceKind(src, axis, .sum); + } + pub fn max(self: *Builder, src: Value, axis: usize) Value { + return self.reduceKind(src, axis, .max); + } + pub fn min(self: *Builder, src: Value, axis: usize) Value { + return self.reduceKind(src, axis, .min); + } + pub fn prod(self: *Builder, src: Value, axis: usize) Value { + return self.reduceKind(src, axis, .prod); + } + + fn scanKind(self: *Builder, src: Value, axis: usize, opts: ScanOpts, comptime kind: ReduceKind) Value { + const combine = struct { + fn c(k: *Builder, element: Value, acc: Value, _: void) Value { + return combineKind(k, element, acc, kind); + } + }.c; + return self.scan({}, .{ .src = src, .axis = axis, .reverse = opts.reverse, .identity = self.kindIdentity(src, kind), .combine = combine }); + } + + pub fn cumsum(self: *Builder, src: Value, axis: usize) Value { + return self.scanKind(src, axis, .{}, .sum); + } + pub fn cumsumOpts(self: *Builder, src: Value, axis: usize, opts: ScanOpts) Value { + return self.scanKind(src, axis, opts, .sum); + } + pub fn cumprod(self: *Builder, src: Value, axis: usize) Value { + return self.scanKind(src, axis, .{}, .prod); + } + pub fn cummax(self: *Builder, src: Value, axis: usize) Value { + return self.scanKind(src, axis, .{}, .max); + } + pub fn cummin(self: *Builder, src: Value, axis: usize) Value { + return self.scanKind(src, axis, .{}, .min); + } + + /// `for iv in [lower, upper) step` carrying `inits`; close with + /// `scope.yield(.{...})`, then read `scope.results`. + pub fn openFor( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + ) ForScope(tupleArity(@TypeOf(inits), "openFor: inits")) { + const N = comptime tupleArity(@TypeOf(inits), "openFor: inits"); + const fields = @typeInfo(@TypeOf(inits)).@"struct".fields; + + // Literals take the type of the first bound that is a Value; all + // three must agree (`AllTypesMatch` on the op). + const ref_ty: *const mlir.Type = if (@TypeOf(lower) == Value) lower.type_() else if (@TypeOf(upper) == Value) upper.type_() else if (@TypeOf(step) == Value) step.type_() else self.scalarTy(.i32); + const lb_v: Value = if (@TypeOf(lower) == Value) lower else self.constMatching(lower, ref_ty); + const ub_v: Value = if (@TypeOf(upper) == Value) upper else self.constMatching(upper, ref_ty); + const step_v: Value = if (@TypeOf(step) == Value) step else self.constMatching(step, ref_ty); + if (!lb_v.type_().eql(ub_v.type_()) or !lb_v.type_().eql(step_v.type_())) { + std.debug.panic("Builder.openFor: bounds must share one type, got {f}, {f}, {f}", .{ lb_v.type_(), ub_v.type_(), step_v.type_() }); + } + + var block_types: [N + 1]*const mlir.Type = undefined; + var block_locs: [N + 1]*const mlir.Location = undefined; + block_types[0] = lb_v.type_(); + block_locs[0] = self.loc(); + var inits_inner: [N]*const mlir.Value = undefined; + inline for (fields, 0..) |f, i| { + const raw = @field(inits, f.name); + const v: Value = if (f.type == Value) raw else self.lift(raw); + block_types[i + 1] = v.type_(); + block_locs[i + 1] = self.loc(); + inits_inner[i] = v.inner; + } + + const body = mlir.Block.init(&block_types, &block_locs); + self.pushBlock(body); + + var carried: [N]Value = undefined; + for (0..N) |i| carried[i] = .{ .inner = body.argument(i + 1), .kernel = self }; + + return .{ + .kernel = self, + .body = body, + .lb_inner = lb_v.inner, + .ub_inner = ub_v.inner, + .step_inner = step_v.inner, + .inits_inner = inits_inner, + .iv = .{ .inner = body.argument(0), .kernel = self }, + .carried = carried, + }; + } + + pub fn openIf(self: *Builder, cond: Value) IfOnlyScope { + const then_block = mlir.Block.init(&.{}, &.{}); + self.pushBlock(then_block); + return .{ + .kernel = self, + .cond_inner = cond.inner, + .then_block = then_block, + }; + } + + pub fn openIfElse( + self: *Builder, + cond: Value, + result_types: anytype, + ) IfScope(tupleArity(@TypeOf(result_types), "openIfElse: result_types")) { + const N = comptime tupleArity(@TypeOf(result_types), "openIfElse: result_types"); + const fields = @typeInfo(@TypeOf(result_types)).@"struct".fields; + + var types: [N]*const mlir.Type = undefined; + inline for (fields, 0..) |f, i| { + if (f.type != *const mlir.Type) + @compileError("openIfElse: every result_type must be *const mlir.Type (use b.scalarTy/tileTy)"); + types[i] = @field(result_types, f.name); + } + + const then_block = mlir.Block.init(&.{}, &.{}); + const else_block = mlir.Block.init(&.{}, &.{}); + self.pushBlock(then_block); + + return .{ + .kernel = self, + .cond_inner = cond.inner, + .then_block = then_block, + .else_block = else_block, + .result_types = types, + }; + } + + /// An unstructured loop carrying `inits`; leave with `scope.breakWith` + /// inside an `if`, and close the body with `scope.yield`. + pub fn openLoop(self: *Builder, inits: anytype) LoopScope(tupleArity(@TypeOf(inits), "openLoop: inits")) { + const N = comptime tupleArity(@TypeOf(inits), "openLoop: inits"); + const fields = @typeInfo(@TypeOf(inits)).@"struct".fields; + + var block_types: [N]*const mlir.Type = undefined; + var block_locs: [N]*const mlir.Location = undefined; + var inits_inner: [N]*const mlir.Value = undefined; + inline for (fields, 0..) |f, i| { + const raw = @field(inits, f.name); + const v: Value = if (f.type == Value) raw else self.lift(raw); + block_types[i] = v.type_(); + block_locs[i] = self.loc(); + inits_inner[i] = v.inner; + } + + const body = mlir.Block.init(&block_types, &block_locs); + self.pushBlock(body); + + var carried: [N]Value = undefined; + for (0..N) |i| carried[i] = .{ .inner = body.argument(i), .kernel = self }; + + return .{ + .kernel = self, + .body = body, + .inits_inner = inits_inner, + .result_types = block_types, + .carried = carried, + }; + } + + pub fn deviceAssert(self: *Builder, condition: Value, message: []const u8) void { + _ = ct.assert_(self.ctx, condition.inner, message, self.loc()).appendTo(self.currentBlock()); + } + + /// Append `return`, verify the module, and serialize it to a NUL-terminated + /// string owned by the kernel's allocator. `results` must be empty: an + /// `entry` returns nothing; outputs reach the host through pointers. + pub fn finish(self: *Builder, results: []const Value) FinishError![:0]const u8 { + std.debug.assert(results.len == 0); + const current = self.currentBlock(); + if (current.terminator() == null) { + _ = ct.return_(self.ctx, &.{}, self.loc()).appendTo(current); + } + + if (!self.module.operation().verify()) { + return error.InvalidMlir; + } + + var al: std.Io.Writer.Allocating = .init(self.allocator); + defer al.deinit(); + + try al.writer.print("{f}", .{self.module.operation()}); + + return try self.allocator.dupeZ(u8, al.written()); + } +}; + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + inline for (dialects_needed) |d| mlir.DialectHandle.fromString(d).insertDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectRoundTrip(ctx: *mlir.Context, ir: [:0]const u8) !void { + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "empty entry round-trips" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "empty"); + defer b.deinit(); + _ = try b.declareArgs(.{ .p = .{ .ptr = .f32 } }); + + const ir = try b.finish(&.{}); + defer std.testing.allocator.free(ir); + + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.module @empty") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "entry @empty(") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "assume") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "div_by<16>") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.entry") == null); + try std.testing.expect(std.mem.indexOf(u8, ir, "return") != null); + try expectRoundTrip(ctx, ir); +} + +test "add_one over pointer tiles matches the XLA milestone kernel" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "add_one"); + defer b.deinit(); + const a = try b.declareArgsOpts(.{ + .in = .{ .ptr_opts = .{ .dtype = .f32, .div_by = null } }, + .out = .{ .ptr_opts = .{ .dtype = .f32, .div_by = null } }, + }, .{}); + + const offsets = b.iota(128, .i32); + const loaded = b.loadPtrOpts(a.in.offset(offsets), .{}); + const sum = loaded.tile.add(1.0); + _ = b.storePtrOpts(a.out.offset(offsets), sum, .{ .token = loaded.token }); + + const ir = try b.finish(&.{}); + defer std.testing.allocator.free(ir); + + try std.testing.expect(std.mem.indexOf(u8, ir, "iota : tile<128xi32>") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "reshape") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "broadcast") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "offset") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "load_ptr_tko weak") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "addf") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "store_ptr_tko weak") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "token = ") != null); + try expectRoundTrip(ctx, ir); +} + +test "vector_add over partition views" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "vector_add"); + defer b.deinit(); + const a = try b.declareArgsOpts(.{ + .x = .{ .ptr = .f32 }, + .y = .{ .ptr = .f32 }, + .out = .{ .ptr = .f32 }, + }, .{ .hints = &.{.{ .arch = .sm_120, .num_worker_warps_per_cta = 4 }} }); + + const shape = [_]i64{ 8192, 128 }; + const strides = [_]i64{ 128, 1 }; + const tile = [_]i64{ 64, 64 }; + const vx = b.partitionView(b.tensorView(a.x, &shape, &strides), &tile, .{}); + const vy = b.partitionView(b.tensorView(a.y, &shape, &strides), &tile, .{}); + const vo = b.partitionView(b.tensorView(a.out, &shape, &strides), &tile, .{}); + + const bid = b.tileBlockId(); + const idx = [_]Value{ bid.x, bid.y }; + const xt = b.load(vx, &idx); + const yt = b.load(vy, &idx); + _ = b.store(xt.add(yt), vo, &idx); + + const ir = try b.finish(&.{}); + defer std.testing.allocator.free(ir); + + try std.testing.expect(std.mem.indexOf(u8, ir, "optimization_hints=") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "make_tensor_view") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "partition_view") != null or std.mem.indexOf(u8, ir, "addf") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "store_view_tko weak") != null); + try expectRoundTrip(ctx, ir); +} + +test "gemm with a for loop and mmaf" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "gemm"); + defer b.deinit(); + const a = try b.declareArgs(.{ .a = .{ .ptr = .bf16 }, .bm = .{ .ptr = .bf16 }, .c = .{ .ptr = .f32 } }); + + const m: i64 = 1024; + const n: i64 = 1024; + const k: i64 = 512; + const pa = b.partitionView(b.tensorView(a.a, &.{ m, k }, &.{ k, 1 }), &.{ 128, 64 }, .{}); + const pb = b.partitionView(b.tensorView(a.bm, &.{ k, n }, &.{ n, 1 }), &.{ 64, 128 }, .{}); + const pc = b.partitionView(b.tensorView(a.c, &.{ m, n }, &.{ n, 1 }), &.{ 128, 128 }, .{}); + + const bid = b.tileBlockId(); + const acc0 = b.zeros(&.{ 128, 128 }, .f32); + + var loop = b.openFor(0, @divExact(k, 64), 1, .{acc0}); + { + const kt = loop.iv; + const acc = loop.carried[0]; + const at = b.load(pa, &.{ bid.x, kt }); + const bt = b.load(pb, &.{ kt, bid.y }); + loop.yield(.{b.mmaf(at, bt, acc)}); + } + _ = b.store(loop.results[0], pc, &.{ bid.x, bid.y }); + + const ir = try b.finish(&.{}); + defer std.testing.allocator.free(ir); + + try std.testing.expect(std.mem.indexOf(u8, ir, "for ") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "iter_values") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "mmaf") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "continue ") != null); + try expectRoundTrip(ctx, ir); +} + +test "if, if-else, loop with break, reduce, scan, cast, select" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "control"); + defer b.deinit(); + // A scalar reaches the kernel through a rank-0 pointer. + const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .n = .{ .ptr = .i32 } }); + const n = b.loadPtr(a.n); + try std.testing.expect(n.isScalar()); + + const v = b.partitionView(b.tensorView(a.x, &.{ 256, 64 }, &.{ 64, 1 }), &.{ 8, 64 }, .{}); + const bid = b.tileBlockId(); + const t = b.load(v, &.{ bid.x, bid.y }); + + // Reductions and scans over the tile. + const row_sum = b.sum(t, 1); + const col_max = b.max(t, 0); + const running = b.cumsum(t, 1); + try std.testing.expectEqual(@as(usize, 1), row_sum.rank()); + try std.testing.expectEqual(@as(i64, 8), row_sum.dim(0)); + try std.testing.expectEqual(@as(i64, 64), col_max.dim(0)); + try std.testing.expectEqual(@as(usize, 2), running.rank()); + + // A variadic reduce: argmax over axis 1 as (max, index). + const cols = b.broadcastTo(b.reshape(b.iota(64, .i32), &.{ 1, 64 }), &.{ 8, 64 }); + const argmax = b.reduceMulti({}, .{ + .srcs = &.{ t, cols }, + .axis = 1, + .identities = &.{ b.identityAttr(.f32, -std.math.inf(f64)), b.identityAttr(.i32, 0) }, + .combine = struct { + fn c(k: *Builder, elements: []const Value, accs: []const Value, _: void) []const Value { + const better = elements[0].gt(accs[0]); + return k.yield(.{ k.select(better, elements[0], accs[0]), k.select(better, elements[1], accs[1]) }); + } + }.c, + }); + try std.testing.expectEqual(@as(usize, 2), argmax.len); + try std.testing.expectEqual(@as(i64, 8), argmax[1].dim(0)); + + // Casts and select. + const as_i32 = t.to(.i32); + const back = as_i32.to(.f32); + const picked = b.where(t.gt(0.0), back, t.neg()); + + // if / if-else carrying a value. + const is_first = bid.x.eq(0); + var only = b.openIf(is_first); + _ = b.store(picked, v, &.{ bid.x, bid.y }); + only.yieldThen(.{}); + + var branch = b.openIfElse(is_first, .{b.tileTy(&.{ 8, 64 }, .f32)}); + branch.yieldThen(.{picked}); + branch.yieldElse(.{t}); + const chosen = branch.results[0]; + + // loop: count up to n, break when reached. + var lp = b.openLoop(.{b.cst(.i32, 0)}); + { + const i = lp.carried[0]; + var done = b.openIf(i.ge(n)); + lp.breakWith(.{i}); + done.yieldThen(.{}); + lp.yield(.{i.add(1)}); + } + _ = lp.results[0]; + + // for with an early `continue` inside an `if`, and i64 bounds from a Value. + var skip = b.openFor(b.cst(.i64, 0), 16, 1, .{b.cst(.i64, 0)}); + { + const acc = skip.carried[0]; + var odd = b.openIf(skip.iv.bitAnd(1).eq(1)); + skip.continueWith(.{acc}); + odd.yieldThen(.{}); + skip.yield(.{acc.add(skip.iv)}); + } + try std.testing.expect(skip.results[0].isScalar()); + + _ = b.store(chosen, v, &.{ bid.x, bid.y }); + + const ir = try b.finish(&.{}); + defer std.testing.allocator.free(ir); + + try std.testing.expect(std.mem.indexOf(u8, ir, "reduce ") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "scan ") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "ftoi") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "itof") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "select") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "loop ") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "break ") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "yield") != null); + try expectRoundTrip(ctx, ir); +} + +test "atomics, print, globals, alloca, dynamic views, strided view, tokens" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "misc"); + defer b.deinit(); + const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .cnt = .{ .ptr = .i32 }, .rows = .{ .ptr = .i32 } }); + b.global("lut", &.{16}, .f32, 1.5, .{ .constant = true }); + + const rows = b.loadPtr(a.rows); + const tv = b.tensorViewDyn(a.x, &.{ .{ .dynamic = rows }, .{ .static = 64 } }, &.{ .{ .static = 64 }, .{ .static = 1 } }); + const extents = b.tensorShape(tv); + try std.testing.expectEqual(@as(usize, 2), extents.len); + const v = b.partitionView(tv, &.{ 8, 64 }, .{ .padding = .nan }); + const tiles = b.indexSpaceShape(v); + try std.testing.expectEqual(@as(usize, 2), tiles.len); + const sv = b.stridedView(tv, &.{ 8, 64 }, &.{ 2, 1 }, .{}); + _ = b.load(sv, &.{ b.cst(.i32, 0), b.cst(.i32, 0) }); + + const bid = b.tileBlockId(); + const loaded = b.loadOpts(v, &.{ bid.x, bid.y }, .{ .hints = &.{.{ .arch = .sm_120, .allow_tma = true }} }); + const tok = b.print("tile %f\n", &.{loaded.tile}); + + const counter = a.cnt.offset(b.iota(8, .i32)); + const old = b.atomicRmw(.add, counter, b.full(&.{8}, 1, .i32), .{ .token = tok }); + const cas = b.atomicCas(counter, old.tile, b.full(&.{8}, 0, .i32), .{ .token = old.token }); + const joined = b.joinTokens(&.{ loaded.token, cas.token, b.makeToken() }); + // Atomic reductions refuse a padded view; the default options must verify. + const unpadded = b.partitionView(tv, &.{ 8, 64 }, .{ .padding = null }); + _ = b.atomicRedView(.addf, unpadded, &.{ bid.x, bid.y }, loaded.tile, .{ .token = joined }); + _ = b.atomicRedView(.addf, unpadded, &.{ bid.x, bid.y }, loaded.tile, .{}); + + // A gather/scatter view: the sparse index is a rank-1 tile of row ids. + const gsv = b.gatherScatterView(tv, &.{ 8, 64 }, 0, .zero); + const gathered = b.load(gsv, &.{ b.iota(8, .i32), b.cst(.i32, 0) }); + try std.testing.expectEqual(@as(i64, 8), gathered.dim(0)); + try std.testing.expectEqual(@as(i64, 64), gathered.dim(1)); + _ = b.store(gathered, gsv, &.{ b.iota(8, .i32), b.cst(.i32, 0) }); + + const lut = b.getGlobal("lut", .f32); + const scratch = b.alloca(.f32, 64, .{}); + _ = b.loadPtr(lut.offset(b.iota(16, .i32))); + _ = b.storePtr(scratch.offset(b.iota(64, .i32)), b.zeros(&.{64}, .f32)); + + const ir = try b.finish(&.{}); + defer std.testing.allocator.free(ir); + + try std.testing.expect(std.mem.indexOf(u8, ir, "global @lut") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "get_global @lut") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "tensor_view") != null); + try expectRoundTrip(ctx, ir); +} diff --git a/kernels/cuda_tile/control_flow.zig b/kernels/cuda_tile/control_flow.zig new file mode 100644 index 00000000..2f971f1d --- /dev/null +++ b/kernels/cuda_tile/control_flow.zig @@ -0,0 +1,155 @@ +//! `cuda_tile` scope wrappers, like `kernels/common/control_flow.zig` but with +//! the dialect's terminators: `continue` ends a `for`/`loop` body, `yield` an +//! `if`, `break` leaves a `loop`. An already-terminated branch is left alone. + +const std = @import("std"); +const mlir = @import("mlir"); +const ct = @import("mlir/dialects/cuda_tile"); +const dsl = @import("kernels/common"); + +pub const tupleArity = dsl.tupleArity; + +fn collect(comptime ValueT: type, comptime N: usize, values: anytype, comptime what: []const u8) [N]*const mlir.Value { + const info = @typeInfo(@TypeOf(values)); + if (info != .@"struct" or !info.@"struct".is_tuple) + @compileError(what ++ " expects a tuple literal"); + if (info.@"struct".fields.len != N) + @compileError(what ++ ": arity must match the scope's declared arity"); + var buf: [N]*const mlir.Value = undefined; + inline for (info.@"struct".fields, 0..) |f, i| { + if (f.type != ValueT) + @compileError(what ++ ": every tuple element must be a Value"); + buf[i] = @field(values, f.name).inner; + } + return buf; +} + +fn terminate(comptime ValueT: type, comptime N: usize, kernel: anytype, block: *mlir.Block, comptime mnemonic: enum { yield, continue_, break_ }, values: anytype, comptime what: []const u8) void { + if (block.terminator() != null) return; + const buf = collect(ValueT, N, values, what); + const op = switch (mnemonic) { + .yield => ct.yield(kernel.ctx, &buf, kernel.loc()), + .continue_ => ct.continue_(kernel.ctx, &buf, kernel.loc()), + .break_ => ct.break_(kernel.ctx, &buf, kernel.loc()), + }; + _ = op.appendTo(block); +} + +/// `for %iv in (lb to ub, step) iter_values(...)`; `yield` closes it. +pub fn ForScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize) type { + return struct { + kernel: *BuilderT, + body: *mlir.Block, + lb_inner: *const mlir.Value, + ub_inner: *const mlir.Value, + step_inner: *const mlir.Value, + inits_inner: [N]*const mlir.Value, + iv: ValueT, + carried: [N]ValueT, + results: [N]ValueT = undefined, + unsigned_cmp: bool = false, + + const Self = @This(); + + /// `continue` in the current block, inside a zero-result `if`: skips + /// the rest of the iteration and terminates that branch. + pub fn continueWith(self: *Self, values: anytype) void { + const k = self.kernel; + terminate(ValueT, N, k, k.currentBlock(), .continue_, values, "ForScope.continueWith"); + } + + pub fn yield(self: *Self, values: anytype) void { + const k = self.kernel; + terminate(ValueT, N, k, self.body, .continue_, values, "ForScope.yield"); + k.popBlock(); + const for_op = ct.for_(k.ctx, self.lb_inner, self.ub_inner, self.step_inner, &self.inits_inner, self.body, self.unsigned_cmp, k.loc()); + _ = for_op.appendTo(k.currentBlock()); + for (0..N) |i| self.results[i] = .{ .inner = for_op.result(i), .kernel = k }; + } + }; +} + +pub fn IfOnlyScope(comptime BuilderT: type, comptime ValueT: type) type { + return struct { + kernel: *BuilderT, + cond_inner: *const mlir.Value, + then_block: *mlir.Block, + + const Self = @This(); + + pub fn yieldThen(self: *Self, values: anytype) void { + const k = self.kernel; + terminate(ValueT, 0, k, self.then_block, .yield, values, "IfOnlyScope.yieldThen"); + k.popBlock(); + _ = ct.if_(k.ctx, self.cond_inner, &.{}, self.then_block, null, k.loc()).appendTo(k.currentBlock()); + } + }; +} + +pub fn IfScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize) type { + return struct { + kernel: *BuilderT, + cond_inner: *const mlir.Value, + then_block: *mlir.Block, + else_block: *mlir.Block, + result_types: [N]*const mlir.Type, + results: [N]ValueT = undefined, + + const Self = @This(); + + pub fn yieldThen(self: *Self, values: anytype) void { + const k = self.kernel; + terminate(ValueT, N, k, self.then_block, .yield, values, "IfScope.yieldThen"); + k.popBlock(); + k.pushBlock(self.else_block); + } + + pub fn yieldElse(self: *Self, values: anytype) void { + const k = self.kernel; + terminate(ValueT, N, k, self.else_block, .yield, values, "IfScope.yieldElse"); + k.popBlock(); + const if_op = ct.if_(k.ctx, self.cond_inner, &self.result_types, self.then_block, self.else_block, k.loc()); + _ = if_op.appendTo(k.currentBlock()); + for (0..N) |i| self.results[i] = .{ .inner = if_op.result(i), .kernel = k }; + } + }; +} + +/// `loop iter_values(...)`: the body ends with `continue` (`yield`) and leaves +/// through `break` (`breakWith`, inside an `if`). `results` are the broken. +pub fn LoopScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize) type { + return struct { + kernel: *BuilderT, + body: *mlir.Block, + inits_inner: [N]*const mlir.Value, + result_types: [N]*const mlir.Type, + carried: [N]ValueT, + results: [N]ValueT = undefined, + + const Self = @This(); + + /// `break` in the current block; terminates the enclosing `if` branch. + pub fn breakWith(self: *Self, values: anytype) void { + const k = self.kernel; + terminate(ValueT, N, k, k.currentBlock(), .break_, values, "LoopScope.breakWith"); + } + + pub fn continueWith(self: *Self, values: anytype) void { + const k = self.kernel; + terminate(ValueT, N, k, k.currentBlock(), .continue_, values, "LoopScope.continueWith"); + } + + pub fn yield(self: *Self, values: anytype) void { + const k = self.kernel; + terminate(ValueT, N, k, self.body, .continue_, values, "LoopScope.yield"); + k.popBlock(); + const loop_op = ct.loop(k.ctx, &self.inits_inner, &self.result_types, self.body, k.loc()); + _ = loop_op.appendTo(k.currentBlock()); + for (0..N) |i| self.results[i] = .{ .inner = loop_op.result(i), .kernel = k }; + } + }; +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/kernels/cuda_tile/dtype.zig b/kernels/cuda_tile/dtype.zig new file mode 100644 index 00000000..4552e662 --- /dev/null +++ b/kernels/cuda_tile/dtype.zig @@ -0,0 +1,103 @@ +const std = @import("std"); +const mlir = @import("mlir"); + +/// Every element type `cuda_tile.tile` accepts (Types.td: +/// CudaTile_TileElementType). `i4`, `f4e2m1fn` need 13.3; `f8e8m0fnu` needs +/// 13.2. `i4` is a tile element only — a pointer cannot point at it. +pub const DType = enum { + i1, + i4, + i8, + i16, + i32, + i64, + f16, + bf16, + f32, + tf32, + f64, + f8e4m3fn, + f8e5m2, + f8e8m0fnu, + f4e2m1fn, + + pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { + return switch (self) { + .i1 => .int(ctx, .i1), + .i4 => .int(ctx, .i4), + .i8 => .int(ctx, .i8), + .i16 => .int(ctx, .i16), + .i32 => .int(ctx, .i32), + .i64 => .int(ctx, .i64), + .f16 => .float(ctx, .f16), + .bf16 => .float(ctx, .bf16), + .f32 => .float(ctx, .f32), + .tf32 => .float(ctx, .tf32), + .f64 => .float(ctx, .f64), + .f8e4m3fn => .float(ctx, .f8e4m3fn), + .f8e5m2 => .float(ctx, .f8e5m2), + .f8e8m0fnu => .float(ctx, .f8e8m0fnu), + .f4e2m1fn => .float(ctx, .f4e2m1fn), + }; + } +}; + +pub fn isFloatDtype(dt: DType) bool { + return switch (dt) { + .f16, .bf16, .f32, .tf32, .f64, .f8e4m3fn, .f8e5m2, .f8e8m0fnu, .f4e2m1fn => true, + else => false, + }; +} + +pub fn dtypeBitwidth(dt: DType) usize { + return switch (dt) { + .i1 => 1, + .i4, .f4e2m1fn => 4, + .i8, .f8e4m3fn, .f8e5m2, .f8e8m0fnu => 8, + .i16, .f16, .bf16 => 16, + .i32, .f32, .tf32 => 32, + .i64, .f64 => 64, + }; +} + +pub fn intBitwidth(dt: DType) u32 { + return switch (dt) { + .i1 => 1, + .i4 => 4, + .i8 => 8, + .i16 => 16, + .i32 => 32, + .i64 => 64, + else => std.debug.panic("intBitwidth: not an integer DType: {s}", .{@tagName(dt)}), + }; +} + +pub fn mlirElemToDType(ctx: *mlir.Context, elem: *const mlir.Type) DType { + inline for (std.meta.fields(DType)) |f| { + const dt = @field(DType, f.name); + if (elem.eql(dt.toMlir(ctx))) return dt; + } + @panic("element type not a recognized CUDA Tile IR DType"); +} + +/// SharedVerifiers.h verifyTileSize: every dimension a power of two and the +/// tile at most 2^24 elements. Checked in Zig so the failure names the shape. +pub fn tileLegal(shape: []const i64) bool { + var elems: i64 = 1; + for (shape) |d| { + if (d <= 0 or (d & (d - 1)) != 0) return false; + elems *= d; + } + return elems <= (1 << 24); +} + +test { + std.testing.refAllDecls(@This()); +} + +test "tileLegal" { + try std.testing.expect(tileLegal(&.{})); + try std.testing.expect(tileLegal(&.{ 64, 128 })); + try std.testing.expect(!tileLegal(&.{3})); + try std.testing.expect(!tileLegal(&.{ 4096, 4096, 2 })); +} diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/BUILD.bazel index c60cd41c..558b03a0 100644 --- a/mlir/dialects/BUILD.bazel +++ b/mlir/dialects/BUILD.bazel @@ -20,6 +20,7 @@ zig_library( deps = [ "//mlir", "//mlir:c", + "//mlir/dialects/cuda_tile", "//mlir/dialects/mosaic_tpu", "//mlir/dialects/stablehlo", "//mlir/dialects/ttir", diff --git a/mlir/dialects/cuda_tile/BUILD.bazel b/mlir/dialects/cuda_tile/BUILD.bazel new file mode 100644 index 00000000..48e50fdc --- /dev/null +++ b/mlir/dialects/cuda_tile/BUILD.bazel @@ -0,0 +1,34 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +# The TileView type interface, which NVIDIA's C API does not expose. +cc_library( + name = "cuda_tile_capi", + srcs = ["cuda_tile_capi.cc"], + hdrs = ["cuda_tile_capi.h"], + strip_include_prefix = ".", + visibility = ["//visibility:private"], + deps = [ + "@cuda_tile//:CudaTileDialect", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "cuda_tile", + import_name = "mlir/dialects/cuda_tile", + main = "cuda_tile.zig", + visibility = ["//visibility:public"], + deps = [ + ":cuda_tile_capi", + "//mlir", + "//mlir:c", + "//stdx", + "@cuda_tile//:cuda_tile_capi", + ], +) + +zig_test( + name = "test", + deps = [":cuda_tile"], +) diff --git a/mlir/dialects/cuda_tile/cuda_tile.zig b/mlir/dialects/cuda_tile/cuda_tile.zig new file mode 100644 index 00000000..ff86836a --- /dev/null +++ b/mlir/dialects/cuda_tile/cuda_tile.zig @@ -0,0 +1,1696 @@ +//! Zig bindings for NVIDIA's `cuda_tile` MLIR dialect (CUDA Tile IR): one +//! function per op in `Ops.td` order. Types and enums go through the dialect's +//! C API; what it has no getter for is parsed from text. + +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub const dialect_namespace = "cuda_tile"; + +fn stringRef(s: []const u8) c.MlirStringRef { + return .{ .data = s.ptr, .length = s.len }; +} + +fn typeFromC(t: c.MlirType) *const mlir.Type { + return @ptrCast(t.ptr orelse @panic("cuda_tile: null MlirType")); +} + +fn attrFromC(a: c.MlirAttribute) *const mlir.Attribute { + return @ptrCast(a.ptr orelse @panic("cuda_tile: null MlirAttribute")); +} + +const null_attr: c.MlirAttribute = .{ .ptr = null }; + +/// `!cuda_tile.ptr` — a typed global-memory pointer. +pub const PointerType = opaque { + const M = mlir.Methods(PointerType, c.MlirType); + + pub const isAFn = c.mlirCudaTileTypeIsAPointerType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn get(ctx: *mlir.Context, pointee_type: *const mlir.Type) *const PointerType { + return @ptrCast(typeFromC(c.mlirCudaTilePointerTypeGet(ctx.ptr(), pointee_type.ptr()))); + } + + pub fn pointee(self: *const PointerType) *const mlir.Type { + return typeFromC(c.mlirCudaTilePointerTypeGetPointeeType(self.ptr())); + } +}; + +pub fn pointerType(ctx: *mlir.Context, pointee_type: *const mlir.Type) *const mlir.Type { + return @ptrCast(PointerType.get(ctx, pointee_type)); +} + +/// `!cuda_tile.tile` — every SSA value that carries data. Rank 0 is +/// the scalar. Dimensions must be powers of two. +pub const TileType = opaque { + const M = mlir.Methods(TileType, c.MlirType); + + pub const isAFn = c.mlirCudaTileTypeIsATileType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn get(ctx: *mlir.Context, shape: []const i64, elem: *const mlir.Type) *const TileType { + return @ptrCast(typeFromC(c.mlirCudaTileTileTypeGet(ctx.ptr(), @intCast(shape.len), shape.ptr, elem.ptr()))); + } + + pub fn elementType(self: *const TileType) *const mlir.Type { + return typeFromC(c.mlirCudaTileTileTypeGetElementType(self.ptr())); + } + + pub fn rank(self: *const TileType) usize { + return @intCast(c.mlirCudaTileTileTypeGetRank(self.ptr())); + } + + pub fn dimension(self: *const TileType, i: usize) i64 { + return c.mlirCudaTileTileTypeGetDimSize(self.ptr(), @intCast(i)); + } +}; + +pub fn tileType(ctx: *mlir.Context, shape: []const i64, elem: *const mlir.Type) *const mlir.Type { + return @ptrCast(TileType.get(ctx, shape, elem)); +} + +/// `!cuda_tile.token` — the ordering token every `*_tko` op produces. +pub const TokenType = opaque { + const M = mlir.Methods(TokenType, c.MlirType); + + pub const isAFn = c.mlirCudaTileTypeIsATokenType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn get(ctx: *mlir.Context) *const TokenType { + return @ptrCast(typeFromC(c.mlirCudaTileTokenTypeGet(ctx.ptr()))); + } +}; + +pub fn tokenType(ctx: *mlir.Context) *const mlir.Type { + return @ptrCast(TokenType.get(ctx)); +} + +/// `!cuda_tile.tensor_view`. `dynamic()` marks a +/// dimension or stride that arrives as an operand of `make_tensor_view`. +pub const TensorViewType = opaque { + const M = mlir.Methods(TensorViewType, c.MlirType); + + pub const isAFn = c.mlirCudaTileTypeIsATensorViewType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn dynamic() i64 { + return c.mlirCudaTileTensorViewTypeGetDynamicSize(); + } + + pub fn get(ctx: *mlir.Context, elem: *const mlir.Type, shape: []const i64, strides: []const i64) *const TensorViewType { + return @ptrCast(typeFromC(c.mlirCudaTileTensorViewTypeGet( + ctx.ptr(), + elem.ptr(), + @intCast(shape.len), + shape.ptr, + @intCast(strides.len), + strides.ptr, + ))); + } + + pub fn elementType(self: *const TensorViewType) *const mlir.Type { + return typeFromC(c.mlirCudaTileTensorViewTypeGetElementType(self.ptr())); + } + + pub fn rank(self: *const TensorViewType) usize { + return @intCast(c.mlirCudaTileTensorViewTypeGetRank(self.ptr())); + } + + pub fn dimension(self: *const TensorViewType, i: usize) i64 { + return c.mlirCudaTileTensorViewTypeGetDimSize(self.ptr(), @intCast(i)); + } + + pub fn stride(self: *const TensorViewType, i: usize) i64 { + return c.mlirCudaTileTensorViewTypeGetStride(self.ptr(), @intCast(i)); + } +}; + +pub fn tensorViewType(ctx: *mlir.Context, elem: *const mlir.Type, shape: []const i64, strides: []const i64) *const mlir.Type { + return @ptrCast(TensorViewType.get(ctx, elem, shape, strides)); +} + +const DimMap = stdx.BoundedArray(i32, mlir.ShapedType.MAX_RANK); + +fn identityDimMap(dim_map: []const i32, rank: usize) DimMap { + var out: DimMap = .empty; + if (dim_map.len > 0) { + out.appendSliceAssumeCapacity(dim_map); + } else { + for (0..rank) |i| out.appendAssumeCapacity(@intCast(i)); + } + return out; +} + +/// `!cuda_tile.partition_view, dim_map=[...]>`. +/// The tile shape, padding and dim_map all live in the type; the op that +/// builds one has no attributes of its own. +pub const PartitionViewType = opaque { + const M = mlir.Methods(PartitionViewType, c.MlirType); + + pub const isAFn = c.mlirCudaTileTypeIsAPartitionViewType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + /// An empty `dim_map` is the identity; the type requires one entry per + /// tile dimension. + pub fn get( + ctx: *mlir.Context, + tile_shape: []const i32, + tensor_view: *const mlir.Type, + dim_map: []const i32, + padding: ?PaddingValue, + ) *const PartitionViewType { + const tile_attr: *const mlir.Attribute = .denseArray(ctx, .i32, tile_shape); + const map = identityDimMap(dim_map, tile_shape.len); + return @ptrCast(typeFromC(c.mlirCudaTilePartitionViewTypeGet( + ctx.ptr(), + tile_attr.ptr(), + tensor_view.ptr(), + @intCast(map.len), + map.constSlice().ptr, + if (padding) |p| p.attribute(ctx).ptr() else null_attr, + ))); + } + + pub fn tensorView(self: *const PartitionViewType) *const mlir.Type { + return typeFromC(c.mlirCudaTilePartitionViewTypeGetTensorView(self.ptr())); + } + + pub fn viewTileType(self: *const PartitionViewType) *const mlir.Type { + return typeFromC(c.mlirCudaTilePartitionViewTypeGetViewTileType(self.ptr())); + } + + pub fn viewIndexRank(self: *const PartitionViewType) usize { + return @intCast(c.mlirCudaTilePartitionViewTypeGetViewIndexRank(self.ptr())); + } +}; + +/// `!cuda_tile.strided_view` (13.3). +pub const StridedViewType = opaque { + const M = mlir.Methods(StridedViewType, c.MlirType); + + pub const isAFn = c.mlirCudaTileTypeIsAStridedViewType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn get( + ctx: *mlir.Context, + tile_shape: []const i32, + traversal_strides: []const i32, + tensor_view: *const mlir.Type, + dim_map: []const i32, + padding: ?PaddingValue, + ) *const StridedViewType { + const tile_attr: *const mlir.Attribute = .denseArray(ctx, .i32, tile_shape); + const strides_attr: *const mlir.Attribute = .denseArray(ctx, .i32, traversal_strides); + const map = identityDimMap(dim_map, tile_shape.len); + return @ptrCast(typeFromC(c.mlirCudaTileStridedViewTypeGet( + ctx.ptr(), + tile_attr.ptr(), + strides_attr.ptr(), + tensor_view.ptr(), + @intCast(map.len), + map.constSlice().ptr, + if (padding) |p| p.attribute(ctx).ptr() else null_attr, + ))); + } + + pub fn viewTileType(self: *const StridedViewType) *const mlir.Type { + return typeFromC(c.mlirCudaTileStridedViewTypeGetViewTileType(self.ptr())); + } + + pub fn viewIndexRank(self: *const StridedViewType) usize { + return @intCast(c.mlirCudaTileStridedViewTypeGetViewIndexRank(self.ptr())); + } +}; + +/// `!cuda_tile.gather_scatter_view, sparse_dim=N>` +/// (13.3). The C API has no getter for it, so it is parsed from text. +pub fn gatherScatterViewType( + ctx: *mlir.Context, + tile_shape: []const i64, + tensor_view: *const mlir.Type, + sparse_dim: u32, + padding: ?PaddingValue, +) *const mlir.Type { + var buf: [1024]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + w.writeAll("!cuda_tile.gather_scatter_view 0) w.writeAll("x") catch unreachable; + w.print("{d}", .{d}) catch unreachable; + } + w.writeAll("), ") catch unreachable; + if (padding) |p| w.print("padding_value={s}, ", .{@tagName(p)}) catch unreachable; + w.print("{f}, sparse_dim={d}>", .{ tensor_view, sparse_dim }) catch unreachable; + const text = w.buffered(); + return mlir.Type.parse(ctx, text) catch std.debug.panic("cuda_tile: cannot parse '{s}'", .{text}); +} + +/// The `tile<...>` type a load through `view` yields, for any view kind — +/// through the dialect's `TileView` type interface (`cuda_tile_capi.cc`), so +/// the gather/scatter view, which has no getters in NVIDIA's C API, works too. +pub fn tileTypeOfView(view: *const mlir.Type) *const mlir.Type { + if (!c.zmlCudaTileTypeIsATileView(view.ptr())) std.debug.panic("cuda_tile: {f} is not a tiled view", .{view}); + return typeFromC(c.zmlCudaTileTileViewGetViewTileType(view.ptr())); +} + +pub fn indexRankOfView(view: *const mlir.Type) usize { + if (!c.zmlCudaTileTypeIsATileView(view.ptr())) std.debug.panic("cuda_tile: {f} is not a tiled view", .{view}); + return @intCast(c.zmlCudaTileTileViewGetViewIndexRank(view.ptr())); +} + +// Enum attributes: built through the C getter from the tag name, so a typo is +// a null attribute rather than a wrong case. + +fn enumAttribute(comptime getter: anytype, ctx: *mlir.Context, name: []const u8) *const mlir.Attribute { + return attrFromC(getter(ctx.ptr(), stringRef(name))); +} + +pub const RoundingMode = enum { + nearest_even, + zero, + negative_inf, + positive_inf, + approx, + full, + nearest_int_to_zero, + + pub fn attribute(self: RoundingMode, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileRoundingModeAttrGet, ctx, @tagName(self)); + } +}; + +pub const Signedness = enum { + signed, + unsigned, + + pub fn attribute(self: Signedness, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileSignednessAttrGet, ctx, @tagName(self)); + } +}; + +pub const IntegerOverflow = enum { + none, + no_signed_wrap, + no_unsigned_wrap, + no_wrap, + + pub fn attribute(self: IntegerOverflow, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileIntegerOverflowAttrGet, ctx, @tagName(self)); + } +}; + +pub const ComparisonOrdering = enum { + unordered, + ordered, + + pub fn attribute(self: ComparisonOrdering, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileComparisonOrderingAttrGet, ctx, @tagName(self)); + } +}; + +pub const ComparisonPredicate = enum { + equal, + not_equal, + less_than, + less_than_or_equal, + greater_than, + greater_than_or_equal, + + pub fn attribute(self: ComparisonPredicate, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileComparisonPredicateAttrGet, ctx, @tagName(self)); + } +}; + +pub const AtomicRMWMode = enum { + @"and", + @"or", + xor, + add, + addf, + max, + min, + umax, + umin, + xchg, + + pub fn attribute(self: AtomicRMWMode, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileAtomicRMWModeAttrGet, ctx, @tagName(self)); + } +}; + +pub const MemoryScope = enum { + tl_blk, + device, + sys, + + pub fn attribute(self: MemoryScope, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileMemoryScopeAttrGet, ctx, @tagName(self)); + } +}; + +pub const MemoryOrdering = enum { + weak, + relaxed, + acquire, + release, + acq_rel, + + pub fn attribute(self: MemoryOrdering, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileMemoryOrderingSemanticsAttrGet, ctx, @tagName(self)); + } +}; + +pub const PaddingValue = enum { + zero, + neg_zero, + nan, + pos_inf, + neg_inf, + + pub fn attribute(self: PaddingValue, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTilePaddingValueAttrGet, ctx, @tagName(self)); + } +}; + +pub const SymbolVisibility = enum { + public, + private, + + pub fn attribute(self: SymbolVisibility, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileSymbolVisibilityAttrGet, ctx, @tagName(self)); + } +}; + +// optimization_hints: per-architecture dictionaries, built from text so +// several architectures can be listed. + +pub const Arch = enum { default, sm_80, sm_86, sm_87, sm_88, sm_89, sm_90, sm_100, sm_103, sm_110, sm_120, sm_121 }; + +/// Hints `entry` accepts: `num_cta_in_cga` (power of two <= 16), +/// `num_worker_warps_per_cta` (4 or 8), `occupancy`. +pub const EntryHint = struct { + arch: Arch = .default, + num_cta_in_cga: ?i32 = null, + num_worker_warps_per_cta: ?i32 = null, + occupancy: ?i32 = null, +}; + +/// Hints `load_view_tko` / `store_view_tko` accept. +pub const LoadStoreHint = struct { + arch: Arch = .default, + allow_tma: ?bool = null, + latency: ?i32 = null, +}; + +/// Hints `load_ptr_tko` / `store_ptr_tko` accept: `latency` only, the +/// pointer arm cannot use TMA. (The dialect drops unsupported keys silently.) +pub const PtrLoadStoreHint = struct { + arch: Arch = .default, + latency: ?i32 = null, +}; + +fn writeHintFields(w: *std.Io.Writer, hint: anytype) !void { + var first = true; + inline for (@typeInfo(@TypeOf(hint)).@"struct".fields) |f| { + if (comptime std.mem.eql(u8, f.name, "arch")) continue; + if (@field(hint, f.name)) |v| { + if (!first) try w.writeAll(", "); + first = false; + switch (@TypeOf(v)) { + bool => try w.print("{s} = {}", .{ f.name, v }), + else => try w.print("{s} = {d}", .{ f.name, v }), + } + } + } +} + +/// `#cuda_tile.optimization_hints` from a +/// slice of `EntryHint` or `LoadStoreHint`. +pub fn optimizationHints(ctx: *mlir.Context, hints: anytype) *const mlir.Attribute { + var buf: [2048]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + w.writeAll("#cuda_tile.optimization_hints<") catch unreachable; + for (hints, 0..) |h, i| { + if (i > 0) w.writeAll(", ") catch unreachable; + w.print("{s} = {{", .{@tagName(h.arch)}) catch unreachable; + writeHintFields(&w, h) catch unreachable; + w.writeAll("}") catch unreachable; + } + w.writeAll(">") catch unreachable; + const text = w.buffered(); + return mlir.Attribute.parse(ctx, text) catch std.debug.panic("cuda_tile: cannot parse '{s}'", .{text}); +} + +// assume predicates: no C getters; parsed from their documented spellings. + +fn parseAttr(ctx: *mlir.Context, comptime fmt: []const u8, args: anytype) *const mlir.Attribute { + var buf: [256]u8 = undefined; + const text = std.fmt.bufPrint(&buf, fmt, args) catch unreachable; + return mlir.Attribute.parse(ctx, text) catch std.debug.panic("cuda_tile: cannot parse '{s}'", .{text}); +} + +/// `#cuda_tile.div_by`, or `` for a per-group claim. +pub fn divBy(ctx: *mlir.Context, divisor: u64, every: ?i64, along: ?i64) *const mlir.Attribute { + if (every) |e| { + return parseAttr(ctx, "#cuda_tile.div_by<{d}, every {d} along {d}>", .{ divisor, e, along orelse 0 }); + } + return parseAttr(ctx, "#cuda_tile.div_by<{d}>", .{divisor}); +} + +pub fn sameElements(ctx: *mlir.Context, values: []const i64) *const mlir.Attribute { + var buf: [256]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + w.writeAll("#cuda_tile.same_elements<[") catch unreachable; + for (values, 0..) |v, i| { + if (i > 0) w.writeAll(", ") catch unreachable; + w.print("{d}", .{v}) catch unreachable; + } + w.writeAll("]>") catch unreachable; + const text = w.buffered(); + return mlir.Attribute.parse(ctx, text) catch std.debug.panic("cuda_tile: cannot parse '{s}'", .{text}); +} + +/// `#cuda_tile.bounded`, either side `?` when null. +pub fn bounded(ctx: *mlir.Context, lb: ?i64, ub: ?i64) *const mlir.Attribute { + var buf: [128]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + w.writeAll("#cuda_tile.bounded<") catch unreachable; + if (lb) |v| w.print("{d}", .{v}) catch unreachable else w.writeAll("?") catch unreachable; + w.writeAll(", ") catch unreachable; + if (ub) |v| w.print("{d}", .{v}) catch unreachable else w.writeAll("?") catch unreachable; + w.writeAll(">") catch unreachable; + const text = w.buffered(); + return mlir.Attribute.parse(ctx, text) catch std.debug.panic("cuda_tile: cannot parse '{s}'", .{text}); +} + +// Constant values: DenseElementsAttr over a tile type, which is a ShapedType. + +pub fn splatAttr(tile_ty: *const mlir.Type, elem_attr: *const mlir.Attribute) *const mlir.Attribute { + return attrFromC(c.mlirDenseElementsAttrSplatGet(tile_ty.ptr(), elem_attr.ptr())); +} + +pub fn intElem(elem_ty: *const mlir.Type, value: i64) *const mlir.Attribute { + return attrFromC(c.mlirIntegerAttrGet(elem_ty.ptr(), value)); +} + +pub fn floatElem(ctx: *mlir.Context, elem_ty: *const mlir.Type, value: f64) *const mlir.Attribute { + return attrFromC(c.mlirFloatAttrDoubleGet(ctx.ptr(), elem_ty.ptr(), value)); +} + +/// A dense `DenseElementsAttr` over `tile_ty` from a raw element buffer; the +/// element type's bit width must be a multiple of 8. +pub fn denseAttr(tile_ty: *const mlir.Type, values: anytype) *const mlir.Attribute { + return .denseElements(@ptrCast(tile_ty), values); +} + +const MakeArgs = mlir.Operation.MakeArgs; + +fn opName(comptime mnemonic: []const u8) []const u8 { + return dialect_namespace ++ "." ++ mnemonic; +} + +/// Attribute lists are passed by value so a helper's temporary outlives the +/// slice handed to `Operation.make`. +const Attrs = stdx.BoundedArray(mlir.NamedAttribute, 8); + +fn attrsOf(list: []const mlir.NamedAttribute) Attrs { + var attrs: Attrs = .empty; + attrs.appendSliceAssumeCapacity(list); + return attrs; +} + +fn unary(ctx: *mlir.Context, comptime mnemonic: []const u8, source: *const mlir.Value, attrs: Attrs, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName(mnemonic), .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{source.type_()} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +fn binary(ctx: *mlir.Context, comptime mnemonic: []const u8, lhs: *const mlir.Value, rhs: *const mlir.Value, attrs: Attrs, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName(mnemonic), .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{lhs.type_()} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +fn convert(ctx: *mlir.Context, comptime mnemonic: []const u8, source: *const mlir.Value, result_type: *const mlir.Type, attrs: Attrs, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName(mnemonic), .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +fn roundingAttrs(ctx: *mlir.Context, rounding: RoundingMode, flush_to_zero: bool) Attrs { + var attrs: Attrs = .empty; + attrs.appendAssumeCapacity(.named(ctx, "rounding_mode", rounding.attribute(ctx))); + if (flush_to_zero) attrs.appendAssumeCapacity(.named(ctx, "flush_to_zero", .unit(ctx))); + return attrs; +} + +fn overflowAttrs(ctx: *mlir.Context, overflow: IntegerOverflow) Attrs { + var attrs: Attrs = .empty; + if (overflow != .none) attrs.appendAssumeCapacity(.named(ctx, "overflow", overflow.attribute(ctx))); + return attrs; +} + +fn ftzAttrs(ctx: *mlir.Context, flush_to_zero: bool) Attrs { + var attrs: Attrs = .empty; + if (flush_to_zero) attrs.appendAssumeCapacity(.named(ctx, "flush_to_zero", .unit(ctx))); + return attrs; +} + +fn i1Like(ctx: *mlir.Context, v: *const mlir.Value) *const mlir.Type { + const t = v.type_().isA(TileType) orelse std.debug.panic("cuda_tile: {f} is not a tile", .{v.type_()}); + var shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..t.rank()) |i| shape.appendAssumeCapacity(t.dimension(i)); + return tileType(ctx, shape.constSlice(), .int(ctx, .i1)); +} + +pub fn absf(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "absf", source, .empty, location); +} + +pub fn addf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, rounding: RoundingMode, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "addf", lhs, rhs, roundingAttrs(ctx, rounding, flush_to_zero), location); +} + +pub fn subf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, rounding: RoundingMode, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "subf", lhs, rhs, roundingAttrs(ctx, rounding, flush_to_zero), location); +} + +pub fn mulf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, rounding: RoundingMode, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "mulf", lhs, rhs, roundingAttrs(ctx, rounding, flush_to_zero), location); +} + +pub fn divf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, rounding: RoundingMode, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "divf", lhs, rhs, roundingAttrs(ctx, rounding, flush_to_zero), location); +} + +pub fn remf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "remf", lhs, rhs, .empty, location); +} + +pub fn negf(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "negf", source, .empty, location); +} + +pub fn fma(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, acc: *const mlir.Value, rounding: RoundingMode, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("fma"), .{ + .operands = .{ .flat = &.{ lhs, rhs, acc } }, + .results = .{ .flat = &.{acc.type_()} }, + .attributes = roundingAttrs(ctx, rounding, flush_to_zero).constSlice(), + .location = location, + }); +} + +fn minMaxAttrs(ctx: *mlir.Context, propagate_nan: bool, flush_to_zero: bool) Attrs { + var attrs: Attrs = .empty; + if (propagate_nan) attrs.appendAssumeCapacity(.named(ctx, "propagate_nan", .unit(ctx))); + if (flush_to_zero) attrs.appendAssumeCapacity(.named(ctx, "flush_to_zero", .unit(ctx))); + return attrs; +} + +pub fn maxf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, propagate_nan: bool, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "maxf", lhs, rhs, minMaxAttrs(ctx, propagate_nan, flush_to_zero), location); +} + +pub fn minf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, propagate_nan: bool, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "minf", lhs, rhs, minMaxAttrs(ctx, propagate_nan, flush_to_zero), location); +} + +pub fn pow(ctx: *mlir.Context, source: *const mlir.Value, exponent: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "pow", source, exponent, .empty, location); +} + +pub fn atan2(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "atan2", x, y, .empty, location); +} + +/// `rounding` null keeps the op's default (13.3 attribute). +pub fn exp(ctx: *mlir.Context, source: *const mlir.Value, rounding: ?RoundingMode, location: *const mlir.Location) *mlir.Operation { + var attrs: Attrs = .empty; + if (rounding) |r| attrs.appendAssumeCapacity(.named(ctx, "rounding_mode", r.attribute(ctx))); + return unary(ctx, "exp", source, attrs, location); +} + +pub fn exp2(ctx: *mlir.Context, source: *const mlir.Value, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "exp2", source, ftzAttrs(ctx, flush_to_zero), location); +} + +pub fn log(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "log", source, .empty, location); +} + +pub fn log2(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "log2", source, .empty, location); +} + +pub fn sqrt(ctx: *mlir.Context, source: *const mlir.Value, rounding: RoundingMode, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "sqrt", source, roundingAttrs(ctx, rounding, flush_to_zero), location); +} + +pub fn rsqrt(ctx: *mlir.Context, source: *const mlir.Value, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "rsqrt", source, ftzAttrs(ctx, flush_to_zero), location); +} + +pub fn sin(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "sin", source, .empty, location); +} + +pub fn cos(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "cos", source, .empty, location); +} + +pub fn tan(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "tan", source, .empty, location); +} + +pub fn sinh(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "sinh", source, .empty, location); +} + +pub fn cosh(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "cosh", source, .empty, location); +} + +/// `rounding` null keeps the op's default (13.2 attribute). Only `approx` +/// and `full` are legal; the upstream verifier reports the others without +/// failing, so they are refused here. +pub fn tanh(ctx: *mlir.Context, source: *const mlir.Value, rounding: ?RoundingMode, location: *const mlir.Location) *mlir.Operation { + var attrs: Attrs = .empty; + if (rounding) |r| { + switch (r) { + .approx, .full => {}, + else => std.debug.panic("cuda_tile.tanh: rounding mode {s} is not allowed (only approx or full)", .{@tagName(r)}), + } + attrs.appendAssumeCapacity(.named(ctx, "rounding_mode", r.attribute(ctx))); + } + return unary(ctx, "tanh", source, attrs, location); +} + +pub fn ceil(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "ceil", source, .empty, location); +} + +pub fn floor(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "floor", source, .empty, location); +} + +pub fn cmpf(ctx: *mlir.Context, predicate: ComparisonPredicate, ordering: ComparisonOrdering, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("cmpf"), .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{i1Like(ctx, lhs)} }, + .attributes = &.{ + .named(ctx, "comparison_predicate", predicate.attribute(ctx)), + .named(ctx, "comparison_ordering", ordering.attribute(ctx)), + }, + .location = location, + }); +} + +pub fn mmaf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, acc: *const mlir.Value, fast_acc: bool, location: *const mlir.Location) *mlir.Operation { + var attrs: Attrs = .empty; + if (fast_acc) attrs.appendAssumeCapacity(.named(ctx, "fast_acc", .unit(ctx))); + return mlir.Operation.make(ctx, opName("mmaf"), .{ + .operands = .{ .flat = &.{ lhs, rhs, acc } }, + .results = .{ .flat = &.{acc.type_()} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// Block-scaled MMA (13.3): scales are plain tiles in logical M/N/K layout. +pub fn mmaf_scaled(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, acc: *const mlir.Value, lhs_scale: *const mlir.Value, rhs_scale: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("mmaf_scaled"), .{ + .operands = .{ .flat = &.{ lhs, rhs, acc, lhs_scale, rhs_scale } }, + .results = .{ .flat = &.{acc.type_()} }, + .location = location, + }); +} + +pub fn absi(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "absi", source, .empty, location); +} + +pub fn addi(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, overflow: IntegerOverflow, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "addi", lhs, rhs, overflowAttrs(ctx, overflow), location); +} + +pub fn subi(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, overflow: IntegerOverflow, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "subi", lhs, rhs, overflowAttrs(ctx, overflow), location); +} + +pub fn muli(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, overflow: IntegerOverflow, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "muli", lhs, rhs, overflowAttrs(ctx, overflow), location); +} + +pub fn mulhii(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "mulhii", x, y, .empty, location); +} + +/// `rounding` null keeps the op's default (truncation). +pub fn divi(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, signedness: Signedness, rounding: ?RoundingMode, location: *const mlir.Location) *mlir.Operation { + var attrs: Attrs = .empty; + attrs.appendAssumeCapacity(.named(ctx, "signedness", signedness.attribute(ctx))); + if (rounding) |r| attrs.appendAssumeCapacity(.named(ctx, "rounding", r.attribute(ctx))); + return binary(ctx, "divi", lhs, rhs, attrs, location); +} + +fn signedBinary(ctx: *mlir.Context, comptime mnemonic: []const u8, lhs: *const mlir.Value, rhs: *const mlir.Value, signedness: Signedness, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, mnemonic, lhs, rhs, attrsOf(&.{.named(ctx, "signedness", signedness.attribute(ctx))}), location); +} + +pub fn remi(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, signedness: Signedness, location: *const mlir.Location) *mlir.Operation { + return signedBinary(ctx, "remi", lhs, rhs, signedness, location); +} + +pub fn maxi(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, signedness: Signedness, location: *const mlir.Location) *mlir.Operation { + return signedBinary(ctx, "maxi", lhs, rhs, signedness, location); +} + +pub fn mini(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, signedness: Signedness, location: *const mlir.Location) *mlir.Operation { + return signedBinary(ctx, "mini", lhs, rhs, signedness, location); +} + +pub fn negi(ctx: *mlir.Context, source: *const mlir.Value, overflow: IntegerOverflow, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "negi", source, overflowAttrs(ctx, overflow), location); +} + +pub fn shli(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, overflow: IntegerOverflow, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "shli", lhs, rhs, overflowAttrs(ctx, overflow), location); +} + +pub fn shri(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, signedness: Signedness, location: *const mlir.Location) *mlir.Operation { + return signedBinary(ctx, "shri", lhs, rhs, signedness, location); +} + +pub fn cmpi(ctx: *mlir.Context, predicate: ComparisonPredicate, signedness: Signedness, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("cmpi"), .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{i1Like(ctx, lhs)} }, + .attributes = &.{ + .named(ctx, "comparison_predicate", predicate.attribute(ctx)), + .named(ctx, "signedness", signedness.attribute(ctx)), + }, + .location = location, + }); +} + +pub fn mmai(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, acc: *const mlir.Value, signedness_lhs: Signedness, signedness_rhs: Signedness, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("mmai"), .{ + .operands = .{ .flat = &.{ lhs, rhs, acc } }, + .results = .{ .flat = &.{acc.type_()} }, + .attributes = &.{ + .named(ctx, "signedness_lhs", signedness_lhs.attribute(ctx)), + .named(ctx, "signedness_rhs", signedness_rhs.attribute(ctx)), + }, + .location = location, + }); +} + +pub fn andi(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "andi", lhs, rhs, .empty, location); +} + +pub fn ori(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "ori", lhs, rhs, .empty, location); +} + +pub fn xori(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "xori", lhs, rhs, .empty, location); +} + +pub fn bitcast(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "bitcast", source, result_type, .empty, location); +} + +pub fn exti(ctx: *mlir.Context, from: *const mlir.Value, signedness: Signedness, to: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "exti", from, to, attrsOf(&.{.named(ctx, "signedness", signedness.attribute(ctx))}), location); +} + +pub fn trunci(ctx: *mlir.Context, from: *const mlir.Value, to: *const mlir.Type, overflow: IntegerOverflow, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "trunci", from, to, overflowAttrs(ctx, overflow), location); +} + +/// `rounding` null keeps the op's default (nearest_even). +pub fn ftof(ctx: *mlir.Context, from: *const mlir.Value, to: *const mlir.Type, rounding: ?RoundingMode, location: *const mlir.Location) *mlir.Operation { + var attrs: Attrs = .empty; + if (rounding) |r| attrs.appendAssumeCapacity(.named(ctx, "rounding_mode", r.attribute(ctx))); + return convert(ctx, "ftof", from, to, attrs, location); +} + +pub fn ftoi(ctx: *mlir.Context, from: *const mlir.Value, signedness: Signedness, rounding: RoundingMode, to: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "ftoi", from, to, attrsOf(&.{ + .named(ctx, "signedness", signedness.attribute(ctx)), + .named(ctx, "rounding_mode", rounding.attribute(ctx)), + }), location); +} + +pub fn itof(ctx: *mlir.Context, from: *const mlir.Value, signedness: Signedness, rounding: RoundingMode, to: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "itof", from, to, attrsOf(&.{ + .named(ctx, "signedness", signedness.attribute(ctx)), + .named(ctx, "rounding_mode", rounding.attribute(ctx)), + }), location); +} + +pub fn int_to_ptr(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "int_to_ptr", source, result_type, .empty, location); +} + +pub fn ptr_to_int(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "ptr_to_int", source, result_type, .empty, location); +} + +pub fn ptr_to_ptr(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "ptr_to_ptr", source, result_type, .empty, location); +} + +/// 13.3: sub-byte tile -> i8 tile. +pub fn pack(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "pack", source, result_type, .empty, location); +} + +/// 13.3: i8 tile -> sub-byte tile. +pub fn unpack(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "unpack", source, result_type, .empty, location); +} + +pub fn constant(ctx: *mlir.Context, value: *const mlir.Attribute, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("constant"), .{ + .results = .{ .flat = &.{result_type} }, + .attributes = &.{.named(ctx, "value", value)}, + .location = location, + }); +} + +pub fn iota(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("iota"), .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn reshape(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "reshape", source, result_type, .empty, location); +} + +pub fn broadcast(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "broadcast", source, result_type, .empty, location); +} + +pub fn cat(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, dim: i64, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("cat"), .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{.named(ctx, "dim", .int(ctx, .i64, dim))}, + .location = location, + }); +} + +pub fn extract(ctx: *mlir.Context, source: *const mlir.Value, indices: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, mlir.ShapedType.MAX_RANK + 1) = .empty; + operands.appendAssumeCapacity(source); + operands.appendSliceAssumeCapacity(indices); + return mlir.Operation.make(ctx, opName("extract"), .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn permute(ctx: *mlir.Context, source: *const mlir.Value, permutation: []const i32, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "permute", source, result_type, attrsOf(&.{.named(ctx, "permutation", .denseArray(ctx, .i32, permutation))}), location); +} + +pub fn select(ctx: *mlir.Context, cond: *const mlir.Value, val_if_true: *const mlir.Value, val_if_false: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("select"), .{ + .operands = .{ .flat = &.{ cond, val_if_true, val_if_false } }, + .results = .{ .flat = &.{val_if_true.type_()} }, + .location = location, + }); +} + +pub fn offset(ctx: *mlir.Context, ptr: *const mlir.Value, off: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "offset", ptr, off, .empty, location); +} + +fn threeI32(ctx: *mlir.Context, comptime mnemonic: []const u8, location: *const mlir.Location) *mlir.Operation { + const i32_tile = tileType(ctx, &.{}, .int(ctx, .i32)); + return mlir.Operation.make(ctx, opName(mnemonic), .{ + .results = .{ .flat = &.{ i32_tile, i32_tile, i32_tile } }, + .location = location, + }); +} + +pub fn get_tile_block_id(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return threeI32(ctx, "get_tile_block_id", location); +} + +pub fn get_num_tile_blocks(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return threeI32(ctx, "get_num_tile_blocks", location); +} + +pub const GlobalOpts = struct { + alignment: ?i64 = null, + constant: bool = false, + visibility: ?SymbolVisibility = null, +}; + +pub fn global(ctx: *mlir.Context, name: []const u8, value: *const mlir.Attribute, opts: GlobalOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: Attrs = .empty; + attrs.appendAssumeCapacity(.named(ctx, "sym_name", .string(ctx, name))); + attrs.appendAssumeCapacity(.named(ctx, "value", value)); + if (opts.alignment) |a| attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, a))); + if (opts.constant) attrs.appendAssumeCapacity(.named(ctx, "constant", .unit(ctx))); + if (opts.visibility) |v| attrs.appendAssumeCapacity(.named(ctx, "symbol_visibility", v.attribute(ctx))); + return mlir.Operation.make(ctx, opName("global"), .{ + .attributes = attrs.constSlice(), + .verify = false, + .location = location, + }); +} + +pub fn get_global(ctx: *mlir.Context, name: []const u8, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("get_global"), .{ + .results = .{ .flat = &.{result_type} }, + .attributes = &.{.named(ctx, "name", .flatSymbolRef(ctx, name))}, + // Symbol lookup needs the enclosing module; verified with it. + .verify = false, + .location = location, + }); +} + +/// `cuda_tile.module @name { ... }`. `body` holds the entries and globals. +pub fn module(ctx: *mlir.Context, name: []const u8, body: *mlir.Block, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("module"), .{ + .attributes = &.{.named(ctx, "sym_name", .string(ctx, name))}, + .blocks = &.{body}, + .verify = false, + .location = location, + }); +} + +pub const EntryArgs = struct { + name: []const u8, + /// The entry block; its arguments are the kernel parameters, every one a + /// rank-0 tile. + block: *mlir.Block, + arg_attrs: ?[]const *const mlir.Attribute = null, + optimization_hints: ?*const mlir.Attribute = null, + location: *const mlir.Location, +}; + +/// `entry @name(...) { ... }` — the kernel. Returns nothing; results reach +/// the host through pointer arguments. +pub fn entry(ctx: *mlir.Context, args: EntryArgs) *mlir.Operation { + var arg_types: stdx.BoundedArray(*const mlir.Type, 256) = .empty; + if (args.block.numArguments() > arg_types.capacity()) std.debug.panic("cuda_tile.entry: {d} parameters, at most {d} supported", .{ args.block.numArguments(), arg_types.capacity() }); + for (0..args.block.numArguments()) |i| arg_types.appendAssumeCapacity(args.block.argument(i).type_()); + + var attrs: Attrs = .empty; + attrs.appendAssumeCapacity(.named(ctx, "sym_name", .string(ctx, args.name))); + attrs.appendAssumeCapacity(.named(ctx, "function_type", .typeAttr(.function(ctx, arg_types.constSlice(), &.{})))); + if (args.arg_attrs) |a| attrs.appendAssumeCapacity(.named(ctx, "arg_attrs", .array(ctx, a))); + if (args.optimization_hints) |h| attrs.appendAssumeCapacity(.named(ctx, "optimization_hints", h)); + + return mlir.Operation.make(ctx, opName("entry"), .{ + .attributes = attrs.constSlice(), + .blocks = &.{args.block}, + .verify = false, + .location = args.location, + }); +} + +fn terminator(ctx: *mlir.Context, comptime mnemonic: []const u8, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName(mnemonic), .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +pub fn return_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return terminator(ctx, "return", values, location); +} + +pub fn yield(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return terminator(ctx, "yield", values, location); +} + +pub fn continue_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return terminator(ctx, "continue", values, location); +} + +pub fn break_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return terminator(ctx, "break", values, location); +} + +/// `for %iv in (%lb to %ub, step %step) iter_values(...)`. `body` takes +/// `[iv, inits...]` and ends with `continue`. +pub fn for_(ctx: *mlir.Context, lower: *const mlir.Value, upper: *const mlir.Value, step: *const mlir.Value, inits: []const *const mlir.Value, body: *mlir.Block, unsigned_cmp: bool, location: *const mlir.Location) *mlir.Operation { + // Built on the state directly: any number of carried values, and outer + // SSA uses only dominate once the op sits in its block, so no per-op + // verify. + var state: mlir.OperationState = .init(opName("for"), location); + state.addOperands(&.{ lower, upper, step }); + state.addOperands(inits); + for (inits) |v| state.addResults(&.{v.type_()}); + if (unsigned_cmp) state.addAttributes(&.{.named(ctx, "unsignedCmp", .unit(ctx))}); + const region = mlir.Region.init(); + region.appendOwnedBlock(body); + state.addOwnedRegions(&.{region}); + return mlir.Operation.init(&state) catch @panic("Failed to create cuda_tile.for"); +} + +/// `if %cond -> (types) { ... } else { ... }`. Branches end with `yield` +/// (or `continue`/`break`/`return` when they leave the enclosing scope). +pub fn if_(ctx: *mlir.Context, cond: *const mlir.Value, result_types: []const *const mlir.Type, then_block: *mlir.Block, else_block: ?*mlir.Block, location: *const mlir.Location) *mlir.Operation { + _ = ctx; + var state: mlir.OperationState = .init(opName("if"), location); + state.addOperands(&.{cond}); + state.addResults(result_types); + const then_region = mlir.Region.init(); + then_region.appendOwnedBlock(then_block); + const else_region = mlir.Region.init(); + if (else_block) |b| else_region.appendOwnedBlock(b); + state.addOwnedRegions(&.{ then_region, else_region }); + return mlir.Operation.init(&state) catch @panic("Failed to create cuda_tile.if"); +} + +/// An unstructured loop over `inits`; the body takes them as arguments and +/// leaves through `continue` (next iteration) or `break` (the results). +pub fn loop(ctx: *mlir.Context, inits: []const *const mlir.Value, result_types: []const *const mlir.Type, body: *mlir.Block, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("loop"), .{ + .operands = .{ .flat = inits }, + .results = .{ .flat = result_types }, + .blocks = &.{body}, + .verify = false, + .location = location, + }); +} + +/// `reduce %operands dim=D identities=[...]` — `body` takes `2N` arguments +/// (element, accumulator per operand) and yields `N` accumulators. Results +/// drop dimension `dim`. +pub fn reduce(ctx: *mlir.Context, operands: []const *const mlir.Value, dim: i32, identities: []const *const mlir.Attribute, body: *mlir.Block, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("reduce"), .{ + .operands = .{ .flat = operands }, + .results = .{ .flat = result_types }, + .blocks = &.{body}, + .attributes = &.{ + .named(ctx, "dim", .int(ctx, .i32, dim)), + .named(ctx, "identities", .array(ctx, identities)), + }, + .verify = false, + .location = location, + }); +} + +/// `scan %operands dim=D reverse=R identities=[...]` — like `reduce`, but the +/// results keep the operand shape. +pub fn scan(ctx: *mlir.Context, operands: []const *const mlir.Value, dim: i32, reverse: bool, identities: []const *const mlir.Attribute, body: *mlir.Block, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("scan"), .{ + .operands = .{ .flat = operands }, + .results = .{ .flat = result_types }, + .blocks = &.{body}, + .attributes = &.{ + .named(ctx, "dim", .int(ctx, .i32, dim)), + .named(ctx, "reverse", .boolean(ctx, reverse)), + .named(ctx, "identities", .array(ctx, identities)), + }, + .verify = false, + .location = location, + }); +} + +pub fn assert_(ctx: *mlir.Context, condition: *const mlir.Value, message: []const u8, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("assert"), .{ + .operands = .{ .flat = &.{condition} }, + .attributes = &.{.named(ctx, "message", .string(ctx, message))}, + .location = location, + }); +} + +/// `%r = assume #predicate, %value` — a refined copy of `value`. +pub fn assume(ctx: *mlir.Context, value: *const mlir.Value, predicate: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("assume"), .{ + .operands = .{ .flat = &.{value} }, + .results = .{ .flat = &.{value.type_()} }, + .attributes = &.{.named(ctx, "predicate", predicate)}, + .location = location, + }); +} + +/// `make_tensor_view %base, shape = [...], strides = [...]`. Static extents +/// live in `result_type`; the operands fill its `?` slots in order. +pub fn make_tensor_view(ctx: *mlir.Context, base: *const mlir.Value, dynamic_shape: []const *const mlir.Value, dynamic_strides: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + var segs: Segments = .{}; + segs.add(&.{base}); + segs.add(dynamic_shape); + segs.add(dynamic_strides); + return mlir.Operation.make(ctx, opName("make_tensor_view"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{segs.attr(ctx)}, + .location = location, + }); +} + +fn viewFromTensorView(ctx: *mlir.Context, comptime mnemonic: []const u8, tensor_view: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, mnemonic, tensor_view, result_type, .empty, location); +} + +/// The tile shape, padding and dim_map are all in `result_type`. +pub fn make_partition_view(ctx: *mlir.Context, tensor_view: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return viewFromTensorView(ctx, "make_partition_view", tensor_view, result_type, location); +} + +pub fn make_strided_view(ctx: *mlir.Context, tensor_view: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return viewFromTensorView(ctx, "make_strided_view", tensor_view, result_type, location); +} + +pub fn make_gather_scatter_view(ctx: *mlir.Context, tensor_view: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return viewFromTensorView(ctx, "make_gather_scatter_view", tensor_view, result_type, location); +} + +/// One rank-0 integer tile per view index dimension. +pub fn get_index_space_shape(ctx: *mlir.Context, view: *const mlir.Value, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("get_index_space_shape"), .{ + .operands = .{ .flat = &.{view} }, + .results = .{ .flat = result_types }, + .location = location, + }); +} + +/// One rank-0 integer tile per tensor dimension. +pub fn get_tensor_shape(ctx: *mlir.Context, tensor_view: *const mlir.Value, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("get_tensor_shape"), .{ + .operands = .{ .flat = &.{tensor_view} }, + .results = .{ .flat = result_types }, + .location = location, + }); +} + +pub const MemOpts = struct { + ordering: MemoryOrdering = .weak, + scope: ?MemoryScope = null, + token: ?*const mlir.Value = null, + optimization_hints: ?*const mlir.Attribute = null, +}; + +fn memAttrs(ctx: *mlir.Context, opts: MemOpts) Attrs { + var attrs: Attrs = .empty; + attrs.appendAssumeCapacity(.named(ctx, "memory_ordering_semantics", opts.ordering.attribute(ctx))); + if (opts.scope) |s| attrs.appendAssumeCapacity(.named(ctx, "memory_scope", s.attribute(ctx))); + if (opts.optimization_hints) |h| attrs.appendAssumeCapacity(.named(ctx, "optimization_hints", h)); + return attrs; +} + +/// An optional operand as a 0- or 1-element segment. Held in a local so the +/// slice handed to `Operation.make` stays valid. +const Seg = struct { + buf: [1]*const mlir.Value = undefined, + len: usize = 0, + + fn slice(self: *const Seg) []const *const mlir.Value { + return self.buf[0..self.len]; + } +}; + +fn optional(v: ?*const mlir.Value) Seg { + var s: Seg = .{}; + if (v) |x| { + s.buf[0] = x; + s.len = 1; + } + return s; +} + +/// Operands of an `AttrSizedOperandSegments` op, flat plus the +/// `operandSegmentSizes` the op's properties expect (a DenseI32ArrayAttr). +const Segments = struct { + values: stdx.BoundedArray(*const mlir.Value, 128) = .empty, + sizes: stdx.BoundedArray(i32, 8) = .empty, + + fn add(self: *Segments, segment: []const *const mlir.Value) void { + self.values.appendSliceAssumeCapacity(segment); + self.sizes.appendAssumeCapacity(@intCast(segment.len)); + } + + fn attr(self: *const Segments, ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, self.sizes.constSlice())); + } +}; + +/// Two results: the tile (the view's tile type) and the token. +pub fn load_view_tko(ctx: *mlir.Context, view: *const mlir.Value, index: []const *const mlir.Value, opts: MemOpts, location: *const mlir.Location) *mlir.Operation { + const tok = optional(opts.token); + var segs: Segments = .{}; + segs.add(&.{view}); + segs.add(index); + segs.add(tok.slice()); + var attrs = memAttrs(ctx, opts); + attrs.appendAssumeCapacity(segs.attr(ctx)); + return mlir.Operation.make(ctx, opName("load_view_tko"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{ tileTypeOfView(view.type_()), tokenType(ctx) } }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// One result: the token. +pub fn store_view_tko(ctx: *mlir.Context, tile: *const mlir.Value, view: *const mlir.Value, index: []const *const mlir.Value, opts: MemOpts, location: *const mlir.Location) *mlir.Operation { + const tok = optional(opts.token); + var segs: Segments = .{}; + segs.add(&.{tile}); + segs.add(&.{view}); + segs.add(index); + segs.add(tok.slice()); + var attrs = memAttrs(ctx, opts); + attrs.appendAssumeCapacity(segs.attr(ctx)); + return mlir.Operation.make(ctx, opName("store_view_tko"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{tokenType(ctx)} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub const LoadPtrOpts = struct { + mask: ?*const mlir.Value = null, + padding: ?*const mlir.Value = null, + mem: MemOpts = .{}, +}; + +/// Gather through a tile of pointers. Two results: the loaded tile and the +/// token. `result_type` is the pointee tile. +pub fn load_ptr_tko(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, opts: LoadPtrOpts, location: *const mlir.Location) *mlir.Operation { + const mask = optional(opts.mask); + const padding = optional(opts.padding); + const tok = optional(opts.mem.token); + var segs: Segments = .{}; + segs.add(&.{source}); + segs.add(mask.slice()); + segs.add(padding.slice()); + segs.add(tok.slice()); + var attrs = memAttrs(ctx, opts.mem); + attrs.appendAssumeCapacity(segs.attr(ctx)); + return mlir.Operation.make(ctx, opName("load_ptr_tko"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{ result_type, tokenType(ctx) } }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub const StorePtrOpts = struct { + mask: ?*const mlir.Value = null, + mem: MemOpts = .{}, +}; + +/// Scatter through a tile of pointers. One result: the token. +pub fn store_ptr_tko(ctx: *mlir.Context, destination: *const mlir.Value, value: *const mlir.Value, opts: StorePtrOpts, location: *const mlir.Location) *mlir.Operation { + const mask = optional(opts.mask); + const tok = optional(opts.mem.token); + var segs: Segments = .{}; + segs.add(&.{destination}); + segs.add(&.{value}); + segs.add(mask.slice()); + segs.add(tok.slice()); + var attrs = memAttrs(ctx, opts.mem); + attrs.appendAssumeCapacity(segs.attr(ctx)); + return mlir.Operation.make(ctx, opName("store_ptr_tko"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{tokenType(ctx)} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub const AtomicOpts = struct { + ordering: MemoryOrdering = .acq_rel, + scope: MemoryScope = .device, + mask: ?*const mlir.Value = null, + token: ?*const mlir.Value = null, +}; + +fn atomicAttrs(ctx: *mlir.Context, opts: AtomicOpts) Attrs { + var attrs: Attrs = .empty; + attrs.appendAssumeCapacity(.named(ctx, "memory_ordering_semantics", opts.ordering.attribute(ctx))); + attrs.appendAssumeCapacity(.named(ctx, "memory_scope", opts.scope.attribute(ctx))); + return attrs; +} + +/// Two results: the old values and the token. +pub fn atomic_cas_tko(ctx: *mlir.Context, pointers: *const mlir.Value, cmp: *const mlir.Value, val: *const mlir.Value, opts: AtomicOpts, location: *const mlir.Location) *mlir.Operation { + const mask = optional(opts.mask); + const tok = optional(opts.token); + var segs: Segments = .{}; + segs.add(&.{pointers}); + segs.add(&.{cmp}); + segs.add(&.{val}); + segs.add(mask.slice()); + segs.add(tok.slice()); + var attrs = atomicAttrs(ctx, opts); + attrs.appendAssumeCapacity(segs.attr(ctx)); + return mlir.Operation.make(ctx, opName("atomic_cas_tko"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{ val.type_(), tokenType(ctx) } }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// Two results: the old values and the token. +pub fn atomic_rmw_tko(ctx: *mlir.Context, pointers: *const mlir.Value, mode: AtomicRMWMode, arg: *const mlir.Value, opts: AtomicOpts, location: *const mlir.Location) *mlir.Operation { + const mask = optional(opts.mask); + const tok = optional(opts.token); + var segs: Segments = .{}; + segs.add(&.{pointers}); + segs.add(&.{arg}); + segs.add(mask.slice()); + segs.add(tok.slice()); + var attrs = atomicAttrs(ctx, opts); + attrs.appendAssumeCapacity(.named(ctx, "mode", mode.attribute(ctx))); + attrs.appendAssumeCapacity(segs.attr(ctx)); + return mlir.Operation.make(ctx, opName("atomic_rmw_tko"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{ arg.type_(), tokenType(ctx) } }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// `atomic_red_view_tko` is relaxed-only, block- or device-scoped, and has +/// no mask operand. +pub const RedViewOpts = struct { + scope: MemoryScope = .device, + token: ?*const mlir.Value = null, +}; + +/// 13.3: an atomic reduction through an unpadded tiled view. One result: +/// the token. `xchg` is not a reduction. +pub fn atomic_red_view_tko(ctx: *mlir.Context, view: *const mlir.Value, index: []const *const mlir.Value, mode: AtomicRMWMode, value: *const mlir.Value, opts: RedViewOpts, location: *const mlir.Location) *mlir.Operation { + if (mode == .xchg) @panic("cuda_tile.atomic_red_view_tko: xchg is not a reduction"); + if (opts.scope == .sys) @panic("cuda_tile.atomic_red_view_tko: only tl_blk and device scopes are allowed"); + const tok = optional(opts.token); + var segs: Segments = .{}; + segs.add(&.{view}); + segs.add(index); + segs.add(&.{value}); + segs.add(tok.slice()); + var attrs: Attrs = .empty; + attrs.appendAssumeCapacity(.named(ctx, "memory_ordering_semantics", MemoryOrdering.relaxed.attribute(ctx))); + attrs.appendAssumeCapacity(.named(ctx, "memory_scope", opts.scope.attribute(ctx))); + attrs.appendAssumeCapacity(.named(ctx, "mode", mode.attribute(ctx))); + attrs.appendAssumeCapacity(segs.attr(ctx)); + return mlir.Operation.make(ctx, opName("atomic_red_view_tko"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{tokenType(ctx)} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// 13.3: `num_elem` elements of the pointee, as a rank-0 pointer tile. +pub fn alloca(ctx: *mlir.Context, num_elem: i64, alignment: i64, is_global: bool, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + var attrs: Attrs = .empty; + attrs.appendAssumeCapacity(.named(ctx, "num_elem", .int(ctx, .i64, num_elem))); + attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, alignment))); + if (is_global) attrs.appendAssumeCapacity(.named(ctx, "global", .unit(ctx))); + return mlir.Operation.make(ctx, opName("alloca"), .{ + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub fn make_token(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("make_token"), .{ + .results = .{ .flat = &.{tokenType(ctx)} }, + .location = location, + }); +} + +pub fn join_tokens(ctx: *mlir.Context, tokens: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("join_tokens"), .{ + .operands = .{ .flat = tokens }, + .results = .{ .flat = &.{tokenType(ctx)} }, + .location = location, + }); +} + +/// Device-side printf. One result: the token. +pub fn print_tko(ctx: *mlir.Context, str: []const u8, args: []const *const mlir.Value, token: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { + const tok = optional(token); + var segs: Segments = .{}; + segs.add(args); + segs.add(tok.slice()); + return mlir.Operation.make(ctx, opName("print_tko"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{tokenType(ctx)} }, + .attributes = &.{ .named(ctx, "str", .string(ctx, str)), segs.attr(ctx) }, + .location = location, + }); +} + +// ============================================================================= +// Tests +// ============================================================================= + +test { + std.testing.refAllDecls(@This()); +} + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + mlir.DialectHandle.fromString(dialect_namespace).insertDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +test "dialect registers and types round-trip" { + const ctx = try testContext(); + defer ctx.deinit(); + + try std.testing.expect(ctx.isRegisteredOperation("cuda_tile.entry")); + try std.testing.expect(ctx.isRegisteredOperation("cuda_tile.load_view_tko")); + try std.testing.expect(ctx.isRegisteredOperation("cuda_tile.mmaf")); + + const f32_ty = mlir.Type.float(ctx, .f32); + const tile = tileType(ctx, &.{ 64, 64 }, f32_ty); + try std.testing.expect(tile.eql(try mlir.Type.parse(ctx, "!cuda_tile.tile<64x64xf32>"))); + try std.testing.expectEqual(@as(usize, 2), tile.isA(TileType).?.rank()); + try std.testing.expectEqual(@as(i64, 64), tile.isA(TileType).?.dimension(1)); + + const ptr = pointerType(ctx, f32_ty); + try std.testing.expect(ptr.eql(try mlir.Type.parse(ctx, "!cuda_tile.ptr"))); + try std.testing.expect(ptr.isA(PointerType).?.pointee().eql(f32_ty)); + + const dyn = TensorViewType.dynamic(); + const tv = tensorViewType(ctx, f32_ty, &.{ dyn, 32 }, &.{ dyn, 1 }); + try std.testing.expect(tv.eql(try mlir.Type.parse(ctx, "!cuda_tile.tensor_view"))); + + const static_tv = tensorViewType(ctx, f32_ty, &.{16}, &.{1}); + const pv: *const mlir.Type = @ptrCast(PartitionViewType.get(ctx, &.{2}, static_tv, &.{}, .zero)); + try std.testing.expect(pv.eql(try mlir.Type.parse(ctx, "!cuda_tile.partition_view>"))); + try std.testing.expect(tileTypeOfView(pv).eql(tileType(ctx, &.{2}, f32_ty))); + try std.testing.expectEqual(@as(usize, 1), indexRankOfView(pv)); + + try std.testing.expect(tokenType(ctx).eql(try mlir.Type.parse(ctx, "!cuda_tile.token"))); +} + +test "enum and hint attributes" { + const ctx = try testContext(); + defer ctx.deinit(); + + try std.testing.expect(RoundingMode.nearest_even.attribute(ctx).eql(try mlir.Attribute.parse(ctx, "#cuda_tile.rounding"))); + const padding = PaddingValue.neg_inf.attribute(ctx); + const padding_text = c.mlirCudaTilePaddingValueAttrGetValue(padding.ptr()); + try std.testing.expectEqualStrings("neg_inf", padding_text.data[0..padding_text.length]); + + const hints = optimizationHints(ctx, &[_]EntryHint{ + .{ .arch = .sm_100, .num_cta_in_cga = 2 }, + .{ .arch = .sm_120, .num_cta_in_cga = 2, .occupancy = 2 }, + }); + try std.testing.expect(hints.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.optimization_hints"))); + + _ = divBy(ctx, 16, null, null); + _ = sameElements(ctx, &.{ 2, 4 }); + _ = bounded(ctx, 0, null); +} + +test "entry round-trip through print and parse" { + const ctx = try testContext(); + defer ctx.deinit(); + + const loc: *const mlir.Location = .unknown(ctx); + const module_op: *mlir.Module = .init(loc); + defer module_op.deinit(); + + const f32_ty = mlir.Type.float(ctx, .f32); + const ptr_tile = tileType(ctx, &.{}, pointerType(ctx, f32_ty)); + const block = mlir.Block.init(&.{ ptr_tile, ptr_tile }, &.{ loc, loc }); + + // out[i] = in[i] + 1 over 128 elements, the shape of cuda-tile's README example. + const idx_ty = tileType(ctx, &.{128}, .int(ctx, .i32)); + const offsets = iota(ctx, idx_ty, loc).appendTo(block); + const ptr1 = tileType(ctx, &.{1}, pointerType(ctx, f32_ty)); + const ptr128 = tileType(ctx, &.{128}, pointerType(ctx, f32_ty)); + const in_r = reshape(ctx, block.argument(0), ptr1, loc).appendTo(block); + const in_b = broadcast(ctx, in_r.result(0), ptr128, loc).appendTo(block); + const in_p = offset(ctx, in_b.result(0), offsets.result(0), loc).appendTo(block); + const f32_128 = tileType(ctx, &.{128}, f32_ty); + const loaded = load_ptr_tko(ctx, in_p.result(0), f32_128, .{}, loc).appendTo(block); + const one = constant(ctx, splatAttr(f32_128, floatElem(ctx, f32_ty, 1.0)), f32_128, loc).appendTo(block); + const sum = addf(ctx, loaded.result(0), one.result(0), .nearest_even, false, loc).appendTo(block); + const out_r = reshape(ctx, block.argument(1), ptr1, loc).appendTo(block); + const out_b = broadcast(ctx, out_r.result(0), ptr128, loc).appendTo(block); + const out_p = offset(ctx, out_b.result(0), offsets.result(0), loc).appendTo(block); + _ = store_ptr_tko(ctx, out_p.result(0), sum.result(0), .{ .mem = .{ .token = loaded.result(1) } }, loc).appendTo(block); + _ = return_(ctx, &.{}, loc).appendTo(block); + + const ct_body = mlir.Block.init(&.{}, &.{}); + _ = entry(ctx, .{ .name = "add_one", .block = block, .location = loc }).appendTo(ct_body); + _ = module(ctx, "m", ct_body, loc).appendTo(module_op.body()); + + try std.testing.expect(module_op.operation().verify()); + + var al: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer al.deinit(); + try al.writer.print("{f}", .{module_op.operation()}); + const text = al.written(); + try std.testing.expect(std.mem.indexOf(u8, text, "cuda_tile.module @m") != null); + try std.testing.expect(std.mem.indexOf(u8, text, "entry @add_one(") != null); + try std.testing.expect(std.mem.indexOf(u8, text, "load_ptr_tko weak") != null); + try std.testing.expect(std.mem.indexOf(u8, text, "addf") != null); + + const parsed = try mlir.Module.parse(ctx, text); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "views, control flow and mma verify" { + const ctx = try testContext(); + defer ctx.deinit(); + + const loc: *const mlir.Location = .unknown(ctx); + const module_op: *mlir.Module = .init(loc); + defer module_op.deinit(); + + const bf16_ty = mlir.Type.float(ctx, .bf16); + const f32_ty = mlir.Type.float(ctx, .f32); + const i32_ty = mlir.Type.int(ctx, .i32); + const bf16_ptr = tileType(ctx, &.{}, pointerType(ctx, bf16_ty)); + const f32_ptr = tileType(ctx, &.{}, pointerType(ctx, f32_ty)); + const block = mlir.Block.init(&.{ bf16_ptr, bf16_ptr, f32_ptr }, &.{ loc, loc, loc }); + + // C[128x128] = A[128x64] @ B[64x128] in 64-wide K tiles. + const tv_a = make_tensor_view(ctx, block.argument(0), &.{}, &.{}, tensorViewType(ctx, bf16_ty, &.{ 128, 64 }, &.{ 64, 1 }), loc).appendTo(block); + const tv_b = make_tensor_view(ctx, block.argument(1), &.{}, &.{}, tensorViewType(ctx, bf16_ty, &.{ 64, 128 }, &.{ 128, 1 }), loc).appendTo(block); + const tv_c = make_tensor_view(ctx, block.argument(2), &.{}, &.{}, tensorViewType(ctx, f32_ty, &.{ 128, 128 }, &.{ 128, 1 }), loc).appendTo(block); + const pv_a: *const mlir.Type = @ptrCast(PartitionViewType.get(ctx, &.{ 64, 64 }, tv_a.result(0).type_(), &.{}, .zero)); + const pv_b: *const mlir.Type = @ptrCast(PartitionViewType.get(ctx, &.{ 64, 64 }, tv_b.result(0).type_(), &.{}, .zero)); + const pv_c: *const mlir.Type = @ptrCast(PartitionViewType.get(ctx, &.{ 64, 64 }, tv_c.result(0).type_(), &.{}, null)); + const va = make_partition_view(ctx, tv_a.result(0), pv_a, loc).appendTo(block); + const vb = make_partition_view(ctx, tv_b.result(0), pv_b, loc).appendTo(block); + const vc = make_partition_view(ctx, tv_c.result(0), pv_c, loc).appendTo(block); + + const bid = get_tile_block_id(ctx, loc).appendTo(block); + const i32_tile = tileType(ctx, &.{}, i32_ty); + const zero = constant(ctx, splatAttr(i32_tile, intElem(i32_ty, 0)), i32_tile, loc).appendTo(block); + const one = constant(ctx, splatAttr(i32_tile, intElem(i32_ty, 1)), i32_tile, loc).appendTo(block); + const acc_ty = tileType(ctx, &.{ 64, 64 }, f32_ty); + const acc0 = constant(ctx, splatAttr(acc_ty, floatElem(ctx, f32_ty, 0.0)), acc_ty, loc).appendTo(block); + + const body = mlir.Block.init(&.{ i32_tile, acc_ty }, &.{ loc, loc }); + const at = load_view_tko(ctx, va.result(0), &.{ bid.result(0), body.argument(0) }, .{}, loc).appendTo(body); + const bt = load_view_tko(ctx, vb.result(0), &.{ body.argument(0), bid.result(1) }, .{}, loc).appendTo(body); + const acc = mmaf(ctx, at.result(0), bt.result(0), body.argument(1), false, loc).appendTo(body); + _ = continue_(ctx, &.{acc.result(0)}, loc).appendTo(body); + const loop_op = for_(ctx, zero.result(0), one.result(0), one.result(0), &.{acc0.result(0)}, body, false, loc).appendTo(block); + + // if %cond -> (tile) { yield } else { yield } + const cond = cmpi(ctx, .equal, .signed, bid.result(0), zero.result(0), loc).appendTo(block); + const then_block = mlir.Block.init(&.{}, &.{}); + _ = yield(ctx, &.{loop_op.result(0)}, loc).appendTo(then_block); + const else_block = mlir.Block.init(&.{}, &.{}); + _ = yield(ctx, &.{acc0.result(0)}, loc).appendTo(else_block); + const if_op = if_(ctx, cond.result(0), &.{acc_ty}, then_block, else_block, loc).appendTo(block); + + _ = store_view_tko(ctx, if_op.result(0), vc.result(0), &.{ bid.result(0), bid.result(1) }, .{}, loc).appendTo(block); + _ = return_(ctx, &.{}, loc).appendTo(block); + + const ct_body = mlir.Block.init(&.{}, &.{}); + _ = entry(ctx, .{ + .name = "gemm", + .block = block, + .optimization_hints = optimizationHints(ctx, &[_]EntryHint{.{ .arch = .sm_120, .num_worker_warps_per_cta = 4 }}), + .location = loc, + }).appendTo(ct_body); + _ = module(ctx, "m", ct_body, loc).appendTo(module_op.body()); + + try std.testing.expect(module_op.operation().verify()); + + var al: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer al.deinit(); + try al.writer.print("{f}", .{module_op.operation()}); + const text = al.written(); + try std.testing.expect(std.mem.indexOf(u8, text, "optimization_hints=") != null); + try std.testing.expect(std.mem.indexOf(u8, text, "load_view_tko weak") != null); + try std.testing.expect(std.mem.indexOf(u8, text, "continue ") != null); + + const parsed = try mlir.Module.parse(ctx, text); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "the verifier rejects what the dialect forbids" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc: *const mlir.Location = .unknown(ctx); + const f32_ty = mlir.Type.float(ctx, .f32); + + // An entry with a rank-1 argument. (Per-op negatives go through + // `Operation.try_make`, whose logged error the test runner counts as a + // failure, so the module-level verifier is the oracle here.) + const module_op: *mlir.Module = .init(loc); + defer module_op.deinit(); + const block = mlir.Block.init(&.{tileType(ctx, &.{4}, f32_ty)}, &.{loc}); + _ = return_(ctx, &.{}, loc).appendTo(block); + const ct_body = mlir.Block.init(&.{}, &.{}); + _ = entry(ctx, .{ .name = "bad", .block = block, .location = loc }).appendTo(ct_body); + _ = module(ctx, "m", ct_body, loc).appendTo(module_op.body()); + try std.testing.expect(!module_op.operation().verify()); +} diff --git a/mlir/dialects/cuda_tile/cuda_tile_capi.cc b/mlir/dialects/cuda_tile/cuda_tile_capi.cc new file mode 100644 index 00000000..755be788 --- /dev/null +++ b/mlir/dialects/cuda_tile/cuda_tile_capi.cc @@ -0,0 +1,17 @@ +#include "cuda_tile_capi.h" + +#include "cuda_tile/Dialect/CudaTile/IR/Types.h" +#include "mlir/CAPI/IR.h" + +bool zmlCudaTileTypeIsATileView(MlirType type) { + return llvm::isa(unwrap(type)); +} + +MlirType zmlCudaTileTileViewGetViewTileType(MlirType type) { + return wrap(llvm::cast(unwrap(type)).getViewTileType()); +} + +intptr_t zmlCudaTileTileViewGetViewIndexRank(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getViewIndexRank()); +} diff --git a/mlir/dialects/cuda_tile/cuda_tile_capi.h b/mlir/dialects/cuda_tile/cuda_tile_capi.h new file mode 100644 index 00000000..4f0f5c70 --- /dev/null +++ b/mlir/dialects/cuda_tile/cuda_tile_capi.h @@ -0,0 +1,24 @@ +// C entry points for what NVIDIA's cuda_tile C API leaves out: the +// `TileView` type interface, shared by the partition, strided and +// gather/scatter views. +#ifndef ZML_MLIR_DIALECTS_CUDA_TILE_CAPI_H_ +#define ZML_MLIR_DIALECTS_CUDA_TILE_CAPI_H_ + +#include +#include + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool zmlCudaTileTypeIsATileView(MlirType type); +MlirType zmlCudaTileTileViewGetViewTileType(MlirType type); +intptr_t zmlCudaTileTileViewGetViewIndexRank(MlirType type); + +#ifdef __cplusplus +} +#endif + +#endif // ZML_MLIR_DIALECTS_CUDA_TILE_CAPI_H_ diff --git a/mlir/dialects/dialects.zig b/mlir/dialects/dialects.zig index 0c88eeaa..8646d840 100644 --- a/mlir/dialects/dialects.zig +++ b/mlir/dialects/dialects.zig @@ -1,5 +1,6 @@ const std = @import("std"); +pub const cuda_tile = @import("mlir/dialects/cuda_tile"); pub const mosaic_tpu = @import("mlir/dialects/mosaic_tpu"); pub const stablehlo = @import("mlir/dialects/stablehlo"); pub const ttir = @import("mlir/dialects/ttir"); diff --git a/third_party/cuda_tile/BUILD.bazel b/third_party/cuda_tile/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/third_party/cuda_tile/cuda_tile.bazel b/third_party/cuda_tile/cuda_tile.bazel new file mode 100644 index 00000000..b19b7a35 --- /dev/null +++ b/third_party/cuda_tile/cuda_tile.bazel @@ -0,0 +1,415 @@ +# XLA's third_party/cuda_tile/cuda_tile.BUILD, plus the C API library it does +# not build (cuda_tile_capi at the bottom). +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +cc_binary( + name = "cuda-tile-tblgen", + srcs = glob([ + "tools/cuda-tile-tblgen/*.cpp", + "tools/cuda-tile-tblgen/*.h", + ]), + deps = [ + "@llvm-project//llvm:Support", + "@llvm-project//llvm:TableGen", + "@llvm-project//mlir:MlirTableGenMain", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TableGen", + ], +) + +exports_files(["LICENSE"]) + +td_library( + name = "CudaTileTdFiles", + srcs = glob(["include/cuda_tile/Dialect/CudaTile/IR/*.td"]), + includes = ["include"], + deps = [ + "@llvm-project//mlir:BuiltinDialectTdFiles", + "@llvm-project//mlir:ControlFlowInterfacesTdFiles", + "@llvm-project//mlir:FunctionInterfacesTdFiles", + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + "@llvm-project//mlir:ViewLikeInterfaceTdFiles", + ], +) + +gentbl_cc_library( + name = "CudaTileDialectIncGen", + tbl_outs = [ + ( + [ + "-gen-dialect-decls", + "-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/Dialect.h.inc", + ), + ( + [ + "-gen-dialect-defs", + "-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/Dialect.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/Dialect.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileInterfacesIncGen", + tbl_outs = [ + ( + ["-gen-attr-interface-decls"], + "include/cuda_tile/Dialect/CudaTile/IR/AttrInterfaces.h.inc", + ), + ( + ["-gen-attr-interface-defs"], + "include/cuda_tile/Dialect/CudaTile/IR/AttrInterfaces.cpp.inc", + ), + ( + ["-gen-type-interface-decls"], + "include/cuda_tile/Dialect/CudaTile/IR/TypeInterfaces.h.inc", + ), + ( + ["-gen-type-interface-defs"], + "include/cuda_tile/Dialect/CudaTile/IR/TypeInterfaces.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/Interfaces.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileTypesIncGen", + tbl_outs = [ + ( + [ + "-gen-typedef-decls", + "-typedefs-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/Types.h.inc", + ), + ( + [ + "-gen-typedef-defs", + "-typedefs-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/Types.cpp.inc", + ), + ( + [ + "-gen-type-constraint-decls", + "-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/TypeConstraints.h.inc", + ), + ( + [ + "-gen-type-constraint-defs", + "-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/TypeConstraints.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/Types.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileAttrDefsIncGen", + tbl_outs = [ + ( + [ + "-gen-attrdef-decls", + "-attrdefs-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/AttrDefs.h.inc", + ), + ( + [ + "-gen-attrdef-defs", + "-attrdefs-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/AttrDefs.cpp.inc", + ), + ( + [ + "-gen-enum-decls", + "-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/Enums.h.inc", + ), + ( + [ + "-gen-enum-defs", + "-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/Enums.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/AttrDefs.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileRemarksIncGen", + tbl_outs = [ + ( + ["-gen-enum-decls"], + "include/cuda_tile/Dialect/CudaTile/IR/TileIRRemarks.h.inc", + ), + ( + ["-gen-enum-defs"], + "include/cuda_tile/Dialect/CudaTile/IR/TileIRRemarks.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/Remarks.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileOpsIncGen", + tbl_outs = [ + ( + ["-gen-op-decls"], + "include/cuda_tile/Dialect/CudaTile/IR/Ops.h.inc", + ), + ( + ["-gen-op-defs"], + "include/cuda_tile/Dialect/CudaTile/IR/Ops.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/Ops.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileOpsCanonicalizationIncGen", + tbl_outs = [ + ( + ["-gen-rewriters"], + "lib/Dialect/CudaTile/IR/OpsCanonicalization.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "lib/Dialect/CudaTile/IR/OpsCanonicalization.td", + deps = [ + ":CudaTileTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + ], +) + +cc_library( + name = "CudaTileDialect", + srcs = glob( + ["lib/Dialect/CudaTile/IR/*.cpp"], + exclude = ["lib/Dialect/CudaTile/IR/CudaTileTesting.cpp"], + ), + hdrs = glob(["include/cuda_tile/Dialect/CudaTile/IR/*.h"]), + includes = [ + "include", + "lib/Dialect/CudaTile/IR", + ], + visibility = ["//visibility:public"], + deps = [ + ":CudaTileAttrDefsIncGen", + ":CudaTileDialectIncGen", + ":CudaTileInterfacesIncGen", + ":CudaTileOpsCanonicalizationIncGen", + ":CudaTileOpsIncGen", + ":CudaTileTypesIncGen", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:BytecodeOpInterface", + "@llvm-project//mlir:ControlFlowInterfaces", + "@llvm-project//mlir:FunctionInterfaces", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:InliningUtils", + "@llvm-project//mlir:SideEffectInterfaces", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:ViewLikeInterface", + ], +) + +td_library( + name = "CudaTileTransformsTdFiles", + srcs = ["include/cuda_tile/Dialect/CudaTile/Transforms/Passes.td"], + includes = ["include"], + deps = [ + "@llvm-project//mlir:PassBaseTdFiles", + ], +) + +gentbl_cc_library( + name = "CudaTileTransformsIncGen", + tbl_outs = [ + ( + [ + "-gen-pass-decls", + "-name=CudaTile", + ], + "include/cuda_tile/Dialect/CudaTile/Transforms/Passes.h.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/Transforms/Passes.td", + deps = [":CudaTileTransformsTdFiles"], +) + +cc_library( + name = "CudaTileTransforms", + srcs = glob(["lib/Dialect/CudaTile/Transforms/*.cpp"]), + hdrs = glob(["include/cuda_tile/Dialect/CudaTile/Transforms/*.h"]), + includes = ["include"], + visibility = ["//visibility:public"], + deps = [ + ":CudaTileDialect", + ":CudaTileTransformsIncGen", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:TransformUtils", + ], +) + +gentbl_cc_library( + name = "CudaTileBytecodeOpsIncGen", + tbl_outs = [ + ( + ["-gen-cuda-tile-bytecode"], + "include/cuda_tile/Bytecode/Writer/Bytecode.inc", + ), + ( + ["-gen-cuda-tile-bytecode-reader"], + "include/cuda_tile/Bytecode/Reader/BytecodeReader.inc", + ), + ], + tblgen = ":cuda-tile-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/Ops.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileBytecodeOpcodesIncGen", + tbl_outs = [ + ( + ["-gen-cuda-tile-opcodes"], + "include/cuda_tile/Bytecode/Common/StaticOpcodes.inc", + ), + ], + tblgen = ":cuda-tile-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/BytecodeOpcodes.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileBytecodeTypeIncGen", + tbl_outs = [ + ( + ["-gen-cuda-tile-type-bytecode"], + "include/cuda_tile/Bytecode/Writer/TypeBytecode.inc", + ), + ( + ["-gen-cuda-tile-type-bytecode-reader"], + "include/cuda_tile/Bytecode/Reader/TypeBytecodeReader.inc", + ), + ], + tblgen = ":cuda-tile-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/BytecodeTypeOpcodes.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileBytecodeAttrIncGen", + tbl_outs = [ + ( + ["-gen-cuda-tile-attr-bytecode"], + "include/cuda_tile/Bytecode/Writer/AttrBytecode.inc", + ), + ], + tblgen = ":cuda-tile-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/BytecodeAttrOpcodes.td", + deps = [":CudaTileTdFiles"], +) + +cc_library( + name = "CudaTileBytecode", + srcs = glob([ + "lib/Bytecode/**/*.cpp", + "lib/Bytecode/**/*.h", + ]), + hdrs = glob([ + "include/cuda_tile/Bytecode/**/*.h", + ]), + includes = [ + "include", + "include/cuda_tile/Bytecode/Common", + "include/cuda_tile/Bytecode/Reader", + "include/cuda_tile/Bytecode/Writer", + ], + visibility = ["//visibility:public"], + deps = [ + ":CudaTileBytecodeAttrIncGen", + ":CudaTileBytecodeOpcodesIncGen", + ":CudaTileBytecodeOpsIncGen", + ":CudaTileBytecodeTypeIncGen", + ":CudaTileDialect", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:TranslateLib", + ], +) + +cc_library( + name = "CudaTileOptimizer", + srcs = ["lib/Dialect/CudaTile/Optimizer/CudaTileOptimizer.cpp"], + hdrs = ["include/cuda_tile/Dialect/CudaTile/Optimizer/CudaTileOptimizer.h"], + includes = ["include"], + visibility = ["//visibility:public"], + deps = [ + ":CudaTileBytecode", + ":CudaTileDialect", + ":CudaTileTransforms", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Parser", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:Transforms", + ], +) + +# The MLIR C API for the dialect: types, enum attributes, bytecode, passes, and +# the dialect handle `mlirGetDialectHandle__cuda_tile__` that ZML resolves at +# comptime. Mirrors lib/CAPI/CMakeLists.txt + lib/CAPI/Dialect/CMakeLists.txt. +cc_library( + name = "cuda_tile_capi", + srcs = [ + "lib/CAPI/Dialect/CudaTileDialect.cpp", + "lib/CAPI/Dialect/CudaTileOptimizer.cpp", + "lib/CAPI/Registration.cpp", + ], + hdrs = glob(["include/cuda_tile-c/**/*.h"]), + includes = ["include"], + visibility = ["//visibility:public"], + deps = [ + ":CudaTileBytecode", + ":CudaTileDialect", + ":CudaTileOptimizer", + ":CudaTileTransforms", + "@llvm-project//mlir:CAPIIR", + ], + # MLIR_DEFINE_CAPI_DIALECT_REGISTRATION expands to a global static + # initializer that the linker would otherwise dead-strip. + alwayslink = True, +) diff --git a/third_party/cuda_tile/repo.bzl b/third_party/cuda_tile/repo.bzl new file mode 100644 index 00000000..59e8d6ab --- /dev/null +++ b/third_party/cuda_tile/repo.bzl @@ -0,0 +1,10 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +# v13.3.3 +def repo(): + git_repository( + name = "cuda_tile", + remote = "https://github.com/NVIDIA/cuda-tile.git", + commit = "af2417041cc939b87ef56d92cfdcf61737c5457e", + build_file = "//third_party/cuda_tile:cuda_tile.bazel", + ) diff --git a/third_party/non_module_deps.bzl b/third_party/non_module_deps.bzl index de6bdc6e..9b8d6530 100644 --- a/third_party/non_module_deps.bzl +++ b/third_party/non_module_deps.bzl @@ -1,6 +1,7 @@ load("//third_party/arocc:repo.bzl", arocc = "repo") load("//third_party/cloud_accelerator_diagnostics:repo.bzl", cloud_accelerator_diagnostics = "repo") load("//third_party/com_google_sentencepiece:repo.bzl", com_google_sentencepiece = "repo") +load("//third_party/cuda_tile:repo.bzl", cuda_tile = "repo") load("//third_party/flashattn:repo.bzl", flashattn = "repo") load("//third_party/flashinfer_cutlass_moe:repo.bzl", flashinfer_cutlass_moe = "repo") load("//third_party/iree:repo.bzl", iree = "repo") @@ -32,6 +33,7 @@ def _non_module_deps_impl(mctx): libvaxis() mosaic_tpu() iree() + cuda_tile() return mctx.extension_metadata( reproducible = True, diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 9f49801d..8a352104 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -146,10 +146,12 @@ zig_library( ":xla_data_upb", "//bazel", "//ffi:zig", + "//kernels/cuda_tile:cuda_tile_builder", "//kernels/mosaic_tpu:mosaic_tpu_builder", "//kernels/triton:triton_builder", "//mlir", "//mlir/dialects", + "//mlir/dialects/cuda_tile", "//mlir/dialects/mosaic_tpu", "//pjrt", "//platforms", diff --git a/zml/kernel.zig b/zml/kernel.zig index b6791b8b..07d7f21b 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -3,6 +3,7 @@ const std = @import("std"); const stdx = @import("stdx"); const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); +const cuda_tile_builder = @import("kernels/cuda_tile/builder"); const mosaic_tpu_builder = @import("kernels/mosaic_tpu/builder"); const tpu_dialect = @import("mlir/dialects/mosaic_tpu"); const triton_builder = @import("kernels/triton/builder"); @@ -388,3 +389,134 @@ pub const mosaic_tpu = struct { ).appendTo(cur.currentScope().block); } }; + +pub const cuda_tile = struct { + pub const Builder = cuda_tile_builder.Builder; + pub const Value = cuda_tile_builder.Value; + pub const DType = cuda_tile_builder.DType; + pub const FinishError = cuda_tile_builder.FinishError; + pub const EntryHint = cuda_tile_builder.EntryHint; + pub const Arch = cuda_tile_builder.Arch; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(&cuda_tile_builder.dialects_needed); + } + + pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i4, .u4 => .i4, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + .f8e8m0 => .f8e8m0fnu, + .f4e2m1 => .f4e2m1fn, + else => std.debug.panic("zml.kernel.cuda_tile.from: dtype {s} has no CUDA Tile IR equivalent", .{@tagName(dt)}), + }; + } + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + /// No `num_warps`/`num_stages`: warp and CTA tuning is + /// `optimization_hints` inside the IR (`Opts.hints`). + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + /// Outputs XLA zeroes before the launch, by position in + /// `spec.outputs`, ascending. + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cuda_tile_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finish(&.{}); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.cuda_tile.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.cudaTile(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .zeroed_outputs = opts.zeroed_outputs, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; + } +}; + +test "cuda_tile kernel emits a module XLA can parse" { + const Cfg = struct { n: i64 }; + const AddOne = cuda_tile.Kernel(Cfg, .{ + .name = "add_one", + .inputs = &.{"x"}, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *cuda_tile.Builder, cfg: Cfg) cuda_tile.FinishError!void { + const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .out = .{ .ptr = .f32 } }); + const vx = b.partitionView(b.tensorView(a.x, &.{cfg.n}, &.{1}), &.{128}, .{}); + const vo = b.partitionView(b.tensorView(a.out, &.{cfg.n}, &.{1}), &.{128}, .{}); + const bid = b.tileBlockId(); + _ = b.store(b.load(vx, &.{bid.x}).add(1.0), vo, &.{bid.x}); + } + }.run, + }); + + const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1024 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.module @add_one") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "entry @add_one(") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "load_view_tko") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "store_view_tko") != null); +} diff --git a/zml/ops.zig b/zml/ops.zig index 2dccdb25..8a095cce 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1086,6 +1086,260 @@ test "triton" { try std.testing.expectEqual(expected_result_b, cpu_result_1.items(f32)[0]); } +pub const CudaTileOps = struct { + name: [:0]const u8, + ir: [:0]const u8, + grid: [3]i32, + /// The Tile IR bytecode version XLA serializes at, "MAJOR.MINOR"; XLA's + /// default is 13.3. + ir_version: ?[]const u8 = null, + /// Result indices XLA zeroes before the launch, ascending. For a kernel + /// that accumulates, or writes less than the whole output. + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: []const dialects.stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, +}; + +/// A `__gpu$xla.gpu.cuda_tile` custom call: XLA assembles `ir` with tileiras +/// and launches `name` over `grid` tile blocks. Every operand and result +/// arrives as one raw device pointer, in order, in the default layout. +pub fn cudaTile(inputs: anytype, outputs: anytype, opts: CudaTileOps) [outputs.len]Tensor { + const mlir_ctx = Compiler.current().mlir_ctx; + + var values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + values[i] = inputs[i].value(); + } + + var res_types: [outputs.len]*const mlir.Type = undefined; + inline for (outputs, 0..) |output, i| { + res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); + } + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 10) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), + .named(mlir_ctx, "kernel_type", .string(mlir_ctx, "cuda_tile")), + .named(mlir_ctx, "ir", .string(mlir_ctx, opts.ir)), + .named(mlir_ctx, "grid_x", .int(mlir_ctx, .i32, opts.grid[0])), + .named(mlir_ctx, "grid_y", .int(mlir_ctx, .i32, opts.grid[1])), + .named(mlir_ctx, "grid_z", .int(mlir_ctx, .i32, opts.grid[2])), + }); + if (opts.ir_version) |v| attrs.appendAssumeCapacity(.named(mlir_ctx, "ir_version", .string(mlir_ctx, v))); + if (opts.zeroed_outputs.len > 0) { + var zeroed: stdx.BoundedArray(*const mlir.Attribute, dialects.stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + for (opts.zeroed_outputs) |i| zeroed.appendAssumeCapacity(.int(mlir_ctx, .i32, i)); + attrs.appendAssumeCapacity(.named(mlir_ctx, "zeroed_outputs", .array(mlir_ctx, zeroed.constSlice()))); + } + const backend_config: *const mlir.Attribute = .dict(mlir_ctx, attrs.constSlice()); + + const op = dialects.stablehlo.custom_call( + mlir_ctx, + &values, + &res_types, + .{ + .call_target_name = "__gpu$xla.gpu.cuda_tile", + .backend_config = .{ .typed_ffi = backend_config }, + .has_side_effect = false, + .output_operand_aliases = opts.output_operand_aliases, + }, + .unknown(mlir_ctx), + ).appendTo(Compiler.current().currentScope().block); + + var outputs_: [outputs.len]Tensor = undefined; + inline for (outputs, 0..) |output, i| { + outputs_[i] = Tensor._result(output, op.result(i)); + } + + return outputs_; +} + +test "cuda_tile" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + if (platform.target != .cuda) return error.SkipZigTest; + + // out[i] = in[i] + 1 over 128 f32; the XLA milestone kernel. + const ir = + \\cuda_tile.module @m { + \\ entry @add_one(%in : tile>, %out : tile>) { + \\ %offsets = iota : tile<128xi32> + \\ %in_r = reshape %in : tile> -> tile<1xptr> + \\ %in_b = broadcast %in_r : tile<1xptr> -> tile<128xptr> + \\ %in_p = offset %in_b, %offsets : tile<128xptr>, tile<128xi32> -> tile<128xptr> + \\ %v, %t0 = load_ptr_tko weak %in_p : tile<128xptr> -> tile<128xf32>, token + \\ %one = constant : tile + \\ %one_r = reshape %one : tile -> tile<1xf32> + \\ %one_b = broadcast %one_r : tile<1xf32> -> tile<128xf32> + \\ %sum = addf %v, %one_b rounding : tile<128xf32> + \\ %out_r = reshape %out : tile> -> tile<1xptr> + \\ %out_b = broadcast %out_r : tile<1xptr> -> tile<128xptr> + \\ %out_p = offset %out_b, %offsets : tile<128xptr>, tile<128xi32> -> tile<128xptr> + \\ %t1 = store_ptr_tko weak %out_p, %sum : tile<128xptr>, tile<128xf32> -> token + \\ return + \\ } + \\} + ; + + const Mod = struct { + pub fn forward(a: Tensor) Tensor { + return cudaTile(.{a}, .{a.shape()}, .{ + .name = "add_one", + .ir = ir, + .grid = .{ 1, 1, 1 }, + })[0]; + } + }; + + const a: zml.Tensor = .init(.{ .n = 128 }, .f32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Mod.forward, .{a}, platform, .{}); + defer exe.deinit(); + + var input: [128]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @floatFromInt(i); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&input)); + defer a_buffer.deinit(); + + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Mod.forward, .{a_buffer}); + defer result.deinit(); + + var host = try result.toSliceAlloc(std.testing.allocator, std.testing.io); + defer host.free(std.testing.allocator); + + for (host.items(f32), 0..) |v, i| { + try std.testing.expectEqual(@as(f32, @floatFromInt(i + 1)), v); + } +} + +test "cuda_tile grid" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + if (platform.target != .cuda) return error.SkipZigTest; + + // Each of the 2x3x4 tile blocks writes its linear id t = x + 2y + 6z + // over out[128*t .. 128*t+128); the input is unused. + const ir = + \\cuda_tile.module @m { + \\ entry @grid(%in : tile>, %out : tile>) { + \\ %bx, %by, %bz = get_tile_block_id : tile + \\ %c2 = constant : tile + \\ %c6 = constant : tile + \\ %c128 = constant : tile + \\ %y2 = muli %by, %c2 : tile + \\ %z6 = muli %bz, %c6 : tile + \\ %xy = addi %bx, %y2 : tile + \\ %t = addi %xy, %z6 : tile + \\ %base = muli %t, %c128 : tile + \\ %offsets = iota : tile<128xi32> + \\ %base_r = reshape %base : tile -> tile<1xi32> + \\ %base_b = broadcast %base_r : tile<1xi32> -> tile<128xi32> + \\ %idx = addi %offsets, %base_b : tile<128xi32> + \\ %out_r = reshape %out : tile> -> tile<1xptr> + \\ %out_b = broadcast %out_r : tile<1xptr> -> tile<128xptr> + \\ %out_p = offset %out_b, %idx : tile<128xptr>, tile<128xi32> -> tile<128xptr> + \\ %tf = itof %t signed rounding : tile -> tile + \\ %tf_r = reshape %tf : tile -> tile<1xf32> + \\ %tf_b = broadcast %tf_r : tile<1xf32> -> tile<128xf32> + \\ %tok = store_ptr_tko weak %out_p, %tf_b : tile<128xptr>, tile<128xf32> -> token + \\ return + \\ } + \\} + ; + + const Mod = struct { + pub fn forward(a: Tensor) Tensor { + return cudaTile(.{a}, .{a.shape()}, .{ + .name = "grid", + .ir = ir, + .grid = .{ 2, 3, 4 }, + })[0]; + } + }; + + const n = 24 * 128; + const a: zml.Tensor = .init(.{ .n = n }, .f32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Mod.forward, .{a}, platform, .{}); + defer exe.deinit(); + + const input = [_]f32{0} ** n; + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&input)); + defer a_buffer.deinit(); + + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Mod.forward, .{a_buffer}); + defer result.deinit(); + + var host = try result.toSliceAlloc(std.testing.allocator, std.testing.io); + defer host.free(std.testing.allocator); + + for (host.items(f32), 0..) |v, i| { + try std.testing.expectEqual(@as(f32, @floatFromInt(i / 128)), v); + } +} + +test "cuda_tile zeroed_outputs" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + if (platform.target != .cuda) return error.SkipZigTest; + + // Writes only the first 64 of 128; `zeroed_outputs` owns the rest. + const ir = + \\cuda_tile.module @m { + \\ entry @half(%in : tile>, %out : tile>) { + \\ %offsets = iota : tile<64xi32> + \\ %in_r = reshape %in : tile> -> tile<1xptr> + \\ %in_b = broadcast %in_r : tile<1xptr> -> tile<64xptr> + \\ %in_p = offset %in_b, %offsets : tile<64xptr>, tile<64xi32> -> tile<64xptr> + \\ %v, %t0 = load_ptr_tko weak %in_p : tile<64xptr> -> tile<64xf32>, token + \\ %one = constant : tile + \\ %one_r = reshape %one : tile -> tile<1xf32> + \\ %one_b = broadcast %one_r : tile<1xf32> -> tile<64xf32> + \\ %sum = addf %v, %one_b rounding : tile<64xf32> + \\ %out_r = reshape %out : tile> -> tile<1xptr> + \\ %out_b = broadcast %out_r : tile<1xptr> -> tile<64xptr> + \\ %out_p = offset %out_b, %offsets : tile<64xptr>, tile<64xi32> -> tile<64xptr> + \\ %tok = store_ptr_tko weak %out_p, %sum : tile<64xptr>, tile<64xf32> -> token + \\ return + \\ } + \\} + ; + + const Mod = struct { + pub fn forward(a: Tensor) Tensor { + return cudaTile(.{a}, .{a.shape()}, .{ + .name = "half", + .ir = ir, + .grid = .{ 1, 1, 1 }, + .zeroed_outputs = &.{0}, + })[0]; + } + }; + + const a: zml.Tensor = .init(.{ .n = 128 }, .f32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Mod.forward, .{a}, platform, .{}); + defer exe.deinit(); + + var input: [128]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @floatFromInt(i); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&input)); + defer a_buffer.deinit(); + + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Mod.forward, .{a_buffer}); + defer result.deinit(); + + var host = try result.toSliceAlloc(std.testing.allocator, std.testing.io); + defer host.free(std.testing.allocator); + + for (host.items(f32), 0..) |v, i| { + const want: f32 = if (i < 64) @floatFromInt(i + 1) else 0; + try std.testing.expectEqual(want, v); + } +} + pub const ScatterArgs = struct { input: Tensor, update: Tensor, From afb3559f7fd8ca4907ae5c9c37f03d326767c51b Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 8 Sep 2026 15:56:58 +0200 Subject: [PATCH 666/881] zml/moe: apply activation thresholds (#781) --- zml/moe/triton.zig | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/zml/moe/triton.zig b/zml/moe/triton.zig index 28c92be0..ed9b26af 100644 --- a/zml/moe/triton.zig +++ b/zml/moe/triton.zig @@ -110,10 +110,15 @@ fn initZeroBiasBuffer(io: std.Io, platform: *const zml.Platform, sharding: zml.S return zml.Buffer.fromSlice(io, platform, zero_slice, sharding); } -fn applyActivation(x: Tensor, mode: Parameters.ActivationMode) Tensor { +fn applyActivation(x: Tensor, mode: Parameters.ActivationMode, activation_threshold: ?f32) Tensor { const mid = @divFloor(x.dim(.out), 2); - const gate = x.slice(.out, .{ .end = mid }); - const up = x.slice(.out, .{ .start = mid }); + var gate = x.slice(.out, .{ .end = mid }); + var up = x.slice(.out, .{ .start = mid }); + if (activation_threshold) |limit_| { + const limit = Tensor.scalar(limit_, x.dtype()); + gate = gate.minimum(limit); + up = up.clamp(limit.negate(), limit); + } return switch (mode) { .silu => gate.silu().mul(up), .relu => x.relu().powByConst(2), @@ -239,8 +244,7 @@ pub fn fusedExpertsImpl( Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), ); - const activated = applyActivation(first_out, options.activation); - + const activated = applyActivation(first_out, options.activation, options.activation_threshold); var activated_quant = activated; a_scale = opts.a2_scale orelse Tensor.scalar(1.0, .f32); if (down.dtype() == .f8e4m3fn) { From 2e5150eb0fbacd8e49b1d794bc07bcc395dbfa0e Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 9 Sep 2026 09:01:10 +0200 Subject: [PATCH 667/881] stdx/json: {Union,TaggedUnion}Helpers (#770) --- examples/llm/models/llama/model.zig | 15 ++- examples/llm/models/llama/session.zig | 2 +- stdx/json.zig | 185 +++++++++++++++++--------- 3 files changed, 134 insertions(+), 68 deletions(-) diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index 0b51b8b7..fd573c32 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -10,10 +10,7 @@ const log = std.log.scoped(.llama); pub const Config = struct { bos_token_id: u32, - eos_token_id: stdx.json.Union(union(enum) { - int: u32, - ints: []u32, - }), + eos_token_id: EosTokens, head_dim: ?u32 = null, hidden_size: u32, num_hidden_layers: u32, @@ -25,6 +22,16 @@ pub const Config = struct { hf_rope_impl: bool = true, tie_word_embeddings: bool = false, rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = .{} }, + + pub const EosTokens = union(enum) { + int: u32, + ints: []u32, + + const Helpers = stdx.json.UnionHelpers(@This()); + pub const jsonParse = Helpers.jsonParse; + pub const jsonParseFromValue = Helpers.jsonParseFromValue; + pub const jsonStringify = Helpers.jsonStringify; + }; }; const Options = struct { diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig index d86a8728..98187a42 100644 --- a/examples/llm/models/llama/session.zig +++ b/examples/llm/models/llama/session.zig @@ -227,7 +227,7 @@ pub const Session = struct { }; fn isEosToken(config: *const model.Config, token_id: u32) bool { - return switch (config.eos_token_id.value) { + return switch (config.eos_token_id) { .int => |eos| token_id == eos, .ints => |eos_list| for (eos_list) |eos| { if (token_id == eos) break true; diff --git a/stdx/json.zig b/stdx/json.zig index dd85eb95..d1143efc 100644 --- a/stdx/json.zig +++ b/stdx/json.zig @@ -17,22 +17,23 @@ const ParseFromValueError = std.json.ParseFromValueError; /// /// ```zig /// const Answer = union { -/// number: i32, -/// numbers: []const i32, +/// number: i32, +/// numbers: []const i32, +/// +/// const Helpers = stdx.json.UnionHelpers(@This()); +/// pub const jsonParse = Helpers.jsonParse; +/// pub const jsonParseFromValue = Helpers.jsonParseFromValue; +/// pub const jsonStringify = Helpers.jsonStringify; /// }; /// /// const Message = struct { /// question: []const u8; -/// answer: stdx.json.Union(Answer); +/// answer: Answer; /// } /// ``` -pub fn Union(comptime T: type) type { +pub fn UnionHelpers(comptime T: type) type { return struct { - const Self = @This(); - - value: T, - - pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!Self { + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!T { return jsonParseFromValue( allocator, try std.json.innerParse(std.json.Value, allocator, source, options), @@ -40,27 +41,23 @@ pub fn Union(comptime T: type) type { ); } - pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!Self { + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!T { inline for (std.meta.fields(T)) |field| { switch (field.type) { - bool => if (source == .bool) return .{ .value = @unionInit(T, field.name, source.bool) }, + bool => if (source == .bool) return @unionInit(T, field.name, source.bool), []const u8 => switch (source) { - .string => |v| { + .string, .number_string => |v| { const dupe = try allocator.dupe(u8, v); - return .{ .value = @unionInit(T, field.name, dupe) }; - }, - .number_string => |v| { - const dupe = try allocator.dupe(u8, v); - return .{ .value = @unionInit(T, field.name, dupe) }; + return @unionInit(T, field.name, dupe); }, else => {}, }, else => switch (@typeInfo(field.type)) { - .int => if (source == .integer) return .{ .value = @unionInit(T, field.name, @intCast(source.integer)) }, - .float => if (source == .float) return .{ .value = @unionInit(T, field.name, @floatCast(source.float)) }, - .@"struct" => if (source == .object) return .{ .value = @unionInit(T, field.name, try std.json.innerParseFromValue(field.type, allocator, source.object, options)) }, + .int => if (source == .integer) return @unionInit(T, field.name, @intCast(source.integer)), + .float => if (source == .float) return @unionInit(T, field.name, @floatCast(source.float)), + .@"struct" => if (source == .object) return @unionInit(T, field.name, try std.json.innerParseFromValue(field.type, allocator, source, options)), inline else => switch (source) { - .number_string, .array => return .{ .value = @unionInit(T, field.name, try std.json.innerParseFromValue(field.type, allocator, source, options)) }, + .number_string, .array => return @unionInit(T, field.name, try std.json.innerParseFromValue(field.type, allocator, source, options)), else => {}, }, }, @@ -69,14 +66,38 @@ pub fn Union(comptime T: type) type { return error.UnknownField; } - pub fn jsonStringify(self: Self, jw: anytype) !void { - return switch (self.value) { + pub fn jsonStringify(self: T, jw: anytype) !void { + return switch (self) { inline else => |v| jw.write(v), }; } }; } +/// Deprecated: prefer importing `stdx.json.UnionHelpers` method into your union. +/// This will remove the nesting required by `stdx.json.Union`. +pub fn Union(comptime T: type) type { + return struct { + const Self = @This(); + + value: T, + + const Helpers = UnionHelpers(T); + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!Self { + return .{ .value = try Helpers.jsonParse(allocator, source, options) }; + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!Self { + return .{ .value = try Helpers.jsonParseFromValue(allocator, source, options) }; + } + + pub fn jsonStringify(self: Self, jw: anytype) !void { + return try Helpers.jsonStringify(self.value, jw); + } + }; +} + /// Handle json fields that can have different Zig types depending on another field in the same message. /// This is translated to a Zig tagged union. /// @@ -93,13 +114,18 @@ pub fn Union(comptime T: type) type { /// /// ```zig /// const Entry = union { -/// faq: struct { question: []const u8, answer: u32 }, -/// address: struct { city: []const u8, zipcode: []const u8 }, +/// faq: struct { question: []const u8, answer: u32 }, +/// address: struct { city: []const u8, zipcode: []const u8 }, +/// +/// const Helpers = stdx.json.TaggedUnionHelpers(@This(), "type"); +/// pub const jsonParse = Helpers.jsonParse; +/// pub const jsonParseFromValue = Helpers.jsonParseFromValue; +/// pub const jsonStringify = Helpers.jsonStringify; /// }; /// -/// const Message = []const stdx.json.TaggedUnion(Entry, "type"); +/// const Message = []const Entry; /// ``` -pub fn TaggedUnion(comptime T: type, comptime tag_name: [:0]const u8) type { +pub fn TaggedUnionHelpers(comptime T: type, comptime tag_name: [:0]const u8) type { comptime { if (@typeInfo(T) != .@"union") { @compileError("TaggedUnion expects a union type, found " ++ @typeName(T)); @@ -112,11 +138,7 @@ pub fn TaggedUnion(comptime T: type, comptime tag_name: [:0]const u8) type { } return struct { - const Self = @This(); - - value: T, - - pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!Self { + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!T { return jsonParseFromValue( allocator, try std.json.innerParse(std.json.Value, allocator, source, options), @@ -124,7 +146,7 @@ pub fn TaggedUnion(comptime T: type, comptime tag_name: [:0]const u8) type { ); } - pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!Self { + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!T { errdefer std.log.warn("failed to parse: {} as {s}", .{ source, @typeName(T) }); if (source != .object) return error.UnexpectedToken; var o = source.object; @@ -136,19 +158,19 @@ pub fn TaggedUnion(comptime T: type, comptime tag_name: [:0]const u8) type { std.log.warn("failed to interpret {s} as a {s}: {}", .{ tag.string, @typeName(field.type), err }); return err; }; - return .{ .value = @unionInit(T, field.name, inner) }; + return @unionInit(T, field.name, inner); } } return error.InvalidEnumTag; } - pub fn jsonStringify(self: Self, jw: anytype) !void { + pub fn jsonStringify(self: T, jw: anytype) !void { try jw.beginObject(); - switch (self.value) { + switch (self) { inline else => |v, tag| { - const field_name = @tagName(tag); + // emit a field containing the enum tag. try jw.objectField(tag_name); - try jw.write(field_name); + try jw.write(@tagName(tag)); switch (@typeInfo(@TypeOf(v))) { .@"struct" => inline for (std.meta.fields(@TypeOf(v))) |field| { try jw.objectField(field.name); @@ -163,6 +185,30 @@ pub fn TaggedUnion(comptime T: type, comptime tag_name: [:0]const u8) type { }; } +/// Deprecated: prefer importing `stdx.json.TaggedUnionHelpers` method into your union. +/// This will remove the nesting required by `stdx.json.TaggedUnion`. +pub fn TaggedUnion(comptime T: type, comptime tag_name: [:0]const u8) type { + return struct { + const Self = @This(); + + value: T, + + const Helpers = TaggedUnionHelpers(T, tag_name); + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!Self { + return .{ .value = try Helpers.jsonParse(allocator, source, options) }; + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!Self { + return .{ .value = try Helpers.jsonParseFromValue(allocator, source, options) }; + } + + pub fn jsonStringify(self: Self, jw: anytype) !void { + return try Helpers.jsonStringify(self.value, jw); + } + }; +} + pub fn NeverNull(comptime T: type, comptime default_value: T) type { return struct { const Self = @This(); @@ -201,11 +247,16 @@ test "union" { flag: bool, ratio: f64, text: []const u8, + + const Helpers = UnionHelpers(@This()); + pub const jsonParse = Helpers.jsonParse; + pub const jsonParseFromValue = Helpers.jsonParseFromValue; + pub const jsonStringify = Helpers.jsonStringify; }; const Message = struct { question: []const u8, - answer: Union(Answer), + answer: Answer, }; var arena = std.heap.ArenaAllocator.init(std.testing.allocator); @@ -215,44 +266,49 @@ test "union" { const number_case = "{\"question\":\"How old are you?\",\"answer\":5}"; const parsed_number = try std.json.parseFromSliceLeaky(Message, allocator, number_case, .{}); try std.testing.expectEqualStrings("How old are you?", parsed_number.question); - try std.testing.expectEqual(@as(i32, 5), parsed_number.answer.value.number); + try std.testing.expectEqual(@as(i32, 5), parsed_number.answer.number); try std.testing.expectFmt(number_case, "{f}", .{std.json.fmt(parsed_number, .{})}); const numbers_case = "{\"question\":\"Count to three.\",\"answer\":[1,2,3]}"; const parsed_numbers = try std.json.parseFromSliceLeaky(Message, allocator, numbers_case, .{}); - try std.testing.expectEqual(@as(usize, 3), parsed_numbers.answer.value.numbers.len); - try std.testing.expectEqual(@as(i32, 1), parsed_numbers.answer.value.numbers[0]); - try std.testing.expectEqual(@as(i32, 2), parsed_numbers.answer.value.numbers[1]); - try std.testing.expectEqual(@as(i32, 3), parsed_numbers.answer.value.numbers[2]); + try std.testing.expectEqual(@as(usize, 3), parsed_numbers.answer.numbers.len); + try std.testing.expectEqual(@as(i32, 1), parsed_numbers.answer.numbers[0]); + try std.testing.expectEqual(@as(i32, 2), parsed_numbers.answer.numbers[1]); + try std.testing.expectEqual(@as(i32, 3), parsed_numbers.answer.numbers[2]); try std.testing.expectFmt(numbers_case, "{f}", .{std.json.fmt(parsed_numbers, .{})}); const flag_case = "{\"question\":\"Are you ready?\",\"answer\":true}"; const parsed_flag = try std.json.parseFromSliceLeaky(Message, allocator, flag_case, .{}); - try std.testing.expectEqual(true, parsed_flag.answer.value.flag); + try std.testing.expectEqual(true, parsed_flag.answer.flag); try std.testing.expectFmt(flag_case, "{f}", .{std.json.fmt(parsed_flag, .{})}); const ratio_case = "{\"question\":\"What is pi?\",\"answer\":3.5}"; const parsed_ratio = try std.json.parseFromSliceLeaky(Message, allocator, ratio_case, .{}); - try std.testing.expectEqual(@as(f64, 3.5), parsed_ratio.answer.value.ratio); + try std.testing.expectEqual(@as(f64, 3.5), parsed_ratio.answer.ratio); try std.testing.expectFmt(ratio_case, "{f}", .{std.json.fmt(parsed_ratio, .{})}); const text_case = "{\"question\":\"Your name?\",\"answer\":\"zml\"}"; const parsed_text = try std.json.parseFromSliceLeaky(Message, allocator, text_case, .{}); - try std.testing.expectEqualStrings("zml", parsed_text.answer.value.text); + try std.testing.expectEqualStrings("zml", parsed_text.answer.text); try std.testing.expectFmt(text_case, "{f}", .{std.json.fmt(parsed_text, .{})}); const null_value = try std.json.parseFromSliceLeaky(std.json.Value, allocator, "null", .{}); - try std.testing.expectError(error.UnknownField, Union(Answer).jsonParseFromValue(allocator, null_value, .{})); + try std.testing.expectError(error.UnknownField, Answer.jsonParseFromValue(allocator, null_value, .{})); } test "strings in union can be freed" { const Answer = union(enum) { string: []const u8, number: u32, + + const Helpers = UnionHelpers(@This()); + pub const jsonParse = Helpers.jsonParse; + pub const jsonParseFromValue = Helpers.jsonParseFromValue; + pub const jsonStringify = Helpers.jsonStringify; }; const Struct = struct { - answer1: Union(Answer), - answer2: Union(Answer), + answer1: Answer, + answer2: Answer, }; const struct_json = \\{"answer1":"I'm 29","answer2": 1e29} @@ -262,17 +318,17 @@ test "strings in union can be freed" { const parsed = try std.json.parseFromValueLeaky(Struct, std.testing.allocator, parsed_value.value, .{}); - try std.testing.expect(parsed.answer1.value == .string); - try std.testing.expectEqualStrings("I'm 29", parsed.answer1.value.string); - std.testing.allocator.free(parsed.answer1.value.string); + try std.testing.expect(parsed.answer1 == .string); + try std.testing.expectEqualStrings("I'm 29", parsed.answer1.string); + std.testing.allocator.free(parsed.answer1.string); - try std.testing.expect(parsed.answer2.value == .string); - try std.testing.expectEqualStrings("1e29", parsed.answer2.value.string); - std.testing.allocator.free(parsed.answer2.value.string); + try std.testing.expect(parsed.answer2 == .string); + try std.testing.expectEqualStrings("1e29", parsed.answer2.string); + std.testing.allocator.free(parsed.answer2.string); } test "tagged union" { - const Entry = union(enum) { + const Message = union(enum) { faq: struct { question: []const u8, answer: u32, @@ -281,9 +337,12 @@ test "tagged union" { city: []const u8, zipcode: []const u8, }, - }; - const Message = TaggedUnion(Entry, "type"); + const Helpers = TaggedUnionHelpers(@This(), "type"); + pub const jsonParse = Helpers.jsonParse; + pub const jsonParseFromValue = Helpers.jsonParseFromValue; + pub const jsonStringify = Helpers.jsonStringify; + }; var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); @@ -291,14 +350,14 @@ test "tagged union" { const faq_case = "{\"type\":\"faq\",\"question\":\"How old are you?\",\"answer\":5}"; const parsed_faq = try std.json.parseFromSliceLeaky(Message, allocator, faq_case, .{}); - try std.testing.expectEqualStrings("How old are you?", parsed_faq.value.faq.question); - try std.testing.expectEqual(@as(u32, 5), parsed_faq.value.faq.answer); + try std.testing.expectEqualStrings("How old are you?", parsed_faq.faq.question); + try std.testing.expectEqual(@as(u32, 5), parsed_faq.faq.answer); try std.testing.expectFmt(faq_case, "{f}", .{std.json.fmt(parsed_faq, .{})}); const address_case = "{\"type\":\"address\",\"city\":\"NYC\",\"zipcode\":\"49130\"}"; const parsed_address = try std.json.parseFromSliceLeaky(Message, allocator, address_case, .{}); - try std.testing.expectEqualStrings("NYC", parsed_address.value.address.city); - try std.testing.expectEqualStrings("49130", parsed_address.value.address.zipcode); + try std.testing.expectEqualStrings("NYC", parsed_address.address.city); + try std.testing.expectEqualStrings("49130", parsed_address.address.zipcode); try std.testing.expectFmt(address_case, "{f}", .{std.json.fmt(parsed_address, .{})}); } From 06074d81f6234b3dc6bd43fb25a8a788a0a07e89 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 9 Sep 2026 09:32:26 +0200 Subject: [PATCH 668/881] zml: support block-128 FP8 quantized input and f32 scale & add rocm computeCapability (#782) Imitating `platform.cuda.computeCapability` as fp8 support is native for e4m3fnuz on gfx942 but e4m3fn for everything else that has hardware support AFAIK. --- zml/nn.zig | 106 ++++++++++++++++++++++++++++++++++++++----- zml/platform.zig | 67 +++++++++++++++++++++++++++ zml/quantization.zig | 104 +++++++++++++++++++++++++++++++++++++++--- 3 files changed, 260 insertions(+), 17 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index 8a300897..d5582f5f 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -106,12 +106,16 @@ test "unpackFp4 expands the requested axis" { /// - **NVFP4**: values `.f4e2m1`, scales `.f8e4m3fn`, block 16 (weight-only bf16 lhs ok) /// - **MXFP4**: values `.f4e2m1`, scales `.f8e8m0fnu`, block 32 /// - **MXFP8**: values `.f8e4m3fn` / `.f8e5m2`, scales `.f8e8m0fnu`, block 32 -/// - TODO: INT4/8 and FP8 with block 128 and per tensor +/// - **Block FP8**: E4M3FN / E4M3FNUZ / E8M0 values, BF16 or F32 128x128 scales +/// - TODO: INT4/8 and FP8 per tensor /// /// Backends: -/// 1. TileIR if CUDA sm>=10 and same lhs/rhs dtype -/// 2. Triton otherwise -/// 3. Unsupported combos fall back to dequant + Dot +/// XLA selects specialized GPU kernels (TileIR, Triton, or block-128 W8A8), +/// with dequant + Dot as the fallback. +/// +/// Axes follow `Tensor.dot`; XLA currently accepts one contracting axis and at +/// most one batch axis. Scales preserve operand axis order, and each scale +/// dimension must divide its corresponding value dimension. /// /// CPU has no specialized path. pub fn scaledDot( @@ -126,7 +130,7 @@ pub fn scaledDot( const Axes = stdx.BoundedArray(i64, constants.MAX_RANK); const result_dtype: DataType = switch (lhs.dtype()) { - .f4e2m1, .f8e4m3, .f8e4m3fn, .f8e5m2, .f8e4m3b11fnuz, .f8e4m3fnuz, .f8e5m2fnuz => .bf16, + .f4e2m1, .f8e4m3, .f8e4m3fn, .f8e5m2, .f8e4m3b11fnuz, .f8e4m3fnuz, .f8e5m2fnuz, .f8e8m0 => .bf16, else => lhs.dtype(), }; var res_shape: Shape = .{ ._dtype = result_dtype }; @@ -190,14 +194,94 @@ pub fn scaledDot( .intArray(mlir_ctx, i64, rhs_batching_axes.constSlice()), }), }); - - const operands: []const Tensor = &.{ lhs, rhs, lhs_scale_operand, rhs_scale_operand }; - - const outs = ops.composite("xla.scaled_dot", operands, &.{res_shape}, scaledDotReference, res_shape, .{ + return ops.composite("xla.scaled_dot", &.{ lhs, rhs, lhs_scale_operand, rhs_scale_operand }, &.{res_shape}, scaledDotReference, res_shape, .{ .composite_attributes = &.{.named(mlir_ctx, "dimension_numbers", dnums)}, - }); + })[0]; +} - return outs[0]; +test "block128 scaled dot layouts" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + // FIXME: Add rocm when we have a PJRT plugin with native fp8 dot support (by the end of Sept 2026) + if (platform.target != .cuda) return error.SkipZigTest; + const dtype: DataType = if (platform.target == .rocm and @import("platform.zig").rocm.computeCapability(platform) == .gfx942) .f8e4m3fnuz else .f8e4m3fn; + const Local = struct { + const Outputs = struct { actual: Tensor, expected: Tensor, linear: Tensor, linear_expected: Tensor }; + + fn dequantize(values: Tensor, scales: Tensor) Tensor { + var expanded = scales.convert(.bf16); + for (0..values.rank()) |axis| { + const factor = @divExact(values.dim(axis), scales.dim(axis)); + const shape = expanded.shape().insert(axis + 1, .{factor}); + const axes = Shape.range(shape.rank(), .i64).remove(axis + 1); + expanded = expanded.broadcast(shape, axes.dims()).reshape(expanded.shape().setDim(axis, values.dim(axis))); + } + return values.convert(.bf16).mul(expanded); + } + + fn forward(x: Tensor, w: Tensor, scales: Tensor, fp8: DataType, prequantized: bool) Outputs { + const weight = w.convert(fp8); + const input = quantization.quantizeBlockFp8(x, .k, fp8); + const linear: Linear = .{ .weight = weight, .tag = Shape.toTag(.k), .quantization = .{ .scheme = .fp8_block128, .scales = scales } }; + return .{ + .linear = linear.forward(x), + .linear_expected = dequantize(input.values, input.scales).dot(dequantize(weight, scales), .k), + .actual = if (prequantized) + scaledDot(input.values, weight, input.scales.convert(scales.dtype()), scales, .k) + else + scaledDot(x, weight, null, scales, .k), + .expected = (if (prequantized) dequantize(input.values, input.scales.convert(scales.dtype())) else x) + .dot(dequantize(weight, scales), .k), + }; + } + }; + inline for (.{ + .{ .{ .m = 16, .k = 256 }, .{ .n = 256, .k = 256 } }, + .{ .{ .b = 2, .m = 16, .k = 256 }, .{ .b = 2, .n = 256, .k = 256 } }, + .{ .{ .k = 256, .b = 2, .m = 3 }, .{ .n = 256, .k = 256, .b = 2 } }, + .{ .{ .b = 2, .k = 256, .s = 2, .m = 3 }, .{ .n = 128, .b = 2, .k = 256 } }, + .{ .{ .b = 2, .m = 3, .k = 256 }, .{ .k = 256, .n = 128 } }, + .{ .{ .k = 128 }, .{ .n = 128, .k = 128 } }, + .{ .{ .m = 3, .k = 256 }, .{ .p = 2, .k = 256, .n = 128 } }, + .{ .{ .m = 3, .k = 128 }, .{ .n = 64, .k = 128 } }, + }) |layout| { + inline for (.{ DataType.f32, DataType.bf16 }) |scale_dtype| { + inline for (.{ false, true }) |prequantized| { + const x: Tensor = .init(layout[0], .bf16); + const w: Tensor = .init(layout[1], .bf16); + const scales: Tensor = .init(w.shape().setDim(.n, std.math.divCeil(i64, w.dim(.n), 128) catch unreachable).setDim(.k, @divExact(w.dim(.k), 128)).withDtype(scale_dtype), scale_dtype); + var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, w, scales, dtype, prequantized }, .{}); + defer exe.deinit(); + try zml.testing.expectEqualShapes(exe.output_shapes[1], exe.output_shapes[0]); + var buffers: [3]zml.Buffer = undefined; + var initialized: usize = 0; + defer for (buffers[0..initialized]) |*buffer| buffer.deinit(); + for ([_]Shape{ x.shape(), w.shape(), scales.shape() }, 0..) |shape, i| { + const slice = try Slice.alloc(allocator, shape); + defer slice.free(allocator); + for (0..shape.count()) |j| { + const value: f32 = if (i == 2) @as(f32, @floatFromInt(1 + (j * 3 + j / 4) % 7)) / 16 else @as(f32, @floatFromInt(@as(i32, @intCast((j * 7 + j / 256) % 13)) - 6)) / 4; + if (shape.dtype() == .f32) { + slice.items(f32)[j] = value; + } else { + slice.items(zml.floats.BFloat16)[j] = .fromF32(value); + } + } + buffers[i] = try zml.Buffer.fromSlice(io, platform, slice, .replicated); + initialized += 1; + } + var output = try zml.testing.autoCall(allocator, io, &exe, Local.forward, .{ buffers[0], buffers[1], buffers[2] }); + defer zml.Buffer.deinitAll(Local.Outputs, &output); + var expected = try output.expected.toSliceAlloc(allocator, io); + defer expected.free(allocator); + try zml.testing.expectClose(io, expected, output.actual, .{ .absolute_tolerance = 0.125, .relative_tolerance = 0.02 }); + var linear_expected = try output.linear_expected.toSliceAlloc(allocator, io); + defer linear_expected.free(allocator); + try zml.testing.expectClose(io, linear_expected, output.linear, .{ .absolute_tolerance = 0.125, .relative_tolerance = 0.02 }); + } + } + } } /// `shape` with every dimension collapsed to 1 diff --git a/zml/platform.zig b/zml/platform.zig index 7106aee0..a99d104e 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -834,6 +834,73 @@ pub const cuda = struct { } }; +pub const rocm = struct { + pub const ComputeCapability = enum { + /// CDNA1: MI100. + gfx908, + /// CDNA2: MI200 series. + gfx90a, + /// CDNA3: MI300 series. + gfx942, + /// CDNA4: MI350 series. + gfx950, + /// RDNA2: RX 6000 series. + gfx1030, + /// RDNA3 (Navi 31): RX 7900 series. + gfx1100, + /// RDNA3 (Navi 32): RX 7800 and RX 7700 series. + gfx1101, + /// RDNA3 (Navi 33): RX 7600 series. + gfx1102, + /// RDNA3 APU: Phoenix. + gfx1103, + /// RDNA3.5: newer Ryzen AI APUs. + gfx1150, + /// RDNA4 (Navi 44): RX 9060 family. + gfx1200, + /// RDNA4 (Navi 48): RX 9070 family. + gfx1201, + + pub const Architecture = enum { + cdna1, + cdna2, + cdna3, + cdna4, + rdna2, + rdna3, + rdna3_5, + rdna4, + }; + + pub fn architecture(self: ComputeCapability) Architecture { + return switch (self) { + .gfx908 => .cdna1, + .gfx90a => .cdna2, + .gfx942 => .cdna3, + .gfx950 => .cdna4, + .gfx1030 => .rdna2, + .gfx1100, .gfx1101, .gfx1102, .gfx1103 => .rdna3, + .gfx1150 => .rdna3_5, + .gfx1200, .gfx1201 => .rdna4, + }; + } + }; + + /// Assumes homogeneous devices. + pub fn computeCapability(platform: *const zml.Platform) ?ComputeCapability { + stdx.debug.assert(platform.target == .rocm, "computeCapability expects .rocm platform, got {}", .{platform.target}); + const device = platform.pjrt_client.devices(platform.pjrt_api)[0]; + const description = device.getDescription(platform.pjrt_api); + + const attributes = description.attributes(platform.pjrt_api); + return for (attributes) |attr| { + if (std.mem.eql(u8, attr.name(), "compute_capability")) { + break std.meta.stringToEnum(ComputeCapability, std.mem.sliceTo(attr.value().string, ':')); + } + } else null; + } +}; + fn dataTypeFromFfiDataType(ffi_dt: pjrt.ffi.DataType) zml.DataType { return switch (ffi_dt) { .bool => .bool, diff --git a/zml/quantization.zig b/zml/quantization.zig index ad25339b..abcdb5ee 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -1,5 +1,6 @@ //! Quantization metadata and scheme classification. const std = @import("std"); + const stdx = @import("stdx"); const DataType = @import("dtype.zig").DataType; @@ -47,8 +48,8 @@ pub const Quantization = struct { /// f8e4m3fn values, one bf16 or f32 scale per output channel, constant along the contraction. /// Emitted by llm-compressor, including for the layers an NVFP4 recipe leaves in FP8. fp8_per_channel, - /// f8e4m3fn values, one bf16 scale per 128x128 tile. The DeepSeek-style FP8 that model - /// vendors publish themselves, under `weight_scale_inv`. + /// f8e4m3fn, f8e4m3fnuz, or f8e8m0 values, one bf16 or f32 scale per 128x128 tile. + /// The DeepSeek-style FP8 that model vendors publish themselves, under `weight_scale_inv`. fp8_block128, /// f8e4m3fn values, one scale for the whole tensor. Spelled `[1, 1]` rather than as a /// scalar: XLA's composite rewriter requires the scale to have the operand's rank. @@ -74,10 +75,11 @@ pub const Quantization = struct { (scale.dtype() == .bf16 or scale.dtype() == .f32) and scale.count() > 1 and scale.rank() == 2 and scale.dim(0) == n and scale.dim(1) == 1, - .fp8_block128 => weight.dtype() == .f8e4m3fn and scale.dtype() == .bf16 and + .fp8_block128 => (weight.dtype() == .f8e4m3fn or weight.dtype() == .f8e4m3fnuz or weight.dtype() == .f8e8m0) and + (scale.dtype() == .bf16 or scale.dtype() == .f32) and scale.count() > 1 and scale.rank() == 2 and - @rem(n, 128) == 0 and @rem(k, 128) == 0 and - scale.dim(0) == @divExact(n, 128) and scale.dim(1) == @divExact(k, 128), + scale.dim(0) == (std.math.divCeil(i64, n, 128) catch unreachable) and + scale.dim(1) == (std.math.divCeil(i64, k, 128) catch unreachable), }; } @@ -105,7 +107,40 @@ pub fn quantizeInput(quantization: Quantization, input: Tensor, axis: Shape.Tag, const global_scale: ?Tensor = if (quantization.input_scale) |scale| scale.asMultiplier() else null; return switch (quantization.scheme) { .nvfp4 => if (supportsNvfp4InputQuantization(platform)) quantizeNvfp4(input, global_scale, axis) else null, - .mxfp8, .mxfp4, .fp8_per_channel, .fp8_block128, .fp8_per_tensor => null, + .fp8_block128 => switch (platform.target) { + .cuda => quantizeBlockFp8(input, axis, .f8e4m3fn), + .rocm => if (platform_mod.rocm.computeCapability(platform)) |capability| switch (capability.architecture()) { + .cdna4, .rdna4 => quantizeBlockFp8(input, axis, .f8e4m3fn), + .cdna3 => quantizeBlockFp8(input, axis, .f8e4m3fnuz), + .cdna1, .cdna2, .rdna2, .rdna3, .rdna3_5 => null, + } else null, + else => null, + }, + .mxfp8, .mxfp4, .fp8_per_channel, .fp8_per_tensor => null, + }; +} + +/// Quantize each 128-value activation block, preserving the input's axes. +/// Scales have the same shape with the contracting dimension divided by 128. +pub fn quantizeBlockFp8(x: Tensor, axis: anytype, dtype: DataType) QuantizedInput { + stdx.debug.assert(dtype == .f8e4m3fn or dtype == .f8e4m3fnuz, "expected E4M3 FP8 dtype, got {s}", .{@tagName(dtype)}); + stdx.debug.assert(@mod(x.dim(axis), 128) == 0, "block FP8 activation width must be divisible by 128, got {f}", .{x.shape()}); + // Match the routed-MoE quantizer's FNUZ range. + const fp8_max: f32 = if (dtype == .f8e4m3fnuz) 224.0 else 448.0; + const grouped = x.convert(.f32).splitAxis(axis, .{ .fp8_ks = -1, .fp8_block = 128 }); + // Put scale arithmetic before the reduction so it can fuse into the producer. + const scales = grouped.abs() + .maximum(.scalar(1e-10, .f32)) + .scale(1.0 / fp8_max) + .max(.fp8_block); + const values = grouped.div(scales.broad(grouped.shape())) + .clamp(.scalar(-fp8_max, .f32), .scalar(fp8_max, .f32)) + .convert(dtype) + .reshape(x.shape().withDtype(dtype)); + return .{ + .values = values, + .scales = scales.reshape(x.shape().setDim(axis, @divExact(x.dim(axis), 128)).withDtype(.f32)), + .global_scale = null, }; } @@ -194,6 +229,20 @@ test "Quantization.Scheme.classify" { .init(.{ .dout = 40, .sc = 48 }, .bf16), )); + // GLM uses f32 scales and permits a partial final 128-row tile. + try expect(@as(?Quantization.Scheme, .fp8_block128), Quantization.Scheme.classify( + .init(.{ .dout = 2048, .d = 6144 }, .f8e4m3fn), + .init(.{ .dout = 16, .sc = 48 }, .f32), + )); + try expect(@as(?Quantization.Scheme, .fp8_block128), Quantization.Scheme.classify( + .init(.{ .dout = 576, .d = 6144 }, .f8e4m3fn), + .init(.{ .dout = 5, .sc = 48 }, .f32), + )); + try expect(@as(?Quantization.Scheme, .fp8_block128), Quantization.Scheme.classify( + .init(.{ .dout = 576, .d = 6144 }, .f8e8m0), + .init(.{ .dout = 5, .sc = 48 }, .f32), + )); + // Mistral's per-tensor FP8: one scale for the whole tensor, rank 0 or [1]. try expect(@as(?Quantization.Scheme, .fp8_per_tensor), Quantization.Scheme.classify(.init(.{ .dout = 4096, .d = 4096 }, .f8e4m3fn), .init(.{}, .f32))); try expect(@as(?Quantization.Scheme, .fp8_per_tensor), Quantization.Scheme.classify(.init(.{ .dout = 4096, .d = 4096 }, .f8e4m3fn), .init(.{ .g = 1 }, .f32))); @@ -276,3 +325,46 @@ test "Quantization.Scheme.classify" { .init(.{ .dout = 1, .sc = 1 }, .bf16), )); } + +test "block FP8 quantization preserves axes and reconstructs constant blocks in FN and FNUZ" { + const zml = @import("zml.zig"); + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + inline for (.{ DataType.f8e4m3fn, DataType.f8e4m3fnuz }) |dtype| { + const Local = struct { + const Outputs = struct { values: Tensor, scales: Tensor }; + fn forward(x: Tensor) Outputs { + const input = quantizeBlockFp8(x, .k, dtype); + return .{ .values = input.values.convert(.f32), .scales = input.scales }; + } + }; + const x: Tensor = .init(.{ .b = 2, .k = 256, .s = 3 }, .bf16); + var exe = try platform.compileFn(allocator, io, Local.forward, .{x}, .{}); + defer exe.deinit(); + try zml.testing.expectEqualShapes(x.shape().withDtype(.f32), exe.output_shapes[0]); + try zml.testing.expectEqualShapes(x.shape().setDim(.k, 2).withDtype(.f32), exe.output_shapes[1]); + var host: [2 * 256 * 3]zml.floats.BFloat16 = undefined; + for (&host, 0..) |*value, i| { + const magnitude: f32 = if ((i / 3) % 256 < 128) 1.0 else 2.0; + value.* = .fromF32(switch (i % 3) { + 0 => 0.0, + 1 => magnitude, + else => -magnitude, + }); + } + var buffer = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&host)); + defer buffer.deinit(); + var output = try zml.testing.autoCall(allocator, io, &exe, Local.forward, .{buffer}); + defer zml.Buffer.deinitAll(Local.Outputs, &output); + var values = try output.values.toSliceAlloc(allocator, io); + defer values.free(allocator); + var scales = try output.scales.toSliceAlloc(allocator, io); + defer scales.free(allocator); + for (scales.constItems(f32)) |scale| try std.testing.expect(std.math.isFinite(scale) and scale > 0); + for (values.constItems(f32), 0..) |value, i| { + const scale_index = (i / (256 * 3)) * 6 + ((i / 3) % 256) / 128 * 3 + i % 3; + try std.testing.expectApproxEqAbs(host[i].toF32(), value * scales.constItems(f32)[scale_index], 1e-6); + } + } +} From 41fce4f2fc0ded823031e1f60b4dbd0a8d60d68b Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 9 Sep 2026 09:32:43 +0200 Subject: [PATCH 669/881] zml/nn: Add zml.nn.forwardQuantized (#783) --- zml/nn.zig | 40 +++++++++++++++++++--------------------- zml/zml.zig | 3 ++- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/zml/nn.zig b/zml/nn.zig index d5582f5f..7d76cf54 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -40,34 +40,32 @@ pub const Linear = struct { } pub fn forward(self: Linear, x: Tensor) Tensor { - const y = self.forwardWeight(x); + if (self.quantization) |q| { + const lhs = x.convert(.bf16); + if (quantization.quantizeInput(q, lhs, self.tag, zml.Compiler.current().platform)) |input| { + return self.forwardQuantized(input, x.dtype()); + } + return self.forwardScaled(lhs, null, null, x.dtype()); + } + const y = x.dot(self.weight, self.tag); return if (self.bias) |bias| y.add(bias.broad(y.shape())) else y; } - fn forwardWeight(self: Linear, x: Tensor) Tensor { - const q = self.quantization orelse return x.dot(self.weight, self.tag); + /// Apply this layer to reusable quantized activation values and scales. + /// Convert to output_dtype before adding bias, which must have the same dtype. + pub fn forwardQuantized(self: Linear, input: quantization.QuantizedInput, output_dtype: DataType) Tensor { + stdx.debug.assert(self.quantization != null, "forwardQuantized requires quantized weights", .{}); + return self.forwardScaled(input.values, input.scales, input.global_scale, output_dtype); + } + fn forwardScaled(self: Linear, lhs: Tensor, lhs_scale: ?Tensor, input_global_scale: ?Tensor, output_dtype: DataType) Tensor { + const q = self.quantization.?; const weight_global_scale: ?Tensor = if (q.global_scale) |s| s.asMultiplier() else null; - const weight = if (isPackedFp4(q.scheme, self.weight.dtype())) unpackFp4(self.weight, self.tag, self.tag) else self.weight; - const scales = if (q.scheme.isMx() and q.scales.dtype() == .u8) - q.scales.bitCast(.f8e8m0) - else - q.scales; - - var lhs = x.convert(.bf16); - var lhs_scale: ?Tensor = null; - var undo_input_scale: ?Tensor = null; - - const platform = zml.Compiler.current().platform; - if (quantization.quantizeInput(q, lhs, self.tag, platform)) |quantized_input| { - lhs = quantized_input.values; - lhs_scale = quantized_input.scales; - undo_input_scale = quantized_input.global_scale; - } - + const scales = if (q.scheme.isMx() and q.scales.dtype() == .u8) q.scales.bitCast(.f8e8m0) else q.scales; const acc = scaledDot(lhs, weight, lhs_scale, scales, self.tag); - return applyGlobalScale(acc, undo_input_scale, weight_global_scale).convert(x.dtype()); + const y = applyGlobalScale(acc, input_global_scale, weight_global_scale).convert(output_dtype); + return if (self.bias) |bias| y.add(bias.broad(y.shape())) else y; } }; diff --git a/zml/zml.zig b/zml/zml.zig index 9461f6a3..995318eb 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -33,7 +33,8 @@ pub const meta = @import("meta.zig"); pub const mlir = @import("mlirx.zig"); pub const moe = @import("moe/moe.zig"); pub const nn = @import("nn.zig"); -pub const Quantization = @import("quantization.zig").Quantization; +pub const quantization = @import("quantization.zig"); +pub const Quantization = quantization.Quantization; pub const ops = @import("ops.zig"); pub const pjrtx = @import("pjrtx.zig"); pub const platform = @import("platform.zig"); From 0f7b3bb4b01c446088d3d882a3c92e9598cffccf Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Wed, 9 Sep 2026 15:17:32 +0200 Subject: [PATCH 670/881] zml/moe: enforce `padded_num_experts` as a power of two (#786) Fix an issue when `padded_num_experts` computed doesn't land as a power of two. Notably for DeepSeek-v4 Pro with its 384 experts, divided by 32 warp is not of power of two, leading to a crash in the triton which expects a power of two number of experts. --- zml/moe/triton.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/moe/triton.zig b/zml/moe/triton.zig index ed9b26af..0e3521af 100644 --- a/zml/moe/triton.zig +++ b/zml/moe/triton.zig @@ -417,7 +417,7 @@ fn alignBlockSize(topk_ids: Tensor, num_experts: i64, block_size_m: i64) struct num_assignments + num_experts * (block_size_m - 1); const max_num_m_blocks = std.math.divCeil(i64, max_num_tokens_padded, block_size_m) catch unreachable; const warp_size: i64 = 32; - const padded_num_experts = (std.math.divCeil(i64, num_experts, warp_size) catch unreachable) * warp_size; + const padded_num_experts: i64 = @intCast(std.math.ceilPowerOfTwoAssert(u64, @intCast(@max(num_experts, warp_size)))); const sort_block_size: i64 = 256; const sort_grid_x: i64 = @min(std.math.divCeil(i64, num_assignments, sort_block_size) catch unreachable, 65535); From f6af7f3e529f3206fc4b8b580832f5a919a8ee4b Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 9 Sep 2026 19:07:33 +0200 Subject: [PATCH 671/881] zml/moe: remove unused metadata (#789) --- examples/llm/models/qwen3_5_moe/inference.zig | 75 ++----------------- examples/llm/models/qwen3_5_moe/model.zig | 18 ++--- examples/llm/models/qwen3_5_moe/session.zig | 13 ---- zml/moe/cutlass_flashinfer.zig | 25 ------- zml/moe/metal.zig | 22 ------ zml/moe/moe.zig | 71 ------------------ zml/moe/mosaic_tpu.zig | 22 ------ zml/moe/triton.zig | 40 ---------- 8 files changed, 12 insertions(+), 274 deletions(-) diff --git a/examples/llm/models/qwen3_5_moe/inference.zig b/examples/llm/models/qwen3_5_moe/inference.zig index a9ff1de0..a8c05bf4 100644 --- a/examples/llm/models/qwen3_5_moe/inference.zig +++ b/examples/llm/models/qwen3_5_moe/inference.zig @@ -10,8 +10,6 @@ const log = std.log.scoped(.qwen3_5_moe); pub const CompilationParameters = struct { kv_cache: model.KvCache, rng: zml.Tensor.Rng, - prefill_moe_metadata: zml.moe.Metadata, - decode_moe_metadata: zml.moe.Metadata, moe_parameters: zml.moe.Parameters, seqlen: u32, shardings: common.Shardings, @@ -22,8 +20,6 @@ pub const CompilationParameters = struct { return .{ .kv_cache = .init(config, 1, seqlen, dtype, .f32, shardings.model), .rng = .init(), - .prefill_moe_metadata = initMoeMetadata(mdl, @intCast(seqlen), 1, moe_backend), - .decode_moe_metadata = initMoeMetadata(mdl, 1, 1, moe_backend), .moe_parameters = .init(.fromBackend(moe_backend, config.text_config.num_experts_per_tok, zml.moe.ActivationMode.silu)), .seqlen = seqlen, .shardings = shardings, @@ -45,7 +41,6 @@ pub const RunArgs = struct { token_index_buffer: *zml.Buffer, active_length_buffer: *zml.Buffer, kv_cache_buffers: *zml.Bufferized(model.KvCache), - moe_metadata_buffers: zml.Bufferized(zml.moe.Metadata), rng_buffers: *zml.Bufferized(zml.Tensor.Rng), layer_index_buffers: []const LayerIndexBuffer, }; @@ -65,9 +60,9 @@ pub const CompiledModel = struct { parameters: CompilationParameters, progress: *std.Progress.Node, ) !CompiledModel { - const prefill = try compileKernel(allocator, io, platform, qwen_model, parameters, parameters.seqlen, parameters.prefill_moe_metadata, "prefill", progress); + const prefill = try compileKernel(allocator, io, platform, qwen_model, parameters, parameters.seqlen, "prefill", progress); errdefer prefill.deinit(); - const decode = try compileKernel(allocator, io, platform, qwen_model, parameters, 1, parameters.decode_moe_metadata, "decode", progress); + const decode = try compileKernel(allocator, io, platform, qwen_model, parameters, 1, "decode", progress); return .{ .loaded_model = loaded_model, .prefill = prefill, @@ -186,7 +181,6 @@ pub fn run(runner: *KernelRunner, args: RunArgs) void { .hidden = hidden_buffer, .token_index = args.token_index_buffer.*, .cache = layer_cache, - .moe_metadata = args.moe_metadata_buffers, }, .outputs = .{ .hidden = &hidden_buffer, .cache = &layer_cache }, }); @@ -208,7 +202,6 @@ pub fn run(runner: *KernelRunner, args: RunArgs) void { .hidden = hidden_buffer, .active_length = args.active_length_buffer.*, .cache = layer_cache, - .moe_metadata = args.moe_metadata_buffers, }, .outputs = .{ .hidden = &hidden_buffer, .cache = &layer_cache }, }); @@ -239,7 +232,6 @@ fn compileKernel( mdl: model.Model, parameters: CompilationParameters, seqlen: usize, - moe_metadata: zml.moe.Metadata, phase: []const u8, progress: *std.Progress.Node, ) !KernelExe { @@ -248,9 +240,9 @@ fn compileKernel( var embed_future = try io.concurrent(compileEmbed, .{ allocator, io, platform, mdl, parameters, seqlen, phase, progress }); errdefer if (embed_future.cancel(io)) |exe| exe.deinit() else |_| {}; - var full_attention_future = try io.concurrent(compileFullAttention, .{ allocator, io, platform, mdl, parameters, seqlen, full_index, moe_metadata, phase, progress }); + var full_attention_future = try io.concurrent(compileFullAttention, .{ allocator, io, platform, mdl, parameters, seqlen, full_index, phase, progress }); errdefer if (full_attention_future.cancel(io)) |exe| exe.deinit() else |_| {}; - var linear_attention_future = try io.concurrent(compileLinearAttention, .{ allocator, io, platform, mdl, parameters, seqlen, linear_index, moe_metadata, phase, progress }); + var linear_attention_future = try io.concurrent(compileLinearAttention, .{ allocator, io, platform, mdl, parameters, seqlen, linear_index, phase, progress }); errdefer if (linear_attention_future.cancel(io)) |exe| exe.deinit() else |_| {}; var sample_future = try io.concurrent(compileSample, .{ allocator, io, platform, mdl, parameters, seqlen, phase, progress }); errdefer if (sample_future.cancel(io)) |exe| exe.deinit() else |_| {}; @@ -279,7 +271,7 @@ fn compileEmbed(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.P }}, parameters, progress, phase, "embedding"); } -fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, layer_index: usize, moe_metadata: zml.moe.Metadata, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardSelfAttn) { +fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, layer_index: usize, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardSelfAttn) { return compileExe(allocator, io, platform, model.TransformerLayer.forwardSelfAttn, .{.{ .layer = mdl.text_model.layers[layer_index], .hidden = hiddenTensor(mdl, seqlen), @@ -290,12 +282,11 @@ fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *con .layer_index = zml.Tensor.init(.{}, .u32), }, .config = mdl.config, - .moe_metadata = moe_metadata, .moe_parameters = parameters.moe_parameters, }}, parameters, progress, phase, "full-attention layer"); } -fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, layer_index: usize, moe_metadata: zml.moe.Metadata, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardLinearAttn) { +fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, layer_index: usize, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardLinearAttn) { return compileExe(allocator, io, platform, model.TransformerLayer.forwardLinearAttn, .{.{ .layer = mdl.text_model.layers[layer_index], .hidden = hiddenTensor(mdl, seqlen), @@ -306,7 +297,6 @@ fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *c .layer_index = zml.Tensor.init(.{}, .u32), }, .config = mdl.config, - .moe_metadata = moe_metadata, .moe_parameters = parameters.moe_parameters, }}, parameters, progress, phase, "linear-attention layer"); } @@ -355,56 +345,3 @@ fn findFirstLayerIndex(layer_types: []const model.LayerType, target: model.Layer for (layer_types, 0..) |layer_type, index| if (layer_type == target) return index; return null; } - -fn initMoeMetadata(qwen_model: model.Model, token_len: usize, batch_size: u32, backend: zml.moe.Backend) zml.moe.Metadata { - var w1_zero_bias_shape: ?zml.Shape = null; - var w2_zero_bias_shape: ?zml.Shape = null; - var first_out_shape: ?zml.Shape = null; - var second_out_shape: ?zml.Shape = null; - - const num_experts_per_tok = qwen_model.config.text_config.num_experts_per_tok; - const num_experts = qwen_model.config.text_config.num_experts; - - for (qwen_model.text_model.layers) |layer| { - const gate_up_shape = zml.Shape.init(.{ - .expert = num_experts, - .out = layer.moe.shared_expert.gate_proj.weight.dim(.dout), - }, .bf16); - const down_shape = zml.Shape.init(.{ - .expert = num_experts, - .out = layer.moe.shared_expert.gate_proj.weight.dim(.d), - }, .bf16); - const first_out = zml.Shape.init(.{ - .total_tokens = batch_size * token_len * num_experts_per_tok, - .out = layer.moe.shared_expert.gate_proj.weight.dim(.dout) * 2, - }, .bf16); - const second_out = zml.Shape.init(.{ - .token = batch_size * token_len, - .topk = num_experts_per_tok, - .out = layer.moe.shared_expert.down_proj.weight.dim(.d), - }, .bf16); - - if (w1_zero_bias_shape == null) { - w1_zero_bias_shape = gate_up_shape; - w2_zero_bias_shape = down_shape; - first_out_shape = first_out; - second_out_shape = second_out; - continue; - } - - if (!w1_zero_bias_shape.?.eql(gate_up_shape) or !w2_zero_bias_shape.?.eql(down_shape) or !first_out_shape.?.eql(first_out) or !second_out_shape.?.eql(second_out)) { - log.warn("MoE bias shapes differ across layers; using shapes from the first layer", .{}); - break; - } - } - - return switch (backend) { - .triton => .init(.{ - .triton = .{ - .w1_zero_bias_shape = w1_zero_bias_shape, - .w2_zero_bias_shape = w2_zero_bias_shape, - }, - }), - .flashinfer_cutlass, .mosaic_tpu, .metal => .init(.fromBackend(backend)), - }; -} diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 45c830f4..f947bb96 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -233,11 +233,10 @@ pub const Model = struct { active_length: zml.Tensor, kv_cache: KvCache, rng: zml.Tensor.Rng, - moe_metadata: zml.moe.Metadata, moe_parameters: zml.moe.Parameters, ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { const tokens = tokens_.withPartialTags(.{.s}); - const new_tokens, const updated_kv_cache, const new_rng = self.text_model.forward(tokens, token_index, active_length, kv_cache, self.config, rng, moe_metadata, moe_parameters); + const new_tokens, const updated_kv_cache, const new_rng = self.text_model.forward(tokens, token_index, active_length, kv_cache, self.config, rng, moe_parameters); return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; } }; @@ -356,7 +355,6 @@ pub const TextModel = struct { kv_cache: KvCache, config: Config, rng: zml.Tensor.Rng, - moe_metadata: zml.moe.Metadata, moe_parameters: zml.moe.Parameters, ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { var hidden_states = EmbedTokens.forward(.{ @@ -366,7 +364,7 @@ pub const TextModel = struct { var updated_kv_cache = kv_cache; for (self.layers, 0..) |layer, i| { - hidden_states, updated_kv_cache = layer.forward(hidden_states, token_index, active_length, updated_kv_cache.atLayer(i), config, moe_metadata, moe_parameters); + hidden_states, updated_kv_cache = layer.forward(hidden_states, token_index, active_length, updated_kv_cache.atLayer(i), config, moe_parameters); } const result = Sampler.sampleTokens(.{ @@ -396,7 +394,6 @@ pub const TransformerLayer = struct { token_index: zml.Tensor, cache: KvCache.SelfAttnCache, config: Config, - moe_metadata: zml.moe.Metadata, moe_parameters: zml.moe.Parameters, }; @@ -411,7 +408,6 @@ pub const TransformerLayer = struct { active_length: zml.Tensor, cache: KvCache.GatedDeltaNetCache, config: Config, - moe_metadata: zml.moe.Metadata, moe_parameters: zml.moe.Parameters, }; @@ -463,7 +459,7 @@ pub const TransformerLayer = struct { const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); - const moe_output = self.moe.forward(normalized_hidden, input.moe_metadata, input.moe_parameters); + const moe_output = self.moe.forward(normalized_hidden, input.moe_parameters); return .{ .hidden = moe_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), @@ -487,7 +483,7 @@ pub const TransformerLayer = struct { const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); - const moe_output = self.moe.forward(normalized_hidden, input.moe_metadata, input.moe_parameters); + const moe_output = self.moe.forward(normalized_hidden, input.moe_parameters); return .{ .hidden = moe_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), @@ -502,7 +498,6 @@ pub const TransformerLayer = struct { active_length: zml.Tensor, kv_cache: KvCache.LayerView, config: Config, - moe_metadata: zml.moe.Metadata, moe_parameters: zml.moe.Parameters, ) struct { zml.Tensor, KvCache } { _ = config; @@ -527,7 +522,7 @@ pub const TransformerLayer = struct { const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); - const moe_output = self.moe.forward(normalized_hidden, moe_metadata, moe_parameters); + const moe_output = self.moe.forward(normalized_hidden, moe_parameters); return .{ moe_output.add(x1).withPartitioning(.{ .d = .replicated }), updated_kv_cache }; } @@ -773,7 +768,7 @@ pub const Moe = struct { _ = allocator; } - pub fn forward(self: Moe, x: zml.Tensor, moe_metadata: zml.moe.Metadata, moe_parameters: zml.moe.Parameters) zml.Tensor { + pub fn forward(self: Moe, x: zml.Tensor, moe_parameters: zml.moe.Parameters) zml.Tensor { const routing_scores, const topk_ids = self.router.forward(x); const moe_output = zml.moe.forwardMoe( @@ -783,7 +778,6 @@ pub const Moe = struct { self.gate_up_proj, self.down_proj, .{}, - moe_metadata, moe_parameters, ) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); diff --git a/examples/llm/models/qwen3_5_moe/session.zig b/examples/llm/models/qwen3_5_moe/session.zig index 4e038531..e1509df5 100644 --- a/examples/llm/models/qwen3_5_moe/session.zig +++ b/examples/llm/models/qwen3_5_moe/session.zig @@ -13,8 +13,6 @@ pub const Session = struct { prefill: inference.KernelRunner, decode: inference.KernelRunner, kv_cache_buffers: zml.Bufferized(model.KvCache), - prefill_moe_metadata_buffers: zml.Bufferized(zml.moe.Metadata), - decode_moe_metadata_buffers: zml.Bufferized(zml.moe.Metadata), rng_buffers: zml.Bufferized(zml.Tensor.Rng), layer_index_buffers: []inference.LayerIndexBuffer, generated_token_slice: zml.Slice, @@ -37,11 +35,6 @@ pub const Session = struct { var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform, shardings.model); errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); - var prefill_moe_metadata_buffers = try compiled_model.params.prefill_moe_metadata.initBuffer(io, platform); - errdefer zml.moe.Metadata.deinitBuffer(&prefill_moe_metadata_buffers); - var decode_moe_metadata_buffers = try compiled_model.params.decode_moe_metadata.initBuffer(io, platform); - errdefer zml.moe.Metadata.deinitBuffer(&decode_moe_metadata_buffers); - const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); var rng_buffers = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); @@ -88,8 +81,6 @@ pub const Session = struct { .prefill = prefill, .decode = decode, .kv_cache_buffers = kv_cache_buffers, - .prefill_moe_metadata_buffers = prefill_moe_metadata_buffers, - .decode_moe_metadata_buffers = decode_moe_metadata_buffers, .rng_buffers = rng_buffers, .layer_index_buffers = layer_index_buffers, .generated_token_slice = generated_token_slice, @@ -106,8 +97,6 @@ pub const Session = struct { self.prefill.deinit(self.allocator); self.decode.deinit(self.allocator); model.KvCache.deinitBuffer(&self.kv_cache_buffers); - zml.moe.Metadata.deinitBuffer(&self.prefill_moe_metadata_buffers); - zml.moe.Metadata.deinitBuffer(&self.decode_moe_metadata_buffers); zml.Tensor.Rng.deinitBuffer(&self.rng_buffers); for (self.layer_index_buffers) |*layer_index_buffer| { switch (layer_index_buffer.*) { @@ -146,7 +135,6 @@ pub const Session = struct { .token_index_buffer = &token_index_buffer, .active_length_buffer = &active_length_buffer, .kv_cache_buffers = &self.kv_cache_buffers, - .moe_metadata_buffers = self.prefill_moe_metadata_buffers, .rng_buffers = &self.rng_buffers, .layer_index_buffers = self.layer_index_buffers, }); @@ -187,7 +175,6 @@ pub const Session = struct { .token_index_buffer = &token_index_buffer, .active_length_buffer = &active_length_buffer, .kv_cache_buffers = &self.kv_cache_buffers, - .moe_metadata_buffers = self.decode_moe_metadata_buffers, .rng_buffers = &self.rng_buffers, .layer_index_buffers = self.layer_index_buffers, }); diff --git a/zml/moe/cutlass_flashinfer.zig b/zml/moe/cutlass_flashinfer.zig index 81dfaffa..d857e344 100644 --- a/zml/moe/cutlass_flashinfer.zig +++ b/zml/moe/cutlass_flashinfer.zig @@ -70,31 +70,6 @@ pub const Parameters = struct { } }; -pub const Metadata = struct { - variant: Variant = .bf16xbf16, - - pub const InitOptions = struct { - variant: Variant = .bf16xbf16, - }; - - pub fn init(opts: InitOptions) Metadata { - return .{ - .variant = opts.variant, - }; - } - - pub fn initBuffer( - _: Metadata, - _: std.Io, - _: *const zml.Platform, - ) !zml.Bufferized(Metadata) { - return {}; - } -}; - -/// NVFP4 scale buffers are borrowed from the model and are not owned by Metadata. -pub fn deinitBuffer(_: *zml.Bufferized(Metadata)) void {} - const Input = struct { hidden_states: zml.Tensor, fc1_weights: zml.Tensor, diff --git a/zml/moe/metal.zig b/zml/moe/metal.zig index eae31e9e..f3b2f07a 100644 --- a/zml/moe/metal.zig +++ b/zml/moe/metal.zig @@ -38,26 +38,6 @@ pub const Parameters = struct { } }; -pub const Metadata = struct { - pub const InitOptions = struct {}; - - pub fn init(opts: InitOptions) Metadata { - _ = opts; - return .{}; - } - - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { - _ = self; - _ = io; - _ = platform; - return {}; - } -}; - -pub fn deinitBuffer(bufferized: *zml.Bufferized(Metadata)) void { - _ = bufferized; -} - const QuantMode = enum { none, fp8, nvfp4 }; fn quantMode(gate_up: Tensor, down: Tensor) !QuantMode { @@ -136,10 +116,8 @@ pub fn fusedExpertsImpl( down: Tensor, topk_weights: Tensor, topk_ids: Tensor, - metadata: Metadata, opts: Options, ) !Tensor { - _ = metadata; if (opts.expert_map != null) return error.InvalidShape; if (opts.w1_bias != null or opts.w2_bias != null) return error.UnsupportedBias; diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index d3b8c567..ccf7446b 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -138,56 +138,6 @@ pub const Parameters = union(Backend) { } }; -pub const Metadata = union(Backend) { - flashinfer_cutlass: cutlass_flashinfer.Metadata, - triton: triton.Metadata, - mosaic_tpu: mosaic_tpu.Metadata, - metal: metal.Metadata, - - pub const InitOptions = union(Backend) { - flashinfer_cutlass: cutlass_flashinfer.Metadata.InitOptions, - triton: triton.Metadata.InitOptions, - mosaic_tpu: mosaic_tpu.Metadata.InitOptions, - metal: metal.Metadata.InitOptions, - - pub fn fromBackend(backend: Backend) InitOptions { - return switch (backend) { - .flashinfer_cutlass => .{ .flashinfer_cutlass = .{} }, - .triton => .{ .triton = .{} }, - .mosaic_tpu => .{ .mosaic_tpu = .{} }, - .metal => .{ .metal = .{} }, - }; - } - }; - - pub fn init(opts: InitOptions) Metadata { - return switch (opts) { - .flashinfer_cutlass => |v| .{ .flashinfer_cutlass = cutlass_flashinfer.Metadata.init(v) }, - .triton => |v| .{ .triton = triton.Metadata.init(v) }, - .mosaic_tpu => |v| .{ .mosaic_tpu = mosaic_tpu.Metadata.init(v) }, - .metal => |v| .{ .metal = metal.Metadata.init(v) }, - }; - } - - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { - return switch (self) { - .flashinfer_cutlass => |metadata| .{ .flashinfer_cutlass = try metadata.initBuffer(io, platform) }, - .triton => |metadata| .{ .triton = try metadata.initBuffer(io, platform) }, - .mosaic_tpu => |metadata| .{ .mosaic_tpu = try metadata.initBuffer(io, platform) }, - .metal => |metadata| .{ .metal = try metadata.initBuffer(io, platform) }, - }; - } - - pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { - switch (self.*) { - .flashinfer_cutlass => |*metadata| cutlass_flashinfer.deinitBuffer(metadata), - .triton => |*metadata| triton.deinitBuffer(metadata), - .mosaic_tpu => |*metadata| mosaic_tpu.deinitBuffer(metadata), - .metal => |*metadata| metal.deinitBuffer(metadata), - } - } -}; - pub const Options = struct { activation_threshold: ?f32 = null, }; @@ -199,7 +149,6 @@ pub fn forwardMoe( gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: Options, - metadata: Metadata, parameters: Parameters, ) !zml.Tensor { const gate_up_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; @@ -422,11 +371,6 @@ pub fn forwardMoe( ); }, .triton => b: { - const triton_metadata = switch (metadata) { - .triton => |v| v, - else => return error.InvalidMetadata, - }; - const global_num_experts = gate_up.weight.dim(.expert); const expert_partition = gate_up.weight.shape().partition(.expert); @@ -437,7 +381,6 @@ pub fn forwardMoe( down.weight, topk_weights, topk_ids, - triton_metadata, .{ .activation = parameters.triton.activation, .global_num_experts = global_num_experts, @@ -488,7 +431,6 @@ pub fn forwardMoe( self.weights_down, self.topk_weights, self.topk_ids, - .{}, .{ .activation = self.activation, .global_num_experts = self.global_num_experts, @@ -524,11 +466,6 @@ pub fn forwardMoe( ); }, .mosaic_tpu => b: { - const tpu_metadata = switch (metadata) { - .mosaic_tpu => |v| v, - else => return error.InvalidMetadata, - }; - const expert_partition = gate_up.weight.shape().partition(.expert); if (expert_partition.eql(.init(.experts))) { @@ -565,7 +502,6 @@ pub fn forwardMoe( self.weights_down, self.topk_weights, self.topk_ids, - .{}, .{ .activation = self.activation, .global_num_experts = self.global_num_experts, @@ -603,7 +539,6 @@ pub fn forwardMoe( down.weight, topk_weights, topk_ids, - tpu_metadata, .{ .activation = parameters.mosaic_tpu.activation, .global_num_experts = gate_up.weight.dim(.expert), @@ -617,18 +552,12 @@ pub fn forwardMoe( .metal => b: { const gate_up_weight_unpacked = unpackedWeight(gate_up); const down_weight_unpacked = unpackedWeight(down); - const metal_metadata = switch (metadata) { - .metal => |v| v, - else => return error.InvalidMetadata, - }; - break :b try metal.fusedExpertsImpl( input, gate_up_weight_unpacked, down_weight_unpacked, topk_weights, topk_ids, - metal_metadata, .{ .activation = parameters.metal.activation, .global_num_experts = gate_up_weight_unpacked.dim(.expert), diff --git a/zml/moe/mosaic_tpu.zig b/zml/moe/mosaic_tpu.zig index 7353c4dd..847033b0 100644 --- a/zml/moe/mosaic_tpu.zig +++ b/zml/moe/mosaic_tpu.zig @@ -46,26 +46,6 @@ pub const Parameters = struct { } }; -pub const Metadata = struct { - pub const InitOptions = struct {}; - - pub fn init(opts: InitOptions) Metadata { - _ = opts; - return .{}; - } - - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { - _ = self; - _ = io; - _ = platform; - return {}; - } -}; - -pub fn deinitBuffer(bufferized: *zml.Bufferized(Metadata)) void { - _ = bufferized; -} - fn validateOptions(opts: Options) !void { if (opts.expert_map != null and opts.global_num_experts == -1) return error.InvalidShape; if (opts.w1_scale != null or opts.w2_scale != null) return error.UnsupportedQuantization; @@ -290,10 +270,8 @@ pub fn fusedExpertsImpl( w2: Tensor, topk_weights: Tensor, topk_ids: Tensor, - metadata: Metadata, opts: Options, ) !Tensor { - _ = metadata; try validateOptions(opts); const b = hidden_states.dim(.b); diff --git a/zml/moe/triton.zig b/zml/moe/triton.zig index 0e3521af..caa9496d 100644 --- a/zml/moe/triton.zig +++ b/zml/moe/triton.zig @@ -73,43 +73,6 @@ pub const Parameters = struct { } }; -pub const Metadata = struct { - w1_zero_bias: ?Tensor = null, - w2_zero_bias: ?Tensor = null, - - pub const InitOptions = struct { - w1_zero_bias_shape: ?Shape = null, - w2_zero_bias_shape: ?Shape = null, - }; - - pub fn init(opts: InitOptions) Metadata { - return .{ - .w1_zero_bias = if (opts.w1_zero_bias_shape) |shape| Tensor.fromShape(shape) else null, - .w2_zero_bias = if (opts.w2_zero_bias_shape) |shape| Tensor.fromShape(shape) else null, - }; - } - - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { - const replicated_sharding = platform.replicated_sharding; - return .{ - .w1_zero_bias = if (self.w1_zero_bias) |tensor| try initZeroBiasBuffer(io, platform, replicated_sharding, tensor.shape()) else null, - .w2_zero_bias = if (self.w2_zero_bias) |tensor| try initZeroBiasBuffer(io, platform, replicated_sharding, tensor.shape()) else null, - }; - } -}; - -pub fn deinitBuffer(bufferized: *zml.Bufferized(Metadata)) void { - if (bufferized.w1_zero_bias) |*buffer| buffer.deinit(); - if (bufferized.w2_zero_bias) |*buffer| buffer.deinit(); -} - -fn initZeroBiasBuffer(io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding, shape: Shape) !zml.Buffer { - var zero_slice: zml.Slice = try .alloc(std.heap.c_allocator, shape); - defer zero_slice.free(std.heap.c_allocator); - @memset(zero_slice.data(), 0); - return zml.Buffer.fromSlice(io, platform, zero_slice, sharding); -} - fn applyActivation(x: Tensor, mode: Parameters.ActivationMode, activation_threshold: ?f32) Tensor { const mid = @divFloor(x.dim(.out), 2); var gate = x.slice(.out, .{ .end = mid }); @@ -140,7 +103,6 @@ pub fn fusedExpertsImpl( w2: Tensor, topk_weights: Tensor, topk_ids: Tensor, - metadata: Metadata, opts: Options, ) !Tensor { try validateOptions(opts); @@ -222,7 +184,6 @@ pub fn fusedExpertsImpl( const b_bias_1 = opts.w1_bias orelse - metadata.w1_zero_bias orelse Tensor.zeroes(Shape.init(.{ .expert = gate_up.dim(.expert), .out = gate_up.dim(.out) }, .bf16)); const b_scale_1 = opts.w1_scale orelse Tensor.scalar(1.0, .f32); @@ -264,7 +225,6 @@ pub fn fusedExpertsImpl( const b_bias_2 = opts.w2_bias orelse - metadata.w2_zero_bias orelse Tensor.zeroes(Shape.init(.{ .expert = down.dim(.expert), .out = down.dim(.out) }, .bf16)); const b_scale_2 = opts.w2_scale orelse Tensor.scalar(1.0, .f32); From 662bc8ea5d784e893f3610651f6903bf48faa335 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Thu, 10 Sep 2026 09:50:57 +0200 Subject: [PATCH 672/881] zml/moe: Simplify triton API and config loader (#790) --- zml/BUILD.bazel | 4 +- zml/meta.zig | 8 + zml/moe/moe.zig | 87 +---- zml/moe/triton.zig | 574 +++++++++++------------------ zml/moe/triton_kernels/config.json | 146 -------- 5 files changed, 251 insertions(+), 568 deletions(-) delete mode 100644 zml/moe/triton_kernels/config.json diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 8a352104..402fb737 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -123,9 +123,7 @@ zig_library( "zml.zig", "//zml/attention/nki:attention", ], - data = [ - "moe/triton_kernels/config.json", - ] + select({ + data = select({ "//platforms:neuron.enabled": [ "//zml/attention/nki:attention_kernel", ], diff --git a/zml/meta.zig b/zml/meta.zig index 3090b609..9ff4d39a 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -231,6 +231,10 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam } to.* = items; }, + .many => if (FromStruct == [*:0]const u8 and ToStruct == FromStruct and From != u8) { + // Tensor axis tags are borrowed immutable strings. + to.* = from; + } else stdx.debug.compileError("zml.meta.mapAlloc doesn't support: {}", .{FromStruct}), else => stdx.debug.compileError("zml.meta.mapAlloc doesn't support: {}", .{FromStruct}), }, .optional => if (from) |f| { @@ -260,6 +264,7 @@ test mapAlloc { array: [2]A, slice: []const A, other: u8, + tag: [*:0]const u8, // We want to allow conversion from comptime to runtime, because Zig type inference works like this. comptime static_val: u8 = 8, comptime static_slice: [2]A = .{ .{ .a = 11 }, .{ .a = 12 } }, @@ -270,6 +275,7 @@ test mapAlloc { array: [2]B, slice: []const B, other: u8, + tag: [*:0]const u8, static_val: u8, static_slice: []B, field_with_empty: struct { B, Empty }, @@ -279,6 +285,7 @@ test mapAlloc { .field = .{ .a = 4 }, .array = .{ .{ .a = 5 }, .{ .a = 6 } }, .other = 7, + .tag = "feature", .slice = &.{ .{ .a = 9 }, .{ .a = 10 } }, .field_with_empty = .{ .{ .a = 9 }, .{} }, }; @@ -292,6 +299,7 @@ test mapAlloc { try testing.expectEqual(5, bb.array[0].b); try testing.expectEqual(6, bb.array[1].b); try testing.expectEqual(7, bb.other); + try testing.expectEqual(aa.tag, bb.tag); try testing.expectEqual(8, bb.static_val); try testing.expectEqual(9, bb.slice[0].b); try testing.expectEqual(10, bb.slice[1].b); diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index ccf7446b..2c422dd0 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -371,97 +371,48 @@ pub fn forwardMoe( ); }, .triton => b: { - const global_num_experts = gate_up.weight.dim(.expert); + const args: triton.FusedExpertsArgs = .{ + .hidden_states = input, + .gate_up = gate_up, + .down = down, + .topk_weights = topk_weights, + .topk_ids = topk_ids, + .activation = parameters.triton.activation, + .activation_threshold = opts.activation_threshold, + }; const expert_partition = gate_up.weight.shape().partition(.expert); if (!expert_partition.eql(.init(.experts))) { - break :b try triton.fusedExpertsImpl( - input, - gate_up.weight, - down.weight, - topk_weights, - topk_ids, - .{ - .activation = parameters.triton.activation, - .global_num_experts = global_num_experts, - .w1_scale = gate_up_scales, - .w2_scale = down_scales, - .w1_bias = gate_up.bias, - .w2_bias = down.bias, - .quant_scheme = quant_scheme, - .activation_threshold = opts.activation_threshold, - }, - ); + break :b try triton.fusedExpertsImpl(args); } break :b zml.ops.manualComputation( (struct { - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - weights_gate_up: zml.Tensor, - weights_down: zml.Tensor, - activation: triton.Parameters.ActivationMode, + args: triton.FusedExpertsArgs, global_num_experts: i64, - bias_gate_up: ?zml.Tensor, - bias_down: ?zml.Tensor, - quant_scheme: ?zml.Quantization.Scheme, - activation_threshold: ?f32, - scales_gate_up: ?zml.Tensor, - scales_down: ?zml.Tensor, fn call(self: @This(), _: zml.Shape) zml.Tensor { - const local_num_experts = self.weights_gate_up.dim(.expert); + const local_args = self.args; + const local_num_experts = local_args.gate_up.weight.dim(.expert); const partition_id = zml.ops.partitionId().convert(.i32); const expert_start = partition_id.scale(local_num_experts).convert(.i32); - // List of global expert ids const global_expert_ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); - // Mapping of local experts to global expert ids, -1 if the global expert is not present in the local partition + // Map global expert ids to local ids, or -1 for experts outside this partition. const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); - const expert_map = local_expert_mask.select( + var mapped_args = local_args; + mapped_args.expert_map = local_expert_mask.select( global_expert_ids.sub(expert_start), zml.Tensor.scalar(-1, .i32), ); - const local_output = triton.fusedExpertsImpl( - self.input, - self.weights_gate_up, - self.weights_down, - self.topk_weights, - self.topk_ids, - .{ - .activation = self.activation, - .global_num_experts = self.global_num_experts, - .expert_map = expert_map, - .w1_scale = self.scales_gate_up, - .w2_scale = self.scales_down, - .w1_bias = self.bias_gate_up, - .w2_bias = self.bias_down, - .quant_scheme = self.quant_scheme, - .activation_threshold = self.activation_threshold, - }, - ) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); - const local_reshaped = local_output.reshape(self.input.shape().dims()).withTags(.{ .b, .s, .d }); + const local_output = triton.fusedExpertsImpl(mapped_args) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); + const local_reshaped = local_output.reshape(local_args.hidden_states.shape().dims()).withTags(.{ .b, .s, .d }); return zml.ops.allReduce(local_reshaped, zml.Tensor.add); } }).call, - .{ - .input = input, - .topk_ids = topk_ids, - .topk_weights = topk_weights, - .weights_gate_up = gate_up.weight, - .weights_down = down.weight, - .activation = parameters.triton.activation, - .global_num_experts = global_num_experts, - .bias_gate_up = gate_up.bias, - .bias_down = down.bias, - .quant_scheme = quant_scheme, - .activation_threshold = opts.activation_threshold, - .scales_gate_up = gate_up_scales, - .scales_down = down_scales, - }, + .{ .args = args, .global_num_experts = gate_up.weight.dim(.expert) }, input.shape(), ); }, diff --git a/zml/moe/triton.zig b/zml/moe/triton.zig index caa9496d..cff06fc1 100644 --- a/zml/moe/triton.zig +++ b/zml/moe/triton.zig @@ -1,7 +1,5 @@ const std = @import("std"); -const bazel = @import("bazel"); -const bazel_builtin = @import("bazel_builtin"); const stdx = @import("stdx"); const zml = @import("../zml.zig"); @@ -20,36 +18,6 @@ test { std.testing.refAllDecls(@This()); } -// ============================================================================= -// Public API -// ============================================================================= - -pub const Options = struct { - inplace: bool = false, - activation: Parameters.ActivationMode = .silu, - apply_router_weight_on_input: bool = false, - quant_scheme: ?zml.Quantization.Scheme = null, - global_num_experts: i64 = -1, - expert_map: ?Tensor = null, - w1_scale: ?Tensor = null, - w2_scale: ?Tensor = null, - w1_zp: ?Tensor = null, - w2_zp: ?Tensor = null, - a1_scale: ?Tensor = null, - a2_scale: ?Tensor = null, - block_shape: ?[]const i64 = null, - w1_bias: ?Tensor = null, - w2_bias: ?Tensor = null, - block_size_m: i64 = 16, - block_size_n: i64 = 64, - block_size_k: i64 = 32, - group_size_m: i64 = 1, - num_warps: i64 = 8, - num_stages: i64 = 4, - dynamic_launch_by_num_tokens: bool = true, - activation_threshold: ?f32 = null, -}; - pub const Parameters = struct { num_experts_per_tok: u32, activation: ActivationMode, @@ -73,6 +41,169 @@ pub const Parameters = struct { } }; +pub const FusedExpertsArgs = struct { + hidden_states: Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + topk_weights: Tensor, + topk_ids: Tensor, + activation: Parameters.ActivationMode = .silu, + expert_map: ?Tensor = null, + activation_threshold: ?f32 = null, +}; + +pub fn fusedExpertsImpl(opts: FusedExpertsArgs) !Tensor { + const hidden_states = opts.hidden_states; + const topk_weights = opts.topk_weights; + const topk_ids = opts.topk_ids; + + if (opts.gate_up.quantizationScheme() != null and opts.gate_up.quantizationScheme().? == .mxfp4) { + const local_topk_ids, const local_topk_weights = if (opts.expert_map) |expert_map| blk: { + const local_num_experts = opts.gate_up.weight.dim(.expert); + const mapped_ids = expert_map + .gather(.{ .expert = topk_ids.convert(.i32) }, .{}) + .withTags(topk_ids.shape().tags()); + const in_range = mapped_ids.cmp(.GE, Tensor.scalar(0, .i32)) + .logical(.AND, mapped_ids.cmp(.LT, Tensor.scalar(local_num_experts, .i32))); + + break :blk .{ + in_range.select(mapped_ids, Tensor.scalar(local_num_experts, .i32)), + in_range.select(topk_weights, Tensor.scalar(0, topk_weights.dtype())), + }; + } else .{ topk_ids, topk_weights }; + + var local_args = opts; + local_args.topk_ids = local_topk_ids; + local_args.topk_weights = local_topk_weights; + return fusedExpertsImpl_fp4(local_args); + } + + const b = hidden_states.dim(.b); + const s = hidden_states.dim(.s); + const num_tokens = b * s; + const launch_config = launchConfigForTokens(num_tokens); + + const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); + const gate_up = opts.gate_up.weight.withTags(.{ .expert, .out, .in }); + const down = opts.down.weight.withTags(.{ .expert, .out, .mid }); + const weights = topk_weights.reshape(.{ .token = num_tokens, .in = topk_weights.dim(.top_expert) }).withTags(.{ .token, .topk }); + const ids = topk_ids.reshape(.{ .token = num_tokens, .in = topk_ids.dim(.top_expert) }).withTags(.{ .token, .topk }); + + stdx.debug.assert(hidden.dtype() == .bf16, "expected BF16 hidden states, got {}", .{hidden.dtype()}); + stdx.debug.assert(gate_up.dtype() == .bf16 or gate_up.dtype() == .f8e4m3fn, "expected BF16 or FP8 E4M3FN gate/up weights, got {}", .{gate_up.dtype()}); + stdx.debug.assert(down.dtype() == .bf16 or down.dtype() == .f8e4m3fn, "expected BF16 or FP8 E4M3FN down weights, got {}", .{down.dtype()}); + stdx.debug.assert(weights.dtype() == .f32 or weights.dtype() == .bf16, "expected FP32 or BF16 routing weights, got {}", .{weights.dtype()}); + stdx.debug.assert(ids.dtype() == .i32, "expected I32 expert ids, got {}", .{ids.dtype()}); + stdx.debug.assert(hidden.dim(.in) == gate_up.dim(.in), "hidden width {} must match gate/up input width {}", .{ hidden.dim(.in), gate_up.dim(.in) }); + stdx.debug.assert(@rem(gate_up.dim(.out), 2) == 0, "gate/up output width must be even, got {}", .{gate_up.dim(.out)}); + stdx.debug.assert(down.dim(.mid) == @divFloor(gate_up.dim(.out), 2), "down input width {} must equal half the gate/up output width {}", .{ down.dim(.mid), gate_up.dim(.out) }); + stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and weights.dim(.token) == hidden.dim(.token), "routing ids and weights must match hidden token count {}, got {} and {}", .{ hidden.dim(.token), ids.dim(.token), weights.dim(.token) }); + stdx.debug.assert(ids.dim(.topk) == weights.dim(.topk), "routing ids and weights must have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), weights.dim(.topk) }); + stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "gate/up and down expert counts must match, got {} and {}", .{ gate_up.dim(.expert), down.dim(.expert) }); + if (opts.expert_map) |expert_map| + stdx.debug.assert(expert_map.dtype() == .i32 and expert_map.rank() == 1, "expected a rank-1 I32 expert map, got rank {} and dtype {}", .{ expert_map.rank(), expert_map.dtype() }); + + const num_experts = if (opts.expert_map) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); + const routing = prepareRouting(ids, num_experts, launch_config.block_size_m); + + const expert_ids = if (opts.expert_map) |expert_map| + expert_map.gather(.{ .expert = routing.expert_ids }, .{}).withTags(.{.g}) + else + routing.expert_ids; + + var hidden_quant = hidden; + var a_scale = Tensor.scalar(1.0, .f32); + + if (gate_up.dtype() == .f8e4m3fn) { + hidden_quant, a_scale = quantizePerTokenGroupFp8(hidden, fp8ActivationGroupSize(hidden)); + } + + const first_cfg = makeFusedMoeConfig( + hidden_quant, + gate_up, + opts.gate_up.quantizationScheme(), + launch_config, + routing.naive_block_assignment, + ids.dim(.topk), + false, + false, + .bf16, + ); + + const gate_up_bias = + opts.gate_up.bias orelse + Tensor.zeroes(Shape.init(.{ .expert = gate_up.dim(.expert), .out = gate_up.dim(.out) }, .bf16)); + + const gate_up_scale = if (opts.gate_up.quantization) |q| q.scales else Tensor.scalar(1.0, .f32); + + const first_out = callFusedMoe( + hidden_quant, + gate_up, + gate_up_bias, + a_scale, + gate_up_scale, + weights, + routing.sorted_token_ids, + expert_ids, + routing.num_tokens_post_padded, + first_cfg, + launch_config, + routing.max_num_tokens_padded, + routing.num_assignments, + Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), + ); + + const activated = applyActivation(first_out, opts.activation, opts.activation_threshold); + var activated_quant = activated; + a_scale = Tensor.scalar(1.0, .f32); + if (down.dtype() == .f8e4m3fn) { + activated_quant, a_scale = quantizePerTokenGroupFp8(activated, fp8ActivationGroupSize(activated)); + } + + const second_cfg = makeFusedMoeConfig( + activated_quant, + down, + opts.gate_up.quantizationScheme(), + launch_config, + routing.naive_block_assignment, + 1, + true, + false, + .bf16, + ); + + const down_bias = + opts.down.bias orelse + Tensor.zeroes(Shape.init(.{ .expert = down.dim(.expert), .out = down.dim(.out) }, .bf16)); + + const down_scale = if (opts.down.quantization) |q| q.scales else Tensor.scalar(1.0, .f32); + + const second_out = callFusedMoe( + activated_quant, + down, + down_bias, + a_scale, + down_scale, + weights, + routing.sorted_token_ids, + expert_ids, + routing.num_tokens_post_padded, + second_cfg, + launch_config, + routing.max_num_tokens_padded, + routing.num_assignments, + Shape.init(.{ .token = b * s, .topk = ids.dim(.topk), .out = down.dim(.out) }, .bf16), + ); + + const output = second_out.sum(.topk).squeeze(.topk); + + return output.reshape(.{ .b = b, .token = s, .out = down.dim(.out) }); +} + +fn isMxFp8(quant_scheme: ?zml.Quantization.Scheme) bool { + return quant_scheme != null and quant_scheme.? == .mxfp8; +} + fn applyActivation(x: Tensor, mode: Parameters.ActivationMode, activation_threshold: ?f32) Tensor { const mid = @divFloor(x.dim(.out), 2); var gate = x.slice(.out, .{ .end = mid }); @@ -89,168 +220,6 @@ fn applyActivation(x: Tensor, mode: Parameters.ActivationMode, activation_thresh }; } -// ============================================================================= -// Top-level entry point -// ============================================================================= - -fn isMxFp8(quant_scheme: ?zml.Quantization.Scheme) bool { - return quant_scheme != null and quant_scheme.? == .mxfp8; -} - -pub fn fusedExpertsImpl( - hidden_states: Tensor, - w1: Tensor, - w2: Tensor, - topk_weights: Tensor, - topk_ids: Tensor, - opts: Options, -) !Tensor { - try validateOptions(opts); - - if (opts.quant_scheme != null and opts.quant_scheme.? == .mxfp4) { - const local_topk_ids, const local_topk_weights = if (opts.expert_map) |expert_map| blk: { - const local_num_experts = w1.dim(.expert); - const mapped_ids = expert_map - .gather(.{ .expert = topk_ids.convert(.i32) }, .{}) - .withTags(topk_ids.shape().tags()); - const in_range = mapped_ids.cmp(.GE, Tensor.scalar(0, .i32)) - .logical(.AND, mapped_ids.cmp(.LT, Tensor.scalar(local_num_experts, .i32))); - - break :blk .{ - in_range.select(mapped_ids, Tensor.scalar(local_num_experts, .i32)), - in_range.select(topk_weights, Tensor.scalar(0, topk_weights.dtype())), - }; - } else .{ topk_ids, topk_weights }; - - return fusedExpertsImpl_fp4( - hidden_states, - local_topk_ids, - local_topk_weights, - w1, - opts.w1_scale.?, - opts.w1_bias, - w2, - opts.w2_scale.?, - opts.w2_bias, - opts.activation_threshold, - ); - } - - const options = applyJsonTokenConfig(opts, hidden_states.dim(0)) catch |err| fallback: { - log.warn("Failed to load MoE launch config from JSON ({}), falling back to built-in token heuristic", .{err}); - break :fallback applyDefaultTokenConfig(opts, hidden_states.dim(0), w1.dim(0)); - }; - const b = hidden_states.dim(.b); - const s = hidden_states.dim(.s); - - const hidden = hidden_states.reshape(.{ .token = b * s, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); - const gate_up = w1.withTags(.{ .expert, .out, .in }); - const down = w2.withTags(.{ .expert, .out, .mid }); - const weights = topk_weights.reshape(.{ .token = b * s, .in = topk_weights.dim(.top_expert) }).withTags(.{ .token, .topk }); - const ids = topk_ids.reshape(.{ .token = b * s, .in = topk_ids.dim(.top_expert) }).withTags(.{ .token, .topk }); - - try validateInputs(hidden, gate_up, down, weights, ids); - - const block_size_m = options.block_size_m; - const num_experts = if (opts.global_num_experts != -1) opts.global_num_experts else gate_up.dim(.expert); - if (opts.expert_map) |expert_map| { - if (expert_map.dtype() != .i32) return error.UnsupportedType; - if (expert_map.rank() != 1 or expert_map.dim(.expert) != num_experts) return error.InvalidShape; - } - const routing = prepareRouting(ids, num_experts, block_size_m); - - const expert_ids = if (opts.expert_map) |expert_map| - expert_map.gather(.{ .expert = routing.expert_ids }, .{}).withTags(.{.g}) - else - routing.expert_ids; - - var hidden_quant = hidden; - var a_scale = opts.a1_scale orelse Tensor.scalar(1.0, .f32); - - if (gate_up.dtype() == .f8e4m3fn) { - hidden_quant, a_scale = quantizePerTokenGroupFp8(hidden, fp8ActivationGroupSize(hidden)); - } - - const first_cfg = makeFusedMoeConfig( - hidden_quant, - gate_up, - options, - routing.naive_block_assignment, - ids.dim(.topk), - false, - false, - .bf16, - ); - - const b_bias_1 = - opts.w1_bias orelse - Tensor.zeroes(Shape.init(.{ .expert = gate_up.dim(.expert), .out = gate_up.dim(.out) }, .bf16)); - - const b_scale_1 = opts.w1_scale orelse Tensor.scalar(1.0, .f32); - - const first_out = callFusedMoe( - hidden_quant, - gate_up, - b_bias_1, - a_scale, - b_scale_1, - weights, - routing.sorted_token_ids, - expert_ids, - routing.num_tokens_post_padded, - first_cfg, - options, - routing.max_num_tokens_padded, - routing.num_assignments, - Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), - ); - - const activated = applyActivation(first_out, options.activation, options.activation_threshold); - var activated_quant = activated; - a_scale = opts.a2_scale orelse Tensor.scalar(1.0, .f32); - if (down.dtype() == .f8e4m3fn) { - activated_quant, a_scale = quantizePerTokenGroupFp8(activated, fp8ActivationGroupSize(activated)); - } - - const second_cfg = makeFusedMoeConfig( - activated_quant, - down, - options, - routing.naive_block_assignment, - 1, - true, - false, - .bf16, - ); - - const b_bias_2 = - opts.w2_bias orelse - Tensor.zeroes(Shape.init(.{ .expert = down.dim(.expert), .out = down.dim(.out) }, .bf16)); - - const b_scale_2 = opts.w2_scale orelse Tensor.scalar(1.0, .f32); - - const second_out = callFusedMoe( - activated_quant, - down, - b_bias_2, - a_scale, - b_scale_2, - weights, - routing.sorted_token_ids, - expert_ids, - routing.num_tokens_post_padded, - second_cfg, - options, - routing.max_num_tokens_padded, - routing.num_assignments, - Shape.init(.{ .token = b * s, .topk = ids.dim(.topk), .out = down.dim(.out) }, .bf16), - ); - - const output = second_out.sum(.topk).squeeze(.topk); - - return output.reshape(.{ .b = b, .token = s, .out = down.dim(.out) }); -} - /// Build the inputs tuple for FusedMoe and invoke it via `K.call(...)`. fn callFusedMoe( a: Tensor, @@ -263,7 +232,7 @@ fn callFusedMoe( expert_ids: Tensor, num_tokens_post_padded: Tensor, cfg: kernels.FusedMoe.Cfg, - options: Options, + launch_config: LaunchConfig, max_num_tokens_padded: i64, num_valid_tokens: i64, output_shape: Shape, @@ -319,8 +288,8 @@ fn callFusedMoe( .{ .cfg = cfg, .grid = .{ @intCast(grid_x), 1, 1 }, - .num_warps = @intCast(options.num_warps), - .num_stages = @intCast(options.num_stages), + .num_warps = @intCast(launch_config.num_warps), + .num_stages = @intCast(launch_config.num_stages), }, ).c; } @@ -505,14 +474,15 @@ fn quantizePerTokenGroupFp8(x: Tensor, group_size: i64) struct { Tensor, Tensor fn makeFusedMoeConfig( a: Tensor, b: Tensor, - opts: Options, + quant_scheme: ?zml.Quantization.Scheme, + launch_config: LaunchConfig, naive_block_assignment: bool, top_k: i64, mul_routed_weight: bool, has_bias: bool, output_dtype: DataType, ) kernels.FusedMoe.Cfg { - var use_fp8 = isMxFp8(opts.quant_scheme); + var use_fp8 = isMxFp8(quant_scheme); if (b.dtype() == .f8e4m3fn) use_fp8 = true; return .{ .a_dtype = toDType(a.dtype()), @@ -522,10 +492,10 @@ fn makeFusedMoeConfig( .b_scale_dtype = if (use_fp8) .bf16 else null, .b_bias_dtype = null, .topk_weights_dtype = null, - .block_size_m = @intCast(opts.block_size_m), - .block_size_n = @intCast(opts.block_size_n), - .block_size_k = @intCast(opts.block_size_k), - .group_size_m = @intCast(opts.group_size_m), + .block_size_m = @intCast(launch_config.block_size_m), + .block_size_n = @intCast(launch_config.block_size_n), + .block_size_k = @intCast(launch_config.block_size_k), + .group_size_m = @intCast(launch_config.group_size_m), .top_k = @intCast(top_k), .naive_block_assignment = naive_block_assignment, .mul_routed_weight = mul_routed_weight, @@ -538,8 +508,7 @@ fn makeFusedMoeConfig( }; } -const DefaultTokenBucket = struct { - tokens: i64, +const LaunchConfig = struct { block_size_m: i64, block_size_n: i64, block_size_k: i64, @@ -548,110 +517,45 @@ const DefaultTokenBucket = struct { num_stages: i64, }; -fn applyDefaultTokenConfig(opts: Options, num_tokens: i64, num_experts: i64) Options { - var out = opts; - if (!opts.dynamic_launch_by_num_tokens) return out; - - // General default policy for NVIDIA bf16/fp16 and fp8 per-tensor. - // Tile sizes scale with batch size: small batches are more memory-bound, - // while larger batches benefit from wider M/N tiles and more warps. - if (num_tokens <= 32) { - out.block_size_m = 16; - } else if (num_tokens <= 96) { - out.block_size_m = 32; - } else if (num_tokens <= 512) { - out.block_size_m = 64; - } else { - out.block_size_m = 128; - } - - out.block_size_n = if (num_tokens <= 64) 64 else 128; - out.block_size_k = if (isMxFp8(opts.quant_scheme) or num_tokens <= 64) 128 else 64; - - const tokens_per_expert = @divFloor(num_tokens, @max(num_experts, 1)); - out.group_size_m = if (tokens_per_expert > 128) 16 else 1; - out.num_warps = if (num_tokens <= 128) 4 else 8; - out.num_stages = if (num_tokens <= 32) 4 else 3; - - return out; -} - -fn getLaunchConfigJsonPath(allocator: std.mem.Allocator) ![]const u8 { - const runfiles = bazel.runfiles(bazel_builtin.current_repository) catch |err| { - log.err("Failed to initialize runfiles for MoE launch config: {}", .{err}); - return err; - }; - - var config_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const config_path = - runfiles.rlocation("zml/zml/moe/triton_kernels/config.json", &config_path_buf) catch null orelse - runfiles.rlocation("zml/zml/moe/triton/triton_kernels/config.json", &config_path_buf) catch |err| { - log.err("Failed to resolve MoE launch config in runfiles: {}", .{err}); - return err; - }; - - const config_json = config_path orelse { - log.warn("MoE launch config is missing from runfiles at both zml/zml/moe/triton_kernels/config.json and zml/zml/moe/triton/triton_kernels/config.json", .{}); - return error.MissingLaunchConfigRunfile; - }; - - return try allocator.dupe(u8, config_json); -} - -const JsonConfigFields = struct { - BLOCK_SIZE_M: i64, - BLOCK_SIZE_N: i64, - BLOCK_SIZE_K: i64, - GROUP_SIZE_M: i64, - num_warps: i64, - num_stages: i64, +const launch_configs = [_]struct { tokens: i64, config: LaunchConfig }{ + .{ .tokens = 1, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 4 } }, + .{ .tokens = 2, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 4 } }, + .{ .tokens = 4, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, + .{ .tokens = 8, .config = .{ .block_size_m = 16, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 16, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 64, .group_size_m = 64, .num_warps = 4, .num_stages = 5 } }, + .{ .tokens = 24, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, + .{ .tokens = 32, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 128, .group_size_m = 1, .num_warps = 4, .num_stages = 2 } }, + .{ .tokens = 48, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 128, .group_size_m = 64, .num_warps = 4, .num_stages = 2 } }, + .{ .tokens = 64, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 128, .group_size_m = 1, .num_warps = 4, .num_stages = 2 } }, + .{ .tokens = 96, .config = .{ .block_size_m = 16, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 128, .config = .{ .block_size_m = 16, .block_size_n = 256, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, + .{ .tokens = 256, .config = .{ .block_size_m = 16, .block_size_n = 256, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, + .{ .tokens = 512, .config = .{ .block_size_m = 32, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 1024, .config = .{ .block_size_m = 64, .block_size_n = 128, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, + .{ .tokens = 1536, .config = .{ .block_size_m = 64, .block_size_n = 128, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, + .{ .tokens = 2048, .config = .{ .block_size_m = 128, .block_size_n = 128, .block_size_k = 64, .group_size_m = 16, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 3072, .config = .{ .block_size_m = 128, .block_size_n = 256, .block_size_k = 64, .group_size_m = 1, .num_warps = 8, .num_stages = 4 } }, + .{ .tokens = 4096, .config = .{ .block_size_m = 128, .block_size_n = 256, .block_size_k = 64, .group_size_m = 16, .num_warps = 8, .num_stages = 4 } }, }; -fn applyJsonTokenConfig(opts: Options, num_tokens: i64) !Options { - var out = opts; - if (!opts.dynamic_launch_by_num_tokens) return out; - - const compilation_context = zml.Compiler.current(); - const io = compilation_context.io; - const allocator = compilation_context.allocator; - - const config_path = try getLaunchConfigJsonPath(allocator); - defer allocator.free(config_path); - const config_json = try std.Io.Dir.cwd().readFileAlloc(io, config_path, allocator, .unlimited); - defer allocator.free(config_json); - const parsed = try std.json.parseFromSlice( - std.json.ArrayHashMap(JsonConfigFields), - allocator, - config_json, - .{}, - ); - defer parsed.deinit(); - - var best_diff: u64 = std.math.maxInt(u64); - var best_m: ?i64 = null; - var best_config: ?JsonConfigFields = null; - - var it = parsed.value.map.iterator(); - while (it.next()) |entry| { - const m = try std.fmt.parseInt(i64, entry.key_ptr.*, 10); - - const diff: u64 = @intCast(@abs(m - num_tokens)); - if (best_m == null or diff < best_diff or (diff == best_diff and m < best_m.?)) { - best_m = m; - best_config = entry.value_ptr.*; - best_diff = diff; +fn launchConfigForTokens(num_tokens: i64) LaunchConfig { + var best = launch_configs[0]; + for (launch_configs[1..]) |candidate| { + // The table is ascending, so ties retain the smaller token bucket. + if (@abs(candidate.tokens - num_tokens) < @abs(best.tokens - num_tokens)) { + best = candidate; } } + return best.config; +} - const cfg = best_config orelse return error.NoMatchingLaunchConfig; - out.block_size_m = cfg.BLOCK_SIZE_M; - out.block_size_n = cfg.BLOCK_SIZE_N; - out.block_size_k = cfg.BLOCK_SIZE_K; - out.group_size_m = cfg.GROUP_SIZE_M; - out.num_warps = cfg.num_warps; - out.num_stages = cfg.num_stages; - - return out; +test "MoE launch config selects the nearest token bucket" { + try std.testing.expectEqualDeep(launchConfigForTokens(4), launchConfigForTokens(6)); + try std.testing.expectEqualDeep(launchConfigForTokens(8), launchConfigForTokens(7)); + try std.testing.expectEqualDeep(launchConfigForTokens(1), launchConfigForTokens(0)); + try std.testing.expectEqualDeep(launchConfigForTokens(4096), launchConfigForTokens(8192)); + try std.testing.expectEqual(@as(i64, 64), launchConfigForTokens(16).group_size_m); + try std.testing.expectEqual(@as(i64, 5), launchConfigForTokens(16).num_stages); } fn fp8ActivationGroupSize(x: Tensor) i64 { @@ -660,45 +564,13 @@ fn fp8ActivationGroupSize(x: Tensor) i64 { return group_size; } -fn validateOptions(opts: Options) !void { - if (opts.inplace) return error.Unimplemented; - if (opts.apply_router_weight_on_input) return error.UnsupportedOption; - if (opts.quant_scheme != null and opts.quant_scheme.? != .mxfp4) return error.UnsupportedQuantization; - if (opts.expert_map != null and opts.global_num_experts == -1) return error.InvalidShape; - if (opts.w1_zp != null or opts.w2_zp != null) return error.UnsupportedOption; - if (opts.a1_scale != null or opts.a2_scale != null or opts.block_shape != null) return error.UnsupportedOption; - if (opts.w1_bias != null or opts.w2_bias != null) return error.UnsupportedOption; -} - -fn validateInputs(hidden: Tensor, gate_up: Tensor, down: Tensor, weights: Tensor, ids: Tensor) !void { - if (hidden.dtype() != .bf16) return error.UnsupportedType; - if (gate_up.dtype() != .bf16 and gate_up.dtype() != .f8e4m3fn) return error.UnsupportedType; - if (down.dtype() != .bf16 and down.dtype() != .f8e4m3fn) return error.UnsupportedType; - if (weights.dtype() != .f32 and weights.dtype() != .bf16) return error.UnsupportedType; - if (ids.dtype() != .i32) return error.UnsupportedType; - if (hidden.dim(.in) != gate_up.dim(.in)) return error.InvalidShape; - if (@rem(gate_up.dim(.out), 2) != 0) return error.InvalidShape; - if (down.dim(.mid) != @divFloor(gate_up.dim(.out), 2)) return error.InvalidShape; - if (ids.dim(.token) != hidden.dim(.token) or weights.dim(.token) != hidden.dim(.token)) return error.InvalidShape; - if (ids.dim(.topk) != weights.dim(.topk)) return error.InvalidShape; - if (gate_up.dim(.expert) != down.dim(.expert)) return error.InvalidShape; -} - // ===== // A16W4 // ===== -pub fn fusedExpertsImpl_fp4( - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - weights_gate_up: zml.Tensor, - scales_gate_up: zml.Tensor, - bias_gate_up: ?zml.Tensor, - weights_down: zml.Tensor, - scales_down: zml.Tensor, - bias_down: ?zml.Tensor, - activation_limit: ?f32, -) !zml.Tensor { +pub fn fusedExpertsImpl_fp4(args: FusedExpertsArgs) !zml.Tensor { + const input = args.hidden_states; + const topk_ids = args.topk_ids; + const topk_weights = args.topk_weights; const x = input.reshape(.{ .token = @divExact(@as(i64, @intCast(input.count())), input.dim(.d)), .d = input.dim(.d), @@ -713,9 +585,9 @@ pub fn fusedExpertsImpl_fp4( const kernel_cfg = getBestConfig( @intCast(num_tokens), @intCast(topk), - @intCast(weights_gate_up.dim(.expert)), + @intCast(args.gate_up.weight.dim(.expert)), ); - const num_experts = weights_gate_up.dim(.expert); + const num_experts = args.gate_up.weight.dim(.expert); const aligned_routing = prepareRouting(flat_topk_ids, num_experts, @intCast(kernel_cfg.block_m)); const routing = prepareFp4Routing( aligned_routing, @@ -728,13 +600,13 @@ pub fn fusedExpertsImpl_fp4( const hidden_shape: zml.Shape = .init(.{ .route = routing.num_rows, - .dout = @divExact(weights_gate_up.dim(.dout), 2), + .dout = @divExact(args.gate_up.weight.dim(.dout), 2), }, .bf16); const hidden = try runGemm( x, - weights_gate_up, - scales_gate_up, + args.gate_up.weight, + args.gate_up.quantization.?.scales, .{ .routing = routing, .weight_contract_tag = zml.Shape.toTag(.d), @@ -742,9 +614,9 @@ pub fn fusedExpertsImpl_fp4( .output_shape = hidden_shape, .gather = routing.sorted_route_ids, .gammas = routing.sorted_weights, - .bias = bias_gate_up, + .bias = args.gate_up.bias, .apply_swiglu = true, - .activation_limit = activation_limit, + .activation_limit = args.activation_threshold, .block_m = kernel_cfg.block_m, .block_n = kernel_cfg.block_n, .block_k = kernel_cfg.block_k, @@ -756,19 +628,19 @@ pub fn fusedExpertsImpl_fp4( const routed_shape: zml.Shape = .init(.{ .route = routing.num_rows, - .d = weights_down.dim(.d), + .d = args.down.weight.dim(.d), }, .bf16); const routed = try runGemm( hidden, - weights_down, - scales_down, + args.down.weight, + args.down.quantization.?.scales, .{ .routing = routing, .weight_contract_tag = zml.Shape.toTag(.dout), .weight_output_tag = zml.Shape.toTag(.d), .output_shape = routed_shape, - .bias = bias_down, + .bias = args.down.bias, .apply_swiglu = false, .activation_limit = 1.0, .block_m = kernel_cfg.block_m, diff --git a/zml/moe/triton_kernels/config.json b/zml/moe/triton_kernels/config.json deleted file mode 100644 index cc853947..00000000 --- a/zml/moe/triton_kernels/config.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "1": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 64, - "GROUP_SIZE_M": 1, - "num_warps": 4, - "num_stages": 4 - }, - "2": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 64, - "GROUP_SIZE_M": 1, - "num_warps": 4, - "num_stages": 4 - }, - "4": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 64, - "GROUP_SIZE_M": 1, - "num_warps": 4, - "num_stages": 3 - }, - "8": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 128, - "GROUP_SIZE_M": 1, - "num_warps": 8, - "num_stages": 3 - }, - "16": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 64, - "BLOCK_SIZE_K": 64, - "GROUP_SIZE_M": 64, - "num_warps": 4, - "num_stages": 5 - }, - "24": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 64, - "BLOCK_SIZE_K": 128, - "GROUP_SIZE_M": 1, - "num_warps": 8, - "num_stages": 2 - }, - "32": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 128, - "GROUP_SIZE_M": 1, - "num_warps": 4, - "num_stages": 2 - }, - "48": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 128, - "GROUP_SIZE_M": 64, - "num_warps": 4, - "num_stages": 2 - }, - "64": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 64, - "BLOCK_SIZE_K": 128, - "GROUP_SIZE_M": 1, - "num_warps": 4, - "num_stages": 2 - }, - "96": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 128, - "GROUP_SIZE_M": 1, - "num_warps": 8, - "num_stages": 3 - }, - "128": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 256, - "BLOCK_SIZE_K": 128, - "GROUP_SIZE_M": 1, - "num_warps": 8, - "num_stages": 2 - }, - "256": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 256, - "BLOCK_SIZE_K": 128, - "GROUP_SIZE_M": 1, - "num_warps": 8, - "num_stages": 2 - }, - "512": { - "BLOCK_SIZE_M": 32, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 128, - "GROUP_SIZE_M": 1, - "num_warps": 8, - "num_stages": 3 - }, - "1024": { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 64, - "GROUP_SIZE_M": 1, - "num_warps": 4, - "num_stages": 3 - }, - "1536": { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 64, - "GROUP_SIZE_M": 1, - "num_warps": 4, - "num_stages": 3 - }, - "2048": { - "BLOCK_SIZE_M": 128, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 64, - "GROUP_SIZE_M": 16, - "num_warps": 8, - "num_stages": 3 - }, - "3072": { - "BLOCK_SIZE_M": 128, - "BLOCK_SIZE_N": 256, - "BLOCK_SIZE_K": 64, - "GROUP_SIZE_M": 1, - "num_warps": 8, - "num_stages": 4 - }, - "4096": { - "BLOCK_SIZE_M": 128, - "BLOCK_SIZE_N": 256, - "BLOCK_SIZE_K": 64, - "GROUP_SIZE_M": 16, - "num_warps": 8, - "num_stages": 4 - } -} From 0ebb0c21a995c673e9a4e34c88a2663a8b3dd3c1 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 10 Sep 2026 13:57:00 +0200 Subject: [PATCH 673/881] platforms: bump plugins for CPU, ROCm, CUDA and Metal (#788) --- MODULE.bazel | 16 +- bin/zml-smi/BUILD.bazel | 2 +- bin/zml-smi/platforms/amdsmi/amdsmi.zig | 2 +- docs/tutorials/write_first_model.md | 2 +- platforms/cpu/BUILD.bazel | 10 +- platforms/cpu/cpu.bzl | 32 +- platforms/cpu/cpu.zig | 2 +- platforms/cuda/BUILD.bazel | 14 +- platforms/cuda/cuda.bzl | 24 +- platforms/cuda/cuda.zig | 6 +- ...da.BUILD.bazel => libzml_cuda.BUILD.bazel} | 10 +- .../cuda/{libpjrt_cuda.h => libzml_cuda.h} | 0 platforms/metal/BUILD.bazel | 6 +- ...l.BUILD.bazel => libzml_metal.BUILD.bazel} | 10 +- platforms/metal/metal.bzl | 10 +- platforms/metal/metal.zig | 4 +- platforms/neuron/BUILD.bazel | 16 +- ....BUILD.bazel => libzml_neuron.BUILD.bazel} | 14 +- .../{libpjrt_neuron.zig => libzml_neuron.zig} | 0 platforms/neuron/neuron.bzl | 6 +- platforms/neuron/neuron.zig | 4 +- platforms/neuron/nki_kernel.zig | 2 +- platforms/oneapi/BUILD.bazel | 8 +- ....BUILD.bazel => libzml_oneapi.BUILD.bazel} | 8 +- platforms/oneapi/oneapi.bzl | 6 +- platforms/oneapi/oneapi.zig | 4 +- platforms/rocm/BUILD.bazel | 10 +- ...cm.BUILD.bazel => libzml_rocm.BUILD.bazel} | 11 +- platforms/rocm/packages.lock.json | 8432 ++++++++++------- platforms/rocm/packages.yaml | 26 +- platforms/rocm/rocm.bzl | 30 +- platforms/rocm/rocm.zig | 4 +- platforms/tpu/BUILD.bazel | 6 +- ...tpu.BUILD.bazel => libzml_tpu.BUILD.bazel} | 8 +- platforms/tpu/tpu.bzl | 6 +- platforms/tpu/tpu.zig | 4 +- tools/BUILD.bazel | 2 +- tools/neuron/BUILD.bazel | 12 +- tools/neuron/ingest.sh | 2 +- tools/neuron/profile.sh | 2 +- tools/neuron/server.sh | 2 +- tools/neuron/summary.sh | 2 +- tools/rocprofv3.sh | 2 +- zml/Compiler.zig | 14 +- zml/attention/nki/BUILD.bazel | 2 +- zml/attention/nki/attention_harness.py | 4 +- 46 files changed, 4944 insertions(+), 3855 deletions(-) rename platforms/cuda/{libpjrt_cuda.BUILD.bazel => libzml_cuda.BUILD.bazel} (96%) rename platforms/cuda/{libpjrt_cuda.h => libzml_cuda.h} (100%) rename platforms/metal/{libpjrt_metal.BUILD.bazel => libzml_metal.BUILD.bazel} (68%) rename platforms/neuron/{libpjrt_neuron.BUILD.bazel => libzml_neuron.BUILD.bazel} (94%) rename platforms/neuron/{libpjrt_neuron.zig => libzml_neuron.zig} (100%) rename platforms/oneapi/{libpjrt_oneapi.BUILD.bazel => libzml_oneapi.BUILD.bazel} (96%) rename platforms/rocm/{libpjrt_rocm.BUILD.bazel => libzml_rocm.BUILD.bazel} (95%) rename platforms/tpu/{libpjrt_tpu.BUILD.bazel => libzml_tpu.BUILD.bazel} (85%) diff --git a/MODULE.bazel b/MODULE.bazel index 974cd6d8..c8402415 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -187,11 +187,11 @@ pip.parse( ) use_repo(pip, "xprof_nightly") -cpu = use_extension("//platforms/cpu:cpu.bzl", "cpu_pjrt_plugin") -use_repo(cpu, "libpjrt_cpu_darwin_amd64", "libpjrt_cpu_darwin_arm64", "libpjrt_cpu_linux_amd64") +cpu = use_extension("//platforms/cpu:cpu.bzl", "cpu_plugin") +use_repo(cpu, "libzml_cpu_darwin_amd64", "libzml_cpu_darwin_arm64", "libzml_cpu_linux_amd64") cuda = use_extension("//platforms/cuda:cuda.bzl", "cuda_packages") -use_repo(cuda, "cuda_nvml_dev_linux_sbsa", "cuda_nvml_dev_linux_x86_64", "cuda_nvtx_linux_sbsa", "cuda_nvtx_linux_x86_64", "libpjrt_cuda_linux_amd64", "libpjrt_cuda_linux_arm64", "zlib1g_linux_amd64", "zlib1g_linux_arm64") +use_repo(cuda, "cuda_nvml_dev_linux_sbsa", "cuda_nvml_dev_linux_x86_64", "cuda_nvtx_linux_sbsa", "cuda_nvtx_linux_x86_64", "libzml_cuda_linux_amd64", "libzml_cuda_linux_arm64", "zlib1g_linux_amd64", "zlib1g_linux_arm64") rocm = use_extension("//platforms/rocm:rocm.bzl", "rocm_packages") use_repo( @@ -211,17 +211,17 @@ use_repo( "amdrocm-sysdeps", "libatomic1", "libdrm-common", - "libpjrt_rocm", + "libzml_rocm", ) oneapi = use_extension("//platforms/oneapi:oneapi.bzl", "oneapi_packages") -use_repo(oneapi, "libpjrt_oneapi") +use_repo(oneapi, "libzml_oneapi") tpu = use_extension("//platforms/tpu:tpu.bzl", "tpu_packages") -use_repo(tpu, "libpjrt_tpu") +use_repo(tpu, "libzml_tpu") neuron = use_extension("//platforms/neuron:neuron.bzl", "neuron_packages") -use_repo(neuron, "aws-neuronx-collectives", "aws-neuronx-runtime-lib", "aws-neuronx-tools", "libgcc_s1", "libgomp1", "libpjrt_neuron", "libstdcpp6") +use_repo(neuron, "aws-neuronx-collectives", "aws-neuronx-runtime-lib", "aws-neuronx-tools", "libgcc_s1", "libgomp1", "libzml_neuron", "libstdcpp6") http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") http_archive( @@ -245,7 +245,7 @@ http_aea_archive( ) metal = use_extension("//platforms/metal:metal.bzl", "metal_packages") -use_repo(metal, "libpjrt_metal") +use_repo(metal, "libzml_metal") non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") use_repo( diff --git a/bin/zml-smi/BUILD.bazel b/bin/zml-smi/BUILD.bazel index ed22f18b..d940a801 100644 --- a/bin/zml-smi/BUILD.bazel +++ b/bin/zml-smi/BUILD.bazel @@ -48,7 +48,7 @@ copy_to_directory( "//platforms/rocm:zmlxrocm", "@amdrocm-amdsmi//:libamd_smi", "@amdrocm-sysdeps//:system_libs", - "@libpjrt_rocm//:amdgpu_ids", + "@libzml_rocm//:amdgpu_ids", ], "@llvm//platforms/config:linux_aarch64": [], }), diff --git a/bin/zml-smi/platforms/amdsmi/amdsmi.zig b/bin/zml-smi/platforms/amdsmi/amdsmi.zig index 538660e8..2d89bc7c 100644 --- a/bin/zml-smi/platforms/amdsmi/amdsmi.zig +++ b/bin/zml-smi/platforms/amdsmi/amdsmi.zig @@ -37,7 +37,7 @@ pub fn init(allocator: std.mem.Allocator) !AmdSmi { const sandbox_path = sandbox.path(&path_buf) orelse return error.AmdSmiUnavailable; var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libamd_smi.so.26" }); + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libamd_smi.so.27" }); var dynlib: std.DynLib = .{ .inner = .{ .handle = std.c.dlopen(path, .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { diff --git a/docs/tutorials/write_first_model.md b/docs/tutorials/write_first_model.md index f57a8f68..b4b1f652 100644 --- a/docs/tutorials/write_first_model.md +++ b/docs/tutorials/write_first_model.md @@ -304,7 +304,7 @@ INFO: Elapsed time: 0.120s, Critical Path: 0.00s INFO: 1 process: 1 internal. INFO: Build completed successfully, 1 total action INFO: Running command line: bazel-bin/examples/simple_layer/simple_layer -info(pjrt): Loaded library: libpjrt_cpu.dylib +info(pjrt): Loaded library: libzml_cpu.dylib debug(zml/module): ******** ZML generated MLIR ******** module @zml attributes {mhlo.num_partitions = 4 : i32, mhlo.num_replicas = 1 : i32} { diff --git a/platforms/cpu/BUILD.bazel b/platforms/cpu/BUILD.bazel index 537c7a8d..2b27ccf9 100644 --- a/platforms/cpu/BUILD.bazel +++ b/platforms/cpu/BUILD.bazel @@ -27,16 +27,16 @@ cc_library( copy_to_directory( name = "sandbox", srcs = select({ - ":darwin_arm64": ["@libpjrt_cpu_darwin_arm64//:libpjrt_cpu"], - ":darwin_amd64": ["@libpjrt_cpu_darwin_amd64//:libpjrt_cpu"], - "@platforms//os:linux": ["@libpjrt_cpu_linux_amd64//:libpjrt_cpu"], + ":darwin_arm64": ["@libzml_cpu_darwin_arm64//:libzml_cpu"], + ":darwin_amd64": ["@libzml_cpu_darwin_amd64//:libzml_cpu"], + "@platforms//os:linux": ["@libzml_cpu_linux_amd64//:libzml_cpu"], }), out = "sandbox/lib", include_external_repositories = ["**"], ) cc_library( - name = "libpjrt_cpu", + name = "libzml_cpu", tags = ["manual"], data = [":sandbox"], defines = ["ZML_RUNTIME_CPU"], @@ -52,7 +52,7 @@ zig_library( "//pjrt", ] + select({ "//platforms:cpu.enabled": [ - ":libpjrt_cpu", + ":libzml_cpu", "//bazel", "//stdx", "@rules_zig//zig/runfiles", diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 7281d2d8..d41a00b0 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -7,38 +7,38 @@ package(default_visibility = ["//visibility:public"]) _BUILD_LINUX = "\n".join([ packages.filegroup( - name = "libpjrt_cpu", - srcs = ["libpjrt_cpu.so"], + name = "libzml_cpu", + srcs = ["libzml_cpu.so"], visibility = ["@zml//platforms/cpu:__subpackages__"], ), ]) _BUILD_DARWIN = packages.filegroup( - name = "libpjrt_cpu", - srcs = ["libpjrt_cpu.dylib"], + name = "libzml_cpu", + srcs = ["libzml_cpu.dylib"], visibility = ["@zml//platforms/cpu:__subpackages__"], ) -def _cpu_pjrt_plugin_impl(mctx): +def _cpu_plugin_impl(mctx): http_archive( - name = "libpjrt_cpu_linux_amd64", + name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "65e631db0f842845e7799d245a414b361a3c3e77bf4cc0547c20c71f28a9fd70", - url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-07-03T00-10-30Z/pjrt-cpu_linux-amd64.tar.gz", + sha256 = "f1670d21b92102b8c7c7f9d0f00d6a7a1a5d90a83d7e28ba5fbeb1f9ce4c170c", + url = "https://mirror.zml.ai/plugins/202609101243.20.1.7ca6884ea2cb/zml-cpu-linux-amd64.tar.zst", ) http_archive( - name = "libpjrt_cpu_darwin_amd64", + name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "22130f752abdaaa0f3ff48e71b9191ce74b29cd3fdb0c4784f04e4c4a436f25e", - url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-07-03T00-10-30Z/pjrt-cpu_darwin-amd64.tar.gz", + sha256 = "d8c7dea830773eb37bb6161f1ed62f69f30bd20a2d669083c5d6a7d60c19844b", + url = "https://mirror.zml.ai/plugins/202609101243.20.1.7ca6884ea2cb/zml-cpu-darwin-amd64.tar.zst", ) http_archive( - name = "libpjrt_cpu_darwin_arm64", + name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "14c85504d801c75fa8d157ce951a2644d8a8d7983346b3ac281aa7f64abf8390", - url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-07-03T00-10-30Z/pjrt-cpu_darwin-arm64.tar.gz", + sha256 = "b8c5a1d15fe3b6ce8520852901ccec3cb12d092b052fdd527449d822833becba", + url = "https://mirror.zml.ai/plugins/202609101243.20.1.7ca6884ea2cb/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( @@ -47,6 +47,6 @@ def _cpu_pjrt_plugin_impl(mctx): root_module_direct_dev_deps = [], ) -cpu_pjrt_plugin = module_extension( - implementation = _cpu_pjrt_plugin_impl, +cpu_plugin = module_extension( + implementation = _cpu_plugin_impl, ) diff --git a/platforms/cpu/cpu.zig b/platforms/cpu/cpu.zig index 83d21246..77a3fa60 100644 --- a/platforms/cpu/cpu.zig +++ b/platforms/cpu/cpu.zig @@ -37,7 +37,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { }; var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "libpjrt_cpu" ++ ext }); + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "libzml_cpu" ++ ext }); break :blk .loadFrom(path); }; } diff --git a/platforms/cuda/BUILD.bazel b/platforms/cuda/BUILD.bazel index 3122bdaa..437f945b 100644 --- a/platforms/cuda/BUILD.bazel +++ b/platforms/cuda/BUILD.bazel @@ -25,10 +25,10 @@ cc_library( ) cc_library( - name = "libpjrt_cuda_repo", + name = "libzml_cuda_repo", deps = select({ - "@llvm//platforms/config:linux_x86_64": ["@libpjrt_cuda_linux_amd64//:libpjrt_cuda"], - "@llvm//platforms/config:linux_aarch64": ["@libpjrt_cuda_linux_arm64//:libpjrt_cuda"], + "@llvm//platforms/config:linux_x86_64": ["@libzml_cuda_linux_amd64//:libzml_cuda"], + "@llvm//platforms/config:linux_aarch64": ["@libzml_cuda_linux_arm64//:libzml_cuda"], }), ) @@ -42,11 +42,11 @@ cc_library( ) cc_library( - name = "libpjrt_cuda", + name = "libzml_cuda", tags = ["manual"], - hdrs = ["libpjrt_cuda.h"], + hdrs = ["libzml_cuda.h"], defines = ["ZML_RUNTIME_CUDA"], - deps = [":libpjrt_cuda_repo"], + deps = [":libzml_cuda_repo"], ) zig_library( @@ -60,7 +60,7 @@ zig_library( "//pjrt", ] + select({ "//platforms:cuda.enabled": [ - ":libpjrt_cuda", + ":libzml_cuda", "//bazel", "//stdx", "@rules_zig//zig/runfiles", diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 2b248e3c..4884f35c 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -213,16 +213,14 @@ _UBUNTU_PACKAGES = { ], } -PJRT_CUDA_RELEASE = "manual-2026-09-08T13-23-00Z" - -_PJRT_CUDA_ASSETS = { +_PLUGINS = { "amd64": { - "sha256": "e86ee894daeeeb83c95347bcda89df4317185d5e9ef9ff9290afa471f57a135a", - "url": "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-cuda_linux-amd64.tar.gz", + "sha256": "2ef0e28330d22ce3039adb1de0d5dabf8bd7126988f9b61ba427946d6c60cb93", + "url": "https://mirror.zml.ai/plugins/202609101243.20.1.7ca6884ea2cb/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "60aae4b73193d980a04d478f85038f4c6069f2dd9559a6549a8f14c420e0d857", - "url": "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-cuda_linux-arm64.tar.gz", + "sha256": "f8180188c4557c526571b90cb54354819043d05c492228a9d1179ac2bef26862", + "url": "https://mirror.zml.ai/plugins/202609101243.20.1.7ca6884ea2cb/zml-cuda-linux-arm64.tar.zst", }, } @@ -349,19 +347,19 @@ def _cuda_impl(mctx): ]), ) - for arch, arch_config in _PJRT_CUDA_ASSETS.items(): + for arch, arch_config in _PLUGINS.items(): http_archive( - name = _repo_name("libpjrt_cuda", arch), - build_file = "libpjrt_cuda.BUILD.bazel", - url = arch_config["url"].format(release = PJRT_CUDA_RELEASE), + name = _repo_name("libzml_cuda", arch), + build_file = "libzml_cuda.BUILD.bazel", + url = arch_config["url"], sha256 = arch_config["sha256"], ) return mctx.extension_metadata( reproducible = True, root_module_direct_deps = [ - "libpjrt_cuda_linux_amd64", - "libpjrt_cuda_linux_arm64", + "libzml_cuda_linux_amd64", + "libzml_cuda_linux_arm64", "cuda_nvml_dev_linux_x86_64", "cuda_nvml_dev_linux_sbsa", "zlib1g_linux_arm64", diff --git a/platforms/cuda/cuda.zig b/platforms/cuda/cuda.zig index 2a711a64..6400290e 100644 --- a/platforms/cuda/cuda.zig +++ b/platforms/cuda/cuda.zig @@ -18,8 +18,8 @@ fn findCudaSandbox( buffer: *[std.Io.Dir.max_path_bytes]u8, ) !?[]const u8 { const candidate = switch (builtin.cpu.arch) { - .aarch64 => "libpjrt_cuda_linux_arm64/sandbox", - .x86_64 => "libpjrt_cuda_linux_amd64/sandbox", + .aarch64 => "libzml_cuda_linux_arm64/sandbox", + .x86_64 => "libzml_cuda_linux_amd64/sandbox", else => return null, }; return try r.rlocation(candidate, buffer); @@ -125,7 +125,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_cuda.so" }); + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_cuda.so" }); break :blk .loadFrom(path); }; } diff --git a/platforms/cuda/libpjrt_cuda.BUILD.bazel b/platforms/cuda/libzml_cuda.BUILD.bazel similarity index 96% rename from platforms/cuda/libpjrt_cuda.BUILD.bazel rename to platforms/cuda/libzml_cuda.BUILD.bazel index 84057fb5..0378ef32 100644 --- a/platforms/cuda/libpjrt_cuda.BUILD.bazel +++ b/platforms/cuda/libzml_cuda.BUILD.bazel @@ -4,8 +4,8 @@ load("@zml//bazel:patchelf.bzl", "patchelf") load("@zml//platforms/cuda:cuda.bzl", "CUDA_COMPAT_FILES") patchelf( - name = "libpjrt_cuda_so", - src = "libpjrt_cuda.so", + name = "libzml_cuda_so", + src = "libzml_cuda.so", add_needed = [ "libzmlxcuda.so.0", ], @@ -18,7 +18,7 @@ patchelf( copy_to_directory( name = "sandbox", srcs = [ - ":libpjrt_cuda_so", + ":libzml_cuda_so", "@zml//platforms/cuda:compat_probe", "@zml//platforms/cuda:zmlxcuda", ] + select({ @@ -67,7 +67,7 @@ copy_to_directory( "compat": "lib/compat", "nvidia/nccl/lib": "lib", "nvvm/lib64": "lib", - "libpjrt_cuda_so": "lib", + "libzml_cuda_so": "lib", "lib/x86_64-linux-gnu": "lib", "lib/aarch64-linux-gnu": "lib", "platforms/cuda/compat_probe": "bin/compat_probe", @@ -79,7 +79,7 @@ copy_to_directory( ) cc_library( - name = "libpjrt_cuda", + name = "libzml_cuda", data = [":sandbox"], linkopts = [ # Defer function call resolution until the function is called diff --git a/platforms/cuda/libpjrt_cuda.h b/platforms/cuda/libzml_cuda.h similarity index 100% rename from platforms/cuda/libpjrt_cuda.h rename to platforms/cuda/libzml_cuda.h diff --git a/platforms/metal/BUILD.bazel b/platforms/metal/BUILD.bazel index 5041fb8c..c4a6c018 100644 --- a/platforms/metal/BUILD.bazel +++ b/platforms/metal/BUILD.bazel @@ -24,11 +24,11 @@ copy_to_directory( ) cc_library( - name = "libpjrt_metal", + name = "libzml_metal", data = [":metal_toolchain"], defines = ["ZML_RUNTIME_METAL"], tags = ["manual"], - deps = ["@libpjrt_metal"], + deps = ["@libzml_metal"], ) zig_library( @@ -41,7 +41,7 @@ zig_library( "//pjrt", ] + select({ "//platforms:metal.enabled": [ - ":libpjrt_metal", + ":libzml_metal", "//bazel", "//stdx", "@rules_zig//zig/runfiles", diff --git a/platforms/metal/libpjrt_metal.BUILD.bazel b/platforms/metal/libzml_metal.BUILD.bazel similarity index 68% rename from platforms/metal/libpjrt_metal.BUILD.bazel rename to platforms/metal/libzml_metal.BUILD.bazel index 9ffa0cc9..777fb9bc 100644 --- a/platforms/metal/libpjrt_metal.BUILD.bazel +++ b/platforms/metal/libzml_metal.BUILD.bazel @@ -2,23 +2,23 @@ load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@bazel_skylib//rules:copy_file.bzl", "copy_file") copy_file( - name = "libpjrt_metal.dylib", - src = "libpjrt_c_api_gpu_plugin.dylib", - out = "lib/libpjrt_c_api_gpu_plugin.dylib", + name = "libzml_metal_dylib", + src = "libzml_metal.dylib", + out = "lib/libzml_metal.dylib", allow_symlink = True, ) copy_to_directory( name = "sandbox", srcs = [ - ":libpjrt_metal.dylib", + ":libzml_metal_dylib", ], add_directory_to_runfiles = False, include_external_repositories = ["**"], ) cc_library( - name = "libpjrt_metal", + name = "libzml_metal", data = [":sandbox"], visibility = ["@zml//platforms/metal:__subpackages__"], ) diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index f1af18c0..26c78880 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -2,15 +2,15 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") def _metal_impl(mctx): http_archive( - name = "libpjrt_metal", - build_file = "libpjrt_metal.BUILD.bazel", - sha256 = "45667d5726844c093b319568d1696694a022fb9d58bba7321822c2544f1417ce", - url = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-09-07T15-23-00Z/pjrt-metal_macos-arm64.tar.zst", + name = "libzml_metal", + build_file = "libzml_metal.BUILD.bazel", + sha256 = "e329f4c89f9e5c64cf46c8665cb89d21ffbd198c4d4e7facdeb41654388521b2", + url = "https://mirror.zml.ai/plugins/202609101243.20.1.7ca6884ea2cb/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( reproducible = True, - root_module_direct_deps = ["libpjrt_metal"], + root_module_direct_deps = ["libzml_metal"], root_module_direct_dev_deps = [], ) diff --git a/platforms/metal/metal.zig b/platforms/metal/metal.zig index e1fbd6c3..6da9be29 100644 --- a/platforms/metal/metal.zig +++ b/platforms/metal/metal.zig @@ -35,7 +35,7 @@ pub fn load(_: std.mem.Allocator, io: std.Io) !*const pjrt.Api { const r = try bazel.runfiles(bazel_builtin.current_repository); var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = try r.rlocation("libpjrt_metal/sandbox", &path_buf) orelse { + const sandbox_path = try r.rlocation("libzml_metal/sandbox", &path_buf) orelse { log.err("Failed to find sandbox path for Metal runtime", .{}); return error.FileNotFound; }; @@ -46,7 +46,7 @@ pub fn load(_: std.mem.Allocator, io: std.Io) !*const pjrt.Api { return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_c_api_gpu_plugin.dylib" }); + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_metal.dylib" }); break :blk pjrt.Api.loadFrom(path); }; } diff --git a/platforms/neuron/BUILD.bazel b/platforms/neuron/BUILD.bazel index 540426db..fada3d5c 100644 --- a/platforms/neuron/BUILD.bazel +++ b/platforms/neuron/BUILD.bazel @@ -15,11 +15,11 @@ exports_files([ # The proxy also initializes the Neuron runtime environment and exposes the # sandboxed compiler binaries to libneuronxla 3.0. zig_shared_library( - name = "libpjrt_neuron", + name = "libzml_neuron", srcs = ["neuron.zig"], - main = "libpjrt_neuron.zig", + main = "libzml_neuron.zig", target_compatible_with = ["@platforms//os:linux"], - visibility = ["@libpjrt_neuron//:__subpackages__"], + visibility = ["@libzml_neuron//:__subpackages__"], zigopts = ["-fno-stack-check"], deps = [ ":libnrt_headers", @@ -49,14 +49,14 @@ zig_binary( runfiles_to_default( name = "neuronx-cc_files", target_compatible_with = ["@platforms//os:linux"], - visibility = ["@libpjrt_neuron//:__subpackages__"], + visibility = ["@libzml_neuron//:__subpackages__"], deps = [":neuronx-cc"], ) runfiles_to_default( name = "nki-cc_files", target_compatible_with = ["@platforms//os:linux"], - visibility = ["@libpjrt_neuron//:__subpackages__"], + visibility = ["@libzml_neuron//:__subpackages__"], deps = [":nki-cc"], ) @@ -90,7 +90,7 @@ cc_library( visibility = [ "//bin/zml-smi:__subpackages__", ], - deps = ["@libpjrt_neuron//:libnrt_headers"], + deps = ["@libzml_neuron//:libnrt_headers"], ) cc_import( @@ -176,7 +176,7 @@ zig_library( ":zmlxneuron", "//bazel", "//stdx", - "@libpjrt_neuron", + "@libzml_neuron", "@rules_zig//zig/runfiles", ], "//conditions:default": [":empty"], @@ -185,7 +185,7 @@ zig_library( zig_library( name = "topology", - data = [":libpjrt_neuron"], + data = [":libzml_neuron"], import_name = "platforms/neuron/topology", main = "topology.zig", tags = ["manual"], diff --git a/platforms/neuron/libpjrt_neuron.BUILD.bazel b/platforms/neuron/libzml_neuron.BUILD.bazel similarity index 94% rename from platforms/neuron/libpjrt_neuron.BUILD.bazel rename to platforms/neuron/libzml_neuron.BUILD.bazel index d8f43146..824accb0 100644 --- a/platforms/neuron/libpjrt_neuron.BUILD.bazel +++ b/platforms/neuron/libzml_neuron.BUILD.bazel @@ -23,16 +23,16 @@ patchelf( ) patchelf( - name = "libpjrt_neuron_so", - src = "@zml//platforms/neuron:libpjrt_neuron", + name = "libzml_neuron_so", + src = "@zml//platforms/neuron:libzml_neuron", set_rpath = "$ORIGIN", - soname = "libpjrt_neuron.so", + soname = "libzml_neuron.so", ) patchelf( name = "neuron_explorer", src = "@aws-neuronx-tools//:neuron-explorer", - add_needed = ["libpjrt_neuron.so"], + add_needed = ["libzml_neuron.so"], set_rpath = "$ORIGIN/../lib", ) @@ -46,7 +46,7 @@ copy_to_directory( srcs = [ ":libneuronpjrt_so", ":libneuronxla_files", - ":libpjrt_neuron_so", + ":libzml_neuron_so", ":neuron_explorer", "@aws-neuronx-collectives//:libnccom", "@aws-neuronx-runtime-lib//:libncfw.patchelf", @@ -96,7 +96,7 @@ copy_to_directory( "platforms/neuron/nki_kernel_compiler.py": "site-packages/nki_kernel_compiler.py", "neuron_explorer": "bin", "libneuronpjrt_so": "lib", - "libpjrt_neuron_so": "lib", + "libzml_neuron_so": "lib", "libnrt.patchelf": "lib", "libncfw.patchelf": "lib", "libnrtucode_extisa": "lib", @@ -130,7 +130,7 @@ cc_library( ) cc_library( - name = "libpjrt_neuron", + name = "libzml_neuron", data = [":sandbox"], visibility = ["@zml//platforms/neuron:__subpackages__"], ) diff --git a/platforms/neuron/libpjrt_neuron.zig b/platforms/neuron/libzml_neuron.zig similarity index 100% rename from platforms/neuron/libpjrt_neuron.zig rename to platforms/neuron/libzml_neuron.zig diff --git a/platforms/neuron/neuron.bzl b/platforms/neuron/neuron.bzl index c6f772fa..8b947c04 100644 --- a/platforms/neuron/neuron.bzl +++ b/platforms/neuron/neuron.bzl @@ -87,8 +87,8 @@ def _neuron_impl(mctx): ]) simple_repository( - name = "libpjrt_neuron", - build_file = ":libpjrt_neuron.BUILD.bazel", + name = "libzml_neuron", + build_file = ":libzml_neuron.BUILD.bazel", ) for repo_name, package in _UBUNTU_PACKAGES.items(): @@ -119,7 +119,7 @@ def _neuron_impl(mctx): "libgcc_s1", "libgomp1", "libstdcpp6", - "libpjrt_neuron", + "libzml_neuron", ], root_module_direct_dev_deps = [], ) diff --git a/platforms/neuron/neuron.zig b/platforms/neuron/neuron.zig index a25e0cf0..f3df8909 100644 --- a/platforms/neuron/neuron.zig +++ b/platforms/neuron/neuron.zig @@ -136,14 +136,14 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { const r = try bazel.runfiles(bazel_builtin.current_repository); var sandbox_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = try r.rlocation("libpjrt_neuron/sandbox", &sandbox_path_buf) orelse { + const sandbox_path = try r.rlocation("libzml_neuron/sandbox", &sandbox_path_buf) orelse { log.err("Failed to find sandbox path for NEURON runtime", .{}); return error.FileNotFound; }; return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_neuron.so" }); + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_neuron.so" }); break :blk .loadFrom(path); }; } diff --git a/platforms/neuron/nki_kernel.zig b/platforms/neuron/nki_kernel.zig index 7bbb60be..4d897be8 100644 --- a/platforms/neuron/nki_kernel.zig +++ b/platforms/neuron/nki_kernel.zig @@ -80,7 +80,7 @@ pub fn compileNkiKernel(allocator: std.mem.Allocator, io: std.Io, kernel: Kernel const r = try bazel.runfiles(bazel_builtin.current_repository); var sandbox_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = try r.rlocation("libpjrt_neuron/sandbox", &sandbox_path_buf) orelse return error.FileNotFound; + const sandbox_path = try r.rlocation("libzml_neuron/sandbox", &sandbox_path_buf) orelse return error.FileNotFound; var source_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const source_path = try r.rlocation(kernel.source_path, &source_path_buf) orelse return error.FileNotFound; diff --git a/platforms/oneapi/BUILD.bazel b/platforms/oneapi/BUILD.bazel index f4d23804..7193eb2a 100644 --- a/platforms/oneapi/BUILD.bazel +++ b/platforms/oneapi/BUILD.bazel @@ -6,7 +6,7 @@ zig_shared_library( main = "zmlxoneapi.zig", shared_lib_name = "libzmlxoneapi.so.0", target_compatible_with = ["@platforms//os:linux"], - visibility = ["@libpjrt_oneapi//:__subpackages__"], + visibility = ["@libzml_oneapi//:__subpackages__"], zigopts = ["-fno-stack-check"], deps = ["//stdx"], ) @@ -18,10 +18,10 @@ cc_library( ) cc_library( - name = "libpjrt_oneapi", + name = "libzml_oneapi", defines = ["ZML_RUNTIME_ONEAPI"], tags = ["manual"], - deps = ["@libpjrt_oneapi"], + deps = ["@libzml_oneapi"], ) zig_library( @@ -37,7 +37,7 @@ zig_library( "@rules_zig//zig/runfiles", ] + select({ "//platforms:oneapi.enabled": [ - ":libpjrt_oneapi", + ":libzml_oneapi", ], "//conditions:default": [":empty"], }), diff --git a/platforms/oneapi/libpjrt_oneapi.BUILD.bazel b/platforms/oneapi/libzml_oneapi.BUILD.bazel similarity index 96% rename from platforms/oneapi/libpjrt_oneapi.BUILD.bazel rename to platforms/oneapi/libzml_oneapi.BUILD.bazel index ea7f51e9..62879ed6 100644 --- a/platforms/oneapi/libpjrt_oneapi.BUILD.bazel +++ b/platforms/oneapi/libzml_oneapi.BUILD.bazel @@ -2,7 +2,7 @@ load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@zml//bazel:patchelf.bzl", "patchelf") patchelf( - name = "libpjrt_oneapi_so", + name = "libzml_oneapi_so", src = "libpjrt_oneapi.so", add_needed = [ "libzmlxoneapi.so.0", @@ -86,7 +86,7 @@ copy_to_directory( ":libiga64_so_2", ":libigc_so_2", ":libigdfcl_so_2", - ":libpjrt_oneapi_so", + ":libzml_oneapi_so", ":libsycl_so", ":libtbb_so_12", ":libze_intel_gpu_so", @@ -121,7 +121,7 @@ copy_to_directory( "libigdfcl_so_2": "lib", "libigc_so_2": "lib", "libdnnl_so_3": "lib", - "libpjrt_oneapi_so": "lib", + "libzml_oneapi_so": "lib", "libsycl_so": "lib", "libtbb_so_12": "lib", ONEAPI_CCL_LIB: "lib", @@ -139,7 +139,7 @@ copy_to_directory( ) cc_library( - name = "libpjrt_oneapi", + name = "libzml_oneapi", data = [":sandbox"], visibility = ["@zml//platforms/oneapi:__subpackages__"], ) diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index fa3b121e..5069f8b5 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -280,7 +280,7 @@ filegroup( )""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), } -_ROOT_MODULE_DIRECT_DEPS = ["libpjrt_oneapi"] +_ROOT_MODULE_DIRECT_DEPS = ["libzml_oneapi"] def _oneapi_impl(mctx): loaded_packages = _read_packages(mctx, [ @@ -288,8 +288,8 @@ def _oneapi_impl(mctx): ]) http_archive( - name = "libpjrt_oneapi", - build_file = "libpjrt_oneapi.BUILD.bazel", + name = "libzml_oneapi", + build_file = "libzml_oneapi.BUILD.bazel", sha256 = PJRT_ONEAPI_ARTIFACT_SHA256, url = PJRT_ONEAPI_ARTIFACT_URL, ) diff --git a/platforms/oneapi/oneapi.zig b/platforms/oneapi/oneapi.zig index 0e411fbf..628c9f5c 100644 --- a/platforms/oneapi/oneapi.zig +++ b/platforms/oneapi/oneapi.zig @@ -55,7 +55,7 @@ pub fn load(_: std.mem.Allocator, io: std.Io) !*const pjrt.Api { const r = try bazel.runfiles(bazel_builtin.current_repository); var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = try r.rlocation("libpjrt_oneapi/sandbox", &path_buf) orelse { + const sandbox_path = try r.rlocation("libzml_oneapi/sandbox", &path_buf) orelse { log.err("Failed to find sandbox path for oneAPI runtime", .{}); return error.FileNotFound; }; @@ -66,7 +66,7 @@ pub fn load(_: std.mem.Allocator, io: std.Io) !*const pjrt.Api { return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_oneapi.so" }); + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_oneapi.so" }); break :blk pjrt.Api.loadFrom(path); }; } diff --git a/platforms/rocm/BUILD.bazel b/platforms/rocm/BUILD.bazel index 34064208..b00be3f3 100644 --- a/platforms/rocm/BUILD.bazel +++ b/platforms/rocm/BUILD.bazel @@ -3,7 +3,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_shared_library") exports_files( ["merge_amdgpu_ids.zig"], - visibility = ["@libpjrt_rocm//:__subpackages__"], + visibility = ["@libzml_rocm//:__subpackages__"], ) zig_shared_library( @@ -13,7 +13,7 @@ zig_shared_library( target_compatible_with = ["@platforms//os:linux"], visibility = [ "//bin/zml-smi:__pkg__", - "@libpjrt_rocm//:__subpackages__", + "@libzml_rocm//:__subpackages__", ], # Use Clang's compiler-rt, but disable stack checking # to avoid requiring on the _zig_probe_stack symbol. @@ -28,10 +28,10 @@ cc_library( ) cc_library( - name = "libpjrt_rocm", + name = "libzml_rocm", defines = ["ZML_RUNTIME_ROCM"], tags = ["manual"], - deps = ["@libpjrt_rocm"], + deps = ["@libzml_rocm"], ) zig_library( @@ -44,7 +44,7 @@ zig_library( "//pjrt", ] + select({ "//platforms:rocm.enabled": [ - ":libpjrt_rocm", + ":libzml_rocm", "//bazel", "//stdx", "@rules_zig//zig/runfiles", diff --git a/platforms/rocm/libpjrt_rocm.BUILD.bazel b/platforms/rocm/libzml_rocm.BUILD.bazel similarity index 95% rename from platforms/rocm/libpjrt_rocm.BUILD.bazel rename to platforms/rocm/libzml_rocm.BUILD.bazel index 504e71b1..3de1dfc4 100644 --- a/platforms/rocm/libpjrt_rocm.BUILD.bazel +++ b/platforms/rocm/libzml_rocm.BUILD.bazel @@ -5,8 +5,8 @@ load("@rules_zig//zig:defs.bzl", "zig_binary") load("@zml//bazel:patchelf.bzl", "patchelf") patchelf( - name = "libpjrt_rocm_so", - src = "libpjrt_rocm.so", + name = "libzml_rocm_so", + src = "libzml_rocm.so", add_needed = [ "libzmlxrocm.so.0", ], @@ -41,10 +41,9 @@ copy_to_directory( name = "sandbox", srcs = [ ":amdgpu_ids", - ":libpjrt_rocm_so", + ":libzml_rocm_so", "@amdrocm-amdsmi//:libamd_smi", "@amdrocm-base//:rocm-core", - "@amdrocm-base//:rocm_smi", "@amdrocm-base//:rocprofiler-register", "@amdrocm-blas//:hipblaslt", "@amdrocm-blas//:hipblaslt_runfiles", @@ -94,7 +93,7 @@ copy_to_directory( "lib/host-math/lib": "lib", "lib/llvm/lib": "lib", "lib/llvm/amdgcn": "amdgcn", - "libpjrt_rocm_so": "lib", + "libzml_rocm_so": "lib", "amdgpu.ids": "share/libdrm/amdgpu.ids", "rccl": "lib", "rocblas": "lib", @@ -116,7 +115,7 @@ copy_to_directory( ) cc_library( - name = "libpjrt_rocm", + name = "libzml_rocm", data = [":sandbox"], visibility = ["@zml//platforms/rocm:__subpackages__"], ) diff --git a/platforms/rocm/packages.lock.json b/platforms/rocm/packages.lock.json index c15018c7..6dce1849 100644 --- a/platforms/rocm/packages.lock.json +++ b/platforms/rocm/packages.lock.json @@ -91,9 +91,9 @@ "arch": "amd64", "dependencies": [ { - "key": "amdrocm-sysdeps7.14_7.14.0-3_amd64", - "name": "amdrocm-sysdeps7.14", - "version": "7.14.0-3" + "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0~20260903-33697653391" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -316,24 +316,24 @@ "version": "3.10.6-1~22.04.1" } ], - "key": "amdrocm-amdsmi7.14_7.14.0-3_amd64", - "name": "amdrocm-amdsmi7.14", - "sha256": "f0004a48ebec4e8e2c059cc196058e870eeb8ab38daa3b58c24e350f7b42b20c", + "key": "amdrocm-amdsmi10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-amdsmi10.1", + "sha256": "9d722a1406c1701e48d93f6c2616eb8b0cf71787de2d074a70124eaae9ea221d", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-amdsmi7.14_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-amdsmi10.1_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sysdeps7.14_7.14.0-3_amd64", - "name": "amdrocm-sysdeps7.14", - "sha256": "6e6ab8b0560f9157ceb2afefe481eddb93789041cbb53137018571d749205a8b", + "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sysdeps10.1", + "sha256": "ccfd6b77ecb56e982bbf18c855c4b402cd642b8fba411112f6499987f4d60ba9", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sysdeps7.14_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sysdeps10.1_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", @@ -779,19 +779,19 @@ "arch": "amd64", "dependencies": [ { - "key": "amdrocm-runtime7.14_7.14.0-3_amd64", - "name": "amdrocm-runtime7.14", - "version": "7.14.0-3" + "key": "amdrocm-runtime10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-llvm7.14_7.14.0-3_amd64", - "name": "amdrocm-llvm7.14", - "version": "7.14.0-3" + "key": "amdrocm-llvm10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sysdeps7.14_7.14.0-3_amd64", - "name": "amdrocm-sysdeps7.14", - "version": "7.14.0-3" + "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0~20260903-33697653391" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -1104,37 +1104,37 @@ "version": "3.10.6-1~22.04.1" } ], - "key": "amdrocm-base7.14_7.14.0-3_amd64", - "name": "amdrocm-base7.14", - "sha256": "4221475e72a3481ae746f7562d6db2094133bb5b9c394fd7f9f1820a5f83695b", + "key": "amdrocm-base10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-base10.1", + "sha256": "2449f0433a9d3b93af4ad4bc42cef5429b3318745e809f83e21b1c20cc838f63", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-base7.14_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-base10.1_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-runtime7.14_7.14.0-3_amd64", - "name": "amdrocm-runtime7.14", - "sha256": "3b940320e839d68e05360ef70e6c4d3222cd135a4b02d68ece2267366330748c", + "key": "amdrocm-runtime10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-runtime10.1", + "sha256": "62accad621bfeb32147c1dff30dfdd9b22ca6bbc6e69c07e9a39d0eca6863e18", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-runtime7.14_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-runtime10.1_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [ { - "key": "amdrocm-runtime7.14_7.14.0-3_amd64", - "name": "amdrocm-runtime7.14", - "version": "7.14.0-3" + "key": "amdrocm-runtime10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sysdeps7.14_7.14.0-3_amd64", - "name": "amdrocm-sysdeps7.14", - "version": "7.14.0-3" + "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0~20260903-33697653391" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -1357,9 +1357,9 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "amdrocm-base7.14_7.14.0-3_amd64", - "name": "amdrocm-base7.14", - "version": "7.14.0-3" + "key": "amdrocm-base10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0~20260903-33697653391" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -1452,13 +1452,13 @@ "version": "3.10.6-1~22.04.1" } ], - "key": "amdrocm-llvm7.14_7.14.0-3_amd64", - "name": "amdrocm-llvm7.14", - "sha256": "46f336f876528d0c2c98503491cb1e1e176986caaf2ea281f84c56c1a8822747", + "key": "amdrocm-llvm10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-llvm10.1", + "sha256": "5f185639c5f2cba6aefe1b1a945b0a05fadb13763ee094e933114e5f153471ae", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-llvm7.14_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-llvm10.1_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", @@ -1651,39 +1651,39 @@ "arch": "amd64", "dependencies": [ { - "key": "amdrocm-blas7.14-gfx950_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx950", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx950", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx950_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx950", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx950", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx950_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx950", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx950", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse-host7.14_7.14.0-3_amd64", - "name": "amdrocm-sparse-host7.14", - "version": "7.14.0-3" + "key": "amdrocm-sparse-host10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse-host10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-runtime7.14_7.14.0-3_amd64", - "name": "amdrocm-runtime7.14", - "version": "7.14.0-3" + "key": "amdrocm-runtime10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-llvm7.14_7.14.0-3_amd64", - "name": "amdrocm-llvm7.14", - "version": "7.14.0-3" + "key": "amdrocm-llvm10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sysdeps7.14_7.14.0-3_amd64", - "name": "amdrocm-sysdeps7.14", - "version": "7.14.0-3" + "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0~20260903-33697653391" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -1906,9 +1906,9 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "amdrocm-base7.14_7.14.0-3_amd64", - "name": "amdrocm-base7.14", - "version": "7.14.0-3" + "key": "amdrocm-base10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0~20260903-33697653391" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -2001,445 +2001,490 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "amdrocm-profiler-base7.14_7.14.0-3_amd64", - "name": "amdrocm-profiler-base7.14", - "version": "7.14.0-3" + "key": "amdrocm-profiler-base10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-profiler-base10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver-host7.14_7.14.0-3_amd64", - "name": "amdrocm-solver-host7.14", - "version": "7.14.0-3" + "key": "amdrocm-solver-host10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver-host10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-math-common7.14_7.14.0-3_amd64", - "name": "amdrocm-math-common7.14", - "version": "7.14.0-3" + "key": "amdrocm-math-common10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-math-common10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas-host7.14_7.14.0-3_amd64", - "name": "amdrocm-blas-host7.14", - "version": "7.14.0-3" + "key": "amdrocm-blas-host10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas-host10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx942_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx942", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx942", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx942_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx942", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx942", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx942_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx942", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx942", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx90a_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx90a", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx90c_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx90c", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx90a_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx90a", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx90c_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx90c", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx90a_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx90a", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx90c_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx90c", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx908_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx908", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx90a", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx908_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx908", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx90a", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx908_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx908", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx90a", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1250_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1250", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx908", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1250_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1250", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx908", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1250_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1250", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx908", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1201_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1201", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx906_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx906", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1201_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1201", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx906_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx906", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1201_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1201", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx906_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx906", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1200_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1200", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx900_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx900", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1200_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1200", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx900_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx900", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1200_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1200", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx900_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx900", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1153_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1153", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1250", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1153_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1153", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1250", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1153_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1153", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1250", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1152_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1152", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1201", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1152_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1152", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1201", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1152_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1152", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1201", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1151_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1151", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1200", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1151_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1151", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1200", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1151_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1151", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1200", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1150_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1150", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1153", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1150_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1150", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1153", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1150_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1150", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1153", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1103_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1103", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1152", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1103_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1103", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1152", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1103_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1103", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1152", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1102_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1102", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1151", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1102_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1102", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1151", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1102_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1102", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1151", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1101_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1101", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1150", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1101_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1101", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1150", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1101_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1101", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1150", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1100_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1100", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1103", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1100_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1100", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1103", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1100_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1100", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1103", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1036_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1036", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1102", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1036_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1036", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1102", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1036_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1036", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1102", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1035_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1035", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1101", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1035_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1035", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1101", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1035_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1035", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1101", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1034_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1034", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1100", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1034_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1034", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1100", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1034_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1034", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1100", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1033_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1033", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1036", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1033_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1033", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1036", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1033_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1033", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1036", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1032_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1032", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1035", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1032_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1032", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1035", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1032_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1032", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1035", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1031_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1031", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1034", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1031_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1031", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1034", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1031_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1031", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1034", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1030_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1030", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1033", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1030_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1030", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1033", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1030_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1030", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1033", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1012_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1012", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1032", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1012_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1012", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1032", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1012_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1012", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1032", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1011_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1011", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1031", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1011_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1011", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1031", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1011_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1011", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1031", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1010_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1010", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1030", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1010_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1010", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1030", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1010_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1010", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1030", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1012", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1012", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1012", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1011", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1011", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1011", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1010", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1010", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1010", + "version": "10.1.0~20260903-33697653391" } ], - "key": "amdrocm-blas7.14_7.14.0-3_amd64", - "name": "amdrocm-blas7.14", - "sha256": "438d5d0d33dcc638b2daefd3e3358b9c5a523dd61188ea23631651dcbb277855", + "key": "amdrocm-blas10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1", + "sha256": "70bf09bab42dc86acc6489580311c267751d848122c13f4af7902b84677e86cb", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas7.14-gfx950_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx950", - "sha256": "fb78b8ae9d4242466168c34c80a4a5aa3d909117c5f13f74dafd24d8f31906ed", + "key": "amdrocm-blas10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx950", + "sha256": "32712f653543a86f71b0631d587ad1818c33b3b549fe5a37ad0cdc2fa5d0ba28", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx950_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx950_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver7.14-gfx950_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx950", - "sha256": "fd7a7ed315f374d130fb2c020dbd0d1894b148b8f83baa8e9f9d15b6dcf81eac", + "key": "amdrocm-solver10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx950", + "sha256": "4f6f3f49115cebeb52c7fd9694f912794fc94a22f632584251107723dc974efd", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx950_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx950_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse7.14-gfx950_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx950", - "sha256": "1d317132ee84eb02e0d30bfad61d90238cae74c7d2bd420606864c70c4644d0d", + "key": "amdrocm-sparse10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx950", + "sha256": "855eb5f6bf63e15d3e03d23ec09eb86deac4df68bb4ec3eb43fc7e8fba6fa633", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx950_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx950_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse-host7.14_7.14.0-3_amd64", - "name": "amdrocm-sparse-host7.14", - "sha256": "44fcdf4dde5b78c5dcc36fe59f6dcf37cb2b6cbcab76a257c8a55e9ac4a246b3", + "key": "amdrocm-sparse-host10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse-host10.1", + "sha256": "3b5cf4021a4c7e3d01bf626cda47c27044aa9f88ce714868dd411cf98a168d6b", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse-host7.14_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse-host10.1_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [ { - "key": "amdrocm-sysdeps7.14_7.14.0-3_amd64", - "name": "amdrocm-sysdeps7.14", - "version": "7.14.0-3" + "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0~20260903-33697653391" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -2462,14 +2507,14 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "amdrocm-runtime7.14_7.14.0-3_amd64", - "name": "amdrocm-runtime7.14", - "version": "7.14.0-3" + "key": "amdrocm-runtime10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-llvm7.14_7.14.0-3_amd64", - "name": "amdrocm-llvm7.14", - "version": "7.14.0-3" + "key": "amdrocm-llvm10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0~20260903-33697653391" }, { "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", @@ -2672,9 +2717,9 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "amdrocm-base7.14_7.14.0-3_amd64", - "name": "amdrocm-base7.14", - "version": "7.14.0-3" + "key": "amdrocm-base10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0~20260903-33697653391" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -2767,3172 +2812,970 @@ "version": "3.10.6-1~22.04.1" } ], - "key": "amdrocm-profiler-base7.14_7.14.0-3_amd64", - "name": "amdrocm-profiler-base7.14", - "sha256": "ae0657637788e4e22f8a8a56b7465751650465efda0f56eb7e1498f24950ac48", + "key": "amdrocm-profiler-base10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-profiler-base10.1", + "sha256": "a11a01214b24e38623907268b9f1d8b7fd1100bed59b4e147d6226ca3a7a3874", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-profiler-base7.14_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-profiler-base10.1_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver-host7.14_7.14.0-3_amd64", - "name": "amdrocm-solver-host7.14", - "sha256": "a90bdb7806459ec274446dac2dafe5355290211068f4f719348ab578e55589a7", + "key": "amdrocm-solver-host10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver-host10.1", + "sha256": "9fcc4c65c59801311f60ea8d111bcfbfaea75fff56c1988d68987aea9c0da4c3", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver-host7.14_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver-host10.1_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-math-common7.14_7.14.0-3_amd64", - "name": "amdrocm-math-common7.14", - "sha256": "fc6c0a3388a0e662c26924668404ac65f4293cc2e1274b69da5aa73837685817", + "key": "amdrocm-math-common10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-math-common10.1", + "sha256": "3a7656ad8f819d060a508c4a95f662cb344b865d4b95c5e99ac2e630b716d824", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-math-common7.14_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-math-common10.1_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas-host7.14_7.14.0-3_amd64", - "name": "amdrocm-blas-host7.14", - "sha256": "6ae820948d8aa9cbcdfcbadcd1bb0b8e2601b08e329463ef88259b354ae77118", + "key": "amdrocm-blas-host10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas-host10.1", + "sha256": "f0c3f8b9a213e3a51871b9304bb021e5bd1ee9030ccf3a002d3a7e95f9672ee0", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas-host7.14_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas-host10.1_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas7.14-gfx942_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx942", - "sha256": "43e62de97c42bc40e78e5a833ca181662233f68af77df3412afbb348a7f07841", + "key": "amdrocm-blas10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx942", + "sha256": "a031f800c28e9681f26d92a044b14f62232c250b89b20dda46a4cf8c6a656400", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx942_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx942_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver7.14-gfx942_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx942", - "sha256": "a186901689b546b9c79189431baee991602528dded1fdea268d70a33b356e8c0", + "key": "amdrocm-solver10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx942", + "sha256": "07d39e2cc097f041eeb26a9161ca51bacd40f1462b2436a6d226dd885c7b00a1", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx942_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx942_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse7.14-gfx942_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx942", - "sha256": "64375a7d1be88879e697fd079f69f2d5b57f9c034d61934ef1a5d8f3948e299a", + "key": "amdrocm-sparse10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx942", + "sha256": "629a30e05403cd6e7a016f519e3bff9cf56eee79ab3a8360a0fa89948f71053d", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx942_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx942_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas7.14-gfx90a_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx90a", - "sha256": "ed695f335ae112920846d3d8d6ccf9eebd0546dff13bbb30bddf7bcbe7acc079", + "key": "amdrocm-blas10.1-gfx90c_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx90c", + "sha256": "5ba71bbc88659437e83cc00b2d47e5e7bfcdd2d805c7cbfeda7a50ff9b06fd7c", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx90a_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx90c_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver7.14-gfx90a_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx90a", - "sha256": "eedd257207dc3a57ce1121577e287dc81132a71b46f1cfa23a8ab83c82d7cf7a", + "key": "amdrocm-solver10.1-gfx90c_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx90c", + "sha256": "5ff7061523f7dff14e4bb8f77ef2a3b8ee03f5053907cc836784b6cabfb48fa4", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx90a_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx90c_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse7.14-gfx90a_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx90a", - "sha256": "cde6903dd58348623343433a6696556cfce427ded9bcf9f94259b29c1f5bae9a", + "key": "amdrocm-sparse10.1-gfx90c_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx90c", + "sha256": "bb714e03c8b27c820a33d0d60ec970ac038316e8dbf042cb87da96acfd6b9820", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx90a_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx90c_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas7.14-gfx908_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx908", - "sha256": "721b7c73c2e8041ae07059c7efad152db6d4be7e1692c78ffae8321ed0f84bdc", + "key": "amdrocm-blas10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx90a", + "sha256": "242c5128627accd2626e2fb4cd9a0ab893fa08e167062decb6166baee98894c5", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx908_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx90a_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver7.14-gfx908_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx908", - "sha256": "30414b9c9e5aef3d59b66d168c7a94c2ab284f52a9250f7df08f5651abf392e5", + "key": "amdrocm-solver10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx90a", + "sha256": "5fb7638b8bcee5b18e697e8346470dfd44a91f8d98abcf7c67f478b75fd802d3", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx908_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx90a_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse7.14-gfx908_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx908", - "sha256": "58611375a235e7e720c759db37d32dafa20aa1519ce711e6e39a790a9e5ab90c", + "key": "amdrocm-sparse10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx90a", + "sha256": "ada1cdc38b161e37925e61d3b3cdd5f5b3a15cb3222a151d459769e8a75b4a77", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx908_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx90a_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas7.14-gfx1250_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1250", - "sha256": "931ab849205b4739d4e2494e77af9ca8e9a4e5b8a24a7bffd4e7ff69b2ae8fed", + "key": "amdrocm-blas10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx908", + "sha256": "db26b9bd2daaa5ffb6a21bc555275ed2ec92c1bd5c1167db00d43a225924b9ad", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1250_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx908_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver7.14-gfx1250_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1250", - "sha256": "d87abf95815e7f6aa183cca6ad9fba9fb9a67b7d93e7e2a8eab1177021a809f9", + "key": "amdrocm-solver10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx908", + "sha256": "63cb1dc258a0fee7fb07604ca67f91c62e61c819700a0e4fdc29c93d89992f62", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1250_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx908_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse7.14-gfx1250_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1250", - "sha256": "0de2d194fed57eddf6bd70ce418f58bf88a5b66b1cea9e7af96c3fe7bec9d1e2", + "key": "amdrocm-sparse10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx908", + "sha256": "c95acca10f7081a3073a25be93be28b21d5192354dec725481421aa49b7ffee8", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1250_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx908_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas7.14-gfx1201_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1201", - "sha256": "df33bae5bf26e5dff8828355a25a25bfdcb076296aeb491ddc1deb9edb151227", + "key": "amdrocm-blas10.1-gfx906_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx906", + "sha256": "a26b3fbe79056c73d2bade44e29ec6f49c66bfde869f5b8118ccc118cd4e33c3", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1201_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx906_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver7.14-gfx1201_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1201", - "sha256": "8f9f8bc4689edb0227152e2bb99654f209705fb99633f49e69d1bf38ff44009d", + "key": "amdrocm-solver10.1-gfx906_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx906", + "sha256": "412dcf592e94a8cbbe836243fbc110361829339d33156b7d4e11b2f8b2e97103", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1201_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx906_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse7.14-gfx1201_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1201", - "sha256": "81f2492452b4bb1bf7f82632ed386df27a90d6feafbeb65640694577aca27c8e", + "key": "amdrocm-sparse10.1-gfx906_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx906", + "sha256": "927ee1317af52aae8db2fd0ce97eacda1de14d2c1e290a09797fb32b2c5737ca", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1201_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx906_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas7.14-gfx1200_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1200", - "sha256": "87358ca164a6519483c55d478fe3ac67ceeed35b4881d7ec4c52b646b1dff715", + "key": "amdrocm-blas10.1-gfx900_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx900", + "sha256": "6f53bdceed424a60184c1ea1569dd2be57499c102f33f70dbafc086165eae955", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1200_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx900_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver7.14-gfx1200_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1200", - "sha256": "291c3bb0c3ca7a61820ee626626eecae9bad2b42a54aeedab2c3a5ccba359090", + "key": "amdrocm-solver10.1-gfx900_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx900", + "sha256": "ec941ec97045c84f01c2bf8f547527f1ed6ee80e9b18af6bec9df70b7f6f8889", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1200_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx900_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse7.14-gfx1200_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1200", - "sha256": "b0324406210d54b925ab5c0df383e4ee80776a792019f4fa9ae0fa16cecc91c3", + "key": "amdrocm-sparse10.1-gfx900_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx900", + "sha256": "0a14ecd08b7067782644c2c20f6c24c18e56513db79a7938900aad676fe3e5dc", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1200_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx900_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas7.14-gfx1153_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1153", - "sha256": "153c4ac2cead0fa5f91c26a6a0471a7572082f38452e3b36d8baf49541579dad", + "key": "amdrocm-blas10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1250", + "sha256": "5e1fa4edf09fdb53468da5d42c94b8253ce28d74cc162890cdc8b8bffe3a0008", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1153_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1250_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver7.14-gfx1153_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1153", - "sha256": "9ee83c070b1eec3dcf2e2ecf5ab31af311f2e6fc22e82f7c0e0811ce90e732f2", + "key": "amdrocm-solver10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1250", + "sha256": "e0c7baa13a70c0e72ad452a7f74ef3f2c3fcd9d0bbd1db1c0f4d42893bb24932", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1153_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1250_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse7.14-gfx1153_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1153", - "sha256": "2595f52211b7ab173be376d14fa0867e9f88baf00ffa0a882c52b42362786148", + "key": "amdrocm-sparse10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1250", + "sha256": "c9a70085ee25b49b0e46a6dd0fa06ec5278c7846b082c19a7cdd139e99fe8686", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1153_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1250_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas7.14-gfx1152_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1152", - "sha256": "b168022b0d9bb346999076aab759db346454bc6e1544e384365f87bdcd46f4d2", + "key": "amdrocm-blas10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1201", + "sha256": "10d913f2ec18d6e16e77c0f6435abdebb8393bc66747ee70ebd7448c6e19b02c", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1152_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1201_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver7.14-gfx1152_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1152", - "sha256": "f83f9da40f6c5aa6a4a8be037fa14c315aadeb57627020cc655bea952c100359", + "key": "amdrocm-solver10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1201", + "sha256": "ea16125989307dbce3aff9b3dded900751616f720a61d445bc633b6f9a9d6315", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1152_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1201_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse7.14-gfx1152_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1152", - "sha256": "5d2d78888d289f0fdc11b70c3d8e5f39489410d0147a56deecc371b272c20584", + "key": "amdrocm-sparse10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1201", + "sha256": "017d35a09886dc97c288d5d8abaa59d24c1c879bf820ddd104e27fe54b90a4ba", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1152_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1201_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas7.14-gfx1151_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1151", - "sha256": "0e3ef5efb688114b036d09ceea520c7d63e6b4e76d3e06551c3894bf3236b26a", + "key": "amdrocm-blas10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1200", + "sha256": "56347ab3dcf22044e6420b998afa866857d5c76cf4223f659100cd4cd725ad6b", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1151_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1200_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver7.14-gfx1151_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1151", - "sha256": "736ab9ee57018110fd6299eeaf092edcc67d9fbf38ea671f1843241174bdf3db", + "key": "amdrocm-solver10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1200", + "sha256": "df27d26d46a78796db5b97243a6073205b94155b81fa41a0d46543c078530539", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1151_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1200_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse7.14-gfx1151_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1151", - "sha256": "735c32e3b322ed32476dd343e4c46bffa7d9ac48c98a53acb538c9f251e6ec75", + "key": "amdrocm-sparse10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1200", + "sha256": "4ff73a8d4186d7b877d8f4647327905817bcb72c4ce55e2c260f4a0841f40614", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1151_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1200_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas7.14-gfx1150_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1150", - "sha256": "1c820ee86dfd2aaea8eecdf2dbdc1a9e162f2acd3e8102c4a49459a315ac76de", + "key": "amdrocm-blas10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1153", + "sha256": "40b06296573a2c23b74ab50247b2466fa3ed51fc7e311686c7451517219e527b", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1150_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1153_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver7.14-gfx1150_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1150", - "sha256": "32930f5dfa59fccbbfa23782d9778e19a7fb3ead1cd2a715d8531a0236502ccf", + "key": "amdrocm-solver10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1153", + "sha256": "bb5f4863fda70d26179e17dd0922493ae2aa45096ecb6d1696897dba5c729b83", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1150_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1153_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse7.14-gfx1150_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1150", - "sha256": "88c3952d7e798691d455701b4c06895e7b8a99cfff62b6a053bfeeefd47f9505", + "key": "amdrocm-sparse10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1153", + "sha256": "cdd2a35b80a85fd95c0b7bd627a6e1cb3dac5d41f277ce89d00613c4cf51428c", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1150_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1153_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas7.14-gfx1103_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1103", - "sha256": "3ea904b6c1aaece72759c30497342e75edf884af93f6fa5b8d1f6010c47009aa", + "key": "amdrocm-blas10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1152", + "sha256": "2cb29cd88839762ed0b7847f1a9a79a6cae0e0ac49b796c0bb4432b427e7b8cc", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1103_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1152_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver7.14-gfx1103_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1103", - "sha256": "7ac5d2d630efb88fe3e3550f07fd564e78d9f6406481eb21e01ddaeaa04ff9d2", + "key": "amdrocm-solver10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1152", + "sha256": "07b9b06b28fd11ae4926efa29597eb444141f365f070dac1154bce12808687d3", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1103_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1152_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse7.14-gfx1103_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1103", - "sha256": "e34e89ab81b6711c7c766d2c8dcc5575a33f6a641cfaafd77f6391f07a6fe5e3", + "key": "amdrocm-sparse10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1152", + "sha256": "ea3bdb9d7a636fa55b003b21faefb3d22629ff63bcb31cd40d13432ae6bdbd04", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1103_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1152_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas7.14-gfx1102_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1102", - "sha256": "b670c3bcff65f8a79da828685da147d2131451351ba6ffadb604f15e456208bd", + "key": "amdrocm-blas10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1151", + "sha256": "857154648bbdac75f539ebe79706a72689988036fae694df042ebe7ea02957a6", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1102_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1151_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver7.14-gfx1102_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1102", - "sha256": "add2034c3d41bf6767b7bffa307ef6d204d2807f400dec343f02f3a61d2628e2", + "key": "amdrocm-solver10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1151", + "sha256": "f957615d37f11493426e278bf19727e45149a5b2551d9d49ff70b8a9a0458811", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1102_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1151_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse7.14-gfx1102_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1102", - "sha256": "00b364d767b504194a070126b02fe6b614018043366cf9a1e199038cf4074f57", + "key": "amdrocm-sparse10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1151", + "sha256": "9b741dfc30bc8a5f0130cfdcbb337c41e2d651d5e077c12b2177158a6f1f01a3", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1102_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1151_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas7.14-gfx1101_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1101", - "sha256": "fdfb768ffa00776bbc96f077ef387ef6a757ebb9b35b9e3454bee410ebd3617b", + "key": "amdrocm-blas10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1150", + "sha256": "ec9eb850c647a69b82cf80ec8f8ea063bf06c6ddb0d2bf9d680803ddd67af1c5", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1101_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1150_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver7.14-gfx1101_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1101", - "sha256": "169fa743505585b7c96e025da8878d956e4268657aec302bfb07e95f891950be", + "key": "amdrocm-solver10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1150", + "sha256": "4bb76ca1d2954da0808635c7172def023b6f80672dbfa299a4269c2b17a41371", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1101_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1150_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse7.14-gfx1101_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1101", - "sha256": "aeecc9ce33b28948ffbc33362784a9bfb9fe74275939191d378df1fccc1dc080", + "key": "amdrocm-sparse10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1150", + "sha256": "abc915dda44d75f699c5f1122049334c73be02da751f3752fff2e7aa5fd68a93", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1101_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1150_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas7.14-gfx1100_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1100", - "sha256": "0203356336f2d90d4e60070b12d71abafa0b24432c43ebe6dbbed3ed4af3f283", + "key": "amdrocm-blas10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1103", + "sha256": "621a18815c4294ef802b0622defa030e5a1bbc23f7195f61abd1c7747f4450c9", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1100_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1103_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver7.14-gfx1100_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1100", - "sha256": "e0a57a6519a7c0e0be684d07409c11127ef1408aeedcc5e0efc8fdd5996e66aa", + "key": "amdrocm-solver10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1103", + "sha256": "c6ff77ba5593df096f3efd524a7e8c8bd2e588274b7e403c9b179e87a76c4709", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1100_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1103_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse7.14-gfx1100_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1100", - "sha256": "692f7ef41e2b6eae1b8bd2fa77435f0783967a0ee34353718ea0179059a22f5c", + "key": "amdrocm-sparse10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1103", + "sha256": "0001e6ac717e66b3dc7efe41246ae4c643f9063a7889bfe39deda66fd6b2bc53", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1100_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1103_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas7.14-gfx1036_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1036", - "sha256": "cb5b86af54a4d10a08ee9775c329e087ef4d98b041706b80363a8713e2348e42", + "key": "amdrocm-blas10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1102", + "sha256": "dbcd542dfaaf7ea141216c54bb980771a158cb09d5f2af83729b062db3233b69", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1036_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1102_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver7.14-gfx1036_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1036", - "sha256": "e702804b6edcde357548e9e72b7dd6625d945a377379193a1db8c072258f837b", + "key": "amdrocm-solver10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1102", + "sha256": "22f9745ca43d9ba3f78585a272b701cd62a813824a7ac711e25ed3814f0f2d96", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1036_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1102_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse7.14-gfx1036_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1036", - "sha256": "eb343fb36c7d1fd2dd46f2388bf8a116339cbca1e6305e3fa68ec6db06614a88", + "key": "amdrocm-sparse10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1102", + "sha256": "71c25b880004010ce89b1d6db2899658360ccb76550320d666d96afcc37639b4", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1036_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1102_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas7.14-gfx1035_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1035", - "sha256": "d1da329e9d6bdf46f095de4d728f0206b01f82767536e6aaa57000e4e5fc3c17", + "key": "amdrocm-blas10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1101", + "sha256": "107a517470ed0667ac6771ce2c6534d0d4dc22bf5669fc59d5a8c434b28ba3b4", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1035_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1101_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver7.14-gfx1035_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1035", - "sha256": "afcba030a7b8fd22dd44e89dffd816a18247f976e9152bcd2151ff814346180a", + "key": "amdrocm-solver10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1101", + "sha256": "115a5371068a7e6f2f418cc2fa556686c3ed7e41a40933798d165e51f34cc30f", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1035_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1101_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse7.14-gfx1035_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1035", - "sha256": "34d0581c35b5cb042ce3bd039ea70d9b4ca7b3ddb02efb895423833ac4d34d26", + "key": "amdrocm-sparse10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1101", + "sha256": "5beb485c1e423bff82e4f8eacb2454aef31c7db5afee58c09eb1267068c0245f", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1035_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1101_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas7.14-gfx1034_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1034", - "sha256": "ee69a306ee6b0e4420fb0d153d20a9bc2c03b1962d6ec7f331320e32ea704fd0", + "key": "amdrocm-blas10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1100", + "sha256": "e0b6684ebeafaac46b0c2b93333b71b4b665b22dc992b2c4ed999eda43971b3e", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1034_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1100_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver7.14-gfx1034_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1034", - "sha256": "aef66e78e7ea247f6f5849d9cc7f395f9db42ef5c8e33b8dab31c1490203d3a9", + "key": "amdrocm-solver10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1100", + "sha256": "bc92ee0916d6355257559e81669e8e406f89b4402f292b8b78284c8295049117", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1034_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1100_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse7.14-gfx1034_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1034", - "sha256": "31f93def4df1d42ba2c8f650701929db005c83fc3c237b72ee99b6f58ff4cd5e", + "key": "amdrocm-sparse10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1100", + "sha256": "91ccf0dd8400e0a795ea09b62412ec531cfcb426f08251348ffb0aedfa05107d", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1034_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1100_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas7.14-gfx1033_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1033", - "sha256": "57510da3b868d360a3c931d60e263bdaf9347626fa8f6814f0b123b2f0358208", + "key": "amdrocm-blas10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1036", + "sha256": "2c34a119a2de5884511933718aa38e410af24c9f7a6bacac06c1dd18a4f0abc1", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1033_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1036_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver7.14-gfx1033_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1033", - "sha256": "66db48a235042d92146745eec1f14370a6d31844c21a8d43ccd607ffcbc2d02d", + "key": "amdrocm-solver10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1036", + "sha256": "d924ee3f4b70cfce1dd44e266ade058a5bd862508824860b202fa03909bb1b4c", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1033_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1036_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse7.14-gfx1033_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1033", - "sha256": "893bcd9b43d56835b9a3a04ea10b0d4fdfb0a2dd7f8c7be74030a9e281a5bb42", + "key": "amdrocm-sparse10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1036", + "sha256": "f3bd7e470e081cc2b7226e98a901c1855c17084e348ce8a9a46a8985a928f680", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1033_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1036_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas7.14-gfx1032_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1032", - "sha256": "0775f39717812228d56cad734a931d81d0222dc086f2dda0cc3055864f62c057", + "key": "amdrocm-blas10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1035", + "sha256": "731831b0f2f3e134548525c2f5a6195fb5511fa7b0b4cfc5b8121a9c47dd5680", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1032_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1035_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver7.14-gfx1032_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1032", - "sha256": "c7016ccff068f1defaef11f43390c87ce9ad7b5a44daf70ca6fc6c64f7fabf05", + "key": "amdrocm-solver10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1035", + "sha256": "789256f456ed6b3888f37f354b7277f75f9463f700363b50c2aad2842662dadc", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1032_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1035_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse7.14-gfx1032_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1032", - "sha256": "19f4afaaee5ab6edbf3d7f5307a2c075a397dbc81bcaca9813cdcea038138821", + "key": "amdrocm-sparse10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1035", + "sha256": "606e905784e54fecc1449a38e5256bf938b80a4bf305f1ce9207dfefdbcbcf52", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1032_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1035_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas7.14-gfx1031_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1031", - "sha256": "b2a9cb9b3cb84b1c856ee44c4d9951c077b6d6a5338bd058c64bba5cd3e7e18b", + "key": "amdrocm-blas10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1034", + "sha256": "f1e9629f774876e4d58544e70c68ff4e8a865398ad6ec9a79e5f5f29fd9c08b0", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1031_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1034_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver7.14-gfx1031_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1031", - "sha256": "ad636af6a4a3cb422cd2983c0ded18f926a51952dd6fabe7c8aedd6205001d20", + "key": "amdrocm-solver10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1034", + "sha256": "e3c0b83128f0d9cfb14d6490ee11c7cd7541a4e047884b0f133e6733d4475c4c", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1031_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1034_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse7.14-gfx1031_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1031", - "sha256": "d8cd61af88b60309d76b584638e6c3b4974c67709db095bd6dc165e5975cf449", + "key": "amdrocm-sparse10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1034", + "sha256": "136009871feac8958f5d06717fd208ed37d15b003e981c58cc4a34874cd253bf", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1031_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1034_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas7.14-gfx1030_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1030", - "sha256": "8aaa8907924c4396fb4e602911691ca69867b99d7ca9e8a39128d7a6027a7392", + "key": "amdrocm-blas10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1033", + "sha256": "ef577d055b5196c03a94dc3ecb7e0dc2620d8c33f622ad3d24a74cf33f685b76", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1030_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1033_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver7.14-gfx1030_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1030", - "sha256": "09452341928fca391c3be8bfca14bafe9b84a2277a22a7a3abf315124ceb387e", + "key": "amdrocm-solver10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1033", + "sha256": "bc2faaef8650a97d0003a6122ed701f4b7ca0ace79815d29fe0588bcfbb23e76", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1030_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1033_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse7.14-gfx1030_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1030", - "sha256": "2d7e48205f5a19290d8d7908cda57123a35e72f452ad6aaa1330a34cf477ebbc", + "key": "amdrocm-sparse10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1033", + "sha256": "8058ebcfa77d219dccbdb9758ddd4203a39775afbbbe02ad634e67b143587be3", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1030_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1033_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas7.14-gfx1012_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1012", - "sha256": "f04222a1088e0080d0a8e4758982e7bf8bb5c0042acf1493e3d4c4489b719594", + "key": "amdrocm-blas10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1032", + "sha256": "8740c616bf9a536d80fb0873e7dc6014040b2c5cdf980e6c45ed192fcada168c", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1012_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1032_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver7.14-gfx1012_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1012", - "sha256": "d322dce522cd973bf0b9cd19166d3c4c67d01f9cf33fca571a8303cdbcbc2ae9", + "key": "amdrocm-solver10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1032", + "sha256": "d091bea4179f5581dccf63b537bbf184cd886853773043ed6b6804dd18800bf2", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1012_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1032_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse7.14-gfx1012_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1012", - "sha256": "1343cbeed7851ee5f1e38723f31bed8af9eb6543d1a80ddf8fd3b87846f9d26f", + "key": "amdrocm-sparse10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1032", + "sha256": "d9ed5b581f3c63192926fc03893f3f7fb5b75b887fb1d68cd502d3658e8c3a96", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1012_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1032_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas7.14-gfx1011_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1011", - "sha256": "2079a6378938c2e07f11e20135849773b2679ddfc3da29eed35dab2dffbbf99a", + "key": "amdrocm-blas10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1031", + "sha256": "998cb3eb22e39c6a4a0934216750e8a5c575e7b34c4f246cebbaea3cc80ade86", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1011_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1031_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver7.14-gfx1011_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1011", - "sha256": "88c4190d84c013a822f937e2322208c4e66a999b35ff82613351114018d12ed8", + "key": "amdrocm-solver10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1031", + "sha256": "2f6d6e1a9b1192a63c9a468699324467de491ae5d9c97efdd3020842b6432b8f", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1011_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1031_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse7.14-gfx1011_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1011", - "sha256": "a62ee0afb6d26bf672039805b2439b8765e1fddfeaceaa8f03661a1a01bfeeea", + "key": "amdrocm-sparse10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1031", + "sha256": "230c6600dd9587138cdcabf3274e04452a1bb5875c126b620cdf79b2f703a5f4", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1011_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1031_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas7.14-gfx1010_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1010", - "sha256": "a1a79437e677a5905f56f4aa4492b228c23e8ffbd95016191b6054e9c3a14372", + "key": "amdrocm-blas10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1030", + "sha256": "7ef186f51905b8d38c52b815a665f5f76a99d036af646c1db48c3ee150d2d238", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas7.14-gfx1010_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1030_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver7.14-gfx1010_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1010", - "sha256": "cc05c8b169ed7b38d49121f9aa9be9a5a4373e967eed26e24162fcd208fa44f4", + "key": "amdrocm-solver10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1030", + "sha256": "62f654b869ba1208db04eb67f4b94904ef330eac8534eb7c9a24c05f2a01a882", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14-gfx1010_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1030_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse7.14-gfx1010_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1010", - "sha256": "9ab745331d7344f0bdd43a906a43e0e5df74d5b7a1af910bfbe03ece7e6e40bf", + "key": "amdrocm-sparse10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1030", + "sha256": "8863c44c88ec1e6aa1d09b0e221ee57ea60fb66b8280535beb14d0c6d12bd7a8", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14-gfx1010_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1030_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1012", + "sha256": "c5e4e2928e67a7d501af132396470afde4c4c279358f2b7a213875a2475d0e87", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1012_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1012", + "sha256": "66d350644d469e3b6f15f481823d91cf5d9dc3de16cff231b885a3a6723ed208", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1012_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1012", + "sha256": "3e30d0262fa41c16e22e86a9ee76908f6b9fb51b04694d553757a3d191c6ac1e", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1012_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1011", + "sha256": "d066c2bcb5ff469ed53fed0e24512cfafb281026b099a31d385ba3891e05f943", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1011_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1011", + "sha256": "c99f5d01203a5a19753790bc8b1dfd6cf6285504855dfbf66de922047d47b9b8", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1011_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1011", + "sha256": "07dcb85b5fdbb062e2b1427b24568b744d0db90ea2c36437b90c2311c4d49db7", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1011_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1010", + "sha256": "645de7ff587ad83f7374f55d9ece589d5d651aaa8d6b9ee0b7ba7525e4e50fd7", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1010_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1010", + "sha256": "c4c344044a99c336ce2a6c72e057ca676576a0587c2124de0511479cbf4c5392", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1010_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1010", + "sha256": "1fd269cdda8880606b595c502756db2f0e7d53be239deacfca23e3a8d52956b2", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1010_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [ { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" + "key": "amdrocm-hipblas-common-dev10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-hipblas-common-dev10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" + "key": "amdrocm-runtime-dev10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-runtime-dev10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" + "key": "amdrocm-llvm-dev10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-llvm-dev10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - } - ], - "key": "amdrocm-blas-dev7.14_7.14.0-3_amd64", - "name": "amdrocm-blas-dev7.14", - "sha256": "37c4dae51923ad0110937c183ce0bbf463b84191c6b86f6a8cf668eeea82d0f0", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-blas-dev7.14_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "amdrocm-dnn7.14-gfx950_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx950", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx950_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx950", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx950_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx950", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx950_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx950", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse-host7.14_7.14.0-3_amd64", - "name": "amdrocm-sparse-host7.14", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-runtime7.14_7.14.0-3_amd64", - "name": "amdrocm-runtime7.14", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-llvm7.14_7.14.0-3_amd64", - "name": "amdrocm-llvm7.14", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sysdeps7.14_7.14.0-3_amd64", - "name": "amdrocm-sysdeps7.14", - "version": "7.14.0-3" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libgcc-11-dev", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", - "name": "libquadmath0", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libubsan1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", - "name": "libtsan0", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", - "name": "gcc-11-base", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", - "name": "liblsan0", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", - "name": "libasan6", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libatomic1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libitm1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgomp1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libstdc++-11-dev", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "amdrocm-base7.14_7.14.0-3_amd64", - "name": "amdrocm-base7.14", - "version": "7.14.0-3" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "amdrocm-profiler-base7.14_7.14.0-3_amd64", - "name": "amdrocm-profiler-base7.14", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver-host7.14_7.14.0-3_amd64", - "name": "amdrocm-solver-host7.14", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-math-common7.14_7.14.0-3_amd64", - "name": "amdrocm-math-common7.14", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas-host7.14_7.14.0-3_amd64", - "name": "amdrocm-blas-host7.14", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-dnn-host7.14_7.14.0-3_amd64", - "name": "amdrocm-dnn-host7.14", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-dnn7.14-gfx942_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx942", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx942_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx942", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx942_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx942", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx942_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx942", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-dnn7.14-gfx90a_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx90a", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx90a_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx90a", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx90a_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx90a", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx90a_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx90a", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-dnn7.14-gfx908_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx908", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx908_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx908", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx908_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx908", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx908_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx908", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-dnn7.14-gfx1201_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx1201", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx1201_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1201", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1201_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1201", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx1201_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1201", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-dnn7.14-gfx1200_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx1200", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx1200_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1200", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1200_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1200", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx1200_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1200", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-dnn7.14-gfx1153_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx1153", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx1153_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1153", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1153_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1153", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx1153_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1153", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-dnn7.14-gfx1152_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx1152", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx1152_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1152", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1152_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1152", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx1152_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1152", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-dnn7.14-gfx1151_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx1151", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx1151_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1151", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1151_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1151", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx1151_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1151", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-dnn7.14-gfx1150_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx1150", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx1150_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1150", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1150_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1150", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx1150_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1150", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-dnn7.14-gfx1102_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx1102", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx1102_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1102", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1102_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1102", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx1102_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1102", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-dnn7.14-gfx1101_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx1101", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx1101_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1101", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1101_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1101", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx1101_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1101", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-dnn7.14-gfx1100_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx1100", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx1100_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1100", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1100_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1100", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx1100_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1100", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-dnn7.14-gfx1030_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx1030", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx1030_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1030", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1030_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1030", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx1030_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1030", - "version": "7.14.0-3" - } - ], - "key": "amdrocm-dnn7.14_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14", - "sha256": "59654345f316a40ad389d984e6a14a22bd00a8f9818343be18cbee7a4bcf663b", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn7.14-gfx950_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx950", - "sha256": "798e334ae094930cb64027cc5a262692095aeb4d94adebf705ebc34ea04d7793", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx950_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn-host7.14_7.14.0-3_amd64", - "name": "amdrocm-dnn-host7.14", - "sha256": "60b5881d4de4965e8c943505bdf3de644b4f654f95d82d4929ce3f81bd543296", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn-host7.14_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn7.14-gfx942_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx942", - "sha256": "6dc1468b464c6bf6c34df884e2a5bdc4a313832e1c80152aa77b5c686a9cda36", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx942_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn7.14-gfx90a_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx90a", - "sha256": "1eb5accc0333896f9d4fb2903ffb7545b61c1c3618de8ee6d4a6676c4139b9c8", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx90a_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn7.14-gfx908_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx908", - "sha256": "5996e133669830de9b1f6bc25b0426c8e844e9b4f181e057396821b76fcf2c51", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx908_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn7.14-gfx1201_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx1201", - "sha256": "bfb6db6d3b202d2267180e7bd9ea408da0fdc7d6ce302b753cb52273222ccd58", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1201_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn7.14-gfx1200_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx1200", - "sha256": "6d7ee796b578339d5446827d7a78acab2bb3c0638b859cde4c10386306afb38d", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1200_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn7.14-gfx1153_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx1153", - "sha256": "a0ac3315e9d8d2198030135110ee69b686cc06d489dd9e012e3d0c7c15da029a", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1153_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn7.14-gfx1152_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx1152", - "sha256": "1ce94ef4c1b6bbede2bc3baea52505522ad3608f4f56756c37c2773cc15d548f", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1152_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn7.14-gfx1151_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx1151", - "sha256": "b09507f24042c4b4c5bb7cc1f23598de9399f40d46c5db9d5e2eba720e980ec6", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1151_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn7.14-gfx1150_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx1150", - "sha256": "389e5a3c4d12b62084fbc2caaed91650c3ad90b15d531ec7a56d7264dc73a008", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1150_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn7.14-gfx1102_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx1102", - "sha256": "3e184a8ba6e6afeaca1ae8c5370bfe7b6fe90099d57746489098067523d16b47", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1102_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn7.14-gfx1101_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx1101", - "sha256": "b2c292835fb4c8250df22fd441bd677d4c195b93e1d1f77c013cbf5a82cd5675", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1101_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn7.14-gfx1100_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx1100", - "sha256": "d8d0d13bfe0a0962081531acdcdbc523edca2dd663600b2764738ef8aac8a42f", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1100_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn7.14-gfx1030_7.14.0-3_amd64", - "name": "amdrocm-dnn7.14-gfx1030", - "sha256": "ccf950df526b22358463fb4bfb8a315bc0df63ab64917bc258bc42a4c40b113f", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-dnn7.14-gfx1030_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "amdrocm-fft7.14-gfx950_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx950", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-fft-host7.14_7.14.0-3_amd64", - "name": "amdrocm-fft-host7.14", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sysdeps7.14_7.14.0-3_amd64", - "name": "amdrocm-sysdeps7.14", - "version": "7.14.0-3" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "amdrocm-runtime7.14_7.14.0-3_amd64", - "name": "amdrocm-runtime7.14", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-llvm7.14_7.14.0-3_amd64", - "name": "amdrocm-llvm7.14", - "version": "7.14.0-3" - }, - { - "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libgcc-11-dev", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", - "name": "libquadmath0", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libubsan1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", - "name": "libtsan0", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", - "name": "gcc-11-base", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", - "name": "liblsan0", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", - "name": "libasan6", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libatomic1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libitm1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgomp1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libstdc++-11-dev", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "amdrocm-base7.14_7.14.0-3_amd64", - "name": "amdrocm-base7.14", - "version": "7.14.0-3" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "amdrocm-fft7.14-gfx942_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx942", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-fft7.14-gfx90a_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx90a", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-fft7.14-gfx908_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx908", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-fft7.14-gfx1250_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1250", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-fft7.14-gfx1201_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1201", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-fft7.14-gfx1200_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1200", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-fft7.14-gfx1153_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1153", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-fft7.14-gfx1152_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1152", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-fft7.14-gfx1151_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1151", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-fft7.14-gfx1150_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1150", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-fft7.14-gfx1103_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1103", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-fft7.14-gfx1102_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1102", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-fft7.14-gfx1101_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1101", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-fft7.14-gfx1100_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1100", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-fft7.14-gfx1036_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1036", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-fft7.14-gfx1035_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1035", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-fft7.14-gfx1034_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1034", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-fft7.14-gfx1033_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1033", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-fft7.14-gfx1032_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1032", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-fft7.14-gfx1031_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1031", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-fft7.14-gfx1030_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1030", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-fft7.14-gfx1012_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1012", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-fft7.14-gfx1011_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1011", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-fft7.14-gfx1010_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1010", - "version": "7.14.0-3" - } - ], - "key": "amdrocm-fft7.14_7.14.0-3_amd64", - "name": "amdrocm-fft7.14", - "sha256": "3e0f943fa7f864b57f0e4654d6263c5e8bcc43bf3c62add2fbadaa20520d7a21", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft7.14-gfx950_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx950", - "sha256": "40682957167935c086ae8f7f30022e778a612fc6cf04e246c3b480109ccf7b1a", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx950_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft-host7.14_7.14.0-3_amd64", - "name": "amdrocm-fft-host7.14", - "sha256": "fb7b39883e51e25fbe9bd46aa4b76b089cd0edfea97be80e8631a85c6c70934d", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft-host7.14_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft7.14-gfx942_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx942", - "sha256": "b141660c1db7ba6882fb35d5ffd4666d5c878a97fe078b80b721813d9249d2d3", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx942_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft7.14-gfx90a_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx90a", - "sha256": "7786d24549a0d698a7505d534b6d020d037e49ff481d937d83fcb1ae4d669c67", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx90a_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft7.14-gfx908_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx908", - "sha256": "bc5999397f7dab4a159d0dfe5272a7db73c7f3647353a79073f8f6cf335473ba", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx908_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft7.14-gfx1250_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1250", - "sha256": "a30b041906961c7ce8b90f2d0e03da94986d06e5374f59671e3323bf2ab38d4b", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1250_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft7.14-gfx1201_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1201", - "sha256": "ab8018b742f52356e84c9662f2d620a39f802e757c79d34498847f52c78be1e2", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1201_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft7.14-gfx1200_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1200", - "sha256": "a43aa5568864e61f669abe0e96a4acfe02c6ada7b0adf7362bc0f288af85d7be", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1200_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft7.14-gfx1153_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1153", - "sha256": "75e65dca76d758b06a8804bb6dd0cda494f5a09bcc336efb1c324ccd99d0e003", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1153_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft7.14-gfx1152_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1152", - "sha256": "dcde969b559140d8eba42459e4a500b372767f4c9951d6986841d2be9c7d23fd", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1152_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft7.14-gfx1151_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1151", - "sha256": "152fea84cdae050455d94380e91b4f890b774f87970460cd66e0badc22262e7c", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1151_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft7.14-gfx1150_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1150", - "sha256": "55721427c464921300572ef3620c237eef35c209aa407b2b953a661a2e8a3c3b", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1150_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft7.14-gfx1103_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1103", - "sha256": "d2355cabc2e1b4ee0c13b1141c9131f3c95e796d3871b063a2cb71b892f701e4", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1103_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft7.14-gfx1102_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1102", - "sha256": "64b3d713c0eff13dc9142acea7f8bda2635cd730ab45edcb05b1a562b52cfc48", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1102_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft7.14-gfx1101_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1101", - "sha256": "df5c62070a5a79ed90bf120dc88f43668620e2ad9d774fbab183f027a65804db", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1101_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft7.14-gfx1100_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1100", - "sha256": "e391f137ad41548f0dc16e13e8605a52f3b5a1e9f2da772bb08c07fb8ed38ae1", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1100_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft7.14-gfx1036_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1036", - "sha256": "0efd59d401afaa6c25df4ba18649053f91a3f9d7f8ca5baf023a063882fdaffb", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1036_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft7.14-gfx1035_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1035", - "sha256": "3ad1e464c73d22ae971ed647337f9854cd13540bcdb2aefdf542551371d9ba01", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1035_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft7.14-gfx1034_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1034", - "sha256": "733de05e2e6b4100f4fe693fa6508ceca84c57e576a177e348f0b27bef7325e3", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1034_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft7.14-gfx1033_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1033", - "sha256": "60a9886a6735887fc0ce51267928d4990e227e2a6c517cce90714f200c91559a", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1033_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft7.14-gfx1032_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1032", - "sha256": "4b2e74a1c35d5f0b3ef83165e6e3ee94030a759980dcec39275ffccf0e79cbc9", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1032_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft7.14-gfx1031_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1031", - "sha256": "5922a31921a36d726e4ef2e51bc78348712c4bd7e4e375031216bbf28ffbf657", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1031_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft7.14-gfx1030_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1030", - "sha256": "cdb09413498d8ccd143f600dd0f4263d9f8c0ac16d4c2a083cf97a4a438ef7f6", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1030_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft7.14-gfx1012_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1012", - "sha256": "ad47313cacbe4f40288dc4ab7ae91d33217c202873e7ebc70c0120e46e619a12", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1012_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft7.14-gfx1011_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1011", - "sha256": "27aba1514f9186f58e0a7ab2da97cb6387c4aaa8ec17b54468eff9681110764e", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1011_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft7.14-gfx1010_7.14.0-3_amd64", - "name": "amdrocm-fft7.14-gfx1010", - "sha256": "e3a018100c31f7dcbcf099a6443814c1a0e2e8377672d2cab72bc3c2d075ab59", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-fft7.14-gfx1010_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "amdrocm-rccl7.14-gfx950_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx950", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-rccl-host7.14_7.14.0-3_amd64", - "name": "amdrocm-rccl-host7.14", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-profiler-base7.14_7.14.0-3_amd64", - "name": "amdrocm-profiler-base7.14", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sysdeps7.14_7.14.0-3_amd64", - "name": "amdrocm-sysdeps7.14", - "version": "7.14.0-3" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "amdrocm-runtime7.14_7.14.0-3_amd64", - "name": "amdrocm-runtime7.14", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-llvm7.14_7.14.0-3_amd64", - "name": "amdrocm-llvm7.14", - "version": "7.14.0-3" - }, - { - "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libgcc-11-dev", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", - "name": "libquadmath0", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libubsan1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", - "name": "libtsan0", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", - "name": "gcc-11-base", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", - "name": "liblsan0", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", - "name": "libasan6", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libatomic1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libitm1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgomp1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libstdc++-11-dev", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "amdrocm-base7.14_7.14.0-3_amd64", - "name": "amdrocm-base7.14", - "version": "7.14.0-3" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "amdrocm-amdsmi7.14_7.14.0-3_amd64", - "name": "amdrocm-amdsmi7.14", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-rccl7.14-gfx942_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx942", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-rccl7.14-gfx90a_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx90a", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-rccl7.14-gfx908_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx908", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-rccl7.14-gfx1250_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1250", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-rccl7.14-gfx1201_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1201", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-rccl7.14-gfx1200_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1200", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-rccl7.14-gfx1151_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1151", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-rccl7.14-gfx1102_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1102", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-rccl7.14-gfx1101_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1101", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-rccl7.14-gfx1100_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1100", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-rccl7.14-gfx1036_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1036", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-rccl7.14-gfx1035_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1035", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-rccl7.14-gfx1034_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1034", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-rccl7.14-gfx1033_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1033", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-rccl7.14-gfx1032_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1032", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-rccl7.14-gfx1031_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1031", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-rccl7.14-gfx1030_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1030", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-rccl7.14-gfx1012_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1012", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-rccl7.14-gfx1011_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1011", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-rccl7.14-gfx1010_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1010", - "version": "7.14.0-3" - } - ], - "key": "amdrocm-rccl7.14_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14", - "sha256": "ab7b72c172252d259bebefb2c325034db4b0e46ecde0191ce50f447c8beeac41", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl7.14-gfx950_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx950", - "sha256": "aa736e0e6b1e32104e80794f3d555828f01c9b28c90c4ca2d4fac0647a8a4565", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx950_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl-host7.14_7.14.0-3_amd64", - "name": "amdrocm-rccl-host7.14", - "sha256": "0d418c1f2a5f15d329e26458a39a395d8236b10cc16397922e1df61e699b90d6", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl-host7.14_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl7.14-gfx942_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx942", - "sha256": "00e33c9f72e853a033641d7d46c0bbf94a7b546eac586fafbf86dab19ce96db4", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx942_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl7.14-gfx90a_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx90a", - "sha256": "54d7d090219946a7631596543582eaa9ad6ffd35a4f56ee86a2ba774628b522e", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx90a_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl7.14-gfx908_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx908", - "sha256": "282008f6ea550e218c93def030e79e890932f9c8d1886ba993071a7896b8ccc9", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx908_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl7.14-gfx1250_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1250", - "sha256": "8013142575c5fad8ed411911e1574cfe165857eded83be2a220478e64dbe8e98", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1250_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl7.14-gfx1201_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1201", - "sha256": "771dc4beedc633de7588849d7c6c2bded9924bb72b5b1ff7e1bdc7fb999beb27", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1201_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl7.14-gfx1200_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1200", - "sha256": "2a4d1d6f12da39ae3bc57d5cdab56e9dc6d28ba335fe532c46bcd6fe9588b1ba", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1200_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl7.14-gfx1151_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1151", - "sha256": "2cdf93160d63d3e329399de716b9261b9c9b6f776b09ffbe7bab4746f7d86ed3", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1151_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl7.14-gfx1102_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1102", - "sha256": "d8782810acb321f94c42a6b0c6334d307917ca2acbbfaeaa0244a563ec6165f5", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1102_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl7.14-gfx1101_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1101", - "sha256": "3c1ebeb61a1e8921c983d9432973ce83d9f919b82e3d8b6dfe8999662c605b6e", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1101_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl7.14-gfx1100_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1100", - "sha256": "ea6da253d619f8e56ea573b980133b3bbd4dfa70e6dd58b339326d00b749da6f", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1100_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl7.14-gfx1036_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1036", - "sha256": "0811c8f1212247711abd50c61b818446926226ea17901ee05ad5edf07ad87916", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1036_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl7.14-gfx1035_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1035", - "sha256": "f4a7924a8032d4cc271541df68ef679e22b6c0ac4643e1d77d7ea742bed09752", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1035_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl7.14-gfx1034_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1034", - "sha256": "8f38d0016c32e18bb254ef97ce597c10a203f471788fa960b946cd10d277397c", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1034_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl7.14-gfx1033_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1033", - "sha256": "d0a8a971fba23dac720b87afc2d6a8c2f4d5df3c92ac6071708bb9a0216d6d86", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1033_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl7.14-gfx1032_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1032", - "sha256": "d5f46671549deb3682ca6f15049d4ec7cd500b19364c6e40d8d455a62927718e", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1032_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl7.14-gfx1031_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1031", - "sha256": "6139cadb284ff1b6d9267fd2780196265c1ff7ee9909d534e7f797fdaf1b01ac", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1031_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl7.14-gfx1030_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1030", - "sha256": "e0257ac0b24a1ea20d78272546cb05fed57342f58d331e128565e658155393a0", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1030_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl7.14-gfx1012_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1012", - "sha256": "060f6cfde9f945271f4dc7bdad87425ff16ef0edd55d172e04950a4a9f112100", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1012_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl7.14-gfx1011_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1011", - "sha256": "bb13e8800079678a4e73871fd34501dbd75c62e4a468b0ef47a21185a19d69bf", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1011_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl7.14-gfx1010_7.14.0-3_amd64", - "name": "amdrocm-rccl7.14-gfx1010", - "sha256": "8d93d6320a5b74e6933cdd523ce2994c5a668fb47c789533fbec3cef3fb6baff", - "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-rccl7.14-gfx1010_7.14.0-3_amd64.deb" - ], - "version": "7.14.0-3" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "amdrocm-llvm-dev7.14_7.14.0-3_amd64", - "name": "amdrocm-llvm-dev7.14", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-llvm7.14_7.14.0-3_amd64", - "name": "amdrocm-llvm7.14", - "version": "7.14.0-3" + "key": "amdrocm-llvm10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-runtime7.14_7.14.0-3_amd64", - "name": "amdrocm-runtime7.14", - "version": "7.14.0-3" + "key": "amdrocm-runtime10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-base7.14_7.14.0-3_amd64", - "name": "amdrocm-base7.14", - "version": "7.14.0-3" + "key": "amdrocm-base10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0~20260903-33697653391" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -6155,9 +3998,9 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "amdrocm-sysdeps7.14_7.14.0-3_amd64", - "name": "amdrocm-sysdeps7.14", - "version": "7.14.0-3" + "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0~20260903-33697653391" }, { "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", @@ -6250,52 +4093,47 @@ "version": "2.35-0ubuntu3.10" } ], - "key": "amdrocm-runtime-dev7.14_7.14.0-3_amd64", - "name": "amdrocm-runtime-dev7.14", - "sha256": "2b3f822939a87291ccf19670c3a0a20472df6aa4ae3042472988afa7de552e8d", + "key": "amdrocm-blas-dev10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas-dev10.1", + "sha256": "1bef633435f6e92942021d66f6ec69f62e1f0f04d23fcd1268df8602365e2f0c", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-runtime-dev7.14_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas-dev10.1_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-llvm-dev7.14_7.14.0-3_amd64", - "name": "amdrocm-llvm-dev7.14", - "sha256": "d5319c83a26687ece983b03da6d68ca6b3dcf3f20313c5c93efd7fd9df9b7de9", + "key": "amdrocm-hipblas-common-dev10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-hipblas-common-dev10.1", + "sha256": "52aa990a98bbcb45ec5b6cc9e392f8cea38a895e0827eb0d6e96eff8a47381f3", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-llvm-dev7.14_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-hipblas-common-dev10.1_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [ { - "key": "amdrocm-solver7.14-gfx950_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx950", - "version": "7.14.0-3" + "key": "amdrocm-llvm-dev10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-llvm-dev10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx950_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx950", - "version": "7.14.0-3" + "key": "amdrocm-llvm10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas-host7.14_7.14.0-3_amd64", - "name": "amdrocm-blas-host7.14", - "version": "7.14.0-3" + "key": "amdrocm-runtime10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-profiler-base7.14_7.14.0-3_amd64", - "name": "amdrocm-profiler-base7.14", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sysdeps7.14_7.14.0-3_amd64", - "name": "amdrocm-sysdeps7.14", - "version": "7.14.0-3" + "key": "amdrocm-base10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0~20260903-33697653391" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -6318,14 +4156,382 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "amdrocm-runtime7.14_7.14.0-3_amd64", - "name": "amdrocm-runtime7.14", - "version": "7.14.0-3" + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" }, { - "key": "amdrocm-llvm7.14_7.14.0-3_amd64", - "name": "amdrocm-llvm7.14", - "version": "7.14.0-3" + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + } + ], + "key": "amdrocm-runtime-dev10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-runtime-dev10.1", + "sha256": "30cc57738702d459a1e13452bb2c1a0a7195c7b31f2e23b9f5c9b7adf216b750", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-runtime-dev10.1_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-llvm-dev10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-llvm-dev10.1", + "sha256": "94255d09d463bd77e01773bb43a7c8a12d64382c283b94a8d8b78e0ed27ef453", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-llvm-dev10.1_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "amdrocm-dnn10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx950", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx950", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx950", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx950", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse-host10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse-host10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-runtime10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-llvm10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" }, { "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", @@ -6528,9 +4734,9 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "amdrocm-base7.14_7.14.0-3_amd64", - "name": "amdrocm-base7.14", - "version": "7.14.0-3" + "key": "amdrocm-base10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0~20260903-33697653391" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -6623,421 +4829,915 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "amdrocm-sparse7.14-gfx950_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx950", - "version": "7.14.0-3" + "key": "amdrocm-profiler-base10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-profiler-base10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse-host7.14_7.14.0-3_amd64", - "name": "amdrocm-sparse-host7.14", - "version": "7.14.0-3" + "key": "amdrocm-solver-host10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver-host10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver-host7.14_7.14.0-3_amd64", - "name": "amdrocm-solver-host7.14", - "version": "7.14.0-3" + "key": "amdrocm-math-common10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-math-common10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-math-common7.14_7.14.0-3_amd64", - "name": "amdrocm-math-common7.14", - "version": "7.14.0-3" + "key": "amdrocm-blas-host10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas-host10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx942_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx942", - "version": "7.14.0-3" + "key": "amdrocm-dnn-host10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn-host10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx942_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx942", - "version": "7.14.0-3" + "key": "amdrocm-dnn10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx942", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx942_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx942", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx942", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx90a_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx90a", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx942", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx90a_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx90a", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx942", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx90a_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx90a", - "version": "7.14.0-3" + "key": "amdrocm-dnn10.1-gfx90c_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx90c", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx908_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx908", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx90c_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx90c", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx908_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx908", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx90c_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx90c", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx908_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx908", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx90c_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx90c", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1250_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1250", - "version": "7.14.0-3" + "key": "amdrocm-dnn10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx90a", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1250_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1250", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx90a", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1250_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1250", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx90a", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1201_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1201", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx90a", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1201_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1201", - "version": "7.14.0-3" + "key": "amdrocm-dnn10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx908", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1201_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1201", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx908", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1200_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1200", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx908", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1200_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1200", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx908", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1200_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1200", - "version": "7.14.0-3" + "key": "amdrocm-dnn10.1-gfx906_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx906", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1153_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1153", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx906_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx906", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1153_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1153", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx906_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx906", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1153_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1153", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx906_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx906", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1152_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1152", - "version": "7.14.0-3" + "key": "amdrocm-dnn10.1-gfx900_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx900", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1152_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1152", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx900_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx900", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1152_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1152", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx900_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx900", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1151_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1151", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx900_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx900", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1151_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1151", - "version": "7.14.0-3" + "key": "amdrocm-dnn10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1250", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1151_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1151", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1250", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1150_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1150", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1250", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1150_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1150", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1250", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1150_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1150", - "version": "7.14.0-3" + "key": "amdrocm-dnn10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1201", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1103_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1103", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1201", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1103_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1103", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1201", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1103_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1103", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1201", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1102_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1102", - "version": "7.14.0-3" + "key": "amdrocm-dnn10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1200", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1102_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1102", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1200", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1102_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1102", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1200", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1101_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1101", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1200", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1101_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1101", - "version": "7.14.0-3" + "key": "amdrocm-dnn10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1153", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1101_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1101", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1153", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1100_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1100", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1153", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1100_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1100", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1153", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1100_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1100", - "version": "7.14.0-3" + "key": "amdrocm-dnn10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1152", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1036_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1036", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1152", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1036_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1036", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1152", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1036_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1036", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1152", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1035_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1035", - "version": "7.14.0-3" + "key": "amdrocm-dnn10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1151", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1035_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1035", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1151", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1035_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1035", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1151", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1034_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1034", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1151", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1034_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1034", - "version": "7.14.0-3" + "key": "amdrocm-dnn10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1150", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1034_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1034", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1150", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1033_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1033", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1150", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1033_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1033", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1150", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1033_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1033", - "version": "7.14.0-3" + "key": "amdrocm-dnn10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1103", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1032_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1032", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1103", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1032_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1032", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1103", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1032_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1032", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1103", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1031_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1031", - "version": "7.14.0-3" + "key": "amdrocm-dnn10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1102", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1031_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1031", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1102", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1031_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1031", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1102", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1030_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1030", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1102", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1030_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1030", - "version": "7.14.0-3" + "key": "amdrocm-dnn10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1101", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1030_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1030", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1101", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1012_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1012", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1101", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1012_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1012", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1101", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1012_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1012", - "version": "7.14.0-3" + "key": "amdrocm-dnn10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1100", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1011_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1011", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1100", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1011_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1011", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1100", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1011_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1011", - "version": "7.14.0-3" + "key": "amdrocm-sparse10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1100", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1010_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1010", - "version": "7.14.0-3" + "key": "amdrocm-dnn10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1036", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1010_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1010", - "version": "7.14.0-3" + "key": "amdrocm-blas10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1036", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1010_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1010", - "version": "7.14.0-3" + "key": "amdrocm-solver10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1036", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1036", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-dnn10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1035", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1035", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1035", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1035", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-dnn10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1034", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1034", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1034", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1034", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-dnn10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1033", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1033", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1033", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1033", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-dnn10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1032", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1032", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1032", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1032", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-dnn10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1031", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1031", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1031", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1031", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-dnn10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1030", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1030", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1030", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1030", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-dnn10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1012", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1012", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1012", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1012", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-dnn10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1011", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1011", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1011", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1011", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-dnn10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1010", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1010", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1010", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1010", + "version": "10.1.0~20260903-33697653391" } ], - "key": "amdrocm-solver7.14_7.14.0-3_amd64", - "name": "amdrocm-solver7.14", - "sha256": "d2440e2d0c746724ef52a61f1f5301bf08eec77078ddf4b732a596502ca7e8f9", + "key": "amdrocm-dnn10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1", + "sha256": "fabb0943631b83b6cec3e430b42ff6f5731de8087e62d68adac1683ee0d9391f", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-solver7.14_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx950", + "sha256": "4cb346e12960579f71d348a7d876e4b563b661ba7d97cc47066ee32593d74f8a", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx950_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn-host10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn-host10.1", + "sha256": "996c4605b993f63e59508cd41f41854f39c9d79105be6a2189a36f9bd3bc00e4", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn-host10.1_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx942", + "sha256": "f60883c8e76e1e896f9c8a3f0ef5315bc3fb475add98a99c3ccfb0b182d1c9e7", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx942_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx90c_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx90c", + "sha256": "914f8c011c3157b09a80b75a6c93da4e208066cc27858f4369b5005cdc9ef379", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx90c_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx90a", + "sha256": "c0e10e44b49fa26dc954599cfe2892d0ccc01f6a7fbbe761b7f5b2d159f45f46", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx90a_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx908", + "sha256": "9416803c73afc49146416549723c1c773477c35eadc5ba61bdfa889a5529cd5f", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx908_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx906_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx906", + "sha256": "518d95df9a8db9e400ab0373d8c15d8aa95b9016da450e44f9aaefc856adda77", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx906_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx900_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx900", + "sha256": "f16c2aa5b65adce059ab714af64c950b4e6d4ebe4601e396d1a6f62aca07473b", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx900_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1250", + "sha256": "ba1a9af51d934a5ce69ed28ae4392e39fd14c0f99607d9913cedc1ad4253d5d5", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1250_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1201", + "sha256": "4af7048c36883c5f8081446386508fa1ec7d1b786924498f0bb374a0b7236570", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1201_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1200", + "sha256": "e54eb6d0352b23bb13ce31bf51418e6b1d48c0fdcb0b5ef1a4789cf88f0a5180", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1200_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1153", + "sha256": "7346c8fa685deb046589ec848cda162f6b4b61e6f46e10af29ca61edb75e0198", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1153_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1152", + "sha256": "a92b0bb304eb8241f03c8c5fa7137c8ff219b685011a8ed522a950e8cb7ab551", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1152_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1151", + "sha256": "036da64732312dd730cc10442001d5606ab16fff39380e2023efc12986bfcfd3", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1151_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1150", + "sha256": "e19418cc9c8520eba50f9a47a5efc6fdf8a899e17da9e180b9d03aafcbc371b9", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1150_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1103", + "sha256": "e45cba470dd199188673f8d76e2f667d9417c49b0460b439a6854b6f3f6f2c7a", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1103_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1102", + "sha256": "d8a35cc470cc3c125d35af9c6bfccec90fe692ea7561753ba5060b48bb5cf9d3", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1102_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1101", + "sha256": "b5614e8e4533d2a9af33e98ba9d2de4c83f74898f6558a217aefb50a69c269c4", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1101_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1100", + "sha256": "374a434a45a6b0f173c96886eadc168dc72b65fde7e23c49ee6766102ce82335", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1100_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1036", + "sha256": "44ef31ce86a418b83c63244b298092b553223f03f5b074a0359ace3f2887f6e8", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1036_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1035", + "sha256": "5dbc1a29d58783dc73c722665c062cf215f2919f3c75c7c3bf25580f40586a95", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1035_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1034", + "sha256": "21f7823c4fd75c4f442283870856f64e94c7a92d86d60d7ca5812256f723d480", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1034_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1033", + "sha256": "8e0b558c18bab1a9757061e3286b3dd0ed54e940c05a41956764fa6fa24f31c4", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1033_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1032", + "sha256": "c7b3c1eed6dc0f5ecbb01e606aeda85390e70a36f681dc0b9d07d5e28a1103e8", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1032_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1031", + "sha256": "708590387b6d623f62dd37f46bf2869ac70003b8906f8e9678e1968b5e97f021", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1031_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1030", + "sha256": "2be1d98c784bd9b0f9d3404e97f396e65c52aa9bdf0deb2ce4e0766831624c92", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1030_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1012", + "sha256": "fdf531b7ba1b5b5481c992917421da3b821693a1a1d58f1174b45c078a53d7fa", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1012_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1011", + "sha256": "f37873e0de4116d7ad6755e3237d09e9083d452f035006b017a8e0ad6f995f82", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1011_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-dnn10.1-gfx1010", + "sha256": "420f999e607c8788d05d14653e5b29c9162b968edfdaf27aa870055b0dae4590", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1010_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" }, { "arch": "amd64", "dependencies": [ { - "key": "amdrocm-sparse7.14-gfx950_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx950", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx950", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx950_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx950", - "version": "7.14.0-3" + "key": "amdrocm-fft-host10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft-host10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx950_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx950", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver-host7.14_7.14.0-3_amd64", - "name": "amdrocm-solver-host7.14", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-math-common7.14_7.14.0-3_amd64", - "name": "amdrocm-math-common7.14", - "version": "7.14.0-3" + "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0~20260903-33697653391" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -7060,29 +5760,14 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "amdrocm-blas-host7.14_7.14.0-3_amd64", - "name": "amdrocm-blas-host7.14", - "version": "7.14.0-3" + "key": "amdrocm-runtime10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-profiler-base7.14_7.14.0-3_amd64", - "name": "amdrocm-profiler-base7.14", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sysdeps7.14_7.14.0-3_amd64", - "name": "amdrocm-sysdeps7.14", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-runtime7.14_7.14.0-3_amd64", - "name": "amdrocm-runtime7.14", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-llvm7.14_7.14.0-3_amd64", - "name": "amdrocm-llvm7.14", - "version": "7.14.0-3" + "key": "amdrocm-llvm10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0~20260903-33697653391" }, { "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", @@ -7285,9 +5970,9 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "amdrocm-base7.14_7.14.0-3_amd64", - "name": "amdrocm-base7.14", - "version": "7.14.0-3" + "key": "amdrocm-base10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0~20260903-33697653391" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -7380,379 +6065,2794 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "amdrocm-sparse-host7.14_7.14.0-3_amd64", - "name": "amdrocm-sparse-host7.14", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx942", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx942_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx942", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx90c_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx90c", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx942_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx942", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx90a", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx942_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx942", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx908", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx90a_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx90a", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx906_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx906", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx90a_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx90a", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx900_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx900", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx90a_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx90a", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1250", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx908_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx908", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1201", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx908_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx908", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1200", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx908_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx908", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1153", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1250_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1250", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1152", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1250_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1250", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1151", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1250_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1250", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1150", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1201_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1201", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1103", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1201_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1201", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1102", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1201_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1201", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1101", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1200_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1200", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1100", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1200_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1200", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1036", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1200_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1200", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1035", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1153_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1153", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1034", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1153_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1153", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1033", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1153_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1153", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1032", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1152_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1152", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1031", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1152_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1152", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1030", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-solver7.14-gfx1152_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1152", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1012", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-sparse7.14-gfx1151_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1151", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1011", + "version": "10.1.0~20260903-33697653391" }, { - "key": "amdrocm-blas7.14-gfx1151_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1151", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1151_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1151", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx1150_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1150", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx1150_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1150", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1150_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1150", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx1103_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1103", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx1103_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1103", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1103_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1103", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx1102_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1102", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx1102_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1102", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1102_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1102", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx1101_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1101", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx1101_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1101", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1101_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1101", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx1100_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1100", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx1100_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1100", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1100_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1100", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx1036_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1036", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx1036_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1036", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1036_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1036", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx1035_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1035", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx1035_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1035", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1035_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1035", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx1034_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1034", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx1034_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1034", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1034_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1034", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx1033_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1033", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx1033_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1033", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1033_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1033", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx1032_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1032", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx1032_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1032", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1032_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1032", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx1031_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1031", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx1031_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1031", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1031_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1031", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx1030_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1030", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx1030_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1030", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1030_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1030", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx1012_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1012", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx1012_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1012", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1012_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1012", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx1011_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1011", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx1011_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1011", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1011_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1011", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-sparse7.14-gfx1010_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14-gfx1010", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-blas7.14-gfx1010_7.14.0-3_amd64", - "name": "amdrocm-blas7.14-gfx1010", - "version": "7.14.0-3" - }, - { - "key": "amdrocm-solver7.14-gfx1010_7.14.0-3_amd64", - "name": "amdrocm-solver7.14-gfx1010", - "version": "7.14.0-3" + "key": "amdrocm-fft10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1010", + "version": "10.1.0~20260903-33697653391" } ], - "key": "amdrocm-sparse7.14_7.14.0-3_amd64", - "name": "amdrocm-sparse7.14", - "sha256": "e2ebd0788d98eb6cef4362dcecf43066f4bef3a07ce98f41e0b7549b4fd93f68", + "key": "amdrocm-fft10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1", + "sha256": "6538002afe1d2a4f8d31e56f3cc83b2cd99092cadcf3324be831d2e7bb16c7bb", "urls": [ - "https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204/pool/main/amdrocm-sparse7.14_7.14.0-3_amd64.deb" + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1_10.1.0~20260903-33697653391_amd64.deb" ], - "version": "7.14.0-3" + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx950", + "sha256": "3ead73072ea39ebfecdb9d0736f50e83124fe2bf52ced522e02c762bc69ab367", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx950_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft-host10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft-host10.1", + "sha256": "56ebdf77beb71f04cc4cf916fc9281ac4f24a09a0d9b4b6801c6052411497b02", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft-host10.1_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx942", + "sha256": "ac2378092eb6af2caeb6c483baadb4737121697675e906a6d55e1d4db30e1e37", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx942_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx90c_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx90c", + "sha256": "a0f062551dec371c5d42ca76ca006860b2e6e80d3a0981beae56069642079a25", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx90c_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx90a", + "sha256": "d4a7a2a4cf2bba3332a6d1331246cc0ba2d4b9633ff69b62517efac22796e4a1", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx90a_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx908", + "sha256": "43209ba15011876fc19e3910775c4b978f7f5c2628d45167c1b1f3defe1c8884", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx908_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx906_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx906", + "sha256": "c290e7eb143b593897dc3ee4b5fbdf5a9ccc5b211499471be13796ad1562ede0", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx906_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx900_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx900", + "sha256": "65dc89e130d932c97838a27123a7fec83433c3e3d1aac617f634a1b4ecc0754b", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx900_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1250", + "sha256": "0d4392b50e74b7ca9c2c6b6263167a5aada416d9415bc3d29ed63c25dfdcfca1", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1250_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1201", + "sha256": "fbb4030d3040af10987cf0652439b1f4f59e6f6b62335392ff9462e17217bd90", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1201_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1200", + "sha256": "c34b93ca4721dd052e7ba00a2450892bca08014686849a2c383bf61e22224e72", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1200_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1153", + "sha256": "b0cd1b3a33e8d6170631f82c5b38e703664750d6ff31ecdf2293c990b557bacb", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1153_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1152", + "sha256": "750516dba73b09a083f7a1ac2cd14097200f2e59c2d0230b3fedddefd3c1f7b8", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1152_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1151", + "sha256": "d647b4847f0780c6e4474fd201571373693c6abbece5577e209c99391ffb90cb", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1151_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1150", + "sha256": "1abe94fe014ac7af44c6547de43b3a14a0db82a5b18fde7456c1a4c144abfffc", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1150_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1103", + "sha256": "a5b3ebadfb033886fd7c637cb8122f397171e511a1d28616da8cf4511a22cbfa", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1103_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1102", + "sha256": "d9b467c06abc268e5ca2d26b8a2c0da64b48f733e54cabdc18b06c18de23de16", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1102_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1101", + "sha256": "9c5e08a8690073a9dd9113f99265321a913e3624fea6d8ed7a0ab65bdf060a60", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1101_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1100", + "sha256": "f85432c5c9282e643212c3692ebc9a0805f5d15be6f13f2432294a699b8a488e", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1100_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1036", + "sha256": "b4fd30b8f178a66727f26baa6459a8e73153853f4a4a79bcf99f45ca2aa1ac91", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1036_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1035", + "sha256": "40305b1b8c983ac3c12a553058deeddc8b47b67c247570214eea10baa1c3c27d", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1035_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1034", + "sha256": "8b8ac4bafef4808c60d2f96a36755c33f167d22a5fec79f9588f5fb8501d80c8", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1034_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1033", + "sha256": "06e3a930cee8af98bb654201afec55d2d7541cad785d2bba9bf50e14bdd88f4b", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1033_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1032", + "sha256": "5e991ac1f2724adb5c13a901404317c67b863a53b507abf8ac2e8be4de57f257", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1032_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1031", + "sha256": "dc5a63b123e63d8c816557e75c3f2b571e6001027941635c5c4af497d9ce3ada", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1031_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1030", + "sha256": "0b4174ac0339450b0c63d9477afeab84fb98310453401e5a73a408e2dbc50eba", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1030_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1012", + "sha256": "1997587ecc2aa1eb380b62b51061b50c2f7ffc00036214162b16191deffa56d5", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1012_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1011", + "sha256": "da905e999e4165475884afcc0eae9f37363ab4cca3ae9423d5e665fcf7b502d2", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1011_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-fft10.1-gfx1010", + "sha256": "4be01dc7c5f64496bf197efefa26859a5ae1f70d6f657e2a7dd5025ed05c4fc5", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1010_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "amdrocm-rccl10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx950", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-rccl-host10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl-host10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-profiler-base10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-profiler-base10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "amdrocm-runtime10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-llvm10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "amdrocm-base10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "amdrocm-amdsmi10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-amdsmi10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-rccl10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx942", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-rccl10.1-gfx90c_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx90c", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-rccl10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx90a", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-rccl10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx908", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-rccl10.1-gfx906_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx906", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-rccl10.1-gfx900_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx900", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-rccl10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1250", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-rccl10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1201", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-rccl10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1200", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-rccl10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1151", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-rccl10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1102", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-rccl10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1101", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-rccl10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1100", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-rccl10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1036", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-rccl10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1035", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-rccl10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1034", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-rccl10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1033", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-rccl10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1032", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-rccl10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1031", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-rccl10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1030", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-rccl10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1012", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-rccl10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1011", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-rccl10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1010", + "version": "10.1.0~20260903-33697653391" + } + ], + "key": "amdrocm-rccl10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1", + "sha256": "790722d01adc9c1bf8c800534068fcb372caf43fe6fbff51ce8fc8e1ab7cc8e5", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx950", + "sha256": "24bc92cdaa7974eaab279e38e1af94478adf57be1e2ab13399e684a2b65a5be7", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx950_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl-host10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl-host10.1", + "sha256": "6444b12d36c247cfc955829de234ef784ee6298beed04a5f5a6bd560fe791be8", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl-host10.1_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx942", + "sha256": "7691821aebf05ff3beeb7db947b60999de23a00b12f585fb3b66462cb71e903d", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx942_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx90c_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx90c", + "sha256": "52564c6944164aa422f2b2cb898ce833e8ae48617c8ac6ee56adc2146e2c10c4", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx90c_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx90a", + "sha256": "774ef85c35b4824f5689f70de9092b9cb18a9a467fec78779404d6ddba7b0426", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx90a_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx908", + "sha256": "416468a8580570e18bf4b82ab276babdd6f5e8e870667d661c4e4f5fb73fa64b", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx908_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx906_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx906", + "sha256": "e3a50d7f0e3104bdaea8dd2e20a1c9ad6cb77ad8147eea9510cad838bf523223", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx906_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx900_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx900", + "sha256": "94d3cefa9c8993b3437fd20cb2f6628deac396d5ae23c9b7c641908ee77c8e50", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx900_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1250", + "sha256": "570a58e63a70e13dad9d14334a48857cb9a759ac60651f36fa2df231eb3e9e6e", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1250_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1201", + "sha256": "5abe6dfe6097048cfa5a22fd6705889cf61363667008fd8cc0bee1f585d69ca7", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1201_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1200", + "sha256": "03278667e2e95a8cbc979e4de10104aecc1b521624dbb043ba2567931469bcec", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1200_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1151", + "sha256": "554fa2650e4619f324562ccbc568f1f44c16cb22816d51680c48c132fd7c8693", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1151_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1102", + "sha256": "021051e9cf9cb49a82647041ec449c35de2e540302b8a0da0200a4933713197c", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1102_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1101", + "sha256": "66563039aeb86fdda465d6be01d0429f205bbc028deaf75048c023ceb167d462", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1101_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1100", + "sha256": "f3a239c13110330a21a84327e15c631c35e7d2522c736899d2a4c7b453084ea2", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1100_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1036", + "sha256": "dce7d2a9c8136e63a33b432e58fa64efb6bf9bc00c39485b2ad472332ce54b22", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1036_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1035", + "sha256": "be1dc0fa1a1d01aa9fe831a18342690383c1497e746162e42b4cf5d6b899357e", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1035_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1034", + "sha256": "b515cda1f55817a4350fad0957dff5693a6b55577cc4923afd2a9a41d09ab56a", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1034_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1033", + "sha256": "70ea2ed56caf665952eb520d059503bca46f45b00c3f34501ace894af09ce9d2", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1033_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1032", + "sha256": "1d99c301bfe389f335fee87b99436bab7da3f11dd56c4b9732330e6f03079d93", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1032_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1031", + "sha256": "e1740e46bd4ada5565cac1ec827971838ee4e23b30eba1ba242e3be47fa1a23d", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1031_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1030", + "sha256": "9608e4772bcd714e8fdca999501539acd36e504114e3cda32ee5d10fcabe82a9", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1030_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1012", + "sha256": "6a4e521f0bbc6d07d49eb8856fae118f78cfd14a13210f2c7926fcd3191f0721", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1012_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1011", + "sha256": "8a05e6996a2c97d37014fb743fe751f0e5f7caf87424f846f64e9e785d315032", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1011_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-rccl10.1-gfx1010", + "sha256": "7098b435b3bba65a309532d154d71d8e477d54a3b28fc8b0b7f976a866f61b67", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1010_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "amdrocm-solver10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx950", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx950", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas-host10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas-host10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-profiler-base10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-profiler-base10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "amdrocm-runtime10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-llvm10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "amdrocm-base10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "amdrocm-sparse10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx950", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse-host10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse-host10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver-host10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver-host10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-math-common10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-math-common10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx942", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx942", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx942", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx90c_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx90c", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx90c_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx90c", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx90c_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx90c", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx90a", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx90a", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx90a", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx908", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx908", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx908", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx906_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx906", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx906_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx906", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx906_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx906", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx900_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx900", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx900_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx900", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx900_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx900", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1250", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1250", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1250", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1201", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1201", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1201", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1200", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1200", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1200", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1153", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1153", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1153", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1152", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1152", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1152", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1151", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1151", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1151", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1150", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1150", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1150", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1103", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1103", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1103", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1102", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1102", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1102", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1101", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1101", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1101", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1100", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1100", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1100", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1036", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1036", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1036", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1035", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1035", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1035", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1034", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1034", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1034", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1033", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1033", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1033", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1032", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1032", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1032", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1031", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1031", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1031", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1030", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1030", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1030", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1012", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1012", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1012", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1011", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1011", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1011", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1010", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1010", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1010", + "version": "10.1.0~20260903-33697653391" + } + ], + "key": "amdrocm-solver10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1", + "sha256": "7fdefce3a45950fd85acd08a7dad35a227be225b4abc544fbb9292cce850df79", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "amdrocm-sparse10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx950", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx950", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx950", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver-host10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver-host10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-math-common10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-math-common10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "amdrocm-blas-host10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas-host10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-profiler-base10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-profiler-base10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-runtime10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-llvm10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "amdrocm-base10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "amdrocm-sparse-host10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse-host10.1", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx942", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx942", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx942", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx90c_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx90c", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx90c_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx90c", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx90c_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx90c", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx90a", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx90a", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx90a", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx908", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx908", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx908", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx906_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx906", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx906_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx906", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx906_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx906", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx900_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx900", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx900_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx900", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx900_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx900", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1250", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1250", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1250", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1201", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1201", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1201", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1200", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1200", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1200", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1153", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1153", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1153", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1152", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1152", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1152", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1151", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1151", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1151", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1150", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1150", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1150", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1103", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1103", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1103", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1102", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1102", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1102", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1101", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1101", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1101", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1100", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1100", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1100", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1036", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1036", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1036", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1035", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1035", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1035", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1034", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1034", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1034", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1033", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1033", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1033", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1032", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1032", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1032", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1031", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1031", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1031", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1030", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1030", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1030", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1012", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1012", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1012", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1011", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1011", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1011", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-sparse10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1-gfx1010", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-blas10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-blas10.1-gfx1010", + "version": "10.1.0~20260903-33697653391" + }, + { + "key": "amdrocm-solver10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-solver10.1-gfx1010", + "version": "10.1.0~20260903-33697653391" + } + ], + "key": "amdrocm-sparse10.1_10.1.0_20260903-33697653391_amd64", + "name": "amdrocm-sparse10.1", + "sha256": "ce618ab8b73fe2d57e6d62f1edec18cd6a850e2bfd33a31fcb6d7bcf0a375a3d", + "urls": [ + "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1_10.1.0~20260903-33697653391_amd64.deb" + ], + "version": "10.1.0~20260903-33697653391" } ], "version": 1 -} \ No newline at end of file +} diff --git a/platforms/rocm/packages.yaml b/platforms/rocm/packages.yaml index 4c8f22a7..57673100 100644 --- a/platforms/rocm/packages.yaml +++ b/platforms/rocm/packages.yaml @@ -5,7 +5,7 @@ version: 1 sources: - channel: stable main - url: https://repo.amd.com/rocm/packages-multi-arch/ubuntu2204 + url: https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391 - channel: jammy main url: https://snapshot.ubuntu.com/ubuntu/20250711T030400Z - channel: jammy-security main @@ -19,15 +19,15 @@ archs: packages: - "libatomic1" - "libdrm-common" - - "amdrocm-amdsmi7.14" - - "amdrocm-base7.14" - - "amdrocm-blas7.14" - - "amdrocm-blas-dev7.14" - - "amdrocm-dnn7.14" - - "amdrocm-fft7.14" - - "amdrocm-llvm7.14" - - "amdrocm-profiler-base7.14" - - "amdrocm-rccl7.14" - - "amdrocm-runtime-dev7.14" - - "amdrocm-solver7.14" - - "amdrocm-sparse7.14" + - "amdrocm-amdsmi10.1" + - "amdrocm-base10.1" + - "amdrocm-blas10.1" + - "amdrocm-blas-dev10.1" + - "amdrocm-dnn10.1" + - "amdrocm-fft10.1" + - "amdrocm-llvm10.1" + - "amdrocm-profiler-base10.1" + - "amdrocm-rccl10.1" + - "amdrocm-runtime-dev10.1" + - "amdrocm-solver10.1" + - "amdrocm-sparse10.1" diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 04abe15c..cc617ebc 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -9,10 +9,10 @@ package(default_visibility = ["//visibility:public"]) load("@rules_cc//cc:cc_library.bzl", "cc_library") """ -_ROCM_VERSION = "7.14" +_ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PJRT_ROCM_URL = "https://github.com/zml/pjrt-artifacts/releases/download/manual-2026-07-20T15-30-00Z/pjrt-rocm_linux-amd64.tar.gz" -_PJRT_ROCM_SHA256 = "6fd0515beb299550e298996f6919db09aec79859feee7362443bf2ebff900d0f" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202609101243.20.1.7ca6884ea2cb/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "d2a311f15532691d7335c0436d4dda13d39e8597e4bb10dbe83b11b02a627546" def _rocm_package_name(name): return name + _ROCM_VERSION @@ -88,18 +88,19 @@ def _rocm_base_build_files(loaded_packages): return { _rocm_package_name("amdrocm-amdsmi"): "\n\n".join([ packages.cc_library(name = "amdsmi", hdrs = ["include/amd_smi/amdsmi.h"], includes = ["include/amd_smi"]), - packages.filegroup(name = "libamd_smi", srcs = ["lib/libamd_smi.so.26"]), + packages.filegroup(name = "libamd_smi", srcs = ["lib/libamd_smi.so.27"]), ]), _rocm_package_name("amdrocm-base"): "\n\n".join([ _rocm_dlopen_patchelf(name = "rocm-core", src = "lib/librocm-core.so.1"), - _rocm_dlopen_patchelf(name = "rocm_smi", src = "lib/librocm_smi64.so.1"), _rocm_dlopen_patchelf(name = "rocprofiler-register", src = "lib/librocprofiler-register.so.0"), ]), _rocm_package_name("amdrocm-blas-dev"): _glob_filegroup("headers", ["include/**"]), _rocm_package_name("amdrocm-blas-host"): "\n\n".join([ _rocm_dlopen_patchelf(name = "hipblas", src = "lib/libhipblas.so.3"), - packages.filegroup(name = "hipblaslt", srcs = ["lib/libhipblaslt.so.1"]), - _rocm_dlopen_patchelf(name = "hipsparselt", src = "lib/libhipsparselt.so.0"), + packages.filegroup(name = "hipblaslt", srcs = [ + "lib/libhipblaslt.so.1", + "lib/liborigami.so.1", + ]), packages.filegroup(name = "rocblas", srcs = ["lib/librocblas.so.5"]), _rocm_dlopen_patchelf(name = "rocroller", src = "lib/librocroller.so.1"), _glob_filegroup("hipblaslt_support", [ @@ -117,7 +118,6 @@ def _rocm_base_build_files(loaded_packages): ]), packages.filegroup(name = "rocblas", srcs = [ "@amdrocm-blas-host//:hipblas", - "@amdrocm-blas-host//:hipsparselt", "@amdrocm-blas-host//:rocblas", ]), packages.filegroup(name = "rocblas_runfiles", srcs = _family_runfile_labels(loaded_packages, "blas")), @@ -151,8 +151,8 @@ def _rocm_base_build_files(loaded_packages): ]), _rocm_package_name("amdrocm-llvm"): "\n\n".join([ packages.filegroup(name = "llvm_libs", srcs = [ - "lib/llvm/lib/libLLVM.so.23.0git", - "lib/llvm/lib/libclang-cpp.so.23.0git", + "lib/llvm/lib/libLLVM.so.24.0git", + "lib/llvm/lib/libclang-cpp.so.24.0git", ]), _glob_filegroup("rocm_device_libs_runfiles", ["lib/llvm/amdgcn/**"]), ]), @@ -262,10 +262,10 @@ def _rocm_impl(mctx): ) http_archive( - name = "libpjrt_rocm", - build_file = "libpjrt_rocm.BUILD.bazel", - url = _PJRT_ROCM_URL, - sha256 = _PJRT_ROCM_SHA256, + name = "libzml_rocm", + build_file = "libzml_rocm.BUILD.bazel", + url = _PLUGIN_ROCM_URL, + sha256 = _PLUGIN_ROCM_SHA256, ) return mctx.extension_metadata( @@ -286,7 +286,7 @@ def _rocm_impl(mctx): "amdrocm-sysdeps", "libatomic1", "libdrm-common", - "libpjrt_rocm", + "libzml_rocm", ], root_module_direct_dev_deps = [], ) diff --git a/platforms/rocm/rocm.zig b/platforms/rocm/rocm.zig index c5820333..c3897dd1 100644 --- a/platforms/rocm/rocm.zig +++ b/platforms/rocm/rocm.zig @@ -41,7 +41,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { const r = try bazel.runfiles(bazel_builtin.current_repository); var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = try r.rlocation("libpjrt_rocm/sandbox", &path_buf) orelse { + const sandbox_path = try r.rlocation("libzml_rocm/sandbox", &path_buf) orelse { log.err("Failed to find sandbox path for ROCm runtime", .{}); return error.FileNotFound; }; @@ -59,7 +59,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { // executing the destructor. Accessing this variable results in a segmentation fault... return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const lib_path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_rocm.so" }); + const lib_path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_rocm.so" }); break :blk .loadFrom(lib_path); }; } diff --git a/platforms/tpu/BUILD.bazel b/platforms/tpu/BUILD.bazel index 3c1a8dff..6b872aec 100644 --- a/platforms/tpu/BUILD.bazel +++ b/platforms/tpu/BUILD.bazel @@ -8,10 +8,10 @@ cc_library( ) cc_library( - name = "libpjrt_tpu", + name = "libzml_tpu", tags = ["manual"], defines = ["ZML_RUNTIME_TPU"], - deps = ["@libpjrt_tpu"], + deps = ["@libzml_tpu"], ) zig_library( @@ -26,7 +26,7 @@ zig_library( "//stdx", ] + select({ "//platforms:tpu.enabled": [ - ":libpjrt_tpu", + ":libzml_tpu", ], "//conditions:default": [":empty"], }), diff --git a/platforms/tpu/libpjrt_tpu.BUILD.bazel b/platforms/tpu/libzml_tpu.BUILD.bazel similarity index 85% rename from platforms/tpu/libpjrt_tpu.BUILD.bazel rename to platforms/tpu/libzml_tpu.BUILD.bazel index 8574fe7e..1186510c 100644 --- a/platforms/tpu/libpjrt_tpu.BUILD.bazel +++ b/platforms/tpu/libzml_tpu.BUILD.bazel @@ -2,15 +2,15 @@ load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@bazel_skylib//rules:copy_file.bzl", "copy_file") copy_file( - name = "libpjrt_tpu.so", + name = "libzml_tpu.so", src = "libtpu/libtpu.so", - out = "lib/libpjrt_tpu.so", + out = "lib/libzml_tpu.so", ) copy_to_directory( name = "sandbox", srcs = [ - "lib/libpjrt_tpu.so", + "lib/libzml_tpu.so", ], add_directory_to_runfiles = False, allow_overwrites = True, @@ -29,7 +29,7 @@ copy_to_directory( ) cc_library( - name = "libpjrt_tpu", + name = "libzml_tpu", data = [":sandbox"], visibility = ["@zml//platforms/tpu:__subpackages__"], ) diff --git a/platforms/tpu/tpu.bzl b/platforms/tpu/tpu.bzl index 08d54ed7..55e2719f 100644 --- a/platforms/tpu/tpu.bzl +++ b/platforms/tpu/tpu.bzl @@ -3,15 +3,15 @@ load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") def _tpu_impl(mctx): # https://storage.googleapis.com/jax-releases/libtpu_releases.html http_archive( - name = "libpjrt_tpu", + name = "libzml_tpu", url = "https://storage.googleapis.com/libtpu-nightly-releases/wheels/libtpu/libtpu-0.0.42.dev20260613+nightly-cp314-cp314t-manylinux_2_31_x86_64.whl", type = "zip", sha256 = "1d0bbb1608154bda6944902abd6f56f956638539d7503db4894927e21a41baed", - build_file = "libpjrt_tpu.BUILD.bazel", + build_file = "libzml_tpu.BUILD.bazel", ) return mctx.extension_metadata( reproducible = True, - root_module_direct_deps = ["libpjrt_tpu"], + root_module_direct_deps = ["libzml_tpu"], root_module_direct_dev_deps = [], ) diff --git a/platforms/tpu/tpu.zig b/platforms/tpu/tpu.zig index 95c0d449..b385bb5c 100644 --- a/platforms/tpu/tpu.zig +++ b/platforms/tpu/tpu.zig @@ -47,14 +47,14 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { const r = try bazel.runfiles(bazel_builtin.current_repository); var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = try r.rlocation("libpjrt_tpu/sandbox", &path_buf) orelse { + const sandbox_path = try r.rlocation("libzml_tpu/sandbox", &path_buf) orelse { log.err("Failed to find sandbox path for TPU runtime", .{}); return error.FileNotFound; }; return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_tpu.so" }); + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_tpu.so" }); break :blk .loadFrom(path); }; } diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 066f76bd..e8c8519d 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -18,7 +18,7 @@ sh_binary( name = "rocprofv3", data = [ "@amdrocm-profiler-base//:rocprofv3_tool", - "@libpjrt_rocm//:sandbox", + "@libzml_rocm//:sandbox", ], srcs = ["rocprofv3.sh"], visibility = ["//visibility:public"], diff --git a/tools/neuron/BUILD.bazel b/tools/neuron/BUILD.bazel index f9e23117..8bfcabed 100644 --- a/tools/neuron/BUILD.bazel +++ b/tools/neuron/BUILD.bazel @@ -3,7 +3,7 @@ load("@rules_shell//shell:sh_binary.bzl", "sh_binary") sh_binary( name = "profile", srcs = ["profile.sh"], - data = ["@libpjrt_neuron//:sandbox"], + data = ["@libzml_neuron//:sandbox"], visibility = ["//visibility:public"], ) @@ -14,7 +14,7 @@ sh_binary( "/tmp/zml-neuron-explorer", "3001", ], - data = ["@libpjrt_neuron//:sandbox"], + data = ["@libzml_neuron//:sandbox"], visibility = ["//visibility:public"], ) @@ -25,27 +25,27 @@ sh_binary( "/tmp/zml-neuron-explorer", "/tmp/zml-neuron-profile", ], - data = ["@libpjrt_neuron//:sandbox"], + data = ["@libzml_neuron//:sandbox"], visibility = ["//visibility:public"], ) sh_binary( name = "summary-txt", srcs = ["summary.sh"], - data = ["@libpjrt_neuron//:sandbox"], + data = ["@libzml_neuron//:sandbox"], visibility = ["//visibility:public"], ) sh_binary( name = "summary-json", srcs = ["summary.sh"], - data = ["@libpjrt_neuron//:sandbox"], + data = ["@libzml_neuron//:sandbox"], visibility = ["//visibility:public"], ) sh_binary( name = "summary-perfetto", srcs = ["summary.sh"], - data = ["@libpjrt_neuron//:sandbox"], + data = ["@libzml_neuron//:sandbox"], visibility = ["//visibility:public"], ) diff --git a/tools/neuron/ingest.sh b/tools/neuron/ingest.sh index 464fbe5a..bb03850d 100755 --- a/tools/neuron/ingest.sh +++ b/tools/neuron/ingest.sh @@ -2,7 +2,7 @@ set -euo pipefail shopt -s nullglob -neuron_explorer_bin="$0.runfiles/+neuron_packages+libpjrt_neuron/sandbox/bin/neuron-explorer" +neuron_explorer_bin="$0.runfiles/+neuron_packages+libzml_neuron/sandbox/bin/neuron-explorer" data_path="$1" default_profile_root="$2" shift 2 diff --git a/tools/neuron/profile.sh b/tools/neuron/profile.sh index db6e10a1..e1118d56 100755 --- a/tools/neuron/profile.sh +++ b/tools/neuron/profile.sh @@ -10,7 +10,7 @@ compile_dir="${run_dir}/compile" mkdir -p "${execution_dir}" "${compile_dir}" -neuron_explorer_bin="$0.runfiles/+neuron_packages+libpjrt_neuron/sandbox/bin/neuron-explorer" +neuron_explorer_bin="$0.runfiles/+neuron_packages+libzml_neuron/sandbox/bin/neuron-explorer" xla_flags="--xla_dump_to=${compile_dir} --xla_dump_hlo_as_text --xla_dump_hlo_as_proto --xla_dump_hlo_pass_re=.*" diff --git a/tools/neuron/server.sh b/tools/neuron/server.sh index b0e57e41..3d0babe1 100755 --- a/tools/neuron/server.sh +++ b/tools/neuron/server.sh @@ -3,7 +3,7 @@ set -euo pipefail data_path="$1" port="$2" -neuron_explorer_bin="$0.runfiles/+neuron_packages+libpjrt_neuron/sandbox/bin/neuron-explorer" +neuron_explorer_bin="$0.runfiles/+neuron_packages+libzml_neuron/sandbox/bin/neuron-explorer" mkdir -p "${data_path}" diff --git a/tools/neuron/summary.sh b/tools/neuron/summary.sh index a4fe50cd..5a1b1461 100755 --- a/tools/neuron/summary.sh +++ b/tools/neuron/summary.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -neuron_explorer_bin="$0.runfiles/+neuron_packages+libpjrt_neuron/sandbox/bin/neuron-explorer" +neuron_explorer_bin="$0.runfiles/+neuron_packages+libzml_neuron/sandbox/bin/neuron-explorer" run_dir="$1" execution_dir="${run_dir}/execution" diff --git a/tools/rocprofv3.sh b/tools/rocprofv3.sh index 1b9c025b..cc7e70bb 100755 --- a/tools/rocprofv3.sh +++ b/tools/rocprofv3.sh @@ -15,7 +15,7 @@ else exit 1 fi -rocm_root="${runfiles_root}/+rocm_packages+libpjrt_rocm/sandbox" +rocm_root="${runfiles_root}/+rocm_packages+libzml_rocm/sandbox" if [[ ! -d "${rocm_root}" ]]; then echo "sandboxed ROCm runtime root not found at ${rocm_root}" >&2 exit 1 diff --git a/zml/Compiler.zig b/zml/Compiler.zig index d8e455ce..d795e9ee 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -740,15 +740,7 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_enable_subchannel_dequantisation_fusion", true, upb_arena); try setXlaOverrideFlag(overrides_map, "xla_gpu_unsupported_enable_triton_multi_output_fusion", true, upb_arena); }, - .rocm => { - // Use lld from libllvm instead of invoking the ld.lld binary. - // This saves us from having to sandbox it. - try setXlaOverrideFlag(overrides_map, "xla_gpu_use_inprocess_lld", true, upb_arena); - - // Do not enable the FUSION command buffer to avoid some weird crashes. - // This is what AMD recommendeds in the meantime. - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "CUBLAS,CUBLASLT,CUSTOM_CALL,CUDNN,DYNAMIC_SLICE_FUSION", upb_arena); - }, + .rocm => {}, .metal => { try setXlaOverrideFlag(overrides_map, "xla_gpu_metal_fast_math", false, upb_arena); }, @@ -781,8 +773,8 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: } switch (platform.target) { - .rocm, .cuda => if (std.c.getenv("ZML_AUTOTUNE_CACHE_DIR")) |path| { - try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_autotuner_cache_dir", std.mem.span(path), upb_arena); + .rocm, .cuda, .oneapi => if (std.c.getenv("ZML_AUTOTUNE_CACHE_DIR")) |path| { + try setXlaOverrideFlag(overrides_map, "xla_gpu_per_fusion_autotune_cache_dir", std.mem.span(path), upb_arena); }, else => {}, } diff --git a/zml/attention/nki/BUILD.bazel b/zml/attention/nki/BUILD.bazel index 12bbb568..a8b165de 100644 --- a/zml/attention/nki/BUILD.bazel +++ b/zml/attention/nki/BUILD.bazel @@ -17,7 +17,7 @@ py_binary( srcs = ["attention_harness.py"], data = [ ":attention.py", - "@libpjrt_neuron//:sandbox", + "@libzml_neuron//:sandbox", ], visibility = ["//visibility:public"], deps = [ diff --git a/zml/attention/nki/attention_harness.py b/zml/attention/nki/attention_harness.py index 372a0fc0..34ce5717 100644 --- a/zml/attention/nki/attention_harness.py +++ b/zml/attention/nki/attention_harness.py @@ -125,10 +125,10 @@ def load_nki_attention_module(): def add_neuron_tools_to_path(): runfiles = Runfiles.Create() python_neuronx_cc = Path( - runfiles.Rlocation("+neuron_packages+libpjrt_neuron/sandbox/bin/python-shims/neuronx-cc") + runfiles.Rlocation("+neuron_packages+libzml_neuron/sandbox/bin/python-shims/neuronx-cc") ) neuronx_cc = Path( - runfiles.Rlocation("+neuron_packages+libpjrt_neuron/sandbox/bin/neuronx-cc") + runfiles.Rlocation("+neuron_packages+libzml_neuron/sandbox/bin/neuronx-cc") ) os.environ["PATH"] = os.pathsep.join( (str(python_neuronx_cc.parent), str(neuronx_cc.parent), os.environ["PATH"]) From 28ea0571e65de5edabe772d475b527f931940eda Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Thu, 10 Sep 2026 19:19:34 +0200 Subject: [PATCH 674/881] zml/moe/triton: remove makeFusedMoeConfig and use clearer names (#792) `makeFusedMoeConfig` was just adding extra complexity and we used `a`, `b` and `c` everywhere. I think it's fine in the triton kernel itself, but outside of it's just confusing IMHO. I removed a bunch of useless parameters also that just end up returning an `Unsupported` error. --- zml/moe/triton.zig | 275 ++++++++-------------- zml/moe/triton_kernels/triton_kernels.zig | 43 ++-- zml/nn.zig | 4 + 3 files changed, 118 insertions(+), 204 deletions(-) diff --git a/zml/moe/triton.zig b/zml/moe/triton.zig index cff06fc1..8ca30b76 100644 --- a/zml/moe/triton.zig +++ b/zml/moe/triton.zig @@ -86,25 +86,25 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) !Tensor { const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); const gate_up = opts.gate_up.weight.withTags(.{ .expert, .out, .in }); const down = opts.down.weight.withTags(.{ .expert, .out, .mid }); - const weights = topk_weights.reshape(.{ .token = num_tokens, .in = topk_weights.dim(.top_expert) }).withTags(.{ .token, .topk }); + const routing_weights = topk_weights.reshape(.{ .token = num_tokens, .in = topk_weights.dim(.top_expert) }).withTags(.{ .token, .topk }); const ids = topk_ids.reshape(.{ .token = num_tokens, .in = topk_ids.dim(.top_expert) }).withTags(.{ .token, .topk }); stdx.debug.assert(hidden.dtype() == .bf16, "expected BF16 hidden states, got {}", .{hidden.dtype()}); stdx.debug.assert(gate_up.dtype() == .bf16 or gate_up.dtype() == .f8e4m3fn, "expected BF16 or FP8 E4M3FN gate/up weights, got {}", .{gate_up.dtype()}); stdx.debug.assert(down.dtype() == .bf16 or down.dtype() == .f8e4m3fn, "expected BF16 or FP8 E4M3FN down weights, got {}", .{down.dtype()}); - stdx.debug.assert(weights.dtype() == .f32 or weights.dtype() == .bf16, "expected FP32 or BF16 routing weights, got {}", .{weights.dtype()}); + stdx.debug.assert(routing_weights.dtype() == .f32 or routing_weights.dtype() == .bf16, "expected FP32 or BF16 routing weights, got {}", .{routing_weights.dtype()}); stdx.debug.assert(ids.dtype() == .i32, "expected I32 expert ids, got {}", .{ids.dtype()}); stdx.debug.assert(hidden.dim(.in) == gate_up.dim(.in), "hidden width {} must match gate/up input width {}", .{ hidden.dim(.in), gate_up.dim(.in) }); stdx.debug.assert(@rem(gate_up.dim(.out), 2) == 0, "gate/up output width must be even, got {}", .{gate_up.dim(.out)}); stdx.debug.assert(down.dim(.mid) == @divFloor(gate_up.dim(.out), 2), "down input width {} must equal half the gate/up output width {}", .{ down.dim(.mid), gate_up.dim(.out) }); - stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and weights.dim(.token) == hidden.dim(.token), "routing ids and weights must match hidden token count {}, got {} and {}", .{ hidden.dim(.token), ids.dim(.token), weights.dim(.token) }); - stdx.debug.assert(ids.dim(.topk) == weights.dim(.topk), "routing ids and weights must have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), weights.dim(.topk) }); + stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "routing ids and weights must match hidden token count {}, got {} and {}", .{ hidden.dim(.token), ids.dim(.token), routing_weights.dim(.token) }); + stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "routing ids and weights must have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), routing_weights.dim(.topk) }); stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "gate/up and down expert counts must match, got {} and {}", .{ gate_up.dim(.expert), down.dim(.expert) }); if (opts.expert_map) |expert_map| stdx.debug.assert(expert_map.dtype() == .i32 and expert_map.rank() == 1, "expected a rank-1 I32 expert map, got rank {} and dtype {}", .{ expert_map.rank(), expert_map.dtype() }); const num_experts = if (opts.expert_map) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); - const routing = prepareRouting(ids, num_experts, launch_config.block_size_m); + const routing = prepareRouting(ids, num_experts, @intCast(launch_config.block_size_m)); const expert_ids = if (opts.expert_map) |expert_map| expert_map.gather(.{ .expert = routing.expert_ids }, .{}).withTags(.{.g}) @@ -112,98 +112,51 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) !Tensor { routing.expert_ids; var hidden_quant = hidden; - var a_scale = Tensor.scalar(1.0, .f32); + var input_scale: ?Tensor = null; if (gate_up.dtype() == .f8e4m3fn) { - hidden_quant, a_scale = quantizePerTokenGroupFp8(hidden, fp8ActivationGroupSize(hidden)); + hidden_quant, input_scale = quantizePerTokenGroupFp8(hidden, fp8ActivationGroupSize(hidden)); } - const first_cfg = makeFusedMoeConfig( - hidden_quant, - gate_up, - opts.gate_up.quantizationScheme(), - launch_config, - routing.naive_block_assignment, - ids.dim(.topk), - false, - false, - .bf16, - ); + const gate_up_out = callFusedMoe(.{ + .input = hidden_quant, + .weight = gate_up, + .bias = opts.gate_up.bias, + .input_scale = input_scale, + .weight_scale = opts.gate_up.quantizationScales(), + .routing = routing, + .expert_ids = expert_ids, + .launch_config = launch_config, + .top_k = @intCast(ids.dim(.topk)), + .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), + }); - const gate_up_bias = - opts.gate_up.bias orelse - Tensor.zeroes(Shape.init(.{ .expert = gate_up.dim(.expert), .out = gate_up.dim(.out) }, .bf16)); - - const gate_up_scale = if (opts.gate_up.quantization) |q| q.scales else Tensor.scalar(1.0, .f32); - - const first_out = callFusedMoe( - hidden_quant, - gate_up, - gate_up_bias, - a_scale, - gate_up_scale, - weights, - routing.sorted_token_ids, - expert_ids, - routing.num_tokens_post_padded, - first_cfg, - launch_config, - routing.max_num_tokens_padded, - routing.num_assignments, - Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), - ); - - const activated = applyActivation(first_out, opts.activation, opts.activation_threshold); + const activated = applyActivation(gate_up_out, opts.activation, opts.activation_threshold); var activated_quant = activated; - a_scale = Tensor.scalar(1.0, .f32); + input_scale = null; if (down.dtype() == .f8e4m3fn) { - activated_quant, a_scale = quantizePerTokenGroupFp8(activated, fp8ActivationGroupSize(activated)); + activated_quant, input_scale = quantizePerTokenGroupFp8(activated, fp8ActivationGroupSize(activated)); } - const second_cfg = makeFusedMoeConfig( - activated_quant, - down, - opts.gate_up.quantizationScheme(), - launch_config, - routing.naive_block_assignment, - 1, - true, - false, - .bf16, - ); + const down_out = callFusedMoe(.{ + .input = activated_quant, + .weight = down, + .bias = opts.down.bias, + .input_scale = input_scale, + .weight_scale = opts.down.quantizationScales(), + .routing_weights = routing_weights, + .routing = routing, + .expert_ids = expert_ids, + .launch_config = launch_config, + .top_k = 1, + .output_shape = Shape.init(.{ .token = b * s, .topk = ids.dim(.topk), .out = down.dim(.out) }, .bf16), + }); - const down_bias = - opts.down.bias orelse - Tensor.zeroes(Shape.init(.{ .expert = down.dim(.expert), .out = down.dim(.out) }, .bf16)); - - const down_scale = if (opts.down.quantization) |q| q.scales else Tensor.scalar(1.0, .f32); - - const second_out = callFusedMoe( - activated_quant, - down, - down_bias, - a_scale, - down_scale, - weights, - routing.sorted_token_ids, - expert_ids, - routing.num_tokens_post_padded, - second_cfg, - launch_config, - routing.max_num_tokens_padded, - routing.num_assignments, - Shape.init(.{ .token = b * s, .topk = ids.dim(.topk), .out = down.dim(.out) }, .bf16), - ); - - const output = second_out.sum(.topk).squeeze(.topk); + const output = down_out.sum(.topk).squeeze(.topk); return output.reshape(.{ .b = b, .token = s, .out = down.dim(.out) }); } -fn isMxFp8(quant_scheme: ?zml.Quantization.Scheme) bool { - return quant_scheme != null and quant_scheme.? == .mxfp8; -} - fn applyActivation(x: Tensor, mode: Parameters.ActivationMode, activation_threshold: ?f32) Tensor { const mid = @divFloor(x.dim(.out), 2); var gate = x.slice(.out, .{ .end = mid }); @@ -221,61 +174,55 @@ fn applyActivation(x: Tensor, mode: Parameters.ActivationMode, activation_thresh } /// Build the inputs tuple for FusedMoe and invoke it via `K.call(...)`. -fn callFusedMoe( - a: Tensor, - b: Tensor, - b_bias: Tensor, - a_scale: Tensor, - b_scale: Tensor, - topk_weights: Tensor, - sorted_token_ids: Tensor, +fn callFusedMoe(opts: struct { + input: Tensor, + weight: Tensor, + bias: ?Tensor = null, + input_scale: ?Tensor = null, + weight_scale: ?Tensor = null, + routing_weights: ?Tensor = null, + routing: Routing, expert_ids: Tensor, - num_tokens_post_padded: Tensor, - cfg: kernels.FusedMoe.Cfg, launch_config: LaunchConfig, - max_num_tokens_padded: i64, - num_valid_tokens: i64, + top_k: usize, output_shape: Shape, -) Tensor { - const block_size_m: i64 = @intCast(cfg.block_size_m); - const block_size_n: i64 = @intCast(cfg.block_size_n); - const m_tokens = a.dim(0); +}) Tensor { + const block_size_m: i64 = @intCast(opts.launch_config.block_size_m); + const block_size_n: i64 = @intCast(opts.launch_config.block_size_n); + const m_tokens = opts.input.dim(0); const em_effective = if (m_tokens < block_size_m) - @min(max_num_tokens_padded, num_valid_tokens * block_size_m) + @min(opts.routing.max_num_tokens_padded, opts.routing.num_assignments * block_size_m) else - max_num_tokens_padded; + opts.routing.max_num_tokens_padded; const grid_x = (std.math.divCeil(i64, em_effective, block_size_m) catch unreachable) * - (std.math.divCeil(i64, b.dim(1), block_size_n) catch unreachable); + (std.math.divCeil(i64, opts.weight.dim(1), block_size_n) catch unreachable); - const stride_asm: i64 = if (cfg.b_scale_dtype != null and a_scale.rank() == 2) a_scale.dim(1) else 0; - const stride_ask: i64 = if (cfg.b_scale_dtype != null and a_scale.rank() == 2) 1 else 0; - const stride_bse: i64 = if (cfg.b_scale_dtype != null and b_scale.rank() == 3) - b_scale.dim(1) * b_scale.dim(2) - else - 0; - const stride_bsk: i64 = if (cfg.b_scale_dtype != null and b_scale.rank() == 3) 1 else 0; - const stride_bsn: i64 = if (cfg.b_scale_dtype != null and b_scale.rank() == 3) b_scale.dim(2) else 0; + const stride_asm: i64 = if (opts.input_scale) |scale| (if (scale.rank() == 2) scale.dim(1) else 0) else 0; + const stride_ask: i64 = if (opts.input_scale) |scale| (if (scale.rank() == 2) 1 else 0) else 0; + const stride_bse: i64 = if (opts.weight_scale) |scale| (if (scale.rank() == 3) scale.dim(1) * scale.dim(2) else 0) else 0; + const stride_bsk: i64 = if (opts.weight_scale) |scale| (if (scale.rank() == 3) 1 else 0) else 0; + const stride_bsn: i64 = if (opts.weight_scale) |scale| (if (scale.rank() == 3) scale.dim(2) else 0) else 0; return kernels.FusedMoe.Kernel.call( .{ - .a_ptr = a, - .b_ptr = b, - .b_bias_ptr = b_bias, - .a_scale_ptr = a_scale, - .b_scale_ptr = b_scale, - .topk_weights_ptr = topk_weights, - .sorted_token_ids_ptr = sorted_token_ids, - .expert_ids_ptr = expert_ids, - .num_tokens_post_padded_ptr = num_tokens_post_padded, - .N_ptr = Tensor.constant(.{ .i64 = b.dim(1) }).reshape(.{1}), - .K_ptr = Tensor.constant(.{ .i64 = b.dim(2) }).reshape(.{1}), + .a_ptr = opts.input, + .b_ptr = opts.weight, + .b_bias_ptr = opts.bias orelse Tensor.scalar(0, .f32), + .a_scale_ptr = opts.input_scale orelse Tensor.scalar(1.0, .f32), + .b_scale_ptr = opts.weight_scale orelse Tensor.scalar(1.0, .f32), + .routing_weights_ptr = opts.routing_weights orelse Tensor.scalar(1.0, .f32), + .sorted_token_ids_ptr = opts.routing.sorted_token_ids, + .expert_ids_ptr = opts.expert_ids, + .num_tokens_post_padded_ptr = opts.routing.num_tokens_post_padded, + .N_ptr = Tensor.constant(.{ .i64 = opts.weight.dim(1) }).reshape(.{1}), + .K_ptr = Tensor.constant(.{ .i64 = opts.weight.dim(2) }).reshape(.{1}), .EM_ptr = Tensor.constant(.{ .i64 = em_effective }).reshape(.{1}), - .num_valid_tokens_ptr = Tensor.constant(.{ .i64 = num_valid_tokens }).reshape(.{1}), - .stride_am_ptr = Tensor.constant(.{ .i64 = a.dim(1) }).reshape(.{1}), - .stride_be_ptr = Tensor.constant(.{ .i64 = b.dim(1) * b.dim(2) }).reshape(.{1}), - .stride_bn_ptr = Tensor.constant(.{ .i64 = b.dim(2) }).reshape(.{1}), - .stride_cm_ptr = Tensor.constant(.{ .i64 = b.dim(.out) }).reshape(.{1}), + .num_valid_tokens_ptr = Tensor.constant(.{ .i64 = opts.routing.num_assignments }).reshape(.{1}), + .stride_am_ptr = Tensor.constant(.{ .i64 = opts.input.dim(1) }).reshape(.{1}), + .stride_be_ptr = Tensor.constant(.{ .i64 = opts.weight.dim(1) * opts.weight.dim(2) }).reshape(.{1}), + .stride_bn_ptr = Tensor.constant(.{ .i64 = opts.weight.dim(2) }).reshape(.{1}), + .stride_cm_ptr = Tensor.constant(.{ .i64 = opts.weight.dim(.out) }).reshape(.{1}), .stride_asm_ptr = Tensor.constant(.{ .i64 = stride_asm }).reshape(.{1}), .stride_ask_ptr = Tensor.constant(.{ .i64 = stride_ask }).reshape(.{1}), .stride_bse_ptr = Tensor.constant(.{ .i64 = stride_bse }).reshape(.{1}), @@ -284,12 +231,27 @@ fn callFusedMoe( .stride_bbe_ptr = Tensor.constant(.{ .i64 = 0 }).reshape(.{1}), .stride_bbn_ptr = Tensor.constant(.{ .i64 = 0 }).reshape(.{1}), }, - .{ .c = output_shape }, + .{ .c = opts.output_shape }, .{ - .cfg = cfg, + .cfg = .{ + .a_dtype = toDType(opts.input.dtype()), + .b_dtype = toDType(opts.weight.dtype()), + .c_dtype = toDType(opts.output_shape.dtype()), + .a_scale_dtype = if (opts.input_scale) |scale| toDType(scale.dtype()) else null, + .b_scale_dtype = if (opts.weight_scale) |scale| toDType(scale.dtype()) else null, + .b_bias_dtype = if (opts.bias) |bias| toDType(bias.dtype()) else null, + .routing_weights_dtype = if (opts.routing_weights) |weights| toDType(weights.dtype()) else null, + .block_size_m = opts.launch_config.block_size_m, + .block_size_n = opts.launch_config.block_size_n, + .block_size_k = opts.launch_config.block_size_k, + .group_size_m = opts.launch_config.group_size_m, + .top_k = opts.top_k, + .naive_block_assignment = opts.routing.naive_block_assignment, + .compute_type = .bf16, + }, .grid = .{ @intCast(grid_x), 1, 1 }, - .num_warps = @intCast(launch_config.num_warps), - .num_stages = @intCast(launch_config.num_stages), + .num_warps = opts.launch_config.num_warps, + .num_stages = opts.launch_config.num_stages, }, ).c; } @@ -471,50 +433,13 @@ fn quantizePerTokenGroupFp8(x: Tensor, group_size: i64) struct { Tensor, Tensor // Config / validation helpers // ============================================================================= -fn makeFusedMoeConfig( - a: Tensor, - b: Tensor, - quant_scheme: ?zml.Quantization.Scheme, - launch_config: LaunchConfig, - naive_block_assignment: bool, - top_k: i64, - mul_routed_weight: bool, - has_bias: bool, - output_dtype: DataType, -) kernels.FusedMoe.Cfg { - var use_fp8 = isMxFp8(quant_scheme); - if (b.dtype() == .f8e4m3fn) use_fp8 = true; - return .{ - .a_dtype = toDType(a.dtype()), - .b_dtype = toDType(b.dtype()), - .c_dtype = toDType(output_dtype), - .a_scale_dtype = if (use_fp8) .bf16 else null, - .b_scale_dtype = if (use_fp8) .bf16 else null, - .b_bias_dtype = null, - .topk_weights_dtype = null, - .block_size_m = @intCast(launch_config.block_size_m), - .block_size_n = @intCast(launch_config.block_size_n), - .block_size_k = @intCast(launch_config.block_size_k), - .group_size_m = @intCast(launch_config.group_size_m), - .top_k = @intCast(top_k), - .naive_block_assignment = naive_block_assignment, - .mul_routed_weight = mul_routed_weight, - .compute_type = .bf16, - .use_fp8_w8a8 = use_fp8, - .use_int8_w8a8 = false, - .use_int8_w8a16 = false, - .per_channel_quant = false, - .has_bias = has_bias, - }; -} - const LaunchConfig = struct { - block_size_m: i64, - block_size_n: i64, - block_size_k: i64, - group_size_m: i64, - num_warps: i64, - num_stages: i64, + block_size_m: usize, + block_size_n: usize, + block_size_k: usize, + group_size_m: usize, + num_warps: i32, + num_stages: i32, }; const launch_configs = [_]struct { tokens: i64, config: LaunchConfig }{ @@ -554,8 +479,8 @@ test "MoE launch config selects the nearest token bucket" { try std.testing.expectEqualDeep(launchConfigForTokens(8), launchConfigForTokens(7)); try std.testing.expectEqualDeep(launchConfigForTokens(1), launchConfigForTokens(0)); try std.testing.expectEqualDeep(launchConfigForTokens(4096), launchConfigForTokens(8192)); - try std.testing.expectEqual(@as(i64, 64), launchConfigForTokens(16).group_size_m); - try std.testing.expectEqual(@as(i64, 5), launchConfigForTokens(16).num_stages); + try std.testing.expectEqual(@as(usize, 64), launchConfigForTokens(16).group_size_m); + try std.testing.expectEqual(@as(i32, 5), launchConfigForTokens(16).num_stages); } fn fp8ActivationGroupSize(x: Tensor) i64 { diff --git a/zml/moe/triton_kernels/triton_kernels.zig b/zml/moe/triton_kernels/triton_kernels.zig index cb913f46..28c3b221 100644 --- a/zml/moe/triton_kernels/triton_kernels.zig +++ b/zml/moe/triton_kernels/triton_kernels.zig @@ -13,8 +13,6 @@ const Builder = tri.Builder; const Value = tri.Value; const DType = tri.DType; -const log = std.log.scoped(.moe_triton); - /// Floor to a multiple of 16 — matches the Python `(v // 16) * 16` guards /// that keep dynamic strides aligned for tt.load/store. fn blockFloor16(v: Value) Value { @@ -175,49 +173,37 @@ pub const FusedMoe = struct { a_scale_dtype: ?DType, b_scale_dtype: ?DType, b_bias_dtype: ?DType, - topk_weights_dtype: ?DType, + routing_weights_dtype: ?DType, block_size_m: usize, block_size_n: usize, block_size_k: usize, group_size_m: usize, top_k: usize, naive_block_assignment: bool, - mul_routed_weight: bool, compute_type: DType, - // Validation flags — the kernel rejects configs that set any of - // these to true (the body only implements the bf16 / no-quant / - // no-bias path). - use_fp8_w8a8: bool, - use_int8_w8a8: bool, - use_int8_w8a16: bool, - per_channel_quant: bool, - has_bias: bool, }; pub const Kernel = tri.Kernel(Cfg, .{ .name = "fused_moe_kernel", .inputs = &.{ - "a_ptr", "b_ptr", "b_bias_ptr", "a_scale_ptr", "b_scale_ptr", - "topk_weights_ptr", "sorted_token_ids_ptr", "expert_ids_ptr", "num_tokens_post_padded_ptr", "N_ptr", - "K_ptr", "EM_ptr", "num_valid_tokens_ptr", "stride_am_ptr", "stride_be_ptr", - "stride_bn_ptr", "stride_cm_ptr", "stride_asm_ptr", "stride_ask_ptr", "stride_bse_ptr", - "stride_bsk_ptr", "stride_bsn_ptr", "stride_bbe_ptr", "stride_bbn_ptr", + "a_ptr", "b_ptr", "b_bias_ptr", "a_scale_ptr", "b_scale_ptr", + "routing_weights_ptr", "sorted_token_ids_ptr", "expert_ids_ptr", "num_tokens_post_padded_ptr", "N_ptr", + "K_ptr", "EM_ptr", "num_valid_tokens_ptr", "stride_am_ptr", "stride_be_ptr", + "stride_bn_ptr", "stride_cm_ptr", "stride_asm_ptr", "stride_ask_ptr", "stride_bse_ptr", + "stride_bsk_ptr", "stride_bsn_ptr", "stride_bbe_ptr", "stride_bbn_ptr", }, .outputs = &.{"c"}, .run = run, }); fn run(b: *Builder, cfg: Cfg) tri.FinishError!void { - if (cfg.use_fp8_w8a8 or cfg.use_int8_w8a8 or cfg.use_int8_w8a16 or cfg.has_bias or cfg.per_channel_quant) { - log.err("fused_moe_kernel: unsupported config (fp8/int8/bias/per_channel)", .{}); - return error.InvalidMlir; - } - const a = try b.declareArgs(.{ .a_ptr = .{ .ptr = cfg.a_dtype }, .b_ptr = .{ .ptr = cfg.b_dtype }, - .b_bias_ptr = .{ .ptr = cfg.b_bias_dtype orelse cfg.c_dtype }, + // The fixed signature requires concrete pointer types even for absent, + // unused inputs. FP32 matches the caller's placeholder tensors. + .b_bias_ptr = .{ .ptr = cfg.b_bias_dtype orelse .f32 }, .a_scale_ptr = .{ .ptr = cfg.a_scale_dtype orelse .f32 }, .b_scale_ptr = .{ .ptr = cfg.b_scale_dtype orelse .f32 }, - .topk_weights_ptr = .{ .ptr = cfg.topk_weights_dtype orelse .f32 }, + .routing_weights_ptr = .{ .ptr = cfg.routing_weights_dtype orelse .f32 }, .sorted_token_ids_ptr = .{ .ptr = .i32 }, .expert_ids_ptr = .{ .ptr = .i32 }, .num_tokens_post_padded_ptr = .{ .ptr = .i32 }, @@ -411,11 +397,10 @@ pub const FusedMoe = struct { } var accumulator = loop.results[2]; - if (cfg.mul_routed_weight) { - const tw_dtype = cfg.topk_weights_dtype orelse .f32; - const tw_other = b.zeros(&.{block_size_m}, tw_dtype); - const tw = b.loadOpts(a.topk_weights_ptr.addPtr(offs_token), .{ .mask = token_mask, .other = tw_other }); - accumulator = accumulator.mul(tw.expandDims(1)); + if (cfg.routing_weights_dtype) |dtype| { + const other = b.zeros(&.{block_size_m}, dtype); + const routing_weights = b.loadOpts(a.routing_weights_ptr.addPtr(offs_token), .{ .mask = token_mask, .other = other }); + accumulator = accumulator.mul(routing_weights.to(.f32).expandDims(1)); } accumulator = accumulator.to(compute_type); diff --git a/zml/nn.zig b/zml/nn.zig index 7d76cf54..8d86f816 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -39,6 +39,10 @@ pub const Linear = struct { return if (self.quantization) |q| q.scheme else null; } + pub fn quantizationScales(self: Linear) ?Tensor { + return if (self.quantization) |q| q.scales else null; + } + pub fn forward(self: Linear, x: Tensor) Tensor { if (self.quantization) |q| { const lhs = x.convert(.bf16); From 432609ed2224d87671d50a9582cb1390c5b9c1dd Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 11 Sep 2026 11:21:20 +0200 Subject: [PATCH 675/881] zml/triton: wire TMA and global scratch memory size in Triton kernels (#797) XLA supports TMA and scratch memory in the triton custom call but we never wired it. It is now solved. --- zml/kernel.zig | 18 +++++++++++++++--- zml/ops.zig | 4 ++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/zml/kernel.zig b/zml/kernel.zig index 07d7f21b..67cfefaf 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -113,6 +113,8 @@ pub const triton = struct { num_warps: i32, output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, debug: bool = false, + /// Bytes for Triton's implicit global scratch argument, across all CTAs. + global_scratch_memory_size: i32 = 0, }; pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { @@ -142,7 +144,7 @@ pub const triton = struct { const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - const tensor_results = ops.triton(inputs_arr, outputs_arr, .{ + const call_opts: ops.TritonOps = .{ .debug = opts.debug, .name = name, .ir = ttir, @@ -150,10 +152,20 @@ pub const triton = struct { .num_stages = opts.num_stages, .num_warps = opts.num_warps, .output_operand_aliases = aliases.constSlice(), - }); + .is_tma_allowed = opts.global_scratch_memory_size > 0, + .global_scratch_memory_size = opts.global_scratch_memory_size, + }; + + const triton_results = if (opts.global_scratch_memory_size > 0) blk: { + // XLA maps the final custom-call result to Triton's implicit + // scratch pointer; it must not appear in the TTIR signature. + const extended = outputs_arr ++ [_]Shape{.init(.{opts.global_scratch_memory_size}, .u8)}; + const res = ops.triton(inputs_arr, extended, call_opts); + break :blk res[0..spec.outputs.len].*; + } else ops.triton(inputs_arr, outputs_arr, call_opts); var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = triton_results[i]; return results; } }; diff --git a/zml/ops.zig b/zml/ops.zig index 8a095cce..33f241f5 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -861,6 +861,8 @@ pub const TritonOps = struct { grid: [3]i32, num_stages: i32, num_warps: i32, + is_tma_allowed: bool = false, + global_scratch_memory_size: i32 = 0, output_operand_aliases: []const dialects.stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, }; @@ -888,6 +890,8 @@ pub fn triton(inputs: anytype, outputs: anytype, opts: TritonOps) [outputs.len]T .named(mlir_ctx, "grid_z", .int(mlir_ctx, .i32, opts.grid[2])), .named(mlir_ctx, "num_stages", .int(mlir_ctx, .i32, opts.num_stages)), .named(mlir_ctx, "num_warps", .int(mlir_ctx, .i32, opts.num_warps)), + .named(mlir_ctx, "is_tma_allowed", .boolean(mlir_ctx, opts.is_tma_allowed)), + .named(mlir_ctx, "global_scratch_memory_size", .int(mlir_ctx, .i32, opts.global_scratch_memory_size)), }); var operands_layouts: [inputs.len][]const usize = undefined; From 46bc6df68496db36e17f3698a69a7c54b5c3adc3 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Fri, 11 Sep 2026 12:55:59 +0200 Subject: [PATCH 676/881] zml/moe: proper FP8 scaling and bias in triton experts (#793) - The current Fp8 scaling wasn't properly finished. We didn't use the scales at all. - Adding a `quantize_input` option that GLM5.3-Flash uses - `applyActivation` wasn't properly using the threshold for relu - The bias was silently ignored for the MoE. --- examples/llm/models/qwen3_5_moe/model.zig | 2 +- kernels/triton/builder.zig | 9 +- zml/BUILD.bazel | 2 +- zml/moe/moe.zig | 11 +- zml/moe/triton.zig | 361 ++++++++++++++++++---- zml/moe/triton_kernels/triton_kernels.zig | 222 ++++++++++++- 6 files changed, 521 insertions(+), 86 deletions(-) diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index f947bb96..84ac6741 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -777,7 +777,7 @@ pub const Moe = struct { routing_scores, self.gate_up_proj, self.down_proj, - .{}, + .{ .quantize_input = false }, moe_parameters, ) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); diff --git a/kernels/triton/builder.zig b/kernels/triton/builder.zig index a1cbf5b1..5ec89cd8 100644 --- a/kernels/triton/builder.zig +++ b/kernels/triton/builder.zig @@ -1318,11 +1318,16 @@ pub const Builder = struct { if (cur_is_float and tgt_is_float) { const cur_bw = dtypeBitwidth(cur_dtype); const tgt_bw = dtypeBitwidth(dtype); - if (tgt_bw > cur_bw) return self.extf(src, dtype); const fp8_involved = (cur_dtype == .f8e4m3fn or cur_dtype == .f8e4m3fnuz or cur_dtype == .f8e5m2 or dtype == .f8e4m3fn or dtype == .f8e4m3fnuz or dtype == .f8e5m2); + if (fp8_involved) { + return self.fpToFpOpts(src, dtype, .{ + .rounding = if (tgt_bw < cur_bw) opts.fp_downcast_rounding orelse .rtne else null, + }); + } + if (tgt_bw > cur_bw) return self.extf(src, dtype); const custom_rounding = opts.fp_downcast_rounding != null and opts.fp_downcast_rounding.? != .rtne; - if (fp8_involved or custom_rounding) { + if (custom_rounding) { return self.fpToFpOpts(src, dtype, .{ .rounding = opts.fp_downcast_rounding orelse .rtne }); } return self.truncf(src, dtype); diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 402fb737..e1803916 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -76,6 +76,7 @@ zig_library( zig_library( name = "zml", srcs = [ + "Compiler.zig", "Sharding.zig", "attention.zig", "attention/attnd.zig", @@ -99,7 +100,6 @@ zig_library( "mem.zig", "meta.zig", "mlirx.zig", - "Compiler.zig", "moe/cutlass_flashinfer.zig", "moe/metal.zig", "moe/moe.zig", diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index 2c422dd0..10aa6247 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -37,10 +37,14 @@ pub const Backend = enum { .flashinfer_cutlass else return error.UnsupportedDataType, - .f8e8m0, .f16, .f32 => .triton, + .f8e4m3fn, .f8e4m3fnuz, .f8e8m0, .f16, .f32 => .triton, else => error.UnsupportedDataType, }, - .rocm, .oneapi => switch (weights_dtype) { + .rocm => switch (weights_dtype) { + .bf16, .f16, .f32, .f8e4m3fn, .f8e4m3fnuz, .f8e8m0 => .triton, + else => error.UnsupportedDataType, + }, + .oneapi => switch (weights_dtype) { .bf16, .f16, .f32 => .triton, else => error.UnsupportedDataType, }, @@ -140,6 +144,8 @@ pub const Parameters = union(Backend) { pub const Options = struct { activation_threshold: ?f32 = null, + /// Quantize activations for Triton FP8 GEMMs; false keeps BF16 activations. + quantize_input: bool, }; pub fn forwardMoe( @@ -379,6 +385,7 @@ pub fn forwardMoe( .topk_ids = topk_ids, .activation = parameters.triton.activation, .activation_threshold = opts.activation_threshold, + .quantize_input = opts.quantize_input, }; const expert_partition = gate_up.weight.shape().partition(.expert); diff --git a/zml/moe/triton.zig b/zml/moe/triton.zig index 8ca30b76..6d6495f4 100644 --- a/zml/moe/triton.zig +++ b/zml/moe/triton.zig @@ -7,7 +7,6 @@ const DataType = zml.DataType; const Tensor = zml.Tensor; const Shape = zml.Shape; const tri = zml.kernel.triton; -const DType = tri.DType; const toDType = tri.from; const a16w4_kernel = @import("triton_kernels/a16w4_kernel.zig"); const kernels = @import("triton_kernels/triton_kernels.zig"); @@ -50,38 +49,54 @@ pub const FusedExpertsArgs = struct { activation: Parameters.ActivationMode = .silu, expert_map: ?Tensor = null, activation_threshold: ?f32 = null, + /// Use FP8 activations for FP8 weights; false keeps BF16 activations. + quantize_input: bool, }; pub fn fusedExpertsImpl(opts: FusedExpertsArgs) !Tensor { const hidden_states = opts.hidden_states; const topk_weights = opts.topk_weights; const topk_ids = opts.topk_ids; - - if (opts.gate_up.quantizationScheme() != null and opts.gate_up.quantizationScheme().? == .mxfp4) { - const local_topk_ids, const local_topk_weights = if (opts.expert_map) |expert_map| blk: { - const local_num_experts = opts.gate_up.weight.dim(.expert); - const mapped_ids = expert_map - .gather(.{ .expert = topk_ids.convert(.i32) }, .{}) - .withTags(topk_ids.shape().tags()); - const in_range = mapped_ids.cmp(.GE, Tensor.scalar(0, .i32)) - .logical(.AND, mapped_ids.cmp(.LT, Tensor.scalar(local_num_experts, .i32))); - - break :blk .{ - in_range.select(mapped_ids, Tensor.scalar(local_num_experts, .i32)), - in_range.select(topk_weights, Tensor.scalar(0, topk_weights.dtype())), - }; - } else .{ topk_ids, topk_weights }; - - var local_args = opts; - local_args.topk_ids = local_topk_ids; - local_args.topk_weights = local_topk_weights; - return fusedExpertsImpl_fp4(local_args); - } - const b = hidden_states.dim(.b); const s = hidden_states.dim(.s); const num_tokens = b * s; - const launch_config = launchConfigForTokens(num_tokens); + var launch_config = launchConfigForTokens(num_tokens); + + const gate_up_scheme = opts.gate_up.quantizationScheme(); + const down_scheme = opts.down.quantizationScheme(); + if (gate_up_scheme) |scheme| switch (scheme) { + .mxfp4 => { + if (down_scheme != .mxfp4) return error.UnsupportedQuantization; + const local_topk_ids, const local_topk_weights = if (opts.expert_map) |expert_map| blk: { + const local_num_experts = opts.gate_up.weight.dim(.expert); + const mapped_ids = expert_map + .gather(.{ .expert = topk_ids.convert(.i32) }, .{}) + .withTags(topk_ids.shape().tags()); + const in_range = mapped_ids.cmp(.GE, Tensor.scalar(0, .i32)) + .logical(.AND, mapped_ids.cmp(.LT, Tensor.scalar(local_num_experts, .i32))); + + break :blk .{ + in_range.select(mapped_ids, Tensor.scalar(local_num_experts, .i32)), + in_range.select(topk_weights, Tensor.scalar(0, topk_weights.dtype())), + }; + } else .{ topk_ids, topk_weights }; + + var local_args = opts; + local_args.topk_ids = local_topk_ids; + local_args.topk_weights = local_topk_weights; + return fusedExpertsImpl_fp4(local_args); + }, + .nvfp4 => return error.UnsupportedQuantization, + .fp8_block128 => launch_config.block_size_k = 128, + .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, + }; + + var down_launch_config = launchConfigForTokens(num_tokens); + if (down_scheme) |scheme| switch (scheme) { + .fp8_block128 => down_launch_config.block_size_k = 128, + .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, + .mxfp4, .nvfp4 => return error.UnsupportedQuantization, + }; const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); const gate_up = opts.gate_up.weight.withTags(.{ .expert, .out, .in }); @@ -90,13 +105,14 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) !Tensor { const ids = topk_ids.reshape(.{ .token = num_tokens, .in = topk_ids.dim(.top_expert) }).withTags(.{ .token, .topk }); stdx.debug.assert(hidden.dtype() == .bf16, "expected BF16 hidden states, got {}", .{hidden.dtype()}); - stdx.debug.assert(gate_up.dtype() == .bf16 or gate_up.dtype() == .f8e4m3fn, "expected BF16 or FP8 E4M3FN gate/up weights, got {}", .{gate_up.dtype()}); - stdx.debug.assert(down.dtype() == .bf16 or down.dtype() == .f8e4m3fn, "expected BF16 or FP8 E4M3FN down weights, got {}", .{down.dtype()}); + stdx.debug.assert(gate_up.dtype() == .bf16 or gate_up.dtype() == .f8e4m3fn or gate_up.dtype() == .f8e4m3fnuz, "expected BF16 or FP8 E4M3FN/FNUZ gate/up weights, got {}", .{gate_up.dtype()}); + stdx.debug.assert(down.dtype() == .bf16 or down.dtype() == .f8e4m3fn or down.dtype() == .f8e4m3fnuz, "expected BF16 or FP8 E4M3FN/FNUZ down weights, got {}", .{down.dtype()}); stdx.debug.assert(routing_weights.dtype() == .f32 or routing_weights.dtype() == .bf16, "expected FP32 or BF16 routing weights, got {}", .{routing_weights.dtype()}); stdx.debug.assert(ids.dtype() == .i32, "expected I32 expert ids, got {}", .{ids.dtype()}); stdx.debug.assert(hidden.dim(.in) == gate_up.dim(.in), "hidden width {} must match gate/up input width {}", .{ hidden.dim(.in), gate_up.dim(.in) }); - stdx.debug.assert(@rem(gate_up.dim(.out), 2) == 0, "gate/up output width must be even, got {}", .{gate_up.dim(.out)}); - stdx.debug.assert(down.dim(.mid) == @divFloor(gate_up.dim(.out), 2), "down input width {} must equal half the gate/up output width {}", .{ down.dim(.mid), gate_up.dim(.out) }); + const activation_reduction: i64 = if (opts.activation == .relu) 1 else 2; + stdx.debug.assert(@rem(gate_up.dim(.out), activation_reduction) == 0, "gate/up output width {} must be divisible by {}", .{ gate_up.dim(.out), activation_reduction }); + stdx.debug.assert(down.dim(.mid) == @divFloor(gate_up.dim(.out), activation_reduction), "down input width {} must match activated width {}", .{ down.dim(.mid), @divFloor(gate_up.dim(.out), activation_reduction) }); stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "routing ids and weights must match hidden token count {}, got {} and {}", .{ hidden.dim(.token), ids.dim(.token), routing_weights.dim(.token) }); stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "routing ids and weights must have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), routing_weights.dim(.topk) }); stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "gate/up and down expert counts must match, got {} and {}", .{ gate_up.dim(.expert), down.dim(.expert) }); @@ -114,8 +130,8 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) !Tensor { var hidden_quant = hidden; var input_scale: ?Tensor = null; - if (gate_up.dtype() == .f8e4m3fn) { - hidden_quant, input_scale = quantizePerTokenGroupFp8(hidden, fp8ActivationGroupSize(hidden)); + if (opts.quantize_input) { + if (gate_up_scheme) |scheme| hidden_quant, input_scale = quantizeFp8Input(hidden, scheme, gate_up.dtype()); } const gate_up_out = callFusedMoe(.{ @@ -127,15 +143,16 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) !Tensor { .routing = routing, .expert_ids = expert_ids, .launch_config = launch_config, + .quant_scheme = gate_up_scheme, .top_k = @intCast(ids.dim(.topk)), .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), }); - const activated = applyActivation(gate_up_out, opts.activation, opts.activation_threshold); - var activated_quant = activated; + const activated = applyExpertActivation(gate_up_out, opts.activation, opts.activation_threshold); + var activated_quant = activated.convert(.bf16); input_scale = null; - if (down.dtype() == .f8e4m3fn) { - activated_quant, input_scale = quantizePerTokenGroupFp8(activated, fp8ActivationGroupSize(activated)); + if (opts.quantize_input) { + if (down_scheme) |scheme| activated_quant, input_scale = quantizeFp8Input(activated, scheme, down.dtype()); } const down_out = callFusedMoe(.{ @@ -147,17 +164,24 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) !Tensor { .routing_weights = routing_weights, .routing = routing, .expert_ids = expert_ids, - .launch_config = launch_config, + .launch_config = down_launch_config, + .quant_scheme = down_scheme, .top_k = 1, .output_shape = Shape.init(.{ .token = b * s, .topk = ids.dim(.topk), .out = down.dim(.out) }, .bf16), }); - const output = down_out.sum(.topk).squeeze(.topk); + const output = down_out.convert(.f32).sum(.topk).squeeze(.topk).convert(.bf16); return output.reshape(.{ .b = b, .token = s, .out = down.dim(.out) }); } -fn applyActivation(x: Tensor, mode: Parameters.ActivationMode, activation_threshold: ?f32) Tensor { +fn applyExpertActivation(input: Tensor, mode: Parameters.ActivationMode, activation_threshold: ?f32) Tensor { + const x = input.convert(.f32); + if (mode == .relu) { + const clipped = if (activation_threshold) |limit| x.minimum(Tensor.scalar(limit, x.dtype())) else x; + return clipped.relu().powByConst(2); + } + const mid = @divFloor(x.dim(.out), 2); var gate = x.slice(.out, .{ .end = mid }); var up = x.slice(.out, .{ .start = mid }); @@ -168,11 +192,72 @@ fn applyActivation(x: Tensor, mode: Parameters.ActivationMode, activation_thresh } return switch (mode) { .silu => gate.silu().mul(up), - .relu => x.relu().powByConst(2), + .relu => unreachable, .gelu => gate.gelu().mul(up), }; } +test "SwiGLU uses FP32 math for BF16 inputs" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + const Local = struct { + fn forward(x: Tensor, threshold: ?f32) Tensor { + return applyExpertActivation(x, .silu, threshold); + } + }; + const x: Tensor = .init(.{ .token = 1, .out = 6 }, .bf16); + const gates = [_]f32{ 0.75, -1.25, 3.5 }; + const ups = [_]f32{ 0.875, -2.25, 4.5 }; + var values: [6]zml.floats.BFloat16 = undefined; + for (gates, ups, 0..) |gate, up, i| { + values[i] = .fromF32(gate); + values[i + 3] = .fromF32(up); + } + var input = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&values)); + defer input.deinit(); + for ([_]?f32{ null, 2 }) |threshold| { + var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, threshold }, .{}); + defer exe.deinit(); + try zml.testing.expectEqualShapes(Shape.init(.{ .token = 1, .out = 3 }, .f32), exe.output_shapes[0]); + var output = try zml.testing.autoCall(allocator, io, &exe, Local.forward, .{input}); + defer output.deinit(); + var actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + for (gates, ups, actual.constItems(f32)) |gate, up, value| { + const g = if (threshold) |limit| @min(gate, limit) else gate; + const u = if (threshold) |limit| std.math.clamp(up, -limit, limit) else up; + try std.testing.expectApproxEqAbs(g / (1 + @exp(-g)) * u, value, 1e-6); + } + } +} + +test "ReLU squared activation preserves width and applies threshold before squaring" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + const Local = struct { + fn forward(x: Tensor, threshold: ?f32) Tensor { + return applyExpertActivation(x, .relu, threshold); + } + }; + const x: Tensor = .init(.{ .token = 1, .out = 5 }, .f32); + const values = [_]f32{ -3, 0, 1, 2, 4 }; + var input = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&values)); + defer input.deinit(); + for ([_]?f32{ null, 2 }) |threshold| { + var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, threshold }, .{}); + defer exe.deinit(); + try zml.testing.expectEqualShapes(x.shape(), exe.output_shapes[0]); + var output = try zml.testing.autoCall(allocator, io, &exe, Local.forward, .{input}); + defer output.deinit(); + var actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + const expected = [_]f32{ 0, 0, 1, 4, if (threshold != null) 4 else 16 }; + try std.testing.expectEqualSlices(f32, &expected, actual.constItems(f32)); + } +} + /// Build the inputs tuple for FusedMoe and invoke it via `K.call(...)`. fn callFusedMoe(opts: struct { input: Tensor, @@ -184,9 +269,14 @@ fn callFusedMoe(opts: struct { routing: Routing, expert_ids: Tensor, launch_config: LaunchConfig, + quant_scheme: ?zml.Quantization.Scheme = null, top_k: usize, output_shape: Shape, }) Tensor { + stdx.debug.assert(opts.quant_scheme != null or (opts.input_scale == null and opts.weight_scale == null), "scales require a quantization scheme", .{}); + for ([_]i64{ opts.weight.dim(1), opts.weight.dim(2), opts.input.dim(1), opts.weight.dim(1) * opts.weight.dim(2), opts.output_shape.dim(-1) }) |dim_or_stride| { + stdx.debug.assert(dim_or_stride > 0 and @mod(dim_or_stride, 16) == 0, "FusedMoe dimensions and matrix strides must be positive multiples of 16, got {}", .{dim_or_stride}); + } const block_size_m: i64 = @intCast(opts.launch_config.block_size_m); const block_size_n: i64 = @intCast(opts.launch_config.block_size_n); const m_tokens = opts.input.dim(0); @@ -194,15 +284,31 @@ fn callFusedMoe(opts: struct { @min(opts.routing.max_num_tokens_padded, opts.routing.num_assignments * block_size_m) else opts.routing.max_num_tokens_padded; + stdx.debug.assert(@mod(em_effective, block_size_m) == 0, "routing capacity {} must be a multiple of block size {}", .{ em_effective, block_size_m }); const grid_x = - (std.math.divCeil(i64, em_effective, block_size_m) catch unreachable) * + @divExact(em_effective, block_size_m) * (std.math.divCeil(i64, opts.weight.dim(1), block_size_n) catch unreachable); + const weight_scale: ?Tensor = if (opts.weight_scale) |scale| blk: { + const scheme = opts.quant_scheme orelse break :blk scale; + break :blk switch (scheme) { + // dot_scaled takes E8M0 encodings as bytes, including native E8M0 tensors. + .mxfp8 => scale.bitCast(.u8), + .fp8_per_channel => scale.reshape(.{ opts.weight.dim(0), opts.weight.dim(1), 1 }), + .fp8_per_tensor => if (scale.count() == 1) scale else scale.reshape(.{ opts.weight.dim(0), 1, 1 }), + .fp8_block128, .mxfp4, .nvfp4 => scale, + }; + } else null; + const stride_asm: i64 = if (opts.input_scale) |scale| (if (scale.rank() == 2) scale.dim(1) else 0) else 0; const stride_ask: i64 = if (opts.input_scale) |scale| (if (scale.rank() == 2) 1 else 0) else 0; - const stride_bse: i64 = if (opts.weight_scale) |scale| (if (scale.rank() == 3) scale.dim(1) * scale.dim(2) else 0) else 0; - const stride_bsk: i64 = if (opts.weight_scale) |scale| (if (scale.rank() == 3) 1 else 0) else 0; - const stride_bsn: i64 = if (opts.weight_scale) |scale| (if (scale.rank() == 3) scale.dim(2) else 0) else 0; + const has_weight_scale_expert_stride = if (weight_scale) |scale| + !(opts.quant_scheme == .fp8_per_tensor and scale.count() == 1) + else + false; + const stride_bse: i64 = if (weight_scale) |scale| (if (scale.rank() == 3 and has_weight_scale_expert_stride) scale.dim(1) * scale.dim(2) else 0) else 0; + const stride_bsk: i64 = if (weight_scale) |scale| (if (scale.rank() == 3) 1 else 0) else 0; + const stride_bsn: i64 = if (weight_scale) |scale| (if (scale.rank() == 3) scale.dim(2) else 0) else 0; return kernels.FusedMoe.Kernel.call( .{ @@ -210,7 +316,7 @@ fn callFusedMoe(opts: struct { .b_ptr = opts.weight, .b_bias_ptr = opts.bias orelse Tensor.scalar(0, .f32), .a_scale_ptr = opts.input_scale orelse Tensor.scalar(1.0, .f32), - .b_scale_ptr = opts.weight_scale orelse Tensor.scalar(1.0, .f32), + .b_scale_ptr = weight_scale orelse Tensor.scalar(1.0, .f32), .routing_weights_ptr = opts.routing_weights orelse Tensor.scalar(1.0, .f32), .sorted_token_ids_ptr = opts.routing.sorted_token_ids, .expert_ids_ptr = opts.expert_ids, @@ -228,8 +334,8 @@ fn callFusedMoe(opts: struct { .stride_bse_ptr = Tensor.constant(.{ .i64 = stride_bse }).reshape(.{1}), .stride_bsk_ptr = Tensor.constant(.{ .i64 = stride_bsk }).reshape(.{1}), .stride_bsn_ptr = Tensor.constant(.{ .i64 = stride_bsn }).reshape(.{1}), - .stride_bbe_ptr = Tensor.constant(.{ .i64 = 0 }).reshape(.{1}), - .stride_bbn_ptr = Tensor.constant(.{ .i64 = 0 }).reshape(.{1}), + .stride_bbe_ptr = Tensor.constant(.{ .i64 = if (opts.bias) |bias| (if (bias.rank() == 2 and bias.dim(0) > 1) bias.dim(1) else 0) else 0 }).reshape(.{1}), + .stride_bbn_ptr = Tensor.constant(.{ .i64 = if (opts.bias) |bias| (if (bias.rank() > 0 and bias.dim(-1) > 1) 1 else 0) else 0 }).reshape(.{1}), }, .{ .c = opts.output_shape }, .{ @@ -238,7 +344,7 @@ fn callFusedMoe(opts: struct { .b_dtype = toDType(opts.weight.dtype()), .c_dtype = toDType(opts.output_shape.dtype()), .a_scale_dtype = if (opts.input_scale) |scale| toDType(scale.dtype()) else null, - .b_scale_dtype = if (opts.weight_scale) |scale| toDType(scale.dtype()) else null, + .b_scale_dtype = if (weight_scale) |scale| toDType(scale.dtype()) else null, .b_bias_dtype = if (opts.bias) |bias| toDType(bias.dtype()) else null, .routing_weights_dtype = if (opts.routing_weights) |weights| toDType(weights.dtype()) else null, .block_size_m = opts.launch_config.block_size_m, @@ -248,6 +354,7 @@ fn callFusedMoe(opts: struct { .top_k = opts.top_k, .naive_block_assignment = opts.routing.naive_block_assignment, .compute_type = .bf16, + .quant_scheme = opts.quant_scheme, }, .grid = .{ @intCast(grid_x), 1, 1 }, .num_warps = opts.launch_config.num_warps, @@ -256,6 +363,131 @@ fn callFusedMoe(opts: struct { ).c; } +test "FP8 routed GEMM with bias matches dequantized weights" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + if (platform.target != .cuda) return error.SkipZigTest; + + const Local = struct { + const Outputs = struct { actual: Tensor, expected: Tensor }; + + fn expandScales(scales: Tensor, shape: Shape) Tensor { + var expanded = scales.convert(.f32); + for (0..shape.rank()) |axis| { + const factor = @divExact(shape.dim(axis), expanded.dim(axis)); + const broad_shape = expanded.shape().insert(axis + 1, .{factor}); + const axes = Shape.range(broad_shape.rank(), .i64).remove(axis + 1); + expanded = expanded.broadcast(broad_shape, axes.dims()) + .reshape(expanded.shape().setDim(axis, shape.dim(axis))); + } + return expanded.withTags(shape.tags()); + } + + fn forward(x: Tensor, w: Tensor, scales: Tensor, ids: Tensor, weights: Tensor, bias: Tensor, scheme: zml.Quantization.Scheme, mx_native: bool, quantize_input: bool) Outputs { + const weight = w.convert(.f8e4m3fn); + var input = x; + var input_scale: ?Tensor = null; + var reference_input = x.convert(.f32); + if (quantize_input) { + input, input_scale = quantizeFp8Input(x, scheme, .f8e4m3fn); + const scale = if (scheme == .mxfp8) input_scale.?.bitCast(.f8e8m0) else input_scale.?; + reference_input = input.convert(.f32).withTags(x.shape().tags()) + .mul(expandScales(scale.convert(.f32), x.shape())); + } + const weight_scale = if (scheme == .mxfp8) blk: { + const native = scales.convert(.f8e8m0); + break :blk if (mx_native) native else native.bitCast(.u8); + } else scales; + const config = launchConfigForTokens(8); + const routing = prepareRouting(ids, w.dim(.expert), @intCast(config.block_size_m)); + const actual = callFusedMoe(.{ + .input = input, + .weight = weight, + .bias = bias, + .input_scale = input_scale, + .weight_scale = weight_scale, + .routing_weights = weights, + .routing = routing, + .expert_ids = routing.expert_ids, + .launch_config = config, + .quant_scheme = scheme, + .top_k = 1, + .output_shape = Shape.init(.{ .token = x.dim(.token), .topk = 1, .out = w.dim(.out) }, .bf16), + }).squeeze(.topk); + const expanded_weight = weight.convert(.f32).mul(expandScales(scales, weight.shape())); + const selected_weight = expanded_weight.gather(.{ .expert = ids.squeeze(.topk) }, .{}); + const dot = reference_input.dot(selected_weight, .in); + const selected_bias = if (bias.rank() == 2) + bias.gather(.{ .expert = ids.squeeze(.topk) }, .{}) + else + bias; + const unweighted = dot.add(selected_bias.convert(.f32).broad(dot.shape())); + const expected = unweighted.mul(weights.squeeze(.topk).convert(.f32).broad(unweighted.shape())).convert(.bf16); + return .{ .actual = actual, .expected = expected }; + } + }; + + for ([_]zml.Quantization.Scheme{ .mxfp8, .fp8_per_tensor, .fp8_per_channel, .fp8_block128 }) |scheme| { + for ([_]bool{ false, true }) |quantize_input| { + for ([_]i64{ 1, 17 }) |tokens| { + const n: i64 = if (scheme == .fp8_block128) 256 else 48; + const scale_dims: [3]i64 = switch (scheme) { + .mxfp8 => .{ 8, n, 8 }, + .fp8_per_tensor => if (tokens == 1) .{ 1, 1, 1 } else .{ 8, 1, 1 }, + .fp8_per_channel => .{ 8, n, 1 }, + .fp8_block128 => .{ 8, 2, 2 }, + else => unreachable, + }; + const x: Tensor = .init(.{ .token = tokens, .in = 256 }, .bf16); + const w: Tensor = .init(.{ .expert = 8, .out = n, .in = 256 }, .bf16); + const scales: Tensor = .init(.{ scale_dims[0], scale_dims[1], scale_dims[2] }, .f32); + const ids: Tensor = .init(.{ .token = tokens, .topk = 1 }, .i32); + const weights: Tensor = .init(.{ .token = tokens, .topk = 1 }, .bf16); + const bias: Tensor = if (tokens == 1) + .init(.{ .out = n }, .bf16) + else + .init(.{ .expert = 8, .out = n }, .f32); + var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, w, scales, ids, weights, bias, scheme, tokens == 1, quantize_input }, .{}); + defer exe.deinit(); + var buffers: [6]zml.Buffer = undefined; + var initialized: usize = 0; + defer for (buffers[0..initialized]) |*buffer| buffer.deinit(); + for ([_]Shape{ x.shape(), w.shape(), scales.shape(), ids.shape(), weights.shape(), bias.shape() }, 0..) |shape, i| { + const slice = try zml.Slice.alloc(allocator, shape); + defer slice.free(allocator); + for (0..shape.count()) |j| { + switch (i) { + 2 => slice.items(f32)[j] = @as(f32, @floatFromInt(@as(u32, 1) << @as(u5, @intCast(j % 4)))) / 16, + 3 => slice.items(i32)[j] = @intCast((j * 3 + 5) % 8), + 5 => { + const value = @as(f32, @floatFromInt(@as(i32, @intCast(j % 17)) - 8)) / 2; + if (shape.dtype() == .f32) { + slice.items(f32)[j] = value; + } else { + slice.items(zml.floats.BFloat16)[j] = .fromF32(value); + } + }, + else => slice.items(zml.floats.BFloat16)[j] = .fromF32(if (i == 4) + @as(f32, @floatFromInt(1 + j % 3)) / 4 + else + @as(f32, @floatFromInt(@as(i32, @intCast((j * 7 + j / 256) % 13)) - 6)) / 4), + } + } + buffers[i] = try zml.Buffer.fromSlice(io, platform, slice, .replicated); + initialized += 1; + } + var output = try zml.testing.autoCall(allocator, io, &exe, Local.forward, .{ buffers[0], buffers[1], buffers[2], buffers[3], buffers[4], buffers[5] }); + defer zml.Buffer.deinitAll(Local.Outputs, &output); + zml.testing.expectClose(io, output.expected, output.actual, .{ .absolute_tolerance = 0.03125, .relative_tolerance = 0.01 }) catch |err| { + log.warn("FP8 routed GEMM failed for scheme={s}, quantize_input={}, tokens={}", .{ @tagName(scheme), quantize_input, tokens }); + return err; + }; + } + } + } +} + const Routing = struct { sorted_token_ids: Tensor, expert_ids: Tensor, @@ -266,6 +498,7 @@ const Routing = struct { }; fn prepareRouting(topk_ids: Tensor, num_experts: i64, block_size_m: i64) Routing { + stdx.debug.assert(block_size_m > 0 and @mod(block_size_m, 16) == 0, "routing block size must be a positive multiple of 16, got {}", .{block_size_m}); const ids = topk_ids.withTags(.{ .token, .topk }).convert(.i32); const num_assignments = ids.dim(.token) * ids.dim(.topk); const sparsity_factor: i64 = 4; @@ -275,7 +508,7 @@ fn prepareRouting(topk_ids: Tensor, num_experts: i64, block_size_m: i64) Routing else if (num_assignments < num_experts) num_assignments * block_size_m else - num_assignments + num_experts * (block_size_m - 1); + (std.math.divCeil(i64, num_assignments + num_experts * (block_size_m - 1), block_size_m) catch unreachable) * block_size_m; const sorted_token_ids, const expert_ids, const num_tokens_post_padded = if (naive_block_assignment) blk: { log.debug("Using naive block assignment for MoE kernels. Num assignments: {d}, Num experts: {d}", .{ num_assignments, num_experts }); @@ -305,8 +538,8 @@ fn alignBlockSize(topk_ids: Tensor, num_experts: i64, block_size_m: i64) struct const max_num_tokens_padded = if (num_assignments < num_experts) num_assignments * block_size_m else - num_assignments + num_experts * (block_size_m - 1); - const max_num_m_blocks = std.math.divCeil(i64, max_num_tokens_padded, block_size_m) catch unreachable; + (std.math.divCeil(i64, num_assignments + num_experts * (block_size_m - 1), block_size_m) catch unreachable) * block_size_m; + const max_num_m_blocks = @divExact(max_num_tokens_padded, block_size_m); const warp_size: i64 = 32; const padded_num_experts: i64 = @intCast(std.math.ceilPowerOfTwoAssert(u64, @intCast(@max(num_experts, warp_size)))); const sort_block_size: i64 = 256; @@ -393,13 +626,19 @@ fn alignBlockSize(topk_ids: Tensor, num_experts: i64, block_size_m: i64) struct return .{ sorted_token_ids, expert_ids, num_tokens_post_padded }; } -fn quantizePerTokenGroupFp8(x: Tensor, group_size: i64) struct { Tensor, Tensor } { +fn quantizeFp8Input(x: Tensor, scheme: zml.Quantization.Scheme, output_dtype: DataType) struct { Tensor, Tensor } { + const group_size: i64, const scale_dtype: DataType, const absmax_epsilon: f32 = switch (scheme) { + .mxfp8 => .{ 32, .u8, 1e-10 }, + .fp8_per_channel, .fp8_per_tensor => .{ x.dim(1), .f32, 1e-10 }, + .fp8_block128 => .{ 128, .f32, 1e-10 }, + .mxfp4, .nvfp4 => unreachable, + }; stdx.debug.assert(x.rank() == 2, "expected a rank-2 activation matrix, got {f}", .{x.shape()}); stdx.debug.assert(@mod(x.dim(1), group_size) == 0, "activation width must be divisible by group size {d}, got {d}", .{ group_size, x.dim(1) }); const groups_per_row = @divExact(x.dim(1), group_size); - const quantized = Tensor.zeroes(Shape.init(.{ .token = x.dim(0), .feature = x.dim(1) }, .f8e4m3fn)); - const scales = Tensor.zeroes(Shape.init(.{ .token = x.dim(0), .group = groups_per_row }, .bf16)); + const quantized = Tensor.zeroes(Shape.init(.{ .token = x.dim(0), .feature = x.dim(1) }, output_dtype)); + const scales = Tensor.zeroes(Shape.init(.{ .token = x.dim(0), .group = groups_per_row }, scale_dtype)); const outs = kernels.PerTokenGroupQuantFp8.Kernel.call( .{ @@ -407,18 +646,16 @@ fn quantizePerTokenGroupFp8(x: Tensor, group_size: i64) struct { Tensor, Tensor .group_size_ptr = Tensor.constant(.{ .i64 = group_size }).reshape(.{1}), .y_num_columns_ptr = Tensor.constant(.{ .i64 = x.dim(1) }).reshape(.{1}), .y_row_stride_ptr = Tensor.constant(.{ .i64 = x.dim(1) }).reshape(.{1}), - .eps_ptr = Tensor.scalar(1e-6, .f32), + .eps_ptr = Tensor.scalar(absmax_epsilon, .f32), }, .{ .y_q = quantized.shape(), .y_s = scales.shape() }, .{ .cfg = .{ .input_dtype = toDType(x.dtype()), - .output_dtype = .f8e4m3fn, - .scale_dtype = .bf16, - .block = @intCast(group_size), - .fp8_min = -448.0, - .fp8_max = 448.0, - .use_ue8m0 = false, + .output_dtype = toDType(output_dtype), + .scale_dtype = toDType(scale_dtype), + .block = std.math.ceilPowerOfTwoAssert(usize, @intCast(group_size)), + .quant_scheme = scheme, }, .grid = .{ @intCast(x.dim(0) * groups_per_row), 1, 1 }, .num_stages = 1, @@ -483,12 +720,6 @@ test "MoE launch config selects the nearest token bucket" { try std.testing.expectEqual(@as(i32, 5), launchConfigForTokens(16).num_stages); } -fn fp8ActivationGroupSize(x: Tensor) i64 { - const group_size: i64 = 128; - stdx.debug.assert(@mod(x.dim(1), group_size) == 0, "FP8 activation width must be divisible by {d}, got {d}", .{ group_size, x.dim(1) }); - return group_size; -} - // ===== // A16W4 // ===== diff --git a/zml/moe/triton_kernels/triton_kernels.zig b/zml/moe/triton_kernels/triton_kernels.zig index 28c3b221..bc7764f2 100644 --- a/zml/moe/triton_kernels/triton_kernels.zig +++ b/zml/moe/triton_kernels/triton_kernels.zig @@ -13,6 +13,8 @@ const Builder = tri.Builder; const Value = tri.Value; const DType = tri.DType; +const log = std.log.scoped(.moe_triton); + /// Floor to a multiple of 16 — matches the Python `(v // 16) * 16` guards /// that keep dynamic strides aligned for tt.load/store. fn blockFloor16(v: Value) Value { @@ -31,9 +33,7 @@ pub const PerTokenGroupQuantFp8 = struct { output_dtype: DType, scale_dtype: DType, block: usize, - fp8_min: f32, - fp8_max: f32, - use_ue8m0: bool, + quant_scheme: zml.Quantization.Scheme, }; pub const Kernel = tri.Kernel(Cfg, .{ .name = "per_token_group_quant_fp8", @@ -54,6 +54,11 @@ pub const PerTokenGroupQuantFp8 = struct { const block: i64 = @intCast(cfg.block); const out_dt = cfg.output_dtype; + const fp8_max: f32 = switch (out_dt) { + .f8e4m3fn => 448.0, + .f8e4m3fnuz => 224.0, + else => return error.InvalidMlir, + }; const scale_dt = cfg.scale_dtype; const group_size = b.load(a.group_size_ptr); @@ -98,21 +103,29 @@ pub const PerTokenGroupQuantFp8 = struct { const absmax = b.max(b.absf(y)).maximum(eps); // scale_raw = _absmax * (1.0 / fp8_max) - const scale_raw = absmax.mul(1.0 / cfg.fp8_max); - const y_s = if (cfg.use_ue8m0) b.exp2(b.ceil(b.log2(scale_raw))) else scale_raw; + const scale_raw = absmax.mul(1.0 / fp8_max); + const y_s: Value, const stored_scale: Value = switch (cfg.quant_scheme) { + .mxfp8 => blk: { + const exponent = b.ceil(b.log2(scale_raw)); + // E8M0 stores the biased exponent as a byte. + break :blk .{ b.exp2(exponent), exponent.add(127).to(.i32).to(.i8) }; + }, + .fp8_per_channel, .fp8_per_tensor, .fp8_block128 => .{ scale_raw, scale_raw.to(scale_dt) }, + .mxfp4, .nvfp4 => unreachable, + }; // y_q = clamp(y / y_s, fp8_min, fp8_max).to(output_dtype) const y_div = y.div(y_s); const clamped = b.clampf( y_div, - b.splat(cfg.fp8_min, &.{block}), - b.splat(cfg.fp8_max, &.{block}), + b.splat(-fp8_max, &.{block}), + b.splat(fp8_max, &.{block}), ); const y_q = clamped.to(out_dt); const y_q_ptrs = y_q_ptr_shifted.addPtr(cols_i64); b.storeOpts(y_q_ptrs, y_q, .{ .mask = mask }); - b.store(y_s_ptr_shifted, y_s.to(scale_dt)); + b.store(y_s_ptr_shifted, stored_scale); } }; @@ -163,8 +176,8 @@ fn writeZerosToOutput( // fused_moe_kernel // ============================================================================= -/// Direct port of Python `fused_moe_kernel`. Bf16 / no-quant / no-bias path -/// only — errors out for the feature flags that aren't implemented yet. +/// Routed GEMM supporting BF16, MXFP8, and per-tensor, per-channel, or +/// 128x128 block-scaled FP8 weights. pub const FusedMoe = struct { pub const Cfg = struct { a_dtype: DType, @@ -181,6 +194,7 @@ pub const FusedMoe = struct { top_k: usize, naive_block_assignment: bool, compute_type: DType, + quant_scheme: ?zml.Quantization.Scheme, }; pub const Kernel = tri.Kernel(Cfg, .{ .name = "fused_moe_kernel", @@ -195,6 +209,16 @@ pub const FusedMoe = struct { .run = run, }); fn run(b: *Builder, cfg: Cfg) tri.FinishError!void { + if (cfg.quant_scheme == .fp8_block128 and cfg.block_size_k != 128) { + log.err("fused_moe_kernel: block-scaled FP8 requires BLOCK_SIZE_K=128", .{}); + return error.InvalidMlir; + } + + if (cfg.quant_scheme == .mxfp8 and cfg.block_size_k % 32 != 0) { + log.err("fused_moe_kernel: MXFP8 requires BLOCK_SIZE_K divisible by 32", .{}); + return error.InvalidMlir; + } + const a = try b.declareArgs(.{ .a_ptr = .{ .ptr = cfg.a_dtype }, .b_ptr = .{ .ptr = cfg.b_dtype }, @@ -232,8 +256,8 @@ pub const FusedMoe = struct { const top_k: i64 = @intCast(cfg.top_k); const compute_type = cfg.compute_type; - // Runtime scalars — load all first, then floor to multiples of 16 - // (Python loads them as a group then runs the // 16 * 16 sequence). + // Matrix dimensions, strides, and padded routing capacity are checked + // for 16-element alignment by the caller. const n_raw = b.load(a.N_ptr); const k_raw = b.load(a.K_ptr); const em_raw = b.load(a.EM_ptr); @@ -349,7 +373,7 @@ pub const FusedMoe = struct { const acc_init = b.zeros(&.{ block_size_m, block_size_n }, .f32); // Loop bounds stay i64 because `k_block` is i64. iter_args order - // matches Python's TTIR (a_ptrs, b_ptrs, accumulator). + // matches Python's TTIR for the matrix and accumulator state. const num_k_iters = k_block.cdiv(block_size_k); var loop = b.openFor(@as(i64, 0), num_k_iters, @as(i64, 1), .{ a_ptrs_init, b_ptrs_init, acc_init }); { @@ -383,7 +407,78 @@ pub const FusedMoe = struct { .other = b.zeros(&.{ block_size_k, block_size_n }, cfg.b_dtype), }); - const new_acc = b.dotOpts(a_val, b_val, acc, .{ + const new_acc = if (cfg.quant_scheme) |scheme| switch (scheme) { + .mxfp8 => scaled: { + const scale_k = @divExact(block_size_k, 32); + const groups = k_iter.mul(scale_k).add(b.arange(0, scale_k, .i64)); + const stride_bse = b.load(a.stride_bse_ptr); + const stride_bsk = b.load(a.stride_bsk_ptr); + const stride_bsn = b.load(a.stride_bsn_ptr); + // dot_scaled expects RHS scales in [N, K/32] order. + const rows = off_experts.mul(stride_bse).add(offs_bn.mul(stride_bsn)).expandDims(1); + const cols = groups.mul(stride_bsk).expandDims(0); + const offsets = b.broadcastTo(rows, &.{ block_size_n, scale_k }) + .add(b.broadcastTo(cols, &.{ block_size_n, scale_k })); + const n_mask = b.broadcastTo(offs_bn.lt(n_block).expandDims(1), &.{ block_size_n, scale_k }); + const k_mask = b.broadcastTo(groups.lt(k_block.cdiv(32)).expandDims(0), &.{ block_size_n, scale_k }); + const scales = b.loadOpts(a.b_scale_ptr.addPtr(offsets), .{ + .mask = n_mask.bitAnd(k_mask), + .other = b.full(&.{ block_size_n, scale_k }, 127, .i8), + }); + const a_scales: ?Value = if (cfg.a_scale_dtype != null) input_scales: { + const a_rows = offs_token.div(top_k).mul(b.load(a.stride_asm_ptr)).expandDims(1); + const a_cols = groups.mul(b.load(a.stride_ask_ptr)).expandDims(0); + const a_offsets = b.broadcastTo(a_rows, &.{ block_size_m, scale_k }) + .add(b.broadcastTo(a_cols, &.{ block_size_m, scale_k })); + const a_mask = b.broadcastTo(token_mask.expandDims(1), &.{ block_size_m, scale_k }) + .bitAnd(b.broadcastTo(groups.lt(k_block.cdiv(32)).expandDims(0), &.{ block_size_m, scale_k })); + break :input_scales b.loadOpts(a.a_scale_ptr.addPtr(a_offsets), .{ + .mask = a_mask, + .other = b.full(&.{ block_size_m, scale_k }, 127, .i8), + }); + } else null; + break :scaled b.dotScaledOpts(a_val, b_val, acc, a_scales, scales, if (a_scales != null) .e4m3 else .bf16, .e4m3, .{}); + }, + .fp8_per_channel, .fp8_per_tensor => b.dotOpts(a_val, if (cfg.a_scale_dtype != null) b_val else b_val.to(.bf16), acc, .{ + .input_precision = .tf32, + .max_num_imprecise_acc = 0, + }), + .fp8_block128 => scaled: { + // Activation scales are [token, K/128] and weight scales are + // [expert, N/128, K/128]. Compute their addresses from the + // loop index; activation scales are optional. + const stride_bse = b.load(a.stride_bse_ptr); + const stride_bsk = b.load(a.stride_bsk_ptr); + const stride_bsn = b.load(a.stride_bsn_ptr); + const b_scale_ptrs = a.b_scale_ptr.addPtr( + off_experts.mul(stride_bse) + .add(offs_bn.div(128).mul(stride_bsn)) + .add(k_iter.mul(stride_bsk)), + ); + const dot = b.dotOpts(a_val, if (cfg.a_scale_dtype != null) b_val else b_val.to(.bf16), b.zeros(&.{ block_size_m, block_size_n }, .f32), .{ + .input_precision = .tf32, + .max_num_imprecise_acc = 0, + }); + const b_s = b.loadOpts(b_scale_ptrs, .{ + .mask = offs_bn.lt(n_block), + .other = b.zeros(&.{block_size_n}, cfg.b_scale_dtype orelse .f32), + }).to(.f32); + var scaled_dot = dot.mul(b_s.expandDims(0)); + if (cfg.a_scale_dtype) |dtype| { + const a_scale_ptrs = a.a_scale_ptr.addPtr( + offs_token.div(top_k).mul(b.load(a.stride_asm_ptr)) + .add(k_iter.mul(b.load(a.stride_ask_ptr))), + ); + const a_s = b.loadOpts(a_scale_ptrs, .{ + .mask = token_mask, + .other = b.zeros(&.{block_size_m}, dtype), + }).to(.f32); + scaled_dot = scaled_dot.mul(a_s.expandDims(1)); + } + break :scaled acc.add(scaled_dot); + }, + .mxfp4, .nvfp4 => unreachable, + } else b.dotOpts(a_val, b_val, acc, .{ .input_precision = .tf32, .max_num_imprecise_acc = 0, }); @@ -393,10 +488,47 @@ pub const FusedMoe = struct { const new_a_ptrs = a_ptrs.addPtr(b.splat(bsk_i32, &.{ block_size_m, block_size_k })); const new_b_ptrs = b_ptrs.addPtr(b.splat(bsk_i32, &.{ block_size_k, block_size_n })); - loop.yield(.{ new_a_ptrs, new_b_ptrs, new_acc }); + loop.yield(.{ + new_a_ptrs, + new_b_ptrs, + new_acc, + }); } var accumulator = loop.results[2]; + if (cfg.quant_scheme == .fp8_per_channel or cfg.quant_scheme == .fp8_per_tensor) { + // These scales are constant along K, so apply them after accumulation. + const expert_offset = off_experts.mul(b.load(a.stride_bse_ptr)); + const offsets = if (cfg.quant_scheme == .fp8_per_channel) + expert_offset.add(offs_bn.mul(b.load(a.stride_bsn_ptr))) + else + b.splat(expert_offset, &.{block_size_n}); + const scales = b.loadOpts(a.b_scale_ptr.addPtr(offsets), .{ + .mask = offs_bn.lt(n_block), + .other = b.zeros(&.{block_size_n}, cfg.b_scale_dtype.?), + }).to(.f32); + accumulator = accumulator.mul(scales.expandDims(0)); + if (cfg.a_scale_dtype) |dtype| { + const offsets_a = offs_token.div(top_k).mul(b.load(a.stride_asm_ptr)); + const scales_a = b.loadOpts(a.a_scale_ptr.addPtr(offsets_a), .{ + .mask = token_mask, + .other = b.zeros(&.{block_size_m}, dtype), + }).to(.f32); + accumulator = accumulator.mul(scales_a.expandDims(1)); + } + } + + if (cfg.b_bias_dtype) |dtype| { + const offsets = off_experts.mul(b.load(a.stride_bbe_ptr)) + .add(offs_bn.mul(b.load(a.stride_bbn_ptr))); + const bias = b.loadOpts(a.b_bias_ptr.addPtr(offsets), .{ + .mask = offs_bn.lt(n_block), + .other = b.zeros(&.{block_size_n}, dtype), + }).to(.f32); + // Bias is in output units and must receive the routing weight too. + accumulator = accumulator.add(bias.expandDims(0)); + } + if (cfg.routing_weights_dtype) |dtype| { const other = b.zeros(&.{block_size_m}, dtype); const routing_weights = b.loadOpts(a.routing_weights_ptr.addPtr(offs_token), .{ .mask = token_mask, .other = other }); @@ -425,6 +557,66 @@ pub const FusedMoe = struct { } }; +test "FP8 activation quantization emits float and E8M0 scales" { + for ([_]zml.Quantization.Scheme{ .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block128 }) |scheme| { + for ([_]DType{ .bf16, .f32 }) |input_dtype| { + const ir = try PerTokenGroupQuantFp8.Kernel.emit(std.testing.allocator, .{ + .input_dtype = input_dtype, + .output_dtype = .f8e4m3fn, + .scale_dtype = if (scheme == .mxfp8) .i8 else .f32, + .block = if (scheme == .mxfp8) 32 else 256, + .quant_scheme = scheme, + }); + defer std.testing.allocator.free(ir); + if (scheme == .mxfp8) { + try std.testing.expect(std.mem.indexOf(u8, ir, "arith.trunci") != null); + } + } + } +} + +test "FusedMoe emits each FP8 scaling path" { + const allocator = std.testing.allocator; + const schemes = [_]?zml.Quantization.Scheme{ null, .mxfp8, .fp8_per_tensor, .fp8_per_channel, .fp8_block128 }; + for (schemes) |scheme| { + for ([_]DType{ .f8e4m3fn, .f8e4m3fnuz }) |fp8_dtype| { + if (scheme == .mxfp8 and fp8_dtype == .f8e4m3fnuz) continue; + for ([_]bool{ false, true }) |quantize_input| { + if (scheme == null and quantize_input) continue; + for ([_]?DType{ null, .bf16, .f32 }) |bias_dtype| { + const ir = try FusedMoe.Kernel.emit(allocator, .{ + .a_dtype = if (quantize_input) fp8_dtype else .bf16, + .b_dtype = if (scheme != null) fp8_dtype else .bf16, + .c_dtype = .bf16, + .a_scale_dtype = if (quantize_input) (if (scheme == .mxfp8) .i8 else .f32) else null, + .b_scale_dtype = if (scheme == .mxfp8) .i8 else if (scheme != null) .f32 else null, + .b_bias_dtype = bias_dtype, + .routing_weights_dtype = .bf16, + .block_size_m = 16, + .block_size_n = 32, + .block_size_k = 128, + .group_size_m = 1, + .top_k = 2, + .naive_block_assignment = false, + .compute_type = .bf16, + .quant_scheme = scheme, + }); + defer allocator.free(ir); + if (scheme == .mxfp8) { + try std.testing.expect(std.mem.indexOf(u8, ir, "tt.dot_scaled") != null); + } else { + try std.testing.expect(std.mem.indexOf(u8, ir, "tt.dot_scaled") == null); + try std.testing.expect(std.mem.indexOf(u8, ir, "tt.dot") != null); + if (scheme != null and !quantize_input) { + try std.testing.expect(std.mem.indexOf(u8, ir, "tt.fp_to_fp") != null); + } + } + } + } + } + } +} + // ============================================================================= // moe_align_block_size_kernel // ============================================================================= From 7204aaacbf3f0fd2581234dc8fd3837b5fae3dbe Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Fri, 11 Sep 2026 12:56:35 +0200 Subject: [PATCH 677/881] zml/moe: Merge MXFP4 & MXFP8 paths (#795) --- examples/llm/models/qwen3_5_moe/model.zig | 6 +- zml/BUILD.bazel | 1 - zml/moe/moe.zig | 16 + zml/moe/triton.zig | 774 +++++----------------- zml/moe/triton_kernels/a16w4_kernel.zig | 189 ------ zml/moe/triton_kernels/triton_kernels.zig | 47 +- 6 files changed, 206 insertions(+), 827 deletions(-) delete mode 100644 zml/moe/triton_kernels/a16w4_kernel.zig diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 84ac6741..5370a6b2 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -777,7 +777,11 @@ pub const Moe = struct { routing_scores, self.gate_up_proj, self.down_proj, - .{ .quantize_input = false }, + .{ + .quantize_input = false, + .gate_up_layout = .split, + .routing_weight_placement = .after_down, + }, moe_parameters, ) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index e1803916..b3125fc5 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -106,7 +106,6 @@ zig_library( "moe/mosaic_tpu.zig", "moe/mosaic_tpu_kernels/gmm_ep.zig", "moe/triton.zig", - "moe/triton_kernels/a16w4_kernel.zig", "moe/triton_kernels/triton_kernels.zig", "nn.zig", "ops.zig", diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index 10aa6247..6e6a0a68 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -146,6 +146,10 @@ pub const Options = struct { activation_threshold: ?f32 = null, /// Quantize activations for Triton FP8 GEMMs; false keeps BF16 activations. quantize_input: bool, + /// Gate/up layout; non-Triton backends require split columns. + gate_up_layout: triton.GateUpLayout, + /// Where routing weights are applied; non-Triton backends require after_down. + routing_weight_placement: triton.RoutingWeightPlacement, }; pub fn forwardMoe( @@ -157,6 +161,16 @@ pub fn forwardMoe( opts: Options, parameters: Parameters, ) !zml.Tensor { + switch (parameters) { + .triton => {}, + .flashinfer_cutlass, .mosaic_tpu, .metal => { + stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); + stdx.debug.assert(opts.gate_up_layout == .split, "Non-Triton MoE backends require split gate/up columns", .{}); + stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); + stdx.debug.assert(opts.activation_threshold == null, "Activation thresholds require the Triton MoE backend", .{}); + }, + } + const gate_up_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; const down_scheme: ?zml.Quantization.Scheme = if (down.quantization) |q| q.scheme else null; if (gate_up_scheme != down_scheme) return error.UnsupportedQuantization; @@ -386,6 +400,8 @@ pub fn forwardMoe( .activation = parameters.triton.activation, .activation_threshold = opts.activation_threshold, .quantize_input = opts.quantize_input, + .gate_up_layout = opts.gate_up_layout, + .routing_weight_placement = opts.routing_weight_placement, }; const expert_partition = gate_up.weight.shape().partition(.expert); diff --git a/zml/moe/triton.zig b/zml/moe/triton.zig index 6d6495f4..9f39cec6 100644 --- a/zml/moe/triton.zig +++ b/zml/moe/triton.zig @@ -8,7 +8,6 @@ const Tensor = zml.Tensor; const Shape = zml.Shape; const tri = zml.kernel.triton; const toDType = tri.from; -const a16w4_kernel = @import("triton_kernels/a16w4_kernel.zig"); const kernels = @import("triton_kernels/triton_kernels.zig"); const log = std.log.scoped(.moe_triton); @@ -40,6 +39,9 @@ pub const Parameters = struct { } }; +pub const GateUpLayout = enum { split, interleaved }; +pub const RoutingWeightPlacement = enum { before_down, after_down }; + pub const FusedExpertsArgs = struct { hidden_states: Tensor, gate_up: zml.nn.Linear, @@ -51,6 +53,8 @@ pub const FusedExpertsArgs = struct { activation_threshold: ?f32 = null, /// Use FP8 activations for FP8 weights; false keeps BF16 activations. quantize_input: bool, + gate_up_layout: GateUpLayout, + routing_weight_placement: RoutingWeightPlacement, }; pub fn fusedExpertsImpl(opts: FusedExpertsArgs) !Tensor { @@ -65,37 +69,16 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) !Tensor { const gate_up_scheme = opts.gate_up.quantizationScheme(); const down_scheme = opts.down.quantizationScheme(); if (gate_up_scheme) |scheme| switch (scheme) { - .mxfp4 => { - if (down_scheme != .mxfp4) return error.UnsupportedQuantization; - const local_topk_ids, const local_topk_weights = if (opts.expert_map) |expert_map| blk: { - const local_num_experts = opts.gate_up.weight.dim(.expert); - const mapped_ids = expert_map - .gather(.{ .expert = topk_ids.convert(.i32) }, .{}) - .withTags(topk_ids.shape().tags()); - const in_range = mapped_ids.cmp(.GE, Tensor.scalar(0, .i32)) - .logical(.AND, mapped_ids.cmp(.LT, Tensor.scalar(local_num_experts, .i32))); - - break :blk .{ - in_range.select(mapped_ids, Tensor.scalar(local_num_experts, .i32)), - in_range.select(topk_weights, Tensor.scalar(0, topk_weights.dtype())), - }; - } else .{ topk_ids, topk_weights }; - - var local_args = opts; - local_args.topk_ids = local_topk_ids; - local_args.topk_weights = local_topk_weights; - return fusedExpertsImpl_fp4(local_args); - }, .nvfp4 => return error.UnsupportedQuantization, .fp8_block128 => launch_config.block_size_k = 128, - .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, + .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, }; var down_launch_config = launchConfigForTokens(num_tokens); if (down_scheme) |scheme| switch (scheme) { .fp8_block128 => down_launch_config.block_size_k = 128, - .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, - .mxfp4, .nvfp4 => return error.UnsupportedQuantization, + .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, + .nvfp4 => return error.UnsupportedQuantization, }; const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); @@ -105,14 +88,16 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) !Tensor { const ids = topk_ids.reshape(.{ .token = num_tokens, .in = topk_ids.dim(.top_expert) }).withTags(.{ .token, .topk }); stdx.debug.assert(hidden.dtype() == .bf16, "expected BF16 hidden states, got {}", .{hidden.dtype()}); - stdx.debug.assert(gate_up.dtype() == .bf16 or gate_up.dtype() == .f8e4m3fn or gate_up.dtype() == .f8e4m3fnuz, "expected BF16 or FP8 E4M3FN/FNUZ gate/up weights, got {}", .{gate_up.dtype()}); - stdx.debug.assert(down.dtype() == .bf16 or down.dtype() == .f8e4m3fn or down.dtype() == .f8e4m3fnuz, "expected BF16 or FP8 E4M3FN/FNUZ down weights, got {}", .{down.dtype()}); + stdx.debug.assert(if (gate_up_scheme == .mxfp4) gate_up.dtype() == .u8 or gate_up.dtype() == .i8 or gate_up.dtype() == .f4e2m1 else gate_up.dtype() == .bf16 or gate_up.dtype() == .f8e4m3fn or gate_up.dtype() == .f8e4m3fnuz, "unsupported gate/up weight dtype {}", .{gate_up.dtype()}); + stdx.debug.assert(if (down_scheme == .mxfp4) down.dtype() == .u8 or down.dtype() == .i8 or down.dtype() == .f4e2m1 else down.dtype() == .bf16 or down.dtype() == .f8e4m3fn or down.dtype() == .f8e4m3fnuz, "unsupported down weight dtype {}", .{down.dtype()}); stdx.debug.assert(routing_weights.dtype() == .f32 or routing_weights.dtype() == .bf16, "expected FP32 or BF16 routing weights, got {}", .{routing_weights.dtype()}); stdx.debug.assert(ids.dtype() == .i32, "expected I32 expert ids, got {}", .{ids.dtype()}); - stdx.debug.assert(hidden.dim(.in) == gate_up.dim(.in), "hidden width {} must match gate/up input width {}", .{ hidden.dim(.in), gate_up.dim(.in) }); + const gate_up_k = gate_up.dim(.in) * @as(i64, if (gate_up_scheme == .mxfp4 and gate_up.dtype() != .f4e2m1) 2 else 1); + const down_k = down.dim(.mid) * @as(i64, if (down_scheme == .mxfp4 and down.dtype() != .f4e2m1) 2 else 1); + stdx.debug.assert(hidden.dim(.in) == gate_up_k, "hidden width {} must match gate/up input width {}", .{ hidden.dim(.in), gate_up_k }); const activation_reduction: i64 = if (opts.activation == .relu) 1 else 2; stdx.debug.assert(@rem(gate_up.dim(.out), activation_reduction) == 0, "gate/up output width {} must be divisible by {}", .{ gate_up.dim(.out), activation_reduction }); - stdx.debug.assert(down.dim(.mid) == @divFloor(gate_up.dim(.out), activation_reduction), "down input width {} must match activated width {}", .{ down.dim(.mid), @divFloor(gate_up.dim(.out), activation_reduction) }); + stdx.debug.assert(down_k == @divFloor(gate_up.dim(.out), activation_reduction), "down input width {} must match activated width {}", .{ down_k, @divFloor(gate_up.dim(.out), activation_reduction) }); stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "routing ids and weights must match hidden token count {}, got {} and {}", .{ hidden.dim(.token), ids.dim(.token), routing_weights.dim(.token) }); stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "routing ids and weights must have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), routing_weights.dim(.topk) }); stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "gate/up and down expert counts must match, got {} and {}", .{ gate_up.dim(.expert), down.dim(.expert) }); @@ -130,7 +115,7 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) !Tensor { var hidden_quant = hidden; var input_scale: ?Tensor = null; - if (opts.quantize_input) { + if (opts.quantize_input and gate_up_scheme != .mxfp4) { if (gate_up_scheme) |scheme| hidden_quant, input_scale = quantizeFp8Input(hidden, scheme, gate_up.dtype()); } @@ -148,10 +133,14 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) !Tensor { .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), }); - const activated = applyExpertActivation(gate_up_out, opts.activation, opts.activation_threshold); + var activated = applyExpertActivation(gate_up_out, opts.activation, opts.activation_threshold, opts.gate_up_layout); + if (opts.routing_weight_placement == .before_down) { + const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); + activated = activated.mul(weights.broad(activated.shape())); + } var activated_quant = activated.convert(.bf16); input_scale = null; - if (opts.quantize_input) { + if (opts.quantize_input and down_scheme != .mxfp4) { if (down_scheme) |scheme| activated_quant, input_scale = quantizeFp8Input(activated, scheme, down.dtype()); } @@ -161,7 +150,7 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) !Tensor { .bias = opts.down.bias, .input_scale = input_scale, .weight_scale = opts.down.quantizationScales(), - .routing_weights = routing_weights, + .routing_weights = if (opts.routing_weight_placement == .after_down) routing_weights else null, .routing = routing, .expert_ids = expert_ids, .launch_config = down_launch_config, @@ -175,7 +164,7 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) !Tensor { return output.reshape(.{ .b = b, .token = s, .out = down.dim(.out) }); } -fn applyExpertActivation(input: Tensor, mode: Parameters.ActivationMode, activation_threshold: ?f32) Tensor { +fn applyExpertActivation(input: Tensor, mode: Parameters.ActivationMode, activation_threshold: ?f32, layout: GateUpLayout) Tensor { const x = input.convert(.f32); if (mode == .relu) { const clipped = if (activation_threshold) |limit| x.minimum(Tensor.scalar(limit, x.dtype())) else x; @@ -183,8 +172,10 @@ fn applyExpertActivation(input: Tensor, mode: Parameters.ActivationMode, activat } const mid = @divFloor(x.dim(.out), 2); - var gate = x.slice(.out, .{ .end = mid }); - var up = x.slice(.out, .{ .start = mid }); + var gate, var up = switch (layout) { + .split => .{ x.slice(.out, .{ .end = mid }), x.slice(.out, .{ .start = mid }) }, + .interleaved => .{ x.slice(.out, .{ .start = 0, .step = 2 }), x.slice(.out, .{ .start = 1, .step = 2 }) }, + }; if (activation_threshold) |limit_| { const limit = Tensor.scalar(limit_, x.dtype()); gate = gate.minimum(limit); @@ -197,37 +188,39 @@ fn applyExpertActivation(input: Tensor, mode: Parameters.ActivationMode, activat }; } -test "SwiGLU uses FP32 math for BF16 inputs" { +test "SwiGLU uses FP32 math for split and interleaved BF16 inputs" { const allocator = std.testing.allocator; const io = std.testing.io; const platform = zml.testing.env(); const Local = struct { - fn forward(x: Tensor, threshold: ?f32) Tensor { - return applyExpertActivation(x, .silu, threshold); + fn forward(x: Tensor, layout: GateUpLayout, threshold: ?f32) Tensor { + return applyExpertActivation(x, .silu, threshold, layout); } }; const x: Tensor = .init(.{ .token = 1, .out = 6 }, .bf16); const gates = [_]f32{ 0.75, -1.25, 3.5 }; const ups = [_]f32{ 0.875, -2.25, 4.5 }; - var values: [6]zml.floats.BFloat16 = undefined; - for (gates, ups, 0..) |gate, up, i| { - values[i] = .fromF32(gate); - values[i + 3] = .fromF32(up); - } - var input = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&values)); - defer input.deinit(); - for ([_]?f32{ null, 2 }) |threshold| { - var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, threshold }, .{}); - defer exe.deinit(); - try zml.testing.expectEqualShapes(Shape.init(.{ .token = 1, .out = 3 }, .f32), exe.output_shapes[0]); - var output = try zml.testing.autoCall(allocator, io, &exe, Local.forward, .{input}); - defer output.deinit(); - var actual = try output.toSliceAlloc(allocator, io); - defer actual.free(allocator); - for (gates, ups, actual.constItems(f32)) |gate, up, value| { - const g = if (threshold) |limit| @min(gate, limit) else gate; - const u = if (threshold) |limit| std.math.clamp(up, -limit, limit) else up; - try std.testing.expectApproxEqAbs(g / (1 + @exp(-g)) * u, value, 1e-6); + for ([_]GateUpLayout{ .split, .interleaved }) |layout| { + var values: [6]zml.floats.BFloat16 = undefined; + for (gates, ups, 0..) |gate, up, i| { + values[if (layout == .split) i else 2 * i] = .fromF32(gate); + values[if (layout == .split) i + 3 else 2 * i + 1] = .fromF32(up); + } + var input = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&values)); + defer input.deinit(); + for ([_]?f32{ null, 2 }) |threshold| { + var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, threshold }, .{}); + defer exe.deinit(); + try zml.testing.expectEqualShapes(Shape.init(.{ .token = 1, .out = 3 }, .f32), exe.output_shapes[0]); + var output = try zml.testing.autoCall(allocator, io, &exe, Local.forward, .{input}); + defer output.deinit(); + var actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + for (gates, ups, actual.constItems(f32)) |gate, up, value| { + const g = if (threshold) |limit| @min(gate, limit) else gate; + const u = if (threshold) |limit| std.math.clamp(up, -limit, limit) else up; + try std.testing.expectApproxEqAbs(g / (1 + @exp(-g)) * u, value, 1e-6); + } } } } @@ -238,7 +231,7 @@ test "ReLU squared activation preserves width and applies threshold before squar const platform = zml.testing.env(); const Local = struct { fn forward(x: Tensor, threshold: ?f32) Tensor { - return applyExpertActivation(x, .relu, threshold); + return applyExpertActivation(x, .relu, threshold, .split); } }; const x: Tensor = .init(.{ .token = 1, .out = 5 }, .f32); @@ -273,8 +266,20 @@ fn callFusedMoe(opts: struct { top_k: usize, output_shape: Shape, }) Tensor { + // Native FP4 tensors expose logical K; the Triton operand is byte-packed. + const weight = if (opts.quant_scheme == .mxfp4 and opts.weight.dtype() == .f4e2m1) + opts.weight.reshape(opts.weight.shape().setDim(2, @divExact(opts.weight.dim(2), 2)).append(.{ .nibble = 2 })).bitCast(.u8) + else + opts.weight; + const weight_k = weight.dim(2) * @as(i64, if (opts.quant_scheme == .mxfp4) 2 else 1); + stdx.debug.assert(opts.input.dim(1) == weight_k, "input width {} must match logical weight K {}", .{ opts.input.dim(1), weight_k }); + if (opts.quant_scheme == .mxfp4) { + stdx.debug.assert(@mod(weight_k, 32) == 0 and opts.weight_scale != null, "MXFP4 requires K divisible by 32 and weight scales", .{}); + stdx.debug.assert(opts.input.dtype() == .bf16 and opts.input_scale == null, "MXFP4 requires BF16 activations without input scales", .{}); + } + stdx.debug.assert(opts.quant_scheme != null or (opts.input_scale == null and opts.weight_scale == null), "scales require a quantization scheme", .{}); - for ([_]i64{ opts.weight.dim(1), opts.weight.dim(2), opts.input.dim(1), opts.weight.dim(1) * opts.weight.dim(2), opts.output_shape.dim(-1) }) |dim_or_stride| { + for ([_]i64{ weight.dim(1), weight.dim(2), opts.input.dim(1), weight.dim(1) * weight.dim(2), opts.output_shape.dim(-1) }) |dim_or_stride| { stdx.debug.assert(dim_or_stride > 0 and @mod(dim_or_stride, 16) == 0, "FusedMoe dimensions and matrix strides must be positive multiples of 16, got {}", .{dim_or_stride}); } const block_size_m: i64 = @intCast(opts.launch_config.block_size_m); @@ -287,16 +292,16 @@ fn callFusedMoe(opts: struct { stdx.debug.assert(@mod(em_effective, block_size_m) == 0, "routing capacity {} must be a multiple of block size {}", .{ em_effective, block_size_m }); const grid_x = @divExact(em_effective, block_size_m) * - (std.math.divCeil(i64, opts.weight.dim(1), block_size_n) catch unreachable); + (std.math.divCeil(i64, weight.dim(1), block_size_n) catch unreachable); const weight_scale: ?Tensor = if (opts.weight_scale) |scale| blk: { const scheme = opts.quant_scheme orelse break :blk scale; break :blk switch (scheme) { // dot_scaled takes E8M0 encodings as bytes, including native E8M0 tensors. - .mxfp8 => scale.bitCast(.u8), - .fp8_per_channel => scale.reshape(.{ opts.weight.dim(0), opts.weight.dim(1), 1 }), - .fp8_per_tensor => if (scale.count() == 1) scale else scale.reshape(.{ opts.weight.dim(0), 1, 1 }), - .fp8_block128, .mxfp4, .nvfp4 => scale, + .mxfp4, .mxfp8 => scale.bitCast(.u8), + .fp8_per_channel => scale.reshape(.{ weight.dim(0), weight.dim(1), 1 }), + .fp8_per_tensor => if (scale.count() == 1) scale else scale.reshape(.{ weight.dim(0), 1, 1 }), + .fp8_block128, .nvfp4 => scale, }; } else null; @@ -313,7 +318,7 @@ fn callFusedMoe(opts: struct { return kernels.FusedMoe.Kernel.call( .{ .a_ptr = opts.input, - .b_ptr = opts.weight, + .b_ptr = weight, .b_bias_ptr = opts.bias orelse Tensor.scalar(0, .f32), .a_scale_ptr = opts.input_scale orelse Tensor.scalar(1.0, .f32), .b_scale_ptr = weight_scale orelse Tensor.scalar(1.0, .f32), @@ -321,14 +326,14 @@ fn callFusedMoe(opts: struct { .sorted_token_ids_ptr = opts.routing.sorted_token_ids, .expert_ids_ptr = opts.expert_ids, .num_tokens_post_padded_ptr = opts.routing.num_tokens_post_padded, - .N_ptr = Tensor.constant(.{ .i64 = opts.weight.dim(1) }).reshape(.{1}), - .K_ptr = Tensor.constant(.{ .i64 = opts.weight.dim(2) }).reshape(.{1}), + .N_ptr = Tensor.constant(.{ .i64 = weight.dim(1) }).reshape(.{1}), + .K_ptr = Tensor.constant(.{ .i64 = weight_k }).reshape(.{1}), .EM_ptr = Tensor.constant(.{ .i64 = em_effective }).reshape(.{1}), .num_valid_tokens_ptr = Tensor.constant(.{ .i64 = opts.routing.num_assignments }).reshape(.{1}), .stride_am_ptr = Tensor.constant(.{ .i64 = opts.input.dim(1) }).reshape(.{1}), - .stride_be_ptr = Tensor.constant(.{ .i64 = opts.weight.dim(1) * opts.weight.dim(2) }).reshape(.{1}), - .stride_bn_ptr = Tensor.constant(.{ .i64 = opts.weight.dim(2) }).reshape(.{1}), - .stride_cm_ptr = Tensor.constant(.{ .i64 = opts.weight.dim(.out) }).reshape(.{1}), + .stride_be_ptr = Tensor.constant(.{ .i64 = weight.dim(1) * weight.dim(2) }).reshape(.{1}), + .stride_bn_ptr = Tensor.constant(.{ .i64 = weight.dim(2) }).reshape(.{1}), + .stride_cm_ptr = Tensor.constant(.{ .i64 = weight.dim(.out) }).reshape(.{1}), .stride_asm_ptr = Tensor.constant(.{ .i64 = stride_asm }).reshape(.{1}), .stride_ask_ptr = Tensor.constant(.{ .i64 = stride_ask }).reshape(.{1}), .stride_bse_ptr = Tensor.constant(.{ .i64 = stride_bse }).reshape(.{1}), @@ -341,7 +346,7 @@ fn callFusedMoe(opts: struct { .{ .cfg = .{ .a_dtype = toDType(opts.input.dtype()), - .b_dtype = toDType(opts.weight.dtype()), + .b_dtype = toDType(weight.dtype()), .c_dtype = toDType(opts.output_shape.dtype()), .a_scale_dtype = if (opts.input_scale) |scale| toDType(scale.dtype()) else null, .b_scale_dtype = if (weight_scale) |scale| toDType(scale.dtype()) else null, @@ -488,6 +493,89 @@ test "FP8 routed GEMM with bias matches dequantized weights" { } } +test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + if (platform.target != .cuda) return error.SkipZigTest; + const Local = struct { + fn forward(x: Tensor, layout: GateUpLayout, placement: RoutingWeightPlacement, storage_dtype: DataType) Tensor { + const fp4 = storage_dtype != .bf16; + const columns = Tensor.arange(.{ .end = 256 }, .i32).withTags(.{.dout}); + const is_gate = switch (layout) { + .split => columns.cmp(.LT, Tensor.scalar(128, .i32)), + .interleaved => columns.remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), + }; + const dtype: DataType = if (fp4) .u8 else .bf16; + // E2M1 codes 1, 2, 3 encode 0.5, 1, 1.5; each byte holds two values. + const gate_value = if (fp4) Tensor.scalar(0x11, .u8) else Tensor.scalar(0.5, .bf16); + const up_value = if (fp4) Tensor.scalar(0x33, .u8) else Tensor.scalar(1.5, .bf16); + const gate_up_values = is_gate.select(gate_value, up_value); + var gate_up: zml.nn.Linear = .{ + .weight = gate_up_values.broad(Shape.init(.{ .expert = 8, .dout = 256, .d = @as(i64, if (fp4) 64 else 128) }, dtype)), + .tag = Shape.toTag(.d), + .quantization = if (fp4) .{ + .scheme = .mxfp4, + .scales = Tensor.scalar(127, .u8).broad(Shape.init(.{ .expert = 8, .dout = 256, .d = 4 }, .u8)), + } else null, + }; + var down: zml.nn.Linear = .{ + .weight = (if (fp4) Tensor.scalar(0x22, .u8) else Tensor.scalar(1, .bf16)).broad(Shape.init(.{ .expert = 8, .d = 128, .dout = @as(i64, if (fp4) 64 else 128) }, dtype)), + .bias = Tensor.scalar(2, .bf16).broad(Shape.init(.{ .expert = 8, .d = 128 }, .bf16)), + .tag = Shape.toTag(.dout), + .quantization = if (fp4) .{ + .scheme = .mxfp4, + .scales = Tensor.scalar(127, .u8).broad(Shape.init(.{ .expert = 8, .d = 128, .dout = 4 }, .u8)), + } else null, + }; + if (storage_dtype == .f4e2m1) { + gate_up.weight = gate_up.weight.bitCast(.f4e2m1).reshape(.{ .expert = 8, .dout = 256, .d = 128 }); + down.weight = down.weight.bitCast(.f4e2m1).reshape(.{ .expert = 8, .d = 128, .dout = 128 }); + gate_up.quantization.?.scales = gate_up.quantization.?.scales.bitCast(.f8e8m0); + down.quantization.?.scales = down.quantization.?.scales.bitCast(.f8e8m0); + } + const route = Tensor.arange(.{ .end = 2 }, .i32).reshape(.{ .b = 1, .s = 1, .top_expert = 2 }) + .broad(Shape.init(.{ .b = 1, .s = x.dim(.s), .top_expert = 2 }, .i32)); + return fusedExpertsImpl(.{ + .hidden_states = x, + .gate_up = gate_up, + .down = down, + .topk_ids = route.addConstant(2), + .topk_weights = route.convert(.f32).addConstant(1).scale(0.25), + .gate_up_layout = layout, + .routing_weight_placement = placement, + .quantize_input = false, + }) catch unreachable; + } + }; + for ([_]DataType{ .bf16, .u8, .f4e2m1 }) |storage_dtype| { + const tokens: i64 = if (storage_dtype == .u8) 17 else 1; + const x: Tensor = .init(.{ .b = 1, .s = tokens, .d = 128 }, .bf16); + const host = try zml.Slice.alloc(allocator, x.shape()); + defer host.free(allocator); + @memset(host.items(zml.floats.BFloat16), .fromF32(1.0 / 128.0)); + var input = try zml.Buffer.fromSlice(io, platform, host, .replicated); + defer input.deinit(); + for ([_]GateUpLayout{ .split, .interleaved }) |layout| { + for ([_]RoutingWeightPlacement{ .before_down, .after_down }) |placement| { + var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, placement, storage_dtype }, .{}); + defer exe.deinit(); + var output = try zml.testing.autoCall(allocator, io, &exe, Local.forward, .{input}); + defer output.deinit(); + var actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + const activation: f32 = 0.5 / (1 + @exp(@as(f32, -0.5))) * 1.5; + // The two routes have weights 0.25 and 0.5. Bias distinguishes placement. + const bias: f32 = if (placement == .before_down) 4 else 1.5; + const expected = 128 * activation * 0.75 + bias; + for (actual.constItems(zml.floats.BFloat16)) |value| { + try std.testing.expectApproxEqAbs(expected, value.toF32(), 0.5); + } + } + } + } +} + const Routing = struct { sorted_token_ids: Tensor, expert_ids: Tensor, @@ -719,545 +807,3 @@ test "MoE launch config selects the nearest token bucket" { try std.testing.expectEqual(@as(usize, 64), launchConfigForTokens(16).group_size_m); try std.testing.expectEqual(@as(i32, 5), launchConfigForTokens(16).num_stages); } - -// ===== -// A16W4 -// ===== -pub fn fusedExpertsImpl_fp4(args: FusedExpertsArgs) !zml.Tensor { - const input = args.hidden_states; - const topk_ids = args.topk_ids; - const topk_weights = args.topk_weights; - const x = input.reshape(.{ - .token = @divExact(@as(i64, @intCast(input.count())), input.dim(.d)), - .d = input.dim(.d), - }); - const num_tokens = x.dim(.token); - const num_routes: i64 = @intCast(topk_ids.count()); - stdx.debug.assert(@mod(num_routes, num_tokens) == 0, "expected {} routing ids to be divisible by {} tokens", .{ num_routes, num_tokens }); - stdx.debug.assert(topk_weights.count() == topk_ids.count(), "expected matching routing id and weight counts, got {} and {}", .{ topk_ids.count(), topk_weights.count() }); - const topk = @divExact(num_routes, num_tokens); - const flat_topk_ids = topk_ids.reshape(.{ .token = num_tokens, .topk = topk }); - const flat_topk_weights = topk_weights.reshape(.{ .token = num_tokens, .topk = topk }); - const kernel_cfg = getBestConfig( - @intCast(num_tokens), - @intCast(topk), - @intCast(args.gate_up.weight.dim(.expert)), - ); - const num_experts = args.gate_up.weight.dim(.expert); - const aligned_routing = prepareRouting(flat_topk_ids, num_experts, @intCast(kernel_cfg.block_m)); - const routing = prepareFp4Routing( - aligned_routing, - flat_topk_ids, - flat_topk_weights, - num_tokens, - num_experts, - @intCast(kernel_cfg.block_m), - ); - - const hidden_shape: zml.Shape = .init(.{ - .route = routing.num_rows, - .dout = @divExact(args.gate_up.weight.dim(.dout), 2), - }, .bf16); - - const hidden = try runGemm( - x, - args.gate_up.weight, - args.gate_up.quantization.?.scales, - .{ - .routing = routing, - .weight_contract_tag = zml.Shape.toTag(.d), - .weight_output_tag = zml.Shape.toTag(.dout), - .output_shape = hidden_shape, - .gather = routing.sorted_route_ids, - .gammas = routing.sorted_weights, - .bias = args.gate_up.bias, - .apply_swiglu = true, - .activation_limit = args.activation_threshold, - .block_m = kernel_cfg.block_m, - .block_n = kernel_cfg.block_n, - .block_k = kernel_cfg.block_k, - .group_m = kernel_cfg.group_m, - .num_warps = kernel_cfg.num_warps, - .num_stages = kernel_cfg.num_stages, - }, - ); - - const routed_shape: zml.Shape = .init(.{ - .route = routing.num_rows, - .d = args.down.weight.dim(.d), - }, .bf16); - - const routed = try runGemm( - hidden, - args.down.weight, - args.down.quantization.?.scales, - .{ - .routing = routing, - .weight_contract_tag = zml.Shape.toTag(.dout), - .weight_output_tag = zml.Shape.toTag(.d), - .output_shape = routed_shape, - .bias = args.down.bias, - .apply_swiglu = false, - .activation_limit = 1.0, - .block_m = kernel_cfg.block_m, - .block_n = kernel_cfg.block_n, - .block_k = kernel_cfg.block_k, - .group_m = kernel_cfg.group_m, - .num_warps = kernel_cfg.num_warps, - .num_stages = kernel_cfg.num_stages, - }, - ); - - const active_routed = routing.active_routes.broad(routed.shape().withDtype(.bool)).select( - routed, - zml.Tensor.zeroes(routed.shape()), - ); - const token_ids = routing.sorted_route_ids.divByConst(routing.topk).withTags(.{.route}); - const output_flat_shape: zml.Shape = .init(.{ .token = routing.num_tokens, .d = input.dim(.d) }, .f32); - const output_flat = zml.Tensor.zeroes(output_flat_shape).scatterSlices( - .{ .token = token_ids }, - active_routed.convert(.f32), - .{}, - ); - - return output_flat.reshape(input.shape().withDtype(.f32)).convert(input.dtype()); -} - -const KernelConf = struct { - block_m: u32, - block_n: u32, - block_k: u32, - group_m: u32, - num_warps: u32, - num_stages: u32, -}; - -const kernel_config_token_buckets = [_]u32{ - 1, 2, 4, 8, 16, 24, 32, 48, 64, - 96, 128, 256, 512, 1024, 1536, 2048, 3072, 4096, -}; - -fn configForTokenBucket(num_tokens: u32) KernelConf { - return switch (num_tokens) { - 1 => .{ - .block_m = 16, - .block_n = 32, - .block_k = 64, - .group_m = 1, - .num_warps = 4, - .num_stages = 4, - }, - 2 => .{ - .block_m = 16, - .block_n = 32, - .block_k = 64, - .group_m = 1, - .num_warps = 4, - .num_stages = 4, - }, - 4 => .{ - .block_m = 16, - .block_n = 32, - .block_k = 64, - .group_m = 1, - .num_warps = 4, - .num_stages = 3, - }, - 8 => .{ - .block_m = 16, - .block_n = 128, - .block_k = 128, - .group_m = 1, - .num_warps = 8, - .num_stages = 3, - }, - 16 => .{ - .block_m = 16, - .block_n = 64, - .block_k = 64, - .group_m = 64, - .num_warps = 4, - .num_stages = 5, - }, - 24 => .{ - .block_m = 16, - .block_n = 64, - .block_k = 128, - .group_m = 1, - .num_warps = 8, - .num_stages = 2, - }, - 32 => .{ - .block_m = 16, - .block_n = 32, - .block_k = 128, - .group_m = 1, - .num_warps = 4, - .num_stages = 2, - }, - 48 => .{ - .block_m = 16, - .block_n = 32, - .block_k = 128, - .group_m = 64, - .num_warps = 4, - .num_stages = 2, - }, - 64 => .{ - .block_m = 16, - .block_n = 64, - .block_k = 128, - .group_m = 1, - .num_warps = 4, - .num_stages = 2, - }, - 96 => .{ - .block_m = 16, - .block_n = 128, - .block_k = 128, - .group_m = 1, - .num_warps = 8, - .num_stages = 3, - }, - 128 => .{ - .block_m = 16, - .block_n = 256, - .block_k = 128, - .group_m = 1, - .num_warps = 8, - .num_stages = 2, - }, - 256 => .{ - .block_m = 16, - .block_n = 256, - .block_k = 128, - .group_m = 1, - .num_warps = 8, - .num_stages = 2, - }, - 512 => .{ - .block_m = 32, - .block_n = 128, - .block_k = 128, - .group_m = 1, - .num_warps = 8, - .num_stages = 3, - }, - 1024 => .{ - .block_m = 64, - .block_n = 128, - .block_k = 64, - .group_m = 1, - .num_warps = 4, - .num_stages = 3, - }, - 1536 => .{ - .block_m = 64, - .block_n = 128, - .block_k = 64, - .group_m = 1, - .num_warps = 4, - .num_stages = 3, - }, - 2048 => .{ - .block_m = 128, - .block_n = 128, - .block_k = 64, - .group_m = 16, - .num_warps = 8, - .num_stages = 3, - }, - 3072 => .{ - .block_m = 128, - .block_n = 256, - .block_k = 64, - .group_m = 1, - .num_warps = 8, - .num_stages = 4, - }, - 4096 => .{ - .block_m = 128, - .block_n = 256, - .block_k = 64, - .group_m = 16, - .num_warps = 8, - .num_stages = 4, - }, - else => unreachable, - }; -} - -fn getBestConfig(num_tokens: u32, topk: u32, num_experts: u32) KernelConf { - const num_routes = std.math.mul(u32, num_tokens, topk) catch std.math.maxInt(u32); - var config = getBestTokenBucketConfig(num_routes); - - if (num_tokens <= 32 and num_routes <= 256 and num_experts <= 64) { - config.block_m = 16; - config.block_n = 256; - config.block_k = 128; - config.group_m = 1; - config.num_warps = 4; - config.num_stages = 2; - } else if (num_tokens <= 64 and num_routes <= 512 and num_experts <= 64) { - config.block_m = 16; - config.block_n = 128; - config.block_k = 128; - config.group_m = 1; - config.num_warps = 4; - config.num_stages = 2; - } - - return config; -} - -fn getBestTokenBucketConfig(num_tokens: u32) KernelConf { - var best_num_tokens = kernel_config_token_buckets[0]; - var best_distance = tokenDistance(num_tokens, best_num_tokens); - - for (kernel_config_token_buckets[1..]) |candidate| { - const distance = tokenDistance(num_tokens, candidate); - if (distance < best_distance or (distance == best_distance and candidate < best_num_tokens)) { - best_num_tokens = candidate; - best_distance = distance; - } - } - - return configForTokenBucket(best_num_tokens); -} - -fn tokenDistance(a: u32, b: u32) u32 { - return if (a >= b) a - b else b - a; -} - -const Fp4Routing = struct { - num_tokens: i64, - num_rows: i64, - topk: i64, - gather_divisor: i64, - grid_m: i64, - sorted_route_ids: zml.Tensor, - sorted_weights: zml.Tensor, - active_routes: zml.Tensor, - tile_experts: zml.Tensor, - tile_starts: zml.Tensor, - tile_ends: zml.Tensor, -}; - -fn prepareFp4Routing( - aligned: Routing, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - num_tokens: i64, - num_experts: i64, - block_m: i64, -) Fp4Routing { - const topk = topk_ids.dim(.topk); - const num_routes = aligned.num_assignments; - const num_rows = if (aligned.naive_block_assignment) - num_routes - else - aligned.max_num_tokens_padded; - const grid_m = if (aligned.naive_block_assignment) - num_routes - else - std.math.divCeil(i64, num_rows, block_m) catch unreachable; - - const sorted_route_candidates = if (aligned.naive_block_assignment) - zml.Tensor.arange(.{ .end = num_routes }, .i32).withTags(.{.route}) - else - aligned.sorted_token_ids.withTags(.{.route}); - const route_index_valid = sorted_route_candidates.cmp(.GE, zml.Tensor.scalar(0, .i32)) - .logical(.AND, sorted_route_candidates.cmp(.LT, zml.Tensor.scalar(num_routes, .i32))); - const sorted_route_ids = route_index_valid.select( - sorted_route_candidates, - zml.Tensor.zeroes(sorted_route_candidates.shape()), - ); - const gather_indices = sorted_route_ids.rename(.{ .route = .sorted_route }); - - const flat_expert_ids = topk_ids.flatten().withTags(.{.route}).convert(.i32); - const sorted_expert_ids = flat_expert_ids - .gather(.{ .route = gather_indices }, .{}) - .rename(.{ .sorted_route = .route }); - const active_routes = route_index_valid - .logical(.AND, sorted_expert_ids.cmp(.GE, zml.Tensor.scalar(0, .i32))) - .logical(.AND, sorted_expert_ids.cmp(.LT, zml.Tensor.scalar(num_experts, .i32))); - - const flat_weights = topk_weights.flatten().withTags(.{.route}); - const gathered_weights = flat_weights - .gather(.{ .route = gather_indices }, .{}) - .rename(.{ .sorted_route = .route }) - .convert(.f32); - const sorted_weights = active_routes.select(gathered_weights, zml.Tensor.zeroes(gathered_weights.shape())); - - const raw_tile_experts = aligned.expert_ids.withTags(.{.tile}).convert(.i32); - const valid_tile_experts = raw_tile_experts.cmp(.GE, zml.Tensor.scalar(0, .i32)) - .logical(.AND, raw_tile_experts.cmp(.LT, zml.Tensor.scalar(num_experts, .i32))); - const tile_experts = valid_tile_experts.select( - raw_tile_experts, - zml.Tensor.zeroes(raw_tile_experts.shape()), - ); - const tile_starts = if (aligned.naive_block_assignment) - zml.Tensor.arange(.{ .end = grid_m }, .i64).withTags(.{.tile}) - else - zml.Tensor.arange(.{ .end = grid_m }, .i64).withTags(.{.tile}).scale(block_m); - const tile_ends = if (aligned.naive_block_assignment) - valid_tile_experts.select(tile_starts.addConstant(1), tile_starts) - else blk: { - const num_tokens_post_padded = aligned.num_tokens_post_padded - .withTags(.{.tile}) - .convert(.i64) - .broad(tile_starts.shape()); - const active_tiles = valid_tile_experts.logical(.AND, tile_starts.cmp(.LT, num_tokens_post_padded)); - break :blk active_tiles.select( - tile_starts.addConstant(block_m).minimum(num_tokens_post_padded), - tile_starts, - ); - }; - - return .{ - .num_tokens = num_tokens, - .num_rows = num_rows, - .topk = topk, - .gather_divisor = topk, - .grid_m = grid_m, - .sorted_route_ids = sorted_route_ids, - .sorted_weights = sorted_weights, - .active_routes = active_routes, - .tile_experts = tile_experts, - .tile_starts = tile_starts, - .tile_ends = tile_ends, - }; -} - -const GemmOpts = struct { - routing: Fp4Routing, - weight_contract_tag: zml.Shape.Tag, - weight_output_tag: zml.Shape.Tag, - output_shape: zml.Shape, - gather: ?zml.Tensor = null, - gammas: ?zml.Tensor = null, - bias: ?zml.Tensor = null, - apply_swiglu: bool = false, - activation_limit: ?f32 = null, - block_m: u32, - block_n: u32, - block_k: u32, - group_m: u32, - num_warps: u32, - num_stages: u32, -}; -fn runGemm( - input: zml.Tensor, - weights: zml.Tensor, - scales: zml.Tensor, - opts: GemmOpts, -) !zml.Tensor { - const input_matrix = input.withTags(.{ .row, .k }); - const contract_k = input_matrix.dim(.k); - const packed_k = weights.dim(opts.weight_contract_tag); - const scale_k = scales.dim(opts.weight_contract_tag); - const n = weights.dim(opts.weight_output_tag); - - stdx.debug.assert(packed_k * 2 == contract_k, "expected packed int4 weight K {} to match activation K {}", .{ packed_k, contract_k }); - stdx.debug.assert(scale_k * 32 == contract_k, "expected MX scale K {} to match activation K {}", .{ scale_k, contract_k }); - const activation_reduction_n: i64 = if (opts.apply_swiglu) 2 else 1; - stdx.debug.assert(@mod(n, activation_reduction_n) == 0, "invalid GEMM output width {}", .{n}); - stdx.debug.assert(opts.output_shape.dim(-1) == @divExact(n, activation_reduction_n), "output shape {f} does not match GEMM N {}", .{ opts.output_shape, n }); - stdx.debug.assert(opts.bias == null, "MXFP4 Triton MoE GEMM bias is not wired yet", .{}); - - const block_m: i32 = @intCast(opts.block_m); - const block_n: i32 = @intCast(opts.block_n); - const block_k: i32 = @intCast(opts.block_k); - // TODO: update the kernel to support uneven K. - if (@mod(contract_k, block_k) != 0) return error.InvalidShape; - const grid_n = std.math.divCeil(i64, n, block_n) catch unreachable; - const has_gammas = opts.gammas != null; - const gathered_input = if (opts.gather) |gather| blk: { - const token_ids = gather.divByConst(opts.routing.gather_divisor).withTags(.{.route}); - break :blk input_matrix.gather(.{ .row = token_ids }, .{}).rename(.{ .route = .row }); - } else input_matrix; - const raw_output_shape = if (opts.apply_swiglu) - opts.output_shape.set(-1, n) - else - opts.output_shape; - - const cfg: a16w4_kernel.Cfg = .{ - .a_dtype = zml.kernel.triton.from(gathered_input.dtype()), - .wp_dtype = packedByteDtype(weights.dtype()), - .ws_dtype = packedByteDtype(scales.dtype()), - .c_dtype = zml.kernel.triton.from(raw_output_shape.dtype()), - .BLOCK_M = block_m, - .BLOCK_N = block_n, - .BLOCK_K = block_k, - .SPLIT_K = 1, - .GROUP_M = @intCast(opts.group_m), - .num_warps = @intCast(opts.num_warps), - .num_stages = @intCast(opts.num_stages), - }; - - var y = a16w4_kernel.Kernel.call( - .{ - .a_ptr = gathered_input, - .wp_ptr = weights, - .ws_ptr = scales, - .tile_expert_ptr = opts.routing.tile_experts, - .tile_mstart_ptr = opts.routing.tile_starts, - .tile_mend_ptr = opts.routing.tile_ends, - .NUM_M_TILES_ptr = scalarI64(opts.routing.grid_m), - .N_ptr = scalarI64(n), - .K_ptr = scalarI64(contract_k), - .stride_am_ptr = scalarI64(contract_k), - .stride_ak_ptr = scalarI64(1), - .stride_we_ptr = scalarI64(n * packed_k), - .stride_wk_ptr = scalarI64(1), - .stride_wn_ptr = scalarI64(packed_k), - .stride_se_ptr = scalarI64(n * scale_k), - .stride_sk_ptr = scalarI64(1), - .stride_sn_ptr = scalarI64(scale_k), - .stride_cm_ptr = scalarI64(raw_output_shape.dim(-1)), - .stride_cn_ptr = scalarI64(1), - }, - .{ .c = raw_output_shape }, - .{ - .cfg = cfg, - .grid = .{ @intCast(opts.routing.grid_m * grid_n), 1, 1 }, - .num_warps = @intCast(opts.num_warps), - .num_stages = @intCast(opts.num_stages), - }, - ).c; - - if (opts.apply_swiglu) { - y = applySwiGlu(y.convert(.f32), opts.activation_limit).convert(opts.output_shape.dtype()); - } - - if (has_gammas) { - const gammas = opts.gammas.?.convert(.f32).appendAxes(.{.dout}).broad(opts.output_shape.withDtype(.f32)); - y = y.convert(.f32).mul(gammas).convert(opts.output_shape.dtype()); - } - - return y; -} - -fn applySwiGlu(input: zml.Tensor, activation_limit: ?f32) zml.Tensor { - var gate = input.slice(.dout, .{ .start = 0, .step = 2 }); - var up = input.slice(.dout, .{ .start = 1, .step = 2 }); - - if (activation_limit) |limit| { - const threshold = zml.Tensor.scalar(limit, .f32); - gate = gate.minimum(threshold); - up = up.clamp(threshold.negate(), threshold); - } - - return gate.silu().mul(up); -} - -fn packedByteDtype(dt: zml.DataType) zml.kernel.triton.DType { - return switch (dt) { - .i8, .u8, .f4e2m1, .f8e8m0 => .i8, - else => zml.kernel.triton.from(dt), - }; -} - -fn scalarI64(v: i64) zml.Tensor { - return zml.Tensor.constant(.{ .i64 = v }).reshape(.{1}); -} diff --git a/zml/moe/triton_kernels/a16w4_kernel.zig b/zml/moe/triton_kernels/a16w4_kernel.zig deleted file mode 100644 index dd5f016d..00000000 --- a/zml/moe/triton_kernels/a16w4_kernel.zig +++ /dev/null @@ -1,189 +0,0 @@ -const std = @import("std"); - -const zml = @import("../../zml.zig"); -const tri = zml.kernel.triton; - -pub const Cfg = struct { - a_dtype: tri.DType = .bf16, - wp_dtype: tri.DType = .i8, - ws_dtype: tri.DType = .i8, - c_dtype: tri.DType = .bf16, - - BLOCK_M: i32 = 16, - BLOCK_N: i32 = 64, - BLOCK_K: i32 = 64, - SPLIT_K: i32 = 1, - GROUP_M: i32 = 1, - num_warps: i32 = 4, - num_stages: i32 = 2, -}; - -pub const Kernel = tri.Kernel(Cfg, .{ - .name = "_gemm_a16mxfp4", - .inputs = &.{ - "a_ptr", - "wp_ptr", - "ws_ptr", - "tile_expert_ptr", - "tile_mstart_ptr", - "tile_mend_ptr", - "NUM_M_TILES_ptr", - "N_ptr", - "K_ptr", - "stride_am_ptr", - "stride_ak_ptr", - "stride_we_ptr", - "stride_wk_ptr", - "stride_wn_ptr", - "stride_se_ptr", - "stride_sk_ptr", - "stride_sn_ptr", - "stride_cm_ptr", - "stride_cn_ptr", - }, - .outputs = &.{"c"}, - .run = run, -}); - -fn run(b: *tri.Builder, cfg: Cfg) tri.FinishError!void { - std.debug.assert(@mod(cfg.BLOCK_K, 32) == 0); - - const a = try b.declareArgs(.{ - .a_ptr = .{ .ptr = cfg.a_dtype }, - .wp_ptr = .{ .ptr = cfg.wp_dtype }, - .ws_ptr = .{ .ptr = cfg.ws_dtype }, - .tile_expert_ptr = .{ .ptr = .i32 }, - .tile_mstart_ptr = .{ .ptr = .i64 }, - .tile_mend_ptr = .{ .ptr = .i64 }, - .NUM_M_TILES_ptr = .{ .ptr = .i64 }, - .N_ptr = .{ .ptr = .i64 }, - .K_ptr = .{ .ptr = .i64 }, - .stride_am_ptr = .{ .ptr = .i64 }, - .stride_ak_ptr = .{ .ptr = .i64 }, - .stride_we_ptr = .{ .ptr = .i64 }, - .stride_wk_ptr = .{ .ptr = .i64 }, - .stride_wn_ptr = .{ .ptr = .i64 }, - .stride_se_ptr = .{ .ptr = .i64 }, - .stride_sk_ptr = .{ .ptr = .i64 }, - .stride_sn_ptr = .{ .ptr = .i64 }, - .stride_cm_ptr = .{ .ptr = .i64 }, - .stride_cn_ptr = .{ .ptr = .i64 }, - .c_ptr = .{ .ptr = cfg.c_dtype }, - }); - - const NUM_M_TILES = b.load(a.NUM_M_TILES_ptr); - const N_val = b.load(a.N_ptr); - const K_val = b.load(a.K_ptr); - const stride_am = b.load(a.stride_am_ptr); - const stride_ak = b.load(a.stride_ak_ptr); - const stride_we = b.load(a.stride_we_ptr); - const stride_wk = b.load(a.stride_wk_ptr); - const stride_wn = b.load(a.stride_wn_ptr); - const stride_se = b.load(a.stride_se_ptr); - const stride_sk = b.load(a.stride_sk_ptr); - const stride_sn = b.load(a.stride_sn_ptr); - const stride_cm = b.load(a.stride_cm_ptr); - const stride_cn = b.load(a.stride_cn_ptr); - - // L2-friendly swizzle: group GROUP_M consecutive m-tiles (usually the same - // expert) and sweep n within them, so each weight column tile is reused. - const pid = b.programId(.x); - const pid_k = b.programId(.y); - const num_pid_n = N_val.cdiv(cfg.BLOCK_N); - const width = b.liftAs(cfg.GROUP_M, .i64).mul(num_pid_n); - const pid_i64 = pid.to(.i64); - const group_id = pid_i64.div(width); - const group_size = NUM_M_TILES.sub(group_id.mul(cfg.GROUP_M)).minimum(cfg.GROUP_M); - const pid_m = group_id.mul(cfg.GROUP_M).add(pid_i64.rem(group_size)); - const pid_n = pid_i64.rem(width).div(group_size); - - const expert = b.load(a.tile_expert_ptr.addPtr(pid_m)); - const m_start = b.load(a.tile_mstart_ptr.addPtr(pid_m)); - const m_end = b.load(a.tile_mend_ptr.addPtr(pid_m)); - - const offs_m = m_start.add(b.arange(0, cfg.BLOCK_M, .i32)); - const offs_n = pid_n.mul(cfg.BLOCK_N).add(b.arange(0, cfg.BLOCK_N, .i32)); - const m_mask = offs_m.lt(m_end); - const n_mask = offs_n.lt(N_val); - - const BLOCK_K_PACKED: i32 = @divExact(cfg.BLOCK_K, 2); - const BLOCK_K_SCALE: i32 = @divExact(cfg.BLOCK_K, 32); - const offs_kh = b.arange(0, BLOCK_K_PACKED, .i32); // packed K rows within a tile - const offs_kg = b.arange(0, BLOCK_K_SCALE, .i32); // scale groups within a tile - - const wp_base = a.wp_ptr.addPtr(expert.mul(stride_we)); - const ws_base = a.ws_ptr.addPtr(expert.mul(stride_se)); - - const acc = b.zeros(&.{ cfg.BLOCK_M, cfg.BLOCK_N }, .f32); - const num_k_tiles = K_val.div(cfg.BLOCK_K); - const pid_k_i64 = pid_k.to(.i64); - var loop = b.openFor(pid_k_i64, num_k_tiles, cfg.SPLIT_K, .{acc}); - { - const t = loop.iv; - const acc_loop = loop.carried[0]; - - const k0 = t.mul(cfg.BLOCK_K); - const offs_ak = k0.add(b.arange(0, cfg.BLOCK_K, .i32)); - const a_mask = m_mask.expandDims(1); - const a_rows = offs_m.expandDims(1); - const a_row_offsets = a_rows.mul(stride_am); - const a_ptrs_col = a.a_ptr.addPtr(a_row_offsets); - const a_cols = offs_ak.expandDims(0); - const a_col_offsets = a_cols.mul(stride_ak); - const a_ptrs_base = b.broadcastTo(a_ptrs_col, &.{ cfg.BLOCK_M, cfg.BLOCK_K }); - const a_col_offsets_full = b.broadcastTo(a_col_offsets, &.{ cfg.BLOCK_M, cfg.BLOCK_K }); - const a_ptrs = a_ptrs_base.addPtr(a_col_offsets_full); - const a_mask_full = b.broadcastTo(a_mask, &.{ cfg.BLOCK_M, cfg.BLOCK_K }); - const a_tile = b.loadOpts(a_ptrs, .{ - .mask = a_mask_full, - .other = b.zeros(&.{ cfg.BLOCK_M, cfg.BLOCK_K }, cfg.a_dtype), - }); // (BM, BK) bf16 - - const pj = k0.div(2).add(offs_kh); - const w_mask = n_mask.expandDims(0); - const w_rows = pj.expandDims(1); - const w_row_offsets = w_rows.mul(stride_wk); - const w_ptrs_col = wp_base.addPtr(w_row_offsets); - const w_cols = offs_n.expandDims(0); - const w_col_offsets = w_cols.mul(stride_wn); - const w_ptrs_base = b.broadcastTo(w_ptrs_col, &.{ BLOCK_K_PACKED, cfg.BLOCK_N }); - const w_col_offsets_full = b.broadcastTo(w_col_offsets, &.{ BLOCK_K_PACKED, cfg.BLOCK_N }); - const w_ptrs = w_ptrs_base.addPtr(w_col_offsets_full); - const w_mask_full = b.broadcastTo(w_mask, &.{ BLOCK_K_PACKED, cfg.BLOCK_N }); - const w = b.loadOpts(w_ptrs, .{ - .mask = w_mask_full, - .other = b.zeros(&.{ BLOCK_K_PACKED, cfg.BLOCK_N }, cfg.wp_dtype), - }); // (BK//2, BN) uint8 - - const kg = k0.div(32).add(offs_kg); - // rhs_scale must be laid out [N, K//32] for dot_scaled - const sc_mask = n_mask.expandDims(1); - const sc_rows = offs_n.expandDims(1); - const sc_row_offsets = sc_rows.mul(stride_sn); - const sc_ptrs_col = ws_base.addPtr(sc_row_offsets); - const sc_cols = kg.expandDims(0); - const sc_col_offsets = sc_cols.mul(stride_sk); - const sc_ptrs_base = b.broadcastTo(sc_ptrs_col, &.{ cfg.BLOCK_N, BLOCK_K_SCALE }); - const sc_col_offsets_full = b.broadcastTo(sc_col_offsets, &.{ cfg.BLOCK_N, BLOCK_K_SCALE }); - const sc_ptrs = sc_ptrs_base.addPtr(sc_col_offsets_full); - const sc_mask_full = b.broadcastTo(sc_mask, &.{ cfg.BLOCK_N, BLOCK_K_SCALE }); - const sc = b.loadOpts(sc_ptrs, .{ - .mask = sc_mask_full, - .other = b.full(&.{ cfg.BLOCK_N, BLOCK_K_SCALE }, 127, cfg.ws_dtype), - }); // (BN, BK//32) e8m0 - const new_acc = b.dotScaledOpts(a_tile, w, acc_loop, null, sc, .bf16, .e2m1, .{ .fast_math = true }); - - loop.yield(.{new_acc}); - } - - const c_rows = offs_m.expandDims(1); - const c_row_offsets = c_rows.mul(stride_cm); - const c_ptrs_col = a.c_ptr.addPtr(c_row_offsets); - const c_cols = offs_n.expandDims(0); - const c_col_offsets = c_cols.mul(stride_cn); - const c_ptrs_base = b.broadcastTo(c_ptrs_col, &.{ cfg.BLOCK_M, cfg.BLOCK_N }); - const c_col_offsets_full = b.broadcastTo(c_col_offsets, &.{ cfg.BLOCK_M, cfg.BLOCK_N }); - const c_ptrs = c_ptrs_base.addPtr(c_col_offsets_full); - const c_mask = m_mask.expandDims(1).bitAnd(n_mask.expandDims(0)); - b.storeOpts(c_ptrs, loop.results[0].to(.bf16), .{ .mask = c_mask }); -} diff --git a/zml/moe/triton_kernels/triton_kernels.zig b/zml/moe/triton_kernels/triton_kernels.zig index bc7764f2..cfcffa4c 100644 --- a/zml/moe/triton_kernels/triton_kernels.zig +++ b/zml/moe/triton_kernels/triton_kernels.zig @@ -176,7 +176,7 @@ fn writeZerosToOutput( // fused_moe_kernel // ============================================================================= -/// Routed GEMM supporting BF16, MXFP8, and per-tensor, per-channel, or +/// Routed GEMM supporting BF16, MXFP4, MXFP8, and per-tensor, per-channel, or /// 128x128 block-scaled FP8 weights. pub const FusedMoe = struct { pub const Cfg = struct { @@ -214,8 +214,8 @@ pub const FusedMoe = struct { return error.InvalidMlir; } - if (cfg.quant_scheme == .mxfp8 and cfg.block_size_k % 32 != 0) { - log.err("fused_moe_kernel: MXFP8 requires BLOCK_SIZE_K divisible by 32", .{}); + if ((cfg.quant_scheme == .mxfp4 or cfg.quant_scheme == .mxfp8) and cfg.block_size_k % 32 != 0) { + log.err("fused_moe_kernel: MXFP4/MXFP8 require BLOCK_SIZE_K divisible by 32", .{}); return error.InvalidMlir; } @@ -252,6 +252,8 @@ pub const FusedMoe = struct { const block_size_m: i64 = @intCast(cfg.block_size_m); const block_size_n: i64 = @intCast(cfg.block_size_n); const block_size_k: i64 = @intCast(cfg.block_size_k); + const weight_packing: i64 = if (cfg.quant_scheme == .mxfp4) 2 else 1; + const block_size_bk = @divExact(block_size_k, weight_packing); const group_size_m: i64 = @intCast(cfg.group_size_m); const top_k: i64 = @intCast(cfg.top_k); const compute_type = cfg.compute_type; @@ -362,12 +364,14 @@ pub const FusedMoe = struct { // addptr) — matches Python's left-to-right evaluation. Defer extsi // for offs_k to after offs_bn's expand_dims/mul (Python emit order). const b_ptr_shifted = a.b_ptr.addPtr(off_experts.mul(stride_be_block)); - const offs_k_col_i32 = b.expandDims(offs_k_i32, 1); + // MXFP4 packs two K values per byte; activation K remains unpacked. + const offs_bk_i32 = b.arange(0, block_size_bk, .i32); + const offs_k_col_i32 = b.expandDims(offs_bk_i32, 1); const offs_bn_row = b.expandDims(offs_bn, 0); const bn_row = offs_bn_row.mul(stride_bn_block); const offs_k_col = offs_k_col_i32.to(.i64); - const bk_term = b.broadcastTo(offs_k_col, &.{ block_size_k, block_size_n }); - const bn_term = b.broadcastTo(bn_row, &.{ block_size_k, block_size_n }); + const bk_term = b.broadcastTo(offs_k_col, &.{ block_size_bk, block_size_n }); + const bn_term = b.broadcastTo(bn_row, &.{ block_size_bk, block_size_n }); const b_ptrs_init = b_ptr_shifted.addPtr(bk_term.add(bn_term)); const acc_init = b.zeros(&.{ block_size_m, block_size_n }, .f32); @@ -397,18 +401,17 @@ pub const FusedMoe = struct { .other = b.zeros(&.{ block_size_m, block_size_k }, cfg.a_dtype), }); - // mask_b = offs_k[:, None] < k_remaining (broadcast to KxN). - const offs_k_col_b = b.expandDims(offs_k_i32, 1).to(.i64); - const offs_k_lt_b = offs_k_col_b.lt(b.splat(k_remaining, &.{ block_size_k, 1 })); - const mask_b = b.broadcastTo(offs_k_lt_b, &.{ block_size_k, block_size_n }); + const offs_k_col_b = b.expandDims(offs_bk_i32, 1).to(.i64); + const offs_k_lt_b = offs_k_col_b.lt(b.splat(k_remaining.cdiv(weight_packing), &.{ block_size_bk, 1 })); + const mask_b = b.broadcastTo(offs_k_lt_b, &.{ block_size_bk, block_size_n }); const b_val = b.loadOpts(b_ptrs, .{ .mask = mask_b, - .other = b.zeros(&.{ block_size_k, block_size_n }, cfg.b_dtype), + .other = b.zeros(&.{ block_size_bk, block_size_n }, cfg.b_dtype), }); const new_acc = if (cfg.quant_scheme) |scheme| switch (scheme) { - .mxfp8 => scaled: { + .mxfp4, .mxfp8 => scaled: { const scale_k = @divExact(block_size_k, 32); const groups = k_iter.mul(scale_k).add(b.arange(0, scale_k, .i64)); const stride_bse = b.load(a.stride_bse_ptr); @@ -437,7 +440,7 @@ pub const FusedMoe = struct { .other = b.full(&.{ block_size_m, scale_k }, 127, .i8), }); } else null; - break :scaled b.dotScaledOpts(a_val, b_val, acc, a_scales, scales, if (a_scales != null) .e4m3 else .bf16, .e4m3, .{}); + break :scaled b.dotScaledOpts(a_val, b_val, acc, a_scales, scales, if (a_scales != null) .e4m3 else .bf16, if (scheme == .mxfp4) .e2m1 else .e4m3, .{}); }, .fp8_per_channel, .fp8_per_tensor => b.dotOpts(a_val, if (cfg.a_scale_dtype != null) b_val else b_val.to(.bf16), acc, .{ .input_precision = .tf32, @@ -477,7 +480,7 @@ pub const FusedMoe = struct { } break :scaled acc.add(scaled_dot); }, - .mxfp4, .nvfp4 => unreachable, + .nvfp4 => unreachable, } else b.dotOpts(a_val, b_val, acc, .{ .input_precision = .tf32, .max_num_imprecise_acc = 0, @@ -486,7 +489,7 @@ pub const FusedMoe = struct { // BLOCK_SIZE_K is `tl.constexpr` → i32 dense splat (stride_ak == 1). const bsk_i32: i32 = @intCast(cfg.block_size_k); const new_a_ptrs = a_ptrs.addPtr(b.splat(bsk_i32, &.{ block_size_m, block_size_k })); - const new_b_ptrs = b_ptrs.addPtr(b.splat(bsk_i32, &.{ block_size_k, block_size_n })); + const new_b_ptrs = b_ptrs.addPtr(b.splat(@as(i32, @intCast(block_size_bk)), &.{ block_size_bk, block_size_n })); loop.yield(.{ new_a_ptrs, @@ -575,21 +578,21 @@ test "FP8 activation quantization emits float and E8M0 scales" { } } -test "FusedMoe emits each FP8 scaling path" { +test "FusedMoe emits each FP8 and MXFP4 scaling path" { const allocator = std.testing.allocator; - const schemes = [_]?zml.Quantization.Scheme{ null, .mxfp8, .fp8_per_tensor, .fp8_per_channel, .fp8_block128 }; + const schemes = [_]?zml.Quantization.Scheme{ null, .mxfp4, .mxfp8, .fp8_per_tensor, .fp8_per_channel, .fp8_block128 }; for (schemes) |scheme| { for ([_]DType{ .f8e4m3fn, .f8e4m3fnuz }) |fp8_dtype| { - if (scheme == .mxfp8 and fp8_dtype == .f8e4m3fnuz) continue; + if ((scheme == .mxfp4 or scheme == .mxfp8) and fp8_dtype == .f8e4m3fnuz) continue; for ([_]bool{ false, true }) |quantize_input| { - if (scheme == null and quantize_input) continue; + if ((scheme == null or scheme == .mxfp4) and quantize_input) continue; for ([_]?DType{ null, .bf16, .f32 }) |bias_dtype| { const ir = try FusedMoe.Kernel.emit(allocator, .{ .a_dtype = if (quantize_input) fp8_dtype else .bf16, - .b_dtype = if (scheme != null) fp8_dtype else .bf16, + .b_dtype = if (scheme == .mxfp4) .i8 else if (scheme != null) fp8_dtype else .bf16, .c_dtype = .bf16, .a_scale_dtype = if (quantize_input) (if (scheme == .mxfp8) .i8 else .f32) else null, - .b_scale_dtype = if (scheme == .mxfp8) .i8 else if (scheme != null) .f32 else null, + .b_scale_dtype = if (scheme == .mxfp4 or scheme == .mxfp8) .i8 else if (scheme != null) .f32 else null, .b_bias_dtype = bias_dtype, .routing_weights_dtype = .bf16, .block_size_m = 16, @@ -602,7 +605,7 @@ test "FusedMoe emits each FP8 scaling path" { .quant_scheme = scheme, }); defer allocator.free(ir); - if (scheme == .mxfp8) { + if (scheme == .mxfp4 or scheme == .mxfp8) { try std.testing.expect(std.mem.indexOf(u8, ir, "tt.dot_scaled") != null); } else { try std.testing.expect(std.mem.indexOf(u8, ir, "tt.dot_scaled") == null); From f75f5bd27eb59528a0ff9afd54db90f3f08c298d Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Fri, 11 Sep 2026 13:07:45 +0200 Subject: [PATCH 678/881] zml/Compiler: Allow providing a Progress.Node (#798) --- zml/Compiler.zig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zml/Compiler.zig b/zml/Compiler.zig index d795e9ee..6a749e81 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -68,6 +68,8 @@ fn mlirRegistry(io: std.Io) *mlir.DialectRegistry { } pub const Options = struct { + /// Incremented once after successful compilation. + progress: std.Progress.Node = .none, shardings: []const Sharding = &.{}, // If null, will be initialized from the target partitioner: ?Sharding.Partitioner = null, @@ -346,6 +348,7 @@ pub fn compile( ); errdefer exe.deinit(); + opts.progress.completeOne(); return exe; } From ecb824e85989107f870fa743264c5b87891950fa Mon Sep 17 00:00:00 2001 From: Alhussein Jamil <85727835+alhussein-jamil@users.noreply.github.com> Date: Fri, 11 Sep 2026 16:29:29 +0200 Subject: [PATCH 679/881] bin/zml-smi: keep NVML polling on one thread so --top does not freeze (#766) ## Summary - `zml-smi --top` was stalling under load because each GPU and the process list hit NVML on their own thread. Those calls serialize on the driver RM lock, so one slow ioctl froze the whole TUI. - All NVIDIA queries now run on a single poller. `nvmlDeviceGetPcieThroughput` (it blocks while sampling) is only taken on the first tick and every fourth tick after that; link speed/width stay on the hot path. - Process-query sizes above 4096 are ignored, and a failed per-process util sample no longer skips the rest of that GPU. --------- Co-authored-by: alhussein jamil --- bin/zml-smi/info/poll_metrics.zig | 12 +++-- bin/zml-smi/platforms/nvml/metrics.zig | 65 ++++++++++++++++++++++---- bin/zml-smi/platforms/nvml/nvml.zig | 23 ++++----- bin/zml-smi/platforms/nvml/process.zig | 15 ++---- 4 files changed, 77 insertions(+), 38 deletions(-) diff --git a/bin/zml-smi/info/poll_metrics.zig b/bin/zml-smi/info/poll_metrics.zig index f0d916ff..62ed94a7 100644 --- a/bin/zml-smi/info/poll_metrics.zig +++ b/bin/zml-smi/info/poll_metrics.zig @@ -1,5 +1,11 @@ const std = @import("std"); +pub fn apply(comptime table: anytype, dest: anytype, src: anytype) void { + inline for (table) |m| { + @field(dest, m.field) = m.query(src) catch null; + } +} + pub fn poll(comptime DB: type, comptime Dev: type, comptime table: anytype) fn (?*std.heap.ArenaAllocator, DB, Dev) void { return struct { fn f(arena: ?*std.heap.ArenaAllocator, db: DB, dev: Dev) void { @@ -7,11 +13,7 @@ pub fn poll(comptime DB: type, comptime Dev: type, comptime table: anytype) fn ( const back = db.back(); back.* = db.front().*; - - inline for (table) |m| { - @field(back, m.field) = m.query(dev) catch null; - } - + apply(table, back, dev); db.swap(); } }.f; diff --git a/bin/zml-smi/platforms/nvml/metrics.zig b/bin/zml-smi/platforms/nvml/metrics.zig index 4e0ee2bd..213dd253 100644 --- a/bin/zml-smi/platforms/nvml/metrics.zig +++ b/bin/zml-smi/platforms/nvml/metrics.zig @@ -1,13 +1,30 @@ const std = @import("std"); const Nvml = @import("nvml.zig"); const device_info = @import("zml-smi/info").device_info; -const DeviceInfo = device_info.DeviceInfo; const GpuInfo = device_info.GpuInfo; +const poll_metrics = @import("zml-smi/info").poll_metrics; const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; const Collector = @import("zml-smi/collector").Collector; -const poll_metrics = @import("zml-smi/info").poll_metrics; const process = @import("process.zig"); +const pcie_every: u8 = 4; + +const Slot = struct { + info: *DoubleBuffer(GpuInfo), + dev: Device, +}; + +const Ctx = struct { + gpa: std.mem.Allocator, + nvml: *const Nvml, + slots: []Slot, + processes: *process.ProcessDoubleBuffer, + last_seen_ts: []u64, + dev_offset: u16, + device_count: u32, + tick: u8 = 0, +}; + pub fn start(collector: *Collector) !void { const nvml = try collector.arena.create(Nvml); nvml.* = try Nvml.init(); @@ -15,6 +32,7 @@ pub fn start(collector: *Collector) !void { const count = try nvml.deviceCount(); const dev_offset: u16 = @intCast(collector.device_infos.items.len); + var slots: std.ArrayList(Slot) = try .initCapacity(collector.arena, count); for (0..count) |i| { const dev = Device.open(nvml, @intCast(i)) catch continue; const initial: GpuInfo = .{ @@ -23,14 +41,42 @@ pub fn start(collector: *Collector) !void { .cuda_driver_version = dev.cudaDriverVersion(collector.arena) catch null, }; const info = try collector.addDevice(.{ .cuda = .{ .values = .{ initial, initial } } }); - try collector.spawnPoll(pollOnce, .{ null, &info.cuda, dev }); + slots.appendAssumeCapacity(.{ .info = &info.cuda, .dev = dev }); } - const processes = try collector.createProcessList(); - try process.init(collector, processes, nvml, dev_offset); + const last_seen_ts = try collector.arena.alloc(u64, count); + @memset(last_seen_ts, 0); + + const ctx = try collector.arena.create(Ctx); + ctx.* = .{ + .gpa = collector.gpa, + .nvml = nvml, + .slots = slots.items, + .processes = try collector.createProcessList(), + .last_seen_ts = last_seen_ts, + .dev_offset = dev_offset, + .device_count = count, + }; + try collector.spawnPoll(pollOnce, .{ctx}); } -const pollOnce = poll_metrics.poll(*DoubleBuffer(GpuInfo), Device, metrics); +fn pollOnce(ctx: *Ctx) void { + const pcie = ctx.tick % pcie_every == 0; + ctx.tick +%= 1; + + for (ctx.slots) |slot| { + pollDevice(slot.info, slot.dev, pcie); + } + process.pollOnce(ctx.gpa, ctx.processes, ctx.nvml, ctx.dev_offset, ctx.device_count, ctx.last_seen_ts); +} + +fn pollDevice(db: *DoubleBuffer(GpuInfo), dev: Device, pcie: bool) void { + const back = db.back(); + back.* = db.front().*; + poll_metrics.apply(metrics, back, dev); + if (pcie) poll_metrics.apply(pcie_metrics, back, dev); + db.swap(); +} const Device = struct { nvml: *const Nvml, @@ -165,9 +211,12 @@ const metrics = .{ .{ .field = "mem_used_bytes", .query = Device.memUsed }, .{ .field = "mem_total_bytes", .query = Device.memTotal }, .{ .field = "mem_bus_width", .query = Device.memBusWidth }, - .{ .field = "pcie_tx_kbps", .query = Device.pcieTx }, - .{ .field = "pcie_rx_kbps", .query = Device.pcieRx }, .{ .field = "pcie_bandwidth_mbps", .query = Device.pcieBandwidth }, .{ .field = "pcie_link_gen", .query = Device.pcieLinkGen }, .{ .field = "pcie_link_width", .query = Device.pcieLinkWidth }, }; + +const pcie_metrics = .{ + .{ .field = "pcie_tx_kbps", .query = Device.pcieTx }, + .{ .field = "pcie_rx_kbps", .query = Device.pcieRx }, +}; diff --git a/bin/zml-smi/platforms/nvml/nvml.zig b/bin/zml-smi/platforms/nvml/nvml.zig index 87d064ce..e2b36982 100644 --- a/bin/zml-smi/platforms/nvml/nvml.zig +++ b/bin/zml-smi/platforms/nvml/nvml.zig @@ -211,6 +211,13 @@ pub fn memBusWidth(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { return width; } +const max_query = 4096; + +fn sizedCount(count: c_uint) ?c_uint { + if (count == 0 or count > max_query) return null; + return count; +} + pub fn computeRunningProcesses(self: Nvml, allocator: std.mem.Allocator, handle: c.nvmlDevice_t) (Error || error{OutOfMemory})![]c.nvmlProcessInfo_t { var count: c_uint = 0; const ret = self.lib.nvmlDeviceGetComputeRunningProcesses_v3(handle, &count, null); @@ -220,9 +227,7 @@ pub fn computeRunningProcesses(self: Nvml, allocator: std.mem.Allocator, handle: } try check(ret); } - if (count == 0) { - return &.{}; - } + count = sizedCount(count) orelse return &.{}; const infos = try allocator.alloc(c.nvmlProcessInfo_t, count); errdefer allocator.free(infos); try check(self.lib.nvmlDeviceGetComputeRunningProcesses_v3(handle, &count, @ptrCast(infos.ptr))); @@ -240,15 +245,11 @@ pub fn graphicsRunningProcesses(self: Nvml, allocator: std.mem.Allocator, handle try check(ret); } - if (count == 0) { - return &.{}; - } - + count = sizedCount(count) orelse return &.{}; const infos = try allocator.alloc(c.nvmlProcessInfo_t, count); errdefer allocator.free(infos); try check(self.lib.nvmlDeviceGetGraphicsRunningProcesses_v3(handle, &count, @ptrCast(infos.ptr))); - return infos; } @@ -263,15 +264,11 @@ pub fn processUtilization(self: Nvml, allocator: std.mem.Allocator, handle: c.nv try check(ret); } - if (count == 0) { - return &.{}; - } - + count = sizedCount(count) orelse return &.{}; const samples = try allocator.alloc(c.nvmlProcessUtilizationSample_t, count); errdefer allocator.free(samples); try check(self.lib.nvmlDeviceGetProcessUtilization(handle, @ptrCast(samples.ptr), &count, last_seen)); - return samples; } diff --git a/bin/zml-smi/platforms/nvml/process.zig b/bin/zml-smi/platforms/nvml/process.zig index 3250a047..5e18cdcb 100644 --- a/bin/zml-smi/platforms/nvml/process.zig +++ b/bin/zml-smi/platforms/nvml/process.zig @@ -1,18 +1,10 @@ const std = @import("std"); const Nvml = @import("nvml.zig"); const pi = @import("zml-smi/info").process_info; -const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); -const Collector = @import("zml-smi/collector").Collector; -pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, nvml: *const Nvml, dev_offset: u16) !void { - const device_count: u32 = nvml.deviceCount() catch 0; - const last_seen_ts = try collector.arena.alloc(u64, device_count); - @memset(last_seen_ts, 0); +pub const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); - try collector.spawnPoll(pollOnce, .{ collector.gpa, list, nvml, dev_offset, device_count, last_seen_ts }); -} - -fn pollOnce(allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, nvml: *const Nvml, dev_offset: u16, device_count: u32, last_seen_ts: []u64) void { +pub fn pollOnce(allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, nvml: *const Nvml, dev_offset: u16, device_count: u32, last_seen_ts: []u64) void { const back = list.back(); back.clearRetainingCapacity(); @@ -29,9 +21,8 @@ fn pollOnce(allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, nvml: *con defer if (graphics.len > 0) allocator.free(graphics); collectFromQuery(allocator, back, idx, graphics); - // Apply utilization samples const last_ts = last_seen_ts[dev_idx]; - const utils = nvml.processUtilization(allocator, handle, last_ts) catch continue; + const utils = nvml.processUtilization(allocator, handle, last_ts) catch &.{}; defer if (utils.len > 0) allocator.free(utils); for (utils) |sample| { From 4788ccc2841232c95ff3745287d5c62b9bb233b9 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 11 Sep 2026 18:28:51 +0200 Subject: [PATCH 680/881] zml/moe: add `triton_mxfp4` backend (#801) This is a kernel that is quite specific to Deepseek V4 Flash Ideally, we would want to unify the triton kernels. --- zml/BUILD.bazel | 2 + zml/moe/moe.zig | 12 + zml/moe/triton_kernels/mxfp4.zig | 378 +++++++++++++++++++++++++++++++ zml/moe/triton_mxfp4.zig | 123 ++++++++++ 4 files changed, 515 insertions(+) create mode 100644 zml/moe/triton_kernels/mxfp4.zig create mode 100644 zml/moe/triton_mxfp4.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index b3125fc5..35587abf 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -100,6 +100,7 @@ zig_library( "mem.zig", "meta.zig", "mlirx.zig", + "moe/triton_mxfp4.zig", "moe/cutlass_flashinfer.zig", "moe/metal.zig", "moe/moe.zig", @@ -107,6 +108,7 @@ zig_library( "moe/mosaic_tpu_kernels/gmm_ep.zig", "moe/triton.zig", "moe/triton_kernels/triton_kernels.zig", + "moe/triton_kernels/mxfp4.zig", "nn.zig", "ops.zig", "pjrtx.zig", diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index 6e6a0a68..7c883cd0 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -4,6 +4,7 @@ const platforms = @import("platforms"); const zml = @import("../zml.zig"); const stdx = zml.stdx; +pub const triton_mxfp4 = @import("triton_mxfp4.zig"); pub const cutlass_flashinfer = @import("cutlass_flashinfer.zig"); pub const metal = @import("metal.zig"); pub const mosaic_tpu = @import("mosaic_tpu.zig"); @@ -21,6 +22,7 @@ pub const ActivationMode = enum { }; pub const Backend = enum { + triton_mxfp4, flashinfer_cutlass, triton, mosaic_tpu, @@ -62,6 +64,7 @@ pub const Backend = enum { pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { return switch (backend) { + .triton_mxfp4 => triton_mxfp4.isAvailable(platform), .flashinfer_cutlass => cutlass_flashinfer.isAvailable(platform), .triton => switch (platform.target) { .cuda, .rocm, .oneapi => true, @@ -74,6 +77,7 @@ pub const Backend = enum { pub fn register(backend: Backend, platform: *zml.Platform) !void { return switch (backend) { + .triton_mxfp4 => {}, .flashinfer_cutlass => cutlass_flashinfer.register(platform), .triton => {}, .mosaic_tpu => {}, @@ -83,12 +87,14 @@ pub const Backend = enum { }; pub const Parameters = union(Backend) { + triton_mxfp4: triton_mxfp4.Parameters, flashinfer_cutlass: cutlass_flashinfer.Parameters, triton: triton.Parameters, mosaic_tpu: mosaic_tpu.Parameters, metal: metal.Parameters, pub const InitOptions = union(Backend) { + triton_mxfp4: triton_mxfp4.Parameters.InitOptions, flashinfer_cutlass: cutlass_flashinfer.Parameters.InitOptions, triton: triton.Parameters.InitOptions, mosaic_tpu: mosaic_tpu.Parameters.InitOptions, @@ -96,6 +102,7 @@ pub const Parameters = union(Backend) { pub fn fromBackend(backend: Backend, num_experts_per_tok: u32, activation: ActivationMode) InitOptions { return switch (backend) { + .triton_mxfp4 => .{ .triton_mxfp4 = .{ .num_experts_per_tok = num_experts_per_tok, .activation = activation } }, .flashinfer_cutlass => .{ .flashinfer_cutlass = .{ .num_experts_per_tok = num_experts_per_tok, .activation = switch (activation) { @@ -134,6 +141,7 @@ pub const Parameters = union(Backend) { pub fn init(opts: InitOptions) Parameters { return switch (opts) { + .triton_mxfp4 => |v| .{ .triton_mxfp4 = triton_mxfp4.Parameters.init(v) }, .flashinfer_cutlass => |v| .{ .flashinfer_cutlass = cutlass_flashinfer.Parameters.init(v) }, .triton => |v| .{ .triton = triton.Parameters.init(v) }, .mosaic_tpu => |v| .{ .mosaic_tpu = mosaic_tpu.Parameters.init(v) }, @@ -169,6 +177,9 @@ pub fn forwardMoe( stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); stdx.debug.assert(opts.activation_threshold == null, "Activation thresholds require the Triton MoE backend", .{}); }, + .triton_mxfp4 => { + stdx.debug.assert(opts.gate_up_layout == .interleaved, "Triton MXFP4 MoE backends require interleaved gate/up columns", .{}); + }, } const gate_up_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; @@ -182,6 +193,7 @@ pub fn forwardMoe( const quant_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; return switch (parameters) { + .triton_mxfp4 => |p| triton_mxfp4.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts, p), .flashinfer_cutlass => b: { if (comptime !platforms.isEnabled(.cuda)) { return error.UnsupportedPlatform; diff --git a/zml/moe/triton_kernels/mxfp4.zig b/zml/moe/triton_kernels/mxfp4.zig new file mode 100644 index 00000000..75359fce --- /dev/null +++ b/zml/moe/triton_kernels/mxfp4.zig @@ -0,0 +1,378 @@ +//! FP4 x FP8 MoE emitted through the ZML Triton DSL. +//! BF16 activations use per-32 FP8 scaling; weights retain per-32 E8M0 scales. +//! Gate/up weight and scale rows are interleaved gate-first. +const std = @import("std"); +const zml = @import("../../zml.zig"); +const tri = zml.kernel.triton; +const B = tri.Builder; +const V = tri.Value; + +pub const Config = struct { + tokens: i64, + hidden: i64, + intermediate: i64, + experts: i64, + global_experts: i64, + topk: i64, + expert_offset: i64 = 0, + // Swiglu clamping. Set to 0 to disable. + swiglu_limit: f32, + /// Scale SwiGLU before FP8 quantization, or scale the down result before reduction. + routing_weight_placement: zml.moe.triton.RoutingWeightPlacement = .before_down, + + pub fn validate(c: Config) !void { + if (c.tokens <= 0 or + c.experts <= 0 or + c.topk <= 0 or c.topk > c.global_experts or + c.expert_offset < 0 or c.expert_offset + c.experts > c.global_experts) + { + return error.InvalidRoutingShape; + } + + if (c.hidden < 128 or @mod(c.hidden, 128) != 0 or + c.intermediate < 128 or @mod(c.intermediate, 128) != 0) + { + return error.InvalidHiddenShape; + } + + if (!std.math.isFinite(c.swiglu_limit) or c.swiglu_limit < 0) { + return error.InvalidScale; + } + } +}; + +pub const Inputs = struct { + x: zml.Tensor, + ids: zml.Tensor, + scales: zml.Tensor, + w1: zml.Tensor, + s1: zml.Tensor, + w2: zml.Tensor, + s2: zml.Tensor, +}; + +pub fn validateInputs(c: Config, a: Inputs) !void { + if (a.s1.rank() != 3 or a.s2.rank() != 3) return error.InvalidInputShape; + if (a.s1.dim(2) < @divTrunc(c.hidden, 32) or + a.s2.dim(2) < @divTrunc(c.intermediate, 32)) return error.InvalidInputShape; + const shapes = .{ + .x = zml.Shape.init(.{ c.tokens, c.hidden }, .bf16), + .ids = zml.Shape.init(.{ c.tokens, c.topk }, .i32), + .scales = zml.Shape.init(.{ c.tokens, c.topk }, .f32), + .w1 = zml.Shape.init(.{ c.experts, 2 * c.intermediate, @divTrunc(c.hidden, 2) }, .u8), + .s1 = zml.Shape.init(.{ c.experts, 2 * c.intermediate, a.s1.dim(2) }, .f8e8m0), + .w2 = zml.Shape.init(.{ c.experts, c.hidden, @divTrunc(c.intermediate, 2) }, .u8), + .s2 = zml.Shape.init(.{ c.experts, c.hidden, a.s2.dim(2) }, .f8e8m0), + }; + inline for (std.meta.fields(Inputs)) |field| { + if (!@field(a, field.name).shape().eql(@field(shapes, field.name))) return error.InvalidInputShape; + } +} + +const bn = 16; +const bk = 512; + +/// Computes an upper bound on the number of expert tiles needed. +/// When there is only one token, reserve a single tile per selected expert. +fn tiles(c: Config) i64 { + if (c.tokens == 1) return c.topk; + return @min(c.tokens * c.topk, @divTrunc(c.tokens * c.topk + c.experts * (bn - 1) + bn - 1, bn)); +} + +/// Returns the next power of two. +fn pow2(x: i64) i64 { + return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(x)) catch unreachable); +} + +fn quantRows(c: Config) i64 { + return if (c.tokens >= 16) 16 else 1; +} + +fn combineBlock(c: Config) i64 { + // Keep at least 512 CTAs before increasing per-CTA reduction work. + const elements = c.tokens * c.hidden; + if (c.tokens > 1 and elements >= 512 * 2048) return 2048; + if (c.tokens > 1 and elements >= 512 * 1024) return 1024; + return 256; +} + +fn ar(b: *B, n: i64) V { + return b.arange(0, n, .i32); +} + +fn ci(b: *B, n: i64) V { + return b.liftAs(n, .i32); +} + +fn ld(b: *B, p: V, mask: V, dt: tri.DType, other: i64) V { + return b.loadOpts(p, .{ .mask = mask, .other = if (p.isTensor()) b.full(p.shape().constSlice(), other, dt) else b.liftAs(other, dt) }); +} + +// Scale rows can be tightly packed or padded. Specialize the GEMM on their +// physical width without imposing an alignment requirement on the tensor. +const GemmConfig = struct { + moe: Config, + scale_stride: i64, +}; + +const Quant = tri.Kernel(Config, .{ .name = "mxfp4_triton_quant", .inputs = &.{"x"}, .outputs = &.{ "q", "s" }, .run = quantInput }); +const Route = tri.Kernel(Config, .{ .name = "mxfp4_triton_route", .inputs = &.{"ids"}, .outputs = &.{ "counts", "pos" }, .run = route }); +const Schedule = tri.Kernel(Config, .{ .name = "mxfp4_triton_schedule", .inputs = &.{ "ids", "counts", "pos" }, .outputs = &.{ "map", "perm", "sched" }, .run = schedule }); +const GateUp = tri.Kernel(GemmConfig, .{ .name = "mxfp4_triton_up", .inputs = &.{ "q", "s", "w", "ws", "ids", "rw", "map", "sched" }, .outputs = &.{ "mid", "ms" }, .run = gateUp }); +const Down = tri.Kernel(GemmConfig, .{ .name = "mxfp4_triton_down", .inputs = &.{ "mid", "ms", "w", "ws", "ids", "sched" }, .outputs = &.{"d"}, .run = down }); +const Combine = tri.Kernel(Config, .{ .name = "mxfp4_triton_combine", .inputs = &.{ "d", "ids", "perm", "rw" }, .outputs = &.{"y"}, .run = combine }); + +pub fn forward(c: Config, a: Inputs) zml.Tensor { + c.validate() catch @panic("Invalid MXFP4 MoE configuration"); + validateInputs(c, a) catch @panic("Invalid MXFP4 tensor layout"); + + const tile_count = tiles(c); + + // Quantize activations + const q = Quant.call( + .{ .x = a.x }, + .{ + .q = .init(.{ c.tokens, c.hidden }, .f8e4m3fn), + .s = .init(.{ c.tokens, @divTrunc(c.hidden, 32) }, .u8), + }, + .{ + .cfg = c, + .grid = .{ @intCast(@divTrunc(c.tokens + quantRows(c) - 1, quantRows(c))), @intCast(@divTrunc(c.hidden, 128)), 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ); + + const dummy = zml.Tensor.scalar(0, .i32); + var map = dummy; + var perm = dummy; + var sched = dummy; + + if (c.tokens != 1) { + const r = Route.call( + .{ .ids = a.ids }, + .{ .counts = .init(.{c.experts}, .i32), .pos = .init(.{c.tokens * c.topk}, .i32) }, + .{ .cfg = c, .grid = .{ @intCast(c.experts), 1, 1 }, .num_warps = 4, .num_stages = 1 }, + ); + + const s = Schedule.call( + .{ .ids = a.ids, .counts = r.counts, .pos = r.pos }, + .{ .map = .init(.{tile_count * bn}, .i32), .perm = .init(.{c.tokens * c.topk}, .i32), .sched = .init(.{2 * tile_count + 1}, .i32) }, + .{ + .cfg = c, + .grid = .{ @intCast(@divTrunc(c.tokens * c.topk + 127, 128)), 1, 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ); + + map = s.map; + perm = s.perm; + sched = s.sched; + } + + // Each narrow tile produces 64 activations: two complete scale groups. + const up_columns: i64 = if (c.tokens <= 8) 64 else 128; + const m = GateUp.call( + .{ .q = q.q, .s = q.s, .w = a.w1, .ws = a.s1.bitCast(.u8), .ids = a.ids, .rw = a.scales, .map = map, .sched = sched }, + .{ .mid = .init(.{ tile_count * bn, c.intermediate }, .f8e4m3fn), .ms = .init(.{ tile_count * bn, @divTrunc(c.intermediate, 32) }, .u8) }, + .{ + .cfg = .{ .moe = c, .scale_stride = a.s1.dim(2) }, + .grid = .{ @intCast(@divTrunc(c.intermediate, up_columns)), @intCast(tile_count), 1 }, + .num_warps = if (c.tokens <= 8) 4 else 8, + .num_stages = 2, + .global_scratch_memory_size = @intCast(128 * @divTrunc(c.intermediate, up_columns) * tile_count), + }, + ); + + const d = Down.call( + .{ .mid = m.mid, .ms = m.ms, .w = a.w2, .ws = a.s2.bitCast(.u8), .ids = a.ids, .sched = sched }, + .{ .d = .init(.{ tile_count * bn, c.hidden }, .bf16) }, + .{ + .cfg = .{ .moe = c, .scale_stride = a.s2.dim(2) }, + .grid = .{ @intCast(@divTrunc(c.hidden, 128)), @intCast(tile_count), 1 }, + .num_warps = if (c.tokens <= 8) 4 else 8, + .num_stages = 2, + .global_scratch_memory_size = @intCast(128 * @divTrunc(c.hidden, 128) * tile_count), + }, + ); + + return Combine.call( + .{ .d = d.d, .ids = a.ids, .perm = perm, .rw = a.scales }, + .{ .y = a.x.shape() }, + .{ + .cfg = c, + .grid = .{ @intCast(c.tokens), @intCast(@divTrunc(c.hidden + combineBlock(c) - 1, combineBlock(c))), 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ).y; +} + +fn quant32(b: *B, x: V) struct { q: V, s: V } { + const rows = x.shape().constSlice()[0]; + const cols = x.shape().constSlice()[1]; + const groups = @divExact(cols, 32); + const blocks = b.reshape(x, &.{ rows * groups, 32 }); + const sf = b.exp2(b.ceil(b.log2(b.maxOpts(b.absf(blocks), .{ .axis = 1 }).maximum(1e-4).mul(1.0 / 448.0)))); + const q = blocks.div(sf.expandDims(1)).maximum(-448.0).minimum(448.0).to(.f8e4m3fn); + const bits = b.shrui(b.bitcast(sf, .i32), b.full(sf.shape().constSlice(), 23, .i32)).bitAnd(255).to(.i8); + return .{ .q = b.reshape(q, &.{ rows, cols }), .s = b.reshape(bits, &.{ rows, groups }) }; +} + +fn quantInput(b: *B, c: Config) tri.FinishError!void { + const a = try b.declareArgs(.{ .x = .{ .ptr = .bf16 }, .q = .{ .ptr = .f8e4m3fn }, .s = .{ .ptr = .i8 } }); + const rows = b.programId(.x).mul(quantRows(c)).add(ar(b, quantRows(c))); + const block = b.programId(.y); + const cols = block.mul(128).add(ar(b, 128)); + const offsets = rows.expandDims(1).mul(c.hidden).add(cols.expandDims(0)); + const mask = rows.lt(c.tokens).expandDims(1); + const x = ld(b, a.x.addPtr(offsets), mask, .bf16, 0).to(.f32); + const q = quant32(b, x); + b.storeOpts(a.q.addPtr(offsets), q.q, .{ .mask = mask }); + const scale_offsets = rows.expandDims(1).mul(@divTrunc(c.hidden, 32)).add(block.mul(4).add(ar(b, 4)).expandDims(0)); + b.storeOpts(a.s.addPtr(scale_offsets), q.s, .{ .mask = mask }); +} + +fn route(b: *B, c: Config) tri.FinishError!void { + const a = try b.declareArgs(.{ .ids = .{ .ptr = .i32 }, .counts = .{ .ptr = .i32 }, .pos = .{ .ptr = .i32 } }); + const e = b.programId(.x); + const p = ar(b, pow2(c.tokens * c.topk)); + const ids = ld(b, a.ids.addPtr(p), p.lt(c.tokens * c.topk), .i32, -1); + const valid = p.lt(c.tokens * c.topk).bitAnd(ids.eq(e.add(c.expert_offset))); + b.storeOpts(a.pos.addPtr(p), valid.to(.i32).cumsum().sub(1), .{ .mask = valid }); + b.store(a.counts.addPtr(e), valid.to(.i32).sum()); +} + +fn schedule(b: *B, c: Config) tri.FinishError!void { + const a = try b.declareArgs(.{ .ids = .{ .ptr = .i32 }, .counts = .{ .ptr = .i32 }, .pos = .{ .ptr = .i32 }, .map = .{ .ptr = .i32 }, .perm = .{ .ptr = .i32 }, .sched = .{ .ptr = .i32 } }); + const nt = tiles(c); + const p = b.programId(.x).mul(128).add(ar(b, 128)); + const e = ld(b, a.ids.addPtr(p), p.lt(c.tokens * c.topk), .i32, -1).sub(ci(b, c.expert_offset)); + const es = ar(b, pow2(c.experts)); + const counts = ld(b, a.counts.addPtr(es), es.lt(c.experts), .i32, 0); + const ts = counts.cdiv(bn); + const starts = ts.cumsum().sub(ts); + var first = b.openIf(b.programId(.x).eq(0)); + b.store(a.sched.addPtr(2 * nt), ts.sum()); + first.yieldThen(.{}); + const live = p.lt(c.tokens * c.topk).bitAnd(e.ge(0)).bitAnd(e.lt(c.experts)); + const safe = e.maximum(0).minimum(pow2(c.experts) - 1); + const pos = ld(b, a.pos.addPtr(p), live, .i32, 0); + const start = starts.gather(safe, 0); + const tile = start.add(pos.div(bn)); + const row = pos.rem(bn); + b.storeOpts(a.perm.addPtr(p), b.select(live, tile.mul(bn).add(row), b.full(&.{128}, -1, .i32)), .{ .mask = p.lt(c.tokens * c.topk) }); + b.storeOpts(a.map.addPtr(tile.mul(bn).add(row)), p, .{ .mask = live }); + b.storeOpts(a.sched.addPtr(tile), e, .{ .mask = live.bitAnd(row.eq(0)) }); + b.storeOpts(a.sched.addPtr(nt).addPtr(tile), counts.gather(safe, 0).sub(pos).minimum(bn), .{ .mask = live.bitAnd(row.eq(0)) }); +} + +fn gateUp(b: *B, c: GemmConfig) tri.FinishError!void { + return gemm(b, c.moe, .gate_up, c.scale_stride); +} + +fn down(b: *B, c: GemmConfig) tri.FinishError!void { + return gemm(b, c.moe, .down, c.scale_stride); +} + +const Projection = enum { gate_up, down }; + +fn gemm(b: *B, c: Config, comptime projection: Projection, scale_stride: i64) tri.FinishError!void { + const is_up = projection == .gate_up; + const a = if (is_up) try b.declareArgs(.{ .q = .{ .ptr = .f8e4m3fn }, .s = .{ .ptr = .i8 }, .w = .{ .ptr = .i8 }, .ws = .{ .ptr = .i8 }, .ids = .{ .ptr = .i32 }, .rw = .{ .ptr = .f32 }, .map = .{ .ptr = .i32 }, .sched = .{ .ptr = .i32 }, .mid = .{ .ptr = .f8e4m3fn }, .ms = .{ .ptr = .i8 } }) else try b.declareArgs(.{ .q = .{ .ptr = .f8e4m3fn }, .s = .{ .ptr = .i8 }, .w = .{ .ptr = .i8 }, .ws = .{ .ptr = .i8 }, .ids = .{ .ptr = .i32 }, .sched = .{ .ptr = .i32 }, .d = .{ .ptr = .bf16 } }); + const block = b.programId(.x); + const tile = b.programId(.y); + const n = ar(b, bn); + const nt = tiles(c); + const expert = if (c.tokens == 1) b.load(a.ids.addPtr(tile)).sub(ci(b, c.expert_offset)) else ld(b, a.sched.addPtr(tile), tile.lt(b.load(a.sched.addPtr(2 * nt))), .i32, 0); + const valid = if (c.tokens == 1) expert.ge(0).bitAnd(expert.lt(c.experts)) else tile.lt(b.load(a.sched.addPtr(2 * nt))); + var live = b.openIf(valid); + const count = if (c.tokens == 1) ci(b, 1) else b.load(a.sched.addPtr(nt).addPtr(tile)); + const pair = if (c.tokens == 1) b.full(&.{bn}, 0, .i32).add(tile) else if (is_up) ld(b, a.map.addPtr(tile.mul(bn).add(n)), n.lt(count), .i32, 0) else n; + const width: i64 = if (is_up and c.tokens > 8) 256 else 128; + const ksize = if (is_up) c.hidden else c.intermediate; + const rows = if (is_up) 2 * c.intermediate else c.hidden; + const m = block.mul(width).add(ar(b, width)); + const k = ar(b, bk); + const ks = ar(b, bk / 32); + const row = if (is_up) pair.div(c.topk) else tile.mul(bn).add(n); + const wd = b.makeTensorDescriptor(a.w.addPtr(expert.to(.i64).mul(@divTrunc(rows * ksize, 2))), &.{ ci(b, rows), ci(b, @divTrunc(ksize, 2)) }, &.{ b.liftAs(@divTrunc(ksize, 2), .i64), b.liftAs(1, .i64) }, .pad_zero, &.{ width, bk / 2 }, .i8); + var loop = b.openFor(0, @divTrunc(ksize + bk - 1, bk), 1, .{b.zeros(&.{ width, bn }, .f32)}); + const sk = loop.iv.mul(bk / 32).add(ks); + const kk = loop.iv.mul(bk).add(k); + const w = b.descriptorLoad(wd, &.{ block.mul(ci(b, width)), loop.iv.mul(bk / 2) }, &.{ width, bk / 2 }, .i8); + const scale_offset = m.expandDims(1).mul(scale_stride).add(sk.expandDims(0)); + const ws = ld(b, a.ws.addPtr(expert.to(.i64).mul(rows * scale_stride)).addPtr(scale_offset), sk.expandDims(0).lt(@divTrunc(ksize, 32)), .i8, 127); + var mask = kk.expandDims(0).lt(ksize); + var smask = sk.expandDims(0).lt(@divTrunc(ksize, 32)); + // The small-batch epilogue leaves padded rows unwritten. + if (is_up or c.tokens <= 8) { + mask = mask.bitAnd(n.lt(count).expandDims(1)); + smask = smask.bitAnd(n.lt(count).expandDims(1)); + } + const x = ld(b, a.q.addPtr(row.expandDims(1).mul(ksize).add(kk.expandDims(0))), mask, .f8e4m3fn, 0); + const xs = ld(b, a.s.addPtr(row.expandDims(1).mul(@divTrunc(ksize, 32)).add(sk.expandDims(0))), smask, .i8, 127); + loop.yield(.{b.dotScaled(w, b.permute(x, &.{ 1, 0 }), loop.carried[0], ws, xs, .e2m1, .e4m3)}); + const epilogue_n: i64 = if (c.tokens == 1) 1 else bn; + const out_n = ar(b, epilogue_n); + const acc = if (c.tokens == 1) live_row: { + // The remaining MMA columns are padding during single-token decode. + // Extract column zero before activation/reduction and output stores. + var v = loop.results[0]; + var cols: i64 = bn; + while (cols > 1) : (cols = @divTrunc(cols, 2)) { + v = b.split(b.reshape(v, &.{ width, @divTrunc(cols, 2), 2 }))[0]; + } + break :live_row b.reshape(v, &.{ 1, width }).to(.bf16); + } else b.permute(loop.results[0], &.{ 1, 0 }).to(.bf16); + if (is_up) { + const columns = @divExact(width, 2); + const halves = b.split(b.reshape(acc.to(.f32), &.{ epilogue_n, columns, 2 })); + var g = halves[0]; + var u = halves[1]; + if (c.swiglu_limit > 0) { + g = g.minimum(c.swiglu_limit); + u = u.minimum(c.swiglu_limit).maximum(-c.swiglu_limit); + } + const ex = b.externElementwise(&.{g.mul(-1.0)}, &.{ epilogue_n, columns }, .f32, "libdevice", "", "__nv_expf"); + var mid = u.mul(b.divRn(g, ex.add(1.0))); + if (c.routing_weight_placement == .before_down) { + const rw = if (c.tokens == 1) b.load(a.rw.addPtr(tile)) else ld(b, a.rw.addPtr(pair), n.lt(count), .f32, 0).expandDims(1); + mid = mid.mul(rw); + } + // Compute scales and quantize directly from the FP32 SwiGLU result. + const q = quant32(b, mid); + const col = block.mul(columns).add(ar(b, columns)); + b.store(a.mid.addPtr(tile.mul(bn).add(out_n).expandDims(1).mul(c.intermediate).add(col.expandDims(0))), q.q); + const scale_cols = block.mul(@divExact(columns, 32)).add(ar(b, @divExact(columns, 32))); + b.store(a.ms.addPtr(tile.mul(bn).add(out_n).expandDims(1).mul(@divTrunc(c.intermediate, 32)).add(scale_cols.expandDims(0))), q.s); + } else b.store(a.d.addPtr(tile.mul(bn).add(out_n).expandDims(1).mul(c.hidden).add(m.expandDims(0))), acc); + live.yieldThen(.{}); +} + +fn combine(b: *B, c: Config) tri.FinishError!void { + return combineWithBn(b, c, bn); +} + +fn combineWithBn(b: *B, c: Config, tile_n: i64) tri.FinishError!void { + const a = try b.declareArgs(.{ .d = .{ .ptr = .bf16 }, .ids = .{ .ptr = .i32 }, .perm = .{ .ptr = .i32 }, .rw = .{ .ptr = .f32 }, .y = .{ .ptr = .bf16 } }); + const t = b.programId(.x); + const block = combineBlock(c); + const h = b.programId(.y).mul(block).add(ar(b, block)); + var acc = b.zeros(&.{block}, .f32); + for (0..@intCast(c.topk)) |s| { + const pair = t.mul(c.topk).add(@as(i64, @intCast(s))); + const row = if (c.tokens == 1) pair.mul(tile_n) else b.load(a.perm.addPtr(pair)); + const valid = if (c.tokens == 1) v: { + const e = b.load(a.ids.addPtr(pair)).sub(ci(b, c.expert_offset)); + break :v e.ge(0).bitAnd(e.lt(c.experts)); + } else row.ge(0); + var value = ld(b, a.d.addPtr(row.mul(c.hidden).add(h)), valid.bitAnd(h.lt(c.hidden)), .bf16, 0).to(.f32); + if (c.routing_weight_placement == .after_down) { + value = value.mul(ld(b, a.rw.addPtr(pair), valid, .f32, 0)); + } + acc = acc.add(value); + } + b.storeOpts(a.y.addPtr(t.mul(c.hidden).add(h)), acc.to(.bf16), .{ .mask = h.lt(c.hidden) }); +} diff --git a/zml/moe/triton_mxfp4.zig b/zml/moe/triton_mxfp4.zig new file mode 100644 index 00000000..644f29d2 --- /dev/null +++ b/zml/moe/triton_mxfp4.zig @@ -0,0 +1,123 @@ +const std = @import("std"); + +const zml = @import("../zml.zig"); +pub const kernels = @import("triton_kernels/mxfp4.zig"); + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + const cc = zml.platform.cuda.computeCapability(platform) orelse return false; + return cc.major == 10; +} + +pub const Parameters = struct { + pub const InitOptions = struct { + num_experts_per_tok: u32, + activation: zml.moe.ActivationMode, + }; + + num_experts_per_tok: u32, + activation: zml.moe.ActivationMode, + + pub fn init(opts: InitOptions) Parameters { + return .{ + .num_experts_per_tok = opts.num_experts_per_tok, + .activation = opts.activation, + }; + } +}; + +/// Row-major MXFP4 E2M1 weights and linear E8M0 block32 scales. +/// BF16 activations are quantized on the GPU to FP8 with per-32 E8M0 scales. +pub fn fusedExperts( + input: zml.Tensor, + ids: zml.Tensor, + weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + options: zml.moe.Options, + parameters: Parameters, +) !zml.Tensor { + if (parameters.activation != .silu) return error.UnsupportedActivation; + if (input.dtype() != .bf16) return error.UnsupportedDataType; + if (gate_up.bias != null or down.bias != null) return error.UnsupportedBias; + const gq = gate_up.quantization orelse return error.UnsupportedQuantization; + const dq = down.quantization orelse return error.UnsupportedQuantization; + if (gq.scheme != .mxfp4 or dq.scheme != .mxfp4) return error.UnsupportedQuantization; + if ((gate_up.weight.dtype() != .u8 and gate_up.weight.dtype() != .i8) or (down.weight.dtype() != .u8 and down.weight.dtype() != .i8)) return error.UnsupportedWeightLayout; + + const expert_parallelism = gate_up.weight.shape().partition(.expert).eql(.init(.experts)); + + const context: Context = .{ + .input = input, + .ids = ids, + .weights = weights, + .w1 = gate_up.weight.bitCast(.u8), + .s1 = gq.scales, + .w2 = down.weight.bitCast(.u8), + .s2 = dq.scales, + .global_experts = gate_up.weight.dim(.expert), + .topk = parameters.num_experts_per_tok, + .limit = options.activation_threshold orelse 0, + .routing_weight_placement = options.routing_weight_placement, + .expert_parallel = expert_parallelism, + }; + + return if (expert_parallelism) + zml.ops.manualComputation(Context.body, context, input.shape()) + else + context.body(input.shape()); +} + +const Context = struct { + input: zml.Tensor, + ids: zml.Tensor, + weights: zml.Tensor, + w1: zml.Tensor, + s1: zml.Tensor, + w2: zml.Tensor, + s2: zml.Tensor, + global_experts: i64, + topk: i64, + limit: f32, + routing_weight_placement: zml.moe.triton.RoutingWeightPlacement, + expert_parallel: bool, + + fn body(self: Context, _: zml.Shape) zml.Tensor { + const experts = self.w1.dim(.expert); + const hidden = self.w2.dim(1); + const intermediate = self.w2.dim(2) * 2; + const tokens: i64 = @intCast(self.input.count() / @as(usize, @intCast(hidden))); + + var ids = self.ids.convert(.i32); + if (self.expert_parallel) { + const partition_id = zml.ops.partitionId().convert(.i32); + ids = ids.sub(partition_id.scale(experts)); + } + + const cfg: kernels.Config = .{ + .tokens = tokens, + .hidden = hidden, + .intermediate = intermediate, + .experts = experts, + .global_experts = self.global_experts, + .topk = self.topk, + .swiglu_limit = self.limit, + .routing_weight_placement = self.routing_weight_placement, + }; + + const result = kernels.forward(cfg, .{ + .x = self.input.reshape(.{ tokens, hidden }), + .ids = ids.reshape(.{ tokens, self.topk }), + .scales = self.weights.convert(.f32).reshape(.{ tokens, self.topk }), + .w1 = self.w1, + .s1 = self.s1, + .w2 = self.w2, + .s2 = self.s2, + }).reshape(self.input.shape().dims()).withTags(self.input.shape()); + + return if (self.expert_parallel) + zml.ops.allReduce(result, zml.Tensor.add) + else + result; + } +}; From b78c0f03d753fec3f84471689c72dcd3959a1d72 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Mon, 14 Sep 2026 09:57:36 +0200 Subject: [PATCH 681/881] kernels/fly: FlyDSL Zig DSL (#799) Adds a Zig front end for the FlyDSL dialect, so AMD GPU kernels can be written in Zig alongside the existing Mosaic, Triton and tile ones, and dispatched the same way. Types and layouts are built through the dialect's own C entry points rather than by printing and reparsing assembly. That holds in both directions: host-side layout literals become attributes directly, and anything the dialect computes is read back through typed accessors. Launch geometry is derived from the device rather than hard-coded, so the same kernels run on both AMD architecture families and the tests stay architecture agnostic. Everything builds and tests on any host, no GPU needed. Also carries one unrelated cleanup, dropping a needless null terminator from the Triton and tile kernel strings. --- MODULE.bazel | 1 + kernels/fly/BUILD.bazel | 25 + kernels/fly/PORTING.md | 83 + kernels/fly/README.md | 188 +++ kernels/fly/builder.zig | 1667 ++++++++++++++++++++ kernels/fly/dtype.zig | 86 + kernels/fly/layout.zig | 544 +++++++ mlir/BUILD.bazel | 1 + mlir/dialects/BUILD.bazel | 2 + mlir/dialects/dialects.zig | 2 + mlir/dialects/fly/BUILD.bazel | 44 + mlir/dialects/fly/attributes.zig | 239 +++ mlir/dialects/fly/fly.zig | 472 ++++++ mlir/dialects/fly/fly_capi.cc | 521 ++++++ mlir/dialects/fly/fly_capi.h | 144 ++ mlir/dialects/fly/rocdl.zig | 164 ++ mlir/dialects/fly/types.zig | 457 ++++++ mlir/dialects/gpu.zig | 89 ++ mlir/mlir.h | 1 + mlir/mlir.zig | 8 + third_party/flydsl/BUILD.bazel | 0 third_party/flydsl/atomic_exchange.patch | 25 + third_party/flydsl/flydsl.bazel | 396 +++++ third_party/flydsl/mlir_property_ref.patch | 384 +++++ third_party/flydsl/repo.bzl | 16 + third_party/flydsl/vector_atomic.patch | 44 + third_party/flydsl/xf32_mfma.patch | 25 + third_party/non_module_deps.bzl | 2 + zml/BUILD.bazel | 2 + zml/kernel.zig | 337 ++++ zml/ops.zig | 188 ++- 31 files changed, 6153 insertions(+), 4 deletions(-) create mode 100644 kernels/fly/BUILD.bazel create mode 100644 kernels/fly/PORTING.md create mode 100644 kernels/fly/README.md create mode 100644 kernels/fly/builder.zig create mode 100644 kernels/fly/dtype.zig create mode 100644 kernels/fly/layout.zig create mode 100644 mlir/dialects/fly/BUILD.bazel create mode 100644 mlir/dialects/fly/attributes.zig create mode 100644 mlir/dialects/fly/fly.zig create mode 100644 mlir/dialects/fly/fly_capi.cc create mode 100644 mlir/dialects/fly/fly_capi.h create mode 100644 mlir/dialects/fly/rocdl.zig create mode 100644 mlir/dialects/fly/types.zig create mode 100644 mlir/dialects/gpu.zig create mode 100644 third_party/flydsl/BUILD.bazel create mode 100644 third_party/flydsl/atomic_exchange.patch create mode 100644 third_party/flydsl/flydsl.bazel create mode 100644 third_party/flydsl/mlir_property_ref.patch create mode 100644 third_party/flydsl/repo.bzl create mode 100644 third_party/flydsl/vector_atomic.patch create mode 100644 third_party/flydsl/xf32_mfma.patch diff --git a/MODULE.bazel b/MODULE.bazel index c8402415..7ac550e7 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -258,6 +258,7 @@ use_repo( "flashattn_linux_arm64", "flashinfer_cutlass_moe_linux_amd64", "flashinfer_cutlass_moe_linux_arm64", + "flydsl", "iree", "libvaxis", "linenoise", diff --git a/kernels/fly/BUILD.bazel b/kernels/fly/BUILD.bazel new file mode 100644 index 00000000..5ee1a1f8 --- /dev/null +++ b/kernels/fly/BUILD.bazel @@ -0,0 +1,25 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "fly_builder", + tags = ["manual"], + srcs = [ + "dtype.zig", + "layout.zig", + ], + import_name = "kernels/fly/builder", + main = "builder.zig", + visibility = ["//visibility:public"], + deps = [ + "//kernels/common", + "//mlir", + "//mlir/dialects", + "//mlir/dialects/fly", + "//stdx", + ], +) + +zig_test( + name = "test", + deps = [":fly_builder"], +) diff --git a/kernels/fly/PORTING.md b/kernels/fly/PORTING.md new file mode 100644 index 00000000..b46d3b97 --- /dev/null +++ b/kernels/fly/PORTING.md @@ -0,0 +1,83 @@ +# Porting a FlyDSL Python kernel to Zig + +The Zig DSL emits the same `fly` ops the Python does, so a port is a +statement-by-statement translation. Work from the pinned FlyDSL tree +(`third_party/flydsl/repo.bzl`, read with `git show :`), not from +FlyDSL HEAD: the plugin only parses what the pin prints. + +## Cheat sheet + +| Python (`flydsl.expr`) | Zig | +|---|---| +| `@flyc.kernel def k(A: Tensor, ...)` | `fly.Kernel(Cfg, .{ .inputs = &.{"a"}, .outputs = &.{"c"}, .run = run })`; `const a = K.args(b)` | +| kernel arg `tiled_copy: fx.TiledCopy` | build it inside `run` (static values cannot be arguments) | +| `fx.thread_idx.x`, `fx.block_idx.x` | `b.threadId(.x)`, `b.blockId(.x)` (`i32`) | +| `A.shape.unpack()` | `a.a.shapeStatic().at(i).leaf.s` | +| `fx.make_layout((8,16),(16,1))` | `fly.L(.{ 8, 16 }, .{ 16, 1 })` (comptime) or `b.static(layout)` | +| `fx.make_ordered_layout((8,16),(1,0))` | `fly.ordered(.{ 8, 16 }, .{ 1, 0 })` | +| `fx.make_tile(64, 8)` / tuple divisor `(64, 8)` | `fly.tile(.{ 64, 8 })` / `.{ 64, 8 }` | +| `fx.make_int_tuple((m, tid))`, coords | `b.intTuple(.{ m, tid })` (`Value`s become `?` leaves) | +| `t[None, None, bx, by]` | `t.slice(.{ null, null, bx, by })` | +| `t[(0, None), None, None]` | `t.slice(.{ .{ 0, null }, null, null })` | +| `t[i]` (no `None`) / `t[i] = v` | `t.at(i)` / `t.set(i, v)` | +| `fx.flat_divide(t, tile)` etc. | `t.flatDivide(tile)`, `.logicalDivide`, `.zippedDivide`, `.tiledDivide` | +| `fx.raked_product(a, b)` etc. | `a.rakedProduct(b)`, `.logicalProduct`, ... | +| `fx.right_inverse(l)`, `fx.composition(l, r)` | `l.rightInverse()`, `l.composition(r)` | +| `fx.get_shape(l)`, `fx.size(l)` (emitted) | `l.emitShape()`, `l.emitSize()` | +| `fx.size(t.shape).unpack()` (host int) | `t.sizeStatic()` | +| `fx.select(tup, [0])`, `fx.get(tup, [1])` | `tup.selectModes(&.{0})`, `tup.get(&.{1})` | +| `fx.make_view((0,0), fx.make_identity_layout((M,N)))` | `b.identity(.{ m, n })` | +| `fx.make_view(ptr, layout)` | `ptr.view(layout)` | +| `fx.make_copy_atom(fx.UniversalCopy128b(), fx.Float32)` | `b.copyAtom(.{ .universal = 128 }, .f32)` | +| `fx.rocdl.BufferCopy32b()` / `BufferCopyLDS128b()` | `.{ .buffer_copy = 32 }` / `.{ .buffer_copy_lds = 128 }` | +| `fx.make_mma_atom(fx.rocdl.MFMA(16,16,4,fx.Float32))` | `b.mmaAtom(try fly.rocdl.MmaOpCDNA3MFMAType.get(ctx, .{ .m = 16, .n = 16, .k = 4, ... }))` | +| `fx.make_tiled_mma(atom, layout)` | `b.tiledMma(atom, layout, null)` | +| `fx.make_tiled_copy_tv(atom, thr, val)` | `b.tiledCopyTV(atom, thr, val)` | +| `fx.make_tiled_copy_A(atom, tiled_mma)` | `b.tiledCopyA(atom, tiled_mma)` | +| `tiled_copy.tile_mn` | `tc.tileMN()` | +| `tiled_mma.tile_size_mnk` etc. | `fly.expect(tm.value.type_(), .tiled_mma).getTileSizeMNK()` | +| `tiled_copy.get_slice(tid).partition_S(t)` | `tc.getSlice(tid).partitionS(t)` | +| `thr_copy.retile(t)` | `thr.retile(t)` | +| `tiled_mma.make_fragment_A(t)` | `tm.makeFragmentA(t)` | +| `fx.make_fragment_like(t[, dtype=fx.Boolean])` | `t.makeFragmentLike(null)` / `t.makeFragmentLike(.i1)` | +| `fx.make_rmem_tensor(8, fx.Float16)` | `b.rmemTensor(fly.L(8, 1), .f16)` | +| `fx.copy(atom, src, dst, pred=p)` | `b.copy(atom, src, dst, .{ .pred = p })` | +| `fx.copy_atom_call(atom, src, dst)` | `b.copyAtomCall(atom, src, dst)` | +| `fx.gemm(atom, d, a, b, c)` | `b.gemm(atom, d, a, bb, c)` | +| `t.load()`, `t.store(v)`, `t.fill(0)` | same names | +| `fx.elem_less(coord, (M, N))` | `coord.elemLess(.{ m, n })` | +| `fx.rocdl.make_buffer_tensor(t)` | `b.bufferTensor(t)` | +| `fx.SharedAllocator().allocate(fx.Array[f32, n, 16])` | `b.sharedArray(.f32, n, 16)` (returns an `n:1` tensor) | +| `gpu.barrier()` | `b.barrier()` | +| `x.shuffle_xor(off, 64)` | `x.shuffleXor(off, 64)` | +| `v.reduce(ReductionOp.ADD)` | `v.reduce(.add)` | +| `v.to(fx.Float32)` | `v.to(.f32)` | +| `x * rrms` (vector × scalar) | `x.mul(rrms)` | +| `fmath.rsqrt(x, fastmath=fast)` | `b.setFastMath(true); b.rsqrt(x)` | +| `lane < RED_SLOTS`, `cond.select(a, b)` | `lane.cmp(.lt, n)`, `cond.select(a, b)` | +| `if tid == 0: ...` | `var s = b.openIf(tid.cmp(.eq, 0)); ...; s.yieldThen(.{});` | +| `for i in range_constexpr(n)` | a plain Zig `for (0..n)` (unrolled at emit time) | +| `for i in range(lo, hi)` with carried values | `b.openFor(lo, hi, 1, .{ init })` + `.yield(.{ ... })` | +| `launch(grid=..., block=(256,1,1))` | `K.call(inputs, outputs, .{ .grid = ..., .threads = 256 })` | + +## Method + +1. Read the kernel and its helpers at the pin; note every `fx.*` primitive. +2. Write the Zig `run` top to bottom. Keep the same op order so the emitted + IR lines up with the Python module when diffing. +3. `bazel test //kernels/fly:test` (or a `K.emit` test): the module must + verify and re-parse. A panic from `fly.: cannot create operation` + lists the operand types; compare them with the Python trace. +4. Run on the ROCm box against a host reference + (`bazel test //zml:test --@zml//platforms:rocm=true ...`). + +## Things that differ + +- Python config (`N`, `dtype_str`, `const_expr(...)` branches) becomes the + kernel `Config`; branch on it with plain `if` in `run`. +- Python lists of vectors across an unrolled loop are Zig arrays of `Value`. +- Python `Int32(x)` / `Float32(x)` constants are `b.constant(.i32, x)`; most + arithmetic methods also take a literal directly (`tid.add(256)`). +- Cached scalars from LDS are read with `t.at(i)`; there is no implicit load. +- bf16 outputs use `arith.truncf`; the Python's manual round-to-nearest bit + trick for pre-gfx950 parts is not reproduced. diff --git a/kernels/fly/README.md b/kernels/fly/README.md new file mode 100644 index 00000000..66b089aa --- /dev/null +++ b/kernels/fly/README.md @@ -0,0 +1,188 @@ +# ZML FlyDSL DSL — Author's Guide + +A Zig builder for AMD's FlyDSL (`fly` / `fly_rocdl` MLIR dialects) that reads +like `flydsl.expr` Python while staying in pure Zig. Kernels are compiled and +launched by ZML's ROCm PJRT plugin through the `__gpu$xla.gpu.fly` custom +call; ZML never links a GPU compiler. + +Two layers define the surface: + +- **Low layer** — `kernels/fly/builder.zig`: `Builder`, `Value`, `TiledCopy`, + `TiledMma`, the comptime layout literals in `layout.zig`, and `DType`. + `Builder.open` takes an `*mlir.Context` from `zml.kernel.fly.newContext()`. +- **High layer** — `zml.kernel.fly.Kernel(Config, spec)`: names the kernel's + inputs and outputs, hands `run` its arguments by name (`K.args(b)`), and + `call`s it from a ZML model. + +The dialect binding (`mlir/dialects/fly`) is generic: every `fly.*` op is +built by name and FlyDSL's own `inferReturnTypes` computes every layout. Zig +only prints and reads the dialect's type syntax; it never does layout algebra. + +## Skeleton of a kernel + +```zig +const zml = @import("zml"); +const fly = zml.kernel.fly; + +pub const Cfg = struct { + thr: fly.Layout = fly.ordered(.{ 8, 16 }, .{ 1, 0 }), // 128 threads + val: fly.Layout = fly.ordered(.{ 1, 4 }, .{ 0, 1 }), // f32x4 along N +}; + +pub const VectorAdd = fly.Kernel(Cfg, .{ + .name = "vector_add", + .inputs = &.{ "a", "b" }, + .outputs = &.{"c"}, + .run = run, +}); + +fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { + const a = VectorAdd.args(b); // .a/.b/.c: row-major !fly.memref views + const shape = a.a.shapeStatic(); + const m = shape.at(0).leaf.s; + const n = shape.at(1).leaf.s; + + const atom = b.copyAtom(.{ .universal = 128 }, .f32); + const tc = b.tiledCopyTV(atom, cfg.thr, cfg.val); // fx.make_tiled_copy_tv + const tid = b.threadId(.x); + const bx = b.blockId(.x); + const by = b.blockId(.y); + + const tile = tc.tileMN(); + const gA = a.a.flatDivide(tile).slice(.{ null, null, bx, by }); // A[None, None, bx, by] + const gB = a.b.flatDivide(tile).slice(.{ null, null, bx, by }); + const gC = a.c.flatDivide(tile).slice(.{ null, null, bx, by }); + const cC = b.identity(.{ m, n }).flatDivide(tile).slice(.{ null, null, bx, by }); + + const thr = tc.getSlice(tid); + const tgA = thr.partitionS(gA); + const tgB = thr.partitionS(gB); + const tgC = thr.partitionD(gC); + const tcC = thr.partitionS(cC).slice(.{ .{ 0, null }, null, null }); + + const rA = tgA.makeFragmentLike(null); + const rB = tgB.makeFragmentLike(null); + const rC = tgC.makeFragmentLike(null); + const pC = tcC.makeFragmentLike(.i1); + for (0..@intCast(pC.sizeStatic())) |i| pC.set(i, tcC.at(i).elemLess(.{ m, n })); + + b.copy(atom, tgA, rA, .{ .pred = pC }); + b.copy(atom, tgB, rB, .{ .pred = pC }); + rC.store(rA.load().add(rB.load())); + b.copy(atom, rC, tgC, .{ .pred = pC }); +} + +// In a model: +const c = VectorAdd.call(.{ .a = a, .b = bt }, .{ .c = a.shape() }, .{ + .cfg = .{}, + .grid = .{ grid_m, grid_n, 1 }, + .threads = 128, +}).c; +``` + +Complete vector-add and tiled-MMA kernels are the tests in `builder.zig` +and `zml/kernel.zig`. + +## The ABI + +`Kernel.emit` produces + +```mlir +module attributes {gpu.container_module} { + gpu.module @zml_fly_kernels { + gpu.func @name(%a: !fly.ptr, ..., %c: !fly.ptr) kernel { ... gpu.return } + } +} +``` + +with one `!fly.ptr` per input, then per output. The plugin checks only the +argument **count**, so the pointer element type is on you: `DType.storageElem` +maps predicates to `i8` because that is how XLA stores them. Static DSL values +(layouts, tiles, atoms) must be built inside the body, never passed as +arguments. `Kernel.call` bridges each pointer to a row-major `!fly.memref` +view of the actual `zml.Shape`, so `K.args(b)` gives tensors. + +Launch geometry lives in `CallOpts`: `grid`, `threads` (threads per block, +converted to wavefronts using the device's wave size — 64 on CDNA, 32 on +RDNA — so it must be a multiple of that), optional `waves_per_eu` (a hard +occupancy clamp), +`shared_mem_bytes` (only for `fly.get_dyn_shared`; static LDS from +`sharedArray` needs none) and `zeroed_args` (indices into inputs-then-outputs; +ignored inside HIP graphs, so zero what you need yourself). + +## Layouts, tiles, coordinates + +`layout.zig` holds comptime literals. `toAttr` turns one into the matching +fly attribute through the dialect's own constructors, and reading a type back +goes through its accessors — neither direction goes through text: + +| Zig | MLIR | +|---|---| +| `fly.it(.{ .{ 2, 4 }, 8 })` | `!fly.int_tuple<((2,4),8)>` | +| `fly.L(.{ 4, 8 }, .{ 1, 4 })` | `!fly.layout<(4,8):(1,4)>` | +| `fly.rowMajor(.{ 4, 8 })` / `fly.colMajor` | `(4,8):(8,1)` / `(4,8):(1,4)` | +| `fly.ordered(.{ 8, 16 }, .{ 1, 0 })` | `fx.make_ordered_layout((8,16),(1,0))`, folded | +| `fly.tile(.{ 128, 64 })`, `fly.tile(.{ null, 8 })` | `!fly.tile<[128\|64]>`, `[*\|8]` | + +`b.static(literal)` materializes one as `fly.static`. Wherever the Python +takes a tuple, the Zig takes a tuple literal: `t.flatDivide(.{ 64, 8 })` is a +tile, `t.slice(.{ null, bid })` is `t[None, bid]`, `b.intTuple(.{ m, tid })` +mixes static ints, `null` (`*`) and dynamic `Value`s (`?` leaves, passed as +operands). `v.shapeStatic()` / `v.sizeStatic()` / `v.elemDType()` read the +inferred type back on the host; `v.emitShape()` / `v.emitSize()` emit ops. + +## Atoms and tiled ops + +- `b.copyAtom(.{ .universal = 128 }, .f32)`, `.{ .buffer_copy = 32 }`, + `.{ .buffer_copy_lds = 128 }`. +- `b.mmaAtom(mma_op)`, where `mma_op` is an MMA op type such as + `fly.rocdl.MmaOpCDNA3MFMAType.get(ctx, .{ .m = 16, .n = 16, .k = 16, ... })`. +- `b.tiledCopyTV(atom, thr, val)`, `b.tiledCopy(atom, layout_tv, tile)`, + `b.tiledCopyA/B/C(copy_atom, tiled_mma)`; `tc.getSlice(tid).partitionS/D`, + `.retile`. +- `b.tiledMma(atom, fly.L(.{ 2, 2, 1 }, .{ 1, 2, 0 }), null)`; + `tm.makeFragmentA/B/C(t)`, `tm.getSlice(tid).partitionA/B/C`. +- `b.copy(atom, src, dst, .{ .pred = p })`, `b.copyAtomCall(atom, src, dst)`, + `b.gemm(atom, d, a, b, c)`. +- `b.bufferTensor(t)` is `fx.rocdl.make_buffer_tensor`: the same tensor over + a CDNA raw-buffer descriptor, for bounds-checked `buffer_copy` atoms. +- Derived layouts come off the type: `fly.expect(tm.value.type_(), .tiled_mma)` + then `.getTileSizeMNK()`, `.getTiledThrValLayoutA()`, ... + +## Memory and scalars + +`t.load()` / `t.store(vec)` move a whole register tensor as a `vector`; +`t.at(coord)` / `t.set(coord, v)` are element accesses (an int tuple for a +coordinate tensor); `t.fill(0)` splats. `b.rmemTensor(layout, dtype)` is +`fx.make_rmem_tensor`; `b.sharedArray(.f32, n, 16)` is a static LDS array +(`fx.SharedAllocator().allocate(fx.Array[...])`). + +Scalars and vectors: `add/sub/mul/div/rem` (a scalar broadcasts against a +vector), `to(dtype)`, `cmp(.lt, x)`, `cond.select(a, b)`, `v.reduce(.add)`, +`v.shuffleXor(off, 64)`, `b.rsqrt(x)`, `b.constant(.f32, 1.0)`, +`b.threadId(.x)` / `b.blockId(.x)` (already `i32`), `b.barrier()`. +`b.setFastMath(true)` tags subsequent float ops `fastmath`. + +Control flow uses `kernels/common`: `b.openFor(lo, hi, step, .{ inits })` +with `.yield(.{ ... })`, `b.openIf(cond)` with `.yieldThen(.{})`, +`b.openIfElse(cond, .{ types })` with `.yieldThen` / `.yieldElse`. + +## Control flow and register tensors + +The ROCm plugin lowers `scf` to `cf` **before** running the Fly passes on a +`gpu.func` entry, and Fly's register-alloca promotion only rewrites a +single-block body. So a kernel that has `openIf` / `openFor` must not hold +register tensors (`rmemTensor`, `makeFragmentLike`, `fill`) — use the SSA +atom calls instead: `b.copyAtomLoad(atom, src)` returns the loaded +`vector`, `b.copyAtomStore(atom, vec, dst)` stores one. Kernels without +control flow (vectorAdd, tiledMma) can use register tensors freely. The +durable fix is on the XLA side (run `convert-scf-to-cf` after the Fly +passes, as FlyDSL's own pipeline does). + +## Debugging + +- A failing `inferReturnTypes` panics with the op name and every operand type + (`fly.make_tiled_copy: cannot create operation ... operand 1: !fly.layout<...>`). +- `Kernel.emit` returns the module text; `bazel test //kernels/fly:test` + builds the example kernels on any host, no GPU needed. +- On the device side, `--xla_dump_to=` dumps the module the plugin saw. diff --git a/kernels/fly/builder.zig b/kernels/fly/builder.zig new file mode 100644 index 00000000..f326246f --- /dev/null +++ b/kernels/fly/builder.zig @@ -0,0 +1,1667 @@ +const std = @import("std"); + +const dialects = @import("mlir/dialects"); +const arith = dialects.arith; +const gpu = dialects.gpu; +const scf = dialects.scf; +const fly = @import("mlir/dialects/fly"); +const dsl = @import("kernels/common"); +const tupleArity = dsl.tupleArity; +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub const layout = @import("layout.zig"); +pub const dtypes = @import("dtype.zig"); + +pub const DType = dtypes.DType; +pub const IntTuple = layout.IntTuple; +pub const Layout = layout.Layout; +pub const Tile = layout.Tile; +pub const it = layout.it; +pub const L = layout.L; +pub const rowMajor = layout.rowMajor; +pub const colMajor = layout.colMajor; +pub const ordered = layout.ordered; +pub const tile = layout.tile; + +pub const dialects_needed = fly.dialects_needed; + +pub const FinishError = error{ InvalidMlir, OutOfMemory } || std.Io.Writer.Error; + +pub const Arch = enum { gfx942, gfx950 }; + +pub const Dim = gpu.Dim; + +pub const gpu_module_name = "zml_fly_kernels"; + +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(Builder); + std.testing.refAllDecls(Value); +} + +// ============================================================================= +// Value +// ============================================================================= + +pub const Value = struct { + inner: *const mlir.Value, + kernel: ?*Builder = null, + + pub const Kind = fly.TypeKind; + + pub fn type_(self: Value) *const mlir.Type { + return self.inner.type_(); + } + + fn kern(self: Value) *Builder { + return self.kernel orelse @panic("fly.Value has no owning Builder; use Builder.* helpers instead"); + } + + /// Derived, never stored: control_flow.zig builds Values with bare + /// `.{ .inner, .kernel }` literals. + pub fn kind(self: Value) Kind { + return fly.typeKind(self.type_()); + } + + pub fn shapeStatic(self: Value) IntTuple { + return self.kern().readIntTuple(fly.layoutLikeShape(self.type_())); + } + + /// Host value, not an op; see `emitSize`. + pub fn sizeStatic(self: Value) i64 { + return self.shapeStatic().product() orelse std.debug.panic("fly: `{f}` has a dynamic shape", .{self.type_()}); + } + + pub fn intTupleStatic(self: Value) IntTuple { + return self.kern().readIntTuple(fly.expect(self.type_(), .int_tuple)); + } + + pub fn elemDType(self: Value) DType { + const t = fly.elemType(self.type_()); + return dtypes.fromMlir(self.kern().ctx, t) orelse std.debug.panic("fly: unsupported element type `{f}`", .{t}); + } + + fn isI64(self: Value) bool { + const int_ty = self.type_().isA(mlir.IntegerType) orelse return false; + return int_ty.width() == 64; + } + + fn isFloatElem(self: Value) bool { + return self.scalarElemType().isFloat(); + } + + /// The element type of a vector, or the type itself. + fn scalarElemType(self: Value) *const mlir.Type { + if (self.type_().isA(mlir.VectorType)) |vec| return vec.shaped().elementType(); + return self.type_(); + } + + fn scalarDType(self: Value) DType { + const t = self.scalarElemType(); + return dtypes.fromMlir(self.kern().ctx, t) orelse std.debug.panic("fly: unsupported scalar type `{f}`", .{t}); + } + + // ------------------------------------------------------- layout algebra + // Every op below takes a Value or a comptime Layout/Tile/tuple literal, + // and FlyDSL infers the result type. + + fn unary(self: Value, comptime mnemonic: []const u8) Value { + const k = self.kern(); + return k.emit(fly.inferred(k.ctx, mnemonic, &.{self.inner}, .empty, k.loc())); + } + + fn binary(self: Value, comptime mnemonic: []const u8, rhs: anytype) Value { + const k = self.kern(); + const r = k.lift(rhs); + return k.emit(fly.inferred(k.ctx, mnemonic, &.{ self.inner, r.inner }, .empty, k.loc())); + } + + pub fn flatDivide(self: Value, tiler: anytype) Value { + return self.binary("flat_divide", tiler); + } + pub fn logicalDivide(self: Value, tiler: anytype) Value { + return self.binary("logical_divide", tiler); + } + pub fn zippedDivide(self: Value, tiler: anytype) Value { + return self.binary("zipped_divide", tiler); + } + pub fn tiledDivide(self: Value, tiler: anytype) Value { + return self.binary("tiled_divide", tiler); + } + pub fn logicalProduct(self: Value, tiler: anytype) Value { + return self.binary("logical_product", tiler); + } + pub fn zippedProduct(self: Value, tiler: anytype) Value { + return self.binary("zipped_product", tiler); + } + pub fn tiledProduct(self: Value, tiler: anytype) Value { + return self.binary("tiled_product", tiler); + } + pub fn flatProduct(self: Value, tiler: anytype) Value { + return self.binary("flat_product", tiler); + } + pub fn blockedProduct(self: Value, tiler: anytype) Value { + return self.binary("blocked_product", tiler); + } + pub fn rakedProduct(self: Value, tiler: anytype) Value { + return self.binary("raked_product", tiler); + } + pub fn composition(self: Value, inner: anytype) Value { + return self.binary("composition", inner); + } + pub fn rightInverse(self: Value) Value { + return self.unary("right_inverse"); + } + pub fn leftInverse(self: Value) Value { + return self.unary("left_inverse"); + } + pub fn coalesce(self: Value) Value { + return self.unary("coalesce"); + } + pub fn complement(self: Value) Value { + return self.unary("complement"); + } + + /// `t[coord]` with a `null` somewhere; `coord` is a tuple literal + /// (`.{ null, null, bx, by }`, nesting allowed) or an int-tuple Value. + pub fn slice(self: Value, coord: anytype) Value { + const k = self.kern(); + const c = k.intTuple(coord); + return k.emit(fly.inferred(k.ctx, "slice", &.{ self.inner, c.inner }, .empty, k.loc())); + } + pub fn dice(self: Value, coord: anytype) Value { + const k = self.kern(); + const c = k.intTuple(coord); + return k.emit(fly.inferred(k.ctx, "dice", &.{ self.inner, c.inner }, .empty, k.loc())); + } + + /// The sub-tuple / sub-layout at a mode path. + pub fn get(self: Value, mode: []const i32) Value { + const k = self.kern(); + return k.emit(fly.get(k.ctx, self.inner, mode, k.loc())); + } + pub fn selectModes(self: Value, indices: []const i32) Value { + const k = self.kern(); + return k.emit(fly.select(k.ctx, self.inner, indices, k.loc())); + } + pub fn take(self: Value, begin: i32, end: i32) Value { + const k = self.kern(); + return k.emit(fly.takeOrGroup(k.ctx, "take", self.inner, begin, end, k.loc())); + } + pub fn group(self: Value, begin: i32, end: i32) Value { + const k = self.kern(); + return k.emit(fly.takeOrGroup(k.ctx, "group", self.inner, begin, end, k.loc())); + } + + /// Emits an op; `shapeStatic` reads the type instead. + pub fn emitShape(self: Value) Value { + return self.unary("get_shape"); + } + pub fn emitStride(self: Value) Value { + return self.unary("get_stride"); + } + pub fn emitLayout(self: Value) Value { + return self.unary("get_layout"); + } + pub fn emitIter(self: Value) Value { + return self.unary("get_iter"); + } + pub fn emitSize(self: Value) Value { + return self.unary("size"); + } + pub fn cosize(self: Value) Value { + return self.unary("cosize"); + } + pub fn productEach(self: Value) Value { + return self.unary("int_tuple_product_each"); + } + + /// A register tensor shaped like `self`. + pub fn makeFragmentLike(self: Value, dtype: ?DType) Value { + const k = self.kern(); + const t: ?*const mlir.Type = if (dtype) |d| d.toMlir(k.ctx) else null; + return k.emit(fly.makeFragmentLike(k.ctx, self.inner, t, k.loc())); + } + + pub fn elemLess(self: Value, rhs: anytype) Value { + const k = self.kern(); + const r = k.intTuple(rhs); + return k.emit(fly.typed(k.ctx, "elem_less", &.{ self.inner, r.inner }, &.{.int(k.ctx, .i1)}, .empty, k.loc())); + } + + // ---------------------------------------------------------------- tensor access + + /// `t[coord]` with no `null` (a coord tensor yields an int tuple). + pub fn at(self: Value, coord: anytype) Value { + const k = self.kern(); + const c = k.intTuple(coord); + return k.emit(fly.inferred(k.ctx, "memref.load", &.{ self.inner, c.inner }, .empty, k.loc())); + } + + pub fn set(self: Value, coord: anytype, value: Value) void { + const k = self.kern(); + const c = k.intTuple(coord); + k.emitNone(fly.effect(k.ctx, "memref.store", &.{ value.inner, self.inner, c.inner }, .empty, k.loc())); + } + + /// The whole register tensor as a `vector`. + pub fn load(self: Value) Value { + return self.unary("memref.load_vec"); + } + + pub fn store(self: Value, vector: Value) void { + const k = self.kern(); + k.emitNone(fly.effect(k.ctx, "memref.store_vec", &.{ vector.inner, self.inner }, .empty, k.loc())); + } + + /// Store a splat vector of the tensor's static size. + pub fn fill(self: Value, value: anytype) void { + const k = self.kern(); + const n = self.sizeStatic(); + const dt = self.elemDType(); + const cst = k.constant(dt, value); + const vec_ty = mlir.Type.vector(&.{n}, dt.toMlir(k.ctx)); + const vec = k.emit(dialects.vector.broadcast(k.ctx, cst.inner, vec_ty, k.loc())); + self.store(vec); + } + + // ---------------------------------------------------------------- pointers + + pub fn addOffset(self: Value, offset: anytype) Value { + const k = self.kern(); + const o = k.intTuple(offset); + return k.emit(fly.inferred(k.ctx, "add_offset", &.{ self.inner, o.inner }, .empty, k.loc())); + } + + /// Scalar load. + pub fn ptrLoad(self: Value) Value { + const k = self.kern(); + const elem = self.elemDType().toMlir(k.ctx); + return k.emit(fly.typed(k.ctx, "ptr.load", &.{self.inner}, &.{elem}, .empty, k.loc())); + } + + pub fn ptrStore(self: Value, value: Value) void { + const k = self.kern(); + k.emitNone(fly.effect(k.ctx, "ptr.store", &.{ value.inner, self.inner }, .empty, k.loc())); + } + + /// A tensor over this pointer. + pub fn view(self: Value, lay: anytype) Value { + return self.binary("make_view", lay); + } + + // ---------------------------------------------------------------- arithmetic + + /// Broadcast a scalar against a vector operand. + fn coerceArith(self: Value, rhs: anytype) struct { Value, Value } { + const k = self.kern(); + const r: Value = if (@TypeOf(rhs) == Value) rhs else k.constLike(rhs, self); + const ln = self.vectorLen(); + const rn = r.vectorLen(); + if (ln == rn) return .{ self, r }; + if (rn == 1) return .{ self, k.emit(dialects.vector.broadcast(k.ctx, r.inner, self.type_(), k.loc())) }; + if (ln == 1) return .{ k.emit(dialects.vector.broadcast(k.ctx, self.inner, r.type_(), k.loc())), r }; + std.debug.panic("fly: cannot combine {f} with {f}", .{ self.type_(), r.type_() }); + } + + pub fn add(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emitFast(if (l.isFloatElem()) arith.addf(k.ctx, l.inner, r.inner, k.loc()) else arith.addi(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn sub(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emitFast(if (l.isFloatElem()) arith.subf(k.ctx, l.inner, r.inner, k.loc()) else arith.subi(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn mul(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emitFast(if (l.isFloatElem()) arith.mulf(k.ctx, l.inner, r.inner, k.loc()) else arith.muli(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn div(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emitFast(if (l.isFloatElem()) arith.divf(k.ctx, l.inner, r.inner, k.loc()) else arith.divsi(k.ctx, l.inner, r.inner, k.loc())); + } + /// Signed. + pub fn rem(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emit(arith.remsi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn toI32(self: Value) Value { + const k = self.kern(); + return k.emit(arith.index_cast(k.ctx, self.inner, .int(k.ctx, .i32), k.loc())); + } + + /// 1 for a scalar. + pub fn vectorLen(self: Value) i64 { + const vec = self.type_().isA(mlir.VectorType) orelse return 1; + return vec.dimension(0); + } + + /// Numeric conversion, scalar or vector. `i1` is treated as unsigned, so a + /// predicate converts to 1, not -1; equal-width float pairs (f16 <-> bf16, + /// the fp8 spellings) go through `arith.convertf`, which is the only op + /// that accepts them. + pub fn to(self: Value, dtype: DType) Value { + const k = self.kern(); + const src = self.scalarDType(); + if (src == dtype) return self; + const n = self.vectorLen(); + const result = if (n == 1) dtype.toMlir(k.ctx) else mlir.Type.vector(&.{n}, dtype.toMlir(k.ctx)); + const unsigned_src = src == .i1; + const op = if (src.isFloat() and dtype.isFloat()) + (if (dtype.bitWidth() > src.bitWidth()) + arith.extf(k.ctx, self.inner, result, k.loc()) + else if (dtype.bitWidth() < src.bitWidth()) + arith.truncf(k.ctx, self.inner, result, k.loc()) + else + arith.convertf(k.ctx, self.inner, result, .{}, k.loc())) + else if (src.isFloat()) + arith.fptosi(k.ctx, self.inner, result, k.loc()) + else if (dtype.isFloat()) + (if (unsigned_src) arith.uitofp(k.ctx, self.inner, result, k.loc()) else arith.sitofp(k.ctx, self.inner, result, k.loc())) + else if (dtype.bitWidth() > src.bitWidth()) + (if (unsigned_src) arith.extui(k.ctx, self.inner, result, k.loc()) else arith.extsi(k.ctx, self.inner, result, k.loc())) + else + arith.trunci(k.ctx, self.inner, result, k.loc()); + return k.emit(op); + } + + pub const Cmp = enum { eq, ne, lt, le, gt, ge }; + + /// Signed integer comparison against a Value or an integer literal. + pub fn cmp(self: Value, pred: Cmp, rhs: anytype) Value { + const k = self.kern(); + const r: Value = if (@TypeOf(rhs) == Value) rhs else k.constLike(rhs, self); + const p: arith.CmpIPredicate = switch (pred) { + .eq => .eq, + .ne => .ne, + .lt => .slt, + .le => .sle, + .gt => .sgt, + .ge => .sge, + }; + return k.emit(arith.cmpi(k.ctx, p, self.inner, r.inner, k.loc())); + } + + /// Ordered. + pub fn cmpf(self: Value, pred: Cmp, rhs: Value) Value { + const k = self.kern(); + const p: arith.CmpFPredicate = switch (pred) { + .eq => .oeq, + .ne => .one, + .lt => .olt, + .le => .ole, + .gt => .ogt, + .ge => .oge, + }; + return k.emit(arith.cmpf(k.ctx, p, self.inner, rhs.inner, k.loc())); + } + + pub fn select(self: Value, a: Value, b: Value) Value { + const k = self.kern(); + return k.emit(arith.select(k.ctx, self.inner, a.inner, b.inner, k.loc())); + } + + /// Lane `k` reads lane `k ^ offset`, within `width` lanes. + pub fn shuffleXor(self: Value, offset: i32, width: i32) Value { + return self.shuffle(.xor, offset, width); + } + + pub const ShuffleMode = enum { xor, up, down, idx }; + + pub fn shuffle(self: Value, mode: ShuffleMode, offset: i32, width: i32) Value { + const k = self.kern(); + const off = k.constant(.i32, offset); + const wid = k.constant(.i32, width); + const mode_attr = switch (mode) { + inline else => |m| fly.parseAttr(k.ctx, "#gpu"), + }; + const op = fly.make(k.ctx, "gpu.shuffle", .{ + .operands = .{ .flat = &.{ self.inner, off.inner, wid.inner } }, + .results = .{ .flat = &.{ self.type_(), .int(k.ctx, .i1) } }, + .attributes = &.{.named(k.ctx, "mode", mode_attr)}, + .location = k.loc(), + }); + return k.emit(op); + } + + pub const ReduceKind = enum { add, mul, minf, maxf, minsi, maxsi, and_, or_, xor }; + + /// Reduce a vector to a scalar. + pub fn reduce(self: Value, reduce_kind: ReduceKind) Value { + const k = self.kern(); + const result = self.scalarElemType(); + const ck: dialects.vector.CombiningKind = switch (reduce_kind) { + .add => .add, + .mul => .mul, + .minf => .minnumf, + .maxf => .maxnumf, + .minsi => .minsi, + .maxsi => .maxsi, + .and_ => .@"and", + .or_ => .@"or", + .xor => .xor, + }; + const op = dialects.vector.reduction(k.ctx, ck, self.inner, result, .{}, k.loc()); + return k.emitFast(op); + } + + /// Static position. + pub fn extract(self: Value, i: i64) Value { + const k = self.kern(); + return k.emit(dialects.vector.extract(k.ctx, self.inner, &.{i}, &.{}, self.scalarElemType(), k.loc())); + } +}; + +// ============================================================================= +// Tiled copy / MMA handles +// ============================================================================= + +/// A `!fly.tiled_copy` plus the tile it was built with. +pub const TiledCopy = struct { + value: Value, + tile: Value, + + pub fn getSlice(self: TiledCopy, thr_idx: Value) ThrCopy { + return .{ .tiled_copy = self.value, .thr = self.value.kern().intTuple(thr_idx) }; + } + + pub fn tileMN(self: TiledCopy) Value { + return self.tile; + } +}; + +/// Per-thread slice of a TiledCopy. +pub const ThrCopy = struct { + tiled_copy: Value, + thr: Value, + + pub fn partitionS(self: ThrCopy, src: Value) Value { + const k = self.tiled_copy.kern(); + return k.emit(fly.inferred(k.ctx, "tiled_copy.partition_src", &.{ self.tiled_copy.inner, src.inner, self.thr.inner }, .empty, k.loc())); + } + + pub fn partitionD(self: ThrCopy, dst: Value) Value { + const k = self.tiled_copy.kern(); + return k.emit(fly.inferred(k.ctx, "tiled_copy.partition_dst", &.{ self.tiled_copy.inner, dst.inner, self.thr.inner }, .empty, k.loc())); + } + + pub fn retile(self: ThrCopy, t: Value) Value { + const k = self.tiled_copy.kern(); + return k.emit(fly.inferred(k.ctx, "tiled_copy.retile", &.{ self.tiled_copy.inner, t.inner }, .empty, k.loc())); + } +}; + +pub const TiledMma = struct { + value: Value, + + pub fn getSlice(self: TiledMma, thr_idx: Value) ThrMma { + return .{ .tiled_mma = self.value, .thr = self.value.kern().intTuple(thr_idx) }; + } + + pub const FragmentOpts = struct { + stages: ?i32 = null, + }; + + fn makeFragment(self: TiledMma, operand: fly.MmaOperand, input: Value, opts: FragmentOpts) Value { + const k = self.value.kern(); + var a: fly.Attrs = .empty; + a.appendAssumeCapacity(.named(k.ctx, "operand_id", Builder.must(fly.attributes.mmaOperandAttr(k.ctx, operand)))); + if (opts.stages) |st| a.appendAssumeCapacity(.named(k.ctx, "stages", .int(k.ctx, .i32, st))); + return k.emit(fly.inferred(k.ctx, "mma.make_fragment", &.{ self.value.inner, input.inner }, a, k.loc())); + } + + /// Per-thread register fragment for the A operand. + pub fn makeFragmentA(self: TiledMma, a: Value) Value { + return self.makeFragment(.a, a, .{}); + } + pub fn makeFragmentB(self: TiledMma, b: Value) Value { + return self.makeFragment(.b, b, .{}); + } + pub fn makeFragmentC(self: TiledMma, c: Value) Value { + return self.makeFragment(.c, c, .{}); + } + pub fn makeFragmentAOpts(self: TiledMma, a: Value, opts: FragmentOpts) Value { + return self.makeFragment(.a, a, opts); + } + pub fn makeFragmentBOpts(self: TiledMma, b: Value, opts: FragmentOpts) Value { + return self.makeFragment(.b, b, opts); + } +}; + +pub const ThrMma = struct { + tiled_mma: Value, + thr: Value, + + fn partition(self: ThrMma, operand: fly.MmaOperand, t: Value) Value { + const k = self.tiled_mma.kern(); + return k.emit(fly.inferred(k.ctx, "tiled_mma.partition", &.{ self.tiled_mma.inner, t.inner, self.thr.inner }, fly.attrs(&.{ + .named(k.ctx, "operand_id", fly.mmaOperandAttr(k.ctx, operand)), + }), k.loc())); + } + + pub fn partitionA(self: ThrMma, a: Value) Value { + return self.partition(.a, a); + } + pub fn partitionB(self: ThrMma, b: Value) Value { + return self.partition(.b, b); + } + pub fn partitionC(self: ThrMma, c: Value) Value { + return self.partition(.c, c); + } +}; + +// ============================================================================= +// Builder +// ============================================================================= + +pub const Builder = struct { + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + ctx: *mlir.Context, + /// `builtin.module attributes {gpu.container_module}`. + module: *mlir.Module, + gpu_module: *mlir.Operation, + gpu_body: *mlir.Block, + name: []const u8, + func_op: ?*mlir.Operation = null, + entry_block: ?*mlir.Block = null, + block_stack: std.ArrayList(*mlir.Block) = .empty, + arg_specs: []const ArgSpec = &.{}, + /// The `!fly.memref` view for tensor arguments, the raw `!fly.ptr` otherwise. + arg_values: []Value = &.{}, + target: Arch = .gfx942, + fast_math: bool = false, + + pub const ArgSpec = struct { + name: []const u8, + dtype: DType, + space: fly.AddressSpace = .global, + /// Row-major tensor argument; null for a bare pointer. + dims: ?[]const i64 = null, + }; + + pub fn open(allocator: std.mem.Allocator, ctx: *mlir.Context, name: []const u8) !Builder { + const loc_: *const mlir.Location = .unknown(ctx); + const module: *mlir.Module = .init(loc_); + errdefer module.deinit(); + module.operation().setAttributeByName("gpu.container_module", .unit(ctx)); + + const gpu_body = mlir.Block.init(&.{}, &.{}); + const gpu_module = gpu.module(ctx, gpu_module_name, gpu_body, loc_); + _ = gpu_module.appendTo(module.body()); + + var arena: std.heap.ArenaAllocator = .init(allocator); + errdefer arena.deinit(); + + return .{ + .allocator = allocator, + .arena = arena, + .ctx = ctx, + .module = module, + .gpu_module = gpu_module, + .gpu_body = gpu_body, + .name = name, + }; + } + + pub fn deinit(self: *Builder) void { + self.module.deinit(); + self.arena.deinit(); + } + + /// Arguments by name. Each field is `.{ .ptr = dtype }` or + /// `.{ .tensor = .{ .dtype = ..., .dims = &.{ ... } } }`. + pub fn declareArgs(self: *Builder, spec: anytype) !dsl.NamedArgs(@TypeOf(spec), Value) { + const Spec = @TypeOf(spec); + const fields = @typeInfo(Spec).@"struct".fields; + var specs: [fields.len]ArgSpec = undefined; + inline for (fields, 0..) |f, i| { + const raw = @field(spec, f.name); + const variant = @typeInfo(@TypeOf(raw)).@"struct".fields[0].name; + const inner = @field(raw, variant); + specs[i] = if (comptime std.mem.eql(u8, variant, "ptr")) + .{ .name = f.name, .dtype = inner } + else if (comptime std.mem.eql(u8, variant, "tensor")) + .{ .name = f.name, .dtype = inner.dtype, .dims = inner.dims, .space = if (@hasField(@TypeOf(inner), "space")) inner.space else .global } + else + @compileError("fly.declareArgs: argument `" ++ f.name ++ "` must be `.{ .ptr = dtype }` or `.{ .tensor = .{ .dtype, .dims } }`"); + } + try self.declareArgsLow(&specs); + var named: dsl.NamedArgs(Spec, Value) = undefined; + inline for (fields, 0..) |f, i| @field(named, f.name) = self.arg(i); + return named; + } + + /// One `!fly.ptr` per spec, in order; tensor + /// arguments are bridged to `!fly.memref` views. + pub fn declareArgsLow(self: *Builder, specs: []const ArgSpec) !void { + std.debug.assert(self.entry_block == null); + const ctx = self.ctx; + const scratch = self.arena.allocator(); + self.arg_specs = try scratch.dupe(ArgSpec, specs); + + const arg_types = try scratch.alloc(*const mlir.Type, specs.len); + const arg_locs = try scratch.alloc(*const mlir.Location, specs.len); + for (arg_types, arg_locs, specs) |*ty_, *a_loc, a| { + ty_.* = self.ptrType(a.dtype.storageElem(), self.addressSpaceAttr(a.space)); + a_loc.* = self.loc().named(ctx, a.name); + } + + const entry = mlir.Block.init(arg_types, arg_locs); + const func_op = gpu.func(ctx, .{ + .name = self.name, + .args = arg_types, + .block = entry, + .location = self.loc(), + }); + _ = func_op.appendTo(self.gpu_body); + self.func_op = func_op; + self.entry_block = entry; + + // The view keeps the storage element type: recasting a predicate + // argument back to `!fly.ptr` would make a copy atom's element + // count `bits / 1` while it still moves `bits / 8` bytes. + self.arg_values = try scratch.alloc(Value, specs.len); + for (specs, 0..) |a, i| { + const raw = self.rawArg(i); + self.arg_values[i] = if (a.dims) |dims| raw.view(self.staticRowMajor(dims)) else raw; + } + } + + pub fn rawArg(self: *Builder, i: usize) Value { + const eb = self.entry_block orelse @panic("Builder.rawArg called before declareArgs"); + return .{ .inner = eb.argument(i), .kernel = self }; + } + + /// A tensor view when the spec has dims, else the raw pointer. + pub fn arg(self: *Builder, i: usize) Value { + return self.arg_values[i]; + } + + pub fn pushBlock(self: *Builder, b: *mlir.Block) void { + self.block_stack.append(self.arena.allocator(), b) catch @panic("Builder.pushBlock OOM"); + } + + pub fn popBlock(self: *Builder) void { + _ = self.block_stack.pop(); + } + + pub fn currentBlock(self: *Builder) *mlir.Block { + if (self.block_stack.items.len > 0) { + return self.block_stack.items[self.block_stack.items.len - 1]; + } + return self.entry_block orelse @panic("Builder has no current block; call declareArgs first"); + } + + pub fn loc(self: *const Builder) *const mlir.Location { + return .unknown(self.ctx); + } + + pub fn emit(self: *Builder, op: *mlir.Operation) Value { + _ = op.appendTo(self.currentBlock()); + return .{ .inner = op.result(0), .kernel = self }; + } + + pub fn emitNone(self: *Builder, op: *mlir.Operation) void { + _ = op.appendTo(self.currentBlock()); + } + + // ---------------------------------------------------------------- types + + /// Arena-allocated. + pub fn readIntTuple(self: *Builder, tuple_ty: *const fly.types.IntTupleType) IntTuple { + if (tuple_ty.isLeaf()) { + return switch (tuple_ty.getLeaf()) { + .static => |v| IntTuple.static(v), + .dynamic, .basis => IntTuple.dyn, + .none => IntTuple.star, + }; + } + const n = tuple_ty.getNumElements(); + const elems = self.alloc(IntTuple, n); + for (0..n) |i| elems[i] = self.readIntTuple(tuple_ty.getElement(i)); + return .{ .tup = elems }; + } + + fn alloc(self: *Builder, comptime T: type, n: usize) []T { + return self.arena.allocator().alloc(T, n) catch @panic("fly: OOM"); + } + + /// The shape and stride of a static `!fly.layout`, as host literals. + pub fn readLayout(self: *Builder, layout_ty: *const mlir.Type) Layout { + const lay = fly.expect(layout_ty, .layout); + const shape = lay.getShape(); + const stride = lay.getStride(); + if (!shape.isStatic() or !stride.isStatic()) { + std.debug.panic("fly: `{f}` is not static", .{layout_ty}); + } + return .{ .shape = self.readIntTuple(shape), .stride = self.readIntTuple(stride) }; + } + + /// A fly type or attribute is rejected only on malformed input, which is + /// a bug in the kernel, not a runtime condition. + pub fn must(result: anytype) @typeInfo(@TypeOf(result)).error_union.payload { + return result catch |err| std.debug.panic("fly: cannot build type ({})", .{err}); + } + + pub fn intTupleType(self: *Builder, t: IntTuple) *const mlir.Type { + return must(fly.types.IntTupleType.get(self.ctx, .{ .attr = must(t.toAttr(self.ctx)) })).type_(); + } + + pub fn layoutType(self: *Builder, lay: Layout) *const mlir.Type { + return must(fly.types.LayoutType.get(self.ctx, .{ .attr = must(lay.toAttr(self.ctx)) })).type_(); + } + + pub fn tileType(self: *Builder, t: Tile) *const mlir.Type { + return must(fly.types.TileType.get(self.ctx, .{ .attr = must(t.toAttr(self.ctx)) })).type_(); + } + + pub fn ptrType(self: *Builder, dtype: DType, space: *const mlir.Attribute) *const mlir.Type { + return must(fly.types.PointerType.get(self.ctx, .{ .elemTy = dtype.toMlir(self.ctx), .addressSpace = space })).type_(); + } + + fn addressSpaceAttr(self: *Builder, space: fly.AddressSpace) *const mlir.Attribute { + return must(fly.attributes.AddressSpaceAttr.get(self.ctx, .{ .addressSpace = space })).attribute(); + } + + /// `!fly.memref`. + pub fn memRefType(self: *Builder, dtype: DType, space: fly.AddressSpace, lay: Layout) *const mlir.Type { + return must(fly.types.MemRefType.get(self.ctx, .{ + .elemTy = dtype.toMlir(self.ctx), + .addressSpace = self.addressSpaceAttr(space), + .layout = must(lay.toAttr(self.ctx)).attribute(), + })).type_(); + } + + // ---------------------------------------------------------------- statics + + /// `fly.static` of an already-built type. + pub fn staticTy(self: *Builder, t: *const mlir.Type) Value { + return self.emit(fly.static(self.ctx, t, self.loc())); + } + + pub fn static(self: *Builder, v: anytype) Value { + const T = @TypeOf(v); + return self.staticTy(if (T == Layout) + self.layoutType(v) + else if (T == Tile) + self.tileType(v) + else if (T == IntTuple) + self.intTupleType(v) + else + @compileError("fly.static: expected a Layout, Tile or IntTuple, got " ++ @typeName(T))); + } + + pub fn staticRowMajor(self: *Builder, dims: []const i64) Value { + if (dims.len == 1) return self.static(Layout{ .shape = .static(dims[0]), .stride = .static(1) }); + const shape = self.alloc(IntTuple, dims.len); + const stride = self.alloc(IntTuple, dims.len); + var acc: i64 = 1; + var i: usize = dims.len; + while (i > 0) { + i -= 1; + shape[i] = .static(dims[i]); + stride[i] = .static(acc); + acc *= dims[i]; + } + return self.static(Layout{ .shape = .{ .tup = shape }, .stride = .{ .tup = stride } }); + } + + /// A Value passes through; a bare tuple literal (`.{ 64, 8 }`) is a tile. + pub fn lift(self: *Builder, v: anytype) Value { + const T = @TypeOf(v); + if (T == Value) return v; + if (T == Layout or T == Tile or T == IntTuple) return self.static(v); + if (@typeInfo(T) == .@"struct" and @typeInfo(T).@"struct".is_tuple) return self.static(tile(v)); + @compileError("fly: expected a Value, Layout, Tile, IntTuple or tuple literal, got " ++ @typeName(T)); + } + + /// A (possibly nested) int tuple from integers (static leaves), `null` + /// (`*`), `IntTuple` literals and `Value`s (dynamic leaves, passed as + /// operands). An `!fly.int_tuple` Value passes through. + pub fn intTuple(self: *Builder, elems: anytype) Value { + if (@TypeOf(elems) == Value and elems.kind() == .int_tuple) return elems; + var operands: stdx.BoundedArray(*const mlir.Value, 32) = .empty; + const spec = self.intTupleSpec(&operands, elems); + const t = self.intTupleType(spec); + return self.emit(fly.makeIntTuple(self.ctx, operands.constSlice(), t, self.loc())); + } + + /// `intTuple`, spelled for coordinates. + pub const coord = intTuple; + + /// The literal for `intTuple`, collecting each dynamic leaf as an operand. + fn intTupleSpec(self: *Builder, operands: *stdx.BoundedArray(*const mlir.Value, 32), v: anytype) IntTuple { + const T = @TypeOf(v); + if (T == Value) { + if (v.kind() == .int_tuple) { + std.debug.panic("fly.intTuple: nested !fly.int_tuple values are not supported; pass scalars", .{}); + } + operands.appendAssumeCapacity(v.inner); + return if (v.isI64()) .dynamic(64, 1) else .dyn; + } + if (T == IntTuple) return v; + if (T == @TypeOf(null)) return .star; + switch (@typeInfo(T)) { + .comptime_int, .int => return .static(@intCast(v)), + .@"struct" => |s| { + if (!s.is_tuple) @compileError("fly.intTuple: expected an integer, null, Value, IntTuple or tuple, got " ++ @typeName(T)); + const elems = self.alloc(IntTuple, s.fields.len); + inline for (s.fields, 0..) |f, i| elems[i] = self.intTupleSpec(operands, @field(v, f.name)); + return .{ .tup = elems }; + }, + else => @compileError("fly.intTuple: expected an integer, null, Value, IntTuple or tuple, got " ++ @typeName(T)), + } + } + + // ---------------------------------------------------------------- scalars & ids + + /// Only the representation `dtype` needs is computed: a float constant + /// outside i64 range (inf, floatMax) must not be forced through + /// `@intFromFloat`, which traps. + pub fn constant(self: *Builder, dtype: DType, value: anytype) Value { + const is_float_lit = switch (@typeInfo(@TypeOf(value))) { + .float, .comptime_float => true, + else => false, + }; + return switch (dtype) { + .f16, .bf16, .f32, .f64 => blk: { + const f: f64 = if (is_float_lit) @floatCast(value) else @floatFromInt(value); + break :blk self.emit(switch (dtype) { + .f16 => arith.constant_float(self.ctx, f, .f16, self.loc()), + .bf16 => arith.constant_float(self.ctx, f, .bf16, self.loc()), + .f32 => arith.constant_float(self.ctx, f, .f32, self.loc()), + else => arith.constant_float(self.ctx, f, .f64, self.loc()), + }); + }, + .i1, .i8, .i16, .i32, .i64 => blk: { + const i: i64 = if (is_float_lit) @intFromFloat(value) else @intCast(value); + break :blk self.emit(arith.constant_int(self.ctx, i, dtype.toMlir(self.ctx), self.loc())); + }, + else => std.debug.panic("fly.constant: no constant for {s}", .{@tagName(dtype)}), + }; + } + + pub fn constIndex(self: *Builder, value: i64) Value { + return self.emit(arith.constant_index(self.ctx, value, self.loc())); + } + + fn idI32(self: *Builder, op: *mlir.Operation) Value { + return self.emit(op).toI32(); + } + + /// Already cast to `i32`: `fly.make_int_tuple` takes only i32/i64. + pub fn threadId(self: *Builder, dim: Dim) Value { + return self.idI32(gpu.thread_id(self.ctx, dim, self.loc())); + } + pub fn blockId(self: *Builder, dim: Dim) Value { + return self.idI32(gpu.block_id(self.ctx, dim, self.loc())); + } + pub fn blockDim(self: *Builder, dim: Dim) Value { + return self.idI32(gpu.block_dim(self.ctx, dim, self.loc())); + } + pub fn gridDim(self: *Builder, dim: Dim) Value { + return self.idI32(gpu.grid_dim(self.ctx, dim, self.loc())); + } + + pub fn barrier(self: *Builder) void { + self.emitNone(gpu.barrier(self.ctx, self.loc())); + } + + /// Tag float arithmetic emitted from now on with `fastmath`. + pub fn setFastMath(self: *Builder, on: bool) void { + self.fast_math = on; + } + + pub fn emitFast(self: *Builder, op: *mlir.Operation) Value { + if (self.fast_math and op.numResults() > 0 and (Value{ .inner = op.result(0), .kernel = self }).isFloatElem()) { + op.setAttributeByName("fastmath", fly.parseAttr(self.ctx, "#arith.fastmath")); + } + return self.emit(op); + } + + pub fn rsqrt(self: *Builder, x: Value) Value { + return self.emitFast(dialects.math.rsqrt(self.ctx, x.inner, self.loc())); + } + pub fn sqrt(self: *Builder, x: Value) Value { + return self.emitFast(dialects.math.sqrt(self.ctx, x.inner, self.loc())); + } + pub fn exp(self: *Builder, x: Value) Value { + return self.emitFast(dialects.math.exp(self.ctx, x.inner, self.loc())); + } + + /// Static LDS, as an `n:1` tensor in shared memory. Each call is its own + /// `fly.make_ptr {allocBytes, allocAlign}` leaf, which the plugin lowers + /// to one LLVM global, so `shared_mem_bytes` may stay 0. + pub fn sharedArray(self: *Builder, dtype: DType, n: i64, alignment: i32) Value { + const bytes: i64 = @divExact(@as(i64, dtype.bitWidth()) * n, 8); + const raw_ty = must(fly.types.PointerType.get(self.ctx, .{ + .elemTy = .int(self.ctx, .i8), + .addressSpace = self.addressSpaceAttr(.shared), + .alignment = must(fly.attributes.AlignAttr.get(self.ctx, .{ .alignment = alignment })), + })).type_(); + const dict: *const mlir.Attribute = .dict(self.ctx, &.{ + .named(self.ctx, "allocBytes", .int(self.ctx, .i64, bytes)), + .named(self.ctx, "allocAlign", .int(self.ctx, .i64, alignment)), + }); + const raw = self.emit(fly.makePtr(self.ctx, &.{}, raw_ty, dict, self.loc())); + const typed = self.recast(raw, dtype); + return typed.view(self.static(Layout{ .shape = .static(n), .stride = .static(1) })); + } + + /// A single atom-sized copy. + pub fn copyAtomCall(self: *Builder, atom: Value, src: Value, dst: Value) void { + self.emitNone(fly.effect(self.ctx, "copy_atom_call", &.{ atom.inner, src.inner, dst.inner }, .empty, self.loc())); + } + + /// One atom-sized load straight into a `vector`. + /// + /// Required over an rmem tensor + `copyAtomCall` + `load` in any kernel + /// with control flow: the plugin lowers `scf` to `cf` before the Fly + /// passes, and Fly's register-alloca promotion only handles one block. + pub fn copyAtomLoad(self: *Builder, atom: Value, src: Value) Value { + const n = src.sizeStatic(); + const dt = src.elemDType(); + const vec_ty = mlir.Type.vector(&.{n}, dt.toMlir(self.ctx)); + const op = fly.make(self.ctx, "fly.copy_atom_call_ssa", .{ + .operands = .{ .flat = &.{ atom.inner, src.inner } }, + .results = .{ .flat = &.{vec_ty} }, + .attributes = &.{.named(self.ctx, "operandSegmentSizes", .denseArray(self.ctx, .i32, &.{ 1, 1, 0, 0 }))}, + .location = self.loc(), + }); + return self.emit(op); + } + + /// One atom-sized store of a `vector`; see `copyAtomLoad`. + pub fn copyAtomStore(self: *Builder, atom: Value, vec: Value, dst: Value) void { + const op = fly.make(self.ctx, "fly.copy_atom_call_ssa", .{ + .operands = .{ .flat = &.{ atom.inner, vec.inner, dst.inner } }, + .attributes = &.{.named(self.ctx, "operandSegmentSizes", .denseArray(self.ctx, .i32, &.{ 1, 1, 1, 0 }))}, + .location = self.loc(), + }); + self.emitNone(op); + } + + // ---------------------------------------------------------------- tensors + + pub fn makeView(self: *Builder, iter: Value, lay: anytype) Value { + _ = self; + return iter.view(lay); + } + + /// A coordinate tensor: value == logical coord. + pub fn identity(self: *Builder, shape: anytype) Value { + const shp = self.intTuple(shape); + const lay = self.emit(fly.inferred(self.ctx, "make_identity_layout", &.{shp.inner}, .empty, self.loc())); + const rank_ = shp.intTupleStatic().rank(); + const zero: IntTuple = if (rank_ == 1) .static(0) else blk: { + const zeros = self.alloc(IntTuple, rank_); + for (zeros) |*z| z.* = .static(0); + break :blk .{ .tup = zeros }; + }; + const base_ty = self.intTupleType(zero); + const base = self.emit(fly.makeIntTuple(self.ctx, &.{}, base_ty, self.loc())); + return base.view(lay); + } + + /// A register tensor; see `copyAtomLoad` before using one in a kernel + /// with control flow. + pub fn rmemTensor(self: *Builder, comptime lay: Layout, dtype: DType) Value { + const lay_v = self.static(lay); + const t = self.memRefType(dtype, .register, lay); + return self.emit(fly.typed(self.ctx, "memref.alloca", &.{lay_v.inner}, &.{t}, .empty, self.loc())); + } + + /// `rmemTensor` with a runtime `Layout`. + pub fn rmemTensorRuntime(self: *Builder, lay: Layout, dtype: DType) Value { + const lay_v = self.static(lay); + const t = self.memRefType(dtype, .register, lay); + return self.emit(fly.typed(self.ctx, "memref.alloca", &.{lay_v.inner}, &.{t}, .empty, self.loc())); + } + + /// Reinterpret a pointer as another element type, keeping the address + /// space, alignment and swizzle. + pub fn recast(self: *Builder, ptr: Value, dtype: DType) Value { + const t = must(fly.ptrWithElem(self.ctx, ptr.type_(), dtype.toMlir(self.ctx))).type_(); + return self.emit(fly.typed(self.ctx, "recast_iter", &.{ptr.inner}, &.{t}, .empty, self.loc())); + } + + // ---------------------------------------------------------------- atoms + + /// The copy-op type an atom wraps, by bit size. + pub const CopyOp = union(enum) { + universal: i32, + buffer_copy: i32, + buffer_copy_lds: i32, + + fn type_(self: CopyOp, ctx: *mlir.Context) *const mlir.Type { + return switch (self) { + .universal => |b| must(fly.types.CopyOpUniversalCopyType.get(ctx, .{ .bitSize = b })).type_(), + .buffer_copy => |b| must(fly.rocdl.CopyOpCDNA3BufferCopyType.get(ctx, .{ .bitSize = b })).type_(), + .buffer_copy_lds => |b| must(fly.rocdl.CopyOpCDNA3BufferCopyLDSType.get(ctx, .{ .bitSize = b })).type_(), + }; + } + }; + + pub fn copyAtom(self: *Builder, op: CopyOp, dtype: DType) Value { + const bits: i32 = @intCast(dtype.bitWidth()); + const t = must(fly.types.CopyAtomType.get(self.ctx, .{ .copyOp = op.type_(self.ctx), .valBits = bits })).type_(); + return self.emit(fly.makeCopyAtom(self.ctx, t, bits, self.loc())); + } + + /// `op` is an MMA op type, such as a `fly.rocdl.MmaOpCDNA3MFMAType`. + pub fn mmaAtom(self: *Builder, op: *const mlir.Type) Value { + const t = must(fly.types.MmaAtomType.get(self.ctx, .{ .mmaOp = op })).type_(); + return self.emit(fly.makeMmaAtom(self.ctx, t, self.loc())); + } + + pub fn tiledCopy(self: *Builder, atom: Value, layout_tv: anytype, tile_mn: anytype) TiledCopy { + const tv = self.lift(layout_tv); + const tl = self.lift(tile_mn); + const v = self.emit(fly.inferred(self.ctx, "make_tiled_copy", &.{ atom.inner, tv.inner, tl.inner }, .empty, self.loc())); + return .{ .value = v, .tile = tl }; + } + + /// raked product -> right inverse -> composition -> make_tiled_copy, + /// with the tiler read off the inferred product shape. + pub fn tiledCopyTV(self: *Builder, atom: Value, thr: Layout, val: Layout) TiledCopy { + const thr_size = thr.size() orelse @panic("fly.tiledCopyTV: thr_layout must be static"); + const val_size = val.size() orelse @panic("fly.tiledCopyTV: val_layout must be static"); + const layout_mn = self.static(thr).rakedProduct(self.static(val)); + const tmp = self.static(Layout{ + .shape = .{ .tup = &.{ .static(thr_size), .static(val_size) } }, + .stride = .{ .tup = &.{ .static(1), .static(thr_size) } }, + }); + const layout_tv = layout_mn.rightInverse().composition(tmp); + const tiler = layout_mn.emitShape().productEach(); + // The tiler as a tile literal: one leaf per mode, `[a|b]`. + const shape = tiler.intTupleStatic(); + const tile_modes = self.alloc(Tile, shape.rank()); + for (tile_modes, 0..) |*m, i| m.* = .{ .leaf = switch (shape.at(i)) { + .leaf => |l| l, + else => std.debug.panic("fly.tiledCopyTV: tiler mode {d} is not a leaf", .{i}), + } }; + return self.tiledCopy(atom, layout_tv, Tile{ .modes = tile_modes }); + } + + /// A TiledCopy matched to an MMA operand: the TV layout and tile size + /// come off the `!fly.tiled_mma` type. + pub fn tiledCopyFor(self: *Builder, operand: fly.MmaOperand, copy_atom: Value, tm: TiledMma) TiledCopy { + const tm_ty = fly.expect(tm.value.type_(), .tiled_mma); + const layout_tv = self.staticTy((switch (operand) { + .a => tm_ty.getTiledThrValLayoutA(), + .b => tm_ty.getTiledThrValLayoutB(), + .c => tm_ty.getTiledThrValLayoutC(), + .d => @panic("fly.tiledCopyFor: no D operand copy"), + }).type_()); + const tile_size = self.staticTy(tm_ty.getTileSizeMNK().type_()); + const modes: [2]i32 = switch (operand) { + .a => .{ 0, 2 }, + .b => .{ 1, 2 }, + .c => .{ 0, 1 }, + .d => unreachable, + }; + const one = self.intTuple(1); + const tile_modes = self.alloc(Tile, modes.len); + for (modes, 0..) |m, i| { + const lay = self.emit(fly.inferred(self.ctx, "make_layout", &.{ tile_size.selectModes(&.{m}).inner, one.inner }, .empty, self.loc())); + tile_modes[i] = .{ .layout = self.readLayout(lay.type_()) }; + } + return self.tiledCopy(copy_atom, layout_tv, Tile{ .modes = tile_modes }); + } + + pub fn tiledCopyA(self: *Builder, copy_atom: Value, tm: TiledMma) TiledCopy { + return self.tiledCopyFor(.a, copy_atom, tm); + } + pub fn tiledCopyB(self: *Builder, copy_atom: Value, tm: TiledMma) TiledCopy { + return self.tiledCopyFor(.b, copy_atom, tm); + } + pub fn tiledCopyC(self: *Builder, copy_atom: Value, tm: TiledMma) TiledCopy { + return self.tiledCopyFor(.c, copy_atom, tm); + } + + /// The same tensor over a CDNA buffer resource + /// (`!fly.ptr`), for hardware bounds-checked + /// accesses and `buffer_copy` atoms. Descriptor size is the maximum. + pub fn bufferTensor(self: *Builder, t: Value) Value { + const ptr = t.emitIter(); + const lay = t.emitLayout(); + const c0 = self.constant(.i16, 0); + const nrec = self.constant(.i64, 0xFFFFFFFF); + // (7 << 12) | (4 << 15): the CDNA descriptor flags FlyDSL and XLA use. + const flags = self.constant(.i32, 0x27000); + const elem = t.elemDType(); + const buf_ty = self.ptrType(elem, (must(fly.rocdl.BufferDescAddressAttr.get(self.ctx))).attribute()); + const buf_ptr = self.emit(fly.makePtr(self.ctx, &.{ ptr.inner, c0.inner, nrec.inner, flags.inner }, buf_ty, null, self.loc())); + return buf_ptr.view(lay); + } + + pub fn tiledMma(self: *Builder, atom: Value, atom_layout: anytype, permutation: ?Value) TiledMma { + const al = self.lift(atom_layout); + const v = if (permutation) |p| + self.emit(fly.inferred(self.ctx, "make_tiled_mma", &.{ atom.inner, al.inner, p.inner }, .empty, self.loc())) + else + self.emit(fly.inferred(self.ctx, "make_tiled_mma", &.{ atom.inner, al.inner }, .empty, self.loc())); + return .{ .value = v }; + } + + pub const CopyOpts = struct { + pred: ?Value = null, + }; + + pub fn copy(self: *Builder, atom: Value, src: Value, dst: Value, opts: CopyOpts) void { + self.emitNone(fly.copy(self.ctx, atom.inner, src.inner, dst.inner, .{ + .pred = if (opts.pred) |p| p.inner else null, + }, self.loc())); + } + + pub fn gemm(self: *Builder, atom: Value, d: Value, a: Value, b: Value, c: Value) void { + self.emitNone(fly.gemm(self.ctx, atom.inner, d.inner, a.inner, b.inner, c.inner, .{}, self.loc())); + } + + // ---------------------------------------------------------------- control flow + + pub fn ForScope(comptime N: usize) type { + return dsl.ForScope(Builder, Value, N); + } + pub const IfOnlyScope = dsl.IfOnlyScope(Builder, Value); + pub fn IfScope(comptime N: usize) type { + return dsl.IfScope(Builder, Value, N); + } + pub fn WhileScope(comptime N: usize, comptime M: usize) type { + return dsl.WhileScope(Builder, Value, N, M); + } + + /// Loop-carried `inits`; the induction variable is `index` unless + /// `lower` is a Value. + pub fn openFor( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + ) ForScope(tupleArity(@TypeOf(inits), "openFor: inits")) { + const N = comptime tupleArity(@TypeOf(inits), "openFor: inits"); + const fields = @typeInfo(@TypeOf(inits)).@"struct".fields; + + const lb_v: Value = if (@TypeOf(lower) == Value) lower else self.constIndex(lower); + const ub_v: Value = if (@TypeOf(upper) == Value) upper else self.constLike(upper, lb_v); + const step_v: Value = if (@TypeOf(step) == Value) step else self.constLike(step, lb_v); + + var block_types: [N + 1]*const mlir.Type = undefined; + var block_locs: [N + 1]*const mlir.Location = undefined; + block_types[0] = lb_v.type_(); + block_locs[0] = self.loc(); + var inits_inner: [N]*const mlir.Value = undefined; + inline for (fields, 0..) |f, i| { + if (f.type != Value) @compileError("openFor: every init must be a Value"); + const v: Value = @field(inits, f.name); + block_types[i + 1] = v.type_(); + block_locs[i + 1] = self.loc(); + inits_inner[i] = v.inner; + } + + const body = mlir.Block.init(&block_types, &block_locs); + self.pushBlock(body); + + var carried: [N]Value = undefined; + for (0..N) |i| carried[i] = .{ .inner = body.argument(i + 1), .kernel = self }; + + return .{ + .kernel = self, + .body = body, + .lb_inner = lb_v.inner, + .ub_inner = ub_v.inner, + .step_inner = step_v.inner, + .inits_inner = inits_inner, + .iv = .{ .inner = body.argument(0), .kernel = self }, + .carried = carried, + }; + } + + pub fn constLike(self: *Builder, value: anytype, like: Value) Value { + if (like.type_().isA(mlir.IndexType) != null) { + const i: i64 = switch (@typeInfo(@TypeOf(value))) { + .float, .comptime_float => @intFromFloat(value), + else => @intCast(value), + }; + return self.constIndex(i); + } + return self.constant(like.scalarDType(), value); + } + + pub fn openIf(self: *Builder, cond: Value) IfOnlyScope { + const then_block = mlir.Block.init(&.{}, &.{}); + self.pushBlock(then_block); + return .{ .kernel = self, .cond_inner = cond.inner, .then_block = then_block }; + } + + pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(tupleArity(@TypeOf(result_types), "openIfElse: result_types")) { + const N = comptime tupleArity(@TypeOf(result_types), "openIfElse: result_types"); + const fields = @typeInfo(@TypeOf(result_types)).@"struct".fields; + var types: [N]*const mlir.Type = undefined; + inline for (fields, 0..) |f, i| { + if (f.type != *const mlir.Type) @compileError("openIfElse: every result_type must be *const mlir.Type"); + types[i] = @field(result_types, f.name); + } + const then_block = mlir.Block.init(&.{}, &.{}); + const else_block = mlir.Block.init(&.{}, &.{}); + self.pushBlock(then_block); + return .{ .kernel = self, .cond_inner = cond.inner, .then_block = then_block, .else_block = else_block, .result_types = types }; + } + + // ---------------------------------------------------------------- finalization + + /// Append `gpu.return`, verify, and print the module (no debug info: XLA + /// scrubs locations). Owned by the builder's allocator. + pub fn finish(self: *Builder) FinishError![:0]const u8 { + _ = gpu.return_(self.ctx, &.{}, self.loc()).appendTo(self.currentBlock()); + + if (!self.module.operation().verify()) { + std.log.err("fly: module failed to verify:\n{f}", .{self.module.operation().fmt(.{ .debug_info = false })}); + return error.InvalidMlir; + } + + var al: std.Io.Writer.Allocating = .init(self.allocator); + defer al.deinit(); + try al.writer.print("{f}", .{self.module.operation().fmt(.{ .debug_info = false })}); + return try self.allocator.dupeZ(u8, al.written()); + } +}; + +// ============================================================================= +// Tests +// ============================================================================= + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + fly.insertDialects(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +/// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the +/// ragged edge. +pub fn emitVectorAdd(b: *Builder, thr_layout: Layout, val_layout: Layout, a_in: Value, b_in: Value, c_out: Value) void { + const shape = a_in.shapeStatic(); + const m = shape.at(0).leaf.s; + const n = shape.at(1).leaf.s; + + const atom = b.copyAtom(.{ .universal = 128 }, .f32); + const tc = b.tiledCopyTV(atom, thr_layout, val_layout); + + const tid = b.threadId(.x); + const bx = b.blockId(.x); + const by = b.blockId(.y); + + const idC = b.identity(.{ m, n }); + const tile_mn = tc.tileMN(); + + const gA = a_in.flatDivide(tile_mn).slice(.{ null, null, bx, by }); + const gB = b_in.flatDivide(tile_mn).slice(.{ null, null, bx, by }); + const gC = c_out.flatDivide(tile_mn).slice(.{ null, null, bx, by }); + const cC = idC.flatDivide(tile_mn).slice(.{ null, null, bx, by }); + + const thr = tc.getSlice(tid); + const tgA = thr.partitionS(gA); + const tgB = thr.partitionS(gB); + const tgC = thr.partitionD(gC); + const tcC = thr.partitionS(cC).slice(.{ .{ 0, null }, null, null }); + + const rA = tgA.makeFragmentLike(null); + const rB = tgB.makeFragmentLike(null); + const rC = tgC.makeFragmentLike(null); + const pC = tcC.makeFragmentLike(.i1); + + for (0..@intCast(pC.sizeStatic())) |i| { + pC.set(i, tcC.at(i).elemLess(.{ m, n })); + } + + b.copy(atom, tgA, rA, .{ .pred = pC }); + b.copy(atom, tgB, rB, .{ .pred = pC }); + rC.store(rA.load().add(rB.load())); + b.copy(atom, rC, tgC, .{ .pred = pC }); +} + +test "vectorAdd builds, verifies and re-parses" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "vector_add"); + defer b.deinit(); + const a = try b.declareArgs(.{ + .a = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 100, 1000 } } }, + .b = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 100, 1000 } } }, + .c = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 100, 1000 } } }, + }); + emitVectorAdd(&b, ordered(.{ 8, 16 }, .{ 1, 0 }), ordered(.{ 1, 4 }, .{ 0, 1 }), a.a, a.b, a.c); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + + const expected = [_][]const u8{ + "module attributes {gpu.container_module}", + "gpu.module @zml_fly_kernels", + "gpu.func @vector_add(%arg0: !fly.ptr, %arg1: !fly.ptr, %arg2: !fly.ptr) kernel", + "fly.static : !fly.layout<(100,1000):(1000,1)>", + "fly.make_copy_atom {valBits = 32 : i32} : !fly.copy_atom, 32>", + "fly.raked_product", + "fly.static : !fly.tile<[8|64]>", + "fly.make_tiled_copy", + "gpu.thread_id x", + "fly.tiled_copy.partition_src", + "fly.tiled_copy.partition_dst", + "fly.make_fragment_like", + "fly.elem_less", + "fly.copy(", + "fly.memref.load_vec", + "arith.addf", + "fly.memref.store_vec", + "gpu.return", + }; + for (expected) |needle| { + if (std.mem.indexOf(u8, ir, needle) == null) { + std.debug.print("missing `{s}` in:\n{s}\n", .{ needle, ir }); + return error.TestUnexpectedResult; + } + } + + const reparsed = try mlir.Module.parse(ctx, ir); + defer reparsed.deinit(); + try std.testing.expect(reparsed.operation().verify()); +} + +/// The matrix atom a device family provides. CDNA has MFMA over f32; RDNA has +/// WMMA, whose verifier requires M=N=K=16 and rejects f32 operands, so the +/// operand type and the K tile change with the family, not just the mnemonic. +pub const MmaFlavor = enum { + cdna3_mfma, + gfx11_wmma, + gfx120x_wmma, + + /// MFMA takes f32 operands at 16x16x4; both WMMAs take f16 at 16x16x16. + pub fn atomType(self: MmaFlavor, ctx: *mlir.Context) *const mlir.Type { + const operand = self.operandDType().toMlir(ctx); + const acc: *const mlir.Type = .float(ctx, .f32); + return switch (self) { + .cdna3_mfma => (fly.rocdl.MmaOpCDNA3MFMAType.get(ctx, .{ + .m = 16, + .n = 16, + .k = 4, + .elemTyA = operand, + .elemTyB = operand, + .elemTyAcc = acc, + }) catch unreachable).type_(), + inline .gfx11_wmma, .gfx120x_wmma => |f| blk: { + const T = if (f == .gfx11_wmma) fly.rocdl.MmaOpGFX11WMMAType else fly.rocdl.MmaOpGFX120XWMMAType; + break :blk (T.get(ctx, .{ + .m = 16, + .n = 16, + .k = 16, + .elemTyA = operand, + .elemTyB = operand, + .elemTyAcc = acc, + }) catch unreachable).type_(); + }, + }; + } + + /// A and B operand type. The accumulator is f32 either way. + pub fn operandDType(self: MmaFlavor) DType { + return switch (self) { + .cdna3_mfma => .f32, + .gfx11_wmma, .gfx120x_wmma => .f16, + }; + } + + /// Threads one atom occupies: MFMA runs on a wave64, WMMA on a wave32. + /// Checked against the dialect's own `mma_atom.thr_layout` below. + pub fn atomThreads(self: MmaFlavor) i32 { + return switch (self) { + .cdna3_mfma => 64, + .gfx11_wmma, .gfx120x_wmma => 32, + }; + } + + /// `emitTiledMma` lays the atom out (2,2,1), so a block is four of them. + pub fn blockThreads(self: MmaFlavor) i32 { + return 4 * self.atomThreads(); + } + + /// The block tile's K, a multiple of the atom's K. + pub fn blockK(self: MmaFlavor) i64 { + return switch (self) { + .cdna3_mfma => 8, + .gfx11_wmma, .gfx120x_wmma => 16, + }; + } + + /// Buffer copies are CDNA-only: FlyROCDL declares no RDNA copy atom, so + /// RDNA moves through the core universal copy over plain tensors. + fn hasBufferCopy(self: MmaFlavor) bool { + return self == .cdna3_mfma; + } +}; + +/// A two-mode tile from extents known only at emit time. +fn tile2(b: *Builder, x: i64, y: i64) Tile { + const modes = b.alloc(Tile, 2); + modes[0] = .{ .leaf = .{ .s = x } }; + modes[1] = .{ .leaf = .{ .s = y } }; + return .{ .modes = modes }; +} + +/// C = A @ B^T in one block of 256 threads, tiled (2,2,1) over the family's +/// matrix atom. +pub fn emitTiledMma(b: *Builder, flavor: MmaFlavor, a_in: Value, b_in: Value, c_out: Value) void { + const block_m = 64; + const block_n = 64; + const block_k = flavor.blockK(); + + const tid = b.threadId(.x); + const bid = b.blockId(.x); + + const A = if (flavor.hasBufferCopy()) b.bufferTensor(a_in) else a_in; + const B = if (flavor.hasBufferCopy()) b.bufferTensor(b_in) else b_in; + const C = if (flavor.hasBufferCopy()) b.bufferTensor(c_out) else c_out; + + const bA = A.zippedDivide(tile2(b, block_m, block_k)).slice(.{ null, bid }); + const bB = B.zippedDivide(tile2(b, block_n, block_k)).slice(.{ null, bid }); + const bC = C.zippedDivide(tile2(b, block_m, block_n)).slice(.{ null, bid }); + + const mma_atom = b.mmaAtom(flavor.atomType(b.ctx)); + const tiled_mma = b.tiledMma(mma_atom, L(.{ 2, 2, 1 }, .{ 1, 2, 0 }), null); + + // A and B move at the operand type, C at the f32 accumulator type. One + // element per instruction: a WMMA fragment's values are strided in memory, + // so a wider copy would move the wrong neighbours. + const copy_ab: Builder.CopyOp = if (flavor.hasBufferCopy()) .{ .buffer_copy = 32 } else .{ .universal = @intCast(flavor.operandDType().bitWidth()) }; + const copy_c: Builder.CopyOp = if (flavor.hasBufferCopy()) .{ .buffer_copy = 32 } else .{ .universal = @intCast(c_out.elemDType().bitWidth()) }; + const atom_ab = b.copyAtom(copy_ab, flavor.operandDType()); + const atom_c = b.copyAtom(copy_c, c_out.elemDType()); + + const tiled_copy_a = b.tiledCopyA(atom_ab, tiled_mma); + const tiled_copy_b = b.tiledCopyB(atom_ab, tiled_mma); + const tiled_copy_c = b.tiledCopyC(atom_c, tiled_mma); + + const thr_copy_a = tiled_copy_a.getSlice(tid); + const thr_copy_b = tiled_copy_b.getSlice(tid); + const thr_copy_c = tiled_copy_c.getSlice(tid); + + const copy_src_a = thr_copy_a.partitionS(bA); + const copy_src_b = thr_copy_b.partitionS(bB); + const copy_dst_c = thr_copy_c.partitionS(bC); + + const frag_a = tiled_mma.makeFragmentA(bA); + const frag_b = tiled_mma.makeFragmentB(bB); + const frag_c = tiled_mma.makeFragmentC(bC); + + const copy_frag_a = thr_copy_a.retile(frag_a); + const copy_frag_b = thr_copy_b.retile(frag_b); + const copy_frag_c = thr_copy_c.retile(frag_c); + + b.copy(atom_ab, copy_src_a, copy_frag_a, .{}); + b.copy(atom_ab, copy_src_b, copy_frag_b, .{}); + + frag_c.fill(0); + b.gemm(mma_atom, frag_c, frag_a, frag_b, frag_c); + + b.copy(atom_c, copy_frag_c, copy_dst_c, .{}); +} + +test "tiledMma builds, verifies and re-parses" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "tiled_mma"); + defer b.deinit(); + const a = try b.declareArgs(.{ + .a = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 64, 8 } } }, + .b = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 64, 8 } } }, + .c = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 64, 64 } } }, + }); + emitTiledMma(&b, .cdna3_mfma, a.a, a.b, a.c); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + + const expected = [_][]const u8{ + "!fly.ptr", + "fly.make_mma_atom : !fly.mma_atom f32>>", + "fly.make_tiled_mma", + "fly.static : !fly.int_tuple<(32,32,4)>", + "fly.mma.make_fragment", + "fly.tiled_copy.retile", + "vector.broadcast", + "fly.gemm(", + }; + for (expected) |needle| { + if (std.mem.indexOf(u8, ir, needle) == null) { + std.debug.print("missing `{s}` in:\n{s}\n", .{ needle, ir }); + return error.TestUnexpectedResult; + } + } + const reparsed = try mlir.Module.parse(ctx, ir); + defer reparsed.deinit(); + try std.testing.expect(reparsed.operation().verify()); +} + +test "MmaFlavor.atomThreads matches the dialect" { + const ctx = try testContext(); + defer ctx.deinit(); + inline for (std.meta.fields(MmaFlavor)) |field| { + const flavor: MmaFlavor = @enumFromInt(field.value); + const atom = try fly.types.MmaAtomType.get(ctx, .{ .mmaOp = flavor.atomType(ctx) }); + // `mma_atom.thr_layout` is the wavefront an atom occupies; the launch + // geometry is derived from it, so a wrong table silently corrupts C. + const threads = switch (atom.getThrLayout().getShape().getLeaf()) { + .static => |v| v, + else => return error.TestUnexpectedResult, + }; + try std.testing.expectEqual(@as(i64, flavor.atomThreads()), threads); + } +} + +test "tiledMma builds for the RDNA WMMA atom" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "tiled_mma_wmma"); + defer b.deinit(); + const k = MmaFlavor.gfx11_wmma.blockK(); + const a = try b.declareArgs(.{ + .a = .{ .tensor = .{ .dtype = .f16, .dims = &.{ 64, k } } }, + .b = .{ .tensor = .{ .dtype = .f16, .dims = &.{ 64, k } } }, + .c = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 64, 64 } } }, + }); + emitTiledMma(&b, .gfx11_wmma, a.a, a.b, a.c); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + + // No buffer descriptor: FlyROCDL declares no RDNA copy atom, so this path + // moves through the core universal copy over plain tensors. + if (std.mem.indexOf(u8, ir, "buffer_desc") != null) { + std.debug.print("unexpected buffer descriptor on the WMMA path:\n{s}\n", .{ir}); + return error.TestUnexpectedResult; + } + for ([_][]const u8{ + "!fly_rocdl.gfx11.wmma<16x16x16, (f16, f16) -> f32", + "!fly.copy_atom, 16>", + "fly.gemm(", + }) |needle| { + if (std.mem.indexOf(u8, ir, needle) == null) { + std.debug.print("missing `{s}` in:\n{s}\n", .{ needle, ir }); + return error.TestUnexpectedResult; + } + } + + const reparsed = try mlir.Module.parse(ctx, ir); + defer reparsed.deinit(); + try std.testing.expect(reparsed.operation().verify()); +} + +test "scalar conversions and constants" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "conv"); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .tensor = .{ .dtype = .f16, .dims = &.{16} } }, + .p = .{ .tensor = .{ .dtype = .i1, .dims = &.{16} } }, + }); + + // A predicate is unsigned: `true` must convert to 1, never -1. + const pred = b.constant(.i32, 3).cmp(.lt, 7); + _ = pred.to(.f32); + _ = pred.to(.i32); + // Equal-width float pairs are neither ext nor trunc. + _ = b.constant(.f16, 1.0).to(.bf16); + // A float constant outside i64 range must not go through @intFromFloat. + _ = b.constant(.f32, -std.math.inf(f32)); + _ = b.constant(.f32, std.math.floatMax(f32)); + // A `.bool` argument keeps its storage element type end to end. + try std.testing.expectEqual(DType.i8, a.p.elemDType()); + try std.testing.expectEqual(DType.f16, a.x.elemDType()); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + for ([_][]const u8{ + "arith.uitofp", + "arith.extui", + "arith.convertf", + "arith.constant 0xFF800000 : f32", + "!fly.memref", + }) |needle| { + if (std.mem.indexOf(u8, ir, needle) == null) { + std.debug.print("missing `{s}` in:\n{s}\n", .{ needle, ir }); + return error.TestUnexpectedResult; + } + } + if (std.mem.indexOf(u8, ir, "arith.sitofp") != null or std.mem.indexOf(u8, ir, "arith.extsi") != null) { + std.debug.print("i1 was sign-extended:\n{s}\n", .{ir}); + return error.TestUnexpectedResult; + } +} + +test "static queries read inferred types" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "q"); + defer b.deinit(); + const a = try b.declareArgs(.{ .x = .{ .tensor = .{ .dtype = .bf16, .dims = &.{ 64, 32 } } } }); + try std.testing.expectEqual(@as(i64, 64 * 32), a.x.sizeStatic()); + try std.testing.expectEqual(DType.bf16, a.x.elemDType()); + try std.testing.expectEqual(Value.Kind.memref, a.x.kind()); + const tiled = a.x.flatDivide(tile(.{ 16, 8 })); + try std.testing.expectEqual(@as(i64, 64 * 32), tiled.sizeStatic()); + try std.testing.expectEqual(@as(usize, 4), tiled.shapeStatic().rank()); +} diff --git a/kernels/fly/dtype.zig b/kernels/fly/dtype.zig new file mode 100644 index 00000000..6962a898 --- /dev/null +++ b/kernels/fly/dtype.zig @@ -0,0 +1,86 @@ +const std = @import("std"); +const mlir = @import("mlir"); + +pub const DType = enum { + i1, + i8, + i16, + i32, + i64, + f16, + bf16, + f32, + f64, + f8e4m3fn, + f8e4m3fnuz, + f8e5m2, + f8e5m2fnuz, + + pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { + return switch (self) { + .i1 => .int(ctx, .i1), + .i8 => .int(ctx, .i8), + .i16 => .int(ctx, .i16), + .i32 => .int(ctx, .i32), + .i64 => .int(ctx, .i64), + .f16 => .float(ctx, .f16), + .bf16 => .float(ctx, .bf16), + .f32 => .float(ctx, .f32), + .f64 => .float(ctx, .f64), + .f8e4m3fn => .float(ctx, .f8e4m3fn), + .f8e4m3fnuz => .float(ctx, .f8e4m3fnuz), + .f8e5m2 => .float(ctx, .f8e5m2), + .f8e5m2fnuz => .float(ctx, .f8e5m2fnuz), + }; + } + + pub fn bitWidth(self: DType) u32 { + return switch (self) { + .i1 => 1, + .i8, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz => 8, + .i16, .f16, .bf16 => 16, + .i32, .f32 => 32, + .i64, .f64 => 64, + }; + } + + pub fn isFloat(self: DType) bool { + return switch (self) { + .f16, .bf16, .f32, .f64, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz => true, + else => false, + }; + } + + /// The element type a kernel argument is stored with on device: + /// predicates live in one byte. A copy atom over `!fly.ptr` would + /// claim `bits / 1` elements while moving `bits / 8` bytes, so an + /// argument keeps its storage type and the body converts explicitly. + pub fn storageElem(self: DType) DType { + return switch (self) { + .i1 => .i8, + else => self, + }; + } +}; + +/// Inverse of `toMlir`. +pub fn fromMlir(ctx: *mlir.Context, ty: *const mlir.Type) ?DType { + inline for (std.meta.fields(DType)) |f| { + const dt: DType = @enumFromInt(f.value); + if (ty.eql(dt.toMlir(ctx))) return dt; + } + return null; +} + +test "dtype types round-trip" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + inline for (std.meta.fields(DType)) |f| { + const dt: DType = @enumFromInt(f.value); + try std.testing.expectEqual(dt, fromMlir(ctx, dt.toMlir(ctx)).?); + } + try std.testing.expectEqual(DType.i8, DType.i1.storageElem()); + try std.testing.expectEqual(DType.f8e4m3fnuz, DType.f8e4m3fnuz.storageElem()); +} diff --git a/kernels/fly/layout.zig b/kernels/fly/layout.zig new file mode 100644 index 00000000..85495ed4 --- /dev/null +++ b/kernels/fly/layout.zig @@ -0,0 +1,544 @@ +const std = @import("std"); + +const fly = @import("mlir/dialects/fly"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +const attributes = fly.attributes; + +/// Enough for any tuple or tile a kernel writes by hand. +const max_modes = 32; + +/// A dynamic integer leaf: `?`, `?{i64}`, `?{div=8}`, `?{i64 div=8}`. +pub const Dyn = struct { + width: u8 = 32, + div: u32 = 1, +}; + +/// Static, dynamic, or the `*` wildcard. +pub const Leaf = union(enum) { + s: i64, + d: Dyn, + none, + + pub fn print(self: Leaf, w: *std.Io.Writer) std.Io.Writer.Error!void { + switch (self) { + .s => |v| try w.print("{d}", .{v}), + .none => try w.writeAll("*"), + .d => |dy| { + try w.writeAll("?"); + if (dy.width != 32 or dy.div != 1) { + try w.writeAll("{"); + if (dy.width != 32) try w.print("i{d}", .{dy.width}); + if (dy.div != 1) { + if (dy.width != 32) try w.writeAll(" "); + try w.print("div={d}", .{dy.div}); + } + try w.writeAll("}"); + } + }, + } + } +}; + +fn leafAttr(leaf: Leaf, ctx: *mlir.Context) mlir.Error!*const attributes.IntAttr { + return switch (leaf) { + .s => |v| .getStatic(ctx, @intCast(v)), + .d => |dy| .getDynamic(ctx, dy.width, @intCast(dy.div)), + .none => .getNone(ctx), + }; +} + +/// A scaled basis element: `vE0`, `2E1E2`. +pub const Basis = struct { + value: Leaf, + modes: []const i32, +}; + +pub const IntTuple = union(enum) { + leaf: Leaf, + basis: Basis, + tup: []const IntTuple, + + pub const dyn: IntTuple = .{ .leaf = .{ .d = .{} } }; + pub const star: IntTuple = .{ .leaf = .none }; + + pub fn static(v: i64) IntTuple { + return .{ .leaf = .{ .s = v } }; + } + + pub fn dynamic(width: u8, div: u32) IntTuple { + return .{ .leaf = .{ .d = .{ .width = width, .div = div } } }; + } + + /// 1 for a leaf. + pub fn rank(self: IntTuple) usize { + return switch (self) { + .tup => |t| t.len, + else => 1, + }; + } + + pub fn isLeaf(self: IntTuple) bool { + return self != .tup; + } + + pub fn at(self: IntTuple, i: usize) IntTuple { + return switch (self) { + .tup => |t| t[i], + else => if (i == 0) self else @panic("fly.IntTuple.at: index out of range on a leaf"), + }; + } + + /// Null if any leaf is dynamic or `*`. + pub fn product(self: IntTuple) ?i64 { + switch (self) { + .leaf => |l| return switch (l) { + .s => |v| v, + else => null, + }, + .basis => return null, + .tup => |t| { + var p: i64 = 1; + for (t) |e| p *= e.product() orelse return null; + return p; + }, + } + } + + pub fn isStatic(self: IntTuple) bool { + return switch (self) { + .leaf => |l| l == .s, + .basis => |b| b.value == .s, + .tup => |t| for (t) |e| { + if (!e.isStatic()) break false; + } else true, + }; + } + + /// At any depth. + pub fn leafCount(self: IntTuple) usize { + return switch (self) { + .tup => |t| blk: { + var n: usize = 0; + for (t) |e| n += e.leafCount(); + break :blk n; + }, + else => 1, + }; + } + + pub fn print(self: IntTuple, w: *std.Io.Writer) std.Io.Writer.Error!void { + switch (self) { + .leaf => |l| try l.print(w), + .basis => |b| { + try b.value.print(w); + for (b.modes) |m| try w.print("E{d}", .{m}); + }, + .tup => |t| { + try w.writeAll("("); + for (t, 0..) |e, i| { + if (i > 0) try w.writeAll(","); + try e.print(w); + } + try w.writeAll(")"); + }, + } + } + + pub fn format(self: IntTuple, w: *std.Io.Writer) std.Io.Writer.Error!void { + try self.print(w); + } + + pub fn toAttr(self: IntTuple, ctx: *mlir.Context) mlir.Error!*const attributes.IntTupleAttr { + return switch (self) { + .leaf => |l| .get(ctx, .{ .value = (try leafAttr(l, ctx)).attribute() }), + .basis => |b| .getBasis(ctx, try leafAttr(b.value, ctx), b.modes), + .tup => |elems| blk: { + var buf: stdx.BoundedArray(*const attributes.IntTupleAttr, max_modes) = .empty; + for (elems) |e| buf.appendAssumeCapacity(try e.toAttr(ctx)); + break :blk .getTuple(ctx, buf.constSlice()); + }, + }; + } + + pub fn eql(a: IntTuple, b: IntTuple) bool { + if (std.meta.activeTag(a) != std.meta.activeTag(b)) return false; + return switch (a) { + .leaf => |l| std.meta.eql(l, b.leaf), + .basis => |x| std.meta.eql(x.value, b.basis.value) and std.mem.eql(i32, x.modes, b.basis.modes), + .tup => |t| t.len == b.tup.len and for (t, b.tup) |x, y| { + if (!x.eql(y)) break false; + } else true, + }; + } +}; + +/// Integers become static leaves, tuples nest, `null` is `*`, an `IntTuple` +/// passes through: `it(.{ .{ 2, 4 }, 8 })` is `((2,4),8)`. +pub fn it(comptime v: anytype) IntTuple { + return comptime itImpl(v); +} + +fn itImpl(comptime v: anytype) IntTuple { + const T = @TypeOf(v); + if (T == IntTuple) return v; + if (T == Leaf) return .{ .leaf = v }; + if (T == @TypeOf(null)) return IntTuple.star; + switch (@typeInfo(T)) { + .comptime_int, .int => return IntTuple.static(v), + .@"struct" => |s| { + if (!s.is_tuple) @compileError("fly.it: expected an integer, null, IntTuple or tuple, got " ++ @typeName(T)); + var elems: [s.fields.len]IntTuple = undefined; + for (s.fields, 0..) |f, i| elems[i] = itImpl(@field(v, f.name)); + const frozen = elems; + return .{ .tup = &frozen }; + }, + else => @compileError("fly.it: expected an integer, null, IntTuple or tuple, got " ++ @typeName(T)), + } +} + +/// `!fly.layout`. +pub const Layout = struct { + shape: IntTuple, + stride: IntTuple, + + pub fn rank(self: Layout) usize { + return self.shape.rank(); + } + + /// Null if dynamic. + pub fn size(self: Layout) ?i64 { + return self.shape.product(); + } + + pub fn print(self: Layout, w: *std.Io.Writer) std.Io.Writer.Error!void { + try self.shape.print(w); + try w.writeAll(":"); + try self.stride.print(w); + } + + pub fn format(self: Layout, w: *std.Io.Writer) std.Io.Writer.Error!void { + try self.print(w); + } + + pub fn toAttr(self: Layout, ctx: *mlir.Context) mlir.Error!*const attributes.LayoutAttr { + return .get(ctx, .{ .shape = try self.shape.toAttr(ctx), .stride = try self.stride.toAttr(ctx) }); + } + + pub fn eql(a: Layout, b: Layout) bool { + return a.shape.eql(b.shape) and a.stride.eql(b.stride); + } +}; + +/// Two congruent tuples: `L(.{ 4, 8 }, .{ 1, 4 })` is `(4,8):(1,4)`. +pub fn L(comptime shape: anytype, comptime stride: anytype) Layout { + return comptime blk: { + const s = it(shape); + const d = it(stride); + checkCongruent(s, d); + break :blk .{ .shape = s, .stride = d }; + }; +} + +fn checkCongruent(comptime a: IntTuple, comptime b: IntTuple) void { + if (a.isLeaf() != b.isLeaf() or a.rank() != b.rank()) { + @compileError("fly.L: shape and stride are not congruent"); + } + if (a == .tup) { + for (a.tup, b.tup) |x, y| checkCongruent(x, y); + } +} + +/// Last mode contiguous: `rowMajor(.{ 4, 8 })` is `(4,8):(8,1)`. +pub fn rowMajor(comptime shape: anytype) Layout { + return comptime blk: { + const s = it(shape); + const n = s.leafCount(); + var order: [n]i32 = undefined; + for (0..n) |i| order[i] = @intCast(n - 1 - i); + break :blk orderedFlat(s, &order); + }; +} + +/// First mode contiguous. +pub fn colMajor(comptime shape: anytype) Layout { + return comptime blk: { + const s = it(shape); + const n = s.leafCount(); + var order: [n]i32 = undefined; + for (0..n) |i| order[i] = @intCast(i); + break :blk orderedFlat(s, &order); + }; +} + +/// `make_ordered_layout` folded at comptime: the smallest order value gets +/// stride 1, the next the product of the previous extents, and so on. +/// `ordered(.{ 8, 16 }, .{ 1, 0 })` is `(8,16):(16,1)`. +pub fn ordered(comptime shape: anytype, comptime order: anytype) Layout { + return comptime blk: { + const s = it(shape); + const o = it(order); + checkCongruent(s, o); + const n = s.leafCount(); + var flat_order: [n]i32 = undefined; + var idx: usize = 0; + flattenOrder(o, &flat_order, &idx); + break :blk orderedFlat(s, &flat_order); + }; +} + +fn flattenOrder(comptime o: IntTuple, out: []i32, idx: *usize) void { + switch (o) { + .leaf => |l| { + out[idx.*] = @intCast(l.s); + idx.* += 1; + }, + .tup => |t| for (t) |e| flattenOrder(e, out, idx), + .basis => @compileError("fly.ordered: basis elements are not allowed in an order"), + } +} + +fn orderedFlat(comptime shape: IntTuple, comptime order: []const i32) Layout { + comptime { + const n = shape.leafCount(); + var extents: [n]i64 = undefined; + var idx: usize = 0; + flattenStatic(shape, &extents, &idx); + var strides: [n]i64 = undefined; + var assigned: [n]bool = @splat(false); + var acc: i64 = 1; + for (0..n) |_| { + var best: ?usize = null; + for (0..n) |i| { + if (assigned[i]) continue; + if (best == null or order[i] < order[best.?]) best = i; + } + strides[best.?] = acc; + acc *= extents[best.?]; + assigned[best.?] = true; + } + var pos: usize = 0; + return .{ .shape = shape, .stride = unflatten(shape, &strides, &pos) }; + } +} + +fn flattenStatic(comptime t: IntTuple, out: []i64, idx: *usize) void { + switch (t) { + .leaf => |l| switch (l) { + .s => |v| { + out[idx.*] = v; + idx.* += 1; + }, + else => @compileError("fly: a folded layout needs a fully static shape"), + }, + .basis => @compileError("fly: a folded layout needs a plain integer shape"), + .tup => |elems| for (elems) |e| flattenStatic(e, out, idx), + } +} + +fn unflatten(comptime like: IntTuple, values: []const i64, pos: *usize) IntTuple { + switch (like) { + .tup => |elems| { + var out: [elems.len]IntTuple = undefined; + for (elems, 0..) |e, i| out[i] = unflatten(e, values, pos); + const frozen = out; + return .{ .tup = &frozen }; + }, + else => { + const v = values[pos.*]; + pos.* += 1; + return IntTuple.static(v); + }, + } +} + +/// A leaf (`8` or `*`), a layout, or a nested tile. +pub const Tile = union(enum) { + leaf: Leaf, + layout: Layout, + modes: []const Tile, + + pub fn print(self: Tile, w: *std.Io.Writer) std.Io.Writer.Error!void { + switch (self) { + .leaf => |l| try l.print(w), + .layout => |l| try l.print(w), + .modes => |m| { + try w.writeAll("["); + for (m, 0..) |e, i| { + if (i > 0) try w.writeAll("|"); + try e.print(w); + } + try w.writeAll("]"); + }, + } + } + + pub fn format(self: Tile, w: *std.Io.Writer) std.Io.Writer.Error!void { + try self.print(w); + } + + /// One mode: an int, a layout, or a nested tile. + fn modeAttr(self: Tile, ctx: *mlir.Context) mlir.Error!*const mlir.Attribute { + return switch (self) { + .leaf => |l| (try leafAttr(l, ctx)).attribute(), + .layout => |l| (try l.toAttr(ctx)).attribute(), + .modes => (try self.toAttr(ctx)).attribute(), + }; + } + + pub fn toAttr(self: Tile, ctx: *mlir.Context) mlir.Error!*const attributes.TileAttr { + const modes = switch (self) { + .modes => |m| m, + else => return .get(ctx, .{ .value = try self.modeAttr(ctx) }), + }; + var buf: stdx.BoundedArray(*const mlir.Attribute, max_modes) = .empty; + for (modes) |m| buf.appendAssumeCapacity(try m.modeAttr(ctx)); + return .getModes(ctx, buf.constSlice()); + } +}; + +/// `tile(.{ 128, 64 })` is `[128|64]`; `tile(.{ null, 8 })` is `[*|8]`. +pub fn tile(comptime v: anytype) Tile { + return comptime tileImpl(v); +} + +fn tileImpl(comptime v: anytype) Tile { + const T = @TypeOf(v); + if (T == Tile) return v; + if (T == Layout) return .{ .layout = v }; + if (T == Leaf) return .{ .leaf = v }; + if (T == @TypeOf(null)) return .{ .leaf = .none }; + switch (@typeInfo(T)) { + .comptime_int, .int => return .{ .leaf = .{ .s = v } }, + .@"struct" => |s| { + if (!s.is_tuple) @compileError("fly.tile: expected an integer, null, Layout, Tile or tuple, got " ++ @typeName(T)); + var elems: [s.fields.len]Tile = undefined; + for (s.fields, 0..) |f, i| elems[i] = tileImpl(@field(v, f.name)); + const frozen = elems; + return .{ .modes = &frozen }; + }, + else => @compileError("fly.tile: expected an integer, null, Layout, Tile or tuple, got " ++ @typeName(T)), + } +} + +/// `S`. +pub const Swizzle = struct { + mask: i32, + base: i32, + shift: i32, + + pub const trivial: Swizzle = .{ .mask = 0, .base = 0, .shift = 0 }; + + pub fn isTrivial(self: Swizzle) bool { + return self.mask == 0 and self.base == 0 and self.shift == 0; + } + + pub fn print(self: Swizzle, w: *std.Io.Writer) std.Io.Writer.Error!void { + try w.print("S<{d},{d},{d}>", .{ self.mask, self.base, self.shift }); + } + + pub fn format(self: Swizzle, w: *std.Io.Writer) std.Io.Writer.Error!void { + try self.print(w); + } + + pub fn toAttr(self: Swizzle, ctx: *mlir.Context) mlir.Error!*const attributes.SwizzleAttr { + return .get(ctx, .{ .mask = self.mask, .base = self.base, .shift = self.shift }); + } +}; + +// ----------------------------------------------------------------------------- +// Tests +// ----------------------------------------------------------------------------- + +fn expectPrints(expected: []const u8, v: anytype) !void { + var buf: [1024]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{v}); + try std.testing.expectEqualStrings(expected, w.buffered()); +} + +test "int tuple literals print in fly syntax" { + try expectPrints("42", it(42)); + try expectPrints("((2,4),8)", it(.{ .{ 2, 4 }, 8 })); + try expectPrints("(?,8)", it(.{ IntTuple.dyn, 8 })); + try expectPrints("?{i64 div=8}", IntTuple.dynamic(64, 8)); + try expectPrints("?{div=4}", IntTuple.dynamic(32, 4)); + try expectPrints("?{i64}", IntTuple.dynamic(64, 1)); + try expectPrints("(*,8)", it(.{ null, 8 })); + try expectPrints("(1,(2,3))", it(.{ 1, .{ 2, 3 } })); +} + +test "layout literals" { + try expectPrints("(4,8):(1,4)", L(.{ 4, 8 }, .{ 1, 4 })); + try expectPrints("((2,4),8):((1,2),8)", L(.{ .{ 2, 4 }, 8 }, .{ .{ 1, 2 }, 8 })); + try expectPrints("(4,8):(8,1)", rowMajor(.{ 4, 8 })); + try expectPrints("(4,8):(1,4)", colMajor(.{ 4, 8 })); + try expectPrints("((2,4),8):((32,8),1)", rowMajor(.{ .{ 2, 4 }, 8 })); + try expectPrints("(8,16):(16,1)", ordered(.{ 8, 16 }, .{ 1, 0 })); + try expectPrints("(1,4):(1,1)", ordered(.{ 1, 4 }, .{ 0, 1 })); + try expectPrints("(4,2,8):(2,1,8)", ordered(.{ 4, 2, 8 }, .{ 1, 0, 2 })); +} + +test "tiles and swizzles" { + try expectPrints("[128|64]", tile(.{ 128, 64 })); + try expectPrints("[*|8]", tile(.{ null, 8 })); + try expectPrints("[(4,8):(1,4)|16]", tile(.{ L(.{ 4, 8 }, .{ 1, 4 }), 16 })); + try expectPrints("[[2|3]|4]", tile(.{ .{ 2, 3 }, 4 })); + try expectPrints("S<3,3,3>", Swizzle{ .mask = 3, .base = 3, .shift = 3 }); +} + +test "static queries" { + try std.testing.expectEqual(@as(?i64, 64), it(.{ .{ 2, 4 }, 8 }).product()); + try std.testing.expectEqual(@as(?i64, null), it(.{ IntTuple.dyn, 8 }).product()); + try std.testing.expectEqual(@as(usize, 2), it(.{ .{ 2, 4 }, 8 }).rank()); + try std.testing.expectEqual(@as(usize, 3), it(.{ .{ 2, 4 }, 8 }).leafCount()); + try std.testing.expect(it(.{ 2, 4 }).eql(it(.{ 2, 4 }))); + try std.testing.expect(!it(.{ 2, 4 }).eql(it(.{ 4, 2 }))); +} + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + fly.insertDialects(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +/// The type built from the host model must be the type the printed literal +/// parses to: the two paths may never drift. +fn expectBuilt(ctx: *mlir.Context, comptime T: type, comptime fmt: []const u8, value: anytype) !void { + var buf: [512]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print(fmt, .{value}); + const parsed = try mlir.Type.parse(ctx, w.buffered()); + const built = try T.get(ctx, .{ .attr = try value.toAttr(ctx) }); + if (!parsed.eql(built.type_())) { + std.debug.print("parsed `{f}` but built `{f}`\n", .{ parsed, built }); + return error.TestUnexpectedResult; + } +} + +test "the host model builds the types its literals print" { + const ctx = try testContext(); + defer ctx.deinit(); + const IntTupleType = fly.types.IntTupleType; + const LayoutType = fly.types.LayoutType; + const TileType = fly.types.TileType; + + try expectBuilt(ctx, IntTupleType, "!fly.int_tuple<{f}>", it(42)); + try expectBuilt(ctx, IntTupleType, "!fly.int_tuple<{f}>", it(.{ .{ 2, 4 }, 8 })); + try expectBuilt(ctx, IntTupleType, "!fly.int_tuple<{f}>", it(.{ IntTuple.dyn, 8 })); + try expectBuilt(ctx, IntTupleType, "!fly.int_tuple<{f}>", IntTuple.dynamic(64, 8)); + try expectBuilt(ctx, IntTupleType, "!fly.int_tuple<{f}>", it(.{ null, 8 })); + try expectBuilt(ctx, IntTupleType, "!fly.int_tuple<{f}>", IntTuple{ .basis = .{ .value = .{ .s = 2 }, .modes = &.{ 1, 2 } } }); + + try expectBuilt(ctx, LayoutType, "!fly.layout<{f}>", L(.{ 4, 8 }, .{ 1, 4 })); + try expectBuilt(ctx, LayoutType, "!fly.layout<{f}>", rowMajor(.{ .{ 2, 4 }, 8 })); + + try expectBuilt(ctx, TileType, "!fly.tile<{f}>", tile(.{ 128, 64 })); + try expectBuilt(ctx, TileType, "!fly.tile<{f}>", tile(.{ null, 8 })); + try expectBuilt(ctx, TileType, "!fly.tile<{f}>", tile(.{ L(.{ 4, 8 }, .{ 1, 4 }), 16 })); + try expectBuilt(ctx, TileType, "!fly.tile<{f}>", tile(.{ .{ 2, 3 }, 4 })); +} diff --git a/mlir/BUILD.bazel b/mlir/BUILD.bazel index 750b0285..e292a030 100644 --- a/mlir/BUILD.bazel +++ b/mlir/BUILD.bazel @@ -15,6 +15,7 @@ cc_library( "@llvm-project//mlir:CAPILLVM", "@llvm-project//mlir:CAPIMath", "@llvm-project//mlir:CAPIMemRef", + "@llvm-project//mlir:CAPIROCDL", "@llvm-project//mlir:CAPISCF", "@llvm-project//mlir:CAPITransforms", "@llvm-project//mlir:CAPIVector", diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/BUILD.bazel index 558b03a0..bcf4deb8 100644 --- a/mlir/dialects/BUILD.bazel +++ b/mlir/dialects/BUILD.bazel @@ -8,6 +8,7 @@ zig_library( "arith.zig", "cf.zig", "func.zig", + "gpu.zig", "math.zig", "memref.zig", "shardy.zig", @@ -21,6 +22,7 @@ zig_library( "//mlir", "//mlir:c", "//mlir/dialects/cuda_tile", + "//mlir/dialects/fly", "//mlir/dialects/mosaic_tpu", "//mlir/dialects/stablehlo", "//mlir/dialects/ttir", diff --git a/mlir/dialects/dialects.zig b/mlir/dialects/dialects.zig index 8646d840..7901ecc5 100644 --- a/mlir/dialects/dialects.zig +++ b/mlir/dialects/dialects.zig @@ -1,6 +1,7 @@ const std = @import("std"); pub const cuda_tile = @import("mlir/dialects/cuda_tile"); +pub const fly = @import("mlir/dialects/fly"); pub const mosaic_tpu = @import("mlir/dialects/mosaic_tpu"); pub const stablehlo = @import("mlir/dialects/stablehlo"); pub const ttir = @import("mlir/dialects/ttir"); @@ -9,6 +10,7 @@ pub const affine = @import("affine.zig"); pub const arith = @import("arith.zig"); pub const cf = @import("cf.zig"); pub const func = @import("func.zig"); +pub const gpu = @import("gpu.zig"); pub const math = @import("math.zig"); pub const memref = @import("memref.zig"); pub const scf = @import("scf.zig"); diff --git a/mlir/dialects/fly/BUILD.bazel b/mlir/dialects/fly/BUILD.bazel new file mode 100644 index 00000000..8f9963df --- /dev/null +++ b/mlir/dialects/fly/BUILD.bazel @@ -0,0 +1,44 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "fly_capi", + srcs = ["fly_capi.cc"], + hdrs = ["fly_capi.h"], + strip_include_prefix = ".", + tags = ["manual"], + visibility = ["//visibility:private"], + deps = [ + "@flydsl//:FlyDialect", + "@flydsl//:FlyROCDLDialect", + "@flydsl//:TiledOpTraits", + "@llvm-project//mlir:CAPIIR", + "@llvm-project//mlir:GPUDialect", + "@llvm-project//mlir:UBDialect", + ], + alwayslink = True, +) + +zig_library( + name = "fly", + tags = ["manual"], + srcs = [ + "attributes.zig", + "rocdl.zig", + "types.zig", + ], + import_name = "mlir/dialects/fly", + main = "fly.zig", + visibility = ["//visibility:public"], + deps = [ + ":fly_capi", + "//mlir", + "//mlir:c", + "//stdx", + ], +) + +zig_test( + name = "test", + deps = [":fly"], +) diff --git a/mlir/dialects/fly/attributes.zig b/mlir/dialects/fly/attributes.zig new file mode 100644 index 00000000..eceb8b0f --- /dev/null +++ b/mlir/dialects/fly/attributes.zig @@ -0,0 +1,239 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); + +/// `#fly.int`, the unified static/dynamic integer leaf. +pub const IntAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAFlyInt; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + + pub fn getStatic(ctx: *mlir.Context, value: i32) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyIntAttrGetStatic(ctx.ptr(), value).ptr orelse return error.InvalidMlir); + } + + pub fn getDynamic(ctx: *mlir.Context, width: i32, divisibility: i32) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyIntAttrGetDynamic(ctx.ptr(), width, divisibility).ptr orelse return error.InvalidMlir); + } + + pub fn getNone(ctx: *mlir.Context) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyIntAttrGetNone(ctx.ptr()).ptr orelse return error.InvalidMlir); + } +}; + +/// `#fly.int_tuple`: a leaf, a basis element, or a nest of those. +pub const IntTupleAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAFlyIntTuple; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + + pub const InitArgs = struct { + value: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyIntTupleAttrGet(ctx.ptr(), args.value.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn getStatic(ctx: *mlir.Context, value: i32) mlir.Error!*const Self { + return get(ctx, .{ .value = (try IntAttr.getStatic(ctx, value)).attribute() }); + } + + pub fn getDynamic(ctx: *mlir.Context, width: i32, divisibility: i32) mlir.Error!*const Self { + return get(ctx, .{ .value = (try IntAttr.getDynamic(ctx, width, divisibility)).attribute() }); + } + + pub fn getNone(ctx: *mlir.Context) mlir.Error!*const Self { + return get(ctx, .{ .value = (try IntAttr.getNone(ctx)).attribute() }); + } + + pub fn getBasis(ctx: *mlir.Context, value: *const IntAttr, modes: []const i32) mlir.Error!*const Self { + const result = c.mlirFlyIntTupleAttrGetBasis(ctx.ptr(), value.ptr(), @intCast(modes.len), modes.ptr); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn getTuple(ctx: *mlir.Context, elements: []const *const Self) mlir.Error!*const Self { + const result = c.mlirFlyIntTupleAttrGetTuple(ctx.ptr(), @intCast(elements.len), @ptrCast(elements.ptr)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `#fly.layout`, a `shape:stride` pair. +pub const LayoutAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAFlyLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + + pub const InitArgs = struct { + shape: *const IntTupleAttr, + stride: *const IntTupleAttr, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyLayoutAttrGet(ctx.ptr(), args.shape.ptr(), args.stride.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `#fly.tile`: a mode, or a `[a|b]` nest of modes. +pub const TileAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAFlyTile; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + + pub const InitArgs = struct { + value: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyTileAttrGet(ctx.ptr(), args.value.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn getModes(ctx: *mlir.Context, modes: []const *const mlir.Attribute) mlir.Error!*const Self { + const result = c.mlirFlyTileAttrGetModes(ctx.ptr(), @intCast(modes.len), @ptrCast(modes.ptr)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `S`. +pub const SwizzleAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAFlySwizzle; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + + pub const InitArgs = struct { + mask: i32, + base: i32, + shift: i32, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlySwizzleAttrGet(ctx.ptr(), args.mask, args.base, args.shift); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `align`, in bytes. +pub const AlignAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAFlyAlign; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + + pub const InitArgs = struct { + alignment: i32, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyAlignAttrGet(ctx.ptr(), args.alignment).ptr orelse return error.InvalidMlir); + } +}; + +pub const AddressSpace = enum(i32) { generic = 0, global = 1, shared = 2, register = 3 }; + +/// `global`, `shared`, ... A pointer or memref may also carry a foreign +/// address-space attribute, such as `#fly_rocdl.buffer_desc`. +pub const AddressSpaceAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAFlyAddressSpace; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + + pub const InitArgs = struct { + addressSpace: AddressSpace, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyAddressSpaceAttrGet(ctx.ptr(), @intFromEnum(args.addressSpace)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +pub const MmaOperand = enum(i32) { a = 0, b = 1, c = 2, d = 3 }; + +pub fn mmaOperandAttr(ctx: *mlir.Context, operand: MmaOperand) mlir.Error!*const mlir.Attribute { + const result = c.mlirFlyMmaOperandAttrGet(ctx.ptr(), @intFromEnum(operand)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); +} + +pub const GemmTraversalOrder = enum(i32) { + kmn = 0, + knm = 1, + mkn = 2, + mnk = 3, + nkm = 4, + nmk = 5, + kmn_serpentine = 6, + knm_serpentine = 7, + mkn_serpentine = 8, + mnk_serpentine = 9, + nkm_serpentine = 10, + nmk_serpentine = 11, +}; + +pub fn gemmTraversalOrderAttr(ctx: *mlir.Context, order: GemmTraversalOrder) mlir.Error!*const mlir.Attribute { + const result = c.mlirFlyGemmTraversalOrderAttrGet(ctx.ptr(), @intFromEnum(order)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); +} diff --git a/mlir/dialects/fly/fly.zig b/mlir/dialects/fly/fly.zig new file mode 100644 index 00000000..6083f308 --- /dev/null +++ b/mlir/dialects/fly/fly.zig @@ -0,0 +1,472 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub const attributes = @import("attributes.zig"); +pub const rocdl = @import("rocdl.zig"); +pub const types = @import("types.zig"); + +pub const AddressSpace = attributes.AddressSpace; +pub const GemmTraversalOrder = attributes.GemmTraversalOrder; +pub const MmaOperand = attributes.MmaOperand; + +/// `gpu`/`rocdl` for `gpu.func` kernels and the ROCDL escape hatches; `ub` +/// because fly ops may fold to `ub.poison`. +pub const dialects_needed = [_][]const u8{ "func", "arith", "scf", "math", "vector", "memref", "llvm", "gpu", "rocdl", "ub", "fly", "fly_rocdl" }; + +pub fn insertDialects(registry: *mlir.DialectRegistry) void { + inline for (dialects_needed) |d| { + mlir.DialectHandle.fromString(d).insertDialect(registry); + } +} + +pub fn opName(comptime mnemonic: []const u8) []const u8 { + return "fly." ++ mnemonic; +} + +/// By value, so a helper's temporary outlives the slice `Operation.make` gets. +pub const Attrs = stdx.BoundedArray(mlir.NamedAttribute, 8); + +pub fn attrs(list: []const mlir.NamedAttribute) Attrs { + var out: Attrs = .empty; + out.appendSliceAssumeCapacity(list); + return out; +} + +/// `Operation.make` that names the op and its operand types on failure: a +/// rejected `inferReturnTypes` otherwise panics with nothing to go on. +pub fn make(ctx: *mlir.Context, name: []const u8, args: mlir.Operation.MakeArgs) *mlir.Operation { + return mlir.Operation.try_make(ctx, name, args) catch |err| { + var buf: [4096]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + if (args.operands) |operands| switch (operands) { + .flat => |vs| for (vs, 0..) |v, i| { + w.print("\n operand {d}: {f}", .{ i, v.type_() }) catch break; + }, + .variadic => |segments| for (segments) |seg| for (seg) |v| { + w.print("\n operand: {f}", .{v.type_()}) catch break; + }, + }; + if (args.results) |results| switch (results) { + .flat => |ts| for (ts, 0..) |t, i| { + w.print("\n result {d}: {f}", .{ i, t }) catch break; + }, + .variadic => {}, + }; + std.debug.panic("{s}: cannot create operation ({}){s}", .{ name, err, w.buffered() }); + }; +} + +/// Result type inferred by FlyDSL. +pub fn inferred(ctx: *mlir.Context, comptime mnemonic: []const u8, operands: []const *const mlir.Value, attributes_: Attrs, location: *const mlir.Location) *mlir.Operation { + return make(ctx, opName(mnemonic), .{ + .operands = .{ .flat = operands }, + .result_type_inference = true, + .attributes = attributes_.constSlice(), + .location = location, + }); +} + +/// Explicit result types. +pub fn typed(ctx: *mlir.Context, comptime mnemonic: []const u8, operands: []const *const mlir.Value, results: []const *const mlir.Type, attributes_: Attrs, location: *const mlir.Location) *mlir.Operation { + return make(ctx, opName(mnemonic), .{ + .operands = .{ .flat = operands }, + .results = .{ .flat = results }, + .attributes = attributes_.constSlice(), + .location = location, + }); +} + +/// No results. +pub fn effect(ctx: *mlir.Context, comptime mnemonic: []const u8, operands: []const *const mlir.Value, attributes_: Attrs, location: *const mlir.Location) *mlir.Operation { + return make(ctx, opName(mnemonic), .{ + .operands = .{ .flat = operands }, + .attributes = attributes_.constSlice(), + .location = location, + }); +} + +/// Panics with the text on failure. +pub fn parseType(ctx: *mlir.Context, text: []const u8) *const mlir.Type { + return mlir.Type.parse(ctx, text) catch std.debug.panic("fly: cannot parse type `{s}`", .{text}); +} + +pub fn parseAttr(ctx: *mlir.Context, text: []const u8) *const mlir.Attribute { + return mlir.Attribute.parse(ctx, text) catch std.debug.panic("fly: cannot parse attribute `{s}`", .{text}); +} + +pub const TypeKind = enum { + memref, + coord_tensor, + ptr, + int_tuple, + layout, + composed_layout, + tile, + swizzle, + copy_atom, + mma_atom, + tiled_copy, + tiled_mma, + other, +}; + +pub fn KindType(comptime kind: TypeKind) type { + return switch (kind) { + .memref => types.MemRefType, + .coord_tensor => types.CoordTensorType, + .ptr => types.PointerType, + .int_tuple => types.IntTupleType, + .layout => types.LayoutType, + .composed_layout => types.ComposedLayoutType, + .tile => types.TileType, + .swizzle => types.SwizzleType, + .copy_atom => types.CopyAtomType, + .mma_atom => types.MmaAtomType, + .tiled_copy => types.TiledCopyType, + .tiled_mma => types.TiledMmaType, + .other => @compileError("fly.KindType: `.other` is not a fly type"), + }; +} + +pub fn typeKind(ty: *const mlir.Type) TypeKind { + inline for (comptime std.meta.tags(TypeKind)) |kind| { + if (kind != .other and ty.isA(KindType(kind)) != null) return kind; + } + return .other; +} + +/// Panics if `ty` is not a `!fly.`. +pub fn expect(ty: *const mlir.Type, comptime kind: TypeKind) *const KindType(kind) { + return ty.isA(KindType(kind)) orelse + std.debug.panic("fly: `{f}` is not a !fly.{s}", .{ ty, @tagName(kind) }); +} + +/// The shape of the outermost plain layout of any layout-carrying type. +pub fn layoutLikeShape(ty: *const mlir.Type) *const types.IntTupleType { + if (ty.isA(types.LayoutType) == null and ty.isA(types.ComposedLayoutType) == null and + ty.isA(types.MemRefType) == null and ty.isA(types.CoordTensorType) == null) + { + std.debug.panic("fly: `{f}` carries no layout", .{ty}); + } + return @ptrCast(c.mlirFlyLayoutLikeTypeGetShape(ty.ptr()).ptr.?); +} + +pub fn elemType(ty: *const mlir.Type) *const mlir.Type { + if (ty.isA(types.MemRefType)) |m| return m.getElemTy(); + if (ty.isA(types.PointerType)) |p| return p.getElemTy(); + std.debug.panic("fly: `{f}` has no element type", .{ty}); +} + +pub fn addressSpace(ty: *const mlir.Type) *const mlir.Attribute { + if (ty.isA(types.MemRefType)) |m| return m.getAddressSpace(); + if (ty.isA(types.PointerType)) |p| return p.getAddressSpace(); + std.debug.panic("fly: `{f}` has no address space", .{ty}); +} + +/// Keeps the address space, alignment and swizzle. +pub fn ptrWithElem(ctx: *mlir.Context, ptr: *const mlir.Type, elem: *const mlir.Type) mlir.Error!*const types.PointerType { + const p = expect(ptr, .ptr); + return types.PointerType.get(ctx, .{ + .elemTy = elem, + .addressSpace = p.getAddressSpace(), + .alignment = p.getAlignment(), + .swizzle = p.getSwizzle(), + }); +} + +/// A value whose whole content lives in its type. +pub fn static(ctx: *mlir.Context, ty: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return typed(ctx, "static", &.{}, &.{ty}, .empty, location); +} + +/// `ty` names the tuple with a `?` per dynamic operand, in order. +pub fn makeIntTuple(ctx: *mlir.Context, dynamic: []const *const mlir.Value, ty: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return typed(ctx, "make_int_tuple", dynamic, &.{ty}, .empty, location); +} + +pub fn makeCopyAtom(ctx: *mlir.Context, ty: *const mlir.Type, val_bits: i32, location: *const mlir.Location) *mlir.Operation { + return typed(ctx, "make_copy_atom", &.{}, &.{ty}, attrs(&.{.named(ctx, "valBits", .int(ctx, .i32, val_bits))}), location); +} + +pub fn makeMmaAtom(ctx: *mlir.Context, ty: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return typed(ctx, "make_mma_atom", &.{}, &.{ty}, .empty, location); +} + +pub fn makeFragmentLike(ctx: *mlir.Context, src: *const mlir.Value, dtype: ?*const mlir.Type, location: *const mlir.Location) *mlir.Operation { + var a: Attrs = .empty; + if (dtype) |t| a.appendAssumeCapacity(.named(ctx, "dtype", .typeAttr(t))); + return inferred(ctx, "make_fragment_like", &.{src}, a, location); +} + +pub const CopyOpts = struct { + pred: ?*const mlir.Value = null, +}; + +pub fn copy(ctx: *mlir.Context, atom: *const mlir.Value, src: *const mlir.Value, dst: *const mlir.Value, opts: CopyOpts, location: *const mlir.Location) *mlir.Operation { + if (opts.pred) |p| return effect(ctx, "copy", &.{ atom, src, dst, p }, .empty, location); + return effect(ctx, "copy", &.{ atom, src, dst }, .empty, location); +} + +pub const GemmOpts = struct { + traversal_order: ?GemmTraversalOrder = null, + traversal_layout: ?*const mlir.Value = null, +}; + +pub fn gemm(ctx: *mlir.Context, atom: *const mlir.Value, d: *const mlir.Value, a: *const mlir.Value, b: *const mlir.Value, cc: *const mlir.Value, opts: GemmOpts, location: *const mlir.Location) *mlir.Operation { + var at: Attrs = .empty; + if (opts.traversal_order) |o| { + const attr = attributes.gemmTraversalOrderAttr(ctx, o) catch @panic("fly.gemm: invalid traversal order"); + at.appendAssumeCapacity(.named(ctx, "traversalOrder", attr)); + } + if (opts.traversal_layout) |l| return effect(ctx, "gemm", &.{ atom, d, a, b, cc, l }, at, location); + return effect(ctx, "gemm", &.{ atom, d, a, b, cc }, at, location); +} + +pub fn get(ctx: *mlir.Context, input: *const mlir.Value, mode: []const i32, location: *const mlir.Location) *mlir.Operation { + return inferred(ctx, "get", &.{input}, attrs(&.{.named(ctx, "mode", .denseArray(ctx, .i32, mode))}), location); +} + +pub fn select(ctx: *mlir.Context, input: *const mlir.Value, indices: []const i32, location: *const mlir.Location) *mlir.Operation { + return inferred(ctx, "select", &.{input}, attrs(&.{.named(ctx, "indices", .denseArray(ctx, .i32, indices))}), location); +} + +pub fn takeOrGroup(ctx: *mlir.Context, comptime mnemonic: []const u8, input: *const mlir.Value, begin: i32, end: i32, location: *const mlir.Location) *mlir.Operation { + return inferred(ctx, mnemonic, &.{input}, attrs(&.{ + .named(ctx, "begin", .int(ctx, .i32, begin)), + .named(ctx, "end", .int(ctx, .i32, end)), + }), location); +} + +/// Explicit pointer type; `fly.make_ptr` infers nothing. +pub fn makePtr(ctx: *mlir.Context, operands: []const *const mlir.Value, ty: *const mlir.Type, dict_attrs: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var a: Attrs = .empty; + if (dict_attrs) |d| a.appendAssumeCapacity(.named(ctx, "dictAttrs", d)); + return typed(ctx, "make_ptr", operands, &.{ty}, a, location); +} + +test { + std.testing.refAllDecls(@This()); + inline for (.{ types, attributes, rocdl }) |module| { + std.testing.refAllDecls(module); + inline for (comptime std.meta.declarations(module)) |decl| { + const value = @field(module, decl.name); + if (@TypeOf(value) == type and @typeInfo(value) == .@"opaque") std.testing.refAllDecls(value); + } + } +} + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + insertDialects(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectPrints(expected: []const u8, value: anytype) !void { + var buf: [512]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(expected, w.buffered()); +} + +test "fly dialects register" { + const ctx = try testContext(); + defer ctx.deinit(); + + try std.testing.expect(ctx.isRegisteredOperation("fly.make_int_tuple")); + try std.testing.expect(ctx.isRegisteredOperation("fly.make_layout")); + try std.testing.expect(ctx.isRegisteredOperation("fly.ptr.load")); + try std.testing.expect(ctx.isRegisteredOperation("fly.copy_atom_call")); + try std.testing.expect(ctx.isRegisteredOperation("gpu.func")); + try std.testing.expect(ctx.isRegisteredOperation("rocdl.sched.barrier")); + try std.testing.expect(ctx.isRegisteredOperation("ub.poison")); +} + +test "fly types parse and print round-trip" { + const ctx = try testContext(); + defer ctx.deinit(); + + const cases = [_][]const u8{ + "!fly.int_tuple<((2, 4), 8)>", + "!fly.int_tuple<((1, 4), (32, 64))>", + "!fly.layout<((2, 3), 3) : ((57, 69), 19)>", + "!fly.memref", + "!fly.memref", + "!fly.ptr", + "!fly.ptr", + "!fly.ptr", + "!fly.memref", + "!fly_rocdl.cdna3.buffer_copy<128>", + "!fly_rocdl.cdna3.buffer_copy_lds<32>", + "!fly_rocdl.cdna3.mfma<16x16x16, (f16, f16) -> f32>", + "!fly.copy_atom, 32>", + "!fly.tile<[8|64]>", + "!fly.coord_tensor<(0,0), (100,1000):(1E0,1E1)>", + }; + for (cases) |src| { + const ty = mlir.Type.parse(ctx, src) catch |err| { + std.debug.print("failed to parse `{s}`: {}\n", .{ src, err }); + return err; + }; + var buf: [512]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{ty}); + const ty2 = try mlir.Type.parse(ctx, w.buffered()); + try std.testing.expect(ty.eql(ty2)); + } +} + +test "types and attributes are built through the C API" { + const ctx = try testContext(); + defer ctx.deinit(); + + const two = try attributes.IntTupleAttr.getStatic(ctx, 2); + const four = try attributes.IntTupleAttr.getStatic(ctx, 4); + const one = try attributes.IntTupleAttr.getStatic(ctx, 1); + const shape = try attributes.IntTupleAttr.getTuple(ctx, &.{ two, four }); + const stride = try attributes.IntTupleAttr.getTuple(ctx, &.{ four, one }); + const layout = try types.LayoutType.get(ctx, .{ .attr = try .get(ctx, .{ .shape = shape, .stride = stride }) }); + try expectPrints("!fly.layout<(2,4):(4,1)>", layout); + try expectPrints("!fly.int_tuple<(2,4)>", layout.getShape()); + try std.testing.expectEqual(@as(usize, 2), layout.getShape().getNumElements()); + try std.testing.expectEqual(types.IntTupleType.Leaf{ .static = 4 }, layout.getShape().getElement(1).getLeaf()); + + const global = try attributes.AddressSpaceAttr.get(ctx, .{ .addressSpace = .global }); + const memref = try types.MemRefType.get(ctx, .{ + .elemTy = .float(ctx, .f16), + .addressSpace = global.attribute(), + .layout = layout.getAttr().attribute(), + }); + try expectPrints("!fly.memref", memref); + try expectPrints("!fly.int_tuple<(2,4)>", layoutLikeShape(memref.type_())); + + const shared = try attributes.AddressSpaceAttr.get(ctx, .{ .addressSpace = .shared }); + const ptr = try types.PointerType.get(ctx, .{ + .elemTy = .int(ctx, .i8), + .addressSpace = shared.attribute(), + .alignment = try .get(ctx, .{ .alignment = 16 }), + }); + try expectPrints("!fly.ptr>", ptr); + try expectPrints("!fly.ptr>", try ptrWithElem(ctx, ptr.type_(), .float(ctx, .f32))); + + const tile = try types.TileType.get(ctx, .{ .attr = try .getModes(ctx, &.{ + (try attributes.IntAttr.getStatic(ctx, 8)).attribute(), + (try attributes.IntAttr.getStatic(ctx, 64)).attribute(), + }) }); + try expectPrints("!fly.tile<[8|64]>", tile); + + const copy_atom = try types.CopyAtomType.get(ctx, .{ + .copyOp = (try types.CopyOpUniversalCopyType.get(ctx, .{ .bitSize = 128 })).type_(), + .valBits = 32, + }); + try expectPrints("!fly.copy_atom, 32>", copy_atom); + + const buffer_desc = try rocdl.BufferDescAddressAttr.get(ctx); + try expectPrints("!fly.ptr", try types.PointerType.get(ctx, .{ + .elemTy = .float(ctx, .f16), + .addressSpace = buffer_desc.attribute(), + })); +} + +test "fly enum and gpu attributes" { + const ctx = try testContext(); + defer ctx.deinit(); + _ = try attributes.mmaOperandAttr(ctx, .a); + _ = try attributes.gemmTraversalOrderAttr(ctx, .kmn); + _ = parseAttr(ctx, "#gpu"); +} + +test "inferred fly ops compute layout algebra" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc: *const mlir.Location = .unknown(ctx); + const module: *mlir.Module = .init(loc); + defer module.deinit(); + const block = module.body(); + + // raked_product((8,16):(16,1), (1,4):(1,1)) — the vectorAdd TV layout. + const thr = static(ctx, parseType(ctx, "!fly.layout<(8,16):(16,1)>"), loc).appendTo(block); + const val = static(ctx, parseType(ctx, "!fly.layout<(1,4):(1,1)>"), loc).appendTo(block); + const mn = inferred(ctx, "raked_product", &.{ thr.result(0), val.result(0) }, .empty, loc).appendTo(block); + const shape = inferred(ctx, "get_shape", &.{mn.result(0)}, .empty, loc).appendTo(block); + const tiler = inferred(ctx, "int_tuple_product_each", &.{shape.result(0)}, .empty, loc).appendTo(block); + + try expectPrints("!fly.int_tuple<(8,64)>", tiler.result(0).type_()); + try std.testing.expect(module.operation().verify()); +} + +test "atom traits" { + const ctx = try testContext(); + defer ctx.deinit(); + + // MFMA 16x16x4 f32 tiled over a (2,2,1) atom layout. + const tm = expect(parseType(ctx, "!fly.tiled_mma f32>>, <(2,2,1):(1,2,0)>>"), .tiled_mma); + try expectPrints("!fly.int_tuple<(32,32,4)>", tm.getTileSizeMNK()); + _ = tm.getTiledThrValLayoutA(); + _ = tm.getThrLayoutVMNK(); + + const atom = try types.MmaAtomType.get(ctx, .{ .mmaOp = (try rocdl.MmaOpCDNA3MFMAType.get(ctx, .{ + .m = 16, + .n = 16, + .k = 4, + .elemTyA = .float(ctx, .f32), + .elemTyB = .float(ctx, .f32), + .elemTyAcc = .float(ctx, .f32), + })).type_() }); + try expectPrints("!fly.mma_atom f32>>", atom); + try expectPrints("!fly.int_tuple<(16,16,4)>", atom.getShapeMNK()); + try expectPrints("!fly.layout<64:1>", atom.getThrLayout()); + + const wmma = try types.MmaAtomType.get(ctx, .{ .mmaOp = (try rocdl.MmaOpGFX11WMMAType.get(ctx, .{ + .m = 16, + .n = 16, + .k = 16, + .elemTyA = .float(ctx, .f16), + .elemTyB = .float(ctx, .f16), + .elemTyAcc = .float(ctx, .f32), + })).type_() }); + try expectPrints("!fly.layout<32:1>", wmma.getThrLayout()); + + const ca = expect(parseType(ctx, "!fly.copy_atom, 32>"), .copy_atom); + _ = ca.getThrValLayoutSrc(); +} + +test "type kinds" { + const ctx = try testContext(); + defer ctx.deinit(); + const cases = .{ + .{ "!fly.memref", TypeKind.memref }, + .{ "!fly.ptr", TypeKind.ptr }, + .{ "!fly.int_tuple<(2,4)>", TypeKind.int_tuple }, + .{ "!fly.layout<(8,4):(4,1)>", TypeKind.layout }, + .{ "!fly.tile<[8|4]>", TypeKind.tile }, + .{ "!fly.copy_atom, 32>", TypeKind.copy_atom }, + .{ "!fly.mma_atom f32>>", TypeKind.mma_atom }, + }; + inline for (cases) |case| { + try std.testing.expectEqual(case[1], typeKind(parseType(ctx, case[0]))); + } + try std.testing.expectEqual(TypeKind.other, typeKind(.float(ctx, .f32))); +} + +test "structural readers" { + const ctx = try testContext(); + defer ctx.deinit(); + const it = expect(parseType(ctx, "!fly.int_tuple<((2,4),?)>"), .int_tuple); + try std.testing.expectEqual(@as(usize, 2), it.getNumElements()); + try std.testing.expect(!it.isLeaf()); + try std.testing.expect(!it.isStatic()); + const inner = it.getElement(0); + try std.testing.expectEqual(@as(usize, 2), inner.getNumElements()); + try std.testing.expectEqual(types.IntTupleType.Leaf{ .static = 4 }, inner.getElement(1).getLeaf()); + try std.testing.expectEqual(types.IntTupleType.Leaf.dynamic, it.getElement(1).getLeaf()); + + const m = parseType(ctx, "!fly.memref o 0 o (64,32):(32,1), align<16>>"); + try expectPrints("!fly.int_tuple<(64,32)>", layoutLikeShape(m)); + try expectPrints("bf16", elemType(m)); +} diff --git a/mlir/dialects/fly/fly_capi.cc b/mlir/dialects/fly/fly_capi.cc new file mode 100644 index 00000000..fe28ff98 --- /dev/null +++ b/mlir/dialects/fly/fly_capi.cc @@ -0,0 +1,521 @@ +#include "fly_capi.h" + +#include "TiledOpTraits.h" +#include "flydsl/Dialect/Fly/IR/FlyDialect.h" +#include "flydsl/Dialect/FlyROCDL/IR/Dialect.h" +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" +#include "mlir/Dialect/GPU/IR/GPUDialect.h" +#include "mlir/Dialect/UB/IR/UBOps.h" + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Fly, fly, mlir::fly::FlyDialect) +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(FlyROCDL, fly_rocdl, + mlir::fly_rocdl::FlyROCDLDialect) +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(UB, ub, mlir::ub::UBDialect) +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(GPU, gpu, mlir::gpu::GPUDialect) + +using mlir::cast; +using mlir::isa; +using namespace mlir::fly; + +namespace { + +mlir::MLIRContext *loadFly(MlirContext ctx) { + mlir::MLIRContext *context = unwrap(ctx); + if (context) context->getOrLoadDialect(); + return context; +} + +mlir::MLIRContext *loadFlyROCDL(MlirContext ctx) { + mlir::MLIRContext *context = unwrap(ctx); + if (context) context->getOrLoadDialect(); + return context; +} + +/// `!attr` and `attr == other` are value arithmetic on a fly IntAttr, not a +/// null test: every handle check here goes through these two helpers. +template +bool isNull(T value) { + return !static_cast(value); +} + +template +T checked(Handle handle, mlir::MLIRContext *ctx) { + auto value = mlir::dyn_cast_if_present(unwrap(handle)); + if (isNull(value) || value.getContext() != ctx) return T{}; + return value; +} + +template +MlirType mmaOpGet(MlirContext ctx, int32_t m, int32_t n, int32_t k, MlirType elemTyA, MlirType elemTyB, MlirType elemTyAcc, Flags... flags) { + mlir::MLIRContext *context = loadFlyROCDL(ctx); + mlir::Type a = unwrap(elemTyA), b = unwrap(elemTyB), acc = unwrap(elemTyAcc); + if (!context || isNull(a) || isNull(b) || isNull(acc)) return {nullptr}; + if (a.getContext() != context || b.getContext() != context || + acc.getContext() != context) + return {nullptr}; + return wrap(static_cast( + Type::get(context, m, n, k, a, b, acc, flags...))); +} + + +} // namespace + +extern "C" { + +#define FLY_ATTR_ISA(Name, Attr) \ + bool mlirAttributeIsAFly##Name(MlirAttribute a) { \ + return mlir::isa_and_nonnull(unwrap(a)); \ + } + +FLY_ATTR_ISA(Int, IntAttr) +FLY_ATTR_ISA(IntTuple, IntTupleAttr) +FLY_ATTR_ISA(Layout, LayoutAttr) +FLY_ATTR_ISA(Tile, TileAttr) +FLY_ATTR_ISA(Swizzle, SwizzleAttr) +FLY_ATTR_ISA(Align, AlignAttr) +FLY_ATTR_ISA(AddressSpace, AddressSpaceAttr) + +#undef FLY_ATTR_ISA + +#define FLY_TYPE_ISA(Name, Type) \ + bool mlirTypeIsAFly##Name(MlirType t) { \ + return mlir::isa_and_nonnull(unwrap(t)); \ + } + +FLY_TYPE_ISA(IntTuple, IntTupleType) +FLY_TYPE_ISA(Layout, LayoutType) +FLY_TYPE_ISA(ComposedLayout, ComposedLayoutType) +FLY_TYPE_ISA(Tile, TileType) +FLY_TYPE_ISA(Swizzle, SwizzleType) +FLY_TYPE_ISA(Pointer, PointerType) +FLY_TYPE_ISA(MemRef, mlir::fly::MemRefType) +FLY_TYPE_ISA(CoordTensor, CoordTensorType) +FLY_TYPE_ISA(CopyAtom, CopyAtomType) +FLY_TYPE_ISA(MmaAtom, MmaAtomType) +FLY_TYPE_ISA(TiledCopy, TiledCopyType) +FLY_TYPE_ISA(TiledMma, TiledMmaType) +FLY_TYPE_ISA(CopyOpUniversalCopy, CopyOpUniversalCopyType) +FLY_TYPE_ISA(ROCDLCopyOpCDNA3BufferCopy, mlir::fly_rocdl::CopyOpCDNA3BufferCopyType) +FLY_TYPE_ISA(ROCDLCopyOpCDNA3BufferCopyLDS, mlir::fly_rocdl::CopyOpCDNA3BufferCopyLDSType) +FLY_TYPE_ISA(ROCDLMmaOpCDNA3MFMA, mlir::fly_rocdl::MmaOpCDNA3_MFMAType) +FLY_TYPE_ISA(ROCDLMmaOpGFX11WMMA, mlir::fly_rocdl::MmaOpGFX11_WMMAType) +FLY_TYPE_ISA(ROCDLMmaOpGFX120XWMMA, mlir::fly_rocdl::MmaOpGFX120X_WMMAType) + +#undef FLY_TYPE_ISA + +MlirAttribute mlirFlyIntAttrGetStatic(MlirContext ctx, int32_t value) { + mlir::MLIRContext *context = loadFly(ctx); + if (!context) return {nullptr}; + return wrap(static_cast(IntAttr::getStatic(context, value))); +} + +MlirAttribute mlirFlyIntAttrGetDynamic(MlirContext ctx, int32_t width, int32_t divisibility) { + mlir::MLIRContext *context = loadFly(ctx); + if (!context) return {nullptr}; + return wrap(static_cast(IntAttr::getDynamic(context, width, divisibility))); +} + +MlirAttribute mlirFlyIntAttrGetNone(MlirContext ctx) { + mlir::MLIRContext *context = loadFly(ctx); + if (!context) return {nullptr}; + return wrap(static_cast(IntAttr::getNone(context))); +} + +MlirAttribute mlirFlyIntTupleAttrGet(MlirContext ctx, MlirAttribute value) { + mlir::MLIRContext *context = loadFly(ctx); + mlir::Attribute inner = unwrap(value); + if (!context || isNull(inner) || inner.getContext() != context) return {nullptr}; + return wrap(static_cast(IntTupleAttr::get(context, inner))); +} + +MlirAttribute mlirFlyIntTupleAttrGetBasis(MlirContext ctx, MlirAttribute value, intptr_t nModes, const int32_t *modes) { + mlir::MLIRContext *context = loadFly(ctx); + if (!context) return {nullptr}; + IntAttr intValue = checked(value, context); + if (isNull(intValue) || (nModes > 0 && !modes)) return {nullptr}; + return wrap(static_cast( + IntTupleAttr::get(intValue, llvm::ArrayRef(modes, nModes)))); +} + +MlirAttribute mlirFlyIntTupleAttrGetTuple(MlirContext ctx, intptr_t nElements, MlirAttribute const *elements) { + mlir::MLIRContext *context = loadFly(ctx); + if (!context || (nElements > 0 && !elements)) return {nullptr}; + llvm::SmallVector values; + values.reserve(nElements); + for (intptr_t i = 0; i < nElements; ++i) { + IntTupleAttr element = checked(elements[i], context); + if (isNull(element)) return {nullptr}; + values.push_back(element); + } + return wrap(static_cast( + IntTupleAttr::get(context, mlir::ArrayAttr::get(context, values)))); +} + +MlirAttribute mlirFlyLayoutAttrGet(MlirContext ctx, MlirAttribute shape, MlirAttribute stride) { + mlir::MLIRContext *context = loadFly(ctx); + if (!context) return {nullptr}; + IntTupleAttr shapeAttr = checked(shape, context); + IntTupleAttr strideAttr = checked(stride, context); + if (isNull(shapeAttr) || isNull(strideAttr)) return {nullptr}; + return wrap(static_cast(LayoutAttr::get(context, shapeAttr, strideAttr))); +} + +MlirAttribute mlirFlyTileAttrGet(MlirContext ctx, MlirAttribute value) { + mlir::MLIRContext *context = loadFly(ctx); + mlir::Attribute inner = unwrap(value); + if (!context || isNull(inner) || inner.getContext() != context) return {nullptr}; + return wrap(static_cast(TileAttr::get(context, inner))); +} + +MlirAttribute mlirFlyTileAttrGetModes(MlirContext ctx, intptr_t nModes, MlirAttribute const *modes) { + mlir::MLIRContext *context = loadFly(ctx); + if (!context || (nModes > 0 && !modes)) return {nullptr}; + llvm::SmallVector values; + values.reserve(nModes); + for (intptr_t i = 0; i < nModes; ++i) { + mlir::Attribute mode = unwrap(modes[i]); + if (isNull(mode) || mode.getContext() != context) return {nullptr}; + values.push_back(mode); + } + return wrap(static_cast( + TileAttr::get(context, mlir::ArrayAttr::get(context, values)))); +} + +MlirAttribute mlirFlySwizzleAttrGet(MlirContext ctx, int32_t mask, int32_t base, int32_t shift) { + mlir::MLIRContext *context = loadFly(ctx); + if (!context) return {nullptr}; + return wrap(static_cast(SwizzleAttr::get(context, mask, base, shift))); +} + +MlirAttribute mlirFlyAlignAttrGet(MlirContext ctx, int32_t alignment) { + mlir::MLIRContext *context = loadFly(ctx); + if (!context) return {nullptr}; + return wrap(static_cast(AlignAttr::get(context, alignment))); +} + +MlirAttribute mlirFlyAddressSpaceAttrGet(MlirContext ctx, int32_t addressSpace) { + mlir::MLIRContext *context = loadFly(ctx); + if (!context) return {nullptr}; + return wrap(static_cast( + AddressSpaceAttr::get(context, static_cast(addressSpace)))); +} + +MlirAttribute mlirFlyMmaOperandAttrGet(MlirContext ctx, int32_t operand) { + mlir::MLIRContext *context = loadFly(ctx); + if (!context) return {nullptr}; + return wrap(static_cast( + MmaOperandAttr::get(context, static_cast(operand)))); +} + +MlirAttribute mlirFlyGemmTraversalOrderAttrGet(MlirContext ctx, int32_t order) { + mlir::MLIRContext *context = loadFly(ctx); + if (!context) return {nullptr}; + return wrap(static_cast(GemmTraversalOrderAttr::get( + context, static_cast(order)))); +} + +MlirType mlirFlyIntTupleTypeGet(MlirContext ctx, MlirAttribute attr) { + mlir::MLIRContext *context = loadFly(ctx); + if (!context) return {nullptr}; + IntTupleAttr value = checked(attr, context); + if (isNull(value)) return {nullptr}; + return wrap(static_cast(IntTupleType::get(value))); +} + +MlirAttribute mlirFlyIntTupleTypeGetAttr(MlirType type) { + return wrap(static_cast(cast(unwrap(type)).getAttr())); +} + +intptr_t mlirFlyIntTupleTypeGetNumElements(MlirType type) { + return cast(unwrap(type)).rank(); +} + +MlirType mlirFlyIntTupleTypeGetElement(MlirType type, intptr_t pos) { + return wrap(static_cast(cast(unwrap(type)).at(pos))); +} + +bool mlirFlyIntTupleTypeIsLeaf(MlirType type) { + return cast(unwrap(type)).isLeaf(); +} + +bool mlirFlyIntTupleTypeIsStatic(MlirType type) { + return cast(unwrap(type)).isStatic(); +} + +bool mlirFlyIntTupleTypeIsStaticLeaf(MlirType type) { + IntTupleAttr attr = cast(unwrap(type)).getAttr(); + return attr.isLeafInt() && attr.getLeafAsInt().isStatic(); +} + +bool mlirFlyIntTupleTypeIsNoneLeaf(MlirType type) { + return cast(unwrap(type)).getAttr().isLeafNone(); +} + +bool mlirFlyIntTupleTypeIsBasisLeaf(MlirType type) { + return cast(unwrap(type)).getAttr().isLeafBasis(); +} + +int64_t mlirFlyIntTupleTypeGetStaticValue(MlirType type) { + return cast(unwrap(type)).getAttr().getLeafAsInt().getValue(); +} + +MlirType mlirFlyLayoutTypeGet(MlirContext ctx, MlirAttribute attr) { + mlir::MLIRContext *context = loadFly(ctx); + if (!context) return {nullptr}; + LayoutAttr value = checked(attr, context); + if (isNull(value)) return {nullptr}; + return wrap(static_cast(LayoutType::get(value))); +} + +MlirAttribute mlirFlyLayoutTypeGetAttr(MlirType type) { + return wrap(static_cast(cast(unwrap(type)).getAttr())); +} + +MlirType mlirFlyLayoutTypeGetShape(MlirType type) { + return wrap(static_cast( + IntTupleType::get(cast(unwrap(type)).getAttr().getShape()))); +} + +MlirType mlirFlyLayoutTypeGetStride(MlirType type) { + return wrap(static_cast( + IntTupleType::get(cast(unwrap(type)).getAttr().getStride()))); +} + +MlirAttribute mlirFlyComposedLayoutTypeGetAttr(MlirType type) { + return wrap(static_cast(cast(unwrap(type)).getAttr())); +} + +MlirType mlirFlyTileTypeGet(MlirContext ctx, MlirAttribute attr) { + mlir::MLIRContext *context = loadFly(ctx); + if (!context) return {nullptr}; + TileAttr value = checked(attr, context); + if (isNull(value)) return {nullptr}; + return wrap(static_cast(TileType::get(value))); +} + +MlirAttribute mlirFlyTileTypeGetAttr(MlirType type) { + return wrap(static_cast(cast(unwrap(type)).getAttr())); +} + +MlirType mlirFlySwizzleTypeGet(MlirContext ctx, MlirAttribute attr) { + mlir::MLIRContext *context = loadFly(ctx); + if (!context) return {nullptr}; + SwizzleAttr value = checked(attr, context); + if (isNull(value)) return {nullptr}; + return wrap(static_cast(SwizzleType::get(value))); +} + +MlirType mlirFlyPointerTypeGet(MlirContext ctx, MlirType elemTy, MlirAttribute addressSpace, MlirAttribute alignment, MlirAttribute swizzle) { + mlir::MLIRContext *context = loadFly(ctx); + mlir::Type element = unwrap(elemTy); + mlir::Attribute space = unwrap(addressSpace); + if (!context || isNull(element) || element.getContext() != context) return {nullptr}; + if (isNull(space) || space.getContext() != context) return {nullptr}; + AlignAttr align = alignment.ptr ? checked(alignment, context) + : AlignAttr::getTrivialAlignment(element); + SwizzleAttr sw = swizzle.ptr ? checked(swizzle, context) + : SwizzleAttr::getTrivialSwizzle(context); + if (isNull(align) || isNull(sw)) return {nullptr}; + return wrap(static_cast( + PointerType::get(context, element, space, align, sw))); +} + +MlirType mlirFlyPointerTypeGetElemTy(MlirType type) { + return wrap(cast(unwrap(type)).getElemTy()); +} + +MlirAttribute mlirFlyPointerTypeGetAddressSpace(MlirType type) { + return wrap(cast(unwrap(type)).getAddressSpace()); +} + +MlirAttribute mlirFlyPointerTypeGetAlignment(MlirType type) { + return wrap(static_cast(cast(unwrap(type)).getAlignment())); +} + +MlirAttribute mlirFlyPointerTypeGetSwizzle(MlirType type) { + return wrap(static_cast(cast(unwrap(type)).getSwizzle())); +} + +MlirType mlirFlyMemRefTypeGet(MlirContext ctx, MlirType elemTy, MlirAttribute addressSpace, MlirAttribute layout, MlirAttribute alignment, MlirAttribute swizzle) { + mlir::MLIRContext *context = loadFly(ctx); + mlir::Type element = unwrap(elemTy); + mlir::Attribute space = unwrap(addressSpace); + mlir::Attribute lay = unwrap(layout); + if (!context || isNull(element) || element.getContext() != context) return {nullptr}; + if (isNull(space) || space.getContext() != context) return {nullptr}; + if (isNull(lay) || lay.getContext() != context) return {nullptr}; + AlignAttr align = alignment.ptr ? checked(alignment, context) + : AlignAttr::getTrivialAlignment(element); + SwizzleAttr sw = swizzle.ptr ? checked(swizzle, context) + : SwizzleAttr::getTrivialSwizzle(context); + if (isNull(align) || isNull(sw)) return {nullptr}; + return wrap(static_cast( + mlir::fly::MemRefType::get(context, element, space, lay, align, sw))); +} + +MlirType mlirFlyMemRefTypeGetElemTy(MlirType type) { + return wrap(cast(unwrap(type)).getElemTy()); +} + +MlirAttribute mlirFlyMemRefTypeGetAddressSpace(MlirType type) { + return wrap(cast(unwrap(type)).getAddressSpace()); +} + +MlirAttribute mlirFlyMemRefTypeGetLayout(MlirType type) { + return wrap(cast(unwrap(type)).getLayout()); +} + +MlirType mlirFlyCoordTensorTypeGet(MlirContext ctx, MlirAttribute base, MlirAttribute layout) { + mlir::MLIRContext *context = loadFly(ctx); + if (!context) return {nullptr}; + IntTupleAttr baseAttr = checked(base, context); + mlir::Attribute lay = unwrap(layout); + if (isNull(baseAttr) || isNull(lay) || lay.getContext() != context) return {nullptr}; + return wrap(static_cast(CoordTensorType::get(context, baseAttr, lay))); +} + +static mlir::Attribute outerLayoutAttr(mlir::Attribute layout) { + while (auto composed = mlir::dyn_cast(layout)) + layout = composed.getOuter(); + return layout; +} + +MlirType mlirFlyLayoutLikeTypeGetShape(MlirType type) { + mlir::Type ty = unwrap(type); + mlir::Attribute layout; + if (auto l = mlir::dyn_cast(ty)) { + layout = l.getAttr(); + } else if (auto c = mlir::dyn_cast(ty)) { + layout = c.getAttr(); + } else if (auto m = mlir::dyn_cast(ty)) { + layout = m.getLayout(); + } else { + layout = cast(ty).getLayout(); + } + return wrap(static_cast( + IntTupleType::get(cast(outerLayoutAttr(layout)).getShape()))); +} + +MlirType mlirFlyCopyAtomTypeGet(MlirContext ctx, MlirType copyOp, int32_t valBits) { + mlir::MLIRContext *context = loadFly(ctx); + mlir::Type op = unwrap(copyOp); + if (!context || isNull(op) || op.getContext() != context) return {nullptr}; + return wrap(static_cast(CopyAtomType::get(context, op, valBits))); +} + +MlirType mlirFlyMmaAtomTypeGet(MlirContext ctx, MlirType mmaOp) { + mlir::MLIRContext *context = loadFly(ctx); + mlir::Type op = unwrap(mmaOp); + if (!context || isNull(op) || op.getContext() != context) return {nullptr}; + return wrap(static_cast(MmaAtomType::get(context, op))); +} + +#define FLY_ATOM_LAYOUT(Name, AtomType, getter) \ + MlirType Name(MlirType type) { \ + return wrap(static_cast(LayoutType::get( \ + cast(cast(unwrap(type)).getter())))); \ + } + +FLY_ATOM_LAYOUT(mlirFlyCopyAtomTypeGetThrLayout, CopyAtomType, getThrLayout) +FLY_ATOM_LAYOUT(mlirFlyCopyAtomTypeGetThrValLayoutSrc, CopyAtomType, getThrValLayoutSrc) +FLY_ATOM_LAYOUT(mlirFlyCopyAtomTypeGetThrValLayoutDst, CopyAtomType, getThrValLayoutDst) +FLY_ATOM_LAYOUT(mlirFlyCopyAtomTypeGetThrValLayoutRef, CopyAtomType, getThrValLayoutRef) +FLY_ATOM_LAYOUT(mlirFlyMmaAtomTypeGetThrLayout, MmaAtomType, getThrLayout) +FLY_ATOM_LAYOUT(mlirFlyMmaAtomTypeGetThrValLayoutA, MmaAtomType, getThrValLayoutA) +FLY_ATOM_LAYOUT(mlirFlyMmaAtomTypeGetThrValLayoutB, MmaAtomType, getThrValLayoutB) +FLY_ATOM_LAYOUT(mlirFlyMmaAtomTypeGetThrValLayoutC, MmaAtomType, getThrValLayoutC) + +#undef FLY_ATOM_LAYOUT + +MlirType mlirFlyMmaAtomTypeGetShapeMNK(MlirType type) { + return wrap(static_cast(IntTupleType::get( + cast(cast(unwrap(type)).getShapeMNK())))); +} + +MlirType mlirFlyTiledCopyTypeGetTiledThrValLayoutSrc(MlirType type) { + auto ty = cast(unwrap(type)); + return wrap(static_cast(LayoutType::get(tiledCopyGetTiledThrValLayoutSrc( + cast(ty.getCopyAtom()), ty.getLayoutThrVal().getAttr(), + ty.getTileMN().getAttr())))); +} + +MlirType mlirFlyTiledCopyTypeGetTiledThrValLayoutDst(MlirType type) { + auto ty = cast(unwrap(type)); + return wrap(static_cast(LayoutType::get(tiledCopyGetTiledThrValLayoutDst( + cast(ty.getCopyAtom()), ty.getLayoutThrVal().getAttr(), + ty.getTileMN().getAttr())))); +} + +MlirType mlirFlyTiledMmaTypeGetTileSizeMNK(MlirType type) { + auto ty = cast(unwrap(type)); + return wrap(static_cast(IntTupleType::get(tiledMmaGetTileSizeMNK( + cast(ty.getMmaAtom()), ty.getAtomLayout().getAttr(), + ty.getPermutation().getAttr())))); +} + +MlirType mlirFlyTiledMmaTypeGetThrLayoutVMNK(MlirType type) { + auto ty = cast(unwrap(type)); + return wrap(static_cast(LayoutType::get(tiledMmaGetThrLayoutVMNK( + cast(ty.getMmaAtom()), ty.getAtomLayout().getAttr())))); +} + +static MlirType tiledMmaTiledThrVal(MlirType tiledMma, MmaOperand operand) { + auto ty = cast(unwrap(tiledMma)); + return wrap(static_cast(LayoutType::get(tiledMmaGetTiledThrValLayout( + cast(ty.getMmaAtom()), ty.getAtomLayout().getAttr(), + ty.getPermutation().getAttr(), operand)))); +} + +MlirType mlirFlyTiledMmaTypeGetTiledThrValLayoutA(MlirType type) { + return tiledMmaTiledThrVal(type, MmaOperand::A); +} + +MlirType mlirFlyTiledMmaTypeGetTiledThrValLayoutB(MlirType type) { + return tiledMmaTiledThrVal(type, MmaOperand::B); +} + +MlirType mlirFlyTiledMmaTypeGetTiledThrValLayoutC(MlirType type) { + return tiledMmaTiledThrVal(type, MmaOperand::C); +} + +MlirType mlirFlyCopyOpUniversalCopyTypeGet(MlirContext ctx, int32_t bitSize) { + mlir::MLIRContext *context = loadFly(ctx); + if (!context) return {nullptr}; + return wrap(static_cast(CopyOpUniversalCopyType::get(context, bitSize))); +} + +MlirAttribute mlirFlyROCDLBufferDescAddressAttrGet(MlirContext ctx) { + mlir::MLIRContext *context = loadFlyROCDL(ctx); + if (!context) return {nullptr}; + return wrap(static_cast( + mlir::fly_rocdl::BufferDescAddressAttr::get(context))); +} + +MlirType mlirFlyROCDLCopyOpCDNA3BufferCopyTypeGet(MlirContext ctx, int32_t bitSize, int32_t cacheModifier) { + mlir::MLIRContext *context = loadFlyROCDL(ctx); + if (!context) return {nullptr}; + return wrap(static_cast( + mlir::fly_rocdl::CopyOpCDNA3BufferCopyType::get(context, bitSize, cacheModifier))); +} + +MlirType mlirFlyROCDLCopyOpCDNA3BufferCopyLDSTypeGet(MlirContext ctx, int32_t bitSize) { + mlir::MLIRContext *context = loadFlyROCDL(ctx); + if (!context) return {nullptr}; + return wrap(static_cast( + mlir::fly_rocdl::CopyOpCDNA3BufferCopyLDSType::get(context, bitSize))); +} + +MlirType mlirFlyROCDLMmaOpCDNA3MFMATypeGet(MlirContext ctx, int32_t m, int32_t n, int32_t k, MlirType elemTyA, MlirType elemTyB, MlirType elemTyAcc) { + return mmaOpGet(ctx, m, n, k, elemTyA, elemTyB, elemTyAcc); +} + +MlirType mlirFlyROCDLMmaOpGFX11WMMATypeGet(MlirContext ctx, int32_t m, int32_t n, int32_t k, MlirType elemTyA, MlirType elemTyB, MlirType elemTyAcc, bool signA, bool signB, bool clamp) { + return mmaOpGet(ctx, m, n, k, elemTyA, elemTyB, elemTyAcc, signA, signB, clamp); +} + +MlirType mlirFlyROCDLMmaOpGFX120XWMMATypeGet(MlirContext ctx, int32_t m, int32_t n, int32_t k, MlirType elemTyA, MlirType elemTyB, MlirType elemTyAcc, bool signA, bool signB, bool clamp) { + return mmaOpGet(ctx, m, n, k, elemTyA, elemTyB, elemTyAcc, signA, signB, clamp); +} + +} // extern "C" diff --git a/mlir/dialects/fly/fly_capi.h b/mlir/dialects/fly/fly_capi.h new file mode 100644 index 00000000..bf92ed47 --- /dev/null +++ b/mlir/dialects/fly/fly_capi.h @@ -0,0 +1,144 @@ +#ifndef ZML_MLIR_DIALECTS_FLY_FLY_CAPI_H +#define ZML_MLIR_DIALECTS_FLY_FLY_CAPI_H + +#include +#include + +#include "mlir-c/IR.h" +#include "mlir-c/Support.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Fly, fly); +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(FlyROCDL, fly_rocdl); +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(UB, ub); +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(GPU, gpu); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyInt(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntAttrGetStatic(MlirContext ctx, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntAttrGetDynamic(MlirContext ctx, int32_t width, int32_t divisibility); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntAttrGetNone(MlirContext ctx); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyIntTuple(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntTupleAttrGet(MlirContext ctx, MlirAttribute value); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntTupleAttrGetBasis(MlirContext ctx, MlirAttribute value, intptr_t nModes, const int32_t *modes); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntTupleAttrGetTuple(MlirContext ctx, intptr_t nElements, MlirAttribute const *elements); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyLayoutAttrGet(MlirContext ctx, MlirAttribute shape, MlirAttribute stride); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyTile(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyTileAttrGet(MlirContext ctx, MlirAttribute value); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyTileAttrGetModes(MlirContext ctx, intptr_t nModes, MlirAttribute const *modes); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlySwizzle(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlySwizzleAttrGet(MlirContext ctx, int32_t mask, int32_t base, int32_t shift); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyAlign(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyAlignAttrGet(MlirContext ctx, int32_t alignment); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyAddressSpace(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyAddressSpaceAttrGet(MlirContext ctx, int32_t addressSpace); + +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyMmaOperandAttrGet(MlirContext ctx, int32_t operand); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyGemmTraversalOrderAttrGet(MlirContext ctx, int32_t order); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyIntTuple(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyIntTupleTypeGet(MlirContext ctx, MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntTupleTypeGetAttr(MlirType type); +MLIR_CAPI_EXPORTED intptr_t mlirFlyIntTupleTypeGetNumElements(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyIntTupleTypeGetElement(MlirType type, intptr_t pos); +MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsLeaf(MlirType type); +MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsStatic(MlirType type); +MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsStaticLeaf(MlirType type); +MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsNoneLeaf(MlirType type); +MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsBasisLeaf(MlirType type); +MLIR_CAPI_EXPORTED int64_t mlirFlyIntTupleTypeGetStaticValue(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyLayout(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyLayoutTypeGet(MlirContext ctx, MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyLayoutTypeGetAttr(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyLayoutTypeGetShape(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyLayoutTypeGetStride(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyComposedLayout(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyComposedLayoutTypeGetAttr(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyTile(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyTileTypeGet(MlirContext ctx, MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyTileTypeGetAttr(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlySwizzle(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlySwizzleTypeGet(MlirContext ctx, MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyPointer(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyPointerTypeGet(MlirContext ctx, MlirType elemTy, MlirAttribute addressSpace, MlirAttribute alignment, MlirAttribute swizzle); +MLIR_CAPI_EXPORTED MlirType mlirFlyPointerTypeGetElemTy(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyPointerTypeGetAddressSpace(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyPointerTypeGetAlignment(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyPointerTypeGetSwizzle(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyMemRef(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyMemRefTypeGet(MlirContext ctx, MlirType elemTy, MlirAttribute addressSpace, MlirAttribute layout, MlirAttribute alignment, MlirAttribute swizzle); +MLIR_CAPI_EXPORTED MlirType mlirFlyMemRefTypeGetElemTy(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyMemRefTypeGetAddressSpace(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyMemRefTypeGetLayout(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyCoordTensor(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyCoordTensorTypeGet(MlirContext ctx, MlirAttribute base, MlirAttribute layout); + +MLIR_CAPI_EXPORTED MlirType mlirFlyLayoutLikeTypeGetShape(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyCopyAtom(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyCopyAtomTypeGet(MlirContext ctx, MlirType copyOp, int32_t valBits); +MLIR_CAPI_EXPORTED MlirType mlirFlyCopyAtomTypeGetThrLayout(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyCopyAtomTypeGetThrValLayoutSrc(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyCopyAtomTypeGetThrValLayoutDst(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyCopyAtomTypeGetThrValLayoutRef(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyMmaAtom(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGet(MlirContext ctx, MlirType mmaOp); +MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetThrLayout(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetShapeMNK(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetThrValLayoutA(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetThrValLayoutB(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetThrValLayoutC(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyTiledCopy(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyTiledCopyTypeGetTiledThrValLayoutSrc(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyTiledCopyTypeGetTiledThrValLayoutDst(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyTiledMma(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyTiledMmaTypeGetTileSizeMNK(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyTiledMmaTypeGetThrLayoutVMNK(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyTiledMmaTypeGetTiledThrValLayoutA(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyTiledMmaTypeGetTiledThrValLayoutB(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyTiledMmaTypeGetTiledThrValLayoutC(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyCopyOpUniversalCopy(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyCopyOpUniversalCopyTypeGet(MlirContext ctx, int32_t bitSize); + +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyROCDLBufferDescAddressAttrGet(MlirContext ctx); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLCopyOpCDNA3BufferCopy(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLCopyOpCDNA3BufferCopyTypeGet(MlirContext ctx, int32_t bitSize, int32_t cacheModifier); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLCopyOpCDNA3BufferCopyLDS(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLCopyOpCDNA3BufferCopyLDSTypeGet(MlirContext ctx, int32_t bitSize); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLMmaOpCDNA3MFMA(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLMmaOpCDNA3MFMATypeGet(MlirContext ctx, int32_t m, int32_t n, int32_t k, MlirType elemTyA, MlirType elemTyB, MlirType elemTyAcc); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLMmaOpGFX11WMMA(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLMmaOpGFX11WMMATypeGet(MlirContext ctx, int32_t m, int32_t n, int32_t k, MlirType elemTyA, MlirType elemTyB, MlirType elemTyAcc, bool signA, bool signB, bool clamp); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLMmaOpGFX120XWMMA(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLMmaOpGFX120XWMMATypeGet(MlirContext ctx, int32_t m, int32_t n, int32_t k, MlirType elemTyA, MlirType elemTyB, MlirType elemTyAcc, bool signA, bool signB, bool clamp); + +#ifdef __cplusplus +} +#endif + +#endif // ZML_MLIR_DIALECTS_FLY_FLY_CAPI_H diff --git a/mlir/dialects/fly/rocdl.zig b/mlir/dialects/fly/rocdl.zig new file mode 100644 index 00000000..a1a19f23 --- /dev/null +++ b/mlir/dialects/fly/rocdl.zig @@ -0,0 +1,164 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); + +/// `#fly_rocdl.buffer_desc`, an address space for buffer-descriptor pointers. +pub const BufferDescAddressAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + + pub fn get(ctx: *mlir.Context) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyROCDLBufferDescAddressAttrGet(ctx.ptr()).ptr orelse return error.InvalidMlir); + } +}; + +/// `!fly_rocdl.cdna3.buffer_copy`. +pub const CopyOpCDNA3BufferCopyType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyROCDLCopyOpCDNA3BufferCopy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + + pub const InitArgs = struct { + bitSize: i32, + /// 0 = cached, 2 = non-temporal. + cacheModifier: i32 = 0, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyROCDLCopyOpCDNA3BufferCopyTypeGet(ctx.ptr(), args.bitSize, args.cacheModifier); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!fly_rocdl.cdna3.buffer_copy_lds`. +pub const CopyOpCDNA3BufferCopyLDSType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyROCDLCopyOpCDNA3BufferCopyLDS; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + + pub const InitArgs = struct { + bitSize: i32, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyROCDLCopyOpCDNA3BufferCopyLDSTypeGet(ctx.ptr(), args.bitSize); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!fly_rocdl.cdna3.mfma`. +pub const MmaOpCDNA3MFMAType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyROCDLMmaOpCDNA3MFMA; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + + pub const InitArgs = struct { + m: i32, + n: i32, + k: i32, + elemTyA: *const mlir.Type, + elemTyB: *const mlir.Type, + elemTyAcc: *const mlir.Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyROCDLMmaOpCDNA3MFMATypeGet(ctx.ptr(), args.m, args.n, args.k, args.elemTyA.ptr(), args.elemTyB.ptr(), args.elemTyAcc.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!fly_rocdl.gfx11.wmma`. +pub const MmaOpGFX11WMMAType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyROCDLMmaOpGFX11WMMA; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + + pub const InitArgs = WmmaInitArgs; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyROCDLMmaOpGFX11WMMATypeGet(ctx.ptr(), args.m, args.n, args.k, args.elemTyA.ptr(), args.elemTyB.ptr(), args.elemTyAcc.ptr(), args.signA, args.signB, args.clamp); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!fly_rocdl.gfx120x.wmma`. +pub const MmaOpGFX120XWMMAType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyROCDLMmaOpGFX120XWMMA; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + + pub const InitArgs = WmmaInitArgs; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyROCDLMmaOpGFX120XWMMATypeGet(ctx.ptr(), args.m, args.n, args.k, args.elemTyA.ptr(), args.elemTyB.ptr(), args.elemTyAcc.ptr(), args.signA, args.signB, args.clamp); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `signA`/`signB`/`clamp` drive the integer WMMA intrinsics and must stay +/// false on the float paths. +pub const WmmaInitArgs = struct { + m: i32, + n: i32, + k: i32, + elemTyA: *const mlir.Type, + elemTyB: *const mlir.Type, + elemTyAcc: *const mlir.Type, + signA: bool = false, + signB: bool = false, + clamp: bool = false, +}; diff --git a/mlir/dialects/fly/types.zig b/mlir/dialects/fly/types.zig new file mode 100644 index 00000000..d974588c --- /dev/null +++ b/mlir/dialects/fly/types.zig @@ -0,0 +1,457 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); + +const attributes = @import("attributes.zig"); + +fn optAttr(value: anytype) c.MlirAttribute { + return if (value) |v| v.ptr() else .{ .ptr = null }; +} + +/// `!fly.int_tuple`. +pub const IntTupleType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyIntTuple; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + + pub const InitArgs = struct { + attr: *const attributes.IntTupleAttr, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyIntTupleTypeGet(ctx.ptr(), args.attr.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn getAttr(self: *const Self) *const attributes.IntTupleAttr { + return @ptrCast(c.mlirFlyIntTupleTypeGetAttr(self.ptr()).ptr.?); + } + + /// 1 for a leaf. + pub fn getNumElements(self: *const Self) usize { + return @intCast(c.mlirFlyIntTupleTypeGetNumElements(self.ptr())); + } + + pub fn getElement(self: *const Self, pos: usize) *const Self { + return @ptrCast(c.mlirFlyIntTupleTypeGetElement(self.ptr(), @intCast(pos)).ptr.?); + } + + pub fn isLeaf(self: *const Self) bool { + return c.mlirFlyIntTupleTypeIsLeaf(self.ptr()); + } + + pub fn isStatic(self: *const Self) bool { + return c.mlirFlyIntTupleTypeIsStatic(self.ptr()); + } + + pub const Leaf = union(enum) { static: i64, dynamic, none, basis }; + + /// Asserts the tuple is a leaf. + pub fn getLeaf(self: *const Self) Leaf { + if (c.mlirFlyIntTupleTypeIsStaticLeaf(self.ptr())) { + return .{ .static = c.mlirFlyIntTupleTypeGetStaticValue(self.ptr()) }; + } + if (c.mlirFlyIntTupleTypeIsNoneLeaf(self.ptr())) return .none; + if (c.mlirFlyIntTupleTypeIsBasisLeaf(self.ptr())) return .basis; + return .dynamic; + } +}; + +/// `!fly.layout`. +pub const LayoutType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + + pub const InitArgs = struct { + attr: *const attributes.LayoutAttr, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyLayoutTypeGet(ctx.ptr(), args.attr.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn getAttr(self: *const Self) *const attributes.LayoutAttr { + return @ptrCast(c.mlirFlyLayoutTypeGetAttr(self.ptr()).ptr.?); + } + + pub fn getShape(self: *const Self) *const IntTupleType { + return @ptrCast(c.mlirFlyLayoutTypeGetShape(self.ptr()).ptr.?); + } + + pub fn getStride(self: *const Self) *const IntTupleType { + return @ptrCast(c.mlirFlyLayoutTypeGetStride(self.ptr()).ptr.?); + } +}; + +/// `!fly.composed_layout`. +pub const ComposedLayoutType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyComposedLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + + pub fn getAttr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirFlyComposedLayoutTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!fly.tile`. +pub const TileType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyTile; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + + pub const InitArgs = struct { + attr: *const attributes.TileAttr, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyTileTypeGet(ctx.ptr(), args.attr.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn getAttr(self: *const Self) *const attributes.TileAttr { + return @ptrCast(c.mlirFlyTileTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!fly.swizzle`. +pub const SwizzleType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlySwizzle; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + + pub const InitArgs = struct { + attr: *const attributes.SwizzleAttr, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirFlySwizzleTypeGet(ctx.ptr(), args.attr.ptr()).ptr orelse return error.InvalidMlir); + } +}; + +/// `!fly.ptr`. +pub const PointerType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyPointer; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + + pub const InitArgs = struct { + elemTy: *const mlir.Type, + addressSpace: *const mlir.Attribute, + alignment: ?*const attributes.AlignAttr = null, + swizzle: ?*const attributes.SwizzleAttr = null, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyPointerTypeGet(ctx.ptr(), args.elemTy.ptr(), args.addressSpace.ptr(), optAttr(args.alignment), optAttr(args.swizzle)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn getElemTy(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirFlyPointerTypeGetElemTy(self.ptr()).ptr.?); + } + + pub fn getAddressSpace(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirFlyPointerTypeGetAddressSpace(self.ptr()).ptr.?); + } + + pub fn getAlignment(self: *const Self) *const attributes.AlignAttr { + return @ptrCast(c.mlirFlyPointerTypeGetAlignment(self.ptr()).ptr.?); + } + + pub fn getSwizzle(self: *const Self) *const attributes.SwizzleAttr { + return @ptrCast(c.mlirFlyPointerTypeGetSwizzle(self.ptr()).ptr.?); + } +}; + +/// `!fly.memref`. +pub const MemRefType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyMemRef; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + + pub const InitArgs = struct { + elemTy: *const mlir.Type, + addressSpace: *const mlir.Attribute, + layout: *const mlir.Attribute, + alignment: ?*const attributes.AlignAttr = null, + swizzle: ?*const attributes.SwizzleAttr = null, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyMemRefTypeGet(ctx.ptr(), args.elemTy.ptr(), args.addressSpace.ptr(), args.layout.ptr(), optAttr(args.alignment), optAttr(args.swizzle)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn getElemTy(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirFlyMemRefTypeGetElemTy(self.ptr()).ptr.?); + } + + pub fn getAddressSpace(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirFlyMemRefTypeGetAddressSpace(self.ptr()).ptr.?); + } + + pub fn getLayout(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirFlyMemRefTypeGetLayout(self.ptr()).ptr.?); + } +}; + +/// `!fly.coord_tensor`. +pub const CoordTensorType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyCoordTensor; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + + pub const InitArgs = struct { + base: *const attributes.IntTupleAttr, + layout: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyCoordTensorTypeGet(ctx.ptr(), args.base.ptr(), args.layout.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!fly.copy_atom`. +pub const CopyAtomType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyCopyAtom; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + + pub const InitArgs = struct { + copyOp: *const mlir.Type, + valBits: i32, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyCopyAtomTypeGet(ctx.ptr(), args.copyOp.ptr(), args.valBits); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn getThrLayout(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyCopyAtomTypeGetThrLayout(self.ptr()).ptr.?); + } + + pub fn getThrValLayoutSrc(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyCopyAtomTypeGetThrValLayoutSrc(self.ptr()).ptr.?); + } + + pub fn getThrValLayoutDst(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyCopyAtomTypeGetThrValLayoutDst(self.ptr()).ptr.?); + } + + pub fn getThrValLayoutRef(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyCopyAtomTypeGetThrValLayoutRef(self.ptr()).ptr.?); + } +}; + +/// `!fly.mma_atom`. +pub const MmaAtomType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyMmaAtom; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + + pub const InitArgs = struct { + mmaOp: *const mlir.Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyMmaAtomTypeGet(ctx.ptr(), args.mmaOp.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn getThrLayout(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyMmaAtomTypeGetThrLayout(self.ptr()).ptr.?); + } + + pub fn getShapeMNK(self: *const Self) *const IntTupleType { + return @ptrCast(c.mlirFlyMmaAtomTypeGetShapeMNK(self.ptr()).ptr.?); + } + + pub fn getThrValLayoutA(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyMmaAtomTypeGetThrValLayoutA(self.ptr()).ptr.?); + } + + pub fn getThrValLayoutB(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyMmaAtomTypeGetThrValLayoutB(self.ptr()).ptr.?); + } + + pub fn getThrValLayoutC(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyMmaAtomTypeGetThrValLayoutC(self.ptr()).ptr.?); + } +}; + +/// `!fly.tiled_copy`. +pub const TiledCopyType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyTiledCopy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + + pub fn getTiledThrValLayoutSrc(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyTiledCopyTypeGetTiledThrValLayoutSrc(self.ptr()).ptr.?); + } + + pub fn getTiledThrValLayoutDst(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyTiledCopyTypeGetTiledThrValLayoutDst(self.ptr()).ptr.?); + } +}; + +/// `!fly.tiled_mma`. +pub const TiledMmaType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyTiledMma; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + + pub fn getTileSizeMNK(self: *const Self) *const IntTupleType { + return @ptrCast(c.mlirFlyTiledMmaTypeGetTileSizeMNK(self.ptr()).ptr.?); + } + + pub fn getThrLayoutVMNK(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyTiledMmaTypeGetThrLayoutVMNK(self.ptr()).ptr.?); + } + + pub fn getTiledThrValLayoutA(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyTiledMmaTypeGetTiledThrValLayoutA(self.ptr()).ptr.?); + } + + pub fn getTiledThrValLayoutB(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyTiledMmaTypeGetTiledThrValLayoutB(self.ptr()).ptr.?); + } + + pub fn getTiledThrValLayoutC(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyTiledMmaTypeGetTiledThrValLayoutC(self.ptr()).ptr.?); + } +}; + +/// `!fly.universal_copy`. +pub const CopyOpUniversalCopyType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyCopyOpUniversalCopy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + + pub const InitArgs = struct { + bitSize: i32, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyCopyOpUniversalCopyTypeGet(ctx.ptr(), args.bitSize); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; diff --git a/mlir/dialects/gpu.zig b/mlir/dialects/gpu.zig new file mode 100644 index 00000000..35538ab2 --- /dev/null +++ b/mlir/dialects/gpu.zig @@ -0,0 +1,89 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub const Dim = enum { x, y, z }; + +pub fn module(ctx: *mlir.Context, name: []const u8, body: *mlir.Block, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "gpu.module", .{ + .blocks = &.{body}, + .attributes = &.{ + .named(ctx, "sym_name", .string(ctx, name)), + }, + .location = location, + .verify = false, + }); +} + +pub const FuncArgs = struct { + name: []const u8, + args: []const *const mlir.Type, + block: *mlir.Block, + location: *const mlir.Location, + kernel: bool = true, + extra_attributes: []const mlir.NamedAttribute = &.{}, +}; + +/// `gpu.func @name(args) kernel { ... gpu.return }`. +pub fn func(ctx: *mlir.Context, args: FuncArgs) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 16) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "sym_name", .string(ctx, args.name)), + .named(ctx, "function_type", .typeAttr(.function(ctx, args.args, &.{}))), + }); + if (args.kernel) attrs.appendAssumeCapacity(.named(ctx, "gpu.kernel", .unit(ctx))); + attrs.appendSliceAssumeCapacity(args.extra_attributes); + return mlir.Operation.make(ctx, "gpu.func", .{ + .blocks = &.{args.block}, + .attributes = attrs.constSlice(), + .location = args.location, + .verify = false, + }); +} + +pub fn return_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "gpu.return", .{ + .operands = .{ .flat = values }, + .location = location, + .verify = false, + }); +} + +fn dimAttr(ctx: *mlir.Context, dim: Dim) *const mlir.Attribute { + return switch (dim) { + inline else => |d| mlir.Attribute.parse(ctx, "#gpu") catch unreachable, + }; +} + +fn idOp(ctx: *mlir.Context, comptime name: []const u8, dim: Dim, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, name, .{ + .results = .{ .flat = &.{.index(ctx)} }, + .attributes = &.{.named(ctx, "dimension", dimAttr(ctx, dim))}, + .location = location, + }); +} + +pub fn thread_id(ctx: *mlir.Context, dim: Dim, location: *const mlir.Location) *mlir.Operation { + return idOp(ctx, "gpu.thread_id", dim, location); +} + +pub fn block_id(ctx: *mlir.Context, dim: Dim, location: *const mlir.Location) *mlir.Operation { + return idOp(ctx, "gpu.block_id", dim, location); +} + +pub fn block_dim(ctx: *mlir.Context, dim: Dim, location: *const mlir.Location) *mlir.Operation { + return idOp(ctx, "gpu.block_dim", dim, location); +} + +pub fn grid_dim(ctx: *mlir.Context, dim: Dim, location: *const mlir.Location) *mlir.Operation { + return idOp(ctx, "gpu.grid_dim", dim, location); +} + +pub fn barrier(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "gpu.barrier", .{ .location = location }); +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/mlir/mlir.h b/mlir/mlir.h index 0effcda9..89043c01 100644 --- a/mlir/mlir.h +++ b/mlir/mlir.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 16d490f0..e87749fe 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -342,6 +342,10 @@ pub const Type = opaque { return @ptrCast(c.mlirTypeParseGet(ctx.ptr(), stringRef(str)).ptr orelse return Error.InvalidMlir); } + pub fn isFloat(self: *const Type) bool { + return c.mlirTypeIsAFloat(self.ptr()); + } + pub fn index(ctx: *Context) *const Type { return @ptrCast(IndexType.get(ctx)); } @@ -463,6 +467,10 @@ pub const IntegerType = opaque { pub const eql = M.eql(c.mlirTypeEqual); pub const format = M.format(c.mlirTypePrint); + pub fn width(self: *const IntegerType) u32 { + return c.mlirIntegerTypeGetWidth(self.ptr()); + } + fn get(ctx: *Context, it: IntegerTypes) *const IntegerType { return exact(ctx, it.bitwidth(), it.signedness()); } diff --git a/third_party/flydsl/BUILD.bazel b/third_party/flydsl/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/third_party/flydsl/atomic_exchange.patch b/third_party/flydsl/atomic_exchange.patch new file mode 100644 index 00000000..7de76db1 --- /dev/null +++ b/third_party/flydsl/atomic_exchange.patch @@ -0,0 +1,25 @@ +diff --git a/include/flydsl/Dialect/Fly/IR/FlyAttrDefs.td b/include/flydsl/Dialect/Fly/IR/FlyAttrDefs.td +--- a/include/flydsl/Dialect/Fly/IR/FlyAttrDefs.td ++++ b/include/flydsl/Dialect/Fly/IR/FlyAttrDefs.td +@@ -43,7 +43,8 @@ + I32EnumAttrCase<"And", 3, "and">, + I32EnumAttrCase<"Or", 4, "or">, + I32EnumAttrCase<"Inc", 5, "inc">, +- I32EnumAttrCase<"Dec", 6, "dec"> ++ I32EnumAttrCase<"Dec", 6, "dec">, ++ I32EnumAttrCase<"Exchange", 7, "exchange"> + ]>; + def Fly_AtomicOpAttr : Fly_EnumAttr {} + +diff --git a/lib/Dialect/Fly/IR/FlyUniversalOps.cpp b/lib/Dialect/Fly/IR/FlyUniversalOps.cpp +--- a/lib/Dialect/Fly/IR/FlyUniversalOps.cpp ++++ b/lib/Dialect/Fly/IR/FlyUniversalOps.cpp +@@ -228,6 +228,8 @@ + return isFloat ? std::nullopt : std::optional(LLVM::AtomicBinOp::uinc_wrap); + case AtomicOp::Dec: + return isFloat ? std::nullopt : std::optional(LLVM::AtomicBinOp::udec_wrap); ++ case AtomicOp::Exchange: ++ return LLVM::AtomicBinOp::xchg; + } + return std::nullopt; + } diff --git a/third_party/flydsl/flydsl.bazel b/third_party/flydsl/flydsl.bazel new file mode 100644 index 00000000..0c06878c --- /dev/null +++ b/third_party/flydsl/flydsl.bazel @@ -0,0 +1,396 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +package( + default_visibility = ["//visibility:public"], + features = [ + "-parse_headers", + "-use_header_modules", + ], + licenses = ["notice"], +) + +exports_files(["LICENSE"]) + +td_library( + name = "td_files", + srcs = glob(["include/flydsl/**/*.td"]), + includes = ["include"], + deps = [ + "@llvm-project//mlir:ArithOpsTdFiles", + "@llvm-project//mlir:BuiltinDialectBytecodeTdFiles", + "@llvm-project//mlir:BytecodeOpInterfaceTdFiles", + "@llvm-project//mlir:FunctionInterfacesTdFiles", + "@llvm-project//mlir:GPUOpsTdFiles", + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:LLVMOpsTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:PassBaseTdFiles", + "@llvm-project//mlir:ROCDLOpsTdFiles", + "@llvm-project//mlir:SCFTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + "@llvm-project//mlir:VectorOpsTdFiles", + ], +) + +gentbl_cc_library( + name = "fly_dialect_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/IR/FlyDialect.cpp.inc": ["--gen-dialect-defs"], + "include/flydsl/Dialect/Fly/IR/FlyDialect.h.inc": ["--gen-dialect-decls"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/IR/FlyDialect.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_interfaces_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/IR/FlyAttrInterfaces.cpp.inc": ["--gen-attr-interface-defs"], + "include/flydsl/Dialect/Fly/IR/FlyAttrInterfaces.h.inc": ["--gen-attr-interface-decls"], + "include/flydsl/Dialect/Fly/IR/FlyOpInterfaces.cpp.inc": ["--gen-op-interface-defs"], + "include/flydsl/Dialect/Fly/IR/FlyOpInterfaces.h.inc": ["--gen-op-interface-decls"], + "include/flydsl/Dialect/Fly/IR/FlyTypeInterfaces.cpp.inc": ["--gen-type-interface-defs"], + "include/flydsl/Dialect/Fly/IR/FlyTypeInterfaces.h.inc": ["--gen-type-interface-decls"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/IR/FlyInterfaces.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_types_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/IR/FlyTypeConstraints.cpp.inc": [ + "--gen-type-constraint-defs", + "--typedefs-dialect=fly", + ], + "include/flydsl/Dialect/Fly/IR/FlyTypeConstraints.h.inc": [ + "--gen-type-constraint-decls", + "--typedefs-dialect=fly", + ], + "include/flydsl/Dialect/Fly/IR/FlyTypeDefs.cpp.inc": [ + "--gen-typedef-defs", + "--typedefs-dialect=fly", + ], + "include/flydsl/Dialect/Fly/IR/FlyTypeDefs.h.inc": [ + "--gen-typedef-decls", + "--typedefs-dialect=fly", + ], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/IR/FlyTypeDefs.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_attrs_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/IR/FlyAttrConstraints.cpp.inc": [ + "--gen-attr-constraint-defs", + "--attrdefs-dialect=fly", + ], + "include/flydsl/Dialect/Fly/IR/FlyAttrConstraints.h.inc": [ + "--gen-attr-constraint-decls", + "--attrdefs-dialect=fly", + ], + "include/flydsl/Dialect/Fly/IR/FlyAttrDefs.cpp.inc": [ + "--gen-attrdef-defs", + "--attrdefs-dialect=fly", + ], + "include/flydsl/Dialect/Fly/IR/FlyAttrDefs.h.inc": [ + "--gen-attrdef-decls", + "--attrdefs-dialect=fly", + ], + "include/flydsl/Dialect/Fly/IR/FlyEnums.cpp.inc": ["--gen-enum-defs"], + "include/flydsl/Dialect/Fly/IR/FlyEnums.h.inc": ["--gen-enum-decls"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/IR/FlyAttrDefs.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_ops_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/IR/FlyOps.cpp.inc": ["--gen-op-defs"], + "include/flydsl/Dialect/Fly/IR/FlyOps.h.inc": ["--gen-op-decls"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/IR/FlyOps.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_passes_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/Transforms/Passes.h.inc": [ + "--gen-pass-decls", + "--name=Fly", + ], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/Transforms/Passes.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_int_tuple_lowering_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/Transforms/IntTupleLowering.cpp.inc": ["--gen-rewriters"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/Transforms/IntTupleLowering.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_layout_lowering_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/Transforms/LayoutLowering.cpp.inc": ["--gen-rewriters"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/Transforms/LayoutLowering.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_memref_lowering_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/Transforms/MemrefLowering.cpp.inc": ["--gen-rewriters"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/Transforms/MemrefLowering.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_rocdl_dialect_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/FlyROCDL/IR/Dialect.cpp.inc": ["--gen-dialect-defs"], + "include/flydsl/Dialect/FlyROCDL/IR/Dialect.h.inc": ["--gen-dialect-decls"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/FlyROCDL/IR/Dialect.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_rocdl_atom_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/FlyROCDL/IR/Atom.cpp.inc": [ + "--gen-typedef-defs", + "--typedefs-dialect=fly_rocdl", + ], + "include/flydsl/Dialect/FlyROCDL/IR/Atom.h.inc": [ + "--gen-typedef-decls", + "--typedefs-dialect=fly_rocdl", + ], + "include/flydsl/Dialect/FlyROCDL/IR/AtomStateEnums.cpp.inc": [ + "--gen-enum-defs", + "--typedefs-dialect=fly_rocdl", + ], + "include/flydsl/Dialect/FlyROCDL/IR/AtomStateEnums.h.inc": [ + "--gen-enum-decls", + "--typedefs-dialect=fly_rocdl", + ], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/FlyROCDL/IR/Atom.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_rocdl_attrs_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/FlyROCDL/IR/AttrDefs.cpp.inc": [ + "--gen-attrdef-defs", + "--attrdefs-dialect=fly_rocdl", + ], + "include/flydsl/Dialect/FlyROCDL/IR/AttrDefs.h.inc": [ + "--gen-attrdef-decls", + "--attrdefs-dialect=fly_rocdl", + ], + "include/flydsl/Dialect/FlyROCDL/IR/AttrEnums.cpp.inc": [ + "--gen-enum-defs", + "--attrdefs-dialect=fly_rocdl", + ], + "include/flydsl/Dialect/FlyROCDL/IR/AttrEnums.h.inc": [ + "--gen-enum-decls", + "--attrdefs-dialect=fly_rocdl", + ], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/FlyROCDL/IR/AttrDefs.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_rocdl_ops_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/FlyROCDL/IR/Ops.cpp.inc": ["--gen-op-defs"], + "include/flydsl/Dialect/FlyROCDL/IR/Ops.h.inc": ["--gen-op-decls"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/FlyROCDL/IR/Ops.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_to_rocdl_passes_inc_gen", + tbl_outs = { + "include/flydsl/Conversion/FlyToROCDL/Passes.h.inc": [ + "--gen-pass-decls", + "--name=FlyToROCDL", + ], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Conversion/FlyToROCDL/Passes.td", + deps = [":td_files"], +) + +cc_library( + name = "FlyDialect", + srcs = [ + "lib/Dialect/Fly/IR/FlyAttrDefs.cpp", + "lib/Dialect/Fly/IR/FlyDialect.cpp", + "lib/Dialect/Fly/IR/FlyOps.cpp", + "lib/Dialect/Fly/IR/FlyTypeDefs.cpp", + "lib/Dialect/Fly/IR/FlyUniversalOps.cpp", + "lib/Dialect/Fly/Utils/IntTupleUtils.cpp", + "lib/Dialect/Fly/Utils/IntUtils.cpp", + "lib/Dialect/Fly/Utils/NormalForm.cpp", + "lib/Dialect/Fly/Utils/PointerUtils.cpp", + ], + hdrs = glob( + [ + "include/flydsl/Dialect/Fly/IR/*.h", + "include/flydsl/Dialect/Fly/Utils/*.h", + "include/flydsl/Dialect/Fly/Utils/*.inc", + ], + exclude = ["include/flydsl/Dialect/Fly/IR/FlyLLVMTranslation.h"], + ), + includes = ["include"], + deps = [ + ":fly_attrs_inc_gen", + ":fly_dialect_inc_gen", + ":fly_interfaces_inc_gen", + ":fly_ops_inc_gen", + ":fly_types_inc_gen", + "@llvm-project//llvm:Support", + "@llvm-project//llvm:ir_headers", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:BytecodeOpInterface", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:GPUDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:SideEffectInterfaces", + ], +) + +cc_library( + name = "FlyTransforms", + srcs = glob(["lib/Dialect/Fly/Transforms/*.cpp"]), + hdrs = glob(["include/flydsl/Dialect/Fly/Transforms/*.h"]), + includes = ["include"], + deps = [ + ":FlyDialect", + ":fly_int_tuple_lowering_inc_gen", + ":fly_layout_lowering_inc_gen", + ":fly_memref_lowering_inc_gen", + ":fly_passes_inc_gen", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:ArithUtils", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:FuncTransforms", + "@llvm-project//mlir:GPUDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:SCFTransforms", + "@llvm-project//mlir:Transforms", + "@llvm-project//mlir:UBDialect", + "@llvm-project//mlir:VectorDialect", + ], +) + +cc_library( + name = "FlyROCDLDialect", + srcs = [ + "lib/Dialect/FlyROCDL/CDNA3/CopyAtom.cpp", + "lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp", + "lib/Dialect/FlyROCDL/CDNA4/CopyAtom.cpp", + "lib/Dialect/FlyROCDL/CDNA4/MmaAtom.cpp", + "lib/Dialect/FlyROCDL/Dialect.cpp", + "lib/Dialect/FlyROCDL/GFX11/MmaAtom.cpp", + "lib/Dialect/FlyROCDL/GFX120X/MmaAtom.cpp", + "lib/Dialect/FlyROCDL/GFX1250/CopyAtom.cpp", + "lib/Dialect/FlyROCDL/GFX1250/MmaAtom.cpp", + "lib/Dialect/FlyROCDL/GFX1250/MmaAtomScale.cpp", + "lib/Dialect/FlyROCDL/Ops.cpp", + ], + hdrs = glob([ + "include/flydsl/Dialect/FlyROCDL/**/*.h", + "lib/Dialect/FlyROCDL/**/*.h", + ]), + includes = ["include"], + deps = [ + ":FlyDialect", + ":fly_rocdl_atom_inc_gen", + ":fly_rocdl_attrs_inc_gen", + ":fly_rocdl_dialect_inc_gen", + ":fly_rocdl_ops_inc_gen", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:ROCDLDialect", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:VectorDialect", + ], +) + +cc_library( + name = "FlyToROCDL", + srcs = ["lib/Conversion/FlyToROCDL/FlyToROCDL.cpp"], + hdrs = glob(["include/flydsl/Conversion/**/*.h"]), + includes = ["include"], + deps = [ + ":FlyDialect", + ":FlyROCDLDialect", + ":fly_to_rocdl_passes_inc_gen", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:ArithUtils", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:GPUDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:ROCDLDialect", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:Transforms", + "@llvm-project//mlir:VectorDialect", + ], +) + +# ZML addition: the per-type accessors FlyDSL's Python bindings use to derive +# tiled TV layouts and MMA tile sizes. They live under lib/Bindings/Python but +# depend only on the dialect (no nanobind). +cc_library( + name = "TiledOpTraits", + srcs = ["lib/Bindings/Python/TiledOpTraits.cpp"], + hdrs = ["lib/Bindings/Python/TiledOpTraits.h"], + includes = ["lib/Bindings/Python"], + deps = [ + ":FlyDialect", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + ], +) diff --git a/third_party/flydsl/mlir_property_ref.patch b/third_party/flydsl/mlir_property_ref.patch new file mode 100644 index 00000000..3833fb78 --- /dev/null +++ b/third_party/flydsl/mlir_property_ref.patch @@ -0,0 +1,384 @@ +diff --git a/lib/Dialect/Fly/IR/FlyOps.cpp b/lib/Dialect/Fly/IR/FlyOps.cpp +--- a/lib/Dialect/Fly/IR/FlyOps.cpp ++++ b/lib/Dialect/Fly/IR/FlyOps.cpp +@@ -192,7 +192,7 @@ + llvm::LogicalResult OP::inferReturnTypes( \ + mlir::MLIRContext *context, std::optional<::mlir::Location> location, \ + mlir::ValueRange operands, mlir::DictionaryAttr attributes, \ +- mlir::OpaqueProperties properties, mlir::RegionRange regions, \ ++ mlir::PropertyRef properties, mlir::RegionRange regions, \ + llvm::SmallVectorImpl &inferredReturnTypes) + + //===----------------------------------------------------------------------===// +diff --git a/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp b/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp +--- a/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp ++++ b/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp +@@ -10,6 +10,7 @@ + #include "mlir/Dialect/SCF/IR/SCF.h" + #include "mlir/Dialect/Vector/IR/VectorOps.h" + #include "mlir/IR/BuiltinAttributes.h" ++#include "mlir/IR/BuiltinOps.h" + #include "mlir/IR/BuiltinTypes.h" + #include "mlir/Pass/Pass.h" + #include "mlir/Transforms/DialectConversion.h" +@@ -63,10 +64,10 @@ + // chosen to avoid collisions with existing globals in the module. Shared by + // the static-LDS `make_ptr` and dynamic-LDS `get_dyn_shared` lowerings. + static LLVM::GlobalOp createLDSGlobal(ConversionPatternRewriter &rewriter, +- gpu::GPUModuleOp moduleOp, Location loc, StringRef prefix, ++ Block *moduleBody, Location loc, StringRef prefix, + int64_t nbytes, int64_t align, unsigned addrSpace) { + llvm::StringSet<> existingNames; +- for (auto globalOp : moduleOp.getBody()->getOps()) ++ for (auto globalOp : moduleBody->getOps()) + existingNames.insert(globalOp.getSymName()); + + unsigned counter = 0; +@@ -74,7 +75,7 @@ + prefix, [&](StringRef candidate) { return existingNames.contains(candidate); }, counter); + + OpBuilder::InsertionGuard guard(rewriter); +- rewriter.setInsertionPointToStart(moduleOp.getBody()); ++ rewriter.setInsertionPointToStart(moduleBody); + + auto arrayTy = LLVM::LLVMArrayType::get(IntegerType::get(rewriter.getContext(), 8), nbytes); + auto globalOp = LLVM::GlobalOp::create(rewriter, loc, arrayTy, +@@ -127,12 +128,17 @@ + return rewriter.notifyMatchFailure( + op, "shared make_ptr requires allocBytes, allocAlign in dictAttrs"); + +- auto moduleOp = op->getParentOfType(); +- if (!moduleOp) ++ Block *moduleBody = nullptr; ++ if (auto gpuModule = op->getParentOfType()) { ++ moduleBody = gpuModule.getBody(); ++ } else if (auto builtinModule = op->getParentOfType()) { ++ moduleBody = builtinModule.getBody(); ++ } ++ if (!moduleBody) + return op->emitError("shared make_ptr must be inside a gpu.module"); + + LLVM::GlobalOp global = +- createLDSGlobal(rewriter, moduleOp, loc, "__shared_alloc", allocBytesAttr.getInt(), ++ createLDSGlobal(rewriter, moduleBody, loc, "__shared_alloc", allocBytesAttr.getInt(), + allocAlignAttr.getInt(), /*addrSpace=*/3); + rewriter.replaceOpWithNewOp(op, global); + return success(); +@@ -169,15 +175,20 @@ + auto flyPtrTy = cast(op.getResult().getType()); + unsigned addrSpace = mapAttrToLLVMAddressSpace(flyPtrTy.getAddressSpace()); + +- auto moduleOp = op->getParentOfType(); +- if (!moduleOp) ++ Block *moduleBody = nullptr; ++ if (auto gpuModule = op->getParentOfType()) { ++ moduleBody = gpuModule.getBody(); ++ } else if (auto builtinModule = op->getParentOfType()) { ++ moduleBody = builtinModule.getBody(); ++ } ++ if (!moduleBody) + return op->emitError("get_dyn_shared must be inside a gpu.module"); + + // Dynamic shared memory has a single logical region per kernel; reuse an + // existing `[0 x i8]` global if one is already present so multiple + // `get_dyn_shared` ops resolve to the same base symbol. + LLVM::GlobalOp sharedGlobal; +- for (auto globalOp : moduleOp.getBody()->getOps()) { ++ for (auto globalOp : moduleBody->getOps()) { + if (auto arrayType = dyn_cast(globalOp.getType())) { + if (globalOp.getAddrSpace() == addrSpace && arrayType.getNumElements() == 0 && + globalOp.getAlignment().value_or(0) == 1024) { +@@ -187,7 +198,7 @@ + } + } + if (!sharedGlobal) { +- sharedGlobal = createLDSGlobal(rewriter, moduleOp, loc, "__dynamic_shared", ++ sharedGlobal = createLDSGlobal(rewriter, moduleBody, loc, "__dynamic_shared", + /*nbytes=*/0, /*align=*/1024, addrSpace); + } + +@@ -402,10 +413,11 @@ + if (isTargetAddressSpace(flyPtrTy.getAddressSpace())) { + BufferFatPtr bp(flyPtrTy, ptr); + Value zero = arith::ConstantIntOp::create(rewriter, loc, 0, 32); ++ Attribute aux = rewriter.getI32IntegerAttr(0); + ArrayAttr noAttrs; + Value loaded = ROCDL::RawPtrBufferLoadOp::create( + rewriter, loc, loadTy, bp.bufferRsrc(rewriter, loc), bp.swizzleByteOffset(rewriter, loc), +- zero, zero, noAttrs, noAttrs, noAttrs); ++ zero, aux, noAttrs, noAttrs, noAttrs); + rewriter.replaceOp(op, loaded); + return success(); + } else { +@@ -447,9 +459,10 @@ + if (isTargetAddressSpace(flyPtrTy.getAddressSpace())) { + BufferFatPtr bp(flyPtrTy, ptr); + Value zero = arith::ConstantIntOp::create(rewriter, loc, 0, 32); ++ Attribute aux = rewriter.getI32IntegerAttr(0); + ArrayAttr noAttrs; + ROCDL::RawPtrBufferStoreOp::create(rewriter, loc, value, bp.bufferRsrc(rewriter, loc), +- bp.swizzleByteOffset(rewriter, loc), zero, zero, noAttrs, ++ bp.swizzleByteOffset(rewriter, loc), zero, aux, noAttrs, + noAttrs, noAttrs); + rewriter.eraseOp(op); + return success(); +@@ -747,10 +760,6 @@ + if (Value tok = op.getAsyncToken()) + asyncTokenType = tok.getType(); + +- // There are two relevant builder signatures in this MLIR: +- // - (kernel, ..., asyncTokenType, asyncDependencies, clusterSize) +- // - (kernel, ..., asyncObject, clusterSize) +- // Pick the one that matches the original op structure. + if (Value asyncObj = adaptor.getAsyncObject()) { + if (!adaptor.getAsyncDependencies().empty()) + return rewriter.notifyMatchFailure( +@@ -758,13 +767,14 @@ + + rewriter.replaceOpWithNewOp( + op, kernelRef, grid, block, adaptor.getDynamicSharedMemorySize(), +- adaptor.getKernelOperands(), asyncObj, clusterSize); ++ adaptor.getKernelOperands(), asyncTokenType, ValueRange{}, asyncObj, clusterSize); + return success(); + } + + rewriter.replaceOpWithNewOp( + op, kernelRef, grid, block, adaptor.getDynamicSharedMemorySize(), +- adaptor.getKernelOperands(), asyncTokenType, adaptor.getAsyncDependencies(), clusterSize); ++ adaptor.getKernelOperands(), asyncTokenType, adaptor.getAsyncDependencies(), ++ /*asyncObject=*/nullptr, clusterSize); + return success(); + } + }; +diff --git a/lib/Dialect/Fly/Transforms/PromoteRegMemToVectorSSA.cpp b/lib/Dialect/Fly/Transforms/PromoteRegMemToVectorSSA.cpp +--- a/lib/Dialect/Fly/Transforms/PromoteRegMemToVectorSSA.cpp ++++ b/lib/Dialect/Fly/Transforms/PromoteRegMemToVectorSSA.cpp +@@ -30,11 +30,6 @@ + template <> struct DenseMapInfo : DenseMapInfo { + using Base = DenseMapInfo; + +- static mlir::fly::MakePtrOp getEmptyKey() { return mlir::fly::MakePtrOp(Base::getEmptyKey()); } +- +- static mlir::fly::MakePtrOp getTombstoneKey() { +- return mlir::fly::MakePtrOp(Base::getTombstoneKey()); +- } + }; + + } // namespace llvm +diff --git a/lib/Dialect/FlyROCDL/CDNA3/CopyAtom.cpp b/lib/Dialect/FlyROCDL/CDNA3/CopyAtom.cpp +--- a/lib/Dialect/FlyROCDL/CDNA3/CopyAtom.cpp ++++ b/lib/Dialect/FlyROCDL/CDNA3/CopyAtom.cpp +@@ -89,7 +89,7 @@ + }; + + // raw buffer load/store cachepolicy (0=cached, 2=nt) +- Value aux = arith::ConstantIntOp::create(builder, loc, getCacheModifier(), 32); ++ Attribute aux = builder.getI32IntegerAttr(getCacheModifier()); + ArrayAttr noAttrs; + + auto srcMemTy = srcTyArg ? dyn_cast(srcTyArg) : fly::MemRefType(); +@@ -313,7 +313,7 @@ + soffset = arith::DivUIOp::create(builder, loc, bits, eight); + } + +- Value zero = arith::ConstantIntOp::create(builder, loc, 0, 32); ++ Attribute aux = builder.getI32IntegerAttr(0); + Value size = arith::ConstantIntOp::create(builder, loc, sizeBytes, 32); + + BufferFatPtr bp(srcMemTy.getPointerType(), src); +@@ -322,7 +322,7 @@ + + ArrayAttr noAttrs; + ROCDL::RawPtrBufferLoadLdsOp::create(builder, loc, srcRsrc, dst, size, srcOff, soffset, immOffset, +- zero, noAttrs, noAttrs, noAttrs); ++ aux, noAttrs, noAttrs, noAttrs); + return success(); + } + +@@ -428,7 +428,7 @@ + soffset = arith::DivUIOp::create(builder, loc, bits, eight); + } + +- Value zero = arith::ConstantIntOp::create(builder, loc, 0, 32); ++ Attribute aux = builder.getI32IntegerAttr(0); + ArrayAttr noAttrs; + + AtomicOp op = getAtomicOp().getValue(); +@@ -437,21 +437,21 @@ + case AtomicOp::Add: + if (!isFloat) + return failure(); +- ROCDL::RawPtrBufferAtomicFaddOp::create(builder, loc, src, dstRsrc, dstOff, soffset, zero, ++ ROCDL::RawPtrBufferAtomicFaddOp::create(builder, loc, src.getType(), src, dstRsrc, dstOff, soffset, aux, + noAttrs, noAttrs, noAttrs); + break; + case AtomicOp::Max: + if (isFloat) +- ROCDL::RawPtrBufferAtomicFmaxOp::create(builder, loc, src, dstRsrc, dstOff, soffset, zero, ++ ROCDL::RawPtrBufferAtomicFmaxOp::create(builder, loc, src.getType(), src, dstRsrc, dstOff, soffset, aux, + noAttrs, noAttrs, noAttrs); + else +- ROCDL::RawPtrBufferAtomicSmaxOp::create(builder, loc, src, dstRsrc, dstOff, soffset, zero, ++ ROCDL::RawPtrBufferAtomicSmaxOp::create(builder, loc, src.getType(), src, dstRsrc, dstOff, soffset, aux, + noAttrs, noAttrs, noAttrs); + break; + case AtomicOp::Min: + if (isFloat) + return failure(); +- ROCDL::RawPtrBufferAtomicUminOp::create(builder, loc, src, dstRsrc, dstOff, soffset, zero, ++ ROCDL::RawPtrBufferAtomicUminOp::create(builder, loc, src.getType(), src, dstRsrc, dstOff, soffset, aux, + noAttrs, noAttrs, noAttrs); + break; + default: +diff --git a/lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp b/lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp +--- a/lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp ++++ b/lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp +@@ -169,8 +169,8 @@ + + #define DISPATCH_MFMA_SSA(M_, K_, PRED, OP) \ + if (m == M_ && n == M_ && k == K_ && (PRED)) { \ +- auto zeroAttr = builder.getI32IntegerAttr(0); \ +- return ROCDL::OP::create(builder, loc, accTy, a, b, c, zeroAttr, zeroAttr, zeroAttr) \ ++ return ROCDL::OP::create(builder, loc, accTy, a, b, c, 0, 0, \ ++ ROCDL::MFMAPermB::none) \ + .getResult(); \ + } + +diff --git a/lib/Dialect/FlyROCDL/Ops.cpp b/lib/Dialect/FlyROCDL/Ops.cpp +--- a/lib/Dialect/FlyROCDL/Ops.cpp ++++ b/lib/Dialect/FlyROCDL/Ops.cpp +@@ -15,7 +15,7 @@ + LogicalResult GetBufferRsrcOp::inferReturnTypes(MLIRContext *context, + std::optional location, + ValueRange operands, DictionaryAttr attributes, +- OpaqueProperties properties, RegionRange regions, ++ PropertyRef properties, RegionRange regions, + SmallVectorImpl &inferredReturnTypes) { + auto ptrTy = dyn_cast(operands[0].getType()); + if (!ptrTy) +diff --git a/lib/Dialect/FlyROCDL/GFX1250/MmaAtom.cpp b/lib/Dialect/FlyROCDL/GFX1250/MmaAtom.cpp +--- a/lib/Dialect/FlyROCDL/GFX1250/MmaAtom.cpp ++++ b/lib/Dialect/FlyROCDL/GFX1250/MmaAtom.cpp +@@ -239,14 +239,12 @@ + Value res; + if constexpr (Variant == WmmaVariant::ModsAllReuse) { + // Float path: no sign/clamp operands. +- res = WmmaOp::create(builder, loc, accTy, +- /*signA=*/false, a, /*signB=*/false, b, +- /*modC=*/(uint16_t)0, c) ++ res = WmmaOp::create(builder, loc, accTy, a, b, ROCDL::WMMACModifier::none, c, ++ /*reuseA=*/false, /*reuseB=*/false) + .getResult(); + } else if constexpr (Variant == WmmaVariant::ModsC) { + // fp8 path: no sign/clamp operands. +- res = WmmaOp::create(builder, loc, accTy, a, b, +- /*modC=*/(uint16_t)0, c, ++ res = WmmaOp::create(builder, loc, accTy, a, b, ROCDL::WMMACModifier::none, c, + /*reuseA=*/false, /*reuseB=*/false) + .getResult(); + } else if constexpr (Variant == WmmaVariant::ModsABClamp) { +diff --git a/lib/Dialect/FlyROCDL/GFX1250/MmaAtomScale.cpp b/lib/Dialect/FlyROCDL/GFX1250/MmaAtomScale.cpp +--- a/lib/Dialect/FlyROCDL/GFX1250/MmaAtomScale.cpp ++++ b/lib/Dialect/FlyROCDL/GFX1250/MmaAtomScale.cpp +@@ -244,15 +244,20 @@ + // fp4-only form; no fmtA/fmtB operands. + if (block16) + return ROCDL::wmma_scale16_f32_32x16x128_f4::create( +- builder, loc, accTy, a, b, /*modC=*/(uint16_t)getModC(), c, +- /*scaleAType=*/(uint32_t)getOpselA(), /*fmtScaleA=*/(uint32_t)0, scaleA, +- /*scaleBType=*/(uint32_t)getOpselB(), /*fmtScaleB=*/(uint32_t)0, scaleB, ++ builder, loc, accTy, a, b, ++ static_cast(getModC()), c, ++ static_cast(getOpselA()), ++ ROCDL::WMMAMatrixScaleFormat::e8, scaleA, ++ static_cast(getOpselB()), ++ ROCDL::WMMAMatrixScaleFormat::e8, scaleB, + /*reuseA=*/getReuseA(), /*reuseB=*/getReuseB()) + .getResult(); + return ROCDL::wmma_scale_f32_32x16x128_f4::create( +- builder, loc, accTy, a, b, /*modC=*/(uint16_t)getModC(), c, +- /*scaleAType=*/(uint32_t)getOpselA(), /*fmtScaleA=*/(uint32_t)0, scaleA, +- /*scaleBType=*/(uint32_t)getOpselB(), /*fmtScaleB=*/(uint32_t)0, scaleB, ++ builder, loc, accTy, a, b, static_cast(getModC()), c, ++ static_cast(getOpselA()), ++ ROCDL::WMMAMatrixScaleFormat::e8, scaleA, ++ static_cast(getOpselB()), ++ ROCDL::WMMAMatrixScaleFormat::e8, scaleB, + /*reuseA=*/getReuseA(), /*reuseB=*/getReuseB()) + .getResult(); + } +@@ -264,17 +269,23 @@ + + if (block16) + return ROCDL::wmma_scale16_f32_16x16x128_f8f6f4::create( +- builder, loc, accTy, /*fmtA=*/*aFmt, a, /*fmtB=*/*bFmt, b, +- /*modC=*/(uint16_t)getModC(), c, +- /*scaleAType=*/(uint32_t)getOpselA(), /*fmtScaleA=*/(uint32_t)0, scaleA, +- /*scaleBType=*/(uint32_t)getOpselB(), /*fmtScaleB=*/(uint32_t)0, scaleB, ++ builder, loc, accTy, static_cast(*aFmt), a, ++ static_cast(*bFmt), b, ++ static_cast(getModC()), c, ++ static_cast(getOpselA()), ++ ROCDL::WMMAMatrixScaleFormat::e8, scaleA, ++ static_cast(getOpselB()), ++ ROCDL::WMMAMatrixScaleFormat::e8, scaleB, + /*reuseA=*/getReuseA(), /*reuseB=*/getReuseB()) + .getResult(); + return ROCDL::wmma_scale_f32_16x16x128_f8f6f4::create( +- builder, loc, accTy, /*fmtA=*/*aFmt, a, /*fmtB=*/*bFmt, b, +- /*modC=*/(uint16_t)getModC(), c, +- /*scaleAType=*/(uint32_t)getOpselA(), /*fmtScaleA=*/(uint32_t)0, scaleA, +- /*scaleBType=*/(uint32_t)getOpselB(), /*fmtScaleB=*/(uint32_t)0, scaleB, ++ builder, loc, accTy, static_cast(*aFmt), a, ++ static_cast(*bFmt), b, ++ static_cast(getModC()), c, ++ static_cast(getOpselA()), ++ ROCDL::WMMAMatrixScaleFormat::e8, scaleA, ++ static_cast(getOpselB()), ++ ROCDL::WMMAMatrixScaleFormat::e8, scaleB, + /*reuseA=*/getReuseA(), /*reuseB=*/getReuseB()) + .getResult(); + } +diff --git a/lib/Dialect/FlyROCDL/CDNA4/MmaAtom.cpp b/lib/Dialect/FlyROCDL/CDNA4/MmaAtom.cpp +--- a/lib/Dialect/FlyROCDL/CDNA4/MmaAtom.cpp ++++ b/lib/Dialect/FlyROCDL/CDNA4/MmaAtom.cpp +@@ -237,21 +237,17 @@ + Value scaleB = LLVM::ExtractValueOp::create( + builder, loc, atomVal, ArrayRef{*getFieldIndex(AtomStateField::ScaleB)}); + +- auto cbszAttr = builder.getI32IntegerAttr(*aTypeCode); +- auto blgpAttr = builder.getI32IntegerAttr(*bTypeCode); +- auto opselAAttr = builder.getI32IntegerAttr(getOpselA()); +- auto opselBAttr = builder.getI32IntegerAttr(getOpselB()); ++ auto cbsz = static_cast(*aTypeCode); ++ auto blgp = static_cast(*bTypeCode); + + if (m == 16 && n == 16 && k == 128) { +- return ROCDL::mfma_scale_f32_16x16x128_f8f6f4::create(builder, loc, accTy, a, b, c, cbszAttr, +- blgpAttr, opselAAttr, scaleA, opselBAttr, +- scaleB) ++ return ROCDL::mfma_scale_f32_16x16x128_f8f6f4::create( ++ builder, loc, accTy, a, b, c, cbsz, blgp, getOpselA(), scaleA, getOpselB(), scaleB) + .getResult(); + } + if (m == 32 && n == 32 && k == 64) { +- return ROCDL::mfma_scale_f32_32x32x64_f8f6f4::create(builder, loc, accTy, a, b, c, cbszAttr, +- blgpAttr, opselAAttr, scaleA, opselBAttr, +- scaleB) ++ return ROCDL::mfma_scale_f32_32x32x64_f8f6f4::create( ++ builder, loc, accTy, a, b, c, cbsz, blgp, getOpselA(), scaleA, getOpselB(), scaleB) + .getResult(); + } + +diff --git a/lib/Dialect/FlyROCDL/GFX1250/CopyAtom.cpp b/lib/Dialect/FlyROCDL/GFX1250/CopyAtom.cpp +--- a/lib/Dialect/FlyROCDL/GFX1250/CopyAtom.cpp ++++ b/lib/Dialect/FlyROCDL/GFX1250/CopyAtom.cpp +@@ -503,7 +503,7 @@ + vector::FromElementsOp::create(builder, loc, VectorType::get({8}, builder.getI32Type()), + ValueRange{zero, zero, zero, zero, zero, zero, zero, zero}); + +- uint32_t cachePolicy = static_cast(getCacheModifier()); ++ Attribute cachePolicy = builder.getI32IntegerAttr(getCacheModifier()); + ArrayAttr noAliasScopes; + if (isLoad) + ROCDL::TensorLoadToLDSOp::create(builder, loc, dgroup0, dgroup1, dg2, dg3, dg4, cachePolicy, diff --git a/third_party/flydsl/repo.bzl b/third_party/flydsl/repo.bzl new file mode 100644 index 00000000..8a8c3be2 --- /dev/null +++ b/third_party/flydsl/repo.bzl @@ -0,0 +1,16 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + git_repository( + name = "flydsl", + remote = "https://github.com/ROCm/FlyDSL.git", + commit = "c62159d0c18e232794a1903f192fc094148f4a43", + build_file = "//third_party/flydsl:flydsl.bazel", + patches = [ + "//third_party/flydsl:atomic_exchange.patch", + "//third_party/flydsl:mlir_property_ref.patch", + "//third_party/flydsl:vector_atomic.patch", + "//third_party/flydsl:xf32_mfma.patch", + ], + patch_args = ["-p1"], + ) diff --git a/third_party/flydsl/vector_atomic.patch b/third_party/flydsl/vector_atomic.patch new file mode 100644 index 00000000..edd23c40 --- /dev/null +++ b/third_party/flydsl/vector_atomic.patch @@ -0,0 +1,44 @@ +diff --git a/lib/Dialect/Fly/IR/FlyUniversalOps.cpp b/lib/Dialect/Fly/IR/FlyUniversalOps.cpp +--- a/lib/Dialect/Fly/IR/FlyUniversalOps.cpp ++++ b/lib/Dialect/Fly/IR/FlyUniversalOps.cpp +@@ -35,28 +35,36 @@ + return FxLayout(FxShape(FxC(1), FxC(getBitSize())), FxStride(FxC(1), FxC(1))); + } + ++static int32_t getAtomicValueBitWidth(Type type) { ++ if (auto vectorType = dyn_cast(type)) { ++ return vectorType.getNumElements() * ++ vectorType.getElementType().getIntOrFloatBitWidth(); ++ } ++ return type.getIntOrFloatBitWidth(); ++} ++ + bool CopyOpUniversalAtomicType::isStatic() const { return true; } + + Value CopyOpUniversalAtomicType::rebuildStaticValue(OpBuilder &builder, Location loc, + Value currentValue) const { + if (currentValue && isa(currentValue.getDefiningOp())) + return nullptr; +- int32_t bits = getValType().getIntOrFloatBitWidth(); ++ int32_t bits = getAtomicValueBitWidth(getValType()); + return MakeCopyAtomOp::create(builder, loc, CopyAtomType::get(*this, bits), bits); + } + + Attribute CopyOpUniversalAtomicType::getThrLayout() const { return FxLayout(FxC(1), FxC(1)); } + + Attribute CopyOpUniversalAtomicType::getThrBitLayoutSrc() const { +- int32_t bits = getValType().getIntOrFloatBitWidth(); ++ int32_t bits = getAtomicValueBitWidth(getValType()); + return FxLayout(FxShape(FxC(1), FxC(bits)), FxStride(FxC(1), FxC(1))); + } + Attribute CopyOpUniversalAtomicType::getThrBitLayoutDst() const { +- int32_t bits = getValType().getIntOrFloatBitWidth(); ++ int32_t bits = getAtomicValueBitWidth(getValType()); + return FxLayout(FxShape(FxC(1), FxC(bits)), FxStride(FxC(1), FxC(1))); + } + Attribute CopyOpUniversalAtomicType::getThrBitLayoutRef() const { +- int32_t bits = getValType().getIntOrFloatBitWidth(); ++ int32_t bits = getAtomicValueBitWidth(getValType()); + return FxLayout(FxShape(FxC(1), FxC(bits)), FxStride(FxC(1), FxC(1))); + } + diff --git a/third_party/flydsl/xf32_mfma.patch b/third_party/flydsl/xf32_mfma.patch new file mode 100644 index 00000000..bc6126a0 --- /dev/null +++ b/third_party/flydsl/xf32_mfma.patch @@ -0,0 +1,25 @@ +diff --git a/lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp b/lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp +--- a/lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp ++++ b/lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp +@@ -109,8 +109,12 @@ + static bool isBF8(Type ty) { return isa(ty); } + + static Type getMfmaABType(MLIRContext *ctx, Type elemTy, int32_t mn, int32_t k = 0) { +- if (elemTy.isF32()) ++ if (elemTy.isF32()) { ++ // v_mfma_f32_32x32x4_xf32 consumes two packed FP32 inputs per lane. ++ if (mn == 32 && k == 4) ++ return VectorType::get({2}, Float32Type::get(ctx)); + return Float32Type::get(ctx); ++ } + if (elemTy.isF16()) + return VectorType::get({mn * k / 64}, Float16Type::get(ctx)); + if (elemTy.isBF16()) { +@@ -179,6 +183,7 @@ + DISPATCH_MFMA_SSA(4, 1, elemTyA.isF32(), mfma_f32_4x4x1f32) + DISPATCH_MFMA_SSA(32, 2, elemTyA.isF32(), mfma_f32_32x32x2f32) + DISPATCH_MFMA_SSA(16, 4, elemTyA.isF32(), mfma_f32_16x16x4f32) ++ DISPATCH_MFMA_SSA(32, 4, elemTyA.isF32(), mfma_f32_32x32x4_xf32) + + DISPATCH_MFMA_SSA(32, 4, elemTyA.isF16(), mfma_f32_32x32x4f16) + DISPATCH_MFMA_SSA(16, 4, elemTyA.isF16(), mfma_f32_16x16x4f16) diff --git a/third_party/non_module_deps.bzl b/third_party/non_module_deps.bzl index 9b8d6530..e83ae6f7 100644 --- a/third_party/non_module_deps.bzl +++ b/third_party/non_module_deps.bzl @@ -4,6 +4,7 @@ load("//third_party/com_google_sentencepiece:repo.bzl", com_google_sentencepiece load("//third_party/cuda_tile:repo.bzl", cuda_tile = "repo") load("//third_party/flashattn:repo.bzl", flashattn = "repo") load("//third_party/flashinfer_cutlass_moe:repo.bzl", flashinfer_cutlass_moe = "repo") +load("//third_party/flydsl:repo.bzl", flydsl = "repo") load("//third_party/iree:repo.bzl", iree = "repo") load("//third_party/libvaxis:repo.bzl", libvaxis = "repo") load("//third_party/linenoise:repo.bzl", linenoise = "repo") @@ -34,6 +35,7 @@ def _non_module_deps_impl(mctx): mosaic_tpu() iree() cuda_tile() + flydsl() return mctx.extension_metadata( reproducible = True, diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 35587abf..92a9bf98 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -146,11 +146,13 @@ zig_library( "//bazel", "//ffi:zig", "//kernels/cuda_tile:cuda_tile_builder", + "//kernels/fly:fly_builder", "//kernels/mosaic_tpu:mosaic_tpu_builder", "//kernels/triton:triton_builder", "//mlir", "//mlir/dialects", "//mlir/dialects/cuda_tile", + "//mlir/dialects/fly", "//mlir/dialects/mosaic_tpu", "//pjrt", "//platforms", diff --git a/zml/kernel.zig b/zml/kernel.zig index 67cfefaf..ca481d65 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -4,6 +4,7 @@ const stdx = @import("stdx"); const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); const cuda_tile_builder = @import("kernels/cuda_tile/builder"); +const fly_builder = @import("kernels/fly/builder"); const mosaic_tpu_builder = @import("kernels/mosaic_tpu/builder"); const tpu_dialect = @import("mlir/dialects/mosaic_tpu"); const triton_builder = @import("kernels/triton/builder"); @@ -12,6 +13,7 @@ const Compiler = @import("Compiler.zig"); const DataType = @import("dtype.zig").DataType; const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); +const platform_ = @import("platform.zig"); const Shape = @import("shape.zig").Shape; const Tensor = @import("tensor.zig").Tensor; @@ -508,6 +510,341 @@ pub const cuda_tile = struct { } }; +pub const fly = struct { + /// Threads per wavefront: 64 on CDNA, 32 on RDNA. + pub fn waveSize(p: *const platform_.Platform) i32 { + const cc = platform_.rocm.computeCapability(p) orelse return 64; + return switch (cc.architecture()) { + .cdna1, .cdna2, .cdna3, .cdna4 => 64, + .rdna2, .rdna3, .rdna3_5, .rdna4 => 32, + }; + } + + /// The plugin sizes a block in wavefronts, so a thread count has to divide + /// evenly: a kernel's layouts are built for an exact number of threads and + /// spare ones would partition out of range. + fn warpsFor(p: *const platform_.Platform, threads: i32) i32 { + const wave = waveSize(p); + if (@rem(threads, wave) != 0) { + std.debug.panic("zml.kernel.fly: {d} threads is not a multiple of the {d}-wide wavefront", .{ threads, wave }); + } + return @divExact(threads, wave); + } + + pub const Builder = fly_builder.Builder; + pub const Value = fly_builder.Value; + pub const DType = fly_builder.DType; + pub const FinishError = fly_builder.FinishError; + pub const TiledCopy = fly_builder.TiledCopy; + pub const TiledMma = fly_builder.TiledMma; + pub const Arch = fly_builder.Arch; + pub const MmaFlavor = fly_builder.MmaFlavor; + + /// The matrix atom this device provides, or null when it has none: WMMA + /// arrived with RDNA3, so gfx1030 has no matrix instruction at all. + pub fn mmaFlavor(p: *const platform_.Platform) ?MmaFlavor { + const cc = platform_.rocm.computeCapability(p) orelse return .cdna3_mfma; + return switch (cc.architecture()) { + .cdna1, .cdna2, .cdna3, .cdna4 => .cdna3_mfma, + .rdna3, .rdna3_5 => .gfx11_wmma, + .rdna4 => .gfx120x_wmma, + .rdna2 => null, + }; + } + pub const layout = fly_builder.layout; + pub const Layout = fly_builder.Layout; + pub const Tile = fly_builder.Tile; + pub const IntTuple = fly_builder.IntTuple; + pub const L = fly_builder.L; + pub const rowMajor = fly_builder.rowMajor; + pub const colMajor = fly_builder.colMajor; + pub const ordered = fly_builder.ordered; + pub const tile = fly_builder.tile; + pub const it = fly_builder.it; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(&fly_builder.dialects_needed); + } + + pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3fnuz => .f8e4m3fnuz, + .f8e5m2 => .f8e5m2, + .f8e5m2fnuz => .f8e5m2fnuz, + else => std.debug.panic("zml.kernel.fly.from: dtype {s} has no Fly equivalent", .{@tagName(dt)}), + }; + } + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + pub const arg_names = spec.inputs ++ spec.outputs; + /// By name, as `!fly.memref` views. + pub const Args = StructOf(arg_names, Value); + + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + /// Threads per block. Converted to the wavefronts the plugin + /// wants using the device's wave size, which is 64 on CDNA and + /// 32 on RDNA, so a kernel states the thread count its layouts + /// were built for and runs on both. + threads: i32, + /// > 0 sets `amdgpu-waves-per-eu = "N, N"`: a hard occupancy clamp. + waves_per_eu: i32 = 0, + /// Required when the body uses `fly.get_dyn_shared`. + shared_mem_bytes: i64 = 0, + /// Inputs first, then outputs; see `zeroedOutput`. A no-op + /// inside command buffers. + zeroed_args: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + /// The arguments `run` was declared with. + pub fn args(b: *Builder) Args { + var out: Args = undefined; + inline for (arg_names, 0..) |fname, i| @field(out, fname) = b.arg(i); + return out; + } + + /// `zeroed_args` index of an output. + pub fn zeroedOutput(comptime f: std.meta.FieldEnum(Outputs)) i32 { + return @intCast(spec.inputs.len + @intFromEnum(f)); + } + + /// `shapes` is inputs, then outputs. + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, shapes: []const Shape) ![:0]const u8 { + std.debug.assert(shapes.len == arg_names.len); + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try fly_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + var specs: [arg_names.len]Builder.ArgSpec = undefined; + inline for (arg_names, 0..) |fname, i| { + const shape = shapes[i]; + specs[i] = .{ + .name = fname, + .dtype = from(shape.dtype()), + .dims = if (shape.rank() > 0) shape.dims() else null, + }; + } + try b.declareArgsLow(&specs); + + try spec.run(&b, cfg); + + return b.finish(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + var shapes: [arg_names.len]Shape = undefined; + inline for (0..spec.inputs.len) |i| shapes[i] = inputs_arr[i].shape(); + inline for (0..spec.outputs.len) |i| shapes[spec.inputs.len + i] = outputs_arr[i]; + + const ir = emit(cur.allocator, opts.cfg, &shapes) catch |err| + std.debug.panic("zml.kernel.fly.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.fly(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .num_warps = warpsFor(cur.platform, opts.threads), + .waves_per_eu = opts.waves_per_eu, + .shared_mem_bytes = opts.shared_mem_bytes, + .zeroed_args = opts.zeroed_args, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; + } +}; + +/// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the +/// ragged edge. +const FlyVectorAdd = struct { + const Cfg = struct { + thr: fly.Layout = fly.ordered(.{ 8, 16 }, .{ 1, 0 }), + val: fly.Layout = fly.ordered(.{ 1, 4 }, .{ 0, 1 }), + }; + + const K = fly.Kernel(Cfg, .{ + .name = "vector_add", + .inputs = &.{ "a", "b" }, + .outputs = &.{"c"}, + .run = run, + }); + + fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { + const a = K.args(b); + fly_builder.emitVectorAdd(b, cfg.thr, cfg.val, a.a, a.b, a.c); + } +}; + +test "fly kernel emits a module XLA can parse" { + const shapes = [_]Shape{ .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32) }; + const ir = try FlyVectorAdd.K.emit(std.testing.allocator, .{}, &shapes); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.func @vector_add(%arg0: !fly.ptr, %arg1: !fly.ptr, %arg2: !fly.ptr) kernel") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "fly.static : !fly.tile<[8|64]>") != null); +} + +/// C = A @ B^T with one MFMA-tiled block. +const FlyTiledMma = struct { + const Cfg = struct { flavor: fly.MmaFlavor }; + const K = fly.Kernel(Cfg, .{ + .name = "tiled_mma", + .inputs = &.{ "a", "b" }, + .outputs = &.{"c"}, + .run = run, + }); + + fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { + const a = K.args(b); + fly_builder.emitTiledMma(b, cfg.flavor, a.a, a.b, a.c); + } +}; + +test "fly kernels run on rocm" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + if (platform.target != .rocm) return error.SkipZigTest; + + const allocator = std.testing.allocator; + + // Compile `forward`, upload both inputs, run, and bring the result back. + const call = struct { + fn f(comptime forward: anytype, p: *const zml.Platform, ta: Tensor, ha: anytype, tb: Tensor, hb: anytype) !zml.Slice { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ ta, tb }, p, .{}); + defer exe.deinit(); + var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), .replicated, std.mem.sliceAsBytes(ha)); + defer ba.deinit(); + var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), .replicated, std.mem.sliceAsBytes(hb)); + defer bb.deinit(); + var out = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ ba, bb }); + defer out.deinit(); + return out.toSliceAlloc(std.testing.allocator, std.testing.io); + } + }.f; + + // vector add: elementwise over a ragged (100, 1000), predicated at the edge. + { + const M, const N = .{ 100, 1000 }; + const Mod = struct { + pub fn forward(a: Tensor, b: Tensor) Tensor { + return FlyVectorAdd.K.call(.{ .a = a, .b = b }, .{ .c = a.shape() }, .{ + .cfg = .{}, + .grid = .{ (M + 8 - 1) / 8, (N + 64 - 1) / 64, 1 }, + .threads = 8 * 16, + }).c; + } + }; + + const ha = try allocator.alloc(f32, M * N); + defer allocator.free(ha); + const hb = try allocator.alloc(f32, M * N); + defer allocator.free(hb); + for (ha, hb, 0..) |*va, *vb, i| { + va.* = @floatFromInt(i % 977); + vb.* = @floatFromInt(1000 + (i % 313)); + } + + var host = try call(Mod.forward, platform, .init(.{ M, N }, .f32), ha, .init(.{ M, N }, .f32), hb); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(ha[i] + hb[i], v); + } + + // tiled mma: C = A @ B^T in one block of 256 threads, over whatever matrix + // atom this device has. CDNA multiplies f32 through MFMA; RDNA3+ has WMMA, + // whose verifier rejects f32 operands, so the operand type and K come from + // the flavour rather than being written into the test. + if (fly.mmaFlavor(platform)) |flavor| switch (flavor) { + inline else => |f| { + const Elem = switch (comptime f.operandDType()) { + .f32 => f32, + .f16 => f16, + else => @compileError("unhandled MMA operand type"), + }; + const M, const N = .{ 64, 64 }; + const Kd = comptime f.blockK(); + const operand_dtype: DataType = switch (comptime f.operandDType()) { + .f32 => .f32, + .f16 => .f16, + else => unreachable, + }; + + const Mod = struct { + pub fn forward(a: Tensor, b: Tensor) Tensor { + return FlyTiledMma.K.call(.{ .a = a, .b = b }, .{ .c = Shape.init(.{ M, N }, .f32) }, .{ + .cfg = .{ .flavor = f }, + .grid = .{ 1, 1, 1 }, + .threads = comptime f.blockThreads(), + }).c; + } + }; + + var ha: [M * Kd]Elem = undefined; + var hb: [N * Kd]Elem = undefined; + for (&ha, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 7) % 13)) - 6; + for (&hb, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 5) % 11)) - 5; + + var host = try call(Mod.forward, platform, .init(.{ M, Kd }, operand_dtype), &ha, .init(.{ N, Kd }, operand_dtype), &hb); + defer host.free(allocator); + + // f16 operands accumulate in f32, so the error is set by the input + // rounding rather than the sum. + const tol: f32 = if (Elem == f16) 1e-2 else 1e-3; + const out = host.items(f32); + for (0..M) |m| for (0..N) |n| { + var acc: f32 = 0; + for (0..Kd) |k| acc += @as(f32, ha[m * Kd + k]) * @as(f32, hb[n * Kd + k]); + try std.testing.expectApproxEqAbs(acc, out[m * N + n], tol); + }; + }, + }; +} + test "cuda_tile kernel emits a module XLA can parse" { const Cfg = struct { n: i64 }; const AddOne = cuda_tile.Kernel(Cfg, .{ diff --git a/zml/ops.zig b/zml/ops.zig index 33f241f5..4cdf9761 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -856,8 +856,8 @@ fn fromMlirOperationWithTags(op: *const mlir.Operation, base: anytype) @TypeOf(b pub const TritonOps = struct { debug: bool = false, - name: [:0]const u8, - ir: [:0]const u8, + name: []const u8, + ir: []const u8, grid: [3]i32, num_stages: i32, num_warps: i32, @@ -927,6 +927,186 @@ pub fn triton(inputs: anytype, outputs: anytype, opts: TritonOps) [outputs.len]T return outputs_; } +pub const FlyOps = struct { + name: []const u8, + /// The whole module text: a `gpu.func @name ... kernel` inside a + /// `gpu.module`, one `!fly.ptr` per operand then per result; or a + /// top-level `func.func @name` in the destination-passing tensor ABI. + ir: []const u8, + grid: [3]i32, + /// Threads per block = num_warps * 64 on CDNA. + num_warps: i32, + waves_per_eu: i32 = 0, + /// Required when the kernel uses `fly.get_dyn_shared`. + shared_mem_bytes: i64 = 0, + /// Indices into the full argument list, operands first, so result `i` is + /// `inputs.len + i`. A no-op inside HIP graphs: a kernel that must see + /// zeroed outputs should zero them itself. + zeroed_args: []const i32 = &.{}, + output_operand_aliases: []const dialects.stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, +}; + +pub fn fly(inputs: anytype, outputs: anytype, opts: FlyOps) [outputs.len]Tensor { + const mlir_ctx = Compiler.current().mlir_ctx; + var arena = std.heap.ArenaAllocator.init(Compiler.current().allocator); + defer arena.deinit(); + + var values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + values[i] = inputs[i].value(); + } + + var res_types: [outputs.len]*const mlir.Type = undefined; + inline for (outputs, 0..) |output, i| { + res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); + } + + // The plugin silently ignores unknown or wrong-typed optional keys, so a + // misspelled one here is not an error, just an override that never lands. + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 10) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), + .named(mlir_ctx, "ir", .string(mlir_ctx, opts.ir)), + .named(mlir_ctx, "num_warps", .int(mlir_ctx, .i32, opts.num_warps)), + .named(mlir_ctx, "grid_x", .int(mlir_ctx, .i32, opts.grid[0])), + .named(mlir_ctx, "grid_y", .int(mlir_ctx, .i32, opts.grid[1])), + .named(mlir_ctx, "grid_z", .int(mlir_ctx, .i32, opts.grid[2])), + }); + if (opts.waves_per_eu > 0) attrs.appendAssumeCapacity(.named(mlir_ctx, "waves_per_eu", .int(mlir_ctx, .i32, opts.waves_per_eu))); + if (opts.shared_mem_bytes > 0) attrs.appendAssumeCapacity(.named(mlir_ctx, "shared_mem_bytes", .int(mlir_ctx, .i64, opts.shared_mem_bytes))); + if (opts.zeroed_args.len > 0) { + const Opts = dialects.stablehlo.CustomCallOpts; + var zeroed: stdx.BoundedArray(*const mlir.Attribute, Opts.MAX_OPERANDS + Opts.MAX_RESULTS) = .empty; + for (opts.zeroed_args) |i| { + std.debug.assert(i >= 0 and i < inputs.len + outputs.len); + zeroed.appendAssumeCapacity(.int(mlir_ctx, .i32, i)); + } + attrs.appendAssumeCapacity(.named(mlir_ctx, "zeroed_outputs", .array(mlir_ctx, zeroed.constSlice()))); + } + const backend_config: *const mlir.Attribute = .dict(mlir_ctx, attrs.constSlice()); + + // The kernel sees raw pointers and assumes row-major: pin the layouts. + var operands_layouts: [inputs.len][]const usize = undefined; + inline for (inputs, 0..) |input, i| { + operands_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(input.rank())).constSlice()) catch unreachable; + } + var results_layouts: [outputs.len][]const usize = undefined; + inline for (outputs, 0..) |output, i| { + results_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(output.rank())).constSlice()) catch unreachable; + } + + const op = dialects.stablehlo.custom_call( + mlir_ctx, + &values, + &res_types, + .{ + .call_target_name = "__gpu$xla.gpu.fly", + .backend_config = .{ .typed_ffi = backend_config }, + .has_side_effect = false, + .operand_layouts = &operands_layouts, + .result_layouts = &results_layouts, + .output_operand_aliases = opts.output_operand_aliases, + }, + .unknown(mlir_ctx), + ).appendTo(Compiler.current().currentScope().block); + + var outputs_: [outputs.len]Tensor = undefined; + inline for (outputs, 0..) |output, i| { + outputs_[i] = Tensor._result(output, op.result(i)); + } + + return outputs_; +} + +test "fly custom call, both entry ABIs" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + if (platform.target != .rocm) return error.SkipZigTest; + + const allocator = std.testing.allocator; + + // Compile `forward` over one f32 input and bring the result back. + const call = struct { + fn f(comptime forward: anytype, p: *const zml.Platform, t: Tensor, h: []const f32) !zml.Slice { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{t}, p, .{}); + defer exe.deinit(); + var buf: zml.Buffer = try .fromBytes(std.testing.io, p, t.shape(), .replicated, std.mem.sliceAsBytes(h)); + defer buf.deinit(); + var out = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{buf}); + defer out.deinit(); + return out.toSliceAlloc(std.testing.allocator, std.testing.io); + } + }.f; + + // Native Fly ABI: a `gpu.func ... kernel` taking one !fly.ptr per operand + // then per result. This is what `zml.kernel.fly` emits. + { + const Mod = struct { + const ir = + \\module attributes {gpu.container_module} { + \\ gpu.module @zml_fly_kernels { + \\ gpu.func @add_one(%a: !fly.ptr, %c: !fly.ptr) kernel { + \\ %tid = gpu.thread_id x + \\ %step = gpu.block_dim x + \\ %n = arith.constant 128 : index + \\ scf.for %i = %tid to %n step %step { + \\ %t = arith.index_cast %i : index to i32 + \\ %off = fly.make_int_tuple(%t) : (i32) -> !fly.int_tuple + \\ %pa = fly.add_offset(%a, %off) : (!fly.ptr, !fly.int_tuple) -> !fly.ptr + \\ %pc = fly.add_offset(%c, %off) : (!fly.ptr, !fly.int_tuple) -> !fly.ptr + \\ %va = fly.ptr.load(%pa) : (!fly.ptr) -> f32 + \\ %one = arith.constant 1.0 : f32 + \\ %vc = arith.addf %va, %one : f32 + \\ fly.ptr.store(%vc, %pc) : (f32, !fly.ptr) -> () + \\ } + \\ gpu.return + \\ } + \\ } + \\} + ; + pub fn forward(a: Tensor) Tensor { + return fly(.{a}, .{a.shape()}, .{ .name = "add_one", .ir = ir, .grid = .{ 1, 1, 1 }, .num_warps = 2 })[0]; + } + }; + + var input: [128]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @floatFromInt(i); + + var host = try call(Mod.forward, platform, .init(.{ .n = 128 }, .f32), &input); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(@as(f32, @floatFromInt(i)) + 1, v); + } + + { + const Mod = struct { + const ir = + \\module { + \\ func.func @add_one(%input: tensor<64xf32>, %output: tensor<64xf32>) -> tensor<64xf32> { + \\ %thread = gpu.thread_id x + \\ %last = arith.constant 63 : index + \\ %i = arith.minui %thread, %last : index + \\ %value = tensor.extract %input[%i] : tensor<64xf32> + \\ %one = arith.constant 1.0 : f32 + \\ %sum = arith.addf %value, %one : f32 + \\ %updated = tensor.insert %sum into %output[%i] : tensor<64xf32> + \\ return %updated : tensor<64xf32> + \\ } + \\} + ; + pub fn forward(a: Tensor) Tensor { + return fly(.{a}, .{a.shape()}, .{ .name = "add_one", .ir = ir, .grid = .{ 1, 1, 1 }, .num_warps = 2 })[0]; + } + }; + + var input: [64]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @floatFromInt(i); + + var host = try call(Mod.forward, platform, .init(.{ .n = 64 }, .f32), &input); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(@as(f32, @floatFromInt(i + 1)), v); + } +} + pub const NeuronNkiOps = struct { name: []const u8, entrypoint: []const u8, @@ -1091,8 +1271,8 @@ test "triton" { } pub const CudaTileOps = struct { - name: [:0]const u8, - ir: [:0]const u8, + name: []const u8, + ir: []const u8, grid: [3]i32, /// The Tile IR bytecode version XLA serializes at, "MAJOR.MINOR"; XLA's /// default is 13.3. From fdb79caa05403f74bb97882f226bc76715507322 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Mon, 14 Sep 2026 15:42:56 +0200 Subject: [PATCH 682/881] kernels/fly: MOE + MLA (#805) Add native FlyDSL kernels for MXFP4 MoE and BF16 sparse MLA on `gfx942`, with SSA builder operations for matrix loops --- kernels/fly/PORTING.md | 35 +- kernels/fly/README.md | 98 +++- kernels/fly/builder.zig | 246 +++++++++- zml/BUILD.bazel | 5 + zml/attention.zig | 1 + zml/attention/fly_kernels/sparse_mla.zig | 458 +++++++++++++++++++ zml/attention/paged_attention.zig | 6 +- zml/attention/sparse_mla.zig | 204 +++++++++ zml/attention/triton_attention.zig | 172 +------ zml/kernel.zig | 1 + zml/moe/fly_kernels/moe.zig | 289 ++++++++++++ zml/moe/fly_kernels/mxfp4.zig | 462 +++++++++++++++++++ zml/moe/fused_experts.zig | 540 ++++++++++++++++++++++ zml/moe/moe.zig | 69 ++- zml/moe/triton.zig | 555 +---------------------- 15 files changed, 2390 insertions(+), 751 deletions(-) create mode 100644 zml/attention/fly_kernels/sparse_mla.zig create mode 100644 zml/attention/sparse_mla.zig create mode 100644 zml/moe/fly_kernels/moe.zig create mode 100644 zml/moe/fly_kernels/mxfp4.zig create mode 100644 zml/moe/fused_experts.zig diff --git a/kernels/fly/PORTING.md b/kernels/fly/PORTING.md index b46d3b97..21673dbe 100644 --- a/kernels/fly/PORTING.md +++ b/kernels/fly/PORTING.md @@ -1,9 +1,10 @@ # Porting a FlyDSL Python kernel to Zig -The Zig DSL emits the same `fly` ops the Python does, so a port is a -statement-by-statement translation. Work from the pinned FlyDSL tree -(`third_party/flydsl/repo.bzl`, read with `git show :`), not from -FlyDSL HEAD: the plugin only parses what the pin prints. +The Zig DSL builds Fly IR directly. Work from the revision and patches in +[the FlyDSL repository rule](../../third_party/flydsl/repo.bzl), reading +Python sources with `git -C show :`. Check the +[ROCm plugin pin](../../platforms/rocm/rocm.bzl) as well: the emitter and +runtime must agree on the dialect version. FlyDSL HEAD may differ from both. ## Cheat sheet @@ -30,7 +31,7 @@ FlyDSL HEAD: the plugin only parses what the pin prints. | `fx.make_view(ptr, layout)` | `ptr.view(layout)` | | `fx.make_copy_atom(fx.UniversalCopy128b(), fx.Float32)` | `b.copyAtom(.{ .universal = 128 }, .f32)` | | `fx.rocdl.BufferCopy32b()` / `BufferCopyLDS128b()` | `.{ .buffer_copy = 32 }` / `.{ .buffer_copy_lds = 128 }` | -| `fx.make_mma_atom(fx.rocdl.MFMA(16,16,4,fx.Float32))` | `b.mmaAtom(try fly.rocdl.MmaOpCDNA3MFMAType.get(ctx, .{ .m = 16, .n = 16, .k = 4, ... }))` | +| `fx.make_mma_atom(fx.rocdl.MFMA(16,16,4,fx.Float32))` | `b.mmaAtom((try fly.rocdl.MmaOpCDNA3MFMAType.get(b.ctx, .{ .m = 16, .n = 16, .k = 4, ... })).type_())` | | `fx.make_tiled_mma(atom, layout)` | `b.tiledMma(atom, layout, null)` | | `fx.make_tiled_copy_tv(atom, thr, val)` | `b.tiledCopyTV(atom, thr, val)` | | `fx.make_tiled_copy_A(atom, tiled_mma)` | `b.tiledCopyA(atom, tiled_mma)` | @@ -63,8 +64,12 @@ FlyDSL HEAD: the plugin only parses what the pin prints. ## Method 1. Read the kernel and its helpers at the pin; note every `fx.*` primitive. -2. Write the Zig `run` top to bottom. Keep the same op order so the emitted - IR lines up with the Python module when diffing. +2. Write the Zig `run` top to bottom, preserving operation order and numeric + conversions. For matrix loops, use SSA copy and MMA calls as described + under [control flow and register tensors](README.md#control-flow-and-register-tensors): + `b.copyAtomLoad` returns vectors, `b.mmaAtomCall` returns the accumulator, + and `b.copyAtomStore` writes vectors. Carry accumulator values through + `openFor`; do not put mutable register tensors across control-flow regions. 3. `bazel test //kernels/fly:test` (or a `K.emit` test): the module must verify and re-parse. A panic from `fly.: cannot create operation` lists the operand types; compare them with the Python trace. @@ -76,8 +81,24 @@ FlyDSL HEAD: the plugin only parses what the pin prints. - Python config (`N`, `dtype_str`, `const_expr(...)` branches) becomes the kernel `Config`; branch on it with plain `if` in `run`. - Python lists of vectors across an unrolled loop are Zig arrays of `Value`. + `v.insert(i, x)` returns a new vector; `v.extract(i)` reads an element. + `v.bitcast(dtype)` preserves total bits and adjusts vector length. +- Keep unsigned semantics explicit for packed bytes and offsets with + `toUnsigned`, `divUnsigned`, `remUnsigned` and `cmpUnsigned`. +- `v.shuffleVector(rhs, indices)` rearranges vector elements within a thread; + `v.shuffle(.up, offset, width)` and `shuffleXor` exchange values across lanes. +- Runtime `while` loops use `b.openWhile(inits, after_types)`, `yieldBefore` + and `yieldAfter`; use a plain Zig `while` only for loops unrolled at emit time. - Python `Int32(x)` / `Float32(x)` constants are `b.constant(.i32, x)`; most arithmetic methods also take a literal directly (`tid.add(256)`). - Cached scalars from LDS are read with `t.at(i)`; there is no implicit load. - bf16 outputs use `arith.truncf`; the Python's manual round-to-nearest bit trick for pre-gfx950 parts is not reproduced. + +## Model kernels + +The [MoE kernels](../../zml/moe/fly_kernels/) include native routing and +MXFP4 GEMV/MFMA implementations. The +[sparse MLA kernel](../../zml/attention/fly_kernels/sparse_mla.zig) uses SSA +matrix accumulators through attention loops. These are examples of the +builder APIs with model tensor layouts and launch wrappers. diff --git a/kernels/fly/README.md b/kernels/fly/README.md index 66b089aa..0f0a86f7 100644 --- a/kernels/fly/README.md +++ b/kernels/fly/README.md @@ -14,9 +14,9 @@ Two layers define the surface: inputs and outputs, hands `run` its arguments by name (`K.args(b)`), and `call`s it from a ZML model. -The dialect binding (`mlir/dialects/fly`) is generic: every `fly.*` op is -built by name and FlyDSL's own `inferReturnTypes` computes every layout. Zig -only prints and reads the dialect's type syntax; it never does layout algebra. +The dialect bindings construct typed MLIR operations. Layout-transforming +operations use FlyDSL's `inferReturnTypes`; static layout literals use typed +attribute constructors and accessors. ## Skeleton of a kernel @@ -83,9 +83,22 @@ const c = VectorAdd.call(.{ .a = a, .b = bt }, .{ .c = a.shape() }, .{ Complete vector-add and tiled-MMA kernels are the tests in `builder.zig` and `zml/kernel.zig`. +Model kernels live beside their consumers: + +- [MoE routing](../../zml/moe/fly_kernels/moe.zig) and + [MXFP4 projections](../../zml/moe/fly_kernels/mxfp4.zig). +- [Sparse MLA](../../zml/attention/fly_kernels/sparse_mla.zig). + +Fly is a separate backend. The [MoE selector](../../zml/moe/moe.zig) and +[sparse-MLA selector](../../zml/attention/sparse_mla.zig) select by platform, +architecture and dtype. The current native kernels target gfx942; their +callers check shape support before routing or launching kernels and use +Triton for unsupported contracts. + ## The ABI -`Kernel.emit` produces +`Kernel.emit(allocator, cfg, shapes)` takes shapes in input-then-output order +and produces ```mlir module attributes {gpu.container_module} { @@ -99,17 +112,22 @@ with one `!fly.ptr` per input, then per output. The plugin checks only the argument **count**, so the pointer element type is on you: `DType.storageElem` maps predicates to `i8` because that is how XLA stores them. Static DSL values (layouts, tiles, atoms) must be built inside the body, never passed as -arguments. `Kernel.call` bridges each pointer to a row-major `!fly.memref` -view of the actual `zml.Shape`, so `K.args(b)` gives tensors. +arguments. `Kernel.call` bridges each non-scalar argument to a row-major +`!fly.memref` view of its `zml.Shape`. Rank-zero arguments remain pointers; +use `ptrLoad` and `ptrStore` for their scalar values. Launch geometry lives in `CallOpts`: `grid`, `threads` (threads per block, converted to wavefronts using the device's wave size — 64 on CDNA, 32 on -RDNA — so it must be a multiple of that), optional `waves_per_eu` (a hard -occupancy clamp), +RDNA — so it must be a multiple of that), optional `waves_per_eu` (sets the +LLVM `amdgpu-waves-per-eu` attribute to `N, N` when positive), `shared_mem_bytes` (only for `fly.get_dyn_shared`; static LDS from `sharedArray` needs none) and `zeroed_args` (indices into inputs-then-outputs; ignored inside HIP graphs, so zero what you need yourself). +`output_operand_aliases` maps output names to input names for buffers that +the kernel reads and writes. Aliasing preserves the input-then-output ABI; +it does not initialize output memory. + ## Layouts, tiles, coordinates `layout.zig` holds comptime literals. `toAttr` turns one into the matching @@ -135,8 +153,9 @@ inferred type back on the host; `v.emitShape()` / `v.emitSize()` emit ops. - `b.copyAtom(.{ .universal = 128 }, .f32)`, `.{ .buffer_copy = 32 }`, `.{ .buffer_copy_lds = 128 }`. -- `b.mmaAtom(mma_op)`, where `mma_op` is an MMA op type such as - `fly.rocdl.MmaOpCDNA3MFMAType.get(ctx, .{ .m = 16, .n = 16, .k = 16, ... })`. +- `b.mmaAtom(mma_op)` takes an MLIR type. Generated MMA type constructors + return a typed wrapper; handle their error and pass the wrapper's + `.type_()` result. - `b.tiledCopyTV(atom, thr, val)`, `b.tiledCopy(atom, layout_tv, tile)`, `b.tiledCopyA/B/C(copy_atom, tiled_mma)`; `tc.getSlice(tid).partitionS/D`, `.retile`. @@ -144,6 +163,9 @@ inferred type back on the host; `v.emitShape()` / `v.emitSize()` emit ops. `tm.makeFragmentA/B/C(t)`, `tm.getSlice(tid).partitionA/B/C`. - `b.copy(atom, src, dst, .{ .pred = p })`, `b.copyAtomCall(atom, src, dst)`, `b.gemm(atom, d, a, b, c)`. +- `b.mmaAtomCall(atom, a, b, c)` takes SSA operands and returns the updated + accumulator. Operand vectors must match the atom's per-lane fragment + types and layout; this call performs no tiled partitioning. - `b.bufferTensor(t)` is `fx.rocdl.make_buffer_tensor`: the same tensor over a CDNA raw-buffer descriptor, for bounds-checked `buffer_copy` atoms. - Derived layouts come off the type: `fly.expect(tm.value.type_(), .tiled_mma)` @@ -161,23 +183,59 @@ Scalars and vectors: `add/sub/mul/div/rem` (a scalar broadcasts against a vector), `to(dtype)`, `cmp(.lt, x)`, `cond.select(a, b)`, `v.reduce(.add)`, `v.shuffleXor(off, 64)`, `b.rsqrt(x)`, `b.constant(.f32, 1.0)`, `b.threadId(.x)` / `b.blockId(.x)` (already `i32`), `b.barrier()`. -`b.setFastMath(true)` tags subsequent float ops `fastmath`. + +Integer types are signless in MLIR. `div`, `rem`, `cmp` and `to` use signed +integer semantics, except that `to` treats predicates as unsigned. +`divUnsigned`, `remUnsigned`, `cmpUnsigned` and `toUnsigned` make unsigned +operations explicit. `bitAnd`, `bitOr`, `bitXor`, `shl` and `shrU` operate +on integer bits. + +`splat` creates a vector, including when its length is one. Numeric casts +preserve vector shape. `bitcast` preserves total bits and adjusts vector +length to the destination element width; scalar bitcasts require equal +widths. Use `extract` to obtain a scalar, `insert` to replace an element and +`shuffleVector` to select elements from two vectors. Lane `shuffle` +supports `xor`, `up`, `down` and `idx` modes with a literal or an `i32` +`Value` offset. + +`maximum` propagates NaNs; `maxNum` selects the numeric operand when only +one operand is NaN. `neg`, `Builder.log` and `Builder.fma` provide floating +operations. Fast math is off by default; `setFastMath(true)` tags subsequent +float operations `fastmath` and permits relaxed numerical semantics. + +`Builder.perm` emits AMD byte permutation. `Builder.cvtPkF32Fp8` decodes +two FP8 bytes from the low or high half of an `i32` word using the CDNA3 +packed conversion instruction. Architecture selection belongs in the +backend before these operations are emitted. + +`Builder.ptrAtomicAdd` adds an `i32` counter and returns its old value. +Use `workgroup` scope for LDS pointers and `agent` scope for global +pointers. The ordering is monotonic; callers provide barriers or kernel +dependencies before consuming completed counters. Control flow uses `kernels/common`: `b.openFor(lo, hi, step, .{ inits })` with `.yield(.{ ... })`, `b.openIf(cond)` with `.yieldThen(.{})`, `b.openIfElse(cond, .{ types })` with `.yieldThen` / `.yieldElse`. +`openWhile` has separate condition and body regions: `before_carried` +feeds `yieldBefore`, `after_carried` feeds `yieldAfter`, and `results` +contains the values returned when the condition becomes false. ## Control flow and register tensors -The ROCm plugin lowers `scf` to `cf` **before** running the Fly passes on a -`gpu.func` entry, and Fly's register-alloca promotion only rewrites a -single-block body. So a kernel that has `openIf` / `openFor` must not hold -register tensors (`rmemTensor`, `makeFragmentLike`, `fill`) — use the SSA -atom calls instead: `b.copyAtomLoad(atom, src)` returns the loaded -`vector`, `b.copyAtomStore(atom, vec, dst)` stores one. Kernels without -control flow (vectorAdd, tiledMma) can use register tensors freely. The -durable fix is on the XLA side (run `convert-scf-to-cf` after the Fly -passes, as FlyDSL's own pipeline does). +The published ROCm plugin lowers `scf` to `cf` **before** running the Fly +passes on a native `gpu.func` entry, and Fly's register-alloca promotion +only rewrites a single-block body. Kernels with runtime control flow must +keep their register state in SSA values: + +- `copyAtomLoad` returns a vector and `copyAtomStore` stores a vector. +- `mmaAtomCall` returns an accumulator without allocating register memory. +- Pass accumulators through `openFor` initial values, `carried`, `yield` + and `results`. Branches and while loops also return SSA values through + their scope objects. + +The MoE and sparse-MLA kernels linked above use this path with the published +runtime. Straight-line kernels can still use `rmemTensor`, +`makeFragmentLike`, `fill` and `gemm`. ## Debugging diff --git a/kernels/fly/builder.zig b/kernels/fly/builder.zig index f326246f..c560d2df 100644 --- a/kernels/fly/builder.zig +++ b/kernels/fly/builder.zig @@ -14,6 +14,7 @@ pub const layout = @import("layout.zig"); pub const dtypes = @import("dtype.zig"); pub const DType = dtypes.DType; +pub const rocdl = fly.rocdl; pub const IntTuple = layout.IntTuple; pub const Layout = layout.Layout; pub const Tile = layout.Tile; @@ -297,11 +298,11 @@ pub const Value = struct { fn coerceArith(self: Value, rhs: anytype) struct { Value, Value } { const k = self.kern(); const r: Value = if (@TypeOf(rhs) == Value) rhs else k.constLike(rhs, self); - const ln = self.vectorLen(); - const rn = r.vectorLen(); - if (ln == rn) return .{ self, r }; - if (rn == 1) return .{ self, k.emit(dialects.vector.broadcast(k.ctx, r.inner, self.type_(), k.loc())) }; - if (ln == 1) return .{ k.emit(dialects.vector.broadcast(k.ctx, self.inner, r.type_(), k.loc())), r }; + if (self.type_().eql(r.type_())) return .{ self, r }; + if (self.type_().isA(mlir.VectorType) != null and r.type_().isA(mlir.VectorType) == null) + return .{ self, k.emit(dialects.vector.broadcast(k.ctx, r.inner, self.type_(), k.loc())) }; + if (self.type_().isA(mlir.VectorType) == null and r.type_().isA(mlir.VectorType) != null) + return .{ k.emit(dialects.vector.broadcast(k.ctx, self.inner, r.type_(), k.loc())), r }; std.debug.panic("fly: cannot combine {f} with {f}", .{ self.type_(), r.type_() }); } @@ -332,6 +333,122 @@ pub const Value = struct { return k.emit(arith.remsi(k.ctx, l.inner, r.inner, k.loc())); } + pub fn bitAnd(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emit(arith.andi(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn bitOr(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emit(arith.ori(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn bitXor(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emit(arith.xori(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn shl(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emit(arith.shli(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn shrU(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emit(arith.shrui(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn divUnsigned(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emit(arith.divui(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn remUnsigned(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emit(arith.remui(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn cmpUnsigned(self: Value, pred: Cmp, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + const p: arith.CmpIPredicate = switch (pred) { + .eq => .eq, + .ne => .ne, + .lt => .ult, + .le => .ule, + .gt => .ugt, + .ge => .uge, + }; + return k.emit(arith.cmpi(k.ctx, p, l.inner, r.inner, k.loc())); + } + + pub fn neg(self: Value) Value { + const k = self.kern(); + return k.emitFast(arith.negf(k.ctx, self.inner, k.loc())); + } + /// Propagates NaNs; use `maxNum` to select the numeric operand instead. + pub fn maximum(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emitFast(arith.maximumf(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn maxNum(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emitFast(arith.maxnumf(k.ctx, l.inner, r.inner, k.loc())); + } + + /// Interpret integer operands and results as unsigned. + pub fn toUnsigned(self: Value, dtype: DType) Value { + const k = self.kern(); + const src = self.scalarDType(); + if (src == dtype) return self; + if (src.isFloat() and dtype.isFloat()) return self.to(dtype); + const result = self.withElemType(dtype.toMlir(k.ctx)); + const op = if (src.isFloat()) + arith.fptoui(k.ctx, self.inner, result, k.loc()) + else if (dtype.isFloat()) + arith.uitofp(k.ctx, self.inner, result, k.loc()) + else if (dtype.bitWidth() > src.bitWidth()) + arith.extui(k.ctx, self.inner, result, k.loc()) + else + arith.trunci(k.ctx, self.inner, result, k.loc()); + return k.emit(op); + } + + pub fn splat(self: Value, n: i64) Value { + const k = self.kern(); + std.debug.assert(self.type_().isA(mlir.VectorType) == null and n > 0); + return k.emit(dialects.vector.broadcast(k.ctx, self.inner, .vector(&.{n}, self.type_()), k.loc())); + } + + /// Preserve total bits, adjusting vector length to the destination element. + pub fn bitcast(self: Value, dtype: DType) Value { + const k = self.kern(); + const src = self.scalarDType(); + if (self.type_().isA(mlir.VectorType)) |vec| { + const n = @divExact(vec.dimension(0) * @as(i64, src.bitWidth()), dtype.bitWidth()); + return k.emit(dialects.vector.bitcast(k.ctx, self.inner, .vector(&.{n}, dtype.toMlir(k.ctx)), k.loc())); + } + std.debug.assert(src.bitWidth() == dtype.bitWidth()); + return k.emit(arith.bitcast(k.ctx, self.inner, dtype.toMlir(k.ctx), k.loc())); + } + + pub fn insert(self: Value, index: i64, value: Value) Value { + const k = self.kern(); + return k.emit(dialects.vector.insert(k.ctx, value.inner, self.inner, &.{index}, &.{}, k.loc())); + } + + pub fn shuffleVector(self: Value, rhs: Value, indices: []const i64) Value { + const k = self.kern(); + return k.emit(dialects.vector.shuffle(k.ctx, self.inner, rhs.inner, indices, .vector(&.{@intCast(indices.len)}, self.scalarElemType()), k.loc())); + } + + fn withElemType(self: Value, elem: *const mlir.Type) *const mlir.Type { + if (self.type_().isA(mlir.VectorType)) |vec| return .vector(&.{vec.dimension(0)}, elem); + return elem; + } + pub fn toI32(self: Value) Value { const k = self.kern(); return k.emit(arith.index_cast(k.ctx, self.inner, .int(k.ctx, .i32), k.loc())); @@ -351,8 +468,7 @@ pub const Value = struct { const k = self.kern(); const src = self.scalarDType(); if (src == dtype) return self; - const n = self.vectorLen(); - const result = if (n == 1) dtype.toMlir(k.ctx) else mlir.Type.vector(&.{n}, dtype.toMlir(k.ctx)); + const result = self.withElemType(dtype.toMlir(k.ctx)); const unsigned_src = src == .i1; const op = if (src.isFloat() and dtype.isFloat()) (if (dtype.bitWidth() > src.bitWidth()) @@ -415,9 +531,9 @@ pub const Value = struct { pub const ShuffleMode = enum { xor, up, down, idx }; - pub fn shuffle(self: Value, mode: ShuffleMode, offset: i32, width: i32) Value { + pub fn shuffle(self: Value, mode: ShuffleMode, offset: anytype, width: i32) Value { const k = self.kern(); - const off = k.constant(.i32, offset); + const off: Value = if (@TypeOf(offset) == Value) offset else k.constant(.i32, offset); const wid = k.constant(.i32, width); const mode_attr = switch (mode) { inline else => |m| fly.parseAttr(k.ctx, "#gpu"), @@ -937,6 +1053,68 @@ pub const Builder = struct { return self.emitFast(dialects.math.exp(self.ctx, x.inner, self.loc())); } + pub fn log(self: *Builder, x: Value) Value { + return self.emitFast(dialects.math.log(self.ctx, x.inner, self.loc())); + } + pub fn fma(self: *Builder, a: Value, b: Value, c: Value) Value { + return self.emitFast(dialects.math.fma(self.ctx, a.inner, b.inner, c.inner, self.loc())); + } + + /// A matrix atom with SSA operands; unlike `gemm`, this needs no register + /// allocas and can carry its accumulator through control flow. + pub fn mmaAtomCall(self: *Builder, atom: Value, a: Value, b: Value, c: Value) Value { + return self.emit(fly.typed(self.ctx, "mma_atom_call_ssa", &.{ atom.inner, a.inner, b.inner, c.inner }, &.{c.type_()}, .empty, self.loc())); + } + + /// AMD byte permutation. ROCDL has no corresponding operation at our pin. + pub fn perm(self: *Builder, hi: Value, lo: Value, selector: Value) Value { + return self.emit(fly.make(self.ctx, "llvm.inline_asm", .{ + .operands = .{ .flat = &.{ hi.inner, lo.inner, selector.inner } }, + .results = .{ .flat = &.{.int(self.ctx, .i32)} }, + .attributes = &.{ + .named(self.ctx, "asm_string", .string(self.ctx, "v_perm_b32 $0, $1, $2, $3")), + .named(self.ctx, "constraints", .string(self.ctx, "=v,v,v,v")), + }, + .location = self.loc(), + })); + } + + /// CDNA3 packed FP8 conversion, selecting two bytes from the low/high word. + pub fn cvtPkF32Fp8(self: *Builder, bits: Value, high: bool) Value { + return self.emit(fly.make(self.ctx, "rocdl.cvt.pk.f32.fp8", .{ + .operands = .{ .flat = &.{bits.inner} }, + .results = .{ .flat = &.{.vector(&.{2}, .float(self.ctx, .f32))} }, + .attributes = &.{.named(self.ctx, "wordSel", .boolean(self.ctx, high))}, + .location = self.loc(), + })); + } + + pub const AtomicScope = enum { workgroup, agent }; + + /// Monotonic i32 addition to a global or shared pointer. Callers provide + /// barriers or kernel dependencies before reading the completed counters. + pub fn ptrAtomicAdd(self: *Builder, ptr: Value, value: Value, scope: AtomicScope) Value { + std.debug.assert(ptr.elemDType() == .i32 and value.type_().eql(DType.i32.toMlir(self.ctx))); + const space: i32 = switch (scope) { + .workgroup => 3, + .agent => 1, + }; + const llvm_ptr = self.emit(fly.inferred(self.ctx, "to_llvm_ptr", &.{ptr.inner}, fly.attrs(&.{ + .named(self.ctx, "llvm_address_space", .int(self.ctx, .i32, space)), + }), self.loc())); + return self.emit(fly.make(self.ctx, "llvm.atomicrmw", .{ + .operands = .{ .flat = &.{ llvm_ptr.inner, value.inner } }, + .results = .{ .flat = &.{value.type_()} }, + .attributes = &.{ + .named(self.ctx, "bin_op", .int(self.ctx, .i64, 1)), // LLVM AtomicBinOp::add. + .named(self.ctx, "ordering", .int(self.ctx, .i64, 2)), // LLVM AtomicOrdering::monotonic. + .named(self.ctx, "syncscope", .string(self.ctx, @tagName(scope))), + .named(self.ctx, "alignment", .int(self.ctx, .i64, 4)), + }, + .location = self.loc(), + })); + } + /// Static LDS, as an `n:1` tensor in shared memory. Each call is its own /// `fly.make_ptr {allocBytes, allocAlign}` leaf, which the plugin lowers /// to one LLVM global, so `shared_mem_bytes` may stay 0. @@ -1257,6 +1435,56 @@ pub const Builder = struct { return .{ .kernel = self, .cond_inner = cond.inner, .then_block = then_block, .else_block = else_block, .result_types = types }; } + pub fn openWhile( + self: *Builder, + inits: anytype, + after_types: anytype, + ) WhileScope( + tupleArity(@TypeOf(inits), "openWhile: inits"), + tupleArity(@TypeOf(after_types), "openWhile: after_types"), + ) { + const N = comptime tupleArity(@TypeOf(inits), "openWhile: inits"); + const M = comptime tupleArity(@TypeOf(after_types), "openWhile: after_types"); + const init_fields = @typeInfo(@TypeOf(inits)).@"struct".fields; + const type_fields = @typeInfo(@TypeOf(after_types)).@"struct".fields; + + var before_types: [N]*const mlir.Type = undefined; + var before_locs: [N]*const mlir.Location = undefined; + var inits_inner: [N]*const mlir.Value = undefined; + inline for (init_fields, 0..) |f, i| { + if (f.type != Value) + @compileError("openWhile: every init must be a Value"); + const v: Value = @field(inits, f.name); + before_types[i] = v.type_(); + before_locs[i] = self.loc(); + inits_inner[i] = v.inner; + } + const before_block = mlir.Block.init(&before_types, &before_locs); + + var after_tys: [M]*const mlir.Type = undefined; + var after_locs: [M]*const mlir.Location = undefined; + inline for (type_fields, 0..) |f, i| { + if (f.type != *const mlir.Type) + @compileError("openWhile: every after_type must be *const mlir.Type"); + after_tys[i] = @field(after_types, f.name); + after_locs[i] = self.loc(); + } + const after_block = mlir.Block.init(&after_tys, &after_locs); + + self.pushBlock(before_block); + var before_carried: [N]Value = undefined; + for (0..N) |i| before_carried[i] = .{ .inner = before_block.argument(i), .kernel = self }; + + return .{ + .kernel = self, + .before_block = before_block, + .after_block = after_block, + .inits_inner = inits_inner, + .after_types = after_tys, + .before_carried = before_carried, + }; + } + // ---------------------------------------------------------------- finalization /// Append `gpu.return`, verify, and print the module (no debug info: XLA diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 92a9bf98..fd63b94e 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -81,9 +81,11 @@ zig_library( "attention.zig", "attention/attnd.zig", "attention/flashattn.zig", + "attention/fly_kernels/sparse_mla.zig", "attention/metal_attention.zig", "attention/mosaic_tpu_kernels/ragged_attention.zig", "attention/paged_attention.zig", + "attention/sparse_mla.zig", "attention/tpu_attention.zig", "attention/triton_attention.zig", "attention/triton_kernels/unified_attention.zig", @@ -102,6 +104,9 @@ zig_library( "mlirx.zig", "moe/triton_mxfp4.zig", "moe/cutlass_flashinfer.zig", + "moe/fly_kernels/moe.zig", + "moe/fly_kernels/mxfp4.zig", + "moe/fused_experts.zig", "moe/metal.zig", "moe/moe.zig", "moe/mosaic_tpu.zig", diff --git a/zml/attention.zig b/zml/attention.zig index 1b492810..c3cb8b47 100644 --- a/zml/attention.zig +++ b/zml/attention.zig @@ -1,6 +1,7 @@ const std = @import("std"); pub const attnd = @import("attention/attnd.zig"); +pub const fly = @import("attention/fly_kernels/sparse_mla.zig"); pub const flashattn = @import("attention/flashattn.zig"); pub const metal = @import("attention/metal_attention.zig"); pub const nki = @import("attention/nki/attention.zig"); diff --git a/zml/attention/fly_kernels/sparse_mla.zig b/zml/attention/fly_kernels/sparse_mla.zig new file mode 100644 index 00000000..f96293e1 --- /dev/null +++ b/zml/attention/fly_kernels/sparse_mla.zig @@ -0,0 +1,458 @@ +const std = @import("std"); +const zml = @import("../../zml.zig"); +const sparse_mla = @import("../sparse_mla.zig"); +const fly = zml.kernel.fly; +const Tensor = zml.Tensor; +const Value = fly.Value; + +pub const Config = struct { + queries: i64, + heads: i64, + pages: i64, + page_size: i64, + topk: i64, + splits: i64, + has_sink: bool, + all_decode: bool, +}; + +const main_inputs: []const [:0]const u8 = &.{ "query", "kv_cache", "sink", "indices", "active", "scale" }; +pub const Main2D = fly.Kernel(Config, .{ + .name = "sparse_mla_2d", + .inputs = main_inputs, + .outputs = &.{"output"}, + .run = runMain2D, +}); +pub const Main3D = fly.Kernel(Config, .{ + .name = "sparse_mla_3d", + .inputs = main_inputs, + .outputs = &.{ "partial_output", "partial_lse" }, + .run = runMain3D, +}); + +pub const ReduceConfig = struct { + queries: i64, + heads: i64, + input_splits: i64, + output_splits: i64, + has_sink: bool, + all_decode: bool, +}; +const reduce_inputs: []const [:0]const u8 = &.{ "input", "input_lse", "sink", "active" }; +pub const Reduce = fly.Kernel(ReduceConfig, .{ + .name = "sparse_mla_reduce", + .inputs = reduce_inputs, + .outputs = &.{"output"}, + .run = runReduce, +}); +pub const ReducePartials = fly.Kernel(ReduceConfig, .{ + .name = "sparse_mla_reduce_partials", + .inputs = reduce_inputs, + .outputs = &.{ "partial_output", "partial_lse" }, + .run = runReducePartials, +}); + +/// Shapes are local to a manual computation. Unsupported contracts fall back +/// before transposes or native launches enter the graph. +pub fn pagedAttention(q: Tensor, kv_cache: Tensor, sink: ?Tensor, indices: Tensor, active: Tensor, opts: sparse_mla.Options, cu_count: usize) ?Tensor { + if (q.rank() != 3 or q.dtype() != .bf16 or q.dim(.hd) != 512 or + q.dim(.q) <= 0 or q.dim(.h) <= 0 or opts.value_rank != 512 or + kv_cache.rank() != 4 or kv_cache.dtype() != .bf16 or kv_cache.dim(.hd) != 512 or + kv_cache.dim(.hkv) != 1 or kv_cache.dim(.page) <= 0 or kv_cache.dim(.k_chunk) <= 0 or + indices.rank() != 2 or indices.dtype() != .i32 or indices.dim(.q) != q.dim(.q) or indices.dim(.topk) <= 0 or + active.count() != 1 or active.dtype() != .i32) return null; + if (sink) |value| { + if (value.count() != @as(usize, @intCast(q.dim(.h)))) return null; + } + const config = sparse_mla.launchConfig(opts, @intCast(indices.dim(.topk)), cu_count); + if (config.block_m != 16 or config.tile_size == 0 or config.tile_size > 16 or + q.count() > std.math.maxInt(i32) / config.num_splits or indices.count() > std.math.maxInt(i32) or + kv_cache.count() / 4 > std.math.maxInt(i32)) return null; + return call( + q.transpose(.{ .q, .h, .hd }), + kv_cache.transpose(.{ .page, .k_chunk, .hkv, .hd }).squeeze(.hkv), + sink, + indices, + active, + opts, + config, + ).transpose(q.shape().tags()); +} + +/// Consumes local, row-major [query, head, 512] and [page, slot, 512] tensors. +/// Values span the full 512 columns; `config` supplies split/reduction settings. +pub fn call(q: Tensor, kv_cache: Tensor, sink: ?Tensor, indices: Tensor, active: Tensor, opts: sparse_mla.Options, config: sparse_mla.Config) Tensor { + std.debug.assert(q.rank() == 3 and q.dim(2) == 512 and q.dtype() == .bf16); + std.debug.assert(opts.value_rank == 512); + std.debug.assert(kv_cache.rank() == 3 and kv_cache.dim(2) == 512 and kv_cache.dtype() == .bf16); + std.debug.assert(indices.rank() == 2 and indices.dim(0) == q.dim(0) and indices.dtype() == .i32); + std.debug.assert(active.count() == 1 and active.dtype() == .i32); + std.debug.assert(config.block_m == 16 and config.tile_size > 0 and config.tile_size <= 16); + std.debug.assert(q.count() <= std.math.maxInt(i32) and indices.count() <= std.math.maxInt(i32)); + std.debug.assert(config.num_splits > 0 and q.count() <= std.math.maxInt(i32) / config.num_splits); + // Each i64 view element contains four adjacent BF16 values without copying. + // View counts are bounded; the kernel forms byte addresses in i64. + std.debug.assert(kv_cache.count() / 4 <= std.math.maxInt(i32)); + const cfg: Config = .{ + .queries = q.dim(0), + .heads = q.dim(1), + .pages = kv_cache.dim(0), + .page_size = kv_cache.dim(1), + .topk = indices.dim(1), + .splits = @intCast(config.num_splits), + .has_sink = sink != null, + .all_decode = opts.all_decode, + }; + const sink_ = if (sink) |value| value.convert(.f32).reshape(.{cfg.heads}) else Tensor.scalar(0, .f32).reshape(.{1}); + const inputs: Main3D.Inputs = .{ + .query = q.reshape(.{ cfg.queries, cfg.heads, 128, 4 }).bitCast(.i64), + .kv_cache = kv_cache.reshape(.{ cfg.pages, cfg.page_size, 128, 4 }).bitCast(.i64), + .sink = sink_, + .indices = indices.reshape(.{ cfg.queries, cfg.topk }), + .active = active.reshape(.{1}), + .scale = Tensor.scalar(opts.scale orelse @as(f32, 1.0 / @sqrt(512.0)), .f32).reshape(.{1}), + }; + const head_blocks = std.math.divCeil(i64, cfg.heads, 16) catch unreachable; + const grid: [3]i32 = .{ @intCast(cfg.queries * head_blocks), @intCast(cfg.splits), 1 }; + if (cfg.splits == 1) { + return Main2D.call(inputs, .{ .output = q.shape() }, .{ .cfg = cfg, .grid = grid, .threads = 256 }).output; + } + const partials = Main3D.call(inputs, .{ + .partial_output = zml.Shape.init(.{ cfg.queries, cfg.heads, cfg.splits, 512 }, .f32), + .partial_lse = zml.Shape.init(.{ cfg.queries, cfg.heads, cfg.splits }, .f32), + }, .{ .cfg = cfg, .grid = grid, .threads = 256 }); + var partial_output = partials.partial_output; + var partial_lse = partials.partial_lse; + var reduce_splits = cfg.splits; + if (cfg.splits > config.grouped_reduce_threshold) { + const output_splits = @divExact(cfg.splits, @as(i64, @intCast(config.splits_per_group))); + const grouped = ReducePartials.call(.{ + .input = partial_output, + .input_lse = partial_lse, + .sink = Tensor.scalar(0, .f32).reshape(.{1}), + .active = active.reshape(.{1}), + }, .{ + .partial_output = zml.Shape.init(.{ cfg.queries, cfg.heads, output_splits, 512 }, .f32), + .partial_lse = zml.Shape.init(.{ cfg.queries, cfg.heads, output_splits }, .f32), + }, .{ + .cfg = .{ .queries = cfg.queries, .heads = cfg.heads, .input_splits = cfg.splits, .output_splits = output_splits, .has_sink = false, .all_decode = cfg.all_decode }, + .grid = .{ @intCast(cfg.queries), @intCast(cfg.heads * output_splits), 1 }, + .threads = 64, + }); + partial_output = grouped.partial_output; + partial_lse = grouped.partial_lse; + reduce_splits = output_splits; + } + return Reduce.call(.{ + .input = partial_output, + .input_lse = partial_lse, + .sink = sink_, + .active = active.reshape(.{1}), + }, .{ .output = q.shape() }, .{ + .cfg = .{ .queries = cfg.queries, .heads = cfg.heads, .input_splits = reduce_splits, .output_splits = 1, .has_sink = cfg.has_sink, .all_decode = cfg.all_decode }, + .grid = .{ @intCast(cfg.queries), @intCast(cfg.heads), 1 }, + .threads = 64, + }).output; +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + return sparse_mla.Backend.auto(platform, .bf16) == .fly; +} + +fn runMain2D(b: *fly.Builder, cfg: Config) fly.FinishError!void { + std.debug.assert(cfg.splits == 1); + try runMain(b, cfg, false, Main2D.args(b)); +} + +fn runMain3D(b: *fly.Builder, cfg: Config) fly.FinishError!void { + try runMain(b, cfg, true, Main3D.args(b)); +} + +fn loadVector(b: *fly.Builder, ptr: Value, offset: Value, comptime dtype: fly.DType, comptime count: i64) Value { + const atom = b.copyAtom(.{ .universal = @intCast(count * @as(i64, dtype.bitWidth())) }, dtype); + return b.copyAtomLoad(atom, ptr.addOffset(offset).view(fly.L(count, 1))); +} + +fn storeVector(b: *fly.Builder, ptr: Value, offset: Value, value: Value, comptime dtype: fly.DType, comptime count: i64) void { + const atom = b.copyAtom(.{ .universal = @intCast(count * @as(i64, dtype.bitWidth())) }, dtype); + b.copyAtomStore(atom, value, ptr.addOffset(offset).view(fly.L(count, 1))); +} + +// One CTA owns one query, sixteen heads and one KV split. The four waves +// partition the 512 contraction dimensions; their score sums meet in LDS. +fn runMain(b: *fly.Builder, cfg: Config, comptime three_d: bool, a: anytype) fly.FinishError!void { + const head_blocks = std.math.divCeil(i64, cfg.heads, 16) catch unreachable; + const tiles = std.math.divCeil(i64, cfg.topk, 16) catch unreachable; + const tiles_per_split = std.math.divCeil(i64, tiles, cfg.splits) catch unreachable; + std.debug.assert(cfg.queries > 0 and cfg.heads > 0 and cfg.topk > 0 and cfg.page_size > 0); + std.debug.assert(cfg.splits > 0 and cfg.splits <= tiles); + const zero = b.constant(.f32, 0.0); + const one = b.constant(.f32, 1.0); + const ninf = b.constant(.f32, -std.math.inf(f32)); + const zero4 = zero.splat(4); + const one4 = one.splat(4); + const ninf4 = ninf.splat(4); + const zero_bf16 = b.constant(.i16, 0).splat(4); + const zero_words = b.constant(.i64, 0).splat(2); + const block = b.blockId(.x).to(.i64); + const split = b.blockId(.y).to(.i64); + const qi = block.divUnsigned(head_blocks); + const head_base = block.remUnsigned(head_blocks).mul(16); + const active = b.constant(.i1, @intFromBool(cfg.all_decode)).bitOr(qi.cmp(.lt, a.active.at(0).to(.i64))); + const output = if (three_d) a.partial_output else a.output; + // Inactive queries leave their preallocated output rows untouched. + var enabled = b.openIf(active); + { + const tid = b.threadId(.x).to(.i64); + const lane = tid.remUnsigned(64); + const wave = tid.divUnsigned(64); + const mn = lane.remUnsigned(16); + const kg4 = lane.divUnsigned(16).mul(4); + const wave128 = wave.mul(128); + const query_head = head_base.add(mn); + const query_word = wave.mul(32).add(lane.divUnsigned(16)); + const valid_query_head = query_head.cmpUnsigned(.lt, cfg.heads); + var q: [8]Value = undefined; + for (0..8) |i| { + var valid = b.openIfElse(valid_query_head, .{zero_bf16.type_()}); + valid.yieldThen(.{a.query.at(.{ qi, query_head, query_word.add(i * 4) }).splat(1).bitcast(.i16)}); + valid.yieldElse(.{zero_bf16}); + q[i] = valid.results[0]; + } + const tile_start = split.mul(tiles).divUnsigned(cfg.splits); + const tile_end = split.add(1).mul(tiles).divUnsigned(cfg.splits); + const loader_key = lane.divUnsigned(4); + const loader_dimension = wave128.add(lane.remUnsigned(4).mul(32)); + const scale = a.scale.at(0).splat(4); + const cache_pointer = a.kv_cache.emitIter(); + const shared = b.sharedArray(.i16, 10496, 16); + const shared_ptr = shared.emitIter(); + const atom = b.mmaAtom((fly.rocdl.MmaOpCDNA3MFMAType.get(b.ctx, .{ + .m = 16, + .n = 16, + .k = 16, + .elemTyA = .float(b.ctx, .bf16), + .elemTyB = .float(b.ctx, .bf16), + .elemTyAcc = .float(b.ctx, .f32), + }) catch return error.InvalidMlir).type_()); + var loop = b.openFor(b.constant(.i64, 0), tiles_per_split, 1, .{ + zero4, zero4, zero4, zero4, zero4, zero4, zero4, zero4, ninf4, zero4, + }); + { + const tile_index = tile_start.add(loop.iv); + const load_pos = tile_index.mul(16).add(loader_key); + const valid_index = tile_index.cmpUnsigned(.lt, tile_end).bitAnd(load_pos.cmpUnsigned(.lt, cfg.topk)); + const invalid_index = b.constant(.i32, -1); + var index = b.openIfElse(valid_index, .{invalid_index.type_()}); + index.yieldThen(.{a.indices.at(.{ qi, load_pos })}); + index.yieldElse(.{invalid_index}); + const position = index.results[0]; + const load_valid = position.cmp(.ge, 0); + // Form the word offset in i64 before multiplying cache dimensions. + const page = position.to(.i64).divUnsigned(cfg.page_size); + const slot = position.to(.i64).remUnsigned(cfg.page_size); + const cache_base = page.mul(cfg.page_size).add(slot).mul(128).add(loader_dimension.divUnsigned(4)); + for (0..4) |i| { + var valid = b.openIfElse(load_valid, .{zero_words.type_()}); + valid.yieldThen(.{loadVector(b, cache_pointer, cache_base.add(i * 2), .i64, 2)}); + valid.yieldElse(.{zero_words}); + const bits = valid.results[0].bitcast(.i16); + const column = loader_dimension.add(i * 8).divUnsigned(8).bitXor(loader_key).mul(8); + storeVector(b, shared_ptr, loader_key.mul(512).add(column), bits, .i16, 8); + } + b.barrier(); + + var qk = zero4; + for (0..8) |i| { + const dimension = wave128.add(kg4).add(i * 16); + const column = dimension.divUnsigned(8).bitXor(mn).mul(8).add(dimension.remUnsigned(8)); + const k = loadVector(b, shared_ptr, mn.mul(512).add(column), .i16, 4); + qk = b.mmaAtomCall(atom, q[i], k, qk); + } + const score_lane = lane.mul(8); + storeVector(b, shared_ptr, wave.mul(512).add(score_lane).add(8192), qk.bitcast(.i16), .i16, 8); + b.barrier(); + var score_parts: [4]Value = undefined; + for (0..4) |w| score_parts[w] = loadVector(b, shared_ptr, score_lane.add(8192 + w * 512), .i16, 8).bitcast(.f32); + const score = score_parts[0].add(score_parts[1]).add(score_parts[2].add(score_parts[3])).mul(scale); + const key_valid = load_valid.to(.i32).shuffle(.idx, mn.mul(4).to(.i32), 64).cmp(.ne, 0); + var new_m = zero4; + var new_l = zero4; + var alpha = zero4; + var probability: [4]Value = undefined; + for (0..4) |i| { + const row = kg4.add(i); + const keep = head_base.add(row).cmpUnsigned(.lt, cfg.heads).bitAnd(key_valid); + const s = keep.select(score.extract(@intCast(i)), ninf); + var max = s; + inline for (.{ 1, 2, 4, 8 }) |shift| max = max.maxNum(max.shuffleXor(shift, 16)); + const old_m = loop.carried[8].extract(@intCast(i)); + const old_l = loop.carried[9].extract(@intCast(i)); + const next_max = old_m.maxNum(max); + const m = next_max.cmpf(.gt, ninf).select(next_max, zero); + const prob = b.exp(s.sub(m)); + var sum = prob; + inline for (.{ 1, 2, 4, 8 }) |shift| sum = sum.add(sum.shuffleXor(shift, 16)); + const rescale = b.exp(old_m.sub(m)); + new_m = new_m.insert(@intCast(i), m); + new_l = new_l.insert(@intCast(i), old_l.mul(rescale).add(sum)); + alpha = alpha.insert(@intCast(i), rescale); + probability[i] = prob; + } + var write_probability = b.openIf(wave.cmp(.eq, 0)); + for (0..4) |i| { + const offset = kg4.add(i).mul(16).add(mn).add(10240); + shared.set(offset, probability[i].to(.bf16).bitcast(.i16)); + } + write_probability.yieldThen(.{}); + b.barrier(); + + const p = loadVector(b, shared_ptr, mn.mul(16).add(kg4).add(10240), .i16, 4); + var next_acc: [8]Value = undefined; + for (0..8) |n| { + const dimension = wave128.add(mn).add(n * 16); + var v = zero_bf16; + for (0..4) |k| { + const row = kg4.add(k); + const column = dimension.divUnsigned(8).bitXor(row).mul(8).add(dimension.remUnsigned(8)); + v = v.insert(@intCast(k), shared.at(row.mul(512).add(column))); + } + next_acc[n] = b.mmaAtomCall(atom, p, v, loop.carried[n].mul(alpha)); + } + // All waves finish consuming K and P before the next tile reuses LDS. + b.barrier(); + loop.yield(.{ next_acc[0], next_acc[1], next_acc[2], next_acc[3], next_acc[4], next_acc[5], next_acc[6], next_acc[7], new_m, new_l }); + } + var denominator = loop.results[9]; + var sink_alpha = one4; + if (!three_d and cfg.has_sink) { + var sink = zero4; + for (0..4) |i| { + const head = head_base.add(kg4).add(i); + var valid = b.openIfElse(head.cmpUnsigned(.lt, cfg.heads), .{ninf.type_()}); + valid.yieldThen(.{a.sink.at(head)}); + valid.yieldElse(.{ninf}); + sink = sink.insert(@intCast(i), valid.results[0]); + } + const max = loop.results[8].maxNum(sink); + const delta = loop.results[8].sub(max); + const center = sink.sub(max); + var sink_probability = zero4; + sink_alpha = zero4; + for (0..4) |i| { + sink_alpha = sink_alpha.insert(@intCast(i), b.exp(delta.extract(@intCast(i)))); + sink_probability = sink_probability.insert(@intCast(i), b.exp(center.extract(@intCast(i)))); + } + denominator = denominator.mul(sink_alpha).add(sink_probability); + } + const has_values = denominator.cmpf(.gt, zero4); + const safe_denominator = has_values.select(denominator, one4); + const inverse = one4.div(safe_denominator); + for (0..8) |n| { + const acc = if (three_d) loop.results[n] else loop.results[n].mul(sink_alpha); + const normalized = has_values.select(acc.mul(inverse), zero4); + const col = wave128.add(mn).add(n * 16); + for (0..4) |i| { + const head = head_base.add(kg4).add(i); + var valid = b.openIf(head.cmpUnsigned(.lt, cfg.heads)); + const value = normalized.extract(@intCast(i)); + if (three_d) { + output.set(.{ qi, head, split, col }, value); + } else { + output.set(.{ qi, head, col }, value.to(.bf16)); + } + valid.yieldThen(.{}); + } + } + if (three_d) { + // Keep logarithms scalar: the published ROCm lowering rejects vector log. + var log_denominator = zero4; + for (0..4) |i| log_denominator = log_denominator.insert(@intCast(i), b.log(safe_denominator.extract(@intCast(i)))); + const lse = has_values.select(loop.results[8].add(log_denominator), ninf4); + var writer = b.openIf(wave.cmp(.eq, 0).bitAnd(mn.cmp(.eq, 0))); + for (0..4) |i| { + const head = head_base.add(kg4).add(i); + var valid = b.openIf(head.cmpUnsigned(.lt, cfg.heads)); + a.partial_lse.set(.{ qi, head, split }, lse.extract(@intCast(i))); + valid.yieldThen(.{}); + } + writer.yieldThen(.{}); + } + enabled.yieldThen(.{}); + } +} + +fn runReduce(b: *fly.Builder, cfg: ReduceConfig) fly.FinishError!void { + std.debug.assert(cfg.output_splits == 1); + runReduction(b, cfg, false, Reduce.args(b)); +} + +fn runReducePartials(b: *fly.Builder, cfg: ReduceConfig) fly.FinishError!void { + std.debug.assert(!cfg.has_sink); + runReduction(b, cfg, true, ReducePartials.args(b)); +} + +fn runReduction(b: *fly.Builder, cfg: ReduceConfig, comptime grouped: bool, a: anytype) void { + const group_splits = @divExact(cfg.input_splits, cfg.output_splits); + std.debug.assert(group_splits > 0 and group_splits <= 64); + const qi = b.blockId(.x).to(.i64); + const head_group = b.blockId(.y).to(.i64); + const head = head_group.divUnsigned(cfg.output_splits); + const group = head_group.remUnsigned(cfg.output_splits); + const group_start = group.mul(group_splits); + const active = b.constant(.i1, @intFromBool(cfg.all_decode)).bitOr(qi.cmp(.lt, a.active.at(0).to(.i64))); + const zero = b.constant(.f32, 0.0); + const one = b.constant(.f32, 1.0); + const ninf = b.constant(.f32, -std.math.inf(f32)); + const zero4 = zero.splat(4); + var enabled = b.openIf(active); + { + const lane = b.threadId(.x).to(.i64); + var valid_split = b.openIfElse(lane.cmpUnsigned(.lt, group_splits), .{ninf.type_()}); + valid_split.yieldThen(.{a.input_lse.at(.{ qi, head, group_start.add(lane) })}); + valid_split.yieldElse(.{ninf}); + const lse = valid_split.results[0]; + var max = lse; + inline for (.{ 32, 16, 8, 4, 2, 1 }) |shift| max = max.maxNum(max.shuffleXor(shift, 64)); + const sink = if (!grouped and cfg.has_sink) a.sink.at(head) else ninf; + const overall_max = max.maxNum(sink); + const has_values = overall_max.cmpf(.gt, ninf); + const safe_max = has_values.select(overall_max, zero); + const weight = b.exp(lse.sub(safe_max)); + var sum = weight; + inline for (.{ 32, 16, 8, 4, 2, 1 }) |shift| sum = sum.add(sum.shuffleXor(shift, 64)); + const sink_weight = if (!grouped and cfg.has_sink) b.exp(sink.sub(safe_max)) else zero; + const denominator = sum.add(sink_weight); + const safe_denominator = denominator.cmpf(.gt, zero).select(denominator, one); + const normalizer = one.div(safe_denominator).splat(4); + const group_flat = qi.mul(cfg.heads).add(head).mul(cfg.input_splits).add(group_start); + const dim0 = lane.mul(8); + var loop = b.openFor(b.constant(.i64, 0), group_splits, 1, .{ zero4, zero4 }); + { + const weight4 = weight.shuffle(.idx, loop.iv.to(.i32), 64).splat(4); + const base = group_flat.add(loop.iv).mul(512).add(dim0); + const part0 = loadVector(b, a.input.emitIter(), base, .f32, 4); + const part1 = loadVector(b, a.input.emitIter(), base.add(4), .f32, 4); + loop.yield(.{ loop.carried[0].add(part0.mul(weight4)), loop.carried[1].add(part1.mul(weight4)) }); + } + const output = if (grouped) a.partial_output else a.output; + const row = qi.mul(cfg.heads).add(head); + const base = if (grouped) row.mul(cfg.output_splits).add(group).mul(512) else row.mul(512); + for (0..2) |i| { + const value = has_values.select(loop.results[i].mul(normalizer), zero4); + const offset = base.add(dim0).add(i * 4); + if (grouped) { + storeVector(b, output.emitIter(), offset, value, .f32, 4); + } else { + storeVector(b, output.emitIter(), offset, value.to(.bf16), .bf16, 4); + } + } + if (grouped) { + var writer = b.openIf(lane.cmp(.eq, 0)); + const lse_value = has_values.select(safe_max.add(b.log(safe_denominator)), ninf); + a.partial_lse.set(.{ qi, head, group }, lse_value); + writer.yieldThen(.{}); + } + enabled.yieldThen(.{}); + } +} diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index c2f73895..e80885a8 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -7,6 +7,7 @@ const flashattn = @import("flashattn.zig"); const metal = @import("metal_attention.zig"); const tpu = @import("tpu_attention.zig"); const triton = @import("triton_attention.zig"); +const sparse_mla = @import("sparse_mla.zig"); const PagedAttention = @This(); @@ -820,7 +821,10 @@ pub fn partialSoftmax(self: zml.Tensor, axis: anytype) PartialSoftmax { } pub const Mla = struct { + pub const Backend = sparse_mla.Backend; + pub const Options = struct { + backend: Mla.Backend = .triton, rope_rank: i64, value_rank: i64, scale: ?f32 = null, @@ -875,7 +879,7 @@ pub const Mla = struct { stdx.debug.assert(latent_kv.dim(.hd) == q.dim(.hd), "expected q and kv cache head dims to match, got q={} kv={}", .{ q.dim(.hd), latent_kv.dim(.hd) }); return switch (parameters) { - .triton => |triton_parameters| triton.paged.pagedSparseMla(triton_parameters, q, latent_kv, sink, topk, tokens_pos, opts), + .triton => |triton_parameters| sparse_mla.pagedAttention(triton_parameters, q, latent_kv, sink, topk, tokens_pos, opts), .stablehlo => |stablehlo_parameters| stablehlo_pagedSparseAttention( q, latent_kv, diff --git a/zml/attention/sparse_mla.zig b/zml/attention/sparse_mla.zig new file mode 100644 index 00000000..964bd571 --- /dev/null +++ b/zml/attention/sparse_mla.zig @@ -0,0 +1,204 @@ +const std = @import("std"); +const stdx = @import("stdx"); +const zml = @import("../zml.zig"); +const triton = @import("triton_attention.zig"); +const fly = @import("fly_kernels/sparse_mla.zig"); +const MlaOptions = @import("paged_attention.zig").Mla.Options; + +pub const Backend = enum { + triton, + fly, + + pub fn auto(platform: *const zml.Platform, dtype: zml.DataType) Backend { + return switch (platform.target) { + .rocm => switch (zml.platform.rocm.computeCapability(platform) orelse return .triton) { + .gfx942 => switch (dtype) { + .bf16 => .fly, + else => .triton, + }, + else => .triton, + }, + else => .triton, + }; + } + + fn call(self: Backend, q: zml.Tensor, kv_cache: zml.Tensor, sink: ?zml.Tensor, topk: zml.Tensor, active_query_count: zml.Tensor, opts: Options) zml.Tensor { + const backend = switch (self) { + .fly => auto(zml.Compiler.current().platform, q.dtype()), + .triton => .triton, + }; + switch (backend) { + .fly => if (fly.pagedAttention(q, kv_cache, sink, topk, active_query_count, opts, triton.getCuCount())) |output| return output, + .triton => {}, + } + return triton.paged.pagedSparseMlaKernel(q, kv_cache, sink, topk, active_query_count, opts); + } +}; + +pub const Options = struct { + qk_rank: usize, + value_rank: usize, + num_heads: usize, + block_size: usize, + rope_rank: usize, + scale: ?f32, + total_q_blocks: usize, + num_kv_splits: ?u8, + all_decode: bool, +}; + +pub const Config = struct { + block_m: usize, + tile_size: usize, + num_splits: usize, + direct_programs: usize, + main_num_warps: usize = 4, + main_num_stages: usize = 2, + grouped_reduce_threshold: usize = 32, + splits_per_group: usize = 16, + grouped_reduce_num_warps: usize = 1, + parallel_reduce_min_splits: usize = std.math.maxInt(usize), + parallel_reduce_num_warps: usize = 1, +}; + +pub fn launchConfig(paged_opts: Options, topk_count: usize, cu_count_: usize) Config { + stdx.debug.assert(paged_opts.total_q_blocks > 0, "sparse MLA requires at least one query", .{}); + stdx.debug.assert(paged_opts.num_heads > 0, "sparse MLA requires at least one query head", .{}); + stdx.debug.assert(topk_count > 0, "sparse MLA requires at least one top-k entry", .{}); + + const padded_heads = std.math.ceilPowerOfTwoAssert(usize, paged_opts.num_heads); + const padded_topk = std.math.ceilPowerOfTwoAssert(usize, topk_count); + var config: Config = .{ + .block_m = @min(padded_heads, 16), + .tile_size = @min(padded_topk, 16), + .num_splits = 1, + .direct_programs = undefined, + }; + const cc = zml.platform.cuda.computeCapability(zml.Compiler.current().platform); + const is_sm103 = if (cc) |value| value.eql(.{ .major = 10, .minor = 3 }) else false; + + // GB300 (sm_103): a single wide query benefits from more head blocks and + // wider sparse tiles. The split selection below still derives each layer's + // split count from query/head/top-k shapes (16 for DSV4 CSA/HCA and 4 for + // its 128-entry full-attention layers). + if (is_sm103 and + paged_opts.all_decode and + paged_opts.total_q_blocks == 1 and + paged_opts.value_rank >= 512) + { + config.block_m = @min(padded_heads, 8); + config.tile_size = @min(padded_topk, 32); + config.parallel_reduce_min_splits = 16; + config.parallel_reduce_num_warps = 2; + } + // Full 256-query prefill chunks already expose enough query parallelism. + // Retain BLOCK_M=16 and the generic split model, but process wider sparse + // tiles. The predicate uses the post-sharding flattened query shape. + if (is_sm103 and + !paged_opts.all_decode and + paged_opts.total_q_blocks >= 256 and + paged_opts.value_rank >= 512) + { + config.tile_size = @min(padded_topk, 32); + } + + const num_tiles = std.math.divCeil(usize, topk_count, config.tile_size) catch unreachable; + const head_blocks = std.math.divCeil(usize, paged_opts.num_heads, config.block_m) catch unreachable; + config.direct_programs = paged_opts.total_q_blocks * head_blocks; + const cu_count = @max(cu_count_, 1); + + if (paged_opts.num_kv_splits) |requested| { + const num_splits: usize = requested; + stdx.debug.assert(std.math.isPowerOfTwo(num_splits), "MLA num_kv_splits ({}) must be a power of two", .{num_splits}); + stdx.debug.assert(num_splits <= 128, "MLA num_kv_splits ({}) must not exceed 128", .{num_splits}); + stdx.debug.assert(num_splits <= num_tiles, "MLA num_kv_splits ({}) must not exceed sparse tile count ({})", .{ num_splits, num_tiles }); + config.num_splits = num_splits; + } else { + var best_cost: usize = std.math.maxInt(usize); + const candidates = [_]usize{ 1, 2, 4, 8, 16, 32, 64, 128 }; + for (candidates) |num_splits| { + if (num_splits > num_tiles) break; + const programs = config.direct_programs * num_splits; + const rounds = std.math.divCeil(usize, programs, cu_count) catch unreachable; + const tiles_per_program = std.math.divCeil(usize, num_tiles, num_splits) catch unreachable; + const cost = rounds * tiles_per_program; + // Candidates are ordered by split count, so ties retain 2D or the lower-overhead 3D launch. + if (cost < best_cost) { + best_cost = cost; + config.num_splits = num_splits; + } + } + } + + return config; +} + +pub fn pagedAttention(parameters: triton.paged.Parameters, q: zml.Tensor, kv_cache: zml.Tensor, sink: ?zml.Tensor, topk: zml.Tensor, tokens_pos: zml.Tensor, opts: MlaOptions) zml.Tensor { + const output_shape = q.shape().set(.hd, opts.value_rank); + return zml.ops.manualComputation( + (struct { + q: zml.Tensor, + kv_cache: zml.Tensor, + sink: ?zml.Tensor, + topk: zml.Tensor, + tokens_pos: zml.Tensor, + block_table: zml.Tensor, + seq_lens: zml.Tensor, + query_start_len: zml.Tensor, + opts: MlaOptions, + options: triton.paged.Options, + + fn body(self: @This(), _: zml.Shape) zml.Tensor { + const block_size = self.kv_cache.dim(.k_chunk); + + const parameters_: triton.paged.Parameters = .{ + .block_table = self.block_table, + .seq_lens = self.seq_lens, + .query_start_len = self.query_start_len, + .options_ = self.options, + }; + + const topk_final = triton.paged.topkToPhysical(parameters_, self.topk, self.tokens_pos, block_size); + const active_query_count = self.query_start_len + .slice(.b, .{ .start = self.query_start_len.dim(.b) - 1 }) + .squeeze(.b); + stdx.debug.assert(topk_final.dim(.q) == self.q.dim(.q), "expected topk q dim ({}) to match q dim ({})", .{ topk_final.dim(.q), self.q.dim(.q) }); + + const num_heads: usize = @intCast(self.q.dim(.h)); + const paged_opts: Options = .{ + .qk_rank = @intCast(self.q.dim(.hd)), + .value_rank = @intCast(self.opts.value_rank), + .num_heads = num_heads, + .block_size = @intCast(self.kv_cache.dim(.k_chunk)), + .rope_rank = @intCast(self.opts.rope_rank), + .scale = self.opts.scale, + .total_q_blocks = @intCast(self.q.dim(.q)), + .num_kv_splits = self.opts.num_kv_splits, + .all_decode = !self.options.is_prefill, + }; + + return self.opts.backend.call( + self.q, + self.kv_cache, + self.sink, + topk_final, + active_query_count, + paged_opts, + ); + } + }).body, + .{ + .q = q, + .kv_cache = kv_cache, + .sink = sink, + .topk = topk, + .tokens_pos = tokens_pos, + .block_table = parameters.block_table, + .seq_lens = parameters.seq_lens, + .query_start_len = parameters.query_start_len, + .opts = opts, + .options = parameters.options_, + }, + output_shape, + ); +} diff --git a/zml/attention/triton_attention.zig b/zml/attention/triton_attention.zig index 360cc87c..c01b380a 100644 --- a/zml/attention/triton_attention.zig +++ b/zml/attention/triton_attention.zig @@ -9,6 +9,7 @@ const MlaOptions = @import("paged_attention.zig").Mla.Options; const kernels = @import("triton_kernels/unified_attention.zig"); const kernels_oneapi = @import("triton_kernels/unified_attention_oneapi.zig"); const mla_kernels = @import("triton_kernels/unified_sparse_mla.zig"); +const sparse_mla = @import("sparse_mla.zig"); const log = std.log.scoped(.@"zml/attention/triton"); @@ -38,96 +39,11 @@ pub const Config2D = struct { total_q_blocks: usize, }; -const SparseMlaConfig = struct { - block_m: usize, - tile_size: usize, - num_splits: usize, - direct_programs: usize, - main_num_warps: usize = 4, - main_num_stages: usize = 2, - grouped_reduce_threshold: usize = 32, - splits_per_group: usize = 16, - grouped_reduce_num_warps: usize = 1, - parallel_reduce_min_splits: usize = std.math.maxInt(usize), - parallel_reduce_num_warps: usize = 1, -}; - fn isCudaComputeCapability(expected: zml.platform.cuda.ComputeCapability) bool { const platform = zml.module.CompilationContext.current().platform; return if (zml.platform.cuda.computeCapability(platform)) |cc| cc.eql(expected) else false; } -fn sparseMlaConfig(paged_opts: paged.PagedSparseMlaOptions, topk_count: usize, cu_count_: usize) SparseMlaConfig { - stdx.debug.assert(paged_opts.total_q_blocks > 0, "sparse MLA requires at least one query", .{}); - stdx.debug.assert(paged_opts.num_heads > 0, "sparse MLA requires at least one query head", .{}); - stdx.debug.assert(topk_count > 0, "sparse MLA requires at least one top-k entry", .{}); - - const padded_heads = std.math.ceilPowerOfTwoAssert(usize, paged_opts.num_heads); - const padded_topk = std.math.ceilPowerOfTwoAssert(usize, topk_count); - var config: SparseMlaConfig = .{ - .block_m = @min(padded_heads, 16), - .tile_size = @min(padded_topk, 16), - .num_splits = 1, - .direct_programs = undefined, - }; - const is_sm103 = isCudaComputeCapability(.{ .major = 10, .minor = 3 }); - - // GB300 (sm_103): a single wide query benefits from more head blocks and - // wider sparse tiles. The split selection below still derives each layer's - // split count from query/head/top-k shapes (16 for DSV4 CSA/HCA and 4 for - // its 128-entry full-attention layers). - if (is_sm103 and - paged_opts.all_decode and - paged_opts.total_q_blocks == 1 and - paged_opts.value_rank >= 512) - { - config.block_m = @min(padded_heads, 8); - config.tile_size = @min(padded_topk, 32); - config.parallel_reduce_min_splits = 16; - config.parallel_reduce_num_warps = 2; - } - // Full 256-query prefill chunks already expose enough query parallelism. - // Retain BLOCK_M=16 and the generic split model, but process wider sparse - // tiles. The predicate uses the post-sharding flattened query shape. - if (is_sm103 and - !paged_opts.all_decode and - paged_opts.total_q_blocks >= 256 and - paged_opts.value_rank >= 512) - { - config.tile_size = @min(padded_topk, 32); - } - - const num_tiles = std.math.divCeil(usize, topk_count, config.tile_size) catch unreachable; - const head_blocks = std.math.divCeil(usize, paged_opts.num_heads, config.block_m) catch unreachable; - config.direct_programs = paged_opts.total_q_blocks * head_blocks; - const cu_count = @max(cu_count_, 1); - - if (paged_opts.num_kv_splits) |requested| { - const num_splits: usize = requested; - stdx.debug.assert(std.math.isPowerOfTwo(num_splits), "MLA num_kv_splits ({}) must be a power of two", .{num_splits}); - stdx.debug.assert(num_splits <= 128, "MLA num_kv_splits ({}) must not exceed 128", .{num_splits}); - stdx.debug.assert(num_splits <= num_tiles, "MLA num_kv_splits ({}) must not exceed sparse tile count ({})", .{ num_splits, num_tiles }); - config.num_splits = num_splits; - } else { - var best_cost: usize = std.math.maxInt(usize); - const candidates = [_]usize{ 1, 2, 4, 8, 16, 32, 64, 128 }; - for (candidates) |num_splits| { - if (num_splits > num_tiles) break; - const programs = config.direct_programs * num_splits; - const rounds = std.math.divCeil(usize, programs, cu_count) catch unreachable; - const tiles_per_program = std.math.divCeil(usize, num_tiles, num_splits) catch unreachable; - const cost = rounds * tiles_per_program; - // Candidates are ordered by split count, so ties retain 2D or the lower-overhead 3D launch. - if (cost < best_cost) { - best_cost = cost; - config.num_splits = num_splits; - } - } - } - - return config; -} - /// Largest `BLOCK_M * HEAD_SIZE_PADDED` the 2D kernel survives. That product is /// the shape of both the query tile and the f32 accumulator carried through the /// tile loop; above it the kernel faults with CUDA_ERROR_MISALIGNED_ADDRESS. @@ -252,7 +168,7 @@ fn select3dConfig(options: paged.PagedAttentionOptions) Config3D { }; } -fn getCuCount() usize { +pub fn getCuCount() usize { const platform = zml.Compiler.current().platform; if (platform.devices.len == 0) return 1; const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return 1; @@ -755,7 +671,7 @@ pub const paged = struct { return output.output; } - fn pagedSparseMlaKernel(q: zml.Tensor, kv_cache: zml.Tensor, sink: ?zml.Tensor, topk: zml.Tensor, active_query_count: zml.Tensor, paged_opts: PagedSparseMlaOptions) zml.Tensor { + pub fn pagedSparseMlaKernel(q: zml.Tensor, kv_cache: zml.Tensor, sink: ?zml.Tensor, topk: zml.Tensor, active_query_count: zml.Tensor, paged_opts: PagedSparseMlaOptions) zml.Tensor { const value_rank: i64 = @intCast(paged_opts.value_rank); const rope_rank: i64 = if (paged_opts.value_rank == paged_opts.qk_rank) 0 else @intCast(paged_opts.rope_rank); stdx.debug.assert(std.math.isPowerOfTwo(paged_opts.value_rank), "sparse MLA value rank ({}) must be a power of two", .{paged_opts.value_rank}); @@ -772,7 +688,7 @@ pub const paged = struct { const use_sink = sink != null; const sm_scale: f32 = paged_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(paged_opts.qk_rank)))); - const config = sparseMlaConfig(paged_opts, @intCast(topk.dim(.topk)), getCuCount()); + const config = sparse_mla.launchConfig(paged_opts, @intCast(topk.dim(.topk)), getCuCount()); const kernel_config: mla_kernels.Config = .{ .q_dtype = triton.from(q.dtype()), @@ -965,86 +881,10 @@ pub const paged = struct { return zml.Tensor.select(valid_topk, physical_topk, zml.Tensor.scalar(-1, .i32).broad(topk_shape)); } - pub const PagedSparseMlaOptions = struct { - qk_rank: usize, - value_rank: usize, - num_heads: usize, - block_size: usize, - rope_rank: usize, - scale: ?f32, - total_q_blocks: usize, - num_kv_splits: ?u8, - all_decode: bool, - }; + pub const PagedSparseMlaOptions = sparse_mla.Options; pub fn pagedSparseMla(parameters: Parameters, q: zml.Tensor, kv_cache: zml.Tensor, sink: ?zml.Tensor, topk: zml.Tensor, tokens_pos: zml.Tensor, opts: MlaOptions) zml.Tensor { - const output_shape = q.shape().set(.hd, opts.value_rank); - return zml.ops.manualComputation( - (struct { - q: zml.Tensor, - kv_cache: zml.Tensor, - sink: ?zml.Tensor, - topk: zml.Tensor, - tokens_pos: zml.Tensor, - block_table: zml.Tensor, - seq_lens: zml.Tensor, - query_start_len: zml.Tensor, - opts: MlaOptions, - options: Options, - - fn body(self: @This(), _: zml.Shape) zml.Tensor { - const block_size = self.kv_cache.dim(.k_chunk); - - const parameters_: Parameters = .{ - .block_table = self.block_table, - .seq_lens = self.seq_lens, - .query_start_len = self.query_start_len, - .options_ = self.options, - }; - - const topk_final = topkToPhysical(parameters_, self.topk, self.tokens_pos, block_size); - const active_query_count = self.query_start_len - .slice(.b, .{ .start = self.query_start_len.dim(.b) - 1 }) - .squeeze(.b); - stdx.debug.assert(topk_final.dim(.q) == self.q.dim(.q), "expected topk q dim ({}) to match q dim ({})", .{ topk_final.dim(.q), self.q.dim(.q) }); - - const num_heads: usize = @intCast(self.q.dim(.h)); - const paged_opts: PagedSparseMlaOptions = .{ - .qk_rank = @intCast(self.q.dim(.hd)), - .value_rank = @intCast(self.opts.value_rank), - .num_heads = num_heads, - .block_size = @intCast(self.kv_cache.dim(.k_chunk)), - .rope_rank = @intCast(self.opts.rope_rank), - .scale = self.opts.scale, - .total_q_blocks = @intCast(self.q.dim(.q)), - .num_kv_splits = self.opts.num_kv_splits, - .all_decode = !self.options.is_prefill, - }; - - return pagedSparseMlaKernel( - self.q, - self.kv_cache, - self.sink, - topk_final, - active_query_count, - paged_opts, - ); - } - }).body, - .{ - .q = q, - .kv_cache = kv_cache, - .sink = sink, - .topk = topk, - .tokens_pos = tokens_pos, - .block_table = parameters.block_table, - .seq_lens = parameters.seq_lens, - .query_start_len = parameters.query_start_len, - .opts = opts, - .options = parameters.options_, - }, - output_shape, - ); + return sparse_mla.pagedAttention(parameters, q, kv_cache, sink, topk, tokens_pos, opts); } }; diff --git a/zml/kernel.zig b/zml/kernel.zig index ca481d65..6535b318 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -539,6 +539,7 @@ pub const fly = struct { pub const TiledMma = fly_builder.TiledMma; pub const Arch = fly_builder.Arch; pub const MmaFlavor = fly_builder.MmaFlavor; + pub const rocdl = fly_builder.rocdl; /// The matrix atom this device provides, or null when it has none: WMMA /// arrived with RDNA3, so gfx1030 has no matrix instruction at all. diff --git a/zml/moe/fly_kernels/moe.zig b/zml/moe/fly_kernels/moe.zig new file mode 100644 index 00000000..3a287fcb --- /dev/null +++ b/zml/moe/fly_kernels/moe.zig @@ -0,0 +1,289 @@ +const std = @import("std"); +const zml = @import("../../zml.zig"); +const shared = @import("../fused_experts.zig"); +const mxfp4 = @import("mxfp4.zig"); +const Tensor = zml.Tensor; +const Shape = zml.Shape; +const fly = zml.kernel.fly; +const Builder = fly.Builder; + +pub const Parameters = shared.Parameters; + +pub fn supports(opts: shared.FusedExpertsArgs) bool { + if (opts.hidden_states.dtype() != .bf16 or opts.gate_up.quantizationScheme() != .mxfp4 or + opts.down.quantizationScheme() != .mxfp4 or opts.gate_up.bias != null or opts.down.bias != null or + opts.routing_weight_placement != .before_down) return false; + const gate_up = opts.gate_up.weight; + const down = opts.down.weight; + if (gate_up.rank() != 3 or down.rank() != 3 or opts.hidden_states.dim(.d) != 5120) return false; + const gate_up_scale = opts.gate_up.quantizationScales() orelse return false; + const down_scale = opts.down.quantizationScales() orelse return false; + const experts = if (opts.expert_map) |map| map.dim(.expert) else gate_up.dim(0); + if (experts < 1 or experts > 512) return false; + inline for (.{ .{ gate_up, gate_up_scale, 4608, 5120 }, .{ down, down_scale, 5120, 2304 } }) |projection| { + const weight, const scale, const n, const k = projection; + const packing_factor: i64 = switch (weight.dtype()) { + .u8, .i8 => 2, + .f4e2m1 => 1, + else => return false, + }; + if (weight.dim(0) < 1 or weight.dim(0) > 512 or weight.dim(1) != n or weight.dim(2) * packing_factor != k or + scale.rank() != 3 or scale.dim(0) != weight.dim(0) or scale.dim(1) != n or scale.dim(2) != k / 32) return false; + switch (scale.dtype()) { + .u8, .i8, .f8e8m0 => {}, + else => return false, + } + } + const routes = opts.hidden_states.dim(.b) * opts.hidden_states.dim(.s) * opts.topk_ids.dim(.top_expert); + return routes > 0 and routes <= std.math.maxInt(i32) / 5120; +} + +pub fn prepareInput(x: Tensor, scheme: ?zml.Quantization.Scheme, _: zml.DataType, _: bool) struct { Tensor, ?Tensor } { + std.debug.assert(scheme == .mxfp4); + return .{ x.convert(.bf16), null }; +} + +pub fn call(opts: shared.GemmOptions) zml.Tensor { + zml.stdx.debug.assert(opts.quant_scheme == .mxfp4 and opts.bias == null and opts.routing_weights == null and opts.input_scale == null, "Fly MoE requires MXFP4 weights, BF16 activations and routing weights before the down projection", .{}); + const weight = if (opts.weight.dtype() == .f4e2m1) + opts.weight.reshape(opts.weight.shape().setDim(2, @divExact(opts.weight.dim(2), 2)).append(.{ .nibble = 2 })).bitCast(.u8) + else + opts.weight; + return mxfp4.call(opts.input, weight, opts.weight_scale.?.bitCast(.u8), opts.routing.sorted_token_ids, opts.expert_ids, opts.routing.num_tokens_post_padded, opts.output_shape, opts.top_k, opts.routing.naive_block_assignment, opts.launch_config.block_size_m); +} + +pub fn prepareRouting(topk_ids: Tensor, num_experts: i64, block_size_m: i64) shared.Routing { + std.debug.assert(num_experts > 0 and num_experts <= 512); + std.debug.assert(block_size_m > 0 and block_size_m <= 128 and @mod(block_size_m, 16) == 0); + const ids = topk_ids.withTags(.{ .token, .topk }).convert(.i32); + const num_assignments = ids.dim(.token) * ids.dim(.topk); + // GEMV handles up to six routes; MFMA shares weights across sorted rows. + const naive_block_assignment = num_assignments <= 6; + const max_num_tokens_padded = if (naive_block_assignment or num_assignments < num_experts) + num_assignments * block_size_m + else + (std.math.divCeil(i64, num_assignments + num_experts * (block_size_m - 1), block_size_m) catch unreachable) * block_size_m; + const sorted_token_ids, const expert_ids, const num_tokens_post_padded = if (naive_block_assignment) blk: { + break :blk .{ + Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)), + ids.reshape(.{ .g = num_assignments }), + Tensor.constant(.{ .i32 = @as(i32, @intCast(max_num_tokens_padded)) }).reshape(.{1}), + }; + } else blk: { + const flat_experts = ids.reshape(.{ .g = num_assignments }); + const aligned = Align.call( + flat_experts, + Tensor.zeroes(Shape.init(.{ .g = max_num_tokens_padded }, .i32)), + Tensor.zeroes(Shape.init(.{ .g = @divExact(max_num_tokens_padded, block_size_m) }, .i32)), + Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)), + Tensor.zeroes(Shape.init(.{ .g = num_experts + 1 }, .i32)), + block_size_m, + ); + const sorted = Sort.call(flat_experts, aligned.sorted_token_ids, aligned.cumsum); + break :blk .{ sorted.sorted_token_ids, aligned.expert_ids, aligned.num_tokens_post_pad }; + }; + return .{ + .sorted_token_ids = sorted_token_ids, + .expert_ids = expert_ids, + .num_tokens_post_padded = num_tokens_post_padded, + .max_num_tokens_padded = max_num_tokens_padded, + .num_assignments = num_assignments, + .naive_block_assignment = naive_block_assignment, + }; +} + +const Align = struct { + const Result = struct { + sorted_token_ids: Tensor, + expert_ids: Tensor, + num_tokens_post_pad: Tensor, + cumsum: Tensor, + }; + + fn supports(ids: Tensor, sorted: Tensor, experts: Tensor, total: Tensor, cumsum: Tensor, block_size: i64) bool { + for ([_]Tensor{ ids, sorted, experts, total, cumsum }) |tensor| { + if (tensor.rank() != 1 or tensor.dtype() != .i32) return false; + } + const num_experts = cumsum.dim(0) - 1; + if (num_experts < 1 or num_experts > 512 or total.dim(0) != 1 or ids.dim(0) > std.math.maxInt(i32) or + block_size <= 0 or block_size > 128 or @mod(block_size, 16) != 0) return false; + const capacity_bound = if (ids.dim(0) < num_experts) + ids.dim(0) * block_size + else + @divTrunc(ids.dim(0) + num_experts * (block_size - 1) + block_size - 1, block_size) * block_size; + return capacity_bound <= std.math.maxInt(i32) and sorted.dim(0) >= capacity_bound and + sorted.dim(0) <= std.math.maxInt(i32) and @mod(sorted.dim(0), block_size) == 0 and + experts.dim(0) == @divTrunc(sorted.dim(0), block_size); + } + + /// All four outputs alias their corresponding input buffers. Invalid expert IDs + /// are ignored, and all unused expert blocks are marked -1. Prefix ends in LDS + /// let each thread find its output block's expert without scanning every expert. + fn call(ids: Tensor, sorted: Tensor, experts: Tensor, total: Tensor, cumsum: Tensor, block_size: i64) Result { + std.debug.assert(Align.supports(ids, sorted, experts, total, cumsum, block_size)); + const cfg: Config = .{ + .routes = ids.dim(0), + .experts = cumsum.dim(0) - 1, + .capacity = sorted.dim(0), + .blocks = experts.dim(0), + .block_size = block_size, + .padding = block_size - 1, + }; + const result = Kernel.call(.{ .ids = ids, .sorted = sorted, .experts = experts, .total = total, .cumsum_in = cumsum }, .{ .sorted_token_ids = sorted.shape(), .expert_ids = experts.shape(), .num_tokens_post_pad = total.shape(), .cumsum = cumsum.shape() }, .{ + .cfg = cfg, + .threads = 512, + .grid = .{ 2, 1, 1 }, + .output_operand_aliases = .{ .sorted_token_ids = .sorted, .expert_ids = .experts, .num_tokens_post_pad = .total, .cumsum = .cumsum_in }, + }); + return .{ .sorted_token_ids = result.sorted_token_ids, .expert_ids = result.expert_ids, .num_tokens_post_pad = result.num_tokens_post_pad, .cumsum = result.cumsum }; + } + + const Config = struct { + routes: i64, + experts: i64, + capacity: i64, + blocks: i64, + block_size: i64, + padding: i64, + }; + + const Kernel = zml.kernel.fly.Kernel(Config, .{ + .name = "fly_moe_align", + .inputs = &.{ "ids", "sorted", "experts", "total", "cumsum_in" }, + .outputs = &.{ "sorted_token_ids", "expert_ids", "num_tokens_post_pad", "cumsum" }, + .run = run, + }); + + fn run(b: *Builder, cfg: Config) fly.FinishError!void { + const args = Kernel.args(b); + const tid = b.threadId(.x); + const tid64 = tid.to(.i64); + const zero = b.constant(.i32, 0); + const one = b.constant(.i32, 1); + const experts = b.constant(.i32, cfg.experts); + // Distinct arrays match the histogram, inclusive ends and wave totals. + const counts = b.sharedArray(.i32, 512, 16); + const ends = b.sharedArray(.i32, 512, 16); + const waves = b.sharedArray(.i32, 8, 16); + var fill = b.openIfElse(b.blockId(.x).cmp(.ne, 0), .{}); + var positions = b.openFor(tid64, cfg.capacity, 512, .{}); + args.sorted_token_ids.set(positions.iv, b.constant(.i32, cfg.routes)); + positions.yield(.{}); + fill.yieldThen(.{}); + + counts.set(tid, zero); + b.barrier(); + var routes = b.openFor(tid64, cfg.routes, 512, .{}); + const id = args.ids.at(routes.iv); + var valid_id = b.openIf(id.cmp(.ge, 0).bitAnd(id.cmp(.lt, cfg.experts))); + _ = b.ptrAtomicAdd(counts.emitIter().addOffset(id), one, .workgroup); + valid_id.yieldThen(.{}); + routes.yield(.{}); + b.barrier(); + + const padded = counts.at(tid).add(cfg.padding).divUnsigned(cfg.block_size).mul(cfg.block_size); + const lane = tid.remUnsigned(64); + const wave = tid.divUnsigned(64); + var prefix = padded; + inline for (.{ 1, 2, 4, 8, 16, 32 }) |shift| { + const peer = prefix.shuffle(.up, shift, 64); + prefix = prefix.add(lane.cmpUnsigned(.ge, shift).select(peer, zero)); + } + var last_lane = b.openIf(lane.cmp(.eq, 63)); + waves.set(wave, prefix); + last_lane.yieldThen(.{}); + b.barrier(); + + var sums = b.openFor(zero, 8, 1, .{ zero, zero }); + const value = waves.at(sums.iv); + const preceding = sums.carried[0].add(sums.iv.cmpUnsigned(.lt, wave).select(value, zero)); + const total = sums.carried[1].add(value); + sums.yield(.{ preceding, total }); + const end = sums.results[0].add(prefix); + const start = end.sub(padded); + ends.set(tid, end); + var has_expert = b.openIf(tid.cmpUnsigned(.lt, cfg.experts)); + args.cumsum.set(tid64, start); + has_expert.yieldThen(.{}); + var first = b.openIf(tid.cmp(.eq, 0)); + args.cumsum.set(b.constant(.i64, cfg.experts), sums.results[1]); + args.num_tokens_post_pad.set(b.constant(.i64, 0), sums.results[1]); + first.yieldThen(.{}); + b.barrier(); + + var blocks = b.openFor(tid64, cfg.blocks, 512, .{}); + const offset = blocks.iv.to(.i32).mul(cfg.block_size); + var block_expert = b.openIfElse(offset.cmpUnsigned(.lt, sums.results[1]), .{zero.type_()}); + // Upper bound of inclusive expert ends handles empty experts without a scan. + var bounds = b.openWhile(.{ zero, experts }, .{ zero.type_(), zero.type_() }); + bounds.yieldBefore(bounds.before_carried[0].cmpUnsigned(.lt, bounds.before_carried[1]), .{ bounds.before_carried[0], bounds.before_carried[1] }); + const lo = bounds.after_carried[0]; + const hi = bounds.after_carried[1]; + const mid = lo.add(hi).shrU(1); + const advance = ends.at(mid).cmpUnsigned(.le, offset); + bounds.yieldAfter(.{ advance.select(mid.add(1), lo), advance.select(hi, mid) }); + block_expert.yieldThen(.{bounds.results[0]}); + block_expert.yieldElse(.{b.constant(.i32, -1)}); + args.expert_ids.set(blocks.iv, block_expert.results[0]); + blocks.yield(.{}); + fill.yieldElse(.{}); + } +}; + +const Sort = struct { + const Result = struct { + sorted_token_ids: Tensor, + cumsum: Tensor, + }; + + fn supports(ids: Tensor, sorted_token_ids: Tensor, cumsum: Tensor) bool { + return ids.rank() == 1 and sorted_token_ids.rank() == 1 and cumsum.rank() == 1 and + ids.dtype() == .i32 and sorted_token_ids.dtype() == .i32 and cumsum.dtype() == .i32 and + cumsum.dim(0) >= 2 and cumsum.dim(0) <= std.math.maxInt(i32) and + ids.dim(0) <= std.math.maxInt(i32) and sorted_token_ids.dim(0) >= ids.dim(0); + } + + /// cumsum contains one starting offset per expert followed by the total padded + /// capacity. Each valid route atomically advances its expert's cursor; ordering + /// within an expert is unspecified. Invalid expert IDs leave both buffers alone. + fn call(ids: Tensor, sorted_token_ids: Tensor, cumsum: Tensor) Result { + std.debug.assert(Sort.supports(ids, sorted_token_ids, cumsum)); + if (ids.dim(0) == 0) return .{ .sorted_token_ids = sorted_token_ids, .cumsum = cumsum }; + const routes = ids.dim(0); + const grid = @min(@divTrunc(routes + 255, 256), 65535); + const cfg: Config = .{ .routes = routes, .experts = cumsum.dim(0) - 1, .step = grid * 256 }; + const result = Kernel.call(.{ .ids = ids, .sorted = sorted_token_ids, .cumsum_in = cumsum }, .{ .sorted_token_ids = sorted_token_ids.shape(), .cumsum = cumsum.shape() }, .{ + .cfg = cfg, + .threads = 256, + .grid = .{ @intCast(grid), 1, 1 }, + .output_operand_aliases = .{ .sorted_token_ids = .sorted, .cumsum = .cumsum_in }, + }); + return .{ .sorted_token_ids = result.sorted_token_ids, .cumsum = result.cumsum }; + } + + const Config = struct { + routes: i64, + experts: i64, + step: i64, + }; + + const Kernel = zml.kernel.fly.Kernel(Config, .{ + .name = "fly_count_and_sort_expert_tokens", + .inputs = &.{ "ids", "sorted", "cumsum_in" }, + .outputs = &.{ "sorted_token_ids", "cumsum" }, + .run = run, + }); + + fn run(b: *Builder, cfg: Config) fly.FinishError!void { + const args = Kernel.args(b); + const start = b.blockId(.x).to(.i64).mul(256).add(b.threadId(.x).to(.i64)); + var routes = b.openFor(start, cfg.routes, cfg.step, .{}); + const id = args.ids.at(routes.iv); + var valid = b.openIf(id.cmp(.ge, 0).bitAnd(id.cmp(.lt, cfg.experts))); + const cursor = args.cumsum.emitIter().addOffset(id.to(.i64)); + const rank = b.ptrAtomicAdd(cursor, b.constant(.i32, 1), .agent); + args.sorted_token_ids.set(rank.to(.i64), routes.iv.to(.i32)); + valid.yieldThen(.{}); + routes.yield(.{}); + } +}; diff --git a/zml/moe/fly_kernels/mxfp4.zig b/zml/moe/fly_kernels/mxfp4.zig new file mode 100644 index 00000000..13c10bf6 --- /dev/null +++ b/zml/moe/fly_kernels/mxfp4.zig @@ -0,0 +1,462 @@ +const std = @import("std"); +const zml = @import("../../zml.zig"); +const Tensor = zml.Tensor; +const fly = zml.kernel.fly; +const Value = fly.Value; +const Builder = fly.Builder; + +/// The caller owns route localization, activation and reduction. +/// `expert_ids` contains local bank indices, including -1 for remote experts. +/// Sorted routes must contain a real prefix followed by padding per expert. +/// `block_m` is their alignment; native tiles contain 16 or 32 rows. +pub fn call( + input: Tensor, + weight: Tensor, + scales: Tensor, + sorted_ids: Tensor, + expert_ids: Tensor, + padded_count: Tensor, + output: zml.Shape, + topk: usize, + naive: bool, + block_m: usize, +) Tensor { + std.debug.assert(input.rank() == 2 and input.dtype() == .bf16 and weight.rank() == 3 and + (weight.dtype() == .u8 or weight.dtype() == .i8) and scales.rank() == 3 and scales.dtype() == .u8 and + sorted_ids.rank() == 1 and sorted_ids.dtype() == .i32 and expert_ids.rank() == 1 and expert_ids.dtype() == .i32 and + padded_count.count() == 1 and padded_count.dtype() == .i32 and output.rank() >= 2 and output.dtype() == .bf16 and + topk > 0 and block_m > 0 and block_m % 16 == 0); + const n = weight.dim(1); + const k = input.dim(1); + std.debug.assert((n == 4608 and k == 5120) or (n == 5120 and k == 2304)); + std.debug.assert(input.dim(0) > 0 and weight.dim(0) > 0 and weight.dim(2) == @divExact(k, 2) and + scales.dim(0) == weight.dim(0) and scales.dim(1) == n and scales.dim(2) == @divExact(k, 32) and + output.dim(-1) == n); + // Keep packed-word counts and route/output indices within signed i32. + std.debug.assert(weight.count() / 4 <= std.math.maxInt(i32) and output.count() <= std.math.maxInt(i32) and + sorted_ids.count() <= std.math.maxInt(i32)); + const routes: i64 = @intCast(output.count() / @as(usize, @intCast(n))); + std.debug.assert(topk <= routes and @mod(@as(usize, @intCast(routes)), topk) == 0 and + input.dim(0) == @divTrunc(routes, @as(i64, @intCast(topk)))); + const capacity: i64 = @intCast(sorted_ids.count()); + const blocks: i64 = @intCast(expert_ids.count()); + if (naive) { + std.debug.assert(blocks == routes); + } else { + std.debug.assert(capacity > 0 and block_m <= capacity and @mod(capacity, @as(i64, @intCast(block_m))) == 0 and + blocks == @divExact(capacity, @as(i64, @intCast(block_m)))); + } + if (naive and routes <= 6) return Gemv.call(input, weight, scales, expert_ids, output, topk); + return callMfma(input, weight, scales, sorted_ids, expert_ids, padded_count, output, topk, naive, block_m); +} + +fn callMfma(input: Tensor, weight: Tensor, scales: Tensor, sorted_ids: Tensor, expert_ids: Tensor, padded_count: Tensor, output: zml.Shape, topk: usize, naive: bool, block_m: usize) Tensor { + const n = weight.dim(1); + const k = input.dim(1); + const routes = input.dim(0) * @as(i64, @intCast(topk)); + const native_m: i64 = if (!naive and block_m % 32 == 0) 32 else 16; + const cfg: Config = .{ + .tokens = input.dim(0), + .experts = weight.dim(0), + .n = n, + .k = k, + .ka = @divExact(k, 4), + .kb = @divExact(k, 8), + .ks = @divExact(k, 32), + .topk = @as(i64, @intCast(topk)), + .routes = routes, + .capacity = @as(i64, @intCast(sorted_ids.count())), + .blocks = @as(i64, @intCast(expert_ids.count())), + .subtiles = @divExact(@as(i64, @intCast(block_m)), native_m), + .native_m = native_m, + .wide = @as(i64, @intFromBool(native_m == 32)), + .stage_elements = native_m * 128, + .shared_elements = native_m * 256, + .naive = @as(i64, @intFromBool(naive)), + }; + const a = input.reshape(.{ input.dim(0), @divExact(k, 4), 4 }).bitCast(.i64); + const b = weight.reshape(.{ weight.dim(0), n, @divExact(k, 8), 4 }).bitCast(.i32); + return Kernel.call(.{ .a = a, .b = b, .scales = scales, .sorted = sorted_ids, .experts = expert_ids, .padded = padded_count.reshape(.{1}) }, .{ + .output = zml.Shape.init(.{ routes, n }, .bf16), + }, .{ + .cfg = cfg, + .threads = 256, + .grid = .{ @intCast(@divExact(n, 64)), @intCast(if (naive) routes else @divExact(cfg.capacity, native_m)), 1 }, + }).output.reshape(output); +} + +const Config = struct { + tokens: i64, + experts: i64, + n: i64, + k: i64, + ka: i64, + kb: i64, + ks: i64, + topk: i64, + routes: i64, + capacity: i64, + blocks: i64, + subtiles: i64, + native_m: i64, + wide: i64, + stage_elements: i64, + shared_elements: i64, + naive: i64, +}; + +const Kernel = zml.kernel.fly.Kernel(Config, .{ + .name = "mxfp4_raw_mfma", + .inputs = &.{ "a", "b", "scales", "sorted", "experts", "padded" }, + .outputs = &.{"output"}, + .run = run, +}); + +fn run(b: *Builder, cfg: Config) fly.FinishError!void { + const args = Kernel.args(b); + const zero = b.constant(.i32, 0); + const fzero = b.constant(.f32, 0.0); + const zeros = fzero.splat(4); + const block_n = b.blockId(.x).to(.i64); + const block_m = b.blockId(.y).to(.i64); + const route_start = block_m.mul(cfg.native_m); + const has_rows = if (cfg.naive != 0) b.constant(.i1, 1) else blk: { + const padded = args.padded.at(b.constant(.i64, 0)).to(.i64); + var has_block = b.openIfElse(route_start.cmpUnsigned(.lt, padded), .{b.constant(.i1, 0).type_()}); + has_block.yieldThen(.{args.sorted.at(route_start).cmp(.ne, cfg.routes)}); + has_block.yieldElse(.{b.constant(.i1, 0)}); + break :blk has_block.results[0]; + }; + var block = b.openIf(has_rows); + const expert_block = if (cfg.naive != 0) block_m else block_m.divUnsigned(cfg.subtiles); + const eid = args.experts.at(expert_block); + const valid_expert = eid.cmp(.ge, 0).bitAnd(eid.cmp(.lt, cfg.experts)); + const thread = b.threadId(.x).to(.i64); + const lane = thread.remUnsigned(64); + const wave = thread.divUnsigned(64); + const mn_lane = lane.remUnsigned(16); + const k_lane = lane.divUnsigned(16); + const out_n = block_n.mul(64).add(wave.mul(16)).add(mn_lane); + // Each expert has a contiguous real prefix before route-count padding. + const second_active = if (cfg.wide != 0) blk: { + const first = args.sorted.at(route_start.add(16)); + break :blk first.cmp(.ge, 0).bitAnd(first.cmp(.lt, cfg.routes)); + } else b.constant(.i1, 0); + + var sums = b.openIfElse(valid_expert, .{ zeros.type_(), zeros.type_() }); + const expert = eid.to(.i64); + const load_row = thread.divUnsigned(16); + const load_chunk = thread.remUnsigned(16); + const a_route = routeForRow(b, args.sorted, block_m, route_start, load_row, cfg.naive != 0); + const a_valid = a_route.cmp(.ge, 0).bitAnd(a_route.cmp(.lt, cfg.routes)); + const a_base = a_route.to(.i64).divUnsigned(cfg.topk).mul(cfg.ka); + var second_route = b.openIfElse(second_active, .{zero.type_()}); + second_route.yieldThen(.{args.sorted.at(route_start.add(load_row).add(16))}); + second_route.yieldElse(.{b.constant(.i32, -1)}); + const a_route_g1 = second_route.results[0]; + const a_valid_g1 = a_route_g1.cmp(.ge, 0).bitAnd(a_route_g1.cmp(.lt, cfg.routes)); + const a_base_g1 = a_route_g1.to(.i64).divUnsigned(cfg.topk).mul(cfg.ka); + + // One allocation, two disjoint ping-pong regions; never two identical leaves. + const shared = b.sharedArray(.i16, cfg.shared_elements, 16); + const load_a = b.copyAtom(.{ .universal = 128 }, .i64); + const load_b = b.copyAtom(.{ .universal = 128 }, .i32); + const copy_shared = b.copyAtom(.{ .universal = 128 }, .i16); + const loader: TileLoader = .{ + .a = args.a, + .weight = args.b, + .scales = args.scales, + .shared = shared, + .load_a = load_a, + .load_b = load_b, + .copy_shared = copy_shared, + .a_base = a_base, + .a_base_g1 = a_base_g1, + .a_valid = a_valid, + .a_valid_g1 = a_valid_g1, + .second_active = second_active, + .a_col = load_chunk.mul(2), + .a_store = load_row.mul(128).add(load_chunk.bitXor(load_row).mul(8)), + .b_base = expert.mul(cfg.n * cfg.kb).add(out_n.mul(cfg.kb)), + .expert = expert, + .out_n = out_n, + .k_lane = k_lane, + }; + const initial = loader.load(b, b.constant(.i64, 0), b.constant(.i64, 0)); + b.barrier(); + const atom = b.mmaAtom((fly.rocdl.MmaOpCDNA3MFMAType.get(b.ctx, .{ + .m = 16, + .n = 16, + .k = 16, + .elemTyA = fly.DType.bf16.toMlir(b.ctx), + .elemTyB = fly.DType.bf16.toMlir(b.ctx), + .elemTyAcc = fly.DType.f32.toMlir(b.ctx), + }) catch return error.InvalidMlir).type_()); + var k = b.openFor(b.constant(.i64, 0), cfg.k, 128, .{ zeros, zeros, initial.words, initial.scale }); + const read_base = k.iv.divUnsigned(128).bitAnd(1).mul(cfg.stage_elements); + const write_base = k.iv.divUnsigned(128).bitAnd(1).bitXor(1).mul(cfg.stage_elements); + var a_fragments: [4]Value = undefined; + var a_fragments_g1: [4]Value = undefined; + inline for (0..4) |u| { + const offset = read_base.add(mn_lane.mul(128)).add(k_lane.mul(4).add(u).bitXor(mn_lane).mul(8)); + a_fragments[u] = b.copyAtomLoad(copy_shared, vectorView(shared, offset, 8)); + } + inline for (0..4) |u| { + const offset = read_base.add(mn_lane.mul(128)).add(k_lane.mul(4).add(u).bitXor(mn_lane).mul(8)).add(2048); + const azero = b.constant(.i16, 0).splat(8); + var second = b.openIfElse(second_active, .{azero.type_()}); + second.yieldThen(.{b.copyAtomLoad(copy_shared, vectorView(shared, offset, 8))}); + second.yieldElse(.{azero}); + a_fragments_g1[u] = second.results[0]; + } + // Prefetch the other LDS region only after all current A fragments are read. + const next_k = k.iv.add(128); + var next = b.openIfElse(next_k.cmpUnsigned(.lt, cfg.k), .{ initial.words.type_(), initial.scale.type_() }); + const next_tile = loader.load(b, next_k, write_base); + next.yieldThen(.{ next_tile.words, next_tile.scale }); + next.yieldElse(.{ b.constant(.i32, 0).splat(4), fzero }); + var acc = k.carried[0]; + var acc_g1 = k.carried[1]; + inline for (0..4) |u| { + const decoded = decodeWeight(b, k.carried[2].extract(u), k.carried[3]); + const a0 = a_fragments[u].shuffleVector(a_fragments[u], &.{ 0, 1, 2, 3 }).bitcast(.bf16); + const a1 = a_fragments[u].shuffleVector(a_fragments[u], &.{ 4, 5, 6, 7 }).bitcast(.bf16); + const w0 = decoded.shuffleVector(decoded, &.{ 0, 1, 2, 3 }); + const w1 = decoded.shuffleVector(decoded, &.{ 4, 5, 6, 7 }); + acc = b.mmaAtomCall(atom, a0, w0, acc); + acc = b.mmaAtomCall(atom, a1, w1, acc); + var second = b.openIfElse(second_active, .{zeros.type_()}); + const a0_g1 = a_fragments_g1[u].shuffleVector(a_fragments_g1[u], &.{ 0, 1, 2, 3 }).bitcast(.bf16); + const a1_g1 = a_fragments_g1[u].shuffleVector(a_fragments_g1[u], &.{ 4, 5, 6, 7 }).bitcast(.bf16); + const first_acc = b.mmaAtomCall(atom, a0_g1, w0, acc_g1); + second.yieldThen(.{b.mmaAtomCall(atom, a1_g1, w1, first_acc)}); + second.yieldElse(.{acc_g1}); + acc_g1 = second.results[0]; + } + // All current reads finish before a later iteration reuses this LDS region. + b.barrier(); + k.yield(.{ acc, acc_g1, next.results[0], next.results[1] }); + sums.yieldThen(.{ k.results[0], k.results[1] }); + sums.yieldElse(.{ zeros, zeros }); + + inline for (0..4) |i| { + storeRow(b, args.output, args.sorted, block_m, route_start, k_lane.mul(4).add(i), out_n, sums.results[0].extract(i), cfg); + } + var store_second = b.openIf(second_active); + inline for (0..4) |i| { + storeRow(b, args.output, args.sorted, block_m, route_start, k_lane.mul(4).add(i + 16), out_n, sums.results[1].extract(i), cfg); + } + store_second.yieldThen(.{}); + block.yieldThen(.{}); +} + +fn routeForRow(b: *Builder, sorted: Value, block: Value, start: Value, row: Value, naive: bool) Value { + return if (naive) row.cmp(.eq, 0).select(block.to(.i32), b.constant(.i32, -1)) else sorted.at(start.add(row)); +} + +fn storeRow(b: *Builder, output: Value, sorted: Value, block: Value, start: Value, row: Value, col: Value, value: Value, cfg: Config) void { + const route = routeForRow(b, sorted, block, start, row, cfg.naive != 0); + var valid = b.openIf(route.cmp(.ge, 0).bitAnd(route.cmp(.lt, cfg.routes))); + output.set(.{ route.to(.i64), col }, value.to(.bf16)); + valid.yieldThen(.{}); +} + +fn vectorView(tensor: Value, offset: Value, comptime n: i64) Value { + return tensor.emitIter().addOffset(offset).view(fly.L(n, 1)); +} + +const TileLoader = struct { + a: Value, + weight: Value, + scales: Value, + shared: Value, + load_a: Value, + load_b: Value, + copy_shared: Value, + a_base: Value, + a_base_g1: Value, + a_valid: Value, + a_valid_g1: Value, + second_active: Value, + a_col: Value, + a_store: Value, + b_base: Value, + expert: Value, + out_n: Value, + k_lane: Value, + + fn load(self: TileLoader, b: *Builder, k: Value, write_base: Value) struct { words: Value, scale: Value } { + self.storeActivation(b, self.a_base, self.a_valid, k, self.a_store.add(write_base)); + var second = b.openIf(self.second_active); + self.storeActivation(b, self.a_base_g1, self.a_valid_g1, k, self.a_store.add(write_base).add(2048)); + second.yieldThen(.{}); + const offset = self.b_base.add(k.divUnsigned(8)).add(self.k_lane.mul(4)); + const words = b.copyAtomLoad(self.load_b, vectorView(self.weight, offset, 4)); + const byte = self.scales.at(.{ self.expert, self.out_n, k.divUnsigned(32).add(self.k_lane) }); + const scale = byte.toUnsigned(.i32); + const finite = scale.cmp(.eq, 0).select(b.constant(.i32, 0x00400000), scale.shl(23)); + return .{ .words = words, .scale = scale.cmp(.eq, 255).select(b.constant(.i32, 0x7fc00000), finite).bitcast(.f32) }; + } + + fn storeActivation(self: TileLoader, b: *Builder, base: Value, valid: Value, k: Value, destination: Value) void { + const zeros = b.constant(.i64, 0).splat(2); + var guarded = b.openIfElse(valid, .{zeros.type_()}); + const offset = base.add(k.divUnsigned(4)).add(self.a_col); + guarded.yieldThen(.{b.copyAtomLoad(self.load_a, vectorView(self.a, offset, 2))}); + guarded.yieldElse(.{zeros}); + b.copyAtomStore(self.copy_shared, guarded.results[0].bitcast(.i16), vectorView(self.shared, destination, 8)); + } +}; + +fn decodeWeight(b: *Builder, word: Value, scale: Value) Value { + const lo = b.constant(.i32, 0x44403800); + const hi = b.constant(.i32, 0x54504c48); + const even = b.perm(hi, lo, word.bitAnd(0x07070707)); + const odd = b.perm(hi, lo, word.shrU(4).bitAnd(0x07070707)); + const ev0 = b.cvtPkF32Fp8(even, false); + const od0 = b.cvtPkF32Fp8(odd, false); + const ev1 = b.cvtPkF32Fp8(even, true); + const od1 = b.cvtPkF32Fp8(odd, true); + const first = ev0.shuffleVector(od0, &.{ 0, 2, 1, 3 }); + const last = ev1.shuffleVector(od1, &.{ 0, 2, 1, 3 }); + const magnitude = first.shuffleVector(last, &.{ 0, 1, 2, 3, 4, 5, 6, 7 }); + var shifts = b.constant(.i32, 0).splat(8); + inline for (0..8) |i| shifts = shifts.insert(i, b.constant(.i32, i * 4)); + const signs = word.splat(8).shrU(shifts).bitAnd(8).shl(28); + // FNUZ has no -0: restore FP4's sign in F32 before scaling and BF16 rounding. + return magnitude.bitcast(.i32).bitOr(signs).bitcast(.f32).mul(scale).to(.bf16); +} + +const Gemv = struct { + // Packed-word GEMV derived from the supplied moe-mi300x/fly_word.py algorithm. + // Several rows share a CTA, and the shorter down projection uses subwaves to + // expose independent outputs without paying for a full wave reduction per row. + const Config = struct { rows: usize, lanes: usize, unroll: usize }; + + /// One local expert ID per original route; negative/out-of-bank IDs write zero. + fn call(x: Tensor, weight: Tensor, scales: Tensor, ids: Tensor, output: zml.Shape, topk: usize) Tensor { + const cfg: Gemv.Config = if (weight.dim(1) == 4608) + .{ .rows = 4, .lanes = 64, .unroll = 1 } + else + .{ .rows = 4, .lanes = 32, .unroll = 5 }; + const rows = cfg.rows; + const lanes = cfg.lanes; + const unroll = cfg.unroll; + const n = weight.dim(1); + const k = x.dim(1); + const routes: i64 = @intCast(ids.count()); + std.debug.assert(x.dtype() == .bf16 and (weight.dtype() == .u8 or weight.dtype() == .i8)); + std.debug.assert(@mod(n, 8) == 0 and @mod(k, 32) == 0); + std.debug.assert(topk > 0 and routes == x.dim(0) * @as(i64, @intCast(topk))); + std.debug.assert(weight.dim(2) * 2 == k); + std.debug.assert(scales.dim(0) == weight.dim(0) and scales.dim(1) == n and scales.dim(2) == @divExact(k, 32)); + const tile_rows = rows * (64 / lanes); + std.debug.assert(@mod(n, @as(i64, @intCast(tile_rows * 8))) == 0); + const fields: EmitConfig = .{ .unroll = unroll, .rows = tile_rows, .lanes = lanes, .stride = unroll * lanes, .ntiles = @divExact(n, @as(i64, @intCast(tile_rows))), .tokens = x.dim(0), .experts = weight.dim(0), .n = n, .k_pairs = @divExact(k, 2), .k_words = @divExact(k, 8), .k_scales = @divExact(k, 32), .routes = routes, .grid = @divExact(n, @as(i64, @intCast(tile_rows))) * routes, .topk = @as(i64, @intCast(topk)) }; + const a = x.reshape(.{ .token = x.dim(0), .in = @divExact(k, 2), .pair = 2 }).bitCast(.i32); + const b = weight.reshape(.{ .expert = weight.dim(0), .out = n, .in = @divExact(weight.dim(2), 4), .word = 4 }).bitCast(.i32); + const matrix_output = zml.Shape.init(.{ .token = routes, .out = n }, output.dtype()); + return Gemv.Kernel.call(.{ .a = a, .b = b, .scales = scales.bitCast(.u8), .ids = ids.reshape(.{routes}) }, .{ + .output = matrix_output, + }, .{ + .cfg = fields, + .threads = @intCast(rows * 64), + .grid = .{ @intCast(fields.grid), 1, 1 }, + }).output.reshape(output); + } + + const EmitConfig = struct { + unroll: usize, + rows: usize, + lanes: usize, + stride: usize, + ntiles: i64, + tokens: i64, + experts: i64, + n: i64, + k_pairs: i64, + k_words: i64, + k_scales: i64, + routes: i64, + grid: i64, + topk: i64, + }; + + const Kernel = zml.kernel.fly.Kernel(EmitConfig, .{ + .name = "mxfp4_gemv", + .inputs = &.{ "a", "b", "scales", "ids" }, + .outputs = &.{"output"}, + .run = Gemv.run, + }); + + fn run(b: *Builder, cfg: EmitConfig) fly.FinishError!void { + const args = Gemv.Kernel.args(b); + const block = b.blockId(.x).to(.i64); + const thread = b.threadId(.x).to(.i64); + const lane = thread.remUnsigned(cfg.lanes); + const rowgroup = thread.divUnsigned(cfg.lanes); + // Preserve the eight-XCD workgroup permutation and original subwave layout. + const pid = block.remUnsigned(8).mul(@divExact(cfg.grid, 8)).add(block.divUnsigned(8)); + const row = pid.remUnsigned(cfg.ntiles).mul(cfg.rows).add(rowgroup); + const route = pid.divUnsigned(cfg.ntiles); + const expert32 = args.ids.at(route); + const expert = expert32.to(.i64); + const token = route.divUnsigned(cfg.topk); + const local = expert32.cmp(.ge, 0).bitAnd(expert32.cmp(.lt, cfg.experts)); + const inrow = row.cmpUnsigned(.lt, cfg.n); + const fzero = b.constant(.f32, 0.0); + var sum = b.openIfElse(local.bitAnd(inrow), .{fzero.type_()}); + var k = b.openFor(lane, cfg.k_words, cfg.stride, .{fzero}); + var carried = k.carried[0]; + for (0..cfg.unroll) |u| { + const pos = k.iv.add(u * cfg.lanes); + var inbounds = b.openIfElse(pos.cmpUnsigned(.lt, cfg.k_words), .{fzero.type_()}); + const word = args.b.at(.{ expert, row, pos }); + const scale8 = args.scales.at(.{ expert, row, pos.divUnsigned(4) }); + const scale = decodeScale(b, scale8); + const low = word.bitAnd(0x07070707); + const high = word.shrU(4).bitAnd(0x07070707); + const even_bits = b.perm(b.constant(.i32, 0xd4d0ccc8), b.constant(.i32, 0xc4c0b800), low); + const odd_bits = b.perm(b.constant(.i32, 0xd4d0ccc8), b.constant(.i32, 0xc4c0b800), high); + const even = even_bits.bitAnd(word.shl(4).bitOr(0x7f7f7f7f)); + const odd = odd_bits.bitAnd(word.bitOr(0x7f7f7f7f)); + // Retain the scalar decoder and its FP32 scale -> BF16 -> FP32 boundary. + const weights = [_]Value{ + b.cvtPkF32Fp8(even, false).mul(scale).to(.bf16).to(.f32), + b.cvtPkF32Fp8(odd, false).mul(scale).to(.bf16).to(.f32), + b.cvtPkF32Fp8(even, true).mul(scale).to(.bf16).to(.f32), + b.cvtPkF32Fp8(odd, true).mul(scale).to(.bf16).to(.f32), + }; + var partial = fzero; + inline for (0..4) |pair| { + const x = args.a.at(.{ token, pos.mul(4).add(pair) }); + const xlo = x.to(.i16).bitcast(.bf16).to(.f32); + const xhi = x.shrU(16).to(.i16).bitcast(.bf16).to(.f32); + const wlo = weights[(pair / 2) * 2].extract(pair % 2); + const whi = weights[(pair / 2) * 2 + 1].extract(pair % 2); + // Odd then even FMA, followed by one add to the K-loop accumulator. + partial = b.fma(xlo, wlo, b.fma(xhi, whi, partial)); + } + inbounds.yieldThen(.{partial.add(carried)}); + inbounds.yieldElse(.{carried}); + carried = inbounds.results[0]; + } + k.yield(.{carried}); + var reduced = k.results[0]; + var shift: i32 = 1; + while (shift < @as(i32, @intCast(cfg.lanes))) : (shift *= 2) { + reduced = reduced.add(reduced.shuffleXor(shift, 64)); + } + sum.yieldThen(.{reduced}); + sum.yieldElse(.{fzero}); + var store = b.openIf(lane.cmp(.eq, 0).bitAnd(inrow)); + args.output.set(.{ route, row }, sum.results[0].to(.bf16)); + store.yieldThen(.{}); + } + + fn decodeScale(b: *Builder, byte: Value) Value { + const scale = byte.toUnsigned(.i32); + const finite = scale.cmp(.eq, 0).select(b.constant(.i32, 0x00400000), scale.shl(23)); + return scale.cmp(.eq, 255).select(b.constant(.i32, 0x7fc00000), finite).bitcast(.f32); + } +}; diff --git a/zml/moe/fused_experts.zig b/zml/moe/fused_experts.zig new file mode 100644 index 00000000..9d57b478 --- /dev/null +++ b/zml/moe/fused_experts.zig @@ -0,0 +1,540 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../zml.zig"); +const Tensor = zml.Tensor; +const Shape = zml.Shape; +const DataType = zml.DataType; +const triton = @import("triton.zig"); +const fly = @import("fly_kernels/moe.zig"); + +const log = std.log.scoped(.moe); + +test { + std.testing.refAllDecls(@This()); +} + +pub const Parameters = struct { + num_experts_per_tok: u32, + activation: ActivationMode, + + pub const ActivationMode = enum { + silu, + relu, + gelu, + }; + + pub const InitOptions = struct { + num_experts_per_tok: u32, + activation: ActivationMode, + }; + + pub fn init(opts: InitOptions) Parameters { + return .{ + .num_experts_per_tok = opts.num_experts_per_tok, + .activation = opts.activation, + }; + } +}; + +pub const GateUpLayout = enum { split, interleaved }; +pub const RoutingWeightPlacement = enum { before_down, after_down }; + +pub const FusedExpertsArgs = struct { + hidden_states: Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + topk_weights: Tensor, + topk_ids: Tensor, + activation: Parameters.ActivationMode = .silu, + expert_map: ?Tensor = null, + activation_threshold: ?f32 = null, + /// Use FP8 activations for FP8 weights; false keeps BF16 activations. + quantize_input: bool, + gate_up_layout: GateUpLayout, + routing_weight_placement: RoutingWeightPlacement, +}; + +pub fn fusedExperts(opts: FusedExpertsArgs, comptime backend: zml.moe.Backend) !Tensor { + const Impl = switch (backend) { + .fly => fly, + .triton => triton, + else => @compileError("unsupported fused-experts backend"), + }; + if (backend == .fly and (!backend.isAvailable(zml.Compiler.current().platform) or !fly.supports(opts))) { + return fusedExperts(opts, .triton); + } + + const hidden_states = opts.hidden_states; + const topk_weights = opts.topk_weights; + const topk_ids = opts.topk_ids; + const b = hidden_states.dim(.b); + const s = hidden_states.dim(.s); + const num_tokens = b * s; + var launch_config = launchConfigForTokens(num_tokens); + + const gate_up_scheme = opts.gate_up.quantizationScheme(); + const down_scheme = opts.down.quantizationScheme(); + if (gate_up_scheme) |scheme| switch (scheme) { + .nvfp4 => return error.UnsupportedQuantization, + .fp8_block128 => launch_config.block_size_k = 128, + .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, + }; + + var down_launch_config = launchConfigForTokens(num_tokens); + if (down_scheme) |scheme| switch (scheme) { + .fp8_block128 => down_launch_config.block_size_k = 128, + .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, + .nvfp4 => return error.UnsupportedQuantization, + }; + + const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); + const gate_up = opts.gate_up.weight.withTags(.{ .expert, .out, .in }); + const down = opts.down.weight.withTags(.{ .expert, .out, .mid }); + const routing_weights = topk_weights.reshape(.{ .token = num_tokens, .in = topk_weights.dim(.top_expert) }).withTags(.{ .token, .topk }); + const ids = topk_ids.reshape(.{ .token = num_tokens, .in = topk_ids.dim(.top_expert) }).withTags(.{ .token, .topk }); + + stdx.debug.assert(hidden.dtype() == .bf16, "expected BF16 hidden states, got {}", .{hidden.dtype()}); + stdx.debug.assert(if (gate_up_scheme == .mxfp4) gate_up.dtype() == .u8 or gate_up.dtype() == .i8 or gate_up.dtype() == .f4e2m1 else gate_up.dtype() == .bf16 or gate_up.dtype() == .f8e4m3fn or gate_up.dtype() == .f8e4m3fnuz, "unsupported gate/up weight dtype {}", .{gate_up.dtype()}); + stdx.debug.assert(if (down_scheme == .mxfp4) down.dtype() == .u8 or down.dtype() == .i8 or down.dtype() == .f4e2m1 else down.dtype() == .bf16 or down.dtype() == .f8e4m3fn or down.dtype() == .f8e4m3fnuz, "unsupported down weight dtype {}", .{down.dtype()}); + stdx.debug.assert(routing_weights.dtype() == .f32 or routing_weights.dtype() == .bf16, "expected FP32 or BF16 routing weights, got {}", .{routing_weights.dtype()}); + stdx.debug.assert(ids.dtype() == .i32, "expected I32 expert ids, got {}", .{ids.dtype()}); + const gate_up_k = gate_up.dim(.in) * @as(i64, if (gate_up_scheme == .mxfp4 and gate_up.dtype() != .f4e2m1) 2 else 1); + const down_k = down.dim(.mid) * @as(i64, if (down_scheme == .mxfp4 and down.dtype() != .f4e2m1) 2 else 1); + stdx.debug.assert(hidden.dim(.in) == gate_up_k, "hidden width {} must match gate/up input width {}", .{ hidden.dim(.in), gate_up_k }); + const activation_reduction: i64 = if (opts.activation == .relu) 1 else 2; + stdx.debug.assert(@rem(gate_up.dim(.out), activation_reduction) == 0, "gate/up output width {} must be divisible by {}", .{ gate_up.dim(.out), activation_reduction }); + stdx.debug.assert(down_k == @divFloor(gate_up.dim(.out), activation_reduction), "down input width {} must match activated width {}", .{ down_k, @divFloor(gate_up.dim(.out), activation_reduction) }); + stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "routing ids and weights must match hidden token count {}, got {} and {}", .{ hidden.dim(.token), ids.dim(.token), routing_weights.dim(.token) }); + stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "routing ids and weights must have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), routing_weights.dim(.topk) }); + stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "gate/up and down expert counts must match, got {} and {}", .{ gate_up.dim(.expert), down.dim(.expert) }); + if (opts.expert_map) |expert_map| + stdx.debug.assert(expert_map.dtype() == .i32 and expert_map.rank() == 1, "expected a rank-1 I32 expert map, got rank {} and dtype {}", .{ expert_map.rank(), expert_map.dtype() }); + + const num_experts = if (opts.expert_map) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); + const routing = Impl.prepareRouting(ids, num_experts, @intCast(launch_config.block_size_m)); + + const expert_ids = if (opts.expert_map) |expert_map| + expert_map.gather(.{ .expert = routing.expert_ids }, .{}).withTags(.{.g}) + else + routing.expert_ids; + + const hidden_quant, const input_scale = Impl.prepareInput(hidden, gate_up_scheme, gate_up.dtype(), opts.quantize_input); + + const gate_up_out = Impl.call(.{ + .input = hidden_quant, + .weight = gate_up, + .bias = opts.gate_up.bias, + .input_scale = input_scale, + .weight_scale = opts.gate_up.quantizationScales(), + .routing = routing, + .expert_ids = expert_ids, + .launch_config = launch_config, + .quant_scheme = gate_up_scheme, + .top_k = @intCast(ids.dim(.topk)), + .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), + }); + + var activated = applyExpertActivation(gate_up_out, opts.activation, opts.activation_threshold, opts.gate_up_layout); + if (opts.routing_weight_placement == .before_down) { + const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); + activated = activated.mul(weights.broad(activated.shape())); + } + const activated_quant, const down_input_scale = Impl.prepareInput(activated, down_scheme, down.dtype(), opts.quantize_input); + + const down_out = Impl.call(.{ + .input = activated_quant, + .weight = down, + .bias = opts.down.bias, + .input_scale = down_input_scale, + .weight_scale = opts.down.quantizationScales(), + .routing_weights = if (opts.routing_weight_placement == .after_down) routing_weights else null, + .routing = routing, + .expert_ids = expert_ids, + .launch_config = down_launch_config, + .quant_scheme = down_scheme, + .top_k = 1, + .output_shape = Shape.init(.{ .token = b * s, .topk = ids.dim(.topk), .out = down.dim(.out) }, .bf16), + }); + + const output = down_out.convert(.f32).sum(.topk).squeeze(.topk).convert(.bf16); + + return output.reshape(.{ .b = b, .token = s, .out = down.dim(.out) }); +} + +fn applyExpertActivation(input: Tensor, mode: Parameters.ActivationMode, activation_threshold: ?f32, layout: GateUpLayout) Tensor { + const x = input.convert(.f32); + if (mode == .relu) { + const clipped = if (activation_threshold) |limit| x.minimum(Tensor.scalar(limit, x.dtype())) else x; + return clipped.relu().powByConst(2); + } + + const mid = @divFloor(x.dim(.out), 2); + var gate, var up = switch (layout) { + .split => .{ x.slice(.out, .{ .end = mid }), x.slice(.out, .{ .start = mid }) }, + .interleaved => .{ x.slice(.out, .{ .start = 0, .step = 2 }), x.slice(.out, .{ .start = 1, .step = 2 }) }, + }; + if (activation_threshold) |limit_| { + const limit = Tensor.scalar(limit_, x.dtype()); + gate = gate.minimum(limit); + up = up.clamp(limit.negate(), limit); + } + return switch (mode) { + .silu => gate.silu().mul(up), + .relu => unreachable, + .gelu => gate.gelu().mul(up), + }; +} + +test "SwiGLU uses FP32 math for split and interleaved BF16 inputs" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + const Local = struct { + fn forward(x: Tensor, layout: GateUpLayout, threshold: ?f32) Tensor { + return applyExpertActivation(x, .silu, threshold, layout); + } + }; + const x: Tensor = .init(.{ .token = 1, .out = 6 }, .bf16); + const gates = [_]f32{ 0.75, -1.25, 3.5 }; + const ups = [_]f32{ 0.875, -2.25, 4.5 }; + for ([_]GateUpLayout{ .split, .interleaved }) |layout| { + var values: [6]zml.floats.BFloat16 = undefined; + for (gates, ups, 0..) |gate, up, i| { + values[if (layout == .split) i else 2 * i] = .fromF32(gate); + values[if (layout == .split) i + 3 else 2 * i + 1] = .fromF32(up); + } + var input = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&values)); + defer input.deinit(); + for ([_]?f32{ null, 2 }) |threshold| { + var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, threshold }, .{}); + defer exe.deinit(); + try zml.testing.expectEqualShapes(Shape.init(.{ .token = 1, .out = 3 }, .f32), exe.output_shapes[0]); + var output = try zml.testing.autoCall(allocator, io, &exe, Local.forward, .{input}); + defer output.deinit(); + var actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + for (gates, ups, actual.constItems(f32)) |gate, up, value| { + const g = if (threshold) |limit| @min(gate, limit) else gate; + const u = if (threshold) |limit| std.math.clamp(up, -limit, limit) else up; + try std.testing.expectApproxEqAbs(g / (1 + @exp(-g)) * u, value, 1e-6); + } + } + } +} + +test "ReLU squared activation preserves width and applies threshold before squaring" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + const Local = struct { + fn forward(x: Tensor, threshold: ?f32) Tensor { + return applyExpertActivation(x, .relu, threshold, .split); + } + }; + const x: Tensor = .init(.{ .token = 1, .out = 5 }, .f32); + const values = [_]f32{ -3, 0, 1, 2, 4 }; + var input = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&values)); + defer input.deinit(); + for ([_]?f32{ null, 2 }) |threshold| { + var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, threshold }, .{}); + defer exe.deinit(); + try zml.testing.expectEqualShapes(x.shape(), exe.output_shapes[0]); + var output = try zml.testing.autoCall(allocator, io, &exe, Local.forward, .{input}); + defer output.deinit(); + var actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + const expected = [_]f32{ 0, 0, 1, 4, if (threshold != null) 4 else 16 }; + try std.testing.expectEqualSlices(f32, &expected, actual.constItems(f32)); + } +} + +/// Build the inputs tuple for FusedMoe and invoke it via `K.call(...)`. +pub const GemmOptions = struct { + input: Tensor, + weight: Tensor, + bias: ?Tensor = null, + input_scale: ?Tensor = null, + weight_scale: ?Tensor = null, + routing_weights: ?Tensor = null, + routing: Routing, + expert_ids: Tensor, + launch_config: LaunchConfig, + quant_scheme: ?zml.Quantization.Scheme = null, + top_k: usize, + output_shape: Shape, +}; + +const callFusedMoe = triton.call; + +test "FP8 routed GEMM with bias matches dequantized weights" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + if (platform.target != .cuda) return error.SkipZigTest; + + const Local = struct { + const Outputs = struct { actual: Tensor, expected: Tensor }; + + fn expandScales(scales: Tensor, shape: Shape) Tensor { + var expanded = scales.convert(.f32); + for (0..shape.rank()) |axis| { + const factor = @divExact(shape.dim(axis), expanded.dim(axis)); + const broad_shape = expanded.shape().insert(axis + 1, .{factor}); + const axes = Shape.range(broad_shape.rank(), .i64).remove(axis + 1); + expanded = expanded.broadcast(broad_shape, axes.dims()) + .reshape(expanded.shape().setDim(axis, shape.dim(axis))); + } + return expanded.withTags(shape.tags()); + } + + fn forward(x: Tensor, w: Tensor, scales: Tensor, ids: Tensor, weights: Tensor, bias: Tensor, scheme: zml.Quantization.Scheme, mx_native: bool, quantize_input: bool) Outputs { + const weight = w.convert(.f8e4m3fn); + var input = x; + var input_scale: ?Tensor = null; + var reference_input = x.convert(.f32); + if (quantize_input) { + input, input_scale = triton.prepareInput(x, scheme, .f8e4m3fn, true); + const scale = if (scheme == .mxfp8) input_scale.?.bitCast(.f8e8m0) else input_scale.?; + reference_input = input.convert(.f32).withTags(x.shape().tags()) + .mul(expandScales(scale.convert(.f32), x.shape())); + } + const weight_scale = if (scheme == .mxfp8) blk: { + const native = scales.convert(.f8e8m0); + break :blk if (mx_native) native else native.bitCast(.u8); + } else scales; + const config = launchConfigForTokens(8); + const routing = triton.prepareRouting(ids, w.dim(.expert), @intCast(config.block_size_m)); + const actual = callFusedMoe(.{ + .input = input, + .weight = weight, + .bias = bias, + .input_scale = input_scale, + .weight_scale = weight_scale, + .routing_weights = weights, + .routing = routing, + .expert_ids = routing.expert_ids, + .launch_config = config, + .quant_scheme = scheme, + .top_k = 1, + .output_shape = Shape.init(.{ .token = x.dim(.token), .topk = 1, .out = w.dim(.out) }, .bf16), + }).squeeze(.topk); + const expanded_weight = weight.convert(.f32).mul(expandScales(scales, weight.shape())); + const selected_weight = expanded_weight.gather(.{ .expert = ids.squeeze(.topk) }, .{}); + const dot = reference_input.dot(selected_weight, .in); + const selected_bias = if (bias.rank() == 2) + bias.gather(.{ .expert = ids.squeeze(.topk) }, .{}) + else + bias; + const unweighted = dot.add(selected_bias.convert(.f32).broad(dot.shape())); + const expected = unweighted.mul(weights.squeeze(.topk).convert(.f32).broad(unweighted.shape())).convert(.bf16); + return .{ .actual = actual, .expected = expected }; + } + }; + + for ([_]zml.Quantization.Scheme{ .mxfp8, .fp8_per_tensor, .fp8_per_channel, .fp8_block128 }) |scheme| { + for ([_]bool{ false, true }) |quantize_input| { + for ([_]i64{ 1, 17 }) |tokens| { + const n: i64 = if (scheme == .fp8_block128) 256 else 48; + const scale_dims: [3]i64 = switch (scheme) { + .mxfp8 => .{ 8, n, 8 }, + .fp8_per_tensor => if (tokens == 1) .{ 1, 1, 1 } else .{ 8, 1, 1 }, + .fp8_per_channel => .{ 8, n, 1 }, + .fp8_block128 => .{ 8, 2, 2 }, + else => unreachable, + }; + const x: Tensor = .init(.{ .token = tokens, .in = 256 }, .bf16); + const w: Tensor = .init(.{ .expert = 8, .out = n, .in = 256 }, .bf16); + const scales: Tensor = .init(.{ scale_dims[0], scale_dims[1], scale_dims[2] }, .f32); + const ids: Tensor = .init(.{ .token = tokens, .topk = 1 }, .i32); + const weights: Tensor = .init(.{ .token = tokens, .topk = 1 }, .bf16); + const bias: Tensor = if (tokens == 1) + .init(.{ .out = n }, .bf16) + else + .init(.{ .expert = 8, .out = n }, .f32); + var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, w, scales, ids, weights, bias, scheme, tokens == 1, quantize_input }, .{}); + defer exe.deinit(); + var buffers: [6]zml.Buffer = undefined; + var initialized: usize = 0; + defer for (buffers[0..initialized]) |*buffer| buffer.deinit(); + for ([_]Shape{ x.shape(), w.shape(), scales.shape(), ids.shape(), weights.shape(), bias.shape() }, 0..) |shape, i| { + const slice = try zml.Slice.alloc(allocator, shape); + defer slice.free(allocator); + for (0..shape.count()) |j| { + switch (i) { + 2 => slice.items(f32)[j] = @as(f32, @floatFromInt(@as(u32, 1) << @as(u5, @intCast(j % 4)))) / 16, + 3 => slice.items(i32)[j] = @intCast((j * 3 + 5) % 8), + 5 => { + const value = @as(f32, @floatFromInt(@as(i32, @intCast(j % 17)) - 8)) / 2; + if (shape.dtype() == .f32) { + slice.items(f32)[j] = value; + } else { + slice.items(zml.floats.BFloat16)[j] = .fromF32(value); + } + }, + else => slice.items(zml.floats.BFloat16)[j] = .fromF32(if (i == 4) + @as(f32, @floatFromInt(1 + j % 3)) / 4 + else + @as(f32, @floatFromInt(@as(i32, @intCast((j * 7 + j / 256) % 13)) - 6)) / 4), + } + } + buffers[i] = try zml.Buffer.fromSlice(io, platform, slice, .replicated); + initialized += 1; + } + var output = try zml.testing.autoCall(allocator, io, &exe, Local.forward, .{ buffers[0], buffers[1], buffers[2], buffers[3], buffers[4], buffers[5] }); + defer zml.Buffer.deinitAll(Local.Outputs, &output); + zml.testing.expectClose(io, output.expected, output.actual, .{ .absolute_tolerance = 0.03125, .relative_tolerance = 0.01 }) catch |err| { + log.warn("FP8 routed GEMM failed for scheme={s}, quantize_input={}, tokens={}", .{ @tagName(scheme), quantize_input, tokens }); + return err; + }; + } + } + } +} + +test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + if (platform.target != .cuda) return error.SkipZigTest; + const Local = struct { + fn forward(x: Tensor, layout: GateUpLayout, placement: RoutingWeightPlacement, storage_dtype: DataType) Tensor { + const fp4 = storage_dtype != .bf16; + const columns = Tensor.arange(.{ .end = 256 }, .i32).withTags(.{.dout}); + const is_gate = switch (layout) { + .split => columns.cmp(.LT, Tensor.scalar(128, .i32)), + .interleaved => columns.remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), + }; + const dtype: DataType = if (fp4) .u8 else .bf16; + // E2M1 codes 1, 2, 3 encode 0.5, 1, 1.5; each byte holds two values. + const gate_value = if (fp4) Tensor.scalar(0x11, .u8) else Tensor.scalar(0.5, .bf16); + const up_value = if (fp4) Tensor.scalar(0x33, .u8) else Tensor.scalar(1.5, .bf16); + const gate_up_values = is_gate.select(gate_value, up_value); + var gate_up: zml.nn.Linear = .{ + .weight = gate_up_values.broad(Shape.init(.{ .expert = 8, .dout = 256, .d = @as(i64, if (fp4) 64 else 128) }, dtype)), + .tag = Shape.toTag(.d), + .quantization = if (fp4) .{ + .scheme = .mxfp4, + .scales = Tensor.scalar(127, .u8).broad(Shape.init(.{ .expert = 8, .dout = 256, .d = 4 }, .u8)), + } else null, + }; + var down: zml.nn.Linear = .{ + .weight = (if (fp4) Tensor.scalar(0x22, .u8) else Tensor.scalar(1, .bf16)).broad(Shape.init(.{ .expert = 8, .d = 128, .dout = @as(i64, if (fp4) 64 else 128) }, dtype)), + .bias = Tensor.scalar(2, .bf16).broad(Shape.init(.{ .expert = 8, .d = 128 }, .bf16)), + .tag = Shape.toTag(.dout), + .quantization = if (fp4) .{ + .scheme = .mxfp4, + .scales = Tensor.scalar(127, .u8).broad(Shape.init(.{ .expert = 8, .d = 128, .dout = 4 }, .u8)), + } else null, + }; + if (storage_dtype == .f4e2m1) { + gate_up.weight = gate_up.weight.bitCast(.f4e2m1).reshape(.{ .expert = 8, .dout = 256, .d = 128 }); + down.weight = down.weight.bitCast(.f4e2m1).reshape(.{ .expert = 8, .d = 128, .dout = 128 }); + gate_up.quantization.?.scales = gate_up.quantization.?.scales.bitCast(.f8e8m0); + down.quantization.?.scales = down.quantization.?.scales.bitCast(.f8e8m0); + } + const route = Tensor.arange(.{ .end = 2 }, .i32).reshape(.{ .b = 1, .s = 1, .top_expert = 2 }) + .broad(Shape.init(.{ .b = 1, .s = x.dim(.s), .top_expert = 2 }, .i32)); + return fusedExperts(.{ + .hidden_states = x, + .gate_up = gate_up, + .down = down, + .topk_ids = route.addConstant(2), + .topk_weights = route.convert(.f32).addConstant(1).scale(0.25), + .gate_up_layout = layout, + .routing_weight_placement = placement, + .quantize_input = false, + }, .triton) catch unreachable; + } + }; + for ([_]DataType{ .bf16, .u8, .f4e2m1 }) |storage_dtype| { + const tokens: i64 = if (storage_dtype == .u8) 17 else 1; + const x: Tensor = .init(.{ .b = 1, .s = tokens, .d = 128 }, .bf16); + const host = try zml.Slice.alloc(allocator, x.shape()); + defer host.free(allocator); + @memset(host.items(zml.floats.BFloat16), .fromF32(1.0 / 128.0)); + var input = try zml.Buffer.fromSlice(io, platform, host, .replicated); + defer input.deinit(); + for ([_]GateUpLayout{ .split, .interleaved }) |layout| { + for ([_]RoutingWeightPlacement{ .before_down, .after_down }) |placement| { + var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, placement, storage_dtype }, .{}); + defer exe.deinit(); + var output = try zml.testing.autoCall(allocator, io, &exe, Local.forward, .{input}); + defer output.deinit(); + var actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + const activation: f32 = 0.5 / (1 + @exp(@as(f32, -0.5))) * 1.5; + // The two routes have weights 0.25 and 0.5. Bias distinguishes placement. + const bias: f32 = if (placement == .before_down) 4 else 1.5; + const expected = 128 * activation * 0.75 + bias; + for (actual.constItems(zml.floats.BFloat16)) |value| { + try std.testing.expectApproxEqAbs(expected, value.toF32(), 0.5); + } + } + } + } +} + +pub const Routing = struct { + sorted_token_ids: Tensor, + expert_ids: Tensor, + num_tokens_post_padded: Tensor, + max_num_tokens_padded: i64, + num_assignments: i64, + naive_block_assignment: bool, +}; + +// ============================================================================= +// Config / validation helpers +// ============================================================================= + +const LaunchConfig = struct { + block_size_m: usize, + block_size_n: usize, + block_size_k: usize, + group_size_m: usize, + num_warps: i32, + num_stages: i32, +}; + +const launch_configs = [_]struct { tokens: i64, config: LaunchConfig }{ + .{ .tokens = 1, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 4 } }, + .{ .tokens = 2, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 4 } }, + .{ .tokens = 4, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, + .{ .tokens = 8, .config = .{ .block_size_m = 16, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 16, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 64, .group_size_m = 64, .num_warps = 4, .num_stages = 5 } }, + .{ .tokens = 24, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, + .{ .tokens = 32, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 128, .group_size_m = 1, .num_warps = 4, .num_stages = 2 } }, + .{ .tokens = 48, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 128, .group_size_m = 64, .num_warps = 4, .num_stages = 2 } }, + .{ .tokens = 64, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 128, .group_size_m = 1, .num_warps = 4, .num_stages = 2 } }, + .{ .tokens = 96, .config = .{ .block_size_m = 16, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 128, .config = .{ .block_size_m = 16, .block_size_n = 256, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, + .{ .tokens = 256, .config = .{ .block_size_m = 16, .block_size_n = 256, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, + .{ .tokens = 512, .config = .{ .block_size_m = 32, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 1024, .config = .{ .block_size_m = 64, .block_size_n = 128, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, + .{ .tokens = 1536, .config = .{ .block_size_m = 64, .block_size_n = 128, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, + .{ .tokens = 2048, .config = .{ .block_size_m = 128, .block_size_n = 128, .block_size_k = 64, .group_size_m = 16, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 3072, .config = .{ .block_size_m = 128, .block_size_n = 256, .block_size_k = 64, .group_size_m = 1, .num_warps = 8, .num_stages = 4 } }, + .{ .tokens = 4096, .config = .{ .block_size_m = 128, .block_size_n = 256, .block_size_k = 64, .group_size_m = 16, .num_warps = 8, .num_stages = 4 } }, +}; + +fn launchConfigForTokens(num_tokens: i64) LaunchConfig { + var best = launch_configs[0]; + for (launch_configs[1..]) |candidate| { + // The table is ascending, so ties retain the smaller token bucket. + if (@abs(candidate.tokens - num_tokens) < @abs(best.tokens - num_tokens)) { + best = candidate; + } + } + return best.config; +} + +test "MoE launch config selects the nearest token bucket" { + try std.testing.expectEqualDeep(launchConfigForTokens(4), launchConfigForTokens(6)); + try std.testing.expectEqualDeep(launchConfigForTokens(8), launchConfigForTokens(7)); + try std.testing.expectEqualDeep(launchConfigForTokens(1), launchConfigForTokens(0)); + try std.testing.expectEqualDeep(launchConfigForTokens(4096), launchConfigForTokens(8192)); + try std.testing.expectEqual(@as(usize, 64), launchConfigForTokens(16).group_size_m); + try std.testing.expectEqual(@as(i32, 5), launchConfigForTokens(16).num_stages); +} diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index 7c883cd0..f4a16714 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -9,6 +9,8 @@ pub const cutlass_flashinfer = @import("cutlass_flashinfer.zig"); pub const metal = @import("metal.zig"); pub const mosaic_tpu = @import("mosaic_tpu.zig"); pub const triton = @import("triton.zig"); +pub const fly = @import("fly_kernels/moe.zig"); +const fused_experts = @import("fused_experts.zig"); pub const triton_kernels = @import("triton_kernels/triton_kernels.zig"); test { @@ -25,6 +27,7 @@ pub const Backend = enum { triton_mxfp4, flashinfer_cutlass, triton, + fly, mosaic_tpu, metal, @@ -62,10 +65,44 @@ pub const Backend = enum { }; } + pub fn autoMxfp4(platform: *const zml.Platform, weights_dtype: zml.DataType) !Backend { + return switch (platform.target) { + .cuda => switch (if (zml.platform.cuda.computeCapability(platform)) |cc| cc.major else 0) { + 10 => switch (weights_dtype) { + .u8, .i8 => .triton_mxfp4, + .f4e2m1 => .triton, + else => error.UnsupportedDataType, + }, + else => switch (weights_dtype) { + .u8, .i8, .f4e2m1 => .triton, + else => error.UnsupportedDataType, + }, + }, + .rocm => switch (zml.platform.rocm.computeCapability(platform) orelse return .triton) { + .gfx942 => switch (weights_dtype) { + .u8, .i8, .f4e2m1 => .fly, + else => error.UnsupportedDataType, + }, + else => switch (weights_dtype) { + .u8, .i8, .f4e2m1 => .triton, + else => error.UnsupportedDataType, + }, + }, + else => switch (weights_dtype) { + .u8, .i8, .f4e2m1 => .triton, + else => error.UnsupportedDataType, + }, + }; + } + pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { return switch (backend) { .triton_mxfp4 => triton_mxfp4.isAvailable(platform), .flashinfer_cutlass => cutlass_flashinfer.isAvailable(platform), + .fly => switch (platform.target) { + .rocm => zml.platform.rocm.computeCapability(platform) == .gfx942, + else => false, + }, .triton => switch (platform.target) { .cuda, .rocm, .oneapi => true, else => false, @@ -79,7 +116,7 @@ pub const Backend = enum { return switch (backend) { .triton_mxfp4 => {}, .flashinfer_cutlass => cutlass_flashinfer.register(platform), - .triton => {}, + .triton, .fly => {}, .mosaic_tpu => {}, .metal => {}, }; @@ -90,6 +127,7 @@ pub const Parameters = union(Backend) { triton_mxfp4: triton_mxfp4.Parameters, flashinfer_cutlass: cutlass_flashinfer.Parameters, triton: triton.Parameters, + fly: fly.Parameters, mosaic_tpu: mosaic_tpu.Parameters, metal: metal.Parameters, @@ -97,6 +135,7 @@ pub const Parameters = union(Backend) { triton_mxfp4: triton_mxfp4.Parameters.InitOptions, flashinfer_cutlass: cutlass_flashinfer.Parameters.InitOptions, triton: triton.Parameters.InitOptions, + fly: fly.Parameters.InitOptions, mosaic_tpu: mosaic_tpu.Parameters.InitOptions, metal: metal.Parameters.InitOptions, @@ -111,14 +150,14 @@ pub const Parameters = union(Backend) { .gelu => .gelu, }, } }, - .triton => .{ .triton = .{ + inline .triton, .fly => |backend_tag| @unionInit(InitOptions, @tagName(backend_tag), .{ .num_experts_per_tok = num_experts_per_tok, .activation = switch (activation) { .silu => .silu, .relu => .relu, .gelu => .gelu, }, - } }, + }), .mosaic_tpu => .{ .mosaic_tpu = .{ .num_experts_per_tok = num_experts_per_tok, .activation = switch (activation) { @@ -143,7 +182,7 @@ pub const Parameters = union(Backend) { return switch (opts) { .triton_mxfp4 => |v| .{ .triton_mxfp4 = triton_mxfp4.Parameters.init(v) }, .flashinfer_cutlass => |v| .{ .flashinfer_cutlass = cutlass_flashinfer.Parameters.init(v) }, - .triton => |v| .{ .triton = triton.Parameters.init(v) }, + inline .triton, .fly => |v, backend_tag| @unionInit(Parameters, @tagName(backend_tag), fused_experts.Parameters.init(v)), .mosaic_tpu => |v| .{ .mosaic_tpu = mosaic_tpu.Parameters.init(v) }, .metal => |v| .{ .metal = metal.Parameters.init(v) }, }; @@ -154,10 +193,10 @@ pub const Options = struct { activation_threshold: ?f32 = null, /// Quantize activations for Triton FP8 GEMMs; false keeps BF16 activations. quantize_input: bool, - /// Gate/up layout; non-Triton backends require split columns. - gate_up_layout: triton.GateUpLayout, - /// Where routing weights are applied; non-Triton backends require after_down. - routing_weight_placement: triton.RoutingWeightPlacement, + /// Gate/up layout; FlashInfer, Mosaic and Metal require split columns. + gate_up_layout: fused_experts.GateUpLayout, + /// Where routing weights are applied; FlashInfer, Mosaic and Metal require after_down. + routing_weight_placement: fused_experts.RoutingWeightPlacement, }; pub fn forwardMoe( @@ -170,7 +209,7 @@ pub fn forwardMoe( parameters: Parameters, ) !zml.Tensor { switch (parameters) { - .triton => {}, + .triton, .fly => {}, .flashinfer_cutlass, .mosaic_tpu, .metal => { stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); stdx.debug.assert(opts.gate_up_layout == .split, "Non-Triton MoE backends require split gate/up columns", .{}); @@ -402,14 +441,14 @@ pub fn forwardMoe( runner_options, ); }, - .triton => b: { - const args: triton.FusedExpertsArgs = .{ + inline .triton, .fly => |p, backend| b: { + const args: fused_experts.FusedExpertsArgs = .{ .hidden_states = input, .gate_up = gate_up, .down = down, .topk_weights = topk_weights, .topk_ids = topk_ids, - .activation = parameters.triton.activation, + .activation = p.activation, .activation_threshold = opts.activation_threshold, .quantize_input = opts.quantize_input, .gate_up_layout = opts.gate_up_layout, @@ -418,12 +457,12 @@ pub fn forwardMoe( const expert_partition = gate_up.weight.shape().partition(.expert); if (!expert_partition.eql(.init(.experts))) { - break :b try triton.fusedExpertsImpl(args); + break :b try fused_experts.fusedExperts(args, backend); } break :b zml.ops.manualComputation( (struct { - args: triton.FusedExpertsArgs, + args: fused_experts.FusedExpertsArgs, global_num_experts: i64, fn call(self: @This(), _: zml.Shape) zml.Tensor { @@ -442,7 +481,7 @@ pub fn forwardMoe( zml.Tensor.scalar(-1, .i32), ); - const local_output = triton.fusedExpertsImpl(mapped_args) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); + const local_output = fused_experts.fusedExperts(mapped_args, backend) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); const local_reshaped = local_output.reshape(local_args.hidden_states.shape().dims()).withTags(.{ .b, .s, .d }); return zml.ops.allReduce(local_reshaped, zml.Tensor.add); } diff --git a/zml/moe/triton.zig b/zml/moe/triton.zig index 9f39cec6..df53311f 100644 --- a/zml/moe/triton.zig +++ b/zml/moe/triton.zig @@ -1,271 +1,20 @@ const std = @import("std"); - const stdx = @import("stdx"); - const zml = @import("../zml.zig"); -const DataType = zml.DataType; const Tensor = zml.Tensor; const Shape = zml.Shape; -const tri = zml.kernel.triton; -const toDType = tri.from; +const DataType = zml.DataType; +const log = std.log.scoped(.moe); const kernels = @import("triton_kernels/triton_kernels.zig"); +const toDType = zml.kernel.triton.from; +const shared = @import("fused_experts.zig"); -const log = std.log.scoped(.moe_triton); +pub const Parameters = shared.Parameters; +pub const GateUpLayout = shared.GateUpLayout; +pub const RoutingWeightPlacement = shared.RoutingWeightPlacement; +pub const FusedExpertsArgs = shared.FusedExpertsArgs; -test { - std.testing.refAllDecls(@This()); -} - -pub const Parameters = struct { - num_experts_per_tok: u32, - activation: ActivationMode, - - pub const ActivationMode = enum { - silu, - relu, - gelu, - }; - - pub const InitOptions = struct { - num_experts_per_tok: u32, - activation: ActivationMode, - }; - - pub fn init(opts: InitOptions) Parameters { - return .{ - .num_experts_per_tok = opts.num_experts_per_tok, - .activation = opts.activation, - }; - } -}; - -pub const GateUpLayout = enum { split, interleaved }; -pub const RoutingWeightPlacement = enum { before_down, after_down }; - -pub const FusedExpertsArgs = struct { - hidden_states: Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - topk_weights: Tensor, - topk_ids: Tensor, - activation: Parameters.ActivationMode = .silu, - expert_map: ?Tensor = null, - activation_threshold: ?f32 = null, - /// Use FP8 activations for FP8 weights; false keeps BF16 activations. - quantize_input: bool, - gate_up_layout: GateUpLayout, - routing_weight_placement: RoutingWeightPlacement, -}; - -pub fn fusedExpertsImpl(opts: FusedExpertsArgs) !Tensor { - const hidden_states = opts.hidden_states; - const topk_weights = opts.topk_weights; - const topk_ids = opts.topk_ids; - const b = hidden_states.dim(.b); - const s = hidden_states.dim(.s); - const num_tokens = b * s; - var launch_config = launchConfigForTokens(num_tokens); - - const gate_up_scheme = opts.gate_up.quantizationScheme(); - const down_scheme = opts.down.quantizationScheme(); - if (gate_up_scheme) |scheme| switch (scheme) { - .nvfp4 => return error.UnsupportedQuantization, - .fp8_block128 => launch_config.block_size_k = 128, - .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, - }; - - var down_launch_config = launchConfigForTokens(num_tokens); - if (down_scheme) |scheme| switch (scheme) { - .fp8_block128 => down_launch_config.block_size_k = 128, - .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, - .nvfp4 => return error.UnsupportedQuantization, - }; - - const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); - const gate_up = opts.gate_up.weight.withTags(.{ .expert, .out, .in }); - const down = opts.down.weight.withTags(.{ .expert, .out, .mid }); - const routing_weights = topk_weights.reshape(.{ .token = num_tokens, .in = topk_weights.dim(.top_expert) }).withTags(.{ .token, .topk }); - const ids = topk_ids.reshape(.{ .token = num_tokens, .in = topk_ids.dim(.top_expert) }).withTags(.{ .token, .topk }); - - stdx.debug.assert(hidden.dtype() == .bf16, "expected BF16 hidden states, got {}", .{hidden.dtype()}); - stdx.debug.assert(if (gate_up_scheme == .mxfp4) gate_up.dtype() == .u8 or gate_up.dtype() == .i8 or gate_up.dtype() == .f4e2m1 else gate_up.dtype() == .bf16 or gate_up.dtype() == .f8e4m3fn or gate_up.dtype() == .f8e4m3fnuz, "unsupported gate/up weight dtype {}", .{gate_up.dtype()}); - stdx.debug.assert(if (down_scheme == .mxfp4) down.dtype() == .u8 or down.dtype() == .i8 or down.dtype() == .f4e2m1 else down.dtype() == .bf16 or down.dtype() == .f8e4m3fn or down.dtype() == .f8e4m3fnuz, "unsupported down weight dtype {}", .{down.dtype()}); - stdx.debug.assert(routing_weights.dtype() == .f32 or routing_weights.dtype() == .bf16, "expected FP32 or BF16 routing weights, got {}", .{routing_weights.dtype()}); - stdx.debug.assert(ids.dtype() == .i32, "expected I32 expert ids, got {}", .{ids.dtype()}); - const gate_up_k = gate_up.dim(.in) * @as(i64, if (gate_up_scheme == .mxfp4 and gate_up.dtype() != .f4e2m1) 2 else 1); - const down_k = down.dim(.mid) * @as(i64, if (down_scheme == .mxfp4 and down.dtype() != .f4e2m1) 2 else 1); - stdx.debug.assert(hidden.dim(.in) == gate_up_k, "hidden width {} must match gate/up input width {}", .{ hidden.dim(.in), gate_up_k }); - const activation_reduction: i64 = if (opts.activation == .relu) 1 else 2; - stdx.debug.assert(@rem(gate_up.dim(.out), activation_reduction) == 0, "gate/up output width {} must be divisible by {}", .{ gate_up.dim(.out), activation_reduction }); - stdx.debug.assert(down_k == @divFloor(gate_up.dim(.out), activation_reduction), "down input width {} must match activated width {}", .{ down_k, @divFloor(gate_up.dim(.out), activation_reduction) }); - stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "routing ids and weights must match hidden token count {}, got {} and {}", .{ hidden.dim(.token), ids.dim(.token), routing_weights.dim(.token) }); - stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "routing ids and weights must have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), routing_weights.dim(.topk) }); - stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "gate/up and down expert counts must match, got {} and {}", .{ gate_up.dim(.expert), down.dim(.expert) }); - if (opts.expert_map) |expert_map| - stdx.debug.assert(expert_map.dtype() == .i32 and expert_map.rank() == 1, "expected a rank-1 I32 expert map, got rank {} and dtype {}", .{ expert_map.rank(), expert_map.dtype() }); - - const num_experts = if (opts.expert_map) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); - const routing = prepareRouting(ids, num_experts, @intCast(launch_config.block_size_m)); - - const expert_ids = if (opts.expert_map) |expert_map| - expert_map.gather(.{ .expert = routing.expert_ids }, .{}).withTags(.{.g}) - else - routing.expert_ids; - - var hidden_quant = hidden; - var input_scale: ?Tensor = null; - - if (opts.quantize_input and gate_up_scheme != .mxfp4) { - if (gate_up_scheme) |scheme| hidden_quant, input_scale = quantizeFp8Input(hidden, scheme, gate_up.dtype()); - } - - const gate_up_out = callFusedMoe(.{ - .input = hidden_quant, - .weight = gate_up, - .bias = opts.gate_up.bias, - .input_scale = input_scale, - .weight_scale = opts.gate_up.quantizationScales(), - .routing = routing, - .expert_ids = expert_ids, - .launch_config = launch_config, - .quant_scheme = gate_up_scheme, - .top_k = @intCast(ids.dim(.topk)), - .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), - }); - - var activated = applyExpertActivation(gate_up_out, opts.activation, opts.activation_threshold, opts.gate_up_layout); - if (opts.routing_weight_placement == .before_down) { - const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); - activated = activated.mul(weights.broad(activated.shape())); - } - var activated_quant = activated.convert(.bf16); - input_scale = null; - if (opts.quantize_input and down_scheme != .mxfp4) { - if (down_scheme) |scheme| activated_quant, input_scale = quantizeFp8Input(activated, scheme, down.dtype()); - } - - const down_out = callFusedMoe(.{ - .input = activated_quant, - .weight = down, - .bias = opts.down.bias, - .input_scale = input_scale, - .weight_scale = opts.down.quantizationScales(), - .routing_weights = if (opts.routing_weight_placement == .after_down) routing_weights else null, - .routing = routing, - .expert_ids = expert_ids, - .launch_config = down_launch_config, - .quant_scheme = down_scheme, - .top_k = 1, - .output_shape = Shape.init(.{ .token = b * s, .topk = ids.dim(.topk), .out = down.dim(.out) }, .bf16), - }); - - const output = down_out.convert(.f32).sum(.topk).squeeze(.topk).convert(.bf16); - - return output.reshape(.{ .b = b, .token = s, .out = down.dim(.out) }); -} - -fn applyExpertActivation(input: Tensor, mode: Parameters.ActivationMode, activation_threshold: ?f32, layout: GateUpLayout) Tensor { - const x = input.convert(.f32); - if (mode == .relu) { - const clipped = if (activation_threshold) |limit| x.minimum(Tensor.scalar(limit, x.dtype())) else x; - return clipped.relu().powByConst(2); - } - - const mid = @divFloor(x.dim(.out), 2); - var gate, var up = switch (layout) { - .split => .{ x.slice(.out, .{ .end = mid }), x.slice(.out, .{ .start = mid }) }, - .interleaved => .{ x.slice(.out, .{ .start = 0, .step = 2 }), x.slice(.out, .{ .start = 1, .step = 2 }) }, - }; - if (activation_threshold) |limit_| { - const limit = Tensor.scalar(limit_, x.dtype()); - gate = gate.minimum(limit); - up = up.clamp(limit.negate(), limit); - } - return switch (mode) { - .silu => gate.silu().mul(up), - .relu => unreachable, - .gelu => gate.gelu().mul(up), - }; -} - -test "SwiGLU uses FP32 math for split and interleaved BF16 inputs" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = zml.testing.env(); - const Local = struct { - fn forward(x: Tensor, layout: GateUpLayout, threshold: ?f32) Tensor { - return applyExpertActivation(x, .silu, threshold, layout); - } - }; - const x: Tensor = .init(.{ .token = 1, .out = 6 }, .bf16); - const gates = [_]f32{ 0.75, -1.25, 3.5 }; - const ups = [_]f32{ 0.875, -2.25, 4.5 }; - for ([_]GateUpLayout{ .split, .interleaved }) |layout| { - var values: [6]zml.floats.BFloat16 = undefined; - for (gates, ups, 0..) |gate, up, i| { - values[if (layout == .split) i else 2 * i] = .fromF32(gate); - values[if (layout == .split) i + 3 else 2 * i + 1] = .fromF32(up); - } - var input = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&values)); - defer input.deinit(); - for ([_]?f32{ null, 2 }) |threshold| { - var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, threshold }, .{}); - defer exe.deinit(); - try zml.testing.expectEqualShapes(Shape.init(.{ .token = 1, .out = 3 }, .f32), exe.output_shapes[0]); - var output = try zml.testing.autoCall(allocator, io, &exe, Local.forward, .{input}); - defer output.deinit(); - var actual = try output.toSliceAlloc(allocator, io); - defer actual.free(allocator); - for (gates, ups, actual.constItems(f32)) |gate, up, value| { - const g = if (threshold) |limit| @min(gate, limit) else gate; - const u = if (threshold) |limit| std.math.clamp(up, -limit, limit) else up; - try std.testing.expectApproxEqAbs(g / (1 + @exp(-g)) * u, value, 1e-6); - } - } - } -} - -test "ReLU squared activation preserves width and applies threshold before squaring" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = zml.testing.env(); - const Local = struct { - fn forward(x: Tensor, threshold: ?f32) Tensor { - return applyExpertActivation(x, .relu, threshold, .split); - } - }; - const x: Tensor = .init(.{ .token = 1, .out = 5 }, .f32); - const values = [_]f32{ -3, 0, 1, 2, 4 }; - var input = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&values)); - defer input.deinit(); - for ([_]?f32{ null, 2 }) |threshold| { - var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, threshold }, .{}); - defer exe.deinit(); - try zml.testing.expectEqualShapes(x.shape(), exe.output_shapes[0]); - var output = try zml.testing.autoCall(allocator, io, &exe, Local.forward, .{input}); - defer output.deinit(); - var actual = try output.toSliceAlloc(allocator, io); - defer actual.free(allocator); - const expected = [_]f32{ 0, 0, 1, 4, if (threshold != null) 4 else 16 }; - try std.testing.expectEqualSlices(f32, &expected, actual.constItems(f32)); - } -} - -/// Build the inputs tuple for FusedMoe and invoke it via `K.call(...)`. -fn callFusedMoe(opts: struct { - input: Tensor, - weight: Tensor, - bias: ?Tensor = null, - input_scale: ?Tensor = null, - weight_scale: ?Tensor = null, - routing_weights: ?Tensor = null, - routing: Routing, - expert_ids: Tensor, - launch_config: LaunchConfig, - quant_scheme: ?zml.Quantization.Scheme = null, - top_k: usize, - output_shape: Shape, -}) Tensor { +pub fn call(opts: shared.GemmOptions) Tensor { // Native FP4 tensors expose logical K; the Triton operand is byte-packed. const weight = if (opts.quant_scheme == .mxfp4 and opts.weight.dtype() == .f4e2m1) opts.weight.reshape(opts.weight.shape().setDim(2, @divExact(opts.weight.dim(2), 2)).append(.{ .nibble = 2 })).bitCast(.u8) @@ -368,224 +117,7 @@ fn callFusedMoe(opts: struct { ).c; } -test "FP8 routed GEMM with bias matches dequantized weights" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = zml.testing.env(); - if (platform.target != .cuda) return error.SkipZigTest; - - const Local = struct { - const Outputs = struct { actual: Tensor, expected: Tensor }; - - fn expandScales(scales: Tensor, shape: Shape) Tensor { - var expanded = scales.convert(.f32); - for (0..shape.rank()) |axis| { - const factor = @divExact(shape.dim(axis), expanded.dim(axis)); - const broad_shape = expanded.shape().insert(axis + 1, .{factor}); - const axes = Shape.range(broad_shape.rank(), .i64).remove(axis + 1); - expanded = expanded.broadcast(broad_shape, axes.dims()) - .reshape(expanded.shape().setDim(axis, shape.dim(axis))); - } - return expanded.withTags(shape.tags()); - } - - fn forward(x: Tensor, w: Tensor, scales: Tensor, ids: Tensor, weights: Tensor, bias: Tensor, scheme: zml.Quantization.Scheme, mx_native: bool, quantize_input: bool) Outputs { - const weight = w.convert(.f8e4m3fn); - var input = x; - var input_scale: ?Tensor = null; - var reference_input = x.convert(.f32); - if (quantize_input) { - input, input_scale = quantizeFp8Input(x, scheme, .f8e4m3fn); - const scale = if (scheme == .mxfp8) input_scale.?.bitCast(.f8e8m0) else input_scale.?; - reference_input = input.convert(.f32).withTags(x.shape().tags()) - .mul(expandScales(scale.convert(.f32), x.shape())); - } - const weight_scale = if (scheme == .mxfp8) blk: { - const native = scales.convert(.f8e8m0); - break :blk if (mx_native) native else native.bitCast(.u8); - } else scales; - const config = launchConfigForTokens(8); - const routing = prepareRouting(ids, w.dim(.expert), @intCast(config.block_size_m)); - const actual = callFusedMoe(.{ - .input = input, - .weight = weight, - .bias = bias, - .input_scale = input_scale, - .weight_scale = weight_scale, - .routing_weights = weights, - .routing = routing, - .expert_ids = routing.expert_ids, - .launch_config = config, - .quant_scheme = scheme, - .top_k = 1, - .output_shape = Shape.init(.{ .token = x.dim(.token), .topk = 1, .out = w.dim(.out) }, .bf16), - }).squeeze(.topk); - const expanded_weight = weight.convert(.f32).mul(expandScales(scales, weight.shape())); - const selected_weight = expanded_weight.gather(.{ .expert = ids.squeeze(.topk) }, .{}); - const dot = reference_input.dot(selected_weight, .in); - const selected_bias = if (bias.rank() == 2) - bias.gather(.{ .expert = ids.squeeze(.topk) }, .{}) - else - bias; - const unweighted = dot.add(selected_bias.convert(.f32).broad(dot.shape())); - const expected = unweighted.mul(weights.squeeze(.topk).convert(.f32).broad(unweighted.shape())).convert(.bf16); - return .{ .actual = actual, .expected = expected }; - } - }; - - for ([_]zml.Quantization.Scheme{ .mxfp8, .fp8_per_tensor, .fp8_per_channel, .fp8_block128 }) |scheme| { - for ([_]bool{ false, true }) |quantize_input| { - for ([_]i64{ 1, 17 }) |tokens| { - const n: i64 = if (scheme == .fp8_block128) 256 else 48; - const scale_dims: [3]i64 = switch (scheme) { - .mxfp8 => .{ 8, n, 8 }, - .fp8_per_tensor => if (tokens == 1) .{ 1, 1, 1 } else .{ 8, 1, 1 }, - .fp8_per_channel => .{ 8, n, 1 }, - .fp8_block128 => .{ 8, 2, 2 }, - else => unreachable, - }; - const x: Tensor = .init(.{ .token = tokens, .in = 256 }, .bf16); - const w: Tensor = .init(.{ .expert = 8, .out = n, .in = 256 }, .bf16); - const scales: Tensor = .init(.{ scale_dims[0], scale_dims[1], scale_dims[2] }, .f32); - const ids: Tensor = .init(.{ .token = tokens, .topk = 1 }, .i32); - const weights: Tensor = .init(.{ .token = tokens, .topk = 1 }, .bf16); - const bias: Tensor = if (tokens == 1) - .init(.{ .out = n }, .bf16) - else - .init(.{ .expert = 8, .out = n }, .f32); - var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, w, scales, ids, weights, bias, scheme, tokens == 1, quantize_input }, .{}); - defer exe.deinit(); - var buffers: [6]zml.Buffer = undefined; - var initialized: usize = 0; - defer for (buffers[0..initialized]) |*buffer| buffer.deinit(); - for ([_]Shape{ x.shape(), w.shape(), scales.shape(), ids.shape(), weights.shape(), bias.shape() }, 0..) |shape, i| { - const slice = try zml.Slice.alloc(allocator, shape); - defer slice.free(allocator); - for (0..shape.count()) |j| { - switch (i) { - 2 => slice.items(f32)[j] = @as(f32, @floatFromInt(@as(u32, 1) << @as(u5, @intCast(j % 4)))) / 16, - 3 => slice.items(i32)[j] = @intCast((j * 3 + 5) % 8), - 5 => { - const value = @as(f32, @floatFromInt(@as(i32, @intCast(j % 17)) - 8)) / 2; - if (shape.dtype() == .f32) { - slice.items(f32)[j] = value; - } else { - slice.items(zml.floats.BFloat16)[j] = .fromF32(value); - } - }, - else => slice.items(zml.floats.BFloat16)[j] = .fromF32(if (i == 4) - @as(f32, @floatFromInt(1 + j % 3)) / 4 - else - @as(f32, @floatFromInt(@as(i32, @intCast((j * 7 + j / 256) % 13)) - 6)) / 4), - } - } - buffers[i] = try zml.Buffer.fromSlice(io, platform, slice, .replicated); - initialized += 1; - } - var output = try zml.testing.autoCall(allocator, io, &exe, Local.forward, .{ buffers[0], buffers[1], buffers[2], buffers[3], buffers[4], buffers[5] }); - defer zml.Buffer.deinitAll(Local.Outputs, &output); - zml.testing.expectClose(io, output.expected, output.actual, .{ .absolute_tolerance = 0.03125, .relative_tolerance = 0.01 }) catch |err| { - log.warn("FP8 routed GEMM failed for scheme={s}, quantize_input={}, tokens={}", .{ @tagName(scheme), quantize_input, tokens }); - return err; - }; - } - } - } -} - -test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = zml.testing.env(); - if (platform.target != .cuda) return error.SkipZigTest; - const Local = struct { - fn forward(x: Tensor, layout: GateUpLayout, placement: RoutingWeightPlacement, storage_dtype: DataType) Tensor { - const fp4 = storage_dtype != .bf16; - const columns = Tensor.arange(.{ .end = 256 }, .i32).withTags(.{.dout}); - const is_gate = switch (layout) { - .split => columns.cmp(.LT, Tensor.scalar(128, .i32)), - .interleaved => columns.remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), - }; - const dtype: DataType = if (fp4) .u8 else .bf16; - // E2M1 codes 1, 2, 3 encode 0.5, 1, 1.5; each byte holds two values. - const gate_value = if (fp4) Tensor.scalar(0x11, .u8) else Tensor.scalar(0.5, .bf16); - const up_value = if (fp4) Tensor.scalar(0x33, .u8) else Tensor.scalar(1.5, .bf16); - const gate_up_values = is_gate.select(gate_value, up_value); - var gate_up: zml.nn.Linear = .{ - .weight = gate_up_values.broad(Shape.init(.{ .expert = 8, .dout = 256, .d = @as(i64, if (fp4) 64 else 128) }, dtype)), - .tag = Shape.toTag(.d), - .quantization = if (fp4) .{ - .scheme = .mxfp4, - .scales = Tensor.scalar(127, .u8).broad(Shape.init(.{ .expert = 8, .dout = 256, .d = 4 }, .u8)), - } else null, - }; - var down: zml.nn.Linear = .{ - .weight = (if (fp4) Tensor.scalar(0x22, .u8) else Tensor.scalar(1, .bf16)).broad(Shape.init(.{ .expert = 8, .d = 128, .dout = @as(i64, if (fp4) 64 else 128) }, dtype)), - .bias = Tensor.scalar(2, .bf16).broad(Shape.init(.{ .expert = 8, .d = 128 }, .bf16)), - .tag = Shape.toTag(.dout), - .quantization = if (fp4) .{ - .scheme = .mxfp4, - .scales = Tensor.scalar(127, .u8).broad(Shape.init(.{ .expert = 8, .d = 128, .dout = 4 }, .u8)), - } else null, - }; - if (storage_dtype == .f4e2m1) { - gate_up.weight = gate_up.weight.bitCast(.f4e2m1).reshape(.{ .expert = 8, .dout = 256, .d = 128 }); - down.weight = down.weight.bitCast(.f4e2m1).reshape(.{ .expert = 8, .d = 128, .dout = 128 }); - gate_up.quantization.?.scales = gate_up.quantization.?.scales.bitCast(.f8e8m0); - down.quantization.?.scales = down.quantization.?.scales.bitCast(.f8e8m0); - } - const route = Tensor.arange(.{ .end = 2 }, .i32).reshape(.{ .b = 1, .s = 1, .top_expert = 2 }) - .broad(Shape.init(.{ .b = 1, .s = x.dim(.s), .top_expert = 2 }, .i32)); - return fusedExpertsImpl(.{ - .hidden_states = x, - .gate_up = gate_up, - .down = down, - .topk_ids = route.addConstant(2), - .topk_weights = route.convert(.f32).addConstant(1).scale(0.25), - .gate_up_layout = layout, - .routing_weight_placement = placement, - .quantize_input = false, - }) catch unreachable; - } - }; - for ([_]DataType{ .bf16, .u8, .f4e2m1 }) |storage_dtype| { - const tokens: i64 = if (storage_dtype == .u8) 17 else 1; - const x: Tensor = .init(.{ .b = 1, .s = tokens, .d = 128 }, .bf16); - const host = try zml.Slice.alloc(allocator, x.shape()); - defer host.free(allocator); - @memset(host.items(zml.floats.BFloat16), .fromF32(1.0 / 128.0)); - var input = try zml.Buffer.fromSlice(io, platform, host, .replicated); - defer input.deinit(); - for ([_]GateUpLayout{ .split, .interleaved }) |layout| { - for ([_]RoutingWeightPlacement{ .before_down, .after_down }) |placement| { - var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, placement, storage_dtype }, .{}); - defer exe.deinit(); - var output = try zml.testing.autoCall(allocator, io, &exe, Local.forward, .{input}); - defer output.deinit(); - var actual = try output.toSliceAlloc(allocator, io); - defer actual.free(allocator); - const activation: f32 = 0.5 / (1 + @exp(@as(f32, -0.5))) * 1.5; - // The two routes have weights 0.25 and 0.5. Bias distinguishes placement. - const bias: f32 = if (placement == .before_down) 4 else 1.5; - const expected = 128 * activation * 0.75 + bias; - for (actual.constItems(zml.floats.BFloat16)) |value| { - try std.testing.expectApproxEqAbs(expected, value.toF32(), 0.5); - } - } - } - } -} - -const Routing = struct { - sorted_token_ids: Tensor, - expert_ids: Tensor, - num_tokens_post_padded: Tensor, - max_num_tokens_padded: i64, - num_assignments: i64, - naive_block_assignment: bool, -}; - -fn prepareRouting(topk_ids: Tensor, num_experts: i64, block_size_m: i64) Routing { +pub fn prepareRouting(topk_ids: Tensor, num_experts: i64, block_size_m: i64) shared.Routing { stdx.debug.assert(block_size_m > 0 and @mod(block_size_m, 16) == 0, "routing block size must be a positive multiple of 16, got {}", .{block_size_m}); const ids = topk_ids.withTags(.{ .token, .topk }).convert(.i32); const num_assignments = ids.dim(.token) * ids.dim(.topk); @@ -618,7 +150,6 @@ fn prepareRouting(topk_ids: Tensor, num_experts: i64, block_size_m: i64) Routing } fn alignBlockSize(topk_ids: Tensor, num_experts: i64, block_size_m: i64) struct { Tensor, Tensor, Tensor } { - log.debug("Using triton kernels to sort and align tokens to experts with block size {d}", .{block_size_m}); const topk_ids_ = topk_ids.withTags(.{ .token, .topk }).convert(.i32); const num_tokens = topk_ids_.dim(.token); const topk = topk_ids_.dim(.topk); @@ -714,6 +245,18 @@ fn alignBlockSize(topk_ids: Tensor, num_experts: i64, block_size_m: i64) struct return .{ sorted_token_ids, expert_ids, num_tokens_post_padded }; } +pub fn prepareInput(x: Tensor, scheme: ?zml.Quantization.Scheme, output_dtype: zml.DataType, quantize: bool) struct { Tensor, ?Tensor } { + if (quantize) { + if (scheme) |scheme_| { + if (scheme_ != .mxfp4) { + const input, const scale = quantizeFp8Input(x, scheme_, output_dtype); + return .{ input, scale }; + } + } + } + return .{ x.convert(.bf16), null }; +} + fn quantizeFp8Input(x: Tensor, scheme: zml.Quantization.Scheme, output_dtype: DataType) struct { Tensor, Tensor } { const group_size: i64, const scale_dtype: DataType, const absmax_epsilon: f32 = switch (scheme) { .mxfp8 => .{ 32, .u8, 1e-10 }, @@ -753,57 +296,3 @@ fn quantizeFp8Input(x: Tensor, scheme: zml.Quantization.Scheme, output_dtype: Da return .{ outs.y_q, outs.y_s }; } - -// ============================================================================= -// Config / validation helpers -// ============================================================================= - -const LaunchConfig = struct { - block_size_m: usize, - block_size_n: usize, - block_size_k: usize, - group_size_m: usize, - num_warps: i32, - num_stages: i32, -}; - -const launch_configs = [_]struct { tokens: i64, config: LaunchConfig }{ - .{ .tokens = 1, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 4 } }, - .{ .tokens = 2, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 4 } }, - .{ .tokens = 4, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, - .{ .tokens = 8, .config = .{ .block_size_m = 16, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, - .{ .tokens = 16, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 64, .group_size_m = 64, .num_warps = 4, .num_stages = 5 } }, - .{ .tokens = 24, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, - .{ .tokens = 32, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 128, .group_size_m = 1, .num_warps = 4, .num_stages = 2 } }, - .{ .tokens = 48, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 128, .group_size_m = 64, .num_warps = 4, .num_stages = 2 } }, - .{ .tokens = 64, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 128, .group_size_m = 1, .num_warps = 4, .num_stages = 2 } }, - .{ .tokens = 96, .config = .{ .block_size_m = 16, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, - .{ .tokens = 128, .config = .{ .block_size_m = 16, .block_size_n = 256, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, - .{ .tokens = 256, .config = .{ .block_size_m = 16, .block_size_n = 256, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, - .{ .tokens = 512, .config = .{ .block_size_m = 32, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, - .{ .tokens = 1024, .config = .{ .block_size_m = 64, .block_size_n = 128, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, - .{ .tokens = 1536, .config = .{ .block_size_m = 64, .block_size_n = 128, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, - .{ .tokens = 2048, .config = .{ .block_size_m = 128, .block_size_n = 128, .block_size_k = 64, .group_size_m = 16, .num_warps = 8, .num_stages = 3 } }, - .{ .tokens = 3072, .config = .{ .block_size_m = 128, .block_size_n = 256, .block_size_k = 64, .group_size_m = 1, .num_warps = 8, .num_stages = 4 } }, - .{ .tokens = 4096, .config = .{ .block_size_m = 128, .block_size_n = 256, .block_size_k = 64, .group_size_m = 16, .num_warps = 8, .num_stages = 4 } }, -}; - -fn launchConfigForTokens(num_tokens: i64) LaunchConfig { - var best = launch_configs[0]; - for (launch_configs[1..]) |candidate| { - // The table is ascending, so ties retain the smaller token bucket. - if (@abs(candidate.tokens - num_tokens) < @abs(best.tokens - num_tokens)) { - best = candidate; - } - } - return best.config; -} - -test "MoE launch config selects the nearest token bucket" { - try std.testing.expectEqualDeep(launchConfigForTokens(4), launchConfigForTokens(6)); - try std.testing.expectEqualDeep(launchConfigForTokens(8), launchConfigForTokens(7)); - try std.testing.expectEqualDeep(launchConfigForTokens(1), launchConfigForTokens(0)); - try std.testing.expectEqualDeep(launchConfigForTokens(4096), launchConfigForTokens(8192)); - try std.testing.expectEqual(@as(usize, 64), launchConfigForTokens(16).group_size_m); - try std.testing.expectEqual(@as(i32, 5), launchConfigForTokens(16).num_stages); -} From 550f2bbee01426de73d5f729b41ed96abc68125b Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 14 Sep 2026 18:19:03 +0200 Subject: [PATCH 683/881] zml/Compiler: allocate inputs/outputs for func.func (#806) When we have a lot of inputs/outputs, it can overflow the defaults we put in the `func.func` helper. --- zml/Compiler.zig | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 6a749e81..d15a542c 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -582,8 +582,16 @@ fn finalizeMlirFunc(compiler: *Compiler, fn_scope: *Scope, input_info: std.Multi output_attrs.* = .dict(mlir_ctx, attrs_list.constSlice()); } + const input_types = try arena.alloc(*const mlir.Type, input_info.len); + for (input_types, 0..) |*input_type, i| input_type.* = fn_scope.block.argument(i).type_(); + + const output_types = try arena.alloc(*const mlir.Type, output_info.len); + for (output_types, output_info.items(.value)) |*output_type, value| output_type.* = value.type_(); + const mlir_func = dialects.func.func(mlir_ctx, .{ .name = "main", + .args = input_types, + .results = output_types, .block = fn_scope.block, .location = .unknown(mlir_ctx), .args_attributes = input_attributes, From 91c8b315022a1a235ef0d57ea162e6d1f4a7e0c5 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 15 Sep 2026 16:07:39 +0200 Subject: [PATCH 684/881] platforms: bump plugins for CPU, ROCm, CUDA and Metal (#809) Bump to `202609151311.37.1.f1b6432d55a0` --- platforms/cpu/cpu.bzl | 12 ++++++------ platforms/cuda/cuda.bzl | 8 ++++---- platforms/metal/metal.bzl | 4 ++-- platforms/rocm/rocm.bzl | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index d41a00b0..63ec0612 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -23,22 +23,22 @@ def _cpu_plugin_impl(mctx): http_archive( name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "f1670d21b92102b8c7c7f9d0f00d6a7a1a5d90a83d7e28ba5fbeb1f9ce4c170c", - url = "https://mirror.zml.ai/plugins/202609101243.20.1.7ca6884ea2cb/zml-cpu-linux-amd64.tar.zst", + sha256 = "084d3113c5bffd11235ca000dc9756596f1aa8ae8adb5ba8c2013850ed151ca2", + url = "https://mirror.zml.ai/plugins/202609151311.37.1.f1b6432d55a0/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "d8c7dea830773eb37bb6161f1ed62f69f30bd20a2d669083c5d6a7d60c19844b", - url = "https://mirror.zml.ai/plugins/202609101243.20.1.7ca6884ea2cb/zml-cpu-darwin-amd64.tar.zst", + sha256 = "479dfbed01449b22e23f29ec6512ec73fad19b5110007a82bb98dccae486a729", + url = "https://mirror.zml.ai/plugins/202609151311.37.1.f1b6432d55a0/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "b8c5a1d15fe3b6ce8520852901ccec3cb12d092b052fdd527449d822833becba", - url = "https://mirror.zml.ai/plugins/202609101243.20.1.7ca6884ea2cb/zml-cpu-darwin-arm64.tar.zst", + sha256 = "f5bab3aed93ae2fd9a6d5878d324a19cd8404993d6441f0258149cd98c228054", + url = "https://mirror.zml.ai/plugins/202609151311.37.1.f1b6432d55a0/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 4884f35c..68d6f420 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -215,12 +215,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "2ef0e28330d22ce3039adb1de0d5dabf8bd7126988f9b61ba427946d6c60cb93", - "url": "https://mirror.zml.ai/plugins/202609101243.20.1.7ca6884ea2cb/zml-cuda-linux-amd64.tar.zst", + "sha256": "fe259dd69c67da5d48bc2bdd1f30a8722c2bbb7171abc0b6f23837b52e29b329", + "url": "https://mirror.zml.ai/plugins/202609151311.37.1.f1b6432d55a0/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "f8180188c4557c526571b90cb54354819043d05c492228a9d1179ac2bef26862", - "url": "https://mirror.zml.ai/plugins/202609101243.20.1.7ca6884ea2cb/zml-cuda-linux-arm64.tar.zst", + "sha256": "0b4843284aee3a5be6a9e33c9db12e11eaa4f9f1393c8569a93290ddb9b47791", + "url": "https://mirror.zml.ai/plugins/202609151311.37.1.f1b6432d55a0/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 26c78880..9a9d9b8a 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "e329f4c89f9e5c64cf46c8665cb89d21ffbd198c4d4e7facdeb41654388521b2", - url = "https://mirror.zml.ai/plugins/202609101243.20.1.7ca6884ea2cb/zml-metal-darwin-arm64.tar.zst", + sha256 = "775504055ce16fc2be8ceaaef843c15b5a051ce3d41f78534392c231adc59a3f", + url = "https://mirror.zml.ai/plugins/202609151311.37.1.f1b6432d55a0/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index cc617ebc..183650e5 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202609101243.20.1.7ca6884ea2cb/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "d2a311f15532691d7335c0436d4dda13d39e8597e4bb10dbe83b11b02a627546" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202609151311.37.1.f1b6432d55a0/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "869f84a01b925c14ba9e49249535e3fd190e9c56dc0426c1e52a12c84acb1d55" def _rocm_package_name(name): return name + _ROCM_VERSION From 94257d37477770420cfcb25b213d8ed5dc8f101b Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Wed, 16 Sep 2026 14:02:46 +0200 Subject: [PATCH 685/881] kernels/cute: Zig CuTe DSL and XLA custom call and bump xprof (#810) Adds a Zig front end for NVIDIA's CuTe DSL, shaped after `cutlass.cute`, plus the `__gpu$xla.gpu.cute` custom call that compiles and launches it on CUDA. - `mlir/dialects/cute_ir`: bindings for the `cute` and `cute_nvgpu` dialects - `zml.ops.cute`: emits the custom call with the textual module, `grid` and `block`. - `kernels/cute` + `zml.kernel.cute.Kernel`: builder, values, tensors with coordinate `get`/`set`, thread/block indices, static smem, sync, `scf`. See `kernels/cute/README.md`. - bump `xprof-nightly` --------- Co-authored-by: Steeve Morin --- MODULE.bazel | 25 +- kernels/cute/BUILD.bazel | 24 + kernels/cute/README.md | 127 + kernels/cute/builder.zig | 950 ++++++ kernels/cute/dtype.zig | 62 + mlir/dialects/BUILD.bazel | 5 +- mlir/dialects/cute_ir/BUILD.bazel | 32 + mlir/dialects/cute_ir/README.md | 107 + mlir/dialects/cute_ir/cute_ir.zig | 2618 +++++++++++++++++ mlir/dialects/cute_ir/cute_nvgpu.zig | 1983 +++++++++++++ .../include/cute_ir-c/Dialect/BUILD.bazel | 24 + .../cute_ir/include/cute_ir-c/Dialect/Cute.h | 23 + .../cute_ir-c/Dialect/CuteAttributes.h | 65 + .../include/cute_ir-c/Dialect/CuteNVGPU.h | 21 + .../cute_ir-c/Dialect/CuteNVGPUTypes.h | 323 ++ .../include/cute_ir-c/Dialect/CuteTypes.h | 115 + .../cute_ir/Dialect/Cute/IR/BUILD.bazel | 92 + .../Dialect/Cute/IR/CuteDialectPrivate.h | 14 + .../cute_ir/Dialect/Cute/IR/CuteOpsPrivate.td | 428 +++ .../cute_ir/Dialect/Cute/IR/CuteTypesAll.td | 2 + .../Dialect/Cute/IR/CuteTypesPrivate.td | 92 + .../cute_ir/Dialect/CuteNVGPU/IR/BUILD.bazel | 88 + .../Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h | 18 + .../Dialect/CuteNVGPU/IR/CuteNVGPUDialect.td | 34 + .../Dialect/CuteNVGPU/IR/CuteNVGPUOps.td | 378 +++ .../Dialect/CuteNVGPU/IR/CuteNVGPUTypes.td | 140 + .../cute_ir/Registration/Registration.h | 46 + mlir/dialects/cute_ir/lib/CAPI/BUILD.bazel | 32 + mlir/dialects/cute_ir/lib/CAPI/Cute.cpp | 17 + .../cute_ir/lib/CAPI/CuteAttributes.cpp | 130 + mlir/dialects/cute_ir/lib/CAPI/CuteNVGPU.cpp | 19 + .../cute_ir/lib/CAPI/CuteNVGPUTypes.cpp | 1039 +++++++ mlir/dialects/cute_ir/lib/CAPI/CuteTypes.cpp | 358 +++ .../lib/Dialect/CuteNVGPU/IR/BUILD.bazel | 12 + .../Dialect/CuteNVGPU/IR/CuteNVGPUDialect.cpp | 135 + .../cute_ir/lib/Dialect/IR/BUILD.bazel | 26 + .../cute_ir/lib/Dialect/IR/CuteDialect.cpp | 59 + .../cute_ir/lib/Dialect/IR/CuteOpsPrivate.cpp | 6 + .../cute_ir/lib/Dialect/IR/CuteTypes.cpp | 5 + .../lib/Dialect/IR/CuteTypesPrivate.cpp | 333 +++ .../cute_ir/lib/Registration/RegisterCute.cpp | 63 + mlir/dialects/cute_ir/test/BUILD.bazel | 23 + .../cute_ir/test/CAPIRegistrationTest.cpp | 96 + .../cute_ir/test/TypeConstructorsTest.c | 70 + mlir/dialects/dialects.zig | 1 + mlir/mlir.zig | 37 +- platforms/cpu/BUILD.bazel | 2 +- platforms/cpu/cpu.bzl | 16 +- platforms/cuda/cuda.bzl | 8 +- platforms/cuda/libzml_cuda.BUILD.bazel | 3 +- platforms/metal/libzml_metal.BUILD.bazel | 11 +- platforms/metal/metal.bzl | 4 +- platforms/rocm/libzml_rocm.BUILD.bazel | 2 +- platforms/rocm/rocm.bzl | 4 +- third_party/cute_ir/BUILD.bazel | 0 third_party/cute_ir/cute_ir.bazel | 197 ++ third_party/cute_ir/repo.bzl | 10 + third_party/cutlass/cutlass.bazel | 111 + third_party/non_module_deps.bzl | 2 + tools/xprof/requirements.lock.txt | 1589 +++++----- zml/BUILD.bazel | 1 + zml/kernel.zig | 137 + zml/ops.zig | 135 + 63 files changed, 11757 insertions(+), 772 deletions(-) create mode 100644 kernels/cute/BUILD.bazel create mode 100644 kernels/cute/README.md create mode 100644 kernels/cute/builder.zig create mode 100644 kernels/cute/dtype.zig create mode 100644 mlir/dialects/cute_ir/BUILD.bazel create mode 100644 mlir/dialects/cute_ir/README.md create mode 100644 mlir/dialects/cute_ir/cute_ir.zig create mode 100644 mlir/dialects/cute_ir/cute_nvgpu.zig create mode 100644 mlir/dialects/cute_ir/include/cute_ir-c/Dialect/BUILD.bazel create mode 100644 mlir/dialects/cute_ir/include/cute_ir-c/Dialect/Cute.h create mode 100644 mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteAttributes.h create mode 100644 mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPU.h create mode 100644 mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPUTypes.h create mode 100644 mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteTypes.h create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/BUILD.bazel create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteDialectPrivate.h create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOpsPrivate.td create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesAll.td create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesPrivate.td create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/BUILD.bazel create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.td create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUOps.td create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUTypes.td create mode 100755 mlir/dialects/cute_ir/include/cute_ir/Registration/Registration.h create mode 100644 mlir/dialects/cute_ir/lib/CAPI/BUILD.bazel create mode 100644 mlir/dialects/cute_ir/lib/CAPI/Cute.cpp create mode 100644 mlir/dialects/cute_ir/lib/CAPI/CuteAttributes.cpp create mode 100644 mlir/dialects/cute_ir/lib/CAPI/CuteNVGPU.cpp create mode 100644 mlir/dialects/cute_ir/lib/CAPI/CuteNVGPUTypes.cpp create mode 100644 mlir/dialects/cute_ir/lib/CAPI/CuteTypes.cpp create mode 100644 mlir/dialects/cute_ir/lib/Dialect/CuteNVGPU/IR/BUILD.bazel create mode 100644 mlir/dialects/cute_ir/lib/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.cpp create mode 100644 mlir/dialects/cute_ir/lib/Dialect/IR/BUILD.bazel create mode 100755 mlir/dialects/cute_ir/lib/Dialect/IR/CuteDialect.cpp create mode 100644 mlir/dialects/cute_ir/lib/Dialect/IR/CuteOpsPrivate.cpp create mode 100644 mlir/dialects/cute_ir/lib/Dialect/IR/CuteTypes.cpp create mode 100644 mlir/dialects/cute_ir/lib/Dialect/IR/CuteTypesPrivate.cpp create mode 100755 mlir/dialects/cute_ir/lib/Registration/RegisterCute.cpp create mode 100644 mlir/dialects/cute_ir/test/BUILD.bazel create mode 100644 mlir/dialects/cute_ir/test/CAPIRegistrationTest.cpp create mode 100644 mlir/dialects/cute_ir/test/TypeConstructorsTest.c create mode 100644 third_party/cute_ir/BUILD.bazel create mode 100644 third_party/cute_ir/cute_ir.bazel create mode 100644 third_party/cute_ir/repo.bzl create mode 100644 third_party/cutlass/cutlass.bazel diff --git a/MODULE.bazel b/MODULE.bazel index 7ac550e7..f598674f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -3,6 +3,7 @@ module( ) bazel_dep(name = "abseil-cpp", version = "20260526.0", repo_name = "com_google_absl") + # Work around abseil-cpp#2071 until the BCR release includes the Clang BMI2 fix. single_version_override( module_name = "abseil-cpp", @@ -12,6 +13,7 @@ single_version_override( ], version = "20260526.0", ) + bazel_dep(name = "bazel_lib", version = "3.2.2") bazel_dep(name = "aspect_rules_py", version = "1.8.4") bazel_dep(name = "bazel_skylib", version = "1.9.0") @@ -22,11 +24,11 @@ bazel_dep(name = "patchelf", version = "0.18.0.bcr.1") bazel_dep(name = "pcre2", version = "10.45") bazel_dep(name = "platforms", version = "1.1.0") bazel_dep(name = "protobuf", version = "35.0", repo_name = "com_google_protobuf") - bazel_dep(name = "pybind11_bazel", version = "3.0.1") bazel_dep(name = "re2", version = "2025-11-05", repo_name = "com_googlesource_code_re2") bazel_dep(name = "rules_cc", version = "0.2.19") bazel_dep(name = "rules_distroless", version = "0.8.0") + # Once the rules_distroless can parse unspaced apt version constraints # Update the rules_distroless version and remove the patch # https://github.com/bazel-contrib/rules_distroless/pull/229 @@ -38,11 +40,13 @@ single_version_override( ], version = "0.8.0", ) + bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep(name = "rules_oci", version = "2.2.7") bazel_dep(name = "rules_proto", version = "7.1.0") bazel_dep(name = "rules_python", version = "1.9.0") bazel_dep(name = "rules_shell", version = "0.6.1") + # repo_name = None prevents rules_swift from eagerly auto-registering toolchains. bazel_dep(name = "rules_swift", version = "3.6.1", repo_name = None) bazel_dep(name = "rules_zig", version = "0.15.1") @@ -61,6 +65,7 @@ single_version_override( ], version = "1.80.0", ) + single_version_override( module_name = "envoy_api", version = "0.0.0-20250128-4de3c74", @@ -181,6 +186,13 @@ use_repo(pip, "huggingface_hub") pip.parse( download_only = True, + # fsspec[gcs] -> gcsfs -> fsspec is a circular dependency, install them together. + experimental_requirement_cycles = { + "fsspec": [ + "fsspec", + "gcsfs", + ], + }, hub_name = "xprof_nightly", python_version = "3.12", requirements_lock = "//tools/xprof:requirements.lock.txt", @@ -221,9 +233,10 @@ tpu = use_extension("//platforms/tpu:tpu.bzl", "tpu_packages") use_repo(tpu, "libzml_tpu") neuron = use_extension("//platforms/neuron:neuron.bzl", "neuron_packages") -use_repo(neuron, "aws-neuronx-collectives", "aws-neuronx-runtime-lib", "aws-neuronx-tools", "libgcc_s1", "libgomp1", "libzml_neuron", "libstdcpp6") +use_repo(neuron, "aws-neuronx-collectives", "aws-neuronx-runtime-lib", "aws-neuronx-tools", "libgcc_s1", "libgomp1", "libstdcpp6", "libzml_neuron") http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + http_archive( name = "sevenzip_macos", build_file_content = """exports_files(["7zz"])""", @@ -232,16 +245,17 @@ http_archive( ) http_aea_archive = use_repo_rule("//bazel:http_aea_archive.bzl", "http_aea_archive") + http_aea_archive( name = "metal_toolchain", archive_decryption_key = "v7VbaT39JmQPp5p4N3HoVPAKcZ5vyhY+nnjMRD79z5E=", + build_file = "//platforms/metal:metal_toolchain.BUILD.bazel", canonical_id = "com.apple.MobileAsset.MetalToolchain-17B54", + includes = ["Metal.xctoolchain/*"], sha256 = "84f1cec3984878294777349e794f7bff0407e501a728bd600dc8605fa36abf6f", urls = [ "https://updates.cdn-apple.com/2025/mobileassets/022-20354/192ADD28-31EE-43C8-972D-E5BED5FB1845/com_apple_MobileAsset_MetalToolchain/637B0E26-E00A-488B-BA3B-CB50A8ECFD74.aar", ], - build_file = "//platforms/metal:metal_toolchain.BUILD.bazel", - includes = ["Metal.xctoolchain/*"], ) metal = use_extension("//platforms/metal:metal.bzl", "metal_packages") @@ -254,6 +268,7 @@ use_repo( "cloud_accelerator_diagnostics", "com_google_sentencepiece", "cuda_tile", + "cute_ir", "flashattn_linux_amd64", "flashattn_linux_arm64", "flashinfer_cutlass_moe_linux_amd64", @@ -300,7 +315,7 @@ use_repo( ) llvm = use_extension("//third_party/xla:llvm.bzl", "llvm") -llvm.configure() +llvm.configure(targets = ["X86", "NVPTX", "AMDGPU", "AArch64"]) use_repo(llvm, "llvm-project") apt = use_extension("@rules_distroless//apt:extensions.bzl", "apt") diff --git a/kernels/cute/BUILD.bazel b/kernels/cute/BUILD.bazel new file mode 100644 index 00000000..97b6ddd9 --- /dev/null +++ b/kernels/cute/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "cute_builder", + tags = ["manual"], + srcs = [ + "dtype.zig", + ], + import_name = "kernels/cute/builder", + main = "builder.zig", + visibility = ["//visibility:public"], + deps = [ + "//kernels/common", + "//mlir", + "//mlir/dialects", + "//mlir/dialects/cute_ir", + "//stdx", + ], +) + +zig_test( + name = "test", + deps = [":cute_builder"], +) diff --git a/kernels/cute/README.md b/kernels/cute/README.md new file mode 100644 index 00000000..367eaaf2 --- /dev/null +++ b/kernels/cute/README.md @@ -0,0 +1,127 @@ +# CuTe DSL kernels in Zig + +`kernels/cute` is a Zig front end for NVIDIA's CuTe DSL, shaped after the +Python `cutlass.cute` API: a kernel takes tensors, indexes them with +coordinates, and reads `thread_idx()` and friends from `cute.arch`. It is the +fourth instance of the `kernels/triton` / `kernels/mosaic_tpu` / +`kernels/cuda_tile` pattern: a `Builder` that owns an MLIR module and appends +ops to a block stack, a `Value` with fluent scalar arithmetic, and a +`zml.kernel.cute.Kernel` wrapper that turns a `run` function into a +`stablehlo.custom_call` XLA compiles in process with `libcute_compiler.so`. + +The dialect binding lives in `mlir/dialects/cute_ir` (`cute` and +`cute_nvgpu`, types through the dialect's C API). + +## A kernel + +The Python notebook's `naive_elementwise_add_kernel`, in Zig: + +```zig +const zml = @import("zml"); +const cute = zml.kernel.cute; + +const Cfg = struct { m: i64, n: i64 }; + +pub const NaiveElementwiseAdd = cute.Kernel(Cfg, .{ + .name = "naive_elementwise_add_kernel", + .inputs = &.{ "gA", "gB" }, + .outputs = &.{"gC"}, + .run = run, +}); + +fn run(b: *cute.Builder, cfg: Cfg) cute.FinishError!void { + // A `tensor` argument is XLA's device pointer composed with a static + // layout; the Python kernel takes `cute.Tensor` the same way. + const t = try b.declareArgs(.{ + .gA = .{ .tensor = .{ .dtype = .f32, .shape = &.{ cfg.m, cfg.n } } }, + .gB = .{ .tensor = .{ .dtype = .f32, .shape = &.{ cfg.m, cfg.n } } }, + .gC = .{ .tensor = .{ .dtype = .f32, .shape = &.{ cfg.m, cfg.n } } }, + }); + + const tidx = b.threadIdx().x; + const bidx = b.blockIdx().x; + const bdim = b.blockDim().x; + const thread_idx = bidx.mul(bdim).add(tidx); + + const n = t.gA.dim(1); + const ni = thread_idx.rem(n); + const mi = thread_idx.div(n); + + // gC[mi, ni] = gA[mi, ni] + gB[mi, ni] + t.gC.set(.{ mi, ni }, t.gA.get(.{ mi, ni }).add(t.gB.get(.{ mi, ni }))); +} + +// In a model (`kernel.launch(grid=..., block=...)`): +const c = NaiveElementwiseAdd.call(.{ .gA = a, .gB = b }, .{ .gC = a.shape() }, .{ + .cfg = .{ .m = 64, .n = 128 }, + .grid = .{ 32, 1, 1 }, + .block = .{ 256, 1, 1 }, +}).gC; +``` + +## Python to Zig + +| `cutlass.cute` | `kernels/cute` | +|--------------------------------------------|-------------------------------------------------| +| `gA: cute.Tensor` kernel argument | `.gA = .{ .tensor = .{ .dtype, .shape } }` | +| `gA[mi, ni]` / `gC[mi, ni] = v` | `gA.get(.{ mi, ni })` / `gC.set(.{ mi, ni }, v)`| +| `cute.make_layout(shape, stride=...)` | `b.makeLayout(shape, .{ .stride = ... })` | +| `cute.make_tensor(ptr, layout)` | `b.makeTensor(ptr, layout)` | +| `cute.arch.thread_idx()` → `(x, y, z)` | `b.threadIdx().x` | +| `block_idx()`, `block_dim()`, `grid_dim()` | `blockIdx()`, `blockDim()`, `gridDim()` | +| `cute.arch.alloc_smem(T, n, align)` | `b.allocSmem(.f32, n, align)` | +| `cute.arch.sync_threads()` | `b.syncThreads()` | +| `x.to(Float32)` | `x.to(.f32)` | +| `if cond:` / `for i in range(...)` | `b.openIf(cond)` / `b.openFor(lb, ub, step, .{})` | + +* **Layouts are static.** `makeLayout` takes Zig slices and emits one + `cute.static`; the tensor keeps its shape and stride in Zig, so `dim`, + `size` and coordinates need no MLIR queries. Coordinates are tuples of + `Value`s and ints; ints stay static in the `!cute.coord` type. +* **Stride defaults differ by origin.** `makeLayout` defaults to CuTe's + compact left-most stride like `cute.make_layout`; a `tensor` argument + defaults to XLA's row-major layout, which is what the buffer holds. +* **Scalars are `i32` and `f32` by default**, like `Int32`/`Float32`; + `cst(dtype, v)` picks another type and `coerce` lifts literals to the other + operand's type. +* **`nvvm` is not linked into ZML.** Thread indices and barriers are emitted + as unregistered ops in generic form; the compiler has the dialect. +* **The module is just the kernel.** `finish` prints `module { func.func + @name(...) {...} }`; the CuTe compiler makes every public function a kernel + entry, so there is no `gpu.module` or host launch function to write. +* **Not covered yet:** dynamic shapes, `local_tile`/`local_partition`, tiled + copies and fragments (`TensorSSA`), TMA and MMA atoms. + +## The custom call + +`ops.cute` emits `__gpu$xla.gpu.cute` with a printed dictionary: `name`, +`kernel_type = "cute"`, `ir` (the textual module), `grid` and `block` +(3-element arrays), optional `zeroed_outputs`. XLA compiles it in process +(`lib/libcute_compiler.so`, which pulls in `lib/cutlass_ir.so`, both from the +sandbox pointed at by `--xla_gpu_cuda_data_dir`), takes the kernel's +dynamic shared memory requirement from the compiler's metadata, and launches +the cubin over `grid` × `block`. Every operand and result is one raw device +pointer in the default layout, operands first; the count is not checked +against the kernel signature, so `inputs`/`outputs` must match `declareArgs`. +A module that carries its own `gpu.module` and host launch still works; +it then owns the launch configuration and `grid`/`block` must be left out. + +To see what XLA received, run with `--xla_dump_to=`: it writes +`.cute.mlir`, `.cute.json` and `.cubin`. + +## Finding what exists + +```sh +# the DSL: Value's fluent methods (add, mul, lt, to, exp, ...) +awk '/^pub const Value = struct/,/^};/' kernels/cute/builder.zig | grep 'pub fn ' + +# the DSL: everything on Tensor and Builder +awk '/^pub const Tensor = struct/,/^};/' kernels/cute/builder.zig | grep 'pub fn ' +awk '/^pub const Builder = struct/,0' kernels/cute/builder.zig | grep ' pub fn ' + +# every bound op of the dialect +grep -n '^pub fn ' mlir/dialects/cute_ir/cute_ir.zig mlir/dialects/cute_ir/cute_nvgpu.zig +``` + +`grep -n '^test "' kernels/cute/builder.zig` lists the tests; each prints a +kernel that round-trips through the verifier. diff --git a/kernels/cute/builder.zig b/kernels/cute/builder.zig new file mode 100644 index 00000000..69b3897d --- /dev/null +++ b/kernels/cute/builder.zig @@ -0,0 +1,950 @@ +const std = @import("std"); + +const cute = @import("mlir/dialects/cute_ir"); +const dialects = @import("mlir/dialects"); +const dsl = @import("kernels/common"); +const tupleArity = dsl.tupleArity; +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +const arith = dialects.arith; +const func = dialects.func; +const math = dialects.math; + +const dtypes = @import("dtype.zig"); +pub const DType = dtypes.DType; +const isFloatDtype = dtypes.isFloatDtype; +const dtypeBitwidth = dtypes.dtypeBitwidth; + +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(Builder); + std.testing.refAllDecls(Value); + std.testing.refAllDecls(Tensor); +} + +/// `nvvm` is not linked into ZML: its ops are emitted unregistered, in +/// generic form; the CuTe compiler has the dialect. +pub const dialects_needed = [_][]const u8{ "func", "cute", "cute_nvgpu", "arith", "scf", "math", "cf" }; + +pub const FinishError = error{InvalidMlir} || std.mem.Allocator.Error || std.Io.Writer.Error; + +pub const MAX_RANK = 8; +pub const Dims = stdx.BoundedArray(i64, MAX_RANK); + +pub const MemorySpace = cute.MemorySpace; + +/// A scalar SSA value (`Int32`, `Float32`, ... in the Python DSL). +pub const Value = struct { + inner: *const mlir.Value, + kernel: ?*Builder = null, + + pub fn type_(self: Value) *const mlir.Type { + return self.inner.type_(); + } + + fn kern(self: Value) *Builder { + return self.kernel orelse @panic("Value has no owning kernel; use Builder.* helpers instead"); + } + + pub fn dtype(self: Value) DType { + return dtypes.mlirToDType(self.kern().ctx, self.type_()); + } + + pub fn isFloat(self: Value) bool { + const t = self.type_(); + inline for (std.meta.fields(mlir.FloatTypes)) |f| { + if (t.isA(mlir.FloatType(@field(mlir.FloatTypes, f.name))) != null) return true; + } + return false; + } + + pub fn isInt(self: Value) bool { + return self.type_().isA(mlir.IntegerType) != null; + } + + pub fn isPtr(self: Value) bool { + return self.type_().isA(cute.PtrType) != null; + } + + pub fn add(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(if (l.isFloat()) arith.addf(k.ctx, l.inner, r.inner, k.loc()) else arith.addi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn sub(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(if (l.isFloat()) arith.subf(k.ctx, l.inner, r.inner, k.loc()) else arith.subi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn mul(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(if (l.isFloat()) arith.mulf(k.ctx, l.inner, r.inner, k.loc()) else arith.muli(k.ctx, l.inner, r.inner, k.loc())); + } + + /// Signed division on integers (Python `//` on Int32). + pub fn div(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(if (l.isFloat()) arith.divf(k.ctx, l.inner, r.inner, k.loc()) else arith.divsi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn rem(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(if (l.isFloat()) arith.remf(k.ctx, l.inner, r.inner, k.loc()) else arith.remsi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn cdiv(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + if (l.isFloat()) @panic("Value.cdiv is integer-only"); + return k.emit(arith.ceildivsi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn neg(self: Value) Value { + const k = self.kern(); + if (self.isFloat()) return k.emit(arith.negf(k.ctx, self.inner, k.loc())); + return k.cst(self.dtype(), 0).sub(self); + } + + pub fn abs(self: Value) Value { + const k = self.kern(); + return k.emit(if (self.isFloat()) math.absf(k.ctx, self.inner, k.loc()) else math.absi(k.ctx, self.inner, k.loc())); + } + + pub fn minimum(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(if (l.isFloat()) arith.minimumf(k.ctx, l.inner, r.inner, k.loc()) else arith.minsi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn maximum(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(if (l.isFloat()) arith.maximumf(k.ctx, l.inner, r.inner, k.loc()) else arith.maxsi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn bitAnd(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(arith.andi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn bitOr(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(arith.ori(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn bitXor(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(arith.xori(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn shl(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(arith.shli(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn shr(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(arith.shrsi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn lt(self: Value, rhs: anytype) Value { + return self.kern().cmp(self, rhs, .slt, .olt); + } + + pub fn le(self: Value, rhs: anytype) Value { + return self.kern().cmp(self, rhs, .sle, .ole); + } + + pub fn gt(self: Value, rhs: anytype) Value { + return self.kern().cmp(self, rhs, .sgt, .ogt); + } + + pub fn ge(self: Value, rhs: anytype) Value { + return self.kern().cmp(self, rhs, .sge, .oge); + } + + pub fn eq(self: Value, rhs: anytype) Value { + return self.kern().cmp(self, rhs, .eq, .oeq); + } + + pub fn ne(self: Value, rhs: anytype) Value { + return self.kern().cmp(self, rhs, .ne, .une); + } + + pub fn exp(self: Value) Value { + const k = self.kern(); + return k.emit(math.exp(k.ctx, self.inner, k.loc())); + } + + pub fn log(self: Value) Value { + const k = self.kern(); + return k.emit(math.log(k.ctx, self.inner, k.loc())); + } + + pub fn sqrt(self: Value) Value { + const k = self.kern(); + return k.emit(math.sqrt(k.ctx, self.inner, k.loc())); + } + + pub fn rsqrt(self: Value) Value { + const k = self.kern(); + return k.emit(math.rsqrt(k.ctx, self.inner, k.loc())); + } + + pub fn tanh(self: Value) Value { + const k = self.kern(); + return k.emit(math.tanh(k.ctx, self.inner, k.loc())); + } + + /// Python `x.to(T)`: int <-> int is signed, int <-> float goes through + /// `sitofp`/`fptosi`, same-width floats through `convertf`. + pub fn to(self: Value, dt: DType) Value { + const k = self.kern(); + const src = self.dtype(); + if (src == dt) return self; + const ty = dt.toMlir(k.ctx); + const src_f = isFloatDtype(src); + const dst_f = isFloatDtype(dt); + const sw = dtypeBitwidth(src); + const dw = dtypeBitwidth(dt); + if (src_f and dst_f) { + if (sw == dw) return k.emit(arith.convertf(k.ctx, self.inner, ty, .{}, k.loc())); + return k.emit(if (dw > sw) arith.extf(k.ctx, self.inner, ty, k.loc()) else arith.truncf(k.ctx, self.inner, ty, k.loc())); + } + if (src_f) return k.emit(arith.fptosi(k.ctx, self.inner, ty, k.loc())); + if (dst_f) return k.emit(arith.sitofp(k.ctx, self.inner, ty, k.loc())); + if (sw == dw) return self; + if (src == .i1) return k.emit(arith.extui(k.ctx, self.inner, ty, k.loc())); + return k.emit(if (dw > sw) arith.extsi(k.ctx, self.inner, ty, k.loc()) else arith.trunci(k.ctx, self.inner, ty, k.loc())); + } +}; + +/// `cute.Layout`: a static shape and stride, carried in Zig so coordinates +/// and element counts need no MLIR queries. +pub const Layout = struct { + inner: *const mlir.Value, + shape: Dims, + stride: Dims, + + pub fn rank(self: Layout) usize { + return self.shape.len; + } + + pub fn size(self: Layout) i64 { + var n: i64 = 1; + for (self.shape.constSlice()) |d| n *= d; + return n; + } + + pub fn dim(self: Layout, i: usize) i64 { + return self.shape.get(i); + } +}; + +/// `cute.Tensor`: an engine (pointer) composed with a layout. `get`/`set` +/// are the Python `t[coord]` / `t[coord] = v`. +pub const Tensor = struct { + inner: *const mlir.Value, + kernel: *Builder, + layout: Layout, + dtype: DType, + space: MemorySpace, + + pub fn shape(self: Tensor) []const i64 { + return self.layout.shape.constSlice(); + } + + pub fn rank(self: Tensor) usize { + return self.layout.rank(); + } + + pub fn size(self: Tensor) i64 { + return self.layout.size(); + } + + pub fn dim(self: Tensor, i: usize) i64 { + return self.layout.dim(i); + } + + /// `t[coord]`: `coord` is a tuple of `Value`s and ints, one per mode. + pub fn get(self: Tensor, coord: anytype) Value { + const k = self.kernel; + const c = k.makeCoord(coord); + return k.emit(cute.memref_load(k.ctx, self.inner, c, self.dtype.toMlir(k.ctx), k.loc())); + } + + /// `t[coord] = value`; the value is converted to the element type. + pub fn set(self: Tensor, coord: anytype, value: anytype) void { + const k = self.kernel; + const c = k.makeCoord(coord); + const v = k.liftAs(value, self.dtype).to(self.dtype); + _ = cute.memref_store(k.ctx, self.inner, c, v.inner, k.loc()).appendTo(k.currentBlock()); + } +}; + +pub const Index3 = struct { x: Value, y: Value, z: Value }; + +pub const ArgSpec = struct { + name: []const u8, + kind: Kind, + + /// XLA passes one raw device pointer per custom-call argument. A `tensor` + /// argument is that pointer composed with a static layout at the top of + /// the kernel, the way the Python DSL passes `cute.Tensor` arguments. + pub const Kind = union(enum) { + ptr: DType, + tensor: TensorSpec, + }; + + pub const TensorSpec = struct { + dtype: DType, + shape: []const i64, + /// Defaults to XLA's row-major layout. + stride: ?[]const i64 = null, + alignment: u64 = 16, + }; +}; + +pub const LayoutOpts = struct { + /// Defaults to CuTe's compact left-most stride, like `cute.make_layout`. + stride: ?[]const i64 = null, +}; + +pub const IfOnlyScope = dsl.IfOnlyScope(Builder, Value); + +pub fn IfScope(comptime N: usize) type { + return dsl.IfScope(Builder, Value, N); +} + +pub fn ForScope(comptime N: usize) type { + return dsl.ForScope(Builder, Value, N); +} + +pub const Builder = struct { + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + ctx: *mlir.Context, + module: *mlir.Module, + name: []const u8, + func_op: ?*mlir.Operation = null, + entry_block: ?*mlir.Block = null, + block_stack: std.ArrayList(*mlir.Block) = .empty, + args: []const ArgSpec = &.{}, + tensors: []?Tensor = &.{}, + + pub fn open(allocator: std.mem.Allocator, ctx: *mlir.Context, name: []const u8) !Builder { + // `nvvm.*` is built unregistered. + ctx.setAllowUnregisteredDialects(true); + const module: *mlir.Module = .init(.unknown(ctx)); + errdefer module.deinit(); + return .{ + .allocator = allocator, + .arena = .init(allocator), + .ctx = ctx, + .module = module, + .name = name, + }; + } + + pub fn deinit(self: *Builder) void { + self.module.deinit(); + self.arena.deinit(); + } + + /// One field per kernel argument: `.{ .a = .{ .tensor = .{ .dtype = .f32, .shape = &.{n} } } }` + /// yields a `Tensor`, `.{ .p = .{ .ptr = .f32 } }` a pointer `Value`. + pub fn declareArgs(self: *Builder, spec: anytype) FinishError!ArgsOf(@TypeOf(spec)) { + const Spec = @TypeOf(spec); + const fields = @typeInfo(Spec).@"struct".fields; + + const arg_specs = try self.arena.allocator().alloc(ArgSpec, fields.len); + inline for (fields, 0..) |f, i| { + const raw = @field(spec, f.name); + const variant = @typeInfo(@TypeOf(raw)).@"struct".fields[0].name; + const inner = @field(raw, variant); + arg_specs[i] = .{ .name = f.name, .kind = switch (@field(std.meta.Tag(ArgSpec.Kind), variant)) { + .ptr => .{ .ptr = inner }, + .tensor => .{ .tensor = .{ + .dtype = inner.dtype, + .shape = inner.shape, + .stride = if (@hasField(@TypeOf(inner), "stride")) inner.stride else null, + .alignment = if (@hasField(@TypeOf(inner), "alignment")) inner.alignment else 16, + } }, + } }; + } + try self.declareArgsLow(arg_specs); + + var named: ArgsOf(Spec) = undefined; + inline for (fields, 0..) |f, i| { + @field(named, f.name) = switch (@FieldType(ArgsOf(Spec), f.name)) { + Tensor => self.tensors[i].?, + else => self.arg(i), + }; + } + return named; + } + + fn ArgsOf(comptime Spec: type) type { + const in = @typeInfo(Spec).@"struct".fields; + comptime var names: [in.len][]const u8 = undefined; + comptime var types: [in.len]type = undefined; + inline for (in, 0..) |f, i| { + names[i] = f.name; + types[i] = if (@hasField(f.type, "tensor")) Tensor else Value; + } + return @Struct(.auto, null, &names, &types, &@splat(.{})); + } + + fn declareArgsLow(self: *Builder, args: []const ArgSpec) FinishError!void { + std.debug.assert(self.entry_block == null); + const ctx = self.ctx; + const scratch = self.arena.allocator(); + + const arg_types = try scratch.alloc(*const mlir.Type, args.len); + const arg_locs = try scratch.alloc(*const mlir.Location, args.len); + for (arg_types, arg_locs, args) |*ty, *l, a| { + ty.* = switch (a.kind) { + .ptr => |dt| self.ptrTy(dt, .gmem, 16), + .tensor => |t| self.ptrTy(t.dtype, .gmem, t.alignment), + } catch return error.InvalidMlir; + l.* = self.loc(); + } + + const entry = mlir.Block.init(arg_types, arg_locs); + const func_op = func.func(ctx, .{ + .name = self.name, + .block = entry, + .results = &.{}, + .location = self.loc(), + .visibility = null, + .verify = false, + .extra_attributes = &.{ + .named(ctx, "cute.kernel", .unit(ctx)), + .named(ctx, "gpu.kernel", .unit(ctx)), + }, + }); + _ = func_op.appendTo(self.module.body()); + + self.func_op = func_op; + self.entry_block = entry; + self.args = args; + + self.tensors = try scratch.alloc(?Tensor, args.len); + for (self.tensors, args, 0..) |*t, a, i| { + t.* = switch (a.kind) { + .ptr => null, + .tensor => |spec| self.makeTensor(self.arg(i), self.makeLayout(spec.shape, .{ + .stride = spec.stride orelse self.rowMajor(spec.shape), + })), + }; + } + } + + pub fn arg(self: *Builder, i: usize) Value { + const eb = self.entry_block orelse @panic("Builder.arg called before declareArgs"); + return .{ .inner = eb.argument(i), .kernel = self }; + } + + pub fn pushBlock(self: *Builder, b: *mlir.Block) void { + self.block_stack.append(self.arena.allocator(), b) catch @panic("Builder.pushBlock OOM"); + } + + pub fn popBlock(self: *Builder) void { + _ = self.block_stack.pop(); + } + + pub fn currentBlock(self: *Builder) *mlir.Block { + if (self.block_stack.items.len > 0) { + return self.block_stack.items[self.block_stack.items.len - 1]; + } + return self.entry_block orelse @panic("Builder has no current block; call declareArgs first"); + } + + pub fn emit(self: *Builder, op: *mlir.Operation) Value { + _ = op.appendTo(self.currentBlock()); + return .{ .inner = op.result(0), .kernel = self }; + } + + pub fn loc(self: *const Builder) *const mlir.Location { + return .unknown(self.ctx); + } + + // ==================== types ==================== + + pub fn ptrTy(self: *Builder, dt: DType, space: MemorySpace, alignment: u64) !*const mlir.Type { + return cute.pointerType(self.ctx, dt.toMlir(self.ctx), space, alignment); + } + + fn algebra(self: *Builder, dims: []const i64) []const u8 { + var out: std.Io.Writer.Allocating = .init(self.arena.allocator()); + if (dims.len == 1) { + out.writer.print("{d}", .{dims[0]}) catch @panic("OOM"); + } else { + out.writer.writeByte('(') catch @panic("OOM"); + for (dims, 0..) |d, i| { + if (i > 0) out.writer.writeByte(',') catch @panic("OOM"); + out.writer.print("{d}", .{d}) catch @panic("OOM"); + } + out.writer.writeByte(')') catch @panic("OOM"); + } + return out.written(); + } + + fn rowMajor(self: *Builder, shape: []const i64) []const i64 { + const s = self.arena.allocator().alloc(i64, shape.len) catch @panic("OOM"); + var acc: i64 = 1; + var i = shape.len; + while (i > 0) : (i -= 1) { + s[i - 1] = acc; + acc *= shape[i - 1]; + } + return s; + } + + fn leftMost(self: *Builder, shape: []const i64) []const i64 { + const s = self.arena.allocator().alloc(i64, shape.len) catch @panic("OOM"); + var acc: i64 = 1; + for (shape, 0..) |d, i| { + s[i] = acc; + acc *= d; + } + return s; + } + + // ==================== cute.core ==================== + + /// `cute.make_layout(shape, stride=...)`, static only. + pub fn makeLayout(self: *Builder, shape: []const i64, opts: LayoutOpts) Layout { + if (shape.len == 0 or shape.len > MAX_RANK) std.debug.panic("makeLayout: rank {d} not in 1..{d}", .{ shape.len, MAX_RANK }); + const stride = opts.stride orelse self.leftMost(shape); + if (stride.len != shape.len) std.debug.panic("makeLayout: stride rank {d} != shape rank {d}", .{ stride.len, shape.len }); + const ctx = self.ctx; + const text = std.fmt.allocPrint(self.arena.allocator(), "{s}:{s}", .{ self.algebra(shape), self.algebra(stride) }) catch @panic("OOM"); + const layout_ty = cute.algebraType(ctx, .layout, text) catch @panic("bad layout"); + const l = self.emit(cute.static(ctx, layout_ty, self.loc())); + return .{ .inner = l.inner, .shape = Dims.fromSlice(shape) catch unreachable, .stride = Dims.fromSlice(stride) catch unreachable }; + } + + /// `cute.make_tensor(ptr, layout)`. + pub fn makeTensor(self: *Builder, ptr: Value, layout: Layout) Tensor { + const pt = ptr.type_().isA(cute.PtrType) orelse std.debug.panic("makeTensor: {f} is not a !cute.ptr", .{ptr.type_()}); + const elem = pt.getValueType() orelse @panic("makeTensor: untyped pointer"); + const memref_ty = (cute.MemRefType.get(self.ctx, .{ .ptr = ptr.type_(), .layout = layout.inner.type_() }) catch @panic("bad memref")).type_(); + const v = self.emit(cute.make_view(self.ctx, ptr.inner, layout.inner, memref_ty, self.loc())); + const space = std.meta.stringToEnum(MemorySpace, pt.getMemorySpace().isA(mlir.StringAttribute).?.value()) orelse .generic; + return .{ .inner = v.inner, .kernel = self, .layout = layout, .dtype = dtypes.mlirToDType(self.ctx, elem), .space = space }; + } + + /// A `!cute.coord` from a tuple of `Value`s and ints; ints stay static in the type. + pub fn makeCoord(self: *Builder, coord: anytype) *const mlir.Value { + const T = @TypeOf(coord); + if (T == Value) return self.makeCoord(.{coord}); + const fields = @typeInfo(T).@"struct".fields; + if (fields.len == 0 or fields.len > MAX_RANK) @compileError("makeCoord: rank must be 1..MAX_RANK"); + var text: std.Io.Writer.Allocating = .init(self.arena.allocator()); + var dyn: stdx.BoundedArray(*const mlir.Value, MAX_RANK) = .empty; + if (fields.len > 1) text.writer.writeByte('(') catch @panic("OOM"); + inline for (fields, 0..) |f, i| { + if (i > 0) text.writer.writeByte(',') catch @panic("OOM"); + const leaf = @field(coord, f.name); + switch (@typeInfo(f.type)) { + .comptime_int, .int => text.writer.print("{d}", .{leaf}) catch @panic("OOM"), + else => { + text.writer.writeByte('?') catch @panic("OOM"); + dyn.appendAssumeCapacity(self.lift(leaf).to(.i32).inner); + }, + } + } + if (fields.len > 1) text.writer.writeByte(')') catch @panic("OOM"); + const ty = cute.algebraType(self.ctx, .coord, text.written()) catch @panic("bad coord"); + return self.emit(cute.make_coord(self.ctx, dyn.constSlice(), ty, self.loc())).inner; + } + + // ==================== cute.arch ==================== + + fn sreg(self: *Builder, comptime reg: []const u8) Index3 { + var out: [3]Value = undefined; + inline for (.{ "x", "y", "z" }, 0..) |axis, i| { + out[i] = self.emit(mlir.Operation.make(self.ctx, "nvvm.read.ptx.sreg." ++ reg ++ "." ++ axis, .{ + .results = .{ .flat = &.{.int(self.ctx, .i32)} }, + .location = self.loc(), + })); + } + return .{ .x = out[0], .y = out[1], .z = out[2] }; + } + + pub fn threadIdx(self: *Builder) Index3 { + return self.sreg("tid"); + } + + pub fn blockIdx(self: *Builder) Index3 { + return self.sreg("ctaid"); + } + + pub fn blockDim(self: *Builder) Index3 { + return self.sreg("ntid"); + } + + pub fn gridDim(self: *Builder) Index3 { + return self.sreg("nctaid"); + } + + /// `cute.arch.sync_threads()`. + pub fn syncThreads(self: *Builder) void { + _ = mlir.Operation.make(self.ctx, "nvvm.barrier", .{ + .attributes = &.{.named(self.ctx, "operandSegmentSizes", .denseArray(self.ctx, .i32, &.{ 0, 0 }))}, + .location = self.loc(), + }).appendTo(self.currentBlock()); + } + + /// `cute.arch.alloc_smem(dtype, size_in_elems, alignment)`: a static + /// shared-memory pointer, sized at compile time. + pub fn allocSmem(self: *Builder, dt: DType, size_in_elems: i64, alignment: ?u64) Value { + const al = alignment orelse @max(1, dtypeBitwidth(dt) / 8); + const ty = self.ptrTy(dt, .smem, al) catch @panic("bad smem pointer"); + return self.emit(cute.nvgpu.arch_alloc_smem(self.ctx, ty, .int(self.ctx, .i32, size_in_elems), self.loc())); + } + + // ==================== scalars ==================== + + pub fn cst(self: *Builder, dt: DType, value: anytype) Value { + const ctx = self.ctx; + const T = @TypeOf(value); + if (isFloatDtype(dt)) { + const f: f64 = switch (@typeInfo(T)) { + .comptime_int, .comptime_float => value, + .int => @floatFromInt(value), + .float => @floatCast(value), + else => @compileError("Builder.cst: unsupported value type " ++ @typeName(T)), + }; + return self.emit(switch (dt) { + inline .f16, .bf16, .f32, .f64, .f8e4m3fn, .f8e5m2 => |ft| arith.constant_float(ctx, f, @field(mlir.FloatTypes, @tagName(ft)), self.loc()), + else => unreachable, + }); + } + const i: i64 = switch (@typeInfo(T)) { + .comptime_int, .int => @intCast(value), + .bool => @intFromBool(value), + .comptime_float, .float => @intFromFloat(value), + else => @compileError("Builder.cst: unsupported value type " ++ @typeName(T)), + }; + if (!fitsInt(i, dt)) std.debug.panic("Builder.cst: {d} does not fit {s}", .{ i, @tagName(dt) }); + return self.emit(arith.constant_int(ctx, i, dt.toMlir(ctx), self.loc())); + } + + fn fitsInt(i: i64, dt: DType) bool { + const bits = dtypeBitwidth(dt); + if (bits >= 64) return true; + const shift: u6 = @intCast(bits - 1); + return i >= -(@as(i64, 1) << shift) and i < (@as(i64, 1) << shift); + } + + fn literalFits(value: anytype, dt: DType) bool { + return switch (@typeInfo(@TypeOf(value))) { + .comptime_int, .int => fitsInt(@intCast(value), dt), + .bool => !isFloatDtype(dt), + else => isFloatDtype(dt), + }; + } + + /// A Zig scalar as a constant: ints become `i32` when they fit, `i64` + /// otherwise; floats `f32`, like the Python DSL's `Int32`/`Float32`. + pub fn lift(self: *Builder, value: anytype) Value { + const T = @TypeOf(value); + if (T == Value) return value; + return switch (@typeInfo(T)) { + .comptime_int, .int => if (literalFits(value, .i32)) self.cst(.i32, value) else self.cst(.i64, value), + .bool => self.cst(.i1, value), + .comptime_float => self.cst(.f32, value), + .float => |info| switch (info.bits) { + 16 => self.cst(.f16, value), + 32 => self.cst(.f32, value), + 64 => self.cst(.f64, value), + else => @compileError("Builder.lift: unsupported float bitwidth"), + }, + else => @compileError("Builder.lift: unsupported type " ++ @typeName(T)), + }; + } + + pub fn liftAs(self: *Builder, value: anytype, dt: DType) Value { + if (@TypeOf(value) == Value) return value; + return self.cst(dt, value); + } + + fn liftLike(self: *Builder, value: anytype, ref: Value) Value { + const dt = ref.dtype(); + return if (literalFits(value, dt)) self.cst(dt, value) else self.lift(value); + } + + /// Lift a literal to the other operand's dtype when it fits, else to its + /// own width; then widen the narrower integer. + pub fn coerce(self: *Builder, a: anytype, b: anytype) struct { Value, Value } { + var av: Value = if (@TypeOf(a) == Value) a else if (@TypeOf(b) == Value) self.liftLike(a, b) else self.lift(a); + var bv: Value = if (@TypeOf(b) == Value) b else self.liftLike(b, av); + if (av.isInt() and bv.isInt()) { + const aw = dtypeBitwidth(av.dtype()); + const bw = dtypeBitwidth(bv.dtype()); + if (aw < bw) av = av.to(bv.dtype()); + if (bw < aw) bv = bv.to(av.dtype()); + } + return .{ av, bv }; + } + + fn cmp(self: *Builder, a: Value, b: anytype, ip: arith.CmpIPredicate, fp: arith.CmpFPredicate) Value { + const l, const r = self.coerce(a, b); + return self.emit(if (l.isFloat()) arith.cmpf(self.ctx, fp, l.inner, r.inner, self.loc()) else arith.cmpi(self.ctx, ip, l.inner, r.inner, self.loc())); + } + + pub fn select(self: *Builder, cond: Value, t: anytype, f: anytype) Value { + const tv, const fv = self.coerce(t, f); + return self.emit(arith.select(self.ctx, cond.inner, tv.inner, fv.inner, self.loc())); + } + + // ==================== scf ==================== + + pub fn openIf(self: *Builder, cond: Value) IfOnlyScope { + const then_block = mlir.Block.init(&.{}, &.{}); + self.pushBlock(then_block); + return .{ .kernel = self, .cond_inner = cond.inner, .then_block = then_block }; + } + + pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(tupleArity(@TypeOf(result_types), "openIfElse: result_types")) { + const N = comptime tupleArity(@TypeOf(result_types), "openIfElse: result_types"); + var types: [N]*const mlir.Type = undefined; + inline for (@typeInfo(@TypeOf(result_types)).@"struct".fields, 0..) |f, i| { + types[i] = @field(result_types, f.name); + } + const then_block = mlir.Block.init(&.{}, &.{}); + const else_block = mlir.Block.init(&.{}, &.{}); + self.pushBlock(then_block); + return .{ .kernel = self, .cond_inner = cond.inner, .then_block = then_block, .else_block = else_block, .result_types = types }; + } + + /// `for iv in range(lower, upper, step)` with loop-carried `inits`. + pub fn openFor(self: *Builder, lower: anytype, upper: anytype, step: anytype, inits: anytype) ForScope(tupleArity(@TypeOf(inits), "openFor: inits")) { + const N = comptime tupleArity(@TypeOf(inits), "openFor: inits"); + const lb = self.lift(lower); + const ub = self.liftAs(upper, lb.dtype()); + const st = self.liftAs(step, lb.dtype()); + + var block_types: [N + 1]*const mlir.Type = undefined; + var block_locs: [N + 1]*const mlir.Location = undefined; + block_types[0] = lb.type_(); + block_locs[0] = self.loc(); + var inits_inner: [N]*const mlir.Value = undefined; + inline for (@typeInfo(@TypeOf(inits)).@"struct".fields, 0..) |f, i| { + const v = self.lift(@field(inits, f.name)); + block_types[i + 1] = v.type_(); + block_locs[i + 1] = self.loc(); + inits_inner[i] = v.inner; + } + const body = mlir.Block.init(&block_types, &block_locs); + self.pushBlock(body); + var carried: [N]Value = undefined; + for (0..N) |i| carried[i] = .{ .inner = body.argument(i + 1), .kernel = self }; + return .{ + .kernel = self, + .body = body, + .lb_inner = lb.inner, + .ub_inner = ub.inner, + .step_inner = st.inner, + .inits_inner = inits_inner, + .iv = .{ .inner = body.argument(0), .kernel = self }, + .carried = carried, + }; + } + + // ==================== module ==================== + + /// The module the CuTe compiler takes: the verified kernel as a public + /// `func.func`, which the compiler turns into the kernel entry. Grid and + /// block travel in the custom call; `block` is also pinned as + /// `nvvm.reqntid`. + pub fn finish(self: *Builder, block: [3]i32) FinishError![:0]const u8 { + const current = self.currentBlock(); + if (current.terminator() == null) { + _ = func.returns(self.ctx, &.{}, self.loc()).appendTo(current); + } + const func_op = self.func_op orelse return error.InvalidMlir; + func_op.setAttributeByName("nvvm.reqntid", .denseArray(self.ctx, .i32, &block)); + if (!self.module.operation().verify()) return error.InvalidMlir; + + var al: std.Io.Writer.Allocating = .init(self.allocator); + defer al.deinit(); + try al.writer.print("{f}", .{self.module.operation()}); + return try self.allocator.dupeZ(u8, al.written()); + } +}; + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + inline for (dialects_needed) |d| mlir.DialectHandle.fromString(d).insertDialect(registry); + mlir.registerFuncExtensions(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectContains(haystack: []const u8, needle: []const u8) !void { + if (std.mem.indexOf(u8, haystack, needle) == null) { + std.debug.print("missing {s} in:\n{s}\n", .{ needle, haystack }); + return error.TestExpectedContains; + } +} + +test "naive elementwise add matches the Python notebook kernel" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "naive_elementwise_add_kernel"); + defer b.deinit(); + const a = try b.declareArgs(.{ + .gA = .{ .tensor = .{ .dtype = .f16, .shape = &.{ 16, 8 } } }, + .gB = .{ .tensor = .{ .dtype = .f16, .shape = &.{ 16, 8 } } }, + .gC = .{ .tensor = .{ .dtype = .f16, .shape = &.{ 16, 8 } } }, + }); + + const tidx = b.threadIdx().x; + const bidx = b.blockIdx().x; + const bdim = b.blockDim().x; + const thread_idx = bidx.mul(bdim).add(tidx); + const n = a.gA.dim(1); + const ni = thread_idx.rem(n); + const mi = thread_idx.div(n); + a.gC.set(.{ mi, ni }, a.gA.get(.{ mi, ni }).add(a.gB.get(.{ mi, ni }))); + + // The module round-trips through the local dialects. + const ir = try b.finish(.{ 128, 1, 1 }); + defer std.testing.allocator.free(ir); + try expectContains(ir, "module {\n func.func @naive_elementwise_add_kernel(%arg0: !cute.ptr>, %arg1: !cute.ptr>, %arg2: !cute.ptr>)"); + try expectContains(ir, "attributes {cute.kernel, gpu.kernel, nvvm.reqntid = array}"); + try expectContains(ir, "\"nvvm.read.ptx.sreg.tid.x\"() : () -> i32"); + try expectContains(ir, "!cute.layout<\"(16,8):(8,1)\">"); + try expectContains(ir, "!cute.coord<\"(?,?)\">"); + try expectContains(ir, "cute.memref.load"); + try expectContains(ir, "cute.memref.store"); + try expectContains(ir, "arith.addf"); + try expectContains(ir, " return\n }\n}\n"); + try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "guard, shared memory, sync, layouts, for loop" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "reverse_blocks"); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .ptr = .f32 }, + .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{4096} } }, + }); + const gx = b.makeTensor(a.x, b.makeLayout(&.{4096}, .{})); + const smem = b.makeTensor(b.allocSmem(.f32, 128, 16), b.makeLayout(&.{128}, .{})); + + const tid = b.threadIdx().x; + const i = b.blockIdx().x.mul(128).add(tid); + smem.set(.{tid}, gx.get(.{i})); + b.syncThreads(); + const j = b.cst(.i32, 127).sub(tid); + var guard = b.openIf(i.lt(4096)); + var acc = b.openFor(0, 4, 1, .{b.cst(.f32, 0.0)}); + acc.yield(.{acc.carried[0].add(smem.get(.{j}).mul(acc.iv.to(.f32)))}); + a.out.set(.{i}, acc.results[0]); + guard.yieldThen(.{}); + + const kernel = try b.finish(.{ 128, 1, 1 }); + defer std.testing.allocator.free(kernel); + try expectContains(kernel, "cute_nvgpu.arch.alloc_smem"); + try expectContains(kernel, "!cute.ptr>"); + try expectContains(kernel, "\"nvvm.barrier\"() {operandSegmentSizes = array} : () -> ()"); + try expectContains(kernel, "scf.if"); + try expectContains(kernel, "scf.for"); + try expectContains(kernel, "!cute.layout<\"4096:1\">"); + const parsed = try mlir.Module.parse(ctx, kernel); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "left-most default stride and static coordinate leaves" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "k"); + defer b.deinit(); + const a = try b.declareArgs(.{ .p = .{ .ptr = .f32 } }); + const t = b.makeTensor(a.p, b.makeLayout(&.{ 4, 4 }, .{})); + try std.testing.expectEqualSlices(i64, &.{ 1, 4 }, t.layout.stride.constSlice()); + try std.testing.expectEqual(@as(i64, 16), t.size()); + const v = t.get(.{ b.threadIdx().x, 3 }); + try std.testing.expect(v.isFloat()); + const kernel = try b.finish(.{ 1, 1, 1 }); + defer std.testing.allocator.free(kernel); + try expectContains(kernel, "!cute.layout<\"(4,4):(1,4)\">"); + try expectContains(kernel, "!cute.coord<\"(?,3)\">"); + const parsed = try mlir.Module.parse(ctx, kernel); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "casts, select, if-else, integer widening" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "k"); + defer b.deinit(); + const a = try b.declareArgs(.{ .p = .{ .tensor = .{ .dtype = .f16, .shape = &.{8} } } }); + const tid = b.threadIdx().x; + + // A literal that does not fit i32 widens the i32 Value instead of truncating. + const big: u32 = 3_000_000_000; + const wide = tid.add(big); + try std.testing.expectEqual(DType.i64, wide.dtype()); + const bound = tid.lt(@as(i64, 5_000_000_000)); + try std.testing.expectEqual(DType.i1, bound.dtype()); + + const f = tid.to(.f32); + try std.testing.expectEqual(DType.f16, f.to(.f16).dtype()); + try std.testing.expectEqual(DType.f64, f.to(.f64).dtype()); + try std.testing.expectEqual(DType.bf16, f.to(.f16).to(.bf16).dtype()); + try std.testing.expectEqual(DType.i32, f.to(.i32).dtype()); + try std.testing.expectEqual(DType.i8, tid.to(.i8).dtype()); + try std.testing.expectEqual(DType.i32, bound.to(.i32).dtype()); + + const picked = b.select(bound, f, 2.0); + var scope = b.openIfElse(tid.lt(4), .{DType.f16.toMlir(ctx)}); + scope.yieldThen(.{picked.to(.f16)}); + scope.yieldElse(.{b.cst(.f16, 1)}); + a.p.set(.{tid}, scope.results[0]); + + const kernel = try b.finish(.{ 1, 1, 1 }); + defer std.testing.allocator.free(kernel); + try expectContains(kernel, "arith.extsi"); + try expectContains(kernel, "arith.constant 3000000000 : i64"); + try expectContains(kernel, "arith.sitofp"); + try expectContains(kernel, "arith.truncf"); + try expectContains(kernel, "arith.extf"); + try expectContains(kernel, "arith.convertf"); + try expectContains(kernel, "arith.fptosi"); + try expectContains(kernel, "arith.trunci"); + try expectContains(kernel, "arith.extui"); + try expectContains(kernel, "arith.select"); + try expectContains(kernel, "} else {"); + const parsed = try mlir.Module.parse(ctx, kernel); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} diff --git a/kernels/cute/dtype.zig b/kernels/cute/dtype.zig new file mode 100644 index 00000000..89916fcf --- /dev/null +++ b/kernels/cute/dtype.zig @@ -0,0 +1,62 @@ +const std = @import("std"); +const mlir = @import("mlir"); + +/// Element types a `!cute.ptr` can point at. +pub const DType = enum { + i1, + i8, + i16, + i32, + i64, + f16, + bf16, + f32, + f64, + f8e4m3fn, + f8e5m2, + + pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { + return switch (self) { + .i1 => .int(ctx, .i1), + .i8 => .int(ctx, .i8), + .i16 => .int(ctx, .i16), + .i32 => .int(ctx, .i32), + .i64 => .int(ctx, .i64), + .f16 => .float(ctx, .f16), + .bf16 => .float(ctx, .bf16), + .f32 => .float(ctx, .f32), + .f64 => .float(ctx, .f64), + .f8e4m3fn => .float(ctx, .f8e4m3fn), + .f8e5m2 => .float(ctx, .f8e5m2), + }; + } +}; + +pub fn isFloatDtype(dt: DType) bool { + return switch (dt) { + .f16, .bf16, .f32, .f64, .f8e4m3fn, .f8e5m2 => true, + else => false, + }; +} + +pub fn dtypeBitwidth(dt: DType) usize { + return switch (dt) { + .i1 => 1, + .i8, .f8e4m3fn, .f8e5m2 => 8, + .i16, .f16, .bf16 => 16, + .i32, .f32 => 32, + .i64, .f64 => 64, + }; +} + +pub fn mlirToDType(ctx: *mlir.Context, t: *const mlir.Type) DType { + inline for (std.meta.fields(DType)) |f| { + const dt = @field(DType, f.name); + if (t.eql(dt.toMlir(ctx))) return dt; + } + std.debug.panic("type {f} is not a CuTe DType", .{t}); +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/BUILD.bazel index bcf4deb8..db4078ac 100644 --- a/mlir/dialects/BUILD.bazel +++ b/mlir/dialects/BUILD.bazel @@ -2,7 +2,6 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "dialects", - tags = ["manual"], srcs = [ "affine.zig", "arith.zig", @@ -11,17 +10,19 @@ zig_library( "gpu.zig", "math.zig", "memref.zig", - "shardy.zig", "scf.zig", + "shardy.zig", "vector.zig", ], import_name = "mlir/dialects", main = "dialects.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ "//mlir", "//mlir:c", "//mlir/dialects/cuda_tile", + "//mlir/dialects/cute_ir", "//mlir/dialects/fly", "//mlir/dialects/mosaic_tpu", "//mlir/dialects/stablehlo", diff --git a/mlir/dialects/cute_ir/BUILD.bazel b/mlir/dialects/cute_ir/BUILD.bazel new file mode 100644 index 00000000..37be8421 --- /dev/null +++ b/mlir/dialects/cute_ir/BUILD.bazel @@ -0,0 +1,32 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "cute_ir", + srcs = ["cute_nvgpu.zig"], + import_name = "mlir/dialects/cute_ir", + main = "cute_ir.zig", + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "//mlir:c", + "//mlir/dialects/cute_ir/include/cute_ir-c/Dialect:cute", + "//mlir/dialects/cute_ir/include/cute_ir-c/Dialect:cute_nvgpu", + "//mlir/dialects/cute_ir/lib/CAPI:cute", + "//mlir/dialects/cute_ir/lib/CAPI:cute_nvgpu", + "//stdx", + ], +) + +zig_test( + name = "zig_test", + deps = [":cute_ir"], +) + +test_suite( + name = "test", + tests = [ + ":zig_test", + "//mlir/dialects/cute_ir/test:capi_registration_test", + "//mlir/dialects/cute_ir/test:type_constructors_test", + ], +) diff --git a/mlir/dialects/cute_ir/README.md b/mlir/dialects/cute_ir/README.md new file mode 100644 index 00000000..9e5ed4f7 --- /dev/null +++ b/mlir/dialects/cute_ir/README.md @@ -0,0 +1,107 @@ +# CuTe Zig Bindings + +Depend on `//mlir/dialects/cute_ir` and import `mlir/dialects/cute_ir`, or use +`@import("mlir/dialects").cute` through the dialect aggregator. + +`cute.registerDialects(registry)` registers both `cute` and `cute_nvgpu`. +Cute builders, types, and attributes live directly in `cute`; CuteNVGPU builders +and types live in `cute.nvgpu`. Cute bindings are in `cute_ir.zig`, and CuteNVGPU +bindings are in `cute_nvgpu.zig`. Public and recovered operations belong to the same `cute` +dialect, without a separate private Zig namespace. + +```zig +const cute = @import("mlir/dialects/cute_ir"); + +const pointer = try cute.PtrType.get(ctx, .{ + .valueType = .float(ctx, .f32), + .memorySpace = .string(ctx, "smem"), + .alignment = 16, +}); +const layout_attr = try cute.LayoutAttr.get(ctx, "4:1"); +const layout = try cute.LayoutType.get(ctx, .{ .attr = layout_attr.attribute() }); +const memref = try cute.MemRefType.get(ctx, .{ + .ptr = pointer.type_(), + .layout = layout.type_(), +}); +const alloc = cute.memref_alloc_smem(ctx, memref.type_(), loc).appendTo(block); +``` + +The checked-in operation builders follow the pinned OSS `CuteOps.td` and the +local `CuteOpsPrivate.td` and `CuteNVGPUOps.td` schemas. Update the bindings when +those schemas change. There is no build-time Zig generator. Attribute and type +wrappers call `CuteAttributes.h/.cpp`, `CuteTypes.h/.cpp`, and +`CuteNVGPUTypes.h/.cpp`. C++ dialect definitions still use TableGen. + +Type constructors call the actual C++ `get`/`getChecked` functions, not a type +parser. Parameters follow ODS names; getters follow the C++ accessor names. +`get` returns `error.InvalidMlir` for invalid parameters or mismatched contexts. +`type_()` converts a typed wrapper to `*const mlir.Type` for operation builders. + +The C API is also available independently of Zig from `lib/CAPI:cute` and +`lib/CAPI:cute_nvgpu`, via `cute_ir-c/Dialect/Cute.h` and `CuteNVGPU.h`. + +Algebra types take an existing algebra attribute, for example +`cute.LayoutType.get(ctx, .{ .attr = layout_attr })`. Attribute constructors +call the C++ algebra parser with the raw expression, without building or parsing +MLIR assembly. `getValue()` returns canonical algebra text owned by the context. +The `algebraAttribute` and `algebraType` helpers dispatch to these C bindings. +Recovered NVGPU payload types take a `StringAttr` containing the +complete payload, including angle brackets, such as ``; the current schemas +preserve those architecture-specific contents without interpreting them. +Type construction uses the typed C API directly, with no temporary MLIR assembly +strings. For explicitly reading existing IR, use MLIR's generic `Type.parse` +or `mlirTypeParseGet`; there are no CuTe-specific parsing helpers. + +Operation builders take explicit parameters, followed by the location, and call +`mlir.Operation.make`. Parameters follow ODS names, with a trailing underscore +where needed to avoid Zig name collisions. Result parameters append `_type` or +`_types`; attributes take `*const mlir.Attribute`. Pass `null` for absent +optional parameters and `&.{}` for empty variadic parameters. +Dots in operation mnemonics become underscores, except for the private +`cute.@"tuple.product"` and `cute.@"tuple.product_each"` builders, whose literal +names distinguish them from the public `tuple_product` and `tuple_product_each`. +Builders compute variadic segment sizes as dense-array attributes and verify +the created operations. To defer verification, use `mlir.Operation.make` +directly with `.verify = false`. + +Builders pass variable-length operand and result groups through +`Operation.make`'s `.variadic` API, without temporary heap allocations or a +fixed operand-count limit in the Zig builder. MLIR still allocates its own +operation storage, and the dialect/compiler determines legal fragment sizes. + +Tests construct public/private/NVGPU operations, verify their registered schemas, +and round-trip MLIR bytecode into a fresh context: + +```sh +bazel test //mlir/dialects/cute_ir:test //mlir/dialects/cute_ir/test:all +``` + +These tests do not invoke the NVIDIA compiler or execute GPU kernels. Recovered +private schemas do not implement NVIDIA's complete semantic verification or +lowering pipeline. + +## Compiler Offset Inspection + +`tools:cute-objdump-offsets` invokes GNU objdump and scans its disassembly for +the eight known private compiler entry-point signatures. It requires Perl +(core modules only) and a GNU objdump supporting AArch64; it neither uses Python +nor loads the library. For example: + +```sh +bazel run //mlir/dialects/cute_ir/tools:cute-objdump-offsets -- \ + --input /absolute/path/to/_cutlass_ir.so --output /tmp/cute-offsets.json +bazel test //mlir/dialects/cute_ir/tools:cute-objdump-offsets-test +``` + +Use `--objdump /path/to/aarch64-linux-gnu-objdump` or `OBJDUMP` to select the +disassembler. The JSON report records the binary SHA-256, symbol availability, +all candidate addresses, and matching instructions. Addresses are ELF virtual +addresses, not file offsets. Missing or ambiguous matches produce exit status +1 and no `offsets` table, while retaining candidates for inspection. + +The signatures come from the native offset generator in `cute-native`, targeting +the 4.6.2 cu13/AArch64 package. This is an independent disassembly-based check +of those signatures, not fresh semantic identification or proof of ABI +compatibility with other builds. Some signatures contain build-specific PC-relative +instructions or neighboring code. This inspection tool does not replace the +native compiler bridge's build-time header generator or runtime binary checks. diff --git a/mlir/dialects/cute_ir/cute_ir.zig b/mlir/dialects/cute_ir/cute_ir.zig new file mode 100644 index 00000000..aa5d84dc --- /dev/null +++ b/mlir/dialects/cute_ir/cute_ir.zig @@ -0,0 +1,2618 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); +const cute = @This(); + +pub const dialect_namespace = "cute"; +pub const nvgpu = @import("cute_nvgpu.zig"); + +pub const AttributeKind = enum { int_tuple, coord, shape, stride, layout, tile, composed_layout, swizzle }; + +pub const CuteType = opaque { + const M = mlir.Methods(CuteType, c.MlirType); + + pub const isAFn = c.mlirTypeIsACuteType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + + pub fn format(self: *const CuteType, writer: *std.Io.Writer) std.Io.Writer.Error!void { + const t: *const mlir.Type = @ptrCast(self); + return t.format(writer); + } +}; + +/// Registers both dialects in the existing MLIR registry. +pub fn registerDialects(registry: *mlir.DialectRegistry) void { + mlir.DialectHandle.fromString(dialect_namespace).insertDialect(registry); + mlir.DialectHandle.fromString(nvgpu.dialect_namespace).insertDialect(registry); +} + +pub const AlgebraKind = AttributeKind; + +/// Constructs the algebra attribute and type through their C APIs. +/// Existing attributes can be passed directly to LayoutType.get, etc. +pub fn algebraType(ctx: *mlir.Context, kind: AlgebraKind, expression: []const u8) !*const mlir.Type { + return switch (kind) { + inline else => |k| blk: { + const T = switch (k) { + .int_tuple => IntTupleType, + .coord => CoordType, + .shape => ShapeType, + .stride => StrideType, + .layout => LayoutType, + .tile => TileType, + .composed_layout => ComposedLayoutType, + .swizzle => SwizzleType, + }; + const attr = try algebraAttribute(ctx, @field(AttributeKind, @tagName(k)), expression); + break :blk (try T.get(ctx, .{ .attr = attr })).type_(); + }, + }; +} + +pub fn algebraAttribute(ctx: *mlir.Context, kind: AttributeKind, expression: []const u8) !*const mlir.Attribute { + return switch (kind) { + inline else => |k| blk: { + const A = switch (k) { + .int_tuple => IntTupleAttr, + .coord => CoordAttr, + .shape => ShapeAttr, + .stride => StrideAttr, + .layout => LayoutAttr, + .tile => TileAttr, + .composed_layout => ComposedLayoutAttr, + .swizzle => SwizzleAttr, + }; + break :blk (try A.get(ctx, expression)).attribute(); + }, + }; +} + +pub const MemorySpace = enum { generic, gmem, cmem, smem, rmem, tmem, dsmem }; + +pub fn pointerType(ctx: *mlir.Context, element_type: ?*const mlir.Type, space: MemorySpace, alignment: u64) !*const mlir.Type { + return (try PtrType.get(ctx, .{ + .valueType = element_type, + .memorySpace = .string(ctx, @tagName(space)), + .alignment = alignment, + })).type_(); +} + +pub fn constrainedIntType(ctx: *mlir.Context, width: enum { i32, i64 }, divisible_by: u64) !*const mlir.Type { + return switch (width) { + .i32 => (try ConstrainedInt32Type.get(ctx, .{ .divisibleBy = divisible_by })).type_(), + .i64 => (try ConstrainedInt64Type.get(ctx, .{ .divisibleBy = divisible_by })).type_(), + }; +} + +/// Constructors take raw CuTe algebra expressions. Getter strings are owned by +/// the MLIR context and remain valid until that context is destroyed. +pub const IntTupleAttr = opaque { + const M = mlir.Methods(IntTupleAttr, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsACuteIntTuple; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + + pub fn get(ctx: *mlir.Context, value: []const u8) mlir.Error!*const IntTupleAttr { + const result = c.mlirCuteIntTupleAttrGet(ctx.ptr(), mlir.stringRef(value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn getValue(self: *const IntTupleAttr) []const u8 { + return mlir.string(c.mlirCuteIntTupleAttrGetValue(self.ptr())); + } + + pub fn attribute(self: *const IntTupleAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn format(self: *const IntTupleAttr, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } +}; + +pub const CoordAttr = opaque { + const M = mlir.Methods(CoordAttr, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsACuteCoord; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + + pub fn get(ctx: *mlir.Context, value: []const u8) mlir.Error!*const CoordAttr { + const result = c.mlirCuteCoordAttrGet(ctx.ptr(), mlir.stringRef(value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn getValue(self: *const CoordAttr) []const u8 { + return mlir.string(c.mlirCuteCoordAttrGetValue(self.ptr())); + } + + pub fn attribute(self: *const CoordAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn format(self: *const CoordAttr, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } +}; + +pub const ShapeAttr = opaque { + const M = mlir.Methods(ShapeAttr, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsACuteShape; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + + pub fn get(ctx: *mlir.Context, value: []const u8) mlir.Error!*const ShapeAttr { + const result = c.mlirCuteShapeAttrGet(ctx.ptr(), mlir.stringRef(value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn getValue(self: *const ShapeAttr) []const u8 { + return mlir.string(c.mlirCuteShapeAttrGetValue(self.ptr())); + } + + pub fn attribute(self: *const ShapeAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn format(self: *const ShapeAttr, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } +}; + +pub const StrideAttr = opaque { + const M = mlir.Methods(StrideAttr, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsACuteStride; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + + pub fn get(ctx: *mlir.Context, value: []const u8) mlir.Error!*const StrideAttr { + const result = c.mlirCuteStrideAttrGet(ctx.ptr(), mlir.stringRef(value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn getValue(self: *const StrideAttr) []const u8 { + return mlir.string(c.mlirCuteStrideAttrGetValue(self.ptr())); + } + + pub fn attribute(self: *const StrideAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn format(self: *const StrideAttr, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } +}; + +pub const LayoutAttr = opaque { + const M = mlir.Methods(LayoutAttr, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsACuteLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + + pub fn get(ctx: *mlir.Context, value: []const u8) mlir.Error!*const LayoutAttr { + const result = c.mlirCuteLayoutAttrGet(ctx.ptr(), mlir.stringRef(value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn getValue(self: *const LayoutAttr) []const u8 { + return mlir.string(c.mlirCuteLayoutAttrGetValue(self.ptr())); + } + + pub fn attribute(self: *const LayoutAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn format(self: *const LayoutAttr, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } +}; + +pub const TileAttr = opaque { + const M = mlir.Methods(TileAttr, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsACuteTile; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + + pub fn get(ctx: *mlir.Context, value: []const u8) mlir.Error!*const TileAttr { + const result = c.mlirCuteTileAttrGet(ctx.ptr(), mlir.stringRef(value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn getValue(self: *const TileAttr) []const u8 { + return mlir.string(c.mlirCuteTileAttrGetValue(self.ptr())); + } + + pub fn attribute(self: *const TileAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn format(self: *const TileAttr, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } +}; + +pub const ComposedLayoutAttr = opaque { + const M = mlir.Methods(ComposedLayoutAttr, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsACuteComposedLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + + pub fn get(ctx: *mlir.Context, value: []const u8) mlir.Error!*const ComposedLayoutAttr { + const result = c.mlirCuteComposedLayoutAttrGet(ctx.ptr(), mlir.stringRef(value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn getValue(self: *const ComposedLayoutAttr) []const u8 { + return mlir.string(c.mlirCuteComposedLayoutAttrGetValue(self.ptr())); + } + + pub fn attribute(self: *const ComposedLayoutAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn format(self: *const ComposedLayoutAttr, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } +}; + +pub const SwizzleAttr = opaque { + const M = mlir.Methods(SwizzleAttr, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsACuteSwizzle; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + + pub fn get(ctx: *mlir.Context, value: []const u8) mlir.Error!*const SwizzleAttr { + const result = c.mlirCuteSwizzleAttrGet(ctx.ptr(), mlir.stringRef(value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn getValue(self: *const SwizzleAttr) []const u8 { + return mlir.string(c.mlirCuteSwizzleAttrGetValue(self.ptr())); + } + + pub fn attribute(self: *const SwizzleAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn format(self: *const SwizzleAttr, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } +}; + +/// `!cute.arith_tuple_iter`. +pub const ArithTupleIteratorType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteArithTupleIterator; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + arithTuple: *const mlir.Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteArithTupleIteratorTypeGet(ctx.ptr(), args.arithTuple.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getArithTuple(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteArithTupleIteratorTypeGetArithTuple(self.ptr()).ptr.?); + } +}; + +/// `!cute.composed_layout`. +pub const ComposedLayoutType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteComposedLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + attr: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteComposedLayoutTypeGet(ctx.ptr(), args.attr.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getAttr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteComposedLayoutTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!cute.i32`. +pub const ConstrainedInt32Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteConstrainedInt32; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + divisibleBy: u64, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteConstrainedInt32TypeGet(ctx.ptr(), args.divisibleBy); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getDivisibleBy(self: *const Self) u64 { + return c.mlirCuteConstrainedInt32TypeGetDivisibleBy(self.ptr()); + } +}; + +/// `!cute.i64`. +pub const ConstrainedInt64Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteConstrainedInt64; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + divisibleBy: u64, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteConstrainedInt64TypeGet(ctx.ptr(), args.divisibleBy); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getDivisibleBy(self: *const Self) u64 { + return c.mlirCuteConstrainedInt64TypeGetDivisibleBy(self.ptr()); + } +}; + +/// `!cute.coord_tensor`. +pub const CoordTensorType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteCoordTensor; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + arithTuple: *const mlir.Type, + layout: *const mlir.Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteCoordTensorTypeGet(ctx.ptr(), args.arithTuple.ptr(), args.layout.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getArithTuple(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteCoordTensorTypeGetArithTuple(self.ptr()).ptr.?); + } + pub fn getLayout(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteCoordTensorTypeGetLayout(self.ptr()).ptr.?); + } +}; + +/// `!cute.coord`. +pub const CoordType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteCoord; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + attr: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteCoordTypeGet(ctx.ptr(), args.attr.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getAttr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCoordTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!cute.fast_divmod_divisor`. +pub const FastDivmodDivisorType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteFastDivmodDivisor; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + width: c_uint, + isPow2: bool, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteFastDivmodDivisorTypeGet(ctx.ptr(), args.width, args.isPow2); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getWidth(self: *const Self) c_uint { + return c.mlirCuteFastDivmodDivisorTypeGetWidth(self.ptr()); + } + pub fn getIsPow2(self: *const Self) bool { + return c.mlirCuteFastDivmodDivisorTypeGetIsPow2(self.ptr()); + } +}; + +/// `!cute.int_tuple`. +pub const IntTupleType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteIntTuple; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + attr: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteIntTupleTypeGet(ctx.ptr(), args.attr.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getAttr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteIntTupleTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!cute.layout`. +pub const LayoutType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + attr: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteLayoutTypeGet(ctx.ptr(), args.attr.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getAttr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteLayoutTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!cute.memref`. +pub const MemRefType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteMemRef; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + ptr: *const mlir.Type, + layout: *const mlir.Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteMemRefTypeGet(ctx.ptr(), args.ptr.ptr(), args.layout.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPtr(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteMemRefTypeGetPtr(self.ptr()).ptr.?); + } + pub fn getLayout(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteMemRefTypeGetLayout(self.ptr()).ptr.?); + } +}; + +/// `!cute.ptr`. +pub const PtrType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACutePtr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valueType: ?*const mlir.Type = null, + memorySpace: *const mlir.Attribute, + alignment: u64, + swizzle: ?*const mlir.Attribute = null, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCutePtrTypeGet(ctx.ptr(), if (args.valueType) |value| value.ptr() else c.MlirType{ .ptr = null }, args.memorySpace.ptr(), args.alignment, if (args.swizzle) |value| value.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValueType(self: *const Self) ?*const mlir.Type { + return @ptrCast(c.mlirCutePtrTypeGetValueType(self.ptr()).ptr); + } + pub fn getMemorySpace(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCutePtrTypeGetMemorySpace(self.ptr()).ptr.?); + } + pub fn getAlignment(self: *const Self) u64 { + return c.mlirCutePtrTypeGetAlignment(self.ptr()); + } + pub fn getSwizzle(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCutePtrTypeGetSwizzle(self.ptr()).ptr); + } +}; + +/// `!cute.shape`. +pub const ShapeType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteShape; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + attr: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteShapeTypeGet(ctx.ptr(), args.attr.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getAttr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteShapeTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!cute.sparse_elem`. +pub const SparseElemType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteSparseElem; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + numLogical: c_int, + physicalType: *const mlir.Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteSparseElemTypeGet(ctx.ptr(), args.numLogical, args.physicalType.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getNumLogical(self: *const Self) c_int { + return c.mlirCuteSparseElemTypeGetNumLogical(self.ptr()); + } + pub fn getPhysicalType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteSparseElemTypeGetPhysicalType(self.ptr()).ptr.?); + } +}; + +/// `!cute.stride`. +pub const StrideType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteStride; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + attr: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteStrideTypeGet(ctx.ptr(), args.attr.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getAttr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteStrideTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!cute.swizzle`. +pub const SwizzleType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteSwizzle; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + attr: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteSwizzleTypeGet(ctx.ptr(), args.attr.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getAttr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSwizzleTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!cute.tile`. +pub const TileType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteTile; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + attr: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteTileTypeGet(ctx.ptr(), args.attr.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getAttr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteTileTypeGetAttr(self.ptr()).ptr.?); + } +}; + +// Builders from the pinned OSS CuteOps.td. +/// `cute.append_to_rank`. Result types are explicit; attributes use mlir.Attribute. +pub fn append_to_rank(ctx: *mlir.Context, input: *const mlir.Value, element: *const mlir.Value, result_type: *const mlir.Type, rank: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.append_to_rank", .{ + .operands = .{ .flat = &.{ input, element } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "rank", rank), + }, + .location = location, + }); +} + +/// `cute.blocked_product`. Result types are explicit; attributes use mlir.Attribute. +pub fn blocked_product(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.blocked_product", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.ceil_div`. Result types are explicit; attributes use mlir.Attribute. +pub fn ceil_div(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.ceil_div", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.coalesce`. Result types are explicit; attributes use mlir.Attribute. +pub fn coalesce(ctx: *mlir.Context, input: *const mlir.Value, target_profile: ?*const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 2) = .empty; + operands.appendSliceAssumeCapacity(&.{input}); + if (target_profile) |value| operands.appendSliceAssumeCapacity(&.{value}); + return mlir.Operation.make(ctx, "cute.coalesce", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.complement`. Result types are explicit; attributes use mlir.Attribute. +pub fn complement(ctx: *mlir.Context, input: *const mlir.Value, cotarget: ?*const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 2) = .empty; + operands.appendSliceAssumeCapacity(&.{input}); + if (cotarget) |value| operands.appendSliceAssumeCapacity(&.{value}); + return mlir.Operation.make(ctx, "cute.complement", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.composed_get_inner`. Result types are explicit; attributes use mlir.Attribute. +pub fn composed_get_inner(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.composed_get_inner", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.composed_get_offset`. Result types are explicit; attributes use mlir.Attribute. +pub fn composed_get_offset(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.composed_get_offset", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.composed_get_outer`. Result types are explicit; attributes use mlir.Attribute. +pub fn composed_get_outer(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.composed_get_outer", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.composition`. Result types are explicit; attributes use mlir.Attribute. +pub fn composition(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.composition", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.cosize`. Result types are explicit; attributes use mlir.Attribute. +pub fn cosize(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, mode: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "cute.cosize", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.crd2idx`. Result types are explicit; attributes use mlir.Attribute. +pub fn crd2idx(ctx: *mlir.Context, coord: *const mlir.Value, shape: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.crd2idx", .{ + .operands = .{ .flat = &.{ coord, shape } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.dice`. Result types are explicit; attributes use mlir.Attribute. +pub fn dice(ctx: *mlir.Context, input: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.dice", .{ + .operands = .{ .flat = &.{ input, coord } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.elem_less`. Result types are explicit; attributes use mlir.Attribute. +pub fn elem_less(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.elem_less", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.equal`. Result types are explicit; attributes use mlir.Attribute. +pub fn equal(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.equal", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.flat_divide`. Result types are explicit; attributes use mlir.Attribute. +pub fn flat_divide(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.flat_divide", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.flat_product`. Result types are explicit; attributes use mlir.Attribute. +pub fn flat_product(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.flat_product", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.flatten`. Result types are explicit; attributes use mlir.Attribute. +pub fn flatten(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.flatten", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.get_layouts_from_tile`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_layouts_from_tile(ctx: *mlir.Context, tile: *const mlir.Value, layouts_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_layouts_from_tile", .{ + .operands = .{ .flat = &.{tile} }, + .results = .{ .variadic = &.{layouts_types} }, + .location = location, + }); +} + +/// `cute.get_leaves`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_leaves(ctx: *mlir.Context, input: *const mlir.Value, results_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_leaves", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .variadic = &.{results_types} }, + .location = location, + }); +} + +/// `cute.get`. Result types are explicit; attributes use mlir.Attribute. +pub fn get(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, mode: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "cute.get", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.get_scalars`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_scalars(ctx: *mlir.Context, cute_value: *const mlir.Value, scalars_types: []const *const mlir.Type, only_dynamic: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (only_dynamic) |value| attributes.appendAssumeCapacity(.named(ctx, "only_dynamic", value)); + return mlir.Operation.make(ctx, "cute.get_scalars", .{ + .operands = .{ .flat = &.{cute_value} }, + .results = .{ .variadic = &.{scalars_types} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.get_shape`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_shape(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_shape", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.get_stride`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_stride(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_stride", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.group_modes`. Result types are explicit; attributes use mlir.Attribute. +pub fn group_modes(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, begin: *const mlir.Attribute, end: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.group_modes", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "begin", begin), + .named(ctx, "end", end), + }, + .location = location, + }); +} + +/// `cute.idx2crd`. Result types are explicit; attributes use mlir.Attribute. +pub fn idx2crd(ctx: *mlir.Context, index: *const mlir.Value, shape: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.idx2crd", .{ + .operands = .{ .flat = &.{ index, shape } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.increment_coord`. Result types are explicit; attributes use mlir.Attribute. +pub fn increment_coord(ctx: *mlir.Context, coord: *const mlir.Value, shape: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.increment_coord", .{ + .operands = .{ .flat = &.{ coord, shape } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.layout_eval`. Result types are explicit; attributes use mlir.Attribute. +pub fn layout_eval(ctx: *mlir.Context, coord: *const mlir.Value, layout: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.layout_eval", .{ + .operands = .{ .flat = &.{ coord, layout } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.left_inverse`. Result types are explicit; attributes use mlir.Attribute. +pub fn left_inverse(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.left_inverse", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.logical_divide`. Result types are explicit; attributes use mlir.Attribute. +pub fn logical_divide(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.logical_divide", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.logical_product`. Result types are explicit; attributes use mlir.Attribute. +pub fn logical_product(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.logical_product", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_composed_layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_composed_layout(ctx: *mlir.Context, inner: *const mlir.Value, offset: *const mlir.Value, outer: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_composed_layout", .{ + .operands = .{ .flat = &.{ inner, offset, outer } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_coord`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_coord(ctx: *mlir.Context, operands_: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_coord", .{ + .operands = .{ .variadic = &.{operands_} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_identity_layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_identity_layout(ctx: *mlir.Context, shape: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_identity_layout", .{ + .operands = .{ .flat = &.{shape} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_int_tuple`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_int_tuple(ctx: *mlir.Context, operands_: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_int_tuple", .{ + .operands = .{ .variadic = &.{operands_} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_layout_like`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_layout_like(ctx: *mlir.Context, src: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_layout_like", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_layout(ctx: *mlir.Context, shape: *const mlir.Value, stride: *const mlir.Value, layout_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_layout", .{ + .operands = .{ .flat = &.{ shape, stride } }, + .results = .{ .flat = &.{layout_type} }, + .location = location, + }); +} + +/// `cute.make_ordered_layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_ordered_layout(ctx: *mlir.Context, shape: *const mlir.Value, order: *const mlir.Value, layout_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_ordered_layout", .{ + .operands = .{ .flat = &.{ shape, order } }, + .results = .{ .flat = &.{layout_type} }, + .location = location, + }); +} + +/// `cute.make_shape`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_shape(ctx: *mlir.Context, operands_: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_shape", .{ + .operands = .{ .variadic = &.{operands_} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_stride`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_stride(ctx: *mlir.Context, operands_: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_stride", .{ + .operands = .{ .variadic = &.{operands_} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_tile`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tile(ctx: *mlir.Context, operands_: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_tile", .{ + .operands = .{ .variadic = &.{operands_} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.prepend_to_rank`. Result types are explicit; attributes use mlir.Attribute. +pub fn prepend_to_rank(ctx: *mlir.Context, input: *const mlir.Value, element: *const mlir.Value, result_type: *const mlir.Type, rank: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.prepend_to_rank", .{ + .operands = .{ .flat = &.{ input, element } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "rank", rank), + }, + .location = location, + }); +} + +/// `cute.print`. Result types are explicit; attributes use mlir.Attribute. +pub fn print(ctx: *mlir.Context, value_: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.print", .{ + .operands = .{ .flat = &.{value_} }, + .location = location, + }); +} + +/// `cute.raked_product`. Result types are explicit; attributes use mlir.Attribute. +pub fn raked_product(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.raked_product", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.recast_layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn recast_layout(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, new_type_bits: *const mlir.Attribute, old_type_bits: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.recast_layout", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = &.{ + .named(ctx, "new_type_bits", new_type_bits), + .named(ctx, "old_type_bits", old_type_bits), + }, + .location = location, + }); +} + +/// `cute.right_inverse`. Result types are explicit; attributes use mlir.Attribute. +pub fn right_inverse(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.right_inverse", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.select`. Result types are explicit; attributes use mlir.Attribute. +pub fn select(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, mode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.select", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "mode", mode), + }, + .location = location, + }); +} + +/// `cute.shape_div`. Result types are explicit; attributes use mlir.Attribute. +pub fn shape_div(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.shape_div", .{ + .operands = .{ .flat = &.{ a, b } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.size`. Result types are explicit; attributes use mlir.Attribute. +pub fn size(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, mode: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "cute.size", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.slice`. Result types are explicit; attributes use mlir.Attribute. +pub fn slice(ctx: *mlir.Context, input: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.slice", .{ + .operands = .{ .flat = &.{ input, coord } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.static`. Result types are explicit; attributes use mlir.Attribute. +pub fn static(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.static", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tile_to_shape`. Result types are explicit; attributes use mlir.Attribute. +pub fn tile_to_shape(ctx: *mlir.Context, input: *const mlir.Value, shape: *const mlir.Value, order: ?*const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 3) = .empty; + operands.appendSliceAssumeCapacity(&.{input}); + operands.appendSliceAssumeCapacity(&.{shape}); + if (order) |value| operands.appendSliceAssumeCapacity(&.{value}); + return mlir.Operation.make(ctx, "cute.tile_to_shape", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tiled_divide`. Result types are explicit; attributes use mlir.Attribute. +pub fn tiled_divide(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tiled_divide", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tiled_product`. Result types are explicit; attributes use mlir.Attribute. +pub fn tiled_product(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tiled_product", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.to_coord`. Result types are explicit; attributes use mlir.Attribute. +pub fn to_coord(ctx: *mlir.Context, src: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.to_coord", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.to_int_tuple`. Result types are explicit; attributes use mlir.Attribute. +pub fn to_int_tuple(ctx: *mlir.Context, src: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.to_int_tuple", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.to_shape`. Result types are explicit; attributes use mlir.Attribute. +pub fn to_shape(ctx: *mlir.Context, src: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.to_shape", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.to_stride`. Result types are explicit; attributes use mlir.Attribute. +pub fn to_stride(ctx: *mlir.Context, src: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.to_stride", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tuple_add`. Result types are explicit; attributes use mlir.Attribute. +pub fn tuple_add(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple_add", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tuple_product_each`. Result types are explicit; attributes use mlir.Attribute. +pub fn tuple_product_each(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple_product_each", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tuple_product`. Result types are explicit; attributes use mlir.Attribute. +pub fn tuple_product(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple_product", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tuple_sub`. Result types are explicit; attributes use mlir.Attribute. +pub fn tuple_sub(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple_sub", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.zipped_divide`. Result types are explicit; attributes use mlir.Attribute. +pub fn zipped_divide(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.zipped_divide", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.zipped_product`. Result types are explicit; attributes use mlir.Attribute. +pub fn zipped_product(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.zipped_product", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +// Builders from CuteOpsPrivate.td. +/// `cute.add_offset`. Result types are explicit; attributes use mlir.Attribute. +pub fn add_offset(ctx: *mlir.Context, src: *const mlir.Value, offset: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.add_offset", .{ + .operands = .{ .flat = &.{ src, offset } }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.apply_swizzle`. Result types are explicit; attributes use mlir.Attribute. +pub fn apply_swizzle(ctx: *mlir.Context, ptr: *const mlir.Value, swizzled_ptr_type: *const mlir.Type, swizzle: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (swizzle) |value| attributes.appendAssumeCapacity(.named(ctx, "swizzle", value)); + return mlir.Operation.make(ctx, "cute.apply_swizzle", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{swizzled_ptr_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.assume`. Result types are explicit; attributes use mlir.Attribute. +pub fn assume(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.assume", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.copy_atom_call`. Result types are explicit; attributes use mlir.Attribute. +pub fn copy_atom_call(ctx: *mlir.Context, atom: *const mlir.Value, src: []const *const mlir.Value, dst: []const *const mlir.Value, pred: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.copy_atom_call", .{ + .operands = .{ .variadic = &.{ + &.{atom}, + src, + dst, + if (pred) |value| &.{value} else &.{}, + } }, + .location = location, + }); +} + +/// `cute.copy.make_fragment`. Result types are explicit; attributes use mlir.Attribute. +pub fn copy_make_fragment(ctx: *mlir.Context, atom: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.copy.make_fragment", .{ + .operands = .{ .flat = &.{ atom, input } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operand_id", operand_id), + }, + .location = location, + }); +} + +/// `cute.copy`. Result types are explicit; attributes use mlir.Attribute. +pub fn copy(ctx: *mlir.Context, copy_atom: *const mlir.Value, src: []const *const mlir.Value, dst: []const *const mlir.Value, pred: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.copy", .{ + .operands = .{ .variadic = &.{ + &.{copy_atom}, + src, + dst, + if (pred) |value| &.{value} else &.{}, + } }, + .location = location, + }); +} + +/// `cute.deref_arith_tuple_iter`. Result types are explicit; attributes use mlir.Attribute. +pub fn deref_arith_tuple_iter(ctx: *mlir.Context, iter: *const mlir.Value, arith_tuple_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.deref_arith_tuple_iter", .{ + .operands = .{ .flat = &.{iter} }, + .results = .{ .flat = &.{arith_tuple_type} }, + .location = location, + }); +} + +/// `cute.deref_desc_iter`. Result types are explicit; attributes use mlir.Attribute. +pub fn deref_desc_iter(ctx: *mlir.Context, iter: *const mlir.Value, value_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.deref_desc_iter", .{ + .operands = .{ .flat = &.{iter} }, + .results = .{ .flat = &.{value_type} }, + .location = location, + }); +} + +/// `cute.derefine`. Result types are explicit; attributes use mlir.Attribute. +pub fn derefine(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.derefine", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.fast_divmod.compute`. Result types are explicit; attributes use mlir.Attribute. +pub fn fast_divmod_compute(ctx: *mlir.Context, dividend: *const mlir.Value, divisor: *const mlir.Value, quotient_type: *const mlir.Type, remainder_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.fast_divmod.compute", .{ + .operands = .{ .flat = &.{ dividend, divisor } }, + .results = .{ .flat = &.{ quotient_type, remainder_type } }, + .location = location, + }); +} + +/// `cute.fast_divmod.create_divisor`. Result types are explicit; attributes use mlir.Attribute. +pub fn fast_divmod_create_divisor(ctx: *mlir.Context, divisor: *const mlir.Value, fast_divmod_divisor_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.fast_divmod.create_divisor", .{ + .operands = .{ .flat = &.{divisor} }, + .results = .{ .flat = &.{fast_divmod_divisor_type} }, + .location = location, + }); +} + +/// `cute.fast_divmod.divide`. Result types are explicit; attributes use mlir.Attribute. +pub fn fast_divmod_divide(ctx: *mlir.Context, dividend: *const mlir.Value, divisor: *const mlir.Value, quotient_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.fast_divmod.divide", .{ + .operands = .{ .flat = &.{ dividend, divisor } }, + .results = .{ .flat = &.{quotient_type} }, + .location = location, + }); +} + +/// `cute.fast_divmod.get_aux`. Result types are explicit; attributes use mlir.Attribute. +pub fn fast_divmod_get_aux(ctx: *mlir.Context, divisor: *const mlir.Value, multiplier_type: *const mlir.Type, shift1_type: *const mlir.Type, shift2_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.fast_divmod.get_aux", .{ + .operands = .{ .flat = &.{divisor} }, + .results = .{ .flat = &.{ multiplier_type, shift1_type, shift2_type } }, + .location = location, + }); +} + +/// `cute.fast_divmod.get_divisor`. Result types are explicit; attributes use mlir.Attribute. +pub fn fast_divmod_get_divisor(ctx: *mlir.Context, fast_divmod_divisor: *const mlir.Value, divisor_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.fast_divmod.get_divisor", .{ + .operands = .{ .flat = &.{fast_divmod_divisor} }, + .results = .{ .flat = &.{divisor_type} }, + .location = location, + }); +} + +/// `cute.fast_divmod.make_divisor`. Result types are explicit; attributes use mlir.Attribute. +pub fn fast_divmod_make_divisor(ctx: *mlir.Context, divisor: *const mlir.Value, multiplier: *const mlir.Value, sh1: *const mlir.Value, sh2: *const mlir.Value, fast_divmod_divisor_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.fast_divmod.make_divisor", .{ + .operands = .{ .flat = &.{ divisor, multiplier, sh1, sh2 } }, + .results = .{ .flat = &.{fast_divmod_divisor_type} }, + .location = location, + }); +} + +/// `cute.filter`. Result types are explicit; attributes use mlir.Attribute. +pub fn filter(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.filter", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.filter_zeros`. Result types are explicit; attributes use mlir.Attribute. +pub fn filter_zeros(ctx: *mlir.Context, input: *const mlir.Value, target_profile: ?*const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 2) = .empty; + operands.appendSliceAssumeCapacity(&.{input}); + if (target_profile) |value| operands.appendSliceAssumeCapacity(&.{value}); + return mlir.Operation.make(ctx, "cute.filter_zeros", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `cute.gemm`. Result types are explicit; attributes use mlir.Attribute. +pub fn gemm(ctx: *mlir.Context, mma_atom: *const mlir.Value, d: *const mlir.Value, a: []const *const mlir.Value, b: []const *const mlir.Value, c_: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.gemm", .{ + .operands = .{ .variadic = &.{ + &.{mma_atom}, + &.{d}, + a, + b, + &.{c_}, + } }, + .location = location, + }); +} + +/// `cute.get_flat_coord`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_flat_coord(ctx: *mlir.Context, index: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_flat_coord", .{ + .operands = .{ .flat = &.{ index, input } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.get_hier_coord`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_hier_coord(ctx: *mlir.Context, index: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_hier_coord", .{ + .operands = .{ .flat = &.{ index, input } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.get_integral_coord`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_integral_coord(ctx: *mlir.Context, index: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_integral_coord", .{ + .operands = .{ .flat = &.{ index, input } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.get_iter`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_iter(ctx: *mlir.Context, source: *const mlir.Value, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_iter", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{ptr_type} }, + .location = location, + }); +} + +/// `cute.get_layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_layout(ctx: *mlir.Context, input: *const mlir.Value, layout_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_layout", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{layout_type} }, + .location = location, + }); +} + +/// `cute.inttoptr`. Result types are explicit; attributes use mlir.Attribute. +pub fn inttoptr(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.inttoptr", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.kernel_smem_size`. Result types are explicit; attributes use mlir.Attribute. +pub fn kernel_smem_size(ctx: *mlir.Context, size_type: *const mlir.Type, kernel_name: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.kernel_smem_size", .{ + .results = .{ .flat = &.{size_type} }, + .attributes = &.{ + .named(ctx, "kernel_name", kernel_name), + }, + .location = location, + }); +} + +/// `cute.load_scaled_index`. Result types are explicit; attributes use mlir.Attribute. +pub fn load_scaled_index(ctx: *mlir.Context, ptr_a: *const mlir.Value, ptr_b: *const mlir.Value, index: *const mlir.Value, stride: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.load_scaled_index", .{ + .operands = .{ .flat = &.{ ptr_a, ptr_b, index, stride } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.local_partition`. Result types are explicit; attributes use mlir.Attribute. +pub fn local_partition(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, index: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.local_partition", .{ + .operands = .{ .flat = &.{ input, tiler, index } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.local_tile`. Result types are explicit; attributes use mlir.Attribute. +pub fn local_tile(ctx: *mlir.Context, input: *const mlir.Value, tile: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, proj: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (proj) |value| attributes.appendAssumeCapacity(.named(ctx, "proj", value)); + return mlir.Operation.make(ctx, "cute.local_tile", .{ + .operands = .{ .flat = &.{ input, tile, coord } }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.make_arith_tuple_iter`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_arith_tuple_iter(ctx: *mlir.Context, value_: ?*const mlir.Value, iter_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_arith_tuple_iter", .{ + .operands = .{ .flat = if (value_) |value| &.{value} else &.{} }, + .results = .{ .flat = &.{iter_type} }, + .location = location, + }); +} + +/// `cute.make_atom`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_atom(ctx: *mlir.Context, values: []const *const mlir.Value, atom_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_atom", .{ + .operands = .{ .variadic = &.{values} }, + .results = .{ .flat = &.{atom_type} }, + .location = location, + }); +} + +/// `cute.make_desc_iter`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_desc_iter(ctx: *mlir.Context, value_: *const mlir.Value, iter_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_desc_iter", .{ + .operands = .{ .flat = &.{value_} }, + .results = .{ .flat = &.{iter_type} }, + .location = location, + }); +} + +/// `cute.make_fragment_like`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_fragment_like(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_fragment_like", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.make_identity_tensor`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_identity_tensor(ctx: *mlir.Context, shape: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_identity_tensor", .{ + .operands = .{ .flat = &.{shape} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_sparse_elem`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_sparse_elem(ctx: *mlir.Context, physical_storage: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_sparse_elem", .{ + .operands = .{ .flat = &.{physical_storage} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_tiled_copy`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tiled_copy(ctx: *mlir.Context, atom: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_tiled_copy", .{ + .operands = .{ .flat = &.{atom} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_tiled_copy_v2`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tiled_copy_v2(ctx: *mlir.Context, atom: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_tiled_copy_v2", .{ + .operands = .{ .flat = &.{atom} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_tiled_mma`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tiled_mma(ctx: *mlir.Context, atom: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_tiled_mma", .{ + .operands = .{ .flat = &.{atom} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_tuple`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tuple(ctx: *mlir.Context, values: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_tuple", .{ + .operands = .{ .variadic = &.{values} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_view`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_view(ctx: *mlir.Context, iter: *const mlir.Value, layout: ?*const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 2) = .empty; + operands.appendSliceAssumeCapacity(&.{iter}); + if (layout) |value| operands.appendSliceAssumeCapacity(&.{value}); + return mlir.Operation.make(ctx, "cute.make_view", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.memref.alloc_smem`. Result types are explicit; attributes use mlir.Attribute. +pub fn memref_alloc_smem(ctx: *mlir.Context, memref_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.memref.alloc_smem", .{ + .results = .{ .flat = &.{memref_type} }, + .location = location, + }); +} + +/// `cute.memref.alloca`. Result types are explicit; attributes use mlir.Attribute. +pub fn memref_alloca(ctx: *mlir.Context, layout: ?*const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.memref.alloca", .{ + .operands = .{ .flat = if (layout) |value| &.{value} else &.{} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.memref.load`. Result types are explicit; attributes use mlir.Attribute. +pub fn memref_load(ctx: *mlir.Context, src: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.memref.load", .{ + .operands = .{ .flat = &.{ src, coord } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.memref.load_vec`. Result types are explicit; attributes use mlir.Attribute. +pub fn memref_load_vec(ctx: *mlir.Context, src: *const mlir.Value, mask: ?*const mlir.Value, pass_thru: ?*const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 3) = .empty; + operands.appendSliceAssumeCapacity(&.{src}); + if (mask) |value| operands.appendSliceAssumeCapacity(&.{value}); + if (pass_thru) |value| operands.appendSliceAssumeCapacity(&.{value}); + return mlir.Operation.make(ctx, "cute.memref.load_vec", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ + 1, + if (mask != null) 1 else 0, + if (pass_thru != null) 1 else 0, + })), + }, + .location = location, + }); +} + +/// `cute.memref.store`. Result types are explicit; attributes use mlir.Attribute. +pub fn memref_store(ctx: *mlir.Context, dst: *const mlir.Value, coord: *const mlir.Value, value_: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.memref.store", .{ + .operands = .{ .flat = &.{ dst, coord, value_ } }, + .location = location, + }); +} + +/// `cute.memref.store_vec`. Result types are explicit; attributes use mlir.Attribute. +pub fn memref_store_vec(ctx: *mlir.Context, value_: *const mlir.Value, dst: *const mlir.Value, mask: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 3) = .empty; + operands.appendSliceAssumeCapacity(&.{value_}); + operands.appendSliceAssumeCapacity(&.{dst}); + if (mask) |value| operands.appendSliceAssumeCapacity(&.{value}); + return mlir.Operation.make(ctx, "cute.memref.store_vec", .{ + .operands = .{ .flat = operands.constSlice() }, + .location = location, + }); +} + +/// `cute.mma_atom_call`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_atom_call(ctx: *mlir.Context, atom: *const mlir.Value, d: *const mlir.Value, a: []const *const mlir.Value, b: []const *const mlir.Value, c_: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.mma_atom_call", .{ + .operands = .{ .variadic = &.{ + &.{atom}, + &.{d}, + a, + b, + &.{c_}, + } }, + .location = location, + }); +} + +/// `cute.mma.make_fragment`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_make_fragment(ctx: *mlir.Context, atom: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.mma.make_fragment", .{ + .operands = .{ .flat = &.{ atom, input } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operand_id", operand_id), + }, + .location = location, + }); +} + +/// `cute.prefetch_atom_call`. Result types are explicit; attributes use mlir.Attribute. +pub fn prefetch_atom_call(ctx: *mlir.Context, atom: *const mlir.Value, src: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.prefetch_atom_call", .{ + .operands = .{ .variadic = &.{ + &.{atom}, + src, + } }, + .location = location, + }); +} + +/// `cute.prefetch`. Result types are explicit; attributes use mlir.Attribute. +pub fn prefetch(ctx: *mlir.Context, prefetch_atom: *const mlir.Value, src: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.prefetch", .{ + .operands = .{ .variadic = &.{ + &.{prefetch_atom}, + src, + } }, + .location = location, + }); +} + +/// `cute.print_tma_desc_im2col`. Result types are explicit; attributes use mlir.Attribute. +pub fn print_tma_desc_im2col(ctx: *mlir.Context, input: *const mlir.Value, fd: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.print_tma_desc_im2col", .{ + .operands = .{ .flat = &.{ input, fd } }, + .location = location, + }); +} + +/// `cute.print_tma_desc_tiled`. Result types are explicit; attributes use mlir.Attribute. +pub fn print_tma_desc_tiled(ctx: *mlir.Context, input: *const mlir.Value, fd: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.print_tma_desc_tiled", .{ + .operands = .{ .flat = &.{ input, fd } }, + .location = location, + }); +} + +/// `cute.print_view`. Result types are explicit; attributes use mlir.Attribute. +pub fn print_view(ctx: *mlir.Context, src: *const mlir.Value, coord: ?*const mlir.Value, verbose: ?*const mlir.Attribute, is_signed: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 2) = .empty; + operands.appendSliceAssumeCapacity(&.{src}); + if (coord) |value| operands.appendSliceAssumeCapacity(&.{value}); + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (verbose) |value| attributes.appendAssumeCapacity(.named(ctx, "verbose", value)); + if (is_signed) |value| attributes.appendAssumeCapacity(.named(ctx, "is_signed", value)); + return mlir.Operation.make(ctx, "cute.print_view", .{ + .operands = .{ .flat = operands.constSlice() }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.ptr.load`. Result types are explicit; attributes use mlir.Attribute. +pub fn ptr_load(ctx: *mlir.Context, ptr: *const mlir.Value, value_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.ptr.load", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{value_type} }, + .location = location, + }); +} + +/// `cute.ptr.store`. Result types are explicit; attributes use mlir.Attribute. +pub fn ptr_store(ctx: *mlir.Context, value_: *const mlir.Value, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.ptr.store", .{ + .operands = .{ .flat = &.{ value_, ptr } }, + .location = location, + }); +} + +/// `cute.ptrtoint`. Result types are explicit; attributes use mlir.Attribute. +pub fn ptrtoint(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.ptrtoint", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.random_delay`. Result types are explicit; attributes use mlir.Attribute. +pub fn random_delay(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.random_delay", .{ + .location = location, + }); +} + +/// `cute.recast_iter`. Result types are explicit; attributes use mlir.Attribute. +pub fn recast_iter(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.recast_iter", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.reduce`. Result types are explicit; attributes use mlir.Attribute. +pub fn reduce(ctx: *mlir.Context, input: *const mlir.Value, shape: *const mlir.Value, init_val: *const mlir.Value, predication: ?*const mlir.Value, output_type: *const mlir.Type, reduced_shape_type: ?*const mlir.Type, reduction_op: *const mlir.Attribute, reduction_profile: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 4) = .empty; + operands.appendSliceAssumeCapacity(&.{input}); + operands.appendSliceAssumeCapacity(&.{shape}); + operands.appendSliceAssumeCapacity(&.{init_val}); + if (predication) |value| operands.appendSliceAssumeCapacity(&.{value}); + var results: stdx.BoundedArray(*const mlir.Type, 2) = .empty; + results.appendSliceAssumeCapacity(&.{output_type}); + if (reduced_shape_type) |value| results.appendSliceAssumeCapacity(&.{value}); + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "reduction_op", reduction_op)); + if (reduction_profile) |value| attributes.appendAssumeCapacity(.named(ctx, "reduction_profile", value)); + return mlir.Operation.make(ctx, "cute.reduce", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = results.constSlice() }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.smem_partition_size`. Result types are explicit; attributes use mlir.Attribute. +pub fn smem_partition_size(ctx: *mlir.Context, size_type: *const mlir.Type, partition_id: *const mlir.Attribute, cumulative: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "partition_id", partition_id)); + if (cumulative) |value| attributes.appendAssumeCapacity(.named(ctx, "cumulative", value)); + return mlir.Operation.make(ctx, "cute.smem_partition_size", .{ + .results = .{ .flat = &.{size_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.stencil_divide`. Result types are explicit; attributes use mlir.Attribute. +pub fn stencil_divide(ctx: *mlir.Context, input: *const mlir.Value, stencil: *const mlir.Value, padding_upper: ?*const mlir.Value, padding_lower: ?*const mlir.Value, traversal_stride: ?*const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 5) = .empty; + operands.appendSliceAssumeCapacity(&.{input}); + operands.appendSliceAssumeCapacity(&.{stencil}); + if (padding_upper) |value| operands.appendSliceAssumeCapacity(&.{value}); + if (padding_lower) |value| operands.appendSliceAssumeCapacity(&.{value}); + if (traversal_stride) |value| operands.appendSliceAssumeCapacity(&.{value}); + return mlir.Operation.make(ctx, "cute.stencil_divide", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ + 1, + 1, + if (padding_upper != null) 1 else 0, + if (padding_lower != null) 1 else 0, + if (traversal_stride != null) 1 else 0, + })), + }, + .location = location, + }); +} + +/// `cute.symbolic`. Result types are explicit; attributes use mlir.Attribute. +pub fn symbolic(ctx: *mlir.Context, symbolic_type: *const mlir.Type, name: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.symbolic", .{ + .results = .{ .flat = &.{symbolic_type} }, + .attributes = &.{ + .named(ctx, "name", name), + }, + .location = location, + }); +} + +/// `cute.tiled.copy.partition_D`. Result types are explicit; attributes use mlir.Attribute. +pub fn tiled_copy_partition_D(ctx: *mlir.Context, tiled_copy: *const mlir.Value, input: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tiled.copy.partition_D", .{ + .operands = .{ .flat = &.{ tiled_copy, input, coord } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tiled.copy.partition_S`. Result types are explicit; attributes use mlir.Attribute. +pub fn tiled_copy_partition_S(ctx: *mlir.Context, tiled_copy: *const mlir.Value, input: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tiled.copy.partition_S", .{ + .operands = .{ .flat = &.{ tiled_copy, input, coord } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tiled.copy.retile`. Result types are explicit; attributes use mlir.Attribute. +pub fn tiled_copy_retile(ctx: *mlir.Context, tiled_copy: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tiled.copy.retile", .{ + .operands = .{ .flat = &.{ tiled_copy, input } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tiled.mma.partition`. Result types are explicit; attributes use mlir.Attribute. +pub fn tiled_mma_partition(ctx: *mlir.Context, tiled_mma: *const mlir.Value, input: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tiled.mma.partition", .{ + .operands = .{ .flat = &.{ tiled_mma, input, coord } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operand_id", operand_id), + }, + .location = location, + }); +} + +/// `cute.tiled.mma.partition_shape`. Result types are explicit; attributes use mlir.Attribute. +pub fn tiled_mma_partition_shape(ctx: *mlir.Context, tiled_mma: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tiled.mma.partition_shape", .{ + .operands = .{ .flat = &.{ tiled_mma, input } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operand_id", operand_id), + }, + .location = location, + }); +} + +/// `cute.tuple_div`. Result types are explicit; attributes use mlir.Attribute. +pub fn tuple_div(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple_div", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tuple_mod`. Result types are explicit; attributes use mlir.Attribute. +pub fn tuple_mod(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple_mod", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tuple_mul`. Result types are explicit; attributes use mlir.Attribute. +pub fn tuple_mul(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple_mul", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tuple.product_each`. Result types are explicit; attributes use mlir.Attribute. +pub fn @"tuple.product_each"(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple.product_each", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tuple.product`. Result types are explicit; attributes use mlir.Attribute. +pub fn @"tuple.product"(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple.product", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.unpack_tuple`. Result types are explicit; attributes use mlir.Attribute. +pub fn unpack_tuple(ctx: *mlir.Context, tuple: *const mlir.Value, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.unpack_tuple", .{ + .operands = .{ .flat = &.{tuple} }, + .results = .{ .variadic = &.{result_types} }, + .location = location, + }); +} + +pub const operation_names: []const []const u8 = &.{ + "cute.append_to_rank", + "cute.blocked_product", + "cute.ceil_div", + "cute.coalesce", + "cute.complement", + "cute.composed_get_inner", + "cute.composed_get_offset", + "cute.composed_get_outer", + "cute.composition", + "cute.cosize", + "cute.crd2idx", + "cute.dice", + "cute.elem_less", + "cute.equal", + "cute.flat_divide", + "cute.flat_product", + "cute.flatten", + "cute.get_layouts_from_tile", + "cute.get_leaves", + "cute.get", + "cute.get_scalars", + "cute.get_shape", + "cute.get_stride", + "cute.group_modes", + "cute.idx2crd", + "cute.increment_coord", + "cute.layout_eval", + "cute.left_inverse", + "cute.logical_divide", + "cute.logical_product", + "cute.make_composed_layout", + "cute.make_coord", + "cute.make_identity_layout", + "cute.make_int_tuple", + "cute.make_layout_like", + "cute.make_layout", + "cute.make_ordered_layout", + "cute.make_shape", + "cute.make_stride", + "cute.make_tile", + "cute.prepend_to_rank", + "cute.print", + "cute.raked_product", + "cute.recast_layout", + "cute.right_inverse", + "cute.select", + "cute.shape_div", + "cute.size", + "cute.slice", + "cute.static", + "cute.tile_to_shape", + "cute.tiled_divide", + "cute.tiled_product", + "cute.to_coord", + "cute.to_int_tuple", + "cute.to_shape", + "cute.to_stride", + "cute.tuple_add", + "cute.tuple_product_each", + "cute.tuple_product", + "cute.tuple_sub", + "cute.zipped_divide", + "cute.zipped_product", + "cute.add_offset", + "cute.apply_swizzle", + "cute.assume", + "cute.copy_atom_call", + "cute.copy.make_fragment", + "cute.copy", + "cute.deref_arith_tuple_iter", + "cute.deref_desc_iter", + "cute.derefine", + "cute.fast_divmod.compute", + "cute.fast_divmod.create_divisor", + "cute.fast_divmod.divide", + "cute.fast_divmod.get_aux", + "cute.fast_divmod.get_divisor", + "cute.fast_divmod.make_divisor", + "cute.filter", + "cute.filter_zeros", + "cute.gemm", + "cute.get_flat_coord", + "cute.get_hier_coord", + "cute.get_integral_coord", + "cute.get_iter", + "cute.get_layout", + "cute.inttoptr", + "cute.kernel_smem_size", + "cute.load_scaled_index", + "cute.local_partition", + "cute.local_tile", + "cute.make_arith_tuple_iter", + "cute.make_atom", + "cute.make_desc_iter", + "cute.make_fragment_like", + "cute.make_identity_tensor", + "cute.make_sparse_elem", + "cute.make_tiled_copy", + "cute.make_tiled_copy_v2", + "cute.make_tiled_mma", + "cute.make_tuple", + "cute.make_view", + "cute.memref.alloc_smem", + "cute.memref.alloca", + "cute.memref.load", + "cute.memref.load_vec", + "cute.memref.store", + "cute.memref.store_vec", + "cute.mma_atom_call", + "cute.mma.make_fragment", + "cute.prefetch_atom_call", + "cute.prefetch", + "cute.print_tma_desc_im2col", + "cute.print_tma_desc_tiled", + "cute.print_view", + "cute.ptr.load", + "cute.ptr.store", + "cute.ptrtoint", + "cute.random_delay", + "cute.recast_iter", + "cute.reduce", + "cute.smem_partition_size", + "cute.stencil_divide", + "cute.symbolic", + "cute.tiled.copy.partition_D", + "cute.tiled.copy.partition_S", + "cute.tiled.copy.retile", + "cute.tiled.mma.partition", + "cute.tiled.mma.partition_shape", + "cute.tuple_div", + "cute.tuple_mod", + "cute.tuple_mul", + "cute.tuple.product_each", + "cute.tuple.product", + "cute.unpack_tuple", +}; + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + registerDialects(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(nvgpu); + inline for (comptime std.meta.declarations(@This())) |decl| { + const T = @field(@This(), decl.name); + if (@TypeOf(T) == type) std.testing.refAllDecls(T); + } + inline for (comptime std.meta.declarations(nvgpu)) |decl| { + const T = @field(nvgpu, decl.name); + if (@TypeOf(T) == type) std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const ctx = try testContext(); + defer ctx.deinit(); + for (cute.operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + for (nvgpu.operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "public and private types through the C API" { + const ctx = try testContext(); + defer ctx.deinit(); + const tuple = try algebraType(ctx, .int_tuple, "(16,32)"); + try std.testing.expect(tuple.eql(try mlir.Type.parse(ctx, "!cute.int_tuple<\"(16,32)\">"))); + try std.testing.expect(tuple.isA(CuteType) != null); + const f32_type = mlir.Type.float(ctx, .f32); + try std.testing.expect((try pointerType(ctx, f32_type, .gmem, 16)).eql(try mlir.Type.parse(ctx, "!cute.ptr>"))); + try std.testing.expect((try constrainedIntType(ctx, .i32, 8)).eql(try mlir.Type.parse(ctx, "!cute.i32"))); + try std.testing.expect((try algebraAttribute(ctx, .layout, "(16,32):(32,1)")).eql(try mlir.Attribute.parse(ctx, "#cute.layout<\"(16,32):(32,1)\">"))); + try std.testing.expect(mlir.Type.int(ctx, .i32).isA(CuteType) == null); + try std.testing.expect((try nvgpu.SmemDescType.get(ctx, .{})).isA(nvgpu.NVGPUType) != null); + const atom = try nvgpu.CopyAtomSIMTSyncCopyType.get(ctx, .{ .payload = .string(ctx, "") }); + try std.testing.expect(atom.type_().eql(try mlir.Type.parse(ctx, "!cute_nvgpu.atom.universal_copy"))); +} + +test "typed constructors and accessors use native type storage" { + const ctx = try testContext(); + defer ctx.deinit(); + const f32_type = mlir.Type.float(ctx, .f32); + const space = mlir.Attribute.string(ctx, "smem"); + const swizzle = try algebraAttribute(ctx, .swizzle, "S<3,4,3>"); + const pointer = try PtrType.get(ctx, .{ + .valueType = f32_type, + .memorySpace = space, + .alignment = 16, + .swizzle = swizzle, + }); + try std.testing.expect(pointer.getValueType().?.eql(f32_type)); + try std.testing.expect(pointer.getMemorySpace().eql(space)); + try std.testing.expectEqual(@as(u64, 16), pointer.getAlignment()); + try std.testing.expect(pointer.getSwizzle().?.eql(swizzle)); + try std.testing.expect(pointer.type_().isA(PtrType) != null); + try std.testing.expect(f32_type.isA(PtrType) == null); + const untyped_pointer = try PtrType.get(ctx, .{ .memorySpace = space, .alignment = 0 }); + try std.testing.expect(untyped_pointer.getValueType() == null); + try std.testing.expect(untyped_pointer.getSwizzle() == null); + + const layout_attr = try algebraAttribute(ctx, .layout, "4:1"); + const layout = try LayoutType.get(ctx, .{ .attr = layout_attr }); + try std.testing.expect(layout.getAttr().eql(layout_attr)); + const memref = try MemRefType.get(ctx, .{ .ptr = pointer.type_(), .layout = layout.type_() }); + try std.testing.expect(memref.getPtr().eql(pointer.type_())); + try std.testing.expect(memref.getLayout().eql(layout.type_())); + const tuple = try algebraType(ctx, .int_tuple, "(4,8)"); + const iterator = try ArithTupleIteratorType.get(ctx, .{ .arithTuple = tuple }); + try std.testing.expect(iterator.getArithTuple().eql(tuple)); + const coords = try CoordTensorType.get(ctx, .{ .arithTuple = tuple, .layout = layout.type_() }); + try std.testing.expect(coords.getArithTuple().eql(tuple)); + try std.testing.expect(coords.getLayout().eql(layout.type_())); + + const i32_type = try ConstrainedInt32Type.get(ctx, .{ .divisibleBy = 8 }); + const i64_type = try ConstrainedInt64Type.get(ctx, .{ .divisibleBy = 16 }); + try std.testing.expectEqual(@as(u64, 8), i32_type.getDivisibleBy()); + try std.testing.expectEqual(@as(u64, 16), i64_type.getDivisibleBy()); + const divisor = try FastDivmodDivisorType.get(ctx, .{ .width = 64, .isPow2 = true }); + try std.testing.expectEqual(@as(c_uint, 64), divisor.getWidth()); + try std.testing.expect(divisor.getIsPow2()); + const sparse = try SparseElemType.get(ctx, .{ .numLogical = 2, .physicalType = f32_type }); + try std.testing.expectEqual(@as(c_int, 2), sparse.getNumLogical()); + try std.testing.expect(sparse.getPhysicalType().eql(f32_type)); + + const payload = mlir.Attribute.string(ctx, ""); + const atom = try nvgpu.CopyAtomSIMTSyncCopyType.get(ctx, .{ .payload = payload }); + try std.testing.expect(atom.getPayload().eql(payload)); + try std.testing.expect(atom.type_().isA(nvgpu.CopyAtomSIMTSyncCopyType) != null); + inline for (.{ nvgpu.SmemDescType, nvgpu.TmaDescriptorTiledType, nvgpu.TmaDescriptorIm2ColType, nvgpu.WorkIdResponseType }) |T| { + try std.testing.expect((try T.get(ctx, .{})).type_().isA(T) != null); + } +} + +test "explicit algebra attribute bindings and type constructors" { + const ctx = try testContext(); + defer ctx.deinit(); + inline for (.{ + .{ IntTupleAttr, IntTupleType, "(16,32)" }, + .{ CoordAttr, CoordType, "(_,?)" }, + .{ ShapeAttr, ShapeType, "(16,32)" }, + .{ StrideAttr, StrideType, "(32,1)" }, + .{ LayoutAttr, LayoutType, "(16,32):(32,1)" }, + .{ TileAttr, TileType, "[(2,3):(1,2);_]" }, + .{ ComposedLayoutAttr, ComposedLayoutType, "S<3,5,4> o 2 o (2,3):(1,2)" }, + .{ SwizzleAttr, SwizzleType, "S<3,4,3>" }, + }) |example| { + const A = example[0]; + const T = example[1]; + const attr = try A.get(ctx, example[2]); + try std.testing.expect(attr.attribute().isA(A) != null); + try std.testing.expect(mlir.Attribute.string(ctx, example[2]).isA(A) == null); + try std.testing.expectError(error.InvalidMlir, A.get(ctx, "invalid")); + const value = attr.getValue(); + try std.testing.expect(attr.eql(try A.get(ctx, value))); + const type_ = try T.get(ctx, .{ .attr = attr.attribute() }); + try std.testing.expect(type_.getAttr().eql(attr.attribute())); + + var text: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer text.deinit(); + try text.writer.print("{f}", .{attr}); + try std.testing.expect(attr.attribute().eql(try mlir.Attribute.parse(ctx, text.written()))); + text.clearRetainingCapacity(); + try text.writer.print("{f}", .{type_}); + try std.testing.expect(type_.type_().eql(try mlir.Type.parse(ctx, text.written()))); + // The getter's view survives other attribute creation in this context. + try std.testing.expectEqualStrings(value, attr.getValue()); + } +} + +test "typed constructors reject invalid parameters" { + const ctx = try testContext(); + defer ctx.deinit(); + const f32_type = mlir.Type.float(ctx, .f32); + const space = mlir.Attribute.string(ctx, "smem"); + try std.testing.expectError(error.InvalidMlir, LayoutType.get(ctx, .{ .attr = space })); + try std.testing.expectError(error.InvalidMlir, ArithTupleIteratorType.get(ctx, .{ .arithTuple = f32_type })); + try std.testing.expectError(error.InvalidMlir, ConstrainedInt32Type.get(ctx, .{ .divisibleBy = 0 })); + try std.testing.expectError(error.InvalidMlir, FastDivmodDivisorType.get(ctx, .{ .width = 8, .isPow2 = false })); + try std.testing.expectError(error.InvalidMlir, SparseElemType.get(ctx, .{ .numLogical = 0, .physicalType = f32_type })); + try std.testing.expectError(error.InvalidMlir, PtrType.get(ctx, .{ .memorySpace = space, .alignment = 3 })); + try std.testing.expectError(error.InvalidMlir, PtrType.get(ctx, .{ .memorySpace = space, .alignment = 16, .swizzle = space })); + const pointer = try pointerType(ctx, f32_type, .smem, 16); + try std.testing.expectError(error.InvalidMlir, MemRefType.get(ctx, .{ .ptr = pointer, .layout = f32_type })); + try std.testing.expectError(error.InvalidMlir, nvgpu.CopyAtomSIMTSyncCopyType.get(ctx, .{ .payload = .unit(ctx) })); + + const other_ctx = try testContext(); + defer other_ctx.deinit(); + try std.testing.expectError(error.InvalidMlir, PtrType.get(other_ctx, .{ .valueType = f32_type, .memorySpace = .string(other_ctx, "smem"), .alignment = 16 })); + try std.testing.expectError(error.InvalidMlir, PtrType.get(other_ctx, .{ .memorySpace = space, .alignment = 16 })); +} + +test "all NVGPU type constructors preserve their storage" { + const ctx = try testContext(); + defer ctx.deinit(); + const payload = mlir.Attribute.string(ctx, ""); + inline for (comptime std.meta.declarations(nvgpu)) |decl| { + const T = @field(nvgpu, decl.name); + if (@TypeOf(T) == type) { + if (@hasDecl(T, "InitArgs")) { + const has_payload = @hasField(T.InitArgs, "payload"); + const type_ = try T.get(ctx, if (has_payload) .{ .payload = payload } else .{}); + try std.testing.expect(type_.type_().isA(T) != null); + try std.testing.expect(type_.type_().isA(nvgpu.NVGPUType) != null); + if (has_payload) try std.testing.expect(type_.getPayload().eql(payload)); + var text: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer text.deinit(); + try text.writer.print("{f}", .{type_}); + try std.testing.expect(type_.type_().eql(try mlir.Type.parse(ctx, text.written()))); + } + } + } +} + +test "dotted private tuple builders remain distinct from public builders" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc = mlir.Location.unknown(ctx); + const tuple_type = try algebraType(ctx, .int_tuple, "(2,(3,4))"); + const tuple = cute.static(ctx, tuple_type, loc); + defer tuple.deinit(); + + inline for (.{ + .{ cute.tuple_product, "cute.tuple_product", "24" }, + .{ cute.@"tuple.product", "cute.tuple.product", "24" }, + .{ cute.tuple_product_each, "cute.tuple_product_each", "(2,12)" }, + .{ cute.@"tuple.product_each", "cute.tuple.product_each", "(2,12)" }, + }) |example| { + const op = example[0](ctx, tuple.result(0), try algebraType(ctx, .int_tuple, example[2]), loc); + defer op.deinit(); + try std.testing.expectEqualStrings(example[1], op.name()); + try std.testing.expect(op.verify()); + } +} + +test "NVGPU builders preserve optional operand segments and attributes" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc = mlir.Location.unknown(ctx); + const gmem = try pointerType(ctx, .float(ctx, .f32), .gmem, 16); + const dsmem = try pointerType(ctx, .float(ctx, .f32), .dsmem, 16); + const barrier = try pointerType(ctx, .int(ctx, .i64), .dsmem, 8); + const block = mlir.Block.init( + &.{ gmem, dsmem, barrier, .int(ctx, .i16), .int(ctx, .i64) }, + &.{ loc, loc, loc, loc, loc }, + ); + defer block.deinit(); + for ([_]bool{ false, true }) |with_mask| { + for ([_]bool{ false, true }) |with_cache| { + const byte_count = mlir.Attribute.int(ctx, .i32, 128); + const op = nvgpu.arch_copy_SM90_bulk_copy_g2s(ctx, block.argument(0), block.argument(1), block.argument(2), if (with_mask) block.argument(3) else null, if (with_cache) block.argument(4) else null, byte_count, loc); + defer op.deinit(); + try std.testing.expect(op.verify()); + try std.testing.expect(op.attributeByName("size").?.eql(byte_count)); + try std.testing.expectEqual(@as(usize, 3) + @intFromBool(with_mask) + @intFromBool(with_cache), op.numOperands()); + try std.testing.expect(op.attributeByName("operandSegmentSizes").?.eql(.denseArray( + ctx, + .i32, + &.{ 1, 1, 1, @intFromBool(with_mask), @intFromBool(with_cache) }, + ))); + if (with_cache) try std.testing.expect(op.operand(op.numOperands() - 1).eql(block.argument(4))); + } + } +} + +test "build public, private and NVGPU operations and round-trip bytecode" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc = mlir.Location.unknown(ctx); + const module = mlir.Module.init(loc); + defer module.deinit(); + const body = module.body(); + + const shape_type = try algebraType(ctx, .shape, "4"); + const stride_type = try algebraType(ctx, .stride, "1"); + const layout_type = try algebraType(ctx, .layout, "4:1"); + const shape = cute.static(ctx, shape_type, loc).appendTo(body); + const stride = cute.static(ctx, stride_type, loc).appendTo(body); + const constructed_shape = cute.make_shape(ctx, &.{}, shape_type, loc).appendTo(body); + try std.testing.expect(constructed_shape.attributeByName("operandSegmentSizes").?.eql(.denseArray(ctx, .i32, &.{0}))); + const leaves = cute.get_leaves(ctx, constructed_shape.result(0), &.{shape_type}, loc).appendTo(body); + try std.testing.expectEqual(@as(usize, 1), leaves.numResults()); + try std.testing.expect(leaves.attributeByName("resultSegmentSizes").?.eql(.denseArray(ctx, .i32, &.{1}))); + const layout = cute.make_layout(ctx, shape.result(0), stride.result(0), layout_type, loc).appendTo(body); + try std.testing.expect(layout.result(0).type_().eql(layout_type)); + + const memref_type = (try MemRefType.get(ctx, .{ + .ptr = try pointerType(ctx, .float(ctx, .f32), .smem, 16), + .layout = layout_type, + })).type_(); + const alloc = cute.memref_alloc_smem(ctx, memref_type, loc).appendTo(body); + const assumed = cute.assume(ctx, alloc.result(0), memref_type, loc).appendTo(body); + const atom_type = (try nvgpu.CopyAtomSIMTSyncCopyType.get(ctx, .{ .payload = .string(ctx, "") })).type_(); + const atom = cute.make_atom(ctx, &.{}, atom_type, loc).appendTo(body); + const copy_op = cute.copy(ctx, atom.result(0), &.{ alloc.result(0), assumed.result(0) }, &.{assumed.result(0)}, null, loc).appendTo(body); + const segments = mlir.Attribute.denseArray(ctx, .i32, &.{ 1, 2, 1, 0 }); + try std.testing.expectEqual(@as(usize, 4), copy_op.numOperands()); + try std.testing.expect(copy_op.attributeByName("operandSegmentSizes").?.eql(segments)); + + const predicate = mlir.Operation.make(ctx, "arith.constant", .{ + .results = .{ .flat = &.{.int(ctx, .i1)} }, + .attributes = &.{.named(ctx, "value", .boolean(ctx, true))}, + .location = loc, + }).appendTo(body); + const predicated_copy = cute.copy(ctx, atom.result(0), &.{alloc.result(0)}, &.{assumed.result(0)}, predicate.result(0), loc).appendTo(body); + try std.testing.expect(predicated_copy.attributeByName("operandSegmentSizes").?.eql(.denseArray(ctx, .i32, &.{ 1, 1, 1, 1 }))); + + // Operand groups have no fixed per-group capacity in the Zig builder. + const many_sources = [_]*const mlir.Value{alloc.result(0)} ** 256; + const large_copy = cute.copy(ctx, atom.result(0), &many_sources, &.{assumed.result(0)}, null, loc).appendTo(body); + try std.testing.expectEqual(@as(usize, 258), large_copy.numOperands()); + try std.testing.expect(large_copy.attributeByName("operandSegmentSizes").?.eql(.denseArray(ctx, .i32, &.{ 1, 256, 1, 0 }))); + + const smem_size = nvgpu.arch_get_dyn_smem_size(ctx, .int(ctx, .i32), loc).appendTo(body); + try std.testing.expectEqualStrings("cute_nvgpu.arch.get_dyn_smem_size", smem_size.name()); + const pointer = try pointerType(ctx, .float(ctx, .f32), .smem, 16); + _ = nvgpu.arch_alloc_smem(ctx, pointer, .int(ctx, .i32, 16), loc).appendTo(body); + try std.testing.expect(module.operation().verify()); + + var text: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer text.deinit(); + try text.writer.print("{f}", .{module.operation()}); + const parsed = try mlir.Module.parse(ctx, text.written()); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); + + var bytes: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer bytes.deinit(); + try module.operation().writeBytecode(null, &bytes.writer); + try std.testing.expect(std.mem.startsWith(u8, bytes.written(), "ML\xefR")); + const reader_ctx = try testContext(); + defer reader_ctx.deinit(); + const round_trip = try mlir.Operation.parse(reader_ctx, bytes.written(), "cute-zig.mlirbc"); + defer round_trip.deinit(); + try std.testing.expect(round_trip.verify()); + var round_trip_text: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer round_trip_text.deinit(); + try round_trip_text.writer.print("{f}", .{round_trip}); + try std.testing.expectEqualStrings(text.written(), round_trip_text.written()); +} + +test "SM120 block-scaled MMA preserves fragment groups and optional selectors" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc = mlir.Location.unknown(ctx); + const i32_type = mlir.Type.int(ctx, .i32); + const f32_type = mlir.Type.float(ctx, .f32); + const arg_types = [_]*const mlir.Type{i32_type} ** 6 ++ + [_]*const mlir.Type{f32_type} ** 4 ++ + [_]*const mlir.Type{.int(ctx, .i8)} ** 2 ++ + [_]*const mlir.Type{.int(ctx, .i16)} ** 2; + const block = mlir.Block.init(&arg_types, &([_]*const mlir.Location{loc} ** 14)); + defer block.deinit(); + const shape = try algebraAttribute(ctx, .shape, "(16,8,64)"); + const fp4 = mlir.Attribute.typeAttr(try mlir.Type.parse(ctx, "f4E2M1FN")); + const scale_type = mlir.Attribute.typeAttr(try mlir.Type.parse(ctx, "f8E8M0FNU")); + for ([_]bool{ false, true }) |with_a| { + for ([_]bool{ false, true }) |with_b| { + const op = nvgpu.arch_mma_SM120_block_scaled( + ctx, + &.{ block.argument(0), block.argument(1), block.argument(2), block.argument(3) }, + &.{ block.argument(4), block.argument(5) }, + &.{ block.argument(6), block.argument(7), block.argument(8), block.argument(9) }, + block.argument(10), + block.argument(11), + if (with_a) block.argument(12) else null, + if (with_b) block.argument(13) else null, + &([_]*const mlir.Type{f32_type} ** 4), + shape, + .int(ctx, .i32, 32), + null, + null, + fp4, + fp4, + scale_type, + loc, + ); + defer op.deinit(); + try std.testing.expect(op.verify()); + try std.testing.expectEqual(@as(usize, 12) + @intFromBool(with_a) + @intFromBool(with_b), op.numOperands()); + for (0..12) |i| try std.testing.expect(op.operand(i).eql(block.argument(i))); + if (with_a) try std.testing.expect(op.operand(12).eql(block.argument(12))); + if (with_b) try std.testing.expect(op.operand(op.numOperands() - 1).eql(block.argument(13))); + try std.testing.expectEqual(@as(usize, 4), op.numResults()); + try std.testing.expect(op.attributeByName("operandSegmentSizes").?.eql(.denseArray( + ctx, + .i32, + &.{ 4, 2, 4, 1, 1, @intFromBool(with_a), @intFromBool(with_b) }, + ))); + } + } +} + +test "private verifier rejects inconsistent operand segments" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc = mlir.Location.unknown(ctx); + const atom_type = (try nvgpu.CopyAtomSIMTSyncCopyType.get(ctx, .{ .payload = .string(ctx, "") })).type_(); + const atom = cute.make_atom(ctx, &.{}, atom_type, loc); + defer atom.deinit(); + const copy_op = cute.copy(ctx, atom.result(0), &.{}, &.{}, null, loc); + defer copy_op.deinit(); + try std.testing.expect(copy_op.verify()); + copy_op.setAttributeByName("operandSegmentSizes", .denseArray(ctx, .i32, &.{ 1, 1, 0, 0 })); + try std.testing.expect(!copy_op.verify()); +} diff --git a/mlir/dialects/cute_ir/cute_nvgpu.zig b/mlir/dialects/cute_ir/cute_nvgpu.zig new file mode 100644 index 00000000..8e41a988 --- /dev/null +++ b/mlir/dialects/cute_ir/cute_nvgpu.zig @@ -0,0 +1,1983 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub const dialect_namespace = "cute_nvgpu"; + +pub const NVGPUType = opaque { + const M = mlir.Methods(NVGPUType, c.MlirType); + + pub const isAFn = c.mlirTypeIsACuteNVGPUType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + + pub fn format(self: *const NVGPUType, writer: *std.Io.Writer) std.Io.Writer.Error!void { + const t: *const mlir.Type = @ptrCast(self); + return t.format(writer); + } +}; + +/// `!cute_nvgpu.atom.bulk_copy_g2s`. +pub const CopyAtomBulkCopyG2SType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomBulkCopyG2S; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomBulkCopyG2STypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.bulk_copy_s2g`. +pub const CopyAtomBulkCopyS2GType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2G; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.bulk_copy_s2s`. +pub const CopyAtomBulkCopyS2SType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2S; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomBulkCopyS2STypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.dsmem_store`. +pub const CopyAtomDsmemStoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomDsmemStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomDsmemStoreTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomDsmemStoreTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.g2r`. +pub const CopyAtomG2RType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomG2R; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomG2RTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.im2col_tma_load`. +pub const CopyAtomIm2ColTmaLoadType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.im2col_tma_store`. +pub const CopyAtomIm2ColTmaStoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.ldsm`. +pub const CopyAtomLdsmType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomLdsm; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomLdsmTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomLdsmTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.non_exec_2d_gather4_tma_load`. +pub const CopyAtomNonExec2DGather4TmaLoadType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExec2DGather4TmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.non_exec_im2col_tma_load`. +pub const CopyAtomNonExecIm2ColTmaLoadType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.non_exec_im2col_tma_store`. +pub const CopyAtomNonExecIm2ColTmaStoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.non_exec_tiled_tma_load`. +pub const CopyAtomNonExecTiledTmaLoadType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.non_exec_tiled_tma_reduce`. +pub const CopyAtomNonExecTiledTmaReduceType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaReduce; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.non_exec_tiled_tma_store`. +pub const CopyAtomNonExecTiledTmaStoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.r2g`. +pub const CopyAtomR2GType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomR2G; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomR2GTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.r2s`. +pub const CopyAtomR2SType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomR2S; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomR2STypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomR2STypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.s2r`. +pub const CopyAtomS2RType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomS2R; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomS2RTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.simt_async_copy`. +pub const CopyAtomSIMTAsyncCopyType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTAsyncCopy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.simt_multimem_ld_reduce`. +pub const CopyAtomSIMTMultimemLdReduceType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemLdReduce; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.simt_multimem_red`. +pub const CopyAtomSIMTMultimemRedType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemRed; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.simt_multimem_st`. +pub const CopyAtomSIMTMultimemStType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemSt; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.universal_copy`. +pub const CopyAtomSIMTSyncCopyType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTSyncCopy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.tmem_load_red`. +pub const CopyAtomSM10xTmemLoadRedType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM10xTmemLoadRed; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.s2t_copy`. +pub const CopyAtomSM100CopyS2TType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM100CopyS2T; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.sm100_s2t_copy_v2`. +pub const CopyAtomSM100S2TCopyV2Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM100S2TCopyV2; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.tmem_load`. +pub const CopyAtomSM100TmemLoadType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM100TmemLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.tmem_store`. +pub const CopyAtomSM100TmemStoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM100TmemStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.stsm`. +pub const CopyAtomStsmType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomStsm; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomStsmTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomStsmTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.tma_load`. +pub const CopyAtomTmaLoadType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomTmaLoadTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.tma_reduce`. +pub const CopyAtomTmaReduceType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomTmaReduce; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomTmaReduceTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.atom.tma_store`. +pub const CopyAtomTmaStoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomTmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomTmaStoreTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.sm80.sparse_mma`. +pub const MmaAtomSM80SparseType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM80Sparse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM80SparseTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.sm80.mma`. +pub const MmaAtomSM80Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM80; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM80TypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.sm89.mma`. +pub const MmaAtomSM89Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM89; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM89TypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM89TypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.sm90.mma`. +pub const MmaAtomSM90Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM90; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM90TypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.sm100.mma_bs_sp`. +pub const MmaAtomSM100UMMABlockScaledSparseType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaledSparse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.sm100.mma_bs`. +pub const MmaAtomSM100UMMABlockScaledType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaled; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.sm100.mma_sp`. +pub const MmaAtomSM100UMMASparseType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM100UMMASparse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.sm100.mma`. +pub const MmaAtomSM100UMMAType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM100UMMA; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM100UMMATypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.SM120.mma_bs`. +pub const MmaAtomSM120BlockScaledType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM120BlockScaled; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.sm103.smem_desc_circular`. +pub const SmemDescCircularSM103Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDescCircularSM103; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSmemDescCircularSM103TypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUSmemDescCircularSM103TypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.smem_desc`. +pub const SmemDescType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDesc; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCuteNVGPUSmemDescTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cute_nvgpu.smem_desc_view`. +pub const SmemDescViewType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDescView; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSmemDescViewTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUSmemDescViewTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.tiled_copy`. +pub const TiledCopyType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUTiledCopy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUTiledCopyTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUTiledCopyTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.tiled_mma`. +pub const TiledMmaType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUTiledMma; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUTiledMmaTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUTiledMmaTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.tma_descriptor_im2col`. +pub const TmaDescriptorIm2ColType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUTmaDescriptorIm2Col; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCuteNVGPUTmaDescriptorIm2ColTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cute_nvgpu.tma_descriptor_tiled`. +pub const TmaDescriptorTiledType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUTmaDescriptorTiled; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCuteNVGPUTmaDescriptorTiledTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cute_nvgpu.atom.universal_fma`. +pub const UniversalFmaAtomType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUUniversalFmaAtom; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + payload: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUUniversalFmaAtomTypeGet(ctx.ptr(), args.payload.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getPayload(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUUniversalFmaAtomTypeGetPayload(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.workid_response`. +pub const WorkIdResponseType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUWorkIdResponse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCuteNVGPUWorkIdResponseTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `cute_nvgpu.arch.alloc_rmem`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_alloc_rmem(ctx: *mlir.Context, input: *const mlir.Value, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.alloc_rmem", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{ptr_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.alloc_smem`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_alloc_smem(ctx: *mlir.Context, ptr_type: *const mlir.Type, input: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.alloc_smem", .{ + .results = .{ .flat = &.{ptr_type} }, + .attributes = &.{ + .named(ctx, "input", input), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM90.bulk_copy_g2s`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM90_bulk_copy_g2s(ctx: *mlir.Context, gmem_data_addr: *const mlir.Value, dsmem_data_addr: *const mlir.Value, dsmem_bar_addr: *const mlir.Value, multicast_mask: ?*const mlir.Value, cache_policy: ?*const mlir.Value, size: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 5) = .empty; + operands.appendSliceAssumeCapacity(&.{gmem_data_addr}); + operands.appendSliceAssumeCapacity(&.{dsmem_data_addr}); + operands.appendSliceAssumeCapacity(&.{dsmem_bar_addr}); + if (multicast_mask) |value| operands.appendSliceAssumeCapacity(&.{value}); + if (cache_policy) |value| operands.appendSliceAssumeCapacity(&.{value}); + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM90.bulk_copy_g2s", .{ + .operands = .{ .flat = operands.constSlice() }, + .attributes = &.{ + .named(ctx, "size", size), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ + 1, + 1, + 1, + if (multicast_mask != null) 1 else 0, + if (cache_policy != null) 1 else 0, + })), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM90.bulk_copy_s2g`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM90_bulk_copy_s2g(ctx: *mlir.Context, smem_data_addr: *const mlir.Value, gmem_data_addr: *const mlir.Value, byte_mask: ?*const mlir.Value, cache_policy: ?*const mlir.Value, size: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 4) = .empty; + operands.appendSliceAssumeCapacity(&.{smem_data_addr}); + operands.appendSliceAssumeCapacity(&.{gmem_data_addr}); + if (byte_mask) |value| operands.appendSliceAssumeCapacity(&.{value}); + if (cache_policy) |value| operands.appendSliceAssumeCapacity(&.{value}); + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM90.bulk_copy_s2g", .{ + .operands = .{ .flat = operands.constSlice() }, + .attributes = &.{ + .named(ctx, "size", size), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ + 1, + 1, + if (byte_mask != null) 1 else 0, + if (cache_policy != null) 1 else 0, + })), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM90.bulk_copy_s2s`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM90_bulk_copy_s2s(ctx: *mlir.Context, smem_data_addr: *const mlir.Value, dsmem_data_addr: *const mlir.Value, dsmem_bar_addr: *const mlir.Value, cta_rank: *const mlir.Value, size: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM90.bulk_copy_s2s", .{ + .operands = .{ .flat = &.{ smem_data_addr, dsmem_data_addr, dsmem_bar_addr, cta_rank } }, + .attributes = &.{ + .named(ctx, "size", size), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM100.copy_s2t`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM100_copy_s2t(ctx: *mlir.Context, src: *const mlir.Value, dst: *const mlir.Value, dp: *const mlir.Attribute, bits: *const mlir.Attribute, cta: *const mlir.Attribute, broadcast: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.copy_s2t", .{ + .operands = .{ .flat = &.{ src, dst } }, + .attributes = &.{ + .named(ctx, "dp", dp), + .named(ctx, "bits", bits), + .named(ctx, "cta", cta), + .named(ctx, "broadcast", broadcast), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM100.tma_load`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM100_tma_load(ctx: *mlir.Context, src_desc: *const mlir.Value, dsmem_data_addr: *const mlir.Value, dsmem_bar_addr: *const mlir.Value, coord: []const *const mlir.Value, multicast_mask: ?*const mlir.Value, offsets: []const *const mlir.Value, cache_policy: ?*const mlir.Value, mode: *const mlir.Attribute, num_cta: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.tma_load", .{ + .operands = .{ .variadic = &.{ + &.{src_desc}, + &.{dsmem_data_addr}, + &.{dsmem_bar_addr}, + coord, + if (multicast_mask) |value| &.{value} else &.{}, + offsets, + if (cache_policy) |value| &.{value} else &.{}, + } }, + .attributes = &.{ + .named(ctx, "mode", mode), + .named(ctx, "num_cta", num_cta), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM100.tma_reduce`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM100_tma_reduce(ctx: *mlir.Context, dst_desc: *const mlir.Value, src_smem_data_addr: *const mlir.Value, coord: []const *const mlir.Value, cache_policy: ?*const mlir.Value, mode: *const mlir.Attribute, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.tma_reduce", .{ + .operands = .{ .variadic = &.{ + &.{dst_desc}, + &.{src_smem_data_addr}, + coord, + if (cache_policy) |value| &.{value} else &.{}, + } }, + .attributes = &.{ + .named(ctx, "mode", mode), + .named(ctx, "kind", kind), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM100.tma_store`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM100_tma_store(ctx: *mlir.Context, dst_desc: *const mlir.Value, src_smem_data_addr: *const mlir.Value, coord: []const *const mlir.Value, cache_policy: ?*const mlir.Value, mode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.tma_store", .{ + .operands = .{ .variadic = &.{ + &.{dst_desc}, + &.{src_smem_data_addr}, + coord, + if (cache_policy) |value| &.{value} else &.{}, + } }, + .attributes = &.{ + .named(ctx, "mode", mode), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM100.tmem_load`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM100_tmem_load(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, num_dp: *const mlir.Attribute, num_b: *const mlir.Attribute, num_rep: *const mlir.Attribute, pack_16: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "num_dp", num_dp)); + attributes.appendAssumeCapacity(.named(ctx, "num_b", num_b)); + attributes.appendAssumeCapacity(.named(ctx, "num_rep", num_rep)); + if (pack_16) |value| attributes.appendAssumeCapacity(.named(ctx, "pack_16", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.tmem_load", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM100.tmem_store`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM100_tmem_store(ctx: *mlir.Context, dst: *const mlir.Value, src: *const mlir.Value, num_dp: *const mlir.Attribute, num_b: *const mlir.Attribute, num_rep: *const mlir.Attribute, expand_16: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "num_dp", num_dp)); + attributes.appendAssumeCapacity(.named(ctx, "num_b", num_b)); + attributes.appendAssumeCapacity(.named(ctx, "num_rep", num_rep)); + if (expand_16) |value| attributes.appendAssumeCapacity(.named(ctx, "expand_16", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.tmem_store", .{ + .operands = .{ .flat = &.{ dst, src } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.arch.get_dyn_smem`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_get_dyn_smem(ctx: *mlir.Context, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.get_dyn_smem", .{ + .results = .{ .flat = &.{ptr_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.get_dyn_smem_size`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_get_dyn_smem_size(ctx: *mlir.Context, size_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.get_dyn_smem_size", .{ + .results = .{ .flat = &.{size_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.make_warp_uniform`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_make_warp_uniform(ctx: *mlir.Context, val: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.make_warp_uniform", .{ + .operands = .{ .flat = &.{val} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.mma.SM120.block_scaled`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_mma_SM120_block_scaled(ctx: *mlir.Context, opA: []const *const mlir.Value, opB: []const *const mlir.Value, opC: []const *const mlir.Value, sfA: *const mlir.Value, sfB: *const mlir.Value, byte_id_a: ?*const mlir.Value, byte_id_b: ?*const mlir.Value, res_types: []const *const mlir.Type, shape_MNK: *const mlir.Attribute, vec_size: *const mlir.Attribute, thread_id_a: ?*const mlir.Attribute, thread_id_b: ?*const mlir.Attribute, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, sf_type: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape_MNK", shape_MNK)); + attributes.appendAssumeCapacity(.named(ctx, "vec_size", vec_size)); + if (thread_id_a) |value| attributes.appendAssumeCapacity(.named(ctx, "thread_id_a", value)); + if (thread_id_b) |value| attributes.appendAssumeCapacity(.named(ctx, "thread_id_b", value)); + attributes.appendAssumeCapacity(.named(ctx, "a_type", a_type)); + attributes.appendAssumeCapacity(.named(ctx, "b_type", b_type)); + attributes.appendAssumeCapacity(.named(ctx, "sf_type", sf_type)); + return mlir.Operation.make(ctx, "cute_nvgpu.arch.mma.SM120.block_scaled", .{ + .operands = .{ .variadic = &.{ + opA, + opB, + opC, + &.{sfA}, + &.{sfB}, + if (byte_id_a) |value| &.{value} else &.{}, + if (byte_id_b) |value| &.{value} else &.{}, + } }, + .results = .{ .variadic = &.{res_types} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.arch.prefetch_tma_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_prefetch_tma_desc(ctx: *mlir.Context, tma_descriptor_ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.prefetch_tma_desc", .{ + .operands = .{ .flat = &.{tma_descriptor_ptr} }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.sm100.alloc_tmem`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_sm100_alloc_tmem(ctx: *mlir.Context, n_cols: *const mlir.Value, dst_ptr: *const mlir.Value, is_two_cta: ?*const mlir.Attribute, exclusive: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (is_two_cta) |value| attributes.appendAssumeCapacity(.named(ctx, "is_two_cta", value)); + if (exclusive) |value| attributes.appendAssumeCapacity(.named(ctx, "exclusive", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.arch.sm100.alloc_tmem", .{ + .operands = .{ .flat = &.{ n_cols, dst_ptr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.arch.sm100.dealloc_tmem`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_sm100_dealloc_tmem(ctx: *mlir.Context, tmem_ptr: *const mlir.Value, n_cols: *const mlir.Value, is_two_cta: ?*const mlir.Attribute, exclusive: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (is_two_cta) |value| attributes.appendAssumeCapacity(.named(ctx, "is_two_cta", value)); + if (exclusive) |value| attributes.appendAssumeCapacity(.named(ctx, "exclusive", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.arch.sm100.dealloc_tmem", .{ + .operands = .{ .flat = &.{ tmem_ptr, n_cols } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.arch.sm100.relinquish_tmem_alloc_permit`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_sm100_relinquish_tmem_alloc_permit(ctx: *mlir.Context, is_two_cta: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (is_two_cta) |value| attributes.appendAssumeCapacity(.named(ctx, "is_two_cta", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.arch.sm100.relinquish_tmem_alloc_permit", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.arch.sm100.retrieve_tmem_ptr`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_sm100_retrieve_tmem_ptr(ctx: *mlir.Context, smem_ptr: *const mlir.Value, tmem_ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.sm100.retrieve_tmem_ptr", .{ + .operands = .{ .flat = &.{smem_ptr} }, + .results = .{ .flat = &.{tmem_ptr_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.get_coord_tensor`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_get_coord_tensor(ctx: *mlir.Context, atom: *const mlir.Value, shape: *const mlir.Value, offset_b: []const *const mlir.Value, layout_b: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.get_coord_tensor", .{ + .operands = .{ .variadic = &.{ + &.{atom}, + &.{shape}, + offset_b, + layout_b, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.get_copy_s2t_smem_desc_view`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_get_copy_s2t_smem_desc_view(ctx: *mlir.Context, atom: *const mlir.Value, view: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.get_copy_s2t_smem_desc_view", .{ + .operands = .{ .flat = &.{ atom, view } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.get_value`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_get_value(ctx: *mlir.Context, atom: *const mlir.Value, result_type: *const mlir.Type, field: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.get_value", .{ + .operands = .{ .flat = &.{atom} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "field", field), + }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_exec_tma`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_exec_tma(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_exec_tma", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_non_exec_2d_gather4_tma_load`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_non_exec_2d_gather4_tma_load(ctx: *mlir.Context, gmem_tensor: *const mlir.Value, gmem_coord_layout: *const mlir.Value, smem_layout: *const mlir.Value, cta_v_map: *const mlir.Value, non_exec_atom_type: *const mlir.Type, tma_tensor_type: *const mlir.Type, kind: *const mlir.Attribute, num_multicast: ?*const mlir.Attribute, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_2d_gather4_tma_load", .{ + .operands = .{ .flat = &.{ gmem_tensor, gmem_coord_layout, smem_layout, cta_v_map } }, + .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_non_exec_im2col_tma_load`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_non_exec_im2col_tma_load(ctx: *mlir.Context, gmem_tensor: *const mlir.Value, smem_layout: *const mlir.Value, cta_v_map: *const mlir.Value, lower_corner_whd: *const mlir.Value, upper_corner_whd: *const mlir.Value, lower_padding_whd: *const mlir.Value, upper_padding_whd: *const mlir.Value, stride_whd: *const mlir.Value, lower_srt: *const mlir.Value, stride_srt: *const mlir.Value, non_exec_atom_type: *const mlir.Type, tma_tensor_type: *const mlir.Type, kind: *const mlir.Attribute, num_multicast: ?*const mlir.Attribute, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_im2col_tma_load", .{ + .operands = .{ .flat = &.{ gmem_tensor, smem_layout, cta_v_map, lower_corner_whd, upper_corner_whd, lower_padding_whd, upper_padding_whd, stride_whd, lower_srt, stride_srt } }, + .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_non_exec_im2col_tma_store`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_non_exec_im2col_tma_store(ctx: *mlir.Context, gmem_tensor: *const mlir.Value, smem_layout: *const mlir.Value, cta_v_map: *const mlir.Value, non_exec_atom_type: *const mlir.Type, tma_tensor_type: *const mlir.Type, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_im2col_tma_store", .{ + .operands = .{ .flat = &.{ gmem_tensor, smem_layout, cta_v_map } }, + .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_non_exec_tiled_tma_load`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_non_exec_tiled_tma_load(ctx: *mlir.Context, gmem_tensor: *const mlir.Value, smem_layout: *const mlir.Value, cta_v_map: *const mlir.Value, non_exec_atom_type: *const mlir.Type, tma_tensor_type: *const mlir.Type, kind: *const mlir.Attribute, num_multicast: ?*const mlir.Attribute, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_tiled_tma_load", .{ + .operands = .{ .flat = &.{ gmem_tensor, smem_layout, cta_v_map } }, + .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_non_exec_tiled_tma_reduce`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_non_exec_tiled_tma_reduce(ctx: *mlir.Context, gmem_tensor: *const mlir.Value, smem_layout: *const mlir.Value, cta_v_map: *const mlir.Value, non_exec_atom_type: *const mlir.Type, tma_tensor_type: *const mlir.Type, kind: *const mlir.Attribute, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_tiled_tma_reduce", .{ + .operands = .{ .flat = &.{ gmem_tensor, smem_layout, cta_v_map } }, + .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_non_exec_tiled_tma_store`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_non_exec_tiled_tma_store(ctx: *mlir.Context, gmem_tensor: *const mlir.Value, smem_layout: *const mlir.Value, cta_v_map: *const mlir.Value, non_exec_atom_type: *const mlir.Type, tma_tensor_type: *const mlir.Type, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_tiled_tma_store", .{ + .operands = .{ .flat = &.{ gmem_tensor, smem_layout, cta_v_map } }, + .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_s2t_copy`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_s2t_copy(ctx: *mlir.Context, copy_s2t_atom: *const mlir.Value, tmem_memref: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_s2t_copy", .{ + .operands = .{ .flat = &.{ copy_s2t_atom, tmem_memref } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_tmem_copy`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_tmem_copy(ctx: *mlir.Context, atom: *const mlir.Value, tmem_memref: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_tmem_copy", .{ + .operands = .{ .flat = &.{ atom, tmem_memref } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.set_value`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_set_value(ctx: *mlir.Context, atom: *const mlir.Value, value_: *const mlir.Value, result_type: *const mlir.Type, field: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.set_value", .{ + .operands = .{ .flat = &.{ atom, value_ } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "field", field), + }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.tma_partition`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_tma_partition(ctx: *mlir.Context, tma_atom: *const mlir.Value, cta_coord: *const mlir.Value, cta_layout: *const mlir.Value, smem_tensor: *const mlir.Value, target_tensors: []const *const mlir.Value, res_smem_tensor_type: *const mlir.Type, res_target_tensors_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.tma_partition", .{ + .operands = .{ .variadic = &.{ + &.{tma_atom}, + &.{cta_coord}, + &.{cta_layout}, + &.{smem_tensor}, + target_tensors, + } }, + .results = .{ .variadic = &.{ + &.{res_smem_tensor_type}, + res_target_tensors_types, + } }, + .location = location, + }); +} + +/// `cute_nvgpu.cast_tma_desc_to_integer`. Result types are explicit; attributes use mlir.Attribute. +pub fn cast_tma_desc_to_integer(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.cast_tma_desc_to_integer", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.copy_tma_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn copy_tma_desc(ctx: *mlir.Context, atom: *const mlir.Value, tma_desc_ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.copy_tma_desc", .{ + .operands = .{ .flat = &.{ atom, tma_desc_ptr } }, + .location = location, + }); +} + +/// `cute_nvgpu.get_grid_constant_pointer`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_grid_constant_pointer(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.get_grid_constant_pointer", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.get_tma_desc_addr`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_tma_desc_addr(ctx: *mlir.Context, src: *const mlir.Value, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.get_tma_desc_addr", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{ptr_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.make_gmma_smem_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_gmma_smem_desc(ctx: *mlir.Context, src: *const mlir.Value, res_type: *const mlir.Type, layout: *const mlir.Attribute, major: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.make_gmma_smem_desc", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "layout", layout), + .named(ctx, "major", major), + }, + .location = location, + }); +} + +/// `cute_nvgpu.make_tma_desc_im2col_at`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tma_desc_im2col_at(ctx: *mlir.Context, gmem_view: *const mlir.Value, smem_layout: *const mlir.Value, cta_value_tile: *const mlir.Value, lower_corner_whd: *const mlir.Value, upper_corner_whd: *const mlir.Value, lower_padding_whd: *const mlir.Value, upper_padding_whd: *const mlir.Value, stride_whd: *const mlir.Value, lower_srt: *const mlir.Value, stride_srt: *const mlir.Value, tma_desc_addr: *const mlir.Value, num_multicast: ?*const mlir.Attribute, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.make_tma_desc_im2col_at", .{ + .operands = .{ .flat = &.{ gmem_view, smem_layout, cta_value_tile, lower_corner_whd, upper_corner_whd, lower_padding_whd, upper_padding_whd, stride_whd, lower_srt, stride_srt, tma_desc_addr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.make_tma_desc_im2col`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tma_desc_im2col(ctx: *mlir.Context, gmem_view: *const mlir.Value, smem_layout: *const mlir.Value, cta_value_tile: *const mlir.Value, lower_corner_whd: *const mlir.Value, upper_corner_whd: *const mlir.Value, lower_padding_whd: *const mlir.Value, upper_padding_whd: *const mlir.Value, stride_whd: *const mlir.Value, lower_srt: *const mlir.Value, stride_srt: *const mlir.Value, result_type: *const mlir.Type, num_multicast: ?*const mlir.Attribute, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.make_tma_desc_im2col", .{ + .operands = .{ .flat = &.{ gmem_view, smem_layout, cta_value_tile, lower_corner_whd, upper_corner_whd, lower_padding_whd, upper_padding_whd, stride_whd, lower_srt, stride_srt } }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.make_tma_desc_tiled_at`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tma_desc_tiled_at(ctx: *mlir.Context, gmem_view: *const mlir.Value, smem_layout: *const mlir.Value, cta_value_tile: *const mlir.Value, tma_desc_addr: *const mlir.Value, traversal_stride: ?*const mlir.Value, mode: *const mlir.Attribute, num_multicast: ?*const mlir.Attribute, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 5) = .empty; + operands.appendSliceAssumeCapacity(&.{gmem_view}); + operands.appendSliceAssumeCapacity(&.{smem_layout}); + operands.appendSliceAssumeCapacity(&.{cta_value_tile}); + operands.appendSliceAssumeCapacity(&.{tma_desc_addr}); + if (traversal_stride) |value| operands.appendSliceAssumeCapacity(&.{value}); + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "mode", mode)); + if (num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.make_tma_desc_tiled_at", .{ + .operands = .{ .flat = operands.constSlice() }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.make_tma_desc_tiled`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tma_desc_tiled(ctx: *mlir.Context, gmem_view: *const mlir.Value, smem_layout: *const mlir.Value, cta_value_tile: *const mlir.Value, traversal_stride: ?*const mlir.Value, result_type: *const mlir.Type, mode: *const mlir.Attribute, num_multicast: ?*const mlir.Attribute, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 4) = .empty; + operands.appendSliceAssumeCapacity(&.{gmem_view}); + operands.appendSliceAssumeCapacity(&.{smem_layout}); + operands.appendSliceAssumeCapacity(&.{cta_value_tile}); + if (traversal_stride) |value| operands.appendSliceAssumeCapacity(&.{value}); + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "mode", mode)); + if (num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.make_tma_desc_tiled", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.make_umma_smem_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_umma_smem_desc(ctx: *mlir.Context, src: *const mlir.Value, nextSrc: ?*const mlir.Value, res_type: *const mlir.Type, layout: *const mlir.Attribute, major: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 2) = .empty; + operands.appendSliceAssumeCapacity(&.{src}); + if (nextSrc) |value| operands.appendSliceAssumeCapacity(&.{value}); + return mlir.Operation.make(ctx, "cute_nvgpu.make_umma_smem_desc", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "layout", layout), + .named(ctx, "major", major), + }, + .location = location, + }); +} + +/// `cute_nvgpu.prefetch_tma_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn prefetch_tma_desc(ctx: *mlir.Context, atom: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.prefetch_tma_desc", .{ + .operands = .{ .flat = &.{atom} }, + .location = location, + }); +} + +/// `cute_nvgpu.sm103.make_umma_smem_desc_circular`. Result types are explicit; attributes use mlir.Attribute. +pub fn sm103_make_umma_smem_desc_circular(ctx: *mlir.Context, src: *const mlir.Value, nextSrc: ?*const mlir.Value, res_type: *const mlir.Type, layout: *const mlir.Attribute, major: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 2) = .empty; + operands.appendSliceAssumeCapacity(&.{src}); + if (nextSrc) |value| operands.appendSliceAssumeCapacity(&.{value}); + return mlir.Operation.make(ctx, "cute_nvgpu.sm103.make_umma_smem_desc_circular", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "layout", layout), + .named(ctx, "major", major), + }, + .location = location, + }); +} + +/// `cute_nvgpu.update_tma_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn update_tma_desc(ctx: *mlir.Context, tma_atom: *const mlir.Value, gmem_tensor: *const mlir.Value, tma_desc_ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.update_tma_desc", .{ + .operands = .{ .flat = &.{ tma_atom, gmem_tensor, tma_desc_ptr } }, + .location = location, + }); +} + +pub const operation_names: []const []const u8 = &.{ + "cute_nvgpu.arch.alloc_rmem", + "cute_nvgpu.arch.alloc_smem", + "cute_nvgpu.arch.copy.SM90.bulk_copy_g2s", + "cute_nvgpu.arch.copy.SM90.bulk_copy_s2g", + "cute_nvgpu.arch.copy.SM90.bulk_copy_s2s", + "cute_nvgpu.arch.copy.SM100.copy_s2t", + "cute_nvgpu.arch.copy.SM100.tma_load", + "cute_nvgpu.arch.copy.SM100.tma_reduce", + "cute_nvgpu.arch.copy.SM100.tma_store", + "cute_nvgpu.arch.copy.SM100.tmem_load", + "cute_nvgpu.arch.copy.SM100.tmem_store", + "cute_nvgpu.arch.get_dyn_smem", + "cute_nvgpu.arch.get_dyn_smem_size", + "cute_nvgpu.arch.make_warp_uniform", + "cute_nvgpu.arch.mma.SM120.block_scaled", + "cute_nvgpu.arch.prefetch_tma_desc", + "cute_nvgpu.arch.sm100.alloc_tmem", + "cute_nvgpu.arch.sm100.dealloc_tmem", + "cute_nvgpu.arch.sm100.relinquish_tmem_alloc_permit", + "cute_nvgpu.arch.sm100.retrieve_tmem_ptr", + "cute_nvgpu.atom.get_coord_tensor", + "cute_nvgpu.atom.get_copy_s2t_smem_desc_view", + "cute_nvgpu.atom.get_value", + "cute_nvgpu.atom.make_exec_tma", + "cute_nvgpu.atom.make_non_exec_2d_gather4_tma_load", + "cute_nvgpu.atom.make_non_exec_im2col_tma_load", + "cute_nvgpu.atom.make_non_exec_im2col_tma_store", + "cute_nvgpu.atom.make_non_exec_tiled_tma_load", + "cute_nvgpu.atom.make_non_exec_tiled_tma_reduce", + "cute_nvgpu.atom.make_non_exec_tiled_tma_store", + "cute_nvgpu.atom.make_s2t_copy", + "cute_nvgpu.atom.make_tmem_copy", + "cute_nvgpu.atom.set_value", + "cute_nvgpu.atom.tma_partition", + "cute_nvgpu.cast_tma_desc_to_integer", + "cute_nvgpu.copy_tma_desc", + "cute_nvgpu.get_grid_constant_pointer", + "cute_nvgpu.get_tma_desc_addr", + "cute_nvgpu.make_gmma_smem_desc", + "cute_nvgpu.make_tma_desc_im2col_at", + "cute_nvgpu.make_tma_desc_im2col", + "cute_nvgpu.make_tma_desc_tiled_at", + "cute_nvgpu.make_tma_desc_tiled", + "cute_nvgpu.make_umma_smem_desc", + "cute_nvgpu.prefetch_tma_desc", + "cute_nvgpu.sm103.make_umma_smem_desc_circular", + "cute_nvgpu.update_tma_desc", +}; diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/BUILD.bazel b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/BUILD.bazel new file mode 100644 index 00000000..02869c71 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "cute", + hdrs = [ + "Cute.h", + "CuteAttributes.h", + "CuteTypes.h", + ], + includes = ["../.."], + deps = ["@llvm-project//mlir:CAPIIR"], +) + +cc_library( + name = "cute_nvgpu", + hdrs = [ + "CuteNVGPU.h", + "CuteNVGPUTypes.h", + ], + includes = ["../.."], + deps = ["@llvm-project//mlir:CAPIIR"], +) diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/Cute.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/Cute.h new file mode 100644 index 00000000..bc2f11a1 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/Cute.h @@ -0,0 +1,23 @@ +//===- Cute.h - C API for the CuTe dialect ----------------------*- C -*-===// + +#ifndef CUTE_IR_C_DIALECT_CUTE_H +#define CUTE_IR_C_DIALECT_CUTE_H + +#include "cute_ir-c/Dialect/CuteAttributes.h" +#include "cute_ir-c/Dialect/CuteTypes.h" +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Cute, cute); + +/// Returns true when `type` belongs to the CuTe dialect. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteType(MlirType type); + +#ifdef __cplusplus +} +#endif + +#endif // CUTE_IR_C_DIALECT_CUTE_H diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteAttributes.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteAttributes.h new file mode 100644 index 00000000..6aecc8cb --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteAttributes.h @@ -0,0 +1,65 @@ +// C API for CuTe algebra attributes. + +#ifndef CUTE_IR_C_DIALECT_CUTE_ATTRIBUTES_H +#define CUTE_IR_C_DIALECT_CUTE_ATTRIBUTES_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Constructors accept raw algebra expressions, not MLIR attribute assembly. +// For example, LayoutAttrGet accepts "(16,32):(32,1)". Invalid expressions +// return a null attribute. GetValue returns canonical algebra text owned by +// the attribute's context; callers must not free it. Getters require an +// attribute accepted by the corresponding IsA predicate. + +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteIntTupleAttrGet(MlirContext context, + MlirStringRef value); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteIntTuple(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirCuteIntTupleAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCoordAttrGet(MlirContext context, + MlirStringRef value); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteCoord(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirCuteCoordAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteShapeAttrGet(MlirContext context, + MlirStringRef value); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirCuteShapeAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteStrideAttrGet(MlirContext context, + MlirStringRef value); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteStride(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirCuteStrideAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteLayoutAttrGet(MlirContext context, + MlirStringRef value); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirCuteLayoutAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteTileAttrGet(MlirContext context, + MlirStringRef value); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteTile(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirCuteTileAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteComposedLayoutAttrGet(MlirContext context, MlirStringRef value); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteComposedLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirCuteComposedLayoutAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteSwizzleAttrGet(MlirContext context, + MlirStringRef value); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteSwizzle(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirCuteSwizzleAttrGetValue(MlirAttribute attr); + +#ifdef __cplusplus +} +#endif + +#endif // CUTE_IR_C_DIALECT_CUTE_ATTRIBUTES_H diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPU.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPU.h new file mode 100644 index 00000000..6e883f3d --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPU.h @@ -0,0 +1,21 @@ +//===- CuteNVGPU.h - C API for the CuTe NVIDIA GPU dialect ------*- C -*-===// + +#ifndef CUTE_IR_C_DIALECT_CUTENVGPU_H +#define CUTE_IR_C_DIALECT_CUTENVGPU_H + +#include "cute_ir-c/Dialect/CuteNVGPUTypes.h" +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(CuteNVGPU, cute_nvgpu); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUType(MlirType type); + +#ifdef __cplusplus +} +#endif + +#endif // CUTE_IR_C_DIALECT_CUTENVGPU_H diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPUTypes.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPUTypes.h new file mode 100644 index 00000000..44c26840 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPUTypes.h @@ -0,0 +1,323 @@ +// C API for CuTe NVIDIA GPU types. +#ifndef CUTE_IR_C_DIALECT_CUTENVGPU_TYPES_H +#define CUTE_IR_C_DIALECT_CUTENVGPU_TYPES_H +#include "mlir-c/IR.h" +#ifdef __cplusplus +extern "C" { +#endif + +// Constructors return a null type for invalid parameters. All handles must +// belong to context. Getters require a type accepted by the matching IsA. + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyG2S(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomBulkCopyG2STypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2G(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2S(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomBulkCopyS2STypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomDsmemStore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomDsmemStoreTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomDsmemStoreTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomG2R(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomG2RTypeGet(MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomG2RTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaLoad(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaStore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomLdsm(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomLdsmTypeGet(MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomLdsmTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomNonExec2DGather4TmaLoad(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaLoad(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaStore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaLoad(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaReduce(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaStore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomR2G(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomR2GTypeGet(MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomR2GTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomR2S(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomR2STypeGet(MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomR2STypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomS2R(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomS2RTypeGet(MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomS2RTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSIMTAsyncCopy(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemLdReduce(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemRed(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemSt(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomSIMTSyncCopy(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSM10xTmemLoadRed(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomSM100CopyS2T(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSM100S2TCopyV2(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSM100TmemLoad(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSM100TmemStore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomStsm(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomStsmTypeGet(MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomStsmTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomTmaLoad(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomTmaLoadTypeGet(MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomTmaLoadTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomTmaReduce(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomTmaReduceTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomTmaReduceTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomTmaStore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomTmaStoreTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomTmaStoreTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM80Sparse(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM80SparseTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM80SparseTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM80(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM80TypeGet(MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM80TypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM89(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM89TypeGet(MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM89TypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM90(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM90TypeGet(MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM90TypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaledSparse(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGet(MlirContext context, + MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaled(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUMmaAtomSM100UMMASparse(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMA(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM100UMMATypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMATypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUMmaAtomSM120BlockScaled(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUSmemDescCircularSM103(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUSmemDescCircularSM103TypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUSmemDescCircularSM103TypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUSmemDesc(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUSmemDescTypeGet(MlirContext context); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUSmemDescView(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUSmemDescViewTypeGet(MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUSmemDescViewTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUTiledCopy(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUTiledCopyTypeGet(MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTiledCopyTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUTiledMma(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUTiledMmaTypeGet(MlirContext context, + MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTiledMmaTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUTmaDescriptorIm2Col(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUTmaDescriptorIm2ColTypeGet(MlirContext context); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUTmaDescriptorTiled(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUTmaDescriptorTiledTypeGet(MlirContext context); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUUniversalFmaAtom(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUUniversalFmaAtomTypeGet( + MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUUniversalFmaAtomTypeGetPayload(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUWorkIdResponse(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUWorkIdResponseTypeGet(MlirContext context); + +#ifdef __cplusplus +} +#endif + +#endif // CUTE_IR_C_DIALECT_CUTENVGPU_TYPES_H diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteTypes.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteTypes.h new file mode 100644 index 00000000..4e22f193 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteTypes.h @@ -0,0 +1,115 @@ +// C API for CuTe types. +#ifndef CUTE_IR_C_DIALECT_CUTE_TYPES_H +#define CUTE_IR_C_DIALECT_CUTE_TYPES_H +#include "mlir-c/IR.h" +#ifdef __cplusplus +extern "C" { +#endif + +// Constructors return a null type for invalid parameters. All handles must +// belong to context. Getters require a type accepted by the matching IsA. + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteArithTupleIterator(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteArithTupleIteratorTypeGet(MlirContext context, MlirType arithTuple); +MLIR_CAPI_EXPORTED MlirType +mlirCuteArithTupleIteratorTypeGetArithTuple(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteComposedLayout(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteComposedLayoutTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteComposedLayoutTypeGetAttr(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteConstrainedInt32(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteConstrainedInt32TypeGet(MlirContext context, uint64_t divisibleBy); +MLIR_CAPI_EXPORTED uint64_t +mlirCuteConstrainedInt32TypeGetDivisibleBy(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteConstrainedInt64(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteConstrainedInt64TypeGet(MlirContext context, uint64_t divisibleBy); +MLIR_CAPI_EXPORTED uint64_t +mlirCuteConstrainedInt64TypeGetDivisibleBy(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteCoordTensor(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTensorTypeGet(MlirContext context, + MlirType arithTuple, + MlirType layout); +MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTensorTypeGetArithTuple(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTensorTypeGetLayout(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteCoord(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCoordTypeGetAttr(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteFastDivmodDivisor(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteFastDivmodDivisorTypeGet( + MlirContext context, unsigned width, bool isPow2); +MLIR_CAPI_EXPORTED unsigned +mlirCuteFastDivmodDivisorTypeGetWidth(MlirType type); +MLIR_CAPI_EXPORTED bool mlirCuteFastDivmodDivisorTypeGetIsPow2(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteIntTuple(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteIntTupleTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteIntTupleTypeGetAttr(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteLayout(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteLayoutTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteLayoutTypeGetAttr(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteMemRef(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteMemRefTypeGet(MlirContext context, + MlirType ptr, + MlirType layout); +MLIR_CAPI_EXPORTED MlirType mlirCuteMemRefTypeGetPtr(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteMemRefTypeGetLayout(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACutePtr(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCutePtrTypeGet(MlirContext context, + MlirType valueType, + MlirAttribute memorySpace, + uint64_t alignment, + MlirAttribute swizzle); +MLIR_CAPI_EXPORTED MlirType mlirCutePtrTypeGetValueType(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirCutePtrTypeGetMemorySpace(MlirType type); +MLIR_CAPI_EXPORTED uint64_t mlirCutePtrTypeGetAlignment(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirCutePtrTypeGetSwizzle(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteShape(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteShapeTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteShapeTypeGetAttr(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteSparseElem(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteSparseElemTypeGet(MlirContext context, + int numLogical, + MlirType physicalType); +MLIR_CAPI_EXPORTED int mlirCuteSparseElemTypeGetNumLogical(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteSparseElemTypeGetPhysicalType(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteStride(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteStrideTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteStrideTypeGetAttr(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteSwizzle(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteSwizzleTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteSwizzleTypeGetAttr(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTile(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteTileTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteTileTypeGetAttr(MlirType type); + +#ifdef __cplusplus +} +#endif + +#endif // CUTE_IR_C_DIALECT_CUTE_TYPES_H diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/BUILD.bazel b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/BUILD.bazel new file mode 100644 index 00000000..ba3f3754 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/BUILD.bazel @@ -0,0 +1,92 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:defs.bzl", "cc_library") + +package(default_visibility = ["//visibility:public"]) + +td_library( + name = "private_tabledefs", + srcs = [ + "CuteOpsPrivate.td", + "CuteTypesAll.td", + "CuteTypesPrivate.td", + ], + includes = ["/mlir/dialects/cute_ir/include"], + deps = [ + "@cute_ir//:dialect_td", + "@llvm-project//mlir:BuiltinDialectTdFiles", + "@llvm-project//mlir:DataLayoutInterfacesTdFiles", + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "ops_private_inc_gen", + includes = ["../../../.."], + tbl_outs = [ + ( + ["-gen-op-decls"], + "CuteOpsPrivate.h.inc", + ), + ( + ["-gen-op-defs"], + "CuteOpsPrivate.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteOpsPrivate.td", + deps = [":private_tabledefs"], +) + +gentbl_cc_library( + name = "types_private_decl_inc_gen", + includes = ["../../../.."], + tbl_outs = [ + ( + [ + "-gen-typedef-decls", + "-typedefs-dialect=cute", + ], + "CuteTypesPrivate.h.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteTypesPrivate.td", + deps = [":private_tabledefs"], +) + +gentbl_cc_library( + name = "types_all_defs_inc_gen", + includes = ["../../../.."], + tbl_outs = [ + ( + [ + "-gen-typedef-defs", + "-typedefs-dialect=cute", + ], + "CuteTypes.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteTypesAll.td", + deps = [":private_tabledefs"], +) + +cc_library( + name = "private_headers", + hdrs = [ + "CuteDialectPrivate.h", + ], + includes = ["../../../.."], + deps = [ + ":ops_private_inc_gen", + ":types_private_decl_inc_gen", + "@cute_ir//:dialect_headers", + ], +) + +alias( + name = "dialect", + actual = ":private_headers", +) diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteDialectPrivate.h b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteDialectPrivate.h new file mode 100644 index 00000000..afe8682d --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteDialectPrivate.h @@ -0,0 +1,14 @@ +//===- CuteDialectPrivate.h - recovered CuTe declarations -----*- C++ -*-===// + +#ifndef CUTE_IR_DIALECT_CUTE_IR_CUTE_DIALECT_PRIVATE_H +#define CUTE_IR_DIALECT_CUTE_IR_CUTE_DIALECT_PRIVATE_H + +#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" + +#define GET_TYPEDEF_CLASSES +#include "cute_ir/Dialect/Cute/IR/CuteTypesPrivate.h.inc" + +#define GET_OP_CLASSES +#include "cute_ir/Dialect/Cute/IR/CuteOpsPrivate.h.inc" + +#endif // CUTE_IR_DIALECT_CUTE_IR_CUTE_DIALECT_PRIVATE_H diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOpsPrivate.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOpsPrivate.td new file mode 100644 index 00000000..e6e5c761 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOpsPrivate.td @@ -0,0 +1,428 @@ +//===- CutePrivateCompatOps.td - proprietary CuTe op schemas -*- tablegen -*-===// + +#ifndef CUTE_IR_DIALECT_CUTE_IR_CUTE_PRIVATE_COMPAT_OPS_TD +#define CUTE_IR_DIALECT_CUTE_IR_CUTE_PRIVATE_COMPAT_OPS_TD + +include "cute_ir/Dialect/Cute/IR/CuteAttrs.td" + +class Cute_PrivateSchemaOp traits = []> + : Cute_Op { + let summary = "Operation recovered from the CuTe-DSL operation registry"; +} + +def Cute_PrivateAddOffsetOp : Cute_PrivateSchemaOp<"add_offset"> { + let arguments = (ins AnyType:$src, AnyType:$offset); + let results = (outs AnyType:$dst); +} + +def Cute_PrivateApplySwizzleOp : Cute_PrivateSchemaOp<"apply_swizzle"> { + let arguments = (ins AnyType:$ptr, OptionalAttr:$swizzle); + let results = (outs AnyType:$swizzled_ptr); +} + +def Cute_PrivateAssumeOp : Cute_PrivateSchemaOp<"assume"> { + let arguments = (ins AnyType:$src); + let results = (outs AnyType:$dst); +} + +def Cute_PrivateCopyOp + : Cute_PrivateSchemaOp<"copy", [AttrSizedOperandSegments]> { + let arguments = (ins AnyType:$copy_atom, Variadic:$src, + Variadic:$dst, Optional:$pred); + let results = (outs); +} + +def Cute_PrivateCopyMakeFragmentOp + : Cute_PrivateSchemaOp<"copy.make_fragment"> { + let arguments = (ins AnyType:$atom, AnyType:$input, I32Attr:$operand_id); + let results = (outs AnyType:$result); +} + +def Cute_PrivateCopyAtomCallOp + : Cute_PrivateSchemaOp<"copy_atom_call", [AttrSizedOperandSegments]> { + let arguments = (ins AnyType:$atom, Variadic:$src, + Variadic:$dst, Optional:$pred); + let results = (outs); +} + +def Cute_PrivateDerefArithTupleIterOp + : Cute_PrivateSchemaOp<"deref_arith_tuple_iter"> { + let arguments = (ins AnyType:$iter); + let results = (outs AnyType:$arith_tuple); +} + +def Cute_PrivateDerefDescIterOp : Cute_PrivateSchemaOp<"deref_desc_iter"> { + let arguments = (ins AnyType:$iter); + let results = (outs AnyType:$value); +} + +def Cute_PrivateDerefineOp : Cute_PrivateSchemaOp<"derefine"> { + let arguments = (ins AnyType:$src); + let results = (outs AnyType:$dst); +} + +def Cute_PrivateFastDivmodComputeOp + : Cute_PrivateSchemaOp<"fast_divmod.compute"> { + let arguments = (ins AnyType:$dividend, AnyType:$divisor); + let results = (outs AnyType:$quotient, AnyType:$remainder); +} + +def Cute_PrivateFastDivmodCreateDivisorOp + : Cute_PrivateSchemaOp<"fast_divmod.create_divisor"> { + let arguments = (ins AnyType:$divisor); + let results = (outs AnyType:$fast_divmod_divisor); +} + +def Cute_PrivateFastDivmodDivideOp + : Cute_PrivateSchemaOp<"fast_divmod.divide"> { + let arguments = (ins AnyType:$dividend, AnyType:$divisor); + let results = (outs AnyType:$quotient); +} + +def Cute_PrivateFastDivmodGetAuxOp + : Cute_PrivateSchemaOp<"fast_divmod.get_aux"> { + let arguments = (ins AnyType:$divisor); + let results = (outs AnyType:$multiplier, AnyInteger:$shift1, + AnyInteger:$shift2); +} + +def Cute_PrivateFastDivmodGetDivisorOp + : Cute_PrivateSchemaOp<"fast_divmod.get_divisor"> { + let arguments = (ins AnyType:$fast_divmod_divisor); + let results = (outs AnyType:$divisor); +} + +def Cute_PrivateFastDivmodMakeDivisorOp + : Cute_PrivateSchemaOp<"fast_divmod.make_divisor"> { + let arguments = (ins AnyType:$divisor, AnyType:$multiplier, + AnyType:$sh1, AnyType:$sh2); + let results = (outs AnyType:$fast_divmod_divisor); +} + +def Cute_PrivateFilterOp : Cute_PrivateSchemaOp<"filter"> { + let arguments = (ins AnyType:$input); + let results = (outs AnyType:$result); +} + +def Cute_PrivateFilterZerosOp : Cute_PrivateSchemaOp<"filter_zeros"> { + let arguments = (ins AnyType:$input, Optional:$target_profile); + let results = (outs AnyType:$res); +} + +def Cute_PrivateGemmOp + : Cute_PrivateSchemaOp<"gemm", [AttrSizedOperandSegments]> { + let arguments = (ins AnyType:$mma_atom, AnyType:$d, Variadic:$a, + Variadic:$b, AnyType:$c); + let results = (outs); +} + +def Cute_PrivateGetFlatCoordOp : Cute_PrivateSchemaOp<"get_flat_coord"> { + let arguments = (ins AnyType:$index, AnyType:$input); + let results = (outs AnyType:$result); +} + +def Cute_PrivateGetHierCoordOp : Cute_PrivateSchemaOp<"get_hier_coord"> { + let arguments = (ins AnyType:$index, AnyType:$input); + let results = (outs AnyType:$result); +} + +def Cute_PrivateGetIntegralCoordOp + : Cute_PrivateSchemaOp<"get_integral_coord"> { + let arguments = (ins AnyType:$index, AnyType:$input); + let results = (outs AnyType:$result); +} + +def Cute_PrivateGetIterOp : Cute_PrivateSchemaOp<"get_iter"> { + let arguments = (ins AnyType:$source); + let results = (outs AnyType:$ptr); +} + +def Cute_PrivateGetLayoutOp : Cute_PrivateSchemaOp<"get_layout"> { + let arguments = (ins AnyType:$input); + let results = (outs AnyType:$layout); +} + +def Cute_PrivateIntToPtrOp : Cute_PrivateSchemaOp<"inttoptr"> { + let arguments = (ins AnyType:$src); + let results = (outs AnyType:$dst); +} + +def Cute_PrivateKernelSmemSizeOp : Cute_PrivateSchemaOp<"kernel_smem_size"> { + let arguments = (ins SymbolRefAttr:$kernel_name); + let results = (outs I64:$size); +} + +def Cute_PrivateLoadScaledIndexOp + : Cute_PrivateSchemaOp<"load_scaled_index"> { + let arguments = (ins AnyType:$ptr_a, AnyType:$ptr_b, AnyType:$index, + AnyType:$stride); + let results = (outs AnyType:$result); +} + +def Cute_PrivateLocalPartitionOp : Cute_PrivateSchemaOp<"local_partition"> { + let arguments = (ins AnyType:$input, AnyType:$tiler, AnyType:$index); + let results = (outs AnyType:$result); +} + +def Cute_PrivateLocalTileOp : Cute_PrivateSchemaOp<"local_tile"> { + let arguments = (ins AnyType:$input, AnyType:$tile, AnyType:$coord, + OptionalAttr:$proj); + let results = (outs AnyType:$result); +} + +def Cute_PrivateMakeArithTupleIterOp + : Cute_PrivateSchemaOp<"make_arith_tuple_iter"> { + let arguments = (ins Optional:$value); + let results = (outs AnyType:$iter); +} + +def Cute_PrivateMakeAtomOp : Cute_PrivateSchemaOp<"make_atom"> { + let arguments = (ins Variadic:$values); + let results = (outs AnyType:$atom); +} + +def Cute_PrivateMakeDescIterOp : Cute_PrivateSchemaOp<"make_desc_iter"> { + let arguments = (ins AnyType:$value); + let results = (outs AnyType:$iter); +} + +def Cute_PrivateMakeFragmentLikeOp + : Cute_PrivateSchemaOp<"make_fragment_like"> { + let arguments = (ins AnyType:$src); + let results = (outs AnyType:$dst); +} + +def Cute_PrivateMakeIdentityTensorOp + : Cute_PrivateSchemaOp<"make_identity_tensor"> { + let arguments = (ins AnyType:$shape); + let results = (outs AnyType:$result); +} + +def Cute_PrivateMakeSparseElemOp : Cute_PrivateSchemaOp<"make_sparse_elem"> { + let arguments = (ins AnyType:$physical_storage); + let results = (outs AnyType:$result); +} + +def Cute_PrivateMakeTiledCopyOp : Cute_PrivateSchemaOp<"make_tiled_copy"> { + let arguments = (ins AnyType:$atom); + let results = (outs AnyType:$result); +} + +def Cute_PrivateMakeTiledCopyV2Op + : Cute_PrivateSchemaOp<"make_tiled_copy_v2"> { + let arguments = (ins AnyType:$atom); + let results = (outs AnyType:$result); +} + +def Cute_PrivateMakeTiledMmaOp : Cute_PrivateSchemaOp<"make_tiled_mma"> { + let arguments = (ins AnyType:$atom); + let results = (outs AnyType:$result); +} + +def Cute_PrivateMakeTupleOp : Cute_PrivateSchemaOp<"make_tuple"> { + let arguments = (ins Variadic:$values); + let results = (outs AnyType:$result); +} + +def Cute_PrivateMakeViewOp : Cute_PrivateSchemaOp<"make_view"> { + let arguments = (ins AnyType:$iter, Optional:$layout); + let results = (outs AnyType:$result); +} + +def Cute_PrivateMemrefAllocSmemOp + : Cute_PrivateSchemaOp<"memref.alloc_smem"> { + let arguments = (ins); + let results = (outs AnyType:$memref); +} + +def Cute_PrivateMemrefAllocaOp : Cute_PrivateSchemaOp<"memref.alloca"> { + let arguments = (ins Optional:$layout); + let results = (outs AnyType:$result); +} + +def Cute_PrivateMemrefLoadOp : Cute_PrivateSchemaOp<"memref.load"> { + let arguments = (ins AnyType:$src, AnyType:$coord); + let results = (outs AnyType:$result); +} + +def Cute_PrivateMemrefLoadVecOp + : Cute_PrivateSchemaOp<"memref.load_vec", [AttrSizedOperandSegments]> { + let arguments = (ins AnyType:$src, Optional:$mask, + Optional:$pass_thru); + let results = (outs AnyVectorOfAnyRank:$result); +} + +def Cute_PrivateMemrefStoreOp : Cute_PrivateSchemaOp<"memref.store"> { + let arguments = (ins AnyType:$dst, AnyType:$coord, AnyType:$value); + let results = (outs); +} + +def Cute_PrivateMemrefStoreVecOp + : Cute_PrivateSchemaOp<"memref.store_vec"> { + let arguments = (ins AnyVectorOfAnyRank:$value, AnyType:$dst, + Optional:$mask); + let results = (outs); +} + +def Cute_PrivateMmaMakeFragmentOp + : Cute_PrivateSchemaOp<"mma.make_fragment"> { + let arguments = (ins AnyType:$atom, AnyType:$input, I32Attr:$operand_id); + let results = (outs AnyType:$result); +} + +def Cute_PrivateMmaAtomCallOp + : Cute_PrivateSchemaOp<"mma_atom_call", [AttrSizedOperandSegments]> { + let arguments = (ins AnyType:$atom, AnyType:$d, Variadic:$a, + Variadic:$b, AnyType:$c); + let results = (outs); +} + +def Cute_PrivatePrefetchOp : Cute_PrivateSchemaOp<"prefetch"> { + let arguments = (ins AnyType:$prefetch_atom, Variadic:$src); + let results = (outs); +} + +def Cute_PrivatePrefetchAtomCallOp + : Cute_PrivateSchemaOp<"prefetch_atom_call"> { + let arguments = (ins AnyType:$atom, Variadic:$src); + let results = (outs); +} + +def Cute_PrivatePrintTmaDescIm2colOp + : Cute_PrivateSchemaOp<"print_tma_desc_im2col"> { + let arguments = (ins AnyType:$input, AnyType:$fd); + let results = (outs); +} + +def Cute_PrivatePrintTmaDescTiledOp + : Cute_PrivateSchemaOp<"print_tma_desc_tiled"> { + let arguments = (ins AnyType:$input, AnyType:$fd); + let results = (outs); +} + +def Cute_PrivatePrintViewOp : Cute_PrivateSchemaOp<"print_view"> { + let arguments = (ins AnyType:$src, Optional:$coord, + OptionalAttr:$verbose, + OptionalAttr:$is_signed); + let results = (outs); +} + +def Cute_PrivatePtrLoadOp : Cute_PrivateSchemaOp<"ptr.load"> { + let arguments = (ins AnyType:$ptr); + let results = (outs AnyType:$value); +} + +def Cute_PrivatePtrStoreOp : Cute_PrivateSchemaOp<"ptr.store"> { + let arguments = (ins AnyType:$value, AnyType:$ptr); + let results = (outs); +} + +def Cute_PrivatePtrToIntOp : Cute_PrivateSchemaOp<"ptrtoint"> { + let arguments = (ins AnyType:$src); + let results = (outs AnyInteger:$dst); +} + +def Cute_PrivateRandomDelayOp : Cute_PrivateSchemaOp<"random_delay"> { + let arguments = (ins); + let results = (outs); +} + +def Cute_PrivateRecastIterOp : Cute_PrivateSchemaOp<"recast_iter"> { + let arguments = (ins AnyType:$src); + let results = (outs AnyType:$dst); +} + +def Cute_PrivateReduceOp : Cute_PrivateSchemaOp<"reduce"> { + let arguments = (ins AnyVectorOfAnyRank:$input, AnyType:$shape, + AnyType:$init_val, + Optional:$predication, + AnyAttr:$reduction_op, + OptionalAttr:$reduction_profile); + let results = (outs AnyType:$output, Optional:$reduced_shape); +} + +def Cute_PrivateSmemPartitionSizeOp + : Cute_PrivateSchemaOp<"smem_partition_size"> { + let arguments = (ins I32Attr:$partition_id, + OptionalAttr:$cumulative); + let results = (outs I64:$size); +} + +def Cute_PrivateStencilDivideOp + : Cute_PrivateSchemaOp<"stencil_divide", [AttrSizedOperandSegments]> { + let arguments = (ins AnyType:$input, AnyType:$stencil, + Optional:$padding_upper, + Optional:$padding_lower, + Optional:$traversal_stride); + let results = (outs AnyType:$result); +} + +def Cute_PrivateSymbolicOp : Cute_PrivateSchemaOp<"symbolic"> { + let arguments = (ins StrAttr:$name); + let results = (outs AnyType:$symbolic); +} + +def Cute_PrivateTiledCopyPartitionDOp + : Cute_PrivateSchemaOp<"tiled.copy.partition_D"> { + let arguments = (ins AnyType:$tiled_copy, AnyType:$input, AnyType:$coord); + let results = (outs AnyType:$result); +} + +def Cute_PrivateTiledCopyPartitionSOp + : Cute_PrivateSchemaOp<"tiled.copy.partition_S"> { + let arguments = (ins AnyType:$tiled_copy, AnyType:$input, AnyType:$coord); + let results = (outs AnyType:$result); +} + +def Cute_PrivateTiledCopyRetileOp + : Cute_PrivateSchemaOp<"tiled.copy.retile"> { + let arguments = (ins AnyType:$tiled_copy, AnyType:$input); + let results = (outs AnyType:$result); +} + +def Cute_PrivateTiledMmaPartitionOp + : Cute_PrivateSchemaOp<"tiled.mma.partition"> { + let arguments = (ins AnyType:$tiled_mma, AnyType:$input, AnyType:$coord, + I32Attr:$operand_id); + let results = (outs AnyType:$result); +} + +def Cute_PrivateTiledMmaPartitionShapeOp + : Cute_PrivateSchemaOp<"tiled.mma.partition_shape"> { + let arguments = (ins AnyType:$tiled_mma, AnyType:$input, + I32Attr:$operand_id); + let results = (outs AnyType:$result); +} + +def Cute_PrivateTupleProductOp : Cute_PrivateSchemaOp<"tuple.product"> { + let arguments = (ins AnyType:$input); + let results = (outs AnyType:$result); +} + +def Cute_PrivateTupleProductEachOp + : Cute_PrivateSchemaOp<"tuple.product_each"> { + let arguments = (ins AnyType:$input); + let results = (outs AnyType:$result); +} + +def Cute_PrivateTupleDivOp : Cute_PrivateSchemaOp<"tuple_div"> { + let arguments = (ins AnyType:$lhs, AnyType:$rhs); + let results = (outs AnyType:$result); +} + +def Cute_PrivateTupleModOp : Cute_PrivateSchemaOp<"tuple_mod"> { + let arguments = (ins AnyType:$lhs, AnyType:$rhs); + let results = (outs AnyType:$result); +} + +def Cute_PrivateTupleMulOp : Cute_PrivateSchemaOp<"tuple_mul"> { + let arguments = (ins AnyType:$lhs, AnyType:$rhs); + let results = (outs AnyType:$result); +} + +def Cute_PrivateUnpackTupleOp : Cute_PrivateSchemaOp<"unpack_tuple"> { + let arguments = (ins AnyType:$tuple); + let results = (outs Variadic:$result); +} + +#endif // CUTE_IR_DIALECT_CUTE_IR_CUTE_PRIVATE_COMPAT_OPS_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesAll.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesAll.td new file mode 100644 index 00000000..4e375294 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesAll.td @@ -0,0 +1,2 @@ +include "cute_ir/Dialect/Cute/IR/CuteTypes.td" +include "cute_ir/Dialect/Cute/IR/CuteTypesPrivate.td" diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesPrivate.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesPrivate.td new file mode 100644 index 00000000..34f7d21e --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesPrivate.td @@ -0,0 +1,92 @@ +#ifndef CUTE_IR_DIALECT_CUTE_IR_CUTE_PRIVATE_COMPAT_TYPES_TD +#define CUTE_IR_DIALECT_CUTE_IR_CUTE_PRIVATE_COMPAT_TYPES_TD + +include "cute_ir/Dialect/Cute/IR/CuteAttrs.td" + +def Cute_ArithTupleIteratorType + : Cute_Type<"ArithTupleIterator", "arith_tuple_iter"> { + let summary = "Iterator over an arithmetic tuple"; + let parameters = (ins + Cute_Param<"IntTupleType", "arithmetic tuple carried by the iterator">:$arithTuple + ); + let hasCustomAssemblyFormat = 1; +} + +def Cute_ConstrainedInt32Type : Cute_Type<"ConstrainedInt32", "i32"> { + let summary = "32-bit integer with a divisibility constraint"; + let parameters = (ins + Cute_Param<"uint64_t", "known positive divisor">:$divisibleBy + ); + let hasCustomAssemblyFormat = 1; + let genVerifyDecl = 1; +} + +def Cute_ConstrainedInt64Type : Cute_Type<"ConstrainedInt64", "i64"> { + let summary = "64-bit integer with a divisibility constraint"; + let parameters = (ins + Cute_Param<"uint64_t", "known positive divisor">:$divisibleBy + ); + let hasCustomAssemblyFormat = 1; + let genVerifyDecl = 1; +} + +def Cute_FastDivmodDivisorType + : Cute_Type<"FastDivmodDivisor", "fast_divmod_divisor"> { + let summary = "Precomputed divisor used by fast divmod operations"; + let parameters = (ins + Cute_Param<"unsigned", "integer width">:$width, + Cute_Param<"bool", "whether the divisor is a power of two">:$isPow2 + ); + let hasCustomAssemblyFormat = 1; + let genVerifyDecl = 1; +} + +def Cute_SparseElemType : Cute_Type<"SparseElem", "sparse_elem"> { + let summary = "Logical sparse element represented by physical storage"; + let parameters = (ins + Cute_Param<"int", "number of logical elements">:$numLogical, + Cute_Param<"mlir::Type", "physical storage type">:$physicalType + ); + let assemblyFormat = "`<` $numLogical `,` qualified($physicalType) `>`"; + let genVerifyDecl = 1; +} + +def Cute_PtrType : Cute_Type<"Ptr", "ptr"> { + let summary = "CuTe pointer with memory-space, alignment, and swizzle metadata"; + let parameters = (ins + Cute_OptionalParam<"mlir::Type", "optional pointed-to value type">:$valueType, + Cute_Param<"mlir::StringAttr", "CuTe address-space mnemonic">:$memorySpace, + Cute_Param<"uint64_t", "byte alignment">:$alignment, + Cute_OptionalParam<"SwizzleAttr", "optional pointer swizzle">:$swizzle + ); + let hasCustomAssemblyFormat = 1; + let genVerifyDecl = 1; +} + +def Cute_MemRefType : Cute_Type<"MemRef", "memref"> { + let summary = "CuTe tensor view pairing a pointer with a static layout"; + let parameters = (ins + Cute_Param<"PtrType", "underlying pointer">:$ptr, + Cute_Param<"mlir::Type", "layout or composed-layout type">:$layout + ); + let hasCustomAssemblyFormat = 1; + let genVerifyDecl = 1; + let extraClassDeclaration = [{ + mlir::Type getValueType() const { return getPtr().getValueType(); } + mlir::StringAttr getMemorySpace() const { return getPtr().getMemorySpace(); } + uint64_t getAlignment() const { return getPtr().getAlignment(); } + SwizzleAttr getSwizzle() const { return getPtr().getSwizzle(); } + }]; +} + +def Cute_CoordTensorType : Cute_Type<"CoordTensor", "coord_tensor"> { + let summary = "Coordinate iterator paired with a CuTe layout"; + let parameters = (ins + Cute_Param<"IntTupleType", "coordinate arithmetic tuple">:$arithTuple, + Cute_Param<"mlir::Type", "layout or composed-layout type">:$layout + ); + let hasCustomAssemblyFormat = 1; + let genVerifyDecl = 1; +} + +#endif // CUTE_IR_DIALECT_CUTE_IR_CUTE_PRIVATE_COMPAT_TYPES_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/BUILD.bazel b/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/BUILD.bazel new file mode 100644 index 00000000..fca978d1 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/BUILD.bazel @@ -0,0 +1,88 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:defs.bzl", "cc_library") + +package(default_visibility = ["//visibility:public"]) + +td_library( + name = "td_files", + srcs = glob(["*.td"]), + includes = ["/mlir/dialects/cute_ir/include"], + deps = ["@llvm-project//mlir:OpBaseTdFiles"], +) + +gentbl_cc_library( + name = "dialect_inc_gen", + includes = ["../../../.."], + tbl_outs = [ + ( + [ + "-gen-dialect-decls", + "-dialect=cute_nvgpu", + ], + "CuteNVGPUDialect.h.inc", + ), + ( + [ + "-gen-dialect-defs", + "-dialect=cute_nvgpu", + ], + "CuteNVGPUDialect.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteNVGPUDialect.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "types_inc_gen", + includes = ["../../../.."], + tbl_outs = [ + ( + [ + "-gen-typedef-decls", + "-typedefs-dialect=cute_nvgpu", + ], + "CuteNVGPUTypes.h.inc", + ), + ( + [ + "-gen-typedef-defs", + "-typedefs-dialect=cute_nvgpu", + ], + "CuteNVGPUTypes.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteNVGPUTypes.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "ops_inc_gen", + includes = ["../../../.."], + tbl_outs = [ + ( + ["-gen-op-decls"], + "CuteNVGPUOps.h.inc", + ), + ( + ["-gen-op-defs"], + "CuteNVGPUOps.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteNVGPUOps.td", + deps = [":td_files"], +) + +cc_library( + name = "headers", + hdrs = ["CuteNVGPUDialect.h"], + includes = ["../../../.."], + deps = [ + ":dialect_inc_gen", + ":ops_inc_gen", + ":types_inc_gen", + ], +) diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h b/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h new file mode 100644 index 00000000..b0e9fa0a --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h @@ -0,0 +1,18 @@ +//===- CuteNVGPUDialect.h - CuTe NVIDIA GPU dialect ------------*- C++ -*-===// + +#ifndef CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_DIALECT_H +#define CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_DIALECT_H + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/DialectImplementation.h" +#include "mlir/IR/OpImplementation.h" + +#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h.inc" + +#define GET_TYPEDEF_CLASSES +#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUTypes.h.inc" + +#define GET_OP_CLASSES +#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUOps.h.inc" + +#endif // CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_DIALECT_H diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.td new file mode 100644 index 00000000..743d4c05 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.td @@ -0,0 +1,34 @@ +//===- CuteNVGPUDialect.td - CuTe NVIDIA GPU dialect -------*- tablegen -*-===// + +#ifndef CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_DIALECT_TD +#define CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_DIALECT_TD + +include "mlir/IR/DialectBase.td" +include "mlir/IR/OpBase.td" + +def CuteNVGPU_Dialect : Dialect { + let name = "cute_nvgpu"; + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let summary = "NVIDIA GPU operations and atom types used by CuTe-DSL"; + let description = [{ + This dialect mirrors the operation and type registry embedded in the + proprietary CuTe-DSL compiler. Proven types receive concrete TypeDefs; + parameterized types remain lossless opaque types until their storage and + verification contracts are fully recovered. + }]; + let extraClassDeclaration = [{ + ::mlir::Type parseType(::mlir::DialectAsmParser &parser) const override; + void printType(::mlir::Type type, + ::mlir::DialectAsmPrinter &printer) const override; + }]; +} + +class CuteNVGPU_Op traits = []> + : Op; + +class CuteNVGPU_Type + : TypeDef { + let mnemonic = typeMnemonic; +} + +#endif // CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_DIALECT_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUOps.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUOps.td new file mode 100644 index 00000000..8826363d --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUOps.td @@ -0,0 +1,378 @@ +//===- CuteNVGPUOps.td - CuTe NVIDIA GPU operations --------*- tablegen -*-===// + +#ifndef CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_OPS_TD +#define CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_OPS_TD + +include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.td" + +class CuteNVGPU_SchemaOp traits = []> + : CuteNVGPU_Op { + let summary = "Operation recovered from the CuTe-DSL operation registry"; +} + +def CuteNVGPU_ArchAllocRmemOp : CuteNVGPU_SchemaOp<"arch.alloc_rmem"> { + let arguments = (ins AnyType:$input); + let results = (outs AnyType:$ptr); +} + +def CuteNVGPU_ArchAllocSmemOp : CuteNVGPU_SchemaOp<"arch.alloc_smem"> { + let arguments = (ins I32Attr:$input); + let results = (outs AnyType:$ptr); +} + +def CuteNVGPU_ArchCopySM90BulkCopyG2SOp + : CuteNVGPU_SchemaOp<"arch.copy.SM90.bulk_copy_g2s", + [AttrSizedOperandSegments]> { + let arguments = (ins AnyType:$gmem_data_addr, AnyType:$dsmem_data_addr, + AnyType:$dsmem_bar_addr, + Optional:$multicast_mask, + Optional:$cache_policy, I32Attr:$size); + let results = (outs); +} + +def CuteNVGPU_ArchCopySM90BulkCopyS2GOp + : CuteNVGPU_SchemaOp<"arch.copy.SM90.bulk_copy_s2g", + [AttrSizedOperandSegments]> { + let arguments = (ins AnyType:$smem_data_addr, AnyType:$gmem_data_addr, + Optional:$byte_mask, + Optional:$cache_policy, I32Attr:$size); + let results = (outs); +} + +def CuteNVGPU_ArchCopySM90BulkCopyS2SOp + : CuteNVGPU_SchemaOp<"arch.copy.SM90.bulk_copy_s2s"> { + let arguments = (ins AnyType:$smem_data_addr, AnyType:$dsmem_data_addr, + AnyType:$dsmem_bar_addr, AnyInteger:$cta_rank, + I32Attr:$size); + let results = (outs); +} + +def CuteNVGPU_ArchCopySM100CopyS2TOp + : CuteNVGPU_SchemaOp<"arch.copy.SM100.copy_s2t"> { + let arguments = (ins AnyType:$src, AnyType:$dst, I32Attr:$dp, I32Attr:$bits, + I32Attr:$cta, AnyAttr:$broadcast); + let results = (outs); +} + +def CuteNVGPU_ArchCopySM100TmaLoadOp + : CuteNVGPU_SchemaOp<"arch.copy.SM100.tma_load", + [AttrSizedOperandSegments]> { + let arguments = (ins AnyType:$src_desc, AnyType:$dsmem_data_addr, + AnyType:$dsmem_bar_addr, Variadic:$coord, + Optional:$multicast_mask, + Variadic:$offsets, + Optional:$cache_policy, + AnyAttr:$mode, I32Attr:$num_cta); + let results = (outs); +} + +def CuteNVGPU_ArchCopySM100TmaReduceOp + : CuteNVGPU_SchemaOp<"arch.copy.SM100.tma_reduce", + [AttrSizedOperandSegments]> { + let arguments = (ins AnyType:$dst_desc, AnyType:$src_smem_data_addr, + Variadic:$coord, + Optional:$cache_policy, + AnyAttr:$mode, AnyAttr:$kind); + let results = (outs); +} + +def CuteNVGPU_ArchCopySM100TmaStoreOp + : CuteNVGPU_SchemaOp<"arch.copy.SM100.tma_store", + [AttrSizedOperandSegments]> { + let arguments = (ins AnyType:$dst_desc, AnyType:$src_smem_data_addr, + Variadic:$coord, + Optional:$cache_policy, AnyAttr:$mode); + let results = (outs); +} + +def CuteNVGPU_ArchCopySM100TmemLoadOp + : CuteNVGPU_SchemaOp<"arch.copy.SM100.tmem_load"> { + let arguments = (ins AnyType:$src, I32Attr:$num_dp, I32Attr:$num_b, + I32Attr:$num_rep, OptionalAttr:$pack_16); + let results = (outs AnyType:$dst); +} + +def CuteNVGPU_ArchCopySM100TmemStoreOp + : CuteNVGPU_SchemaOp<"arch.copy.SM100.tmem_store"> { + let arguments = (ins AnyType:$dst, AnyType:$src, I32Attr:$num_dp, + I32Attr:$num_b, I32Attr:$num_rep, + OptionalAttr:$expand_16); + let results = (outs); +} + +def CuteNVGPU_ArchGetDynSmemOp + : CuteNVGPU_SchemaOp<"arch.get_dyn_smem"> { + let arguments = (ins); + let results = (outs AnyType:$ptr); +} + +def CuteNVGPU_ArchGetDynSmemSizeOp + : CuteNVGPU_SchemaOp<"arch.get_dyn_smem_size"> { + let arguments = (ins); + let results = (outs I32:$size); +} + +def CuteNVGPU_ArchMakeWarpUniformOp + : CuteNVGPU_SchemaOp<"arch.make_warp_uniform"> { + let arguments = (ins AnyInteger:$val); + let results = (outs AnyInteger:$res); +} + +def CuteNVGPU_ArchMmaSM120BlockScaledOp + : CuteNVGPU_SchemaOp<"arch.mma.SM120.block_scaled", + [AttrSizedOperandSegments]> { + let arguments = (ins Variadic:$opA, Variadic:$opB, + Variadic:$opC, AnyType:$sfA, AnyType:$sfB, + Optional:$byte_id_a, + Optional:$byte_id_b, + AnyAttr:$shape_MNK, I32Attr:$vec_size, + OptionalAttr:$thread_id_a, + OptionalAttr:$thread_id_b, + TypeAttr:$a_type, TypeAttr:$b_type, TypeAttr:$sf_type); + let results = (outs Variadic:$res); +} + +def CuteNVGPU_ArchPrefetchTmaDescOp + : CuteNVGPU_SchemaOp<"arch.prefetch_tma_desc"> { + let arguments = (ins AnyType:$tma_descriptor_ptr); + let results = (outs); +} + +def CuteNVGPU_ArchSM100AllocTmemOp + : CuteNVGPU_SchemaOp<"arch.sm100.alloc_tmem"> { + let arguments = (ins AnyInteger:$n_cols, AnyType:$dst_ptr, + OptionalAttr:$is_two_cta, + OptionalAttr:$exclusive); + let results = (outs); +} + +def CuteNVGPU_ArchSM100DeallocTmemOp + : CuteNVGPU_SchemaOp<"arch.sm100.dealloc_tmem"> { + let arguments = (ins AnyType:$tmem_ptr, AnyInteger:$n_cols, + OptionalAttr:$is_two_cta, + OptionalAttr:$exclusive); + let results = (outs); +} + +def CuteNVGPU_ArchSM100RelinquishTmemAllocPermitOp + : CuteNVGPU_SchemaOp<"arch.sm100.relinquish_tmem_alloc_permit"> { + let arguments = (ins OptionalAttr:$is_two_cta); + let results = (outs); +} + +def CuteNVGPU_ArchSM100RetrieveTmemPtrOp + : CuteNVGPU_SchemaOp<"arch.sm100.retrieve_tmem_ptr"> { + let arguments = (ins AnyType:$smem_ptr); + let results = (outs AnyType:$tmem_ptr); +} + +def CuteNVGPU_AtomMakeExecTmaOp + : CuteNVGPU_SchemaOp<"atom.make_exec_tma"> { + let arguments = (ins AnyType:$input); + let results = (outs AnyType:$result); +} + +def CuteNVGPU_AtomMakeNonExec2DGather4TmaLoadOp + : CuteNVGPU_SchemaOp<"atom.make_non_exec_2d_gather4_tma_load"> { + let arguments = (ins AnyType:$gmem_tensor, AnyType:$gmem_coord_layout, + AnyType:$smem_layout, AnyType:$cta_v_map, + AnyAttr:$kind, OptionalAttr:$num_multicast, + OptionalAttr:$tma_format); + let results = (outs AnyType:$non_exec_atom, AnyType:$tma_tensor); +} + +def CuteNVGPU_AtomMakeNonExecIm2ColTmaLoadOp + : CuteNVGPU_SchemaOp<"atom.make_non_exec_im2col_tma_load"> { + let arguments = (ins AnyType:$gmem_tensor, AnyType:$smem_layout, + AnyType:$cta_v_map, AnyType:$lower_corner_whd, + AnyType:$upper_corner_whd, AnyType:$lower_padding_whd, + AnyType:$upper_padding_whd, AnyType:$stride_whd, + AnyType:$lower_srt, AnyType:$stride_srt, + AnyAttr:$kind, OptionalAttr:$num_multicast, + OptionalAttr:$tma_format); + let results = (outs AnyType:$non_exec_atom, AnyType:$tma_tensor); +} + +def CuteNVGPU_AtomMakeNonExecIm2ColTmaStoreOp + : CuteNVGPU_SchemaOp<"atom.make_non_exec_im2col_tma_store"> { + let arguments = (ins AnyType:$gmem_tensor, AnyType:$smem_layout, + AnyType:$cta_v_map, + OptionalAttr:$tma_format); + let results = (outs AnyType:$non_exec_atom, AnyType:$tma_tensor); +} + +def CuteNVGPU_AtomMakeNonExecTiledTmaLoadOp + : CuteNVGPU_SchemaOp<"atom.make_non_exec_tiled_tma_load"> { + let arguments = (ins AnyType:$gmem_tensor, AnyType:$smem_layout, + AnyType:$cta_v_map, AnyAttr:$kind, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format); + let results = (outs AnyType:$non_exec_atom, AnyType:$tma_tensor); +} + +def CuteNVGPU_AtomMakeNonExecTiledTmaReduceOp + : CuteNVGPU_SchemaOp<"atom.make_non_exec_tiled_tma_reduce"> { + let arguments = (ins AnyType:$gmem_tensor, AnyType:$smem_layout, + AnyType:$cta_v_map, AnyAttr:$kind, + OptionalAttr:$tma_format); + let results = (outs AnyType:$non_exec_atom, AnyType:$tma_tensor); +} + +def CuteNVGPU_AtomMakeNonExecTiledTmaStoreOp + : CuteNVGPU_SchemaOp<"atom.make_non_exec_tiled_tma_store"> { + let arguments = (ins AnyType:$gmem_tensor, AnyType:$smem_layout, + AnyType:$cta_v_map, + OptionalAttr:$tma_format); + let results = (outs AnyType:$non_exec_atom, AnyType:$tma_tensor); +} + +def CuteNVGPU_AtomMakeS2TCopyOp + : CuteNVGPU_SchemaOp<"atom.make_s2t_copy"> { + let arguments = (ins AnyType:$copy_s2t_atom, AnyType:$tmem_memref); + let results = (outs AnyType:$result); +} + +def CuteNVGPU_AtomMakeTmemCopyOp + : CuteNVGPU_SchemaOp<"atom.make_tmem_copy"> { + let arguments = (ins AnyType:$atom, AnyType:$tmem_memref); + let results = (outs AnyType:$result); +} + +def CuteNVGPU_AtomGetCoordTensorOp + : CuteNVGPU_SchemaOp<"atom.get_coord_tensor", + [SameVariadicOperandSize]> { + let arguments = (ins AnyType:$atom, AnyType:$shape, + Variadic:$offset_b, + Variadic:$layout_b); + let results = (outs AnyType:$result); +} + +def CuteNVGPU_AtomGetCopyS2TSmemDescViewOp + : CuteNVGPU_SchemaOp<"atom.get_copy_s2t_smem_desc_view"> { + let arguments = (ins AnyType:$atom, AnyType:$view); + let results = (outs AnyType:$result); +} + +def CuteNVGPU_AtomGetValueOp + : CuteNVGPU_SchemaOp<"atom.get_value"> { + let arguments = (ins AnyType:$atom, AnyAttr:$field); + let results = (outs AnyType:$result); +} + +def CuteNVGPU_AtomSetValueOp + : CuteNVGPU_SchemaOp<"atom.set_value"> { + let arguments = (ins AnyType:$atom, AnyType:$value, AnyAttr:$field); + let results = (outs AnyType:$result); +} + +def CuteNVGPU_AtomTmaPartitionOp + : CuteNVGPU_SchemaOp<"atom.tma_partition"> { + let arguments = (ins AnyType:$tma_atom, AnyType:$cta_coord, + AnyType:$cta_layout, AnyType:$smem_tensor, + Variadic:$target_tensors); + let results = (outs AnyType:$res_smem_tensor, + Variadic:$res_target_tensors); +} + +def CuteNVGPU_CastTmaDescToIntegerOp + : CuteNVGPU_SchemaOp<"cast_tma_desc_to_integer"> { + let arguments = (ins AnyType:$src); + let results = (outs AnyInteger:$dst); +} + +def CuteNVGPU_CopyTmaDescOp : CuteNVGPU_SchemaOp<"copy_tma_desc"> { + let arguments = (ins AnyType:$atom, AnyType:$tma_desc_ptr); + let results = (outs); +} + +def CuteNVGPU_GetGridConstantPointerOp + : CuteNVGPU_SchemaOp<"get_grid_constant_pointer"> { + let arguments = (ins AnyType:$input); + let results = (outs AnyType:$result); +} + +def CuteNVGPU_GetTmaDescAddrOp + : CuteNVGPU_SchemaOp<"get_tma_desc_addr"> { + let arguments = (ins AnyType:$src); + let results = (outs AnyType:$ptr); +} + +def CuteNVGPU_MakeGmmaSmemDescOp + : CuteNVGPU_SchemaOp<"make_gmma_smem_desc"> { + let arguments = (ins AnyType:$src, AnyAttr:$layout, AnyAttr:$major); + let results = (outs AnyType:$res); +} + +def CuteNVGPU_MakeTmaDescIm2ColAtOp + : CuteNVGPU_SchemaOp<"make_tma_desc_im2col_at"> { + let arguments = (ins AnyType:$gmem_view, AnyType:$smem_layout, + AnyType:$cta_value_tile, AnyType:$lower_corner_whd, + AnyType:$upper_corner_whd, AnyType:$lower_padding_whd, + AnyType:$upper_padding_whd, AnyType:$stride_whd, + AnyType:$lower_srt, AnyType:$stride_srt, + AnyType:$tma_desc_addr, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format); + let results = (outs); +} + +def CuteNVGPU_MakeTmaDescIm2ColOp + : CuteNVGPU_SchemaOp<"make_tma_desc_im2col"> { + let arguments = (ins AnyType:$gmem_view, AnyType:$smem_layout, + AnyType:$cta_value_tile, AnyType:$lower_corner_whd, + AnyType:$upper_corner_whd, AnyType:$lower_padding_whd, + AnyType:$upper_padding_whd, AnyType:$stride_whd, + AnyType:$lower_srt, AnyType:$stride_srt, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format); + let results = (outs AnyType:$result); +} + +def CuteNVGPU_MakeTmaDescTiledAtOp + : CuteNVGPU_SchemaOp<"make_tma_desc_tiled_at"> { + let arguments = (ins AnyType:$gmem_view, AnyType:$smem_layout, + AnyType:$cta_value_tile, AnyType:$tma_desc_addr, + Optional:$traversal_stride, AnyAttr:$mode, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format); + let results = (outs); +} + +def CuteNVGPU_MakeTmaDescTiledOp + : CuteNVGPU_SchemaOp<"make_tma_desc_tiled"> { + let arguments = (ins AnyType:$gmem_view, AnyType:$smem_layout, + AnyType:$cta_value_tile, + Optional:$traversal_stride, AnyAttr:$mode, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format); + let results = (outs AnyType:$result); +} + +def CuteNVGPU_SM103MakeUmmaSmemDescCircularOp + : CuteNVGPU_SchemaOp<"sm103.make_umma_smem_desc_circular"> { + let arguments = (ins AnyType:$src, Optional:$nextSrc, + AnyAttr:$layout, AnyAttr:$major); + let results = (outs AnyType:$res); +} + +def CuteNVGPU_MakeUmmaSmemDescOp + : CuteNVGPU_SchemaOp<"make_umma_smem_desc"> { + let arguments = (ins AnyType:$src, Optional:$nextSrc, + AnyAttr:$layout, AnyAttr:$major); + let results = (outs AnyType:$res); +} + +def CuteNVGPU_PrefetchTmaDescOp + : CuteNVGPU_SchemaOp<"prefetch_tma_desc"> { + let arguments = (ins AnyType:$atom); + let results = (outs); +} + +def CuteNVGPU_UpdateTmaDescOp + : CuteNVGPU_SchemaOp<"update_tma_desc"> { + let arguments = (ins AnyType:$tma_atom, AnyType:$gmem_tensor, + AnyType:$tma_desc_ptr); + let results = (outs); +} + +#endif // CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_OPS_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUTypes.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUTypes.td new file mode 100644 index 00000000..ff56a095 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUTypes.td @@ -0,0 +1,140 @@ +//===- CuteNVGPUTypes.td - CuTe NVIDIA GPU types -----------*- tablegen -*-===// + +#ifndef CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_TYPES_TD +#define CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_TYPES_TD + +include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.td" + +def CuteNVGPU_SmemDescType : CuteNVGPU_Type<"SmemDesc", "smem_desc"> { + let summary = "Unparameterized shared-memory descriptor"; + let assemblyFormat = ""; +} + +def CuteNVGPU_TmaDescriptorTiledType + : CuteNVGPU_Type<"TmaDescriptorTiled", "tma_descriptor_tiled"> { + let summary = "Unparameterized tiled TMA descriptor"; + let assemblyFormat = ""; +} + +def CuteNVGPU_TmaDescriptorIm2ColType + : CuteNVGPU_Type<"TmaDescriptorIm2Col", "tma_descriptor_im2col"> { + let summary = "Unparameterized im2col TMA descriptor"; + let assemblyFormat = ""; +} + +def CuteNVGPU_WorkIdResponseType + : CuteNVGPU_Type<"WorkIdResponse", "workid_response"> { + let summary = "Tensor-memory work identifier response"; + let assemblyFormat = ""; +} + +class CuteNVGPU_PayloadType + : CuteNVGPU_Type { + let summary = "CuTe-DSL NVIDIA GPU type with lossless architecture payload"; + let parameters = (ins + "mlir::StringAttr":$payload + ); + let hasCustomAssemblyFormat = 1; +} + +def CuteNVGPU_CopyAtomBulkCopyG2SType + : CuteNVGPU_PayloadType<"CopyAtomBulkCopyG2S", "atom.bulk_copy_g2s">; +def CuteNVGPU_CopyAtomBulkCopyS2GType + : CuteNVGPU_PayloadType<"CopyAtomBulkCopyS2G", "atom.bulk_copy_s2g">; +def CuteNVGPU_CopyAtomBulkCopyS2SType + : CuteNVGPU_PayloadType<"CopyAtomBulkCopyS2S", "atom.bulk_copy_s2s">; +def CuteNVGPU_CopyAtomDsmemStoreType + : CuteNVGPU_PayloadType<"CopyAtomDsmemStore", "atom.dsmem_store">; +def CuteNVGPU_CopyAtomG2RType + : CuteNVGPU_PayloadType<"CopyAtomG2R", "atom.g2r">; +def CuteNVGPU_CopyAtomLdsmType + : CuteNVGPU_PayloadType<"CopyAtomLdsm", "atom.ldsm">; +def CuteNVGPU_CopyAtomNonExec2DGather4TmaLoadType + : CuteNVGPU_PayloadType<"CopyAtomNonExec2DGather4TmaLoad", + "atom.non_exec_2d_gather4_tma_load">; +def CuteNVGPU_CopyAtomNonExecIm2ColTmaLoadType + : CuteNVGPU_PayloadType<"CopyAtomNonExecIm2ColTmaLoad", + "atom.non_exec_im2col_tma_load">; +def CuteNVGPU_CopyAtomNonExecIm2ColTmaStoreType + : CuteNVGPU_PayloadType<"CopyAtomNonExecIm2ColTmaStore", + "atom.non_exec_im2col_tma_store">; +def CuteNVGPU_CopyAtomNonExecTiledTmaLoadType + : CuteNVGPU_PayloadType<"CopyAtomNonExecTiledTmaLoad", + "atom.non_exec_tiled_tma_load">; +def CuteNVGPU_CopyAtomNonExecTiledTmaReduceType + : CuteNVGPU_PayloadType<"CopyAtomNonExecTiledTmaReduce", + "atom.non_exec_tiled_tma_reduce">; +def CuteNVGPU_CopyAtomNonExecTiledTmaStoreType + : CuteNVGPU_PayloadType<"CopyAtomNonExecTiledTmaStore", + "atom.non_exec_tiled_tma_store">; +def CuteNVGPU_CopyAtomR2GType + : CuteNVGPU_PayloadType<"CopyAtomR2G", "atom.r2g">; +def CuteNVGPU_CopyAtomR2SType + : CuteNVGPU_PayloadType<"CopyAtomR2S", "atom.r2s">; +def CuteNVGPU_CopyAtomS2RType + : CuteNVGPU_PayloadType<"CopyAtomS2R", "atom.s2r">; +def CuteNVGPU_CopyAtomSIMTAsyncCopyType + : CuteNVGPU_PayloadType<"CopyAtomSIMTAsyncCopy", "atom.simt_async_copy">; +def CuteNVGPU_CopyAtomSIMTSyncCopyType + : CuteNVGPU_PayloadType<"CopyAtomSIMTSyncCopy", "atom.universal_copy">; +def CuteNVGPU_CopyAtomSM100CopyS2TType + : CuteNVGPU_PayloadType<"CopyAtomSM100CopyS2T", "atom.s2t_copy">; +def CuteNVGPU_CopyAtomSM100TmemLoadType + : CuteNVGPU_PayloadType<"CopyAtomSM100TmemLoad", "atom.tmem_load">; +def CuteNVGPU_CopyAtomSM100TmemStoreType + : CuteNVGPU_PayloadType<"CopyAtomSM100TmemStore", "atom.tmem_store">; +def CuteNVGPU_CopyAtomSM10xTmemLoadRedType + : CuteNVGPU_PayloadType<"CopyAtomSM10xTmemLoadRed", "atom.tmem_load_red">; +def CuteNVGPU_CopyAtomStsmType + : CuteNVGPU_PayloadType<"CopyAtomStsm", "atom.stsm">; +def CuteNVGPU_MmaAtomSM100UMMABlockScaledType + : CuteNVGPU_PayloadType<"MmaAtomSM100UMMABlockScaled", "sm100.mma_bs">; +def CuteNVGPU_MmaAtomSM100UMMAType + : CuteNVGPU_PayloadType<"MmaAtomSM100UMMA", "sm100.mma">; +def CuteNVGPU_MmaAtomSM120BlockScaledType + : CuteNVGPU_PayloadType<"MmaAtomSM120BlockScaled", "SM120.mma_bs">; +def CuteNVGPU_MmaAtomSM80Type + : CuteNVGPU_PayloadType<"MmaAtomSM80", "sm80.mma">; +def CuteNVGPU_MmaAtomSM89Type + : CuteNVGPU_PayloadType<"MmaAtomSM89", "sm89.mma">; +def CuteNVGPU_MmaAtomSM90Type + : CuteNVGPU_PayloadType<"MmaAtomSM90", "sm90.mma">; +def CuteNVGPU_SmemDescViewType + : CuteNVGPU_PayloadType<"SmemDescView", "smem_desc_view">; +def CuteNVGPU_TiledCopyType + : CuteNVGPU_PayloadType<"TiledCopy", "tiled_copy">; +def CuteNVGPU_TiledMmaType + : CuteNVGPU_PayloadType<"TiledMma", "tiled_mma">; +def CuteNVGPU_UniversalFmaAtomType + : CuteNVGPU_PayloadType<"UniversalFmaAtom", "atom.universal_fma">; +def CuteNVGPU_CopyAtomIm2ColTmaLoadType + : CuteNVGPU_PayloadType<"CopyAtomIm2ColTmaLoad", "atom.im2col_tma_load">; +def CuteNVGPU_CopyAtomIm2ColTmaStoreType + : CuteNVGPU_PayloadType<"CopyAtomIm2ColTmaStore", "atom.im2col_tma_store">; +def CuteNVGPU_CopyAtomSIMTMultimemLdReduceType + : CuteNVGPU_PayloadType<"CopyAtomSIMTMultimemLdReduce", + "atom.simt_multimem_ld_reduce">; +def CuteNVGPU_CopyAtomSIMTMultimemRedType + : CuteNVGPU_PayloadType<"CopyAtomSIMTMultimemRed", "atom.simt_multimem_red">; +def CuteNVGPU_CopyAtomSIMTMultimemStType + : CuteNVGPU_PayloadType<"CopyAtomSIMTMultimemSt", "atom.simt_multimem_st">; +def CuteNVGPU_CopyAtomSM100S2TCopyV2Type + : CuteNVGPU_PayloadType<"CopyAtomSM100S2TCopyV2", "atom.sm100_s2t_copy_v2">; +def CuteNVGPU_CopyAtomTmaLoadType + : CuteNVGPU_PayloadType<"CopyAtomTmaLoad", "atom.tma_load">; +def CuteNVGPU_CopyAtomTmaReduceType + : CuteNVGPU_PayloadType<"CopyAtomTmaReduce", "atom.tma_reduce">; +def CuteNVGPU_CopyAtomTmaStoreType + : CuteNVGPU_PayloadType<"CopyAtomTmaStore", "atom.tma_store">; +def CuteNVGPU_MmaAtomSM100UMMABlockScaledSparseType + : CuteNVGPU_PayloadType<"MmaAtomSM100UMMABlockScaledSparse", + "sm100.mma_bs_sp">; +def CuteNVGPU_MmaAtomSM100UMMASparseType + : CuteNVGPU_PayloadType<"MmaAtomSM100UMMASparse", "sm100.mma_sp">; +def CuteNVGPU_MmaAtomSM80SparseType + : CuteNVGPU_PayloadType<"MmaAtomSM80Sparse", "sm80.sparse_mma">; +def CuteNVGPU_SmemDescCircularSM103Type + : CuteNVGPU_PayloadType<"SmemDescCircularSM103", + "sm103.smem_desc_circular">; + +#endif // CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_TYPES_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Registration/Registration.h b/mlir/dialects/cute_ir/include/cute_ir/Registration/Registration.h new file mode 100755 index 00000000..2e27e123 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Registration/Registration.h @@ -0,0 +1,46 @@ +// clang-format off +/*************************************************************************************************** + * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +// clang-format on + +#ifndef CUTE_IR_REGISTRATION_REGISTRATION_H +#define CUTE_IR_REGISTRATION_REGISTRATION_H + +#include "mlir/IR/DialectRegistry.h" + +namespace mlir::cutlass_compiler::cute { + +void registerCuteDialects(DialectRegistry ®istry); +void registerCutePasses(); + +} // namespace mlir::cutlass_compiler::cute + +#endif // CUTE_IR_REGISTRATION_REGISTRATION_H diff --git a/mlir/dialects/cute_ir/lib/CAPI/BUILD.bazel b/mlir/dialects/cute_ir/lib/CAPI/BUILD.bazel new file mode 100644 index 00000000..448294dd --- /dev/null +++ b/mlir/dialects/cute_ir/lib/CAPI/BUILD.bazel @@ -0,0 +1,32 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "cute", + srcs = [ + "Cute.cpp", + "CuteAttributes.cpp", + "CuteTypes.cpp", + ], + deps = [ + "//mlir/dialects/cute_ir/include/cute_ir-c/Dialect:cute", + "//mlir/dialects/cute_ir/lib/Dialect/IR:dialect", + "@llvm-project//mlir:CAPIIR", + ], + alwayslink = True, +) + +cc_library( + name = "cute_nvgpu", + srcs = [ + "CuteNVGPU.cpp", + "CuteNVGPUTypes.cpp", + ], + deps = [ + "//mlir/dialects/cute_ir/include/cute_ir-c/Dialect:cute_nvgpu", + "//mlir/dialects/cute_ir/lib/Dialect/CuteNVGPU/IR:dialect", + "@llvm-project//mlir:CAPIIR", + ], + alwayslink = True, +) diff --git a/mlir/dialects/cute_ir/lib/CAPI/Cute.cpp b/mlir/dialects/cute_ir/lib/CAPI/Cute.cpp new file mode 100644 index 00000000..6956b2aa --- /dev/null +++ b/mlir/dialects/cute_ir/lib/CAPI/Cute.cpp @@ -0,0 +1,17 @@ +//===- Cute.cpp - C API for the CuTe dialect -----------------------------===// + +#include "cute_ir-c/Dialect/Cute.h" + +#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Cute, cute, + mlir::cutlass_compiler::cute::CuteDialect) + +bool mlirTypeIsACuteType(MlirType type) { + mlir::Type cppType = unwrap(type); + return cppType && + cppType.getDialect().getNamespace() == + mlir::cutlass_compiler::cute::CuteDialect::getDialectNamespace(); +} diff --git a/mlir/dialects/cute_ir/lib/CAPI/CuteAttributes.cpp b/mlir/dialects/cute_ir/lib/CAPI/CuteAttributes.cpp new file mode 100644 index 00000000..4b7b267c --- /dev/null +++ b/mlir/dialects/cute_ir/lib/CAPI/CuteAttributes.cpp @@ -0,0 +1,130 @@ +// C++ to C bindings for CuTe algebra attributes. + +#include "cute_ir-c/Dialect/CuteAttributes.h" + +#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Support.h" + +namespace cute = mlir::cutlass_compiler::cute; + +namespace { +template +MlirAttribute getAlgebraAttribute(MlirContext context, MlirStringRef value) { + auto *ctx = unwrap(context); + if (!ctx || (!value.data && value.length)) + return {nullptr}; + auto algebra = + cutegen::from_string(unwrap(value).str()); + if (!algebra) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(Attr::get(ctx, std::move(*algebra))); +} + +template MlirStringRef getAlgebraValue(MlirAttribute attr) { + auto value = llvm::cast(unwrap(attr)); + // Intern the serialization so the returned view outlives this function. + auto text = mlir::StringAttr::get(value.getContext(), + cutegen::to_string(value.getRef())); + return wrap(text.getValue()); +} +} // namespace + +MlirAttribute mlirCuteIntTupleAttrGet(MlirContext context, + MlirStringRef value) { + return getAlgebraAttribute(context, value); +} + +bool mlirAttributeIsACuteIntTuple(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteIntTupleAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteCoordAttrGet(MlirContext context, MlirStringRef value) { + return getAlgebraAttribute(context, value); +} + +bool mlirAttributeIsACuteCoord(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteCoordAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteShapeAttrGet(MlirContext context, MlirStringRef value) { + return getAlgebraAttribute(context, value); +} + +bool mlirAttributeIsACuteShape(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteShapeAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteStrideAttrGet(MlirContext context, MlirStringRef value) { + return getAlgebraAttribute(context, value); +} + +bool mlirAttributeIsACuteStride(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteStrideAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteLayoutAttrGet(MlirContext context, MlirStringRef value) { + return getAlgebraAttribute(context, value); +} + +bool mlirAttributeIsACuteLayout(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteLayoutAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteTileAttrGet(MlirContext context, MlirStringRef value) { + return getAlgebraAttribute(context, value); +} + +bool mlirAttributeIsACuteTile(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteTileAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteComposedLayoutAttrGet(MlirContext context, + MlirStringRef value) { + return getAlgebraAttribute(context, value); +} + +bool mlirAttributeIsACuteComposedLayout(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteComposedLayoutAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteSwizzleAttrGet(MlirContext context, MlirStringRef value) { + return getAlgebraAttribute(context, value); +} + +bool mlirAttributeIsACuteSwizzle(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteSwizzleAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} diff --git a/mlir/dialects/cute_ir/lib/CAPI/CuteNVGPU.cpp b/mlir/dialects/cute_ir/lib/CAPI/CuteNVGPU.cpp new file mode 100644 index 00000000..1de2f1df --- /dev/null +++ b/mlir/dialects/cute_ir/lib/CAPI/CuteNVGPU.cpp @@ -0,0 +1,19 @@ +//===- CuteNVGPU.cpp - C API for the CuTe NVIDIA GPU dialect ------------===// + +#include "cute_ir-c/Dialect/CuteNVGPU.h" + +#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h" +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION( + CuteNVGPU, cute_nvgpu, mlir::cutlass_compiler::cute_nvgpu::CuteNVGPUDialect) + +bool mlirTypeIsACuteNVGPUType(MlirType type) { + mlir::Type cppType = unwrap(type); + if (!cppType) + return false; + if (auto opaque = mlir::dyn_cast(cppType)) + return opaque.getDialectNamespace() == "cute_nvgpu"; + return cppType.getDialect().getNamespace() == "cute_nvgpu"; +} diff --git a/mlir/dialects/cute_ir/lib/CAPI/CuteNVGPUTypes.cpp b/mlir/dialects/cute_ir/lib/CAPI/CuteNVGPUTypes.cpp new file mode 100644 index 00000000..5072b643 --- /dev/null +++ b/mlir/dialects/cute_ir/lib/CAPI/CuteNVGPUTypes.cpp @@ -0,0 +1,1039 @@ +// C++ to C bindings for CuTe NVIDIA GPU types. +#include "cute_ir-c/Dialect/CuteNVGPUTypes.h" +#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h" +#include "mlir/CAPI/IR.h" + +namespace cute_nvgpu = mlir::cutlass_compiler::cute_nvgpu; + +bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyG2S(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomBulkCopyG2STypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomBulkCopyG2SType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2G(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomBulkCopyS2GType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2S(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomBulkCopyS2STypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomBulkCopyS2SType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomDsmemStore(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomDsmemStoreTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomDsmemStoreType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUCopyAtomDsmemStoreTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomG2R(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomG2RTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomG2RType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUCopyAtomG2RTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaLoad(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomIm2ColTmaLoadType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaStore(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomIm2ColTmaStoreType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomLdsm(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomLdsmTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomLdsmType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUCopyAtomLdsmTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomNonExec2DGather4TmaLoad(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap( + cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType::get(ctx, payloadValue)); +} +MlirAttribute +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaLoad(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap( + cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType::get(ctx, payloadValue)); +} +MlirAttribute +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaStore(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap( + cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType::get(ctx, payloadValue)); +} +MlirAttribute +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaLoad(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap( + cute_nvgpu::CopyAtomNonExecTiledTmaLoadType::get(ctx, payloadValue)); +} +MlirAttribute +mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaReduce(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap( + cute_nvgpu::CopyAtomNonExecTiledTmaReduceType::get(ctx, payloadValue)); +} +MlirAttribute +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaStore(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap( + cute_nvgpu::CopyAtomNonExecTiledTmaStoreType::get(ctx, payloadValue)); +} +MlirAttribute +mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomR2G(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomR2GTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomR2GType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUCopyAtomR2GTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomR2S(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomR2STypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomR2SType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUCopyAtomR2STypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomS2R(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomS2RTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomS2RType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUCopyAtomS2RTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSIMTAsyncCopy(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSIMTAsyncCopyType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemLdReduce(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap( + cute_nvgpu::CopyAtomSIMTMultimemLdReduceType::get(ctx, payloadValue)); +} +MlirAttribute +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemRed(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSIMTMultimemRedType::get(ctx, payloadValue)); +} +MlirAttribute +mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemSt(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSIMTMultimemStType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSIMTSyncCopy(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSIMTSyncCopyType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSM10xTmemLoadRed(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSM10xTmemLoadRedType::get(ctx, payloadValue)); +} +MlirAttribute +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSM100CopyS2T(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSM100CopyS2TType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSM100S2TCopyV2(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSM100S2TCopyV2Type::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSM100TmemLoad(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSM100TmemLoadType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSM100TmemStore(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSM100TmemStoreType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomStsm(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomStsmTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomStsmType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUCopyAtomStsmTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomTmaLoad(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomTmaLoadTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomTmaLoadType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUCopyAtomTmaLoadTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomTmaReduce(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomTmaReduceTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomTmaReduceType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUCopyAtomTmaReduceTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomTmaStore(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomTmaStoreTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomTmaStoreType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUCopyAtomTmaStoreTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPayload())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM80Sparse(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM80SparseTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM80SparseType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM80SparseTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM80(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM80TypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM80Type::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM80TypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPayload())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM89(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM89TypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM89Type::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM89TypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPayload())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM90(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM90TypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM90Type::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM90TypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPayload())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaledSparse(MlirType type) { + return llvm::isa_and_nonnull< + cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType>(unwrap(type)); +} +MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType::get( + ctx, payloadValue)); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaled(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap( + cute_nvgpu::MmaAtomSM100UMMABlockScaledType::get(ctx, payloadValue)); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMASparse(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM100UMMASparseType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMA(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMATypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM100UMMAType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM100UMMATypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPayload())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM120BlockScaled(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM120BlockScaledType::get(ctx, payloadValue)); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUSmemDescCircularSM103(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUSmemDescCircularSM103TypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::SmemDescCircularSM103Type::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUSmemDescCircularSM103TypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getPayload())); +} + +bool mlirTypeIsACuteNVGPUSmemDesc(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUSmemDescTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute_nvgpu::SmemDescType::get(ctx)); +} + +bool mlirTypeIsACuteNVGPUSmemDescView(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUSmemDescViewTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::SmemDescViewType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUSmemDescViewTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPayload())); +} + +bool mlirTypeIsACuteNVGPUTiledCopy(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUTiledCopyTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::TiledCopyType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUTiledCopyTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPayload())); +} + +bool mlirTypeIsACuteNVGPUTiledMma(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUTiledMmaTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::TiledMmaType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUTiledMmaTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPayload())); +} + +bool mlirTypeIsACuteNVGPUTmaDescriptorIm2Col(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUTmaDescriptorIm2ColTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute_nvgpu::TmaDescriptorIm2ColType::get(ctx)); +} + +bool mlirTypeIsACuteNVGPUTmaDescriptorTiled(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUTmaDescriptorTiledTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute_nvgpu::TmaDescriptorTiledType::get(ctx)); +} + +bool mlirTypeIsACuteNVGPUUniversalFmaAtom(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUUniversalFmaAtomTypeGet(MlirContext context, + MlirAttribute payload) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto payloadValue = + llvm::dyn_cast_if_present(unwrap(payload)); + if (!payloadValue || payloadValue.getContext() != ctx) + return {nullptr}; + return wrap(cute_nvgpu::UniversalFmaAtomType::get(ctx, payloadValue)); +} +MlirAttribute mlirCuteNVGPUUniversalFmaAtomTypeGetPayload(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPayload())); +} + +bool mlirTypeIsACuteNVGPUWorkIdResponse(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUWorkIdResponseTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute_nvgpu::WorkIdResponseType::get(ctx)); +} diff --git a/mlir/dialects/cute_ir/lib/CAPI/CuteTypes.cpp b/mlir/dialects/cute_ir/lib/CAPI/CuteTypes.cpp new file mode 100644 index 00000000..5172112a --- /dev/null +++ b/mlir/dialects/cute_ir/lib/CAPI/CuteTypes.cpp @@ -0,0 +1,358 @@ +// C++ to C bindings for CuTe types. +#include "cute_ir-c/Dialect/CuteTypes.h" +#include "cute_ir/Dialect/Cute/IR/CuteDialectPrivate.h" +#include "mlir/CAPI/IR.h" + +namespace cute = mlir::cutlass_compiler::cute; + +bool mlirTypeIsACuteArithTupleIterator(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteArithTupleIteratorTypeGet(MlirContext context, + MlirType arithTuple) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto arithTupleValue = + llvm::dyn_cast_if_present(unwrap(arithTuple)); + if (!arithTupleValue || arithTupleValue.getContext() != ctx) + return {nullptr}; + return wrap(cute::ArithTupleIteratorType::get(ctx, arithTupleValue)); +} +MlirType mlirCuteArithTupleIteratorTypeGetArithTuple(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getArithTuple())); +} + +bool mlirTypeIsACuteComposedLayout(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteComposedLayoutTypeGet(MlirContext context, + MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto attrValue = + llvm::dyn_cast_if_present(unwrap(attr)); + if (!attrValue || attrValue.getContext() != ctx) + return {nullptr}; + return wrap(cute::ComposedLayoutType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + attrValue)); +} +MlirAttribute mlirCuteComposedLayoutTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteConstrainedInt32(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteConstrainedInt32TypeGet(MlirContext context, + uint64_t divisibleBy) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute::ConstrainedInt32Type::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + divisibleBy)); +} +uint64_t mlirCuteConstrainedInt32TypeGetDivisibleBy(MlirType type) { + return llvm::cast(unwrap(type)).getDivisibleBy(); +} + +bool mlirTypeIsACuteConstrainedInt64(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteConstrainedInt64TypeGet(MlirContext context, + uint64_t divisibleBy) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute::ConstrainedInt64Type::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + divisibleBy)); +} +uint64_t mlirCuteConstrainedInt64TypeGetDivisibleBy(MlirType type) { + return llvm::cast(unwrap(type)).getDivisibleBy(); +} + +bool mlirTypeIsACuteCoordTensor(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteCoordTensorTypeGet(MlirContext context, MlirType arithTuple, + MlirType layout) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto arithTupleValue = + llvm::dyn_cast_if_present(unwrap(arithTuple)); + if (!arithTupleValue || arithTupleValue.getContext() != ctx) + return {nullptr}; + auto layoutValue = unwrap(layout); + if (!layoutValue || layoutValue.getContext() != ctx) + return {nullptr}; + return wrap(cute::CoordTensorType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + arithTupleValue, layoutValue)); +} +MlirType mlirCuteCoordTensorTypeGetArithTuple(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getArithTuple())); +} +MlirType mlirCuteCoordTensorTypeGetLayout(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getLayout())); +} + +bool mlirTypeIsACuteCoord(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteCoordTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto attrValue = llvm::dyn_cast_if_present(unwrap(attr)); + if (!attrValue || attrValue.getContext() != ctx) + return {nullptr}; + return wrap(cute::CoordType::get(ctx, attrValue)); +} +MlirAttribute mlirCuteCoordTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteFastDivmodDivisor(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteFastDivmodDivisorTypeGet(MlirContext context, unsigned width, + bool isPow2) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute::FastDivmodDivisorType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, width, + isPow2)); +} +unsigned mlirCuteFastDivmodDivisorTypeGetWidth(MlirType type) { + return llvm::cast(unwrap(type)).getWidth(); +} +bool mlirCuteFastDivmodDivisorTypeGetIsPow2(MlirType type) { + return llvm::cast(unwrap(type)).getIsPow2(); +} + +bool mlirTypeIsACuteIntTuple(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteIntTupleTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto attrValue = llvm::dyn_cast_if_present(unwrap(attr)); + if (!attrValue || attrValue.getContext() != ctx) + return {nullptr}; + return wrap(cute::IntTupleType::get(ctx, attrValue)); +} +MlirAttribute mlirCuteIntTupleTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteLayout(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteLayoutTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto attrValue = llvm::dyn_cast_if_present(unwrap(attr)); + if (!attrValue || attrValue.getContext() != ctx) + return {nullptr}; + return wrap(cute::LayoutType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + attrValue)); +} +MlirAttribute mlirCuteLayoutTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteMemRef(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteMemRefTypeGet(MlirContext context, MlirType ptr, + MlirType layout) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto ptrValue = llvm::dyn_cast_if_present(unwrap(ptr)); + if (!ptrValue || ptrValue.getContext() != ctx) + return {nullptr}; + auto layoutValue = unwrap(layout); + if (!layoutValue || layoutValue.getContext() != ctx) + return {nullptr}; + return wrap(cute::MemRefType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + ptrValue, layoutValue)); +} +MlirType mlirCuteMemRefTypeGetPtr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPtr())); +} +MlirType mlirCuteMemRefTypeGetLayout(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getLayout())); +} + +bool mlirTypeIsACutePtr(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCutePtrTypeGet(MlirContext context, MlirType valueType, + MlirAttribute memorySpace, uint64_t alignment, + MlirAttribute swizzle) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueTypeValue = unwrap(valueType); + if (valueType.ptr && (!valueTypeValue || valueTypeValue.getContext() != ctx)) + return {nullptr}; + auto memorySpaceValue = + llvm::dyn_cast_if_present(unwrap(memorySpace)); + if (!memorySpaceValue || memorySpaceValue.getContext() != ctx) + return {nullptr}; + auto swizzleValue = + llvm::dyn_cast_if_present(unwrap(swizzle)); + if (swizzle.ptr && (!swizzleValue || swizzleValue.getContext() != ctx)) + return {nullptr}; + return wrap(cute::PtrType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valueTypeValue, memorySpaceValue, alignment, swizzleValue)); +} +MlirType mlirCutePtrTypeGetValueType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValueType())); +} +MlirAttribute mlirCutePtrTypeGetMemorySpace(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getMemorySpace())); +} +uint64_t mlirCutePtrTypeGetAlignment(MlirType type) { + return llvm::cast(unwrap(type)).getAlignment(); +} +MlirAttribute mlirCutePtrTypeGetSwizzle(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getSwizzle())); +} + +bool mlirTypeIsACuteShape(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteShapeTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto attrValue = llvm::dyn_cast_if_present(unwrap(attr)); + if (!attrValue || attrValue.getContext() != ctx) + return {nullptr}; + return wrap(cute::ShapeType::get(ctx, attrValue)); +} +MlirAttribute mlirCuteShapeTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteSparseElem(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteSparseElemTypeGet(MlirContext context, int numLogical, + MlirType physicalType) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto physicalTypeValue = unwrap(physicalType); + if (!physicalTypeValue || physicalTypeValue.getContext() != ctx) + return {nullptr}; + return wrap(cute::SparseElemType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + numLogical, physicalTypeValue)); +} +int mlirCuteSparseElemTypeGetNumLogical(MlirType type) { + return llvm::cast(unwrap(type)).getNumLogical(); +} +MlirType mlirCuteSparseElemTypeGetPhysicalType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPhysicalType())); +} + +bool mlirTypeIsACuteStride(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteStrideTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto attrValue = llvm::dyn_cast_if_present(unwrap(attr)); + if (!attrValue || attrValue.getContext() != ctx) + return {nullptr}; + return wrap(cute::StrideType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + attrValue)); +} +MlirAttribute mlirCuteStrideTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteSwizzle(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteSwizzleTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto attrValue = llvm::dyn_cast_if_present(unwrap(attr)); + if (!attrValue || attrValue.getContext() != ctx) + return {nullptr}; + return wrap(cute::SwizzleType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + attrValue)); +} +MlirAttribute mlirCuteSwizzleTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteTile(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteTileTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto attrValue = llvm::dyn_cast_if_present(unwrap(attr)); + if (!attrValue || attrValue.getContext() != ctx) + return {nullptr}; + return wrap(cute::TileType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + attrValue)); +} +MlirAttribute mlirCuteTileTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} diff --git a/mlir/dialects/cute_ir/lib/Dialect/CuteNVGPU/IR/BUILD.bazel b/mlir/dialects/cute_ir/lib/Dialect/CuteNVGPU/IR/BUILD.bazel new file mode 100644 index 00000000..f6f289f3 --- /dev/null +++ b/mlir/dialects/cute_ir/lib/Dialect/CuteNVGPU/IR/BUILD.bazel @@ -0,0 +1,12 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "dialect", + srcs = ["CuteNVGPUDialect.cpp"], + deps = [ + "//mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR:headers", + "@llvm-project//mlir:IR", + ], +) diff --git a/mlir/dialects/cute_ir/lib/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.cpp b/mlir/dialects/cute_ir/lib/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.cpp new file mode 100644 index 00000000..5a2086a1 --- /dev/null +++ b/mlir/dialects/cute_ir/lib/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.cpp @@ -0,0 +1,135 @@ +//===- CuteNVGPUDialect.cpp - CuTe NVIDIA GPU dialect --------------------===// + +#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h" + +#include "mlir/IR/Builders.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/DialectImplementation.h" + +#include "llvm/ADT/TypeSwitch.h" + +using namespace mlir; +using namespace mlir::cutlass_compiler::cute_nvgpu; + +#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.cpp.inc" + +#define GET_TYPEDEF_CLASSES +#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUTypes.cpp.inc" + +#define GET_OP_CLASSES +#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUOps.cpp.inc" + +#define CUTE_NVGPU_PAYLOAD_TYPES(M) \ + M(CopyAtomBulkCopyG2SType) \ + M(CopyAtomBulkCopyS2GType) \ + M(CopyAtomBulkCopyS2SType) \ + M(CopyAtomDsmemStoreType) \ + M(CopyAtomG2RType) \ + M(CopyAtomLdsmType) \ + M(CopyAtomNonExec2DGather4TmaLoadType) \ + M(CopyAtomNonExecIm2ColTmaLoadType) \ + M(CopyAtomNonExecIm2ColTmaStoreType) \ + M(CopyAtomNonExecTiledTmaLoadType) \ + M(CopyAtomNonExecTiledTmaReduceType) \ + M(CopyAtomNonExecTiledTmaStoreType) \ + M(CopyAtomR2GType) \ + M(CopyAtomR2SType) \ + M(CopyAtomS2RType) \ + M(CopyAtomSIMTAsyncCopyType) \ + M(CopyAtomSIMTSyncCopyType) \ + M(CopyAtomSM100CopyS2TType) \ + M(CopyAtomSM100TmemLoadType) \ + M(CopyAtomSM100TmemStoreType) \ + M(CopyAtomSM10xTmemLoadRedType) \ + M(CopyAtomStsmType) \ + M(MmaAtomSM100UMMABlockScaledType) \ + M(MmaAtomSM100UMMAType) \ + M(MmaAtomSM120BlockScaledType) \ + M(MmaAtomSM80Type) \ + M(MmaAtomSM89Type) \ + M(MmaAtomSM90Type) \ + M(SmemDescViewType) \ + M(TiledCopyType) \ + M(TiledMmaType) \ + M(UniversalFmaAtomType) \ + M(CopyAtomIm2ColTmaLoadType) \ + M(CopyAtomIm2ColTmaStoreType) \ + M(CopyAtomSIMTMultimemLdReduceType) \ + M(CopyAtomSIMTMultimemRedType) \ + M(CopyAtomSIMTMultimemStType) \ + M(CopyAtomSM100S2TCopyV2Type) \ + M(CopyAtomTmaLoadType) \ + M(CopyAtomTmaReduceType) \ + M(CopyAtomTmaStoreType) \ + M(MmaAtomSM100UMMABlockScaledSparseType) \ + M(MmaAtomSM100UMMASparseType) \ + M(MmaAtomSM80SparseType) \ + M(SmemDescCircularSM103Type) + +#define DEFINE_PAYLOAD_TYPE(TYPE) \ + Type TYPE::parse(AsmParser &parser) { \ + parser.emitError(parser.getCurrentLocation(), \ + "payload types are parsed by CuteNVGPUDialect"); \ + return {}; \ + } \ + void TYPE::print(AsmPrinter &printer) const { \ + printer << getPayload().getValue(); \ + } +CUTE_NVGPU_PAYLOAD_TYPES(DEFINE_PAYLOAD_TYPE) +#undef DEFINE_PAYLOAD_TYPE + +template +static Type maybeParsePayloadType(MLIRContext *context, StringRef spec) { + StringRef remainder = spec; + if (!remainder.consume_front(TypeT::getMnemonic())) + return {}; + if (!remainder.empty() && !remainder.starts_with("<")) + return {}; + return TypeT::get(context, StringAttr::get(context, remainder)); +} + +Type CuteNVGPUDialect::parseType(DialectAsmParser &parser) const { + StringRef spec = parser.getFullSymbolSpec(); + if (spec == SmemDescType::getMnemonic() || + spec == TmaDescriptorTiledType::getMnemonic() || + spec == TmaDescriptorIm2ColType::getMnemonic() || + spec == WorkIdResponseType::getMnemonic()) { + StringRef mnemonic; + Type type; + OptionalParseResult result = generatedTypeParser(parser, &mnemonic, type); + return result.has_value() && succeeded(*result) ? type : Type(); + } + +#define TRY_PAYLOAD_TYPE(TYPE) \ + if (Type type = maybeParsePayloadType(getContext(), spec)) \ + return type; + CUTE_NVGPU_PAYLOAD_TYPES(TRY_PAYLOAD_TYPE) +#undef TRY_PAYLOAD_TYPE + + StringAttr dialectNamespace = + StringAttr::get(getContext(), getDialectNamespace()); + return OpaqueType::get(dialectNamespace, spec); +} + +void CuteNVGPUDialect::printType(Type type, DialectAsmPrinter &printer) const { + if (succeeded(generatedTypePrinter(type, printer))) + return; + llvm_unreachable("unexpected non-CuteNVGPU type"); +} + +void CuteNVGPUDialect::initialize() { + addTypes< +#define GET_TYPEDEF_LIST +#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUTypes.cpp.inc" + >(); + addOperations< +#define GET_OP_LIST +#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUOps.cpp.inc" + >(); + + // Preserve every not-yet-structured private type spelling exactly. MLIR's + // default hook stores the complete symbol specification in OpaqueType. + allowUnknownTypes(); +} + +#undef CUTE_NVGPU_PAYLOAD_TYPES diff --git a/mlir/dialects/cute_ir/lib/Dialect/IR/BUILD.bazel b/mlir/dialects/cute_ir/lib/Dialect/IR/BUILD.bazel new file mode 100644 index 00000000..3ccc2f2d --- /dev/null +++ b/mlir/dialects/cute_ir/lib/Dialect/IR/BUILD.bazel @@ -0,0 +1,26 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "dialect", + srcs = [ + "CuteDialect.cpp", + "CuteOpsPrivate.cpp", + "CuteTypes.cpp", + "CuteTypesPrivate.cpp", + ], + deps = [ + "//mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR:private_headers", + "//mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR:types_all_defs_inc_gen", + "@cute_ir//:dialect_support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:MathDialect", + "@llvm-project//mlir:NVVMDialect", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:UBDialect", + ], +) diff --git a/mlir/dialects/cute_ir/lib/Dialect/IR/CuteDialect.cpp b/mlir/dialects/cute_ir/lib/Dialect/IR/CuteDialect.cpp new file mode 100755 index 00000000..083f0347 --- /dev/null +++ b/mlir/dialects/cute_ir/lib/Dialect/IR/CuteDialect.cpp @@ -0,0 +1,59 @@ +// clang-format off +/*************************************************************************************************** + * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +// clang-format on + +#include "cute_ir/Dialect/Cute/IR/CuteDialectPrivate.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/IR/Builders.h" +#include "mlir/IR/DialectImplementation.h" + +using namespace mlir; +using namespace mlir::cutlass_compiler::cute; + +#include "cute_ir/Dialect/Cute/IR/CuteDialect.cpp.inc" +#include "cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.cpp.inc" + +void CuteDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "cute_ir/Dialect/Cute/IR/CuteOps.cpp.inc" + >(); + addOperations< +#define GET_OP_LIST +#include "cute_ir/Dialect/Cute/IR/CuteOpsPrivate.cpp.inc" + >(); + CuteDialect::registerCuteAttributes(this); + CuteDialect::registerCuteTypes(this); +} + +// registerCuteAttributes is implemented in CuteAttributes.cpp. diff --git a/mlir/dialects/cute_ir/lib/Dialect/IR/CuteOpsPrivate.cpp b/mlir/dialects/cute_ir/lib/Dialect/IR/CuteOpsPrivate.cpp new file mode 100644 index 00000000..3492d685 --- /dev/null +++ b/mlir/dialects/cute_ir/lib/Dialect/IR/CuteOpsPrivate.cpp @@ -0,0 +1,6 @@ +//===- CutePrivateCompatOps.cpp - recovered CuTe operations --------------===// + +#include "cute_ir/Dialect/Cute/IR/CuteDialectPrivate.h" + +#define GET_OP_CLASSES +#include "cute_ir/Dialect/Cute/IR/CuteOpsPrivate.cpp.inc" diff --git a/mlir/dialects/cute_ir/lib/Dialect/IR/CuteTypes.cpp b/mlir/dialects/cute_ir/lib/Dialect/IR/CuteTypes.cpp new file mode 100644 index 00000000..039f341c --- /dev/null +++ b/mlir/dialects/cute_ir/lib/Dialect/IR/CuteTypes.cpp @@ -0,0 +1,5 @@ +//===- CuteTypes.cpp - combined OSS and recovered CuTe types -------------===// + +#include "cute_ir/Dialect/Cute/IR/CuteDialectPrivate.h" + +#include "cute_ir/lib/Dialect/IR/CuteTypes.cpp" diff --git a/mlir/dialects/cute_ir/lib/Dialect/IR/CuteTypesPrivate.cpp b/mlir/dialects/cute_ir/lib/Dialect/IR/CuteTypesPrivate.cpp new file mode 100644 index 00000000..676b2340 --- /dev/null +++ b/mlir/dialects/cute_ir/lib/Dialect/IR/CuteTypesPrivate.cpp @@ -0,0 +1,333 @@ +#include "cute_ir/Dialect/Cute/IR/CuteDialectPrivate.h" + +#include "mlir/IR/DialectImplementation.h" + +#include "llvm/Support/MathExtras.h" + +using namespace mlir; +using namespace mlir::cutlass_compiler::cute; + +//===----------------------------------------------------------------------===// +// CuTe-DSL runtime and tensor-view types +//===----------------------------------------------------------------------===// + +static IntTupleType parseIntTuplePayload(AsmParser &parser) { + std::string text; + if (failed(parser.parseString(&text))) + return {}; + auto value = cutegen::from_string(text); + if (!value) { + parser.emitError(parser.getCurrentLocation(), + "failed to parse arithmetic tuple from \"") + << text << '"'; + return {}; + } + return IntTupleType::get(parser.getContext(), std::move(*value)); +} + +static Type parseLayoutPayload(AsmParser &parser) { + std::string text; + if (failed(parser.parseString(&text))) + return {}; + if (auto value = cutegen::from_string(text)) + return LayoutType::get(parser.getContext(), std::move(*value)); + if (auto value = cutegen::from_string(text)) + return ComposedLayoutType::get(parser.getContext(), std::move(*value)); + parser.emitError(parser.getCurrentLocation(), "failed to parse layout from \"") + << text << '"'; + return {}; +} + +static void printLayoutPayload(AsmPrinter &printer, Type layout) { + if (auto plain = llvm::dyn_cast(layout)) { + printer.printString(cutegen::to_string(plain.getRef())); + return; + } + auto composed = llvm::cast(layout); + printer.printString(cutegen::to_string(composed.getRef())); +} + +Type ArithTupleIteratorType::parse(AsmParser &parser) { + if (failed(parser.parseLess())) + return {}; + IntTupleType tuple = parseIntTuplePayload(parser); + if (!tuple || failed(parser.parseGreater())) + return {}; + return get(parser.getContext(), tuple); +} + +void ArithTupleIteratorType::print(AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getArithTuple().getRef())); + printer << '>'; +} + +template +static Type parseConstrainedInt(AsmParser &parser) { + uint64_t divisibleBy = 1; + if (succeeded(parser.parseOptionalLess())) { + if (failed(parser.parseKeyword("divby")) || + failed(parser.parseInteger(divisibleBy)) || + failed(parser.parseGreater())) + return {}; + } + return ConstrainedType::getChecked( + [&] { return parser.emitError(parser.getCurrentLocation()); }, + parser.getContext(), divisibleBy); +} + +template +static void printConstrainedInt(ConstrainedType type, AsmPrinter &printer) { + if (type.getDivisibleBy() != 1) + printer << "'; +} + +Type ConstrainedInt32Type::parse(AsmParser &parser) { + return parseConstrainedInt(parser); +} + +void ConstrainedInt32Type::print(AsmPrinter &printer) const { + printConstrainedInt(*this, printer); +} + +LogicalResult ConstrainedInt32Type::verify( + function_ref emitError, uint64_t divisibleBy) { + return divisibleBy > 0 ? success() + : emitError() << "expects divisible_by > 0"; +} + +Type ConstrainedInt64Type::parse(AsmParser &parser) { + return parseConstrainedInt(parser); +} + +void ConstrainedInt64Type::print(AsmPrinter &printer) const { + printConstrainedInt(*this, printer); +} + +LogicalResult ConstrainedInt64Type::verify( + function_ref emitError, uint64_t divisibleBy) { + return divisibleBy > 0 ? success() + : emitError() << "expects divisible_by > 0"; +} + +Type FastDivmodDivisorType::parse(AsmParser &parser) { + unsigned width; + if (failed(parser.parseLess()) || failed(parser.parseInteger(width))) + return {}; + bool isPow2 = false; + if (succeeded(parser.parseOptionalComma())) { + if (failed(parser.parseKeyword("is_pow_2"))) + return {}; + isPow2 = true; + } + if (failed(parser.parseGreater())) + return {}; + return getChecked([&] { return parser.emitError(parser.getCurrentLocation()); }, + parser.getContext(), width, isPow2); +} + +void FastDivmodDivisorType::print(AsmPrinter &printer) const { + printer << '<' << getWidth(); + if (getIsPow2()) + printer << ", is_pow_2"; + printer << '>'; +} + +LogicalResult FastDivmodDivisorType::verify( + function_ref emitError, unsigned width, + bool /*isPow2*/) { + return width == 32 || width == 64 + ? success() + : emitError() << "expects width to be 32 or 64, got " << width; +} + +LogicalResult SparseElemType::verify( + function_ref emitError, int numLogical, + Type physicalType) { + if (numLogical <= 0) + return emitError() << "expects num_logical > 0"; + if (!physicalType) + return emitError() << "expects a physical storage type"; + return success(); +} + +namespace { +struct ParsedPtrFields { + Type valueType; + StringAttr memorySpace; + uint64_t alignment = 0; + SwizzleAttr swizzle; +}; +} // namespace + +static FailureOr parseInlineSwizzle(AsmParser &parser) { + int64_t bits, base, shift; + if (failed(parser.parseLess()) || failed(parser.parseInteger(bits)) || + failed(parser.parseComma()) || failed(parser.parseInteger(base)) || + failed(parser.parseComma()) || failed(parser.parseInteger(shift)) || + failed(parser.parseGreater())) + return failure(); + std::string text = "S<" + std::to_string(bits) + "," + + std::to_string(base) + "," + std::to_string(shift) + ">"; + auto value = cutegen::from_string(text); + if (!value) { + parser.emitError(parser.getCurrentLocation(), "invalid pointer swizzle ") + << text; + return failure(); + } + return SwizzleAttr::get(parser.getContext(), std::move(*value)); +} + +static FailureOr parsePtrFields(AsmParser &parser, + bool expectLayout, + Type *layout) { + ParsedPtrFields fields; + OptionalParseResult valueResult = parser.parseOptionalType(fields.valueType); + if (valueResult.has_value() && failed(*valueResult)) + return failure(); + if (valueResult.has_value() && failed(parser.parseComma())) + return failure(); + + StringRef memorySpace; + if (failed(parser.parseKeyword(&memorySpace))) + return failure(); + fields.memorySpace = StringAttr::get(parser.getContext(), memorySpace); + + while (succeeded(parser.parseOptionalComma())) { + if (succeeded(parser.parseOptionalKeyword("align"))) { + if (failed(parser.parseLess()) || + failed(parser.parseInteger(fields.alignment)) || + failed(parser.parseGreater())) + return failure(); + continue; + } + if (succeeded(parser.parseOptionalKeyword("S"))) { + FailureOr swizzle = parseInlineSwizzle(parser); + if (failed(swizzle)) + return failure(); + fields.swizzle = *swizzle; + continue; + } + if (expectLayout && layout && !*layout) { + *layout = parseLayoutPayload(parser); + if (!*layout) + return failure(); + continue; + } + parser.emitError(parser.getCurrentLocation(), + "expected align, swizzle, or layout"); + return failure(); + } + return fields; +} + +static void printPtrFields(AsmPrinter &printer, PtrType pointer) { + if (pointer.getValueType()) { + printer.printType(pointer.getValueType()); + printer << ", "; + } + printer << pointer.getMemorySpace().getValue(); + if (pointer.getAlignment() != 0) + printer << ", align<" << pointer.getAlignment() << '>'; + if (pointer.getSwizzle()) + printer << ", " << cutegen::to_string(pointer.getSwizzle().getRef()); +} + +Type PtrType::parse(AsmParser &parser) { + if (failed(parser.parseLess())) + return {}; + FailureOr fields = parsePtrFields(parser, false, nullptr); + if (failed(fields) || failed(parser.parseGreater())) + return {}; + return getChecked([&] { return parser.emitError(parser.getCurrentLocation()); }, + parser.getContext(), fields->valueType, + fields->memorySpace, fields->alignment, fields->swizzle); +} + +void PtrType::print(AsmPrinter &printer) const { + printer << '<'; + printPtrFields(printer, *this); + printer << '>'; +} + +LogicalResult PtrType::verify(function_ref emitError, + Type /*valueType*/, StringAttr memorySpace, + uint64_t alignment, SwizzleAttr /*swizzle*/) { + if (!memorySpace) + return emitError() << "expects an address space"; + StringRef space = memorySpace.getValue(); + if (space != "generic" && space != "gmem" && space != "cmem" && + space != "smem" && space != "rmem" && space != "tmem" && + space != "dsmem") + return emitError() << "invalid address space '" << space << "'"; + if (alignment != 0 && !llvm::isPowerOf2_64(alignment)) + return emitError() << "expects power-of-two alignment, got " << alignment; + return success(); +} + +Type mlir::cutlass_compiler::cute::MemRefType::parse(AsmParser &parser) { + if (failed(parser.parseLess())) + return {}; + Type layout; + FailureOr fields = parsePtrFields(parser, true, &layout); + if (failed(fields) || !layout || failed(parser.parseGreater())) + return {}; + PtrType pointer = PtrType::getChecked( + [&] { return parser.emitError(parser.getCurrentLocation()); }, + parser.getContext(), fields->valueType, fields->memorySpace, + fields->alignment, fields->swizzle); + if (!pointer) + return {}; + return getChecked([&] { return parser.emitError(parser.getCurrentLocation()); }, + parser.getContext(), pointer, layout); +} + +void mlir::cutlass_compiler::cute::MemRefType::print( + AsmPrinter &printer) const { + printer << '<'; + printPtrFields(printer, getPtr()); + printer << ", "; + printLayoutPayload(printer, getLayout()); + printer << '>'; +} + +LogicalResult mlir::cutlass_compiler::cute::MemRefType::verify( + function_ref emitError, PtrType pointer, + Type layout) { + if (!pointer) + return emitError() << "expects a CuTe pointer"; + if (!llvm::isa(layout)) + return emitError() << "expects a CuTe layout or composed layout"; + return success(); +} + +Type CoordTensorType::parse(AsmParser &parser) { + if (failed(parser.parseLess())) + return {}; + IntTupleType tuple = parseIntTuplePayload(parser); + if (!tuple || failed(parser.parseComma())) + return {}; + Type layout = parseLayoutPayload(parser); + if (!layout || failed(parser.parseGreater())) + return {}; + return getChecked([&] { return parser.emitError(parser.getCurrentLocation()); }, + parser.getContext(), tuple, layout); +} + +void CoordTensorType::print(AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getArithTuple().getRef())); + printer << ", "; + printLayoutPayload(printer, getLayout()); + printer << '>'; +} + +LogicalResult CoordTensorType::verify( + function_ref emitError, IntTupleType tuple, + Type layout) { + if (!tuple) + return emitError() << "expects an arithmetic tuple"; + if (!llvm::isa(layout)) + return emitError() << "expects a CuTe layout or composed layout"; + return success(); +} diff --git a/mlir/dialects/cute_ir/lib/Registration/RegisterCute.cpp b/mlir/dialects/cute_ir/lib/Registration/RegisterCute.cpp new file mode 100755 index 00000000..c382e735 --- /dev/null +++ b/mlir/dialects/cute_ir/lib/Registration/RegisterCute.cpp @@ -0,0 +1,63 @@ +// clang-format off +/*************************************************************************************************** + * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +// clang-format on + +#include "cute_ir/Conversion/CuteToBase/Passes.h" +#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" +#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h" +#include "cute_ir/Dialect/Cute/Transforms/Passes.h" +#include "cute_ir/Registration/Registration.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Dialect/GPU/IR/GPUDialect.h" +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/Dialect/Math/IR/Math.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/Dialect/UB/IR/UBOps.h" + +using namespace mlir; + +namespace mlir::cutlass_compiler::cute { + +void registerCuteDialects(DialectRegistry ®istry) { + registry.insert(); +} + +void registerCutePasses() { + registerCuteTransformsPasses(); + registerCuteConversionPasses(); +} + +} // namespace mlir::cutlass_compiler::cute diff --git a/mlir/dialects/cute_ir/test/BUILD.bazel b/mlir/dialects/cute_ir/test/BUILD.bazel new file mode 100644 index 00000000..4f1d7484 --- /dev/null +++ b/mlir/dialects/cute_ir/test/BUILD.bazel @@ -0,0 +1,23 @@ +load("@rules_cc//cc:defs.bzl", "cc_test") + +package(default_visibility = ["//visibility:public"]) + +cc_test( + name = "type_constructors_test", + srcs = ["TypeConstructorsTest.c"], + deps = [ + "//mlir/dialects/cute_ir/lib/CAPI:cute", + "//mlir/dialects/cute_ir/lib/CAPI:cute_nvgpu", + "@llvm-project//mlir:CAPIIR", + ], +) + +cc_test( + name = "capi_registration_test", + srcs = ["CAPIRegistrationTest.cpp"], + deps = [ + "//mlir/dialects/cute_ir/lib/CAPI:cute", + "//mlir/dialects/cute_ir/lib/CAPI:cute_nvgpu", + "@llvm-project//mlir:CAPIIR", + ], +) diff --git a/mlir/dialects/cute_ir/test/CAPIRegistrationTest.cpp b/mlir/dialects/cute_ir/test/CAPIRegistrationTest.cpp new file mode 100644 index 00000000..8e0496c7 --- /dev/null +++ b/mlir/dialects/cute_ir/test/CAPIRegistrationTest.cpp @@ -0,0 +1,96 @@ +//===- CAPIRegistrationTest.cpp - CuTe C API smoke test ------------------===// + +#include "cute_ir-c/Dialect/Cute.h" +#include "cute_ir-c/Dialect/CuteNVGPU.h" +#include "mlir-c/IR.h" + +#include +#include + +namespace { + +int fail(MlirContext context, const char *message) { + std::fprintf(stderr, "%s\n", message); + mlirContextDestroy(context); + return 1; +} + +void appendString(MlirStringRef chunk, void *userData) { + static_cast(userData)->append(chunk.data, chunk.length); +} + +} // namespace + +int main() { + MlirContext context = mlirContextCreate(); + + MlirDialectHandle cute = mlirGetDialectHandle__cute__(); + MlirDialectHandle cuteNVGPU = mlirGetDialectHandle__cute_nvgpu__(); + mlirDialectHandleRegisterDialect(cute, context); + mlirDialectHandleRegisterDialect(cuteNVGPU, context); + + if (mlirDialectIsNull(mlirDialectHandleLoadDialect(cute, context))) + return fail(context, "failed to load the cute dialect"); + if (mlirDialectIsNull(mlirDialectHandleLoadDialect(cuteNVGPU, context))) + return fail(context, "failed to load the cute_nvgpu dialect"); + + MlirType cutePublicType = mlirTypeParseGet( + context, mlirStringRefCreateFromCString("!cute.int_tuple<\"1\">")); + if (mlirTypeIsNull(cutePublicType) || !mlirTypeIsACuteType(cutePublicType)) + return fail(context, "failed to parse an OSS cute type"); + + MlirType cutePrivateType = + mlirTypeParseGet(context, mlirStringRefCreateFromCString("!cute.i32")); + if (mlirTypeIsNull(cutePrivateType) || !mlirTypeIsACuteType(cutePrivateType)) + return fail(context, "failed to parse a recovered cute type"); + + MlirType cuteNVGPUType = mlirTypeParseGet( + context, mlirStringRefCreateFromCString("!cute_nvgpu.smem_desc")); + if (mlirTypeIsNull(cuteNVGPUType) || !mlirTypeIsACuteNVGPUType(cuteNVGPUType)) + return fail(context, "failed to parse a cute_nvgpu type"); + + if (!mlirContextIsRegisteredOperation( + context, mlirStringRefCreateFromCString("cute.add_offset"))) + return fail(context, "recovered cute operation is not registered"); + if (!mlirContextIsRegisteredOperation( + context, + mlirStringRefCreateFromCString("cute_nvgpu.arch.alloc_rmem"))) + return fail(context, "cute_nvgpu operation is not registered"); + + const char *privateOpsModule = R"mlir( +module { + %alloc = "cute.memref.alloc_smem"() : () -> !cute.i32 + %assumed = "cute.assume"(%alloc) : (!cute.i32) -> !cute.i32 + %tuple = "cute.make_tuple"(%alloc, %assumed) + : (!cute.i32, !cute.i32) -> !cute.i32 + "cute.copy"(%tuple, %alloc, %assumed) + <{operandSegmentSizes = array}> + : (!cute.i32, !cute.i32, !cute.i32) -> () +} +)mlir"; + MlirModule module = mlirModuleCreateParse( + context, mlirStringRefCreateFromCString(privateOpsModule)); + if (mlirModuleIsNull(module)) + return fail(context, "failed to parse recovered cute operations"); + bool privateOpsAreValid = mlirOperationVerify(mlirModuleGetOperation(module)); + std::string bytecode; + mlirOperationWriteBytecode(mlirModuleGetOperation(module), appendString, + &bytecode); + mlirModuleDestroy(module); + if (!privateOpsAreValid) + return fail(context, "recovered cute operations failed verification"); + + MlirOperation roundTripped = mlirOperationCreateParse( + context, mlirStringRefCreate(bytecode.data(), bytecode.size()), + mlirStringRefCreateFromCString("private-ops.mlirbc")); + if (mlirOperationIsNull(roundTripped)) + return fail(context, "failed to read recovered operations from bytecode"); + bool bytecodeIsValid = mlirOperationVerify(roundTripped); + mlirOperationDestroy(roundTripped); + if (!bytecodeIsValid) + return fail(context, + "recovered operations failed after bytecode round trip"); + + mlirContextDestroy(context); + return 0; +} diff --git a/mlir/dialects/cute_ir/test/TypeConstructorsTest.c b/mlir/dialects/cute_ir/test/TypeConstructorsTest.c new file mode 100644 index 00000000..317a7193 --- /dev/null +++ b/mlir/dialects/cute_ir/test/TypeConstructorsTest.c @@ -0,0 +1,70 @@ +#include "cute_ir-c/Dialect/Cute.h" +#include "cute_ir-c/Dialect/CuteNVGPU.h" +#include "mlir-c/BuiltinAttributes.h" +#include "mlir-c/BuiltinTypes.h" + +#include +#include + +#define CHECK(condition) \ + do { \ + if (!(condition)) { \ + fprintf(stderr, "failed at line %d: %s\n", __LINE__, #condition); \ + mlirContextDestroy(context); \ + return 1; \ + } \ + } while (0) + +int main(void) { + MlirContext context = mlirContextCreate(); + MlirType nullType = {NULL}; + MlirAttribute nullAttr = {NULL}; + MlirType f32 = mlirF32TypeGet(context); + MlirAttribute space = + mlirStringAttrGet(context, mlirStringRefCreateFromCString("smem")); + MlirAttribute layoutAttr = + mlirCuteLayoutAttrGet(context, mlirStringRefCreateFromCString("4:1")); + CHECK(mlirAttributeIsACuteLayout(layoutAttr)); + CHECK(!mlirAttributeIsACuteLayout(nullAttr)); + CHECK(!mlirAttributeIsACuteLayout(space)); + MlirStringRef layoutValue = mlirCuteLayoutAttrGetValue(layoutAttr); + CHECK(layoutValue.length == 3 && memcmp(layoutValue.data, "4:1", 3) == 0); + CHECK(mlirAttributeIsNull(mlirCuteLayoutAttrGet( + context, mlirStringRefCreateFromCString("invalid")))); + + // Constructors load their dialect and build storage without parsing a type. + MlirType pointer = mlirCutePtrTypeGet(context, f32, space, 16, nullAttr); + CHECK(mlirTypeIsACutePtr(pointer)); + CHECK(mlirTypeEqual(mlirCutePtrTypeGetValueType(pointer), f32)); + CHECK(mlirAttributeEqual(mlirCutePtrTypeGetMemorySpace(pointer), space)); + CHECK(mlirCutePtrTypeGetAlignment(pointer) == 16); + CHECK(mlirAttributeIsNull(mlirCutePtrTypeGetSwizzle(pointer))); + CHECK(!mlirTypeIsACutePtr(f32)); + CHECK(!mlirTypeIsACutePtr(nullType)); + CHECK( + mlirTypeIsNull(mlirCutePtrTypeGet(context, f32, nullAttr, 16, nullAttr))); + + MlirType integer = mlirCuteConstrainedInt32TypeGet(context, 8); + CHECK(mlirTypeIsACuteConstrainedInt32(integer)); + CHECK(mlirCuteConstrainedInt32TypeGetDivisibleBy(integer) == 8); + + MlirType layout = mlirCuteLayoutTypeGet(context, layoutAttr); + CHECK(mlirTypeIsACuteLayout(layout)); + CHECK(mlirAttributeEqual(mlirCuteLayoutTypeGetAttr(layout), layoutAttr)); + CHECK(mlirTypeIsNull(mlirCuteLayoutTypeGet(context, space))); + MlirType memref = mlirCuteMemRefTypeGet(context, pointer, layout); + CHECK(mlirTypeIsACuteMemRef(memref)); + CHECK(mlirTypeEqual(mlirCuteMemRefTypeGetPtr(memref), pointer)); + CHECK(mlirTypeEqual(mlirCuteMemRefTypeGetLayout(memref), layout)); + + MlirType descriptor = mlirCuteNVGPUSmemDescTypeGet(context); + CHECK(mlirTypeIsACuteNVGPUSmemDesc(descriptor)); + MlirAttribute payload = + mlirStringAttrGet(context, mlirStringRefCreateFromCString("")); + MlirType atom = mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGet(context, payload); + CHECK(mlirTypeIsACuteNVGPUCopyAtomSIMTSyncCopy(atom)); + CHECK(mlirAttributeEqual( + mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetPayload(atom), payload)); + mlirContextDestroy(context); + return 0; +} diff --git a/mlir/dialects/dialects.zig b/mlir/dialects/dialects.zig index 7901ecc5..d90c36a5 100644 --- a/mlir/dialects/dialects.zig +++ b/mlir/dialects/dialects.zig @@ -1,6 +1,7 @@ const std = @import("std"); pub const cuda_tile = @import("mlir/dialects/cuda_tile"); +pub const cute = @import("mlir/dialects/cute_ir"); pub const fly = @import("mlir/dialects/fly"); pub const mosaic_tpu = @import("mlir/dialects/mosaic_tpu"); pub const stablehlo = @import("mlir/dialects/stablehlo"); diff --git a/mlir/mlir.zig b/mlir/mlir.zig index e87749fe..25270d8d 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -9,6 +9,39 @@ test { std.testing.refAllDecls(@This()); } +test "variadic operation groups preserve order and dense array segment sizes" { + const ctx = try Context.init(.{}); + defer ctx.deinit(); + ctx.setAllowUnregisteredDialects(true); + const loc = Location.unknown(ctx); + const i32_type = Type.int(ctx, .i32); + const i64_type = Type.int(ctx, .i64); + const block = Block.init(&.{ i32_type, i64_type, i32_type }, &.{ loc, loc, loc }); + defer block.deinit(); + const op = Operation.make(ctx, "test.segmented", .{ + .operands = .{ .variadic = &.{ + &.{block.argument(0)}, + &.{}, + &.{ block.argument(1), block.argument(2) }, + } }, + .results = .{ .variadic = &.{ + &.{}, + &.{ i64_type, i32_type }, + &.{i64_type}, + } }, + .location = loc, + }); + defer op.deinit(); + try std.testing.expectEqual(@as(usize, 3), op.numOperands()); + for (0..3) |i| try std.testing.expect(op.operand(i).eql(block.argument(i))); + try std.testing.expectEqual(@as(usize, 3), op.numResults()); + try std.testing.expect(op.result(0).type_().eql(i64_type)); + try std.testing.expect(op.result(1).type_().eql(i32_type)); + try std.testing.expect(op.result(2).type_().eql(i64_type)); + try std.testing.expect(op.attributeByName("operandSegmentSizes").?.eql(.denseArray(ctx, .i32, &.{ 1, 0, 2 }))); + try std.testing.expect(op.attributeByName("resultSegmentSizes").?.eql(.denseArray(ctx, .i32, &.{ 0, 2, 1 }))); +} + pub const Error = error{ /// Invalid Mlir was created. InvalidMlir, @@ -1288,7 +1321,7 @@ pub const Operation = opaque { sizes.appendAssumeCapacity(@intCast(segment_operands.len)); } state.addAttributes(&.{ - .named(ctx, "operandSegmentSizes", .denseElements(RankedTensorType.get(&.{@intCast(sizes.len)}, .int(ctx, .i32), null).shaped(), sizes.constSlice())), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, sizes.constSlice())), }); }, }; @@ -1301,7 +1334,7 @@ pub const Operation = opaque { sizes.appendAssumeCapacity(@intCast(segment_results.len)); } state.addAttributes(&.{ - .named(ctx, "resultSegmentSizes", .denseElements(RankedTensorType.get(&.{@intCast(sizes.len)}, .int(ctx, .i32), null).shaped(), sizes.constSlice())), + .named(ctx, "resultSegmentSizes", .denseArray(ctx, .i32, sizes.constSlice())), }); }, }; diff --git a/platforms/cpu/BUILD.bazel b/platforms/cpu/BUILD.bazel index 2b27ccf9..c75db16d 100644 --- a/platforms/cpu/BUILD.bazel +++ b/platforms/cpu/BUILD.bazel @@ -31,7 +31,7 @@ copy_to_directory( ":darwin_amd64": ["@libzml_cpu_darwin_amd64//:libzml_cpu"], "@platforms//os:linux": ["@libzml_cpu_linux_amd64//:libzml_cpu"], }), - out = "sandbox/lib", + out = "sandbox", include_external_repositories = ["**"], ) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 63ec0612..4be74384 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -8,14 +8,14 @@ package(default_visibility = ["//visibility:public"]) _BUILD_LINUX = "\n".join([ packages.filegroup( name = "libzml_cpu", - srcs = ["libzml_cpu.so"], + srcs = ["lib/libzml_cpu.so"], visibility = ["@zml//platforms/cpu:__subpackages__"], ), ]) _BUILD_DARWIN = packages.filegroup( name = "libzml_cpu", - srcs = ["libzml_cpu.dylib"], + srcs = ["lib/libzml_cpu.dylib"], visibility = ["@zml//platforms/cpu:__subpackages__"], ) @@ -23,22 +23,22 @@ def _cpu_plugin_impl(mctx): http_archive( name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "084d3113c5bffd11235ca000dc9756596f1aa8ae8adb5ba8c2013850ed151ca2", - url = "https://mirror.zml.ai/plugins/202609151311.37.1.f1b6432d55a0/zml-cpu-linux-amd64.tar.zst", + sha256 = "5d6783ccd775a74dbe8d03e9f35e7a5a422ef051f56bb4266ac918bcb5f56970", + url = "https://mirror.zml.ai/plugins/202609151715.39.1.4dc71c717185/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "479dfbed01449b22e23f29ec6512ec73fad19b5110007a82bb98dccae486a729", - url = "https://mirror.zml.ai/plugins/202609151311.37.1.f1b6432d55a0/zml-cpu-darwin-amd64.tar.zst", + sha256 = "3535dee8b26bcf3e1906779b103bcbf8903c88ca93b85487d98fd2181e4f99be", + url = "https://mirror.zml.ai/plugins/202609151715.39.1.4dc71c717185/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "f5bab3aed93ae2fd9a6d5878d324a19cd8404993d6441f0258149cd98c228054", - url = "https://mirror.zml.ai/plugins/202609151311.37.1.f1b6432d55a0/zml-cpu-darwin-arm64.tar.zst", + sha256 = "bf5c036ba2a8195f8449119fa6138c7b83d7382c5732c2149d74a7b901095ffd", + url = "https://mirror.zml.ai/plugins/202609151715.39.1.4dc71c717185/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 68d6f420..ef5a75ec 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -215,12 +215,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "fe259dd69c67da5d48bc2bdd1f30a8722c2bbb7171abc0b6f23837b52e29b329", - "url": "https://mirror.zml.ai/plugins/202609151311.37.1.f1b6432d55a0/zml-cuda-linux-amd64.tar.zst", + "sha256": "e698804410e260f48ffdea425295c6551f3466eb8ea571ce07f9c9b43b0b034c", + "url": "https://mirror.zml.ai/plugins/202609151715.39.1.4dc71c717185/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "0b4843284aee3a5be6a9e33c9db12e11eaa4f9f1393c8569a93290ddb9b47791", - "url": "https://mirror.zml.ai/plugins/202609151311.37.1.f1b6432d55a0/zml-cuda-linux-arm64.tar.zst", + "sha256": "ba5717ec04c25761b6efe7f77ae89542c27ec4e56f5e3e548a0d39f8b7863aac", + "url": "https://mirror.zml.ai/plugins/202609151715.39.1.4dc71c717185/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/cuda/libzml_cuda.BUILD.bazel b/platforms/cuda/libzml_cuda.BUILD.bazel index 0378ef32..71cdb0be 100644 --- a/platforms/cuda/libzml_cuda.BUILD.bazel +++ b/platforms/cuda/libzml_cuda.BUILD.bazel @@ -5,7 +5,7 @@ load("@zml//platforms/cuda:cuda.bzl", "CUDA_COMPAT_FILES") patchelf( name = "libzml_cuda_so", - src = "libzml_cuda.so", + src = "lib/libzml_cuda.so", add_needed = [ "libzmlxcuda.so.0", ], @@ -18,6 +18,7 @@ patchelf( copy_to_directory( name = "sandbox", srcs = [ + "lib/cutlass_ir.so", ":libzml_cuda_so", "@zml//platforms/cuda:compat_probe", "@zml//platforms/cuda:zmlxcuda", diff --git a/platforms/metal/libzml_metal.BUILD.bazel b/platforms/metal/libzml_metal.BUILD.bazel index 777fb9bc..7bac767d 100644 --- a/platforms/metal/libzml_metal.BUILD.bazel +++ b/platforms/metal/libzml_metal.BUILD.bazel @@ -1,17 +1,10 @@ load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") -load("@bazel_skylib//rules:copy_file.bzl", "copy_file") - -copy_file( - name = "libzml_metal_dylib", - src = "libzml_metal.dylib", - out = "lib/libzml_metal.dylib", - allow_symlink = True, -) +load("@rules_cc//cc:cc_library.bzl", "cc_library") copy_to_directory( name = "sandbox", srcs = [ - ":libzml_metal_dylib", + "lib/libzml_metal.dylib", ], add_directory_to_runfiles = False, include_external_repositories = ["**"], diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 9a9d9b8a..3eea63c9 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "775504055ce16fc2be8ceaaef843c15b5a051ce3d41f78534392c231adc59a3f", - url = "https://mirror.zml.ai/plugins/202609151311.37.1.f1b6432d55a0/zml-metal-darwin-arm64.tar.zst", + sha256 = "c081e43b3c265728a7345e6735335c115bb53d7158bc594d192e611e90494fd4", + url = "https://mirror.zml.ai/plugins/202609151715.39.1.4dc71c717185/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/libzml_rocm.BUILD.bazel b/platforms/rocm/libzml_rocm.BUILD.bazel index 3de1dfc4..0a18c130 100644 --- a/platforms/rocm/libzml_rocm.BUILD.bazel +++ b/platforms/rocm/libzml_rocm.BUILD.bazel @@ -6,7 +6,7 @@ load("@zml//bazel:patchelf.bzl", "patchelf") patchelf( name = "libzml_rocm_so", - src = "libzml_rocm.so", + src = "lib/libzml_rocm.so", add_needed = [ "libzmlxrocm.so.0", ], diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 183650e5..a34e9da7 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202609151311.37.1.f1b6432d55a0/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "869f84a01b925c14ba9e49249535e3fd190e9c56dc0426c1e52a12c84acb1d55" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202609151715.39.1.4dc71c717185/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "c4fd5b602407f478525987e20cc9f306731046c336397eef3568dd2dfd8bb158" def _rocm_package_name(name): return name + _ROCM_VERSION diff --git a/third_party/cute_ir/BUILD.bazel b/third_party/cute_ir/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/third_party/cute_ir/cute_ir.bazel b/third_party/cute_ir/cute_ir.bazel new file mode 100644 index 00000000..7a9de330 --- /dev/null +++ b/third_party/cute_ir/cute_ir.bazel @@ -0,0 +1,197 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:defs.bzl", "cc_library") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "cutegen", + hdrs = glob(["cutegen/include/cutegen/**/*.hpp"]), + includes = ["cutegen/include"], + deps = [ + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:MathDialect", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:UBDialect", + "@llvm-project//mlir:NVVMDialect", + ], +) + +td_library( + name = "dialect_td", + srcs = [ + "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteAttrs.td", + "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteDialect.td", + "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td", + "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.td", + "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypes.td", + ], + includes = ["cute_ir/include"], + deps = [ + "@llvm-project//mlir:BuiltinDialectTdFiles", + "@llvm-project//mlir:DataLayoutInterfacesTdFiles", + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "dialect_inc_gen", + includes = ["."], + tbl_outs = [ + ( + ["-gen-dialect-decls", "-dialect=cute"], + "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteDialect.h.inc", + ), + ( + ["-gen-dialect-defs", "-dialect=cute"], + "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteDialect.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteDialect.td", + deps = [":dialect_td"], +) + +gentbl_cc_library( + name = "type_interfaces_inc_gen", + includes = ["."], + tbl_outs = [ + ( + ["-gen-type-interface-decls"], + "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.h.inc", + ), + ( + ["-gen-type-interface-defs"], + "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.td", + deps = [":dialect_td"], +) + +gentbl_cc_library( + name = "attrs_inc_gen", + includes = ["."], + tbl_outs = [ + ( + ["-gen-attrdef-decls", "-attrdefs-dialect=cute"], + "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteAttrs.h.inc", + ), + ( + ["-gen-attrdef-defs", "-attrdefs-dialect=cute"], + "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteAttrs.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteAttrs.td", + deps = [":dialect_td"], +) + +gentbl_cc_library( + name = "ops_inc_gen", + includes = ["."], + tbl_outs = [ + ( + ["-gen-op-decls"], + "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.h.inc", + ), + ( + ["-gen-op-defs"], + "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td", + deps = [":dialect_td"], +) + +# gentbl_cc_library( +# name = "ops_inc_gen", +# includes = ["cute_ir/include"], +# tbl_outs = [ +# ( +# ["-gen-op-decls"], +# "cute_ir/Dialect/Cute/IR/CuteOps.h.inc", +# ), +# ( +# ["-gen-op-defs"], +# "cute_ir/Dialect/Cute/IR/CuteOps.cpp.inc", +# ), +# ], +# tblgen = "@llvm-project//mlir:mlir-tblgen", +# td_file = "CuteOps.td", +# deps = [":tabledefs"], +# ) + +gentbl_cc_library( + name = "types_decl_inc_gen", + includes = ["cute_ir/include"], + tbl_outs = [ + ( + ["-gen-typedef-decls", "-typedefs-dialect=cute"], + "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypes.h.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypes.td", + deps = [":dialect_td"], +) + +gentbl_cc_library( + name = "types_defs_inc_gen", + includes = ["cute_ir/include"], + tbl_outs = [ + ( + ["-gen-typedef-defs", "-typedefs-dialect=cute"], + "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypes.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypes.td", + deps = [":dialect_td"], +) + +cc_library( + name = "dialect_headers", + hdrs = [ + "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteDialect.h", + ], + includes = ["cute_ir/include"], + textual_hdrs = ["cute_ir/lib/Dialect/IR/CuteTypes.cpp"], + deps = [ + ":attrs_inc_gen", + ":cutegen", + ":dialect_inc_gen", + ":ops_inc_gen", + ":type_interfaces_inc_gen", + ":types_decl_inc_gen", + ], +) + +cc_library( + name = "dialect_support", + srcs = [ + "cute_ir/lib/Dialect/IR/CuteAttributes.cpp", + "cute_ir/lib/Dialect/IR/CuteOps.cpp", + ], + includes = ["cute_ir/include"], + deps = [":dialect_headers"], +) + +cc_library( + name = "dialect", + srcs = [ + "cute_ir/lib/Dialect/Cute/IR/CuteDialect.cpp", + "cute_ir/lib/Dialect/Cute/IR/CuteTypes.cpp", + ], + deps = [ + ":dialect_headers", + ":dialect_support", + ":types_defs_inc_gen", + ], +) diff --git a/third_party/cute_ir/repo.bzl b/third_party/cute_ir/repo.bzl new file mode 100644 index 00000000..89de3670 --- /dev/null +++ b/third_party/cute_ir/repo.bzl @@ -0,0 +1,10 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + git_repository( + name = "cute_ir", + remote = "https://github.com/NVIDIA/cutlass.git", + commit = "147295a3d4b75f3aeff247c25b8927cea9a7006a", + build_file = Label("//third_party/cute_ir:cute_ir.bazel"), + strip_prefix = "cutlass_compiler", + ) diff --git a/third_party/cutlass/cutlass.bazel b/third_party/cutlass/cutlass.bazel new file mode 100644 index 00000000..e009a6d4 --- /dev/null +++ b/third_party/cutlass/cutlass.bazel @@ -0,0 +1,111 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:defs.bzl", "cc_library") + +td_library( + name = "dialect_td", + srcs = [ + "include/cute_ir/Dialect/Cute/IR/CuteAttrs.td", + "include/cute_ir/Dialect/Cute/IR/CuteDialect.td", + "include/cute_ir/Dialect/Cute/IR/CuteOps.td", + "include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.td", + "include/cute_ir/Dialect/Cute/IR/CuteTypes.td", + ], + includes = ["include"], + deps = [ + "@llvm-project//mlir:BuiltinDialectTdFiles", + "@llvm-project//mlir:DataLayoutInterfacesTdFiles", + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "dialect_inc_gen", + strip_include_prefix = ".", + tbl_outs = [ + ( + ["-gen-dialect-decls", "-dialect=cute"], + "include/cute_ir/Dialect/Cute/IR/CuteDialect.h.inc", + ), + ( + ["-gen-dialect-defs", "-dialect=cute"], + "include/cute_ir/Dialect/Cute/IR/CuteDialect.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cute_ir/Dialect/Cute/IR/CuteDialect.td", + deps = [":dialect_td"], +) + +gentbl_cc_library( + name = "type_interfaces_inc_gen", + strip_include_prefix = ".", + tbl_outs = [ + ( + ["-gen-type-interface-decls"], + "include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.h.inc", + ), + ( + ["-gen-type-interface-defs"], + "include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.td", + deps = [":dialect_td"], +) + +gentbl_cc_library( + name = "attrs_inc_gen", + strip_include_prefix = ".", + tbl_outs = [ + ( + ["-gen-attrdef-decls", "-attrdefs-dialect=cute"], + "include/cute_ir/Dialect/Cute/IR/CuteAttrs.h.inc", + ), + ( + ["-gen-attrdef-defs", "-attrdefs-dialect=cute"], + "include/cute_ir/Dialect/Cute/IR/CuteAttrs.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cute_ir/Dialect/Cute/IR/CuteAttrs.td", + deps = [":dialect_td"], +) + +gentbl_cc_library( + name = "ops_inc_gen", + strip_include_prefix = ".", + tbl_outs = [ + ( + ["-gen-op-decls"], + "include/cute_ir/Dialect/Cute/IR/CuteOps.h.inc", + ), + ( + ["-gen-op-defs"], + "include/cute_ir/Dialect/Cute/IR/CuteOps.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cute_ir/Dialect/Cute/IR/CuteOps.td", + deps = [":dialect_td"], +) + +cc_library( + name = "headers", + hdrs = [ + hdr for hdr in [ + "include/cute_ir/Dialect/Cute/IR/CuteDialect.h", + "include/cute_ir/Dialect/Cute/IR/CuteOps.h", + "include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.h", + ] + ], + strip_include_prefix = "include", + deps = [ + ":attrs_inc_gen", + ":dialect_inc_gen", + ":ops_inc_gen", + ":type_interfaces_inc_gen", + ], +) diff --git a/third_party/non_module_deps.bzl b/third_party/non_module_deps.bzl index e83ae6f7..fd8d82fe 100644 --- a/third_party/non_module_deps.bzl +++ b/third_party/non_module_deps.bzl @@ -2,6 +2,7 @@ load("//third_party/arocc:repo.bzl", arocc = "repo") load("//third_party/cloud_accelerator_diagnostics:repo.bzl", cloud_accelerator_diagnostics = "repo") load("//third_party/com_google_sentencepiece:repo.bzl", com_google_sentencepiece = "repo") load("//third_party/cuda_tile:repo.bzl", cuda_tile = "repo") +load("//third_party/cute_ir:repo.bzl", cute_ir = "repo") load("//third_party/flashattn:repo.bzl", flashattn = "repo") load("//third_party/flashinfer_cutlass_moe:repo.bzl", flashinfer_cutlass_moe = "repo") load("//third_party/flydsl:repo.bzl", flydsl = "repo") @@ -36,6 +37,7 @@ def _non_module_deps_impl(mctx): iree() cuda_tile() flydsl() + cute_ir() return mctx.extension_metadata( reproducible = True, diff --git a/tools/xprof/requirements.lock.txt b/tools/xprof/requirements.lock.txt index 7ca0f132..6bf9f9bc 100644 --- a/tools/xprof/requirements.lock.txt +++ b/tools/xprof/requirements.lock.txt @@ -8,126 +8,126 @@ aiohappyeyeballs==2.7.1 \ --hash=sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d \ --hash=sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472 # via aiohttp -aiohttp==3.14.1 \ - --hash=sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5 \ - --hash=sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983 \ - --hash=sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521 \ - --hash=sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340 \ - --hash=sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d \ - --hash=sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a \ - --hash=sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4 \ - --hash=sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a \ - --hash=sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f \ - --hash=sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee \ - --hash=sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8 \ - --hash=sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb \ - --hash=sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397 \ - --hash=sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05 \ - --hash=sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8 \ - --hash=sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09 \ - --hash=sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2 \ - --hash=sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba \ - --hash=sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf \ - --hash=sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271 \ - --hash=sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5 \ - --hash=sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847 \ - --hash=sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264 \ - --hash=sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf \ - --hash=sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6 \ - --hash=sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df \ - --hash=sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035 \ - --hash=sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126 \ - --hash=sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6 \ - --hash=sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35 \ - --hash=sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4 \ - --hash=sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333 \ - --hash=sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203 \ - --hash=sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c \ - --hash=sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1 \ - --hash=sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251 \ - --hash=sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365 \ - --hash=sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b \ - --hash=sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621 \ - --hash=sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94 \ - --hash=sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da \ - --hash=sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491 \ - --hash=sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe \ - --hash=sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d \ - --hash=sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080 \ - --hash=sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42 \ - --hash=sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c \ - --hash=sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397 \ - --hash=sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9 \ - --hash=sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8 \ - --hash=sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345 \ - --hash=sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3 \ - --hash=sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602 \ - --hash=sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2 \ - --hash=sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966 \ - --hash=sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192 \ - --hash=sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95 \ - --hash=sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3 \ - --hash=sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b \ - --hash=sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444 \ - --hash=sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6 \ - --hash=sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573 \ - --hash=sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af \ - --hash=sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15 \ - --hash=sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe \ - --hash=sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2 \ - --hash=sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496 \ - --hash=sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876 \ - --hash=sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817 \ - --hash=sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448 \ - --hash=sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e \ - --hash=sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6 \ - --hash=sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd \ - --hash=sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f \ - --hash=sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe \ - --hash=sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c \ - --hash=sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca \ - --hash=sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c \ - --hash=sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa \ - --hash=sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc \ - --hash=sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0 \ - --hash=sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0 \ - --hash=sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2 \ - --hash=sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844 \ - --hash=sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719 \ - --hash=sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1 \ - --hash=sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3 \ - --hash=sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178 \ - --hash=sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3 \ - --hash=sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95 \ - --hash=sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730 \ - --hash=sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842 \ - --hash=sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd \ - --hash=sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d \ - --hash=sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96 \ - --hash=sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85 \ - --hash=sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1 \ - --hash=sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199 \ - --hash=sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a \ - --hash=sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588 \ - --hash=sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec \ - --hash=sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004 \ - --hash=sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480 \ - --hash=sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04 \ - --hash=sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8 \ - --hash=sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce \ - --hash=sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087 \ - --hash=sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505 \ - --hash=sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780 \ - --hash=sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4 \ - --hash=sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d \ - --hash=sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca \ - --hash=sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665 \ - --hash=sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296 \ - --hash=sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c \ - --hash=sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a \ - --hash=sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7 \ - --hash=sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451 \ - --hash=sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3 +aiohttp==3.14.3 \ + --hash=sha256:03cd2bde3d7f085b64e549c985f4bb928cad7e8ecf5323bfca320db548d81b39 \ + --hash=sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043 \ + --hash=sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b \ + --hash=sha256:0fdea2281997af69da84c77ffa6f5938a0285f21fb3887c249d67419ca865b3d \ + --hash=sha256:11fb37ef075669eee52ab1928fbf6e1741fada40409fa309ebde9607a962aebf \ + --hash=sha256:134ac5ddcf61c6fad984b9a5727d83492ada43d63471db20fb73042c13fca62f \ + --hash=sha256:152516815ef926786a0b6ae2b8f1fd2e0c71582dee0b435636865316fd4891b7 \ + --hash=sha256:1576145bdceeb92382d899751e12743a3a5b8e460a841e3e50543859e54864dc \ + --hash=sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559 \ + --hash=sha256:16ea7e24c309fb7c0bbd505d149abe4fe4dccfb8db911db7dbec0921bc889a6f \ + --hash=sha256:18c441d0a8fca6de8d1f546849b9f0ab20d435993e2c5b59562b2fae6be2f929 \ + --hash=sha256:18cb43369747b2ae007bd2655fb8e63a099c2ff1d207962943636dac989b3147 \ + --hash=sha256:1b59533861b70a2185c8f4f350f791f39d64358ef6944ce71c5240c9ec0982c9 \ + --hash=sha256:1c5281acc88b92396f88c7e1e2748f8466689df22b80170e4f51efa712fb47a8 \ + --hash=sha256:1c5ec8fb1bcc31a8466f74aaf26c345d5c386fa4bd08a3f0eb9c7a4a3fe8b5bf \ + --hash=sha256:1caa7b0d05f3e3a36f87788c59e970a7ee1cefcfcbb924a9f138c4a6551c9cb7 \ + --hash=sha256:21c016079415ed3fd676963e9793700a566d85dbbd6bfc564b9b2d209147dcc8 \ + --hash=sha256:2498f0fe69ead802f9675beca44a7c21c62fdaa4ec5145ea1c3ad6edbee29f85 \ + --hash=sha256:25bd2708db6bdf6a6630dd37bdcdfcb47c4434d22ac69c64665b802910140b30 \ + --hash=sha256:270d3dace9ca2f10f0da5d8ebe519b7a310fc6112ed916e32df5866df0888553 \ + --hash=sha256:2e1161602f45a54de2ce0905243a95f58cb42dcd378402f3697f5e0b21e9d2e7 \ + --hash=sha256:2e9878ae68e4a5f1c0abe4dd497dbc3d51946f5837b56759e2a02e78fa90ef86 \ + --hash=sha256:30402d03a7c0ff52bce290b57e564e9079fd9d0cb545c8aba73f86a103162d2e \ + --hash=sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a \ + --hash=sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c \ + --hash=sha256:38901a84da3ce22249f6e860bf8f90d141bcab7da090cc398f8bb58c0e44b7da \ + --hash=sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5 \ + --hash=sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d \ + --hash=sha256:3ae5b3a59436d089b5395d910121a390feed4d00578eb95a0fd1a329fe963100 \ + --hash=sha256:3d4f72af88ac2474bb5bca640030320e3d38a0163a1d7533500e87be458eef71 \ + --hash=sha256:3f42e9b78301f11c8f861746175d8b9c1ccef713fcad9eab396e2f6db8ed4a22 \ + --hash=sha256:42a67efc36300d052fb4508a53e8b6901b9284b599ae63945c377569c5fcc1e1 \ + --hash=sha256:48d67b87db6279c044760787eb01f6413032c2e6f3ba1cafaa492b1c8e578479 \ + --hash=sha256:498c6c623134f8e09a3c4e60bcd607a0b4590dd7dbf08dd40851b27cbb520ccb \ + --hash=sha256:49f7325beb0f85ef4aef5f48f490269575f83e6e2acad00a1d80b807eb027062 \ + --hash=sha256:4e3ac92d90e92773b2362d506068e9a948192bd553e743c5b2429e28527c8661 \ + --hash=sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427 \ + --hash=sha256:5373dc80ad1aa2fb9ad95c83f24eef418bbda3a61375f128e5b0192e4f3f9b32 \ + --hash=sha256:53e5179d8abb5710f8e83ba207c41c8d1261fcffd4616500e15ca2b7a33be10a \ + --hash=sha256:53e7b4ce82b54a8bcc71b3b67a5cbd177ca1d7f592cbc92cd38b7349f73482db \ + --hash=sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42 \ + --hash=sha256:54cfcdee2770dac994417cbb0ee1f3eb0e7cb6b30c79bf44f2c02ff79ec5124a \ + --hash=sha256:55bdcc472aafe2de4a253045cc128007a64f1e0264fb675791e132ea5edaa3bd \ + --hash=sha256:56f355e79f71aef2a85c80305cc915f894b170dba76de5fe84f6351939b83c06 \ + --hash=sha256:5895ef58c4620afe02fa16044f023dc4dafec08158f9d08874a46a7dbc0341b8 \ + --hash=sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228 \ + --hash=sha256:5f08ec777f35ee70720233b8b9811d3bb5d728137f30ac91b7457709c3261ac0 \ + --hash=sha256:614c61d478b83953e261d02bb2df750f17227cd33ef8002945bf5aebbde21919 \ + --hash=sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee \ + --hash=sha256:6debfa7312ff9d4c124dc71d72e9a0a4b9e0879e48ba6fcb42bef5c3300289e2 \ + --hash=sha256:7041d52c3a7fa20c9e8c182b534704abb19502c8bdcbde7ab23bfda6f642394f \ + --hash=sha256:70c987b27534f9ae1a723f47ae921571d616da21d3208282bf4c52af5164ac43 \ + --hash=sha256:74ab5b6a9fb13e873e5a90946588baecaf488745e1db1a4a5c433f971f035098 \ + --hash=sha256:78253b573e6ffab5028924fc98bc281aae05445969982a10864bc360dea2016c \ + --hash=sha256:7a75aa63cbf9b21cfaf60dc2657e19df2c2867d91707d653fee171ffeedd1371 \ + --hash=sha256:8800c996b01c2772a783e3e46f3e1abd5823029adca0df54231960de9bfefa5b \ + --hash=sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0 \ + --hash=sha256:8a5fd34f7f7410d1730d5c2ba873cacb2eed3fede366feb268a70ba22581ed8f \ + --hash=sha256:8b3b60de05f3dcb6f6a00f818bb2ec781cee4de0645f59ccaf99b1d1823b6100 \ + --hash=sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529 \ + --hash=sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc \ + --hash=sha256:9aa6e61fdf20105c4144e755bd586008ff450791d67b1c8146fdc15959c4d51c \ + --hash=sha256:9d9edccfe496b476db5f398d97b865e9a6752bcf8aec4eef8390ce20fb64bb41 \ + --hash=sha256:9fc7b5bfec6573f3ae844f457fdde5adeb713f8b8e4a81ad64fc207b49383716 \ + --hash=sha256:a0dc483c00da8b673abbb367eb6f8d8f4bcec30eb58529ea13cb42e7fd2dfa33 \ + --hash=sha256:a3a8296e7ab5c295f53f1041487cb088e1480775aafbf7fe545d93b770a0f96f \ + --hash=sha256:a3e22975f905b89a55a488c2a08f2fdb2186175349e917d48985cc468a3d4c6e \ + --hash=sha256:a4af35c443e0b1a1bd6a8af3f3485d7fda15c142751a00f3ff8090f0b93346fa \ + --hash=sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b \ + --hash=sha256:ac74facc01463f138b0da5580329cfcc82818dea5656e83ddcd11268fc12ff80 \ + --hash=sha256:ad4c8b7488d745d2ca4838ebd8ae5ba9b56341d30b1da43640e4ce87f9f49646 \ + --hash=sha256:b014a6ed7cf912e787149fdc529166d3ceabac23f26efeea3158c9aba2354e7e \ + --hash=sha256:b20032766aedf6261c7a566585a40867d092ac03a0d81592d5370ef9b054f99b \ + --hash=sha256:b2466434105a4e03113c36ec775cc2ebe6676b62eae326fa670bb607ef788c1c \ + --hash=sha256:b304db572b4368edd8dda8a2274f73156fe15558fca4a917cb8a09fc47af5963 \ + --hash=sha256:ba59d59aba08ac02fc03b0c8983ccd5ee39a199d0552ce9e6d2b4845b34d59ae \ + --hash=sha256:bd52f811e65f6fb634b1047159657c98f52b407f8efec907bcfc09da9a4c0a25 \ + --hash=sha256:bdd0e2834dce1a26c1bbe26464861e16bbe217042cbff619247c11594472518c \ + --hash=sha256:c23ec8ee9d5ab2f5421f9c7fffce208435607af27fd46d4a44e031954352838f \ + --hash=sha256:c39846c3aad97a8530c89d7a3869a8f8e9e3762c6ac0504481e5c80948f7e807 \ + --hash=sha256:c3c200cf9757edd785051dc699c7ecbec22110dbfcb3fefc7a9f9695eda8ea7a \ + --hash=sha256:c7d3a97c678d34fc5b59da671ee9cd630096ddc643e7b5a30d54a2a6f3574d3f \ + --hash=sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d \ + --hash=sha256:cc7cb243a68167172f48c1fd43cee91ec4b1d40cefd190edd43369d1a6bc9c82 \ + --hash=sha256:ccd4893707b3e2a13e39c90d43cf80edf2e4d0457935bcc103bf2346214c3f15 \ + --hash=sha256:cd817772b2fcf2b8c0905795318485f9ec16eae60b29feb7f4c77085311637f0 \ + --hash=sha256:cda5fd5c95ad7a125a2e8464acc78b98b94c475a3780d6aa0aa157c93f470f4d \ + --hash=sha256:cef89a58e628c4efcac3275c2d68083f82426dcdc89c1492a6f654f9f7ea6ab9 \ + --hash=sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19 \ + --hash=sha256:d6088ec9894113802bddb3c09e974929aed2c7b3a8c456219b8aab4481f1a239 \ + --hash=sha256:d6218d92e450824e9b4881f44e8c09f1853b490f9a64130801024a4793b1b3b0 \ + --hash=sha256:d77640cc618c1d99fc4f8589c0f24a730adfa54eb1e57ef7bf0c8dfb78da898c \ + --hash=sha256:d7d2deec16eeedf55f2c7cf75b521ea3856a5177e123844f8fd0f114ce252cb5 \ + --hash=sha256:db332af25642007330fca8be5c4d194caf2bea7a7fc84415aff3497af5dfee6b \ + --hash=sha256:dd54d0e8717de95939766febac482ac0474d8ac3b048115f9f2b1d23a16e7db4 \ + --hash=sha256:ddcac3c6b382e81f1dd0499199d4136b877beb4cb5ef770bbbfba56c4b8f55d2 \ + --hash=sha256:df82f3787c940c94986b34222d59c9e38843fba85139f36e85255a82ad5355a9 \ + --hash=sha256:dfa68deb2a443bdaa3ea5297b0699c1464f08aef3812b486d1348eee61b07dc0 \ + --hash=sha256:dff9461ec275f22135650d5ba4b4931a11f3958df7dfbb8db630000d4dee0883 \ + --hash=sha256:e1e74298bab6ee0d6e749ed4fd1901c7e604bdda32c03d787a2cc71c46d0433d \ + --hash=sha256:e2667f0bbe7eb6c74eae5e9691441ad186e5845ca3cff63230fc09c4e7514f5d \ + --hash=sha256:e3be98a7c30b8c25d573dafba7171d66dfb05ee6a9070fc46535464ff97700a6 \ + --hash=sha256:e568e14940c09955aa51f4e645b6daa18a581c5dcfcd73744dcc86a856e3ced3 \ + --hash=sha256:e72ee89e28d907a18f46959b4eb0bb06701cc7f8cf4366e00029e2ccfaaf5924 \ + --hash=sha256:e92eb8acc45eb6a9f4935071a77edf5b85cc6f8dfad5cd99e97653c26593cdde \ + --hash=sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787 \ + --hash=sha256:eac645b09bcfdf73df7536331f0678c1086ea250981118ddb5199e17ccef72bb \ + --hash=sha256:eb0495d778817619273c108784292be161a924b9f5ae5cbbc70a2caa6838250b \ + --hash=sha256:ebe8e504f058fe91223351cecd2d9d6946c9d241bb0250d898ffbdf584cc72b0 \ + --hash=sha256:ed099d105449c4f9e84f24af203cd131349d4761d8813fa7e02c32e7128cd910 \ + --hash=sha256:f0f177d1b195b9e06376cfd7d308d8a1b920909a609d03ac82a8c73bbb16d3b9 \ + --hash=sha256:f3d2669fe7dec7fc359ecdb5984b29b50d85d5d00f8c1cb61de4f4a24ee42627 \ + --hash=sha256:f4e05329faa0ea1a404b37de4f034fd2c2defcca06a68dc6745e4e56c88e8a48 \ + --hash=sha256:f53bcd52f585e1ac3e590d61434eb61f9a88c38df041b4ea126d97144344a77b \ + --hash=sha256:f55119f7bf25f49ed210f6096090715da24f2943c62102448915fde3c62877ce \ + --hash=sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a \ + --hash=sha256:f8fb78a83c9e5f741ca3a68cfb455c1f5bb83b4e7249a3848b3cd78d0a8563b0 \ + --hash=sha256:fa9467a8113aa69d3d7c55a70ef0b7c636010a40993f3df9d9d0d73b3eb7ef24 \ + --hash=sha256:fd51ebf9d3a00c074df4ede271023f4d2dba289bcc740b88191872716014e3c5 # via gcsfs aiosignal==1.4.0 \ --hash=sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e \ @@ -137,258 +137,337 @@ attrs==26.1.0 \ --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 \ --hash=sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32 # via aiohttp -certifi==2026.6.17 \ - --hash=sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432 \ - --hash=sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db +certifi==2026.7.22 \ + --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \ + --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55 # via requests -cffi==2.1.0 \ - --hash=sha256:02cb7ff33ded4f1532476731f89ede53e2e488a8e6205515a82144246ffa7dcc \ - --hash=sha256:03e9810d18c646077e501f661b682fbf5dee4676048527ca3cffe66faa9960dd \ - --hash=sha256:0520e1f4c35f44e209cbbb421b67eec42e6a157f59444dfb6058874ff3610e5d \ - --hash=sha256:0582a58f3051372229ca8e7f5f589f9e5632678208d8636fea3676711fdf7fe5 \ - --hash=sha256:0611e7ebf90573a535ebdc33ae9da222d037853983e13359f580fab781ca017f \ - --hash=sha256:0a42c688d19fca6e095a53c6a6e2295a5b050a8b289f109adab02a9e61a25de6 \ - --hash=sha256:0a96b74cda968eebbad56d973efe5098974f0a9fb323865bf99ea1fd24e3e64c \ - --hash=sha256:10537b1df4967ca26d21e5072d7d54188354483b91dc75058968d3f0cf13fbda \ - --hash=sha256:11b3fb55f4f8ad92274ed26705f65d8f91457de71f5380061eb6d125a768fecd \ - --hash=sha256:15faec4adfff450819f3aee0e2e02c812de6edb88203aa58807955db2003472a \ - --hash=sha256:164bff1657b2a74f0b6d54e11c9b375bc97b931f2ca9c43fcf875838da1570dd \ - --hash=sha256:1854b724d00f6654c742097d5387569021be12d3a0f770eae1df8f8acfcc6acd \ - --hash=sha256:19c54ac121cad98450b4896fa9a43ee0180d57bc4bc911a33db6cab1efab6cd3 \ - --hash=sha256:1b96bfe2c4bd825681b7d311ad6d9b7280a091f43e8f63da5729638083cd3bfb \ - --hash=sha256:1e9f50d192a3e525b15a75ab5114e442d83d657b7ec29182a991bc9a88fd3a66 \ - --hash=sha256:1ff3456eab0d889592d1936d6125bbfbc7ae4d3354a700f8bd80450a66445d4d \ - --hash=sha256:2282cd5e38aa8accd03e99d1256af8411c84cdbee6a89d841b563fdbd1f3e50f \ - --hash=sha256:276f20fffd7b396e12516ba8edf9509210ac248cbbc5acbc39cd512f9f59ebe6 \ - --hash=sha256:2b71d409cccee78310ab5dec549aed052aaea483346e282c7b02362596e01bb0 \ - --hash=sha256:2e9dabb9abcb7ad15938c7196ad5c1718a4e6d33cc79b4c0209bdb64c4a54a5c \ - --hash=sha256:30b65779d598c370374fefabf138d456fd6f3216bfa7bedfab1ba82025b0cd93 \ - --hash=sha256:33eb1ad83ebe8f313e0df035c406227d55a79456704a863fad9842136af5ad7d \ - --hash=sha256:35aaea0c7ee0e58a5cd8c2fd1a48fdf7ece0d2699b7ecdda08194e9ce5dd9b3d \ - --hash=sha256:3681e031db29958a7502f5c0c9d6bbc4c36cb20f7b104086fa642d1799631ff8 \ - --hash=sha256:379de10ce1ba048b1448599d1b37b24caee16309d1ac98d3982fc997f768700b \ - --hash=sha256:37f525a7e7e50c017fdebe58b787be310ad59357ae43a053943a6e1a6c526001 \ - --hash=sha256:3b926723c13eba9f81d2ef3820d63aeceec3b2d4639906047bf675cb8a7a500d \ - --hash=sha256:3d7f118b5adbfdfead90c25822690b02bc8074fba949bb7858bec4ebd55adb43 \ - --hash=sha256:46b1c8db8f6122420f32d02fffb924c2fe9bc772d228c7c711748fff56aabb2b \ - --hash=sha256:47ff3a8bfd8cb9da1af7524b965127095055654c177fcfc7578debcb015eecd0 \ - --hash=sha256:4d433a51f1870e43a13b6732f92aaf540ff77c2015097c78556f75a2d6c030e0 \ - --hash=sha256:4f26194e3d95e06501b942642855aed4f953d55e95d7d01b7c4483db3ecff458 \ - --hash=sha256:510aeeeac94811b138077451da1fb18b308a5feab47dd2b603af55804155e1c8 \ - --hash=sha256:5972433ad71a9e46516584ef60a0fda12d9dc459938d1539c3ddecf9bdc1368d \ - --hash=sha256:5ecbd0499275d57506d397eebe1981cee87b47fcd9ef5c22cab7ed7644a39a94 \ - --hash=sha256:6274dcb2d15cef48daa73ed1be5a40d501d74dccd0cd6db364776d12cb6ba022 \ - --hash=sha256:63960549e4f8dc41e31accb97b975abaecfc44c03e396c093a6436763c2ea7db \ - --hash=sha256:64c753a0f87a256020004f37a1c8c02c480e725f910f0b2a0f3f07debd1b2479 \ - --hash=sha256:6af371f3767faeffc6ac1ef57cdfd25844403e9d3f476c5537caee499de96376 \ - --hash=sha256:6ca4919c6e4f89aa99c42510b42cf54596892c00b3f9077f6bdd1505e24b9c8d \ - --hash=sha256:6d194185eabd279f1c05ebe3504265ddfc5ad2b58d0714f7db9f01da592e9eb6 \ - --hash=sha256:702c436735fbe99d59ada02a1f65cfc0d31c0ee8b7290912f8fbc5cd1e4b16c3 \ - --hash=sha256:716ff8ec22f20b4d988b12884086bcef0fc99737043e503f7a3935a6be99b1ea \ - --hash=sha256:762f99479dcb369f60ab9017ad4ab97a36a1dd7c1ee5a3b15db0f4b8659120cd \ - --hash=sha256:7762faa47e8ff7eb80bd261d9a7d8eea2d8baa69de5e95b70c1f338bbe712f02 \ - --hash=sha256:78474632761faa0fb96f30b1c928c84ebcf68713cbb80d15bab09dfe61640fde \ - --hash=sha256:799416bae98336e400981ff6e532d67d5c709cfb30afb79865a1315f94b0e224 \ - --hash=sha256:7d034dcffa09e9a46c93fa3a3be402096cb5354ac6e41ab8e5cc9cd8b642ad76 \ - --hash=sha256:7d28dff1db6764108bc30788d85d61c876beff416d9a49cb9dd7c5a9f34f5804 \ - --hash=sha256:7d3538f9c0e50670f4deb93dbb696576e60590369cae2faf7de681e597a8a1f1 \ - --hash=sha256:7d5980a3433d4b71a5e120f9dd551403d7824e31e2e67124fe2769c404c06913 \ - --hash=sha256:7ea6b3e2c4250ff1de21c630fe72d0f63eb95c2c32ffbf64a358cf4a8836d714 \ - --hash=sha256:86cf8755a791f72c85dc287128cc62d4f24d392e3f1e15837245623f4a33cccc \ - --hash=sha256:88023dfe18799507b73f1dbb0d14326a17465de1bc9c9c7655c22845e9ddc3a2 \ - --hash=sha256:89095c1968b4ba8285840e131bf2891b09ae137fe2146905acae0354fbce1b5e \ - --hash=sha256:8d35c139744adb3e727cd51b1a18324bbe44b8bd41bf8322bca4d41289f48eda \ - --hash=sha256:8e74a6135550c4748af665b1b1118b6aab33b1fc6a16f9aff630af107c3b4512 \ - --hash=sha256:8f9ec95b8a043d3dfbc74d9abc6f7baf524dd27a8dc160b0a32ff9cdab650c28 \ - --hash=sha256:90bec57cf82089383bd06a605b3eb8daebf7e5a668520beaf6e327a83a947699 \ - --hash=sha256:95f2954c2c9473d892eca6e0409f3568b37ab62a8eedb122461f73cc273476e3 \ - --hash=sha256:961be50688f7fba2fa65f63712d3b9b341a22311f5253460ce933f52f0de1c8c \ - --hash=sha256:98fff996e983a36d3aa2eca83af40c5821202e7e6f32d13ae94e3d2286f10cfe \ - --hash=sha256:9b8f0f26ca4e7513c534d351eca551947d053fac438f2a04ac96d882909b0d3a \ - --hash=sha256:9d72af0cf10a76a600a9690078fe31c63b9588c8e86bf9fd353f713c84b5db0f \ - --hash=sha256:9d8272c0e483b024e1b9ad029821470ed8ec65631dbd90217469da0e7cd89f1c \ - --hash=sha256:a016194dbe13d14ee9556e734b772d8d67b947092b268d757fd4290e3ba2dfc2 \ - --hash=sha256:a5781494d4d400a3f47f8f1da94b324f6e6b440a53387774002890a2a2f4b50f \ - --hash=sha256:a95b05f9baf29b91171b3a8bd2020b028835243e7b0ff6bb23e2a3c228518b1b \ - --hash=sha256:aa7a1b53a2a4452ada2d1b5dade9960b2522f1e61293a811a077439e39029565 \ - --hash=sha256:ac0f1a2d0cfa7eea3f2aaf006ab6e70e8feeb16b75d65b7e5939982ca2f11056 \ - --hash=sha256:af5e2915d41fe6c961694d7bfdc8562942638200f3ce2765dfb8b745cf997629 \ - --hash=sha256:b6422532152adf4e59b110cb2808cee7a033800952f5c036b4af047ee43199e7 \ - --hash=sha256:b65f590ef2a44640f9a05dbb548a429b4ade77913ce683ac8b1480777658a6c0 \ - --hash=sha256:ba00f661f8ba35d075c937174e27c2c421cec3942fd2e0ea3e66996757c0fdd9 \ - --hash=sha256:bccbbb5ee76a61f9d99b5bf3846a51d7fca4b6a732fe46f89295610edaf41853 \ - --hash=sha256:bf01d8c84cbea96b944c73b22182e6c7c432b3475632b8111dbfdc95ddad6e13 \ - --hash=sha256:bf5c6cf48238b0eb4c086978c492ad1cbc22373fc5b2d7353b3a598ce6db887a \ - --hash=sha256:c16914df9fb7f500e440e6875fa23ff5e0b31db01fa9c06af98d59a91f0dc2e4 \ - --hash=sha256:c351efb95e832a853a29361675f33a7ce53de1a109cd73fd47af0712213aa4ce \ - --hash=sha256:c4165821e131d6d4ca444347c2b694e2311bcfa3fe5a861cc72968f28867beac \ - --hash=sha256:c5f5df567f6eb216de69be06ce55c8b714090fae02b18a3b40da8163b8c5fa9c \ - --hash=sha256:c941bb58d5a6e1c3892d86e42927ed6c180302f07e6d395d08c416e594b98b46 \ - --hash=sha256:c97f080ea627e2863524c5af3836e2270b5f5dfff1f104392b959f8df0c5d384 \ - --hash=sha256:cb96698e3c7413d906ce83f8ffd245ec1bd94707541f299d0ce4d6b0193e982b \ - --hash=sha256:cbb7640ce37159548d2147b5b8c241f962143d4c71231431820783f4dc78f210 \ - --hash=sha256:cdf2448aab5f661c9315308ec8b93f4e8a1a67a3c733f8631067a2b67d5913dc \ - --hash=sha256:d2117334c3af3bdcb9a88522b844a2bdb5efdc4f71c6c822df55486ae1c3347a \ - --hash=sha256:d53d10f7da99ae46f7373b9150393e9c5eab9b224909982b43832668de4779f5 \ - --hash=sha256:d9fafc5aa2e2a39aaf7f8cc0c1f044a9b07fca12e558dca53a3cc5c654ad67a7 \ - --hash=sha256:db3eb7d46527159a878ec3460e9d40615bc25ba337d477db681aea6e4f05c5d2 \ - --hash=sha256:dbf7c7a88e2bac086f06d14577332760bdeecc42bdec8ac4077f6260557d9326 \ - --hash=sha256:df2b82571a1b30f58a87bf4e5a9e78d2b1eff6c6ce8fd3aa3757221f93f0863f \ - --hash=sha256:df92f2aba50eb4d96718b68ef76f2e57a57b54f2fa62333496d16c6d585a85ca \ - --hash=sha256:eb4e8997a49aa2c08a3e43c9045d224448b8941d88e7ac163c7d383e560cbf98 \ - --hash=sha256:efc1cdd798b1aaf39b4610bba7aad28c9bea9b910f25c784ccf9ec1fa719d1f9 \ - --hash=sha256:f146d154428a2523f9cc7936c02353c2459b8f6cf07d3cd1ee1c0a611109c5d5 \ - --hash=sha256:f5bce581e6b8c235e566a14768a943b172ada3ed73537bb0c0be1edee312d4e7 \ - --hash=sha256:f9912624a0c0b834b7520d7769b3644453aabc0a7e1c839da7359f050750e9bc \ - --hash=sha256:fb62edb5bb52cca65fab91a63afa7561607120d26090a7e8fda6fb9f064726da \ - --hash=sha256:ff067a8d8d880e7809e4ac88eb009bb848870115317b306666502ccad30b147f +cffi==2.1.1 \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 # via cryptography -charset-normalizer==3.4.9 \ - --hash=sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380 \ - --hash=sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62 \ - --hash=sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c \ - --hash=sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226 \ - --hash=sha256:0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5 \ - --hash=sha256:0fa1aec2d32bcc03c8fa0f6f1712caad1adc38509f31142112e5c9daf5b9c833 \ - --hash=sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b \ - --hash=sha256:16d10d789dd9bcca1173c95af82c58433122564b7bc39385124be735a35cbe99 \ - --hash=sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501 \ - --hash=sha256:1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec \ - --hash=sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698 \ - --hash=sha256:231ddcbb35e2ff8973e1365db41fe0572662893b99a05deb183b68ad4c0c8bd4 \ - --hash=sha256:253a4a220747e8b5faf57ec320c4f5efb0cef05f647420bf267143ec15dba10a \ - --hash=sha256:280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3 \ - --hash=sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2 \ - --hash=sha256:304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a \ - --hash=sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e \ - --hash=sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4 \ - --hash=sha256:375b83ed0aecfce76c16d198fbc21f3b11b337d68662bea0a995046682a11419 \ - --hash=sha256:3c09a49d6cde137258beb3d551994a2927fd35ad5cf96aed573f61bbd67c5f84 \ - --hash=sha256:3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da \ - --hash=sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519 \ - --hash=sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe \ - --hash=sha256:432786d3561e69aeeae6c7e8648964ce0ad05736120135601f87ac26b9c83381 \ - --hash=sha256:43b9e366a31fdd1c87d0eb08f579b4a82b723ea54338f040d6b4e518a026ea29 \ - --hash=sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614 \ - --hash=sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0 \ - --hash=sha256:476743fe6dfe14a2da12e3ac79125dc84a3b2cf8094369a47a1529b0cd8549fe \ - --hash=sha256:4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29 \ - --hash=sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0 \ - --hash=sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917 \ - --hash=sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9 \ - --hash=sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32 \ - --hash=sha256:58150c9f9b9a552505912d182ccdf26f6396fb6094816ceebcbb20eecabaed94 \ - --hash=sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63 \ - --hash=sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd \ - --hash=sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198 \ - --hash=sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde \ - --hash=sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012 \ - --hash=sha256:6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1 \ - --hash=sha256:65a7ff3f705e57d392f7261b6d0550fe137c3019477431f1c355e0db0a7d3e15 \ - --hash=sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b \ - --hash=sha256:67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993 \ - --hash=sha256:68ce9f4d6b26d5ccbf7fd4459bf75f74a0a146677ebba80597df60cbdb20e6f4 \ - --hash=sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5 \ - --hash=sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8 \ - --hash=sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35 \ - --hash=sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642 \ - --hash=sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2 \ - --hash=sha256:79580094b00d1789d1f93ea55bc43cb2f611910c72235b7657f3482ddcc1b22d \ - --hash=sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9 \ - --hash=sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c \ - --hash=sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33 \ - --hash=sha256:871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db \ - --hash=sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf \ - --hash=sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9 \ - --hash=sha256:8c041122946b7ba21bb32c45b1aa57b1be35527690aeb3c5c234521085632eee \ - --hash=sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84 \ - --hash=sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44 \ - --hash=sha256:920079c3f7456fa213e0829ed2073aaa727fd39d889ead5b4f35d0de5460d04f \ - --hash=sha256:93d59d504b230e83c7a843251681959a0b6a9cd76f6e146ce1b8a80eb8739af9 \ - --hash=sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9 \ - --hash=sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177 \ - --hash=sha256:9bb41182d93ea91f60b4bc8fbf4c820c69ef8a12ab2d917f3f1834f1acad07e8 \ - --hash=sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b \ - --hash=sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39 \ - --hash=sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41 \ - --hash=sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0 \ - --hash=sha256:aa99adc8f081b475a12843953db36831eaf83ec33eb46a90629ca6a5de45a616 \ - --hash=sha256:ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d \ - --hash=sha256:ad41ba96094304aa090f5a30cb6e4fb3b3f1c264c523394b4c39bbacc4dc92ba \ - --hash=sha256:b5314963fce9b0b12743891de876e724997864ee22aa496f903f426c7e2fa5b2 \ - --hash=sha256:bcf74c1df76758a395bf0af608c04c82257523f55c9868b334f06270d0f2112b \ - --hash=sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9 \ - --hash=sha256:c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b \ - --hash=sha256:c1225416b463483160e4af85d5fc3a9690ccb53fd4b1865a6437825f5ede3209 \ - --hash=sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48 \ - --hash=sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046 \ - --hash=sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632 \ - --hash=sha256:cd6280cf040f233bd7d3407b743b4b4c74f70e8e1c4199cb112a62c941c0772a \ - --hash=sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2 \ - --hash=sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1 \ - --hash=sha256:ddf4af30b417d9fe16481e9b81c27ab2a7cde1ff7ba3e85653b02db7d145dc7b \ - --hash=sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990 \ - --hash=sha256:df7276909358e5635ae203673ab7e509ddd224225a8d6b0790bf13eb2bde1cc5 \ - --hash=sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b \ - --hash=sha256:e9701d0049d92c16703a42771b98d560b95248949f23f8cf7b4eddd201814fb9 \ - --hash=sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534 \ - --hash=sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81 \ - --hash=sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a \ - --hash=sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d \ - --hash=sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf \ - --hash=sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115 +charset-normalizer==3.5.1 \ + --hash=sha256:00668ebb0609751758682eb0b5857e7c35b9f00e84dfdef062e103244ec94d45 \ + --hash=sha256:012a22b88a77ca2e59b98ac5889b0deb604147666032f45e6d6e217634d2550d \ + --hash=sha256:01e93745f7f219b703b60ba7afead36cfc4242782be5af484673fc500df12da5 \ + --hash=sha256:04368edf83514385ffc3e1cfd4546e595f4f1272dd23ba437a93a9cc3741d47b \ + --hash=sha256:0722590aabf9dc6a6c0343d523c05458fa2b5047dbe6302fd526bb570600753f \ + --hash=sha256:07ffd07412fc5d5e84cd8952acf9ff7e4ed7a708e69d1bada19d8ba91711353f \ + --hash=sha256:09a7bba9f739468c8e78c36a75c33768e53cb1959fc638f510454c14683f00d5 \ + --hash=sha256:0b2b1b3fa5670c127b246df1d0c059defd41f689a868a3b9d79df9b1cac42d22 \ + --hash=sha256:0c6dfb5ca6723eeed15aa8e564a014d69fcb8812f94eef11fe3631e0508199f5 \ + --hash=sha256:0d929fc574b4d6fd9e7c0f5c2ede8716a41911923aa7fa5fce38e0818aa4a1ac \ + --hash=sha256:13e3afe97712e8887cd516e960c63f0b93122971e5b5e4b2622fe7701771e838 \ + --hash=sha256:15f024313246a4ed976c60f440bb8d257815513a681d212ff74fd46f7d715a90 \ + --hash=sha256:195ce897c6153c0700078142cf8efe3e6454ca4cf4357499e4078dfd83396626 \ + --hash=sha256:19a3dd5aa73cef1c99687c4fc57db016a9c17104ae1185da88ba566a5d3bebe4 \ + --hash=sha256:1d1c7a53a6c2103925cdd6d7229f8c567379f211c869793df679f2e9f738c369 \ + --hash=sha256:1f5883d77fd409a261abb5dc8ccbe335720d798b1de4abb3b1d47ccbbc76b53b \ + --hash=sha256:21b82d8082f6f5e7f456ef0bd16323d08de1266efbfeb476e64b2a91d1471a4e \ + --hash=sha256:252d099029bcbea642f2a06c4ed5046bdf8b5a8150b64afa5e027e88b106e5ee \ + --hash=sha256:256dd4d85d9e4dc595e2bc983c980e73f62ddeb3165c58b4c3dfe78c5c8548c1 \ + --hash=sha256:26422d45fd13551cf564c58932f7d72b4f58b93b0fcf18c35ba6be12b46bb102 \ + --hash=sha256:2679de311c7946dde5d3b6f44941844133ff5c7cb86099c0061ab1e8901c20a8 \ + --hash=sha256:29880d17a8eb0b5cfdfd8944b468322928059aa35f1f5fa8ff22b149ec0b42f8 \ + --hash=sha256:2bced4061f000f7187254a02ad3433ae17eaf991747ceea2f478422590a5bba9 \ + --hash=sha256:2e9cf9253119d8e5d111f05d71626786fd3d6193817316eab1ca088cdb8593cf \ + --hash=sha256:2f06b7eae9dbe77fe1d644ca244dad508de8d302870a43f3c559b521270938a0 \ + --hash=sha256:2f293479cce755c75f1697e87c409b7ae4c555c7dfecb6e988ad13abba943031 \ + --hash=sha256:329fc3ccb63ad22d867d84c2adea759a64079a37ba4a343433b02c7a2816871e \ + --hash=sha256:343fb4f2821043bd87095f7b08a1a181febc8e36ac64212143bbfd0a0e1bc235 \ + --hash=sha256:3588e376b3ea2eea84976f67273d679f229e24c66dce7b82ae45aef04ff6e072 \ + --hash=sha256:35aea775dc2bd5f54cd84a1cd2696cc3207c479cb9cf0bd346f0d343e4300ddb \ + --hash=sha256:35fe081843b35aad20ffeccec3eeffbe637b15d14f3fb22cc1b59cd8ec17e93c \ + --hash=sha256:36047af20e17097c3bb9476c2b7655f2f7aa51322c0ba58c07695bedf755a950 \ + --hash=sha256:3617ac3cfd8b9888f145ad89dd6e692285834b0201c6074a5eeaad3fd4d668c2 \ + --hash=sha256:366ec70f5547c640d3ce1985722490f23faf4eb5216a7eeba78277490e78dacb \ + --hash=sha256:394fea06235c8543390050ed5f529187074b029fb027213f6c46ac11ab5d950e \ + --hash=sha256:3d27167433c0d5f18dc850f07d0b3816221984fecdc405d6c157a6f0b8f8e9e6 \ + --hash=sha256:3e5e1224c0a6a90e05843e07adfec669edebec17801c67072f51e59561d63c0b \ + --hash=sha256:41876ee62a3dddf48ff1121ad8f0798032aa03f2fd35f21f34a4cab14f18d8d2 \ + --hash=sha256:433c5a81eade63b47e522303bad236f59dba55ea6951746f5558355eeed8c75d \ + --hash=sha256:4582c27e8c889d64811987b5967fbd3ae0c823fe1fd933b543d55ac20bb475fa \ + --hash=sha256:485a0d363cafefcd2538a73c7c838daa2035f09b2c9f9b5e3133f80c6aeb84c2 \ + --hash=sha256:494b70049a4d69aec6e8137c13af4cf8db8c9f9820a1392ac293b0dd2987a818 \ + --hash=sha256:496846868fea80e479324862fa877f02411f2fd0f83b79ccee2607aa68b2a032 \ + --hash=sha256:4abdc5f9ad448c1ecbfae2974b820535d6bc6e7eef63babbab3d81cf46968c71 \ + --hash=sha256:4b599739b93b2cbeded49645ae3c8d1405c29ddfbceac1545c87a3f9580a9e96 \ + --hash=sha256:4bea7f8ebe90bbd7f0e4a2de42ca6924ba23e3e76418c408ff82f1d46fabd687 \ + --hash=sha256:4c4fb141a727957c93edfe5c32a26ceb6b5f6461d67146e2d39f51e16170bea8 \ + --hash=sha256:4c9548dc78002099910abaebc0a72ac58b7d30931869e0351c09b507dff4ece3 \ + --hash=sha256:4d26f14f041e83dd8edfd61f4cd4fa7285d31798b5bf1f28e70c367ba6c41d61 \ + --hash=sha256:4f298bdadb8f0b9e5672877f647d1be9373ef5320c9e2f049795e26cad28b6a9 \ + --hash=sha256:52ec005752a56ae79547a05c0139ca2501a0c866390b6115008456b9f0e7cde1 \ + --hash=sha256:55261ac0d2941c42f196dd576f543d87a8ee03cd6f5e30dfb4d807b2e3b9121a \ + --hash=sha256:56490c595a28b1bb27dfc583e816152a9767721ef58b2c03b13f954d2f707420 \ + --hash=sha256:58d3e12c88e0950bca850ae1f7c256055c097639c2edb9eb123af9807d8b15e4 \ + --hash=sha256:58d4aa13a59c969dbfdf9e6a9560e242cbfd9e8a8f50c2747714df1a423adf65 \ + --hash=sha256:59171c6e45bf07d0d5cab3b0bf81d945035530f6873398b3b531c31184d46663 \ + --hash=sha256:5b6d1386bf0096d26d3a863dc0a487a5b4eb9aa93cf5ba69683d29dde6b9d60f \ + --hash=sha256:5c0ea61a470e070686aa30892fed79e297d2c8d0ab46b8bcdf027d38c51da591 \ + --hash=sha256:5c84bec0ab5ae0c64bfe73a7d2adcb5ce73b467523fc27fd6a28ab2aa6cbe35a \ + --hash=sha256:5ca0555312ae2fe82715cada7fac375530c2f3349e1eaa1bcb33d0283ac79a18 \ + --hash=sha256:5d8531a6569d025f68e2321e7638fb7978f23db58e5f69f56913837aae03816e \ + --hash=sha256:5e2d0e146dcb57034f8b97dc58d2d512cb90aba253960ce449f695fec6a82c6f \ + --hash=sha256:5fc45d653ea8c9a20479167e11d4a0f8cb2fa3470737ab6f9c827532313187b7 \ + --hash=sha256:6117b84ea48435e5356dc737f5121485c30920ba43375fa7b434fd753df0eac3 \ + --hash=sha256:6199d5606e2bbf2b096cf64d03f8b6790c91081d5ac866b8e7bb6422738cc60c \ + --hash=sha256:62b55f6722735a6c472f88361cde6640608773d9443cebdbb51abf436a1fcdd3 \ + --hash=sha256:687c9ca3035544b113bea2055e180af96fb63c0c476e22a9180f51925186e7b7 \ + --hash=sha256:6b7430cf5728e68f6c462254009a6ef4086e1bea43cf2f57aa9c55fb4f50ff96 \ + --hash=sha256:6ba32c4d2abf1d2fe7cf27d280f4cca5664233b0f885549c7761719eb977f486 \ + --hash=sha256:6c9cdde8becb25a7fde49924511aa2644d6f8081cc8df8e9452724303348d8e3 \ + --hash=sha256:6df0ec430f9a831772c23ca5a224cba36517a58a84bb32c32bb59a9fa67c47f6 \ + --hash=sha256:6e2912d4babbc65196ac13c2f53468dc57fb8b9c25ef913e8c59ddf7c6dc0e1b \ + --hash=sha256:6e5e4d73d588ca5ed09df1b7dcd1b203d1df3c542e3f50d126c947d432b10731 \ + --hash=sha256:70055ff39b97c99e7ae40ea3e393fb62aa2e44dbd9b29f8d14f42fb0025c3959 \ + --hash=sha256:706bfd38730a5ac7a365793269a00f4e988178cec121391f4248d84ad8c972e9 \ + --hash=sha256:7235dc28fc6dd9d832ac7c7bce95367dedb85929f17368a0c2bee1e080b9acbf \ + --hash=sha256:774d157f112367ff4abd29019f38f023c24e00e56edc7829c20e358a5a913ad8 \ + --hash=sha256:77efcff2b23071c349402ac1066667a3d011f62398d81408c9b88ad991747c9e \ + --hash=sha256:789b8982559ae28dad2356519f841655756cdcd96616410590ae0b17454ee64f \ + --hash=sha256:7ac76cf9afd34929d76eb7fcb63be476a4853d8a96f0dcf2d0db68a0cbdf9885 \ + --hash=sha256:7c0c10730342b0c9b35dd1d619beb8214e520bd96a1f870f452680b238aab3e0 \ + --hash=sha256:823f82903d189af463d7df250ef1f7f696f3cee08cc8d91deb565e8d425f6506 \ + --hash=sha256:838648accb3a7fd9803fd45c87bce8509648eb0c11bc34e216141300977244f2 \ + --hash=sha256:854066be00447fa8de2ccbbe893e2ffc4b123ef16d897af794c1e18bd4a714b0 \ + --hash=sha256:85d5855daafc240cc045c026d7a15fd198a09b0fc8ff6f5ecbb5297b509cb11e \ + --hash=sha256:85de3134b5379856e323ba37c19c9256d39425f7b76a63af52b09fb4664c2e8f \ + --hash=sha256:87e4f41d375c0b9be2fb5251aee4b8a689169e134535aed81bf085c3b647451e \ + --hash=sha256:88ca277405c2d3b71c4e1c2ee0e7966e807bcba86a69d11e19ba199d18ae4491 \ + --hash=sha256:88e85ab89cb822c1e635f51d6d32e488f94e002e70e2f492bdb8b945543f345a \ + --hash=sha256:8ac8c94b6539074e0f40899301273ac8402b9b3e01c7b7ba269ff30340aaaf20 \ + --hash=sha256:8fe532b3c966d1fb794e0698e4589d0444017ae77fc0b31edea13c0e35bcc449 \ + --hash=sha256:9085f87b0e38a2b92b8923059b4e8789fe40d9279712d15dcc670048d77079af \ + --hash=sha256:90b7481fb62fbe172c558bc6fd1c4c98d82004a54a7551f20e11ac9bf0b8708c \ + --hash=sha256:92caef967d287a407085d61176fce4012b1dd62daed4eb6d5ceb26d3d2538712 \ + --hash=sha256:9362dd90aa7dab48c0054a21187791ccf05473f7dba5d92b8033ae62164675e7 \ + --hash=sha256:94d78ecec2605a8d0398b0f365d5f12a63248438516f5dac536a5eff7337df4a \ + --hash=sha256:94fbf1c0c6cc0d3d5e50f9a9313a8cdca90dd696d34b381cd1704f8c9e939f20 \ + --hash=sha256:950f23cb393f85543777b0433f082cddd25b51ab398eac7971146495679efe5f \ + --hash=sha256:96eefc178f8636b9c760c5829345307fd81cfae9ab1e80997dbddeb0f54ee9a3 \ + --hash=sha256:96fef3e886d6a9874b14f27fc193fbdc69d5d8035783d86aa4e1cea594e695f9 \ + --hash=sha256:977cdbd483a9cff38179bea4fd754289a6f2195c7abd414aba85410b3e66cc5e \ + --hash=sha256:978eab16f55b4ab2c2a745be9a0a840bf8f09a7f227d9c76eb30214d078865a5 \ + --hash=sha256:994e883d17c559cdfd38c84003c8b27d25424a1077272a17e7cd27bfe0bf57b2 \ + --hash=sha256:9ac4444d8d4fd4c4bd08bf451ed3167aa9e7ec6cdb41b648794f1d1103652e36 \ + --hash=sha256:9b5db6052055d34d41230fb78d7c439c23dc536a9896f6cb039e8dd92cfc1263 \ + --hash=sha256:9d9a0dc7cbe9bec24c3f767c9122c41fe5a1bc43f47cd099d00d393e09769de4 \ + --hash=sha256:9dbdd9205662134957cf0c324f639bdc5031c0ca056e2369e238db75187c0f11 \ + --hash=sha256:9eea3ab2597a5e65fe65296e2d6a84570845a6b55532d90333d740d48bbc850a \ + --hash=sha256:a2028475ba855475b8b4d3cfeb4994269c967aea8b9892dfba907f4263a863a3 \ + --hash=sha256:a3a370082ce34d0612f421e15fe011c53bb1feff21a26d06ad4fb244dab5a375 \ + --hash=sha256:a545775cfe815855ea32d7c27731d79da358ef2055b4a25830231b1622dd18aa \ + --hash=sha256:a5cbd90ecf0fc62e64726917ad083b73001f0563657a87ec3c0b504e277dc90d \ + --hash=sha256:a6d095662e73e74f0a49988e0593373e243e3a52e27bfeea0a859e88acf4a0f5 \ + --hash=sha256:a6dac12ff6b846103483683f60c5f8fee205121adc58ffd87e90a90a3af69e99 \ + --hash=sha256:a951ad59cad9145664a730d3036b40b844e74d2d3683da40111463cd3a83845d \ + --hash=sha256:aa1099b956fb795e686d073568f6dc002a0bb89765ea6d5b055dd7d9bf1b116c \ + --hash=sha256:aa2bb0b37202dca27175591f761108b5d34096ade1191ffe4808bdf6b1571488 \ + --hash=sha256:aae2ee51122d3ae968a3837d97dc24a0aeebb0dea23694422cd172bd30017cd6 \ + --hash=sha256:ab743e9bc90c1f73552ec33e10e3331315acd2c397b36065b591b0181de533cc \ + --hash=sha256:ac00177c4831ffa650f8609e4bdddd5fe09c03b1c0c47acece7e6ea20421598b \ + --hash=sha256:ac13b004224fb341e1e25a1ed5e19d32f57cdb2a403e01f003b46f051a550f6f \ + --hash=sha256:acaf604462bf330b0d07e7a07c1d6e4adac79e5fb13e9c5140590542cafacc00 \ + --hash=sha256:ae31a1a1db2ee6cc2942fccaf695c934bc7f3db9f2133a3fef1f367cf1a4ab10 \ + --hash=sha256:ae4a097991662cd4fff0ddc74e0fe7874f82e00042fa0ea00855645ed0c79598 \ + --hash=sha256:aea996a6aba25260827c9ea511d1addfde2da9eb686ac961838509086188b7e6 \ + --hash=sha256:b39b69b347e5e47a3b5b8cfc005c68c1ba347474e3960236c4944a8ecd174962 \ + --hash=sha256:b54e7e13267d49ffbfe68e25b3cbd774dab38fa37238f71265e91b36146eb21c \ + --hash=sha256:b9af956078716df40d985fb0dfeb2c2120c5ca92ba4ff4b388acfd01cdc14d08 \ + --hash=sha256:ba2f37ee79e6338845261a3c5b1784e5d1acdff2c0785b284f1b633033d136ab \ + --hash=sha256:ba501e667c17d8411f98e67a022d9604ef179aff0e459b7e292c796837c13573 \ + --hash=sha256:baf3775a2635e5a11fbd5e4e64ee69c7e86875d224a5c72aca4c141064589a90 \ + --hash=sha256:bb57753e36e4855b8ca375069482250a6246372331a3e4f3407eaebb007443f5 \ + --hash=sha256:bd6c173f04743d483881bffa1478d5a4624475b8cd1d2194956a75548e191c18 \ + --hash=sha256:be47f99644b208bff7766314013f9acf57b056b04191d570d68ad14022cf5b1d \ + --hash=sha256:c010f5581d9c612804cc59fcf7b524b707fbcb72828551237ab545bb5c7034af \ + --hash=sha256:c1dcc36dcb96abc02236e182d17e0f71430152a6c2c7447421da2d2dc144edea \ + --hash=sha256:c428c6c31eb5f4277d7f8eccaf767fbd548ddd5ce3c8b4f4cbbfab3d96b5904c \ + --hash=sha256:c658c50ac0c98cd755a2dd50b7977d3bca7df401dcc47fbdfa87db53ef7d4e8b \ + --hash=sha256:c71fb0d56c920c269cd3e2e3fe7c610e3f1fdb21a6ce60efa6430ff63676cea6 \ + --hash=sha256:c7b742bf31c88566b4bb6335a7f393bb322e580b6bb98df7bd0c25e6e3519ce8 \ + --hash=sha256:cc0329df4caaceb950d2f580b5ac716a377f7059624a0bafaeaf8a218c6ed774 \ + --hash=sha256:cc5d36d96478aa9c60654bd932525bf32964c62a7281eafdf16d85003a8d6004 \ + --hash=sha256:ce854f5f478050ade5a238731c4ca985a7d3b3cb53ff600a9b5c3b689b5f0a7a \ + --hash=sha256:ced3fdd71aaa83ce593746c2edb42b7a59cb4c19c8b5c407781c72e493aae55a \ + --hash=sha256:cee5dd7c6fb5dd52a0fe2a740f9bc6e3593f5f8b1788bde49de02086f30182b2 \ + --hash=sha256:cfa1c0cc3a8f9f53f1243a5a99ac36fd003880199383b37672e86ddda9cb07e2 \ + --hash=sha256:d1ee1e296209fdce05b81b663250eefa02213a2da7b41bf26f7829b8ba3545aa \ + --hash=sha256:d59b75732e9b6f27388e10c14b0259cc5f2e48c78627d185e6a177b58ad3cffe \ + --hash=sha256:d63600d620ad0064c3a748b950ac5ea38a80190e5498532efefa4b7b3f1da1f3 \ + --hash=sha256:dd732602a7009217f658d5863d12d79d373a4de0eebc111094bcdd3bb8e0a6cc \ + --hash=sha256:e06efa066f7dbadbc84ebc126a97c452a6451dfcf589d89d788484949e1cf795 \ + --hash=sha256:e199fb99720074809a7720f1c0b4d919eea8b87e88713e0f8f602f7bef543d9d \ + --hash=sha256:e4b018dc5a0eee4676e38fe84a47a427816c590b93b55d9025274ec4d6ffc2dc \ + --hash=sha256:e6621fb2a4988d6e53eedc455e5903e2679f3967b8acb3d639f1b63c14a2e893 \ + --hash=sha256:e71c909f353863b2b89c83de2ebed71ea6d0df8a6ef65a128193c5e650766bef \ + --hash=sha256:e90251c0c7bdd54a100a0dce3c07b7e637278c93af29dbf78ebb89a58c4bac7d \ + --hash=sha256:e9fbdce1e47394b09bc9f26ab117dfc8d6491977a11d86f592bb42c779db2fda \ + --hash=sha256:eb12fb2ba69ffa05f8695f61c69e591dc4b4a12ac3757ac8af8adb259bf56d17 \ + --hash=sha256:eda059b6bc8bc0812d626fd91a7ce01bf583df0a61296eff390fd94141a34e30 \ + --hash=sha256:f03ac127268b43ef4fe9e6ab6794a6794b49485a0cc0c1db79876d2f33f75bc7 \ + --hash=sha256:f298e218441525d3794428b4c8b8fb8662c6d3ea79925d4807ee6b9a96a3bca5 \ + --hash=sha256:f5542f9b941279d82d41eb0aa9f98eba36fe4df5c7086c651df7944935b37182 \ + --hash=sha256:f6f7deae3feb4edfa2efaf7c574fe88cbf055038a6abdb40188e4fff66d5699f \ + --hash=sha256:f9b1e28d0e8dbfa858abdba91d6b547beaf2df1a59bec6da6faae7b96a4991a9 \ + --hash=sha256:f9f8405c2c758532c74fed975dbee57be1f31a6e865c031870c79a6ed3212ada \ + --hash=sha256:fa48b1b63d639f9483e0633e092f5851e2348c352f1f9bb6c8182f87884ef876 \ + --hash=sha256:fb78f6e7fcd8ad785d28cd577168bc1aaee827b25bb8755638f694794ea98f0a \ + --hash=sha256:fbc597639158fd7c14d55e808718848319540f51b0e6746e3eefa59723a4a348 \ + --hash=sha256:fce8cbd4997efeb450bd298b54f755dcdff18d496f7a5ddbb4867c6d7c88fdc3 \ + --hash=sha256:fd0350afdc3aabd5576f60ea109228bd5538139713c7b094c5cd27c73a98bc6f \ + --hash=sha256:fd0a274c0e5f9a21565cd9d3dd749b61f96b7aa1e20a93aa1ba4029518f2e5c0 \ + --hash=sha256:fdb8a068947befafba9952162645dc2fecaeb400e64584829ed5e9b2fbe21a7f # via requests cheroot==11.1.2 \ --hash=sha256:0f6c0ba05c00fbc869fb46b1de4ec2384e1d85418ae963d3bc10ae83b688dbfa \ --hash=sha256:bfb70c49663f63b0440f2b54dbc6b0d1650e56dfe4e2641f59b2c6f727b44aca # via xprof-nightly -cryptography==49.0.0 \ - --hash=sha256:026ac7423e6fa66872d3bf889be5974507da3944f866f704fa200eadacd00001 \ - --hash=sha256:07cab27cc7b7e0fd28e5e26bb9eeedde5c135c868b46de4a27845abe94af6122 \ - --hash=sha256:084ef1af862eb07ec46d25f68689f2102a9fc0e05ce7b80f14f5fe51e4eef0f6 \ - --hash=sha256:0b82e28ee398a386f0807bba7884d30f25218855690f45115831bcce5d90822c \ - --hash=sha256:0e959b578856a3924bc0cbb710fc12c387b9412a951389f3ca61704a9e25f325 \ - --hash=sha256:0f21641cf4b30fca7aee061ced0ec7ad7b073518088b7c9969a297c0ae796c69 \ - --hash=sha256:196ecd6a36e4e9aa10270393bb98d8df88fccee0bf1e5128b91ae4eb4375896d \ - --hash=sha256:2400ef9c9e2299a25614eb1dea3db54a69b1349efd043bfac9c67630d136df36 \ - --hash=sha256:28d8b15e6275f12c8a207dc309dfa957903c927d08d0cc937ee3f63f200693cc \ - --hash=sha256:2afe9051da7ae7bd5905da5a949280c7d2bb75682e188f650a9d0f2756b834c6 \ - --hash=sha256:2eda353d8a27bcbcaa4cbed18994a74ab4d19a2ca897db188ea269ab9b71419b \ - --hash=sha256:32703d93296f5c1f4b53349ad3a250c2cae0fdecd3a3dd5d47e616d8d616af27 \ - --hash=sha256:33cd0565932807baddb67b96dbee92f2c374b5c89dee09fd74079aeb8c8dba61 \ - --hash=sha256:35b151772baff2c74cba7fa290ceaff4c3b11c0c881eb93eb5dbc05a7cfbba18 \ - --hash=sha256:36d1709f992593689b45bda411498d62c6e365f2ca00b84657d4dadd24de16db \ - --hash=sha256:42b0684e0e40cf26122427802486f6d93aea593612603a94fbf260c7eb1e9c1b \ - --hash=sha256:4ae387c9cb68ea569ca17e490d66d8142b81c3cc814bf179974b7d146e490bbb \ - --hash=sha256:53ecee2e23f7169b6117e99fc8a944e5e50f79e69758a83b52a00cb98ab2b2d2 \ - --hash=sha256:66ec79c3904820572d7e987abdf304281f141d37ad9a489b8e97066e7b9b6459 \ - --hash=sha256:67e1d20ad9ef3a563c59ef22e7a8a0b8210bd26604369ea4a30a7c66aefe504e \ - --hash=sha256:6f2debedf9ca60cf1d5bd466475638af5130f89965605cd818484d19987d3a21 \ - --hash=sha256:6fc361c34fb6aac015ce19435876635e5c6d21db31998b0920f675f131e043b8 \ - --hash=sha256:73a205dce83953d131a4aa1e0fd917a2fd1c5b1eef251e9d7152efefcbf5caf7 \ - --hash=sha256:7abcee80084cda3f7691f3eb1ce480d8df49cec637b429aa35986c1de71738aa \ - --hash=sha256:8c25ceb16df5b9435f3f6a9829204985b0e0cbee3b48aacd432c7d2c850b44d9 \ - --hash=sha256:966fe0e9c67490071f14c0d2b1cb2dfb3023c5ce39457343931415f08382f2db \ - --hash=sha256:9e82dcc8e56052715fb18b2429e3bca4823b1629136a2084fc45a9a5cecb9b64 \ - --hash=sha256:b20133d204d2bb56ba047642199603876c872026ca53e79c35b83772ab2cc505 \ - --hash=sha256:b39efa323140595abd3ecca8529d321ae50f55f3aa3ba9cc81ea56a6011953d5 \ - --hash=sha256:b47db11c2c3525083296069b98ac5221907455e989ae0c2e3008bde851921615 \ - --hash=sha256:b87e65d263b3e5d3bb92a57e2a6638e2f31110fa7aa890c7b2dbba42248d0a3f \ - --hash=sha256:b970c6da94d5bb18629db453d14f2a1300f6bf59b61e9b82377931ef95504866 \ - --hash=sha256:be9fcb48a55f023493482827d4f459bd263cc20efde64f204b97c123201850c6 \ - --hash=sha256:c2bc30226390d60ea19d9f82b19db005fe0452154a23c1c410c12ea801e43561 \ - --hash=sha256:c83782480a4a9da4d0feb51950131ba32e12e70813848b3343f6e18c28a66838 \ - --hash=sha256:cbc77da8c523d5abd028635ba850a6966fcee2c82e2bf65a41d1d8afe0f98be9 \ - --hash=sha256:ccac2bfebc306b862133e3bb71f3f6ee8bb525240089b2d952e4144b3a6d5da7 \ - --hash=sha256:d0527ce944105f257f605a827d6ebead966c752038b6e8656abb9c5edee6fc68 \ - --hash=sha256:d8ecde755e2e91bf773fc94e8c9d730cd7f2007004cb492263a794ec3899a1c8 \ - --hash=sha256:e3fb64c420688e5319ae25113a354015abbd8dffbfbc41781a1ea66fc7622ac3 \ - --hash=sha256:e5dfc1e64de5677cec922ffa8da89c546d0415bf6efdf081842e5d44c84e1f0e \ - --hash=sha256:ec5e529fb80935c94fe7b729f9972b50e351a0e6b50aa294fd5cabb109fcc29a \ - --hash=sha256:f37d847238971164fdbc68ade6f6574aecc9c0af714190e2083429ff68f4ce9d \ - --hash=sha256:f78ff2c9ed8dc2d036b0f4d640e22522213d047c1b14e61205a7e55c80a494d4 \ - --hash=sha256:f89660a348f4f78a92366240a61404e337586ef7f5909a2fef59ca88ef505493 \ - --hash=sha256:fc1e275c2f1d97b1a6450b8b0ea3ebfa6e087a611c2b26cb2404d48588abab7b +cryptography==50.0.1 \ + --hash=sha256:01f41478cf33fc605a6a089cd56d28b45c6c0b45a1928b61797f2621a04bac71 \ + --hash=sha256:05ba322c4da95b262a212c345af888ef2c37c88c0509756ea00a0e6d68850f23 \ + --hash=sha256:16c5ecd954b3330ebfb6605eca4fd952da8bef376551d5cc264534e3770a9ee6 \ + --hash=sha256:2a93d05e34d5f67fba6f891fe85d929999baa7195e853923ea6d7576c9e68c5e \ + --hash=sha256:2b34d76a652ea2b6faf777c35df230c5637842cd904e04f16230c3f9f03e4361 \ + --hash=sha256:2ebbfb0f1fed745e91796e3e1080a1440423fdae8ece1b995a1d80883a409054 \ + --hash=sha256:30a125032e5642a21ff816e021152bd4e7e94f03eff3f4b7fca41cd22bc3110f \ + --hash=sha256:330fbb252391c596f1ae42c5754449dc924e6ad012dca8efe0d703f9f2d12ec6 \ + --hash=sha256:359e62deae718bce96170e223fdcb6357e4fbd3bb7a3a75f4430763532560e49 \ + --hash=sha256:407fe2b6db00939c05c0e945e9914238f2f0a430974839429dafc82b1ee6bee5 \ + --hash=sha256:42be3bb70596b3abe4ac097b75be223e8b3ab614a0e5de068e3dcc54d71d6149 \ + --hash=sha256:4c4188f7c0cf655be5c06342b817ed0f9595b69ffa2b12026e5353eed29dea88 \ + --hash=sha256:51593d180cf6d179bde5c5d065bed81386b1f381656ae7d042b7ffc87a9895ad \ + --hash=sha256:51afcfceb15597cf2635068e4ac9a56b2abde622edde17f37d85fd7b5306497a \ + --hash=sha256:53e279950892dc102c6b4e52af03ae5ea92fac572a1ddab78ca73a997f62b69f \ + --hash=sha256:55d16b1ef3ee0958d893a977b19777887e546c9954ea81b200c3301a864013f2 \ + --hash=sha256:5dd9bda1c12b4162f6ff568eeb5e0ff956c28d14406e875cfe8a63a2d414ff20 \ + --hash=sha256:5fe002589592ed749ce77fe0695fcbd3500dd61d7d6db5858a7544c612fa8e45 \ + --hash=sha256:5fe939deeb161024a6be98229c953b6591fef1f41214497a78fe793a244c017f \ + --hash=sha256:693c99b49bd37d0d096e4334c10232c77248c415b98d35236094cdf96d57258b \ + --hash=sha256:76de83fbd91ac49c0feaaa983d0748fd7a53176afac5fb3bf7478d244f0eb527 \ + --hash=sha256:79bf008d1f9af6071c797ad133e39915dfee7614f18f18f4db9072eb715064a3 \ + --hash=sha256:804728ce710890870f3aaa344b2e161172d258d768ac139d02cfd9092d0d94e6 \ + --hash=sha256:8921d58f426793c5f1b47f0b59575780de9a095214958d0eb37d909593db8367 \ + --hash=sha256:8df2de9102026855887e4587084f6eabd80ed0f345b8ad8a7ac27ab9bf4723e0 \ + --hash=sha256:9cb3cb952cf5a8abd50c782a98a89d71699715e802fe349704b47f2425b42a94 \ + --hash=sha256:9dde0a357190eb3b1da1bb9ab750e9c85cba82ca5977aa0836cbb94e92611239 \ + --hash=sha256:9ebcdd5519be9b652a46f507817a74591774fc3d6923ac364e4dfa64e36b291b \ + --hash=sha256:a0b1a59e3a089064a0ec309e9428c8e3ae4e161419d20ac33600767e83fc658a \ + --hash=sha256:a255449073358275b64b67d3f595f268bbef70e72b6edb65e0c70c735bf739c9 \ + --hash=sha256:a8f40ea47330e71b594a7e246898f93177c259490c63183dbaf9e571d71ed9a5 \ + --hash=sha256:ac02b07824d4d1001bd4367599f839c19cb171924c796e52c23508ac14c2c0cc \ + --hash=sha256:aed8db4f6d71c51efb89530e12d9464e7bf2923d46c3205dc794a2a93f8c0648 \ + --hash=sha256:b8f852c65863251b9e3a1b8c150ce21e59b522dbb6a7d4bc80e680d38388e986 \ + --hash=sha256:be224a65493ec5b74a158ff22a5522ce4a5ca1e543c647a3a4730d4a09e5f959 \ + --hash=sha256:ca83d00d9e69cd5eb63f2e69c3a5a59e0cecae5ae14c6ae0b35830fe3b37bad0 \ + --hash=sha256:cbf74a81765ee67413503ca6e26dcc4f6f5a519822436cc0a1b97aab6c1b8a17 \ + --hash=sha256:d63ae8f6481fec907ac0f588eee8a90aefde112c633131fe540e5711ddbb5a4e \ + --hash=sha256:e22dfed744bd4002e909464cb23d2f0b05c6f3113a79ef2e9864a53db737c733 \ + --hash=sha256:e2ca8fd1b6b4b82a1c4cb02841d0837e3c12336c2e24b520ab8ab3b969733d8f \ + --hash=sha256:e74591e283fe6eb956416c929eb58262a719fe0311fd9054c62c3350ed8760d8 \ + --hash=sha256:f74455bb086a85d5e81246412602aaa97ed095e504cd40dd261ef50be42205bf \ + --hash=sha256:fb4b9672d389c738b175c4166e78310f8a70358886aacd9173ee03a85ffdc671 \ + --hash=sha256:fc3ed7ebd2a8c96f5b166de0ab9b624996bef3b07bbeb19364dfb78222c22c80 \ + --hash=sha256:fd3718b960d0b5dd213cdf03f3bcb7000e69dda0de8b956061947ff6bcff5558 \ + --hash=sha256:ff838d62ec1bfce4f9ba7fa16f4a7b554cd8d0c299e6be37502161a660c84eef # via google-auth decorator==5.3.1 \ --hash=sha256:4cbcdd55a6efadb9dbea26b858f4fb3264567b52d69ca0d25b721b553f60ea82 \ @@ -536,27 +615,27 @@ frozenlist==1.8.0 \ # via # aiohttp # aiosignal -fsspec==2026.6.0 \ - --hash=sha256:02e0b71817df9b2169dc30a16832045764def1191b43dcff5bb85bdee212d2a1 \ - --hash=sha256:f5bac145310fe30e16e1471bd6840b2d990d609e872251d7e674241822abf01a +fsspec[gcs]==2026.7.0 \ + --hash=sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279 \ + --hash=sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88 # via # etils # gcsfs # xprof-nightly -gcsfs==2026.7.0 \ - --hash=sha256:2df36ce1e9010e76dc4295774030495a97496838483c619e9f63bc0ab7bdc770 \ - --hash=sha256:8ce6c08ea64302d8fd4bc23a615ccd9e7752541cde1a88ec8362109bd79cde78 - # via xprof-nightly -google-api-core[grpc]==2.31.0 \ - --hash=sha256:2be84ee0f584c48e6bde1b36766e23348b361fb7e55e56135fc76ce1c397f9c2 \ - --hash=sha256:ef79fb3784c71cbac89cbd03301ba0c8fb8ad2aa95d7f9204dd9628f7adf59ab +gcsfs==2026.8.1 \ + --hash=sha256:66e6316690b17e1887a4f4bc0902eee53f5b3fd860cc3f0377d65c5be3313b1f \ + --hash=sha256:738da7dcc74817c7668d032f932598482a23cce1b061c537e54278be44d2fff8 + # via fsspec +google-api-core[grpc]==2.37.0 \ + --hash=sha256:cf58f220aa797f1ffdda52194c4c7d72efeced09297c2976529f8135f6d85b9a \ + --hash=sha256:d84042a0034cce9c4304e17d2e5e9966c2fcb1a851b73799291f687a67470c8b # via # google-cloud-core # google-cloud-storage # google-cloud-storage-control -google-auth==2.55.2 \ - --hash=sha256:97ae7790ff740f2bc9db60eb864a7804f4ac19f5f02c38b3d942f2fea6e9b9ae \ - --hash=sha256:d715f265f2cafc6a5f1bf0dc19870d20e3119f6f6682785a250bce3d03d38a3b +google-auth==2.58.0 \ + --hash=sha256:55e30cf15e737de92c5323d78cda8a83fcd57e7ffbaf900c4600039fd60a80fd \ + --hash=sha256:8a9c4645bb4c8e91668fb1934b95ae6a8687084232753639220ba9bf04a1610d # via # gcsfs # google-api-core @@ -564,23 +643,23 @@ google-auth==2.55.2 \ # google-cloud-core # google-cloud-storage # google-cloud-storage-control -google-auth-oauthlib==1.4.0 \ - --hash=sha256:18b5e28880eb8eba9065c436becdc0ee8e4b59117a73a510679c82f70cd363d2 \ - --hash=sha256:251314f213a9ee46a5ae73988e84fd7cca8bb68e7ecf4bfd45940f9e7f51d070 +google-auth-oauthlib==1.4.1 \ + --hash=sha256:1a83f5f2a8421dedadaa3caf25b3a710dddf85a33a63144be41c2fc79174b106 \ + --hash=sha256:a1be43ec69fe563ac9b2c4d6fc4334b323b21cbdc59a638b5fa34dd4d5a2a348 # via gcsfs -google-cloud-core==2.6.0 \ - --hash=sha256:6d63ac8e5eca6d9e4319d0a1e2265fadcd7f1049904378caecfa01cf52dd869e \ - --hash=sha256:e76149739f90fac1fc6757c09f47eaccb3145b54adbd7759b0f7c4b235f46c83 +google-cloud-core==2.7.0 \ + --hash=sha256:874aaf89765db87a9b911b7a2ca7c5068554868eed9e75c7766affe342a2913d \ + --hash=sha256:c18a250904cfdda021eb3ae8b8238c9f9ca272a4cbbfb5cba946b3fe3022eed1 # via google-cloud-storage -google-cloud-storage==3.12.0 \ - --hash=sha256:03ae9847c6babb368f35f054126b8a08cbc0e3266efb990eb17b9926a45cf3be \ - --hash=sha256:3880773754ddf7c27567b04e2a4d193950b6b99429f37b9097d873686e95b09c +google-cloud-storage==3.14.1 \ + --hash=sha256:8f0fe2b74bd80ddcbd67247a3b47b81035a0a4f448569ed358fa40a46c940118 \ + --hash=sha256:b24e74b493c60b19b83462933a83bb831e45b6ca4924b0d75eac8e176b58b3a7 # via # gcsfs # xprof-nightly -google-cloud-storage-control==1.12.0 \ - --hash=sha256:20f7f1252fa5635d47e12f746aa69d719e31eb9405cbbd14cdfba317db27d421 \ - --hash=sha256:49090d03532c0c84c6246a5fd490c31fd4bbffb4c7771c0a6744ec23a194a5f6 +google-cloud-storage-control==1.15.0 \ + --hash=sha256:46c8c801ff0425a1bd59bc28899a8242e15f6bbb06a4f2304cbd169c7d26cb29 \ + --hash=sha256:e3d50276f0aed0c33ebcf3c6221e933889e042df70a56886ddffe59c66889183 # via gcsfs google-crc32c==1.8.0 \ --hash=sha256:014a7e68d623e9a4222d663931febc3033c5c7c9730785727de2a81f87d5bab8 \ @@ -619,96 +698,106 @@ google-crc32c==1.8.0 \ # via # google-cloud-storage # google-resumable-media -google-resumable-media==2.10.0 \ - --hash=sha256:88152884bee37b2bf36a0ab81ad8c7fd12212c9803dd981d77c1b35b02d34e7c \ - --hash=sha256:e324bc9d0fdae4c52a08ae90456edc4e71ece858399e1217ac0eb3a51d6bc6ee +google-resumable-media==2.10.2 \ + --hash=sha256:1de441703cd298d75a419bfdc0066e9fc7b0a1de630df96eea8ce8f5c759358c \ + --hash=sha256:e3cedc827a4ea41e216582d74346f1fb9fceb625a8c3c53912f2ca1d663334d7 # via google-cloud-storage -googleapis-common-protos[grpc]==1.75.0 \ - --hash=sha256:53a062ff3c32552fbd62c11fe23768b78e4ddf0494d5e5fd97d3f4689c75fbbd \ - --hash=sha256:961ed60399c457ceb0ee8f285a84c870aabc9c6a832b9d37bb281b5bebde43ed +googleapis-common-protos[grpc]==1.75.3 \ + --hash=sha256:57c435ac2c68b108999b6db075d9053e4d7a936ba57b4a3d45667b1346f1738a \ + --hash=sha256:a018d2bf098ca9fb6faa08d5bb780e2a2c2f73c566f069761331386c9596d3f2 # via # google-api-core # grpc-google-iam-v1 # grpcio-status -grpc-google-iam-v1==0.14.4 \ - --hash=sha256:392b3796947ed6334e61171d9ab06bf7eb357f554e5fc7556ad7aab6d0e17038 \ - --hash=sha256:412facc320fcbd94034b4df3d557662051d4d8adfa86e0ddb4dca70a3f739964 +grpc-google-iam-v1==0.14.5 \ + --hash=sha256:07fd3a9fafb586588e771831fbfc8f6597050181d0c3b45e039d18b8fdc1aab5 \ + --hash=sha256:0f5e680b20aa0a9441e68c769da04d94d70fca4e43751a82d8abb8aa6a7181ca # via google-cloud-storage-control -grpcio==1.81.1 \ - --hash=sha256:0490c30c261eded63f3f354979f9dc4502a9fb944cccb60cd9dc85f5a7349854 \ - --hash=sha256:0a37165cc80b1a368384b383e63a4c38116a10467ae44c904d2d7468c4470ec2 \ - --hash=sha256:12b7524c88d4026d3dcb7b0ebe16b6714f3b4af402ddd0f0639ab064a00c87c3 \ - --hash=sha256:15641444eca4a29358107b3dceb74c1c6305c55c822fd199b458aaea4068a7fb \ - --hash=sha256:1b22c80559854b789a01fd89e8929b3798a156c0829b5282a8939f33ad4115ad \ - --hash=sha256:1e123f9b37edb8375fd74130d1f69c944bbf0a7b06761ae7211154b8759e94d2 \ - --hash=sha256:24c8e57504c8f45b237e40b99262d181071e5099a07053695b75d97bb53053a0 \ - --hash=sha256:2c2e2ae6867c2966b8daccc836d54a13218e0007e9a490aeb81dd05be64d22d7 \ - --hash=sha256:30e825f6848d9f18bba350ed6c75c1b02a0b5184474a31db9a32b1fa66fd8c79 \ - --hash=sha256:3768a5ff1b2125e6f552e561b6b2dca0e64982d8949689b4df145cf8b98d7821 \ - --hash=sha256:3ad74f8bb1a18963914c5452d289422830b39459e8776ebbcd207be1fbfb1d94 \ - --hash=sha256:410482da976329fe5f4067270401b12cf2bd552ff8020f054ecfaddb5475f9d6 \ - --hash=sha256:428bec0161b48d8cf583c068591bc0016d0d9cfff52462b72b3884861ea768c5 \ - --hash=sha256:506f48f2f9c29b143fca3dad7b0d518c188b6c9648c75a2ae6e2d9f2c13a060b \ - --hash=sha256:58ad1131c300d3c9b933802b3cc4dc69d380822935ba50b28703156ea826fbf7 \ - --hash=sha256:592b5fee597faa91cce2dd294dd7d9a1c83d76c4dbf877e33ec1adb866b2fbed \ - --hash=sha256:61233fe8951e5c85dff81c2458b6528624760166946b5b47ea150a589168411f \ - --hash=sha256:62481553b1793a27e9b9c3cf9e5bd483ef045ca72462592074b46d42b0c4d9b9 \ - --hash=sha256:6282caffb41ec326d4cb67ca9cf53b739d1b2f975a2acb498c7418e9f7d9a416 \ - --hash=sha256:69ef28e54fc85397f91b8c19592b8ef3d81952080366914823bd8572a2958120 \ - --hash=sha256:6f9a0c9c1cc15c112d1c053064fd032b64917062292c3d70aea280e02ae10b77 \ - --hash=sha256:6fa10a767143a5e82e8eaab53918af0cd8909a57a27f8cb2288b80a613ac671b \ - --hash=sha256:766bc7c9a9c340342f4c864ccbda8e78111e4751f13b895812b9c148fb79e9d0 \ - --hash=sha256:78e29211f26da2fdd0e9c6d2b79f489476140cf7029b6a64808ade7ca4156a42 \ - --hash=sha256:819edbdcb42ab8598b494bcf0222684bbb7a3c772bd1b1f0be7e029a6063c28e \ - --hash=sha256:85b10a45b8993d195c4f3ff57025b8d1e11834909ee475c403bfa60cb4caefaf \ - --hash=sha256:88268ca418cacea64cecb0d1d600d3c6b3a8038fcba02e1e205178c5b1f47661 \ - --hash=sha256:8b39472beafc0bdcafc4c8c73ad082ebfdb449d566897a61e7acb4fa88089115 \ - --hash=sha256:8ea1936c26b99999b27479853039a7f34713f56c49375ad52b38535ec93a796c \ - --hash=sha256:98a07f9bf591e3a8919797bee1c53f026ba4acd587e5a4404c8e57c9ec36b2a5 \ - --hash=sha256:a185a04039df6cae8648bc8ab6d6fde7bf94f7188ecf7828e76ac52eef1e41d6 \ - --hash=sha256:a35009284d0d3d5c2c9601c164a911b8b4331608d98a9a66d47d97bb2f522b70 \ - --hash=sha256:a3acb384427816dd5d470f47e62137b87f74da694faa8a50147012cf40df276a \ - --hash=sha256:aa2ba7d2ad6df4d80127cea65e5b8d5e2c3adbf153ff4804452836328aca7c54 \ - --hash=sha256:b10e1ff4756ed27d5a29d7fc79cfce7ef1ff56ad20025b89bac7cf79e09abbbe \ - --hash=sha256:b137f4bf3ada9dc44d411478decc6ff09a79ed30b306cd2abaa98408c3588137 \ - --hash=sha256:b259a04a737cb3496be0901328eb8b7552ed8df4865d8c8f1cf1bffcfc0776a3 \ - --hash=sha256:b427c19380991a4eaab2f6144b64b99b412043314c6bf4ab544f97bb31ee4190 \ - --hash=sha256:bb693b1e3d9a2f3fd228e2110daf4b5aeedb36761ca1e4282f74725f6d89f611 \ - --hash=sha256:c261d74b1a945cf895a9d6eccd1685a8e837531beaab782da4d630a8d12deffb \ - --hash=sha256:c5bf2dc311127d91230cc79b92188c082634a06cf66c5234db49a43b910183b0 \ - --hash=sha256:ca1cc11d82677b9662082e5478b7528e2b7db7beaa6bdff42bd62789d81be399 \ - --hash=sha256:d4b2dddfc219f54f956ccd53cf76a1d338ffe68fc7f2849ec9c7feb9927ff692 \ - --hash=sha256:d71d30f2d92f67d944631c523713934fee37292469e182ebcd2c1dd8a64ce53f \ - --hash=sha256:d865db4a6318e1c1bea83292e0ed231090538fc4ca45425b0f0480eb338bbc6e \ - --hash=sha256:e2aa72e3ce1770317ef534f63d397b55e130725f5149bd36077c3b539019db27 \ - --hash=sha256:e3657301562ac3cb8018d30d0d3ebfa39932239f7b5703422057ef14b69949f5 \ - --hash=sha256:e64dd101d380a115cc5a0c7856788adb535f1a4e21fc543775602f8be95180ae \ - --hash=sha256:e8ca6a1fcdb2943c9cbc1804a1baf3acb6071d72a471591678ded84218006e14 \ - --hash=sha256:edb59506291b647a30884b1d51a599d605f40b20af4a7dc3d33786a47a31de60 \ - --hash=sha256:f9a0ebbe45c29b5e5866593c12b78bd9035f0f0f0d4bc8361680cd580d99db49 +grpcio==1.84.0 \ + --hash=sha256:026d757df86c5b7a41de8200b9a2cda454aaa5004cb0c7e3374c66eb82f61499 \ + --hash=sha256:06619ba1515e5ee69fb2a514e95dd8be05ce74cb3928d5b34f87f87c86fe3c27 \ + --hash=sha256:08735e3d08d24ab3132cf87e2e5dea8746cabcc7d676c2b0b7362f195feef9d9 \ + --hash=sha256:0d532ade4486dad9b302ffa4d4683d67561051c26d17c4023322845e9fa10140 \ + --hash=sha256:158c1c11cfb61b4849c3caf4d52de6f5ecd376e14446feb4a90dc95a90d616f5 \ + --hash=sha256:15bb76489e337fc492685c9758e2fd4d4ab516b901ad830dc5a91987decf00be \ + --hash=sha256:19aaf172fc2edbefccce3f6e92c5150975dbe56c45744e9e87cf72ebdf85bfbe \ + --hash=sha256:209414080da8c20af94df1395b635da52dd57b5edc9e917e1deca0dc1c4bb55e \ + --hash=sha256:210e4c32f907045eb8158273e60c6ab69a3947697df6245dbda381f26c59485b \ + --hash=sha256:23e6e8e8a75cff88e0a793bfd3becea03a13e2763ae90c1ff573bc19ca5b429a \ + --hash=sha256:27b8b36200a9fbee6e120246f4a8a41657549107ef19fb2c819c4b2fd524f39a \ + --hash=sha256:28d2609691da93051e998495108bbddd2a9f7a561253bae94828d81290f30c15 \ + --hash=sha256:2c024da73b296f040b8360e60bd73a659b230093684a438da0e1260f34cc724e \ + --hash=sha256:393d8a78bff6731ecc5ad2151a821f8fbc1709b137ebb9c25a4ef399fbdcc914 \ + --hash=sha256:3d6a82c4fc6c85f2fb7572c86bdb86f84c97b6580e5f6599f711800bac48a5d8 \ + --hash=sha256:3de427b05f244ba2c2a9bdc67e7a6731c8340811524ecc4435466549f8af1d17 \ + --hash=sha256:406583b4e8fb2282ebd392e12b963e601c1f82e07125a8c2cb5b144e7e024796 \ + --hash=sha256:4119efa6519871719ad81f33bc95ab87857dcb1c5801f30a6e592f2c41164169 \ + --hash=sha256:42959bd50dd660ffc3f2a9bec15a6da4f9aaa0dda555d59ff2d2e80b908456a8 \ + --hash=sha256:455ed6083353b8e938f1d58c765eab2fbb165731e5b507be30fee344915a2a11 \ + --hash=sha256:465eef3d17e59ad22a556fc0138f7c7c799df426734344daec42c797d49fda99 \ + --hash=sha256:47ecf0d9b81d981f07b61bd89eced9d2582f5eaacc3aaa36ad27f81aef70a27f \ + --hash=sha256:49717e857899f4136d7657bf5aded61ac479110a075438290923a4d86af7cd02 \ + --hash=sha256:4aaeceeb7fa7d824c322d1ec3208c8495c88478a927295553235435fc49043ad \ + --hash=sha256:57dc36a5ab0e676f5f6e171de2917fd0aef73f32a9aaf23956bfe19997a30bd1 \ + --hash=sha256:5933a052946873d01a42119a05420d669bdca436aeba2d1851988ccb12b421c0 \ + --hash=sha256:5deda5b4bf62769eb98c119cca43d40e1231e34846b19db5cdea821d446a2253 \ + --hash=sha256:61386101ecaa096b694d0dd278caf99a56aeec78440cc17e918eef0b50f2d567 \ + --hash=sha256:659728f20fc7a0933ed7b1945435e31014b97ab8a5a7edcbaa70da4794aeb191 \ + --hash=sha256:70bb4ce8be0c5606bec259cbd7152374470396413b7863a658a08c849e6b29ff \ + --hash=sha256:71fd60e6e426d293d0a2f685115ad0a0845117602cf13605a4be7524fb5f7bba \ + --hash=sha256:756ea5c2da00fa65c930284892d2a9706828704ca3ba40b4c51c4834eb39fcfd \ + --hash=sha256:800b7e00d92553313c0463c200087930aa78678ec1d528193aeb50906f55989b \ + --hash=sha256:82da34ae4f639c73ac46e521e00c0a49bf86f717b9fb1f405f133e98731e38dc \ + --hash=sha256:8e1a45d174b6b8589f51dce1cea804aa6c1f72c9c80cba91ae2caabeb6d90540 \ + --hash=sha256:8e3f508d0e9e6236ba2f08d56e33355e434e785e813149a1b8477d3edf69779d \ + --hash=sha256:986e9751d416d7a6eaa2fecdac38da63153d63a4b340ba7d624889c490451500 \ + --hash=sha256:9b73836ba0e16fcbb57c31cf6cbc2907c8d8c790b83679df454b74bd15e0be04 \ + --hash=sha256:9bab4cf571653a8afffb83ce21aa27b51dfe629b526b7b6adec35491fe1fc2ea \ + --hash=sha256:a71d24f40b0cc6798feaa978c7411dc1135b7018e9fc0442db611c139bf58344 \ + --hash=sha256:a9383401d9f116f98cacd4eba6c505a6edb80ba65badfc8e8ed8ae64983bcc44 \ + --hash=sha256:b44f0a0fc7bc6677d38cc80bca1a32814ce6c8f200fb8b3c1a61c9d77eaefbf3 \ + --hash=sha256:b5c6f20d657ae09ae4e30d9d3a21edd13f1219d58cc6f999b9d1bb63be9c1baa \ + --hash=sha256:b61692f0069b3eee2fc8a3a1b7f6c044df9e03fede6ce69b3ca832e1c39f26c5 \ + --hash=sha256:b8c62888c3e49debf37ad9773e3c02f77b0c1e811f8fb0962f2b6c3bbab5b97a \ + --hash=sha256:bd8ea8eb3817b226057cc1c0e7ec4b378dcda52043b972b6ff12b1152178967d \ + --hash=sha256:c5559b492007dc09b4de9b95dab05f0b5e53547aad230cf07e46c7dd017a3be5 \ + --hash=sha256:d0fdd25faece8a1f95e8a3a8006e29701b5cf8dadb4a8132e68f3134637004a5 \ + --hash=sha256:e094dd21f077af8194923fc263cad872eaa1802bb0156fd7e5ae18e99cd86715 \ + --hash=sha256:e41c3993eee896c617dbd8a505085d28b6e84a0445ed9a1f40f95808473cf678 \ + --hash=sha256:e88d304f094f4937bc27ec6a435e218a084168f11ec630c8d5d39b431d08d81d \ + --hash=sha256:e90e3bdf7b5eac005fef631adae9cafde16f922def207b80a7c46b253c18ad20 \ + --hash=sha256:ed2c1493c44d0932f1e55fdb5d1ead658c68288ec5d51b8c4928422d98633ef9 \ + --hash=sha256:edb6f87fc60ff438557291501b3e16c7a77c3b01a52d782cf276dccc7c5dd89c \ + --hash=sha256:efb29f8633bf6630dc89de4fe0353ac3d7e4b70ef7b6e29fb40f00e68c127fa5 \ + --hash=sha256:f6c972474ce691aca74e58d17625450cef153dc4760364cadeb167983ea6d589 \ + --hash=sha256:f6d178ba6dc8e82976c184b65fddde172d054c17237993a3e083efe4f134d55b \ + --hash=sha256:f9a456bdbed52a01c9ab8423bdebab04a5363c78676edc55ab9b58bd13bdf9e1 \ + --hash=sha256:fbdbcd06986ede3ce584083b1dc2afe6808e8943e5cf50ad11183c03aceda25a \ + --hash=sha256:fc66cb50c93554b86db0b6625ab5c6e9051dbf8847c08d93c84918e02e413fb7 \ + --hash=sha256:fff5ef3fe1bba7d6147e5f19e01e5e122ac2c076486887ddcb8d42e663400fbe # via # google-api-core # google-cloud-storage-control # googleapis-common-protos # grpc-google-iam-v1 # grpcio-status -grpcio-status==1.81.1 \ - --hash=sha256:08072fa9995f4a95c647fc6f4f85e2411573d00087bcabdf30f260114338f232 \ - --hash=sha256:9389a03e746017b10f0630c064289201458f3ce01f5d7ef4b0bebc1ef6cf82ad +grpcio-status==1.84.0 \ + --hash=sha256:0c182ca0d6e60acbfd0e14499cf39a155e4827a1c3fd9f7638e49af15a74c30a \ + --hash=sha256:5caf28ba7184b81f618b5f7f094859fd2541bf429d2189bbbcd715c9c2cdcee2 # via google-api-core gviz-api==1.10.0 \ --hash=sha256:846692dd8cc73224fc31b18e41589bd934e1cc05090c6576af4b4b26c2e71b90 \ --hash=sha256:a05055fed8c279f34f4b496eace7648c7fe9c1b06851e8a36e748541f1adbb05 # via xprof-nightly -idna==3.18 \ - --hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \ - --hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848 +idna==3.19 \ + --hash=sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15 \ + --hash=sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4 # via # requests # yarl -jaraco-functools==4.5.0 \ - --hash=sha256:3bb5665ea4a020cf78a7040e89154c77edadb3ca74f366479669c5999aa70b03 \ - --hash=sha256:79ce39246eddbde4b3a03b77ea5f0f7878dc669b166a66cf3fa8e266aa3fa2f4 +jaraco-functools==4.6.0 \ + --hash=sha256:880c577ec9720b3a052d5bc611fb9f2269b3d87902ef42440df443b88e443280 \ + --hash=sha256:99e3dc0060c5cbe8fcd1cdb36258e2a65ca40f1566b2033b12abb1bb44dd3c30 # via cheroot markupsafe==3.0.3 \ --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f \ @@ -807,153 +896,178 @@ more-itertools==11.1.0 \ # via # cheroot # jaraco-functools -multidict==6.7.1 \ - --hash=sha256:026d264228bcd637d4e060844e39cdc60f86c479e463d49075dedc21b18fbbe0 \ - --hash=sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9 \ - --hash=sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581 \ - --hash=sha256:067343c68cd6612d375710f895337b3a98a033c94f14b9a99eff902f205424e2 \ - --hash=sha256:08ccb2a6dc72009093ebe7f3f073e5ec5964cba9a706fa94b1a1484039b87941 \ - --hash=sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3 \ - --hash=sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43 \ - --hash=sha256:0d17522c37d03e85c8098ec8431636309b2682cf12e58f4dbc76121fb50e4962 \ - --hash=sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1 \ - --hash=sha256:0e697826df7eb63418ee190fd06ce9f1803593bb4b9517d08c60d9b9a7f69d8f \ - --hash=sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c \ - --hash=sha256:121a34e5bfa410cdf2c8c49716de160de3b1dbcd86b49656f5681e4543bcd1a8 \ - --hash=sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa \ - --hash=sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6 \ - --hash=sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c \ - --hash=sha256:17207077e29342fdc2c9a82e4b306f1127bf1ea91f8b71e02d4798a70bb99991 \ - --hash=sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262 \ - --hash=sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd \ - --hash=sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d \ - --hash=sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d \ - --hash=sha256:1fa6609d0364f4f6f58351b4659a1f3e0e898ba2a8c5cac04cb2c7bc556b0bc5 \ - --hash=sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3 \ - --hash=sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601 \ - --hash=sha256:24c0cf81544ca5e17cfcb6e482e7a82cd475925242b308b890c9452a074d4505 \ - --hash=sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0 \ - --hash=sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292 \ - --hash=sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed \ - --hash=sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362 \ - --hash=sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511 \ - --hash=sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23 \ - --hash=sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2 \ - --hash=sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb \ - --hash=sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e \ - --hash=sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582 \ - --hash=sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0 \ - --hash=sha256:3943debf0fbb57bdde5901695c11094a9a36723e5c03875f87718ee15ca2f4d2 \ - --hash=sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e \ - --hash=sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d \ - --hash=sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65 \ - --hash=sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a \ - --hash=sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd \ - --hash=sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d \ - --hash=sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108 \ - --hash=sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177 \ - --hash=sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144 \ - --hash=sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5 \ - --hash=sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd \ - --hash=sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5 \ - --hash=sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060 \ - --hash=sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37 \ - --hash=sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56 \ - --hash=sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df \ - --hash=sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963 \ - --hash=sha256:5884a04f4ff56c6120f6ccf703bdeb8b5079d808ba604d4d53aec0d55dc33568 \ - --hash=sha256:59bc83d3f66b41dac1e7460aac1d196edc70c9ba3094965c467715a70ecb46db \ - --hash=sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118 \ - --hash=sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84 \ - --hash=sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f \ - --hash=sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889 \ - --hash=sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71 \ - --hash=sha256:65573858d27cdeaca41893185677dc82395159aa28875a8867af66532d413a8f \ - --hash=sha256:6704fa2b7453b2fb121740555fa1ee20cd98c4d011120caf4d2b8d4e7c76eec0 \ - --hash=sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7 \ - --hash=sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048 \ - --hash=sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8 \ - --hash=sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49 \ - --hash=sha256:6f77ce314a29263e67adadc7e7c1bc699fcb3a305059ab973d038f87caa42ed0 \ - --hash=sha256:749aa54f578f2e5f439538706a475aa844bfa8ef75854b1401e6e528e4937cf9 \ - --hash=sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59 \ - --hash=sha256:7dfb78d966b2c906ae1d28ccf6e6712a3cd04407ee5088cd276fe8cb42186190 \ - --hash=sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709 \ - --hash=sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d \ - --hash=sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c \ - --hash=sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e \ - --hash=sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2 \ - --hash=sha256:8affcf1c98b82bc901702eb73b6947a1bfa170823c153fe8a47b5f5f02e48e40 \ - --hash=sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3 \ - --hash=sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee \ - --hash=sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609 \ - --hash=sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c \ - --hash=sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445 \ - --hash=sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1 \ - --hash=sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a \ - --hash=sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5 \ - --hash=sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31 \ - --hash=sha256:974e72a2474600827abaeda71af0c53d9ebbc3c2eb7da37b37d7829ae31232d8 \ - --hash=sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33 \ - --hash=sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7 \ - --hash=sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca \ - --hash=sha256:98c5787b0a0d9a41d9311eae44c3b76e6753def8d8870ab501320efe75a6a5f8 \ - --hash=sha256:9b0d9b91d1aa44db9c1f1ecd0d9d2ae610b2f4f856448664e01a3b35899f3f92 \ - --hash=sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733 \ - --hash=sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429 \ - --hash=sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9 \ - --hash=sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4 \ - --hash=sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6 \ - --hash=sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2 \ - --hash=sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172 \ - --hash=sha256:a9fc4caa29e2e6ae408d1c450ac8bf19892c5fca83ee634ecd88a53332c59981 \ - --hash=sha256:aa23b001d968faef416ff70dc0f1ab045517b9b42a90edd3e9bcdb06479e31d5 \ - --hash=sha256:ac1c665bad8b5d762f5f85ebe4d94130c26965f11de70c708c75671297c776de \ - --hash=sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52 \ - --hash=sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7 \ - --hash=sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c \ - --hash=sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2 \ - --hash=sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6 \ - --hash=sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf \ - --hash=sha256:bb08271280173720e9fea9ede98e5231defcbad90f1624bea26f32ec8a956e2f \ - --hash=sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b \ - --hash=sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961 \ - --hash=sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a \ - --hash=sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3 \ - --hash=sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b \ - --hash=sha256:c524c6fb8fc342793708ab111c4dbc90ff9abd568de220432500e47e990c0358 \ - --hash=sha256:c5f0c21549ab432b57dcc82130f388d84ad8179824cc3f223d5e7cfbfd4143f6 \ - --hash=sha256:c6b3228e1d80af737b72925ce5fb4daf5a335e49cd7ab77ed7b9fdfbf58c526e \ - --hash=sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1 \ - --hash=sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c \ - --hash=sha256:c93c3db7ea657dd4637d57e74ab73de31bccefe144d3d4ce370052035bc85fb5 \ - --hash=sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53 \ - --hash=sha256:cdea2e7b2456cfb6694fb113066fd0ec7ea4d67e3a35e1f4cbeea0b448bf5872 \ - --hash=sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e \ - --hash=sha256:cf37cbe5ced48d417ba045aca1b21bafca67489452debcde94778a576666a1df \ - --hash=sha256:d4f49cb5661344764e4c7c7973e92a47a59b8fc19b6523649ec9dc4960e58a03 \ - --hash=sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8 \ - --hash=sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a \ - --hash=sha256:d82dd730a95e6643802f4454b8fdecdf08667881a9c5670db85bc5a56693f122 \ - --hash=sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a \ - --hash=sha256:dd96c01a9dcd4889dcfcf9eb5544ca0c77603f239e3ffab0524ec17aea9a93ee \ - --hash=sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32 \ - --hash=sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3 \ - --hash=sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489 \ - --hash=sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23 \ - --hash=sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34 \ - --hash=sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75 \ - --hash=sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8 \ - --hash=sha256:eb351f72c26dc9abe338ca7294661aa22969ad8ffe7ef7d5541d19f368dc854a \ - --hash=sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d \ - --hash=sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855 \ - --hash=sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b \ - --hash=sha256:f537b55778cd3cbee430abe3131255d3a78202e0f9ea7ffc6ada893a4bcaeea4 \ - --hash=sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4 \ - --hash=sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d \ - --hash=sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0 \ - --hash=sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba \ - --hash=sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19 +multidict==6.8.0 \ + --hash=sha256:003a3bddb32915c3f67096ea41d24e53edf710edb65a1f5d0c70ab40b0e4d20b \ + --hash=sha256:00be37bde741bf60871082cd347a093218c44886e99231b7516671c70f2c280d \ + --hash=sha256:029897732a9c798737457e382bf84e8c64237eff224a90aea2639f4413c45e4e \ + --hash=sha256:05c2e90c5289c5f7436ba2c25812a5fbdaa1c1bc11c8d8d3bbf64f5cd7c633dd \ + --hash=sha256:071da134651b04a8507dfb331ac0988f376337c2aea59486bf20989fb5b5a64e \ + --hash=sha256:088b04a66b3c1fce6fe4d771ec184a0426262d0b86709c908477b4ac7965df40 \ + --hash=sha256:093167d22a8c95af30f597b8a5686f20a14512989942d4be804d119899caca20 \ + --hash=sha256:0935971bffd0b479fc90c4811ca787703e93fcb6afea939a375dfc80285ab368 \ + --hash=sha256:095f62ea4e7a3be2f6c567ab695ce10e950f2adb905c1bec82281593e0b2d2ad \ + --hash=sha256:0b143d53590e89f43153d81d505a8448d4d57354354385aef8a51d67ffefa27e \ + --hash=sha256:0c1c4debad7337627b86837abdf0237ca3cb3d7e17de7eab0177c263878546d4 \ + --hash=sha256:0eca15d627e942ce186a935061f1568cc46c02e97c419c8da802df2be9f917d8 \ + --hash=sha256:0ef606c15cac6c90279acf34120784b6f36662cbf382defd3955cd8f1115336b \ + --hash=sha256:10456943903744ae1249728161c96bd9d2f7eb5ee17fcc2ffda2dc32e1bb36c7 \ + --hash=sha256:11d71490bf4bbff1141b14b93af419ad68c56b60bea9277fcb3f94dcca4796eb \ + --hash=sha256:122adc7c46ac1e31ecfc7f81b2530533dccafdba70f5d741649f87e336c63384 \ + --hash=sha256:13967dca8b2f33230a1427b52438326bb1c9101a1df22a3309ed3fcbbb3c96f0 \ + --hash=sha256:13e26f59f0eecfc5f67c663ad550ffdaf62c0f657547cde387f6c86af1c9449e \ + --hash=sha256:15db8e6cab5f4cc9241bc56e69fdf3452cf49c10ee3c7977c742e68a275b3786 \ + --hash=sha256:18f0e06360c3e451a3ab800355773c8d125a758238d780c800b0ee5e90ee903c \ + --hash=sha256:1969971900b0871530f9b62280dcc2d75688e74d2a69262bc01faf2b96c78f04 \ + --hash=sha256:1b8986d4313dcee7c932837d16a535f1840b827bac1ea7c5c4c80751d0423794 \ + --hash=sha256:1bdb9b8fba5a9aef673ec90db3f55b1ce743f2fbdea4d37dc04d14ccdfc153ff \ + --hash=sha256:1f57c414be82490bc0e0305fdb834186229b2d9b6a35fa0afd1eb1a772d125ab \ + --hash=sha256:1f66fe6a021173d0d47968491791966b9f3e6d61115f2491744aa0c07a6e67af \ + --hash=sha256:202436df907c15adbb94360296c425ea53cf8968a5d2cff9b5b9790ae1972b33 \ + --hash=sha256:2196ba6df392c3574acadd14ef87550f3611349c8618564de324b806a7a31cee \ + --hash=sha256:22a310ad37672a261e55a8b5e28d0ae08cfb68abb1f46418ccd19835c3b8e836 \ + --hash=sha256:23c9ee89967b6a9b4048acb3b93b660ed714ce9c8bf3bbe652959bc120dc02dc \ + --hash=sha256:2622fe114c0bd66ca5c461859357587f5a5e35ee5ff49fc5643d1bc78dbb41c6 \ + --hash=sha256:26a7aafc992e78872e2c8c1f7248c0e01139cf9020a7781b0c064fa566832712 \ + --hash=sha256:27747162712e85c84598d364425dbf1714ff335bdb6ba3171c4e5081196e8916 \ + --hash=sha256:29631224698de1e42abc8fa7658d830e0aed0029785144b5832b695da5adef2f \ + --hash=sha256:29b6e7bc4442a56cf8e0dc1cabf3fdc77cd533568d6829fc76a1effd2ce332ec \ + --hash=sha256:29be9fd289e9ab8f480996ea2f686e1654b80242033843cb11691688329423f1 \ + --hash=sha256:2ba9933e8f35fe4a70f540b837254c4055da82dc3a9e500a8f95e61498083a15 \ + --hash=sha256:2cc66abb85e2108c9ff8a1c0d20fa260bf690bbb33caef4ff3ecb2c2cbdfff5d \ + --hash=sha256:2cd560498ae8e1bcc955643c1d78eb8e338226d07a983c656ea8c4443d3eec0f \ + --hash=sha256:2f79cc3e8039a8cf5c77e0811b0807953fd52d0863b9b76970b20d696dc64a78 \ + --hash=sha256:2f8a4b0b4d639d525928c7f30de527bfdf9ead6e44a5e8cb9c50aced5e4590cb \ + --hash=sha256:307c1acd812fe897e7fbe10c6758822e8c04be4e7c60a9f54901cdf8b5ab8bc3 \ + --hash=sha256:3126f2a96704505aa4e92a72d6e8a5d7f29d40a987ced8bf69e29d71dfc71fbc \ + --hash=sha256:31e8901637e20ccb3cf8f8848b5d0f7a00462bf5b34f7cf3dcbb2753b18e8b39 \ + --hash=sha256:346ac52e56bcda320c0dcdfdd081947ed7cada33afea4e2284bef7b0733bff9b \ + --hash=sha256:348bb85e2038b40c007383616d73f734869063772372519549ebd7da1723d1a4 \ + --hash=sha256:3533a03e4e789baf6a286e7b0b1b6da3f3d7c3eab569686ee29ee1d8b52e2cb4 \ + --hash=sha256:35977263d9bf506dbc65349f63b3b8c91606d4abc110990945e3b94bc671319c \ + --hash=sha256:397599503b718f0137f26d3f6532d6955069cd2e5917c47ef581495bc2529ff8 \ + --hash=sha256:3bafff8598f0528017ddc74194e5451d5c22d046c98935f8f86247b0f286e4f8 \ + --hash=sha256:3d1f48582686a0a3b81e9b43234766cc96697df72081af3f48107bd3f34d34e5 \ + --hash=sha256:4261863fc8b5ab1b815ede94e592e94c6af5b04616014929057e61859e7382a9 \ + --hash=sha256:43a4b56555bbcf8af161e7c7682bd93eec10f068c95844511864c018c8e5e13b \ + --hash=sha256:45cc39ba50fb0754a4359b90f8229ae08598fe2266abe3521b4e5a9ba916534a \ + --hash=sha256:46029e6e27a3ec0dc55b53f58df82d10f04c5e111f78248279b530bedad2c30a \ + --hash=sha256:48ea524a25a1cd5972cf293bc95713918cba0bcd6fa9b992d906c857c546abe2 \ + --hash=sha256:4ee953a5ebaeed38dc21cc032ed17a9d9782802e00042200497ab4b01b0bf7c0 \ + --hash=sha256:54af1266710cb0f305127ae0b970aff8d208057f8a29cd6e1db99b0114947035 \ + --hash=sha256:560b211fc3bd4a1e1c6de44f6d38113bf5b410dfc89a4c0d2a3c0edbf1a0dfb8 \ + --hash=sha256:563661919f603374c40cf45ffcd25535c12b8954203569a2ab1cee5265871cf4 \ + --hash=sha256:563d6500ca80dac7bba6f48a78e0ffd87e21a7d4d24642c6503a2ddccd70c110 \ + --hash=sha256:59e539c4eb4d3a53b0e630a6ba2b2f2824732b5e73f90e30a280f12fde157b15 \ + --hash=sha256:5bbbb696c8024475b1877d14ce20d5f1cc05b8f6d786cea0fe3aa7fedc02e891 \ + --hash=sha256:5caf684986a2490628f059a99dd107b566a2d34cf947f8eb8387e0500a1f90c5 \ + --hash=sha256:5cd4637ce76312ba1e05eb9c5193fec231f64fee0944e135fa1e951242355b37 \ + --hash=sha256:610c7637bc36b90f39e6c66f710f93d57018f83d53e1e187caaa218c6892b95f \ + --hash=sha256:628ff11e6720f90acd0c305dfa3339f04a783a20de8cda6ac333ba46447261e8 \ + --hash=sha256:62b8e291a4f7edbf7cde7a43d831d893ba443a1b627498b53581943b0e348feb \ + --hash=sha256:6300d5176647145ba1e22991c924fb29743e54b4d7b8bc85a0d3ec0e55e189cb \ + --hash=sha256:64eaeda36ee8d88f9e8616a587a8c66a663283cf6e0dcf013c1ddd8c758e4aef \ + --hash=sha256:658f5a1895b804423d97b22d06fc0d0b171c7c01dcc3aa9c8faf0c0e26a249a5 \ + --hash=sha256:65c85c79f5a2c04fbbc18f006c014674dc5fdf270cb978d8862c82c6f694e60c \ + --hash=sha256:68186a2d4051c8ffd17be33553bea2ec9bbc8ef860fe2980a221d96126296f31 \ + --hash=sha256:68d40b2bace413f3231f5729d3fcfb1837fd31c4907e241b5d43211bfd76f3c2 \ + --hash=sha256:69708fecaa88bcb2341397b49fc95057a835b02a3670c551b37f95dd79e64e3a \ + --hash=sha256:69b3e519a132bb943b0daae15fc8c2168706b17f826481d32a32a5e784b129e3 \ + --hash=sha256:6b62b7e0025aa48dec11e125e655d1157985a5fdcec04b1ad500101ad072b891 \ + --hash=sha256:714597cb5d5e15a8a449d2ae23c45b486a9e8fa33c462c7a33d7f35b65d92943 \ + --hash=sha256:758233648ac47b07c575224c4eadd73c8929c3b4c31e2afcfea935fde1cda735 \ + --hash=sha256:75daa15ca16d6285eb2e104b2f05ee6f8d9836c68da3ce5c85f615a0450eed0e \ + --hash=sha256:77745725125d01fd613b6db043362aa7c6bfbfdb23d45dbfc3d92bf58160af62 \ + --hash=sha256:7941ef106ca1f2c62314a13c7ed913bcf49641f3efdc12864d588e17870920ac \ + --hash=sha256:7a2573d0fd34f361a4a14e54d8cda3a91ac4e55fbf0d719698024f3b09c5b147 \ + --hash=sha256:7a62e302fc8cd6aa8972207e7e951d1fdee7c1dda18568305041d19f0e2c00f5 \ + --hash=sha256:7bb0dad75068fee80fcb60f88569722c199d8656a16706702dc6e3b786819c90 \ + --hash=sha256:7bc7003991ebd368a20d05228137a37b3d3066751f3ea1e4f7b8efe8e752f2f5 \ + --hash=sha256:7d26dc8f070c0ec5579e987fa615ffd6883086106eefdff9e10d160fc5630630 \ + --hash=sha256:8125e60f3c70e323ac07dd8b3635f7b3bbc5c3a9ac04ae5988f668ff7ae28a18 \ + --hash=sha256:8180b635290a75af8478f1b3e9810135381ae24833293fe77b85c1c21ff842ab \ + --hash=sha256:82780eb8bf59e8fb25dd081fde6e058805045d6374a7f2f877effc826ca4434b \ + --hash=sha256:835d5a90b11d1f5f8200ff3cc8316bded76eebebc92436398947a27657e645e7 \ + --hash=sha256:83ff054b04915be5c15680da6c6012474a2cc2bf534129a0e8c6a99f17ba7238 \ + --hash=sha256:8457aff3c12a89a8e1c4674de5c777857fbc429f40fe117a3d29538547cbc364 \ + --hash=sha256:847d6082ae694dc95e548acb201bc100e1cfa96513bc71fdcb86f709dad6c435 \ + --hash=sha256:883284137e25318ed9735b742ae46341a864888fae28e8b6314c4f84da080f08 \ + --hash=sha256:887f9a975996032c686719eb7b3e1e7942fab5079c2b778bbd9afe9a9d78244f \ + --hash=sha256:8890c89d662560e51c55ac1304d6f919b23942abe9ae1127cb1de9aa6132fa52 \ + --hash=sha256:88a6df88567680504ae28bfa7a1f2f64243d91e79a40b2c92ef42efc531e23da \ + --hash=sha256:8d1046b5427dcafe6e8a0e07527dd74f1ee694006160162f53f3a17f15aad3b4 \ + --hash=sha256:8daafaa0b2eb43f76898ced78b1e0fb91b38c4fa50da516c18067f2a2d578c20 \ + --hash=sha256:8dc2d9c3a924ed14166e63650b2cf9f59e7821743bdd50b23802bd97ca09bde5 \ + --hash=sha256:90c10b22860dbd09982d0b8993b66231a861bea2993d4a817ff35273f6ea285a \ + --hash=sha256:91fa75d0a693832106d98f66c849f034f21c828d14437f1fb97d3784aab89e84 \ + --hash=sha256:930c6058047410e3edff445f5a6e4457f2e089042dede00e2d18ce06f3ceae2e \ + --hash=sha256:9442b14eec262a1f74369bbd07e75bc5155105164649a4b9fbc1ebc7b8fb0b14 \ + --hash=sha256:95c27b4f3f04320fc44e338573f40c5c956b504a7fcf081a157fd0b02579311c \ + --hash=sha256:9606f583e7acaf61e7b3f56074e14037b9af7cb194590edfc0114b3ae5931ff7 \ + --hash=sha256:962f18c59a000f30b084ea2e6b8001521bb315efd4e5f10acf9fb36f366b7882 \ + --hash=sha256:9caef53b20a105c0d66518a34be2f71b2783de8d091767575ef86f6ea422236d \ + --hash=sha256:9e37024b41d7a7e7e9cce14b248d54707c21c2a2ea30a47b71bdcefcafec00f2 \ + --hash=sha256:a5a7ee1217949ddd43c6b7bcf70d5c22193bb50e8c695386de5905325e93ce9f \ + --hash=sha256:a5e1583c14775580da05641240ce0d93f36ce3ddef3d5083a827468b0bcfe874 \ + --hash=sha256:a9e246f67ac038568b854ed7c5578e4c6af1f742359901a8fcc3603ff1358df6 \ + --hash=sha256:ab83fdd8cf307353edba9c427c17a3a021c2522d690f5633dd9f72d28b48ccca \ + --hash=sha256:ac746cb365bac1c462da9e3e6ab8904a8efe2217a56b0b2e3d9480f41d2b2602 \ + --hash=sha256:ad474c11d851b6fc97cb625e4822bc0cbd567fc07dc2602e28faec5a36b42bbb \ + --hash=sha256:b03ca066b47b18b205cc080dca6f76cbd159f8cdd33a02a0700164c13b37e463 \ + --hash=sha256:b1cd4d66ce894a45482e1ac2837c31d0bd447df35065e542b60055aa2d00404b \ + --hash=sha256:b25426f9f6ed402835617c8f23609a47045f91ecff365eb6734817e039a8ed25 \ + --hash=sha256:b367c342327717d644db4c0ddb37ceb655c84822215ea0773a3a36911b74b71d \ + --hash=sha256:b7e62b8fc7bd6cad007b9f2e0ad9c8d4854c06350d5f51e1a439dd18b510ecac \ + --hash=sha256:b8b7aa75146266fd3e2a2437cf69ae188688c04ab8665b163d4257b46c1e0c83 \ + --hash=sha256:bb36381e1f9f9d06eba2f10bdd438e5d20c07d5b55e1a3eee30b9f44cbf52316 \ + --hash=sha256:bb8c7da8c861391f7ae48e3593762be2dabe405109e01aec520fbe1a6d15d14b \ + --hash=sha256:bb9a60b7faa5d37c426fa91cf4d6738182a1f2755b9fab7c9c64cd466c4ce51e \ + --hash=sha256:be007d1aee2cbd530347dcafedb400891a3b5f1bd7135f95cf5d5b330b5219ee \ + --hash=sha256:be569fff1d85cd29391c431c5641c8772acb75bbdc61e60a8e82fceb9023d385 \ + --hash=sha256:bea7df027015856ba5d0a88e3b4777ff8cb5c66b58fc108050fe79d4dd9d4d2d \ + --hash=sha256:c0fe437a6d2f36aac2b49517057776575b5bf359df314cca20d230a6e139c089 \ + --hash=sha256:c2b2a96cf1dd99fe7867be4c013314225f4d5786e6685906e29932d42aca6f11 \ + --hash=sha256:c2c5fd0fd39574ccd58e1a52565b341aff522c5c836f1b3eb7605c371e61f52c \ + --hash=sha256:c46a08bf070d6849fed483e9d9833f9d06aecb8382ed985be0b38508b3ae958e \ + --hash=sha256:c5f3a2af441670d80ce5fdf13b6c1b421fc1fc7fc5182d58ac7486738bb2b742 \ + --hash=sha256:c60e50bc5b07faac92fd3a20fa21cc8cf3e3f7204d2867b206c73293ebc19101 \ + --hash=sha256:c68e0c0649d17c2d0339e3674e86a4aeba4a7e6b21c1e394cf947a95433b31d0 \ + --hash=sha256:c9c98d2f0126ba84cb45601eed97ff67ff767e19ae6eb3c31b02827b54d700e5 \ + --hash=sha256:ca52b9ec80851366197577154c862c4c4c7036ca76ae94cef5cb59c5cfeab944 \ + --hash=sha256:cbd86f9787c5e2f5fd27d8b21458222f107347c6731c4e93dde68f554b466a2d \ + --hash=sha256:d0264f8d5cb0a803f650a6a8572dfa0cd1e099a2234c588dc8fb220b415b865f \ + --hash=sha256:d0be2b832435001bc623ca7f1499ca1a853d4f082fb61221a80ce71132f50b26 \ + --hash=sha256:d244cf6b52b5ba1c34c3832f4652a668ebb36d95949b96eed9a1c54d916a90dd \ + --hash=sha256:d2d236b8a44ae91536a12ebcb996bdb31cf27425f36b4d05c87f2ba2716050ba \ + --hash=sha256:d3da668e903c934ed0b587ecacfed6901f6ae6384a6e975887592b61845e78bc \ + --hash=sha256:d6dc7804c50fabd28644d4d18a4b20aad3681b3e64f3acd3182b330ca73f7a32 \ + --hash=sha256:d7e5ba0a0153e35fbce9c51df530c8b4cb0c3012b46a04ff9a048441a269c2ed \ + --hash=sha256:d8a5ac357ac283490a8d1899b0383355fd1f8634b14ba0d59e4c0dd97db85556 \ + --hash=sha256:da1c112c5784ccd9d32cd90be6739fee32644e874eff6ae8f0497cba3e352e58 \ + --hash=sha256:dc911ae6152e455b16a2a1a626aa6cd612fa01efb9d0a4ab3f5cf328b911483d \ + --hash=sha256:e0db3a4d1e264e225037a6023888972c25206a96e016021a5bea41c9a939f2a9 \ + --hash=sha256:e192018b732f7b168e6604cbdf40fa8e05c996693b9eb445a0d8a73f4b77c5d3 \ + --hash=sha256:e37b744849fb631bb52e3dadde35ffeee365a6c41cf71257b5b7acc9cd83fd38 \ + --hash=sha256:e41226ecf607f062fe34a2f4cf64ad3a89e3a0180dc800b463b6b14c06dd10dc \ + --hash=sha256:e418ec99574ca24365ca96546af285c2b021a1a072478a79f0e3cc3b08837154 \ + --hash=sha256:e6ec7d37841609a691b96a10b4fde386c7cd93ebbb939f59c9f23325ee788395 \ + --hash=sha256:e886ef8c9879105fe4fc99417447b3a5f35d1131412ce839470bd2089fe2043f \ + --hash=sha256:e8e1e895e23818d343e4ae7dd95a0a556fdeaf8b471acf1c0a39b93c6f54d478 \ + --hash=sha256:e9dc7b4ff6ef184504b49ef9a4113d49a646653b2ce89f5f48c1f57cdf6ba081 \ + --hash=sha256:ea880d441be7c510106bc56064be39266d948aef94ad4955e8784690019a5d9f \ + --hash=sha256:eabb03dc3e4ed6333ecd1cc9826ec80e7a98b5506deeb832d7260c8e44166d23 \ + --hash=sha256:ec0a4d066356054d569a66e0a94691a2058b680be5e710298f61db11a3c4609f \ + --hash=sha256:edda19aff836ec515caafc09ea53d2ab144a041f09ee9a7cefcbd3ae4e976256 \ + --hash=sha256:f1f4a220db6ed7c8fd16b6d644ffd1f082651693204daf3275e049fadc849e39 \ + --hash=sha256:f25b61a708bd276e8cbb6afcbbf1b8e793a3be70ba0a842d0b8692020f83b706 \ + --hash=sha256:f2fa3d3b1c933d4bcb8fd2018700d5e7235c52f2ab8c88d22286965c5c0f00f8 \ + --hash=sha256:f3071e6515cc63714d014da8f738ae9fa3997c476203f3cd46de380c2376ed7b \ + --hash=sha256:f3a0a31189acf6703307397c6139ddabd734c20c5ef92649fc93e473df6615a3 \ + --hash=sha256:f7eefd0233a7c33ca980a5cfef26f1e9b5e2137839e752a99963696729f12d91 \ + --hash=sha256:f8b09b25e0f4dc2ea9e2adbb1cc3ba11a94d6fa3dd978ae659c8743052e1afbc \ + --hash=sha256:f8d7b66c9e09c0bb0add2b5895e646b62a0849e71155066f215523de6b95cbe6 \ + --hash=sha256:fa6c2880709c84457de104385b704fc28860f27e442ad13966fc4af8e714fe9c \ + --hash=sha256:fc5460940f50dff00731b4132366840ba9685286ea88ea104b661899084f3fea \ + --hash=sha256:fd789a294d8e098528be29b2669b83005ce569339f8cef167fc0274c3115c34c # via # aiohttp # yarl @@ -961,6 +1075,10 @@ oauthlib==3.3.1 \ --hash=sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9 \ --hash=sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1 # via requests-oauthlib +opentelemetry-api==1.44.0 \ + --hash=sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a \ + --hash=sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef + # via google-api-core propcache==0.5.2 \ --hash=sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427 \ --hash=sha256:04dc2390d9edbbaef7461f33322555976ffddf0b650a038649d026358714e6c5 \ @@ -1086,21 +1204,21 @@ propcache==0.5.2 \ # via # aiohttp # yarl -proto-plus==1.28.0 \ - --hash=sha256:38e5696342835b08fc116f30a25665b29531cda9d5d5643e9b81fc312385abd9 \ - --hash=sha256:a630604310899e73c59ec302e5765c058d412b2f090b9c79c8822589f14955b8 +proto-plus==1.28.4 \ + --hash=sha256:4b01341272f8a348db3f003b6143109f83ab43091019d5181b3fcdf500ab32aa \ + --hash=sha256:5ff7ecad828e032a491fcb86947801768e32237f99dd049b649965b892ae9a63 # via # google-api-core # google-cloud-storage-control -protobuf==7.35.1 \ - --hash=sha256:11d6b0ec246892d85215b0a13ca6e0233cf5284b68f0ac02646427f4ff88a799 \ - --hash=sha256:230a75ddfc2de4806e56696ce9640c1cdfdb6543b7cfce98d42a4c0a0e7bdb87 \ - --hash=sha256:24f857477359a85c0c235261b8ba905fd51b2562f4a64ca1df5473f29850cbf6 \ - --hash=sha256:353652e4efd0bca5b5fc2656abf8307ef351f0cf938c9eba09f0e09c20a25c30 \ - --hash=sha256:4bc97768d8fe4ad6743c8a19403e314511ed9f6d13205b687e52421c023ac1b9 \ - --hash=sha256:74758715c53d7158fb76caf4f0cfdacc5329a4b1bb994f865d6cf302d413a1c4 \ - --hash=sha256:b73f9489a4b8b1c9cb1f8ed951c736392592edb24b9d6819f36d2e10b171d5b4 \ - --hash=sha256:ce115a26fe0c39a2c29973d914d327e516a6455464489fe3cd1e51a1b354f81a +protobuf==7.36.1 \ + --hash=sha256:0b53ce95272aad50ad25d7ff03373743209822e8ba42ea7fad27d2bee1547d00 \ + --hash=sha256:39c518c05586c016d7874ff6079ee115bcec1ea5fbb1d177fbf7867ef4c67e44 \ + --hash=sha256:3cf2ee25d006cee57294a1196ea43b37feb78e0dcd1e8af5c1aeddb777655aca \ + --hash=sha256:43d3d37b1eb24c113b9b7d02008cac44e423f00b611b7781ae998d7623972969 \ + --hash=sha256:51139351435d9b43d88a55eaa49fb6f737fbb478fb0cbf2cf694d1a04a9d3363 \ + --hash=sha256:7d951e46b3f963d6c264c367c437921de9d5aedd9c3f9612b9077736b4e3ad5c \ + --hash=sha256:97198b77e369a0abd8e262b8f6c7266c55ddb796a3a12c76d7b8881188ed83aa \ + --hash=sha256:d0f6470f0ce2b84e3feaea2d4b816378b37ba4d4aa08a274305373de93e2d524 # via # google-api-core # google-cloud-storage-control @@ -1109,9 +1227,9 @@ protobuf==7.35.1 \ # grpcio-status # proto-plus # xprof-nightly -pyasn1==0.6.3 \ - --hash=sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf \ - --hash=sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde +pyasn1==0.6.4 \ + --hash=sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81 \ + --hash=sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b # via pyasn1-modules pyasn1-modules==0.4.2 \ --hash=sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a \ @@ -1133,9 +1251,9 @@ requests-oauthlib==2.0.0 \ --hash=sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36 \ --hash=sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9 # via google-auth-oauthlib -setuptools==83.0.0 \ - --hash=sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef \ - --hash=sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3 +setuptools==69.5.1 \ + --hash=sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987 \ + --hash=sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32 # via xprof-nightly six==1.17.0 \ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ @@ -1155,6 +1273,7 @@ typing-extensions==4.16.0 \ # aiosignal # etils # grpcio + # opentelemetry-api urllib3==2.7.0 \ --hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \ --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 @@ -1165,117 +1284,117 @@ werkzeug==3.1.8 \ --hash=sha256:63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50 \ --hash=sha256:9bad61a4268dac112f1c5cd4630a56ede601b6ed420300677a869083d70a4c44 # via xprof-nightly -xprof-nightly==2.23.3a20260707 \ - --hash=sha256:006db5a8c1d6f31b21d80456d098e5ac6156c90cb3d88910d5a60d71f993ae02 \ - --hash=sha256:4d531ac724f16f3bbe318220ccf2fed19d01fddfa1a49568334d9797f9df7b03 \ - --hash=sha256:aba882245c392cd997ced825b7a320d9cced99e9b289452fd65ddfde745ec44c \ - --hash=sha256:da0078ef75c6504a3733ab0f72d5f8846d781e512bbf7c36dabedb7433f2708d +xprof-nightly==2.24.2a20260915 \ + --hash=sha256:096d108cfad9b22842a57774fb1d386a0680d10f5e55ac421d81f3473c869866 \ + --hash=sha256:225031fb920efa291fb3b81a59452519ca39fafbb6853a187b13a6c56065156e \ + --hash=sha256:a37a653582f93d018d5ab0210415b8b090f6c5d3885f15db6426817bfde48a71 \ + --hash=sha256:b089f56f87f8524e40a400b266d826a670fef8f2fe2bfcd256bbf64ec773b779 # via -r tools/xprof/requirements.in -yarl==1.24.2 \ - --hash=sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b \ - --hash=sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30 \ - --hash=sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc \ - --hash=sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f \ - --hash=sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae \ - --hash=sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8 \ - --hash=sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75 \ - --hash=sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a \ - --hash=sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c \ - --hash=sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461 \ - --hash=sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44 \ - --hash=sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b \ - --hash=sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727 \ - --hash=sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9 \ - --hash=sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd \ - --hash=sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67 \ - --hash=sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420 \ - --hash=sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db \ - --hash=sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50 \ - --hash=sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b \ - --hash=sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50 \ - --hash=sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9 \ - --hash=sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1 \ - --hash=sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488 \ - --hash=sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2 \ - --hash=sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f \ - --hash=sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d \ - --hash=sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003 \ - --hash=sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536 \ - --hash=sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a \ - --hash=sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a \ - --hash=sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa \ - --hash=sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f \ - --hash=sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e \ - --hash=sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035 \ - --hash=sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12 \ - --hash=sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe \ - --hash=sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4 \ - --hash=sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294 \ - --hash=sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7 \ - --hash=sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761 \ - --hash=sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643 \ - --hash=sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413 \ - --hash=sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57 \ - --hash=sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36 \ - --hash=sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14 \ - --hash=sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd \ - --hash=sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5 \ - --hash=sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656 \ - --hash=sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad \ - --hash=sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c \ - --hash=sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0 \ - --hash=sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992 \ - --hash=sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342 \ - --hash=sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1 \ - --hash=sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf \ - --hash=sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024 \ - --hash=sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986 \ - --hash=sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb \ - --hash=sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d \ - --hash=sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543 \ - --hash=sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d \ - --hash=sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed \ - --hash=sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617 \ - --hash=sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996 \ - --hash=sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8 \ - --hash=sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2 \ - --hash=sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3 \ - --hash=sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535 \ - --hash=sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630 \ - --hash=sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215 \ - --hash=sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592 \ - --hash=sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf \ - --hash=sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b \ - --hash=sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac \ - --hash=sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0 \ - --hash=sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92 \ - --hash=sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122 \ - --hash=sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1 \ - --hash=sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8 \ - --hash=sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576 \ - --hash=sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8 \ - --hash=sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712 \ - --hash=sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1 \ - --hash=sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2 \ - --hash=sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b \ - --hash=sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a \ - --hash=sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53 \ - --hash=sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1 \ - --hash=sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d \ - --hash=sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208 \ - --hash=sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0 \ - --hash=sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c \ - --hash=sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607 \ - --hash=sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c \ - --hash=sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8 \ - --hash=sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39 \ - --hash=sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f \ - --hash=sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8 \ - --hash=sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90 \ - --hash=sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45 \ - --hash=sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2 \ - --hash=sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056 \ - --hash=sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14 +yarl==1.24.5 \ + --hash=sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36 \ + --hash=sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331 \ + --hash=sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871 \ + --hash=sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498 \ + --hash=sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780 \ + --hash=sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027 \ + --hash=sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224 \ + --hash=sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144 \ + --hash=sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76 \ + --hash=sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3 \ + --hash=sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb \ + --hash=sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740 \ + --hash=sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec \ + --hash=sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e \ + --hash=sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a \ + --hash=sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24 \ + --hash=sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950 \ + --hash=sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95 \ + --hash=sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb \ + --hash=sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928 \ + --hash=sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c \ + --hash=sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41 \ + --hash=sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e \ + --hash=sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550 \ + --hash=sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f \ + --hash=sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b \ + --hash=sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e \ + --hash=sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede \ + --hash=sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad \ + --hash=sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6 \ + --hash=sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104 \ + --hash=sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2 \ + --hash=sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba \ + --hash=sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9 \ + --hash=sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12 \ + --hash=sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1 \ + --hash=sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027 \ + --hash=sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385 \ + --hash=sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2 \ + --hash=sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840 \ + --hash=sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2 \ + --hash=sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4 \ + --hash=sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc \ + --hash=sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c \ + --hash=sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213 \ + --hash=sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba \ + --hash=sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448 \ + --hash=sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0 \ + --hash=sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6 \ + --hash=sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966 \ + --hash=sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750 \ + --hash=sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621 \ + --hash=sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13 \ + --hash=sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0 \ + --hash=sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58 \ + --hash=sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54 \ + --hash=sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075 \ + --hash=sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4 \ + --hash=sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440 \ + --hash=sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f \ + --hash=sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4 \ + --hash=sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c \ + --hash=sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f \ + --hash=sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d \ + --hash=sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7 \ + --hash=sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9 \ + --hash=sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723 \ + --hash=sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047 \ + --hash=sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da \ + --hash=sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293 \ + --hash=sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b \ + --hash=sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61 \ + --hash=sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca \ + --hash=sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed \ + --hash=sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a \ + --hash=sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a \ + --hash=sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688 \ + --hash=sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16 \ + --hash=sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d \ + --hash=sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077 \ + --hash=sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce \ + --hash=sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd \ + --hash=sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d \ + --hash=sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88 \ + --hash=sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5 \ + --hash=sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75 \ + --hash=sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1 \ + --hash=sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f \ + --hash=sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b \ + --hash=sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff \ + --hash=sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9 \ + --hash=sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f \ + --hash=sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0 \ + --hash=sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5 \ + --hash=sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25 \ + --hash=sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6 \ + --hash=sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00 \ + --hash=sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373 \ + --hash=sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd \ + --hash=sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1 \ + --hash=sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2 \ + --hash=sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d \ + --hash=sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba \ + --hash=sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104 # via aiohttp zipp==4.1.0 \ --hash=sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f \ diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index fd63b94e..99052d1d 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -151,6 +151,7 @@ zig_library( "//bazel", "//ffi:zig", "//kernels/cuda_tile:cuda_tile_builder", + "//kernels/cute:cute_builder", "//kernels/fly:fly_builder", "//kernels/mosaic_tpu:mosaic_tpu_builder", "//kernels/triton:triton_builder", diff --git a/zml/kernel.zig b/zml/kernel.zig index 6535b318..692dd3da 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -4,6 +4,7 @@ const stdx = @import("stdx"); const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); const cuda_tile_builder = @import("kernels/cuda_tile/builder"); +const cute_builder = @import("kernels/cute/builder"); const fly_builder = @import("kernels/fly/builder"); const mosaic_tpu_builder = @import("kernels/mosaic_tpu/builder"); const tpu_dialect = @import("mlir/dialects/mosaic_tpu"); @@ -510,6 +511,114 @@ pub const cuda_tile = struct { } }; +pub const cute = struct { + pub const Builder = cute_builder.Builder; + pub const Value = cute_builder.Value; + pub const DType = cute_builder.DType; + pub const FinishError = cute_builder.FinishError; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(&cute_builder.dialects_needed); + } + + pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + else => std.debug.panic("zml.kernel.cute.from: dtype {s} has no CuTe equivalent", .{@tagName(dt)}), + }; + } + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + /// `kernel.launch(grid=, block=)` in the Python DSL; dynamic shared + /// memory is what the kernel declares. + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + block: [3]i32, + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, block: [3]i32) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cute_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + // XLA passes one pointer per input then per output; it does + // not check the count against the kernel. + const expected = spec.inputs.len + spec.outputs.len; + if (b.args.len != expected) { + std.debug.panic("zml.kernel.cute.Kernel({s}): declareArgs declared {d} arguments, inputs + outputs are {d}", .{ name, b.args.len, expected }); + } + + return b.finish(block); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg, opts.block) catch |err| + std.debug.panic("zml.kernel.cute.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .block = opts.block, + .zeroed_outputs = opts.zeroed_outputs, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; + } +}; + pub const fly = struct { /// Threads per wavefront: 64 on CDNA, 32 on RDNA. pub fn waveSize(p: *const platform_.Platform) i32 { @@ -846,6 +955,34 @@ test "fly kernels run on rocm" { }; } +test "cute kernel emits the module the CuTe compiler takes" { + const Cfg = struct { n: i64 }; + const AddOne = cute.Kernel(Cfg, .{ + .name = "add_one", + .inputs = &.{"x"}, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *cute.Builder, cfg: Cfg) cute.FinishError!void { + const a = try b.declareArgs(.{ + .x = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, + .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, + }); + const i = b.blockIdx().x.mul(b.blockDim().x).add(b.threadIdx().x); + var guard = b.openIf(i.lt(cfg.n)); + a.out.set(.{i}, a.x.get(.{i}).add(1.0)); + guard.yieldThen(.{}); + } + }.run, + }); + + const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1000 }, .{ 128, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "module {\n func.func @add_one(") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "nvvm.reqntid = array") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cute.memref.load") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); +} + test "cuda_tile kernel emits a module XLA can parse" { const Cfg = struct { n: i64 }; const AddOne = cuda_tile.Kernel(Cfg, .{ diff --git a/zml/ops.zig b/zml/ops.zig index 4cdf9761..217f04b5 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1337,6 +1337,141 @@ pub fn cudaTile(inputs: anytype, outputs: anytype, opts: CudaTileOps) [outputs.l return outputs_; } +pub const CuteOps = struct { + /// The kernel symbol. + name: []const u8, + /// Textual CuTe DSL module: either public `func.func` kernels, launched + /// with `grid`/`block`, or a `gpu.module` plus a host launch function + /// that owns the launch configuration. + ir: []const u8, + grid: ?[3]i32 = null, + block: ?[3]i32 = null, + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: []const dialects.stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, +}; + +/// A `__gpu$xla.gpu.cute` custom call: XLA compiles `ir` with the CuTe compiler +/// and launches `name`. Every operand and result arrives as one raw device +/// pointer, in order, in the default layout. +pub fn cute(inputs: anytype, outputs: anytype, opts: CuteOps) [outputs.len]Tensor { + const mlir_ctx = Compiler.current().mlir_ctx; + + var values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + values[i] = inputs[i].value(); + } + + var res_types: [outputs.len]*const mlir.Type = undefined; + inline for (outputs, 0..) |output, i| { + res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); + } + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), + .named(mlir_ctx, "kernel_type", .string(mlir_ctx, "cute")), + .named(mlir_ctx, "ir", .string(mlir_ctx, opts.ir)), + }); + inline for (.{ "grid", "block" }) |key| { + if (@field(opts, key)) |dims| { + var elems: [3]*const mlir.Attribute = undefined; + for (&elems, dims) |*e, d| e.* = .int(mlir_ctx, .i32, d); + attrs.appendAssumeCapacity(.named(mlir_ctx, key, .array(mlir_ctx, &elems))); + } + } + if (opts.zeroed_outputs.len > 0) { + var zeroed: stdx.BoundedArray(*const mlir.Attribute, dialects.stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + for (opts.zeroed_outputs) |i| zeroed.appendAssumeCapacity(.int(mlir_ctx, .i32, i)); + attrs.appendAssumeCapacity(.named(mlir_ctx, "zeroed_outputs", .array(mlir_ctx, zeroed.constSlice()))); + } + const backend_config: *const mlir.Attribute = .dict(mlir_ctx, attrs.constSlice()); + + const op = dialects.stablehlo.custom_call( + mlir_ctx, + &values, + &res_types, + .{ + .call_target_name = "__gpu$xla.gpu.cute", + .backend_config = .{ .typed_ffi = backend_config }, + .has_side_effect = false, + .output_operand_aliases = opts.output_operand_aliases, + }, + .unknown(mlir_ctx), + ).appendTo(Compiler.current().currentScope().block); + + var outputs_: [outputs.len]Tensor = undefined; + inline for (outputs, 0..) |output, i| { + outputs_[i] = Tensor._result(output, op.result(i)); + } + + return outputs_; +} + +test "cute" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + if (platform.target != .cuda) return error.SkipZigTest; + + // out[i] = in[i] + 1 over 128 f32, in the CuTe DSL's frontend form. + const ir = + \\module attributes {gpu.container_module} { + \\ gpu.module @kernels { + \\ cuda.kernel @add_one(%arg0: !cute.ptr>, %arg1: !cute.ptr>) attributes {cute.kernel, gpu.kernel, nvvm.reqntid = array} { + \\ %i = nvvm.read.ptx.sreg.tid.x : i32 + \\ %shape = cute.make_shape() : () -> !cute.shape<"128"> + \\ %lay = cute.make_layout(%shape) : !cute.layout<"128:1"> + \\ %coord = cute.make_coord(%i) : (i32) -> !cute.coord<"?"> + \\ %in = cute.make_view(%arg0, %lay) : !cute.memref, "128:1"> + \\ %out = cute.make_view(%arg1, %lay) : !cute.memref, "128:1"> + \\ %x = cute.memref.load(%in, %coord) : (!cute.memref, "128:1">, !cute.coord<"?">) -> f32 + \\ %one = arith.constant 1.0 : f32 + \\ %y = arith.addf %x, %one : f32 + \\ cute.memref.store(%out, %coord, %y) : (!cute.memref, "128:1">, !cute.coord<"?">, f32) -> () + \\ return + \\ } + \\ } + \\ func.func @launch(%arg0: !cute.ptr>, %arg1: !cute.ptr>) -> i32 attributes {llvm.emit_c_interface} { + \\ %smem = cute.kernel_smem_size @kernels::@add_one : i64 + \\ %c0_i64 = arith.constant 0 : i64 + \\ %stream = cuda.cast %c0_i64 : i64 -> !cuda.stream + \\ %c1 = arith.constant 1 : i32 + \\ %c128 = arith.constant 128 : i32 + \\ %cfg = cuda.launch_cfg.create (blockDim = (%c128, %c1, %c1), dynamicSmemBytes = %smem, gridDim = (%c1, %c1, %c1), stream = %stream) : i32, i32, i32, i64, i32, i32, i32, !cuda.stream -> !cuda.launch_cfg + \\ %r = cuda.launch_ex @kernels::@add_one<%cfg> (%arg0, %arg1) : !cuda.launch_cfg, (!cute.ptr>, !cute.ptr>) -> !cuda.result + \\ %status = cuda.cast %r : !cuda.result -> i32 + \\ return %status : i32 + \\ } + \\} + ; + + const Mod = struct { + pub fn forward(a: Tensor) Tensor { + return cute(.{a}, .{a.shape()}, .{ .name = "add_one", .ir = ir })[0]; + } + }; + + const a: zml.Tensor = .init(.{ .n = 128 }, .f32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Mod.forward, .{a}, platform, .{}); + defer exe.deinit(); + + var input: [128]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @floatFromInt(i); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&input)); + defer a_buffer.deinit(); + + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Mod.forward, .{a_buffer}); + defer result.deinit(); + + var host = try result.toSliceAlloc(std.testing.allocator, std.testing.io); + defer host.free(std.testing.allocator); + + for (host.items(f32), 0..) |v, i| { + try std.testing.expectEqual(@as(f32, @floatFromInt(i + 1)), v); + } +} + test "cuda_tile" { const zml = @import("zml.zig"); const platform = zml.testing.env(); From 4adb31b4d6e62865acca76d8b84c5e022b3c3dec Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 16 Sep 2026 14:53:44 +0200 Subject: [PATCH 686/881] platforms: bump plugins for CPU, ROCm, CUDA and Metal (#812) Bump to build `202609161226.43.1.b992d049cec2` --- platforms/cpu/cpu.bzl | 6 +++--- platforms/cuda/cuda.bzl | 8 ++++---- platforms/metal/metal.bzl | 4 ++-- platforms/rocm/rocm.bzl | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 4be74384..929102ad 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -24,21 +24,21 @@ def _cpu_plugin_impl(mctx): name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, sha256 = "5d6783ccd775a74dbe8d03e9f35e7a5a422ef051f56bb4266ac918bcb5f56970", - url = "https://mirror.zml.ai/plugins/202609151715.39.1.4dc71c717185/zml-cpu-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202609161226.43.1.b992d049cec2/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "3535dee8b26bcf3e1906779b103bcbf8903c88ca93b85487d98fd2181e4f99be", - url = "https://mirror.zml.ai/plugins/202609151715.39.1.4dc71c717185/zml-cpu-darwin-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202609161226.43.1.b992d049cec2/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "bf5c036ba2a8195f8449119fa6138c7b83d7382c5732c2149d74a7b901095ffd", - url = "https://mirror.zml.ai/plugins/202609151715.39.1.4dc71c717185/zml-cpu-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202609161226.43.1.b992d049cec2/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index ef5a75ec..5cb12c1c 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -215,12 +215,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "e698804410e260f48ffdea425295c6551f3466eb8ea571ce07f9c9b43b0b034c", - "url": "https://mirror.zml.ai/plugins/202609151715.39.1.4dc71c717185/zml-cuda-linux-amd64.tar.zst", + "sha256": "5f464f17a514b871945728951d68a729f64782593b691feb6ced3626f322ff7f", + "url": "https://mirror.zml.ai/plugins/202609161226.43.1.b992d049cec2/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "ba5717ec04c25761b6efe7f77ae89542c27ec4e56f5e3e548a0d39f8b7863aac", - "url": "https://mirror.zml.ai/plugins/202609151715.39.1.4dc71c717185/zml-cuda-linux-arm64.tar.zst", + "sha256": "07521ec4647d1ce4ef8f611b7e65a7ccae5b99b4bc05795c01f3b5a6563cb2a5", + "url": "https://mirror.zml.ai/plugins/202609161226.43.1.b992d049cec2/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 3eea63c9..d9908a98 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "c081e43b3c265728a7345e6735335c115bb53d7158bc594d192e611e90494fd4", - url = "https://mirror.zml.ai/plugins/202609151715.39.1.4dc71c717185/zml-metal-darwin-arm64.tar.zst", + sha256 = "da6e5e880b950d91cb036b3b68ebd7cd26a8e265b20ff58ea4b52dc242a365bd", + url = "https://mirror.zml.ai/plugins/202609161226.43.1.b992d049cec2/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index a34e9da7..ae600a05 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202609151715.39.1.4dc71c717185/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "c4fd5b602407f478525987e20cc9f306731046c336397eef3568dd2dfd8bb158" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202609161226.43.1.b992d049cec2/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "2dfdacb5f58141d8c5e618a69794df6b7f4fa213e7748e1e891106f629467084" def _rocm_package_name(name): return name + _ROCM_VERSION From ace231388d01ee93c079a448f9ec06bc91822980 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Wed, 16 Sep 2026 17:48:02 +0200 Subject: [PATCH 687/881] platforms/cpu: fix sandbox lib path after plugin layout change (#819) The CPU plugin archives now ship the shared library under lib/, and copy_to_directory outputs to sandbox/ instead of sandbox/lib. Look up the sandbox directory in runfiles and join lib/ when building the library path, matching metal, rocm and cuda. --- platforms/cpu/cpu.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platforms/cpu/cpu.zig b/platforms/cpu/cpu.zig index 77a3fa60..25d8bdd3 100644 --- a/platforms/cpu/cpu.zig +++ b/platforms/cpu/cpu.zig @@ -24,7 +24,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { const r = try bazel.runfiles(bazel_builtin.current_repository); var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = try r.rlocation("zml/platforms/cpu/sandbox/lib", &path_buf) orelse { + const sandbox_path = try r.rlocation("zml/platforms/cpu/sandbox", &path_buf) orelse { log.err("Failed to find sandbox path for CPU runtime", .{}); return error.FileNotFound; }; @@ -37,7 +37,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { }; var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "libzml_cpu" ++ ext }); + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_cpu" ++ ext }); break :blk .loadFrom(path); }; } From 7adc862cbd1b780a4ac10c1cb0602d271f29f3a3 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 16 Sep 2026 17:57:40 +0200 Subject: [PATCH 688/881] pjrt: rename ambiguous field of `CreateViewOfDeviceBufferArgs` (#814) --- pjrt/pjrt.zig | 4 ++-- zml/platform.zig | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 5f71050e..37e9fb13 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -523,7 +523,7 @@ pub const Client = opaque { } pub const CreateViewOfDeviceBufferArgs = struct { - data: *anyopaque, + device_buffer_ptr: *anyopaque, dims: []const i64, element_type: BufferType, layout: MemoryLayout, @@ -539,7 +539,7 @@ pub const Client = opaque { const layout = args.layout.toCStruct(); const ret = try api.call(.PJRT_Client_CreateViewOfDeviceBuffer, .{ .client = self.inner(), - .device_buffer_ptr = @constCast(args.data), + .device_buffer_ptr = @constCast(args.device_buffer_ptr), .dims = args.dims.ptr, .num_dims = args.dims.len, .element_type = @intFromEnum(args.element_type), diff --git a/zml/platform.zig b/zml/platform.zig index a99d104e..a9e076c0 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -963,7 +963,7 @@ fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { } else return error.MemoryNotFound; var pjrt_buffer = try pjrt_client.createViewOfDeviceBuffer(pjrt_api, .{ - .data = buffer.data, + .device_buffer_ptr = buffer.data, .dims = shape.dims(), .element_type = pjrtx.bufferTypeFromDtype(shape.dtype()), .device = device, From ae9ce333edbe82dad76ef898cc6f1099e99fd001 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 16 Sep 2026 17:57:40 +0200 Subject: [PATCH 689/881] zml/io: introduce support of memory in the loader (#815) This only works on OneAPI and Cuda for now. For other platform we would need a writer that allocates host pinned uninitialized buffers for the shards and write directly to them. --- zml/io.zig | 78 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 69 insertions(+), 9 deletions(-) diff --git a/zml/io.zig b/zml/io.zig index ee020b7e..436aab71 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -27,6 +27,7 @@ pub const TensorStore = struct { pub const Binding = struct { tensors: []*safetensors.Tensor, transformed: bool, + memory: Memory.Kind = .default, }; registry: *safetensors.TensorRegistry, @@ -161,7 +162,19 @@ pub const TensorStore = struct { } else false; } + pub const CreateTensorOpts = struct { + memory: Memory.Kind = .default, + }; + pub fn maybeCreateTensor(self: View, subkey: []const u8, tagz: anytype, partitioning: anytype) ?Tensor { + return self.maybeCreateTensorInternal(subkey, tagz, partitioning, .{}); + } + + pub fn maybeCreateHostPinnedTensor(self: View, subkey: []const u8, tagz: anytype, partitioning: anytype) ?Tensor { + return self.maybeCreateTensorInternal(subkey, tagz, partitioning, .{ .memory = .host_pinned }); + } + + fn maybeCreateTensorInternal(self: View, subkey: []const u8, tagz: anytype, partitioning: anytype, opts: CreateTensorOpts) ?Tensor { var buffer: [256]u8 = undefined; const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); const source = self.store.dupeSource(key) orelse return null; @@ -175,13 +188,18 @@ pub const TensorStore = struct { shape = applyPartitioning(shape, partitioning); const tensor: Tensor = .fromShape(shape); - self.store.putSourcesNoClobber(tensor.id, .{ .tensors = sources, .transformed = false }) catch |e| std.debug.panic("Not handling {} errors", .{e}); + self.store.putSourcesNoClobber(tensor.id, .{ .tensors = sources, .transformed = false, .memory = opts.memory }) catch |e| std.debug.panic("Not handling {} errors", .{e}); return tensor; } pub fn createTensor(self: View, subkey: []const u8, tagz: anytype, partitioning: anytype) Tensor { - return self.maybeCreateTensor(subkey, tagz, partitioning) orelse + return self.maybeCreateTensorInternal(subkey, tagz, partitioning, .{}) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); + } + + pub fn createHostPinnedTensor(self: View, subkey: []const u8, tagz: anytype, partitioning: anytype) Tensor { + return self.maybeCreateTensorInternal(subkey, tagz, partitioning, .{ .memory = .host_pinned }) orelse stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); } @@ -417,14 +435,14 @@ pub const Loader = struct { }; stdx.debug.assert(!sources.transformed and sources.tensors.len == 1, "Tensor {} is transformed or has {} sources; `load` only streams single-source tensors", .{ tensor.id, sources.tensors.len }); - self.loadSingleInner(io, sources.tensors[0], tensor.shape(), buffer, shardings, opts) catch |e| { + self.loadSingleInner(io, sources.tensors[0], tensor.shape(), buffer, sources.memory, shardings, opts) catch |e| { log.err("Errors are not handled in `defaultCallback`, got {}", .{e}); unreachable; }; } fn loadSingle(self: *Loader, io: std.Io, source: *safetensors.Tensor, shape: Shape, buffer: *Buffer, loaded: *bool, shardings: []const Sharding, opts: LoadOpts) void { - self.loadSingleInner(io, source, shape, buffer, shardings, opts) catch |e| { + self.loadSingleInner(io, source, shape, buffer, .device, shardings, opts) catch |e| { log.err("Failed to load tensor {s}: {}", .{ source.name, e }); loaded.* = false; return; @@ -438,6 +456,7 @@ pub const Loader = struct { source: *safetensors.Tensor, shape: Shape, buffer: *Buffer, + memory: Memory.Kind, shardings: []const Sharding, opts: LoadOpts, ) !void { @@ -459,6 +478,7 @@ pub const Loader = struct { shape, sharding, buffer, + memory, ); defer writer.deinit(self.allocator); @@ -621,14 +641,16 @@ pub const MemoryWriter = union(enum) { shape: Shape, sharding: Sharding, buffer: *Buffer, + memory: Memory.Kind, ) !MemoryWriter { return switch (platform.target) { .cuda, .oneapi => .{ - .direct = try .init(allocator, io, platform, pools, dma_allocators, dma_chunk_size, shape, sharding, buffer), - }, - .rocm, .tpu, .neuron, .cpu, .metal => .{ - .buffered = try .init(allocator, io, platform, shape, sharding, buffer), + .direct = try .init(allocator, io, platform, pools, dma_allocators, dma_chunk_size, shape, sharding, buffer, memory), }, + .rocm, .tpu, .neuron, .cpu, .metal => if (memory == .host_pinned) + std.debug.panic("Host pinned memory is not supported on {}", .{platform.target}) + else + .{ .buffered = try .init(allocator, io, platform, shape, sharding, buffer) }, }; } @@ -1133,6 +1155,7 @@ pub const DirectMemoryWriter = struct { shape: Shape, sharding: Sharding, buffer: *Buffer, + memory: Memory.Kind, ) !DirectMemoryWriter { const ordered_devices = sharding.devicesInCanonicalOrder(); var shard_writers = try allocator.alloc(DirectShardWriter, ordered_devices.len); @@ -1150,7 +1173,7 @@ pub const DirectMemoryWriter = struct { const pool = &pools[device.id]; const shard_dma_allocator = dma_allocators[device.id].allocator(); - const pjrt_mem = platform.devices[device.id].memory(.default).?; + const pjrt_mem = platform.devices[device.id].memory(memory).?; shard_writers[i] = try .init(shard_dma_allocator, io, pjrt_mem, pool, placement.shape); @@ -1381,6 +1404,20 @@ pub const DirectMemoryWriter = struct { } }; +fn buildMesh2( + allocator: std.mem.Allocator, + target: @import("platform.zig").Target, + devices: []const @import("platform.zig").Device, +) !Sharding.PhysicalMesh { + if (devices.len < 2) return error.NotEnoughDevices; + const topology: Sharding.PhysicalMesh.Tree = .axis(.link_x, .{ .mesh = .torus }, &.{ + .device(devices[0]), + .device(devices[1]), + }); + + return Sharding.PhysicalMesh.fromTree(allocator, target, topology); +} + fn buildMesh2x2( allocator: std.mem.Allocator, target: @import("platform.zig").Target, @@ -1449,6 +1486,7 @@ const DirectMemoryWriterDeviceTest = struct { writable_slice_min_len: usize = 128, pool_chunks: usize = 4, pool_chunk_size: usize = 1 << 20, + memory: Memory.Kind = .default, }; allocator: std.mem.Allocator, @@ -1467,6 +1505,7 @@ const DirectMemoryWriterDeviceTest = struct { scenario.writable_slice_min_len, scenario.pool_chunks, scenario.pool_chunk_size, + scenario.memory, ); } @@ -1479,6 +1518,7 @@ const DirectMemoryWriterDeviceTest = struct { writable_slice_min_len: usize, pool_chunks: usize, pool_chunk_size: usize, + memory: Memory.Kind, ) !void { const slice = try Slice.alloc(self.allocator, shape); defer slice.free(self.allocator); @@ -1514,6 +1554,7 @@ const DirectMemoryWriterDeviceTest = struct { shape, sharding, &written_buffer, + memory, ); defer writer.deinit(); defer written_buffer.deinit(); @@ -1676,3 +1717,22 @@ test "DirectMemoryWriter: 3D topology folded model + replicated batch" { }, }); } + +test "MemoryWriter can produce a host pinned buffer" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "host_pinned", + .create_options = .{ + .physical_mesh = .{ .custom = buildMesh2 }, + }, + .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32) + .withPartitioning(.{ .batch = .replicated, .model = .model }), + .logical_mesh = .mesh(.{ .model = .high_bandwidth }), + .strategy = .parseBindings(.{ .model = .link_x }), + .memory = .host_pinned, + }); +} From 78a89679fa1c959f58a32099fe50a8e969819d08 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 16 Sep 2026 17:57:41 +0200 Subject: [PATCH 690/881] zml/buffer: introduce simple `createView` (#817) `Buffer.createView` can be used to create a view from a host pinned buffer and use it as if it was one the device. This avoids XLA limitations of doing operations directly on host pinned buffers. --- pjrt/pjrt.zig | 34 ++++++++++++++++++++++++++++++++++ zml/buffer.zig | 24 ++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 37e9fb13..8efdeae3 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -1071,6 +1071,35 @@ pub const MemoryLayout = union(MemoryLayoutType) { }, }; } + + fn fromCStruct(self: c.PJRT_Buffer_MemoryLayout) MemoryLayout { + return switch (self.type) { + c.PJRT_Buffer_MemoryLayout_Type_Tiled => b: { + const tiled = self.unnamed_0.tiled; + + const minor_to_major: []const i64 = if (tiled.minor_to_major == null) &.{} else tiled.minor_to_major[0..tiled.minor_to_major_size]; + const tile_dims_sizes: []const usize = if (tiled.tile_dim_sizes == null) &.{} else tiled.tile_dim_sizes[0..tiled.num_tiles]; + + const tile_dims: []const i64 = if (tiled.tile_dims == null) &.{} else td: { + var tile_dim_len: usize = 0; + for (tile_dims_sizes) |size| tile_dim_len += size; + break :td tiled.tile_dims[0..tile_dim_len]; + }; + + break :b .{ + .tiled = .{ + .minor_to_major = minor_to_major, + .tile_dims = tile_dims, + .tile_dims_sizes = tile_dims_sizes, + }, + }; + }, + c.PJRT_Buffer_MemoryLayout_Type_Strides => .{ + .strides = .{ .byte_strides = self.unnamed_0.strides.byte_strides[0..self.unnamed_0.strides.num_byte_strides] }, + }, + else => unreachable, + }; + } }; pub const DefaultMemoryLayout = struct { @@ -1310,6 +1339,11 @@ pub const Buffer = opaque { .buffer = self.inner(), }); } + + pub fn memoryLayout(self: *const Buffer, api: *const Api) ApiError!MemoryLayout { + const ret = try api.call(.PJRT_Buffer_GetMemoryLayout, .{ .buffer = self.inner() }); + return .fromCStruct(ret.layout); + } }; pub const Event = opaque { diff --git a/zml/buffer.zig b/zml/buffer.zig index a620efd7..a687943e 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -336,6 +336,30 @@ pub const Buffer = struct { pub fn opaqueDevicePtr(self: Buffer, device_id: usize) *anyopaque { return self._shards.get(device_id).opaqueDeviceMemoryDataPointer(self._platform.pjrt_api) catch unreachable; } + + /// Creates a view of the given buffer + pub fn createView(self: Buffer) !Buffer { + const platform = self._platform; + + var view_shards_buffer: [MAX_NUM_SHARDS]*pjrt.Buffer = undefined; + var view_shard_list: std.ArrayList(*pjrt.Buffer) = .initBuffer(&view_shards_buffer); + errdefer for (view_shard_list.items) |shard| { + shard.deinit(platform.pjrt_api); + }; + + for (self._shards.constSlice()) |shard| { + const view_shard = try platform.pjrt_client.createViewOfDeviceBuffer(platform.pjrt_api, .{ + .device_buffer_ptr = try shard.opaqueDeviceMemoryDataPointer(platform.pjrt_api), + .dims = shard.dimensions(platform.pjrt_api), + .element_type = shard.elementType(platform.pjrt_api), + .layout = try shard.memoryLayout(platform.pjrt_api), + .device = try shard.device(platform.pjrt_api), + }); + view_shard_list.appendAssumeCapacity(view_shard); + } + + return fromPjrtBuffers(platform, self._shape, self._sharding, view_shard_list.items); + } }; test "device round-trip" { From 51235e7eb69dd287cfda82150690eab251c2ed23 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 16 Sep 2026 18:13:08 +0200 Subject: [PATCH 691/881] zml: block32 FP8 with E8M0 scales, and an explicit Linear output dtype (#811) Two things, the second one falling out of the first. **`fp8_block32`**: E4M3FN weights with one scale per 32x32 tile, the layout DeepSeek V4.1 ships. `quantizeBlockFp8` takes a block size and a scale dtype; the E8M0 path floors amax at 1e-4 and rounds the exponent up, so the activation side is MXFP8: one E8M0 scale per row per 32 contracted values. That pairing is what XLA's fp8_block_gemv arm needs to hand both scales to `tt.dot_scaled` (`Fp8BlockGemvScaleInMma` requires E8M0 on *both* sides); F32 activation scales would quietly fall to the per-block multiply. The Triton MoE path gets the same scheme at `block_size_k = 32`. `Linear.forward` also stops rounding the activation to BF16 before quantizing, so amax is computed on the input as given. **An explicit output dtype** on `scaledDot` and `Linear.forward`. Inference cannot answer what an FP8 x FP8 product should return: the accumulation is F32 either way, so BF16 or F32 back is the caller's choice. `forwardScaled` now hands the caller's type to `scaledDot` rather than rounding through BF16 and converting after. Narrow outputs are rejected. Separately, `Linear.forward` converts the weight and bias to the activation dtype. `dotGeneral` asserts both operands share one, which broke the moment a model raised its activation to F32 against a BF16 weight; the wrappers this API replaced converted for exactly that reason. **Breaking:** `Linear.forward` takes a second argument and `scaledDot` a fifth. --------- Co-authored-by: Raphael Nahoum --- docs/howtos/howto_torch2zml.md | 6 +- examples/llm/models/llama/model.zig | 16 ++--- examples/llm/models/llama_tests.zig | 10 +++- examples/llm/models/qwen3_5/model.zig | 26 ++++---- examples/llm/models/qwen3_5_moe/model.zig | 30 +++++----- platforms/cpu/cpu.bzl | 6 +- platforms/cuda/cuda.bzl | 8 +-- platforms/metal/metal.bzl | 4 +- platforms/rocm/rocm.bzl | 4 +- zml/moe/fused_experts.zig | 7 ++- zml/moe/triton.zig | 17 +++++- zml/moe/triton_kernels/triton_kernels.zig | 32 ++++++---- zml/nn.zig | 43 ++++++++------ zml/quantization.zig | 72 +++++++++++++++-------- 14 files changed, 172 insertions(+), 109 deletions(-) diff --git a/docs/howtos/howto_torch2zml.md b/docs/howtos/howto_torch2zml.md index d4c1fffd..d05e2b6f 100644 --- a/docs/howtos/howto_torch2zml.md +++ b/docs/howtos/howto_torch2zml.md @@ -259,10 +259,10 @@ const Mlp = struct { down_proj: zml.nn.Linear, pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { - const proj = self.up_proj.forward(x); - var output = self.gate_proj.forward(x); + const proj = self.up_proj.forward(x, x.dtype()); + var output = self.gate_proj.forward(x, x.dtype()); output = output.silu().mul(proj); - return self.down_proj.forward(output); + return self.down_proj.forward(output, x.dtype()); } }; ``` diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index fd573c32..b86c6d6b 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -367,7 +367,7 @@ pub const LmHead = struct { var logits = blk: { if (self.lm_head) |lm_head| { - break :blk lm_head.forward(hidden).rename(.{ .dout = .d }); + break :blk lm_head.forward(hidden, hidden.dtype()).rename(.{ .dout = .d }); } else { break :blk self.embed_tokens.weight.withTags(.{ .voc, .d }).dot(hidden, .d); } @@ -491,10 +491,10 @@ const Mlp = struct { } pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { - const proj = self.up_proj.forward(x); - var output = self.gate_proj.forward(x); + const proj = self.up_proj.forward(x, x.dtype()); + var output = self.gate_proj.forward(x, x.dtype()); output = output.silu().mul(proj).rename(.{ .dout = .d }); - return self.down_proj.forward(output); + return self.down_proj.forward(output, output.dtype()); } }; @@ -557,9 +557,9 @@ const SelfAttn = struct { // This avoids paying gather-style collectives independently for each projection. const x_qkv = x.withPartitioning(.{ .d = .replicated }); - var q = self.q_proj.forward(x_qkv).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); - var k = self.k_proj.forward(x_qkv).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); - var v = self.v_proj.forward(x_qkv).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); + var q = self.q_proj.forward(x_qkv, x_qkv.dtype()).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); + var k = self.k_proj.forward(x_qkv, x_qkv.dtype()).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); + var v = self.v_proj.forward(x_qkv, x_qkv.dtype()).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); // In self-attention, .s axis is used both for keys and queries. const pos_index = b: { @@ -603,7 +603,7 @@ const SelfAttn = struct { ); const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); - const delta = self.o_proj.forward(attn) + const delta = self.o_proj.forward(attn, attn.dtype()) .rename(.{ .dout = .d }) .withPartitioning(.{ .d = .replicated }); return .{ delta, new_kv_cache }; diff --git a/examples/llm/models/llama_tests.zig b/examples/llm/models/llama_tests.zig index 5b5de5f7..537c04a4 100644 --- a/examples/llm/models/llama_tests.zig +++ b/examples/llm/models/llama_tests.zig @@ -108,7 +108,15 @@ fn testLayer( var out_buffer_expected = try loadBufferFromStore(allocator, io, platform, activation_store, out_key, sharding); defer out_buffer_expected.deinit(); - const exe = try platform.compileFn(allocator, io, @TypeOf(layer).forward, .{ layer, in_tensor }, .{ .shardings = &.{sharding} }); + // `zml.nn.Linear.forward` takes an explicit output dtype; every other layer here + // is `fn (self, Tensor) Tensor`. + const Layer = @TypeOf(layer); + const Call = struct { + fn forward(l: Layer, x: zml.Tensor) zml.Tensor { + return if (Layer == zml.nn.Linear) l.forward(x, x.dtype()) else Layer.forward(l, x); + } + }; + const exe = try platform.compileFn(allocator, io, Call.forward, .{ layer, in_tensor }, .{ .shardings = &.{sharding} }); defer exe.deinit(); var args = try exe.args(allocator); diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 22fb4d92..5a2a41ac 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -260,7 +260,7 @@ pub const Sampler = struct { pub fn sampleTokens(input: Input) Output { const self = input.sampler; const x = self.norm.forward(input.hidden); - const logits = self.lm_head.forward(x.withPartialTags(.{.d})).rename(.{ .dout = .voc }); + const logits = self.lm_head.forward(x.withPartialTags(.{.d}), x.dtype()).rename(.{ .dout = .voc }); const next_tokens, const new_rng = zml.nn.sampleTokens(logits, self.gen_options.sampling_strategy, input.rng); return .{ .tokens = next_tokens.convert(.u32), @@ -523,11 +523,11 @@ pub const Mlp = struct { } pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { - const up_projed = self.up_proj.forward(x); - const gate = self.gate_proj.forward(x); + const up_projed = self.up_proj.forward(x, x.dtype()); + const gate = self.gate_proj.forward(x, x.dtype()); const hidden = gate.silu().mul(up_projed); - const output = self.down_proj.forward(hidden); + const output = self.down_proj.forward(hidden, hidden.dtype()); return output; } }; @@ -578,7 +578,7 @@ pub const SelfAttn = struct { } fn projectQAndGate(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { - const q_proj = self.q_proj.forward(x).splitAxis(.dout, .{ .h = self.num_heads, .hd = 2 * self.head_dim }); + const q_proj = self.q_proj.forward(x, x.dtype()).splitAxis(.dout, .{ .h = self.num_heads, .hd = 2 * self.head_dim }); const q, var gate = q_proj.chunkExact(.hd, 2); gate = gate.merge(.{ .d_out_proj = .{ .h, .hd } }); return .{ q, gate }; @@ -589,8 +589,8 @@ pub const SelfAttn = struct { if (self.num_kv_heads > 0) break :b self.num_kv_heads; break :b self.num_heads; }; - const k = self.k_proj.forward(x).splitAxis(.dout, .{ .h = num_kv_heads, .hd = self.head_dim }); - const v = self.v_proj.forward(x).splitAxis(.dout, .{ .h = num_kv_heads, .hd = self.head_dim }); + const k = self.k_proj.forward(x, x.dtype()).splitAxis(.dout, .{ .h = num_kv_heads, .hd = self.head_dim }); + const v = self.v_proj.forward(x, x.dtype()).splitAxis(.dout, .{ .h = num_kv_heads, .hd = self.head_dim }); return .{ k, v }; } @@ -667,7 +667,7 @@ pub const SelfAttn = struct { const gated_output = attn_output.mul(gate.sigmoid()); const projected_output = self.o_proj - .forward(gated_output.rename(.{ .d_out_proj = .d })) + .forward(gated_output.rename(.{ .d_out_proj = .d }), gated_output.dtype()) .rename(.{ .dout = .d }) .withPartitioning(.{ .d = .replicated }); @@ -874,7 +874,7 @@ pub const GatedDeltaNet = struct { const left_pad = self.conv_kernel_size - 1; const x_in = x.withPartitioning(.{ .d = .replicated }); - const projected_qkv = self.in_proj_qkv.forward(x_in) + const projected_qkv = self.in_proj_qkv.forward(x_in, x_in.dtype()) .rename(.{ .dout = .mix }) .withPartitioning(.{ .s = .replicated, .mix = .model }); const use_cached_state = x.dim(.s) == 1 and left_pad > 0; @@ -908,10 +908,10 @@ pub const GatedDeltaNet = struct { } mixed_qkv = mixed_qkv.withPartitioning(.{ .s = .replicated, .mix = .model }); - const z = self.in_proj_z.forward(x_in) + const z = self.in_proj_z.forward(x_in, x_in.dtype()) .splitAxis(.dout, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }); - const b = self.in_proj_b.forward(x_in).rename(.{ .dout = .vh }); - const a = self.in_proj_a.forward(x_in).rename(.{ .dout = .vh }); + const b = self.in_proj_b.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }); + const a = self.in_proj_a.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }); const query = mixed_qkv .slice(.mix, .{ .start = 0, .end = key_dim }) @@ -957,7 +957,7 @@ pub const GatedDeltaNet = struct { .rename(.{ .d = .vhd }); const output = self.out_proj - .forward(core_attn_out_normed.merge(.{ .d = .{ .vh, .vhd } })) + .forward(core_attn_out_normed.merge(.{ .d = .{ .vh, .vhd } }), core_attn_out_normed.dtype()) .rename(.{ .dout = .d }) .withPartitioning(.{ .d = .replicated }); const updated_cache = cache.update( diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 5370a6b2..7daa9750 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -265,7 +265,7 @@ pub const Sampler = struct { pub fn sampleTokens(input: Input) Output { const self = input.sampler; const x = self.norm.forward(input.hidden); - const logits = self.lm_head.forward(x.withPartialTags(.{.d})).rename(.{ .dout = .voc }); + const logits = self.lm_head.forward(x.withPartialTags(.{.d}), x.dtype()).rename(.{ .dout = .voc }); const next_tokens, const new_rng = zml.nn.sampleTokens(logits, self.gen_options.sampling_strategy, input.rng); return .{ .tokens = next_tokens.convert(.u32), @@ -583,7 +583,7 @@ pub const SelfAttn = struct { } fn projectQAndGate(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { - const q_proj = self.q_proj.forward(x) + const q_proj = self.q_proj.forward(x, x.dtype()) .splitAxis(.dout, .{ .h = self.num_heads, .hd = 2 * self.head_dim }); const q, var gate = q_proj.chunkExact(.hd, 2); gate = gate.merge(.{ .d_out_proj = .{ .h, .hd } }); @@ -592,9 +592,9 @@ pub const SelfAttn = struct { fn projectKV(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; - const k = self.k_proj.forward(x) + const k = self.k_proj.forward(x, x.dtype()) .splitAxis(.dout, .{ .h = num_kv_heads, .hd = self.head_dim }); - const v = self.v_proj.forward(x) + const v = self.v_proj.forward(x, x.dtype()) .splitAxis(.dout, .{ .h = num_kv_heads, .hd = self.head_dim }); return .{ k, v }; } @@ -650,7 +650,7 @@ pub const SelfAttn = struct { ).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); const gated_output = attn_output.mul(gate.sigmoid()); - const projected_output = self.o_proj.forward(gated_output.rename(.{ .d_out_proj = .d })).rename(.{ .dout = .d }).withPartitioning(.{ .d = .replicated }); + const projected_output = self.o_proj.forward(gated_output.rename(.{ .d_out_proj = .d }), gated_output.dtype()).rename(.{ .dout = .d }).withPartitioning(.{ .d = .replicated }); return .{ projected_output, new_kv_cache }; } @@ -692,11 +692,11 @@ pub const Mlp = struct { } pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { - const up_projed = self.up_proj.forward(x); - const gate = self.gate_proj.forward(x); + const up_projed = self.up_proj.forward(x, x.dtype()); + const gate = self.gate_proj.forward(x, x.dtype()); const hidden = gate.silu().mul(up_projed).rename(.{ .dout = .d }); - const output = self.down_proj.forward(hidden); + const output = self.down_proj.forward(hidden, hidden.dtype()); return output; } }; @@ -721,7 +721,7 @@ const Router = struct { } pub fn forward(self: Router, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { - const router_logits = self.router.forward(x).convert(.f32); + const router_logits = self.router.forward(x, .f32); const routing = router_logits.topK(.{ .top_expert = .expert }, self.num_experts_per_tok, .{}); const topk_ids = routing.indices.convert(.i32); const router_scores = routing.values.softmax(.top_expert); @@ -785,7 +785,7 @@ pub const Moe = struct { moe_parameters, ) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); - const shared_gate = self.shared_expert_gate.forward(x).sigmoid().broad(x.shape()); + const shared_gate = self.shared_expert_gate.forward(x, x.dtype()).sigmoid().broad(x.shape()); const shared = self.shared_expert.forward(x).rename(.{ .dout = .d }).mul(shared_gate).withPartitioning(.{ .b = .replicated, .s = .replicated, @@ -998,7 +998,7 @@ pub const GatedDeltaNet = struct { const left_pad = self.conv_kernel_size - 1; const x_in = x.withPartitioning(.{ .d = .replicated }); - const projected_qkv = self.in_proj_qkv.forward(x_in) + const projected_qkv = self.in_proj_qkv.forward(x_in, x_in.dtype()) .rename(.{ .dout = .mix }).withPartitioning(.{ .s = .replicated, .mix = .model }); const use_cached_state = x.dim(.s) == 1 and left_pad > 0; const conv_input = if (use_cached_state) @@ -1031,11 +1031,11 @@ pub const GatedDeltaNet = struct { } mixed_qkv = mixed_qkv.withPartitioning(.{ .s = .replicated, .mix = .model }); - const z = self.in_proj_z.forward(x_in) + const z = self.in_proj_z.forward(x_in, x_in.dtype()) .splitAxis(.dout, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }) .withPartitioning(.{ .s = .replicated, .vh = .model, .vhd = .replicated }); - const b = self.in_proj_b.forward(x_in).rename(.{ .dout = .vh }).withPartitioning(.{ .s = .replicated, .vh = .model }); - const a = self.in_proj_a.forward(x_in).rename(.{ .dout = .vh }).withPartitioning(.{ .s = .replicated, .vh = .model }); + const b = self.in_proj_b.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }).withPartitioning(.{ .s = .replicated, .vh = .model }); + const a = self.in_proj_a.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }).withPartitioning(.{ .s = .replicated, .vh = .model }); const query = mixed_qkv .slice(.mix, .{ .start = 0, .end = key_dim }) @@ -1075,7 +1075,7 @@ pub const GatedDeltaNet = struct { .rename(.{ .d = .vhd }) .withPartitioning(.{ .s = .replicated, .vh = .model, .vhd = .replicated }); - const output = self.out_proj.forward(core_attn_out_normed.merge(.{ .d = .{ .vh, .vhd } })) + const output = self.out_proj.forward(core_attn_out_normed.merge(.{ .d = .{ .vh, .vhd } }), core_attn_out_normed.dtype()) .rename(.{ .dout = .d }).withPartitioning(.{ .d = .replicated }); const updated_cache = cache.update( if (use_cached_state) buildUpdatedConvState(conv_input, left_pad) else buildUpdatedConvStateFromPrefix(projected_qkv, left_pad, active_length), diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 929102ad..39f7bd09 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -24,21 +24,21 @@ def _cpu_plugin_impl(mctx): name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, sha256 = "5d6783ccd775a74dbe8d03e9f35e7a5a422ef051f56bb4266ac918bcb5f56970", - url = "https://mirror.zml.ai/plugins/202609161226.43.1.b992d049cec2/zml-cpu-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202609161604.45.1.80cbf79212e9/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "3535dee8b26bcf3e1906779b103bcbf8903c88ca93b85487d98fd2181e4f99be", - url = "https://mirror.zml.ai/plugins/202609161226.43.1.b992d049cec2/zml-cpu-darwin-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202609161604.45.1.80cbf79212e9/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "bf5c036ba2a8195f8449119fa6138c7b83d7382c5732c2149d74a7b901095ffd", - url = "https://mirror.zml.ai/plugins/202609161226.43.1.b992d049cec2/zml-cpu-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202609161604.45.1.80cbf79212e9/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 5cb12c1c..d7634924 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -215,12 +215,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "5f464f17a514b871945728951d68a729f64782593b691feb6ced3626f322ff7f", - "url": "https://mirror.zml.ai/plugins/202609161226.43.1.b992d049cec2/zml-cuda-linux-amd64.tar.zst", + "sha256": "45492852cb39295cdc14032c757af2c1edd8a494c737e00be70b5359cc883db5", + "url": "https://mirror.zml.ai/plugins/202609161604.45.1.80cbf79212e9/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "07521ec4647d1ce4ef8f611b7e65a7ccae5b99b4bc05795c01f3b5a6563cb2a5", - "url": "https://mirror.zml.ai/plugins/202609161226.43.1.b992d049cec2/zml-cuda-linux-arm64.tar.zst", + "sha256": "7f1dccc1179073d37270ae905623f5eff1ca0ca9f48cd2e127f0ce2e0e6f1d70", + "url": "https://mirror.zml.ai/plugins/202609161604.45.1.80cbf79212e9/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index d9908a98..ade601ee 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "da6e5e880b950d91cb036b3b68ebd7cd26a8e265b20ff58ea4b52dc242a365bd", - url = "https://mirror.zml.ai/plugins/202609161226.43.1.b992d049cec2/zml-metal-darwin-arm64.tar.zst", + sha256 = "aa67157ffd996e1a244a7a79c940cf15ac97919f0104617e70f523155d16388a", + url = "https://mirror.zml.ai/plugins/202609161604.45.1.80cbf79212e9/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index ae600a05..f1287726 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202609161226.43.1.b992d049cec2/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "2dfdacb5f58141d8c5e618a69794df6b7f4fa213e7748e1e891106f629467084" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202609161604.45.1.80cbf79212e9/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "11f8afa002ac7d26a640577e74f984f71aa4f1b9eb657c587b21c8348baf5f18" def _rocm_package_name(name): return name + _ROCM_VERSION diff --git a/zml/moe/fused_experts.zig b/zml/moe/fused_experts.zig index 9d57b478..026f4865 100644 --- a/zml/moe/fused_experts.zig +++ b/zml/moe/fused_experts.zig @@ -6,8 +6,9 @@ const zml = @import("../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; const DataType = zml.DataType; -const triton = @import("triton.zig"); const fly = @import("fly_kernels/moe.zig"); +const triton = @import("triton.zig"); +const callFusedMoe = triton.call; const log = std.log.scoped(.moe); @@ -79,12 +80,14 @@ pub fn fusedExperts(opts: FusedExpertsArgs, comptime backend: zml.moe.Backend) ! if (gate_up_scheme) |scheme| switch (scheme) { .nvfp4 => return error.UnsupportedQuantization, .fp8_block128 => launch_config.block_size_k = 128, + .fp8_block32 => launch_config.block_size_k = 32, .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, }; var down_launch_config = launchConfigForTokens(num_tokens); if (down_scheme) |scheme| switch (scheme) { .fp8_block128 => down_launch_config.block_size_k = 128, + .fp8_block32 => down_launch_config.block_size_k = 32, .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, .nvfp4 => return error.UnsupportedQuantization, }; @@ -266,8 +269,6 @@ pub const GemmOptions = struct { output_shape: Shape, }; -const callFusedMoe = triton.call; - test "FP8 routed GEMM with bias matches dequantized weights" { const allocator = std.testing.allocator; const io = std.testing.io; diff --git a/zml/moe/triton.zig b/zml/moe/triton.zig index df53311f..7af7f809 100644 --- a/zml/moe/triton.zig +++ b/zml/moe/triton.zig @@ -1,19 +1,20 @@ const std = @import("std"); + const stdx = @import("stdx"); + const zml = @import("../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; const DataType = zml.DataType; -const log = std.log.scoped(.moe); -const kernels = @import("triton_kernels/triton_kernels.zig"); const toDType = zml.kernel.triton.from; const shared = @import("fused_experts.zig"); - pub const Parameters = shared.Parameters; pub const GateUpLayout = shared.GateUpLayout; pub const RoutingWeightPlacement = shared.RoutingWeightPlacement; pub const FusedExpertsArgs = shared.FusedExpertsArgs; +const kernels = @import("triton_kernels/triton_kernels.zig"); +const log = std.log.scoped(.moe); pub fn call(opts: shared.GemmOptions) Tensor { // Native FP4 tensors expose logical K; the Triton operand is byte-packed. const weight = if (opts.quant_scheme == .mxfp4 and opts.weight.dtype() == .f4e2m1) @@ -27,6 +28,13 @@ pub fn call(opts: shared.GemmOptions) Tensor { stdx.debug.assert(opts.input.dtype() == .bf16 and opts.input_scale == null, "MXFP4 requires BF16 activations without input scales", .{}); } + if (opts.quant_scheme == .fp8_block32) { + const scales = opts.weight_scale orelse stdx.debug.panic("block32 FP8 requires weight scales", .{}); + stdx.debug.assert(weight.dtype() == .f8e4m3fn and @mod(weight.dim(1), 32) == 0 and @mod(weight_k, 32) == 0, "block32 FP8 requires E4M3FN weights with N and K divisible by 32", .{}); + stdx.debug.assert(scales.rank() == 3 and scales.dim(0) == weight.dim(0) and scales.dim(1) == @divExact(weight.dim(1), 32) and scales.dim(2) == @divExact(weight_k, 32), "block32 FP8 requires scales shaped [expert, N/32, K/32], got {f}", .{scales.shape()}); + stdx.debug.assert(scales.dtype() == .f8e8m0 or scales.dtype() == .bf16 or scales.dtype() == .f32, "unsupported block32 FP8 scale dtype {}", .{scales.dtype()}); + } + stdx.debug.assert(opts.quant_scheme != null or (opts.input_scale == null and opts.weight_scale == null), "scales require a quantization scheme", .{}); for ([_]i64{ weight.dim(1), weight.dim(2), opts.input.dim(1), weight.dim(1) * weight.dim(2), opts.output_shape.dim(-1) }) |dim_or_stride| { stdx.debug.assert(dim_or_stride > 0 and @mod(dim_or_stride, 16) == 0, "FusedMoe dimensions and matrix strides must be positive multiples of 16, got {}", .{dim_or_stride}); @@ -50,6 +58,8 @@ pub fn call(opts: shared.GemmOptions) Tensor { .mxfp4, .mxfp8 => scale.bitCast(.u8), .fp8_per_channel => scale.reshape(.{ weight.dim(0), weight.dim(1), 1 }), .fp8_per_tensor => if (scale.count() == 1) scale else scale.reshape(.{ weight.dim(0), 1, 1 }), + // Tiled FP8 uses ordinary dot products with FP32 scale arithmetic. + .fp8_block32 => scale.convert(.f32), .fp8_block128, .nvfp4 => scale, }; } else null; @@ -262,6 +272,7 @@ fn quantizeFp8Input(x: Tensor, scheme: zml.Quantization.Scheme, output_dtype: Da .mxfp8 => .{ 32, .u8, 1e-10 }, .fp8_per_channel, .fp8_per_tensor => .{ x.dim(1), .f32, 1e-10 }, .fp8_block128 => .{ 128, .f32, 1e-10 }, + .fp8_block32 => .{ 32, .f32, 1e-4 }, .mxfp4, .nvfp4 => unreachable, }; stdx.debug.assert(x.rank() == 2, "expected a rank-2 activation matrix, got {f}", .{x.shape()}); diff --git a/zml/moe/triton_kernels/triton_kernels.zig b/zml/moe/triton_kernels/triton_kernels.zig index cfcffa4c..2377929b 100644 --- a/zml/moe/triton_kernels/triton_kernels.zig +++ b/zml/moe/triton_kernels/triton_kernels.zig @@ -111,6 +111,17 @@ pub const PerTokenGroupQuantFp8 = struct { break :blk .{ b.exp2(exponent), exponent.add(127).to(.i32).to(.i8) }; }, .fp8_per_channel, .fp8_per_tensor, .fp8_block128 => .{ scale_raw, scale_raw.to(scale_dt) }, + .fp8_block32 => blk: { + // Match quantizeBlockFp8(..., .f8e8m0), but store the scales as FP32 here. + // It is unclear whether power-of-two scaling is best for this kernel, + // but it keeps activation quantization consistent with that function. + // Round up from the FP32 bits, including values just above a power of two. + const bits = b.bitcast(scale_raw, .i32); + const exponent = bits.div(0x800000); + const fractional = bits.bitAnd(0x7fffff).ne(0).to(.i32); + const scale = b.bitcast(exponent.add(fractional).mul(0x800000), .f32); + break :blk .{ scale, scale.to(scale_dt) }; + }, .mxfp4, .nvfp4 => unreachable, }; @@ -209,8 +220,9 @@ pub const FusedMoe = struct { .run = run, }); fn run(b: *Builder, cfg: Cfg) tri.FinishError!void { - if (cfg.quant_scheme == .fp8_block128 and cfg.block_size_k != 128) { - log.err("fused_moe_kernel: block-scaled FP8 requires BLOCK_SIZE_K=128", .{}); + const fp8_block_size: usize = if (cfg.quant_scheme == .fp8_block32) 32 else 128; + if ((cfg.quant_scheme == .fp8_block128 or cfg.quant_scheme == .fp8_block32) and cfg.block_size_k != fp8_block_size) { + log.err("fused_moe_kernel: block-scaled FP8 requires BLOCK_SIZE_K={}", .{fp8_block_size}); return error.InvalidMlir; } @@ -446,16 +458,16 @@ pub const FusedMoe = struct { .input_precision = .tf32, .max_num_imprecise_acc = 0, }), - .fp8_block128 => scaled: { - // Activation scales are [token, K/128] and weight scales are - // [expert, N/128, K/128]. Compute their addresses from the + .fp8_block32, .fp8_block128 => scaled: { + // Activation scales are [token, K/block] and weight scales are + // [expert, N/block, K/block]. Compute their addresses from the // loop index; activation scales are optional. const stride_bse = b.load(a.stride_bse_ptr); const stride_bsk = b.load(a.stride_bsk_ptr); const stride_bsn = b.load(a.stride_bsn_ptr); const b_scale_ptrs = a.b_scale_ptr.addPtr( off_experts.mul(stride_bse) - .add(offs_bn.div(128).mul(stride_bsn)) + .add(offs_bn.div(fp8_block_size).mul(stride_bsn)) .add(k_iter.mul(stride_bsk)), ); const dot = b.dotOpts(a_val, if (cfg.a_scale_dtype != null) b_val else b_val.to(.bf16), b.zeros(&.{ block_size_m, block_size_n }, .f32), .{ @@ -561,13 +573,13 @@ pub const FusedMoe = struct { }; test "FP8 activation quantization emits float and E8M0 scales" { - for ([_]zml.Quantization.Scheme{ .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block128 }) |scheme| { + for ([_]zml.Quantization.Scheme{ .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block128, .fp8_block32 }) |scheme| { for ([_]DType{ .bf16, .f32 }) |input_dtype| { const ir = try PerTokenGroupQuantFp8.Kernel.emit(std.testing.allocator, .{ .input_dtype = input_dtype, .output_dtype = .f8e4m3fn, .scale_dtype = if (scheme == .mxfp8) .i8 else .f32, - .block = if (scheme == .mxfp8) 32 else 256, + .block = if (scheme == .mxfp8 or scheme == .fp8_block32) 32 else 256, .quant_scheme = scheme, }); defer std.testing.allocator.free(ir); @@ -580,7 +592,7 @@ test "FP8 activation quantization emits float and E8M0 scales" { test "FusedMoe emits each FP8 and MXFP4 scaling path" { const allocator = std.testing.allocator; - const schemes = [_]?zml.Quantization.Scheme{ null, .mxfp4, .mxfp8, .fp8_per_tensor, .fp8_per_channel, .fp8_block128 }; + const schemes = [_]?zml.Quantization.Scheme{ null, .mxfp4, .mxfp8, .fp8_per_tensor, .fp8_per_channel, .fp8_block128, .fp8_block32 }; for (schemes) |scheme| { for ([_]DType{ .f8e4m3fn, .f8e4m3fnuz }) |fp8_dtype| { if ((scheme == .mxfp4 or scheme == .mxfp8) and fp8_dtype == .f8e4m3fnuz) continue; @@ -597,7 +609,7 @@ test "FusedMoe emits each FP8 and MXFP4 scaling path" { .routing_weights_dtype = .bf16, .block_size_m = 16, .block_size_n = 32, - .block_size_k = 128, + .block_size_k = if (scheme == .fp8_block32) 32 else 128, .group_size_m = 1, .top_k = 2, .naive_block_assignment = false, diff --git a/zml/nn.zig b/zml/nn.zig index 8d86f816..7c23f7ce 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -43,20 +43,19 @@ pub const Linear = struct { return if (self.quantization) |q| q.scales else null; } - pub fn forward(self: Linear, x: Tensor) Tensor { + pub fn forward(self: Linear, x: Tensor, out_dtype: DataType) Tensor { if (self.quantization) |q| { - const lhs = x.convert(.bf16); - if (quantization.quantizeInput(q, lhs, self.tag, zml.Compiler.current().platform)) |input| { - return self.forwardQuantized(input, x.dtype()); + if (quantization.quantizeInput(q, x, self.tag, zml.Compiler.current().platform)) |input| { + return self.forwardQuantized(input, out_dtype); } - return self.forwardScaled(lhs, null, null, x.dtype()); + return self.forwardScaled(x.convert(.bf16), null, null, out_dtype); } - const y = x.dot(self.weight, self.tag); - return if (self.bias) |bias| y.add(bias.broad(y.shape())) else y; + const y = x.dot(self.weight.convert(x.dtype()), self.tag).convert(out_dtype); + return if (self.bias) |bias| y.add(bias.convert(y.dtype()).broad(y.shape())) else y; } /// Apply this layer to reusable quantized activation values and scales. - /// Convert to output_dtype before adding bias, which must have the same dtype. + /// Convert to output_dtype before adding bias. pub fn forwardQuantized(self: Linear, input: quantization.QuantizedInput, output_dtype: DataType) Tensor { stdx.debug.assert(self.quantization != null, "forwardQuantized requires quantized weights", .{}); return self.forwardScaled(input.values, input.scales, input.global_scale, output_dtype); @@ -67,9 +66,9 @@ pub const Linear = struct { const weight_global_scale: ?Tensor = if (q.global_scale) |s| s.asMultiplier() else null; const weight = if (isPackedFp4(q.scheme, self.weight.dtype())) unpackFp4(self.weight, self.tag, self.tag) else self.weight; const scales = if (q.scheme.isMx() and q.scales.dtype() == .u8) q.scales.bitCast(.f8e8m0) else q.scales; - const acc = scaledDot(lhs, weight, lhs_scale, scales, self.tag); + const acc = scaledDot(lhs, weight, lhs_scale, scales, output_dtype, self.tag); const y = applyGlobalScale(acc, input_global_scale, weight_global_scale).convert(output_dtype); - return if (self.bias) |bias| y.add(bias.broad(y.shape())) else y; + return if (self.bias) |bias| y.add(bias.convert(y.dtype()).broad(y.shape())) else y; } }; @@ -120,22 +119,28 @@ test "unpackFp4 expands the requested axis" { /// dimension must divide its corresponding value dimension. /// /// CPU has no specialized path. +/// +/// `out_dtype` is the element type of the returned tensor; only BF16 and F32 stay on +/// the fused path. It buys no accuracy: the kernels accumulate in F32 either way. pub fn scaledDot( lhs: Tensor, rhs: Tensor, lhs_scale: ?Tensor, rhs_scale: Tensor, + out_dtype: DataType, args: anytype, ) Tensor { + // A narrow output would be an unscaled cast of the F32 accumulator + stdx.debug.assert( + out_dtype.isFloat() and out_dtype.bitSizeOf() >= 16, + "scaledDot output dtype must be a 16-bit-or-wider float, got {s}", + .{@tagName(out_dtype)}, + ); const dot_axes = lhs.dotAxes(rhs, args); const Axes = stdx.BoundedArray(i64, constants.MAX_RANK); - const result_dtype: DataType = switch (lhs.dtype()) { - .f4e2m1, .f8e4m3, .f8e4m3fn, .f8e5m2, .f8e4m3b11fnuz, .f8e4m3fnuz, .f8e5m2fnuz, .f8e8m0 => .bf16, - else => lhs.dtype(), - }; - var res_shape: Shape = .{ ._dtype = result_dtype }; + var res_shape: Shape = .{ ._dtype = out_dtype }; var lhs_batching_axes: Axes = .empty; var rhs_batching_axes: Axes = .empty; for (dot_axes.batching.constSlice()) |b_axes| { @@ -224,15 +229,15 @@ test "block128 scaled dot layouts" { fn forward(x: Tensor, w: Tensor, scales: Tensor, fp8: DataType, prequantized: bool) Outputs { const weight = w.convert(fp8); - const input = quantization.quantizeBlockFp8(x, .k, fp8); + const input = quantization.quantizeBlockFp8(x, .k, 128, fp8, .f32); const linear: Linear = .{ .weight = weight, .tag = Shape.toTag(.k), .quantization = .{ .scheme = .fp8_block128, .scales = scales } }; return .{ - .linear = linear.forward(x), + .linear = linear.forward(x, .bf16), .linear_expected = dequantize(input.values, input.scales).dot(dequantize(weight, scales), .k), .actual = if (prequantized) - scaledDot(input.values, weight, input.scales.convert(scales.dtype()), scales, .k) + scaledDot(input.values, weight, input.scales.convert(scales.dtype()), scales, .bf16, .k) else - scaledDot(x, weight, null, scales, .k), + scaledDot(x, weight, null, scales, .bf16, .k), .expected = (if (prequantized) dequantize(input.values, input.scales.convert(scales.dtype())) else x) .dot(dequantize(weight, scales), .k), }; diff --git a/zml/quantization.zig b/zml/quantization.zig index abcdb5ee..fb95b9f6 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -51,6 +51,8 @@ pub const Quantization = struct { /// f8e4m3fn, f8e4m3fnuz, or f8e8m0 values, one bf16 or f32 scale per 128x128 tile. /// The DeepSeek-style FP8 that model vendors publish themselves, under `weight_scale_inv`. fp8_block128, + /// E4M3FN weights with E8M0, BF16 or F32 scales per 32x32 tile. + fp8_block32, /// f8e4m3fn values, one scale for the whole tensor. Spelled `[1, 1]` rather than as a /// scalar: XLA's composite rewriter requires the scale to have the operand's rank. fp8_per_tensor, @@ -75,6 +77,9 @@ pub const Quantization = struct { (scale.dtype() == .bf16 or scale.dtype() == .f32) and scale.count() > 1 and scale.rank() == 2 and scale.dim(0) == n and scale.dim(1) == 1, + .fp8_block32 => weight.dtype() == .f8e4m3fn and + (scale.dtype() == .f8e8m0 or ((scale.dtype() == .bf16 or scale.dtype() == .f32) and scale.count() > 1)) and + scale.rank() == 2 and scale.dim(0) * 32 == n and scale.dim(1) * 32 == k, .fp8_block128 => (weight.dtype() == .f8e4m3fn or weight.dtype() == .f8e4m3fnuz or weight.dtype() == .f8e8m0) and (scale.dtype() == .bf16 or scale.dtype() == .f32) and scale.count() > 1 and scale.rank() == 2 and @@ -106,40 +111,61 @@ pub const QuantizedInput = struct { pub fn quantizeInput(quantization: Quantization, input: Tensor, axis: Shape.Tag, platform: *const Platform) ?QuantizedInput { const global_scale: ?Tensor = if (quantization.input_scale) |scale| scale.asMultiplier() else null; return switch (quantization.scheme) { - .nvfp4 => if (supportsNvfp4InputQuantization(platform)) quantizeNvfp4(input, global_scale, axis) else null, - .fp8_block128 => switch (platform.target) { - .cuda => quantizeBlockFp8(input, axis, .f8e4m3fn), - .rocm => if (platform_mod.rocm.computeCapability(platform)) |capability| switch (capability.architecture()) { - .cdna4, .rdna4 => quantizeBlockFp8(input, axis, .f8e4m3fn), - .cdna3 => quantizeBlockFp8(input, axis, .f8e4m3fnuz), - .cdna1, .cdna2, .rdna2, .rdna3, .rdna3_5 => null, - } else null, - else => null, + .nvfp4 => if (supportsNvfp4InputQuantization(platform)) quantizeNvfp4(input.convert(.bf16), global_scale, axis) else null, + .fp8_block32, .fp8_block128 => blk: { + const dtype: DataType = switch (platform.target) { + .cuda => .f8e4m3fn, + .rocm => if (platform_mod.rocm.computeCapability(platform)) |capability| switch (capability.architecture()) { + .cdna4, .rdna4 => .f8e4m3fn, + .cdna3 => .f8e4m3fnuz, + .cdna1, .cdna2, .rdna2, .rdna3, .rdna3_5 => return null, + } else return null, + else => return null, + }; + break :blk switch (quantization.scheme) { + .fp8_block32 => quantizeBlockFp8(input, axis, 32, dtype, .f8e8m0), + .fp8_block128 => quantizeBlockFp8(input.convert(.bf16), axis, 128, dtype, .f32), + else => unreachable, + }; }, .mxfp8, .mxfp4, .fp8_per_channel, .fp8_per_tensor => null, }; } -/// Quantize each 128-value activation block, preserving the input's axes. -/// Scales have the same shape with the contracting dimension divided by 128. -pub fn quantizeBlockFp8(x: Tensor, axis: anytype, dtype: DataType) QuantizedInput { - stdx.debug.assert(dtype == .f8e4m3fn or dtype == .f8e4m3fnuz, "expected E4M3 FP8 dtype, got {s}", .{@tagName(dtype)}); - stdx.debug.assert(@mod(x.dim(axis), 128) == 0, "block FP8 activation width must be divisible by 128, got {f}", .{x.shape()}); +/// Quantize activation blocks, preserving the input's axes. +/// E8M0 scales round upward to a power of two to avoid overflowing FP8 values. +pub fn quantizeBlockFp8(x: Tensor, axis: anytype, block_size: i64, dtype: DataType, scale_dtype: DataType) QuantizedInput { + stdx.debug.assert(block_size > 0, "block FP8 size must be positive, got {}", .{block_size}); + stdx.debug.assert(@mod(x.dim(axis), block_size) == 0, "block FP8 activation width must be divisible by {}, got {f}", .{ block_size, x.shape() }); // Match the routed-MoE quantizer's FNUZ range. - const fp8_max: f32 = if (dtype == .f8e4m3fnuz) 224.0 else 448.0; - const grouped = x.convert(.f32).splitAxis(axis, .{ .fp8_ks = -1, .fp8_block = 128 }); - // Put scale arithmetic before the reduction so it can fuse into the producer. - const scales = grouped.abs() - .maximum(.scalar(1e-10, .f32)) - .scale(1.0 / fp8_max) - .max(.fp8_block); + const fp8_max: f32 = switch (dtype) { + .f8e4m3fn => 448.0, + .f8e4m3fnuz => 224.0, + else => stdx.debug.panic("expected E4M3 FP8 dtype, got {s}", .{@tagName(dtype)}), + }; + const grouped = x.convert(.f32).splitAxis(axis, .{ .fp8_ks = -1, .fp8_block = block_size }); + const scales = switch (scale_dtype) { + // Put scale arithmetic before the reduction so it can fuse into the producer. + .f32 => grouped.abs() + .maximum(.scalar(1e-10, .f32)) + .scale(1.0 / fp8_max) + .max(.fp8_block), + .f8e8m0 => blk: { + const raw_scale = grouped.abs().maximum(.scalar(1e-4, .f32)).scale(1.0 / fp8_max).max(.fp8_block); + const bits = raw_scale.bitCast(.u32); + const exponent = bits.shiftRightLogical(.scalar(23, .u32)); + const fractional = bits.logical(.AND, .scalar(0x7fffff, .u32)).cmp(.NE, .scalar(0, .u32)).convert(.u32); + break :blk exponent.add(fractional).shiftLeft(.scalar(23, .u32)).bitCast(.f32); + }, + else => stdx.debug.panic("expected F32 or E8M0 scale dtype, got {s}", .{@tagName(scale_dtype)}), + }; const values = grouped.div(scales.broad(grouped.shape())) .clamp(.scalar(-fp8_max, .f32), .scalar(fp8_max, .f32)) .convert(dtype) .reshape(x.shape().withDtype(dtype)); return .{ .values = values, - .scales = scales.reshape(x.shape().setDim(axis, @divExact(x.dim(axis), 128)).withDtype(.f32)), + .scales = scales.reshape(x.shape().setDim(axis, @divExact(x.dim(axis), block_size)).withDtype(.f32)).convert(scale_dtype), .global_scale = null, }; } @@ -335,7 +361,7 @@ test "block FP8 quantization preserves axes and reconstructs constant blocks in const Local = struct { const Outputs = struct { values: Tensor, scales: Tensor }; fn forward(x: Tensor) Outputs { - const input = quantizeBlockFp8(x, .k, dtype); + const input = quantizeBlockFp8(x, .k, 128, dtype, .f32); return .{ .values = input.values.convert(.f32), .scales = input.scales }; } }; From df3a43d29fd81ccbee05722111ccd2d50e42565e Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Wed, 16 Sep 2026 21:50:22 +0200 Subject: [PATCH 692/881] bazel: add zig_options rule (#820) Introduce `zig_option` and `zig_options` rules to generate the bazel equivalent of option modules plugged on bazel flags. On `string` flags, if the flag has a `allowed_values`, the `string` is converted into a `enum` in zig code. Also, port platforms to this and remove the ugly `defines` hack to comptime test for platforms enablement. In the process, port `arocc` to the `zig_options`. --- bazel/zig_options.bzl | 205 ++++++++++++++++++++++++++++++++++ platforms/BUILD.bazel | 13 ++- platforms/cpu/BUILD.bazel | 14 +-- platforms/cpu/cpu.zig | 4 +- platforms/cuda/BUILD.bazel | 16 +-- platforms/cuda/cuda.zig | 3 +- platforms/metal/BUILD.bazel | 17 +-- platforms/metal/metal.h | 1 + platforms/metal/metal.zig | 3 +- platforms/neuron/BUILD.bazel | 19 +--- platforms/neuron/neuron.zig | 3 +- platforms/oneapi/BUILD.bazel | 10 +- platforms/oneapi/oneapi.h | 1 + platforms/oneapi/oneapi.zig | 3 +- platforms/platforms.zig | 7 +- platforms/rocm/BUILD.bazel | 11 +- platforms/rocm/rocm.h | 1 + platforms/rocm/rocm.zig | 3 +- platforms/tpu/BUILD.bazel | 14 +-- platforms/tpu/tpu.zig | 4 +- third_party/arocc/arocc.bazel | 95 +++++++++++----- 21 files changed, 333 insertions(+), 114 deletions(-) create mode 100644 bazel/zig_options.bzl create mode 100644 platforms/metal/metal.h create mode 100644 platforms/oneapi/oneapi.h create mode 100644 platforms/rocm/rocm.h diff --git a/bazel/zig_options.bzl b/bazel/zig_options.bzl new file mode 100644 index 00000000..76d09a19 --- /dev/null +++ b/bazel/zig_options.bzl @@ -0,0 +1,205 @@ +"""Build settings for Zig options.""" + +load("@bazel_skylib//lib:types.bzl", "types") +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load("@rules_zig//zig:defs.bzl", "zig_library") + +ZigOption = provider( + doc = "A Zig option's configured value and allowed string values.", + fields = { + "module_name": "The Bazel module declaring the flag, or empty when unavailable.", + "value": "The configured build setting value.", + "type": "The Zig integer type, or None for other flag types.", + "allowed_values": "Allowed values for string flags; an empty list allows any string. None for other flag types.", + "enum_name": "Optional exported enum type name for a constrained string flag.", + "nullable": "Whether an unconstrained string emits null for an empty value.", + }, +) + +_MAKE_VARIABLE_ATTR = attr.string( + doc = "Optional Make variable name exposed to rules that list this flag in toolchains.", +) + +_MODULE_NAME_ATTR = attr.string( + doc = "Declaring module name, captured by zig_option during package evaluation.", +) + +_SCOPE_ATTR = attr.string( + doc = "The scope indicates where a flag can propagate to", + default = "target", +) + +def _flag_impl(ctx): + value = ctx.build_setting_value + providers = [ + BuildSettingInfo(value = value), + ZigOption( + module_name = ctx.attr.module_name, + value = value, + type = getattr(ctx.attr, "type", None), + allowed_values = getattr(ctx.attr, "values", None), + enum_name = getattr(ctx.attr, "enum_name", ""), + nullable = getattr(ctx.attr, "nullable", False), + ), + ] + if ctx.attr.make_variable: + providers.append(platform_common.TemplateVariableInfo({ + ctx.attr.make_variable: str(value).lower() if type(value) == "bool" else str(value), + })) + return providers + +def _string_impl(ctx): + if ctx.attr.nullable and ctx.attr.values: + fail("nullable strings cannot have enum values") + if ctx.attr.enum_name and not ctx.attr.values: + fail("enum_name requires nonempty values") + if ctx.attr.values and ctx.build_setting_value not in ctx.attr.values: + fail("{}: invalid value '{}'; expected one of {}".format( + ctx.label, + ctx.build_setting_value, + ctx.attr.values, + )) + return _flag_impl(ctx) + +def _zig_string(value): + # JSON and Zig differ in their control-character and Unicode escapes. + escaped = [] + for char in value.elems(): + part = json.encode(char)[1:-1] + if part.startswith("\\u"): + part = "\\u{" + part[2:] + "}" + escaped.append(part.replace("\\b", "\\x08").replace("\\f", "\\x0c")) + return '"' + "".join(escaped) + '"' + +def _zig_options_impl(ctx): + lines = [] + names = {} + for setting in ctx.attr.settings: + option = setting[ZigOption] + name = setting.label.name + if name in names: + fail("Duplicate Zig declaration: {}".format(name)) + names[name] = True + value = option.value + if types.is_bool(value): + zig_type = "bool" + literal = "true" if value else "false" + elif types.is_int(value): + zig_type = option.type + literal = str(value) + elif option.allowed_values: + zig_type = "enum { " + ", ".join(["@" + _zig_string(v) for v in option.allowed_values]) + " }" + if option.enum_name: + if option.enum_name in names: + fail("Duplicate Zig declaration: {}".format(option.enum_name)) + names[option.enum_name] = True + lines.append("pub const @{} = {};".format(_zig_string(option.enum_name), zig_type)) + zig_type = "@" + _zig_string(option.enum_name) + literal = ".@" + _zig_string(value) + else: + zig_type = "?[]const u8" if option.nullable else "[]const u8" + literal = "null" if option.nullable and not value else _zig_string(value) + lines.append("pub const @{}: {} = {};".format(_zig_string(name), zig_type, literal)) + ctx.actions.write(ctx.outputs.out, "\n".join(lines) + "\n") + return [DefaultInfo(files = depset([ctx.outputs.out]))] + +_zig_options_source = rule( + implementation = _zig_options_impl, + attrs = { + "out": attr.output(mandatory = True), + "settings": attr.label_list( + mandatory = True, + allow_files = False, + providers = [ZigOption], + ), + }, +) + +def zig_options(name, settings, **kwargs): + """Generate an options module and expose it as a zig_library. + + Args: + name: Library target name; the generated source is .zig. + settings: Option flags; target names become quoted Zig constant names. + **kwargs: Attributes forwarded to zig_library, except main which is generated. + """ + if "main" in kwargs: + fail("zig_options generates its own main source") + source_attrs = {key: kwargs[key] for key in ["testonly", "tags", "compatible_with", "restricted_to", "target_compatible_with"] if key in kwargs} + _zig_options_source( + name = name + "_source", + settings = settings, + out = name + ".zig", + visibility = ["//visibility:private"], + **source_attrs + ) + zig_library(name = name, main = ":" + name + "_source", **kwargs) + +def zig_option(name, build_setting_default, type = None, values = None, enum_name = None, **kwargs): + """Define a command-line option, inferring its kind from the default. + + Args: + name: Flag target name and generated Zig constant name. + build_setting_default: Integer, boolean, or string default value. + type: Zig integer type; defaults to usize. Only valid for integers. + values: Allowed string values, emitted as enum fields when nonempty. + enum_name: Optional exported enum type name; requires a string with nonempty values. + **kwargs: Attributes forwarded to the underlying flag rule. + """ + if type != None and not types.is_int(build_setting_default): + fail("type is only supported for integer options") + if values != None and not types.is_string(build_setting_default): + fail("values is only supported for string options") + if enum_name != None and (not types.is_string(build_setting_default) or not values): + fail("enum_name requires a string option with nonempty values") + if types.is_bool(build_setting_default): + flag = bool_flag + elif types.is_int(build_setting_default): + flag = int_flag + kwargs["type"] = type if type != None else "usize" + elif types.is_string(build_setting_default): + flag = string_flag + kwargs["values"] = values if values != None else [] + kwargs["enum_name"] = enum_name if enum_name != None else "" + else: + fail("Zig options require an integer, boolean, or string default") + flag(name = name, build_setting_default = build_setting_default, module_name = native.module_name() or "", **kwargs) + +int_flag = rule( + implementation = _flag_impl, + build_setting = config.int(flag = True), + attrs = { + "type": attr.string(default = "usize", doc = "Zig integer type, such as usize, u32, or i64."), + "module_name": _MODULE_NAME_ATTR, + "make_variable": _MAKE_VARIABLE_ATTR, + "scope": _SCOPE_ATTR, + }, + doc = "An integer build setting that can be set on the command line", +) + +bool_flag = rule( + implementation = _flag_impl, + build_setting = config.bool(flag = True), + attrs = { + "module_name": _MODULE_NAME_ATTR, + "make_variable": _MAKE_VARIABLE_ATTR, + "scope": _SCOPE_ATTR, + }, + doc = "A boolean build setting that can be set on the command line", +) + +string_flag = rule( + implementation = _string_impl, + build_setting = config.string(flag = True), + attrs = { + "nullable": attr.bool(doc = "Emit an optional string; an empty value becomes null. Cannot be combined with values."), + "enum_name": attr.string(doc = "Optional exported Zig enum type name; requires nonempty values."), + "module_name": _MODULE_NAME_ATTR, + "values": attr.string_list( + doc = "The list of allowed values for this setting. An error is raised if any other value is given.", + ), + "make_variable": _MAKE_VARIABLE_ATTR, + "scope": _SCOPE_ATTR, + }, + doc = "A string-typed build setting that can be set on the command line", +) diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index 19900cd4..a454d361 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -1,5 +1,5 @@ -load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") load("@rules_zig//zig:defs.bzl", "zig_library") +load("//bazel:zig_options.bzl", "zig_option", "zig_options") PLATFORMS = { "cpu": True, @@ -12,13 +12,21 @@ PLATFORMS = { } [ - bool_flag( + zig_option( name = runtime, build_setting_default = default, + scope = "universal", ) for runtime, default in PLATFORMS.items() ] +zig_options( + name = "options", + import_name = "platforms/options", + settings = [":" + runtime for runtime in PLATFORMS], + visibility = ["//platforms:__subpackages__"], +) + config_setting( name = "rocm.enabled", constraint_values = [ @@ -60,6 +68,7 @@ zig_library( tags = ["manual"], visibility = ["//visibility:public"], deps = [ + ":options", "//pjrt", ] + [ "//platforms/{}".format(runtime) diff --git a/platforms/cpu/BUILD.bazel b/platforms/cpu/BUILD.bazel index c75db16d..b592d3bb 100644 --- a/platforms/cpu/BUILD.bazel +++ b/platforms/cpu/BUILD.bazel @@ -18,12 +18,6 @@ config_setting( ], ) -cc_library( - name = "empty", - tags = ["manual"], - defines = ["ZML_RUNTIME_CPU_DISABLED"], -) - copy_to_directory( name = "sandbox", srcs = select({ @@ -37,19 +31,19 @@ copy_to_directory( cc_library( name = "libzml_cpu", - tags = ["manual"], data = [":sandbox"], - defines = ["ZML_RUNTIME_CPU"], + tags = ["manual"], ) zig_library( name = "cpu", - tags = ["manual"], import_name = "platforms/cpu", main = "cpu.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ "//pjrt", + "//platforms:options", ] + select({ "//platforms:cpu.enabled": [ ":libzml_cpu", @@ -57,6 +51,6 @@ zig_library( "//stdx", "@rules_zig//zig/runfiles", ], - "//conditions:default": [":empty"], + "//conditions:default": [], }), ) diff --git a/platforms/cpu/cpu.zig b/platforms/cpu/cpu.zig index 25d8bdd3..de4c7bd2 100644 --- a/platforms/cpu/cpu.zig +++ b/platforms/cpu/cpu.zig @@ -3,15 +3,15 @@ const builtin = @import("builtin"); const bazel = @import("bazel"); const bazel_builtin = @import("bazel_builtin"); -const c = @import("c"); const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); const runfiles = @import("runfiles"); const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/platforms/cpu"); pub fn isEnabled() bool { - return @hasDecl(c, "ZML_RUNTIME_CPU"); + return platforms_options.cpu; } pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { diff --git a/platforms/cuda/BUILD.bazel b/platforms/cuda/BUILD.bazel index 437f945b..b337cc9a 100644 --- a/platforms/cuda/BUILD.bazel +++ b/platforms/cuda/BUILD.bazel @@ -1,5 +1,5 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_shared_library", "zig_binary"); +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_shared_library") zig_binary( name = "compat_probe", @@ -18,12 +18,6 @@ zig_shared_library( deps = ["//stdx"], ) -cc_library( - name = "empty", - tags = ["manual"], - defines = ["ZML_RUNTIME_CUDA_DISABLED"], -) - cc_library( name = "libzml_cuda_repo", deps = select({ @@ -43,21 +37,21 @@ cc_library( cc_library( name = "libzml_cuda", - tags = ["manual"], hdrs = ["libzml_cuda.h"], - defines = ["ZML_RUNTIME_CUDA"], + tags = ["manual"], deps = [":libzml_cuda_repo"], ) zig_library( name = "cuda", - tags = ["manual"], srcs = ["compat_probe.zig"], import_name = "platforms/cuda", main = "cuda.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ "//pjrt", + "//platforms:options", ] + select({ "//platforms:cuda.enabled": [ ":libzml_cuda", @@ -65,6 +59,6 @@ zig_library( "//stdx", "@rules_zig//zig/runfiles", ], - "//conditions:default": [":empty"], + "//conditions:default": [], }), ) diff --git a/platforms/cuda/cuda.zig b/platforms/cuda/cuda.zig index 6400290e..8f1a03a8 100644 --- a/platforms/cuda/cuda.zig +++ b/platforms/cuda/cuda.zig @@ -5,6 +5,7 @@ const bazel = @import("bazel"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); const stdx = @import("stdx"); const compat_probe = @import("compat_probe.zig"); @@ -26,7 +27,7 @@ fn findCudaSandbox( } pub fn isEnabled() bool { - return @hasDecl(c, "ZML_RUNTIME_CUDA"); + return platforms_options.cuda; } pub fn needsCudaCompat(io: std.Io, sandbox_path: []const u8) !bool { diff --git a/platforms/metal/BUILD.bazel b/platforms/metal/BUILD.bazel index c4a6c018..c6f1f3c8 100644 --- a/platforms/metal/BUILD.bazel +++ b/platforms/metal/BUILD.bazel @@ -2,31 +2,25 @@ load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library") -cc_library( - name = "empty", - defines = ["ZML_RUNTIME_METAL_DISABLED"], - tags = ["manual"], -) - copy_to_directory( name = "metal_toolchain", srcs = [ - "@metal_toolchain//:metal_compile_tools", "@metal_toolchain//:metal_compile_runtime", + "@metal_toolchain//:metal_compile_tools", ], out = "metal_toolchain", + exec_compatible_with = ["@platforms//os:macos"], + include_external_repositories = ["**"], replace_prefixes = { "Metal.xctoolchain/usr/metal/32023": ".", }, - include_external_repositories = ["**"], - exec_compatible_with = ["@platforms//os:macos"], target_compatible_with = ["@platforms//os:macos"], ) cc_library( name = "libzml_metal", + hdrs = ["metal.h"], data = [":metal_toolchain"], - defines = ["ZML_RUNTIME_METAL"], tags = ["manual"], deps = ["@libzml_metal"], ) @@ -39,6 +33,7 @@ zig_library( visibility = ["//visibility:public"], deps = [ "//pjrt", + "//platforms:options", ] + select({ "//platforms:metal.enabled": [ ":libzml_metal", @@ -46,6 +41,6 @@ zig_library( "//stdx", "@rules_zig//zig/runfiles", ], - "//conditions:default": [":empty"], + "//conditions:default": [], }), ) diff --git a/platforms/metal/metal.h b/platforms/metal/metal.h new file mode 100644 index 00000000..933e4f31 --- /dev/null +++ b/platforms/metal/metal.h @@ -0,0 +1 @@ + #include diff --git a/platforms/metal/metal.zig b/platforms/metal/metal.zig index 6da9be29..e3052168 100644 --- a/platforms/metal/metal.zig +++ b/platforms/metal/metal.zig @@ -5,12 +5,13 @@ const bazel = @import("bazel"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/platforms/metal"); pub fn isEnabled() bool { - return @hasDecl(c, "ZML_RUNTIME_METAL"); + return platforms_options.metal; } fn setMetalToolchainEnv(r: anytype) !void { diff --git a/platforms/neuron/BUILD.bazel b/platforms/neuron/BUILD.bazel index fada3d5c..d427fd47 100644 --- a/platforms/neuron/BUILD.bazel +++ b/platforms/neuron/BUILD.bazel @@ -24,9 +24,9 @@ zig_shared_library( deps = [ ":libnrt_headers", ":neuron_nrt", - ":zmlxneuron", "//bazel", "//pjrt", + "//platforms:options", "//stdx", "@rules_zig//zig/runfiles", "@xla//xla/pjrt/c:pjrt_c_api_hdrs", @@ -71,18 +71,6 @@ cc_library( ], ) -cc_library( - name = "empty", - defines = ["ZML_RUNTIME_NEURON_DISABLED"], - tags = ["manual"], -) - -cc_library( - name = "zmlxneuron", - defines = ["ZML_RUNTIME_NEURON"], - tags = ["manual"], -) - cc_library( name = "libnrt_headers", hdrs = ["nrt.h"], @@ -169,17 +157,17 @@ zig_library( visibility = ["//visibility:public"], deps = [ "//pjrt", + "//platforms:options", ] + select({ "//platforms:neuron.enabled": [ ":libnrt_headers", ":neuron_nrt", - ":zmlxneuron", "//bazel", "//stdx", "@libzml_neuron", "@rules_zig//zig/runfiles", ], - "//conditions:default": [":empty"], + "//conditions:default": [], }), ) @@ -192,6 +180,5 @@ zig_library( visibility = ["//visibility:public"], deps = [ ":neuron", - ":zmlxneuron", ], ) diff --git a/platforms/neuron/neuron.zig b/platforms/neuron/neuron.zig index f3df8909..995a97f8 100644 --- a/platforms/neuron/neuron.zig +++ b/platforms/neuron/neuron.zig @@ -5,6 +5,7 @@ const bazel = @import("bazel"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/platforms/neuron"); @@ -95,7 +96,7 @@ pub fn instance() !Instance { } pub fn isEnabled() bool { - return @hasDecl(c, "ZML_RUNTIME_NEURON"); + return platforms_options.neuron; } fn hasNeuronDevice(io: std.Io) bool { diff --git a/platforms/oneapi/BUILD.bazel b/platforms/oneapi/BUILD.bazel index 7193eb2a..c4e8eb32 100644 --- a/platforms/oneapi/BUILD.bazel +++ b/platforms/oneapi/BUILD.bazel @@ -11,15 +11,8 @@ zig_shared_library( deps = ["//stdx"], ) -cc_library( - name = "empty", - defines = ["ZML_RUNTIME_ONEAPI_DISABLED"], - tags = ["manual"], -) - cc_library( name = "libzml_oneapi", - defines = ["ZML_RUNTIME_ONEAPI"], tags = ["manual"], deps = ["@libzml_oneapi"], ) @@ -33,12 +26,13 @@ zig_library( deps = [ "//bazel", "//pjrt", + "//platforms:options", "//stdx", "@rules_zig//zig/runfiles", ] + select({ "//platforms:oneapi.enabled": [ ":libzml_oneapi", ], - "//conditions:default": [":empty"], + "//conditions:default": [], }), ) diff --git a/platforms/oneapi/oneapi.h b/platforms/oneapi/oneapi.h new file mode 100644 index 00000000..933e4f31 --- /dev/null +++ b/platforms/oneapi/oneapi.h @@ -0,0 +1 @@ + #include diff --git a/platforms/oneapi/oneapi.zig b/platforms/oneapi/oneapi.zig index 628c9f5c..2a2262b7 100644 --- a/platforms/oneapi/oneapi.zig +++ b/platforms/oneapi/oneapi.zig @@ -5,12 +5,13 @@ const bazel = @import("bazel"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/platforms/oneapi"); pub fn isEnabled() bool { - return @hasDecl(c, "ZML_RUNTIME_ONEAPI"); + return platforms_options.oneapi; } fn hasOneApiDevice(io: std.Io) bool { diff --git a/platforms/platforms.zig b/platforms/platforms.zig index 38093ce7..1e347a5c 100644 --- a/platforms/platforms.zig +++ b/platforms/platforms.zig @@ -4,9 +4,10 @@ const cpu = @import("platforms/cpu"); const cuda = @import("platforms/cuda"); const metal = @import("platforms/metal"); const neuron = @import("platforms/neuron"); -const pjrt = @import("pjrt"); -const rocm = @import("platforms/rocm"); const oneapi = @import("platforms/oneapi"); +const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); +const rocm = @import("platforms/rocm"); const tpu = @import("platforms/tpu"); const platforms = @This(); @@ -32,7 +33,7 @@ pub const Platform = enum { pub fn isEnabled(target: Platform) bool { return switch (target) { - inline else => |tag| @field(platforms, @tagName(tag)).isEnabled(), + inline else => |tag| @field(platforms_options, @tagName(tag)), }; } diff --git a/platforms/rocm/BUILD.bazel b/platforms/rocm/BUILD.bazel index b00be3f3..91651372 100644 --- a/platforms/rocm/BUILD.bazel +++ b/platforms/rocm/BUILD.bazel @@ -21,15 +21,9 @@ zig_shared_library( deps = ["//stdx"], ) -cc_library( - name = "empty", - defines = ["ZML_RUNTIME_ROCM_DISABLED"], - tags = ["manual"], -) - cc_library( name = "libzml_rocm", - defines = ["ZML_RUNTIME_ROCM"], + hdrs = ["rocm.h"], tags = ["manual"], deps = ["@libzml_rocm"], ) @@ -42,6 +36,7 @@ zig_library( visibility = ["//visibility:public"], deps = [ "//pjrt", + "//platforms:options", ] + select({ "//platforms:rocm.enabled": [ ":libzml_rocm", @@ -49,6 +44,6 @@ zig_library( "//stdx", "@rules_zig//zig/runfiles", ], - "//conditions:default": [":empty"], + "//conditions:default": [], }), ) diff --git a/platforms/rocm/rocm.h b/platforms/rocm/rocm.h new file mode 100644 index 00000000..933e4f31 --- /dev/null +++ b/platforms/rocm/rocm.h @@ -0,0 +1 @@ + #include diff --git a/platforms/rocm/rocm.zig b/platforms/rocm/rocm.zig index c3897dd1..4f054caa 100644 --- a/platforms/rocm/rocm.zig +++ b/platforms/rocm/rocm.zig @@ -5,13 +5,14 @@ const bazel = @import("bazel"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); const runfiles = @import("runfiles"); const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/platforms/rocm"); pub fn isEnabled() bool { - return @hasDecl(c, "ZML_RUNTIME_ROCM"); + return platforms_options.rocm; } fn hasRocmDevices(io: std.Io) bool { diff --git a/platforms/tpu/BUILD.bazel b/platforms/tpu/BUILD.bazel index 6b872aec..f3063c7a 100644 --- a/platforms/tpu/BUILD.bazel +++ b/platforms/tpu/BUILD.bazel @@ -1,42 +1,36 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library") -cc_library( - name = "empty", - tags = ["manual"], - defines = ["ZML_RUNTIME_TPU_DISABLED"], -) - cc_library( name = "libzml_tpu", tags = ["manual"], - defines = ["ZML_RUNTIME_TPU"], deps = ["@libzml_tpu"], ) zig_library( name = "tpu", - tags = ["manual"], import_name = "platforms/tpu", main = "tpu.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ "//bazel", "//pjrt", + "//platforms:options", "//stdx", ] + select({ "//platforms:tpu.enabled": [ ":libzml_tpu", ], - "//conditions:default": [":empty"], + "//conditions:default": [], }), ) zig_library( name = "ragged_paged", - tags = ["manual"], import_name = "platforms/tpu/ragged_paged", main = "ragged_paged.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ "//kernels/mosaic_tpu:mosaic_tpu_builder", diff --git a/platforms/tpu/tpu.zig b/platforms/tpu/tpu.zig index b385bb5c..bf859e8a 100644 --- a/platforms/tpu/tpu.zig +++ b/platforms/tpu/tpu.zig @@ -3,14 +3,14 @@ const builtin = @import("builtin"); const bazel = @import("bazel"); const bazel_builtin = @import("bazel_builtin"); -const c = @import("c"); const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/platforms/tpu"); pub fn isEnabled() bool { - return @hasDecl(c, "ZML_RUNTIME_TPU"); + return platforms_options.tpu; } /// Check if running on Google Compute Engine, because TPUs will poll the diff --git a/third_party/arocc/arocc.bazel b/third_party/arocc/arocc.bazel index 71c1fd6e..8a643feb 100644 --- a/third_party/arocc/arocc.bazel +++ b/third_party/arocc/arocc.bazel @@ -1,40 +1,83 @@ load("@bazel_skylib//rules:run_binary.bzl", "run_binary") -load("@bazel_skylib//rules:write_file.bzl", "write_file") load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") +load("@zml//bazel:zig_options.bzl", "zig_option", "zig_options") -write_file( - name = "system_defaults_zig", - out = "system_defaults.zig", - content = ["""\ -pub const enable_linker_build_id: bool = false; -pub const linker: []const u8 = ""; -pub const sysroot: []const u8 = ""; -pub const gcc_install_prefix: []const u8 = ""; -pub const rtlib: []const u8 = ""; -pub const unwindlib: []const u8 = ""; -"""], +zig_option( + name = "enable_linker_build_id", + build_setting_default = False, ) -zig_library( +zig_option( + name = "linker", + build_setting_default = "", +) + +zig_option( + name = "sysroot", + build_setting_default = "", +) + +zig_option( + name = "gcc_install_prefix", + build_setting_default = "", +) + +zig_option( + name = "rtlib", + build_setting_default = "", +) + +zig_option( + name = "unwindlib", + build_setting_default = "", +) + +zig_options( name = "system_defaults", - main = ":system_defaults_zig", + settings = [ + ":enable_linker_build_id", + ":linker", + ":sysroot", + ":gcc_install_prefix", + ":rtlib", + ":unwindlib", + ], ) -write_file( - name = "aro_options_zig", - out = "aro_options.zig", - content = ["""\ -pub const enable_tracy: bool = false; -pub const enable_tracy_callstack: bool = false; -pub const enable_tracy_allocation: bool = false; -pub const debug_allocations: bool = false; -pub const version_str: []const u8 = ""; -"""], +zig_option( + name = "enable_tracy", + build_setting_default = False, ) -zig_library( +zig_option( + name = "enable_tracy_callstack", + build_setting_default = False, +) + +zig_option( + name = "enable_tracy_allocation", + build_setting_default = False, +) + +zig_option( + name = "debug_allocations", + build_setting_default = False, +) + +zig_option( + name = "version_str", + build_setting_default = "", +) + +zig_options( name = "build_options", - main = ":aro_options_zig", + settings = [ + ":enable_tracy", + ":enable_tracy_callstack", + ":enable_tracy_allocation", + ":debug_allocations", + ":version_str", + ], ) zig_library( From 6de80971d232279c8aa0cbb9fef5a67bce17f7e4 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Wed, 16 Sep 2026 21:51:38 +0200 Subject: [PATCH 693/881] platforms/oneapi: add missing header in oneapi target --- platforms/oneapi/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/platforms/oneapi/BUILD.bazel b/platforms/oneapi/BUILD.bazel index c4e8eb32..aaa06989 100644 --- a/platforms/oneapi/BUILD.bazel +++ b/platforms/oneapi/BUILD.bazel @@ -13,6 +13,7 @@ zig_shared_library( cc_library( name = "libzml_oneapi", + hdrs = ["oneapi.h"], tags = ["manual"], deps = ["@libzml_oneapi"], ) From d60a11e2a2ba57d8110bdef41dc2de741aed46ac Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Wed, 16 Sep 2026 22:27:34 +0200 Subject: [PATCH 694/881] zml: add support for zio (#821) Add a [zio](https://github.com/lalinsky/zio) dependency. We require ZML to be aware of which `std.Io` implementation it's running because we use `threadlocal` to store the current `zml.Compiler` instance when compiling a graph (to make the DSL pleasant to use). This is our only `threadlocal` usage but it is one too much. So we must make it compatible with `zio.TaskLocal`. Not a big deal, but this requires opt-in from the user. And so `zio` support can be opt-in enabled by setting the build flag (default is `std`): ``` --@zml//zml:io_impl=zio ``` Sadly there is no `std.Io` response to `threadlocal` (or task local). Hopefully this will be fixed in the future. Note that this creates problem if using `std.Io.Evented`. --- MODULE.bazel | 3 +- MODULE.bazel.lock | 3 +- third_party/non_module_deps.bzl | 4 +-- third_party/zio/BUILD.bazel | 0 third_party/zio/progress-parent-file.patch | 17 ++++++++++ third_party/zio/repo.bzl | 11 +++++++ third_party/zio/zio.bazel | 38 ++++++++++++++++++++++ zml/BUILD.bazel | 22 +++++++++++-- zml/Compiler.zig | 30 ++++++++++++----- 9 files changed, 114 insertions(+), 14 deletions(-) create mode 100644 third_party/zio/BUILD.bazel create mode 100644 third_party/zio/progress-parent-file.patch create mode 100644 third_party/zio/repo.bzl create mode 100644 third_party/zio/zio.bazel diff --git a/MODULE.bazel b/MODULE.bazel index f598674f..9e4d1338 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -14,7 +14,7 @@ single_version_override( version = "20260526.0", ) -bazel_dep(name = "bazel_lib", version = "3.2.2") +bazel_dep(name = "bazel_lib", version = "3.7.2") bazel_dep(name = "aspect_rules_py", version = "1.8.4") bazel_dep(name = "bazel_skylib", version = "1.9.0") @@ -285,6 +285,7 @@ use_repo( "uucode", "xla", "zigimg", + "zio", ) xla = use_extension("//third_party/xla:xla.bzl", "xla") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 841260f0..a83ce7bb 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -71,7 +71,8 @@ "https://bcr.bazel.build/modules/bazel_lib/3.0.0/MODULE.bazel": "22b70b80ac89ad3f3772526cd9feee2fa412c2b01933fea7ed13238a448d370d", "https://bcr.bazel.build/modules/bazel_lib/3.1.0/MODULE.bazel": "6809765c14e3c766a9b9286c7b0ec56ed87a73326e48fe01749f0c0fdcfe3287", "https://bcr.bazel.build/modules/bazel_lib/3.2.2/MODULE.bazel": "e2c890c8a515d6bca9c66d47718aa9e44b458fde64ec7204b8030bf2d349058c", - "https://bcr.bazel.build/modules/bazel_lib/3.2.2/source.json": "9e84e115c20e14652c5c21401ae85ff4daa8702e265b5c0b3bf89353f17aa212", + "https://bcr.bazel.build/modules/bazel_lib/3.7.2/MODULE.bazel": "cbc920fff00a55b2fd663d40850560a6fb17fd5ef3a9f3d6cbf0d32908dd8a40", + "https://bcr.bazel.build/modules/bazel_lib/3.7.2/source.json": "7a9c2b06a113c4962d329bb68befdb90d0447788a19aabaa32b476a0458e7d28", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", diff --git a/third_party/non_module_deps.bzl b/third_party/non_module_deps.bzl index fd8d82fe..4e5f49a3 100644 --- a/third_party/non_module_deps.bzl +++ b/third_party/non_module_deps.bzl @@ -17,6 +17,7 @@ load("//third_party/translate-c:repo.bzl", translate_c = "repo") load("//third_party/uucode:repo.bzl", uucode = "repo") load("//third_party/xla:repo.bzl", xla = "repo") load("//third_party/zigimg:repo.bzl", zigimg = "repo") +load("//third_party/zio:repo.bzl", zio = "repo") def _non_module_deps_impl(mctx): cloud_accelerator_diagnostics() @@ -38,11 +39,10 @@ def _non_module_deps_impl(mctx): cuda_tile() flydsl() cute_ir() + zio() return mctx.extension_metadata( reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], ) non_module_deps = module_extension( diff --git a/third_party/zio/BUILD.bazel b/third_party/zio/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/third_party/zio/progress-parent-file.patch b/third_party/zio/progress-parent-file.patch new file mode 100644 index 00000000..2d92fd06 --- /dev/null +++ b/third_party/zio/progress-parent-file.patch @@ -0,0 +1,17 @@ +diff --git a/src/io.zig b/src/io.zig +--- a/src/io.zig ++++ b/src/io.zig +@@ -1853,7 +1853,12 @@ fn childKillImpl(_: ?*anyopaque, child: *std.process.Child) void { + } + + fn progressParentFileImpl(_: ?*anyopaque) std.Progress.ParentFileError!Io.File { +- @panic("progressParentFile: not supported"); ++ var environ: Io.Threaded.Environ = .{ .process_environ = processEnviron() }; ++ environ.scan(.failing); ++ var file = try environ.zig_progress_file; ++ // Inherited Windows handles are not associated with zio's IOCP port. ++ if (builtin.os.tag == .windows) flagsWritePollable(&file.flags, false); ++ return file; + } + + fn nowImpl(_: ?*anyopaque, clock: Io.Clock) Io.Timestamp { diff --git a/third_party/zio/repo.bzl b/third_party/zio/repo.bzl new file mode 100644 index 00000000..76280f54 --- /dev/null +++ b/third_party/zio/repo.bzl @@ -0,0 +1,11 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + git_repository( + name = "zio", + remote = "https://github.com/lalinsky/zio", + commit = "54cb18f05d3feaec431353de85d5d10dac93d37b", + build_file = Label("//third_party/zio:zio.bazel"), + patches = [Label("//third_party/zio:progress-parent-file.patch")], + patch_args = ["-p1"], + ) diff --git a/third_party/zio/zio.bazel b/third_party/zio/zio.bazel new file mode 100644 index 00000000..6314c5a0 --- /dev/null +++ b/third_party/zio/zio.bazel @@ -0,0 +1,38 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") +load("@zml//bazel:zig_options.bzl", "zig_option", "zig_options") + +zig_option( + name = "backend", + build_setting_default = "", + nullable = True, +) + +zig_option( + name = "resolve_beneath_mode", + build_setting_default = "strict", + enum_name = "ResolveBeneathMode", + values = ["strict", "best_effort"], +) + +zig_option(name = "no_hacks", build_setting_default = False) +zig_option(name = "task_migration", build_setting_default = True) +zig_option(name = "scheduler_metrics", build_setting_default = False) + +zig_options( + name = "zio_options", + settings = [ + ":backend", + ":resolve_beneath_mode", + ":no_hacks", + ":task_migration", + ":scheduler_metrics", + ], +) + +zig_library( + name = "zio", + main = "src/zio.zig", + srcs = glob(["src/**/*.zig"]), + deps = [":zio_options"], + visibility = ["//visibility:public"], +) diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 99052d1d..81fc1146 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -1,6 +1,22 @@ load("@com_google_protobuf//bazel:upb_c_proto_library.bzl", "upb_c_proto_library") load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") +load("//bazel:zig_options.bzl", "zig_option", "zig_options") + +zig_option( + name = "io_impl", + build_setting_default = "std", + values = [ + "std", + "zio", + ], +) + +zig_options( + name = "zml_options", + import_name = "zml/options", + settings = [":io_impl"], +) upb_c_proto_library( name = "xla_data_upb", @@ -102,7 +118,6 @@ zig_library( "mem.zig", "meta.zig", "mlirx.zig", - "moe/triton_mxfp4.zig", "moe/cutlass_flashinfer.zig", "moe/fly_kernels/moe.zig", "moe/fly_kernels/mxfp4.zig", @@ -112,8 +127,9 @@ zig_library( "moe/mosaic_tpu.zig", "moe/mosaic_tpu_kernels/gmm_ep.zig", "moe/triton.zig", - "moe/triton_kernels/triton_kernels.zig", "moe/triton_kernels/mxfp4.zig", + "moe/triton_kernels/triton_kernels.zig", + "moe/triton_mxfp4.zig", "nn.zig", "ops.zig", "pjrtx.zig", @@ -148,6 +164,7 @@ zig_library( ":profiling_traceme_lib", ":xla_compile_options_upb", ":xla_data_upb", + ":zml_options", "//bazel", "//ffi:zig", "//kernels/cuda_tile:cuda_tile_builder", @@ -169,6 +186,7 @@ zig_library( "//vfs", "//zml/tokenizer", "@rules_zig//zig/runfiles", + "@zio", ] + select({ "//platforms:neuron.enabled": [ "//platforms/neuron:nki_kernel", diff --git a/zml/Compiler.zig b/zml/Compiler.zig index d15a542c..b635cc1d 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -6,6 +6,8 @@ const mlir = @import("mlir"); const pjrt = @import("pjrt"); const stdx = @import("stdx"); const upb = @import("upb"); +const zio = @import("zio"); +const zml_options = @import("zml/options"); const Buffer = @import("buffer.zig").Buffer; const DataType = @import("dtype.zig").DataType; @@ -44,7 +46,9 @@ manual_computation_depth: usize = 0, channel_id: i64 = 0, composite_id: i64 = 0, +var _current_zio: zio.TaskLocal(*Compiler) = .{}; threadlocal var _current: ?*Compiler = null; + var mlir_global_init_mutex: std.Io.Mutex = .init; var mlir_global_registry: ?*mlir.DialectRegistry = null; @@ -182,7 +186,12 @@ pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, } pub fn deinit(self: *Compiler) void { - if (_current == self) _current = null; + switch (zml_options.io_impl) { + .std => { + if (_current == self) _current = null; + }, + .zio => {}, + } std.debug.assert(self.scopes.len == 0); self.mlir_pass_manager.deinit(); self.module.deinit(); @@ -201,11 +210,14 @@ pub fn deactivate(self: *Compiler) void { } pub fn current() *Compiler { - return _current.?; + return currentOrNull().?; } pub fn currentOrNull() ?*Compiler { - return _current; + return switch (zml_options.io_impl) { + .std => _current, + .zio => _current_zio.get(), + }; } pub fn currentScope(self: *Compiler) *Scope { @@ -443,11 +455,13 @@ fn emitMlir(compiler: *Compiler, comptime func: anytype, args: std.meta.ArgsTupl var input_info = try createBlockArguments(compiler, fn_scope, &args); errdefer input_info.deinit(compiler.allocator); - var result = result: { - compiler.activate(); - defer compiler.deactivate(); - - break :result @call(.auto, func, args); + var result = switch (zml_options.io_impl) { + .std => blk: { + compiler.activate(); + defer compiler.deactivate(); + break :blk @call(.auto, func, args); + }, + .zio => _current_zio.scoped(compiler, func, args), }; var output_info = try collectOutputInfo(compiler, fn_scope, &result); From f09060e5bffb72d7e743421eac500aac07e7b88d Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 17 Sep 2026 10:02:57 +0200 Subject: [PATCH 695/881] //tools/xprof: bump to xprof 2.32.2 (#823) I found a non-nightly version that seems to work well for us. Previous nightly versions we were using is not hosted anymore on Pypi.org --- MODULE.bazel | 4 +- tools/xprof/BUILD.bazel | 2 +- tools/xprof/main.py | 13 + tools/xprof/requirements.in | 2 +- tools/xprof/requirements.lock.txt | 545 ++++++++++++++++-------------- 5 files changed, 315 insertions(+), 251 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 9e4d1338..78fef70a 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -193,11 +193,11 @@ pip.parse( "gcsfs", ], }, - hub_name = "xprof_nightly", + hub_name = "xprof", python_version = "3.12", requirements_lock = "//tools/xprof:requirements.lock.txt", ) -use_repo(pip, "xprof_nightly") +use_repo(pip, "xprof") cpu = use_extension("//platforms/cpu:cpu.bzl", "cpu_plugin") use_repo(cpu, "libzml_cpu_darwin_amd64", "libzml_cpu_darwin_arm64", "libzml_cpu_linux_amd64") diff --git a/tools/xprof/BUILD.bazel b/tools/xprof/BUILD.bazel index 3b4dc40e..285f7003 100644 --- a/tools/xprof/BUILD.bazel +++ b/tools/xprof/BUILD.bazel @@ -7,7 +7,7 @@ py_binary( main = "main.py", tags = ["manual"], visibility = ["//visibility:public"], - deps = ["@xprof_nightly//xprof_nightly:pkg"], + deps = ["@xprof//xprof:pkg"], ) uv_lock( diff --git a/tools/xprof/main.py b/tools/xprof/main.py index 468c5603..79ed3ccf 100644 --- a/tools/xprof/main.py +++ b/tools/xprof/main.py @@ -1,8 +1,20 @@ +import os import sys +from pathlib import Path +from xprof import profile_plugin from xprof.cli.xprof_cli import main as xprof_main +def _configure_static_dir() -> None: + static_dir = Path(profile_plugin.__file__).parent / "static" + # Bazel runfiles contain a real directory whose files are symlinks into the + # repository cache. Resolve a file so XProf's traversal check sees the same + # base path as the assets it opens. + resolved_static_dir = (static_dir / "index.html").resolve().parent + os.environ.setdefault("XPROF_STATIC_DIR", str(resolved_static_dir)) + + def _argv() -> list[str]: user_args = sys.argv[1:] if user_args and not user_args[0].startswith("-"): @@ -25,5 +37,6 @@ def _argv() -> list[str]: if __name__ == "__main__": + _configure_static_dir() sys.argv = _argv() raise SystemExit(xprof_main()) diff --git a/tools/xprof/requirements.in b/tools/xprof/requirements.in index 666e8afb..7a238fea 100644 --- a/tools/xprof/requirements.in +++ b/tools/xprof/requirements.in @@ -1 +1 @@ -xprof-nightly +xprof==2.23.2 diff --git a/tools/xprof/requirements.lock.txt b/tools/xprof/requirements.lock.txt index 6bf9f9bc..f3ac6542 100644 --- a/tools/xprof/requirements.lock.txt +++ b/tools/xprof/requirements.lock.txt @@ -3,7 +3,7 @@ absl-py==2.5.0 \ --hash=sha256:0c996f25c0490700fadabe6351630f6111534fa0ae252cc6d2014ea3b141135f \ --hash=sha256:0f17b89f2a4eaaedc4f28c622998aa690564b3012a396a4ffad0821007fe03ba - # via xprof-nightly + # via xprof aiohappyeyeballs==2.7.1 \ --hash=sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d \ --hash=sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472 @@ -420,7 +420,7 @@ charset-normalizer==3.5.1 \ cheroot==11.1.2 \ --hash=sha256:0f6c0ba05c00fbc869fb46b1de4ec2384e1d85418ae963d3bc10ae83b688dbfa \ --hash=sha256:bfb70c49663f63b0440f2b54dbc6b0d1650e56dfe4e2641f59b2c6f727b44aca - # via xprof-nightly + # via xprof cryptography==50.0.1 \ --hash=sha256:01f41478cf33fc605a6a089cd56d28b45c6c0b45a1928b61797f2621a04bac71 \ --hash=sha256:05ba322c4da95b262a212c345af888ef2c37c88c0509756ea00a0e6d68850f23 \ @@ -476,11 +476,11 @@ decorator==5.3.1 \ etils[epath]==1.14.0 \ --hash=sha256:8136e7f4c4173cd0af0ca5481c4475152f0b8686192951eefa60ee8711e1ede4 \ --hash=sha256:b5df7341f54dbe1405a4450b2741207b4a8c279780402b45f87202b94dfc52b4 - # via xprof-nightly + # via xprof fire==0.7.1 \ --hash=sha256:3b208f05c736de98fb343310d090dcc4d8c78b2a89ea4f32b837c586270a9cbf \ --hash=sha256:e43fd8a5033a9001e7e2973bab96070694b9f12f2e0ecf96d4683971b5ab1882 - # via xprof-nightly + # via xprof frozenlist==1.8.0 \ --hash=sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686 \ --hash=sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0 \ @@ -621,7 +621,7 @@ fsspec[gcs]==2026.7.0 \ # via # etils # gcsfs - # xprof-nightly + # xprof gcsfs==2026.8.1 \ --hash=sha256:66e6316690b17e1887a4f4bc0902eee53f5b3fd860cc3f0377d65c5be3313b1f \ --hash=sha256:738da7dcc74817c7668d032f932598482a23cce1b061c537e54278be44d2fff8 @@ -656,7 +656,7 @@ google-cloud-storage==3.14.1 \ --hash=sha256:b24e74b493c60b19b83462933a83bb831e45b6ca4924b0d75eac8e176b58b3a7 # via # gcsfs - # xprof-nightly + # xprof google-cloud-storage-control==1.15.0 \ --hash=sha256:46c8c801ff0425a1bd59bc28899a8242e15f6bbb06a4f2304cbd169c7d26cb29 \ --hash=sha256:e3d50276f0aed0c33ebcf3c6221e933889e042df70a56886ddffe59c66889183 @@ -788,7 +788,7 @@ grpcio-status==1.84.0 \ gviz-api==1.10.0 \ --hash=sha256:846692dd8cc73224fc31b18e41589bd934e1cc05090c6576af4b4b26c2e71b90 \ --hash=sha256:a05055fed8c279f34f4b496eace7648c7fe9c1b06851e8a36e748541f1adbb05 - # via xprof-nightly + # via xprof idna==3.19 \ --hash=sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15 \ --hash=sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4 @@ -1079,128 +1079,145 @@ opentelemetry-api==1.44.0 \ --hash=sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a \ --hash=sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef # via google-api-core -propcache==0.5.2 \ - --hash=sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427 \ - --hash=sha256:04dc2390d9edbbaef7461f33322555976ffddf0b650a038649d026358714e6c5 \ - --hash=sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa \ - --hash=sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7 \ - --hash=sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a \ - --hash=sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0 \ - --hash=sha256:0fd59b5af35f74da48d905dcbad55449ba13be91823cb05a9bd590bbf5b61660 \ - --hash=sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94 \ - --hash=sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917 \ - --hash=sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42 \ - --hash=sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3 \ - --hash=sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa \ - --hash=sha256:1ca071adabaab6e9219924bbe00af821f1ee7de113a9eca1cdc292de3d120f4d \ - --hash=sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33 \ - --hash=sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a \ - --hash=sha256:2293949b855ce597f2826452d17c2d545fb5622379c4ea6fdf525e9b8e8a2511 \ - --hash=sha256:26a4dca084132874e639895c3135dfad5eb20bae209f62d1aeb31b03e601c3c0 \ - --hash=sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84 \ - --hash=sha256:29cbaac5ea0212663e6845e04b5e188d5a6ae6dd919810ac835bf1d3b42c3f4c \ - --hash=sha256:29f9309a2e42b0d273be006fdb4be2d6c39a47f6f57d8fb1cf9f81481df81b66 \ - --hash=sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821 \ - --hash=sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb \ - --hash=sha256:2f8ea531c794b9d6274acd4e8d2c2ebcac590a4361d27482edd3010b79f1325e \ - --hash=sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853 \ - --hash=sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56 \ - --hash=sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55 \ - --hash=sha256:3b199b9b2b3d6a7edf3183ba8a9a137a22b97f7df525feb5ae1eccf026d2a9c6 \ - --hash=sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704 \ - --hash=sha256:44e488ef40dbb452700b2b1f8188934121f6648f52c295055662d2191959ff82 \ - --hash=sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f \ - --hash=sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64 \ - --hash=sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999 \ - --hash=sha256:4621064bbf28fa77ff64dd5d94367c04684c67d3a5bf1dff25f0cd0d98a38f3b \ - --hash=sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb \ - --hash=sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d \ - --hash=sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4 \ - --hash=sha256:54adaa85a22078d1e306304a40984dc5be99d599bf3dc0a24dc98f7daeab89ab \ - --hash=sha256:552ffadf6ad409844bc5919c42a0a83d88314cedddaea0e41e80a8b8fffe881f \ - --hash=sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03 \ - --hash=sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5 \ - --hash=sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba \ - --hash=sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979 \ - --hash=sha256:5aaa2b923c1944ac8febd6609cb373540a5563e7cbcb0fd770f75dace2eb817b \ - --hash=sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144 \ - --hash=sha256:5fcb98e7598b1ee0addab320d90f65b530297a867dbfe9de52ea838077e16e3d \ - --hash=sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e \ - --hash=sha256:66ea454f095ddf5b6b14f56c064c0941c4788be11e18d2464cf643bf7203ff67 \ - --hash=sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117 \ - --hash=sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa \ - --hash=sha256:6bf3be92233808fcd338eba0fb4d0b59ec5772af4f4ecfcec450d1bfc0f8b5eb \ - --hash=sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96 \ - --hash=sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5 \ - --hash=sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476 \ - --hash=sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191 \ - --hash=sha256:74b70780220e2dd89175ca24b81b68b67c83db499ae611e7f2313cb329801c78 \ - --hash=sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078 \ - --hash=sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837 \ - --hash=sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a \ - --hash=sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba \ - --hash=sha256:8114f28879e0904748e831c3a7774261bd9e75f49be089f389a76f959dcd13fe \ - --hash=sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c \ - --hash=sha256:823581fd5cb08b12a48bfa11fe962a7916766b6170c17b028fbdf762b85eb9bf \ - --hash=sha256:85341b12b9d55bad0bded24cac341bb34289469e03a11f3f583ea1cc1db0326c \ - --hash=sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9 \ - --hash=sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8 \ - --hash=sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe \ - --hash=sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031 \ - --hash=sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913 \ - --hash=sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d \ - --hash=sha256:949c91d1a990cf3b2e8188dfcfb25005e0b834a06c63fa4ef9f360878ce21ecf \ - --hash=sha256:95f1e3f4760d404b13c9976c0229b2b49a3c8e2c62a9ce92efdd2b11ada75e3f \ - --hash=sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539 \ - --hash=sha256:a0e399a2eccb91ed18721f86aa85757727400b6865c89e88934781deb9c8498b \ - --hash=sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285 \ - --hash=sha256:a4840ab0ae0216d952f4b53dc6d0b992bfc2bedbfe360bdd9b548bc184c08959 \ - --hash=sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d \ - --hash=sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4 \ - --hash=sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f \ - --hash=sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836 \ - --hash=sha256:b05d643f944a8c3c4bd86d65ffd87bf3264b617f87791940302bc474d2ff5274 \ - --hash=sha256:b96db7141a592cbc968daf1feea83a118e6ab378af4abbc72b248c895414c22d \ - --hash=sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f \ - --hash=sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e \ - --hash=sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe \ - --hash=sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1 \ - --hash=sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a \ - --hash=sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39 \ - --hash=sha256:c6844ba6364fb12f403928a82cfd295ab103a2b315c77c747b2dbe4a41894ea7 \ - --hash=sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a \ - --hash=sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164 \ - --hash=sha256:cc1177027eda740fdb152706bd215a3f124e3eea15afc39f2cb9fe351b50619e \ - --hash=sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2 \ - --hash=sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0 \ - --hash=sha256:cd416c1de191973c52ff1a12a57446bfc7642797b282d7caf2162d7d1b8aa9a0 \ - --hash=sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335 \ - --hash=sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568 \ - --hash=sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4 \ - --hash=sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80 \ - --hash=sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2 \ - --hash=sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370 \ - --hash=sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4 \ - --hash=sha256:d5a81be28596d6559f6131ef33e10200de6e17643b3c74ce03f9eb103be6ae8b \ - --hash=sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42 \ - --hash=sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a \ - --hash=sha256:decfca4c79dd53ebab484b00cc4b6717d8c369f86e74aa4ca395a64ac651495e \ - --hash=sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757 \ - --hash=sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825 \ - --hash=sha256:e4294d04a94dcab1b3bccd8b66d962dcad411a1d19414b2a41d1445f1de32ad0 \ - --hash=sha256:e59bc9e66329185b93dab73f210f1a37f81cb40f321501db8017c9aea15dba27 \ - --hash=sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf \ - --hash=sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f \ - --hash=sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d \ - --hash=sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366 \ - --hash=sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc \ - --hash=sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c \ - --hash=sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7 \ - --hash=sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702 \ - --hash=sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098 \ - --hash=sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751 \ - --hash=sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e \ - --hash=sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6 +propcache==0.5.4 \ + --hash=sha256:004e685b315646c410771836e72a44f143bbe624f29653a42687815069a303d5 \ + --hash=sha256:02c0a34f16889cf800f10f0247a564d8ce6eeab6ffcd7c87198f769067eb8432 \ + --hash=sha256:03969626faf0783a592dfa17e28eac06018bd0b44dafae6943d53b92421a7f72 \ + --hash=sha256:03b229037d25b801e7af53fd52b9fc49d9439b036fca1e087e02780631adfa97 \ + --hash=sha256:0951315a6b3142ee2167404d707743f0157c110091342b1aa0accac5cf0e4acf \ + --hash=sha256:0a095db8e15a6020db149ecbed6461939fe74f6acaa3ae8b702a1fe8c38cd983 \ + --hash=sha256:0c889f6fa84957bc7e8b4eab71fd16a0455068d5045e3aa40c733071d2b2fd77 \ + --hash=sha256:0d21d0d2c82bbfeb1677a9711f38df968f9837576102bb4add1bd449d28d88f1 \ + --hash=sha256:10ef33a68a61ce317e095fd2e202a592ea92392b90944a78c993f0d9a73ab06c \ + --hash=sha256:12682126712ddc19b70ff819debbd279e58adf1f0c8f8f8138c18ade2044b284 \ + --hash=sha256:135036c5cfc93864affb0f9af9a27e5d7a71cb7bd745e7b6dbfc2d56cc30e827 \ + --hash=sha256:13e52b6e0bde97dee98ab66552dbff2931649c96f1ac432eac299fe689ec373b \ + --hash=sha256:141fdbd73748db0cf7636035030aaac383d2efde8f34e7bc24594cc776d225b8 \ + --hash=sha256:146f48a9e4812611a7581003b1a39de56c34967046310c4171a68ef908c9a745 \ + --hash=sha256:174507f82d3594622acb1dd2dafecf2d899d6d506335494e7107767bf05f3aae \ + --hash=sha256:1783582065a1f07f9d9ee1e992e13f15d7dc8fb1eb3a7476d43eb3f2e69d26bb \ + --hash=sha256:17a7400cec0256f0a71ae71f9da398f9894c956ff6668a1c9d317b3367316320 \ + --hash=sha256:1b2f3bec4261a94019575481c726c29850f72e27907773c75b1de421e20e9f9d \ + --hash=sha256:1d759d05634f1b038fb625a66662a8c85e5a8fec912da381b5149ddac107482b \ + --hash=sha256:1df8d8561b21465c5dd56110a01caf897e026d065b4b84e98a488209094272ec \ + --hash=sha256:213bb68d9ced5cf2bf717b1071bf2b09b4b04c426256f9fe6d054c60318424c4 \ + --hash=sha256:23278f808cd81d5ada7184a76606b925fb3389c60e1077b2cd7da7b1fcf0553c \ + --hash=sha256:251c63dd46a0659bb875cb254dc4c1e79ee91a847c737cd62373295afc2235dc \ + --hash=sha256:279655a16973f1ee2bd2fe79973137681642fd9ae0d89215bba263726eb0dc3a \ + --hash=sha256:2814ecd8e818f487bee4b0f921bc4d1c176cc5fc71ac0f072d0fa67eda4ac14b \ + --hash=sha256:286867fb156488c251a3721766e380ac4495e4fd6b51aaa1403d89ce7f4359d9 \ + --hash=sha256:2dba2f02d2d5c09ef8a0e6c1a42aeaa451f4be9898cb00b04fe98717da2eb23b \ + --hash=sha256:30cc1cebaf9aef49db06357a50398323ae04d70460c0491837d026ab7d6452ea \ + --hash=sha256:31eb43ba2edc704ab2ec27815315dd8a19def0fb16215be4cfe8d32fe78ffd51 \ + --hash=sha256:350b272b2279f4135a64fc0c304a5d08e28a137c9573442c606152446638a831 \ + --hash=sha256:36c0d9db44b523ef93d03341b1c42d69ff01d673c053d1b1c6c3a363bcaa39ba \ + --hash=sha256:3af0c8642b2da4815d86e631232ac8286e17644fad907c19508aa8e7cb4ba8ad \ + --hash=sha256:3cd3a7edb6b95b9b33998135ebfa18d709da82290fb8f27c858970b5a12c8b56 \ + --hash=sha256:3d605bb239b796e82a81c6709548b2bd460ab73b4590cb0c83de8a2dd9694d0f \ + --hash=sha256:3e413d7a4a9b4866b7a761d6060d434b64d23cd35122eda3b026a0bbe8196b25 \ + --hash=sha256:3eb2e820e8e2101407da93f17c57cbb7d225461955fc60105daaba14cd421ee2 \ + --hash=sha256:3fa15757fea1dfcd5b7745cad9f4638929605531bd4018ab2adff7955f1a403d \ + --hash=sha256:3fc24f209c1b7f7f688b66b98293954f5504279760999b58920ee12dd8471c1d \ + --hash=sha256:4054acf80d40456a0537f2913b349718649d8d6458a14ab7f48d0ce28c30869d \ + --hash=sha256:40e94adb1e7d39ff28a8bd8d8b8fbd1df6b9f40976dbe379134f1ce058e532dd \ + --hash=sha256:420162a77f94eb1cf5ef7893f500016dabd548e73de956785a1dd899cc73006a \ + --hash=sha256:425f8cc86ab5018b4b8d4a23bc8e74d964bd3d757c3702e301aa79be76c53f6c \ + --hash=sha256:44149f46500a0a41b95b4d99c2e586a77319539730607b9892974a092788b111 \ + --hash=sha256:445ee3bfb46e85838387fb3c536a73cc0b994dc192b004e40e170adc54aa2a7e \ + --hash=sha256:45488d1a5f9ab5bd90aaa1ca20f50fe1922b8ffad71a2009d2adf41355897aac \ + --hash=sha256:45bebbe252550fec975ba3b62bc6f931643cfd3b5464ef47619cf3fef154e01c \ + --hash=sha256:45bf2e730ab8905d0527fe05a86500f406e64305c34cc81ebe64b4617cab9760 \ + --hash=sha256:48cb48c5346a97de792254af77715aa2529c2a1ebc5f586aa0aae44a02f1fe57 \ + --hash=sha256:4a1f4f5ffa55dce6307631f3cb2948e117e665966ea512e0d502b16c24f567e7 \ + --hash=sha256:4cfe0a92ae30151869e67a4b5f5e105e4e03ad30b3f38e5211b5bf77d0881993 \ + --hash=sha256:4d86476a935c88963d9b8e1a9a0d38188790e9622169bfbafa173046846709d3 \ + --hash=sha256:4e985382be6d15da8d0c2710a6fa7b9070fc9ecdeefb7f580e88373984ec8be3 \ + --hash=sha256:4f2d880ff60f45898f4acfa152aac8d04e3ee627d90ff4003491bf92239d5757 \ + --hash=sha256:4fbc1a15dc8cd1689508758d626b372b1f09d28d9577667feaf9e6bfcd8efcbc \ + --hash=sha256:50e337653721d20ead710da33bf44487fbe8a0db8782714b60306481e9f95b51 \ + --hash=sha256:53eaa697c4d0422ff4cb714d00231b43352064d97b944033b30c1d57cc506ec0 \ + --hash=sha256:56fc3f7599528db40b1efa0889a620116e2704144495273d66066e8164e45838 \ + --hash=sha256:58134228927cee6c047d626c08e60a81be604a20578a12ce752cc5c9a84d4826 \ + --hash=sha256:594eb4c6ec35e7179b058481f4e9f02521b56de16fa577c4b85c76fb1bf8a9f8 \ + --hash=sha256:5cacf3c9efd09df409dc33654dd077e1c245ba8fb747b0f0236ef41b7c49b589 \ + --hash=sha256:60a64cbccaa11b7760ce705a14ada17ba459e7ca9f23ba587eb013821032d7ef \ + --hash=sha256:62530ca89187827e4a4fe733f971abe81a7542eeea48ff61995f19b64d7199c8 \ + --hash=sha256:62c60aec739ed00124573cce1178138fd690c7676352d67a37328c1cf51d7468 \ + --hash=sha256:69fc35c0779522da366c563e5faf203ffc1f8ff0021d5b1337fa4efa5be73177 \ + --hash=sha256:6af4693716bfb03f1752ef1b30faa593db2c01d5272e9b8564a1549452a979ab \ + --hash=sha256:6c7599df2b57ebeea8de011b5f2f7b85de95e76037d43d34b95e328430275487 \ + --hash=sha256:6e9368e87a3efc285e559131092c5db643eb8e56de4ee42064d5baec22ef2bb5 \ + --hash=sha256:6f0093ac3e9daada202c2082439d414a625c57184727a46e112a3fb2a81cb788 \ + --hash=sha256:7177c43eddf10a0893c4fec52ebb408fdcd7f7d63962caace9180d8f81b14ece \ + --hash=sha256:720cf832eb2d0b0dfee129cb3335a26f6ce3cc45ee1187e8f0731758caa16792 \ + --hash=sha256:770e8209d018175fc0063936fa9583b6d27e88c5ad31543f3383d66080efdd62 \ + --hash=sha256:7a8d5ff04eb1f85698a78d20c62a14676e7b960dcafde09a388d60ad377d355d \ + --hash=sha256:7b9100a93b372418d8688f3f2a3e5b45c64d70ca4d6176e121aca1e3bfc1e32f \ + --hash=sha256:7cc528e760a8af06f2b13e9b9f362cd90c7c718ea61228a96dbd31ba16ed7f47 \ + --hash=sha256:7ffafcbfc7b549ab940047e505c831eabac5e67de53e1bc174adbc5285c55944 \ + --hash=sha256:87a3caecf8095e48dc72f84bfa42e23a848cf410cc9cc13031fba4869b706a21 \ + --hash=sha256:886b59c4d28ca97dd23b025fdfc50a0356be934efbbbca89ad26230067f86fe5 \ + --hash=sha256:8876b39961e33d912afe3c1bee18ee564fdad0206f873cc15d522756b7f50737 \ + --hash=sha256:897d1ddf6716e8f47200f7aad9a0efa6cc7586df66c6defa572f9eab379c078e \ + --hash=sha256:8a1fc236528c457cd739c88abe823da851b7ab645d72792f88658114cc340c12 \ + --hash=sha256:8a235f73d6e020855dc29dff012d920c02ee0feab8d73a24185a7569f4be1161 \ + --hash=sha256:8f911c395cef73c510bac566da9507bb6a43e7763d0c79138dc60ee53f11207e \ + --hash=sha256:96f7c5c15656040ddcbc51e56dc59b58aa25999d743c126abd425b9766ab43e9 \ + --hash=sha256:978f28401afbc76cdc3df9e1717b4229a06b626a1dcc75db4e1f2beb3884c3e9 \ + --hash=sha256:98914de2c4d7f0f9f4a8c6ea4bf05841f4175796941e3ef7d47eb718f22311fb \ + --hash=sha256:9a2a8a50a93dee0268a860a07fa3b4bd968f8ce4dbd794957da772f395368526 \ + --hash=sha256:9cbfff4423eef4cc6cafc021469641a2b835f610b2647a6c5281903e21b8670d \ + --hash=sha256:9e9ab13760aa8b6d0881ae7cb04fd891d8d490cd2554ea8e79bb278399169bcc \ + --hash=sha256:9f3551b8a35c1df3e7ea4d2d86edee15f0dde1bddd434a71744048683544d0ef \ + --hash=sha256:9f86f7259efe2c951f43e57d471c9b41daa5bfc7db9f67189059cf1ae6d77fd9 \ + --hash=sha256:9fb0a5be8d9aa213150e8d8148a42aca4984b285bcad1e69587dc4298edd929b \ + --hash=sha256:a219f0ac59817a9114dd2aa57c13180f993e819ba658c7ddab4b66ed1ee0d370 \ + --hash=sha256:a419ee85e654927baabda3929c03c0cc1112bf472ff0dfd6142f4e3a81ca4162 \ + --hash=sha256:a4d7a54719b67338a305dca2ce6aafe366817df94ddfd4b5514374356f5ca546 \ + --hash=sha256:a5793c7698a53f56f4a1889a4737c7eeb1b7ad0842fa6b1abca22913ff79c8c1 \ + --hash=sha256:a5e8ef588c109725dc713ba69aadcac00a1ef90c2ce9c0a8c7075128f569f47f \ + --hash=sha256:a74bfa37147cc08fb29df10bd9c16f40fa7f860cd3a6d2fff853323a94f6e17f \ + --hash=sha256:ada748108a43d29b7c328ba7db3755327cd94f028bcc1a7ee3f0addcfacd9c38 \ + --hash=sha256:ae58f361bd5dae942717c65d3413b478c70aea9c462599e7b9adad3731db3894 \ + --hash=sha256:b28f41fa3b8c6900457f858ec5b03998f3a6d535fbc1bb2edec5961ea05ec429 \ + --hash=sha256:b3083bfe87f95c756e610bd8025f26cbd1cd4aaa03a422f2d65efb7a97cd53d8 \ + --hash=sha256:b61805357d966680acf68b3b6d49772631ed9df44ebece10ff1460e117a7da8a \ + --hash=sha256:b77c313314524ca9c38fbd70f73515d04597ac58c40c939bc0e71eeb4abff680 \ + --hash=sha256:bee7d3aed13d56f54e681df38c3a23031bc9e3863f687d9d598825c9146acd7d \ + --hash=sha256:c02c0e570c5c7e077b0181a9f3cdb7d4c3617d1cda6b5c95bd5d34022923d82c \ + --hash=sha256:c174bfd1c48a1b51a3078e95586dde718374bac79719ab3541ec9e74aec40574 \ + --hash=sha256:c2ba30a89035b57b73e00475de948521602f543d79ce01db10b04b36c4c76fc8 \ + --hash=sha256:c3e98c55bde2bcf7db3c70d1aed7ae9aa8aebbf19a250c66645cde44cdb8b867 \ + --hash=sha256:c3ef2818d63bc86071e9d2989ae75a1bc32b8f7059cfd9f5abbbee70c32e2ed6 \ + --hash=sha256:c83acbce9f2b5e3f5f5eda9e53d2001fed22fcdfef81274a9e02d8fd53b70a30 \ + --hash=sha256:c9281e922c072158c91974d4589f1dbe0fee6d467f284c28e463f9f5a4d933f4 \ + --hash=sha256:cc07876cfb079b6f6f36d21ce75784ad6c2c6b563eeac0ed26c2fa2669b85df9 \ + --hash=sha256:ccf4f7a79e26bb7efb06ecd50c177833b71df05cbc748701372325e6bcc17f6f \ + --hash=sha256:cdee8205a44d0be91bbac4c41b95d86641b72dfc7aef1279400e4fda3f26a937 \ + --hash=sha256:ceb3e879afac028f93d272c957814695dc5569e4904262dbee92f6c41bd5e4a3 \ + --hash=sha256:d1f5a500bfcbb2c0ab85e98a0dcd70f5899d34efe365a0187700369a79603031 \ + --hash=sha256:d42a9a856a4a6e2f6c10f1318c07e7daa498d6593abe745c71dae4521a26ca39 \ + --hash=sha256:d83b12902eb8bce151259c86c03ba746600b2d994543de46e370cecf96c452f2 \ + --hash=sha256:d8e017eeb7482bed34cdb0d61cf2bcfc88d104bbab296a17cd16a6af8aabc70e \ + --hash=sha256:db3ae52ccc150dbc84704e9d642743897f3e1c54742ff34cacb661e52e3818a9 \ + --hash=sha256:dbab5f5ff6897c81f355d079010cdae85b02e5a0b518b5251523b8ad8ae9ac3c \ + --hash=sha256:dc4242ca653c9b30ab51c5f8193323e7bc0928f897ee9103201e59a43abcb72e \ + --hash=sha256:dcbf346a318a5e30063f547630b02bb787ce2f45b6368d5da143660b6a3835d8 \ + --hash=sha256:dd2ac8f5b643454c2cc6b6118b13da16e88f4a6434fc3ba61aca384029f04f36 \ + --hash=sha256:e1d52a05dc417279f7e5c7618c5dfbbc29923aaf9bc0a5c1802ddcebf54c61a0 \ + --hash=sha256:e6720ba44ad7e72174314d0e1fb0172494cff5c73a3a8a2159c3d2402ff15565 \ + --hash=sha256:e738ab81179510ce79b2eac9a6ecf47feffd9e76d1c72e403005dddb6e36c06c \ + --hash=sha256:e904d4d01f36bd6e197590be1533c44e06058771e0746dd073a8ebb3ef880858 \ + --hash=sha256:e9f165403b81fea7e89c932d89046a1e3d9a3a60e8d7ef2f249dccdcb0982bf5 \ + --hash=sha256:ec6a85f424afa8d23e0d9a094e5dbb6eda01da91c92b9183cd433768247ffc97 \ + --hash=sha256:ee19113bce2f3acd46432050688b70f61acd6857d75abb9ec96341b7e9ced123 \ + --hash=sha256:ef3b928d9c984322b5c44e6964d8dbc653da87d2d8ee1647fa6da43072e650a9 \ + --hash=sha256:f273dcf7149a50527c4fd1f55cfe9eac0f60753f5af544b4c9352578e20c0874 \ + --hash=sha256:f5470694918830da62fac9e69133b53d23b736d7070e587b27a4a2be37e08e68 \ + --hash=sha256:f574e460d1c8a08384a016fdb09ccf3543433263ed6b2f97104f979e64ea57c2 \ + --hash=sha256:f85915e00dcb1cd9f2f890ead064ed40a27df06f0db65be427b29482ae357572 \ + --hash=sha256:fc2461ecc45f17893f8207e73b46ea8ba93e33630e51cf4af3fbc21d47462b1a \ + --hash=sha256:ff6b113f50bc066a698db5d944d2c6dc7507168dd3341e255a8892fd0715a558 # via # aiohttp # yarl @@ -1226,7 +1243,7 @@ protobuf==7.36.1 \ # grpc-google-iam-v1 # grpcio-status # proto-plus - # xprof-nightly + # xprof pyasn1==0.6.4 \ --hash=sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81 \ --hash=sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b @@ -1254,13 +1271,13 @@ requests-oauthlib==2.0.0 \ setuptools==69.5.1 \ --hash=sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987 \ --hash=sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32 - # via xprof-nightly + # via xprof six==1.17.0 \ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 # via # gviz-api - # xprof-nightly + # xprof termcolor==3.3.0 \ --hash=sha256:348871ca648ec6a9a983a13ab626c0acce02f515b9e1983332b17af7979521c5 \ --hash=sha256:cf642efadaf0a8ebbbf4bc7a31cec2f9b5f21a9f726f4ccbb08192c9c26f43a5 @@ -1274,127 +1291,161 @@ typing-extensions==4.16.0 \ # etils # grpcio # opentelemetry-api -urllib3==2.7.0 \ - --hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \ - --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 +urllib3==2.8.0 \ + --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \ + --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63 # via # requests - # xprof-nightly + # xprof werkzeug==3.1.8 \ --hash=sha256:63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50 \ --hash=sha256:9bad61a4268dac112f1c5cd4630a56ede601b6ed420300677a869083d70a4c44 - # via xprof-nightly -xprof-nightly==2.24.2a20260915 \ - --hash=sha256:096d108cfad9b22842a57774fb1d386a0680d10f5e55ac421d81f3473c869866 \ - --hash=sha256:225031fb920efa291fb3b81a59452519ca39fafbb6853a187b13a6c56065156e \ - --hash=sha256:a37a653582f93d018d5ab0210415b8b090f6c5d3885f15db6426817bfde48a71 \ - --hash=sha256:b089f56f87f8524e40a400b266d826a670fef8f2fe2bfcd256bbf64ec773b779 + # via xprof +xprof==2.23.2 \ + --hash=sha256:38e809a29b8226183143d7adf59321f96f6b09ce5a0db0810162a2d9742187fa \ + --hash=sha256:921bc9b1d28ea78d7c63614e28b88dadbe535cf5b6c07c7b65a829a329635a9e \ + --hash=sha256:cdd3387cc357f08713df10b1032977d01c478690e7601f833de13a8ef763aa7b \ + --hash=sha256:d2b802594cf853c509e502a63403c481e607630a98d9ace4cb5fb275c707b5f5 # via -r tools/xprof/requirements.in -yarl==1.24.5 \ - --hash=sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36 \ - --hash=sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331 \ - --hash=sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871 \ - --hash=sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498 \ - --hash=sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780 \ - --hash=sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027 \ - --hash=sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224 \ - --hash=sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144 \ - --hash=sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76 \ - --hash=sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3 \ - --hash=sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb \ - --hash=sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740 \ - --hash=sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec \ - --hash=sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e \ - --hash=sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a \ - --hash=sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24 \ - --hash=sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950 \ - --hash=sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95 \ - --hash=sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb \ - --hash=sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928 \ - --hash=sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c \ - --hash=sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41 \ - --hash=sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e \ - --hash=sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550 \ - --hash=sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f \ - --hash=sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b \ - --hash=sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e \ - --hash=sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede \ - --hash=sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad \ - --hash=sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6 \ - --hash=sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104 \ - --hash=sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2 \ - --hash=sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba \ - --hash=sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9 \ - --hash=sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12 \ - --hash=sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1 \ - --hash=sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027 \ - --hash=sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385 \ - --hash=sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2 \ - --hash=sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840 \ - --hash=sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2 \ - --hash=sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4 \ - --hash=sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc \ - --hash=sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c \ - --hash=sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213 \ - --hash=sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba \ - --hash=sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448 \ - --hash=sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0 \ - --hash=sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6 \ - --hash=sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966 \ - --hash=sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750 \ - --hash=sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621 \ - --hash=sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13 \ - --hash=sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0 \ - --hash=sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58 \ - --hash=sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54 \ - --hash=sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075 \ - --hash=sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4 \ - --hash=sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440 \ - --hash=sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f \ - --hash=sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4 \ - --hash=sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c \ - --hash=sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f \ - --hash=sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d \ - --hash=sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7 \ - --hash=sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9 \ - --hash=sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723 \ - --hash=sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047 \ - --hash=sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da \ - --hash=sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293 \ - --hash=sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b \ - --hash=sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61 \ - --hash=sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca \ - --hash=sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed \ - --hash=sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a \ - --hash=sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a \ - --hash=sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688 \ - --hash=sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16 \ - --hash=sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d \ - --hash=sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077 \ - --hash=sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce \ - --hash=sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd \ - --hash=sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d \ - --hash=sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88 \ - --hash=sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5 \ - --hash=sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75 \ - --hash=sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1 \ - --hash=sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f \ - --hash=sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b \ - --hash=sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff \ - --hash=sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9 \ - --hash=sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f \ - --hash=sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0 \ - --hash=sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5 \ - --hash=sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25 \ - --hash=sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6 \ - --hash=sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00 \ - --hash=sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373 \ - --hash=sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd \ - --hash=sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1 \ - --hash=sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2 \ - --hash=sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d \ - --hash=sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba \ - --hash=sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104 +yarl==1.25.1 \ + --hash=sha256:0136d640dfa9b0523853e411430a99f8a91eca85774c6420285a33b755bc6de3 \ + --hash=sha256:03dd38de09bc213e9a8b29761eec33ee1d5318dac0e49d8af36e4d27830e23a7 \ + --hash=sha256:0a191bfdb30a79b98e5d175d75285f9fcb78bf0e46ba5efda042e1c72071a0de \ + --hash=sha256:0a66db89ea473abeac4b70523cafd94db3772380e565f9d28af7a179b7af71fa \ + --hash=sha256:0ae12ff2b805fa02c4dab838005caef735e39986322698c48588d3beacb65c62 \ + --hash=sha256:0f12afda4eea8c8994a76d4df1875c765194f5fbe8a9d197929ea303caee29ec \ + --hash=sha256:10b2fd95332f0d716d5eee3c9fb2ce8eada19082de7fee83d32e37992fd75c26 \ + --hash=sha256:126a2533570c554719ca40a1288fdee1700b6bc82e7131aa69fa85252d92e651 \ + --hash=sha256:12b6bc4906e11f5e1a1cdcb12296e7afbd366c783cc8073403cd2fb74334e453 \ + --hash=sha256:142c06c4d6a35ee3ec5da08499805e879cb3ca7c1fbfbecb0140fe72403818d6 \ + --hash=sha256:14b79a30a93a3ce2e8832603fd0ab780ada281b0ba5110b519a634f2d7d7d1fc \ + --hash=sha256:17c9877a89fb6e2bca6f9087eb24cd7fb434653946ef5075e470d23d49b52287 \ + --hash=sha256:192a866877a49993949ef1975864ad8728bea28ee810f6abe1a0729c2b500426 \ + --hash=sha256:1ab7618921a93767387a4b83776f751588f5b5ae9bb5bc96620e2e2e00bca868 \ + --hash=sha256:1e80dcf1446e1b080b1932b0d103c464a04112f5bc31f0f983ad418172063cde \ + --hash=sha256:1f51020b2eb8a003c84925638ec63c21a750a4bddd3a22ec8eac6a742dadf1b9 \ + --hash=sha256:1fb2a01ba8cd9c5d2c5dc1ec35e0fc951d04b4f037541d4ac090c993ce58b3d7 \ + --hash=sha256:2239a02249d9326655419e0168a28ca9008938eaab31dc29fc875c217927a6c0 \ + --hash=sha256:23bf5b403c879a54964e0feac7285688e04bb220074878d737d331522da0a5bf \ + --hash=sha256:24ce942011a61953e7d313438038f4d32ff21387b775f58a957f7a07dd55ef95 \ + --hash=sha256:25868beca8b6765f8f7d0e11fe6dd7c66dd4b0793b9500286d20cc92352126a5 \ + --hash=sha256:287e99ff5aa4dc1c7630bfc683ded6f106d756c99dec432a2d7f197a784f51c6 \ + --hash=sha256:29273edf1530e397bd07cb784db1fbe0d2590b77569f2e24679a9c0a2d763b94 \ + --hash=sha256:2b49375d22299b0a834c2bca72f39aaecc270d96fb24c30424899676f487b22a \ + --hash=sha256:30eec96e8a91bd588ce897c9543f6d5d8d34b28fbcba28a4dedf20ebeae9fe57 \ + --hash=sha256:319e070a01db9920fb63761843f96a104c8e2b9427266731810dc1e22595b17c \ + --hash=sha256:35dcbea443fafb3eece757ad4e514560ddeb6c34cfae1582c620d7b293d7feee \ + --hash=sha256:3f4d48a6112712973e676bd792121fee470e432d749177162d9949d5c9460a1b \ + --hash=sha256:3feb99222553a8cbedfa52c2f59dd84c3f50d5b582c728d522caf8d72769a54b \ + --hash=sha256:419f392a1da624877975709e3864dfe833af6cc7671b39318086d456e288380c \ + --hash=sha256:42a66563d8cc056ee32e6191e05097a7b2b3bc302e0bc3133daf8710eb18bd26 \ + --hash=sha256:48796ea00a303961507dc6c8437c4b325a6fc3f95f7c36c71b91ea9a8150963c \ + --hash=sha256:4bd6340d20ae2c7ca719b87b426e808e90743b676d05d4c26c4fb5ca71f41184 \ + --hash=sha256:4ca89e4e21854ed27ec753297dde84b16c9f8e53b14a4866fb44457d643c19f8 \ + --hash=sha256:4d781294bb815ecb5ea57ff6bbf8038e0a31a95fdf3e1788f66e0dc100d64b58 \ + --hash=sha256:4f1c91f5a5980a937ff8e238e98e6897e1ad74a4b1e2c0d68c73b5ffbb3f5c0b \ + --hash=sha256:564fdc7085d2245ab84f88882fdb1d6ac0723124bff6ded35bfb1c00f812630d \ + --hash=sha256:59ba3a6e1aa8cfe5adf4bd270fd965db21955401b7ca6f1696010c55ed4daec2 \ + --hash=sha256:5df89f769cc8ff94c3d7e7603386fba309d25ce5240132d26c15baa8d0e96c4c \ + --hash=sha256:632da579b2d879f6bad20f2cfa35ded1efe2f4f77f8abb26a6234a5b236acd2f \ + --hash=sha256:65b5b2066651b7432d389e9799d979c703bcc6ef44266bb8153ef54e91e4aab3 \ + --hash=sha256:664ec6a520b74a1df2810666eb67695fcb77fa663e6ea0a25aaf2e529cb24dfa \ + --hash=sha256:681c758b0490f9e96b78e5fa8e8dc6e648e9185bb6eaebe73183c33ea0c445f3 \ + --hash=sha256:683e362b8ba453080f7489c66f4ea794e751c35b72e7eab3575ef784c2fbc7fb \ + --hash=sha256:68782fdb4027b8d1eee25ec35e9a6db05e863b899eb0310b3a33b6c3fef55707 \ + --hash=sha256:6efaf45df6a849cef613a03a94c845647456662f85438c886bb67a9c027c8c2c \ + --hash=sha256:71f42c5b9a948c113bbdebfa544598321431d064ff959d32e99b1feb61d68345 \ + --hash=sha256:72849d892954be4d09e569b8b831ac39ce58417fedc767d4308a0fe542018a40 \ + --hash=sha256:72c34ac7ad4314c19362d5ce27626dcc8429bd30bbf8c179f4234078851f9492 \ + --hash=sha256:734f6e5400352ac4254456003d462866c684703570929cff7a7bde015d0cb371 \ + --hash=sha256:75baa6cf9b6d1c52f3e111a130e202fd8cf0a5b3a066c3f73d615e885092e4ec \ + --hash=sha256:77716e245c90f058466a05e6a465bb8600f767a8f4b18b4d40f3aff958e5f73c \ + --hash=sha256:77e5099b99b37f3cf79c246998ca9f7313a78054cd1809ec46bc1afad47e1c4c \ + --hash=sha256:783dd1467083f4d3f7722ad6a313f24c173e7571372738fcb7a6e6d1ba48df25 \ + --hash=sha256:7a5c3115595995779ee21f2567035793911c3802a43c74f3fbb0314929ec67ac \ + --hash=sha256:7c88edaec8c349ad4c5ad4c486a3defcc4b80ceb2f074436ffa0a87caf5e76a6 \ + --hash=sha256:7cb414a73e21a7ab58254926073f2930cb22f5b4314ea4260a687e2b3fd4dce3 \ + --hash=sha256:7d42e7e3ca399555578b4d617e3a6ecf13371b3743a115995fa010c7bf341459 \ + --hash=sha256:7d575b54cb3863ef9bc290ea4b009999d55dc237326131e4853cf33e888fee03 \ + --hash=sha256:7e4de3ac4adbad3d0bc7c6f4360a7dbff5de2f15e3b723be3198074e17fd9c40 \ + --hash=sha256:7efc9f082dfed77c316edffa9deb52888e1bc6789171887cc1f68e06d65465c8 \ + --hash=sha256:80a063f8297fc796296f00f100be520f209b23dc98f93ce8eba6ee7122598209 \ + --hash=sha256:80e47012e730da131c9f059c80936783f9659aae22dc31c03c0595590d11ed54 \ + --hash=sha256:83d4a37e4b95da4d8bda930d6d35b75b4cdadbacbb4980cae290ea3100b5d51d \ + --hash=sha256:83e9f4a25085bd4b7214701a0794ff1f50fc633ffb8bdfebf07abdd81c2db126 \ + --hash=sha256:85a18376073f8a39aa07be34f9fc77e2869aa72c55c441efdd2cf79a0407504d \ + --hash=sha256:87796fedc3ba97ec14fab55acb48584276e6c1e4c1e89c422bda62c838e754a9 \ + --hash=sha256:882569ff613758cac762a457a5d72d6e211b28d4bcfea89d1d71ea942b02eac0 \ + --hash=sha256:8ce4d6ccafb33d39bd78444612d14938ead674c25702ded2ee9c54a47735d225 \ + --hash=sha256:8e7d98cdbb6d71e726f7d525952867096053d1f290dd4e3c50d7d313a136f414 \ + --hash=sha256:8ee202350cf57abf0e9502a41601841019c25d3db7ff52d980aaf31446254059 \ + --hash=sha256:8fb0eb4955adf0579001581f2f71a126e8781ba61bcd120f127b0401163c6c2d \ + --hash=sha256:90c30ed53546da833c700115c0064c22120d1b1560f474699fd31f22dd668233 \ + --hash=sha256:9489e6abf47ba37f332075a91444c7cfedb03e6ce99fbb2f116bfe1ce810da3b \ + --hash=sha256:94d7aa6debf92a1dd14cb5280b083a764169a13cfb23a452111160274ed989f4 \ + --hash=sha256:98d370568f393215d605304cdb77b3d5539bd192c75b623c7304c42c8d6d8273 \ + --hash=sha256:9b1bdaae98bc016825dd3c9d8ee1832f829b3341f9cc6ebd1a1b0a7fef7367cc \ + --hash=sha256:9d693bf4bf534e9ba3ae2780cfd577f5135629f7b5ac653490859d0b77864865 \ + --hash=sha256:9d6ed3d17bccce4c05343e1ca8da13bc5c02c812a4e7282ddd05e8769322d3fc \ + --hash=sha256:9e23c82b63cd7652fc24d33ed6cc17099d607aa3b4fc4ddc75e95062f3d82df4 \ + --hash=sha256:a1daf47cd95a7c3a63456336bc5aaa8c86dd3a47d07ed3d0e76132ae4666a5a1 \ + --hash=sha256:a1e32763e641a1566507d90a8d3b19bfc3cc04a9d4e5ae3e32189874ed4b58a3 \ + --hash=sha256:a2059a2d891bd156bc5184e7ab7a56e78a84dfcfdeac8c501b552533ad1c36ee \ + --hash=sha256:a2ed0ba415ccdf08f14bf544cb78346d0f76086707ffee24921a2c84dbf1305a \ + --hash=sha256:a3faadac7d812ddac258feb57b9846b60c1b437c4f4b9ad42595c6f6fe4390df \ + --hash=sha256:a5877f2255aab518ebe528289037699201d5dc5f045f2396cb30aa02db22f57f \ + --hash=sha256:a78b50b4f7918a3de71105d5c0b93bbc57bb8339a4d03a9dfd449f9068e76f3d \ + --hash=sha256:a8c2b841478068440d8b733005d13a5ef535b9928cbc05f17182d410f32ba449 \ + --hash=sha256:a9ca696eb02e5c02a8afd872ada510eba9b7fe6e68b9572c2e9a9b1941e31e2e \ + --hash=sha256:aa4ed3dd308548f9e707d9caaf005d2d7f8c1e7868f858dfeb47fe76e16b391d \ + --hash=sha256:ab2054c5531af2a9ba7b69b8ec91e4f884420e83a8c5e579b013084cb57e5e5d \ + --hash=sha256:abb1384477f5901d436b5d2e5465954de46ea6098f59163d243660b5c4461d35 \ + --hash=sha256:acae6b45d1ace09b6ba3876da43b88366ef368f73b988c7f57e14231753d4420 \ + --hash=sha256:acfa7e22aa6c6e7a5996a41d275bfa01efa7ea56ab890590280e9063e2cf5c1b \ + --hash=sha256:af4ea5b37403ef4e30f3927eaed540db942bde01d8d3ff083527c0704d1c9c68 \ + --hash=sha256:b10dd0557ba422715b5206b3743192135a6022acca8baec51aa127d0a75db8fe \ + --hash=sha256:b13b88747769537f3d32e89e3a735da10c0a9e35d7322928c701b5f93d3afffd \ + --hash=sha256:b51c159a9794633f5e0db7ecec7b2b6e3734eca1f5d17dc989ff3552a43ff78b \ + --hash=sha256:b5402a340723fa7da00b5cff987ddab61276be6d11251ea71ae02bcac54890d8 \ + --hash=sha256:b7abffdf37af1cec6a2ad69b827aa84320db5894791bc8ed932dc93fb274b7e9 \ + --hash=sha256:b8075fe90bc08e40b8b8a1874fab42ee4c7b56af05c5886e9cc841397f916908 \ + --hash=sha256:bc3ac7bf569f6b64dad04dd7808c7872dae8a97df657856eac05e9b7e3614a85 \ + --hash=sha256:bd0912757081f89b107d6c00b2ff8a194401b0b87eadcf4481de2b865a8fd44f \ + --hash=sha256:bdc8d8b8c22e9e43ac68316b5e6cf083dec537f4ec213cb4aa967b583bc3fa64 \ + --hash=sha256:be80550d9bfe83d9b62398a37081a90434e6df2d978ec345c3d2820de6beddab \ + --hash=sha256:c6f117789d22dce188e5754e8bc65b7e6ebf8cb73963b9fa761f672a5883769d \ + --hash=sha256:ca32926d7d77bcc8838425c4c95e040a3ace1cb7dfdae599013458dcda2607ca \ + --hash=sha256:cce0727fd5ac04d372fa9bbfde9febc2bcf209aadfcf0468e45dec72719895d1 \ + --hash=sha256:d0f1489233a254bb3643d2f05de7d59019254d81daeca6b9162fe9edef57e0c7 \ + --hash=sha256:d1c557dfd5e3db046053a0bdc72261ade790ebe8e2c7a41b36b0ca1f14cb95f3 \ + --hash=sha256:d21f0fa80a02d05299207eeaafef345d812ace96d5306e4ef265e1d419a615fa \ + --hash=sha256:d45673badd08456d0340e9364eddafe1c53a9d2896424294de4d7dd71ad3ee57 \ + --hash=sha256:d5add7b4ca7afeea91d52e4d4e4db3b1fe9885b71f07054560d8c4296b7441a2 \ + --hash=sha256:d5f90e44653c4e0f78501ed9bb7d3fce835a8d62b7c6ed0cb16557534087e743 \ + --hash=sha256:d7306dee25b8a0e737363f347362b875094b4dc4e367311470656ae420fdbf8e \ + --hash=sha256:dbcef5a9119ef653653132cccaf999b30a0af6f33bb0a4ba80bec30056868487 \ + --hash=sha256:def538065f9e4d4cf1ae164bd59aba00dfa84f03923e0de4c3788f252d6bcd17 \ + --hash=sha256:df23df54b5114a17c2d0ef192433e2e5a9f0c5178c32375e90b7cfc965f349d0 \ + --hash=sha256:dfbf531053a0935f2e871bcd4753f90313688772ff8c017f5ea402e315a78c1f \ + --hash=sha256:e029648f9c951db30e98a7d7ec90835db88ec4b32820efe2a9bdc2287e032eb6 \ + --hash=sha256:e07595c7d6f4db270ceede356a1bd1c07a34f1c26f958d1ed0cd7b48e0d2bba3 \ + --hash=sha256:e12c538e00e7c1b286a07061046b90e8124e6a9793efae2c70db6a4aad07faad \ + --hash=sha256:e546fe1d4a93ebc2910f0d768baff19faa09843ab3f2036a67ed6e69fae4419d \ + --hash=sha256:e5637ca8d0bd7fb72648a6c7934af4baaccb697657f7438c9d264fc2abb8b0b1 \ + --hash=sha256:e636b64d24fd9c38053c5e389a1174c66361fa49dcfd220f4dd35b4abde7cb89 \ + --hash=sha256:e7011b8fb8c4054bf0c12e5edc6cd83778b0028e99ce59b18586ed036f92cfdc \ + --hash=sha256:e80f557716fd765439577131e526b8942ffc2c07bdbc5e39fa62f660ba1e963f \ + --hash=sha256:e92b6bcc741b86d67606c40d3cb9c7cc8e6c737f81e31f4a94efc204456c92e3 \ + --hash=sha256:eb96ed1ae6c7d072d60840c0434aef07a2df611812810807fbc54263a6053e9a \ + --hash=sha256:eda19ea5ee88742f47a2340816e6f2d40b53bed3ab5b69794769f36af9f35bb4 \ + --hash=sha256:ef74070ac553c59eb4f04258722066d6c6135b7baa03b2e9f2da65c096e96d98 \ + --hash=sha256:efb01a106f971cb3752856bca2318bbdf7f01bd8823779c461586cbe5ffd5258 \ + --hash=sha256:f074e8d4aa0a5798920ddb6de3d08b228c614ff3724c3e8bd7577f4bafea867b \ + --hash=sha256:f38a70074041d3b7e138e452799f5174198bae5bd5ab2000917badf403908c5f \ + --hash=sha256:f41753a76f4f63927d03a0d8ba8f5ce0f2083bec29a8cfaccc55371b1564b96b \ + --hash=sha256:f53dcd26694f148f738edc052b5a69234833e739f10f4c3287bdfd8ec0f7b326 \ + --hash=sha256:f61964f235a43738bfac50da46fc4254943a7eea3051aeb0b6fc7c992c29fadc \ + --hash=sha256:fe01645169a2112aa1d4ebc3e4c5f029c5c8f97adfc32e5d37c993b39a994d75 # via aiohttp zipp==4.1.0 \ --hash=sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f \ From cf7c93c81becf6d4131d07fe9561a16c0f240ba0 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 17 Sep 2026 12:47:28 +0200 Subject: [PATCH 696/881] zml/io: use direct memory writer on ROCm (#825) --- zml/io.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zml/io.zig b/zml/io.zig index 436aab71..ba0873d5 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -644,10 +644,10 @@ pub const MemoryWriter = union(enum) { memory: Memory.Kind, ) !MemoryWriter { return switch (platform.target) { - .cuda, .oneapi => .{ + .cuda, .rocm, .oneapi => .{ .direct = try .init(allocator, io, platform, pools, dma_allocators, dma_chunk_size, shape, sharding, buffer, memory), }, - .rocm, .tpu, .neuron, .cpu, .metal => if (memory == .host_pinned) + .tpu, .neuron, .cpu, .metal => if (memory == .host_pinned) std.debug.panic("Host pinned memory is not supported on {}", .{platform.target}) else .{ .buffered = try .init(allocator, io, platform, shape, sharding, buffer) }, From 5b481836546e93bc5e11597b17effeb33cfe4bbb Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 17 Sep 2026 12:48:01 +0200 Subject: [PATCH 697/881] zml/quantization: recognize FNUZ block-32 FP8 (#826) Allow block-32 scales to accompany E4M3FNUZ weights after load-time normalization on CDNA3. The tensor layout and quantization scheme stay unchanged, so existing Linear paths can use native FP8 conversion without materializing software-decoded FN weights. --- zml/quantization.zig | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/zml/quantization.zig b/zml/quantization.zig index fb95b9f6..5357610d 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -51,7 +51,7 @@ pub const Quantization = struct { /// f8e4m3fn, f8e4m3fnuz, or f8e8m0 values, one bf16 or f32 scale per 128x128 tile. /// The DeepSeek-style FP8 that model vendors publish themselves, under `weight_scale_inv`. fp8_block128, - /// E4M3FN weights with E8M0, BF16 or F32 scales per 32x32 tile. + /// E4M3FN or E4M3FNUZ weights with E8M0, BF16 or F32 scales per 32x32 tile. fp8_block32, /// f8e4m3fn values, one scale for the whole tensor. Spelled `[1, 1]` rather than as a /// scalar: XLA's composite rewriter requires the scale to have the operand's rank. @@ -77,7 +77,7 @@ pub const Quantization = struct { (scale.dtype() == .bf16 or scale.dtype() == .f32) and scale.count() > 1 and scale.rank() == 2 and scale.dim(0) == n and scale.dim(1) == 1, - .fp8_block32 => weight.dtype() == .f8e4m3fn and + .fp8_block32 => (weight.dtype() == .f8e4m3fn or weight.dtype() == .f8e4m3fnuz) and (scale.dtype() == .f8e8m0 or ((scale.dtype() == .bf16 or scale.dtype() == .f32) and scale.count() > 1)) and scale.rank() == 2 and scale.dim(0) * 32 == n and scale.dim(1) * 32 == k, .fp8_block128 => (weight.dtype() == .f8e4m3fn or weight.dtype() == .f8e4m3fnuz or weight.dtype() == .f8e8m0) and @@ -255,6 +255,13 @@ test "Quantization.Scheme.classify" { .init(.{ .dout = 40, .sc = 48 }, .bf16), )); + inline for (.{ .f8e4m3fn, .f8e4m3fnuz }) |dtype| { + try expect(@as(?Quantization.Scheme, .fp8_block32), Quantization.Scheme.classify( + .init(.{ .dout = 1280, .d = 5120 }, dtype), + .init(.{ .dout = 40, .sc = 160 }, .f8e8m0), + )); + } + // GLM uses f32 scales and permits a partial final 128-row tile. try expect(@as(?Quantization.Scheme, .fp8_block128), Quantization.Scheme.classify( .init(.{ .dout = 2048, .d = 6144 }, .f8e4m3fn), From b31dca5b12327014be06ac9a362a340d86e96e70 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 17 Sep 2026 23:52:43 +0200 Subject: [PATCH 698/881] bazel: make lto work again with bazel runfiles init (#822) in LTO, the linker would garbage collect that exported symbol since it wouldn't be used inside the final binary forcing it undefined via a link flag prevents it from being removed. --- bazel/BUILD.bazel | 6 ++++++ bazel/bazel.zig | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/bazel/BUILD.bazel b/bazel/BUILD.bazel index 6292f235..edbef692 100644 --- a/bazel/BUILD.bazel +++ b/bazel/BUILD.bazel @@ -1,11 +1,17 @@ load("@rules_zig//zig:defs.bzl", "zig_library") +cc_library( + name = "retain_runfiles_init", + linkopts = ["-Wl,-u,_bazel_runfiles_init"], +) + zig_library( name = "bazel", tags = ["manual"], main = "bazel.zig", visibility = ["//visibility:public"], deps = [ + ":retain_runfiles_init", "//stdx", "@rules_zig//zig/runfiles", ], diff --git a/bazel/bazel.zig b/bazel/bazel.zig index 614d2fa7..0cf23580 100644 --- a/bazel/bazel.zig +++ b/bazel/bazel.zig @@ -12,7 +12,7 @@ const init_array_section = switch (builtin.object_format) { else => "", }; -export const _ linksection(init_array_section) = &struct { +export const _bazel_runfiles_init linksection(init_array_section) = &struct { fn call(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { _ = argc; _ = envp; From ebb981315712e6dda3a85aa180b85ecd982a369e Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 18 Sep 2026 10:17:10 +0200 Subject: [PATCH 699/881] zml/platform: disable async dispatch on cuda (#828) This has a negative impact on single graph forward --- zml/platform.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/platform.zig b/zml/platform.zig index a9e076c0..c81198e5 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -727,7 +727,7 @@ pub const CreateOptions = struct { allocator: Allocator = .{ .bfc = .{} }, /// The PJRT C API still exposes this under legacy /// `use_tfrt_gpu_client` name. - gpu_async_dispatch: bool = true, + gpu_async_dispatch: bool = false, // TODO support all of https://github.com/openxla/xla/blob/3d31c48c719d331d432132b3e0c2c5ce52650675/xla/pjrt/c/pjrt_c_api_gpu_internal.cc#L76-L86 // visible_devices: []const i64 = &.{}, // node_id From 243dcda29b24620f002c8eede847f1e098e3d028 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Fri, 18 Sep 2026 10:33:00 +0200 Subject: [PATCH 700/881] zml/attention/triton: expose tokenToSequence (#818) --- zml/attention/triton_attention.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zml/attention/triton_attention.zig b/zml/attention/triton_attention.zig index c01b380a..27d5cd90 100644 --- a/zml/attention/triton_attention.zig +++ b/zml/attention/triton_attention.zig @@ -6,10 +6,10 @@ const zml = @import("../zml.zig"); const triton = zml.kernel.triton; const AttentionOptions = @import("paged_attention.zig").AttentionOptions; const MlaOptions = @import("paged_attention.zig").Mla.Options; +const sparse_mla = @import("sparse_mla.zig"); const kernels = @import("triton_kernels/unified_attention.zig"); const kernels_oneapi = @import("triton_kernels/unified_attention_oneapi.zig"); const mla_kernels = @import("triton_kernels/unified_sparse_mla.zig"); -const sparse_mla = @import("sparse_mla.zig"); const log = std.log.scoped(.@"zml/attention/triton"); @@ -831,7 +831,7 @@ pub const paged = struct { return out.output.reshape(out_shape); } - fn tokenToSequence(query_start_len: zml.Tensor, query_count: i64) zml.Tensor { + pub fn tokenToSequence(query_start_len: zml.Tensor, query_count: i64) zml.Tensor { const sequence_count = query_start_len.dim(.b) - 1; const starts = query_start_len.slice(.b, .{ .end = sequence_count }).rename(.{ .b = .seq }); const ends = query_start_len.slice(.b, .{ .start = 1 }).rename(.{ .b = .seq }); From 86e55c0c91d74be8963e67e31fd374d7aef56b5d Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Fri, 18 Sep 2026 16:22:21 +0200 Subject: [PATCH 701/881] zml/mem: use UninitializedBufferAllocator for ROCm (#832) It's faster than using the default pageable memory allocator with the latest version of the plugin, --- zml/mem.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zml/mem.zig b/zml/mem.zig index a0f853a0..5ba82c69 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -22,8 +22,8 @@ pub const DmaAllocator = union(enum) { pub fn init(parent: std.mem.Allocator, device: *const Device) DmaAllocator { return switch (device.platform.target) { .cuda => .{ .dmam = .init(parent, device.platform) }, - .oneapi, .tpu => .{ .uib = .init(device.memory(.host_pinned).?) }, - .rocm, .cpu, .neuron, .metal => .{ .passthrough = parent }, + .rocm, .oneapi, .tpu => .{ .uib = .init(device.memory(.host_pinned).?) }, + .cpu, .neuron, .metal => .{ .passthrough = parent }, }; } From 1f0c465f9c4ec23074cc9aa5cbb7dbda3193ad60 Mon Sep 17 00:00:00 2001 From: Olivier Boulant Date: Fri, 18 Sep 2026 16:27:00 +0200 Subject: [PATCH 702/881] bazel: fix runfiles initializer retention on macOS (#829) --- bazel/BUILD.bazel | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bazel/BUILD.bazel b/bazel/BUILD.bazel index edbef692..434ecfa9 100644 --- a/bazel/BUILD.bazel +++ b/bazel/BUILD.bazel @@ -1,14 +1,19 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library") cc_library( name = "retain_runfiles_init", - linkopts = ["-Wl,-u,_bazel_runfiles_init"], + # Mach-O adds an underscore to the exported Zig symbol. + linkopts = select({ + "@platforms//os:macos": ["-Wl,-u,__bazel_runfiles_init"], + "//conditions:default": ["-Wl,-u,_bazel_runfiles_init"], + }), ) zig_library( name = "bazel", - tags = ["manual"], main = "bazel.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ ":retain_runfiles_init", From 00d5fcfb00b105d03a2d871807924e9e42eb5cf4 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 18 Sep 2026 16:34:20 +0200 Subject: [PATCH 703/881] kernels/triton/builder: add explicit cast to avoid compilation error (#831) --- kernels/triton/builder.zig | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kernels/triton/builder.zig b/kernels/triton/builder.zig index 5ec89cd8..702fba0d 100644 --- a/kernels/triton/builder.zig +++ b/kernels/triton/builder.zig @@ -1161,7 +1161,9 @@ pub const Builder = struct { } else { c, _ = self.broadcast(c, xv); } - return self.select(c, xv, yv); + // Arithmetic broadcasting can widen i1 when the selected values are + // integers. The select predicate must remain boolean. + return self.select(c.to(.i1), xv, yv); } pub fn divsi(self: *Builder, lhs: Value, rhs: Value) Value { From a6c4464dba83a56aacbd65ea566e7852b4b30356 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 18 Sep 2026 17:22:07 +0200 Subject: [PATCH 704/881] zml/platforms: use vmm by default on CUDA and ROCm (#833) So that we benefit from `SKIP_TEMP` --- zml/platform.zig | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/zml/platform.zig b/zml/platform.zig index c81198e5..90917339 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -709,11 +709,12 @@ pub const CreateOptions = struct { // bump memory fraction from XLA defaults of 75% to 90%. // Even on a 8GB GPU it should leave enough space for the platform driver/runtime. // https://github.com/openxla/xla/blob/3e87afa11a865cf91137522492918ad18bfe5b7c/xla/pjrt/plugin/xla_gpu/xla_gpu_allocator_config.h#L25-L60 - xla_gpu: XlaGpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.90 } } }, + rocm: XlaGpu = .{ .allocator = .{ .vmm = .{ .memory_fraction = 0.90 } } }, + cuda: XlaGpu = .{ .allocator = .{ .vmm = .{ .memory_fraction = 0.90 } } }, tpu: struct {} = .{}, neuron: struct {} = .{}, - oneapi: struct {} = .{}, - metal: struct {} = .{}, + oneapi: XlaGpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.90 } } }, + metal: XlaGpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.90 } } }, pub const Cpu = struct { device_count: u32, @@ -740,6 +741,8 @@ pub const CreateOptions = struct { bfc: Options, /// use cudaMallocAsync async: Options, + /// use virtual memory allocator + vmm: VmmOptions, /// use raw cuMalloc platform, @@ -748,6 +751,10 @@ pub const CreateOptions = struct { memory_fraction: f32 = 0.90, collective_memory_size_mb: i64 = 0, }; + + pub const VmmOptions = struct { + memory_fraction: f32 = 0.90, + }; }; fn writeNamedValues(self: XlaGpu, target: Target, values: *std.ArrayList(pjrt.NamedValue)) void { @@ -759,7 +766,7 @@ pub const CreateOptions = struct { values.appendAssumeCapacity(.init(.string, "allocator", switch (self.allocator) { .bfc => "bfc", .async => "cuda_async", - .platform => unreachable, + .platform, .vmm => unreachable, })); values.appendAssumeCapacity(.init(.bool, "preallocate", opt.preallocate)); if (opt.memory_fraction > 0) { @@ -769,6 +776,12 @@ pub const CreateOptions = struct { values.appendAssumeCapacity(.init(.int64, "collective_memory_size", opt.collective_memory_size_mb * 1024 * 1024)); } }, + .vmm => |opt| { + values.appendAssumeCapacity(.init(.string, "allocator", "vmm")); + if (opt.memory_fraction > 0) { + values.appendAssumeCapacity(.init(.float, "memory_fraction", opt.memory_fraction)); + } + }, } switch (target) { .cuda => values.appendAssumeCapacity(.init(.bool, "use_tfrt_gpu_client", self.gpu_async_dispatch)), @@ -782,7 +795,10 @@ pub const CreateOptions = struct { values.shrinkRetainingCapacity(0); switch (target) { .cpu => self.cpu.writeNamedValues(&values), - .cuda, .rocm, .oneapi, .metal => self.xla_gpu.writeNamedValues(target, &values), + .cuda => self.cuda.writeNamedValues(target, &values), + .rocm => self.rocm.writeNamedValues(target, &values), + .oneapi => self.oneapi.writeNamedValues(target, &values), + .metal => self.metal.writeNamedValues(target, &values), inline else => |t| { stdx.debug.assertComptime(@hasField(CreateOptions, @tagName(t)), "zml.platform.CreateOptions doesn't list target {s}", .{@tagName(t)}); const options = @field(self, @tagName(t)); From 8bae3485f56c1c6375a3252e32509465b912f69c Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 18 Sep 2026 17:22:07 +0200 Subject: [PATCH 705/881] zml/Compiler: enable optimizations flags (#834) * `xla_gpu_experimental_use_collective_kernels=CONCURRENT` allows parallel execution * `xla_gpu_experimental_use_collective_kernels=SKIP_TEMP` avoids graph updates * `xla_gpu_experimental_use_collective_kernels` disables custom collectives while we are looking for a fix * `xla_gpu_enable_command_buffer` adds collectives to the graphs --- zml/Compiler.zig | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/zml/Compiler.zig b/zml/Compiler.zig index b635cc1d..2dfb080a 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -764,8 +764,19 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_scaled_dot_with_tile_ir", true, upb_arena); try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_enable_subchannel_dequantisation_fusion", true, upb_arena); try setXlaOverrideFlag(overrides_map, "xla_gpu_unsupported_enable_triton_multi_output_fusion", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_scheduling_mode", "CONCURRENT", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_update_mode", "SKIP_TEMP", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_collective_kernels", "", upb_arena); + // Add collectives to the default list + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES,CONDITIONAL,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); + }, + .rocm => { + try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_scheduling_mode", "CONCURRENT", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_update_mode", "SKIP_TEMP", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_collective_kernels", "", upb_arena); + // Add collectives to the default list + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES,CONDITIONAL,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); }, - .rocm => {}, .metal => { try setXlaOverrideFlag(overrides_map, "xla_gpu_metal_fast_math", false, upb_arena); }, From c5d476d5e852447f20a6e212ef560dc23d08aadb Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 18 Sep 2026 18:44:39 +0200 Subject: [PATCH 706/881] zml/testing: fix environment init (#836) --- zml/testing.zig | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/zml/testing.zig b/zml/testing.zig index dd8cd2a8..213eae9f 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -14,11 +14,15 @@ var _platform: ?*const Platform = null; pub fn env() *const Platform { if (!builtin.is_test) @compileError("Cannot use zml.testing.env outside of a test block"); if (_platform == null) { + const memory_fraction = 0.90; const platform = Platform.auto( std.heap.c_allocator, std.testing.io, .{ - .xla_gpu = .{ .allocator = .{ .bfc = .{ .preallocate = false, .memory_fraction = 0.85 } } }, + .cuda = .{ .allocator = .{ .vmm = .{ .memory_fraction = memory_fraction } } }, + .rocm = .{ .allocator = .{ .vmm = .{ .memory_fraction = memory_fraction } } }, + .metal = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = memory_fraction } } }, + .oneapi = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = memory_fraction } } }, }, ) catch @panic("Pjrt not available"); From fc0eb7d8788b061a28f57fa3b61c6e71e4fbff2f Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 18 Sep 2026 20:00:56 +0200 Subject: [PATCH 707/881] platforms: bump plugins to 202609181631.57.1.f601577e1ab1 (#837) Notably, adds LTO --- MODULE.bazel | 7 +++ platforms/cpu/BUILD.bazel | 4 ++ platforms/cpu/cpu.bzl | 39 +++++++++++--- platforms/cpu/packages.lock.json | 65 ++++++++++++++++++++++ platforms/cpu/packages.yaml | 16 ++++++ platforms/cuda/cuda.bzl | 17 ++++-- platforms/cuda/libzml_cuda.BUILD.bazel | 4 ++ platforms/cuda/packages.lock.json | 74 ++++++++++++++++++++++---- platforms/cuda/packages.yaml | 3 ++ platforms/metal/metal.bzl | 4 +- platforms/rocm/libzml_rocm.BUILD.bazel | 1 + platforms/rocm/packages.lock.json | 44 ++++++++++++--- platforms/rocm/packages.yaml | 3 ++ platforms/rocm/rocm.bzl | 5 +- 14 files changed, 254 insertions(+), 32 deletions(-) create mode 100644 platforms/cpu/packages.lock.json create mode 100644 platforms/cpu/packages.yaml diff --git a/MODULE.bazel b/MODULE.bazel index 78fef70a..dc1b4f30 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -320,6 +320,13 @@ llvm.configure(targets = ["X86", "NVPTX", "AMDGPU", "AArch64"]) use_repo(llvm, "llvm-project") apt = use_extension("@rules_distroless//apt:extensions.bzl", "apt") +apt.install( + name = "apt_cpu", + lock = "//platforms/cpu:packages.lock.json", + manifest = "//platforms/cpu:packages.yaml", +) +use_repo(apt, "apt_cpu") + apt.install( name = "apt_cuda", lock = "//platforms/cuda:packages.lock.json", diff --git a/platforms/cpu/BUILD.bazel b/platforms/cpu/BUILD.bazel index b592d3bb..418c7837 100644 --- a/platforms/cpu/BUILD.bazel +++ b/platforms/cpu/BUILD.bazel @@ -27,6 +27,10 @@ copy_to_directory( }), out = "sandbox", include_external_repositories = ["**"], + replace_prefixes = { + "libzml_cpu_so": "lib", + "usr/lib/x86_64-linux-gnu": "lib", + }, ) cc_library( diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 39f7bd09..a7fbf624 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -1,4 +1,5 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("//bazel:http_deb_archive.bzl", "http_deb_archive") load("//platforms:packages.bzl", "packages") _BUILD_FILE_DEFAULT_VISIBILITY = """\ @@ -6,9 +7,15 @@ package(default_visibility = ["//visibility:public"]) """ _BUILD_LINUX = "\n".join([ + packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), + packages.patchelf( + name = "libzml_cpu_so", + src = "lib/libzml_cpu.so", + set_rpath = "$ORIGIN", + ), packages.filegroup( name = "libzml_cpu", - srcs = ["lib/libzml_cpu.so"], + srcs = [":libzml_cpu_so", "@llvm-libunwind1//:libunwind"], visibility = ["@zml//platforms/cpu:__subpackages__"], ), ]) @@ -20,30 +27,46 @@ _BUILD_DARWIN = packages.filegroup( ) def _cpu_plugin_impl(mctx): + loaded_packages = packages.read(mctx, ["@zml//platforms/cpu:packages.lock.json"]) + pkg = loaded_packages["llvm-libunwind1"]["amd64"] + http_deb_archive( + name = "llvm-libunwind1", + urls = pkg["urls"], + sha256 = pkg["sha256"], + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + packages.filegroup( + name = "libunwind", + srcs = ["usr/lib/x86_64-linux-gnu/libunwind.so.1"], + ), + ) + http_archive( name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "5d6783ccd775a74dbe8d03e9f35e7a5a422ef051f56bb4266ac918bcb5f56970", - url = "https://mirror.zml.ai/plugins/202609161604.45.1.80cbf79212e9/zml-cpu-linux-amd64.tar.zst", + sha256 = "e825d8d1031ef536f5a1ac2c1eb0a10f8af83168ffb14c3cc8b26aee25ed1f87", + url = "https://mirror.zml.ai/plugins/202609181726.59.1.6ccbaab9cce9/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "3535dee8b26bcf3e1906779b103bcbf8903c88ca93b85487d98fd2181e4f99be", - url = "https://mirror.zml.ai/plugins/202609161604.45.1.80cbf79212e9/zml-cpu-darwin-amd64.tar.zst", + sha256 = "5da35f0e471a410f3a3edba95f8a849ea017c1ed32c6fab0239a25a5d21acad4", + url = "https://mirror.zml.ai/plugins/202609181726.59.1.6ccbaab9cce9/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "bf5c036ba2a8195f8449119fa6138c7b83d7382c5732c2149d74a7b901095ffd", - url = "https://mirror.zml.ai/plugins/202609161604.45.1.80cbf79212e9/zml-cpu-darwin-arm64.tar.zst", + sha256 = "b0f6a8a1486780a9433b08153d71e0e31ed0e4cde60f200346c687c9346b2d53", + url = "https://mirror.zml.ai/plugins/202609181726.59.1.6ccbaab9cce9/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( reproducible = True, - root_module_direct_deps = "all", + root_module_direct_deps = [ + "libzml_cpu_linux_amd64", + "libzml_cpu_darwin_amd64", + "libzml_cpu_darwin_arm64", + ], root_module_direct_dev_deps = [], ) diff --git a/platforms/cpu/packages.lock.json b/platforms/cpu/packages.lock.json new file mode 100644 index 00000000..776c49b1 --- /dev/null +++ b/platforms/cpu/packages.lock.json @@ -0,0 +1,65 @@ +{ + "packages": [ + { + "arch": "amd64", + "dependencies": [ + { + "key": "libc6_2.36-9-p-deb12u10_amd64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + } + ], + "key": "llvm-libunwind1_1-23.1.2_-p--p-20260916013112-p-21a77e7bb6bf-1_exp1_20260916133123.74_amd64", + "name": "llvm-libunwind1", + "sha256": "7fe1e2f61f6db688b53a4015a2d391f4474e0c0aec76b4dc262fe91a43854e84", + "urls": [ + "https://apt.llvm.org/bookworm/pool/main/l/llvm-toolchain-23/llvm-libunwind1_23.1.2~++20260916013112+21a77e7bb6bf-1~exp1~20260916133123.74_amd64.deb" + ], + "version": "1:23.1.2~++20260916013112+21a77e7bb6bf-1~exp1~20260916133123.74" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libc6_2.36-9-p-deb12u10_amd64", + "name": "libc6", + "sha256": "5dc83256f10ca4d0f2a53dd6583ffd0d0e319af30074ea6c82fb0ae77bd16365", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/glibc/libc6_2.36-9+deb12u10_amd64.deb" + ], + "version": "2.36-9+deb12u10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", + "name": "libgcc-s1", + "sha256": "3016e62cb4b7cd8038822870601f5ed131befe942774d0f745622cc77d8a88f7", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/libgcc-s1_12.2.0-14+deb12u1_amd64.deb" + ], + "version": "12.2.0-14+deb12u1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", + "name": "gcc-12-base", + "sha256": "1896a2aacf4ad681ff5eacc24a5b0ca4d5d9c9b9c9e4b6de5197bc1e116ea619", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/gcc-12-base_12.2.0-14+deb12u1_amd64.deb" + ], + "version": "12.2.0-14+deb12u1" + } + ], + "version": 1 +} \ No newline at end of file diff --git a/platforms/cpu/packages.yaml b/platforms/cpu/packages.yaml new file mode 100644 index 00000000..b8ce6251 --- /dev/null +++ b/platforms/cpu/packages.yaml @@ -0,0 +1,16 @@ +# +# bazel run @apt_cpu//:lock +# +version: 1 + +sources: + - channel: bookworm main + url: https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z + - channel: llvm-toolchain-bookworm-23 main + url: https://apt.llvm.org/bookworm + +archs: + - "amd64" + +packages: + - "llvm-libunwind1" # LLVM 23 runtime from llvm-toolchain-bookworm-23. diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index d7634924..de1f6928 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -202,6 +202,15 @@ NVSHMEM_PACKAGES = { } _UBUNTU_PACKAGES = { + "llvm-libunwind1": [ + """filegroup( + name = "libunwind", + srcs = select({ + "@llvm//platforms/config:linux_x86_64": ["usr/lib/x86_64-linux-gnu/libunwind.so.1"], + "@llvm//platforms/config:linux_aarch64": ["usr/lib/aarch64-linux-gnu/libunwind.so.1"], + }), + )""", + ], "zlib1g": [ """filegroup( name = "zlib1g", @@ -215,12 +224,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "45492852cb39295cdc14032c757af2c1edd8a494c737e00be70b5359cc883db5", - "url": "https://mirror.zml.ai/plugins/202609161604.45.1.80cbf79212e9/zml-cuda-linux-amd64.tar.zst", + "sha256": "f10315ea243b67a575be634e9ff012e5d9eb6fead849c9faeba4edf12bd8a19d", + "url": "https://mirror.zml.ai/plugins/202609181726.59.1.6ccbaab9cce9/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "7f1dccc1179073d37270ae905623f5eff1ca0ca9f48cd2e127f0ce2e0e6f1d70", - "url": "https://mirror.zml.ai/plugins/202609161604.45.1.80cbf79212e9/zml-cuda-linux-arm64.tar.zst", + "sha256": "7e3fd96f3d81c95f8da9c8165a67a5e0925393081bbcdfbe0fca361546d0a93d", + "url": "https://mirror.zml.ai/plugins/202609181726.59.1.6ccbaab9cce9/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/cuda/libzml_cuda.BUILD.bazel b/platforms/cuda/libzml_cuda.BUILD.bazel index 71cdb0be..e79ab900 100644 --- a/platforms/cuda/libzml_cuda.BUILD.bazel +++ b/platforms/cuda/libzml_cuda.BUILD.bazel @@ -39,6 +39,7 @@ copy_to_directory( "@libnvjitlink_linux_x86_64//:libnvjitlink", "@libnvshmem_linux_x86_64//:libnvshmem", "@libnvvm_linux_x86_64//:libnvvm", + "@llvm-libunwind1_linux_amd64//:libunwind", "@nccl_linux_amd64//:nccl", "@zlib1g_linux_amd64//:zlib1g", ], @@ -58,6 +59,7 @@ copy_to_directory( "@libnvjitlink_linux_sbsa//:libnvjitlink", "@libnvshmem_linux_sbsa//:libnvshmem", "@libnvvm_linux_sbsa//:libnvvm", + "@llvm-libunwind1_linux_arm64//:libunwind", "@nccl_linux_arm64//:nccl", "@zlib1g_linux_arm64//:zlib1g", ], @@ -71,6 +73,8 @@ copy_to_directory( "libzml_cuda_so": "lib", "lib/x86_64-linux-gnu": "lib", "lib/aarch64-linux-gnu": "lib", + "usr/lib/x86_64-linux-gnu": "lib", + "usr/lib/aarch64-linux-gnu": "lib", "platforms/cuda/compat_probe": "bin/compat_probe", "platforms/cuda/libzmlxcuda": "lib/libzmlxcuda", } | { diff --git a/platforms/cuda/packages.lock.json b/platforms/cuda/packages.lock.json index 1c39a387..f3f7c4b2 100755 --- a/platforms/cuda/packages.lock.json +++ b/platforms/cuda/packages.lock.json @@ -19,13 +19,13 @@ "version": "12.2.0-14+deb12u1" } ], - "key": "zlib1g_1-1.2.13.dfsg-1_amd64", - "name": "zlib1g", - "sha256": "d7dd1d1411fedf27f5e27650a6eff20ef294077b568f4c8c5e51466dc7c08ce4", + "key": "llvm-libunwind1_1-23.1.2_-p--p-20260916013112-p-21a77e7bb6bf-1_exp1_20260916133123.74_amd64", + "name": "llvm-libunwind1", + "sha256": "7fe1e2f61f6db688b53a4015a2d391f4474e0c0aec76b4dc262fe91a43854e84", "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_amd64.deb" + "https://apt.llvm.org/bookworm/pool/main/l/llvm-toolchain-23/llvm-libunwind1_23.1.2~++20260916013112+21a77e7bb6bf-1~exp1~20260916133123.74_amd64.deb" ], - "version": "1:1.2.13.dfsg-1" + "version": "1:23.1.2~++20260916013112+21a77e7bb6bf-1~exp1~20260916133123.74" }, { "arch": "amd64", @@ -60,6 +60,33 @@ ], "version": "12.2.0-14+deb12u1" }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "libc6_2.36-9-p-deb12u10_amd64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + } + ], + "key": "zlib1g_1-1.2.13.dfsg-1_amd64", + "name": "zlib1g", + "sha256": "d7dd1d1411fedf27f5e27650a6eff20ef294077b568f4c8c5e51466dc7c08ce4", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_amd64.deb" + ], + "version": "1:1.2.13.dfsg-1" + }, { "arch": "arm64", "dependencies": [ @@ -79,13 +106,13 @@ "version": "12.2.0-14+deb12u1" } ], - "key": "zlib1g_1-1.2.13.dfsg-1_arm64", - "name": "zlib1g", - "sha256": "52b8b8a145bbe1956bba82034f77022cbef0c3d0885c9e32d9817a7932fe1913", + "key": "llvm-libunwind1_1-23.1.2_-p--p-20260916013112-p-21a77e7bb6bf-1_exp1_20260916133123.74_arm64", + "name": "llvm-libunwind1", + "sha256": "d5643512d5108eb0320486a740879223632fb8058ce321654d66604ba4a0439f", "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_arm64.deb" + "https://apt.llvm.org/bookworm/pool/main/l/llvm-toolchain-23/llvm-libunwind1_23.1.2~++20260916013112+21a77e7bb6bf-1~exp1~20260916133123.74_arm64.deb" ], - "version": "1:1.2.13.dfsg-1" + "version": "1:23.1.2~++20260916013112+21a77e7bb6bf-1~exp1~20260916133123.74" }, { "arch": "arm64", @@ -119,6 +146,33 @@ "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/gcc-12-base_12.2.0-14+deb12u1_arm64.deb" ], "version": "12.2.0-14+deb12u1" + }, + { + "arch": "arm64", + "dependencies": [ + { + "key": "libc6_2.36-9-p-deb12u10_arm64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_arm64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_arm64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + } + ], + "key": "zlib1g_1-1.2.13.dfsg-1_arm64", + "name": "zlib1g", + "sha256": "52b8b8a145bbe1956bba82034f77022cbef0c3d0885c9e32d9817a7932fe1913", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_arm64.deb" + ], + "version": "1:1.2.13.dfsg-1" } ], "version": 1 diff --git a/platforms/cuda/packages.yaml b/platforms/cuda/packages.yaml index aaa3c42f..6828e7b5 100644 --- a/platforms/cuda/packages.yaml +++ b/platforms/cuda/packages.yaml @@ -6,10 +6,13 @@ version: 1 sources: - channel: bookworm main url: https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z + - channel: llvm-toolchain-bookworm-23 main + url: https://apt.llvm.org/bookworm archs: - "amd64" - "arm64" packages: + - "llvm-libunwind1" # LLVM 23 runtime from llvm-toolchain-bookworm-23. - "zlib1g" diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index ade601ee..b498d6a3 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "aa67157ffd996e1a244a7a79c940cf15ac97919f0104617e70f523155d16388a", - url = "https://mirror.zml.ai/plugins/202609161604.45.1.80cbf79212e9/zml-metal-darwin-arm64.tar.zst", + sha256 = "135fb7a562efb23ed25cad21cdedd33c56aaa9e7a202f928d46176047ee0234c", + url = "https://mirror.zml.ai/plugins/202609181726.59.1.6ccbaab9cce9/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/libzml_rocm.BUILD.bazel b/platforms/rocm/libzml_rocm.BUILD.bazel index 0a18c130..7996b001 100644 --- a/platforms/rocm/libzml_rocm.BUILD.bazel +++ b/platforms/rocm/libzml_rocm.BUILD.bazel @@ -70,6 +70,7 @@ copy_to_directory( "@amdrocm-sparse//:rocsparse", "@amdrocm-sysdeps//:system_libs", "@libatomic1", + "@llvm-libunwind1//:libunwind", "@zml//platforms/rocm:zmlxrocm", ], add_directory_to_runfiles = True, diff --git a/platforms/rocm/packages.lock.json b/platforms/rocm/packages.lock.json index 6dce1849..8eb071d7 100644 --- a/platforms/rocm/packages.lock.json +++ b/platforms/rocm/packages.lock.json @@ -24,13 +24,13 @@ "version": "12.3.0-1ubuntu1~22.04" } ], - "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libatomic1", - "sha256": "56ec991983c7c89a2c15bd5134534531d722ec5b51a60679f7ceaa1a09b37a09", + "key": "llvm-libunwind1_1-23.1.2_-p--p-20260916113350-p-ba019b7aafce-1_exp1_20260916233400.76_amd64", + "name": "llvm-libunwind1", + "sha256": "72f83f34f22e01e1219c9600f9e8562505afc1eabfb5b0cfae5f4603d0694720", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libatomic1_12.3.0-1ubuntu1~22.04_amd64.deb" + "https://apt.llvm.org/jammy/pool/main/l/llvm-toolchain-23/llvm-libunwind1_23.1.2~++20260916113350+ba019b7aafce-1~exp1~20260916233400.76_amd64.deb" ], - "version": "12.3.0-1ubuntu1~22.04" + "version": "1:23.1.2~++20260916113350+ba019b7aafce-1~exp1~20260916233400.76" }, { "arch": "amd64", @@ -76,6 +76,38 @@ ], "version": "12.3.0-1ubuntu1~22.04" }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + } + ], + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "sha256": "56ec991983c7c89a2c15bd5134534531d722ec5b51a60679f7ceaa1a09b37a09", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libatomic1_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, { "arch": "amd64", "dependencies": [], @@ -8855,4 +8887,4 @@ } ], "version": 1 -} +} \ No newline at end of file diff --git a/platforms/rocm/packages.yaml b/platforms/rocm/packages.yaml index 57673100..d3c8bbcd 100644 --- a/platforms/rocm/packages.yaml +++ b/platforms/rocm/packages.yaml @@ -4,6 +4,8 @@ version: 1 sources: + - channel: llvm-toolchain-jammy-23 main + url: https://apt.llvm.org/jammy - channel: stable main url: https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391 - channel: jammy main @@ -17,6 +19,7 @@ archs: - "amd64" packages: + - "llvm-libunwind1" # LLVM 23 runtime from llvm-toolchain-jammy-23. - "libatomic1" - "libdrm-common" - "amdrocm-amdsmi10.1" diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index f1287726..c0b5bbb3 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202609161604.45.1.80cbf79212e9/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "11f8afa002ac7d26a640577e74f984f71aa4f1b9eb657c587b21c8348baf5f18" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202609181726.59.1.6ccbaab9cce9/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "f9aa7ed2c7082972192740bd3f292eed8bff98f4561c221dbbb7413b433f04ac" def _rocm_package_name(name): return name + _ROCM_VERSION @@ -21,6 +21,7 @@ def _rocm_repo_name(package_name): return package_name.replace(_ROCM_VERSION, "") _UBUNTU_PACKAGES = { + "llvm-libunwind1": packages.filegroup(name = "libunwind", srcs = ["usr/lib/x86_64-linux-gnu/libunwind.so.1"]), "libatomic1": packages.filegroup(name = "libatomic1", srcs = ["usr/lib/x86_64-linux-gnu/libatomic.so.1"]), "libdrm-common": packages.filegroup(name = "amdgpu_ids", srcs = ["usr/share/libdrm/amdgpu.ids"]), } From 63d8204be27d61b4ea0e5b5a8b846005c66436a2 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 21 Sep 2026 18:59:29 +0200 Subject: [PATCH 708/881] zml: add Platform.io_impl to enable threaded or zio at runtime (#841) This enables runtime selection of the std.Io runtime. This also fixes the compilation hangs by using zio.blockInPlace for CPU bound ops. This may require some more `zio.blockInPlace`, so don't merge until we're sure. --- MODULE.bazel | 7 +++- third_party/zio/repo.bzl | 2 +- zml/BUILD.bazel | 17 ---------- zml/Compiler.zig | 56 +++++++++++++++++-------------- zml/attention/paged_attention.zig | 4 ++- zml/platform.zig | 10 +++++- 6 files changed, 50 insertions(+), 46 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index dc1b4f30..0c2f9217 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -316,7 +316,12 @@ use_repo( ) llvm = use_extension("//third_party/xla:llvm.bzl", "llvm") -llvm.configure(targets = ["X86", "NVPTX", "AMDGPU", "AArch64"]) +llvm.configure(targets = [ + "X86", + "NVPTX", + "AMDGPU", + "AArch64", +]) use_repo(llvm, "llvm-project") apt = use_extension("@rules_distroless//apt:extensions.bzl", "apt") diff --git a/third_party/zio/repo.bzl b/third_party/zio/repo.bzl index 76280f54..54aca677 100644 --- a/third_party/zio/repo.bzl +++ b/third_party/zio/repo.bzl @@ -4,7 +4,7 @@ def repo(): git_repository( name = "zio", remote = "https://github.com/lalinsky/zio", - commit = "54cb18f05d3feaec431353de85d5d10dac93d37b", + commit = "417757935ef6d0f0a2396d4023a44ca82ac1901c", build_file = Label("//third_party/zio:zio.bazel"), patches = [Label("//third_party/zio:progress-parent-file.patch")], patch_args = ["-p1"], diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 81fc1146..57adbdae 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -1,22 +1,6 @@ load("@com_google_protobuf//bazel:upb_c_proto_library.bzl", "upb_c_proto_library") load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") -load("//bazel:zig_options.bzl", "zig_option", "zig_options") - -zig_option( - name = "io_impl", - build_setting_default = "std", - values = [ - "std", - "zio", - ], -) - -zig_options( - name = "zml_options", - import_name = "zml/options", - settings = [":io_impl"], -) upb_c_proto_library( name = "xla_data_upb", @@ -164,7 +148,6 @@ zig_library( ":profiling_traceme_lib", ":xla_compile_options_upb", ":xla_data_upb", - ":zml_options", "//bazel", "//ffi:zig", "//kernels/cuda_tile:cuda_tile_builder", diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 2dfb080a..7400c842 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -7,7 +7,6 @@ const pjrt = @import("pjrt"); const stdx = @import("stdx"); const upb = @import("upb"); const zio = @import("zio"); -const zml_options = @import("zml/options"); const Buffer = @import("buffer.zig").Buffer; const DataType = @import("dtype.zig").DataType; @@ -46,7 +45,6 @@ manual_computation_depth: usize = 0, channel_id: i64 = 0, composite_id: i64 = 0, -var _current_zio: zio.TaskLocal(*Compiler) = .{}; threadlocal var _current: ?*Compiler = null; var mlir_global_init_mutex: std.Io.Mutex = .init; @@ -186,12 +184,7 @@ pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, } pub fn deinit(self: *Compiler) void { - switch (zml_options.io_impl) { - .std => { - if (_current == self) _current = null; - }, - .zio => {}, - } + if (_current == self) _current = null; std.debug.assert(self.scopes.len == 0); self.mlir_pass_manager.deinit(); self.module.deinit(); @@ -214,10 +207,7 @@ pub fn current() *Compiler { } pub fn currentOrNull() ?*Compiler { - return switch (zml_options.io_impl) { - .std => _current, - .zio => _current_zio.get(), - }; + return _current; } pub fn currentScope(self: *Compiler) *Scope { @@ -286,6 +276,24 @@ pub fn compile( comptime func: anytype, args: std.meta.ArgsTuple(@TypeOf(func)), opts: Options, +) Error!Exe { + return switch (platform.io_impl) { + .threaded => compileInternal(allocator, io, platform, func, args, opts), + .zio => try zio.blockInPlace(struct { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const Platform, args_: @TypeOf(args), opts_: Options) Error!Exe { + return try compileInternal(allocator_, io_, platform_, func, args_, opts_); + } + }.call, .{ allocator, io, platform, args, opts }), + }; +} + +pub fn compileInternal( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + comptime func: anytype, + args: std.meta.ArgsTuple(@TypeOf(func)), + opts: Options, ) Error!Exe { // TODO: Here we have somewhat of a requirement // Emitting MLIR requires to have the compiler context available at all times using `Compiler.current()`. @@ -294,22 +302,23 @@ pub fn compile( // I think the correct implementation would be to dispatch `emitMlir` to a thread pool, then wait for the result // asynchronously using the provided Io. For now, we'll simply make that blocking as it's not a big deal but keep // in mind we might want to revisit that later. - _ = io; - var st_io: std.Io.Threaded = .init_single_threaded; - defer st_io.deinit(); + // + // This is NOT compatible with std.Io.Evented. + // It is compatible with zio, hence the blockInPlace above. const span_name = try tracer.formatSpanName(allocator, "zml.module.compile", .{ .program_name = opts.program_name, .arg_count = args.len, }); + defer allocator.free(span_name); var span = tracer.Span.start(span_name); defer span.end(); - var compiler: Compiler = .init(allocator, st_io.io(), platform, opts); + var compiler: Compiler = .init(allocator, io, platform, opts); defer compiler.deinit(); - var result = emitMlir(&compiler, func, args) catch |err| switch (err) { + var result = compiler.emitMlir(func, args) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => unreachable, }; @@ -343,7 +352,7 @@ pub fn compile( var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); - const loaded_executable = try compileModuleToPjrtExecutable(arena.allocator(), st_io.io(), platform, compiler.module, compiler.partitioning, opts); + const loaded_executable = try compileModuleToPjrtExecutable(arena.allocator(), io, platform, compiler.module, compiler.partitioning, opts); log.debug("\n******** ZML generated MLIR ********\n{f}", .{compiler.module.operation()}); const exe = try Exe.init( @@ -455,13 +464,10 @@ fn emitMlir(compiler: *Compiler, comptime func: anytype, args: std.meta.ArgsTupl var input_info = try createBlockArguments(compiler, fn_scope, &args); errdefer input_info.deinit(compiler.allocator); - var result = switch (zml_options.io_impl) { - .std => blk: { - compiler.activate(); - defer compiler.deactivate(); - break :blk @call(.auto, func, args); - }, - .zio => _current_zio.scoped(compiler, func, args), + var result = blk: { + compiler.activate(); + defer compiler.deactivate(); + break :blk @call(.auto, func, args); }; var output_info = try collectOutputInfo(compiler, fn_scope, &result); diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index e80885a8..d3a378ae 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -5,9 +5,9 @@ const stdx = @import("stdx"); const zml = @import("../zml.zig"); const flashattn = @import("flashattn.zig"); const metal = @import("metal_attention.zig"); +const sparse_mla = @import("sparse_mla.zig"); const tpu = @import("tpu_attention.zig"); const triton = @import("triton_attention.zig"); -const sparse_mla = @import("sparse_mla.zig"); const PagedAttention = @This(); @@ -322,6 +322,7 @@ test "Backend.auto selects mosaic_tpu on TPU" { .physical_mesh = undefined, .replicated_sharding = undefined, .shardings = .empty, + .io_impl = .threaded, }; try std.testing.expectEqual(Backend.mosaic_tpu, Backend.auto(&platform)); @@ -339,6 +340,7 @@ test "Backend.auto selects triton on oneAPI" { .physical_mesh = undefined, .replicated_sharding = undefined, .shardings = .empty, + .io_impl = .threaded, }; try std.testing.expectEqual(Backend.triton, Backend.auto(&platform)); diff --git a/zml/platform.zig b/zml/platform.zig index 90917339..1706309e 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -3,8 +3,8 @@ const builtin = @import("builtin"); const c = @import("c"); const pjrt = @import("pjrt"); -pub const Target = @import("platforms").Platform; const stdx = @import("stdx"); +pub const Target = @import("platforms").Platform; const attention = @import("attention.zig"); const constants = @import("constants.zig"); @@ -285,9 +285,15 @@ pub const Platform = struct { physical_mesh: zml.Sharding.PhysicalMesh, replicated_sharding: zml.Sharding, shardings: std.StringArrayHashMapUnmanaged(zml.Sharding), + io_impl: IoImpl, pub const MAX_NUM_DEVICES: u16 = if (Target.tpu.isEnabled()) 64 else 32; + pub const IoImpl = enum { + threaded, + zio, + }; + pub fn init(allocator: std.mem.Allocator, io: std.Io, target: Target, options: CreateOptions) !*Platform { const api = try loadOrGetApi(allocator, io, target); @@ -324,6 +330,7 @@ pub const Platform = struct { .memories = undefined, .physical_mesh = undefined, .replicated_sharding = undefined, + .io_impl = options.io_impl, }; break :platform platform; }; @@ -715,6 +722,7 @@ pub const CreateOptions = struct { neuron: struct {} = .{}, oneapi: XlaGpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.90 } } }, metal: XlaGpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.90 } } }, + io_impl: Platform.IoImpl = .threaded, pub const Cpu = struct { device_count: u32, From d0cf9562a2d01720c0d4a3ab2e9ed9013b8a6281 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 21 Sep 2026 19:26:09 +0200 Subject: [PATCH 709/881] zml: enable nccl user buffers by default With SKIP_TEMP, captured NCCL collectives can retain stale physical mappings if VMM reclaims their backing under memory pressure, causing silent corruption. Enable user buffers to assign color 1 and receive the reclaim exemption: https://github.com/openxla/xla/pull/46029 --- zml/Compiler.zig | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 7400c842..778fb21d 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -775,6 +775,11 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_collective_kernels", "", upb_arena); // Add collectives to the default list try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES,CONDITIONAL,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); + // With SKIP_TEMP, captured NCCL collectives can retain stale physical mappings + // if VMM reclaims their backing under memory pressure, causing silent corruption. + // Enable user buffers to assign color 1 and receive the reclaim exemption: + // https://github.com/openxla/xla/pull/46029 + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_nccl_user_buffers", true, upb_arena); }, .rocm => { try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_scheduling_mode", "CONCURRENT", upb_arena); @@ -782,6 +787,11 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_collective_kernels", "", upb_arena); // Add collectives to the default list try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES,CONDITIONAL,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); + // With SKIP_TEMP, captured NCCL collectives can retain stale physical mappings + // if VMM reclaims their backing under memory pressure, causing silent corruption. + // Enable user buffers to assign color 1 and receive the reclaim exemption: + // https://github.com/openxla/xla/pull/46029 + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_nccl_user_buffers", true, upb_arena); }, .metal => { try setXlaOverrideFlag(overrides_map, "xla_gpu_metal_fast_math", false, upb_arena); From 397aa43e0b11f34eabae0a7d9ca5e548bcc0d1a1 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 21 Sep 2026 19:42:46 +0200 Subject: [PATCH 710/881] pjrt: properly interpret error when an event is ready (#844) A ready event can still contain an error that we need to report. --- pjrt/pjrt.zig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 8efdeae3..17339f66 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -1372,7 +1372,8 @@ pub const Event = opaque { pub fn await(self: *Event, api: *const Api, io: std.Io) ApiError!void { if (self.isReady(api)) { - return; + const err = self.getEventError(api) orelse return; + return interpretPjrtError(api, err, "PJRT_Event_Await"); } const Ctx = struct { From 71396f389a5cb5e85d46bbe2eab4a1e2acf5086d Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Tue, 22 Sep 2026 16:31:08 +0200 Subject: [PATCH 711/881] zml/moe: fix cutlass expert local routing (#845) Prevent non local experts to fallback on expert 0. --- zml/moe/moe.zig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index f4a16714..ded94167 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -284,11 +284,11 @@ pub fn forwardMoe( .logical(.AND, self.topk_ids.cmp(.LT, expert_end)); const local_topk_ids = local_route_mask.select( self.topk_ids.sub(expert_start), - zml.Tensor.scalar(0, .i32), + zml.Tensor.scalar(-1, .i32), ); const local_topk_weights = local_route_mask.select( self.topk_weights, - zml.Tensor.scalar(0, self.topk_weights.dtype()), + zml.Tensor.scalar(-1, self.topk_weights.dtype()), ); const local_output = cutlass_flashinfer.fusedExpertsNvfp4( @@ -386,11 +386,11 @@ pub fn forwardMoe( .logical(.AND, self.topk_ids.cmp(.LT, expert_end)); const local_topk_ids = local_route_mask.select( self.topk_ids.sub(expert_start), - zml.Tensor.scalar(0, .i32), + zml.Tensor.scalar(-1, .i32), ); const local_topk_weights = local_route_mask.select( self.topk_weights, - zml.Tensor.scalar(0, self.topk_weights.dtype()), + zml.Tensor.scalar(-1, self.topk_weights.dtype()), ); const local_output = cutlass_flashinfer.fusedExpertsBf16( From ca9a342377bedf1de3d05ea0a63b87658b650e11 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 22 Sep 2026 21:05:06 +0200 Subject: [PATCH 712/881] platforms: update plugins and tune ROCm graph execution (#848) Update CPU, CUDA, Metal, and ROCm plugin references to release 202609221707.75.1.0cf3c5a71e94. Default ROCm to one HIP graph stream and one hardware queue per device to reduce synchronization overhead and throughput variability. Disable rocprofiler queue interposition by default, while preserving explicit environment overrides. Restrict collective command-buffer capture to all-reduce and all-gather. Other collective types execute outside HIP graphs. --- platforms/cpu/cpu.bzl | 6 +++--- platforms/cuda/cuda.bzl | 8 ++++---- platforms/metal/metal.bzl | 4 ++-- platforms/rocm/rocm.bzl | 4 ++-- platforms/rocm/rocm.zig | 7 +++++++ zml/Compiler.zig | 2 ++ 6 files changed, 20 insertions(+), 11 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index a7fbf624..89b44492 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -43,21 +43,21 @@ def _cpu_plugin_impl(mctx): name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, sha256 = "e825d8d1031ef536f5a1ac2c1eb0a10f8af83168ffb14c3cc8b26aee25ed1f87", - url = "https://mirror.zml.ai/plugins/202609181726.59.1.6ccbaab9cce9/zml-cpu-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202609221707.75.1.0cf3c5a71e94/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "5da35f0e471a410f3a3edba95f8a849ea017c1ed32c6fab0239a25a5d21acad4", - url = "https://mirror.zml.ai/plugins/202609181726.59.1.6ccbaab9cce9/zml-cpu-darwin-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202609221707.75.1.0cf3c5a71e94/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "b0f6a8a1486780a9433b08153d71e0e31ed0e4cde60f200346c687c9346b2d53", - url = "https://mirror.zml.ai/plugins/202609181726.59.1.6ccbaab9cce9/zml-cpu-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202609221707.75.1.0cf3c5a71e94/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index de1f6928..f173e8ab 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -224,12 +224,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "f10315ea243b67a575be634e9ff012e5d9eb6fead849c9faeba4edf12bd8a19d", - "url": "https://mirror.zml.ai/plugins/202609181726.59.1.6ccbaab9cce9/zml-cuda-linux-amd64.tar.zst", + "sha256": "a36251f0e8dbff5135a7434d7ae89ec2ec50767825400f38e9ec7578937db043", + "url": "https://mirror.zml.ai/plugins/202609221707.75.1.0cf3c5a71e94/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "7e3fd96f3d81c95f8da9c8165a67a5e0925393081bbcdfbe0fca361546d0a93d", - "url": "https://mirror.zml.ai/plugins/202609181726.59.1.6ccbaab9cce9/zml-cuda-linux-arm64.tar.zst", + "sha256": "a73bc23e230df4ae3dce6a709b0006dfd17f968d8cec6474f833a8409542b72f", + "url": "https://mirror.zml.ai/plugins/202609221707.75.1.0cf3c5a71e94/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index b498d6a3..9c8f628f 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "135fb7a562efb23ed25cad21cdedd33c56aaa9e7a202f928d46176047ee0234c", - url = "https://mirror.zml.ai/plugins/202609181726.59.1.6ccbaab9cce9/zml-metal-darwin-arm64.tar.zst", + sha256 = "3797e34249e59175665e36370b7b71a6983ccc28cdc272fafa1d25424a47f5d4", + url = "https://mirror.zml.ai/plugins/202609221707.75.1.0cf3c5a71e94/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index c0b5bbb3..4f94d748 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202609181726.59.1.6ccbaab9cce9/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "f9aa7ed2c7082972192740bd3f292eed8bff98f4561c221dbbb7413b433f04ac" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202609221707.75.1.0cf3c5a71e94/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "c5aacc6e6f5c14b1b65c352e45c329a4481de1b152794988a3c243325c82da7c" def _rocm_package_name(name): return name + _ROCM_VERSION diff --git a/platforms/rocm/rocm.zig b/platforms/rocm/rocm.zig index 4f054caa..402d5504 100644 --- a/platforms/rocm/rocm.zig +++ b/platforms/rocm/rocm.zig @@ -25,6 +25,13 @@ fn hasRocmDevices(io: std.Io) bool { fn setupRocmEnv(rocm_data_dir: []const u8) !void { var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; _ = c.setenv("ROCM_PATH", try stdx.Io.Dir.path.bufJoinZ(&buf, &.{rocm_data_dir}), 1); // must be zero terminated + + // Use one HIP graph stream to reduce cross-stream synchronization overhead. + _ = c.setenv("DEBUG_HIP_FORCE_GRAPH_QUEUES", "1", 0); + // Share one hardware queue per device so auxiliary barriers stay on the kernel queue. + _ = c.setenv("GPU_MAX_HW_QUEUES", "1", 0); + // Disable rocprofiler queue interposition to avoid its dispatch overhead outside profiling. + _ = c.setenv("ROCPROFILER_QUEUE_INTERPOSITION", "0", 0); } pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 778fb21d..4d885473 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -785,6 +785,8 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_scheduling_mode", "CONCURRENT", upb_arena); try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_update_mode", "SKIP_TEMP", upb_arena); try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_collective_kernels", "", upb_arena); + // Only capture all-reduce and all-gather collectives in HIP graphs; other collective types execute outside command buffers since they are unsupported in HIP graphs. + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_collectives_command_buffer_filter", "ALLREDUCE,ALLGATHER", upb_arena); // Add collectives to the default list try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES,CONDITIONAL,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); // With SKIP_TEMP, captured NCCL collectives can retain stale physical mappings From e1356ee4edbac0548d7d25ad6590841b5cd9293b Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 23 Sep 2026 00:17:41 +0200 Subject: [PATCH 713/881] zml: Compiler.pushLoc (#846) Add explicit debug info in ZML. Model author can now add `pushLocation` to mark specific sections of the IR --------- Co-authored-by: Guillaume Wenzek --- mlir/dialects/stablehlo/stablehlo.zig | 3 +- mlir/mlir.zig | 75 ++++++++++++- stdx/meta.zig | 21 ++++ zml/Compiler.zig | 137 ++++++++++++++++++++++- zml/kernel.zig | 2 +- zml/ops.zig | 151 +++++++++++++++----------- zml/tensor.zig | 131 +++++++++++----------- 7 files changed, 387 insertions(+), 133 deletions(-) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index ad4b5dbb..efae3018 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -1474,6 +1474,7 @@ pub fn allReduce( reducer_block: *mlir.Block, replica_groups: *const mlir.Attribute, channel_handle: *const mlir.Attribute, + location: *const mlir.Location, ) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.all_reduce", .{ .operands = .{ .variadic = &.{inputs} }, @@ -1485,6 +1486,6 @@ pub fn allReduce( .named(ctx, "use_global_device_ids", .unit(ctx)), }, .verify = true, - .location = .unknown(ctx), + .location = location, }); } diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 25270d8d..d759dbfa 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -348,7 +348,7 @@ pub const Location = opaque { pub fn namedFmt(loc: *const Location, ctx: *Context, comptime fmt: []const u8, args: anytype) *const Location { var buf: [256]u8 = undefined; - return loc.named(loc, ctx, std.fmt.bufPrint(&buf, fmt, args) catch blk: { + return loc.named(ctx, std.fmt.bufPrint(&buf, fmt, args) catch blk: { buf[buf.len - 3 ..].* = "...".*; break :blk &buf; }); @@ -361,6 +361,77 @@ pub const Location = opaque { pub fn unknown(ctx: *Context) *const Location { return @ptrCast(c.mlirLocationUnknownGet(ctx.ptr()).ptr); } + + pub const Tagged = union(enum) { + unknown, + named: *const Named, + callsite: *const Callsite, + file_line_col: *const FileLineCol, + fused: *const Fused, + }; + + pub fn inspect(location: *const Location) Tagged { + const loc_ptr = location.ptr(); + return if (c.mlirLocationIsAUnknown(loc_ptr)) + .unknown + else if (c.mlirLocationIsAName(loc_ptr)) + .{ .named = @ptrCast(location) } + else if (c.mlirLocationIsACallSite(loc_ptr)) + .{ .callsite = @ptrCast(location) } + else if (c.mlirLocationIsAFileLineColRange(loc_ptr)) + .{ .file_line_col = @ptrCast(location) } + else if (c.mlirLocationIsAFused(loc_ptr)) + .{ .fused = @ptrCast(location) } + else + // Theoritically the C++ API has "OpaqueLoc" but it's not part of C bindings, + // I think in this case it's fine to fall back to .unknown + // https://mlir.llvm.org/docs/Dialects/Builtin/#opaqueloc + .unknown; + } + + pub const Named = opaque { + pub fn name(location: *const Named) []const u8 { + const identifier: *const Identifier = @ptrCast(c.mlirLocationNameGetName(.{ .ptr = location }).ptr); + return identifier.str(); + } + + pub fn childLoc(location: *const Named) *const Location { + return @ptrCast(c.mlirLocationNameGetChildLoc(.{ .ptr = location }).ptr); + } + }; + + pub const Callsite = opaque { + pub fn callee(location: *const Callsite) *const Location { + return @ptrCast(c.mlirLocationCallSiteGetCallee(.{ .ptr = location }).ptr); + } + + pub fn caller(location: *const Callsite) *const Location { + return @ptrCast(c.mlirLocationCallSiteGetCaller(.{ .ptr = location }).ptr); + } + }; + + pub const FileLineCol = opaque { + pub fn src(location: *const FileLineCol) *const std.builtin.SourceLocation { + const file_id: *const Identifier = @ptrCast(c.mlirLocationFileLineColRangeGetFilename(.{ .ptr = location }).ptr); + return .{ + .file = file_id.str(), + .line = @truncate(c.mlirLocationFileLineColRangeGetStartLine(.{ .ptr = location })), + .column = @truncate(c.mlirLocationFileLineColRangeGetStartColumn(.{ .ptr = location })), + }; + } + }; + + pub const Fused = opaque { + pub fn len(location: *const Fused) u32 { + return @intCast(c.mlirLocationFusedGetNumLocations(.{ .ptr = location })); + } + + pub fn locations(location: *const Fused, allocator: std.mem.Allocator) error{OutOfMemory}![]*const Location { + const result = try allocator.alloc(*const Location, location.len()); + c.mlirLocationFusedGetLocations(.{ .ptr = location }, @ptrCast(result.ptr)); + return result; + } + }; }; pub const Type = opaque { @@ -1304,7 +1375,7 @@ pub const Operation = opaque { attributes: ?[]const NamedAttribute = null, blocks: ?[]const *Block = null, verify: bool = true, - location: ?*const Location = null, + location: ?*const Location, }; pub fn try_make(ctx: *Context, name_: []const u8, args: MakeArgs) !*Operation { diff --git a/stdx/meta.zig b/stdx/meta.zig index afaaa917..cb9aaeba 100644 --- a/stdx/meta.zig +++ b/stdx/meta.zig @@ -187,3 +187,24 @@ pub fn Tail(comptime Tuple: type) type { else => @compileError("Tail works on tuple type"), }; } + +pub fn fnName(comptime func: anytype) []const u8 { + const T = @TypeOf(func); + std.debug.assert(@typeInfo(T) == .@"fn"); + const wrapper_name = @typeName(Wrapper(func)); + + var it = std.mem.splitSequence(u8, wrapper_name, ".Wrapper((function '"); + _ = it.first(); + const rest = it.next().?; + return rest[0 .. rest.len - 3]; +} + +fn Wrapper(comptime arg: anytype) type { + return struct { + const stuff = arg; + }; +} + +test fnName { + try std.testing.expectEqualStrings("Tail", fnName(Tail)); +} diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 4d885473..619676c1 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -24,7 +24,7 @@ const Partitioning = Sharding.Partitioning; const Tensor = @import("tensor.zig").Tensor; const Compiler = @This(); -const log = std.log.scoped(.@"zml/compiler"); +const log = std.log.scoped(.@"zml/Compiler"); allocator: std.mem.Allocator, io: std.Io, @@ -41,6 +41,8 @@ mlir_known_types: std.enums.EnumArray(DataType, *const mlir.Type), scopes: stdx.BoundedArray(Scope, 16) = .empty, manual_computation_depth: usize = 0, +unknown_location: *const mlir.Location, +location: *const mlir.Location, channel_id: i64 = 0, composite_id: i64 = 0, @@ -135,7 +137,8 @@ pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, var mlir_ctx = mlir.Context.init(.{ .registry = mlir_registry, .threading = false }) catch unreachable; mlir_ctx.loadAllAvailableDialects(); - const module = mlir.Module.init(.unknown(mlir_ctx)); + const unknown_location: *const mlir.Location = .unknown(mlir_ctx); + const module = mlir.Module.init(unknown_location); module.operation().setAttributeByName("sym_name", .string(mlir_ctx, opts.program_name)); const pass_manager = mlir.PassManager.init(mlir_ctx); @@ -180,6 +183,8 @@ pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, .module = module, .platform = platform, .partitioning = partitioning, + .location = unknown_location, + .unknown_location = unknown_location, }; } @@ -220,6 +225,74 @@ pub fn pushBlock(self: *Compiler, block: *mlir.Block) *Scope { return self.currentScope(); } +/// Change `Compiler.location` by appending a frame to the current stack of locations +/// Must be followed by a `defer compiler.popLocation` +pub fn pushLocation(self: *Compiler, location: std.builtin.SourceLocation, name: []const u8) void { + var callee: *const mlir.Location = .fromSrc(self.mlir_ctx, location); + if (name.len > 0) callee = callee.named(self.mlir_ctx, name); + self.location = if (self.location == self.unknown_location) callee else .callSite(callee, self.location); +} + +/// see `zml.Compiler.pushLocation` +pub fn pushLocationFmt(self: *Compiler, location: std.builtin.SourceLocation, comptime fmt: []const u8, args: anytype) void { + const name = std.fmt.allocPrint(self.allocator, fmt, args) catch { + return self.pushLocation(location, ""); + }; + defer self.allocator.free(name); + + return self.pushLocation(location, name); +} + +/// Remove last call frame pushed by `zml.Compiler.pushLocation` +pub fn popLocation(self: *Compiler) void { + switch (self.location.inspect()) { + .named => { + // We only pushed a named, when there was no parent location + self.location = self.unknown_location; + return; + }, + .callsite => |callsite| { + self.location = callsite.caller(); + return; + }, + .unknown, .file_line_col, .fused => @panic("unexpected location metadata"), + } +} + +test pushLocation { + const mlir_registry = mlirRegistry(std.testing.io); + var mlir_ctx = mlir.Context.init(.{ .registry = mlir_registry, .threading = false }) catch unreachable; + mlir_ctx.loadAllAvailableDialects(); + + const unknown_location: *const mlir.Location = .unknown(mlir_ctx); + + var compiler: Compiler = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + .arena = undefined, + .mlir_registry = mlir_registry, + .mlir_ctx = mlir_ctx, + .mlir_pass_manager = undefined, + .mlir_known_types = undefined, + .module = undefined, + .platform = undefined, + .partitioning = undefined, + .location = unknown_location, + .unknown_location = unknown_location, + }; + + compiler.pushLocation(@src(), "loc1"); + const loc1 = compiler.location; + + compiler.pushLocation(@src(), "loc2"); + compiler.popLocation(); + + try std.testing.expectEqual(loc1, compiler.location); + compiler.popLocation(); + + try std.testing.expectEqual(unknown_location, compiler.location); +} + pub fn nextChannelId(self: *Compiler) i64 { self.channel_id += 1; return self.channel_id; @@ -344,7 +417,7 @@ pub fn compileInternal( compiler.mlir_pass_manager.runOnOp(compiler.module.operation()) catch |err| switch (err) { error.MlirUnexpected => { - std.log.err("Failed to canonicalize invalid mlir: \n {f} \n ", .{compiler.module.operation()}); + compiler.handleCompilationError(io, compiler.arena.allocator(), opts, err, "Failed to canonicalize invalid MLIR"); @panic("ZML generated invalid mlir. Please open a bug report"); }, }; @@ -352,7 +425,11 @@ pub fn compileInternal( var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); - const loaded_executable = try compileModuleToPjrtExecutable(arena.allocator(), io, platform, compiler.module, compiler.partitioning, opts); + const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), io, platform, compiler.module, compiler.partitioning, opts) catch |err| { + compiler.handleCompilationError(io, compiler.arena.allocator(), opts, err, "Pjrt failed to compile the following MLIR"); + return err; + }; + log.debug("\n******** ZML generated MLIR ********\n{f}", .{compiler.module.operation()}); const exe = try Exe.init( @@ -848,3 +925,55 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: return loaded_executable; } + +fn handleCompilationError(compiler: *Compiler, io: std.Io, allocator: std.mem.Allocator, opts: Options, err: anyerror, msg: []const u8) void { + const module_op = compiler.module.operation().fmt(.{ .debug_info = true, .debug_info_pretty_form = true }); + const truncated_msg = "..."; + var too_big: bool = true; + + const buffer: []u8 = allocator.alloc(u8, 8192) catch { + log.err("{s} ({}):\n{f}", .{ msg, err, module_op }); + return; + }; + defer allocator.free(buffer); + + var buffer_writer: std.Io.Writer = .fixed(buffer); + const module_mlir: []const u8 = module_mlir: { + module_op.format(&buffer_writer) catch { + @memcpy(buffer[8192 - truncated_msg.len ..], truncated_msg); + too_big = true; + break :module_mlir buffer; + }; + too_big = false; + break :module_mlir buffer_writer.buffered(); + }; + + log.err("{s} ({}):\n{s}", .{ msg, err, module_mlir }); + if (!too_big) return; + + if (opts.xla_dump_to == null) { + log.warn("To see the full .mlir, set `zml.Compiler.Options.xla_dump_to`", .{}); + return; + } + const xla_dir = opts.xla_dump_to.?; + const dir = std.Io.Dir.openDirAbsolute(io, xla_dir, .{}) catch dir: { + std.Io.Dir.createDirAbsolute(io, xla_dir, .default_dir) catch |e| { + log.err("failed to dump mlir to {s}: {}", .{ xla_dir, e }); + return; + }; + break :dir std.Io.Dir.openDirAbsolute(io, xla_dir, .{}) catch unreachable; + }; + const filename = std.fmt.allocPrint(allocator, "{s}.mlir", .{opts.program_name}) catch return; + const mlir_file = dir.createFile(io, filename, .{ .truncate = true }) catch |e| { + log.err("failed to dump mlir to {s}/{s}: {}", .{ xla_dir, filename, e }); + return; + }; + + // Reuse the previously allocated buffer + var mlir_file_writer = mlir_file.writer(io, buffer); + module_op.format(&mlir_file_writer.interface) catch |e| { + log.err("Partial mlir dump at: {s}/{s} ({})", .{ xla_dir, filename, e }); + return; + }; + log.warn("Full .mlir at: {s}/{s}", .{ xla_dir, filename }); +} diff --git a/zml/kernel.zig b/zml/kernel.zig index 692dd3da..41c0a4ea 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -400,7 +400,7 @@ pub const mosaic_tpu = struct { .additional_attributes = args.additional_attributes, .output_operand_aliases = args.aliases, }, - .unknown(cur.mlir_ctx), + cur.location, ).appendTo(cur.currentScope().block); } }; diff --git a/zml/ops.zig b/zml/ops.zig index 217f04b5..59a454f9 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -82,7 +82,7 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@T block_types[i + input_tensors.len] = mlirx.Type.rankedTensor(mlir_ctx, scalar_shape); } - const block_locs: [2 * input_tensors.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); + const block_locs: [2 * input_tensors.len]*const mlir.Location = @splat(ctx.unknown_location); const block = mlir.Block.init(&block_types, &block_locs); errdefer block.deinit(); @@ -90,6 +90,9 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@T const reducer_scope = ctx.pushBlock(block); defer reducer_scope.pop(); + ctx.pushLocation(@src(), stdx.meta.fnName(func)); + defer ctx.popLocation(); + inline for (0..input_tensors.len) |i| { reducer_scope.registerTensorAsBlockArgument(args[i].left.id, i); reducer_scope.registerTensorAsBlockArgument(args[i].right.id, i + input_tensors.len); @@ -109,7 +112,7 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@T } } - _ = dialects.stablehlo.returns(mlir_ctx, &reduced_values, .unknown(mlir_ctx)).appendTo(block); + _ = dialects.stablehlo.returns(mlir_ctx, &reduced_values, ctx.location).appendTo(block); break :b block; }; @@ -144,6 +147,7 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@T reducer_block, replica_groups_attr, channel_handle_attr, + ctx.location, ).appendTo(ctx.currentScope().block); return switch (@typeInfo(@TypeOf(inputs))) { @@ -193,7 +197,7 @@ pub fn partitionId() Tensor { const ctx = Compiler.current(); const op = mlir.Operation.make(ctx.mlir_ctx, "stablehlo.partition_id", .{ .results = .{ .flat = &.{mlirx.Type.rankedTensor(ctx.mlir_ctx, Shape.scalar(.u32))} }, - .location = .unknown(ctx.mlir_ctx), + .location = ctx.location, }).appendTo(ctx.currentScope().block); return Tensor._result(.init(.{}, .u32), op.result(0)); } @@ -223,13 +227,16 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func block_types[i + inits.len] = mlirx.Type.rankedTensor(mlir_ctx, args[i].right.shape()); } - const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); + const block_locs: [2 * inits.len]*const mlir.Location = @splat(compiler.unknown_location); const reduce_block = mlir.Block.init(&block_types, &block_locs); errdefer reduce_block.deinit(); const reduce_scope = compiler.pushBlock(reduce_block); defer reduce_scope.pop(); + compiler.pushLocation(@src(), stdx.meta.fnName(func)); + defer compiler.popLocation(); + inline for (0..inits.len) |i| { reduce_scope.registerTensorAsBlockArgument(args[i].left.id, i); reduce_scope.registerTensorAsBlockArgument(args[i].right.id, i + inits.len); @@ -242,7 +249,7 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func result_values[i] = result[i].value(); } - const block_result = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)); + const block_result = dialects.stablehlo.returns(mlir_ctx, &result_values, compiler.location); _ = block_result.appendTo(reduce_block); break :b .{ reduce_block, result }; }; @@ -260,7 +267,7 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func .named(mlir_ctx, "dimensions", .denseArray(mlir_ctx, .i64, axes_)), }, .verify = true, - .location = .unknown(mlir_ctx), + .location = compiler.location, }).appendTo(caller_scope.block); // `stablehlo.reduce` drops axes. We want to avoid that to propagate tags. @@ -286,7 +293,7 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func reduce_op.result(i), broadcasting_axes.slice()[0 .. reduced_shape.rank() - axes_.len], mlirx.Type.rankedTensor(mlir_ctx, reduced_shape), - .unknown(mlir_ctx), + compiler.location, ).appendTo(caller_scope.block); result[i] = Tensor._result(reduced_shape, broad_op.result(0)); @@ -330,7 +337,7 @@ pub fn reduceWindow(N: comptime_int, inputs: [N]Tensor, inits: [N]Tensor, opts: block_types[i + N] = mlirx.Type.rankedTensor(mlir_ctx, args[i].right.shape()); } - const block_locs: [2 * N]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); + const block_locs: [2 * N]*const mlir.Location = @splat(compiler.unknown_location); const reduce_block = mlir.Block.init(&block_types, &block_locs); errdefer reduce_block.deinit(); @@ -349,7 +356,7 @@ pub fn reduceWindow(N: comptime_int, inputs: [N]Tensor, inits: [N]Tensor, opts: result_values[i] = result[i].value(); } - _ = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)).appendTo(reduce_block); + _ = dialects.stablehlo.returns(mlir_ctx, &result_values, compiler.location).appendTo(reduce_block); break :b .{ reduce_block, result }; }; var input_values: [inputs.len]*const mlir.Value = undefined; @@ -376,7 +383,7 @@ pub fn reduceWindow(N: comptime_int, inputs: [N]Tensor, inits: [N]Tensor, opts: )), }, .verify = true, - .location = .unknown(mlir_ctx), + .location = compiler.location, }).appendTo(compiler.currentScope().block); inline for (0..result.len) |i| { @@ -409,7 +416,7 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytyp block_types[2 * i + 1] = mlirx.Type.rankedTensor(mlir_ctx, args[i].right.shape()); } - const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); + const block_locs: [2 * inputs.len]*const mlir.Location = @splat(compiler.unknown_location); const sort_block = mlir.Block.init(&block_types, &block_locs); errdefer sort_block.deinit(); @@ -423,7 +430,7 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytyp var result = @call(.auto, func, args ++ context); - _ = dialects.stablehlo.return_(mlir_ctx, result.value(), .unknown(mlir_ctx)).appendTo(sort_block); + _ = dialects.stablehlo.return_(mlir_ctx, result.value(), compiler.location).appendTo(sort_block); break :b sort_block; }; @@ -441,7 +448,7 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytyp .named(mlir_ctx, "is_stable", .boolean(mlir_ctx, is_stable)), }, .verify = true, - .location = .unknown(mlir_ctx), + .location = compiler.location, }).appendTo(compiler.currentScope().block); var result: [inputs.len]Tensor = undefined; @@ -470,7 +477,8 @@ pub fn @"while"( const allocator = arena.allocator(); const mlir_ctx = comp.mlir_ctx; - const location = mlir.Location.unknown(mlir_ctx); + comp.pushLocation(@src(), @typeName(While)); + defer comp.popLocation(); // Force to materialize tensor.value() before we push a new scope. var captured_context: While = undefined; @@ -493,7 +501,7 @@ pub fn @"while"( for (flat_operands) |input| { operands_info.appendAssumeCapacity(.{ .type = mlirx.Type.rankedTensor(mlir_ctx, input._shape), - .location = location, + .location = comp.location, .value = input.value(), }); } @@ -505,6 +513,9 @@ pub fn @"while"( const scope = comp.pushBlock(block); defer scope.pop(); + comp.pushLocationFmt(@src(), "{s}.cond", .{@typeName(While)}); + defer comp.popLocation(); + // Interpret initial_state as the block argument for (0.., flat_operands) |i, input| { scope.registerTensorAsBlockArgument(input.id, i); @@ -513,7 +524,7 @@ pub fn @"while"( const cond: Tensor = captured_context.cond(initial_state); stdx.debug.assert(cond.rank() == 0 and cond.dtype() == .bool, "zml.ops.while expects cond to return a scalar bool Tensor, got {f}", .{cond}); - _ = dialects.stablehlo.return_(mlir_ctx, cond.value(), location).appendTo(block); + _ = dialects.stablehlo.return_(mlir_ctx, cond.value(), comp.location).appendTo(block); break :b block; }; @@ -524,6 +535,9 @@ pub fn @"while"( const scope = comp.pushBlock(block); defer scope.pop(); + comp.pushLocationFmt(@src(), "{s}.body", .{@typeName(While)}); + defer comp.popLocation(); + // Interpret operands as the block argument scope.id_to_argument.ensureUnusedCapacity(scope.arena.allocator(), flat_operands.len) catch @panic("OOM"); for (0.., flat_operands) |i, input| { @@ -541,7 +555,7 @@ pub fn @"while"( } const result_values = meta.collectAlloc(Tensor.value, {}, allocator, &result) catch @panic("OOM"); - _ = dialects.stablehlo.returns(mlir_ctx, result_values, location).appendTo(block); + _ = dialects.stablehlo.returns(mlir_ctx, result_values, comp.location).appendTo(block); break :b .{ block, result }; }; @@ -551,7 +565,7 @@ pub fn @"while"( operands_info.items(.type), cond_block, body_block, - location, + comp.location, ).appendTo(comp.currentScope().block); const AssignResultCtx = struct { @@ -661,7 +675,8 @@ pub fn @"if"( stdx.debug.assert(pred.dtype() == .bool and pred.count() == 1, "zml.ops.if expects the condition to have exactly one element of dtype .bool, got {f}", .{pred}); - var arena = std.heap.ArenaAllocator.init(Compiler.current().allocator); + const compiler = Compiler.current(); + var arena = std.heap.ArenaAllocator.init(compiler.allocator); defer arena.deinit(); const allocator = arena.allocator(); @@ -675,7 +690,10 @@ pub fn @"if"( }.capture, arena.allocator(), {}, if_captures, &blkctx) catch unreachable; const mlir_ctx = Compiler.current().mlir_ctx; - const loc: *const mlir.Location = .unknown(mlir_ctx); + const loc: *const mlir.Location = compiler.location; + + compiler.pushLocation(@src(), @typeName(If)); + defer compiler.popLocation(); const true_branch, const true_branch_block = b: { const block = mlir.Block.init(&.{}, &.{}); @@ -760,13 +778,15 @@ pub fn if2( ) @TypeOf(on_true) { stdx.debug.assert(pred.dtype() == .bool and pred.count() == 1, "zml.ops.if expects the condition to have exactly one element of dtype .bool, got {f}", .{pred}); - var arena = std.heap.ArenaAllocator.init(Compiler.current().allocator); + var compiler = Compiler.current(); + + var arena = std.heap.ArenaAllocator.init(compiler.allocator); defer arena.deinit(); const allocator = arena.allocator(); - const mlir_ctx = Compiler.current().mlir_ctx; - const loc: *const mlir.Location = .unknown(mlir_ctx); + const mlir_ctx = compiler.mlir_ctx; + const loc: *const mlir.Location = compiler.location; const true_values = meta.collectAlloc(Tensor.value, {}, allocator, &on_true) catch @panic("OOM"); defer allocator.free(true_values); @@ -774,7 +794,7 @@ pub fn if2( const block = mlir.Block.init(&.{}, &.{}); errdefer block.deinit(); - const scope = Compiler.current().pushBlock(block); + const scope = compiler.pushBlock(block); defer scope.pop(); _ = dialects.stablehlo.returns(mlir_ctx, true_values, loc).appendTo(block); break :b block; @@ -786,7 +806,7 @@ pub fn if2( const block = mlir.Block.init(&.{}, &.{}); errdefer block.deinit(); - const scope = Compiler.current().pushBlock(block); + const scope = compiler.pushBlock(block); defer scope.pop(); _ = dialects.stablehlo.returns(mlir_ctx, false_values, loc).appendTo(block); @@ -800,7 +820,7 @@ pub fn if2( .location = loc, .verify = false, }); - _ = op.appendTo(Compiler.current().currentScope().block); + _ = op.appendTo(compiler.currentScope().block); return fromMlirOperationWithTags(op, on_true); } @@ -868,8 +888,9 @@ pub const TritonOps = struct { /// Generate an MLIR call to the given member function with the given tensors. pub fn triton(inputs: anytype, outputs: anytype, opts: TritonOps) [outputs.len]Tensor { - const mlir_ctx = Compiler.current().mlir_ctx; - var arena = std.heap.ArenaAllocator.init(Compiler.current().allocator); + const compiler = Compiler.current(); + const mlir_ctx = compiler.mlir_ctx; + var arena = std.heap.ArenaAllocator.init(compiler.allocator); defer arena.deinit(); var values: [inputs.len]*const mlir.Value = undefined; @@ -916,8 +937,8 @@ pub fn triton(inputs: anytype, outputs: anytype, opts: TritonOps) [outputs.len]T .result_layouts = &results_layouts, .output_operand_aliases = opts.output_operand_aliases, }, - .unknown(mlir_ctx), - ).appendTo(Compiler.current().currentScope().block); + compiler.location, + ).appendTo(compiler.currentScope().block); var outputs_: [outputs.len]Tensor = undefined; inline for (outputs, 0..) |output, i| { @@ -947,8 +968,9 @@ pub const FlyOps = struct { }; pub fn fly(inputs: anytype, outputs: anytype, opts: FlyOps) [outputs.len]Tensor { - const mlir_ctx = Compiler.current().mlir_ctx; - var arena = std.heap.ArenaAllocator.init(Compiler.current().allocator); + const compiler = Compiler.current(); + const mlir_ctx = compiler.mlir_ctx; + var arena = std.heap.ArenaAllocator.init(compiler.allocator); defer arena.deinit(); var values: [inputs.len]*const mlir.Value = undefined; @@ -1007,8 +1029,8 @@ pub fn fly(inputs: anytype, outputs: anytype, opts: FlyOps) [outputs.len]Tensor .result_layouts = &results_layouts, .output_operand_aliases = opts.output_operand_aliases, }, - .unknown(mlir_ctx), - ).appendTo(Compiler.current().currentScope().block); + compiler.location, + ).appendTo(compiler.currentScope().block); var outputs_: [outputs.len]Tensor = undefined; inline for (outputs, 0..) |output, i| { @@ -1198,7 +1220,7 @@ pub fn neuronNki(inputs: anytype, outputs: anytype, opts: NeuronNkiOps) [outputs .named(mlir_ctx, "backend_config", .string(mlir_ctx, compiled_backend_config)), }, }, - .unknown(mlir_ctx), + ctx.location, ).appendTo(ctx.currentScope().block); var outputs_: [outputs.len]Tensor = undefined; @@ -1287,7 +1309,8 @@ pub const CudaTileOps = struct { /// and launches `name` over `grid` tile blocks. Every operand and result /// arrives as one raw device pointer, in order, in the default layout. pub fn cudaTile(inputs: anytype, outputs: anytype, opts: CudaTileOps) [outputs.len]Tensor { - const mlir_ctx = Compiler.current().mlir_ctx; + const compiler = Compiler.current(); + const mlir_ctx = compiler.mlir_ctx; var values: [inputs.len]*const mlir.Value = undefined; inline for (0..inputs.len) |i| { @@ -1326,8 +1349,8 @@ pub fn cudaTile(inputs: anytype, outputs: anytype, opts: CudaTileOps) [outputs.l .has_side_effect = false, .output_operand_aliases = opts.output_operand_aliases, }, - .unknown(mlir_ctx), - ).appendTo(Compiler.current().currentScope().block); + compiler.location, + ).appendTo(compiler.currentScope().block); var outputs_: [outputs.len]Tensor = undefined; inline for (outputs, 0..) |output, i| { @@ -1354,7 +1377,9 @@ pub const CuteOps = struct { /// and launches `name`. Every operand and result arrives as one raw device /// pointer, in order, in the default layout. pub fn cute(inputs: anytype, outputs: anytype, opts: CuteOps) [outputs.len]Tensor { - const mlir_ctx = Compiler.current().mlir_ctx; + const compiler = Compiler.current(); + + const mlir_ctx = compiler.mlir_ctx; var values: [inputs.len]*const mlir.Value = undefined; inline for (0..inputs.len) |i| { @@ -1396,8 +1421,8 @@ pub fn cute(inputs: anytype, outputs: anytype, opts: CuteOps) [outputs.len]Tenso .has_side_effect = false, .output_operand_aliases = opts.output_operand_aliases, }, - .unknown(mlir_ctx), - ).appendTo(Compiler.current().currentScope().block); + compiler.location, + ).appendTo(compiler.currentScope().block); var outputs_: [outputs.len]Tensor = undefined; inline for (outputs, 0..) |output, i| { @@ -1681,10 +1706,11 @@ pub fn scatter( context: anytype, opts: Tensor.ScatterOpts, ) stdx.meta.FnReturn(func) { - var arena = std.heap.ArenaAllocator.init(Compiler.current().allocator); + const compiler = Compiler.current(); + var arena = std.heap.ArenaAllocator.init(compiler.allocator); defer arena.deinit(); - const mlir_ctx = Compiler.current().mlir_ctx; + const mlir_ctx = compiler.mlir_ctx; const update_block, var result = b: { const ArgsTypes: [inputs.len]type = @splat(ScatterArgs); @@ -1700,11 +1726,11 @@ pub fn scatter( block_types[i + inputs.len] = mlirx.Type.rankedTensor(mlir_ctx, args[i].update.shape()); } - const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); + const block_locs: [2 * inputs.len]*const mlir.Location = @splat(compiler.unknown_location); const update_block = mlir.Block.init(&block_types, &block_locs); errdefer update_block.deinit(); - const scope = Compiler.current().pushBlock(update_block); + const scope = compiler.pushBlock(update_block); defer scope.pop(); inline for (0..inputs.len) |i| { @@ -1719,7 +1745,7 @@ pub fn scatter( result_values[i] = result[i].value(); } - _ = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)).appendTo(update_block); + _ = dialects.stablehlo.returns(mlir_ctx, &result_values, compiler.location).appendTo(update_block); break :b .{ update_block, result }; }; @@ -1787,8 +1813,8 @@ pub fn scatter( .indices_are_sorted = opts.indices_are_sorted, .unique_indices = opts.indices_are_unique, }, - .unknown(mlir_ctx), - ).appendTo(Compiler.current().currentScope().block); + compiler.location, + ).appendTo(compiler.currentScope().block); inline for (0..result.len) |i| { result[i] = Tensor._result(inputs[i].shape(), op.result(i)); @@ -1994,7 +2020,8 @@ pub const GatherAxisKind = enum { batching, offset, collapsed, indices }; pub const GatherOpts = struct { indices_are_sorted: bool = false }; pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, opts: GatherOpts) Tensor { - const mlir_ctx = Compiler.current().mlir_ctx; + const compiler = Compiler.current(); + const mlir_ctx = compiler.mlir_ctx; stdx.debug.assert(idx_axes.len > 0, "gather expects 1 or more axes to operate one, received none. Example: `x.gather(.a, indices, .{{}})`", .{}); for (idx_axes, 0..) |a, i| { @@ -2100,8 +2127,8 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, .index_vector_dim = indices.axis(.coord), .indices_are_sorted = opts.indices_are_sorted, }, - .unknown(mlir_ctx), - ).appendTo(Compiler.current().currentScope().block); + compiler.location, + ).appendTo(compiler.currentScope().block); const mlir_shape = Tensor.fromMlirValue(gather_op.result(0)).shape(); stdx.debug.assert(mlir_shape.eql(res_shape), "gather expects that batching indices appear in the same order in 'self' and 'indices', got: self={f}, indices={f}. You should transpose one or the other.", .{ self, indices }); @@ -2325,10 +2352,10 @@ pub fn composite( { const block_types = ctx.alloc(*const mlir.Type, inputs.len); const block_locs = ctx.alloc(*const mlir.Location, inputs.len); + @memset(block_locs, ctx.unknown_location); for (inputs, 0..) |t, i| { block_types[i] = mlirx.Type.rankedTensor(mlir_ctx, t.shape()); - block_locs[i] = mlir.Location.unknown(mlir_ctx); } const block = mlir.Block.init(block_types, block_locs); @@ -2353,14 +2380,14 @@ pub fn composite( rvals[i] = t.value(); } - _ = dialects.func.returns(mlir_ctx, rvals, .unknown(mlir_ctx)).appendTo(block); + _ = dialects.func.returns(mlir_ctx, rvals, ctx.location).appendTo(block); } scope.pop(); _ = dialects.func.func(mlir_ctx, .{ .name = decomp_name, .block = block, - .location = .unknown(mlir_ctx), + .location = ctx.location, .visibility = .private, .verify = false, }).appendTo(ctx.module.body()); @@ -2386,7 +2413,7 @@ pub fn composite( .composite_attributes = opts.composite_attributes, .version = opts.version, }, - .unknown(mlir_ctx), + ctx.location, ).appendTo(ctx.currentScope().block); const out_tensors = ctx.alloc(Tensor, outputs.len); @@ -2590,7 +2617,7 @@ fn manualComputationInternal( block_types[i] = mlirx.Type.rankedTensor(ctx.mlir_ctx, input_shape); } const block_locs = try arena.alloc(*const mlir.Location, input_shapes.len); - @memset(block_locs, mlir.Location.unknown(ctx.mlir_ctx)); + @memset(block_locs, ctx.unknown_location); const manual_block = mlir.Block.init(block_types, block_locs); errdefer manual_block.deinit(); @@ -2621,7 +2648,7 @@ fn manualComputationInternal( _ = mlir.Operation.make(ctx.mlir_ctx, "sdy.return", .{ .operands = .{ .flat = local_output_values }, .verify = false, - .location = .unknown(ctx.mlir_ctx), + .location = ctx.location, }).appendTo(manual_block); const global_result_types = try arena.alloc(*const mlir.Type, outputs.len); @@ -2639,7 +2666,7 @@ fn manualComputationInternal( .named(ctx.mlir_ctx, "manual_axes", manual_axes_attr), }, .verify = true, - .location = .unknown(ctx.mlir_ctx), + .location = ctx.location, }).appendTo(scope.block); // Use the compiler allocator to return memory to the parent @@ -2665,7 +2692,7 @@ fn manualComputationInternal( .named(ctx.mlir_ctx, "mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")), }, }, - .unknown(ctx.mlir_ctx), + ctx.location, ).appendTo(scope.block); local_input_values[i] = full_to_shard.result(0); } @@ -2707,7 +2734,7 @@ fn manualComputationInternal( .named(ctx.mlir_ctx, "mhlo.sharding", gspmd_attr), }, }, - .unknown(ctx.mlir_ctx), + ctx.location, ).appendTo(scope.block); global_values[i] = shard_to_full.result(0); } @@ -2716,7 +2743,7 @@ fn manualComputationInternal( ctx.mlir_ctx, global_values, global_types, - .unknown(ctx.mlir_ctx), + ctx.location, ).appendTo(scope.block); const sharded_outputs = ctx.alloc(Tensor, outputs.len); @@ -3190,7 +3217,7 @@ pub fn typedCustomCall( .output_operand_aliases = opts.output_operand_aliases orelse &.{}, .additional_attributes = customCallAdditionalAttributes(ctx, opts), }, - .unknown(ctx.mlir_ctx), + ctx.location, ).appendTo(ctx.currentScope().block); if (ctx.manual_computation_depth > 0 and ctx.partitioning.partitioner == .gspmd) { diff --git a/zml/tensor.zig b/zml/tensor.zig index cca909ff..38f9c548 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -178,6 +178,7 @@ pub const Tensor = struct { .attributes = &.{ .named(ctx.mlir_ctx, "sharding", attr), }, + .location = ctx.location, }).appendTo(currentBlock()); break :blk op.result(0); }, @@ -194,7 +195,7 @@ pub const Tensor = struct { .named(ctx.mlir_ctx, "mhlo.sharding", attr), }, }, - .unknown(ctx.mlir_ctx), + ctx.location, ).appendTo(currentBlock()); break :blk op.result(0); }, @@ -231,7 +232,7 @@ pub const Tensor = struct { .named(ctx.mlir_ctx, "mhlo.frontend_attributes", frontend_attributes), }, }, - .unknown(ctx.mlir_ctx), + ctx.location, ).appendTo(currentBlock()); const res = _result(self._shape, op.result(0)); @@ -423,7 +424,7 @@ pub const Tensor = struct { /// Returns a Tensor containing the absolute value of each element of the input Tensor. pub fn abs(self: Tensor) Tensor { - const op = dialects.stablehlo.abs(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = dialects.stablehlo.abs(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); const dt = switch (self.dtype()) { .c64 => .f32, .c128 => .f64, @@ -458,7 +459,7 @@ pub const Tensor = struct { mlirCtx(), self.value(), mlirx.Type.rankedTensor(mlirCtx(), res_shape), - .unknown(mlirCtx()), + currentLoc(), ).appendTo(currentBlock()); return _result(res_shape, op.result(0)); @@ -471,26 +472,26 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise number of leading 0 bits in the input Tensor. pub fn countLeadingZeros(self: Tensor) Tensor { - const op = dialects.stablehlo.count_leading_zeros(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = dialects.stablehlo.count_leading_zeros(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing booleans indicating if each element of the input Tensor is finite. pub fn isFinite(self: Tensor) Tensor { - const op = dialects.stablehlo.is_finite(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = dialects.stablehlo.is_finite(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape.withDtype(.bool), op.result(0)); } /// Returns a Tensor containing the element-wise number of bits set in the input Tensor. pub fn popcnt(self: Tensor) Tensor { stdx.debug.assert(self.dtype().isInteger(), "popcnt expects tensor type to be an integer, got {}", .{self.dtype()}); - const op = dialects.stablehlo.popcnt(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = dialects.stablehlo.popcnt(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing the sign of the input Tensor element-wise. pub fn sign(self: Tensor) Tensor { - const op = dialects.stablehlo.sign(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = dialects.stablehlo.sign(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -553,7 +554,7 @@ pub const Tensor = struct { pub fn cholesky(self: Tensor, lower: bool) Tensor { stdx.debug.assert(self.rank() <= 2, "cholesky expects tensor rank to be <= 2, got {}", .{self.rank()}); - const op = dialects.stablehlo.cholesky(mlirCtx(), self.value(), lower, .unknown(mlirCtx())).appendTo(currentBlock()); + const op = dialects.stablehlo.cholesky(mlirCtx(), self.value(), lower, currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -562,7 +563,7 @@ pub const Tensor = struct { stdx.debug.assert(self.dtype() == other.dtype(), "triangularSolve expects tensors to be of the same type, got {} and {}", .{ self.dtype(), other.dtype() }); stdx.debug.assert(self.rank() >= 2 and self.rank() == other.rank(), "triangularSolve expects tensors to have the same rank >= 2, got {} and {}", .{ self.rank(), other.rank() }); - const op = dialects.stablehlo.triangular_solve(mlirCtx(), self.value(), other.value(), opts, .unknown(mlirCtx())).appendTo(currentBlock()); + const op = dialects.stablehlo.triangular_solve(mlirCtx(), self.value(), other.value(), opts, currentLoc()).appendTo(currentBlock()); return _result(other._shape, op.result(0)); } @@ -570,7 +571,7 @@ pub const Tensor = struct { pub fn roundNearestAfz(self: Tensor) Tensor { stdx.debug.assert(self.dtype().isFloat(), "roundNearestAfz expects tensor type to be a float, got {}", .{self.dtype()}); - const op = dialects.stablehlo.round_nearest_afz(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = dialects.stablehlo.round_nearest_afz(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -578,7 +579,7 @@ pub const Tensor = struct { pub fn roundNearestEven(self: Tensor) Tensor { stdx.debug.assert(self.dtype().isFloat(), "roundNearestEven expects tensor type to be a float, got {}", .{self.dtype()}); - const op = dialects.stablehlo.round_nearest_even(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = dialects.stablehlo.round_nearest_even(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -587,7 +588,7 @@ pub const Tensor = struct { stdx.debug.assert(re._shape.eql(im._shape), "complex expects tensor shapes to match, got {f} and {f}", .{ re._shape, im._shape }); stdx.debug.assert(re.dtype() == .f32 or re.dtype() == .f64, "complex expects tensors type to be f32 or f64, got {}", .{re.dtype()}); - const op = dialects.stablehlo.complex(mlirCtx(), re.value(), im.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = dialects.stablehlo.complex(mlirCtx(), re.value(), im.value(), currentLoc()).appendTo(currentBlock()); const dt: DataType = if (re.dtype() == .f32) .c64 else .c128; return _result(re._shape.withDtype(dt), op.result(0)); } @@ -607,7 +608,7 @@ pub const Tensor = struct { .c128 => .f64, else => unreachable, }; - const op = dialects.stablehlo.real(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = dialects.stablehlo.real(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape.withDtype(dt), op.result(0)); } @@ -628,7 +629,7 @@ pub const Tensor = struct { .c128 => .f64, else => unreachable, }; - const op = dialects.stablehlo.imag(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = dialects.stablehlo.imag(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape.withDtype(dt), op.result(0)); } @@ -672,7 +673,7 @@ pub const Tensor = struct { }, }; - const op = dialects.stablehlo.fft(mlirCtx(), self.value(), opts, .unknown(mlirCtx())).appendTo(currentBlock()); + const op = dialects.stablehlo.fft(mlirCtx(), self.value(), opts, currentLoc()).appendTo(currentBlock()); return _result(sh, op.result(0)); } @@ -740,7 +741,7 @@ pub const Tensor = struct { self._state.value(), mlirx.Type.rankedTensor(mlirCtx(), self._state.shape()), mlirx.Type.rankedTensor(mlirCtx(), sh), - .unknown(mlirCtx()), + currentLoc(), ).appendTo(currentBlock()); return .{ self.update(op.result(0)), _result(sh, op.result(1)) }; } @@ -864,7 +865,7 @@ pub const Tensor = struct { const a = Tensor.constant(DataType.Value.init(sh.dtype(), opts.mean)); const b = Tensor.constant(DataType.Value.init(sh.dtype(), opts.stddev)); const res_tensor_shape = Tensor.constantTensor(Shape.init(.{sh.rank()}, .i64), std.mem.sliceAsBytes(sh.dims())); - const op = dialects.stablehlo.rng(mlirCtx(), a.value(), b.value(), res_tensor_shape.value(), .NORMAL, .unknown(mlirCtx())).appendTo(currentBlock()); + const op = dialects.stablehlo.rng(mlirCtx(), a.value(), b.value(), res_tensor_shape.value(), .NORMAL, currentLoc()).appendTo(currentBlock()); return _result(sh, op.result(0)); } @@ -970,7 +971,7 @@ pub const Tensor = struct { stdx.debug.assert(1 <= exponent_bits, "reducePrecision expects 'exponent_bits' to be >= 1, got {}", .{exponent_bits}); stdx.debug.assert(0 <= mantissa_bits, "reducePrecision expects 'mantissa_bits' to be positive, got {}", .{mantissa_bits}); - const op = dialects.stablehlo.reduce_precision(mlirCtx(), self.value(), exponent_bits, mantissa_bits, .unknown(mlirCtx())).appendTo(currentBlock()); + const op = dialects.stablehlo.reduce_precision(mlirCtx(), self.value(), exponent_bits, mantissa_bits, currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -1063,7 +1064,7 @@ pub const Tensor = struct { other.value(), mlirx.Type.rankedTensor(mlirCtx(), new_shape), used_opts, - .unknown(mlirCtx()), + currentLoc(), ).appendTo(currentBlock()); return _result(new_shape, op.result(0)); @@ -1239,12 +1240,12 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise floor operation of the input Tensor. pub fn floor(self: Tensor) Tensor { - return _result(self._shape, dialects.stablehlo.floor(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()).result(0)); + return _result(self._shape, dialects.stablehlo.floor(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()).result(0)); } /// Returns a Tensor containing the element-wise ceil operation of the input Tensor. pub fn ceil(self: Tensor) Tensor { - return _result(self._shape, dialects.stablehlo.ceil(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()).result(0)); + return _result(self._shape, dialects.stablehlo.ceil(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()).result(0)); } /// Returns a Tensor containing the element-wise conversion to another type. @@ -1254,7 +1255,7 @@ pub const Tensor = struct { } const res_type = mlirx.Type.rankedTensor(mlirCtx(), self.shape().withDtype(to)); - const op = dialects.stablehlo.convert(mlirCtx(), self.value(), res_type, .unknown(mlirCtx())).appendTo(currentBlock()); + const op = dialects.stablehlo.convert(mlirCtx(), self.value(), res_type, currentLoc()).appendTo(currentBlock()); return _result(self._shape.withDtype(to), op.result(0)); } @@ -1381,13 +1382,13 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise rounding operation of the input Tensor. pub fn round(self: Tensor) Tensor { - const round_op = dialects.stablehlo.round_nearest_even(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const round_op = dialects.stablehlo.round_nearest_even(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, round_op.result(0)); } /// Returns a Tensor containing the element-wise clamping operation of the input Tensor. pub fn clamp(self: Tensor, min_: Tensor, max_: Tensor) Tensor { - const op = dialects.stablehlo.clamp(mlirCtx(), min_.value(), self.value(), max_.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = dialects.stablehlo.clamp(mlirCtx(), min_.value(), self.value(), max_.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -1566,14 +1567,14 @@ pub const Tensor = struct { .rhs_contracting_dimensions = rhs_contracting_axes.constSlice(), .dot_precision = .fast, }, - .unknown(mlirCtx()), + currentLoc(), ).appendTo(currentBlock()); return _result(res_shape, op.result(0)); } /// Returns a Tensor containing the sigmoid function applied to each element of the input Tensor. pub fn sigmoid(self: Tensor) Tensor { - const op = dialects.stablehlo.logistic(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = dialects.stablehlo.logistic(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -1871,7 +1872,7 @@ pub const Tensor = struct { self.value(), mlirx.Type.rankedTensor(mlirCtx(), res_shape), .{ .permutation = toI64(permutation).constSlice() }, - .unknown(mlirCtx()), + currentLoc(), ).appendTo(currentBlock()); return _result(res_shape, op.result(0)); } @@ -1902,7 +1903,7 @@ pub const Tensor = struct { mlirCtx(), self.value(), mlirx.Type.rankedTensor(mlirCtx(), new_shape), - .unknown(mlirCtx()), + currentLoc(), ).appendTo(currentBlock()); return _result(new_shape, reshaped_val.result(0)); } @@ -1918,7 +1919,7 @@ pub const Tensor = struct { mlirCtx(), self.value(), mlirx.Type.rankedTensor(mlirCtx(), new_shape), - .unknown(mlirCtx()), + currentLoc(), ).appendTo(currentBlock()); return _result(new_shape, reshaped_val.result(0)); } @@ -2070,7 +2071,7 @@ pub const Tensor = struct { self.value(), new_shape.dims(), start_indices[0..rk], - .unknown(mlirCtx()), + currentLoc(), ).appendTo(currentBlock()); const res = _result(new_shape, op.result(0)); @@ -2094,15 +2095,15 @@ pub const Tensor = struct { res_shape = res_shape.setDim(a, std.math.divCeil(i64, args.end - args.start, args.step) catch unreachable); } - const ctx = mlirCtx(); + const ctx = Compiler.current(); const slice_op = dialects.stablehlo.slice( - ctx, + ctx.mlir_ctx, self.value(), start_indices[0..self.rank()], limit_indices[0..self.rank()], strides[0..self.rank()], - mlirx.Type.rankedTensor(ctx, res_shape), - .unknown(ctx), + mlirx.Type.rankedTensor(ctx.mlir_ctx, res_shape), + ctx.location, ).appendTo(currentBlock()); var res = _result(res_shape, slice_op.result(0)); @@ -2207,7 +2208,7 @@ pub const Tensor = struct { } const res_shape = tensors[0]._shape.set(a, concatenated_dim); - const op = dialects.stablehlo.concatenate(mlirCtx(), buffer, a, .unknown(mlirCtx())).appendTo(currentBlock()); + const op = dialects.stablehlo.concatenate(mlirCtx(), buffer, a, currentLoc()).appendTo(currentBlock()); // log.debug("concatenate({}, {}, {d}) -> {d}", .{ tensors[0], tensors[1], a, res_shape }); return _result(res_shape, op.result(0)); } @@ -2373,55 +2374,55 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise negation of the input Tensor. pub fn negate(self: Tensor) Tensor { - const negate_op = dialects.stablehlo.negate(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const negate_op = dialects.stablehlo.negate(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, negate_op.result(0)); } /// Returns a Tensor containing the element-wise cosine of the input Tensor. pub fn cos(self: Tensor) Tensor { - const cosine_op = dialects.stablehlo.cosine(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const cosine_op = dialects.stablehlo.cosine(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, cosine_op.result(0)); } /// Returns a Tensor containing the element-wise sine of the input Tensor. pub fn sin(self: Tensor) Tensor { - const sine_op = dialects.stablehlo.sine(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const sine_op = dialects.stablehlo.sine(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, sine_op.result(0)); } /// Returns a Tensor containing the element-wise exponential operation of the input Tensor. pub fn exp(self: Tensor) Tensor { - const op = dialects.stablehlo.exponential(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = dialects.stablehlo.exponential(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing the element-wise logarithm operation of the input Tensor. pub fn log(self: Tensor) Tensor { - const op = dialects.stablehlo.log(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = dialects.stablehlo.log(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing the element-wise square-root of the input Tensor. pub fn sqrt(self: Tensor) Tensor { - const sqrt_op = dialects.stablehlo.sqrt(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const sqrt_op = dialects.stablehlo.sqrt(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, sqrt_op.result(0)); } /// Returns a Tensor containing the element-wise reverse square-root of the input Tensor. pub fn rsqrt(self: Tensor) Tensor { - const rsqrt_op = dialects.stablehlo.rsqrt(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const rsqrt_op = dialects.stablehlo.rsqrt(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, rsqrt_op.result(0)); } /// Returns a Tensor containing the element-wise hyperbolic tangent of the input Tensor. pub fn tanh(self: Tensor) Tensor { - const tanh_op = dialects.stablehlo.tanh(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const tanh_op = dialects.stablehlo.tanh(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, tanh_op.result(0)); } /// Returns a Tensor containing the element-wise exponential minus one operation of the input Tensor. pub fn exponentialMinusOne(self: Tensor) Tensor { - const expm1_op = dialects.stablehlo.exponential_minus_one(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const expm1_op = dialects.stablehlo.exponential_minus_one(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, expm1_op.result(0)); } @@ -2442,7 +2443,7 @@ pub const Tensor = struct { mlirCtx(), 0, mlirx.Type.rankedTensor(mlirCtx(), sh), - .unknown(mlirCtx()), + currentLoc(), ).appendTo(currentBlock()); var res = _result(sh, op.result(0)); @@ -2495,7 +2496,7 @@ pub const Tensor = struct { mlirCtx(), a, mlirx.Type.rankedTensor(mlirCtx(), res_shape), - .unknown(mlirCtx()), + currentLoc(), ).appendTo(currentBlock()); return _result(res_shape, op.result(0)); } @@ -2517,7 +2518,7 @@ pub const Tensor = struct { mlirCtx(), 0, mlirx.Type.rankedTensor(mlirCtx(), sh), - .unknown(mlirCtx()), + currentLoc(), ).appendTo(currentBlock()); var res = _result(sh, iota_op.result(0)); @@ -2566,7 +2567,7 @@ pub const Tensor = struct { &.{}, mlirx.Type.fromDType(mlirCtx(), val.dtype()), val.asBytes(), - .unknown(mlirCtx()), + currentLoc(), ).appendTo(currentBlock()); return _result(.init(&.{}, val.dtype()), op.result(0)); } @@ -2583,9 +2584,9 @@ pub const Tensor = struct { null, null, ); - const buffer = dialects.stablehlo.createBuffer(ctx.mlir_ctx, buffer_type, .unknown(ctx.mlir_ctx)) + const buffer = dialects.stablehlo.createBuffer(ctx.mlir_ctx, buffer_type, ctx.location) .appendTo(currentBlock()); - const tensor = dialects.stablehlo.unpin(ctx.mlir_ctx, buffer.result(0), .unknown(ctx.mlir_ctx)) + const tensor = dialects.stablehlo.unpin(ctx.mlir_ctx, buffer.result(0), ctx.location) .appendTo(currentBlock()); return _result(sh, tensor.result(0)); } @@ -2594,7 +2595,7 @@ pub const Tensor = struct { pub fn constantTensor(sh: Shape, bytes_: []const u8) Tensor { const elem_type = mlirx.Type.fromDType(mlirCtx(), sh.dtype()); //const elem_type = mlirx.denseElementAttrType(val.dtype()) orelse std.debug.panic("constantTensor expects a dtype that can be serialized to MLIR, like f32 or i32, got {f}", .{val.shape()}); - const constant_op = dialects.stablehlo.constant(mlirCtx(), sh.dims(), elem_type, bytes_, .unknown(mlirCtx())).appendTo(currentBlock()); + const constant_op = dialects.stablehlo.constant(mlirCtx(), sh.dims(), elem_type, bytes_, currentLoc()).appendTo(currentBlock()); return _result(sh, constant_op.result(0)); } @@ -2630,7 +2631,7 @@ pub const Tensor = struct { return _result(res_shape, self.value()); } const result_type = mlirx.Type.rankedTensor(mlirCtx(), res_shape); - const broadcast_op = dialects.stablehlo.broadcast_in_dim(mlirCtx(), self.value(), axes_, result_type, .unknown(mlirCtx())).appendTo(currentBlock()); + const broadcast_op = dialects.stablehlo.broadcast_in_dim(mlirCtx(), self.value(), axes_, result_type, currentLoc()).appendTo(currentBlock()); return _result(res_shape, broadcast_op.result(0)); } @@ -2690,7 +2691,7 @@ pub const Tensor = struct { ctx.mlir_ctx, &.{self.value()}, &.{self.value().type_()}, - .unknown(ctx.mlir_ctx), + ctx.location, ).appendTo(ctx.currentScope().block); return _result(self.shape(), op.result(0)); @@ -2700,7 +2701,7 @@ pub const Tensor = struct { pub fn reshape(self: Tensor, output_shape_: anytype) Tensor { const output_shape = self._shape.reshape(output_shape_); const tensor_type = mlirx.Type.rankedTensor(mlirCtx(), output_shape); - const reshape_value = dialects.stablehlo.reshape(mlirCtx(), self.value(), tensor_type, .unknown(mlirCtx())).appendTo(currentBlock()); + const reshape_value = dialects.stablehlo.reshape(mlirCtx(), self.value(), tensor_type, currentLoc()).appendTo(currentBlock()); return _result(output_shape, reshape_value.result(0)); } @@ -2742,7 +2743,7 @@ pub const Tensor = struct { mlirCtx(), self.value(), Tensor.scalar(padding_value, self.dtype()).value(), - .unknown(mlirCtx()), + currentLoc(), .{ .low = low[0..rk], .high = high[0..rk], .interior = interior[0..rk] }, ).appendTo(currentBlock()); @@ -2788,7 +2789,7 @@ pub const Tensor = struct { pub fn reverse(self: Tensor, axes_: anytype) Tensor { const actual_axes = self._shape.axes(axes_); - const reverse_op = dialects.stablehlo.reverse(mlirCtx(), self.value(), toI64(actual_axes.constSlice()).constSlice(), .unknown(mlirCtx())).appendTo(currentBlock()); + const reverse_op = dialects.stablehlo.reverse(mlirCtx(), self.value(), toI64(actual_axes.constSlice()).constSlice(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, reverse_op.result(0)); } @@ -2983,7 +2984,7 @@ pub const Tensor = struct { .index_vector_dim = index_coord_axis, .indices_are_sorted = opts.indices_are_sorted, }, - .unknown(mlirCtx()), + currentLoc(), ).appendTo(currentBlock()); return _result(res_shape, gather_op.result(0)); } @@ -3628,7 +3629,7 @@ pub const Tensor = struct { .has_side_effect = false, .backend_config = .{ .original = backend_config }, }, - .unknown(ctx.mlir_ctx), + ctx.location, ).appendTo(currentBlock()); var values = Tensor._result(values_shape, op.result(0)); @@ -4005,7 +4006,7 @@ pub const Tensor = struct { self.value(), update.value(), offset_values[0..self.rank()], - .unknown(mlirCtx()), + currentLoc(), ).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -4177,7 +4178,7 @@ pub const Tensor = struct { other.value(), dialects.stablehlo.ComparisonDirection.init(mlirCtx(), direction).getValue(), getComparisonType(mlirCtx(), self.dtype()).getValue(), - .unknown(mlirCtx()), + currentLoc(), ).appendTo(currentBlock()); return _result(self._shape.withDtype(.bool), op.result(0)); @@ -4349,7 +4350,7 @@ pub const Tensor = struct { bool_tensor.value(), on_true.value(), on_false.value(), - .unknown(mlirCtx()), + currentLoc(), ).appendTo(currentBlock()); return _result(on_true._shape, op.result(0)); @@ -4361,7 +4362,7 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise not logical operation of the input Tensor. pub fn not(self: Tensor) Tensor { - const op = dialects.stablehlo.not(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = dialects.stablehlo.not(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -4607,6 +4608,10 @@ pub const Tensor = struct { return Compiler.current().mlir_ctx; } + fn currentLoc() *const mlir.Location { + return Compiler.current().location; + } + fn currentBlock() *mlir.Block { return Compiler.current().currentScope().block; } From 11d49d8bc56ee793f8435e617e946a2da265e684 Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Wed, 23 Sep 2026 16:57:04 +0200 Subject: [PATCH 714/881] zml/tokenizer: add normalizer support (#847) Support normalization from HF JSON and typed normalizer sequences. Changes: - Patch IREE memory ownership and chunk-boundary bugs - Add regression tests for IREE bugs. --- ...x-normalizer-lifetime-and-boundaries.patch | 74 ++++ third_party/iree/repo.bzl | 1 + zml/tokenizer/iree/BUILD.bazel | 13 +- zml/tokenizer/iree/tokenizer.zig | 329 ++++++++++++++++++ zml/tokenizer/tokenizer.zig | 36 ++ 5 files changed, 452 insertions(+), 1 deletion(-) create mode 100644 third_party/iree/fix-normalizer-lifetime-and-boundaries.patch diff --git a/third_party/iree/fix-normalizer-lifetime-and-boundaries.patch b/third_party/iree/fix-normalizer-lifetime-and-boundaries.patch new file mode 100644 index 00000000..194a7b87 --- /dev/null +++ b/third_party/iree/fix-normalizer-lifetime-and-boundaries.patch @@ -0,0 +1,74 @@ +diff --git a/runtime/src/iree/tokenizer/normalizer/sequence.c b/runtime/src/iree/tokenizer/normalizer/sequence.c +--- a/runtime/src/iree/tokenizer/normalizer/sequence.c ++++ b/runtime/src/iree/tokenizer/normalizer/sequence.c +@@ -188,6 +188,14 @@ + "%" PRIhsz " exceeds maximum %" PRIu32, + flattened_count, + IREE_TOKENIZER_NORMALIZER_SEQUENCE_MAX_DEPTH); ++ } ++ ++ // Allocate before transferring ownership, including flattening nested ++ // sequences. On failure the caller must retain all original children. ++ iree_tokenizer_normalizer_sequence_t* normalizer = NULL; ++ if (flattened_count != 1) { ++ IREE_RETURN_IF_ERROR(iree_allocator_malloc( ++ allocator, sizeof(*normalizer), (void**)&normalizer)); + } + + // Build flattened children array, transferring ownership as we go. +@@ -232,18 +240,6 @@ + state_size = iree_host_align(state_size, iree_max_align_t); + child_state_offsets[i] = state_size; + state_size += iree_tokenizer_normalizer_state_size(flattened_children[i]); +- } +- +- // Allocate the sequence normalizer. +- iree_tokenizer_normalizer_sequence_t* normalizer = NULL; +- iree_status_t status = iree_allocator_malloc(allocator, sizeof(*normalizer), +- (void**)&normalizer); +- if (!iree_status_is_ok(status)) { +- // Cleanup: we've taken ownership of the children, so free them on failure. +- for (iree_host_size_t i = 0; i < flattened_count; ++i) { +- iree_tokenizer_normalizer_free(flattened_children[i]); +- } +- return status; + } + + // Initialize base. +@@ -566,6 +562,15 @@ + if (tile_size > IREE_TOKENIZER_NORMALIZER_SEQUENCE_MIN_BATCH) { + // Limit tile to what we can handle in scratch. + tile_size = IREE_TOKENIZER_NORMALIZER_SEQUENCE_MIN_BATCH; ++ // Never split a UTF-8 codepoint between child process calls. ++ while (tile_size > 0 && ++ ((uint8_t)remaining_input.data[tile_size] & 0xC0) == 0x80) { ++ --tile_size; ++ } ++ } ++ iree_tokenizer_normalizer_flags_t tile_flags = flags; ++ if (tile_size < remaining_input.size) { ++ tile_flags &= ~IREE_TOKENIZER_NORMALIZER_FLAG_SEGMENT_END; + } + + // Process one tile through all children. +@@ -574,7 +579,7 @@ + iree_host_size_t tile_consumed = 0; + iree_host_size_t tile_written = 0; + IREE_RETURN_IF_ERROR(iree_tokenizer_normalizer_sequence_process_tile( +- state, tile_input, scratch, scratch_size, remaining_output, flags, ++ state, tile_input, scratch, scratch_size, remaining_output, tile_flags, + &tile_consumed, &tile_written)); + + if (tile_consumed == 0 && tile_written == 0) { +diff --git a/runtime/src/iree/tokenizer/normalizer/strip.c b/runtime/src/iree/tokenizer/normalizer/strip.c +--- a/runtime/src/iree/tokenizer/normalizer/strip.c ++++ b/runtime/src/iree/tokenizer/normalizer/strip.c +@@ -353,7 +353,7 @@ + } + + // Phase 3: Handle segment end. +- if (is_segment_end) { ++ if (is_segment_end && in_ptr == in_end) { + // Pending whitespace is trailing - discard it. + iree_tokenizer_normalizer_strip_clear_pending(state); + state->at_start = true; diff --git a/third_party/iree/repo.bzl b/third_party/iree/repo.bzl index 860118ac..75c4eacb 100644 --- a/third_party/iree/repo.bzl +++ b/third_party/iree/repo.bzl @@ -17,6 +17,7 @@ def repo(): "//third_party/iree:fix-added-token-matching.patch", "//third_party/iree:match-hf-tokenizer.patch", "//third_party/iree:laguna-tokenizer-compat.patch", + "//third_party/iree:fix-normalizer-lifetime-and-boundaries.patch", ], patch_args = ["-p1"], ) diff --git a/zml/tokenizer/iree/BUILD.bazel b/zml/tokenizer/iree/BUILD.bazel index ebed7df4..a49ccd61 100644 --- a/zml/tokenizer/iree/BUILD.bazel +++ b/zml/tokenizer/iree/BUILD.bazel @@ -7,8 +7,19 @@ zig_library( visibility = ["//visibility:public"], deps = [ "@iree//runtime/src/iree/tokenizer", + "@iree//runtime/src/iree/tokenizer:normalizer", + "@iree//runtime/src/iree/tokenizer/format/huggingface:normalizer_json", "@iree//runtime/src/iree/tokenizer/format/huggingface:tokenizer_json", - "@iree//runtime/src/iree/tokenizer/vocab:vocab", + "@iree//runtime/src/iree/tokenizer/normalizer:lowercase", + "@iree//runtime/src/iree/tokenizer/normalizer:nfc", + "@iree//runtime/src/iree/tokenizer/normalizer:nfd", + "@iree//runtime/src/iree/tokenizer/normalizer:nfkd", + "@iree//runtime/src/iree/tokenizer/normalizer:regex_replace", + "@iree//runtime/src/iree/tokenizer/normalizer:replace", + "@iree//runtime/src/iree/tokenizer/normalizer:sequence", + "@iree//runtime/src/iree/tokenizer/normalizer:strip", + "@iree//runtime/src/iree/tokenizer/normalizer:strip_accents", + "@iree//runtime/src/iree/tokenizer/vocab", ], ) diff --git a/zml/tokenizer/iree/tokenizer.zig b/zml/tokenizer/iree/tokenizer.zig index 179ca9bd..f2a62fe6 100644 --- a/zml/tokenizer/iree/tokenizer.zig +++ b/zml/tokenizer/iree/tokenizer.zig @@ -131,6 +131,14 @@ pub const Tokenizer = struct { return Decoder.init(self, self.allocator); } + pub fn normalizer(self: *const Tokenizer, sequence: []const Normalizer.Kind) !Normalizer { + return Normalizer.init(self.allocator, sequence); + } + + pub fn normalizerFromHuggingFaceJson(_: *const Tokenizer, json: []const u8) !Normalizer { + return Normalizer.fromHuggingFaceJson(json); + } + pub fn tokenId(self: *const Tokenizer, token: []const u8) ?u32 { const vocab = c.iree_tokenizer_vocab(self.inner); if (vocab == null) return null; @@ -476,6 +484,327 @@ pub const Tokenizer = struct { }; }; +pub const Normalizer = struct { + pub const Kind = union(enum) { + /// Unicode Normalization Form C. + NFC, + /// Unicode Normalization Form D. + NFD, + /// Unicode Normalization Form KD. + NFKD, + /// Removes Unicode combining marks. + /// filters out characters in the Unicode Mark category (Mn, Mc, Me). + StripAccents, + /// Converts text to lowercase. + Lowercase, + /// Substitutes occurrences of a literal or regular-expression pattern with the specified content. + Replace: struct { + kind: enum { + regex, + exact, + }, + pattern: []const u8, + content: []const u8, + }, + /// Removes leading and/or trailing whitespace. + Strip: enum(u8) { + left = 0b10, + right = 0b01, + both = 0b11, + }, + }; + + sequence: *c.iree_tokenizer_normalizer_t, + sequence_size: usize, + + // Supported normalizer types as of `4d4e97d00f099a21f38eeff26f82a6d9e3643a11`: + // - Sequence: Chains multiple normalizers in order + // - Lowercase: Unicode case folding + // - Strip: Leading/trailing whitespace removal + // - Prepend: Prefix string insertion + // - StripAccents: Removes combining marks (without NFD) + // - BertNormalizer: Combined BERT normalization pipeline + pub fn fromHuggingFaceJson(json: []const u8) !Normalizer { + var normalizer: ?*c.iree_tokenizer_normalizer_t = null; + errdefer if (normalizer) |ptr| c.iree_tokenizer_normalizer_free(ptr); + + try checkOk(c.iree_tokenizer_huggingface_parse_normalizer(.{ .data = json.ptr, .size = json.len }, c.iree_allocator_system(), &normalizer)); + if (normalizer == null) return error.NormalizerAllocationFailed; + + const sequence_size: usize = @intCast(c.iree_tokenizer_normalizer_state_size(normalizer.?)); + + return .{ + .sequence = normalizer.?, + .sequence_size = sequence_size, + }; + } + + fn createNormalizer(normalizer_kind: Kind) !?*c.iree_tokenizer_normalizer_t { + var norm: ?*c.iree_tokenizer_normalizer_t = null; + + const iree_allocator = c.iree_allocator_system(); + + const status = switch (normalizer_kind) { + .NFC => c.iree_tokenizer_normalizer_nfc_allocate(iree_allocator, &norm), + .NFD => c.iree_tokenizer_normalizer_nfd_allocate(iree_allocator, &norm), + .NFKD => c.iree_tokenizer_normalizer_nfkd_allocate(iree_allocator, &norm), + .StripAccents => c.iree_tokenizer_normalizer_strip_accents_allocate(iree_allocator, &norm), + .Lowercase => c.iree_tokenizer_normalizer_lowercase_allocate(iree_allocator, &norm), + .Replace => |rule| switch (rule.kind) { + .regex => c.iree_tokenizer_normalizer_regex_replace_allocate( + .{ .data = rule.pattern.ptr, .size = rule.pattern.len }, + .{ .data = rule.content.ptr, .size = rule.content.len }, + iree_allocator, + &norm, + ), + .exact => c.iree_tokenizer_normalizer_replace_allocate( + .{ .data = rule.pattern.ptr, .size = rule.pattern.len }, + .{ .data = rule.content.ptr, .size = rule.content.len }, + iree_allocator, + &norm, + ), + }, + .Strip => |direction| b: { + const strip_left = @intFromEnum(direction) & 0b10 != 0; + const strip_right = @intFromEnum(direction) & 0b01 != 0; + break :b c.iree_tokenizer_normalizer_strip_allocate(strip_left, strip_right, iree_allocator, &norm); + }, + }; + + try checkOk(status); + if (norm == null) return error.NormalizerAllocationFailed; + return norm; + } + + pub fn init(allocator: std.mem.Allocator, sequence: []const Kind) !Normalizer { + if (sequence.len == 0) return error.EmptySequence; + + if (sequence.len == 1) { + // IREE requires a sequence of at least two normalizer. + const norm = try createNormalizer(sequence[0]) orelse unreachable; + const sequence_size: usize = @intCast(c.iree_tokenizer_normalizer_state_size(norm)); + + return .{ + .sequence = norm, + .sequence_size = sequence_size, + }; + } + + var normalizers: std.ArrayList(*c.iree_tokenizer_normalizer_t) = try .initCapacity(allocator, sequence.len); + defer normalizers.deinit(allocator); + errdefer for (normalizers.items) |normalizer| c.iree_tokenizer_normalizer_free(normalizer); + + for (sequence) |normalizer| { + normalizers.appendAssumeCapacity(try createNormalizer(normalizer) orelse unreachable); + } + + var normalizer_sequence: ?*c.iree_tokenizer_normalizer_t = null; + // IREE takes ownership of all the normalizers. + try checkOk(c.iree_tokenizer_normalizer_sequence_allocate(normalizers.items.ptr, normalizers.items.len, c.iree_allocator_system(), &normalizer_sequence)); + errdefer c.iree_tokenizer_normalizer_free(normalizer_sequence); + normalizers.clearRetainingCapacity(); + + if (normalizer_sequence == null) return error.NormalizerAllocationFailed; + + const sequence_size: usize = @intCast(c.iree_tokenizer_normalizer_state_size(normalizer_sequence.?)); + if (sequence_size == 0) return error.NormalizerAllocationFailed; + + return .{ + .sequence = normalizer_sequence.?, + .sequence_size = sequence_size, + }; + } + + pub fn deinit(self: *Normalizer) void { + c.iree_tokenizer_normalizer_free(self.sequence); + } + + pub fn normalize(self: *Normalizer, allocator: std.mem.Allocator, text: []const u8) ![]const u8 { + const storage: []align(16) u8 = try allocator.alignedAlloc(u8, .@"16", self.sequence_size); + defer allocator.free(storage); + + var state: ?*c.iree_tokenizer_normalizer_state_t = null; + defer if (state) |ptr| c.iree_tokenizer_normalizer_state_deinitialize(ptr); + + try checkOk(c.iree_tokenizer_normalizer_state_initialize(self.sequence, storage.ptr, &state)); + if (state == null) return error.NormalizationFailed; + + var buffer: [2048]u8 = undefined; + var remaining = text; + + var writer: std.Io.Writer.Allocating = .init(allocator); + defer writer.deinit(); + + while (remaining.len > 0) { + var written: usize = 0; + var consumed: usize = 0; + + try checkOk(c.iree_tokenizer_normalizer_state_process( + state.?, + .{ .data = remaining.ptr, .size = remaining.len }, + .{ .data = &buffer, .size = buffer.len }, + c.IREE_TOKENIZER_NORMALIZER_FLAG_SEGMENT_END, + &consumed, + &written, + )); + + if (consumed == 0 and written == 0) return error.NormalizationFailed; + + remaining = remaining[consumed..]; + try writer.writer.writeAll(buffer[0..written]); + } + + var has_pending = true; + while (has_pending) { + var written: usize = 0; + + try checkOk(c.iree_tokenizer_normalizer_state_finalize(state.?, .{ .data = &buffer, .size = buffer.len }, &written)); + has_pending = c.iree_tokenizer_normalizer_state_has_pending(state.?); + + try writer.writer.writeAll(buffer[0..written]); + + if (written == 0 and has_pending) return error.NormalizationFailed; + } + + return writer.toOwnedSlice(); + } +}; + +test "normalizer from Hugging Face" { + const allocator = std.testing.allocator; + + const json = + \\{ + \\ "type": "Sequence", + \\ "normalizers": [ + \\ {"type":"NFKD"}, + \\ {"type":"StripAccents"}, + \\ {"type":"Lowercase"}, + \\ {"type":"Replace","pattern":{"Regex":"[ \\t\\r\\n]+"},"content":" "}, + \\ {"type":"Replace","pattern":{"Regex":"^ $"},"content":"\ue000"}, + \\ {"type":"Strip","strip_left":true,"strip_right":true}, + \\ {"type":"Replace","pattern":{"String":"\ue000"},"content":" "} + \\ ] + \\} + ; + + var normalizer = try Normalizer.fromHuggingFaceJson(json); + defer normalizer.deinit(); + + const TestCase = struct { + input: []const u8, + expected: []const u8, + }; + + for ([_]TestCase{ + .{ .input = " hello", .expected = "hello" }, + .{ .input = "hello ", .expected = "hello" }, + .{ .input = " hello ", .expected = "hello" }, + .{ .input = "\nworld\r\n", .expected = "world" }, + .{ .input = "ÉCOLE", .expected = "ecole" }, + .{ .input = " CAFÉ\tAU\nLAIT ", .expected = "cafe au lait" }, + .{ .input = "\t \r\n", .expected = " " }, + .{ .input = "", .expected = "" }, + }) |case| { + const res = try normalizer.normalize(allocator, case.input); + defer allocator.free(res); + + try std.testing.expectEqualSlices(u8, case.expected, res); + } +} + +test "normalizer sequence" { + const allocator = std.testing.allocator; + + var normalizer = try Normalizer.init(allocator, &.{ + .{ .Strip = .both }, + .Lowercase, + }); + defer normalizer.deinit(); + + const res = try normalizer.normalize(allocator, " helLO WoRLD "); + defer allocator.free(res); + + try std.testing.expectEqualSlices(u8, "hello world", res); +} + +test "normalizer preserves whitespace across output and sequence boundaries" { + const allocator = std.testing.allocator; + for ([_][]const Normalizer.Kind{ + &.{.{ .Strip = .both }}, + &.{ .Lowercase, .{ .Strip = .both } }, + &.{ .Lowercase, .{ .Strip = .both }, .NFC }, + }) |steps| { + var normalizer = try Normalizer.init(allocator, steps); + defer normalizer.deinit(); + for ([_]usize{ 63, 64, 2047, 2048, 4096 }) |prefix_len| { + const input = try allocator.alloc(u8, prefix_len + 4); + defer allocator.free(input); + @memset(input[0..prefix_len], 'a'); + @memcpy(input[prefix_len..], " b "); + const output = try normalizer.normalize(allocator, input); + defer allocator.free(output); + try std.testing.expectEqualSlices(u8, input[0 .. prefix_len + 2], output); + } + } +} + +test "normalizer sequence preserves Unicode across tiles" { + const allocator = std.testing.allocator; + var normalizer = try Normalizer.fromHuggingFaceJson( + \\{"type":"Sequence","normalizers":[{"type":"Lowercase"},{"type":"Strip","strip_left":true,"strip_right":true}]} + ); + defer normalizer.deinit(); + for ([_][]const u8{ "é", "€", "😀" }) |codepoint| { + for (61..65) |prefix_len| { + const input = try allocator.alloc(u8, prefix_len + codepoint.len + 1); + defer allocator.free(input); + @memset(input[0..prefix_len], 'a'); + @memcpy(input[prefix_len..][0..codepoint.len], codepoint); + input[input.len - 1] = 'b'; + const output = try normalizer.normalize(allocator, input); + defer allocator.free(output); + try std.testing.expectEqualSlices(u8, input, output); + } + } +} + +test "normalizer sequence preserves composition across tiles" { + const allocator = std.testing.allocator; + var normalizer = try Normalizer.init(allocator, &.{ .NFC, .Lowercase }); + defer normalizer.deinit(); + const input = "a" ** 63 ++ "e\u{301}b"; + const output = try normalizer.normalize(allocator, input); + defer allocator.free(output); + try std.testing.expectEqualSlices(u8, "a" ** 63 ++ "éb", output); +} + +test "normalizer sequence allocation failure retains child ownership" { + const FailAllocator = struct { + fn control(_: ?*anyopaque, _: c.iree_allocator_command_t, _: ?*const anyopaque, _: [*c]?*anyopaque) callconv(.c) c.iree_status_t { + return @ptrFromInt(c.IREE_STATUS_RESOURCE_EXHAUSTED); + } + }; + // Include a nested sequence: failure must preserve its shell and children. + var nested = try Normalizer.init(std.testing.allocator, &.{ .NFC, .Lowercase }); + defer nested.deinit(); + var strip = try Normalizer.init(std.testing.allocator, &.{.{ .Strip = .both }}); + defer strip.deinit(); + const children = [_]*c.iree_tokenizer_normalizer_t{ nested.sequence, strip.sequence }; + var sequence: ?*c.iree_tokenizer_normalizer_t = null; + const status = c.iree_tokenizer_normalizer_sequence_allocate(&children, children.len, .{ + .self = null, + .ctl = FailAllocator.control, + }, &sequence); + defer c.iree_status_free(status); + try std.testing.expect(status != null); + try std.testing.expectEqual(@as(u32, c.IREE_STATUS_RESOURCE_EXHAUSTED), statusCode(status)); + try std.testing.expect(sequence == null); + const output = try nested.normalize(std.testing.allocator, "HELLO"); + defer std.testing.allocator.free(output); + try std.testing.expectEqualStrings("hello", output); +} + fn tokenOutputWriter(allocator: std.mem.Allocator) std.Io.Writer.Allocating { return std.Io.Writer.Allocating.initAligned(allocator, .of(u32)); } diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig index b0aae1c6..3b6b1e99 100644 --- a/zml/tokenizer/tokenizer.zig +++ b/zml/tokenizer/tokenizer.zig @@ -14,6 +14,28 @@ const Tokenizers = enum { }; pub const Tokenizer = union(Tokenizers) { + pub const Normalizer = union(Tokenizers) { + pub const Kind = iree.Normalizer.Kind; + + iree: iree.Normalizer, + sentencepiece: void, + homemade: void, + + pub fn deinit(self: *Normalizer) void { + switch (self.*) { + .iree => |*n| n.deinit(), + else => {}, + } + } + + pub fn normalize(self: *Normalizer, allocator: std.mem.Allocator, text: []const u8) ![]const u8 { + return switch (self.*) { + .iree => |*n| n.normalize(allocator, text), + else => error.UnsupportedNormalizer, + }; + } + }; + pub const Encoder = union(Tokenizers) { iree: iree.Tokenizer.Encoder, sentencepiece: sentencepiece.Encoder, @@ -142,6 +164,20 @@ pub const Tokenizer = union(Tokenizers) { }; } + pub fn normalizer(self: *const Tokenizer, sequence: []const Normalizer.Kind) !Normalizer { + return switch (self.*) { + .iree => |*backend| .{ .iree = try backend.normalizer(sequence) }, + else => error.UnsupportedNormalizer, + }; + } + + pub fn normalizerFromHuggingFaceJson(self: *const Tokenizer, json: []const u8) !Normalizer { + return switch (self.*) { + .iree => |*backend| .{ .iree = try backend.normalizerFromHuggingFaceJson(json) }, + else => error.UnsupportedNormalizer, + }; + } + pub fn tokenId(self: *const Tokenizer, token: []const u8) ?u32 { return switch (self.*) { inline else => |v| v.tokenId(token), From 1e045ec017c273f5d5c1ae48ce6bbaa1d0768e47 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Wed, 23 Sep 2026 17:54:37 +0200 Subject: [PATCH 715/881] mlir/dialects/cute_ir: carry the DSL compiler's CuTe dialects (#849) The CuTe dialects no longer build on a git checkout with a private overlay. - Every operation, type, attribute and enum of the compiler's cute and cute_nvgpu dialects is typed, with its documentation, operand constraints, traits and result inference. Nothing is kept as an opaque payload, and the compiler's IR round-trips through it unchanged. - The definitions are generated following `REGENERATING.md`, from the release's Python bindings and by probing the compiler. - The C API and the Zig builders follow: atom types take their parameters instead of a text payload, constrained integers are one type, and pointers take an address space enum. --- kernels/cute/builder.zig | 3 +- mlir/dialects/cute_ir/BUILD.bazel | 6 +- mlir/dialects/cute_ir/README.md | 125 +- mlir/dialects/cute_ir/REGENERATING.md | 232 + mlir/dialects/cute_ir/cute_ir.zig | 2663 +++++----- mlir/dialects/cute_ir/cute_nvgpu.zig | 4294 ++++++++++++++--- .../include/cute_ir-c/Dialect/BUILD.bazel | 2 + .../cute_ir/include/cute_ir-c/Dialect/Cute.h | 1 + .../Dialect/CuteCompilerAttributes.h | 225 + .../include/cute_ir-c/Dialect/CuteNVGPU.h | 1 + .../cute_ir-c/Dialect/CuteNVGPUAttributes.h | 480 ++ .../cute_ir-c/Dialect/CuteNVGPUTypes.h | 989 +++- .../include/cute_ir-c/Dialect/CuteTypes.h | 110 +- .../cute_ir/Dialect/Cute/IR/BUILD.bazel | 128 +- .../cute_ir/Dialect/Cute/IR/CuteAttrsAll.td | 7 + .../Dialect/Cute/IR/CuteAttrsCompiler.td | 220 + .../Dialect/Cute/IR/CuteDialectPrivate.h | 14 - .../cute_ir/Dialect/Cute/IR/CuteEnums.td | 70 + .../cute_ir/Dialect/Cute/IR/CuteInference.h | 24 + .../cute_ir/Dialect/Cute/IR/CuteOps.td | 3795 +++++++++++++++ .../cute_ir/Dialect/Cute/IR/CuteOpsPrivate.td | 428 -- .../cute_ir/Dialect/Cute/IR/CuteTypesAll.td | 5 +- .../Dialect/Cute/IR/CuteTypesCompiler.td | 269 ++ .../Dialect/Cute/IR/CuteTypesPrivate.td | 92 - .../cute_ir/Dialect/CuteNVGPU/IR/BUILD.bazel | 56 +- .../Dialect/CuteNVGPU/IR/CuteNVGPUAttrs.td | 132 + .../Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h | 19 + .../Dialect/CuteNVGPU/IR/CuteNVGPUDialect.td | 10 +- .../Dialect/CuteNVGPU/IR/CuteNVGPUEnums.td | 683 +++ .../Dialect/CuteNVGPU/IR/CuteNVGPUOps.td | 2372 +++++++-- .../Dialect/CuteNVGPU/IR/CuteNVGPUTypes.td | 802 ++- .../cute_ir/Registration/Registration.h | 46 - mlir/dialects/cute_ir/lib/CAPI/BUILD.bazel | 2 + .../lib/CAPI/CuteCompilerAttributes.cpp | 672 +++ .../cute_ir/lib/CAPI/CuteNVGPUAttributes.cpp | 1183 +++++ .../cute_ir/lib/CAPI/CuteNVGPUTypes.cpp | 3157 ++++++++++-- mlir/dialects/cute_ir/lib/CAPI/CuteTypes.cpp | 300 +- .../lib/Dialect/CuteNVGPU/IR/BUILD.bazel | 3 + .../Dialect/CuteNVGPU/IR/CuteNVGPUDialect.cpp | 1714 ++++++- .../cute_ir/lib/Dialect/IR/BUILD.bazel | 16 +- .../cute_ir/lib/Dialect/IR/CuteDialect.cpp | 59 - .../cute_ir/lib/Dialect/IR/CuteInference.cpp | 2982 ++++++++++++ .../cute_ir/lib/Dialect/IR/CuteOps.cpp | 9 + .../cute_ir/lib/Dialect/IR/CuteOpsPrivate.cpp | 6 - .../cute_ir/lib/Dialect/IR/CuteTypes.cpp | 1598 +++++- .../lib/Dialect/IR/CuteTypesPrivate.cpp | 333 -- .../cute_ir/lib/Registration/RegisterCute.cpp | 63 - mlir/dialects/cute_ir/test/BUILD.bazel | 23 - .../cute_ir/test/CAPIRegistrationTest.cpp | 96 - .../cute_ir/test/TypeConstructorsTest.c | 70 - platforms/cpu/cpu.bzl | 6 +- platforms/cuda/cuda.bzl | 8 +- platforms/metal/metal.bzl | 4 +- platforms/rocm/rocm.bzl | 4 +- third_party/cute_ir/cute_ir.bazel | 187 +- .../cutegen_dynamic_divisibility.patch | 243 + third_party/cute_ir/repo.bzl | 8 +- zml/nn.zig | 23 +- zml/quantization.zig | 3 + 59 files changed, 26561 insertions(+), 4514 deletions(-) create mode 100644 mlir/dialects/cute_ir/REGENERATING.md create mode 100644 mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteCompilerAttributes.h create mode 100644 mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPUAttributes.h create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteAttrsAll.td create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteAttrsCompiler.td delete mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteDialectPrivate.h create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteEnums.td create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteInference.h create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td delete mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOpsPrivate.td create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesCompiler.td delete mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesPrivate.td create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUAttrs.td create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUEnums.td delete mode 100755 mlir/dialects/cute_ir/include/cute_ir/Registration/Registration.h create mode 100644 mlir/dialects/cute_ir/lib/CAPI/CuteCompilerAttributes.cpp create mode 100644 mlir/dialects/cute_ir/lib/CAPI/CuteNVGPUAttributes.cpp delete mode 100755 mlir/dialects/cute_ir/lib/Dialect/IR/CuteDialect.cpp create mode 100644 mlir/dialects/cute_ir/lib/Dialect/IR/CuteInference.cpp create mode 100644 mlir/dialects/cute_ir/lib/Dialect/IR/CuteOps.cpp delete mode 100644 mlir/dialects/cute_ir/lib/Dialect/IR/CuteOpsPrivate.cpp delete mode 100644 mlir/dialects/cute_ir/lib/Dialect/IR/CuteTypesPrivate.cpp delete mode 100755 mlir/dialects/cute_ir/lib/Registration/RegisterCute.cpp delete mode 100644 mlir/dialects/cute_ir/test/BUILD.bazel delete mode 100644 mlir/dialects/cute_ir/test/CAPIRegistrationTest.cpp delete mode 100644 mlir/dialects/cute_ir/test/TypeConstructorsTest.c create mode 100644 third_party/cute_ir/cutegen_dynamic_divisibility.patch diff --git a/kernels/cute/builder.zig b/kernels/cute/builder.zig index 69b3897d..d59fa439 100644 --- a/kernels/cute/builder.zig +++ b/kernels/cute/builder.zig @@ -542,8 +542,7 @@ pub const Builder = struct { const elem = pt.getValueType() orelse @panic("makeTensor: untyped pointer"); const memref_ty = (cute.MemRefType.get(self.ctx, .{ .ptr = ptr.type_(), .layout = layout.inner.type_() }) catch @panic("bad memref")).type_(); const v = self.emit(cute.make_view(self.ctx, ptr.inner, layout.inner, memref_ty, self.loc())); - const space = std.meta.stringToEnum(MemorySpace, pt.getMemorySpace().isA(mlir.StringAttribute).?.value()) orelse .generic; - return .{ .inner = v.inner, .kernel = self, .layout = layout, .dtype = dtypes.mlirToDType(self.ctx, elem), .space = space }; + return .{ .inner = v.inner, .kernel = self, .layout = layout, .dtype = dtypes.mlirToDType(self.ctx, elem), .space = pt.getAddressSpace() }; } /// A `!cute.coord` from a tuple of `Value`s and ints; ints stay static in the type. diff --git a/mlir/dialects/cute_ir/BUILD.bazel b/mlir/dialects/cute_ir/BUILD.bazel index 37be8421..3b310559 100644 --- a/mlir/dialects/cute_ir/BUILD.bazel +++ b/mlir/dialects/cute_ir/BUILD.bazel @@ -24,9 +24,5 @@ zig_test( test_suite( name = "test", - tests = [ - ":zig_test", - "//mlir/dialects/cute_ir/test:capi_registration_test", - "//mlir/dialects/cute_ir/test:type_constructors_test", - ], + tests = [":zig_test"], ) diff --git a/mlir/dialects/cute_ir/README.md b/mlir/dialects/cute_ir/README.md index 9e5ed4f7..d75d0950 100644 --- a/mlir/dialects/cute_ir/README.md +++ b/mlir/dialects/cute_ir/README.md @@ -1,20 +1,37 @@ -# CuTe Zig Bindings +# CuTe dialects and Zig bindings + +`mlir/dialects/cute_ir` is the CuTe dialect of nvidia-cutlass-dsl 4.8.0. The +dialect, attributes, public types and `cutegen` layout algebra come verbatim +from the CUTLASS release's `cutlass_compiler/` tree (`@cute_ir`, +pinned in `third_party/cute_ir`, one patch giving cutegen the +`?{div=N}` leaf); this tree carries what differs from it: `CuteTypes.cpp` with +the compiler's remaining types, attributes and enums and ratio strides +allowed, the compiler's attributes, types and enums beyond the release +(`CuteAttrsCompiler.td`, `CuteTypesCompiler.td`, `CuteEnums.td` and the +`CuteNVGPU*.td` files), and every operation of the compiler's registry in +`CuteOps.td` and `CuteNVGPUOps.td`. + +Updating to a new DSL release means refreshing the compiler entry-point +offsets and regenerating the `.td` files from the release's Python bindings +and compiler. The C API (except the algebra attributes, `CuteAttributes.*`) +and the Zig types, attributes, enums and operation builders are generated +from those `.td` files and follow them. + +## Zig Depend on `//mlir/dialects/cute_ir` and import `mlir/dialects/cute_ir`, or use `@import("mlir/dialects").cute` through the dialect aggregator. `cute.registerDialects(registry)` registers both `cute` and `cute_nvgpu`. Cute builders, types, and attributes live directly in `cute`; CuteNVGPU builders -and types live in `cute.nvgpu`. Cute bindings are in `cute_ir.zig`, and CuteNVGPU -bindings are in `cute_nvgpu.zig`. Public and recovered operations belong to the same `cute` -dialect, without a separate private Zig namespace. +and types live in `cute.nvgpu`. ```zig const cute = @import("mlir/dialects/cute_ir"); const pointer = try cute.PtrType.get(ctx, .{ .valueType = .float(ctx, .f32), - .memorySpace = .string(ctx, "smem"), + .addressSpace = .smem, .alignment = 16, }); const layout_attr = try cute.LayoutAttr.get(ctx, "4:1"); @@ -26,82 +43,38 @@ const memref = try cute.MemRefType.get(ctx, .{ const alloc = cute.memref_alloc_smem(ctx, memref.type_(), loc).appendTo(block); ``` -The checked-in operation builders follow the pinned OSS `CuteOps.td` and the -local `CuteOpsPrivate.td` and `CuteNVGPUOps.td` schemas. Update the bindings when -those schemas change. There is no build-time Zig generator. Attribute and type -wrappers call `CuteAttributes.h/.cpp`, `CuteTypes.h/.cpp`, and -`CuteNVGPUTypes.h/.cpp`. C++ dialect definitions still use TableGen. - -Type constructors call the actual C++ `get`/`getChecked` functions, not a type -parser. Parameters follow ODS names; getters follow the C++ accessor names. +Type and attribute constructors call the C++ `get`/`getChecked` functions, +not a parser. `InitArgs` fields are the ODS parameter names in lower camel +case; getters are the C++ accessor names without underscores. Optional and defaulted parameters have defaults in `InitArgs`. Enums +are Zig enums with the compiler's values (the C API takes the integer value). `get` returns `error.InvalidMlir` for invalid parameters or mismatched contexts. `type_()` converts a typed wrapper to `*const mlir.Type` for operation builders. +Algebra types take an algebra attribute, `cute.LayoutType.get(ctx, .{ .attr = +layout_attr })`; algebra attribute constructors call the C++ algebra parser +with the raw expression and `getValue()` returns the canonical text. NVGPU atom +types take their typed parameters, such as `.valType` and `.copyBits`; +`cute.nvgpu.typeFromPayload(T, ctx, "")` parses one from the text after +its mnemonic. For reading existing IR use MLIR's generic `Type.parse`. + +Operation builders take the operands, then the result types, then the +attributes, then the location, and call `mlir.Operation.make`. Parameters +follow ODS names, with a trailing underscore where needed. Result parameters +append `_type` or `_types`; attributes take `*const mlir.Attribute`. Pass `null` +for absent optional parameters and `&.{}` for empty variadic ones. Dots in +mnemonics become underscores. Variable-length operand and result groups go +through `Operation.make`'s `.variadic` API, which writes the segment-size +attributes. Builders verify the created operations; to defer verification, use +`mlir.Operation.make` directly with `.verify = false`. The C API is also available independently of Zig from `lib/CAPI:cute` and -`lib/CAPI:cute_nvgpu`, via `cute_ir-c/Dialect/Cute.h` and `CuteNVGPU.h`. - -Algebra types take an existing algebra attribute, for example -`cute.LayoutType.get(ctx, .{ .attr = layout_attr })`. Attribute constructors -call the C++ algebra parser with the raw expression, without building or parsing -MLIR assembly. `getValue()` returns canonical algebra text owned by the context. -The `algebraAttribute` and `algebraType` helpers dispatch to these C bindings. -Recovered NVGPU payload types take a `StringAttr` containing the -complete payload, including angle brackets, such as ``; the current schemas -preserve those architecture-specific contents without interpreting them. -Type construction uses the typed C API directly, with no temporary MLIR assembly -strings. For explicitly reading existing IR, use MLIR's generic `Type.parse` -or `mlirTypeParseGet`; there are no CuTe-specific parsing helpers. - -Operation builders take explicit parameters, followed by the location, and call -`mlir.Operation.make`. Parameters follow ODS names, with a trailing underscore -where needed to avoid Zig name collisions. Result parameters append `_type` or -`_types`; attributes take `*const mlir.Attribute`. Pass `null` for absent -optional parameters and `&.{}` for empty variadic parameters. -Dots in operation mnemonics become underscores, except for the private -`cute.@"tuple.product"` and `cute.@"tuple.product_each"` builders, whose literal -names distinguish them from the public `tuple_product` and `tuple_product_each`. -Builders compute variadic segment sizes as dense-array attributes and verify -the created operations. To defer verification, use `mlir.Operation.make` -directly with `.verify = false`. - -Builders pass variable-length operand and result groups through -`Operation.make`'s `.variadic` API, without temporary heap allocations or a -fixed operand-count limit in the Zig builder. MLIR still allocates its own -operation storage, and the dialect/compiler determines legal fragment sizes. - -Tests construct public/private/NVGPU operations, verify their registered schemas, -and round-trip MLIR bytecode into a fresh context: +`lib/CAPI:cute_nvgpu`, via `cute_ir-c/Dialect/Cute.h` and `CuteNVGPU.h`; +constructors return a null handle for invalid parameters. ```sh -bazel test //mlir/dialects/cute_ir:test //mlir/dialects/cute_ir/test:all +bazel test //mlir/dialects/cute_ir:test ``` -These tests do not invoke the NVIDIA compiler or execute GPU kernels. Recovered -private schemas do not implement NVIDIA's complete semantic verification or -lowering pipeline. - -## Compiler Offset Inspection - -`tools:cute-objdump-offsets` invokes GNU objdump and scans its disassembly for -the eight known private compiler entry-point signatures. It requires Perl -(core modules only) and a GNU objdump supporting AArch64; it neither uses Python -nor loads the library. For example: - -```sh -bazel run //mlir/dialects/cute_ir/tools:cute-objdump-offsets -- \ - --input /absolute/path/to/_cutlass_ir.so --output /tmp/cute-offsets.json -bazel test //mlir/dialects/cute_ir/tools:cute-objdump-offsets-test -``` - -Use `--objdump /path/to/aarch64-linux-gnu-objdump` or `OBJDUMP` to select the -disassembler. The JSON report records the binary SHA-256, symbol availability, -all candidate addresses, and matching instructions. Addresses are ELF virtual -addresses, not file offsets. Missing or ambiguous matches produce exit status -1 and no `offsets` table, while retaining candidates for inspection. - -The signatures come from the native offset generator in `cute-native`, targeting -the 4.6.2 cu13/AArch64 package. This is an independent disassembly-based check -of those signatures, not fresh semantic identification or proof of ABI -compatibility with other builds. Some signatures contain build-specific PC-relative -instructions or neighboring code. This inspection tool does not replace the -native compiler bridge's build-time header generator or runtime binary checks. +These tests only check the bindings: every bound operation is registered and +every type and attribute rebuilds from its getters. The dialect itself is +tested where it is maintained. The dialects here do not implement the +compiler's semantic verification or lowering. diff --git a/mlir/dialects/cute_ir/REGENERATING.md b/mlir/dialects/cute_ir/REGENERATING.md new file mode 100644 index 00000000..949ed51c --- /dev/null +++ b/mlir/dialects/cute_ir/REGENERATING.md @@ -0,0 +1,232 @@ +# Regenerating the dialect definitions + +These are instructions for an agent. They describe what this directory must +contain after a CuTe DSL release bump, what is available to derive it, and how +to check the result. How to get there is left to you: combine the sources as +you see fit, write throwaway scripts, probe the compiler. Scripts and +intermediate tables are not committed; only the definitions are. + +Paths below are relative to this directory (the one holding this file). + +## Goal + +Every dialect this directory carries (`cute`, `cute_nvgpu`, and `cuda` when +`include/cute_ir/Dialect/Cuda` exists) must describe the DSL compiler's +registry as completely and as precisely as the release allows: every +operation, type, attribute and enum the compiler registers, with every fact +that can be established about each of them. Nothing is opaque unless it cannot +be recovered, and each remaining gap is stated in the file where it occurs. + +The DSL compiler stays the authority for semantics and lowering. These +definitions exist so that IR can be built, parsed, printed and type-inferred +without it, and so that anything built here is accepted by it unchanged. + +## What lives here + +- **Taken from the release archive.** Its `cutlass_compiler/` tree provides the + cute dialect declaration, the public types and attributes and their `.td` + files, the type interfaces, `CuteDialect.h`, `CuteAttributes.cpp`, + `CuteDialect.cpp` and the `cutegen` layout algebra. The build reads them + from the archive; they are never copied or edited here. One patch on the + archive (`cutegen_dynamic_divisibility.patch`, next to the archive's + repository rule) teaches `cutegen` the `?{div=N}` leaf. Refresh it if it no + longer applies. The archive's `CuteOps.td` is not used: its operations + differ from the compiler's. +- **Maintained here.** `CuteTypes.cpp` started as the release's copy and holds + the parsers, printers and verifiers of every cute type, fixed wherever the + release's parsing diverges from the compiler's. `CuteInference.{h,cpp}` holds + result type inference; the other dialect `.cpp` files and the `BUILD.bazel` + files complete them. Edit them as the new definitions require. +- **Generated: what you produce.** Every `.td` file under `include/` that is not + a dialect declaration: the operations (`CuteOps.td`, `CuteNVGPUOps.td`, + `CudaOps.td`), the compiler's types, attributes and enums beyond the release + (`CuteTypesCompiler.td`, `CuteAttrsCompiler.td`, `CuteEnums.td`, + `CuteNVGPUTypes.td`, `CuteNVGPUAttrs.td`, `CuteNVGPUEnums.td`, `CudaTypes.td`, + `CudaAttrs.td`, `CudaEnums.td`), and any new `.td` you need to split them + sensibly. The dialect declarations (`*Dialect.td`) change only as far as the + generated files need. +- **Language bindings, if present.** When the tree carries bindings derived + from these definitions (a C API under `lib/CAPI`, builder files next to this + directory), regenerate them from the records of the new definitions + (`llvm-tblgen --dump-json` gives them all; do not regex the `.td` files). + When a class changes shape (opaque to parametric, merged, split, renamed + accessors), change the bindings to match, and keep the existing + convenience helpers working where it costs little. + +Consumers outside this directory use the dialect classes and must keep +compiling: search the repository for the dialect namespaces +(`cutlass_compiler::cute`, `cute_nvgpu`, `cuda`) and for the headers under +`include/cute_ir`. When a definition changes shape, update those users in the +same change. Tighter operand constraints also break consumers that built IR +with loose types, and so does an operation that stops inferring its results: +fix the consumer, not the constraint. + +## Sources + +Pin everything to one release version, read from the repository rule that +fetches the release archive. The same version names the Python wheels. + +The packaging below is the one of the release these instructions were last +run against. Releases move things around: when a source is missing or has +changed shape, find where the same fact now lives, and update this file. + +| Source | Where | What it gives | +|---|---|---| +| Release archive | `https://github.com/NVIDIA/cutlass/archive/refs/tags/v.tar.gz`, `cutlass_compiler/` | The release's own `.td` files: what is already defined, their style, the parameter C++ types and helpers (`Cute_Param`, interfaces). Anything defined there is not redefined here. | +| Operation bindings | wheel `nvidia-cutlass-dsl-libs-base`, `cutlass/_mlir/dialects/__ops_gen.py` | Per operation: name, class name, docstring (summary, description, examples), operands and results in order with single, optional or variadic kind, segment attributes, attribute names with their ODS constraint (`AttrBuilder.get('')`), whether the builder infers result types, region count, successors. | +| Enum bindings | same wheel, `__enum_gen.py` | Every enum: cases, integer values, spellings, and for the ones that are attributes, the mnemonic they print under. | +| Compiler library | wheel `nvidia-cutlass-dsl-libs-cu13` (or `-cu12`), `cutlass/_mlir/_mlir_libs/_cutlass_ir.cu13.*.so` | Its strings: type and attribute class names, parameter names and C++ types (from the generated parser messages `failed to parse parameter '

' which is to be a `), mnemonics, verifier messages and constraint summaries. The strings are not in declaration order, and parameters read by custom directives have no message at all. | +| Typed Python classes | the same `.so`, as `cutlass._mlir.dialects.cute`, `cute_nvgpu`, `cuda` | For every compiler type and attribute: a `get(...)` with its parameters in order, and getters. Building a value and printing it shows the canonical syntax of each parameter value. | +| The examples | the release archive's `examples/python/CuTeDSL` | Real compiler-accepted modules, the round-trip corpus (see below). | +| The live compiler | the same `.so`, loaded from Python | Ground truth for everything above. | + +Loading the compiler. The `.so` only runs on Linux, on the architecture it was +built for. Two ways: + +- **Full install** (needed for the typed classes): a venv with + `nvidia-cutlass-dsl==` installed, then `import cutlass` before + anything else. +- **Bare wheels** (generic API only): unpack the two wheels into one directory, + put its `nvidia_cutlass_dsl/dsl_packages` on `sys.path`, `import + cutlass._mlir._mlir_libs`. The typed classes fail to import this way. + +Then use `cutlass._mlir.ir` with a fresh `ir.Context()`. The compiler +segfaults on some malformed inputs (some bitlayouts, some bad inference +inputs): run probes in a forked process. + +What the live compiler tells you: + +- `ir.Type.parse` / `ir.Attribute.parse` accept or reject a spelling, and on + rejection say where and what they expected; `str()` gives the canonical + printed form. Together with the typed classes this establishes each syntax: + the generated parser messages name each missing parameter, and the bespoke + custom directives (`128x128x32`, `32 DP`, `x32`, trailing flags such as + `mcast` or `t`) show up as `expected ...` errors. Every mnemonic a parser + knows can be found by trying every identifier-like string of the `.so`. +- `ir.Module.parse` on generic-form operations with wrong types yields the + verifier's constraint for each operand and result, in the compiler's words + (`operand #0 must be ... , but got ...`). Probe each operand with one + instance of every type class to learn exactly which classes it accepts. +- `op.has_trait(ir.IsTerminatorTrait)`, `ir.NoTerminatorTrait`; constructing + `ir.InferTypeOpInterface(op)` or `ir.MemoryEffectsOpInterface(op)` fails when + the operation lacks the interface. +- `ir.InferTypeOpInterface().inferReturnTypes(operands=..., + attributes=..., context=...)` is the inference oracle, with operand values + taken as block arguments of a dummy region. It cannot take properties: + for operations with attributes, parse and verify a module instead. +- Loop-invariant code motion hoists `Pure` operations, `canonicalize` removes + unused ones that do not write, and `cse` merges ones that at most read. +- Printing a parsed generic operation shows its custom assembly. + +The round-trip corpus: run every example of the archive's +`examples/python/CuTeDSL` with `CUTE_DSL_KEEP=ir-debug`, which keeps each +compiled module's IR; set `CUTE_DSL_ARCH` for the architecture-restricted +ones. This gives hundreds of real modules covering far more operations than +the fixtures in the repository (the testdata under `tools/` and the kernels' +fixtures), which are added to it. + +When sources disagree, the live compiler wins; record the disagreement in a +comment. + +## What each definition must carry + +Operations: + +- The operation name, and the binding's class name as the def name. +- `summary` and `description` from the docstring, kept whole, including its + examples. Reflow nothing that is code. Every example must parse and verify in + the compiler; fix the ones that do not and say so. +- Every operand and result, in order, with its name, its kind (single, + `Optional`, `Variadic`), and a type constraint that accepts exactly the type + classes the compiler's verifier accepts, with the verifier's wording as its + summary. Where a class or interface is not defined here or in the release, + or no probe type satisfies it, the predicate accepts any type and the file + says so. Constraints on types of unregistered dialects (such as `ptr`) must + also match the opaque form those types take here. A constraint must never + reject what the compiler accepts. +- Every attribute with its constraint: the ODS constraint the bindings name, + or the generated enum or dialect attribute it corresponds to; `OptionalAttr` + and `DefaultValuedAttr` as the bindings show. +- `AttrSizedOperandSegments` / `AttrSizedResultSegments`, regions (count and + kind), successors, terminator traits. +- `Pure` (or the memory effects) when established; none when unknown. +- `InferTypeOpInterface` exactly on the operations whose result types the + compiler infers. ODS generates the inference when all results are buildable + types; the others are written in `CuteInference.cpp`. The compiler's + inference is not a function of types alone: it folds constant operands and + keeps divisibility that `cutegen` drops. Where a rule is not recovered, + inference must fail rather than guess, and those operations override + `refineReturnTypes` so written result types stand: compiler-accepted IR is + never rejected for want of an inference rule. +- Operations keep the generic assembly form unless an `assemblyFormat` has been + checked to round-trip against the compiler (see below); the custom syntax + stays in the description. + +Types and attributes: + +- Every class the compiler has that the release lacks, with its mnemonic, + summary, parameters in declaration order (from the typed `get()`s and the + order parsing asks for them) with their C++ types and optional or defaulted + status, and a printed example in the description. C++ types of parameters + read by custom directives are not observable: choose the natural one. +- The syntax exactly as the compiler prints and parses it: a declarative + `assemblyFormat` when it can express it, otherwise a custom parser and + printer. Parametric beats opaque: a verbatim string payload is acceptable + only when the syntax cannot be reproduced, and the file says why. A hidden + parameter that never prints is left out: keeping it would make a built value + differ from its own printed text. +- The compiler's type aliases (`!memref_smem_f16`, `!copy_simt`, ...) are part + of its printed form: reproduce them through the dialect's + `OpAsmDialectInterface`. +- Verifiers (`genVerifyDecl`) where the compiler rejects parameter values, when + the rule is known. Large legality tables (instruction shapes and forms) may + be left to the compiler; the file says which. +- Enums as `I32EnumAttr` with the compiler's values and spellings, and their + `EnumAttr` with the compiler's mnemonic and `` form. + +Every generated file starts with a short header naming the release version and +the sources it came from, saying it is generated, and listing what it leaves +out or leaves unconstrained, and why. + +ODS pitfalls met so far: + +- Naming `InferTypeOpInterface` in the traits of an operation with buildable + results suppresses the `inferReturnTypes` ODS would generate. +- `mlir-tblgen -gen-enum-decls` emits every enum it sees, so a dialect's enums + file must not include another dialect's. +- The release's `CuteDialect.h` includes the attributes before the types, so + enums the attributes use must be emitted ahead of the attributes' include. +- A mnemonic that varies with a value (`!cute.i32`, `!cute.i7`) needs a hook + in front of the generated type parser. +- An attribute whose body is not in `<...>` prints in the opaque form + (`#dialect<...>`); its pretty form does not parse. + +## Checking the result + +A change is done when all of these hold: + +1. **Coverage.** Every operation in the bindings, every enum, and every + mnemonic the compiler's parsers know is either defined here, defined by the + release, or listed in the file's header as left out with the reason. +2. **Round trip against the compiler,** on the corpus and on sweeps of + spellings for every type and attribute (combinations of parameter values): + whatever the compiler accepts, this dialect accepts, and prints text the + compiler reads back to its own canonical form. Where the compiler's printed + form is itself reparseable, this dialect prints it byte-identical. Where + the compiler prints text its own parser rejects, or its parser drops data, + the reparseable, lossless form wins; list those cases. +3. **Inference agrees** with the compiler on every result the corpus and + crafted oracle cases produce. The inference test holds recorded compiler + results, since the compiler cannot be asked at test time: add a case for + every inferred operation the emitted modules do not already cover. +4. **The repository's tests pass**, including the dialect's own tests and the + tests of every consumer found above (kernel emitters, host tools, language + bindings), built with the tree's usual Bazel configuration. +5. **Both copies agree.** When the same dialect is carried by more than one + repository, the shared files are identical in each, and only + repository-specific files (`BUILD.bazel`, bindings) differ. + +Report, at the end: what changed in each file, the coverage figures, the +round-trip and inference figures, what is still opaque or unconstrained and +why, the test results, and anything in these instructions that was wrong, +missing or unclear. diff --git a/mlir/dialects/cute_ir/cute_ir.zig b/mlir/dialects/cute_ir/cute_ir.zig index aa5d84dc..3e9ff08f 100644 --- a/mlir/dialects/cute_ir/cute_ir.zig +++ b/mlir/dialects/cute_ir/cute_ir.zig @@ -71,21 +71,26 @@ pub fn algebraAttribute(ctx: *mlir.Context, kind: AttributeKind, expression: []c }; } -pub const MemorySpace = enum { generic, gmem, cmem, smem, rmem, tmem, dsmem }; +/// The address space of a pointer or memref. +pub const MemorySpace = AddressSpace; pub fn pointerType(ctx: *mlir.Context, element_type: ?*const mlir.Type, space: MemorySpace, alignment: u64) !*const mlir.Type { return (try PtrType.get(ctx, .{ .valueType = element_type, - .memorySpace = .string(ctx, @tagName(space)), + .addressSpace = space, .alignment = alignment, })).type_(); } pub fn constrainedIntType(ctx: *mlir.Context, width: enum { i32, i64 }, divisible_by: u64) !*const mlir.Type { - return switch (width) { - .i32 => (try ConstrainedInt32Type.get(ctx, .{ .divisibleBy = divisible_by })).type_(), - .i64 => (try ConstrainedInt64Type.get(ctx, .{ .divisibleBy = divisible_by })).type_(), - }; + return (try ConstrainedIntType.get(ctx, .{ + .divisibility = std.math.cast(i64, divisible_by) orelse return error.InvalidMlir, + .width = switch (width) { + .i32 => 32, + .i64 => 64, + }, + .isPow2 = false, + })).type_(); } /// Constructors take raw CuTe algebra expressions. Getter strings are owned by @@ -298,24 +303,70 @@ pub const SwizzleAttr = opaque { } }; -/// `!cute.arith_tuple_iter`. +// Enums, generated from the dialect's .td files; values are the compiler's. + +/// Address spaces for CuTe memrefs and pointers. +pub const AddressSpace = enum(u32) { + generic = 0, + gmem = 1, + smem = 3, + rmem = 5, + tmem = 6, + dsmem = 7, + cmem = 4, +}; + +/// Copy operand ID. +pub const CopyOperand = enum(u32) { + S = 0, + D = 1, +}; + +/// Major mode for MMA operations. +pub const MajorMode = enum(u32) { + k = 0, + mn = 1, +}; + +/// MMA operand ID. +pub const MmaOperand = enum(u32) { + A = 0, + B = 1, + C = 2, + D = 3, + SFA = 4, + SFB = 5, + E = 6, +}; + +/// Op for cute reduce operations. +pub const ReductionOp = enum(u32) { + ADD = 0, + MUL = 1, + MIN = 2, + MAX = 3, +}; + +// Types and attributes, generated from the dialect's .td files. + +/// `!cute.arith_tuple_iter`: Iterator over an arithmetic tuple. pub const ArithTupleIteratorType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteArithTupleIterator; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "arith_tuple_iter"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { arithTuple: *const mlir.Type, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { const result = c.mlirCuteArithTupleIteratorTypeGet(ctx.ptr(), args.arithTuple.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); @@ -325,24 +376,24 @@ pub const ArithTupleIteratorType = opaque { } }; -/// `!cute.composed_layout`. +/// `!cute.composed_layout`: CuTe composed layout: A ∘ offset ∘ B. pub const ComposedLayoutType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteComposedLayout; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "composed_layout"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { attr: *const mlir.Attribute, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { const result = c.mlirCuteComposedLayoutTypeGet(ctx.ptr(), args.attr.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); @@ -352,71 +403,52 @@ pub const ComposedLayoutType = opaque { } }; -/// `!cute.i32`. -pub const ConstrainedInt32Type = opaque { +/// `!cute.i`: Integer of any width with known divisibility and power of two. +pub const ConstrainedIntType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteConstrainedInt32; + pub const isAFn = c.mlirTypeIsACuteConstrainedInt; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - divisibleBy: u64, + divisibility: i64, + width: c_uint, + isPow2: bool, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteConstrainedInt32TypeGet(ctx.ptr(), args.divisibleBy); + const result = c.mlirCuteConstrainedIntTypeGet(ctx.ptr(), args.divisibility, args.width, args.isPow2); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getDivisibleBy(self: *const Self) u64 { - return c.mlirCuteConstrainedInt32TypeGetDivisibleBy(self.ptr()); + pub fn getDivisibility(self: *const Self) i64 { + return c.mlirCuteConstrainedIntTypeGetDivisibility(self.ptr()); + } + pub fn getWidth(self: *const Self) c_uint { + return c.mlirCuteConstrainedIntTypeGetWidth(self.ptr()); + } + pub fn getIsPow2(self: *const Self) bool { + return c.mlirCuteConstrainedIntTypeGetIsPow2(self.ptr()); } }; -/// `!cute.i64`. -pub const ConstrainedInt64Type = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteConstrainedInt64; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - divisibleBy: u64, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteConstrainedInt64TypeGet(ctx.ptr(), args.divisibleBy); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn getDivisibleBy(self: *const Self) u64 { - return c.mlirCuteConstrainedInt64TypeGetDivisibleBy(self.ptr()); - } -}; - -/// `!cute.coord_tensor`. +/// `!cute.coord_tensor`: Coordinate tensor: an arithmetic tuple with a layout. pub const CoordTensorType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteCoordTensor; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "coord_tensor"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -424,7 +456,6 @@ pub const CoordTensorType = opaque { arithTuple: *const mlir.Type, layout: *const mlir.Type, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { const result = c.mlirCuteCoordTensorTypeGet(ctx.ptr(), args.arithTuple.ptr(), args.layout.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); @@ -437,24 +468,24 @@ pub const CoordTensorType = opaque { } }; -/// `!cute.coord`. +/// `!cute.coord`: Scalar integer, underscore wildcard, or recursive tuple of coordinates. pub const CoordType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteCoord; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "coord"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { attr: *const mlir.Attribute, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { const result = c.mlirCuteCoordTypeGet(ctx.ptr(), args.attr.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); @@ -464,17 +495,18 @@ pub const CoordType = opaque { } }; -/// `!cute.fast_divmod_divisor`. +/// `!cute.fast_divmod_divisor`: Precomputed divisor of the fast divmod operations. pub const FastDivmodDivisorType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteFastDivmodDivisor; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "fast_divmod_divisor"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -482,7 +514,6 @@ pub const FastDivmodDivisorType = opaque { width: c_uint, isPow2: bool, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { const result = c.mlirCuteFastDivmodDivisorTypeGet(ctx.ptr(), args.width, args.isPow2); return @ptrCast(result.ptr orelse return error.InvalidMlir); @@ -495,24 +526,24 @@ pub const FastDivmodDivisorType = opaque { } }; -/// `!cute.int_tuple`. +/// `!cute.int_tuple`: Scalar integer or recursive tuple of integers. pub const IntTupleType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteIntTuple; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "int_tuple"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { attr: *const mlir.Attribute, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { const result = c.mlirCuteIntTupleTypeGet(ctx.ptr(), args.attr.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); @@ -522,24 +553,24 @@ pub const IntTupleType = opaque { } }; -/// `!cute.layout`. +/// `!cute.layout`: CuTe layout: a shape/stride pair. pub const LayoutType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteLayout; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "layout"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { attr: *const mlir.Attribute, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { const result = c.mlirCuteLayoutTypeGet(ctx.ptr(), args.attr.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); @@ -549,17 +580,18 @@ pub const LayoutType = opaque { } }; -/// `!cute.memref`. +/// `!cute.memref`: Tensor view: a pointer with a layout. pub const MemRefType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteMemRef; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "memref"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -567,7 +599,6 @@ pub const MemRefType = opaque { ptr: *const mlir.Type, layout: *const mlir.Type, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { const result = c.mlirCuteMemRefTypeGet(ctx.ptr(), args.ptr.ptr(), args.layout.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); @@ -580,36 +611,37 @@ pub const MemRefType = opaque { } }; -/// `!cute.ptr`. +/// `!cute.ptr`: Pointer with address space, alignment, swizzle and bit layout. pub const PtrType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACutePtr; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "ptr"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { valueType: ?*const mlir.Type = null, - memorySpace: *const mlir.Attribute, + addressSpace: AddressSpace, alignment: u64, swizzle: ?*const mlir.Attribute = null, + bitlayout: ?*const mlir.Attribute = null, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCutePtrTypeGet(ctx.ptr(), if (args.valueType) |value| value.ptr() else c.MlirType{ .ptr = null }, args.memorySpace.ptr(), args.alignment, if (args.swizzle) |value| value.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCutePtrTypeGet(ctx.ptr(), if (args.valueType) |v| v.ptr() else c.MlirType{ .ptr = null }, @intFromEnum(args.addressSpace), args.alignment, if (args.swizzle) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, if (args.bitlayout) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValueType(self: *const Self) ?*const mlir.Type { return @ptrCast(c.mlirCutePtrTypeGetValueType(self.ptr()).ptr); } - pub fn getMemorySpace(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCutePtrTypeGetMemorySpace(self.ptr()).ptr.?); + pub fn getAddressSpace(self: *const Self) AddressSpace { + return @enumFromInt(c.mlirCutePtrTypeGetAddressSpace(self.ptr())); } pub fn getAlignment(self: *const Self) u64 { return c.mlirCutePtrTypeGetAlignment(self.ptr()); @@ -617,26 +649,29 @@ pub const PtrType = opaque { pub fn getSwizzle(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCutePtrTypeGetSwizzle(self.ptr()).ptr); } + pub fn getBitlayout(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCutePtrTypeGetBitlayout(self.ptr()).ptr); + } }; -/// `!cute.shape`. +/// `!cute.shape`: Scalar integer or recursive tuple of shape extents. pub const ShapeType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteShape; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "shape"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { attr: *const mlir.Attribute, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { const result = c.mlirCuteShapeTypeGet(ctx.ptr(), args.attr.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); @@ -646,17 +681,18 @@ pub const ShapeType = opaque { } }; -/// `!cute.sparse_elem`. +/// `!cute.sparse_elem`: Logical sparse element stored in a physical type. pub const SparseElemType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteSparseElem; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "sparse_elem"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -664,7 +700,6 @@ pub const SparseElemType = opaque { numLogical: c_int, physicalType: *const mlir.Type, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { const result = c.mlirCuteSparseElemTypeGet(ctx.ptr(), args.numLogical, args.physicalType.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); @@ -677,24 +712,24 @@ pub const SparseElemType = opaque { } }; -/// `!cute.stride`. +/// `!cute.stride`: Scalar or recursive tuple of stride elements. pub const StrideType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteStride; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "stride"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { attr: *const mlir.Attribute, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { const result = c.mlirCuteStrideTypeGet(ctx.ptr(), args.attr.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); @@ -704,24 +739,24 @@ pub const StrideType = opaque { } }; -/// `!cute.swizzle`. +/// `!cute.swizzle`: CuTe swizzle: S. pub const SwizzleType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteSwizzle; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "swizzle"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { attr: *const mlir.Attribute, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { const result = c.mlirCuteSwizzleTypeGet(ctx.ptr(), args.attr.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); @@ -731,24 +766,24 @@ pub const SwizzleType = opaque { } }; -/// `!cute.tile`. +/// `!cute.tile`: CuTe tile: a recursive sequence of layouts and underscores. pub const TileType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteTile; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "tile"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { attr: *const mlir.Attribute, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { const result = c.mlirCuteTileTypeGet(ctx.ptr(), args.attr.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); @@ -758,7 +793,631 @@ pub const TileType = opaque { } }; -// Builders from the pinned OSS CuteOps.td. +/// `!cute.tiled_copy`: Copy atom tiled over threads and values. +pub const TiledCopyType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteTiledCopy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "tiled_copy"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + copyAtom: *const mlir.Type, + layoutCopyTv: *const mlir.Attribute, + tilerMn: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteTiledCopyTypeGet(ctx.ptr(), args.copyAtom.ptr(), args.layoutCopyTv.ptr(), args.tilerMn.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getCopyAtom(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteTiledCopyTypeGetCopyAtom(self.ptr()).ptr.?); + } + pub fn getLayoutCopyTv(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteTiledCopyTypeGetLayoutCopyTv(self.ptr()).ptr.?); + } + pub fn getTilerMn(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteTiledCopyTypeGetTilerMn(self.ptr()).ptr.?); + } + pub const getAtom = getCopyAtom; +}; + +/// `!cute.tiled_copy_v2`: V2 copy atom tiled over threads and values. +pub const TiledCopyV2Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteTiledCopyV2; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "tiled_copy_v2"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + copyAtom: *const mlir.Type, + layoutCopyTv: *const mlir.Attribute, + tilerMn: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteTiledCopyV2TypeGet(ctx.ptr(), args.copyAtom.ptr(), args.layoutCopyTv.ptr(), args.tilerMn.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getCopyAtom(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteTiledCopyV2TypeGetCopyAtom(self.ptr()).ptr.?); + } + pub fn getLayoutCopyTv(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteTiledCopyV2TypeGetLayoutCopyTv(self.ptr()).ptr.?); + } + pub fn getTilerMn(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteTiledCopyV2TypeGetTilerMn(self.ptr()).ptr.?); + } +}; + +/// `!cute.tiled_mma`: MMA atom tiled over MNK. +pub const TiledMmaType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteTiledMma; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "tiled_mma"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + mmaAtom: *const mlir.Type, + atomLayoutMNK: *const mlir.Attribute, + permutationMNK: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteTiledMmaTypeGet(ctx.ptr(), args.mmaAtom.ptr(), args.atomLayoutMNK.ptr(), if (args.permutationMNK) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getMmaAtom(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteTiledMmaTypeGetMmaAtom(self.ptr()).ptr.?); + } + pub fn getAtomLayoutMNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteTiledMmaTypeGetAtomLayoutMNK(self.ptr()).ptr.?); + } + pub fn getPermutationMNK(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteTiledMmaTypeGetPermutationMNK(self.ptr()).ptr); + } + pub const getAtom = getMmaAtom; + pub const getAtomLayoutMnk = getAtomLayoutMNK; + pub const getPermutationMnk = getPermutationMNK; +}; + +/// `!cute.tuple`: Non-empty tuple of types. +pub const TupleType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteTuple; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "tuple"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + types: []const *const mlir.Type, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteTupleTypeGet(ctx.ptr(), @intCast(args.types.len), @ptrCast(args.types.ptr)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getNumTypes(self: *const Self) usize { + return @intCast(c.mlirCuteTupleTypeGetNumTypes(self.ptr())); + } + pub fn getType(self: *const Self, pos: usize) *const mlir.Type { + return @ptrCast(c.mlirCuteTupleTypeGetType(self.ptr(), @intCast(pos)).ptr.?); + } +}; + +/// `#cute.bitlayout`: Bit layout of a pointer's elements within a storage chunk. +pub const BitLayoutAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteBitLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "bitlayout"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + layout: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteBitLayoutAttrGet(ctx.ptr(), args.layout.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getLayout(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteBitLayoutAttrGetLayout(self.ptr()).ptr.?); + } +}; + +/// `#cute.copy_atom`: Thread and value layouts of a copy atom. +pub const CopyAtomAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteCopyAtom; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "copy_atom"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + thrId: *const mlir.Attribute, + layoutSrc: *const mlir.Attribute, + layoutDst: *const mlir.Attribute, + layoutRef: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteCopyAtomAttrGet(ctx.ptr(), args.thrId.ptr(), args.layoutSrc.ptr(), args.layoutDst.ptr(), args.layoutRef.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getThrId(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomAttrGetThrId(self.ptr()).ptr.?); + } + pub fn getLayoutSrc(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomAttrGetLayoutSrc(self.ptr()).ptr.?); + } + pub fn getLayoutDst(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomAttrGetLayoutDst(self.ptr()).ptr.?); + } + pub fn getLayoutRef(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomAttrGetLayoutRef(self.ptr()).ptr.?); + } +}; + +/// `#cute.copy_atom_v2`: Thread and value layouts and fragments of a V2 copy atom. +pub const CopyAtomV2Attr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteCopyAtomV2; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "copy_atom_v2"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + thrId: *const mlir.Attribute, + layoutSrcTV: *const mlir.Attribute, + layoutDstTV: *const mlir.Attribute, + frgSrc: *const mlir.Attribute, + frgDst: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteCopyAtomV2AttrGet(ctx.ptr(), args.thrId.ptr(), args.layoutSrcTV.ptr(), args.layoutDstTV.ptr(), args.frgSrc.ptr(), args.frgDst.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getThrId(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomV2AttrGetThrId(self.ptr()).ptr.?); + } + pub fn getLayoutSrcTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomV2AttrGetLayoutSrcTV(self.ptr()).ptr.?); + } + pub fn getLayoutDstTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomV2AttrGetLayoutDstTV(self.ptr()).ptr.?); + } + pub fn getFrgSrc(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomV2AttrGetFrgSrc(self.ptr()).ptr.?); + } + pub fn getFrgDst(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomV2AttrGetFrgDst(self.ptr()).ptr.?); + } +}; + +/// `#cute.mma_atom`: Shapes, thread-value layouts and fragments of an MMA atom. +pub const MmaAtomAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteMmaAtom; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "mma_atom"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMNK: *const mlir.Attribute, + thrId: *const mlir.Attribute, + layoutATV: *const mlir.Attribute, + layoutBTV: *const mlir.Attribute, + layoutCTV: *const mlir.Attribute, + shapeAMK: *const mlir.Attribute, + shapeBNK: *const mlir.Attribute, + shapeCMN: *const mlir.Attribute, + frgA: *const mlir.Attribute, + frgB: *const mlir.Attribute, + frgC: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteMmaAtomAttrGet(ctx.ptr(), args.shapeMNK.ptr(), args.thrId.ptr(), args.layoutATV.ptr(), args.layoutBTV.ptr(), args.layoutCTV.ptr(), args.shapeAMK.ptr(), args.shapeBNK.ptr(), args.shapeCMN.ptr(), args.frgA.ptr(), args.frgB.ptr(), args.frgC.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getShapeMNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetShapeMNK(self.ptr()).ptr.?); + } + pub fn getThrId(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetThrId(self.ptr()).ptr.?); + } + pub fn getLayoutATV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetLayoutATV(self.ptr()).ptr.?); + } + pub fn getLayoutBTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetLayoutBTV(self.ptr()).ptr.?); + } + pub fn getLayoutCTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetLayoutCTV(self.ptr()).ptr.?); + } + pub fn getShapeAMK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetShapeAMK(self.ptr()).ptr.?); + } + pub fn getShapeBNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetShapeBNK(self.ptr()).ptr.?); + } + pub fn getShapeCMN(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetShapeCMN(self.ptr()).ptr.?); + } + pub fn getFrgA(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetFrgA(self.ptr()).ptr.?); + } + pub fn getFrgB(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetFrgB(self.ptr()).ptr.?); + } + pub fn getFrgC(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetFrgC(self.ptr()).ptr.?); + } +}; + +/// `#cute.mx_mma_atom`: MMA atom attribute with scale-factor operands. +pub const MxMmaAtomAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteMxMmaAtom; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "mx_mma_atom"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMNK: *const mlir.Attribute, + thrId: *const mlir.Attribute, + layoutATV: *const mlir.Attribute, + layoutSFATV: *const mlir.Attribute, + layoutBTV: *const mlir.Attribute, + layoutSFBTV: *const mlir.Attribute, + layoutCTV: *const mlir.Attribute, + shapeAMK: *const mlir.Attribute, + shapeSFAMK: *const mlir.Attribute, + shapeBNK: *const mlir.Attribute, + shapeSFBNK: *const mlir.Attribute, + shapeCMN: *const mlir.Attribute, + frgA: *const mlir.Attribute, + frgSFA: *const mlir.Attribute, + frgB: *const mlir.Attribute, + frgSFB: *const mlir.Attribute, + frgC: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteMxMmaAtomAttrGet(ctx.ptr(), args.shapeMNK.ptr(), args.thrId.ptr(), args.layoutATV.ptr(), args.layoutSFATV.ptr(), args.layoutBTV.ptr(), args.layoutSFBTV.ptr(), args.layoutCTV.ptr(), args.shapeAMK.ptr(), args.shapeSFAMK.ptr(), args.shapeBNK.ptr(), args.shapeSFBNK.ptr(), args.shapeCMN.ptr(), args.frgA.ptr(), args.frgSFA.ptr(), args.frgB.ptr(), args.frgSFB.ptr(), args.frgC.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getShapeMNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeMNK(self.ptr()).ptr.?); + } + pub fn getThrId(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetThrId(self.ptr()).ptr.?); + } + pub fn getLayoutATV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetLayoutATV(self.ptr()).ptr.?); + } + pub fn getLayoutSFATV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetLayoutSFATV(self.ptr()).ptr.?); + } + pub fn getLayoutBTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetLayoutBTV(self.ptr()).ptr.?); + } + pub fn getLayoutSFBTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetLayoutSFBTV(self.ptr()).ptr.?); + } + pub fn getLayoutCTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetLayoutCTV(self.ptr()).ptr.?); + } + pub fn getShapeAMK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeAMK(self.ptr()).ptr.?); + } + pub fn getShapeSFAMK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeSFAMK(self.ptr()).ptr.?); + } + pub fn getShapeBNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeBNK(self.ptr()).ptr.?); + } + pub fn getShapeSFBNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeSFBNK(self.ptr()).ptr.?); + } + pub fn getShapeCMN(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeCMN(self.ptr()).ptr.?); + } + pub fn getFrgA(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetFrgA(self.ptr()).ptr.?); + } + pub fn getFrgSFA(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetFrgSFA(self.ptr()).ptr.?); + } + pub fn getFrgB(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetFrgB(self.ptr()).ptr.?); + } + pub fn getFrgSFB(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetFrgSFB(self.ptr()).ptr.?); + } + pub fn getFrgC(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetFrgC(self.ptr()).ptr.?); + } +}; + +/// `#cute.reduction_op`: Op for cute reduce operations. +pub const ReductionOpAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteReductionOp; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "reduction_op"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ReductionOp, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteReductionOpAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) ReductionOp { + return @enumFromInt(c.mlirCuteReductionOpAttrGetValue(self.ptr())); + } +}; + +/// `#cute.sparse_mma_atom`: MMA atom attribute with a sparsity-metadata operand. +pub const SparseMmaAtomAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteSparseMmaAtom; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "sparse_mma_atom"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMNK: *const mlir.Attribute, + thrId: *const mlir.Attribute, + layoutATV: *const mlir.Attribute, + layoutBTV: *const mlir.Attribute, + layoutCTV: *const mlir.Attribute, + layoutETV: *const mlir.Attribute, + shapeAMK: *const mlir.Attribute, + shapeBNK: *const mlir.Attribute, + shapeCMN: *const mlir.Attribute, + shapeEMK: *const mlir.Attribute, + frgA: *const mlir.Attribute, + frgB: *const mlir.Attribute, + frgC: *const mlir.Attribute, + frgE: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteSparseMmaAtomAttrGet(ctx.ptr(), args.shapeMNK.ptr(), args.thrId.ptr(), args.layoutATV.ptr(), args.layoutBTV.ptr(), args.layoutCTV.ptr(), args.layoutETV.ptr(), args.shapeAMK.ptr(), args.shapeBNK.ptr(), args.shapeCMN.ptr(), args.shapeEMK.ptr(), args.frgA.ptr(), args.frgB.ptr(), args.frgC.ptr(), args.frgE.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getShapeMNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetShapeMNK(self.ptr()).ptr.?); + } + pub fn getThrId(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetThrId(self.ptr()).ptr.?); + } + pub fn getLayoutATV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetLayoutATV(self.ptr()).ptr.?); + } + pub fn getLayoutBTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetLayoutBTV(self.ptr()).ptr.?); + } + pub fn getLayoutCTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetLayoutCTV(self.ptr()).ptr.?); + } + pub fn getLayoutETV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetLayoutETV(self.ptr()).ptr.?); + } + pub fn getShapeAMK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetShapeAMK(self.ptr()).ptr.?); + } + pub fn getShapeBNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetShapeBNK(self.ptr()).ptr.?); + } + pub fn getShapeCMN(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetShapeCMN(self.ptr()).ptr.?); + } + pub fn getShapeEMK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetShapeEMK(self.ptr()).ptr.?); + } + pub fn getFrgA(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetFrgA(self.ptr()).ptr.?); + } + pub fn getFrgB(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetFrgB(self.ptr()).ptr.?); + } + pub fn getFrgC(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetFrgC(self.ptr()).ptr.?); + } + pub fn getFrgE(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetFrgE(self.ptr()).ptr.?); + } +}; + +/// `#cute.sparse_mx_mma_atom`: MMA atom attribute with scale-factor and metadata operands. +pub const SparseMxMmaAtomAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteSparseMxMmaAtom; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "sparse_mx_mma_atom"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMNK: *const mlir.Attribute, + thrId: *const mlir.Attribute, + layoutATV: *const mlir.Attribute, + layoutSFATV: *const mlir.Attribute, + layoutBTV: *const mlir.Attribute, + layoutSFBTV: *const mlir.Attribute, + layoutCTV: *const mlir.Attribute, + layoutETV: *const mlir.Attribute, + shapeAMK: *const mlir.Attribute, + shapeSFAMK: *const mlir.Attribute, + shapeBNK: *const mlir.Attribute, + shapeSFBNK: *const mlir.Attribute, + shapeCMN: *const mlir.Attribute, + shapeEMK: *const mlir.Attribute, + frgA: *const mlir.Attribute, + frgSFA: *const mlir.Attribute, + frgB: *const mlir.Attribute, + frgSFB: *const mlir.Attribute, + frgC: *const mlir.Attribute, + frgE: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteSparseMxMmaAtomAttrGet(ctx.ptr(), args.shapeMNK.ptr(), args.thrId.ptr(), args.layoutATV.ptr(), args.layoutSFATV.ptr(), args.layoutBTV.ptr(), args.layoutSFBTV.ptr(), args.layoutCTV.ptr(), args.layoutETV.ptr(), args.shapeAMK.ptr(), args.shapeSFAMK.ptr(), args.shapeBNK.ptr(), args.shapeSFBNK.ptr(), args.shapeCMN.ptr(), args.shapeEMK.ptr(), args.frgA.ptr(), args.frgSFA.ptr(), args.frgB.ptr(), args.frgSFB.ptr(), args.frgC.ptr(), args.frgE.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getShapeMNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeMNK(self.ptr()).ptr.?); + } + pub fn getThrId(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetThrId(self.ptr()).ptr.?); + } + pub fn getLayoutATV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutATV(self.ptr()).ptr.?); + } + pub fn getLayoutSFATV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutSFATV(self.ptr()).ptr.?); + } + pub fn getLayoutBTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutBTV(self.ptr()).ptr.?); + } + pub fn getLayoutSFBTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutSFBTV(self.ptr()).ptr.?); + } + pub fn getLayoutCTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutCTV(self.ptr()).ptr.?); + } + pub fn getLayoutETV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutETV(self.ptr()).ptr.?); + } + pub fn getShapeAMK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeAMK(self.ptr()).ptr.?); + } + pub fn getShapeSFAMK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeSFAMK(self.ptr()).ptr.?); + } + pub fn getShapeBNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeBNK(self.ptr()).ptr.?); + } + pub fn getShapeSFBNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeSFBNK(self.ptr()).ptr.?); + } + pub fn getShapeCMN(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeCMN(self.ptr()).ptr.?); + } + pub fn getShapeEMK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeEMK(self.ptr()).ptr.?); + } + pub fn getFrgA(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgA(self.ptr()).ptr.?); + } + pub fn getFrgSFA(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgSFA(self.ptr()).ptr.?); + } + pub fn getFrgB(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgB(self.ptr()).ptr.?); + } + pub fn getFrgSFB(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgSFB(self.ptr()).ptr.?); + } + pub fn getFrgC(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgC(self.ptr()).ptr.?); + } + pub fn getFrgE(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgE(self.ptr()).ptr.?); + } +}; + +// Operation builders, generated from the operations' .td. +/// `cute.add_offset`. Result types are explicit; attributes use mlir.Attribute. +pub fn add_offset(ctx: *mlir.Context, src: *const mlir.Value, offset: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.add_offset", .{ + .operands = .{ .flat = &.{ src, offset } }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + /// `cute.append_to_rank`. Result types are explicit; attributes use mlir.Attribute. pub fn append_to_rank(ctx: *mlir.Context, input: *const mlir.Value, element: *const mlir.Value, result_type: *const mlir.Type, rank: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute.append_to_rank", .{ @@ -771,10 +1430,31 @@ pub fn append_to_rank(ctx: *mlir.Context, input: *const mlir.Value, element: *co }); } +/// `cute.apply_swizzle`. Result types are explicit; attributes use mlir.Attribute. +pub fn apply_swizzle(ctx: *mlir.Context, ptr: *const mlir.Value, swizzled_ptr_type: *const mlir.Type, swizzle: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (swizzle) |value| attributes.appendAssumeCapacity(.named(ctx, "swizzle", value)); + return mlir.Operation.make(ctx, "cute.apply_swizzle", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{swizzled_ptr_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.assume`. Result types are explicit; attributes use mlir.Attribute. +pub fn assume(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.assume", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + /// `cute.blocked_product`. Result types are explicit; attributes use mlir.Attribute. -pub fn blocked_product(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +pub fn blocked_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute.blocked_product", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, + .operands = .{ .flat = &.{ input, tiler } }, .results = .{ .flat = &.{result_type} }, .location = location, }); @@ -791,24 +1471,21 @@ pub fn ceil_div(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir /// `cute.coalesce`. Result types are explicit; attributes use mlir.Attribute. pub fn coalesce(ctx: *mlir.Context, input: *const mlir.Value, target_profile: ?*const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - var operands: stdx.BoundedArray(*const mlir.Value, 2) = .empty; - operands.appendSliceAssumeCapacity(&.{input}); - if (target_profile) |value| operands.appendSliceAssumeCapacity(&.{value}); return mlir.Operation.make(ctx, "cute.coalesce", .{ - .operands = .{ .flat = operands.constSlice() }, + .operands = .{ .variadic = &.{ + &.{input}, + if (target_profile) |value| &.{value} else &.{}, + } }, .results = .{ .flat = &.{result_type} }, .location = location, }); } /// `cute.complement`. Result types are explicit; attributes use mlir.Attribute. -pub fn complement(ctx: *mlir.Context, input: *const mlir.Value, cotarget: ?*const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - var operands: stdx.BoundedArray(*const mlir.Value, 2) = .empty; - operands.appendSliceAssumeCapacity(&.{input}); - if (cotarget) |value| operands.appendSliceAssumeCapacity(&.{value}); +pub fn complement(ctx: *mlir.Context, input: *const mlir.Value, cotarget: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute.complement", .{ - .operands = .{ .flat = operands.constSlice() }, - .results = .{ .flat = &.{result_type} }, + .operands = .{ .flat = &.{ input, cotarget } }, + .results = .{ .flat = &.{res_type} }, .location = location, }); } @@ -841,564 +1518,19 @@ pub fn composed_get_outer(ctx: *mlir.Context, input: *const mlir.Value, result_t } /// `cute.composition`. Result types are explicit; attributes use mlir.Attribute. -pub fn composition(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +pub fn composition(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute.composition", .{ .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{result_type} }, + .results = .{ .flat = &.{res_type} }, .location = location, }); } -/// `cute.cosize`. Result types are explicit; attributes use mlir.Attribute. -pub fn cosize(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, mode: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); - return mlir.Operation.make(ctx, "cute.cosize", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute.crd2idx`. Result types are explicit; attributes use mlir.Attribute. -pub fn crd2idx(ctx: *mlir.Context, coord: *const mlir.Value, shape: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.crd2idx", .{ - .operands = .{ .flat = &.{ coord, shape } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.dice`. Result types are explicit; attributes use mlir.Attribute. -pub fn dice(ctx: *mlir.Context, input: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.dice", .{ - .operands = .{ .flat = &.{ input, coord } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.elem_less`. Result types are explicit; attributes use mlir.Attribute. -pub fn elem_less(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.elem_less", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.equal`. Result types are explicit; attributes use mlir.Attribute. -pub fn equal(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.equal", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.flat_divide`. Result types are explicit; attributes use mlir.Attribute. -pub fn flat_divide(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.flat_divide", .{ - .operands = .{ .flat = &.{ input, tiler } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.flat_product`. Result types are explicit; attributes use mlir.Attribute. -pub fn flat_product(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.flat_product", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.flatten`. Result types are explicit; attributes use mlir.Attribute. -pub fn flatten(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.flatten", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.get_layouts_from_tile`. Result types are explicit; attributes use mlir.Attribute. -pub fn get_layouts_from_tile(ctx: *mlir.Context, tile: *const mlir.Value, layouts_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.get_layouts_from_tile", .{ - .operands = .{ .flat = &.{tile} }, - .results = .{ .variadic = &.{layouts_types} }, - .location = location, - }); -} - -/// `cute.get_leaves`. Result types are explicit; attributes use mlir.Attribute. -pub fn get_leaves(ctx: *mlir.Context, input: *const mlir.Value, results_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.get_leaves", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .variadic = &.{results_types} }, - .location = location, - }); -} - -/// `cute.get`. Result types are explicit; attributes use mlir.Attribute. -pub fn get(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, mode: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); - return mlir.Operation.make(ctx, "cute.get", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute.get_scalars`. Result types are explicit; attributes use mlir.Attribute. -pub fn get_scalars(ctx: *mlir.Context, cute_value: *const mlir.Value, scalars_types: []const *const mlir.Type, only_dynamic: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (only_dynamic) |value| attributes.appendAssumeCapacity(.named(ctx, "only_dynamic", value)); - return mlir.Operation.make(ctx, "cute.get_scalars", .{ - .operands = .{ .flat = &.{cute_value} }, - .results = .{ .variadic = &.{scalars_types} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute.get_shape`. Result types are explicit; attributes use mlir.Attribute. -pub fn get_shape(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.get_shape", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.get_stride`. Result types are explicit; attributes use mlir.Attribute. -pub fn get_stride(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.get_stride", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.group_modes`. Result types are explicit; attributes use mlir.Attribute. -pub fn group_modes(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, begin: *const mlir.Attribute, end: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.group_modes", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "begin", begin), - .named(ctx, "end", end), - }, - .location = location, - }); -} - -/// `cute.idx2crd`. Result types are explicit; attributes use mlir.Attribute. -pub fn idx2crd(ctx: *mlir.Context, index: *const mlir.Value, shape: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.idx2crd", .{ - .operands = .{ .flat = &.{ index, shape } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.increment_coord`. Result types are explicit; attributes use mlir.Attribute. -pub fn increment_coord(ctx: *mlir.Context, coord: *const mlir.Value, shape: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.increment_coord", .{ - .operands = .{ .flat = &.{ coord, shape } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.layout_eval`. Result types are explicit; attributes use mlir.Attribute. -pub fn layout_eval(ctx: *mlir.Context, coord: *const mlir.Value, layout: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.layout_eval", .{ - .operands = .{ .flat = &.{ coord, layout } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.left_inverse`. Result types are explicit; attributes use mlir.Attribute. -pub fn left_inverse(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.left_inverse", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.logical_divide`. Result types are explicit; attributes use mlir.Attribute. -pub fn logical_divide(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.logical_divide", .{ - .operands = .{ .flat = &.{ input, tiler } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.logical_product`. Result types are explicit; attributes use mlir.Attribute. -pub fn logical_product(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.logical_product", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.make_composed_layout`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_composed_layout(ctx: *mlir.Context, inner: *const mlir.Value, offset: *const mlir.Value, outer: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_composed_layout", .{ - .operands = .{ .flat = &.{ inner, offset, outer } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.make_coord`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_coord(ctx: *mlir.Context, operands_: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_coord", .{ - .operands = .{ .variadic = &.{operands_} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.make_identity_layout`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_identity_layout(ctx: *mlir.Context, shape: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_identity_layout", .{ - .operands = .{ .flat = &.{shape} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.make_int_tuple`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_int_tuple(ctx: *mlir.Context, operands_: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_int_tuple", .{ - .operands = .{ .variadic = &.{operands_} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.make_layout_like`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_layout_like(ctx: *mlir.Context, src: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_layout_like", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.make_layout`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_layout(ctx: *mlir.Context, shape: *const mlir.Value, stride: *const mlir.Value, layout_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_layout", .{ - .operands = .{ .flat = &.{ shape, stride } }, - .results = .{ .flat = &.{layout_type} }, - .location = location, - }); -} - -/// `cute.make_ordered_layout`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_ordered_layout(ctx: *mlir.Context, shape: *const mlir.Value, order: *const mlir.Value, layout_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_ordered_layout", .{ - .operands = .{ .flat = &.{ shape, order } }, - .results = .{ .flat = &.{layout_type} }, - .location = location, - }); -} - -/// `cute.make_shape`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_shape(ctx: *mlir.Context, operands_: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_shape", .{ - .operands = .{ .variadic = &.{operands_} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.make_stride`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_stride(ctx: *mlir.Context, operands_: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_stride", .{ - .operands = .{ .variadic = &.{operands_} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.make_tile`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_tile(ctx: *mlir.Context, operands_: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_tile", .{ - .operands = .{ .variadic = &.{operands_} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.prepend_to_rank`. Result types are explicit; attributes use mlir.Attribute. -pub fn prepend_to_rank(ctx: *mlir.Context, input: *const mlir.Value, element: *const mlir.Value, result_type: *const mlir.Type, rank: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.prepend_to_rank", .{ - .operands = .{ .flat = &.{ input, element } }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "rank", rank), - }, - .location = location, - }); -} - -/// `cute.print`. Result types are explicit; attributes use mlir.Attribute. -pub fn print(ctx: *mlir.Context, value_: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.print", .{ - .operands = .{ .flat = &.{value_} }, - .location = location, - }); -} - -/// `cute.raked_product`. Result types are explicit; attributes use mlir.Attribute. -pub fn raked_product(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.raked_product", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.recast_layout`. Result types are explicit; attributes use mlir.Attribute. -pub fn recast_layout(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, new_type_bits: *const mlir.Attribute, old_type_bits: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.recast_layout", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .attributes = &.{ - .named(ctx, "new_type_bits", new_type_bits), - .named(ctx, "old_type_bits", old_type_bits), - }, - .location = location, - }); -} - -/// `cute.right_inverse`. Result types are explicit; attributes use mlir.Attribute. -pub fn right_inverse(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.right_inverse", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.select`. Result types are explicit; attributes use mlir.Attribute. -pub fn select(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, mode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.select", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "mode", mode), - }, - .location = location, - }); -} - -/// `cute.shape_div`. Result types are explicit; attributes use mlir.Attribute. -pub fn shape_div(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.shape_div", .{ - .operands = .{ .flat = &.{ a, b } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.size`. Result types are explicit; attributes use mlir.Attribute. -pub fn size(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, mode: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); - return mlir.Operation.make(ctx, "cute.size", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute.slice`. Result types are explicit; attributes use mlir.Attribute. -pub fn slice(ctx: *mlir.Context, input: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.slice", .{ - .operands = .{ .flat = &.{ input, coord } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.static`. Result types are explicit; attributes use mlir.Attribute. -pub fn static(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.static", .{ - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.tile_to_shape`. Result types are explicit; attributes use mlir.Attribute. -pub fn tile_to_shape(ctx: *mlir.Context, input: *const mlir.Value, shape: *const mlir.Value, order: ?*const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - var operands: stdx.BoundedArray(*const mlir.Value, 3) = .empty; - operands.appendSliceAssumeCapacity(&.{input}); - operands.appendSliceAssumeCapacity(&.{shape}); - if (order) |value| operands.appendSliceAssumeCapacity(&.{value}); - return mlir.Operation.make(ctx, "cute.tile_to_shape", .{ - .operands = .{ .flat = operands.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.tiled_divide`. Result types are explicit; attributes use mlir.Attribute. -pub fn tiled_divide(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.tiled_divide", .{ - .operands = .{ .flat = &.{ input, tiler } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.tiled_product`. Result types are explicit; attributes use mlir.Attribute. -pub fn tiled_product(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.tiled_product", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.to_coord`. Result types are explicit; attributes use mlir.Attribute. -pub fn to_coord(ctx: *mlir.Context, src: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.to_coord", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.to_int_tuple`. Result types are explicit; attributes use mlir.Attribute. -pub fn to_int_tuple(ctx: *mlir.Context, src: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.to_int_tuple", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.to_shape`. Result types are explicit; attributes use mlir.Attribute. -pub fn to_shape(ctx: *mlir.Context, src: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.to_shape", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.to_stride`. Result types are explicit; attributes use mlir.Attribute. -pub fn to_stride(ctx: *mlir.Context, src: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.to_stride", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.tuple_add`. Result types are explicit; attributes use mlir.Attribute. -pub fn tuple_add(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.tuple_add", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.tuple_product_each`. Result types are explicit; attributes use mlir.Attribute. -pub fn tuple_product_each(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.tuple_product_each", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.tuple_product`. Result types are explicit; attributes use mlir.Attribute. -pub fn tuple_product(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.tuple_product", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.tuple_sub`. Result types are explicit; attributes use mlir.Attribute. -pub fn tuple_sub(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.tuple_sub", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.zipped_divide`. Result types are explicit; attributes use mlir.Attribute. -pub fn zipped_divide(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.zipped_divide", .{ - .operands = .{ .flat = &.{ input, tiler } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.zipped_product`. Result types are explicit; attributes use mlir.Attribute. -pub fn zipped_product(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.zipped_product", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -// Builders from CuteOpsPrivate.td. -/// `cute.add_offset`. Result types are explicit; attributes use mlir.Attribute. -pub fn add_offset(ctx: *mlir.Context, src: *const mlir.Value, offset: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.add_offset", .{ - .operands = .{ .flat = &.{ src, offset } }, - .results = .{ .flat = &.{dst_type} }, - .location = location, - }); -} - -/// `cute.apply_swizzle`. Result types are explicit; attributes use mlir.Attribute. -pub fn apply_swizzle(ctx: *mlir.Context, ptr: *const mlir.Value, swizzled_ptr_type: *const mlir.Type, swizzle: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (swizzle) |value| attributes.appendAssumeCapacity(.named(ctx, "swizzle", value)); - return mlir.Operation.make(ctx, "cute.apply_swizzle", .{ - .operands = .{ .flat = &.{ptr} }, - .results = .{ .flat = &.{swizzled_ptr_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute.assume`. Result types are explicit; attributes use mlir.Attribute. -pub fn assume(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.assume", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .location = location, - }); -} - -/// `cute.copy_atom_call`. Result types are explicit; attributes use mlir.Attribute. -pub fn copy_atom_call(ctx: *mlir.Context, atom: *const mlir.Value, src: []const *const mlir.Value, dst: []const *const mlir.Value, pred: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.copy_atom_call", .{ +/// `cute.copy`. Result types are explicit; attributes use mlir.Attribute. +pub fn copy(ctx: *mlir.Context, copy_atom: *const mlir.Value, src: []const *const mlir.Value, dst: []const *const mlir.Value, pred: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.copy", .{ .operands = .{ .variadic = &.{ - &.{atom}, + &.{copy_atom}, src, dst, if (pred) |value| &.{value} else &.{}, @@ -1419,11 +1551,11 @@ pub fn copy_make_fragment(ctx: *mlir.Context, atom: *const mlir.Value, input: *c }); } -/// `cute.copy`. Result types are explicit; attributes use mlir.Attribute. -pub fn copy(ctx: *mlir.Context, copy_atom: *const mlir.Value, src: []const *const mlir.Value, dst: []const *const mlir.Value, pred: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.copy", .{ +/// `cute.copy_atom_call`. Result types are explicit; attributes use mlir.Attribute. +pub fn copy_atom_call(ctx: *mlir.Context, atom: *const mlir.Value, src: []const *const mlir.Value, dst: []const *const mlir.Value, pred: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.copy_atom_call", .{ .operands = .{ .variadic = &.{ - &.{copy_atom}, + &.{atom}, src, dst, if (pred) |value| &.{value} else &.{}, @@ -1432,6 +1564,27 @@ pub fn copy(ctx: *mlir.Context, copy_atom: *const mlir.Value, src: []const *cons }); } +/// `cute.cosize`. Result types are explicit; attributes use mlir.Attribute. +pub fn cosize(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, mode: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "cute.cosize", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.crd2idx`. Result types are explicit; attributes use mlir.Attribute. +pub fn crd2idx(ctx: *mlir.Context, coord: *const mlir.Value, layout: *const mlir.Value, index_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.crd2idx", .{ + .operands = .{ .flat = &.{ coord, layout } }, + .results = .{ .flat = &.{index_type} }, + .location = location, + }); +} + /// `cute.deref_arith_tuple_iter`. Result types are explicit; attributes use mlir.Attribute. pub fn deref_arith_tuple_iter(ctx: *mlir.Context, iter: *const mlir.Value, arith_tuple_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute.deref_arith_tuple_iter", .{ @@ -1442,10 +1595,10 @@ pub fn deref_arith_tuple_iter(ctx: *mlir.Context, iter: *const mlir.Value, arith } /// `cute.deref_desc_iter`. Result types are explicit; attributes use mlir.Attribute. -pub fn deref_desc_iter(ctx: *mlir.Context, iter: *const mlir.Value, value_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +pub fn deref_desc_iter(ctx: *mlir.Context, iter: *const mlir.Value, value__type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute.deref_desc_iter", .{ .operands = .{ .flat = &.{iter} }, - .results = .{ .flat = &.{value_type} }, + .results = .{ .flat = &.{value__type} }, .location = location, }); } @@ -1459,6 +1612,36 @@ pub fn derefine(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mli }); } +/// `cute.dice`. Result types are explicit; attributes use mlir.Attribute. +pub fn dice(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, coord: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.dice", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "coord", coord), + }, + .location = location, + }); +} + +/// `cute.elem_less`. Result types are explicit; attributes use mlir.Attribute. +pub fn elem_less(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.elem_less", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.equal`. Result types are explicit; attributes use mlir.Attribute. +pub fn equal(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.equal", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + /// `cute.fast_divmod.compute`. Result types are explicit; attributes use mlir.Attribute. pub fn fast_divmod_compute(ctx: *mlir.Context, dividend: *const mlir.Value, divisor: *const mlir.Value, quotient_type: *const mlir.Type, remainder_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute.fast_divmod.compute", .{ @@ -1524,16 +1707,34 @@ pub fn filter(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const /// `cute.filter_zeros`. Result types are explicit; attributes use mlir.Attribute. pub fn filter_zeros(ctx: *mlir.Context, input: *const mlir.Value, target_profile: ?*const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - var operands: stdx.BoundedArray(*const mlir.Value, 2) = .empty; - operands.appendSliceAssumeCapacity(&.{input}); - if (target_profile) |value| operands.appendSliceAssumeCapacity(&.{value}); return mlir.Operation.make(ctx, "cute.filter_zeros", .{ - .operands = .{ .flat = operands.constSlice() }, + .operands = .{ .variadic = &.{ + &.{input}, + if (target_profile) |value| &.{value} else &.{}, + } }, .results = .{ .flat = &.{res_type} }, .location = location, }); } +/// `cute.flat_divide`. Result types are explicit; attributes use mlir.Attribute. +pub fn flat_divide(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.flat_divide", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.flat_product`. Result types are explicit; attributes use mlir.Attribute. +pub fn flat_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.flat_product", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + /// `cute.gemm`. Result types are explicit; attributes use mlir.Attribute. pub fn gemm(ctx: *mlir.Context, mma_atom: *const mlir.Value, d: *const mlir.Value, a: []const *const mlir.Value, b: []const *const mlir.Value, c_: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute.gemm", .{ @@ -1548,6 +1749,18 @@ pub fn gemm(ctx: *mlir.Context, mma_atom: *const mlir.Value, d: *const mlir.Valu }); } +/// `cute.get`. Result types are explicit; attributes use mlir.Attribute. +pub fn get(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, mode: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "cute.get", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + /// `cute.get_flat_coord`. Result types are explicit; attributes use mlir.Attribute. pub fn get_flat_coord(ctx: *mlir.Context, index: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute.get_flat_coord", .{ @@ -1593,6 +1806,91 @@ pub fn get_layout(ctx: *mlir.Context, input: *const mlir.Value, layout_type: *co }); } +/// `cute.get_layouts_from_tile`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_layouts_from_tile(ctx: *mlir.Context, tile: *const mlir.Value, layouts_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_layouts_from_tile", .{ + .operands = .{ .flat = &.{tile} }, + .results = .{ .variadic = &.{ + layouts_types, + } }, + .location = location, + }); +} + +/// `cute.get_leaves`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_leaves(ctx: *mlir.Context, input: *const mlir.Value, results__types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_leaves", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .variadic = &.{ + results__types, + } }, + .location = location, + }); +} + +/// `cute.get_scalars`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_scalars(ctx: *mlir.Context, cute_value: *const mlir.Value, scalars_types: []const *const mlir.Type, only_dynamic: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (only_dynamic) |value| attributes.appendAssumeCapacity(.named(ctx, "only_dynamic", value)); + return mlir.Operation.make(ctx, "cute.get_scalars", .{ + .operands = .{ .flat = &.{cute_value} }, + .results = .{ .variadic = &.{ + scalars_types, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.get_shape`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_shape(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_shape", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.get_stride`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_stride(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_stride", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.group_modes`. Result types are explicit; attributes use mlir.Attribute. +pub fn group_modes(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, begin: *const mlir.Attribute, end: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.group_modes", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "begin", begin), + .named(ctx, "end", end), + }, + .location = location, + }); +} + +/// `cute.idx2crd`. Result types are explicit; attributes use mlir.Attribute. +pub fn idx2crd(ctx: *mlir.Context, index: *const mlir.Value, shape: *const mlir.Value, coord_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.idx2crd", .{ + .operands = .{ .flat = &.{ index, shape } }, + .results = .{ .flat = &.{coord_type} }, + .location = location, + }); +} + +/// `cute.increment_coord`. Result types are explicit; attributes use mlir.Attribute. +pub fn increment_coord(ctx: *mlir.Context, coord: *const mlir.Value, shape: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.increment_coord", .{ + .operands = .{ .flat = &.{ coord, shape } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + /// `cute.inttoptr`. Result types are explicit; attributes use mlir.Attribute. pub fn inttoptr(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute.inttoptr", .{ @@ -1613,6 +1911,15 @@ pub fn kernel_smem_size(ctx: *mlir.Context, size_type: *const mlir.Type, kernel_ }); } +/// `cute.left_inverse`. Result types are explicit; attributes use mlir.Attribute. +pub fn left_inverse(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.left_inverse", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + /// `cute.load_scaled_index`. Result types are explicit; attributes use mlir.Attribute. pub fn load_scaled_index(ctx: *mlir.Context, ptr_a: *const mlir.Value, ptr_b: *const mlir.Value, index: *const mlir.Value, stride: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute.load_scaled_index", .{ @@ -1643,10 +1950,30 @@ pub fn local_tile(ctx: *mlir.Context, input: *const mlir.Value, tile: *const mli }); } +/// `cute.logical_divide`. Result types are explicit; attributes use mlir.Attribute. +pub fn logical_divide(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.logical_divide", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.logical_product`. Result types are explicit; attributes use mlir.Attribute. +pub fn logical_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.logical_product", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + /// `cute.make_arith_tuple_iter`. Result types are explicit; attributes use mlir.Attribute. pub fn make_arith_tuple_iter(ctx: *mlir.Context, value_: ?*const mlir.Value, iter_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute.make_arith_tuple_iter", .{ - .operands = .{ .flat = if (value_) |value| &.{value} else &.{} }, + .operands = .{ .variadic = &.{ + if (value_) |value| &.{value} else &.{}, + } }, .results = .{ .flat = &.{iter_type} }, .location = location, }); @@ -1655,12 +1982,34 @@ pub fn make_arith_tuple_iter(ctx: *mlir.Context, value_: ?*const mlir.Value, ite /// `cute.make_atom`. Result types are explicit; attributes use mlir.Attribute. pub fn make_atom(ctx: *mlir.Context, values: []const *const mlir.Value, atom_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute.make_atom", .{ - .operands = .{ .variadic = &.{values} }, + .operands = .{ .variadic = &.{ + values, + } }, .results = .{ .flat = &.{atom_type} }, .location = location, }); } +/// `cute.make_composed_layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_composed_layout(ctx: *mlir.Context, inner: *const mlir.Value, offset: *const mlir.Value, outer: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_composed_layout", .{ + .operands = .{ .flat = &.{ inner, offset, outer } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_coord`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_coord(ctx: *mlir.Context, dynamicElements: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_coord", .{ + .operands = .{ .variadic = &.{ + dynamicElements, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + /// `cute.make_desc_iter`. Result types are explicit; attributes use mlir.Attribute. pub fn make_desc_iter(ctx: *mlir.Context, value_: *const mlir.Value, iter_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute.make_desc_iter", .{ @@ -1679,6 +2028,15 @@ pub fn make_fragment_like(ctx: *mlir.Context, src: *const mlir.Value, dst_type: }); } +/// `cute.make_identity_layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_identity_layout(ctx: *mlir.Context, shape: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_identity_layout", .{ + .operands = .{ .flat = &.{shape} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + /// `cute.make_identity_tensor`. Result types are explicit; attributes use mlir.Attribute. pub fn make_identity_tensor(ctx: *mlir.Context, shape: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute.make_identity_tensor", .{ @@ -1688,6 +2046,58 @@ pub fn make_identity_tensor(ctx: *mlir.Context, shape: *const mlir.Value, result }); } +/// `cute.make_int_tuple`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_int_tuple(ctx: *mlir.Context, dynamicElements: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_int_tuple", .{ + .operands = .{ .variadic = &.{ + dynamicElements, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_layout(ctx: *mlir.Context, shape: ?*const mlir.Value, stride: ?*const mlir.Value, layout_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_layout", .{ + .operands = .{ .variadic = &.{ + if (shape) |value| &.{value} else &.{}, + if (stride) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{layout_type} }, + .location = location, + }); +} + +/// `cute.make_layout_like`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_layout_like(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_layout_like", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.make_ordered_layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_ordered_layout(ctx: *mlir.Context, shape: *const mlir.Value, order: *const mlir.Value, layout_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_ordered_layout", .{ + .operands = .{ .flat = &.{ shape, order } }, + .results = .{ .flat = &.{layout_type} }, + .location = location, + }); +} + +/// `cute.make_shape`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_shape(ctx: *mlir.Context, dynamicElements: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_shape", .{ + .operands = .{ .variadic = &.{ + dynamicElements, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + /// `cute.make_sparse_elem`. Result types are explicit; attributes use mlir.Attribute. pub fn make_sparse_elem(ctx: *mlir.Context, physical_storage: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute.make_sparse_elem", .{ @@ -1697,6 +2107,28 @@ pub fn make_sparse_elem(ctx: *mlir.Context, physical_storage: *const mlir.Value, }); } +/// `cute.make_stride`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_stride(ctx: *mlir.Context, dynamicElements: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_stride", .{ + .operands = .{ .variadic = &.{ + dynamicElements, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_tile`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tile(ctx: *mlir.Context, dynamicElements: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_tile", .{ + .operands = .{ .variadic = &.{ + dynamicElements, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + /// `cute.make_tiled_copy`. Result types are explicit; attributes use mlir.Attribute. pub fn make_tiled_copy(ctx: *mlir.Context, atom: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute.make_tiled_copy", .{ @@ -1727,7 +2159,9 @@ pub fn make_tiled_mma(ctx: *mlir.Context, atom: *const mlir.Value, result_type: /// `cute.make_tuple`. Result types are explicit; attributes use mlir.Attribute. pub fn make_tuple(ctx: *mlir.Context, values: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute.make_tuple", .{ - .operands = .{ .variadic = &.{values} }, + .operands = .{ .variadic = &.{ + values, + } }, .results = .{ .flat = &.{result_type} }, .location = location, }); @@ -1735,11 +2169,11 @@ pub fn make_tuple(ctx: *mlir.Context, values: []const *const mlir.Value, result_ /// `cute.make_view`. Result types are explicit; attributes use mlir.Attribute. pub fn make_view(ctx: *mlir.Context, iter: *const mlir.Value, layout: ?*const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - var operands: stdx.BoundedArray(*const mlir.Value, 2) = .empty; - operands.appendSliceAssumeCapacity(&.{iter}); - if (layout) |value| operands.appendSliceAssumeCapacity(&.{value}); return mlir.Operation.make(ctx, "cute.make_view", .{ - .operands = .{ .flat = operands.constSlice() }, + .operands = .{ .variadic = &.{ + &.{iter}, + if (layout) |value| &.{value} else &.{}, + } }, .results = .{ .flat = &.{result_type} }, .location = location, }); @@ -1756,7 +2190,9 @@ pub fn memref_alloc_smem(ctx: *mlir.Context, memref_type: *const mlir.Type, loca /// `cute.memref.alloca`. Result types are explicit; attributes use mlir.Attribute. pub fn memref_alloca(ctx: *mlir.Context, layout: ?*const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute.memref.alloca", .{ - .operands = .{ .flat = if (layout) |value| &.{value} else &.{} }, + .operands = .{ .variadic = &.{ + if (layout) |value| &.{value} else &.{}, + } }, .results = .{ .flat = &.{result_type} }, .location = location, }); @@ -1773,20 +2209,13 @@ pub fn memref_load(ctx: *mlir.Context, src: *const mlir.Value, coord: *const mli /// `cute.memref.load_vec`. Result types are explicit; attributes use mlir.Attribute. pub fn memref_load_vec(ctx: *mlir.Context, src: *const mlir.Value, mask: ?*const mlir.Value, pass_thru: ?*const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - var operands: stdx.BoundedArray(*const mlir.Value, 3) = .empty; - operands.appendSliceAssumeCapacity(&.{src}); - if (mask) |value| operands.appendSliceAssumeCapacity(&.{value}); - if (pass_thru) |value| operands.appendSliceAssumeCapacity(&.{value}); return mlir.Operation.make(ctx, "cute.memref.load_vec", .{ - .operands = .{ .flat = operands.constSlice() }, + .operands = .{ .variadic = &.{ + &.{src}, + if (mask) |value| &.{value} else &.{}, + if (pass_thru) |value| &.{value} else &.{}, + } }, .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ - 1, - if (mask != null) 1 else 0, - if (pass_thru != null) 1 else 0, - })), - }, .location = location, }); } @@ -1801,12 +2230,24 @@ pub fn memref_store(ctx: *mlir.Context, dst: *const mlir.Value, coord: *const ml /// `cute.memref.store_vec`. Result types are explicit; attributes use mlir.Attribute. pub fn memref_store_vec(ctx: *mlir.Context, value_: *const mlir.Value, dst: *const mlir.Value, mask: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { - var operands: stdx.BoundedArray(*const mlir.Value, 3) = .empty; - operands.appendSliceAssumeCapacity(&.{value_}); - operands.appendSliceAssumeCapacity(&.{dst}); - if (mask) |value| operands.appendSliceAssumeCapacity(&.{value}); return mlir.Operation.make(ctx, "cute.memref.store_vec", .{ - .operands = .{ .flat = operands.constSlice() }, + .operands = .{ .variadic = &.{ + &.{value_}, + &.{dst}, + if (mask) |value| &.{value} else &.{}, + } }, + .location = location, + }); +} + +/// `cute.mma.make_fragment`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_make_fragment(ctx: *mlir.Context, atom: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.mma.make_fragment", .{ + .operands = .{ .flat = &.{ atom, input } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operand_id", operand_id), + }, .location = location, }); } @@ -1825,14 +2266,13 @@ pub fn mma_atom_call(ctx: *mlir.Context, atom: *const mlir.Value, d: *const mlir }); } -/// `cute.mma.make_fragment`. Result types are explicit; attributes use mlir.Attribute. -pub fn mma_make_fragment(ctx: *mlir.Context, atom: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.mma.make_fragment", .{ - .operands = .{ .flat = &.{ atom, input } }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "operand_id", operand_id), - }, +/// `cute.prefetch`. Result types are explicit; attributes use mlir.Attribute. +pub fn prefetch(ctx: *mlir.Context, prefetch_atom: *const mlir.Value, src: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.prefetch", .{ + .operands = .{ .variadic = &.{ + &.{prefetch_atom}, + src, + } }, .location = location, }); } @@ -1848,13 +2288,28 @@ pub fn prefetch_atom_call(ctx: *mlir.Context, atom: *const mlir.Value, src: []co }); } -/// `cute.prefetch`. Result types are explicit; attributes use mlir.Attribute. -pub fn prefetch(ctx: *mlir.Context, prefetch_atom: *const mlir.Value, src: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.prefetch", .{ +/// `cute.prepend_to_rank`. Result types are explicit; attributes use mlir.Attribute. +pub fn prepend_to_rank(ctx: *mlir.Context, input: *const mlir.Value, element: *const mlir.Value, result_type: *const mlir.Type, rank: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.prepend_to_rank", .{ + .operands = .{ .flat = &.{ input, element } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "rank", rank), + }, + .location = location, + }); +} + +/// `cute.print`. Result types are explicit; attributes use mlir.Attribute. +pub fn print(ctx: *mlir.Context, values: []const *const mlir.Value, fmt: ?*const mlir.Attribute, operand_signed: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (fmt) |value| attributes.appendAssumeCapacity(.named(ctx, "fmt", value)); + if (operand_signed) |value| attributes.appendAssumeCapacity(.named(ctx, "operand_signed", value)); + return mlir.Operation.make(ctx, "cute.print", .{ .operands = .{ .variadic = &.{ - &.{prefetch_atom}, - src, + values, } }, + .attributes = attributes.constSlice(), .location = location, }); } @@ -1877,24 +2332,24 @@ pub fn print_tma_desc_tiled(ctx: *mlir.Context, input: *const mlir.Value, fd: *c /// `cute.print_view`. Result types are explicit; attributes use mlir.Attribute. pub fn print_view(ctx: *mlir.Context, src: *const mlir.Value, coord: ?*const mlir.Value, verbose: ?*const mlir.Attribute, is_signed: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - var operands: stdx.BoundedArray(*const mlir.Value, 2) = .empty; - operands.appendSliceAssumeCapacity(&.{src}); - if (coord) |value| operands.appendSliceAssumeCapacity(&.{value}); var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; if (verbose) |value| attributes.appendAssumeCapacity(.named(ctx, "verbose", value)); if (is_signed) |value| attributes.appendAssumeCapacity(.named(ctx, "is_signed", value)); return mlir.Operation.make(ctx, "cute.print_view", .{ - .operands = .{ .flat = operands.constSlice() }, + .operands = .{ .variadic = &.{ + &.{src}, + if (coord) |value| &.{value} else &.{}, + } }, .attributes = attributes.constSlice(), .location = location, }); } /// `cute.ptr.load`. Result types are explicit; attributes use mlir.Attribute. -pub fn ptr_load(ctx: *mlir.Context, ptr: *const mlir.Value, value_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +pub fn ptr_load(ctx: *mlir.Context, ptr: *const mlir.Value, value__type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute.ptr.load", .{ .operands = .{ .flat = &.{ptr} }, - .results = .{ .flat = &.{value_type} }, + .results = .{ .flat = &.{value__type} }, .location = location, }); } @@ -1916,6 +2371,15 @@ pub fn ptrtoint(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mli }); } +/// `cute.raked_product`. Result types are explicit; attributes use mlir.Attribute. +pub fn raked_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.raked_product", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + /// `cute.random_delay`. Result types are explicit; attributes use mlir.Attribute. pub fn random_delay(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute.random_delay", .{ @@ -1932,27 +2396,91 @@ pub fn recast_iter(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const }); } +/// `cute.recast_layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn recast_layout(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, new_type_bits: *const mlir.Attribute, old_type_bits: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.recast_layout", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = &.{ + .named(ctx, "new_type_bits", new_type_bits), + .named(ctx, "old_type_bits", old_type_bits), + }, + .location = location, + }); +} + /// `cute.reduce`. Result types are explicit; attributes use mlir.Attribute. pub fn reduce(ctx: *mlir.Context, input: *const mlir.Value, shape: *const mlir.Value, init_val: *const mlir.Value, predication: ?*const mlir.Value, output_type: *const mlir.Type, reduced_shape_type: ?*const mlir.Type, reduction_op: *const mlir.Attribute, reduction_profile: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - var operands: stdx.BoundedArray(*const mlir.Value, 4) = .empty; - operands.appendSliceAssumeCapacity(&.{input}); - operands.appendSliceAssumeCapacity(&.{shape}); - operands.appendSliceAssumeCapacity(&.{init_val}); - if (predication) |value| operands.appendSliceAssumeCapacity(&.{value}); - var results: stdx.BoundedArray(*const mlir.Type, 2) = .empty; - results.appendSliceAssumeCapacity(&.{output_type}); - if (reduced_shape_type) |value| results.appendSliceAssumeCapacity(&.{value}); var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; attributes.appendAssumeCapacity(.named(ctx, "reduction_op", reduction_op)); if (reduction_profile) |value| attributes.appendAssumeCapacity(.named(ctx, "reduction_profile", value)); return mlir.Operation.make(ctx, "cute.reduce", .{ - .operands = .{ .flat = operands.constSlice() }, - .results = .{ .flat = results.constSlice() }, + .operands = .{ .variadic = &.{ + &.{input}, + &.{shape}, + &.{init_val}, + if (predication) |value| &.{value} else &.{}, + } }, + .results = .{ .variadic = &.{ + &.{output_type}, + if (reduced_shape_type) |value| &.{value} else &.{}, + } }, .attributes = attributes.constSlice(), .location = location, }); } +/// `cute.right_inverse`. Result types are explicit; attributes use mlir.Attribute. +pub fn right_inverse(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.right_inverse", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.select`. Result types are explicit; attributes use mlir.Attribute. +pub fn select(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, mode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.select", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "mode", mode), + }, + .location = location, + }); +} + +/// `cute.shape_div`. Result types are explicit; attributes use mlir.Attribute. +pub fn shape_div(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.shape_div", .{ + .operands = .{ .flat = &.{ a, b } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.size`. Result types are explicit; attributes use mlir.Attribute. +pub fn size(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, mode: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "cute.size", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.slice`. Result types are explicit; attributes use mlir.Attribute. +pub fn slice(ctx: *mlir.Context, input: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.slice", .{ + .operands = .{ .flat = &.{ input, coord } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + /// `cute.smem_partition_size`. Result types are explicit; attributes use mlir.Attribute. pub fn smem_partition_size(ctx: *mlir.Context, size_type: *const mlir.Type, partition_id: *const mlir.Attribute, cumulative: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; @@ -1965,26 +2493,25 @@ pub fn smem_partition_size(ctx: *mlir.Context, size_type: *const mlir.Type, part }); } +/// `cute.static`. Result types are explicit; attributes use mlir.Attribute. +pub fn static(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.static", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + /// `cute.stencil_divide`. Result types are explicit; attributes use mlir.Attribute. pub fn stencil_divide(ctx: *mlir.Context, input: *const mlir.Value, stencil: *const mlir.Value, padding_upper: ?*const mlir.Value, padding_lower: ?*const mlir.Value, traversal_stride: ?*const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - var operands: stdx.BoundedArray(*const mlir.Value, 5) = .empty; - operands.appendSliceAssumeCapacity(&.{input}); - operands.appendSliceAssumeCapacity(&.{stencil}); - if (padding_upper) |value| operands.appendSliceAssumeCapacity(&.{value}); - if (padding_lower) |value| operands.appendSliceAssumeCapacity(&.{value}); - if (traversal_stride) |value| operands.appendSliceAssumeCapacity(&.{value}); return mlir.Operation.make(ctx, "cute.stencil_divide", .{ - .operands = .{ .flat = operands.constSlice() }, + .operands = .{ .variadic = &.{ + &.{input}, + &.{stencil}, + if (padding_upper) |value| &.{value} else &.{}, + if (padding_lower) |value| &.{value} else &.{}, + if (traversal_stride) |value| &.{value} else &.{}, + } }, .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ - 1, - 1, - if (padding_upper != null) 1 else 0, - if (padding_lower != null) 1 else 0, - if (traversal_stride != null) 1 else 0, - })), - }, .location = location, }); } @@ -2000,6 +2527,15 @@ pub fn symbolic(ctx: *mlir.Context, symbolic_type: *const mlir.Type, name: *cons }); } +/// `cute.tile_to_shape`. Result types are explicit; attributes use mlir.Attribute. +pub fn tile_to_shape(ctx: *mlir.Context, block: *const mlir.Value, trg_shape: *const mlir.Value, ord_shape: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tile_to_shape", .{ + .operands = .{ .flat = &.{ block, trg_shape, ord_shape } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + /// `cute.tiled.copy.partition_D`. Result types are explicit; attributes use mlir.Attribute. pub fn tiled_copy_partition_D(ctx: *mlir.Context, tiled_copy: *const mlir.Value, input: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute.tiled.copy.partition_D", .{ @@ -2051,6 +2587,60 @@ pub fn tiled_mma_partition_shape(ctx: *mlir.Context, tiled_mma: *const mlir.Valu }); } +/// `cute.tiled_divide`. Result types are explicit; attributes use mlir.Attribute. +pub fn tiled_divide(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tiled_divide", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tiled_product`. Result types are explicit; attributes use mlir.Attribute. +pub fn tiled_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tiled_product", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.to_int_tuple`. Result types are explicit; attributes use mlir.Attribute. +pub fn to_int_tuple(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.to_int_tuple", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.tuple.product`. Result types are explicit; attributes use mlir.Attribute. +pub fn tuple_product(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple.product", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tuple.product_each`. Result types are explicit; attributes use mlir.Attribute. +pub fn tuple_product_each(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple.product_each", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tuple_add`. Result types are explicit; attributes use mlir.Attribute. +pub fn tuple_add(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple_add", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + /// `cute.tuple_div`. Result types are explicit; attributes use mlir.Attribute. pub fn tuple_div(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute.tuple_div", .{ @@ -2078,19 +2668,10 @@ pub fn tuple_mul(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Va }); } -/// `cute.tuple.product_each`. Result types are explicit; attributes use mlir.Attribute. -pub fn @"tuple.product_each"(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.tuple.product_each", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.tuple.product`. Result types are explicit; attributes use mlir.Attribute. -pub fn @"tuple.product"(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.tuple.product", .{ - .operands = .{ .flat = &.{input} }, +/// `cute.tuple_sub`. Result types are explicit; attributes use mlir.Attribute. +pub fn tuple_sub(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple_sub", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, .results = .{ .flat = &.{result_type} }, .location = location, }); @@ -2100,13 +2681,36 @@ pub fn @"tuple.product"(ctx: *mlir.Context, input: *const mlir.Value, result_typ pub fn unpack_tuple(ctx: *mlir.Context, tuple: *const mlir.Value, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute.unpack_tuple", .{ .operands = .{ .flat = &.{tuple} }, - .results = .{ .variadic = &.{result_types} }, + .results = .{ .variadic = &.{ + result_types, + } }, + .location = location, + }); +} + +/// `cute.zipped_divide`. Result types are explicit; attributes use mlir.Attribute. +pub fn zipped_divide(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.zipped_divide", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.zipped_product`. Result types are explicit; attributes use mlir.Attribute. +pub fn zipped_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.zipped_product", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, .location = location, }); } pub const operation_names: []const []const u8 = &.{ + "cute.add_offset", "cute.append_to_rank", + "cute.apply_swizzle", + "cute.assume", "cute.blocked_product", "cute.ceil_div", "cute.coalesce", @@ -2115,69 +2719,17 @@ pub const operation_names: []const []const u8 = &.{ "cute.composed_get_offset", "cute.composed_get_outer", "cute.composition", + "cute.copy", + "cute.copy.make_fragment", + "cute.copy_atom_call", "cute.cosize", "cute.crd2idx", - "cute.dice", - "cute.elem_less", - "cute.equal", - "cute.flat_divide", - "cute.flat_product", - "cute.flatten", - "cute.get_layouts_from_tile", - "cute.get_leaves", - "cute.get", - "cute.get_scalars", - "cute.get_shape", - "cute.get_stride", - "cute.group_modes", - "cute.idx2crd", - "cute.increment_coord", - "cute.layout_eval", - "cute.left_inverse", - "cute.logical_divide", - "cute.logical_product", - "cute.make_composed_layout", - "cute.make_coord", - "cute.make_identity_layout", - "cute.make_int_tuple", - "cute.make_layout_like", - "cute.make_layout", - "cute.make_ordered_layout", - "cute.make_shape", - "cute.make_stride", - "cute.make_tile", - "cute.prepend_to_rank", - "cute.print", - "cute.raked_product", - "cute.recast_layout", - "cute.right_inverse", - "cute.select", - "cute.shape_div", - "cute.size", - "cute.slice", - "cute.static", - "cute.tile_to_shape", - "cute.tiled_divide", - "cute.tiled_product", - "cute.to_coord", - "cute.to_int_tuple", - "cute.to_shape", - "cute.to_stride", - "cute.tuple_add", - "cute.tuple_product_each", - "cute.tuple_product", - "cute.tuple_sub", - "cute.zipped_divide", - "cute.zipped_product", - "cute.add_offset", - "cute.apply_swizzle", - "cute.assume", - "cute.copy_atom_call", - "cute.copy.make_fragment", - "cute.copy", "cute.deref_arith_tuple_iter", "cute.deref_desc_iter", "cute.derefine", + "cute.dice", + "cute.elem_less", + "cute.equal", "cute.fast_divmod.compute", "cute.fast_divmod.create_divisor", "cute.fast_divmod.divide", @@ -2186,23 +2738,47 @@ pub const operation_names: []const []const u8 = &.{ "cute.fast_divmod.make_divisor", "cute.filter", "cute.filter_zeros", + "cute.flat_divide", + "cute.flat_product", "cute.gemm", + "cute.get", "cute.get_flat_coord", "cute.get_hier_coord", "cute.get_integral_coord", "cute.get_iter", "cute.get_layout", + "cute.get_layouts_from_tile", + "cute.get_leaves", + "cute.get_scalars", + "cute.get_shape", + "cute.get_stride", + "cute.group_modes", + "cute.idx2crd", + "cute.increment_coord", "cute.inttoptr", "cute.kernel_smem_size", + "cute.left_inverse", "cute.load_scaled_index", "cute.local_partition", "cute.local_tile", + "cute.logical_divide", + "cute.logical_product", "cute.make_arith_tuple_iter", "cute.make_atom", + "cute.make_composed_layout", + "cute.make_coord", "cute.make_desc_iter", "cute.make_fragment_like", + "cute.make_identity_layout", "cute.make_identity_tensor", + "cute.make_int_tuple", + "cute.make_layout", + "cute.make_layout_like", + "cute.make_ordered_layout", + "cute.make_shape", "cute.make_sparse_elem", + "cute.make_stride", + "cute.make_tile", "cute.make_tiled_copy", "cute.make_tiled_copy_v2", "cute.make_tiled_mma", @@ -2214,33 +2790,51 @@ pub const operation_names: []const []const u8 = &.{ "cute.memref.load_vec", "cute.memref.store", "cute.memref.store_vec", - "cute.mma_atom_call", "cute.mma.make_fragment", - "cute.prefetch_atom_call", + "cute.mma_atom_call", "cute.prefetch", + "cute.prefetch_atom_call", + "cute.prepend_to_rank", + "cute.print", "cute.print_tma_desc_im2col", "cute.print_tma_desc_tiled", "cute.print_view", "cute.ptr.load", "cute.ptr.store", "cute.ptrtoint", + "cute.raked_product", "cute.random_delay", "cute.recast_iter", + "cute.recast_layout", "cute.reduce", + "cute.right_inverse", + "cute.select", + "cute.shape_div", + "cute.size", + "cute.slice", "cute.smem_partition_size", + "cute.static", "cute.stencil_divide", "cute.symbolic", + "cute.tile_to_shape", "cute.tiled.copy.partition_D", "cute.tiled.copy.partition_S", "cute.tiled.copy.retile", "cute.tiled.mma.partition", "cute.tiled.mma.partition_shape", + "cute.tiled_divide", + "cute.tiled_product", + "cute.to_int_tuple", + "cute.tuple.product", + "cute.tuple.product_each", + "cute.tuple_add", "cute.tuple_div", "cute.tuple_mod", "cute.tuple_mul", - "cute.tuple.product_each", - "cute.tuple.product", + "cute.tuple_sub", "cute.unpack_tuple", + "cute.zipped_divide", + "cute.zipped_product", }; fn testContext() !*mlir.Context { @@ -2273,346 +2867,143 @@ test "every bound operation is registered" { try std.testing.expect(!ctx.allowUnregisteredDialects()); } -test "public and private types through the C API" { - const ctx = try testContext(); - defer ctx.deinit(); - const tuple = try algebraType(ctx, .int_tuple, "(16,32)"); - try std.testing.expect(tuple.eql(try mlir.Type.parse(ctx, "!cute.int_tuple<\"(16,32)\">"))); - try std.testing.expect(tuple.isA(CuteType) != null); - const f32_type = mlir.Type.float(ctx, .f32); - try std.testing.expect((try pointerType(ctx, f32_type, .gmem, 16)).eql(try mlir.Type.parse(ctx, "!cute.ptr>"))); - try std.testing.expect((try constrainedIntType(ctx, .i32, 8)).eql(try mlir.Type.parse(ctx, "!cute.i32"))); - try std.testing.expect((try algebraAttribute(ctx, .layout, "(16,32):(32,1)")).eql(try mlir.Attribute.parse(ctx, "#cute.layout<\"(16,32):(32,1)\">"))); - try std.testing.expect(mlir.Type.int(ctx, .i32).isA(CuteType) == null); - try std.testing.expect((try nvgpu.SmemDescType.get(ctx, .{})).isA(nvgpu.NVGPUType) != null); - const atom = try nvgpu.CopyAtomSIMTSyncCopyType.get(ctx, .{ .payload = .string(ctx, "") }); - try std.testing.expect(atom.type_().eql(try mlir.Type.parse(ctx, "!cute_nvgpu.atom.universal_copy"))); -} - -test "typed constructors and accessors use native type storage" { - const ctx = try testContext(); - defer ctx.deinit(); - const f32_type = mlir.Type.float(ctx, .f32); - const space = mlir.Attribute.string(ctx, "smem"); - const swizzle = try algebraAttribute(ctx, .swizzle, "S<3,4,3>"); - const pointer = try PtrType.get(ctx, .{ - .valueType = f32_type, - .memorySpace = space, - .alignment = 16, - .swizzle = swizzle, - }); - try std.testing.expect(pointer.getValueType().?.eql(f32_type)); - try std.testing.expect(pointer.getMemorySpace().eql(space)); - try std.testing.expectEqual(@as(u64, 16), pointer.getAlignment()); - try std.testing.expect(pointer.getSwizzle().?.eql(swizzle)); - try std.testing.expect(pointer.type_().isA(PtrType) != null); - try std.testing.expect(f32_type.isA(PtrType) == null); - const untyped_pointer = try PtrType.get(ctx, .{ .memorySpace = space, .alignment = 0 }); - try std.testing.expect(untyped_pointer.getValueType() == null); - try std.testing.expect(untyped_pointer.getSwizzle() == null); - - const layout_attr = try algebraAttribute(ctx, .layout, "4:1"); - const layout = try LayoutType.get(ctx, .{ .attr = layout_attr }); - try std.testing.expect(layout.getAttr().eql(layout_attr)); - const memref = try MemRefType.get(ctx, .{ .ptr = pointer.type_(), .layout = layout.type_() }); - try std.testing.expect(memref.getPtr().eql(pointer.type_())); - try std.testing.expect(memref.getLayout().eql(layout.type_())); - const tuple = try algebraType(ctx, .int_tuple, "(4,8)"); - const iterator = try ArithTupleIteratorType.get(ctx, .{ .arithTuple = tuple }); - try std.testing.expect(iterator.getArithTuple().eql(tuple)); - const coords = try CoordTensorType.get(ctx, .{ .arithTuple = tuple, .layout = layout.type_() }); - try std.testing.expect(coords.getArithTuple().eql(tuple)); - try std.testing.expect(coords.getLayout().eql(layout.type_())); - - const i32_type = try ConstrainedInt32Type.get(ctx, .{ .divisibleBy = 8 }); - const i64_type = try ConstrainedInt64Type.get(ctx, .{ .divisibleBy = 16 }); - try std.testing.expectEqual(@as(u64, 8), i32_type.getDivisibleBy()); - try std.testing.expectEqual(@as(u64, 16), i64_type.getDivisibleBy()); - const divisor = try FastDivmodDivisorType.get(ctx, .{ .width = 64, .isPow2 = true }); - try std.testing.expectEqual(@as(c_uint, 64), divisor.getWidth()); - try std.testing.expect(divisor.getIsPow2()); - const sparse = try SparseElemType.get(ctx, .{ .numLogical = 2, .physicalType = f32_type }); - try std.testing.expectEqual(@as(c_int, 2), sparse.getNumLogical()); - try std.testing.expect(sparse.getPhysicalType().eql(f32_type)); - - const payload = mlir.Attribute.string(ctx, ""); - const atom = try nvgpu.CopyAtomSIMTSyncCopyType.get(ctx, .{ .payload = payload }); - try std.testing.expect(atom.getPayload().eql(payload)); - try std.testing.expect(atom.type_().isA(nvgpu.CopyAtomSIMTSyncCopyType) != null); - inline for (.{ nvgpu.SmemDescType, nvgpu.TmaDescriptorTiledType, nvgpu.TmaDescriptorIm2ColType, nvgpu.WorkIdResponseType }) |T| { - try std.testing.expect((try T.get(ctx, .{})).type_().isA(T) != null); - } -} - -test "explicit algebra attribute bindings and type constructors" { +test "generated types and attributes rebuild from their getters" { + @setEvalBranchQuota(100_000); const ctx = try testContext(); defer ctx.deinit(); inline for (.{ - .{ IntTupleAttr, IntTupleType, "(16,32)" }, - .{ CoordAttr, CoordType, "(_,?)" }, - .{ ShapeAttr, ShapeType, "(16,32)" }, - .{ StrideAttr, StrideType, "(32,1)" }, - .{ LayoutAttr, LayoutType, "(16,32):(32,1)" }, - .{ TileAttr, TileType, "[(2,3):(1,2);_]" }, - .{ ComposedLayoutAttr, ComposedLayoutType, "S<3,5,4> o 2 o (2,3):(1,2)" }, - .{ SwizzleAttr, SwizzleType, "S<3,4,3>" }, + .{ ArithTupleIteratorType, "!cute.arith_tuple_iter<\"(?,?{div=16})\">" }, + .{ ComposedLayoutType, "!cute.composed_layout<\"(4,5):(1,4) o 2 o (2,3):(1,2)\">" }, + .{ ConstrainedIntType, "!cute.i16" }, + .{ CoordTensorType, "!cute.coord_tensor<\"(?,?,?)\", \"((512,1),1):((1/2@0,0),0)\">" }, + .{ CoordType, "!cute.coord<\"1\">" }, + .{ FastDivmodDivisorType, "!cute.fast_divmod_divisor<32>" }, + .{ IntTupleType, "!cute.int_tuple<\"1\">" }, + .{ LayoutType, "!cute.layout<\"(2,3):(1,2)\">" }, + .{ MemRefType, "!cute.memref, S<2,4,3>, \"((8,16),(32,1),(1,3)):((32,256),(1,0),(0,4096))\">" }, + .{ PtrType, "!cute.ptr, S<2,4,3>>" }, + .{ ShapeType, "!cute.shape<\"(2,3)\">" }, + .{ SparseElemType, "!cute.sparse_elem<2, vector<1xf16>>" }, + .{ StrideType, "!cute.stride<\"1\">" }, + .{ SwizzleType, "!cute.swizzle<\"S<0,4,3>\">" }, + .{ TileType, "!cute.tile<\"[(2,3):(1,2)]\">" }, + .{ TiledCopyType, "!cute.tiled_copy, layout_copy_tv = <\"(32,4):(4,1)\">, tiler_mn = <\"[4:1;32:1]\">>" }, + .{ TiledCopyV2Type, "!cute.tiled_copy_v2, layout_copy_tv = <\"(1,8):(0,1)\">, tiler_mn = <\"[8:1]\">>" }, + .{ TiledMmaType, "!cute.tiled_mma f32 >, atom_layout_MNK = <\"(16,16,1):(16,1,0)\">, permutation_MNK = <\"[(16,4):(4,1);(16,4):(4,1);_]\">>" }, + .{ TupleType, "!cute.tuple" }, + .{ BitLayoutAttr, "#cute.bitlayout<<\"(8,4):(1,8)\">>" }, + .{ CopyAtomAttr, "#cute.copy_atom, layout_tv = (<\"(1,8):(0,1)\">, <\"(1,8):(0,1)\">, <\"(1,8):(0,1)\">)>" }, + .{ CopyAtomV2Attr, "#cute.copy_atom_v2, layout_tv = (<\"(1,8):(0,1)\">, <\"(1,8):(0,1)\">), frg = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, + .{ MmaAtomAttr, "#cute.mma_atom<<\"(16,8,16)\">, <\"32:1\">, layout_TV = (<\"1:0\">, <\"1:0\">, <\"1:0\">), shapes = (<\"(16,16)\">, <\"(8,16)\">, <\"(16,8)\">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, + .{ MxMmaAtomAttr, "#cute.mx_mma_atom<<\"(16,8,16)\">, <\"32:1\">, layout_TV = (<\"1:0\">, <\"1:0\">, <\"1:0\">, <\"1:0\">, <\"1:0\">), shapes = (<\"(16,16)\">, <\"(8,16)\">, <\"(16,8)\">, <\"1\">, <\"2\">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, + .{ SparseMmaAtomAttr, "#cute.sparse_mma_atom<<\"(16,8,16)\">, <\"32:1\">, layout_TV = (<\"1:0\">, <\"1:0\">, <\"1:0\">, <\"1:0\">), shapes = (<\"(16,16)\">, <\"(8,16)\">, <\"(16,8)\">, <\"1\">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, + .{ SparseMxMmaAtomAttr, "#cute.sparse_mx_mma_atom<<\"(16,8,16)\">, <\"32:1\">, layout_TV = (<\"1:0\">, <\"2:0\">, <\"3:0\">, <\"4:0\">, <\"5:0\">, <\"6:0\">), shapes = (<\"1\">, <\"2\">, <\"3\">, <\"4\">, <\"5\">, <\"6\">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, + .{ nvgpu.CopyAtomBulkCopyG2SType, "!cute_nvgpu.atom.bulk_copy_g2s" }, + .{ nvgpu.CopyAtomBulkCopyS2GType, "!cute_nvgpu.atom.bulk_copy_s2g" }, + .{ nvgpu.CopyAtomBulkCopyS2SType, "!cute_nvgpu.atom.bulk_copy_s2s" }, + .{ nvgpu.CopyAtomDsmemStoreType, "!cute_nvgpu.atom.dsmem_store" }, + .{ nvgpu.CopyAtomG2RType, "!cute_nvgpu.atom.g2r" }, + .{ nvgpu.CopyAtomIm2ColTmaLoadType, "!cute_nvgpu.atom.im2col_tma_load mcast = true tma_gbasis = <\"(64,128):(1@1,1@0)\">>" }, + .{ nvgpu.CopyAtomIm2ColTmaStoreType, "!cute_nvgpu.atom.im2col_tma_store tma_gbasis = <\"(32,128):(1@1,1@0)\">>" }, + .{ nvgpu.CopyAtomLdsmType, "!cute_nvgpu.atom.ldsm, sz_pattern = u16, num_matrices = 4, t>" }, + .{ nvgpu.CopyAtomNonExec2DGather4TmaLoadType, "!cute_nvgpu.atom.non_exec_2d_gather4_tma_load, tma_format = F16_RN>" }, + .{ nvgpu.CopyAtomNonExec2DScatter4TmaStoreType, "!cute_nvgpu.atom.non_exec_2d_scatter4_tma_store, tma_format = F16_RN>" }, + .{ nvgpu.CopyAtomNonExecIm2ColTmaLoadType, "!cute_nvgpu.atom.non_exec_im2col_tma_load, tma_format = F16_RN>" }, + .{ nvgpu.CopyAtomNonExecIm2ColTmaStoreType, "!cute_nvgpu.atom.non_exec_im2col_tma_store, tma_format = F16_RN>" }, + .{ nvgpu.CopyAtomNonExecTiledTmaLoadType, "!cute_nvgpu.atom.non_exec_tiled_tma_load, tma_format = BF16_RN>" }, + .{ nvgpu.CopyAtomNonExecTiledTmaReduceType, "!cute_nvgpu.atom.non_exec_tiled_tma_reduce, tma_format = F32_RN>" }, + .{ nvgpu.CopyAtomNonExecTiledTmaStoreType, "!cute_nvgpu.atom.non_exec_tiled_tma_store, tma_format = BF16_RN>" }, + .{ nvgpu.CopyAtomR2GType, "!cute_nvgpu.atom.r2g" }, + .{ nvgpu.CopyAtomR2SType, "!cute_nvgpu.atom.r2s" }, + .{ nvgpu.CopyAtomS2RType, "!cute_nvgpu.atom.s2r" }, + .{ nvgpu.CopyAtomSIMTAsyncCopyType, "!cute_nvgpu.atom.simt_async_copy" }, + .{ nvgpu.CopyAtomSIMTMultimemLdReduceType, "!cute_nvgpu.atom.simt_multimem_ld_reduce, acquire, gpu>" }, + .{ nvgpu.CopyAtomSIMTMultimemRedType, "!cute_nvgpu.atom.simt_multimem_red" }, + .{ nvgpu.CopyAtomSIMTMultimemStType, "!cute_nvgpu.atom.simt_multimem_st" }, + .{ nvgpu.CopyAtomSIMTSyncCopyType, "!cute_nvgpu.atom.universal_copy" }, + .{ nvgpu.CopyAtomSM100CopyS2TType, "!cute_nvgpu.atom.s2t_copy" }, + .{ nvgpu.CopyAtomSM100S2TCopyV2Type, "!cute_nvgpu.atom.sm100_s2t_copy_v2" }, + .{ nvgpu.CopyAtomSM100TmemLoadType, "!cute_nvgpu.atom.tmem_load" }, + .{ nvgpu.CopyAtomSM100TmemStoreType, "!cute_nvgpu.atom.tmem_store" }, + .{ nvgpu.CopyAtomSM107TmemLoadSPCompressType, "!cute_nvgpu.atom.tmem_load_spcompress" }, + .{ nvgpu.CopyAtomSM10xTmemLoadRedType, "!cute_nvgpu.atom.tmem_load_red" }, + .{ nvgpu.CopyAtomStsmType, "!cute_nvgpu.atom.stsm, num_matrices = 4, n>" }, + .{ nvgpu.CopyAtomTmaLoadType, "!cute_nvgpu.atom.tma_load mcast = true tma_gbasis = <\"(64,1):(1@0,1@1)\">>" }, + .{ nvgpu.CopyAtomTmaReduceType, "!cute_nvgpu.atom.tma_reduce tma_gbasis = <\"(32,128):(1@1,1@0)\">>" }, + .{ nvgpu.CopyAtomTmaStoreType, "!cute_nvgpu.atom.tma_store tma_gbasis = <\"(64,16):(1@0,1@1)\">>" }, + .{ nvgpu.MmaAtomSM100UMMABlockScaledSparseType, "!cute_nvgpu.sm100.mma_bs_sp<128x128x64, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, sparse_metadata_format = tid, frag_kind = ss, vec_size = 32>" }, + .{ nvgpu.MmaAtomSM100UMMABlockScaledType, "!cute_nvgpu.sm100.mma_bs<128x128x96, num_cta = 1, ab_major = (k, k), elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E8M0FNU, frag_kind = ss, vec_size = 16, arch_promote = sm_103>" }, + .{ nvgpu.MmaAtomSM100UMMASparseType, "!cute_nvgpu.sm100.mma_sp<128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f16, f16, f32), e_type = i8, sparse_metadata_format = tid, frag_kind = ss, c_scale_exp = 0>" }, + .{ nvgpu.MmaAtomSM100UMMAType, "!cute_nvgpu.sm100.mma<128x128x16, num_cta = 1, ab_major = (k, mn), elem_type = (f16, f16, f32), frag_kind = ss, c_scale_exp = 0>" }, + .{ nvgpu.MmaAtomSM107UMMABlockScaledSparseType, "!cute_nvgpu.sm107.mma_bs_sp<128x128x64, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, sparse_metadata_format = tid, frag_kind = ss, vec_size = 32, ab_collector_op = (discard, discard)>" }, + .{ nvgpu.MmaAtomSM107UMMABlockScaledType, "!cute_nvgpu.sm107.mma_bs<128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, frag_kind = ss, vec_size = 32, ab_collector_op = (discard, use), int_overflow = >" }, + .{ nvgpu.MmaAtomSM107UMMASparseType, "!cute_nvgpu.sm107.mma_sp<128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f16, f16, f32), e_type = i8, sparse_metadata_format = tid, frag_kind = ss, c_scale_exp = 0, ab_collector_op = (discard, discard)>" }, + .{ nvgpu.MmaAtomSM107UMMAType, "!cute_nvgpu.sm107.mma<128x128x16, num_cta = 1, ab_major = (k, mn), elem_type = (f16, f16, f32), frag_kind = ss, c_scale_exp = 0, ab_collector_op = (lastuse, fill)>" }, + .{ nvgpu.MmaAtomSM120BlockScaledType, "!cute_nvgpu.SM120.mma_bs<16x8x64, vec_size = 32, elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E8M0FNU, use_sf_layout_TV = true>" }, + .{ nvgpu.MmaAtomSM80SparseType, "!cute_nvgpu.sm80.sparse_mma<16x8x64, elem_type = (i8, i8, i32), md_format = tid, int_overflow = >" }, + .{ nvgpu.MmaAtomSM80Type, "!cute_nvgpu.sm80.mma<16x8x32, elem_type = (i8, i8, i32), int_overflow = , binary_op = >" }, + .{ nvgpu.MmaAtomSM89Type, "!cute_nvgpu.sm89.mma<16x8x32, elem_type = (f8E4M3FN, f8E4M3FN, f32) >" }, + .{ nvgpu.MmaAtomSM90Type, "!cute_nvgpu.sm90.mma<64x128x32, ab_major = (k, k), elem_type = (i8, i8, i32), frag_kind = ss, int_overflow = , a_neg, b_neg>" }, + .{ nvgpu.SmemDescCircularSM103Type, "!cute_nvgpu.sm103.smem_desc_circular<<\"(8,128,2):(128,1,1024)\">>" }, + .{ nvgpu.SmemDescCircularSM107Type, "!cute_nvgpu.sm107.smem_desc_circular<<\"(8,128,2):(128,1,1024)\">>" }, + .{ nvgpu.SmemDescSM107Type, "!cute_nvgpu.sm107.smem_desc" }, + .{ nvgpu.SmemDescType, "!cute_nvgpu.smem_desc" }, + .{ nvgpu.SmemDescViewType, "!cute_nvgpu.smem_desc_view" }, + .{ nvgpu.TmaDescriptorIm2ColType, "!cute_nvgpu.tma_descriptor_im2col" }, + .{ nvgpu.TmaDescriptorTiledType, "!cute_nvgpu.tma_descriptor_tiled" }, + .{ nvgpu.UniversalFmaAtomType, "!cute_nvgpu.atom.universal_fma<1x1x1, (f32, f32) -> f32 >" }, + .{ nvgpu.WorkIdResponseType, "!cute_nvgpu.workid_response" }, + .{ nvgpu.NotImplementedFrgAttr, "#cute_nvgpu.not_implemented_frg<>" }, + .{ nvgpu.RmemFrgAttr, "#cute_nvgpu.rmem_frg" }, + .{ nvgpu.SM100CircularSmemFrgAttr, "#cute_nvgpu.arch.sm100.circular_smem_frg" }, + .{ nvgpu.SM100SmemFrgAttr, "#cute_nvgpu.arch.sm100.smem_frg" }, + .{ nvgpu.SM100TmemEFrgAttr, "#cute_nvgpu.arch.sm100.tmem_e_frg" }, + .{ nvgpu.SM100TmemFrgAttr, "#cute_nvgpu.arch.sm100.tmem_frg" }, + .{ nvgpu.SM100TmemSfFrgAttr, "#cute_nvgpu.arch.sm100.tmem_sf_frg" }, + .{ nvgpu.SM107SmemFrgAttr, "#cute_nvgpu.arch.sm107.smem_frg" }, + .{ nvgpu.SM90SmemFrgAttr, "#cute_nvgpu.arch.sm90.smem_frg" }, }) |example| { - const A = example[0]; - const T = example[1]; - const attr = try A.get(ctx, example[2]); - try std.testing.expect(attr.attribute().isA(A) != null); - try std.testing.expect(mlir.Attribute.string(ctx, example[2]).isA(A) == null); - try std.testing.expectError(error.InvalidMlir, A.get(ctx, "invalid")); - const value = attr.getValue(); - try std.testing.expect(attr.eql(try A.get(ctx, value))); - const type_ = try T.get(ctx, .{ .attr = attr.attribute() }); - try std.testing.expect(type_.getAttr().eql(attr.attribute())); - + const T = example[0]; + const is_type = @hasDecl(T, "type_"); + const parsed = if (is_type) try mlir.Type.parse(ctx, example[1]) else try mlir.Attribute.parse(ctx, example[1]); + const value = parsed.isA(T) orelse return error.TestUnexpectedResult; + try std.testing.expect((try rebuild(T, ctx, value)).eql(value)); var text: std.Io.Writer.Allocating = .init(std.testing.allocator); defer text.deinit(); - try text.writer.print("{f}", .{attr}); - try std.testing.expect(attr.attribute().eql(try mlir.Attribute.parse(ctx, text.written()))); - text.clearRetainingCapacity(); - try text.writer.print("{f}", .{type_}); - try std.testing.expect(type_.type_().eql(try mlir.Type.parse(ctx, text.written()))); - // The getter's view survives other attribute creation in this context. - try std.testing.expectEqualStrings(value, attr.getValue()); + try text.writer.print("{f}", .{value}); + try std.testing.expect(parsed.eql(if (is_type) try mlir.Type.parse(ctx, text.written()) else try mlir.Attribute.parse(ctx, text.written()))); } -} - -test "typed constructors reject invalid parameters" { - const ctx = try testContext(); - defer ctx.deinit(); - const f32_type = mlir.Type.float(ctx, .f32); - const space = mlir.Attribute.string(ctx, "smem"); - try std.testing.expectError(error.InvalidMlir, LayoutType.get(ctx, .{ .attr = space })); - try std.testing.expectError(error.InvalidMlir, ArithTupleIteratorType.get(ctx, .{ .arithTuple = f32_type })); - try std.testing.expectError(error.InvalidMlir, ConstrainedInt32Type.get(ctx, .{ .divisibleBy = 0 })); - try std.testing.expectError(error.InvalidMlir, FastDivmodDivisorType.get(ctx, .{ .width = 8, .isPow2 = false })); - try std.testing.expectError(error.InvalidMlir, SparseElemType.get(ctx, .{ .numLogical = 0, .physicalType = f32_type })); - try std.testing.expectError(error.InvalidMlir, PtrType.get(ctx, .{ .memorySpace = space, .alignment = 3 })); - try std.testing.expectError(error.InvalidMlir, PtrType.get(ctx, .{ .memorySpace = space, .alignment = 16, .swizzle = space })); - const pointer = try pointerType(ctx, f32_type, .smem, 16); - try std.testing.expectError(error.InvalidMlir, MemRefType.get(ctx, .{ .ptr = pointer, .layout = f32_type })); - try std.testing.expectError(error.InvalidMlir, nvgpu.CopyAtomSIMTSyncCopyType.get(ctx, .{ .payload = .unit(ctx) })); - - const other_ctx = try testContext(); - defer other_ctx.deinit(); - try std.testing.expectError(error.InvalidMlir, PtrType.get(other_ctx, .{ .valueType = f32_type, .memorySpace = .string(other_ctx, "smem"), .alignment = 16 })); - try std.testing.expectError(error.InvalidMlir, PtrType.get(other_ctx, .{ .memorySpace = space, .alignment = 16 })); -} - -test "all NVGPU type constructors preserve their storage" { - const ctx = try testContext(); - defer ctx.deinit(); - const payload = mlir.Attribute.string(ctx, ""); - inline for (comptime std.meta.declarations(nvgpu)) |decl| { - const T = @field(nvgpu, decl.name); - if (@TypeOf(T) == type) { - if (@hasDecl(T, "InitArgs")) { - const has_payload = @hasField(T.InitArgs, "payload"); - const type_ = try T.get(ctx, if (has_payload) .{ .payload = payload } else .{}); - try std.testing.expect(type_.type_().isA(T) != null); - try std.testing.expect(type_.type_().isA(nvgpu.NVGPUType) != null); - if (has_payload) try std.testing.expect(type_.getPayload().eql(payload)); - var text: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer text.deinit(); - try text.writer.print("{f}", .{type_}); - try std.testing.expect(type_.type_().eql(try mlir.Type.parse(ctx, text.written()))); + // Every case of every enum attribute. + inline for (.{ cute, nvgpu }) |namespace| { + inline for (comptime std.meta.declarations(namespace)) |decl| { + const T = @field(namespace, decl.name); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque" and @hasDecl(T, "InitArgs")) { + const fields = @typeInfo(T.InitArgs).@"struct".fields; + if (fields.len == 1 and comptime std.mem.eql(u8, fields[0].name, "value")) { + inline for (@typeInfo(fields[0].type).@"enum".fields) |case| { + const attr = try T.get(ctx, .{ .value = @enumFromInt(case.value) }); + try std.testing.expectEqual(case.value, @intFromEnum(attr.getValue())); + try std.testing.expect((try rebuild(T, ctx, attr)).eql(attr)); + } + } } } } + try std.testing.expect((try nvgpu.SmemDescType.get(ctx, .{})).type_().isA(nvgpu.NVGPUType) != null); } -test "dotted private tuple builders remain distinct from public builders" { - const ctx = try testContext(); - defer ctx.deinit(); - const loc = mlir.Location.unknown(ctx); - const tuple_type = try algebraType(ctx, .int_tuple, "(2,(3,4))"); - const tuple = cute.static(ctx, tuple_type, loc); - defer tuple.deinit(); - - inline for (.{ - .{ cute.tuple_product, "cute.tuple_product", "24" }, - .{ cute.@"tuple.product", "cute.tuple.product", "24" }, - .{ cute.tuple_product_each, "cute.tuple_product_each", "(2,12)" }, - .{ cute.@"tuple.product_each", "cute.tuple.product_each", "(2,12)" }, - }) |example| { - const op = example[0](ctx, tuple.result(0), try algebraType(ctx, .int_tuple, example[2]), loc); - defer op.deinit(); - try std.testing.expectEqualStrings(example[1], op.name()); - try std.testing.expect(op.verify()); - } -} - -test "NVGPU builders preserve optional operand segments and attributes" { - const ctx = try testContext(); - defer ctx.deinit(); - const loc = mlir.Location.unknown(ctx); - const gmem = try pointerType(ctx, .float(ctx, .f32), .gmem, 16); - const dsmem = try pointerType(ctx, .float(ctx, .f32), .dsmem, 16); - const barrier = try pointerType(ctx, .int(ctx, .i64), .dsmem, 8); - const block = mlir.Block.init( - &.{ gmem, dsmem, barrier, .int(ctx, .i16), .int(ctx, .i64) }, - &.{ loc, loc, loc, loc, loc }, - ); - defer block.deinit(); - for ([_]bool{ false, true }) |with_mask| { - for ([_]bool{ false, true }) |with_cache| { - const byte_count = mlir.Attribute.int(ctx, .i32, 128); - const op = nvgpu.arch_copy_SM90_bulk_copy_g2s(ctx, block.argument(0), block.argument(1), block.argument(2), if (with_mask) block.argument(3) else null, if (with_cache) block.argument(4) else null, byte_count, loc); - defer op.deinit(); - try std.testing.expect(op.verify()); - try std.testing.expect(op.attributeByName("size").?.eql(byte_count)); - try std.testing.expectEqual(@as(usize, 3) + @intFromBool(with_mask) + @intFromBool(with_cache), op.numOperands()); - try std.testing.expect(op.attributeByName("operandSegmentSizes").?.eql(.denseArray( - ctx, - .i32, - &.{ 1, 1, 1, @intFromBool(with_mask), @intFromBool(with_cache) }, - ))); - if (with_cache) try std.testing.expect(op.operand(op.numOperands() - 1).eql(block.argument(4))); +fn rebuild(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T { + var args: T.InitArgs = undefined; + var elements: [8]*const mlir.Type = undefined; + inline for (@typeInfo(T.InitArgs).@"struct".fields) |field| { + const name = [_]u8{comptime std.ascii.toUpper(field.name[0])} ++ field.name[1..]; + if (field.type == []const *const mlir.Type) { + const count = @field(T, "getNum" ++ name)(value); + for (elements[0..count], 0..) |*element, i| element.* = @field(T, "get" ++ name[0 .. name.len - 1])(value, i); + @field(args, field.name) = elements[0..count]; + } else { + @field(args, field.name) = @field(T, "get" ++ name)(value); } } -} - -test "build public, private and NVGPU operations and round-trip bytecode" { - const ctx = try testContext(); - defer ctx.deinit(); - const loc = mlir.Location.unknown(ctx); - const module = mlir.Module.init(loc); - defer module.deinit(); - const body = module.body(); - - const shape_type = try algebraType(ctx, .shape, "4"); - const stride_type = try algebraType(ctx, .stride, "1"); - const layout_type = try algebraType(ctx, .layout, "4:1"); - const shape = cute.static(ctx, shape_type, loc).appendTo(body); - const stride = cute.static(ctx, stride_type, loc).appendTo(body); - const constructed_shape = cute.make_shape(ctx, &.{}, shape_type, loc).appendTo(body); - try std.testing.expect(constructed_shape.attributeByName("operandSegmentSizes").?.eql(.denseArray(ctx, .i32, &.{0}))); - const leaves = cute.get_leaves(ctx, constructed_shape.result(0), &.{shape_type}, loc).appendTo(body); - try std.testing.expectEqual(@as(usize, 1), leaves.numResults()); - try std.testing.expect(leaves.attributeByName("resultSegmentSizes").?.eql(.denseArray(ctx, .i32, &.{1}))); - const layout = cute.make_layout(ctx, shape.result(0), stride.result(0), layout_type, loc).appendTo(body); - try std.testing.expect(layout.result(0).type_().eql(layout_type)); - - const memref_type = (try MemRefType.get(ctx, .{ - .ptr = try pointerType(ctx, .float(ctx, .f32), .smem, 16), - .layout = layout_type, - })).type_(); - const alloc = cute.memref_alloc_smem(ctx, memref_type, loc).appendTo(body); - const assumed = cute.assume(ctx, alloc.result(0), memref_type, loc).appendTo(body); - const atom_type = (try nvgpu.CopyAtomSIMTSyncCopyType.get(ctx, .{ .payload = .string(ctx, "") })).type_(); - const atom = cute.make_atom(ctx, &.{}, atom_type, loc).appendTo(body); - const copy_op = cute.copy(ctx, atom.result(0), &.{ alloc.result(0), assumed.result(0) }, &.{assumed.result(0)}, null, loc).appendTo(body); - const segments = mlir.Attribute.denseArray(ctx, .i32, &.{ 1, 2, 1, 0 }); - try std.testing.expectEqual(@as(usize, 4), copy_op.numOperands()); - try std.testing.expect(copy_op.attributeByName("operandSegmentSizes").?.eql(segments)); - - const predicate = mlir.Operation.make(ctx, "arith.constant", .{ - .results = .{ .flat = &.{.int(ctx, .i1)} }, - .attributes = &.{.named(ctx, "value", .boolean(ctx, true))}, - .location = loc, - }).appendTo(body); - const predicated_copy = cute.copy(ctx, atom.result(0), &.{alloc.result(0)}, &.{assumed.result(0)}, predicate.result(0), loc).appendTo(body); - try std.testing.expect(predicated_copy.attributeByName("operandSegmentSizes").?.eql(.denseArray(ctx, .i32, &.{ 1, 1, 1, 1 }))); - - // Operand groups have no fixed per-group capacity in the Zig builder. - const many_sources = [_]*const mlir.Value{alloc.result(0)} ** 256; - const large_copy = cute.copy(ctx, atom.result(0), &many_sources, &.{assumed.result(0)}, null, loc).appendTo(body); - try std.testing.expectEqual(@as(usize, 258), large_copy.numOperands()); - try std.testing.expect(large_copy.attributeByName("operandSegmentSizes").?.eql(.denseArray(ctx, .i32, &.{ 1, 256, 1, 0 }))); - - const smem_size = nvgpu.arch_get_dyn_smem_size(ctx, .int(ctx, .i32), loc).appendTo(body); - try std.testing.expectEqualStrings("cute_nvgpu.arch.get_dyn_smem_size", smem_size.name()); - const pointer = try pointerType(ctx, .float(ctx, .f32), .smem, 16); - _ = nvgpu.arch_alloc_smem(ctx, pointer, .int(ctx, .i32, 16), loc).appendTo(body); - try std.testing.expect(module.operation().verify()); - - var text: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer text.deinit(); - try text.writer.print("{f}", .{module.operation()}); - const parsed = try mlir.Module.parse(ctx, text.written()); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); - - var bytes: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer bytes.deinit(); - try module.operation().writeBytecode(null, &bytes.writer); - try std.testing.expect(std.mem.startsWith(u8, bytes.written(), "ML\xefR")); - const reader_ctx = try testContext(); - defer reader_ctx.deinit(); - const round_trip = try mlir.Operation.parse(reader_ctx, bytes.written(), "cute-zig.mlirbc"); - defer round_trip.deinit(); - try std.testing.expect(round_trip.verify()); - var round_trip_text: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer round_trip_text.deinit(); - try round_trip_text.writer.print("{f}", .{round_trip}); - try std.testing.expectEqualStrings(text.written(), round_trip_text.written()); -} - -test "SM120 block-scaled MMA preserves fragment groups and optional selectors" { - const ctx = try testContext(); - defer ctx.deinit(); - const loc = mlir.Location.unknown(ctx); - const i32_type = mlir.Type.int(ctx, .i32); - const f32_type = mlir.Type.float(ctx, .f32); - const arg_types = [_]*const mlir.Type{i32_type} ** 6 ++ - [_]*const mlir.Type{f32_type} ** 4 ++ - [_]*const mlir.Type{.int(ctx, .i8)} ** 2 ++ - [_]*const mlir.Type{.int(ctx, .i16)} ** 2; - const block = mlir.Block.init(&arg_types, &([_]*const mlir.Location{loc} ** 14)); - defer block.deinit(); - const shape = try algebraAttribute(ctx, .shape, "(16,8,64)"); - const fp4 = mlir.Attribute.typeAttr(try mlir.Type.parse(ctx, "f4E2M1FN")); - const scale_type = mlir.Attribute.typeAttr(try mlir.Type.parse(ctx, "f8E8M0FNU")); - for ([_]bool{ false, true }) |with_a| { - for ([_]bool{ false, true }) |with_b| { - const op = nvgpu.arch_mma_SM120_block_scaled( - ctx, - &.{ block.argument(0), block.argument(1), block.argument(2), block.argument(3) }, - &.{ block.argument(4), block.argument(5) }, - &.{ block.argument(6), block.argument(7), block.argument(8), block.argument(9) }, - block.argument(10), - block.argument(11), - if (with_a) block.argument(12) else null, - if (with_b) block.argument(13) else null, - &([_]*const mlir.Type{f32_type} ** 4), - shape, - .int(ctx, .i32, 32), - null, - null, - fp4, - fp4, - scale_type, - loc, - ); - defer op.deinit(); - try std.testing.expect(op.verify()); - try std.testing.expectEqual(@as(usize, 12) + @intFromBool(with_a) + @intFromBool(with_b), op.numOperands()); - for (0..12) |i| try std.testing.expect(op.operand(i).eql(block.argument(i))); - if (with_a) try std.testing.expect(op.operand(12).eql(block.argument(12))); - if (with_b) try std.testing.expect(op.operand(op.numOperands() - 1).eql(block.argument(13))); - try std.testing.expectEqual(@as(usize, 4), op.numResults()); - try std.testing.expect(op.attributeByName("operandSegmentSizes").?.eql(.denseArray( - ctx, - .i32, - &.{ 4, 2, 4, 1, 1, @intFromBool(with_a), @intFromBool(with_b) }, - ))); - } - } -} - -test "private verifier rejects inconsistent operand segments" { - const ctx = try testContext(); - defer ctx.deinit(); - const loc = mlir.Location.unknown(ctx); - const atom_type = (try nvgpu.CopyAtomSIMTSyncCopyType.get(ctx, .{ .payload = .string(ctx, "") })).type_(); - const atom = cute.make_atom(ctx, &.{}, atom_type, loc); - defer atom.deinit(); - const copy_op = cute.copy(ctx, atom.result(0), &.{}, &.{}, null, loc); - defer copy_op.deinit(); - try std.testing.expect(copy_op.verify()); - copy_op.setAttributeByName("operandSegmentSizes", .denseArray(ctx, .i32, &.{ 1, 1, 0, 0 })); - try std.testing.expect(!copy_op.verify()); + return T.get(ctx, args); } diff --git a/mlir/dialects/cute_ir/cute_nvgpu.zig b/mlir/dialects/cute_ir/cute_nvgpu.zig index 8e41a988..7d7ac63d 100644 --- a/mlir/dialects/cute_ir/cute_nvgpu.zig +++ b/mlir/dialects/cute_ir/cute_nvgpu.zig @@ -4,6 +4,8 @@ const c = @import("c"); const mlir = @import("mlir"); const stdx = @import("stdx"); +const cute = @import("cute_ir.zig"); + pub const dialect_namespace = "cute_nvgpu"; pub const NVGPUType = opaque { @@ -20,1129 +22,2751 @@ pub const NVGPUType = opaque { } }; -/// `!cute_nvgpu.atom.bulk_copy_g2s`. +/// Parses `!cute_nvgpu.`, such as `` for +/// `CopyAtomSIMTSyncCopyType`, and returns it as a `T`. +pub fn typeFromPayload(comptime T: type, ctx: *mlir.Context, payload: []const u8) mlir.Error!*const T { + var buffer: [1024]u8 = undefined; + const text = std.fmt.bufPrint(&buffer, "!" ++ dialect_namespace ++ "." ++ T.mnemonic ++ "{s}", .{payload}) catch return error.InvalidMlir; + return (try mlir.Type.parse(ctx, text)).isA(T) orelse error.InvalidMlir; +} + +// Enums, generated from the dialect's .td files; values are the compiler's. + +/// NVGpu arch number like sm_103. +pub const Arch = enum(u32) { + sm_52 = 520, + sm_70 = 700, + sm_75 = 750, + sm_80 = 800, + sm_86 = 860, + sm_90 = 900, + sm_100 = 1000, + sm_103 = 1030, + sm_107 = 1070, +}; + +/// Fields stored in Bulk Load Copy Atom type. +pub const AtomCopyFieldBulkCopyG2S = enum(u32) { + tma_bar = 1, + mcast_mask = 2, + cache_policy = 7, +}; + +/// Fields stored in Bulk Store Copy Atom type. +pub const AtomCopyFieldBulkCopyS2G = enum(u32) { + byte_mask = 4, + cache_policy = 7, +}; + +/// Fields stored in Bulk CTA to Cluster Copy Atom type. +pub const AtomCopyFieldBulkCopyS2S = enum(u32) { + tma_bar = 1, + cta_rank = 5, +}; + +/// Fields stored in CopyAtomDsmemStoreType. +pub const AtomCopyFieldDsmemStore = enum(u32) { + mbar_ptr = 8, +}; + +/// Fields stored in CopyAtomG2RType. +pub const AtomCopyFieldLoadGlobal = enum(u32) { + cache_policy = 7, +}; + +/// Fields stored in CopyAtomNonExec2DGather4TmaLoadType. +pub const AtomCopyFieldNonExec2DGather4TmaLoad = enum(u32) { + tma_desc = 3, +}; + +/// Fields stored in CopyAtomNonExec2DScatter4TmaStoreType. +pub const AtomCopyFieldNonExec2DScatter4TmaStore = enum(u32) { + tma_desc = 3, +}; + +/// Fields stored in CopyAtomNonExecIm2ColTmaLoadType. +pub const AtomCopyFieldNonExecIm2ColTmaLoad = enum(u32) { + im2col_tma_desc = 6, +}; + +/// Fields stored in CopyAtomNonExecIm2ColTmaStoreType. +pub const AtomCopyFieldNonExecIm2ColTmaStore = enum(u32) { + im2col_tma_desc = 6, +}; + +/// Fields stored in CopyAtomNonExecTiledTmaLoadType. +pub const AtomCopyFieldNonExecTiledTmaLoad = enum(u32) { + tma_desc = 3, +}; + +/// Fields stored in CopyAtomNonExecTiledTmaReduceType. +pub const AtomCopyFieldNonExecTiledTmaReduce = enum(u32) { + tma_desc = 3, +}; + +/// Fields stored in CopyAtomNonExecTiledTmaStoreType. +pub const AtomCopyFieldNonExecTiledTmaStore = enum(u32) { + tma_desc = 3, +}; + +/// Fields stored in CopyAtomR2GType. +pub const AtomCopyFieldStoreGlobal = enum(u32) { + cache_policy = 7, +}; + +/// Fields stored in Tma Load Copy Atom type. +pub const AtomCopyFieldTmaLoad = enum(u32) { + tma_descriptor_ptr = 0, + tma_bar = 1, + mcast_mask = 2, + cache_policy = 7, + g_stride = 9, +}; + +/// Fields stored in Tma Reduce Copy Atom type. +pub const AtomCopyFieldTmaReduce = enum(u32) { + tma_descriptor_ptr = 0, + cache_policy = 7, + g_stride = 9, +}; + +/// Fields stored in Tma Store Copy Atom type. +pub const AtomCopyFieldTmaStore = enum(u32) { + tma_descriptor_ptr = 0, + cache_policy = 7, + g_stride = 9, +}; + +/// Fields stored in MMA Atom types for SM100. +pub const AtomMmaFieldSM100 = enum(u32) { + accum_c = 0, + neg_a = 1, + neg_b = 2, + disable_output_lane = 6, +}; + +/// Fields stored in MMA Atom types for SM100 (block scaled). +pub const AtomMmaFieldSM100BlockScaled = enum(u32) { + accum_c = 0, + neg_a = 1, + neg_b = 2, + sf_a = 3, + sf_b = 4, +}; + +/// Fields stored in MMA Atom types for SM100 (block scaled). +pub const AtomMmaFieldSM100BlockScaledSparse = enum(u32) { + accum_c = 0, + neg_a = 1, + neg_b = 2, + sf_a = 3, + sf_b = 4, + e = 5, +}; + +/// Fields stored in sparse MMA Atom types for SM100. +pub const AtomMmaFieldSM100Sparse = enum(u32) { + accum_c = 0, + neg_a = 1, + neg_b = 2, + e = 5, +}; + +/// Fields stored in MMA Atom types for SM120 (block scaled). +pub const AtomMmaFieldSM120BlockScaled = enum(u32) { + sf_a = 0, + sf_b = 1, +}; + +/// Fields stored in MMA Atom types for SM80. +pub const AtomMmaFieldSM80Sparse = enum(u32) { + e = 0, +}; + +/// Fields stored in MMA Atom types for SM90. +pub const AtomMmaFieldSM90 = enum(u32) { + accum_c = 0, +}; + +/// Binary operation for single-bit MMA operations. +pub const BinaryOp = enum(u32) { + none = 0, + xor_popc = 1, + and_popc = 2, +}; + +/// Cute Cache Eviction Priority kind. +pub const CacheEvictionPriority = enum(u32) { + EVICT_NORMAL = 0, + EVICT_FIRST = 1, + EVICT_LAST = 2, + EVICT_UNCHANGED = 3, + NO_ALLOCATE = 4, +}; + +/// Broadcast modes for the different utccp instructions. +pub const CopyS2TBroadcast = enum(u32) { + none = 0, + lw_0213 = 1, + lw_0123 = 2, + x4 = 3, +}; + +/// The various kinds of TMA loads in gather/scatter mode. +pub const GatherScatterTmaLoad = enum(u32) { + sm_100 = 0, + sm_100_multicast = 1, + sm_100_2sm = 2, + sm_100_2sm_multicast = 3, +}; + +/// The various kinds of TMA loads in im2col mode. +pub const Im2ColTmaLoad = enum(u32) { + sm_90 = 0, + sm_90_multicast = 1, + sm_100_2sm = 2, + sm_100_2sm_multicast = 3, +}; + +/// CuTe L2 prefetch size options. +pub const L2PrefetchSize = enum(u32) { + NONE = 0, + RESERVED = 1, + SIZE_64B = 2, + SIZE_128B = 3, + SIZE_256B = 4, +}; + +/// multimem ld_reduce accumulation precision kind. +pub const LdReduceAccPrecisionKind = enum(u32) { + F16 = 0, + F32 = 1, +}; + +/// LDSM's sz pattern, describing the bit size. +pub const LdsmSzPattern = enum(u32) { + u16 = 0, + u4to8 = 1, + s4to8 = 2, + u2to4 = 3, + s2to4 = 4, + u4x16p64to8 = 5, + u6x16p32to8 = 6, + u8 = 7, +}; + +/// Cache modes for the load instructions. +pub const LoadCacheMode = enum(u32) { + always = 0, + global = 1, + streaming = 2, + last_use = 3, + none = 4, +}; + +/// MMA overflow options. +pub const MMAIntOverflow = enum(u32) { + satfinite = 1, + wrapped = 0, +}; + +/// Memory Ordering kind. +pub const MemOrderKind = enum(u32) { + WEAK = 0, + RELAXED = 1, + ACQUIRE = 2, + RELEASE = 3, + ACQ_REL = 4, + SC = 5, + MMIO = 6, + CONSTANT = 7, + VOLATILE = 8, +}; + +/// Cute Memory Scope kind. +pub const MemScopeKind = enum(u32) { + CTA = 0, + CLUSTER = 1, + GPU = 2, + SYS = 3, +}; + +/// Enums for the mma collector op. +pub const MmaCollectorOp = enum(u32) { + discard = 0, + lastuse = 1, + fill = 2, + use = 3, +}; + +/// Enums for the mma frag type. +pub const MmaFragKind = enum(u32) { + smem_desc = 0, + sparse_smem_desc = 1, + tmem = 2, + tmem_ws = 3, + tmem_e = 4, + tmem_sf = 5, + rmem = 6, +}; + +/// Op for the TMASTORE instruction. +pub const ReductionKind = enum(u32) { + ADD = 0, + MIN = 1, + MAX = 2, + INC = 3, + DEC = 4, + AND = 5, + OR = 6, + XOR = 7, +}; + +/// CuTe Shared Space kind. +pub const SharedSpace = enum(u32) { + CTA = 0, + CLUSTER = 1, +}; + +/// Metadata format used in sparse MMA. +pub const SparseMetadataFormat = enum(u32) { + tid = 0, +}; + +/// Cache modes for the store instructions. +pub const StoreCacheMode = enum(u32) { + write_back = 0, + global = 1, + streaming = 2, + write_through = 3, + none = 4, +}; + +/// The various kinds of TMA loads in tiled mode. +pub const TiledTmaLoad = enum(u32) { + sm_90 = 0, + sm_90_multicast = 1, + sm_100_2sm = 2, + sm_100_2sm_multicast = 3, +}; + +/// Bits 74-71 of the TMA descriptor. +pub const TmaDataFormat = enum(u32) { + U8 = 0, + U16 = 1, + U32 = 2, + S32 = 3, + U64 = 4, + S64 = 5, + F16_RN = 6, + F32_RN = 7, + F32_FTZ_RN = 8, + F64_RN = 9, + BF16_RN = 10, + U4 = 11, + U4_UNPACK_U8 = 12, + U6_UNPACK_U8 = 13, + TF32_RN = 14, + TF32_FTZ_RN = 15, +}; + +/// Modes for the TMA load instruction. +pub const TmaLoadMode = enum(u32) { + tiled = 0, + im2col = 1, + w128 = 2, + w = 3, + gather4 = 4, +}; + +/// Modes for the TMA store instruction. +pub const TmaStoreMode = enum(u32) { + tiled = 0, + im2col = 1, + scatter4 = 2, +}; + +/// Modes for the Tmem allocation. +pub const TmemAllocMode = enum(u32) { + Interleaved = 0, + NonInterleaved = 1, + Duplicated = 2, + SfDuplicated_4x1 = 3, + SfDuplicated_2x2 = 4, +}; + +/// Reduce operation for the Tmem load instruction. +pub const TmemLoadRedOp = enum(u32) { + max = 0, + maxabs = 1, + min = 2, + minabs = 3, +}; + +// Types and attributes, generated from the dialect's .td files. + +/// `!cute_nvgpu.atom.bulk_copy_g2s`: Bulk (TMA non-tensor) copy atom. pub const CopyAtomBulkCopyG2SType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomBulkCopyG2S; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.bulk_copy_g2s"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + copyBits: c_int = 0, + mcast: bool = false, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomBulkCopyG2STypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomBulkCopyG2STypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.mcast); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetValType(self.ptr()).ptr.?); + } + pub fn getCopyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetCopyBits(self.ptr()); + } + pub fn getMcast(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetMcast(self.ptr()); } }; -/// `!cute_nvgpu.atom.bulk_copy_s2g`. +/// `!cute_nvgpu.atom.bulk_copy_s2g`: Bulk (TMA non-tensor) copy atom. pub const CopyAtomBulkCopyS2GType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2G; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.bulk_copy_s2g"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + copyBits: c_int = 0, + mask: bool = false, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.mask); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetValType(self.ptr()).ptr.?); + } + pub fn getCopyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetCopyBits(self.ptr()); + } + pub fn getMask(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetMask(self.ptr()); } }; -/// `!cute_nvgpu.atom.bulk_copy_s2s`. +/// `!cute_nvgpu.atom.bulk_copy_s2s`: Bulk (TMA non-tensor) copy atom. pub const CopyAtomBulkCopyS2SType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2S; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.bulk_copy_s2s"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + copyBits: c_int = 0, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomBulkCopyS2STypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomBulkCopyS2STypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetValType(self.ptr()).ptr.?); + } + pub fn getCopyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetCopyBits(self.ptr()); } }; -/// `!cute_nvgpu.atom.dsmem_store`. +/// `!cute_nvgpu.atom.dsmem_store`: Distributed shared-memory store atom. pub const CopyAtomDsmemStoreType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomDsmemStore; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.dsmem_store"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + copyBits: c_int, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomDsmemStoreTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomDsmemStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomDsmemStoreTypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomDsmemStoreTypeGetValType(self.ptr()).ptr.?); + } + pub fn getCopyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomDsmemStoreTypeGetCopyBits(self.ptr()); } }; -/// `!cute_nvgpu.atom.g2r`. +/// `!cute_nvgpu.atom.g2r`: Global-to-register load atom. pub const CopyAtomG2RType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomG2R; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.g2r"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + copyBits: c_int, + memOrder: MemOrderKind = .WEAK, + memScope: MemScopeKind = .CTA, + l2PrefetchSize: L2PrefetchSize = .NONE, + l1CacheEvictPriority: CacheEvictionPriority = .EVICT_NORMAL, + loadCacheMode: LoadCacheMode = .always, + sharedSpace: SharedSpace = .CTA, + invariant: bool = false, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomG2RTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomG2RTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.memOrder), @intFromEnum(args.memScope), @intFromEnum(args.l2PrefetchSize), @intFromEnum(args.l1CacheEvictPriority), @intFromEnum(args.loadCacheMode), @intFromEnum(args.sharedSpace), args.invariant); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetValType(self.ptr()).ptr.?); + } + pub fn getCopyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomG2RTypeGetCopyBits(self.ptr()); + } + pub fn getMemOrder(self: *const Self) MemOrderKind { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomG2RTypeGetMemOrder(self.ptr())); + } + pub fn getMemScope(self: *const Self) MemScopeKind { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomG2RTypeGetMemScope(self.ptr())); + } + pub fn getL2PrefetchSize(self: *const Self) L2PrefetchSize { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomG2RTypeGetL2PrefetchSize(self.ptr())); + } + pub fn getL1CacheEvictPriority(self: *const Self) CacheEvictionPriority { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomG2RTypeGetL1CacheEvictPriority(self.ptr())); + } + pub fn getLoadCacheMode(self: *const Self) LoadCacheMode { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomG2RTypeGetLoadCacheMode(self.ptr())); + } + pub fn getSharedSpace(self: *const Self) SharedSpace { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomG2RTypeGetSharedSpace(self.ptr())); + } + pub fn getInvariant(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomG2RTypeGetInvariant(self.ptr()); } }; -/// `!cute_nvgpu.atom.im2col_tma_load`. +/// `!cute_nvgpu.atom.im2col_tma_load`: Executable im2col TMA load atom. pub const CopyAtomIm2ColTmaLoadType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaLoad; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.im2col_tma_load"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + copyBits: c_int, + numCta: c_int, + gStride: *const mlir.Type, + mcast: bool = false, + tmaGbasis: ?*const mlir.Type = null, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.numCta, args.gStride.ptr(), args.mcast, if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetValType(self.ptr()).ptr.?); + } + pub fn getCopyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetCopyBits(self.ptr()); + } + pub fn getNumCta(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetNumCta(self.ptr()); + } + pub fn getGStride(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetGStride(self.ptr()).ptr.?); + } + pub fn getMcast(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetMcast(self.ptr()); + } + pub fn getTmaGbasis(self: *const Self) ?*const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetTmaGbasis(self.ptr()).ptr); } }; -/// `!cute_nvgpu.atom.im2col_tma_store`. +/// `!cute_nvgpu.atom.im2col_tma_store`: Executable im2col TMA store atom. pub const CopyAtomIm2ColTmaStoreType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaStore; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.im2col_tma_store"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + copyBits: c_int, + gStride: *const mlir.Type, + tmaGbasis: ?*const mlir.Type = null, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetValType(self.ptr()).ptr.?); + } + pub fn getCopyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetCopyBits(self.ptr()); + } + pub fn getGStride(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetGStride(self.ptr()).ptr.?); + } + pub fn getTmaGbasis(self: *const Self) ?*const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetTmaGbasis(self.ptr()).ptr); } }; -/// `!cute_nvgpu.atom.ldsm`. +/// `!cute_nvgpu.atom.ldsm`: ldmatrix atom. pub const CopyAtomLdsmType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomLdsm; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.ldsm"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + mode: *const mlir.Attribute, + szPattern: LdsmSzPattern, + numMatrices: c_int, + transpose: bool = false, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomLdsmTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomLdsmTypeGet(ctx.ptr(), args.valType.ptr(), args.mode.ptr(), @intFromEnum(args.szPattern), args.numMatrices, args.transpose); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomLdsmTypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomLdsmTypeGetValType(self.ptr()).ptr.?); + } + pub fn getMode(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomLdsmTypeGetMode(self.ptr()).ptr.?); + } + pub fn getSzPattern(self: *const Self) LdsmSzPattern { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomLdsmTypeGetSzPattern(self.ptr())); + } + pub fn getNumMatrices(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomLdsmTypeGetNumMatrices(self.ptr()); + } + pub fn getTranspose(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomLdsmTypeGetTranspose(self.ptr()); } }; -/// `!cute_nvgpu.atom.non_exec_2d_gather4_tma_load`. +/// `!cute_nvgpu.atom.non_exec_2d_gather4_tma_load`: Non-executable 2-D gather4 TMA load atom. pub const CopyAtomNonExec2DGather4TmaLoadType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExec2DGather4TmaLoad; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.non_exec_2d_gather4_tma_load"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + kind: GatherScatterTmaLoad, + valType: *const mlir.Type, + copyBits: c_int, + tmaGbasis: *const mlir.Type, + tmaFormat: ?TmaDataFormat = null, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGet(ctx.ptr(), @intFromEnum(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetPayload(self.ptr()).ptr.?); + pub fn getKind(self: *const Self) GatherScatterTmaLoad { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetKind(self.ptr())); + } + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetValType(self.ptr()).ptr.?); + } + pub fn getCopyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetCopyBits(self.ptr()); + } + pub fn getTmaGbasis(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaGbasis(self.ptr()).ptr.?); + } + pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { + const value = c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaFormat(self.ptr()); + return if (value < 0) null else @enumFromInt(value); } }; -/// `!cute_nvgpu.atom.non_exec_im2col_tma_load`. +/// `!cute_nvgpu.atom.non_exec_2d_scatter4_tma_store`: Non-executable 2-D scatter4 TMA store atom. +pub const CopyAtomNonExec2DScatter4TmaStoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExec2DScatter4TmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.non_exec_2d_scatter4_tma_store"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int, + tmaGbasis: *const mlir.Type, + tmaFormat: ?TmaDataFormat = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetValType(self.ptr()).ptr.?); + } + pub fn getCopyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetCopyBits(self.ptr()); + } + pub fn getTmaGbasis(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaGbasis(self.ptr()).ptr.?); + } + pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { + const value = c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaFormat(self.ptr()); + return if (value < 0) null else @enumFromInt(value); + } +}; + +/// `!cute_nvgpu.atom.non_exec_im2col_tma_load`: Non-executable im2col TMA load atom. pub const CopyAtomNonExecIm2ColTmaLoadType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaLoad; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.non_exec_im2col_tma_load"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + kind: Im2ColTmaLoad, + valType: *const mlir.Type, + copyBits: c_int, + tmaGbasis: *const mlir.Type, + tmaFormat: ?TmaDataFormat = null, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGet(ctx.ptr(), @intFromEnum(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetPayload(self.ptr()).ptr.?); + pub fn getKind(self: *const Self) Im2ColTmaLoad { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetKind(self.ptr())); + } + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetValType(self.ptr()).ptr.?); + } + pub fn getCopyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetCopyBits(self.ptr()); + } + pub fn getTmaGbasis(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaGbasis(self.ptr()).ptr.?); + } + pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { + const value = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaFormat(self.ptr()); + return if (value < 0) null else @enumFromInt(value); } }; -/// `!cute_nvgpu.atom.non_exec_im2col_tma_store`. +/// `!cute_nvgpu.atom.non_exec_im2col_tma_store`: Non-executable im2col TMA store atom. pub const CopyAtomNonExecIm2ColTmaStoreType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaStore; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.non_exec_im2col_tma_store"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + copyBits: c_int, + tmaGbasis: *const mlir.Type, + tmaFormat: ?TmaDataFormat = null, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetValType(self.ptr()).ptr.?); + } + pub fn getCopyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetCopyBits(self.ptr()); + } + pub fn getTmaGbasis(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaGbasis(self.ptr()).ptr.?); + } + pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { + const value = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaFormat(self.ptr()); + return if (value < 0) null else @enumFromInt(value); } }; -/// `!cute_nvgpu.atom.non_exec_tiled_tma_load`. +/// `!cute_nvgpu.atom.non_exec_tiled_tma_load`: Non-executable tiled TMA load atom. pub const CopyAtomNonExecTiledTmaLoadType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaLoad; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.non_exec_tiled_tma_load"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + kind: TiledTmaLoad, + valType: *const mlir.Type, + copyBits: c_int, + tmaGbasis: *const mlir.Type, + tmaFormat: ?TmaDataFormat = null, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGet(ctx.ptr(), @intFromEnum(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetPayload(self.ptr()).ptr.?); + pub fn getKind(self: *const Self) TiledTmaLoad { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetKind(self.ptr())); + } + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetValType(self.ptr()).ptr.?); + } + pub fn getCopyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetCopyBits(self.ptr()); + } + pub fn getTmaGbasis(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaGbasis(self.ptr()).ptr.?); + } + pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { + const value = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaFormat(self.ptr()); + return if (value < 0) null else @enumFromInt(value); } }; -/// `!cute_nvgpu.atom.non_exec_tiled_tma_reduce`. +/// `!cute_nvgpu.atom.non_exec_tiled_tma_reduce`: Non-executable tiled TMA reduce atom. pub const CopyAtomNonExecTiledTmaReduceType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaReduce; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.non_exec_tiled_tma_reduce"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + kind: ReductionKind, + valType: *const mlir.Type, + copyBits: c_int, + tmaGbasis: *const mlir.Type, + tmaFormat: ?TmaDataFormat = null, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGet(ctx.ptr(), @intFromEnum(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetPayload(self.ptr()).ptr.?); + pub fn getKind(self: *const Self) ReductionKind { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetKind(self.ptr())); + } + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetValType(self.ptr()).ptr.?); + } + pub fn getCopyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetCopyBits(self.ptr()); + } + pub fn getTmaGbasis(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaGbasis(self.ptr()).ptr.?); + } + pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { + const value = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaFormat(self.ptr()); + return if (value < 0) null else @enumFromInt(value); } }; -/// `!cute_nvgpu.atom.non_exec_tiled_tma_store`. +/// `!cute_nvgpu.atom.non_exec_tiled_tma_store`: Non-executable tiled TMA store atom. pub const CopyAtomNonExecTiledTmaStoreType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaStore; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.non_exec_tiled_tma_store"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + copyBits: c_int, + tmaGbasis: *const mlir.Type, + tmaFormat: ?TmaDataFormat = null, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetValType(self.ptr()).ptr.?); + } + pub fn getCopyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetCopyBits(self.ptr()); + } + pub fn getTmaGbasis(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaGbasis(self.ptr()).ptr.?); + } + pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { + const value = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaFormat(self.ptr()); + return if (value < 0) null else @enumFromInt(value); } }; -/// `!cute_nvgpu.atom.r2g`. +/// `!cute_nvgpu.atom.r2g`: Register-to-global store atom. pub const CopyAtomR2GType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomR2G; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.r2g"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + copyBits: c_int, + memOrder: MemOrderKind = .WEAK, + memScope: MemScopeKind = .CTA, + l1CacheEvictPriority: CacheEvictionPriority = .EVICT_NORMAL, + storeCacheMode: StoreCacheMode = .write_back, + sharedSpace: SharedSpace = .CTA, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomR2GTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomR2GTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.memOrder), @intFromEnum(args.memScope), @intFromEnum(args.l1CacheEvictPriority), @intFromEnum(args.storeCacheMode), @intFromEnum(args.sharedSpace)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetValType(self.ptr()).ptr.?); + } + pub fn getCopyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomR2GTypeGetCopyBits(self.ptr()); + } + pub fn getMemOrder(self: *const Self) MemOrderKind { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2GTypeGetMemOrder(self.ptr())); + } + pub fn getMemScope(self: *const Self) MemScopeKind { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2GTypeGetMemScope(self.ptr())); + } + pub fn getL1CacheEvictPriority(self: *const Self) CacheEvictionPriority { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2GTypeGetL1CacheEvictPriority(self.ptr())); + } + pub fn getStoreCacheMode(self: *const Self) StoreCacheMode { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2GTypeGetStoreCacheMode(self.ptr())); + } + pub fn getSharedSpace(self: *const Self) SharedSpace { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2GTypeGetSharedSpace(self.ptr())); } }; -/// `!cute_nvgpu.atom.r2s`. +/// `!cute_nvgpu.atom.r2s`: Register-to-shared store atom. pub const CopyAtomR2SType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomR2S; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.r2s"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + copyBits: c_int, + memOrder: MemOrderKind = .WEAK, + memScope: MemScopeKind = .CTA, + sharedSpace: SharedSpace = .CTA, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomR2STypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomR2STypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.memOrder), @intFromEnum(args.memScope), @intFromEnum(args.sharedSpace)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomR2STypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomR2STypeGetValType(self.ptr()).ptr.?); + } + pub fn getCopyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomR2STypeGetCopyBits(self.ptr()); + } + pub fn getMemOrder(self: *const Self) MemOrderKind { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2STypeGetMemOrder(self.ptr())); + } + pub fn getMemScope(self: *const Self) MemScopeKind { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2STypeGetMemScope(self.ptr())); + } + pub fn getSharedSpace(self: *const Self) SharedSpace { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2STypeGetSharedSpace(self.ptr())); } }; -/// `!cute_nvgpu.atom.s2r`. +/// `!cute_nvgpu.atom.s2r`: Shared-to-register load atom. pub const CopyAtomS2RType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomS2R; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.s2r"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + copyBits: c_int, + memOrder: MemOrderKind = .WEAK, + memScope: MemScopeKind = .CTA, + sharedSpace: SharedSpace = .CTA, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomS2RTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomS2RTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.memOrder), @intFromEnum(args.memScope), @intFromEnum(args.sharedSpace)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetValType(self.ptr()).ptr.?); + } + pub fn getCopyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomS2RTypeGetCopyBits(self.ptr()); + } + pub fn getMemOrder(self: *const Self) MemOrderKind { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomS2RTypeGetMemOrder(self.ptr())); + } + pub fn getMemScope(self: *const Self) MemScopeKind { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomS2RTypeGetMemScope(self.ptr())); + } + pub fn getSharedSpace(self: *const Self) SharedSpace { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomS2RTypeGetSharedSpace(self.ptr())); } }; -/// `!cute_nvgpu.atom.simt_async_copy`. +/// `!cute_nvgpu.atom.simt_async_copy`: cp.async copy atom. pub const CopyAtomSIMTAsyncCopyType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTAsyncCopy; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.simt_async_copy"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + cache: LoadCacheMode, + copyBits: c_int, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGet(ctx.ptr(), args.valType.ptr(), @intFromEnum(args.cache), args.copyBits); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetValType(self.ptr()).ptr.?); + } + pub fn getCache(self: *const Self) LoadCacheMode { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCache(self.ptr())); + } + pub fn getCopyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCopyBits(self.ptr()); } }; -/// `!cute_nvgpu.atom.simt_multimem_ld_reduce`. +/// `!cute_nvgpu.atom.simt_multimem_ld_reduce`: multimem.ld_reduce atom. pub const CopyAtomSIMTMultimemLdReduceType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemLdReduce; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.simt_multimem_ld_reduce"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + copyBits: c_int = 0, + reduction: ReductionKind = .ADD, + ldReduceAccPrecision: ?*const mlir.Attribute = null, + memOrder: MemOrderKind = .RELAXED, + memScope: MemScopeKind = .SYS, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.reduction), if (args.ldReduceAccPrecision) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, @intFromEnum(args.memOrder), @intFromEnum(args.memScope)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetValType(self.ptr()).ptr.?); + } + pub fn getCopyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetCopyBits(self.ptr()); + } + pub fn getReduction(self: *const Self) ReductionKind { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetReduction(self.ptr())); + } + pub fn getLdReduceAccPrecision(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetLdReduceAccPrecision(self.ptr()).ptr); + } + pub fn getMemOrder(self: *const Self) MemOrderKind { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemOrder(self.ptr())); + } + pub fn getMemScope(self: *const Self) MemScopeKind { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemScope(self.ptr())); } }; -/// `!cute_nvgpu.atom.simt_multimem_red`. +/// `!cute_nvgpu.atom.simt_multimem_red`: multimem.red atom. pub const CopyAtomSIMTMultimemRedType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemRed; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.simt_multimem_red"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + copyBits: c_int, + reduction: ReductionKind = .ADD, + memOrder: MemOrderKind = .RELAXED, + memScope: MemScopeKind = .SYS, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.reduction), @intFromEnum(args.memOrder), @intFromEnum(args.memScope)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetValType(self.ptr()).ptr.?); + } + pub fn getCopyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetCopyBits(self.ptr()); + } + pub fn getReduction(self: *const Self) ReductionKind { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetReduction(self.ptr())); + } + pub fn getMemOrder(self: *const Self) MemOrderKind { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemOrder(self.ptr())); + } + pub fn getMemScope(self: *const Self) MemScopeKind { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemScope(self.ptr())); } }; -/// `!cute_nvgpu.atom.simt_multimem_st`. +/// `!cute_nvgpu.atom.simt_multimem_st`: multimem.st atom. pub const CopyAtomSIMTMultimemStType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemSt; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.simt_multimem_st"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + copyBits: c_int = 0, + memOrder: MemOrderKind = .RELAXED, + memScope: MemScopeKind = .SYS, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.memOrder), @intFromEnum(args.memScope)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetValType(self.ptr()).ptr.?); + } + pub fn getCopyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetCopyBits(self.ptr()); + } + pub fn getMemOrder(self: *const Self) MemOrderKind { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemOrder(self.ptr())); + } + pub fn getMemScope(self: *const Self) MemScopeKind { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemScope(self.ptr())); } }; -/// `!cute_nvgpu.atom.universal_copy`. +/// `!cute_nvgpu.atom.universal_copy`: Universal (SIMT) copy atom. pub const CopyAtomSIMTSyncCopyType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTSyncCopy; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.universal_copy"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + copyBits: c_int = 0, + srcSpace: cute.AddressSpace = .generic, + dstSpace: cute.AddressSpace = .generic, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.srcSpace), @intFromEnum(args.dstSpace)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetValType(self.ptr()).ptr.?); + } + pub fn getCopyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetCopyBits(self.ptr()); + } + pub fn getSrcSpace(self: *const Self) cute.AddressSpace { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetSrcSpace(self.ptr())); + } + pub fn getDstSpace(self: *const Self) cute.AddressSpace { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetDstSpace(self.ptr())); } }; -/// `!cute_nvgpu.atom.tmem_load_red`. -pub const CopyAtomSM10xTmemLoadRedType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM10xTmemLoadRed; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - payload: *const mlir.Attribute, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet(ctx.ptr(), args.payload.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetPayload(self.ptr()).ptr.?); - } -}; - -/// `!cute_nvgpu.atom.s2t_copy`. +/// `!cute_nvgpu.atom.s2t_copy`: tcgen05.cp (shared to tensor memory) atom. pub const CopyAtomSM100CopyS2TType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM100CopyS2T; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.s2t_copy"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + numDp: c_int, + numBit: c_int, + numCta: c_int, + broadcast: CopyS2TBroadcast = .none, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numCta, @intFromEnum(args.broadcast)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetValType(self.ptr()).ptr.?); + } + pub fn getNumDp(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumDp(self.ptr()); + } + pub fn getNumBit(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumBit(self.ptr()); + } + pub fn getNumCta(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumCta(self.ptr()); + } + pub fn getBroadcast(self: *const Self) CopyS2TBroadcast { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetBroadcast(self.ptr())); } }; -/// `!cute_nvgpu.atom.sm100_s2t_copy_v2`. +/// `!cute_nvgpu.atom.sm100_s2t_copy_v2`: tcgen05.cp atom with the shared-memory major mode. pub const CopyAtomSM100S2TCopyV2Type = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM100S2TCopyV2; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.sm100_s2t_copy_v2"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + numDp: c_int, + numBit: c_int, + numCta: c_int, + smemMajor: cute.MajorMode, + broadcast: CopyS2TBroadcast = .none, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numCta, @intFromEnum(args.smemMajor), @intFromEnum(args.broadcast)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetValType(self.ptr()).ptr.?); + } + pub fn getNumDp(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumDp(self.ptr()); + } + pub fn getNumBit(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumBit(self.ptr()); + } + pub fn getNumCta(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumCta(self.ptr()); + } + pub fn getSmemMajor(self: *const Self) cute.MajorMode { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetSmemMajor(self.ptr())); + } + pub fn getBroadcast(self: *const Self) CopyS2TBroadcast { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetBroadcast(self.ptr())); } }; -/// `!cute_nvgpu.atom.tmem_load`. +/// `!cute_nvgpu.atom.tmem_load`: tcgen05.ld atom. pub const CopyAtomSM100TmemLoadType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM100TmemLoad; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.tmem_load"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + numDp: c_int, + numBit: c_int, + numRep: c_uint, + pack16b: bool = false, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, args.pack16b); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetValType(self.ptr()).ptr.?); + } + pub fn getNumDp(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumDp(self.ptr()); + } + pub fn getNumBit(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumBit(self.ptr()); + } + pub fn getNumRep(self: *const Self) c_uint { + return c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumRep(self.ptr()); + } + pub fn getPack16b(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetPack16b(self.ptr()); } }; -/// `!cute_nvgpu.atom.tmem_store`. +/// `!cute_nvgpu.atom.tmem_store`: tcgen05.st atom. pub const CopyAtomSM100TmemStoreType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM100TmemStore; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.tmem_store"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + numDp: c_int, + numBit: c_int, + numRep: c_uint, + expand16b: bool = false, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, args.expand16b); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetValType(self.ptr()).ptr.?); + } + pub fn getNumDp(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumDp(self.ptr()); + } + pub fn getNumBit(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumBit(self.ptr()); + } + pub fn getNumRep(self: *const Self) c_uint { + return c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumRep(self.ptr()); + } + pub fn getExpand16b(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetExpand16b(self.ptr()); } }; -/// `!cute_nvgpu.atom.stsm`. +/// `!cute_nvgpu.atom.tmem_load_spcompress`: SM107 tensor-memory load with sparse compression. +pub const CopyAtomSM107TmemLoadSPCompressType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM107TmemLoadSPCompress; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.tmem_load_spcompress"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + numDp: c_int, + numBit: c_int, + numRep: c_uint, + redOp: TmemLoadRedOp, + red: bool = false, + nan: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, @intFromEnum(args.redOp), args.red, args.nan); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetValType(self.ptr()).ptr.?); + } + pub fn getNumDp(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumDp(self.ptr()); + } + pub fn getNumBit(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumBit(self.ptr()); + } + pub fn getNumRep(self: *const Self) c_uint { + return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumRep(self.ptr()); + } + pub fn getRedOp(self: *const Self) TmemLoadRedOp { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRedOp(self.ptr())); + } + pub fn getRed(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRed(self.ptr()); + } + pub fn getNan(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNan(self.ptr()); + } +}; + +/// `!cute_nvgpu.atom.tmem_load_red`: tcgen05.ld.red atom. +pub const CopyAtomSM10xTmemLoadRedType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM10xTmemLoadRed; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.tmem_load_red"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + numDp: c_int, + numBit: c_int, + numRep: c_uint, + redOp: TmemLoadRedOp, + nan: bool = false, + halfSplitOff: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, @intFromEnum(args.redOp), args.nan, if (args.halfSplitOff) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetValType(self.ptr()).ptr.?); + } + pub fn getNumDp(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumDp(self.ptr()); + } + pub fn getNumBit(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumBit(self.ptr()); + } + pub fn getNumRep(self: *const Self) c_uint { + return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumRep(self.ptr()); + } + pub fn getRedOp(self: *const Self) TmemLoadRedOp { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetRedOp(self.ptr())); + } + pub fn getNan(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNan(self.ptr()); + } + pub fn getHalfSplitOff(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetHalfSplitOff(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.atom.stsm`: stmatrix atom. pub const CopyAtomStsmType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomStsm; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.stsm"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + mode: *const mlir.Attribute, + numMatrices: c_int, + transpose: bool = false, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomStsmTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomStsmTypeGet(ctx.ptr(), args.valType.ptr(), args.mode.ptr(), args.numMatrices, args.transpose); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomStsmTypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomStsmTypeGetValType(self.ptr()).ptr.?); + } + pub fn getMode(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomStsmTypeGetMode(self.ptr()).ptr.?); + } + pub fn getNumMatrices(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomStsmTypeGetNumMatrices(self.ptr()); + } + pub fn getTranspose(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomStsmTypeGetTranspose(self.ptr()); } }; -/// `!cute_nvgpu.atom.tma_load`. +/// `!cute_nvgpu.atom.tma_load`: Executable TMA load atom. pub const CopyAtomTmaLoadType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomTmaLoad; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.tma_load"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + sparsity: c_int = 1, + copyBits: c_int, + mode: TmaLoadMode, + numCta: c_int, + gStride: *const mlir.Type, + mcast: bool = false, + tmaGbasis: ?*const mlir.Type = null, + override: bool = false, + noFullyOobTile: bool = false, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomTmaLoadTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomTmaLoadTypeGet(ctx.ptr(), args.valType.ptr(), args.sparsity, args.copyBits, @intFromEnum(args.mode), args.numCta, args.gStride.ptr(), args.mcast, if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }, args.override, args.noFullyOobTile); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetValType(self.ptr()).ptr.?); + } + pub fn getSparsity(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetSparsity(self.ptr()); + } + pub fn getCopyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetCopyBits(self.ptr()); + } + pub fn getMode(self: *const Self) TmaLoadMode { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetMode(self.ptr())); + } + pub fn getNumCta(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetNumCta(self.ptr()); + } + pub fn getGStride(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetGStride(self.ptr()).ptr.?); + } + pub fn getMcast(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetMcast(self.ptr()); + } + pub fn getTmaGbasis(self: *const Self) ?*const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetTmaGbasis(self.ptr()).ptr); + } + pub fn getOverride(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetOverride(self.ptr()); + } + pub fn getNoFullyOobTile(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetNoFullyOobTile(self.ptr()); } }; -/// `!cute_nvgpu.atom.tma_reduce`. +/// `!cute_nvgpu.atom.tma_reduce`: Executable TMA reduce atom. pub const CopyAtomTmaReduceType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomTmaReduce; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.tma_reduce"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + copyBits: c_int, + mode: TmaStoreMode, + kind: ReductionKind, + gStride: *const mlir.Type, + tmaGbasis: ?*const mlir.Type = null, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomTmaReduceTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomTmaReduceTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.mode), @intFromEnum(args.kind), args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetValType(self.ptr()).ptr.?); + } + pub fn getCopyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetCopyBits(self.ptr()); + } + pub fn getMode(self: *const Self) TmaStoreMode { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetMode(self.ptr())); + } + pub fn getKind(self: *const Self) ReductionKind { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetKind(self.ptr())); + } + pub fn getGStride(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetGStride(self.ptr()).ptr.?); + } + pub fn getTmaGbasis(self: *const Self) ?*const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetTmaGbasis(self.ptr()).ptr); } }; -/// `!cute_nvgpu.atom.tma_store`. +/// `!cute_nvgpu.atom.tma_store`: Executable TMA store atom. pub const CopyAtomTmaStoreType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomTmaStore; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.tma_store"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + valType: *const mlir.Type, + sparsity: c_int = 1, + copyBits: c_int, + mode: TmaStoreMode, + gStride: *const mlir.Type, + tmaGbasis: ?*const mlir.Type = null, + override: bool = false, + noFullyOobTile: bool = false, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomTmaStoreTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUCopyAtomTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.sparsity, args.copyBits, @intFromEnum(args.mode), args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }, args.override, args.noFullyOobTile); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetPayload(self.ptr()).ptr.?); + pub fn getValType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetValType(self.ptr()).ptr.?); + } + pub fn getSparsity(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetSparsity(self.ptr()); + } + pub fn getCopyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetCopyBits(self.ptr()); + } + pub fn getMode(self: *const Self) TmaStoreMode { + return @enumFromInt(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetMode(self.ptr())); + } + pub fn getGStride(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetGStride(self.ptr()).ptr.?); + } + pub fn getTmaGbasis(self: *const Self) ?*const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetTmaGbasis(self.ptr()).ptr); + } + pub fn getOverride(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetOverride(self.ptr()); + } + pub fn getNoFullyOobTile(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetNoFullyOobTile(self.ptr()); } }; -/// `!cute_nvgpu.sm80.sparse_mma`. -pub const MmaAtomSM80SparseType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM80Sparse; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - payload: *const mlir.Attribute, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM80SparseTypeGet(ctx.ptr(), args.payload.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetPayload(self.ptr()).ptr.?); - } -}; - -/// `!cute_nvgpu.sm80.mma`. -pub const MmaAtomSM80Type = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM80; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - payload: *const mlir.Attribute, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM80TypeGet(ctx.ptr(), args.payload.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetPayload(self.ptr()).ptr.?); - } -}; - -/// `!cute_nvgpu.sm89.mma`. -pub const MmaAtomSM89Type = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM89; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - payload: *const mlir.Attribute, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM89TypeGet(ctx.ptr(), args.payload.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM89TypeGetPayload(self.ptr()).ptr.?); - } -}; - -/// `!cute_nvgpu.sm90.mma`. -pub const MmaAtomSM90Type = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM90; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - payload: *const mlir.Attribute, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM90TypeGet(ctx.ptr(), args.payload.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetPayload(self.ptr()).ptr.?); - } -}; - -/// `!cute_nvgpu.sm100.mma_bs_sp`. +/// `!cute_nvgpu.sm100.mma_bs_sp`: SM100 block-scaled sparse UMMA MMA atom. pub const MmaAtomSM100UMMABlockScaledSparseType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaledSparse; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "sm100.mma_bs_sp"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + shapeMnk: *const mlir.Attribute, + numCta: c_int, + aMajor: cute.MajorMode, + bMajor: cute.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + sfType: *const mlir.Type, + sparseMetadataFormat: SparseMetadataFormat, + aFragKind: MmaFragKind, + vecSize: c_int, + archPromote: Arch = .sm_100, + intOverflow: ?*const mlir.Attribute = null, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @intFromEnum(args.sparseMetadataFormat), @intFromEnum(args.aFragKind), args.vecSize, @intFromEnum(args.archPromote), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetPayload(self.ptr()).ptr.?); + pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn getNumCta(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetNumCta(self.ptr()); + } + pub fn getAMajor(self: *const Self) cute.MajorMode { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAMajor(self.ptr())); + } + pub fn getBMajor(self: *const Self) cute.MajorMode { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBMajor(self.ptr())); + } + pub fn getAType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAType(self.ptr()).ptr.?); + } + pub fn getBType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBType(self.ptr()).ptr.?); + } + pub fn getCType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetCType(self.ptr()).ptr.?); + } + pub fn getSfType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSfType(self.ptr()).ptr.?); + } + pub fn getSparseMetadataFormat(self: *const Self) SparseMetadataFormat { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSparseMetadataFormat(self.ptr())); + } + pub fn getAFragKind(self: *const Self) MmaFragKind { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAFragKind(self.ptr())); + } + pub fn getVecSize(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetVecSize(self.ptr()); + } + pub fn getArchPromote(self: *const Self) Arch { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetArchPromote(self.ptr())); + } + pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetIntOverflow(self.ptr()).ptr); } }; -/// `!cute_nvgpu.sm100.mma_bs`. +/// `!cute_nvgpu.sm100.mma_bs`: SM100 block-scaled UMMA MMA atom. pub const MmaAtomSM100UMMABlockScaledType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaled; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "sm100.mma_bs"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + shapeMnk: *const mlir.Attribute, + numCta: c_int, + aMajor: cute.MajorMode, + bMajor: cute.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + sfType: *const mlir.Type, + aFragKind: MmaFragKind, + vecSize: c_int, + archPromote: Arch = .sm_100, + intOverflow: ?*const mlir.Attribute = null, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @intFromEnum(args.aFragKind), args.vecSize, @intFromEnum(args.archPromote), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetPayload(self.ptr()).ptr.?); + pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn getNumCta(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetNumCta(self.ptr()); + } + pub fn getAMajor(self: *const Self) cute.MajorMode { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAMajor(self.ptr())); + } + pub fn getBMajor(self: *const Self) cute.MajorMode { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBMajor(self.ptr())); + } + pub fn getAType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAType(self.ptr()).ptr.?); + } + pub fn getBType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBType(self.ptr()).ptr.?); + } + pub fn getCType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetCType(self.ptr()).ptr.?); + } + pub fn getSfType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetSfType(self.ptr()).ptr.?); + } + pub fn getAFragKind(self: *const Self) MmaFragKind { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAFragKind(self.ptr())); + } + pub fn getVecSize(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetVecSize(self.ptr()); + } + pub fn getArchPromote(self: *const Self) Arch { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetArchPromote(self.ptr())); + } + pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetIntOverflow(self.ptr()).ptr); } }; -/// `!cute_nvgpu.sm100.mma_sp`. +/// `!cute_nvgpu.sm100.mma_sp`: SM100 sparse UMMA MMA atom. pub const MmaAtomSM100UMMASparseType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM100UMMASparse; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "sm100.mma_sp"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + shapeMnk: *const mlir.Attribute, + numCta: c_int, + aMajor: cute.MajorMode, + bMajor: cute.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + eType: *const mlir.Type, + sparseMetadataFormat: SparseMetadataFormat, + aFragKind: MmaFragKind, + cScaleExp: c_int, + intOverflow: ?*const mlir.Attribute = null, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.eType.ptr(), @intFromEnum(args.sparseMetadataFormat), @intFromEnum(args.aFragKind), args.cScaleExp, if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetPayload(self.ptr()).ptr.?); + pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn getNumCta(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetNumCta(self.ptr()); + } + pub fn getAMajor(self: *const Self) cute.MajorMode { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAMajor(self.ptr())); + } + pub fn getBMajor(self: *const Self) cute.MajorMode { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBMajor(self.ptr())); + } + pub fn getAType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAType(self.ptr()).ptr.?); + } + pub fn getBType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBType(self.ptr()).ptr.?); + } + pub fn getCType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCType(self.ptr()).ptr.?); + } + pub fn getEType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetEType(self.ptr()).ptr.?); + } + pub fn getSparseMetadataFormat(self: *const Self) SparseMetadataFormat { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetSparseMetadataFormat(self.ptr())); + } + pub fn getAFragKind(self: *const Self) MmaFragKind { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAFragKind(self.ptr())); + } + pub fn getCScaleExp(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCScaleExp(self.ptr()); + } + pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetIntOverflow(self.ptr()).ptr); } }; -/// `!cute_nvgpu.sm100.mma`. +/// `!cute_nvgpu.sm100.mma`: SM100 UMMA MMA atom. pub const MmaAtomSM100UMMAType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM100UMMA; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "sm100.mma"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + shapeMnk: *const mlir.Attribute, + numCta: c_int, + aMajor: cute.MajorMode, + bMajor: cute.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + aFragKind: MmaFragKind, + cScaleExp: c_int, + intOverflow: ?*const mlir.Attribute = null, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM100UMMATypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUMmaAtomSM100UMMATypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @intFromEnum(args.aFragKind), args.cScaleExp, if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetPayload(self.ptr()).ptr.?); + pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn getNumCta(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetNumCta(self.ptr()); + } + pub fn getAMajor(self: *const Self) cute.MajorMode { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAMajor(self.ptr())); + } + pub fn getBMajor(self: *const Self) cute.MajorMode { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetBMajor(self.ptr())); + } + pub fn getAType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAType(self.ptr()).ptr.?); + } + pub fn getBType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetBType(self.ptr()).ptr.?); + } + pub fn getCType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetCType(self.ptr()).ptr.?); + } + pub fn getAFragKind(self: *const Self) MmaFragKind { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAFragKind(self.ptr())); + } + pub fn getCScaleExp(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetCScaleExp(self.ptr()); + } + pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetIntOverflow(self.ptr()).ptr); } }; -/// `!cute_nvgpu.SM120.mma_bs`. +/// `!cute_nvgpu.sm107.mma_bs_sp`: SM107 block-scaled sparse UMMA MMA atom. +pub const MmaAtomSM107UMMABlockScaledSparseType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaledSparse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "sm107.mma_bs_sp"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + numCta: c_int, + aMajor: cute.MajorMode, + bMajor: cute.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + sfType: *const mlir.Type, + sparseMetadataFormat: SparseMetadataFormat, + aFragKind: MmaFragKind, + vecSize: c_int, + aCollectorOp: MmaCollectorOp, + bCollectorOp: MmaCollectorOp, + intOverflow: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @intFromEnum(args.sparseMetadataFormat), @intFromEnum(args.aFragKind), args.vecSize, @intFromEnum(args.aCollectorOp), @intFromEnum(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn getNumCta(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetNumCta(self.ptr()); + } + pub fn getAMajor(self: *const Self) cute.MajorMode { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAMajor(self.ptr())); + } + pub fn getBMajor(self: *const Self) cute.MajorMode { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBMajor(self.ptr())); + } + pub fn getAType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAType(self.ptr()).ptr.?); + } + pub fn getBType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBType(self.ptr()).ptr.?); + } + pub fn getCType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetCType(self.ptr()).ptr.?); + } + pub fn getSfType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSfType(self.ptr()).ptr.?); + } + pub fn getSparseMetadataFormat(self: *const Self) SparseMetadataFormat { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSparseMetadataFormat(self.ptr())); + } + pub fn getAFragKind(self: *const Self) MmaFragKind { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAFragKind(self.ptr())); + } + pub fn getVecSize(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetVecSize(self.ptr()); + } + pub fn getACollectorOp(self: *const Self) MmaCollectorOp { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetACollectorOp(self.ptr())); + } + pub fn getBCollectorOp(self: *const Self) MmaCollectorOp { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBCollectorOp(self.ptr())); + } + pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetIntOverflow(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.sm107.mma_bs`: SM107 block-scaled UMMA MMA atom. +pub const MmaAtomSM107UMMABlockScaledType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaled; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "sm107.mma_bs"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + numCta: c_int, + aMajor: cute.MajorMode, + bMajor: cute.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + sfType: *const mlir.Type, + aFragKind: MmaFragKind, + vecSize: c_int, + aCollectorOp: MmaCollectorOp, + bCollectorOp: MmaCollectorOp, + intOverflow: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @intFromEnum(args.aFragKind), args.vecSize, @intFromEnum(args.aCollectorOp), @intFromEnum(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn getNumCta(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetNumCta(self.ptr()); + } + pub fn getAMajor(self: *const Self) cute.MajorMode { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAMajor(self.ptr())); + } + pub fn getBMajor(self: *const Self) cute.MajorMode { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBMajor(self.ptr())); + } + pub fn getAType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAType(self.ptr()).ptr.?); + } + pub fn getBType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBType(self.ptr()).ptr.?); + } + pub fn getCType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetCType(self.ptr()).ptr.?); + } + pub fn getSfType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetSfType(self.ptr()).ptr.?); + } + pub fn getAFragKind(self: *const Self) MmaFragKind { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAFragKind(self.ptr())); + } + pub fn getVecSize(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetVecSize(self.ptr()); + } + pub fn getACollectorOp(self: *const Self) MmaCollectorOp { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetACollectorOp(self.ptr())); + } + pub fn getBCollectorOp(self: *const Self) MmaCollectorOp { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBCollectorOp(self.ptr())); + } + pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetIntOverflow(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.sm107.mma_sp`: SM107 sparse UMMA MMA atom. +pub const MmaAtomSM107UMMASparseType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM107UMMASparse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "sm107.mma_sp"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + numCta: c_int, + aMajor: cute.MajorMode, + bMajor: cute.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + eType: *const mlir.Type, + sparseMetadataFormat: SparseMetadataFormat, + aFragKind: MmaFragKind, + cScaleExp: c_int, + aCollectorOp: MmaCollectorOp, + bCollectorOp: MmaCollectorOp, + intOverflow: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.eType.ptr(), @intFromEnum(args.sparseMetadataFormat), @intFromEnum(args.aFragKind), args.cScaleExp, @intFromEnum(args.aCollectorOp), @intFromEnum(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn getNumCta(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetNumCta(self.ptr()); + } + pub fn getAMajor(self: *const Self) cute.MajorMode { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAMajor(self.ptr())); + } + pub fn getBMajor(self: *const Self) cute.MajorMode { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBMajor(self.ptr())); + } + pub fn getAType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAType(self.ptr()).ptr.?); + } + pub fn getBType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBType(self.ptr()).ptr.?); + } + pub fn getCType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCType(self.ptr()).ptr.?); + } + pub fn getEType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetEType(self.ptr()).ptr.?); + } + pub fn getSparseMetadataFormat(self: *const Self) SparseMetadataFormat { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetSparseMetadataFormat(self.ptr())); + } + pub fn getAFragKind(self: *const Self) MmaFragKind { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAFragKind(self.ptr())); + } + pub fn getCScaleExp(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCScaleExp(self.ptr()); + } + pub fn getACollectorOp(self: *const Self) MmaCollectorOp { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetACollectorOp(self.ptr())); + } + pub fn getBCollectorOp(self: *const Self) MmaCollectorOp { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBCollectorOp(self.ptr())); + } + pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetIntOverflow(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.sm107.mma`: SM107 UMMA MMA atom. +pub const MmaAtomSM107UMMAType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM107UMMA; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "sm107.mma"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + numCta: c_int, + aMajor: cute.MajorMode, + bMajor: cute.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + aFragKind: MmaFragKind, + cScaleExp: c_int, + aCollectorOp: MmaCollectorOp, + bCollectorOp: MmaCollectorOp, + intOverflow: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM107UMMATypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @intFromEnum(args.aFragKind), args.cScaleExp, @intFromEnum(args.aCollectorOp), @intFromEnum(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn getNumCta(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetNumCta(self.ptr()); + } + pub fn getAMajor(self: *const Self) cute.MajorMode { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAMajor(self.ptr())); + } + pub fn getBMajor(self: *const Self) cute.MajorMode { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBMajor(self.ptr())); + } + pub fn getAType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAType(self.ptr()).ptr.?); + } + pub fn getBType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBType(self.ptr()).ptr.?); + } + pub fn getCType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetCType(self.ptr()).ptr.?); + } + pub fn getAFragKind(self: *const Self) MmaFragKind { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAFragKind(self.ptr())); + } + pub fn getCScaleExp(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetCScaleExp(self.ptr()); + } + pub fn getACollectorOp(self: *const Self) MmaCollectorOp { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetACollectorOp(self.ptr())); + } + pub fn getBCollectorOp(self: *const Self) MmaCollectorOp { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBCollectorOp(self.ptr())); + } + pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetIntOverflow(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.SM120.mma_bs`: SM120 block-scaled mma.sync MMA atom. pub const MmaAtomSM120BlockScaledType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM120BlockScaled; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "SM120.mma_bs"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + shapeMnk: *const mlir.Attribute, + vecSize: c_int, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + sfType: *const mlir.Type, + useSfLayoutTV: bool, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.vecSize, args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), args.useSfLayoutTV); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetPayload(self.ptr()).ptr.?); + pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn getVecSize(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetVecSize(self.ptr()); + } + pub fn getAType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetAType(self.ptr()).ptr.?); + } + pub fn getBType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetBType(self.ptr()).ptr.?); + } + pub fn getCType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetCType(self.ptr()).ptr.?); + } + pub fn getSfType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetSfType(self.ptr()).ptr.?); + } + pub fn getUseSfLayoutTV(self: *const Self) bool { + return c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetUseSfLayoutTV(self.ptr()); } }; -/// `!cute_nvgpu.sm103.smem_desc_circular`. +/// `!cute_nvgpu.sm80.sparse_mma`: SM80 sparse mma.sync MMA atom. +pub const MmaAtomSM80SparseType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM80Sparse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "sm80.sparse_mma"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + sparseMetadataFormat: SparseMetadataFormat, + intOverflow: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM80SparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @intFromEnum(args.sparseMetadataFormat), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn getAType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetAType(self.ptr()).ptr.?); + } + pub fn getBType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetBType(self.ptr()).ptr.?); + } + pub fn getCType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetCType(self.ptr()).ptr.?); + } + pub fn getSparseMetadataFormat(self: *const Self) SparseMetadataFormat { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetSparseMetadataFormat(self.ptr())); + } + pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetIntOverflow(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.sm80.mma`: SM80 mma.sync MMA atom. +pub const MmaAtomSM80Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM80; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "sm80.mma"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + intOverflow: ?*const mlir.Attribute = null, + binaryOp: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM80TypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, if (args.binaryOp) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn getAType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetAType(self.ptr()).ptr.?); + } + pub fn getBType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetBType(self.ptr()).ptr.?); + } + pub fn getCType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetCType(self.ptr()).ptr.?); + } + pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetIntOverflow(self.ptr()).ptr); + } + pub fn getBinaryOp(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetBinaryOp(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.sm89.mma`: SM89 fp8 mma.sync MMA atom. +pub const MmaAtomSM89Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM89; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "sm89.mma"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM89TypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM89TypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn getAType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM89TypeGetAType(self.ptr()).ptr.?); + } + pub fn getBType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM89TypeGetBType(self.ptr()).ptr.?); + } + pub fn getCType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM89TypeGetCType(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.sm90.mma`: SM90 wgmma MMA atom. +pub const MmaAtomSM90Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM90; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "sm90.mma"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + aMajor: cute.MajorMode, + bMajor: cute.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + aFragKind: MmaFragKind, + intOverflow: ?*const mlir.Attribute = null, + aNeg: bool = false, + bNeg: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM90TypeGet(ctx.ptr(), args.shapeMnk.ptr(), @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @intFromEnum(args.aFragKind), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, args.aNeg, args.bNeg); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn getAMajor(self: *const Self) cute.MajorMode { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM90TypeGetAMajor(self.ptr())); + } + pub fn getBMajor(self: *const Self) cute.MajorMode { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM90TypeGetBMajor(self.ptr())); + } + pub fn getAType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetAType(self.ptr()).ptr.?); + } + pub fn getBType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetBType(self.ptr()).ptr.?); + } + pub fn getCType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetCType(self.ptr()).ptr.?); + } + pub fn getAFragKind(self: *const Self) MmaFragKind { + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM90TypeGetAFragKind(self.ptr())); + } + pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetIntOverflow(self.ptr()).ptr); + } + pub fn getANeg(self: *const Self) bool { + return c.mlirCuteNVGPUMmaAtomSM90TypeGetANeg(self.ptr()); + } + pub fn getBNeg(self: *const Self) bool { + return c.mlirCuteNVGPUMmaAtomSM90TypeGetBNeg(self.ptr()); + } +}; + +/// `!cute_nvgpu.sm103.smem_desc_circular`: Circular shared-memory descriptor over a block layout in bytes. pub const SmemDescCircularSM103Type = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDescCircularSM103; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "sm103.smem_desc_circular"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + blockLayoutBytes: *const mlir.Attribute, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSmemDescCircularSM103TypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUSmemDescCircularSM103TypeGet(ctx.ptr(), args.blockLayoutBytes.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUSmemDescCircularSM103TypeGetPayload(self.ptr()).ptr.?); + pub fn getBlockLayoutBytes(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUSmemDescCircularSM103TypeGetBlockLayoutBytes(self.ptr()).ptr.?); } }; -/// `!cute_nvgpu.smem_desc`. +/// `!cute_nvgpu.sm107.smem_desc_circular`: Circular shared-memory descriptor over a block layout in bytes. +pub const SmemDescCircularSM107Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDescCircularSM107; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "sm107.smem_desc_circular"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + blockLayoutBytes: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSmemDescCircularSM107TypeGet(ctx.ptr(), args.blockLayoutBytes.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getBlockLayoutBytes(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUSmemDescCircularSM107TypeGetBlockLayoutBytes(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.sm107.smem_desc`: SM107 shared-memory descriptor. +pub const SmemDescSM107Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDescSM107; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "sm107.smem_desc"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCuteNVGPUSmemDescSM107TypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cute_nvgpu.smem_desc`: UMMA/GMMA shared-memory descriptor. pub const SmemDescType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDesc; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "smem_desc"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { _ = args; const result = c.mlirCuteNVGPUSmemDescTypeGet(ctx.ptr()); @@ -1150,103 +2774,53 @@ pub const SmemDescType = opaque { } }; -/// `!cute_nvgpu.smem_desc_view`. +/// `!cute_nvgpu.smem_desc_view`: Shared-memory descriptor iterator with a layout. pub const SmemDescViewType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDescView; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "smem_desc_view"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + desc: *const mlir.Type, + layout: *const mlir.Attribute, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSmemDescViewTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUSmemDescViewTypeGet(ctx.ptr(), args.desc.ptr(), args.layout.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUSmemDescViewTypeGetPayload(self.ptr()).ptr.?); + pub fn getDesc(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUSmemDescViewTypeGetDesc(self.ptr()).ptr.?); + } + pub fn getLayout(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUSmemDescViewTypeGetLayout(self.ptr()).ptr.?); } }; -/// `!cute_nvgpu.tiled_copy`. -pub const TiledCopyType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUTiledCopy; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - payload: *const mlir.Attribute, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTiledCopyTypeGet(ctx.ptr(), args.payload.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUTiledCopyTypeGetPayload(self.ptr()).ptr.?); - } -}; - -/// `!cute_nvgpu.tiled_mma`. -pub const TiledMmaType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUTiledMma; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - payload: *const mlir.Attribute, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTiledMmaTypeGet(ctx.ptr(), args.payload.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUTiledMmaTypeGetPayload(self.ptr()).ptr.?); - } -}; - -/// `!cute_nvgpu.tma_descriptor_im2col`. +/// `!cute_nvgpu.tma_descriptor_im2col`: Im2col TMA descriptor. pub const TmaDescriptorIm2ColType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUTmaDescriptorIm2Col; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "tma_descriptor_im2col"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { _ = args; const result = c.mlirCuteNVGPUTmaDescriptorIm2ColTypeGet(ctx.ptr()); @@ -1254,22 +2828,22 @@ pub const TmaDescriptorIm2ColType = opaque { } }; -/// `!cute_nvgpu.tma_descriptor_tiled`. +/// `!cute_nvgpu.tma_descriptor_tiled`: Tiled TMA descriptor. pub const TmaDescriptorTiledType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUTmaDescriptorTiled; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "tma_descriptor_tiled"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { _ = args; const result = c.mlirCuteNVGPUTmaDescriptorTiledTypeGet(ctx.ptr()); @@ -1277,49 +2851,61 @@ pub const TmaDescriptorTiledType = opaque { } }; -/// `!cute_nvgpu.atom.universal_fma`. +/// `!cute_nvgpu.atom.universal_fma`: Universal FMA MMA atom. pub const UniversalFmaAtomType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUUniversalFmaAtom; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom.universal_fma"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - payload: *const mlir.Attribute, + shapeMnk: *const mlir.Attribute, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUUniversalFmaAtomTypeGet(ctx.ptr(), args.payload.ptr()); + const result = c.mlirCuteNVGPUUniversalFmaAtomTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPayload(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUUniversalFmaAtomTypeGetPayload(self.ptr()).ptr.?); + pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUUniversalFmaAtomTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn getAType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUUniversalFmaAtomTypeGetAType(self.ptr()).ptr.?); + } + pub fn getBType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUUniversalFmaAtomTypeGetBType(self.ptr()).ptr.?); + } + pub fn getCType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUUniversalFmaAtomTypeGetCType(self.ptr()).ptr.?); } }; -/// `!cute_nvgpu.workid_response`. +/// `!cute_nvgpu.workid_response`: Cluster launch control work-id response. pub const WorkIdResponseType = opaque { const Self = @This(); const M = mlir.Methods(Self, c.MlirType); pub const isAFn = c.mlirTypeIsACuteNVGPUWorkIdResponse; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "workid_response"; pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { return self.type_().format(writer); } - pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { _ = args; const result = c.mlirCuteNVGPUWorkIdResponseTypeGet(ctx.ptr()); @@ -1327,6 +2913,1393 @@ pub const WorkIdResponseType = opaque { } }; +/// `#cute_nvgpu.atom_copy_field_bulkg2s`: Fields stored in Bulk Load Copy Atom type. +pub const AtomCopyFieldBulkCopyG2SAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyG2S; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_bulkg2s"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldBulkCopyG2S, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) AtomCopyFieldBulkCopyG2S { + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.atom_copy_field_bulks2g`: Fields stored in Bulk Store Copy Atom type. +pub const AtomCopyFieldBulkCopyS2GAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2G; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_bulks2g"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldBulkCopyS2G, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) AtomCopyFieldBulkCopyS2G { + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.atom_copy_field_bulks2s`: Fields stored in Bulk CTA to Cluster Copy Atom type. +pub const AtomCopyFieldBulkCopyS2SAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2S; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_bulks2s"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldBulkCopyS2S, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) AtomCopyFieldBulkCopyS2S { + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.atom_copy_field_dsmem_store`: Fields stored in CopyAtomDsmemStoreType. +pub const AtomCopyFieldDsmemStoreAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldDsmemStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_dsmem_store"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldDsmemStore, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) AtomCopyFieldDsmemStore { + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.atom_copy_field_g2r`: Fields stored in CopyAtomG2RType. +pub const AtomCopyFieldLoadGlobalAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldLoadGlobal; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_g2r"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldLoadGlobal, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) AtomCopyFieldLoadGlobal { + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.atom_copy_field_non_exec_2d_gather4_tma_load`: Fields stored in CopyAtomNonExec2DGather4TmaLoadType. +pub const AtomCopyFieldNonExec2DGather4TmaLoadAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_non_exec_2d_gather4_tma_load"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldNonExec2DGather4TmaLoad, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) AtomCopyFieldNonExec2DGather4TmaLoad { + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.atom_copy_field_non_exec_2d_scatter4_tma_store`: Fields stored in CopyAtomNonExec2DScatter4TmaStoreType. +pub const AtomCopyFieldNonExec2DScatter4TmaStoreAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_non_exec_2d_scatter4_tma_store"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldNonExec2DScatter4TmaStore, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) AtomCopyFieldNonExec2DScatter4TmaStore { + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.atom_copy_field_non_exec_im2col_tma_load`: Fields stored in CopyAtomNonExecIm2ColTmaLoadType. +pub const AtomCopyFieldNonExecIm2ColTmaLoadAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_non_exec_im2col_tma_load"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldNonExecIm2ColTmaLoad, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) AtomCopyFieldNonExecIm2ColTmaLoad { + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.atom_copy_field_non_exec_im2col_tma_store`: Fields stored in CopyAtomNonExecIm2ColTmaStoreType. +pub const AtomCopyFieldNonExecIm2ColTmaStoreAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_non_exec_im2col_tma_store"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldNonExecIm2ColTmaStore, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) AtomCopyFieldNonExecIm2ColTmaStore { + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.atom_copy_field_non_exec_tma_load`: Fields stored in CopyAtomNonExecTiledTmaLoadType. +pub const AtomCopyFieldNonExecTiledTmaLoadAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_non_exec_tma_load"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldNonExecTiledTmaLoad, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) AtomCopyFieldNonExecTiledTmaLoad { + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.atom_copy_field_non_exec_tma_reduce`: Fields stored in CopyAtomNonExecTiledTmaReduceType. +pub const AtomCopyFieldNonExecTiledTmaReduceAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaReduce; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_non_exec_tma_reduce"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldNonExecTiledTmaReduce, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) AtomCopyFieldNonExecTiledTmaReduce { + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.atom_copy_field_non_exec_tma_store`: Fields stored in CopyAtomNonExecTiledTmaStoreType. +pub const AtomCopyFieldNonExecTiledTmaStoreAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_non_exec_tma_store"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldNonExecTiledTmaStore, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) AtomCopyFieldNonExecTiledTmaStore { + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.atom_copy_field_r2g`: Fields stored in CopyAtomR2GType. +pub const AtomCopyFieldStoreGlobalAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldStoreGlobal; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_r2g"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldStoreGlobal, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) AtomCopyFieldStoreGlobal { + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.atom_copy_field_tmaload`: Fields stored in Tma Load Copy Atom type. +pub const AtomCopyFieldTmaLoadAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_tmaload"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldTmaLoad, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) AtomCopyFieldTmaLoad { + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.atom_copy_field_tmareduce`: Fields stored in Tma Reduce Copy Atom type. +pub const AtomCopyFieldTmaReduceAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldTmaReduce; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_tmareduce"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldTmaReduce, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) AtomCopyFieldTmaReduce { + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.atom_copy_field_tmastore`: Fields stored in Tma Store Copy Atom type. +pub const AtomCopyFieldTmaStoreAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldTmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_tmastore"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldTmaStore, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) AtomCopyFieldTmaStore { + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.atom_mma_field_sm100`: Fields stored in MMA Atom types for SM100. +pub const AtomMmaFieldSM100Attr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM100; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom_mma_field_sm100"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomMmaFieldSM100, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomMmaFieldSM100AttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) AtomMmaFieldSM100 { + return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM100AttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.atom_mma_field_sm100_block_scaled`: Fields stored in MMA Atom types for SM100 (block scaled). +pub const AtomMmaFieldSM100BlockScaledAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaled; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom_mma_field_sm100_block_scaled"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomMmaFieldSM100BlockScaled, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) AtomMmaFieldSM100BlockScaled { + return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.atom_mma_field_sm100_block_scaled_sparse`: Fields stored in MMA Atom types for SM100 (block scaled). +pub const AtomMmaFieldSM100BlockScaledSparseAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaledSparse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom_mma_field_sm100_block_scaled_sparse"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomMmaFieldSM100BlockScaledSparse, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) AtomMmaFieldSM100BlockScaledSparse { + return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.atom_mma_field_sm100_sparse`: Fields stored in sparse MMA Atom types for SM100. +pub const AtomMmaFieldSM100SparseAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM100Sparse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom_mma_field_sm100_sparse"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomMmaFieldSM100Sparse, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) AtomMmaFieldSM100Sparse { + return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.atom_mma_field_sm120_block_scaled`: Fields stored in MMA Atom types for SM120 (block scaled). +pub const AtomMmaFieldSM120BlockScaledAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM120BlockScaled; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom_mma_field_sm120_block_scaled"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomMmaFieldSM120BlockScaled, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) AtomMmaFieldSM120BlockScaled { + return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.atom_mma_field_sm80_sparse`: Fields stored in MMA Atom types for SM80. +pub const AtomMmaFieldSM80SparseAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM80Sparse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom_mma_field_sm80_sparse"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomMmaFieldSM80Sparse, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) AtomMmaFieldSM80Sparse { + return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.atom_mma_field_sm90`: Fields stored in MMA Atom types for SM90. +pub const AtomMmaFieldSM90Attr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM90; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "atom_mma_field_sm90"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomMmaFieldSM90, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomMmaFieldSM90AttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) AtomMmaFieldSM90 { + return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM90AttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.bin_op`: Binary operation for single-bit MMA operations. +pub const BinaryOpAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUBinaryOp; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "bin_op"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: BinaryOp, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUBinaryOpAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) BinaryOp { + return @enumFromInt(c.mlirCuteNVGPUBinaryOpAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.copy_s2t_broadcast_mode`: Broadcast modes for the different utccp instructions. +pub const CopyS2TBroadcastAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUCopyS2TBroadcast; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "copy_s2t_broadcast_mode"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: CopyS2TBroadcast, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyS2TBroadcastAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) CopyS2TBroadcast { + return @enumFromInt(c.mlirCuteNVGPUCopyS2TBroadcastAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.gather_scatter_tma_load`: The various kinds of TMA loads in gather/scatter mode. +pub const GatherScatterTmaLoadAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUGatherScatterTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "gather_scatter_tma_load"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: GatherScatterTmaLoad, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUGatherScatterTmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) GatherScatterTmaLoad { + return @enumFromInt(c.mlirCuteNVGPUGatherScatterTmaLoadAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.im2col_tma_load`: The various kinds of TMA loads in im2col mode. +pub const Im2ColTmaLoadAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUIm2ColTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "im2col_tma_load"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Im2ColTmaLoad, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUIm2ColTmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) Im2ColTmaLoad { + return @enumFromInt(c.mlirCuteNVGPUIm2ColTmaLoadAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.ld_reduce_acc_precision_kind`: multimem ld_reduce accumulation precision kind. +pub const LdReduceAccPrecisionKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPULdReduceAccPrecisionKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "ld_reduce_acc_precision_kind"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: LdReduceAccPrecisionKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPULdReduceAccPrecisionKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) LdReduceAccPrecisionKind { + return @enumFromInt(c.mlirCuteNVGPULdReduceAccPrecisionKindAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.ldsm_sz_pattern`: LDSM's sz pattern, describing the bit size. +pub const LdsmSzPatternAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPULdsmSzPattern; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "ldsm_sz_pattern"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: LdsmSzPattern, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPULdsmSzPatternAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) LdsmSzPattern { + return @enumFromInt(c.mlirCuteNVGPULdsmSzPatternAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.load_cache_mode`: Cache modes for the load instructions. +pub const LoadCacheModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPULoadCacheMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "load_cache_mode"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: LoadCacheMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPULoadCacheModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) LoadCacheMode { + return @enumFromInt(c.mlirCuteNVGPULoadCacheModeAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.mma_int_overflow`: MMA overflow options. +pub const MMAIntOverflowAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUMMAIntOverflow; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "mma_int_overflow"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMAIntOverflow, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMMAIntOverflowAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) MMAIntOverflow { + return @enumFromInt(c.mlirCuteNVGPUMMAIntOverflowAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.major`: Major mode for MMA operations. +pub const MajorModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUMajorMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "major"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: cute.MajorMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMajorModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) cute.MajorMode { + return @enumFromInt(c.mlirCuteNVGPUMajorModeAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.mma_collector_op`: Enums for the mma collector op. +pub const MmaCollectorOpAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUMmaCollectorOp; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "mma_collector_op"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MmaCollectorOp, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaCollectorOpAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) MmaCollectorOp { + return @enumFromInt(c.mlirCuteNVGPUMmaCollectorOpAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.mma_frag_kind`: Enums for the mma frag type. +pub const MmaFragKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUMmaFragKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "mma_frag_kind"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MmaFragKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaFragKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) MmaFragKind { + return @enumFromInt(c.mlirCuteNVGPUMmaFragKindAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.not_implemented_frg`: Fragment of an atom that has none. +pub const NotImplementedFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUNotImplementedFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "not_implemented_frg"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCuteNVGPUNotImplementedFrgAttrGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `#cute_nvgpu.tma_reduce_kind`: Op for the TMASTORE instruction. +pub const ReductionKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUReductionKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tma_reduce_kind"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ReductionKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUReductionKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) ReductionKind { + return @enumFromInt(c.mlirCuteNVGPUReductionKindAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.rmem_frg`: Register-memory fragment of an MMA operand. +pub const RmemFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPURmemFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "rmem_frg"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + valueType: *const mlir.Type, + operand: cute.MmaOperand, + deriveElemType: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPURmemFrgAttrGet(ctx.ptr(), args.valueType.ptr(), @intFromEnum(args.operand), args.deriveElemType); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValueType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPURmemFrgAttrGetValueType(self.ptr()).ptr.?); + } + pub fn getOperand(self: *const Self) cute.MmaOperand { + return @enumFromInt(c.mlirCuteNVGPURmemFrgAttrGetOperand(self.ptr())); + } + pub fn getDeriveElemType(self: *const Self) bool { + return c.mlirCuteNVGPURmemFrgAttrGetDeriveElemType(self.ptr()); + } +}; + +/// `#cute_nvgpu.arch.sm100.circular_smem_frg`: SM100 circular shared-memory fragment. +pub const SM100CircularSmemFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUSM100CircularSmemFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "arch.sm100.circular_smem_frg"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + majorMode: cute.MajorMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSM100CircularSmemFrgAttrGet(ctx.ptr(), @intFromEnum(args.majorMode)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getMajorMode(self: *const Self) cute.MajorMode { + return @enumFromInt(c.mlirCuteNVGPUSM100CircularSmemFrgAttrGetMajorMode(self.ptr())); + } +}; + +/// `#cute_nvgpu.arch.sm100.smem_frg`: SM100 shared-memory fragment. +pub const SM100SmemFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUSM100SmemFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "arch.sm100.smem_frg"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + majorMode: cute.MajorMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSM100SmemFrgAttrGet(ctx.ptr(), @intFromEnum(args.majorMode)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getMajorMode(self: *const Self) cute.MajorMode { + return @enumFromInt(c.mlirCuteNVGPUSM100SmemFrgAttrGetMajorMode(self.ptr())); + } +}; + +/// `#cute_nvgpu.arch.sm100.tmem_e_frg`: SM100 tensor-memory sparse-metadata fragment. +pub const SM100TmemEFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUSM100TmemEFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "arch.sm100.tmem_e_frg"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + aType: *const mlir.Type, + eType: *const mlir.Type, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSM100TmemEFrgAttrGet(ctx.ptr(), args.aType.ptr(), args.eType.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getAType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUSM100TmemEFrgAttrGetAType(self.ptr()).ptr.?); + } + pub fn getEType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUSM100TmemEFrgAttrGetEType(self.ptr()).ptr.?); + } +}; + +/// `#cute_nvgpu.arch.sm100.tmem_frg`: SM100 tensor-memory fragment. +pub const SM100TmemFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUSM100TmemFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "arch.sm100.tmem_frg"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + dataType: *const mlir.Type, + storageType: *const mlir.Type, + ctaGroup: c_int, + tmemAllocMode: TmemAllocMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSM100TmemFrgAttrGet(ctx.ptr(), args.dataType.ptr(), args.storageType.ptr(), args.ctaGroup, @intFromEnum(args.tmemAllocMode)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getDataType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUSM100TmemFrgAttrGetDataType(self.ptr()).ptr.?); + } + pub fn getStorageType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUSM100TmemFrgAttrGetStorageType(self.ptr()).ptr.?); + } + pub fn getCtaGroup(self: *const Self) c_int { + return c.mlirCuteNVGPUSM100TmemFrgAttrGetCtaGroup(self.ptr()); + } + pub fn getTmemAllocMode(self: *const Self) TmemAllocMode { + return @enumFromInt(c.mlirCuteNVGPUSM100TmemFrgAttrGetTmemAllocMode(self.ptr())); + } +}; + +/// `#cute_nvgpu.arch.sm100.tmem_sf_frg`: SM100 tensor-memory scale-factor fragment. +pub const SM100TmemSfFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUSM100TmemSfFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "arch.sm100.tmem_sf_frg"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + sfType: *const mlir.Type, + sfVecSize: c_int, + ctaGroup: c_int, + isSfa: bool, + tmemAllocMode: TmemAllocMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSM100TmemSfFrgAttrGet(ctx.ptr(), args.sfType.ptr(), args.sfVecSize, args.ctaGroup, args.isSfa, @intFromEnum(args.tmemAllocMode)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getSfType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUSM100TmemSfFrgAttrGetSfType(self.ptr()).ptr.?); + } + pub fn getSfVecSize(self: *const Self) c_int { + return c.mlirCuteNVGPUSM100TmemSfFrgAttrGetSfVecSize(self.ptr()); + } + pub fn getCtaGroup(self: *const Self) c_int { + return c.mlirCuteNVGPUSM100TmemSfFrgAttrGetCtaGroup(self.ptr()); + } + pub fn getIsSfa(self: *const Self) bool { + return c.mlirCuteNVGPUSM100TmemSfFrgAttrGetIsSfa(self.ptr()); + } + pub fn getTmemAllocMode(self: *const Self) TmemAllocMode { + return @enumFromInt(c.mlirCuteNVGPUSM100TmemSfFrgAttrGetTmemAllocMode(self.ptr())); + } +}; + +/// `#cute_nvgpu.arch.sm107.smem_frg`: SM107 shared-memory fragment. +pub const SM107SmemFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUSM107SmemFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "arch.sm107.smem_frg"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + majorMode: cute.MajorMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSM107SmemFrgAttrGet(ctx.ptr(), @intFromEnum(args.majorMode)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getMajorMode(self: *const Self) cute.MajorMode { + return @enumFromInt(c.mlirCuteNVGPUSM107SmemFrgAttrGetMajorMode(self.ptr())); + } +}; + +/// `#cute_nvgpu.arch.sm90.smem_frg`: SM90 shared-memory fragment. +pub const SM90SmemFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUSM90SmemFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "arch.sm90.smem_frg"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + majorMode: cute.MajorMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSM90SmemFrgAttrGet(ctx.ptr(), @intFromEnum(args.majorMode)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getMajorMode(self: *const Self) cute.MajorMode { + return @enumFromInt(c.mlirCuteNVGPUSM90SmemFrgAttrGetMajorMode(self.ptr())); + } +}; + +/// `#cute_nvgpu.tiled_tma_load`: The various kinds of TMA loads in tiled mode. +pub const TiledTmaLoadAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUTiledTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tiled_tma_load"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TiledTmaLoad, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUTiledTmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) TiledTmaLoad { + return @enumFromInt(c.mlirCuteNVGPUTiledTmaLoadAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.tma_data_format`: Bits 74-71 of the TMA descriptor. +pub const TmaDataFormatAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUTmaDataFormat; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tma_data_format"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TmaDataFormat, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUTmaDataFormatAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) TmaDataFormat { + return @enumFromInt(c.mlirCuteNVGPUTmaDataFormatAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.tma_load_mode`: Modes for the TMA load instruction. +pub const TmaLoadModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUTmaLoadMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tma_load_mode"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TmaLoadMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUTmaLoadModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) TmaLoadMode { + return @enumFromInt(c.mlirCuteNVGPUTmaLoadModeAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.tma_store_mode`: Modes for the TMA store instruction. +pub const TmaStoreModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUTmaStoreMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tma_store_mode"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TmaStoreMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUTmaStoreModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) TmaStoreMode { + return @enumFromInt(c.mlirCuteNVGPUTmaStoreModeAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.tmem_alloc_mode`: Modes for the Tmem allocation. +pub const TmemAllocModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUTmemAllocMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tmem_alloc_mode"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TmemAllocMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUTmemAllocModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) TmemAllocMode { + return @enumFromInt(c.mlirCuteNVGPUTmemAllocModeAttrGetValue(self.ptr())); + } +}; + +/// `#cute_nvgpu.tmem_load_red_op`: Reduce operation for the Tmem load instruction. +pub const TmemLoadRedOpAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUTmemLoadRedOp; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tmem_load_red_op"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TmemLoadRedOp, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUTmemLoadRedOpAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) TmemLoadRedOp { + return @enumFromInt(c.mlirCuteNVGPUTmemLoadRedOpAttrGetValue(self.ptr())); + } +}; + +// Operation builders, generated from the operations' .td. /// `cute_nvgpu.arch.alloc_rmem`. Result types are explicit; attributes use mlir.Attribute. pub fn arch_alloc_rmem(ctx: *mlir.Context, input: *const mlir.Value, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute_nvgpu.arch.alloc_rmem", .{ @@ -1347,63 +4320,6 @@ pub fn arch_alloc_smem(ctx: *mlir.Context, ptr_type: *const mlir.Type, input: *c }); } -/// `cute_nvgpu.arch.copy.SM90.bulk_copy_g2s`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_copy_SM90_bulk_copy_g2s(ctx: *mlir.Context, gmem_data_addr: *const mlir.Value, dsmem_data_addr: *const mlir.Value, dsmem_bar_addr: *const mlir.Value, multicast_mask: ?*const mlir.Value, cache_policy: ?*const mlir.Value, size: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - var operands: stdx.BoundedArray(*const mlir.Value, 5) = .empty; - operands.appendSliceAssumeCapacity(&.{gmem_data_addr}); - operands.appendSliceAssumeCapacity(&.{dsmem_data_addr}); - operands.appendSliceAssumeCapacity(&.{dsmem_bar_addr}); - if (multicast_mask) |value| operands.appendSliceAssumeCapacity(&.{value}); - if (cache_policy) |value| operands.appendSliceAssumeCapacity(&.{value}); - return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM90.bulk_copy_g2s", .{ - .operands = .{ .flat = operands.constSlice() }, - .attributes = &.{ - .named(ctx, "size", size), - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ - 1, - 1, - 1, - if (multicast_mask != null) 1 else 0, - if (cache_policy != null) 1 else 0, - })), - }, - .location = location, - }); -} - -/// `cute_nvgpu.arch.copy.SM90.bulk_copy_s2g`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_copy_SM90_bulk_copy_s2g(ctx: *mlir.Context, smem_data_addr: *const mlir.Value, gmem_data_addr: *const mlir.Value, byte_mask: ?*const mlir.Value, cache_policy: ?*const mlir.Value, size: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - var operands: stdx.BoundedArray(*const mlir.Value, 4) = .empty; - operands.appendSliceAssumeCapacity(&.{smem_data_addr}); - operands.appendSliceAssumeCapacity(&.{gmem_data_addr}); - if (byte_mask) |value| operands.appendSliceAssumeCapacity(&.{value}); - if (cache_policy) |value| operands.appendSliceAssumeCapacity(&.{value}); - return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM90.bulk_copy_s2g", .{ - .operands = .{ .flat = operands.constSlice() }, - .attributes = &.{ - .named(ctx, "size", size), - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ - 1, - 1, - if (byte_mask != null) 1 else 0, - if (cache_policy != null) 1 else 0, - })), - }, - .location = location, - }); -} - -/// `cute_nvgpu.arch.copy.SM90.bulk_copy_s2s`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_copy_SM90_bulk_copy_s2s(ctx: *mlir.Context, smem_data_addr: *const mlir.Value, dsmem_data_addr: *const mlir.Value, dsmem_bar_addr: *const mlir.Value, cta_rank: *const mlir.Value, size: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM90.bulk_copy_s2s", .{ - .operands = .{ .flat = &.{ smem_data_addr, dsmem_data_addr, dsmem_bar_addr, cta_rank } }, - .attributes = &.{ - .named(ctx, "size", size), - }, - .location = location, - }); -} - /// `cute_nvgpu.arch.copy.SM100.copy_s2t`. Result types are explicit; attributes use mlir.Attribute. pub fn arch_copy_SM100_copy_s2t(ctx: *mlir.Context, src: *const mlir.Value, dst: *const mlir.Value, dp: *const mlir.Attribute, bits: *const mlir.Attribute, cta: *const mlir.Attribute, broadcast: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.copy_s2t", .{ @@ -1500,6 +4416,93 @@ pub fn arch_copy_SM100_tmem_store(ctx: *mlir.Context, dst: *const mlir.Value, sr }); } +/// `cute_nvgpu.arch.copy.SM107.tma_load_override`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM107_tma_load_override(ctx: *mlir.Context, src_desc: *const mlir.Value, dsmem_data_addr: *const mlir.Value, dsmem_bar_addr: *const mlir.Value, override_addr: *const mlir.Value, tensor_size: []const *const mlir.Value, lower_stride: []const *const mlir.Value, upper_stride: ?*const mlir.Value, coord: []const *const mlir.Value, multicast_mask: ?*const mlir.Value, cache_policy: ?*const mlir.Value, mode: *const mlir.Attribute, num_cta: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM107.tma_load_override", .{ + .operands = .{ .variadic = &.{ + &.{src_desc}, + &.{dsmem_data_addr}, + &.{dsmem_bar_addr}, + &.{override_addr}, + tensor_size, + lower_stride, + if (upper_stride) |value| &.{value} else &.{}, + coord, + if (multicast_mask) |value| &.{value} else &.{}, + if (cache_policy) |value| &.{value} else &.{}, + } }, + .attributes = &.{ + .named(ctx, "mode", mode), + .named(ctx, "num_cta", num_cta), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM107.tma_store_override`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM107_tma_store_override(ctx: *mlir.Context, dst_desc: *const mlir.Value, src_smem_data_addr: *const mlir.Value, override_addr: *const mlir.Value, tensor_size: []const *const mlir.Value, lower_stride: []const *const mlir.Value, upper_stride: ?*const mlir.Value, coord: []const *const mlir.Value, cache_policy: ?*const mlir.Value, mode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM107.tma_store_override", .{ + .operands = .{ .variadic = &.{ + &.{dst_desc}, + &.{src_smem_data_addr}, + &.{override_addr}, + tensor_size, + lower_stride, + if (upper_stride) |value| &.{value} else &.{}, + coord, + if (cache_policy) |value| &.{value} else &.{}, + } }, + .attributes = &.{ + .named(ctx, "mode", mode), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM90.bulk_copy_g2s`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM90_bulk_copy_g2s(ctx: *mlir.Context, gmem_data_addr: *const mlir.Value, dsmem_data_addr: *const mlir.Value, dsmem_bar_addr: *const mlir.Value, multicast_mask: ?*const mlir.Value, cache_policy: ?*const mlir.Value, size: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM90.bulk_copy_g2s", .{ + .operands = .{ .variadic = &.{ + &.{gmem_data_addr}, + &.{dsmem_data_addr}, + &.{dsmem_bar_addr}, + if (multicast_mask) |value| &.{value} else &.{}, + if (cache_policy) |value| &.{value} else &.{}, + } }, + .attributes = &.{ + .named(ctx, "size", size), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM90.bulk_copy_s2g`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM90_bulk_copy_s2g(ctx: *mlir.Context, smem_data_addr: *const mlir.Value, gmem_data_addr: *const mlir.Value, byte_mask: ?*const mlir.Value, cache_policy: ?*const mlir.Value, size: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM90.bulk_copy_s2g", .{ + .operands = .{ .variadic = &.{ + &.{smem_data_addr}, + &.{gmem_data_addr}, + if (byte_mask) |value| &.{value} else &.{}, + if (cache_policy) |value| &.{value} else &.{}, + } }, + .attributes = &.{ + .named(ctx, "size", size), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM90.bulk_copy_s2s`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM90_bulk_copy_s2s(ctx: *mlir.Context, smem_data_addr: *const mlir.Value, dsmem_data_addr: *const mlir.Value, dsmem_bar_addr: *const mlir.Value, cta_rank: *const mlir.Value, size: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM90.bulk_copy_s2s", .{ + .operands = .{ .flat = &.{ smem_data_addr, dsmem_data_addr, dsmem_bar_addr, cta_rank } }, + .attributes = &.{ + .named(ctx, "size", size), + }, + .location = location, + }); +} + /// `cute_nvgpu.arch.get_dyn_smem`. Result types are explicit; attributes use mlir.Attribute. pub fn arch_get_dyn_smem(ctx: *mlir.Context, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute_nvgpu.arch.get_dyn_smem", .{ @@ -1525,6 +4528,74 @@ pub fn arch_make_warp_uniform(ctx: *mlir.Context, val: *const mlir.Value, res_ty }); } +/// `cute_nvgpu.arch.mma.SM107.umma`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_mma_SM107_umma(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, cd: *const mlir.Value, idesc: *const mlir.Value, accum: *const mlir.Value, disable_output_lane: *const mlir.Value, num_cta: *const mlir.Attribute, scale: *const mlir.Attribute, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, shape_k: *const mlir.Attribute, a_collector_op: *const mlir.Attribute, b_collector_op: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.mma.SM107.umma", .{ + .operands = .{ .flat = &.{ a, b, cd, idesc, accum, disable_output_lane } }, + .attributes = &.{ + .named(ctx, "num_cta", num_cta), + .named(ctx, "scale", scale), + .named(ctx, "a_type", a_type), + .named(ctx, "b_type", b_type), + .named(ctx, "shape_k", shape_k), + .named(ctx, "a_collector_op", a_collector_op), + .named(ctx, "b_collector_op", b_collector_op), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.mma.SM107.umma_block_scaled`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_mma_SM107_umma_block_scaled(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, cd: *const mlir.Value, sf_a: *const mlir.Value, sf_b: *const mlir.Value, idesc: *const mlir.Value, accum: *const mlir.Value, num_cta: *const mlir.Attribute, vec_size: *const mlir.Attribute, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, shape_k: *const mlir.Attribute, a_collector_op: *const mlir.Attribute, b_collector_op: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.mma.SM107.umma_block_scaled", .{ + .operands = .{ .flat = &.{ a, b, cd, sf_a, sf_b, idesc, accum } }, + .attributes = &.{ + .named(ctx, "num_cta", num_cta), + .named(ctx, "vec_size", vec_size), + .named(ctx, "a_type", a_type), + .named(ctx, "b_type", b_type), + .named(ctx, "shape_k", shape_k), + .named(ctx, "a_collector_op", a_collector_op), + .named(ctx, "b_collector_op", b_collector_op), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_mma_SM107_umma_block_scaled_sparse(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, cd: *const mlir.Value, sf_a: *const mlir.Value, sf_b: *const mlir.Value, e_op: *const mlir.Value, idesc: *const mlir.Value, accum: *const mlir.Value, num_cta: *const mlir.Attribute, vec_size: *const mlir.Attribute, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, shape_k: *const mlir.Attribute, a_collector_op: *const mlir.Attribute, b_collector_op: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse", .{ + .operands = .{ .flat = &.{ a, b, cd, sf_a, sf_b, e_op, idesc, accum } }, + .attributes = &.{ + .named(ctx, "num_cta", num_cta), + .named(ctx, "vec_size", vec_size), + .named(ctx, "a_type", a_type), + .named(ctx, "b_type", b_type), + .named(ctx, "shape_k", shape_k), + .named(ctx, "a_collector_op", a_collector_op), + .named(ctx, "b_collector_op", b_collector_op), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.mma.SM107.umma_sparse`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_mma_SM107_umma_sparse(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, cd: *const mlir.Value, e: *const mlir.Value, idesc: *const mlir.Value, accum: *const mlir.Value, num_cta: *const mlir.Attribute, scale: *const mlir.Attribute, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, shape_k: *const mlir.Attribute, a_collector_op: *const mlir.Attribute, b_collector_op: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.mma.SM107.umma_sparse", .{ + .operands = .{ .flat = &.{ a, b, cd, e, idesc, accum } }, + .attributes = &.{ + .named(ctx, "num_cta", num_cta), + .named(ctx, "scale", scale), + .named(ctx, "a_type", a_type), + .named(ctx, "b_type", b_type), + .named(ctx, "shape_k", shape_k), + .named(ctx, "a_collector_op", a_collector_op), + .named(ctx, "b_collector_op", b_collector_op), + }, + .location = location, + }); +} + /// `cute_nvgpu.arch.mma.SM120.block_scaled`. Result types are explicit; attributes use mlir.Attribute. pub fn arch_mma_SM120_block_scaled(ctx: *mlir.Context, opA: []const *const mlir.Value, opB: []const *const mlir.Value, opC: []const *const mlir.Value, sfA: *const mlir.Value, sfB: *const mlir.Value, byte_id_a: ?*const mlir.Value, byte_id_b: ?*const mlir.Value, res_types: []const *const mlir.Type, shape_MNK: *const mlir.Attribute, vec_size: *const mlir.Attribute, thread_id_a: ?*const mlir.Attribute, thread_id_b: ?*const mlir.Attribute, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, sf_type: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; @@ -1545,7 +4616,9 @@ pub fn arch_mma_SM120_block_scaled(ctx: *mlir.Context, opA: []const *const mlir. if (byte_id_a) |value| &.{value} else &.{}, if (byte_id_b) |value| &.{value} else &.{}, } }, - .results = .{ .variadic = &.{res_types} }, + .results = .{ .variadic = &.{ + res_types, + } }, .attributes = attributes.constSlice(), .location = location, }); @@ -1603,13 +4676,13 @@ pub fn arch_sm100_retrieve_tmem_ptr(ctx: *mlir.Context, smem_ptr: *const mlir.Va } /// `cute_nvgpu.atom.get_coord_tensor`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_get_coord_tensor(ctx: *mlir.Context, atom: *const mlir.Value, shape: *const mlir.Value, offset_b: []const *const mlir.Value, layout_b: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +pub fn atom_get_coord_tensor(ctx: *mlir.Context, atom: *const mlir.Value, shape: *const mlir.Value, offset_b: ?*const mlir.Value, layout_b: ?*const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute_nvgpu.atom.get_coord_tensor", .{ .operands = .{ .variadic = &.{ &.{atom}, &.{shape}, - offset_b, - layout_b, + if (offset_b) |value| &.{value} else &.{}, + if (layout_b) |value| &.{value} else &.{}, } }, .results = .{ .flat = &.{result_type} }, .location = location, @@ -1638,10 +4711,14 @@ pub fn atom_get_value(ctx: *mlir.Context, atom: *const mlir.Value, result_type: } /// `cute_nvgpu.atom.make_exec_tma`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_make_exec_tma(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +pub fn atom_make_exec_tma(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, override: ?*const mlir.Attribute, no_fully_oob_tile: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (override) |value| attributes.appendAssumeCapacity(.named(ctx, "override", value)); + if (no_fully_oob_tile) |value| attributes.appendAssumeCapacity(.named(ctx, "no_fully_oob_tile", value)); return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_exec_tma", .{ .operands = .{ .flat = &.{input} }, .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), .location = location, }); } @@ -1660,6 +4737,18 @@ pub fn atom_make_non_exec_2d_gather4_tma_load(ctx: *mlir.Context, gmem_tensor: * }); } +/// `cute_nvgpu.atom.make_non_exec_2d_scatter4_tma_store`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_non_exec_2d_scatter4_tma_store(ctx: *mlir.Context, gmem_tensor: *const mlir.Value, gmem_index_layout: *const mlir.Value, smem_layout: *const mlir.Value, cta_v_map: *const mlir.Value, non_exec_atom_type: *const mlir.Type, tma_tensor_type: *const mlir.Type, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_2d_scatter4_tma_store", .{ + .operands = .{ .flat = &.{ gmem_tensor, gmem_index_layout, smem_layout, cta_v_map } }, + .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + /// `cute_nvgpu.atom.make_non_exec_im2col_tma_load`. Result types are explicit; attributes use mlir.Attribute. pub fn atom_make_non_exec_im2col_tma_load(ctx: *mlir.Context, gmem_tensor: *const mlir.Value, smem_layout: *const mlir.Value, cta_v_map: *const mlir.Value, lower_corner_whd: *const mlir.Value, upper_corner_whd: *const mlir.Value, lower_padding_whd: *const mlir.Value, upper_padding_whd: *const mlir.Value, stride_whd: *const mlir.Value, lower_srt: *const mlir.Value, stride_srt: *const mlir.Value, non_exec_atom_type: *const mlir.Type, tma_tensor_type: *const mlir.Type, kind: *const mlir.Attribute, num_multicast: ?*const mlir.Attribute, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; @@ -1734,6 +4823,15 @@ pub fn atom_make_s2t_copy(ctx: *mlir.Context, copy_s2t_atom: *const mlir.Value, }); } +/// `cute_nvgpu.atom.make_tma_residue_tensor`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_tma_residue_tensor(ctx: *mlir.Context, tma_atom: *const mlir.Value, gmem_tensor: *const mlir.Value, residue_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_tma_residue_tensor", .{ + .operands = .{ .flat = &.{ tma_atom, gmem_tensor } }, + .results = .{ .flat = &.{residue_type} }, + .location = location, + }); +} + /// `cute_nvgpu.atom.make_tmem_copy`. Result types are explicit; attributes use mlir.Attribute. pub fn atom_make_tmem_copy(ctx: *mlir.Context, atom: *const mlir.Value, tmem_memref: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_tmem_copy", .{ @@ -1755,6 +4853,15 @@ pub fn atom_set_value(ctx: *mlir.Context, atom: *const mlir.Value, value_: *cons }); } +/// `cute_nvgpu.atom.sm107.get_copy_s2t_smem_desc_view`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_sm107_get_copy_s2t_smem_desc_view(ctx: *mlir.Context, atom: *const mlir.Value, view: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.sm107.get_copy_s2t_smem_desc_view", .{ + .operands = .{ .flat = &.{ atom, view } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + /// `cute_nvgpu.atom.tma_partition`. Result types are explicit; attributes use mlir.Attribute. pub fn atom_tma_partition(ctx: *mlir.Context, tma_atom: *const mlir.Value, cta_coord: *const mlir.Value, cta_layout: *const mlir.Value, smem_tensor: *const mlir.Value, target_tensors: []const *const mlir.Value, res_smem_tensor_type: *const mlir.Type, res_target_tensors_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cute_nvgpu.atom.tma_partition", .{ @@ -1821,18 +4928,6 @@ pub fn make_gmma_smem_desc(ctx: *mlir.Context, src: *const mlir.Value, res_type: }); } -/// `cute_nvgpu.make_tma_desc_im2col_at`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_tma_desc_im2col_at(ctx: *mlir.Context, gmem_view: *const mlir.Value, smem_layout: *const mlir.Value, cta_value_tile: *const mlir.Value, lower_corner_whd: *const mlir.Value, upper_corner_whd: *const mlir.Value, lower_padding_whd: *const mlir.Value, upper_padding_whd: *const mlir.Value, stride_whd: *const mlir.Value, lower_srt: *const mlir.Value, stride_srt: *const mlir.Value, tma_desc_addr: *const mlir.Value, num_multicast: ?*const mlir.Attribute, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); - if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); - return mlir.Operation.make(ctx, "cute_nvgpu.make_tma_desc_im2col_at", .{ - .operands = .{ .flat = &.{ gmem_view, smem_layout, cta_value_tile, lower_corner_whd, upper_corner_whd, lower_padding_whd, upper_padding_whd, stride_whd, lower_srt, stride_srt, tma_desc_addr } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - /// `cute_nvgpu.make_tma_desc_im2col`. Result types are explicit; attributes use mlir.Attribute. pub fn make_tma_desc_im2col(ctx: *mlir.Context, gmem_view: *const mlir.Value, smem_layout: *const mlir.Value, cta_value_tile: *const mlir.Value, lower_corner_whd: *const mlir.Value, upper_corner_whd: *const mlir.Value, lower_padding_whd: *const mlir.Value, upper_padding_whd: *const mlir.Value, stride_whd: *const mlir.Value, lower_srt: *const mlir.Value, stride_srt: *const mlir.Value, result_type: *const mlir.Type, num_multicast: ?*const mlir.Attribute, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; @@ -1846,20 +4941,13 @@ pub fn make_tma_desc_im2col(ctx: *mlir.Context, gmem_view: *const mlir.Value, sm }); } -/// `cute_nvgpu.make_tma_desc_tiled_at`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_tma_desc_tiled_at(ctx: *mlir.Context, gmem_view: *const mlir.Value, smem_layout: *const mlir.Value, cta_value_tile: *const mlir.Value, tma_desc_addr: *const mlir.Value, traversal_stride: ?*const mlir.Value, mode: *const mlir.Attribute, num_multicast: ?*const mlir.Attribute, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - var operands: stdx.BoundedArray(*const mlir.Value, 5) = .empty; - operands.appendSliceAssumeCapacity(&.{gmem_view}); - operands.appendSliceAssumeCapacity(&.{smem_layout}); - operands.appendSliceAssumeCapacity(&.{cta_value_tile}); - operands.appendSliceAssumeCapacity(&.{tma_desc_addr}); - if (traversal_stride) |value| operands.appendSliceAssumeCapacity(&.{value}); - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "mode", mode)); +/// `cute_nvgpu.make_tma_desc_im2col_at`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tma_desc_im2col_at(ctx: *mlir.Context, gmem_view: *const mlir.Value, smem_layout: *const mlir.Value, cta_value_tile: *const mlir.Value, lower_corner_whd: *const mlir.Value, upper_corner_whd: *const mlir.Value, lower_padding_whd: *const mlir.Value, upper_padding_whd: *const mlir.Value, stride_whd: *const mlir.Value, lower_srt: *const mlir.Value, stride_srt: *const mlir.Value, tma_desc_addr: *const mlir.Value, num_multicast: ?*const mlir.Attribute, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; if (num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); - return mlir.Operation.make(ctx, "cute_nvgpu.make_tma_desc_tiled_at", .{ - .operands = .{ .flat = operands.constSlice() }, + return mlir.Operation.make(ctx, "cute_nvgpu.make_tma_desc_im2col_at", .{ + .operands = .{ .flat = &.{ gmem_view, smem_layout, cta_value_tile, lower_corner_whd, upper_corner_whd, lower_padding_whd, upper_padding_whd, stride_whd, lower_srt, stride_srt, tma_desc_addr } }, .attributes = attributes.constSlice(), .location = location, }); @@ -1867,30 +4955,49 @@ pub fn make_tma_desc_tiled_at(ctx: *mlir.Context, gmem_view: *const mlir.Value, /// `cute_nvgpu.make_tma_desc_tiled`. Result types are explicit; attributes use mlir.Attribute. pub fn make_tma_desc_tiled(ctx: *mlir.Context, gmem_view: *const mlir.Value, smem_layout: *const mlir.Value, cta_value_tile: *const mlir.Value, traversal_stride: ?*const mlir.Value, result_type: *const mlir.Type, mode: *const mlir.Attribute, num_multicast: ?*const mlir.Attribute, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - var operands: stdx.BoundedArray(*const mlir.Value, 4) = .empty; - operands.appendSliceAssumeCapacity(&.{gmem_view}); - operands.appendSliceAssumeCapacity(&.{smem_layout}); - operands.appendSliceAssumeCapacity(&.{cta_value_tile}); - if (traversal_stride) |value| operands.appendSliceAssumeCapacity(&.{value}); var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; attributes.appendAssumeCapacity(.named(ctx, "mode", mode)); if (num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); return mlir.Operation.make(ctx, "cute_nvgpu.make_tma_desc_tiled", .{ - .operands = .{ .flat = operands.constSlice() }, + .operands = .{ .variadic = &.{ + &.{gmem_view}, + &.{smem_layout}, + &.{cta_value_tile}, + if (traversal_stride) |value| &.{value} else &.{}, + } }, .results = .{ .flat = &.{result_type} }, .attributes = attributes.constSlice(), .location = location, }); } +/// `cute_nvgpu.make_tma_desc_tiled_at`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tma_desc_tiled_at(ctx: *mlir.Context, gmem_view: *const mlir.Value, smem_layout: *const mlir.Value, cta_value_tile: *const mlir.Value, tma_desc_addr: *const mlir.Value, traversal_stride: ?*const mlir.Value, mode: *const mlir.Attribute, num_multicast: ?*const mlir.Attribute, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "mode", mode)); + if (num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.make_tma_desc_tiled_at", .{ + .operands = .{ .variadic = &.{ + &.{gmem_view}, + &.{smem_layout}, + &.{cta_value_tile}, + &.{tma_desc_addr}, + if (traversal_stride) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + /// `cute_nvgpu.make_umma_smem_desc`. Result types are explicit; attributes use mlir.Attribute. pub fn make_umma_smem_desc(ctx: *mlir.Context, src: *const mlir.Value, nextSrc: ?*const mlir.Value, res_type: *const mlir.Type, layout: *const mlir.Attribute, major: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - var operands: stdx.BoundedArray(*const mlir.Value, 2) = .empty; - operands.appendSliceAssumeCapacity(&.{src}); - if (nextSrc) |value| operands.appendSliceAssumeCapacity(&.{value}); return mlir.Operation.make(ctx, "cute_nvgpu.make_umma_smem_desc", .{ - .operands = .{ .flat = operands.constSlice() }, + .operands = .{ .variadic = &.{ + &.{src}, + if (nextSrc) |value| &.{value} else &.{}, + } }, .results = .{ .flat = &.{res_type} }, .attributes = &.{ .named(ctx, "layout", layout), @@ -1910,11 +5017,43 @@ pub fn prefetch_tma_desc(ctx: *mlir.Context, atom: *const mlir.Value, location: /// `cute_nvgpu.sm103.make_umma_smem_desc_circular`. Result types are explicit; attributes use mlir.Attribute. pub fn sm103_make_umma_smem_desc_circular(ctx: *mlir.Context, src: *const mlir.Value, nextSrc: ?*const mlir.Value, res_type: *const mlir.Type, layout: *const mlir.Attribute, major: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - var operands: stdx.BoundedArray(*const mlir.Value, 2) = .empty; - operands.appendSliceAssumeCapacity(&.{src}); - if (nextSrc) |value| operands.appendSliceAssumeCapacity(&.{value}); return mlir.Operation.make(ctx, "cute_nvgpu.sm103.make_umma_smem_desc_circular", .{ - .operands = .{ .flat = operands.constSlice() }, + .operands = .{ .variadic = &.{ + &.{src}, + if (nextSrc) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "layout", layout), + .named(ctx, "major", major), + }, + .location = location, + }); +} + +/// `cute_nvgpu.sm107.make_umma_smem_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn sm107_make_umma_smem_desc(ctx: *mlir.Context, src: *const mlir.Value, nextSrc: ?*const mlir.Value, res_type: *const mlir.Type, layout: *const mlir.Attribute, major: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.sm107.make_umma_smem_desc", .{ + .operands = .{ .variadic = &.{ + &.{src}, + if (nextSrc) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "layout", layout), + .named(ctx, "major", major), + }, + .location = location, + }); +} + +/// `cute_nvgpu.sm107.make_umma_smem_desc_circular`. Result types are explicit; attributes use mlir.Attribute. +pub fn sm107_make_umma_smem_desc_circular(ctx: *mlir.Context, src: *const mlir.Value, nextSrc: ?*const mlir.Value, res_type: *const mlir.Type, layout: *const mlir.Attribute, major: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.sm107.make_umma_smem_desc_circular", .{ + .operands = .{ .variadic = &.{ + &.{src}, + if (nextSrc) |value| &.{value} else &.{}, + } }, .results = .{ .flat = &.{res_type} }, .attributes = &.{ .named(ctx, "layout", layout), @@ -1935,18 +5074,24 @@ pub fn update_tma_desc(ctx: *mlir.Context, tma_atom: *const mlir.Value, gmem_ten pub const operation_names: []const []const u8 = &.{ "cute_nvgpu.arch.alloc_rmem", "cute_nvgpu.arch.alloc_smem", - "cute_nvgpu.arch.copy.SM90.bulk_copy_g2s", - "cute_nvgpu.arch.copy.SM90.bulk_copy_s2g", - "cute_nvgpu.arch.copy.SM90.bulk_copy_s2s", "cute_nvgpu.arch.copy.SM100.copy_s2t", "cute_nvgpu.arch.copy.SM100.tma_load", "cute_nvgpu.arch.copy.SM100.tma_reduce", "cute_nvgpu.arch.copy.SM100.tma_store", "cute_nvgpu.arch.copy.SM100.tmem_load", "cute_nvgpu.arch.copy.SM100.tmem_store", + "cute_nvgpu.arch.copy.SM107.tma_load_override", + "cute_nvgpu.arch.copy.SM107.tma_store_override", + "cute_nvgpu.arch.copy.SM90.bulk_copy_g2s", + "cute_nvgpu.arch.copy.SM90.bulk_copy_s2g", + "cute_nvgpu.arch.copy.SM90.bulk_copy_s2s", "cute_nvgpu.arch.get_dyn_smem", "cute_nvgpu.arch.get_dyn_smem_size", "cute_nvgpu.arch.make_warp_uniform", + "cute_nvgpu.arch.mma.SM107.umma", + "cute_nvgpu.arch.mma.SM107.umma_block_scaled", + "cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse", + "cute_nvgpu.arch.mma.SM107.umma_sparse", "cute_nvgpu.arch.mma.SM120.block_scaled", "cute_nvgpu.arch.prefetch_tma_desc", "cute_nvgpu.arch.sm100.alloc_tmem", @@ -1958,26 +5103,31 @@ pub const operation_names: []const []const u8 = &.{ "cute_nvgpu.atom.get_value", "cute_nvgpu.atom.make_exec_tma", "cute_nvgpu.atom.make_non_exec_2d_gather4_tma_load", + "cute_nvgpu.atom.make_non_exec_2d_scatter4_tma_store", "cute_nvgpu.atom.make_non_exec_im2col_tma_load", "cute_nvgpu.atom.make_non_exec_im2col_tma_store", "cute_nvgpu.atom.make_non_exec_tiled_tma_load", "cute_nvgpu.atom.make_non_exec_tiled_tma_reduce", "cute_nvgpu.atom.make_non_exec_tiled_tma_store", "cute_nvgpu.atom.make_s2t_copy", + "cute_nvgpu.atom.make_tma_residue_tensor", "cute_nvgpu.atom.make_tmem_copy", "cute_nvgpu.atom.set_value", + "cute_nvgpu.atom.sm107.get_copy_s2t_smem_desc_view", "cute_nvgpu.atom.tma_partition", "cute_nvgpu.cast_tma_desc_to_integer", "cute_nvgpu.copy_tma_desc", "cute_nvgpu.get_grid_constant_pointer", "cute_nvgpu.get_tma_desc_addr", "cute_nvgpu.make_gmma_smem_desc", - "cute_nvgpu.make_tma_desc_im2col_at", "cute_nvgpu.make_tma_desc_im2col", - "cute_nvgpu.make_tma_desc_tiled_at", + "cute_nvgpu.make_tma_desc_im2col_at", "cute_nvgpu.make_tma_desc_tiled", + "cute_nvgpu.make_tma_desc_tiled_at", "cute_nvgpu.make_umma_smem_desc", "cute_nvgpu.prefetch_tma_desc", "cute_nvgpu.sm103.make_umma_smem_desc_circular", + "cute_nvgpu.sm107.make_umma_smem_desc", + "cute_nvgpu.sm107.make_umma_smem_desc_circular", "cute_nvgpu.update_tma_desc", }; diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/BUILD.bazel b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/BUILD.bazel index 02869c71..3833369f 100644 --- a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/BUILD.bazel +++ b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/BUILD.bazel @@ -7,6 +7,7 @@ cc_library( hdrs = [ "Cute.h", "CuteAttributes.h", + "CuteCompilerAttributes.h", "CuteTypes.h", ], includes = ["../.."], @@ -17,6 +18,7 @@ cc_library( name = "cute_nvgpu", hdrs = [ "CuteNVGPU.h", + "CuteNVGPUAttributes.h", "CuteNVGPUTypes.h", ], includes = ["../.."], diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/Cute.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/Cute.h index bc2f11a1..dd896824 100644 --- a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/Cute.h +++ b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/Cute.h @@ -4,6 +4,7 @@ #define CUTE_IR_C_DIALECT_CUTE_H #include "cute_ir-c/Dialect/CuteAttributes.h" +#include "cute_ir-c/Dialect/CuteCompilerAttributes.h" #include "cute_ir-c/Dialect/CuteTypes.h" #include "mlir-c/IR.h" diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteCompilerAttributes.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteCompilerAttributes.h new file mode 100644 index 00000000..f3b4375a --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteCompilerAttributes.h @@ -0,0 +1,225 @@ +// C API for the CuTe attributes beyond the algebra, generated from the +// dialect's .td files. +#ifndef CUTE_IR_C_DIALECT_CUTE_COMPILER_ATTRIBUTES_H +#define CUTE_IR_C_DIALECT_CUTE_COMPILER_ATTRIBUTES_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles, and std::optional enums a +// negative value. Enums are their integer values (the dialect's *Enums.td). +// Getters require a value accepted by the matching IsA. + +// `#cute.bitlayout`: Bit layout of a pointer's elements within a storage chunk. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteBitLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteBitLayoutAttrGet(MlirContext context, + MlirAttribute layout); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteBitLayoutAttrGetLayout(MlirAttribute attr); + +// `#cute.copy_atom`: Thread and value layouts of a copy atom. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteCopyAtom(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCopyAtomAttrGet( + MlirContext context, MlirAttribute thrId, MlirAttribute layoutSrc, + MlirAttribute layoutDst, MlirAttribute layoutRef); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomAttrGetThrId(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomAttrGetLayoutSrc(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomAttrGetLayoutDst(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomAttrGetLayoutRef(MlirAttribute attr); + +// `#cute.copy_atom_v2`: Thread and value layouts and fragments of a V2 copy +// atom. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteCopyAtomV2(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCopyAtomV2AttrGet( + MlirContext context, MlirAttribute thrId, MlirAttribute layoutSrcTV, + MlirAttribute layoutDstTV, MlirAttribute frgSrc, MlirAttribute frgDst); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomV2AttrGetThrId(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomV2AttrGetLayoutSrcTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomV2AttrGetLayoutDstTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomV2AttrGetFrgSrc(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomV2AttrGetFrgDst(MlirAttribute attr); + +// `#cute.mma_atom`: Shapes, thread-value layouts and fragments of an MMA atom. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteMmaAtom(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutBTV, MlirAttribute layoutCTV, + MlirAttribute shapeAMK, MlirAttribute shapeBNK, MlirAttribute shapeCMN, + MlirAttribute frgA, MlirAttribute frgB, MlirAttribute frgC); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetShapeMNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetThrId(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetLayoutATV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetLayoutBTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetLayoutCTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetShapeAMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetShapeBNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetShapeCMN(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMmaAtomAttrGetFrgA(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMmaAtomAttrGetFrgB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMmaAtomAttrGetFrgC(MlirAttribute attr); + +// `#cute.mx_mma_atom`: MMA atom attribute with scale-factor operands. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteMxMmaAtom(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMxMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutSFATV, MlirAttribute layoutBTV, + MlirAttribute layoutSFBTV, MlirAttribute layoutCTV, MlirAttribute shapeAMK, + MlirAttribute shapeSFAMK, MlirAttribute shapeBNK, MlirAttribute shapeSFBNK, + MlirAttribute shapeCMN, MlirAttribute frgA, MlirAttribute frgSFA, + MlirAttribute frgB, MlirAttribute frgSFB, MlirAttribute frgC); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetShapeMNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetThrId(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetLayoutATV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetLayoutSFATV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetLayoutBTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetLayoutSFBTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetLayoutCTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetShapeAMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetShapeSFAMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetShapeBNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetShapeSFBNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetShapeCMN(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetFrgA(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetFrgSFA(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetFrgB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetFrgSFB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetFrgC(MlirAttribute attr); + +// `#cute.reduction_op`: Op for cute reduce operations. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteReductionOp(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteReductionOpAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteReductionOpAttrGetValue(MlirAttribute attr); + +// `#cute.sparse_mma_atom`: MMA atom attribute with a sparsity-metadata operand. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteSparseMmaAtom(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteSparseMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutBTV, MlirAttribute layoutCTV, + MlirAttribute layoutETV, MlirAttribute shapeAMK, MlirAttribute shapeBNK, + MlirAttribute shapeCMN, MlirAttribute shapeEMK, MlirAttribute frgA, + MlirAttribute frgB, MlirAttribute frgC, MlirAttribute frgE); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetShapeMNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetThrId(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetLayoutATV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetLayoutBTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetLayoutCTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetLayoutETV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetShapeAMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetShapeBNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetShapeCMN(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetShapeEMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetFrgA(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetFrgB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetFrgC(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetFrgE(MlirAttribute attr); + +// `#cute.sparse_mx_mma_atom`: MMA atom attribute with scale-factor and metadata +// operands. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteSparseMxMmaAtom(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteSparseMxMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutSFATV, MlirAttribute layoutBTV, + MlirAttribute layoutSFBTV, MlirAttribute layoutCTV, MlirAttribute layoutETV, + MlirAttribute shapeAMK, MlirAttribute shapeSFAMK, MlirAttribute shapeBNK, + MlirAttribute shapeSFBNK, MlirAttribute shapeCMN, MlirAttribute shapeEMK, + MlirAttribute frgA, MlirAttribute frgSFA, MlirAttribute frgB, + MlirAttribute frgSFB, MlirAttribute frgC, MlirAttribute frgE); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeMNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetThrId(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetLayoutATV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetLayoutSFATV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetLayoutBTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetLayoutSFBTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetLayoutCTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetLayoutETV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeAMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeSFAMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeBNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeSFBNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeCMN(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeEMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetFrgA(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetFrgSFA(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetFrgB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetFrgSFB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetFrgC(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetFrgE(MlirAttribute attr); + +#ifdef __cplusplus +} +#endif + +#endif // CUTE_IR_C_DIALECT_CUTE_COMPILER_ATTRIBUTES_H diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPU.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPU.h index 6e883f3d..175435f5 100644 --- a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPU.h +++ b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPU.h @@ -3,6 +3,7 @@ #ifndef CUTE_IR_C_DIALECT_CUTENVGPU_H #define CUTE_IR_C_DIALECT_CUTENVGPU_H +#include "cute_ir-c/Dialect/CuteNVGPUAttributes.h" #include "cute_ir-c/Dialect/CuteNVGPUTypes.h" #include "mlir-c/IR.h" diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPUAttributes.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPUAttributes.h new file mode 100644 index 00000000..c382d546 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPUAttributes.h @@ -0,0 +1,480 @@ +// C API for the CuTe NVIDIA GPU attributes, generated from the dialect's .td +// files. +#ifndef CUTE_IR_C_DIALECT_CUTENVGPU_ATTRIBUTES_H +#define CUTE_IR_C_DIALECT_CUTENVGPU_ATTRIBUTES_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles, and std::optional enums a +// negative value. Enums are their integer values (the dialect's *Enums.td). +// Getters require a value accepted by the matching IsA. + +// `#cute_nvgpu.atom_copy_field_bulkg2s`: Fields stored in Bulk Load Copy Atom +// type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyG2S(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_bulks2g`: Fields stored in Bulk Store Copy Atom +// type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2G(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_bulks2s`: Fields stored in Bulk CTA to Cluster +// Copy Atom type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2S(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_dsmem_store`: Fields stored in +// CopyAtomDsmemStoreType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldDsmemStore(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_g2r`: Fields stored in CopyAtomG2RType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldLoadGlobal(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_2d_gather4_tma_load`: Fields stored in +// CopyAtomNonExec2DGather4TmaLoadType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoad( + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGetValue( + MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_2d_scatter4_tma_store`: Fields stored +// in CopyAtomNonExec2DScatter4TmaStoreType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStore( + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGetValue( + MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_im2col_tma_load`: Fields stored in +// CopyAtomNonExecIm2ColTmaLoadType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoad(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_im2col_tma_store`: Fields stored in +// CopyAtomNonExecIm2ColTmaStoreType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaStore(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_tma_load`: Fields stored in +// CopyAtomNonExecTiledTmaLoadType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaLoad(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_tma_reduce`: Fields stored in +// CopyAtomNonExecTiledTmaReduceType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaReduce(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_tma_store`: Fields stored in +// CopyAtomNonExecTiledTmaStoreType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaStore(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_r2g`: Fields stored in CopyAtomR2GType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldStoreGlobal(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_tmaload`: Fields stored in Tma Load Copy Atom +// type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldTmaLoad(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_tmareduce`: Fields stored in Tma Reduce Copy +// Atom type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldTmaReduce(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_tmastore`: Fields stored in Tma Store Copy Atom +// type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldTmaStore(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm100`: Fields stored in MMA Atom types for +// SM100. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM100(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM100AttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM100AttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm100_block_scaled`: Fields stored in MMA Atom +// types for SM100 (block scaled). +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaled(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm100_block_scaled_sparse`: Fields stored in MMA +// Atom types for SM100 (block scaled). +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaledSparse(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm100_sparse`: Fields stored in sparse MMA Atom +// types for SM100. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM100Sparse(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm120_block_scaled`: Fields stored in MMA Atom +// types for SM120 (block scaled). +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM120BlockScaled(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm80_sparse`: Fields stored in MMA Atom types for +// SM80. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM80Sparse(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm90`: Fields stored in MMA Atom types for SM90. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM90(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM90AttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM90AttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.bin_op`: Binary operation for single-bit MMA operations. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVGPUBinaryOp(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUBinaryOpAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUBinaryOpAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.copy_s2t_broadcast_mode`: Broadcast modes for the different +// utccp instructions. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUCopyS2TBroadcast(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyS2TBroadcastAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyS2TBroadcastAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.gather_scatter_tma_load`: The various kinds of TMA loads in +// gather/scatter mode. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUGatherScatterTmaLoad(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUGatherScatterTmaLoadAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUGatherScatterTmaLoadAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.im2col_tma_load`: The various kinds of TMA loads in im2col mode. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUIm2ColTmaLoad(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUIm2ColTmaLoadAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUIm2ColTmaLoadAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.ld_reduce_acc_precision_kind`: multimem ld_reduce accumulation +// precision kind. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPULdReduceAccPrecisionKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPULdReduceAccPrecisionKindAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPULdReduceAccPrecisionKindAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.ldsm_sz_pattern`: LDSM's sz pattern, describing the bit size. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPULdsmSzPattern(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPULdsmSzPatternAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPULdsmSzPatternAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.load_cache_mode`: Cache modes for the load instructions. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPULoadCacheMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPULoadCacheModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPULoadCacheModeAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.mma_int_overflow`: MMA overflow options. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUMMAIntOverflow(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMMAIntOverflowAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMMAIntOverflowAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.major`: Major mode for MMA operations. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVGPUMajorMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMajorModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMajorModeAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.mma_collector_op`: Enums for the mma collector op. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUMmaCollectorOp(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaCollectorOpAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaCollectorOpAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.mma_frag_kind`: Enums for the mma frag type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUMmaFragKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaFragKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaFragKindAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.not_implemented_frg`: Fragment of an atom that has none. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUNotImplementedFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUNotImplementedFrgAttrGet(MlirContext context); + +// `#cute_nvgpu.tma_reduce_kind`: Op for the TMASTORE instruction. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUReductionKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUReductionKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUReductionKindAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.rmem_frg`: Register-memory fragment of an MMA operand. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVGPURmemFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPURmemFrgAttrGet(MlirContext context, MlirType valueType, + uint32_t operand, bool deriveElemType); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPURmemFrgAttrGetValueType(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPURmemFrgAttrGetOperand(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPURmemFrgAttrGetDeriveElemType(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm100.circular_smem_frg`: SM100 circular shared-memory +// fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM100CircularSmemFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUSM100CircularSmemFrgAttrGet( + MlirContext context, uint32_t majorMode); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUSM100CircularSmemFrgAttrGetMajorMode(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm100.smem_frg`: SM100 shared-memory fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM100SmemFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUSM100SmemFrgAttrGet(MlirContext context, uint32_t majorMode); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUSM100SmemFrgAttrGetMajorMode(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm100.tmem_e_frg`: SM100 tensor-memory sparse-metadata +// fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM100TmemEFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUSM100TmemEFrgAttrGet( + MlirContext context, MlirType aType, MlirType eType); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUSM100TmemEFrgAttrGetAType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUSM100TmemEFrgAttrGetEType(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm100.tmem_frg`: SM100 tensor-memory fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM100TmemFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUSM100TmemFrgAttrGet( + MlirContext context, MlirType dataType, MlirType storageType, int ctaGroup, + uint32_t tmemAllocMode); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUSM100TmemFrgAttrGetDataType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUSM100TmemFrgAttrGetStorageType(MlirAttribute attr); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUSM100TmemFrgAttrGetCtaGroup(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUSM100TmemFrgAttrGetTmemAllocMode(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm100.tmem_sf_frg`: SM100 tensor-memory scale-factor +// fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM100TmemSfFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUSM100TmemSfFrgAttrGet( + MlirContext context, MlirType sfType, int sfVecSize, int ctaGroup, + bool isSfa, uint32_t tmemAllocMode); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUSM100TmemSfFrgAttrGetSfType(MlirAttribute attr); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUSM100TmemSfFrgAttrGetSfVecSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUSM100TmemSfFrgAttrGetCtaGroup(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUSM100TmemSfFrgAttrGetIsSfa(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUSM100TmemSfFrgAttrGetTmemAllocMode(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm107.smem_frg`: SM107 shared-memory fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM107SmemFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUSM107SmemFrgAttrGet(MlirContext context, uint32_t majorMode); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUSM107SmemFrgAttrGetMajorMode(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm90.smem_frg`: SM90 shared-memory fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM90SmemFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUSM90SmemFrgAttrGet(MlirContext context, uint32_t majorMode); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUSM90SmemFrgAttrGetMajorMode(MlirAttribute attr); + +// `#cute_nvgpu.tiled_tma_load`: The various kinds of TMA loads in tiled mode. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUTiledTmaLoad(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTiledTmaLoadAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUTiledTmaLoadAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.tma_data_format`: Bits 74-71 of the TMA descriptor. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUTmaDataFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTmaDataFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUTmaDataFormatAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.tma_load_mode`: Modes for the TMA load instruction. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUTmaLoadMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTmaLoadModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUTmaLoadModeAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.tma_store_mode`: Modes for the TMA store instruction. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUTmaStoreMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTmaStoreModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUTmaStoreModeAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.tmem_alloc_mode`: Modes for the Tmem allocation. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUTmemAllocMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTmemAllocModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUTmemAllocModeAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.tmem_load_red_op`: Reduce operation for the Tmem load +// instruction. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUTmemLoadRedOp(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTmemLoadRedOpAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUTmemLoadRedOpAttrGetValue(MlirAttribute attr); + +#ifdef __cplusplus +} +#endif + +#endif // CUTE_IR_C_DIALECT_CUTENVGPU_ATTRIBUTES_H diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPUTypes.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPUTypes.h index 44c26840..f4060a1a 100644 --- a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPUTypes.h +++ b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPUTypes.h @@ -1,317 +1,1000 @@ -// C API for CuTe NVIDIA GPU types. +// C API for the CuTe NVIDIA GPU types, generated from the dialect's .td files. #ifndef CUTE_IR_C_DIALECT_CUTENVGPU_TYPES_H #define CUTE_IR_C_DIALECT_CUTENVGPU_TYPES_H + #include "mlir-c/IR.h" + #ifdef __cplusplus extern "C" { #endif -// Constructors return a null type for invalid parameters. All handles must -// belong to context. Getters require a type accepted by the matching IsA. +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles, and std::optional enums a +// negative value. Enums are their integer values (the dialect's *Enums.td). +// Getters require a value accepted by the matching IsA. +// `!cute_nvgpu.atom.bulk_copy_g2s`: Bulk (TMA non-tensor) copy atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyG2S(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomBulkCopyG2STypeGet( - MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetPayload(MlirType type); + MlirContext context, MlirType valType, int copyBits, bool mcast); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetMcast(MlirType type); +// `!cute_nvgpu.atom.bulk_copy_s2g`: Bulk (TMA non-tensor) copy atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2G(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGet( - MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetPayload(MlirType type); + MlirContext context, MlirType valType, int copyBits, bool mask); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetMask(MlirType type); +// `!cute_nvgpu.atom.bulk_copy_s2s`: Bulk (TMA non-tensor) copy atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2S(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomBulkCopyS2STypeGet( - MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetPayload(MlirType type); + MlirContext context, MlirType valType, int copyBits); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetCopyBits(MlirType type); +// `!cute_nvgpu.atom.dsmem_store`: Distributed shared-memory store atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomDsmemStore(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomDsmemStoreTypeGet( - MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomDsmemStoreTypeGetPayload(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomG2R(MlirType type); + MlirContext context, MlirType valType, int copyBits); MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomG2RTypeGet(MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomG2RTypeGetPayload(MlirType type); +mlirCuteNVGPUCopyAtomDsmemStoreTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomDsmemStoreTypeGetCopyBits(MlirType type); +// `!cute_nvgpu.atom.g2r`: Global-to-register load atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomG2R(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomG2RTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t memOrder, + uint32_t memScope, uint32_t l2PrefetchSize, uint32_t l1CacheEvictPriority, + uint32_t loadCacheMode, uint32_t sharedSpace, bool invariant); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomG2RTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomG2RTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomG2RTypeGetMemOrder(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomG2RTypeGetMemScope(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomG2RTypeGetL2PrefetchSize(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomG2RTypeGetL1CacheEvictPriority(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomG2RTypeGetLoadCacheMode(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomG2RTypeGetSharedSpace(MlirType type); +MLIR_CAPI_EXPORTED bool mlirCuteNVGPUCopyAtomG2RTypeGetInvariant(MlirType type); + +// `!cute_nvgpu.atom.im2col_tma_load`: Executable im2col TMA load atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaLoad(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGet( - MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetPayload(MlirType type); + MlirContext context, MlirType valType, int copyBits, int numCta, + MlirType gStride, bool mcast, MlirType tmaGbasis); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetGStride(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetMcast(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetTmaGbasis(MlirType type); +// `!cute_nvgpu.atom.im2col_tma_store`: Executable im2col TMA store atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaStore(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGet( - MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetPayload(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomLdsm(MlirType type); + MlirContext context, MlirType valType, int copyBits, MlirType gStride, + MlirType tmaGbasis); MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomLdsmTypeGet(MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomLdsmTypeGetPayload(MlirType type); +mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetGStride(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetTmaGbasis(MlirType type); +// `!cute_nvgpu.atom.ldsm`: ldmatrix atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomLdsm(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomLdsmTypeGet( + MlirContext context, MlirType valType, MlirAttribute mode, + uint32_t szPattern, int numMatrices, bool transpose); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomLdsmTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomLdsmTypeGetMode(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomLdsmTypeGetSzPattern(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomLdsmTypeGetNumMatrices(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomLdsmTypeGetTranspose(MlirType type); + +// `!cute_nvgpu.atom.non_exec_2d_gather4_tma_load`: Non-executable 2-D gather4 +// TMA load atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomNonExec2DGather4TmaLoad(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGet( - MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetPayload(MlirType type); + MlirContext context, uint32_t kind, MlirType valType, int copyBits, + MlirType tmaGbasis, int64_t tmaFormat); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetKind(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED int64_t +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaFormat(MlirType type); +// `!cute_nvgpu.atom.non_exec_2d_scatter4_tma_store`: Non-executable 2-D +// scatter4 TMA store atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomNonExec2DScatter4TmaStore(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGet(MlirContext context, + MlirType valType, + int copyBits, + MlirType tmaGbasis, + int64_t tmaFormat); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED int64_t +mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaFormat(MlirType type); + +// `!cute_nvgpu.atom.non_exec_im2col_tma_load`: Non-executable im2col TMA load +// atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaLoad(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGet( - MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetPayload(MlirType type); + MlirContext context, uint32_t kind, MlirType valType, int copyBits, + MlirType tmaGbasis, int64_t tmaFormat); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetKind(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED int64_t +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaFormat(MlirType type); +// `!cute_nvgpu.atom.non_exec_im2col_tma_store`: Non-executable im2col TMA store +// atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaStore(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGet( - MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetPayload(MlirType type); + MlirContext context, MlirType valType, int copyBits, MlirType tmaGbasis, + int64_t tmaFormat); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED int64_t +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaFormat(MlirType type); +// `!cute_nvgpu.atom.non_exec_tiled_tma_load`: Non-executable tiled TMA load +// atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaLoad(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGet( - MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetPayload(MlirType type); + MlirContext context, uint32_t kind, MlirType valType, int copyBits, + MlirType tmaGbasis, int64_t tmaFormat); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetKind(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED int64_t +mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaFormat(MlirType type); +// `!cute_nvgpu.atom.non_exec_tiled_tma_reduce`: Non-executable tiled TMA reduce +// atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaReduce(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGet( - MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetPayload(MlirType type); + MlirContext context, uint32_t kind, MlirType valType, int copyBits, + MlirType tmaGbasis, int64_t tmaFormat); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetKind(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED int64_t +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaFormat(MlirType type); +// `!cute_nvgpu.atom.non_exec_tiled_tma_store`: Non-executable tiled TMA store +// atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaStore(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGet( - MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetPayload(MlirType type); + MlirContext context, MlirType valType, int copyBits, MlirType tmaGbasis, + int64_t tmaFormat); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED int64_t +mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaFormat(MlirType type); +// `!cute_nvgpu.atom.r2g`: Register-to-global store atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomR2G(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomR2GTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t memOrder, + uint32_t memScope, uint32_t l1CacheEvictPriority, uint32_t storeCacheMode, + uint32_t sharedSpace); MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomR2GTypeGet(MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomR2GTypeGetPayload(MlirType type); +mlirCuteNVGPUCopyAtomR2GTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomR2GTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomR2GTypeGetMemOrder(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomR2GTypeGetMemScope(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomR2GTypeGetL1CacheEvictPriority(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomR2GTypeGetStoreCacheMode(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomR2GTypeGetSharedSpace(MlirType type); +// `!cute_nvgpu.atom.r2s`: Register-to-shared store atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomR2S(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomR2STypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t memOrder, + uint32_t memScope, uint32_t sharedSpace); MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomR2STypeGet(MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomR2STypeGetPayload(MlirType type); +mlirCuteNVGPUCopyAtomR2STypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomR2STypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomR2STypeGetMemOrder(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomR2STypeGetMemScope(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomR2STypeGetSharedSpace(MlirType type); +// `!cute_nvgpu.atom.s2r`: Shared-to-register load atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomS2R(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomS2RTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t memOrder, + uint32_t memScope, uint32_t sharedSpace); MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomS2RTypeGet(MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomS2RTypeGetPayload(MlirType type); +mlirCuteNVGPUCopyAtomS2RTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomS2RTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomS2RTypeGetMemOrder(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomS2RTypeGetMemScope(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomS2RTypeGetSharedSpace(MlirType type); +// `!cute_nvgpu.atom.simt_async_copy`: cp.async copy atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomSIMTAsyncCopy(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGet( - MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetPayload(MlirType type); + MlirContext context, MlirType valType, uint32_t cache, int copyBits); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCache(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCopyBits(MlirType type); +// `!cute_nvgpu.atom.simt_multimem_ld_reduce`: multimem.ld_reduce atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemLdReduce(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGet( - MlirContext context, MlirAttribute payload); + MlirContext context, MlirType valType, int copyBits, uint32_t reduction, + MlirAttribute ldReduceAccPrecision, uint32_t memOrder, uint32_t memScope); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetReduction(MlirType type); MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetPayload(MlirType type); +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetLdReduceAccPrecision( + MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemOrder(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemScope(MlirType type); +// `!cute_nvgpu.atom.simt_multimem_red`: multimem.red atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemRed(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGet( - MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetPayload(MlirType type); + MlirContext context, MlirType valType, int copyBits, uint32_t reduction, + uint32_t memOrder, uint32_t memScope); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetReduction(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemOrder(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemScope(MlirType type); +// `!cute_nvgpu.atom.simt_multimem_st`: multimem.st atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemSt(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGet( - MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetPayload(MlirType type); + MlirContext context, MlirType valType, int copyBits, uint32_t memOrder, + uint32_t memScope); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemOrder(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemScope(MlirType type); +// `!cute_nvgpu.atom.universal_copy`: Universal (SIMT) copy atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomSIMTSyncCopy(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGet( - MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetPayload(MlirType type); - -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUCopyAtomSM10xTmemLoadRed(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet( - MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetPayload(MlirType type); + MlirContext context, MlirType valType, int copyBits, uint32_t srcSpace, + uint32_t dstSpace); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetSrcSpace(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetDstSpace(MlirType type); +// `!cute_nvgpu.atom.s2t_copy`: tcgen05.cp (shared to tensor memory) atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomSM100CopyS2T(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGet( - MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetPayload(MlirType type); + MlirContext context, MlirType valType, int numDp, int numBit, int numCta, + uint32_t broadcast); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumDp(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumBit(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetBroadcast(MlirType type); +// `!cute_nvgpu.atom.sm100_s2t_copy_v2`: tcgen05.cp atom with the shared-memory +// major mode. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomSM100S2TCopyV2(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGet( - MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetPayload(MlirType type); + MlirContext context, MlirType valType, int numDp, int numBit, int numCta, + uint32_t smemMajor, uint32_t broadcast); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumDp(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumBit(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetSmemMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetBroadcast(MlirType type); +// `!cute_nvgpu.atom.tmem_load`: tcgen05.ld atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomSM100TmemLoad(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGet( - MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetPayload(MlirType type); + MlirContext context, MlirType valType, int numDp, int numBit, + unsigned numRep, bool pack16b); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumDp(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumBit(MlirType type); +MLIR_CAPI_EXPORTED unsigned +mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumRep(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetPack16b(MlirType type); +// `!cute_nvgpu.atom.tmem_store`: tcgen05.st atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomSM100TmemStore(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGet( - MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetPayload(MlirType type); + MlirContext context, MlirType valType, int numDp, int numBit, + unsigned numRep, bool expand16b); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumDp(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumBit(MlirType type); +MLIR_CAPI_EXPORTED unsigned +mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumRep(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetExpand16b(MlirType type); +// `!cute_nvgpu.atom.tmem_load_spcompress`: SM107 tensor-memory load with sparse +// compression. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSM107TmemLoadSPCompress(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGet( + MlirContext context, MlirType valType, int numDp, int numBit, + unsigned numRep, uint32_t redOp, bool red, bool nan); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumDp(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumBit(MlirType type); +MLIR_CAPI_EXPORTED unsigned +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumRep(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRedOp(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRed(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNan(MlirType type); + +// `!cute_nvgpu.atom.tmem_load_red`: tcgen05.ld.red atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSM10xTmemLoadRed(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet( + MlirContext context, MlirType valType, int numDp, int numBit, + unsigned numRep, uint32_t redOp, bool nan, MlirAttribute halfSplitOff); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumDp(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumBit(MlirType type); +MLIR_CAPI_EXPORTED unsigned +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumRep(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetRedOp(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNan(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetHalfSplitOff(MlirType type); + +// `!cute_nvgpu.atom.stsm`: stmatrix atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomStsm(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomStsmTypeGet( + MlirContext context, MlirType valType, MlirAttribute mode, int numMatrices, + bool transpose); MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomStsmTypeGet(MlirContext context, MlirAttribute payload); +mlirCuteNVGPUCopyAtomStsmTypeGetValType(MlirType type); MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomStsmTypeGetPayload(MlirType type); +mlirCuteNVGPUCopyAtomStsmTypeGetMode(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomStsmTypeGetNumMatrices(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomStsmTypeGetTranspose(MlirType type); +// `!cute_nvgpu.atom.tma_load`: Executable TMA load atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomTmaLoad(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomTmaLoadTypeGet( + MlirContext context, MlirType valType, int sparsity, int copyBits, + uint32_t mode, int numCta, MlirType gStride, bool mcast, MlirType tmaGbasis, + bool override, bool noFullyOobTile); MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomTmaLoadTypeGet(MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomTmaLoadTypeGetPayload(MlirType type); +mlirCuteNVGPUCopyAtomTmaLoadTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomTmaLoadTypeGetSparsity(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomTmaLoadTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomTmaLoadTypeGetMode(MlirType type); +MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomTmaLoadTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomTmaLoadTypeGetGStride(MlirType type); +MLIR_CAPI_EXPORTED bool mlirCuteNVGPUCopyAtomTmaLoadTypeGetMcast(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomTmaLoadTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomTmaLoadTypeGetOverride(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomTmaLoadTypeGetNoFullyOobTile(MlirType type); +// `!cute_nvgpu.atom.tma_reduce`: Executable TMA reduce atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomTmaReduce(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomTmaReduceTypeGet( - MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomTmaReduceTypeGetPayload(MlirType type); + MlirContext context, MlirType valType, int copyBits, uint32_t mode, + uint32_t kind, MlirType gStride, MlirType tmaGbasis); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomTmaReduceTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomTmaReduceTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomTmaReduceTypeGetMode(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomTmaReduceTypeGetKind(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomTmaReduceTypeGetGStride(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomTmaReduceTypeGetTmaGbasis(MlirType type); +// `!cute_nvgpu.atom.tma_store`: Executable TMA store atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomTmaStore(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomTmaStoreTypeGet( - MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomTmaStoreTypeGetPayload(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM80Sparse(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM80SparseTypeGet( - MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM80SparseTypeGetPayload(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM80(MlirType type); + MlirContext context, MlirType valType, int sparsity, int copyBits, + uint32_t mode, MlirType gStride, MlirType tmaGbasis, bool override, + bool noFullyOobTile); MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM80TypeGet(MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM80TypeGetPayload(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM89(MlirType type); +mlirCuteNVGPUCopyAtomTmaStoreTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomTmaStoreTypeGetSparsity(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomTmaStoreTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomTmaStoreTypeGetMode(MlirType type); MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM89TypeGet(MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM89TypeGetPayload(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM90(MlirType type); +mlirCuteNVGPUCopyAtomTmaStoreTypeGetGStride(MlirType type); MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM90TypeGet(MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM90TypeGetPayload(MlirType type); +mlirCuteNVGPUCopyAtomTmaStoreTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomTmaStoreTypeGetOverride(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomTmaStoreTypeGetNoFullyOobTile(MlirType type); +// `!cute_nvgpu.sm100.mma_bs_sp`: SM100 block-scaled sparse UMMA MMA atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaledSparse(MlirType type); MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGet(MlirContext context, - MlirAttribute payload); +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType sfType, uint32_t sparseMetadataFormat, uint32_t aFragKind, + int vecSize, uint32_t archPromote, MlirAttribute intOverflow); MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetPayload(MlirType type); +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSfType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSparseMetadataFormat( + MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetVecSize(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetArchPromote(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetIntOverflow(MlirType type); +// `!cute_nvgpu.sm100.mma_bs`: SM100 block-scaled UMMA MMA atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaled(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGet( - MlirContext context, MlirAttribute payload); + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType sfType, uint32_t aFragKind, int vecSize, uint32_t archPromote, + MlirAttribute intOverflow); MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetPayload(MlirType type); +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetSfType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetVecSize(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetArchPromote(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetIntOverflow(MlirType type); +// `!cute_nvgpu.sm100.mma_sp`: SM100 sparse UMMA MMA atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMASparse(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGet( - MlirContext context, MlirAttribute payload); + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType eType, uint32_t sparseMetadataFormat, uint32_t aFragKind, + int cScaleExp, MlirAttribute intOverflow); MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetPayload(MlirType type); +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetEType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetSparseMetadataFormat(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCScaleExp(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetIntOverflow(MlirType type); +// `!cute_nvgpu.sm100.mma`: SM100 UMMA MMA atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMA(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM100UMMATypeGet( - MlirContext context, MlirAttribute payload); + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + uint32_t aFragKind, int cScaleExp, MlirAttribute intOverflow); MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM100UMMATypeGetPayload(MlirType type); +mlirCuteNVGPUMmaAtomSM100UMMATypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM100UMMATypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMATypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMATypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMATypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMATypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMATypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMATypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM100UMMATypeGetCScaleExp(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMATypeGetIntOverflow(MlirType type); +// `!cute_nvgpu.sm107.mma_bs_sp`: SM107 block-scaled sparse UMMA MMA atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaledSparse(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType sfType, uint32_t sparseMetadataFormat, uint32_t aFragKind, + int vecSize, uint32_t aCollectorOp, uint32_t bCollectorOp, + MlirAttribute intOverflow); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSfType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSparseMetadataFormat( + MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetVecSize(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetACollectorOp( + MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBCollectorOp( + MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetIntOverflow(MlirType type); + +// `!cute_nvgpu.sm107.mma_bs`: SM107 block-scaled UMMA MMA atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaled(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType sfType, uint32_t aFragKind, int vecSize, uint32_t aCollectorOp, + uint32_t bCollectorOp, MlirAttribute intOverflow); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetSfType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetVecSize(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetACollectorOp(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBCollectorOp(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetIntOverflow(MlirType type); + +// `!cute_nvgpu.sm107.mma_sp`: SM107 sparse UMMA MMA atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUMmaAtomSM107UMMASparse(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType eType, uint32_t sparseMetadataFormat, uint32_t aFragKind, + int cScaleExp, uint32_t aCollectorOp, uint32_t bCollectorOp, + MlirAttribute intOverflow); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetEType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetSparseMetadataFormat(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCScaleExp(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetACollectorOp(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBCollectorOp(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetIntOverflow(MlirType type); + +// `!cute_nvgpu.sm107.mma`: SM107 UMMA MMA atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM107UMMA(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM107UMMATypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + uint32_t aFragKind, int cScaleExp, uint32_t aCollectorOp, + uint32_t bCollectorOp, MlirAttribute intOverflow); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMATypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM107UMMATypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMATypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMATypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMATypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMATypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMATypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMATypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM107UMMATypeGetCScaleExp(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMATypeGetACollectorOp(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMATypeGetBCollectorOp(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMATypeGetIntOverflow(MlirType type); + +// `!cute_nvgpu.SM120.mma_bs`: SM120 block-scaled mma.sync MMA atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM120BlockScaled(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGet( - MlirContext context, MlirAttribute payload); + MlirContext context, MlirAttribute shapeMnk, int vecSize, MlirType aType, + MlirType bType, MlirType cType, MlirType sfType, bool useSfLayoutTV); MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetPayload(MlirType type); +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetVecSize(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetSfType(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetUseSfLayoutTV(MlirType type); +// `!cute_nvgpu.sm80.sparse_mma`: SM80 sparse mma.sync MMA atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM80Sparse(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM80SparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, MlirType aType, MlirType bType, + MlirType cType, uint32_t sparseMetadataFormat, MlirAttribute intOverflow); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM80SparseTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM80SparseTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM80SparseTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM80SparseTypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM80SparseTypeGetSparseMetadataFormat(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM80SparseTypeGetIntOverflow(MlirType type); + +// `!cute_nvgpu.sm80.mma`: SM80 mma.sync MMA atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM80(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM80TypeGet( + MlirContext context, MlirAttribute shapeMnk, MlirType aType, MlirType bType, + MlirType cType, MlirAttribute intOverflow, MlirAttribute binaryOp); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM80TypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM80TypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM80TypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM80TypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM80TypeGetIntOverflow(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM80TypeGetBinaryOp(MlirType type); + +// `!cute_nvgpu.sm89.mma`: SM89 fp8 mma.sync MMA atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM89(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM89TypeGet(MlirContext context, MlirAttribute shapeMnk, + MlirType aType, MlirType bType, MlirType cType); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM89TypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM89TypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM89TypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM89TypeGetCType(MlirType type); + +// `!cute_nvgpu.sm90.mma`: SM90 wgmma MMA atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM90(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM90TypeGet( + MlirContext context, MlirAttribute shapeMnk, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + uint32_t aFragKind, MlirAttribute intOverflow, bool aNeg, bool bNeg); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM90TypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM90TypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM90TypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM90TypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM90TypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM90TypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM90TypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM90TypeGetIntOverflow(MlirType type); +MLIR_CAPI_EXPORTED bool mlirCuteNVGPUMmaAtomSM90TypeGetANeg(MlirType type); +MLIR_CAPI_EXPORTED bool mlirCuteNVGPUMmaAtomSM90TypeGetBNeg(MlirType type); + +// `!cute_nvgpu.sm103.smem_desc_circular`: Circular shared-memory descriptor +// over a block layout in bytes. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUSmemDescCircularSM103(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUSmemDescCircularSM103TypeGet( - MlirContext context, MlirAttribute payload); + MlirContext context, MlirAttribute blockLayoutBytes); MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUSmemDescCircularSM103TypeGetPayload(MlirType type); +mlirCuteNVGPUSmemDescCircularSM103TypeGetBlockLayoutBytes(MlirType type); +// `!cute_nvgpu.sm107.smem_desc_circular`: Circular shared-memory descriptor +// over a block layout in bytes. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUSmemDescCircularSM107(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUSmemDescCircularSM107TypeGet( + MlirContext context, MlirAttribute blockLayoutBytes); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUSmemDescCircularSM107TypeGetBlockLayoutBytes(MlirType type); + +// `!cute_nvgpu.sm107.smem_desc`: SM107 shared-memory descriptor. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUSmemDescSM107(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUSmemDescSM107TypeGet(MlirContext context); + +// `!cute_nvgpu.smem_desc`: UMMA/GMMA shared-memory descriptor. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUSmemDesc(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUSmemDescTypeGet(MlirContext context); +// `!cute_nvgpu.smem_desc_view`: Shared-memory descriptor iterator with a +// layout. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUSmemDescView(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUSmemDescViewTypeGet(MlirContext context, MlirAttribute payload); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUSmemDescViewTypeGet( + MlirContext context, MlirType desc, MlirAttribute layout); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUSmemDescViewTypeGetDesc(MlirType type); MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUSmemDescViewTypeGetPayload(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUTiledCopy(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUTiledCopyTypeGet(MlirContext context, MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUTiledCopyTypeGetPayload(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUTiledMma(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUTiledMmaTypeGet(MlirContext context, - MlirAttribute payload); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUTiledMmaTypeGetPayload(MlirType type); +mlirCuteNVGPUSmemDescViewTypeGetLayout(MlirType type); +// `!cute_nvgpu.tma_descriptor_im2col`: Im2col TMA descriptor. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUTmaDescriptorIm2Col(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUTmaDescriptorIm2ColTypeGet(MlirContext context); +// `!cute_nvgpu.tma_descriptor_tiled`: Tiled TMA descriptor. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUTmaDescriptorTiled(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUTmaDescriptorTiledTypeGet(MlirContext context); +// `!cute_nvgpu.atom.universal_fma`: Universal FMA MMA atom. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUUniversalFmaAtom(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUUniversalFmaAtomTypeGet( - MlirContext context, MlirAttribute payload); + MlirContext context, MlirAttribute shapeMnk, MlirType aType, MlirType bType, + MlirType cType); MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUUniversalFmaAtomTypeGetPayload(MlirType type); +mlirCuteNVGPUUniversalFmaAtomTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUUniversalFmaAtomTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUUniversalFmaAtomTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUUniversalFmaAtomTypeGetCType(MlirType type); +// `!cute_nvgpu.workid_response`: Cluster launch control work-id response. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUWorkIdResponse(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUWorkIdResponseTypeGet(MlirContext context); diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteTypes.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteTypes.h index 4e22f193..0a4bd93d 100644 --- a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteTypes.h +++ b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteTypes.h @@ -1,38 +1,44 @@ -// C API for CuTe types. +// C API for the CuTe types, generated from the dialect's .td files. #ifndef CUTE_IR_C_DIALECT_CUTE_TYPES_H #define CUTE_IR_C_DIALECT_CUTE_TYPES_H + #include "mlir-c/IR.h" + #ifdef __cplusplus extern "C" { #endif -// Constructors return a null type for invalid parameters. All handles must -// belong to context. Getters require a type accepted by the matching IsA. +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles, and std::optional enums a +// negative value. Enums are their integer values (the dialect's *Enums.td). +// Getters require a value accepted by the matching IsA. +// `!cute.arith_tuple_iter`: Iterator over an arithmetic tuple. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteArithTupleIterator(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteArithTupleIteratorTypeGet(MlirContext context, MlirType arithTuple); MLIR_CAPI_EXPORTED MlirType mlirCuteArithTupleIteratorTypeGetArithTuple(MlirType type); +// `!cute.composed_layout`: CuTe composed layout: A ∘ offset ∘ B. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteComposedLayout(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteComposedLayoutTypeGet(MlirContext context, MlirAttribute attr); MLIR_CAPI_EXPORTED MlirAttribute mlirCuteComposedLayoutTypeGetAttr(MlirType type); -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteConstrainedInt32(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteConstrainedInt32TypeGet(MlirContext context, uint64_t divisibleBy); -MLIR_CAPI_EXPORTED uint64_t -mlirCuteConstrainedInt32TypeGetDivisibleBy(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteConstrainedInt64(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteConstrainedInt64TypeGet(MlirContext context, uint64_t divisibleBy); -MLIR_CAPI_EXPORTED uint64_t -mlirCuteConstrainedInt64TypeGetDivisibleBy(MlirType type); +// `!cute.i`: Integer of any width with known divisibility and power of two. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteConstrainedInt(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteConstrainedIntTypeGet(MlirContext context, + int64_t divisibility, + unsigned width, + bool isPow2); +MLIR_CAPI_EXPORTED int64_t +mlirCuteConstrainedIntTypeGetDivisibility(MlirType type); +MLIR_CAPI_EXPORTED unsigned mlirCuteConstrainedIntTypeGetWidth(MlirType type); +MLIR_CAPI_EXPORTED bool mlirCuteConstrainedIntTypeGetIsPow2(MlirType type); +// `!cute.coord_tensor`: Coordinate tensor: an arithmetic tuple with a layout. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteCoordTensor(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTensorTypeGet(MlirContext context, MlirType arithTuple, @@ -40,11 +46,15 @@ MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTensorTypeGet(MlirContext context, MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTensorTypeGetArithTuple(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTensorTypeGetLayout(MlirType type); +// `!cute.coord`: Scalar integer, underscore wildcard, or recursive tuple of +// coordinates. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteCoord(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTypeGet(MlirContext context, MlirAttribute attr); MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCoordTypeGetAttr(MlirType type); +// `!cute.fast_divmod_divisor`: Precomputed divisor of the fast divmod +// operations. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteFastDivmodDivisor(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteFastDivmodDivisorTypeGet( MlirContext context, unsigned width, bool isPow2); @@ -52,16 +62,19 @@ MLIR_CAPI_EXPORTED unsigned mlirCuteFastDivmodDivisorTypeGetWidth(MlirType type); MLIR_CAPI_EXPORTED bool mlirCuteFastDivmodDivisorTypeGetIsPow2(MlirType type); +// `!cute.int_tuple`: Scalar integer or recursive tuple of integers. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteIntTuple(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteIntTupleTypeGet(MlirContext context, MlirAttribute attr); MLIR_CAPI_EXPORTED MlirAttribute mlirCuteIntTupleTypeGetAttr(MlirType type); +// `!cute.layout`: CuTe layout: a shape/stride pair. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteLayout(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteLayoutTypeGet(MlirContext context, MlirAttribute attr); MLIR_CAPI_EXPORTED MlirAttribute mlirCuteLayoutTypeGetAttr(MlirType type); +// `!cute.memref`: Tensor view: a pointer with a layout. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteMemRef(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteMemRefTypeGet(MlirContext context, MlirType ptr, @@ -69,22 +82,24 @@ MLIR_CAPI_EXPORTED MlirType mlirCuteMemRefTypeGet(MlirContext context, MLIR_CAPI_EXPORTED MlirType mlirCuteMemRefTypeGetPtr(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteMemRefTypeGetLayout(MlirType type); +// `!cute.ptr`: Pointer with address space, alignment, swizzle and bit layout. MLIR_CAPI_EXPORTED bool mlirTypeIsACutePtr(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCutePtrTypeGet(MlirContext context, - MlirType valueType, - MlirAttribute memorySpace, - uint64_t alignment, - MlirAttribute swizzle); +MLIR_CAPI_EXPORTED MlirType mlirCutePtrTypeGet( + MlirContext context, MlirType valueType, uint32_t addressSpace, + uint64_t alignment, MlirAttribute swizzle, MlirAttribute bitlayout); MLIR_CAPI_EXPORTED MlirType mlirCutePtrTypeGetValueType(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute mlirCutePtrTypeGetMemorySpace(MlirType type); +MLIR_CAPI_EXPORTED uint32_t mlirCutePtrTypeGetAddressSpace(MlirType type); MLIR_CAPI_EXPORTED uint64_t mlirCutePtrTypeGetAlignment(MlirType type); MLIR_CAPI_EXPORTED MlirAttribute mlirCutePtrTypeGetSwizzle(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirCutePtrTypeGetBitlayout(MlirType type); +// `!cute.shape`: Scalar integer or recursive tuple of shape extents. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteShape(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteShapeTypeGet(MlirContext context, MlirAttribute attr); MLIR_CAPI_EXPORTED MlirAttribute mlirCuteShapeTypeGetAttr(MlirType type); +// `!cute.sparse_elem`: Logical sparse element stored in a physical type. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteSparseElem(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteSparseElemTypeGet(MlirContext context, int numLogical, @@ -93,21 +108,76 @@ MLIR_CAPI_EXPORTED int mlirCuteSparseElemTypeGetNumLogical(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteSparseElemTypeGetPhysicalType(MlirType type); +// `!cute.stride`: Scalar or recursive tuple of stride elements. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteStride(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteStrideTypeGet(MlirContext context, MlirAttribute attr); MLIR_CAPI_EXPORTED MlirAttribute mlirCuteStrideTypeGetAttr(MlirType type); +// `!cute.swizzle`: CuTe swizzle: S. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteSwizzle(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteSwizzleTypeGet(MlirContext context, MlirAttribute attr); MLIR_CAPI_EXPORTED MlirAttribute mlirCuteSwizzleTypeGetAttr(MlirType type); +// `!cute.tile`: CuTe tile: a recursive sequence of layouts and underscores. MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTile(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteTileTypeGet(MlirContext context, MlirAttribute attr); MLIR_CAPI_EXPORTED MlirAttribute mlirCuteTileTypeGetAttr(MlirType type); +// `!cute.tiled_copy`: Copy atom tiled over threads and values. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTiledCopy(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledCopyTypeGet(MlirContext context, + MlirType copyAtom, + MlirAttribute layoutCopyTv, + MlirAttribute tilerMn); +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledCopyTypeGetCopyAtom(MlirType type); +// Former name of mlirCuteTiledCopyTypeGetCopyAtom. +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledCopyTypeGetAtom(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledCopyTypeGetLayoutCopyTv(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteTiledCopyTypeGetTilerMn(MlirType type); + +// `!cute.tiled_copy_v2`: V2 copy atom tiled over threads and values. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTiledCopyV2(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteTiledCopyV2TypeGet(MlirContext context, MlirType copyAtom, + MlirAttribute layoutCopyTv, MlirAttribute tilerMn); +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledCopyV2TypeGetCopyAtom(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledCopyV2TypeGetLayoutCopyTv(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledCopyV2TypeGetTilerMn(MlirType type); + +// `!cute.tiled_mma`: MMA atom tiled over MNK. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTiledMma(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledMmaTypeGet( + MlirContext context, MlirType mmaAtom, MlirAttribute atomLayoutMNK, + MlirAttribute permutationMNK); +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledMmaTypeGetMmaAtom(MlirType type); +// Former name of mlirCuteTiledMmaTypeGetMmaAtom. +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledMmaTypeGetAtom(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledMmaTypeGetAtomLayoutMNK(MlirType type); +// Former name of mlirCuteTiledMmaTypeGetAtomLayoutMNK. +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledMmaTypeGetAtomLayoutMnk(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledMmaTypeGetPermutationMNK(MlirType type); +// Former name of mlirCuteTiledMmaTypeGetPermutationMNK. +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledMmaTypeGetPermutationMnk(MlirType type); + +// `!cute.tuple`: Non-empty tuple of types. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTuple(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteTupleTypeGet(MlirContext context, + intptr_t numTypes, + MlirType const *types); +MLIR_CAPI_EXPORTED intptr_t mlirCuteTupleTypeGetNumTypes(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteTupleTypeGetType(MlirType type, + intptr_t pos); + #ifdef __cplusplus } #endif diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/BUILD.bazel b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/BUILD.bazel index ba3f3754..ed80bac3 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/BUILD.bazel +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/BUILD.bazel @@ -3,44 +3,79 @@ load("@rules_cc//cc:defs.bzl", "cc_library") package(default_visibility = ["//visibility:public"]) +# The release's dialect, attributes and types come from the archive; the DSL +# compiler's remaining attributes, types and enums are CuteAttrsCompiler.td, +# CuteTypesCompiler.td and CuteEnums.td, and every one of its operations is +# CuteOps.td, generated from the DSL's Python bindings and compiler. td_library( - name = "private_tabledefs", + name = "tabledefs", srcs = [ - "CuteOpsPrivate.td", + "CuteAttrsAll.td", + "CuteAttrsCompiler.td", + "CuteEnums.td", + "CuteOps.td", "CuteTypesAll.td", - "CuteTypesPrivate.td", + "CuteTypesCompiler.td", ], includes = ["/mlir/dialects/cute_ir/include"], - deps = [ - "@cute_ir//:dialect_td", - "@llvm-project//mlir:BuiltinDialectTdFiles", - "@llvm-project//mlir:DataLayoutInterfacesTdFiles", - "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", - "@llvm-project//mlir:OpBaseTdFiles", - "@llvm-project//mlir:SideEffectInterfacesTdFiles", - ], + deps = ["@cute_ir//:cute_ir_dialect_td"], ) gentbl_cc_library( - name = "ops_private_inc_gen", + name = "enums_inc_gen", includes = ["../../../.."], tbl_outs = [ ( - ["-gen-op-decls"], - "CuteOpsPrivate.h.inc", + ["-gen-enum-decls"], + "CuteEnums.h.inc", ), ( - ["-gen-op-defs"], - "CuteOpsPrivate.cpp.inc", + ["-gen-enum-defs"], + "CuteEnums.cpp.inc", ), ], tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "CuteOpsPrivate.td", - deps = [":private_tabledefs"], + td_file = "CuteEnums.td", + deps = [":tabledefs"], ) gentbl_cc_library( - name = "types_private_decl_inc_gen", + name = "attrs_inc_gen", + includes = ["../../../.."], + tbl_outs = [ + ( + [ + "-gen-attrdef-decls", + "-attrdefs-dialect=cute", + ], + "CuteAttrDecls.h.inc", + ), + ( + [ + "-gen-attrdef-defs", + "-attrdefs-dialect=cute", + ], + "CuteAttrs.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteAttrsAll.td", + deps = [":tabledefs"], +) + +# CuteDialect.h includes the attribute declarations first: they bring the +# enums the attributes and types use. +genrule( + name = "attrs_h_inc", + srcs = ["CuteAttrDecls.h.inc"], + outs = ["CuteAttrs.h.inc"], + cmd = "(echo '#ifdef GET_ATTRDEF_CLASSES'; " + + "echo '#include \"cute_ir/Dialect/Cute/IR/CuteEnums.h.inc\"'; " + + "echo '#endif'; cat $<) > $@", +) + +gentbl_cc_library( + name = "types_inc_gen", includes = ["../../../.."], tbl_outs = [ ( @@ -48,18 +83,8 @@ gentbl_cc_library( "-gen-typedef-decls", "-typedefs-dialect=cute", ], - "CuteTypesPrivate.h.inc", + "CuteTypes.h.inc", ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "CuteTypesPrivate.td", - deps = [":private_tabledefs"], -) - -gentbl_cc_library( - name = "types_all_defs_inc_gen", - includes = ["../../../.."], - tbl_outs = [ ( [ "-gen-typedef-defs", @@ -70,23 +95,44 @@ gentbl_cc_library( ], tblgen = "@llvm-project//mlir:mlir-tblgen", td_file = "CuteTypesAll.td", - deps = [":private_tabledefs"], + deps = [":tabledefs"], +) + +gentbl_cc_library( + name = "ops_inc_gen", + includes = ["../../../.."], + tbl_outs = [ + ( + ["-gen-op-decls"], + "CuteOps.h.inc", + ), + ( + ["-gen-op-defs"], + "CuteOps.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteOps.td", + deps = [":tabledefs"], ) cc_library( - name = "private_headers", + name = "headers", hdrs = [ - "CuteDialectPrivate.h", + "CuteAttrs.h.inc", + "CuteInference.h", ], includes = ["../../../.."], deps = [ - ":ops_private_inc_gen", - ":types_private_decl_inc_gen", - "@cute_ir//:dialect_headers", + ":attrs_inc_gen", + ":enums_inc_gen", + ":ops_inc_gen", + ":types_inc_gen", + "@cute_ir//:cute_ir_dialect_header", + "@cute_ir//:cutegen", + "@llvm-project//mlir:DataLayoutInterfaces", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:SideEffectInterfaces", ], ) - -alias( - name = "dialect", - actual = ":private_headers", -) diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteAttrsAll.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteAttrsAll.td new file mode 100644 index 00000000..ea4264cb --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteAttrsAll.td @@ -0,0 +1,7 @@ +// The dialect has one generated attribute parser and printer, so the +// release's attributes, the compiler's and its enum attributes are generated +// together from here. + +include "cute_ir/Dialect/Cute/IR/CuteAttrs.td" +include "cute_ir/Dialect/Cute/IR/CuteAttrsCompiler.td" +include "cute_ir/Dialect/Cute/IR/CuteEnums.td" diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteAttrsCompiler.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteAttrsCompiler.td new file mode 100644 index 00000000..7ce36eb5 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteAttrsCompiler.td @@ -0,0 +1,220 @@ +// The cute dialect's attributes beyond the release, generated from the +// nvidia-cutlass-dsl 4.8.0 release; do not edit. +// +// Sources: the compiler library (`_cutlass_ir`): its generated parser +// messages give every class, its parameters in declaration order and their +// C++ types; the live compiler, loaded from Python, gives the syntax (what it +// prints for what it parses) and the verifier messages. +// +// The fragment parameters are `FragmentAttrInterface` in the compiler, an +// interface neither the release nor this tree defines: they are plain +// attributes here (the cute_nvgpu `*_frg` attributes). +// +// The reduction_op enum attribute is in CuteEnums.td. + +#ifndef CUTE_IR_GENERATED_CUTE_ATTRS_COMPILER_TD +#define CUTE_IR_GENERATED_CUTE_ATTRS_COMPILER_TD + +include "cute_ir/Dialect/Cute/IR/CuteAttrs.td" + +def Cute_BitLayoutAttr : Cute_Attr<"BitLayout", "bitlayout"> { + let summary = "Bit layout of a pointer's elements within a storage chunk"; + let description = [{ + A two-mode layout from (element bit, element index) to the bit offset in + a storage chunk. Used by `!cute.ptr` (`bitlayout<"...">`). + + ```mlir + #cute.bitlayout<<"(8,4):(1,8)">> + ``` + }]; + let parameters = (ins + Cute_Param<"LayoutAttr", "(element bit, element index) -> bit">:$layout + ); + let assemblyFormat = "`<` $layout `>`"; + let genVerifyDecl = 1; +} + +def Cute_CopyAtomAttr : Cute_Attr<"CopyAtom", "copy_atom"> { + let summary = "Thread and value layouts of a copy atom"; + let description = [{ + ```mlir + #cute.copy_atom, layout_tv = (<"(1,8):(0,1)">, <"(1,8):(0,1)">, <"(1,8):(0,1)">)> + ``` + }]; + let parameters = (ins + Cute_Param<"LayoutAttr", "thread id layout">:$thr_id, + Cute_Param<"LayoutAttr", "source thread-value layout">:$layout_src, + Cute_Param<"LayoutAttr", "destination thread-value layout">:$layout_dst, + Cute_Param<"LayoutAttr", "reference thread-value layout">:$layout_ref + ); + let assemblyFormat = [{ + `<` `thr_id` `=` $thr_id `,` `layout_tv` `=` + `(` $layout_src `,` $layout_dst `,` $layout_ref `)` `>` + }]; +} + +def Cute_CopyAtomV2Attr : Cute_Attr<"CopyAtomV2", "copy_atom_v2"> { + let summary = "Thread and value layouts and fragments of a V2 copy atom"; + let description = [{ + ```mlir + #cute.copy_atom_v2, layout_tv = (<"(1,8):(0,1)">, <"(1,8):(0,1)">), frg = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)> + ``` + }]; + let parameters = (ins + Cute_Param<"LayoutAttr", "thread id layout">:$thr_id, + Cute_Param<"LayoutAttr", "source thread-value layout">:$layoutSrc_TV, + Cute_Param<"LayoutAttr", "destination thread-value layout">:$layoutDst_TV, + Cute_Param<"mlir::Attribute", "source fragment">:$frgSrc, + Cute_Param<"mlir::Attribute", "destination fragment">:$frgDst + ); + let assemblyFormat = [{ + `<` `thr_id` `=` $thr_id `,` `layout_tv` `=` + `(` $layoutSrc_TV `,` $layoutDst_TV `)` `,` + `frg` `=` `(` $frgSrc `,` $frgDst `)` `>` + }]; +} + +def Cute_MmaAtomAttr : Cute_Attr<"MmaAtom", "mma_atom"> { + let summary = "Shapes, thread-value layouts and fragments of an MMA atom"; + let description = [{ + ```mlir + #cute.mma_atom<<"(16,8,16)">, <"32:1">, layout_TV = (<"1:0">, <"1:0">, <"1:0">), shapes = (<"(16,16)">, <"(8,16)">, <"(16,8)">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)> + ``` + }]; + let parameters = (ins + Cute_Param<"ShapeAttr", "MNK shape">:$shape_MNK, + Cute_Param<"LayoutAttr", "thread id layout">:$thr_id, + Cute_Param<"LayoutAttr", "A thread-value layout">:$layoutA_TV, + Cute_Param<"LayoutAttr", "B thread-value layout">:$layoutB_TV, + Cute_Param<"LayoutAttr", "C thread-value layout">:$layoutC_TV, + Cute_Param<"ShapeAttr", "A shape (MK)">:$shapeA_MK, + Cute_Param<"ShapeAttr", "B shape (NK)">:$shapeB_NK, + Cute_Param<"ShapeAttr", "C shape (MN)">:$shapeC_MN, + Cute_Param<"mlir::Attribute", "A fragment">:$frgA, + Cute_Param<"mlir::Attribute", "B fragment">:$frgB, + Cute_Param<"mlir::Attribute", "C fragment">:$frgC + ); + let assemblyFormat = [{ + `<` $shape_MNK `,` $thr_id `,` + `layout_TV` `=` `(` $layoutA_TV `,` $layoutB_TV `,` $layoutC_TV `)` `,` + `shapes` `=` `(` $shapeA_MK `,` $shapeB_NK `,` $shapeC_MN `)` + `frgs` `=` `(` $frgA `,` $frgB `,` $frgC `)` `>` + }]; +} + +def Cute_MxMmaAtomAttr : Cute_Attr<"MxMmaAtom", "mx_mma_atom"> { + let summary = "MMA atom attribute with scale-factor operands"; + let description = [{ + Operands in the order A, SFA, B, SFB, C. + + ```mlir + #cute.mx_mma_atom<<"(16,8,16)">, <"32:1">, layout_TV = (<"1:0">, <"1:0">, <"1:0">, <"1:0">, <"1:0">), shapes = (<"(16,16)">, <"(8,16)">, <"(16,8)">, <"1">, <"2">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)> + ``` + }]; + let parameters = (ins + Cute_Param<"ShapeAttr", "MNK shape">:$shape_MNK, + Cute_Param<"LayoutAttr", "thread id layout">:$thr_id, + Cute_Param<"LayoutAttr", "A thread-value layout">:$layoutA_TV, + Cute_Param<"LayoutAttr", "SFA thread-value layout">:$layoutSFA_TV, + Cute_Param<"LayoutAttr", "B thread-value layout">:$layoutB_TV, + Cute_Param<"LayoutAttr", "SFB thread-value layout">:$layoutSFB_TV, + Cute_Param<"LayoutAttr", "C thread-value layout">:$layoutC_TV, + Cute_Param<"ShapeAttr", "A shape (MK)">:$shapeA_MK, + Cute_Param<"ShapeAttr", "SFA shape (MK)">:$shapeSFA_MK, + Cute_Param<"ShapeAttr", "B shape (NK)">:$shapeB_NK, + Cute_Param<"ShapeAttr", "SFB shape (NK)">:$shapeSFB_NK, + Cute_Param<"ShapeAttr", "C shape (MN)">:$shapeC_MN, + Cute_Param<"mlir::Attribute", "A fragment">:$frgA, + Cute_Param<"mlir::Attribute", "SFA fragment">:$frgSFA, + Cute_Param<"mlir::Attribute", "B fragment">:$frgB, + Cute_Param<"mlir::Attribute", "SFB fragment">:$frgSFB, + Cute_Param<"mlir::Attribute", "C fragment">:$frgC + ); + let assemblyFormat = [{ + `<` $shape_MNK `,` $thr_id `,` + `layout_TV` `=` `(` $layoutA_TV `,` $layoutSFA_TV `,` $layoutB_TV `,` + $layoutSFB_TV `,` $layoutC_TV `)` `,` + `shapes` `=` `(` $shapeA_MK `,` $shapeSFA_MK `,` $shapeB_NK `,` + $shapeSFB_NK `,` $shapeC_MN `)` + `frgs` `=` `(` $frgA `,` $frgSFA `,` $frgB `,` $frgSFB `,` $frgC `)` `>` + }]; +} + +def Cute_SparseMmaAtomAttr : Cute_Attr<"SparseMmaAtom", "sparse_mma_atom"> { + let summary = "MMA atom attribute with a sparsity-metadata operand"; + let description = [{ + Operands in the order A, B, C, E. + + ```mlir + #cute.sparse_mma_atom<<"(16,8,16)">, <"32:1">, layout_TV = (<"1:0">, <"1:0">, <"1:0">, <"1:0">), shapes = (<"(16,16)">, <"(8,16)">, <"(16,8)">, <"1">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)> + ``` + }]; + let parameters = (ins + Cute_Param<"ShapeAttr", "MNK shape">:$shape_MNK, + Cute_Param<"LayoutAttr", "thread id layout">:$thr_id, + Cute_Param<"LayoutAttr", "A thread-value layout">:$layoutA_TV, + Cute_Param<"LayoutAttr", "B thread-value layout">:$layoutB_TV, + Cute_Param<"LayoutAttr", "C thread-value layout">:$layoutC_TV, + Cute_Param<"LayoutAttr", "E thread-value layout">:$layoutE_TV, + Cute_Param<"ShapeAttr", "A shape (MK)">:$shapeA_MK, + Cute_Param<"ShapeAttr", "B shape (NK)">:$shapeB_NK, + Cute_Param<"ShapeAttr", "C shape (MN)">:$shapeC_MN, + Cute_Param<"ShapeAttr", "E shape (MK)">:$shapeE_MK, + Cute_Param<"mlir::Attribute", "A fragment">:$frgA, + Cute_Param<"mlir::Attribute", "B fragment">:$frgB, + Cute_Param<"mlir::Attribute", "C fragment">:$frgC, + Cute_Param<"mlir::Attribute", "E fragment">:$frgE + ); + let assemblyFormat = [{ + `<` $shape_MNK `,` $thr_id `,` + `layout_TV` `=` `(` $layoutA_TV `,` $layoutB_TV `,` $layoutC_TV `,` + $layoutE_TV `)` `,` + `shapes` `=` `(` $shapeA_MK `,` $shapeB_NK `,` $shapeC_MN `,` $shapeE_MK `)` + `frgs` `=` `(` $frgA `,` $frgB `,` $frgC `,` $frgE `)` `>` + }]; +} + +def Cute_SparseMxMmaAtomAttr + : Cute_Attr<"SparseMxMmaAtom", "sparse_mx_mma_atom"> { + let summary = "MMA atom attribute with scale-factor and metadata operands"; + let description = [{ + Operands in the order A, SFA, B, SFB, C, E. + + ```mlir + #cute.sparse_mx_mma_atom<<"(16,8,16)">, <"32:1">, layout_TV = (<"1:0">, <"2:0">, <"3:0">, <"4:0">, <"5:0">, <"6:0">), shapes = (<"1">, <"2">, <"3">, <"4">, <"5">, <"6">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)> + ``` + }]; + let parameters = (ins + Cute_Param<"ShapeAttr", "MNK shape">:$shape_MNK, + Cute_Param<"LayoutAttr", "thread id layout">:$thr_id, + Cute_Param<"LayoutAttr", "A thread-value layout">:$layoutA_TV, + Cute_Param<"LayoutAttr", "SFA thread-value layout">:$layoutSFA_TV, + Cute_Param<"LayoutAttr", "B thread-value layout">:$layoutB_TV, + Cute_Param<"LayoutAttr", "SFB thread-value layout">:$layoutSFB_TV, + Cute_Param<"LayoutAttr", "C thread-value layout">:$layoutC_TV, + Cute_Param<"LayoutAttr", "E thread-value layout">:$layoutE_TV, + Cute_Param<"ShapeAttr", "A shape (MK)">:$shapeA_MK, + Cute_Param<"ShapeAttr", "SFA shape (MK)">:$shapeSFA_MK, + Cute_Param<"ShapeAttr", "B shape (NK)">:$shapeB_NK, + Cute_Param<"ShapeAttr", "SFB shape (NK)">:$shapeSFB_NK, + Cute_Param<"ShapeAttr", "C shape (MN)">:$shapeC_MN, + Cute_Param<"ShapeAttr", "E shape (MK)">:$shapeE_MK, + Cute_Param<"mlir::Attribute", "A fragment">:$frgA, + Cute_Param<"mlir::Attribute", "SFA fragment">:$frgSFA, + Cute_Param<"mlir::Attribute", "B fragment">:$frgB, + Cute_Param<"mlir::Attribute", "SFB fragment">:$frgSFB, + Cute_Param<"mlir::Attribute", "C fragment">:$frgC, + Cute_Param<"mlir::Attribute", "E fragment">:$frgE + ); + let assemblyFormat = [{ + `<` $shape_MNK `,` $thr_id `,` + `layout_TV` `=` `(` $layoutA_TV `,` $layoutSFA_TV `,` $layoutB_TV `,` + $layoutSFB_TV `,` $layoutC_TV `,` $layoutE_TV `)` `,` + `shapes` `=` `(` $shapeA_MK `,` $shapeSFA_MK `,` $shapeB_NK `,` + $shapeSFB_NK `,` $shapeC_MN `,` $shapeE_MK `)` + `frgs` `=` `(` $frgA `,` $frgSFA `,` $frgB `,` $frgSFB `,` $frgC `,` + $frgE `)` `>` + }]; +} + +#endif // CUTE_IR_GENERATED_CUTE_ATTRS_COMPILER_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteDialectPrivate.h b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteDialectPrivate.h deleted file mode 100644 index afe8682d..00000000 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteDialectPrivate.h +++ /dev/null @@ -1,14 +0,0 @@ -//===- CuteDialectPrivate.h - recovered CuTe declarations -----*- C++ -*-===// - -#ifndef CUTE_IR_DIALECT_CUTE_IR_CUTE_DIALECT_PRIVATE_H -#define CUTE_IR_DIALECT_CUTE_IR_CUTE_DIALECT_PRIVATE_H - -#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" - -#define GET_TYPEDEF_CLASSES -#include "cute_ir/Dialect/Cute/IR/CuteTypesPrivate.h.inc" - -#define GET_OP_CLASSES -#include "cute_ir/Dialect/Cute/IR/CuteOpsPrivate.h.inc" - -#endif // CUTE_IR_DIALECT_CUTE_IR_CUTE_DIALECT_PRIVATE_H diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteEnums.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteEnums.td new file mode 100644 index 00000000..d3464172 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteEnums.td @@ -0,0 +1,70 @@ +// The cute dialect's enums, generated from the nvidia-cutlass-dsl 4.8.0 +// enum bindings (`_cute_enum_gen.py`); do not edit. +// +// Case values and spellings are the compiler's. An enum the bindings build +// as a dialect attribute also has its EnumAttr, printed `#`. + +#ifndef CUTE_IR_GENERATED_CUTE_ENUMS_TD +#define CUTE_IR_GENERATED_CUTE_ENUMS_TD + +include "cute_ir/Dialect/Cute/IR/CuteDialect.td" +include "mlir/IR/EnumAttr.td" + +def Cute_AddressSpaceEnum : I32EnumAttr<"AddressSpace", "Address spaces for CuTe memrefs and pointers", [ + I32EnumAttrCase<"generic", 0, "generic">, + I32EnumAttrCase<"gmem", 1, "gmem">, + I32EnumAttrCase<"smem", 3, "smem">, + I32EnumAttrCase<"rmem", 5, "rmem">, + I32EnumAttrCase<"tmem", 6, "tmem">, + I32EnumAttrCase<"dsmem", 7, "dsmem">, + I32EnumAttrCase<"cmem", 4, "cmem"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute"; + let genSpecializedAttr = 0; +} + +def Cute_CopyOperandEnum : I32EnumAttr<"CopyOperand", "Copy operand ID", [ + I32EnumAttrCase<"S", 0, "S">, + I32EnumAttrCase<"D", 1, "D"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute"; + let genSpecializedAttr = 0; +} + +def Cute_MajorModeEnum : I32EnumAttr<"MajorMode", "Major mode for MMA operations", [ + I32EnumAttrCase<"k", 0, "k">, + I32EnumAttrCase<"mn", 1, "mn"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute"; + let genSpecializedAttr = 0; +} + +def Cute_MmaOperandEnum : I32EnumAttr<"MmaOperand", "MMA operand ID", [ + I32EnumAttrCase<"A", 0, "A">, + I32EnumAttrCase<"B", 1, "B">, + I32EnumAttrCase<"C", 2, "C">, + I32EnumAttrCase<"D", 3, "D">, + I32EnumAttrCase<"SFA", 4, "SFA">, + I32EnumAttrCase<"SFB", 5, "SFB">, + I32EnumAttrCase<"E", 6, "E"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute"; + let genSpecializedAttr = 0; +} + +def Cute_ReductionOpEnum : I32EnumAttr<"ReductionOp", "Op for cute reduce operations", [ + I32EnumAttrCase<"ADD", 0, "ADD">, + I32EnumAttrCase<"MUL", 1, "MUL">, + I32EnumAttrCase<"MIN", 2, "MIN">, + I32EnumAttrCase<"MAX", 3, "MAX"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute"; + let genSpecializedAttr = 0; +} + +def Cute_ReductionOpAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute"; + let assemblyFormat = "`<` $value `>`"; +} + +#endif // CUTE_IR_GENERATED_CUTE_ENUMS_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteInference.h b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteInference.h new file mode 100644 index 00000000..26fbfb69 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteInference.h @@ -0,0 +1,24 @@ +//===- CuteInference.h - result types the compiler infers --------*- C++ -*-===// + +#ifndef CUTE_IR_DIALECT_CUTE_IR_CUTE_INFERENCE_H +#define CUTE_IR_DIALECT_CUTE_IR_CUTE_INFERENCE_H + +#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" + +namespace mlir::cutlass_compiler::cute { +// make_tiled_copy_D: `atom` tiled over `tiled`'s destination thread-value +// layout and tiler, or null when the atom is not one this tree knows. +Type tiledCopyD(Type atom, TiledCopyType tiled); + +// Result types of operations the compiler does not infer (their builders +// take the type): what the DSL passes, or null when not computable here. +Type makeLayoutResultType(Type shape, Type stride); // stride may be null +Type makeViewResultType(Type iter, Type layout); +Type getResultType(Type input, ArrayRef mode); +Type assumeResultType(Type src); +Type makeArithTupleIterResultType(Type tuple); +Type fastDivmodCreateDivisorResultType(Type divisor); +Type fastDivmodComputeResultType(Type divisor); // both results +} // namespace mlir::cutlass_compiler::cute + +#endif // CUTE_IR_DIALECT_CUTE_IR_CUTE_INFERENCE_H diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td new file mode 100644 index 00000000..768fbf9f --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td @@ -0,0 +1,3795 @@ +// The cute dialect's operations, generated from the nvidia-cutlass-dsl 4.8.0 release; do not edit. +// +// Sources: the operation bindings (`_cute_ops_gen.py`: names, classes, +// docstrings, operand/result/attribute structure, inference) and the live +// compiler (`_cutlass_ir` loaded from Python): each operand and result +// constraint carries the verifier's wording and accepts every type class the +// verifier let through when probed with one instance of every type; traits +// and interfaces are the registered operation's; memory effects come from +// loop-invariant code motion (hoisted: Pure), canonicalize (dead: no write) +// and CSE (merged: at most reads). Operations whose effects include writes +// or could not be told apart carry none (unknown effects). +// +// Operations keep the generic assembly form; the compiler's custom syntax is +// shown in each description. +// +// Constraints not recovered: +// cute.deref_desc_iter operand #0: no probe type accepted; any type allowed +// cute.make_desc_iter operand #0: no probe type accepted; any type allowed + +#ifndef CUTE_IR_GENERATED_CUTE_OPS_TD +#define CUTE_IR_GENERATED_CUTE_OPS_TD + +include "cute_ir/Dialect/Cute/IR/CuteAttrs.td" +include "cute_ir/Dialect/Cute/IR/CuteEnums.td" + +include "mlir/Interfaces/InferTypeOpInterface.td" +include "mlir/Interfaces/SideEffectInterfaces.td" + +// Operand and result constraints, in the compiler's words. +def Cute_32BitSignlessIntegerOr64BitSignless : Type, "32-bit signless integer or 64-bit signless integer">; +def Cute_AMemoryReferenceTypeThatRepresentsATensor : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self))">, "A memory reference type that represents a tensor view with an iterator and layout or CuTe Layout Type or Cute shape type">; +def Cute_AMemoryReferenceTypeThatRepresentsATensor2 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self))">, "A memory reference type that represents a tensor view with an iterator and layout or Cute coord_tensor type">; +def Cute_AMemoryReferenceTypeThatRepresentsATensor3 : Type($_self))">, "A memory reference type that represents a tensor view with an iterator and layout">; +def Cute_ATupleIsAFixedLengthOrderedList : Type($_self))">, "A tuple is a fixed-length, ordered list consisting of multiple elements.">; +def Cute_ATypeInterfaceForDescriptorIteratorTypes : Type, "A type interface for descriptor iterator types.">; // not recovered: no pool type passed +def Cute_ATypeInterfaceForMemoryDescriptorsSuchAs : Type, "A type interface for memory descriptors such as smem desc etc.">; // not recovered: no pool type passed +def Cute_ArithmeticTupleType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self))">, "Arithmetic tuple type">; +def Cute_AtomTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledMmaType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM120BlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomLdsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomS2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTAsyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemLdReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemStType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100S2TCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomStsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM10xTmemLoadRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTSyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::UniversalFmaAtomType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80SparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM89Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM90Type>($_self))">, "AtomTypeInterface instance">; +def Cute_CopyAtomTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomLdsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomS2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTAsyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemLdReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemStType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomStsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM10xTmemLoadRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTSyncCopyType>($_self))">, "CopyAtomTypeInterface instance">; +def Cute_CopyAtomTypeInterfaceInstanceOrCopyAtomTypeInterfaceV2Instance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomLdsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomS2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTAsyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemLdReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemStType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100S2TCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomStsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM10xTmemLoadRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTSyncCopyType>($_self))">, "CopyAtomTypeInterface instance or CopyAtomTypeInterfaceV2 instance">; +def Cute_CopyAtomTypeInterfaceV2Instance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100S2TCopyV2Type>($_self))">, "CopyAtomTypeInterfaceV2 instance">; +def Cute_CuTeComposedLayoutType : Type($_self))">, "CuTe composed layout type">; +def Cute_CuTeIteratorTypeWithIntTupleAsIts : Type($_self))">, "CuTe iterator type with int_tuple as its underlying implementation">; +def Cute_CuTeLayoutType : Type($_self))">, "CuTe Layout Type">; +def Cute_CuTeLayoutTypeOrCuteSwizzleType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self))">, "CuTe Layout Type or Cute swizzle type">; +def Cute_CuTeLayoutTypeOrSwizzleWithValidParameters : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self))">, "CuTe Layout Type or swizzle with valid parameters that can be represented by 32-bit masks">; +def Cute_CuTePointerTo32BitSignlessInteger : Type($_self))">, "CuTe pointer to 32-bit signless integer">; +def Cute_CuTePointerType : Type($_self))">, " CuTe pointer type ">; +def Cute_CuTePointerTypeOrAMemoryReferenceType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::FastDivmodDivisorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ConstrainedIntType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::PtrType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledMmaType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM120BlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100S2TCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80SparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM90Type>($_self) || ::llvm::isa<::mlir::IntegerType>($_self) || ::llvm::isa<::mlir::FloatType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMArrayType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMPointerType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMStructType>($_self) || ($_self.getDialect().getNamespace() == \"ptr\" || (::llvm::isa<::mlir::OpaqueType>($_self) && ::llvm::cast<::mlir::OpaqueType>($_self).getDialectNamespace() == \"ptr\")) || ::llvm::isa<::mlir::VectorType>($_self))">, " CuTe pointer type or A memory reference type that represents a tensor view with an iterator and layout or Integer type with constraints. or CuTe fast divmod divisor type or loadable maybe-static CuTe type with runtime storage or LLVM type with size">; +def Cute_CuTePointerTypeOrLLVMPointerType : Type($_self) || ::llvm::isa<::mlir::LLVM::LLVMPointerType>($_self))">, " CuTe pointer type or LLVM pointer type">; +def Cute_CuTeSparseElementType : Type($_self))">, " CuTe sparse element type">; +def Cute_CuteCoordTensorType : Type($_self))">, "Cute coord_tensor type">; +def Cute_CuteCoordType : Type($_self))">, "Cute coord type">; +def Cute_CuteShapeType : Type($_self))">, "Cute shape type">; +def Cute_CuteShapeTypeOrIntTupleAsIntegerOr : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self))">, "Cute shape type or IntTuple as integer or tuple of IntTuple(s) or Cute coord type">; +def Cute_CuteShapeTypeOrIntTupleAsIntegerOr2 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, "Cute shape type or IntTuple as integer or tuple of IntTuple(s) or Cute coord type or CuTe Layout Type or CuTe composed layout type or ViewTypeInterface instance">; +def Cute_CuteStrideType : Type($_self))">, "Cute stride type">; +def Cute_CuteTileType : Type($_self))">, "Cute tile type">; +def Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant : Type($_self))">, "Divisor for fast unsigned integer divmods by invariant integers.">; +def Cute_GeneralizedIteratorTypeOfCuTe : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::PtrType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescSM107Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescType>($_self))">, " Generalized iterator type of CuTe ">; +def Cute_GeneralizedLayoutTypeOfCuTe : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self))">, " Generalized layout type of CuTe ">; +def Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, " Generalized layout type of CuTe or ViewTypeInterface instance">; +def Cute_I32I64OrConstrainedInteger : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || $_self.isSignlessInteger(32) || $_self.isSignlessInteger(64))">, "i32, i64 or Constrained integer">; +def Cute_IntTupleAsIntegerOrTupleOfIntTupleS : Type($_self))">, "IntTuple as integer or tuple of IntTuple(s)">; +def Cute_IntTupleAsIntegerOrTupleOfIntTupleS2 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self))">, "IntTuple as integer or tuple of IntTuple(s) or Cute shape type">; +def Cute_IntTupleTypeThatDepthIs0 : Type($_self))">, "Int tuple type that depth is 0">; +def Cute_Integer : Type($_self))">, "integer">; +def Cute_IntegerOrIntegerTypeWithConstraints : Type($_self) || ::llvm::isa<::mlir::IntegerType>($_self))">, "integer or Integer type with constraints. ">; +def Cute_IntegerTypeWithConstraints : Type($_self))">, " Integer type with constraints. ">; +def Cute_MaybeStaticTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ComposedLayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledMmaType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM120BlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomLdsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomS2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTAsyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemLdReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemStType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100S2TCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomStsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM10xTmemLoadRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTSyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::UniversalFmaAtomType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80SparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM89Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM90Type>($_self))">, "MaybeStaticTypeInterface instance">; +def Cute_MemrefInGenericOrRegisterMemOrSmem : Type($_self))">, "memref in generic or register mem or smem">; +def Cute_MemrefInSmemWithStaticAndNonComposed : Type($_self))">, "memref in smem with static and non-composed layout">; +def Cute_MmaAtomTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM120BlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::UniversalFmaAtomType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80SparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM89Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM90Type>($_self))">, "MmaAtomTypeInterface instance">; +def Cute_OfRanks1 : Type($_self))">, " of ranks 1">; +def Cute_OfRanks1OrFloatingPointOrInteger : Type($_self) || ::llvm::isa<::mlir::FloatType>($_self) || ::llvm::isa<::mlir::VectorType>($_self))">, " of ranks 1 or floating-point or integer">; +def Cute_PointerTypeInterfaceInstance : Type($_self))">, "PointerTypeInterface instance">; +def Cute_PrefetchAtomTypeInterfaceOrTiledCopyTypeWithPrefetchAtomTypeInterfaceCopyAtom : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self))">, "PrefetchAtomTypeInterface or TiledCopyType with PrefetchAtomTypeInterface copy atom">; +def Cute_RefinableTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ComposedLayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ConstrainedIntType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::PtrType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescSM107Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, "RefinableTypeInterface instance">; +def Cute_RefinableTypeInterfaceInstanceOrInteger : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ComposedLayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ConstrainedIntType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::PtrType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescSM107Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self) || ::llvm::isa<::mlir::IntegerType>($_self))">, "RefinableTypeInterface instance or integer">; +def Cute_StaticLayout : Type($_self))">, "static layout">; +def Cute_StaticShape : Type($_self))">, "static shape">; +def Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self))">, "Tile-like types, e.g. LayoutType, ShapeType, TileType">; +def Cute_TiledCopyAtomType : Type($_self))">, "Tiled copy atom type.">; +def Cute_TiledCopyAtomTypeOrTiledCopyV2 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyV2Type>($_self))">, "Tiled copy atom type. or Tiled copy V2 atom type.">; +def Cute_TiledCopyV2AtomType : Type($_self))">, "Tiled copy V2 atom type.">; +def Cute_TiledMMAAtomType : Type($_self))">, "Tiled MMA atom type.">; +def Cute_TypeInterfaceForCuteTypesCanBePrinted : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ComposedLayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::PtrType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self) || ::llvm::isa<::mlir::IntegerType>($_self) || ::llvm::isa<::mlir::FloatType>($_self))">, "Type interface for cute types can be printed by `cute.print`">; +def Cute_ValueTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self))">, "ValueTypeInterface instance">; +def Cute_ValueTypeInterfaceInstanceOrIntegerTypeWithConstraints : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ConstrainedIntType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self))">, "ValueTypeInterface instance or Integer type with constraints. ">; +def Cute_ValueTypeInterfaceInstanceOrIntegerTypeWithConstraintsOr : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ConstrainedIntType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || $_self.isSignlessInteger(8) || $_self.isSignlessInteger(16) || $_self.isSignlessInteger(32) || $_self.isSignlessInteger(64) || ::llvm::isa<::mlir::IndexType>($_self))">, "ValueTypeInterface instance or Integer type with constraints. or 8/16/32/64-bit integer or index">; +def Cute_VectorOf1BitSignlessIntegerValuesOf : Type($_self))">, "vector of 1-bit signless integer values of ranks 1">; +def Cute_ViewTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, "ViewTypeInterface instance">; +def Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, "ViewTypeInterface instance or Generalized layout type of CuTe ">; +def Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe2 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, "ViewTypeInterface instance or Generalized layout type of CuTe or Cute tile type">; +def Cute_XTupleLikeType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self))">, "XTuple-like type">; +def Cute_XTupleLikeTypeOrCuTeLayoutType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self))">, "XTuple-like type or CuTe Layout Type">; + +// Operations whose result types are inferred in C++. Verification compares +// them with the inferred ones where the rule is recovered for the operands; +// elsewhere the types written stand, as the compiler accepted them. +class Cute_InferredOp traits = []> + : Cute_Op]> { + let extraClassDefinition = [{ + ::llvm::LogicalResult $cppClass::refineReturnTypes( + ::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, + ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, + ::mlir::PropertyRef properties, ::mlir::RegionRange regions, + ::llvm::SmallVectorImpl<::mlir::Type> &returnTypes) { + ::llvm::SmallVector<::mlir::Type, 4> inferred; + if (::mlir::failed(inferReturnTypes(context, ::std::nullopt, operands, + attributes, properties, regions, + inferred))) + return ::mlir::success(); + if (!isCompatibleReturnTypes(inferred, returnTypes)) + return ::mlir::emitOptionalError( + location, "'", getOperationName(), "' op inferred type(s) ", + inferred, " are incompatible with return type(s) of operation ", + returnTypes); + return ::mlir::success(); + } + }]; +} + +def Cute_AddOffsetOp : Cute_InferredOp<"add_offset", [Pure]> { + let description = [{ + The `cute.add_offset` operation accepts an input IteratorTypeInterface and + a corresponding offset type which must be an + arithmetic tuple. It returns a new SSA value as type IteratorTypeInterface. + + When src operand is a sparse pointer, the logical offset must be a multiple + of the sparsity factor. The actual offset added to the src operand is + equal to the logical offset divided by the sparsity factor. + + `add_offset(%it, N)` returns a new iterator pointing to an object with an + offset of`N` elements. It is similar to C++'s iterator arithmetic. + + Example: + + ```mlir + // Add offset to 1-D linear pointer + %ptr = cute.add_offset(%0, %1) : (!cute.ptr>, !cute.int_tuple<"?">) -> !cute.ptr> + + // Add offset to 1-D linear pointer with static offset + %off = cute.static : !cute.int_tuple<"1012"> + %ptr = cute.add_offset(%0, %off) : (!cute.ptr>) -> !cute.ptr> + + // Add offset as arithmetic tuple to N-D nested arithmetic tuple + %arith_tuple = cute.add_offset(%0, %1) : (!cute.int_tuple<"(?,(3,?))">, !cute.int_tuple<"(3,(1,?))">) -> !cute.int_tuple<"(?,(4,?))"> + + // Add offset as arithmetic tuple to N-D nested arithmetic tuple with static offset + %arith_tuple = cute.add_offset(%0, "(3,(1,3))") : (!cute.int_tuple<"(?,(3,?))">) -> !cute.int_tuple<"(?,(4,?))"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedIteratorTypeOfCuTe:$src, + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$offset + ); + let results = (outs + Cute_GeneralizedIteratorTypeOfCuTe:$dst + ); +} +def Cute_AlgorithmCopyOp : Cute_Op<"copy", [AttrSizedOperandSegments]> { + let description = [{ + CuTe's copy algorithm copies the elements of a source Tensor into the elements + of a destination Tensor. + + See the file cutlass/media/docs/cute/04_algorithms.md in the CUTLASS repository + for more information. + + A Tensor (cute.memref) encapsulates the data type, data location, and possibly + also the shape and stride of the tensor at compile time. As a result, copy can + and does dispatch, based on the types of its arguments, to use any of various + synchronous or asynchronous hardware copy instructions. + + An .copy accepts CuteCopyAtomTypeInterface as first parameter. To keep consistent + interface with CUTLASS C++, it could be either atom.tiled_copy or atom.universal_copy. + + Optional argument 'pred' is used to guard memory accesses performed in copy. + Predication works in granularity of atom_v. For src tensor whose shape is: + `((atom_v, rest_v), rest_m, rest_n, ...)`, the pred tensor shape should be: + `((rest_v), rest_m, rest_n, ...)`, we denoted this as the canonical format. + If the pred tensor's value equals 0, the corresponding copy will be skipped. + + When using atom.universal_copy instead of atom.tiled_copy, we allow the layout of + `pred` to deviate from the canonical format. Instead, the layout shape of `pred` + must match the layout shape of `src` or `dst`. Additionally, the `copy_bits` + parameter cannot be set for atom.universal_copy when predicate is present. + + The operands `src` and `dst` are variadic, each containing a variable number of tensors: + - For regular copy, `src` and `dst` each contain a single tensor. + - For copy with auxiliary operands, they contain the main tensor followed by + auxiliary tensors. For example: + - For load with row-wise reduction, `dst` = [data, stat]. + - For SPARSIFY (structured sparsity compression), `dst` = [data, metadata]. + - For TMA gather4, `src` = [coord0, coord1, coord2, coord3] (four 2D coordinate tensors). + - For TMA scatter4, `dst` = [coord0, coord1, coord2, coord3] (four 2D coordinate tensors). + + Example: + + ```mlir + cute.copy(%tiled_copy, %src, %dst) : ( + !cute.tiled_copy, + layout_copy_tv = <"((4,2),(64,128)):((0,64),(128,1))">, + tiler_mn = <"[2048:1;1:1;4:1;1:1]">>, + !cute.coord_tensor<"(0,0)", "(((64, 128), 1), 1, 1, 2):(((1@0, 1@1), 0), 0, 0, 64@0)">, + !cute.memref, "(((64,128),1),1,1,2):(((1,64),0),0,0,8192)">) + + // atom.universal_copy with canonicalize layout format + cute.copy(%copy_atom, %src, %dst, %pred) : ( + !cute_nvgpu.atom.universal_copy, + !cute.memref, "((1,1),2,4):((0,0),1,2)">, + !cute.memref, "((1,1),2,4):((0,0),1,2)">, + !cute.memref, "((1),2,4):((0),1,2)">) + + // atom.universal_copy with relaxed layout format + cute.copy(%atom, %arg0, %arg1, %arg2) : ( + !cute_nvgpu.atom.universal_copy, + !cute.memref, "(4,4):(1,4)">, + !cute.memref, + !cute.memref, "(4,4):(1,4)">) + + ``` + }]; + let arguments = (ins + Cute_CopyAtomTypeInterfaceInstanceOrCopyAtomTypeInterfaceV2Instance:$copy_atom, + Variadic:$src, + Variadic:$dst, + Optional:$pred + ); + let results = (outs + ); +} +def Cute_AlgorithmGemmOp : Cute_Op<"gemm", [AttrSizedOperandSegments]> { + let description = [{ + CuTe's gemm algorithm takes 1 mma atom and 4 inputs(cute.memref/cute_nvgpu.smem_desc_view) + as input. + It performs the general matrix multiplication: + `D += A * B + C`. + Layout carried by operads determines the exact form of the gemm computation. + Generally speaking, there are 5 valid forms: + 1. `(V) x (V) => (V)`. + The element-wise product of vectors. Dispatches to FMA or MMA. + In this case, `rank(A) = rank(B) = rank(C) = rank(D) = 1`. + 2. `(M) x (N) => (M,N)`. + The outer product of vectors. Dispatches to [3] with new mode K=(1). + In this case, `rank(A) = rank(B) = 1` and `rank(C) = rank(D) = 2`. + 3. `(M,K) x (N,K) => (M,N)`. + The product of matrices. Dispatches to [5] with MMA vector-mode V. + In this case, `rank(A) = rank(B) = rank(C) = rank(D) = 2`. + 4. `(V,M) x (V,N) => (V,M,N)`. + The batched outer product of vectors. Accounts for register reuse and dispatches to [1] for each (m,n). + In this case, `rank(A) = rank(B) = 2` and `rank(C) = rank(D) = 3`. + 5. `(V,M,K) x (V,N,K) => (V,M,N)`. + The batched product of matrices. Dispatches to [4] for each (k). + In this case, `rank(A) = rank(B) = rank(C) = rank(D) = 3`. + + Currently, only the form in item 5 is supported. + + The operands `a` and `b` are variadic, each containing a variable number of tensors: + - For regular GEMM, `a` and `b` contain the GEMM A and B tensors respectively. + - For GEMM with auxiliary operands, `a` and `b` contain the GEMM A and B tensors followed by their respective auxiliary tensors. For example: + - For BlockScaledGemm, `a` = [A, SFA] and `b` = [B, SFB]. + - For SparseGemm, `a` = [A, E] and `b` = [B]. + - For BlockScaledSparseGemm, `a` = [A, SFA, E] and `b` = [B, SFB]. + See the file cutlass/media/docs/cute/04_algorithms.md and include/cute/algorithm/gemm.hpp in the CUTLASS repository for more information. + + Example (regular GEMM): + ```mlir + cute.gemm(%mma, %d, %a, %b, %c) { + } : (!cute.tiled_mma, + atom_layout_MNK = <"(1,1,1)">>, + !cute.memref, "((128,128),1,1):((131072,1),0,0)">, + !cute_nvgpu.smem_desc_view o 0 o ((128,16),1,4):((64,1),0,16)">, + !cute_nvgpu.smem_desc_view o 0 o ((128,16),1,4):((64,1),0,16)">, + !cute.memref, "((128,128),1,1):((131072,1),0,0)">) + ``` + + Example (block-scaled GEMM): + ```mlir + cute.gemm(%mma, %d, [%a, %sfa], [%b, %sfb], %c) : ( + !cute.tiled_mma, + atom_layout_MNK = <"(1,2,1):(0,1,0)">>, + !cute.memref, "((128,128),1,1):((65536,1),0,0)">, + [!cute_nvgpu.smem_desc_view, + !cute.memref, "((((32,4),4),(16,4)),1):((((262144,4),8388608),(0,1)),0)">], + [!cute_nvgpu.smem_desc_view, + !cute.memref, "((((32,4),4),(16,4)),1):((((262144,4),8388608),(0,1)),0)">], + !cute.memref, "((128,128),1,1):((65536,1),0,0)">) + ``` + }]; + let arguments = (ins + Cute_MmaAtomTypeInterfaceInstance:$mma_atom, + Cute_ViewTypeInterfaceInstance:$d, + Variadic:$a, + Variadic:$b, + Cute_ViewTypeInterfaceInstance:$c + ); + let results = (outs + ); +} +def Cute_AlgorithmPrefetchOp : Cute_Op<"prefetch"> { + let summary = "CuTe's prefetch algorithm prefetches the elements of source Tensor(s) into cache."; + let description = [{ + A Tensor encapsulates the data type, data location, and possibly + also the shape and stride of the tensor at compile time. As a result, prefetch can + and does dispatch, based on the types of its arguments, to use any of various + synchronous or asynchronous hardware prefetch instructions. + + An .prefetch accepts CutePrefetchAtomTypeinterface as first parameter. + + The `src` operand is variadic: + - For regular TMA prefetch, `src` is a single tensor. + - For 2D `tile::gather4` mode, `src` is `[data_coord_tensor, gmem_index_tensor]`, + mirroring `cute.copy` for the same mode. + + Example: + + ```mlir + cute.prefetch(%tiled_copy, %src) : ( + !cute.tiled_copy, + layout_copy_tv = <"((4,2),(64,128)):((0,64),(128,1))">, + tiler_mn = <"[2048:1;1:1;4:1;1:1]">>, + !cute.coord_tensor<"(0,0)", "(((64, 128), 1), 1, 1, 2):(((1@0, 1@1), 0), 0, 0, 64@0)">) + + // gather4 mode: data tensor + gmem index tensor + cute.prefetch(%tiled_copy, %src_data, %src_idx) : ( + !cute.tiled_copy, ...>, + !cute.coord_tensor<...>, + !cute.coord_tensor<...>) + ``` + }]; + let arguments = (ins + Cute_PrefetchAtomTypeInterfaceOrTiledCopyTypeWithPrefetchAtomTypeInterfaceCopyAtom:$prefetch_atom, + Variadic:$src + ); + let results = (outs + ); +} +def Cute_AppendToRankOp : Cute_InferredOp<"append_to_rank", [Pure]> { + let summary = "Append element to input to rank N."; + let description = [{ + Example: + + ```mlir + %shp = cute.make_shape() : !cute.shape<"(5,4,3)"> + %elem = cute.make_shape() : !cute.shape<"(2,1)"> + %new_layout = cute.append_to_rank<5> (%shp, %elem) + : (..., ...) -> !cute.shape<"(5,4,3,(2,1),(2,1))"> + ``` + }]; + let arguments = (ins + Cute_ValueTypeInterfaceInstance:$input, + Cute_ValueTypeInterfaceInstance:$element, + SI32Attr:$rank + ); + let results = (outs + Cute_ValueTypeInterfaceInstance:$result + ); +} +def Cute_ApplySwizzleOp : Cute_InferredOp<"apply_swizzle", [Pure]> { + let description = [{ + The `apply_swizzle` operation applies a swizzle transformation to a pointer's + address, producing a new pointer with the swizzled address. This is necessary + for dereferencing pointers that use swizzled memory layouts, such as those used + for improved memory access patterns on GPUs. + + Two forms are supported: + + 1. **CutePtrType form**: The swizzle is encoded in the pointer type itself. + No explicit swizzle operand is required or allowed. + + ```mlir + %swizzled = cute.apply_swizzle(%ptr) : !cute.ptr, S<3,4,3>> + ``` + + 2. **llvm.ptr form**: The swizzle is passed as an optional attribute in the + attribute dictionary. If the swizzle attribute is absent, the operation + is a no-op (identity swizzle). + + ```mlir + // With explicit swizzle: + %swizzled = cute.apply_swizzle(%ptr) { S<3,4,3> } : !llvm.ptr<3> + // Identity (no-op): + %swizzled = cute.apply_swizzle(%ptr) : !llvm.ptr<3> + ``` + + If the input pointer type has a non-identity swizzle attribute, this operation + computes the swizzled address by applying the swizzle transformation to the + pointer's address bits. The transformation is typically an XOR-based permutation + of certain address bits, as defined by the swizzle parameters. + + Swizzle is parameterized by three values: + - MBase: The number of least-significant bits to keep constant (not affected by the swizzle) + - BBits: The number of bits in the mask (the width of the swizzle region) + - SShift: The distance to shift the mask (positive values shift right, negative values shift left) + + The swizzle transformation operates on the address as follows: + + 0bxxxxxxxxxxxxxxxYYYxxxxxxxZZZxxxx + ^--^ MBase: least-significant bits kept constant + ^-^ ^-^ BBits: width of the mask (YYY and ZZZ) + ^---------^ SShift: shift distance for the YYY mask + + The swizzled address is computed by XOR'ing the ZZZ and YYY bitfields: + + Example: + + ``` + 0bxxxxxxxxxxxxxxxxYYxxxxxxxxxZZxxx + + the result is + 0bxxxxxxxxxxxxxxxxYYxxxxxxxxxAAxxx where AA = ZZ `xor` YY + ``` + + This operation is typically used just before dereferencing a pointer, + ensuring that the correct physical memory location is accessed according + to the swizzle pattern. If the pointer type has an identity swizzle, + this operation is a no-op and returns the original pointer. + }]; + let arguments = (ins + Cute_CuTePointerTypeOrLLVMPointerType:$ptr, + OptionalAttr:$swizzle + ); + let results = (outs + Cute_CuTePointerTypeOrLLVMPointerType:$swizzled_ptr + ); +} +def Cute_AssumeOp : Cute_Op<"assume", [Pure]> { + let description = [{ + `cute.assume` represents assumptions such as divisibility for an SSA value. + unlike `llvm.assume`, `cute.assume` produces a strong typed value as output. + It is limited to support all constrains defined by `ConstrainedIntType`. + + Please see [constrained integer type](### ConstrainedIntType) for details. + + Example: + ```mlir + // Claim %1 is divisible by 8 + %1 = cute.assume(%0 : i32) : !cute.i32 + + ``` + + Note: + + // xxx: need to revisit + + ```mlir + %0 = ... -> !cute.i32 + if (%cond) { + %1 = cute.assume(%0 : !cute.i32) : !cute.i32 + ... // %1 has the assumption. + + // %0 is not assumed here + %data = load %ptr[%0] : ? + } + + // %0 here does not have any assumption. + ``` + }]; + let arguments = (ins + Cute_Integer:$src + ); + let results = (outs + Cute_IntegerTypeWithConstraints:$dst + ); +} +def Cute_BlockedProductOp : Cute_InferredOp<"blocked_product", [Pure]> { + let description = [{ + This operation performs "blocked_product" on the input and produce a blocked_product result, + and it automatically infer the "blocked_product" layout for the result type. + + Example: + + ```mlir + %input = cute.make_layout() : !cute.layout<"(3,4):(4,1)"> + %tiler = cute.make_layout() : !cute.layout<"(2,5):(1,2)"> + %bprod = cute.blocked_product(%input, %tiler) + : (!cute.layout<"(3,4):(4,1)">, !cute.layout<"(2,5):(1,2)">) + -> !cute.layout<"((3,2),(4,5)):((4,12),(1,24))"> + ``` + + Example with dynamic composed_layout: + + ``` + %bprod = cute.blocked_product(%input, %tiler) + : (!cute.composed_layout<"(?):(?) o ? o (5,4):(4,1)">, !cute.layout<"(?,?):(?,?)">) + -> !cute.composed_layout<"(?):(?) o ? o ((5,?),(4,?)):((4,?{div=20}),(1,?{div=20}))"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$input, + Cute_CuTeLayoutType:$tiler + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_CeilDivOp : Cute_InferredOp<"ceil_div", [Pure]> { + let arguments = (ins + Cute_IntTupleAsIntegerOrTupleOfIntTupleS2:$input, + Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$tiler + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS2:$result + ); +} +def Cute_CoalesceOp : Cute_InferredOp<"coalesce", [Pure]> { + let summary = "`coalesce` op simplifies layout by combining modes that are possible to combine."; + let description = [{ + It does not respect the shape of the layout but preserve total size. + + ```mlir + // coalesce modes with contiguous/packed strides from left to right + %0 = cute.coalesce(%arg0) + : (!cute.layout<"((4,5),3):((1,4),20)">) -> !cute.layout<"60:1"> + + // coalesce from left to right but not right to left + %0 = cute.coalesce(%arg0) + : (!cute.layout<"(3,(4,5)):(20,(1,4))">) -> !cute.layout<"(3,20):(20,1)"> + %0 = cute.coalesce(%arg0, %arg1) + : (!cute.layout<"(3,(4,5)):(20,(1,4))">, !cute.coord<"(1,1)">) -> !cute.layout<"(3,20):(20,1)"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance:$input, + Optional:$target_profile + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance:$result + ); +} +def Cute_ComplementOp : Cute_InferredOp<"complement", [Pure]> { + let summary = "Produces a disjoint layout."; + let description = [{ + The complement of a layout L is a layout L* such that (L,L*) achieves greater surjectivity. + That is, L* maps to elements in the codomain of L that are not in the image of L. + + It's worth to note that (L,L*) isn't alway surjective. + + Examples: + ``` + // trivial complement of L under it's domain is 1:0 + %res = cute.complement(%arg0, %arg1) + : (!cute.layout<"3:1">, !cute.shape<"3">) -> !cute.layout<"1:0"> + + // 3:2 is strided layout. complement of 3:2 under 6 is 2:1 filling stride + // (L,L*) => (3,2):(2,1) is a surjective layout + %res = cute.complement(%arg0, %arg1) + : (!cute.layout<"3:2">, !cute.shape<"6">) -> !cute.layout<"2:1"> + + %res = cute.complement(%arg0, %arg1) + : (!cute.layout<"(3,(5,2)):(1,(3,15))">, !cute.shape<"(6,20)">) -> !cute.layout<"4:30"> + ``` + }]; + let arguments = (ins + Cute_CuTeLayoutType:$input, + Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$cotarget + ); + let results = (outs + Cute_CuTeLayoutType:$res + ); +} +def Cute_ComposedGetInnerOp : Cute_InferredOp<"composed_get_inner", [Pure]> { + let summary = "Gets inner layout from a layout."; + let description = [{ + Example: + + ```mlir + %layout = cute.make_layout(.., ..): !cute.layout<"(32,64):(1,32)>"> + %layout_a = cute.composed_get_layout_a(%layout) : (!cute.composed_layout<"(64,32):(1,64) o 0 o (32,64):(1,32)>">) -> !cute.layout<"(64,32):(1,64)"> + %offset = cute.composed_get_offset(%layout) : (!cute.composed_layout<"S<3,4,3> o 7 o (32,64):(1,32)>">) -> !cute.shape<"7"> + %layout_b = cute.composed_get_layout_b(%layout) : (!cute.composed_layout<"(64,32):(1,64) o 0 o (32,64):(1,32)>">) -> !cute.layout<"(32,64):(1,32)"> + ``` + }]; + let arguments = (ins + Cute_CuTeComposedLayoutType:$input + ); + let results = (outs + Cute_CuTeLayoutTypeOrCuteSwizzleType:$result + ); +} +def Cute_ComposedGetOffsetOp : Cute_InferredOp<"composed_get_offset", [Pure]> { + let summary = "Gets offset from a layout."; + let description = [{ + Example: + + ```mlir + %layout = cute.make_layout(.., ..): !cute.layout<"(32,64):(1,32)>"> + %layout_a = cute.composed_get_layout_a(%layout) : (!cute.composed_layout<"(64,32):(1,64) o 0 o (32,64):(1,32)>">) -> !cute.layout<"(64,32):(1,64)"> + %offset = cute.composed_get_offset(%layout) : (!cute.composed_layout<"S<3,4,3> o 7 o (32,64):(1,32)>">) -> !cute.shape<"7"> + %layout_b = cute.composed_get_layout_b(%layout) : (!cute.composed_layout<"(64,32):(1,64) o 0 o (32,64):(1,32)>">) -> !cute.layout<"(32,64):(1,32)"> + ``` + }]; + let arguments = (ins + Cute_CuTeComposedLayoutType:$input + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$result + ); +} +def Cute_ComposedGetOuterOp : Cute_InferredOp<"composed_get_outer", [Pure]> { + let summary = "Gets layout outer from composed layout."; + let description = [{ + Example: + + ```mlir + %layout_outer = cute.composed_get_outer(%layout) + : (!cute.composed_layout<"(64,32):(1,64) o 0 o (32,64):(1,32)>">) -> !cute.layout<"(32,64):(1,32)"> + ``` + }]; + let arguments = (ins + Cute_CuTeComposedLayoutType:$input + ); + let results = (outs + Cute_CuTeLayoutType:$result + ); +} +def Cute_CompositionOp : Cute_InferredOp<"composition", [Pure]> { + let description = [{ + The `composition` operation computes the composition of two layouts + (`!cute.layout`), in the following called `lhs` and `rhs`. + + Examples: + ``` + // Composition of two static layouts. + %0 = cute.composition(%arg0, %arg1) + : (!cute.layout<"(20):(2)">, !cute.layout<"(5,4):(4,1)">) -> !cute.layout<"(5,4):(8,2)"> + + // Composition of a static and a dynamic layout. + %1 = cute.composition(%arg0, %arg1) + : (!cute.layout<"(20):(2)">, !cute.layout<"(?,?):(?,?)">) -> !cute.layout<"(?,?):(?{div=2},?{div=2})"> + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$lhs, + Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$rhs + ); + let results = (outs + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$res + ); +} +def Cute_CopyAtomCallOp : Cute_Op<"copy_atom_call", [AttrSizedOperandSegments]> { + let summary = "`cute.copy_atom_call` op executes a copy atom operation with the given operands."; + let description = [{ + Following src/dst layout of atom are valid: + * ((atom_v, rest_v)) + * ((atom_v)) + * (atom_v) + + Both `src` and `dst` operands are variadic, containing a variable number of tensors: + + - For regular copy, `src` and `dst` each contain a single tensor. + - For copy with auxiliary operands, they contain the main tensor followed by + auxiliary tensors. For example: + + - For load with row-wise reduction, `dst` = [data, stat]. + - For SPARSIFY (structured sparsity compression), `dst` = [data, metadata]. + - For TMA gather4, `src` = [coord0, coord1, coord2, coord3] (four 2D coordinate tensors). + - For TMA scatter4, `dst` = [coord0, coord1, coord2, coord3] (four 2D coordinate tensors). + + Examples: + + ```mlir + // Regular copy atom operation + cute.copy_atom_call(%copy_load, %src_v, %dst_v) + : (!cute_nvgpu.atom.tmem_load, + !cute.memref, "(((2,32),1)):(((1,131072),0))">, + !cute.memref, "((2,1)):((1,0))">) + -> () + ``` + + ```mlir + // TMA gather4: combine four 2D coordinate tensors into single destination + cute.copy_atom_call(%tma_gather4_atom, [%coord0, %coord1, %coord2, %coord3], %dst_v) + : (!cute_nvgpu.atom.tma_gather4<...>, + [!cute.coord_tensor<...>, !cute.coord_tensor<...>, + !cute.coord_tensor<...>, !cute.coord_tensor<...>], + !cute.memref) + -> () + ``` + }]; + let arguments = (ins + Cute_CopyAtomTypeInterfaceInstanceOrCopyAtomTypeInterfaceV2Instance:$atom, + Variadic:$src, + Variadic:$dst, + Optional:$pred + ); + let results = (outs + ); +} +def Cute_CopyMakeFragmentOp : Cute_InferredOp<"copy.make_fragment", [Pure]> { + let description = [{ + Construct a copy fragment for the S or D side from either a partitioned view or an abstract TV + description. Intended for use between `cute.tiled.copy.partition_{S,D}` and `cute.copy`, but + also supports layout/shape inputs for RMEM/TMEM fragment construction similar to MMA. + + The op accepts: + 1. an operand id `S` or `D` (printed as a keyword); + 2. a copy atom value; + 3. either a partitioned view or a layout/shape to build the fragment. + + Returns a new ViewTypeInterface derived value to be consumed by `cute.copy`. + + Examples: + ```mlir + // Load from global memory (gmem -> rmem): build S and D fragments from S-partition + %atom_load = cute.make_atom() : () -> !cute_nvgpu.atom.universal_copy + %tiled_copy_load = cute.make_tiled_copy(%atom_load) + : !cute.tiled_copy, + layout_copy_tv = <"(128,8):(1,128)">, tiler_mn = <"[128;8]">> + %tGSrc_load = cute.tiled.copy.partition_S(%tiled_copy_load, %gSrc, %tid_coord) + : (!cute.tiled_copy, + layout_copy_tv = <"(128,8):(1,128)">, tiler_mn = <"[128;8]">>, + !cute.memref, "(128,128):(?{div=8},1)">, !cute.coord<"?">) + -> !cute.memref, "((8,1),1,16):((1,0),0,8)"> + + %frgS_load = cute.copy.make_fragment S (%atom_load, %tGSrc_load) : ( + !cute_nvgpu.atom.universal_copy, + !cute.memref, "((8,1),1,16):((1,0),0,8)"> + ) -> !cute.memref, "((8,1),1,16):((1,0),0,8)"> + + %rLoad = cute.copy.make_fragment D (%atom_load, %tGSrc_load) : ( + !cute_nvgpu.atom.universal_copy, + !cute.memref, "((8,1),1,16):((1,0),0,8)" + ) -> !cute.memref, "((8,1),1,16):((1,0),0,8)" + ``` + }]; + let arguments = (ins + Cute_CopyAtomTypeInterfaceInstanceOrCopyAtomTypeInterfaceV2Instance:$atom, + Cute_AMemoryReferenceTypeThatRepresentsATensor:$input, + Cute_CopyOperandEnum:$operand_id + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_CosizeOp : Cute_InferredOp<"cosize", [Pure]> { + let description = [{ + This operation calculates the cosize of a layout. + * For a layout `l`, cosize(l) = l(size(l) - 1) + 1. + * For a layout `l` with mode `mode`, cosize(l) = cosize(get(l)). + * For a composed layout 'l = a o b', cosize(l) = cosize(b). + + The `mode` attribute specifies the nested mode in the type for cosize calculation. + It indexes a sub-scope within the input type and returns the cosize of that sub-scope. + The rank of `mode` must be less than or equal to the depth of the type. + If `mode` is empty, the operation calculates the cosize of the entire type. + + ### Examples + + ```mlir + %input: !cute.composed_layout<"(100):(1) o 0 o (3,4):(4,1)"> + %0 = cute.cosize(%input) + : !cute.composed_layout<"(100):(1) o 0 o (3,4):(4,1)"> + // cosize(%input) = cosize((3,4):(4,1)), ret is 12 + ``` + + Example with mode: + ```mlir + %input: !cute.layout<"((4,3),((3,(2,2)),2)):((1,4),((12,(36,72)),144))"> + %0 = cute.cosize(%input) <{mode=[1,0,1]}> + : (!cute.layout<"((4,3),((3,(2,2)),2)):((1,4),((12,(36,72)),144))">) -> !cute.int_tuple<"109"> + // cosize(%input) = cosize((2,2):(36,72)), ret is 109 + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, + DefaultValuedAttr:$mode + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$result + ); +} +def Cute_Crd2IdxOp : Cute_InferredOp<"crd2idx", [Pure]> { + let description = [{ + Compute the index of a coordinate with respect to the given layout. I.e., + compute the natural coordinate of `coord` wrt. to the shape + and then compute the dot product with the strides. + + Examples: + ```mlir + // %crd0 is already a natural coordinate for %shape. + %0 = cute.crd2idx(%crd, %layout): (!cute.coord<"(?,(?,?))">, !cute.layout<"(?,(?,?)):(?,(?,?))">) -> (i32) + // Compute the natural coordinate, then dot product. + %1 = cute.crd2idx(%crd, %layout): (!cute.coord<"(?,?)">, !cute.layout<"(?,(?,?)):(?,(?,?))">) -> (i32) + // Static and dynamic shape/strides are supported. + %3 = cute.crd2idx(%crd, %layout): (!cute.coord<"(?,?)">, !cute.layout<"(4,2):(1,2)">) -> (i32) + // Strides with scaled basis produces a tuple. + %5 = cute.crd2idx(%crd, %layout) : (!cute.coord<"(?,?,?)">, !cute.layout<"(10,20,30):(1@0,1@1,20@1)">) -> (!cute.int_tuple<"(?,?)">) + ``` + }]; + let arguments = (ins + Cute_CuteCoordType:$coord, + Cute_GeneralizedLayoutTypeOfCuTe:$layout + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$index + ); +} +def Cute_DereferenceArithTupleIteratorOp : Cute_InferredOp<"deref_arith_tuple_iter", [Pure]> { + let summary = "Dereferences an arithmetic tuple iterator to return the int_tuple value it holds."; + let description = [{ + Example: + + ```mlir + %0 = cute.deref_arith_tuple_iter(%arg0) : !cute.arith_tuple_iter<"(?,4)"> -> !cute.int_tuple<"(?,4)"> + ``` + }]; + let arguments = (ins + Cute_CuTeIteratorTypeWithIntTupleAsIts:$iter + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$arith_tuple + ); +} +def Cute_DereferenceDescriptorIteratorOp : Cute_InferredOp<"deref_desc_iter", [Pure]> { + let description = [{ + This Operation dereferences a descriptor iterator to return the descriptor + value it holds. + }]; + let arguments = (ins + Cute_ATypeInterfaceForDescriptorIteratorTypes:$iter + ); + let results = (outs + AnyType:$value + ); +} +def Cute_DerefineOp : Cute_Op<"derefine", [Pure]> { + let description = [{ + The `cute.derefine` operation discard static information from modes + of a given cute algebra type like shape, stride, layout, coord or tile into + modes with less static information. + + For each mode, it either cast it into dynamic mode or keep it unchanged. In future, + it can be extended to support removal of `divby N` from dynamic mode. + + Casting dynamic mode into static mode is illegal. + + E.g. + + ```mlir + %dyn_0 = cute.derefine(%0) : !cute.shape<"(3,5)"> to !cute.shape<"(?,5)"> + %dyn_1 = cute.derefine(%1) : !cute.stride<"(?,5)"> to !cute.stride<"(?,?)"> + %dyn_2 = cute.derefine(%2) : !cute.coord<"(8,_)"> to !cute.coord<"(?,_)"> + ``` + }]; + let arguments = (ins + Cute_RefinableTypeInterfaceInstance:$src + ); + let results = (outs + Cute_RefinableTypeInterfaceInstanceOrInteger:$dst + ); +} +def Cute_DiceOp : Cute_InferredOp<"dice", [Pure]> { + let description = [{ + This operation dices a type implementing the `CuteValueTypeInterface` (ShapeType, IntTupleType, + LayoutType,...) using a static coordinate attribute. The profile of said coordinate must be + weakly congruent with that of `input`. Modes associated with an underscore are discarded in the + result. + }]; + let arguments = (ins + Cute_ValueTypeInterfaceInstance:$input, + Cute_CoordAttr:$coord + ); + let results = (outs + Cute_ValueTypeInterfaceInstance:$result + ); +} +def Cute_ElemLessOp : Cute_Op<"elem_less", [Pure]> { + let description = [{ + The cute.elem_less operation compares every elements in Lhs to the + corresponding elements in Rhs. The result is 1 if every Lhs elements + are less than corresponding Rhs elements, otherwise result is 0. + + `ElemLessOp` is useful when we want to know if a coordinate is out of bound: + + ```mlir + %is_in_bound = cute.elem_less(%coord, %shape) : !cute.coord<"(?,?)">, !cute.shape<"(?,?)"> + ``` + }]; + let arguments = (ins + Cute_CuteShapeTypeOrIntTupleAsIntegerOr:$lhs, + Cute_CuteShapeTypeOrIntTupleAsIntegerOr:$rhs + ); + let results = (outs + I1:$result + ); +} +def Cute_EqualOp : Cute_Op<"equal", [Pure]> { + let description = [{ + Checks two dynamic cute values for equality. Full static values are considered + equal only when their types are the same. + + Example: + + ```mlir + %1 = cute.make_shape : !cute.shape<"(32, 64)"> + %2 = cute.make_shape : !cute.shape<"(32, 64)"> + %result = cute.equal %1, %2 : !cute.shape<"(32,64)">, !cute.shape<"(32,64)"> + ``` + + The generated code only checks profile positions in which at least one side has a dynamic value. + Static comparisons are performed at compile time by the operation folder. + + ```mlir + %1 = cute.make_shape : !cute.shape<"(?, 64)"> + %2 = cute.make_shape : !cute.shape<"(?, 64)"> + %result = cute.equal %1, %2 : !cute.shape<"(?,64)">, !cute.shape<"(?,64)"> + ``` + + This would result in the first element being checked for equality in the runtime representation of the shapes. + + Note that this is dynamic if either of the operands will force that operand to be checked. So, + + ```mlir + %1 = cute.make_shape : !cute.shape<"(?, 64)"> + %2 = cute.make_shape : !cute.shape<"(64, ?)"> + %result = cute.equal %1, %2 : !cute.shape<"(?,64)">, !cute.shape<"(64,?)"> + ``` + + will result in both dimensions being checked. + }]; + let arguments = (ins + Cute_ValueTypeInterfaceInstance:$lhs, + Cute_ValueTypeInterfaceInstance:$rhs + ); + let results = (outs + I1:$result + ); +} +def Cute_FastDivmodComputeOp : Cute_Op<"fast_divmod.compute", [Pure]> { + let description = [{ + This operation performs a fast divmod using a fast divmod divisor. It computes the quotient and + the remainder without emitting division instructions. The quotient and the remainder are of the + same width as the divisor. + }]; + let arguments = (ins + Cute_32BitSignlessIntegerOr64BitSignless:$dividend, + Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant:$divisor + ); + let results = (outs + Cute_32BitSignlessIntegerOr64BitSignless:$quotient, + Cute_32BitSignlessIntegerOr64BitSignless:$remainder + ); +} +def Cute_FastDivmodCreateDivisorOp : Cute_Op<"fast_divmod.create_divisor", [Pure]> { + let description = [{ + This operation creates a fast divmod divisor from an integer. It is meant to be pre-computed, + ideally in host code, and then used throughout the lifetime of a device kernel for performing + divmods without emitting division instructions. + }]; + let arguments = (ins + Cute_32BitSignlessIntegerOr64BitSignless:$divisor + ); + let results = (outs + Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant:$fast_divmod_divisor + ); +} +def Cute_FastDivmodDivideOp : Cute_Op<"fast_divmod.divide", [Pure]> { + let description = [{ + This operation performs a fast divide using a fast divmod divisor. It computes the quotient + without emitting division instructions. The quotient is of the same width as the divisor. + }]; + let arguments = (ins + Cute_32BitSignlessIntegerOr64BitSignless:$dividend, + Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant:$divisor + ); + let results = (outs + Cute_32BitSignlessIntegerOr64BitSignless:$quotient + ); +} +def Cute_FastDivmodGetAuxOp : Cute_InferredOp<"fast_divmod.get_aux", [Pure]> { + let arguments = (ins + Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant:$divisor + ); + let results = (outs + Cute_32BitSignlessIntegerOr64BitSignless:$multiplier, + I8:$shift1, + I8:$shift2 + ); +} +def Cute_FastDivmodGetDivisorOp : Cute_InferredOp<"fast_divmod.get_divisor", [Pure]> { + let arguments = (ins + Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant:$fast_divmod_divisor + ); + let results = (outs + Cute_32BitSignlessIntegerOr64BitSignless:$divisor + ); +} +def Cute_FastDivmodMakeDivisorOp : Cute_Op<"fast_divmod.make_divisor", [Pure]> { + let arguments = (ins + Cute_32BitSignlessIntegerOr64BitSignless:$divisor, + Cute_32BitSignlessIntegerOr64BitSignless:$multiplier, + I8:$sh1, + I8:$sh2 + ); + let results = (outs + Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant:$fast_divmod_divisor + ); +} +def Cute_FilterOp : Cute_InferredOp<"filter", [Pure]> { + let summary = "This operation filters out dimensions with 0-strides and 1-sizes from a layout or view."; + let description = [{ + When working with tensor layouts, some dimensions may have stride 0 (indicating + broadcasting) or size 1 (singleton dimensions). This operation creates a new layout + or view with these dimensions removed, which can simplify subsequent operations. + + OptionalOperand `target_profile` is the user indicated filter control, with this operand, + the original layout's stride is ignored in the zero-stride selection. + But the `target_profile` will not take effect currently. This is a placeholder for future use. + + This operation accepts both layout and view types as valid input. + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance:$input + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance:$result + ); +} +def Cute_FilterZerosOp : Cute_InferredOp<"filter_zeros", [Pure]> { + let description = [{ + This operation produces a transformed layout or view by replacing the shape + of any mode with stride zero and non-unit extent with 1. This is commonly + applied prior to copy operations to prevent redundant or duplicated + data movement arising from broadcasted dimensions. + + The optional operand `target_profile` allows the user to specify a stride + filter profile; when provided, the selection of zero-stride dimensions is + determined by this stride profile rather than the original layout's stride + information. + + This operation accepts both layout and view types as input. + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance:$input, + Optional:$target_profile + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance:$res + ); +} +def Cute_FlatDivideOp : Cute_InferredOp<"flat_divide", [Pure]> { + let summary = "This operation performs a flat_divide on a view or a layout by a tiler."; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, + Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$tiler + ); + let results = (outs + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_FlatProductOp : Cute_InferredOp<"flat_product", [Pure]> { + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$input, + Cute_CuTeLayoutType:$tiler + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_GetFlatCoordOp : Cute_InferredOp<"get_flat_coord", [Pure]> { + let description = [{ + This OP accept a scalar index and a normal layout, + returns a flat coord. + + + The "flat" here means the reversed coordinate will be depth=1 coordinate. + This is useful when we just care about the rank semantic of the layout and don't care about the detail profile. + + + Example: + + ```mlir + %layout = cute.make_layout() : !cute.layout<"(5,(4,3),2):(1,(5,20),60)"> + %index = arith.constant 117 : i32 + %flat_coord = cute.get_flat_coord(%index, %layout) : (..., ...) -> !cute.coord<"(2,11,1)"> + %hier_coord = cute.get_hire_coord(%index, %layout) : (..., ...) -> !cute.coord<"(2,(3,2),1)"> + %integral_coord = cute.get_integral_coord(%index, %layout) : (..., ...) -> !cute.coord<"117"> + ``` + }]; + let arguments = (ins + Cute_32BitSignlessIntegerOr64BitSignless:$index, + Cute_CuTeLayoutType:$input + ); + let results = (outs + Cute_CuteCoordType:$result + ); +} +def Cute_GetHierCoordOp : Cute_InferredOp<"get_hier_coord", [Pure]> { + let description = [{ + This OP accept a scalar index and a normal layout, + returns a hier coord. + + + The "hier" here means the reversed coordinate will have the same profile with the given layout. + This is useful when we care about both the rank semantic and the detail layout profile. + + + Example: + + ```mlir + %layout = cute.make_layout() : !cute.layout<"(5,(4,3),2):(1,(5,20),60)"> + %index = arith.constant 117 : i32 + %flat_coord = cute.get_flat_coord(%index, %layout) : (..., ...) -> !cute.coord<"(2,11,1)"> + %hier_coord = cute.get_hire_coord(%index, %layout) : (..., ...) -> !cute.coord<"(2,(3,2),1)"> + %integral_coord = cute.get_integral_coord(%index, %layout) : (..., ...) -> !cute.coord<"117"> + ``` + }]; + let arguments = (ins + Cute_32BitSignlessIntegerOr64BitSignless:$index, + Cute_CuTeLayoutType:$input + ); + let results = (outs + Cute_CuteCoordType:$result + ); +} +def Cute_GetIntegralCoordOp : Cute_InferredOp<"get_integral_coord", [Pure]> { + let description = [{ + This OP accept a scalar index and a normal layout, + returns a integral coord. + + + The "integral" here means the reversed coordinate will be a "scalar". + This is useful when we don't care about the layout's semantic and profile and we just want to know the offset in "domain". + + + Example: + + ```mlir + %layout = cute.make_layout() : !cute.layout<"(5,(4,3),2):(1,(5,20),60)"> + %index = arith.constant 117 : i32 + %flat_coord = cute.get_flat_coord(%index, %layout) : (..., ...) -> !cute.coord<"(2,11,1)"> + %hier_coord = cute.get_hire_coord(%index, %layout) : (..., ...) -> !cute.coord<"(2,(3,2),1)"> + %integral_coord = cute.get_integral_coord(%index, %layout) : (..., ...) -> !cute.coord<"117"> + ``` + }]; + let arguments = (ins + Cute_32BitSignlessIntegerOr64BitSignless:$index, + Cute_CuTeLayoutType:$input + ); + let results = (outs + Cute_CuteCoordType:$result + ); +} +def Cute_GetIterOp : Cute_InferredOp<"get_iter", [Pure]> { + let description = [{ + The "cute.get_iter" gets the underlying iterator from memref or coord_tensor. + It is solely as step during lowering, it has no side effects. + + Example: + + ```mlir + %ptr = cute.get_iter(%0) : !cute.memref, "((64,128),2):((1,1),64)"> + + // Or for TMA tensor + + %arith_tuple = cute.get_iter(%0) : !cute.coord_tensor<"(128, 64)", "((64,64),1):((1@0,1@0),64@1)"> + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstance:$source + ); + let results = (outs + Cute_GeneralizedIteratorTypeOfCuTe:$ptr + ); +} +def Cute_GetLayoutOp : Cute_InferredOp<"get_layout", [Pure]> { + let summary = "Gets the layout out of a `CuteViewTypeInterface`."; + let description = [{ + Example: + + ```mlir + %0 = cute.get_layout(%arg0) : !cute.memref + + %0 = cute.get_layout(%arg0) : !cute.memref o 0 o (256,?):(?,1)"> + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstance:$input + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$layout + ); +} +def Cute_GetLayoutsFromTileOp : Cute_InferredOp<"get_layouts_from_tile", [Pure]> { + let description = [{ + This operation extracts the layout elements out of a tile. A tile is a + layout, an underscore, or a tuple of tiles. + + Examples: + + - the type of the tile is !cute.tile<"[1:0;2:1]"> + 2 layouts are extracted of type !cute.layout<"1:0"> and !cute.layout<"2:1">, respectively + - the type of the tile is !cute.tile<[_;[1:0;2:1];3:1]> + 3 layouts are extracted of type !cute.layout<"1:0">, !cute.layout<"2:1">, and + !cute.layout<"3:1">, respectively + }]; + let arguments = (ins + Cute_CuteTileType:$tile + ); + let results = (outs + Variadic:$layouts + ); +} +def Cute_GetLeavesOp : Cute_InferredOp<"get_leaves", [Pure]> { + let description = [{ + The `cute.get_leaves` operation extracts all leaf values from a hierarchical tuple into + a flat sequence of results. Each result corresponds to a leaf value found in the input tuple, + preserving the order of a depth-first traversal. + + The operation handles nested tuples of any depth, including tuples containing other tuples. + The type of each result matches the type of the corresponding leaf value in the input. + + Example: + ```mlir + // Extract leaves from a hierarchical shape tuple + %0:3 = cute.get_leaves(%hier) : !cute.shape<"(3,?{div=4},5)"> + -> (!cute.shape<"3">, !cute.shape<"?{div=4}">, !cute.shape<"5">) + + // Extract leaves from a nested int tuple + %1:4 = cute.get_leaves(%t) : !cute.int_tuple<"((1,2),(3,4))"> + -> (!cute.int_tuple<"1">, !cute.int_tuple<"2">, + !cute.int_tuple<"3">, !cute.int_tuple<"4">) + ``` + }]; + let arguments = (ins + Cute_XTupleLikeType:$input + ); + let results = (outs + Variadic:$results_ + ); +} +def Cute_GetOp : Cute_Op<"get", [Pure]> { + let description = [{ + This operation returns a potentially nested mode out of a type implementing + the `CuteValueTypeInterface`, e.g. a shape, a tuple, a layout,... + + The optional `mode` attribute indices are used to select the portion of the + input type. The number of indices in the `mode` attribute should be less + than or equal to the depth of type. If `mode` is empty, a copy of the input + value is returned. + + Example: + + ```mlir + %shape = cute.make_shape() : () -> !cute.shape<"(1, (2, 4), 8)"> + %mode0 = cute.get(%shape) <{mode=[1]}> + : !cute.shape<"(1,(2,4),8)"> -> !cute.shape<"(2,4)"> + %mode1 = cute.get(%shape) <{mode=[1,1]}> + : !cute.shape<"(1,(2,4),8)"> -> !cute.shape<"4"> + ``` + }]; + let arguments = (ins + Cute_ValueTypeInterfaceInstance:$input, + OptionalAttr:$mode + ); + let results = (outs + Cute_ValueTypeInterfaceInstanceOrIntegerTypeWithConstraintsOr:$result + ); +} +def Cute_GetScalarsOp : Cute_InferredOp<"get_scalars", [Pure]> { + let summary = "Gets the scalars out of a value with CuteValueTypeInterface."; + let description = [{ + This operation returns only the dynamic scalars when `only_dynamic` is set. + + A "scalar" in this context refers to a single value used as part of the + state of a type. In general, each state value held by a type will be + considered a scalar if it is printed in the text format. This includes: + - static integers + - dynamic integers + - swizzle parameters + - scaled basis value (static integer, ratio (num + denom), dynamic integer) + - scaled basis modes + - indexed pointer address (always a dynamic value) + - indexed pointer stride (dynamic integer or dynamic) + + Note that underscore values in coordinate types are NOT considered scalars. + + Example: + + ```mlir + %1 = cute.get_shape(%0) : (!cute.layout<"(256,?):(?,1)">) -> !cute.shape<"(256,?)"> + %2 = cute.get_scalars(%1) <{only_dynamic}> : (!cute.shape<"(256,?)">) -> i32 + ``` + }]; + let arguments = (ins + Cute_ValueTypeInterfaceInstanceOrIntegerTypeWithConstraints:$cute_value, + UnitAttr:$only_dynamic + ); + let results = (outs + Variadic:$scalars + ); +} +def Cute_GetShapeOp : Cute_InferredOp<"get_shape", [Pure]> { + let summary = "get shape from view, layout or tile"; + let description = [{ + Example: + + ```mlir + %shape = cute.get_shape(%layout) : (!cute.layout<"(32,64):(1,32)>">) -> !cute.shape<"(32,64)"> + %shape = cute.get_shape(%tile) + : (!cute.tile<"[(32,64):(1,32);3:1]>">) -> !cute.shape<"((32,64),3)"> + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe2:$input + ); + let results = (outs + Cute_CuteShapeType:$result + ); +} +def Cute_GetStrideOp : Cute_InferredOp<"get_stride", [Pure]> { + let summary = "get stride from layout."; + let description = [{ + Please be noted that stride is not generalized for view and layout because composed layout + doesn't have generalized layout concept. + + Example: + + ```mlir + %stride = cute.get_stride(%layout) : (!cute.layout<"(32,64):(1,32)>">) -> !cute.stride<"(1, 32)"> + ``` + }]; + let arguments = (ins + Cute_CuTeLayoutType:$input + ); + let results = (outs + Cute_CuteStrideType:$result + ); +} +def Cute_GroupModesOp : Cute_InferredOp<"group_modes", [Pure]> { + let description = [{ + This operation will make the input view/layout's [B,E) modes + into one new mode with one-depth deeper. + + Example: + + ```mlir + %res = cute.group_modes(%arg0) <0,1> + : (!cute.coord_tensor<"(0,0)", "(512,64,2):(1@1,1@0,64@0)">) + -> !cute.coord_tensor<"(0,0)", "((512),64,2):((1@1),1@0,64@0)"> + %res = cute.group_modes(%arg0) <0,2> + : (!cute.composed_layout<"(3,4,5):(6,7,8) o (0,1,2) o (9,9,9):(1@1,1@0,1@2)">) + -> !cute.composed_layout<"(3,4,5):(6,7,8) o (0,1,2) o ((9,9),9):((1@1,1@0),1@2)"> + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, + I32Attr:$begin, + I32Attr:$end + ); + let results = (outs + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_Idx2CrdOp : Cute_InferredOp<"idx2crd", [Pure]> { + let description = [{ + compute the coordinate corresponding to an index with respect to the given shape. I.e., + compute the coordinate that maps to the given index using given shape and default stride. + Note: This only works for compact layout. + + Examples: + ```mlir + // Convert a linear index to coordinate using shape and default stride. + %0 = cute.idx2crd(%index, %shape): (!cute.int_tuple<"?">, !cute.shape<"(4,2)">) -> (!cute.coord<"(?,?)">) + // Static and dynamic shape are supported. + %1 = cute.idx2crd(%index, %shape): (!cute.int_tuple<"?">, !cute.shape<"(?,(?,?))">) -> (!cute.coord<"(?,(?,?))">) + ``` + }]; + let arguments = (ins + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$index, + Cute_CuteShapeType:$shape + ); + let results = (outs + Cute_CuteCoordType:$coord + ); +} +def Cute_IncrementCoordOp : Cute_InferredOp<"increment_coord", [Pure]> { + let summary = "This operation colexicographically increments a coord within a coord space defined by a shape."; + let description = [{ + Example: + + ```mlir + // With static coord and shape + %coord1 = cute.increment_coord(%coord0, %shape) : + (!cute.coord<"(0,0)">, !cute.shape<"(2,2)">) -> !cute.coord<"(1,0)"> + %coord2 = cute.increment_coord(%coord1, %shape) : + (!cute.coord<"(1,0)">, !cute.shape<"(2,2)">) -> !cute.coord<"(0,1)"> + + // With dynamic coord or shape + %coord1 = cute.increment_coord(%coord0, %shape) : + (!cute.coord<"(?,?)">, !cute.shape<"(?,8)">) -> !cute.coord<"(?,?)"> + ``` + }]; + let arguments = (ins + Cute_CuteCoordType:$coord, + Cute_CuteShapeType:$shape + ); + let results = (outs + Cute_CuteCoordType:$result + ); +} +def Cute_IntToPtrOp : Cute_Op<"inttoptr", [Pure]> { + let description = [{ + The `cute.inttoptr` operation accepts an integer value and cast to cute + pointer value with align=1 (byte) by default. + + To create ptr with align > 1, we must use constrained int type with divby > 1. + Alignment of result ptr is the same as `divby` of constrained int type + + Example: + + ```mlir + // Convert to cute.ptr with unrestricted integer type. Can only have align<1> + %ptr = cute.inttoptr(%0) : i64 to !cute.ptr> + + // Convert to cute.ptr with constrained integer type. + %aligned_ptr = cute.inttoptr(%0) : !cute.i64 to !cute.ptr> + ``` + }]; + let arguments = (ins + Cute_IntegerOrIntegerTypeWithConstraints:$src + ); + let results = (outs + Cute_PointerTypeInterfaceInstance:$dst + ); +} +def Cute_KernelSmemSizeOp : Cute_Op<"kernel_smem_size", [Pure]> { + let description = [{ + This host-side operation gets the static smem size from a kernel function + and returns the size as an i64 integer. + }]; + let arguments = (ins + SymbolRefAttr:$kernel_name + ); + let results = (outs + I64:$size + ); +} +def Cute_LeftInverseOp : Cute_InferredOp<"left_inverse", [Pure]> { + let arguments = (ins + Cute_CuTeLayoutType:$input + ); + let results = (outs + Cute_CuTeLayoutType:$result + ); +} +def Cute_LoadScaledIndexOp : Cute_InferredOp<"load_scaled_index", [MemoryEffects<[MemRead]>]> { + let description = [{ + The `cute.load_scaled_index` operation adds the offset (%ptr_b[%index] * %stride) to a + cute pointer %ptr_a. %ptr_a can be any cute pointer type, while %ptr_b must be an I32 cute pointer. + The behavior is undefined if %ptr_b[%index] results in an out-of-bound access. + The result is a cute pointer that has identical type as %ptr_a except that its alignment is derived + from ptr_a's alignment and the stride. + + Example: + + ```mlir + %ptr = cute.load_scaled_index(%src_ptr, %table_ptr, %index, %stride) + : (!cute.ptr, !cute.ptr, !cute.int_tuple<"?">, !cute.int_tuple<"?{div=16}">) + -> !cute.ptr + ``` + }]; + let arguments = (ins + Cute_CuTePointerType:$ptr_a, + Cute_CuTePointerTo32BitSignlessInteger:$ptr_b, + Cute_IntTupleTypeThatDepthIs0:$index, + Cute_IntTupleTypeThatDepthIs0:$stride + ); + let results = (outs + Cute_CuTePointerType:$result + ); +} +def Cute_LocalPartitionOp : Cute_InferredOp<"local_partition", [Pure]> { + let description = [{ + The `local_partition` operation tiles a tensor according to the flat shape of a layout + that provides the coordinate of the target index. + This is typical at the Thread level where data is partitioned across repeated patterns + of threads. + + Example: + + ```mlir + %local_partition = cute.local_partition(%input, %tile, %index) + : (!cute.memref, "(16,?):(1,16)">, !cute.layout<"(2,4)">, i32) + -> !cute.memref + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstance:$input, + Cute_StaticLayout:$tiler, + Cute_32BitSignlessIntegerOr64BitSignless:$index + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_LocalTileOp : Cute_InferredOp<"local_tile", [Pure]> { + let description = [{ + The `local_tile` operation tiles a tensor according to a tiler and use a coord to + index into the remainder, keeping the tile. + + This is typical at the CTA level where tiles of data are extracted: + + ```mlir + %data = // ( M, N) + %crd = cute.make_coord(blockIdx.x,blockIdx.y) : !cute.coord<"(?,?)"> + %cta_data = cute.local_tile(%data, %tile, %crd) : ... // (_32,_64) + ``` + + Example: + + ```mlir + %0 = cute.local_tile(%alloc, %tile, %coord) + : (!cute.memref, "(128,128):(65536,1)">, + !cute.tile<"[128:1;32:1]">, + !cute.coord<"(_,_)">) + -> !cute.memref, "(128,32,1,4):(65536,1,0,32)"> + + %1 = cute.local_tile(%arg0, %shape, %coord_1) + : (!cute.coord_tensor<"(0,0,0)", "(256,64,1):(1@1,1@0,1@2)">, + !cute.shape<"(256,64)">, + !cute.coord<"(0,_,_)">) + -> !cute.coord_tensor<"(0,0,0)", "(256,64,1,1):(1@1,1@0,0,1@2)"> + ``` + + Utilizing a flat coordinate '(_)' will flatten the resulting tile. + + ```mlir + %0 = cute.local_tile(%alloc, %tile, %coord) + : (!cute.memref, "(128,128):(65536,1)">, + !cute.tile<"[128:1;32:1]">, + !cute.coord<"(_)">) + -> !cute.memref, "(128,32,1,4):(65536,1,0,32)"> + + %1 = cute.local_tile(%arg0, %shape, %coord_1) + : (!cute.coord_tensor<"(0,0,0)", "(256,64,1):(1@1,1@0,1@2)">, + !cute.shape<"(256,64)">, + !cute.coord<"(_)">) + -> !cute.coord_tensor<"(0,0,0)", "(256,64,1,1,1):(1@1,1@0,0,0,1@2)"> + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstance:$input, + Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$tile, + Cute_CuteCoordType:$coord, + OptionalAttr:$proj + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_LogicalDivideOp : Cute_InferredOp<"logical_divide", [Pure]> { + let description = [{ + This operation performs "logical_divide" on the input and produces a logical_divided result, + and it automatically infers the "logical_divided" layout for the result type. + + Example: + + ```mlir + %alloc = cute.memref.alloc() : !cute.memref + %shape = cute.make_shape() : () -> !cute.shape<"(4,8)"> + %zdiv = cute.logical_divide(%alloc, %shape) + : (!cute.memref, !cute.shape<"(4,8)">) + -> !cute.memref + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, + Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$tiler + ); + let results = (outs + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_LogicalProductOp : Cute_InferredOp<"logical_product", [Pure]> { + let description = [{ + This operation performs "logical_product" on the input and produce a logical_product result, + and it automatically infer the "logical_product" layout for the result type. + + Example: + + ```mlir + %input = cute.make_layout() : !cute.layout<"(3,4):(4,1)"> + %tiler = cute.make_layout() : !cute.layout<"(2,5):(1,2)"> + %prod = cute.logical_product(%input, %tiler) + : (!cute.layout<"(3,4):(4,1)">, !cute.layout<"(2,5):(1,2)">) + -> !cute.layout<"((3,4),(2,5)):((4,1),(12,24))"> + ``` + + Example with dynamic composed_layout: + + ``` + %prod = cute.logical_product(%input, %tiler) + : (!cute.composed_layout<"(?):(?) o ? o (5,4):(4,1)">, + !cute.layout<"(?,?):(?,?)">) + -> !cute.composed_layout<"(?):(?) o ? o ((5,4),(?,?)):((4,1),(?{div=20},?{div=20}))"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$input, + Cute_CuTeLayoutType:$tiler + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_MakeArithTupleIteratorOp : Cute_Op<"make_arith_tuple_iter", [Pure]> { + let description = [{ + This Operation makes an arithmetic tuple iterator (`!cute.arith_tuple_iter<...>`) + from an int_tuple value (`!cute.int_tuple<...>`) if provided, or it is empty + when all the modes are static. + + Its verifier checks that the value type of the result iterator matches the + operand value type if provided. + }]; + let arguments = (ins + Optional:$value + ); + let results = (outs + Cute_CuTeIteratorTypeWithIntTupleAsIts:$iter + ); +} +def Cute_MakeAtomOp : Cute_Op<"make_atom", [Pure]> { + let summary = "`cute.make_atom` op is generic constructor of SSA value of copy or mma atom type."; + let description = [{ + It optionally takes variadic list of SSA values as well as named attribute to + initialize corresponding fields of copy or mma atom. + + Examples: + + ```mlir + // create copy atom which doesn't require initialization + %0 = cute.make_atom() : () + -> !cute_nvgpu.atom.tmem_load + + // create mma atom and initialize it with SSA values + %1 = cute.make_atom(%accum_c, %a_neg, %b_neg) : (i1, i1, i1) + -> !cute_nvgpu.sm100.mma<128x128x16, + num_cta = 1, + ab_major = (k,k), + elem_type = (f16,f16,f32), + frag_kind = ss, + c_scale_exp = 0> + ``` + }]; + let arguments = (ins + Variadic:$values + ); + let results = (outs + Cute_AtomTypeInterfaceInstance:$atom + ); +} +def Cute_MakeComposedLayoutOp : Cute_InferredOp<"make_composed_layout", [Pure]> { + let summary = "The `make_composed_layout` operation constructs composed layout with"; + let description = [{ + * (inner) affined layout or swizzle layout + * offset + * (outer) layout + + In CuTe algebra, composed layout represent composition of + + ```math + L_{inner} o offset o L_{outer} + ``` + + where outer layout defined the domain of the layout which is visible to user. For given input coordinate + in domain of outer layout, composed layout maps coordinate to codomain by + + ```math + L_{inner}(L_{outer}(coordinate) + offset) + ``` + + Composed Layout is useful in cases where composition() does not or cannot apply to LayoutA and LayoutB. + For example, when the "divisibility condition" in shape_div is violated in composition(LayoutA, LayoutB). + + This ComposedLayout provides similar functionality to Layout including tiling, partitioning, + coordinate-to-index mapping and layout manipulations, but is not considered a "normal" layout. + For example, this layout provides shape() and size() functions, but does not provide stride() functions. + Mostly, the similar functionality is accomplished by applying each operation to LayoutB only + as LayoutB defines the domain. + + ### Examples: + + ```mlir + // Swizzle composed layout + %inner : !cute.swizzle<"S<3,4,3>"> + %layout = cute.make_composed_layout(%inner, %offset, %outer) + : !cute.composed_layout<"S<3,4,3> o 5 o (?,?):(?,?)"> + + // Linearized composed layout + %inner : !cute.layout<"((?,?),(?,?)):((?,?),(?,?))"> + %layout = cute.make_composed_layout(%inner, %offset, %outer) + : !cute.composed_layout<"((?,?),(?,?)):((?,?),(?,?)) o ? o (1,2):(3,4)"> + ``` + }]; + let arguments = (ins + Cute_CuTeLayoutTypeOrSwizzleWithValidParameters:$inner, + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$offset, + Cute_CuTeLayoutType:$outer + ); + let results = (outs + Cute_CuTeComposedLayoutType:$result + ); +} +def Cute_MakeCoordOp : Cute_Op<"make_coord", [Pure]> { + let description = [{ + Constructs a coord. The operands correspond to the + coord's dynamic elements, ordered from left to right. + The number of operands should match the count of dynamic elements in the + resulting coord type. + + Example: + + ```mlir + // Definitions for %0, %1 come earlier + %shape = cute.make_coord(%0, %1) : (i32, i32) + -> !cute.coord<"((?,32),?)"> + ``` + }]; + let arguments = (ins + Variadic:$dynamicElements + ); + let results = (outs + Cute_CuteCoordType:$result + ); +} +def Cute_MakeDescriptorIteratorOp : Cute_Op<"make_desc_iter", [Pure]> { + let description = [{ + This Operation makes a descriptor iterator from a descriptor value. Its + verifier verifies that the value Type of the result Type implementing the + iterator Type Interface is the same as the Type as the input operand. + }]; + let arguments = (ins + Cute_ATypeInterfaceForMemoryDescriptorsSuchAs:$value + ); + let results = (outs + AnyType:$iter + ); +} +def Cute_MakeFragmentLikeOp : Cute_InferredOp<"make_fragment_like", [Pure]> { + let summary = "A deprecated operation but used by collective DSL (keep it here to avoid breaking changes)"; + let description = [{ + TODO: remove this operation in the future + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$src + ); + let results = (outs + Cute_CuTeLayoutType:$dst + ); +} +def Cute_MakeIdentityLayoutOp : Cute_InferredOp<"make_identity_layout", [Pure]> { + let summary = "This operation makes an identity layout with the given shape."; + let description = [{ + Example: + ```mlir + %sha = cute.make_shape() : () -> !cute.shape<"(2,(3,4),5)"> + %lay = cute.make_identity_layout(%sha) : !cute.layout<"(2,(3,4),5):(1@0,(1@0@1,1@1@1),1@2)"> + ``` + }]; + let arguments = (ins + Cute_CuteShapeType:$shape + ); + let results = (outs + Cute_CuTeLayoutType:$result + ); +} +def Cute_MakeIdentityTensorOp : Cute_InferredOp<"make_identity_tensor", [Pure]> { + let description = [{ + The cute.make_identity_tensor operation construct an identity_tensor + based on the given shape. + + Example to make an identity_tensor from a static shape: + + ```mlir + %identity_tensor = cute.make_identity_tensor(%shape) + : !cute.coord_tensor<"(0,(0,0))", "(2,(3,4)):(1@0,(1@0@1,1@1@1))"> + ``` + + Example to make an identity_tensor from a dynamic shape: + + ```mlir + %identity_tensor = cute.make_identity_tensor(%shape) + : !cute.coord_tensor<"((0,0),(0,0))", "((?,?),(?,?)):((1@0@0,1@1@0),(1@0@1,1@1@1))"> + ``` + }]; + let arguments = (ins + Cute_CuteShapeType:$shape + ); + let results = (outs + Cute_CuteCoordTensorType:$result + ); +} +def Cute_MakeIntTupleOp : Cute_Op<"make_int_tuple", [Pure]> { + let description = [{ + Constructs a int_tuple. The operands correspond to the + int_tuple's dynamic elements, ordered from left to right. + The number of operands should match the count of dynamic elements in the + resulting int_tuple type. + + Example: + + ```mlir + // Definitions for %0, %1 come earlier + %shape = cute.make_int_tuple(%0, %1) : (i32, i32) + -> !cute.int_tuple<"((?,32),?)"> + ``` + }]; + let arguments = (ins + Variadic:$dynamicElements + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$result + ); +} +def Cute_MakeLayoutLikeOp : Cute_InferredOp<"make_layout_like", [Pure]> { + let description = [{ + Creates a compact memory layout that: + 1. Has a congruent shape with the source layout + 2. Has strides in the same order of major-ness as the input layout + 3. Mode 0 strides are `LayoutLeft` (column major) with zero strides preserved from the input layout + + Mode-0 constraints are applied due to its common usage with MMA_Atoms and Copy_Atoms + that need to create RMEM fragments. + + Example: + ```mlir + %0 = cute.make_layout_like(%layout) + : !cute.layout<"(4,8):(1,1024)"> to !cute.layout<"(4,8):(1,4)"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$src + ); + let results = (outs + Cute_CuTeLayoutType:$dst + ); +} +def Cute_MakeLayoutOp : Cute_Op<"make_layout", [AttrSizedOperandSegments, Pure]> { + let description = [{ + The `make_layout` operation generates a layout based on provided shape and stride. + If the stride component is omitted in the operation's result type + (e.g., `!cute.layout<"(?,(?,?))">`), the operation defaults to a column-major + layout, and missing stride details are automatically inferred to complete the type + of the resulting layout. + + The shape or stride operand must be supplied if they contain dynamic elements. + However, if the shape or stride is entirely static, corresponding operands can be + neglected. + + When the result type does not specify stride information (like `!cute.layout<"(?,?)">` + ), it is not necessary to provide a stride operand, even if the stride to be inferred + includes dynamic components (as in the inferred type `!cute.layout<"(?,?):(1,?)">`). + + ### Examples: + + ```mlir + // Shape and stride are both dynamic + %layout = cute.make_layout(%shape, %stride) : !cute.layout<"(?,64):(1,?)"> + + // Shape and stride are both static + %layout = cute.make_layout() : !cute.layout<"(32,64):(1,32)"> + + // Stride is omitted in the result type, implying it should be inferred + %layout = cute.make_layout(%shape) : !cute.layout<"(?,(?,?))"> + ``` + }]; + let arguments = (ins + Optional:$shape, + Optional:$stride + ); + let results = (outs + Cute_CuTeLayoutType:$layout + ); +} +def Cute_MakeOrderedLayoutOp : Cute_InferredOp<"make_ordered_layout", [Pure]> { + let description = [{ + The `make_ordered_layout` operation generates a compact layout based on provided `shape` + and stride is calculated by `shape` and `order`. + + Different from `make_layout`, `shape` operand is required, `order` is optional. When `order` + isn't set, default stride growth order is from left to right. Supports dynamic values in `shape`, + while currently `order` is restricted to static. Rank of `order` should be equal to rank of + shape but no need for congruent. + + Note that numbers in `order` are just used for relative order relationship, e.g. (67,46,23,32) + is equivalent to (4,3,1,2), (3,2,4) is equivalent to (2,1,3). + + ### Examples: + + ```mlir + %shape = cute.make_shape() : !cute.shape<"(32,64)"> + %shape_dyn = cute.make_shape() : !cute.shape<"(32,?)"> + %order = cute.make_shape() : !cute.shape<"(1,2)"> + %layout = cute.make_ordered_layout(%shape, %order) + : (!cute.shape<"(32,64)">, !cute.shape<"(1,2)">) -> !cute.layout<"(32,64):(1,32)"> + %layout_dyn = cute.make_ordered_layout(%shape_dyn, %order) + : (!cute.shape<"(32,?)">, !cute.shape<"(1,2)">) -> !cute.layout<"(32,?):(1,32)"> + + %order = cute.make_shape() : !cute.shape<"(2,1)"> + %layout = cute.make_ordered_layout(%shape, %order) + : (!cute.shape<"(32,64)">, !cute.shape<"(2,1)">) -> !cute.layout<"(32,64):(64,1)"> + %layout_dyn = cute.make_ordered_layout(%shape_dyn, %order) + : (!cute.shape<"(32,?)">, !cute.shape<"(2,1)">) -> !cute.layout<"(32,?):(?,1)"> + + %shape = cute.make_shape() : !cute.shape<"(2,(2,2),2)"> + %order = cute.make_shape() : !cute.shape<"(0,(2,3),1)"> + %layout = cute.make_ordered_layout(%shape, %order) : !cute.layout<"(2,(2,2),2):(1,(4,8),2)"> + + %order = cute.make_shape() : !cute.shape<"(2,1,3)"> + %layout = cute.make_ordered_layout(%shape, %order) : !cute.layout<"(2,(2,2),2):(4,(1,2),8)"> + }]; + let arguments = (ins + Cute_CuteShapeType:$shape, + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$order + ); + let results = (outs + Cute_CuTeLayoutType:$layout + ); +} +def Cute_MakeShapeOp : Cute_Op<"make_shape", [Pure]> { + let description = [{ + Constructs a shape. The operands correspond to the + shape's dynamic elements, ordered from left to right. + The number of operands should match the count of dynamic elements in the + resulting shape type. + + Example: + + ```mlir + // Definitions for %0, %1 come earlier + %shape = cute.make_shape(%0, %1) : (i32, i32) + -> !cute.shape<"((?,32),?)"> + ``` + }]; + let arguments = (ins + Variadic:$dynamicElements + ); + let results = (outs + Cute_CuteShapeType:$result + ); +} +def Cute_MakeSparseElemOp : Cute_Op<"make_sparse_elem", [Pure]> { + let summary = "Creates a sparse element from a physical storage."; + let description = [{ + Example: + + ```mlir + %v0 = arith.constant 8 : i8 + %v1 = arith.constant 31 : i8 + %physical_storage = vector.from_elements(vector<2xf16>, %v0, %v1) + %sparse_elem = cute.make_sparse_elem(%physical_storage) : (!cute.sparse_elem<4, vector<2xf16>>) + ``` + }]; + let arguments = (ins + AnyType:$physical_storage + ); + let results = (outs + Cute_CuTeSparseElementType:$result + ); +} +def Cute_MakeStrideOp : Cute_Op<"make_stride", [Pure]> { + let description = [{ + Constructs a stride. The operands correspond to the + stride's dynamic elements, ordered from left to right. + The number of operands should match the count of dynamic elements in the + resulting stride type. + + Example: + + ```mlir + // Definitions for %0, %1 come earlier + %shape = cute.make_stride(%0, %1) : (i32, i32) + -> !cute.stride<"((?,32),?)"> + ``` + }]; + let arguments = (ins + Variadic:$dynamicElements + ); + let results = (outs + Cute_CuteStrideType:$result + ); +} +def Cute_MakeTileOp : Cute_Op<"make_tile", [Pure]> { + let description = [{ + Constructs a tile. The operands correspond to the + tile's dynamic elements, ordered from left to right. + The number of operands should match the count of dynamic elements in the + resulting tile type. + + Example: + + ```mlir + // Definitions for %0, %1 come earlier + %shape = cute.make_tile(%0, %1) : (i32, i32) + -> !cute.tile<"((?,32),?)"> + ``` + }]; + let arguments = (ins + Variadic:$dynamicElements + ); + let results = (outs + Cute_CuteTileType:$result + ); +} +def Cute_MakeTiledCopyOp : Cute_Op<"make_tiled_copy", [Pure]> { + let description = [{ + This operation creates a new copy atom instance by tiling a "base" copy atom + instance. Copy atoms are used by the CuTe algorithm layer during lowering to + generate the appropriate `cute_nvgpu.arch` operations. + + Example: + The below values are origin from cutlass SM100_f16f16_UMMA kernel, the original code: + auto tAgA_mkl = cta_tma_a.partition_S(cta_mma.partition_A(gA_mkl))(_,_0{},_0{},_0{},_,_,_); + All the values are from the "cta_tma_a". + ```mlir + %0 = cute.make_atom(%tma_desc_ptr, %tma_bar) : (!cute.ptr>!cute.ptr>) + -> !cute_nvgpu.atom.tma_load + %1 = cute.make_tiled_copy(%0) + : !cute.tiled_copy, + layout_copy_tv = <"((4,2),(64,128)):((0,64),(128,1))">, + tiler_mn = <"[2048:1;1:1;4:1;1:1]">> + ``` + }]; + let arguments = (ins + Cute_CopyAtomTypeInterfaceInstance:$atom + ); + let results = (outs + Cute_TiledCopyAtomType:$result + ); +} +def Cute_MakeTiledCopyV2Op : Cute_Op<"make_tiled_copy_v2", [Pure]> { + let description = [{ + This operation creates a V2 tiled copy instance by tiling a V2 copy atom. + V2 copy atoms use fragment attributes for operand-specific behavior. + + Example: + ```mlir + %0 = cute.make_atom() : () -> !cute_nvgpu.atom.sm100_s2t_copy_v2 + %1 = cute.make_tiled_copy_v2(%0) + : !cute.tiled_copy_v2, + layout_copy_tv = <"((32,4),(2,32)):((0,1),(128,4))">, + tiler_mn = <"[(4,32):(32,1);2:1]">> + ``` + }]; + let arguments = (ins + Cute_CopyAtomTypeInterfaceV2Instance:$atom + ); + let results = (outs + Cute_TiledCopyV2AtomType:$result + ); +} +def Cute_MakeTiledMmaOp : Cute_Op<"make_tiled_mma", [Pure]> { + let description = [{ + This operation creates a new MMA atom instance by tiling a "base" MMA atom + instance. MMA atoms are used by the CuTe algorithm layer during lowering to + generate the appropriate `cute.arch` operations. + + Example: + ```mlir + %0 = cute.make_atom(%accum, %a_neg, %b_neg) : (i1, i1, i1) + -> !cute_nvgpu.sm100.mma<64x64x16, num_cta = 1, ab_major = (k,mn), elem_type = (f16,f16,f16), frag_kind = ss, c_scale_exp = 0> + %1 = cute.make_tiled_mma(%0) + : !cute.tiled_mma, + atom_layout_MNK = <"(1,1,1)">> + ``` + }]; + let arguments = (ins + Cute_MmaAtomTypeInterfaceInstance:$atom + ); + let results = (outs + Cute_TiledMMAAtomType:$result + ); +} +def Cute_MakeTupleOp : Cute_Op<"make_tuple", [Pure]> { + let description = [{ + Creates a tuple from multiple values. The number and types of the + parameters must match the number and types of the result tuple. + }]; + let arguments = (ins + Variadic:$values + ); + let results = (outs + Cute_ATupleIsAFixedLengthOrderedList:$result + ); +} +def Cute_MakeViewOp : Cute_Op<"make_view", [Pure]> { + let description = [{ + The cute.make_view operation construct `CuteViewTypeInterface` + (i.e., cute.memref or cute.coord_tensor) with given `iter` and `layout`. + Internally, we infer the exact return type by checking `iter`'s type: + - if `iter` is cute.ptr, return type is `CuteMemRefType` + - if `iter` is cute.tuple, return type is `CuteCoordTensorType` + + `iter` and `layout` could contain both static and dynamic values. + + `MakeViewOp` usually is the result of a previous operation + (e.g., cute.slice, cute.zipped_divide), where creating a new + `CuteViewTypeInterface` is needed. + + `MakeViewOp` does not involve allocating buffers of any kind. + + Example to make cute.coord_tensor with an iter of cute.tuple type: + + ```mlir + %coord_tensor= cute.make_view(%iter, %layout) : !cute.coord_tensor<"(0,2)", "(30):(20@1)"> + ``` + + Example to make cute.memref with an iter of cute.ptr type: + + ```mlir + %memref= cute.make_view(%iter, %layout) : !cute.memref + ``` + + Static Layout + + ```mlir + %memref= cute.make_view(%iter) : !cute.memref + ``` + + Composed+Static Layout + + ```mlir + %memref= cute.make_view(%iter) : !cute.memref o 9 o (64):(16)"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedIteratorTypeOfCuTe:$iter, + Optional:$layout + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_MemRefAllocSmemOp : Cute_Op<"memref.alloc_smem"> { + let description = [{ + The memref.alloc_smem operation is utilized for the allocation of static + shared memory. The cosize of the memref layout, which must be static, + determines the size of the memory allocated by this operation. + + ```mlir + %0 = cute.memref.alloc_smem() : !cute.memref + ``` + }]; + let arguments = (ins + ); + let results = (outs + Cute_MemrefInSmemWithStaticAndNonComposed:$memref + ); +} +def Cute_MemRefAllocaOp : Cute_Op<"memref.alloca"> { + let description = [{ + The memref.alloca operation is used to allocate shared/stack memory, which is automatically + released when control returns from the closest surrounding operation's region. This + operation only supports memref with the address space reg, and attempting to allocate + memory with other address spaces will result in an error. + + The size of memory allocated is determined by the layout operand's cosize, which can + be either static or dynamic. The layout operand can be omitted when the memref layout + is static. + + If the memref is sparse, the allocated memory size will be layout operand's cosize + divided by sparsity factor. + + ```mlir + // %dynamic_layout : !cute.layout<"(?,24)"> + %0 = cute.memref.alloca(%dynamic_layout) : !cute.memref + // static layout + %1 = cute.memref.alloca() : !cute.memref + ``` + }]; + let arguments = (ins + Optional:$layout + ); + let results = (outs + Cute_MemrefInGenericOrRegisterMemOrSmem:$result + ); +} +def Cute_MemRefLoadOp : Cute_InferredOp<"memref.load", [MemoryEffects<[MemRead]>]> { + let description = [{ + The `memref.load` operation loads one element at the specified logical coordinate from a memref + in gmem/rmem/smem/generic memory spaces to a register. This operation performs the following steps: + + 1. Converts the logical coordinate to a linear index using `cute.crd2idx` + 2. Retrieves the iterator from the memref using `cute.get_iter` + 3. Adds the offset to the iterator using `cute.add_offset` + 4. For CuteMemRefType, applies swizzling if needed, converts to LLVM pointer, and performs the load + 5. For CoordTensorType, returns the tuple directly + + The element type generally cannot be a sub-byte type (no f4, f6, i1, i2 etc.), with the exception + of i1 which has special handling. + + NOTE: For tf32 types, the operation treat it as f32 even bitwidth of MLIR type is 19. + + Example: + + ```mlir + // Load a fp32 + %coord = cute.make_coord(%c1_i32, %c2_i32) : (i32, i32) -> !cute.coord<"(?,?)"> + %0 = cute.memref.load(%memref, %coord) : (!cute.memref, !cute.coord<"(?,?)">) -> f32 + + // Load a tf32 value (will be handled with special bitwise casting) + %1 = cute.memref.load(%tf32_memref, %coord) : (!cute.memref, !cute.coord<"(?,?)">) -> tf32 + ``` + }]; + let arguments = (ins + Cute_AMemoryReferenceTypeThatRepresentsATensor2:$src, + Cute_CuteCoordType:$coord + ); + let results = (outs + AnyType:$result + ); +} +def Cute_MemRefLoadVecOp : Cute_InferredOp<"memref.load_vec", [AttrSizedOperandSegments, MemoryEffects<[MemRead]>]> { + let description = [{ + The `memref.load_vec` operation loads all elements in the logical coordinate + space from a memref generic or register memory to a 1D vector. Only memrefs with + static layouts are accepted. + + The resulting vector has the same element type as the memref, with a size + equal to the **size** (not **cosize**) of the memref layout. + + An optional mask vector (of the same size as the output vector) can be + provided to mask out elements. When the mask is present, an optional + pass_thru vector can also be provided to fill in values for elements that + are masked out, and thus the result is equivalent to a regular memref load + followed by a select between the loaded and the passthru values, predicated + on the same mask, except that the masked-off lanes are not accessed. + + Example: + + ```mlir + // Load vector + %0 = cute.memref.load_vec(%memref) : (!cute.memref) -> vector<32xf32> + + // Load vector from swizzled memory + %0 = cute.memref.load_vec %memref : !cute.memref, "((32,1),1,1):((1,0),0,0)"> + + // Load vector with mask and pass_thru + %0 = cute.memref.load_vec(%memref, %mask, %pass_thru) + : (!cute.memref, vector<32xi1>, vector<32xf32>) -> vector<32xf32> + ``` + }]; + let arguments = (ins + Cute_StaticShape:$src, + Optional:$mask, + Optional:$pass_thru + ); + let results = (outs + Cute_OfRanks1:$result + ); +} +def Cute_MemRefStoreOp : Cute_Op<"memref.store"> { + let description = [{ + The `memref.store` operation stores a register value to a memref in gmem/rmem/smem/generic at given logical coordinate. + This operation performs the following steps: + + 1. Converts the logical coordinate to a linear index using `cute.crd2idx` + 2. Retrieves the iterator from the memref using `cute.get_iter` + 3. Adds the offset to the iterator using `cute.add_offset` + 4. Converts the pointer to the appropriate LLVM pointer type and performs the store + + The element type generally cannot be a sub-byte type (no f4, f6, i1, i2 etc.), with the exception + of i1 which has special handling. + + NOTE: For tf32 types, the operation treats it as f32 even though the bitwidth of the MLIR type is 19. + This special handling allows for proper conversion between tf32 and f32 representations during store operations. + + Example: + + ```mlir + // Store a fp32 + %coord = cute.make_coord(%c1_i32, %c2_i32) : (i32, i32) -> !cute.coord<"(?,?)"> + cute.memref.store(%memref, %coord, %value) : (!cute.memref, !cute.coord<"(?,?)">, f32) + + // Store a tf32 value (will be handled with special bitwise casting) + cute.memref.store(%tf32_memref, %coord, %tf32_value) : (!cute.memref, !cute.coord<"(?,?)">, tf32) + ``` + }]; + let arguments = (ins + Cute_AMemoryReferenceTypeThatRepresentsATensor3:$dst, + Cute_CuteCoordType:$coord, + AnyType:$value + ); + let results = (outs + ); +} +def Cute_MemRefStoreVecOp : Cute_Op<"memref.store_vec"> { + let description = [{ + The `memref.store_vec` operation stores all elements of a 1D vector to a to the logical coordinate + space of memref in generic or register memory. Only memrefs with static layouts are accepted. + + The vector has the same element type as the memref, with a size equal to + the **size** (not **cosize**) of the memref layout. + + An optional mask vector (of the same size as the input vector) can be + provided to mask out elements. + + ```mlir + // Store vector + %0 = cute.memref.store_vec(%value, %memref) : (vector<32xf32>, !cute.memref) -> () + + // Store vector to swizzled memory + %0 = cute.memref.store_vec %value, %memref : !cute.memref, "((32,1),1,1):((1,0),0,0)"> + + // Store vector with mask + %0 = cute.memref.store_vec(%value, %memref, %mask) + : (vector<32xf32>, !cute.memref, vector<32xi1>) -> () + ``` + }]; + let arguments = (ins + Cute_OfRanks1:$value, + Cute_StaticShape:$dst, + Optional:$mask + ); + let results = (outs + ); +} +def Cute_MmaAtomCallOp : Cute_Op<"mma_atom_call", [AttrSizedOperandSegments]> { + let summary = "`cute.mma_atom_call` op executes a mma atom operation with the given operands."; + let description = [{ + The operands `a` and `b` are variadic, each containing a variable number of tensors: + - For regular MMA, `a` and `b` contain the MMA A and B tensors respectively. + - For MMA with auxiliary operands, `a` and `b` contain the MMA A and B tensors followed by their respective auxiliary tensors. For example: + - For BlockScaledMMA, `a` = [A, SFA] and `b` = [B, SFB]. + - For SparseMMA, `a` = [A, E] and `b` = [B]. + - For BlockScaledSparseMMA, `a` = [A, SFA, E] and `b` = [B, SFB]. + Example (regular MMA): + + ```mlir + // Call a mma atom operation + cute.mma_atom_call(%mma_atom, %d, %a, %b, %c) + : (!cute_nvgpu.sm100.mma<128x128x16, num_cta = 1, ab_major = (k,k), + elem_type = (f16,f16,f32), + frag_kind = ss, + c_scale_exp = 0>, + !cute.memref, "((128,128)):((131072,1))">, + !cute_nvgpu.smem_desc_view, + !cute_nvgpu.smem_desc_view, + !cute.memref, "((128,128)):((131072,1))">) + -> () + ``` + + Example (block-scaled MMA): + + ```mlir + cute.mma_atom_call(%mma_atom, %d, [%a, %sfa], [%b, %sfb], %c) + : (!cute_nvgpu.sm100.mma_bs<256x128x64, num_cta = 2, ab_major = (k,k), + elem_type = (f4E2M1FN, f4E2M1FN, f32), + sf_type = f8E4M3FN, frag_kind = ss, vec_size = 16>, + !cute.memref, "((128,128)):((65536,1))">, + [!cute_nvgpu.smem_desc_view, + !cute.memref, "((((32,4),4),(16,4))):((((262144,4),8388608),(0,1)))">], + [!cute_nvgpu.smem_desc_view, + !cute.memref, "((((32,4),4),(16,4))):((((262144,4),8388608),(0,1)))">], + !cute.memref, "((128,128)):((65536,1))">) + -> () + ``` + + Example (block-scaled MMA): + + ```mlir + cute.mma_atom_call(%mma_atom, %d, [%a, %sfa], [%b, %sfb], %c) + : (!cute_nvgpu.sm100.mma_bs<256x128x64, num_cta = 2, ab_major = (k,k), + elem_type = (f4E2M1FN, f4E2M1FN, f32), + sf_type = f8E4M3FN, frag_kind = ss, vec_size = 16>, + !cute.memref, "((128,128)):((65536,1))">, + [!cute_nvgpu.smem_desc_view, + !cute.memref, "((((32,4),4),(16,4))):((((262144,4),8388608),(0,1)))">], + [!cute_nvgpu.smem_desc_view, + !cute.memref, "((((32,4),4),(16,4))):((((262144,4),8388608),(0,1)))">], + !cute.memref, "((128,128)):((65536,1))">) + -> () + ``` + + Example (block-scaled MMA): + + ```mlir + cute.mma_atom_call(%mma_atom, %d, [%a, %sfa], [%b, %sfb], %c) + : (!cute_nvgpu.sm100.mma_bs<256x128x64, num_cta = 2, ab_major = (k,k), + elem_type = (f4E2M1FN, f4E2M1FN, f32), + sf_type = f8E4M3FN, frag_kind = ss, vec_size = 16>, + !cute.memref, "((128,128)):((65536,1))">, + [!cute_nvgpu.smem_desc_view, + !cute.memref, "((((32,4),4),(16,4))):((((262144,4),8388608),(0,1)))">], + [!cute_nvgpu.smem_desc_view, + !cute.memref, "((((32,4),4),(16,4))):((((262144,4),8388608),(0,1)))">], + !cute.memref, "((128,128)):((65536,1))">) + -> () + ``` + }]; + let arguments = (ins + Cute_MmaAtomTypeInterfaceInstance:$atom, + Cute_ViewTypeInterfaceInstance:$d, + Variadic:$a, + Variadic:$b, + Cute_ViewTypeInterfaceInstance:$c + ); + let results = (outs + ); +} +def Cute_MmaMakeFragmentOp : Cute_InferredOp<"mma.make_fragment", [Pure]> { + let description = [{ + Given a partitioned tensor, the `cute.mma.make_fragment` operation returns + a tensor with the correct memory space for the specified operand and dot + atom. This op is usually used between partition and gemm. It is the + "prepare" step before gemm. + + `cute.tiled.mma.partition` returns a view into the specified tensor, but it + does not copy the data to the correct memory space. This is done by + `cute.mma.make_fragment` and explicit memcpy operations. Whether a memcpy + is needed or not depend on the result memory space. + + * RMEM: The result tensor is a newly allocated tensor with unspecified + contents. A memcpy is needed. + * SMEM: The result tensor is a view into the input tensor in the form of a + CuteSmemDescViewType. A memcpy is not needed. + * TMEM: The result tensor uses a dummy TMEM engine (fake pointer 0x0) and + must be set correctly later, aligned with Cute C++. + + Instead of a tensor, this op also accepts cute.layout (for RMEM/TMEM) and + cute.shape (for RMEM/TMEM). + + Example: + + ```mlir + // Example 1: Full example for SM80. + // Partition a tensor for operand A, which is stored in SMEM. + %partitioned = cute.tiled.mma.partition A(%atom, %smem_A, %tid_coord) + : (!cute.tiled_mma, + atom_layout_MNK = <"(2,2,1)">>, + !cute.memref, S<2,4,3>, "(128,32):(32,1)">, + !cute.coord<"?">) + -> !cute.memref, S<2,4,3>, "((2,2,2),4,2):((1,256,8),1024,16)"> + + // The tensor is still in SMEM, but sm80.mma expects A in registers. + // Prepare a tensor with the correct shape and strides in RMEM. + %fragment = cute.mma.make_fragment A (%3, %partitioned) + : (!cute.tiled_mma, + atom_layout_MNK = <"(2,2,1)">>, + !cute.memref, S<2,4,3>, "((2,2,2),4,2):((1,256,8),1024,16)">) + -> !cute.memref, "((2,2,2),4,2):((1,2,4),16,8)"> + + // Copy the data from SMEM to RMEM. + cute.copy(..., %partitioned, %fragment) + + // Example 2: Make fragment for SM100 (SMEM). + %fragment = cute.mma.make_fragment A (%0, %1) + : (!cute.tiled_mma, + atom_layout_MNK = <"(1,1,1):(0,0,0)">>, + !cute.memref, "((128,16),2,32):((512,1),65536,16)">) + -> !cute_nvgpu.smem_desc_view + + // Example 3: Make fragment for SM100 (TMEM) from layout. + %fragment = cute.mma.make_fragment A (%0, %1) + : (!cute.tiled_mma, + atom_layout_MNK = <"(1,1,1):(0,0,0)">>, + !cute.layout<"((64,16),1,4):((1,64),0,1024)">) + -> !cute.memref, "((128,16),1,4):((131072,1),0,16)"> + + // Example 3: Make fragment for SM100 (TMEM) from shape. + %fragment = cute.mma.make_fragment A (%0, %1) + : (!cute.tiled_mma, + atom_layout_MNK = <"(1,1,1):(0,0,0)">>, + !cute.shape<"((64,16),1,4)">) + -> !cute.memref, "((128,16),1,4):((131072,1),0,16)"> + ``` + }]; + let arguments = (ins + Cute_MmaAtomTypeInterfaceInstance:$atom, + Cute_AMemoryReferenceTypeThatRepresentsATensor:$input, + Cute_MmaOperandEnum:$operand_id + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_PrefetchAtomCallOp : Cute_Op<"prefetch_atom_call", [MemoryEffects<[MemRead]>]> { + let summary = "`cute.exec_prefetch_atom` op executes a prefetch atom operation with the given operands."; + let description = [{ + Following src layout of atom are valid + * ((atom_v, rest_v)) + * ((atom_v)) + * (atom_v) + + The `src` operand is variadic: + - For regular TMA prefetch, `src` is a single tensor. + - For 2D `tile::gather4` mode, `src` is `[data_coord_tensor, gmem_index_tensor]`. + + Examples: + + ```mlir + // Call a prefetch atom operation + cute.prefetch_atom_call(%copy_tma_load, %src_v) + : (!cute_nvgpu.atom.tma_load, + %src: !cute.coord_tensor<"(0,0)", "(((64,128),1),1,1,2):(((1@0,1@1),0),0,0,64@0)">) + -> () + + // gather4 prefetch: data tensor + gmem index tensor + cute.prefetch_atom_call(%gather4_atom, %src_v, %idx_v) + : (!cute_nvgpu.atom.tma_load>, + !cute.coord_tensor<...>, + !cute.coord_tensor<...>) + -> () + ``` + }]; + let arguments = (ins + Cute_PrefetchAtomTypeInterfaceOrTiledCopyTypeWithPrefetchAtomTypeInterfaceCopyAtom:$atom, + Variadic:$src + ); + let results = (outs + ); +} +def Cute_PrependToRankOp : Cute_InferredOp<"prepend_to_rank", [Pure]> { + let summary = "Prepend element to input to rank N."; + let description = [{ + Example: + + ```mlir + %shp = cute.make_shape() : !cute.shape<"(3,4,5)"> + %elem = cute.make_shape() : !cute.shape<"(1,2)"> + %new_layout = cute.prepend_to_rank<5> (%shp, %elem) + : (!cute.shape<"(3,4,5)">, !cute.shape<"1">) -> !cute.shape<"((1,2),(1,2),3,4,5)"> + ``` + }]; + let arguments = (ins + Cute_ValueTypeInterfaceInstance:$input, + Cute_ValueTypeInterfaceInstance:$element, + SI32Attr:$rank + ); + let results = (outs + Cute_ValueTypeInterfaceInstance:$result + ); +} +def Cute_PrintOp : Cute_Op<"print"> { + let summary = "Debug operation to print out value of cute data type."; + let description = [{ + Supported following cute data types + * shape, stride, layout, tile + * View: memref, coord_tensor + * Iterator: ptr, tuple + + Examples: + + ```mlir + cute.print(%0) : !cute.shape<"(3,5)"> + + cute.print(%1) : !cute.layout<"(5,(3,?)):(?,(4,5))"> + + // variadic operands without format string + cute.print(%0, %1) : !cute.shape<"(3,5)">, !cute.layout<"(5,(3,?)):(?,(4,5))"> + + // with format string + cute.print("{} -> {}", %0, %1) : !cute.shape<"(3,5)">, !cute.layout<"(5,(3,?)):(?,(4,5))"> + ``` + + The optional `operand_signed` attribute carries the DSL-level signedness of + each operand, one entry per value. It exists because DSL integers lower to + signless MLIR integer types, so an unsigned value is indistinguishable from + a signed one by type alone. When an entry is false, a generic `{}` + placeholder for that integer operand is rendered with an unsigned specifier + (`%u`/`%llu`) instead of `%d`/`%lld`. When absent, all operands are treated + as signed, and an explicit `%u` in the format string still forces unsigned. + }]; + let arguments = (ins + Variadic:$values, + OptionalAttr:$fmt, + OptionalAttr:$operand_signed + ); + let results = (outs + ); +} +def Cute_PrintTMADescIm2ColOp : Cute_Op<"print_tma_desc_im2col"> { + let summary = "This operation interprets the input i64 as a pointer to a TmaDescriptorInternal."; + let description = [{ + Example: + ```mlir + %tma_desc_a_host = cute_nvgpu.make_tma_desc_im2col( + %tensor, + %slayout, + %cta_v_tile, + %lower_corner_whd, + %upper_corner_whd, + %lower_padding_whd, + %upper_padding_whd, + %stride_whd, + %lower_srt, + %stride_srt) + <{host = true}> : ( + !cute.memref, "((8,8,1,4),64,1):((64,512,4096,4096),1,0)">, + !cute.composed_layout<"S<2,4,3> o 0 o ((8,16),(32,1)):((32,256),(1,0))">, + !cute.layout<"(128,32):(1@0,1@1)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(1,1,1)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(1,1,1)">) -> !cute.ptr> + %tma_desc_a_host_ptr_i64 = cute_nvgpu.cast_tma_desc_to_integer(%tma_desc_a_host: !cute.ptr) -> i64 + %fd = arith.constant 1 : i32 // stdout + cute.print_tma_desc_im2col(%tma_desc_a_host_ptr_i64, %fd) + }]; + let arguments = (ins + I64:$input, + I32:$fd + ); + let results = (outs + ); +} +def Cute_PrintTMADescTiledOp : Cute_Op<"print_tma_desc_tiled"> { + let summary = "This operation interprets the input i64 as a pointer to a TmaDescriptorInternal."; + let description = [{ + Example: + ```mlir + %tma_desc_a_host = cute_nvgpu.make_tma_desc_tiled(%a_tensor, %smem_ablayout_128_64_k_major, %cta_v_tile_128_64_k_major) + <{mode = host = true}> : + ( + !cute.memref, "(?,?,?) : (?,1,?)">, + !cute.composed_layout<"S<3, 4, 3> o 0 o ((128,64),1,1):((64,1),0,0)">, + !cute.layout<"((128,64),1,1):((1@0,1@1),0,0)"> + ) -> !cute.ptr + %tma_desc_a_host_ptr_i64 = cute_nvgpu.cast_tma_desc_to_integer(%tma_desc_a_host: !cute.ptr) -> i64 + %fd = arith.constant 1 : i32 // stdout + cute.print_tma_desc_tiled(%tma_desc_a_host_ptr_i64, %fd) + ``` + }]; + let arguments = (ins + I64:$input, + I32:$fd + ); + let results = (outs + ); +} +def Cute_PrintViewOp : Cute_Op<"print_view"> { + let summary = "Debug operation to print cute view types."; + let description = [{ + Currently supports cute.memref and cute.tensor. MemRefs with both static layout + and dynamic layout are accepted. An optional coord parameter can be used to + specify the element to be printed. If no coord is provided, all elements will + be printed. + + The `verbose` attribute controls the output format: + - When `verbose` is true (default), it prints the full details of the view + including memory address, shape, and strides. + - When `verbose` is false, it prints in a more compact torch-style matrix format. + + The `signed` attribute is used for integer data types to control whether values + should be interpreted as signed or unsigned. This attribute is ignored for + floating point types and coord tensors. + + Examples: + + ```mlir + cute.print_view(%0) : !cute.memref + output: + Memref raw_ptr(0x0000152397fffc80: f32, rmem, align<4>) o (2,2):(2,1) = ( + (0,0)=1.000000 + (0,1)=2.000000 + (1,0)=3.000000 + (1,1)=4.000000 + ) + + cute.print_view(%1) : !cute.memref + output: + Memref raw_ptr(0x0000148eabfffbf0: f32, rmem, align<4>) o (2,(2,2)):(4,(2,1)) = ( + (0,(0,0))=1.000000 + (0,(0,1))=2.000000 + (0,(1,0))=3.000000 + (0,(1,1))=4.000000 + (1,(0,0))=5.000000 + (1,(0,1))=6.000000 + (1,(1,0))=7.000000 + (1,(1,1))=8.000000 + ) + + cute.print_view(%3,%coord) : !cute.memref,!cute.coord<"(?,(?,?))"> + output: + Memref raw_ptr(0x0000150ce3fffc90: f32, rmem, align<4>) o (2,(2,2)):(4,(2,1)), coord (1,(1,0)) = (7.000000) + + // Using verbose=false to print in torch-style format + cute.print_view(%0) {verbose = false} : !cute.memref + + output: + tensor(raw_ptr(0x000000003746a2e0: f64, generic, align<8>) o (8,5,3):(15,3,1), data= + [[[ 1.426091, -0.279034, -1.635684, -0.026360, -0.732545, ], + [-0.733555, 0.886134, -1.418841, -0.203192, -1.836270, ], + ... + [-1.172012, 1.678573, -0.004324, 0.441605, 1.484385, ], + [-0.550848, 1.455505, -0.968547, 1.233722, 0.812084, ]], + + [[ 0.762814, -0.641657, 0.843695, -1.380960, 0.261200, ], + [ 1.500137, -0.921287, 1.001725, 0.106540, -1.326231, ], + ... + [-0.878001, -0.465694, -1.204548, -0.407944, -0.092967, ], + [ 1.035407, 0.043012, -1.959517, 0.075913, 0.893392, ]], + + [[-0.129989, 0.750885, 0.219408, 0.442481, -0.508588, ], + [ 0.791268, -0.547978, 0.194177, 0.652729, 2.642380, ], + ... + [-0.674993, -1.017359, -1.272791, 0.829529, 0.852879, ], + [-0.256070, -0.287006, 1.900199, -0.492318, -0.658134, ]]]) + + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstance:$src, + Optional:$coord, + DefaultValuedAttr:$verbose, + DefaultValuedAttr:$is_signed + ); + let results = (outs + ); +} +def Cute_PtrLoadOp : Cute_InferredOp<"ptr.load", [MemoryEffects<[MemRead]>]> { + let summary = "Loads an LLVM or CuTe value from the given `!cute.ptr`."; + let arguments = (ins + Cute_CuTePointerType:$ptr + ); + let results = (outs + Cute_CuTePointerTypeOrAMemoryReferenceType:$value + ); +} +def Cute_PtrStoreOp : Cute_Op<"ptr.store"> { + let description = [{ + Stores the given CuTe value at the provided pointer. Handles any + LLVM-storable type, including those that can be stored with + `llvm.store` *and* CuTe types. + }]; + let arguments = (ins + Cute_CuTePointerTypeOrAMemoryReferenceType:$value, + Cute_CuTePointerType:$ptr + ); + let results = (outs + ); +} +def Cute_PtrToIntOp : Cute_Op<"ptrtoint", [Pure]> { + let description = [{ + The `cute.ptrtoint` operation accepts a cute pointer value and casts it to an + integer value. The pointer must have byte alignment, meaning subbyte types + must have explicit alignment specified (e.g., align<1> or higher). + + Example: + + ```mlir + %int_aligned = cute.ptrtoint(%ptr) : !cute.ptr> to i64 + ``` + }]; + let arguments = (ins + Cute_PointerTypeInterfaceInstance:$src + ); + let results = (outs + Cute_Integer:$dst + ); +} +def Cute_RakedProductOp : Cute_InferredOp<"raked_product", [Pure]> { + let description = [{ + This operation performs "raked_product" on the input and produce a raked_product result, + and it automatically infer the "raked_product" layout for the result type. + + Example: + + ```mlir + %input = cute.make_layout() : !cute.layout<"(3,4):(4,1)"> + %tiler = cute.make_layout() : !cute.layout<"(2,5):(1,2)"> + %prod = cute.raked_product(%arg0, %arg1) + : (!cute.layout<"(3,4):(4,1)">, !cute.layout<"(2,5):(1,2)">) + -> !cute.layout<"((2,3),(5,4)):((12,4),(24,1))"> + ``` + + Example with dynamic composed_layout: + + ``` + %prod = cute.raked_product(%input, %tiler) + : (!cute.composed_layout<"(?):(?) o ? o (5,4):(4,1)">, + !cute.layout<"(?,?):(?,?)">) + -> !cute.composed_layout<"(?):(?) o ? o ((?,5),(?,4)):((?{div=20},4),(?{div=20},1))"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$input, + Cute_CuTeLayoutType:$tiler + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_RandomDelayOp : Cute_Op<"random_delay"> { + let arguments = (ins + ); + let results = (outs + ); +} +def Cute_RecastIterOp : Cute_Op<"recast_iter", [Pure]> { + let description = [{ + This `cute.recast_iter` is similar to CuTe C++'s recast_ptr, convert two iterators + in certain aspects like type/swizzle for convenience. + But this operation has some usage limitation for correctness, + like it's not allowed for TMA tensor's iterator and it's not allowed to convert align attribute. + In general, it's not allowed to convert two different iterator kinds (PtrType <-> IntTupleType) + + Example: + + ```mlir + %ptr = cute.recast_iter(%0) : !cute.ptr, S<3,4,3>> + to !cute.ptr, S<2,4,3>> + ``` + }]; + let arguments = (ins + Cute_CuTePointerType:$src + ); + let results = (outs + Cute_CuTePointerType:$dst + ); +} +def Cute_RecastLayoutOp : Cute_InferredOp<"recast_layout", [Pure]> { + let summary = "Example:"; + let description = [{ + ```mlir + # upcast<8> operation + %0 = cute.recast_layout<32, 4>(%arg0) + : !cute.layout<"(128,64):(1,128)"> to !cute.layout<"(16,64):(1,16)"> + # upcast<2> for composed layout + %0 = cute.recast_layout<32, 16>(%arg0) + : !cute.composed_layout<"((6,7),8):((10,12),1) o 6 o (8,2):(1,4)"> + to !cute.composed_layout<"((6,7),4):((5,6),1) o 3 o (4,2):(1,2)"> + # downcast<8> operation + %0 = cute.recast_layout<8, 32>(%arg0) + : !!cute.layout<"(16,64):(1,16)"> to cute.layout<"(128,64):(1,128)"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$src, + I32Attr:$new_type_bits, + I32Attr:$old_type_bits + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$dst + ); +} +def Cute_ReduceOp : Cute_InferredOp<"reduce", [Pure]> { + let description = [{ + The `cute.reduce` operation reduces elements of a vector according to a given + shape and reduction operator, optionally guided by a reduction profile and/or a + predication mask. + + The reduction is performed over the dimensions of the input vector as described + by the provided shape. By default, all dimensions are reduced to a scalar. If a + reduction profile is specified, it determines which dimensions are kept (marked + with '_') and which are reduced (marked with an integer or tuple structure). The + result is either a scalar or a lower-rank vector, along with the reduced shape + if applicable. + + An optional predication vector (of the same size as the input vector) can be + provided to mask out elements from the reduction, enabling conditional + reduction. + + The reduction operator (e.g., MIN, MAX, ADD, MUL) is specified via the `op` + attribute. + + Examples: + ```mlir + // Reduce all elements to a scalar (minimum) + %reduced = cute.reduce(%input, %shape, %init) + { op = #cute.reduction_op } + : (vector<20xf32>, !cute.shape<"(4,5)">, f32) -> f32 + + // Reduce all elements to a scalar (maximum) with predication + %reduced = cute.reduce(%input, %shape, %init, %pred) + { op = #cute.reduction_op } + : (vector<20xf32>, !cute.shape<"(4,5)">, f32, vector<20xi1>) -> f32 + + // Reduce all but the second dimension (profile keeps dimension 1) + %reduced, %red_shp = cute.reduce(%input, %shape, %init) + { op = #cute.reduction_op, profile = #cute.coord<"(1,_,1)"> } + : (vector<60xf32>, !cute.shape<"(5,4,3)">, f32) -> (vector<4xf32>, !cute.shape<"(4)>") + + // Reduce a nested dimension (profile keeps outer and first inner dimension) + %reduced, %red_shp = cute.reduce(%input, %shape, %init) + { op = #cute.reduction_op, profile = #cute.coord<"(_,(_,1))"> } + : (vector<30xf32>, !cute.shape<"(5,(3,2))">, f32) -> (vector<15xf32>, !cute.shape<"(5,3)>") + ``` + The operation supports both flat and nested shapes and profiles, enabling reductions over complex multidimensional data layouts. + }]; + let arguments = (ins + Cute_OfRanks1:$input, + AnyType:$shape, + AnyType:$init_val, + Optional:$predication, + Cute_ReductionOpAttr:$reduction_op, + OptionalAttr:$reduction_profile + ); + let results = (outs + Cute_OfRanks1OrFloatingPointOrInteger:$output, + Optional:$reduced_shape + ); +} +def Cute_RightInverseOp : Cute_InferredOp<"right_inverse", [Pure]> { + let description = [{ + This operation builds the right inverse of a layout. + The current implementation only supports static layout. + An error is returned when the layout contains dynamic values. + + Example: + + ```mlir + %layout = cute.make_layout(.., ..): !cute.layout<"(8, 4):(4, 1)"> + %inv_layout = cute.right_inverse %layout : (!cute.layout<"(8, 4):(4, 1)">) -> !cute.layout<"(4, 8):(8, 1)"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$input + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_SelectOp : Cute_InferredOp<"select", [Pure]> { + let description = [{ + This operation extracts specific dimensions out of a type implementing + the `CuteValueTypeInterface`, e.g. a shape, a coordinate, a tuple, a layout, ... + + The `mode` attribute indices are used to select the portion of the input type. + The indices in the `mode` attribute can be permutated or repeated, but should + be less than the rank of type. + + Example: + + ```mlir + %shape = cute.make_shape() : () -> !cute.shape<"(1, (2, 4), 8)"> + %mode0 = cute.select<[1]>(%shape) + : (!cute.shape<"(1,(2,4),8)">) -> !cute.shape<"((2, 4))"> + %mode1 = cute.select<[0,1]>(%shape) + : (!cute.shape<"(1,(2,4),8)">) -> !cute.shape<"(1, (2,4))"> + %mode2 = cute.select<[2,0]>(%shape) + : (!cute.shape<"(1,(2,4),8)">) -> !cute.shape<"(8, 1)"> + %mode3 = cute.select<[2,2]>(%shape) + : (!cute.shape<"(1,(2,4),8)">) -> !cute.shape<"(8, 8)"> + ``` + }]; + let arguments = (ins + Cute_ValueTypeInterfaceInstance:$input, + DenseI32ArrayAttr:$mode + ); + let results = (outs + Cute_ValueTypeInterfaceInstance:$result + ); +} +def Cute_ShapeDivOp : Cute_InferredOp<"shape_div", [Pure]> { + let description = [{ + Division for Shapes. + Case Tuple Tuple: + Perform shape_div element-wise + Case Tuple Int: + Fold the division of b across each element of a + Example: shape_div((4,5,6),40) -> shape_div((1,5,6),10) -> shape_div((1,1,6),2) -> (1,1,3) + Case Int Tuple: + Return shape_div(a, product(b)) + Case Int Int: + Enforce the divisibility condition a % b == 0 || b % a == 0 when possible + Return a / b with rounding away from 0 (that is, 1 or -1 when a < b) + + ```mlir + %res = cute.shape_div(%a, %b) + : (!cute.int_tuple<"(4,5)">, !cute.int_tuple<"(40)">) -> !cute.int_tuple<"(1,1)"> + ``` + }]; + let arguments = (ins + Cute_CuteShapeType:$a, + Cute_CuteShapeType:$b + ); + let results = (outs + Cute_CuteShapeType:$result + ); +} +def Cute_SizeOp : Cute_InferredOp<"size", [Pure]> { + let summary = "This operation computes the size of a cute value type by multiplying its dimensions."; + let description = [{ + The `mode` attribute specifies the nested mode in the type for size calculation. + It indexes a sub-scope within the input type and returns the size of that sub-scope. + The rank of `mode` must be less than or equal to the depth of the type. + If `mode` is empty, the operation calculates the size of the entire type. + + ### Examples + + ```mlir + %shape = cute.make_shape() : () -> !cute.shape<"(1,(2,4),9)"> + %size = cute.size(%shape) <{mode=[1]}> : (!cute.shape<"(1,(2,4),9)">) -> !cute.int_tuple<"8"> + ``` + + In this example, the mode attribute is set to [1], which selects the second dimension of the + shape (1,(2,4),9). The size of this dimension is calculated as 2 * 4 = 8. + + ```mlir + %shape = cute.make_shape() : () -> !cute.shape<"(1,(2,4),8)"> + %size = cute.size(%shape) : (!cute.shape<"(1,(2,4),8)">) -> !cute.int_tuple<"64"> + ``` + + In this example, the mode attribute is empty, so the operation calculates the size of the entire + shape (1,(2,4),8). The size is calculated as 1 * 2 * 4 * 8 = 64. + }]; + let arguments = (ins + Cute_CuteShapeTypeOrIntTupleAsIntegerOr2:$input, + DefaultValuedAttr:$mode + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$result + ); +} +def Cute_SliceOp : Cute_InferredOp<"slice", [Pure]> { + let description = [{ + This operation takes slices of a input memref according to the `coord`. Note that + `!coord<"(_,_,k)">` means "create a slice that views (i, j, k) for all valid i, + all valid j, and a specific value of k. + + The `coord` elements can be static, dynamic (represented by "?"), or underscore + (similar to the ":" (the colon punctuation) in Python or Fortran array slices). + However, the mode of `coord` should be less than or equal to the mode of `input` + memref layout; otherwise, a runtime error will occur. + + Example: + + ```mlir + %slice = cute.slice(%alloc, %coord) + : !cute.memref, !cute.coord<"(0, (_, _), 0)"> + ``` + + Example with dynamic coord: + + ```mlir + %slice = cute.slice(%alloc, %coord) + : !cute.memref, !cute.coord<"(_,(64,?),0)"> + ``` + + Example with scaled basis in layout: + + ```mlir + %slice = cute.slice(%coord_tensor, %coord) + : !cute.coord_tensor<"(0,0)", "(10,20,30):(1@0,1@1,20@1)">, !cute.coord<"(0,2,_)"> + ``` + Example with swizzled layout: + + ```mlir + %slice = cute.slice(%swizzle_layout, %coord) + : (!cute.composed_layout<"S<1,5,1> o 0 o ((2,4),(8,4)):((1,16),(2,64))">, + !cute.coord<"((?,?),(_,_))">) -> !cute.composed_layout<"(8,(2,2)):(2,(?,128))"> + ``` + + Example with PDSL memref slice (swizzle in units of bytes): + + ```mlir + %res_memref = !cute.slice(%memref, %crd) : !cute.memref, "(64,32):(32,1)">, !cute.coord + ``` + + Example with PISL memref slice (swizzle in units of elements): + + ```mlir + %res_memref = !cute.slice(%memref, %crd) : !cute.memref o 0 o (64,32):(32,1)">, !cute.coord + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, + Cute_CuteCoordType:$coord + ); + let results = (outs + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_SmemPartitionSizeOp : Cute_Op<"smem_partition_size", [Pure]> { + let description = [{ + This operation queries the static smem layout for one partition in the + current kernel and returns the partition size as an i64 integer. It is only + valid on the kernel side; host-side total smem queries should use + `cute.kernel_smem_size`. + }]; + let arguments = (ins + I32Attr:$partition_id, + UnitAttr:$cumulative + ); + let results = (outs + I64:$size + ); +} +def Cute_StaticOp : Cute_Op<"static", [Pure]> { + let description = [{ + Creates a compile-time constant value with a fully static CuTe type. This operation takes no + operands and produces a single result of one of the following CuTe types: + + - !cute.int_tuple: A tuple of static integers + - !cute.coord: A coordinate with static dimensions + - !cute.stride: A stride with static values + - !cute.shape: A shape with static dimensions + - !cute.tile: A tile with static dimensions and layout + - !cute.layout: A layout with static shape and strides + - !cute.composed_layout: A composed layout with static inner/outer layouts + + The type of the result must be fully static (contain no dynamic/unknown values). + + Some atom types can be fully static with no runtime states, e.g. `UniversalFma`. + + ### Examples: + + ```mlir + // Create a static shape with value 5 + %0 = cute.static : !cute.shape<"5"> + + // Create a static 2D layout with shape (32,64) and strides (64,1) + %1 = cute.static : !cute.layout<"(32,64):(64,1)"> + ``` + }]; + let arguments = (ins + ); + let results = (outs + Cute_MaybeStaticTypeInterfaceInstance:$result + ); +} +def Cute_StencilDivideOp : Cute_InferredOp<"stencil_divide", [AttrSizedOperandSegments, Pure]> { + let summary = "A convolution has the following input tensors:"; + let description = [{ + * Activation tensor (Act): `((N,(D,H,W,...)), (C,(1,1,1)))` + * Filter tensor (Flt): `(K,(C,(T,R,S, ...)))` + * Output tensor (Out): `((N,(Z,P,Q, ...)), K)` + + Where + + * `N` := number of samples in the signal + * `D`, `H`, `W`, ... := spatial dimensions of the activation tensor + * `C` := channel dimension of the activation tensor + * `K` := channel dimension of the filter and output tensor + * `T`,`R`,`S`,... := spoke dimensions of the filter tensor + * `Z`,`P`,`Q`,... := spatial dimensions of the output tensor + + The number of spatial dimensions must DHW... must match both TRS... and ZPQ... + + As is evident in the tensor shapes, these cannot be issued to a GEMM without transformation, since + there are no logical M, N, and K modes into which we can group the tensor modes. + + Every spoke of the filter tensor (TRS) will be applied to some (offset) + view of the activation tensor, thus expanding the logical size of the activation tensor. + Additionally, a similar logical transform of the spatial dimensions can be encoded as + a function of the padding, dilations, traversal strides, and filter spokes. + This gets us to our im2col transform: + + Im2col transform affects the component shapes/strides of the activation tensor in the following way: + + * `ZPQ` Shape : changes `DHW` domain with formula `(1 + (DHW + pad - (((TRS-1) * dilation) + 1)) / traversal_stride)` + * `TRS` Shape : `TRS` domain instead of `(1,1,1)` + * `ZPQ` Strides : Original `DHW` strides get scaled by traversal strides `DHW` + * `TRS` Strides : Original `DHW` strides get scaled by dilation `DHW`. + + With this transform applied, we end up with a set of input and output tensors that are logically + consistent in their `MNK` dimensions, thus allowing us to dispatch to a GEMM. + + * im2col activation layout: `((N,(Z,P,Q)), (C,(T,R,S)))` // logical (M,K) + * filter layout : `( K, (C,(T,R,S)))` // logical (N,K) + * output layout : `((N,(Z,P,Q)), K )` // logical (M,N) + + `stencil_divide` performs this operation given these parameters. + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, + Cute_CuteTileType:$stencil, + Optional:$padding_upper, + Optional:$padding_lower, + Optional:$traversal_stride + ); + let results = (outs + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_SymbolicOp : Cute_Op<"symbolic", [Pure]> { + let description = [{ + A symbol in CuTe IR represent an abstracted value which could be materialized + later in a pass. If it's not materialized, the behavior is undefined. + + Example: + + ```mlir + %0 = cute.symbolic("tile_m") : i32 + %1 = cute.symbolic("cluster_size") : !cute.shape<"(?,2)"> + ``` + }]; + let arguments = (ins + StrAttr:$name + ); + let results = (outs + AnyType:$symbolic + ); +} +def Cute_TileToShapeOp : Cute_InferredOp<"tile_to_shape", [Pure]> { + let description = [{ + The `tile_to_shape` operation is similar to blocked_product, but specifies the + result shape instead of the product shape, which is more convenient in certain + circumstances. + + The op has three operands: + 1. `block`: The layout to repeat. + 2. `trg_shape`: The target shape of the result + 3. `ord_shape`: The order of the modes of `trg_shape` to tile layout `block` with. + E.g. (_2,_1,_3) will cause @a layout to repeat across the second mode first, + the first mode second, and the third mode last. Currently only supports static order. + + Example: + + ```mlir + %0 = cute.tile_to_shape(%block, %trg_shape, %ord_shape) + : (!cute.layout<"(128,128):(128,1)">, + !cute.shape<"(1024,512)">, + !cute.shape<"(2,1)">) + -> !cute.layout<"((128,8),(128,4)):((128,65536),(1,16384))"> + + %0 = cute.tile_to_shape(%block, %trg_shape, %ord_shape) + : (!cute.layout<"((32,4),(16,4)):((16,4),(0,1))">, + !cute.shape<"(256,512,2)">, + !cute.shape<"(1,2,3)">) + -> !cute.layout<"((32,4,2),(16,4,8),2):((16,4,512),(0,1,1024),8192)"> + + %0 = cute.tile_to_shape(%block, %trg_shape, %ord_shape) + : (!cute.layout<"((32,4),(16,4)):((16,4),(0,1))">, + !cute.shape<"(256,512,2)">, + !cute.shape<"(2,1,3)">) + -> !cute.layout<"((32,4,2),(16,4,8),2):((16,4,4096),(0,1,512),8192)"> + + %0 = cute.tile_to_shape(%block, %trg_shape, %ord_shape) + : (!cute.composed_layout<"S<3,4,3> o 0 o ((32,4),(?,4)):((16,4),(0,1))">, + !cute.shape<"(256,?,2)">, + !cute.shape<"(3,2,1)">) + -> !cute.composed_layout<"S<3,4,3> o 0 o ((32,4,2),(?,4,?),2):((16,4,?{div=1024}),(0,1,1024),512)"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$block, + Cute_CuteShapeType:$trg_shape, + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$ord_shape + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_TiledCopyPartitionDOp : Cute_InferredOp<"tiled.copy.partition_D", [Pure]> { + let description = [{ + The `cute.tiled.copy.partition_X` operation accepts: + 1. cute.tiled_copy; + 2. A ViewTypeInterface; + 3. cute.coord. + returns a new ViewTypeInterface type (MemrefType/CoordTensorType) with partitioned layout + and sliced offset (sliced offset will be implemented in the lowering). + + The new partitioned layout has the following meaning: ((vector_mode), rest_m, rest_n). vector_mode + represents the number of instructions issued and the number of elements processed per + instruction; in our example below, we have (2, 1) : (1, 0), meaning that each + instruction is processing two data. Rest_m is the number of iterations + along the first mode, a single iteration in our example. Finally, rest_n is the + number of iterations along the second mode, 128. + + Example: + + ```mlir + %dst_partitioned = cute.tiled.copy.partition_D(...) + : (!cute.tiled_copy, + layout_copy_tv = <"((32,4),(2,32)):((0,1),(128,4))">, + tiler_mn = <"[(4,32):(32,1);2:1]">>, + !cute.memref, "(128,256):(256,1)">, + !cute.coord<"?">) + -> !cute.memref, "((2,1),1,128):((1,0),0,2)"> + ``` + }]; + let arguments = (ins + Cute_TiledCopyAtomTypeOrTiledCopyV2:$tiled_copy, + Cute_ViewTypeInterfaceInstance:$input, + Cute_CuteCoordType:$coord + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_TiledCopyPartitionSOp : Cute_InferredOp<"tiled.copy.partition_S", [Pure]> { + let description = [{ + The `cute.tiled.copy.partition_X` operation accepts: + 1. cute.tiled_copy; + 2. A ViewTypeInterface; + 3. cute.coord. + returns a new ViewTypeInterface type (MemrefType/CoordTensorType) with partitioned layout + and sliced offset (sliced offset will be implemented in the lowering). + + The new partitioned layout has the following meaning: ((vector_mode), rest_m, rest_n). vector_mode + represents the number of instructions issued and the number of elements processed per + instruction; in our example below, we have (2, 1) : (1, 0), meaning that each + instruction is processing two data. Rest_m is the number of iterations + along the first mode, a single iteration in our example. Finally, rest_n is the + number of iterations along the second mode, 128. + + Example: + + ```mlir + %dst_partitioned = cute.tiled.copy.partition_D(...) + : (!cute.tiled_copy, + layout_copy_tv = <"((32,4),(2,32)):((0,1),(128,4))">, + tiler_mn = <"[(4,32):(32,1);2:1]">>, + !cute.memref, "(128,256):(256,1)">, + !cute.coord<"?">) + -> !cute.memref, "((2,1),1,128):((1,0),0,2)"> + ``` + }]; + let arguments = (ins + Cute_TiledCopyAtomTypeOrTiledCopyV2:$tiled_copy, + Cute_ViewTypeInterfaceInstance:$input, + Cute_CuteCoordType:$coord + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_TiledCopyRetileOp : Cute_InferredOp<"tiled.copy.retile", [Pure]> { + let description = [{ + The cute.tiled.copy.retile operation is used to reorganize the layout of a tensor + that has already been tiled or partitioned. This is typically necessary when + two TiledMma or TiledCopy instances are connected, accessing the same memory, + but require different logical views of that memory. + + ### Use Case + + Consider a scenario where a TiledMma instance is the consumer, and a TiledCopy + instance is the producer. The TiledMma instance partitions the memory, creating + a logical view that suits its needs. However, this view may not be compatible with + the TiledCopy instance. To resolve this, the cute.tiled.copy.retile operation is used + to retile the TiledCopy instance, creating a new logical view that is compatible with + the TiledMma instance. + + ``` + // consumer + partition_A(%tiled_mma, ...) + %mma_view = partition_A(%tiled_mma, %sa_tensor, %idx) + + // producer + %tiled_copy = make_tiled_copy_A(%copy_atom, %tiled_mma) + // retile %mma_view to meet needs of tiled_copy + %lds_view = retile(%tiled_copy, %mma_view) + ``` + + ### Operation + + The cute.tiled.copy.retile operation takes two inputs: + * A cute.tiled_copy instance, which represents the tensor to be retiled. + * A CuTeTypeWithLayout derived type, which represents the tensor that has already been tiled or partitioned. + + + Example: + + ```mlir + %retiled = cute.tiled.copy.retile(%tiled_copy, %src_tensor) + : (!cute.tiled_copy, num_matrices=4>, + layout_copy_tv = <"((4,8,4),((2,2,4),1)):((128,4,1),((64,32,512),0))">, + tiler_mn = <"[(4,16):(32,1);32:1]">>, + !cute.memref) + -> !cute.memref + ``` + }]; + let arguments = (ins + Cute_TiledCopyAtomTypeOrTiledCopyV2:$tiled_copy, + Cute_ViewTypeInterfaceInstance:$input + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_TiledDivideOp : Cute_InferredOp<"tiled_divide", [Pure]> { + let summary = "This operation performs a tiled_divide on a view or a layout by a tiler."; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, + Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$tiler + ); + let results = (outs + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_TiledMmaPartitionOp : Cute_InferredOp<"tiled.mma.partition", [Pure]> { + let description = [{ + `cute.tiled.mma.partition` reorganizes a tensor's layout to match what the + MMA atom expects. The operation takes 4 arguments: + + 1. The type of operand. E.g., `CUTE_MMA_OPERAND_A` or `CUTE_MMA_OPERAND_B`. + 2. A `!cute.tiled_mma` value that specifies the kind of MMA. + 3. The unpartitioned operand tensor (ViewTypeInterface-derived SSA value). + 4. The `!cute.coord` value of the execution unit for which to partition the + the input tensor. + + The coordinate type depends on the MMA atom and the operand memory space. + Some examples: + * SM80: thread ID + * SM90: It depends... + - When partitioning A with "frag_kind = ss": warp-group ID + - When partitioning A with "frag_kind = rs": thread ID + - When partitioning B: warp-group ID + - When partitioning C/D: thread ID + * SM100: CTA ID within the cluster (always 0 for 1-CTA mode) + + The operation returns a new ViewTypeInterface-derived value + (CuteMemrefType/CuteCoordTensorType) with partitioned layout and sliced + offset (sliced offset will be implemented in the lowering). + + Example: + + ```mlir + // Partition an operand A tensor of shape 128x32 (residing in shared + // memory) for a !tiled_mma atom that operates on shape 32x16x16. (I.e., + // four SM80 MMA atoms with shape 16x8x16.) The result tensor is a view + // into the A tensor that represents the part that thread %tid_coord is + // operating on. + // + // The last two modes of the result tensor are 4,2. This is because the + // tiled atom is applied 4 times in M direction and 2 times in K direction. + // + // K=0:15 K=16:31 + // ┌────────────────────┬────────────────────┐ + // M=0:31 │ tiled atom (0,0) │ tiled atom (0,1) │ + // ├────────────────────┼────────────────────┤ + // M=32:63 │ tiled atom (1,0) │ tiled atom (1,1) │ + // ├────────────────────┼────────────────────┤ + // M=64:95 │ tiled atom (2,0) │ tiled atom (2,1) │ + // ├────────────────────┼────────────────────┤ + // M=96:127 │ tiled atom (3,0) │ tiled atom (3,1) │ + // └────────────────────┴────────────────────┘ + // + // For each tiled atom application of size 32x16x16, thread %tid_coord gets + // a subview of size 2,2,2 of operand A. (128 threads are cooperatively + // computing the !tiled_mma of shape 32x16x16, but an individual thread + // does not need access to the full 32x16 matrix A part.) + // + %partitioned = cute.tiled.mma.partition A(%0, %1, %tid_coord) + : (!cute.tiled_mma, + atom_layout_MNK = <"(2,2,1)">>, + !cute.memref, S<2,4,3>, "(128,32):(32,1)">, + !cute.coord<"?">) + -> !cute.memref, S<2,4,3>, "((2,2,2),4,2):((1,256,8),1024,16)"> + + // Partition an operand A tensor of shape 256x512 for an SM100 !tiled_mma. + // There are 256/128=2 tiled atom applications in M direction and + // 512/16=32 tiled atom applications in K direction. The atom specifies a + // single CTA, so that CTA get the entire A tensor. The coordinate is not + // really need in this case, but we pass it anyway to keep the API uniform. + %partitioned = cute.tiled.mma.partition A (%0, %1, %cta_coord) + : (!cute.tiled_mma, + atom_layout_MNK = <"(1,1,1):(0,0,0)">>, + !cute.memref, "(256,512):(512,1)">, + !cute.coord<"0">) + -> !cute.memref, "((128,16),2,32):((512,1),65536,16)"> + ``` + }]; + let arguments = (ins + Cute_TiledMMAAtomType:$tiled_mma, + Cute_ViewTypeInterfaceInstance:$input, + Cute_CuteCoordType:$coord, + Cute_MmaOperandEnum:$operand_id + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_TiledMmaPartitionShapeOp : Cute_InferredOp<"tiled.mma.partition_shape", [Pure]> { + let description = [{ + This operation partitions a shape into an MMA shape. For example, it takes a logical (M,K,...) + shape and tiles it into a shape like (MMA,MMA_M,MMA_K,...) where + - MMA is the MMA mode, for example (128,16) for an MMA instruction of shape 128x128x16 + - MMA_M is the number of times the MMA repeats in the M-mode + - MMA_K is the number of times the MMA repeats in the K-mode + - ... denotes the "rest" + }]; + let arguments = (ins + Cute_TiledMMAAtomType:$tiled_mma, + Cute_CuteShapeType:$input, + Cute_MmaOperandEnum:$operand_id + ); + let results = (outs + Cute_CuteShapeType:$result + ); +} +def Cute_TiledProductOp : Cute_InferredOp<"tiled_product", [Pure]> { + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$input, + Cute_CuTeLayoutType:$tiler + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_ToIntTupleOp : Cute_InferredOp<"to_int_tuple", [Pure]> { + let description = [{ + The `cute.cast_to_int_tuple` operation casts from any x-tuple type (shape, stride, coord) to an int-tuple type. + The operation preserves the values but changes their type to int-tuple. + + cast will fail if input types contains modes not supported by int-tuple, e.g. _, scaled based, etc. + + Example: + + ```mlir + // Cast from shape to int_tuple + %0 = cute.to_int_tuple(%shape) : !cute.shape<"(8,4)"> to !cute.int_tuple<"(8,4)"> + + // Cast from stride to int_tuple + %1 = cute.to_int_tuple(%stride) : !cute.stride<"(8,4)"> to !cute.int_tuple<"(8,4)"> + + // Cast from coord to int_tuple + %2 = cute.to_int_tuple(%coord) : !cute.coord<"(8,4)"> to !cute.int_tuple<"(8,4)"> + + // Cast from stride to int_tuple preserving divisibility + %3 = cute.to_int_tuple(%stride) : !cute.stride<"(8,?{div=4})"> to !cute.int_tuple<"(8,?{div=4})"> + ``` + }]; + let arguments = (ins + Cute_XTupleLikeType:$src + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$dst + ); +} +def Cute_TupleAddOp : Cute_InferredOp<"tuple_add", [Pure]> { + let description = [{ + The `cute.tuple_add` operation performs element-wise addition between two CuTe x-tuple types. + It accepts two input operands and returns a result of the same kind, where each element in the result + is the sum of the corresponding elements in the input tuples. + + This operation applies subtraction recursively for nested tuples, matching corresponding elements + at each level of the tuple hierarchy. + + Example: + + ```mlir + // Element-wise add two arithmetic tuples + %result = cute.tuple_add(%a, %b) + : (!cute.int_tuple<"(?,(3,?))">, !cute.int_tuple<"(3,(1,?))">) -> !cute.int_tuple<"(?,(4,?))"> + + // Element-wise add with static values + %b = cute.static : !cute.int_tuple<"(3,(1,3))"> + %result = cute.tuple_add(%a, %b) : (!cute.int_tuple<"(?,(3,?))">) -> !cute.int_tuple<"(?,(4,?))"> + ``` + }]; + let arguments = (ins + Cute_ArithmeticTupleType:$lhs, + Cute_ArithmeticTupleType:$rhs + ); + let results = (outs + Cute_ArithmeticTupleType:$result + ); +} +def Cute_TupleDivOp : Cute_InferredOp<"tuple_div", [Pure]> { + let description = [{ + The `cute.tuple_div` operation performs element-wise division between two CuTe x-tuple types. + It accepts two input operands and returns a result of the same kind, where each element in the result + is the quotient of the corresponding elements in the input tuples. + + This operation applies division recursively for nested tuples, matching corresponding elements + at each level of the tuple hierarchy. + + Example: + + ```mlir + // Element-wise divide two arithmetic tuples + %result = cute.tuple_div(%a, %b) + : (!cute.int_tuple<"(?,(3,?))">, !cute.int_tuple<"(3,(1,?))">) -> !cute.int_tuple<"(?,(3,?))"> + + // Element-wise divide with static values + %b = cute.static : !cute.int_tuple<"(3,(1,3))"> + %result = cute.tuple_div(%a, %b) : (!cute.int_tuple<"(?,(3,?))">) -> !cute.int_tuple<"(?,(0,?))"> + ``` + }]; + let arguments = (ins + Cute_ArithmeticTupleType:$lhs, + Cute_ArithmeticTupleType:$rhs + ); + let results = (outs + Cute_ArithmeticTupleType:$result + ); +} +def Cute_TupleModOp : Cute_InferredOp<"tuple_mod", [Pure]> { + let description = [{ + The `cute.tuple_mod` operation computes the element-wise modulo (remainder) + between two CuTe x-tuple types. Both operands must be of the same tuple kind + (e.g., `!cute.int_tuple`), and the result is a tuple of the same structure + and type. + + For each corresponding element in the input tuples, the operation computes + the remainder of the division of the left-hand side element by the right-hand + side element. If the tuples are nested, the operation is applied recursively, + matching elements at each level of the tuple hierarchy. + + This operation supports both static and dynamic tuple elements. When static + values are provided, the modulo is computed at compile time; otherwise, it is + performed at runtime. + + **Examples:** + + ```mlir + // Element-wise modulo of two arithmetic tuples + %result = cute.tuple_mod(%a, %b) + : (!cute.int_tuple<"(?,(3,?))">, !cute.int_tuple<"(3,(1,?))">) + -> !cute.int_tuple<"(?,(0,?))"> + + // Element-wise modulo with static values + %b = cute.static : !cute.int_tuple<"(3,(1,3))"> + %result = cute.tuple_mod(%a, %b) + : (!cute.int_tuple<"(?,(3,?))">) -> !cute.int_tuple<"(?,(1,?))"> + ``` + + **Semantics:** + - The operation traverses the tuple structure recursively. + - For each pair of corresponding elements `(x, y)`, the result is `x % y`. + - The structure and rank of the result tuple matches that of the operands. + - If the tuple contains nested tuples, the modulo is applied to each + corresponding sub-tuple. + + This operation is useful for broadcasting, tiling, and other tensor algebra + applications where modular arithmetic over structured tuples is required. + }]; + let arguments = (ins + Cute_ArithmeticTupleType:$lhs, + Cute_ArithmeticTupleType:$rhs + ); + let results = (outs + Cute_ArithmeticTupleType:$result + ); +} +def Cute_TupleMulOp : Cute_InferredOp<"tuple_mul", [Pure]> { + let description = [{ + The `cute.tuple_mul` operation performs element-wise multiplication between two CuTe x-tuple types. + It accepts two input operands and returns a result of the same kind, where each element in the result + is the product of the corresponding elements in the input tuples. + + This operation applies multiplication recursively for nested tuples, matching corresponding elements + at each level of the tuple hierarchy. + + Example: + + ```mlir + // Element-wise multiply two arithmetic tuples + %result = cute.tuple_mul(%a, %b) + : (!cute.int_tuple<"(?,(3,?))">, !cute.int_tuple<"(3,(1,?))">) -> !cute.int_tuple<"(?,(3,?))"> + + // Element-wise multiply with static values + %b = cute.static : !cute.int_tuple<"(3,(1,3))"> + %result = cute.tuple_mul(%a, %b) : (!cute.int_tuple<"(?,(3,?))">) -> !cute.int_tuple<"(?,(3,?))"> + ``` + }]; + let arguments = (ins + Cute_ArithmeticTupleType:$lhs, + Cute_ArithmeticTupleType:$rhs + ); + let results = (outs + Cute_ArithmeticTupleType:$result + ); +} +def Cute_TupleProductEachOp : Cute_InferredOp<"tuple.product_each", [Pure]> { + let summary = "This operation performs \"product_each\" on each element in a tuple"; + let description = [{ + Example: + + ```mlir + %tuple = cute.make_int_tuple() : !cute.int_tuple<"(2,(5,4))"> + %prod = cute.tuple.product_each(%tuple) : (!cute.int_tuple<"(2,(5,4))">) -> !cute.int_tuple<"(2, 20)"> + ``` + }]; + let arguments = (ins + Cute_IntTupleAsIntegerOrTupleOfIntTupleS2:$input + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS2:$result + ); +} +def Cute_TupleProductOp : Cute_InferredOp<"tuple.product", [Pure]> { + let summary = "This operation performs \"product\" on all elements in a tuple"; + let description = [{ + Example: + + ```mlir + %tuple = cute.make_int_tuple() : !cute.int_tuple<"(2,(5,4))"> + %prod = cute.product(%tuple) : (!cute.int_tuple<"(2,(5,4))">) -> !cute.int_tuple<"40"> + ``` + }]; + let arguments = (ins + Cute_IntTupleAsIntegerOrTupleOfIntTupleS2:$input + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS2:$result + ); +} +def Cute_TupleSubOp : Cute_InferredOp<"tuple_sub", [Pure]> { + let description = [{ + The `cute.tuple_sub` operation performs element-wise subtraction between two CuTe x-tuple types. + It accepts two input operands and returns a result of the same kind, where each element in the result + is the difference of the corresponding elements in the input tuples. + + This operation applies subtraction recursively for nested tuples, matching corresponding elements + at each level of the tuple hierarchy. + + Example: + + ```mlir + // Element-wise subtract two arithmetic tuples + %result = cute.tuple_sub(%a, %b) + : (!cute.int_tuple<"(?,(3,?))">, !cute.int_tuple<"(3,(1,?))">) -> !cute.int_tuple<"(?,(2,?))"> + + // Element-wise subtract with static values + %b = cute.static : !cute.int_tuple<"(3,(1,3))"> + %result = cute.tuple_sub(%a, %b) : (!cute.int_tuple<"(?,(3,?))">) -> !cute.int_tuple<"(?,(2,?))"> + ``` + }]; + let arguments = (ins + Cute_ArithmeticTupleType:$lhs, + Cute_ArithmeticTupleType:$rhs + ); + let results = (outs + Cute_ArithmeticTupleType:$result + ); +} +def Cute_UnpackTupleOp : Cute_Op<"unpack_tuple", [Pure]> { + let description = [{ + Unpacks a tuple into its individual components. The number and types + of the components must match the number and types of the results. + }]; + let arguments = (ins + Cute_ATupleIsAFixedLengthOrderedList:$tuple + ); + let results = (outs + Variadic:$result + ); +} +def Cute_ZippedDivideOp : Cute_InferredOp<"zipped_divide", [Pure]> { + let description = [{ + This operation performs "zipped_divide" on the input and produce a zipped_divided result, + and it automatically infer the "zipped_divided" layout for the result type. + + Example: + + ```mlir + %alloc = cute.memref.alloc() : !cute.memref + %shape = cute.make_shape() : () -> !cute.shape<"(4,8)"> + %zdiv = cute.zipped_divide(%alloc, %shape) + : !cute.memref, !cute.shape<"(4,8)"> + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, + Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$tiler + ); + let results = (outs + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_ZippedProductOp : Cute_InferredOp<"zipped_product", [Pure]> { + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$input, + Cute_CuTeLayoutType:$tiler + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$result + ); +} + +#endif // CUTE_IR_GENERATED_CUTE_OPS_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOpsPrivate.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOpsPrivate.td deleted file mode 100644 index e6e5c761..00000000 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOpsPrivate.td +++ /dev/null @@ -1,428 +0,0 @@ -//===- CutePrivateCompatOps.td - proprietary CuTe op schemas -*- tablegen -*-===// - -#ifndef CUTE_IR_DIALECT_CUTE_IR_CUTE_PRIVATE_COMPAT_OPS_TD -#define CUTE_IR_DIALECT_CUTE_IR_CUTE_PRIVATE_COMPAT_OPS_TD - -include "cute_ir/Dialect/Cute/IR/CuteAttrs.td" - -class Cute_PrivateSchemaOp traits = []> - : Cute_Op { - let summary = "Operation recovered from the CuTe-DSL operation registry"; -} - -def Cute_PrivateAddOffsetOp : Cute_PrivateSchemaOp<"add_offset"> { - let arguments = (ins AnyType:$src, AnyType:$offset); - let results = (outs AnyType:$dst); -} - -def Cute_PrivateApplySwizzleOp : Cute_PrivateSchemaOp<"apply_swizzle"> { - let arguments = (ins AnyType:$ptr, OptionalAttr:$swizzle); - let results = (outs AnyType:$swizzled_ptr); -} - -def Cute_PrivateAssumeOp : Cute_PrivateSchemaOp<"assume"> { - let arguments = (ins AnyType:$src); - let results = (outs AnyType:$dst); -} - -def Cute_PrivateCopyOp - : Cute_PrivateSchemaOp<"copy", [AttrSizedOperandSegments]> { - let arguments = (ins AnyType:$copy_atom, Variadic:$src, - Variadic:$dst, Optional:$pred); - let results = (outs); -} - -def Cute_PrivateCopyMakeFragmentOp - : Cute_PrivateSchemaOp<"copy.make_fragment"> { - let arguments = (ins AnyType:$atom, AnyType:$input, I32Attr:$operand_id); - let results = (outs AnyType:$result); -} - -def Cute_PrivateCopyAtomCallOp - : Cute_PrivateSchemaOp<"copy_atom_call", [AttrSizedOperandSegments]> { - let arguments = (ins AnyType:$atom, Variadic:$src, - Variadic:$dst, Optional:$pred); - let results = (outs); -} - -def Cute_PrivateDerefArithTupleIterOp - : Cute_PrivateSchemaOp<"deref_arith_tuple_iter"> { - let arguments = (ins AnyType:$iter); - let results = (outs AnyType:$arith_tuple); -} - -def Cute_PrivateDerefDescIterOp : Cute_PrivateSchemaOp<"deref_desc_iter"> { - let arguments = (ins AnyType:$iter); - let results = (outs AnyType:$value); -} - -def Cute_PrivateDerefineOp : Cute_PrivateSchemaOp<"derefine"> { - let arguments = (ins AnyType:$src); - let results = (outs AnyType:$dst); -} - -def Cute_PrivateFastDivmodComputeOp - : Cute_PrivateSchemaOp<"fast_divmod.compute"> { - let arguments = (ins AnyType:$dividend, AnyType:$divisor); - let results = (outs AnyType:$quotient, AnyType:$remainder); -} - -def Cute_PrivateFastDivmodCreateDivisorOp - : Cute_PrivateSchemaOp<"fast_divmod.create_divisor"> { - let arguments = (ins AnyType:$divisor); - let results = (outs AnyType:$fast_divmod_divisor); -} - -def Cute_PrivateFastDivmodDivideOp - : Cute_PrivateSchemaOp<"fast_divmod.divide"> { - let arguments = (ins AnyType:$dividend, AnyType:$divisor); - let results = (outs AnyType:$quotient); -} - -def Cute_PrivateFastDivmodGetAuxOp - : Cute_PrivateSchemaOp<"fast_divmod.get_aux"> { - let arguments = (ins AnyType:$divisor); - let results = (outs AnyType:$multiplier, AnyInteger:$shift1, - AnyInteger:$shift2); -} - -def Cute_PrivateFastDivmodGetDivisorOp - : Cute_PrivateSchemaOp<"fast_divmod.get_divisor"> { - let arguments = (ins AnyType:$fast_divmod_divisor); - let results = (outs AnyType:$divisor); -} - -def Cute_PrivateFastDivmodMakeDivisorOp - : Cute_PrivateSchemaOp<"fast_divmod.make_divisor"> { - let arguments = (ins AnyType:$divisor, AnyType:$multiplier, - AnyType:$sh1, AnyType:$sh2); - let results = (outs AnyType:$fast_divmod_divisor); -} - -def Cute_PrivateFilterOp : Cute_PrivateSchemaOp<"filter"> { - let arguments = (ins AnyType:$input); - let results = (outs AnyType:$result); -} - -def Cute_PrivateFilterZerosOp : Cute_PrivateSchemaOp<"filter_zeros"> { - let arguments = (ins AnyType:$input, Optional:$target_profile); - let results = (outs AnyType:$res); -} - -def Cute_PrivateGemmOp - : Cute_PrivateSchemaOp<"gemm", [AttrSizedOperandSegments]> { - let arguments = (ins AnyType:$mma_atom, AnyType:$d, Variadic:$a, - Variadic:$b, AnyType:$c); - let results = (outs); -} - -def Cute_PrivateGetFlatCoordOp : Cute_PrivateSchemaOp<"get_flat_coord"> { - let arguments = (ins AnyType:$index, AnyType:$input); - let results = (outs AnyType:$result); -} - -def Cute_PrivateGetHierCoordOp : Cute_PrivateSchemaOp<"get_hier_coord"> { - let arguments = (ins AnyType:$index, AnyType:$input); - let results = (outs AnyType:$result); -} - -def Cute_PrivateGetIntegralCoordOp - : Cute_PrivateSchemaOp<"get_integral_coord"> { - let arguments = (ins AnyType:$index, AnyType:$input); - let results = (outs AnyType:$result); -} - -def Cute_PrivateGetIterOp : Cute_PrivateSchemaOp<"get_iter"> { - let arguments = (ins AnyType:$source); - let results = (outs AnyType:$ptr); -} - -def Cute_PrivateGetLayoutOp : Cute_PrivateSchemaOp<"get_layout"> { - let arguments = (ins AnyType:$input); - let results = (outs AnyType:$layout); -} - -def Cute_PrivateIntToPtrOp : Cute_PrivateSchemaOp<"inttoptr"> { - let arguments = (ins AnyType:$src); - let results = (outs AnyType:$dst); -} - -def Cute_PrivateKernelSmemSizeOp : Cute_PrivateSchemaOp<"kernel_smem_size"> { - let arguments = (ins SymbolRefAttr:$kernel_name); - let results = (outs I64:$size); -} - -def Cute_PrivateLoadScaledIndexOp - : Cute_PrivateSchemaOp<"load_scaled_index"> { - let arguments = (ins AnyType:$ptr_a, AnyType:$ptr_b, AnyType:$index, - AnyType:$stride); - let results = (outs AnyType:$result); -} - -def Cute_PrivateLocalPartitionOp : Cute_PrivateSchemaOp<"local_partition"> { - let arguments = (ins AnyType:$input, AnyType:$tiler, AnyType:$index); - let results = (outs AnyType:$result); -} - -def Cute_PrivateLocalTileOp : Cute_PrivateSchemaOp<"local_tile"> { - let arguments = (ins AnyType:$input, AnyType:$tile, AnyType:$coord, - OptionalAttr:$proj); - let results = (outs AnyType:$result); -} - -def Cute_PrivateMakeArithTupleIterOp - : Cute_PrivateSchemaOp<"make_arith_tuple_iter"> { - let arguments = (ins Optional:$value); - let results = (outs AnyType:$iter); -} - -def Cute_PrivateMakeAtomOp : Cute_PrivateSchemaOp<"make_atom"> { - let arguments = (ins Variadic:$values); - let results = (outs AnyType:$atom); -} - -def Cute_PrivateMakeDescIterOp : Cute_PrivateSchemaOp<"make_desc_iter"> { - let arguments = (ins AnyType:$value); - let results = (outs AnyType:$iter); -} - -def Cute_PrivateMakeFragmentLikeOp - : Cute_PrivateSchemaOp<"make_fragment_like"> { - let arguments = (ins AnyType:$src); - let results = (outs AnyType:$dst); -} - -def Cute_PrivateMakeIdentityTensorOp - : Cute_PrivateSchemaOp<"make_identity_tensor"> { - let arguments = (ins AnyType:$shape); - let results = (outs AnyType:$result); -} - -def Cute_PrivateMakeSparseElemOp : Cute_PrivateSchemaOp<"make_sparse_elem"> { - let arguments = (ins AnyType:$physical_storage); - let results = (outs AnyType:$result); -} - -def Cute_PrivateMakeTiledCopyOp : Cute_PrivateSchemaOp<"make_tiled_copy"> { - let arguments = (ins AnyType:$atom); - let results = (outs AnyType:$result); -} - -def Cute_PrivateMakeTiledCopyV2Op - : Cute_PrivateSchemaOp<"make_tiled_copy_v2"> { - let arguments = (ins AnyType:$atom); - let results = (outs AnyType:$result); -} - -def Cute_PrivateMakeTiledMmaOp : Cute_PrivateSchemaOp<"make_tiled_mma"> { - let arguments = (ins AnyType:$atom); - let results = (outs AnyType:$result); -} - -def Cute_PrivateMakeTupleOp : Cute_PrivateSchemaOp<"make_tuple"> { - let arguments = (ins Variadic:$values); - let results = (outs AnyType:$result); -} - -def Cute_PrivateMakeViewOp : Cute_PrivateSchemaOp<"make_view"> { - let arguments = (ins AnyType:$iter, Optional:$layout); - let results = (outs AnyType:$result); -} - -def Cute_PrivateMemrefAllocSmemOp - : Cute_PrivateSchemaOp<"memref.alloc_smem"> { - let arguments = (ins); - let results = (outs AnyType:$memref); -} - -def Cute_PrivateMemrefAllocaOp : Cute_PrivateSchemaOp<"memref.alloca"> { - let arguments = (ins Optional:$layout); - let results = (outs AnyType:$result); -} - -def Cute_PrivateMemrefLoadOp : Cute_PrivateSchemaOp<"memref.load"> { - let arguments = (ins AnyType:$src, AnyType:$coord); - let results = (outs AnyType:$result); -} - -def Cute_PrivateMemrefLoadVecOp - : Cute_PrivateSchemaOp<"memref.load_vec", [AttrSizedOperandSegments]> { - let arguments = (ins AnyType:$src, Optional:$mask, - Optional:$pass_thru); - let results = (outs AnyVectorOfAnyRank:$result); -} - -def Cute_PrivateMemrefStoreOp : Cute_PrivateSchemaOp<"memref.store"> { - let arguments = (ins AnyType:$dst, AnyType:$coord, AnyType:$value); - let results = (outs); -} - -def Cute_PrivateMemrefStoreVecOp - : Cute_PrivateSchemaOp<"memref.store_vec"> { - let arguments = (ins AnyVectorOfAnyRank:$value, AnyType:$dst, - Optional:$mask); - let results = (outs); -} - -def Cute_PrivateMmaMakeFragmentOp - : Cute_PrivateSchemaOp<"mma.make_fragment"> { - let arguments = (ins AnyType:$atom, AnyType:$input, I32Attr:$operand_id); - let results = (outs AnyType:$result); -} - -def Cute_PrivateMmaAtomCallOp - : Cute_PrivateSchemaOp<"mma_atom_call", [AttrSizedOperandSegments]> { - let arguments = (ins AnyType:$atom, AnyType:$d, Variadic:$a, - Variadic:$b, AnyType:$c); - let results = (outs); -} - -def Cute_PrivatePrefetchOp : Cute_PrivateSchemaOp<"prefetch"> { - let arguments = (ins AnyType:$prefetch_atom, Variadic:$src); - let results = (outs); -} - -def Cute_PrivatePrefetchAtomCallOp - : Cute_PrivateSchemaOp<"prefetch_atom_call"> { - let arguments = (ins AnyType:$atom, Variadic:$src); - let results = (outs); -} - -def Cute_PrivatePrintTmaDescIm2colOp - : Cute_PrivateSchemaOp<"print_tma_desc_im2col"> { - let arguments = (ins AnyType:$input, AnyType:$fd); - let results = (outs); -} - -def Cute_PrivatePrintTmaDescTiledOp - : Cute_PrivateSchemaOp<"print_tma_desc_tiled"> { - let arguments = (ins AnyType:$input, AnyType:$fd); - let results = (outs); -} - -def Cute_PrivatePrintViewOp : Cute_PrivateSchemaOp<"print_view"> { - let arguments = (ins AnyType:$src, Optional:$coord, - OptionalAttr:$verbose, - OptionalAttr:$is_signed); - let results = (outs); -} - -def Cute_PrivatePtrLoadOp : Cute_PrivateSchemaOp<"ptr.load"> { - let arguments = (ins AnyType:$ptr); - let results = (outs AnyType:$value); -} - -def Cute_PrivatePtrStoreOp : Cute_PrivateSchemaOp<"ptr.store"> { - let arguments = (ins AnyType:$value, AnyType:$ptr); - let results = (outs); -} - -def Cute_PrivatePtrToIntOp : Cute_PrivateSchemaOp<"ptrtoint"> { - let arguments = (ins AnyType:$src); - let results = (outs AnyInteger:$dst); -} - -def Cute_PrivateRandomDelayOp : Cute_PrivateSchemaOp<"random_delay"> { - let arguments = (ins); - let results = (outs); -} - -def Cute_PrivateRecastIterOp : Cute_PrivateSchemaOp<"recast_iter"> { - let arguments = (ins AnyType:$src); - let results = (outs AnyType:$dst); -} - -def Cute_PrivateReduceOp : Cute_PrivateSchemaOp<"reduce"> { - let arguments = (ins AnyVectorOfAnyRank:$input, AnyType:$shape, - AnyType:$init_val, - Optional:$predication, - AnyAttr:$reduction_op, - OptionalAttr:$reduction_profile); - let results = (outs AnyType:$output, Optional:$reduced_shape); -} - -def Cute_PrivateSmemPartitionSizeOp - : Cute_PrivateSchemaOp<"smem_partition_size"> { - let arguments = (ins I32Attr:$partition_id, - OptionalAttr:$cumulative); - let results = (outs I64:$size); -} - -def Cute_PrivateStencilDivideOp - : Cute_PrivateSchemaOp<"stencil_divide", [AttrSizedOperandSegments]> { - let arguments = (ins AnyType:$input, AnyType:$stencil, - Optional:$padding_upper, - Optional:$padding_lower, - Optional:$traversal_stride); - let results = (outs AnyType:$result); -} - -def Cute_PrivateSymbolicOp : Cute_PrivateSchemaOp<"symbolic"> { - let arguments = (ins StrAttr:$name); - let results = (outs AnyType:$symbolic); -} - -def Cute_PrivateTiledCopyPartitionDOp - : Cute_PrivateSchemaOp<"tiled.copy.partition_D"> { - let arguments = (ins AnyType:$tiled_copy, AnyType:$input, AnyType:$coord); - let results = (outs AnyType:$result); -} - -def Cute_PrivateTiledCopyPartitionSOp - : Cute_PrivateSchemaOp<"tiled.copy.partition_S"> { - let arguments = (ins AnyType:$tiled_copy, AnyType:$input, AnyType:$coord); - let results = (outs AnyType:$result); -} - -def Cute_PrivateTiledCopyRetileOp - : Cute_PrivateSchemaOp<"tiled.copy.retile"> { - let arguments = (ins AnyType:$tiled_copy, AnyType:$input); - let results = (outs AnyType:$result); -} - -def Cute_PrivateTiledMmaPartitionOp - : Cute_PrivateSchemaOp<"tiled.mma.partition"> { - let arguments = (ins AnyType:$tiled_mma, AnyType:$input, AnyType:$coord, - I32Attr:$operand_id); - let results = (outs AnyType:$result); -} - -def Cute_PrivateTiledMmaPartitionShapeOp - : Cute_PrivateSchemaOp<"tiled.mma.partition_shape"> { - let arguments = (ins AnyType:$tiled_mma, AnyType:$input, - I32Attr:$operand_id); - let results = (outs AnyType:$result); -} - -def Cute_PrivateTupleProductOp : Cute_PrivateSchemaOp<"tuple.product"> { - let arguments = (ins AnyType:$input); - let results = (outs AnyType:$result); -} - -def Cute_PrivateTupleProductEachOp - : Cute_PrivateSchemaOp<"tuple.product_each"> { - let arguments = (ins AnyType:$input); - let results = (outs AnyType:$result); -} - -def Cute_PrivateTupleDivOp : Cute_PrivateSchemaOp<"tuple_div"> { - let arguments = (ins AnyType:$lhs, AnyType:$rhs); - let results = (outs AnyType:$result); -} - -def Cute_PrivateTupleModOp : Cute_PrivateSchemaOp<"tuple_mod"> { - let arguments = (ins AnyType:$lhs, AnyType:$rhs); - let results = (outs AnyType:$result); -} - -def Cute_PrivateTupleMulOp : Cute_PrivateSchemaOp<"tuple_mul"> { - let arguments = (ins AnyType:$lhs, AnyType:$rhs); - let results = (outs AnyType:$result); -} - -def Cute_PrivateUnpackTupleOp : Cute_PrivateSchemaOp<"unpack_tuple"> { - let arguments = (ins AnyType:$tuple); - let results = (outs Variadic:$result); -} - -#endif // CUTE_IR_DIALECT_CUTE_IR_CUTE_PRIVATE_COMPAT_OPS_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesAll.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesAll.td index 4e375294..a8cfdb29 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesAll.td +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesAll.td @@ -1,2 +1,5 @@ +// The dialect has one generated type parser and printer, so the release's +// types and the compiler's are generated together from here. + include "cute_ir/Dialect/Cute/IR/CuteTypes.td" -include "cute_ir/Dialect/Cute/IR/CuteTypesPrivate.td" +include "cute_ir/Dialect/Cute/IR/CuteTypesCompiler.td" diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesCompiler.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesCompiler.td new file mode 100644 index 00000000..3b240ad2 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesCompiler.td @@ -0,0 +1,269 @@ +// The cute dialect's types beyond the release, generated from the +// nvidia-cutlass-dsl 4.8.0 release; do not edit. +// +// Sources: the compiler library (`_cutlass_ir`): its generated parser +// messages give the classes with a declarative syntax, their parameters in +// declaration order and their C++ types; the typed Python bindings of the +// types (`get` signatures and getters) give the parameters of the others; the +// live compiler, loaded from Python, gives the syntax (what it prints for what +// it parses, including every default it leaves out) and the verifier +// messages. +// +// Differences from the compiler, where its C++ types are not defined here: +// - The atoms of tiled_copy / tiled_copy_v2 / tiled_mma are +// `CopyAtomTypeInterface` / `CopyAtomTypeInterfaceV2` / +// `MmaAtomTypeInterface` in the compiler; plain types here. +// - ptr, memref and i have bespoke parsers in the compiler, so their +// parameter names and order are not recorded; the order here follows the +// Python `get` of each. +// - coord_tensor and memref layouts are `cutegen::layout` in the compiler's +// messages but accept composed layouts; they are a layout or composed +// layout type here. +// Verifier rules not reproduced: the bitlayout's cosize and element-count +// rules and TiledMMA's "requires static permutations of MNK" (neither +// triggered by any probe). The memref's sparse density rule ("layout must be +// dense but cannot be divided by sparsity") is inferred from probes: every +// probe agrees with it. + +#ifndef CUTE_IR_GENERATED_CUTE_TYPES_COMPILER_TD +#define CUTE_IR_GENERATED_CUTE_TYPES_COMPILER_TD + +include "cute_ir/Dialect/Cute/IR/CuteTypes.td" +include "cute_ir/Dialect/Cute/IR/CuteAttrsCompiler.td" +include "cute_ir/Dialect/Cute/IR/CuteEnums.td" + +def Cute_PtrType : Cute_Type<"Ptr", "ptr"> { + let summary = "Pointer with address space, alignment, swizzle and bit layout"; + let description = [{ + `<[value_type, ][space][, align][, S][, bitlayout<"L">]>`, the + fields in this order. Defaults are left out when printing and filled in + when parsing: + + - the address space is `generic` (always printed); + - the alignment is the value type's natural one: its width in bytes for + an integer or float that is a whole number of bytes, 1 for `i1`, 0 + (none) for other sub-byte widths, the physical type's bytes for a + `!cute.sparse_elem`, and 1 for any other type or no type; + - the swizzle is the identity `S<0,4,3>`; + - the bit layout is `(b,n):(1,s)` for an integer or float of `b` bits + (`s` = 8 for `i1`, else `b`) with `n` the least count making a chunk of + whole bytes (32-bit chunks in `tmem`), and + `(pb,(L,n)):(1,(0,pb))` for a `!cute.sparse_elem` of `pb` bits. + + A sparse pointer (the Python `sparsity`) is a `!cute.sparse_elem` value + type. + + ```mlir + !cute.ptr, S<2,4,3>> + !cute.ptr + !cute.ptr> + ``` + }]; + let parameters = (ins + Cute_OptionalParam<"mlir::Type", "pointee type">:$value_type, + Cute_Param<"AddressSpace", "address space">:$address_space, + Cute_Param<"uint64_t", "alignment in bytes; 0 only as a sub-byte type's natural alignment">:$alignment, + Cute_OptionalParam<"SwizzleAttr", "swizzle; null for the identity">:$swizzle, + Cute_OptionalParam<"BitLayoutAttr", "bit layout; null for the default">:$bitlayout + ); + let hasCustomAssemblyFormat = 1; + let genVerifyDecl = 1; + let extraClassDeclaration = [{ + /// The alignment a pointer to `valueType` has when none is spelled. + static uint64_t getNaturalAlignment(mlir::Type valueType); + /// The bit layout a pointer has when none is spelled (null if none). + static BitLayoutAttr getDefaultBitLayout(mlir::Type valueType, + AddressSpace addressSpace); + }]; +} + +def Cute_MemRefType : Cute_Type<"MemRef", "memref"> { + let summary = "Tensor view: a pointer with a layout"; + let description = [{ + `][, S], "layout"[, bitlayout<"L">]>`: + the pointer fields as in `!cute.ptr` (the value type is required), then the + layout or composed layout, then a non-default bit layout. A composed + layout whose inner part is the identity swizzle is read as its outer + layout. + + ```mlir + !cute.memref, S<2,4,3>, "((8,16),(32,1),(1,3)):((32,256),(1,0),(0,4096))"> + !cute.memref> + ``` + }]; + let parameters = (ins + Cute_Param<"PtrType", "pointer">:$ptr, + Cute_Param<"mlir::Type", "layout or composed-layout type">:$layout + ); + let hasCustomAssemblyFormat = 1; + let genVerifyDecl = 1; + let extraClassDeclaration = [{ + mlir::Type getValueType() const { return getPtr().getValueType(); } + AddressSpace getAddressSpace() const { return getPtr().getAddressSpace(); } + uint64_t getAlignment() const { return getPtr().getAlignment(); } + SwizzleAttr getSwizzle() const { return getPtr().getSwizzle(); } + }]; +} + +def Cute_CoordTensorType : Cute_Type<"CoordTensor", "coord_tensor"> { + let summary = "Coordinate tensor: an arithmetic tuple with a layout"; + let description = [{ + ```mlir + !cute.coord_tensor<"(?,?,?)", "((512,1),1):((1/2@0,0),0)"> + ``` + }]; + let parameters = (ins + Cute_Param<"IntTupleType", "origin arithmetic tuple">:$arith_tuple, + Cute_Param<"mlir::Type", "layout or composed-layout type">:$layout + ); + let hasCustomAssemblyFormat = 1; + let genVerifyDecl = 1; +} + +def Cute_ArithTupleIteratorType + : Cute_Type<"ArithTupleIterator", "arith_tuple_iter"> { + let summary = "Iterator over an arithmetic tuple"; + let description = [{ + ```mlir + !cute.arith_tuple_iter<"(?,?{div=16})"> + ``` + }]; + let parameters = (ins + Cute_Param<"IntTupleType", "current arithmetic tuple">:$arith_tuple + ); + // `$arith_tuple` alone would print a space after the mnemonic. + let hasCustomAssemblyFormat = 1; +} + +// `!cute.i`: one mnemonic per width, which the generated type dispatch +// cannot match, so the class has no mnemonic and CuteTypes.cpp routes +// `i` keywords to it. +def Cute_ConstrainedIntType : TypeDef { + let summary = "Integer of any width with known divisibility and power of two"; + let description = [{ + `!cute.i[<[pow2, ]divby D>]`; `divby 1` is left out (the default), + `divby 0` and negative divisors are kept. The compiler also reads + `!cute_nvgpu.i` as this type. + + ```mlir + !cute.i32 + !cute.i16 + !cute.i64 + ``` + }]; + // The compiler's type name for it. + let typeName = "cute.ConstrainedInt"; + let parameters = (ins + Cute_Param<"int64_t", "known divisor">:$divisibility, + Cute_Param<"unsigned", "integer width">:$width, + Cute_Param<"bool", "known to be a power of two">:$is_pow2 + ); + let extraClassDeclaration = [{ + static mlir::Type parse(mlir::AsmParser &parser, unsigned width); + void print(mlir::AsmPrinter &printer) const; + }]; +} + +def Cute_FastDivmodDivisorType + : Cute_Type<"FastDivmodDivisor", "fast_divmod_divisor"> { + let summary = "Precomputed divisor of the fast divmod operations"; + let description = [{ + ```mlir + !cute.fast_divmod_divisor<32> + !cute.fast_divmod_divisor<64, is_pow_2> + ``` + }]; + let parameters = (ins + Cute_Param<"unsigned", "integer width">:$width, + Cute_Param<"bool", "whether the divisor is a power of two">:$is_pow_2 + ); + let hasCustomAssemblyFormat = 1; +} + +def Cute_SparseElemType : Cute_Type<"SparseElem", "sparse_elem"> { + let summary = "Logical sparse element stored in a physical type"; + let description = [{ + `num_logical` logical elements stored as `physical_type` (an integer, + float or vector of them). + + ```mlir + !cute.sparse_elem<2, vector<1xf16>> + ``` + }]; + let parameters = (ins + Cute_Param<"int", "number of logical elements">:$num_logical, + Cute_Param<"mlir::Type", "physical storage type">:$physical_type + ); + let assemblyFormat = "`<` $num_logical `,` $physical_type `>`"; + let genVerifyDecl = 1; +} + +def Cute_TiledCopyType : Cute_Type<"TiledCopy", "tiled_copy"> { + let summary = "Copy atom tiled over threads and values"; + let description = [{ + ```mlir + !cute.tiled_copy, layout_copy_tv = <"(32,4):(4,1)">, tiler_mn = <"[4:1;32:1]">> + ``` + }]; + let parameters = (ins + Cute_Param<"mlir::Type", "copy atom">:$copy_atom, + Cute_Param<"LayoutAttr", "thread-value layout">:$layout_copy_tv, + Cute_Param<"TileAttr", "MN tiler">:$tiler_mn + ); + let assemblyFormat = [{ + `<` $copy_atom `,` `layout_copy_tv` `=` $layout_copy_tv `,` + `tiler_mn` `=` $tiler_mn `>` + }]; +} + +def Cute_TiledCopyV2Type : Cute_Type<"TiledCopyV2", "tiled_copy_v2"> { + let summary = "V2 copy atom tiled over threads and values"; + let description = [{ + ```mlir + !cute.tiled_copy_v2, layout_copy_tv = <"(1,8):(0,1)">, tiler_mn = <"[8:1]">> + ``` + }]; + let parameters = (ins + Cute_Param<"mlir::Type", "V2 copy atom">:$copy_atom, + Cute_Param<"LayoutAttr", "thread-value layout">:$layout_copy_tv, + Cute_Param<"TileAttr", "MN tiler">:$tiler_mn + ); + let assemblyFormat = [{ + `<` $copy_atom `,` `layout_copy_tv` `=` $layout_copy_tv `,` + `tiler_mn` `=` $tiler_mn `>` + }]; +} + +def Cute_TiledMmaType : Cute_Type<"TiledMma", "tiled_mma"> { + let summary = "MMA atom tiled over MNK"; + let description = [{ + The permutation is optional (absent, not `[_;_;_]`, when not given). + + ```mlir + !cute.tiled_mma f32 >, atom_layout_MNK = <"(16,16,1):(16,1,0)">, permutation_MNK = <"[(16,4):(4,1);(16,4):(4,1);_]">> + ``` + }]; + let parameters = (ins + Cute_Param<"mlir::Type", "mma atom">:$mma_atom, + Cute_Param<"LayoutAttr", "rank-3 atom layout over MNK">:$atom_layout_MNK, + Cute_OptionalParam<"TileAttr", "rank-3 permutation over MNK">:$permutation_MNK + ); + let assemblyFormat = [{ + `<` $mma_atom `,` `atom_layout_MNK` `=` $atom_layout_MNK + (`,` `permutation_MNK` `=` $permutation_MNK^)? `>` + }]; + let genVerifyDecl = 1; +} + +def Cute_TupleType : Cute_Type<"Tuple", "tuple"> { + let summary = "Non-empty tuple of types"; + let description = [{ + ```mlir + !cute.tuple + ``` + }]; + let parameters = (ins ArrayRefParameter<"mlir::Type", "element types">:$types); + let assemblyFormat = "`<` $types `>`"; +} + +#endif // CUTE_IR_GENERATED_CUTE_TYPES_COMPILER_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesPrivate.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesPrivate.td deleted file mode 100644 index 34f7d21e..00000000 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesPrivate.td +++ /dev/null @@ -1,92 +0,0 @@ -#ifndef CUTE_IR_DIALECT_CUTE_IR_CUTE_PRIVATE_COMPAT_TYPES_TD -#define CUTE_IR_DIALECT_CUTE_IR_CUTE_PRIVATE_COMPAT_TYPES_TD - -include "cute_ir/Dialect/Cute/IR/CuteAttrs.td" - -def Cute_ArithTupleIteratorType - : Cute_Type<"ArithTupleIterator", "arith_tuple_iter"> { - let summary = "Iterator over an arithmetic tuple"; - let parameters = (ins - Cute_Param<"IntTupleType", "arithmetic tuple carried by the iterator">:$arithTuple - ); - let hasCustomAssemblyFormat = 1; -} - -def Cute_ConstrainedInt32Type : Cute_Type<"ConstrainedInt32", "i32"> { - let summary = "32-bit integer with a divisibility constraint"; - let parameters = (ins - Cute_Param<"uint64_t", "known positive divisor">:$divisibleBy - ); - let hasCustomAssemblyFormat = 1; - let genVerifyDecl = 1; -} - -def Cute_ConstrainedInt64Type : Cute_Type<"ConstrainedInt64", "i64"> { - let summary = "64-bit integer with a divisibility constraint"; - let parameters = (ins - Cute_Param<"uint64_t", "known positive divisor">:$divisibleBy - ); - let hasCustomAssemblyFormat = 1; - let genVerifyDecl = 1; -} - -def Cute_FastDivmodDivisorType - : Cute_Type<"FastDivmodDivisor", "fast_divmod_divisor"> { - let summary = "Precomputed divisor used by fast divmod operations"; - let parameters = (ins - Cute_Param<"unsigned", "integer width">:$width, - Cute_Param<"bool", "whether the divisor is a power of two">:$isPow2 - ); - let hasCustomAssemblyFormat = 1; - let genVerifyDecl = 1; -} - -def Cute_SparseElemType : Cute_Type<"SparseElem", "sparse_elem"> { - let summary = "Logical sparse element represented by physical storage"; - let parameters = (ins - Cute_Param<"int", "number of logical elements">:$numLogical, - Cute_Param<"mlir::Type", "physical storage type">:$physicalType - ); - let assemblyFormat = "`<` $numLogical `,` qualified($physicalType) `>`"; - let genVerifyDecl = 1; -} - -def Cute_PtrType : Cute_Type<"Ptr", "ptr"> { - let summary = "CuTe pointer with memory-space, alignment, and swizzle metadata"; - let parameters = (ins - Cute_OptionalParam<"mlir::Type", "optional pointed-to value type">:$valueType, - Cute_Param<"mlir::StringAttr", "CuTe address-space mnemonic">:$memorySpace, - Cute_Param<"uint64_t", "byte alignment">:$alignment, - Cute_OptionalParam<"SwizzleAttr", "optional pointer swizzle">:$swizzle - ); - let hasCustomAssemblyFormat = 1; - let genVerifyDecl = 1; -} - -def Cute_MemRefType : Cute_Type<"MemRef", "memref"> { - let summary = "CuTe tensor view pairing a pointer with a static layout"; - let parameters = (ins - Cute_Param<"PtrType", "underlying pointer">:$ptr, - Cute_Param<"mlir::Type", "layout or composed-layout type">:$layout - ); - let hasCustomAssemblyFormat = 1; - let genVerifyDecl = 1; - let extraClassDeclaration = [{ - mlir::Type getValueType() const { return getPtr().getValueType(); } - mlir::StringAttr getMemorySpace() const { return getPtr().getMemorySpace(); } - uint64_t getAlignment() const { return getPtr().getAlignment(); } - SwizzleAttr getSwizzle() const { return getPtr().getSwizzle(); } - }]; -} - -def Cute_CoordTensorType : Cute_Type<"CoordTensor", "coord_tensor"> { - let summary = "Coordinate iterator paired with a CuTe layout"; - let parameters = (ins - Cute_Param<"IntTupleType", "coordinate arithmetic tuple">:$arithTuple, - Cute_Param<"mlir::Type", "layout or composed-layout type">:$layout - ); - let hasCustomAssemblyFormat = 1; - let genVerifyDecl = 1; -} - -#endif // CUTE_IR_DIALECT_CUTE_IR_CUTE_PRIVATE_COMPAT_TYPES_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/BUILD.bazel b/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/BUILD.bazel index fca978d1..9cf051ca 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/BUILD.bazel +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/BUILD.bazel @@ -7,7 +7,12 @@ td_library( name = "td_files", srcs = glob(["*.td"]), includes = ["/mlir/dialects/cute_ir/include"], - deps = ["@llvm-project//mlir:OpBaseTdFiles"], + deps = [ + "//mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR:tabledefs", + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], ) gentbl_cc_library( @@ -58,6 +63,51 @@ gentbl_cc_library( deps = [":td_files"], ) +# The compiler's nvgpu enums (CuteNVGPUEnums.td does not include the cute +# enums, which the cute dialect emits). +gentbl_cc_library( + name = "enums_inc_gen", + includes = ["../../../.."], + tbl_outs = [ + ( + ["-gen-enum-decls"], + "CuteNVGPUEnums.h.inc", + ), + ( + ["-gen-enum-defs"], + "CuteNVGPUEnums.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteNVGPUEnums.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "attrs_inc_gen", + includes = ["../../../.."], + tbl_outs = [ + ( + [ + "-gen-attrdef-decls", + "-attrdefs-dialect=cute_nvgpu", + ], + "CuteNVGPUAttrs.h.inc", + ), + ( + [ + "-gen-attrdef-defs", + "-attrdefs-dialect=cute_nvgpu", + ], + "CuteNVGPUAttrs.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteNVGPUAttrs.td", + deps = [":td_files"], +) + +# Every cute_nvgpu operation of the compiler's registry. gentbl_cc_library( name = "ops_inc_gen", includes = ["../../../.."], @@ -81,8 +131,12 @@ cc_library( hdrs = ["CuteNVGPUDialect.h"], includes = ["../../../.."], deps = [ + ":attrs_inc_gen", ":dialect_inc_gen", + ":enums_inc_gen", ":ops_inc_gen", ":types_inc_gen", + "//mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR:headers", + "@llvm-project//mlir:InferTypeOpInterface", ], ) diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUAttrs.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUAttrs.td new file mode 100644 index 00000000..94c2cbdf --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUAttrs.td @@ -0,0 +1,132 @@ +//===- CuteNVGPUAttrs.td - CuTe NVIDIA GPU attributes ------*- tablegen -*-===// +// +// Generated for nvidia-cutlass-dsl 4.8.0 from the DSL compiler +// (`_cutlass_ir.cu13` parser messages and probing its attribute parser and +// printer) and its enum bindings; do not edit by hand. +// +// The enum attributes are CuteNVGPUEnums.td; `#cute_nvgpu.major`, over the +// cute dialect's MajorMode, is here so that the enum generator run over +// CuteNVGPUEnums.td emits the nvgpu enums only. The fragment attributes are +// `key = value` lists in a fixed order; a defaulted field is printed only +// when set. +// +//===----------------------------------------------------------------------===// + +#ifndef CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_ATTRS_TD +#define CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_ATTRS_TD + +include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.td" +include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUEnums.td" +include "cute_ir/Dialect/Cute/IR/CuteEnums.td" + +def CuteNVGPU_MajorModeAttr + : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +class CuteNVGPU_Attr + : AttrDef { + let mnemonic = attrMnemonic; +} + +// `true` / `false`. +class CuteNVGPU_BoolParam : AttrOrTypeParameter<"bool", ""> { + let parser = "::mlir::cutlass_compiler::cute_nvgpu::parseBool($_parser)"; + let printer = "$_printer << ($_self ? \"true\" : \"false\")"; +} +def CuteNVGPU_Bool : CuteNVGPU_BoolParam; +def CuteNVGPU_FalseByDefault : CuteNVGPU_BoolParam { + let defaultValue = "false"; +} + +def CuteNVGPU_RmemFrgAttr : CuteNVGPU_Attr<"RmemFrg", "rmem_frg"> { + let summary = "Register-memory fragment of an MMA operand"; + let description = [{ + `#cute_nvgpu.rmem_frg` + }]; + let parameters = (ins + "::mlir::Type":$value_type, + "::mlir::cutlass_compiler::cute::MmaOperand":$operand, + CuteNVGPU_FalseByDefault:$derive_elem_type + ); + let assemblyFormat = [{ + `<` `value_type` `=` $value_type `,` `operand` `=` $operand + (`,` `derive_elem_type` `=` $derive_elem_type^)? `>` + }]; +} + +class CuteNVGPU_SmemFrg + : CuteNVGPU_Attr { + let summary = arch # " shared-memory fragment"; + let description = "`#cute_nvgpu." # attrMnemonic # "`"; + let parameters = (ins "::mlir::cutlass_compiler::cute::MajorMode":$major_mode); + let assemblyFormat = "`<` `major` `=` $major_mode `>`"; +} + +def CuteNVGPU_SM90SmemFrgAttr + : CuteNVGPU_SmemFrg<"SM90SmemFrg", "arch.sm90.smem_frg", "SM90">; +def CuteNVGPU_SM100SmemFrgAttr + : CuteNVGPU_SmemFrg<"SM100SmemFrg", "arch.sm100.smem_frg", "SM100">; +def CuteNVGPU_SM100CircularSmemFrgAttr + : CuteNVGPU_SmemFrg<"SM100CircularSmemFrg", "arch.sm100.circular_smem_frg", + "SM100 circular">; +def CuteNVGPU_SM107SmemFrgAttr + : CuteNVGPU_SmemFrg<"SM107SmemFrg", "arch.sm107.smem_frg", "SM107">; + +def CuteNVGPU_SM100TmemFrgAttr + : CuteNVGPU_Attr<"SM100TmemFrg", "arch.sm100.tmem_frg"> { + let summary = "SM100 tensor-memory fragment"; + let description = [{ + `#cute_nvgpu.arch.sm100.tmem_frg` + }]; + let parameters = (ins + "::mlir::Type":$data_type, + "::mlir::Type":$storage_type, + "int":$cta_group, + "TmemAllocMode":$tmem_alloc_mode + ); + let assemblyFormat = [{ + `<` `data_type` `=` $data_type `,` `storage_type` `=` $storage_type `,` + `cta_group` `=` $cta_group `,` `tmem_alloc_mode` `=` $tmem_alloc_mode `>` + }]; +} + +def CuteNVGPU_SM100TmemSfFrgAttr + : CuteNVGPU_Attr<"SM100TmemSfFrg", "arch.sm100.tmem_sf_frg"> { + let summary = "SM100 tensor-memory scale-factor fragment"; + let description = [{ + `#cute_nvgpu.arch.sm100.tmem_sf_frg` + }]; + let parameters = (ins + "::mlir::Type":$sf_type, + "int":$sf_vec_size, + "int":$cta_group, + CuteNVGPU_Bool:$is_sfa, + "TmemAllocMode":$tmem_alloc_mode + ); + let assemblyFormat = [{ + `<` `sf_type` `=` $sf_type `,` `sf_vec_size` `=` $sf_vec_size `,` + `cta_group` `=` $cta_group `,` `is_sfa` `=` $is_sfa `,` + `tmem_alloc_mode` `=` $tmem_alloc_mode `>` + }]; +} + +def CuteNVGPU_SM100TmemEFrgAttr + : CuteNVGPU_Attr<"SM100TmemEFrg", "arch.sm100.tmem_e_frg"> { + let summary = "SM100 tensor-memory sparse-metadata fragment"; + let description = [{ + `#cute_nvgpu.arch.sm100.tmem_e_frg` + }]; + let parameters = (ins "::mlir::Type":$a_type, "::mlir::Type":$e_type); + let assemblyFormat = "`<` `a_type` `=` $a_type `,` `e_type` `=` $e_type `>`"; +} + +def CuteNVGPU_NotImplementedFrgAttr + : CuteNVGPU_Attr<"NotImplementedFrg", "not_implemented_frg"> { + let summary = "Fragment of an atom that has none"; + let description = "`#cute_nvgpu.not_implemented_frg<>`"; + let assemblyFormat = "`<` `>`"; +} + +#endif // CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_ATTRS_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h b/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h index b0e9fa0a..0011c3de 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h @@ -3,12 +3,31 @@ #ifndef CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_DIALECT_H #define CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_DIALECT_H +#include + +#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" #include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinAttributes.h" #include "mlir/IR/DialectImplementation.h" #include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/InferTypeOpInterface.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" #include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h.inc" +#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUEnums.h.inc" + +namespace mlir::cutlass_compiler::cute_nvgpu { +/// Parses `true` or `false`. +FailureOr parseBool(AsmParser &parser); +/// The TMA data format a non-executable TMA atom takes for an element type +/// when none is given, if there is one. +std::optional defaultTmaDataFormat(Type elementType); +} // namespace mlir::cutlass_compiler::cute_nvgpu + +#define GET_ATTRDEF_CLASSES +#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUAttrs.h.inc" + #define GET_TYPEDEF_CLASSES #include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUTypes.h.inc" diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.td index 743d4c05..88fe7fd4 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.td +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.td @@ -3,6 +3,7 @@ #ifndef CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_DIALECT_TD #define CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_DIALECT_TD +include "mlir/IR/AttrTypeBase.td" include "mlir/IR/DialectBase.td" include "mlir/IR/OpBase.td" @@ -11,16 +12,17 @@ def CuteNVGPU_Dialect : Dialect { let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; let summary = "NVIDIA GPU operations and atom types used by CuTe-DSL"; let description = [{ - This dialect mirrors the operation and type registry embedded in the - proprietary CuTe-DSL compiler. Proven types receive concrete TypeDefs; - parameterized types remain lossless opaque types until their storage and - verification contracts are fully recovered. + This dialect mirrors the operation, type and attribute registry of + nvidia-cutlass-dsl 4.8.0, embedded in the proprietary CuTe-DSL compiler. + The type parser is hand-written only to accept `!cute_nvgpu.iN`, the + compiler's alias of `!cute.iN`. }]; let extraClassDeclaration = [{ ::mlir::Type parseType(::mlir::DialectAsmParser &parser) const override; void printType(::mlir::Type type, ::mlir::DialectAsmPrinter &printer) const override; }]; + let useDefaultAttributePrinterParser = 1; } class CuteNVGPU_Op traits = []> diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUEnums.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUEnums.td new file mode 100644 index 00000000..3a350562 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUEnums.td @@ -0,0 +1,683 @@ +// The cute_nvgpu dialect's enums, generated from the nvidia-cutlass-dsl 4.8.0 +// enum bindings (`_cute_nvgpu_enum_gen.py`); do not edit. +// +// Case values and spellings are the compiler's. An enum the bindings build +// as a dialect attribute also has its EnumAttr, printed `#`. +// The cute enums the nvgpu bindings repeat are the cute dialect's; their +// nvgpu attributes are in CuteNVGPUAttrs.td, so that -gen-enum-decls +// over this file emits the nvgpu enums only. + +#ifndef CUTE_IR_GENERATED_CUTENVGPU_ENUMS_TD +#define CUTE_IR_GENERATED_CUTENVGPU_ENUMS_TD + +include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.td" +include "mlir/IR/EnumAttr.td" + +def CuteNVGPU_BinaryOpEnum : I32EnumAttr<"BinaryOp", "Binary operation for single-bit MMA operations", [ + I32EnumAttrCase<"none", 0, "none">, + I32EnumAttrCase<"xor_popc", 1, "xor_popc">, + I32EnumAttrCase<"and_popc", 2, "and_popc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldBulkCopyG2SEnum : I32EnumAttr<"AtomCopyFieldBulkCopyG2S", "Fields stored in Bulk Load Copy Atom type", [ + I32EnumAttrCase<"tma_bar", 1, "tma_bar">, + I32EnumAttrCase<"mcast_mask", 2, "mcast_mask">, + I32EnumAttrCase<"cache_policy", 7, "cache_policy"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldBulkCopyS2GEnum : I32EnumAttr<"AtomCopyFieldBulkCopyS2G", "Fields stored in Bulk Store Copy Atom type", [ + I32EnumAttrCase<"byte_mask", 4, "byte_mask">, + I32EnumAttrCase<"cache_policy", 7, "cache_policy"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldBulkCopyS2SEnum : I32EnumAttr<"AtomCopyFieldBulkCopyS2S", "Fields stored in Bulk CTA to Cluster Copy Atom type", [ + I32EnumAttrCase<"tma_bar", 1, "tma_bar">, + I32EnumAttrCase<"cta_rank", 5, "cta_rank"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldDsmemStoreEnum : I32EnumAttr<"AtomCopyFieldDsmemStore", "Fields stored in CopyAtomDsmemStoreType", [ + I32EnumAttrCase<"mbar_ptr", 8, "mbar_ptr"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldLoadGlobalEnum : I32EnumAttr<"AtomCopyFieldLoadGlobal", "Fields stored in CopyAtomG2RType", [ + I32EnumAttrCase<"cache_policy", 7, "cache_policy"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldNonExec2DGather4TmaLoadEnum : I32EnumAttr<"AtomCopyFieldNonExec2DGather4TmaLoad", "Fields stored in CopyAtomNonExec2DGather4TmaLoadType", [ + I32EnumAttrCase<"tma_desc", 3, "tma_desc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldNonExec2DScatter4TmaStoreEnum : I32EnumAttr<"AtomCopyFieldNonExec2DScatter4TmaStore", "Fields stored in CopyAtomNonExec2DScatter4TmaStoreType", [ + I32EnumAttrCase<"tma_desc", 3, "tma_desc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldNonExecIm2ColTmaLoadEnum : I32EnumAttr<"AtomCopyFieldNonExecIm2ColTmaLoad", "Fields stored in CopyAtomNonExecIm2ColTmaLoadType", [ + I32EnumAttrCase<"im2col_tma_desc", 6, "im2col_tma_desc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldNonExecIm2ColTmaStoreEnum : I32EnumAttr<"AtomCopyFieldNonExecIm2ColTmaStore", "Fields stored in CopyAtomNonExecIm2ColTmaStoreType", [ + I32EnumAttrCase<"im2col_tma_desc", 6, "im2col_tma_desc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldNonExecTiledTmaLoadEnum : I32EnumAttr<"AtomCopyFieldNonExecTiledTmaLoad", "Fields stored in CopyAtomNonExecTiledTmaLoadType", [ + I32EnumAttrCase<"tma_desc", 3, "tma_desc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldNonExecTiledTmaReduceEnum : I32EnumAttr<"AtomCopyFieldNonExecTiledTmaReduce", "Fields stored in CopyAtomNonExecTiledTmaReduceType", [ + I32EnumAttrCase<"tma_desc", 3, "tma_desc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldNonExecTiledTmaStoreEnum : I32EnumAttr<"AtomCopyFieldNonExecTiledTmaStore", "Fields stored in CopyAtomNonExecTiledTmaStoreType", [ + I32EnumAttrCase<"tma_desc", 3, "tma_desc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldStoreGlobalEnum : I32EnumAttr<"AtomCopyFieldStoreGlobal", "Fields stored in CopyAtomR2GType", [ + I32EnumAttrCase<"cache_policy", 7, "cache_policy"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldTmaLoadEnum : I32EnumAttr<"AtomCopyFieldTmaLoad", "Fields stored in Tma Load Copy Atom type", [ + I32EnumAttrCase<"tma_descriptor_ptr", 0, "tma_descriptor_ptr">, + I32EnumAttrCase<"tma_bar", 1, "tma_bar">, + I32EnumAttrCase<"mcast_mask", 2, "mcast_mask">, + I32EnumAttrCase<"cache_policy", 7, "cache_policy">, + I32EnumAttrCase<"g_stride", 9, "g_stride"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldTmaReduceEnum : I32EnumAttr<"AtomCopyFieldTmaReduce", "Fields stored in Tma Reduce Copy Atom type", [ + I32EnumAttrCase<"tma_descriptor_ptr", 0, "tma_descriptor_ptr">, + I32EnumAttrCase<"cache_policy", 7, "cache_policy">, + I32EnumAttrCase<"g_stride", 9, "g_stride"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldTmaStoreEnum : I32EnumAttr<"AtomCopyFieldTmaStore", "Fields stored in Tma Store Copy Atom type", [ + I32EnumAttrCase<"tma_descriptor_ptr", 0, "tma_descriptor_ptr">, + I32EnumAttrCase<"cache_policy", 7, "cache_policy">, + I32EnumAttrCase<"g_stride", 9, "g_stride"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomMmaFieldSM80SparseEnum : I32EnumAttr<"AtomMmaFieldSM80Sparse", "Fields stored in MMA Atom types for SM80", [ + I32EnumAttrCase<"e", 0, "e"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomMmaFieldSM90Enum : I32EnumAttr<"AtomMmaFieldSM90", "Fields stored in MMA Atom types for SM90", [ + I32EnumAttrCase<"accum_c", 0, "accum_c"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomMmaFieldSM100BlockScaledEnum : I32EnumAttr<"AtomMmaFieldSM100BlockScaled", "Fields stored in MMA Atom types for SM100 (block scaled)", [ + I32EnumAttrCase<"accum_c", 0, "accum_c">, + I32EnumAttrCase<"neg_a", 1, "neg_a">, + I32EnumAttrCase<"neg_b", 2, "neg_b">, + I32EnumAttrCase<"sf_a", 3, "sf_a">, + I32EnumAttrCase<"sf_b", 4, "sf_b"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomMmaFieldSM100BlockScaledSparseEnum : I32EnumAttr<"AtomMmaFieldSM100BlockScaledSparse", "Fields stored in MMA Atom types for SM100 (block scaled)", [ + I32EnumAttrCase<"accum_c", 0, "accum_c">, + I32EnumAttrCase<"neg_a", 1, "neg_a">, + I32EnumAttrCase<"neg_b", 2, "neg_b">, + I32EnumAttrCase<"sf_a", 3, "sf_a">, + I32EnumAttrCase<"sf_b", 4, "sf_b">, + I32EnumAttrCase<"e", 5, "e"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomMmaFieldSM100Enum : I32EnumAttr<"AtomMmaFieldSM100", "Fields stored in MMA Atom types for SM100", [ + I32EnumAttrCase<"accum_c", 0, "accum_c">, + I32EnumAttrCase<"neg_a", 1, "neg_a">, + I32EnumAttrCase<"neg_b", 2, "neg_b">, + I32EnumAttrCase<"disable_output_lane", 6, "disable_output_lane"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomMmaFieldSM100SparseEnum : I32EnumAttr<"AtomMmaFieldSM100Sparse", "Fields stored in sparse MMA Atom types for SM100", [ + I32EnumAttrCase<"accum_c", 0, "accum_c">, + I32EnumAttrCase<"neg_a", 1, "neg_a">, + I32EnumAttrCase<"neg_b", 2, "neg_b">, + I32EnumAttrCase<"e", 5, "e"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomMmaFieldSM120BlockScaledEnum : I32EnumAttr<"AtomMmaFieldSM120BlockScaled", "Fields stored in MMA Atom types for SM120 (block scaled)", [ + I32EnumAttrCase<"sf_a", 0, "sf_a">, + I32EnumAttrCase<"sf_b", 1, "sf_b"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_CacheEvictionPriorityEnum : I32EnumAttr<"CacheEvictionPriority", "Cute Cache Eviction Priority kind", [ + I32EnumAttrCase<"EVICT_NORMAL", 0, "evict_normal">, + I32EnumAttrCase<"EVICT_FIRST", 1, "evict_first">, + I32EnumAttrCase<"EVICT_LAST", 2, "evict_last">, + I32EnumAttrCase<"EVICT_UNCHANGED", 3, "evict_unchanged">, + I32EnumAttrCase<"NO_ALLOCATE", 4, "no_allocate"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_CopyS2TBroadcastEnum : I32EnumAttr<"CopyS2TBroadcast", "Broadcast modes for the different utccp instructions", [ + I32EnumAttrCase<"none", 0, "none">, + I32EnumAttrCase<"lw_0213", 1, "lw_0213">, + I32EnumAttrCase<"lw_0123", 2, "lw_0123">, + I32EnumAttrCase<"x4", 3, "x4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_GatherScatterTmaLoadEnum : I32EnumAttr<"GatherScatterTmaLoad", "The various kinds of TMA loads in gather/scatter mode", [ + I32EnumAttrCase<"sm_100", 0, "sm_100">, + I32EnumAttrCase<"sm_100_multicast", 1, "sm_100_multicast">, + I32EnumAttrCase<"sm_100_2sm", 2, "sm_100_2sm">, + I32EnumAttrCase<"sm_100_2sm_multicast", 3, "sm_100_2sm_multicast"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_ArchEnum : I32EnumAttr<"Arch", "NVGpu arch number like sm_103", [ + I32EnumAttrCase<"sm_52", 520, "sm_52">, + I32EnumAttrCase<"sm_70", 700, "sm_70">, + I32EnumAttrCase<"sm_75", 750, "sm_75">, + I32EnumAttrCase<"sm_80", 800, "sm_80">, + I32EnumAttrCase<"sm_86", 860, "sm_86">, + I32EnumAttrCase<"sm_90", 900, "sm_90">, + I32EnumAttrCase<"sm_100", 1000, "sm_100">, + I32EnumAttrCase<"sm_103", 1030, "sm_103">, + I32EnumAttrCase<"sm_107", 1070, "sm_107"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_Im2ColTmaLoadEnum : I32EnumAttr<"Im2ColTmaLoad", "The various kinds of TMA loads in im2col mode", [ + I32EnumAttrCase<"sm_90", 0, "sm_90">, + I32EnumAttrCase<"sm_90_multicast", 1, "sm_90_multicast">, + I32EnumAttrCase<"sm_100_2sm", 2, "sm_100_2sm">, + I32EnumAttrCase<"sm_100_2sm_multicast", 3, "sm_100_2sm_multicast"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_L2PrefetchSizeEnum : I32EnumAttr<"L2PrefetchSize", "CuTe L2 prefetch size options", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"RESERVED", 1, "reserved">, + I32EnumAttrCase<"SIZE_64B", 2, "size_64b">, + I32EnumAttrCase<"SIZE_128B", 3, "size_128b">, + I32EnumAttrCase<"SIZE_256B", 4, "size_256b"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_LdReduceAccPrecisionKindEnum : I32EnumAttr<"LdReduceAccPrecisionKind", "multimem ld_reduce accumulation precision kind", [ + I32EnumAttrCase<"F16", 0, "F16">, + I32EnumAttrCase<"F32", 1, "F32"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_LdsmSzPatternEnum : I32EnumAttr<"LdsmSzPattern", "LDSM's sz pattern, describing the bit size", [ + I32EnumAttrCase<"u16", 0, "u16">, + I32EnumAttrCase<"u4to8", 1, "u4to8">, + I32EnumAttrCase<"s4to8", 2, "s4to8">, + I32EnumAttrCase<"u2to4", 3, "u2to4">, + I32EnumAttrCase<"s2to4", 4, "s2to4">, + I32EnumAttrCase<"u4x16p64to8", 5, "u4x16p64to8">, + I32EnumAttrCase<"u6x16p32to8", 6, "u6x16p32to8">, + I32EnumAttrCase<"u8", 7, "u8"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_LoadCacheModeEnum : I32EnumAttr<"LoadCacheMode", "Cache modes for the load instructions", [ + I32EnumAttrCase<"always", 0, "always">, + I32EnumAttrCase<"global", 1, "global">, + I32EnumAttrCase<"streaming", 2, "streaming">, + I32EnumAttrCase<"last_use", 3, "last_use">, + I32EnumAttrCase<"none", 4, "none"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_MMAIntOverflowEnum : I32EnumAttr<"MMAIntOverflow", "MMA overflow options", [ + I32EnumAttrCase<"satfinite", 1, "satfinite">, + I32EnumAttrCase<"wrapped", 0, "wrapped"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_MemOrderKindEnum : I32EnumAttr<"MemOrderKind", "Memory Ordering kind", [ + I32EnumAttrCase<"WEAK", 0, "weak">, + I32EnumAttrCase<"RELAXED", 1, "relaxed">, + I32EnumAttrCase<"ACQUIRE", 2, "acquire">, + I32EnumAttrCase<"RELEASE", 3, "release">, + I32EnumAttrCase<"ACQ_REL", 4, "acq_rel">, + I32EnumAttrCase<"SC", 5, "sc">, + I32EnumAttrCase<"MMIO", 6, "mmio">, + I32EnumAttrCase<"CONSTANT", 7, "constant">, + I32EnumAttrCase<"VOLATILE", 8, "volatile"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_MemScopeKindEnum : I32EnumAttr<"MemScopeKind", "Cute Memory Scope kind", [ + I32EnumAttrCase<"CTA", 0, "cta">, + I32EnumAttrCase<"CLUSTER", 1, "cluster">, + I32EnumAttrCase<"GPU", 2, "gpu">, + I32EnumAttrCase<"SYS", 3, "sys"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_MmaCollectorOpEnum : I32EnumAttr<"MmaCollectorOp", "Enums for the mma collector op", [ + I32EnumAttrCase<"discard", 0, "discard">, + I32EnumAttrCase<"lastuse", 1, "lastuse">, + I32EnumAttrCase<"fill", 2, "fill">, + I32EnumAttrCase<"use", 3, "use"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_MmaFragKindEnum : I32EnumAttr<"MmaFragKind", "Enums for the mma frag type", [ + I32EnumAttrCase<"smem_desc", 0, "smem_desc">, + I32EnumAttrCase<"sparse_smem_desc", 1, "sparse_smem_desc">, + I32EnumAttrCase<"tmem", 2, "tmem">, + I32EnumAttrCase<"tmem_ws", 3, "tmem_ws">, + I32EnumAttrCase<"tmem_e", 4, "tmem_e">, + I32EnumAttrCase<"tmem_sf", 5, "tmem_sf">, + I32EnumAttrCase<"rmem", 6, "rmem"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_ReductionKindEnum : I32EnumAttr<"ReductionKind", "Op for the TMASTORE instruction", [ + I32EnumAttrCase<"ADD", 0, "ADD">, + I32EnumAttrCase<"MIN", 1, "MIN">, + I32EnumAttrCase<"MAX", 2, "MAX">, + I32EnumAttrCase<"INC", 3, "INC">, + I32EnumAttrCase<"DEC", 4, "DEC">, + I32EnumAttrCase<"AND", 5, "AND">, + I32EnumAttrCase<"OR", 6, "OR">, + I32EnumAttrCase<"XOR", 7, "XOR"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_SharedSpaceEnum : I32EnumAttr<"SharedSpace", "CuTe Shared Space kind", [ + I32EnumAttrCase<"CTA", 0, "cta">, + I32EnumAttrCase<"CLUSTER", 1, "cluster"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_SparseMetadataFormatEnum : I32EnumAttr<"SparseMetadataFormat", "Metadata format used in sparse MMA", [ + I32EnumAttrCase<"tid", 0, "tid"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_StoreCacheModeEnum : I32EnumAttr<"StoreCacheMode", "Cache modes for the store instructions", [ + I32EnumAttrCase<"write_back", 0, "write_back">, + I32EnumAttrCase<"global", 1, "global">, + I32EnumAttrCase<"streaming", 2, "streaming">, + I32EnumAttrCase<"write_through", 3, "write_through">, + I32EnumAttrCase<"none", 4, "none"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_TiledTmaLoadEnum : I32EnumAttr<"TiledTmaLoad", "The various kinds of TMA loads in tiled mode", [ + I32EnumAttrCase<"sm_90", 0, "sm_90">, + I32EnumAttrCase<"sm_90_multicast", 1, "sm_90_multicast">, + I32EnumAttrCase<"sm_100_2sm", 2, "sm_100_2sm">, + I32EnumAttrCase<"sm_100_2sm_multicast", 3, "sm_100_2sm_multicast"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_TmaDataFormatEnum : I32EnumAttr<"TmaDataFormat", "Bits 74-71 of the TMA descriptor", [ + I32EnumAttrCase<"U8", 0, "U8">, + I32EnumAttrCase<"U16", 1, "U16">, + I32EnumAttrCase<"U32", 2, "U32">, + I32EnumAttrCase<"S32", 3, "S32">, + I32EnumAttrCase<"U64", 4, "U64">, + I32EnumAttrCase<"S64", 5, "S64">, + I32EnumAttrCase<"F16_RN", 6, "F16_RN">, + I32EnumAttrCase<"F32_RN", 7, "F32_RN">, + I32EnumAttrCase<"F32_FTZ_RN", 8, "F32_FTZ_RN">, + I32EnumAttrCase<"F64_RN", 9, "F64_RN">, + I32EnumAttrCase<"BF16_RN", 10, "BF16_RN">, + I32EnumAttrCase<"U4", 11, "U4">, + I32EnumAttrCase<"U4_UNPACK_U8", 12, "U4_UNPACK_U8">, + I32EnumAttrCase<"U6_UNPACK_U8", 13, "U6_UNPACK_U8">, + I32EnumAttrCase<"TF32_RN", 14, "TF32_RN">, + I32EnumAttrCase<"TF32_FTZ_RN", 15, "TF32_FTZ_RN"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_TmaLoadModeEnum : I32EnumAttr<"TmaLoadMode", "Modes for the TMA load instruction", [ + I32EnumAttrCase<"tiled", 0, "tiled">, + I32EnumAttrCase<"im2col", 1, "im2col">, + I32EnumAttrCase<"w128", 2, "w128">, + I32EnumAttrCase<"w", 3, "w">, + I32EnumAttrCase<"gather4", 4, "gather4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_TmaStoreModeEnum : I32EnumAttr<"TmaStoreMode", "Modes for the TMA store instruction", [ + I32EnumAttrCase<"tiled", 0, "tiled">, + I32EnumAttrCase<"im2col", 1, "im2col">, + I32EnumAttrCase<"scatter4", 2, "scatter4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_TmemAllocModeEnum : I32EnumAttr<"TmemAllocMode", "Modes for the Tmem allocation", [ + I32EnumAttrCase<"Interleaved", 0, "Interleaved">, + I32EnumAttrCase<"NonInterleaved", 1, "NonInterleaved">, + I32EnumAttrCase<"Duplicated", 2, "Duplicated">, + I32EnumAttrCase<"SfDuplicated_4x1", 3, "SfDuplicated_4x1">, + I32EnumAttrCase<"SfDuplicated_2x2", 4, "SfDuplicated_2x2"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_TmemLoadRedOpEnum : I32EnumAttr<"TmemLoadRedOp", "Reduce operation for the Tmem load instruction", [ + I32EnumAttrCase<"max", 0, "max">, + I32EnumAttrCase<"maxabs", 1, "maxabs">, + I32EnumAttrCase<"min", 2, "min">, + I32EnumAttrCase<"minabs", 3, "minabs"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_CopyS2TBroadcastAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldBulkCopyG2SAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldBulkCopyS2GAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldBulkCopyS2SAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldDsmemStoreAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldLoadGlobalAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldNonExec2DGather4TmaLoadAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldNonExec2DScatter4TmaStoreAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldNonExecIm2ColTmaLoadAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldNonExecIm2ColTmaStoreAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldNonExecTiledTmaLoadAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldNonExecTiledTmaReduceAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldNonExecTiledTmaStoreAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldStoreGlobalAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldTmaLoadAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldTmaReduceAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldTmaStoreAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomMmaFieldSM80SparseAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomMmaFieldSM90Attr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomMmaFieldSM100Attr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomMmaFieldSM100BlockScaledAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomMmaFieldSM100BlockScaledSparseAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomMmaFieldSM100SparseAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomMmaFieldSM120BlockScaledAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_BinaryOpAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_GatherScatterTmaLoadAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_Im2ColTmaLoadAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_LdReduceAccPrecisionKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_LdsmSzPatternAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_LoadCacheModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_MMAIntOverflowAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_MmaCollectorOpAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_MmaFragKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_ReductionKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_TiledTmaLoadAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_TmaDataFormatAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_TmaLoadModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_TmaStoreModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_TmemAllocModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_TmemLoadRedOpAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +#endif // CUTE_IR_GENERATED_CUTENVGPU_ENUMS_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUOps.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUOps.td index 8826363d..da52eccc 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUOps.td +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUOps.td @@ -1,378 +1,2012 @@ -//===- CuteNVGPUOps.td - CuTe NVIDIA GPU operations --------*- tablegen -*-===// +// The cute_nvgpu dialect's operations, generated from the nvidia-cutlass-dsl 4.8.0 release; do not edit. +// +// Sources: the operation bindings (`_cute_nvgpu_ops_gen.py`: names, classes, +// docstrings, operand/result/attribute structure, inference) and the live +// compiler (`_cutlass_ir` loaded from Python): each operand and result +// constraint carries the verifier's wording and accepts every type class the +// verifier let through when probed with one instance of every type; traits +// and interfaces are the registered operation's; memory effects come from +// loop-invariant code motion (hoisted: Pure), canonicalize (dead: no write) +// and CSE (merged: at most reads). Operations whose effects include writes +// or could not be told apart carry none (unknown effects). +// +// Operations keep the generic assembly form; the compiler's custom syntax is +// shown in each description. +// +// Constraints not recovered: +// cute_nvgpu.arch.mma.SM107.umma_block_scaled operand #3: no probe type accepted; any type allowed +// cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse operand #3: no probe type accepted; any type allowed +// cute_nvgpu.arch.mma.SM107.umma_sparse operand #3: no probe type accepted; any type allowed -#ifndef CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_OPS_TD -#define CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_OPS_TD +#ifndef CUTE_IR_GENERATED_CUTENVGPU_OPS_TD +#define CUTE_IR_GENERATED_CUTENVGPU_OPS_TD -include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.td" +include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUAttrs.td" +include "cute_ir/Dialect/Cute/IR/CuteAttrs.td" +include "cute_ir/Dialect/Cute/IR/CuteEnums.td" -class CuteNVGPU_SchemaOp traits = []> - : CuteNVGPU_Op { - let summary = "Operation recovered from the CuTe-DSL operation registry"; +include "mlir/Interfaces/InferTypeOpInterface.td" +include "mlir/Interfaces/SideEffectInterfaces.td" + +// Operand and result constraints, in the compiler's words. +def CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor : Type($_self))">, "A memory reference type that represents a tensor view with an iterator and layout">; +def CuteNVGPU_AllowableDescriptorTypesForS2TSrcOperand : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescType>($_self))">, "allowable descriptor types for S2T src operand">; +def CuteNVGPU_AllowableDescriptorTypesForSM107BlockscaledUMMAA : Type($_self))">, "allowable descriptor types for SM107 Blockscaled UMMA A operand">, BuildableType<"::mlir::cutlass_compiler::cute_nvgpu::SmemDescSM107Type::get($_builder.getContext())">; +def CuteNVGPU_AllowableDescriptorTypesForSM107UMMAAOperand : Type($_self))">, "allowable descriptor types for SM107 UMMA A operand">, BuildableType<"::mlir::cutlass_compiler::cute_nvgpu::SmemDescSM107Type::get($_builder.getContext())">; +def CuteNVGPU_AllowableRegisterTypesForSM120BlockScaledMMA : Type, "allowable register types for SM120 block scaled MMA operations">; +def CuteNVGPU_AllowableRegisterTypesForSM120BlockScaledMMA2 : Type($_self))">, "allowable register types for SM120 block scaled MMA operations">; +def CuteNVGPU_AllowableRegisterTypesForSM120BlockScaledMMA3 : Type, "allowable register types for SM120 block scaled MMA operations">; +def CuteNVGPU_AtomTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledMmaType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM120BlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomLdsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomS2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTAsyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemLdReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemStType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100S2TCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomStsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM10xTmemLoadRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTSyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::UniversalFmaAtomType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80SparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM89Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM90Type>($_self))">, "AtomTypeInterface instance">; +def CuteNVGPU_CopyAtomTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomLdsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomS2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTAsyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemLdReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemStType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomStsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM10xTmemLoadRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTSyncCopyType>($_self))">, "CopyAtomTypeInterface instance">; +def CuteNVGPU_CuTeComposedLayoutType : Type($_self))">, "CuTe composed layout type">; +def CuteNVGPU_CuTeLayoutType : Type($_self))">, "CuTe Layout Type">; +def CuteNVGPU_CuTeMemrefToMemorySpacesSmem : Type($_self))">, "CuTe memref to memory spaces (smem)">; +def CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA : Type($_self))">, "CuTe pointer to allowable types for SM100 UMMA C/D operands in spaces (tmem)">; +def CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA2 : Type, "CuTe pointer to allowable types for SM100 UMMA E operand in spaces (tmem)">; // not recovered: no pool type passed +def CuteNVGPU_CuTePointerToAllowableTypesForSM107UMMA : Type, "CuTe pointer to allowable types for SM107 UMMA SF A/B operands in spaces (tmem)">; // not recovered: no pool type passed +def CuteNVGPU_CuTePointerToAllowableTypesForSM90MMA : Type($_self))">, "CuTe pointer to allowable types for SM90 MMA A/B operands in spaces (smem) with byte alignment 1">; +def CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte : Type($_self))">, "CuTe pointer to global memory with 64-byte alignment">; +def CuteNVGPU_CuTePointerToGlobalMemoryWithByteAlignment : Type($_self))">, "CuTe pointer to global memory with byte alignment 1">; +def CuteNVGPU_CuTePointerToMemorySpacesRmem : Type($_self))">, "CuTe pointer to memory spaces (rmem)">; +def CuteNVGPU_CuTePointerToMemorySpacesSmem : Type($_self))">, "CuTe pointer to memory spaces (smem)">; +def CuteNVGPU_CuTePointerToMemorySpacesSmemOrCuTe : Type($_self))">, "CuTe pointer to memory spaces (smem) or CuTe pointer to memory spaces (gmem)">; +def CuteNVGPU_CuTePointerToMemorySpacesSmemWithByte : Type($_self))">, "CuTe pointer to memory spaces (smem) with byte alignment 1">; +def CuteNVGPU_CuTePointerToMemorySpacesSmemWithByte2 : Type($_self))">, "CuTe pointer to memory spaces (smem) with byte alignment 128">; +def CuteNVGPU_CuTePointerToMemorySpacesTmem : Type($_self))">, "CuTe pointer to memory spaces (tmem)">; +def CuteNVGPU_CuTePointerToSharedMemoryWith8Byte : Type($_self))">, "CuTe pointer to shared memory with 8-byte alignment">; +def CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase : Type($_self))">, "CuTe pointer to shared memory with swizzle base atom compatible alignment with byte alignment 1">; +def CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase2 : Type($_self))">, "CuTe pointer to shared memory with swizzle base atom compatible alignment">; +def CuteNVGPU_CuTePointerToSharedOrDistributedSharedMemory : Type($_self))">, "CuTe pointer to shared or distributed shared memory with 8-byte alignment">; +def CuteNVGPU_CuTePointerType : Type($_self))">, " CuTe pointer type ">; +def CuteNVGPU_CuteCoordTensorType : Type($_self))">, "Cute coord_tensor type">; +def CuteNVGPU_CuteCoordType : Type($_self))">, "Cute coord type">; +def CuteNVGPU_CuteShapeType : Type($_self))">, "Cute shape type">; +def CuteNVGPU_CuteSm107SmemDescType : Type($_self))">, "Cute sm107.smem_desc type">, BuildableType<"::mlir::cutlass_compiler::cute_nvgpu::SmemDescSM107Type::get($_builder.getContext())">; +def CuteNVGPU_CuteSmemDescType : Type($_self))">, "Cute smem_desc type">, BuildableType<"::mlir::cutlass_compiler::cute_nvgpu::SmemDescType::get($_builder.getContext())">; +def CuteNVGPU_CuteSmemDescViewType : Type($_self))">, "Cute smem_desc_view type">; +def CuteNVGPU_GeneralizedLayoutTypeOfCuTe : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self))">, " Generalized layout type of CuTe ">; +def CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS : Type($_self))">, "IntTuple as integer or tuple of IntTuple(s)">; +def CuteNVGPU_Integer : Type($_self))">, "integer">; +def CuteNVGPU_SM100BlackwellSharedToTensorMemoryCopyAtom : Type($_self))">, "SM100 (Blackwell) shared-to-tensor memory copy atom type via tcgen05.cp.">; +def CuteNVGPU_SM100BlackwellSharedToTensorMemoryCopyAtom2 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self))">, "SM100 (Blackwell) shared-to-tensor memory copy atom type via tcgen05.cp. or Tiled copy atom type.">; +def CuteNVGPU_SM90PointerToTMATensorDescriptor : Type($_self))">, "SM90+ pointer to TMA tensor descriptor">; +def CuteNVGPU_SM90TMATensorMemoryAcceleratorLoadAtomType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self))">, "SM90+ TMA (Tensor Memory Accelerator) load atom type via cp.async.bulk. or SM90+ TMA im2col mode load atom type via cp.async.bulk.tensor. or SM90+ TMA (Tensor Memory Accelerator) store atom type via cp.async.bulk. or SM90+ TMA im2col mode store atom type via cp.async.bulk.tensor. or SM90+ TMA store with atomic reduction atom type via cp.reduce.async.bulk. or Type interface for non-executable TMA atom types">; +def CuteNVGPU_SignlessIntegerOrIndexOrCuTeLayoutType : Type($_self) || ::llvm::isa<::mlir::IntegerType>($_self) || ::llvm::isa<::mlir::IndexType>($_self))">, "signless integer or index or CuTe Layout Type">; +def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA : Type($_self))">, "The non-executable copy atom type for TMA loads in 2D gather4 mode">; +def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA2 : Type($_self))">, "The non-executable copy atom type for TMA stores in 2D scatter4 mode">; +def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA3 : Type($_self))">, "The non-executable copy atom type for TMA loads in im2col mode">; +def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA4 : Type($_self))">, "The non-executable copy atom type for TMA stores in im2col mode">; +def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA5 : Type($_self))">, "The non-executable copy atom type for TMA loads in tiled mode">; +def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA6 : Type($_self))">, "The non-executable copy atom type for TMA reduce in tiled mode">; +def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA7 : Type($_self))">, "The non-executable copy atom type for TMA stores in tiled mode">; +def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA8 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self))">, "The non-executable copy atom type for TMA loads in tiled mode or The non-executable copy atom type for TMA stores in tiled mode">; +def CuteNVGPU_TiledCopyAtomType : Type($_self))">, "Tiled copy atom type.">; +def CuteNVGPU_TypeInterfaceForNonExecutableTMAAtomTypes : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self))">, "Type interface for non-executable TMA atom types">; +def CuteNVGPU_VectorOf32BitSignlessIntegerValuesOf : Type($_self))">, "vector of 32-bit signless integer values of length 1/2/4/8/16/32/64/128">; +def CuteNVGPU_VectorOf32BitSignlessIntegerValuesOf2 : Type($_self))">, "vector of 32-bit signless integer values of length 4/8">; +def CuteNVGPU_ViewTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, "ViewTypeInterface instance">; + +// Attribute constraints the bindings name but do not describe. +def CuteNVGPU_ArchCopySM100CopyS2TBitAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{128, 256}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { + let storageType = "::mlir::IntegerAttr"; + let returnType = "::mlir::IntegerAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_ArchCopySM100CopyS2TCtaAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{1, 2}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { + let storageType = "::mlir::IntegerAttr"; + let returnType = "::mlir::IntegerAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_ArchCopySM100CopyS2TDpAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{4, 32, 64, 128}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { + let storageType = "::mlir::IntegerAttr"; + let returnType = "::mlir::IntegerAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_ArchCopySM100TMEMNumBAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{32, 64, 128, 256}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { + let storageType = "::mlir::IntegerAttr"; + let returnType = "::mlir::IntegerAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_ArchCopySM100TMEMNumDpAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{16, 32}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { + let storageType = "::mlir::IntegerAttr"; + let returnType = "::mlir::IntegerAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_ArchCopySM100TMEMNumRepAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{1, 2, 4, 8, 16, 32, 64, 128}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { + let storageType = "::mlir::IntegerAttr"; + let returnType = "::mlir::IntegerAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_ArchCopySM100TmaLoadOpNumCTAAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{1, 2}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { + let storageType = "::mlir::IntegerAttr"; + let returnType = "::mlir::IntegerAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_SM120BlockScaledABTypeAttr : Attr($_self) && (::llvm::isa<::mlir::Float8E4M3FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float8E5M2Type>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float4E2M1FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float6E2M3FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float6E3M2FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of allowable types for SM120 block scaled MMA A/B operands"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_SM120BlockScaledSFTypeAttr : Attr($_self) && (::llvm::isa<::mlir::Float8E4M3FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float8E8M0FNUType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of allowable types for SM120 block scaled MMA A/B scale factors"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_TmaLoadOrStoreModeAttr : Attr($_self)">, "TMA load or store mode attribute"> { + let storageType = "::mlir::Attribute"; + let returnType = "::mlir::Attribute"; + let convertFromStorage = "$_self"; } -def CuteNVGPU_ArchAllocRmemOp : CuteNVGPU_SchemaOp<"arch.alloc_rmem"> { - let arguments = (ins AnyType:$input); - let results = (outs AnyType:$ptr); +// Operations whose result types are inferred in C++. Verification compares +// them with the inferred ones where the rule is recovered for the operands; +// elsewhere the types written stand, as the compiler accepted them. +class CuteNVGPU_InferredOp traits = []> + : CuteNVGPU_Op]> { + let extraClassDefinition = [{ + ::llvm::LogicalResult $cppClass::refineReturnTypes( + ::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, + ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, + ::mlir::PropertyRef properties, ::mlir::RegionRange regions, + ::llvm::SmallVectorImpl<::mlir::Type> &returnTypes) { + ::llvm::SmallVector<::mlir::Type, 4> inferred; + if (::mlir::failed(inferReturnTypes(context, ::std::nullopt, operands, + attributes, properties, regions, + inferred))) + return ::mlir::success(); + if (!isCompatibleReturnTypes(inferred, returnTypes)) + return ::mlir::emitOptionalError( + location, "'", getOperationName(), "' op inferred type(s) ", + inferred, " are incompatible with return type(s) of operation ", + returnTypes); + return ::mlir::success(); + } + }]; } -def CuteNVGPU_ArchAllocSmemOp : CuteNVGPU_SchemaOp<"arch.alloc_smem"> { - let arguments = (ins I32Attr:$input); - let results = (outs AnyType:$ptr); +def CuteNVGPU_ArchAllocRmemOp : CuteNVGPU_Op<"arch.alloc_rmem", [MemoryEffects<[MemAlloc]>]> { + let description = [{ + This operation allocate a per-thread private buffer on the stack, with its size + determined by either the provided element count or the static layout. + + Examples: + + ```mlir + %c32_i32 = arith.constant 32 : i32 + %0 = cute_nvgpu.arch.alloc_rmem(%c32_i32) : i32 -> !cute.ptr + + // %static_layout : !cute.layout<"(8,4):(1,8)"> + %1 = cute_nvgpu.arch.alloc_rmem(%static_layout) : !cute.layout<"(8,4):(1,8)"> -> !cute.ptr + + // %dynamic_layout : !cute.layout<"(?,4):(1,?)"> + %2 = cute_nvgpu.arch.alloc_rmem(%dynamic_layout) : !cute.layout<"(?,4):(1,?)"> -> !cute.ptr + ``` + }]; + let arguments = (ins + CuteNVGPU_SignlessIntegerOrIndexOrCuTeLayoutType:$input + ); + let results = (outs + CuteNVGPU_CuTePointerToMemorySpacesRmem:$ptr + ); +} +def CuteNVGPU_ArchAllocSmemOp : CuteNVGPU_Op<"arch.alloc_smem", [MemoryEffects<[MemAlloc]>]> { + let description = [{ + This operation allocate static shared memory based on either a specified + element count or a static layout. + + Examples: + + ```mlir + // size provided by element count + %0 = cute_nvgpu.arch.alloc_smem(1024) : !cute.ptr> + + // size provided by static layout + %1 = cute_nvgpu.arch.alloc_smem("(8,4):(1,8)") : !cute.ptr> + ``` + }]; + let arguments = (ins + I32Attr:$input + ); + let results = (outs + CuteNVGPU_CuTePointerToMemorySpacesSmem:$ptr + ); +} +def CuteNVGPU_ArchCopySM90BulkCopyG2SOp : CuteNVGPU_Op<"arch.copy.SM90.bulk_copy_g2s", [AttrSizedOperandSegments]> { + let description = [{ + The `arch.copy.SM90.bulk_copy_g2s` operation performs a bulk load from global + to shared memory. + + The `gmem_data_addr` operand is a 16-byte aligned pointer to the source data + in global memory. + + The `dsmem_data_addr` operand is a 16-byte aligned pointer to the destination data + in shared cluster memory. + + The `dsmem_bar_addr` operand is a 8-byte aligned pointer to the barrier in shared + memory. + + The `size` operand specifies the bytes copied by BLKCP, aligned to 16 bytes. + + The `multicast_mask` operand is used when multicasting. + + The `cache_policy` operand is used to indicate the eviction priority of the L2 cache. + + Examples: + ```mlir + %gmem = ... : !cute.ptr> + %smem = ... : !cute.ptr> + %bar = ... : !cute.ptr> + + // Without multicast mask + %0 = cute_nvgpu.arch.copy.SM90.bulk_copy_g2s( + %gmem : !cute.ptr>, + %smem : !cute.ptr>, + %bar : !cute.ptr> + ) size = 128 + + // With multicast mask + %1 = cute_nvgpu.arch.copy.SM90.bulk_copy_g2s( + %gmem : !cute.ptr>, + %smem : !cute.ptr>, + %bar : !cute.ptr> + ) size = 128 mask = %mask + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToGlobalMemoryWithByteAlignment:$gmem_data_addr, + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase:$dsmem_data_addr, + CuteNVGPU_CuTePointerToSharedMemoryWith8Byte:$dsmem_bar_addr, + Optional:$multicast_mask, + Optional:$cache_policy, + I32Attr:$size + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM90BulkCopyS2GOp : CuteNVGPU_Op<"arch.copy.SM90.bulk_copy_s2g", [AttrSizedOperandSegments]> { + let description = [{ + The `arch.copy.SM90.bulk_copy_s2g` operation performs a bulk load from shared + to global memory. + + The `smem_data_addr` operand is a 16-byte aligned pointer to the sourced data + in shared memory. + + The `gmem_data_addr` operand is a 16-byte aligned pointer to the estination data + in global memory. + + The `size` operand specifies the bytes copied by BLKCP, aligned to 16 bytes. + + The `byte_mask` operand is specifies whether the i-th byte of each 16-byte wide + chunk of source data is copied to the destination. + + The `cache_policy` operand is used to indicate the eviction priority of the L2 cache. + + Examples: + ```mlir + %smem = ... : !cute.ptr> + %gmem = ... : !cute.ptr> + + // Without byte mask + %0 = cute_nvgpu.arch.copy.SM90.bulk_copy_s2g( + %smem : !cute.ptr>, + %gmem : !cute.ptr>, + ) size = 128 + + // With byte mask + %1 = cute_nvgpu.arch.copy.SM90.bulk_copy_s2g( + %smem : !cute.ptr>, + %gmem : !cute.ptr>, + ) size = 128 byte_mask = %mask + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase:$smem_data_addr, + CuteNVGPU_CuTePointerToGlobalMemoryWithByteAlignment:$gmem_data_addr, + Optional:$byte_mask, + Optional:$cache_policy, + I32Attr:$size + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM90BulkCopyS2SOp : CuteNVGPU_Op<"arch.copy.SM90.bulk_copy_s2s"> { + let description = [{ + The `arch.copy.SM90.bulk_copy_s2s` operation performs a bulk load from shared + cta to shared cluster. + + The `smem_data_addr` operand is a 16-byte aligned pointer to the source data + in shared memory. + + The `dsmem_data_addr` operand is a 16-byte aligned pointer to the destination data + in shared cluster memory. + + The `dsmem_bar_addr` operand is a 8-byte aligned pointer to the barrier in shared + memory. + + The `cta_rank` operand specifies the dst cta rank in cluster for BLKCP. + + The `size` attr specifies the bytes copied by BLKCP, aligned to 16 bytes. + + Examples: + ```mlir + %smem = ... : !cute.ptr> + %dsmem = ... : !cute.ptr> + %bar = ... : !cute.ptr> + %cta = ... : i32 + + %0 = cute_nvgpu.arch.copy.SM90.bulk_copy_s2s( + %smem : !cute.ptr>, + %dsmem : !cute.ptr>, + %bar : !cute.ptr>, + %cta : i32 + ) size = 128 + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase:$smem_data_addr, + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase:$dsmem_data_addr, + CuteNVGPU_CuTePointerToSharedMemoryWith8Byte:$dsmem_bar_addr, + I32:$cta_rank, + I32Attr:$size + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM100CopyS2TOp : CuteNVGPU_Op<"arch.copy.SM100.copy_s2t"> { + let description = [{ + See the file cutlass/include/cute/arch/copy_sm100.hpp in the CUTLASS + repository for more information. + }]; + let arguments = (ins + CuteNVGPU_AllowableDescriptorTypesForS2TSrcOperand:$src, + CuteNVGPU_CuTePointerToMemorySpacesTmem:$dst, + CuteNVGPU_ArchCopySM100CopyS2TDpAttr:$dp, + CuteNVGPU_ArchCopySM100CopyS2TBitAttr:$bits, + CuteNVGPU_ArchCopySM100CopyS2TCtaAttr:$cta, + CuteNVGPU_CopyS2TBroadcastAttr:$broadcast + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM100TmaLoadOp : CuteNVGPU_Op<"arch.copy.SM100.tma_load", [AttrSizedOperandSegments]> { + let description = [{ + The `arch.copy.SM100.tma_load` operation performs a tensor load from global + to shared memory. + + The attribute `mode` indicates which mode is selected. It can be: + + - tiled + - im2col for image-to-column loads + - w128 for activation-stationary convolution MMA mode (requires multicast) + - w for weight-stationary convolution MMA mode (requires multicast) + - gather4 for gather mode + + The attribute `num_cta` specifies the scope of the instruction. + + The `src_desc` operand is a 64-byte aligned pointer to the source tensor in + global memory. + + The `dsmem_data_addr` is a 128-byte aligned pointer to the destination data + address in shared memory. + + The `dsmem_bar_addr` is an 8-byte aligned pointer to the barrier in shared + or distributed shared memory. + + The `coord` operand holds the tensor coordinates (up to 5 dimensions). + + The `multicast_mask` operand is used when multicasting. + + The `offsets` operand holds offsets for im2col, w128, or w modes. + + The `cache_policy` operand is used to indicate the eviction priority of the L2 cache. + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte:$src_desc, + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase:$dsmem_data_addr, + CuteNVGPU_CuTePointerToSharedOrDistributedSharedMemory:$dsmem_bar_addr, + Variadic:$coord, + Optional:$multicast_mask, + Variadic:$offsets, + Optional:$cache_policy, + CuteNVGPU_TmaLoadModeAttr:$mode, + CuteNVGPU_ArchCopySM100TmaLoadOpNumCTAAttr:$num_cta + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM100TmaReduceOp : CuteNVGPU_Op<"arch.copy.SM100.tma_reduce", [AttrSizedOperandSegments]> { + let description = [{ + The `arch.copy.SM100.tma_reduce` operation performs a tensor store from shared memory + to global with reduction. + + The attribute `mode` indicates which mode is selected. It can be: + + - tiled + - im2col for image-to-column loads + + The attribute `Kind` indicates which kind of reduce op is selected. It can be: + + - ADD + - MIN + - MAX + - INC + - DEC + - AND + - OR + - XOR + + The `dst_desc` is a 64-byte aligned pointer to the destination data + address in global memory. + + The `src_smem_data_addr` operand is a 128-byte aligned pointer to the source address in + shared memory. + + The `coord` operand holds the tensor coordinates (up to 5 dimensions). + + The `cache_policy` operand is used to indicate the eviction priority of the L2 cache. + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte:$dst_desc, + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase2:$src_smem_data_addr, + Variadic:$coord, + Optional:$cache_policy, + CuteNVGPU_TmaStoreModeAttr:$mode, + CuteNVGPU_ReductionKindAttr:$kind + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM100TmaStoreOp : CuteNVGPU_Op<"arch.copy.SM100.tma_store", [AttrSizedOperandSegments]> { + let description = [{ + The `arch.copy.SM100.tma_store` operation performs a tensor store from shared memory + to global. + + The attribute `mode` indicates which mode is selected. It can be: + + - tiled + - im2col for image-to-column loads + - scatter4 for scatter mode + + The `dst_desc` is a 64-byte aligned pointer to the destination data + address in global memory. + + The `src_smem_data_addr` operand is a 128-byte aligned pointer to the source address in + shared memory. + + The `coord` operand holds the tensor coordinates (up to 5 dimensions). + + The `cache_policy` operand is used to indicate the eviction priority of the L2 cache. + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte:$dst_desc, + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase2:$src_smem_data_addr, + Variadic:$coord, + Optional:$cache_policy, + CuteNVGPU_TmaStoreModeAttr:$mode + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM100TmemLoadOp : CuteNVGPU_Op<"arch.copy.SM100.tmem_load", [MemoryEffects<[MemRead]>]> { + let description = [{ + See the file cutlass/include/cute/arch/copy_sm100.hpp in the CUTLASS + repository for more information. + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToMemorySpacesTmem:$src, + CuteNVGPU_ArchCopySM100TMEMNumDpAttr:$num_dp, + CuteNVGPU_ArchCopySM100TMEMNumBAttr:$num_b, + CuteNVGPU_ArchCopySM100TMEMNumRepAttr:$num_rep, + UnitAttr:$pack_16 + ); + let results = (outs + CuteNVGPU_VectorOf32BitSignlessIntegerValuesOf:$dst + ); +} +def CuteNVGPU_ArchCopySM100TmemStoreOp : CuteNVGPU_Op<"arch.copy.SM100.tmem_store"> { + let description = [{ + See the file cutlass/include/cute/arch/copy_sm100.hpp in the CUTLASS + repository for more information. + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToMemorySpacesTmem:$dst, + CuteNVGPU_VectorOf32BitSignlessIntegerValuesOf:$src, + CuteNVGPU_ArchCopySM100TMEMNumDpAttr:$num_dp, + CuteNVGPU_ArchCopySM100TMEMNumBAttr:$num_b, + CuteNVGPU_ArchCopySM100TMEMNumRepAttr:$num_rep, + UnitAttr:$expand_16 + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM107TmaLoadOverrideOp : CuteNVGPU_Op<"arch.copy.SM107.tma_load_override", [AttrSizedOperandSegments]> { + let description = [{ + The `arch.copy.SM107.tma_load_override` operation performs a TMA tensor load + from global to shared memory with runtime override of address, dimensions, + and strides. + + In addition to the normal TMA load operands, this op carries: + - `override_addr`: global pointer overriding the descriptor's base address + - `tensor_size`: per-mode sizes (encoded as size-1, i16) + - `lower_stride`: per-mode strides in 16B units (i32, modes 1..N-1) + - `upper_stride`: packed upper 4 bits of up to 4 strides (i16) + + Coordinates are passed as zeros (override form ignores them). + + Example: + + Suppose we want to load a 2D tile from a tensor with logical sizes + `{S0, S1}` and per-mode element strides `{1, ES1}` (mode 0 is contiguous, + so its stride is not passed). Let `EBITS` be the value type's bit-width + and `elemsPer16B = 128 / EBITS`. Mode 0's size is in `tensor_size[0]`, + mode 1's size is in `tensor_size[1]`, and mode 1's stride is split between + `lower_stride[0]` and the nibble at bits `[0:4]` of `upper_stride`. + + The encoding (matching `CopyAtomTmaLoadType::lowerOverrideToArch`) is: + + ```mlir + // ---- tensor_size: trunc(S_i - 1), one entry per mode ---- + %s0_m1 = arith.subi %S0_i32, %c1_i32 : i32 // S0 - 1 + %s1_m1 = arith.subi %S1_i32, %c1_i32 : i32 // S1 - 1 + %ts0 = arith.trunci %s0_m1 : i32 to i16 + %ts1 = arith.trunci %s1_m1 : i32 to i16 + + // ---- stride in 16B units: ES_i (element stride, i64) >> log2(elemsPer16B) + %sh = arith.constant log2(elemsPer16B) : i64 + %s1_16b = arith.shrui %ES1_i64, %sh : i64 // mode 1, stride in 16B + + // ---- lower_stride[idx]: low 32 bits of stride_in_16B (idx = mode - 1) ---- + %ls1 = arith.trunci %s1_16b : i64 to i32 // idx = 0 (mode 1) + + // ---- upper_stride: packed nibbles, OR-accumulated, truncated to i16 ---- + // nibble_i = (stride_in_16B[mode i] >> 32) & 0xF + // upper |= nibble_i << (4 * (i - 1)) + %c32 = arith.constant 32 : i64 + %c0xF = arith.constant 15 : i64 + %hi_1 = arith.shrui %s1_16b, %c32 : i64 // upper bits of mode 1 + %nib_1 = arith.andi %hi_1, %c0xF : i64 + %sh_1 = arith.constant 0 : i64 // 4 * (1 - 1) = 0 + %shf_1 = arith.shli %nib_1, %sh_1 : i64 + %acc = arith.ori %c0_i64, %shf_1 : i64 // accumulate over modes + %us = arith.trunci %acc : i64 to i16 + + // ---- the override TMA load itself ---- + cute_nvgpu.arch.copy.SM107.tma_load_override( + %tma_desc : !cute.ptr>, // src TMA descriptor + %dst_smem : !cute.ptr>, // dst data buffer + %dst_bar : !cute.ptr>, // mbarrier + %override_addr : !cute.ptr>, // runtime base addr override + [%c0_i32, %c0_i32] : i32, i32) // coords (ignored: zeros) + tensor_size = %ts0, %ts1 // (size - 1) per mode, i16 + lower_stride = %ls1 // strides for modes 1..N-1, in 16B units, low 32b (i32) + upper_stride = %us // packed upper 4 bits of up to 4 strides (i16) + cache_policy = %policy // L2 eviction policy (i64) + mode = num_cta = 1 + ``` + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte:$src_desc, + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase:$dsmem_data_addr, + CuteNVGPU_CuTePointerToSharedOrDistributedSharedMemory:$dsmem_bar_addr, + CuteNVGPU_CuTePointerType:$override_addr, + Variadic:$tensor_size, + Variadic:$lower_stride, + Optional:$upper_stride, + Variadic:$coord, + Optional:$multicast_mask, + Optional:$cache_policy, + CuteNVGPU_TmaLoadModeAttr:$mode, + CuteNVGPU_ArchCopySM100TmaLoadOpNumCTAAttr:$num_cta + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM107TmaStoreOverrideOp : CuteNVGPU_Op<"arch.copy.SM107.tma_store_override", [AttrSizedOperandSegments]> { + let description = [{ + The `arch.copy.SM107.tma_store_override` operation performs a TMA tensor store + from shared memory to global with runtime override of address, dimensions, + and strides. + + In addition to the normal TMA store operands, this op carries: + - `override_addr`: global pointer overriding the descriptor's base address + - `tensor_size`: per-mode sizes (encoded as size-1, i16) + - `lower_stride`: per-mode strides in 16B units (i32, modes 1..N-1) + - `upper_stride`: packed upper 4 bits of up to 4 strides (i16) + + Example + The example of tma_load_override applies here except that tma_store_override + doesn't have the mbar argument. + + Coordinates are passed as zeros (override form ignores them). + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte:$dst_desc, + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase2:$src_smem_data_addr, + CuteNVGPU_CuTePointerType:$override_addr, + Variadic:$tensor_size, + Variadic:$lower_stride, + Optional:$upper_stride, + Variadic:$coord, + Optional:$cache_policy, + CuteNVGPU_TmaStoreModeAttr:$mode + ); + let results = (outs + ); +} +def CuteNVGPU_ArchGetDynamicSmemOp : CuteNVGPU_Op<"arch.get_dyn_smem", [MemoryEffects<[MemAlloc]>]> { + let description = [{ + This operation acquires the pointer to the dynamic shared memory, whose size + is determined at the kernel launch time. + }]; + let arguments = (ins + ); + let results = (outs + CuteNVGPU_CuTePointerToMemorySpacesSmem:$ptr + ); +} +def CuteNVGPU_ArchGetDynamicSmemSizeOp : CuteNVGPU_Op<"arch.get_dyn_smem_size", [Pure]> { + let description = [{ + This operation gets the size in bytes of the dynamic shared memory that was + specified at kernel launch time. This can be used for bounds checking during + shared memory allocation. + }]; + let arguments = (ins + ); + let results = (outs + I32:$size + ); +} +def CuteNVGPU_ArchMakeWarpUniformOp : CuteNVGPU_Op<"arch.make_warp_uniform", [Pure]> { + let description = [{ + This operation is to reach a similar goal as canonical_warp_idx_sync() in cutlass C++, which is + typically used to hint to ptxas to use uniform register to store current value so as to avoid + redundant R2URs in mainloop. It's actually a shortcut usage of `nvvm.shfl.sync idx 31, val, 0, -1`. + + Example: + + ```mlir + %thread_id = nvvm.read.ptx.sreg.tid.x : i32 + %warp_size_log2 = arith.constant 5 : i32 + %warp_id = arith.shrsi %thread_id, %warp_size_log2 : i32 + %warp_id_uniform = cute_nvgpu.arch.make_warp_uniform %warp_id : i32 + ``` + And this IR will be lowered to: + ```mlir + %thread_id = nvvm.read.ptx.sreg.tid.x : i32 + %warp_size_log2 = arith.constant 5 : i32 + %warp_id = arith.shrsi %thread_id, %warp_size_log2 : i32 + %warp_full_mask = arith.constant -1 : i32 + %c0_i32 = arith.constant 0 : i32 + %warp_id_uniform = nvvm.shfl.sync idx %c0_i32, %warp_id, %c0_i32, %warp_full_mask : i32 -> i32 + ``` + }]; + let arguments = (ins + I32:$val + ); + let results = (outs + I32:$res + ); +} +def CuteNVGPU_ArchMmaSM107UMMABlockScaledOp : CuteNVGPU_Op<"arch.mma.SM107.umma_block_scaled"> { + let description = [{ + The `cute_nvgpu.arch.mma.SM107.umma_block_scaled` operation performs a matrix + multiply-accumulate operation. This arch op contains scaling factor for A/B. + Matrix elements for operand A can reside in tensor memory or shared memory. + Operand B always reside in shared memory. Elements for the accumulator input + and output (CD) always reside in tensor memory. The `accum` operand controls + whether the matrix multiplication result is added to data in `C` (`accum` = `true`) + or overwrites the data in `C` (`accum` = `false`). The `sf_a/b` operands indicate + the scaling factor of A/B and they always reside in tensor memory. The `vec_size` + attribute indicates the scaling vector size. + See the file cutlass/include/cute/arch/mma_sm100_umma.hpp in the CUTLASS + repository for more information. + Examples: + ```mlir + cute_nvgpu.arch.mma.SM107.umma_block_scaled(%A, %B, %C, %sf_a, %sf_b, %idesc, %accum) + { num_cta = 1, vec_size = 32, a_type = f8E5M2, shape_k = 32 } + : (!cute_nvgpu.smem_desc, !cute_nvgpu.smem_desc, !cute.ptr, !cute.ptr, + !cute.ptr, i32, i1) + ``` + }]; + let arguments = (ins + CuteNVGPU_AllowableDescriptorTypesForSM107BlockscaledUMMAA:$a, + CuteNVGPU_CuteSm107SmemDescType:$b, + CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA:$cd, + CuteNVGPU_CuTePointerToAllowableTypesForSM107UMMA:$sf_a, + AnyType:$sf_b, + AnyType:$idesc, + AnyType:$accum, + APIntAttr:$num_cta, + I32Attr:$vec_size, + TypeAttr:$a_type, + TypeAttr:$b_type, + I32Attr:$shape_k, + CuteNVGPU_MmaCollectorOpAttr:$a_collector_op, + CuteNVGPU_MmaCollectorOpAttr:$b_collector_op + ); + let results = (outs + ); +} +def CuteNVGPU_ArchMmaSM107UMMABlockScaledSparseOp : CuteNVGPU_Op<"arch.mma.SM107.umma_block_scaled_sparse"> { + let description = [{ + The `cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse` operation performs a matrix + multiply-accumulate operation. This arch op contains scaling factor for A/B and supports + structured sparsity. Matrix elements for operand A can reside in tensor memory or shared memory. + Operand B always reside in shared memory. Elements for the accumulator input + and output (CD) always reside in tensor memory. The `accum` operand controls + whether the matrix multiplication result is added to data in `C` (`accum` = `true`) + or overwrites the data in `C` (`accum` = `false`). The `sf_a/b` operands indicate + the scaling factor of A/B and they always reside in tensor memory. The `e_op` operand + provides the sparse metadata that describes the sparsity positional information of the matrix, + enabling efficient computation with structured sparse matrices. The `vec_size` + attribute indicates the scaling vector size. + See the file cutlass/include/cute/arch/mma_sm100_umma.hpp in the CUTLASS + repository for more information. + Examples: + ```mlir + cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse(%A, %B, %C, %sf_a, %sf_b, %E, %idesc, %accum) + { num_cta = 1, vec_size = 32, a_type = f8E5M2, shape_k = 32 } + : (!cute_nvgpu.sm107.smem_desc, !cute_nvgpu.sm107.smem_desc, !cute.ptr, !cute.ptr, + !cute.ptr, !cute.ptr>, i32, i1) + ``` + }]; + let arguments = (ins + CuteNVGPU_AllowableDescriptorTypesForSM107BlockscaledUMMAA:$a, + CuteNVGPU_CuteSm107SmemDescType:$b, + CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA:$cd, + CuteNVGPU_CuTePointerToAllowableTypesForSM107UMMA:$sf_a, + AnyType:$sf_b, + AnyType:$e_op, + AnyType:$idesc, + AnyType:$accum, + APIntAttr:$num_cta, + I32Attr:$vec_size, + TypeAttr:$a_type, + TypeAttr:$b_type, + I32Attr:$shape_k, + CuteNVGPU_MmaCollectorOpAttr:$a_collector_op, + CuteNVGPU_MmaCollectorOpAttr:$b_collector_op + ); + let results = (outs + ); +} +def CuteNVGPU_ArchMmaSM107UMMAOp : CuteNVGPU_Op<"arch.mma.SM107.umma"> { + let description = [{ + The `cute_nvgpu.arch.mma.SM107.umma` operation performs a matrix multiply- + accumulate operation. + Matrix elements for operand A can reside in either shared memory or tensor + memory (tmem). As such, the argument for operand A can be either a shared + memory descriptor or a pointer to tensor memory. + Elements for operand B always reside in shared memory. Elements for the + accumulator input and output (CD) always reside in tensor memory. The + `accum` operand controls whether the matrix multiplication result is added + to data in `C` (`accum` = `true`) or overwrites the data in `C` (`accum` = + `false`). Needs instruction desc according to nvvm backend and an additional + `a_type` for mmaKind inference. + See the file cutlass/include/cute/arch/mma_sm100_umma.hpp in the CUTLASS + repository for more information. + Examples: + ```mlir + cute_nvgpu.arch.mma.SM107.umma(%A, %B, %C, %idesc, %accum) + { num_cta = 1, ab_type = (f8E5M2, f8E5M2), shape_k = 64, ab_collector_op = (discard, discard) } + : (!cute_nvgpu.smem_desc, !cute_nvgpu.smem_desc, !cute.ptr, i32, i1) -> () + ``` + }]; + let arguments = (ins + CuteNVGPU_AllowableDescriptorTypesForSM107UMMAAOperand:$a, + CuteNVGPU_CuteSm107SmemDescType:$b, + CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA:$cd, + I32:$idesc, + I1:$accum, + CuteNVGPU_VectorOf32BitSignlessIntegerValuesOf2:$disable_output_lane, + APIntAttr:$num_cta, + APIntAttr:$scale, + TypeAttr:$a_type, + TypeAttr:$b_type, + I32Attr:$shape_k, + CuteNVGPU_MmaCollectorOpAttr:$a_collector_op, + CuteNVGPU_MmaCollectorOpAttr:$b_collector_op + ); + let results = (outs + ); +} +def CuteNVGPU_ArchMmaSM107UMMASparseOp : CuteNVGPU_Op<"arch.mma.SM107.umma_sparse"> { + let description = [{ + The `cute_nvgpu.arch.mma.SM107.umma_sparse` operation performs a matrix + multiply-accumulate operation where matrix A is structured sparse. + Matrix elements for operand A can reside in either shared memory or tensor + memory (tmem). As such, the argument for operand A can be either a shared + memory descriptor or a pointer to tensor memory. Operand E holds sparse + metadata that reside in tensor memory. + Elements for operand B always reside in shared memory. Elements for the + accumulator input and output (CD) always reside in tensor memory. The + `accum` operand controls whether the matrix multiplication result is added + to data in `C` (`accum` = `true`) or overwrites the data in `C` (`accum` = + `false`). Needs instruction desc according to nvvm backend and an additional + `a_type` for mmaKind inference. + See the file cutlass/include/cute/arch/mma_sm100_umma.hpp in the CUTLASS + repository for more information. + Examples: + ```mlir + cute_nvgpu.arch.mma.SM107.umma_sparse(%A, %B, %C, %E, %idesc, %accum) + {num_cta = 1 : i32, scale = 0 : i32, ab_type = (f16, f16), shape_k = 32 : i32, ab_collector_op = (discard, discard)} + : (!cute_nvgpu.sm107.smem_desc, !cute_nvgpu.sm107.smem_desc, !cute.ptr, + !cute.ptr, i32, i1) -> () + ``` + }]; + let arguments = (ins + CuteNVGPU_AllowableDescriptorTypesForSM107UMMAAOperand:$a, + CuteNVGPU_CuteSm107SmemDescType:$b, + CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA:$cd, + CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA2:$e, + AnyType:$idesc, + AnyType:$accum, + APIntAttr:$num_cta, + APIntAttr:$scale, + TypeAttr:$a_type, + TypeAttr:$b_type, + I32Attr:$shape_k, + CuteNVGPU_MmaCollectorOpAttr:$a_collector_op, + CuteNVGPU_MmaCollectorOpAttr:$b_collector_op + ); + let results = (outs + ); +} +def CuteNVGPU_ArchMmaSM120BlockScaledOp : CuteNVGPU_Op<"arch.mma.SM120.block_scaled", [AttrSizedOperandSegments]> { + let description = [{ + The `cute_nvgpu.arch.mma.SM120.block_scaled` operation performs a matrix + multiply-accumulate operation. This arch op contains scaling factor for A/B. + Matrix elements for A/B/C/D/SFA/SFB and accumulator are always in registers. + + C is always F32 under current SM120 arch, so there is no `c_type` here. + + See the file cutlass/include/cute/arch/mma_sm120.hpp in the CUTLASS repository + for more information. + + Examples: + ```mlir + cute_nvgpu.arch.mma.SM120.block_scaled + A[%a0, %a1, %a2, %a3] B[%b0, %b1] C[%c0, %c1, %c2, %c3] + sfA(%scaleAData) sfB(%scaleBData) + {shape_MNK=<"(16,8,32)">, vec_size = 32, a_type = f4E2M1FN, b_type = f4E2M1FN, sf_type = f8E8M0FNU} + : A[i32, i32, i32, i32] B[i32, i32] C[f32, f32, f32, f32] + sfA(i8) sfB(i8) -> (f32, f32, f32, f32) + ``` + }]; + let arguments = (ins + Variadic:$opA, + Variadic:$opB, + Variadic:$opC, + CuteNVGPU_AllowableRegisterTypesForSM120BlockScaledMMA3:$sfA, + CuteNVGPU_AllowableRegisterTypesForSM120BlockScaledMMA3:$sfB, + Optional:$byte_id_a, + Optional:$byte_id_b, + Cute_ShapeAttr:$shape_MNK, + I32Attr:$vec_size, + DefaultValuedAttr:$thread_id_a, + DefaultValuedAttr:$thread_id_b, + CuteNVGPU_SM120BlockScaledABTypeAttr:$a_type, + CuteNVGPU_SM120BlockScaledABTypeAttr:$b_type, + CuteNVGPU_SM120BlockScaledSFTypeAttr:$sf_type + ); + let results = (outs + Variadic:$res + ); +} +def CuteNVGPU_ArchPrefetchTmaDescOp : CuteNVGPU_Op<"arch.prefetch_tma_desc"> { + let description = [{ + The `arch.prefetch_tma_desc` operation does prefetch TMA descriptor using generic addressing + (i.e. no specific state space: const or param). + + Example: + + ```mlir + cute_nvgpu.arch.prefetch_tma_desc(%tma_a) : (!cute.ptr>) -> () + ``` + }]; + let arguments = (ins + CuteNVGPU_SM90PointerToTMATensorDescriptor:$tma_descriptor_ptr + ); + let results = (outs + ); +} +def CuteNVGPU_ArchSM100AllocTmemOp : CuteNVGPU_Op<"arch.sm100.alloc_tmem"> { + let summary = "The `arch.sm100.alloc_tmem` operation performs dynamic allocation of TMEM."; + let description = [{ + It dynamically allocates storage on TMEM with requested `n_cols` and stores + the allocated TMEM pointer into the shared memory buffer at the shared memory + location specified by `dst_ptr`. + + This operation is blocking, meaning it will wait until enough memory is available. + The action writting pointer value into shared memory is not synchronized. It's + user's responsibility to synchronize to make sure it's visible to all threads in the CTA. + + The allocation may fail if one of following conditions is true: + 1. The requested allocation size is not a multiple of 32 columns or greater + than 512 columns or not a power of 2, or + 2. If an attempt is made to allocate after executing the + `cute_nvgpu.arch.sm100.relinquish_tmem_alloc_permit` operation. + + This operation must be issued by a single fully active warp of the CTA and + must not be issued by more than one warp simultaneously. For repeated + allocations, the same warp must be used to issue all allocations. All + `alloc_tmem` operations must be issued by the same warp. + + For the allocation of 2 CTAs, one warp from each of the peer CTAs must + collectively perform the allocation, and the issuing warp must make sure that + peer CTA is launched and is still active. The participating warps from both + CTAs should have the same logical warp ID. There is no explicit need for + synchronization between these 2 CTAs before executing this operation. + + The behavior of this operation is undefined if all the threads in the warp do + not use the same values of `n_cols`, or if any thread in the warp has exited. + + The meaning of each argument is as follows: + - `n_cols`: The number of columns to be allocated. The unit of allocation is + 32 columns and the number of columns must be a power of 2. It must be within + the range [32, 512]. The number of columns allocated should not increase + between any two allocations in the execution order within the CTA. Failing to + meet these conditions may lead to undefined behavior. + - `dst_ptr`: Pointer to shared memory where the result TMEM pointer will be + written. It should have a natural alignment of 32-bits. + - `is_two_cta`: Specifies the number of CTAs involved in the allocation. + + Examples: + + ```mlir + cute_nvgpu.arch.sm100.alloc_tmem(%n_cols, %res_shm_ptr) [cta_1] + : i32, !cute.ptr> + + cute_nvgpu.arch.sm100.alloc_tmem(%n_cols, %res_shm_ptr) [cta_2] + : i32, !cute.ptr> + ``` + }]; + let arguments = (ins + I32:$n_cols, + CuteNVGPU_CuTePointerToMemorySpacesSmem:$dst_ptr, + UnitAttr:$is_two_cta, + UnitAttr:$exclusive + ); + let results = (outs + ); +} +def CuteNVGPU_ArchSM100DeallocTmemOp : CuteNVGPU_Op<"arch.sm100.dealloc_tmem"> { + let description = [{ + The `arch.sm100.dealloc_tmem` operation deallocates the Tensor Memory + specified by the TMEM address `tmem_ptr` and slice count. The address + `tmem_ptr` must point to a previously allocated TMEM. If the + deallocation fails, a runtime error occurs. + + This operation should only be performed by a single fully active warp within + the CTA. It should not be carried out by multiple warps simultaneously. The + warp ID that does alloc and dealloc can be different, but all `dealloc_tmem` + operations must be issued by the same warp. + + For the 2 CTAs deallocation, one warp from each of the peer CTAs must + collectively perform the de-allocation, and the issuing warp must make sure + that peer CTA is launched and is still active. The warps from both CTAs should + have the same logical warp ID. Prior explicit synchronization between the two + CTAs is not required before executing this operation. + + The behavior of this operation is undefined if all the threads in the warp do + not use the same values of `n_cols`, or if any thread in the warp has exited. + + The meaning of each argument is as follows: + - `tmem_ptr`: The base address of the TMEM buffer being deallocated. + - `n_cols`: The number of columns to be deallocated. The unit of deallocation + is 32 columns and the number of columns must be a power of 2. It must be within + the range [32, 512]. Failing to meet these conditions may lead to undefined + behavior. + - `is_two_cta`: Specifies the number of CTAs involved in the deallocation. + + Examples: + + ```mlir + cute_nvgpu.arch.sm100.dealloc_tmem(%tmem_ptr, %n_cols) [cta_1] + : !cute.ptr>, i32 + + cute_nvgpu.arch.sm100.dealloc_tmem(%tmem_ptr, %n_cols) [cta_2] + : !cute.ptr>, i32 + ``` + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToMemorySpacesTmem:$tmem_ptr, + I32:$n_cols, + UnitAttr:$is_two_cta, + UnitAttr:$exclusive + ); + let results = (outs + ); +} +def CuteNVGPU_ArchSM100RelinquishTmemAllocPermitOp : CuteNVGPU_Op<"arch.sm100.relinquish_tmem_alloc_permit"> { + let description = [{ + This `arch.sm100.relinquish_tmem_alloc_permit` specifies that the CTA of the + executing thread is relinquishing the right to allocate Tensor Memory so that + other CTAs can allocate. + + Once this operation is executed, any subsequent attempts by the same CTA to + allocate TMEM are considered illegal. + + Example: + + ```mlir + cute_nvgpu.arch.sm100.relinquish_tmem_alloc_permit [cta_1] + ``` + }]; + let arguments = (ins + UnitAttr:$is_two_cta + ); + let results = (outs + ); +} +def CuteNVGPU_ArchSM100RetrieveTmemPtrOp : CuteNVGPU_Op<"arch.sm100.retrieve_tmem_ptr", [MemoryEffects<[MemRead]>]> { + let description = [{ + The `arch.sm100.retrieve_tmem_ptr` operation is a convenient shorthand for + loading the allocated TMEM pointer from shared memory and casting it into a + TMEM pointer. It includes a correctness check for the TMEM pointer and + returns a null pointer if the TMEM pointer is invalid. + + Example: + + ```mlir + %tmem_ptr = cute_nvgpu.arch.sm100.retrieve_tmem_ptr(%res_shm_ptr) + : !cute.ptr> -> !cute.ptr> + ``` + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToMemorySpacesSmem:$smem_ptr + ); + let results = (outs + CuteNVGPU_CuTePointerToMemorySpacesTmem:$tmem_ptr + ); +} +def CuteNVGPU_AtomCopyMakeExecTmaOp : CuteNVGPU_InferredOp<"atom.make_exec_tma", [Pure]> { + let arguments = (ins + CuteNVGPU_TypeInterfaceForNonExecutableTMAAtomTypes:$input, + UnitAttr:$override, + UnitAttr:$no_fully_oob_tile + ); + let results = (outs + CuteNVGPU_CopyAtomTypeInterfaceInstance:$result + ); +} +def CuteNVGPU_AtomCopyMakeNonExec2DGather4TmaLoadOp : CuteNVGPU_InferredOp<"atom.make_non_exec_2d_gather4_tma_load", [Pure]> { + let summary = "This operation creates"; + let description = [{ + 1. a non-executable atom for TMA loads in tiled gather4 mode (.tile::gather4) + 2. the TMA tensor mapping the logical coordinates of gmem_tensor to TMA coordinate offsets + + The non-executable atom, which wraps a TMA descriptor and attaches type information to it for + safety, can then be used to create an executable one using + cute_nvgpu.atom.copy.make_exec_tma. It can also be used directly for TiledCopy-based + partitioning of source and destination tensors. + }]; + let arguments = (ins + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, + CuteNVGPU_CuTeLayoutType:$gmem_coord_layout, + CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_v_map, + CuteNVGPU_GatherScatterTmaLoadAttr:$kind, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA:$non_exec_atom, + CuteNVGPU_CuteCoordTensorType:$tma_tensor + ); +} +def CuteNVGPU_AtomCopyMakeNonExec2DScatter4TmaStoreOp : CuteNVGPU_InferredOp<"atom.make_non_exec_2d_scatter4_tma_store", [Pure]> { + let summary = "This operation creates"; + let description = [{ + 1. a non-executable atom for TMA stores in tiled scatter4 mode (.tile::scatter4) + 2. the TMA tensor mapping the logical coordinates of the scatter index layout to TMA coordinate offsets + + The non-executable atom, which wraps a TMA descriptor and attaches type information to it for + safety, can then be used to create an executable one using + cute_nvgpu.atom.copy.make_exec_tma. It can also be used directly for TiledCopy-based + partitioning of source and destination tensors. + }]; + let arguments = (ins + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, + CuteNVGPU_CuTeLayoutType:$gmem_index_layout, + CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_v_map, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA2:$non_exec_atom, + CuteNVGPU_CuteCoordTensorType:$tma_tensor + ); +} +def CuteNVGPU_AtomCopyMakeNonExecIm2ColTmaLoadOp : CuteNVGPU_InferredOp<"atom.make_non_exec_im2col_tma_load", [Pure]> { + let summary = "This operation creates"; + let description = [{ + 1. a non-executable atom for TMA loads in im2col mode + 2. the TMA tensor mapping the logical coordinates of gmem_tensor to TMA coordinate offsets + + The non-executable atom, which wraps a TMA descriptor and attaches type information to it for + safety, can then be used to create an executable one using + cute_nvgpu.atom.copy.make_exec_tma_load. It can also be used directly for TiledCopy-based + partitioning of source and destination tensors. + }]; + let arguments = (ins + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, + CuteNVGPU_CuTeComposedLayoutType:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_v_map, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_corner_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$upper_corner_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_padding_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$upper_padding_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$stride_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_srt, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$stride_srt, + CuteNVGPU_Im2ColTmaLoadAttr:$kind, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA3:$non_exec_atom, + CuteNVGPU_CuteCoordTensorType:$tma_tensor + ); +} +def CuteNVGPU_AtomCopyMakeNonExecIm2ColTmaStoreOp : CuteNVGPU_InferredOp<"atom.make_non_exec_im2col_tma_store", [Pure]> { + let summary = "This operation creates"; + let description = [{ + 1. a non-executable atom for TMA stores in im2col mode + 2. the TMA tensor mapping the logical coordinates of gmem_tensor to TMA coordinate offsets + + The non-executable atom, which wraps a TMA descriptor and attaches type information to it for + safety, can then be used to create an executable one using + cute_nvgpu.atom.copy.make_exec_tma_store. It can also be used directly for TiledCopy-based + partitioning of source and destination tensors. + }]; + let arguments = (ins + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, + CuteNVGPU_CuTeComposedLayoutType:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_v_map, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA4:$non_exec_atom, + CuteNVGPU_CuteCoordTensorType:$tma_tensor + ); +} +def CuteNVGPU_AtomCopyMakeNonExecTiledTmaLoadOp : CuteNVGPU_InferredOp<"atom.make_non_exec_tiled_tma_load", [Pure]> { + let summary = "This operation creates"; + let description = [{ + 1. a non-executable atom for TMA loads in tiled mode + 2. the TMA tensor mapping the logical coordinates of gmem_tensor to TMA coordinate offsets + + The non-executable atom, which wraps a TMA descriptor and attaches type information to it for + safety, can then be used to create an executable one using + cute_nvgpu.atom.copy.make_exec_tma. It can also be used directly for TiledCopy-based + partitioning of source and destination tensors. + }]; + let arguments = (ins + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, + CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_v_map, + CuteNVGPU_TiledTmaLoadAttr:$kind, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA5:$non_exec_atom, + CuteNVGPU_CuteCoordTensorType:$tma_tensor + ); +} +def CuteNVGPU_AtomCopyMakeNonExecTiledTmaReduceOp : CuteNVGPU_InferredOp<"atom.make_non_exec_tiled_tma_reduce", [Pure]> { + let summary = "This operation creates"; + let description = [{ + 1. a non-executable atom for TMA reduce in tiled mode + 2. the TMA tensor mapping the logical coordinates of gmem_tensor to TMA coordinate offsets + + The non-executable atom, which wraps a TMA descriptor and attaches type information to it for safety, can then be used to create an executable one using + cute_nvgpu.atom.copy.make_exec_tma. It can also be used directly for TiledCopy-based partitioning of source and destination tensors. + }]; + let arguments = (ins + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, + CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_v_map, + CuteNVGPU_ReductionKindAttr:$kind, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA6:$non_exec_atom, + CuteNVGPU_CuteCoordTensorType:$tma_tensor + ); +} +def CuteNVGPU_AtomCopyMakeNonExecTiledTmaStoreOp : CuteNVGPU_InferredOp<"atom.make_non_exec_tiled_tma_store", [Pure]> { + let summary = "This operation creates"; + let description = [{ + 1. a non-executable atom for TMA store in tiled mode + 2. the TMA tensor mapping the logical coordinates of gmem_tensor to TMA coordinate offsets + + The non-executable atom, which wraps a TMA descriptor and attaches type information to it for + safety, can then be used to create an executable one using + cute_nvgpu.atom.copy.make_exec_tma. It can also be used directly for TiledCopy-based + partitioning of source and destination tensors. + }]; + let arguments = (ins + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, + CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_v_map, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA7:$non_exec_atom, + CuteNVGPU_CuteCoordTensorType:$tma_tensor + ); +} +def CuteNVGPU_AtomCopyMakeS2TCopyOp : CuteNVGPU_InferredOp<"atom.make_s2t_copy", [Pure]> { + let description = [{ + This operation creates a new copy atom instance by tiling an utccp copy atom + instance with a tmem memref. + + Example: + ```mlir + %copy_s2t_atom = cute_nvgpu.atom.make_s2t_copy() : !cute_nvgpu.atom.s2t_copy + %tmem_memref = cute.memref.alloca() : !cute.memref, "(128,32):(65536,1)"> + %0 = cute_nvgpu.atom.make_s2t_copy(%copy_s2t_atom, %tmem_memref) + : (!cute_nvgpu.atom.s2t_copy, + !cute.memref, "(128,32):(65536,1)">) + -> !cute.tiled_copy, + layout_copy_tv = <"(1,(128,8)):(0,(1,128))">, + tiler_mn = <"[128:1;8:1]">> + ``` + }]; + let arguments = (ins + CuteNVGPU_SM100BlackwellSharedToTensorMemoryCopyAtom:$copy_s2t_atom, + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$tmem_memref + ); + let results = (outs + CuteNVGPU_TiledCopyAtomType:$result + ); +} +def CuteNVGPU_AtomCopyMakeTmemCopyOp : CuteNVGPU_InferredOp<"atom.make_tmem_copy", [Pure]> { + let description = [{ + This operation creates a new copy atom instance by tiling a load tmem copy atom + instance with a tmem memref. + + Example: + ```mlir + // %tmem_memref : !cute.memref, "(128,32):(65536,1)"> + %load_atom = cute.make_atom() : !cute_nvgpu.atom.tmem_load + %0 = cute_nvgpu.atom.make_tmem_copy(%load_atom, %tmem_memref) + : (!cute_nvgpu.atom.tmem_load, + !cute.memref, "(128,32):(65536,1)">) + -> !cute.tiled_copy, + layout_copy_tv = <"((32,4),(32,32)):((0,1),(128,4))">, + tiler_mn = <"[(4,32):(32,1);32:1]">> + ``` + }]; + let arguments = (ins + CuteNVGPU_CopyAtomTypeInterfaceInstance:$atom, + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$tmem_memref + ); + let results = (outs + CuteNVGPU_TiledCopyAtomType:$result + ); +} +def CuteNVGPU_AtomGetCoordTensorOp : CuteNVGPU_InferredOp<"atom.get_coord_tensor", [SameVariadicOperandSize, Pure]> { + let description = [{ + This operation creates a get tma coord atom instance from copy atom and shapes. + The atom will be called before algo copy(tma copy) to provide tma coord for algo copy. + In Cute C++, copy_atom cache stride info during make_tiled_copy and the stride + info is associated with shapes. shapes in argument will make a congruent comparison + with cache first and then construct tma coord with stride. + + Example: + ```mlir + %5 = cute_nvgpu.atom.get_coord_tensor(%4, %5) + : (!cute_nvgpu.atom.tma_load>, + !cute.shape<128, 64>) + -> !cute.coord_tensor<"(0,0)", "(512,64,2):(1@1,1@0,64@0)"> + ``` + + ```mlir + %6 = cute_nvgpu.atom.get_coord_tensor(%4, %5, %2, %3) + : (!cute_nvgpu.atom.tma_load>, + !cute.shape<"(500, 128)">, + !cute.int_tuple<"(0, 0)">, + !cute.layout<"(128, 256): (1@0, 1@1) + -> !cute.coord_tensor<"(0,0)", "(500,128):(1@1,1@0) o (0,0) o (128,256):(1@0,1@1)"> + ``` + }]; + let arguments = (ins + CuteNVGPU_AtomTypeInterfaceInstance:$atom, + CuteNVGPU_CuteShapeType:$shape, + Optional:$offset_b, + Optional:$layout_b + ); + let results = (outs + CuteNVGPU_CuteCoordTensorType:$result + ); +} +def CuteNVGPU_AtomGetCopyS2TSmemDescViewOp : CuteNVGPU_InferredOp<"atom.get_copy_s2t_smem_desc_view", [Pure]> { + let description = [{ + This operation creates a SmemDescView instance from the given UtccpAtomType and + the SmemView. The given SmemView must match with the Utccp's TiledCopy src partitioned layout. + The result will be used as the src of algo.copy. + + Example: + ```mlir + %res_0 = cute_nvgpu.atom.get_copy_s2t_smem_desc_view(%atom_0, %src_0) + : (!cute_nvgpu.atom.s2t_copy, + !cute.memref, S<2,4,3>, "(((128,4),1),2,(4,2)):(((16,1),0),2048,(4,4096))">) + -> !cute_nvgpu.smem_desc_view + ``` + }]; + let arguments = (ins + CuteNVGPU_SM100BlackwellSharedToTensorMemoryCopyAtom2:$atom, + CuteNVGPU_CuTeMemrefToMemorySpacesSmem:$view + ); + let results = (outs + CuteNVGPU_CuteSmemDescViewType:$result + ); +} +def CuteNVGPU_AtomGetCopyS2TSmemDescViewSM107Op : CuteNVGPU_InferredOp<"atom.sm107.get_copy_s2t_smem_desc_view", [Pure]> { + let description = [{ + This operation creates a SmemDescView instance from the given UtccpAtomType and + the SmemView. The given SmemView must match with the Utccp's TiledCopy src partitioned layout. + The result will be used as the src of algo.copy. + + Example: + ```mlir + %res_0 = cute_nvgpu.atom.sm107.get_copy_s2t_smem_desc_view(%atom_0, %src_0) + : (!cute_nvgpu.atom.s2t_copy, + !cute.memref, S<2,4,3>, "(((128,4),1),2,(4,2)):(((16,1),0),2048,(4,4096))">) + -> !cute_nvgpu.smem_desc_view + ``` + }]; + let arguments = (ins + CuteNVGPU_SM100BlackwellSharedToTensorMemoryCopyAtom2:$atom, + CuteNVGPU_CuTeMemrefToMemorySpacesSmem:$view + ); + let results = (outs + CuteNVGPU_CuteSmemDescViewType:$result + ); +} +def CuteNVGPU_AtomGetValueOp : CuteNVGPU_Op<"atom.get_value", [Pure]> { + let description = [{ + CuTe atoms represent both compile-time and runtime information for an + architecture-specific `copy` or `MMA` operation. For runtime information, + an atom can be thought of as similar to a C/C++ `struct` with predefined + member variables. Instead of defining operations to get and set the member + variables of each architecture atom, the CuTe dialect uses `get_value` and + `set_value` operations to uniformly handle retrieving and updating atom + member variables (hereafter referred to as "fields"). + The fields of each architecture-specific atom are identified by an enum + attribute. See the documentation for the operations that create an atom + or the attribute associated with the atom type for the expected enum values + and associated operand types. + The `field` attribute must correspond to the type of the `atom` operand. + For example, if the `atom` type is `!cute_nvgpu.sm100.mma`, the enum + attribute must be `#cute_nvgpu.atom_mma_field_sm100`. The verifier for this + operation will also compare the value `Type` to that expected by the + specific field. + + Example: + ```mlir + %5 = cute_nvgpu.atom.get_value(%4) : (!cute_nvgpu.sm100.mma<64x64x16, + num_cta = 1, + ab_major = (k,mn), + elem_type = (f16,f16,f16), + frag_kind = ss, + c_scale_exp = 0>) -> i1 + ``` + }]; + let arguments = (ins + CuteNVGPU_AtomTypeInterfaceInstance:$atom, + AnyAttr:$field + ); + let results = (outs + AnyType:$result + ); +} +def CuteNVGPU_AtomMakeTmaResidueTensorOp : CuteNVGPU_InferredOp<"atom.make_tma_residue_tensor", [Pure]> { + let description = [{ + Builds the per-TMA-mode "remaining elements" coord tensor that + `cute.copy` consumes on the override path. The op takes a non-executable + TMA atom (whose type carries `tma_gbasis`) and the corresponding gmem + tensor, and produces a coord tensor whose + + - iter is gmem tensor's shape combined and ordered according to tma gbasis. + - layout has the same shape as the tma tensor while the stride is negated. + + The residue tensor tracks the remaining shape to copy during TMA override, + e.g, slicing the residue tensor with a cta coordinate gets the remaining + shape for it given the negated stride. + + Example: + ```mlir + %residue = cute_nvgpu.atom.make_tma_residue_tensor(%non_exec_atom, %gmem) + : (!cute_nvgpu.atom.non_exec_tiled_tma_load>, + !cute.memref, "(?,?):(?,1)">) + -> !cute.coord_tensor<"(?,?)", "(?,?):(-1@1,-1@0)"> + ``` + }]; + let arguments = (ins + CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA8:$tma_atom, + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor + ); + let results = (outs + CuteNVGPU_CuteCoordTensorType:$residue + ); +} +def CuteNVGPU_AtomSetValueOp : CuteNVGPU_InferredOp<"atom.set_value", [Pure]> { + let description = [{ + CuTe atoms represent both compile-time and runtime information for an + architecture-specific `copy` or `MMA` operation. For runtime information, + an atom can be thought of as similar to a C/C++ `struct` with predefined + member variables. Instead of defining operations to get and set the member + variables of each architecture atom, the CuTe dialect uses `get_value` and + `set_value` operations to uniformly handle retrieving and updating atom + member variables (hereafter referred to as "fields"). + The fields of each architecture-specific atom are identified by an enum + attribute. See the documentation for the operations that create an atom + or the attribute associated with the atom type for the expected enum values + and associated operand types. + The `field` attribute must correspond to the type of the `atom` operand. + For example, if the `atom` type is `!cute_nvgpu.sm100.mma`, the enum + attribute must be `#cute_nvgpu.atom_mma_field_sm100`. The verifier for this + operation will also compare the value `Type` to that expected by the + specific field. + + Example: + ```mlir + %2 = cute_nvgpu.atom.set_value(%0, %no_accum) : + (!cute_nvgpu.sm100.mma<64x64x16, + num_cta = 1, + ab_major = (k,mn), + elem_type = (f16,f16,f16), + frag_kind = ss, + c_scale_exp = 0>, i1) + ``` + }]; + let arguments = (ins + CuteNVGPU_AtomTypeInterfaceInstance:$atom, + AnyType:$value, + AnyAttr:$field + ); + let results = (outs + CuteNVGPU_AtomTypeInterfaceInstance:$result + ); +} +def CuteNVGPU_AtomTmaPartitionOp : CuteNVGPU_InferredOp<"atom.tma_partition", [Pure]> { + let description = [{ + tma_partition is the new TMA routine for the dynamic and preferred cluster feature. + tma_partition op requires below arguments: + 1. tma_atom; + 2. cta_coord in the multicast dimension; + 3. cta_layout in the multicast dimension; + 4. smem_memref; + 5. variadic CuteViewTypeInterface tensors (one or more coord_tensor / memref), + covering the following cases: + 5.a: for regular TMA load/store, a single TMA tensor as input + 5.b: for tile::gather4 mode, a 2D TMA coord tensor plus the corresponding + gmem index tensor + and it produces: + 1. partitioned_smem_memref; + 2. one partitioned coord_tensor / memref per input, with the following cases: + 2.a: for regular TMA load/store, one partitioned TMA tensor + 2.b: for tile::gather4 mode, the partitioned TMA coord tensor plus the + partitioned gmem index tensor + + The return values are the src and dst arguments of algo.copy(atom, src, dst). + The second and the third input arguments can be runtime values, so the multicast + can be runtime determined. + + This operation is also compatible for the old static cluster TMA routines. + You only need to feed the cta_coord and cta_layout with static values, + then you can get compile-time multicast computation. + + The target_tensors operand is variadic and accepts any CuteViewTypeInterface + type (coord_tensor or memref). When a single tensor is provided, brackets + are omitted for backward compatibility. When multiple tensors are provided, + they are grouped in brackets. + + Example (single tensor): + ```mlir + %res_smem, %res_gmem = cute_nvgpu.atom.tma_partition( + %tma_atom, %cta_coord, %cta_layout, %stensor, %gtensor + ) : (!cute_nvgpu.atom.tma_load, + !cute.coord<"?">, + !cute.layout<"?:1">, + !cute.memref, S<3,4,3>, "(((128,16),1,4),8):(((64,1),0,16),8192)">, + !cute.coord_tensor<"(0,?,0)", "(((128,16),1,4),?,?,?):(((1@1,1@0),0,16@0),256@1,64@0,1@2)">) + ``` + + Example (multiple tensors): + ```mlir + %res_smem, %res_gmem0, %res_gmem1 = cute_nvgpu.atom.tma_partition( + %tma_atom, %cta_coord, %cta_layout, %stensor, [%gtensor0, %gtensor1] + ) : (!cute_nvgpu.atom.tma_load<...>, + !cute.coord<"?">, + !cute.layout<"?:1">, + !cute.memref, + [!cute.coord_tensor<...>, !cute.memref]) + -> (!cute.memref, + [!cute.coord_tensor<...>, !cute.memref]) + ``` + }]; + let arguments = (ins + CuteNVGPU_SM90TMATensorMemoryAcceleratorLoadAtomType:$tma_atom, + CuteNVGPU_CuteCoordType:$cta_coord, + CuteNVGPU_CuTeLayoutType:$cta_layout, + CuteNVGPU_CuTeMemrefToMemorySpacesSmem:$smem_tensor, + Variadic:$target_tensors + ); + let results = (outs + CuteNVGPU_CuTeMemrefToMemorySpacesSmem:$res_smem_tensor, + Variadic:$res_target_tensors + ); +} +def CuteNVGPU_CastTMADescToIntegerOp : CuteNVGPU_Op<"cast_tma_desc_to_integer", [Pure]> { + let summary = "cast_tma_desc_to_integer cast a tma_desc into an integer."; + let description = [{ + Example: + ```mlir + %23 = cute_nvgpu.cast_tma_desc_to_integer(%11 : !cute.ptr>) -> i64 + ``` + }]; + let arguments = (ins + CuteNVGPU_SM90PointerToTMATensorDescriptor:$src + ); + let results = (outs + CuteNVGPU_Integer:$dst + ); +} +def CuteNVGPU_CopyTMADescriptorOp : CuteNVGPU_Op<"copy_tma_desc"> { + let description = [{ + This operation copies the TMA descriptor from a non-executable TMA atom to the pointer. + The pointer could be a smem or gmem pointer and should be aligned to at least the size of TMA descriptor. + }]; + let arguments = (ins + CuteNVGPU_TypeInterfaceForNonExecutableTMAAtomTypes:$atom, + CuteNVGPU_CuTePointerToMemorySpacesSmemOrCuTe:$tma_desc_ptr + ); + let results = (outs + ); +} +def CuteNVGPU_GetGridConstantPointerOp : CuteNVGPU_InferredOp<"get_grid_constant_pointer", [Pure]> { + let description = [{ + 'cute_nvgpu.get_grid_constant_pointer' takes a grid constant value argument + from a kernel and returns a `cute.ptr` whose pointee type matches the + input. + + Before device ABI conversion, the input may be an intermediate result that + function finalization will promote to a grid-constant kernel argument. + Direct kernel arguments are verified immediately; intermediate provenance + is validated when the physical kernel signature is finalized. The result + type and alignment are fully inferred from the input type by + `InferTypeOpInterface`. + Example (TMA descriptor): + func.func @kernel( + %tma_desc_a : !cute_nvgpu.tma_descriptor_tiled {cute_nvgpu.grid_constant}) + attributes {cute.kernel} { + %tma_desc_a_ptr = cute_nvgpu.get_grid_constant_pointer (%tma_desc_a) : + !cute.ptr>; + ... + } + + Example (arbitrary value type): + func.func @kernel( + %params : !llvm.struct<(i64, i64)> {cute_nvgpu.grid_constant}) + attributes {cute.kernel} { + %params_ptr = cute_nvgpu.get_grid_constant_pointer (%params) : + !cute.ptr, generic>; + ... + } + }]; + let arguments = (ins + AnyType:$input + ); + let results = (outs + CuteNVGPU_CuTePointerType:$result + ); +} +def CuteNVGPU_GetTmaDescAddrOp : CuteNVGPU_Op<"get_tma_desc_addr", [Pure]> { + let description = [{ + The `cute_nvgpu.get_tma_desc_addr` operation accepts an input copy atom, + extracts the address within copy atom, then returns it as the address + of current copy atom's TMA desc in a single SSA value. The address within + input copy atom should have global/generic address space, and should be aligned to 64B. + Example: + + ```mlir + %1 = cute_nvgpu.get_tma_desc_addr(%cpy_atom : !cute_nvgpu.atom.tma_load) -> !cute.ptr> + ``` + }]; + let arguments = (ins + CuteNVGPU_CopyAtomTypeInterfaceInstance:$src + ); + let results = (outs + CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte:$ptr + ); +} +def CuteNVGPU_MakeGMMASmemDescOp : CuteNVGPU_Op<"make_gmma_smem_desc", [Pure]> { + let description = [{ + The `cute_nvgpu.make_gmma_smem_desc` operation accepts an input cute.ptr alongside a cute.layout and a major attr, + then constructs the shared memory descriptor and returns it. The layout must be describing + the buffer pointed to by the input pointer and must carry valid swizzle information. + For GMMA, there are 4 supported swizzle variants: + + ``` + swizzle | cooresponding enum in gmma smem desc + S<0, 4, 3> | SWIZZLE_NONE + S<1, 4, 3> | SWIZZLE_32B + S<2, 4, 3> | SWIZZLE_64B + S<3, 4, 3> | SWIZZLE_128B + ``` + The cute.ptr must carry shared address space and must be aligned to 16B. + + Example: + + ```mlir + %1 = cute_nvgpu.make_gmma_smem_desc(%arg1 : !cute.ptr, S<3,4,3>>) + layout = <"((64,16),1):((64,1),0)">, major = #cute_nvgpu.major + -> !cute_nvgpu.smem_desc + ``` + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToAllowableTypesForSM90MMA:$src, + Cute_LayoutAttr:$layout, + CuteNVGPU_MajorModeAttr:$major + ); + let results = (outs + CuteNVGPU_CuteSmemDescType:$res + ); +} +def CuteNVGPU_MakeTmaDescIm2ColAtOp : CuteNVGPU_Op<"make_tma_desc_im2col_at"> { + let description = [{ + In-place variant of Cute_MakeTmaDescIm2ColOp. The descriptor is stored + into the address passed as the last operand. + + Operands (identical semantics to the im2col op) + - All operands of MakeTmaDescIm2ColOp (see above). + - Value tma_desc_addr – destination address for the descriptor. + + Optional attributes (identical semantics to the im2col op) + - num_multicast, tma_format + + Results: none – descriptor is written to the supplied address. + + Example: + ```mlir + cute_nvgpu.make_tma_desc_im2col_at( + %tensor, + %slayout, + %cta_v_tile, + %lower_corner_whd, + %upper_corner_whd, + %lower_padding_whd, + %upper_padding_whd, + %stride_whd, + %lower_srt, + %stride_srt, + %tma_desc_addr) + <{num_multicast = 1 tma_format = #cute_nvgpu.tma_data_format}> : ( + !cute.memref, "((8,8,1,4),64,1):((64,512,4096,4096),1,0)">, + !cute.composed_layout<"S<2,4,3> o 0 o ((8,16),(32,1)):((32,256),(1,0))">, + !cute.layout<"(128,32):(1@0,1@1)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(1,1,1)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(1,1,1)">, + !cute.ptr>) + ``` + }]; + let arguments = (ins + CuteNVGPU_ViewTypeInterfaceInstance:$gmem_view, + CuteNVGPU_CuTeComposedLayoutType:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_value_tile, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_corner_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$upper_corner_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_padding_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$upper_padding_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$stride_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_srt, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$stride_srt, + CuteNVGPU_CuTePointerType:$tma_desc_addr, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format + ); + let results = (outs + ); +} +def CuteNVGPU_MakeTmaDescIm2ColOp : CuteNVGPU_Op<"make_tma_desc_im2col", [Pure]> { + let summary = "Create a TMA descriptor in im2col mode, suitable for convolutional forward passes."; + let description = [{ + Operands + - Value gmem_view, smem_layout, cta_value_tile – as in the tiled op. + - Value lower_corner_whd / upper_corner_whd – start & end (W,H,D) + positions of the + receptive-field window. + - Value lower_padding_whd / upper_padding_whd – padding applied + before/after each dim. + - Value stride_whd – traversal stride in WHD. + - Value lower_srt / stride_srt – split-K traversal + parameters (SRT). + + Optional attributes + - num_multicast : i32 + - tma_format : CuteTmaDataFormatAttr + + Result + - Cute pointer to the generated descriptor. + + Example: + ```mlir + %desc = cute_nvgpu.make_tma_desc_im2col( + %tensor, + %slayout, + %cta_v_tile, + %lower_corner_whd, + %upper_corner_whd, + %lower_padding_whd, + %upper_padding_whd, + %stride_whd, + %lower_srt, + %stride_srt) + <{num_multicast = 1 tma_format = #cute_nvgpu.tma_data_format}> : ( + !cute.memref, "((8,8,1,4),64,1):((64,512,4096,4096),1,0)">, + !cute.composed_layout<"S<2,4,3> o 0 o ((8,16),(32,1)):((32,256),(1,0))">, + !cute.layout<"(128,32):(1@0,1@1)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(1,1,1)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(1,1,1)">) -> !cute.ptr> + ``` + }]; + let arguments = (ins + CuteNVGPU_ViewTypeInterfaceInstance:$gmem_view, + CuteNVGPU_CuTeComposedLayoutType:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_value_tile, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_corner_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$upper_corner_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_padding_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$upper_padding_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$stride_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_srt, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$stride_srt, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_SM90PointerToTMATensorDescriptor:$result + ); +} +def CuteNVGPU_MakeTmaDescTiledAtOp : CuteNVGPU_Op<"make_tma_desc_tiled_at"> { + let summary = "In-place version of Cute_MakeTmaDescTiledOp."; + let description = [{ + Operands (identical semantics to the Tiled op) + - Value gmem_view – Global-memory tensor view. + - Value smem_layout – SMEM box layout. + - Value cta_value_tile – CTA-value-tile mapping. + - Value tma_desc_addr – Cute pointer that names the memory location + where the TMA descriptor should be written. + - Value traversal_stride – Cute int tuple that defines the traversal stride. + If omitted, the traversal stride is set to 1 for all dimensions. + + Optional attributes (identical semantics to the Tiled op) + - mode, num_multicast, tma_format + + Results: none – descriptor is written to the supplied address. + + Example: + ```mlir + cute_nvgpu.make_tma_desc_tiled_at(%a_tensor, %a_smem_clayout, %a_cta_v_tile, %tma_desc_addr) + <{mode = }> : (!cute.memref, "(?,?,?) : (1,?,?)">, + !cute.composed_layout<"S<3,4,3> o 0 o (((64,64),2)):(((1,64,4096))">, + !cute.layout<"(((64,64),2)):(((1@0,1@1),64@0))">, + !cute.ptr>) + + cute_nvgpu.make_tma_desc_tiled_at(%sfa_tensor, %sfa_smem_clayout, %sfa_cta_v_tile, %existing_tma_desc) + <{mode = num_multicast = 2 tma_format = #cute_nvgpu.tma_data_format}> : ( + !cute.memref, "((32,4,?),(32,4,?)):((16,4,?),(0,1,512))">, + !cute.composed_layout<"S<0,4,3> o 0 o ((((32,4),1),(32,1)),1,(4,1))">, + !cute.layout<"(((32,4),32),1,4):(((1@0@0,1@1@0),1@0@1),0,1@1@1)">, + !cute.ptr>) + + cute_nvgpu.make_tma_desc_tiled(%view, %slayout, %cta_v_tile, %tma_desc_addr %traversal_stride) + <{mode = }> : ( + !cute.memref, "((?,?,?,?),?):((?,?,?,?),?)">, + !cute.composed_layout<"S<2,4,3> o 0 o (128,32):(32,1)">, + !cute.layout<"((128),32):((1@0@0),1@1)">, + !cute.ptr>, + !cute.int_tuple<"(1,?,?,?)">) + ``` + }]; + let arguments = (ins + CuteNVGPU_ViewTypeInterfaceInstance:$gmem_view, + CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_value_tile, + CuteNVGPU_CuTePointerType:$tma_desc_addr, + Optional:$traversal_stride, + CuteNVGPU_TmaLoadModeAttr:$mode, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format + ); + let results = (outs + ); +} +def CuteNVGPU_MakeTmaDescTiledOp : CuteNVGPU_Op<"make_tma_desc_tiled", [Pure]> { + let summary = "Create a TMA descriptor in tiled load mode."; + let description = [{ + Operands + - Value gmem_view – Cute view that describes the tensor in global memory. + - Value smem_layout – Composed/regular Cute layout that defines the + rectangular box in shared memory (SMEM tile). + - Value cta_value_tile – Cute layout that maps CTA threads to elements + within the SMEM tile. + - Value traversal_stride – Cute int tuple that defines the traversal stride. + If omitted, the traversal stride is set to 1 for all dimensions. + + Optional attributes + - mode : CuteTmaLoadModeAttr or CuteTmaStoreModeAttr (defaults to #cute_nvgpu.tma_load_mode) + - num_multicast : i32 (defaults to 1; 1 ⇒ no multicast) + - tma_format : CuteTmaDataFormatAttr (element format written to + bits 74–71 of the HW TMA + descriptor; if omitted an + appropriate default is deduced) + + Result + - Cute pointer to the freshly created TMA-descriptor object. + + Example: + ```mlir + %tma_desc_a = cute_nvgpu.make_tma_desc_tiled(%a_tensor, %a_smem_clayout, %a_cta_v_tile) + <{mode = #cute_nvgpu.tma_load_mode}> : (!cute.memref, "(?,?,?) : (1,?,?)">, + !cute.composed_layout<"S<3,4,3> o 0 o (((64,64),2)):(((1,64),4096))">, + !cute.layout<"(((64,64),2)):(((1@0,1@1),64@0))">) -> !cute.ptr> + + %tma_desc_sfa = cute_nvgpu.make_tma_desc_tiled(%sfa_tensor, %sfa_smem_clayout, %sfa_cta_v_tile) + <{mode = #cute_nvgpu.tma_load_mode num_multicast = 2 tma_format = #cute_nvgpu.tma_data_format}> : ( + !cute.memref, "((32,4,?),(32,4,?)):((16,4,?),(0,1,512))">, + !cute.composed_layout<"S<0,4,3> o 0 o ((((32,4),1),(32,1)),1,(4,1)):((((16,4),512),(0,1)),0,(1,512))">, + !cute.layout<"(((32,4),32),1,4):(((1@0@0,1@1@0),1@0@1),0,1@1@1)">) -> !cute.ptr> + + %tma_desc_strided_c = cute_nvgpu.make_tma_desc_tiled(%view, %slayout, %cta_v_tile, %traversal_stride) + <{mode = #cute_nvgpu.tma_load_mode}> : ( + !cute.memref, "((?,?,?,?),?):((?,?,?,?),?)">, + !cute.composed_layout<"S<2,4,3> o 0 o (128,32):(32,1)">, + !cute.layout<"((128),32):((1@0@0),1@1)">, + !cute.int_tuple<"(1,?,?,?)">) -> !cute.ptr> + ``` + }]; + let arguments = (ins + CuteNVGPU_ViewTypeInterfaceInstance:$gmem_view, + CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_value_tile, + Optional:$traversal_stride, + CuteNVGPU_TmaLoadOrStoreModeAttr:$mode, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_SM90PointerToTMATensorDescriptor:$result + ); +} +def CuteNVGPU_MakeUMMASM107SmemDescOp : CuteNVGPU_Op<"sm107.make_umma_smem_desc", [Pure]> { + let summary = "Example:"; + let description = [{ + ```mlir + %1 = cute_nvgpu.sm107.make_umma_smem_desc(%arg1 : !cute.ptr) + layout = <"S<3, 4, 3> o 0 o ((64,16),1):((64,1),0)">, major = #cute_nvgpu.major + -> !cute_nvgpu.sm107.smem_desc + ``` + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToMemorySpacesSmem:$src, + Optional:$nextSrc, + Cute_LayoutAttr:$layout, + CuteNVGPU_MajorModeAttr:$major + ); + let results = (outs + CuteNVGPU_CuteSm107SmemDescType:$res + ); +} +def CuteNVGPU_MakeUMMASmemDescCircularSM103Op : CuteNVGPU_InferredOp<"sm103.make_umma_smem_desc_circular", [Pure]> { + let summary = "Example:"; + let description = [{ + ```mlir + %1 = cute_nvgpu.sm103.make_umma_smem_desc_circular(%arg1 : !cute.ptr) + layout = <"S<3, 4, 3> o 0 o ((64,16),1):((64,1),0)">, major = #cute_nvgpu.major + -> !cute_nvgpu.smem_desc_circular + ``` + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToMemorySpacesSmemWithByte:$src, + Optional:$nextSrc, + Cute_LayoutAttr:$layout, + CuteNVGPU_MajorModeAttr:$major + ); + let results = (outs + AnyType:$res + ); +} +def CuteNVGPU_MakeUMMASmemDescCircularSM107Op : CuteNVGPU_InferredOp<"sm107.make_umma_smem_desc_circular", [Pure]> { + let summary = "Example:"; + let description = [{ + ```mlir + %1 = cute_nvgpu.sm103.make_umma_smem_desc_circular(%arg1 : !cute.ptr) + layout = <"S<3, 4, 3> o 0 o ((64,16),1):((64,1),0)">, major = #cute_nvgpu.major + -> !cute_nvgpu.smem_desc_circular + ``` + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToMemorySpacesSmemWithByte:$src, + Optional:$nextSrc, + Cute_LayoutAttr:$layout, + CuteNVGPU_MajorModeAttr:$major + ); + let results = (outs + AnyType:$res + ); +} +def CuteNVGPU_MakeUMMASmemDescOp : CuteNVGPU_Op<"make_umma_smem_desc", [Pure]> { + let summary = "Example:"; + let description = [{ + ```mlir + %1 = cute_nvgpu.make_umma_smem_desc(%arg1 : !cute.ptr) + layout = <"S<3, 4, 3> o 0 o ((64,16),1):((64,1),0)">, major = #cute_nvgpu.major + -> !cute_nvgpu.smem_desc + ``` + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToMemorySpacesSmemWithByte:$src, + Optional:$nextSrc, + Cute_LayoutAttr:$layout, + CuteNVGPU_MajorModeAttr:$major + ); + let results = (outs + CuteNVGPU_CuteSmemDescType:$res + ); +} +def CuteNVGPU_PrefetchTmaDescriptorOp : CuteNVGPU_Op<"prefetch_tma_desc"> { + let arguments = (ins + CuteNVGPU_TypeInterfaceForNonExecutableTMAAtomTypes:$atom + ); + let results = (outs + ); +} +def CuteNVGPU_UpdateTMADescOp : CuteNVGPU_Op<"update_tma_desc"> { + let description = [{ + This operation updates the TMA descriptor in smem or gmem with the information from given gmem tensor. + Specifically, the following fields of the TMA descriptor will be updated: + 1. the global tensor base address will be updated with the base address of the global memory tensor. + 2. the tensor shapes will be updated with the shape of the global memory tensor. + 3. the tensor strides will be updated with the stride of the global memory tensor. + Other fields of the TMA descriptor will be left unchanged. + The TMA descriptor pointer specifies the address of the tma descriptor to be updated. + It could be a smem or gmem pointer. The pointer should be aligned to at least 64B. + Example: + ```mlir + cute_nvgpu.update_tma_desc(%tma_atom, %gmem_tensor, %smem_ptr) : ( + !cute_nvgpu.atom.non_exec_tiled_tma_load>, + !cute.memref, "(?,?,?) : (1,?,?)">, + !cute.ptr>) + -> () + + cute_nvgpu.update_tma_desc(%tma_atom, %gmem_tensor, %gmem_ptr) : ( + !cute_nvgpu.atom.non_exec_tiled_tma_load>, + !cute.memref, "(?,?,?) : (?,1,?)">, + !cute.ptr>) + -> () + ``` + }]; + let arguments = (ins + CuteNVGPU_TypeInterfaceForNonExecutableTMAAtomTypes:$tma_atom, + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, + CuteNVGPU_CuTePointerToMemorySpacesSmemOrCuTe:$tma_desc_ptr + ); + let results = (outs + ); } -def CuteNVGPU_ArchCopySM90BulkCopyG2SOp - : CuteNVGPU_SchemaOp<"arch.copy.SM90.bulk_copy_g2s", - [AttrSizedOperandSegments]> { - let arguments = (ins AnyType:$gmem_data_addr, AnyType:$dsmem_data_addr, - AnyType:$dsmem_bar_addr, - Optional:$multicast_mask, - Optional:$cache_policy, I32Attr:$size); - let results = (outs); -} - -def CuteNVGPU_ArchCopySM90BulkCopyS2GOp - : CuteNVGPU_SchemaOp<"arch.copy.SM90.bulk_copy_s2g", - [AttrSizedOperandSegments]> { - let arguments = (ins AnyType:$smem_data_addr, AnyType:$gmem_data_addr, - Optional:$byte_mask, - Optional:$cache_policy, I32Attr:$size); - let results = (outs); -} - -def CuteNVGPU_ArchCopySM90BulkCopyS2SOp - : CuteNVGPU_SchemaOp<"arch.copy.SM90.bulk_copy_s2s"> { - let arguments = (ins AnyType:$smem_data_addr, AnyType:$dsmem_data_addr, - AnyType:$dsmem_bar_addr, AnyInteger:$cta_rank, - I32Attr:$size); - let results = (outs); -} - -def CuteNVGPU_ArchCopySM100CopyS2TOp - : CuteNVGPU_SchemaOp<"arch.copy.SM100.copy_s2t"> { - let arguments = (ins AnyType:$src, AnyType:$dst, I32Attr:$dp, I32Attr:$bits, - I32Attr:$cta, AnyAttr:$broadcast); - let results = (outs); -} - -def CuteNVGPU_ArchCopySM100TmaLoadOp - : CuteNVGPU_SchemaOp<"arch.copy.SM100.tma_load", - [AttrSizedOperandSegments]> { - let arguments = (ins AnyType:$src_desc, AnyType:$dsmem_data_addr, - AnyType:$dsmem_bar_addr, Variadic:$coord, - Optional:$multicast_mask, - Variadic:$offsets, - Optional:$cache_policy, - AnyAttr:$mode, I32Attr:$num_cta); - let results = (outs); -} - -def CuteNVGPU_ArchCopySM100TmaReduceOp - : CuteNVGPU_SchemaOp<"arch.copy.SM100.tma_reduce", - [AttrSizedOperandSegments]> { - let arguments = (ins AnyType:$dst_desc, AnyType:$src_smem_data_addr, - Variadic:$coord, - Optional:$cache_policy, - AnyAttr:$mode, AnyAttr:$kind); - let results = (outs); -} - -def CuteNVGPU_ArchCopySM100TmaStoreOp - : CuteNVGPU_SchemaOp<"arch.copy.SM100.tma_store", - [AttrSizedOperandSegments]> { - let arguments = (ins AnyType:$dst_desc, AnyType:$src_smem_data_addr, - Variadic:$coord, - Optional:$cache_policy, AnyAttr:$mode); - let results = (outs); -} - -def CuteNVGPU_ArchCopySM100TmemLoadOp - : CuteNVGPU_SchemaOp<"arch.copy.SM100.tmem_load"> { - let arguments = (ins AnyType:$src, I32Attr:$num_dp, I32Attr:$num_b, - I32Attr:$num_rep, OptionalAttr:$pack_16); - let results = (outs AnyType:$dst); -} - -def CuteNVGPU_ArchCopySM100TmemStoreOp - : CuteNVGPU_SchemaOp<"arch.copy.SM100.tmem_store"> { - let arguments = (ins AnyType:$dst, AnyType:$src, I32Attr:$num_dp, - I32Attr:$num_b, I32Attr:$num_rep, - OptionalAttr:$expand_16); - let results = (outs); -} - -def CuteNVGPU_ArchGetDynSmemOp - : CuteNVGPU_SchemaOp<"arch.get_dyn_smem"> { - let arguments = (ins); - let results = (outs AnyType:$ptr); -} - -def CuteNVGPU_ArchGetDynSmemSizeOp - : CuteNVGPU_SchemaOp<"arch.get_dyn_smem_size"> { - let arguments = (ins); - let results = (outs I32:$size); -} - -def CuteNVGPU_ArchMakeWarpUniformOp - : CuteNVGPU_SchemaOp<"arch.make_warp_uniform"> { - let arguments = (ins AnyInteger:$val); - let results = (outs AnyInteger:$res); -} - -def CuteNVGPU_ArchMmaSM120BlockScaledOp - : CuteNVGPU_SchemaOp<"arch.mma.SM120.block_scaled", - [AttrSizedOperandSegments]> { - let arguments = (ins Variadic:$opA, Variadic:$opB, - Variadic:$opC, AnyType:$sfA, AnyType:$sfB, - Optional:$byte_id_a, - Optional:$byte_id_b, - AnyAttr:$shape_MNK, I32Attr:$vec_size, - OptionalAttr:$thread_id_a, - OptionalAttr:$thread_id_b, - TypeAttr:$a_type, TypeAttr:$b_type, TypeAttr:$sf_type); - let results = (outs Variadic:$res); -} - -def CuteNVGPU_ArchPrefetchTmaDescOp - : CuteNVGPU_SchemaOp<"arch.prefetch_tma_desc"> { - let arguments = (ins AnyType:$tma_descriptor_ptr); - let results = (outs); -} - -def CuteNVGPU_ArchSM100AllocTmemOp - : CuteNVGPU_SchemaOp<"arch.sm100.alloc_tmem"> { - let arguments = (ins AnyInteger:$n_cols, AnyType:$dst_ptr, - OptionalAttr:$is_two_cta, - OptionalAttr:$exclusive); - let results = (outs); -} - -def CuteNVGPU_ArchSM100DeallocTmemOp - : CuteNVGPU_SchemaOp<"arch.sm100.dealloc_tmem"> { - let arguments = (ins AnyType:$tmem_ptr, AnyInteger:$n_cols, - OptionalAttr:$is_two_cta, - OptionalAttr:$exclusive); - let results = (outs); -} - -def CuteNVGPU_ArchSM100RelinquishTmemAllocPermitOp - : CuteNVGPU_SchemaOp<"arch.sm100.relinquish_tmem_alloc_permit"> { - let arguments = (ins OptionalAttr:$is_two_cta); - let results = (outs); -} - -def CuteNVGPU_ArchSM100RetrieveTmemPtrOp - : CuteNVGPU_SchemaOp<"arch.sm100.retrieve_tmem_ptr"> { - let arguments = (ins AnyType:$smem_ptr); - let results = (outs AnyType:$tmem_ptr); -} - -def CuteNVGPU_AtomMakeExecTmaOp - : CuteNVGPU_SchemaOp<"atom.make_exec_tma"> { - let arguments = (ins AnyType:$input); - let results = (outs AnyType:$result); -} - -def CuteNVGPU_AtomMakeNonExec2DGather4TmaLoadOp - : CuteNVGPU_SchemaOp<"atom.make_non_exec_2d_gather4_tma_load"> { - let arguments = (ins AnyType:$gmem_tensor, AnyType:$gmem_coord_layout, - AnyType:$smem_layout, AnyType:$cta_v_map, - AnyAttr:$kind, OptionalAttr:$num_multicast, - OptionalAttr:$tma_format); - let results = (outs AnyType:$non_exec_atom, AnyType:$tma_tensor); -} - -def CuteNVGPU_AtomMakeNonExecIm2ColTmaLoadOp - : CuteNVGPU_SchemaOp<"atom.make_non_exec_im2col_tma_load"> { - let arguments = (ins AnyType:$gmem_tensor, AnyType:$smem_layout, - AnyType:$cta_v_map, AnyType:$lower_corner_whd, - AnyType:$upper_corner_whd, AnyType:$lower_padding_whd, - AnyType:$upper_padding_whd, AnyType:$stride_whd, - AnyType:$lower_srt, AnyType:$stride_srt, - AnyAttr:$kind, OptionalAttr:$num_multicast, - OptionalAttr:$tma_format); - let results = (outs AnyType:$non_exec_atom, AnyType:$tma_tensor); -} - -def CuteNVGPU_AtomMakeNonExecIm2ColTmaStoreOp - : CuteNVGPU_SchemaOp<"atom.make_non_exec_im2col_tma_store"> { - let arguments = (ins AnyType:$gmem_tensor, AnyType:$smem_layout, - AnyType:$cta_v_map, - OptionalAttr:$tma_format); - let results = (outs AnyType:$non_exec_atom, AnyType:$tma_tensor); -} - -def CuteNVGPU_AtomMakeNonExecTiledTmaLoadOp - : CuteNVGPU_SchemaOp<"atom.make_non_exec_tiled_tma_load"> { - let arguments = (ins AnyType:$gmem_tensor, AnyType:$smem_layout, - AnyType:$cta_v_map, AnyAttr:$kind, - OptionalAttr:$num_multicast, - OptionalAttr:$tma_format); - let results = (outs AnyType:$non_exec_atom, AnyType:$tma_tensor); -} - -def CuteNVGPU_AtomMakeNonExecTiledTmaReduceOp - : CuteNVGPU_SchemaOp<"atom.make_non_exec_tiled_tma_reduce"> { - let arguments = (ins AnyType:$gmem_tensor, AnyType:$smem_layout, - AnyType:$cta_v_map, AnyAttr:$kind, - OptionalAttr:$tma_format); - let results = (outs AnyType:$non_exec_atom, AnyType:$tma_tensor); -} - -def CuteNVGPU_AtomMakeNonExecTiledTmaStoreOp - : CuteNVGPU_SchemaOp<"atom.make_non_exec_tiled_tma_store"> { - let arguments = (ins AnyType:$gmem_tensor, AnyType:$smem_layout, - AnyType:$cta_v_map, - OptionalAttr:$tma_format); - let results = (outs AnyType:$non_exec_atom, AnyType:$tma_tensor); -} - -def CuteNVGPU_AtomMakeS2TCopyOp - : CuteNVGPU_SchemaOp<"atom.make_s2t_copy"> { - let arguments = (ins AnyType:$copy_s2t_atom, AnyType:$tmem_memref); - let results = (outs AnyType:$result); -} - -def CuteNVGPU_AtomMakeTmemCopyOp - : CuteNVGPU_SchemaOp<"atom.make_tmem_copy"> { - let arguments = (ins AnyType:$atom, AnyType:$tmem_memref); - let results = (outs AnyType:$result); -} - -def CuteNVGPU_AtomGetCoordTensorOp - : CuteNVGPU_SchemaOp<"atom.get_coord_tensor", - [SameVariadicOperandSize]> { - let arguments = (ins AnyType:$atom, AnyType:$shape, - Variadic:$offset_b, - Variadic:$layout_b); - let results = (outs AnyType:$result); -} - -def CuteNVGPU_AtomGetCopyS2TSmemDescViewOp - : CuteNVGPU_SchemaOp<"atom.get_copy_s2t_smem_desc_view"> { - let arguments = (ins AnyType:$atom, AnyType:$view); - let results = (outs AnyType:$result); -} - -def CuteNVGPU_AtomGetValueOp - : CuteNVGPU_SchemaOp<"atom.get_value"> { - let arguments = (ins AnyType:$atom, AnyAttr:$field); - let results = (outs AnyType:$result); -} - -def CuteNVGPU_AtomSetValueOp - : CuteNVGPU_SchemaOp<"atom.set_value"> { - let arguments = (ins AnyType:$atom, AnyType:$value, AnyAttr:$field); - let results = (outs AnyType:$result); -} - -def CuteNVGPU_AtomTmaPartitionOp - : CuteNVGPU_SchemaOp<"atom.tma_partition"> { - let arguments = (ins AnyType:$tma_atom, AnyType:$cta_coord, - AnyType:$cta_layout, AnyType:$smem_tensor, - Variadic:$target_tensors); - let results = (outs AnyType:$res_smem_tensor, - Variadic:$res_target_tensors); -} - -def CuteNVGPU_CastTmaDescToIntegerOp - : CuteNVGPU_SchemaOp<"cast_tma_desc_to_integer"> { - let arguments = (ins AnyType:$src); - let results = (outs AnyInteger:$dst); -} - -def CuteNVGPU_CopyTmaDescOp : CuteNVGPU_SchemaOp<"copy_tma_desc"> { - let arguments = (ins AnyType:$atom, AnyType:$tma_desc_ptr); - let results = (outs); -} - -def CuteNVGPU_GetGridConstantPointerOp - : CuteNVGPU_SchemaOp<"get_grid_constant_pointer"> { - let arguments = (ins AnyType:$input); - let results = (outs AnyType:$result); -} - -def CuteNVGPU_GetTmaDescAddrOp - : CuteNVGPU_SchemaOp<"get_tma_desc_addr"> { - let arguments = (ins AnyType:$src); - let results = (outs AnyType:$ptr); -} - -def CuteNVGPU_MakeGmmaSmemDescOp - : CuteNVGPU_SchemaOp<"make_gmma_smem_desc"> { - let arguments = (ins AnyType:$src, AnyAttr:$layout, AnyAttr:$major); - let results = (outs AnyType:$res); -} - -def CuteNVGPU_MakeTmaDescIm2ColAtOp - : CuteNVGPU_SchemaOp<"make_tma_desc_im2col_at"> { - let arguments = (ins AnyType:$gmem_view, AnyType:$smem_layout, - AnyType:$cta_value_tile, AnyType:$lower_corner_whd, - AnyType:$upper_corner_whd, AnyType:$lower_padding_whd, - AnyType:$upper_padding_whd, AnyType:$stride_whd, - AnyType:$lower_srt, AnyType:$stride_srt, - AnyType:$tma_desc_addr, - OptionalAttr:$num_multicast, - OptionalAttr:$tma_format); - let results = (outs); -} - -def CuteNVGPU_MakeTmaDescIm2ColOp - : CuteNVGPU_SchemaOp<"make_tma_desc_im2col"> { - let arguments = (ins AnyType:$gmem_view, AnyType:$smem_layout, - AnyType:$cta_value_tile, AnyType:$lower_corner_whd, - AnyType:$upper_corner_whd, AnyType:$lower_padding_whd, - AnyType:$upper_padding_whd, AnyType:$stride_whd, - AnyType:$lower_srt, AnyType:$stride_srt, - OptionalAttr:$num_multicast, - OptionalAttr:$tma_format); - let results = (outs AnyType:$result); -} - -def CuteNVGPU_MakeTmaDescTiledAtOp - : CuteNVGPU_SchemaOp<"make_tma_desc_tiled_at"> { - let arguments = (ins AnyType:$gmem_view, AnyType:$smem_layout, - AnyType:$cta_value_tile, AnyType:$tma_desc_addr, - Optional:$traversal_stride, AnyAttr:$mode, - OptionalAttr:$num_multicast, - OptionalAttr:$tma_format); - let results = (outs); -} - -def CuteNVGPU_MakeTmaDescTiledOp - : CuteNVGPU_SchemaOp<"make_tma_desc_tiled"> { - let arguments = (ins AnyType:$gmem_view, AnyType:$smem_layout, - AnyType:$cta_value_tile, - Optional:$traversal_stride, AnyAttr:$mode, - OptionalAttr:$num_multicast, - OptionalAttr:$tma_format); - let results = (outs AnyType:$result); -} - -def CuteNVGPU_SM103MakeUmmaSmemDescCircularOp - : CuteNVGPU_SchemaOp<"sm103.make_umma_smem_desc_circular"> { - let arguments = (ins AnyType:$src, Optional:$nextSrc, - AnyAttr:$layout, AnyAttr:$major); - let results = (outs AnyType:$res); -} - -def CuteNVGPU_MakeUmmaSmemDescOp - : CuteNVGPU_SchemaOp<"make_umma_smem_desc"> { - let arguments = (ins AnyType:$src, Optional:$nextSrc, - AnyAttr:$layout, AnyAttr:$major); - let results = (outs AnyType:$res); -} - -def CuteNVGPU_PrefetchTmaDescOp - : CuteNVGPU_SchemaOp<"prefetch_tma_desc"> { - let arguments = (ins AnyType:$atom); - let results = (outs); -} - -def CuteNVGPU_UpdateTmaDescOp - : CuteNVGPU_SchemaOp<"update_tma_desc"> { - let arguments = (ins AnyType:$tma_atom, AnyType:$gmem_tensor, - AnyType:$tma_desc_ptr); - let results = (outs); -} - -#endif // CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_OPS_TD +#endif // CUTE_IR_GENERATED_CUTENVGPU_OPS_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUTypes.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUTypes.td index ff56a095..00fa734e 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUTypes.td +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUTypes.td @@ -1,140 +1,750 @@ //===- CuteNVGPUTypes.td - CuTe NVIDIA GPU types -----------*- tablegen -*-===// +// +// Generated for nvidia-cutlass-dsl 4.8.0 from the DSL compiler +// (`_cutlass_ir.cu13` parser messages and verifier strings, its typed Python +// type classes, and probing its type parser and printer); do not edit by hand. +// +// Every cute_nvgpu type of the compiler is here, with its parameters in the +// compiler's order and C++ types. Parameters the compiler parses with custom +// directives (the `MxNxK` shape, `frag_kind`, `D DP`, `xR`, `t|n`, the sm90 +// negation flags) are named after the typed getters or the directive. +// +// Syntax is the compiler's, byte for byte, except where its printer emits text +// its own parser rejects; the parsers here also accept that text, and the +// printers here then emit the form the compiler parses back to the same type: +// - atom.universal_copy with copy_bits 0 and an address space prints `0 b`; +// - atom.simt_multimem_* print a default field when a later one is not; +// - sm100.mma_bs(_sp) print `arch_promote` when `int_overflow` is present. +// +// Verifiers carry the compiler's rules on parameter values where they are +// known, with its messages; instruction legality (the MMA shape and type +// tables, multimem PTX forms, ldmatrix/stmatrix configurations) is left to the +// compiler. +// +// Left out: the hidden `atom` attribute of the mma atoms' typed get()s. It +// overrides the atom's layouts, is never printed, and so cannot survive text; +// every mma atom parsed from text has it unset, which is what these types +// model. +// +//===----------------------------------------------------------------------===// #ifndef CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_TYPES_TD #define CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_TYPES_TD include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.td" +class CuteNVGPU_CustomType + : CuteNVGPU_Type { + let hasCustomAssemblyFormat = 1; +} + +defvar CuteNVGPU_Ty = "::mlir::Type"; +defvar CuteNVGPU_Shape = "::mlir::cutlass_compiler::cute::ShapeAttr"; +defvar CuteNVGPU_Layout = "::mlir::cutlass_compiler::cute::LayoutAttr"; +defvar CuteNVGPU_LayoutTy = "::mlir::cutlass_compiler::cute::LayoutType"; +defvar CuteNVGPU_StrideTy = "::mlir::cutlass_compiler::cute::StrideType"; +defvar CuteNVGPU_Major = "::mlir::cutlass_compiler::cute::MajorMode"; +defvar CuteNVGPU_Space = "::mlir::cutlass_compiler::cute::AddressSpace"; + +//===----------------------------------------------------------------------===// +// Descriptors +//===----------------------------------------------------------------------===// + def CuteNVGPU_SmemDescType : CuteNVGPU_Type<"SmemDesc", "smem_desc"> { - let summary = "Unparameterized shared-memory descriptor"; + let summary = "UMMA/GMMA shared-memory descriptor"; + let description = "`!cute_nvgpu.smem_desc`"; let assemblyFormat = ""; } +def CuteNVGPU_SmemDescSM107Type + : CuteNVGPU_Type<"SmemDescSM107", "sm107.smem_desc"> { + let summary = "SM107 shared-memory descriptor"; + let description = "`!cute_nvgpu.sm107.smem_desc`"; + let assemblyFormat = ""; +} + +class CuteNVGPU_SmemDescCircular + : CuteNVGPU_Type { + let summary = "Circular shared-memory descriptor over a block layout in bytes"; + let description = [{ + The block layout is static, has three modes, a 128-byte mode 1 and is + mode-1-major compact, with at least two blocks in mode 2: + + `!cute_nvgpu.}] # typeMnemonic # [{<<"(8,128,2):(128,1,1024)">>` + }]; + let parameters = (ins CuteNVGPU_Layout:$block_layout_bytes); + let assemblyFormat = "`<` $block_layout_bytes `>`"; + let genVerifyDecl = 1; +} + +def CuteNVGPU_SmemDescCircularSM103Type + : CuteNVGPU_SmemDescCircular<"SmemDescCircularSM103", + "sm103.smem_desc_circular">; +def CuteNVGPU_SmemDescCircularSM107Type + : CuteNVGPU_SmemDescCircular<"SmemDescCircularSM107", + "sm107.smem_desc_circular">; + +def CuteNVGPU_SmemDescViewType + : CuteNVGPU_CustomType<"SmemDescView", "smem_desc_view"> { + let summary = "Shared-memory descriptor iterator with a layout"; + let description = [{ + The descriptor is a smem_desc, sm107.smem_desc or smem_desc_circular type; + the layout is a plain layout, as a string. + + `!cute_nvgpu.smem_desc_view` + }]; + let parameters = (ins CuteNVGPU_Ty:$desc, CuteNVGPU_Layout:$layout); + let genVerifyDecl = 1; +} + def CuteNVGPU_TmaDescriptorTiledType : CuteNVGPU_Type<"TmaDescriptorTiled", "tma_descriptor_tiled"> { - let summary = "Unparameterized tiled TMA descriptor"; + let summary = "Tiled TMA descriptor"; + let description = "`!cute_nvgpu.tma_descriptor_tiled`"; let assemblyFormat = ""; } def CuteNVGPU_TmaDescriptorIm2ColType : CuteNVGPU_Type<"TmaDescriptorIm2Col", "tma_descriptor_im2col"> { - let summary = "Unparameterized im2col TMA descriptor"; + let summary = "Im2col TMA descriptor"; + let description = "`!cute_nvgpu.tma_descriptor_im2col`"; let assemblyFormat = ""; } def CuteNVGPU_WorkIdResponseType : CuteNVGPU_Type<"WorkIdResponse", "workid_response"> { - let summary = "Tensor-memory work identifier response"; + let summary = "Cluster launch control work-id response"; + let description = "`!cute_nvgpu.workid_response`"; let assemblyFormat = ""; } -class CuteNVGPU_PayloadType - : CuteNVGPU_Type { - let summary = "CuTe-DSL NVIDIA GPU type with lossless architecture payload"; +//===----------------------------------------------------------------------===// +// SIMT copy atoms +//===----------------------------------------------------------------------===// + +def CuteNVGPU_CopyAtomSIMTSyncCopyType + : CuteNVGPU_CustomType<"CopyAtomSIMTSyncCopy", "atom.universal_copy"> { + let summary = "Universal (SIMT) copy atom"; + let description = [{ + `copy_bits` 0 means the element width and is kept as 0; the address + spaces default to generic. Each field needs the previous ones. + + `!cute_nvgpu.atom.universal_copy` + }]; let parameters = (ins - "mlir::StringAttr":$payload + CuteNVGPU_Ty:$val_type, + DefaultValuedParameter<"int", "0">:$copy_bits, + DefaultValuedParameter:$src_space, + DefaultValuedParameter:$dst_space ); - let hasCustomAssemblyFormat = 1; + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomSIMTAsyncCopyType + : CuteNVGPU_CustomType<"CopyAtomSIMTAsyncCopy", "atom.simt_async_copy"> { + let summary = "cp.async copy atom"; + let description = [{ + `copy_bits` absent or 0 parses as the element width and is always printed. + + `!cute_nvgpu.atom.simt_async_copy` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "LoadCacheMode":$cache, + "int":$copy_bits + ); +} + +def CuteNVGPU_CopyAtomDsmemStoreType + : CuteNVGPU_CustomType<"CopyAtomDsmemStore", "atom.dsmem_store"> { + let summary = "Distributed shared-memory store atom"; + let description = [{ + `copy_bits` absent or 0 parses as the element width and is always printed. + + `!cute_nvgpu.atom.dsmem_store` + }]; + let parameters = (ins CuteNVGPU_Ty:$val_type, "int":$copy_bits); + let genVerifyDecl = 1; +} + +class CuteNVGPU_BulkCopy + : CuteNVGPU_CustomType { + let summary = "Bulk (TMA non-tensor) copy atom"; + let description = [{ + Space-separated groups, printed when not default (`copy_bits` 0). + + `!cute_nvgpu.}] # typeMnemonic # "<" # example # ">`"; + let parameters = !con( + (ins CuteNVGPU_Ty:$val_type, DefaultValuedParameter<"int", "0">:$copy_bits), + !if(!empty(flag), (ins), + !dag(ins, [DefaultValuedParameter<"bool", "false">], [flag]))); } def CuteNVGPU_CopyAtomBulkCopyG2SType - : CuteNVGPU_PayloadType<"CopyAtomBulkCopyG2S", "atom.bulk_copy_g2s">; + : CuteNVGPU_BulkCopy<"CopyAtomBulkCopyG2S", "atom.bulk_copy_g2s", "mcast", + "f32 copy_bits = 128 mcast = true">; def CuteNVGPU_CopyAtomBulkCopyS2GType - : CuteNVGPU_PayloadType<"CopyAtomBulkCopyS2G", "atom.bulk_copy_s2g">; + : CuteNVGPU_BulkCopy<"CopyAtomBulkCopyS2G", "atom.bulk_copy_s2g", "mask", + "f32 copy_bits = 128 mask = true">; def CuteNVGPU_CopyAtomBulkCopyS2SType - : CuteNVGPU_PayloadType<"CopyAtomBulkCopyS2S", "atom.bulk_copy_s2s">; -def CuteNVGPU_CopyAtomDsmemStoreType - : CuteNVGPU_PayloadType<"CopyAtomDsmemStore", "atom.dsmem_store">; + : CuteNVGPU_BulkCopy<"CopyAtomBulkCopyS2S", "atom.bulk_copy_s2s", "", + "f32 copy_bits = 128">; + +// g2r, r2g, r2s, s2r: ``, space separated, +// fields in parameter order and printed when not default; copy_bits absent or +// 0 parses as the element width and is always printed. + def CuteNVGPU_CopyAtomG2RType - : CuteNVGPU_PayloadType<"CopyAtomG2R", "atom.g2r">; -def CuteNVGPU_CopyAtomLdsmType - : CuteNVGPU_PayloadType<"CopyAtomLdsm", "atom.ldsm">; -def CuteNVGPU_CopyAtomNonExec2DGather4TmaLoadType - : CuteNVGPU_PayloadType<"CopyAtomNonExec2DGather4TmaLoad", - "atom.non_exec_2d_gather4_tma_load">; -def CuteNVGPU_CopyAtomNonExecIm2ColTmaLoadType - : CuteNVGPU_PayloadType<"CopyAtomNonExecIm2ColTmaLoad", - "atom.non_exec_im2col_tma_load">; -def CuteNVGPU_CopyAtomNonExecIm2ColTmaStoreType - : CuteNVGPU_PayloadType<"CopyAtomNonExecIm2ColTmaStore", - "atom.non_exec_im2col_tma_store">; -def CuteNVGPU_CopyAtomNonExecTiledTmaLoadType - : CuteNVGPU_PayloadType<"CopyAtomNonExecTiledTmaLoad", - "atom.non_exec_tiled_tma_load">; -def CuteNVGPU_CopyAtomNonExecTiledTmaReduceType - : CuteNVGPU_PayloadType<"CopyAtomNonExecTiledTmaReduce", - "atom.non_exec_tiled_tma_reduce">; -def CuteNVGPU_CopyAtomNonExecTiledTmaStoreType - : CuteNVGPU_PayloadType<"CopyAtomNonExecTiledTmaStore", - "atom.non_exec_tiled_tma_store">; + : CuteNVGPU_CustomType<"CopyAtomG2R", "atom.g2r"> { + let summary = "Global-to-register load atom"; + let description = [{ + `!cute_nvgpu.atom.g2r`, + `!cute_nvgpu.atom.g2r` (invariant only with + the default memory features) + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$copy_bits, + DefaultValuedParameter<"MemOrderKind", "MemOrderKind::WEAK">:$mem_order, + DefaultValuedParameter<"MemScopeKind", "MemScopeKind::CTA">:$mem_scope, + DefaultValuedParameter<"L2PrefetchSize", "L2PrefetchSize::NONE">:$l2_prefetch_size, + DefaultValuedParameter<"CacheEvictionPriority", "CacheEvictionPriority::EVICT_NORMAL">:$l1_cache_evict_priority, + DefaultValuedParameter<"LoadCacheMode", "LoadCacheMode::always">:$load_cache_mode, + DefaultValuedParameter<"SharedSpace", "SharedSpace::CTA">:$shared_space, + DefaultValuedParameter<"bool", "false">:$invariant + ); + let genVerifyDecl = 1; +} + def CuteNVGPU_CopyAtomR2GType - : CuteNVGPU_PayloadType<"CopyAtomR2G", "atom.r2g">; + : CuteNVGPU_CustomType<"CopyAtomR2G", "atom.r2g"> { + let summary = "Register-to-global store atom"; + let description = [{ + `!cute_nvgpu.atom.r2g` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$copy_bits, + DefaultValuedParameter<"MemOrderKind", "MemOrderKind::WEAK">:$mem_order, + DefaultValuedParameter<"MemScopeKind", "MemScopeKind::CTA">:$mem_scope, + DefaultValuedParameter<"CacheEvictionPriority", "CacheEvictionPriority::EVICT_NORMAL">:$l1_cache_evict_priority, + DefaultValuedParameter<"StoreCacheMode", "StoreCacheMode::write_back">:$store_cache_mode, + DefaultValuedParameter<"SharedSpace", "SharedSpace::CTA">:$shared_space + ); + let genVerifyDecl = 1; +} + +class CuteNVGPU_SharedCopy + : CuteNVGPU_CustomType { + let summary = what # " atom"; + let description = [{ + `!cute_nvgpu.}] # typeMnemonic # [{` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$copy_bits, + DefaultValuedParameter<"MemOrderKind", "MemOrderKind::WEAK">:$mem_order, + DefaultValuedParameter<"MemScopeKind", "MemScopeKind::CTA">:$mem_scope, + DefaultValuedParameter<"SharedSpace", "SharedSpace::CTA">:$shared_space + ); + let genVerifyDecl = 1; +} + def CuteNVGPU_CopyAtomR2SType - : CuteNVGPU_PayloadType<"CopyAtomR2S", "atom.r2s">; + : CuteNVGPU_SharedCopy<"CopyAtomR2S", "atom.r2s", "Register-to-shared store">; def CuteNVGPU_CopyAtomS2RType - : CuteNVGPU_PayloadType<"CopyAtomS2R", "atom.s2r">; -def CuteNVGPU_CopyAtomSIMTAsyncCopyType - : CuteNVGPU_PayloadType<"CopyAtomSIMTAsyncCopy", "atom.simt_async_copy">; -def CuteNVGPU_CopyAtomSIMTSyncCopyType - : CuteNVGPU_PayloadType<"CopyAtomSIMTSyncCopy", "atom.universal_copy">; -def CuteNVGPU_CopyAtomSM100CopyS2TType - : CuteNVGPU_PayloadType<"CopyAtomSM100CopyS2T", "atom.s2t_copy">; -def CuteNVGPU_CopyAtomSM100TmemLoadType - : CuteNVGPU_PayloadType<"CopyAtomSM100TmemLoad", "atom.tmem_load">; -def CuteNVGPU_CopyAtomSM100TmemStoreType - : CuteNVGPU_PayloadType<"CopyAtomSM100TmemStore", "atom.tmem_store">; -def CuteNVGPU_CopyAtomSM10xTmemLoadRedType - : CuteNVGPU_PayloadType<"CopyAtomSM10xTmemLoadRed", "atom.tmem_load_red">; -def CuteNVGPU_CopyAtomStsmType - : CuteNVGPU_PayloadType<"CopyAtomStsm", "atom.stsm">; -def CuteNVGPU_MmaAtomSM100UMMABlockScaledType - : CuteNVGPU_PayloadType<"MmaAtomSM100UMMABlockScaled", "sm100.mma_bs">; -def CuteNVGPU_MmaAtomSM100UMMAType - : CuteNVGPU_PayloadType<"MmaAtomSM100UMMA", "sm100.mma">; -def CuteNVGPU_MmaAtomSM120BlockScaledType - : CuteNVGPU_PayloadType<"MmaAtomSM120BlockScaled", "SM120.mma_bs">; -def CuteNVGPU_MmaAtomSM80Type - : CuteNVGPU_PayloadType<"MmaAtomSM80", "sm80.mma">; -def CuteNVGPU_MmaAtomSM89Type - : CuteNVGPU_PayloadType<"MmaAtomSM89", "sm89.mma">; -def CuteNVGPU_MmaAtomSM90Type - : CuteNVGPU_PayloadType<"MmaAtomSM90", "sm90.mma">; -def CuteNVGPU_SmemDescViewType - : CuteNVGPU_PayloadType<"SmemDescView", "smem_desc_view">; -def CuteNVGPU_TiledCopyType - : CuteNVGPU_PayloadType<"TiledCopy", "tiled_copy">; -def CuteNVGPU_TiledMmaType - : CuteNVGPU_PayloadType<"TiledMma", "tiled_mma">; -def CuteNVGPU_UniversalFmaAtomType - : CuteNVGPU_PayloadType<"UniversalFmaAtom", "atom.universal_fma">; -def CuteNVGPU_CopyAtomIm2ColTmaLoadType - : CuteNVGPU_PayloadType<"CopyAtomIm2ColTmaLoad", "atom.im2col_tma_load">; -def CuteNVGPU_CopyAtomIm2ColTmaStoreType - : CuteNVGPU_PayloadType<"CopyAtomIm2ColTmaStore", "atom.im2col_tma_store">; -def CuteNVGPU_CopyAtomSIMTMultimemLdReduceType - : CuteNVGPU_PayloadType<"CopyAtomSIMTMultimemLdReduce", - "atom.simt_multimem_ld_reduce">; -def CuteNVGPU_CopyAtomSIMTMultimemRedType - : CuteNVGPU_PayloadType<"CopyAtomSIMTMultimemRed", "atom.simt_multimem_red">; + : CuteNVGPU_SharedCopy<"CopyAtomS2R", "atom.s2r", "Shared-to-register load">; + +// Multimem atoms: positional `, ORDER, SCOPE>`; a trailing +// run of defaults is omitted (copy_bits 0, ADD, no accumulation precision, +// relaxed, sys). + def CuteNVGPU_CopyAtomSIMTMultimemStType - : CuteNVGPU_PayloadType<"CopyAtomSIMTMultimemSt", "atom.simt_multimem_st">; + : CuteNVGPU_CustomType<"CopyAtomSIMTMultimemSt", "atom.simt_multimem_st"> { + let summary = "multimem.st atom"; + let description = [{ + `copy_bits` 0 (the element width) is kept as 0. + + `!cute_nvgpu.atom.simt_multimem_st` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + DefaultValuedParameter<"int", "0">:$copy_bits, + DefaultValuedParameter<"MemOrderKind", "MemOrderKind::RELAXED">:$mem_order, + DefaultValuedParameter<"MemScopeKind", "MemScopeKind::SYS">:$mem_scope + ); +} + +def CuteNVGPU_CopyAtomSIMTMultimemRedType + : CuteNVGPU_CustomType<"CopyAtomSIMTMultimemRed", "atom.simt_multimem_red"> { + let summary = "multimem.red atom"; + let description = [{ + `copy_bits` absent or 0 parses as the element width and is always printed. + + `!cute_nvgpu.atom.simt_multimem_red` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$copy_bits, + DefaultValuedParameter<"ReductionKind", "ReductionKind::ADD">:$reduction, + DefaultValuedParameter<"MemOrderKind", "MemOrderKind::RELAXED">:$mem_order, + DefaultValuedParameter<"MemScopeKind", "MemScopeKind::SYS">:$mem_scope + ); +} + +def CuteNVGPU_CopyAtomSIMTMultimemLdReduceType + : CuteNVGPU_CustomType<"CopyAtomSIMTMultimemLdReduce", + "atom.simt_multimem_ld_reduce"> { + let summary = "multimem.ld_reduce atom"; + let description = [{ + `copy_bits` 0 (the element width) is kept as 0. The accumulation + precision must be given to give a memory order or scope. + + `!cute_nvgpu.atom.simt_multimem_ld_reduce, acquire, gpu>` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + DefaultValuedParameter<"int", "0">:$copy_bits, + DefaultValuedParameter<"ReductionKind", "ReductionKind::ADD">:$reduction, + OptionalParameter<"LdReduceAccPrecisionKindAttr">:$ld_reduce_acc_precision, + DefaultValuedParameter<"MemOrderKind", "MemOrderKind::RELAXED">:$mem_order, + DefaultValuedParameter<"MemScopeKind", "MemScopeKind::SYS">:$mem_scope + ); +} + +//===----------------------------------------------------------------------===// +// Tensor-memory copy atoms: `` +//===----------------------------------------------------------------------===// + +class CuteNVGPU_TmemCopy + : CuteNVGPU_CustomType { + let summary = what; + let description = [{ + `!cute_nvgpu.}] # typeMnemonic # "`"; + let parameters = !con( + (ins CuteNVGPU_Ty:$val_type, "int":$num_dp, "int":$num_bit, + "unsigned":$num_rep), + !dag(ins, [OptionalParameter<"::mlir::UnitAttr">], [flag])); + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomSM100TmemLoadType + : CuteNVGPU_TmemCopy<"CopyAtomSM100TmemLoad", "atom.tmem_load", + "tcgen05.ld atom", "pack16b">; +def CuteNVGPU_CopyAtomSM100TmemStoreType + : CuteNVGPU_TmemCopy<"CopyAtomSM100TmemStore", "atom.tmem_store", + "tcgen05.st atom", "expand16b">; + +def CuteNVGPU_CopyAtomSM10xTmemLoadRedType + : CuteNVGPU_CustomType<"CopyAtomSM10xTmemLoadRed", "atom.tmem_load_red"> { + let summary = "tcgen05.ld.red atom"; + let description = [{ + `half_split_off` prints as an integer attribute without spaces around `=`. + + `!cute_nvgpu.atom.tmem_load_red` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$num_dp, + "int":$num_bit, + "unsigned":$num_rep, + "TmemLoadRedOp":$red_op, + OptionalParameter<"::mlir::UnitAttr">:$nan, + OptionalParameter<"::mlir::IntegerAttr">:$half_split_off + ); + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomSM107TmemLoadSPCompressType + : CuteNVGPU_CustomType<"CopyAtomSM107TmemLoadSPCompress", + "atom.tmem_load_spcompress"> { + let summary = "SM107 tensor-memory load with sparse compression"; + let description = [{ + `!cute_nvgpu.atom.tmem_load_spcompress` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$num_dp, + "int":$num_bit, + "unsigned":$num_rep, + "TmemLoadRedOp":$red_op, + OptionalParameter<"::mlir::UnitAttr">:$red, + OptionalParameter<"::mlir::UnitAttr">:$nan + ); + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomSM100CopyS2TType + : CuteNVGPU_CustomType<"CopyAtomSM100CopyS2T", "atom.s2t_copy"> { + let summary = "tcgen05.cp (shared to tensor memory) atom"; + let description = [{ + The broadcast mode `none` is omitted. + + `!cute_nvgpu.atom.s2t_copy` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$num_dp, + "int":$num_bit, + "int":$num_cta, + DefaultValuedParameter<"CopyS2TBroadcast", "CopyS2TBroadcast::none">:$broadcast + ); + let genVerifyDecl = 1; +} + def CuteNVGPU_CopyAtomSM100S2TCopyV2Type - : CuteNVGPU_PayloadType<"CopyAtomSM100S2TCopyV2", "atom.sm100_s2t_copy_v2">; + : CuteNVGPU_CustomType<"CopyAtomSM100S2TCopyV2", "atom.sm100_s2t_copy_v2"> { + let summary = "tcgen05.cp atom with the shared-memory major mode"; + let description = [{ + The broadcast mode `none` is omitted. + + `!cute_nvgpu.atom.sm100_s2t_copy_v2` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$num_dp, + "int":$num_bit, + "int":$num_cta, + CuteNVGPU_Major:$smem_major, + DefaultValuedParameter<"CopyS2TBroadcast", "CopyS2TBroadcast::none">:$broadcast + ); + let genVerifyDecl = 1; +} + +//===----------------------------------------------------------------------===// +// ldmatrix / stmatrix +//===----------------------------------------------------------------------===// + +def CuteNVGPU_CopyAtomLdsmType + : CuteNVGPU_CustomType<"CopyAtomLdsm", "atom.ldsm"> { + let summary = "ldmatrix atom"; + let description = [{ + The last field is `t` (transposed) or `n`. + + `!cute_nvgpu.atom.ldsm, sz_pattern = u16, num_matrices = 4, t>` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + CuteNVGPU_Shape:$mode, + "LdsmSzPattern":$sz_pattern, + "int":$num_matrices, + OptionalParameter<"::mlir::UnitAttr">:$transpose + ); +} + +def CuteNVGPU_CopyAtomStsmType + : CuteNVGPU_CustomType<"CopyAtomStsm", "atom.stsm"> { + let summary = "stmatrix atom"; + let description = [{ + The last field is `t` (transposed) or `n`. + + `!cute_nvgpu.atom.stsm, num_matrices = 4, n>` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + CuteNVGPU_Shape:$mode, + "int":$num_matrices, + OptionalParameter<"::mlir::UnitAttr">:$transpose + ); +} + +//===----------------------------------------------------------------------===// +// TMA atoms +//===----------------------------------------------------------------------===// + +// Executable: comma-separated fields up to g_stride, then space-separated +// groups printed when set (booleans when true, sparsity when not 1). + def CuteNVGPU_CopyAtomTmaLoadType - : CuteNVGPU_PayloadType<"CopyAtomTmaLoad", "atom.tma_load">; -def CuteNVGPU_CopyAtomTmaReduceType - : CuteNVGPU_PayloadType<"CopyAtomTmaReduce", "atom.tma_reduce">; + : CuteNVGPU_CustomType<"CopyAtomTmaLoad", "atom.tma_load"> { + let summary = "Executable TMA load atom"; + let description = [{ + `!cute_nvgpu.atom.tma_load mcast = true tma_gbasis = <"(64,1):(1@0,1@1)">>` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + DefaultValuedParameter<"int", "1">:$sparsity, + "int":$copy_bits, + "TmaLoadMode":$mode, + "int":$num_cta, + CuteNVGPU_StrideTy:$g_stride, + DefaultValuedParameter<"bool", "false">:$mcast, + OptionalParameter:$tma_gbasis, + DefaultValuedParameter<"bool", "false">:$override, + DefaultValuedParameter<"bool", "false">:$no_fully_oob_tile + ); + let genVerifyDecl = 1; +} + def CuteNVGPU_CopyAtomTmaStoreType - : CuteNVGPU_PayloadType<"CopyAtomTmaStore", "atom.tma_store">; -def CuteNVGPU_MmaAtomSM100UMMABlockScaledSparseType - : CuteNVGPU_PayloadType<"MmaAtomSM100UMMABlockScaledSparse", - "sm100.mma_bs_sp">; -def CuteNVGPU_MmaAtomSM100UMMASparseType - : CuteNVGPU_PayloadType<"MmaAtomSM100UMMASparse", "sm100.mma_sp">; + : CuteNVGPU_CustomType<"CopyAtomTmaStore", "atom.tma_store"> { + let summary = "Executable TMA store atom"; + let description = [{ + `!cute_nvgpu.atom.tma_store tma_gbasis = <"(64,16):(1@0,1@1)">>` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + DefaultValuedParameter<"int", "1">:$sparsity, + "int":$copy_bits, + "TmaStoreMode":$mode, + CuteNVGPU_StrideTy:$g_stride, + OptionalParameter:$tma_gbasis, + DefaultValuedParameter<"bool", "false">:$override, + DefaultValuedParameter<"bool", "false">:$no_fully_oob_tile + ); + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomTmaReduceType + : CuteNVGPU_CustomType<"CopyAtomTmaReduce", "atom.tma_reduce"> { + let summary = "Executable TMA reduce atom"; + let description = [{ + `!cute_nvgpu.atom.tma_reduce tma_gbasis = <"(32,128):(1@1,1@0)">>` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$copy_bits, + "TmaStoreMode":$mode, + "ReductionKind":$kind, + CuteNVGPU_StrideTy:$g_stride, + OptionalParameter:$tma_gbasis + ); +} + +def CuteNVGPU_CopyAtomIm2ColTmaLoadType + : CuteNVGPU_CustomType<"CopyAtomIm2ColTmaLoad", "atom.im2col_tma_load"> { + let summary = "Executable im2col TMA load atom"; + let description = [{ + `!cute_nvgpu.atom.im2col_tma_load mcast = true tma_gbasis = <"(64,128):(1@1,1@0)">>` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$copy_bits, + "int":$num_cta, + CuteNVGPU_StrideTy:$g_stride, + DefaultValuedParameter<"bool", "false">:$mcast, + OptionalParameter:$tma_gbasis + ); +} + +def CuteNVGPU_CopyAtomIm2ColTmaStoreType + : CuteNVGPU_CustomType<"CopyAtomIm2ColTmaStore", "atom.im2col_tma_store"> { + let summary = "Executable im2col TMA store atom"; + let description = [{ + `!cute_nvgpu.atom.im2col_tma_store tma_gbasis = <"(32,128):(1@1,1@0)">>` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$copy_bits, + CuteNVGPU_StrideTy:$g_stride, + OptionalParameter:$tma_gbasis + ); +} + +// Non-executable: `<[kind, ]T, copy_bits = N, tma_gbasis = <"L">[, tma_format = F]>`. +// An omitted format is the element type's (F16_RN, BF16_RN, F32_RN, TF32_RN, +// F64_RN, U4 for 4-bit, U8 up to 8 bits, U16, U32/S32, U64/S64); it is always +// printed, and empty when the element type has none. + +class CuteNVGPU_NonExecTma + : CuteNVGPU_CustomType { + let summary = "Non-executable " # what # " atom"; + let description = "`!cute_nvgpu." # typeMnemonic # "<" # example # ">`"; + defvar common = (ins + CuteNVGPU_Ty:$val_type, + "int":$copy_bits, + CuteNVGPU_LayoutTy:$tma_gbasis, + "std::optional":$tma_format + ); + let parameters = !if(!empty(kindType), common, + !con((ins kindType:$kind), common)); + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomNonExecTiledTmaLoadType + : CuteNVGPU_NonExecTma<"CopyAtomNonExecTiledTmaLoad", + "atom.non_exec_tiled_tma_load", "tiled TMA load", "TiledTmaLoad", + "sm_90, bf16, copy_bits = 1024, tma_gbasis = <\"(64,1):(1@0,1@1)\">, tma_format = BF16_RN">; +def CuteNVGPU_CopyAtomNonExecTiledTmaStoreType + : CuteNVGPU_NonExecTma<"CopyAtomNonExecTiledTmaStore", + "atom.non_exec_tiled_tma_store", "tiled TMA store", "", + "bf16, copy_bits = 1024, tma_gbasis = <\"(64,1):(1@0,1@1)\">, tma_format = BF16_RN">; +def CuteNVGPU_CopyAtomNonExecTiledTmaReduceType + : CuteNVGPU_NonExecTma<"CopyAtomNonExecTiledTmaReduce", + "atom.non_exec_tiled_tma_reduce", "tiled TMA reduce", "ReductionKind", + "ADD, f32, copy_bits = 1024, tma_gbasis = <\"(64,1):(1@0,1@1)\">, tma_format = F32_RN">; +def CuteNVGPU_CopyAtomNonExecIm2ColTmaLoadType + : CuteNVGPU_NonExecTma<"CopyAtomNonExecIm2ColTmaLoad", + "atom.non_exec_im2col_tma_load", "im2col TMA load", "Im2ColTmaLoad", + "sm_90, f16, copy_bits = 1024, tma_gbasis = <\"(64,1):(1@0,1@1)\">, tma_format = F16_RN">; +def CuteNVGPU_CopyAtomNonExecIm2ColTmaStoreType + : CuteNVGPU_NonExecTma<"CopyAtomNonExecIm2ColTmaStore", + "atom.non_exec_im2col_tma_store", "im2col TMA store", "", + "f16, copy_bits = 1024, tma_gbasis = <\"(64,1):(1@0,1@1)\">, tma_format = F16_RN">; +def CuteNVGPU_CopyAtomNonExec2DGather4TmaLoadType + : CuteNVGPU_NonExecTma<"CopyAtomNonExec2DGather4TmaLoad", + "atom.non_exec_2d_gather4_tma_load", "2-D gather4 TMA load", + "GatherScatterTmaLoad", + "sm_100, f16, copy_bits = 1024, tma_gbasis = <\"(64,1):(1@0,1@1)\">, tma_format = F16_RN">; +def CuteNVGPU_CopyAtomNonExec2DScatter4TmaStoreType + : CuteNVGPU_NonExecTma<"CopyAtomNonExec2DScatter4TmaStore", + "atom.non_exec_2d_scatter4_tma_store", "2-D scatter4 TMA store", "", + "f16, copy_bits = 1024, tma_gbasis = <\"(4,64):(1@0,1@1)\">, tma_format = F16_RN">; + +//===----------------------------------------------------------------------===// +// MMA atoms. The shape is `MxNxK` (a static rank-3 shape); `elem_type` lists +// A, B and C; `frag_kind` is the A fragment kind: `ss` (smem_desc), `rs` +// (rmem, sm90) or `ts` (tmem, sm100 and sm107). +//===----------------------------------------------------------------------===// + +class CuteNVGPU_MmaAtom + : CuteNVGPU_CustomType { + let summary = what # " MMA atom"; + let description = "`!cute_nvgpu." # typeMnemonic # "<" # example # ">`"; + let genVerifyDecl = 1; +} + +def CuteNVGPU_UniversalFmaAtomType + : CuteNVGPU_MmaAtom<"UniversalFmaAtom", "atom.universal_fma", "Universal FMA", + "1x1x1, (f32, f32) -> f32 "> { + let parameters = (ins CuteNVGPU_Shape:$shape_mnk, CuteNVGPU_Ty:$a_type, + CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type); +} + +def CuteNVGPU_MmaAtomSM80Type + : CuteNVGPU_MmaAtom<"MmaAtomSM80", "sm80.mma", "SM80 mma.sync", + "16x8x32, elem_type = (i8, i8, i32), int_overflow = "> { + let description = [{ + `binary_op` needs `int_overflow` before it. + + `!cute_nvgpu.sm80.mma<16x8x32, elem_type = (i8, i8, i32), int_overflow = , binary_op = >` + }]; + let parameters = (ins + CuteNVGPU_Shape:$shape_mnk, + CuteNVGPU_Ty:$a_type, CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type, + OptionalParameter<"MMAIntOverflowAttr">:$int_overflow, + OptionalParameter<"BinaryOpAttr">:$binary_op + ); +} + def CuteNVGPU_MmaAtomSM80SparseType - : CuteNVGPU_PayloadType<"MmaAtomSM80Sparse", "sm80.sparse_mma">; -def CuteNVGPU_SmemDescCircularSM103Type - : CuteNVGPU_PayloadType<"SmemDescCircularSM103", - "sm103.smem_desc_circular">; + : CuteNVGPU_MmaAtom<"MmaAtomSM80Sparse", "sm80.sparse_mma", + "SM80 sparse mma.sync", + "16x8x64, elem_type = (i8, i8, i32), md_format = tid, int_overflow = "> { + let parameters = (ins + CuteNVGPU_Shape:$shape_mnk, + CuteNVGPU_Ty:$a_type, CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type, + "SparseMetadataFormat":$sparse_metadata_format, + OptionalParameter<"MMAIntOverflowAttr">:$int_overflow + ); +} + +def CuteNVGPU_MmaAtomSM89Type + : CuteNVGPU_MmaAtom<"MmaAtomSM89", "sm89.mma", "SM89 fp8 mma.sync", + "16x8x32, elem_type = (f8E4M3FN, f8E4M3FN, f32) "> { + let parameters = (ins CuteNVGPU_Shape:$shape_mnk, CuteNVGPU_Ty:$a_type, + CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type); +} + +def CuteNVGPU_MmaAtomSM90Type + : CuteNVGPU_MmaAtom<"MmaAtomSM90", "sm90.mma", "SM90 wgmma", + "64x128x16, ab_major = (k, mn), elem_type = (f16, f16, f32), frag_kind = rs, a_neg, b_neg"> { + let description = [{ + The trailing `int_overflow`, `a_neg` and `b_neg` parse in any order and + print in this one. + + `!cute_nvgpu.sm90.mma<64x128x32, ab_major = (k, k), elem_type = (i8, i8, i32), frag_kind = ss, int_overflow = , a_neg, b_neg>` + }]; + let parameters = (ins + CuteNVGPU_Shape:$shape_mnk, + CuteNVGPU_Major:$a_major, CuteNVGPU_Major:$b_major, + CuteNVGPU_Ty:$a_type, CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type, + "MmaFragKind":$a_frag_kind, + OptionalParameter<"MMAIntOverflowAttr">:$int_overflow, + OptionalParameter<"::mlir::UnitAttr">:$a_neg, + OptionalParameter<"::mlir::UnitAttr">:$b_neg + ); +} + +def CuteNVGPU_MmaAtomSM120BlockScaledType + : CuteNVGPU_MmaAtom<"MmaAtomSM120BlockScaled", "SM120.mma_bs", + "SM120 block-scaled mma.sync", + "16x8x64, vec_size = 32, elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E8M0FNU, use_sf_layout_TV = true"> { + let parameters = (ins + CuteNVGPU_Shape:$shape_mnk, + "int":$vec_size, + CuteNVGPU_Ty:$a_type, CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type, + CuteNVGPU_Ty:$sf_type, + "bool":$use_sf_layout_TV + ); +} + +// UMMA (tcgen05.mma): `]>`. arch_promote (sm100 +// block-scaled only) defaults to sm_100 and is omitted then; the sm107 atoms +// require ab_collector_op. + +class CuteNVGPU_Umma + : CuteNVGPU_MmaAtom { + let parameters = !con( + (ins CuteNVGPU_Shape:$shape_mnk, "int":$num_cta, + CuteNVGPU_Major:$a_major, CuteNVGPU_Major:$b_major, + CuteNVGPU_Ty:$a_type, CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type), + !if(sf, (ins CuteNVGPU_Ty:$sf_type), (ins)), + !if(!and(sp, !not(sf)), (ins CuteNVGPU_Ty:$e_type), (ins)), + !if(sp, (ins "SparseMetadataFormat":$sparse_metadata_format), (ins)), + (ins "MmaFragKind":$a_frag_kind), + !if(bs, (ins "int":$vec_size), (ins "int":$c_scale_exp)), + !if(!and(bs, !not(sm107)), + (ins DefaultValuedParameter<"Arch", "Arch::sm_100">:$arch_promote), (ins)), + !if(sm107, (ins "MmaCollectorOp":$a_collector_op, + "MmaCollectorOp":$b_collector_op), (ins)), + (ins OptionalParameter<"MMAIntOverflowAttr">:$int_overflow)); +} + +def CuteNVGPU_MmaAtomSM100UMMAType + : CuteNVGPU_Umma<"MmaAtomSM100UMMA", "sm100.mma", "SM100 UMMA", + "128x128x16, num_cta = 1, ab_major = (k, mn), elem_type = (f16, f16, f32), frag_kind = ss, c_scale_exp = 0", + 0, 0, 0, 0>; +def CuteNVGPU_MmaAtomSM100UMMASparseType + : CuteNVGPU_Umma<"MmaAtomSM100UMMASparse", "sm100.mma_sp", "SM100 sparse UMMA", + "128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f16, f16, f32), e_type = i8, sparse_metadata_format = tid, frag_kind = ss, c_scale_exp = 0", + 0, 1, 0, 0>; +def CuteNVGPU_MmaAtomSM100UMMABlockScaledType + : CuteNVGPU_Umma<"MmaAtomSM100UMMABlockScaled", "sm100.mma_bs", + "SM100 block-scaled UMMA", + "128x128x96, num_cta = 1, ab_major = (k, k), elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E8M0FNU, frag_kind = ss, vec_size = 16, arch_promote = sm_103", + 1, 0, 1, 0>; +def CuteNVGPU_MmaAtomSM100UMMABlockScaledSparseType + : CuteNVGPU_Umma<"MmaAtomSM100UMMABlockScaledSparse", "sm100.mma_bs_sp", + "SM100 block-scaled sparse UMMA", + "128x128x64, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, sparse_metadata_format = tid, frag_kind = ss, vec_size = 32", + 1, 1, 1, 0>; +def CuteNVGPU_MmaAtomSM107UMMAType + : CuteNVGPU_Umma<"MmaAtomSM107UMMA", "sm107.mma", "SM107 UMMA", + "128x128x16, num_cta = 1, ab_major = (k, mn), elem_type = (f16, f16, f32), frag_kind = ss, c_scale_exp = 0, ab_collector_op = (lastuse, fill)", + 0, 0, 0, 1>; +def CuteNVGPU_MmaAtomSM107UMMASparseType + : CuteNVGPU_Umma<"MmaAtomSM107UMMASparse", "sm107.mma_sp", "SM107 sparse UMMA", + "128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f16, f16, f32), e_type = i8, sparse_metadata_format = tid, frag_kind = ss, c_scale_exp = 0, ab_collector_op = (discard, discard)", + 0, 1, 0, 1>; +def CuteNVGPU_MmaAtomSM107UMMABlockScaledType + : CuteNVGPU_Umma<"MmaAtomSM107UMMABlockScaled", "sm107.mma_bs", + "SM107 block-scaled UMMA", + "128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, frag_kind = ss, vec_size = 32, ab_collector_op = (discard, use), int_overflow = ", + 1, 0, 1, 1>; +def CuteNVGPU_MmaAtomSM107UMMABlockScaledSparseType + : CuteNVGPU_Umma<"MmaAtomSM107UMMABlockScaledSparse", "sm107.mma_bs_sp", + "SM107 block-scaled sparse UMMA", + "128x128x64, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, sparse_metadata_format = tid, frag_kind = ss, vec_size = 32, ab_collector_op = (discard, discard)", + 1, 1, 1, 1>; #endif // CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_TYPES_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Registration/Registration.h b/mlir/dialects/cute_ir/include/cute_ir/Registration/Registration.h deleted file mode 100755 index 2e27e123..00000000 --- a/mlir/dialects/cute_ir/include/cute_ir/Registration/Registration.h +++ /dev/null @@ -1,46 +0,0 @@ -// clang-format off -/*************************************************************************************************** - * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: BSD-3-Clause - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -// clang-format on - -#ifndef CUTE_IR_REGISTRATION_REGISTRATION_H -#define CUTE_IR_REGISTRATION_REGISTRATION_H - -#include "mlir/IR/DialectRegistry.h" - -namespace mlir::cutlass_compiler::cute { - -void registerCuteDialects(DialectRegistry ®istry); -void registerCutePasses(); - -} // namespace mlir::cutlass_compiler::cute - -#endif // CUTE_IR_REGISTRATION_REGISTRATION_H diff --git a/mlir/dialects/cute_ir/lib/CAPI/BUILD.bazel b/mlir/dialects/cute_ir/lib/CAPI/BUILD.bazel index 448294dd..213c3f0c 100644 --- a/mlir/dialects/cute_ir/lib/CAPI/BUILD.bazel +++ b/mlir/dialects/cute_ir/lib/CAPI/BUILD.bazel @@ -7,6 +7,7 @@ cc_library( srcs = [ "Cute.cpp", "CuteAttributes.cpp", + "CuteCompilerAttributes.cpp", "CuteTypes.cpp", ], deps = [ @@ -21,6 +22,7 @@ cc_library( name = "cute_nvgpu", srcs = [ "CuteNVGPU.cpp", + "CuteNVGPUAttributes.cpp", "CuteNVGPUTypes.cpp", ], deps = [ diff --git a/mlir/dialects/cute_ir/lib/CAPI/CuteCompilerAttributes.cpp b/mlir/dialects/cute_ir/lib/CAPI/CuteCompilerAttributes.cpp new file mode 100644 index 00000000..d60e59e5 --- /dev/null +++ b/mlir/dialects/cute_ir/lib/CAPI/CuteCompilerAttributes.cpp @@ -0,0 +1,672 @@ +// C API for the CuTe attributes beyond the algebra, generated from the +// dialect's .td files. +#include "cute_ir-c/Dialect/CuteCompilerAttributes.h" + +#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" +#include "mlir/CAPI/IR.h" + +namespace cute = mlir::cutlass_compiler::cute; + +namespace { +// Unwraps `handle` into `out`: false when it is of another context or not a +// T, or when it is null and not optional. +template +bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { + auto value = unwrap(handle); + if (!value) + return optional; + if (value.getContext() != ctx) + return false; + out = llvm::dyn_cast(value); + return static_cast(out); +} +} // namespace + +bool mlirAttributeIsACuteBitLayout(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteBitLayoutAttrGet(MlirContext context, + MlirAttribute layout) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::LayoutAttr layoutValue; + if (!unwrapAs(ctx, layout, false, layoutValue)) + return {nullptr}; + return wrap(cute::BitLayoutAttr::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + layoutValue)); +} +MlirAttribute mlirCuteBitLayoutAttrGetLayout(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayout())); +} + +bool mlirAttributeIsACuteCopyAtom(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteCopyAtomAttrGet(MlirContext context, MlirAttribute thrId, + MlirAttribute layoutSrc, + MlirAttribute layoutDst, + MlirAttribute layoutRef) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; + if (!unwrapAs(ctx, thrId, false, thrIdValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutSrcValue; + if (!unwrapAs(ctx, layoutSrc, false, layoutSrcValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutDstValue; + if (!unwrapAs(ctx, layoutDst, false, layoutDstValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutRefValue; + if (!unwrapAs(ctx, layoutRef, false, layoutRefValue)) + return {nullptr}; + return wrap(cute::CopyAtomAttr::get(ctx, thrIdValue, layoutSrcValue, + layoutDstValue, layoutRefValue)); +} +MlirAttribute mlirCuteCopyAtomAttrGetThrId(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getThrId())); +} +MlirAttribute mlirCuteCopyAtomAttrGetLayoutSrc(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutSrc())); +} +MlirAttribute mlirCuteCopyAtomAttrGetLayoutDst(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutDst())); +} +MlirAttribute mlirCuteCopyAtomAttrGetLayoutRef(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutRef())); +} + +bool mlirAttributeIsACuteCopyAtomV2(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute +mlirCuteCopyAtomV2AttrGet(MlirContext context, MlirAttribute thrId, + MlirAttribute layoutSrcTV, MlirAttribute layoutDstTV, + MlirAttribute frgSrc, MlirAttribute frgDst) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; + if (!unwrapAs(ctx, thrId, false, thrIdValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutSrcTVValue; + if (!unwrapAs(ctx, layoutSrcTV, false, layoutSrcTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutDstTVValue; + if (!unwrapAs(ctx, layoutDstTV, false, layoutDstTVValue)) + return {nullptr}; + mlir::Attribute frgSrcValue; + if (!unwrapAs(ctx, frgSrc, false, frgSrcValue)) + return {nullptr}; + mlir::Attribute frgDstValue; + if (!unwrapAs(ctx, frgDst, false, frgDstValue)) + return {nullptr}; + return wrap(cute::CopyAtomV2Attr::get(ctx, thrIdValue, layoutSrcTVValue, + layoutDstTVValue, frgSrcValue, + frgDstValue)); +} +MlirAttribute mlirCuteCopyAtomV2AttrGetThrId(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getThrId())); +} +MlirAttribute mlirCuteCopyAtomV2AttrGetLayoutSrcTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutSrc_TV())); +} +MlirAttribute mlirCuteCopyAtomV2AttrGetLayoutDstTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutDst_TV())); +} +MlirAttribute mlirCuteCopyAtomV2AttrGetFrgSrc(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgSrc())); +} +MlirAttribute mlirCuteCopyAtomV2AttrGetFrgDst(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgDst())); +} + +bool mlirAttributeIsACuteMmaAtom(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutBTV, MlirAttribute layoutCTV, + MlirAttribute shapeAMK, MlirAttribute shapeBNK, MlirAttribute shapeCMN, + MlirAttribute frgA, MlirAttribute frgB, MlirAttribute frgC) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMNKValue; + if (!unwrapAs(ctx, shapeMNK, false, shapeMNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; + if (!unwrapAs(ctx, thrId, false, thrIdValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutATVValue; + if (!unwrapAs(ctx, layoutATV, false, layoutATVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutBTVValue; + if (!unwrapAs(ctx, layoutBTV, false, layoutBTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutCTVValue; + if (!unwrapAs(ctx, layoutCTV, false, layoutCTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeAMKValue; + if (!unwrapAs(ctx, shapeAMK, false, shapeAMKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeBNKValue; + if (!unwrapAs(ctx, shapeBNK, false, shapeBNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeCMNValue; + if (!unwrapAs(ctx, shapeCMN, false, shapeCMNValue)) + return {nullptr}; + mlir::Attribute frgAValue; + if (!unwrapAs(ctx, frgA, false, frgAValue)) + return {nullptr}; + mlir::Attribute frgBValue; + if (!unwrapAs(ctx, frgB, false, frgBValue)) + return {nullptr}; + mlir::Attribute frgCValue; + if (!unwrapAs(ctx, frgC, false, frgCValue)) + return {nullptr}; + return wrap(cute::MmaAtomAttr::get( + ctx, shapeMNKValue, thrIdValue, layoutATVValue, layoutBTVValue, + layoutCTVValue, shapeAMKValue, shapeBNKValue, shapeCMNValue, frgAValue, + frgBValue, frgCValue)); +} +MlirAttribute mlirCuteMmaAtomAttrGetShapeMNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShape_MNK())); +} +MlirAttribute mlirCuteMmaAtomAttrGetThrId(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getThrId())); +} +MlirAttribute mlirCuteMmaAtomAttrGetLayoutATV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutA_TV())); +} +MlirAttribute mlirCuteMmaAtomAttrGetLayoutBTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutB_TV())); +} +MlirAttribute mlirCuteMmaAtomAttrGetLayoutCTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutC_TV())); +} +MlirAttribute mlirCuteMmaAtomAttrGetShapeAMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeA_MK())); +} +MlirAttribute mlirCuteMmaAtomAttrGetShapeBNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeB_NK())); +} +MlirAttribute mlirCuteMmaAtomAttrGetShapeCMN(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeC_MN())); +} +MlirAttribute mlirCuteMmaAtomAttrGetFrgA(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgA())); +} +MlirAttribute mlirCuteMmaAtomAttrGetFrgB(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgB())); +} +MlirAttribute mlirCuteMmaAtomAttrGetFrgC(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgC())); +} + +bool mlirAttributeIsACuteMxMmaAtom(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteMxMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutSFATV, MlirAttribute layoutBTV, + MlirAttribute layoutSFBTV, MlirAttribute layoutCTV, MlirAttribute shapeAMK, + MlirAttribute shapeSFAMK, MlirAttribute shapeBNK, MlirAttribute shapeSFBNK, + MlirAttribute shapeCMN, MlirAttribute frgA, MlirAttribute frgSFA, + MlirAttribute frgB, MlirAttribute frgSFB, MlirAttribute frgC) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMNKValue; + if (!unwrapAs(ctx, shapeMNK, false, shapeMNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; + if (!unwrapAs(ctx, thrId, false, thrIdValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutATVValue; + if (!unwrapAs(ctx, layoutATV, false, layoutATVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutSFATVValue; + if (!unwrapAs(ctx, layoutSFATV, false, layoutSFATVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutBTVValue; + if (!unwrapAs(ctx, layoutBTV, false, layoutBTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutSFBTVValue; + if (!unwrapAs(ctx, layoutSFBTV, false, layoutSFBTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutCTVValue; + if (!unwrapAs(ctx, layoutCTV, false, layoutCTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeAMKValue; + if (!unwrapAs(ctx, shapeAMK, false, shapeAMKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeSFAMKValue; + if (!unwrapAs(ctx, shapeSFAMK, false, shapeSFAMKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeBNKValue; + if (!unwrapAs(ctx, shapeBNK, false, shapeBNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeSFBNKValue; + if (!unwrapAs(ctx, shapeSFBNK, false, shapeSFBNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeCMNValue; + if (!unwrapAs(ctx, shapeCMN, false, shapeCMNValue)) + return {nullptr}; + mlir::Attribute frgAValue; + if (!unwrapAs(ctx, frgA, false, frgAValue)) + return {nullptr}; + mlir::Attribute frgSFAValue; + if (!unwrapAs(ctx, frgSFA, false, frgSFAValue)) + return {nullptr}; + mlir::Attribute frgBValue; + if (!unwrapAs(ctx, frgB, false, frgBValue)) + return {nullptr}; + mlir::Attribute frgSFBValue; + if (!unwrapAs(ctx, frgSFB, false, frgSFBValue)) + return {nullptr}; + mlir::Attribute frgCValue; + if (!unwrapAs(ctx, frgC, false, frgCValue)) + return {nullptr}; + return wrap(cute::MxMmaAtomAttr::get( + ctx, shapeMNKValue, thrIdValue, layoutATVValue, layoutSFATVValue, + layoutBTVValue, layoutSFBTVValue, layoutCTVValue, shapeAMKValue, + shapeSFAMKValue, shapeBNKValue, shapeSFBNKValue, shapeCMNValue, frgAValue, + frgSFAValue, frgBValue, frgSFBValue, frgCValue)); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetShapeMNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShape_MNK())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetThrId(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getThrId())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutATV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutA_TV())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutSFATV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutSFA_TV())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutBTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutB_TV())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutSFBTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutSFB_TV())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutCTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutC_TV())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetShapeAMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeA_MK())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetShapeSFAMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeSFA_MK())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetShapeBNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeB_NK())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetShapeSFBNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeSFB_NK())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetShapeCMN(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeC_MN())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetFrgA(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgA())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetFrgSFA(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgSFA())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetFrgB(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgB())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetFrgSFB(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgSFB())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetFrgC(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgC())); +} + +bool mlirAttributeIsACuteReductionOp(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteReductionOpAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute::symbolizeReductionOp(value); + if (!valueValue) + return {nullptr}; + return wrap(cute::ReductionOpAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteReductionOpAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteSparseMmaAtom(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutBTV, MlirAttribute layoutCTV, + MlirAttribute layoutETV, MlirAttribute shapeAMK, MlirAttribute shapeBNK, + MlirAttribute shapeCMN, MlirAttribute shapeEMK, MlirAttribute frgA, + MlirAttribute frgB, MlirAttribute frgC, MlirAttribute frgE) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMNKValue; + if (!unwrapAs(ctx, shapeMNK, false, shapeMNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; + if (!unwrapAs(ctx, thrId, false, thrIdValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutATVValue; + if (!unwrapAs(ctx, layoutATV, false, layoutATVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutBTVValue; + if (!unwrapAs(ctx, layoutBTV, false, layoutBTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutCTVValue; + if (!unwrapAs(ctx, layoutCTV, false, layoutCTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutETVValue; + if (!unwrapAs(ctx, layoutETV, false, layoutETVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeAMKValue; + if (!unwrapAs(ctx, shapeAMK, false, shapeAMKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeBNKValue; + if (!unwrapAs(ctx, shapeBNK, false, shapeBNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeCMNValue; + if (!unwrapAs(ctx, shapeCMN, false, shapeCMNValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeEMKValue; + if (!unwrapAs(ctx, shapeEMK, false, shapeEMKValue)) + return {nullptr}; + mlir::Attribute frgAValue; + if (!unwrapAs(ctx, frgA, false, frgAValue)) + return {nullptr}; + mlir::Attribute frgBValue; + if (!unwrapAs(ctx, frgB, false, frgBValue)) + return {nullptr}; + mlir::Attribute frgCValue; + if (!unwrapAs(ctx, frgC, false, frgCValue)) + return {nullptr}; + mlir::Attribute frgEValue; + if (!unwrapAs(ctx, frgE, false, frgEValue)) + return {nullptr}; + return wrap(cute::SparseMmaAtomAttr::get( + ctx, shapeMNKValue, thrIdValue, layoutATVValue, layoutBTVValue, + layoutCTVValue, layoutETVValue, shapeAMKValue, shapeBNKValue, + shapeCMNValue, shapeEMKValue, frgAValue, frgBValue, frgCValue, + frgEValue)); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeMNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShape_MNK())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetThrId(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getThrId())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetLayoutATV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutA_TV())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetLayoutBTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutB_TV())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetLayoutCTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutC_TV())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetLayoutETV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutE_TV())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeAMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeA_MK())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeBNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeB_NK())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeCMN(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeC_MN())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeEMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeE_MK())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetFrgA(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgA())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetFrgB(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgB())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetFrgC(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgC())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetFrgE(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgE())); +} + +bool mlirAttributeIsACuteSparseMxMmaAtom(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutSFATV, MlirAttribute layoutBTV, + MlirAttribute layoutSFBTV, MlirAttribute layoutCTV, MlirAttribute layoutETV, + MlirAttribute shapeAMK, MlirAttribute shapeSFAMK, MlirAttribute shapeBNK, + MlirAttribute shapeSFBNK, MlirAttribute shapeCMN, MlirAttribute shapeEMK, + MlirAttribute frgA, MlirAttribute frgSFA, MlirAttribute frgB, + MlirAttribute frgSFB, MlirAttribute frgC, MlirAttribute frgE) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMNKValue; + if (!unwrapAs(ctx, shapeMNK, false, shapeMNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; + if (!unwrapAs(ctx, thrId, false, thrIdValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutATVValue; + if (!unwrapAs(ctx, layoutATV, false, layoutATVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutSFATVValue; + if (!unwrapAs(ctx, layoutSFATV, false, layoutSFATVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutBTVValue; + if (!unwrapAs(ctx, layoutBTV, false, layoutBTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutSFBTVValue; + if (!unwrapAs(ctx, layoutSFBTV, false, layoutSFBTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutCTVValue; + if (!unwrapAs(ctx, layoutCTV, false, layoutCTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutETVValue; + if (!unwrapAs(ctx, layoutETV, false, layoutETVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeAMKValue; + if (!unwrapAs(ctx, shapeAMK, false, shapeAMKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeSFAMKValue; + if (!unwrapAs(ctx, shapeSFAMK, false, shapeSFAMKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeBNKValue; + if (!unwrapAs(ctx, shapeBNK, false, shapeBNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeSFBNKValue; + if (!unwrapAs(ctx, shapeSFBNK, false, shapeSFBNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeCMNValue; + if (!unwrapAs(ctx, shapeCMN, false, shapeCMNValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeEMKValue; + if (!unwrapAs(ctx, shapeEMK, false, shapeEMKValue)) + return {nullptr}; + mlir::Attribute frgAValue; + if (!unwrapAs(ctx, frgA, false, frgAValue)) + return {nullptr}; + mlir::Attribute frgSFAValue; + if (!unwrapAs(ctx, frgSFA, false, frgSFAValue)) + return {nullptr}; + mlir::Attribute frgBValue; + if (!unwrapAs(ctx, frgB, false, frgBValue)) + return {nullptr}; + mlir::Attribute frgSFBValue; + if (!unwrapAs(ctx, frgSFB, false, frgSFBValue)) + return {nullptr}; + mlir::Attribute frgCValue; + if (!unwrapAs(ctx, frgC, false, frgCValue)) + return {nullptr}; + mlir::Attribute frgEValue; + if (!unwrapAs(ctx, frgE, false, frgEValue)) + return {nullptr}; + return wrap(cute::SparseMxMmaAtomAttr::get( + ctx, shapeMNKValue, thrIdValue, layoutATVValue, layoutSFATVValue, + layoutBTVValue, layoutSFBTVValue, layoutCTVValue, layoutETVValue, + shapeAMKValue, shapeSFAMKValue, shapeBNKValue, shapeSFBNKValue, + shapeCMNValue, shapeEMKValue, frgAValue, frgSFAValue, frgBValue, + frgSFBValue, frgCValue, frgEValue)); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeMNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShape_MNK())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetThrId(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getThrId())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutATV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutA_TV())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutSFATV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutSFA_TV())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutBTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutB_TV())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutSFBTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutSFB_TV())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutCTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutC_TV())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutETV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutE_TV())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeAMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeA_MK())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeSFAMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeSFA_MK())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeBNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeB_NK())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeSFBNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeSFB_NK())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeCMN(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeC_MN())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeEMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeE_MK())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgA(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgA())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgSFA(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgSFA())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgB(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgB())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgSFB(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgSFB())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgC(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgC())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgE(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgE())); +} diff --git a/mlir/dialects/cute_ir/lib/CAPI/CuteNVGPUAttributes.cpp b/mlir/dialects/cute_ir/lib/CAPI/CuteNVGPUAttributes.cpp new file mode 100644 index 00000000..e4633940 --- /dev/null +++ b/mlir/dialects/cute_ir/lib/CAPI/CuteNVGPUAttributes.cpp @@ -0,0 +1,1183 @@ +// C API for the CuTe NVIDIA GPU attributes, generated from the dialect's .td +// files. +#include "cute_ir-c/Dialect/CuteNVGPUAttributes.h" + +#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h" +#include "mlir/CAPI/IR.h" + +namespace cute_nvgpu = mlir::cutlass_compiler::cute_nvgpu; + +namespace { +// Unwraps `handle` into `out`: false when it is of another context or not a +// T, or when it is null and not optional. +template +bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { + auto value = unwrap(handle); + if (!value) + return optional; + if (value.getContext() != ctx) + return false; + out = llvm::dyn_cast(value); + return static_cast(out); +} +} // namespace + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyG2S(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldBulkCopyG2S( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldBulkCopyG2SAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2G(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldBulkCopyS2G( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldBulkCopyS2GAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2S(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldBulkCopyS2S( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldBulkCopyS2SAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldDsmemStore(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldDsmemStore( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldDsmemStoreAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldLoadGlobal(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldLoadGlobal( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldLoadGlobalAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoad( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExec2DGather4TmaLoadAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExec2DGather4TmaLoad(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldNonExec2DGather4TmaLoadAttr::get( + ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGetValue( + MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStore( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExec2DScatter4TmaStoreAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExec2DScatter4TmaStore(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldNonExec2DScatter4TmaStoreAttr::get( + ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGetValue( + MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoad( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExecIm2ColTmaLoadAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExecIm2ColTmaLoad(value); + if (!valueValue) + return {nullptr}; + return wrap( + cute_nvgpu::AtomCopyFieldNonExecIm2ColTmaLoadAttr::get(ctx, *valueValue)); +} +uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaStore( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExecIm2ColTmaStoreAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExecIm2ColTmaStore(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldNonExecIm2ColTmaStoreAttr::get( + ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGetValue( + MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaLoad( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExecTiledTmaLoadAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExecTiledTmaLoad(value); + if (!valueValue) + return {nullptr}; + return wrap( + cute_nvgpu::AtomCopyFieldNonExecTiledTmaLoadAttr::get(ctx, *valueValue)); +} +uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaReduce( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExecTiledTmaReduceAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExecTiledTmaReduce(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldNonExecTiledTmaReduceAttr::get( + ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGetValue( + MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaStore( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExecTiledTmaStoreAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExecTiledTmaStore(value); + if (!valueValue) + return {nullptr}; + return wrap( + cute_nvgpu::AtomCopyFieldNonExecTiledTmaStoreAttr::get(ctx, *valueValue)); +} +uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldStoreGlobal(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldStoreGlobal( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldStoreGlobalAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldTmaLoad(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldTmaLoad( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldTmaLoadAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldTmaReduce(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldTmaReduce( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldTmaReduceAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldTmaStore(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldTmaStore( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldTmaStoreAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM100(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomMmaFieldSM100AttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomMmaFieldSM100(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomMmaFieldSM100Attr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomMmaFieldSM100AttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaled(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomMmaFieldSM100BlockScaled(value); + if (!valueValue) + return {nullptr}; + return wrap( + cute_nvgpu::AtomMmaFieldSM100BlockScaledAttr::get(ctx, *valueValue)); +} +uint32_t +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaledSparse( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomMmaFieldSM100BlockScaledSparseAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomMmaFieldSM100BlockScaledSparse(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomMmaFieldSM100BlockScaledSparseAttr::get( + ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGetValue( + MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM100Sparse(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomMmaFieldSM100Sparse( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomMmaFieldSM100SparseAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM120BlockScaled(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomMmaFieldSM120BlockScaled(value); + if (!valueValue) + return {nullptr}; + return wrap( + cute_nvgpu::AtomMmaFieldSM120BlockScaledAttr::get(ctx, *valueValue)); +} +uint32_t +mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM80Sparse(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomMmaFieldSM80Sparse( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomMmaFieldSM80SparseAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM90(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomMmaFieldSM90AttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomMmaFieldSM90(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomMmaFieldSM90Attr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomMmaFieldSM90AttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUBinaryOp(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUBinaryOpAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeBinaryOp(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::BinaryOpAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUBinaryOpAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUCopyS2TBroadcast(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUCopyS2TBroadcastAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeCopyS2TBroadcast(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyS2TBroadcastAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUCopyS2TBroadcastAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUGatherScatterTmaLoad(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUGatherScatterTmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeGatherScatterTmaLoad( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::GatherScatterTmaLoadAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUGatherScatterTmaLoadAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUIm2ColTmaLoad(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUIm2ColTmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeIm2ColTmaLoad(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::Im2ColTmaLoadAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUIm2ColTmaLoadAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPULdReduceAccPrecisionKind(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPULdReduceAccPrecisionKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLdReduceAccPrecisionKind( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::LdReduceAccPrecisionKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPULdReduceAccPrecisionKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPULdsmSzPattern(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPULdsmSzPatternAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLdsmSzPattern(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::LdsmSzPatternAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPULdsmSzPatternAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPULoadCacheMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPULoadCacheModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLoadCacheMode(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::LoadCacheModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPULoadCacheModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUMMAIntOverflow(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUMMAIntOverflowAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMMAIntOverflow(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::MMAIntOverflowAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUMMAIntOverflowAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUMajorMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUMajorModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::MajorModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUMajorModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUMmaCollectorOp(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUMmaCollectorOpAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::MmaCollectorOpAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUMmaCollectorOpAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUMmaFragKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUMmaFragKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::MmaFragKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUMmaFragKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUNotImplementedFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUNotImplementedFrgAttrGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute_nvgpu::NotImplementedFrgAttr::get(ctx)); +} + +bool mlirAttributeIsACuteNVGPUReductionKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUReductionKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeReductionKind(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::ReductionKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUReductionKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPURmemFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPURmemFrgAttrGet(MlirContext context, + MlirType valueType, uint32_t operand, + bool deriveElemType) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valueTypeValue; + if (!unwrapAs(ctx, valueType, false, valueTypeValue)) + return {nullptr}; + auto operandValue = + ::mlir::cutlass_compiler::cute::symbolizeMmaOperand(operand); + if (!operandValue) + return {nullptr}; + return wrap(cute_nvgpu::RmemFrgAttr::get(ctx, valueTypeValue, *operandValue, + deriveElemType)); +} +MlirType mlirCuteNVGPURmemFrgAttrGetValueType(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getValueType())); +} +uint32_t mlirCuteNVGPURmemFrgAttrGetOperand(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getOperand()); +} +bool mlirCuteNVGPURmemFrgAttrGetDeriveElemType(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getDeriveElemType(); +} + +bool mlirAttributeIsACuteNVGPUSM100CircularSmemFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM100CircularSmemFrgAttrGet(MlirContext context, + uint32_t majorMode) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto majorModeValue = + ::mlir::cutlass_compiler::cute::symbolizeMajorMode(majorMode); + if (!majorModeValue) + return {nullptr}; + return wrap(cute_nvgpu::SM100CircularSmemFrgAttr::get(ctx, *majorModeValue)); +} +uint32_t mlirCuteNVGPUSM100CircularSmemFrgAttrGetMajorMode(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getMajorMode()); +} + +bool mlirAttributeIsACuteNVGPUSM100SmemFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM100SmemFrgAttrGet(MlirContext context, + uint32_t majorMode) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto majorModeValue = + ::mlir::cutlass_compiler::cute::symbolizeMajorMode(majorMode); + if (!majorModeValue) + return {nullptr}; + return wrap(cute_nvgpu::SM100SmemFrgAttr::get(ctx, *majorModeValue)); +} +uint32_t mlirCuteNVGPUSM100SmemFrgAttrGetMajorMode(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getMajorMode()); +} + +bool mlirAttributeIsACuteNVGPUSM100TmemEFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM100TmemEFrgAttrGet(MlirContext context, + MlirType aType, + MlirType eType) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type eTypeValue; + if (!unwrapAs(ctx, eType, false, eTypeValue)) + return {nullptr}; + return wrap(cute_nvgpu::SM100TmemEFrgAttr::get(ctx, aTypeValue, eTypeValue)); +} +MlirType mlirCuteNVGPUSM100TmemEFrgAttrGetAType(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getAType())); +} +MlirType mlirCuteNVGPUSM100TmemEFrgAttrGetEType(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getEType())); +} + +bool mlirAttributeIsACuteNVGPUSM100TmemFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM100TmemFrgAttrGet(MlirContext context, + MlirType dataType, + MlirType storageType, + int ctaGroup, + uint32_t tmemAllocMode) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type dataTypeValue; + if (!unwrapAs(ctx, dataType, false, dataTypeValue)) + return {nullptr}; + mlir::Type storageTypeValue; + if (!unwrapAs(ctx, storageType, false, storageTypeValue)) + return {nullptr}; + auto tmemAllocModeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemAllocMode( + tmemAllocMode); + if (!tmemAllocModeValue) + return {nullptr}; + return wrap(cute_nvgpu::SM100TmemFrgAttr::get( + ctx, dataTypeValue, storageTypeValue, ctaGroup, *tmemAllocModeValue)); +} +MlirType mlirCuteNVGPUSM100TmemFrgAttrGetDataType(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getDataType())); +} +MlirType mlirCuteNVGPUSM100TmemFrgAttrGetStorageType(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getStorageType())); +} +int mlirCuteNVGPUSM100TmemFrgAttrGetCtaGroup(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getCtaGroup(); +} +uint32_t mlirCuteNVGPUSM100TmemFrgAttrGetTmemAllocMode(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getTmemAllocMode()); +} + +bool mlirAttributeIsACuteNVGPUSM100TmemSfFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM100TmemSfFrgAttrGet(MlirContext context, + MlirType sfType, int sfVecSize, + int ctaGroup, bool isSfa, + uint32_t tmemAllocMode) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type sfTypeValue; + if (!unwrapAs(ctx, sfType, false, sfTypeValue)) + return {nullptr}; + auto tmemAllocModeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemAllocMode( + tmemAllocMode); + if (!tmemAllocModeValue) + return {nullptr}; + return wrap(cute_nvgpu::SM100TmemSfFrgAttr::get( + ctx, sfTypeValue, sfVecSize, ctaGroup, isSfa, *tmemAllocModeValue)); +} +MlirType mlirCuteNVGPUSM100TmemSfFrgAttrGetSfType(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getSfType())); +} +int mlirCuteNVGPUSM100TmemSfFrgAttrGetSfVecSize(MlirAttribute attr) { + return llvm::cast(unwrap(attr)) + .getSfVecSize(); +} +int mlirCuteNVGPUSM100TmemSfFrgAttrGetCtaGroup(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getCtaGroup(); +} +bool mlirCuteNVGPUSM100TmemSfFrgAttrGetIsSfa(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getIsSfa(); +} +uint32_t mlirCuteNVGPUSM100TmemSfFrgAttrGetTmemAllocMode(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getTmemAllocMode()); +} + +bool mlirAttributeIsACuteNVGPUSM107SmemFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM107SmemFrgAttrGet(MlirContext context, + uint32_t majorMode) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto majorModeValue = + ::mlir::cutlass_compiler::cute::symbolizeMajorMode(majorMode); + if (!majorModeValue) + return {nullptr}; + return wrap(cute_nvgpu::SM107SmemFrgAttr::get(ctx, *majorModeValue)); +} +uint32_t mlirCuteNVGPUSM107SmemFrgAttrGetMajorMode(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getMajorMode()); +} + +bool mlirAttributeIsACuteNVGPUSM90SmemFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM90SmemFrgAttrGet(MlirContext context, + uint32_t majorMode) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto majorModeValue = + ::mlir::cutlass_compiler::cute::symbolizeMajorMode(majorMode); + if (!majorModeValue) + return {nullptr}; + return wrap(cute_nvgpu::SM90SmemFrgAttr::get(ctx, *majorModeValue)); +} +uint32_t mlirCuteNVGPUSM90SmemFrgAttrGetMajorMode(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getMajorMode()); +} + +bool mlirAttributeIsACuteNVGPUTiledTmaLoad(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUTiledTmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTiledTmaLoad(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::TiledTmaLoadAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUTiledTmaLoadAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUTmaDataFormat(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUTmaDataFormatAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::TmaDataFormatAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUTmaDataFormatAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUTmaLoadMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUTmaLoadModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaLoadMode(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::TmaLoadModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUTmaLoadModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUTmaStoreMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUTmaStoreModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaStoreMode(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::TmaStoreModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUTmaStoreModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUTmemAllocMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUTmemAllocModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemAllocMode(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::TmemAllocModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUTmemAllocModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUTmemLoadRedOp(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUTmemLoadRedOpAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemLoadRedOp(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::TmemLoadRedOpAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUTmemLoadRedOpAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} diff --git a/mlir/dialects/cute_ir/lib/CAPI/CuteNVGPUTypes.cpp b/mlir/dialects/cute_ir/lib/CAPI/CuteNVGPUTypes.cpp index 5072b643..181419da 100644 --- a/mlir/dialects/cute_ir/lib/CAPI/CuteNVGPUTypes.cpp +++ b/mlir/dialects/cute_ir/lib/CAPI/CuteNVGPUTypes.cpp @@ -1,30 +1,55 @@ -// C++ to C bindings for CuTe NVIDIA GPU types. +// C API for the CuTe NVIDIA GPU types, generated from the dialect's .td files. #include "cute_ir-c/Dialect/CuteNVGPUTypes.h" + #include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h" #include "mlir/CAPI/IR.h" namespace cute_nvgpu = mlir::cutlass_compiler::cute_nvgpu; +namespace { +// Unwraps `handle` into `out`: false when it is of another context or not a +// T, or when it is null and not optional. +template +bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { + auto value = unwrap(handle); + if (!value) + return optional; + if (value.getContext() != ctx) + return false; + out = llvm::dyn_cast(value); + return static_cast(out); +} +} // namespace + bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyG2S(MlirType type) { return llvm::isa_and_nonnull( unwrap(type)); } MlirType mlirCuteNVGPUCopyAtomBulkCopyG2STypeGet(MlirContext context, - MlirAttribute payload) { + MlirType valType, int copyBits, + bool mcast) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap(cute_nvgpu::CopyAtomBulkCopyG2SType::get(ctx, payloadValue)); + return wrap(cute_nvgpu::CopyAtomBulkCopyG2SType::get(ctx, valTypeValue, + copyBits, mcast)); } -MlirAttribute mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetPayload(MlirType type) { - return wrap(static_cast( +MlirType mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetValType(MlirType type) { + return wrap(static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getValType())); +} +int mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +bool mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetMcast(MlirType type) { + return llvm::cast(unwrap(type)) + .getMcast(); } bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2G(MlirType type) { @@ -32,21 +57,30 @@ bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2G(MlirType type) { unwrap(type)); } MlirType mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGet(MlirContext context, - MlirAttribute payload) { + MlirType valType, int copyBits, + bool mask) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap(cute_nvgpu::CopyAtomBulkCopyS2GType::get(ctx, payloadValue)); + return wrap(cute_nvgpu::CopyAtomBulkCopyS2GType::get(ctx, valTypeValue, + copyBits, mask)); } -MlirAttribute mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetPayload(MlirType type) { - return wrap(static_cast( +MlirType mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetValType(MlirType type) { + return wrap(static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getValType())); +} +int mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +bool mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetMask(MlirType type) { + return llvm::cast(unwrap(type)) + .getMask(); } bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2S(MlirType type) { @@ -54,21 +88,26 @@ bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2S(MlirType type) { unwrap(type)); } MlirType mlirCuteNVGPUCopyAtomBulkCopyS2STypeGet(MlirContext context, - MlirAttribute payload) { + MlirType valType, + int copyBits) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap(cute_nvgpu::CopyAtomBulkCopyS2SType::get(ctx, payloadValue)); + return wrap( + cute_nvgpu::CopyAtomBulkCopyS2SType::get(ctx, valTypeValue, copyBits)); } -MlirAttribute mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetPayload(MlirType type) { - return wrap(static_cast( +MlirType mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetValType(MlirType type) { + return wrap(static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getValType())); +} +int mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); } bool mlirTypeIsACuteNVGPUCopyAtomDsmemStore(MlirType type) { @@ -76,41 +115,111 @@ bool mlirTypeIsACuteNVGPUCopyAtomDsmemStore(MlirType type) { unwrap(type)); } MlirType mlirCuteNVGPUCopyAtomDsmemStoreTypeGet(MlirContext context, - MlirAttribute payload) { + MlirType valType, + int copyBits) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap(cute_nvgpu::CopyAtomDsmemStoreType::get(ctx, payloadValue)); + return wrap(cute_nvgpu::CopyAtomDsmemStoreType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits)); } -MlirAttribute mlirCuteNVGPUCopyAtomDsmemStoreTypeGetPayload(MlirType type) { - return wrap(static_cast( +MlirType mlirCuteNVGPUCopyAtomDsmemStoreTypeGetValType(MlirType type) { + return wrap(static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getValType())); +} +int mlirCuteNVGPUCopyAtomDsmemStoreTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); } bool mlirTypeIsACuteNVGPUCopyAtomG2R(MlirType type) { return llvm::isa_and_nonnull(unwrap(type)); } -MlirType mlirCuteNVGPUCopyAtomG2RTypeGet(MlirContext context, - MlirAttribute payload) { +MlirType mlirCuteNVGPUCopyAtomG2RTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t memOrder, + uint32_t memScope, uint32_t l2PrefetchSize, uint32_t l1CacheEvictPriority, + uint32_t loadCacheMode, uint32_t sharedSpace, bool invariant) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap(cute_nvgpu::CopyAtomG2RType::get(ctx, payloadValue)); + auto memOrderValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); + if (!memOrderValue) + return {nullptr}; + auto memScopeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); + if (!memScopeValue) + return {nullptr}; + auto l2PrefetchSizeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeL2PrefetchSize( + l2PrefetchSize); + if (!l2PrefetchSizeValue) + return {nullptr}; + auto l1CacheEvictPriorityValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeCacheEvictionPriority( + l1CacheEvictPriority); + if (!l1CacheEvictPriorityValue) + return {nullptr}; + auto loadCacheModeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLoadCacheMode( + loadCacheMode); + if (!loadCacheModeValue) + return {nullptr}; + auto sharedSpaceValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSharedSpace(sharedSpace); + if (!sharedSpaceValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomG2RType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits, *memOrderValue, *memScopeValue, + *l2PrefetchSizeValue, *l1CacheEvictPriorityValue, *loadCacheModeValue, + *sharedSpaceValue, invariant)); } -MlirAttribute mlirCuteNVGPUCopyAtomG2RTypeGetPayload(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getPayload())); +MlirType mlirCuteNVGPUCopyAtomG2RTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValType())); +} +int mlirCuteNVGPUCopyAtomG2RTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)).getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomG2RTypeGetMemOrder(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMemOrder()); +} +uint32_t mlirCuteNVGPUCopyAtomG2RTypeGetMemScope(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMemScope()); +} +uint32_t mlirCuteNVGPUCopyAtomG2RTypeGetL2PrefetchSize(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getL2PrefetchSize()); +} +uint32_t mlirCuteNVGPUCopyAtomG2RTypeGetL1CacheEvictPriority(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getL1CacheEvictPriority()); +} +uint32_t mlirCuteNVGPUCopyAtomG2RTypeGetLoadCacheMode(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getLoadCacheMode()); +} +uint32_t mlirCuteNVGPUCopyAtomG2RTypeGetSharedSpace(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getSharedSpace()); +} +bool mlirCuteNVGPUCopyAtomG2RTypeGetInvariant(MlirType type) { + return llvm::cast(unwrap(type)).getInvariant(); } bool mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaLoad(MlirType type) { @@ -118,21 +227,53 @@ bool mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaLoad(MlirType type) { unwrap(type)); } MlirType mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGet(MlirContext context, - MlirAttribute payload) { + MlirType valType, + int copyBits, int numCta, + MlirType gStride, bool mcast, + MlirType tmaGbasis) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap(cute_nvgpu::CopyAtomIm2ColTmaLoadType::get(ctx, payloadValue)); + ::mlir::cutlass_compiler::cute::StrideType gStrideValue; + if (!unwrapAs(ctx, gStride, false, gStrideValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, true, tmaGbasisValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomIm2ColTmaLoadType::get( + ctx, valTypeValue, copyBits, numCta, gStrideValue, mcast, + tmaGbasisValue)); } -MlirAttribute mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetPayload(MlirType type) { - return wrap(static_cast( +MlirType mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetValType(MlirType type) { + return wrap(static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getValType())); +} +int mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +int mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumCta(); +} +MlirType mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetGStride(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getGStride())); +} +bool mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetMcast(MlirType type) { + return llvm::cast(unwrap(type)) + .getMcast(); +} +MlirType mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); } bool mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaStore(MlirType type) { @@ -140,251 +281,688 @@ bool mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaStore(MlirType type) { unwrap(type)); } MlirType mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGet(MlirContext context, - MlirAttribute payload) { + MlirType valType, + int copyBits, + MlirType gStride, + MlirType tmaGbasis) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap(cute_nvgpu::CopyAtomIm2ColTmaStoreType::get(ctx, payloadValue)); + ::mlir::cutlass_compiler::cute::StrideType gStrideValue; + if (!unwrapAs(ctx, gStride, false, gStrideValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, true, tmaGbasisValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomIm2ColTmaStoreType::get( + ctx, valTypeValue, copyBits, gStrideValue, tmaGbasisValue)); } -MlirAttribute mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetPayload(MlirType type) { - return wrap(static_cast( +MlirType mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetValType(MlirType type) { + return wrap(static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getValType())); +} +int mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +MlirType mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetGStride(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getGStride())); +} +MlirType mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); } bool mlirTypeIsACuteNVGPUCopyAtomLdsm(MlirType type) { return llvm::isa_and_nonnull(unwrap(type)); } -MlirType mlirCuteNVGPUCopyAtomLdsmTypeGet(MlirContext context, - MlirAttribute payload) { +MlirType mlirCuteNVGPUCopyAtomLdsmTypeGet(MlirContext context, MlirType valType, + MlirAttribute mode, + uint32_t szPattern, int numMatrices, + bool transpose) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap(cute_nvgpu::CopyAtomLdsmType::get(ctx, payloadValue)); + ::mlir::cutlass_compiler::cute::ShapeAttr modeValue; + if (!unwrapAs(ctx, mode, false, modeValue)) + return {nullptr}; + auto szPatternValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLdsmSzPattern(szPattern); + if (!szPatternValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomLdsmType::get( + ctx, valTypeValue, modeValue, *szPatternValue, numMatrices, + transpose ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr())); } -MlirAttribute mlirCuteNVGPUCopyAtomLdsmTypeGetPayload(MlirType type) { +MlirType mlirCuteNVGPUCopyAtomLdsmTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValType())); +} +MlirAttribute mlirCuteNVGPUCopyAtomLdsmTypeGetMode(MlirType type) { return wrap(static_cast( - llvm::cast(unwrap(type)).getPayload())); + llvm::cast(unwrap(type)).getMode())); +} +uint32_t mlirCuteNVGPUCopyAtomLdsmTypeGetSzPattern(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getSzPattern()); +} +int mlirCuteNVGPUCopyAtomLdsmTypeGetNumMatrices(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumMatrices(); +} +bool mlirCuteNVGPUCopyAtomLdsmTypeGetTranspose(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getTranspose()); } bool mlirTypeIsACuteNVGPUCopyAtomNonExec2DGather4TmaLoad(MlirType type) { return llvm::isa_and_nonnull( unwrap(type)); } -MlirType -mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGet(MlirContext context, - MlirAttribute payload) { +MlirType mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGet( + MlirContext context, uint32_t kind, MlirType valType, int copyBits, + MlirType tmaGbasis, int64_t tmaFormat) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + auto kindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeGatherScatterTmaLoad(kind); + if (!kindValue) return {nullptr}; - return wrap( - cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType::get(ctx, payloadValue)); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) + return {nullptr}; + std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> + tmaFormatValue; + if (tmaFormat >= 0) { + tmaFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( + static_cast(tmaFormat)); + if (!tmaFormatValue) + return {nullptr}; + } + return wrap(cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + *kindValue, valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); } -MlirAttribute -mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetPayload(MlirType type) { - return wrap(static_cast( +uint32_t +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetKind(MlirType type) { + return static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getKind()); +} +MlirType +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetCopyBits(MlirType type) { + return llvm::cast( + unwrap(type)) + .getCopyBits(); +} +MlirType +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} +int64_t +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaFormat(MlirType type) { + auto value = + llvm::cast(unwrap(type)) + .getTmaFormat(); + return value ? static_cast(*value) : -1; +} + +bool mlirTypeIsACuteNVGPUCopyAtomNonExec2DScatter4TmaStore(MlirType type) { + return llvm::isa_and_nonnull< + cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGet( + MlirContext context, MlirType valType, int copyBits, MlirType tmaGbasis, + int64_t tmaFormat) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) + return {nullptr}; + std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> + tmaFormatValue; + if (tmaFormat >= 0) { + tmaFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( + static_cast(tmaFormat)); + if (!tmaFormatValue) + return {nullptr}; + } + return wrap(cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); +} +MlirType +mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetCopyBits( + MlirType type) { + return llvm::cast( + unwrap(type)) + .getCopyBits(); +} +MlirType +mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getTmaGbasis())); +} +int64_t +mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaFormat(MlirType type) { + auto value = llvm::cast( + unwrap(type)) + .getTmaFormat(); + return value ? static_cast(*value) : -1; } bool mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaLoad(MlirType type) { return llvm::isa_and_nonnull( unwrap(type)); } -MlirType -mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGet(MlirContext context, - MlirAttribute payload) { +MlirType mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGet( + MlirContext context, uint32_t kind, MlirType valType, int copyBits, + MlirType tmaGbasis, int64_t tmaFormat) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + auto kindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeIm2ColTmaLoad(kind); + if (!kindValue) return {nullptr}; - return wrap( - cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType::get(ctx, payloadValue)); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) + return {nullptr}; + std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> + tmaFormatValue; + if (tmaFormat >= 0) { + tmaFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( + static_cast(tmaFormat)); + if (!tmaFormatValue) + return {nullptr}; + } + return wrap(cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + *kindValue, valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); } -MlirAttribute -mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetPayload(MlirType type) { - return wrap(static_cast( +uint32_t mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetKind(MlirType type) { + return static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getKind()); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} +int64_t +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaFormat(MlirType type) { + auto value = + llvm::cast(unwrap(type)) + .getTmaFormat(); + return value ? static_cast(*value) : -1; } bool mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaStore(MlirType type) { return llvm::isa_and_nonnull( unwrap(type)); } -MlirType -mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGet(MlirContext context, - MlirAttribute payload) { +MlirType mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGet(MlirContext context, + MlirType valType, + int copyBits, + MlirType tmaGbasis, + int64_t tmaFormat) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap( - cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType::get(ctx, payloadValue)); + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) + return {nullptr}; + std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> + tmaFormatValue; + if (tmaFormat >= 0) { + tmaFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( + static_cast(tmaFormat)); + if (!tmaFormatValue) + return {nullptr}; + } + return wrap(cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); } -MlirAttribute -mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetPayload(MlirType type) { - return wrap(static_cast( +MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetValType(MlirType type) { + return wrap(static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getValType())); +} +int mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} +int64_t +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaFormat(MlirType type) { + auto value = + llvm::cast(unwrap(type)) + .getTmaFormat(); + return value ? static_cast(*value) : -1; } bool mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaLoad(MlirType type) { return llvm::isa_and_nonnull( unwrap(type)); } -MlirType -mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGet(MlirContext context, - MlirAttribute payload) { +MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGet( + MlirContext context, uint32_t kind, MlirType valType, int copyBits, + MlirType tmaGbasis, int64_t tmaFormat) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + auto kindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTiledTmaLoad(kind); + if (!kindValue) return {nullptr}; - return wrap( - cute_nvgpu::CopyAtomNonExecTiledTmaLoadType::get(ctx, payloadValue)); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) + return {nullptr}; + std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> + tmaFormatValue; + if (tmaFormat >= 0) { + tmaFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( + static_cast(tmaFormat)); + if (!tmaFormatValue) + return {nullptr}; + } + return wrap(cute_nvgpu::CopyAtomNonExecTiledTmaLoadType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + *kindValue, valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); } -MlirAttribute -mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetPayload(MlirType type) { - return wrap(static_cast( +uint32_t mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetKind(MlirType type) { + return static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getKind()); +} +MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} +int64_t +mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaFormat(MlirType type) { + auto value = + llvm::cast(unwrap(type)) + .getTmaFormat(); + return value ? static_cast(*value) : -1; } bool mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaReduce(MlirType type) { return llvm::isa_and_nonnull( unwrap(type)); } -MlirType -mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGet(MlirContext context, - MlirAttribute payload) { +MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGet( + MlirContext context, uint32_t kind, MlirType valType, int copyBits, + MlirType tmaGbasis, int64_t tmaFormat) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + auto kindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeReductionKind(kind); + if (!kindValue) return {nullptr}; - return wrap( - cute_nvgpu::CopyAtomNonExecTiledTmaReduceType::get(ctx, payloadValue)); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) + return {nullptr}; + std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> + tmaFormatValue; + if (tmaFormat >= 0) { + tmaFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( + static_cast(tmaFormat)); + if (!tmaFormatValue) + return {nullptr}; + } + return wrap(cute_nvgpu::CopyAtomNonExecTiledTmaReduceType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + *kindValue, valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); } -MlirAttribute -mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetPayload(MlirType type) { - return wrap(static_cast( +uint32_t mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetKind(MlirType type) { + return static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getKind()); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} +int64_t +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaFormat(MlirType type) { + auto value = + llvm::cast(unwrap(type)) + .getTmaFormat(); + return value ? static_cast(*value) : -1; } bool mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaStore(MlirType type) { return llvm::isa_and_nonnull( unwrap(type)); } -MlirType -mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGet(MlirContext context, - MlirAttribute payload) { +MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGet(MlirContext context, + MlirType valType, + int copyBits, + MlirType tmaGbasis, + int64_t tmaFormat) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap( - cute_nvgpu::CopyAtomNonExecTiledTmaStoreType::get(ctx, payloadValue)); + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) + return {nullptr}; + std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> + tmaFormatValue; + if (tmaFormat >= 0) { + tmaFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( + static_cast(tmaFormat)); + if (!tmaFormatValue) + return {nullptr}; + } + return wrap(cute_nvgpu::CopyAtomNonExecTiledTmaStoreType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); } -MlirAttribute -mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetPayload(MlirType type) { - return wrap(static_cast( +MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetValType(MlirType type) { + return wrap(static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getValType())); +} +int mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} +int64_t +mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaFormat(MlirType type) { + auto value = + llvm::cast(unwrap(type)) + .getTmaFormat(); + return value ? static_cast(*value) : -1; } bool mlirTypeIsACuteNVGPUCopyAtomR2G(MlirType type) { return llvm::isa_and_nonnull(unwrap(type)); } -MlirType mlirCuteNVGPUCopyAtomR2GTypeGet(MlirContext context, - MlirAttribute payload) { +MlirType mlirCuteNVGPUCopyAtomR2GTypeGet(MlirContext context, MlirType valType, + int copyBits, uint32_t memOrder, + uint32_t memScope, + uint32_t l1CacheEvictPriority, + uint32_t storeCacheMode, + uint32_t sharedSpace) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap(cute_nvgpu::CopyAtomR2GType::get(ctx, payloadValue)); + auto memOrderValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); + if (!memOrderValue) + return {nullptr}; + auto memScopeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); + if (!memScopeValue) + return {nullptr}; + auto l1CacheEvictPriorityValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeCacheEvictionPriority( + l1CacheEvictPriority); + if (!l1CacheEvictPriorityValue) + return {nullptr}; + auto storeCacheModeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeStoreCacheMode( + storeCacheMode); + if (!storeCacheModeValue) + return {nullptr}; + auto sharedSpaceValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSharedSpace(sharedSpace); + if (!sharedSpaceValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomR2GType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits, *memOrderValue, *memScopeValue, + *l1CacheEvictPriorityValue, *storeCacheModeValue, *sharedSpaceValue)); } -MlirAttribute mlirCuteNVGPUCopyAtomR2GTypeGetPayload(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getPayload())); +MlirType mlirCuteNVGPUCopyAtomR2GTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValType())); +} +int mlirCuteNVGPUCopyAtomR2GTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)).getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomR2GTypeGetMemOrder(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMemOrder()); +} +uint32_t mlirCuteNVGPUCopyAtomR2GTypeGetMemScope(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMemScope()); +} +uint32_t mlirCuteNVGPUCopyAtomR2GTypeGetL1CacheEvictPriority(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getL1CacheEvictPriority()); +} +uint32_t mlirCuteNVGPUCopyAtomR2GTypeGetStoreCacheMode(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getStoreCacheMode()); +} +uint32_t mlirCuteNVGPUCopyAtomR2GTypeGetSharedSpace(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getSharedSpace()); } bool mlirTypeIsACuteNVGPUCopyAtomR2S(MlirType type) { return llvm::isa_and_nonnull(unwrap(type)); } -MlirType mlirCuteNVGPUCopyAtomR2STypeGet(MlirContext context, - MlirAttribute payload) { +MlirType mlirCuteNVGPUCopyAtomR2STypeGet(MlirContext context, MlirType valType, + int copyBits, uint32_t memOrder, + uint32_t memScope, + uint32_t sharedSpace) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap(cute_nvgpu::CopyAtomR2SType::get(ctx, payloadValue)); + auto memOrderValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); + if (!memOrderValue) + return {nullptr}; + auto memScopeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); + if (!memScopeValue) + return {nullptr}; + auto sharedSpaceValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSharedSpace(sharedSpace); + if (!sharedSpaceValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomR2SType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits, *memOrderValue, *memScopeValue, + *sharedSpaceValue)); } -MlirAttribute mlirCuteNVGPUCopyAtomR2STypeGetPayload(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getPayload())); +MlirType mlirCuteNVGPUCopyAtomR2STypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValType())); +} +int mlirCuteNVGPUCopyAtomR2STypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)).getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomR2STypeGetMemOrder(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMemOrder()); +} +uint32_t mlirCuteNVGPUCopyAtomR2STypeGetMemScope(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMemScope()); +} +uint32_t mlirCuteNVGPUCopyAtomR2STypeGetSharedSpace(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getSharedSpace()); } bool mlirTypeIsACuteNVGPUCopyAtomS2R(MlirType type) { return llvm::isa_and_nonnull(unwrap(type)); } -MlirType mlirCuteNVGPUCopyAtomS2RTypeGet(MlirContext context, - MlirAttribute payload) { +MlirType mlirCuteNVGPUCopyAtomS2RTypeGet(MlirContext context, MlirType valType, + int copyBits, uint32_t memOrder, + uint32_t memScope, + uint32_t sharedSpace) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap(cute_nvgpu::CopyAtomS2RType::get(ctx, payloadValue)); + auto memOrderValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); + if (!memOrderValue) + return {nullptr}; + auto memScopeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); + if (!memScopeValue) + return {nullptr}; + auto sharedSpaceValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSharedSpace(sharedSpace); + if (!sharedSpaceValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomS2RType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits, *memOrderValue, *memScopeValue, + *sharedSpaceValue)); } -MlirAttribute mlirCuteNVGPUCopyAtomS2RTypeGetPayload(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getPayload())); +MlirType mlirCuteNVGPUCopyAtomS2RTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValType())); +} +int mlirCuteNVGPUCopyAtomS2RTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)).getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomS2RTypeGetMemOrder(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMemOrder()); +} +uint32_t mlirCuteNVGPUCopyAtomS2RTypeGetMemScope(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMemScope()); +} +uint32_t mlirCuteNVGPUCopyAtomS2RTypeGetSharedSpace(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getSharedSpace()); } bool mlirTypeIsACuteNVGPUCopyAtomSIMTAsyncCopy(MlirType type) { @@ -392,69 +970,161 @@ bool mlirTypeIsACuteNVGPUCopyAtomSIMTAsyncCopy(MlirType type) { unwrap(type)); } MlirType mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGet(MlirContext context, - MlirAttribute payload) { + MlirType valType, + uint32_t cache, + int copyBits) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap(cute_nvgpu::CopyAtomSIMTAsyncCopyType::get(ctx, payloadValue)); + auto cacheValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLoadCacheMode(cache); + if (!cacheValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSIMTAsyncCopyType::get( + ctx, valTypeValue, *cacheValue, copyBits)); } -MlirAttribute mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetPayload(MlirType type) { - return wrap(static_cast( +MlirType mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetValType(MlirType type) { + return wrap(static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getValType())); +} +uint32_t mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCache(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getCache()); +} +int mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); } bool mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemLdReduce(MlirType type) { return llvm::isa_and_nonnull( unwrap(type)); } -MlirType -mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGet(MlirContext context, - MlirAttribute payload) { +MlirType mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t reduction, + MlirAttribute ldReduceAccPrecision, uint32_t memOrder, uint32_t memScope) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap( - cute_nvgpu::CopyAtomSIMTMultimemLdReduceType::get(ctx, payloadValue)); + auto reductionValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeReductionKind(reduction); + if (!reductionValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::LdReduceAccPrecisionKindAttr + ldReduceAccPrecisionValue; + if (!unwrapAs(ctx, ldReduceAccPrecision, true, ldReduceAccPrecisionValue)) + return {nullptr}; + auto memOrderValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); + if (!memOrderValue) + return {nullptr}; + auto memScopeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); + if (!memScopeValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSIMTMultimemLdReduceType::get( + ctx, valTypeValue, copyBits, *reductionValue, ldReduceAccPrecisionValue, + *memOrderValue, *memScopeValue)); +} +MlirType +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetReduction(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getReduction()); } MlirAttribute -mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetPayload(MlirType type) { +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetLdReduceAccPrecision( + MlirType type) { return wrap(static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getLdReduceAccPrecision())); +} +uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemOrder(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getMemOrder()); +} +uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemScope(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getMemScope()); } bool mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemRed(MlirType type) { return llvm::isa_and_nonnull( unwrap(type)); } -MlirType mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGet(MlirContext context, - MlirAttribute payload) { +MlirType mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t reduction, + uint32_t memOrder, uint32_t memScope) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap(cute_nvgpu::CopyAtomSIMTMultimemRedType::get(ctx, payloadValue)); + auto reductionValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeReductionKind(reduction); + if (!reductionValue) + return {nullptr}; + auto memOrderValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); + if (!memOrderValue) + return {nullptr}; + auto memScopeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); + if (!memScopeValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSIMTMultimemRedType::get( + ctx, valTypeValue, copyBits, *reductionValue, *memOrderValue, + *memScopeValue)); } -MlirAttribute -mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetPayload(MlirType type) { - return wrap(static_cast( +MlirType mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetValType(MlirType type) { + return wrap(static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getValType())); +} +int mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetReduction(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getReduction()); +} +uint32_t mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemOrder(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getMemOrder()); +} +uint32_t mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemScope(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getMemScope()); } bool mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemSt(MlirType type) { @@ -462,21 +1132,46 @@ bool mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemSt(MlirType type) { unwrap(type)); } MlirType mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGet(MlirContext context, - MlirAttribute payload) { + MlirType valType, + int copyBits, + uint32_t memOrder, + uint32_t memScope) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap(cute_nvgpu::CopyAtomSIMTMultimemStType::get(ctx, payloadValue)); + auto memOrderValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); + if (!memOrderValue) + return {nullptr}; + auto memScopeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); + if (!memScopeValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSIMTMultimemStType::get( + ctx, valTypeValue, copyBits, *memOrderValue, *memScopeValue)); } -MlirAttribute mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetPayload(MlirType type) { - return wrap(static_cast( +MlirType mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetValType(MlirType type) { + return wrap(static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getValType())); +} +int mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemOrder(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getMemOrder()); +} +uint32_t mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemScope(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getMemScope()); } bool mlirTypeIsACuteNVGPUCopyAtomSIMTSyncCopy(MlirType type) { @@ -484,44 +1179,47 @@ bool mlirTypeIsACuteNVGPUCopyAtomSIMTSyncCopy(MlirType type) { unwrap(type)); } MlirType mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGet(MlirContext context, - MlirAttribute payload) { + MlirType valType, + int copyBits, + uint32_t srcSpace, + uint32_t dstSpace) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap(cute_nvgpu::CopyAtomSIMTSyncCopyType::get(ctx, payloadValue)); + auto srcSpaceValue = + ::mlir::cutlass_compiler::cute::symbolizeAddressSpace(srcSpace); + if (!srcSpaceValue) + return {nullptr}; + auto dstSpaceValue = + ::mlir::cutlass_compiler::cute::symbolizeAddressSpace(dstSpace); + if (!dstSpaceValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSIMTSyncCopyType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits, *srcSpaceValue, *dstSpaceValue)); } -MlirAttribute mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetPayload(MlirType type) { - return wrap(static_cast( +MlirType mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetValType(MlirType type) { + return wrap(static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getValType())); } - -bool mlirTypeIsACuteNVGPUCopyAtomSM10xTmemLoadRed(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); +int mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); } -MlirType mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet(MlirContext context, - MlirAttribute payload) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomSM10xTmemLoadRedType::get(ctx, payloadValue)); +uint32_t mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetSrcSpace(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getSrcSpace()); } -MlirAttribute -mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetPayload(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getPayload())); +uint32_t mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetDstSpace(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getDstSpace()); } bool mlirTypeIsACuteNVGPUCopyAtomSM100CopyS2T(MlirType type) { @@ -529,21 +1227,46 @@ bool mlirTypeIsACuteNVGPUCopyAtomSM100CopyS2T(MlirType type) { unwrap(type)); } MlirType mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGet(MlirContext context, - MlirAttribute payload) { + MlirType valType, int numDp, + int numBit, int numCta, + uint32_t broadcast) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap(cute_nvgpu::CopyAtomSM100CopyS2TType::get(ctx, payloadValue)); + auto broadcastValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeCopyS2TBroadcast( + broadcast); + if (!broadcastValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSM100CopyS2TType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, numDp, numBit, numCta, *broadcastValue)); } -MlirAttribute mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetPayload(MlirType type) { - return wrap(static_cast( +MlirType mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetValType(MlirType type) { + return wrap(static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getValType())); +} +int mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumDp(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumDp(); +} +int mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumBit(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumBit(); +} +int mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumCta(); +} +uint32_t mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetBroadcast(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBroadcast()); } bool mlirTypeIsACuteNVGPUCopyAtomSM100S2TCopyV2(MlirType type) { @@ -551,21 +1274,56 @@ bool mlirTypeIsACuteNVGPUCopyAtomSM100S2TCopyV2(MlirType type) { unwrap(type)); } MlirType mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGet(MlirContext context, - MlirAttribute payload) { + MlirType valType, int numDp, + int numBit, int numCta, + uint32_t smemMajor, + uint32_t broadcast) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap(cute_nvgpu::CopyAtomSM100S2TCopyV2Type::get(ctx, payloadValue)); + auto smemMajorValue = + ::mlir::cutlass_compiler::cute::symbolizeMajorMode(smemMajor); + if (!smemMajorValue) + return {nullptr}; + auto broadcastValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeCopyS2TBroadcast( + broadcast); + if (!broadcastValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSM100S2TCopyV2Type::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, numDp, numBit, numCta, *smemMajorValue, *broadcastValue)); } -MlirAttribute mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetPayload(MlirType type) { - return wrap(static_cast( +MlirType mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetValType(MlirType type) { + return wrap(static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getValType())); +} +int mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumDp(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumDp(); +} +int mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumBit(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumBit(); +} +int mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumCta(); +} +uint32_t mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetSmemMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getSmemMajor()); +} +uint32_t mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetBroadcast(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBroadcast()); } bool mlirTypeIsACuteNVGPUCopyAtomSM100TmemLoad(MlirType type) { @@ -573,21 +1331,42 @@ bool mlirTypeIsACuteNVGPUCopyAtomSM100TmemLoad(MlirType type) { unwrap(type)); } MlirType mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGet(MlirContext context, - MlirAttribute payload) { + MlirType valType, int numDp, + int numBit, unsigned numRep, + bool pack16b) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap(cute_nvgpu::CopyAtomSM100TmemLoadType::get(ctx, payloadValue)); + return wrap(cute_nvgpu::CopyAtomSM100TmemLoadType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, numDp, numBit, numRep, + pack16b ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr())); } -MlirAttribute mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetPayload(MlirType type) { - return wrap(static_cast( +MlirType mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetValType(MlirType type) { + return wrap(static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getValType())); +} +int mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumDp(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumDp(); +} +int mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumBit(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumBit(); +} +unsigned mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumRep(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumRep(); +} +bool mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetPack16b(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getPack16b()); } bool mlirTypeIsACuteNVGPUCopyAtomSM100TmemStore(MlirType type) { @@ -595,321 +1374,1760 @@ bool mlirTypeIsACuteNVGPUCopyAtomSM100TmemStore(MlirType type) { unwrap(type)); } MlirType mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGet(MlirContext context, - MlirAttribute payload) { + MlirType valType, int numDp, + int numBit, unsigned numRep, + bool expand16b) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap(cute_nvgpu::CopyAtomSM100TmemStoreType::get(ctx, payloadValue)); + return wrap(cute_nvgpu::CopyAtomSM100TmemStoreType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, numDp, numBit, numRep, + expand16b ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr())); } -MlirAttribute mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetPayload(MlirType type) { - return wrap(static_cast( +MlirType mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetValType(MlirType type) { + return wrap(static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getValType())); +} +int mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumDp(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumDp(); +} +int mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumBit(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumBit(); +} +unsigned mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumRep(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumRep(); +} +bool mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetExpand16b(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getExpand16b()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSM107TmemLoadSPCompress(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGet( + MlirContext context, MlirType valType, int numDp, int numBit, + unsigned numRep, uint32_t redOp, bool red, bool nan) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto redOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemLoadRedOp(redOp); + if (!redOpValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, numDp, numBit, numRep, *redOpValue, + red ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr(), + nan ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr())); +} +MlirType +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumDp(MlirType type) { + return llvm::cast( + unwrap(type)) + .getNumDp(); +} +int mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumBit(MlirType type) { + return llvm::cast( + unwrap(type)) + .getNumBit(); +} +unsigned +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumRep(MlirType type) { + return llvm::cast( + unwrap(type)) + .getNumRep(); +} +uint32_t +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRedOp(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getRedOp()); +} +bool mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRed(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getRed()); +} +bool mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNan(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getNan()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSM10xTmemLoadRed(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet( + MlirContext context, MlirType valType, int numDp, int numBit, + unsigned numRep, uint32_t redOp, bool nan, MlirAttribute halfSplitOff) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto redOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemLoadRedOp(redOp); + if (!redOpValue) + return {nullptr}; + ::mlir::IntegerAttr halfSplitOffValue; + if (!unwrapAs(ctx, halfSplitOff, true, halfSplitOffValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSM10xTmemLoadRedType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, numDp, numBit, numRep, *redOpValue, + nan ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr(), halfSplitOffValue)); +} +MlirType mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumDp(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumDp(); +} +int mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumBit(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumBit(); +} +unsigned mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumRep(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumRep(); +} +uint32_t mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetRedOp(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getRedOp()); +} +bool mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNan(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getNan()); +} +MlirAttribute +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetHalfSplitOff(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getHalfSplitOff())); } bool mlirTypeIsACuteNVGPUCopyAtomStsm(MlirType type) { return llvm::isa_and_nonnull(unwrap(type)); } -MlirType mlirCuteNVGPUCopyAtomStsmTypeGet(MlirContext context, - MlirAttribute payload) { +MlirType mlirCuteNVGPUCopyAtomStsmTypeGet(MlirContext context, MlirType valType, + MlirAttribute mode, int numMatrices, + bool transpose) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap(cute_nvgpu::CopyAtomStsmType::get(ctx, payloadValue)); + ::mlir::cutlass_compiler::cute::ShapeAttr modeValue; + if (!unwrapAs(ctx, mode, false, modeValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomStsmType::get( + ctx, valTypeValue, modeValue, numMatrices, + transpose ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr())); } -MlirAttribute mlirCuteNVGPUCopyAtomStsmTypeGetPayload(MlirType type) { +MlirType mlirCuteNVGPUCopyAtomStsmTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValType())); +} +MlirAttribute mlirCuteNVGPUCopyAtomStsmTypeGetMode(MlirType type) { return wrap(static_cast( - llvm::cast(unwrap(type)).getPayload())); + llvm::cast(unwrap(type)).getMode())); +} +int mlirCuteNVGPUCopyAtomStsmTypeGetNumMatrices(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumMatrices(); +} +bool mlirCuteNVGPUCopyAtomStsmTypeGetTranspose(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getTranspose()); } bool mlirTypeIsACuteNVGPUCopyAtomTmaLoad(MlirType type) { return llvm::isa_and_nonnull(unwrap(type)); } -MlirType mlirCuteNVGPUCopyAtomTmaLoadTypeGet(MlirContext context, - MlirAttribute payload) { +MlirType mlirCuteNVGPUCopyAtomTmaLoadTypeGet( + MlirContext context, MlirType valType, int sparsity, int copyBits, + uint32_t mode, int numCta, MlirType gStride, bool mcast, MlirType tmaGbasis, + bool override, bool noFullyOobTile) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap(cute_nvgpu::CopyAtomTmaLoadType::get(ctx, payloadValue)); + auto modeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaLoadMode(mode); + if (!modeValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute::StrideType gStrideValue; + if (!unwrapAs(ctx, gStride, false, gStrideValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, true, tmaGbasisValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomTmaLoadType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, sparsity, copyBits, *modeValue, numCta, gStrideValue, mcast, + tmaGbasisValue, override, noFullyOobTile)); } -MlirAttribute mlirCuteNVGPUCopyAtomTmaLoadTypeGetPayload(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getPayload())); +MlirType mlirCuteNVGPUCopyAtomTmaLoadTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValType())); +} +int mlirCuteNVGPUCopyAtomTmaLoadTypeGetSparsity(MlirType type) { + return llvm::cast(unwrap(type)) + .getSparsity(); +} +int mlirCuteNVGPUCopyAtomTmaLoadTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomTmaLoadTypeGetMode(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMode()); +} +int mlirCuteNVGPUCopyAtomTmaLoadTypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)).getNumCta(); +} +MlirType mlirCuteNVGPUCopyAtomTmaLoadTypeGetGStride(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getGStride())); +} +bool mlirCuteNVGPUCopyAtomTmaLoadTypeGetMcast(MlirType type) { + return llvm::cast(unwrap(type)).getMcast(); +} +MlirType mlirCuteNVGPUCopyAtomTmaLoadTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} +bool mlirCuteNVGPUCopyAtomTmaLoadTypeGetOverride(MlirType type) { + return llvm::cast(unwrap(type)) + .getOverride(); +} +bool mlirCuteNVGPUCopyAtomTmaLoadTypeGetNoFullyOobTile(MlirType type) { + return llvm::cast(unwrap(type)) + .getNoFullyOobTile(); } bool mlirTypeIsACuteNVGPUCopyAtomTmaReduce(MlirType type) { return llvm::isa_and_nonnull(unwrap(type)); } MlirType mlirCuteNVGPUCopyAtomTmaReduceTypeGet(MlirContext context, - MlirAttribute payload) { + MlirType valType, int copyBits, + uint32_t mode, uint32_t kind, + MlirType gStride, + MlirType tmaGbasis) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap(cute_nvgpu::CopyAtomTmaReduceType::get(ctx, payloadValue)); + auto modeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaStoreMode(mode); + if (!modeValue) + return {nullptr}; + auto kindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeReductionKind(kind); + if (!kindValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute::StrideType gStrideValue; + if (!unwrapAs(ctx, gStride, false, gStrideValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, true, tmaGbasisValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomTmaReduceType::get( + ctx, valTypeValue, copyBits, *modeValue, *kindValue, gStrideValue, + tmaGbasisValue)); } -MlirAttribute mlirCuteNVGPUCopyAtomTmaReduceTypeGetPayload(MlirType type) { - return wrap(static_cast( +MlirType mlirCuteNVGPUCopyAtomTmaReduceTypeGetValType(MlirType type) { + return wrap(static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getValType())); +} +int mlirCuteNVGPUCopyAtomTmaReduceTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomTmaReduceTypeGetMode(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMode()); +} +uint32_t mlirCuteNVGPUCopyAtomTmaReduceTypeGetKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getKind()); +} +MlirType mlirCuteNVGPUCopyAtomTmaReduceTypeGetGStride(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getGStride())); +} +MlirType mlirCuteNVGPUCopyAtomTmaReduceTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); } bool mlirTypeIsACuteNVGPUCopyAtomTmaStore(MlirType type) { return llvm::isa_and_nonnull(unwrap(type)); } MlirType mlirCuteNVGPUCopyAtomTmaStoreTypeGet(MlirContext context, - MlirAttribute payload) { + MlirType valType, int sparsity, + int copyBits, uint32_t mode, + MlirType gStride, + MlirType tmaGbasis, bool override, + bool noFullyOobTile) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) return {nullptr}; - return wrap(cute_nvgpu::CopyAtomTmaStoreType::get(ctx, payloadValue)); -} -MlirAttribute mlirCuteNVGPUCopyAtomTmaStoreTypeGetPayload(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getPayload())); -} - -bool mlirTypeIsACuteNVGPUMmaAtomSM80Sparse(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteNVGPUMmaAtomSM80SparseTypeGet(MlirContext context, - MlirAttribute payload) { - auto *ctx = unwrap(context); - if (!ctx) + auto modeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaStoreMode(mode); + if (!modeValue) return {nullptr}; - ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + ::mlir::cutlass_compiler::cute::StrideType gStrideValue; + if (!unwrapAs(ctx, gStride, false, gStrideValue)) return {nullptr}; - return wrap(cute_nvgpu::MmaAtomSM80SparseType::get(ctx, payloadValue)); -} -MlirAttribute mlirCuteNVGPUMmaAtomSM80SparseTypeGetPayload(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getPayload())); -} - -bool mlirTypeIsACuteNVGPUMmaAtomSM80(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteNVGPUMmaAtomSM80TypeGet(MlirContext context, - MlirAttribute payload) { - auto *ctx = unwrap(context); - if (!ctx) + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, true, tmaGbasisValue)) return {nullptr}; - ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) - return {nullptr}; - return wrap(cute_nvgpu::MmaAtomSM80Type::get(ctx, payloadValue)); + return wrap(cute_nvgpu::CopyAtomTmaStoreType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, sparsity, copyBits, *modeValue, gStrideValue, + tmaGbasisValue, override, noFullyOobTile)); } -MlirAttribute mlirCuteNVGPUMmaAtomSM80TypeGetPayload(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getPayload())); +MlirType mlirCuteNVGPUCopyAtomTmaStoreTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValType())); } - -bool mlirTypeIsACuteNVGPUMmaAtomSM89(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); +int mlirCuteNVGPUCopyAtomTmaStoreTypeGetSparsity(MlirType type) { + return llvm::cast(unwrap(type)) + .getSparsity(); } -MlirType mlirCuteNVGPUMmaAtomSM89TypeGet(MlirContext context, - MlirAttribute payload) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) - return {nullptr}; - return wrap(cute_nvgpu::MmaAtomSM89Type::get(ctx, payloadValue)); +int mlirCuteNVGPUCopyAtomTmaStoreTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); } -MlirAttribute mlirCuteNVGPUMmaAtomSM89TypeGetPayload(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getPayload())); +uint32_t mlirCuteNVGPUCopyAtomTmaStoreTypeGetMode(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMode()); } - -bool mlirTypeIsACuteNVGPUMmaAtomSM90(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); +MlirType mlirCuteNVGPUCopyAtomTmaStoreTypeGetGStride(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getGStride())); } -MlirType mlirCuteNVGPUMmaAtomSM90TypeGet(MlirContext context, - MlirAttribute payload) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) - return {nullptr}; - return wrap(cute_nvgpu::MmaAtomSM90Type::get(ctx, payloadValue)); +MlirType mlirCuteNVGPUCopyAtomTmaStoreTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); } -MlirAttribute mlirCuteNVGPUMmaAtomSM90TypeGetPayload(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getPayload())); +bool mlirCuteNVGPUCopyAtomTmaStoreTypeGetOverride(MlirType type) { + return llvm::cast(unwrap(type)) + .getOverride(); +} +bool mlirCuteNVGPUCopyAtomTmaStoreTypeGetNoFullyOobTile(MlirType type) { + return llvm::cast(unwrap(type)) + .getNoFullyOobTile(); } bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaledSparse(MlirType type) { return llvm::isa_and_nonnull< cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType>(unwrap(type)); } -MlirType -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGet(MlirContext context, - MlirAttribute payload) { +MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType sfType, uint32_t sparseMetadataFormat, uint32_t aFragKind, + int vecSize, uint32_t archPromote, MlirAttribute intOverflow) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) return {nullptr}; - return wrap(cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType::get( - ctx, payloadValue)); + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + mlir::Type sfTypeValue; + if (!unwrapAs(ctx, sfType, false, sfTypeValue)) + return {nullptr}; + auto sparseMetadataFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSparseMetadataFormat( + sparseMetadataFormat); + if (!sparseMetadataFormatValue) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + auto archPromoteValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeArch(archPromote); + if (!archPromoteValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, sfTypeValue, *sparseMetadataFormatValue, *aFragKindValue, + vecSize, *archPromoteValue, intOverflowValue)); } MlirAttribute -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetPayload(MlirType type) { +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetShapeMnk(MlirType type) { return wrap(static_cast( llvm::cast( unwrap(type)) - .getPayload())); + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetNumCta(MlirType type) { + return llvm::cast( + unwrap(type)) + .getNumCta(); +} +uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getAMajor()); +} +uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getBMajor()); +} +MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getAType())); +} +MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getBType())); +} +MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getCType())); +} +MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSfType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getSfType())); +} +uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSparseMetadataFormat( + MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getSparseMetadataFormat()); +} +uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getAFragKind()); +} +int mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetVecSize( + MlirType type) { + return llvm::cast( + unwrap(type)) + .getVecSize(); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetArchPromote( + MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getArchPromote()); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetIntOverflow( + MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getIntOverflow())); } bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaled(MlirType type) { return llvm::isa_and_nonnull( unwrap(type)); } -MlirType -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGet(MlirContext context, - MlirAttribute payload) { +MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType sfType, uint32_t aFragKind, int vecSize, uint32_t archPromote, + MlirAttribute intOverflow) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) return {nullptr}; - return wrap( - cute_nvgpu::MmaAtomSM100UMMABlockScaledType::get(ctx, payloadValue)); + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + mlir::Type sfTypeValue; + if (!unwrapAs(ctx, sfType, false, sfTypeValue)) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + auto archPromoteValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeArch(archPromote); + if (!archPromoteValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM100UMMABlockScaledType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, sfTypeValue, *aFragKindValue, vecSize, *archPromoteValue, + intOverflowValue)); } MlirAttribute -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetPayload(MlirType type) { +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetShapeMnk(MlirType type) { return wrap(static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumCta(); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAMajor()); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBMajor()); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getCType())); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetSfType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getSfType())); +} +uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAFragKind()); +} +int mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetVecSize(MlirType type) { + return llvm::cast(unwrap(type)) + .getVecSize(); +} +uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetArchPromote(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getArchPromote()); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getIntOverflow())); } bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMASparse(MlirType type) { return llvm::isa_and_nonnull( unwrap(type)); } -MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGet(MlirContext context, - MlirAttribute payload) { +MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType eType, uint32_t sparseMetadataFormat, uint32_t aFragKind, + int cScaleExp, MlirAttribute intOverflow) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) return {nullptr}; - return wrap(cute_nvgpu::MmaAtomSM100UMMASparseType::get(ctx, payloadValue)); + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + mlir::Type eTypeValue; + if (!unwrapAs(ctx, eType, false, eTypeValue)) + return {nullptr}; + auto sparseMetadataFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSparseMetadataFormat( + sparseMetadataFormat); + if (!sparseMetadataFormatValue) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM100UMMASparseType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, eTypeValue, *sparseMetadataFormatValue, *aFragKindValue, + cScaleExp, intOverflowValue)); } -MlirAttribute mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetPayload(MlirType type) { +MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetShapeMnk(MlirType type) { return wrap(static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumCta(); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAMajor()); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBMajor()); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getCType())); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetEType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getEType())); +} +uint32_t +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetSparseMetadataFormat(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getSparseMetadataFormat()); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAFragKind()); +} +int mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCScaleExp(MlirType type) { + return llvm::cast(unwrap(type)) + .getCScaleExp(); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getIntOverflow())); } bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMA(MlirType type) { return llvm::isa_and_nonnull(unwrap(type)); } -MlirType mlirCuteNVGPUMmaAtomSM100UMMATypeGet(MlirContext context, - MlirAttribute payload) { +MlirType mlirCuteNVGPUMmaAtomSM100UMMATypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + uint32_t aFragKind, int cScaleExp, MlirAttribute intOverflow) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) return {nullptr}; - return wrap(cute_nvgpu::MmaAtomSM100UMMAType::get(ctx, payloadValue)); + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM100UMMAType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, *aFragKindValue, cScaleExp, intOverflowValue)); } -MlirAttribute mlirCuteNVGPUMmaAtomSM100UMMATypeGetPayload(MlirType type) { +MlirAttribute mlirCuteNVGPUMmaAtomSM100UMMATypeGetShapeMnk(MlirType type) { return wrap(static_cast( - llvm::cast(unwrap(type)).getPayload())); + llvm::cast(unwrap(type)) + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM100UMMATypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)).getNumCta(); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMATypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getAMajor()); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMATypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getBMajor()); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMATypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMATypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMATypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCType())); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMATypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAFragKind()); +} +int mlirCuteNVGPUMmaAtomSM100UMMATypeGetCScaleExp(MlirType type) { + return llvm::cast(unwrap(type)) + .getCScaleExp(); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM100UMMATypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getIntOverflow())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaledSparse(MlirType type) { + return llvm::isa_and_nonnull< + cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType>(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType sfType, uint32_t sparseMetadataFormat, uint32_t aFragKind, + int vecSize, uint32_t aCollectorOp, uint32_t bCollectorOp, + MlirAttribute intOverflow) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + mlir::Type sfTypeValue; + if (!unwrapAs(ctx, sfType, false, sfTypeValue)) + return {nullptr}; + auto sparseMetadataFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSparseMetadataFormat( + sparseMetadataFormat); + if (!sparseMetadataFormatValue) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + auto aCollectorOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( + aCollectorOp); + if (!aCollectorOpValue) + return {nullptr}; + auto bCollectorOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( + bCollectorOp); + if (!bCollectorOpValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, sfTypeValue, *sparseMetadataFormatValue, *aFragKindValue, + vecSize, *aCollectorOpValue, *bCollectorOpValue, intOverflowValue)); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetNumCta(MlirType type) { + return llvm::cast( + unwrap(type)) + .getNumCta(); +} +uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getAMajor()); +} +uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getBMajor()); +} +MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getAType())); +} +MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getBType())); +} +MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getCType())); +} +MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSfType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getSfType())); +} +uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSparseMetadataFormat( + MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getSparseMetadataFormat()); +} +uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getAFragKind()); +} +int mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetVecSize( + MlirType type) { + return llvm::cast( + unwrap(type)) + .getVecSize(); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetACollectorOp( + MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getACollectorOp()); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBCollectorOp( + MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getBCollectorOp()); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetIntOverflow( + MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getIntOverflow())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaled(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType sfType, uint32_t aFragKind, int vecSize, uint32_t aCollectorOp, + uint32_t bCollectorOp, MlirAttribute intOverflow) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + mlir::Type sfTypeValue; + if (!unwrapAs(ctx, sfType, false, sfTypeValue)) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + auto aCollectorOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( + aCollectorOp); + if (!aCollectorOpValue) + return {nullptr}; + auto bCollectorOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( + bCollectorOp); + if (!bCollectorOpValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM107UMMABlockScaledType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, sfTypeValue, *aFragKindValue, vecSize, *aCollectorOpValue, + *bCollectorOpValue, intOverflowValue)); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumCta(); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAMajor()); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBMajor()); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getCType())); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetSfType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getSfType())); +} +uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAFragKind()); +} +int mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetVecSize(MlirType type) { + return llvm::cast(unwrap(type)) + .getVecSize(); +} +uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetACollectorOp(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getACollectorOp()); +} +uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBCollectorOp(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBCollectorOp()); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getIntOverflow())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM107UMMASparse(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType eType, uint32_t sparseMetadataFormat, uint32_t aFragKind, + int cScaleExp, uint32_t aCollectorOp, uint32_t bCollectorOp, + MlirAttribute intOverflow) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + mlir::Type eTypeValue; + if (!unwrapAs(ctx, eType, false, eTypeValue)) + return {nullptr}; + auto sparseMetadataFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSparseMetadataFormat( + sparseMetadataFormat); + if (!sparseMetadataFormatValue) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + auto aCollectorOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( + aCollectorOp); + if (!aCollectorOpValue) + return {nullptr}; + auto bCollectorOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( + bCollectorOp); + if (!bCollectorOpValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM107UMMASparseType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, eTypeValue, *sparseMetadataFormatValue, *aFragKindValue, + cScaleExp, *aCollectorOpValue, *bCollectorOpValue, intOverflowValue)); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumCta(); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAMajor()); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBMajor()); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getCType())); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetEType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getEType())); +} +uint32_t +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetSparseMetadataFormat(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getSparseMetadataFormat()); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAFragKind()); +} +int mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCScaleExp(MlirType type) { + return llvm::cast(unwrap(type)) + .getCScaleExp(); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetACollectorOp(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getACollectorOp()); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBCollectorOp(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBCollectorOp()); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getIntOverflow())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM107UMMA(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMATypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + uint32_t aFragKind, int cScaleExp, uint32_t aCollectorOp, + uint32_t bCollectorOp, MlirAttribute intOverflow) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + auto aCollectorOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( + aCollectorOp); + if (!aCollectorOpValue) + return {nullptr}; + auto bCollectorOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( + bCollectorOp); + if (!bCollectorOpValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM107UMMAType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, *aFragKindValue, cScaleExp, *aCollectorOpValue, + *bCollectorOpValue, intOverflowValue)); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM107UMMATypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM107UMMATypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)).getNumCta(); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMATypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getAMajor()); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMATypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getBMajor()); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMATypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMATypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMATypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCType())); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMATypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAFragKind()); +} +int mlirCuteNVGPUMmaAtomSM107UMMATypeGetCScaleExp(MlirType type) { + return llvm::cast(unwrap(type)) + .getCScaleExp(); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMATypeGetACollectorOp(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getACollectorOp()); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMATypeGetBCollectorOp(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBCollectorOp()); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM107UMMATypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getIntOverflow())); } bool mlirTypeIsACuteNVGPUMmaAtomSM120BlockScaled(MlirType type) { return llvm::isa_and_nonnull( unwrap(type)); } -MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGet(MlirContext context, - MlirAttribute payload) { +MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGet( + MlirContext context, MlirAttribute shapeMnk, int vecSize, MlirType aType, + MlirType bType, MlirType cType, MlirType sfType, bool useSfLayoutTV) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) return {nullptr}; - return wrap(cute_nvgpu::MmaAtomSM120BlockScaledType::get(ctx, payloadValue)); + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + mlir::Type sfTypeValue; + if (!unwrapAs(ctx, sfType, false, sfTypeValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM120BlockScaledType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, vecSize, aTypeValue, bTypeValue, cTypeValue, sfTypeValue, + useSfLayoutTV)); } MlirAttribute -mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetPayload(MlirType type) { +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetShapeMnk(MlirType type) { return wrap(static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetVecSize(MlirType type) { + return llvm::cast(unwrap(type)) + .getVecSize(); +} +MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getCType())); +} +MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetSfType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getSfType())); +} +bool mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetUseSfLayoutTV(MlirType type) { + return llvm::cast(unwrap(type)) + .getUseSfLayout_TV(); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM80Sparse(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM80SparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, MlirType aType, MlirType bType, + MlirType cType, uint32_t sparseMetadataFormat, MlirAttribute intOverflow) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + auto sparseMetadataFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSparseMetadataFormat( + sparseMetadataFormat); + if (!sparseMetadataFormatValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM80SparseType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, aTypeValue, bTypeValue, cTypeValue, + *sparseMetadataFormatValue, intOverflowValue)); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM80SparseTypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getShapeMnk())); +} +MlirType mlirCuteNVGPUMmaAtomSM80SparseTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM80SparseTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM80SparseTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCType())); +} +uint32_t +mlirCuteNVGPUMmaAtomSM80SparseTypeGetSparseMetadataFormat(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getSparseMetadataFormat()); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM80SparseTypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getIntOverflow())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM80(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM80TypeGet(MlirContext context, + MlirAttribute shapeMnk, MlirType aType, + MlirType bType, MlirType cType, + MlirAttribute intOverflow, + MlirAttribute binaryOp) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::BinaryOpAttr binaryOpValue; + if (!unwrapAs(ctx, binaryOp, true, binaryOpValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM80Type::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, aTypeValue, bTypeValue, cTypeValue, intOverflowValue, + binaryOpValue)); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM80TypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getShapeMnk())); +} +MlirType mlirCuteNVGPUMmaAtomSM80TypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM80TypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM80TypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCType())); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM80TypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getIntOverflow())); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM80TypeGetBinaryOp(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBinaryOp())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM89(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM89TypeGet(MlirContext context, + MlirAttribute shapeMnk, MlirType aType, + MlirType bType, MlirType cType) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM89Type::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, aTypeValue, bTypeValue, cTypeValue)); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM89TypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getShapeMnk())); +} +MlirType mlirCuteNVGPUMmaAtomSM89TypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM89TypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM89TypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCType())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM90(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM90TypeGet( + MlirContext context, MlirAttribute shapeMnk, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + uint32_t aFragKind, MlirAttribute intOverflow, bool aNeg, bool bNeg) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM90Type::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, *aFragKindValue, intOverflowValue, + aNeg ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr(), + bNeg ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr())); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM90TypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getShapeMnk())); +} +uint32_t mlirCuteNVGPUMmaAtomSM90TypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getAMajor()); +} +uint32_t mlirCuteNVGPUMmaAtomSM90TypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getBMajor()); +} +MlirType mlirCuteNVGPUMmaAtomSM90TypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM90TypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM90TypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCType())); +} +uint32_t mlirCuteNVGPUMmaAtomSM90TypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getAFragKind()); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM90TypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getIntOverflow())); +} +bool mlirCuteNVGPUMmaAtomSM90TypeGetANeg(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getANeg()); +} +bool mlirCuteNVGPUMmaAtomSM90TypeGetBNeg(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getBNeg()); } bool mlirTypeIsACuteNVGPUSmemDescCircularSM103(MlirType type) { return llvm::isa_and_nonnull( unwrap(type)); } -MlirType mlirCuteNVGPUSmemDescCircularSM103TypeGet(MlirContext context, - MlirAttribute payload) { +MlirType +mlirCuteNVGPUSmemDescCircularSM103TypeGet(MlirContext context, + MlirAttribute blockLayoutBytes) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + ::mlir::cutlass_compiler::cute::LayoutAttr blockLayoutBytesValue; + if (!unwrapAs(ctx, blockLayoutBytes, false, blockLayoutBytesValue)) return {nullptr}; - return wrap(cute_nvgpu::SmemDescCircularSM103Type::get(ctx, payloadValue)); + return wrap(cute_nvgpu::SmemDescCircularSM103Type::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + blockLayoutBytesValue)); } -MlirAttribute mlirCuteNVGPUSmemDescCircularSM103TypeGetPayload(MlirType type) { +MlirAttribute +mlirCuteNVGPUSmemDescCircularSM103TypeGetBlockLayoutBytes(MlirType type) { return wrap(static_cast( llvm::cast(unwrap(type)) - .getPayload())); + .getBlockLayoutBytes())); +} + +bool mlirTypeIsACuteNVGPUSmemDescCircularSM107(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType +mlirCuteNVGPUSmemDescCircularSM107TypeGet(MlirContext context, + MlirAttribute blockLayoutBytes) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::LayoutAttr blockLayoutBytesValue; + if (!unwrapAs(ctx, blockLayoutBytes, false, blockLayoutBytesValue)) + return {nullptr}; + return wrap(cute_nvgpu::SmemDescCircularSM107Type::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + blockLayoutBytesValue)); +} +MlirAttribute +mlirCuteNVGPUSmemDescCircularSM107TypeGetBlockLayoutBytes(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getBlockLayoutBytes())); +} + +bool mlirTypeIsACuteNVGPUSmemDescSM107(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUSmemDescSM107TypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute_nvgpu::SmemDescSM107Type::get(ctx)); } bool mlirTypeIsACuteNVGPUSmemDesc(MlirType type) { @@ -926,61 +3144,29 @@ MlirType mlirCuteNVGPUSmemDescTypeGet(MlirContext context) { bool mlirTypeIsACuteNVGPUSmemDescView(MlirType type) { return llvm::isa_and_nonnull(unwrap(type)); } -MlirType mlirCuteNVGPUSmemDescViewTypeGet(MlirContext context, - MlirAttribute payload) { +MlirType mlirCuteNVGPUSmemDescViewTypeGet(MlirContext context, MlirType desc, + MlirAttribute layout) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + mlir::Type descValue; + if (!unwrapAs(ctx, desc, false, descValue)) return {nullptr}; - return wrap(cute_nvgpu::SmemDescViewType::get(ctx, payloadValue)); + ::mlir::cutlass_compiler::cute::LayoutAttr layoutValue; + if (!unwrapAs(ctx, layout, false, layoutValue)) + return {nullptr}; + return wrap(cute_nvgpu::SmemDescViewType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + descValue, layoutValue)); } -MlirAttribute mlirCuteNVGPUSmemDescViewTypeGetPayload(MlirType type) { +MlirType mlirCuteNVGPUSmemDescViewTypeGetDesc(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getDesc())); +} +MlirAttribute mlirCuteNVGPUSmemDescViewTypeGetLayout(MlirType type) { return wrap(static_cast( - llvm::cast(unwrap(type)).getPayload())); -} - -bool mlirTypeIsACuteNVGPUTiledCopy(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteNVGPUTiledCopyTypeGet(MlirContext context, - MlirAttribute payload) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) - return {nullptr}; - return wrap(cute_nvgpu::TiledCopyType::get(ctx, payloadValue)); -} -MlirAttribute mlirCuteNVGPUTiledCopyTypeGetPayload(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getPayload())); -} - -bool mlirTypeIsACuteNVGPUTiledMma(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteNVGPUTiledMmaTypeGet(MlirContext context, - MlirAttribute payload) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) - return {nullptr}; - return wrap(cute_nvgpu::TiledMmaType::get(ctx, payloadValue)); -} -MlirAttribute mlirCuteNVGPUTiledMmaTypeGetPayload(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getPayload())); + llvm::cast(unwrap(type)).getLayout())); } bool mlirTypeIsACuteNVGPUTmaDescriptorIm2Col(MlirType type) { @@ -1011,20 +3197,45 @@ bool mlirTypeIsACuteNVGPUUniversalFmaAtom(MlirType type) { return llvm::isa_and_nonnull(unwrap(type)); } MlirType mlirCuteNVGPUUniversalFmaAtomTypeGet(MlirContext context, - MlirAttribute payload) { + MlirAttribute shapeMnk, + MlirType aType, MlirType bType, + MlirType cType) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto payloadValue = - llvm::dyn_cast_if_present(unwrap(payload)); - if (!payloadValue || payloadValue.getContext() != ctx) + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) return {nullptr}; - return wrap(cute_nvgpu::UniversalFmaAtomType::get(ctx, payloadValue)); + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + return wrap(cute_nvgpu::UniversalFmaAtomType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, aTypeValue, bTypeValue, cTypeValue)); } -MlirAttribute mlirCuteNVGPUUniversalFmaAtomTypeGetPayload(MlirType type) { +MlirAttribute mlirCuteNVGPUUniversalFmaAtomTypeGetShapeMnk(MlirType type) { return wrap(static_cast( - llvm::cast(unwrap(type)).getPayload())); + llvm::cast(unwrap(type)) + .getShapeMnk())); +} +MlirType mlirCuteNVGPUUniversalFmaAtomTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAType())); +} +MlirType mlirCuteNVGPUUniversalFmaAtomTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBType())); +} +MlirType mlirCuteNVGPUUniversalFmaAtomTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCType())); } bool mlirTypeIsACuteNVGPUWorkIdResponse(MlirType type) { diff --git a/mlir/dialects/cute_ir/lib/CAPI/CuteTypes.cpp b/mlir/dialects/cute_ir/lib/CAPI/CuteTypes.cpp index 5172112a..986cbb09 100644 --- a/mlir/dialects/cute_ir/lib/CAPI/CuteTypes.cpp +++ b/mlir/dialects/cute_ir/lib/CAPI/CuteTypes.cpp @@ -1,10 +1,26 @@ -// C++ to C bindings for CuTe types. +// C API for the CuTe types, generated from the dialect's .td files. #include "cute_ir-c/Dialect/CuteTypes.h" -#include "cute_ir/Dialect/Cute/IR/CuteDialectPrivate.h" + +#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" #include "mlir/CAPI/IR.h" namespace cute = mlir::cutlass_compiler::cute; +namespace { +// Unwraps `handle` into `out`: false when it is of another context or not a +// T, or when it is null and not optional. +template +bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { + auto value = unwrap(handle); + if (!value) + return optional; + if (value.getContext() != ctx) + return false; + out = llvm::dyn_cast(value); + return static_cast(out); +} +} // namespace + bool mlirTypeIsACuteArithTupleIterator(MlirType type) { return llvm::isa_and_nonnull(unwrap(type)); } @@ -14,9 +30,8 @@ MlirType mlirCuteArithTupleIteratorTypeGet(MlirContext context, if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto arithTupleValue = - llvm::dyn_cast_if_present(unwrap(arithTuple)); - if (!arithTupleValue || arithTupleValue.getContext() != ctx) + ::mlir::cutlass_compiler::cute::IntTupleType arithTupleValue; + if (!unwrapAs(ctx, arithTuple, false, arithTupleValue)) return {nullptr}; return wrap(cute::ArithTupleIteratorType::get(ctx, arithTupleValue)); } @@ -34,9 +49,8 @@ MlirType mlirCuteComposedLayoutTypeGet(MlirContext context, if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto attrValue = - llvm::dyn_cast_if_present(unwrap(attr)); - if (!attrValue || attrValue.getContext() != ctx) + ::mlir::cutlass_compiler::cute::ComposedLayoutAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) return {nullptr}; return wrap(cute::ComposedLayoutType::getChecked( [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, @@ -47,38 +61,26 @@ MlirAttribute mlirCuteComposedLayoutTypeGetAttr(MlirType type) { llvm::cast(unwrap(type)).getAttr())); } -bool mlirTypeIsACuteConstrainedInt32(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); +bool mlirTypeIsACuteConstrainedInt(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); } -MlirType mlirCuteConstrainedInt32TypeGet(MlirContext context, - uint64_t divisibleBy) { +MlirType mlirCuteConstrainedIntTypeGet(MlirContext context, + int64_t divisibility, unsigned width, + bool isPow2) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - return wrap(cute::ConstrainedInt32Type::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - divisibleBy)); + return wrap(cute::ConstrainedIntType::get(ctx, divisibility, width, isPow2)); } -uint64_t mlirCuteConstrainedInt32TypeGetDivisibleBy(MlirType type) { - return llvm::cast(unwrap(type)).getDivisibleBy(); +int64_t mlirCuteConstrainedIntTypeGetDivisibility(MlirType type) { + return llvm::cast(unwrap(type)).getDivisibility(); } - -bool mlirTypeIsACuteConstrainedInt64(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); +unsigned mlirCuteConstrainedIntTypeGetWidth(MlirType type) { + return llvm::cast(unwrap(type)).getWidth(); } -MlirType mlirCuteConstrainedInt64TypeGet(MlirContext context, - uint64_t divisibleBy) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cute::ConstrainedInt64Type::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - divisibleBy)); -} -uint64_t mlirCuteConstrainedInt64TypeGetDivisibleBy(MlirType type) { - return llvm::cast(unwrap(type)).getDivisibleBy(); +bool mlirCuteConstrainedIntTypeGetIsPow2(MlirType type) { + return llvm::cast(unwrap(type)).getIsPow2(); } bool mlirTypeIsACuteCoordTensor(MlirType type) { @@ -90,12 +92,11 @@ MlirType mlirCuteCoordTensorTypeGet(MlirContext context, MlirType arithTuple, if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto arithTupleValue = - llvm::dyn_cast_if_present(unwrap(arithTuple)); - if (!arithTupleValue || arithTupleValue.getContext() != ctx) + ::mlir::cutlass_compiler::cute::IntTupleType arithTupleValue; + if (!unwrapAs(ctx, arithTuple, false, arithTupleValue)) return {nullptr}; - auto layoutValue = unwrap(layout); - if (!layoutValue || layoutValue.getContext() != ctx) + mlir::Type layoutValue; + if (!unwrapAs(ctx, layout, false, layoutValue)) return {nullptr}; return wrap(cute::CoordTensorType::getChecked( [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, @@ -118,8 +119,8 @@ MlirType mlirCuteCoordTypeGet(MlirContext context, MlirAttribute attr) { if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto attrValue = llvm::dyn_cast_if_present(unwrap(attr)); - if (!attrValue || attrValue.getContext() != ctx) + ::mlir::cutlass_compiler::cute::CoordAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) return {nullptr}; return wrap(cute::CoordType::get(ctx, attrValue)); } @@ -137,15 +138,13 @@ MlirType mlirCuteFastDivmodDivisorTypeGet(MlirContext context, unsigned width, if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - return wrap(cute::FastDivmodDivisorType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, width, - isPow2)); + return wrap(cute::FastDivmodDivisorType::get(ctx, width, isPow2)); } unsigned mlirCuteFastDivmodDivisorTypeGetWidth(MlirType type) { return llvm::cast(unwrap(type)).getWidth(); } bool mlirCuteFastDivmodDivisorTypeGetIsPow2(MlirType type) { - return llvm::cast(unwrap(type)).getIsPow2(); + return llvm::cast(unwrap(type)).getIsPow_2(); } bool mlirTypeIsACuteIntTuple(MlirType type) { @@ -156,8 +155,8 @@ MlirType mlirCuteIntTupleTypeGet(MlirContext context, MlirAttribute attr) { if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto attrValue = llvm::dyn_cast_if_present(unwrap(attr)); - if (!attrValue || attrValue.getContext() != ctx) + ::mlir::cutlass_compiler::cute::IntTupleAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) return {nullptr}; return wrap(cute::IntTupleType::get(ctx, attrValue)); } @@ -174,8 +173,8 @@ MlirType mlirCuteLayoutTypeGet(MlirContext context, MlirAttribute attr) { if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto attrValue = llvm::dyn_cast_if_present(unwrap(attr)); - if (!attrValue || attrValue.getContext() != ctx) + ::mlir::cutlass_compiler::cute::LayoutAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) return {nullptr}; return wrap(cute::LayoutType::getChecked( [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, @@ -195,11 +194,11 @@ MlirType mlirCuteMemRefTypeGet(MlirContext context, MlirType ptr, if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto ptrValue = llvm::dyn_cast_if_present(unwrap(ptr)); - if (!ptrValue || ptrValue.getContext() != ctx) + ::mlir::cutlass_compiler::cute::PtrType ptrValue; + if (!unwrapAs(ctx, ptr, false, ptrValue)) return {nullptr}; - auto layoutValue = unwrap(layout); - if (!layoutValue || layoutValue.getContext() != ctx) + mlir::Type layoutValue; + if (!unwrapAs(ctx, layout, false, layoutValue)) return {nullptr}; return wrap(cute::MemRefType::getChecked( [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, @@ -218,34 +217,37 @@ bool mlirTypeIsACutePtr(MlirType type) { return llvm::isa_and_nonnull(unwrap(type)); } MlirType mlirCutePtrTypeGet(MlirContext context, MlirType valueType, - MlirAttribute memorySpace, uint64_t alignment, - MlirAttribute swizzle) { + uint32_t addressSpace, uint64_t alignment, + MlirAttribute swizzle, MlirAttribute bitlayout) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueTypeValue = unwrap(valueType); - if (valueType.ptr && (!valueTypeValue || valueTypeValue.getContext() != ctx)) + mlir::Type valueTypeValue; + if (!unwrapAs(ctx, valueType, true, valueTypeValue)) return {nullptr}; - auto memorySpaceValue = - llvm::dyn_cast_if_present(unwrap(memorySpace)); - if (!memorySpaceValue || memorySpaceValue.getContext() != ctx) + auto addressSpaceValue = + ::mlir::cutlass_compiler::cute::symbolizeAddressSpace(addressSpace); + if (!addressSpaceValue) return {nullptr}; - auto swizzleValue = - llvm::dyn_cast_if_present(unwrap(swizzle)); - if (swizzle.ptr && (!swizzleValue || swizzleValue.getContext() != ctx)) + ::mlir::cutlass_compiler::cute::SwizzleAttr swizzleValue; + if (!unwrapAs(ctx, swizzle, true, swizzleValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::BitLayoutAttr bitlayoutValue; + if (!unwrapAs(ctx, bitlayout, true, bitlayoutValue)) return {nullptr}; return wrap(cute::PtrType::getChecked( [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - valueTypeValue, memorySpaceValue, alignment, swizzleValue)); + valueTypeValue, *addressSpaceValue, alignment, swizzleValue, + bitlayoutValue)); } MlirType mlirCutePtrTypeGetValueType(MlirType type) { return wrap(static_cast( llvm::cast(unwrap(type)).getValueType())); } -MlirAttribute mlirCutePtrTypeGetMemorySpace(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getMemorySpace())); +uint32_t mlirCutePtrTypeGetAddressSpace(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getAddressSpace()); } uint64_t mlirCutePtrTypeGetAlignment(MlirType type) { return llvm::cast(unwrap(type)).getAlignment(); @@ -254,6 +256,10 @@ MlirAttribute mlirCutePtrTypeGetSwizzle(MlirType type) { return wrap(static_cast( llvm::cast(unwrap(type)).getSwizzle())); } +MlirAttribute mlirCutePtrTypeGetBitlayout(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBitlayout())); +} bool mlirTypeIsACuteShape(MlirType type) { return llvm::isa_and_nonnull(unwrap(type)); @@ -263,8 +269,8 @@ MlirType mlirCuteShapeTypeGet(MlirContext context, MlirAttribute attr) { if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto attrValue = llvm::dyn_cast_if_present(unwrap(attr)); - if (!attrValue || attrValue.getContext() != ctx) + ::mlir::cutlass_compiler::cute::ShapeAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) return {nullptr}; return wrap(cute::ShapeType::get(ctx, attrValue)); } @@ -282,8 +288,8 @@ MlirType mlirCuteSparseElemTypeGet(MlirContext context, int numLogical, if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto physicalTypeValue = unwrap(physicalType); - if (!physicalTypeValue || physicalTypeValue.getContext() != ctx) + mlir::Type physicalTypeValue; + if (!unwrapAs(ctx, physicalType, false, physicalTypeValue)) return {nullptr}; return wrap(cute::SparseElemType::getChecked( [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, @@ -305,8 +311,8 @@ MlirType mlirCuteStrideTypeGet(MlirContext context, MlirAttribute attr) { if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto attrValue = llvm::dyn_cast_if_present(unwrap(attr)); - if (!attrValue || attrValue.getContext() != ctx) + ::mlir::cutlass_compiler::cute::StrideAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) return {nullptr}; return wrap(cute::StrideType::getChecked( [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, @@ -325,8 +331,8 @@ MlirType mlirCuteSwizzleTypeGet(MlirContext context, MlirAttribute attr) { if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto attrValue = llvm::dyn_cast_if_present(unwrap(attr)); - if (!attrValue || attrValue.getContext() != ctx) + ::mlir::cutlass_compiler::cute::SwizzleAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) return {nullptr}; return wrap(cute::SwizzleType::getChecked( [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, @@ -345,8 +351,8 @@ MlirType mlirCuteTileTypeGet(MlirContext context, MlirAttribute attr) { if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto attrValue = llvm::dyn_cast_if_present(unwrap(attr)); - if (!attrValue || attrValue.getContext() != ctx) + ::mlir::cutlass_compiler::cute::TileAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) return {nullptr}; return wrap(cute::TileType::getChecked( [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, @@ -356,3 +362,147 @@ MlirAttribute mlirCuteTileTypeGetAttr(MlirType type) { return wrap(static_cast( llvm::cast(unwrap(type)).getAttr())); } + +bool mlirTypeIsACuteTiledCopy(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteTiledCopyTypeGet(MlirContext context, MlirType copyAtom, + MlirAttribute layoutCopyTv, + MlirAttribute tilerMn) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type copyAtomValue; + if (!unwrapAs(ctx, copyAtom, false, copyAtomValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutCopyTvValue; + if (!unwrapAs(ctx, layoutCopyTv, false, layoutCopyTvValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::TileAttr tilerMnValue; + if (!unwrapAs(ctx, tilerMn, false, tilerMnValue)) + return {nullptr}; + return wrap(cute::TiledCopyType::get(ctx, copyAtomValue, layoutCopyTvValue, + tilerMnValue)); +} +MlirType mlirCuteTiledCopyTypeGetCopyAtom(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCopyAtom())); +} +MlirType mlirCuteTiledCopyTypeGetAtom(MlirType type) { + return mlirCuteTiledCopyTypeGetCopyAtom(type); +} +MlirAttribute mlirCuteTiledCopyTypeGetLayoutCopyTv(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getLayoutCopyTv())); +} +MlirAttribute mlirCuteTiledCopyTypeGetTilerMn(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getTilerMn())); +} + +bool mlirTypeIsACuteTiledCopyV2(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteTiledCopyV2TypeGet(MlirContext context, MlirType copyAtom, + MlirAttribute layoutCopyTv, + MlirAttribute tilerMn) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type copyAtomValue; + if (!unwrapAs(ctx, copyAtom, false, copyAtomValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutCopyTvValue; + if (!unwrapAs(ctx, layoutCopyTv, false, layoutCopyTvValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::TileAttr tilerMnValue; + if (!unwrapAs(ctx, tilerMn, false, tilerMnValue)) + return {nullptr}; + return wrap(cute::TiledCopyV2Type::get(ctx, copyAtomValue, layoutCopyTvValue, + tilerMnValue)); +} +MlirType mlirCuteTiledCopyV2TypeGetCopyAtom(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCopyAtom())); +} +MlirAttribute mlirCuteTiledCopyV2TypeGetLayoutCopyTv(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getLayoutCopyTv())); +} +MlirAttribute mlirCuteTiledCopyV2TypeGetTilerMn(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getTilerMn())); +} + +bool mlirTypeIsACuteTiledMma(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteTiledMmaTypeGet(MlirContext context, MlirType mmaAtom, + MlirAttribute atomLayoutMNK, + MlirAttribute permutationMNK) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type mmaAtomValue; + if (!unwrapAs(ctx, mmaAtom, false, mmaAtomValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr atomLayoutMNKValue; + if (!unwrapAs(ctx, atomLayoutMNK, false, atomLayoutMNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::TileAttr permutationMNKValue; + if (!unwrapAs(ctx, permutationMNK, true, permutationMNKValue)) + return {nullptr}; + return wrap(cute::TiledMmaType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + mmaAtomValue, atomLayoutMNKValue, permutationMNKValue)); +} +MlirType mlirCuteTiledMmaTypeGetMmaAtom(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getMmaAtom())); +} +MlirType mlirCuteTiledMmaTypeGetAtom(MlirType type) { + return mlirCuteTiledMmaTypeGetMmaAtom(type); +} +MlirAttribute mlirCuteTiledMmaTypeGetAtomLayoutMNK(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAtomLayout_MNK())); +} +MlirAttribute mlirCuteTiledMmaTypeGetAtomLayoutMnk(MlirType type) { + return mlirCuteTiledMmaTypeGetAtomLayoutMNK(type); +} +MlirAttribute mlirCuteTiledMmaTypeGetPermutationMNK(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPermutation_MNK())); +} +MlirAttribute mlirCuteTiledMmaTypeGetPermutationMnk(MlirType type) { + return mlirCuteTiledMmaTypeGetPermutationMNK(type); +} + +bool mlirTypeIsACuteTuple(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteTupleTypeGet(MlirContext context, intptr_t numTypes, + MlirType const *types) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + llvm::SmallVector typesValue; + for (intptr_t i = 0; i < numTypes; ++i) { + mlir::Type element; + if (!unwrapAs(ctx, types[i], false, element)) + return {nullptr}; + typesValue.push_back(element); + } + return wrap(cute::TupleType::get(ctx, typesValue)); +} +intptr_t mlirCuteTupleTypeGetNumTypes(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getTypes().size()); +} +MlirType mlirCuteTupleTypeGetType(MlirType type, intptr_t pos) { + return wrap(llvm::cast(unwrap(type)).getTypes()[pos]); +} diff --git a/mlir/dialects/cute_ir/lib/Dialect/CuteNVGPU/IR/BUILD.bazel b/mlir/dialects/cute_ir/lib/Dialect/CuteNVGPU/IR/BUILD.bazel index f6f289f3..708508fa 100644 --- a/mlir/dialects/cute_ir/lib/Dialect/CuteNVGPU/IR/BUILD.bazel +++ b/mlir/dialects/cute_ir/lib/Dialect/CuteNVGPU/IR/BUILD.bazel @@ -6,7 +6,10 @@ cc_library( name = "dialect", srcs = ["CuteNVGPUDialect.cpp"], deps = [ + "//mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR:headers", "//mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR:headers", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:AsmParser", "@llvm-project//mlir:IR", ], ) diff --git a/mlir/dialects/cute_ir/lib/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.cpp b/mlir/dialects/cute_ir/lib/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.cpp index 5a2086a1..7c2e6bc3 100644 --- a/mlir/dialects/cute_ir/lib/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.cpp +++ b/mlir/dialects/cute_ir/lib/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.cpp @@ -1,123 +1,1653 @@ //===- CuteNVGPUDialect.cpp - CuTe NVIDIA GPU dialect --------------------===// +// +// Parsers, printers and verifiers of the cute_nvgpu types that the declarative +// format cannot express. The syntax and the verifier messages are the DSL +// compiler's (see CuteNVGPUTypes.td). #include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h" +#include +#include + +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/TypeSwitch.h" +#include "mlir/AsmParser/AsmParser.h" #include "mlir/IR/Builders.h" #include "mlir/IR/BuiltinTypes.h" #include "mlir/IR/DialectImplementation.h" -#include "llvm/ADT/TypeSwitch.h" - using namespace mlir; using namespace mlir::cutlass_compiler::cute_nvgpu; +namespace cute = mlir::cutlass_compiler::cute; #include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.cpp.inc" +#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUEnums.cpp.inc" + +#define GET_ATTRDEF_CLASSES +#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUAttrs.cpp.inc" + #define GET_TYPEDEF_CLASSES #include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUTypes.cpp.inc" #define GET_OP_CLASSES #include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUOps.cpp.inc" -#define CUTE_NVGPU_PAYLOAD_TYPES(M) \ - M(CopyAtomBulkCopyG2SType) \ - M(CopyAtomBulkCopyS2GType) \ - M(CopyAtomBulkCopyS2SType) \ - M(CopyAtomDsmemStoreType) \ - M(CopyAtomG2RType) \ - M(CopyAtomLdsmType) \ - M(CopyAtomNonExec2DGather4TmaLoadType) \ - M(CopyAtomNonExecIm2ColTmaLoadType) \ - M(CopyAtomNonExecIm2ColTmaStoreType) \ - M(CopyAtomNonExecTiledTmaLoadType) \ - M(CopyAtomNonExecTiledTmaReduceType) \ - M(CopyAtomNonExecTiledTmaStoreType) \ - M(CopyAtomR2GType) \ - M(CopyAtomR2SType) \ - M(CopyAtomS2RType) \ - M(CopyAtomSIMTAsyncCopyType) \ - M(CopyAtomSIMTSyncCopyType) \ - M(CopyAtomSM100CopyS2TType) \ - M(CopyAtomSM100TmemLoadType) \ - M(CopyAtomSM100TmemStoreType) \ - M(CopyAtomSM10xTmemLoadRedType) \ - M(CopyAtomStsmType) \ - M(MmaAtomSM100UMMABlockScaledType) \ - M(MmaAtomSM100UMMAType) \ - M(MmaAtomSM120BlockScaledType) \ - M(MmaAtomSM80Type) \ - M(MmaAtomSM89Type) \ - M(MmaAtomSM90Type) \ - M(SmemDescViewType) \ - M(TiledCopyType) \ - M(TiledMmaType) \ - M(UniversalFmaAtomType) \ - M(CopyAtomIm2ColTmaLoadType) \ - M(CopyAtomIm2ColTmaStoreType) \ - M(CopyAtomSIMTMultimemLdReduceType) \ - M(CopyAtomSIMTMultimemRedType) \ - M(CopyAtomSIMTMultimemStType) \ - M(CopyAtomSM100S2TCopyV2Type) \ - M(CopyAtomTmaLoadType) \ - M(CopyAtomTmaReduceType) \ - M(CopyAtomTmaStoreType) \ - M(MmaAtomSM100UMMABlockScaledSparseType) \ - M(MmaAtomSM100UMMASparseType) \ - M(MmaAtomSM80SparseType) \ - M(SmemDescCircularSM103Type) - -#define DEFINE_PAYLOAD_TYPE(TYPE) \ - Type TYPE::parse(AsmParser &parser) { \ - parser.emitError(parser.getCurrentLocation(), \ - "payload types are parsed by CuteNVGPUDialect"); \ - return {}; \ - } \ - void TYPE::print(AsmPrinter &printer) const { \ - printer << getPayload().getValue(); \ - } -CUTE_NVGPU_PAYLOAD_TYPES(DEFINE_PAYLOAD_TYPE) -#undef DEFINE_PAYLOAD_TYPE - -template -static Type maybeParsePayloadType(MLIRContext *context, StringRef spec) { - StringRef remainder = spec; - if (!remainder.consume_front(TypeT::getMnemonic())) - return {}; - if (!remainder.empty() && !remainder.starts_with("<")) - return {}; - return TypeT::get(context, StringAttr::get(context, remainder)); +FailureOr mlir::cutlass_compiler::cute_nvgpu::parseBool(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + StringRef kw; + if (p.parseKeyword(&kw)) return failure(); + if (kw == "true" || kw == "false") return kw == "true"; + return p.emitError(loc, "expected `true` or `false`"); } -Type CuteNVGPUDialect::parseType(DialectAsmParser &parser) const { - StringRef spec = parser.getFullSymbolSpec(); - if (spec == SmemDescType::getMnemonic() || - spec == TmaDescriptorTiledType::getMnemonic() || - spec == TmaDescriptorIm2ColType::getMnemonic() || - spec == WorkIdResponseType::getMnemonic()) { - StringRef mnemonic; - Type type; - OptionalParseResult result = generatedTypeParser(parser, &mnemonic, type); - return result.has_value() && succeeded(*result) ? type : Type(); +//===----------------------------------------------------------------------===// +// Helpers +//===----------------------------------------------------------------------===// + +namespace { + +using EmitFn = llvm::function_ref; + +template +ParseResult field(AsmParser &p, T &out) { + FailureOr v = FieldParser::parse(p); + if (failed(v)) return failure(); + out = *v; + return success(); +} +ParseResult field(AsmParser &p, Type &out) { return p.parseType(out); } +ParseResult field(AsmParser &p, int &out) { return p.parseInteger(out); } +ParseResult field(AsmParser &p, bool &out) { + FailureOr v = parseBool(p); + if (failed(v)) return failure(); + out = *v; + return success(); +} + +// `key = value`. +template +ParseResult keyField(AsmParser &p, StringRef key, T &out) { + return failure(p.parseKeyword(key) || p.parseEqual() || field(p, out)); +} +// `, key = value`. +template +ParseResult nextKeyField(AsmParser &p, StringRef key, T &out) { + return failure(p.parseComma() || keyField(p, key, out)); +} +// An optional ` key = value` group. +template +ParseResult optKeyField(AsmParser &p, StringRef key, T &out) { + if (failed(p.parseOptionalKeyword(key))) return success(); + return failure(p.parseEqual() || field(p, out)); +} +// `N word`, as in `128 b` or `32 DP`. +ParseResult countOf(AsmParser &p, StringRef word, int &out) { + return failure(p.parseInteger(out) || p.parseKeyword(word)); +} +// `(a, b)`. +template +ParseResult pairOf(AsmParser &p, T &a, T &b) { + return failure(p.parseLParen() || field(p, a) || p.parseComma() || + field(p, b) || p.parseRParen()); +} +// `elem_type = (A, B, C)`. +ParseResult elemTypes(AsmParser &p, Type &a, Type &b, Type &c) { + return failure(p.parseKeyword("elem_type") || p.parseEqual() || + p.parseLParen() || p.parseType(a) || p.parseComma() || + p.parseType(b) || p.parseComma() || p.parseType(c) || + p.parseRParen()); +} +void printElemTypes(AsmPrinter &p, Type a, Type b, Type c) { + p << "elem_type = (" << a << ", " << b << ", " << c << ")"; +} +// `xR`. +ParseResult repeat(AsmParser &p, unsigned &out) { + SMLoc loc = p.getCurrentLocation(); + StringRef kw; + if (p.parseKeyword(&kw)) return failure(); + if (!kw.consume_front("x")) + return p.emitError(loc, "failed to parse parameter 'num_rep' which is to start with `x`"); + if (kw.getAsInteger(10, out)) + return p.emitError(loc, "failed to parse parameter 'num_rep' which is to be a `unsigned`"); + return success(); +} + +template +void printIfNot(AsmPrinter &p, StringRef key, E value, E dflt) { + if (value != dflt) p << ' ' << key << " = " << stringifyEnum(value); +} +template +void printAttrField(AsmPrinter &p, StringRef key, A attr) { + if (!attr) return; + p << ", " << key << " = "; + p.printStrippedAttrOrType(attr); +} + +// The width of an element: integers and floats, 64 for index and pointers, +// a sparse element's share of its physical storage. +int elementBits(Type t) { + if (auto s = llvm::dyn_cast(t)) + return s.getNumLogical() ? elementBits(s.getPhysicalType()) / s.getNumLogical() : 0; + if (t.isTF32()) return 32; + if (t.isIntOrFloat()) return t.getIntOrFloatBitWidth(); + if (llvm::isa(t)) return 64; + return 0; +} +int orElementBits(int bits, Type t) { return bits ? bits : elementBits(t); } + +// The integers of a flat tuple `(a,b,c)`. +std::optional> flatInts(StringRef text) { + if (!text.consume_front("(") || !text.consume_back(")")) return std::nullopt; + SmallVector out; + for (StringRef part : llvm::split(text, ',')) { + int64_t v; + if (part.getAsInteger(10, v)) return std::nullopt; + out.push_back(v); } + return out; +} -#define TRY_PAYLOAD_TYPE(TYPE) \ - if (Type type = maybeParsePayloadType(getContext(), spec)) \ - return type; - CUTE_NVGPU_PAYLOAD_TYPES(TRY_PAYLOAD_TYPE) -#undef TRY_PAYLOAD_TYPE +// `MxNxK`. +ParseResult parseShapeMNK(AsmParser &p, cute::ShapeAttr &out) { + SMLoc loc = p.getCurrentLocation(); + SmallVector dims; + if (p.parseDimensionList(dims, /*allowDynamic=*/false, /*withTrailingX=*/false)) + return p.emitError(loc, "can not parse shape_MNK"); + auto shape = cutegen::from_string( + "(" + llvm::join(llvm::map_range(dims, [](int64_t d) { return std::to_string(d); }), ",") + ")"); + if (!shape) return p.emitError(loc, "can not parse shape_MNK"); + out = cute::ShapeAttr::get(p.getContext(), *shape); + return success(); +} +void printShapeMNK(AsmPrinter &p, cute::ShapeAttr shape) { + std::string text = cutegen::to_string(shape.getRef()); + StringRef body(text); + body.consume_front("("); + body.consume_back(")"); + for (char c : body) p << (c == ',' ? 'x' : c); +} +LogicalResult verifyShapeMNK(EmitFn emitError, cute::ShapeAttr shape) { + std::string text = cutegen::to_string(shape.getRef()); + auto dims = flatInts(text); + if (!dims || dims->size() != 3) + return emitError() << "expects flatten rank-3 tuple of shape, but got " << text; + return success(); +} - StringAttr dialectNamespace = - StringAttr::get(getContext(), getDialectNamespace()); - return OpaqueType::get(dialectNamespace, spec); +// The A fragment kind: `ss` (shared memory), `rs` (registers) or `ts` +// (tensor memory). +ParseResult parseFragKind(AsmParser &p, MmaFragKind other, MmaFragKind &out) { + SMLoc loc = p.getCurrentLocation(); + StringRef kw; + bool ts = other == MmaFragKind::tmem; + if (p.parseKeyword("frag_kind") || p.parseEqual() || p.parseKeyword(&kw)) + return failure(); + if (kw == "ss") out = MmaFragKind::smem_desc; + else if (kw == (ts ? "ts" : "rs")) out = other; + else return p.emitError(loc, ts ? "expects `ts` or `ss`" : "expects `rs` or `ss`"); + return success(); +} +StringRef fragKindText(MmaFragKind k) { + switch (k) { + case MmaFragKind::smem_desc: return "ss"; + case MmaFragKind::rmem: return "rs"; + case MmaFragKind::tmem: return "ts"; + default: return stringifyEnum(k); + } +} + +// `t` or `n`. +ParseResult parseTranspose(AsmParser &p, UnitAttr &out) { + SMLoc loc = p.getCurrentLocation(); + StringRef kw; + if (p.parseKeyword(&kw)) return failure(); + if (kw != "t" && kw != "n") return p.emitError(loc, "expects `t` or `n`"); + out = kw == "t" ? UnitAttr::get(p.getContext()) : UnitAttr(); + return success(); +} + +bool isPow2AtLeast8(int v) { return v >= 8 && llvm::isPowerOf2_32(v); } + +} // namespace + +//===----------------------------------------------------------------------===// +// Descriptors +//===----------------------------------------------------------------------===// + +// sm103/sm107.smem_desc_circular: static `(B,128,N):(128,1,128*B)`, N >= 2. +static LogicalResult verifyCircular(EmitFn emitError, cute::LayoutAttr layout) { + std::string text = cutegen::to_string(layout.getRef()); + StringRef shapeText, strideText; + std::tie(shapeText, strideText) = StringRef(text).split(':'); + if (StringRef(text).contains('?')) + return emitError() << "block layout must be fully static but got " << text; + auto shape = flatInts(shapeText), stride = flatInts(strideText); + if (!shape || shape->size() != 3 || !stride || stride->size() != 3) + return emitError() << "expects block layout to have 3 modes, but got " << text; + if ((*shape)[1] != 128) + return emitError() << "expects block layout to have 128B 1st mode, but got " << text; + if ((*stride)[1] != 1 || (*stride)[0] != 128 || (*stride)[2] != 128 * (*shape)[0]) + return emitError() << "expects mode-1 major compact layout, but got " << text; + if ((*shape)[2] < 2) + return emitError() << "Circular SMEM descriptor requires at least 2 128B blocks since " + "MMA may span 2 buffers, but got " << text; + return success(); +} +LogicalResult SmemDescCircularSM103Type::verify(EmitFn emitError, cute::LayoutAttr layout) { + return verifyCircular(emitError, layout); +} +LogicalResult SmemDescCircularSM107Type::verify(EmitFn emitError, cute::LayoutAttr layout) { + return verifyCircular(emitError, layout); +} + +Type SmemDescViewType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type desc; + std::string text; + if (p.parseLess() || p.parseType(desc) || p.parseComma()) return {}; + SMLoc layoutLoc = p.getCurrentLocation(); + if (p.parseString(&text) || p.parseGreater()) return {}; + auto layout = cutegen::from_string(text); + if (!layout) { + p.emitError(layoutLoc, "failed to parse SmemDescViewType parameter 'layout' as `cutegen::layout`"); + return {}; + } + return getChecked([&] { return p.emitError(loc); }, p.getContext(), desc, + cute::LayoutAttr::get(p.getContext(), *layout)); +} +void SmemDescViewType::print(AsmPrinter &p) const { + p << '<' << getDesc() << ", "; + p.printString(cutegen::to_string(getLayout().getRef())); + p << '>'; +} +LogicalResult SmemDescViewType::verify(EmitFn emitError, Type desc, cute::LayoutAttr) { + if (!llvm::isa(desc)) + return emitError() << "expected SmemDescType, SmemDescCircularSM103Type, " + "SmemDescCircularSM107Type or SmemDescSM107Type"; + return success(); +} + +//===----------------------------------------------------------------------===// +// SIMT copy atoms +//===----------------------------------------------------------------------===// + +// ``; the compiler prints +// `` for copy_bits 0, which is accepted too. +Type CopyAtomSIMTSyncCopyType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int bits = 0; + auto src = cute::AddressSpace::generic, dst = src; + if (p.parseLess() || p.parseType(t)) return {}; + // `src_space` / `dst_space` groups; one alone is what the compiler prints + // when the other is generic (or copy_bits is 0), so it is read too. + auto spaces = [&]() -> ParseResult { + if (succeeded(p.parseOptionalKeyword("dst_space"))) + return failure(p.parseEqual() || field(p, dst)); + if (p.parseKeyword("src_space") || p.parseEqual() || field(p, src)) return failure(); + if (succeeded(p.parseOptionalComma())) return keyField(p, "dst_space", dst); + return success(); + }; + if (succeeded(p.parseOptionalComma())) { + OptionalParseResult n = p.parseOptionalInteger(bits); + if (n.has_value() && (failed(*n) || p.parseKeyword("b"))) return {}; + if ((!n.has_value() || succeeded(p.parseOptionalComma())) && spaces()) return {}; + } + if (p.parseGreater()) return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, bits, src, dst); +} +void CopyAtomSIMTSyncCopyType::print(AsmPrinter &p) const { + auto generic = cute::AddressSpace::generic; + bool spaces = getSrcSpace() != generic || getDstSpace() != generic; + p << '<' << getValType(); + if (getCopyBits() || spaces) p << ", " << getCopyBits() << " b"; + if (spaces) p << ", src_space = " << stringifyEnum(getSrcSpace()); + if (getDstSpace() != generic) p << ", dst_space = " << stringifyEnum(getDstSpace()); + p << '>'; +} +LogicalResult CopyAtomSIMTSyncCopyType::verify(EmitFn emitError, Type, int bits, + cute::AddressSpace, cute::AddressSpace) { + if (bits != 0 && !isPow2AtLeast8(bits)) + return emitError() << "expected copy_bits to be 0 (auto) or a power of 2 (>= 8), but got " + << bits; + return success(); +} + +// ``. +Type CopyAtomSIMTAsyncCopyType::parse(AsmParser &p) { + Type t; + int bits = 0; + LoadCacheMode cache; + if (p.parseLess() || p.parseType(t) || nextKeyField(p, "cache", cache)) return {}; + if (succeeded(p.parseOptionalComma()) && countOf(p, "b", bits)) return {}; + if (p.parseGreater()) return {}; + return get(p.getContext(), t, cache, orElementBits(bits, t)); +} +void CopyAtomSIMTAsyncCopyType::print(AsmPrinter &p) const { + p << '<' << getValType() << ", cache = " << stringifyEnum(getCache()) << ", " + << getCopyBits() << " b>"; +} + +// ``. +Type CopyAtomDsmemStoreType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int bits = 0; + if (p.parseLess() || p.parseType(t)) return {}; + if (succeeded(p.parseOptionalComma()) && countOf(p, "b", bits)) return {}; + if (p.parseGreater()) return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, + orElementBits(bits, t)); +} +void CopyAtomDsmemStoreType::print(AsmPrinter &p) const { + p << '<' << getValType() << ", " << getCopyBits() << " b>"; +} +LogicalResult CopyAtomDsmemStoreType::verify(EmitFn emitError, Type, int bits) { + if (bits != 0 && bits != 32 && bits != 64 && bits != 128) + return emitError() << "expects copy_bits to be one of (0, 32, 64, 128), but got " << bits; + return success(); +} + +// Bulk copies: ``. +Type CopyAtomBulkCopyG2SType::parse(AsmParser &p) { + Type t; + int bits = 0; + bool mcast = false; + if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || + optKeyField(p, "mcast", mcast) || p.parseGreater()) + return {}; + return get(p.getContext(), t, bits, mcast); +} +void CopyAtomBulkCopyG2SType::print(AsmPrinter &p) const { + p << '<' << getValType(); + if (getCopyBits()) p << " copy_bits = " << getCopyBits(); + if (getMcast()) p << " mcast = true"; + p << '>'; +} +Type CopyAtomBulkCopyS2GType::parse(AsmParser &p) { + Type t; + int bits = 0; + bool mask = false; + if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || + optKeyField(p, "mask", mask) || p.parseGreater()) + return {}; + return get(p.getContext(), t, bits, mask); +} +void CopyAtomBulkCopyS2GType::print(AsmPrinter &p) const { + p << '<' << getValType(); + if (getCopyBits()) p << " copy_bits = " << getCopyBits(); + if (getMask()) p << " mask = true"; + p << '>'; +} +Type CopyAtomBulkCopyS2SType::parse(AsmParser &p) { + Type t; + int bits = 0; + if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || + p.parseGreater()) + return {}; + return get(p.getContext(), t, bits); +} +void CopyAtomBulkCopyS2SType::print(AsmPrinter &p) const { + p << '<' << getValType(); + if (getCopyBits()) p << " copy_bits = " << getCopyBits(); + p << '>'; +} + +// g2r, r2g, r2s, s2r: ``. +static LogicalResult verifyCopyBits(EmitFn emitError, int bits) { + if (!isPow2AtLeast8(bits)) + return emitError() << "expects copy_bits to be a power of 2 (>= 8), but got " << bits; + return success(); +} + +Type CopyAtomG2RType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int bits = 0; + auto order = MemOrderKind::WEAK; + auto scope = MemScopeKind::CTA; + auto prefetch = L2PrefetchSize::NONE; + auto evict = CacheEvictionPriority::EVICT_NORMAL; + auto cache = LoadCacheMode::always; + auto shared = SharedSpace::CTA; + bool invariant = false; + if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || + optKeyField(p, "mem_order", order) || optKeyField(p, "mem_scope", scope) || + optKeyField(p, "l2_prefetch_size", prefetch) || + optKeyField(p, "l1_cache_evict_priority", evict) || + optKeyField(p, "load_cache_mode", cache) || optKeyField(p, "shared_space", shared) || + optKeyField(p, "invariant", invariant) || p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, + orElementBits(bits, t), order, scope, prefetch, evict, cache, shared, + invariant); +} +void CopyAtomG2RType::print(AsmPrinter &p) const { + p << '<' << getValType() << " copy_bits = " << getCopyBits(); + printIfNot(p, "mem_order", getMemOrder(), MemOrderKind::WEAK); + printIfNot(p, "mem_scope", getMemScope(), MemScopeKind::CTA); + printIfNot(p, "l2_prefetch_size", getL2PrefetchSize(), L2PrefetchSize::NONE); + printIfNot(p, "l1_cache_evict_priority", getL1CacheEvictPriority(), + CacheEvictionPriority::EVICT_NORMAL); + printIfNot(p, "load_cache_mode", getLoadCacheMode(), LoadCacheMode::always); + printIfNot(p, "shared_space", getSharedSpace(), SharedSpace::CTA); + if (getInvariant()) p << " invariant = true"; + p << '>'; +} +LogicalResult CopyAtomG2RType::verify(EmitFn emitError, Type, int bits, MemOrderKind order, + MemScopeKind scope, L2PrefetchSize prefetch, + CacheEvictionPriority evict, LoadCacheMode cache, + SharedSpace, bool invariant) { + if (failed(verifyCopyBits(emitError, bits))) return failure(); + if (invariant && (order != MemOrderKind::WEAK || scope != MemScopeKind::CTA || + prefetch != L2PrefetchSize::NONE || + evict != CacheEvictionPriority::EVICT_NORMAL || + cache != LoadCacheMode::always)) + return emitError() << "invariant_load is not allowed when the other non-default memory " + "features (e.g., mem_order) are set to non-default values"; + if (order != MemOrderKind::WEAK && cache != LoadCacheMode::always) + return emitError() << "expects load_cache_mode to be 'always' when mem_order is not " + "'weak', but got '" + << stringifyEnum(cache) << "' and '" << stringifyEnum(order) << "'"; + return success(); +} + +Type CopyAtomR2GType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int bits = 0; + auto order = MemOrderKind::WEAK; + auto scope = MemScopeKind::CTA; + auto evict = CacheEvictionPriority::EVICT_NORMAL; + auto cache = StoreCacheMode::write_back; + auto shared = SharedSpace::CTA; + if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || + optKeyField(p, "mem_order", order) || optKeyField(p, "mem_scope", scope) || + optKeyField(p, "l1_cache_evict_priority", evict) || + optKeyField(p, "store_cache_mode", cache) || optKeyField(p, "shared_space", shared) || + p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, + orElementBits(bits, t), order, scope, evict, cache, shared); +} +void CopyAtomR2GType::print(AsmPrinter &p) const { + p << '<' << getValType() << " copy_bits = " << getCopyBits(); + printIfNot(p, "mem_order", getMemOrder(), MemOrderKind::WEAK); + printIfNot(p, "mem_scope", getMemScope(), MemScopeKind::CTA); + printIfNot(p, "l1_cache_evict_priority", getL1CacheEvictPriority(), + CacheEvictionPriority::EVICT_NORMAL); + printIfNot(p, "store_cache_mode", getStoreCacheMode(), StoreCacheMode::write_back); + printIfNot(p, "shared_space", getSharedSpace(), SharedSpace::CTA); + p << '>'; +} +LogicalResult CopyAtomR2GType::verify(EmitFn emitError, Type, int bits, MemOrderKind order, + MemScopeKind, CacheEvictionPriority, + StoreCacheMode cache, SharedSpace) { + if (failed(verifyCopyBits(emitError, bits))) return failure(); + if (order != MemOrderKind::WEAK && cache != StoreCacheMode::write_back) + return emitError() << "expects store_cache_mode to be 'write_back' when mem_order is " + "not 'weak', but got '" + << stringifyEnum(cache) << "' and '" << stringifyEnum(order) << "'"; + return success(); +} + +template +static Type parseSharedCopy(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int bits = 0; + auto order = MemOrderKind::WEAK; + auto scope = MemScopeKind::CTA; + auto shared = SharedSpace::CTA; + if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || + optKeyField(p, "mem_order", order) || optKeyField(p, "mem_scope", scope) || + optKeyField(p, "shared_space", shared) || p.parseGreater()) + return {}; + return T::getChecked([&] { return p.emitError(loc); }, p.getContext(), t, + orElementBits(bits, t), order, scope, shared); +} +template +static void printSharedCopy(AsmPrinter &p, T type) { + p << '<' << type.getValType() << " copy_bits = " << type.getCopyBits(); + printIfNot(p, "mem_order", type.getMemOrder(), MemOrderKind::WEAK); + printIfNot(p, "mem_scope", type.getMemScope(), MemScopeKind::CTA); + printIfNot(p, "shared_space", type.getSharedSpace(), SharedSpace::CTA); + p << '>'; +} +Type CopyAtomR2SType::parse(AsmParser &p) { return parseSharedCopy(p); } +void CopyAtomR2SType::print(AsmPrinter &p) const { printSharedCopy(p, *this); } +LogicalResult CopyAtomR2SType::verify(EmitFn emitError, Type, int bits, MemOrderKind, + MemScopeKind, SharedSpace) { + return verifyCopyBits(emitError, bits); +} +Type CopyAtomS2RType::parse(AsmParser &p) { return parseSharedCopy(p); } +void CopyAtomS2RType::print(AsmPrinter &p) const { printSharedCopy(p, *this); } +LogicalResult CopyAtomS2RType::verify(EmitFn emitError, Type, int bits, MemOrderKind, + MemScopeKind, SharedSpace) { + return verifyCopyBits(emitError, bits); +} + +//===----------------------------------------------------------------------===// +// Multimem atoms: `, ORDER, SCOPE>`, positional. Parsing +// takes each field by its kind, which also reads what the compiler prints +// when it drops a default field before a set one. +//===----------------------------------------------------------------------===// + +namespace { +struct Multimem { + int bits = 0; + ReductionKind red = ReductionKind::ADD; + LdReduceAccPrecisionKindAttr acc; + MemOrderKind order = MemOrderKind::RELAXED; + MemScopeKind scope = MemScopeKind::SYS; +}; + +ParseResult parseMultimem(AsmParser &p, Type &t, Multimem &m, bool hasRed, bool hasAcc) { + if (p.parseLess() || p.parseType(t)) return failure(); + while (succeeded(p.parseOptionalComma())) { + SMLoc loc = p.getCurrentLocation(); + StringRef kw; + OptionalParseResult bits = p.parseOptionalInteger(m.bits); + if (bits.has_value()) { + if (failed(*bits) || p.parseKeyword("b")) return failure(); + continue; + } + if (hasAcc && succeeded(p.parseOptionalLess())) { + LdReduceAccPrecisionKind kind; + if (field(p, kind) || p.parseGreater()) return failure(); + m.acc = LdReduceAccPrecisionKindAttr::get(p.getContext(), kind); + continue; + } + if (p.parseKeyword(&kw)) return failure(); + if (auto r = symbolizeReductionKind(kw); r && hasRed) m.red = *r; + else if (auto o = symbolizeMemOrderKind(kw)) m.order = *o; + else if (auto s = symbolizeMemScopeKind(kw)) m.scope = *s; + else return p.emitError(loc, "unexpected `") << kw << "`"; + } + return p.parseGreater(); +} + +// Prints the fields up to the last one that is not default. +void printMultimem(AsmPrinter &p, Type t, const Multimem &m, bool bitsSet, bool hasRed, + bool hasAcc) { + int last = m.scope != MemScopeKind::SYS ? 4 + : m.order != MemOrderKind::RELAXED ? 3 + : m.acc ? 2 + : hasRed && m.red != ReductionKind::ADD ? 1 + : bitsSet ? 0 + : -1; + p << '<' << t; + if (last >= 0) p << ", " << m.bits << " b"; + if (hasRed && last >= 1) p << ", " << stringifyEnum(m.red); + if (hasAcc && last >= 2 && m.acc) { + p << ", "; + p.printStrippedAttrOrType(m.acc); + } + if (last >= 3) p << ", " << stringifyEnum(m.order); + if (last >= 4) p << ", " << stringifyEnum(m.scope); + p << '>'; +} +} // namespace + +Type CopyAtomSIMTMultimemStType::parse(AsmParser &p) { + Type t; + Multimem m; + if (parseMultimem(p, t, m, /*hasRed=*/false, /*hasAcc=*/false)) return {}; + return get(p.getContext(), t, m.bits, m.order, m.scope); +} +void CopyAtomSIMTMultimemStType::print(AsmPrinter &p) const { + Multimem m{getCopyBits(), ReductionKind::ADD, {}, getMemOrder(), getMemScope()}; + printMultimem(p, getValType(), m, getCopyBits() != 0, false, false); +} + +Type CopyAtomSIMTMultimemRedType::parse(AsmParser &p) { + Type t; + Multimem m; + if (parseMultimem(p, t, m, /*hasRed=*/true, /*hasAcc=*/false)) return {}; + // An unset width is the element's, at least 32 bits (the f16x2 form). + int bits = m.bits ? m.bits : std::max(elementBits(t), 32); + return get(p.getContext(), t, bits, m.red, m.order, m.scope); +} +void CopyAtomSIMTMultimemRedType::print(AsmPrinter &p) const { + Multimem m{getCopyBits(), getReduction(), {}, getMemOrder(), getMemScope()}; + printMultimem(p, getValType(), m, /*bitsSet=*/true, true, false); +} + +Type CopyAtomSIMTMultimemLdReduceType::parse(AsmParser &p) { + Type t; + Multimem m; + if (parseMultimem(p, t, m, /*hasRed=*/true, /*hasAcc=*/true)) return {}; + return get(p.getContext(), t, m.bits, m.red, m.acc, m.order, m.scope); +} +void CopyAtomSIMTMultimemLdReduceType::print(AsmPrinter &p) const { + Multimem m{getCopyBits(), getReduction(), getLdReduceAccPrecision(), getMemOrder(), + getMemScope()}; + printMultimem(p, getValType(), m, getCopyBits() != 0, true, true); +} + +//===----------------------------------------------------------------------===// +// Tensor-memory atoms: `` +//===----------------------------------------------------------------------===// + +static ParseResult parseTmemHead(AsmParser &p, Type &t, int &dp, int &bit, unsigned &rep) { + return failure(p.parseLess() || p.parseType(t) || p.parseComma() || + countOf(p, "DP", dp) || p.parseComma() || countOf(p, "bit", bit) || + p.parseComma() || repeat(p, rep)); +} +static void printTmemHead(AsmPrinter &p, Type t, int dp, int bit, unsigned rep) { + p << '<' << t << ", " << dp << " DP, " << bit << " bit, x" << rep; +} +// An optional `, word`. +static ParseResult parseFlag(AsmParser &p, StringRef word, UnitAttr &out) { + if (failed(p.parseOptionalComma())) return success(); + if (p.parseKeyword(word)) return failure(); + out = UnitAttr::get(p.getContext()); + return success(); +} + +static LogicalResult verifyTmemCopy(EmitFn emitError, int dp, int bit, unsigned rep) { + if (dp != 16 && dp != 32) + return emitError() << "expects the number of data paths to be one of [16,32], but got " << dp; + if (bit != 32 && bit != 64 && bit != 128 && bit != 256) + return emitError() << "expects the number of bits to be one of [32,64,128,256], but got " << bit; + if (rep < 1 || rep > 128 || !llvm::isPowerOf2_32(rep)) + return emitError() << "expects the number of repetitions to be one of " + "[1,2,4,8,16,32,64,128], but got " << rep; + if (dp == 32 && bit != 32) + return emitError() << "only 32b patterns are supported when the number of data paths " + "is 32, but got " << bit; + if (dp == 16 && bit == 256 && rep >= 64) + return emitError() << "a repetition of 64 or 128 is not supported when the number of " + "data paths is 16 and the number of bits is 256"; + if (dp == 16 && bit == 128 && rep == 128) + return emitError() << "a repetition of 128 is not supported when the number of data " + "paths is 16 and the number of bits is 128"; + return success(); +} + +Type CopyAtomSM100TmemLoadType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int dp, bit; + unsigned rep; + UnitAttr pack; + if (parseTmemHead(p, t, dp, bit, rep) || parseFlag(p, "pack16b", pack) || p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, bit, rep, pack); +} +void CopyAtomSM100TmemLoadType::print(AsmPrinter &p) const { + printTmemHead(p, getValType(), getNumDp(), getNumBit(), getNumRep()); + p << (getPack16b() ? ", pack16b>" : ">"); +} +LogicalResult CopyAtomSM100TmemLoadType::verify(EmitFn emitError, Type, int dp, int bit, + unsigned rep, UnitAttr) { + return verifyTmemCopy(emitError, dp, bit, rep); +} + +Type CopyAtomSM100TmemStoreType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int dp, bit; + unsigned rep; + UnitAttr expand; + if (parseTmemHead(p, t, dp, bit, rep) || parseFlag(p, "expand16b", expand) || + p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, bit, rep, expand); +} +void CopyAtomSM100TmemStoreType::print(AsmPrinter &p) const { + printTmemHead(p, getValType(), getNumDp(), getNumBit(), getNumRep()); + p << (getExpand16b() ? ", expand16b>" : ">"); +} +LogicalResult CopyAtomSM100TmemStoreType::verify(EmitFn emitError, Type, int dp, int bit, + unsigned rep, UnitAttr) { + return verifyTmemCopy(emitError, dp, bit, rep); +} + +// `<..., OP[, nan][, half_split_off = N[ : T]]>`. +Type CopyAtomSM10xTmemLoadRedType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int dp, bit; + unsigned rep; + TmemLoadRedOp op; + UnitAttr nan; + IntegerAttr split; + if (parseTmemHead(p, t, dp, bit, rep) || p.parseComma() || field(p, op)) return {}; + while (succeeded(p.parseOptionalComma())) { + if (!nan && !split && succeeded(p.parseOptionalKeyword("nan"))) { + nan = UnitAttr::get(p.getContext()); + continue; + } + if (p.parseKeyword("half_split_off") || p.parseEqual() || p.parseAttribute(split)) + return {}; + } + if (p.parseGreater()) return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, bit, rep, op, nan, + split); +} +void CopyAtomSM10xTmemLoadRedType::print(AsmPrinter &p) const { + printTmemHead(p, getValType(), getNumDp(), getNumBit(), getNumRep()); + p << ", " << stringifyEnum(getRedOp()); + if (getNan()) p << ", nan"; + if (getHalfSplitOff()) p << ", half_split_off=" << getHalfSplitOff(); + p << '>'; +} +LogicalResult CopyAtomSM10xTmemLoadRedType::verify(EmitFn emitError, Type t, int dp, int bit, + unsigned rep, TmemLoadRedOp op, UnitAttr nan, + IntegerAttr split) { + if (!(t.isF32() || t.isInteger(32))) + return emitError() << "expect val_type to be u32, s32, or f32, but got '" << t << "'"; + if (nan && !t.isF32()) + return emitError() << "expect val_type to be f32 when nan is specified"; + if ((op == TmemLoadRedOp::minabs || op == TmemLoadRedOp::maxabs) && !t.isF32()) + return emitError() << "expect val_type to be f32 when red_op is minabs or maxabs"; + if (bit != 32) return emitError() << "only 32b patterns are supported, but got " << bit; + if (rep < 2 || rep > 128 || !llvm::isPowerOf2_32(rep)) + return emitError() << "expects the number of repetitions to be one of " + "[2,4,8,16,32,64,128], but got " << rep; + if (split && dp != 16) + return emitError() << "expect half_split_off to be set only when mode is 16dp 32bit"; + return success(); +} + +// `<..., OP[, red[, nan]]>`. +Type CopyAtomSM107TmemLoadSPCompressType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int dp, bit; + unsigned rep; + TmemLoadRedOp op; + UnitAttr red, nan; + if (parseTmemHead(p, t, dp, bit, rep) || p.parseComma() || field(p, op) || + parseFlag(p, "red", red) || (red && parseFlag(p, "nan", nan)) || p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, bit, rep, op, red, + nan); +} +void CopyAtomSM107TmemLoadSPCompressType::print(AsmPrinter &p) const { + printTmemHead(p, getValType(), getNumDp(), getNumBit(), getNumRep()); + p << ", " << stringifyEnum(getRedOp()); + if (getRed()) p << ", red"; + if (getNan()) p << ", nan"; + p << '>'; +} +LogicalResult CopyAtomSM107TmemLoadSPCompressType::verify(EmitFn emitError, Type t, int dp, + int bit, unsigned rep, TmemLoadRedOp, + UnitAttr red, UnitAttr nan) { + if (dp != 32 || bit != 32) + return emitError() << "expect `num_dp` and `num_bit` to be 32, but got " << dp << " and " + << bit; + if (!t.isF32()) return emitError() << "expect val_type to be f32, but got '" << t << "'"; + if (rep < 4 || rep > 128 || !llvm::isPowerOf2_32(rep)) + return emitError() << "expect the number of repetitions to be one of " + "[4,8,16,32,64,128], but got " << rep; + if (nan && !red) return emitError() << "expect `red` to be true when `nan` is true"; + return success(); +} + +// The UTCCP shapes of tcgen05.cp. +static bool isUtccp(int dp, int bit, CopyS2TBroadcast b) { + using B = CopyS2TBroadcast; + return (b == B::none && (dp == 4 || dp == 128) && (bit == 128 || bit == 256)) || + (b == B::x4 && dp == 32 && bit == 128) || + ((b == B::lw_0213 || b == B::lw_0123) && dp == 64 && bit == 128); +} + +// ``. +Type CopyAtomSM100CopyS2TType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int dp, bit, cta; + auto bcast = CopyS2TBroadcast::none; + if (p.parseLess() || p.parseType(t) || p.parseComma() || countOf(p, "DP", dp) || + p.parseComma() || countOf(p, "bit", bit) || p.parseComma() || countOf(p, "cta", cta)) + return {}; + if (succeeded(p.parseOptionalComma()) && field(p, bcast)) return {}; + if (p.parseGreater()) return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, bit, cta, bcast); +} +void CopyAtomSM100CopyS2TType::print(AsmPrinter &p) const { + p << '<' << getValType() << ", " << getNumDp() << " DP, " << getNumBit() << " bit, " + << getNumCta() << " cta"; + if (getBroadcast() != CopyS2TBroadcast::none) p << ", " << stringifyEnum(getBroadcast()); + p << '>'; +} +LogicalResult CopyAtomSM100CopyS2TType::verify(EmitFn emitError, Type, int dp, int bit, int cta, + CopyS2TBroadcast b) { + if (cta != 1 && cta != 2) return emitError() << "Expect cta is 1/2 but got: " << cta; + if (!isUtccp(dp, bit, b)) + return emitError() << "The given UTCCP params: dp " << dp << ", bits " << bit + << ", broadcast " << stringifyEnum(b) << " is unsupported or illegal."; + return success(); +} + +// ``. +Type CopyAtomSM100S2TCopyV2Type::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int dp, bit, cta; + cute::MajorMode major; + auto bcast = CopyS2TBroadcast::none; + if (p.parseLess() || p.parseType(t) || nextKeyField(p, "num_dp", dp) || + nextKeyField(p, "num_bit", bit) || nextKeyField(p, "num_cta", cta) || + nextKeyField(p, "smem_major", major)) + return {}; + if (succeeded(p.parseOptionalComma()) && keyField(p, "broadcast", bcast)) return {}; + if (p.parseGreater()) return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, bit, cta, major, + bcast); +} +void CopyAtomSM100S2TCopyV2Type::print(AsmPrinter &p) const { + p << '<' << getValType() << ", num_dp = " << getNumDp() << ", num_bit = " << getNumBit() + << ", num_cta = " << getNumCta() << ", smem_major = " << stringifyEnum(getSmemMajor()); + if (getBroadcast() != CopyS2TBroadcast::none) + p << ", broadcast = " << stringifyEnum(getBroadcast()); + p << '>'; +} +LogicalResult CopyAtomSM100S2TCopyV2Type::verify(EmitFn emitError, Type, int dp, int bit, + int cta, cute::MajorMode, + CopyS2TBroadcast b) { + if (cta != 1 && cta != 2) return emitError() << "expects cta to be 1 or 2, but got " << cta; + if (!isUtccp(dp, bit, b)) + return emitError() << "expects valid UTCCP params, but got dp " << dp << ", bits " << bit + << ", broadcast " << stringifyEnum(b); + return success(); +} + +//===----------------------------------------------------------------------===// +// ldmatrix / stmatrix +//===----------------------------------------------------------------------===// + +Type CopyAtomLdsmType::parse(AsmParser &p) { + Type t; + cute::ShapeAttr mode; + LdsmSzPattern sz; + int n; + UnitAttr trans; + if (p.parseLess() || keyField(p, "val_type", t) || nextKeyField(p, "mode", mode) || + nextKeyField(p, "sz_pattern", sz) || nextKeyField(p, "num_matrices", n) || + p.parseComma() || parseTranspose(p, trans) || p.parseGreater()) + return {}; + return get(p.getContext(), t, mode, sz, n, trans); +} +void CopyAtomLdsmType::print(AsmPrinter &p) const { + p << "" : ", n>"); +} + +Type CopyAtomStsmType::parse(AsmParser &p) { + Type t; + cute::ShapeAttr mode; + int n; + UnitAttr trans; + if (p.parseLess() || p.parseType(t) || nextKeyField(p, "mode", mode) || + nextKeyField(p, "num_matrices", n) || p.parseComma() || parseTranspose(p, trans) || + p.parseGreater()) + return {}; + return get(p.getContext(), t, mode, n, trans); +} +void CopyAtomStsmType::print(AsmPrinter &p) const { + p << '<' << getValType() << ", mode = "; + p.printStrippedAttrOrType(getMode()); + p << ", num_matrices = " << getNumMatrices() << (getTranspose() ? ", t>" : ", n>"); +} + +//===----------------------------------------------------------------------===// +// TMA atoms +//===----------------------------------------------------------------------===// + +static LogicalResult verifySparsity(EmitFn emitError, int sparsity) { + if (sparsity != 1 && sparsity != 2 && sparsity != 4 && sparsity != 8 && sparsity != 16) + return emitError() << "expects sparsity to be one of (1, 2, 4, 8, 16), but got " << sparsity; + return success(); +} +// `'; +} +LogicalResult CopyAtomTmaLoadType::verify(EmitFn emitError, Type, int sparsity, int, + TmaLoadMode, int, cute::StrideType, bool, + cute::LayoutType, bool, bool) { + return verifySparsity(emitError, sparsity); +} + +Type CopyAtomTmaStoreType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int sparsity = 1, bits; + TmaStoreMode mode; + cute::StrideType stride; + cute::LayoutType gbasis; + bool override = false, noOob = false; + if (parseTmaHead(p, t, &sparsity, bits) || nextKeyField(p, "mode", mode) || + nextKeyField(p, "g_stride", stride) || optKeyField(p, "tma_gbasis", gbasis) || + optKeyField(p, "override", override) || optKeyField(p, "no_fully_oob_tile", noOob) || + p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, sparsity, bits, mode, + stride, gbasis, override, noOob); +} +void CopyAtomTmaStoreType::print(AsmPrinter &p) const { + printTmaHead(p, getValType(), getSparsity(), getCopyBits()); + p << ", mode = " << stringifyEnum(getMode()); + printTmaTail(p, getGStride(), getTmaGbasis()); + printTrue(p, "override", getOverride()); + printTrue(p, "no_fully_oob_tile", getNoFullyOobTile()); + p << '>'; +} +LogicalResult CopyAtomTmaStoreType::verify(EmitFn emitError, Type, int sparsity, int, + TmaStoreMode, cute::StrideType, cute::LayoutType, + bool, bool) { + return verifySparsity(emitError, sparsity); +} + +Type CopyAtomTmaReduceType::parse(AsmParser &p) { + Type t; + int bits; + TmaStoreMode mode; + ReductionKind kind; + cute::StrideType stride; + cute::LayoutType gbasis; + if (parseTmaHead(p, t, nullptr, bits) || nextKeyField(p, "mode", mode) || + nextKeyField(p, "kind", kind) || nextKeyField(p, "g_stride", stride) || + optKeyField(p, "tma_gbasis", gbasis) || p.parseGreater()) + return {}; + return get(p.getContext(), t, bits, mode, kind, stride, gbasis); +} +void CopyAtomTmaReduceType::print(AsmPrinter &p) const { + printTmaHead(p, getValType(), 1, getCopyBits()); + p << ", mode = " << stringifyEnum(getMode()) << ", kind = " << stringifyEnum(getKind()); + printTmaTail(p, getGStride(), getTmaGbasis()); + p << '>'; +} + +Type CopyAtomIm2ColTmaLoadType::parse(AsmParser &p) { + Type t; + int bits, ctas; + cute::StrideType stride; + cute::LayoutType gbasis; + bool mcast = false; + if (parseTmaHead(p, t, nullptr, bits) || nextKeyField(p, "num_cta", ctas) || + nextKeyField(p, "g_stride", stride) || optKeyField(p, "mcast", mcast) || + optKeyField(p, "tma_gbasis", gbasis) || p.parseGreater()) + return {}; + return get(p.getContext(), t, bits, ctas, stride, mcast, gbasis); +} +void CopyAtomIm2ColTmaLoadType::print(AsmPrinter &p) const { + printTmaHead(p, getValType(), 1, getCopyBits()); + p << ", num_cta = " << getNumCta() << ", g_stride = "; + p.printStrippedAttrOrType(getGStride()); + printTrue(p, "mcast", getMcast()); + if (getTmaGbasis()) { + p << " tma_gbasis = "; + p.printStrippedAttrOrType(getTmaGbasis()); + } + p << '>'; +} + +Type CopyAtomIm2ColTmaStoreType::parse(AsmParser &p) { + Type t; + int bits; + cute::StrideType stride; + cute::LayoutType gbasis; + if (parseTmaHead(p, t, nullptr, bits) || nextKeyField(p, "g_stride", stride) || + optKeyField(p, "tma_gbasis", gbasis) || p.parseGreater()) + return {}; + return get(p.getContext(), t, bits, stride, gbasis); +} +void CopyAtomIm2ColTmaStoreType::print(AsmPrinter &p) const { + printTmaHead(p, getValType(), 1, getCopyBits()); + printTmaTail(p, getGStride(), getTmaGbasis()); + p << '>'; +} + +// The TMA data format of an element type when none is given. +std::optional mlir::cutlass_compiler::cute_nvgpu::defaultTmaDataFormat(Type t) { + if (auto s = llvm::dyn_cast(t)) t = s.getPhysicalType(); + if (t.isF16()) return TmaDataFormat::F16_RN; + if (t.isBF16()) return TmaDataFormat::BF16_RN; + if (t.isF32()) return TmaDataFormat::F32_RN; + if (t.isTF32()) return TmaDataFormat::TF32_RN; + if (t.isF64()) return TmaDataFormat::F64_RN; + if (!t.isIntOrFloat()) return std::nullopt; + unsigned bits = t.getIntOrFloatBitWidth(); + bool isSigned = t.isSignedInteger(); + if (bits == 4) return TmaDataFormat::U4; + if (bits <= 8) return TmaDataFormat::U8; + if (bits == 16) return TmaDataFormat::U16; + if (bits == 32) return isSigned ? TmaDataFormat::S32 : TmaDataFormat::U32; + if (bits == 64) return isSigned ? TmaDataFormat::S64 : TmaDataFormat::U64; + return std::nullopt; +} + +namespace { +// `<[kind, ]T, copy_bits = N, tma_gbasis = L[, tma_format = F]>`. +template +ParseResult parseNonExecTma(AsmParser &p, Kind *kind, Type &t, int &bits, + cute::LayoutType &gbasis, std::optional &format) { + if (p.parseLess()) return failure(); + if (kind && (field(p, *kind) || p.parseComma())) return failure(); + if (p.parseType(t) || nextKeyField(p, "copy_bits", bits) || + nextKeyField(p, "tma_gbasis", gbasis)) + return failure(); + format = defaultTmaDataFormat(t); + if (succeeded(p.parseOptionalComma())) { + // Empty for an element type without a format, as the compiler prints it. + if (p.parseKeyword("tma_format") || p.parseEqual()) return failure(); + if (succeeded(p.parseOptionalGreater())) { + format = std::nullopt; + return success(); + } + TmaDataFormat f; + if (field(p, f)) return failure(); + format = f; + } + return p.parseGreater(); +} +template +void printNonExecTma(AsmPrinter &p, T type) { + p << type.getValType() << ", copy_bits = " << type.getCopyBits() << ", tma_gbasis = "; + p.printStrippedAttrOrType(type.getTmaGbasis()); + p << ", tma_format = "; + if (type.getTmaFormat()) p << stringifyEnum(*type.getTmaFormat()); + p << '>'; +} +LogicalResult verifyNonExecTma(EmitFn emitError, int bits) { + if (bits == 0) return emitError() << "invalid copy_bits, got 0"; + return success(); +} +} // namespace + +#define NON_EXEC_TMA_WITH_KIND(TYPE, KIND) \ + Type TYPE::parse(AsmParser &p) { \ + SMLoc loc = p.getCurrentLocation(); \ + KIND kind; \ + Type t; \ + int bits; \ + cute::LayoutType gbasis; \ + std::optional format; \ + if (parseNonExecTma(p, &kind, t, bits, gbasis, format)) return {}; \ + return getChecked([&] { return p.emitError(loc); }, p.getContext(), kind, t, \ + bits, gbasis, format); \ + } \ + void TYPE::print(AsmPrinter &p) const { \ + p << '<' << stringifyEnum(getKind()) << ", "; \ + printNonExecTma(p, *this); \ + } \ + LogicalResult TYPE::verify(EmitFn emitError, KIND, Type, int bits, \ + cute::LayoutType, std::optional) { \ + return verifyNonExecTma(emitError, bits); \ + } +#define NON_EXEC_TMA(TYPE) \ + Type TYPE::parse(AsmParser &p) { \ + SMLoc loc = p.getCurrentLocation(); \ + Type t; \ + int bits; \ + cute::LayoutType gbasis; \ + std::optional format; \ + if (parseNonExecTma(p, nullptr, t, bits, gbasis, format)) return {}; \ + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, bits, \ + gbasis, format); \ + } \ + void TYPE::print(AsmPrinter &p) const { \ + p << '<'; \ + printNonExecTma(p, *this); \ + } \ + LogicalResult TYPE::verify(EmitFn emitError, Type, int bits, cute::LayoutType, \ + std::optional) { \ + return verifyNonExecTma(emitError, bits); \ + } +NON_EXEC_TMA_WITH_KIND(CopyAtomNonExecTiledTmaLoadType, TiledTmaLoad) +NON_EXEC_TMA_WITH_KIND(CopyAtomNonExecTiledTmaReduceType, ReductionKind) +NON_EXEC_TMA_WITH_KIND(CopyAtomNonExecIm2ColTmaLoadType, Im2ColTmaLoad) +NON_EXEC_TMA_WITH_KIND(CopyAtomNonExec2DGather4TmaLoadType, GatherScatterTmaLoad) +NON_EXEC_TMA(CopyAtomNonExecTiledTmaStoreType) +NON_EXEC_TMA(CopyAtomNonExecIm2ColTmaStoreType) +NON_EXEC_TMA(CopyAtomNonExec2DScatter4TmaStoreType) +#undef NON_EXEC_TMA_WITH_KIND +#undef NON_EXEC_TMA + +//===----------------------------------------------------------------------===// +// MMA atoms +//===----------------------------------------------------------------------===// + +Type UniversalFmaAtomType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + cute::ShapeAttr shape; + Type a, b, c; + if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || p.parseLParen() || + p.parseType(a) || p.parseComma() || p.parseType(b) || p.parseRParen() || + p.parseArrow() || p.parseType(c) || p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, a, b, c); +} +void UniversalFmaAtomType::print(AsmPrinter &p) const { + p << '<'; + printShapeMNK(p, getShapeMnk()); + p << ", (" << getAType() << ", " << getBType() << ") -> " << getCType() << " >"; +} +LogicalResult UniversalFmaAtomType::verify(EmitFn emitError, cute::ShapeAttr shape, Type, Type, + Type) { + return verifyShapeMNK(emitError, shape); +} + +// ``. +Type MmaAtomSM80Type::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + cute::ShapeAttr shape; + Type a, b, c; + MMAIntOverflowAttr overflow; + BinaryOpAttr op; + if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || elemTypes(p, a, b, c)) + return {}; + if (succeeded(p.parseOptionalComma()) && + (keyField(p, "int_overflow", overflow) || + (succeeded(p.parseOptionalComma()) && keyField(p, "binary_op", op)))) + return {}; + if (p.parseGreater()) return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, a, b, c, overflow, + op); +} +void MmaAtomSM80Type::print(AsmPrinter &p) const { + p << '<'; + printShapeMNK(p, getShapeMnk()); + p << ", "; + printElemTypes(p, getAType(), getBType(), getCType()); + printAttrField(p, "int_overflow", getIntOverflow()); + printAttrField(p, "binary_op", getBinaryOp()); + p << " >"; +} +LogicalResult MmaAtomSM80Type::verify(EmitFn emitError, cute::ShapeAttr shape, Type, Type, Type, + MMAIntOverflowAttr, BinaryOpAttr) { + return verifyShapeMNK(emitError, shape); +} + +// ``. +Type MmaAtomSM80SparseType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + cute::ShapeAttr shape; + Type a, b, c; + SparseMetadataFormat format; + MMAIntOverflowAttr overflow; + if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || elemTypes(p, a, b, c) || + nextKeyField(p, "md_format", format)) + return {}; + if (succeeded(p.parseOptionalComma()) && keyField(p, "int_overflow", overflow)) return {}; + if (p.parseGreater()) return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, a, b, c, format, + overflow); +} +void MmaAtomSM80SparseType::print(AsmPrinter &p) const { + p << '<'; + printShapeMNK(p, getShapeMnk()); + p << ", "; + printElemTypes(p, getAType(), getBType(), getCType()); + p << ", md_format = " << stringifyEnum(getSparseMetadataFormat()); + printAttrField(p, "int_overflow", getIntOverflow()); + p << " >"; +} +LogicalResult MmaAtomSM80SparseType::verify(EmitFn emitError, cute::ShapeAttr shape, Type, Type, + Type, SparseMetadataFormat, MMAIntOverflowAttr) { + return verifyShapeMNK(emitError, shape); +} + +Type MmaAtomSM89Type::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + cute::ShapeAttr shape; + Type a, b, c; + if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || elemTypes(p, a, b, c) || + p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, a, b, c); +} +void MmaAtomSM89Type::print(AsmPrinter &p) const { + p << '<'; + printShapeMNK(p, getShapeMnk()); + p << ", "; + printElemTypes(p, getAType(), getBType(), getCType()); + p << " >"; +} +LogicalResult MmaAtomSM89Type::verify(EmitFn emitError, cute::ShapeAttr shape, Type, Type, + Type) { + return verifyShapeMNK(emitError, shape); +} + +// ``. +Type MmaAtomSM90Type::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + cute::ShapeAttr shape; + cute::MajorMode aMajor, bMajor; + Type a, b, c; + MmaFragKind frag; + MMAIntOverflowAttr overflow; + UnitAttr aNeg, bNeg; + if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || + p.parseKeyword("ab_major") || p.parseEqual() || pairOf(p, aMajor, bMajor) || + p.parseComma() || elemTypes(p, a, b, c) || p.parseComma() || + parseFragKind(p, MmaFragKind::rmem, frag)) + return {}; + while (succeeded(p.parseOptionalComma())) { + SMLoc at = p.getCurrentLocation(); + if (succeeded(p.parseOptionalKeyword("a_neg"))) aNeg = UnitAttr::get(p.getContext()); + else if (succeeded(p.parseOptionalKeyword("b_neg"))) bNeg = UnitAttr::get(p.getContext()); + else if (succeeded(p.parseOptionalKeyword("int_overflow"))) { + if (p.parseEqual() || field(p, overflow)) return {}; + } else { + p.emitError(at, "expected `int_overflow`, `a_neg`, or `b_neg`"); + return {}; + } + } + if (p.parseGreater()) return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, aMajor, bMajor, a, + b, c, frag, overflow, aNeg, bNeg); +} +void MmaAtomSM90Type::print(AsmPrinter &p) const { + p << '<'; + printShapeMNK(p, getShapeMnk()); + p << ", ab_major = (" << stringifyEnum(getAMajor()) << ", " << stringifyEnum(getBMajor()) + << "), "; + printElemTypes(p, getAType(), getBType(), getCType()); + p << ", frag_kind = " << fragKindText(getAFragKind()); + printAttrField(p, "int_overflow", getIntOverflow()); + if (getANeg()) p << ", a_neg"; + if (getBNeg()) p << ", b_neg"; + p << '>'; +} +LogicalResult MmaAtomSM90Type::verify(EmitFn emitError, cute::ShapeAttr shape, cute::MajorMode, + cute::MajorMode, Type, Type, Type, MmaFragKind, + MMAIntOverflowAttr, UnitAttr, UnitAttr) { + return verifyShapeMNK(emitError, shape); +} + +// ``. +Type MmaAtomSM120BlockScaledType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + cute::ShapeAttr shape; + int vec; + Type a, b, c, sf; + bool useTV; + if (p.parseLess() || parseShapeMNK(p, shape) || nextKeyField(p, "vec_size", vec) || + p.parseComma() || elemTypes(p, a, b, c) || nextKeyField(p, "sf_type", sf) || + nextKeyField(p, "use_sf_layout_TV", useTV) || p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, vec, a, b, c, sf, + useTV); +} +void MmaAtomSM120BlockScaledType::print(AsmPrinter &p) const { + p << '<'; + printShapeMNK(p, getShapeMnk()); + p << ", vec_size = " << getVecSize() << ", "; + printElemTypes(p, getAType(), getBType(), getCType()); + p << ", sf_type = " << getSfType() << ", use_sf_layout_TV = " + << (getUseSfLayout_TV() ? "true" : "false") << '>'; +} +LogicalResult MmaAtomSM120BlockScaledType::verify(EmitFn emitError, cute::ShapeAttr shape, int, + Type, Type, Type, Type, bool) { + return verifyShapeMNK(emitError, shape); +} + +// UMMA atoms: one grammar, fields present by kind (see CuteNVGPUTypes.td). +namespace { +struct Umma { + bool sf, sp, bs, sm107; + cute::ShapeAttr shape; + int numCta = 1; + cute::MajorMode aMajor, bMajor; + Type a, b, c, sfType, eType; + SparseMetadataFormat metadata = SparseMetadataFormat::tid; + MmaFragKind frag = MmaFragKind::smem_desc; + int scaleOrVec = 0; + Arch arch = Arch::sm_100; + MmaCollectorOp aCollector = MmaCollectorOp::discard, bCollector = MmaCollectorOp::discard; + MMAIntOverflowAttr overflow; +}; + +ParseResult parseUmma(AsmParser &p, Umma &u) { + if (p.parseLess() || parseShapeMNK(p, u.shape) || nextKeyField(p, "num_cta", u.numCta) || + p.parseComma() || p.parseKeyword("ab_major") || p.parseEqual() || + pairOf(p, u.aMajor, u.bMajor) || p.parseComma() || elemTypes(p, u.a, u.b, u.c)) + return failure(); + if (u.sf && nextKeyField(p, "sf_type", u.sfType)) return failure(); + if (u.sp && !u.sf && nextKeyField(p, "e_type", u.eType)) return failure(); + if (u.sp && nextKeyField(p, "sparse_metadata_format", u.metadata)) return failure(); + if (p.parseComma() || parseFragKind(p, MmaFragKind::tmem, u.frag) || + nextKeyField(p, u.bs ? "vec_size" : "c_scale_exp", u.scaleOrVec)) + return failure(); + bool comma = succeeded(p.parseOptionalComma()); + if (comma && u.bs && !u.sm107 && succeeded(p.parseOptionalKeyword("arch_promote"))) { + if (p.parseEqual() || field(p, u.arch)) return failure(); + comma = succeeded(p.parseOptionalComma()); + } + if (u.sm107) { + if (!comma) return p.parseComma(); + if (p.parseKeyword("ab_collector_op") || p.parseEqual() || + pairOf(p, u.aCollector, u.bCollector)) + return failure(); + comma = succeeded(p.parseOptionalComma()); + } + if (comma && keyField(p, "int_overflow", u.overflow)) return failure(); + return p.parseGreater(); +} + +void printUmma(AsmPrinter &p, const Umma &u) { + p << '<'; + printShapeMNK(p, u.shape); + p << ", num_cta = " << u.numCta << ", ab_major = (" << stringifyEnum(u.aMajor) << ", " + << stringifyEnum(u.bMajor) << "), "; + printElemTypes(p, u.a, u.b, u.c); + if (u.sf) p << ", sf_type = " << u.sfType; + if (u.sp && !u.sf) p << ", e_type = " << u.eType; + if (u.sp) p << ", sparse_metadata_format = " << stringifyEnum(u.metadata); + p << ", frag_kind = " << fragKindText(u.frag) << (u.bs ? ", vec_size = " : ", c_scale_exp = ") + << u.scaleOrVec; + if (u.bs && !u.sm107 && (u.arch != Arch::sm_100 || u.overflow)) + p << ", arch_promote = " << stringifyEnum(u.arch); + if (u.sm107) + p << ", ab_collector_op = (" << stringifyEnum(u.aCollector) << ", " + << stringifyEnum(u.bCollector) << ")"; + printAttrField(p, "int_overflow", u.overflow); + p << '>'; +} + +// `sourced` picks the atom's wording of the tensor-memory A rule. +LogicalResult verifyUmma(EmitFn emitError, cute::ShapeAttr shape, int numCta, MmaFragKind frag, + cute::MajorMode aMajor, int cScaleExp = 0, bool sourced = false) { + if (failed(verifyShapeMNK(emitError, shape))) return failure(); + if (numCta != 1 && numCta != 2) + return emitError() << "expects `num_cta` equals to 1 or 2, but got " << numCta; + if (cScaleExp > 15) + return emitError() << "expects `c_scale_exp` in range [0, 15], but got " << cScaleExp; + if (frag == MmaFragKind::tmem && aMajor != cute::MajorMode::k) + return emitError() << (sourced ? "a operand must be k-major when sourced from tmem" + : "a operand must be of major-mode k when coming from tmem"); + return success(); +} +} // namespace + +Type MmaAtomSM100UMMAType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Umma u{false, false, false, false}; + if (parseUmma(p, u)) return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, u.numCta, + u.aMajor, u.bMajor, u.a, u.b, u.c, u.frag, u.scaleOrVec, u.overflow); +} +void MmaAtomSM100UMMAType::print(AsmPrinter &p) const { + Umma u{false, false, false, false}; + u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), + u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), + u.frag = getAFragKind(), u.scaleOrVec = getCScaleExp(), u.overflow = getIntOverflow(); + printUmma(p, u); +} +LogicalResult MmaAtomSM100UMMAType::verify(EmitFn emitError, cute::ShapeAttr shape, int numCta, + cute::MajorMode aMajor, cute::MajorMode, Type, Type, + Type, MmaFragKind frag, int cScaleExp, + MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor, cScaleExp); +} + +Type MmaAtomSM100UMMASparseType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Umma u{false, true, false, false}; + if (parseUmma(p, u)) return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, u.numCta, + u.aMajor, u.bMajor, u.a, u.b, u.c, u.eType, u.metadata, u.frag, + u.scaleOrVec, u.overflow); +} +void MmaAtomSM100UMMASparseType::print(AsmPrinter &p) const { + Umma u{false, true, false, false}; + u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), + u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), + u.eType = getEType(), u.metadata = getSparseMetadataFormat(), u.frag = getAFragKind(), + u.scaleOrVec = getCScaleExp(), u.overflow = getIntOverflow(); + printUmma(p, u); +} +LogicalResult MmaAtomSM100UMMASparseType::verify(EmitFn emitError, cute::ShapeAttr shape, + int numCta, cute::MajorMode aMajor, cute::MajorMode, + Type, Type, Type, Type, SparseMetadataFormat, + MmaFragKind frag, int cScaleExp, + MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor, cScaleExp); +} + +Type MmaAtomSM100UMMABlockScaledType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Umma u{true, false, true, false}; + if (parseUmma(p, u)) return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, u.numCta, + u.aMajor, u.bMajor, u.a, u.b, u.c, u.sfType, u.frag, u.scaleOrVec, u.arch, + u.overflow); +} +void MmaAtomSM100UMMABlockScaledType::print(AsmPrinter &p) const { + Umma u{true, false, true, false}; + u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), + u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), + u.sfType = getSfType(), u.frag = getAFragKind(), u.scaleOrVec = getVecSize(), + u.arch = getArchPromote(), u.overflow = getIntOverflow(); + printUmma(p, u); +} +LogicalResult MmaAtomSM100UMMABlockScaledType::verify(EmitFn emitError, cute::ShapeAttr shape, + int numCta, cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, Type, + MmaFragKind frag, int, Arch, + MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor, 0, /*sourced=*/true); +} + +Type MmaAtomSM100UMMABlockScaledSparseType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Umma u{true, true, true, false}; + if (parseUmma(p, u)) return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, u.numCta, + u.aMajor, u.bMajor, u.a, u.b, u.c, u.sfType, u.metadata, u.frag, + u.scaleOrVec, u.arch, u.overflow); +} +void MmaAtomSM100UMMABlockScaledSparseType::print(AsmPrinter &p) const { + Umma u{true, true, true, false}; + u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), + u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), + u.sfType = getSfType(), u.metadata = getSparseMetadataFormat(), u.frag = getAFragKind(), + u.scaleOrVec = getVecSize(), u.arch = getArchPromote(), u.overflow = getIntOverflow(); + printUmma(p, u); +} +LogicalResult MmaAtomSM100UMMABlockScaledSparseType::verify( + EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, cute::MajorMode, + Type, Type, Type, Type, SparseMetadataFormat, MmaFragKind frag, int, Arch, + MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor); +} + +Type MmaAtomSM107UMMAType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Umma u{false, false, false, true}; + if (parseUmma(p, u)) return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, u.numCta, + u.aMajor, u.bMajor, u.a, u.b, u.c, u.frag, u.scaleOrVec, u.aCollector, + u.bCollector, u.overflow); +} +void MmaAtomSM107UMMAType::print(AsmPrinter &p) const { + Umma u{false, false, false, true}; + u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), + u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), + u.frag = getAFragKind(), u.scaleOrVec = getCScaleExp(), u.aCollector = getACollectorOp(), + u.bCollector = getBCollectorOp(), u.overflow = getIntOverflow(); + printUmma(p, u); +} +LogicalResult MmaAtomSM107UMMAType::verify(EmitFn emitError, cute::ShapeAttr shape, int numCta, + cute::MajorMode aMajor, cute::MajorMode, Type, Type, + Type, MmaFragKind frag, int cScaleExp, + MmaCollectorOp, MmaCollectorOp, MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor, cScaleExp); +} + +Type MmaAtomSM107UMMASparseType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Umma u{false, true, false, true}; + if (parseUmma(p, u)) return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, u.numCta, + u.aMajor, u.bMajor, u.a, u.b, u.c, u.eType, u.metadata, u.frag, + u.scaleOrVec, u.aCollector, u.bCollector, u.overflow); +} +void MmaAtomSM107UMMASparseType::print(AsmPrinter &p) const { + Umma u{false, true, false, true}; + u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), + u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), + u.eType = getEType(), u.metadata = getSparseMetadataFormat(), u.frag = getAFragKind(), + u.scaleOrVec = getCScaleExp(), u.aCollector = getACollectorOp(), + u.bCollector = getBCollectorOp(), u.overflow = getIntOverflow(); + printUmma(p, u); +} +LogicalResult MmaAtomSM107UMMASparseType::verify(EmitFn emitError, cute::ShapeAttr shape, + int numCta, cute::MajorMode aMajor, cute::MajorMode, + Type, Type, Type, Type, SparseMetadataFormat, + MmaFragKind frag, int cScaleExp, MmaCollectorOp, + MmaCollectorOp, MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor, cScaleExp); +} + +Type MmaAtomSM107UMMABlockScaledType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Umma u{true, false, true, true}; + if (parseUmma(p, u)) return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, u.numCta, + u.aMajor, u.bMajor, u.a, u.b, u.c, u.sfType, u.frag, u.scaleOrVec, + u.aCollector, u.bCollector, u.overflow); +} +void MmaAtomSM107UMMABlockScaledType::print(AsmPrinter &p) const { + Umma u{true, false, true, true}; + u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), + u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), + u.sfType = getSfType(), u.frag = getAFragKind(), u.scaleOrVec = getVecSize(), + u.aCollector = getACollectorOp(), u.bCollector = getBCollectorOp(), + u.overflow = getIntOverflow(); + printUmma(p, u); +} +LogicalResult MmaAtomSM107UMMABlockScaledType::verify(EmitFn emitError, cute::ShapeAttr shape, + int numCta, cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, Type, + MmaFragKind frag, int, MmaCollectorOp, + MmaCollectorOp, MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor, 0, /*sourced=*/true); +} + +Type MmaAtomSM107UMMABlockScaledSparseType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Umma u{true, true, true, true}; + if (parseUmma(p, u)) return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, u.numCta, + u.aMajor, u.bMajor, u.a, u.b, u.c, u.sfType, u.metadata, u.frag, + u.scaleOrVec, u.aCollector, u.bCollector, u.overflow); +} +void MmaAtomSM107UMMABlockScaledSparseType::print(AsmPrinter &p) const { + Umma u{true, true, true, true}; + u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), + u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), + u.sfType = getSfType(), u.metadata = getSparseMetadataFormat(), u.frag = getAFragKind(), + u.scaleOrVec = getVecSize(), u.aCollector = getACollectorOp(), + u.bCollector = getBCollectorOp(), u.overflow = getIntOverflow(); + printUmma(p, u); +} +LogicalResult MmaAtomSM107UMMABlockScaledSparseType::verify( + EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, cute::MajorMode, + Type, Type, Type, Type, SparseMetadataFormat, MmaFragKind frag, int, MmaCollectorOp, + MmaCollectorOp, MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor, 0, /*sourced=*/true); +} + +//===----------------------------------------------------------------------===// +// Dialect +//===----------------------------------------------------------------------===// + +// `!cute_nvgpu.iN[]` is the compiler's alias of `!cute.iN[]`. +Type CuteNVGPUDialect::parseType(DialectAsmParser &parser) const { + SMLoc loc = parser.getCurrentLocation(); + StringRef mnemonic; + Type type; + OptionalParseResult result = generatedTypeParser(parser, &mnemonic, type); + if (result.has_value()) return succeeded(*result) ? type : Type(); + StringRef width = mnemonic; + if (width.consume_front("i") && !width.empty() && llvm::all_of(width, llvm::isDigit)) { + std::string spec = ("!cute." + mnemonic).str(); + int64_t divisor; + if (succeeded(parser.parseOptionalLess())) { + if (parser.parseKeyword("divby") || parser.parseInteger(divisor) || parser.parseGreater()) + return {}; + spec += ""; + } + return mlir::parseType(spec, getContext()); + } + parser.emitError(loc) << "unknown type `" << mnemonic << "` in dialect `" + << getNamespace() << "`"; + return {}; } void CuteNVGPUDialect::printType(Type type, DialectAsmPrinter &printer) const { - if (succeeded(generatedTypePrinter(type, printer))) - return; + if (succeeded(generatedTypePrinter(type, printer))) return; llvm_unreachable("unexpected non-CuteNVGPU type"); } void CuteNVGPUDialect::initialize() { + addAttributes< +#define GET_ATTRDEF_LIST +#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUAttrs.cpp.inc" + >(); addTypes< #define GET_TYPEDEF_LIST #include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUTypes.cpp.inc" @@ -126,10 +1656,4 @@ void CuteNVGPUDialect::initialize() { #define GET_OP_LIST #include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUOps.cpp.inc" >(); - - // Preserve every not-yet-structured private type spelling exactly. MLIR's - // default hook stores the complete symbol specification in OpaqueType. - allowUnknownTypes(); } - -#undef CUTE_NVGPU_PAYLOAD_TYPES diff --git a/mlir/dialects/cute_ir/lib/Dialect/IR/BUILD.bazel b/mlir/dialects/cute_ir/lib/Dialect/IR/BUILD.bazel index 3ccc2f2d..ba8a35ff 100644 --- a/mlir/dialects/cute_ir/lib/Dialect/IR/BUILD.bazel +++ b/mlir/dialects/cute_ir/lib/Dialect/IR/BUILD.bazel @@ -2,18 +2,22 @@ load("@rules_cc//cc:defs.bzl", "cc_library") package(default_visibility = ["//visibility:public"]) +# CuteTypes.cpp is the release's with the compiler's types, attributes and +# enums added and ratio strides in scaled bases allowed (`1/2@0`, how a 2-CTA +# TMA partition addresses its halves). cc_library( name = "dialect", srcs = [ - "CuteDialect.cpp", - "CuteOpsPrivate.cpp", + "CuteInference.cpp", + "CuteOps.cpp", "CuteTypes.cpp", - "CuteTypesPrivate.cpp", + "@cute_ir//:cute_ir/lib/Dialect/IR/CuteAttributes.cpp", + "@cute_ir//:cute_ir/lib/Dialect/IR/CuteDialect.cpp", ], deps = [ - "//mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR:private_headers", - "//mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR:types_all_defs_inc_gen", - "@cute_ir//:dialect_support", + "//mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR:headers", + "//mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR:headers", + "@llvm-project//llvm:Support", "@llvm-project//mlir:ArithDialect", "@llvm-project//mlir:IR", "@llvm-project//mlir:InferTypeOpInterface", diff --git a/mlir/dialects/cute_ir/lib/Dialect/IR/CuteDialect.cpp b/mlir/dialects/cute_ir/lib/Dialect/IR/CuteDialect.cpp deleted file mode 100755 index 083f0347..00000000 --- a/mlir/dialects/cute_ir/lib/Dialect/IR/CuteDialect.cpp +++ /dev/null @@ -1,59 +0,0 @@ -// clang-format off -/*************************************************************************************************** - * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: BSD-3-Clause - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -// clang-format on - -#include "cute_ir/Dialect/Cute/IR/CuteDialectPrivate.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/Builders.h" -#include "mlir/IR/DialectImplementation.h" - -using namespace mlir; -using namespace mlir::cutlass_compiler::cute; - -#include "cute_ir/Dialect/Cute/IR/CuteDialect.cpp.inc" -#include "cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.cpp.inc" - -void CuteDialect::initialize() { - addOperations< -#define GET_OP_LIST -#include "cute_ir/Dialect/Cute/IR/CuteOps.cpp.inc" - >(); - addOperations< -#define GET_OP_LIST -#include "cute_ir/Dialect/Cute/IR/CuteOpsPrivate.cpp.inc" - >(); - CuteDialect::registerCuteAttributes(this); - CuteDialect::registerCuteTypes(this); -} - -// registerCuteAttributes is implemented in CuteAttributes.cpp. diff --git a/mlir/dialects/cute_ir/lib/Dialect/IR/CuteInference.cpp b/mlir/dialects/cute_ir/lib/Dialect/IR/CuteInference.cpp new file mode 100644 index 00000000..e3440079 --- /dev/null +++ b/mlir/dialects/cute_ir/lib/Dialect/IR/CuteInference.cpp @@ -0,0 +1,2982 @@ +//===- CuteInference.cpp - result types the DSL compiler would infer -----===// +// +// Every operation the DSL compiler infers result types for, inferred here +// over cutegen, so a module built in this tree carries the types the compiler +// expects. Where the compiler's rule is not recovered for some inputs (other +// atoms, tilings or layouts than those covered), inference fails rather than +// guesses. test:dialect_inference_test holds these to the compiler's choice. + +#include +#include +#include +#include +#include +#include +#include + +#include "cute_ir/Dialect/Cute/IR/CuteInference.h" +#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/TypeSwitch.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Matchers.h" + +using namespace mlir; +using namespace mlir::cutlass_compiler::cute; +namespace cg = cutegen; +namespace cute_nvgpu = mlir::cutlass_compiler::cute_nvgpu; +// The dialect's memref, not the builtin one. +using CuteMemRefType = ::mlir::cutlass_compiler::cute::MemRefType; + +namespace { + +// Composed layouts that turned out plain are typed as plain layouts, as the +// compiler types them. +Type layoutType(MLIRContext *ctx, cg::layout l) { + return LayoutType::get(ctx, std::move(l)); +} +Type layoutType(MLIRContext *ctx, cg::composed_layout l) { + if (l.is_normal_layout()) return LayoutType::get(ctx, l.layout_b()); + return ComposedLayoutType::get(ctx, std::move(l)); +} + +// `f` runs over the algebra value of a layout or composed layout type; the +// result keeps the input's kind (see rebuild for views). +template +Type mapLayout(MLIRContext *ctx, Type layout, F &&f) { + if (auto l = llvm::dyn_cast(layout)) { + auto r = f(l.getRef()); + if (!cg::is_valid(r)) return {}; + return layoutType(ctx, std::move(r)); + } + if (auto l = llvm::dyn_cast(layout)) { + auto r = f(l.getRef()); + if (!cg::is_valid(r)) return {}; + return ComposedLayoutType::get(ctx, std::move(r)); + } + return {}; +} + +// What carries a layout: the layout itself, a memref (pointer + layout) or a +// coordinate tensor (arithmetic tuple + layout). +struct Carrier { + Type layout; + PtrType ptr; // memref + IntTupleType iter; // coord_tensor + Type desc; // cute_nvgpu.smem_desc_view +}; + +std::optional carrierOf(Type t) { + if (llvm::isa(t)) return Carrier{t, {}, {}, {}}; + if (auto m = llvm::dyn_cast(t)) + return Carrier{m.getLayout(), m.getPtr(), {}, {}}; + if (auto c = llvm::dyn_cast(t)) + return Carrier{c.getLayout(), {}, c.getArithTuple(), {}}; + if (auto v = llvm::dyn_cast(t)) + return Carrier{LayoutType::get(t.getContext(), v.getLayout()), {}, {}, v.getDesc()}; + return std::nullopt; +} + +Type rebuild(MLIRContext *ctx, const Carrier &c, Type layout) { + if (!layout) return {}; + // A view's layout is plain when its composed layout turned out plain. + if (auto l = llvm::dyn_cast(layout); l && (c.ptr || c.iter || c.desc)) + layout = layoutType(ctx, l.getRef()); + if (c.ptr) return CuteMemRefType::get(ctx, c.ptr, layout); + if (c.iter) return CoordTensorType::get(ctx, c.iter, layout); + if (c.desc) { + auto l = llvm::dyn_cast(layout); + return l ? cute_nvgpu::SmemDescViewType::get(ctx, c.desc, l.getAttr()) : Type(); + } + return layout; +} + +// The bit width of an element type; tf32 counts as its 32-bit storage. +int64_t bitsOfType(Type t) { + if (t.isTF32()) return 32; + return t.isIntOrFloat() ? t.getIntOrFloatBitWidth() : 0; +} + +// The alignment a pointer keeps after moving by `offset` elements: the gcd +// with what is known of the offset, a static value or a divisibility; the +// element's natural alignment when less than a byte is known. +uint64_t movedAlignment(PtrType ptr, const cg::int_tuple &offset) { + Type elem = ptr.getValueType(); + uint64_t bits = elem && elem.isIntOrFloat() ? bitsOfType(elem) : 8; + uint64_t natural = PtrType::getNaturalAlignment(elem); + uint64_t align = ptr.getAlignment() ? ptr.getAlignment() : natural; + uint64_t known = 0; + if (cg::holds_int(offset)) { + int64_t v = offset.as_int64(); + if (v == 0) return ptr.getAlignment(); + known = static_cast(v < 0 ? -v : v); + } else if (cg::holds_dynamic_int(offset)) { + known = static_cast( + std::get(offset).get_properties().divisibility); + } else { + return natural; + } + uint64_t result = std::gcd(align * 8, known * bits) / 8; + return result ? result : natural; +} + +PtrType movedPtr(PtrType ptr, const cg::int_tuple &offset) { + return PtrType::get(ptr.getContext(), ptr.getValueType(), ptr.getAddressSpace(), + movedAlignment(ptr, offset), ptr.getSwizzle(), + ptr.getBitlayout()); +} + +// A coordinate with its underscores read as zero, for evaluating a layout. +std::optional zeroUnderscores(const cg::coord &c) { + std::string text = cg::to_string(c); + std::replace(text.begin(), text.end(), '_', '0'); + return cg::from_string(text); +} + +// Slicing a carrier: the layout is sliced and the iterator moves by the +// layout evaluated at the coordinate. +Type sliceCarrier(MLIRContext *ctx, const Carrier &c, const cg::coord &coord) { + cg::int_tuple offset; + Type layout; + if (auto l = llvm::dyn_cast(c.layout)) { + auto [sliced, off] = cg::slice_and_offset(coord, l.getRef()); + if (!cg::is_valid(sliced)) return {}; + layout = layoutType(ctx, std::move(sliced)); + offset = off; + } else if (auto l = llvm::dyn_cast(c.layout)) { + // The composed layout's offset takes the move; the iterator stays. + auto sliced = cg::slice(coord, l.getRef()); + if (!cg::is_valid(sliced)) return {}; + return rebuild(ctx, c, layoutType(ctx, std::move(sliced))); + } else { + return {}; + } + if (c.ptr) return CuteMemRefType::get(ctx, movedPtr(c.ptr, offset), layout); + if (c.iter) { + auto sum = cg::arith_tuple_sum(c.iter.getRef(), offset); + if (!cg::is_valid(sum)) return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), + layout); + } + // A descriptor iterator does not move. + return rebuild(ctx, c, layout); +} + +// The compiler writes a stride of 0 on every mode of extent 1 that a divide +// produces; cutegen keeps the stride the algebra gave it. +cg::layout zeroUnitModes(cg::layout l) { + if (cg::holds_leaf(l.shape())) { + if (cg::holds_int(l.shape()) && l.shape().as_int64() == 1) { + if (auto unit = cg::from_string("1:0")) return *unit; + } + return l; + } + // Shapes and strides are congruent, so walk both spellings together. + auto shape = cg::to_string(l.shape()); + auto stride = cg::to_string(l.stride()); + auto fs = cg::flatten(l.shape()); + auto ft = cg::flatten(l.stride()); + if (cg::rank(fs) != cg::rank(ft)) return l; + bool changed = false; + for (size_t i = 0; i < cg::rank(fs); ++i) { + if (cg::holds_int(fs[i]) && fs[i].as_int64() == 1 && + !(cg::holds_int(ft[i]) && ft[i].as_int64() == 0)) { + changed = true; + } + } + if (!changed) return l; + // Rewrite the stride text leaf by leaf, in flattened order. + std::string out; + size_t leaf = 0; + auto stridesAreLeaves = [&](size_t i) { + return cg::holds_int(fs[i]) && fs[i].as_int64() == 1; + }; + size_t i = 0; + while (i < stride.size()) { + char ch = stride[i]; + if (ch == '(' || ch == ')' || ch == ',') { + out += ch; + ++i; + continue; + } + size_t j = i; + int depth = 0; + while (j < stride.size() && + !((stride[j] == ',' || stride[j] == ')') && depth == 0)) { + if (stride[j] == '<' || stride[j] == '{') ++depth; + if (stride[j] == '>' || stride[j] == '}') --depth; + ++j; + } + out += stridesAreLeaves(leaf) ? std::string("0") : stride.substr(i, j - i); + ++leaf; + i = j; + } + auto rebuilt = cg::from_string(shape + ":" + out); + return rebuilt ? *rebuilt : l; +} + +const cg::layout &outerOf(const cg::layout &l) { return l; } +cg::layout outerOf(const cg::composed_layout &l) { return l.layout_b(); } + +cg::composed_layout zeroUnitModes(cg::composed_layout l) { + if (l.is_a_swizzle()) + return cg::composed_layout(l.swizzle_a(), l.offset(), zeroUnitModes(l.layout_b())); + return cg::composed_layout(l.layout_a(), l.offset(), zeroUnitModes(l.layout_b())); +} + +template +LogicalResult infer(std::optional loc, SmallVectorImpl &out, + F &&f) { + Type t = f(); + if (!t) return emitOptionalError(loc, "cannot infer the result type"); + out.push_back(t); + return success(); +} + +// `like`'s nesting with its leaves replaced, in order, by `leaves`. +template +std::string nestLike(const T &like, llvm::ArrayRef leaves, size_t &i) { + if (!cg::holds_vector(like)) return i < leaves.size() ? leaves[i++] : "x"; + std::string s = "("; + for (size_t k = 0; k < cg::rank(like); ++k) + s += (k ? "," : "") + nestLike(like[k], leaves, i); + return s + ")"; +} + +template +void leavesOf(const T &t, std::vector &out) { + if (!cg::holds_vector(t)) return out.push_back(cg::to_string(t)); + for (size_t k = 0; k < cg::rank(t); ++k) leavesOf(t[k], out); +} + +bool isOne(const std::string &leaf) { return leaf == "1"; } + +// The plain layout of a layout, or the outer layout of a composed one. +std::optional outerLayout(Type t) { + if (auto l = llvm::dyn_cast(t)) return l.getRef(); + if (auto c = llvm::dyn_cast(t)) return c.getRef().layout_b(); + return std::nullopt; +} + +// The plain layout of a layout or swizzled composed layout. +std::optional swizzledOuter(Type t) { + if (auto l = llvm::dyn_cast(t)) return l.getRef(); + auto c = llvm::dyn_cast(t); + if (c && c.getRef().is_a_swizzle()) return c.getRef().layout_b(); + return std::nullopt; +} + +// A dynamic integer's spelling: `?`, with its width when 64 and its +// divisibility when above 1. +std::string dynamicInt(int64_t div, bool wide) { + std::string props = std::string(wide ? "i64" : "") + (wide && div > 1 ? " " : "") + + (div > 1 ? "div=" + std::to_string(div) : ""); + return props.empty() ? "?" : "?{" + props + "}"; +} + +// idx2crd of a scalar `idx` over `shape` (column-major), keeping what a +// dynamic index's divisibility tells of each coordinate, as the compiler +// does (cutegen drops it): a divisible mode's coordinate is 0; a mode whose +// extent the divisibility divides keeps it; the rest know nothing. +std::optional splitIndex(const cg::int_tuple &idx, const cg::shape &shape) { + std::vector leaves, crd; + leavesOf(shape, leaves); + bool known = cg::holds_int(idx); + if (!known && !cg::holds_dynamic_int(idx)) return std::nullopt; + int64_t v = known ? idx.as_int64() : 0, div = 1; + bool wide = false; + if (!known) { + const auto &p = std::get(idx).get_properties(); + div = p.divisibility; + wide = p.width == 64; + } + for (size_t i = 0; i < leaves.size(); ++i) { + int64_t n = 0; + bool staticExtent = !StringRef(leaves[i]).getAsInteger(10, n) && n > 0; + bool last = i + 1 == leaves.size(); + if (known) { + if (!staticExtent) return std::nullopt; + crd.push_back(std::to_string(last ? v : v % n)); + v /= n; + } else if (last) { + crd.push_back(dynamicInt(div, wide)); + } else if (staticExtent && div % n == 0) { + crd.push_back("0"); + div /= n; + } else { + crd.push_back(dynamicInt(staticExtent && n % div == 0 ? div : 1, wide)); + div = 1; + } + } + size_t i = 0; + return nestLike(shape, crd, i); +} +} // namespace + +// ---------------------------------------------------------------- cute + +LogicalResult AddOffsetOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto offset = llvm::dyn_cast(a.getOffset().getType()); + if (!offset) return emitOptionalError(loc, "offset must be an int tuple"); + return infer(loc, out, [&]() -> Type { + Type src = a.getSrc().getType(); + if (auto p = llvm::dyn_cast(src)) return movedPtr(p, offset.getRef()); + if (auto it = llvm::dyn_cast(src)) { + auto sum = cg::arith_tuple_sum(it.getArithTuple().getRef(), + offset.getRef()); + if (!cg::is_valid(sum)) return {}; + return ArithTupleIteratorType::get(ctx, IntTupleType::get(ctx, sum)); + } + return {}; + }); +} + +namespace { +// Pads `input` to `rank` modes with `element`, after (append) or before. +template +LogicalResult inferToRank(MLIRContext *ctx, std::optional loc, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, SmallVectorImpl &out, + bool append) { + typename Op::Adaptor a(operands, attrs, props); + int32_t rank = a.getRank(); + Type in = a.getInput().getType(), e = a.getElement().getType(); + return infer(loc, out, [&]() -> Type { + if (rank <= 0) return {}; + return llvm::TypeSwitch(in) + .Case([&](auto ty) -> Type { + using T = decltype(ty); + // A composed layout is padded with plain layouts. + using E = std::conditional_t, LayoutType, T>; + auto el = llvm::dyn_cast(e); + if (!el) return {}; + auto r = append ? cg::append_to_rank_N(rank, ty.getRef(), el.getRef()) + : cg::prepend_to_rank_N(rank, ty.getRef(), el.getRef()); + if (!cg::is_valid(r)) return {}; + return T::get(ctx, std::move(r)); + }) + .Default([](Type) -> Type { return {}; }); + }); +} +} // namespace + +LogicalResult AppendToRankOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferToRank(ctx, loc, operands, attrs, props, out, true); +} + +LogicalResult PrependToRankOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferToRank(ctx, loc, operands, attrs, props, out, false); +} + +Type mlir::cutlass_compiler::cute::assumeResultType(Type src) { + auto constrained = llvm::dyn_cast(src); + for (unsigned width : {64u, 32u}) + if (src.isInteger(width) || (constrained && constrained.getWidth() == width)) + return ConstrainedIntType::get(src.getContext(), 1, width, false); + return {}; +} + +LogicalResult CeilDivOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + Type in = a.getInput().getType(), tiler = a.getTiler().getType(); + return infer(loc, out, [&]() -> Type { + // cutegen divides shapes; int tuples go through shape and back. + cg::shape shape; + bool tuple = false; + if (auto s = llvm::dyn_cast(in)) { + shape = s.getRef(); + } else if (auto t = llvm::dyn_cast(in)) { + shape = cg::rec_var_cast(t.getRef()); + tuple = true; + } else { + return {}; + } + cg::shape r; + if (auto t = llvm::dyn_cast(tiler)) { + r = cg::ceil_div(shape, t.getRef()); + } else if (auto t = llvm::dyn_cast(tiler)) { + r = cg::ceil_div(shape, t.getRef()); + } else if (auto t = llvm::dyn_cast(tiler)) { + r = cg::ceil_div(shape, cg::rec_var_cast(t.getRef())); + } else { + return {}; + } + if (!cg::is_valid(r)) return {}; + if (tuple) return IntTupleType::get(ctx, cg::rec_var_cast(r)); + return ShapeType::get(ctx, std::move(r)); + }); +} + +LogicalResult CoalesceOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto c = carrierOf(a.getInput().getType()); + if (!c) return emitOptionalError(loc, "input carries no layout"); + CoordType profile; + if (a.getTargetProfile()) + profile = llvm::dyn_cast(a.getTargetProfile().getType()); + return infer(loc, out, [&]() -> Type { + return rebuild(ctx, *c, mapLayout(ctx, c->layout, [&](const auto &l) { + return profile ? cg::coalesce(l, profile.getRef()) : cg::coalesce(l); + })); + }); +} + +LogicalResult ComposedGetOuterOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto c = llvm::dyn_cast(operands[0].getType()); + if (!c) return {}; + return LayoutType::get(ctx, c.getRef().layout_b()); + }); +} + +LogicalResult CompositionOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto c = carrierOf(a.getLhs().getType()); + if (!c) return emitOptionalError(loc, "lhs carries no layout"); + Type rhs = a.getRhs().getType(); + return infer(loc, out, [&]() -> Type { + return rebuild(ctx, *c, llvm::TypeSwitch(rhs) + .Case([&](auto r) { + return mapLayout(ctx, c->layout, [&](const auto &l) { + return cg::composition(l, r.getRef()); + }); + }) + .Default([](Type) -> Type { return {}; })); + }); +} + +LogicalResult Crd2IdxOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto coord = llvm::dyn_cast(a.getCoord().getType()); + if (!coord) return emitOptionalError(loc, "coord must be a coord"); + return infer(loc, out, [&]() -> Type { + auto c = zeroUnderscores(coord.getRef()); + if (!c) return {}; + Type layout = a.getLayout().getType(); + cg::int_tuple idx; + if (auto l = llvm::dyn_cast(layout)) { + idx = cg::layout_eval(*c, l.getRef()); + } else if (auto s = llvm::dyn_cast(layout)) { + idx = cg::crd2idx(*c, s.getRef()); + } else { + return {}; + } + if (!cg::is_valid(idx)) return {}; + return IntTupleType::get(ctx, std::move(idx)); + }); +} + +LogicalResult DereferenceArithTupleIteratorOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto it = llvm::dyn_cast(operands[0].getType()); + return it ? it.getArithTuple() : Type{}; + }); +} + +LogicalResult DiceOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + const cg::coord &coord = a.getCoordAttr().getRef(); + Type in = a.getInput().getType(); + return infer(loc, out, [&]() -> Type { + if (auto c = carrierOf(in)) { + return rebuild(ctx, *c, mapLayout(ctx, c->layout, [&](const auto &l) { + return cg::dice(coord, l); + })); + } + return llvm::TypeSwitch(in) + .Case([&](auto ty) -> Type { + auto r = cg::dice(coord, ty.getRef()); + if (!cg::is_valid(r)) return {}; + return decltype(ty)::get(ctx, std::move(r)); + }) + .Default([](Type) -> Type { return {}; }); + }); +} + +Type mlir::cutlass_compiler::cute::fastDivmodComputeResultType(Type divisor) { + auto d = llvm::dyn_cast(divisor); + return d ? IntegerType::get(divisor.getContext(), d.getWidth()) : Type(); +} + +Type mlir::cutlass_compiler::cute::fastDivmodCreateDivisorResultType(Type divisor) { + auto i = llvm::dyn_cast(divisor); + return i ? FastDivmodDivisorType::get(divisor.getContext(), i.getWidth(), false) : Type(); +} + +LogicalResult FilterZerosOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto c = carrierOf(a.getInput().getType()); + if (!c) return emitOptionalError(loc, "input carries no layout"); + return infer(loc, out, [&]() -> Type { + return rebuild(ctx, *c, mapLayout(ctx, c->layout, [&](const auto &l) { + return cg::filter_zeros(l); + })); + }); +} + +namespace { +// zeroUnitModes on all but the last `keep` modes (of mode 1 when `zipped`): +// the modes a divide leaves untouched keep their strides. +cg::layout zeroUnitModesHead(const cg::layout &l, size_t keep, bool zipped) { + if (!keep || !cg::holds_vector(l.shape())) return zeroUnitModes(l); + if (zipped) { + if (cg::rank(l.shape()) != 2) return zeroUnitModes(l); + return cg::make_layout(std::vector{ + zeroUnitModes(cg::get(l, 0)), zeroUnitModesHead(cg::get(l, 1), keep, false)}); + } + size_t n = cg::rank(l.shape()); + if (keep > n) return l; + std::vector modes; + for (size_t i = 0; i < n; ++i) + modes.push_back(i < n - keep ? zeroUnitModes(cg::get(l, i)) : cg::get(l, i)); + return cg::make_layout(modes); +} +cg::composed_layout zeroUnitModesHead(const cg::composed_layout &l, size_t keep, bool zipped) { + cg::layout b = zeroUnitModesHead(l.layout_b(), keep, zipped); + if (l.is_a_swizzle()) return cg::composed_layout(l.swizzle_a(), l.offset(), b); + return cg::composed_layout(l.layout_a(), l.offset(), b); +} + +template +LogicalResult inferDivide(MLIRContext *ctx, std::optional loc, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, SmallVectorImpl &out, + F &÷, bool zipped = false) { + typename Op::Adaptor a(operands, attrs, props); + auto c = carrierOf(a.getInput().getType()); + if (!c) return emitOptionalError(loc, "input carries no layout"); + Type tiler = a.getTiler().getType(); + return infer(loc, out, [&]() -> Type { + return rebuild(ctx, *c, llvm::TypeSwitch(tiler) + .template Case([&](auto t) { + // Unit modes the tiler passes through (`_` or extent 1) keep their + // nonzero strides, which zeroUnitModes does not model: not covered. + if constexpr (std::is_same_v) { + auto outer = outerLayout(c->layout); + for (size_t i = 0; outer && cg::holds_vector(t.getRef()) && i < cg::rank(t.getRef()) && + cg::holds_vector(outer->shape()) && i < cg::rank(outer->shape()); + ++i) { + std::string m = cg::to_string(t.getRef()[i]); + auto tl = cg::from_string(m); + bool pass = m == "_" || (tl && cg::holds_int(cg::size(*tl)) && + cg::size(*tl).as_int64() == 1 && cg::to_string(tl->stride()) != "0"); + std::vector shapes, strides; + leavesOf(outer->shape()[i], shapes); + leavesOf(outer->stride()[i], strides); + for (size_t k = 0; pass && k < shapes.size() && k < strides.size(); ++k) + if (shapes[k] == "1" && strides[k] != "0") return Type(); + } + } + return mapLayout(ctx, c->layout, [&](const auto &l) { + // A tile or shape tiler leaves the input's trailing modes alone. + size_t keep = 0; + if constexpr (!std::is_same_v) { + size_t in = cg::rank(outerOf(l).shape()), r = cg::rank(t.getRef()); + if (cg::holds_vector(outerOf(l).shape()) && r < in) keep = in - r; + } + return zeroUnitModesHead(divide(l, t.getRef()), keep, zipped); + }); + }) + .Default([](Type) -> Type { return {}; })); + }); +} +} // namespace + +LogicalResult FlatDivideOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferDivide(ctx, loc, operands, attrs, props, out, + [](const auto &l, const auto &t) { return cg::flat_divide(l, t); }); +} + +LogicalResult LogicalDivideOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferDivide(ctx, loc, operands, attrs, props, out, + [](const auto &l, const auto &t) { return cg::logical_divide(l, t); }); +} + +LogicalResult TiledDivideOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferDivide(ctx, loc, operands, attrs, props, out, + [](const auto &l, const auto &t) { return cg::tiled_divide(l, t); }); +} + +LogicalResult ZippedDivideOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferDivide(ctx, loc, operands, attrs, props, out, + [](const auto &l, const auto &t) { return cg::zipped_divide(l, t); }, true); +} + +Type mlir::cutlass_compiler::cute::getResultType(Type in, ArrayRef mode) { + MLIRContext *ctx = in.getContext(); + return llvm::TypeSwitch(in) + .Case([&](auto ty) -> Type { + auto r = cg::get(ty.getRef(), mode); + if (!cg::is_valid(r)) return {}; + if constexpr (std::is_same_v) + return layoutType(ctx, std::move(r)); + else + return decltype(ty)::get(ctx, std::move(r)); + }) + .Default([](Type) -> Type { return {}; }); +} + +LogicalResult GetIterOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + Type src = operands[0].getType(); + if (auto m = llvm::dyn_cast(src)) return m.getPtr(); + if (auto c = llvm::dyn_cast(src)) + return ArithTupleIteratorType::get(ctx, c.getArithTuple()); + if (auto v = llvm::dyn_cast(src)) return v.getDesc(); + return {}; + }); +} + +LogicalResult GetLayoutOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto c = carrierOf(operands[0].getType()); + return c ? c->layout : Type{}; + }); +} + +LogicalResult GetLeavesOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return llvm::TypeSwitch(operands[0].getType()) + .Case( + [&](auto ty) -> LogicalResult { + auto flat = cg::flatten(ty.getRef()); + for (size_t i = 0; i < cg::rank(flat); ++i) + out.push_back(decltype(ty)::get(ctx, flat[i])); + return success(); + }) + .Case([&](TileType ty) -> LogicalResult { + // A tile's leaves: its layouts, and `_` as a tile. + auto flat = cg::flatten(ty.getRef()); + for (size_t i = 0; i < cg::rank(flat); ++i) { + flat[i].visit([&](const auto &e) { + using T = std::decay_t; + if constexpr (cg::is_layout::value) + out.push_back(LayoutType::get(ctx, e)); + else if constexpr (!std::is_same_v>) + out.push_back(TileType::get(ctx, flat[i])); + }); + } + return success(); + }) + .Default([&](Type t) -> LogicalResult { + return emitOptionalError(loc, "cannot take the leaves of ", t); + }); +} + +LogicalResult GetScalarsOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + bool onlyDynamic = a.getOnlyDynamic(); + return llvm::TypeSwitch(a.getCuteValue().getType()) + .Case([&](auto ty) -> LogicalResult { + SmallVector scalars; + cg::collect_scalar_types(ctx, ty.getRef(), scalars, onlyDynamic); + out.append(scalars.begin(), scalars.end()); + return success(); + }) + .Default([&](Type t) -> LogicalResult { + return emitOptionalError(loc, "cannot take the scalars of ", t); + }); +} + +LogicalResult GetShapeOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + Type in = operands[0].getType(); + if (auto t = llvm::dyn_cast(in)) + return ShapeType::get(ctx, cg::extract_shape_from_tile(t.getRef())); + auto c = carrierOf(in); + if (!c) return {}; + if (auto l = llvm::dyn_cast(c->layout)) + return ShapeType::get(ctx, l.getRef().shape()); + if (auto l = llvm::dyn_cast(c->layout)) + return ShapeType::get(ctx, l.getRef().shape()); + return {}; + }); +} + +LogicalResult GroupModesOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto c = carrierOf(a.getInput().getType()); + if (!c) return emitOptionalError(loc, "input carries no layout"); + int begin = static_cast(a.getBegin()), end = static_cast(a.getEnd()); + return infer(loc, out, [&]() -> Type { + return rebuild(ctx, *c, mapLayout(ctx, c->layout, [&](const auto &l) { + return cg::group(begin, end, l); + })); + }); +} + +LogicalResult LocalTileOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto c = carrierOf(a.getInput().getType()); + auto tile = llvm::dyn_cast(a.getTile().getType()); + auto coord = llvm::dyn_cast(a.getCoord().getType()); + if (!c || !tile || !coord) + return emitOptionalError(loc, "expects a layout carrier, a tile and a coord"); + return infer(loc, out, [&]() -> Type { + // CuTe's inner_partition: divide by the tile, take the coordinate in the + // rest modes, and flatten the two modes into one rank. + cg::tile tiler = tile.getRef(); + cg::coord crd = coord.getRef(); + if (CoordAttr proj = a.getProjAttr()) { + tiler = cg::dice(proj.getRef(), tiler); + crd = cg::dice(proj.getRef(), crd); + } + auto layout = llvm::dyn_cast(c->layout); + if (!layout) return {}; + cg::layout zipped = cg::zipped_divide(layout.getRef(), tiler); + if (!cg::is_valid(zipped)) return {}; + cg::layout tiles = cg::get(zipped, 0), rest = cg::get(zipped, 1); + auto [sliced, offset] = cg::slice_and_offset(crd, rest); + if (!cg::is_valid(sliced)) return {}; + // One rank: the tile modes, then what is left of the rest modes. + std::vector modes; + for (size_t i = 0; i < cg::rank(tiles.shape()); ++i) + modes.push_back(cg::get(tiles, i)); + if (cg::holds_leaf(sliced.shape())) { + modes.push_back(sliced); + } else { + for (size_t i = 0; i < cg::rank(sliced.shape()); ++i) + modes.push_back(cg::get(sliced, i)); + } + // The input's modes beyond the tiler that the coordinate keeps are + // untouched and keep their strides. + size_t keep = 0, in = cg::rank(layout.getRef().shape()), r = cg::rank(tiler); + for (size_t i = r; cg::holds_vector(crd) && i < in && i < cg::rank(crd); ++i) + keep += cg::to_string(crd[i]) == "_"; + cg::layout flat = zeroUnitModesHead(cg::make_layout(modes), keep, false); + Type result = layoutType(ctx, std::move(flat)); + if (c->ptr) return CuteMemRefType::get(ctx, movedPtr(c->ptr, offset), result); + if (c->iter) { + auto sum = cg::arith_tuple_sum(c->iter.getRef(), offset); + if (!cg::is_valid(sum)) return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), + result); + } + return result; + }); +} + +Type mlir::cutlass_compiler::cute::makeArithTupleIterResultType(Type tuple) { + auto t = llvm::dyn_cast(tuple); + return t ? ArithTupleIteratorType::get(tuple.getContext(), t) : Type(); +} + +LogicalResult MakeComposedLayoutOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto offset = llvm::dyn_cast(a.getOffset().getType()); + auto outer = llvm::dyn_cast(a.getOuter().getType()); + if (!offset || !outer) + return emitOptionalError(loc, "expects an int tuple offset and a layout"); + return infer(loc, out, [&]() -> Type { + Type inner = a.getInner().getType(); + if (auto l = llvm::dyn_cast(inner)) + return ComposedLayoutType::get(ctx, cg::composed_layout(l.getRef(), offset.getRef(), outer.getRef())); + if (auto s = llvm::dyn_cast(inner)) + return ComposedLayoutType::get(ctx, cg::composed_layout(s.getRef(), offset.getRef(), outer.getRef())); + return {}; + }); +} + +LogicalResult MakeIdentityLayoutOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto s = llvm::dyn_cast(operands[0].getType()); + if (!s) return {}; + cg::layout l = cg::make_identity_layout(s.getRef()); + return cg::is_valid(l) ? LayoutType::get(ctx, std::move(l)) : Type{}; + }); +} + +LogicalResult MakeOrderedLayoutOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto shape = llvm::dyn_cast(a.getShape().getType()); + auto order = llvm::dyn_cast(a.getOrder().getType()); + if (!shape || !order) return emitOptionalError(loc, "expects a shape and an order"); + return infer(loc, out, [&]() -> Type { + cg::layout l(shape.getRef(), + cg::compact_order(shape.getRef(), order.getRef())); + return cg::is_valid(l) ? LayoutType::get(ctx, std::move(l)) : Type{}; + }); +} + +Type mlir::cutlass_compiler::cute::makeViewResultType(Type iter, Type layout) { + MLIRContext *ctx = iter.getContext(); + if (!llvm::isa(layout)) return {}; + if (auto p = llvm::dyn_cast(iter)) return CuteMemRefType::get(ctx, p, layout); + if (auto it = llvm::dyn_cast(iter)) + return CoordTensorType::get(ctx, it.getArithTuple(), layout); + auto plain = llvm::dyn_cast(layout); + if (plain && llvm::isa(iter)) + return cute_nvgpu::SmemDescViewType::get(ctx, iter, plain.getAttr()); + return {}; +} + +LogicalResult MemRefLoadOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto m = llvm::dyn_cast(operands[0].getType()); + return m ? m.getValueType() : Type{}; + }); +} + +LogicalResult MemRefLoadVecOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + return infer(loc, out, [&]() -> Type { + auto m = llvm::dyn_cast(a.getSrc().getType()); + if (!m) return {}; + auto l = llvm::dyn_cast(m.getLayout()); + if (!l) return {}; + auto n = cg::size(l.getRef()); + if (!cg::holds_int(n)) return {}; + return VectorType::get({n.as_int64()}, m.getValueType()); + }); +} + +LogicalResult SizeOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + ArrayRef mode; + if (auto m = a.getModeAttr()) mode = m.asArrayRef(); + Type in = a.getInput().getType(); + if (auto c = carrierOf(in)) in = c->layout; + return infer(loc, out, [&]() -> Type { + return llvm::TypeSwitch(in) + .Case( + [&](auto ty) -> Type { + auto r = cg::rec_var_cast( + cg::size(cg::get(ty.getRef(), mode))); + if (!cg::is_valid(r)) return {}; + return IntTupleType::get(ctx, std::move(r)); + }) + .Default([](Type) -> Type { return {}; }); + }); +} + +LogicalResult SliceOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto coord = llvm::dyn_cast(a.getCoord().getType()); + if (!coord) return emitOptionalError(loc, "coord must be a coord"); + Type in = a.getInput().getType(); + return infer(loc, out, [&]() -> Type { + if (auto c = carrierOf(in)) return sliceCarrier(ctx, *c, coord.getRef()); + return llvm::TypeSwitch(in) + .Case([&](auto ty) -> Type { + auto r = cg::slice(coord.getRef(), ty.getRef()); + if (!cg::is_valid(r)) return {}; + return decltype(ty)::get(ctx, std::move(r)); + }) + .Default([](Type) -> Type { return {}; }); + }); +} + +LogicalResult TileToShapeOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto shape = llvm::dyn_cast(a.getTrgShape().getType()); + auto order = llvm::dyn_cast(a.getOrdShape().getType()); + if (!shape || !order) return emitOptionalError(loc, "expects a shape and an order"); + return infer(loc, out, [&]() -> Type { + return mapLayout(ctx, a.getBlock().getType(), [&](const auto &l) { + return cg::tile_to_shape(l, shape.getRef(), order.getRef()); + }); + }); +} + +LogicalResult ToIntTupleOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + return llvm::TypeSwitch(operands[0].getType()) + .Case([&](auto ty) -> Type { + return IntTupleType::get(ctx, cg::rec_var_cast(ty.getRef())); + }) + .Default([](Type) -> Type { return {}; }); + }); +} + +namespace { +template +LogicalResult inferTupleArith(MLIRContext *ctx, std::optional loc, + ValueRange operands, SmallVectorImpl &out, + F &&f) { + return infer(loc, out, [&]() -> Type { + return llvm::TypeSwitch(operands[0].getType()) + .Case([&](auto lhs) -> Type { + using T = decltype(lhs); + auto rhs = llvm::dyn_cast(operands[1].getType()); + if (!rhs) return {}; + auto r = f.template operator()(lhs.getRef(), rhs.getRef()); + if (!cg::is_valid(r)) return {}; + return T::get(ctx, std::move(r)); + }) + .Default([](Type) -> Type { return {}; }); + }); +} +} // namespace + +LogicalResult TupleAddOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return inferTupleArith(ctx, loc, operands, out, [](const auto &a, const auto &b) { + return cg::arith_tuple_sum(a, b); + }); +} + +LogicalResult TupleMulOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return inferTupleArith(ctx, loc, operands, out, [](const auto &a, const auto &b) { + return cg::arith_tuple_mul(a, b); + }); +} + +LogicalResult TupleDivOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return inferTupleArith(ctx, loc, operands, out, [](const auto &a, const auto &b) { + return cg::arith_tuple_div(a, b); + }); +} + +// Without a stride the layout is compact column-major, unit modes at 0. +Type mlir::cutlass_compiler::cute::makeLayoutResultType(Type shape, Type stride) { + auto s = llvm::dyn_cast(shape); + if (!s) return {}; + MLIRContext *ctx = shape.getContext(); + if (stride) { + auto st = llvm::dyn_cast(stride); + if (!st) return {}; + return LayoutType::get(ctx, cg::layout(s.getRef(), st.getRef())); + } + cg::layout l(s.getRef(), cg::compact_col_major(s.getRef())); + return LayoutType::get(ctx, zeroUnitModes(l)); +} + +namespace { +// `MxNxK` and `num_cta` of an SM100 block-scaled or plain mma atom. +struct MmaAtom { + int64_t m = 0, n = 0, k = 0, ctas = 1; + Type c; // accumulator element + bool aTmem = false; // frag_kind = ts +}; +std::optional mmaAtomOf(Type tiledMma) { + auto tiled = llvm::dyn_cast(tiledMma); + if (!tiled) return std::nullopt; + // One atom, not permuted: other tilings are not covered here. + auto n = cg::size(tiled.getAtomLayout_MNK().getRef()); + TileAttr perm = tiled.getPermutation_MNK(); + if (!cg::holds_int(n) || n.as_int64() != 1 || + (perm && cg::to_string(perm.getRef()) != "[_;_;_]")) + return std::nullopt; + MmaAtom atom; + ShapeAttr shape; + if (auto bs = llvm::dyn_cast(tiled.getMmaAtom())) { + shape = bs.getShapeMnk(); + atom.ctas = bs.getNumCta(); + atom.c = bs.getCType(); + atom.aTmem = bs.getAFragKind() == cute_nvgpu::MmaFragKind::tmem; + } else if (auto mma = llvm::dyn_cast(tiled.getMmaAtom())) { + shape = mma.getShapeMnk(); + atom.ctas = mma.getNumCta(); + atom.c = mma.getCType(); + atom.aTmem = mma.getAFragKind() == cute_nvgpu::MmaFragKind::tmem; + } else { + return std::nullopt; + } + if (cg::rank(shape.getRef()) != 3) return std::nullopt; + int64_t *mnk[] = {&atom.m, &atom.n, &atom.k}; + for (int i = 0; i < 3; ++i) { + auto e = cg::get(shape.getRef(), i); + if (!cg::holds_int(e)) return std::nullopt; + *mnk[i] = e.as_int64(); + } + return atom; +} +} // namespace + +// partition_A/B/C of the trivial tiled mma: the leading two modes are divided +// by the per-CTA atom shape, the CTA's share of the first is taken at the +// thread coordinate, and what remains follows the atom modes. +LogicalResult TiledMmaPartitionOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto atom = mmaAtomOf(a.getTiledMma().getType()); + auto c = carrierOf(a.getInput().getType()); + auto coord = llvm::dyn_cast(a.getCoord().getType()); + auto which = llvm::dyn_cast_or_null(a.getOperandIdAttr()); + if (!atom || !c || !coord || !which) + return emitOptionalError(loc, "expects a tiled mma, a tensor, a coord and an operand"); + return infer(loc, out, [&]() -> Type { + auto layout = llvm::dyn_cast(c->layout); + if (!layout) return {}; + int64_t a0, a1; + switch (which.getInt()) { + case 0: a0 = atom->m / atom->ctas; a1 = atom->k; break; // A: (M, K) + case 1: a0 = atom->n / atom->ctas; a1 = atom->k; break; // B: (N, K) + case 2: a0 = atom->m / atom->ctas; a1 = atom->n; break; // C: (M, N) + default: return {}; + } + const cg::layout &l = layout.getRef(); + if (cg::rank(l.shape()) < 2) return {}; + auto tile = cg::from_string("[" + std::to_string(a0) + ":1;" + + std::to_string(a1) + ":1]"); + if (!tile) return {}; + cg::layout two = cg::make_layout(std::vector{cg::get(l, 0), cg::get(l, 1)}); + cg::layout divided = cg::zipped_divide(two, *tile); + if (!cg::is_valid(divided)) return {}; + cg::layout atoms = cg::get(divided, 0), rest = cg::get(divided, 1); + // The first rest mode holds the CTA index innermost. + auto ctaTile = cg::from_string("[" + std::to_string(atom->ctas) + ":1]"); + if (!ctaTile) return {}; + cg::layout rest0 = cg::zipped_divide(cg::get(rest, 0), *ctaTile); + if (!cg::is_valid(rest0) || cg::rank(rest0.shape()) != 2) return {}; + // A lone CTA has nothing to select: its coordinate is 0 whatever the + // thread index says. + std::string which_cta = atom->ctas == 1 ? "0" : cg::to_string(coord.getRef()); + auto [rest0Sliced, offset] = cg::slice_and_offset( + *cg::from_string("(" + which_cta + ",_)"), rest0); + if (!cg::is_valid(rest0Sliced)) return {}; + // What the slice leaves is one mode, not a tuple around it. + while (cg::holds_vector(rest0Sliced.shape()) && cg::rank(rest0Sliced.shape()) == 1) + rest0Sliced = cg::get(rest0Sliced, 0); + std::vector modes{atoms, rest0Sliced, cg::get(rest, 1)}; + for (size_t i = 2; i < cg::rank(l.shape()); ++i) modes.push_back(cg::get(l, i)); + // Modes beyond M and K (or N) are untouched. + Type result = layoutType(ctx, zeroUnitModesHead(cg::make_layout(modes), modes.size() - 3, false)); + if (c->ptr) return CuteMemRefType::get(ctx, movedPtr(c->ptr, offset), result); + if (c->iter) { + auto sum = cg::arith_tuple_sum(c->iter.getRef(), offset); + if (!cg::is_valid(sum)) return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), result); + } + return result; + }); +} + +// The fragment an mma reads: for A and B a descriptor view over the staged +// smem tile, atom modes collapsed and strides in 16-byte units; for C the +// accumulator in tensor memory, one lane per row and one column per element. +LogicalResult MmaMakeFragmentOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto which = llvm::dyn_cast_or_null(a.getOperandIdAttr()); + auto atom = mmaAtomOf(a.getAtom().getType()); + // A in tensor memory (frag_kind = ts) is not covered. + if (!which || !atom || (which.getInt() == 0 && atom->aTmem)) + return emitOptionalError(loc, "expects an SM100 tiled mma and an operand id"); + Type in = a.getInput().getType(); + return infer(loc, out, [&]() -> Type { + if (which.getInt() == 2) { + auto shape = llvm::dyn_cast(in); + if (!shape) return {}; + const cg::shape &s = shape.getRef(); + if (cg::rank(s) < 1 || !cg::holds_vector(cg::get(s, 0))) return {}; + auto m = cg::size(cg::get(s, 0), 0), n = cg::size(cg::get(s, 0), 1); + if (!cg::holds_int(m) || !cg::holds_int(n)) return {}; + // ((M,N), rest...): rows on lanes, columns contiguous, stages by N + // columns; unit modes at 0. A 64-row share of a 2-CTA atom puts the + // second half of the columns on lanes 64 and up. + int64_t cols = n.as_int64(); + // Mode 0 is flattened to its (M,N) extents. + std::string text = "((" + cg::to_string(m) + "," + cg::to_string(n) + ")", + stride = "((65536,1)"; + if (atom->m / atom->ctas == 64) { + if (atom->ctas != 2 || m.as_int64() != 64 || cols % 2) return {}; + cols /= 2; + text = "((64,(" + std::to_string(cols) + ",2))"; + stride = "((65536,(1,4194304))"; + } else if (atom->m / atom->ctas != 128) { + return {}; + } + for (size_t i = 1; i < cg::rank(s); ++i) { + auto e = cg::size(s, i); + text += "," + cg::to_string(cg::get(s, i)); + stride += "," + ((cg::holds_int(e) && e.as_int64() == 1) + ? std::string("0") : std::to_string(cols)); + } + auto l = cg::from_string(text + "):" + stride + ")"); + if (!l) return {}; + auto ptr = PtrType::get(ctx, atom->c, AddressSpace::tmem, 1, {}, {}); + return CuteMemRefType::get(ctx, ptr, LayoutType::get(ctx, *l)); + } + auto m = llvm::dyn_cast(in); + if (!m) return {}; + cg::layout l; + if (auto plain = llvm::dyn_cast(m.getLayout())) l = plain.getRef(); + else if (auto composed = llvm::dyn_cast(m.getLayout())) l = composed.getRef().layout_b(); + else return {}; + // The atom mode collapses to one; the rest modes follow. + int64_t bits = bitsOfType(m.getValueType()); + std::string shape = "(1", stride = "(0"; + for (size_t i = 1; i < cg::rank(l.shape()); ++i) { + auto e = cg::size(l.shape(), i); + auto st = cg::get(l.stride(), i); + if (!cg::holds_int(e) || !cg::holds_int(st)) return {}; + shape += "," + std::to_string(e.as_int64()); + stride += "," + std::to_string(st.as_int64() * bits / 128); + } + auto view = cg::from_string(shape + "):" + stride + ")"); + if (!view) return {}; + return cute_nvgpu::SmemDescViewType::get( + ctx, cute_nvgpu::SmemDescType::get(ctx), LayoutAttr::get(ctx, *view)); + }); +} + +LogicalResult TupleSubOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return inferTupleArith(ctx, loc, operands, out, [](const auto &a, const auto &b) { + return cg::arith_tuple_sub(a, b); + }); +} + +LogicalResult TupleModOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + if (failed(inferTupleArith(ctx, loc, operands, out, [](const auto &a, const auto &b) { + return cg::arith_tuple_mod(a, b); + }))) + return failure(); + // A dynamic leaf modulo a static one: 0 when its divisibility is a + // multiple, itself when a divisor, else unknown (cutegen drops this). + std::vector lhs, rhs, res; + auto leaves = [](Type t, std::vector &l) { + if (auto i = llvm::dyn_cast(t)) leavesOf(i.getRef(), l); + if (auto s = llvm::dyn_cast(t)) leavesOf(s.getRef(), l); + }; + leaves(operands[0].getType(), lhs); + leaves(operands[1].getType(), rhs); + leaves(out[0], res); + if (lhs.size() != rhs.size() || lhs.size() != res.size()) return success(); + for (size_t i = 0; i < res.size(); ++i) { + int64_t n, div = 1; + if (lhs[i].front() != '?' || StringRef(rhs[i]).getAsInteger(10, n) || n <= 0) continue; + size_t d = lhs[i].find("div="); + if (d != std::string::npos) StringRef(lhs[i]).substr(d + 4).take_while(llvm::isDigit).getAsInteger(10, div); + res[i] = div % n == 0 ? "0" : n % div == 0 ? lhs[i] : dynamicInt(1, lhs[i].find("i64") != std::string::npos); + } + size_t i = 0; + Type t = out[0]; + std::string text; + if (auto it = llvm::dyn_cast(t)) text = nestLike(it.getRef(), res, i); + else text = nestLike(llvm::cast(t).getRef(), res, i); + if (llvm::isa(t)) { + if (auto r = cg::from_string(text)) out[0] = IntTupleType::get(ctx, *r); + } else if (auto r = cg::from_string(text)) { + out[0] = ShapeType::get(ctx, *r); + } + return success(); +} + +// ---------------------------------------------------------------- more cute + +namespace { + +template +LogicalResult inferProduct(MLIRContext *ctx, std::optional loc, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, SmallVectorImpl &out, F &&f) { + typename Op::Adaptor a(operands, attrs, props); + auto tiler = llvm::dyn_cast(a.getTiler().getType()); + if (!tiler || !cg::is_int_or_dynamic_int_only(tiler.getRef().stride())) + return emitOptionalError(loc, "expects an integer-strided layout tiler"); + Type in = a.getInput().getType(); + return infer(loc, out, [&]() -> Type { + auto outer = outerLayout(in); + if (!outer || !cg::is_static(*outer) || + (!cg::is_int_or_dynamic_int_only(outer->stride()) && !cg::is_static(tiler.getRef()))) + return {}; + return mapLayout(ctx, in, [&](const auto &l) { return f(l, tiler.getRef()); }); + }); +} + +// A coordinate per leaf of `shape`: 0 where the extent is 1, else dynamic. +std::optional dynamicCoordLike(const cg::shape &shape) { + std::vector leaves; + leavesOf(shape, leaves); + for (auto &leaf : leaves) leaf = isOne(leaf) ? "0" : "?"; + size_t i = 0; + return cg::from_string(nestLike(shape, leaves, i)); +} + +// Moves a carrier's iterator by `offset` and gives it `layout`. +Type moved(MLIRContext *ctx, const Carrier &c, const cg::int_tuple &offset, Type layout) { + if (c.ptr) return CuteMemRefType::get(ctx, movedPtr(c.ptr, offset), layout); + if (c.iter) { + auto sum = cg::arith_tuple_sum(c.iter.getRef(), offset); + if (!cg::is_valid(sum)) return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), layout); + } + return rebuild(ctx, c, layout); +} + +} // namespace + +#define CUTE_PRODUCT(Op, fn) \ + LogicalResult Op::inferReturnTypes( \ + MLIRContext *ctx, std::optional loc, ValueRange operands, \ + DictionaryAttr attrs, PropertyRef props, RegionRange, \ + SmallVectorImpl &out) { \ + return inferProduct(ctx, loc, operands, attrs, props, out, \ + [](const auto &l, const auto &t) { return cg::fn(l, t); }); \ + } +CUTE_PRODUCT(LogicalProductOp, logical_product) +CUTE_PRODUCT(ZippedProductOp, zipped_product) +CUTE_PRODUCT(TiledProductOp, tiled_product) +CUTE_PRODUCT(FlatProductOp, flat_product) +CUTE_PRODUCT(BlockedProductOp, blocked_product) +#undef CUTE_PRODUCT + +// Unlike the other products, the raked one leaves unit modes at stride 0. +LogicalResult RakedProductOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, SmallVectorImpl &out) { + return inferProduct(ctx, loc, operands, attrs, props, out, + [](const auto &l, const auto &t) { return zeroUnitModes(cg::raked_product(l, t)); }); +} + +// The pointer's type is kept: the compiler records no swizzle on it. +LogicalResult ApplySwizzleOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, SmallVectorImpl &out) { + out.push_back(operands[0].getType()); + return success(); +} + +LogicalResult ComplementOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto in = llvm::dyn_cast(a.getInput().getType()); + Type co = a.getCotarget().getType(); + return infer(loc, out, [&]() -> Type { + if (!in) return {}; + const cg::layout &l = in.getRef(); + if (!cg::is_int_or_dynamic_int_only(l.stride()) || + (!cg::is_static(l.stride()) && cg::rank(l.shape()) > 1)) + return {}; + cg::shape shape; + if (auto s = llvm::dyn_cast(co)) shape = s.getRef(); + else if (auto c = llvm::dyn_cast(co)) shape = c.getRef().shape(); + else return {}; + cg::layout r = cg::complement(l, shape); + return cg::is_valid(r) ? LayoutType::get(ctx, std::move(r)) : Type(); + }); +} + +LogicalResult ComposedGetInnerOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto c = llvm::dyn_cast(operands[0].getType()); + if (!c) return {}; + if (c.getRef().is_a_swizzle()) return SwizzleType::get(ctx, c.getRef().swizzle_a()); + return LayoutType::get(ctx, c.getRef().layout_a()); + }); +} + +LogicalResult ComposedGetOffsetOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto c = llvm::dyn_cast(operands[0].getType()); + return c ? IntTupleType::get(ctx, c.getRef().offset()) : Type(); + }); +} + +LogicalResult CosizeOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + ArrayRef mode = a.getMode(); + auto c = carrierOf(a.getInput().getType()); + return infer(loc, out, [&]() -> Type { + if (!c) return {}; + cg::int_tuple r; + if (auto l = llvm::dyn_cast(c->layout)) { + r = cg::rec_var_cast(cg::cosize(cg::get(l.getRef(), mode))); + } else { + // An affine inner layout has no cosize. + auto cl = llvm::cast(c->layout); + if (!cl.getRef().is_a_swizzle()) return {}; + r = cg::rec_var_cast(cg::cosize(cg::get(cl.getRef(), mode))); + } + return cg::is_valid(r) ? IntTupleType::get(ctx, std::move(r)) : Type(); + }); +} + +// The multiplier and the two shifts of the divisor's magic numbers. +LogicalResult FastDivmodGetAuxOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + Type i = fastDivmodComputeResultType(operands[0].getType()); + if (!i) return emitOptionalError(loc, "expects a fast divmod divisor"); + out.append({i, IntegerType::get(ctx, 8), IntegerType::get(ctx, 8)}); + return success(); +} + +LogicalResult FastDivmodGetDivisorOp::inferReturnTypes( + MLIRContext *, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&] { return fastDivmodComputeResultType(operands[0].getType()); }); +} + +LogicalResult FilterOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + auto c = carrierOf(operands[0].getType()); + if (!c) return emitOptionalError(loc, "input carries no layout"); + return infer(loc, out, [&]() -> Type { + Type layout; + if (auto l = llvm::dyn_cast(c->layout)) { + layout = layoutType(ctx, cg::filter(l.getRef())); + } else { + // The outer layout is filtered; the inner function is kept. + const cg::composed_layout &cl = llvm::cast(c->layout).getRef(); + cg::layout f = cg::filter(cl.layout_b()); + if (!cg::is_valid(f)) return {}; + layout = layoutType(ctx, cl.is_a_swizzle() ? cg::composed_layout(cl.swizzle_a(), cl.offset(), f) + : cg::composed_layout(cl.layout_a(), cl.offset(), f)); + } + return rebuild(ctx, *c, layout); + }); +} + +namespace { +// Layout::get_hier_coord: each leaf's coordinate is (index / stride) % shape. +// A constant index gives static coordinates where the leaf is static; any +// other index gives dynamic ones, 0 on leaves of extent 1. +std::optional> hierCoord(Value index, const cg::layout &l) { + std::vector shapes, strides; + leavesOf(l.shape(), shapes); + leavesOf(l.stride(), strides); + if (shapes.size() != strides.size()) return std::nullopt; + APInt value; + bool constant = matchPattern(index, m_ConstantInt(&value)); + std::vector crd; + for (size_t i = 0; i < shapes.size(); ++i) { + int64_t n = 0, d = 0; + bool staticShape = !StringRef(shapes[i]).getAsInteger(10, n); + if (staticShape && n == 1) { + crd.push_back("0"); + } else if (!constant) { + crd.push_back("?"); + } else if (StringRef(strides[i]).getAsInteger(10, d) || d <= 0) { + return std::nullopt; // not recovered for dynamic or zero strides + } else { + int64_t q = value.getSExtValue() / d; + crd.push_back(staticShape ? std::to_string(q % n) : "?"); + } + } + return crd; +} + +// Column-major index of static coordinates in `shape`, or "?". +std::string colMajor(const std::vector &crd, const std::vector &shapes, + size_t begin, size_t end) { + int64_t idx = 0, scale = 1; + for (size_t i = begin; i < end; ++i) { + int64_t c, n; + if (StringRef(crd[i]).getAsInteger(10, c)) return "?"; + idx += c * scale; + if (i + 1 < end && StringRef(shapes[i]).getAsInteger(10, n)) return "?"; + scale *= n; + } + return std::to_string(idx); +} +} // namespace + +LogicalResult GetHierCoordOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto layout = llvm::dyn_cast(a.getInput().getType()); + return infer(loc, out, [&]() -> Type { + auto crd = layout ? hierCoord(a.getIndex(), layout.getRef()) : std::nullopt; + if (!crd) return {}; + size_t i = 0; + auto c = cg::from_string(nestLike(layout.getRef().shape(), *crd, i)); + return c ? CoordType::get(ctx, std::move(*c)) : Type(); + }); +} + +// One coordinate per mode: the mode's hierarchical coordinate, column-major. +LogicalResult GetFlatCoordOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto layout = llvm::dyn_cast(a.getInput().getType()); + return infer(loc, out, [&]() -> Type { + auto crd = layout ? hierCoord(a.getIndex(), layout.getRef()) : std::nullopt; + if (!crd) return {}; + const cg::shape &shape = layout.getRef().shape(); + std::vector shapes, modes; + leavesOf(shape, shapes); + if (!cg::holds_vector(shape)) { + modes.push_back(colMajor(*crd, shapes, 0, shapes.size())); + } else { + size_t begin = 0; + for (size_t m = 0; m < cg::rank(shape); ++m) { + std::vector sub; + leavesOf(shape[m], sub); + modes.push_back(colMajor(*crd, shapes, begin, begin + sub.size())); + begin += sub.size(); + } + } + auto c = cg::from_string(cg::holds_vector(shape) ? "(" + llvm::join(modes, ",") + ")" + : modes[0]); + return c ? CoordType::get(ctx, std::move(*c)) : Type(); + }); +} + +LogicalResult GetIntegralCoordOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto layout = llvm::dyn_cast(a.getInput().getType()); + return infer(loc, out, [&]() -> Type { + auto crd = layout ? hierCoord(a.getIndex(), layout.getRef()) : std::nullopt; + if (!crd) return {}; + std::vector shapes; + leavesOf(layout.getRef().shape(), shapes); + auto c = cg::from_string(colMajor(*crd, shapes, 0, shapes.size())); + return c ? CoordType::get(ctx, std::move(*c)) : Type(); + }); +} + +LogicalResult GetLayoutsFromTileOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + auto tile = llvm::dyn_cast(operands[0].getType()); + if (!tile) return emitOptionalError(loc, "expects a tile"); + auto flat = cg::flatten(tile.getRef()); + for (size_t i = 0; i < cg::rank(flat); ++i) { + flat[i].visit([&](const auto &e) { + if constexpr (cg::is_layout>::value) + out.push_back(LayoutType::get(ctx, e)); + }); + } + if (out.empty()) return emitOptionalError(loc, "the tile holds no layout"); + return success(); +} + +LogicalResult GetStrideOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto l = llvm::dyn_cast(operands[0].getType()); + return l ? StrideType::get(ctx, l.getRef().stride()) : Type(); + }); +} + +LogicalResult Idx2CrdOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto index = llvm::dyn_cast(a.getIndex().getType()); + auto shape = llvm::dyn_cast(a.getShape().getType()); + return infer(loc, out, [&]() -> Type { + if (!index || !shape) return {}; + size_t r = cg::rank(index.getRef()); + if (r > 1 && r != cg::rank(shape.getRef())) return {}; + if (auto split = cg::holds_dynamic_int(index.getRef()) + ? splitIndex(index.getRef(), shape.getRef()) : std::nullopt) { + auto c = cg::from_string(*split); + return c ? CoordType::get(ctx, std::move(*c)) : Type(); + } + auto c = cg::rec_var_cast(cg::idx2crd(index.getRef(), shape.getRef())); + return cg::is_valid(c) ? CoordType::get(ctx, std::move(c)) : Type(); + }); +} + +LogicalResult IncrementCoordOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto coord = llvm::dyn_cast(a.getCoord().getType()); + auto shape = llvm::dyn_cast(a.getShape().getType()); + return infer(loc, out, [&]() -> Type { + if (!coord || !shape || !cg::is_congruent(coord.getRef(), shape.getRef()) || + cg::has_underscore(coord.getRef())) + return {}; + auto c = cg::increment_coord(coord.getRef(), shape.getRef()); + return cg::is_valid(c) ? CoordType::get(ctx, std::move(c)) : Type(); + }); +} + +namespace { +template +LogicalResult inferInverse(MLIRContext *ctx, std::optional loc, + Type input, bool staticStride, SmallVectorImpl &out, + F &&f) { + return infer(loc, out, [&]() -> Type { + auto l = llvm::dyn_cast(input); + if (!l || !cg::is_static(l.getRef().shape()) || + (staticStride && !cg::is_static(l.getRef().stride()))) + return {}; + cg::layout r = f(l.getRef()); + return cg::is_valid(r) ? LayoutType::get(ctx, std::move(r)) : Type(); + }); +} +} // namespace + +LogicalResult LeftInverseOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return inferInverse(ctx, loc, operands[0].getType(), true, out, + [](const cg::layout &l) { return cg::left_inverse(l); }); +} + +LogicalResult RightInverseOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return inferInverse(ctx, loc, operands[0].getType(), false, out, + [](const cg::layout &l) { return cg::right_inverse(l); }); +} + +LogicalResult LoadScaledIndexOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, SmallVectorImpl &out) { + out.push_back(operands[0].getType()); + return success(); +} + +// CuTe's local_partition: the tensor divided by the tiler's shape, sliced at +// the thread's (dynamic) coordinate in the tile. +LogicalResult LocalPartitionOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto c = carrierOf(a.getInput().getType()); + auto tiler = llvm::dyn_cast(a.getTiler().getType()); + return infer(loc, out, [&]() -> Type { + auto layout = c ? llvm::dyn_cast(c->layout) : LayoutType(); + if (!layout || !tiler || !cg::is_static(tiler.getRef())) return {}; + cg::shape tile = cg::product_each(tiler.getRef().shape()); + cg::layout zipped = cg::zipped_divide(layout.getRef(), tile); + auto thread = dynamicCoordLike(tile); + if (!cg::is_valid(zipped) || !thread) return {}; + auto crd = cg::from_string("(" + cg::to_string(*thread) + ",_)"); + if (!crd) return {}; + auto [sliced, offset] = cg::slice_and_offset(*crd, zipped); + if (!cg::is_valid(sliced)) return {}; + // The slice keeps the rest mode's own modes, not a tuple around them. + if (cg::holds_vector(sliced.shape()) && cg::rank(sliced.shape()) == 1) + sliced = cg::get(sliced, 0); + return moved(ctx, *c, offset, LayoutType::get(ctx, zeroUnitModes(sliced))); + }); +} + +// CuTe's make_fragment_like: the first mode compact column-major with its +// stride-0 modes kept, the other modes ordered by their strides (dynamic +// ones last) after it; column-major when there is one mode or the first +// mode's shape is dynamic. +LogicalResult MakeFragmentLikeOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + auto src = swizzledOuter(operands[0].getType()); + return infer(loc, out, [&]() -> Type { + if (!src) return {}; + const cg::shape &shape = src->shape(); + std::vector shapes, strides; + leavesOf(shape, shapes); + leavesOf(src->stride(), strides); + if (shapes.size() != strides.size()) return {}; + size_t first = shapes.size(); // leaves of mode 0 + bool ordered = cg::holds_vector(shape) && cg::rank(shape) > 1 && cg::is_static(shape[0]); + if (ordered) { + std::vector m0; + leavesOf(shape[0], m0); + first = m0.size(); + } + // The order the leaves are laid out in. + std::vector order(shapes.size()); + std::iota(order.begin(), order.end(), 0); + auto key = [&](size_t i) -> int64_t { + int64_t v; + return StringRef(strides[i]).getAsInteger(10, v) ? INT64_MAX : v; + }; + if (ordered) + std::stable_sort(order.begin() + first, order.end(), + [&](size_t x, size_t y) { return key(x) < key(y); }); + std::vector result(shapes.size()); + cg::int_tuple step = *cg::from_string("1"); + for (size_t i : order) { + if (isOne(shapes[i]) || (ordered && i < first && strides[i] == "0")) { + result[i] = "0"; + continue; + } + result[i] = cg::to_string(step); + auto n = cg::from_string(shapes[i]); + if (!n) return {}; + step = cg::arith_tuple_mul(step, *n); + } + size_t i = 0; + auto l = cg::from_string(cg::to_string(shape) + ":" + nestLike(shape, result, i)); + return l ? LayoutType::get(ctx, std::move(*l)) : Type(); + }); +} + +LogicalResult MakeIdentityTensorOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto s = llvm::dyn_cast(operands[0].getType()); + if (!s) return {}; + std::vector leaves; + leavesOf(s.getRef(), leaves); + for (auto &leaf : leaves) leaf = "0"; + size_t i = 0; + auto zero = cg::from_string(nestLike(s.getRef(), leaves, i)); + cg::layout l = cg::make_identity_layout(s.getRef()); + if (!zero || !cg::is_valid(l)) return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, *zero), LayoutType::get(ctx, l)); + }); +} + +LogicalResult MakeLayoutLikeOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + Type src = operands[0].getType(); + cg::layout r; + if (auto l = llvm::dyn_cast(src)) { + if (cg::has_scaled_basis(l.getRef().stride())) return {}; + r = cg::make_layout_like(l.getRef()); + } else if (auto c = llvm::dyn_cast(src)) { + if (!c.getRef().is_a_swizzle() || cg::has_scaled_basis(c.getRef().layout_b().stride())) + return {}; + r = cg::make_layout_like(c.getRef()); + } else { + return {}; + } + return cg::is_valid(r) ? LayoutType::get(ctx, std::move(r)) : Type(); + }); +} + +LogicalResult PtrLoadOp::inferReturnTypes( + MLIRContext *, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto p = llvm::dyn_cast(operands[0].getType()); + return p ? p.getValueType() : Type(); + }); +} + +LogicalResult RecastLayoutOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + int newBits = a.getNewTypeBits(), oldBits = a.getOldTypeBits(); + return infer(loc, out, [&]() -> Type { + if (newBits <= 0 || oldBits <= 0) return {}; + auto outer = outerLayout(a.getSrc().getType()); + if (!outer || !cg::is_int_or_dynamic_int_only(outer->stride())) return {}; + return mapLayout(ctx, a.getSrc().getType(), + [&](const auto &l) { return cg::recast(newBits, oldBits, l); }); + }); +} + +// Without a profile the whole vector reduces to a scalar; with one, the +// modes it marks reduce and the others (`_`) remain, as a vector and shape. +LogicalResult ReduceOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto vec = llvm::dyn_cast(a.getInput().getType()); + if (!vec) return emitOptionalError(loc, "expects a vector input"); + if (!a.getReductionProfileAttr()) { + out.push_back(vec.getElementType()); + return success(); + } + auto shape = llvm::dyn_cast(a.getShape().getType()); + if (!shape) return emitOptionalError(loc, "expects a shape"); + cg::shape rest = cg::slice(a.getReductionProfileAttr().getRef(), shape.getRef()); + auto n = cg::size(rest); + if (!cg::is_valid(rest) || !cg::holds_int(n)) + return emitOptionalError(loc, "cannot reduce the shape"); + out.push_back(VectorType::get({n.as_int64()}, vec.getElementType())); + out.push_back(ShapeType::get(ctx, std::move(rest))); + return success(); +} + +LogicalResult SelectOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + ArrayRef mode = a.getMode(); + return infer(loc, out, [&]() -> Type { + return llvm::TypeSwitch(a.getInput().getType()) + .Case([&](auto ty) -> Type { + size_t r = cg::rank(ty.getRef()); + for (int32_t m : mode) + if (m < 0 || static_cast(m) >= r) return {}; + auto res = cg::select(ty.getRef(), mode); + if (!cg::is_valid(res)) return {}; + return decltype(ty)::get(ctx, std::move(res)); + }) + .Default([](Type) -> Type { return {}; }); + }); +} + +LogicalResult ShapeDivOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto x = llvm::dyn_cast(operands[0].getType()); + auto y = llvm::dyn_cast(operands[1].getType()); + if (!x || !y) return {}; + cg::shape r = cg::shape_div(x.getRef(), y.getRef()); + return cg::is_valid(r) ? ShapeType::get(ctx, std::move(r)) : Type(); + }); +} + +// The compiler aborts on every input tried (bad_alloc, segfault): no rule. +LogicalResult StencilDivideOp::inferReturnTypes( + MLIRContext *, std::optional loc, ValueRange, DictionaryAttr, + PropertyRef, RegionRange, SmallVectorImpl &) { + return emitOptionalError(loc, "cute.stencil_divide inference is not recovered"); +} + +// No type of the compiler's registry implements the descriptor iterator +// interface this needs, so the compiler rejects every input. +LogicalResult DereferenceDescriptorIteratorOp::inferReturnTypes( + MLIRContext *, std::optional loc, ValueRange, DictionaryAttr, + PropertyRef, RegionRange, SmallVectorImpl &) { + return emitOptionalError(loc, "expects a descriptor iterator"); +} + +LogicalResult TupleProductOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + return llvm::TypeSwitch(operands[0].getType()) + .Case([&](auto ty) -> Type { + auto r = cg::product(ty.getRef()); + return cg::is_valid(r) ? decltype(ty)::get(ctx, std::move(r)) : Type(); + }) + .Default([](Type) -> Type { return {}; }); + }); +} + +LogicalResult TupleProductEachOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + return llvm::TypeSwitch(operands[0].getType()) + .Case([&](auto ty) -> Type { + auto r = cg::product_each(ty.getRef()); + return cg::is_valid(r) ? decltype(ty)::get(ctx, std::move(r)) : Type(); + }) + .Default([](Type) -> Type { return {}; }); + }); +} + +// A SIMT copy's fragment is the tensor itself; the rule for other atoms is +// not recovered. +LogicalResult CopyMakeFragmentOp::inferReturnTypes( + MLIRContext *, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto tiled = llvm::dyn_cast(a.getAtom().getType()); + Type in = a.getInput().getType(); + if (!tiled || !llvm::isa(tiled.getCopyAtom()) || + !llvm::isa(in)) + return emitOptionalError(loc, "copy fragment inference covers tiled SIMT copies only"); + out.push_back(in); + return success(); +} + +// The atom's shape per CTA, then how often the atom repeats over the tile. +LogicalResult TiledMmaPartitionShapeOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto atom = mmaAtomOf(a.getTiledMma().getType()); + auto shape = llvm::dyn_cast(a.getInput().getType()); + auto which = llvm::dyn_cast_or_null(a.getOperandIdAttr()); + return infer(loc, out, [&]() -> Type { + if (!atom || !shape || !which || !cg::holds_vector(shape.getRef()) || + cg::rank(shape.getRef()) != 2) + return {}; + int64_t full[2], part[2]; + switch (which.getInt()) { + case 0: full[0] = atom->m; full[1] = atom->k; part[0] = atom->m / atom->ctas; break; + case 1: full[0] = atom->n; full[1] = atom->k; part[0] = atom->n / atom->ctas; break; + case 2: full[0] = atom->m; full[1] = atom->n; part[0] = atom->m / atom->ctas; break; + default: return {}; + } + part[1] = full[1]; + std::string rest; + for (int i = 0; i < 2; ++i) { + auto n = cg::size(shape.getRef(), i); + if (!cg::holds_int(n) || n.as_int64() % full[i]) return {}; + rest += "," + std::to_string(n.as_int64() / full[i]); + } + auto r = cg::from_string("((" + std::to_string(part[0]) + "," + + std::to_string(part[1]) + ")" + rest + ")"); + return r ? ShapeType::get(ctx, std::move(*r)) : Type(); + }); +} + +// ---------------------------------------------------------------- copy atoms + +namespace { + +std::optional layoutOf(const std::string &text) { + return cg::from_string(text); +} + +template +int64_t asInt(const V &v) { + return cg::holds_int(v) ? v.as_int64() : -1; +} + + +// TMEM addresses count lanes in the upper half-word: 2^21 bits per lane. +constexpr int64_t kTmemLaneBits = int64_t(1) << 21; + +// A copy atom's traits as CuTe spells them (cute/atom/copy_traits_*.hpp): +// the thread count and the (thr,val) -> bit layouts, and for tensor memory +// atoms the bit -> address layout. +struct CopyTraits { + int64_t threads = 1; + int64_t bits = 8; + cg::layout src, dst, ref, valId; +}; + +std::optional copyTraitsOf(Type atom) { + CopyTraits t; + auto set = [&](std::string src, std::string dst, bool refIsSrc, + std::string valId = "") -> std::optional { + auto s = layoutOf(src), d = layoutOf(dst); + if (!s || !d) return std::nullopt; + t.src = *s; + t.dst = *d; + t.ref = refIsSrc ? *s : *d; + if (!valId.empty()) { + auto v = layoutOf(valId); + if (!v) return std::nullopt; + t.valId = *v; + } + return t; + }; + auto n = [](int64_t v) { return std::to_string(v); }; + if (auto u = llvm::dyn_cast(atom)) { + t.bits = bitsOfType(u.getValType()); + std::string l = "(1," + n(u.getCopyBits() ? u.getCopyBits() : t.bits) + "):(0,1)"; + return set(l, l, true); + } + if (auto s = llvm::dyn_cast(atom)) { + // SM90_U32xN_STSM_N and SM90_U16x2N_STSM_T: the SM75 ldmatrix layouts + // with source and destination swapped. + t.bits = bitsOfType(s.getValType()); + t.threads = 32; + int64_t matrices = s.getNumMatrices(); + bool transposed = bool(s.getTranspose()); + std::string ldsmSrc, ldsmDst; + if (!transposed) { + switch (matrices) { + case 1: ldsmSrc = "((8,4),128):((128,0),1)"; ldsmDst = "(32,32):(32,1)"; break; + case 2: ldsmSrc = "((16,2),128):((128,0),1)"; ldsmDst = "(32,(32,2)):(32,(1,1024))"; break; + case 4: ldsmSrc = "(32,128):(128,1)"; ldsmDst = "(32,(32,4)):(32,(1,1024))"; break; + default: return std::nullopt; + } + } else { + switch (matrices) { + case 1: ldsmSrc = "((8,4),128):((128,0),1)"; ldsmDst = "((4,8),(16,2)):((256,16),(1,128))"; break; + case 2: ldsmSrc = "((16,2),128):((128,0),1)"; ldsmDst = "((4,8),(16,2,2)):((256,16),(1,128,1024))"; break; + case 4: ldsmSrc = "(32,128):(128,1)"; ldsmDst = "((4,8),(16,2,4)):((256,16),(1,128,1024))"; break; + default: return std::nullopt; + } + } + return set(ldsmDst, ldsmSrc, true); + } + // Tensor memory loads and stores: the tmem side is the reference; a store + // is a load with source and destination swapped. + auto tmem = [&](Type val, int64_t dp, int64_t bit, int64_t x, bool load) -> std::optional { + t.bits = bitsOfType(val); + t.threads = 32; + std::string mem, reg, valId; + if (dp == 32 && bit == 32) { + mem = "(32," + n(1024 * x) + "):(0,1)"; + reg = "(32," + n(32 * x) + "):(" + n(32 * x) + ",1)"; + valId = "(" + n(32 * x) + ",32):(1," + n(kTmemLaneBits) + ")"; + } else if (dp == 16 && bit == 256) { + mem = "(32," + n(4096 * x) + "):(0,1)"; + reg = x == 1 ? "((4,8),(64,2)):((64,256),(1,2048))" + : "((4,8),(64,2," + n(x) + ")):((64," + n(256 * x) + "),(1," + n(2048 * x) + ",256))"; + valId = "(" + n(256 * x) + ",16):(1," + n(kTmemLaneBits) + ")"; + } else { + return std::nullopt; + } + return load ? set(mem, reg, true, valId) : set(reg, mem, false, valId); + }; + // 16-bit packing and split reductions are not covered. + if (auto l = llvm::dyn_cast(atom)) + if (!l.getPack16b()) + return tmem(l.getValType(), l.getNumDp(), l.getNumBit(), l.getNumRep(), true); + if (auto l = llvm::dyn_cast(atom)) + if (!l.getHalfSplitOff()) + return tmem(l.getValType(), l.getNumDp(), l.getNumBit(), l.getNumRep(), true); + if (auto st = llvm::dyn_cast(atom)) + if (!st.getExpand16b()) + return tmem(st.getValType(), st.getNumDp(), st.getNumBit(), st.getNumRep(), false); + if (auto s = llvm::dyn_cast(atom)) { + // : SM100_UTCCP_4x32dp128bit_{C}cta. + t.bits = bitsOfType(s.getValType()); + int64_t dp = s.getNumDp(), bit = s.getNumBit(); + t.threads = s.getNumCta(); + if (dp != 32 || bit != 128 || s.getBroadcast() != cute_nvgpu::CopyS2TBroadcast::x4 || + t.threads < 1) + return std::nullopt; + return set("(" + n(t.threads) + ",(32,128,4)):(0,(1,32,0))", + "(" + n(t.threads) + ",16384):(0,1)", false, + "(32,128,4):(" + n(kTmemLaneBits) + ",1," + n(32 * kTmemLaneBits) + ")"); + } + return std::nullopt; +} + +// A tiled copy: its atom in element units, the (thr,val) -> tile layout and +// the tiler over MN. +struct TiledCopy { + Type atom; + CopyTraits traits; + cg::layout tv; + cg::tile tiler; + int64_t atomThreads = 0, atomVals = 0, tiledThreads = 0; +}; + +std::optional tiledCopyOf(Type t) { + auto tiled = llvm::dyn_cast(t); + if (!tiled) return std::nullopt; + auto traits = copyTraitsOf(tiled.getCopyAtom()); + if (!traits) return std::nullopt; + TiledCopy tc; + tc.atom = tiled.getCopyAtom(); + tc.traits = *traits; + // Copy_Atom recasts the bit layouts to the value type. + tc.traits.src = cg::upcast(traits->bits, traits->src); + tc.traits.dst = cg::upcast(traits->bits, traits->dst); + tc.traits.ref = cg::upcast(traits->bits, traits->ref); + if (!cg::is_valid(tc.traits.src) || !cg::is_valid(tc.traits.dst) || + !cg::is_valid(tc.traits.ref)) + return std::nullopt; + tc.tv = tiled.getLayoutCopyTv().getRef(); + tc.tiler = tiled.getTilerMn().getRef(); + tc.atomThreads = asInt(cg::size(tc.traits.ref, 0)); + tc.atomVals = asInt(cg::size(tc.traits.ref, 1)); + tc.tiledThreads = asInt(cg::size(tc.tv, 0)); + if (tc.atomThreads <= 0 || tc.atomVals <= 0 || tc.tiledThreads <= 0) + return std::nullopt; + return tc; +} + +// The modes of a layout, one leaf mode as itself. +std::vector modesOf(const cg::layout &l) { + std::vector out; + if (!cg::holds_vector(l.shape())) return {l}; + for (size_t i = 0; i < cg::rank(l.shape()); ++i) out.push_back(cg::get(l, i)); + return out; +} + +// `l` with its first mode composed with `f` and the others kept: CuTe's +// tensor.compose(f, _). +cg::layout composeFirst(const cg::layout &l, const cg::layout &f) { + std::vector modes = modesOf(l); + modes[0] = cg::composition(modes[0], f); + return cg::make_layout(modes); +} + +// ((a,b),(c,d)) -> ((a,c),(b,d)) +cg::layout zip2(const cg::layout &l) { + return cg::make_layout(std::vector{ + cg::make_layout(std::vector{cg::get(l, 0, 0), cg::get(l, 1, 0)}), + cg::make_layout(std::vector{cg::get(l, 0, 1), cg::get(l, 1, 1)})}); +} + +cg::coord profileOf(const char *text) { + return *cg::from_string(text); +} + +// TiledCopy::tile2thrfrg: ((Tile),(Rest...)) -> (Thr, (FrgV, FrgX), Rest...) +// through the atom's (src or dst) thread-value map `ref2trg`. +cg::layout tile2thrfrg(const TiledCopy &tc, const cg::layout &tiled, + const cg::layout &ref2trg, bool flatRest = true) { + auto atomShape = cg::from_string( + "(" + std::to_string(tc.atomThreads) + "," + std::to_string(tc.atomVals) + ")"); + if (!atomShape) return cg::layout(cg::cg_error_t{}); + cg::layout atomTV = cg::zipped_divide(tc.tv, *atomShape); + if (!cg::is_valid(atomTV)) return atomTV; + cg::layout trgTV = composeFirst(atomTV, ref2trg); + cg::layout thrval2mn = cg::coalesce(zip2(trgTV), profileOf("(1,(1,1))")); + if (!cg::is_valid(thrval2mn)) return thrval2mn; + cg::layout tv = cg::composition(cg::get(tiled, 0), thrval2mn); + if (!cg::is_valid(tv)) return tv; + std::vector modes{cg::get(tv, 0), cg::get(tv, 1)}; + // The rest modes follow; a rank-1 tensor keeps its one rest mode whole. + if (!flatRest) modes.push_back(cg::get(tiled, 1)); + else for (const cg::layout &m : modesOf(cg::get(tiled, 1))) modes.push_back(m); + return cg::make_layout(modes); +} + +// TiledCopy::tidfrg_S/D: (M,N,...) -> (Thr, (FrgV, FrgX), (RestM, RestN, ...)). +cg::layout tidfrg(const TiledCopy &tc, const cg::layout &tensor, bool src) { + cg::layout tiled = cg::zipped_divide(tensor, tc.tiler); + if (!cg::is_valid(tiled)) return tiled; + cg::layout ref2trg = cg::composition(cg::right_inverse(tc.traits.ref), + src ? tc.traits.src : tc.traits.dst); + bool flatRest = cg::holds_vector(tensor.shape()) && cg::rank(tensor.shape()) > 1; + return tile2thrfrg(tc, tiled, ref2trg, flatRest); +} + +// ThrCopy::partition_S/D: the thread's slice of tidfrg. +Type partitionCopy(MLIRContext *ctx, const TiledCopy &tc, const Carrier &c, + const cg::coord &thread, bool src) { + auto layout = llvm::dyn_cast(c.layout); + if (!layout) return {}; + cg::layout tv = tidfrg(tc, layout.getRef(), src); + if (!cg::is_valid(tv)) return {}; + std::string coord = "(" + cg::to_string(thread); + for (size_t i = 1; i < cg::rank(tv.shape()); ++i) coord += ",_"; + coord += ")"; + auto crd = cg::from_string(coord); + if (!crd) return {}; + auto [sliced, offset] = cg::slice_and_offset(*crd, tv); + if (!cg::is_valid(sliced)) return {}; + // The tensor's modes beyond the tiler are untouched and keep their strides. + size_t in = cg::rank(layout.getRef().shape()), r = cg::rank(tc.tiler); + size_t keep = cg::holds_vector(layout.getRef().shape()) && in > r ? in - r : 0; + Type result = LayoutType::get(ctx, zeroUnitModesHead(sliced, keep, false)); + if (c.ptr) return CuteMemRefType::get(ctx, movedPtr(c.ptr, offset), result); + if (c.iter) { + auto sum = cg::arith_tuple_sum(c.iter.getRef(), offset); + if (!cg::is_valid(sum)) return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), result); + } + return rebuild(ctx, c, result); +} + +// The shapes of a tiler's modes, as one shape. +std::optional tilerShape(const cg::tile &tiler) { + std::string text = cg::to_string(tiler); + if (text.size() < 2 || text.front() != '[') return std::nullopt; + SmallVector parts; + StringRef(text).drop_front().drop_back().split(parts, ';'); + std::string shape = "("; + for (size_t i = 0; i < parts.size(); ++i) { + auto l = cg::from_string(parts[i].str()); + if (!l) return std::nullopt; + shape += (i ? "," : "") + cg::to_string(l->shape()); + } + return cg::from_string(shape + ")"); +} + +// TiledCopy::retile: (V, RestM, RestN, ...) of one thread's values regrouped +// into the atom's values. +Type retileCopy(MLIRContext *ctx, const TiledCopy &tc, const Carrier &c) { + auto layout = llvm::dyn_cast(c.layout); + if (!layout) return {}; + const cg::layout &l = layout.getRef(); + size_t R = cg::rank(l.shape()); + int64_t V = asInt(cg::size(l, 0)); + auto shape = tilerShape(tc.tiler); + if (V <= 0 || !shape) return {}; + cg::layout inv = cg::right_inverse(tc.tv).with_shape(*shape); + cg::layout frgMN = cg::upcast(tc.tiledThreads * V, inv); + if (!cg::is_valid(frgMN)) return {}; + auto vLayout = layoutOf(std::to_string(V) + ":1"); + auto atomVals = layoutOf(std::to_string(tc.atomVals) + ":1"); + if (!vLayout || !atomVals) return {}; + cg::layout frgV = cg::zipped_divide( + cg::logical_product(*vLayout, cg::right_inverse(frgMN)), *atomVals); + if (!cg::is_valid(frgV)) return {}; + std::string tiler = "(" + std::to_string(V); + auto each = cg::product_each(frgMN.shape()); + for (size_t i = 0; i < cg::rank(each); ++i) tiler += "," + cg::to_string(each[i]); + auto tilerShape = cg::from_string(tiler + ")"); + if (!tilerShape) return {}; + cg::layout tiled = cg::zipped_divide(l, *tilerShape); + if (!cg::is_valid(tiled)) return {}; + cg::layout v = composeFirst(tiled, frgV); + if (!cg::is_valid(v)) return {}; + // v(_, (0, _, _...)): the first rest mode is the one the tile added. + std::vector modes{cg::get(v, 0)}; + std::vector rest = modesOf(cg::get(v, 1)); + for (size_t i = 1; i < rest.size() && i < R; ++i) modes.push_back(rest[i]); + return rebuild(ctx, c, LayoutType::get(ctx, zeroUnitModes(cg::make_layout(modes)))); +} + +// make_cotiled_copy: the (thr,val) -> tile layout and the tiler of a copy +// whose atom addresses `data` directly. +std::optional> cotiledCopy(const cg::layout &atomTV, + const cg::layout &data) { + auto unit = layoutOf("1:0"); + cg::layout invData = cg::make_layout(std::vector{cg::left_inverse(data), *unit}); + cg::layout tvData = cg::composition(invData, atomTV); + if (!cg::is_valid(tvData)) return std::nullopt; + auto flat = cg::product_each(data.shape()); + size_t R = cg::rank(flat); + std::string flatText = cg::to_string(flat); + std::string tilerText = "["; + for (size_t i = 0; i < R; ++i) { + std::string stride = "("; + for (size_t j = 0; j < R; ++j) stride += std::string(j ? "," : "") + (i == j ? "1" : "0"); + auto pick = layoutOf(flatText + ":" + stride + ")"); + if (!pick) return std::nullopt; + cg::layout mode = cg::filter(cg::composition(*pick, tvData)); + if (!cg::is_valid(mode)) return std::nullopt; + tilerText += (i ? ";" : "") + cg::to_string(mode); + } + auto tiler = cg::from_string(tilerText + "]"); + auto flatShape = cg::from_string(flatText); + if (!tiler || !flatShape) return std::nullopt; + cg::layout tile2data = cg::composition(cg::layout(*flatShape), *tiler); + cg::layout tv = cg::composition(cg::left_inverse(tile2data), tvData); + if (!cg::is_valid(tv)) return std::nullopt; + return std::make_pair(tv, *tiler); +} + +Type tiledCopyType(MLIRContext *ctx, Type atom, const cg::layout &tv, const cg::tile &tiler) { + return TiledCopyType::get(ctx, atom, LayoutAttr::get(ctx, tv), + TileAttr::get(ctx, tiler)); +} + +// ---------------------------------------------------------------- TMA + +struct TmaFormat { + std::string name; + int64_t bits = 0; +}; + +// The descriptor's data format: the attribute when given, else the element's. +std::optional tmaFormatOf(Type elem, Attribute attr) { + TmaFormat f; + if (attr) { + auto format = llvm::dyn_cast(attr); + if (!format) return std::nullopt; + f.name = cute_nvgpu::stringifyTmaDataFormat(format.getValue()).str(); + StringRef n = f.name; + if (n == "F16_RN" || n == "BF16_RN") f.bits = 16; + else if (n == "F32_RN" || n == "F32_FTZ_RN" || n == "INT32" || n == "TF32") f.bits = 32; + else if (n == "F64_RN" || n == "INT64") f.bits = 64; + else if (!n.drop_until(llvm::isDigit).take_while(llvm::isDigit).getAsInteger(10, f.bits)) {} + else return std::nullopt; + return f; + } + f.bits = bitsOfType(elem); + if (elem.isTF32()) f.name = "TF32_RN"; + else if (elem.isF16()) f.name = "F16_RN"; + else if (elem.isBF16()) f.name = "BF16_RN"; + else if (elem.isF32()) f.name = "F32_RN"; + else if (elem.isF64()) f.name = "F64_RN"; + else f.name = "U" + std::to_string(f.bits); + return f; +} + +// CuTe's coalesce_256: which adjacent modes of a flattened layout merge into +// one TMA box dimension, the merged extent staying within 256. +std::vector> coalesce256Groups(const cg::layout &l) { + auto fs = cg::flatten(l.shape()); + auto ft = cg::flatten(l.stride()); + if (!cg::holds_vector(fs)) return {{0}}; + std::vector> groups; + std::vector cur{0}; + int64_t curS = asInt(fs[0]), curD = asInt(ft[0]); + for (size_t i = 1; i < cg::rank(fs); ++i) { + int64_t s = asInt(fs[i]), d = asInt(ft[i]); + if (curS == 1) { + cur = {i}; curS = s; curD = d; + } else if (s == 1) { + continue; + } else if (curS > 0 && curD >= 0 && s > 0 && d >= 0 && curS * curD == d && + s * curS <= 256) { + cur.push_back(i); + curS *= s; + } else { + groups.push_back(cur); + cur = {i}; curS = s; curD = d; + } + } + groups.push_back(cur); + return groups; +} + +// The layout with `shape`'s leaves regrouped by `groups`, over `stride`'s +// leaves: what composing with the box shape does, keeping every basis. +std::optional regroup(const cg::layout &shape, const cg::layout &stride, + const std::vector> &groups) { + auto fs = cg::flatten(shape.shape()); + auto ft = cg::flatten(stride.stride()); + auto leaf = [&](const auto &flat, size_t i) { + return cg::holds_vector(flat) ? cg::to_string(flat[i]) : cg::to_string(flat); + }; + std::string shapes, strides; + for (size_t g = 0; g < groups.size(); ++g) { + if (g) { shapes += ","; strides += ","; } + if (groups[g].size() == 1) { + shapes += leaf(fs, groups[g][0]); + strides += leaf(ft, groups[g][0]); + continue; + } + shapes += "("; strides += "("; + for (size_t k = 0; k < groups[g].size(); ++k) { + if (k) { shapes += ","; strides += ","; } + shapes += leaf(fs, groups[g][k]); + strides += leaf(ft, groups[g][k]); + } + shapes += ")"; strides += ")"; + } + return layoutOf("(" + shapes + "):(" + strides + ")"); +} + +// A layout composed with a coordinate layout: each mode of `rhs` selects, +// through its basis stride, the mode of `lhs` it indexes. +cg::layout composeCoord(const cg::layout &lhs, const cg::layout &rhs) { + std::vector modes; + for (const cg::layout &m : modesOf(rhs)) { + const auto &d = m.stride(); + if (cg::holds_int(d)) { + modes.push_back(m); + continue; + } + if (!cg::holds_scaled_basis(d)) return cg::layout(cg::cg_error_t{}); + const auto &basis = std::get(d); + if (!basis.value_holds_int()) return cg::layout(cg::cg_error_t{}); + auto scale = layoutOf(cg::to_string(m.shape()) + ":" + + std::to_string(basis.static_integral_value())); + if (!scale) return cg::layout(cg::cg_error_t{}); + cg::layout sub = cg::get(lhs, basis.modes()); + cg::layout composed = cg::composition(sub, *scale); + if (!cg::is_valid(composed)) return composed; + modes.push_back(composed); + } + return cg::make_layout(modes); +} + +// Leaves of congruent shape, stride and basis, with the path to each. +template +void walkLeaves(const cg::shape &s, const cg::stride &d, const cg::stride &e, + std::vector &path, F &&f) { + if (cg::holds_vector(s)) { + for (size_t i = 0; i < cg::rank(s); ++i) { + path.push_back(i); + walkLeaves(s[i], d[i], e[i], path, f); + path.pop_back(); + } + return; + } + f(s, d, e, path); +} + +bool basisAt(const cg::stride &leaf, const std::vector &path) { + return cg::holds_scaled_basis(leaf) && + std::get(leaf).modes() == path; +} + +// Which mode of a stride tuple carries the basis `path`, if any. +std::optional modeWithBasis(const cg::stride &strides, const std::vector &path) { + for (size_t j = 0; j < cg::rank(strides); ++j) { + auto flat = cg::flatten(strides[j]); + if (!cg::holds_vector(flat)) { + if (basisAt(flat, path)) return j; + continue; + } + for (size_t k = 0; k < cg::rank(flat); ++k) + if (basisAt(flat[k], path)) return j; + } + return std::nullopt; +} + +// construct_tma_gbasis: the TMA box shape with, per mode, the gmem modes it +// runs over. +cg::layout tmaGbasis(const cg::layout &gmem, int64_t elemBits, int64_t tmaBits, + const cg::layout &smem, const cg::layout &ctaVMap) { + auto fail = [] { return cg::layout(cg::cg_error_t{}); }; + cg::layout inv = cg::right_inverse(smem); + cg::layout full = cg::coalesce(cg::composition(ctaVMap, inv)); + if (!cg::is_valid(full)) return full; + std::vector fullModes = modesOf(full); + // Keep the leading modes that step through gmem one element at a time. + size_t smemRank = 0; + for (; smemRank < fullModes.size(); ++smemRank) { + const auto &d = fullModes[smemRank].stride(); + if (!cg::holds_scaled_basis(d)) break; + const auto &b = std::get(d); + if (!b.value_holds_int() || b.static_integral_value() != 1) break; + } + if (smemRank == 0) return fail(); + fullModes.resize(smemRank); + cg::layout sidx2gmode = cg::make_layout(fullModes); + cg::layout tileGstride = cg::recast(tmaBits, elemBits, composeCoord(gmem, sidx2gmode)); + if (!cg::is_valid(tileGstride)) return tileGstride; + cg::layout gbasis = cg::make_identity_layout(gmem.shape()); + cg::layout tileGbasisTmp = composeCoord(gbasis, sidx2gmode); + if (!cg::is_valid(tileGbasisTmp)) return tileGbasisTmp; + auto regrouped = regroup(tileGstride, tileGbasisTmp, coalesce256Groups(tileGstride)); + if (!regrouped) return fail(); + cg::layout tmaGbasisTile = *regrouped; + // Gmem modes the tile misses still shape the descriptor, as size-1 modes. + cg::layout gmemT = cg::recast(tmaBits, elemBits, gmem); + if (!cg::is_valid(gmemT)) return gmemT; + std::vector modes = modesOf(tmaGbasisTile); + auto flatTile = cg::flatten(tmaGbasisTile.stride()); + std::vector path; + bool ok = true; + walkLeaves(gmemT.shape(), gmemT.stride(), gbasis.stride(), path, + [&](const auto &s, const auto &d, const auto &e, const std::vector &p) { + if (asInt(s) == 1 || asInt(d) == 0) return; + bool found = false; + if (!cg::holds_vector(flatTile)) { + found = basisAt(flatTile, p); + } else { + for (size_t k = 0; k < cg::rank(flatTile) && !found; ++k) + found = basisAt(flatTile[k], p); + } + if (found) return; + auto extra = layoutOf("1:" + cg::to_string(e)); + if (!extra) ok = false; + else modes.push_back(*extra); + }); + if (!ok) return fail(); + cg::layout result = cg::make_layout(modes); + if (modes.size() > 4) result = cg::group(4, modes.size(), result); + return result; +} + +// The TMA tensor's layout: gmem shape over TMA coordinate bases, scaled to +// the descriptor's element. +std::optional tmaTensorLayout(const cg::layout &gmem, int64_t elemBits, + int64_t tmaBits, const cg::layout &gbasis) { + cg::layout basis = cg::make_identity_layout(gmem.shape()); + const auto &tmaStrides = gbasis.stride(); + bool ok = true; + std::string text; + std::vector path; + // Written in the shape's own nesting. + std::function write = + [&](const cg::shape &s, const cg::stride &d, const cg::stride &e) { + if (cg::holds_vector(s)) { + text += "("; + for (size_t i = 0; i < cg::rank(s); ++i) { + if (i) text += ","; + path.push_back(i); + write(s[i], d[i], e[i]); + path.pop_back(); + } + text += ")"; + return; + } + if (asInt(s) == 1 || asInt(d) == 0) { text += "0"; return; } + auto j = cg::holds_vector(tmaStrides) ? modeWithBasis(tmaStrides, path) + : (basisAt(tmaStrides, path) ? std::optional(0) : std::nullopt); + if (!j) { text += "0"; return; } + if (*j == 0) { + int64_t stride = asInt(d); + if (stride <= 0) { ok = false; return; } + int64_t num = stride * elemBits, den = tmaBits; + int64_t g = std::gcd(num, den); + num /= g; den /= g; + text += std::to_string(num) + (den == 1 ? "" : "/" + std::to_string(den)) + "@0"; + return; + } + auto mode = cg::holds_vector(tmaStrides) ? tmaStrides[*j] : tmaStrides; + auto flat = cg::flatten(mode); + if (cg::holds_vector(flat) && cg::rank(flat) != 1) { ok = false; return; } + text += "1@" + std::to_string(*j); + }; + write(gmem.shape(), gmem.stride(), basis.stride()); + if (!ok) return std::nullopt; + return layoutOf(cg::to_string(gmem.shape()) + ":" + text); +} + +// The smem tile layout without its swizzle. +std::optional plainLayoutOf(Type t) { + if (auto l = llvm::dyn_cast(t)) return l.getRef(); + if (auto c = llvm::dyn_cast(t)) return c.getRef().layout_b(); + return std::nullopt; +} + +struct TmaAtomInfo { + int64_t elemBits = 0, copyBits = 0; +}; + +std::optional tmaAtomInfoOf(Type atom) { + TmaAtomInfo info; + if (auto l = llvm::dyn_cast(atom)) { + info.elemBits = bitsOfType(l.getValType()); + info.copyBits = l.getCopyBits(); + } else if (auto s = llvm::dyn_cast(atom)) { + info.elemBits = bitsOfType(s.getValType()); + info.copyBits = s.getCopyBits(); + } else if (auto r = llvm::dyn_cast(atom)) { + info.elemBits = bitsOfType(r.getValType()); + info.copyBits = r.getCopyBits(); + } else if (auto l = llvm::dyn_cast(atom)) { + info.elemBits = bitsOfType(l.getValType()); + info.copyBits = l.getCopyBits(); + } else if (auto s = llvm::dyn_cast(atom)) { + info.elemBits = bitsOfType(s.getValType()); + info.copyBits = s.getCopyBits(); + } else { + return std::nullopt; + } + if (info.elemBits <= 0 || info.copyBits <= 0) return std::nullopt; + return info; +} + + +// make_non_exec_tiled_tma_load/store: the atom and the TMA tensor. +LogicalResult inferTmaAtom(MLIRContext *ctx, std::optional loc, + Type gmemType, Type smemType, Type ctaVMapType, + Attribute kind, Attribute format, SmallVectorImpl &out) { + auto gmem = llvm::dyn_cast(gmemType); + auto smem = plainLayoutOf(smemType); + auto ctaVMap = llvm::dyn_cast(ctaVMapType); + if (!gmem || !smem || !ctaVMap) + return emitOptionalError(loc, "expects a gmem tensor, an smem layout and a CTA value map"); + auto glayout = llvm::dyn_cast(gmem.getLayout()); + if (!glayout) return emitOptionalError(loc, "gmem tensor needs a plain layout"); + Type elem = gmem.getValueType(); + auto fmt = tmaFormatOf(elem, format); + if (!fmt || fmt->bits <= 0) return emitOptionalError(loc, "unknown TMA data format"); + int64_t elemBits = bitsOfType(elem); + cg::layout gbasis = tmaGbasis(glayout.getRef(), elemBits, fmt->bits, *smem, ctaVMap.getRef()); + if (!cg::is_valid(gbasis)) return emitOptionalError(loc, "cannot build the TMA basis"); + auto tensor = tmaTensorLayout(glayout.getRef(), elemBits, fmt->bits, gbasis); + if (!tensor) return emitOptionalError(loc, "cannot build the TMA tensor"); + int64_t copyBits = asInt(cg::size(gbasis)) * fmt->bits; + auto gbasisType = LayoutType::get(ctx, gbasis); + std::optional dataFormat = + cute_nvgpu::symbolizeTmaDataFormat(fmt->name); + Type atom; + if (auto k = llvm::dyn_cast_or_null(kind)) { + atom = cute_nvgpu::CopyAtomNonExecTiledTmaLoadType::get(ctx, k.getValue(), elem, copyBits, + gbasisType, dataFormat); + } else if (auto k = llvm::dyn_cast_or_null(kind)) { + atom = cute_nvgpu::CopyAtomNonExecTiledTmaReduceType::get(ctx, k.getValue(), elem, copyBits, + gbasisType, dataFormat); + } else { + atom = cute_nvgpu::CopyAtomNonExecTiledTmaStoreType::get(ctx, elem, copyBits, gbasisType, + dataFormat); + } + std::string zeros = "("; + for (size_t i = 0; i < cg::rank(gbasis.shape()); ++i) zeros += i ? ",0" : "0"; + auto iter = cg::from_string(zeros + ")"); + if (!iter) return emitOptionalError(loc, "cannot build the TMA coordinate"); + out.push_back(atom); + out.push_back(CoordTensorType::get(ctx, IntTupleType::get(ctx, *iter), + LayoutType::get(ctx, *tensor))); + return success(); +} + +} // namespace + +// ---------------------------------------------------------------- cute copy ops + +// TiledCopy::get_layoutD_TV over the reference tile, then the tiled copy of +// another atom with that layout (make_tiled_copy_D). +Type mlir::cutlass_compiler::cute::tiledCopyD(Type atom, TiledCopyType tiled) { + auto tc = tiledCopyOf(tiled); + if (!tc) return {}; + auto shape = tilerShape(tc->tiler); + if (!shape) return {}; + auto refShape = cg::from_string("(" + cg::to_string(*shape) + ",1)"); + if (!refShape) return {}; + cg::layout refD(*refShape); + cg::layout ref2trg = cg::composition(cg::right_inverse(tc->traits.ref), tc->traits.dst); + cg::layout tv = tile2thrfrg(*tc, refD, ref2trg); + if (!cg::is_valid(tv) || cg::rank(tv.shape()) < 2) return {}; + cg::layout layoutTV = + zeroUnitModes(cg::make_layout(std::vector{cg::get(tv, 0), cg::get(tv, 1)})); + MLIRContext *ctx = atom.getContext(); + return TiledCopyType::get(ctx, atom, LayoutAttr::get(ctx, layoutTV), + tiled.getTilerMn()); +} + +LogicalResult TiledCopyPartitionSOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto tc = tiledCopyOf(a.getTiledCopy().getType()); + auto c = carrierOf(a.getInput().getType()); + auto coord = llvm::dyn_cast(a.getCoord().getType()); + if (!tc || !c || !coord) + return emitOptionalError(loc, "expects a tiled copy, a tensor and a thread coord"); + return infer(loc, out, [&] { return partitionCopy(ctx, *tc, *c, coord.getRef(), true); }); +} + +LogicalResult TiledCopyPartitionDOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto tc = tiledCopyOf(a.getTiledCopy().getType()); + auto c = carrierOf(a.getInput().getType()); + auto coord = llvm::dyn_cast(a.getCoord().getType()); + if (!tc || !c || !coord) + return emitOptionalError(loc, "expects a tiled copy, a tensor and a thread coord"); + return infer(loc, out, [&] { return partitionCopy(ctx, *tc, *c, coord.getRef(), false); }); +} + +LogicalResult TiledCopyRetileOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto tc = tiledCopyOf(a.getTiledCopy().getType()); + auto c = carrierOf(a.getInput().getType()); + if (!tc || !c) return emitOptionalError(loc, "expects a tiled copy and a tensor"); + return infer(loc, out, [&] { return retileCopy(ctx, *tc, *c); }); +} + +// ---------------------------------------------------------------- cute_nvgpu + + +namespace mlir::cutlass_compiler::cute_nvgpu { + +LogicalResult AtomSetValueOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, SmallVectorImpl &out) { + out.push_back(operands[0].getType()); + return success(); +} + +// The executable atom of a TMA descriptor: the element, copy size and basis +// carry over; the kind becomes the mode and, for loads, the CTA count. +LogicalResult AtomCopyMakeExecTmaOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + bool over = a.getOverride(), noOob = a.getNoFullyOobTile(); + Type in = a.getInput().getType(); + auto stride = cg::from_string("()"); + if (!stride) return emitOptionalError(loc, "cannot build the TMA stride"); + auto gStride = StrideType::get(ctx, *stride); + if (auto load = llvm::dyn_cast(in)) { + TiledTmaLoad kind = load.getKind(); + bool twoSm = kind == TiledTmaLoad::sm_100_2sm || kind == TiledTmaLoad::sm_100_2sm_multicast; + bool mcast = kind == TiledTmaLoad::sm_90_multicast || kind == TiledTmaLoad::sm_100_2sm_multicast; + out.push_back(CopyAtomTmaLoadType::get(ctx, load.getValType(), /*sparsity=*/1, + load.getCopyBits(), TmaLoadMode::tiled, twoSm ? 2 : 1, + gStride, mcast, load.getTmaGbasis(), over, noOob)); + return success(); + } + if (auto store = llvm::dyn_cast(in)) { + out.push_back(CopyAtomTmaStoreType::get(ctx, store.getValType(), /*sparsity=*/1, + store.getCopyBits(), TmaStoreMode::tiled, gStride, + store.getTmaGbasis(), over, noOob)); + return success(); + } + if (auto red = llvm::dyn_cast(in)) { + out.push_back(CopyAtomTmaReduceType::get(ctx, red.getValType(), red.getCopyBits(), + TmaStoreMode::tiled, red.getKind(), gStride, + red.getTmaGbasis())); + return success(); + } + if (auto load = llvm::dyn_cast(in)) { + Im2ColTmaLoad kind = load.getKind(); + bool twoSm = kind == Im2ColTmaLoad::sm_100_2sm || kind == Im2ColTmaLoad::sm_100_2sm_multicast; + bool mcast = kind == Im2ColTmaLoad::sm_90_multicast || kind == Im2ColTmaLoad::sm_100_2sm_multicast; + out.push_back(CopyAtomIm2ColTmaLoadType::get(ctx, load.getValType(), load.getCopyBits(), + twoSm ? 2 : 1, gStride, mcast, load.getTmaGbasis())); + return success(); + } + if (auto store = llvm::dyn_cast(in)) { + out.push_back(CopyAtomIm2ColTmaStoreType::get(ctx, store.getValType(), store.getCopyBits(), + gStride, store.getTmaGbasis())); + return success(); + } + // The gather/scatter atoms' executable forms are not recovered. + return emitOptionalError(loc, "expects a non-executable tiled or im2col TMA atom"); +} + + +LogicalResult AtomCopyMakeNonExecTiledTmaLoadOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + return inferTmaAtom(ctx, loc, a.getGmemTensor().getType(), + a.getSmemLayout().getType(), a.getCtaVMap().getType(), + a.getKindAttr(), a.getTmaFormatAttr(), out); +} + +LogicalResult AtomCopyMakeNonExecTiledTmaStoreOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + return inferTmaAtom(ctx, loc, a.getGmemTensor().getType(), + a.getSmemLayout().getType(), a.getCtaVMap().getType(), + /*kind=*/nullptr, a.getTmaFormatAttr(), out); +} + +LogicalResult AtomCopyMakeNonExecTiledTmaReduceOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + return inferTmaAtom(ctx, loc, a.getGmemTensor().getType(), + a.getSmemLayout().getType(), a.getCtaVMap().getType(), + a.getKindAttr(), a.getTmaFormatAttr(), out); +} + +// tma_partition: the smem tile's contiguous vectors, one instruction each, +// read through every tensor; a multicast CTA starts at its share. +LogicalResult AtomTmaPartitionOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto info = tmaAtomInfoOf(a.getTmaAtom().getType()); + auto coord = llvm::dyn_cast(a.getCtaCoord().getType()); + auto ctaLayout = llvm::dyn_cast(a.getCtaLayout().getType()); + auto smem = carrierOf(a.getSmemTensor().getType()); + if (!info || !coord || !ctaLayout || !smem) + return emitOptionalError(loc, "expects a TMA atom, a CTA coord and layout, and tensors"); + auto smemLayout = llvm::dyn_cast_or_null(smem->layout); + if (!smemLayout || !cg::holds_vector(smemLayout.getRef().shape())) + return emitOptionalError(loc, "smem tensor needs a plain layout of rank >= 1"); + int64_t numVal = info->copyBits / info->elemBits; + cg::layout inv = cg::right_inverse(cg::get(smemLayout.getRef(), 0)); + int64_t b = asInt(cg::size(inv)); + int64_t n = asInt(cg::size(smemLayout.getRef(), 0)); + if (b <= 0 || n <= 0 || n % b || numVal <= 0) + return emitOptionalError(loc, "cannot tile the smem vector"); + // tile_to_shape: the vector repeated over the tile, broadcast modes at 0. + cg::layout layoutV = inv; + if (n != b) { + auto rest = layoutOf(std::to_string(n / b) + ":1"); + layoutV = cg::logical_product(inv, *rest); + if (!cg::is_valid(layoutV)) return emitOptionalError(loc, "cannot tile the smem vector"); + } + auto tmaV = layoutOf(std::to_string(numVal) + ":1"); + layoutV = cg::logical_divide(layoutV, *tmaV); + if (!cg::is_valid(layoutV)) return emitOptionalError(loc, "cannot divide the smem vector"); + // The multicast offset inside the instruction's vector. + int64_t ctas = asInt(cg::cosize(ctaLayout.getRef())); + std::optional multicast; + if (ctas > 1) { + // The CTA layout scaled by the share: (cosize:share) o cta_layout. + auto share = layoutOf(std::to_string(ctas) + ":" + std::to_string(numVal / ctas)); + cg::layout scaled = cg::composition(*share, ctaLayout.getRef()); + if (!cg::is_valid(scaled)) return emitOptionalError(loc, "cannot scale the CTA layout"); + multicast = cg::layout_eval(coord.getRef(), scaled); + } + SmallVector results; + auto partition = [&](Type tensor) -> Type { + auto c = carrierOf(tensor); + if (!c) return {}; + auto layout = llvm::dyn_cast(c->layout); + if (!layout) return {}; + cg::layout v = cg::coalesce(composeFirst(layout.getRef(), layoutV), profileOf("((1,1))")); + if (!cg::is_valid(v)) return {}; + // Only the vector's unit modes get a zero stride; the rest keep theirs. + Type result = LayoutType::get(ctx, zeroUnitModesHead(v, cg::rank(v.shape()) - 1, false)); + if (!multicast) return rebuild(ctx, *c, result); + // The index split over the vector's modes, column-major, keeping what + // its divisibility says of each coordinate. + auto vec = cg::get(v.shape(), 0); + if (!cg::holds_vector(vec)) return {}; + auto split = splitIndex(*multicast, vec[0]); + if (!split) return {}; + std::string crd = "((" + *split + ",0)"; + for (size_t i = 1; i < cg::rank(v.shape()); ++i) crd += ",0"; + auto at = cg::from_string(crd + ")"); + if (!at) return {}; + cg::int_tuple offset = cg::layout_eval(*at, v); + if (c->ptr) return CuteMemRefType::get(ctx, movedPtr(c->ptr, offset), result); + if (c->iter) { + auto sum = cg::arith_tuple_sum(c->iter.getRef(), offset); + if (!cg::is_valid(sum)) return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), result); + } + return rebuild(ctx, *c, result); + }; + results.push_back(partition(a.getSmemTensor().getType())); + for (Value target : a.getTargetTensors()) results.push_back(partition(target.getType())); + for (Type t : results) + if (!t) return emitOptionalError(loc, "cannot partition for TMA"); + out.append(results.begin(), results.end()); + return success(); +} + +namespace { +// make_cotiled_copy over a tensor memory tensor for an atom whose thread +// layout is `atomT`. +Type tmemTiledCopy(MLIRContext *ctx, std::optional loc, Type atom, + Type tmem, const std::string &atomT, SmallVectorImpl &out) { + auto traits = copyTraitsOf(atom); + auto m = llvm::dyn_cast(tmem); + auto layout = m ? llvm::dyn_cast(m.getLayout()) : LayoutType(); + if (!traits || !layout || !cg::is_valid(traits->valId)) return {}; + auto t = layoutOf(atomT); + if (!t) return {}; + cg::layout v = cg::coalesce(cg::upcast(traits->bits, traits->valId)); + if (!cg::is_valid(v)) return {}; + auto tiled = cotiledCopy(cg::make_layout(std::vector{*t, v}), layout.getRef()); + if (!tiled) return {}; + return tiledCopyType(ctx, atom, tiled->first, tiled->second); +} +} // namespace + +// make_tmem_copy: four warps, each at the same place in its own lane quarter. +LogicalResult AtomCopyMakeTmemCopyOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto traits = copyTraitsOf(a.getAtom().getType()); + if (!traits || traits->bits <= 0) return emitOptionalError(loc, "expects a tmem atom"); + int64_t lane = kTmemLaneBits / traits->bits; + return infer(loc, out, [&] { + return tmemTiledCopy(ctx, loc, a.getAtom().getType(), a.getTmemMemref().getType(), + "(32,4):(0," + std::to_string(32 * lane) + ")", out); + }); +} + +// make_s2t_copy (make_utccp_copy): one thread per CTA. +LogicalResult AtomCopyMakeS2TCopyOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto traits = copyTraitsOf(a.getCopyS2tAtom().getType()); + if (!traits) return emitOptionalError(loc, "expects an s2t atom"); + return infer(loc, out, [&] { + return tmemTiledCopy(ctx, loc, a.getCopyS2tAtom().getType(), + a.getTmemMemref().getType(), + std::to_string(traits->threads) + ":0", out); + }); +} + +namespace { +// The smem side of an s2t copy as descriptor units: the tile recast to 128 +// bits, viewed through `desc`. +LogicalResult s2tSmemDescView(std::optional loc, Type atom, Type view, + Type desc, SmallVectorImpl &out) { + if (auto tiled = llvm::dyn_cast(atom)) atom = tiled.getCopyAtom(); + auto m = llvm::dyn_cast(view); + auto layout = m ? llvm::dyn_cast(m.getLayout()) : LayoutType(); + if (!llvm::isa(atom) || !layout) + return emitOptionalError(loc, "expects an s2t atom and an smem tensor"); + // The compiler wants mode 0 as (values, 1). + const cg::layout &l = layout.getRef(); + auto m0 = cg::get(l.shape(), 0); + if (!cg::holds_vector(m0) || cg::rank(m0) != 2 || !cg::holds_int(cg::size(m0, 1)) || + cg::size(m0, 1).as_int64() != 1) + return emitOptionalError(loc, "expects mode 0 as (values, 1)"); + return infer(loc, out, [&]() -> Type { + cg::layout v = cg::recast(128, bitsOfType(m.getValueType()), l); + if (!cg::is_valid(v)) return {}; + return SmemDescViewType::get(atom.getContext(), desc, LayoutAttr::get(atom.getContext(), v)); + }); +} + +// An operation whose inference rule is not recovered: the compiler either +// aborts on the inputs tried or needs inputs no probe produced. +LogicalResult unrecovered(std::optional loc, StringRef op) { + return emitOptionalError(loc, op, " result inference is not recovered"); +} +} // namespace + +LogicalResult AtomGetCopyS2TSmemDescViewOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + return s2tSmemDescView(loc, a.getAtom().getType(), a.getView().getType(), + SmemDescType::get(ctx), out); +} + +LogicalResult AtomGetCopyS2TSmemDescViewSM107Op::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + return s2tSmemDescView(loc, a.getAtom().getType(), a.getView().getType(), + SmemDescSM107Type::get(ctx), out); +} + +// A generic-space pointer to the parameter, at its type's natural alignment. +LogicalResult GetGridConstantPointerOp::inferReturnTypes( + MLIRContext *ctx, std::optional, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + Type t = operands[0].getType(); + bool desc = llvm::isa(t); + out.push_back(PtrType::get(ctx, t, AddressSpace::generic, + desc ? 64 : PtrType::getNaturalAlignment(t), {}, {})); + return success(); +} + +// The residues of the TMA tensor's coordinates: one per box mode, starting +// at the extent of the gmem mode it runs over and counting down. +LogicalResult AtomMakeTmaResidueTensorOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + Type atom = a.getTmaAtom().getType(), gbasis; + if (auto l = llvm::dyn_cast(atom)) gbasis = l.getTmaGbasis(); + if (auto s = llvm::dyn_cast(atom)) gbasis = s.getTmaGbasis(); + auto basis = llvm::dyn_cast_or_null(gbasis); + auto m = llvm::dyn_cast(a.getGmemTensor().getType()); + auto gmem = m ? llvm::dyn_cast(m.getLayout()) : LayoutType(); + return infer(loc, out, [&]() -> Type { + if (!basis || !gmem || !cg::holds_vector(basis.getRef().stride())) return {}; + const cg::stride &modes = basis.getRef().stride(); + // Each box mode's one basis: the gmem leaf it runs over. + std::vector> paths; + for (size_t j = 0; j < cg::rank(modes); ++j) { + auto flat = cg::flatten(modes[j]); + const cg::stride &leaf = cg::holds_vector(flat) && cg::rank(flat) == 1 ? flat[0] : flat; + if (!cg::holds_scaled_basis(leaf)) return {}; + paths.push_back(std::get(leaf).modes()); + } + std::vector iter(paths.size()), strides; + std::vector path; + const cg::layout &g = gmem.getRef(); + walkLeaves(g.shape(), g.stride(), g.stride(), path, + [&](const auto &s, const auto &, const auto &, const std::vector &p) { + auto j = std::find(paths.begin(), paths.end(), p) - paths.begin(); + if (j == static_cast(paths.size())) return strides.push_back("0"); + iter[j] = cg::to_string(s); + strides.push_back("-1@" + std::to_string(j)); + }); + for (const std::string &e : iter) + if (e.empty()) return {}; + size_t i = 0; + auto layout = cg::from_string(cg::to_string(g.shape()) + ":" + + nestLike(g.shape(), strides, i)); + auto start = cg::from_string("(" + llvm::join(iter, ",") + ")"); + if (!layout || !start) return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, *start), LayoutType::get(ctx, *layout)); + }); +} + +LogicalResult AtomGetCoordTensorOp::inferReturnTypes( + MLIRContext *, std::optional loc, ValueRange, DictionaryAttr, + PropertyRef, RegionRange, SmallVectorImpl &) { + return unrecovered(loc, "cute_nvgpu.atom.get_coord_tensor"); +} + +// The im2col and gather/scatter TMA atoms and the circular descriptors +// validate inputs (canonical UMMA layouts, im2col corners) whose rules are +// not recovered. +#define CUTE_UNRECOVERED(Op, name) \ + LogicalResult Op::inferReturnTypes( \ + MLIRContext *, std::optional loc, ValueRange, DictionaryAttr, \ + PropertyRef, RegionRange, SmallVectorImpl &) { \ + return unrecovered(loc, name); \ + } +CUTE_UNRECOVERED(AtomCopyMakeNonExec2DGather4TmaLoadOp, "cute_nvgpu.atom.make_non_exec_2d_gather4_tma_load") +CUTE_UNRECOVERED(AtomCopyMakeNonExec2DScatter4TmaStoreOp, "cute_nvgpu.atom.make_non_exec_2d_scatter4_tma_store") +CUTE_UNRECOVERED(AtomCopyMakeNonExecIm2ColTmaLoadOp, "cute_nvgpu.atom.make_non_exec_im2col_tma_load") +CUTE_UNRECOVERED(AtomCopyMakeNonExecIm2ColTmaStoreOp, "cute_nvgpu.atom.make_non_exec_im2col_tma_store") +CUTE_UNRECOVERED(MakeUMMASmemDescCircularSM103Op, "cute_nvgpu.sm103.make_umma_smem_desc_circular") +CUTE_UNRECOVERED(MakeUMMASmemDescCircularSM107Op, "cute_nvgpu.sm107.make_umma_smem_desc_circular") +#undef CUTE_UNRECOVERED + +} // namespace mlir::cutlass_compiler::cute_nvgpu diff --git a/mlir/dialects/cute_ir/lib/Dialect/IR/CuteOps.cpp b/mlir/dialects/cute_ir/lib/Dialect/IR/CuteOps.cpp new file mode 100644 index 00000000..e28b9e8a --- /dev/null +++ b/mlir/dialects/cute_ir/lib/Dialect/IR/CuteOps.cpp @@ -0,0 +1,9 @@ +//===- CuteOps.cpp - the compiler's CuTe operations -----------------------===// + +#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" +// Operand constraints name cute_nvgpu and LLVM types. +#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h" +#include "mlir/Dialect/LLVMIR/LLVMTypes.h" + +#define GET_OP_CLASSES +#include "cute_ir/Dialect/Cute/IR/CuteOps.cpp.inc" diff --git a/mlir/dialects/cute_ir/lib/Dialect/IR/CuteOpsPrivate.cpp b/mlir/dialects/cute_ir/lib/Dialect/IR/CuteOpsPrivate.cpp deleted file mode 100644 index 3492d685..00000000 --- a/mlir/dialects/cute_ir/lib/Dialect/IR/CuteOpsPrivate.cpp +++ /dev/null @@ -1,6 +0,0 @@ -//===- CutePrivateCompatOps.cpp - recovered CuTe operations --------------===// - -#include "cute_ir/Dialect/Cute/IR/CuteDialectPrivate.h" - -#define GET_OP_CLASSES -#include "cute_ir/Dialect/Cute/IR/CuteOpsPrivate.cpp.inc" diff --git a/mlir/dialects/cute_ir/lib/Dialect/IR/CuteTypes.cpp b/mlir/dialects/cute_ir/lib/Dialect/IR/CuteTypes.cpp index 039f341c..bd37b7bd 100644 --- a/mlir/dialects/cute_ir/lib/Dialect/IR/CuteTypes.cpp +++ b/mlir/dialects/cute_ir/lib/Dialect/IR/CuteTypes.cpp @@ -1,5 +1,1597 @@ -//===- CuteTypes.cpp - combined OSS and recovered CuTe types -------------===// +// clang-format off +/*************************************************************************************************** + * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +// clang-format on -#include "cute_ir/Dialect/Cute/IR/CuteDialectPrivate.h" +#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" +#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h" -#include "cute_ir/lib/Dialect/IR/CuteTypes.cpp" +#include "mlir/Dialect/LLVMIR/LLVMTypes.h" +#include "mlir/IR/Builders.h" +#include "mlir/IR/DialectImplementation.h" +#include "mlir/IR/OpImplementation.h" + +#include "llvm/ADT/TypeSwitch.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/MathExtras.h" + +#include + +using namespace mlir; +using namespace mlir::cutlass_compiler::cute; + +namespace cg = cutegen; + +// The compiler's parser of the algebra types reads `<"text">` leniently: a +// missing or unreadable payload, including a bare mnemonic, gives the default +// value (`()`, `():()`, `[]`, `S<0,4,3>`) instead of an error. +template +static T parseLenientPayload(mlir::AsmParser &parser) { + std::string str; + if (succeeded(parser.parseOptionalLess()) && + succeeded(parser.parseOptionalString(&str)) && + succeeded(parser.parseOptionalGreater())) + return cutegen::from_string(str).value_or(T{}); + return T{}; +} + +// The compiler's integer type is `!cute.i`, a mnemonic per width that +// the generated dispatch cannot match. The generated CuteDialect::parseType +// and printType pass the dialect parser and printer, so these overloads of +// the renamed dispatch functions catch them first (defined below). +static OptionalParseResult generatedCuteTypeParser(DialectAsmParser &parser, + StringRef *mnemonic, + Type &value); +static LogicalResult generatedCuteTypePrinter(Type type, + DialectAsmPrinter &printer); +#define generatedTypeParser generatedCuteTypeParser +#define generatedTypePrinter generatedCuteTypePrinter + +// Emit the tablegen-generated type storage / parsing machinery (also drives +// CuteDialect::parseType / printType via useDefaultTypePrinterParser). +#define GET_TYPEDEF_CLASSES +#include "cute_ir/Dialect/Cute/IR/CuteTypes.cpp.inc" +#undef generatedTypeParser +#undef generatedTypePrinter + +//===----------------------------------------------------------------------===// +// CuteDialect — type registration +//===----------------------------------------------------------------------===// + +namespace { +namespace cute = ::mlir::cutlass_compiler::cute; +namespace nv = ::mlir::cutlass_compiler::cute_nvgpu; + +std::string printed(Type type) { + std::string text; + llvm::raw_string_ostream os(text); + type.print(os); + return text; +} + +// The compiler's name for a tiled copy's atom; empty when it has none. +std::string copyAliasName(Type atom) { + return llvm::TypeSwitch(atom) + .Case([](auto) { return "copy_simt"; }) + .Case([](auto) { return "copy_ldgsts"; }) + .Case([](auto) { return "copy_dsmem_store"; }) + .Case([](auto) { return "copy_blkcp_g2s"; }) + .Case([](auto) { return "copy_blkcp_s2g"; }) + .Case([](auto) { return "copy_blkcp_s2s"; }) + .Case([](auto) { return "copy_g2r"; }) + .Case([](auto) { return "copy_r2g"; }) + .Case([](auto) { return "copy_r2s"; }) + .Case([](auto) { return "copy_s2r"; }) + .Case([](auto) { return "copy_utccp"; }) + .Case( + [](auto t) { return "copy_ldtm_" + std::to_string(t.getNumBit()); }) + .Case( + [](auto t) { return "copy_sttm_" + std::to_string(t.getNumBit()); }) + .Case([](auto t) { + return "copy_ldtm_red_" + std::to_string(t.getNumBit()); + }) + .Case([](auto t) { + return "copy_ldtm_spcompress_" + std::to_string(t.getNumBit()); + }) + .Case([](auto t) { + return "copy_ldsm_" + std::to_string(t.getNumMatrices()); + }) + .Case([](auto t) { + return "copy_stsm_" + std::to_string(t.getNumMatrices()); + }) + .Case([](auto t) { + return "copy_tmaldg_" + std::to_string(t.getCopyBits() / 8) + "b"; + }) + .Case([](auto t) { + return "copy_tmastg_" + std::to_string(t.getCopyBits() / 8) + "b"; + }) + .Case([](auto t) { + return "copy_tmaredg_" + std::to_string(t.getCopyBits() / 8) + "b"; + }) + .Default([](Type) { return "copy_unknown_type"; }); +} + +// `mma____xx`; empty for a type that is not an MMA atom. +std::string mmaAliasName(Type atom) { + auto name = [](auto t) { + std::string shape; + auto mnk = cutegen::flatten(t.getShapeMnk().getRef()); + for (size_t i = 0; i < cutegen::rank(mnk); ++i) + shape += (i ? "x" : "") + cutegen::to_string(cutegen::get(mnk, i)); + return "mma_" + printed(t.getAType()) + "_" + printed(t.getBType()) + "_" + + printed(t.getCType()) + "_" + shape; + }; + return llvm::TypeSwitch(atom) + .Case(name) + .Default([](Type) { return std::string(); }); +} + +// The compiler's type aliases: `!memref__` for a memref, +// `!copy_` for a tiled copy and `!mma____xx` for a +// tiled MMA (MLIR escapes and numbers them). Other types have none. +struct CuteOpAsmInterface : public OpAsmDialectInterface { + using OpAsmDialectInterface::OpAsmDialectInterface; + AliasResult getAlias(Type type, raw_ostream &os) const override { + std::string name; + if (auto memref = llvm::dyn_cast(type)) + name = "memref_" + + stringifyAddressSpace(memref.getPtr().getAddressSpace()).str() + + "_" + printed(memref.getPtr().getValueType()); + else if (auto copy = llvm::dyn_cast(type)) + name = copyAliasName(copy.getCopyAtom()); + else if (auto mma = llvm::dyn_cast(type)) + name = mmaAliasName(mma.getMmaAtom()); + if (name.empty()) + return AliasResult::NoAlias; + os << name; + return AliasResult::OverridableAlias; + } +}; +} // namespace + +void CuteDialect::registerCuteTypes(CuteDialect *dialect) { + dialect->addTypes< +#define GET_TYPEDEF_LIST +#include "cute_ir/Dialect/Cute/IR/CuteTypes.cpp.inc" + >(); + dialect->addInterfaces(); +} + +//===----------------------------------------------------------------------===// +// IntTupleType — custom assembly format +// + +// Grammar: < " cutegen-int-tuple-string " > (read leniently, see above) +//===----------------------------------------------------------------------===// + +mlir::Type IntTupleType::parse(mlir::AsmParser &parser) { + return get(parser.getContext(), + IntTupleAttr::get(parser.getContext(), + parseLenientPayload(parser))); +} + +void IntTupleType::print(mlir::AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getRef())); + printer << '>'; +} + +bool IntTupleType::isStatic() const { return cutegen::is_static(getRef()); } + +mlir::Attribute IntTupleType::getValueAttr() const { return getAttr(); } + +llvm::TypeSize +IntTupleType::getTypeSize(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); +} + +llvm::TypeSize +IntTupleType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +IntTupleType::getABIAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +IntTupleType::getPreferredAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypePreferredAlignment( + cg::get_llvm_type(builder, getRef())); +} + +std::optional +IntTupleType::getIndexBitwidth(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); +} + +bool IntTupleType::areCompatible( + mlir::DataLayoutEntryListRef oldLayout, + mlir::DataLayoutEntryListRef newLayout, + mlir::DataLayoutSpecInterface newSpec, + const mlir::DataLayoutIdentifiedEntryMap &identified) const { + OpBuilder builder(getContext()); + // The lowering can be a bare integer (scalar/depth-0 dynamic leaves), which + // does not implement DataLayoutTypeInterface; only delegate when it does and + // otherwise fall back to the interface default ("compatible"). + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); + } + return true; +} + +llvm::LogicalResult +IntTupleType::verifyEntries(mlir::DataLayoutEntryListRef entries, + mlir::Location loc) const { + OpBuilder builder(getContext()); + // See areCompatible: bare-integer lowerings cannot be queried, so only + // delegate when the lowered type implements the interface. + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.verifyEntries(entries, loc); + } + return mlir::success(); +} + +//===----------------------------------------------------------------------===// +// CoordType — custom assembly format +//===----------------------------------------------------------------------===// + +mlir::Type CoordType::parse(mlir::AsmParser &parser) { + return get(parser.getContext(), + CoordAttr::get(parser.getContext(), + parseLenientPayload(parser))); +} + +void CoordType::print(mlir::AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getRef())); + printer << '>'; +} + +bool CoordType::isStatic() const { return cutegen::is_static(getRef()); } + +mlir::Attribute CoordType::getValueAttr() const { return getAttr(); } + +llvm::TypeSize +CoordType::getTypeSize(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); +} + +llvm::TypeSize +CoordType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); +} + +uint64_t CoordType::getABIAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +CoordType::getPreferredAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypePreferredAlignment( + cg::get_llvm_type(builder, getRef())); +} + +std::optional +CoordType::getIndexBitwidth(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); +} + +bool CoordType::areCompatible( + mlir::DataLayoutEntryListRef oldLayout, + mlir::DataLayoutEntryListRef newLayout, + mlir::DataLayoutSpecInterface newSpec, + const mlir::DataLayoutIdentifiedEntryMap &identified) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); + } + return true; +} + +llvm::LogicalResult +CoordType::verifyEntries(mlir::DataLayoutEntryListRef entries, + mlir::Location loc) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.verifyEntries(entries, loc); + } + return mlir::success(); +} + +//===----------------------------------------------------------------------===// +// ShapeType — custom assembly format +//===----------------------------------------------------------------------===// + +mlir::Type ShapeType::parse(mlir::AsmParser &parser) { + return get(parser.getContext(), + ShapeAttr::get(parser.getContext(), + parseLenientPayload(parser))); +} + +void ShapeType::print(mlir::AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getRef())); + printer << '>'; +} + +bool ShapeType::isStatic() const { return cutegen::is_static(getRef()); } + +mlir::Attribute ShapeType::getValueAttr() const { return getAttr(); } + +llvm::TypeSize +ShapeType::getTypeSize(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); +} + +llvm::TypeSize +ShapeType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); +} + +uint64_t ShapeType::getABIAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +ShapeType::getPreferredAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypePreferredAlignment( + cg::get_llvm_type(builder, getRef())); +} + +std::optional +ShapeType::getIndexBitwidth(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); +} + +bool ShapeType::areCompatible( + mlir::DataLayoutEntryListRef oldLayout, + mlir::DataLayoutEntryListRef newLayout, + mlir::DataLayoutSpecInterface newSpec, + const mlir::DataLayoutIdentifiedEntryMap &identified) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); + } + return true; +} + +llvm::LogicalResult +ShapeType::verifyEntries(mlir::DataLayoutEntryListRef entries, + mlir::Location loc) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.verifyEntries(entries, loc); + } + return mlir::success(); +} + +//===----------------------------------------------------------------------===// +// StrideType — custom assembly format +//===----------------------------------------------------------------------===// + +mlir::Type StrideType::parse(mlir::AsmParser &parser) { + auto loc = parser.getCurrentLocation(); + auto attr = StrideAttr::get(parser.getContext(), + parseLenientPayload(parser)); + return getChecked([&] { return parser.emitError(loc); }, parser.getContext(), + attr); +} + +void StrideType::print(mlir::AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getRef())); + printer << '>'; +} + +bool StrideType::isStatic() const { return cutegen::is_static(getRef()); } + +mlir::Attribute StrideType::getValueAttr() const { return getAttr(); } + +// The compiler accepts every stride cutegen reads, mixed integer and +// scaled-basis leaves and mixed-depth basis paths included. +mlir::LogicalResult +StrideType::verify(llvm::function_ref emitError, + StrideAttr attr) { + return mlir::success(); +} + +llvm::TypeSize +StrideType::getTypeSize(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); +} + +llvm::TypeSize +StrideType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +StrideType::getABIAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +StrideType::getPreferredAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypePreferredAlignment( + cg::get_llvm_type(builder, getRef())); +} + +std::optional +StrideType::getIndexBitwidth(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); +} + +bool StrideType::areCompatible( + mlir::DataLayoutEntryListRef oldLayout, + mlir::DataLayoutEntryListRef newLayout, + mlir::DataLayoutSpecInterface newSpec, + const mlir::DataLayoutIdentifiedEntryMap &identified) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); + } + return true; +} + +llvm::LogicalResult +StrideType::verifyEntries(mlir::DataLayoutEntryListRef entries, + mlir::Location loc) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.verifyEntries(entries, loc); + } + return mlir::success(); +} + +//===----------------------------------------------------------------------===// +// LayoutType — custom assembly format +//===----------------------------------------------------------------------===// + +mlir::Type LayoutType::parse(mlir::AsmParser &parser) { + auto loc = parser.getCurrentLocation(); + auto attr = LayoutAttr::get(parser.getContext(), + parseLenientPayload(parser)); + return getChecked([&] { return parser.emitError(loc); }, parser.getContext(), + attr); +} + +void LayoutType::print(mlir::AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getRef())); + printer << '>'; +} + +bool LayoutType::isStatic() const { return cutegen::is_static(getRef()); } + +mlir::Attribute LayoutType::getValueAttr() const { return getAttr(); } + +/// Verify LayoutType storage invariants. cutegen leaves these +/// unchecked at construction time for performance; enforce them at the +/// IR boundary. +mlir::LogicalResult +LayoutType::verify(llvm::function_ref emitError, + LayoutAttr attr) { + auto const &layout = attr.getRef(); + if (cutegen::has_error(layout)) { + return emitError() << "expects no error(`x`) in layout, but got \"" + << cutegen::to_string(layout) << "\""; + } + if (cutegen::any_leaf_is(layout.shape(), [](auto const &e) { + return cutegen::holds_int(e) && cutegen::get_int(e) <= 0; + })) { + return emitError() << "expects positive shape mode, but got \"" + << cutegen::to_string(layout) << "\""; + } + if (!cutegen::is_congruent(layout.shape(), layout.stride())) { + return emitError() + << "expects shape and stride profile to match, but got \"" + << cutegen::to_string(layout) << "\""; + } + return mlir::success(); +} + +llvm::TypeSize +LayoutType::getTypeSize(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); +} + +llvm::TypeSize +LayoutType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +LayoutType::getABIAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +LayoutType::getPreferredAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypePreferredAlignment( + cg::get_llvm_type(builder, getRef())); +} + +std::optional +LayoutType::getIndexBitwidth(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); +} + +bool LayoutType::areCompatible( + mlir::DataLayoutEntryListRef oldLayout, + mlir::DataLayoutEntryListRef newLayout, + mlir::DataLayoutSpecInterface newSpec, + const mlir::DataLayoutIdentifiedEntryMap &identified) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); + } + return true; +} + +llvm::LogicalResult +LayoutType::verifyEntries(mlir::DataLayoutEntryListRef entries, + mlir::Location loc) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.verifyEntries(entries, loc); + } + return mlir::success(); +} + +//===----------------------------------------------------------------------===// +// TileType — custom assembly format +//===----------------------------------------------------------------------===// + +mlir::Type TileType::parse(mlir::AsmParser &parser) { + auto loc = parser.getCurrentLocation(); + auto attr = TileAttr::get(parser.getContext(), + parseLenientPayload(parser)); + return getChecked([&] { return parser.emitError(loc); }, parser.getContext(), + attr); +} + +void TileType::print(mlir::AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getRef())); + printer << '>'; +} + +bool TileType::isStatic() const { return cutegen::is_static(getRef()); } + +mlir::Attribute TileType::getValueAttr() const { return getAttr(); } + +// Like strides, the layouts of a tile are not restricted further. +mlir::LogicalResult +TileType::verify(llvm::function_ref emitError, + TileAttr attr) { + return mlir::success(); +} + +llvm::TypeSize +TileType::getTypeSize(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); +} + +llvm::TypeSize +TileType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); +} + +uint64_t TileType::getABIAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +TileType::getPreferredAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypePreferredAlignment( + cg::get_llvm_type(builder, getRef())); +} + +std::optional +TileType::getIndexBitwidth(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); +} + +bool TileType::areCompatible( + mlir::DataLayoutEntryListRef oldLayout, + mlir::DataLayoutEntryListRef newLayout, + mlir::DataLayoutSpecInterface newSpec, + const mlir::DataLayoutIdentifiedEntryMap &identified) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); + } + return true; +} + +llvm::LogicalResult +TileType::verifyEntries(mlir::DataLayoutEntryListRef entries, + mlir::Location loc) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.verifyEntries(entries, loc); + } + return mlir::success(); +} + +//===----------------------------------------------------------------------===// +// ComposedLayoutType — custom assembly format +//===----------------------------------------------------------------------===// + +mlir::Type ComposedLayoutType::parse(mlir::AsmParser &parser) { + // Capture the start location so verify() failures (positivity, + // congruence) are attributed to the type literal. + auto loc = parser.getCurrentLocation(); + if (parser.parseLess()) { + return {}; + } + std::string str; + if (parser.parseString(&str)) { + return {}; + } + auto opt = cutegen::from_string(str); + if (!opt) { + parser.emitError(parser.getCurrentLocation(), + "failed to parse composed_layout from \"" + str + "\""); + return {}; + } + if (parser.parseGreater()) { + return {}; + } + auto attr = ComposedLayoutAttr::get(parser.getContext(), std::move(*opt)); + return ComposedLayoutType::getChecked([&] { return parser.emitError(loc); }, + parser.getContext(), attr); +} + +void ComposedLayoutType::print(mlir::AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getRef())); + printer << '>'; +} + +bool ComposedLayoutType::isStatic() const { + // No free cutegen::is_static(composed_layout_t) exists — check each part. + const auto &cl = getRef(); + bool b_ok = cutegen::is_static(cl.layout_b()); + bool off_ok = cutegen::is_static(cl.offset()); + bool a_ok = cl.is_a_swizzle() || cutegen::is_static(cl.layout_a()); + return b_ok && off_ok && a_ok; +} + +mlir::Attribute ComposedLayoutType::getValueAttr() const { return getAttr(); } + +/// Verify ComposedLayoutType storage invariants: no error sentinel, +/// positive static shape leaves, addable offset, and a sum that's +/// composable into the inner layout (scalar for swizzle, scalar or +/// weakly congruent for affine). +mlir::LogicalResult ComposedLayoutType::verify( + llvm::function_ref emitError, + ComposedLayoutAttr attr) { + auto const &composed = attr.getRef(); + if (cutegen::has_error(composed)) { + return emitError() << "expects no error(`x`) in layout, but got \"" + << cutegen::to_string(composed) << "\""; + } + // A trivial wrap (identity swizzle, no offset) is accepted and prints as + // its plain layout, as the compiler does. + auto hasNonPositiveShape = [](cutegen::layout const &lyt) { + return cutegen::any_leaf_is(lyt.shape(), [](auto const &e) { + return cutegen::holds_int(e) && cutegen::get_int(e) <= 0; + }); + }; + if (hasNonPositiveShape(composed.layout_b()) || + (composed.is_a_affine() && hasNonPositiveShape(composed.layout_a()))) { + return emitError() << "expects positive shape mode, but got \"" + << cutegen::to_string(composed) << "\""; + } + + // Addable: the offset and layout_eval(0, outer) must be compatible + // under arith-tuple addition (no scaled-basis-vs-integer mismatch + // or rank disagreement). + auto offset_b = cutegen::layout_eval(0, composed.layout_b()); + auto sum = + cutegen::arith_tuple_sum(composed.offset(), offset_b); + if (cutegen::has_error(sum)) { + return emitError() << "expects offset and `layout_eval(0, outer)` to be " + "addable, but got " + << cutegen::to_string(composed.offset()) << " and " + << cutegen::to_string(offset_b); + } + + // Swizzle inner: the arith-tuple sum must be a scalar (int or + // dynamic int) — swizzles can only consume scalar indices, so any + // scaled-basis structure surviving from the offset or outer is a + // composability error. + if (composed.is_a_swizzle() && !cutegen::holds_int_or_dynamic_int(sum)) { + return emitError() << "swizzle layout expects scalar `offset` and " + "`outer` without scaled basis, but got " + << cutegen::to_string(composed.offset()) << " and " + << cutegen::to_string(composed.layout_b()); + } + + // Affine inner: the sum either is a scalar (and layout_a indexes + // it directly) or matches layout_a's shape profile so each leaf + // dispatches to the corresponding mode of A. + if (composed.is_a_affine() && !cutegen::holds_int_or_dynamic_int(sum) && + !cutegen::weakly_congruent(sum, composed.layout_a().shape())) { + return emitError() + << "expects arith sum of `layout_eval(0, outer)` and `offset` to be " + "scalar or weakly congruent to inner shape, but got sum " + << cutegen::to_string(sum) << " and inner shape " + << cutegen::to_string(composed.layout_a().shape()); + } + + return mlir::success(); +} + +llvm::TypeSize +ComposedLayoutType::getTypeSize(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); +} + +llvm::TypeSize ComposedLayoutType::getTypeSizeInBits( + const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +ComposedLayoutType::getABIAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); +} + +uint64_t ComposedLayoutType::getPreferredAlignment( + const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypePreferredAlignment( + cg::get_llvm_type(builder, getRef())); +} + +std::optional ComposedLayoutType::getIndexBitwidth( + const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); +} + +bool ComposedLayoutType::areCompatible( + mlir::DataLayoutEntryListRef oldLayout, + mlir::DataLayoutEntryListRef newLayout, + mlir::DataLayoutSpecInterface newSpec, + const mlir::DataLayoutIdentifiedEntryMap &identified) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); + } + return true; +} + +llvm::LogicalResult +ComposedLayoutType::verifyEntries(mlir::DataLayoutEntryListRef entries, + mlir::Location loc) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.verifyEntries(entries, loc); + } + return mlir::success(); +} + +//===----------------------------------------------------------------------===// +// SwizzleType — custom assembly format +//===----------------------------------------------------------------------===// + +mlir::Type SwizzleType::parse(mlir::AsmParser &parser) { + auto loc = parser.getCurrentLocation(); + auto attr = SwizzleAttr::get(parser.getContext(), + parseLenientPayload(parser)); + return getChecked([&] { return parser.emitError(loc); }, parser.getContext(), + attr); +} + +void SwizzleType::print(mlir::AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getRef())); + printer << '>'; +} + +bool SwizzleType::isStatic() const { return true; } + +mlir::Attribute SwizzleType::getValueAttr() const { return getAttr(); } + +/// Verify SwizzleType storage invariants. Defense-in-depth — the +/// string parser already rejects invalid swizzle parameters via +/// cutegen::from_string, but programmatic +/// construction goes around the parser, so re-check here. +/// +/// cutegen::swizzle::is_valid (swizzle.hpp) enforces two +/// invariants on S: +/// 1. |num_shift| + num_bits + num_base <= 32 — parameters +/// must fit within the swizzle's 32-bit working width. +/// 2. |num_shift| >= num_bits — the shift amount must cover +/// the mask width so the Y and Z masks do not overlap. +mlir::LogicalResult +SwizzleType::verify(llvm::function_ref emitError, + SwizzleAttr attr) { + auto const &sw = attr.getRef(); + uint32_t numBits = sw.num_bits(); + uint32_t numBase = sw.num_base(); + int32_t numShift = sw.num_shift(); + uint32_t absShift = static_cast(std::abs(numShift)); + constexpr uint32_t kBitWidth = 32; + + if (absShift + numBits + numBase > kBitWidth) { + return emitError() << "expects |num_shift| + num_bits + num_base <= " + << kBitWidth << ", but got " << absShift << " + " + << numBits << " + " << numBase << " = " + << (absShift + numBits + numBase) << " for swizzle " + << cutegen::to_string(sw); + } + if (absShift < numBits) { + return emitError() << "expects |num_shift| >= num_bits, but got " + << absShift << " < " << numBits << " for swizzle " + << cutegen::to_string(sw); + } + return mlir::success(); +} + +// A swizzle is a compile-time-only object whose runtime representation is an +// empty LLVM struct (mirroring convertSwizzleType in CuteTypeConverter.cpp), so +// the data-layout queries delegate to that struct rather than to +// cg::get_llvm_type. +static mlir::Type getSwizzleLLVMType(mlir::MLIRContext *ctx) { + return LLVM::LLVMStructType::getLiteral(ctx, SmallVector{}); +} + +llvm::TypeSize +SwizzleType::getTypeSize(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + return dataLayout.getTypeSize(getSwizzleLLVMType(getContext())); +} + +llvm::TypeSize +SwizzleType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + return dataLayout.getTypeSizeInBits(getSwizzleLLVMType(getContext())); +} + +uint64_t +SwizzleType::getABIAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + return dataLayout.getTypeABIAlignment(getSwizzleLLVMType(getContext())); +} + +uint64_t +SwizzleType::getPreferredAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + return dataLayout.getTypePreferredAlignment(getSwizzleLLVMType(getContext())); +} + +std::optional +SwizzleType::getIndexBitwidth(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + return dataLayout.getTypeIndexBitwidth(getSwizzleLLVMType(getContext())); +} + +bool SwizzleType::areCompatible( + mlir::DataLayoutEntryListRef oldLayout, + mlir::DataLayoutEntryListRef newLayout, + mlir::DataLayoutSpecInterface newSpec, + const mlir::DataLayoutIdentifiedEntryMap &identified) const { + if (auto dlTy = llvm::dyn_cast( + getSwizzleLLVMType(getContext()))) { + return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); + } + return true; +} + +llvm::LogicalResult +SwizzleType::verifyEntries(mlir::DataLayoutEntryListRef entries, + mlir::Location loc) const { + if (auto dlTy = llvm::dyn_cast( + getSwizzleLLVMType(getContext()))) { + return dlTy.verifyEntries(entries, loc); + } + return mlir::success(); +} + +//===----------------------------------------------------------------------===// +// The DSL compiler's types and attributes (CuteTypesCompiler.td, +// CuteAttrsCompiler.td) +//===----------------------------------------------------------------------===// + +#include "cute_ir/Dialect/Cute/IR/CuteEnums.cpp.inc" + +static OptionalParseResult generatedCuteTypeParser(DialectAsmParser &parser, + StringRef *mnemonic, + Type &value) { + OptionalParseResult result = generatedCuteTypeParser( + static_cast(parser), mnemonic, value); + if (result.has_value()) + return result; + StringRef digits = *mnemonic; + unsigned width; + if (!digits.consume_front("i") || digits.empty() || + !llvm::all_of(digits, llvm::isDigit) || digits.getAsInteger(10, width)) + return std::nullopt; + value = ConstrainedIntType::parse(parser, width); + return success(!!value); +} + +static LogicalResult generatedCuteTypePrinter(Type type, + DialectAsmPrinter &printer) { + if (auto integer = llvm::dyn_cast(type)) { + integer.print(printer); + return success(); + } + return generatedCuteTypePrinter(type, static_cast(printer)); +} + +// Bits of an integer or float (tf32 is stored in 32), or of a vector of them. +static unsigned storageBits(Type type) { + if (auto integer = llvm::dyn_cast_or_null(type)) + return integer.getWidth(); + if (auto fp = llvm::dyn_cast_or_null(type)) + return fp.isTF32() ? 32 : fp.getWidth(); + if (auto vector = llvm::dyn_cast_or_null(type)) + return vector.getNumElements() * storageBits(vector.getElementType()); + return 0; +} + +static bool isIdentitySwizzle(SwizzleAttr swizzle) { + return !swizzle || cutegen::to_string(swizzle.getRef()) == "S<0,4,3>"; +} + +uint64_t PtrType::getNaturalAlignment(Type valueType) { + unsigned bits = 0; + if (auto sparse = llvm::dyn_cast_or_null(valueType)) + bits = storageBits(sparse.getPhysicalType()); + else if (llvm::isa_and_nonnull(valueType)) + bits = storageBits(valueType); + if (bits <= 1) + return 1; + return bits % 8 ? 0 : bits / 8; +} + +BitLayoutAttr PtrType::getDefaultBitLayout(Type valueType, + AddressSpace addressSpace) { + unsigned chunk = addressSpace == AddressSpace::tmem ? 32 : 8; + std::string text; + if (auto sparse = llvm::dyn_cast_or_null(valueType)) { + unsigned bits = storageBits(sparse.getPhysicalType()); + if (!bits) + return {}; + text = llvm::formatv("({0},({1},{2})):(1,(0,{0}))", bits, + sparse.getNumLogical(), chunk / std::gcd(bits, chunk)) + .str(); + } else if (llvm::isa_and_nonnull(valueType)) { + unsigned bits = storageBits(valueType); + unsigned stride = bits == 1 ? 8 : bits; + text = llvm::formatv("({0},{1}):(1,{2})", bits, + chunk / std::gcd(stride, chunk), stride) + .str(); + } else { + return {}; + } + auto layout = cutegen::from_string(text); + if (!layout) + return {}; + MLIRContext *ctx = valueType.getContext(); + return BitLayoutAttr::get(ctx, LayoutAttr::get(ctx, std::move(*layout))); +} + +// The bits of a chunk: the largest extent times stride of the element mode. +static std::optional chunkBits(const cutegen::layout &layout) { + auto elements = cutegen::get(layout, 1); + auto shape = cutegen::flatten(elements.shape()); + auto stride = cutegen::flatten(elements.stride()); + if (cutegen::holds_int(shape) && cutegen::holds_int(stride)) + return shape.as_int64() * stride.as_int64(); + int64_t bits = 0; + for (size_t i = 0; i < cutegen::rank(shape); ++i) { + auto extent = cutegen::get(shape, i); + auto step = cutegen::get(stride, i); + if (!cutegen::holds_int(extent) || !cutegen::holds_int(step)) + return std::nullopt; + bits = std::max(bits, extent.as_int64() * step.as_int64()); + } + return bits; +} + +LogicalResult BitLayoutAttr::verify(function_ref emitError, + LayoutAttr layout) { + const cutegen::layout &l = layout.getRef(); + if (!cutegen::is_static(l)) + return emitError() << "bitlayout must be fully static but got " << layout; + if (cutegen::rank(l) != 2) + return emitError() << "bitlayout must have 2 modes corresponding to " + "(element bit, element index), but got " + << cutegen::rank(l); + auto elements = cutegen::size(l, 1); + if (cutegen::holds_int(elements) && elements.as_int64() == 0) + return emitError() + << "bitlayout must contain at least one logical element, but got 0"; + return success(); +} + +//===- ptr and memref -----------------------------------------------------===// + +namespace { +struct PtrFields { + Type valueType; + AddressSpace space = AddressSpace::generic; + std::optional alignment; + SwizzleAttr swizzle; + BitLayoutAttr bitlayout; +}; +} // namespace + +// `bitlayout<"layout">`, the keyword already read. +static BitLayoutAttr parseBitLayout(AsmParser &parser) { + std::string text; + if (failed(parser.parseLess()) || failed(parser.parseString(&text)) || + failed(parser.parseGreater())) + return {}; + auto layout = cutegen::from_string(text); + if (!layout) { + parser.emitError(parser.getCurrentLocation(), "expects cg::layout"); + return {}; + } + MLIRContext *ctx = parser.getContext(); + return BitLayoutAttr::getChecked( + [&] { return parser.emitError(parser.getCurrentLocation()); }, ctx, + LayoutAttr::get(ctx, std::move(*layout))); +} + +// `S`, the `S` already read. +static SwizzleAttr parseInlineSwizzle(AsmParser &parser) { + int64_t bits, base, shift; + if (failed(parser.parseLess()) || failed(parser.parseInteger(bits)) || + failed(parser.parseComma()) || failed(parser.parseInteger(base)) || + failed(parser.parseComma()) || failed(parser.parseInteger(shift)) || + failed(parser.parseGreater())) + return {}; + auto value = cutegen::from_string( + llvm::formatv("S<{0},{1},{2}>", bits, base, shift).str()); + if (!value) { + parser.emitError(parser.getCurrentLocation(), "invalid swizzle parameters"); + return {}; + } + return SwizzleAttr::get(parser.getContext(), std::move(*value)); +} + +// A layout or composed layout spelled as a string; a composed layout that is +// a plain one (identity swizzle, no offset) is read as that layout. +static Type parseLayoutPayload(AsmParser &parser) { + std::string text; + llvm::SMLoc loc = parser.getCurrentLocation(); + if (failed(parser.parseString(&text))) + return {}; + MLIRContext *ctx = parser.getContext(); + auto emitError = [&] { return parser.emitError(loc); }; + if (auto value = cutegen::from_string(text)) + return LayoutType::getChecked(emitError, ctx, + LayoutAttr::get(ctx, std::move(*value))); + if (auto value = cutegen::from_string(text)) { + if (value->is_normal_layout()) + return LayoutType::getChecked(emitError, ctx, + LayoutAttr::get(ctx, value->layout_b())); + return ComposedLayoutType::getChecked( + emitError, ctx, ComposedLayoutAttr::get(ctx, std::move(*value))); + } + parser.emitError(parser.getCurrentLocation(), "layout parse failure"); + return {}; +} + +static void printLayoutPayload(AsmPrinter &printer, Type layout) { + if (auto plain = llvm::dyn_cast(layout)) + printer.printString(cutegen::to_string(plain.getRef())); + else + printer.printString( + cutegen::to_string(llvm::cast(layout).getRef())); +} + +static LogicalResult memrefLayoutExpected(AsmParser &parser) { + return parser.emitError(parser.getCurrentLocation(), + "failed to parse CuteMemRefType parameter `,` " + "expected, layout cannot be omitted."); +} + +// `[type, ][space][, align][, S][, bitlayout<"L">]` in this order; +// a memref requires the type and ends with its layout string instead of the +// bit layout, which follows the layout. +static FailureOr parsePtrFields(AsmParser &parser, Type *layout) { + PtrFields fields; + OptionalParseResult typed = parser.parseOptionalType(fields.valueType); + if (typed.has_value() && failed(*typed)) + return failure(); + if (layout && !typed.has_value()) + return memrefLayoutExpected(parser); + enum { kSpace, kAlign, kSwizzle, kBitLayout, kDone } next = kSpace; + bool first = !typed.has_value(); + while (true) { + if (!first && failed(parser.parseOptionalComma())) + break; + StringRef keyword; + if (failed(parser.parseOptionalKeyword(&keyword))) { + if (layout) { + if (!(*layout = parseLayoutPayload(parser))) + return failure(); + return fields; + } + if (first) + break; + parser.emitError(parser.getCurrentLocation(), + "expected a pointer field after `,`"); + return failure(); + } + first = false; + if (auto space = symbolizeAddressSpace(keyword); + space && next <= kSpace) { + fields.space = *space; + next = kAlign; + } else if (keyword == "align" && next <= kAlign) { + uint64_t alignment; + if (failed(parser.parseLess()) || failed(parser.parseInteger(alignment)) || + failed(parser.parseGreater())) + return failure(); + fields.alignment = alignment; + next = kSwizzle; + } else if (keyword == "S" && next <= kSwizzle) { + if (!(fields.swizzle = parseInlineSwizzle(parser))) + return failure(); + next = kBitLayout; + } else if (keyword == "bitlayout" && !layout && next <= kBitLayout) { + if (!(fields.bitlayout = parseBitLayout(parser))) + return failure(); + next = kDone; + } else { + parser.emitError(parser.getCurrentLocation(), "unexpected pointer field `") + << keyword << "`"; + return failure(); + } + } + if (layout) + return memrefLayoutExpected(parser); + return fields; +} + +// Defaults are left out: they are what the parser fills in. +static PtrType buildPtr(AsmParser &parser, const PtrFields &fields) { + SwizzleAttr swizzle = isIdentitySwizzle(fields.swizzle) ? SwizzleAttr() + : fields.swizzle; + BitLayoutAttr bitlayout = fields.bitlayout; + if (bitlayout && + bitlayout == PtrType::getDefaultBitLayout(fields.valueType, fields.space)) + bitlayout = {}; + return PtrType::getChecked( + [&] { return parser.emitError(parser.getCurrentLocation()); }, + parser.getContext(), fields.valueType, fields.space, + fields.alignment.value_or(PtrType::getNaturalAlignment(fields.valueType)), + swizzle, bitlayout); +} + +static void printPtrFields(AsmPrinter &printer, PtrType ptr) { + if (ptr.getValueType()) { + printer.printType(ptr.getValueType()); + printer << ", "; + } + printer << stringifyAddressSpace(ptr.getAddressSpace()); + if (ptr.getAlignment() != PtrType::getNaturalAlignment(ptr.getValueType())) + printer << ", align<" << ptr.getAlignment() << '>'; + if (!isIdentitySwizzle(ptr.getSwizzle())) + printer << ", " << cutegen::to_string(ptr.getSwizzle().getRef()); +} + +static void printBitLayout(AsmPrinter &printer, PtrType ptr) { + BitLayoutAttr bitlayout = ptr.getBitlayout(); + if (!bitlayout || bitlayout == PtrType::getDefaultBitLayout( + ptr.getValueType(), ptr.getAddressSpace())) + return; + printer << ", bitlayout<"; + printer.printString(cutegen::to_string(bitlayout.getLayout().getRef())); + printer << '>'; +} + +Type PtrType::parse(AsmParser &parser) { + if (failed(parser.parseLess())) + return {}; + FailureOr fields = parsePtrFields(parser, nullptr); + if (failed(fields) || failed(parser.parseGreater())) + return {}; + return buildPtr(parser, *fields); +} + +void PtrType::print(AsmPrinter &printer) const { + printer << '<'; + printPtrFields(printer, *this); + printBitLayout(printer, *this); + printer << '>'; +} + +LogicalResult PtrType::verify(function_ref emitError, + Type valueType, AddressSpace addressSpace, + uint64_t alignment, SwizzleAttr swizzle, + BitLayoutAttr bitlayout) { + if (alignment == 0 && getNaturalAlignment(valueType) != 0) + return emitError() << "expects alignment >= 1, got 0"; + bool tmem = addressSpace == AddressSpace::tmem; + if (tmem && swizzle && swizzle.getRef().num_bits() != 0) + return emitError() << "tmem pointer does not support non-identity swizzle"; + BitLayoutAttr defaultBitLayout = getDefaultBitLayout(valueType, addressSpace); + BitLayoutAttr effective = bitlayout ? bitlayout : defaultBitLayout; + if (!effective) + return success(); + if (auto bits = chunkBits(effective.getLayout().getRef())) { + if (*bits % 8 != 0) + return emitError() << "bitlayout chunk must be byte aligned, but size is " + << *bits << " bits"; + if (tmem && (*bits < 32 || !llvm::isPowerOf2_64(*bits))) + return emitError() + << "tmem pointer only supports 32 bit chunks that is a power of " + "2, got " + << *bits; + } + if (llvm::isa_and_nonnull(valueType) && + effective != defaultBitLayout) + return emitError() << "expects bitlayout to be " << defaultBitLayout + << " for valueType '" << valueType << "'but got " + << effective; + return success(); +} + +// Whether the offsets of `layout` are whole aligned blocks of `sparsity` +// elements: the contiguous run from offset 0 (stride-1 mode, then each mode +// whose stride is the run so far) is a multiple of it, and so is every other +// stride. +static bool isDenseForSparsity(const cutegen::layout &layout, + int64_t sparsity) { + if (!cutegen::is_static(layout)) + return false; + auto shape = cutegen::flatten(layout.shape()); + auto stride = cutegen::flatten(layout.stride()); + std::vector> modes; // (stride, extent) + auto add = [&](const auto &extent, const auto &step) { + if (extent.as_int64() != 1 && step.as_int64() != 0) + modes.emplace_back(step.as_int64(), extent.as_int64()); + }; + if (cutegen::holds_int(shape)) { + add(shape, stride); + } else { + for (size_t i = 0; i < cutegen::rank(shape); ++i) + add(cutegen::get(shape, i), cutegen::get(stride, i)); + } + llvm::sort(modes, [](auto a, auto b) { + return std::abs(a.first) < std::abs(b.first); + }); + int64_t run = 1; + bool contiguous = true; + for (auto [step, extent] : modes) { + if (contiguous && step == run) { + run *= extent; + continue; + } + contiguous = false; + if (step % sparsity != 0) + return false; + } + return run % sparsity == 0; +} + +Type mlir::cutlass_compiler::cute::MemRefType::parse(AsmParser &parser) { + if (failed(parser.parseLess())) + return {}; + Type layout; + FailureOr fields = parsePtrFields(parser, &layout); + if (failed(fields)) + return {}; + if (succeeded(parser.parseOptionalComma())) { + if (failed(parser.parseKeyword("bitlayout")) || + !(fields->bitlayout = parseBitLayout(parser))) + return {}; + } + if (failed(parser.parseGreater())) + return {}; + PtrType ptr = buildPtr(parser, *fields); + if (!ptr) + return {}; + return getChecked([&] { return parser.emitError(parser.getCurrentLocation()); }, + parser.getContext(), ptr, layout); +} + +void mlir::cutlass_compiler::cute::MemRefType::print( + AsmPrinter &printer) const { + printer << '<'; + printPtrFields(printer, getPtr()); + printer << ", "; + printLayoutPayload(printer, getLayout()); + printBitLayout(printer, getPtr()); + printer << '>'; +} + +LogicalResult mlir::cutlass_compiler::cute::MemRefType::verify( + function_ref emitError, PtrType ptr, Type layout) { + if (!ptr) + return emitError() << "expects a CuTe pointer"; + if (!llvm::isa(layout)) + return emitError() << "expects a CuTe layout or composed layout"; + Type valueType = ptr.getValueType(); + if (!valueType || !(valueType.isIntOrIndexOrFloat() || + llvm::isa(valueType))) + return emitError() + << "expects value type to be int, float, ptr or sparse_elem, but got '" + << valueType << "'"; + if (auto sparse = llvm::dyn_cast(valueType)) { + int64_t sparsity = sparse.getNumLogical(); + bool dense; + if (auto composed = llvm::dyn_cast(layout)) { + // A swizzle may not move elements within a block. + const auto &c = composed.getRef(); + dense = c.is_a_swizzle() && + (int64_t(1) << c.swizzle_a().num_base()) % sparsity == 0 && + cutegen::holds_int(c.offset()) && + c.offset().as_int64() % sparsity == 0 && + isDenseForSparsity(c.layout_b(), sparsity); + } else { + dense = isDenseForSparsity(llvm::cast(layout).getRef(), + sparsity); + } + if (!dense) + return emitError() + << "layout must be dense but cannot be divided by sparsity"; + } + if (auto composed = llvm::dyn_cast(layout)) { + if (!isIdentitySwizzle(ptr.getSwizzle()) && composed.getRef().is_a_swizzle() && + !composed.getRef().swizzle_a().is_identity()) + return emitError() << "cannot have both a swizzle on the pointer and a " + "non-identity swizzle in the layout"; + } else if (cutegen::has_scaled_basis( + llvm::cast(layout).getRef().stride())) { + return emitError() << "expected iterator and layout codomain to have " + "congruent profiles, but got iterator '" + << ptr << "' and layout layout: " + << cutegen::to_string( + llvm::cast(layout).getRef()); + } + return success(); +} + +//===- arith_tuple_iter, coord_tensor, i, fast_divmod_divisor, ... -----===// + +Type ArithTupleIteratorType::parse(AsmParser &parser) { + auto tuple = IntTupleType::parse(parser); + return tuple ? get(parser.getContext(), llvm::cast(tuple)) + : Type(); +} + +void ArithTupleIteratorType::print(AsmPrinter &printer) const { + getArithTuple().print(printer); +} + +Type CoordTensorType::parse(AsmParser &parser) { + std::string text; + if (failed(parser.parseLess()) || failed(parser.parseString(&text))) + return {}; + auto tuple = cutegen::from_string(text); + if (!tuple) { + parser.emitError(parser.getCurrentLocation(), + "failed to parse CoordTensorType parameter 'arith_tuple' " + "which is to be a `cutegen::int_tuple`"); + return {}; + } + Type layout; + if (failed(parser.parseComma()) || !(layout = parseLayoutPayload(parser)) || + failed(parser.parseGreater())) + return {}; + return getChecked([&] { return parser.emitError(parser.getCurrentLocation()); }, + parser.getContext(), + IntTupleType::get(parser.getContext(), std::move(*tuple)), + layout); +} + +void CoordTensorType::print(AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getArithTuple().getRef())); + printer << ", "; + printLayoutPayload(printer, getLayout()); + printer << '>'; +} + +LogicalResult CoordTensorType::verify( + function_ref emitError, IntTupleType arithTuple, + Type layout) { + if (!arithTuple) + return emitError() << "expects an arithmetic tuple"; + if (!llvm::isa(layout)) + return emitError() << "expects a CuTe layout or composed layout"; + return success(); +} + +// `<[pow2, ]divby D>`, or nothing for a plain integer. +Type ConstrainedIntType::parse(AsmParser &parser, unsigned width) { + int64_t divisibility = 1; + bool isPow2 = false; + if (succeeded(parser.parseOptionalLess())) { + isPow2 = succeeded(parser.parseOptionalKeyword("pow2")); + bool divided = !isPow2 || succeeded(parser.parseOptionalComma()); + if (divided && (failed(parser.parseKeyword("divby")) || + failed(parser.parseInteger(divisibility)))) + return {}; + if (failed(parser.parseGreater())) + return {}; + } + return get(parser.getContext(), divisibility, width, isPow2); +} + +void ConstrainedIntType::print(AsmPrinter &printer) const { + printer << 'i' << getWidth(); + if (!getIsPow2() && getDivisibility() == 1) + return; + printer << '<'; + if (getIsPow2()) + printer << "pow2"; + if (getDivisibility() != 1) + printer << (getIsPow2() ? ", " : "") << "divby " << getDivisibility(); + printer << '>'; +} + +Type FastDivmodDivisorType::parse(AsmParser &parser) { + unsigned width; + if (failed(parser.parseLess()) || failed(parser.parseInteger(width))) + return {}; + bool isPow2 = false; + if (succeeded(parser.parseOptionalComma())) { + if (failed(parser.parseKeyword("is_pow_2"))) + return {}; + isPow2 = true; + } + if (failed(parser.parseGreater())) + return {}; + return get(parser.getContext(), width, isPow2); +} + +void FastDivmodDivisorType::print(AsmPrinter &printer) const { + printer << '<' << getWidth(); + if (getIsPow_2()) + printer << ", is_pow_2"; + printer << '>'; +} + +LogicalResult SparseElemType::verify( + function_ref emitError, int numLogical, + Type physicalType) { + if (numLogical <= 0) + return emitError() << "sparse element type logical count must be > 0"; + Type element = physicalType; + if (auto vector = llvm::dyn_cast_or_null(physicalType)) + element = vector.getElementType(); + if (!llvm::isa_and_nonnull(element)) + return emitError() << "unsupported physical element type '" << physicalType + << "'"; + return success(); +} + +LogicalResult TiledMmaType::verify(function_ref emitError, + Type mmaAtom, LayoutAttr atomLayoutMNK, + TileAttr permutationMNK) { + if (cutegen::rank(atomLayoutMNK.getRef()) != 3) + return emitError() << "TiledMMA requires rank-3 atomLayoutMNK"; + if (permutationMNK && cutegen::rank(permutationMNK.getRef()) != 3) + return emitError() << "TiledMMA requires rank-3 permutationMNK"; + return success(); +} diff --git a/mlir/dialects/cute_ir/lib/Dialect/IR/CuteTypesPrivate.cpp b/mlir/dialects/cute_ir/lib/Dialect/IR/CuteTypesPrivate.cpp deleted file mode 100644 index 676b2340..00000000 --- a/mlir/dialects/cute_ir/lib/Dialect/IR/CuteTypesPrivate.cpp +++ /dev/null @@ -1,333 +0,0 @@ -#include "cute_ir/Dialect/Cute/IR/CuteDialectPrivate.h" - -#include "mlir/IR/DialectImplementation.h" - -#include "llvm/Support/MathExtras.h" - -using namespace mlir; -using namespace mlir::cutlass_compiler::cute; - -//===----------------------------------------------------------------------===// -// CuTe-DSL runtime and tensor-view types -//===----------------------------------------------------------------------===// - -static IntTupleType parseIntTuplePayload(AsmParser &parser) { - std::string text; - if (failed(parser.parseString(&text))) - return {}; - auto value = cutegen::from_string(text); - if (!value) { - parser.emitError(parser.getCurrentLocation(), - "failed to parse arithmetic tuple from \"") - << text << '"'; - return {}; - } - return IntTupleType::get(parser.getContext(), std::move(*value)); -} - -static Type parseLayoutPayload(AsmParser &parser) { - std::string text; - if (failed(parser.parseString(&text))) - return {}; - if (auto value = cutegen::from_string(text)) - return LayoutType::get(parser.getContext(), std::move(*value)); - if (auto value = cutegen::from_string(text)) - return ComposedLayoutType::get(parser.getContext(), std::move(*value)); - parser.emitError(parser.getCurrentLocation(), "failed to parse layout from \"") - << text << '"'; - return {}; -} - -static void printLayoutPayload(AsmPrinter &printer, Type layout) { - if (auto plain = llvm::dyn_cast(layout)) { - printer.printString(cutegen::to_string(plain.getRef())); - return; - } - auto composed = llvm::cast(layout); - printer.printString(cutegen::to_string(composed.getRef())); -} - -Type ArithTupleIteratorType::parse(AsmParser &parser) { - if (failed(parser.parseLess())) - return {}; - IntTupleType tuple = parseIntTuplePayload(parser); - if (!tuple || failed(parser.parseGreater())) - return {}; - return get(parser.getContext(), tuple); -} - -void ArithTupleIteratorType::print(AsmPrinter &printer) const { - printer << '<'; - printer.printString(cutegen::to_string(getArithTuple().getRef())); - printer << '>'; -} - -template -static Type parseConstrainedInt(AsmParser &parser) { - uint64_t divisibleBy = 1; - if (succeeded(parser.parseOptionalLess())) { - if (failed(parser.parseKeyword("divby")) || - failed(parser.parseInteger(divisibleBy)) || - failed(parser.parseGreater())) - return {}; - } - return ConstrainedType::getChecked( - [&] { return parser.emitError(parser.getCurrentLocation()); }, - parser.getContext(), divisibleBy); -} - -template -static void printConstrainedInt(ConstrainedType type, AsmPrinter &printer) { - if (type.getDivisibleBy() != 1) - printer << "'; -} - -Type ConstrainedInt32Type::parse(AsmParser &parser) { - return parseConstrainedInt(parser); -} - -void ConstrainedInt32Type::print(AsmPrinter &printer) const { - printConstrainedInt(*this, printer); -} - -LogicalResult ConstrainedInt32Type::verify( - function_ref emitError, uint64_t divisibleBy) { - return divisibleBy > 0 ? success() - : emitError() << "expects divisible_by > 0"; -} - -Type ConstrainedInt64Type::parse(AsmParser &parser) { - return parseConstrainedInt(parser); -} - -void ConstrainedInt64Type::print(AsmPrinter &printer) const { - printConstrainedInt(*this, printer); -} - -LogicalResult ConstrainedInt64Type::verify( - function_ref emitError, uint64_t divisibleBy) { - return divisibleBy > 0 ? success() - : emitError() << "expects divisible_by > 0"; -} - -Type FastDivmodDivisorType::parse(AsmParser &parser) { - unsigned width; - if (failed(parser.parseLess()) || failed(parser.parseInteger(width))) - return {}; - bool isPow2 = false; - if (succeeded(parser.parseOptionalComma())) { - if (failed(parser.parseKeyword("is_pow_2"))) - return {}; - isPow2 = true; - } - if (failed(parser.parseGreater())) - return {}; - return getChecked([&] { return parser.emitError(parser.getCurrentLocation()); }, - parser.getContext(), width, isPow2); -} - -void FastDivmodDivisorType::print(AsmPrinter &printer) const { - printer << '<' << getWidth(); - if (getIsPow2()) - printer << ", is_pow_2"; - printer << '>'; -} - -LogicalResult FastDivmodDivisorType::verify( - function_ref emitError, unsigned width, - bool /*isPow2*/) { - return width == 32 || width == 64 - ? success() - : emitError() << "expects width to be 32 or 64, got " << width; -} - -LogicalResult SparseElemType::verify( - function_ref emitError, int numLogical, - Type physicalType) { - if (numLogical <= 0) - return emitError() << "expects num_logical > 0"; - if (!physicalType) - return emitError() << "expects a physical storage type"; - return success(); -} - -namespace { -struct ParsedPtrFields { - Type valueType; - StringAttr memorySpace; - uint64_t alignment = 0; - SwizzleAttr swizzle; -}; -} // namespace - -static FailureOr parseInlineSwizzle(AsmParser &parser) { - int64_t bits, base, shift; - if (failed(parser.parseLess()) || failed(parser.parseInteger(bits)) || - failed(parser.parseComma()) || failed(parser.parseInteger(base)) || - failed(parser.parseComma()) || failed(parser.parseInteger(shift)) || - failed(parser.parseGreater())) - return failure(); - std::string text = "S<" + std::to_string(bits) + "," + - std::to_string(base) + "," + std::to_string(shift) + ">"; - auto value = cutegen::from_string(text); - if (!value) { - parser.emitError(parser.getCurrentLocation(), "invalid pointer swizzle ") - << text; - return failure(); - } - return SwizzleAttr::get(parser.getContext(), std::move(*value)); -} - -static FailureOr parsePtrFields(AsmParser &parser, - bool expectLayout, - Type *layout) { - ParsedPtrFields fields; - OptionalParseResult valueResult = parser.parseOptionalType(fields.valueType); - if (valueResult.has_value() && failed(*valueResult)) - return failure(); - if (valueResult.has_value() && failed(parser.parseComma())) - return failure(); - - StringRef memorySpace; - if (failed(parser.parseKeyword(&memorySpace))) - return failure(); - fields.memorySpace = StringAttr::get(parser.getContext(), memorySpace); - - while (succeeded(parser.parseOptionalComma())) { - if (succeeded(parser.parseOptionalKeyword("align"))) { - if (failed(parser.parseLess()) || - failed(parser.parseInteger(fields.alignment)) || - failed(parser.parseGreater())) - return failure(); - continue; - } - if (succeeded(parser.parseOptionalKeyword("S"))) { - FailureOr swizzle = parseInlineSwizzle(parser); - if (failed(swizzle)) - return failure(); - fields.swizzle = *swizzle; - continue; - } - if (expectLayout && layout && !*layout) { - *layout = parseLayoutPayload(parser); - if (!*layout) - return failure(); - continue; - } - parser.emitError(parser.getCurrentLocation(), - "expected align, swizzle, or layout"); - return failure(); - } - return fields; -} - -static void printPtrFields(AsmPrinter &printer, PtrType pointer) { - if (pointer.getValueType()) { - printer.printType(pointer.getValueType()); - printer << ", "; - } - printer << pointer.getMemorySpace().getValue(); - if (pointer.getAlignment() != 0) - printer << ", align<" << pointer.getAlignment() << '>'; - if (pointer.getSwizzle()) - printer << ", " << cutegen::to_string(pointer.getSwizzle().getRef()); -} - -Type PtrType::parse(AsmParser &parser) { - if (failed(parser.parseLess())) - return {}; - FailureOr fields = parsePtrFields(parser, false, nullptr); - if (failed(fields) || failed(parser.parseGreater())) - return {}; - return getChecked([&] { return parser.emitError(parser.getCurrentLocation()); }, - parser.getContext(), fields->valueType, - fields->memorySpace, fields->alignment, fields->swizzle); -} - -void PtrType::print(AsmPrinter &printer) const { - printer << '<'; - printPtrFields(printer, *this); - printer << '>'; -} - -LogicalResult PtrType::verify(function_ref emitError, - Type /*valueType*/, StringAttr memorySpace, - uint64_t alignment, SwizzleAttr /*swizzle*/) { - if (!memorySpace) - return emitError() << "expects an address space"; - StringRef space = memorySpace.getValue(); - if (space != "generic" && space != "gmem" && space != "cmem" && - space != "smem" && space != "rmem" && space != "tmem" && - space != "dsmem") - return emitError() << "invalid address space '" << space << "'"; - if (alignment != 0 && !llvm::isPowerOf2_64(alignment)) - return emitError() << "expects power-of-two alignment, got " << alignment; - return success(); -} - -Type mlir::cutlass_compiler::cute::MemRefType::parse(AsmParser &parser) { - if (failed(parser.parseLess())) - return {}; - Type layout; - FailureOr fields = parsePtrFields(parser, true, &layout); - if (failed(fields) || !layout || failed(parser.parseGreater())) - return {}; - PtrType pointer = PtrType::getChecked( - [&] { return parser.emitError(parser.getCurrentLocation()); }, - parser.getContext(), fields->valueType, fields->memorySpace, - fields->alignment, fields->swizzle); - if (!pointer) - return {}; - return getChecked([&] { return parser.emitError(parser.getCurrentLocation()); }, - parser.getContext(), pointer, layout); -} - -void mlir::cutlass_compiler::cute::MemRefType::print( - AsmPrinter &printer) const { - printer << '<'; - printPtrFields(printer, getPtr()); - printer << ", "; - printLayoutPayload(printer, getLayout()); - printer << '>'; -} - -LogicalResult mlir::cutlass_compiler::cute::MemRefType::verify( - function_ref emitError, PtrType pointer, - Type layout) { - if (!pointer) - return emitError() << "expects a CuTe pointer"; - if (!llvm::isa(layout)) - return emitError() << "expects a CuTe layout or composed layout"; - return success(); -} - -Type CoordTensorType::parse(AsmParser &parser) { - if (failed(parser.parseLess())) - return {}; - IntTupleType tuple = parseIntTuplePayload(parser); - if (!tuple || failed(parser.parseComma())) - return {}; - Type layout = parseLayoutPayload(parser); - if (!layout || failed(parser.parseGreater())) - return {}; - return getChecked([&] { return parser.emitError(parser.getCurrentLocation()); }, - parser.getContext(), tuple, layout); -} - -void CoordTensorType::print(AsmPrinter &printer) const { - printer << '<'; - printer.printString(cutegen::to_string(getArithTuple().getRef())); - printer << ", "; - printLayoutPayload(printer, getLayout()); - printer << '>'; -} - -LogicalResult CoordTensorType::verify( - function_ref emitError, IntTupleType tuple, - Type layout) { - if (!tuple) - return emitError() << "expects an arithmetic tuple"; - if (!llvm::isa(layout)) - return emitError() << "expects a CuTe layout or composed layout"; - return success(); -} diff --git a/mlir/dialects/cute_ir/lib/Registration/RegisterCute.cpp b/mlir/dialects/cute_ir/lib/Registration/RegisterCute.cpp deleted file mode 100755 index c382e735..00000000 --- a/mlir/dialects/cute_ir/lib/Registration/RegisterCute.cpp +++ /dev/null @@ -1,63 +0,0 @@ -// clang-format off -/*************************************************************************************************** - * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: BSD-3-Clause - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -// clang-format on - -#include "cute_ir/Conversion/CuteToBase/Passes.h" -#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" -#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h" -#include "cute_ir/Dialect/Cute/Transforms/Passes.h" -#include "cute_ir/Registration/Registration.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/GPU/IR/GPUDialect.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/Math/IR/Math.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Dialect/UB/IR/UBOps.h" - -using namespace mlir; - -namespace mlir::cutlass_compiler::cute { - -void registerCuteDialects(DialectRegistry ®istry) { - registry.insert(); -} - -void registerCutePasses() { - registerCuteTransformsPasses(); - registerCuteConversionPasses(); -} - -} // namespace mlir::cutlass_compiler::cute diff --git a/mlir/dialects/cute_ir/test/BUILD.bazel b/mlir/dialects/cute_ir/test/BUILD.bazel deleted file mode 100644 index 4f1d7484..00000000 --- a/mlir/dialects/cute_ir/test/BUILD.bazel +++ /dev/null @@ -1,23 +0,0 @@ -load("@rules_cc//cc:defs.bzl", "cc_test") - -package(default_visibility = ["//visibility:public"]) - -cc_test( - name = "type_constructors_test", - srcs = ["TypeConstructorsTest.c"], - deps = [ - "//mlir/dialects/cute_ir/lib/CAPI:cute", - "//mlir/dialects/cute_ir/lib/CAPI:cute_nvgpu", - "@llvm-project//mlir:CAPIIR", - ], -) - -cc_test( - name = "capi_registration_test", - srcs = ["CAPIRegistrationTest.cpp"], - deps = [ - "//mlir/dialects/cute_ir/lib/CAPI:cute", - "//mlir/dialects/cute_ir/lib/CAPI:cute_nvgpu", - "@llvm-project//mlir:CAPIIR", - ], -) diff --git a/mlir/dialects/cute_ir/test/CAPIRegistrationTest.cpp b/mlir/dialects/cute_ir/test/CAPIRegistrationTest.cpp deleted file mode 100644 index 8e0496c7..00000000 --- a/mlir/dialects/cute_ir/test/CAPIRegistrationTest.cpp +++ /dev/null @@ -1,96 +0,0 @@ -//===- CAPIRegistrationTest.cpp - CuTe C API smoke test ------------------===// - -#include "cute_ir-c/Dialect/Cute.h" -#include "cute_ir-c/Dialect/CuteNVGPU.h" -#include "mlir-c/IR.h" - -#include -#include - -namespace { - -int fail(MlirContext context, const char *message) { - std::fprintf(stderr, "%s\n", message); - mlirContextDestroy(context); - return 1; -} - -void appendString(MlirStringRef chunk, void *userData) { - static_cast(userData)->append(chunk.data, chunk.length); -} - -} // namespace - -int main() { - MlirContext context = mlirContextCreate(); - - MlirDialectHandle cute = mlirGetDialectHandle__cute__(); - MlirDialectHandle cuteNVGPU = mlirGetDialectHandle__cute_nvgpu__(); - mlirDialectHandleRegisterDialect(cute, context); - mlirDialectHandleRegisterDialect(cuteNVGPU, context); - - if (mlirDialectIsNull(mlirDialectHandleLoadDialect(cute, context))) - return fail(context, "failed to load the cute dialect"); - if (mlirDialectIsNull(mlirDialectHandleLoadDialect(cuteNVGPU, context))) - return fail(context, "failed to load the cute_nvgpu dialect"); - - MlirType cutePublicType = mlirTypeParseGet( - context, mlirStringRefCreateFromCString("!cute.int_tuple<\"1\">")); - if (mlirTypeIsNull(cutePublicType) || !mlirTypeIsACuteType(cutePublicType)) - return fail(context, "failed to parse an OSS cute type"); - - MlirType cutePrivateType = - mlirTypeParseGet(context, mlirStringRefCreateFromCString("!cute.i32")); - if (mlirTypeIsNull(cutePrivateType) || !mlirTypeIsACuteType(cutePrivateType)) - return fail(context, "failed to parse a recovered cute type"); - - MlirType cuteNVGPUType = mlirTypeParseGet( - context, mlirStringRefCreateFromCString("!cute_nvgpu.smem_desc")); - if (mlirTypeIsNull(cuteNVGPUType) || !mlirTypeIsACuteNVGPUType(cuteNVGPUType)) - return fail(context, "failed to parse a cute_nvgpu type"); - - if (!mlirContextIsRegisteredOperation( - context, mlirStringRefCreateFromCString("cute.add_offset"))) - return fail(context, "recovered cute operation is not registered"); - if (!mlirContextIsRegisteredOperation( - context, - mlirStringRefCreateFromCString("cute_nvgpu.arch.alloc_rmem"))) - return fail(context, "cute_nvgpu operation is not registered"); - - const char *privateOpsModule = R"mlir( -module { - %alloc = "cute.memref.alloc_smem"() : () -> !cute.i32 - %assumed = "cute.assume"(%alloc) : (!cute.i32) -> !cute.i32 - %tuple = "cute.make_tuple"(%alloc, %assumed) - : (!cute.i32, !cute.i32) -> !cute.i32 - "cute.copy"(%tuple, %alloc, %assumed) - <{operandSegmentSizes = array}> - : (!cute.i32, !cute.i32, !cute.i32) -> () -} -)mlir"; - MlirModule module = mlirModuleCreateParse( - context, mlirStringRefCreateFromCString(privateOpsModule)); - if (mlirModuleIsNull(module)) - return fail(context, "failed to parse recovered cute operations"); - bool privateOpsAreValid = mlirOperationVerify(mlirModuleGetOperation(module)); - std::string bytecode; - mlirOperationWriteBytecode(mlirModuleGetOperation(module), appendString, - &bytecode); - mlirModuleDestroy(module); - if (!privateOpsAreValid) - return fail(context, "recovered cute operations failed verification"); - - MlirOperation roundTripped = mlirOperationCreateParse( - context, mlirStringRefCreate(bytecode.data(), bytecode.size()), - mlirStringRefCreateFromCString("private-ops.mlirbc")); - if (mlirOperationIsNull(roundTripped)) - return fail(context, "failed to read recovered operations from bytecode"); - bool bytecodeIsValid = mlirOperationVerify(roundTripped); - mlirOperationDestroy(roundTripped); - if (!bytecodeIsValid) - return fail(context, - "recovered operations failed after bytecode round trip"); - - mlirContextDestroy(context); - return 0; -} diff --git a/mlir/dialects/cute_ir/test/TypeConstructorsTest.c b/mlir/dialects/cute_ir/test/TypeConstructorsTest.c deleted file mode 100644 index 317a7193..00000000 --- a/mlir/dialects/cute_ir/test/TypeConstructorsTest.c +++ /dev/null @@ -1,70 +0,0 @@ -#include "cute_ir-c/Dialect/Cute.h" -#include "cute_ir-c/Dialect/CuteNVGPU.h" -#include "mlir-c/BuiltinAttributes.h" -#include "mlir-c/BuiltinTypes.h" - -#include -#include - -#define CHECK(condition) \ - do { \ - if (!(condition)) { \ - fprintf(stderr, "failed at line %d: %s\n", __LINE__, #condition); \ - mlirContextDestroy(context); \ - return 1; \ - } \ - } while (0) - -int main(void) { - MlirContext context = mlirContextCreate(); - MlirType nullType = {NULL}; - MlirAttribute nullAttr = {NULL}; - MlirType f32 = mlirF32TypeGet(context); - MlirAttribute space = - mlirStringAttrGet(context, mlirStringRefCreateFromCString("smem")); - MlirAttribute layoutAttr = - mlirCuteLayoutAttrGet(context, mlirStringRefCreateFromCString("4:1")); - CHECK(mlirAttributeIsACuteLayout(layoutAttr)); - CHECK(!mlirAttributeIsACuteLayout(nullAttr)); - CHECK(!mlirAttributeIsACuteLayout(space)); - MlirStringRef layoutValue = mlirCuteLayoutAttrGetValue(layoutAttr); - CHECK(layoutValue.length == 3 && memcmp(layoutValue.data, "4:1", 3) == 0); - CHECK(mlirAttributeIsNull(mlirCuteLayoutAttrGet( - context, mlirStringRefCreateFromCString("invalid")))); - - // Constructors load their dialect and build storage without parsing a type. - MlirType pointer = mlirCutePtrTypeGet(context, f32, space, 16, nullAttr); - CHECK(mlirTypeIsACutePtr(pointer)); - CHECK(mlirTypeEqual(mlirCutePtrTypeGetValueType(pointer), f32)); - CHECK(mlirAttributeEqual(mlirCutePtrTypeGetMemorySpace(pointer), space)); - CHECK(mlirCutePtrTypeGetAlignment(pointer) == 16); - CHECK(mlirAttributeIsNull(mlirCutePtrTypeGetSwizzle(pointer))); - CHECK(!mlirTypeIsACutePtr(f32)); - CHECK(!mlirTypeIsACutePtr(nullType)); - CHECK( - mlirTypeIsNull(mlirCutePtrTypeGet(context, f32, nullAttr, 16, nullAttr))); - - MlirType integer = mlirCuteConstrainedInt32TypeGet(context, 8); - CHECK(mlirTypeIsACuteConstrainedInt32(integer)); - CHECK(mlirCuteConstrainedInt32TypeGetDivisibleBy(integer) == 8); - - MlirType layout = mlirCuteLayoutTypeGet(context, layoutAttr); - CHECK(mlirTypeIsACuteLayout(layout)); - CHECK(mlirAttributeEqual(mlirCuteLayoutTypeGetAttr(layout), layoutAttr)); - CHECK(mlirTypeIsNull(mlirCuteLayoutTypeGet(context, space))); - MlirType memref = mlirCuteMemRefTypeGet(context, pointer, layout); - CHECK(mlirTypeIsACuteMemRef(memref)); - CHECK(mlirTypeEqual(mlirCuteMemRefTypeGetPtr(memref), pointer)); - CHECK(mlirTypeEqual(mlirCuteMemRefTypeGetLayout(memref), layout)); - - MlirType descriptor = mlirCuteNVGPUSmemDescTypeGet(context); - CHECK(mlirTypeIsACuteNVGPUSmemDesc(descriptor)); - MlirAttribute payload = - mlirStringAttrGet(context, mlirStringRefCreateFromCString("")); - MlirType atom = mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGet(context, payload); - CHECK(mlirTypeIsACuteNVGPUCopyAtomSIMTSyncCopy(atom)); - CHECK(mlirAttributeEqual( - mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetPayload(atom), payload)); - mlirContextDestroy(context); - return 0; -} diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 89b44492..cf3c27c8 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -43,21 +43,21 @@ def _cpu_plugin_impl(mctx): name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, sha256 = "e825d8d1031ef536f5a1ac2c1eb0a10f8af83168ffb14c3cc8b26aee25ed1f87", - url = "https://mirror.zml.ai/plugins/202609221707.75.1.0cf3c5a71e94/zml-cpu-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202609231543.88.1.e2f61df0be45/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "5da35f0e471a410f3a3edba95f8a849ea017c1ed32c6fab0239a25a5d21acad4", - url = "https://mirror.zml.ai/plugins/202609221707.75.1.0cf3c5a71e94/zml-cpu-darwin-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202609231543.88.1.e2f61df0be45/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "b0f6a8a1486780a9433b08153d71e0e31ed0e4cde60f200346c687c9346b2d53", - url = "https://mirror.zml.ai/plugins/202609221707.75.1.0cf3c5a71e94/zml-cpu-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202609231543.88.1.e2f61df0be45/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index f173e8ab..4f3a31f5 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -224,12 +224,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "a36251f0e8dbff5135a7434d7ae89ec2ec50767825400f38e9ec7578937db043", - "url": "https://mirror.zml.ai/plugins/202609221707.75.1.0cf3c5a71e94/zml-cuda-linux-amd64.tar.zst", + "sha256": "b5ad9b8d37e59457b219a5c6a025d6bea598f3415019c0003a6c5f659d673028", + "url": "https://mirror.zml.ai/plugins/202609231543.88.1.e2f61df0be45/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "a73bc23e230df4ae3dce6a709b0006dfd17f968d8cec6474f833a8409542b72f", - "url": "https://mirror.zml.ai/plugins/202609221707.75.1.0cf3c5a71e94/zml-cuda-linux-arm64.tar.zst", + "sha256": "470a14e7fc9973ac732fb7eb868599adb1549a2aefa01ed3b2497a15ed8a678b", + "url": "https://mirror.zml.ai/plugins/202609231543.88.1.e2f61df0be45/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 9c8f628f..b97ea6b7 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "3797e34249e59175665e36370b7b71a6983ccc28cdc272fafa1d25424a47f5d4", - url = "https://mirror.zml.ai/plugins/202609221707.75.1.0cf3c5a71e94/zml-metal-darwin-arm64.tar.zst", + sha256 = "219654b57a70eb982f0b25d39de301e66f752e71b55df23bdaa25d12f011eba2", + url = "https://mirror.zml.ai/plugins/202609231543.88.1.e2f61df0be45/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 4f94d748..a6e056c7 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202609221707.75.1.0cf3c5a71e94/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "c5aacc6e6f5c14b1b65c352e45c329a4481de1b152794988a3c243325c82da7c" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202609231543.88.1.e2f61df0be45/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "02274718670a0c523301003b3e6d045178219bb7017a179b4a229f8b9cee8391" def _rocm_package_name(name): return name + _ROCM_VERSION diff --git a/third_party/cute_ir/cute_ir.bazel b/third_party/cute_ir/cute_ir.bazel index 7a9de330..0c0aed19 100644 --- a/third_party/cute_ir/cute_ir.bazel +++ b/third_party/cute_ir/cute_ir.bazel @@ -3,32 +3,49 @@ load("@rules_cc//cc:defs.bzl", "cc_library") package(default_visibility = ["//visibility:public"]) +licenses(["notice"]) # BSD-3-Clause + +# The CuTe dialect of the cutlass_compiler/ tree, as released. The tree that +# builds it (//mlir/dialects/cute_ir) adds the DSL compiler's own attributes, +# types and operations and compiles these sources with them. +CUTE_IR_INCLUDE = "cute_ir/include" + +exports_files([ + "LICENSE.txt", + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteDialect.h", + "cute_ir/lib/Dialect/IR/CuteAttributes.cpp", + "cute_ir/lib/Dialect/IR/CuteDialect.cpp", +]) + + +# cutegen, the CuTe layout algebra, with the `?{div=N}` leaf added by +# cutegen_dynamic_divisibility.patch: a dynamic extent the DSL knows to be a +# multiple of N, which the release's cutegen cannot read. cc_library( name = "cutegen", - hdrs = glob(["cutegen/include/cutegen/**/*.hpp"]), + hdrs = glob(["cutegen/include/cutegen/*.hpp"]), includes = ["cutegen/include"], deps = [ "@llvm-project//llvm:Support", - "@llvm-project//mlir:IR", "@llvm-project//mlir:ArithDialect", - "@llvm-project//mlir:MathDialect", + "@llvm-project//mlir:IR", "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:MathDialect", + "@llvm-project//mlir:NVVMDialect", "@llvm-project//mlir:SCFDialect", "@llvm-project//mlir:UBDialect", - "@llvm-project//mlir:NVVMDialect", ], ) td_library( - name = "dialect_td", + name = "cute_ir_dialect_td", srcs = [ - "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteAttrs.td", - "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteDialect.td", - "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td", - "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.td", - "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypes.td", + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteAttrs.td", + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteDialect.td", + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.td", + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteTypes.td", ], - includes = ["cute_ir/include"], + includes = [CUTE_IR_INCLUDE], deps = [ "@llvm-project//mlir:BuiltinDialectTdFiles", "@llvm-project//mlir:DataLayoutInterfacesTdFiles", @@ -39,159 +56,49 @@ td_library( ) gentbl_cc_library( - name = "dialect_inc_gen", - includes = ["."], + name = "cute_ir_dialect_inc_gen", + includes = [CUTE_IR_INCLUDE], tbl_outs = [ ( ["-gen-dialect-decls", "-dialect=cute"], - "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteDialect.h.inc", + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteDialect.h.inc", ), ( ["-gen-dialect-defs", "-dialect=cute"], - "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteDialect.cpp.inc", + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteDialect.cpp.inc", ), ], tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteDialect.td", - deps = [":dialect_td"], + td_file = CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteDialect.td", + deps = [":cute_ir_dialect_td"], ) gentbl_cc_library( - name = "type_interfaces_inc_gen", - includes = ["."], + name = "cute_ir_type_interfaces_inc_gen", + includes = [CUTE_IR_INCLUDE], tbl_outs = [ ( ["-gen-type-interface-decls"], - "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.h.inc", + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.h.inc", ), ( ["-gen-type-interface-defs"], - "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.cpp.inc", + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.cpp.inc", ), ], tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.td", - deps = [":dialect_td"], -) - -gentbl_cc_library( - name = "attrs_inc_gen", - includes = ["."], - tbl_outs = [ - ( - ["-gen-attrdef-decls", "-attrdefs-dialect=cute"], - "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteAttrs.h.inc", - ), - ( - ["-gen-attrdef-defs", "-attrdefs-dialect=cute"], - "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteAttrs.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteAttrs.td", - deps = [":dialect_td"], -) - -gentbl_cc_library( - name = "ops_inc_gen", - includes = ["."], - tbl_outs = [ - ( - ["-gen-op-decls"], - "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.h.inc", - ), - ( - ["-gen-op-defs"], - "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td", - deps = [":dialect_td"], -) - -# gentbl_cc_library( -# name = "ops_inc_gen", -# includes = ["cute_ir/include"], -# tbl_outs = [ -# ( -# ["-gen-op-decls"], -# "cute_ir/Dialect/Cute/IR/CuteOps.h.inc", -# ), -# ( -# ["-gen-op-defs"], -# "cute_ir/Dialect/Cute/IR/CuteOps.cpp.inc", -# ), -# ], -# tblgen = "@llvm-project//mlir:mlir-tblgen", -# td_file = "CuteOps.td", -# deps = [":tabledefs"], -# ) - -gentbl_cc_library( - name = "types_decl_inc_gen", - includes = ["cute_ir/include"], - tbl_outs = [ - ( - ["-gen-typedef-decls", "-typedefs-dialect=cute"], - "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypes.h.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypes.td", - deps = [":dialect_td"], -) - -gentbl_cc_library( - name = "types_defs_inc_gen", - includes = ["cute_ir/include"], - tbl_outs = [ - ( - ["-gen-typedef-defs", "-typedefs-dialect=cute"], - "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypes.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypes.td", - deps = [":dialect_td"], + td_file = CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.td", + deps = [":cute_ir_dialect_td"], ) +# The header alone: the attribute, type and operation class declarations it +# includes are generated by the building tree, which adds the compiler's own. cc_library( - name = "dialect_headers", - hdrs = [ - "cute_ir/include/cute_ir/Dialect/Cute/IR/CuteDialect.h", - ], - includes = ["cute_ir/include"], - textual_hdrs = ["cute_ir/lib/Dialect/IR/CuteTypes.cpp"], + name = "cute_ir_dialect_header", + hdrs = [CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteDialect.h"], + includes = [CUTE_IR_INCLUDE], deps = [ - ":attrs_inc_gen", - ":cutegen", - ":dialect_inc_gen", - ":ops_inc_gen", - ":type_interfaces_inc_gen", - ":types_decl_inc_gen", - ], -) - -cc_library( - name = "dialect_support", - srcs = [ - "cute_ir/lib/Dialect/IR/CuteAttributes.cpp", - "cute_ir/lib/Dialect/IR/CuteOps.cpp", - ], - includes = ["cute_ir/include"], - deps = [":dialect_headers"], -) - -cc_library( - name = "dialect", - srcs = [ - "cute_ir/lib/Dialect/Cute/IR/CuteDialect.cpp", - "cute_ir/lib/Dialect/Cute/IR/CuteTypes.cpp", - ], - deps = [ - ":dialect_headers", - ":dialect_support", - ":types_defs_inc_gen", + ":cute_ir_dialect_inc_gen", + ":cute_ir_type_interfaces_inc_gen", ], ) diff --git a/third_party/cute_ir/cutegen_dynamic_divisibility.patch b/third_party/cute_ir/cutegen_dynamic_divisibility.patch new file mode 100644 index 00000000..4f479d60 --- /dev/null +++ b/third_party/cute_ir/cutegen_dynamic_divisibility.patch @@ -0,0 +1,243 @@ +--- a/cutlass_compiler/cutegen/include/cutegen/dynamic.hpp ++++ b/cutlass_compiler/cutegen/include/cutegen/dynamic.hpp +@@ -73,13 +73,20 @@ + * @brief Width (in bits) of the associated dynamic integer + */ + int32_t width = default_dynamic_int_bitwidth(); ++ /** ++ * @brief Known positive divisor of the associated dynamic integer; 1 when ++ * nothing is known. Spelled `div=N` in the dynamic leaf's properties. ++ */ ++ static_int_t divisibility = 1; + /// Constructor + constexpr dynamic_int_properties_t() = default; + explicit dynamic_int_properties_t(int32_t w); ++ dynamic_int_properties_t(int32_t w, static_int_t div); + /// Equality operators + bool operator==(const dynamic_int_properties_t& rhs) const; + bool operator!=(const dynamic_int_properties_t& rhs) const; + dynamic_int_properties_t with_width(int32_t w) const; ++ dynamic_int_properties_t with_divisibility(static_int_t div) const; + bool is_default() const; + bool are_compatible(static_int_t v) const; + static_int_t minimum() const; +@@ -475,11 +482,25 @@ + static std::string to_str(const dynamic_t& d) + { + std::string s("?"); +- if(d.get_width() != default_dynamic_int_bitwidth()) ++ const bool has_width = d.get_width() != default_dynamic_int_bitwidth(); ++ const bool has_div = d.get_properties().divisibility != 1; ++ if(has_width || has_div) + { + s.append("{"); +- s.append("i"); +- s.append(std::to_string(d.get_width())); ++ if(has_width) ++ { ++ s.append("i"); ++ s.append(std::to_string(d.get_width())); ++ } ++ if(has_div) ++ { ++ if(has_width) ++ { ++ s.append(" "); ++ } ++ s.append("div="); ++ s.append(std::to_string(d.get_properties().divisibility)); ++ } + s.append("}"); + } + return s; +@@ -553,6 +574,22 @@ + return; + } + } ++ else if(kw.value() == "div") ++ { ++ auto eq = parse_type(); ++ if(!eq.has_value()) ++ { ++ // Error: missing '=' after 'div' ++ return; ++ } ++ auto div = parse_type(); ++ if(!div.has_value() || div.value() <= 0) ++ { ++ // Error: no positive value for the divisor ++ return; ++ } ++ prop.divisibility = div.value(); ++ } + else + { + // Error: an unexpected keyword was found +@@ -572,6 +609,7 @@ + static void encode_value(const dynamic_t& d, TBuffer& buffer) + { + encode(buffer, d.get_width()); ++ encode(buffer, d.get_properties().divisibility); + } + }; + +@@ -586,7 +624,10 @@ + if(!w.has_value() || + (widths.end() == std::find(widths.begin(), widths.end(), w.value()))) + return std::nullopt; +- return dynamic_t(w.value()); ++ const auto div = decode(it, end); ++ if(!div.has_value() || div.value() <= 0) ++ return std::nullopt; ++ return dynamic_t(dynamic_int_properties_t(w.value(), div.value())); + } + }; + +@@ -816,45 +857,72 @@ + struct property_policy_c_t + { + using int_t = static_int_t; ++ // The divisibility a result keeps: a sum or difference is a multiple of ++ // the gcd of its operands' divisors, a product of their product, and a ++ // static operand counts as its own absolute value. ++ static static_int_t gcd_div(static_int_t a, static_int_t b) ++ { ++ a = a < 0 ? -a : a; ++ b = b < 0 ? -b : b; ++ while(b != 0) ++ { ++ static_int_t t = a % b; ++ a = b; ++ b = t; ++ } ++ return a == 0 ? 1 : a; ++ } ++ static dynamic_int_properties_t with_div(int32_t width, static_int_t div) ++ { ++ return dynamic_int_properties_t(width, div <= 0 ? 1 : div); ++ } + //-------------------------------------------------------------------------- + // resolve_add() + static arith_resolve_t resolve_add(const dynamic_t& lhs, const dynamic_t& rhs) + { + auto res_width = std::max(lhs.get_width(), rhs.get_width()); +- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); ++ auto div = gcd_div(lhs.get_properties().divisibility, ++ rhs.get_properties().divisibility); ++ return arith_op_desc_t(with_div(res_width, div), res_width); + } + template , bool> = true> + static arith_resolve_t resolve_add(const dynamic_t& lhs, TInt rhs) + { + auto res_width = std::max(lhs.get_width(), in_range(rhs) ? 32 : 64); +- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); ++ auto div = gcd_div(lhs.get_properties().divisibility, static_int_t(rhs)); ++ return arith_op_desc_t(with_div(res_width, div), res_width); + } + static arith_resolve_t resolve_sub(const dynamic_t& lhs, const dynamic_t& rhs) + { + auto res_width = std::max(lhs.get_width(), rhs.get_width()); +- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); ++ auto div = gcd_div(lhs.get_properties().divisibility, ++ rhs.get_properties().divisibility); ++ return arith_op_desc_t(with_div(res_width, div), res_width); + } + template , bool> = true> + static arith_resolve_t resolve_sub(const dynamic_t& lhs, TInt rhs) + { + auto res_width = std::max(lhs.get_width(), in_range(rhs) ? 32 : 64); +- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); ++ auto div = gcd_div(lhs.get_properties().divisibility, static_int_t(rhs)); ++ return arith_op_desc_t(with_div(res_width, div), res_width); + } + template , bool> = true> + static arith_resolve_t resolve_sub(TInt lhs, const dynamic_t& rhs) + { + auto res_width = std::max(rhs.get_width(), in_range(lhs) ? 32 : 64); +- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); ++ auto div = gcd_div(rhs.get_properties().divisibility, static_int_t(lhs)); ++ return arith_op_desc_t(with_div(res_width, div), res_width); + } + //-------------------------------------------------------------------------- + // resolve_mul() + static arith_resolve_t resolve_mul(const dynamic_t& lhs, const dynamic_t& rhs) + { + auto res_width = std::max(lhs.get_width(), rhs.get_width()); +- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); ++ auto div = lhs.get_properties().divisibility * rhs.get_properties().divisibility; ++ return arith_op_desc_t(with_div(res_width, div), res_width); + } + template , bool> = true> +@@ -870,7 +938,9 @@ + return src_operand_index_t(0); + } + auto res_width = std::max(lhs.get_width(), in_range(rhs) ? 32 : 64); +- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); ++ auto div = lhs.get_properties().divisibility * ++ static_int_t(rhs < 0 ? -rhs : rhs); ++ return arith_op_desc_t(with_div(res_width, div), res_width); + } + //-------------------------------------------------------------------------- + // resolve_mod() +@@ -939,7 +1009,9 @@ + return src_operand_index_t(0); + } + auto res_width = std::max(lhs.get_width(), in_range(rhs) ? 32 : 64); +- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); ++ auto d = lhs.get_properties().divisibility; ++ auto r = static_int_t(rhs < 0 ? -rhs : rhs); ++ return arith_op_desc_t(with_div(res_width, d % r == 0 ? d / r : 1), res_width); + } + template , bool> = true> +@@ -1259,12 +1331,18 @@ + + inline dynamic_int_properties_t::dynamic_int_properties_t(int32_t w) : + width(w) ++{ ++} ++ ++inline dynamic_int_properties_t::dynamic_int_properties_t(int32_t w, static_int_t div) : ++ width(w), ++ divisibility(div) + { + } + + inline bool dynamic_int_properties_t::operator==(const dynamic_int_properties_t& rhs) const + { +- return (width == rhs.width); ++ return (width == rhs.width) && (divisibility == rhs.divisibility); + } + + inline bool dynamic_int_properties_t::operator!=(const dynamic_int_properties_t& rhs) const +@@ -1284,9 +1362,16 @@ + return r; + } + ++inline dynamic_int_properties_t dynamic_int_properties_t::with_divisibility(static_int_t div) const ++{ ++ dynamic_int_properties_t r = *this; ++ r.divisibility = div; ++ return r; ++} ++ + inline bool dynamic_int_properties_t::are_compatible(static_int_t v) const + { +- return in_range_of_width(v, width); ++ return in_range_of_width(v, width) && (v % divisibility == 0); + } + + inline static_int_t dynamic_int_properties_t::minimum() const +@@ -1313,7 +1398,7 @@ + + inline bool dynamic_int_properties_t::is_constrained() const + { +- return false; ++ return divisibility != 1; + } + + inline dynamic_t::dynamic_t(int32_t width_) : diff --git a/third_party/cute_ir/repo.bzl b/third_party/cute_ir/repo.bzl index 89de3670..8ccba581 100644 --- a/third_party/cute_ir/repo.bzl +++ b/third_party/cute_ir/repo.bzl @@ -1,10 +1,16 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") def repo(): + # The cutlass_compiler/ tree of the CUTLASS v4.8.0 release: the open-source + # part of the CuTe dialect that mlir/dialects/cute_ir builds on. git_repository( name = "cute_ir", remote = "https://github.com/NVIDIA/cutlass.git", - commit = "147295a3d4b75f3aeff247c25b8927cea9a7006a", + commit = "098de2a652cf8f00fd70b2df54051c7eccbb855a", build_file = Label("//third_party/cute_ir:cute_ir.bazel"), strip_prefix = "cutlass_compiler", + # The DSL spells a dynamic leaf it knows to be a multiple of N as + # `?{div=N}`; the release's cutegen reads only the width. + patches = [Label("//third_party/cute_ir:cutegen_dynamic_divisibility.patch")], + patch_args = ["-p2"], ) diff --git a/zml/nn.zig b/zml/nn.zig index 7c23f7ce..9d46a5ba 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -66,7 +66,7 @@ pub const Linear = struct { const weight_global_scale: ?Tensor = if (q.global_scale) |s| s.asMultiplier() else null; const weight = if (isPackedFp4(q.scheme, self.weight.dtype())) unpackFp4(self.weight, self.tag, self.tag) else self.weight; const scales = if (q.scheme.isMx() and q.scales.dtype() == .u8) q.scales.bitCast(.f8e8m0) else q.scales; - const acc = scaledDot(lhs, weight, lhs_scale, scales, output_dtype, self.tag); + const acc = scaledDot(lhs, weight, lhs_scale, scales, output_dtype, self.tag, .{ .rhs_scale_swizzled = q.swizzled_scales }); const y = applyGlobalScale(acc, input_global_scale, weight_global_scale).convert(output_dtype); return if (self.bias) |bias| y.add(bias.convert(y.dtype()).broad(y.shape())) else y; } @@ -129,6 +129,7 @@ pub fn scaledDot( rhs_scale: Tensor, out_dtype: DataType, args: anytype, + opts: ScaledDotOpts, ) Tensor { // A narrow output would be an unscaled cast of the F32 accumulator stdx.debug.assert( @@ -201,11 +202,25 @@ pub fn scaledDot( .intArray(mlir_ctx, i64, rhs_batching_axes.constSlice()), }), }); + const attributes = [_]mlir.NamedAttribute{ + .named(mlir_ctx, "dimension_numbers", dnums), + .named(mlir_ctx, "rhs_scale_swizzled", .boolean(mlir_ctx, true)), + }; + if (opts.rhs_scale_swizzled) { + stdx.debug.assert(rhs.rank() == 2 and rhs_scale.rank() == 2 and rhs_contracting_axes.get(0) == 1, "scaledDot: a swizzled rhs scale needs a [n, k] rhs, got {f}", .{rhs}); + } return ops.composite("xla.scaled_dot", &.{ lhs, rhs, lhs_scale_operand, rhs_scale_operand }, &.{res_shape}, scaledDotReference, res_shape, .{ - .composite_attributes = &.{.named(mlir_ctx, "dimension_numbers", dnums)}, + .composite_attributes = if (opts.rhs_scale_swizzled) &attributes else attributes[0..1], })[0]; } +pub const ScaledDotOpts = struct { + /// The rhs scale's `[n, k / block]` grid is stored in the 128x4 blocked layout of Blackwell's block-scaled mma, + /// each device's slice on its own. + // TODO: fine for now, but we should find a better way to express how a scale is laid out. + rhs_scale_swizzled: bool = false, +}; + test "block128 scaled dot layouts" { const allocator = std.testing.allocator; const io = std.testing.io; @@ -235,9 +250,9 @@ test "block128 scaled dot layouts" { .linear = linear.forward(x, .bf16), .linear_expected = dequantize(input.values, input.scales).dot(dequantize(weight, scales), .k), .actual = if (prequantized) - scaledDot(input.values, weight, input.scales.convert(scales.dtype()), scales, .bf16, .k) + scaledDot(input.values, weight, input.scales.convert(scales.dtype()), scales, .bf16, .k, .{}) else - scaledDot(x, weight, null, scales, .bf16, .k), + scaledDot(x, weight, null, scales, .bf16, .k, .{}), .expected = (if (prequantized) dequantize(input.values, input.scales.convert(scales.dtype())) else x) .dot(dequantize(weight, scales), .k), }; diff --git a/zml/quantization.zig b/zml/quantization.zig index 5357610d..dc82f558 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -17,6 +17,9 @@ pub const Quantization = struct { scales: Tensor, global_scale: ?GlobalScale = null, input_scale: ?GlobalScale = null, + /// HACK: `scales` is stored in the 128x4 blocked layout of Blackwell's block-scaled mma. + // Something cleaner will need to be done to differentiate different kinds of swizzling + swizzled_scales: bool = false, /// A per-tensor scale and the polarity its producer wrote it in pub const GlobalScale = struct { From 58cda1283ba793f6af7e56329f99d953eca3ef6d Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Wed, 23 Sep 2026 23:17:48 +0200 Subject: [PATCH 716/881] buffer: ensure proper deinitialization of host buffer events (#854) --- zml/buffer.zig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zml/buffer.zig b/zml/buffer.zig index a687943e..5b86cae3 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -294,6 +294,7 @@ pub const Buffer = struct { const maybe_event = try self._shards.get(shard_index).toHostBuffer(self._platform.pjrt_api, destination); if (maybe_event) |event| { + defer event.deinit(self._platform.pjrt_api); try event.await(self._platform.pjrt_api, io); } } @@ -314,6 +315,7 @@ pub const Buffer = struct { const sub_slice = placement.shardSlice(device.coords, slice); const maybe_event = try self._shards.get(shard_index).toHostBuffer(self._platform.pjrt_api, shard_slice.data()); if (maybe_event) |event| { + defer event.deinit(self._platform.pjrt_api); try event.await(self._platform.pjrt_api, io); } From 50e68877a640b1728e0b5d2b51c93cf89d6dfca3 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 24 Sep 2026 10:31:19 +0200 Subject: [PATCH 717/881] ci: use bazel --experimental_disk_cache_gc_max_size (#824) --- .github/workflows/ci.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 43964175..c996de04 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -84,6 +84,13 @@ jobs: common --test_output=streamed common --color=yes common --show_timestamps + build --experimental_disk_cache_gc_max_size=100G + build --experimental_disk_cache_gc_max_age=14d + build --experimental_disk_cache_gc_idle_delay=1m + common --experimental_action_cache_gc_max_age=14d + common --experimental_action_cache_gc_idle_delay=1m + common --repo_contents_cache_gc_max_age=14d + common --repo_contents_cache_gc_idle_delay=1m ${{ !matrix.self_hosted && secrets.BUILDBUDDY_CI_API_KEY != '' && 'common --config=remote' || '' }} ${{ !matrix.self_hosted && secrets.BUILDBUDDY_CI_API_KEY != '' && format('common:remote --remote_header=x-buildbuddy-api-key={0}', secrets.BUILDBUDDY_CI_API_KEY) || '' }} ${{ !matrix.self_hosted && secrets.BUILDBUDDY_CI_API_KEY != '' && 'common:remote --remote_download_outputs=minimal' || '' }} From e2da790b1c800d7b5fb03463efc446f249082c9f Mon Sep 17 00:00:00 2001 From: Louis Lechevalier Date: Thu, 24 Sep 2026 13:12:08 +0200 Subject: [PATCH 718/881] zml/moe: CuTe MoE mxfp4 backend (#853) Introduces the CuTe mxfp4 moe backend. The kernels use the cute dsl and are optimized (and so at the moment constraint to its shapes). Introduces the cute launch host func and so the associated cuda dialect in zml. --- kernels/cute/README.md | 39 +- kernels/cute/builder.zig | 2331 ++++++++++++++++++++- kernels/cute/dtype.zig | 9 +- mlir/dialects/BUILD.bazel | 1 + mlir/dialects/cuda.zig | 141 ++ mlir/dialects/dialects.zig | 1 + third_party/cute_ir/repo.bzl | 12 +- third_party/cute_ir/slice_tensor.patch | 45 + zml/BUILD.bazel | 4 + zml/kernel.zig | 87 + zml/moe/cute_kernels/moe.zig | 172 ++ zml/moe/cute_kernels/mxfp4.zig | 263 +++ zml/moe/cute_kernels/persistent_mxfp4.zig | 2018 ++++++++++++++++++ zml/moe/cute_mxfp4.zig | 123 ++ zml/moe/moe.zig | 15 +- zml/moe/triton_kernels/mxfp4.zig | 107 +- zml/moe/triton_mxfp4.zig | 14 +- zml/ops.zig | 10 +- 18 files changed, 5323 insertions(+), 69 deletions(-) create mode 100644 mlir/dialects/cuda.zig create mode 100644 third_party/cute_ir/slice_tensor.patch create mode 100644 zml/moe/cute_kernels/moe.zig create mode 100644 zml/moe/cute_kernels/mxfp4.zig create mode 100644 zml/moe/cute_kernels/persistent_mxfp4.zig create mode 100644 zml/moe/cute_mxfp4.zig diff --git a/kernels/cute/README.md b/kernels/cute/README.md index 367eaaf2..2c968a96 100644 --- a/kernels/cute/README.md +++ b/kernels/cute/README.md @@ -71,8 +71,18 @@ const c = NaiveElementwiseAdd.call(.{ .gA = a, .gB = b }, .{ .gC = a.shape() }, | `block_idx()`, `block_dim()`, `grid_dim()` | `blockIdx()`, `blockDim()`, `gridDim()` | | `cute.arch.alloc_smem(T, n, align)` | `b.allocSmem(.f32, n, align)` | | `cute.arch.sync_threads()` | `b.syncThreads()` | +| `cute.arch.shuffle_sync_bfly(v, lane)` | `b.shuffleXor(v, lane)` | +| `cute.arch.griddepcontrol_launch_dependents()` | `b.launchDependents()` | | `x.to(Float32)` | `x.to(.f32)` | | `if cond:` / `for i in range(...)` | `b.openIf(cond)` / `b.openFor(lb, ub, step, .{})` | +| `cpasync.make_tiled_tma_atom(...)` | `b.makeTiledTmaLoadAtom(...)` / `...Typed(...)` | +| `cpasync.tma_partition(...)` | `b.tmaPartition(...)` | +| `cute.copy(atom, src, dst)` | `b.copy(atom, src, dst, pred)` | +| `cute.make_tiled_mma(atom)` | `b.makeTiledMma(atom, result_type)` | +| `tiled_mma.make_fragment_A/B/C(...)` | `b.mmaMakeFragment(...)` | +| `cute.gemm(...)` | `b.gemm(...)` | +| `cute.arch.alloc_tmem(...)` | `b.allocTmem(...)` | +| `cute.arch.retrieve_tmem_ptr(...)` | `b.retrieveTmemPtr(...)` | * **Layouts are static.** `makeLayout` takes Zig slices and emits one `cute.static`; the tensor keeps its shape and stride in Zig, so `dim`, @@ -89,8 +99,33 @@ const c = NaiveElementwiseAdd.call(.{ .gA = a, .gB = b }, .{ .gC = a.shape() }, * **The module is just the kernel.** `finish` prints `module { func.func @name(...) {...} }`; the CuTe compiler makes every public function a kernel entry, so there is no `gpu.module` or host launch function to write. -* **Not covered yet:** dynamic shapes, `local_tile`/`local_partition`, tiled - copies and fragments (`TensorSSA`), TMA and MMA atoms. +* **Explicit result types:** Zig covers local tiling, TMA partitioning, tiled + copies, tensor-memory fragments, and SM100 block-scaled MMA. CuTe derives + their layouts in the compiler, so callers spell out the resulting CuTe type + with `View` instead of relying on Python's dynamic type objects. +* **Not covered yet:** dynamic tensor shapes and automatic CuTe layout algebra. +* **Typed variants.** Operations whose result type CuTe derives exist in two + forms: one takes the result type as MLIR text (`localTile`, `slice`, + `tmaPartition`, ...), the `...Typed` one takes an `*mlir.Type` assembled + with `layoutSpec`, `memrefType` and `coordTensorType`. +* **Architecture sections.** `Builder` groups TMA (`cute.nvgpu.cpasync`, + SM90+) and tcgen05 / tensor memory (`cute.nvgpu.tcgen05`, SM100) in their + own sections; mbarriers, fences and bulk-copy groups sit with `cute.arch`. + Each doc comment names its minimum architecture when it is above SM80. + +## Programs with a host launch + +Kernels that need launch-time CuTe objects (TMA descriptors, tiled MMA atoms, +scheduler parameters) use `zml.kernel.cute.Program` instead of `Kernel`. Its +`run` callback emits one `gpu.module` with the `cuda.kernel`s +(`b.beginFunction(name, .cuda_kernel)`), then a public host `func.func` +(`b.beginFunction(name, .host)`) that builds those objects from the XLA +buffers and calls `b.launchEx`. The custom call carries no grid or block: +XLA runs the host function with the real buffers and launches (or records in +a command buffer) whatever it launched. `CallOpts.scalars` are passed to the +host function after the buffers. `zml/moe/cute_kernels/persistent_mxfp4.zig` +is the reference: a warp-specialized SM100 block-scaled GEMM whose host +function builds four TMA tensor maps and a PDL launch. ## The custom call diff --git a/kernels/cute/builder.zig b/kernels/cute/builder.zig index d59fa439..69900224 100644 --- a/kernels/cute/builder.zig +++ b/kernels/cute/builder.zig @@ -8,8 +8,11 @@ const mlir = @import("mlir"); const stdx = @import("stdx"); const arith = dialects.arith; +const cuda = dialects.cuda; const func = dialects.func; +const gpu = dialects.gpu; const math = dialects.math; +const vector = dialects.vector; const dtypes = @import("dtype.zig"); pub const DType = dtypes.DType; @@ -21,11 +24,13 @@ test { std.testing.refAllDecls(Builder); std.testing.refAllDecls(Value); std.testing.refAllDecls(Tensor); + std.testing.refAllDecls(View); + std.testing.refAllDecls(Atom); } /// `nvvm` is not linked into ZML: its ops are emitted unregistered, in /// generic form; the CuTe compiler has the dialect. -pub const dialects_needed = [_][]const u8{ "func", "cute", "cute_nvgpu", "arith", "scf", "math", "cf" }; +pub const dialects_needed = [_][]const u8{ "func", "gpu", "cute", "cute_nvgpu", "arith", "scf", "math", "cf", "vector" }; pub const FinishError = error{InvalidMlir} || std.mem.Allocator.Error || std.Io.Writer.Error; @@ -34,6 +39,43 @@ pub const Dims = stdx.BoundedArray(i64, MAX_RANK); pub const MemorySpace = cute.MemorySpace; +/// A CuTe algebra expression assembled from Zig values. Kernel authors use +/// `Builder.expr`, `Builder.basis`, and `Builder.layoutSpec`; only this DSL +/// layer serializes the expression required by CuTe's C API. +pub const AlgebraExpr = struct { + text: []const u8, +}; + +pub const AlgebraToken = enum { + /// CuTe's slice placeholder (`_`). + all, + /// A dynamic algebra leaf (`?`). + dynamic, +}; + +pub const ScaledBasis = struct { + numerator: i64, + denominator: i64 = 1, + modes: [MAX_RANK]u8 = @splat(0), + mode_count: u8, +}; + +/// A dynamic CuTe integer whose runtime value is known to be divisible by a +/// static amount. CuTe carries this fact in dependent coordinate-tensor types +/// after tiling and slicing, for example `?{div=128}`. +pub const ConstrainedDynamic = struct { + divisible_by: u64, +}; + +pub const LayoutSpec = struct { + shape: AlgebraExpr, + stride: AlgebraExpr, + + pub fn payload(self: LayoutSpec, b: *Builder) []const u8 { + return std.fmt.allocPrint(b.arena.allocator(), "{s}:{s}", .{ self.shape.text, self.stride.text }) catch @panic("OOM"); + } +}; + /// A scalar SSA value (`Int32`, `Float32`, ... in the Python DSL). pub const Value = struct { inner: *const mlir.Value, @@ -158,6 +200,18 @@ pub const Value = struct { return k.emit(arith.shrsi(k.ctx, l.inner, r.inner, k.loc())); } + pub fn shrLogical(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(arith.shrui(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn bitCast(self: Value, dt: DType) Value { + const k = self.kern(); + if (dtypeBitwidth(self.dtype()) != dtypeBitwidth(dt)) @panic("Value.bitCast requires equal bit widths"); + return k.emit(arith.bitcast(k.ctx, self.inner, dt.toMlir(k.ctx), k.loc())); + } + pub fn lt(self: Value, rhs: anytype) Value { return self.kern().cmp(self, rhs, .slt, .olt); } @@ -192,6 +246,22 @@ pub const Value = struct { return k.emit(math.log(k.ctx, self.inner, k.loc())); } + pub fn log2(self: Value) Value { + const k = self.kern(); + return k.emit(math.log2(k.ctx, self.inner, k.loc())); + } + + pub fn exp2(self: Value) Value { + const k = self.kern(); + return k.emit(math.exp2(k.ctx, self.inner, k.loc())); + } + + pub fn ceil(self: Value) Value { + const k = self.kern(); + if (!self.isFloat()) @panic("Value.ceil is float-only"); + return k.emit(math.ceil(k.ctx, self.inner, k.loc())); + } + pub fn sqrt(self: Value) Value { const k = self.kern(); return k.emit(math.sqrt(k.ctx, self.inner, k.loc())); @@ -285,16 +355,196 @@ pub const Tensor = struct { } /// `t[coord] = value`; the value is converted to the element type. - pub fn set(self: Tensor, coord: anytype, value: anytype) void { + pub fn set(self: Tensor, coord: anytype, value_: anytype) void { const k = self.kernel; const c = k.makeCoord(coord); - const v = k.liftAs(value, self.dtype).to(self.dtype); + const v = k.liftAs(value_, self.dtype).to(self.dtype); _ = cute.memref_store(k.ctx, self.inner, c, v.inner, k.loc()).appendTo(k.currentBlock()); } + + /// Erase Zig's tracked layout while retaining the MLIR memref. CuTe + /// layout algebra derives the layouts returned by tiling/partitioning; + /// those values are represented by `View` because Zig cannot recompute + /// their dependent types. + pub fn view(self: Tensor) View { + return .{ .inner = self.inner, .kernel = self.kernel }; + } + + pub fn value(self: Tensor) Value { + return .{ .inner = self.inner, .kernel = self.kernel }; + } }; +/// A CuTe tensor or pointer with a layout derived by the CuTe compiler. +pub const View = struct { + inner: *const mlir.Value, + kernel: *Builder, + + pub fn type_(self: View) *const mlir.Type { + return self.inner.type_(); + } + + pub fn value(self: View) Value { + return .{ .inner = self.inner, .kernel = self.kernel }; + } + + pub fn get(self: View, coord: anytype, result_dtype: DType) Value { + const c = self.kernel.makeCoord(coord); + return self.kernel.emit(cute.memref_load(self.kernel.ctx, self.inner, c, result_dtype.toMlir(self.kernel.ctx), self.kernel.loc())); + } + + pub fn set(self: View, coord: anytype, value_: Value) void { + self.kernel.emitVoid(cute.memref_store(self.kernel.ctx, self.inner, self.kernel.makeCoord(coord), value_.inner, self.kernel.loc())); + } +}; + +/// A copy/MMA atom or a tiled atom. Atom fields are immutable SSA values; +/// `setField` returns the atom carrying the updated runtime field. +pub const Atom = struct { + inner: *const mlir.Value, + kernel: *Builder, + + pub fn type_(self: Atom) *const mlir.Type { + return self.inner.type_(); + } + + pub fn value(self: Atom) Value { + return .{ .inner = self.inner, .kernel = self.kernel }; + } + + pub fn setField(self: Atom, field: []const u8, value_: anytype) Atom { + const T = @TypeOf(value_); + const new_value = if (T == Value or T == Tensor or T == View or T == Atom or T == Layout) + self.kernel.asValue(value_) + else + self.kernel.lift(value_); + const result = self.kernel.emit(cute.nvgpu.atom_set_value( + self.kernel.ctx, + self.inner, + new_value.inner, + self.type_(), + .string(self.kernel.ctx, field), + self.kernel.loc(), + )); + return .{ .inner = result.inner, .kernel = self.kernel }; + } +}; + +/// Attribute payload accepted by target-specific CuTe operations. +pub const Attr = union(enum) { + str: []const u8, + text: []const u8, + raw: *const mlir.Attribute, + + pub fn get(self: Attr, ctx: *mlir.Context) *const mlir.Attribute { + return switch (self) { + .str => |s| .string(ctx, s), + .text => |s| mlir.Attribute.parse(ctx, s) catch std.debug.panic("invalid MLIR attribute: {s}", .{s}), + .raw => |a| a, + }; + } +}; + +pub const TmaDescriptor = struct { + atom: Atom, + tensor: View, +}; + +pub const TmaPartition = struct { + smem: View, + targets: []const View, +}; + +/// Element encoding of a TMA transfer when it differs from the tensor dtype. +pub const TmaFormat = enum { + default, + u4_unpack_u8, + u16, +}; + +/// Typed description of a tiled TMA atom: the element type, the bits moved +/// per copy, the basis of one box, and the coordinate layout of the tensor. +pub const TmaConfig = struct { + dtype: DType, + copy_bits: u32, + global_basis: LayoutSpec, + coordinate_layout: LayoutSpec, + coordinate_rank: u8, + format: TmaFormat = .default, + num_multicast: u32 = 1, +}; + +pub const BlockScaledMma = struct { + atom: Atom, + tiled: Atom, +}; + +/// Shape of the SM100 block-scaled MXFP4 x MXFP8 MMA instruction. +pub const BlockScaledMmaConfig = struct { + m: u16 = 128, + n: u16, + k: u16 = 32, + num_cta: u8 = 1, + vec_size: u8 = 32, +}; + +fn dtypeName(dtype: DType) []const u8 { + return switch (dtype) { + .i1 => "i1", + .i8 => "i8", + .i16 => "i16", + .i32 => "i32", + .i64 => "i64", + .f16 => "f16", + .bf16 => "bf16", + .f32 => "f32", + .f64 => "f64", + .f4e2m1fn => "f4E2M1FN", + .f8e4m3fn => "f8E4M3FN", + .f8e5m2 => "f8E5M2", + .f8e8m0fnu => "f8E8M0FNU", + }; +} + +fn formatName(format: TmaFormat, dtype: DType) []const u8 { + return switch (format) { + .u4_unpack_u8 => "U4_UNPACK_U8", + .u16 => "U16", + .default => switch (dtype) { + .f8e4m3fn => "U8", + .f32 => "F32_RN", + else => @panic("TMA format must be specified for this dtype"), + }, + }; +} + +fn tmaFormatAttribute(b: *Builder, format: TmaFormat) ?*const mlir.Attribute { + return switch (format) { + .default => null, + .u4_unpack_u8 => b.parseAttribute("#cute_nvgpu.tma_data_format"), + .u16 => b.parseAttribute("#cute_nvgpu.tma_data_format"), + }; +} + +fn blockScaledPayload(b: *Builder, cfg: BlockScaledMmaConfig) []const u8 { + return std.fmt.allocPrint( + b.arena.allocator(), + "<{d}x{d}x{d}, num_cta = {d}, ab_major = (k, k), elem_type = (f4E2M1FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, frag_kind = ss, vec_size = {d}>", + .{ cfg.m, cfg.n, cfg.k, cfg.num_cta, cfg.vec_size }, + ) catch @panic("OOM"); +} + pub const Index3 = struct { x: Value, y: Value, z: Value }; +pub const LaunchConfig = struct { + inner: *const mlir.Value, + kernel: *Builder, + + pub fn value(self: LaunchConfig) Value { + return .{ .inner = self.inner, .kernel = self.kernel }; + } +}; + pub const ArgSpec = struct { name: []const u8, kind: Kind, @@ -305,6 +555,20 @@ pub const ArgSpec = struct { pub const Kind = union(enum) { ptr: DType, tensor: TensorSpec, + /// A type constructed through a dialect/type API. + mlir_type: MlirTypeSpec, + /// A by-value kernel ABI argument such as a CUDA tensor-map atom. + raw: RawSpec, + }; + + pub const MlirTypeSpec = struct { + type_: *const mlir.Type, + grid_constant: bool = false, + }; + + pub const RawSpec = struct { + type_text: []const u8, + grid_constant: bool = false, }; pub const TensorSpec = struct { @@ -331,12 +595,20 @@ pub fn ForScope(comptime N: usize) type { return dsl.ForScope(Builder, Value, N); } +pub fn WhileScope(comptime N: usize, comptime M: usize) type { + return dsl.WhileScope(Builder, Value, N, M); +} + pub const Builder = struct { + pub const FunctionKind = enum { func_only_kernel, cuda_kernel, host }; + allocator: std.mem.Allocator, arena: std.heap.ArenaAllocator, ctx: *mlir.Context, module: *mlir.Module, + gpu_body: ?*mlir.Block = null, name: []const u8, + function_kind: FunctionKind = .func_only_kernel, func_op: ?*mlir.Operation = null, entry_block: ?*mlir.Block = null, block_stack: std.ArrayList(*mlir.Block) = .empty, @@ -357,6 +629,29 @@ pub const Builder = struct { }; } + /// Open a complete CuTe program. Unlike `open`, which emits the compact + /// func.func-only ABI consumed directly by XLA, a program owns a + /// `gpu.module` for one or more `cuda.kernel` operations and can append a + /// host `func.func` that constructs CuTe launch objects and calls + /// `cuda.launch_ex`. + pub fn openProgram(allocator: std.mem.Allocator, ctx: *mlir.Context, name: []const u8) !Builder { + var self = try open(allocator, ctx, name); + self.module.operation().setAttributeByName("gpu.container_module", .unit(ctx)); + const body = mlir.Block.init(&.{}, &.{}); + const gpu_module = gpu.module(ctx, "kernels", body, self.loc()); + _ = gpu_module.appendTo(self.module.body()); + self.gpu_body = body; + self.function_kind = .cuda_kernel; + return self; + } + + /// Start another function in a complete program after `endFunction`. + pub fn beginFunction(self: *Builder, name: []const u8, kind: FunctionKind) void { + std.debug.assert(self.entry_block == null and self.func_op == null); + self.name = name; + self.function_kind = kind; + } + pub fn deinit(self: *Builder) void { self.module.deinit(); self.arena.deinit(); @@ -381,6 +676,14 @@ pub const Builder = struct { .stride = if (@hasField(@TypeOf(inner), "stride")) inner.stride else null, .alignment = if (@hasField(@TypeOf(inner), "alignment")) inner.alignment else 16, } }, + .mlir_type => .{ .mlir_type = .{ + .type_ = inner.type_, + .grid_constant = if (@hasField(@TypeOf(inner), "grid_constant")) inner.grid_constant else false, + } }, + .raw => .{ .raw = .{ + .type_text = inner.type_text, + .grid_constant = if (@hasField(@TypeOf(inner), "grid_constant")) inner.grid_constant else false, + } }, } }; } try self.declareArgsLow(arg_specs); @@ -417,24 +720,69 @@ pub const Builder = struct { ty.* = switch (a.kind) { .ptr => |dt| self.ptrTy(dt, .gmem, 16), .tensor => |t| self.ptrTy(t.dtype, .gmem, t.alignment), + .mlir_type => |typed| typed.type_, + .raw => |raw| self.parseType(raw.type_text), } catch return error.InvalidMlir; l.* = self.loc(); } const entry = mlir.Block.init(arg_types, arg_locs); - const func_op = func.func(ctx, .{ - .name = self.name, - .block = entry, - .results = &.{}, - .location = self.loc(), - .visibility = null, - .verify = false, - .extra_attributes = &.{ - .named(ctx, "cute.kernel", .unit(ctx)), - .named(ctx, "gpu.kernel", .unit(ctx)), - }, - }); - _ = func_op.appendTo(self.module.body()); + const empty_dict: *const mlir.Attribute = .dict(ctx, &.{}); + const arg_attrs = try scratch.alloc(*const mlir.Attribute, args.len); + var any_arg_attr = false; + for (arg_attrs, args) |*attr, a| { + attr.* = switch (a.kind) { + .mlir_type => |typed| if (typed.grid_constant) .dict(ctx, &.{ + .named(ctx, "cute_nvgpu.grid_constant", .unit(ctx)), + }) else empty_dict, + .raw => |raw| if (raw.grid_constant) .dict(ctx, &.{ + .named(ctx, "cute_nvgpu.grid_constant", .unit(ctx)), + }) else empty_dict, + else => empty_dict, + }; + if (attr.* != empty_dict) any_arg_attr = true; + } + + const func_op = switch (self.function_kind) { + .func_only_kernel => func.func(ctx, .{ + .name = self.name, + .block = entry, + .args_attributes = if (any_arg_attr) arg_attrs else null, + .results = &.{}, + .location = self.loc(), + .visibility = null, + .verify = false, + .extra_attributes = &.{ + .named(ctx, "cute.kernel", .unit(ctx)), + .named(ctx, "gpu.kernel", .unit(ctx)), + }, + }), + .cuda_kernel => cuda.kernel(ctx, .{ + .name = self.name, + .function_type = .function(ctx, arg_types, &.{}), + .body = entry, + .arg_attrs = .array(ctx, arg_attrs), + .extra_attributes = &.{ + .named(ctx, "cute.kernel", .unit(ctx)), + .named(ctx, "gpu.kernel", .unit(ctx)), + }, + .location = self.loc(), + }), + .host => func.func(ctx, .{ + .name = self.name, + .block = entry, + .args_attributes = if (any_arg_attr) arg_attrs else null, + .results = &.{.int(ctx, .i32)}, + .location = self.loc(), + .visibility = null, + .verify = false, + .extra_attributes = &.{.named(ctx, "llvm.emit_c_interface", .unit(ctx))}, + }), + }; + _ = func_op.appendTo(if (self.function_kind == .cuda_kernel) + self.gpu_body orelse @panic("cuda_kernel requires Builder.openProgram") + else + self.module.body()); self.func_op = func_op; self.entry_block = entry; @@ -447,6 +795,8 @@ pub const Builder = struct { .tensor => |spec| self.makeTensor(self.arg(i), self.makeLayout(spec.shape, .{ .stride = spec.stride orelse self.rowMajor(spec.shape), })), + .mlir_type => null, + .raw => null, }; } } @@ -476,16 +826,255 @@ pub const Builder = struct { return .{ .inner = op.result(0), .kernel = self }; } + /// Append an operation which does not return an SSA value. + pub fn emitVoid(self: *Builder, op: *mlir.Operation) void { + std.debug.assert(op.numResults() == 0); + _ = op.appendTo(self.currentBlock()); + } + + /// Append a multi-result operation and return all of its SSA values. + /// + /// Most of the small CuTe kernels only need one-result operations, but + /// TMA descriptor construction and TMA partitioning return an atom and a + /// coordinate tensor together. Keeping both results from one operation + /// is required: rebuilding either value changes the descriptor identity. + pub fn emitResults(self: *Builder, comptime N: usize, op: *mlir.Operation) [N]Value { + std.debug.assert(op.numResults() == N); + _ = op.appendTo(self.currentBlock()); + var result: [N]Value = undefined; + for (&result, 0..) |*value, i| value.* = .{ .inner = op.result(i), .kernel = self }; + return result; + } + + /// Convert any DSL handle into a scalar wrapper without changing its + /// MLIR type. This keeps the low-level generated bindings usable while + /// callers work with `Tensor`, `View`, and `Atom` handles. + pub fn asValue(self: *Builder, value_: anytype) Value { + return switch (@TypeOf(value_)) { + Value => value_, + Tensor => value_.value(), + View => value_.value(), + Atom => value_.value(), + LaunchConfig => value_.value(), + Layout => .{ .inner = value_.inner, .kernel = self }, + else => @compileError("expected a CuTe DSL handle, got " ++ @typeName(@TypeOf(value_))), + }; + } + + fn innerSlice(self: *Builder, values: anytype) []const *const mlir.Value { + const out = self.arena.allocator().alloc(*const mlir.Value, values.len) catch @panic("OOM"); + switch (@typeInfo(@TypeOf(values))) { + .@"struct" => { + inline for (values, 0..) |value, i| out[i] = self.asValue(value).inner; + }, + else => { + for (values, out) |value, *dst| dst.* = self.asValue(value).inner; + }, + } + return out; + } + pub fn loc(self: *const Builder) *const mlir.Location { return .unknown(self.ctx); } + pub fn setFunctionAttribute(self: *Builder, name: []const u8, attribute: *const mlir.Attribute) void { + const function = self.func_op orelse @panic("setFunctionAttribute called before declareArgs"); + function.setAttributeByName(name, attribute); + } + + // ==================== cuda host launch ==================== + + pub fn kernelSmemSize(self: *Builder, kernel_name: []const u8) Value { + const symbol = std.fmt.allocPrint(self.arena.allocator(), "@kernels::@{s}", .{kernel_name}) catch @panic("OOM"); + return self.emit(cute.kernel_smem_size( + self.ctx, + .int(self.ctx, .i64), + self.parseAttribute(symbol), + self.loc(), + )); + } + + /// Convert the stream placeholder used by a generated host entry into a + /// CUDA stream. The native CuTe host bridge substitutes XLA's current + /// stream when it executes the wrapper. + pub fn cudaStream(self: *Builder) Value { + return self.emit(cuda.cast(self.ctx, cuda.streamType(self.ctx), self.cst(.i64, 0).inner, self.loc())); + } + + pub const LaunchOptions = struct { + grid: [3]Value, + block: [3]Value, + dynamic_smem: Value, + stream: Value, + cluster: ?[3]Value = null, + cooperative: bool = false, + use_pdl: bool = false, + }; + + /// Build the mutable CUDA launch configuration used by `cuda.launch_ex`. + /// The segment attribute is written explicitly because the CUDA dialect is + /// supplied by the runtime CuTe compiler and is intentionally unregistered + /// in ZML's lightweight construction context. + pub fn makeLaunchConfig(self: *Builder, opts: LaunchOptions) LaunchConfig { + const config = self.emit(cuda.launch_cfg_create(self.ctx, .{ + .max_attrs = 17, + .block = .{ opts.block[0].inner, opts.block[1].inner, opts.block[2].inner }, + .dynamic_smem = opts.dynamic_smem.inner, + .grid = .{ opts.grid[0].inner, opts.grid[1].inner, opts.grid[2].inner }, + .stream = opts.stream.inner, + .location = self.loc(), + })); + if (opts.cluster) |cluster| { + self.emitVoid(cuda.launch_cfg_cluster_dim(self.ctx, config.inner, .{ cluster[0].inner, cluster[1].inner, cluster[2].inner }, self.loc())); + } + self.emitVoid(cuda.launch_cfg_cooperative(self.ctx, config.inner, self.cst(.i32, @intFromBool(opts.cooperative)).inner, self.loc())); + self.emitVoid(cuda.launch_cfg_programmatic_stream_serialization_allowed(self.ctx, config.inner, self.cst(.i32, @intFromBool(opts.use_pdl)).inner, self.loc())); + return .{ .inner = config.inner, .kernel = self }; + } + + /// Request a CUDA kernel launch from a generated host function. The host + /// bridge records this request; XLA performs or command-buffer-records the + /// actual launch on its stream. + pub fn launchEx(self: *Builder, kernel_name: []const u8, config: LaunchConfig, arguments: anytype) Value { + const values = self.innerSlice(arguments); + const symbol = std.fmt.allocPrint(self.arena.allocator(), "@kernels::@{s}", .{kernel_name}) catch @panic("OOM"); + return self.emit(cuda.launch_ex(self.ctx, .{ + .config = config.inner, + .inputs = values, + .callee = self.parseAttribute(symbol), + .assume_kernel_attr = self.targetAttribute("#cuda.assume_kernel_attr"), + .location = self.loc(), + })); + } + + pub fn cudaResultStatus(self: *Builder, result: Value) Value { + return self.emit(cuda.cast(self.ctx, .int(self.ctx, .i32), result.inner, self.loc())); + } + + pub fn returnHostStatus(self: *Builder, status: Value) void { + self.emitVoid(func.returns(self.ctx, &.{status.inner}, self.loc())); + } + + /// Finish the current function but retain the surrounding module so a + /// host launch function can be emitted next. + pub fn endFunction(self: *Builder, block: ?[3]i32) void { + const current = self.currentBlock(); + if (current.terminator() == null) { + const terminator = if (self.function_kind == .cuda_kernel) + cuda.return_(self.ctx, &.{}, self.loc()) + else + func.returns(self.ctx, &.{}, self.loc()); + _ = terminator.appendTo(current); + } + if (block) |dims| { + const function = self.func_op orelse @panic("endFunction before declareArgs"); + function.setAttributeByName("nvvm.reqntid", .denseArray(self.ctx, .i32, &dims)); + } + self.func_op = null; + self.entry_block = null; + self.block_stack.clearRetainingCapacity(); + self.args = &.{}; + self.tensors = &.{}; + } + + pub fn finishProgram(self: *Builder) FinishError![:0]const u8 { + std.debug.assert(self.entry_block == null and self.func_op == null); + if (!self.module.operation().verify()) return error.InvalidMlir; + return self.renderModule(true); + } + + const target_attr_marker = "__zml_cute_target_attr__"; + + fn renderModule(self: *Builder, generic: bool) FinishError![:0]const u8 { + var printed: std.Io.Writer.Allocating = .init(self.allocator); + defer printed.deinit(); + // Generic form is stable across the lightweight construction dialect + // and NVIDIA's runtime compiler dialect. In particular, the two + // versions currently use different custom assembly printers for + // `cute.make_composed_layout`. + if (generic) { + try printed.writer.print("{f}", .{self.module.operation().fmt(.{ .print_generic_op_form = true })}); + } else { + try printed.writer.print("{f}", .{self.module.operation()}); + } + + // A target attribute is initially a quoted StringAttr so the local + // verifier can carry it through an AnyAttr field. Replace the whole + // quoted string with its CuTe assembly before handing the module to + // the real compiler, which owns the attribute parser. + var output: std.Io.Writer.Allocating = .init(self.allocator); + defer output.deinit(); + var rest = printed.written(); + while (std.mem.indexOf(u8, rest, target_attr_marker)) |marker| { + if (marker == 0 or rest[marker - 1] != '"') return error.InvalidMlir; + try output.writer.writeAll(rest[0 .. marker - 1]); + const payload = rest[marker + target_attr_marker.len ..]; + const end = std.mem.indexOfScalar(u8, payload, '"') orelse return error.InvalidMlir; + try output.writer.writeAll(payload[0..end]); + rest = payload[end + 1 ..]; + } + try output.writer.writeAll(rest); + return try self.allocator.dupeZ(u8, output.written()); + } + // ==================== types ==================== pub fn ptrTy(self: *Builder, dt: DType, space: MemorySpace, alignment: u64) !*const mlir.Type { return cute.pointerType(self.ctx, dt.toMlir(self.ctx), space, alignment); } + pub fn swizzledPtrTy(self: *Builder, dt: DType, space: MemorySpace, alignment: u64, swizzle: Swizzle) !*const mlir.Type { + const text = std.fmt.allocPrint(self.arena.allocator(), "S<{d},{d},{d}>", .{ swizzle.bits, swizzle.base, swizzle.shift }) catch @panic("OOM"); + const attr = try cute.SwizzleAttr.get(self.ctx, text); + return (try cute.PtrType.get(self.ctx, .{ + .valueType = dt.toMlir(self.ctx), + .addressSpace = space, + .alignment = alignment, + .swizzle = attr.attribute(), + })).type_(); + } + + pub fn intToPtr(self: *Builder, address: anytype, dt: DType, space: MemorySpace, alignment: u64) Value { + const ptr_type = self.ptrTy(dt, space, alignment) catch @panic("invalid CuTe pointer type"); + return self.emit(cute.inttoptr(self.ctx, self.lift(address).inner, ptr_type, self.loc())); + } + + /// Reinterpret a pointer without changing its address. This is the typed + /// counterpart of Python CuTe's `recast_ptr`; packed MXFP4 buffers use it + /// to expose their byte storage to a scalar correctness implementation. + pub fn recastPointer(self: *Builder, pointer: anytype, dt: DType, space: MemorySpace, alignment: u64) Value { + const result_type = self.ptrTy(dt, space, alignment) catch @panic("invalid recast pointer type"); + return self.emit(cute.recast_iter(self.ctx, self.asValue(pointer).inner, result_type, self.loc())); + } + + pub fn recastSwizzledPointer(self: *Builder, pointer: anytype, dt: DType, space: MemorySpace, alignment: u64, swizzle: Swizzle) Value { + const result_type = self.swizzledPtrTy(dt, space, alignment, swizzle) catch @panic("invalid swizzled pointer type"); + return self.emit(cute.recast_iter(self.ctx, self.asValue(pointer).inner, result_type, self.loc())); + } + + /// Parse a CuTe/CuTe-NVGPU type that has no compact C-API constructor. + /// This is primarily used for architecture-specific atom and descriptor + /// types whose complete layout is part of the type itself. + pub fn parseType(self: *Builder, text: []const u8) *const mlir.Type { + return mlir.Type.parse(self.ctx, text) catch std.debug.panic("invalid MLIR type: {s}", .{text}); + } + + /// Parse a target-specific MLIR attribute. + pub fn parseAttribute(self: *Builder, text: []const u8) *const mlir.Attribute { + return mlir.Attribute.parse(self.ctx, text) catch std.debug.panic("invalid MLIR attribute: {s}", .{text}); + } + + /// Preserve an attribute owned by the runtime CuTe compiler when the + /// lightweight ZML MLIR context has no parser hook for that attribute. + /// It is represented as a verifier-safe string while constructing the + /// module and restored to target assembly by `finish`/`finishProgram`. + pub fn targetAttribute(self: *Builder, text: []const u8) *const mlir.Attribute { + if (std.mem.indexOfScalar(u8, text, '"') != null) @panic("targetAttribute cannot contain a quote"); + const encoded = std.fmt.allocPrint(self.arena.allocator(), "__zml_cute_target_attr__{s}", .{text}) catch @panic("OOM"); + return .string(self.ctx, encoded); + } + fn algebra(self: *Builder, dims: []const i64) []const u8 { var out: std.Io.Writer.Allocating = .init(self.arena.allocator()); if (dims.len == 1) { @@ -501,6 +1090,178 @@ pub const Builder = struct { return out.written(); } + fn writeAlgebra(self: *Builder, writer: *std.Io.Writer, value: anytype) void { + const T = @TypeOf(value); + if (T == AlgebraExpr) { + writer.writeAll(value.text) catch @panic("OOM"); + return; + } + if (T == AlgebraToken) { + writer.writeByte(switch (value) { + .all => '_', + .dynamic => '?', + }) catch @panic("OOM"); + return; + } + if (T == ScaledBasis) { + writer.print("{d}", .{value.numerator}) catch @panic("OOM"); + if (value.denominator != 1) writer.print("/{d}", .{value.denominator}) catch @panic("OOM"); + for (value.modes[0..value.mode_count]) |mode| writer.print("@{d}", .{mode}) catch @panic("OOM"); + return; + } + if (T == ConstrainedDynamic) { + if (value.divisible_by == 0) @panic("CuTe constrained dynamic divisor must be positive"); + writer.print("?{{div={d}}}", .{value.divisible_by}) catch @panic("OOM"); + return; + } + switch (@typeInfo(T)) { + .comptime_int, .int => writer.print("{d}", .{value}) catch @panic("OOM"), + .@"struct" => |info| { + writer.writeByte('(') catch @panic("OOM"); + inline for (info.fields, 0..) |field, i| { + if (i != 0) writer.writeByte(',') catch @panic("OOM"); + self.writeAlgebra(writer, @field(value, field.name)); + } + writer.writeByte(')') catch @panic("OOM"); + }, + else => @compileError("unsupported CuTe algebra value " ++ @typeName(T)), + } + } + + /// Build a CuTe algebra expression from integers, nested Zig tuples, + /// `AlgebraToken`s and `ScaledBasis` values. + pub fn expr(self: *Builder, value: anytype) AlgebraExpr { + var out: std.Io.Writer.Allocating = .init(self.arena.allocator()); + self.writeAlgebra(&out.writer, value); + return .{ .text = out.written() }; + } + + /// `coefficient @ mode...`, including rational coefficients such as + /// `1/2@0`. The mode path maps a coordinate into a nested CuTe mode. + pub fn basis(self: *Builder, numerator: i64, denominator: i64, modes: anytype) ScaledBasis { + _ = self; + if (denominator <= 0) @panic("CuTe scaled-basis denominator must be positive"); + const fields = @typeInfo(@TypeOf(modes)).@"struct".fields; + if (fields.len == 0 or fields.len > MAX_RANK) @panic("CuTe scaled basis needs 1..MAX_RANK modes"); + var result: ScaledBasis = .{ .numerator = numerator, .denominator = denominator, .mode_count = fields.len }; + inline for (fields, 0..) |field, i| result.modes[i] = @intCast(@field(modes, field.name)); + return result; + } + + pub fn layoutSpec(self: *Builder, shape: anytype, stride: anytype) LayoutSpec { + return .{ .shape = self.expr(shape), .stride = self.expr(stride) }; + } + + pub fn layoutType(self: *Builder, spec: LayoutSpec) *const mlir.Type { + return cute.algebraType(self.ctx, .layout, spec.payload(self)) catch @panic("invalid CuTe layout"); + } + + pub fn composedLayoutType(self: *Builder, swizzle: Swizzle, offset: anytype, outer: LayoutSpec) *const mlir.Type { + const payload = std.fmt.allocPrint( + self.arena.allocator(), + "S<{d},{d},{d}> o {s} o {s}", + .{ swizzle.bits, swizzle.base, swizzle.shift, self.expr(offset).text, outer.payload(self) }, + ) catch @panic("OOM"); + return cute.algebraType(self.ctx, .composed_layout, payload) catch @panic("invalid CuTe composed layout"); + } + + pub fn memrefType(self: *Builder, dtype: DType, space: MemorySpace, alignment: u64, layout_type: *const mlir.Type) *const mlir.Type { + const pointer = self.ptrTy(dtype, space, alignment) catch @panic("invalid CuTe pointer type"); + return (cute.MemRefType.get(self.ctx, .{ .ptr = pointer, .layout = layout_type }) catch @panic("invalid CuTe memref type")).type_(); + } + + pub fn memrefTypeFromPointer(self: *Builder, pointer_type: *const mlir.Type, layout_type: *const mlir.Type) *const mlir.Type { + return (cute.MemRefType.get(self.ctx, .{ .ptr = pointer_type, .layout = layout_type }) catch @panic("invalid CuTe memref type")).type_(); + } + + pub fn staticLayout(self: *Builder, spec: LayoutSpec) View { + const result = self.emit(cute.static(self.ctx, self.layoutType(spec), self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Materialize a rational scaled-basis layout used by block-scale TMA. + /// The local standalone LayoutAttr constructor only accepts integer + /// strides, while CoordTensorType owns the more general coordinate + /// algebra. Recovering its layout through the typed C API avoids exposing + /// that parser distinction to kernel code. + pub fn staticCoordinateLayout(self: *Builder, rank: usize, spec: LayoutSpec) View { + const tensor_type = self.coordTensorType(rank, spec); + const coords = tensor_type.isA(cute.CoordTensorType) orelse @panic("invalid coordinate tensor type"); + const result = self.emit(cute.static(self.ctx, coords.getLayout(), self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub const Swizzle = struct { bits: i64, base: i64, shift: i64 }; + + pub fn staticSwizzle(self: *Builder, spec: Swizzle) View { + const payload = std.fmt.allocPrint(self.arena.allocator(), "S<{d},{d},{d}>", .{ spec.bits, spec.base, spec.shift }) catch @panic("OOM"); + const ty = cute.algebraType(self.ctx, .swizzle, payload) catch @panic("invalid CuTe swizzle"); + const result = self.emit(cute.static(self.ctx, ty, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Construct `swizzle o offset o outer_layout` through the CuTe dialect. + pub fn makeComposedLayout(self: *Builder, swizzle: Swizzle, offset: anytype, outer: LayoutSpec) View { + const inner = self.staticSwizzle(swizzle); + const offset_expr = self.expr(offset); + const offset_ty = cute.algebraType(self.ctx, .int_tuple, offset_expr.text) catch @panic("invalid CuTe composed-layout offset"); + const offset_value = self.emit(cute.make_int_tuple(self.ctx, &.{}, offset_ty, self.loc())); + const outer_value = self.staticLayout(outer); + const result_ty = self.composedLayoutType(swizzle, offset, outer); + const result = self.emit(cute.make_composed_layout(self.ctx, inner.inner, offset_value.inner, outer_value.inner, result_ty, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Compose a raw pointer and any CuTe layout value. The memref result type + /// is obtained through the dialect C API rather than parsed from assembly. + pub fn makeTensorView(self: *Builder, pointer: anytype, layout: anytype) View { + const p = self.asValue(pointer); + const l = self.asValue(layout); + const result_ty = (cute.MemRefType.get(self.ctx, .{ .ptr = p.type_(), .layout = l.type_() }) catch @panic("invalid CuTe tensor view")).type_(); + const result = self.emit(cute.make_view(self.ctx, p.inner, l.inner, result_ty, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn coordTensorType(self: *Builder, rank: usize, layout: LayoutSpec) *const mlir.Type { + if (rank == 0 or rank > MAX_RANK) @panic("coordTensorType rank must be in 1..MAX_RANK"); + var zeros: std.Io.Writer.Allocating = .init(self.arena.allocator()); + if (rank > 1) zeros.writer.writeByte('(') catch @panic("OOM"); + for (0..rank) |i| { + if (i != 0) zeros.writer.writeByte(',') catch @panic("OOM"); + zeros.writer.writeByte('0') catch @panic("OOM"); + } + if (rank > 1) zeros.writer.writeByte(')') catch @panic("OOM"); + const tuple = cute.algebraType(self.ctx, .int_tuple, zeros.written()) catch @panic("invalid coordinate tuple"); + const payload = layout.payload(self); + // The standalone LayoutAttr C API intentionally restricts scaled + // basis coefficients to integers. CoordTensorType additionally + // accepts the rational basis used by block-scaled scale-factor maps. + // Keep that parser workaround inside the DSL type constructor. + if (std.mem.indexOfScalar(u8, payload, '/') != null) { + const type_text = std.fmt.allocPrint(self.arena.allocator(), "!cute.coord_tensor<\"{s}\", \"{s}\">", .{ zeros.written(), payload }) catch @panic("OOM"); + return self.parseType(type_text); + } + return (cute.CoordTensorType.get(self.ctx, .{ .arithTuple = tuple, .layout = self.layoutType(layout) }) catch @panic("invalid coordinate tensor")).type_(); + } + + /// Coordinate-tensor type whose coordinate tuple contains dynamic and + /// static leaves. `tuple_payload` is CuTe algebra such as `(0,?,?)`; + /// layouts remain structured `LayoutSpec` values in Zig. + pub fn coordTensorTypePayload(self: *Builder, tuple_payload: []const u8, layout: LayoutSpec) *const mlir.Type { + const attr = cute.IntTupleAttr.get(self.ctx, tuple_payload) catch std.debug.panic("invalid coordinate tuple {s}", .{tuple_payload}); + const tuple = (cute.IntTupleType.get(self.ctx, .{ .attr = attr.attribute() }) catch @panic("invalid coordinate tuple type")).type_(); + return (cute.CoordTensorType.get(self.ctx, .{ .arithTuple = tuple, .layout = self.layoutType(layout) }) catch @panic("invalid coordinate tensor")).type_(); + } + + /// Coordinate-tensor type with an explicit arithmetic-tuple expression. + /// This represents the divisibility information produced by CuTe tiling + /// without requiring kernel code to spell textual MLIR types. + pub fn coordTensorTypeWithTuple(self: *Builder, tuple: anytype, layout: LayoutSpec) *const mlir.Type { + const tuple_text = self.expr(tuple).text; + const tuple_type = cute.algebraType(self.ctx, .int_tuple, tuple_text) catch std.debug.panic("invalid coordinate tuple {s}", .{tuple_text}); + return (cute.CoordTensorType.get(self.ctx, .{ .arithTuple = tuple_type, .layout = self.layoutType(layout) }) catch @panic("invalid coordinate tensor")).type_(); + } + fn rowMajor(self: *Builder, shape: []const i64) []const i64 { const s = self.arena.allocator().alloc(i64, shape.len) catch @panic("OOM"); var acc: i64 = 1; @@ -524,6 +1285,16 @@ pub const Builder = struct { // ==================== cute.core ==================== + /// Materialize a value whose complete static CuTe type is already known. + /// + /// Nested layouts and integer tuples produced by CuTe's layout algebra + /// cannot be represented by the flat `Layout` helper. Keeping this + /// escape hatch typed still lets MLIR verify every consumer. + pub fn staticValue(self: *Builder, type_text: []const u8) View { + const result = self.emit(cute.static(self.ctx, self.parseType(type_text), self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + /// `cute.make_layout(shape, stride=...)`, static only. pub fn makeLayout(self: *Builder, shape: []const i64, opts: LayoutOpts) Layout { if (shape.len == 0 or shape.len > MAX_RANK) std.debug.panic("makeLayout: rank {d} not in 1..{d}", .{ shape.len, MAX_RANK }); @@ -545,31 +1316,1070 @@ pub const Builder = struct { return .{ .inner = v.inner, .kernel = self, .layout = layout, .dtype = dtypes.mlirToDType(self.ctx, elem), .space = pt.getAddressSpace() }; } - /// A `!cute.coord` from a tuple of `Value`s and ints; ints stay static in the type. - pub fn makeCoord(self: *Builder, coord: anytype) *const mlir.Value { + fn writeCoord(self: *Builder, writer: *std.Io.Writer, dyn: *stdx.BoundedArray(*const mlir.Value, MAX_RANK), coord: anytype) void { const T = @TypeOf(coord); - if (T == Value) return self.makeCoord(.{coord}); - const fields = @typeInfo(T).@"struct".fields; - if (fields.len == 0 or fields.len > MAX_RANK) @compileError("makeCoord: rank must be 1..MAX_RANK"); + if (T == Value) { + writer.writeByte('?') catch @panic("OOM"); + dyn.append(self.lift(coord).to(.i32).inner) catch @panic("makeCoord: too many dynamic leaves"); + return; + } + if (T == AlgebraToken) { + writer.writeByte(switch (coord) { + .all => '_', + .dynamic => @panic("makeCoord: AlgebraToken.dynamic needs an SSA value"), + }) catch @panic("OOM"); + return; + } + switch (@typeInfo(T)) { + .comptime_int, .int => writer.print("{d}", .{coord}) catch @panic("OOM"), + .@"struct" => |info| { + if (info.fields.len == 0) @compileError("makeCoord: empty tuple"); + if (info.fields.len > 1) writer.writeByte('(') catch @panic("OOM"); + inline for (info.fields, 0..) |field, i| { + if (i != 0) writer.writeByte(',') catch @panic("OOM"); + self.writeCoord(writer, dyn, @field(coord, field.name)); + } + if (info.fields.len > 1) writer.writeByte(')') catch @panic("OOM"); + }, + else => @compileError("makeCoord: unsupported coordinate leaf " ++ @typeName(T)), + } + } + + /// A `!cute.coord` from nested tuples of `Value`s, integers, and `.all` + /// placeholders. Integer leaves stay static in the dependent CuTe type. + pub fn makeCoord(self: *Builder, coord: anytype) *const mlir.Value { + if (@TypeOf(coord) == Value) return self.makeCoord(.{coord}); var text: std.Io.Writer.Allocating = .init(self.arena.allocator()); var dyn: stdx.BoundedArray(*const mlir.Value, MAX_RANK) = .empty; - if (fields.len > 1) text.writer.writeByte('(') catch @panic("OOM"); - inline for (fields, 0..) |f, i| { - if (i > 0) text.writer.writeByte(',') catch @panic("OOM"); - const leaf = @field(coord, f.name); - switch (@typeInfo(f.type)) { - .comptime_int, .int => text.writer.print("{d}", .{leaf}) catch @panic("OOM"), - else => { - text.writer.writeByte('?') catch @panic("OOM"); - dyn.appendAssumeCapacity(self.lift(leaf).to(.i32).inner); - }, - } - } - if (fields.len > 1) text.writer.writeByte(')') catch @panic("OOM"); + self.writeCoord(&text.writer, &dyn, coord); const ty = cute.algebraType(self.ctx, .coord, text.written()) catch @panic("bad coord"); return self.emit(cute.make_coord(self.ctx, dyn.constSlice(), ty, self.loc())).inner; } + /// A dynamic/static CuTe integer tuple, used for pointer arithmetic in + /// tensor and shared memory. It accepts the same nested Zig values as a + /// coordinate, but preserves integer-tuple semantics. + pub fn makeIntTuple(self: *Builder, values: anytype) View { + var text: std.Io.Writer.Allocating = .init(self.arena.allocator()); + var dyn: stdx.BoundedArray(*const mlir.Value, MAX_RANK) = .empty; + self.writeCoord(&text.writer, &dyn, values); + const ty = cute.algebraType(self.ctx, .int_tuple, text.written()) catch @panic("bad integer tuple"); + const value = self.emit(cute.make_int_tuple(self.ctx, dyn.constSlice(), ty, self.loc())); + return .{ .inner = value.inner, .kernel = self }; + } + + /// A one-element integer tuple holding a value known to be a multiple of + /// `divisor`. Ops that derive alignment from an offset, such as + /// `cute.add_offset`, need that to keep the operand's alignment. + pub fn makeIntTupleDivBy(self: *Builder, value: anytype, comptime divisor: u32) View { + const constrained = self.assumeDivBy(value, divisor); + const payload = std.fmt.allocPrint(self.arena.allocator(), "!cute.int_tuple<\"?{{div={d}}}\">", .{divisor}) catch @panic("OOM"); + const tuple = self.emit(cute.make_int_tuple(self.ctx, &.{constrained.inner}, self.parseType(payload), self.loc())); + return .{ .inner = tuple.inner, .kernel = self }; + } + + /// Refine an integer SSA value with the divisibility information CuTe + /// derives while slicing a tiled coordinate tensor. + pub fn assumeDivBy(self: *Builder, value: anytype, comptime divisor: u32) Value { + if (divisor == 0) @compileError("assumeDivBy divisor must be non-zero"); + const type_text = std.fmt.allocPrint(self.arena.allocator(), "!cute.i32", .{divisor}) catch @panic("OOM"); + return self.emit(cute.assume(self.ctx, self.lift(value).to(.i32).inner, self.parseType(type_text), self.loc())); + } + + /// Build a coordinate tensor with an explicit dependent arithmetic-tuple + /// type. This is the source-level equivalent of the tuple produced by + /// `slice` after TMA partitioning (for example `?{div=128}`). + pub fn makeCoordTensorWithTuple(self: *Builder, values: anytype, tuple_payload: []const u8, layout: anytype) View { + const tuple_text = std.fmt.allocPrint(self.arena.allocator(), "!cute.int_tuple<\"{s}\">", .{tuple_payload}) catch @panic("OOM"); + const tuple_type = self.parseType(tuple_text); + const tuple = self.emit(cute.make_int_tuple(self.ctx, self.innerSlice(values), tuple_type, self.loc())); + const iterator_type = (cute.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = tuple_type }) catch @panic("invalid arithmetic tuple iterator")).type_(); + const iterator = self.emit(cute.make_arith_tuple_iter(self.ctx, tuple.inner, iterator_type, self.loc())); + const layout_value = self.asValue(layout); + const result_type = (cute.CoordTensorType.get(self.ctx, .{ .arithTuple = tuple_type, .layout = layout_value.type_() }) catch @panic("invalid coordinate tensor")).type_(); + const result = self.emit(cute.make_view(self.ctx, iterator.inner, layout_value.inner, result_type, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Build a coordinate tensor from a runtime origin and a static layout. + /// This is the direct form of the `get_iter`/`make_arith_tuple_iter`/ + /// `make_view` sequence emitted by the Python CuTe front end after a tile + /// is selected from a larger coordinate tensor. + pub fn makeCoordTensor(self: *Builder, origin: anytype, layout: anytype) View { + const tuple = self.makeIntTuple(origin); + const layout_value = self.asValue(layout); + const iterator_type = (cute.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = tuple.type_() }) catch @panic("invalid arithmetic tuple iterator")).type_(); + const iterator = self.emit(cute.make_arith_tuple_iter(self.ctx, tuple.inner, iterator_type, self.loc())); + const result_type = (cute.CoordTensorType.get(self.ctx, .{ .arithTuple = tuple.type_(), .layout = layout_value.type_() }) catch @panic("invalid coordinate tensor")).type_(); + const result = self.emit(cute.make_view(self.ctx, iterator.inner, layout_value.inner, result_type, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Rebuild a coordinate tensor from the arithmetic-tuple iterator of an + /// existing tensor. CuTe's Python front end emits this sequence after it + /// slices a TMA-partitioned coordinate tensor: + /// + /// get_iter -> deref_arith_tuple_iter -> get_leaves -> make_int_tuple + /// -> make_arith_tuple_iter -> make_view + /// + /// Keeping that sequence explicit matters for TMA. The iterator carries + /// the descriptor coordinates selected by `slice`; constructing a fresh + /// tuple from the scheduler indices would lose CuTe's coordinate algebra. + pub fn rebuildCoordTensorIterator( + self: *Builder, + comptime leaf_count: usize, + input: anytype, + tuple_payload: []const u8, + comptime leaf_divisibility: [leaf_count]u32, + layout: anytype, + ) View { + if (leaf_count == 0 or leaf_count > MAX_RANK) @panic("invalid coordinate tuple leaf count"); + const source = self.asValue(input); + const source_type = source.type_().isA(cute.CoordTensorType) orelse + std.debug.panic("rebuildCoordTensorIterator: expected coordinate tensor, got {f}", .{source.type_()}); + const source_tuple_type = source_type.getArithTuple(); + const source_iterator_type = (cute.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = source_tuple_type }) catch + @panic("invalid source arithmetic tuple iterator")).type_(); + const source_iterator = self.emit(cute.get_iter(self.ctx, source.inner, source_iterator_type, self.loc())); + const source_tuple = self.emit(cute.deref_arith_tuple_iter( + self.ctx, + source_iterator.inner, + source_tuple_type, + self.loc(), + )); + + const leaf_types = self.arena.allocator().alloc(*const mlir.Type, leaf_count) catch @panic("OOM"); + // 0 marks a leaf the producer knows to be statically zero, 1 a leaf + // with no known divisibility. + inline for (leaf_divisibility, 0..) |divisor, i| { + const leaf_payload = if (divisor == 0) + "0" + else if (divisor == 1) + "?" + else + std.fmt.allocPrint(self.arena.allocator(), "?{{div={d}}}", .{divisor}) catch @panic("OOM"); + leaf_types[i] = cute.algebraType(self.ctx, .int_tuple, leaf_payload) catch + std.debug.panic("invalid dynamic integer tuple leaf {s}", .{leaf_payload}); + } + const leaves = self.emitResults(leaf_count, cute.get_leaves( + self.ctx, + source_tuple.inner, + leaf_types, + self.loc(), + )); + + const rebuilt_tuple_type = cute.algebraType(self.ctx, .int_tuple, tuple_payload) catch + std.debug.panic("invalid rebuilt coordinate tuple {s}", .{tuple_payload}); + const rebuilt_tuple = self.emit(cute.make_int_tuple( + self.ctx, + self.innerSlice(leaves[0..]), + rebuilt_tuple_type, + self.loc(), + )); + const rebuilt_iterator_type = (cute.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = rebuilt_tuple_type }) catch + @panic("invalid rebuilt arithmetic tuple iterator")).type_(); + const rebuilt_iterator = self.emit(cute.make_arith_tuple_iter( + self.ctx, + rebuilt_tuple.inner, + rebuilt_iterator_type, + self.loc(), + )); + const layout_value = self.asValue(layout); + const result_type = (cute.CoordTensorType.get(self.ctx, .{ + .arithTuple = rebuilt_tuple_type, + .layout = layout_value.type_(), + }) catch @panic("invalid rebuilt coordinate tensor")).type_(); + const result = self.emit(cute.make_view( + self.ctx, + rebuilt_iterator.inner, + layout_value.inner, + result_type, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + /// A coordinate made only of CuTe placeholders, such as `(_,_,_)`. + /// Placeholders select every element of a mode and therefore cannot be + /// represented by Zig integer values passed to `makeCoord`. + pub fn staticCoord(self: *Builder, payload: []const u8) View { + const type_text = std.fmt.allocPrint(self.arena.allocator(), "!cute.coord<\"{s}\">", .{payload}) catch @panic("OOM"); + const result = self.emit(cute.make_coord(self.ctx, &.{}, self.parseType(type_text), self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Static `cute.make_shape`; `payload` is CuTe algebra such as + /// `((128,16),1,1,2)`. Dynamic leaves can be added when a kernel needs + /// them, but all current SM100 instruction tiles are static. + pub fn makeShape(self: *Builder, payload: []const u8) View { + const ty_text = std.fmt.allocPrint(self.arena.allocator(), "!cute.shape<\"{s}\">", .{payload}) catch @panic("OOM"); + const value = self.emit(cute.make_shape(self.ctx, &.{}, self.parseType(ty_text), self.loc())); + return .{ .inner = value.inner, .kernel = self }; + } + + /// Dynamic `cute.make_shape`. The result type describes which leaves are + /// runtime values (for example `!cute.shape<"(?,?,?)">`). Keeping these + /// leaves dynamic is significant to CuTe's TMA layout algebra when a + /// runtime extent happens to be one. + pub fn makeShapeValues(self: *Builder, values: anytype, result_type: *const mlir.Type) View { + const value = self.emit(cute.make_shape(self.ctx, self.innerSlice(values), result_type, self.loc())); + return .{ .inner = value.inner, .kernel = self }; + } + + pub fn tileToShapeTyped(self: *Builder, input: anytype, shape: anytype, order: View, result_type: *const mlir.Type) View { + const result = self.emit(cute.tile_to_shape( + self.ctx, + self.asValue(input).inner, + self.asValue(shape).inner, + order.inner, + result_type, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Static `cute.make_tile` from mode extents. + pub fn makeTile(self: *Builder, shape: []const i64) View { + var payload: std.Io.Writer.Allocating = .init(self.arena.allocator()); + payload.writer.writeByte('[') catch @panic("OOM"); + for (shape, 0..) |extent, i| { + if (i != 0) payload.writer.writeByte(';') catch @panic("OOM"); + payload.writer.print("{d}:1", .{extent}) catch @panic("OOM"); + } + payload.writer.writeByte(']') catch @panic("OOM"); + const ty_text = std.fmt.allocPrint(self.arena.allocator(), "!cute.tile<\"{s}\">", .{payload.written()}) catch @panic("OOM"); + const value = self.emit(cute.make_tile(self.ctx, &.{}, self.parseType(ty_text), self.loc())); + return .{ .inner = value.inner, .kernel = self }; + } + + /// Construct a target atom whose complete architecture payload is + /// encoded in `atom_type` (for example `!cute_nvgpu.sm100.mma_bs<...>`). + pub fn makeAtom(self: *Builder, atom_type: []const u8, values: anytype) Atom { + const result = self.emit(cute.make_atom(self.ctx, self.innerSlice(values), self.parseType(atom_type), self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn makeTiledMma(self: *Builder, atom: Atom, result_type: []const u8) Atom { + const result = self.emit(cute.make_tiled_mma(self.ctx, atom.inner, self.parseType(result_type), self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Allocate a statically typed shared-memory tensor. This is used for + /// swizzled layouts which cannot be represented by `Layout` alone. + pub fn allocSmemView(self: *Builder, memref_type: []const u8) View { + const result = self.emit(cute.memref_alloc_smem(self.ctx, self.parseType(memref_type), self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Allocate shared memory with a type assembled by the Zig DSL. This is + /// the non-string counterpart of `allocSmemView` used by source kernels. + pub fn allocSmemTyped(self: *Builder, dtype: DType, alignment: u64, layout_type: *const mlir.Type) View { + const memref_type = self.memrefType(dtype, .smem, alignment, layout_type); + const result = self.emit(cute.memref_alloc_smem(self.ctx, memref_type, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Allocate one compiler-visible shared-memory storage object. The field + /// descriptions use the CuTe host compiler's `name:size:offset` format; + /// they let its shared-memory sizing/layout pass reproduce a Python + /// `SharedStorage` struct while Zig keeps typed pointers to its fields. + pub fn allocSmemStorage( + self: *Builder, + dtype: DType, + alignment: u64, + layout_type: *const mlir.Type, + partition_id: i32, + fields: []const []const u8, + ) View { + const field_attrs = self.arena.allocator().alloc(*const mlir.Attribute, fields.len) catch @panic("OOM"); + for (fields, field_attrs) |field, *attr| attr.* = .string(self.ctx, field); + const memref_type = self.memrefType(dtype, .smem, alignment, layout_type); + const result = self.emit(mlir.Operation.make(self.ctx, "cute.memref.alloca", .{ + .results = .{ .flat = &.{memref_type} }, + .attributes = &.{ + .named(self.ctx, "smem.partition_id", .int(self.ctx, .i32, partition_id)), + .named(self.ctx, "smem.struct_fields", .array(self.ctx, field_attrs)), + }, + .location = self.loc(), + })); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Allocate a register-memory tensor whose layout is represented by a + /// normal CuTe layout value. + pub fn allocRmemTyped(self: *Builder, dtype: DType, alignment: u64, layout: anytype) View { + const layout_value = self.asValue(layout); + const memref_type = self.memrefType(dtype, .rmem, alignment, layout_value.type_()); + const result = self.emit(cute.memref_alloca(self.ctx, layout_value.inner, memref_type, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn makeView(self: *Builder, pointer: anytype, layout: anytype, result_type: []const u8) View { + const result = self.emit(cute.make_view( + self.ctx, + self.asValue(pointer).inner, + self.asValue(layout).inner, + self.parseType(result_type), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn makeViewTyped(self: *Builder, pointer: anytype, layout: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute.make_view( + self.ctx, + self.asValue(pointer).inner, + self.asValue(layout).inner, + result_type, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn makeViewNoLayout(self: *Builder, pointer: anytype, result_type: []const u8) View { + const result = self.emit(cute.make_view(self.ctx, self.asValue(pointer).inner, null, self.parseType(result_type), self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn localTile(self: *Builder, input: anytype, tile: anytype, coord: anytype, result_type: []const u8, projection: ?Attr) View { + const coord_value = if (@TypeOf(coord) == View) coord.inner else self.makeCoord(coord); + const result = self.emit(cute.local_tile( + self.ctx, + self.asValue(input).inner, + self.asValue(tile).inner, + coord_value, + self.parseType(result_type), + if (projection) |p| p.get(self.ctx) else null, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn localTileTyped(self: *Builder, input: anytype, tile: anytype, coord: anytype, result_type: *const mlir.Type, projection: ?Attr) View { + const coord_value = if (@TypeOf(coord) == View) coord.inner else self.makeCoord(coord); + const result = self.emit(cute.local_tile( + self.ctx, + self.asValue(input).inner, + self.asValue(tile).inner, + coord_value, + result_type, + if (projection) |p| p.get(self.ctx) else null, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn localPartition(self: *Builder, input: anytype, tiler: anytype, index: anytype, result_type: []const u8) View { + const result = self.emit(cute.local_partition( + self.ctx, + self.asValue(input).inner, + self.asValue(tiler).inner, + self.makeCoord(index), + self.parseType(result_type), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Apply CuTe layout algebra while keeping the compiler-derived result + /// type explicit. These operations are the building blocks used between + /// TMA partitioning and the block-scaled MMA fragments. + pub fn slice(self: *Builder, input: anytype, coord: anytype, result_type: []const u8) View { + const result = self.emit(cute.slice( + self.ctx, + self.asValue(input).inner, + self.makeCoord(coord), + self.parseType(result_type), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn sliceTyped(self: *Builder, input: anytype, coord: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute.slice( + self.ctx, + self.asValue(input).inner, + self.makeCoord(coord), + result_type, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Rebuild a tensor with the same iterator and a new layout. This is the + /// tensor form of CuTe's `group_modes`: grouping changes only the layout, + /// while the backing pointer or arithmetic-tuple iterator stays intact. + /// The local CUTLASS schema exposes `cute.group_modes` only for layouts, + /// so expressing the equivalent algebra this way also remains portable. + pub fn withLayout(self: *Builder, input: anytype, layout: anytype) View { + const value = self.asValue(input); + const layout_value = self.asValue(layout); + const input_type = value.type_(); + const result_type: *const mlir.Type = if (input_type.isA(cute.MemRefType)) |memref| + (cute.MemRefType.get(self.ctx, .{ .ptr = memref.getPtr(), .layout = layout_value.type_() }) catch @panic("invalid regrouped memref")).type_() + else if (input_type.isA(cute.CoordTensorType)) |coords| + (cute.CoordTensorType.get(self.ctx, .{ .arithTuple = coords.getArithTuple(), .layout = layout_value.type_() }) catch @panic("invalid regrouped coordinate tensor")).type_() + else + std.debug.panic("withLayout: expected memref or coordinate tensor, got {f}", .{input_type}); + + const iterator_type: *const mlir.Type = if (input_type.isA(cute.MemRefType)) |memref| + memref.getPtr() + else if (input_type.isA(cute.CoordTensorType)) |coords| + (cute.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = coords.getArithTuple() }) catch @panic("invalid arithmetic tuple iterator")).type_() + else + unreachable; + const iterator = self.emit(cute.get_iter(self.ctx, value.inner, iterator_type, self.loc())); + const result = self.emit(cute.make_view(self.ctx, iterator.inner, layout_value.inner, result_type, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn groupModes(self: *Builder, input: anytype, begin: u32, end: u32, result_type: []const u8) View { + const result = self.emit(cute.group_modes( + self.ctx, + self.asValue(input).inner, + self.parseType(result_type), + .int(self.ctx, .i32, begin), + .int(self.ctx, .i32, end), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn tiledDivide(self: *Builder, input: anytype, tiler: anytype, result_type: []const u8) View { + const result = self.emit(cute.tiled_divide( + self.ctx, + self.asValue(input).inner, + self.asValue(tiler).inner, + self.parseType(result_type), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn flatDivide(self: *Builder, input: anytype, tiler: anytype, result_type: []const u8) View { + const result = self.emit(cute.flat_divide( + self.ctx, + self.asValue(input).inner, + self.asValue(tiler).inner, + self.parseType(result_type), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn coalesce(self: *Builder, input: anytype, target_profile: ?View, result_type: []const u8) View { + const result = self.emit(cute.coalesce( + self.ctx, + self.asValue(input).inner, + if (target_profile) |target| target.inner else null, + self.parseType(result_type), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn filterZeros(self: *Builder, input: anytype, target_profile: ?View, result_type: []const u8) View { + const result = self.emit(cute.filter_zeros( + self.ctx, + self.asValue(input).inner, + if (target_profile) |target| target.inner else null, + self.parseType(result_type), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn getLayout(self: *Builder, input: anytype, result_type: []const u8) View { + const result = self.emit(cute.get_layout(self.ctx, self.asValue(input).inner, self.parseType(result_type), self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn getIter(self: *Builder, input: anytype, result_type: []const u8) View { + const result = self.emit(cute.get_iter(self.ctx, self.asValue(input).inner, self.parseType(result_type), self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Typed form of `cute.get_iter`, used when the pointer type is already + /// available through the CuTe C API and no textual type is necessary. + pub fn getIterTyped(self: *Builder, input: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute.get_iter(self.ctx, self.asValue(input).inner, result_type, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn getShape(self: *Builder, input: anytype, result_type: []const u8) View { + const result = self.emit(cute.get_shape(self.ctx, self.asValue(input).inner, self.parseType(result_type), self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn addOffset(self: *Builder, input: anytype, offset: anytype, result_type: []const u8) View { + const result = self.emit(cute.add_offset( + self.ctx, + self.asValue(input).inner, + self.asValue(offset).inner, + self.parseType(result_type), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn addOffsetTyped(self: *Builder, input: anytype, offset: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute.add_offset( + self.ctx, + self.asValue(input).inner, + self.asValue(offset).inner, + result_type, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn recastIter(self: *Builder, input: anytype, result_type: []const u8) View { + const result = self.emit(cute.recast_iter(self.ctx, self.asValue(input).inner, self.parseType(result_type), self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn appendToRank(self: *Builder, input: anytype, element: anytype, rank: u32, result_type: []const u8) View { + const result = self.emit(cute.append_to_rank( + self.ctx, + self.asValue(input).inner, + self.asValue(element).inner, + self.parseType(result_type), + .int(self.ctx, .i32, rank), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn makeTiledCopy(self: *Builder, atom: Atom, result_type: []const u8) Atom { + const result = self.emit(cute.make_tiled_copy(self.ctx, atom.inner, self.parseType(result_type), self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn tiledCopyPartition(self: *Builder, tiled: Atom, input: anytype, coord: anytype, destination: bool, result_type: []const u8) View { + const op = if (destination) + cute.tiled_copy_partition_D(self.ctx, tiled.inner, self.asValue(input).inner, self.makeCoord(coord), self.parseType(result_type), self.loc()) + else + cute.tiled_copy_partition_S(self.ctx, tiled.inner, self.asValue(input).inner, self.makeCoord(coord), self.parseType(result_type), self.loc()); + const result = self.emit(op); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn tiledCopyRetile(self: *Builder, tiled: Atom, input: anytype, result_type: []const u8) View { + const result = self.emit(cute.tiled_copy_retile( + self.ctx, + tiled.inner, + self.asValue(input).inner, + self.parseType(result_type), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn tiledMmaPartition(self: *Builder, tiled: Atom, input: anytype, coord: anytype, operand_id: u2, result_type: []const u8) View { + const result = self.emit(cute.tiled_mma_partition( + self.ctx, + tiled.inner, + self.asValue(input).inner, + self.makeCoord(coord), + self.parseType(result_type), + .int(self.ctx, .i32, operand_id), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn tiledMmaPartitionTyped(self: *Builder, tiled: Atom, input: anytype, coord: anytype, operand_id: u2, result_type: *const mlir.Type) View { + const result = self.emit(cute.tiled_mma_partition( + self.ctx, + tiled.inner, + self.asValue(input).inner, + self.makeCoord(coord), + result_type, + .int(self.ctx, .i32, operand_id), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn mmaMakeFragment(self: *Builder, tiled: Atom, input: anytype, operand_id: u2, result_type: []const u8) View { + const result = self.emit(cute.mma_make_fragment( + self.ctx, + tiled.inner, + self.asValue(input).inner, + self.parseType(result_type), + .int(self.ctx, .i32, operand_id), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn copy(self: *Builder, atom: Atom, src: anytype, dst: anytype, pred: ?Value) void { + self.emitVoid(cute.copy( + self.ctx, + atom.inner, + self.innerSlice(src), + self.innerSlice(dst), + if (pred) |p| p.inner else null, + self.loc(), + )); + } + + pub fn gemm(self: *Builder, tiled: Atom, d: anytype, a: anytype, rhs: anytype, c: anytype) void { + self.emitVoid(cute.gemm( + self.ctx, + tiled.inner, + self.asValue(d).inner, + self.innerSlice(a), + self.innerSlice(rhs), + self.asValue(c).inner, + self.loc(), + )); + } + + pub fn vectorExtract(self: *Builder, source: Value, lane: i64, dtype: DType) Value { + return self.emit(vector.extract( + self.ctx, + source.inner, + &.{lane}, + &.{}, + dtype.toMlir(self.ctx), + self.loc(), + )); + } + + // ==================== cute.nvgpu.cpasync: TMA (SM90+) ==================== + // The host function builds tiled TMA atoms (Python `make_tiled_tma_atom`); + // the kernel receives them as grid-constant arguments and makes them + // executable before issuing copies. + + pub const TmaLoadOptions = struct { + atom_type: []const u8, + tensor_type: []const u8, + kind: Attr = .{ .str = "tma_load" }, + num_multicast: u32 = 1, + tma_format: ?Attr = null, + }; + + /// Build the host-visible TMA descriptor and its coordinate tensor. The + /// result types are supplied explicitly because they depend on CuTe's + /// layout algebra and may contain divisibility constraints. + pub fn makeTiledTmaLoadAtom(self: *Builder, gmem: anytype, smem_layout: anytype, cta_map: anytype, opts: TmaLoadOptions) TmaDescriptor { + const results = self.emitResults(2, cute.nvgpu.atom_make_non_exec_tiled_tma_load( + self.ctx, + self.asValue(gmem).inner, + self.asValue(smem_layout).inner, + self.asValue(cta_map).inner, + self.parseType(opts.atom_type), + self.parseType(opts.tensor_type), + opts.kind.get(self.ctx), + .int(self.ctx, .i32, opts.num_multicast), + if (opts.tma_format) |format| format.get(self.ctx) else null, + self.loc(), + )); + return .{ + .atom = .{ .inner = results[0].inner, .kernel = self }, + .tensor = .{ .inner = results[1].inner, .kernel = self }, + }; + } + + /// Typed counterpart of `makeTiledTmaLoadAtom`: the atom and coordinate + /// tensor types are assembled from `config`. + pub fn makeTiledTmaLoadAtomTyped(self: *Builder, gmem: anytype, smem_layout: anytype, cta_map: anytype, config: TmaConfig) TmaDescriptor { + const results = self.emitResults(2, cute.nvgpu.atom_make_non_exec_tiled_tma_load( + self.ctx, + self.asValue(gmem).inner, + self.asValue(smem_layout).inner, + self.asValue(cta_map).inner, + self.tmaLoadAtomType(config), + self.coordTensorType(config.coordinate_rank, config.coordinate_layout), + self.parseAttribute("#cute_nvgpu.tiled_tma_load"), + .int(self.ctx, .i32, config.num_multicast), + tmaFormatAttribute(self, config.format), + self.loc(), + )); + return .{ + .atom = .{ .inner = results[0].inner, .kernel = self }, + .tensor = .{ .inner = results[1].inner, .kernel = self }, + }; + } + + pub const TmaStoreOptions = struct { + atom_type: []const u8, + tensor_type: []const u8, + tma_format: ?Attr = null, + }; + + /// Host-visible tiled TMA store atom and its coordinate tensor. + pub fn makeTiledTmaStoreAtom(self: *Builder, gmem: anytype, smem_layout: anytype, cta_map: anytype, opts: TmaStoreOptions) TmaDescriptor { + const results = self.emitResults(2, cute.nvgpu.atom_make_non_exec_tiled_tma_store( + self.ctx, + self.asValue(gmem).inner, + self.asValue(smem_layout).inner, + self.asValue(cta_map).inner, + self.parseType(opts.atom_type), + self.parseType(opts.tensor_type), + if (opts.tma_format) |format| format.get(self.ctx) else null, + self.loc(), + )); + return .{ + .atom = .{ .inner = results[0].inner, .kernel = self }, + .tensor = .{ .inner = results[1].inner, .kernel = self }, + }; + } + + /// Typed counterpart of `makeTiledTmaStoreAtom`. + pub fn makeTiledTmaStoreAtomTyped(self: *Builder, gmem: anytype, smem_layout: anytype, cta_map: anytype, config: TmaConfig) TmaDescriptor { + const results = self.emitResults(2, cute.nvgpu.atom_make_non_exec_tiled_tma_store( + self.ctx, + self.asValue(gmem).inner, + self.asValue(smem_layout).inner, + self.asValue(cta_map).inner, + self.tmaStoreAtomType(config), + self.coordTensorType(config.coordinate_rank, config.coordinate_layout), + tmaFormatAttribute(self, config.format), + self.loc(), + )); + return .{ + .atom = .{ .inner = results[0].inner, .kernel = self }, + .tensor = .{ .inner = results[1].inner, .kernel = self }, + }; + } + + /// Host-side tiled TMA load atom type described by `config`. + pub fn tmaLoadAtomType(self: *Builder, config: TmaConfig) *const mlir.Type { + return self.parseType(std.fmt.allocPrint( + self.arena.allocator(), + "!cute_nvgpu.atom.non_exec_tiled_tma_load, tma_format = {s}>", + .{ dtypeName(config.dtype), config.copy_bits, config.global_basis.payload(self), formatName(config.format, config.dtype) }, + ) catch @panic("OOM")); + } + + /// Executable device-side form of a tiled TMA load atom. `make_exec_tma` + /// preserves the tensor-map payload and adds the launch mode used by + /// `copy`. + pub fn tmaLoadExecAtomType(self: *Builder, config: TmaConfig) *const mlir.Type { + return self.parseType(std.fmt.allocPrint( + self.arena.allocator(), + "!cute_nvgpu.atom.tma_load<{s}, copy_bits = {d}, mode = tiled, num_cta = {d}, g_stride = <\"()\"> tma_gbasis = <\"{s}\">>", + .{ dtypeName(config.dtype), config.copy_bits, config.num_multicast, config.global_basis.payload(self) }, + ) catch @panic("OOM")); + } + + /// Host-side tiled TMA store atom type described by `config`. + pub fn tmaStoreAtomType(self: *Builder, config: TmaConfig) *const mlir.Type { + return self.parseType(std.fmt.allocPrint( + self.arena.allocator(), + "!cute_nvgpu.atom.non_exec_tiled_tma_store<{s}, copy_bits = {d}, tma_gbasis = <\"{s}\">, tma_format = {s}>", + .{ dtypeName(config.dtype), config.copy_bits, config.global_basis.payload(self), formatName(config.format, config.dtype) }, + ) catch @panic("OOM")); + } + + /// Executable device-side form of a tiled TMA store atom. + pub fn tmaStoreExecAtomType(self: *Builder, config: TmaConfig) *const mlir.Type { + return self.parseType(std.fmt.allocPrint( + self.arena.allocator(), + "!cute_nvgpu.atom.tma_store<{s}, copy_bits = {d}, mode = tiled, g_stride = <\"()\"> tma_gbasis = <\"{s}\">>", + .{ dtypeName(config.dtype), config.copy_bits, config.global_basis.payload(self) }, + ) catch @panic("OOM")); + } + + pub fn prefetchTmaDesc(self: *Builder, descriptor: Atom) void { + self.emitVoid(cute.nvgpu.prefetch_tma_desc(self.ctx, descriptor.inner, self.loc())); + } + + pub fn makeExecTma(self: *Builder, descriptor: Atom, result_type: []const u8) Atom { + const result = self.emit(cute.nvgpu.atom_make_exec_tma(self.ctx, descriptor.inner, self.parseType(result_type), null, null, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn makeExecTmaTyped(self: *Builder, descriptor: Atom, result_type: *const mlir.Type) Atom { + const result = self.emit(cute.nvgpu.atom_make_exec_tma(self.ctx, descriptor.inner, result_type, null, null, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Attach the mbarrier that an executable TMA load atom completes. + pub fn setTmaBarrier(self: *Builder, atom: Atom, barrier: Value) Atom { + const result = self.emit(cute.nvgpu.atom_set_value( + self.ctx, + atom.inner, + barrier.inner, + atom.type_(), + self.parseAttribute("#cute_nvgpu.atom_copy_field_tmaload"), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn tmaPartition( + self: *Builder, + descriptor: Atom, + cta_coord: anytype, + cta_layout: anytype, + smem: anytype, + targets: anytype, + smem_type: []const u8, + target_types: []const []const u8, + ) TmaPartition { + if (targets.len != target_types.len) @panic("tmaPartition: target count does not match result type count"); + const result_types = self.arena.allocator().alloc(*const mlir.Type, target_types.len) catch @panic("OOM"); + const views = self.arena.allocator().alloc(View, target_types.len) catch @panic("OOM"); + for (target_types, result_types) |type_text, *ty| ty.* = self.parseType(type_text); + const op = cute.nvgpu.atom_tma_partition( + self.ctx, + descriptor.inner, + self.makeCoord(cta_coord), + self.asValue(cta_layout).inner, + self.asValue(smem).inner, + self.innerSlice(targets), + self.parseType(smem_type), + result_types, + self.loc(), + ); + _ = op.appendTo(self.currentBlock()); + for (views, 0..) |*view, i| view.* = .{ .inner = op.result(i + 1), .kernel = self }; + return .{ .smem = .{ .inner = op.result(0), .kernel = self }, .targets = views }; + } + + /// Typed form of `tmaPartition`. The dependent result types are built by + /// the Zig CuTe DSL instead of being round-tripped through textual MLIR. + pub fn tmaPartitionTyped( + self: *Builder, + descriptor: Atom, + cta_coord: anytype, + cta_layout: anytype, + smem: anytype, + targets: anytype, + smem_type: *const mlir.Type, + target_types: []const *const mlir.Type, + ) TmaPartition { + if (targets.len != target_types.len) @panic("tmaPartitionTyped: target count does not match result type count"); + const views = self.arena.allocator().alloc(View, target_types.len) catch @panic("OOM"); + const op = cute.nvgpu.atom_tma_partition( + self.ctx, + descriptor.inner, + self.makeCoord(cta_coord), + self.asValue(cta_layout).inner, + self.asValue(smem).inner, + self.innerSlice(targets), + smem_type, + target_types, + self.loc(), + ); + _ = op.appendTo(self.currentBlock()); + for (views, 0..) |*view, i| view.* = .{ .inner = op.result(i + 1), .kernel = self }; + return .{ .smem = .{ .inner = op.result(0), .kernel = self }, .targets = views }; + } + + // ==================== cute.nvgpu.tcgen05 (SM100) ==================== + + /// Tiled MMA type of the SM100 block-scaled MXFP4 x MXFP8 atom. + pub fn blockScaledMmaType(self: *Builder, cfg: BlockScaledMmaConfig) *const mlir.Type { + return self.parseType(std.fmt.allocPrint( + self.arena.allocator(), + "!cute.tiled_mma, permutation_MNK = <\"[_;_;_]\">>", + .{blockScaledPayload(self, cfg)}, + ) catch @panic("OOM")); + } + + /// SS, K-major MXFP4 x MXFP8 block-scaled atom and its tiled MMA, as the + /// CuTe DSL builds them. Both scale-factor pointers start at `scale_ptr` + /// and are patched after tensor-memory allocation. + pub fn makeBlockScaledMma(self: *Builder, cfg: BlockScaledMmaConfig, scale_ptr: Value) BlockScaledMma { + if (cfg.m != 128 or cfg.k != 32 or cfg.n < 8 or cfg.n > 256 or cfg.n % 8 != 0) { + std.debug.panic("unsupported SM100 block-scaled MMA shape {d}x{d}x{d}", .{ cfg.m, cfg.n, cfg.k }); + } + const disabled = self.cst(.i1, false); + const atom_type = std.fmt.allocPrint(self.arena.allocator(), "!cute_nvgpu.sm100.mma_bs{s}", .{blockScaledPayload(self, cfg)}) catch @panic("OOM"); + const atom = self.makeAtom(atom_type, .{ disabled, disabled, disabled, scale_ptr, scale_ptr }); + const tiled = self.emit(cute.make_tiled_mma(self.ctx, atom.inner, self.blockScaledMmaType(cfg), self.loc())); + return .{ .atom = atom, .tiled = .{ .inner = tiled.inner, .kernel = self } }; + } + + /// Enable (or disable) accumulation into C for the next block-scaled MMA. + pub fn setMmaAccumulate(self: *Builder, mma: Atom, enabled: Value) Atom { + const result = self.emit(cute.nvgpu.atom_set_value( + self.ctx, + mma.inner, + enabled.inner, + mma.type_(), + self.parseAttribute("#cute_nvgpu.atom_mma_field_sm100_block_scaled"), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Commit the preceding tcgen05 MMA operations to an mbarrier. One elected + /// lane of the MMA warp issues this after each K-stage. + pub fn tcgen05Commit(self: *Builder, barrier: Value) void { + self.emitVoid(mlir.Operation.make(self.ctx, "nvvm.tcgen05.commit", .{ + .operands = .{ .flat = &.{self.sharedBarrierPtr(barrier).inner} }, + .attributes = &.{.named(self.ctx, "group", self.parseAttribute("#nvvm.cta_group"))}, + .location = self.loc(), + })); + } + + /// Wait until asynchronous tensor-memory loads have reached registers. + pub fn fenceTmemLoad(self: *Builder) void { + self.emitVoid(mlir.Operation.make(self.ctx, "nvvm.tcgen05.wait", .{ + .attributes = &.{.named(self.ctx, "kind", self.parseAttribute("#nvvm.tcgen05_wait"))}, + .location = self.loc(), + })); + } + + pub const TmemOptions = struct { two_cta: bool = false, exclusive: bool = false }; + + pub fn allocTmem(self: *Builder, columns: u16, holding_ptr: anytype, opts: TmemOptions) void { + self.emitVoid(cute.nvgpu.arch_sm100_alloc_tmem( + self.ctx, + self.cst(.i32, columns).inner, + self.asValue(holding_ptr).inner, + if (opts.two_cta) .unit(self.ctx) else null, + if (opts.exclusive) .unit(self.ctx) else null, + self.loc(), + )); + } + + pub fn retrieveTmemPtr(self: *Builder, holding_ptr: anytype, dtype: DType, alignment: u64) Value { + const ptr_type = self.ptrTy(dtype, .tmem, alignment) catch @panic("invalid tensor-memory pointer type"); + return self.emit(cute.nvgpu.arch_sm100_retrieve_tmem_ptr(self.ctx, self.asValue(holding_ptr).inner, ptr_type, self.loc())); + } + + pub fn relinquishTmemAllocPermit(self: *Builder, opts: TmemOptions) void { + self.emitVoid(cute.nvgpu.arch_sm100_relinquish_tmem_alloc_permit(self.ctx, if (opts.two_cta) .unit(self.ctx) else null, self.loc())); + } + + pub fn deallocTmem(self: *Builder, pointer: anytype, columns: u16, opts: TmemOptions) void { + self.emitVoid(cute.nvgpu.arch_sm100_dealloc_tmem( + self.ctx, + self.asValue(pointer).inner, + self.cst(.i32, columns).inner, + if (opts.two_cta) .unit(self.ctx) else null, + if (opts.exclusive) .unit(self.ctx) else null, + self.loc(), + )); + } + + pub const TmemLoadOptions = struct { + num_dp: u16, + num_b: u16, + num_rep: u16, + pack_16: bool = false, + }; + + pub fn tmemLoad(self: *Builder, src: anytype, result_type: []const u8, opts: TmemLoadOptions) View { + const result = self.emit(cute.nvgpu.arch_copy_SM100_tmem_load( + self.ctx, + self.asValue(src).inner, + self.parseType(result_type), + .int(self.ctx, .i32, opts.num_dp), + .int(self.ctx, .i32, opts.num_b), + .int(self.ctx, .i32, opts.num_rep), + if (opts.pack_16) .boolean(self.ctx, true) else null, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Direct tensor-memory load into an SSA vector. Higher-level tiled-copy + /// helpers remain preferable when their dependent layouts are available. + pub fn tmemLoadVector(self: *Builder, src: anytype, lanes: i64, opts: TmemLoadOptions) Value { + // The NVVM-facing operation transports raw 32-bit register words. + // Callers reinterpret each lane as the accumulator element type. + const result_type = mlir.Type.vector(&.{lanes}, DType.i32.toMlir(self.ctx)); + return self.emit(cute.nvgpu.arch_copy_SM100_tmem_load( + self.ctx, + self.asValue(src).inner, + result_type, + .int(self.ctx, .i32, opts.num_dp), + .int(self.ctx, .i32, opts.num_b), + .int(self.ctx, .i32, opts.num_rep), + if (opts.pack_16) .boolean(self.ctx, true) else null, + self.loc(), + )); + } + + pub fn makeS2tCopy(self: *Builder, atom: Atom, tmem: anytype, result_type: []const u8) Atom { + const result = self.emit(cute.nvgpu.atom_make_s2t_copy( + self.ctx, + atom.inner, + self.asValue(tmem).inner, + self.parseType(result_type), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn makeTmemCopy(self: *Builder, atom: Atom, tmem: anytype, result_type: []const u8) Atom { + const result = self.emit(cute.nvgpu.atom_make_tmem_copy( + self.ctx, + atom.inner, + self.asValue(tmem).inner, + self.parseType(result_type), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn s2tSmemDescriptor(self: *Builder, atom: Atom, input: anytype, result_type: []const u8) View { + const result = self.emit(cute.nvgpu.atom_get_copy_s2t_smem_desc_view( + self.ctx, + atom.inner, + self.asValue(input).inner, + self.parseType(result_type), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Raw SM100 TMA load. The descriptor address, destinations and + /// coordinates come from the executable atom and partition operations. + pub fn tmaLoad( + self: *Builder, + descriptor_addr: Value, + data_addr: Value, + barrier_addr: Value, + coords: []const Value, + multicast_mask: ?Value, + offsets: []const Value, + cache_policy: ?Value, + mode: *const mlir.Attribute, + num_cta: u8, + ) void { + self.emitVoid(cute.nvgpu.arch_copy_SM100_tma_load( + self.ctx, + descriptor_addr.inner, + data_addr.inner, + barrier_addr.inner, + self.innerSlice(coords), + if (multicast_mask) |v| v.inner else null, + self.innerSlice(offsets), + if (cache_policy) |v| v.inner else null, + mode, + .int(self.ctx, .i32, num_cta), + self.loc(), + )); + } + + /// Raw SM100 TMA store. + pub fn tmaStore( + self: *Builder, + descriptor_addr: Value, + data_addr: Value, + coords: []const Value, + cache_policy: ?Value, + mode: *const mlir.Attribute, + ) void { + self.emitVoid(cute.nvgpu.arch_copy_SM100_tma_store( + self.ctx, + descriptor_addr.inner, + data_addr.inner, + self.innerSlice(coords), + if (cache_policy) |v| v.inner else null, + mode, + self.loc(), + )); + } + // ==================== cute.arch ==================== fn sreg(self: *Builder, comptime reg: []const u8) Index3 { @@ -599,10 +2409,204 @@ pub const Builder = struct { return self.sreg("nctaid"); } + /// Mark a scalar as identical in every lane of its warp. CuTe uses this + /// fact when lowering role branches that contain warp-collective TMA and + /// tensor-memory operations. + pub fn makeWarpUniform(self: *Builder, value: anytype) Value { + const scalar = self.asValue(value); + // The CuTe Python frontend legalizes predicates through an i32: + // extui i1 -> i32; make_warp_uniform i32; cmpi ne 0 + // `arch.make_warp_uniform` on i1 parses, but the native SM100 + // pipeline does not lower it all the way to NVVM. + const uniform_input = if (scalar.dtype() == .i1) scalar.to(.i32) else scalar; + const uniform = self.emit(cute.nvgpu.arch_make_warp_uniform( + self.ctx, + uniform_input.inner, + uniform_input.type_(), + self.loc(), + )); + return if (scalar.dtype() == .i1) uniform.ne(0) else uniform; + } + /// `cute.arch.sync_threads()`. pub fn syncThreads(self: *Builder) void { _ = mlir.Operation.make(self.ctx, "nvvm.barrier", .{ - .attributes = &.{.named(self.ctx, "operandSegmentSizes", .denseArray(self.ctx, .i32, &.{ 0, 0 }))}, + .attributes = &.{.named(self.ctx, "operandSegmentSizes", .denseArray(self.ctx, .i32, &.{ 0, 0, 0 }))}, + .location = self.loc(), + }).appendTo(self.currentBlock()); + } + + /// Synchronize a fixed subset of the CTA on a named hardware barrier. + /// `participants` must be a warp multiple and every participating thread + /// must execute the same barrier id. + pub fn namedBarrier(self: *Builder, id: i32, participants: i32) void { + // The embedded CuTe compiler currently reads NVVM through a different + // MLIR revision. A generic `nvvm.barrier` with optional operands is + // accepted, but those operands are discarded and it lowers to + // `bar.sync 0`. Constant inline PTX preserves the named barrier ABI + // used by CUTLASS (`2,160` for TMEM allocation and `1,128` for the + // epilogue pipeline). + const assembly = std.fmt.allocPrint( + self.arena.allocator(), + "bar.sync {d}, {d};", + .{ id, participants }, + ) catch @panic("namedBarrier OOM"); + _ = mlir.Operation.make(self.ctx, "llvm.inline_asm", .{ + .attributes = &.{ + .named(self.ctx, "asm_string", .string(self.ctx, assembly)), + .named(self.ctx, "constraints", .string(self.ctx, "")), + .named(self.ctx, "has_side_effects", .unit(self.ctx)), + }, + .location = self.loc(), + }).appendTo(self.currentBlock()); + } + + /// Build the CuTe first-class divisor used by CUTLASS's persistent tile + /// scheduler. Its arithmetic is lowered to the same multiply/shift + /// sequence as Python CuTe DSL rather than an integer div/rem pair. + pub fn fastDivmodCreate(self: *Builder, divisor_: anytype) Value { + const divisor = self.lift(divisor_).to(.i32); + return self.emit(cute.fast_divmod_create_divisor( + self.ctx, + divisor.inner, + self.parseType("!cute.fast_divmod_divisor<32>"), + self.loc(), + )); + } + + /// Return `{ quotient, remainder }` for a CUTLASS FastDivmod divisor. + pub fn fastDivmod(self: *Builder, dividend_: anytype, divisor: Value) [2]Value { + const dividend = self.lift(dividend_).to(.i32); + return self.emitResults(2, cute.fast_divmod_compute( + self.ctx, + dividend.inner, + divisor.inner, + DType.i32.toMlir(self.ctx), + DType.i32.toMlir(self.ctx), + self.loc(), + )); + } + + /// Elect one lane from the active warp. Used by TMA issue, tensor-memory + /// allocation, and tcgen05 commit operations. + pub fn electSync(self: *Builder) Value { + return self.emit(mlir.Operation.make(self.ctx, "nvvm.elect.sync", .{ + .results = .{ .flat = &.{.int(self.ctx, .i1)} }, + .location = self.loc(), + })); + } + + /// `cute.arch.shuffle_sync_bfly`: exchange a scalar with the lane at + /// `lane_id ^ offset` in the current warp. + /// Warp-wide maximum of a non-negative float, as one `redux.sync` instead + /// of a shuffle reduction: IEEE bit patterns of non-negative floats order + /// like unsigned integers. + pub fn warpMaxNonNegative(self: *Builder, value: Value) Value { + const dtype = value.dtype(); + if (dtypeBitwidth(dtype) != 32) @panic("warpMaxNonNegative requires a 32-bit scalar"); + const bits = if (dtype == .i32) value else value.bitCast(.i32); + const reduced = self.emit(mlir.Operation.make(self.ctx, "llvm.inline_asm", .{ + .operands = .{ .flat = &.{bits.inner} }, + .results = .{ .flat = &.{.int(self.ctx, .i32)} }, + .attributes = &.{ + .named(self.ctx, "asm_string", .string(self.ctx, "redux.sync.max.u32 $0, $1, 0xffffffff;")), + .named(self.ctx, "constraints", .string(self.ctx, "=r,r")), + }, + .location = self.loc(), + })); + return if (dtype == .i32) reduced else reduced.bitCast(dtype); + } + + pub fn shuffleXor(self: *Builder, value: Value, offset: i32) Value { + const dtype = value.dtype(); + if (dtypeBitwidth(dtype) != 32) @panic("shuffleXor currently requires a 32-bit scalar"); + const bits = if (dtype == .i32) value else value.bitCast(.i32); + const shuffled = self.emit(mlir.Operation.make(self.ctx, "llvm.inline_asm", .{ + .operands = .{ .flat = &.{ + bits.inner, + self.cst(.i32, offset).inner, + } }, + .results = .{ .flat = &.{.int(self.ctx, .i32)} }, + .attributes = &.{ + .named(self.ctx, "asm_string", .string(self.ctx, "shfl.sync.bfly.b32 $0, $1, $2, 31, 0xffffffff;")), + .named(self.ctx, "constraints", .string(self.ctx, "=r,r,r")), + }, + .location = self.loc(), + })); + return if (dtype == .i32) shuffled else shuffled.bitCast(dtype); + } + + /// `cute.arch.shuffle_sync`: read a scalar from lane `lane` of the + /// current warp. Every lane must participate. + pub fn shuffleIdx(self: *Builder, value: Value, lane: anytype) Value { + const dtype = value.dtype(); + if (dtypeBitwidth(dtype) != 32) @panic("shuffleIdx currently requires a 32-bit scalar"); + const bits = if (dtype == .i32) value else value.bitCast(.i32); + const shuffled = self.emit(mlir.Operation.make(self.ctx, "llvm.inline_asm", .{ + .operands = .{ .flat = &.{ + bits.inner, + self.liftAs(lane, .i32).inner, + } }, + .results = .{ .flat = &.{.int(self.ctx, .i32)} }, + .attributes = &.{ + .named(self.ctx, "asm_string", .string(self.ctx, "shfl.sync.idx.b32 $0, $1, $2, 31, 0xffffffff;")), + .named(self.ctx, "constraints", .string(self.ctx, "=r,r,r")), + }, + .location = self.loc(), + })); + return if (dtype == .i32) shuffled else shuffled.bitCast(dtype); + } + + /// Four FP32 values rounded to FP8 E4M3 (saturating) and packed into one + /// 32-bit word, `a` in the lowest byte. + pub fn packFp8x4(self: *Builder, a: Value, b: Value, c: Value, d: Value) Value { + return self.emit(mlir.Operation.make(self.ctx, "llvm.inline_asm", .{ + .operands = .{ .flat = &.{ a.to(.f32).inner, b.to(.f32).inner, c.to(.f32).inner, d.to(.f32).inner } }, + .results = .{ .flat = &.{.int(self.ctx, .i32)} }, + .attributes = &.{ + .named(self.ctx, "asm_string", .string(self.ctx, "{ .reg .b16 lo, hi; cvt.rn.satfinite.e4m3x2.f32 lo, $2, $1; cvt.rn.satfinite.e4m3x2.f32 hi, $4, $3; mov.b32 $0, {lo, hi}; }")), + .named(self.ctx, "constraints", .string(self.ctx, "=r,f,f,f,f")), + }, + .location = self.loc(), + })); + } + + /// One-operand FP32 PTX instruction, e.g. `ex2.approx.ftz.f32`. + pub fn unaryF32(self: *Builder, comptime instruction: []const u8, value: Value) Value { + return self.emit(mlir.Operation.make(self.ctx, "llvm.inline_asm", .{ + .operands = .{ .flat = &.{value.to(.f32).inner} }, + .results = .{ .flat = &.{DType.f32.toMlir(self.ctx)} }, + .attributes = &.{ + .named(self.ctx, "asm_string", .string(self.ctx, instruction ++ " $0, $1;")), + .named(self.ctx, "constraints", .string(self.ctx, "=f,f")), + }, + .location = self.loc(), + })); + } + + /// Signal programmatic dependent launches after this grid has consumed + /// its inputs. The caller must also launch the producer/consumer chain + /// with PDL enabled. + pub fn launchDependents(self: *Builder) void { + _ = mlir.Operation.make(self.ctx, "llvm.inline_asm", .{ + .attributes = &.{ + .named(self.ctx, "asm_string", .string(self.ctx, "griddepcontrol.launch_dependents;")), + .named(self.ctx, "constraints", .string(self.ctx, "")), + .named(self.ctx, "has_side_effects", .unit(self.ctx)), + }, + .location = self.loc(), + }).appendTo(self.currentBlock()); + } + + /// Wait until the preceding grid has made its programmatic dependency + /// available. This is the consumer-side half of a PDL launch chain. + pub fn waitForDependency(self: *Builder) void { + _ = mlir.Operation.make(self.ctx, "llvm.inline_asm", .{ + .attributes = &.{ + .named(self.ctx, "asm_string", .string(self.ctx, "griddepcontrol.wait;")), + .named(self.ctx, "constraints", .string(self.ctx, "")), + .named(self.ctx, "has_side_effects", .unit(self.ctx)), + }, .location = self.loc(), }).appendTo(self.currentBlock()); } @@ -615,6 +2619,124 @@ pub const Builder = struct { return self.emit(cute.nvgpu.arch_alloc_smem(self.ctx, ty, .int(self.ctx, .i32, size_in_elems), self.loc())); } + // mbarriers, async-proxy fences and bulk-copy groups. + + /// mbarrier operands are `!llvm.ptr<3>` for NVVM. + fn sharedBarrierPtr(self: *Builder, barrier: Value) Value { + return self.emit(mlir.Operation.make(self.ctx, "builtin.unrealized_conversion_cast", .{ + .operands = .{ .flat = &.{barrier.inner} }, + .results = .{ .flat = &.{self.parseType("!llvm.ptr<3>")} }, + .location = self.loc(), + })); + } + + /// Initialize a shared-memory mbarrier expecting `arrivals` per phase. + pub fn mbarrierInit(self: *Builder, barrier: Value, arrivals: u32) void { + self.emitVoid(mlir.Operation.make(self.ctx, "nvvm.mbarrier.init", .{ + .operands = .{ .flat = &.{ self.sharedBarrierPtr(barrier).inner, self.cst(.i32, arrivals).inner } }, + .attributes = &.{.named(self.ctx, "layout", self.parseAttribute("#nvvm.mbarrier_layout"))}, + .location = self.loc(), + })); + } + + /// Arrive and expect `bytes` of asynchronous transactions (SM90+), as a + /// TMA producer does before issuing its copies. + pub fn mbarrierArriveExpectTx(self: *Builder, barrier: Value, bytes: u32) void { + self.emitVoid(mlir.Operation.make(self.ctx, "nvvm.mbarrier.arrive.expect_tx", .{ + .operands = .{ .flat = &.{ self.sharedBarrierPtr(barrier).inner, self.cst(.i32, bytes).inner } }, + .attributes = &.{ + .named(self.ctx, "operandSegmentSizes", .denseArray(self.ctx, .i32, &.{ 1, 1, 0, 0 })), + .named(self.ctx, "relaxed", .boolean(self.ctx, false)), + .named(self.ctx, "scope", self.parseAttribute("#nvvm.mem_scope")), + }, + .location = self.loc(), + })); + } + + pub fn mbarrierArrive(self: *Builder, barrier: Value, count: u32) void { + self.emitVoid(mlir.Operation.make(self.ctx, "nvvm.mbarrier.arrive", .{ + .operands = .{ .flat = &.{ self.sharedBarrierPtr(barrier).inner, self.cst(.i32, count).inner } }, + .attributes = &.{ + .named(self.ctx, "operandSegmentSizes", .denseArray(self.ctx, .i32, &.{ 1, 1, 0 })), + .named(self.ctx, "relaxed", .boolean(self.ctx, false)), + .named(self.ctx, "scope", self.parseAttribute("#nvvm.mem_scope")), + }, + .location = self.loc(), + })); + } + + /// Test whether phase `phase` has completed, without suspending (SM90+). + pub fn mbarrierWaitParity(self: *Builder, barrier: Value, phase: Value) Value { + return self.emit(mlir.Operation.make(self.ctx, "nvvm.mbarrier.wait.parity", .{ + .operands = .{ .flat = &.{ self.sharedBarrierPtr(barrier).inner, phase.inner } }, + .results = .{ .flat = &.{.int(self.ctx, .i1)} }, + .attributes = &.{ + .named(self.ctx, "kind", self.parseAttribute("#nvvm.mbar_wait")), + .named(self.ctx, "scope", self.parseAttribute("#nvvm.mbar_scope")), + }, + .location = self.loc(), + })); + } + + /// Wait for phase `phase`, letting the warp sleep for up to + /// `suspend_time` cycles per attempt (SM90+). + pub fn mbarrierTryWaitParity(self: *Builder, barrier: Value, phase: Value, suspend_time: u32) void { + self.emitVoid(mlir.Operation.make(self.ctx, "nvvm.mbarrier.try_wait.parity", .{ + .operands = .{ .flat = &.{ self.sharedBarrierPtr(barrier).inner, phase.inner, self.cst(.i32, suspend_time).inner } }, + .attributes = &.{.named(self.ctx, "useIntrinsic", .boolean(self.ctx, false))}, + .location = self.loc(), + })); + } + + /// Wait for a pipeline phase with the two-step sequence of CuTe's + /// `PipelineTmaUmma`: test the barrier without suspending, then suspend + /// only when the phase is still outstanding. The initial test matters when + /// a producer reaches a full ring and must yield to the consumer that + /// recycles its next stage. + pub fn mbarrierWait(self: *Builder, barrier: Value, phase: Value) void { + const ready = self.mbarrierWaitParity(barrier, phase); + var pending = self.openIf(ready.eq(false)); + self.mbarrierTryWaitParity(barrier, phase, 10_000_000); + pending.yieldThen(.{}); + } + + /// Make mbarrier initialization visible to the async proxy (SM90+). + pub fn fenceMbarrierInit(self: *Builder) void { + self.emitVoid(mlir.Operation.make(self.ctx, "nvvm.fence.mbarrier.init", .{ .location = self.loc() })); + } + + /// Publish ordinary shared-memory writes to the asynchronous (TMA) proxy (SM90+). + pub fn fenceProxyAsyncShared(self: *Builder) void { + self.emitVoid(mlir.Operation.make(self.ctx, "nvvm.fence.proxy", .{ + .attributes = &.{ + .named(self.ctx, "kind", self.parseAttribute("#nvvm.proxy_kind")), + .named(self.ctx, "space", self.parseAttribute("#nvvm.shared_space")), + }, + .location = self.loc(), + })); + } + + /// Close the current group of bulk asynchronous copies, e.g. TMA stores (SM90+). + pub fn cpAsyncBulkCommitGroup(self: *Builder) void { + self.emitVoid(mlir.Operation.make(self.ctx, "nvvm.cp.async.bulk.commit.group", .{ .location = self.loc() })); + } + + /// Wait until at most `pending` bulk copy groups are in flight; with + /// `read`, only until their sources have been read (SM90+). + pub fn cpAsyncBulkWaitGroup(self: *Builder, pending: u32, read: bool) void { + var attrs: [2]mlir.NamedAttribute = undefined; + var len: usize = 1; + attrs[0] = .named(self.ctx, "group", .int(self.ctx, .i32, pending)); + if (read) { + attrs[1] = .named(self.ctx, "read", .unit(self.ctx)); + len += 1; + } + self.emitVoid(mlir.Operation.make(self.ctx, "nvvm.cp.async.bulk.wait_group", .{ + .attributes = attrs[0..len], + .location = self.loc(), + })); + } + // ==================== scalars ==================== pub fn cst(self: *Builder, dt: DType, value: anytype) Value { @@ -624,11 +2746,12 @@ pub const Builder = struct { const f: f64 = switch (@typeInfo(T)) { .comptime_int, .comptime_float => value, .int => @floatFromInt(value), + .bool => @floatFromInt(@intFromBool(value)), .float => @floatCast(value), else => @compileError("Builder.cst: unsupported value type " ++ @typeName(T)), }; return self.emit(switch (dt) { - inline .f16, .bf16, .f32, .f64, .f8e4m3fn, .f8e5m2 => |ft| arith.constant_float(ctx, f, @field(mlir.FloatTypes, @tagName(ft)), self.loc()), + inline .f16, .bf16, .f32, .f64, .f4e2m1fn, .f8e4m3fn, .f8e5m2, .f8e8m0fnu => |ft| arith.constant_float(ctx, f, @field(mlir.FloatTypes, @tagName(ft)), self.loc()), else => unreachable, }); } @@ -643,6 +2766,7 @@ pub const Builder = struct { } fn fitsInt(i: i64, dt: DType) bool { + if (dt == .i1) return i == 0 or i == 1; const bits = dtypeBitwidth(dt); if (bits >= 64) return true; const shift: u6 = @intCast(bits - 1); @@ -764,6 +2888,53 @@ pub const Builder = struct { }; } + /// `scf.while` with explicit loop-carried values. The before region sees + /// `inits`, calls `yieldBefore(condition, forwarded)`, then the after + /// region sees `forwarded` and calls `yieldAfter(next_inits)`. This is the + /// control-flow shape emitted by the Python CuTe persistent scheduler. + pub fn openWhile( + self: *Builder, + inits: anytype, + after_types: anytype, + ) WhileScope( + tupleArity(@TypeOf(inits), "openWhile: inits"), + tupleArity(@TypeOf(after_types), "openWhile: after_types"), + ) { + const N = comptime tupleArity(@TypeOf(inits), "openWhile: inits"); + const M = comptime tupleArity(@TypeOf(after_types), "openWhile: after_types"); + + var init_types: [N]*const mlir.Type = undefined; + var init_locs: [N]*const mlir.Location = undefined; + var inits_inner: [N]*const mlir.Value = undefined; + inline for (@typeInfo(@TypeOf(inits)).@"struct".fields, 0..) |f, i| { + const v = self.lift(@field(inits, f.name)); + init_types[i] = v.type_(); + init_locs[i] = self.loc(); + inits_inner[i] = v.inner; + } + + var result_types: [M]*const mlir.Type = undefined; + var result_locs: [M]*const mlir.Location = undefined; + inline for (@typeInfo(@TypeOf(after_types)).@"struct".fields, 0..) |f, i| { + result_types[i] = @field(after_types, f.name); + result_locs[i] = self.loc(); + } + + const before = mlir.Block.init(&init_types, &init_locs); + const after = mlir.Block.init(&result_types, &result_locs); + self.pushBlock(before); + var before_carried: [N]Value = undefined; + for (0..N) |i| before_carried[i] = .{ .inner = before.argument(i), .kernel = self }; + return .{ + .kernel = self, + .before_block = before, + .after_block = after, + .inits_inner = inits_inner, + .after_types = result_types, + .before_carried = before_carried, + }; + } + // ==================== module ==================== /// The module the CuTe compiler takes: the verified kernel as a public @@ -779,10 +2950,7 @@ pub const Builder = struct { func_op.setAttributeByName("nvvm.reqntid", .denseArray(self.ctx, .i32, &block)); if (!self.module.operation().verify()) return error.InvalidMlir; - var al: std.Io.Writer.Allocating = .init(self.allocator); - defer al.deinit(); - try al.writer.print("{f}", .{self.module.operation()}); - return try self.allocator.dupeZ(u8, al.written()); + return self.renderModule(false); } }; @@ -859,6 +3027,9 @@ test "guard, shared memory, sync, layouts, for loop" { const i = b.blockIdx().x.mul(128).add(tid); smem.set(.{tid}, gx.get(.{i})); b.syncThreads(); + b.namedBarrier(2, 128); + b.launchDependents(); + b.waitForDependency(); const j = b.cst(.i32, 127).sub(tid); var guard = b.openIf(i.lt(4096)); var acc = b.openFor(0, 4, 1, .{b.cst(.f32, 0.0)}); @@ -870,7 +3041,10 @@ test "guard, shared memory, sync, layouts, for loop" { defer std.testing.allocator.free(kernel); try expectContains(kernel, "cute_nvgpu.arch.alloc_smem"); try expectContains(kernel, "!cute.ptr>"); - try expectContains(kernel, "\"nvvm.barrier\"() {operandSegmentSizes = array} : () -> ()"); + try expectContains(kernel, "\"nvvm.barrier\"() {operandSegmentSizes = array} : () -> ()"); + try expectContains(kernel, "bar.sync 2, 128;"); + try expectContains(kernel, "griddepcontrol.launch_dependents;"); + try expectContains(kernel, "griddepcontrol.wait;"); try expectContains(kernel, "scf.if"); try expectContains(kernel, "scf.for"); try expectContains(kernel, "!cute.layout<\"4096:1\">"); @@ -917,6 +3091,7 @@ test "casts, select, if-else, integer widening" { try std.testing.expectEqual(DType.i1, bound.dtype()); const f = tid.to(.f32); + const shuffled = b.shuffleXor(f, 16); try std.testing.expectEqual(DType.f16, f.to(.f16).dtype()); try std.testing.expectEqual(DType.f64, f.to(.f64).dtype()); try std.testing.expectEqual(DType.bf16, f.to(.f16).to(.bf16).dtype()); @@ -924,7 +3099,7 @@ test "casts, select, if-else, integer widening" { try std.testing.expectEqual(DType.i8, tid.to(.i8).dtype()); try std.testing.expectEqual(DType.i32, bound.to(.i32).dtype()); - const picked = b.select(bound, f, 2.0); + const picked = b.select(bound, shuffled, 2.0); var scope = b.openIfElse(tid.lt(4), .{DType.f16.toMlir(ctx)}); scope.yieldThen(.{picked.to(.f16)}); scope.yieldElse(.{b.cst(.f16, 1)}); @@ -942,8 +3117,80 @@ test "casts, select, if-else, integer widening" { try expectContains(kernel, "arith.trunci"); try expectContains(kernel, "arith.extui"); try expectContains(kernel, "arith.select"); + try expectContains(kernel, "shfl.sync.bfly.b32"); try expectContains(kernel, "} else {"); const parsed = try mlir.Module.parse(ctx, kernel); defer parsed.deinit(); try std.testing.expect(parsed.operation().verify()); } + +test "complete program emits a Zig-built CUDA host launch" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.openProgram(std.testing.allocator, ctx, "program_add"); + defer b.deinit(); + const device = try b.declareArgs(.{ + .input = .{ .tensor = .{ .dtype = .f32, .shape = &.{128} } }, + .output = .{ .tensor = .{ .dtype = .f32, .shape = &.{128} } }, + }); + const tid = b.threadIdx().x; + device.output.set(.{tid}, device.input.get(.{tid}).add(1.0)); + b.endFunction(.{ 128, 1, 1 }); + + b.beginFunction("program_add", .host); + _ = try b.declareArgs(.{ + .input = .{ .ptr = DType.f32 }, + .output = .{ .ptr = DType.f32 }, + }); + const one = b.cst(.i32, 1); + const config = b.makeLaunchConfig(.{ + .grid = .{ one, one, one }, + .block = .{ b.cst(.i32, 128), one, one }, + .dynamic_smem = b.kernelSmemSize("program_add"), + .stream = b.cudaStream(), + .use_pdl = true, + }); + const launched = b.launchEx("program_add", config, .{ b.arg(0), b.arg(1) }); + b.returnHostStatus(b.cudaResultStatus(launched)); + b.endFunction(null); + + const ir = try b.finishProgram(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "\"gpu.module\""); + try expectContains(ir, "cuda.kernel"); + try expectContains(ir, "sym_name = \"program_add\""); + try expectContains(ir, "cuda.launch_ex"); +} + +test "SM100 block-scaled MMA atoms use the CuTe DSL instruction shapes" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + inline for (dialects_needed) |dialect| mlir.DialectHandle.fromString(dialect).insertDialect(registry); + mlir.registerFuncExtensions(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + + var b = try Builder.open(std.testing.allocator, ctx, "sm100_blockscaled_atoms"); + defer b.deinit(); + _ = try b.declareArgs(.{ .descriptor = .{ .raw = .{ + .type_text = "!cute_nvgpu.atom.non_exec_tiled_tma_load, tma_format = U8>", + .grid_constant = true, + } } }); + + const sf_ptr = b.intToPtr(0, .f8e8m0fnu, .tmem, 1); + _ = b.makeBlockScaledMma(.{ .n = 16 }, sf_ptr); + _ = b.makeBlockScaledMma(.{ .n = 128 }, sf_ptr); + + const ir = try b.finish(.{ 192, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "sm100.mma_bs<128x16x32") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "sm100.mma_bs<128x128x32") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cute.make_tiled_mma") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cute_nvgpu.grid_constant") != null); + + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} diff --git a/kernels/cute/dtype.zig b/kernels/cute/dtype.zig index 89916fcf..6dc9cc14 100644 --- a/kernels/cute/dtype.zig +++ b/kernels/cute/dtype.zig @@ -12,8 +12,10 @@ pub const DType = enum { bf16, f32, f64, + f4e2m1fn, f8e4m3fn, f8e5m2, + f8e8m0fnu, pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { return switch (self) { @@ -26,15 +28,17 @@ pub const DType = enum { .bf16 => .float(ctx, .bf16), .f32 => .float(ctx, .f32), .f64 => .float(ctx, .f64), + .f4e2m1fn => .float(ctx, .f4e2m1fn), .f8e4m3fn => .float(ctx, .f8e4m3fn), .f8e5m2 => .float(ctx, .f8e5m2), + .f8e8m0fnu => .float(ctx, .f8e8m0fnu), }; } }; pub fn isFloatDtype(dt: DType) bool { return switch (dt) { - .f16, .bf16, .f32, .f64, .f8e4m3fn, .f8e5m2 => true, + .f16, .bf16, .f32, .f64, .f4e2m1fn, .f8e4m3fn, .f8e5m2, .f8e8m0fnu => true, else => false, }; } @@ -42,7 +46,8 @@ pub fn isFloatDtype(dt: DType) bool { pub fn dtypeBitwidth(dt: DType) usize { return switch (dt) { .i1 => 1, - .i8, .f8e4m3fn, .f8e5m2 => 8, + .f4e2m1fn => 4, + .i8, .f8e4m3fn, .f8e5m2, .f8e8m0fnu => 8, .i16, .f16, .bf16 => 16, .i32, .f32 => 32, .i64, .f64 => 64, diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/BUILD.bazel index db4078ac..3b7bf6ce 100644 --- a/mlir/dialects/BUILD.bazel +++ b/mlir/dialects/BUILD.bazel @@ -6,6 +6,7 @@ zig_library( "affine.zig", "arith.zig", "cf.zig", + "cuda.zig", "func.zig", "gpu.zig", "math.zig", diff --git a/mlir/dialects/cuda.zig b/mlir/dialects/cuda.zig new file mode 100644 index 00000000..1f75d39b --- /dev/null +++ b/mlir/dialects/cuda.zig @@ -0,0 +1,141 @@ +//! Builders for the NVIDIA CUDA host-program dialect used by CuTe DSL. +//! +//! The dialect itself is registered by NVIDIA's runtime compiler. ZML builds +//! these operations in an allow-unregistered context, so constructors mirror +//! the ODS schemas from NVIDIA CUTLASS DSL's generated `_cuda_ops_gen.py`. +const std = @import("std"); + +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub const dialect_namespace = "cuda"; + +pub fn streamType(ctx: *mlir.Context) *const mlir.Type { + return mlir.Type.parse(ctx, "!cuda.stream") catch @panic("CUDA stream type is unavailable"); +} + +pub fn resultType(ctx: *mlir.Context) *const mlir.Type { + return mlir.Type.parse(ctx, "!cuda.result") catch @panic("CUDA result type is unavailable"); +} + +pub fn launchConfigType(ctx: *mlir.Context, max_attrs: u32) *const mlir.Type { + var buffer: [64]u8 = undefined; + const text = std.fmt.bufPrint(&buffer, "!cuda.launch_cfg", .{max_attrs}) catch @panic("CUDA launch-config type overflow"); + return mlir.Type.parse(ctx, text) catch @panic("CUDA launch-config type is unavailable"); +} + +pub const KernelArgs = struct { + name: []const u8, + function_type: *const mlir.Type, + body: *mlir.Block, + arg_attrs: ?*const mlir.Attribute = null, + extra_attributes: []const mlir.NamedAttribute = &.{}, + location: *const mlir.Location, +}; + +pub fn kernel(ctx: *mlir.Context, args: KernelArgs) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 16) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "sym_name", .string(ctx, args.name)), + .named(ctx, "function_type", .typeAttr(args.function_type)), + }); + if (args.arg_attrs) |attr| attrs.appendAssumeCapacity(.named(ctx, "arg_attrs", attr)); + attrs.appendSliceAssumeCapacity(args.extra_attributes); + return mlir.Operation.make(ctx, "cuda.kernel", .{ + .blocks = &.{args.body}, + .attributes = attrs.constSlice(), + .location = args.location, + .verify = false, + }); +} + +pub fn return_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.return", .{ + .operands = .{ .flat = values }, + .location = location, + .verify = false, + }); +} + +pub fn cast(ctx: *mlir.Context, dst: *const mlir.Type, src: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.cast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst} }, + .location = location, + .verify = false, + }); +} + +pub const LaunchConfigCreateArgs = struct { + max_attrs: u32, + block: [3]*const mlir.Value, + dynamic_smem: *const mlir.Value, + grid: [3]*const mlir.Value, + stream: *const mlir.Value, + location: *const mlir.Location, +}; + +pub fn launch_cfg_create(ctx: *mlir.Context, args: LaunchConfigCreateArgs) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.create", .{ + .operands = .{ .flat = &.{ + args.block[0], args.block[1], args.block[2], args.dynamic_smem, + args.grid[0], args.grid[1], args.grid[2], args.stream, + } }, + .results = .{ .flat = &.{launchConfigType(ctx, args.max_attrs)} }, + .attributes = &.{.named(ctx, "maxNumAttrs", .int(ctx, .i32, args.max_attrs))}, + .location = args.location, + .verify = false, + }); +} + +pub fn launch_cfg_cluster_dim(ctx: *mlir.Context, config: *const mlir.Value, dims: [3]*const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.cluster_dim", .{ + .operands = .{ .flat = &.{ config, dims[0], dims[1], dims[2] } }, + .location = location, + .verify = false, + }); +} + +fn launchCfgFlag(ctx: *mlir.Context, comptime name: []const u8, config: *const mlir.Value, enabled: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{ config, enabled } }, + .location = location, + .verify = false, + }); +} + +pub fn launch_cfg_cooperative(ctx: *mlir.Context, config: *const mlir.Value, enabled: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return launchCfgFlag(ctx, "cuda.launch_cfg.cooperative", config, enabled, location); +} + +pub fn launch_cfg_programmatic_stream_serialization_allowed(ctx: *mlir.Context, config: *const mlir.Value, enabled: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return launchCfgFlag(ctx, "cuda.launch_cfg.programmatic_stream_serialization_allowed", config, enabled, location); +} + +pub const LaunchExArgs = struct { + config: *const mlir.Value, + inputs: []const *const mlir.Value, + callee: *const mlir.Attribute, + assume_kernel_attr: ?*const mlir.Attribute = null, + location: *const mlir.Location, +}; + +pub fn launch_ex(ctx: *mlir.Context, args: LaunchExArgs) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 128) = .empty; + operands.appendAssumeCapacity(args.config); + operands.appendSliceAssumeCapacity(args.inputs); + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attrs.appendAssumeCapacity(.named(ctx, "callee", args.callee)); + if (args.assume_kernel_attr) |attr| attrs.appendAssumeCapacity(.named(ctx, "assume_kernel_attr", attr)); + return mlir.Operation.make(ctx, "cuda.launch_ex", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = &.{resultType(ctx)} }, + .attributes = attrs.constSlice(), + .location = args.location, + .verify = false, + }); +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/mlir/dialects/dialects.zig b/mlir/dialects/dialects.zig index d90c36a5..b5fd8d7b 100644 --- a/mlir/dialects/dialects.zig +++ b/mlir/dialects/dialects.zig @@ -10,6 +10,7 @@ pub const ttir = @import("mlir/dialects/ttir"); pub const affine = @import("affine.zig"); pub const arith = @import("arith.zig"); pub const cf = @import("cf.zig"); +pub const cuda = @import("cuda.zig"); pub const func = @import("func.zig"); pub const gpu = @import("gpu.zig"); pub const math = @import("math.zig"); diff --git a/third_party/cute_ir/repo.bzl b/third_party/cute_ir/repo.bzl index 8ccba581..6beb5df0 100644 --- a/third_party/cute_ir/repo.bzl +++ b/third_party/cute_ir/repo.bzl @@ -9,8 +9,14 @@ def repo(): commit = "098de2a652cf8f00fd70b2df54051c7eccbb855a", build_file = Label("//third_party/cute_ir:cute_ir.bazel"), strip_prefix = "cutlass_compiler", - # The DSL spells a dynamic leaf it knows to be a multiple of N as - # `?{div=N}`; the release's cutegen reads only the width. - patches = [Label("//third_party/cute_ir:cutegen_dynamic_divisibility.patch")], + patches = [ + # `cute.slice` upstream only infers tuple and layout results; the + # MoE kernels slice memrefs and coordinate tensors with explicit + # result types. + Label("//third_party/cute_ir:slice_tensor.patch"), + # The DSL spells a dynamic leaf it knows to be a multiple of N as + # `?{div=N}`; the release's cutegen reads only the width. + Label("//third_party/cute_ir:cutegen_dynamic_divisibility.patch"), + ], patch_args = ["-p2"], ) diff --git a/third_party/cute_ir/slice_tensor.patch b/third_party/cute_ir/slice_tensor.patch new file mode 100644 index 00000000..173a795b --- /dev/null +++ b/third_party/cute_ir/slice_tensor.patch @@ -0,0 +1,45 @@ +diff --git a/cutlass_compiler/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td b/cutlass_compiler/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td +--- a/cutlass_compiler/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td ++++ b/cutlass_compiler/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td +@@ -2990,7 +2990,7 @@ + + def Cute_SliceOp : Cute_Op<"slice", [ + Pure, +- DeclareOpInterfaceMethods ++ DeclareOpInterfaceMethods + ]> { + let cuteSpecGroup = "LayoutAlgebra"; + let summary = "Slice a value, keeping modes where the coordinate is `_`"; +@@ -3154,11 +3154,11 @@ + }]; + + let arguments = (ins +- Cute_Arg:$input, ++ Cute_Arg:$input, + Cute_Arg:$coord + ); + let results = (outs +- Cute_Arg:$result ++ Cute_Arg:$result + ); + let assemblyFormat = [{ `(` operands `)` attr-dict `:` type(operands) }]; + } +diff --git a/cutlass_compiler/cute_ir/lib/Dialect/IR/CuteOps.cpp b/cutlass_compiler/cute_ir/lib/Dialect/IR/CuteOps.cpp +--- a/cutlass_compiler/cute_ir/lib/Dialect/IR/CuteOps.cpp ++++ b/cutlass_compiler/cute_ir/lib/Dialect/IR/CuteOps.cpp +@@ -1327,6 +1327,15 @@ + }); + } + ++LogicalResult SliceOp::refineReturnTypes( ++ MLIRContext *, std::optional, ValueRange, DictionaryAttr, ++ PropertyRef, RegionRange, SmallVectorImpl &) { ++ // Tensor views are private CuTe-DSL types. Their result type is supplied ++ // explicitly by the front end, so no public layout-algebra refinement is ++ // required here. ++ return success(); ++} ++ + //===----------------------------------------------------------------------===// + // DiceOp (cute.dice) + //===----------------------------------------------------------------------===// diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 57adbdae..55c76f49 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -102,6 +102,10 @@ zig_library( "mem.zig", "meta.zig", "mlirx.zig", + "moe/cute_kernels/moe.zig", + "moe/cute_kernels/mxfp4.zig", + "moe/cute_kernels/persistent_mxfp4.zig", + "moe/cute_mxfp4.zig", "moe/cutlass_flashinfer.zig", "moe/fly_kernels/moe.zig", "moe/fly_kernels/mxfp4.zig", diff --git a/zml/kernel.zig b/zml/kernel.zig index 41c0a4ea..7c018cfb 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -513,9 +513,18 @@ pub const cuda_tile = struct { pub const cute = struct { pub const Builder = cute_builder.Builder; + pub const ArgSpec = cute_builder.ArgSpec; pub const Value = cute_builder.Value; + pub const View = cute_builder.View; + pub const Atom = cute_builder.Atom; pub const DType = cute_builder.DType; + pub const AlgebraToken = cute_builder.AlgebraToken; + pub const LayoutSpec = cute_builder.LayoutSpec; + pub const ConstrainedDynamic = cute_builder.ConstrainedDynamic; pub const FinishError = cute_builder.FinishError; + pub const TmaConfig = cute_builder.TmaConfig; + pub const TmaFormat = cute_builder.TmaFormat; + pub const BlockScaledMmaConfig = cute_builder.BlockScaledMmaConfig; pub fn newContext() std.mem.Allocator.Error!*mlir.Context { return makeKernelContext(&cute_builder.dialects_needed); @@ -532,8 +541,10 @@ pub const cute = struct { .bf16 => .bf16, .f32 => .f32, .f64 => .f64, + .f4e2m1 => .f4e2m1fn, .f8e4m3fn => .f8e4m3fn, .f8e5m2 => .f8e5m2, + .f8e8m0 => .f8e8m0fnu, else => std.debug.panic("zml.kernel.cute.from: dtype {s} has no CuTe equivalent", .{@tagName(dt)}), }; } @@ -617,6 +628,82 @@ pub const cute = struct { } }; } + + fn ProgramSpec(comptime Config: type) type { + return struct { + /// Public host entry selected by XLA. The callback may give the + /// nested CUDA kernel a different (private) symbol. + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + /// A complete CuTe program: one or more `cuda.kernel` operations inside + /// a `gpu.module`, followed by a public `func.func` which constructs the + /// launch-time CuTe objects and calls `cuda.launch_ex`. + /// + /// This is the source-level equivalent of invoking a Python `@cute.jit` + /// object whose `__call__` builds TMA descriptors and launches an + /// `@cute.kernel`. XLA executes the generated host function on its CUDA + /// stream, so the custom call deliberately carries no external grid or + /// block dimensions. + pub fn Program( + comptime ConfigT: type, + comptime spec: ProgramSpec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + zeroed_outputs: []const i32 = &.{}, + scalars: []const i64 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cute_builder.Builder.openProgram(allocator, ctx, name); + defer b.deinit(); + try spec.run(&b, cfg); + return b.finishProgram(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.cute.Program({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .zeroed_outputs = opts.zeroed_outputs, + .scalars = opts.scalars, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; + } }; pub const fly = struct { diff --git a/zml/moe/cute_kernels/moe.zig b/zml/moe/cute_kernels/moe.zig new file mode 100644 index 00000000..f9d48499 --- /dev/null +++ b/zml/moe/cute_kernels/moe.zig @@ -0,0 +1,172 @@ +//! SM100 CuTe MXFP8 x MXFP4 MoE, the Zig port of the CuTe DSL `NativeMoE`: +//! routing, routed MXFP8 input quantization, persistent up GEMM with a fused +//! SwiGLU + MXFP8 epilogue, persistent down GEMM storing rows in route order, +//! and the top-k reduction. Routing weights are applied before the down +//! projection, with the reference model's BF16 rounding. +const std = @import("std"); + +const zml = @import("../../zml.zig"); +const triton_mxfp4 = @import("../triton_mxfp4.zig"); +pub const boundary = @import("mxfp4.zig"); +pub const persistent = @import("persistent_mxfp4.zig"); + +pub const Inputs = struct { + /// `[tokens, hidden]` BF16. + x: zml.Tensor, + /// `[tokens, topk]` FP32. + routing_weights: zml.Tensor, + w1: zml.Tensor, + /// `w1` scales in the 128x4 layout of `packWeightScales`. + s1: zml.Tensor, + w2: zml.Tensor, + /// `w2` scales in the 128x4 layout of `packWeightScales`. + s2: zml.Tensor, + /// `[tokens, topk]` local expert ids. + ids: zml.Tensor, +}; + +/// The kernels are tuned for the DeepSeek V4.1 expert geometry, shared by the +/// main MoE and the D-Spark drafter. Expert count and top-k are parameters. +pub fn isSupported(hidden: i64, intermediate: i64) bool { + return hidden == 5120 and intermediate == 2304; +} + +/// Route each token/top-k pair to its own one-row group instead of grouping +/// routes by expert. A token's top-k experts are distinct, so for a single +/// token grouping cannot share any weight tile and only adds the routing +/// kernels. Python `grouped=False`. +fn isDirect(tokens: i64) bool { + return tokens == 1; +} + +/// GEMM tile N, which is also the routed expert-group size. Wider tiles +/// reuse each weight tile across more routes but pad more rows per expert, +/// so they only pay off once experts receive tens of routes. +fn tileN(tokens: i64, experts: i64, topk: i64) i64 { + // Direct routing keeps one row per group: the narrowest MMA wastes the + // least of each B tile and leaves room for a 12-stage A/B ring. + if (isDirect(tokens)) return 8; + if (tokens < 512) return 16; + // Prefill: each group re-reads the whole expert weight matrix, so widen + // the tile once experts hold enough routes to fill it. Half the routes + // are remote under expert parallelism, hence the conservative estimate. + // The thresholds are measured on the DeepSeek V4.1 expert geometry. + const routes_per_expert = @divTrunc(tokens * topk, 2 * experts); + if (routes_per_expert >= 128) return 128; + if (routes_per_expert >= 32) return 64; + return 32; +} + +/// One persistent CTA per SM. Each CTA needs the full 227 KiB of shared +/// memory, so launching more CTAs than SMs (e.g. 160 on a 152-SM GB300) +/// runs the surplus as a second wave and nearly doubles the GEMM time. +fn persistentCtas() i64 { + const platform = zml.Compiler.current().platform; + if (platform.devices.len == 0) return 148; + const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return 148; + return if (attribute.int64 > 0) attribute.int64 else 148; +} + +/// Upper bound on the number of `group`-row expert groups for all routes. +fn capacity(tokens: i64, topk: i64, experts: i64, group: i64) i64 { + return @divTrunc(tokens * topk + group - 1, group) + experts - 1; +} + +/// Where each route lives in the grouped GEMM operands. +const Routing = struct { + /// Expert groups, i.e. the GEMMs' L extent. + groups: i64, + /// Route -> physical routed row. Null for direct routing, where route + /// `r` is row `r`. + route_map: ?zml.Tensor, + /// Physical routed row -> route. Direct routing does not read it. + route_inverse: zml.Tensor, + /// `[group experts | group sizes | active group count]` for grouped + /// routing, `[route experts]` for direct routing. + schedule: zml.Tensor, +}; + +fn route(tokens: i64, hidden: i64, intermediate: i64, experts: i64, topk: i64, n: i64, direct: bool, ids: zml.Tensor) Routing { + const routes = tokens * topk; + const flat_ids = ids.convert(.i32).reshape(.{routes}); + if (direct) return .{ .groups = routes, .route_map = null, .route_inverse = flat_ids, .schedule = flat_ids }; + const groups = capacity(tokens, topk, experts, n); + const cfg: triton_mxfp4.kernels.Config = .{ + .tokens = tokens, + .hidden = hidden, + .intermediate = intermediate, + .experts = experts, + .global_experts = experts, + .topk = topk, + // Unused: only the routing and scheduling kernels of that config run. + .swiglu_limit = 0, + }; + const schedule = triton_mxfp4.kernels.scheduleForCute(cfg, flat_ids.reshape(.{ tokens, topk }), groups, n); + return .{ .groups = groups, .route_map = schedule.route_map, .route_inverse = schedule.route_inverse, .schedule = schedule.schedule }; +} + +/// `[tokens, hidden]` BF16 output of the MoE layer. +pub fn forward(tokens: i64, hidden: i64, intermediate: i64, experts: i64, topk: i64, swiglu_limit: f32, a: Inputs) zml.Tensor { + return forwardWithTile(tokens, hidden, intermediate, experts, topk, swiglu_limit, a, null); +} + +/// `forward` with an explicit GEMM tile N, for tuning. `null` selects it from +/// the batch size. +pub fn forwardWithTile(tokens: i64, hidden: i64, intermediate: i64, experts: i64, topk: i64, swiglu_limit: f32, a: Inputs, tile_n: ?i64) zml.Tensor { + if (!isSupported(hidden, intermediate)) @panic("unsupported SM100 CuTe MoE shape"); + const n = tile_n orelse tileN(tokens, experts, topk); + const direct = isDirect(tokens); + const routing = route(tokens, hidden, intermediate, experts, topk, n, direct, a.ids); + const routes = tokens * topk; + + const up_cfg: persistent.Config = .{ + .experts = experts, + .m = 2 * intermediate, + .n = n, + .k = hidden, + .groups = routing.groups, + .persistent_ctas = persistentCtas(), + .epilogue = .swiglu_mxfp8, + .routes = routes, + .direct = direct, + .swiglu_limit = swiglu_limit, + }; + var down_cfg = up_cfg; + down_cfg.m = hidden; + down_cfg.k = intermediate; + down_cfg.epilogue = .route_rows; + + const q1 = boundary.groupedQuantize(a.x, routing.route_map orelse routing.schedule, .{ + .tokens = tokens, + .hidden = hidden, + .topk = topk, + .group_size = up_cfg.groupRows(), + .capacity = routing.groups, + .direct = direct, + }); + const q2 = persistent.upQuantized(up_cfg, .{ + .weight = a.w1, + .schedule = routing.schedule, + .input_quant = q1.q, + .weight_scale = a.s1.bitCast(.u8), + .input_scale = q1.s, + }, routing.route_inverse, a.routing_weights.reshape(.{routes})); + const down = persistent.downRows(down_cfg, .{ + .weight = a.w2, + .schedule = routing.schedule, + .input_quant = q2.q, + .weight_scale = a.s2.bitCast(.u8), + .input_scale = q2.s, + }, routing.route_inverse); + return boundary.combine(down, routing.route_map orelse routing.schedule, .{ + .tokens = tokens, + .hidden = hidden, + .topk = topk, + .columns_per_cta = boundary.combineColumns(tokens, hidden), + }); +} + +test { + _ = boundary; + _ = persistent; +} diff --git a/zml/moe/cute_kernels/mxfp4.zig b/zml/moe/cute_kernels/mxfp4.zig new file mode 100644 index 00000000..98b16ccf --- /dev/null +++ b/zml/moe/cute_kernels/mxfp4.zig @@ -0,0 +1,263 @@ +//! CuTe boundary kernels around the persistent MXFP4 GEMMs of +//! `persistent_mxfp4.zig`: routed MXFP8 input quantization and the top-k +//! reduction. +const std = @import("std"); + +const zml = @import("../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; + +pub const GroupedQuantConfig = struct { + tokens: i64, + hidden: i64, + topk: i64, + /// Rows stored per expert group. + group_size: i64, + capacity: i64, + /// Route `r` goes to row `r * group_size`; `route_map` is not read. + direct: bool = false, +}; + +pub const CombineConfig = struct { + tokens: i64, + hidden: i64, + topk: i64, + columns_per_cta: i64, +}; + +// Both boundary kernels are programs rather than bare kernels so their launch +// carries the programmatic-dependent-launch attribute: they then start while +// the previous kernel drains, and wait on `griddepcontrol` before their first +// read. A bare kernel is launched by XLA without that attribute. +const GroupedQuantize = cute.Program(GroupedQuantConfig, .{ + .name = "mxfp4_cute_grouped_quantize", + .inputs = &.{ "x", "route_map" }, + .outputs = &.{ "q", "s" }, + .run = groupedQuantizeProgram, +}); + +const Combine = cute.Program(CombineConfig, .{ + .name = "mxfp4_cute_combine", + .inputs = &.{ "d", "routes" }, + .outputs = &.{"y"}, + .run = combineProgram, +}); + +const quantize_kernel_name = "mxfp4_cute_grouped_quantize_kernel"; +const combine_kernel_name = "mxfp4_cute_combine_kernel"; + +/// Host launch of a boundary kernel: forward the buffer pointers, with PDL. +fn launchBoundary(b: *B, kernel_name: [:0]const u8, grid: [3]i32, args: anytype) cute.FinishError!void { + const one = b.cst(.i32, 1); + const config = b.makeLaunchConfig(.{ + .grid = .{ b.cst(.i32, grid[0]), b.cst(.i32, grid[1]), b.cst(.i32, grid[2]) }, + .block = .{ b.cst(.i32, 128), one, one }, + .dynamic_smem = b.kernelSmemSize(kernel_name), + .stream = b.cudaStream(), + .cluster = .{ one, one, one }, + .use_pdl = true, + }); + const launch = b.launchEx(kernel_name, config, args); + b.returnHostStatus(b.cudaResultStatus(launch)); +} + +/// Quantize each token once, then scatter it to the physical rows assigned by +/// the routing. Scale bytes use the 128x4 layout consumed by the SM100 +/// block-scaled MMA scale-factor tensor map. +pub fn groupedQuantize(x: zml.Tensor, route_map: zml.Tensor, cfg: GroupedQuantConfig) struct { q: zml.Tensor, s: zml.Tensor } { + std.debug.assert(cfg.group_size == 8 or cfg.group_size == 16 or cfg.group_size == 32 or cfg.group_size == 64 or cfg.group_size == 128); + const result = GroupedQuantize.call( + .{ .x = x, .route_map = route_map }, + .{ + .q = .init(.{ cfg.capacity * cfg.group_size, cfg.hidden }, .f8e4m3fn), + .s = .init(.{ cfg.capacity, @divExact(cfg.hidden, 32) * 128 }, .u8), + }, + .{ .cfg = cfg }, + ); + return .{ .q = result.q, .s = result.s }; +} + +/// Amortize launch overhead for large prefills while retaining enough CTAs +/// to saturate the GPU for decode and small batches. +pub fn combineColumns(tokens: i64, hidden: i64) i64 { + const elements = tokens * hidden; + if (tokens > 1 and elements >= 512 * 2048) return 2048; + if (tokens > 1 and elements >= 512 * 1024) return 1024; + // Two columns per thread is the smallest tile: 128 threads, one 32-bit + // word each. + return 256; +} + +/// Sum the `[tokens * topk, hidden]` BF16 route rows of each token into BF16. +/// Routing weights were already applied before the down projection. +/// `routes` is negative for routes this rank does not compute (expert +/// parallelism): their rows were never written. +pub fn combine(d: zml.Tensor, routes: zml.Tensor, cfg: CombineConfig) zml.Tensor { + return Combine.call( + .{ .d = d, .routes = routes }, + .{ .y = .init(.{ cfg.tokens, cfg.hidden }, .bf16) }, + .{ .cfg = cfg }, + ).y; +} + +/// Python `prepare_input_vector`: each thread quantizes four consecutive +/// values and stores them as one packed word, so eight lanes cover one +/// 32-value MX block. +fn groupedQuantizeProgram(b: *B, cfg: GroupedQuantConfig) cute.FinishError!void { + b.beginFunction(quantize_kernel_name, .cuda_kernel); + try groupedQuantizeKernel(b, cfg); + b.endFunction(.{ 128, 1, 1 }); + + b.beginFunction(GroupedQuantize.name, .host); + // Host parameters carry the custom call's element types; the kernel reads + // four FP8 values per 32-bit word. + const host = try b.declareArgs(.{ + .x = .{ .ptr = cute.DType.bf16 }, + .route_map = .{ .ptr = cute.DType.i32 }, + .q = .{ .ptr = cute.DType.f8e4m3fn }, + .s = .{ .ptr = cute.DType.i8 }, + }); + try launchBoundary(b, quantize_kernel_name, .{ @intCast(@divExact(cfg.hidden, 4 * 128)), @intCast(cfg.tokens), 1 }, .{ + host.x, host.route_map, b.recastPointer(host.q, .i32, .gmem, 16), host.s, + }); + b.endFunction(null); +} + +fn groupedQuantizeKernel(b: *B, cfg: GroupedQuantConfig) cute.FinishError!void { + std.debug.assert(@rem(cfg.hidden, 4 * 128) == 0); + const a = try b.declareArgs(.{ + .x = .{ .tensor = .{ .dtype = .bf16, .shape = &.{ cfg.tokens, cfg.hidden } } }, + .route_map = .{ .tensor = .{ .dtype = .i32, .shape = &.{cfg.tokens * cfg.topk} } }, + // Four FP8 values per word. + .q = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.capacity * cfg.group_size, @divExact(cfg.hidden, 4) } } }, + .s = .{ .tensor = .{ .dtype = .i8, .shape = &.{ cfg.capacity, @divExact(cfg.hidden, 32) * 128 } } }, + }); + + const tid = b.threadIdx().x; + // This grid may have started before its producer finished: the routing and + // the previous layer write what it reads below. + b.waitForDependency(); + const token = b.blockIdx().y; + const word = b.blockIdx().x.mul(128).add(tid); + var x: [4]cute.Value = undefined; + for (&x, 0..) |*value, i| value.* = a.x.get(.{ token, word.mul(4).add(@as(i32, @intCast(i))) }).to(.f32); + + var maximum = x[0].abs().maximum(x[1].abs()).maximum(x[2].abs().maximum(x[3].abs())); + inline for (.{ 1, 2, 4 }) |offset| { + maximum = maximum.maximum(b.shuffleXor(maximum, offset)); + } + const raw_bits = maximum.maximum(1e-4).mul(1.0 / 448.0).bitCast(.i32); + const exponent = raw_bits.shrLogical(23).add(raw_bits.bitAnd(0x7fffff).ne(0).to(.i32)); + const inverse_scale = b.cst(.i32, 254).sub(exponent).shl(23).bitCast(.f32); + const packed_word = b.packFp8x4(x[0].mul(inverse_scale), x[1].mul(inverse_scale), x[2].mul(inverse_scale), x[3].mul(inverse_scale)); + + const scale_col = word.div(8); + for (0..@intCast(cfg.topk)) |slot| { + const route = token.mul(cfg.topk).add(slot); + const row = if (cfg.direct) route.mul(cfg.group_size) else a.route_map.get(.{route}); + // Routes of other expert-parallel ranks are not scheduled. + var scheduled = if (cfg.direct) null else b.openIf(row.ge(0)); + a.q.set(.{ row, word }, packed_word); + + var leader = b.openIf(tid.rem(8).eq(0)); + const scale_row = row.div(cfg.group_size); + const row_in_group = row.rem(cfg.group_size); + const swizzled_col = scale_col.div(4).mul(512) + .add(row_in_group.rem(32).mul(16)) + .add(row_in_group.div(32).mul(4)) + .add(scale_col.rem(4)); + a.s.set(.{ scale_row, swizzled_col }, exponent.to(.i8)); + leader.yieldThen(.{}); + if (scheduled) |*s| s.yieldThen(.{}); + } + // Only now are the rows this CTA owns complete: release the persistent + // GEMM, which waits on this signal before its first TMA read. + b.launchDependents(); +} + +/// Each thread owns two adjacent BF16 columns, read and written as one +/// 32-bit word: a warp then moves a full 128-byte line per access. +fn combineProgram(b: *B, cfg: CombineConfig) cute.FinishError!void { + b.beginFunction(combine_kernel_name, .cuda_kernel); + try combineKernel(b, cfg); + b.endFunction(.{ 128, 1, 1 }); + + b.beginFunction(Combine.name, .host); + // The kernel reads and writes two BF16 columns per 32-bit word. + const host = try b.declareArgs(.{ + .d = .{ .ptr = cute.DType.bf16 }, + .routes = .{ .ptr = cute.DType.i32 }, + .y = .{ .ptr = cute.DType.bf16 }, + }); + try launchBoundary(b, combine_kernel_name, .{ + @intCast(@divTrunc(cfg.hidden + cfg.columns_per_cta - 1, cfg.columns_per_cta)), + @intCast(cfg.tokens), + 1, + }, .{ b.recastPointer(host.d, .i32, .gmem, 16), host.routes, b.recastPointer(host.y, .i32, .gmem, 16) }); + b.endFunction(null); +} + +fn combineKernel(b: *B, cfg: CombineConfig) cute.FinishError!void { + const pairs = @divExact(cfg.hidden, 2); + const a = try b.declareArgs(.{ + .d = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.tokens * cfg.topk, pairs } } }, + .routes = .{ .tensor = .{ .dtype = .i32, .shape = &.{cfg.tokens * cfg.topk} } }, + .y = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.tokens, pairs } } }, + }); + + // The down projection's rows arrive from a grid this one may overlap. + b.waitForDependency(); + const token = b.blockIdx().y; + const pairs_per_cta = @divExact(cfg.columns_per_cta, 2); + const first_pair = b.blockIdx().x.mul(pairs_per_cta).add(b.threadIdx().x); + for (0..@intCast(@divExact(pairs_per_cta, 128))) |chunk| { + const pair = first_pair.add(chunk * 128); + var in_bounds = b.openIf(pair.lt(pairs)); + var low = b.cst(.f32, 0); + var high = b.cst(.f32, 0); + for (0..@intCast(cfg.topk)) |slot| { + const route = token.mul(cfg.topk).add(slot); + // Routes of other expert-parallel ranks were never written. + var computed = b.openIfElse(a.routes.get(.{route}).ge(0), .{ cute.DType.f32.toMlir(b.ctx), cute.DType.f32.toMlir(b.ctx) }); + const word = a.d.get(.{ route, pair }); + computed.yieldThen(.{ + low.add(word.shl(16).bitCast(.f32)), + high.add(word.bitAnd(@as(i32, @bitCast(@as(u32, 0xffff0000)))).bitCast(.f32)), + }); + computed.yieldElse(.{ low, high }); + low = computed.results[0]; + high = computed.results[1]; + } + const low_bits = low.to(.bf16).to(.f32).bitCast(.i32).shrLogical(16); + const high_bits = high.to(.bf16).to(.f32).bitCast(.i32).bitAnd(@as(i32, @bitCast(@as(u32, 0xffff0000)))); + a.y.set(.{ token, pair }, low_bits.bitOr(high_bits)); + in_bounds.yieldThen(.{}); + } + b.launchDependents(); +} + +test "MXFP4 CuTe boundary kernels emit for decode and prefill batches" { + inline for (.{ @as(i64, 1), 6, 96, 16_384 }) |tokens| { + const quant_ir = try GroupedQuantize.emit(std.testing.allocator, .{ + .tokens = tokens, + .hidden = 5120, + .topk = 6, + .group_size = 16, + .capacity = @divTrunc(tokens * 6 + 15, 16) + 191, + }); + defer std.testing.allocator.free(quant_ir); + try std.testing.expect(std.mem.indexOf(u8, quant_ir, "griddepcontrol.launch_dependents;") != null); + try std.testing.expect(std.mem.indexOf(u8, quant_ir, "cuda.launch_ex") != null); + try std.testing.expect(std.mem.indexOf(u8, quant_ir, "cvt.rn.satfinite.e4m3x2.f32") != null); + + const combine_ir = try Combine.emit(std.testing.allocator, .{ + .tokens = tokens, + .hidden = 5120, + .topk = 6, + .columns_per_cta = combineColumns(tokens, 5120), + }); + defer std.testing.allocator.free(combine_ir); + try std.testing.expect(std.mem.indexOf(u8, combine_ir, "cuda.launch_ex") != null); + try std.testing.expect(std.mem.indexOf(u8, combine_ir, "scf.if") != null); + } +} diff --git a/zml/moe/cute_kernels/persistent_mxfp4.zig b/zml/moe/cute_kernels/persistent_mxfp4.zig new file mode 100644 index 00000000..1f4b25cb --- /dev/null +++ b/zml/moe/cute_kernels/persistent_mxfp4.zig @@ -0,0 +1,2018 @@ +//! Source-level SM100 MXFP4 grouped GEMM. +//! +//! This file is the Zig CuTe-DSL counterpart of the CuTe DSL +//! `Sm100BlockScaledPersistentDenseGemmKernel` with its fused MoE epilogues. +//! It emits a complete CuTe program: the nested `cuda.kernel` consumes the +//! objects constructed by the public host `func.func`, and the host function +//! creates the block-scaled MMA objects, four TMA tensor maps, persistent +//! scheduler parameters, and the PDL-enabled launch. +//! +//! One CTA computes a 128xN output tile of one expert group: warp 5 drives +//! the TMA ring, warp 4 issues the S2T scale copies and UMMA, and warps 0-3 +//! drain the accumulator from tensor memory into one of the two epilogues. +const std = @import("std"); +const mlir = @import("mlir"); + +const zml = @import("../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const V = cute.Value; + +const accumulator_stages = 2; + +/// Shared-memory plan of the Python kernel for one tile width, copied from +/// its generated `SharedStorage` (`smem.struct_fields`). The A/B ring depth +/// is whatever fits next to the two C epilogue stages; the fused epilogues +/// only use the start of `sC` as scratch. +pub const Tile = struct { + n: i64, + stages: usize, + smem_bytes: i64, + tmem_columns: u16, + empty_barriers: i64, + accumulator_full: i64, + accumulator_empty: i64, + tmem_dealloc: i64, + tmem_holding: i64, + c: i64, + a: i64, + b: i64, + sfa: i64, + sfb: i64, + /// Bytes reserved at `c`. Only the epilogue scratch is used; the wide + /// prefill tiles keep just that instead of two full C stages. + c_bytes: ?i64 = null, + + pub fn of(comptime n: i64) Tile { + return switch (n) { + 8 => .{ + .n = 8, + .stages = 12, + .smem_bytes = 230400, + .tmem_columns = 64, + .empty_barriers = 96, + .accumulator_full = 192, + .accumulator_empty = 208, + .tmem_dealloc = 224, + .tmem_holding = 232, + .c = 1024, + .a = 9216, + .b = 205824, + .sfa = 218112, + .sfb = 224256, + }, + 16 => .{ + .n = 16, + .stages = 11, + .smem_bytes = 232448, + .tmem_columns = 64, + .empty_barriers = 88, + .accumulator_full = 176, + .accumulator_empty = 192, + .tmem_dealloc = 208, + .tmem_holding = 216, + .c = 1024, + .a = 17408, + .b = 197632, + .sfa = 220160, + .sfb = 226304, + }, + 32 => .{ + .n = 32, + .stages = 9, + .smem_bytes = 228352, + .tmem_columns = 128, + .empty_barriers = 72, + .accumulator_full = 144, + .accumulator_empty = 160, + .tmem_dealloc = 176, + .tmem_holding = 184, + .c = 1024, + .a = 33792, + .b = 181248, + .sfa = 218112, + .sfb = 223232, + }, + // Prefill tiles. One expert weight tile feeds 64 or 128 routed + // rows instead of 32, which is what keeps large batches from + // re-reading the expert weights once per group. They hold only + // the epilogue scratch at `c`, so the A/B ring stays deep. + 64 => .{ + .n = 64, + .stages = 8, + .smem_bytes = 206848, + .tmem_columns = 256, + .empty_barriers = 64, + .accumulator_full = 128, + .accumulator_empty = 144, + .tmem_dealloc = 160, + .tmem_holding = 168, + .c = 1024, + .c_bytes = 512, + .a = 2048, + .sfa = 133120, + .sfb = 137216, + .b = 141312, + }, + 128 => .{ + .n = 128, + .stages = 6, + .smem_bytes = 204800, + .tmem_columns = 512, + .empty_barriers = 48, + .accumulator_full = 96, + .accumulator_empty = 112, + .tmem_dealloc = 128, + .tmem_holding = 136, + .c = 1024, + .c_bytes = 512, + .a = 2048, + .b = 100352, + .sfa = 198656, + .sfb = 201728, + }, + else => @compileError("unsupported MXFP4 tile N"), + }; + } + + /// Routed rows an epilogue pass handles: one per warp lane. + pub fn chunkRows(self: Tile) i64 { + return @min(self.n, 32); + } + + /// Epilogue passes over the tile width. + pub fn chunks(self: Tile) usize { + return @intCast(@divExact(self.n, self.chunkRows())); + } + + const a_stage_bytes = 128 * 128; + const sf_stage_bytes = 512; + + fn bStageBytes(self: Tile) i64 { + return self.n * 128; + } + + fn cStageElements(self: Tile) i64 { + return 128 * self.n; + } + + /// TMA bytes landing in one A/B stage: unpacked FP4 A is counted as + /// 128x128 nibbles, then the FP8 B tile and both 512-byte scale atoms. + fn transactionBytes(self: Tile) u32 { + return @intCast(8192 + self.bStageBytes() + 2 * sf_stage_bytes); + } + + fn fields(self: Tile, b: *B) []const []const u8 { + const arena = b.arena.allocator(); + const s: i64 = @intCast(self.stages); + const out = arena.alloc([]const u8, 11) catch @panic("OOM"); + const Field = struct { []const u8, i64, i64 }; + const list = [_]Field{ + .{ "ab_full_mbar_ptr", 8 * s, 0 }, + .{ "ab_empty_mbar_ptr", 8 * s, self.empty_barriers }, + .{ "acc_full_mbar_ptr", 16, self.accumulator_full }, + .{ "acc_empty_mbar_ptr", 16, self.accumulator_empty }, + .{ "tmem_dealloc_mbar", 8, self.tmem_dealloc }, + .{ "tmem_holding_buf", 4, self.tmem_holding }, + .{ "sC", self.c_bytes orelse accumulator_stages * self.cStageElements() * 4, self.c }, + .{ "sA", s * a_stage_bytes, self.a }, + .{ "sB", s * self.bStageBytes(), self.b }, + .{ "sSFA", s * sf_stage_bytes, self.sfa }, + .{ "sSFB", s * sf_stage_bytes, self.sfb }, + }; + for (list, out) |field, *text| { + text.* = std.fmt.allocPrint(arena, "{s}:{d}:{d}", .{ field[0], field[1], field[2] }) catch @panic("OOM"); + } + return out; + } +}; + +const StageStorage = struct { + a: V, + b: V, + sfa: V, + sfb: V, + c: V, + load_barriers: V, + mma_barriers: V, + accumulator_full_barriers: V, + accumulator_empty_barriers: V, +}; + +const StageBuffers = struct { + a_destination: cute.View, + b_destination: cute.View, + sfa_destination: cute.View, + sfb_destination: cute.View, + load_barrier: V, + mma_barrier: V, +}; + +const StagedOperands = struct { + a: cute.View, + b: cute.View, + sfa: cute.View, + sfb: cute.View, +}; + +/// What the epilogue warps do with each finished 128xN accumulator tile. +pub const Epilogue = enum { + /// Up projection: clamped SwiGLU of the interleaved gate/up rows, BF16 + /// rounding and the routing weight as in the reference model, then MXFP8 + /// quantization straight into the down GEMM's input and 128x4 scale + /// layouts. Python `fused=1`. + swiglu_mxfp8, + /// Down projection: BF16 rows stored in route order `[routes, m]`, + /// skipping padding rows. Python `fused=2`. BF16 halves the traffic of + /// this store and of the top-k reduction that reads it back; the routed + /// partial sums are rounded exactly like the reference finalize. + route_rows, +}; + +pub const Config = struct { + experts: i64, + /// GEMM output rows (4608 for gate/up, 5120 for down). + m: i64, + /// Tile N: rows in one routed expert group. + n: i64 = 16, + /// Reduction dimension (5120 for gate/up, 2304 for down). + k: i64, + groups: i64, + /// Resident CTAs of the static persistent tile scheduler: one per SM. + persistent_ctas: i64 = 148, + epilogue: Epilogue, + /// Token/top-k routes; sizes the route-indexed epilogue operands. + routes: i64, + /// Clamp applied to both halves of the fused SwiGLU. Emitted as an IR + /// constant, so a threaded value costs exactly what a literal did. + swiglu_limit: f32 = 10.0, + /// Ungrouped routing: group `g` is route `g` and holds one live row. + /// `schedule` is then just the `[routes]` expert ids; its size and + /// active-count sections and `route_inverse` are not read. + direct: bool = false, + + /// Rows stored per group in the routed operands: one whole N tile, of + /// which direct routing fills only the first. A tensor map whose row mode + /// had extent one would lose that mode altogether — the TMA atom infers a + /// zero stride for it — and every group would then read the same row. + pub fn groupRows(self: Config) i64 { + return self.n; + } + + pub fn routedRows(self: Config) i64 { + return self.groups * self.groupRows(); + } +}; + +/// `schedule` is the routing schedule `[group experts | group sizes | active +/// group count]`; the kernel reads the first and last sections in place. +const inputs = [_][:0]const u8{ + "weight", + "schedule", + "input_quant", + "weight_scale", + "input_scale", +}; + +/// `route_inverse` maps each physical routed row to its token/top-k route; +/// entries of padding rows are never read. +const UpQuantized = cute.Program(Config, .{ + .name = "zml_mxfp4_up_swiglu_zig", + .inputs = &(inputs ++ [_][:0]const u8{ "route_inverse", "routing_weights" }), + .outputs = &.{ "q", "s" }, + .run = buildProgram, +}); + +const DownRows = cute.Program(Config, .{ + .name = "zml_mxfp4_down_rows_zig", + .inputs = &(inputs ++ [_][:0]const u8{"route_inverse"}), + .outputs = &.{"output"}, + .run = buildProgram, +}); + +pub const Inputs = struct { + weight: zml.Tensor, + schedule: zml.Tensor, + input_quant: zml.Tensor, + weight_scale: zml.Tensor, + input_scale: zml.Tensor, +}; + +/// Up projection with the SwiGLU + MXFP8 epilogue: the routed FP8 rows +/// `[groups * n, m / 2]` and their 128x4 scales `[groups, m / 64 * 128]`, +/// ready for the down GEMM. `routing_weights` is `[routes]` FP32. +pub fn upQuantized(cfg: Config, a: Inputs, route_inverse: zml.Tensor, routing_weights: zml.Tensor) struct { q: zml.Tensor, s: zml.Tensor } { + std.debug.assert(cfg.epilogue == .swiglu_mxfp8); + const activations = @divExact(cfg.m, 2); + const result = UpQuantized.call(.{ + .weight = a.weight, + .schedule = a.schedule, + .input_quant = a.input_quant, + .weight_scale = a.weight_scale, + .input_scale = a.input_scale, + .route_inverse = route_inverse, + .routing_weights = routing_weights, + }, .{ + .q = .init(.{ cfg.routedRows(), activations }, .f8e4m3fn), + .s = .init(.{ cfg.groups, @divExact(activations, 32) * 128 }, .u8), + }, .{ .cfg = cfg, .scalars = &.{ cfg.m, cfg.n, cfg.k, cfg.groups, cfg.experts } }); + return .{ .q = result.q, .s = result.s }; +} + +/// Down projection storing FP32 `[routes, m]` in token/top-k order. +pub fn downRows(cfg: Config, a: Inputs, route_inverse: zml.Tensor) zml.Tensor { + std.debug.assert(cfg.epilogue == .route_rows); + return DownRows.call(.{ + .weight = a.weight, + .schedule = a.schedule, + .input_quant = a.input_quant, + .weight_scale = a.weight_scale, + .input_scale = a.input_scale, + .route_inverse = route_inverse, + }, .{ .output = .init(.{ cfg.routes, cfg.m }, .bf16) }, .{ .cfg = cfg, .scalars = &.{ cfg.m, cfg.n, cfg.k, cfg.groups, cfg.experts } }).output; +} + +fn typed(type_: *const mlir.Type) struct { mlir_type: cute.ArgSpec.MlirTypeSpec } { + return .{ .mlir_type = .{ .type_ = type_ } }; +} + +fn gridTyped(type_: *const mlir.Type) struct { mlir_type: cute.ArgSpec.MlirTypeSpec } { + return .{ .mlir_type = .{ .type_ = type_, .grid_constant = true } }; +} + +/// The two projections share this kernel but get distinct symbols, so a +/// profile shows them as separate rows instead of one merged entry. +fn deviceName(comptime epilogue: Epilogue) [:0]const u8 { + return switch (epilogue) { + .swiglu_mxfp8 => "mxfp4_sm100_persistent_grouped_gemm_up_swiglu", + .route_rows => "mxfp4_sm100_persistent_grouped_gemm_down_rows", + }; +} + +fn buildProgram(b: *B, cfg: Config) cute.FinishError!void { + return switch (cfg.n) { + inline 8, 16, 32, 64, 128 => |n| switch (cfg.epilogue) { + inline else => |epilogue| buildTiledProgram(b, cfg, comptime Tile.of(n), epilogue), + }, + else => std.debug.panic("unsupported MXFP4 tile N {d}", .{cfg.n}), + }; +} + +/// Types and layouts shared by the device ABI and the host launch. +fn Layouts(comptime tile: Tile) type { + return struct { + const n = tile.n; + const stages = tile.stages; + const ab_swizzle: B.Swizzle = .{ .bits = 3, .base = 4, .shift = 3 }; + + fn aCoord(b: *B, cfg: Config) cute.LayoutSpec { + return b.layoutSpec(.{ cfg.m, cfg.k, cfg.experts }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); + } + fn bCoord(b: *B, cfg: Config) cute.LayoutSpec { + return b.layoutSpec(.{ cfg.groupRows(), cfg.k, cfg.groups }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); + } + fn sfCoord(b: *B) cute.LayoutSpec { + return b.layoutSpec( + .{ .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ 1, cute.AlgebraToken.dynamic } }, + .{ + .{ .{ b.basis(8, 1, .{0}), b.basis(2, 1, .{0}) }, b.basis(1, 1, .{1}) }, + .{ .{ 0, b.basis(1, 2, .{0}) }, b.basis(1, 1, .{2}) }, + .{ 0, b.basis(1, 1, .{3}) }, + }, + ); + } + fn aBasis(b: *B) cute.LayoutSpec { + return b.layoutSpec(.{ 128, 128, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); + } + fn bBasis(b: *B) cute.LayoutSpec { + return b.layoutSpec(.{ 128, n, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); + } + fn sfBasis(b: *B) cute.LayoutSpec { + return b.layoutSpec( + .{ .{ 2, 4, 32 }, 1, 1, 1 }, + .{ .{ b.basis(1, 1, .{ 1, 0, 1 }), b.basis(1, 1, .{ 1, 0, 0 }), b.basis(1, 1, .{ 0, 0, 0 }) }, b.basis(1, 1, .{ 1, 0 }), b.basis(1, 1, .{ 1, 1 }), b.basis(1, 1, .{ 1, 2 }) }, + ); + } + fn tmaA(b: *B, cfg: Config) cute.TmaConfig { + return .{ .dtype = .f4e2m1fn, .copy_bits = 65536, .global_basis = aBasis(b), .coordinate_layout = aCoord(b, cfg), .coordinate_rank = 3, .format = .u4_unpack_u8 }; + } + fn tmaB(b: *B, cfg: Config) cute.TmaConfig { + return .{ .dtype = .f8e4m3fn, .copy_bits = @intCast(tile.bStageBytes() * 8), .global_basis = bBasis(b), .coordinate_layout = bCoord(b, cfg), .coordinate_rank = 3 }; + } + fn tmaSf(b: *B) cute.TmaConfig { + return .{ .dtype = .f8e8m0fnu, .copy_bits = 4096, .global_basis = sfBasis(b), .coordinate_layout = sfCoord(b), .coordinate_rank = 4, .format = .u16 }; + } + // Shared-memory layouts, in the same element units as Python. + fn aSmem(b: *B, comptime staged: bool) cute.LayoutSpec { + return if (staged) + b.layoutSpec(.{ .{ 128, 32 }, 1, 4, stages }, .{ .{ 128, 1 }, 0, 32, Tile.a_stage_bytes }) + else + b.layoutSpec(.{ .{ 128, 32 }, 1, 4 }, .{ .{ 128, 1 }, 0, 32 }); + } + fn bSmem(b: *B, comptime staged: bool) cute.LayoutSpec { + return if (staged) + b.layoutSpec(.{ .{ n, 32 }, 1, 4, stages }, .{ .{ 128, 1 }, 0, 32, tile.bStageBytes() }) + else + b.layoutSpec(.{ .{ n, 32 }, 1, 4 }, .{ .{ 128, 1 }, 0, 32 }); + } + fn sfSmem(b: *B, comptime staged: bool) cute.LayoutSpec { + return if (staged) + b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 1 }, .{ 32, 1 } }, 1, 4, stages }, .{ .{ .{ .{ 16, 4 }, 0 }, .{ 0, 0 } }, 0, 1, Tile.sf_stage_bytes }) + else + b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 1 }, .{ 32, 1 } }, 1, 4 }, .{ .{ .{ .{ 16, 4 }, 0 }, .{ 0, 0 } }, 0, 1 }); + } + }; +} + +/// Device memrefs of the route-indexed epilogue operands. +const EpilogueTypes = struct { + sizes: *const mlir.Type, + inverse: *const mlir.Type, + weights: *const mlir.Type, + q: *const mlir.Type, + s: *const mlir.Type, + rows: *const mlir.Type, + + fn of(b: *B, cfg: Config) EpilogueTypes { + const activations = @divExact(cfg.m, 2); + return .{ + // Group sizes follow the group experts inside `schedule`. + .sizes = b.memrefType(.i32, .gmem, 4, b.layoutType(b.layoutSpec(cfg.groups, 1))), + .inverse = b.memrefType(.i32, .gmem, 16, b.layoutType(b.layoutSpec(cfg.routedRows(), 1))), + .weights = b.memrefType(.f32, .gmem, 16, b.layoutType(b.layoutSpec(@max(cfg.routes, 1), 1))), + .q = b.memrefType(.f8e4m3fn, .gmem, 16, b.layoutType(b.layoutSpec(.{ cfg.routedRows(), activations }, .{ activations, 1 }))), + .s = b.memrefType(.i8, .gmem, 16, b.layoutType(b.layoutSpec(.{ cfg.groups, @divExact(activations, 32) * 128 }, .{ @divExact(activations, 32) * 128, 1 }))), + .rows = b.memrefType(.bf16, .gmem, 16, b.layoutType(b.layoutSpec(.{ @max(cfg.routes, 1), cfg.m }, .{ cfg.m, 1 }))), + }; + } +}; + +fn buildTiledProgram(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: Epilogue) cute.FinishError!void { + @setEvalBranchQuota(20_000); + std.debug.assert(@rem(cfg.m, 128) == 0); + std.debug.assert(@rem(cfg.k, 128) == 0); + const L = Layouts(tile); + const public_name = b.name; + + const ids_memref = b.memrefType(.i32, .gmem, 16, b.layoutType(b.layoutSpec(cfg.groups, 1))); + // The active group count is the last schedule entry, 4-byte aligned. + const active_memref = b.memrefType(.i32, .gmem, 4, b.layoutType(b.layoutSpec(1, 1))); + const et = EpilogueTypes.of(b, cfg); + const i32_type = cute.DType.i32.toMlir(b.ctx); + const mma_type = b.blockScaledMmaType(.{ .n = @intCast(tile.n) }); + const mma_sfb_type = b.blockScaledMmaType(.{ .n = 128 }); + + // Device ABI. The host function below constructs every non-buffer value. + b.beginFunction(deviceName(epilogue), .cuda_kernel); + const d = try b.declareArgs(switch (epilogue) { + .swiglu_mxfp8 => .{ + .expert_ids = typed(ids_memref), + .active_count = typed(active_memref), + .mma = typed(mma_type), + .mma_sfb = typed(mma_sfb_type), + .tma_a = gridTyped(b.tmaLoadAtomType(L.tmaA(b, cfg))), + .gA = typed(b.coordTensorType(3, L.aCoord(b, cfg))), + .tma_b = gridTyped(b.tmaLoadAtomType(L.tmaB(b, cfg))), + .gB = typed(b.coordTensorType(3, L.bCoord(b, cfg))), + .tma_sfa = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), + .gSFA = typed(b.coordTensorType(4, L.sfCoord(b))), + .tma_sfb = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), + .gSFB = typed(b.coordTensorType(4, L.sfCoord(b))), + .group_sizes = typed(et.sizes), + .route_inverse = typed(et.inverse), + .routing_weights = typed(et.weights), + .q = typed(et.q), + .s = typed(et.s), + .tiles_m = typed(i32_type), + }, + .route_rows => .{ + .expert_ids = typed(ids_memref), + .active_count = typed(active_memref), + .mma = typed(mma_type), + .mma_sfb = typed(mma_sfb_type), + .tma_a = gridTyped(b.tmaLoadAtomType(L.tmaA(b, cfg))), + .gA = typed(b.coordTensorType(3, L.aCoord(b, cfg))), + .tma_b = gridTyped(b.tmaLoadAtomType(L.tmaB(b, cfg))), + .gB = typed(b.coordTensorType(3, L.bCoord(b, cfg))), + .tma_sfa = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), + .gSFA = typed(b.coordTensorType(4, L.sfCoord(b))), + .tma_sfb = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), + .gSFB = typed(b.coordTensorType(4, L.sfCoord(b))), + .group_sizes = typed(et.sizes), + .route_inverse = typed(et.inverse), + .rows = typed(et.rows), + .tiles_m = typed(i32_type), + }, + }); + b.setFunctionAttribute("nvvm.minctasm", .int(b.ctx, .i32, 1)); + b.setFunctionAttribute("smem.partition_num", .int(b.ctx, .i32, 2)); + b.setFunctionAttribute("smem.hint_smem_base_uniform", .unit(b.ctx)); + b.setFunctionAttribute("cu_attrs", b.parseAttribute("{max_dynamic_shared_size_bytes = #cuda.dev_max_shared_memory_optin, non_portable_cluster_size_allowed = 1 : i32}")); + + // Prefetch every tensor-map descriptor before the specialized warps begin + // the persistent pipeline. This is the work PDL buys us: it runs while the + // producing grid is still writing the operands. + const warp = b.makeWarpUniform(b.makeWarpUniform(b.threadIdx().x.div(32))); + var tma_warp = b.openIf(warp.eq(5)); + inline for (.{ d.tma_a, d.tma_b, d.tma_sfa, d.tma_sfb }) |value| { + const descriptor: cute.Atom = .{ .inner = value.inner, .kernel = b }; + b.prefetchTmaDesc(descriptor); + } + tma_warp.yieldThen(.{}); + // Everything below reads the schedule and the quantized activations the + // preceding grid produces, so the whole block waits here. + b.waitForDependency(); + buildUmmaDevice(b, cfg, tile, epilogue, d); + // The epilogue has written every row this CTA owns. + b.launchDependents(); + b.endFunction(.{ 192, 1, 1 }); + + // Public host ABI: the XLA operands, the outputs, then the problem shape. + // XLA owns packed storage as signless bytes. Recast only inside the host + // program when constructing typed tensor maps. + b.beginFunction(public_name, .host); + const operands = .{ + .weight = .{ .ptr = cute.DType.i8 }, + .schedule = .{ .ptr = cute.DType.i32 }, + .input_quant = .{ .ptr = cute.DType.f8e4m3fn }, + .weight_scale = .{ .ptr = cute.DType.i8 }, + .input_scale = .{ .ptr = cute.DType.i8 }, + }; + const shape = .{ + .problem_m = typed(i32_type), + .problem_n = typed(i32_type), + .problem_k = typed(i32_type), + .problem_l = typed(i32_type), + .expert_count = typed(i32_type), + }; + _ = try b.declareArgs(switch (epilogue) { + .swiglu_mxfp8 => .{ + .weight = operands.weight, + .schedule = operands.schedule, + .input_quant = operands.input_quant, + .weight_scale = operands.weight_scale, + .input_scale = operands.input_scale, + .route_inverse = .{ .ptr = cute.DType.i32 }, + .routing_weights = .{ .ptr = cute.DType.f32 }, + .q = .{ .ptr = cute.DType.f8e4m3fn }, + .s = .{ .ptr = cute.DType.i8 }, + .problem_m = shape.problem_m, + .problem_n = shape.problem_n, + .problem_k = shape.problem_k, + .problem_l = shape.problem_l, + .expert_count = shape.expert_count, + }, + .route_rows => .{ + .weight = operands.weight, + .schedule = operands.schedule, + .input_quant = operands.input_quant, + .weight_scale = operands.weight_scale, + .input_scale = operands.input_scale, + .route_inverse = .{ .ptr = cute.DType.i32 }, + .output = .{ .ptr = cute.DType.bf16 }, + .problem_m = shape.problem_m, + .problem_n = shape.problem_n, + .problem_k = shape.problem_k, + .problem_l = shape.problem_l, + .expert_count = shape.expert_count, + }, + }); + try buildHostLaunch(b, cfg, tile, epilogue); + b.endFunction(null); +} + +/// Host argument positions: the five mainloop operands, the epilogue's +/// extra inputs and outputs, then the five problem-shape scalars. +fn HostArgs(comptime epilogue: Epilogue) type { + return struct { + const weight = 0; + const schedule = 1; + const input_quant = 2; + const weight_scale = 3; + const input_scale = 4; + const route_inverse = 5; + const routing_weights = 6; + const output = switch (epilogue) { + .swiglu_mxfp8 => 7, + .route_rows => 6, + }; + const scales = 8; + const problem_m = switch (epilogue) { + .swiglu_mxfp8 => 9, + .route_rows => 7, + }; + const problem_n = problem_m + 1; + const problem_k = problem_m + 2; + const problem_l = problem_m + 3; + const expert_count = problem_m + 4; + }; +} + +/// Complete staged shared-memory tensors, as Python's `PipelineTmaUmma` +/// consumers see them. Per-stage views are slices of these. +fn stagedOperands(b: *B, comptime tile: Tile, storage: StageStorage) StagedOperands { + const L = Layouts(tile); + const a_full = b.makeViewTyped( + b.recastSwizzledPointer(storage.a, .i8, .smem, 1024, L.ab_swizzle), + b.staticLayout(L.aSmem(b, true)), + b.memrefTypeFromPointer(b.swizzledPtrTy(.i8, .smem, 1024, L.ab_swizzle) catch @panic("bad A shared pointer"), b.layoutType(L.aSmem(b, true))), + ); + const b_full = b.makeViewTyped( + b.recastSwizzledPointer(storage.b, .f8e4m3fn, .smem, 1024, L.ab_swizzle), + b.staticLayout(L.bSmem(b, true)), + b.memrefTypeFromPointer(b.swizzledPtrTy(.f8e4m3fn, .smem, 1024, L.ab_swizzle) catch @panic("bad B shared pointer"), b.layoutType(L.bSmem(b, true))), + ); + const sfa_full = b.makeViewTyped(storage.sfa, b.staticLayout(L.sfSmem(b, true)), b.memrefType(.f8e8m0fnu, .smem, 1024, b.layoutType(L.sfSmem(b, true)))); + const sfb_full = b.makeViewTyped(storage.sfb, b.staticLayout(L.sfSmem(b, true)), b.memrefType(.f8e8m0fnu, .smem, 1024, b.layoutType(L.sfSmem(b, true)))); + return .{ .a = a_full, .b = b_full, .sfa = sfa_full, .sfb = sfb_full }; +} + +fn stageBuffersAt(b: *B, comptime tile: Tile, storage: StageStorage, stage_index: V) StageBuffers { + const L = Layouts(tile); + // Keep the shared operands as staged tensors and select the current stage + // with CuTe slicing. This is the representation used by the Python + // PipelineTmaUmma implementation; rebuilding a memref from a runtime + // pointer loses the staged descriptor semantics at ring wrap. + const full = stagedOperands(b, tile, storage); + const all = cute.AlgebraToken.all; + const a = b.sliceTyped(full.a, .{ all, all, all, stage_index }, b.memrefTypeFromPointer(b.swizzledPtrTy(.i8, .smem, 1024, L.ab_swizzle) catch @panic("bad A stage pointer"), b.layoutType(L.aSmem(b, false)))); + const rhs = b.sliceTyped(full.b, .{ all, all, all, stage_index }, b.memrefTypeFromPointer(b.swizzledPtrTy(.f8e4m3fn, .smem, 1024, L.ab_swizzle) catch @panic("bad B stage pointer"), b.layoutType(L.bSmem(b, false)))); + // One scale stage is 512 bytes, so slicing a stage out of the ring keeps + // that alignment, not the array's. + const sf_stage_align = Tile.sf_stage_bytes; + const sfa = b.sliceTyped(full.sfa, .{ all, all, all, stage_index }, b.memrefType(.f8e8m0fnu, .smem, sf_stage_align, b.layoutType(L.sfSmem(b, false)))); + const sfb = b.sliceTyped(full.sfb, .{ all, all, all, stage_index }, b.memrefType(.f8e8m0fnu, .smem, sf_stage_align, b.layoutType(L.sfSmem(b, false)))); + + // `tma_partition` preserves a singleton CTA mode. Keep it in the copy + // views: it is part of the executable TMA atom's expected algebra even + // for a one-CTA cluster. + const a_partition_layout = b.staticLayout(b.layoutSpec(.{ .{ Tile.a_stage_bytes, 1 }, .{1} }, .{ .{ 1, 0 }, .{0} })); + const b_partition_layout = b.staticLayout(b.layoutSpec(.{ .{ tile.bStageBytes(), 1 }, .{1} }, .{ .{ 1, 0 }, .{0} })); + const sf_partition_layout = b.staticLayout(b.layoutSpec(.{ .{ Tile.sf_stage_bytes, 1 }, .{1} }, .{ .{ 1, 0 }, .{0} })); + + return .{ + .a_destination = b.withLayout(a, a_partition_layout), + .b_destination = b.withLayout(rhs, b_partition_layout), + .sfa_destination = b.withLayout(sfa, sf_partition_layout), + .sfb_destination = b.withLayout(sfb, sf_partition_layout), + .load_barrier = barrierAt(b, storage.load_barriers, stage_index), + .mma_barrier = barrierAt(b, storage.mma_barriers, stage_index), + }; +} + +/// Issue the four transactions that fill one mainloop stage. The caller is +/// already restricted to the TMA warp; one elected lane owns the mbarrier +/// arrival. +fn issueTmaStage( + b: *B, + comptime tile: Tile, + comptime reduction_tiles: usize, + partitioned_a_target: cute.View, + partitioned_b_target: cute.View, + stage: StageBuffers, + k_tile: V, + tile_m: V, + group: V, + expert: V, + exec_a: cute.Atom, + exec_b: cute.Atom, + exec_sfa: cute.Atom, + exec_sfb: cute.Atom, +) void { + const n = tile.n; + const all = cute.AlgebraToken.all; + const a_copy_layout = b.staticLayout(b.layoutSpec( + .{ .{ .{ 128, 128 }, 1 }, .{1} }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, .{0} }, + )); + const b_copy_layout = b.staticLayout(b.layoutSpec( + .{ .{ .{ 128, n }, 1 }, .{1} }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, .{0} }, + )); + const sf_copy_spec = b.layoutSpec( + .{ .{ 512, 1 }, .{1} }, + .{ .{ b.basis(1, 2, .{0}), 0 }, .{0} }, + ); + // Follow the same coordinate algebra as the Python kernel. TMA + // partitioning produces a tensor over (tile, M, K, expert/group); first + // select the output tile and expert/group, then select the K stage. The + // remaining iterator carries the tensor-map coordinates needed by copy. + const a_tile_k = b.sliceTyped( + partitioned_a_target, + .{ all, tile_m, all, expert }, + b.coordTensorTypePayload("(0,?{div=128},?)", b.layoutSpec( + .{ .{ .{ 128, 128 }, 1 }, reduction_tiles }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(128, 1, .{0}) }, + )), + ); + const a_tile = b.sliceTyped( + a_tile_k, + .{ all, k_tile }, + b.coordTensorTypePayload("(?{div=128},?{div=128},?)", b.layoutSpec( + .{.{ .{ 128, 128 }, 1 }}, + .{.{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }}, + )), + ); + const a_source = b.rebuildCoordTensorIterator(3, a_tile, "(?{div=128},?{div=128},?)", .{ 128, 128, 1 }, a_copy_layout); + // B's tile spans all of a group's rows, so its N-tile index is zero. Pass + // it as a value rather than a literal: a statically zero leaf collapses out + // of the coordinate tuple, and the TMA lowering then shifts the remaining + // tensor-map coordinates, silently reading group zero for every group. + const b_tile_n = b.cst(.i32, 0); + const b_tile_k = b.sliceTyped( + partitioned_b_target, + .{ all, b_tile_n, all, group }, + b.coordTensorTypePayload(fmt(b, "(0,?{{div={d}}},?)", .{n}), b.layoutSpec( + .{ .{ .{ 128, n }, 1 }, reduction_tiles }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(128, 1, .{0}) }, + )), + ); + const b_tile = b.sliceTyped( + b_tile_k, + .{ all, k_tile }, + b.coordTensorTypePayload(fmt(b, "(?{{div=128}},?{{div={d}}},?)", .{n}), b.layoutSpec( + .{.{ .{ 128, n }, 1 }}, + .{.{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }}, + )), + ); + const b_source = b.rebuildCoordTensorIterator(3, b_tile, fmt(b, "(?{{div=128}},?{{div={d}}},?)", .{n}), .{ 128, n, 1 }, b_copy_layout); + const sfa_source = b.makeCoordTensor(.{ 0, tile_m, k_tile, expert }, b.staticCoordinateLayout(4, sf_copy_spec)); + const sfb_source = b.makeCoordTensor(.{ 0, b.cst(.i32, 0), k_tile, group }, b.staticCoordinateLayout(4, sf_copy_spec)); + + var elected = b.openIf(b.electSync()); + b.mbarrierArriveExpectTx(stage.load_barrier, tile.transactionBytes()); + elected.yieldThen(.{}); + b.copy(b.setTmaBarrier(exec_a, stage.load_barrier), .{a_source}, .{stage.a_destination}, null); + b.copy(b.setTmaBarrier(exec_b, stage.load_barrier), .{b_source}, .{stage.b_destination}, null); + b.copy(b.setTmaBarrier(exec_sfa, stage.load_barrier), .{sfa_source}, .{stage.sfa_destination}, null); + b.copy(b.setTmaBarrier(exec_sfb, stage.load_barrier), .{sfb_source}, .{stage.sfb_destination}, null); +} + +fn barrierAt(b: *B, base: V, stage: V) V { + return b.addOffsetTyped( + base, + b.makeIntTuple(.{stage}), + b.ptrTy(.i64, .smem, 8) catch @panic("bad pipeline barrier pointer"), + ).value(); +} + +const PipelinePosition = struct { + index: V, + phase: V, +}; + +/// Scalar form of CUTLASS's StaticPersistentTileScheduler state for the +/// one-CTA DeepSeek specialization. Python carries the decoded M/N/L +/// coordinate, validity predicate, linear work index, and executed-tile +/// count through every role's `scf.while`. Keeping the same state shape is +/// important to the CuTe software-pipeline passes, even though N is always +/// zero for one tile per group. +const PersistentWork = struct { + m: V, + n: V, + group: V, + valid: V, + linear: V, + executed: V, +}; + +/// Direct routing (one token) runs one single-row group per route. Routes +/// owned by other expert-parallel ranks carry expert -1: the persistent tiles +/// cover the valid routes only, in route order, so they do not occupy a wave. +const DirectRoutes = struct { + expert_ids: cute.View, + routes: usize, + + fn count(self: DirectRoutes, b: *B) V { + var valid = b.cst(.i32, 0); + for (0..self.routes) |r| valid = valid.add(self.expertAt(b, r).ge(0).to(.i32)); + return valid; + } + + /// Route of the `k`-th valid route. + fn nth(self: DirectRoutes, b: *B, k: V) V { + var seen = b.cst(.i32, 0); + var route = b.cst(.i32, 0); + for (0..self.routes) |r| { + const valid = self.expertAt(b, r).ge(0); + route = b.select(valid.bitAnd(seen.eq(k)), b.cst(.i32, @as(i32, @intCast(r))), route); + seen = seen.add(valid.to(.i32)); + } + return route; + } + + fn expertAt(self: DirectRoutes, b: *B, r: usize) V { + return self.expert_ids.get(.{b.cst(.i32, @as(i32, @intCast(r)))}, .i32); + } +}; + +fn persistentWorkAt( + b: *B, + linear: V, + executed: V, + active: V, + tiles_m: V, + div_m: V, + div_n: V, + direct: ?DirectRoutes, +) PersistentWork { + // StaticPersistentTileScheduler uses two first-class FastDivmod objects + // to decode the column-major (M, N=1, L) problem layout. Preserve those + // operations even though ordinary rem/div is mathematically equivalent; + // CuTe's scheduler reconstruction and loop canonicalization key off these + // values and carry them through every persistent loop. + const group_and_m = b.fastDivmod(linear, div_m); + const group_and_n = b.fastDivmod(group_and_m[0], div_n); + const group = if (direct) |routes| routes.nth(b, group_and_n[0]) else group_and_n[0]; + return .{ + .m = b.makeWarpUniform(group_and_m[1]), + .n = b.makeWarpUniform(group_and_n[1]), + .group = b.makeWarpUniform(group), + .valid = b.makeWarpUniform(linear.lt(active.mul(tiles_m))), + .linear = linear, + .executed = executed, + }; +} + +fn advancePersistentWork( + b: *B, + linear: V, + executed: V, + persistent_clusters: V, + active: V, + tiles_m: V, + div_m: V, + div_n: V, + direct: ?DirectRoutes, +) PersistentWork { + return persistentWorkAt(b, linear.add(persistent_clusters), executed.add(1), active, tiles_m, div_m, div_n, direct); +} + +/// Equivalent to CUTLASS PipelineState.advance(). Keep the index and phase +/// as explicit loop-carried SSA values instead of recovering them with +/// division/remainder; the generated CuTe program then has the same circular +/// state transition as the Python DSL. +fn advancePipeline(b: *B, index: V, phase: V, comptime stages: usize) PipelinePosition { + const advanced = index.add(1); + const wraps = advanced.eq(stages); + const next_index = b.select(wraps, 0, advanced); + var next_phase = b.openIfElse(wraps, .{phase.type_()}); + next_phase.yieldThen(.{phase.bitXor(1)}); + next_phase.yieldElse(.{phase}); + return .{ .index = next_index, .phase = next_phase.results[0] }; +} + +fn peekBarrierIfRemaining(b: *B, base: V, count: V, limit: usize, state: PipelinePosition) V { + var peek = b.openIfElse(count.lt(limit), .{cute.DType.i1.toMlir(b.ctx)}); + peek.yieldThen(.{b.mbarrierWaitParity(barrierAt(b, base, state.index), state.phase)}); + peek.yieldElse(.{b.cst(.i1, true)}); + return peek.results[0]; +} + +fn accumulatorTmemAt(b: *B, comptime tile: Tile, tmem: V, stage: V) V { + // The stage offset is a whole tile width, which `cute.add_offset` needs to + // know: it derives the result alignment from what it knows of the offset. + return b.addOffsetTyped( + tmem, + b.makeIntTupleDivBy(stage.mul(tile.n), @intCast(tile.n)), + b.ptrTy(.f32, .tmem, 16) catch @panic("bad accumulator TMEM pointer"), + ).value(); +} + +/// `cute.add_offset` infers the alignment a pointer keeps as the gcd of the +/// base alignment and the offset, and `cute.recast_iter` may not change it, so +/// the field's own alignment has to be what the offset leaves. +fn sharedPointerAt( + b: *B, + base: V, + comptime byte_offset: i64, + comptime dtype: cute.DType, + comptime alignment: u64, +) V { + const moved = comptime movedAlignment(byte_offset); + comptime std.debug.assert(moved >= alignment); + const byte_ptr = b.addOffsetTyped( + base, + b.makeIntTuple(.{byte_offset}), + b.ptrTy(.i8, .smem, moved) catch @panic("bad shared byte pointer"), + ); + return b.recastPointer(byte_ptr, dtype, .smem, moved); +} + +/// Alignment of `shared_base + byte_offset`, which the shared storage is +/// allocated with. +fn movedAlignment(comptime byte_offset: i64) u64 { + return offsetAlignment(1024, byte_offset, 1); +} + +/// `cute.add_offset` keeps the gcd of the operand alignment and the offset, +/// and the op's result type has to say so. +fn offsetAlignment(base_bytes: u64, offset_elements: i64, element_bytes: u64) u64 { + if (offset_elements == 0) return base_bytes; + const moved = @as(u64, @intCast(@abs(offset_elements))) * element_bytes; + return @max(std.math.gcd(base_bytes, moved), element_bytes); +} + +/// Warp 5 runs independently over every persistent output tile. Keep the +/// producer index and phase as loop-carried values across tiles, exactly like +/// CUTLASS PipelineState. Only the per-tile count is reset to zero. +fn runTmaProducer( + b: *B, + comptime tile: Tile, + comptime k_tiles: usize, + storage: StageStorage, + active: V, + tiles_m: V, + expert_ids: cute.View, + direct: ?DirectRoutes, + partitioned_a_target: cute.View, + partitioned_b_target: cute.View, + exec_a: cute.Atom, + exec_b: cute.Atom, + exec_sfa: cute.Atom, + exec_sfb: cute.Atom, +) void { + const initial_index = b.cst(.i32, 0); + const initial_phase = b.cst(.i32, 1); + const initial_count = b.cst(.i32, 0); + const persistent_clusters = b.gridDim().z; + const cta_m = b.blockIdx().x; + const cta_n = b.blockIdx().y; + const cta_l = b.cst(.i32, 0); + const div_m = b.fastDivmodCreate(tiles_m); + const div_n = b.fastDivmodCreate(1); + const initial_work = persistentWorkAt(b, b.blockIdx().z, b.cst(.i32, 0), active, tiles_m, div_m, div_n, direct); + const carried = .{ + initial_work.m, initial_work.n, initial_work.group, initial_work.valid, + initial_count, initial_index, initial_phase, persistent_clusters, + initial_work.linear, cta_m, cta_n, cta_l, + initial_work.executed, tiles_m, active, div_m, + div_n, + }; + var tiles = b.openWhile(carried, typesOf(carried)); + tiles.yieldBefore(tiles.before_carried[3], tupleOf(tiles.before_carried)); + const tile_m = tiles.after_carried[0]; + const group = tiles.after_carried[2]; + const expert = expert_ids.get(.{group}, .i32); + const first_state: PipelinePosition = .{ + .index = tiles.after_carried[5], + .phase = tiles.after_carried[6], + }; + const first_ready = b.mbarrierWaitParity( + barrierAt(b, storage.mma_barriers, first_state.index), + first_state.phase, + ); + var stages = b.openFor( + 0, + k_tiles, + 1, + .{ first_ready, b.cst(.i32, 0), first_state.index, first_state.phase }, + ); + const k = stages.carried[1]; + const state: PipelinePosition = .{ + .index = stages.carried[2], + .phase = stages.carried[3], + }; + const buffers = stageBuffersAt(b, tile, storage, state.index); + var pending = b.openIf(stages.carried[0].eq(false)); + b.mbarrierTryWaitParity(buffers.mma_barrier, state.phase, 10_000_000); + pending.yieldThen(.{}); + issueTmaStage(b, tile, k_tiles, partitioned_a_target, partitioned_b_target, buffers, k, tile_m, group, expert, exec_a, exec_b, exec_sfa, exec_sfb); + const next_count = stages.carried[1].add(1); + const next_state = advancePipeline(b, state.index, state.phase, tile.stages); + const next_ready = peekBarrierIfRemaining(b, storage.mma_barriers, next_count, k_tiles, next_state); + stages.yield(.{ next_ready, next_count, next_state.index, next_state.phase }); + const next_work = advancePersistentWork( + b, + tiles.after_carried[8], + tiles.after_carried[12], + tiles.after_carried[7], + tiles.after_carried[14], + tiles.after_carried[13], + tiles.after_carried[15], + tiles.after_carried[16], + direct, + ); + tiles.yieldAfter(.{ + next_work.m, + next_work.n, + next_work.group, + next_work.valid, + stages.results[1], + stages.results[2], + stages.results[3], + tiles.after_carried[7], + next_work.linear, + tiles.after_carried[9], + tiles.after_carried[10], + tiles.after_carried[11], + next_work.executed, + tiles.after_carried[13], + tiles.after_carried[14], + tiles.after_carried[15], + tiles.after_carried[16], + }); + + // PipelineTmaUmma.producer_tail: drain every empty barrier before the TMA + // warp exits so no outstanding tcgen05 arrival outlives the CTA. + var tail_state: PipelinePosition = .{ + .index = tiles.results[5], + .phase = tiles.results[6], + }; + inline for (0..tile.stages) |stage| { + b.mbarrierTryWaitParity( + barrierAt(b, storage.mma_barriers, tail_state.index), + tail_state.phase, + 10_000_000, + ); + if (stage + 1 < tile.stages) { + tail_state = advancePipeline(b, tail_state.index, tail_state.phase, tile.stages); + } + } +} + +fn Tuple(comptime T: type, comptime len: usize) type { + return std.meta.Tuple(&([_]type{T} ** len)); +} + +fn arity(comptime T: type) usize { + return @typeInfo(T).@"struct".fields.len; +} + +/// The MLIR types of a tuple of values, as `openWhile` expects them. +fn typesOf(values: anytype) Tuple(*const mlir.Type, arity(@TypeOf(values))) { + const len = comptime arity(@TypeOf(values)); + var out: Tuple(*const mlir.Type, len) = undefined; + inline for (0..len) |i| out[i] = values[i].type_(); + return out; +} + +/// Forward a whole loop-carried array, as `yieldBefore` expects a tuple. +fn tupleOf(array: anytype) Tuple(@TypeOf(array[0]), array.len) { + var out: Tuple(@TypeOf(array[0]), array.len) = undefined; + inline for (0..array.len) |i| out[i] = array[i]; + return out; +} + +/// SM100 MXFP4 mainloop objects of one pipeline stage, produced by +/// `StagedMxfp4Mainloop.atStage`. The dependent CuTe result types are +/// centralized here; callers only provide tensors and the allocated +/// tensor-memory pointer. +const Mxfp4Mainloop = struct { + a: V, + b: V, + accumulator: V, + sfa: V, + sfb: V, + sfa_copy: cute.Atom, + sfb_copy: cute.Atom, + sfa_source: V, + sfb_source: V, + sfa_target: V, + sfb_target: V, + + pub fn copyScales(self: Mxfp4Mainloop, builder: *B) void { + builder.copy(self.sfa_copy, .{self.sfa_source}, .{self.sfa_target}, null); + builder.copy(self.sfb_copy, .{self.sfb_source}, .{self.sfb_target}, null); + } +}; + +/// Pipeline-wide SM100 MXFP4 mainloop objects. CUTLASS derives the UMMA and +/// S2T descriptors from the complete staged tensors once, then slices those +/// descriptors at the current PipelineState index. Keeping the stage mode in +/// the descriptor is required when the circular buffer wraps back to stage 0. +const StagedMxfp4Mainloop = struct { + a: cute.View, + b: cute.View, + sfa_source: cute.View, + sfb_source: cute.View, + sfa: V, + sfb: V, + sfa_copy: cute.Atom, + sfb_copy: cute.Atom, + sfa_target: V, + sfb_target: V, + /// MMA tile N: accumulator columns per stage. + n: i64, + + pub fn atStage(self: StagedMxfp4Mainloop, b: *B, index: V, accumulator_tmem: V) Mxfp4Mainloop { + const all = cute.AlgebraToken.all; + const a = b.sliceTyped( + self.a, + .{ all, all, all, index }, + b.parseType("!cute_nvgpu.smem_desc_view"), + ); + const rhs = b.sliceTyped( + self.b, + .{ all, all, all, index }, + b.parseType("!cute_nvgpu.smem_desc_view"), + ); + const sfa_stage = b.sliceTyped( + self.sfa_source, + .{ all, all, all, all, index }, + b.parseType("!cute_nvgpu.smem_desc_view"), + ); + const sfb_stage = b.sliceTyped( + self.sfb_source, + .{ all, all, all, all, index }, + b.parseType("!cute_nvgpu.smem_desc_view"), + ); + const source_grouped_layout = b.staticValue("!cute.layout<\"(((32,1,1,4),1),(1,1,1)):(((1,1,1,0),0),(0,0,0))\">"); + const source_grouped_ty = "!cute_nvgpu.smem_desc_view"; + const sfa_source = b.makeView( + b.getIter(sfa_stage, "!cute_nvgpu.smem_desc"), + source_grouped_layout, + source_grouped_ty, + ); + const sfb_source = b.makeView( + b.getIter(sfb_stage, "!cute_nvgpu.smem_desc"), + source_grouped_layout, + source_grouped_ty, + ); + const acc_payload = fmt(b, "((128,{d}),1,1):((65536,1),0,0)", .{self.n}); + const acc_layout = b.staticValue(fmt(b, "!cute.layout<\"{s}\">", .{acc_payload})); + const accumulator = b.makeView( + accumulator_tmem, + acc_layout, + fmt(b, "!cute.memref, \"{s}\">", .{acc_payload}), + ); + return .{ + .a = a.value(), + .b = rhs.value(), + .accumulator = accumulator.value(), + .sfa = self.sfa, + .sfb = self.sfb, + .sfa_copy = self.sfa_copy, + .sfb_copy = self.sfb_copy, + .sfa_source = sfa_source.value(), + .sfb_source = sfb_source.value(), + .sfa_target = self.sfa_target, + .sfb_target = self.sfb_target, + }; + } +}; + +/// Build the same pipeline-wide descriptors as Python's +/// `Sm100BlockScaledPersistentDenseGemmKernel`: `make_fragment_A/B` and the +/// S2T partitions see the complete staged tensors. `atStage` performs the +/// per-iteration slicing afterwards. +fn makeMxfp4StagedMainloop( + b: *B, + tiled_mma: cute.Atom, + comptime n: i64, + comptime stages: usize, + smem_a: V, + smem_b: V, + smem_sfa: V, + smem_sfb: V, + scale_tmem: V, +) StagedMxfp4Mainloop { + // Descriptor strides are in 16-byte units: a 128x128 unpacked-FP4 A + // stage is 16 KiB and an Nx128 FP8 B stage is N*128 bytes. + const frag_a = b.mmaMakeFragment( + tiled_mma, + smem_a, + 0, + fmt(b, "!cute_nvgpu.smem_desc_view", .{stages}), + ); + const frag_b = b.mmaMakeFragment( + tiled_mma, + smem_b, + 1, + fmt(b, "!cute_nvgpu.smem_desc_view", .{ stages, n * 8 }), + ); + + // TMEM columns: two N-wide accumulator stages, then 16 SFA and 16 SFB + // columns, matching num_accumulator_tmem_cols/num_sfa_tmem_cols in Python. + const sf_layout = b.staticValue("!cute.layout<\"((((32,4),4),(32,1)),1,4):((((262144,4),8388608),(0,0)),0,1)\">"); + const sfa_col = b.addOffset(scale_tmem, b.staticValue(fmt(b, "!cute.int_tuple<\"{d}\">", .{2 * n})), "!cute.ptr>"); + const sfb_col = b.addOffset(scale_tmem, b.staticValue(fmt(b, "!cute.int_tuple<\"{d}\">", .{2 * n + 16})), "!cute.ptr>"); + const sfa_ptr = b.recastIter(sfa_col, "!cute.ptr>"); + const sfb_ptr = b.recastIter(sfb_col, "!cute.ptr>"); + const sf_memref = "!cute.memref, \"((((32,4),4),(32,1)),1,4):((((262144,4),8388608),(0,0)),0,1)\">"; + const sfa = b.makeView(sfa_ptr, sf_layout, sf_memref); + const sfb = b.makeView(sfb_ptr, sf_layout, sf_memref); + + const smem_filtered_ty = fmt(b, "!cute.memref, \"((((32,4),1),(1,1)),1,4,{d}):((((16,4),0),(0,0)),0,1,512)\">", .{stages}); + const tmem_filtered_ty = "!cute.memref, \"((((32,4),4),(1,1)),1,4):((((262144,4),8388608),(0,0)),0,1)\">"; + const sfa_smem_filtered = b.filterZeros(smem_sfa, null, smem_filtered_ty); + const sfb_smem_filtered = b.filterZeros(smem_sfb, null, smem_filtered_ty); + const sfa_tmem_filtered = b.filterZeros(sfa, null, tmem_filtered_ty); + const sfb_tmem_filtered = b.filterZeros(sfb, null, tmem_filtered_ty); + + const copy_atom = b.makeAtom("!cute_nvgpu.atom.s2t_copy", .{}); + const copy_ty = "!cute.tiled_copy, layout_copy_tv = <\"(1,(32,(4,4),4)):(0,(1,(512,32),128))\">, tiler_mn = <\"[512:1;1:0;4:1]\">>"; + const sfa_copy = b.makeS2tCopy(copy_atom, sfa_tmem_filtered, copy_ty); + const sfb_copy = b.makeS2tCopy(copy_atom, sfb_tmem_filtered, copy_ty); + + const source_partition_ty = fmt(b, "!cute.memref, \"(((32,4,4,4),1),1,1,1,{d}):(((16,1,4,0),0),0,0,0,512)\">", .{stages}); + const source_desc_ty = fmt(b, "!cute_nvgpu.smem_desc_view", .{stages}); + const target_partition_ty = "!cute.memref, \"(((32,4,(4,4)),1),1,1,1):(((262144,1,(4,8388608)),0),0,0,0)\">"; + const target_grouped_ty = "!cute.memref, \"(((32,4,(4,4)),1),(1,1,1)):(((262144,1,(4,8388608)),0),(0,0,0))\">"; + const sfa_src_partition = b.tiledCopyPartition(sfa_copy, sfa_smem_filtered, .{0}, false, source_partition_ty); + const sfb_src_partition = b.tiledCopyPartition(sfb_copy, sfb_smem_filtered, .{0}, false, source_partition_ty); + const sfa_desc = b.s2tSmemDescriptor(sfa_copy, sfa_src_partition, source_desc_ty); + const sfb_desc = b.s2tSmemDescriptor(sfb_copy, sfb_src_partition, source_desc_ty); + const sfa_dst_partition = b.tiledCopyPartition(sfa_copy, sfa_tmem_filtered, .{0}, true, target_partition_ty); + const sfb_dst_partition = b.tiledCopyPartition(sfb_copy, sfb_tmem_filtered, .{0}, true, target_partition_ty); + const target_grouped_layout = b.staticValue("!cute.layout<\"(((32,4,(4,4)),1),(1,1,1)):(((262144,1,(4,8388608)),0),(0,0,0))\">"); + const sfa_dst = b.makeView(b.getIter(sfa_dst_partition, "!cute.ptr>"), target_grouped_layout, target_grouped_ty); + const sfb_dst = b.makeView(b.getIter(sfb_dst_partition, "!cute.ptr>"), target_grouped_layout, target_grouped_ty); + + return .{ + .a = frag_a, + .b = frag_b, + .sfa_source = sfa_desc, + .sfb_source = sfb_desc, + .sfa = sfa.value(), + .sfb = sfb.value(), + .sfa_copy = sfa_copy, + .sfb_copy = sfb_copy, + .sfa_target = sfa_dst.value(), + .sfb_target = sfb_dst.value(), + .n = n, + }; +} + +fn fmt(b: *B, comptime format: []const u8, args: anytype) []const u8 { + return std.fmt.allocPrint(b.arena.allocator(), format, args) catch @panic("OOM"); +} + +/// Warp 4 drains the A/B ring into one of two independent accumulator stages. +fn runMmaConsumer( + b: *B, + comptime tile: Tile, + comptime k_tiles: usize, + storage: StageStorage, + tiled_mma: cute.Atom, + tmem_holding: V, + active: V, + tiles_m: V, +) void { + const tmem = b.retrieveTmemPtr(tmem_holding, .f32, 16); + // Match Python's construction order: derive A/B UMMA fragments and S2T + // scale descriptors from the complete staged tensors once. The K loop + // slices these compiler-derived descriptors by PipelineState.index. + const staged = stagedOperands(b, tile, storage); + const staged_mainloop = makeMxfp4StagedMainloop( + b, + tiled_mma, + tile.n, + tile.stages, + staged.a.value(), + staged.b.value(), + staged.sfa.value(), + staged.sfb.value(), + tmem, + ); + const persistent_clusters = b.gridDim().z; + const div_m = b.fastDivmodCreate(tiles_m); + const div_n = b.fastDivmodCreate(1); + // The MMA warp never reads the group, only the tile count. + const initial_work = persistentWorkAt(b, b.blockIdx().z, b.cst(.i32, 0), active, tiles_m, div_m, div_n, null); + const carried = .{ + initial_work.m, initial_work.n, initial_work.group, initial_work.valid, + b.cst(.i32, 0), b.cst(.i32, 0), b.cst(.i32, 0), tiled_mma.value(), + b.cst(.i32, 0), b.cst(.i32, 0), b.cst(.i32, 1), persistent_clusters, + initial_work.linear, b.blockIdx().x, b.blockIdx().y, b.cst(.i32, 0), + initial_work.executed, tiles_m, active, div_m, + div_n, + }; + var tiles = b.openWhile(carried, typesOf(carried)); + tiles.yieldBefore(tiles.before_carried[3], tupleOf(tiles.before_carried)); + // Python peeks the first AB-full token before it blocks on an available + // accumulator stage. Preserve that ordering so the two independent + // pipelines enter the K loop with the same outstanding operations. + const first_state: PipelinePosition = .{ + .index = tiles.after_carried[5], + .phase = tiles.after_carried[6], + }; + const first_ready = b.mbarrierWaitParity( + barrierAt(b, storage.load_barriers, first_state.index), + first_state.phase, + ); + const accumulator_state: PipelinePosition = .{ + .index = tiles.after_carried[9], + .phase = tiles.after_carried[10], + }; + const accumulator_stage = accumulator_state.index; + const accumulator_empty = barrierAt(b, storage.accumulator_empty_barriers, accumulator_stage); + b.mbarrierTryWaitParity(accumulator_empty, accumulator_state.phase, 10_000_000); + + const persistent_mma: cute.Atom = .{ .inner = tiles.after_carried[7].inner, .kernel = b }; + const initial_mma = b.setMmaAccumulate(persistent_mma, b.cst(.i1, false)); + var stages = b.openFor( + 0, + k_tiles, + 1, + // Match PipelineTmaUmma's Python loop-carried ABI exactly. The + // software-pipeline pass recognizes the leading try-wait token and + // keeps the MMA atom immediately after it. + .{ first_ready, initial_mma.value(), b.cst(.i32, 0), first_state.index, first_state.phase }, + ); + const k = stages.carried[2]; + const state: PipelinePosition = .{ .index = stages.carried[3], .phase = stages.carried[4] }; + var pending = b.openIf(stages.carried[0].eq(false)); + b.mbarrierTryWaitParity(barrierAt(b, storage.load_barriers, state.index), state.phase, 10_000_000); + pending.yieldThen(.{}); + const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, accumulator_stage); + const mainloop = staged_mainloop.atStage(b, state.index, accumulator_tmem); + mainloop.copyScales(b); + const loop_mma: cute.Atom = .{ .inner = stages.carried[1].inner, .kernel = b }; + const mma = b.setMmaAccumulate(loop_mma, k.ne(0)); + b.gemm( + mma, + mainloop.accumulator, + .{ mainloop.a, mainloop.sfa }, + .{ mainloop.b, mainloop.sfb }, + mainloop.accumulator, + ); + var commit_lane = b.openIf(b.electSync()); + b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index)); + commit_lane.yieldThen(.{}); + const next_count = stages.carried[2].add(1); + const next_state = advancePipeline(b, state.index, state.phase, tile.stages); + const next_ready = peekBarrierIfRemaining(b, storage.load_barriers, next_count, k_tiles, next_state); + stages.yield(.{ next_ready, mma.value(), next_count, next_state.index, next_state.phase }); + + var full_lane = b.openIf(b.electSync()); + b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage)); + full_lane.yieldThen(.{}); + const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); + const next_acc_count = tiles.after_carried[8].add(1); + const next_work = advancePersistentWork( + b, + tiles.after_carried[12], + tiles.after_carried[16], + tiles.after_carried[11], + tiles.after_carried[18], + tiles.after_carried[17], + tiles.after_carried[19], + tiles.after_carried[20], + null, + ); + tiles.yieldAfter(.{ + next_work.m, + next_work.n, + next_work.group, + next_work.valid, + stages.results[2], + stages.results[3], + stages.results[4], + stages.results[1], + next_acc_count, + next_accumulator_state.index, + next_accumulator_state.phase, + tiles.after_carried[11], + next_work.linear, + tiles.after_carried[13], + tiles.after_carried[14], + tiles.after_carried[15], + next_work.executed, + tiles.after_carried[17], + tiles.after_carried[18], + tiles.after_carried[19], + tiles.after_carried[20], + }); + + // Match PipelineUmmaAsync.producer_tail. The state points at the next + // accumulator slot; advance to the last slot used by this producer and + // wait until the epilogue has released it before the MMA warp exits. + var accumulator_tail: PipelinePosition = .{ + .index = tiles.results[9], + .phase = tiles.results[10], + }; + inline for (0..accumulator_stages - 1) |_| { + accumulator_tail = advancePipeline(b, accumulator_tail.index, accumulator_tail.phase, accumulator_stages); + } + b.mbarrierTryWaitParity( + barrierAt(b, storage.accumulator_empty_barriers, accumulator_tail.index), + accumulator_tail.phase, + 10_000_000, + ); +} + +fn view(b: *B, value: V) cute.View { + return .{ .inner = value.inner, .kernel = b }; +} + +/// Epilogue operands; which fields are set depends on the `Epilogue`. +const EpilogueOperands = struct { + group_sizes: cute.View, + route_inverse: cute.View, + routing_weights: ?cute.View = null, + q: ?cute.View = null, + s: ?cute.View = null, + rows: ?cute.View = null, + direct: bool, + /// `Config.groupRows`. + group_rows: i64 = 0, + /// `Config.swiglu_limit`. + swiglu_limit: f32 = 10.0, +}; + +/// Warps 0-3 drain each TMEM accumulator stage into registers, release it to +/// the MMA warp, then write the tile out according to `epilogue`. +fn runEpilogue( + b: *B, + comptime tile: Tile, + comptime epilogue: Epilogue, + storage: StageStorage, + tmem_holding: V, + active: V, + tiles_m: V, + direct: ?DirectRoutes, + operands: EpilogueOperands, +) void { + const n = tile.n; + const tid = b.threadIdx().x; + const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); + const tmem = b.retrieveTmemPtr(tmem_holding, .f32, 16); + + const div_m = b.fastDivmodCreate(tiles_m); + const div_n = b.fastDivmodCreate(1); + const initial_work = persistentWorkAt(b, b.blockIdx().z, b.cst(.i32, 0), active, tiles_m, div_m, div_n, direct); + const carried = .{ + initial_work.m, initial_work.n, initial_work.group, initial_work.valid, + b.cst(.i32, 0), b.cst(.i32, 0), b.cst(.i32, 0), initial_work.linear, + initial_work.executed, + }; + var tiles = b.openWhile(carried, .{ + initial_work.m.type_(), + initial_work.n.type_(), + initial_work.group.type_(), + cute.DType.i32.toMlir(b.ctx), + cute.DType.i32.toMlir(b.ctx), + cute.DType.i32.toMlir(b.ctx), + initial_work.linear.type_(), + initial_work.executed.type_(), + }); + tiles.yieldBefore(tiles.before_carried[3], .{ + tiles.before_carried[0], + tiles.before_carried[1], + tiles.before_carried[2], + tiles.before_carried[4], + tiles.before_carried[5], + tiles.before_carried[6], + tiles.before_carried[7], + tiles.before_carried[8], + }); + const tile_m = tiles.after_carried[0]; + const group = tiles.after_carried[2]; + const accumulator_state: PipelinePosition = .{ + .index = tiles.after_carried[4], + .phase = tiles.after_carried[5], + }; + const stage = accumulator_state.index; + b.mbarrierTryWaitParity(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, 10_000_000); + + const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, stage); + const warp_tmem = b.addOffsetTyped( + accumulator_tmem, + b.makeIntTupleDivBy(warp.mul(32 * 65536), 4), + b.ptrTy(.f32, .tmem, 16) catch @panic("bad epilogue TMEM pointer"), + ).value(); + // A warp lane holds one routed row, so tiles wider than a warp are drained + // 32 rows at a time. The accumulator stage is released once its last chunk + // is in registers. + const chunk_rows: usize = comptime @intCast(tile.chunkRows()); + const tile_chunks = comptime tile.chunks(); + inline for (0..tile_chunks) |chunk| { + const chunk_tmem = if (chunk == 0) warp_tmem else b.addOffsetTyped( + warp_tmem, + b.makeIntTuple(.{@as(i32, @intCast(chunk * chunk_rows))}), + b.ptrTy(.f32, .tmem, offsetAlignment(16, @intCast(chunk * chunk_rows), 4)) catch @panic("bad epilogue TMEM pointer"), + ).value(); + // Route metadata does not depend on the accumulator. + const routed = routedLanes(b, tile, operands, group, chunk); + const values = b.tmemLoadVector(chunk_tmem, @intCast(chunk_rows), .{ .num_dp = 32, .num_b = 32, .num_rep = @intCast(chunk_rows) }); + b.fenceTmemLoad(); + if (chunk + 1 == tile_chunks) { + // Registers now own the accumulator values, so all 128 epilogue + // threads release this TMEM stage before the stores. + b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), 1); + } + var columns: [@intCast(@min(n, 32))]V = undefined; + inline for (&columns, 0..) |*column, i| column.* = b.vectorExtract(values, i, .i32).bitCast(.f32); + switch (epilogue) { + .swiglu_mxfp8 => storeTileSwiglu(b, tile, storage, operands, routed, tile_m, group, &columns, chunk), + .route_rows => storeTileRows(b, tile, operands, routed, tile_m, &columns, chunk), + } + } + const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); + const next_work = advancePersistentWork(b, tiles.after_carried[6], tiles.after_carried[7], b.gridDim().z, active, tiles_m, div_m, div_n, direct); + tiles.yieldAfter(.{ + next_work.m, + next_work.n, + next_work.group, + next_work.valid, + tiles.after_carried[3].add(1), + next_accumulator_state.index, + next_accumulator_state.phase, + next_work.linear, + next_work.executed, + }); + + var release_warp = b.openIf(warp.eq(0)); + // Permit release and deallocation are warp collective as well. + b.relinquishTmemAllocPermit(.{}); + release_warp.yieldThen(.{}); + b.namedBarrier(1, 128); + var dealloc_warp = b.openIf(warp.eq(0)); + b.deallocTmem(tmem, tile.tmem_columns, .{}); + dealloc_warp.yieldThen(.{}); +} + +/// E8M0 exponent of the power-of-two MX scale for a block maximum, rounded +/// up so that `maximum / scale <= 448`, and the matching inverse scale. +fn mxScale(b: *B, maximum: V) struct { exponent: V, inverse: V } { + const raw_bits = maximum.maximum(1e-4).mul(1.0 / 448.0).bitCast(.i32); + const exponent = raw_bits.shrLogical(23).add(raw_bits.bitAnd(0x7fffff).ne(0).to(.i32)); + return .{ .exponent = exponent, .inverse = b.cst(.i32, 254).sub(exponent).shl(23).bitCast(.f32) }; +} + +/// Per-row routing data of one expert group, spread across a warp: lane `l` +/// holds routed row `l` of the group. Columns index it with `shuffleIdx`. +const RoutedLanes = struct { + count: V, + /// Token/top-k route of the lane's row; 0 for padding rows. + route: V, + /// Routing weight of the lane's row (up projection only). + weight: ?V, +}; + +fn routedLanes(b: *B, comptime tile: Tile, operands: EpilogueOperands, group: V, comptime chunk: usize) RoutedLanes { + // Only the up projection applies routing weights. + const weights = operands.routing_weights; + if (operands.direct) return .{ + .count = b.cst(.i32, 1), + .route = group, + .weight = if (weights) |w| w.get(.{group}, .f32) else null, + }; + const lane = b.threadIdx().x.rem(32); + const count = operands.group_sizes.get(.{group}, .i32); + // Lanes past the tile width alias row 0, which is always live. Wide tiles + // run one 32-row chunk at a time, so lane `l` holds row `32 * chunk + l`. + const chunk_rows = comptime tile.chunkRows(); + const in_chunk = b.select(lane.lt(@as(i32, @intCast(chunk_rows))), lane, b.cst(.i32, 0)); + const row_in_group = if (chunk == 0) in_chunk else in_chunk.add(@as(i32, @intCast(@as(i64, @intCast(chunk)) * chunk_rows))); + const live = row_in_group.lt(count); + const inverse = operands.route_inverse.get(.{group.mul(tile.n).add(row_in_group)}, .i32); + const route = b.select(live, inverse, b.cst(.i32, 0)); + const weight = if (weights) |w| w.get(.{route}, .f32) else null; + return .{ .count = count, .route = route, .weight = weight }; +} + +/// Python `fused_epilogue`: thread `t` owns output row `128 * tile_m + t` +/// of the interleaved projection, so even threads hold a gate and their odd +/// neighbour the matching up value. Each even thread computes one SwiGLU +/// activation per routed row; 32 consecutive activations span two warps, +/// whose partial maxima meet in shared memory. +/// +/// At decode sizes most rows of a group are padding, whose outputs are never +/// read. `count` is uniform across the CTA, so whole 8-row chunks of padding +/// are skipped; per-row branches would serialize the shuffle chains of +/// independent rows. Do not duplicate this body under a full-group branch: +/// ptxas then loses track of convergence at the tile loop's warp-uniform +/// shuffles, and lanes still outside the gate-lane stores read garbage. +fn storeTileSwiglu(b: *B, comptime tile: Tile, storage: StageStorage, operands: EpilogueOperands, routed: RoutedLanes, tile_m: V, group: V, columns: []const V, comptime chunk: usize) void { + // Direct routing has exactly one live row per group. + if (operands.direct) + swigluRows(b, tile, storage, operands, routed, tile_m, group, columns, 1, false, chunk) + else + swigluRows(b, tile, storage, operands, routed, tile_m, group, columns, comptime @intCast(tile.chunkRows()), true, chunk); +} + +const swiglu_chunk = 8; + +/// The first `live` rows of `storeTileSwiglu`; with `skip_padding`, chunks +/// of rows past `count` are skipped. +fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: EpilogueOperands, routed: RoutedLanes, tile_m: V, group: V, columns: []const V, comptime live: usize, comptime skip_padding: bool, comptime chunk: usize) void { + const n = comptime tile.chunkRows(); + // First routed row of this epilogue pass. + const chunk_first: i32 = comptime @intCast(chunk * @as(usize, @intCast(tile.chunkRows()))); + const tid = b.threadIdx().x; + const lane = tid.rem(32); + const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); + const is_gate = tid.rem(2).eq(0); + const first_row = group.mul(operands.group_rows); + const f32_type = cute.DType.f32.toMlir(b.ctx); + // The TMA C staging buffer is unused by this epilogue. + const scratch_layout = b.layoutSpec(.{ 4, n }, .{ n, 1 }); + const scratch = b.makeViewTyped(storage.c, b.staticLayout(scratch_layout), b.memrefType(.f32, .smem, 1024, b.layoutType(scratch_layout))); + const row_chunks = std.math.divCeil(usize, live, swiglu_chunk) catch unreachable; + + var activations: [@intCast(n)]V = undefined; + // Lane `c` collects the warp maximum of column `c`. + var lane_maximum = b.cst(.f32, 0); + for (0..row_chunks) |row_chunk| { + const first = row_chunk * swiglu_chunk; + const last = @min(first + swiglu_chunk, live); + // Row 0 of an active group is always live. + const guarded = skip_padding and (row_chunk > 0 or chunk > 0); + const types: [swiglu_chunk + 1]*const mlir.Type = @splat(f32_type); + // `@TypeOf` does not evaluate its operand. + var guard: ?@TypeOf(b.openIfElse(routed.count, tupleOf(types))) = if (guarded) b.openIfElse(routed.count.gt(chunk_first + @as(i32, @intCast(first))), tupleOf(types)) else null; + var chunk_maximum = lane_maximum; + for (columns[first..last], activations[first..last], first..) |value, *activation, column| { + const c: i32 = @intCast(column); + // The reference model rounds the projection to BF16. + const limit = operands.swiglu_limit; + const gate = value.to(.bf16).to(.f32).minimum(limit); + const up_value = b.shuffleXor(value, 1).to(.bf16).to(.f32).maximum(-limit).minimum(limit); + // up * SiLU(gate), with the hardware approximations Python's + // fastmath selects: exp(-g) = 2^(-g * log2 e). + const exp_neg = b.unaryF32("ex2.approx.ftz.f32", gate.mul(-std.math.log2e)); + var result = up_value.mul(gate).mul(b.unaryF32("rcp.approx.ftz.f32", exp_neg.add(1.0))); + result = result.mul(b.shuffleIdx(routed.weight.?, c)).to(.bf16).to(.f32); + // Odd threads hold no activation and must not raise the maxima. + activation.* = b.select(is_gate, result, b.cst(.f32, 0)); + const maximum = b.warpMaxNonNegative(activation.*.abs()); + chunk_maximum = b.select(lane.eq(c), maximum, chunk_maximum); + } + if (guard) |*g| { + var then_values: [swiglu_chunk + 1]V = @splat(chunk_maximum); + for (activations[first..last], then_values[0 .. last - first]) |activation, *t| t.* = activation; + g.yieldThen(tupleOf(then_values)); + // Padding chunk: zero activations, maxima unchanged. The zero is + // emitted in the else block, where it is used. + var else_values: [swiglu_chunk + 1]V = @splat(b.cst(.f32, 0)); + else_values[swiglu_chunk] = lane_maximum; + g.yieldElse(tupleOf(else_values)); + for (activations[first..last], g.results[0 .. last - first]) |*activation, result| activation.* = result; + lane_maximum = g.results[swiglu_chunk]; + } else { + lane_maximum = chunk_maximum; + } + } + var owner = b.openIf(lane.lt(@as(i32, @intCast(n)))); + scratch.set(.{ warp, lane }, lane_maximum); + owner.yieldThen(.{}); + b.namedBarrier(1, 128); + + // Lane `c` turns the two half-block maxima of column `c` into its scale. + const pair = warp.bitAnd(~@as(i32, 1)); + const lane_column = b.select(lane.lt(@as(i32, @intCast(n))), lane, b.cst(.i32, 0)); + const scale = mxScale(b, scratch.get(.{ pair, lane_column }, .f32).maximum(scratch.get(.{ pair.add(1), lane_column }, .f32))); + // The next tile reuses the scratch maxima. + b.namedBarrier(1, 128); + + const activation_index = tile_m.mul(64).add(tid.div(2)); + for (0..row_chunks) |row_chunk| { + const first = row_chunk * swiglu_chunk; + const last = @min(first + swiglu_chunk, live); + var guard: ?@TypeOf(b.openIf(routed.count)) = if (skip_padding and (chunk > 0 or first > 0)) b.openIf(routed.count.gt(chunk_first + @as(i32, @intCast(first)))) else null; + var inverses: [swiglu_chunk]V = undefined; + for (inverses[0 .. last - first], first..) |*inverse, column| inverse.* = b.shuffleIdx(scale.inverse, @as(i32, @intCast(column))); + var gate_lane = b.openIf(is_gate); + for (activations[first..last], inverses[0 .. last - first], first..) |activation, inverse, column| { + const quantized = activation.mul(inverse).maximum(-448.0).minimum(448.0).to(.f8e4m3fn); + operands.q.?.set(.{ first_row.add(chunk_first + @as(i32, @intCast(column))), activation_index }, quantized); + } + gate_lane.yieldThen(.{}); + if (guard) |*g| g.yieldThen(.{}); + } + + // Warps 0 and 2 each own one 32-activation block of this tile. + var scale_lane = b.openIf(warp.rem(2).eq(0).bitAnd(lane.add(chunk_first).lt(routed.count))); + const block = tile_m.mul(2).add(warp.div(2)); + const swizzled = block.div(4).mul(512).add(lane.mul(16)).add(chunk * 4).add(block.rem(4)); + operands.s.?.set(.{ group, swizzled }, scale.exponent.to(.i8)); + scale_lane.yieldThen(.{}); +} + +/// Python `down_epilogue`: store each live routed row at its token/top-k +/// route, so the final reduction reads rows in route order. +fn storeTileRows(b: *B, comptime tile: Tile, operands: EpilogueOperands, routed: RoutedLanes, tile_m: V, columns: []const V, comptime chunk: usize) void { + const chunk_first: i32 = comptime @intCast(chunk * @as(usize, @intCast(tile.chunkRows()))); + const output_column = tile_m.mul(128).add(b.threadIdx().x); + if (operands.direct) { + operands.rows.?.set(.{ routed.route, output_column }, columns[0].to(.bf16)); + return; + } + var routes: [32]V = undefined; + for (routes[0..columns.len], 0..) |*route, column| route.* = b.shuffleIdx(routed.route, @as(i32, @intCast(column))); + for (columns, routes[0..columns.len], 0..) |value, route, column| { + var live: ?@TypeOf(b.openIf(routed.count)) = if (column == 0 and chunk == 0) null else b.openIf(routed.count.gt(chunk_first + @as(i32, @intCast(column)))); + operands.rows.?.set(.{ route, output_column }, value.to(.bf16)); + if (live) |*l| l.yieldThen(.{}); + } +} + +fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: Epilogue, d: anytype) void { + const L = Layouts(tile); + const n = tile.n; + const stages = tile.stages; + // Allocate the complete Python SharedStorage object as one compiler-visible + // memref. `cute.kernel_smem_size` only accounts for memref allocations; + // treating every field as an architecture pointer left the launch at zero + // dynamic shared bytes and made the first TMA instruction fault. + const shared = b.allocSmemStorage( + .i8, + 1024, + b.layoutType(b.layoutSpec(.{tile.smem_bytes}, .{1})), + 1, + tile.fields(b), + ); + const shared_base = b.getIterTyped( + shared, + b.ptrTy(.i8, .smem, 1024) catch @panic("bad shared base pointer"), + ).value(); + + const tmem_holding = sharedPointerAt(b, shared_base, tile.tmem_holding, .i32, 8); + const storage: StageStorage = .{ + .c = sharedPointerAt(b, shared_base, tile.c, .f32, 1024), + .a = sharedPointerAt(b, shared_base, tile.a, .i8, 1024), + .b = sharedPointerAt(b, shared_base, tile.b, .f8e4m3fn, 1024), + .sfa = sharedPointerAt(b, shared_base, tile.sfa, .f8e8m0fnu, 1024), + .sfb = sharedPointerAt(b, shared_base, tile.sfb, .f8e8m0fnu, 1024), + .load_barriers = sharedPointerAt(b, shared_base, 0, .i64, 1024), + .mma_barriers = sharedPointerAt(b, shared_base, tile.empty_barriers, .i64, 8), + .accumulator_full_barriers = sharedPointerAt(b, shared_base, tile.accumulator_full, .i64, 16), + .accumulator_empty_barriers = sharedPointerAt(b, shared_base, tile.accumulator_empty, .i64, 16), + }; + + const tid = b.threadIdx().x; + const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); + const tiled_mma: cute.Atom = .{ .inner = d.mma.inner, .kernel = b }; + + // Reproduce the Python CuTe path for the two matrix operands: + // coordinate tensor -> local_tile -> tiled_mma.partition -> group modes + // -> tma_partition. All dependent result types come from Zig layout + // objects; this kernel does not embed textual MLIR. + const m_tiles = @divExact(cfg.m, 128); + const k_tiles = @divExact(cfg.k, 128); + const a_local_spec = b.layoutSpec( + .{ 128, 128, m_tiles, k_tiles, cfg.experts }, + .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(128, 1, .{1}), b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, + ); + // The group tile covers all of B's rows, so the N-tile mode has extent one + // and its stride is inferred as zero. Direct routing is no different: the + // tensor map still spans a whole tile of rows, zero-filling all but one. + const b_local_spec = b.layoutSpec( + .{ n, 128, 1, k_tiles, cfg.groups }, + .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), 0, b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, + ); + const all_tiles = .{ cute.AlgebraToken.all, cute.AlgebraToken.all, cute.AlgebraToken.all }; + const tiled_a = b.localTileTyped(d.gA, b.makeTile(&.{ 128, 128 }), all_tiles, b.coordTensorType(3, a_local_spec), null); + const tiled_b = b.localTileTyped(d.gB, b.makeTile(&.{ n, 128 }), all_tiles, b.coordTensorType(3, b_local_spec), null); + + const a_mma_spec = b.layoutSpec( + .{ .{ 128, 32 }, 1, 4, m_tiles, k_tiles, cfg.experts }, + .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(32, 1, .{0}), b.basis(128, 1, .{1}), b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, + ); + const b_mma_spec = b.layoutSpec( + .{ .{ n, 32 }, 1, 4, 1, k_tiles, cfg.groups }, + .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(32, 1, .{0}), 0, b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, + ); + const mma_a = b.tiledMmaPartitionTyped(tiled_mma, tiled_a, .{0}, 0, b.coordTensorType(3, a_mma_spec)); + const mma_b = b.tiledMmaPartitionTyped(tiled_mma, tiled_b, .{0}, 1, b.coordTensorType(3, b_mma_spec)); + + const a_grouped_layout = b.staticLayout(b.layoutSpec( + .{ .{ .{ 128, 32 }, 1, 4 }, m_tiles, k_tiles, cfg.experts }, + .{ .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(32, 1, .{0}) }, b.basis(128, 1, .{1}), b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, + )); + const b_grouped_layout = b.staticLayout(b.layoutSpec( + .{ .{ .{ n, 32 }, 1, 4 }, 1, k_tiles, cfg.groups }, + .{ .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(32, 1, .{0}) }, b.basis(n, 1, .{1}), b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, + )); + const staged = stagedOperands(b, tile, storage); + const smem_a_grouped = b.withLayout(staged.a, b.staticLayout(b.layoutSpec( + .{ .{ .{ 128, 32 }, 1, 4 }, stages }, + .{ .{ .{ 128, 1 }, 0, 32 }, Tile.a_stage_bytes }, + ))); + const smem_b_grouped = b.withLayout(staged.b, b.staticLayout(b.layoutSpec( + .{ .{ .{ n, 32 }, 1, 4 }, stages }, + .{ .{ .{ 128, 1 }, 0, 32 }, tile.bStageBytes() }, + ))); + const one_cta = b.staticLayout(b.layoutSpec(.{1}, .{0})); + const a_partition_layout = b.layoutSpec(.{ .{ Tile.a_stage_bytes, 1 }, stages }, .{ .{ 1, 0 }, Tile.a_stage_bytes }); + const b_partition_layout = b.layoutSpec(.{ .{ tile.bStageBytes(), 1 }, stages }, .{ .{ 1, 0 }, tile.bStageBytes() }); + const a_target_layout = b.layoutSpec( + .{ .{ .{ 128, 128 }, 1 }, m_tiles, k_tiles, cfg.experts }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(128, 1, .{1}), b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, + ); + const b_target_layout = b.layoutSpec( + .{ .{ .{ 128, n }, 1 }, 1, k_tiles, cfg.groups }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(n, 1, .{1}), b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, + ); + const partitioned_a = b.tmaPartitionTyped( + .{ .inner = d.tma_a.inner, .kernel = b }, + .{0}, + one_cta, + smem_a_grouped, + .{b.withLayout(mma_a, a_grouped_layout)}, + b.memrefTypeFromPointer(b.swizzledPtrTy(.i8, .smem, 1024, L.ab_swizzle) catch @panic("bad smem pointer"), b.layoutType(a_partition_layout)), + &.{b.coordTensorType(3, a_target_layout)}, + ); + const partitioned_b = b.tmaPartitionTyped( + .{ .inner = d.tma_b.inner, .kernel = b }, + .{0}, + one_cta, + smem_b_grouped, + .{b.withLayout(mma_b, b_grouped_layout)}, + b.memrefTypeFromPointer(b.swizzledPtrTy(.f8e4m3fn, .smem, 1024, L.ab_swizzle) catch @panic("bad smem pointer"), b.layoutType(b_partition_layout)), + &.{b.coordTensorType(3, b_target_layout)}, + ); + + const exec_a = b.makeExecTmaTyped(.{ .inner = d.tma_a.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaA(b, cfg))); + const exec_b = b.makeExecTmaTyped(.{ .inner = d.tma_b.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaB(b, cfg))); + const exec_sfa = b.makeExecTmaTyped(.{ .inner = d.tma_sfa.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaSf(b))); + const exec_sfb = b.makeExecTmaTyped(.{ .inner = d.tma_sfb.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaSf(b))); + + const expert_ids: cute.View = .{ .inner = d.expert_ids.inner, .kernel = b }; + const active_count: cute.View = .{ .inner = d.active_count.inner, .kernel = b }; + const direct: ?DirectRoutes = if (cfg.direct) .{ .expert_ids = expert_ids, .routes = @intCast(cfg.groups) } else null; + const active = if (direct) |routes| routes.count(b) else active_count.get(.{0}, .i32); + + // Match CUTLASS PipelineTmaUmma/PipelineUmmaAsync initialization. Warp 0 + // owns initialization, and one elected lane initializes each complete + // barrier array. mbarrier.init is not warp collective: issuing it from + // every lane races on the same shared state and breaks the first phase + // transition when the A/B ring wraps. + var init_warp = b.openIf(warp.eq(0)); + const barrier_arrays = .{ + .{ storage.load_barriers, stages, 1 }, + .{ storage.mma_barriers, stages, 1 }, + .{ storage.accumulator_full_barriers, accumulator_stages, 1 }, + .{ storage.accumulator_empty_barriers, accumulator_stages, 128 }, + }; + inline for (barrier_arrays) |array| { + var lane = b.openIf(b.electSync()); + inline for (0..array[1]) |stage| { + b.mbarrierInit(barrierAt(b, array[0], b.cst(.i32, stage)), array[2]); + } + lane.yieldThen(.{}); + } + b.fenceMbarrierInit(); + init_warp.yieldThen(.{}); + b.syncThreads(); + + // Each role walks the same persistent tile sequence independently. The + // AB and accumulator mbarriers carry all cross-warp dependencies. + var tma_warp = b.openIf(warp.eq(5)); + switch (k_tiles) { + inline 18, 40 => |kt| runTmaProducer(b, tile, kt, storage, active, d.tiles_m, expert_ids, direct, partitioned_a.targets[0], partitioned_b.targets[0], exec_a, exec_b, exec_sfa, exec_sfb), + else => std.debug.panic("unsupported MXFP4 reduction tile count {d}", .{k_tiles}), + } + tma_warp.yieldThen(.{}); + + var mma_warp = b.openIf(warp.eq(4)); + // Python's MMA role reaches its own TmemAllocator.wait_for_alloc site. + b.namedBarrier(2, 160); + switch (k_tiles) { + inline 18, 40 => |kt| runMmaConsumer(b, tile, kt, storage, tiled_mma, tmem_holding, active, d.tiles_m), + else => unreachable, + } + mma_warp.yieldThen(.{}); + + var epilogue_warps = b.openIf(warp.lt(4)); + // TmemAllocator.allocate is issued collectively by warp 0. The four + // epilogue warps then meet the MMA warp at named barrier 2. + var alloc_warp = b.openIf(warp.eq(0)); + b.allocTmem(tile.tmem_columns, tmem_holding, .{}); + alloc_warp.yieldThen(.{}); + b.namedBarrier(2, 160); + switch (epilogue) { + .swiglu_mxfp8 => runEpilogue(b, tile, epilogue, storage, tmem_holding, active, d.tiles_m, direct, .{ + .group_sizes = view(b, d.group_sizes), + .route_inverse = view(b, d.route_inverse), + .routing_weights = view(b, d.routing_weights), + .q = view(b, d.q), + .s = view(b, d.s), + .direct = cfg.direct, + .group_rows = cfg.groupRows(), + .swiglu_limit = cfg.swiglu_limit, + }), + .route_rows => runEpilogue(b, tile, epilogue, storage, tmem_holding, active, d.tiles_m, direct, .{ + .group_sizes = view(b, d.group_sizes), + .route_inverse = view(b, d.route_inverse), + .rows = view(b, d.rows), + .direct = cfg.direct, + }), + } + epilogue_warps.yieldThen(.{}); +} + +fn buildHostLaunch(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: Epilogue) cute.FinishError!void { + const L = Layouts(tile); + const host_arg = HostArgs(epilogue); + const n = tile.n; + const a_layout = b.layoutSpec(.{ cfg.m, cfg.k, cfg.experts }, .{ cfg.k, 1, cfg.m * cfg.k }); + const b_layout = b.layoutSpec(.{ cfg.groupRows(), cfg.k, cfg.groups }, .{ cfg.k, 1, cfg.k * cfg.groupRows() }); + // `cute.tile_to_shape` derives that the dynamic strides step whole + // 512-byte scale atoms, and the result type has to carry it. + const sf_stride: cute.ConstrainedDynamic = .{ .divisible_by = Tile.sf_stage_bytes }; + const dynamic_sf_layout = b.layoutSpec( + .{ .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ 1, cute.AlgebraToken.dynamic } }, + .{ .{ .{ 16, 4 }, sf_stride }, .{ .{ 0, 1 }, 512 }, .{ 0, sf_stride } }, + ); + + const gA = b.makeTensorView(b.recastPointer(b.arg(host_arg.weight), .f4e2m1fn, .gmem, 16), b.staticLayout(a_layout)); + // `schedule` is `[group experts | group sizes | active count]`. + const schedule = b.arg(host_arg.schedule); + const ids = b.makeTensorView(schedule, b.staticLayout(b.layoutSpec(cfg.groups, 1))); + const active_ptr = b.addOffsetTyped( + schedule, + b.makeIntTuple(.{2 * cfg.groups}), + b.ptrTy(.i32, .gmem, offsetAlignment(16, 2 * cfg.groups, 4)) catch @panic("bad active count pointer"), + ); + const active = b.makeTensorView(active_ptr, b.staticLayout(b.layoutSpec(1, 1))); + const gB = b.makeTensorView(b.arg(host_arg.input_quant), b.staticLayout(b_layout)); + // Match Python's host construction instead of materializing these two + // layouts with `cute.static`. In particular, SFB's M extent is one; CuTe + // must still retain that mode while encoding the tensor map. + const sf_atom_layout = b.staticLayout(b.layoutSpec( + .{ .{ 32, 4 }, .{ 32, 4 } }, + .{ .{ 16, 4 }, .{ 0, 1 } }, + )); + // The Zig dialect binding exposes the zero-based form of Python CuTe's + // `(2, 1, 3)` tile-to-shape order. + const sf_order = b.makeIntTuple(.{ 1, 0, 2 }); + const sfa_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_m), b.arg(host_arg.problem_k), b.arg(host_arg.expert_count) }, b.parseType("!cute.shape<\"(?,?,?)\">")); + const sfb_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_n), b.arg(host_arg.problem_k), b.arg(host_arg.problem_l) }, b.parseType("!cute.shape<\"(?,?,?)\">")); + const dynamic_sfa_layout = b.tileToShapeTyped(sf_atom_layout, sfa_shape, sf_order, b.layoutType(dynamic_sf_layout)); + const dynamic_sfb_layout = b.tileToShapeTyped(sf_atom_layout, sfb_shape, sf_order, b.layoutType(dynamic_sf_layout)); + const gSFA = b.makeTensorView(b.recastPointer(b.arg(host_arg.weight_scale), .f8e8m0fnu, .gmem, 16), dynamic_sfa_layout); + const gSFB = b.makeTensorView(b.recastPointer(b.arg(host_arg.input_scale), .f8e8m0fnu, .gmem, 16), dynamic_sfb_layout); + + const zero_tmem = b.intToPtr(0, .f8e8m0fnu, .tmem, 1); + const mma = b.makeBlockScaledMma(.{ .n = @intCast(n) }, zero_tmem); + const mma_sfb = b.makeBlockScaledMma(.{ .n = 128 }, zero_tmem); + + const a_smem = b.makeComposedLayout(L.ab_swizzle, 0, L.aSmem(b, false)); + const b_smem = b.makeComposedLayout(L.ab_swizzle, 0, L.bSmem(b, false)); + const sf_smem = b.staticLayout(L.sfSmem(b, false)); + const a_map = b.staticLayout(b.layoutSpec(.{ .{ 128, 32 }, 1, 4 }, .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0, b.basis(32, 1, .{1}) })); + const b_map = b.staticLayout(b.layoutSpec(.{ .{ n, 32 }, 1, 4 }, .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0, b.basis(32, 1, .{1}) })); + const sf_map = b.staticLayout(b.layoutSpec( + .{ .{ .{ 32, 4 }, 32 }, 1, 4 }, + .{ .{ .{ b.basis(1, 1, .{ 0, 0, 0 }), b.basis(1, 1, .{ 1, 0, 0 }) }, b.basis(1, 1, .{ 0, 0, 1 }) }, 0, b.basis(1, 1, .{ 1, 0, 1 }) }, + )); + + const tma_a = b.makeTiledTmaLoadAtomTyped(gA.value(), a_smem.value(), a_map.value(), L.tmaA(b, cfg)); + const tma_b = b.makeTiledTmaLoadAtomTyped(gB.value(), b_smem.value(), b_map.value(), L.tmaB(b, cfg)); + const tma_sfa = b.makeTiledTmaLoadAtomTyped(gSFA.value(), sf_smem.value(), sf_map.value(), L.tmaSf(b)); + const tma_sfb = b.makeTiledTmaLoadAtomTyped(gSFB.value(), sf_smem.value(), sf_map.value(), L.tmaSf(b)); + + const one = b.cst(.i32, 1); + const tiles_m = b.cst(.i32, @divExact(cfg.m, 128)); + const grid_z = b.cst(.i32, @min(@divExact(cfg.m, 128) * cfg.groups, cfg.persistent_ctas)); + const config = b.makeLaunchConfig(.{ + .grid = .{ one, one, grid_z }, + .block = .{ b.cst(.i32, 192), one, one }, + .dynamic_smem = b.kernelSmemSize(deviceName(epilogue)), + .stream = b.cudaStream(), + .cluster = .{ one, one, one }, + .use_pdl = true, + }); + const mainloop = .{ + ids, active, mma.tiled, mma_sfb.tiled, tma_a.atom, tma_a.tensor, tma_b.atom, tma_b.tensor, + tma_sfa.atom, tma_sfa.tensor, tma_sfb.atom, tma_sfb.tensor, + }; + const activations = @divExact(cfg.m, 2); + const group_sizes = b.makeTensorView( + b.addOffsetTyped(schedule, b.makeIntTuple(.{cfg.groups}), b.ptrTy(.i32, .gmem, offsetAlignment(16, cfg.groups, 4)) catch @panic("bad group size pointer")), + b.staticLayout(b.layoutSpec(cfg.groups, 1)), + ); + const launch = switch (epilogue) { + .swiglu_mxfp8 => b.launchEx(deviceName(epilogue), config, mainloop ++ .{ + group_sizes, + b.makeTensorView(b.arg(host_arg.route_inverse), b.staticLayout(b.layoutSpec(cfg.routedRows(), 1))), + b.makeTensorView(b.arg(host_arg.routing_weights), b.staticLayout(b.layoutSpec(@max(cfg.routes, 1), 1))), + b.makeTensorView(b.arg(host_arg.output), b.staticLayout(b.layoutSpec(.{ cfg.routedRows(), activations }, .{ activations, 1 }))), + b.makeTensorView(b.arg(host_arg.scales), b.staticLayout(b.layoutSpec(.{ cfg.groups, @divExact(activations, 32) * 128 }, .{ @divExact(activations, 32) * 128, 1 }))), + tiles_m, + }), + .route_rows => b.launchEx(deviceName(epilogue), config, mainloop ++ .{ + group_sizes, + b.makeTensorView(b.arg(host_arg.route_inverse), b.staticLayout(b.layoutSpec(cfg.routedRows(), 1))), + b.makeTensorView(b.arg(host_arg.output), b.staticLayout(b.layoutSpec(.{ @max(cfg.routes, 1), cfg.m }, .{ cfg.m, 1 }))), + tiles_m, + }), + }; + b.returnHostStatus(b.cudaResultStatus(launch)); +} + +test "MXFP4 GEMM programs emit their host launch for every tile width" { + inline for (.{ 8, 16, 32 }) |n| { + inline for (.{ .{ UpQuantized, Epilogue.swiglu_mxfp8, 4608, 5120 }, .{ DownRows, Epilogue.route_rows, 5120, 2304 } }) |program| { + const cfg: Config = .{ + .experts = 192, + .m = program[2], + .n = n, + .k = program[3], + .groups = 227, + .epilogue = program[1], + .routes = 576, + .direct = n == 8, + }; + const ir = try program[0].emit(std.testing.allocator, cfg); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, deviceName(program[1])) != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda.launch_ex") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "make_non_exec_tiled_tma_load") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, std.fmt.comptimePrint("sm100.mma_bs<128x{d}x32", .{n})) != null); + } + } +} diff --git a/zml/moe/cute_mxfp4.zig b/zml/moe/cute_mxfp4.zig new file mode 100644 index 00000000..10fb3ec7 --- /dev/null +++ b/zml/moe/cute_mxfp4.zig @@ -0,0 +1,123 @@ +//! Blackwell MXFP4 MoE backend with persistent CuTe GEMMs. +const zml = @import("../zml.zig"); +const triton_mxfp4 = @import("triton_mxfp4.zig"); +pub const kernels = @import("cute_kernels/moe.zig"); + +// `moe.zig`'s `refAllDecls` reaches this file but not its imports, so the +// kernel emit tests are only collected if `kernels` is referenced here. +test { + _ = kernels; +} + +pub const Parameters = triton_mxfp4.Parameters; + +/// Pack E8M0 weight scales in the exact 128x4 swizzle consumed by the TMA +/// scale-factor descriptor. Loaders apply this once at load time. +pub fn packWeightScales(scales: zml.Tensor) zml.Tensor { + const experts = scales.dim(0); + const rows = scales.dim(1); + const groups = scales.dim(2); + var packed_scales = scales.reshape(.{ experts, @divExact(rows, 128), 4, 32, @divExact(groups, 4), 4 }) + .transpose(.{ 0, 1, 4, 3, 2, 5 }); + return packed_scales.reshape(scales.shape()).withPartitioning(.{ .expert = .experts }); +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + const cc = zml.platform.cuda.computeCapability(platform) orelse return false; + return cc.major == 10; +} + +pub fn fusedExperts( + input: zml.Tensor, + ids: zml.Tensor, + weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + options: zml.moe.Options, + parameters: Parameters, +) !zml.Tensor { + if (parameters.activation != .silu) return error.UnsupportedActivation; + if (input.dtype() != .bf16) return error.UnsupportedDataType; + if (gate_up.bias != null or down.bias != null) return error.UnsupportedBias; + const gq = gate_up.quantization orelse return error.UnsupportedQuantization; + const dq = down.quantization orelse return error.UnsupportedQuantization; + if (gq.scheme != .mxfp4 or dq.scheme != .mxfp4) return error.UnsupportedQuantization; + // Weight storage for mxfp4 in HF is expressed as u8 or i8 + if ((gate_up.weight.dtype() != .u8 and gate_up.weight.dtype() != .i8) or + (down.weight.dtype() != .u8 and down.weight.dtype() != .i8)) + { + return error.UnsupportedWeightLayout; + } + + const expert_parallelism = gate_up.weight.shape().partition(.expert).eql(.init(.experts)); + const hidden = down.weight.dim(1); + const intermediate = down.weight.dim(2) * 2; + // The CuTe GEMMs are specialized on one set of dimensions + if (!kernels.isSupported(hidden, intermediate)) return error.UnsupportedShape; + const activation_threshold = options.activation_threshold orelse return error.UnsupportedActivation; + // The SwiGLU clamp is a kernel parameter, but the routing weight is + // multiplied in the up epilogue, before the down projection: applying it + // after would have to move into the down epilogue or the reduction. + if (options.routing_weight_placement != .before_down) return error.UnsupportedRoutingWeightPlacement; + const context: Context = .{ + .input = input, + .ids = ids, + .weights = weights, + .w1 = gate_up.weight.bitCast(.u8), + .s1 = gq.scales, + .w2 = down.weight.bitCast(.u8), + .s2 = dq.scales, + .topk = parameters.num_experts_per_tok, + .swiglu_limit = activation_threshold, + .expert_parallel = expert_parallelism, + }; + return if (expert_parallelism) + zml.ops.manualComputation(Context.body, context, input.shape()) + else + context.body(input.shape()); +} + +const Context = struct { + input: zml.Tensor, + ids: zml.Tensor, + weights: zml.Tensor, + w1: zml.Tensor, + s1: zml.Tensor, + w2: zml.Tensor, + s2: zml.Tensor, + topk: i64, + swiglu_limit: f32, + expert_parallel: bool, + + fn body(self: Context, _: zml.Shape) zml.Tensor { + const experts = self.w1.dim(.expert); + const hidden = self.w2.dim(1); + const intermediate = self.w2.dim(2) * 2; + var ids = self.ids.convert(.i32).reshape(.{ .token = .auto, .topk = self.topk }); + const tokens = ids.dim(.token); + const routing_weights = self.weights.convert(.f32).reshape(.{ tokens, self.topk }); + if (self.expert_parallel) { + const partition_id = zml.ops.partitionId().convert(.i32); + const expert_start = partition_id.scale(experts); + const expert_end = expert_start.addConstant(experts); + // Routes of other ranks get expert -1 and are not computed here. + const local = ids.cmp(.GE, expert_start).logical(.AND, ids.cmp(.LT, expert_end)); + ids = local.select(ids.sub(expert_start), .scalar(-1, .i32)); + } + const result = kernels.forward(tokens, hidden, intermediate, experts, self.topk, self.swiglu_limit, .{ + .x = self.input.reshape(.{ tokens, hidden }), + .routing_weights = routing_weights, + .w1 = self.w1, + .s1 = self.s1, + .w2 = self.w2, + .s2 = self.s2, + .ids = ids, + }).reshape(self.input.shape().dims()).withTags(self.input.shape()); + + return if (self.expert_parallel) + zml.ops.allReduce(result, zml.Tensor.add) + else + result; + } +}; diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index ded94167..8adaad69 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -5,6 +5,7 @@ const platforms = @import("platforms"); const zml = @import("../zml.zig"); const stdx = zml.stdx; pub const triton_mxfp4 = @import("triton_mxfp4.zig"); +pub const cute_mxfp4 = @import("cute_mxfp4.zig"); pub const cutlass_flashinfer = @import("cutlass_flashinfer.zig"); pub const metal = @import("metal.zig"); pub const mosaic_tpu = @import("mosaic_tpu.zig"); @@ -24,6 +25,7 @@ pub const ActivationMode = enum { }; pub const Backend = enum { + cute_mxfp4, triton_mxfp4, flashinfer_cutlass, triton, @@ -69,7 +71,7 @@ pub const Backend = enum { return switch (platform.target) { .cuda => switch (if (zml.platform.cuda.computeCapability(platform)) |cc| cc.major else 0) { 10 => switch (weights_dtype) { - .u8, .i8 => .triton_mxfp4, + .u8, .i8 => if (cute_mxfp4.isAvailable(platform)) .cute_mxfp4 else .triton_mxfp4, .f4e2m1 => .triton, else => error.UnsupportedDataType, }, @@ -98,6 +100,7 @@ pub const Backend = enum { pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { return switch (backend) { .triton_mxfp4 => triton_mxfp4.isAvailable(platform), + .cute_mxfp4 => cute_mxfp4.isAvailable(platform), .flashinfer_cutlass => cutlass_flashinfer.isAvailable(platform), .fly => switch (platform.target) { .rocm => zml.platform.rocm.computeCapability(platform) == .gfx942, @@ -114,7 +117,7 @@ pub const Backend = enum { pub fn register(backend: Backend, platform: *zml.Platform) !void { return switch (backend) { - .triton_mxfp4 => {}, + .cute_mxfp4, .triton_mxfp4 => {}, .flashinfer_cutlass => cutlass_flashinfer.register(platform), .triton, .fly => {}, .mosaic_tpu => {}, @@ -124,6 +127,7 @@ pub const Backend = enum { }; pub const Parameters = union(Backend) { + cute_mxfp4: cute_mxfp4.Parameters, triton_mxfp4: triton_mxfp4.Parameters, flashinfer_cutlass: cutlass_flashinfer.Parameters, triton: triton.Parameters, @@ -132,6 +136,7 @@ pub const Parameters = union(Backend) { metal: metal.Parameters, pub const InitOptions = union(Backend) { + cute_mxfp4: cute_mxfp4.Parameters.InitOptions, triton_mxfp4: triton_mxfp4.Parameters.InitOptions, flashinfer_cutlass: cutlass_flashinfer.Parameters.InitOptions, triton: triton.Parameters.InitOptions, @@ -141,7 +146,7 @@ pub const Parameters = union(Backend) { pub fn fromBackend(backend: Backend, num_experts_per_tok: u32, activation: ActivationMode) InitOptions { return switch (backend) { - .triton_mxfp4 => .{ .triton_mxfp4 = .{ .num_experts_per_tok = num_experts_per_tok, .activation = activation } }, + inline .cute_mxfp4, .triton_mxfp4 => |backend_tag| @unionInit(InitOptions, @tagName(backend_tag), .{ .num_experts_per_tok = num_experts_per_tok, .activation = activation }), .flashinfer_cutlass => .{ .flashinfer_cutlass = .{ .num_experts_per_tok = num_experts_per_tok, .activation = switch (activation) { @@ -180,6 +185,7 @@ pub const Parameters = union(Backend) { pub fn init(opts: InitOptions) Parameters { return switch (opts) { + .cute_mxfp4 => |v| .{ .cute_mxfp4 = cute_mxfp4.Parameters.init(v) }, .triton_mxfp4 => |v| .{ .triton_mxfp4 = triton_mxfp4.Parameters.init(v) }, .flashinfer_cutlass => |v| .{ .flashinfer_cutlass = cutlass_flashinfer.Parameters.init(v) }, inline .triton, .fly => |v, backend_tag| @unionInit(Parameters, @tagName(backend_tag), fused_experts.Parameters.init(v)), @@ -216,7 +222,7 @@ pub fn forwardMoe( stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); stdx.debug.assert(opts.activation_threshold == null, "Activation thresholds require the Triton MoE backend", .{}); }, - .triton_mxfp4 => { + .cute_mxfp4, .triton_mxfp4 => { stdx.debug.assert(opts.gate_up_layout == .interleaved, "Triton MXFP4 MoE backends require interleaved gate/up columns", .{}); }, } @@ -232,6 +238,7 @@ pub fn forwardMoe( const quant_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; return switch (parameters) { + .cute_mxfp4 => |p| cute_mxfp4.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts, p), .triton_mxfp4 => |p| triton_mxfp4.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts, p), .flashinfer_cutlass => b: { if (comptime !platforms.isEnabled(.cuda)) { diff --git a/zml/moe/triton_kernels/mxfp4.zig b/zml/moe/triton_kernels/mxfp4.zig index 75359fce..4da51a5a 100644 --- a/zml/moe/triton_kernels/mxfp4.zig +++ b/zml/moe/triton_kernels/mxfp4.zig @@ -117,11 +117,77 @@ const GemmConfig = struct { const Quant = tri.Kernel(Config, .{ .name = "mxfp4_triton_quant", .inputs = &.{"x"}, .outputs = &.{ "q", "s" }, .run = quantInput }); const Route = tri.Kernel(Config, .{ .name = "mxfp4_triton_route", .inputs = &.{"ids"}, .outputs = &.{ "counts", "pos" }, .run = route }); -const Schedule = tri.Kernel(Config, .{ .name = "mxfp4_triton_schedule", .inputs = &.{ "ids", "counts", "pos" }, .outputs = &.{ "map", "perm", "sched" }, .run = schedule }); +const RouteAtomic = tri.Kernel(Config, .{ .name = "mxfp4_triton_route_atomic", .inputs = &.{ "ids", "zeros" }, .outputs = &.{ "counts", "pos" }, .run = routeAtomic }); +// The same device code serves both backends; only the group geometry differs. +const ScheduleConfig = struct { + moe: Config, + /// Expert groups the schedule may fill. + capacity: i64, + /// Routed rows per expert group, i.e. the consuming GEMM's tile N. + group: i64, +}; +// Two symbols for one kernel: both backends can run in the same process, and +// distinct names keep them apart in profiles. +const Schedule = tri.Kernel(ScheduleConfig, .{ .name = "mxfp4_triton_schedule", .inputs = &.{ "ids", "counts", "pos" }, .outputs = &.{ "map", "perm", "sched" }, .run = scheduleGroups }); const GateUp = tri.Kernel(GemmConfig, .{ .name = "mxfp4_triton_up", .inputs = &.{ "q", "s", "w", "ws", "ids", "rw", "map", "sched" }, .outputs = &.{ "mid", "ms" }, .run = gateUp }); const Down = tri.Kernel(GemmConfig, .{ .name = "mxfp4_triton_down", .inputs = &.{ "mid", "ms", "w", "ws", "ids", "sched" }, .outputs = &.{"d"}, .run = down }); const Combine = tri.Kernel(Config, .{ .name = "mxfp4_triton_combine", .inputs = &.{ "d", "ids", "perm", "rw" }, .outputs = &.{"y"}, .run = combine }); +/// MXFP4 routing for the CuTe backend (`zml/moe/cute_mxfp4.zig`), whose +/// persistent GEMMs consume it. It reuses this backend's `route` and +/// `schedule` kernels with `group`-row expert groups (the CuTe GEMM tile N) +/// and a compact `capacity`, instead of the fixed `bn` tiles used here. +/// `route_map` maps routes to routed rows, `route_inverse` rows back to +/// routes, and `schedule` is `[group experts | group sizes | active group +/// count]`. +pub fn scheduleForCute(c: Config, ids: zml.Tensor, capacity: i64, group: i64) struct { + route_inverse: zml.Tensor, + route_map: zml.Tensor, + schedule: zml.Tensor, +} { + c.validate() catch @panic("Invalid MXFP4 MoE configuration"); + std.debug.assert(capacity > 0); + const routes = c.tokens * c.topk; + // `route` rescans every route once per expert: cheapest for decode, but + // quadratic. Past a few thousand tokens, count with atomics instead. + const r: struct { counts: zml.Tensor, pos: zml.Tensor } = if (routes >= atomic_route_threshold) blk: { + const counted = RouteAtomic.call( + .{ .ids = ids, .zeros = zml.Tensor.zeroes(.init(.{c.experts}, .i32)) }, + .{ .counts = .init(.{c.experts}, .i32), .pos = .init(.{routes}, .i32) }, + .{ + .cfg = c, + .grid = .{ @intCast(@divTrunc(routes + route_block - 1, route_block)), 1, 1 }, + .num_warps = 4, + .num_stages = 1, + .output_operand_aliases = .{ .counts = .zeros }, + }, + ); + break :blk .{ .counts = counted.counts, .pos = counted.pos }; + } else blk: { + const scanned = Route.call( + .{ .ids = ids }, + .{ .counts = .init(.{c.experts}, .i32), .pos = .init(.{routes}, .i32) }, + .{ .cfg = c, .grid = .{ @intCast(c.experts), 1, 1 }, .num_warps = 4, .num_stages = 1 }, + ); + break :blk .{ .counts = scanned.counts, .pos = scanned.pos }; + }; + const s = Schedule.call( + .{ .ids = ids, .counts = r.counts, .pos = r.pos }, + .{ + .map = .init(.{capacity * group}, .i32), + .perm = .init(.{c.tokens * c.topk}, .i32), + .sched = .init(.{2 * capacity + 1}, .i32), + }, + .{ + .cfg = .{ .moe = c, .capacity = capacity, .group = group }, + .grid = .{ @intCast(@divTrunc(c.tokens * c.topk + 127, 128)), 1, 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ); + return .{ .route_inverse = s.map, .route_map = s.perm, .schedule = s.sched }; +} + pub fn forward(c: Config, a: Inputs) zml.Tensor { c.validate() catch @panic("Invalid MXFP4 MoE configuration"); validateInputs(c, a) catch @panic("Invalid MXFP4 tensor layout"); @@ -159,7 +225,7 @@ pub fn forward(c: Config, a: Inputs) zml.Tensor { .{ .ids = a.ids, .counts = r.counts, .pos = r.pos }, .{ .map = .init(.{tile_count * bn}, .i32), .perm = .init(.{c.tokens * c.topk}, .i32), .sched = .init(.{2 * tile_count + 1}, .i32) }, .{ - .cfg = c, + .cfg = .{ .moe = c, .capacity = tile_count, .group = bn }, .grid = .{ @intCast(@divTrunc(c.tokens * c.topk + 127, 128)), 1, 1 }, .num_warps = 4, .num_stages = 1, @@ -244,14 +310,35 @@ fn route(b: *B, c: Config) tri.FinishError!void { b.store(a.counts.addPtr(e), valid.to(.i32).sum()); } -fn schedule(b: *B, c: Config) tri.FinishError!void { +const route_block = 1024; +const atomic_route_threshold = 6 * 2048; + +/// Routing for large batches, used by `scheduleForCute`. +/// Each route takes the next free slot of its expert with an atomic +/// increment of `counts` (zero-initialized through `zeros`). Slots within an +/// expert are therefore unordered, which the grouped GEMMs do not care +/// about: every routed row is computed independently. +fn routeAtomic(b: *B, c: Config) tri.FinishError!void { + const a = try b.declareArgs(.{ .ids = .{ .ptr = .i32 }, .zeros = .{ .ptr = .i32 }, .counts = .{ .ptr = .i32 }, .pos = .{ .ptr = .i32 } }); + const p = b.programId(.x).mul(route_block).add(ar(b, route_block)); + const in_range = p.lt(c.tokens * c.topk); + const e = ld(b, a.ids.addPtr(p), in_range, .i32, -1).sub(ci(b, c.expert_offset)); + const valid = in_range.bitAnd(e.ge(0)).bitAnd(e.lt(c.experts)); + const slot = b.atomicRmwOpts(.add, a.counts.addPtr(e.maximum(0)), b.full(&.{route_block}, 1, .i32), .{ .mask = valid, .sem = .relaxed }); + b.storeOpts(a.pos.addPtr(p), slot, .{ .mask = valid }); +} + +fn scheduleGroups(b: *B, cfg: ScheduleConfig) tri.FinishError!void { + const c = cfg.moe; + const nt = cfg.capacity; + // Route positions are i32 tensors; keep the group size in that type. + const group: i32 = @intCast(cfg.group); const a = try b.declareArgs(.{ .ids = .{ .ptr = .i32 }, .counts = .{ .ptr = .i32 }, .pos = .{ .ptr = .i32 }, .map = .{ .ptr = .i32 }, .perm = .{ .ptr = .i32 }, .sched = .{ .ptr = .i32 } }); - const nt = tiles(c); const p = b.programId(.x).mul(128).add(ar(b, 128)); const e = ld(b, a.ids.addPtr(p), p.lt(c.tokens * c.topk), .i32, -1).sub(ci(b, c.expert_offset)); const es = ar(b, pow2(c.experts)); const counts = ld(b, a.counts.addPtr(es), es.lt(c.experts), .i32, 0); - const ts = counts.cdiv(bn); + const ts = counts.cdiv(group); const starts = ts.cumsum().sub(ts); var first = b.openIf(b.programId(.x).eq(0)); b.store(a.sched.addPtr(2 * nt), ts.sum()); @@ -260,12 +347,12 @@ fn schedule(b: *B, c: Config) tri.FinishError!void { const safe = e.maximum(0).minimum(pow2(c.experts) - 1); const pos = ld(b, a.pos.addPtr(p), live, .i32, 0); const start = starts.gather(safe, 0); - const tile = start.add(pos.div(bn)); - const row = pos.rem(bn); - b.storeOpts(a.perm.addPtr(p), b.select(live, tile.mul(bn).add(row), b.full(&.{128}, -1, .i32)), .{ .mask = p.lt(c.tokens * c.topk) }); - b.storeOpts(a.map.addPtr(tile.mul(bn).add(row)), p, .{ .mask = live }); + const tile = start.add(pos.div(group)); + const row = pos.rem(group); + b.storeOpts(a.perm.addPtr(p), b.select(live, tile.mul(group).add(row), b.full(&.{128}, -1, .i32)), .{ .mask = p.lt(c.tokens * c.topk) }); + b.storeOpts(a.map.addPtr(tile.mul(group).add(row)), p, .{ .mask = live }); b.storeOpts(a.sched.addPtr(tile), e, .{ .mask = live.bitAnd(row.eq(0)) }); - b.storeOpts(a.sched.addPtr(nt).addPtr(tile), counts.gather(safe, 0).sub(pos).minimum(bn), .{ .mask = live.bitAnd(row.eq(0)) }); + b.storeOpts(a.sched.addPtr(nt).addPtr(tile), counts.gather(safe, 0).sub(pos).minimum(group), .{ .mask = live.bitAnd(row.eq(0)) }); } fn gateUp(b: *B, c: GemmConfig) tri.FinishError!void { diff --git a/zml/moe/triton_mxfp4.zig b/zml/moe/triton_mxfp4.zig index 644f29d2..7ead08fd 100644 --- a/zml/moe/triton_mxfp4.zig +++ b/zml/moe/triton_mxfp4.zig @@ -43,7 +43,12 @@ pub fn fusedExperts( const gq = gate_up.quantization orelse return error.UnsupportedQuantization; const dq = down.quantization orelse return error.UnsupportedQuantization; if (gq.scheme != .mxfp4 or dq.scheme != .mxfp4) return error.UnsupportedQuantization; - if ((gate_up.weight.dtype() != .u8 and gate_up.weight.dtype() != .i8) or (down.weight.dtype() != .u8 and down.weight.dtype() != .i8)) return error.UnsupportedWeightLayout; + // Weight storage for mxfp4 in HF is expressed as u8 or i8 + if ((gate_up.weight.dtype() != .u8 and gate_up.weight.dtype() != .i8) or + (down.weight.dtype() != .u8 and down.weight.dtype() != .i8)) + { + return error.UnsupportedWeightLayout; + } const expert_parallelism = gate_up.weight.shape().partition(.expert).eql(.init(.experts)); @@ -86,9 +91,8 @@ const Context = struct { const experts = self.w1.dim(.expert); const hidden = self.w2.dim(1); const intermediate = self.w2.dim(2) * 2; - const tokens: i64 = @intCast(self.input.count() / @as(usize, @intCast(hidden))); - - var ids = self.ids.convert(.i32); + var ids = self.ids.convert(.i32).reshape(.{ .token = .auto, .topk = self.topk }); + const tokens = ids.dim(.token); if (self.expert_parallel) { const partition_id = zml.ops.partitionId().convert(.i32); ids = ids.sub(partition_id.scale(experts)); @@ -107,7 +111,7 @@ const Context = struct { const result = kernels.forward(cfg, .{ .x = self.input.reshape(.{ tokens, hidden }), - .ids = ids.reshape(.{ tokens, self.topk }), + .ids = ids, .scales = self.weights.convert(.f32).reshape(.{ tokens, self.topk }), .w1 = self.w1, .s1 = self.s1, diff --git a/zml/ops.zig b/zml/ops.zig index 59a454f9..1e04de12 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1370,6 +1370,7 @@ pub const CuteOps = struct { grid: ?[3]i32 = null, block: ?[3]i32 = null, zeroed_outputs: []const i32 = &.{}, + scalars: []const i64 = &.{}, output_operand_aliases: []const dialects.stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, }; @@ -1391,7 +1392,7 @@ pub fn cute(inputs: anytype, outputs: anytype, opts: CuteOps) [outputs.len]Tenso res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); } - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; attrs.appendSliceAssumeCapacity(&.{ .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), .named(mlir_ctx, "kernel_type", .string(mlir_ctx, "cute")), @@ -1409,6 +1410,13 @@ pub fn cute(inputs: anytype, outputs: anytype, opts: CuteOps) [outputs.len]Tenso for (opts.zeroed_outputs) |i| zeroed.appendAssumeCapacity(.int(mlir_ctx, .i32, i)); attrs.appendAssumeCapacity(.named(mlir_ctx, "zeroed_outputs", .array(mlir_ctx, zeroed.constSlice()))); } + if (opts.scalars.len > 0) { + const allocator = Compiler.current().allocator; + const scalar_attrs = allocator.alloc(*const mlir.Attribute, opts.scalars.len) catch @panic("OOM"); + defer allocator.free(scalar_attrs); + for (opts.scalars, scalar_attrs) |value, *attribute| attribute.* = .int(mlir_ctx, .i64, value); + attrs.appendAssumeCapacity(.named(mlir_ctx, "scalars", .array(mlir_ctx, scalar_attrs))); + } const backend_config: *const mlir.Attribute = .dict(mlir_ctx, attrs.constSlice()); const op = dialects.stablehlo.custom_call( From 778a5ab3c48e840f797bbbfa94eb78863a643b72 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Thu, 24 Sep 2026 17:56:20 +0200 Subject: [PATCH 719/881] zml/compiler: enable experimental all-reduce kernels on CUDA and bump plugins (#852) Enable XLA's experimental all-reduce kernels on CUDA (custom one-shot / two-shot kernels instead of NCCL) --- platforms/cpu/cpu.bzl | 8 ++++---- platforms/cuda/cuda.bzl | 8 ++++---- platforms/metal/metal.bzl | 4 ++-- platforms/rocm/rocm.bzl | 4 ++-- zml/Compiler.zig | 2 +- zml/io.zig | 2 +- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index cf3c27c8..b3fa8dc6 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -42,22 +42,22 @@ def _cpu_plugin_impl(mctx): http_archive( name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "e825d8d1031ef536f5a1ac2c1eb0a10f8af83168ffb14c3cc8b26aee25ed1f87", - url = "https://mirror.zml.ai/plugins/202609231543.88.1.e2f61df0be45/zml-cpu-linux-amd64.tar.zst", + sha256 = "16f0040b1805864fc0c3e13fe15f8ffccaf885a59b4f625666d2618583c0ac50", + url = "https://mirror.zml.ai/plugins/202609241513.97.1.78be97663f43/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "5da35f0e471a410f3a3edba95f8a849ea017c1ed32c6fab0239a25a5d21acad4", - url = "https://mirror.zml.ai/plugins/202609231543.88.1.e2f61df0be45/zml-cpu-darwin-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202609241513.97.1.78be97663f43/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "b0f6a8a1486780a9433b08153d71e0e31ed0e4cde60f200346c687c9346b2d53", - url = "https://mirror.zml.ai/plugins/202609231543.88.1.e2f61df0be45/zml-cpu-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202609241513.97.1.78be97663f43/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 4f3a31f5..3c021b94 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -224,12 +224,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "b5ad9b8d37e59457b219a5c6a025d6bea598f3415019c0003a6c5f659d673028", - "url": "https://mirror.zml.ai/plugins/202609231543.88.1.e2f61df0be45/zml-cuda-linux-amd64.tar.zst", + "sha256": "58e07d6766d3f9b1262d9ab7f4a6aadf5358df517afd75d0952a305c65497361", + "url": "https://mirror.zml.ai/plugins/202609241513.97.1.78be97663f43/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "470a14e7fc9973ac732fb7eb868599adb1549a2aefa01ed3b2497a15ed8a678b", - "url": "https://mirror.zml.ai/plugins/202609231543.88.1.e2f61df0be45/zml-cuda-linux-arm64.tar.zst", + "sha256": "8118bb6ce9b6753cf6f35e026b5798e28cdca328ae236c63822a6fd8d9d5514f", + "url": "https://mirror.zml.ai/plugins/202609241513.97.1.78be97663f43/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index b97ea6b7..49bd84f4 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "219654b57a70eb982f0b25d39de301e66f752e71b55df23bdaa25d12f011eba2", - url = "https://mirror.zml.ai/plugins/202609231543.88.1.e2f61df0be45/zml-metal-darwin-arm64.tar.zst", + sha256 = "d5f51655129ec9ac289560eefd9e273e4a5780ed58fe4d4acf725e65e0e8b7c9", + url = "https://mirror.zml.ai/plugins/202609241513.97.1.78be97663f43/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index a6e056c7..97e431f8 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202609231543.88.1.e2f61df0be45/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "02274718670a0c523301003b3e6d045178219bb7017a179b4a229f8b9cee8391" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202609241513.97.1.78be97663f43/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "abffc6b901d3132d112d8858a808553839de174d35035b8798919426f74b3db6" def _rocm_package_name(name): return name + _ROCM_VERSION diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 619676c1..f946e982 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -849,7 +849,7 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: try setXlaOverrideFlag(overrides_map, "xla_gpu_unsupported_enable_triton_multi_output_fusion", true, upb_arena); try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_scheduling_mode", "CONCURRENT", upb_arena); try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_update_mode", "SKIP_TEMP", upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_collective_kernels", "", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_collective_kernels", "COLLECTIVE_KERNEL_ALL_REDUCE", upb_arena); // Add collectives to the default list try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES,CONDITIONAL,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); // With SKIP_TEMP, captured NCCL collectives can retain stale physical mappings diff --git a/zml/io.zig b/zml/io.zig index ba0873d5..a6ff262d 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -430,7 +430,7 @@ pub const Loader = struct { fn defaultCallback(self: *Loader, io: std.Io, tensor: *const Tensor, buffer: *Buffer, store: *const TensorStore, shardings: []const Sharding, opts: LoadOpts) void { const sources = store.getSourcesById(tensor.id) orelse { - std.log.warn("Failed to get sources for tensor with id: {}", .{tensor.id}); + std.log.debug("Failed to get sources for tensor with id: {}", .{tensor.id}); return; }; stdx.debug.assert(!sources.transformed and sources.tensors.len == 1, "Tensor {} is transformed or has {} sources; `load` only streams single-source tensors", .{ tensor.id, sources.tensors.len }); From 90f914ad07bb0598e6212102056fc07bae3cfff0 Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Thu, 24 Sep 2026 17:57:54 +0200 Subject: [PATCH 720/881] platforms/cuda: use info level instead of warn (#827) --- platforms/cuda/cuda.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platforms/cuda/cuda.zig b/platforms/cuda/cuda.zig index 8f1a03a8..f1c9d1e4 100644 --- a/platforms/cuda/cuda.zig +++ b/platforms/cuda/cuda.zig @@ -114,7 +114,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { }; if (cudaCompat) { - log.warn("Detected NVIDIA GPU that requires CUDA compatibility libraries.", .{}); + log.info("Detected NVIDIA GPU that requires CUDA compatibility libraries.", .{}); var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "compat", "libcuda.so.1" }); _ = std.c.dlopen(path, .{ .NOW = true }) orelse { From cabcdd8f59d4fcba73cdfffc7d31023faf6c1ead Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Thu, 24 Sep 2026 20:13:04 +0200 Subject: [PATCH 721/881] platforms/oneapi: Use latest pjrt artifact (#855) --- platforms/oneapi/libzml_oneapi.BUILD.bazel | 11 ++++++++++- platforms/oneapi/oneapi.bzl | 4 ++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/platforms/oneapi/libzml_oneapi.BUILD.bazel b/platforms/oneapi/libzml_oneapi.BUILD.bazel index 62879ed6..67de8022 100644 --- a/platforms/oneapi/libzml_oneapi.BUILD.bazel +++ b/platforms/oneapi/libzml_oneapi.BUILD.bazel @@ -11,6 +11,7 @@ patchelf( "dlopen": "zmlxoneapi_dlopen", }, set_rpath = "$ORIGIN", + soname = "libzml_oneapi.so", ) patchelf( @@ -68,15 +69,23 @@ patchelf( ) ONEAPI_VERSION = "2026.1" + ONEAPI_CCL_LIB = "ccl/2022.1/lib" + ONEAPI_MPI_LIB = "mpi/2021.18/lib" + ONEAPI_MPI_LIBFABRIC_LIB = "mpi/2021.18/opt/mpi/libfabric/lib" + ONEAPI_TCM_VERSION = "1.5" + ONEAPI_UMF_VERSION = "1.1" ONEAPI_COMPILER_LIB = "compiler/{}/lib".format(ONEAPI_VERSION) + ONEAPI_MKL_LIB = "mkl/{}/lib".format(ONEAPI_VERSION) + ONEAPI_TCM_LIB = "tcm/{}/lib".format(ONEAPI_TCM_VERSION) + ONEAPI_UMF_LIB = "umf/{}/lib".format(ONEAPI_UMF_VERSION) copy_to_directory( @@ -86,10 +95,10 @@ copy_to_directory( ":libiga64_so_2", ":libigc_so_2", ":libigdfcl_so_2", - ":libzml_oneapi_so", ":libsycl_so", ":libtbb_so_12", ":libze_intel_gpu_so", + ":libzml_oneapi_so", "@intel-oneapi-ccl-2022.1//:ccl_runtime", "@intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1//:sycl_runtime", "@intel-oneapi-compiler-shared-runtime-2026.1//:compiler_runtime", diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index 5069f8b5..0ab5b1a0 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -1,8 +1,8 @@ load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") -PJRT_ONEAPI_RELEASE = "manual-2026-08-17T19-00-00Z" -PJRT_ONEAPI_ARTIFACT_SHA256 = "af68f460376245013c6d42b661c24e15685dea2aa9ea9f8267d1878ef0c363a1" +PJRT_ONEAPI_RELEASE = "manual-2026-09-08T13-23-00Z" +PJRT_ONEAPI_ARTIFACT_SHA256 = "c0c429f435a31207cb976636a9fa2d11b420ddc65bdde325e0ae736afb47a9b7" PJRT_ONEAPI_ARTIFACT_URL = "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-oneapi_linux-amd64.tar.gz".format( release = PJRT_ONEAPI_RELEASE, ) From 9b7d8fb58cecc598255bf13208a77f0ecf9e0cfa Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 24 Sep 2026 22:21:44 +0200 Subject: [PATCH 722/881] bazel: rework zig_options to be config_settting compatible zig_option can now take a `flag` dependency to expose existing flags or `bool`, `int or `string` for config_setting flags --- bazel/zig_options.bzl | 177 ++++++++++++++-------------------- platforms/BUILD.bazel | 73 ++++++++------ platforms/cpu/cpu.zig | 2 +- platforms/cuda/cuda.zig | 2 +- platforms/metal/metal.zig | 2 +- platforms/neuron/neuron.zig | 2 +- platforms/oneapi/oneapi.zig | 2 +- platforms/platforms.zig | 2 +- platforms/rocm/rocm.zig | 2 +- platforms/tpu/tpu.zig | 2 +- third_party/arocc/arocc.bazel | 22 ++--- third_party/zio/zio.bazel | 20 ++-- 12 files changed, 149 insertions(+), 159 deletions(-) diff --git a/bazel/zig_options.bzl b/bazel/zig_options.bzl index 76d09a19..88ab40bb 100644 --- a/bazel/zig_options.bzl +++ b/bazel/zig_options.bzl @@ -7,8 +7,9 @@ load("@rules_zig//zig:defs.bzl", "zig_library") ZigOption = provider( doc = "A Zig option's configured value and allowed string values.", fields = { + "name": "The flag's target name, or this option's target name for explicit values.", "module_name": "The Bazel module declaring the flag, or empty when unavailable.", - "value": "The configured build setting value.", + "value": "The configured build setting or explicit value.", "type": "The Zig integer type, or None for other flag types.", "allowed_values": "Allowed values for string flags; an empty list allows any string. None for other flag types.", "enum_name": "Optional exported enum type name for a constrained string flag.", @@ -16,50 +17,43 @@ ZigOption = provider( }, ) -_MAKE_VARIABLE_ATTR = attr.string( - doc = "Optional Make variable name exposed to rules that list this flag in toolchains.", -) +_FlagInfo = provider(fields = ["allowed_values"]) -_MODULE_NAME_ATTR = attr.string( - doc = "Declaring module name, captured by zig_option during package evaluation.", -) +def _flag_info_impl(_target, ctx): + return [_FlagInfo(allowed_values = getattr(ctx.rule.attr, "values", []))] -_SCOPE_ATTR = attr.string( - doc = "The scope indicates where a flag can propagate to", - default = "target", -) +_flag_info = aspect(implementation = _flag_info_impl) -def _flag_impl(ctx): - value = ctx.build_setting_value - providers = [ - BuildSettingInfo(value = value), - ZigOption( - module_name = ctx.attr.module_name, - value = value, - type = getattr(ctx.attr, "type", None), - allowed_values = getattr(ctx.attr, "values", None), - enum_name = getattr(ctx.attr, "enum_name", ""), - nullable = getattr(ctx.attr, "nullable", False), - ), - ] - if ctx.attr.make_variable: - providers.append(platform_common.TemplateVariableInfo({ - ctx.attr.make_variable: str(value).lower() if type(value) == "bool" else str(value), - })) - return providers - -def _string_impl(ctx): - if ctx.attr.nullable and ctx.attr.values: +def _zig_option_impl(ctx): + flag = ctx.attr.flag + if ctx.attr.value_source == "flag": + if flag == None: + fail("flag must resolve to a build setting") + value = flag[BuildSettingInfo].value + else: + value = getattr(ctx.attr, ctx.attr.value_source) + if not (types.is_bool(value) or types.is_int(value) or types.is_string(value)): + fail("Zig options require an integer, boolean, or string value") + allowed_values = None + if types.is_string(value): + allowed_values = flag[_FlagInfo].allowed_values if flag else [] + if ctx.attr.type and not types.is_int(value): + fail("type is only supported for integer options") + if ctx.attr.nullable and not types.is_string(value): + fail("nullable is only supported for string options") + if ctx.attr.nullable and allowed_values: fail("nullable strings cannot have enum values") - if ctx.attr.enum_name and not ctx.attr.values: - fail("enum_name requires nonempty values") - if ctx.attr.values and ctx.build_setting_value not in ctx.attr.values: - fail("{}: invalid value '{}'; expected one of {}".format( - ctx.label, - ctx.build_setting_value, - ctx.attr.values, - )) - return _flag_impl(ctx) + if ctx.attr.enum_name and not allowed_values: + fail("enum_name requires a string flag with nonempty values") + return [ZigOption( + name = flag.label.name if flag else ctx.label.name, + module_name = "", + value = value, + type = (ctx.attr.type or "usize") if types.is_int(value) else None, + allowed_values = allowed_values, + enum_name = ctx.attr.enum_name, + nullable = ctx.attr.nullable, + )] def _zig_string(value): # JSON and Zig differ in their control-character and Unicode escapes. @@ -76,7 +70,7 @@ def _zig_options_impl(ctx): names = {} for setting in ctx.attr.settings: option = setting[ZigOption] - name = setting.label.name + name = option.name if name in names: fail("Duplicate Zig declaration: {}".format(name)) names[name] = True @@ -120,7 +114,7 @@ def zig_options(name, settings, **kwargs): Args: name: Library target name; the generated source is .zig. - settings: Option flags; target names become quoted Zig constant names. + settings: zig_option targets; flag names or explicit option names become quoted Zig constants. **kwargs: Attributes forwarded to zig_library, except main which is generated. """ if "main" in kwargs: @@ -135,71 +129,42 @@ def zig_options(name, settings, **kwargs): ) zig_library(name = name, main = ":" + name + "_source", **kwargs) -def zig_option(name, build_setting_default, type = None, values = None, enum_name = None, **kwargs): - """Define a command-line option, inferring its kind from the default. - - Args: - name: Flag target name and generated Zig constant name. - build_setting_default: Integer, boolean, or string default value. - type: Zig integer type; defaults to usize. Only valid for integers. - values: Allowed string values, emitted as enum fields when nonempty. - enum_name: Optional exported enum type name; requires a string with nonempty values. - **kwargs: Attributes forwarded to the underlying flag rule. - """ - if type != None and not types.is_int(build_setting_default): - fail("type is only supported for integer options") - if values != None and not types.is_string(build_setting_default): - fail("values is only supported for string options") - if enum_name != None and (not types.is_string(build_setting_default) or not values): - fail("enum_name requires a string option with nonempty values") - if types.is_bool(build_setting_default): - flag = bool_flag - elif types.is_int(build_setting_default): - flag = int_flag - kwargs["type"] = type if type != None else "usize" - elif types.is_string(build_setting_default): - flag = string_flag - kwargs["values"] = values if values != None else [] - kwargs["enum_name"] = enum_name if enum_name != None else "" - else: - fail("Zig options require an integer, boolean, or string default") - flag(name = name, build_setting_default = build_setting_default, module_name = native.module_name() or "", **kwargs) - -int_flag = rule( - implementation = _flag_impl, - build_setting = config.int(flag = True), - attrs = { - "type": attr.string(default = "usize", doc = "Zig integer type, such as usize, u32, or i64."), - "module_name": _MODULE_NAME_ATTR, - "make_variable": _MAKE_VARIABLE_ATTR, - "scope": _SCOPE_ATTR, - }, - doc = "An integer build setting that can be set on the command line", -) - -bool_flag = rule( - implementation = _flag_impl, - build_setting = config.bool(flag = True), - attrs = { - "module_name": _MODULE_NAME_ATTR, - "make_variable": _MAKE_VARIABLE_ATTR, - "scope": _SCOPE_ATTR, - }, - doc = "A boolean build setting that can be set on the command line", -) - -string_flag = rule( - implementation = _string_impl, - build_setting = config.string(flag = True), +_zig_option = rule( + implementation = _zig_option_impl, attrs = { + "flag": attr.label( + providers = [BuildSettingInfo], + aspects = [_flag_info], + doc = "Boolean, integer, or string build setting. An aspect discovers allowed string values.", + ), + "bool": attr.bool(), + "string": attr.string(), + "int": attr.int(), + "value_source": attr.string(mandatory = True, values = ["flag", "bool", "string", "int"]), + "type": attr.string(doc = "Zig integer type; defaults to usize. Only valid for integer options."), "nullable": attr.bool(doc = "Emit an optional string; an empty value becomes null. Cannot be combined with values."), "enum_name": attr.string(doc = "Optional exported Zig enum type name; requires nonempty values."), - "module_name": _MODULE_NAME_ATTR, - "values": attr.string_list( - doc = "The list of allowed values for this setting. An error is raised if any other value is given.", - ), - "make_variable": _MAKE_VARIABLE_ATTR, - "scope": _SCOPE_ATTR, }, - doc = "A string-typed build setting that can be set on the command line", + doc = "Expose a build setting or explicit value as a Zig option, named after the flag or this target respectively.", ) + +def zig_option(name, flag = None, bool = None, string = None, int = None, **kwargs): + """Expose exactly one build setting or explicit value as a Zig option. + + Args: + name: Target name; also the generated Zig constant name for explicit values. + flag: Boolean, integer, or string build setting to read. + bool: Explicit boolean value, which may use select(). + string: Explicit string value, which may use select(). + int: Explicit integer value, which may use select(). + **kwargs: Attributes forwarded to the rule, including type, nullable, and enum_name. + """ + sources = {key: value for key, value in { + "flag": flag, + "bool": bool, + "string": string, + "int": int, + }.items() if value != None} + if len(sources) != 1: + fail("zig_option requires exactly one of flag, bool, string, or int") + _zig_option(name = name, value_source = sources.keys()[0], **dict(sources, **kwargs)) diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index a454d361..104bdab8 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -1,3 +1,4 @@ +load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") load("@rules_zig//zig:defs.bzl", "zig_library") load("//bazel:zig_options.bzl", "zig_option", "zig_options") @@ -12,30 +13,63 @@ PLATFORMS = { } [ - zig_option( + bool_flag( name = runtime, build_setting_default = default, - scope = "universal", ) for runtime, default in PLATFORMS.items() ] +[ + zig_option( + name = "{}_enabled".format(runtime), + bool = select({ + "//platforms:{}.enabled".format(runtime): True, + "//conditions:default": False, + }), + visibility = ["//platforms:__pkg__"], + ) + for runtime in PLATFORMS +] + zig_options( name = "options", import_name = "platforms/options", - settings = [":" + runtime for runtime in PLATFORMS], + settings = [":{}_enabled".format(runtime) for runtime in PLATFORMS], visibility = ["//platforms:__subpackages__"], ) -config_setting( - name = "rocm.enabled", - constraint_values = [ - "@platforms//cpu:x86_64", - "@platforms//os:linux", - ], - flag_values = {":rocm": "True"}, - visibility = ["//visibility:public"], -) +# x86_64 and aarch64 +[ + config_setting( + name = "{}.enabled".format(runtime), + flag_values = {":{}".format(runtime): "True"}, + visibility = ["//visibility:public"], + ) + for runtime in [ + "cpu", + "cuda", + ] +] + +# x86_64 only +[ + config_setting( + name = "{}.enabled".format(runtime), + constraint_values = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], + flag_values = {":{}".format(runtime): "True"}, + visibility = ["//visibility:public"], + ) + for runtime in [ + "rocm", + "tpu", + "neuron", + "oneapi", + ] +] config_setting( name = "metal.enabled", @@ -47,21 +81,6 @@ config_setting( visibility = ["//visibility:public"], ) -[ - config_setting( - name = "{}.enabled".format(runtime), - flag_values = {":{}".format(runtime): "True"}, - visibility = ["//visibility:public"], - ) - for runtime in [ - "cpu", - "cuda", - "tpu", - "neuron", - "oneapi", - ] -] - zig_library( name = "platforms", main = "platforms.zig", diff --git a/platforms/cpu/cpu.zig b/platforms/cpu/cpu.zig index de4c7bd2..523cd119 100644 --- a/platforms/cpu/cpu.zig +++ b/platforms/cpu/cpu.zig @@ -11,7 +11,7 @@ const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/platforms/cpu"); pub fn isEnabled() bool { - return platforms_options.cpu; + return platforms_options.cpu_enabled; } pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { diff --git a/platforms/cuda/cuda.zig b/platforms/cuda/cuda.zig index f1c9d1e4..562fdc56 100644 --- a/platforms/cuda/cuda.zig +++ b/platforms/cuda/cuda.zig @@ -27,7 +27,7 @@ fn findCudaSandbox( } pub fn isEnabled() bool { - return platforms_options.cuda; + return platforms_options.cuda_enabled; } pub fn needsCudaCompat(io: std.Io, sandbox_path: []const u8) !bool { diff --git a/platforms/metal/metal.zig b/platforms/metal/metal.zig index e3052168..ddde2a30 100644 --- a/platforms/metal/metal.zig +++ b/platforms/metal/metal.zig @@ -11,7 +11,7 @@ const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/platforms/metal"); pub fn isEnabled() bool { - return platforms_options.metal; + return platforms_options.metal_enabled; } fn setMetalToolchainEnv(r: anytype) !void { diff --git a/platforms/neuron/neuron.zig b/platforms/neuron/neuron.zig index 995a97f8..039cc566 100644 --- a/platforms/neuron/neuron.zig +++ b/platforms/neuron/neuron.zig @@ -96,7 +96,7 @@ pub fn instance() !Instance { } pub fn isEnabled() bool { - return platforms_options.neuron; + return platforms_options.neuron_enabled; } fn hasNeuronDevice(io: std.Io) bool { diff --git a/platforms/oneapi/oneapi.zig b/platforms/oneapi/oneapi.zig index 2a2262b7..c5654741 100644 --- a/platforms/oneapi/oneapi.zig +++ b/platforms/oneapi/oneapi.zig @@ -11,7 +11,7 @@ const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/platforms/oneapi"); pub fn isEnabled() bool { - return platforms_options.oneapi; + return platforms_options.oneapi_enabled; } fn hasOneApiDevice(io: std.Io) bool { diff --git a/platforms/platforms.zig b/platforms/platforms.zig index 1e347a5c..3f0c56fd 100644 --- a/platforms/platforms.zig +++ b/platforms/platforms.zig @@ -33,7 +33,7 @@ pub const Platform = enum { pub fn isEnabled(target: Platform) bool { return switch (target) { - inline else => |tag| @field(platforms_options, @tagName(tag)), + inline else => |tag| @field(platforms_options, @tagName(tag) ++ "_enabled"), }; } diff --git a/platforms/rocm/rocm.zig b/platforms/rocm/rocm.zig index 402d5504..59a4de79 100644 --- a/platforms/rocm/rocm.zig +++ b/platforms/rocm/rocm.zig @@ -12,7 +12,7 @@ const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/platforms/rocm"); pub fn isEnabled() bool { - return platforms_options.rocm; + return platforms_options.rocm_enabled; } fn hasRocmDevices(io: std.Io) bool { diff --git a/platforms/tpu/tpu.zig b/platforms/tpu/tpu.zig index bf859e8a..3c8143bb 100644 --- a/platforms/tpu/tpu.zig +++ b/platforms/tpu/tpu.zig @@ -10,7 +10,7 @@ const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/platforms/tpu"); pub fn isEnabled() bool { - return platforms_options.tpu; + return platforms_options.tpu_enabled; } /// Check if running on Google Compute Engine, because TPUs will poll the diff --git a/third_party/arocc/arocc.bazel b/third_party/arocc/arocc.bazel index 8a643feb..853399f5 100644 --- a/third_party/arocc/arocc.bazel +++ b/third_party/arocc/arocc.bazel @@ -4,32 +4,32 @@ load("@zml//bazel:zig_options.bzl", "zig_option", "zig_options") zig_option( name = "enable_linker_build_id", - build_setting_default = False, + bool = False, ) zig_option( name = "linker", - build_setting_default = "", + string = "", ) zig_option( name = "sysroot", - build_setting_default = "", + string = "", ) zig_option( name = "gcc_install_prefix", - build_setting_default = "", + string = "", ) zig_option( name = "rtlib", - build_setting_default = "", + string = "", ) zig_option( name = "unwindlib", - build_setting_default = "", + string = "", ) zig_options( @@ -46,27 +46,27 @@ zig_options( zig_option( name = "enable_tracy", - build_setting_default = False, + bool = False, ) zig_option( name = "enable_tracy_callstack", - build_setting_default = False, + bool = False, ) zig_option( name = "enable_tracy_allocation", - build_setting_default = False, + bool = False, ) zig_option( name = "debug_allocations", - build_setting_default = False, + bool = False, ) zig_option( name = "version_str", - build_setting_default = "", + string = "", ) zig_options( diff --git a/third_party/zio/zio.bazel b/third_party/zio/zio.bazel index 6314c5a0..61cd6ef4 100644 --- a/third_party/zio/zio.bazel +++ b/third_party/zio/zio.bazel @@ -1,28 +1,34 @@ +load("@bazel_skylib//rules:common_settings.bzl", "string_flag") load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") load("@zml//bazel:zig_options.bzl", "zig_option", "zig_options") zig_option( name = "backend", - build_setting_default = "", + string = "", nullable = True, ) -zig_option( +string_flag( name = "resolve_beneath_mode", build_setting_default = "strict", - enum_name = "ResolveBeneathMode", values = ["strict", "best_effort"], ) -zig_option(name = "no_hacks", build_setting_default = False) -zig_option(name = "task_migration", build_setting_default = True) -zig_option(name = "scheduler_metrics", build_setting_default = False) +zig_option( + name = "resolve_beneath_mode_option", + flag = ":resolve_beneath_mode", + enum_name = "ResolveBeneathMode", +) + +zig_option(name = "no_hacks", bool = False) +zig_option(name = "task_migration", bool = True) +zig_option(name = "scheduler_metrics", bool = False) zig_options( name = "zio_options", settings = [ ":backend", - ":resolve_beneath_mode", + ":resolve_beneath_mode_option", ":no_hacks", ":task_migration", ":scheduler_metrics", From 94e79b1ea7c1a814f306c16dd9da74e92a81c8d7 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 21 Sep 2026 19:46:48 +0000 Subject: [PATCH 723/881] third_party/zio: add dynamic page size support this enables GB300 (notably) to work --- third_party/zio/repo.bzl | 5 +- third_party/zio/runtime-page-size.patch | 297 ++++++++++++++++++++++++ 2 files changed, 301 insertions(+), 1 deletion(-) create mode 100644 third_party/zio/runtime-page-size.patch diff --git a/third_party/zio/repo.bzl b/third_party/zio/repo.bzl index 54aca677..4f6e2d78 100644 --- a/third_party/zio/repo.bzl +++ b/third_party/zio/repo.bzl @@ -6,6 +6,9 @@ def repo(): remote = "https://github.com/lalinsky/zio", commit = "417757935ef6d0f0a2396d4023a44ca82ac1901c", build_file = Label("//third_party/zio:zio.bazel"), - patches = [Label("//third_party/zio:progress-parent-file.patch")], + patches = [ + Label("//third_party/zio:progress-parent-file.patch"), + Label("//third_party/zio:runtime-page-size.patch"), + ], patch_args = ["-p1"], ) diff --git a/third_party/zio/runtime-page-size.patch b/third_party/zio/runtime-page-size.patch new file mode 100644 index 00000000..5a1db751 --- /dev/null +++ b/third_party/zio/runtime-page-size.patch @@ -0,0 +1,297 @@ +diff --git a/src/coro/stack.zig b/src/coro/stack.zig +--- a/src/coro/stack.zig ++++ b/src/coro/stack.zig +@@ -10,7 +10,12 @@ + + const log = @import("../common.zig").log; + +-pub const page_size = if (builtin.os.tag == .freestanding) 1 else std.heap.page_size_min; ++// Pointer types need a comptime alignment; mapping geometry needs the OS page size. ++pub const page_alignment = if (builtin.os.tag == .freestanding) 1 else std.heap.page_size_min; ++ ++pub fn pageSize() usize { ++ return if (builtin.os.tag == .freestanding) 1 else std.heap.pageSize(); ++} + + // Signal type changed from c_int to enum in Zig 0.16 + const is_pre_016 = builtin.zig_version.major == 0 and builtin.zig_version.minor < 16; +@@ -24,7 +29,7 @@ + var old_sigbus_action: posix.Sigaction = undefined; + + pub const StackInfo = extern struct { +- allocation_ptr: [*]align(page_size) u8, // deallocation_stack on Windows (TEB offset 0x1478) ++ allocation_ptr: [*]align(page_alignment) u8, // deallocation_stack on Windows (TEB offset 0x1478) + base: usize, // stack_base on Windows (TEB offset 0x08) + limit: usize, // stack_limit on Windows (TEB offset 0x10) + allocation_len: usize, +@@ -45,6 +50,7 @@ + } + + fn stackAllocPosix(info: *StackInfo, maximum_size: usize, committed_size: usize) error{OutOfMemory}!void { ++ const page_size = pageSize(); + // Round maximum_size up to page boundary, then add guard page + const aligned_max = std.mem.alignForward(usize, maximum_size, page_size); + // Ensure we allocate at least 2 pages (guard + usable space) +@@ -102,7 +108,7 @@ + // Commit initial portion at top of stack + const stack_top = @intFromPtr(allocation.ptr) + size; + const initial_commit_start = stack_top - commit_size; +- const initial_region: [*]align(page_size) u8 = @ptrFromInt(initial_commit_start); ++ const initial_region: [*]align(page_alignment) u8 = @ptrFromInt(initial_commit_start); + posix.mprotect(initial_region[0..commit_size], posix.PROT.READ | posix.PROT.WRITE) catch |err| { + log.err("Failed to mprotect stack region (commit_size={d}): {}", .{ commit_size, err }); + return error.OutOfMemory; +@@ -160,6 +166,7 @@ + /// overflow faults on the guard page and crashes (no SIGSEGV growth handler is + /// installed, see setupStackGrowth). + fn stackAllocOpenBSD(info: *StackInfo, size: usize) error{OutOfMemory}!void { ++ const page_size = pageSize(); + const allocation = posix.mmap( + null, + size, +@@ -175,7 +182,7 @@ + + // Turn the lowest page into a PROT_NONE guard page. The rest of the mapping + // keeps its MAP_STACK flag, so the stack pointer stays valid for syscalls. +- const guard: [*]align(page_size) u8 = allocation.ptr; ++ const guard: [*]align(page_alignment) u8 = allocation.ptr; + posix.mprotect(guard[0..page_size], posix.PROT.NONE) catch |err| { + log.err("Failed to mprotect OpenBSD stack guard page: {}", .{err}); + return error.OutOfMemory; +@@ -197,7 +204,7 @@ + /// header. Pages between and below slots stay PROT_NONE, so guard pages cost + /// nothing extra. POSIX-only (the slab pool is comptime-disabled elsewhere); + /// OpenBSD is excluded because MAP_STACK cannot start as PROT_NONE. +-pub fn slabReserve(len: usize) error{OutOfMemory}![]align(page_size) u8 { ++pub fn slabReserve(len: usize) error{OutOfMemory}![]align(page_alignment) u8 { + const prot_flags = posix.PROT.NONE | posix.PROT.MAX(posix.PROT.READ | posix.PROT.WRITE); + var map_flags = posix.MAP.PRIVATE | posix.MAP.ANONYMOUS; + if (builtin.os.tag == .linux or builtin.os.tag == .netbsd) { +@@ -216,7 +223,7 @@ + } + + // Commit the header page for the slab bookkeeping. +- posix.mprotect(allocation.ptr[0..page_size], posix.PROT.READ | posix.PROT.WRITE) catch |err| { ++ posix.mprotect(allocation.ptr[0..pageSize()], posix.PROT.READ | posix.PROT.WRITE) catch |err| { + log.err("Failed to commit stack slab header page: {}", .{err}); + return error.OutOfMemory; + }; +@@ -224,7 +231,7 @@ + return allocation; + } + +-pub fn slabFree(mem: []align(page_size) u8) void { ++pub fn slabFree(mem: []align(page_alignment) u8) void { + posix.munmap(mem) catch {}; + } + +@@ -234,7 +241,8 @@ + /// exactly the layout stackAllocPosix produces, so growth, overflow + /// detection, and pooling treat both kinds identically. One mprotect per + /// cold slot; a recycled slot pays no syscalls at all. +-pub fn stackInitSlot(info: *StackInfo, slot: []align(page_size) u8, committed_size: usize, owner_tag: usize) error{OutOfMemory}!void { ++pub fn stackInitSlot(info: *StackInfo, slot: []align(page_alignment) u8, committed_size: usize, owner_tag: usize) error{OutOfMemory}!void { ++ const page_size = pageSize(); + // Commit at least one page so the pool's FreeNode always fits. + const commit_size = @max(std.mem.alignForward(usize, committed_size, page_size), page_size); + if (commit_size > slot.len - page_size) { +@@ -244,7 +252,7 @@ + + const stack_top = @intFromPtr(slot.ptr) + slot.len; + const initial_commit_start = stack_top - commit_size; +- const initial_region: [*]align(page_size) u8 = @ptrFromInt(initial_commit_start); ++ const initial_region: [*]align(page_alignment) u8 = @ptrFromInt(initial_commit_start); + posix.mprotect(initial_region[0..commit_size], posix.PROT.READ | posix.PROT.WRITE) catch |err| { + log.err("Failed to commit slab stack slot (commit_size={d}): {}", .{ commit_size, err }); + return error.OutOfMemory; +@@ -299,7 +307,7 @@ + const committed_size = info.base - info.limit; + if (committed_size == 0) return; + +- const addr: [*]align(page_size) u8 = @ptrFromInt(committed_start); ++ const addr: [*]align(page_alignment) u8 = @ptrFromInt(committed_start); + + // MADV_FREE is available on Linux 4.5+, macOS, FreeBSD, NetBSD + // It allows lazy reclamation - physical pages are freed when system needs memory +@@ -332,6 +340,7 @@ + /// Mode .grow: Grow by 1.5x current size in 64KB chunks + /// Mode .full: Commit all remaining uncommitted stack + fn stackExtendPosix(info: *StackInfo, mode: StackExtendMode) error{StackOverflow}!void { ++ const page_size = pageSize(); + // OpenBSD stacks are fully committed at allocation time (MAP_STACK regions + // cannot be grown via mprotect), so there is nothing to extend. + if (builtin.os.tag == .openbsd) return; +@@ -341,7 +350,7 @@ + // Calculate new limit based on mode + const new_limit = switch (mode) { + .grow => blk: { +- const chunk_size = 64 * 1024; ++ const chunk_size = @max(64 * 1024, page_size); + const growth_factor_num = 3; + const growth_factor_den = 2; + +@@ -375,7 +384,7 @@ + // Commit the memory region + const commit_start = std.mem.alignBackward(usize, new_limit, page_size); + const commit_size = info.limit - commit_start; +- const addr: [*]align(page_size) u8 = @ptrFromInt(commit_start); ++ const addr: [*]align(page_alignment) u8 = @ptrFromInt(commit_start); + posix.mprotect(addr[0..commit_size], posix.PROT.READ | posix.PROT.WRITE) catch { + return error.StackOverflow; + }; +@@ -385,6 +394,7 @@ + } + + fn stackAllocWindows(info: *StackInfo, maximum_size: usize, committed_size: usize) error{OutOfMemory}!void { ++ const page_size = pageSize(); + // Round sizes up to page boundary + const commit_size = std.mem.alignForward(usize, committed_size, page_size); + const max_size = std.mem.alignForward(usize, maximum_size, page_size); +@@ -457,7 +467,7 @@ + + // Setup alternate stack for this thread if not already done + if (!altstack_installed) { +- const mem = try std.heap.page_allocator.alignedAlloc(u8, .fromByteUnits(page_size), altstack_size); ++ const mem = try std.heap.page_allocator.alignedAlloc(u8, .fromByteUnits(page_alignment), altstack_size); + errdefer std.heap.page_allocator.free(mem); + + var stack = posix.stack_t{ +@@ -599,7 +609,7 @@ + + // Stack layout: [guard_page][uncommitted][committed] + const stack_base = @intFromPtr(stack_info.allocation_ptr); +- const guard_page_end = stack_base + page_size; ++ const guard_page_end = stack_base + pageSize(); + const uncommitted_start = guard_page_end; + const uncommitted_end = stack_info.limit; + +@@ -631,7 +641,7 @@ + const stack_base = @intFromPtr(stack_info.allocation_ptr); + const stack_size = stack_info.allocation_len; + const committed = stack_info.base - stack_info.limit; +- const is_guard_page_fault = fault_addr >= stack_base and fault_addr < stack_base + page_size; ++ const is_guard_page_fault = fault_addr >= stack_base and fault_addr < stack_base + pageSize(); + + const msg = std.fmt.bufPrint( + &buf, +@@ -670,7 +680,7 @@ + // Verify at least the requested amount was committed (minus the owner + // tag reserve, which lives in committed memory above `base` on POSIX). + // Note: RtlCreateUserStack on Windows may commit more than requested +- const commit_size_rounded = std.mem.alignForward(usize, committed_size, page_size); ++ const commit_size_rounded = std.mem.alignForward(usize, committed_size, pageSize()); + const actual_committed = stack.base - stack.limit; + try std.testing.expect(actual_committed + owner_tag_reserve >= commit_size_rounded); + +diff --git a/src/coro/stack_pool.zig b/src/coro/stack_pool.zig +--- a/src/coro/stack_pool.zig ++++ b/src/coro/stack_pool.zig +@@ -33,7 +33,7 @@ + /// of a standalone stack mapping. + const Slab = struct { + next: ?*Slab, +- memory: []align(stack.page_size) u8, ++ memory: []align(stack.page_alignment) u8, + /// Slots handed out at least once. Only the newest slab still carves. + carved: usize, + /// Slots currently acquired by live coroutines. +@@ -114,9 +114,10 @@ + last_shrink: Timestamp, + + pub fn init(config: Config) StackPool { ++ const page_size = stack.pageSize(); + // Same slot layout as stackAllocPosix: usable size rounded to pages, + // plus the (never committed) guard page, at least two pages total. +- const aligned_max = std.mem.alignForward(usize, config.maximum_size, stack.page_size); ++ const aligned_max = std.mem.alignForward(usize, config.maximum_size, page_size); + return .{ + .config = config, + .mutex = .init(), +@@ -125,7 +126,7 @@ + .pool_size = 0, + .slabs = null, + .partial = null, +- .slot_size = @max(aligned_max + stack.page_size, stack.page_size * 2), ++ .slot_size = @max(aligned_max + page_size, page_size * 2), + .in_use = 0, + .epoch_peak = 0, + .retain_target = 0, +@@ -388,11 +389,12 @@ + /// The new slot is counted as in use on its slab; the caller accounts + /// the pool-wide acquire. + fn carveSlotLocked(self: *StackPool) ?StackInfo { ++ const page_size = stack.pageSize(); + const slab = blk: { + if (self.slabs) |s| { + if (s.carved < self.config.slab_slots) break :blk s; + } +- const len = stack.page_size + self.config.slab_slots * self.slot_size; ++ const len = page_size + self.config.slab_slots * self.slot_size; + const mem = stack.slabReserve(len) catch return null; + const s: *Slab = @ptrCast(@alignCast(mem.ptr)); + s.* = .{ .next = self.slabs, .memory = mem, .carved = 0, .in_use = 0, .free = null, .partial_prev = null, .partial_next = null }; +@@ -400,8 +402,8 @@ + break :blk s; + }; + +- const offset = stack.page_size + slab.carved * self.slot_size; +- const slot: []align(stack.page_size) u8 = @alignCast(slab.memory[offset .. offset + self.slot_size]); ++ const offset = page_size + slab.carved * self.slot_size; ++ const slot: []align(stack.page_alignment) u8 = @alignCast(slab.memory[offset .. offset + self.slot_size]); + var stack_info: StackInfo = undefined; + stack.stackInitSlot(&stack_info, slot, self.config.committed_size, @intFromPtr(slab)) catch return null; + slab.carved += 1; +@@ -428,7 +430,7 @@ + /// The slab this allocation was carved from, or null for an individually + /// mapped stack. Must run under mutex (the chain is mutated by carving + /// and shrinking). +- fn slabOfLocked(self: *StackPool, ptr: [*]align(stack.page_size) u8) ?*Slab { ++ fn slabOfLocked(self: *StackPool, ptr: [*]align(stack.page_alignment) u8) ?*Slab { + const addr = @intFromPtr(ptr); + var slab = self.slabs; + while (slab) |s| : (slab = s.next) { +@@ -771,3 +773,36 @@ + pool.shrink(.fromSeconds(30)); + try std.testing.expectEqual(0, pool.pool_size); + } ++ ++test "StackPool: runtime page alignment and full growth" { ++ if (!slab_supported) return error.SkipZigTest; ++ ++ for ([_]usize{ 0, 2 }) |slab_slots| { ++ var pool = StackPool.init(.{ ++ .maximum_size = 8 * 1024 * 1024, ++ .committed_size = 256 * 1024, ++ .slab_slots = slab_slots, ++ }); ++ defer pool.deinit(); ++ ++ // Keep slots live to exercise each offset and cross a slab boundary. ++ var infos: [3]StackInfo = undefined; ++ var acquired: usize = 0; ++ defer for (infos[0..acquired]) |info| pool.release(info); ++ for (&infos) |*info| { ++ info.* = try pool.acquire(); ++ acquired += 1; ++ if (slab_slots != 0) { ++ try std.testing.expect(pool.slabOfLocked(info.allocation_ptr) != null); ++ } ++ try std.testing.expectEqual(0, @intFromPtr(info.allocation_ptr) % stack.pageSize()); ++ try std.testing.expectEqual(0, info.limit % stack.pageSize()); ++ try stack.stackExtend(info, .grow); ++ try stack.stackExtend(info, .full); ++ try std.testing.expectEqual(@intFromPtr(info.allocation_ptr) + stack.pageSize(), info.limit); ++ const memory: [*]u8 = @ptrFromInt(info.limit); ++ memory[0] = 0xAA; ++ memory[info.base - info.limit - 1] = 0xBB; ++ } ++ } ++} +diff --git a/src/stack_test.zig b/src/stack_test.zig +--- /dev/null ++++ b/src/stack_test.zig +@@ -0,0 +1,4 @@ ++comptime { ++ _ = @import("coro/stack.zig"); ++ _ = @import("coro/stack_pool.zig"); ++} From 2397718ca38133916833435d6aab9156d9919417 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Fri, 25 Sep 2026 07:15:52 +0000 Subject: [PATCH 724/881] zml: enable xla_gpu_cudnn_gemm_fusion_level=2 on cuda --- zml/Compiler.zig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zml/Compiler.zig b/zml/Compiler.zig index f946e982..b67b81f2 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -857,6 +857,8 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: // Enable user buffers to assign color 1 and receive the reclaim exemption: // https://github.com/openxla/xla/pull/46029 try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_nccl_user_buffers", true, upb_arena); + // Enable for both Blackwell+ and Ampere+ + try setXlaOverrideFlag(overrides_map, "xla_gpu_cudnn_gemm_fusion_level", 2, upb_arena); }, .rocm => { try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_scheduling_mode", "CONCURRENT", upb_arena); From c6530f08d39d9c502bd2b3c7035d73bbaf7196c9 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Fri, 25 Sep 2026 13:52:26 +0200 Subject: [PATCH 725/881] workspace: bump plugins (#857) --- platforms/cpu/cpu.bzl | 6 +++--- platforms/cuda/cuda.bzl | 8 ++++---- platforms/metal/metal.bzl | 4 ++-- platforms/rocm/rocm.bzl | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index b3fa8dc6..06bde701 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -43,21 +43,21 @@ def _cpu_plugin_impl(mctx): name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, sha256 = "16f0040b1805864fc0c3e13fe15f8ffccaf885a59b4f625666d2618583c0ac50", - url = "https://mirror.zml.ai/plugins/202609241513.97.1.78be97663f43/zml-cpu-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202609250917.103.1.3c8a2e14f8d9/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "5da35f0e471a410f3a3edba95f8a849ea017c1ed32c6fab0239a25a5d21acad4", - url = "https://mirror.zml.ai/plugins/202609241513.97.1.78be97663f43/zml-cpu-darwin-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202609250917.103.1.3c8a2e14f8d9/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "b0f6a8a1486780a9433b08153d71e0e31ed0e4cde60f200346c687c9346b2d53", - url = "https://mirror.zml.ai/plugins/202609241513.97.1.78be97663f43/zml-cpu-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202609250917.103.1.3c8a2e14f8d9/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 3c021b94..1390abab 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -224,12 +224,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "58e07d6766d3f9b1262d9ab7f4a6aadf5358df517afd75d0952a305c65497361", - "url": "https://mirror.zml.ai/plugins/202609241513.97.1.78be97663f43/zml-cuda-linux-amd64.tar.zst", + "sha256": "2b10512d132339c16b38c0e9c76e4271b1b1f533e7988fb4ba019c5864df00db", + "url": "https://mirror.zml.ai/plugins/202609250917.103.1.3c8a2e14f8d9/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "8118bb6ce9b6753cf6f35e026b5798e28cdca328ae236c63822a6fd8d9d5514f", - "url": "https://mirror.zml.ai/plugins/202609241513.97.1.78be97663f43/zml-cuda-linux-arm64.tar.zst", + "sha256": "0ed505366b24ea0fc4b4d1137a5c7e76c6dffb056414d93f7f355e2d7ac09475", + "url": "https://mirror.zml.ai/plugins/202609250917.103.1.3c8a2e14f8d9/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 49bd84f4..ecc454a7 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "d5f51655129ec9ac289560eefd9e273e4a5780ed58fe4d4acf725e65e0e8b7c9", - url = "https://mirror.zml.ai/plugins/202609241513.97.1.78be97663f43/zml-metal-darwin-arm64.tar.zst", + sha256 = "176b9d08cba22871a5c53e33da26235509612f6ce22247e3ee3d6ea8f5d206b0", + url = "https://mirror.zml.ai/plugins/202609250917.103.1.3c8a2e14f8d9/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 97e431f8..6c272eb0 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202609241513.97.1.78be97663f43/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "abffc6b901d3132d112d8858a808553839de174d35035b8798919426f74b3db6" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202609250917.103.1.3c8a2e14f8d9/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "5b4a3a37b6e55c221a4a8f6f859d979e8f0fab28b09c1e15d81a556da13c53b9" def _rocm_package_name(name): return name + _ROCM_VERSION From 90618b8a7d1eb79779509a485dea40e802274656 Mon Sep 17 00:00:00 2001 From: Louis Lechevalier Date: Mon, 28 Sep 2026 18:36:44 +0200 Subject: [PATCH 726/881] zml/ops: Up tensors size to trigger the gemm fusion dot lowering in XLA (#863) --- zml/ops.zig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zml/ops.zig b/zml/ops.zig index 1e04de12..adaa3966 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -762,8 +762,8 @@ test "if" { { const pred: Tensor = .init(.{}, .i32); - const a: Tensor = .init(.{ 4, 4 }, .f32); - const b: Tensor = .init(.{ 4, 4 }, .f32); + const a: Tensor = .init(.{ 8, 8 }, .f32); + const b: Tensor = .init(.{ 8, 8 }, .f32); const mod = try platform.compileFn(allocator, std.testing.io, IfMod._fwd, .{ pred, a, b }, .{}); defer mod.deinit(); } @@ -842,8 +842,8 @@ test if2 { { const pred: Tensor = .init(.{}, .i32); - const a: Tensor = .init(.{ 4, 4 }, .f32); - const b: Tensor = .init(.{ 4, 4 }, .f32); + const a: Tensor = .init(.{ 8, 8 }, .f32); + const b: Tensor = .init(.{ 8, 8 }, .f32); const mod = try platform.compileFn(allocator, std.testing.io, IfMod._fwd, .{ pred, a, b }, .{}); defer mod.deinit(); } From 74c590b0b0b4ae44f94e4a3e5d6fc4f36e2bcfe9 Mon Sep 17 00:00:00 2001 From: Louis Lechevalier Date: Mon, 28 Sep 2026 18:53:26 +0200 Subject: [PATCH 727/881] zml/moe: Refactoring MoE selection and packing contracts (#862) Refactoring of the MoE selection and add repacking contract required by the model inits. This solve a design problem in the model init : - Before, backend selection required the resulting MoE after repacking (dtype), and repacking required to know the selected backend. - Now, backend selection is based on raw weights loaded from HF (quantization scheme and dtype). Then it induces the packing. Add useful structs for packing that are specific to MoE (`ExpertsLayout`). --- examples/llm/models/qwen3_5_moe/model.zig | 7 +- zml/moe/cute_kernels/moe.zig | 4 +- zml/moe/cute_mxfp4.zig | 11 -- zml/moe/fused_experts.zig | 26 ++--- zml/moe/moe.zig | 134 ++++++++++++---------- zml/moe/triton.zig | 2 +- 6 files changed, 90 insertions(+), 94 deletions(-) diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 7daa9750..f4096bfb 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -150,9 +150,9 @@ pub const LoadedModel = struct { progress: *std.Progress.Node, ) !inference.CompiledModel { _ = backend; - const moe_dtype = self.inner.text_model.layers[0].moe.gate_up_proj.weight.dtype(); - log.info("Moe dtype : {}", .{moe_dtype}); - const moe_backend = try zml.moe.Backend.auto(platform, moe_dtype); + const moe_gate_up = self.inner.text_model.layers[0].moe.gate_up_proj; + log.info("Moe dtype : {}", .{moe_gate_up.weight.dtype()}); + const moe_backend = try zml.moe.Backend.auto(platform, moe_gate_up.quantizationScheme(), moe_gate_up.weight.dtype()); const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), moe_backend, shardings); return inference.CompiledModel.init(allocator, io, platform, self, self.inner, params, progress); } @@ -779,7 +779,6 @@ pub const Moe = struct { self.down_proj, .{ .quantize_input = false, - .gate_up_layout = .split, .routing_weight_placement = .after_down, }, moe_parameters, diff --git a/zml/moe/cute_kernels/moe.zig b/zml/moe/cute_kernels/moe.zig index f9d48499..ca41474d 100644 --- a/zml/moe/cute_kernels/moe.zig +++ b/zml/moe/cute_kernels/moe.zig @@ -16,10 +16,10 @@ pub const Inputs = struct { /// `[tokens, topk]` FP32. routing_weights: zml.Tensor, w1: zml.Tensor, - /// `w1` scales in the 128x4 layout of `packWeightScales`. + /// `w1` scales in the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). s1: zml.Tensor, w2: zml.Tensor, - /// `w2` scales in the 128x4 layout of `packWeightScales`. + /// `w2` scales in the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). s2: zml.Tensor, /// `[tokens, topk]` local expert ids. ids: zml.Tensor, diff --git a/zml/moe/cute_mxfp4.zig b/zml/moe/cute_mxfp4.zig index 10fb3ec7..3e2d0edf 100644 --- a/zml/moe/cute_mxfp4.zig +++ b/zml/moe/cute_mxfp4.zig @@ -11,17 +11,6 @@ test { pub const Parameters = triton_mxfp4.Parameters; -/// Pack E8M0 weight scales in the exact 128x4 swizzle consumed by the TMA -/// scale-factor descriptor. Loaders apply this once at load time. -pub fn packWeightScales(scales: zml.Tensor) zml.Tensor { - const experts = scales.dim(0); - const rows = scales.dim(1); - const groups = scales.dim(2); - var packed_scales = scales.reshape(.{ experts, @divExact(rows, 128), 4, 32, @divExact(groups, 4), 4 }) - .transpose(.{ 0, 1, 4, 3, 2, 5 }); - return packed_scales.reshape(scales.shape()).withPartitioning(.{ .expert = .experts }); -} - pub fn isAvailable(platform: *const zml.Platform) bool { if (platform.target != .cuda) return false; const cc = zml.platform.cuda.computeCapability(platform) orelse return false; diff --git a/zml/moe/fused_experts.zig b/zml/moe/fused_experts.zig index 026f4865..9d492e10 100644 --- a/zml/moe/fused_experts.zig +++ b/zml/moe/fused_experts.zig @@ -39,7 +39,7 @@ pub const Parameters = struct { } }; -pub const GateUpLayout = enum { split, interleaved }; +pub const ProjectionLayout = enum { concatenated, interleaved }; pub const RoutingWeightPlacement = enum { before_down, after_down }; pub const FusedExpertsArgs = struct { @@ -53,7 +53,7 @@ pub const FusedExpertsArgs = struct { activation_threshold: ?f32 = null, /// Use FP8 activations for FP8 weights; false keeps BF16 activations. quantize_input: bool, - gate_up_layout: GateUpLayout, + gate_up_layout: ProjectionLayout, routing_weight_placement: RoutingWeightPlacement, }; @@ -166,7 +166,7 @@ pub fn fusedExperts(opts: FusedExpertsArgs, comptime backend: zml.moe.Backend) ! return output.reshape(.{ .b = b, .token = s, .out = down.dim(.out) }); } -fn applyExpertActivation(input: Tensor, mode: Parameters.ActivationMode, activation_threshold: ?f32, layout: GateUpLayout) Tensor { +fn applyExpertActivation(input: Tensor, mode: Parameters.ActivationMode, activation_threshold: ?f32, layout: ProjectionLayout) Tensor { const x = input.convert(.f32); if (mode == .relu) { const clipped = if (activation_threshold) |limit| x.minimum(Tensor.scalar(limit, x.dtype())) else x; @@ -175,7 +175,7 @@ fn applyExpertActivation(input: Tensor, mode: Parameters.ActivationMode, activat const mid = @divFloor(x.dim(.out), 2); var gate, var up = switch (layout) { - .split => .{ x.slice(.out, .{ .end = mid }), x.slice(.out, .{ .start = mid }) }, + .concatenated => .{ x.slice(.out, .{ .end = mid }), x.slice(.out, .{ .start = mid }) }, .interleaved => .{ x.slice(.out, .{ .start = 0, .step = 2 }), x.slice(.out, .{ .start = 1, .step = 2 }) }, }; if (activation_threshold) |limit_| { @@ -190,23 +190,23 @@ fn applyExpertActivation(input: Tensor, mode: Parameters.ActivationMode, activat }; } -test "SwiGLU uses FP32 math for split and interleaved BF16 inputs" { +test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { const allocator = std.testing.allocator; const io = std.testing.io; const platform = zml.testing.env(); const Local = struct { - fn forward(x: Tensor, layout: GateUpLayout, threshold: ?f32) Tensor { + fn forward(x: Tensor, layout: ProjectionLayout, threshold: ?f32) Tensor { return applyExpertActivation(x, .silu, threshold, layout); } }; const x: Tensor = .init(.{ .token = 1, .out = 6 }, .bf16); const gates = [_]f32{ 0.75, -1.25, 3.5 }; const ups = [_]f32{ 0.875, -2.25, 4.5 }; - for ([_]GateUpLayout{ .split, .interleaved }) |layout| { + for ([_]ProjectionLayout{ .concatenated, .interleaved }) |layout| { var values: [6]zml.floats.BFloat16 = undefined; for (gates, ups, 0..) |gate, up, i| { - values[if (layout == .split) i else 2 * i] = .fromF32(gate); - values[if (layout == .split) i + 3 else 2 * i + 1] = .fromF32(up); + values[if (layout == .concatenated) i else 2 * i] = .fromF32(gate); + values[if (layout == .concatenated) i + 3 else 2 * i + 1] = .fromF32(up); } var input = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&values)); defer input.deinit(); @@ -233,7 +233,7 @@ test "ReLU squared activation preserves width and applies threshold before squar const platform = zml.testing.env(); const Local = struct { fn forward(x: Tensor, threshold: ?f32) Tensor { - return applyExpertActivation(x, .relu, threshold, .split); + return applyExpertActivation(x, .relu, threshold, .concatenated); } }; const x: Tensor = .init(.{ .token = 1, .out = 5 }, .f32); @@ -400,11 +400,11 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { const platform = zml.testing.env(); if (platform.target != .cuda) return error.SkipZigTest; const Local = struct { - fn forward(x: Tensor, layout: GateUpLayout, placement: RoutingWeightPlacement, storage_dtype: DataType) Tensor { + fn forward(x: Tensor, layout: ProjectionLayout, placement: RoutingWeightPlacement, storage_dtype: DataType) Tensor { const fp4 = storage_dtype != .bf16; const columns = Tensor.arange(.{ .end = 256 }, .i32).withTags(.{.dout}); const is_gate = switch (layout) { - .split => columns.cmp(.LT, Tensor.scalar(128, .i32)), + .concatenated => columns.cmp(.LT, Tensor.scalar(128, .i32)), .interleaved => columns.remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), }; const dtype: DataType = if (fp4) .u8 else .bf16; @@ -457,7 +457,7 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { @memset(host.items(zml.floats.BFloat16), .fromF32(1.0 / 128.0)); var input = try zml.Buffer.fromSlice(io, platform, host, .replicated); defer input.deinit(); - for ([_]GateUpLayout{ .split, .interleaved }) |layout| { + for ([_]ProjectionLayout{ .concatenated, .interleaved }) |layout| { for ([_]RoutingWeightPlacement{ .before_down, .after_down }) |placement| { var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, placement, storage_dtype }, .{}); defer exe.deinit(); diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index 8adaad69..7fa57104 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -13,11 +13,30 @@ pub const triton = @import("triton.zig"); pub const fly = @import("fly_kernels/moe.zig"); const fused_experts = @import("fused_experts.zig"); pub const triton_kernels = @import("triton_kernels/triton_kernels.zig"); +pub const ProjectionLayout = fused_experts.ProjectionLayout; test { std.testing.refAllDecls(@This()); } +/// How a backend expects the expert weights to be stored +/// The packer writes this layout and forwardMoe reads it +pub const ExpertsLayout = struct { + /// Column order of the fused gate/up projection. + gate_up: ProjectionLayout, + /// Backend specific transformation of the weights and scales. + packing: Packing, + + pub const Packing = enum { + /// Experts stacked as stored in the checkpoint. + plain, + /// Block scales swizzled to the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). + swizzled_scales, + /// Weights, block scales and global scales in the FlashInfer CUTLASS NVFP4 layout. + flashinfer_nvfp4, + }; +}; + pub const ActivationMode = enum { silu, relu, @@ -33,70 +52,53 @@ pub const Backend = enum { mosaic_tpu, metal, - pub fn auto(platform: *const zml.Platform, weights_dtype: zml.DataType) !Backend { + pub fn auto(platform: *const zml.Platform, scheme: ?zml.Quantization.Scheme, dtype: zml.DataType) !Backend { + // Keep the dtype as an argument because non scheme-specific backends may depend on it later + _ = dtype; return switch (platform.target) { - .cuda => switch (weights_dtype) { - .bf16 => if (cutlass_flashinfer.isAvailable(platform)) - .flashinfer_cutlass - else - .triton, - .f4e2m1 => if (cutlass_flashinfer.isNvfp4Supported(platform)) - .flashinfer_cutlass - else - return error.UnsupportedDataType, - .f8e4m3fn, .f8e4m3fnuz, .f8e8m0, .f16, .f32 => .triton, - else => error.UnsupportedDataType, + .cuda => b: { + const s = scheme orelse break :b .triton; + break :b switch (s) { + .mxfp4 => if (cute_mxfp4.isAvailable(platform)) + .cute_mxfp4 + else if (triton_mxfp4.isAvailable(platform)) + .triton_mxfp4 + else + .triton, + .nvfp4 => if (cutlass_flashinfer.isNvfp4Supported(platform)) + .flashinfer_cutlass + else + error.UnsupportedQuantization, + .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block128, .fp8_block32 => .triton, + }; }, - .rocm => switch (weights_dtype) { - .bf16, .f16, .f32, .f8e4m3fn, .f8e4m3fnuz, .f8e8m0 => .triton, - else => error.UnsupportedDataType, + .rocm => b: { + const s = scheme orelse break :b .triton; + break :b switch (s) { + .mxfp4 => if (zml.platform.rocm.computeCapability(platform) == .gfx942) .fly else .triton, + .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block128, .fp8_block32 => .triton, + .nvfp4 => error.UnsupportedQuantization, + }; }, - .oneapi => switch (weights_dtype) { - .bf16, .f16, .f32 => .triton, - else => error.UnsupportedDataType, + .oneapi => b: { + const s = scheme orelse break :b .triton; + break :b switch (s) { + .mxfp4 => .triton, + .nvfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block128, .fp8_block32 => error.UnsupportedQuantization, + }; }, - .tpu => switch (weights_dtype) { - .bf16, .f16, .f32 => .mosaic_tpu, - else => error.UnsupportedDataType, - }, - .metal => switch (weights_dtype) { - .bf16, .f16, .f32, .f4e2m1, .u8, .f8e4m3fn => .metal, - else => error.UnsupportedDataType, + .tpu => if (scheme == null) .mosaic_tpu else error.UnsupportedQuantization, + .metal => b: { + const s = scheme orelse break :b .metal; + break :b switch (s) { + .nvfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block128, .fp8_block32 => .metal, + .mxfp4 => error.UnsupportedQuantization, + }; }, else => error.UnimplementedMoEBackend, }; } - pub fn autoMxfp4(platform: *const zml.Platform, weights_dtype: zml.DataType) !Backend { - return switch (platform.target) { - .cuda => switch (if (zml.platform.cuda.computeCapability(platform)) |cc| cc.major else 0) { - 10 => switch (weights_dtype) { - .u8, .i8 => if (cute_mxfp4.isAvailable(platform)) .cute_mxfp4 else .triton_mxfp4, - .f4e2m1 => .triton, - else => error.UnsupportedDataType, - }, - else => switch (weights_dtype) { - .u8, .i8, .f4e2m1 => .triton, - else => error.UnsupportedDataType, - }, - }, - .rocm => switch (zml.platform.rocm.computeCapability(platform) orelse return .triton) { - .gfx942 => switch (weights_dtype) { - .u8, .i8, .f4e2m1 => .fly, - else => error.UnsupportedDataType, - }, - else => switch (weights_dtype) { - .u8, .i8, .f4e2m1 => .triton, - else => error.UnsupportedDataType, - }, - }, - else => switch (weights_dtype) { - .u8, .i8, .f4e2m1 => .triton, - else => error.UnsupportedDataType, - }, - }; - } - pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { return switch (backend) { .triton_mxfp4 => triton_mxfp4.isAvailable(platform), @@ -115,6 +117,17 @@ pub const Backend = enum { }; } + /// The layout contract the expert weights must have for a given backend with scheme + pub fn expertsLayout(backend: Backend, scheme: ?zml.Quantization.Scheme) !ExpertsLayout { + return switch (backend) { + .cute_mxfp4 => if (scheme == .mxfp4) .{ .gate_up = .interleaved, .packing = .swizzled_scales } else error.UnsupportedQuantization, + .triton_mxfp4 => if (scheme == .mxfp4) .{ .gate_up = .interleaved, .packing = .plain } else error.UnsupportedQuantization, + .flashinfer_cutlass => if (scheme == .nvfp4) .{ .gate_up = .concatenated, .packing = .flashinfer_nvfp4 } else error.UnsupportedQuantization, + .mosaic_tpu, .metal => .{ .gate_up = .concatenated, .packing = .plain }, + .triton, .fly => if (scheme == .mxfp4) .{ .gate_up = .interleaved, .packing = .plain } else .{ .gate_up = .concatenated, .packing = .plain }, + }; + } + pub fn register(backend: Backend, platform: *zml.Platform) !void { return switch (backend) { .cute_mxfp4, .triton_mxfp4 => {}, @@ -199,8 +212,6 @@ pub const Options = struct { activation_threshold: ?f32 = null, /// Quantize activations for Triton FP8 GEMMs; false keeps BF16 activations. quantize_input: bool, - /// Gate/up layout; FlashInfer, Mosaic and Metal require split columns. - gate_up_layout: fused_experts.GateUpLayout, /// Where routing weights are applied; FlashInfer, Mosaic and Metal require after_down. routing_weight_placement: fused_experts.RoutingWeightPlacement, }; @@ -215,16 +226,12 @@ pub fn forwardMoe( parameters: Parameters, ) !zml.Tensor { switch (parameters) { - .triton, .fly => {}, + .triton, .fly, .cute_mxfp4, .triton_mxfp4 => {}, .flashinfer_cutlass, .mosaic_tpu, .metal => { stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); - stdx.debug.assert(opts.gate_up_layout == .split, "Non-Triton MoE backends require split gate/up columns", .{}); stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); stdx.debug.assert(opts.activation_threshold == null, "Activation thresholds require the Triton MoE backend", .{}); }, - .cute_mxfp4, .triton_mxfp4 => { - stdx.debug.assert(opts.gate_up_layout == .interleaved, "Triton MXFP4 MoE backends require interleaved gate/up columns", .{}); - }, } const gate_up_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; @@ -449,6 +456,7 @@ pub fn forwardMoe( ); }, inline .triton, .fly => |p, backend| b: { + const layout = try backend.expertsLayout(gate_up_scheme); const args: fused_experts.FusedExpertsArgs = .{ .hidden_states = input, .gate_up = gate_up, @@ -458,7 +466,7 @@ pub fn forwardMoe( .activation = p.activation, .activation_threshold = opts.activation_threshold, .quantize_input = opts.quantize_input, - .gate_up_layout = opts.gate_up_layout, + .gate_up_layout = layout.gate_up, .routing_weight_placement = opts.routing_weight_placement, }; const expert_partition = gate_up.weight.shape().partition(.expert); diff --git a/zml/moe/triton.zig b/zml/moe/triton.zig index 7af7f809..7bb0d2be 100644 --- a/zml/moe/triton.zig +++ b/zml/moe/triton.zig @@ -9,7 +9,7 @@ const DataType = zml.DataType; const toDType = zml.kernel.triton.from; const shared = @import("fused_experts.zig"); pub const Parameters = shared.Parameters; -pub const GateUpLayout = shared.GateUpLayout; +pub const ProjectionLayout = shared.ProjectionLayout; pub const RoutingWeightPlacement = shared.RoutingWeightPlacement; pub const FusedExpertsArgs = shared.FusedExpertsArgs; const kernels = @import("triton_kernels/triton_kernels.zig"); From 337ce099fc6f1f7c757449ab195807fdfa7f1111 Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Tue, 29 Sep 2026 10:29:37 +0200 Subject: [PATCH 728/881] zml/tokenizer: add function to retrieve the token corresponding using its ID (#851) --- zml/tokenizer/iree/tokenizer.zig | 23 +++++++++++++++++++ zml/tokenizer/sentencepiece/sentencepiece.i | 2 ++ zml/tokenizer/sentencepiece/sentencepiece.zig | 5 ++++ zml/tokenizer/tokenizer.zig | 6 +++++ 4 files changed, 36 insertions(+) diff --git a/zml/tokenizer/iree/tokenizer.zig b/zml/tokenizer/iree/tokenizer.zig index f2a62fe6..bc986d4b 100644 --- a/zml/tokenizer/iree/tokenizer.zig +++ b/zml/tokenizer/iree/tokenizer.zig @@ -149,6 +149,18 @@ pub const Tokenizer = struct { return if (id < 0) null else @intCast(id); } + pub fn decode(self: *const Tokenizer, token_id: u32) ?[]const u8 { + const vocab = c.iree_tokenizer_vocab(self.inner); + if (vocab == null) return null; + + if (token_id > std.math.maxInt(i32)) return null; + + const text = c.iree_tokenizer_vocab_token_text(vocab, @intCast(token_id)); + if (text.data == null) return null; + + return text.data[0..text.size]; + } + pub const Encoder = struct { tokenizer: *const Tokenizer, state_storage: []u8, @@ -1232,6 +1244,17 @@ test "tokenId lookup" { try std.testing.expect(tokenizer.tokenId("") == null); } +test "tokenId <-> decode" { + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + for ([_][]const u8{ "[UNK]", "hello", "world", "foo", "bar" }) |text| { + try std.testing.expectEqualSlices(u8, text, tokenizer.decode(tokenizer.tokenId(text).?).?); + } +} + test "writer with pre-allocated output writer" { // Pass an already-allocated aligned output writer to the encoder stream. const allocator = std.testing.allocator; diff --git a/zml/tokenizer/sentencepiece/sentencepiece.i b/zml/tokenizer/sentencepiece/sentencepiece.i index 01476b59..d89489c7 100644 --- a/zml/tokenizer/sentencepiece/sentencepiece.i +++ b/zml/tokenizer/sentencepiece/sentencepiece.i @@ -85,6 +85,8 @@ namespace sentencepiece { virtual sentencepiece::util::Status Encode(absl::string_view input, std::vector *ids) const; virtual sentencepiece::util::Status Decode(const std::vector &ids, std::string *detokenized) const; virtual int PieceToId(absl::string_view piece) const; + virtual const std::string &IdToPiece(int id) const; + virtual int GetPieceSize() const; virtual int unk_id() const; virtual int bos_id() const; virtual int eos_id() const; diff --git a/zml/tokenizer/sentencepiece/sentencepiece.zig b/zml/tokenizer/sentencepiece/sentencepiece.zig index 9ba19001..a09e0c0a 100644 --- a/zml/tokenizer/sentencepiece/sentencepiece.zig +++ b/zml/tokenizer/sentencepiece/sentencepiece.zig @@ -195,4 +195,9 @@ pub const SentencePieceProcessor = opaque { pub fn tokenId(self: *SentencePieceProcessor, token: []const u8) u32 { return @intCast(c.SentencePieceProcessor_PieceToId(@ptrCast(self), ffi.ZigSlice.from(token))); } + + pub fn decode(self: *SentencePieceProcessor, token_id: u32) ?[]const u8 { + if (token_id >= @as(u32, @intCast(c.SentencePieceProcessor_GetPieceSize(@ptrCast(self))))) return null; + return ffi.ZigSlice.to(u8, c.SentencePieceProcessor_IdToPiece(@ptrCast(self), @intCast(token_id))); + } }; diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig index 3b6b1e99..781df814 100644 --- a/zml/tokenizer/tokenizer.zig +++ b/zml/tokenizer/tokenizer.zig @@ -183,6 +183,12 @@ pub const Tokenizer = union(Tokenizers) { inline else => |v| v.tokenId(token), }; } + + pub fn decode(self: *const Tokenizer, token_id: u32) ?[]const u8 { + return switch (self.*) { + inline else => |v| v.decode(token_id), + }; + } }; test { From 8745ef8b3055442b9ba0ef15974854c7300f89ad Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 29 Sep 2026 12:43:12 +0200 Subject: [PATCH 729/881] Compiler: disable cudnn fusion (#867) Too many problems for now in XLA: - https://github.com/zml/zml/pull/863 - a bf16 dot failed We have fixes incoming for those, but unclear whether it's enough --- zml/Compiler.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/Compiler.zig b/zml/Compiler.zig index b67b81f2..547ea6b3 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -858,7 +858,7 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: // https://github.com/openxla/xla/pull/46029 try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_nccl_user_buffers", true, upb_arena); // Enable for both Blackwell+ and Ampere+ - try setXlaOverrideFlag(overrides_map, "xla_gpu_cudnn_gemm_fusion_level", 2, upb_arena); + // try setXlaOverrideFlag(overrides_map, "xla_gpu_cudnn_gemm_fusion_level", 2, upb_arena); }, .rocm => { try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_scheduling_mode", "CONCURRENT", upb_arena); From 94278ee4ed6d10bad45d28b1aa12e27753e09ab7 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 29 Sep 2026 12:45:01 +0200 Subject: [PATCH 730/881] zml/quantization: Improve FP8 quantization (#859) The changed code needs to round up the scale to the next power of two. The math is: ``` (bits + 0x7fffff) & 0xff800000 ``` Since 0x7fffff is \(2^{23}-1\): - When \(m=0\), adding it fills the fraction bits without carrying into the exponent. - When \(m>0\), the addition carries exactly once into the exponent. - The mask then clears all 23 fraction bits. --- zml/quantization.zig | 65 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/zml/quantization.zig b/zml/quantization.zig index dc82f558..b8beb352 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -155,10 +155,10 @@ pub fn quantizeBlockFp8(x: Tensor, axis: anytype, block_size: i64, dtype: DataTy .max(.fp8_block), .f8e8m0 => blk: { const raw_scale = grouped.abs().maximum(.scalar(1e-4, .f32)).scale(1.0 / fp8_max).max(.fp8_block); + // Round a positive F32 scale up to a power of two. Adding all + // mantissa bits carries into the exponent iff the mantissa is nonzero. const bits = raw_scale.bitCast(.u32); - const exponent = bits.shiftRightLogical(.scalar(23, .u32)); - const fractional = bits.logical(.AND, .scalar(0x7fffff, .u32)).cmp(.NE, .scalar(0, .u32)).convert(.u32); - break :blk exponent.add(fractional).shiftLeft(.scalar(23, .u32)).bitCast(.f32); + break :blk bits.addConstant(0x7fffff).logical(.AND, .scalar(0xff800000, .u32)).bitCast(.f32); }, else => stdx.debug.panic("expected F32 or E8M0 scale dtype, got {s}", .{@tagName(scale_dtype)}), }; @@ -404,3 +404,62 @@ test "block FP8 quantization preserves axes and reconstructs constant blocks in } } } + +test "block FP8 E8M0 scales preserve power-of-two boundaries and nonminor axes" { + const zml = @import("zml.zig"); + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + inline for (.{ DataType.f8e4m3fn, DataType.f8e4m3fnuz }) |dtype| { + const Local = struct { + const Outputs = struct { values: Tensor, scales: Tensor }; + fn forward(x: Tensor) Outputs { + const input = quantizeBlockFp8(x, .k, 32, dtype, .f8e8m0); + return .{ .values = input.values.convert(.f32), .scales = input.scales.convert(.f32) }; + } + }; + const x: Tensor = .init(.{ .b = 2, .k = 256, .s = 3 }, .f32); + var exe = try platform.compileFn(allocator, io, Local.forward, .{x}, .{}); + defer exe.deinit(); + try zml.testing.expectEqualShapes(x.shape().withDtype(.f32), exe.output_shapes[0]); + try zml.testing.expectEqualShapes(x.shape().setDim(.k, 8).withDtype(.f32), exe.output_shapes[1]); + const fp8_max: f32 = if (dtype == .f8e4m3fn) 448.0 else 224.0; + var host: [2 * 256 * 3]f32 = undefined; + var maxima = [_]f32{0} ** (2 * 8 * 3); + for (&host, 0..) |*value, i| { + const block = (i / 3) % 256 / 32; + const scale_index = (i / (256 * 3)) * 24 + block * 3 + i % 3; + const boundary = fp8_max * std.math.pow(f32, 2, @as(f32, @floatFromInt(block)) - 4); + const bits: u32 = @bitCast(boundary); + const magnitude: f32 = switch (i % 3) { + 0 => @bitCast(bits - 1), + 1 => boundary, + else => @bitCast(bits + 1), + }; + value.* = if (block == 0) 0 else if (block == 1) 1e-30 else magnitude * (if (i % 2 == 0) @as(f32, 1) else -1); + maxima[scale_index] = @max(maxima[scale_index], @abs(value.*)); + } + var expected_scales: [2 * 8 * 3]f32 = undefined; + for (&expected_scales, maxima) |*scale, maximum| { + const raw = @max(maximum, 1e-4) * (1.0 / fp8_max); + const bits: u32 = @bitCast(raw); + const exponent = (bits >> 23) + @intFromBool(bits & 0x7fffff != 0); + scale.* = @bitCast(exponent << 23); + } + var buffer = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&host)); + defer buffer.deinit(); + var output = try zml.testing.autoCall(allocator, io, &exe, Local.forward, .{buffer}); + defer zml.Buffer.deinitAll(Local.Outputs, &output); + var values = try output.values.toSliceAlloc(allocator, io); + defer values.free(allocator); + var scales = try output.scales.toSliceAlloc(allocator, io); + defer scales.free(allocator); + try std.testing.expectEqualSlices(f32, &expected_scales, scales.constItems(f32)); + for (values.constItems(f32), 0..) |value, i| { + const scale_index = (i / (256 * 3)) * 24 + ((i / 3) % 256) / 32 * 3 + i % 3; + const normalized = std.math.clamp(host[i] / expected_scales[scale_index], -fp8_max, fp8_max); + const expected = if (dtype == .f8e4m3fn) zml.floats.Float8E4M3FN.fromF32(normalized).toF32() else zml.floats.Float8E4M3FNUZ.fromF32(normalized).toF32(); + try std.testing.expectEqual(expected, value); + } + } +} From 8392254013ba5858dcc74ecf8a2a62052a0279d6 Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Tue, 29 Sep 2026 13:57:37 +0200 Subject: [PATCH 731/881] zml/attention: support sink (#864) Required for Mimo 2.6 --- zml/attention.zig | 2 +- zml/attention/paged_attention.zig | 3 ++- zml/attention/triton_attention.zig | 22 ++++++++++++++-------- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/zml/attention.zig b/zml/attention.zig index c3cb8b47..2740c547 100644 --- a/zml/attention.zig +++ b/zml/attention.zig @@ -171,7 +171,7 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T break :b attn_output; }, .attnd => attnd.causalAttention(q, k, v, token_index, metadata.attnd, parameters.attnd), - .nki => |params| nki.attention(q, k, v, token_index, params), + .nki => nki.attention(q, k, v, token_index, parameters.nki), .cuda_fa2 => flashattn.fa2.attention(q, k, v, token_index, metadata.cuda_fa2, parameters.cuda_fa2), .cuda_fa3 => flashattn.fa3.attention(q, k, v, token_index, metadata.cuda_fa3, parameters.cuda_fa3), .metal_fa => metal.attention(q, k, v, token_index, metadata.metal_fa), diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index d3a378ae..818a19b3 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -203,6 +203,7 @@ pub const AttentionOptions = struct { is_causal: bool = true, sliding_window: i32 = -1, scale: ?f32 = null, + sink: ?zml.Tensor = null, }; pub const KvCache = union(enum) { @@ -542,7 +543,7 @@ test pagedAttention { // cu_fa2 creates new buffers while other reuse triton buffers. defer if (backend == .cuda_fa2) zml.Buffer.deinitAll(Parameters, ¶meters_d); - var output_d = try zml.testing.autoCall(allocator, io, &exe, pagedAttention, .{ parameters_d, q, new_k, new_v, kv_cache_d }); + var output_d = try zml.testing.autoCall(allocator, io, &exe, pagedAttention, .{ parameters_d, q, new_k, new_v, kv_cache_d, .{} }); defer output_d.deinit(); results_per_backend.set(backend, try output_d.toSliceAlloc(allocator, io)); } diff --git a/zml/attention/triton_attention.zig b/zml/attention/triton_attention.zig index 27d5cd90..00876276 100644 --- a/zml/attention/triton_attention.zig +++ b/zml/attention/triton_attention.zig @@ -356,7 +356,7 @@ pub const paged = struct { .use_alibi_slopes = false, .use_qq_bias = false, .use_softcap = false, - .use_sinks = false, + .use_sinks = (opts.sink != null), .sliding_window = @intCast(paged_attention_opts.sliding_window), .block_q = @intCast(config.block_q), .block_m = @intCast(config.block_m), @@ -377,12 +377,14 @@ pub const paged = struct { const scale: f32 = paged_attention_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))); const num_seqs = parameters.block_table.dim(0); + const sink = if (opts.sink) |sink| sink.convert(.f32) else dummy; + const output = kernels.KernelUnifiedAttention2dPtr.Kernel.call( .{ .query_ptr = q, .key_cache_ptr = k_cache, .value_cache_ptr = v_cache, - .sink_ptr = dummy, + .sink_ptr = sink, .block_tables_ptr = parameters.block_table, .seq_lens_ptr = parameters.seq_lens, .alibi_slopes_ptr = dummy, @@ -434,7 +436,7 @@ pub const paged = struct { .use_alibi_slopes = false, .use_qq_bias = false, .use_softcap = false, - .use_sinks = false, + .use_sinks = (opts.sink != null), .sliding_window = @intCast(paged_attention_opts.sliding_window), .block_q = @intCast(config.attention.block_q), .block_m = @intCast(config.attention.block_m), @@ -472,12 +474,15 @@ pub const paged = struct { @intCast(paged_attention_opts.num_kv_heads), @intCast(config.attention.num_segments_per_seq), }; + + const sink = if (opts.sink) |sink| sink.convert(.f32) else dummy; + const attn_output = kernels.KernelUnifiedAttention3dPtr.Kernel.call( .{ .query_ptr = q, .key_cache_ptr = k_cache, .value_cache_ptr = v_cache, - .sink_ptr = dummy, + .sink_ptr = sink, .block_tables_ptr = parameters.block_table, .seq_lens_ptr = parameters.seq_lens, .alibi_slopes_ptr = dummy, @@ -545,8 +550,6 @@ pub const paged = struct { /// the SIMD16-tuned kernel (compile-time KV strides, cached loads, same-page fast /// path) while reusing the shared segment-reduce kernel. See unified_attention_oneapi.zig. pub fn pagedAttention3dOneapi(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, paged_attention_opts: PagedAttentionOptions) zml.Tensor { - _ = opts; - const config = select3dConfig(paged_attention_opts); const head_size_padded: i64 = @intCast(std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim)); @@ -566,7 +569,7 @@ pub const paged = struct { .use_alibi_slopes = false, .use_qq_bias = false, .use_softcap = false, - .use_sinks = false, + .use_sinks = (opts.sink != null), .sliding_window = @intCast(paged_attention_opts.sliding_window), .block_q = @intCast(config.attention.block_q), .block_m = @intCast(config.attention.block_m), @@ -606,12 +609,15 @@ pub const paged = struct { const num_seqs = parameters.block_table.dim(0); const attn_grid: [3]i32 = .{ @intCast(config.attention.total_q_blocks), @intCast(paged_attention_opts.num_kv_heads), @intCast(config.attention.num_segments_per_seq) }; + + const sink = if (opts.sink) |sink| sink.convert(.f32) else dummy; + const attn_output = kernels_oneapi.KernelUnifiedAttention3dPtr.Kernel.call( .{ .query_ptr = q, .key_cache_ptr = k_cache, .value_cache_ptr = v_cache, - .sink_ptr = dummy, + .sink_ptr = sink, .block_tables_ptr = parameters.block_table, .seq_lens_ptr = parameters.seq_lens, .alibi_slopes_ptr = dummy, From 35d5bf9282d1e12e075def08f17761fbe9b66375 Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Tue, 29 Sep 2026 14:37:42 +0200 Subject: [PATCH 732/881] zml/tokenizer: remove homemade tokenizer (#870) --- zml/tokenizer/BUILD.bazel | 5 +- zml/tokenizer/homemade.zig | 1243 ----------------------------------- zml/tokenizer/tokenizer.zig | 7 - 3 files changed, 2 insertions(+), 1253 deletions(-) delete mode 100644 zml/tokenizer/homemade.zig diff --git a/zml/tokenizer/BUILD.bazel b/zml/tokenizer/BUILD.bazel index da1c8bda..f2cbc132 100644 --- a/zml/tokenizer/BUILD.bazel +++ b/zml/tokenizer/BUILD.bazel @@ -3,10 +3,9 @@ load("@zml//bazel:zig_srcs.bzl", "zig_srcs") zig_library( name = "tokenizer", - tags = ["manual"], - srcs = ["homemade.zig"], import_name = "zml/tokenizer", main = "tokenizer.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ "//ffi:zig", @@ -29,7 +28,7 @@ zig_binary( zig_test( name = "test", deps = [ - ":tokenizer" + ":tokenizer", ], ) diff --git a/zml/tokenizer/homemade.zig b/zml/tokenizer/homemade.zig deleted file mode 100644 index e2b142f7..00000000 --- a/zml/tokenizer/homemade.zig +++ /dev/null @@ -1,1243 +0,0 @@ -//! Text tokenizer implementations -//! Disclaimer this is not a very robust implementation: -//! In particular the normalization is pretty minimalist, only works with ascii, and don't do unicode normalization. -//! Mostly used for testing models that don't have an official HF/sentencepiece tokenizer. -const std = @import("std"); -const testing = std.testing; -const builtin = @import("builtin"); - -const stdx = @import("stdx"); - -const log = std.log.scoped(.@"zml/tokenizer"); - -/// Byte Pair Encoding tokenizer generally used for LLM. -pub const Tokenizer = struct { - tokens: [][]const u8, - token_lookup: std.StringHashMapUnmanaged(u32), - special_tokens: SpecialTokens, - - scores: []f32, - max_token_len: u32, - normalizer: ?Normalizer, - // Allows to split unknown unicode characters into bytes. - byte_fallback: bool = false, - - arena_state: std.heap.ArenaAllocator, - vocab_size: u32, - next_token_id: u32 = 0, - - pub const SpecialTokens = struct { - eos: u32, - bos: u32, - unk: u32, - pad: u32 = std.math.maxInt(u32), - hard_space: u32 = std.math.maxInt(u32), - }; - - pub fn init( - allocator: std.mem.Allocator, - vocab_size: u32, - max_token_len: u32, - normalizer: ?Normalizer, - special_tokens: SpecialTokens, - alloc_tokens: bool, - ) !Tokenizer { - var arena_state = std.heap.ArenaAllocator.init(allocator); - errdefer arena_state.deinit(); - const arena = arena_state.allocator(); - - var token_lookup: std.StringHashMapUnmanaged(u32) = .empty; - errdefer token_lookup.deinit(arena); - - try token_lookup.ensureTotalCapacity(arena, @intCast(vocab_size)); - - const tokens: [][]const u8 = if (alloc_tokens) try arena.alloc([]const u8, vocab_size) else &.{}; - errdefer if (alloc_tokens) arena.free(tokens); - - const scores: []f32 = if (alloc_tokens) try arena.alloc(f32, vocab_size) else &.{}; - errdefer if (alloc_tokens) arena.free(scores); - - return .{ - .tokens = tokens, - .scores = scores, - .max_token_len = max_token_len, - .token_lookup = token_lookup, - .arena_state = arena_state, - .normalizer = normalizer, - .vocab_size = vocab_size, - .special_tokens = special_tokens, - }; - } - - pub fn deinit(self: Tokenizer) void { - self.arena_state.deinit(); - } - - pub fn encoder(self: *Tokenizer) !Encoder { - return Encoder.init(self); - } - - pub fn decoder(self: *Tokenizer) !Decoder { - return Decoder.init(self); - } - - /// Reads a new word directly into the tokenizer arena. - pub fn readTokenInto(self: *Tokenizer, score: f32, len: usize, tok_reader: *std.Io.Reader) !void { - const arena = self.arena_state.allocator(); - - const token = try arena.alloc(u8, len); - try tok_reader.readSliceAll(token); - return self.addOwnedToken(score, token); - } - - /// Adds a new token (and copy it) - pub fn addToken(self: *Tokenizer, score: f32, token: []const u8) !void { - const arena = self.arena_state.allocator(); - return self.addOwnedToken(score, try arena.dupe(u8, token)); - } - - /// Adds a new token (without copying it) - pub fn addOwnedToken(self: *Tokenizer, score: f32, token: []const u8) void { - const i = self.next_token_id; - std.debug.assert(i < self.vocab_size); - self.next_token_id += 1; - - self.scores[i] = score; - self.tokens[i] = token; - const v = self.token_lookup.getOrPutAssumeCapacity(token); - if (!v.found_existing) { - v.value_ptr.* = i; - } - } - - pub fn addOwnedTokenByIndex(self: *Tokenizer, i: u32, score: f32, token: []const u8) void { - std.debug.assert(i < self.vocab_size); - self.next_token_id += 1; - self.scores[i] = score; - self.tokens[i] = token; - const v = self.token_lookup.getOrPutAssumeCapacity(token); - if (!v.found_existing) { - v.value_ptr.* = @intCast(i); - } - } - - pub fn lookup(self: *const Tokenizer, str: []const u8) ?u32 { - return self.token_lookup.get(str); - } - - pub fn tokenId(self: *const Tokenizer, token: []const u8) ?u32 { - return self.token_lookup.get(token); - } - - pub const EncodeOptions = struct { - /// Should the beginning of sentence '' token be added. - add_bos: bool = true, - add_eos: bool = false, - pad_to: u32 = 0, - // Print tokenization intermediary steps. - debug: bool = false, - }; - - pub fn encode(self: *const Tokenizer, allocator: std.mem.Allocator, raw: []const u8, options: EncodeOptions) ![]u32 { - if (options.debug) log.debug("Tokenizer.encode('{s}')", .{raw}); - const input = if (self.normalizer) |n| try n.normalize(allocator, raw) else raw; - defer if (self.normalizer) |_| allocator.free(input); - if (options.debug) log.debug("Tokenizer.encode.normalize -> '{s}'", .{input}); - - // Allocate a buffer that can fit all indices as well as extra character if requested. - // We then slice it so that the token merging code doesn't see the bos token. - const tok_buff_alloc = try allocator.alloc(u32, @max(options.pad_to, input.len + 2)); - const tok_buff = if (options.add_bos) tok_buff_alloc[1..] else tok_buff_alloc; - - const MergeState = union(enum) { ready: u32, nope, hard_space, idk }; - const mergeable = try allocator.alloc(MergeState, tok_buff.len); - - var num_tokens: usize = 0; - var it: CharTokenIterator = .{ .input = input }; - while (try it.nextCodepointToken(self)) |token| : (num_tokens += 1) { - tok_buff[num_tokens] = token; - mergeable[num_tokens] = if (token == self.special_tokens.hard_space) - .hard_space - else - .idk; - } - - var stable_prefix: usize = 0; - var stable_off: usize = 0; - while (true) { - // This code is a bit overcomplicated cause I'm abstracting over two algorithms: - // BPE and sentencepiece unigram model. - // Normally BPE is pre-split on spaces then the regular merge algorithm is applied. - // With unigram model you work at sentence level and you handle spaces as you would any other bytes, - // hoping the final tokens mostly align with spaces. - // This seemed like a good idea, but is kinda bad because I had to add special code to speed up BPE - // by detecting when the first "word" is treated and can be safely removed from sequence. - // Also it doesn't work well with BPE vocab which have multi-space tokens (for indentation) - // and have custom splitting rules. - // This is fine for now cause we now have bindings to HF tokenizers for complexe use cases - // and are only using this for tinyllama/gguf models. - // If we come back to use this in production, the implementation would gain in speed/clarity - // by splitting in two. - // The merging token logic isn't that complicated anyway. - - // Step by step visualization of the progress. - if (options.debug) { - var _debug_buf: [256]u8 = undefined; - var debug_progress: std.Io.Writer = .fixed(&_debug_buf); - self.decodeWithOpts(tok_buff[0..num_tokens], &debug_progress, .{ .sep = "|" }) catch {}; - log.debug("tokens: {any} -> {s}", .{ tok_buff[0..num_tokens], debug_progress.buffered() }); - } - var best_score: f32 = -1e10; - var best_token: u32 = 0; - var best_idx: ?usize = null; - var input_off: usize = stable_off; - - // Find best tokens to merge in all available tokens - for (stable_prefix..num_tokens - 1) |i| { - if (tok_buff[i] == self.special_tokens.unk) { - input_off += 1; - continue; - } - const cur_tok = self.tokens[tok_buff[i]]; - defer input_off += cur_tok.len; - - // Lookup merge for current token, if not already done. - switch (mergeable[i]) { - .nope => continue, - .ready => {}, - .hard_space => { - // Since tokens are not allowed to merge through hard sep, - // we don't need to merge the sentence-wide best token. - // We can just merge the best token since beginning. - if (best_idx != null) break; - // OTOH if there was no merge possible since beginning, - // we can skip the beginning in future iterations. - stable_prefix = i + 1; - stable_off = input_off + cur_tok.len; - continue; - }, - .idk => { - - // Special tokens can't be concatenated. - if (builtin.mode == .Debug and tok_buff[i] != self.special_tokens.unk) { - // Detects memory corruption of tokens. - if (cur_tok.len == 0 or cur_tok.len > self.max_token_len) @panic("Token looks corrupted !"); - - if (!std.mem.eql(u8, cur_tok, input[input_off..][0..cur_tok.len])) { - log.err("current token '{s}' not found in input string '{s}' !", .{ cur_tok, input[input_off..] }); - @panic("invalid tokenization"); - } - } - const next_tok = self.tokens[tok_buff[i + 1]]; - // if `next_tok` is `.unk`, length is 1; otherwise, it's the length of the token. - const next_tok_len = if (tok_buff[i + 1] == self.special_tokens.unk) 1 else next_tok.len; - const concat_tokens = input[input_off..][0 .. cur_tok.len + next_tok_len]; - // Save the result - mergeable[i] = if (self.lookup(concat_tokens)) |tok| - .{ .ready = tok } - else - .nope; - }, - } - - switch (mergeable[i]) { - .idk, .hard_space => unreachable, - .nope => continue, - .ready => |tok| { - if (self.scores[tok] > best_score) { - best_score = self.scores[tok]; - best_token = tok; - best_idx = i; - } - }, - } - } - - if (best_idx) |bidx| { - // Apply the merge. - tok_buff[bidx] = best_token; - std.mem.copyForwards(u32, tok_buff[bidx + 1 ..], tok_buff[bidx + 2 .. num_tokens]); - std.mem.copyForwards(MergeState, mergeable[bidx + 1 ..], mergeable[bidx + 2 .. num_tokens]); - num_tokens -= 1; - // We got two new merge lookups to do. - mergeable[bidx] = .idk; - if (bidx > 0 and mergeable[bidx - 1] != .hard_space) mergeable[bidx - 1] = .idk; - } else { - // No merge candidate => we are done ! - break; - } - } - - if (options.add_eos) { - tok_buff[num_tokens] = self.special_tokens.eos; - num_tokens += 1; - } - if (options.add_bos) { - tok_buff_alloc[0] = self.special_tokens.bos; - num_tokens += 1; - } - if (num_tokens < options.pad_to) { - for (num_tokens..options.pad_to) |i| { - tok_buff_alloc[i] = self.special_tokens.pad; - } - num_tokens = options.pad_to; - } - - // Release extra memory we don't need anymore. - allocator.free(mergeable); - _ = allocator.resize(tok_buff_alloc, num_tokens); - return tok_buff_alloc[0..num_tokens]; - } - - /// Returns a slice corresponding to the given id. Handles unknown ids and special ids. - pub fn lookupPiece(self: *const Tokenizer, id: usize) []const u8 { - return if (id == self.special_tokens.bos or id == self.special_tokens.eos or id == self.special_tokens.pad) - "" - else if (id == self.special_tokens.unk) - "" - else if (id > self.tokens.len) - "" // this means we received an invalid id, but we didn't want to panic. - else - self.tokens[id]; - } - - /// Converts the given slice of tokens back into bytes. - /// Note that if the tokenizer allows sub-unicode bytes, it's possible - /// the output is not valid utf8. - pub fn decode(self: *const Tokenizer, input: []const u32, writer: *std.Io.Writer) std.Io.Writer.Error!void { - try self.decodeWithOpts(input, writer, .{}); - } - - pub fn decodeWithOpts( - self: *const Tokenizer, - input: []const u32, - output: *std.Io.Writer, - opts: struct { sep: []const u8 = "" }, - ) std.Io.Writer.Error!void { - const escaped = if (self.normalizer) |n| n.escapedSpace() else null; - // Flag used to indicate if the first dummy whitespace has been consumed. - var first_token: bool = true; - for (input) |id| { - // Retrieve the slice corresponding to the id. - var piece = self.lookupPiece(id); - - // Convert `▁` to a regular space. - if (escaped) |escspc| { - // we modify piece inside the loop, so we can use it in the condition - while (std.mem.startsWith(u8, piece, escspc)) { - piece = piece[escspc.len..]; - // don't output a space at beginning of text. - if (!first_token) try output.writeByte(' '); - } - } - - try output.writeAll(piece); - first_token = false; - try output.writeAll(opts.sep); - } - } - - /// Some tokenizers have bytes encoded in hex like this: "<0x40>". - /// This break the tokenization algorithm because the input text - /// will contain "@" not "<0x40>", - /// and if the input contains "<0x40>" it needs to not be treated as a single byte. - /// So we replace byte fallbacks strings, by their corresponding character. - /// This enables the normal tokenization algorithm to work. - pub fn rewriteByteFallbackTokens(tokenizer: *Tokenizer) !void { - tokenizer.byte_fallback = true; - var single_bytes = try tokenizer.arena_state.allocator().alloc(u8, 256); - var byte_fallback_buf = "<0x00>".*; - - for (0..256) |i| { - const c: u8 = @truncate(i); - single_bytes[i] = c; - - // First lookup the byte fallback entry. - // Note: we assume upper case, but we could try both upper and lower case if needed. - var writer: std.Io.Writer = .fixed(byte_fallback_buf[3..5]); - try writer.printInt(c, 16, .upper, .{ .fill = '0', .width = 2 }); - const entry = tokenizer.token_lookup.getEntry(&byte_fallback_buf) orelse { - log.err("Tokenizer has \"byte_fallback\" = true, but doesn't contains the byte fallback token {s}", .{byte_fallback_buf}); - return error.InvalidInput; - }; - - // Check if the character is already present in the vocab. - // In that case, nothing to do, - // but note that the fallback token will be "unreachable", - // ie there is no way the tokenizer can produce it. - if (tokenizer.token_lookup.get(&.{c})) |_| continue; - - const idx: u32 = entry.value_ptr.*; - tokenizer.token_lookup.removeByPtr(entry.key_ptr); - tokenizer.addOwnedTokenByIndex(idx, tokenizer.scores[idx], single_bytes[i .. i + 1]); - } - } -}; - -test Tokenizer { - const allocator = std.testing.allocator; - const special_tokens: Tokenizer.SpecialTokens = .{ - .unk = 0, - .bos = 1, - .eos = 2, - }; - - var tokenizer = try Tokenizer.init(allocator, 10, 5, null, special_tokens, true); - defer tokenizer.deinit(); - - try tokenizer.addToken(10, "hello"); - try tokenizer.addToken(3.5, "world"); - - try testing.expect(tokenizer.lookup("hello") == 0); - try testing.expect(tokenizer.lookup("world") == 1); - - // TODO: test Tokenizer.decode, Tokenizer.encode, Tokenizer.readTokenInto -} - -pub const Encoder = struct { - inner: *Tokenizer, - arena: std.heap.ArenaAllocator, - current_ids: []const u32 = &.{}, - - fn init(inner: *Tokenizer) !Encoder { - var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); - // Warmup the arena. Page allocator is expensive, avoid calling it for small reallocs. - _ = try arena.allocator().alloc(u32, 4096); - std.debug.assert(arena.reset(.retain_capacity)); - return .{ .inner = inner, .arena = arena }; - } - - pub fn reset(self: *Encoder) void { - self.current_ids = &.{}; - std.debug.assert(self.arena.reset(.retain_capacity)); - } - - pub fn deinit(self: *Encoder) void { - self.arena.deinit(); - } - - pub fn encode(self: *Encoder, input: []const u8) ![]const u32 { - self.reset(); - const res = try self.inner.encode(self.arena.allocator(), input, .{ - .add_bos = true, - .add_eos = false, - .pad_to = 0, - // Print tokenization intermediary steps. - .debug = false, - }); - self.current_ids = res; - return res; - } - - pub fn ids(self: *const Encoder) []const u32 { - return self.current_ids; - } -}; - -pub const Decoder = struct { - const StringBuffer = stdx.BoundedArray(u8, 128); - const TokensIdsBuffer = stdx.BoundedArray(u32, 4); - - inner: *Tokenizer, - arena: std.heap.ArenaAllocator, - - current_string: ?[]const u8 = null, - last_string: StringBuffer = .empty, - last_token_ids: TokensIdsBuffer = .empty, - - fn init(inner: *Tokenizer) !Decoder { - var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); - // Warmup the arena. Page allocator is expensive, avoid calling it for small reallocs. - _ = try arena.allocator().alloc(u32, 4096); - std.debug.assert(arena.reset(.retain_capacity)); - return .{ .inner = inner, .arena = arena }; - } - - pub fn deinit(self: *Decoder) void { - self.arena.deinit(); - } - - pub fn reset(self: *Decoder) void { - std.debug.assert(self.arena.reset(.retain_capacity)); - self.current_string = null; - } - - pub fn decode(self: *Decoder, ids: []const u32) ![]const u8 { - // TODO: revisite tokenizer api to use std.Io.Writer. - self.reset(); - // Reuse the same warmup than in init, to maximize contiguous writes. - var arena_writer = std.Io.Writer.Allocating.initCapacity(self.arena.allocator(), 4096) catch unreachable; - try self.inner.decode(ids, &arena_writer.writer); - self.current_string = arena_writer.written(); - return self.current_string.?; - } - - pub fn string(self: *const Decoder) []const u8 { - return self.current_string orelse ""; - } - - pub fn next(self: *Decoder, token_id: u32) !?[]const u8 { - if (self.last_token_ids.len >= self.last_token_ids.capacity()) { - _ = self.last_token_ids.orderedRemove(0); - } - self.last_token_ids.appendAssumeCapacity(token_id); - const new_string = try self.decode(self.last_token_ids.constSlice()); - if (self.last_string.len == 0) { - self.last_string = try StringBuffer.fromSlice(new_string); - return new_string; - } - var view = try std.unicode.Utf8View.init(self.last_string.constSlice()); - var it = view.iterator(); - while (it.nextCodepointSlice()) |cp| { - const start = it.i - cp.len; - if (std.mem.startsWith(u8, new_string, self.last_string.constSlice()[start..])) { - const chunk = new_string[self.last_string.len - start ..]; - self.last_string = try StringBuffer.fromSlice(new_string); - return chunk; - } - } - return null; - } -}; - -/// Given a slice, split it in the most simple tokens using the given tokenizer tokens. -/// The output of this can be used to initialize the tokenization algorithm. -/// Normally we split the input text into utf8 codepoint, -/// but if we find an unknown codepoint we either split it in bytes, or use the special "unknown" token, -/// depending on the tokenizer configuration. -const CharTokenIterator = struct { - state: union(enum) { by_codepoint, by_byte: u8 } = .by_codepoint, - input: []const u8, - - fn nextCodepointToken(self: *CharTokenIterator, tokenizer: *const Tokenizer) error{ TruncatedInput, Utf8InvalidStartByte }!?u32 { - if (self.input.len == 0) return null; - return switch (self.state) { - .by_byte => |*byte_left| { - const idx = tokenizer.lookup(self.input[0..1]) orelse { - // Normally this has been caught when calling `rewriteByteFallbackTokens`. - std.debug.panic("Tokenizer has \"byte_fallback\" = true, but doesn't contains the byte fallback for token '<0x{X:02}>'", .{self.input[0]}); - }; - - self.input = self.input[1..]; - byte_left.* -|= 1; - if (byte_left.* == 0) self.state = .by_codepoint; - return idx; - }, - .by_codepoint => { - // Try to lookup valid utf8 codepoint first. - const utf8_len = try std.unicode.utf8ByteSequenceLength(self.input[0]); - if (self.input.len < utf8_len) return error.TruncatedInput; - if (tokenizer.lookup(self.input[0..utf8_len])) |idx| { - self.input = self.input[utf8_len..]; - return idx; - } - - // Otherwise split in bytes if it's allowed. - if (tokenizer.byte_fallback) { - // TODO: replace this by a continue statement next time we bump Zig. - self.state = .{ .by_byte = utf8_len }; - return self.nextCodepointToken(tokenizer); - } - - // Or mark the full utf8 codepoint as unknown. - log.debug("Token not found for char '{s}'", .{self.input[0..utf8_len]}); - self.input = self.input[utf8_len..]; - return tokenizer.special_tokens.unk; - }, - }; - } -}; - -test CharTokenIterator { - const special_tokens: Tokenizer.SpecialTokens = .{ .unk = 0, .bos = 1, .eos = 2 }; - var tokenizer = try Tokenizer.init(std.testing.allocator, 16, 4, null, special_tokens, true); - defer tokenizer.deinit(); - - tokenizer.addOwnedToken(1.0, ""); // 0 - tokenizer.addOwnedToken(1.0, ""); // 1 - tokenizer.addOwnedToken(1.0, ""); // 2 - tokenizer.addOwnedToken(1.0, "ζ"); // 3 - tokenizer.addOwnedToken(1.0, &.{0xE2}); // 4: ℳ, first byte - tokenizer.addOwnedToken(1.0, &.{0x84}); // 5: ℳ, second byte - tokenizer.addOwnedToken(1.0, &.{0xB3}); // 6: ℳ, third byte - tokenizer.addOwnedToken(1.0, "L"); // 7 - - // No byte fallback - { - tokenizer.byte_fallback = false; - var it: CharTokenIterator = .{ .input = "ζℳL" }; - var res: stdx.BoundedArray(u32, 8) = .empty; - while (try it.nextCodepointToken(&tokenizer)) |token| { - res.appendAssumeCapacity(token); - } - try std.testing.expectEqualSlices(u32, &[_]u32{ 3, 0, 7 }, res.constSlice()); - } - - // with byte fallback - { - tokenizer.byte_fallback = true; - var it: CharTokenIterator = .{ .input = "ζℳL" }; - var res: stdx.BoundedArray(u32, 8) = .empty; - while (try it.nextCodepointToken(&tokenizer)) |token| { - res.appendAssumeCapacity(token); - } - try std.testing.expectEqualSlices(u32, &[_]u32{ 3, 4, 5, 6, 7 }, res.constSlice()); - } -} - -/// Text normalizer. -/// Most tokenizer assumes the input text have been prepocessed with on of those. -pub const Normalizer = struct { - /// Space token used by sentencepiece derived tokenizer. - pub const sentencepiece_space = "▁"; // \xe2\x96\x81 - - _whitespace: stdx.BoundedArray(u8, 8) = .empty, - - flags: packed struct { - remove_extra_whitespaces: bool, - add_dummy_prefix: bool, - add_dummy_suffix: bool, - /// Cheap lower casing. - /// TODO: try to match Python "lower" - lower_case_ascii: bool, - /// cheap ascii punct splitting. - // doing this processing ahead of time simplifies the logic - split_on_punct_ascii: bool, - }, - - pub fn init(flags: @FieldType(Normalizer, "flags"), escaped_whitespace: ?[]const u8) Normalizer { - var res: Normalizer = .{ .flags = flags }; - if (escaped_whitespace) |escaped| { - res._whitespace.appendSliceAssumeCapacity(escaped); - } - return res; - } - - pub inline fn escapedSpace(self: Normalizer) ?[]const u8 { - return if (self._whitespace.len > 1) self._whitespace.constSlice() else null; - } - - fn addSlice(allocator: std.mem.Allocator, data: []const u8, consumed: usize, normalized: *std.ArrayList(u8), normalized_to_origin: *std.ArrayList(usize)) !void { - try normalized.appendSlice(allocator, data); - for (data) |_| try normalized_to_origin.append(allocator, consumed); - } - - pub const Result = struct { - /// Normalized string - normalized: []const u8, - /// Mapping between chars in the original string and chars in the new string - normalized_to_origin: []const usize, - - pub fn deinit(self: Result, allocator: std.mem.Allocator) void { - allocator.free(self.normalized); - allocator.free(self.normalized_to_origin); - } - }; - - /// Simplifed version of Sentencepiece normalizer. - /// - /// Llama2 uses a normalizer called "identity" so this basically only handles trailing - /// whitespaces and replaces whitespace with the "▁" (U+2581) character. - pub fn normalize(self: Normalizer, allocator: std.mem.Allocator, input: []const u8) ![]const u8 { - const res = try self.normalizeWithMapping(allocator, input); - allocator.free(res.normalized_to_origin); - return res.normalized; - } - - /// Returns both the normalized string and a mapping between the normalized string and the original. - pub fn normalizeWithMapping(self: Normalizer, allocator: std.mem.Allocator, input: []const u8) !Result { - // Number of bytes consumed from the input. - var consumed: usize = 0; - var trimmed_input = input; - - // Skip leading whitespaces. - if (self.flags.remove_extra_whitespaces) { - while (trimmed_input.len != 0) { - if (trimmed_input[0] != ' ') break; - trimmed_input = trimmed_input[1..]; - consumed += 1; - } - } - - // If the trimmed input is empty, we are done. - if (trimmed_input.len == 0) { - return .{ .normalized = &.{}, .normalized_to_origin = &.{} }; - } - - // Pre-allocate outputs - const space = self.escapedSpace() orelse " "; - const overhead = if (self.flags.split_on_punct_ascii) space.len + 1 else space.len; - var normalized: std.ArrayList(u8) = try .initCapacity(allocator, trimmed_input.len * overhead + 2 * space.len); - errdefer normalized.deinit(allocator); - var normalized_to_origin: std.ArrayList(usize) = try .initCapacity(allocator, normalized.capacity); - errdefer normalized_to_origin.deinit(allocator); - - // If the spec asks for it, add a whitespace at the beginning. - if (self.flags.add_dummy_prefix) try addSlice(allocator, space, consumed, &normalized, &normalized_to_origin); - - var is_prev_space: bool = true; - var is_prev_word: bool = false; - - while (trimmed_input.len != 0) { - // NOTE(Corendos): This might feel weird but normally the slice we get comes from a normalizing process and can contain multiple codepoints. - // Since we have an "identity" normalizer, each slice is actually a unicode character. - const multibyte_length = try std.unicode.utf8ByteSequenceLength(trimmed_input[0]); - var slice = trimmed_input[0..multibyte_length]; - const origin = consumed; - consumed += multibyte_length; - trimmed_input = trimmed_input[multibyte_length..]; - - if (self.flags.remove_extra_whitespaces and is_prev_space) { - while (slice.len > 0 and slice[0] == ' ') { - slice = slice[1..]; - } - if (slice.len == 0) continue; - } - is_prev_space = slice[slice.len - 1] == ' '; - - if (slice.len == 1) ascii: { - // The more advanced logic only works with ascii atm - var byte = slice[0]; - if (self.escapedSpace() != null and byte == ' ') { - // replace the space token by the special token - try addSlice(allocator, space, origin, &normalized, &normalized_to_origin); - is_prev_word = false; - break :ascii; - } else if (self.flags.split_on_punct_ascii) { - if (is_prev_word and isPunct(slice)) { - // Insert a space, but continue handling the rest - try addSlice(allocator, space, origin, &normalized, &normalized_to_origin); - } - } - if (self.flags.lower_case_ascii) { - byte = std.ascii.toLower(byte); - } - try normalized.append(allocator, byte); - try normalized_to_origin.append(allocator, origin); - } else { - // we can safely copy to the output. - try addSlice(allocator, slice, origin, &normalized, &normalized_to_origin); - } - is_prev_word = !is_prev_space and !isPunct(slice); - } - - // Skip trailing whitespaces - if (self.flags.remove_extra_whitespaces) { - while (std.mem.endsWith(u8, normalized.items, space)) { - const length = normalized.items.len - space.len; - consumed = normalized_to_origin.items[length]; - try normalized.resize(allocator, length); - try normalized_to_origin.resize(allocator, length); - } - } - - try normalized_to_origin.append(allocator, consumed); - - std.debug.assert(normalized_to_origin.items.len == normalized.items.len + 1); - - if (self.flags.add_dummy_suffix) try addSlice(allocator, space, consumed, &normalized, &normalized_to_origin); - - return .{ - .normalized_to_origin = try normalized_to_origin.toOwnedSlice(allocator), - .normalized = try normalized.toOwnedSlice(allocator), - }; - } - - pub fn wellKnown(impl: KnownImplementation) Normalizer { - return switch (impl) { - .sentencepiece => init(.{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = true, - .add_dummy_suffix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - }, sentencepiece_space), - .llama3 => init(.{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = false, - .add_dummy_suffix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - }, null), - .gpt2 => init(.{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = true, - .add_dummy_suffix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - }, null), - }; - } - - pub fn fromHfJson(config: std.json.ObjectMap) error{InvalidNormalizerJson}!Normalizer { - var normalizer: Normalizer = .{ .flags = .{ - .remove_extra_whitespaces = false, - .add_dummy_suffix = false, - .add_dummy_prefix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - } }; - - // Normalizer config can be a single normalizer, or a sequence of normalizers. - const maybe_steps = objectGet(config, .array, "normalizers"); - const steps = if (maybe_steps) |st| st.items else &.{std.json.Value{ .object = config }}; - - for (steps) |step_val| { - if (step_val != .object) { - return error.InvalidNormalizerJson; - } - const step = step_val.object; - - const step_type = objectGet(step, .string, "type") orelse { - return error.InvalidNormalizerJson; - }; - if (std.mem.eql(u8, "Prepend", step_type)) { - normalizer.flags.add_dummy_prefix = true; - } else if (std.mem.eql(u8, "Append", step_type)) { - normalizer.flags.add_dummy_suffix = true; - } else if (std.mem.eql(u8, "Replace", step_type)) { - const pattern = objectGet(step, .object, "pattern") orelse return error.InvalidNormalizerJson; - const str_pattern = objectGet(pattern, .string, "String") orelse return error.InvalidNormalizerJson; - - if (std.mem.eql(u8, str_pattern, " ")) { - normalizer._whitespace.appendSliceAssumeCapacity( - objectGet(step, .string, "content") orelse return error.InvalidNormalizerJson, - ); - } else { - log.warn("Normalizer Replace pattern not supported: '{s}' -> '{s}'", .{ str_pattern, objectGet(pattern, .string, "content") orelse "" }); - } - } else { - log.warn("Unknown normalizer type: {s}", .{step_type}); - } - } - - return normalizer; - } - - test "Normalizer.fromHfJson" { - const config_json = - \\{ - \\ "type": "Sequence", - \\ "normalizers": [ - \\ { - \\ "type": "Prepend", - \\ "prepend": "▁" - \\ }, - \\ { - \\ "type": "Replace", - \\ "pattern": { - \\ "String": " " - \\ }, - \\ "content": "▁" - \\ } - \\ ] - \\} - ; - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const config = try std.json.parseFromSliceLeaky(std.json.Value, arena.allocator(), config_json, .{}); - const normalizer = try Normalizer.fromHfJson(config.object); - - const expected = Normalizer{ - ._whitespace = .{ .buffer = [_]u8{ 0xe2, 0x96, 0x81 } ++ @as([5]u8, @splat(0)), .len = 3 }, - .flags = .{ - .remove_extra_whitespaces = false, - .add_dummy_prefix = true, - .add_dummy_suffix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - }, - }; - try std.testing.expectEqual(expected.flags, normalizer.flags); - try std.testing.expectEqualStrings(expected.escapedSpace().?, normalizer.escapedSpace().?); - } -}; -pub const KnownImplementation = enum(u8) { - sentencepiece, - gpt2, - llama3, -}; - -fn isPunct(unicode_char: []const u8) bool { - // TODO use unicode categories - if (unicode_char.len > 1) return false; - - return switch (unicode_char[0]) { - ' ', '\t' => false, - 0...8 => true, - 10...31 => true, - '!'...'/' => true, - ':'...'@' => true, - '['...'`' => true, - '{'...'~' => true, - else => false, - }; -} - -test Normalizer { - { - const n: Normalizer = .{ .flags = .{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = true, - .add_dummy_suffix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - } }; - const res = try n.normalizeWithMapping(testing.allocator, "Hellŏ world!"); - defer res.deinit(testing.allocator); - - try testing.expectEqualSlices(u8, " Hellŏ world!", res.normalized); - try testing.expectEqualSlices( - usize, - // H e l l ŏ ␣ w o r l d ! - &.{ 0, 0, 1, 2, 3, 4, 4, 6, 8, 9, 10, 11, 12, 13, 14 }, - res.normalized_to_origin, - ); - } - - { - const n: Normalizer = .{ .flags = .{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = true, - .add_dummy_suffix = true, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - } }; - const res = try n.normalize(testing.allocator, "Hello world!"); - defer testing.allocator.free(res); - - try testing.expectEqualSlices(u8, " Hello world! ", res); - } - - { - const n = Normalizer.init( - .{ - .remove_extra_whitespaces = false, - .add_dummy_prefix = true, - .add_dummy_suffix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - }, - Normalizer.sentencepiece_space, - ); - const res = try n.normalize(testing.allocator, "Hello world!"); - defer testing.allocator.free(res); - - try testing.expectEqualSlices(u8, "▁Hello▁▁world!", res); - } - - { - const n: Normalizer = .{ .flags = .{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = false, - .add_dummy_suffix = true, - .lower_case_ascii = true, - .split_on_punct_ascii = false, - } }; - const res = try n.normalize(testing.allocator, "Hello world!"); - defer testing.allocator.free(res); - - try testing.expectEqualSlices(u8, "hello world! ", res); - } - - { - const n: Normalizer = .{ .flags = .{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = false, - .add_dummy_suffix = true, - .lower_case_ascii = false, - .split_on_punct_ascii = true, - } }; - const res = try n.normalize(testing.allocator, "Hello world!"); - defer testing.allocator.free(res); - - try testing.expectEqualSlices(u8, "Hello world ! ", res); - } -} - -/// gpt2 had their own way of storing text. -/// Unfortunately this has contaminated other models. -/// This implementation precompupte a mapping between bytes encoded with GPT2 algorithm, -/// into utf8 bytes, and do lookups at runtime. -pub const Gpt2TextDecoder = struct { - const Code = stdx.BoundedArray(u8, 2); - - // TODO: benchmark this is more efficient than doing the conversion at runtime. - code_to_byte: std.AutoArrayHashMapUnmanaged(Code, u8) = .empty, - allocator: std.mem.Allocator, - - pub fn init(allocator: std.mem.Allocator) !Gpt2TextDecoder { - var self = Gpt2TextDecoder{ - .allocator = allocator, - }; - try self.code_to_byte.ensureTotalCapacity(allocator, 256); - errdefer unreachable; - - var n: usize = 0; - for (0..256) |index| { - var code: Code = .{ .buffer = .{ 0, 0 }, .len = 0 }; // 0-init - const i: u8 = @intCast(index); - if (isPrintableByte(i)) { - if (std.ascii.isAscii(i)) { - code.appendAssumeCapacity(i); - } else { - const codepoint: u21 = @as(u21, @intCast(i)); - code.len = @intCast(std.unicode.utf8Encode(codepoint, &code.buffer) catch unreachable); - } - } else { - const codepoint: u21 = 256 + @as(u21, @intCast(n)); - code.len = @intCast(std.unicode.utf8Encode(codepoint, &code.buffer) catch unreachable); - n += 1; - } - - self.code_to_byte.putAssumeCapacityNoClobber(code, i); - } - return self; - } - - pub fn deinit(self: *Gpt2TextDecoder) void { - self.code_to_byte.deinit(self.allocator); - } - - /// Transform bytes representing text under the gpt2 encoding, - /// and write to the `unicode` buffer utf-8 bytes. - pub fn decode(self: Gpt2TextDecoder, bytes: []const u8, writer: *std.Io.Writer) (error{InvalidInput} || std.Io.Writer.Error)!void { - var it = std.unicode.Utf8Iterator{ .i = 0, .bytes = bytes }; - while (it.nextCodepointSlice()) |codepoint| { - const code: Code = switch (codepoint.len) { - 1 => .{ .buffer = .{ codepoint[0], 0 }, .len = 1 }, // 0-init - 2 => .{ .buffer = .{ codepoint[0], codepoint[1] }, .len = 2 }, - else => return error.InvalidInput, - }; - const byte = self.code_to_byte.get(code) orelse return error.InvalidInput; - try writer.writeByte(byte); - } - } - - inline fn isPrintableByte(c: u8) bool { - return ('!' <= c and c <= '~') or (0xa1 <= c and c <= 0xac) or (0xae <= c and c <= 0xff); - } -}; - -test Gpt2TextDecoder { - var decoder = try Gpt2TextDecoder.init(testing.allocator); - defer decoder.deinit(); - - var buf: [128]u8 = undefined; - - // Ascii is not changed. - { - var out: std.Io.Writer = .fixed(&buf); - try decoder.decode("getTitle", &out); - try testing.expectEqualStrings("getTitle", out.buffered()); - } - - { - var out: std.Io.Writer = .fixed(&buf); - try decoder.decode("Ċ", &out); - try testing.expectEqualStrings("\n", out.buffered()); - } - - // Leading space are represented with 'Ġ' - { - var out: std.Io.Writer = .fixed(&buf); - try decoder.decode("ĠUINavigationController", &out); - try testing.expectEqualStrings(" UINavigationController", out.buffered()); - } - // Russian is wild - { - var out: std.Io.Writer = .fixed(&buf); - try decoder.decode("ĠÑĢабоÑĤ", &out); - try testing.expectEqualStrings(" работ", out.buffered()); - } -} - -/// Open a json file in HF format and load the vocab from it. -pub fn fromHfJson(allocator: std.mem.Allocator, io: std.Io, tokenizer_path: []const u8) !Tokenizer { - const file = try std.fs.cwd().openFile(tokenizer_path, .{}); - defer file.close(); - - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - const file_content = b: { - var reader = file.reader(io, &.{}); - var writer: std.Io.Writer.Allocating = .init(allocator); - defer writer.deinit(); - _ = try reader.interface.streamRemaining(&writer.writer); - break :b try writer.toOwnedSlice(); - }; - defer allocator.free(file_content); - - const info = try std.json.parseFromSliceLeaky(std.json.Value, arena, file_content, .{ - .duplicate_field_behavior = .use_last, - }); - const main_object = switch (info) { - .object => |obj| if (obj.get("added_tokens") == null or obj.get("model") == null) { - return error.InvalidFormat; - } else obj, - else => return error.InvalidFormat, - }; - - const model = objectGet(main_object, .object, "model") orelse return error.InvalidFormat; - const vocab = objectGet(model, .object, "vocab") orelse return error.InvalidFormat; - const added_tokens = if (objectGet(main_object, .array, "added_tokens")) |added| added.items else &.{}; - const vocab_size: u32 = @intCast(vocab.count() + added_tokens.len); - - const normalizer = if (objectGet(main_object, .object, "normalizer")) |normalizer_config| - try Normalizer.fromHfJson(normalizer_config) - else - Normalizer.wellKnown(.llama3); - - // delay init of special tokens. - var tokenizer = try Tokenizer.init(allocator, vocab_size, 256, normalizer, undefined, true); - errdefer tokenizer.deinit(); - - // Buffer containing all concatenated tokens. - // Reserve a big chunk, to avoid grow event, but release over-allocated memory. - var all_tokens: std.Io.Writer.Allocating = try .initCapacity(tokenizer.arena_state.allocator(), file_content.len); - const original_alloc = all_tokens.writer.buffer.ptr; - // A re-alloc event here means we have invalidated all slices inside the tokenizer. - // If this is too annoying we could switch to a custom type instead of slices. - defer { - std.debug.assert(all_tokens.writer.buffer.ptr == original_alloc); - } - - // gpt2 based tokenizer got a special way of encoding unicode. - // we don't know in advance if this will be used by this tokenizer or not. - // so we assume it is the case, but if we find some unicode character, - // outside of the range used by gpt2 we know it was wrong, and start over. - var is_gpt2_vocab: bool = true; - var gpt2_decoder = try Gpt2TextDecoder.init(allocator); - defer gpt2_decoder.deinit(); - var it = vocab.iterator(); - while (it.next()) |kv| { - const n = all_tokens.writer.buffered().len; - gpt2_decoder.decode(kv.key_ptr.*, &all_tokens.writer) catch |err| { - switch (err) { - error.InvalidInput => { - is_gpt2_vocab = false; - break; - }, - error.WriteFailed => return error.OutOfMemory, - } - }; - const idx: u32 = @intCast(kv.value_ptr.*.integer); - const token = all_tokens.written()[n..]; - tokenizer.addOwnedTokenByIndex(idx, @floatFromInt(vocab_size - idx), token); - } - - if (!is_gpt2_vocab) { - // We where wrong, this is not a gpt2 vocab, start over, - // and reset the tokenizer state. - tokenizer.next_token_id = 0; - tokenizer.token_lookup.clearRetainingCapacity(); - all_tokens.clearRetainingCapacity(); - it = vocab.iterator(); - while (it.next()) |kv| { - const idx: u32 = @intCast(kv.value_ptr.*.integer); - const token = try dup(&all_tokens, kv.key_ptr.*); - tokenizer.addOwnedTokenByIndex(idx, @floatFromInt(vocab_size - idx), token); - } - } - - // More tokens, typically added during fine tuning of the model. - for (added_tokens) |token_obj| { - if (token_obj != .object) return error.InvalidFormat; - const v = objectGet(token_obj.object, .string, "content") orelse return error.InvalidFormat; - const id: u32 = @intCast(objectGet(token_obj.object, .integer, "id") orelse return error.InvalidFormat); - const n = all_tokens.written().len; - try if (is_gpt2_vocab) - gpt2_decoder.decode(v, &all_tokens.writer) - else - all_tokens.writer.writeAll(v); - const token = all_tokens.written()[n..]; - tokenizer.addOwnedTokenByIndex(id, 0, token); - } - // We won't add more tokens here, let release. - _ = try all_tokens.toOwnedSlice(); - - var unk = tokenizer.lookup(""); - if (objectGet(model, .integer, "unk_token")) |unk_tok| { - unk = @intCast(unk_tok); - } - - tokenizer.special_tokens = .{ - // TODO allow users to specify special tokens or read them from a tokenizer_config.json file - .bos = tokenizer.lookup("") orelse tokenizer.lookup("<|begin_of_text|>") orelse @panic("bos token not found !"), - .eos = tokenizer.lookup("") orelse tokenizer.lookup("<|end_of_text|>") orelse @panic("eos token not found !"), - .unk = unk orelse std.math.maxInt(u32), - }; - - const byte_fallback = objectGet(model, .bool, "byte_fallback") orelse false; - if (!byte_fallback and unk == null) { - // GPT2 tokenizer have byte fallback already encoded in the model, - // but the json generally don't have the field set. - // We can detect it though because they don't specify an unknown token. - if (is_gpt2_vocab) { - tokenizer.byte_fallback = true; - } else { - log.warn("The given tokenizer can't handle unknown token: no unknown token was set, and byte_fallback is disabled too ! The tokenizer will panic when facing unknown tokens.", .{}); - } - } else if (byte_fallback) { - try tokenizer.rewriteByteFallbackTokens(); - } - return tokenizer; -} - -/// Returns a copy of the given string, stored inside the given ArrayList. -fn dup(allocating: *std.Io.Writer.Allocating, str: []const u8) std.Io.Writer.Error![]const u8 { - const n = allocating.written().len; - try allocating.writer.writeAll(str); - return allocating.written()[n..]; -} - -/// Returns the given entry in a json object only if it has the right type. -fn objectGet( - object: std.json.ObjectMap, - comptime kind: std.meta.FieldEnum(std.json.Value), - key: []const u8, -) ?@FieldType(std.json.Value, @tagName(kind)) { - const val = object.get(key) orelse return null; - if (val != kind) return null; - return @field(val, @tagName(kind)); -} - -pub fn fromTinyLlamaFile(allocator: std.mem.Allocator, io: std.Io, tokenizer_path: []const u8, vocab_size: u32) !Tokenizer { - const tokenizer_file = try std.fs.cwd().openFile(tokenizer_path, .{}); - defer tokenizer_file.close(); - var read_buff: [4096]u8 = undefined; - var tok_reader = tokenizer_file.reader(io, &read_buff); - const r: *std.Io.Reader = &tok_reader.interface; - - const max_token_len = try readValueLE(u32, r); - const special_tokens: Tokenizer.SpecialTokens = .{ - .unk = 0, - .bos = 1, - .eos = 2, - }; - var tokenizer = try Tokenizer.init(allocator, vocab_size, max_token_len, null, special_tokens, true); - var i: u32 = 0; - while (readToken(&tokenizer, r)) : (i += 1) { - // Pass - } else |_| { - if (i < vocab_size) { - log.info("Read {d} words out of {d}", .{ i, vocab_size }); - } - tokenizer.vocab_size = i; - } - try tokenizer.rewriteByteFallbackTokens(); - return tokenizer; -} - -fn readToken(tokenizer: *Tokenizer, tok_reader: *std.Io.Reader) !void { - const score: f32 = try readValueLE(f32, tok_reader); - const len: usize = try readValueLE(u32, tok_reader); - try tokenizer.readTokenInto(score, len, tok_reader); -} - -fn readValueLE(T: type, reader: *std.Io.Reader) !T { - var res: [1]T = undefined; - try reader.readSliceEndian(T, &res, .little); - return res[0]; -} diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig index 781df814..fc304b81 100644 --- a/zml/tokenizer/tokenizer.zig +++ b/zml/tokenizer/tokenizer.zig @@ -3,14 +3,11 @@ const std = @import("std"); pub const iree = @import("iree"); pub const sentencepiece = @import("sentencepiece"); -pub const homemade = @import("homemade.zig"); - const log = std.log.scoped(.@"zml/tokenizer"); const Tokenizers = enum { iree, sentencepiece, - homemade, }; pub const Tokenizer = union(Tokenizers) { @@ -19,7 +16,6 @@ pub const Tokenizer = union(Tokenizers) { iree: iree.Normalizer, sentencepiece: void, - homemade: void, pub fn deinit(self: *Normalizer) void { switch (self.*) { @@ -39,7 +35,6 @@ pub const Tokenizer = union(Tokenizers) { pub const Encoder = union(Tokenizers) { iree: iree.Tokenizer.Encoder, sentencepiece: sentencepiece.Encoder, - homemade: homemade.Encoder, pub fn deinit(self: *Encoder) void { switch (self.*) { @@ -81,7 +76,6 @@ pub const Tokenizer = union(Tokenizers) { pub const Decoder = union(Tokenizers) { iree: iree.Tokenizer.Decoder, sentencepiece: sentencepiece.Decoder, - homemade: homemade.Decoder, pub fn deinit(self: *Decoder) void { switch (self.*) { @@ -126,7 +120,6 @@ pub const Tokenizer = union(Tokenizers) { iree: iree.Tokenizer, sentencepiece: *sentencepiece.SentencePieceProcessor, - homemade: *homemade.Tokenizer, pub fn fromFile(allocator: std.mem.Allocator, io: std.Io, model: []const u8) !Tokenizer { if (std.mem.endsWith(u8, model, ".pb")) { From f37e845672dfc8f1c05381d8b2421dfbb85ae595 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 29 Sep 2026 18:23:08 +0200 Subject: [PATCH 733/881] zml/safetensors: support hf snapshots (#872) This PR adds support for models downloaded via `hf download` without the `--local-dir` flag. This is done by keeping the given path instead of getting it from the `std.Io.File`. The problem with `File.realpath` is that in the case of models downloaded from HF, the files points to blobs whose name are hex strings, thus losing the required file extension. Unfortunately, we rely on the file extension to detect the kind of repository the path represents. I also added `//examples/io:playground` to the `//:completion` target so that we have completion when working on it. --- BUILD.bazel | 1 + examples/io/BUILD.bazel | 1 + examples/io/main.zig | 2 +- zml/safetensors.zig | 33 +++++++++++++++------------------ 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 19061490..cefaac75 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -39,6 +39,7 @@ zls_completion( "//examples/llm", "//examples/mnist", "//examples/sharding", + "//examples/io:playground", "//stdx", "//zml", "//zml/tokenizer/iree", diff --git a/examples/io/BUILD.bazel b/examples/io/BUILD.bazel index 64051f18..d471f74d 100644 --- a/examples/io/BUILD.bazel +++ b/examples/io/BUILD.bazel @@ -7,4 +7,5 @@ zig_binary( data = [], main = "main.zig", deps = ["//zml"], + visibility = ["//visibility:public"], ) diff --git a/examples/io/main.zig b/examples/io/main.zig index 5675010d..29215866 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -246,7 +246,7 @@ fn printTree(io: std.Io, writer: *std.Io.Writer, dir: std.Io.Dir, prefix: []cons const extension = if (is_last) " " else "│ "; const size: u64 = switch (entry.kind) { - .file => blk: { + .file, .sym_link => blk: { const stat = try dir.statFile(io, entry.name, .{}); break :blk stat.size; }, diff --git a/zml/safetensors.zig b/zml/safetensors.zig index 6a35a795..63791ea9 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -24,8 +24,9 @@ pub fn fetchRegistry( io: std.Io, repo: std.Io.Dir, model: std.Io.File, + model_path: []const u8, ) !TensorRegistry { - const file_type = try resolveFiletype(io, model); + const file_type = resolveFiletype(model_path); return switch (file_type) { .index => blk: { @@ -284,9 +285,9 @@ pub const TensorRegistry = struct { io: std.Io, repo: std.Io.Dir, ) !TensorRegistry { - const entrypoint = try resolveModelEntrypoint(io, repo); - defer entrypoint.close(io); - return try fetchRegistry(allocator, io, repo, entrypoint); + const entrypoint_file, const entrypoint_path = try resolveModelEntrypoint(io, repo); + defer entrypoint_file.close(io); + return try fetchRegistry(allocator, io, repo, entrypoint_file, entrypoint_path); } pub fn fromPath( @@ -297,14 +298,13 @@ pub const TensorRegistry = struct { var repo = try resolveModelRepo(io, path); defer repo.close(io); - const entrypoint = if (std.mem.endsWith(u8, path, ".safetensors.index.json") or - std.mem.endsWith(u8, path, ".safetensors")) - try repo.openFile(io, path, .{ .mode = .read_only }) + const entrypoint_file, const entrypoint_path = if (std.mem.endsWith(u8, path, ".safetensors.index.json") or std.mem.endsWith(u8, path, ".safetensors")) + .{ try repo.openFile(io, path, .{ .mode = .read_only }), path } else try resolveModelEntrypoint(io, repo); - defer entrypoint.close(io); + defer entrypoint_file.close(io); - return try fetchRegistry(allocator, io, repo, entrypoint); + return try fetchRegistry(allocator, io, repo, entrypoint_file, entrypoint_path); } pub fn deinit(self: *TensorRegistry) void { @@ -550,13 +550,10 @@ fn parseSafetensorsIndexFiles( } } -pub fn resolveFiletype(io: std.Io, file: std.Io.File) !FileType { - var path_buf: [1024]u8 = undefined; - const path_len = try file.realPath(io, &path_buf); - - if (std.mem.endsWith(u8, path_buf[0..path_len], ".safetensors.index.json")) { +pub fn resolveFiletype(path: []const u8) FileType { + if (std.mem.endsWith(u8, path, ".safetensors.index.json")) { return .index; - } else if (std.mem.endsWith(u8, path_buf[0..path_len], ".safetensors")) { + } else if (std.mem.endsWith(u8, path, ".safetensors")) { return .safetensors; } else { return .unknown; @@ -597,18 +594,18 @@ pub fn resolveModelRepo( return dir; } -pub fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!std.Io.File { +pub fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!struct { std.Io.File, []const u8 } { { const index_name = "model.safetensors.index.json"; if (repo.openFile(io, index_name, .{ .mode = .read_only })) |index_file| { - return index_file; + return .{ index_file, "model.safetensors.index.json" }; } else |_| {} } { const model_name = "model.safetensors"; if (repo.openFile(io, model_name, .{ .mode = .read_only })) |model_file| { - return model_file; + return .{ model_file, "model.safetensors" }; } else |_| {} } From a34428e05c50e2f84c87d7a7acf31c911dbd70de Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Wed, 30 Sep 2026 18:24:24 +0200 Subject: [PATCH 734/881] workspace: bump plugins (#876) --- platforms/cpu/cpu.bzl | 12 ++++++------ platforms/cuda/cuda.bzl | 8 ++++---- platforms/metal/metal.bzl | 4 ++-- platforms/rocm/rocm.bzl | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 06bde701..b3c63138 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -42,22 +42,22 @@ def _cpu_plugin_impl(mctx): http_archive( name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "16f0040b1805864fc0c3e13fe15f8ffccaf885a59b4f625666d2618583c0ac50", - url = "https://mirror.zml.ai/plugins/202609250917.103.1.3c8a2e14f8d9/zml-cpu-linux-amd64.tar.zst", + sha256 = "32432b7e4b9894a1de2e70e2f064093df0996679587adee2ebb1c930682d4251", + url = "https://mirror.zml.ai/plugins/202609301537.129.1.ac4bc67c46bf/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "5da35f0e471a410f3a3edba95f8a849ea017c1ed32c6fab0239a25a5d21acad4", - url = "https://mirror.zml.ai/plugins/202609250917.103.1.3c8a2e14f8d9/zml-cpu-darwin-amd64.tar.zst", + sha256 = "be60fe17befb5809f943346490f10c171c039c1893ca87ec624bc8649ea4740d", + url = "https://mirror.zml.ai/plugins/202609301537.129.1.ac4bc67c46bf/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "b0f6a8a1486780a9433b08153d71e0e31ed0e4cde60f200346c687c9346b2d53", - url = "https://mirror.zml.ai/plugins/202609250917.103.1.3c8a2e14f8d9/zml-cpu-darwin-arm64.tar.zst", + sha256 = "79e5212f139a8214e3ea88fa31c85d24401af21441a3324f06269e66ef15b7be", + url = "https://mirror.zml.ai/plugins/202609301537.129.1.ac4bc67c46bf/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 1390abab..449d2bc4 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -224,12 +224,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "2b10512d132339c16b38c0e9c76e4271b1b1f533e7988fb4ba019c5864df00db", - "url": "https://mirror.zml.ai/plugins/202609250917.103.1.3c8a2e14f8d9/zml-cuda-linux-amd64.tar.zst", + "sha256": "fd84f28d7697ff7c364f1a67c406f7436d123cf6ebb4039d968548d1f13ed70c", + "url": "https://mirror.zml.ai/plugins/202609301537.129.1.ac4bc67c46bf/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "0ed505366b24ea0fc4b4d1137a5c7e76c6dffb056414d93f7f355e2d7ac09475", - "url": "https://mirror.zml.ai/plugins/202609250917.103.1.3c8a2e14f8d9/zml-cuda-linux-arm64.tar.zst", + "sha256": "df853190c31465b67bd1eedb916b955d4395880fd5752b616bc8885799bd5122", + "url": "https://mirror.zml.ai/plugins/202609301537.129.1.ac4bc67c46bf/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index ecc454a7..aefcbe71 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "176b9d08cba22871a5c53e33da26235509612f6ce22247e3ee3d6ea8f5d206b0", - url = "https://mirror.zml.ai/plugins/202609250917.103.1.3c8a2e14f8d9/zml-metal-darwin-arm64.tar.zst", + sha256 = "d3aa3df41f14622624a9ed53b9574d5c8a7947543003d1b6c8625ec6ce683c53", + url = "https://mirror.zml.ai/plugins/202609301537.129.1.ac4bc67c46bf/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 6c272eb0..77cdf80b 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202609250917.103.1.3c8a2e14f8d9/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "5b4a3a37b6e55c221a4a8f6f859d979e8f0fab28b09c1e15d81a556da13c53b9" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202609301537.129.1.ac4bc67c46bf/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "483a34bdb6bb1844af0afaaca87b326d8c7f8bf85352c73ccdca5e06ea40a534" def _rocm_package_name(name): return name + _ROCM_VERSION From 4d6c61f5953afa6677610eb5672b79f6b7fe4e91 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 30 Sep 2026 19:24:34 +0200 Subject: [PATCH 735/881] platforms/oneapi: update pjrt plugin (#877) --- platforms/oneapi/libzml_oneapi.BUILD.bazel | 2 +- platforms/oneapi/oneapi.bzl | 13 ++++++------- zml/Compiler.zig | 7 +++---- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/platforms/oneapi/libzml_oneapi.BUILD.bazel b/platforms/oneapi/libzml_oneapi.BUILD.bazel index 67de8022..eb08e652 100644 --- a/platforms/oneapi/libzml_oneapi.BUILD.bazel +++ b/platforms/oneapi/libzml_oneapi.BUILD.bazel @@ -3,7 +3,7 @@ load("@zml//bazel:patchelf.bzl", "patchelf") patchelf( name = "libzml_oneapi_so", - src = "libpjrt_oneapi.so", + src = "lib/libzml_oneapi.so", add_needed = [ "libzmlxoneapi.so.0", ], diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index 0ab5b1a0..b6436a93 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -1,15 +1,14 @@ load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") -PJRT_ONEAPI_RELEASE = "manual-2026-09-08T13-23-00Z" -PJRT_ONEAPI_ARTIFACT_SHA256 = "c0c429f435a31207cb976636a9fa2d11b420ddc65bdde325e0ae736afb47a9b7" -PJRT_ONEAPI_ARTIFACT_URL = "https://github.com/zml/pjrt-artifacts/releases/download/{release}/pjrt-oneapi_linux-amd64.tar.gz".format( - release = PJRT_ONEAPI_RELEASE, -) +_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202609301700.135.1.f7a73483aa8a/zml-oneapi-linux-amd64.tar.zst" +_PLUGIN_ONEAPI_SHA256 = "2437fd3abc23913f1b3d38ea74c3ddd04a73961b56f89165ba4346cfd75eacde" ONEAPI_VERSION = "2026.1" + # www.intel.com/content/www/us/en/developer/tools/oneapi/onednn-download.html ONEAPI_DNNL_VERSION = "2026.0" + # https://github.com/uxlfoundation/onetbb ONEAPI_TBB_VERSION = "2023.1" ONEAPI_TCM_VERSION = "1.5" @@ -290,8 +289,8 @@ def _oneapi_impl(mctx): http_archive( name = "libzml_oneapi", build_file = "libzml_oneapi.BUILD.bazel", - sha256 = PJRT_ONEAPI_ARTIFACT_SHA256, - url = PJRT_ONEAPI_ARTIFACT_URL, + sha256 = _PLUGIN_ONEAPI_SHA256, + url = _PLUGIN_ONEAPI_URL, ) for pkg_name, build_file_content in _UBUNTU_PACKAGES.items(): diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 547ea6b3..57a357e4 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -881,12 +881,11 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: // More efficient for the allgather/broadcast implementation of the collective permute. try setXlaOverrideFlag(overrides_map, "xla_gpu_collective_permute_connected_components", true, upb_arena); try setXlaOverrideFlag(overrides_map, "xla_gpu_autotune_level", 0, upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "CUBLASLT,CUSTOM_CALL,FUSION", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_scheduling_mode", "CONCURRENT", upb_arena); - // Not supported by OneAPI - try setXlaOverrideFlag(overrides_map, "xla_disable_hlo_passes", "scan-rewriter", upb_arena); + // Grouped GEMM still needs a oneDNN lowering. try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_ragged_dot_grouped_gemm", false, upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_cub_radix_sort", false, upb_arena); }, else => {}, } From 676f26a4e6b1d9f54e46622ec3e02e7791d8d024 Mon Sep 17 00:00:00 2001 From: Olivier Boulant Date: Wed, 30 Sep 2026 22:28:09 +0200 Subject: [PATCH 736/881] platforms/cuda: fix libunwind downloads blocking Linux ARM64 build (#875) The error it fixes: ```bash bazel build //llmd:upload_sentry_symbols_linux_amd64 //llmd:upload_sentry_symbols_linux_arm64 --config=remote ... ERROR: no such package '@@zml++cuda_packages+llvm-libunwind1_linux_arm64//': java.io.IOException: Error downloading [https://apt.llvm.org/bookworm/pool/main/l/llvm-toolchain-23/llvm-libunwind1_23.1.2~++20260916013112+21a77e7bb6bf-1~exp1~20260916133123.74_arm64.deb] to /Users/oboulant/Library/Caches/bazel/_bazel_oboulant/50730f1e2ab11dae9d7f564452062ab5/external/zml++cuda_packages+llvm-libunwind1_linux_arm64/.downloaded.llvm-libunwind1_23.1.2~++20260916013112+21a77e7bb6bf-1~exp1~20260916133123.74_arm64.deb: GET returned 404 Not Found ... ``` The `platforms/cuda/packages.lock.json` was updated with `bazel run @apt_cuda//:lock`. --- platforms/cuda/packages.lock.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/platforms/cuda/packages.lock.json b/platforms/cuda/packages.lock.json index f3f7c4b2..65ab6ca1 100755 --- a/platforms/cuda/packages.lock.json +++ b/platforms/cuda/packages.lock.json @@ -19,13 +19,13 @@ "version": "12.2.0-14+deb12u1" } ], - "key": "llvm-libunwind1_1-23.1.2_-p--p-20260916013112-p-21a77e7bb6bf-1_exp1_20260916133123.74_amd64", + "key": "llvm-libunwind1_1-23.1.2_-p--p-20260920033443-p-85ac56026243-1_exp1_20260920033605.79_amd64", "name": "llvm-libunwind1", - "sha256": "7fe1e2f61f6db688b53a4015a2d391f4474e0c0aec76b4dc262fe91a43854e84", + "sha256": "aa690e56d52dc2798491167a06a879cb047b4f294cc953bd4c9f7267b0b0cecd", "urls": [ - "https://apt.llvm.org/bookworm/pool/main/l/llvm-toolchain-23/llvm-libunwind1_23.1.2~++20260916013112+21a77e7bb6bf-1~exp1~20260916133123.74_amd64.deb" + "https://apt.llvm.org/bookworm/pool/main/l/llvm-toolchain-23/llvm-libunwind1_23.1.2~++20260920033443+85ac56026243-1~exp1~20260920033605.79_amd64.deb" ], - "version": "1:23.1.2~++20260916013112+21a77e7bb6bf-1~exp1~20260916133123.74" + "version": "1:23.1.2~++20260920033443+85ac56026243-1~exp1~20260920033605.79" }, { "arch": "amd64", @@ -106,13 +106,13 @@ "version": "12.2.0-14+deb12u1" } ], - "key": "llvm-libunwind1_1-23.1.2_-p--p-20260916013112-p-21a77e7bb6bf-1_exp1_20260916133123.74_arm64", + "key": "llvm-libunwind1_1-23.1.2_-p--p-20260920033443-p-85ac56026243-1_exp1_20260920033605.79_arm64", "name": "llvm-libunwind1", - "sha256": "d5643512d5108eb0320486a740879223632fb8058ce321654d66604ba4a0439f", + "sha256": "a34362f952c5c815e715ab9b9b9c4776f6690683cb2037f04cf71322f1822e3b", "urls": [ - "https://apt.llvm.org/bookworm/pool/main/l/llvm-toolchain-23/llvm-libunwind1_23.1.2~++20260916013112+21a77e7bb6bf-1~exp1~20260916133123.74_arm64.deb" + "https://apt.llvm.org/bookworm/pool/main/l/llvm-toolchain-23/llvm-libunwind1_23.1.2~++20260920033443+85ac56026243-1~exp1~20260920033605.79_arm64.deb" ], - "version": "1:23.1.2~++20260916013112+21a77e7bb6bf-1~exp1~20260916133123.74" + "version": "1:23.1.2~++20260920033443+85ac56026243-1~exp1~20260920033605.79" }, { "arch": "arm64", From 8cbd3bd38c6a39543d96fd8f1dd71008da93e370 Mon Sep 17 00:00:00 2001 From: Olivier Boulant Date: Thu, 1 Oct 2026 18:41:45 +0200 Subject: [PATCH 737/881] platform/metal: support toolchain extraction on Linux x86_64 (#880) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Even if `exec_compatible_with = ["@platforms//os:macos"],` is removed from `platforms/metal/BUILD.bazel`, fetching the pinned Metal toolchain on Linux currently fails: ``` ERROR: /home/oboulant/.cache/bazel/_bazel_oboulant/4ea653214b23f6728bdb9f5b40de2f50/external/zml+/bazel/http_aea_archive.bzl:20:13: An error occurred during the fetch of repository 'zml++http_aea_archive+metal_toolchain': Traceback (most recent call last): File "/home/oboulant/.cache/bazel/_bazel_oboulant/4ea653214b23f6728bdb9f5b40de2f50/external/zml+/bazel/http_aea_archive.bzl", line 43, column 19, in _http_aea_archive_impl _require_macos(rctx) File "/home/oboulant/.cache/bazel/_bazel_oboulant/4ea653214b23f6728bdb9f5b40de2f50/external/zml+/bazel/http_aea_archive.bzl", line 20, column 13, in _require_macos fail("http_aea_archive is macOS-only; current repository OS is %r" % rctx.os.name) Error in fail: http_aea_archive is macOS-only; current repository OS is "linux" ``` Add checksum-pinned `ipsw` and `7-Zip` binaries for Linux x86_64. `bazel/http_aea_archive.bzl` now selects the right binaries according to running host. It preserves the existing macOS extraction path. Linux ARM64 remains uncovered. Validation: ```bash bazel build \ # selects compiler binaries from the extracted external repository \ # filegroup defined in platforms/metal/metal_toolchain.BUILD.bazel \ @metal_toolchain//:metal_compile_tools \ # selects the compiler’s runtime files \ # filegroup also defined in platforms/metal/metal_toolchain.BUILD.bazel \ @metal_toolchain//:metal_compile_runtime \ # copies compiler binaries and runtime files into the layout consumed by ZML \ //platforms/metal:metal_toolchain \ # selects macOS ARM64 as the build target \ --platforms=//platforms:macos_arm64 \ --lockfile_mode=off ``` --------- Co-authored-by: Corentin Kerisit --- MODULE.bazel | 38 +++++++++++- bazel/http_aea_archive.bzl | 112 ++++++++++++++++++++++++------------ platforms/metal/BUILD.bazel | 1 - 3 files changed, 111 insertions(+), 40 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 0c2f9217..71056c62 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -238,12 +238,48 @@ use_repo(neuron, "aws-neuronx-collectives", "aws-neuronx-runtime-lib", "aws-neur http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") http_archive( - name = "sevenzip_macos", + name = "sevenzip_darwin_amd64", build_file_content = """exports_files(["7zz"])""", sha256 = "26aa75bc262bb10bf0805617b95569c3035c2c590a99f7db55c7e9607b2685e0", urls = ["https://www.7-zip.org/a/7z2501-mac.tar.xz"], ) +# Host tools for the portable AEA extraction path. +http_archive( + name = "sevenzip_darwin_arm64", + build_file_content = """exports_files(["7zz"])""", + sha256 = "26aa75bc262bb10bf0805617b95569c3035c2c590a99f7db55c7e9607b2685e0", + urls = ["https://www.7-zip.org/a/7z2501-mac.tar.xz"], +) + +http_archive( + name = "sevenzip_linux_amd64", + build_file_content = """exports_files(["7zz"])""", + sha256 = "4ca3b7c6f2f67866b92622818b58233dc70367be2f36b498eb0bdeaaa44b53f4", + urls = ["https://www.7-zip.org/a/7z2501-linux-x64.tar.xz"], +) + +http_archive( + name = "ipsw_linux_amd64", + build_file_content = """exports_files(["ipsw"])""", + sha256 = "923c1caa44d3adaf2b792725c6209e879a13ab68af462ae5adfe1d611e2f3653", + urls = ["https://github.com/blacktop/ipsw/releases/download/v3.1.729/ipsw_3.1.729_linux_x86_64.tar.gz"], +) + +http_archive( + name = "sevenzip_linux_arm64", + build_file_content = """exports_files(["7zz"])""", + sha256 = "39c5140f02ce4436599303c59a149f654cb1bbc47cdc105a942120d747ae040d", + urls = ["https://www.7-zip.org/a/7z2501-linux-arm64.tar.xz"], +) + +http_archive( + name = "ipsw_linux_arm64", + build_file_content = """exports_files(["ipsw"])""", + sha256 = "fdef33729a0cb1d6be0cbffe677a97fb167e6af07fda3eafd03137ce607e92b4", + urls = ["https://github.com/blacktop/ipsw/releases/download/v3.1.729/ipsw_3.1.729_linux_arm64.tar.gz"], +) + http_aea_archive = use_repo_rule("//bazel:http_aea_archive.bzl", "http_aea_archive") http_aea_archive( diff --git a/bazel/http_aea_archive.bzl b/bazel/http_aea_archive.bzl index 867bc504..439a6a41 100644 --- a/bazel/http_aea_archive.bzl +++ b/bazel/http_aea_archive.bzl @@ -1,4 +1,6 @@ -"""Repository rule for macOS Apple Encrypted Archive.""" +"""Repository rule for downloading and extracting Apple Encrypted Archive assets on macOS and Linux AMD64/ARM64.""" + +load("@bazel_lib//lib:repo_utils.bzl", "repo_utils") def _fail_result(step, result): if result.return_code != 0: @@ -14,33 +16,63 @@ def _execute(rctx, step, argv, quiet = True): _fail_result(step, result) return result -def _require_macos(rctx): - os_name = rctx.os.name.lower() - if "mac" not in os_name and "darwin" not in os_name: - fail("http_aea_archive is macOS-only; current repository OS is %r" % rctx.os.name) - -def _single_restore_dmg(rctx): - restore_dir = rctx.path("AssetData/Restore") - if not restore_dir.exists: - fail("aa patch did not produce AssetData/Restore") +def _host_tools(rctx): + platform = repo_utils.platform(rctx) + if platform not in ["darwin_amd64", "darwin_arm64", "linux_amd64", "linux_arm64"]: + fail("http_aea_archive supports macOS and Linux AMD64/ARM64 repository hosts; got %s/%s" % (rctx.os.name, rctx.os.arch)) + ipsw = Label("@ipsw_{}//:ipsw".format(platform)) if platform.startswith("linux_") else None + sevenzip = Label("@sevenzip_{}//:7zz".format(platform)) + return (ipsw, sevenzip) +def _single_restore_dmg(restore_dirs): dmgs = [] entries = [] - for entry in restore_dir.readdir(): - entries.append(entry.basename) - if not entry.is_dir and entry.basename.endswith(".dmg"): - dmgs.append(entry) + for restore_dir in restore_dirs: + if not restore_dir.exists: + continue + for entry in restore_dir.readdir(): + entries.append(str(entry)) + if not entry.is_dir and entry.basename.endswith(".dmg"): + dmgs.append(entry) if len(dmgs) != 1: - fail("expected exactly one restore DMG in AssetData/Restore, got %d; entries: %s" % ( + fail("expected exactly one DMG under AssetData/Restore, got %d; entries: %s" % ( len(dmgs), ", ".join(sorted(entries)), )) return dmgs[0] +def _linux_restore_dirs(rctx, ipsw, aar): + # Create an explicit empty config. It avoids loading the user's ipsw configuration. + rctx.file("ipsw-config.yaml", "{}\n") + _execute(rctx, "ipsw ota extract", [ + rctx.path(ipsw), + "--config", + rctx.path("ipsw-config.yaml"), + "ota", + "extract", + aar, + "--key-val", + rctx.attr.archive_decryption_key, + "--pattern", + "AssetData/Restore/.*\\.dmg$", # pattern to match DMG files under AssetData/Restore + "--output", + rctx.path("ipsw-output"), + "--confirm", + "--no-color", + ]) + output = rctx.path("ipsw-output") + restore_dirs = [] + if output.exists: + # ipsw prefixes extracted paths with an asset-specific directory. + for entry in output.readdir(): + if entry.is_dir: + restore_dirs.append(entry.get_child("AssetData", "Restore")) + return restore_dirs + def _http_aea_archive_impl(rctx): - _require_macos(rctx) + ipsw, sevenzip = _host_tools(rctx) urls = [] if rctx.attr.url: @@ -54,7 +86,7 @@ def _http_aea_archive_impl(rctx): if not rctx.attr.archive_decryption_key: fail("archive_decryption_key is required") - aar = rctx.path("asset.aar") + aar = rctx.path("asset.aar") # path to the downloaded Apple Encrypted Archive (AAR) file download_kwargs = { "canonical_id": rctx.attr.canonical_id, "integrity": rctx.attr.integrity, @@ -71,31 +103,39 @@ def _http_aea_archive_impl(rctx): rctx.download(**download_kwargs) - _execute(rctx, "aa patch", [ - "/usr/bin/aa", - "patch", - "-i", - aar, - "-key-value", - "base64:%s" % rctx.attr.archive_decryption_key, - "-src", - "/var/empty", - "-dst", - rctx.path("."), - ]) - rctx.delete("asset.aar") - - dmg = _single_restore_dmg(rctx) + if ipsw: + restore_dirs = _linux_restore_dirs(rctx, ipsw, aar) + else: + _execute(rctx, "aa patch", [ + "/usr/bin/aa", + "patch", + "-i", + aar, + "-key-value", + "base64:%s" % rctx.attr.archive_decryption_key, + "-src", + "/var/empty", + "-dst", + rctx.path("."), + ]) + restore_dirs = [rctx.path("AssetData/Restore")] + dmg = _single_restore_dmg(restore_dirs) extract_args = [ - rctx.path(rctx.attr._sevenzip), + rctx.path(sevenzip), "x", "-y", dmg, ] extract_args.extend(rctx.attr.includes) _execute(rctx, "7z extract dmg", extract_args) + + # Central cleanup of temporary files and directories after extraction. + # If absent, rctx.delete will handle it gracefully. + rctx.delete("asset.aar") rctx.delete("AssetData") + rctx.delete("ipsw-output") + rctx.delete("ipsw-config.yaml") rctx.template("BUILD.bazel", rctx.attr.build_file) @@ -119,10 +159,6 @@ http_aea_archive = repository_rule( default = [], doc = "Patterns to pass to 7z for selective extraction from the DMG. If empty, extracts everything.", ), - "_sevenzip": attr.label( - allow_single_file = True, - default = Label("@sevenzip_macos//:7zz"), - ), }, - doc = "Downloads and extracts a macOS AEA-wrapped AppleArchive asset", + doc = "Downloads and extracts an AEA-wrapped AppleArchive asset on macOS and Linux AMD64/ARM64.", ) diff --git a/platforms/metal/BUILD.bazel b/platforms/metal/BUILD.bazel index c6f1f3c8..48bca771 100644 --- a/platforms/metal/BUILD.bazel +++ b/platforms/metal/BUILD.bazel @@ -9,7 +9,6 @@ copy_to_directory( "@metal_toolchain//:metal_compile_tools", ], out = "metal_toolchain", - exec_compatible_with = ["@platforms//os:macos"], include_external_repositories = ["**"], replace_prefixes = { "Metal.xctoolchain/usr/metal/32023": ".", From e2a882077d3d53fb2027bfcd438390f4e4583126 Mon Sep 17 00:00:00 2001 From: Louis Lechevalier Date: Fri, 2 Oct 2026 10:13:50 +0200 Subject: [PATCH 738/881] zml/attention: CuTe flashmla kernel for sm100, sparse MLA related backend (#882) - Implement the flashmla kernel for sm100 using the cute dsl in zig. - Extend the cute dsl with specialized sm100 (tcgen05) ops. - Introduces a new cute backend which calls the cute kernel. - Uses a new cache for MLA (latent or quantized). --- kernels/cute/builder.zig | 473 ++++-- mlir/dialects/BUILD.bazel | 2 + mlir/dialects/dialects.zig | 2 + mlir/dialects/llvm_dialect.zig | 177 +++ mlir/dialects/nvvm.zig | 419 +++++ zml/BUILD.bazel | 2 + zml/attention.zig | 1 + zml/attention/cute_kernels/flashmla_sm100.zig | 1355 +++++++++++++++++ zml/attention/cute_kernels/sparse_mla.zig | 266 ++++ zml/attention/paged_attention.zig | 142 +- zml/attention/sparse_mla.zig | 79 +- 11 files changed, 2695 insertions(+), 223 deletions(-) create mode 100644 mlir/dialects/llvm_dialect.zig create mode 100644 mlir/dialects/nvvm.zig create mode 100644 zml/attention/cute_kernels/flashmla_sm100.zig create mode 100644 zml/attention/cute_kernels/sparse_mla.zig diff --git a/kernels/cute/builder.zig b/kernels/cute/builder.zig index 69900224..ce28b3f4 100644 --- a/kernels/cute/builder.zig +++ b/kernels/cute/builder.zig @@ -11,7 +11,9 @@ const arith = dialects.arith; const cuda = dialects.cuda; const func = dialects.func; const gpu = dialects.gpu; +const llvm = dialects.llvm; const math = dialects.math; +const nvvm = dialects.nvvm; const vector = dialects.vector; const dtypes = @import("dtype.zig"); @@ -170,6 +172,14 @@ pub const Value = struct { return k.emit(if (l.isFloat()) arith.maximumf(k.ctx, l.inner, r.inner, k.loc()) else arith.maxsi(k.ctx, l.inner, r.inner, k.loc())); } + /// Float max that ignores NaN operands (`arith.maxnumf`, a single `max.f32`), unlike + /// the NaN-propagating `maximum`. + pub fn maxnum(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(arith.maxnumf(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn bitAnd(self: Value, rhs: anytype) Value { const k = self.kern(); const l, const r = k.coerce(self, rhs); @@ -1040,6 +1050,12 @@ pub const Builder = struct { return self.emit(cute.inttoptr(self.ctx, self.lift(address).inner, ptr_type, self.loc())); } + /// `cute.ptrtoint`: the integer address of a pointer (`i32` for shared + /// memory, `i64` for global memory), for hand-written PTX operands. + pub fn ptrToInt(self: *Builder, pointer: anytype, dt: DType) Value { + return self.emit(cute.ptrtoint(self.ctx, self.asValue(pointer).inner, dt.toMlir(self.ctx), self.loc())); + } + /// Reinterpret a pointer without changing its address. This is the typed /// counterpart of Python CuTe's `recast_ptr`; packed MXFP4 buffers use it /// to expose their byte storage to a scalar correctness implementation. @@ -2211,19 +2227,12 @@ pub const Builder = struct { /// Commit the preceding tcgen05 MMA operations to an mbarrier. One elected /// lane of the MMA warp issues this after each K-stage. pub fn tcgen05Commit(self: *Builder, barrier: Value) void { - self.emitVoid(mlir.Operation.make(self.ctx, "nvvm.tcgen05.commit", .{ - .operands = .{ .flat = &.{self.sharedBarrierPtr(barrier).inner} }, - .attributes = &.{.named(self.ctx, "group", self.parseAttribute("#nvvm.cta_group"))}, - .location = self.loc(), - })); + self.emitVoid(nvvm.tcgen05_commit(self.ctx, self.sharedBarrierPtr(barrier).inner, .cta_1, self.loc())); } /// Wait until asynchronous tensor-memory loads have reached registers. pub fn fenceTmemLoad(self: *Builder) void { - self.emitVoid(mlir.Operation.make(self.ctx, "nvvm.tcgen05.wait", .{ - .attributes = &.{.named(self.ctx, "kind", self.parseAttribute("#nvvm.tcgen05_wait"))}, - .location = self.loc(), - })); + self.emitVoid(nvvm.tcgen05_wait(self.ctx, .load, self.loc())); } pub const TmemOptions = struct { two_cta: bool = false, exclusive: bool = false }; @@ -2385,10 +2394,7 @@ pub const Builder = struct { fn sreg(self: *Builder, comptime reg: []const u8) Index3 { var out: [3]Value = undefined; inline for (.{ "x", "y", "z" }, 0..) |axis, i| { - out[i] = self.emit(mlir.Operation.make(self.ctx, "nvvm.read.ptx.sreg." ++ reg ++ "." ++ axis, .{ - .results = .{ .flat = &.{.int(self.ctx, .i32)} }, - .location = self.loc(), - })); + out[i] = self.emit(nvvm.read_sreg(self.ctx, reg ++ "." ++ axis, self.loc())); } return .{ .x = out[0], .y = out[1], .z = out[2] }; } @@ -2430,10 +2436,7 @@ pub const Builder = struct { /// `cute.arch.sync_threads()`. pub fn syncThreads(self: *Builder) void { - _ = mlir.Operation.make(self.ctx, "nvvm.barrier", .{ - .attributes = &.{.named(self.ctx, "operandSegmentSizes", .denseArray(self.ctx, .i32, &.{ 0, 0, 0 }))}, - .location = self.loc(), - }).appendTo(self.currentBlock()); + _ = nvvm.barrier(self.ctx, self.loc()).appendTo(self.currentBlock()); } /// Synchronize a fixed subset of the CTA on a named hardware barrier. @@ -2451,14 +2454,7 @@ pub const Builder = struct { "bar.sync {d}, {d};", .{ id, participants }, ) catch @panic("namedBarrier OOM"); - _ = mlir.Operation.make(self.ctx, "llvm.inline_asm", .{ - .attributes = &.{ - .named(self.ctx, "asm_string", .string(self.ctx, assembly)), - .named(self.ctx, "constraints", .string(self.ctx, "")), - .named(self.ctx, "has_side_effects", .unit(self.ctx)), - }, - .location = self.loc(), - }).appendTo(self.currentBlock()); + _ = llvm.inline_asm(self.ctx, assembly, &.{}, null, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); } /// Build the CuTe first-class divisor used by CUTLASS's persistent tile @@ -2490,10 +2486,7 @@ pub const Builder = struct { /// Elect one lane from the active warp. Used by TMA issue, tensor-memory /// allocation, and tcgen05 commit operations. pub fn electSync(self: *Builder) Value { - return self.emit(mlir.Operation.make(self.ctx, "nvvm.elect.sync", .{ - .results = .{ .flat = &.{.int(self.ctx, .i1)} }, - .location = self.loc(), - })); + return self.emit(nvvm.elect_sync(self.ctx, self.loc())); } /// `cute.arch.shuffle_sync_bfly`: exchange a scalar with the lane at @@ -2505,15 +2498,7 @@ pub const Builder = struct { const dtype = value.dtype(); if (dtypeBitwidth(dtype) != 32) @panic("warpMaxNonNegative requires a 32-bit scalar"); const bits = if (dtype == .i32) value else value.bitCast(.i32); - const reduced = self.emit(mlir.Operation.make(self.ctx, "llvm.inline_asm", .{ - .operands = .{ .flat = &.{bits.inner} }, - .results = .{ .flat = &.{.int(self.ctx, .i32)} }, - .attributes = &.{ - .named(self.ctx, "asm_string", .string(self.ctx, "redux.sync.max.u32 $0, $1, 0xffffffff;")), - .named(self.ctx, "constraints", .string(self.ctx, "=r,r")), - }, - .location = self.loc(), - })); + const reduced = self.emit(llvm.inline_asm(self.ctx, "redux.sync.max.u32 $0, $1, 0xffffffff;", &.{bits.inner}, .int(self.ctx, .i32), .{ .constraints = "=r,r" }, self.loc())); return if (dtype == .i32) reduced else reduced.bitCast(dtype); } @@ -2521,18 +2506,7 @@ pub const Builder = struct { const dtype = value.dtype(); if (dtypeBitwidth(dtype) != 32) @panic("shuffleXor currently requires a 32-bit scalar"); const bits = if (dtype == .i32) value else value.bitCast(.i32); - const shuffled = self.emit(mlir.Operation.make(self.ctx, "llvm.inline_asm", .{ - .operands = .{ .flat = &.{ - bits.inner, - self.cst(.i32, offset).inner, - } }, - .results = .{ .flat = &.{.int(self.ctx, .i32)} }, - .attributes = &.{ - .named(self.ctx, "asm_string", .string(self.ctx, "shfl.sync.bfly.b32 $0, $1, $2, 31, 0xffffffff;")), - .named(self.ctx, "constraints", .string(self.ctx, "=r,r,r")), - }, - .location = self.loc(), - })); + const shuffled = self.emit(llvm.inline_asm(self.ctx, "shfl.sync.bfly.b32 $0, $1, $2, 31, 0xffffffff;", &.{ bits.inner, self.cst(.i32, offset).inner }, .int(self.ctx, .i32), .{ .constraints = "=r,r,r" }, self.loc())); return if (dtype == .i32) shuffled else shuffled.bitCast(dtype); } @@ -2542,73 +2516,39 @@ pub const Builder = struct { const dtype = value.dtype(); if (dtypeBitwidth(dtype) != 32) @panic("shuffleIdx currently requires a 32-bit scalar"); const bits = if (dtype == .i32) value else value.bitCast(.i32); - const shuffled = self.emit(mlir.Operation.make(self.ctx, "llvm.inline_asm", .{ - .operands = .{ .flat = &.{ - bits.inner, - self.liftAs(lane, .i32).inner, - } }, - .results = .{ .flat = &.{.int(self.ctx, .i32)} }, - .attributes = &.{ - .named(self.ctx, "asm_string", .string(self.ctx, "shfl.sync.idx.b32 $0, $1, $2, 31, 0xffffffff;")), - .named(self.ctx, "constraints", .string(self.ctx, "=r,r,r")), - }, - .location = self.loc(), - })); + const shuffled = self.emit(llvm.inline_asm(self.ctx, "shfl.sync.idx.b32 $0, $1, $2, 31, 0xffffffff;", &.{ bits.inner, self.liftAs(lane, .i32).inner }, .int(self.ctx, .i32), .{ .constraints = "=r,r,r" }, self.loc())); return if (dtype == .i32) shuffled else shuffled.bitCast(dtype); } /// Four FP32 values rounded to FP8 E4M3 (saturating) and packed into one /// 32-bit word, `a` in the lowest byte. pub fn packFp8x4(self: *Builder, a: Value, b: Value, c: Value, d: Value) Value { - return self.emit(mlir.Operation.make(self.ctx, "llvm.inline_asm", .{ - .operands = .{ .flat = &.{ a.to(.f32).inner, b.to(.f32).inner, c.to(.f32).inner, d.to(.f32).inner } }, - .results = .{ .flat = &.{.int(self.ctx, .i32)} }, - .attributes = &.{ - .named(self.ctx, "asm_string", .string(self.ctx, "{ .reg .b16 lo, hi; cvt.rn.satfinite.e4m3x2.f32 lo, $2, $1; cvt.rn.satfinite.e4m3x2.f32 hi, $4, $3; mov.b32 $0, {lo, hi}; }")), - .named(self.ctx, "constraints", .string(self.ctx, "=r,f,f,f,f")), - }, - .location = self.loc(), - })); + return self.emit(llvm.inline_asm( + self.ctx, + "{ .reg .b16 lo, hi; cvt.rn.satfinite.e4m3x2.f32 lo, $2, $1; cvt.rn.satfinite.e4m3x2.f32 hi, $4, $3; mov.b32 $0, {lo, hi}; }", + &.{ a.to(.f32).inner, b.to(.f32).inner, c.to(.f32).inner, d.to(.f32).inner }, + .int(self.ctx, .i32), + .{ .constraints = "=r,f,f,f,f" }, + self.loc(), + )); } /// One-operand FP32 PTX instruction, e.g. `ex2.approx.ftz.f32`. pub fn unaryF32(self: *Builder, comptime instruction: []const u8, value: Value) Value { - return self.emit(mlir.Operation.make(self.ctx, "llvm.inline_asm", .{ - .operands = .{ .flat = &.{value.to(.f32).inner} }, - .results = .{ .flat = &.{DType.f32.toMlir(self.ctx)} }, - .attributes = &.{ - .named(self.ctx, "asm_string", .string(self.ctx, instruction ++ " $0, $1;")), - .named(self.ctx, "constraints", .string(self.ctx, "=f,f")), - }, - .location = self.loc(), - })); + return self.emit(llvm.inline_asm(self.ctx, instruction ++ " $0, $1;", &.{value.to(.f32).inner}, DType.f32.toMlir(self.ctx), .{ .constraints = "=f,f" }, self.loc())); } /// Signal programmatic dependent launches after this grid has consumed /// its inputs. The caller must also launch the producer/consumer chain /// with PDL enabled. pub fn launchDependents(self: *Builder) void { - _ = mlir.Operation.make(self.ctx, "llvm.inline_asm", .{ - .attributes = &.{ - .named(self.ctx, "asm_string", .string(self.ctx, "griddepcontrol.launch_dependents;")), - .named(self.ctx, "constraints", .string(self.ctx, "")), - .named(self.ctx, "has_side_effects", .unit(self.ctx)), - }, - .location = self.loc(), - }).appendTo(self.currentBlock()); + _ = llvm.inline_asm(self.ctx, "griddepcontrol.launch_dependents;", &.{}, null, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); } /// Wait until the preceding grid has made its programmatic dependency /// available. This is the consumer-side half of a PDL launch chain. pub fn waitForDependency(self: *Builder) void { - _ = mlir.Operation.make(self.ctx, "llvm.inline_asm", .{ - .attributes = &.{ - .named(self.ctx, "asm_string", .string(self.ctx, "griddepcontrol.wait;")), - .named(self.ctx, "constraints", .string(self.ctx, "")), - .named(self.ctx, "has_side_effects", .unit(self.ctx)), - }, - .location = self.loc(), - }).appendTo(self.currentBlock()); + _ = llvm.inline_asm(self.ctx, "griddepcontrol.wait;", &.{}, null, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); } /// `cute.arch.alloc_smem(dtype, size_in_elems, alignment)`: a static @@ -2621,71 +2561,41 @@ pub const Builder = struct { // mbarriers, async-proxy fences and bulk-copy groups. - /// mbarrier operands are `!llvm.ptr<3>` for NVVM. + /// mbarrier operands are `!llvm.ptr<3>` for NVVM. A barrier is either a CuTe + /// shared-memory pointer or a raw `i32` shared-memory address. fn sharedBarrierPtr(self: *Builder, barrier: Value) Value { + if (barrier.isInt()) return self.addressPtr(barrier, .shared); return self.emit(mlir.Operation.make(self.ctx, "builtin.unrealized_conversion_cast", .{ .operands = .{ .flat = &.{barrier.inner} }, - .results = .{ .flat = &.{self.parseType("!llvm.ptr<3>")} }, + .results = .{ .flat = &.{llvm.pointerType(self.ctx, .shared)} }, .location = self.loc(), })); } /// Initialize a shared-memory mbarrier expecting `arrivals` per phase. pub fn mbarrierInit(self: *Builder, barrier: Value, arrivals: u32) void { - self.emitVoid(mlir.Operation.make(self.ctx, "nvvm.mbarrier.init", .{ - .operands = .{ .flat = &.{ self.sharedBarrierPtr(barrier).inner, self.cst(.i32, arrivals).inner } }, - .attributes = &.{.named(self.ctx, "layout", self.parseAttribute("#nvvm.mbarrier_layout"))}, - .location = self.loc(), - })); + self.emitVoid(nvvm.mbarrier_init(self.ctx, self.sharedBarrierPtr(barrier).inner, self.cst(.i32, arrivals).inner, self.loc())); } /// Arrive and expect `bytes` of asynchronous transactions (SM90+), as a /// TMA producer does before issuing its copies. pub fn mbarrierArriveExpectTx(self: *Builder, barrier: Value, bytes: u32) void { - self.emitVoid(mlir.Operation.make(self.ctx, "nvvm.mbarrier.arrive.expect_tx", .{ - .operands = .{ .flat = &.{ self.sharedBarrierPtr(barrier).inner, self.cst(.i32, bytes).inner } }, - .attributes = &.{ - .named(self.ctx, "operandSegmentSizes", .denseArray(self.ctx, .i32, &.{ 1, 1, 0, 0 })), - .named(self.ctx, "relaxed", .boolean(self.ctx, false)), - .named(self.ctx, "scope", self.parseAttribute("#nvvm.mem_scope")), - }, - .location = self.loc(), - })); + self.emitVoid(nvvm.mbarrier_arrive_expect_tx(self.ctx, self.sharedBarrierPtr(barrier).inner, self.cst(.i32, bytes).inner, .cta, self.loc())); } pub fn mbarrierArrive(self: *Builder, barrier: Value, count: u32) void { - self.emitVoid(mlir.Operation.make(self.ctx, "nvvm.mbarrier.arrive", .{ - .operands = .{ .flat = &.{ self.sharedBarrierPtr(barrier).inner, self.cst(.i32, count).inner } }, - .attributes = &.{ - .named(self.ctx, "operandSegmentSizes", .denseArray(self.ctx, .i32, &.{ 1, 1, 0 })), - .named(self.ctx, "relaxed", .boolean(self.ctx, false)), - .named(self.ctx, "scope", self.parseAttribute("#nvvm.mem_scope")), - }, - .location = self.loc(), - })); + self.emitVoid(nvvm.mbarrier_arrive(self.ctx, self.sharedBarrierPtr(barrier).inner, self.cst(.i32, count).inner, .cta, self.loc())); } /// Test whether phase `phase` has completed, without suspending (SM90+). pub fn mbarrierWaitParity(self: *Builder, barrier: Value, phase: Value) Value { - return self.emit(mlir.Operation.make(self.ctx, "nvvm.mbarrier.wait.parity", .{ - .operands = .{ .flat = &.{ self.sharedBarrierPtr(barrier).inner, phase.inner } }, - .results = .{ .flat = &.{.int(self.ctx, .i1)} }, - .attributes = &.{ - .named(self.ctx, "kind", self.parseAttribute("#nvvm.mbar_wait")), - .named(self.ctx, "scope", self.parseAttribute("#nvvm.mbar_scope")), - }, - .location = self.loc(), - })); + return self.emit(nvvm.mbarrier_wait_parity(self.ctx, self.sharedBarrierPtr(barrier).inner, phase.inner, .@"try", .cta, self.loc())); } /// Wait for phase `phase`, letting the warp sleep for up to /// `suspend_time` cycles per attempt (SM90+). pub fn mbarrierTryWaitParity(self: *Builder, barrier: Value, phase: Value, suspend_time: u32) void { - self.emitVoid(mlir.Operation.make(self.ctx, "nvvm.mbarrier.try_wait.parity", .{ - .operands = .{ .flat = &.{ self.sharedBarrierPtr(barrier).inner, phase.inner, self.cst(.i32, suspend_time).inner } }, - .attributes = &.{.named(self.ctx, "useIntrinsic", .boolean(self.ctx, false))}, - .location = self.loc(), - })); + self.emitVoid(nvvm.mbarrier_try_wait_parity(self.ctx, self.sharedBarrierPtr(barrier).inner, phase.inner, self.cst(.i32, suspend_time).inner, self.loc())); } /// Wait for a pipeline phase with the two-step sequence of CuTe's @@ -2702,39 +2612,294 @@ pub const Builder = struct { /// Make mbarrier initialization visible to the async proxy (SM90+). pub fn fenceMbarrierInit(self: *Builder) void { - self.emitVoid(mlir.Operation.make(self.ctx, "nvvm.fence.mbarrier.init", .{ .location = self.loc() })); + self.emitVoid(nvvm.fence_mbarrier_init(self.ctx, self.loc())); } /// Publish ordinary shared-memory writes to the asynchronous (TMA) proxy (SM90+). pub fn fenceProxyAsyncShared(self: *Builder) void { - self.emitVoid(mlir.Operation.make(self.ctx, "nvvm.fence.proxy", .{ - .attributes = &.{ - .named(self.ctx, "kind", self.parseAttribute("#nvvm.proxy_kind")), - .named(self.ctx, "space", self.parseAttribute("#nvvm.shared_space")), - }, - .location = self.loc(), - })); + self.emitVoid(nvvm.fence_proxy(self.ctx, .@"async.shared", .cta, self.loc())); } /// Close the current group of bulk asynchronous copies, e.g. TMA stores (SM90+). pub fn cpAsyncBulkCommitGroup(self: *Builder) void { - self.emitVoid(mlir.Operation.make(self.ctx, "nvvm.cp.async.bulk.commit.group", .{ .location = self.loc() })); + self.emitVoid(nvvm.cp_async_bulk_commit_group(self.ctx, self.loc())); } /// Wait until at most `pending` bulk copy groups are in flight; with /// `read`, only until their sources have been read (SM90+). pub fn cpAsyncBulkWaitGroup(self: *Builder, pending: u32, read: bool) void { - var attrs: [2]mlir.NamedAttribute = undefined; - var len: usize = 1; - attrs[0] = .named(self.ctx, "group", .int(self.ctx, .i32, pending)); - if (read) { - attrs[1] = .named(self.ctx, "read", .unit(self.ctx)); - len += 1; + self.emitVoid(nvvm.cp_async_bulk_wait_group(self.ctx, pending, read, self.loc())); + } + + // ==================== raw-address SM90/SM100 primitives ==================== + // Hand-scheduled kernels address shared memory with `i32` offsets in the + // shared window, global memory with `i64` addresses and tensor memory with + // `i32` TMEM addresses (lane << 16 | column). These helpers emit the NVVM / + // LLVM operations for them directly (no inline PTX). + + pub const AddressSpace = llvm.AddressSpace; + + /// Typed LLVM pointer (`llvm.inttoptr`) for an integer address in `space`. + pub fn addressPtr(self: *Builder, address: Value, comptime space: AddressSpace) Value { + return self.emit(llvm.inttoptr(self.ctx, address.inner, llvm.pointerType(self.ctx, space), self.loc())); + } + + fn wordsType(self: *Builder, comptime n: usize) *const mlir.Type { + const word = mlir.Type.int(self.ctx, .i32); + return if (n == 1) word else mlir.Type.vector(&.{n}, word); + } + + fn splitWords(self: *Builder, comptime n: usize, v: Value) [n]Value { + var out: [n]Value = undefined; + if (n == 1) { + out[0] = v; + } else { + for (0..n) |i| out[i] = self.vectorExtract(v, @intCast(i), .i32); } - self.emitVoid(mlir.Operation.make(self.ctx, "nvvm.cp.async.bulk.wait_group", .{ - .attributes = attrs[0..len], - .location = self.loc(), - })); + return out; + } + + fn joinWords(self: *Builder, comptime n: usize, words: [n]Value) Value { + if (n == 1) return words[0]; + var inner: [n]*const mlir.Value = undefined; + for (words, &inner) |w, *x| x.* = w.inner; + return self.emit(vector.from_elements(self.ctx, &inner, self.wordsType(n), self.loc())); + } + + pub const LoadOptions = struct { + /// Read-only for the whole kernel (global: non-coherent `ld.global.nc`). + invariant: bool = false, + }; + + /// Load `n` (1, 2 or 4) consecutive 32-bit words from a typed pointer. + pub fn loadWordsPtr(self: *Builder, comptime n: usize, ptr: Value, opts: LoadOptions) [n]Value { + const v = self.emit(llvm.load(self.ctx, ptr.inner, self.wordsType(n), .{ .alignment = 4 * n, .invariant = opts.invariant }, self.loc())); + return self.splitWords(n, v); + } + + /// Load `n` (1, 2 or 4) consecutive 32-bit words at an integer address. + pub fn loadWords(self: *Builder, comptime n: usize, address: Value, comptime space: AddressSpace, opts: LoadOptions) [n]Value { + return self.loadWordsPtr(n, self.addressPtr(address, space), opts); + } + + /// Store `n` (1, 2 or 4) consecutive 32-bit words through a typed pointer. + pub fn storeWordsPtr(self: *Builder, comptime n: usize, ptr: Value, words: [n]Value) void { + self.emitVoid(llvm.store(self.ctx, self.joinWords(n, words).inner, ptr.inner, .{ .alignment = 4 * n }, self.loc())); + } + + /// Store `n` (1, 2 or 4) consecutive 32-bit words at an integer address. + pub fn storeWords(self: *Builder, comptime n: usize, address: Value, comptime space: AddressSpace, words: [n]Value) void { + self.storeWordsPtr(n, self.addressPtr(address, space), words); + } + + /// Store the low 16 bits of an `i32`. + pub fn storeHalf(self: *Builder, address: Value, comptime space: AddressSpace, word: Value) void { + self.emitVoid(llvm.store(self.ctx, word.to(.i16).inner, self.addressPtr(address, space).inner, .{ .alignment = 2 }, self.loc())); + } + + /// `bar.sync id, threads`: named barrier over `threads` threads. + pub fn barrierSync(self: *Builder, id: i32, threads: i32) void { + _ = self.callIntrinsic("llvm.nvvm.barrier.cta.sync.aligned.count", &.{ self.cst(.i32, id), self.cst(.i32, threads) }, null); + } + + /// `bar.red.or.pred`: named barrier over `threads` threads that also returns whether + /// `predicate` (i1) is true in any of them. + pub fn barrierReduceOr(self: *Builder, id: i32, threads: i32, predicate: Value) Value { + return self.callIntrinsic("llvm.nvvm.barrier.cta.red.or.aligned.count", &.{ self.cst(.i32, id), self.cst(.i32, threads), predicate }, mlir.Type.int(self.ctx, .i1)).?; + } + + /// `bar.warp.sync 0xffffffff`. + pub fn syncWarp(self: *Builder) void { + self.emitVoid(nvvm.bar_warp_sync(self.ctx, self.cst(.i32, -1).inner, self.loc())); + } + + /// Non-blocking test of mbarrier phase `parity` (i1). + pub fn mbarrierTestParity(self: *Builder, barrier: Value, parity: Value) Value { + return self.emit(nvvm.mbarrier_wait_parity(self.ctx, self.sharedBarrierPtr(barrier).inner, parity.inner, .@"test", .cta, self.loc())); + } + + /// Call an LLVM intrinsic by name (e.g. an `llvm.nvvm.*` intrinsic the NVVM dialect of + /// the CuTe compiler does not model). + pub fn callIntrinsic(self: *Builder, comptime name: []const u8, operands: []const Value, result: ?*const mlir.Type) ?Value { + const alloc = self.arena.allocator(); + const inner = alloc.alloc(*const mlir.Value, operands.len) catch @panic("OOM"); + for (operands, inner) |o, *x| x.* = o.inner; + const op = llvm.call_intrinsic(self.ctx, name, inner, result, self.loc()); + if (result == null) { + self.emitVoid(op); + return null; + } + return self.emit(op); + } + + /// 16-byte `cp.async.cg` global -> shared. With `valid` (i1) false, the destination is + /// zero-filled and `src` is not read (src-size 0). + pub fn cpAsync16(self: *Builder, dst: Value, src: Value, valid: ?Value) void { + const d = self.addressPtr(dst, .shared); + const g = self.addressPtr(src, .global); + if (valid) |v| { + _ = self.callIntrinsic("llvm.nvvm.cp.async.cg.shared.global.16.s", &.{ d, g, self.select(v, self.cst(.i32, 16), self.cst(.i32, 0)) }, null); + } else { + _ = self.callIntrinsic("llvm.nvvm.cp.async.cg.shared.global.16", &.{ d, g }, null); + } + } + + /// Arrive on `barrier` once this thread's prior cp.async copies have landed; with + /// `noinc`, the arrival is counted in the barrier's expected count. + pub fn cpAsyncMbarrierArrive(self: *Builder, barrier: Value, comptime noinc: bool) void { + _ = self.callIntrinsic(if (noinc) "llvm.nvvm.cp.async.mbarrier.arrive.noinc.shared" else "llvm.nvvm.cp.async.mbarrier.arrive.shared", &.{self.sharedBarrierPtr(barrier)}, null); + } + + /// Wait until all of this thread's cp.async copies have landed. + pub fn cpAsyncWaitAll(self: *Builder) void { + self.emitVoid(nvvm.cp_async_commit_group(self.ctx, self.loc())); + self.emitVoid(nvvm.cp_async_wait_group(self.ctx, 0, self.loc())); + } + + /// Prefetch the global line of `address` (i64) into L2 with evict-last priority. + pub fn prefetchL2EvictLast(self: *Builder, address: Value) void { + _ = self.callIntrinsic("llvm.nvvm.prefetch.global.L2.evict.last", &.{self.addressPtr(address, .global)}, null); + } + + /// Warp-wide TMEM allocation of `columns`; the base address is written to `holder` (i32 smem). + pub fn tmemAllocAt(self: *Builder, holder: Value, columns: u32) void { + self.emitVoid(nvvm.tcgen05_alloc(self.ctx, self.addressPtr(holder, .shared).inner, self.cst(.i32, columns).inner, self.loc())); + } + + pub fn tmemDeallocAt(self: *Builder, taddr: Value, columns: u32) void { + self.emitVoid(nvvm.tcgen05_dealloc(self.ctx, self.addressPtr(taddr, .tensor).inner, self.cst(.i32, columns).inner, self.loc())); + } + + pub const Tcgen05Fence = nvvm.Tcgen05FenceKind; + + /// `tcgen05.fence::{before,after}_thread_sync`. + pub fn tcgen05Fence(self: *Builder, comptime kind: Tcgen05Fence) void { + self.emitVoid(nvvm.tcgen05_fence(self.ctx, kind, self.loc())); + } + + /// `tcgen05.wait::st`. + pub fn fenceTmemStore(self: *Builder) void { + self.emitVoid(nvvm.tcgen05_wait(self.ctx, .store, self.loc())); + } + + /// `tcgen05.mma.cta_group::1.kind::f16`: D (TMEM at `d`) = A . B (+ D when + /// `accumulate`, i1), with A and B given by shared-memory descriptors (i64) and the + /// instruction descriptor `idesc` (i32). + pub fn tcgen05MmaF16(self: *Builder, d: Value, adesc: Value, bdesc: Value, idesc: Value, accumulate: Value) void { + self.emitVoid(nvvm.tcgen05_mma(self.ctx, self.addressPtr(d, .tensor).inner, adesc.inner, bdesc.inner, idesc.inner, accumulate.inner, .f16, .cta_1, self.loc())); + } + + /// `tcgen05.ld.32x32b.x{n}`: thread i of the warp gets TMEM lane (warp base + i), + /// `n` consecutive 32-bit columns from `taddr`. + pub fn tmemLoad32x32b(self: *Builder, comptime n: usize, taddr: Value) [n]Value { + const v = self.emit(nvvm.tcgen05_ld(self.ctx, self.addressPtr(taddr, .tensor).inner, self.wordsType(n), .shape_32x32b, self.loc())); + return self.splitWords(n, v); + } + + /// `tcgen05.st.32x32b.x{n}` (see `tmemLoad32x32b`). + pub fn tmemStore32x32b(self: *Builder, comptime n: usize, taddr: Value, words: [n]Value) void { + self.emitVoid(nvvm.tcgen05_st(self.ctx, self.addressPtr(taddr, .tensor).inner, self.joinWords(n, words).inner, .shape_32x32b, self.loc())); + } + + /// Rank of this CTA in its cluster. + pub fn clusterCtaRank(self: *Builder) Value { + return self.emit(nvvm.read_sreg(self.ctx, "cluster.ctarank", self.loc())); + } + + /// The same shared-memory location (i32 address) in CTA `rank` of the cluster, as a + /// `shared::cluster` pointer. + pub fn mapaShared(self: *Builder, address: Value, rank: Value) Value { + return self.emit(nvvm.mapa(self.ctx, self.addressPtr(address, .shared).inner, rank.inner, llvm.pointerType(self.ctx, .shared_cluster), self.loc())); + } + + /// Full cluster barrier (arrive.release + wait.acquire, aligned). + pub fn clusterSync(self: *Builder) void { + self.emitVoid(nvvm.cluster_arrive(self.ctx, true, self.loc())); + self.emitVoid(nvvm.cluster_wait(self.ctx, true, self.loc())); + } + + /// Warp-wide maximum of an f32 (`redux.sync.max.f32`, SM100). + pub fn warpMaxF32(self: *Builder, value: Value) Value { + return self.callIntrinsic("llvm.nvvm.redux.sync.fmax", &.{ value, self.cst(.i32, -1) }, mlir.Type.float(self.ctx, .f32)).?; + } + + /// Fast approximate 2^x (`ex2.approx.ftz.f32`). + pub fn exp2Approx(self: *Builder, value: Value) Value { + return self.emit(nvvm.ex2(self.ctx, value.inner, true, self.loc())); + } + + /// Fast approximate 1/x (`rcp.approx.ftz.f32`). + pub fn rcpApprox(self: *Builder, value: Value) Value { + return self.emit(nvvm.rcp_approx_ftz(self.ctx, value.inner, self.loc())); + } + + /// `vector<2 x dt>`, e.g. a bf16x2 pair packed in 32 bits. + fn pairType(self: *Builder, comptime dt: DType) *const mlir.Type { + return mlir.Type.vector(&.{2}, dt.toMlir(self.ctx)); + } + + fn bitcastTo(self: *Builder, value: Value, ty: *const mlir.Type) Value { + return self.emit(llvm.bitcast(self.ctx, value.inner, ty, self.loc())); + } + + /// Two FP8 e4m3 (low 16 bits of `pair`, i32) -> bf16x2 packed in an i32. + pub fn e4m3x2ToBf16x2(self: *Builder, pair: Value) Value { + const v = self.callIntrinsic("llvm.nvvm.e4m3x2.to.bf16x2.rn", &.{pair.to(.i16)}, self.pairType(.bf16)).?; + return self.bitcastTo(v, mlir.Type.int(self.ctx, .i32)); + } + + /// Two FP4 e2m1 (the low byte of `pair`, the even element in the low nibble) + /// -> bf16x2 packed in an i32. + pub fn e2m1x2ToBf16x2(self: *Builder, pair: Value) Value { + const v = self.callIntrinsic("llvm.nvvm.e2m1x2.to.bf16x2.rn", &.{pair.to(.i16)}, self.pairType(.bf16)).?; + return self.bitcastTo(v, mlir.Type.int(self.ctx, .i32)); + } + + /// Eight FP4 e2m1 (one word, even elements in the low nibbles) -> four bf16x2 packed + /// in i32s. The pairs are read as the low bytes of the 16-bit halves of `word` and + /// `word >> 8`, which ptxas maps to operand selectors (one shift per word). + pub fn e2m1x8ToBf16x2x4(self: *Builder, word: Value) [4]Value { + const halves = self.pairType(.i16); + const even = self.bitcastTo(word, halves); + const odd = self.bitcastTo(word.shrLogical(8), halves); + const pairs = [4]Value{ + self.vectorExtract(even, 0, .i16), self.vectorExtract(odd, 0, .i16), + self.vectorExtract(even, 1, .i16), self.vectorExtract(odd, 1, .i16), + }; + var out: [4]Value = undefined; + for (pairs, &out) |pair, *o| o.* = self.e2m1x2ToBf16x2(pair); + return out; + } + + /// bf16x2 * bf16x2 (round to nearest), both packed in i32. + pub fn mulBf16x2(self: *Builder, a: Value, b: Value) Value { + const ty = self.pairType(.bf16); + const p = self.emit(llvm.fmul(self.ctx, self.bitcastTo(a, ty).inner, self.bitcastTo(b, ty).inner, ty, self.loc())); + return self.bitcastTo(p, mlir.Type.int(self.ctx, .i32)); + } + + /// Two f32 -> a packed pair of `dt` (bf16 or f16) in an i32, `lo` in the low half. + pub fn packHalf2(self: *Builder, lo: Value, hi: Value, comptime dt: DType) Value { + const pair = self.emit(vector.from_elements(self.ctx, &.{ lo.to(dt).inner, hi.to(dt).inner }, self.pairType(dt), self.loc())); + return self.bitcastTo(pair, mlir.Type.int(self.ctx, .i32)); + } + + /// A packed pair of `dt` (bf16 or f16) in an i32 -> two f32 (low half first). + pub fn unpackHalf2(self: *Builder, word: Value, comptime dt: DType) [2]Value { + const pair = self.bitcastTo(word, self.pairType(dt)); + return .{ self.vectorExtract(pair, 0, dt).to(.f32), self.vectorExtract(pair, 1, dt).to(.f32) }; + } + + /// f32 -> bf16 bits (round to nearest even) in the low 16 bits of an i32. + pub fn f32ToBf16Bits(self: *Builder, value: Value) Value { + return self.emit(arith.extui(self.ctx, value.to(.bf16).bitCast(.i16).inner, mlir.Type.int(self.ctx, .i32), self.loc())); + } + + /// An i32 equal to `value` at run time that the compiler cannot constant-fold + /// (scaled by `%nctaid.z`, which is 1 for 2-D grids): e.g. a loop bound that must + /// keep a loop rolled (small code, fewer instruction-cache misses). + pub fn opaqueI32(self: *Builder, value: i32) Value { + return self.gridDim().z.mul(value); } // ==================== scalars ==================== diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/BUILD.bazel index 3b7bf6ce..9938dbaa 100644 --- a/mlir/dialects/BUILD.bazel +++ b/mlir/dialects/BUILD.bazel @@ -9,8 +9,10 @@ zig_library( "cuda.zig", "func.zig", "gpu.zig", + "llvm_dialect.zig", "math.zig", "memref.zig", + "nvvm.zig", "scf.zig", "shardy.zig", "vector.zig", diff --git a/mlir/dialects/dialects.zig b/mlir/dialects/dialects.zig index b5fd8d7b..e64b8601 100644 --- a/mlir/dialects/dialects.zig +++ b/mlir/dialects/dialects.zig @@ -13,8 +13,10 @@ pub const cf = @import("cf.zig"); pub const cuda = @import("cuda.zig"); pub const func = @import("func.zig"); pub const gpu = @import("gpu.zig"); +pub const llvm = @import("llvm_dialect.zig"); pub const math = @import("math.zig"); pub const memref = @import("memref.zig"); +pub const nvvm = @import("nvvm.zig"); pub const scf = @import("scf.zig"); pub const shardy = @import("shardy.zig"); pub const vector = @import("vector.zig"); diff --git a/mlir/dialects/llvm_dialect.zig b/mlir/dialects/llvm_dialect.zig new file mode 100644 index 00000000..9ca44ae2 --- /dev/null +++ b/mlir/dialects/llvm_dialect.zig @@ -0,0 +1,177 @@ +const std = @import("std"); + +const mlir = @import("mlir"); + +// ============================================================================= +// Types +// ============================================================================= + +/// Address spaces of `!llvm.ptr` on NVPTX. +pub const AddressSpace = enum(u32) { + generic = 0, + global = 1, + shared = 3, + constant = 4, + local = 5, + /// Tensor memory (SM100 TMEM). + tensor = 6, + /// Shared memory of any CTA of the cluster (`shared::cluster`). + shared_cluster = 7, +}; + +/// `!llvm.ptr`, an opaque pointer. +/// +/// Parsed rather than built with `mlirLLVMPointerTypeGet`: the type must come from the +/// LLVM dialect registered in `ctx`, which may be another MLIR build than the C API's +/// (e.g. in the CuTe compiler context), and loading the C API's dialect there would +/// clash with it. +pub fn pointerType(ctx: *mlir.Context, space: AddressSpace) *const mlir.Type { + var buf: [32]u8 = undefined; + const text = std.fmt.bufPrint(&buf, "!llvm.ptr<{d}>", .{@intFromEnum(space)}) catch unreachable; + return mlir.Type.parse(ctx, text) catch std.debug.panic("failed to parse LLVM pointer type '{s}'", .{text}); +} + +// ============================================================================= +// Casts and arithmetic +// ============================================================================= + +/// llvm.inttoptr — an integer address as a pointer of type `result_type`. +pub fn inttoptr(ctx: *mlir.Context, address: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.inttoptr", .{ + .operands = .{ .flat = &.{address} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// llvm.bitcast — reinterpret `value` as `result_type` (same bit width). +pub fn bitcast(ctx: *mlir.Context, value: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.bitcast", .{ + .operands = .{ .flat = &.{value} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// llvm.fmul — floating-point (or floating-point vector) product of type `result_type`. +pub fn fmul(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.fmul", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +// ============================================================================= +// Memory +// ============================================================================= + +pub const LoadOptions = struct { + /// Alignment in bytes (none: the natural alignment of the type). + alignment: ?u64 = null, + /// The memory is not written while the kernel runs (`!invariant.load`; on NVPTX + /// global memory this selects the non-coherent `ld.global.nc`). + invariant: bool = false, +}; + +/// llvm.load — a value of type `result_type` through `pointer`. +pub fn load(ctx: *mlir.Context, pointer: *const mlir.Value, result_type: *const mlir.Type, opts: LoadOptions, location: *const mlir.Location) *mlir.Operation { + var attrs: [2]mlir.NamedAttribute = undefined; + var len: usize = 0; + if (opts.alignment) |alignment| { + attrs[len] = .named(ctx, "alignment", .int(ctx, .i64, alignment)); + len += 1; + } + if (opts.invariant) { + attrs[len] = .named(ctx, "invariant", .unit(ctx)); + len += 1; + } + return mlir.Operation.make(ctx, "llvm.load", .{ + .operands = .{ .flat = &.{pointer} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs[0..len], + .location = location, + }); +} + +pub const StoreOptions = struct { + /// Alignment in bytes (none: the natural alignment of the type). + alignment: ?u64 = null, +}; + +/// llvm.store — `value` through `pointer`. +pub fn store(ctx: *mlir.Context, value: *const mlir.Value, pointer: *const mlir.Value, opts: StoreOptions, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + var len: usize = 0; + if (opts.alignment) |alignment| { + attrs[len] = .named(ctx, "alignment", .int(ctx, .i64, alignment)); + len += 1; + } + return mlir.Operation.make(ctx, "llvm.store", .{ + .operands = .{ .flat = &.{ value, pointer } }, + .attributes = attrs[0..len], + .location = location, + }); +} + +// ============================================================================= +// Intrinsics and inline assembly +// ============================================================================= + +/// llvm.call_intrinsic — call the LLVM intrinsic `name` (e.g. `llvm.nvvm.*`), with an +/// optional result of type `result_type`. +pub fn call_intrinsic( + ctx: *mlir.Context, + name: []const u8, + operands: []const *const mlir.Value, + result_type: ?*const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.call_intrinsic", .{ + .operands = .{ .flat = operands }, + .results = .{ .flat = if (result_type) |r| &.{r} else &.{} }, + .attributes = &.{ + .named(ctx, "intrin", .string(ctx, name)), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ @intCast(operands.len), 0 })), + .named(ctx, "op_bundle_sizes", .denseArray(ctx, .i32, &.{})), + }, + .location = location, + }); +} + +pub const InlineAsmOptions = struct { + /// LLVM constraint string, e.g. `"=r,r"` (outputs first). + constraints: []const u8 = "", + /// The assembly has effects beyond its outputs (it is never removed or merged). + has_side_effects: bool = false, +}; + +/// llvm.inline_asm — the PTX (or other target) assembly `asm_string`, with `$i` +/// placeholders bound to the result then `operands`. +pub fn inline_asm( + ctx: *mlir.Context, + asm_string: []const u8, + operands: []const *const mlir.Value, + result_type: ?*const mlir.Type, + opts: InlineAsmOptions, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: [3]mlir.NamedAttribute = undefined; + attrs[0] = .named(ctx, "asm_string", .string(ctx, asm_string)); + attrs[1] = .named(ctx, "constraints", .string(ctx, opts.constraints)); + var len: usize = 2; + if (opts.has_side_effects) { + attrs[len] = .named(ctx, "has_side_effects", .unit(ctx)); + len += 1; + } + return mlir.Operation.make(ctx, "llvm.inline_asm", .{ + .operands = .{ .flat = operands }, + .results = .{ .flat = if (result_type) |r| &.{r} else &.{} }, + .attributes = attrs[0..len], + .location = location, + }); +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/mlir/dialects/nvvm.zig b/mlir/dialects/nvvm.zig new file mode 100644 index 00000000..6cf5d71d --- /dev/null +++ b/mlir/dialects/nvvm.zig @@ -0,0 +1,419 @@ +const std = @import("std"); + +const mlir = @import("mlir"); + +// ============================================================================= +// Enum attributes — parsed from their textual `#nvvm.` form. +// ============================================================================= + +fn enumAttribute(ctx: *mlir.Context, comptime mnemonic: []const u8, value: []const u8) *const mlir.Attribute { + var buf: [96]u8 = undefined; + const text = std.fmt.bufPrint(&buf, "#nvvm." ++ mnemonic ++ "<{s}>", .{value}) catch unreachable; + return mlir.Attribute.parse(ctx, text) catch std.debug.panic("failed to parse NVVM attribute '{s}'", .{text}); +} + +fn EnumAttribute(comptime mnemonic: []const u8, comptime Tag: type) type { + return struct { + pub fn attribute(self: Tag, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(ctx, mnemonic, @tagName(self)); + } + }; +} + +/// `#nvvm.cta_group<...>`: CTAs cooperating on a tcgen05 operation. +pub const CtaGroup = enum { + cta_1, + cta_2, + pub const attribute = EnumAttribute("cta_group", CtaGroup).attribute; +}; + +/// `#nvvm.tcgen05_fence<...>`: `tcgen05.fence::{before,after}_thread_sync`. +pub const Tcgen05FenceKind = enum { + before, + after, + pub const attribute = EnumAttribute("tcgen05_fence", Tcgen05FenceKind).attribute; +}; + +/// `#nvvm.tcgen05_wait<...>`: `tcgen05.wait::{ld,st}`. +pub const Tcgen05WaitKind = enum { + load, + store, + pub const attribute = EnumAttribute("tcgen05_wait", Tcgen05WaitKind).attribute; +}; + +/// `#nvvm.tcgen05_mma_kind<...>`: input types of `tcgen05.mma`. +pub const Tcgen05MmaKind = enum { + f16, + tf32, + f8f6f4, + i8, + pub const attribute = EnumAttribute("tcgen05_mma_kind", Tcgen05MmaKind).attribute; +}; + +/// `#nvvm.tcgen05_ldst_shape<...>`: lane/bit shape of `tcgen05.ld` / `tcgen05.st`. +pub const Tcgen05LdStShape = enum { + shape_16x64b, + shape_16x128b, + shape_16x256b, + shape_32x32b, + shape_16x32bx2, + pub const attribute = EnumAttribute("tcgen05_ldst_shape", Tcgen05LdStShape).attribute; +}; + +/// `#nvvm.mem_scope<...>`: scope of a memory operation. +pub const MemScope = enum { + cta, + cluster, + gpu, + sys, + pub const attribute = EnumAttribute("mem_scope", MemScope).attribute; +}; + +/// `#nvvm.mbar_wait<...>`: `test` (non-blocking) or `try` (may suspend) wait. +pub const MBarrierWaitKind = enum { + @"test", + @"try", + pub const attribute = EnumAttribute("mbar_wait", MBarrierWaitKind).attribute; +}; + +/// `#nvvm.mbar_scope<...>`: scope of an mbarrier wait. +pub const MBarrierScope = enum { + cta, + cluster, + pub const attribute = EnumAttribute("mbar_scope", MBarrierScope).attribute; +}; + +/// `#nvvm.proxy_kind<...>`: memory proxy of `fence.proxy`. +pub const ProxyKind = enum { + alias, + async, + @"async.global", + @"async.shared", + pub const attribute = EnumAttribute("proxy_kind", ProxyKind).attribute; +}; + +/// `#nvvm.shared_space<...>`: `shared::cta` or `shared::cluster`. +pub const SharedSpace = enum { + cta, + cluster, + pub const attribute = EnumAttribute("shared_space", SharedSpace).attribute; +}; + +// ============================================================================= +// Special registers and thread synchronization +// ============================================================================= + +/// nvvm.read.ptx.sreg. — an i32 special register, e.g. `tid.x`, `ctaid.y`, +/// `cluster.ctarank`. +pub fn read_sreg(ctx: *mlir.Context, comptime name: []const u8, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg." ++ name, .{ + .results = .{ .flat = &.{.int(ctx, .i32)} }, + .location = location, + }); +} + +/// nvvm.barrier — `bar.sync 0` over the whole CTA. +pub fn barrier(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.barrier", .{ + .attributes = &.{.named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ 0, 0, 0 }))}, + .location = location, + }); +} + +/// nvvm.bar.warp.sync — `bar.warp.sync mask` (i32 lane mask, -1 for the full warp). +pub fn bar_warp_sync(ctx: *mlir.Context, mask: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.bar.warp.sync", .{ + .operands = .{ .flat = &.{mask} }, + .location = location, + }); +} + +/// nvvm.elect.sync — i1, true in one elected lane of the warp. +pub fn elect_sync(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.elect.sync", .{ + .results = .{ .flat = &.{.int(ctx, .i1)} }, + .location = location, + }); +} + +// ============================================================================= +// mbarrier +// ============================================================================= + +/// nvvm.mbarrier.init — initialize the shared mbarrier at `ptr` (`!llvm.ptr<3>`) +/// expecting `count` (i32) arrivals per phase. +pub fn mbarrier_init(ctx: *mlir.Context, ptr: *const mlir.Value, count: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mbarrier.init", .{ + .operands = .{ .flat = &.{ ptr, count } }, + .attributes = &.{.named(ctx, "layout", enumAttribute(ctx, "mbarrier_layout", "v0"))}, + .location = location, + }); +} + +/// nvvm.mbarrier.arrive — arrive `count` (i32) times (release semantics). +pub fn mbarrier_arrive(ctx: *mlir.Context, ptr: *const mlir.Value, count: *const mlir.Value, scope: MemScope, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive", .{ + .operands = .{ .flat = &.{ ptr, count } }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ 1, 1, 0 })), + .named(ctx, "relaxed", .boolean(ctx, false)), + .named(ctx, "scope", scope.attribute(ctx)), + }, + .location = location, + }); +} + +/// nvvm.mbarrier.arrive.expect_tx — arrive and expect `bytes` (i32) of asynchronous +/// transactions (SM90+). +pub fn mbarrier_arrive_expect_tx(ctx: *mlir.Context, ptr: *const mlir.Value, bytes: *const mlir.Value, scope: MemScope, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive.expect_tx", .{ + .operands = .{ .flat = &.{ ptr, bytes } }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ 1, 1, 0, 0 })), + .named(ctx, "relaxed", .boolean(ctx, false)), + .named(ctx, "scope", scope.attribute(ctx)), + }, + .location = location, + }); +} + +/// nvvm.mbarrier.wait.parity — i1, whether phase `parity` (i32) has completed +/// (`test`: never suspends; `try`: may suspend for a system-dependent time). +pub fn mbarrier_wait_parity( + ctx: *mlir.Context, + ptr: *const mlir.Value, + parity: *const mlir.Value, + kind: MBarrierWaitKind, + scope: MBarrierScope, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mbarrier.wait.parity", .{ + .operands = .{ .flat = &.{ ptr, parity } }, + .results = .{ .flat = &.{.int(ctx, .i1)} }, + .attributes = &.{ + .named(ctx, "kind", kind.attribute(ctx)), + .named(ctx, "scope", scope.attribute(ctx)), + }, + .location = location, + }); +} + +/// nvvm.mbarrier.try_wait.parity — loop until phase `parity` (i32) has completed, +/// suspending up to `suspend_time` (i32, cycles) per attempt. +pub fn mbarrier_try_wait_parity( + ctx: *mlir.Context, + ptr: *const mlir.Value, + parity: *const mlir.Value, + suspend_time: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mbarrier.try_wait.parity", .{ + .operands = .{ .flat = &.{ ptr, parity, suspend_time } }, + .attributes = &.{.named(ctx, "useIntrinsic", .boolean(ctx, false))}, + .location = location, + }); +} + +/// nvvm.fence.mbarrier.init — make mbarrier initialization visible to the async proxy. +pub fn fence_mbarrier_init(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.fence.mbarrier.init", .{ .location = location }); +} + +/// nvvm.fence.proxy — order memory accesses across proxies (e.g. generic writes +/// before asynchronous TMA / tcgen05 reads of shared memory). +pub fn fence_proxy(ctx: *mlir.Context, kind: ProxyKind, space: SharedSpace, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.fence.proxy", .{ + .attributes = &.{ + .named(ctx, "kind", kind.attribute(ctx)), + .named(ctx, "space", space.attribute(ctx)), + }, + .location = location, + }); +} + +// ============================================================================= +// Asynchronous copies +// ============================================================================= + +/// nvvm.cp.async.commit.group — close the current group of cp.async copies. +pub fn cp_async_commit_group(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.cp.async.commit.group", .{ .location = location }); +} + +/// nvvm.cp.async.wait.group — wait until at most `pending` cp.async groups are in flight. +pub fn cp_async_wait_group(ctx: *mlir.Context, pending: u32, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.cp.async.wait.group", .{ + .attributes = &.{.named(ctx, "n", .int(ctx, .i32, pending))}, + .location = location, + }); +} + +/// nvvm.cp.async.bulk.commit.group — close the current group of bulk copies (SM90+). +pub fn cp_async_bulk_commit_group(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.commit.group", .{ .location = location }); +} + +/// nvvm.cp.async.bulk.wait_group — wait until at most `pending` bulk groups are in +/// flight; with `read`, only until their sources have been read. +pub fn cp_async_bulk_wait_group(ctx: *mlir.Context, pending: u32, read: bool, location: *const mlir.Location) *mlir.Operation { + var attrs: [2]mlir.NamedAttribute = undefined; + attrs[0] = .named(ctx, "group", .int(ctx, .i32, pending)); + var len: usize = 1; + if (read) { + attrs[len] = .named(ctx, "read", .unit(ctx)); + len += 1; + } + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.wait_group", .{ + .attributes = attrs[0..len], + .location = location, + }); +} + +// ============================================================================= +// tcgen05 (SM100 tensor cores and tensor memory) +// ============================================================================= + +/// nvvm.tcgen05.alloc — allocate `columns` (i32) TMEM columns for the warp; the base +/// address is written to `holder` (`!llvm.ptr<3>`). +pub fn tcgen05_alloc(ctx: *mlir.Context, holder: *const mlir.Value, columns: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.tcgen05.alloc", .{ + .operands = .{ .flat = &.{ holder, columns } }, + .location = location, + }); +} + +/// nvvm.tcgen05.dealloc — free `columns` (i32) TMEM columns at `taddr` (`!llvm.ptr<6>`). +pub fn tcgen05_dealloc(ctx: *mlir.Context, taddr: *const mlir.Value, columns: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.tcgen05.dealloc", .{ + .operands = .{ .flat = &.{ taddr, columns } }, + .location = location, + }); +} + +/// nvvm.tcgen05.fence — `tcgen05.fence::{before,after}_thread_sync`. +pub fn tcgen05_fence(ctx: *mlir.Context, kind: Tcgen05FenceKind, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.tcgen05.fence", .{ + .attributes = &.{.named(ctx, "kind", kind.attribute(ctx))}, + .location = location, + }); +} + +/// nvvm.tcgen05.wait — wait for this thread's tcgen05 loads or stores. +pub fn tcgen05_wait(ctx: *mlir.Context, kind: Tcgen05WaitKind, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.tcgen05.wait", .{ + .attributes = &.{.named(ctx, "kind", kind.attribute(ctx))}, + .location = location, + }); +} + +/// nvvm.tcgen05.commit — arrive on the mbarrier at `barrier` (`!llvm.ptr<3>`) once the +/// preceding tcgen05 MMAs complete. +pub fn tcgen05_commit(ctx: *mlir.Context, barrier_ptr: *const mlir.Value, group: CtaGroup, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.tcgen05.commit", .{ + .operands = .{ .flat = &.{barrier_ptr} }, + .attributes = &.{.named(ctx, "group", group.attribute(ctx))}, + .location = location, + }); +} + +/// nvvm.tcgen05.mma — D (TMEM, `!llvm.ptr<6>`) = A * B (+ D when `accumulate`, i1), +/// A and B given by shared-memory descriptors (i64), `idesc` the instruction descriptor (i32). +pub fn tcgen05_mma( + ctx: *mlir.Context, + d: *const mlir.Value, + a_desc: *const mlir.Value, + b_desc: *const mlir.Value, + idesc: *const mlir.Value, + accumulate: *const mlir.Value, + kind: Tcgen05MmaKind, + cta_group: CtaGroup, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma", .{ + .operands = .{ .flat = &.{ d, a_desc, b_desc, idesc, accumulate } }, + .attributes = &.{ + .named(ctx, "kind", kind.attribute(ctx)), + .named(ctx, "ctaGroup", cta_group.attribute(ctx)), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ 1, 1, 1, 1, 1, 0, 0 })), + }, + .location = location, + }); +} + +/// nvvm.tcgen05.ld — load from TMEM at `taddr` (`!llvm.ptr<6>`) into `result_type` +/// (i32 or `vector`). +pub fn tcgen05_ld(ctx: *mlir.Context, taddr: *const mlir.Value, result_type: *const mlir.Type, shape: Tcgen05LdStShape, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.tcgen05.ld", .{ + .operands = .{ .flat = &.{taddr} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{.named(ctx, "shape", shape.attribute(ctx))}, + .location = location, + }); +} + +/// nvvm.tcgen05.st — store `value` (i32 or `vector`) to TMEM at `taddr`. +pub fn tcgen05_st(ctx: *mlir.Context, taddr: *const mlir.Value, value: *const mlir.Value, shape: Tcgen05LdStShape, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.tcgen05.st", .{ + .operands = .{ .flat = &.{ taddr, value } }, + .attributes = &.{.named(ctx, "shape", shape.attribute(ctx))}, + .location = location, + }); +} + +// ============================================================================= +// Clusters +// ============================================================================= + +/// nvvm.mapa — the shared-memory location `ptr` (`!llvm.ptr<3>`) in CTA `rank` (i32) +/// of the cluster, as a `result_type` (`!llvm.ptr<7>`) pointer. +pub fn mapa(ctx: *mlir.Context, ptr: *const mlir.Value, rank: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mapa", .{ + .operands = .{ .flat = &.{ ptr, rank } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// nvvm.cluster.arrive — `barrier.cluster.arrive` (release), `.aligned` when every +/// thread of the warp executes it. +pub fn cluster_arrive(ctx: *mlir.Context, aligned: bool, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.cluster.arrive", .{ + .attributes = if (aligned) &.{.named(ctx, "aligned", .unit(ctx))} else &.{}, + .location = location, + }); +} + +/// nvvm.cluster.wait — `barrier.cluster.wait` (acquire). +pub fn cluster_wait(ctx: *mlir.Context, aligned: bool, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.cluster.wait", .{ + .attributes = if (aligned) &.{.named(ctx, "aligned", .unit(ctx))} else &.{}, + .location = location, + }); +} + +// ============================================================================= +// Math +// ============================================================================= + +/// nvvm.ex2 — approximate 2^x (f32), flushing denormals when `ftz`. +pub fn ex2(ctx: *mlir.Context, value: *const mlir.Value, ftz: bool, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.ex2", .{ + .operands = .{ .flat = &.{value} }, + .results = .{ .flat = &.{mlir.Type.float(ctx, .f32)} }, + .attributes = &.{.named(ctx, "ftz", .boolean(ctx, ftz))}, + .location = location, + }); +} + +/// nvvm.rcp.approx.ftz.f — approximate 1/x (f32), flushing denormals. +pub fn rcp_approx_ftz(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.rcp.approx.ftz.f", .{ + .operands = .{ .flat = &.{value} }, + .results = .{ .flat = &.{mlir.Type.float(ctx, .f32)} }, + .location = location, + }); +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 55c76f49..50b7254b 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -81,6 +81,8 @@ zig_library( "attention.zig", "attention/attnd.zig", "attention/flashattn.zig", + "attention/cute_kernels/flashmla_sm100.zig", + "attention/cute_kernels/sparse_mla.zig", "attention/fly_kernels/sparse_mla.zig", "attention/metal_attention.zig", "attention/mosaic_tpu_kernels/ragged_attention.zig", diff --git a/zml/attention.zig b/zml/attention.zig index 2740c547..8549ba89 100644 --- a/zml/attention.zig +++ b/zml/attention.zig @@ -1,6 +1,7 @@ const std = @import("std"); pub const attnd = @import("attention/attnd.zig"); +pub const cute = @import("attention/cute_kernels/sparse_mla.zig"); pub const fly = @import("attention/fly_kernels/sparse_mla.zig"); pub const flashattn = @import("attention/flashattn.zig"); pub const metal = @import("attention/metal_attention.zig"); diff --git a/zml/attention/cute_kernels/flashmla_sm100.zig b/zml/attention/cute_kernels/flashmla_sm100.zig new file mode 100644 index 00000000..ed03b394 --- /dev/null +++ b/zml/attention/cute_kernels/flashmla_sm100.zig @@ -0,0 +1,1355 @@ +//! Sparse MLA decode / prefill for DeepSeek-V4.1 on SM100: persistent CTAs, +//! each processing query tokens t = cta, cta + ctas, ... (one token per +//! cluster when a token's key blocks are split across a cluster). +//! +//! Swapped-operand formulation (the 16 real heads are the MMA N dimension): +//! S^T [64 keys, 16 heads] = K_blk [64, 512] . Q^T tcgen05 M=64 N=16, K-major A/B +//! O^T [512, 16] += V_blk^T [512, 64] . P^T tcgen05 M=128 N=16, MN-major A +//! K and V are the same dequantized BF16 tile (SW128 K-major, 64 KiB), read +//! through two descriptors. Nothing is materialized in global memory and there +//! is no split-KV reduction (a cluster merges its partials through DSMEM). +//! +//! Warp roles (384 threads): +//! 0-3 softmax (lazy rescale, one barrier vote per block), O rescale (TMEM), epilogue +//! 4 MMA issue (one elected lane) +//! 5-6 loaders: cp.async of the raw rows and scales of each key block (zero-fill +//! for invalid rows) +//! 7 front: per token, validated indices and block counts (one token ahead), Q +//! 8-11 FP8/FP4 -> BF16 dequantization into the tile, in 128-column parts +//! Key blocks are numbered across the CTA's tokens: block g uses buffer g & 1 +//! with parity (g >> 1) & 1. Per-token state (indices, Q, O in TMEM) is +//! double-buffered by token parity so consecutive tokens overlap. +const std = @import("std"); + +const zml = @import("../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const V = cute.Value; + +pub const Cfg = struct { + queries: i64, + window_capacity: i64, + compressed_capacity: i64, + window_slots: i64, + compressed_slots: i64, + scale: f64, + /// CTAs per query token (a thread-block cluster): each takes a contiguous + /// range of the token's key blocks and the partial results are merged + /// through distributed shared memory, inside the kernel. Power of two. + /// Clusters of more than one CTA require `ctas == queries`. + cluster: i64 = 1, + /// Persistent CTAs (clusters): token t is processed by CTA t % ctas. + ctas: i64 = 0, + + fn gridCtas(c: Cfg) i64 { + return if (c.ctas > 0) @min(c.ctas, c.queries) else c.queries; + } +}; + +pub const Program = cute.Program(Cfg, .{ + .name = "flashmla_sm100", + .inputs = &.{ "q", "wv", "ws", "cv", "cs", "wi", "ci", "lengths", "sink", "active" }, + .outputs = &.{"out"}, + .run = program, +}); + +const kernel_name = "flashmla_sm100_kernel"; +const H = 16; +const D = 512; +const BK = 64; +const threads = 384; +const log2e = 1.4426950408889634; +/// Lazy rescale: the running max only moves when a block exceeds it by this +/// many log2 units, so P stays below 2^32 (BF16 P, F32 l and O keep their +/// relative precision) and O is rarely rescaled. +const rescale_threshold = 32.0; + +// Shared memory plan (bytes). +const smem = struct { + const tile = [2]i32{ 0, 65536 }; + /// Q of token parity kb (SW128 K-major, the B operand of QK). + const q = [2]i32{ 131072, 147456 }; + /// Raw (quantized) rows: a ring of slots of 64 rows x 288 bytes (see `rawBase`). + /// An FP4 block takes one slot (256 data bytes, then the 32 scale bytes of each + /// row); an FP8 block takes two (row bytes 0-255, then 256-511; its 16 scale + /// bytes follow the second slot's data). + const raw_slot = 64 * 288; + const scale_off = 256; + const p = [2]i32{ 221184, 223232 }; + const red = [2]i32{ 225280, 225536 }; + const red_sum = [2]i32{ 225792, 226048 }; + const bars = 226304; + const holder = bars + 8 * 40; + /// Per token parity: physical row per candidate (or -1), window [128] then + /// compressed [512], followed by the token's block counts (window, compressed). + const idx = holder + 16; + const idx_bytes = 640 * 4 + 16; + const counts = 640 * 4; + /// Sink logits (f32 [16], prefetched at start). + const sink = idx + 2 * idx_bytes; + const total = sink + H * 4; + /// Output staging, bf16 [16 heads][512] (reuses Q after the CTA's last QK). + const ostage = q[0]; + /// Cluster merge (after every block of the cluster is done), in the raw ring: + /// per-head max and sum of this CTA, merge weights [16 CTAs][16 heads], and the + /// inbox of (m, l) per head from every rank [16 ranks][32] f32. + const stats = 163840; + const weights = stats + 2 * H * 4; + const stats_in = weights + 16 * H * 4; + /// Cluster merge: O^T [512][16] f32 of this CTA (reuses tile 0 after the last PV), + /// and the inbox of the dims this rank owns, [ranks][512 / C][16] f32 (tile 0, upper half). + const obuf = 0; + const inbox = 32768; +}; + +const Bar = enum(i32) { + raw_full = 0, // [4] + raw_empty = 4, // [4] + tile_full = 8, + s_full = 10, + p_full = 12, + pv_done = 14, + idx_ready = 16, + idx_free = 18, + o_free = 20, + q_free = 22, + q_ready = 24, +}; + +/// Arrivals on idx_free: softmax warps (4), MMA (1), loaders (2), dequant warps (4). +const idx_consumers = 11; + +fn bar(b: *B, base: V, comptime which: Bar, buf: anytype) V { + return base.add(smem.bars + 8 * @intFromEnum(which)).add(b.lift(buf).mul(8)); +} + +// TMEM columns: O^T of token parity ob in 64 * ob (4 chunks of 128 dims x 16 +// heads); S^T of buffer `buf` in 128 + 64 * buf: four partial sums over 128 +// latent dims each, 16 columns apart (independent accumulators so the QK MMAs pipeline). +const tmem_cols = 256; +const tmem_s = 128; +const qk_parts = 4; + +const Format = enum { + fp8, + fp4, + fn rowBytes(f: Format) i32 { + return if (f == .fp8) 512 else 256; + } + fn scaleBytes(f: Format) i32 { + return if (f == .fp8) 16 else 32; + } +}; + +const Globals = struct { + q: V, + wv: V, + ws: V, + cv: V, + cs: V, + wi: V, + ci: V, + lengths: V, + sink: V, + active: V, + out: V, +}; + +const Ctx = struct { + b: *B, + cfg: Cfg, + g: Globals, + s: V, // shared base + tid: V, + warp: V, + lane: V, + taddr: V, + rank: V, // CTA rank in the token's cluster + cta: V, // cluster index: first token + ntok: V, // tokens of this CTA + // Per token (see `forToken`). + k: V = undefined, // token ordinal within the CTA + t: V = undefined, // query token + kb: V = undefined, // token parity (index buffer, O buffer) + ib: V = undefined, // shared address of the token's index buffer + nwb: V = undefined, // window blocks + ncb: V = undefined, // compressed blocks + b0: V = undefined, // this CTA's key blocks [b0, b1) of the token + b1: V = undefined, + n: V = undefined, // b1 - b0; local block j is block b0 + j of the token + nwl: V = undefined, // local window (FP8) blocks: the first nwl local blocks + g0: V = undefined, // CTA-wide number of the token's first local block + slot0: V = undefined, // CTA-wide raw-slot use of the token's first local block +}; + +/// The per-token context of token ordinal k; `counts` waits for the token's +/// index buffer and loads the block counts. +fn forToken(c0: Ctx, k: V, g0: V, slot0: V, comptime counts: bool) Ctx { + const b = c0.b; + var c = c0; + c.k = k; + c.t = c0.cta.add(k.mul(@as(i32, @intCast(c0.cfg.gridCtas())))); + c.kb = k.bitAnd(1); + c.ib = c0.s.add(smem.idx).add(c.kb.mul(smem.idx_bytes)); + c.g0 = g0; + c.slot0 = slot0; + if (counts) { + b.mbarrierWait(bar(b, c.s, .idx_ready, c.kb), parity(k)); + const nc = b.loadWords(2, c.ib.add(smem.counts), .shared, .{}); + c.nwb = nc[0]; + c.ncb = nc[1]; + const nb = c.nwb.add(c.ncb); + const log_c: i32 = @intCast(std.math.log2_int(u64, @intCast(c0.cfg.cluster))); + c.b0 = c0.rank.mul(nb).shrLogical(log_c); + c.b1 = c0.rank.add(1).mul(nb).shrLogical(log_c); + c.n = c.b1.sub(c.b0); + c.nwl = c.b1.minimum(c.nwb).sub(c.b0).maximum(0); + } + return c; +} + +/// This warp is done with the token's index buffer. +fn releaseIndices(c: Ctx) void { + const b = c.b; + b.syncWarp(); + var l0 = b.openIf(c.lane.eq(0)); + b.mbarrierArrive(bar(b, c.s, .idx_free, c.kb), 1); + l0.yieldThen(.{}); +} + +/// First raw-slot use of local block j: FP8 (window) blocks come first and use two. +fn slotUse(c: Ctx, j: V) V { + return c.slot0.add(j).add(j.minimum(c.nwl)); +} + +/// With one token per CTA the second Q buffer is unused: the ring starts there and +/// has 4 slots (two FP8 blocks in flight); otherwise 3 slots after it. +fn oneTokenPerCta(cfg: Cfg) bool { + return cfg.gridCtas() == cfg.queries; +} + +fn rawBase(cfg: Cfg) i32 { + return if (oneTokenPerCta(cfg)) smem.q[1] else smem.q[1] + 16384; +} + +fn rawSlots(cfg: Cfg) i32 { + return if (oneTokenPerCta(cfg)) 4 else 3; +} + +fn slotAddr(c: Ctx, use: V) V { + return c.s.add(rawBase(c.cfg)).add(use.rem(rawSlots(c.cfg)).mul(smem.raw_slot)); +} + +fn scaleAddr(c: Ctx, use: V) V { + return slotAddr(c, use).add(smem.scale_off); +} + +fn slotParity(c: Ctx, use: V) V { + return use.div(rawSlots(c.cfg)).bitAnd(1); +} + +fn parity(j: V) V { + return j.shrLogical(1).bitAnd(1); +} + +fn tupleOf(comptime n: usize, arr: [n]V) std.meta.Tuple(&([_]type{V} ** n)) { + var t: std.meta.Tuple(&([_]type{V} ** n)) = undefined; + inline for (0..n) |i| t[i] = arr[i]; + return t; +} + +fn program(b: *B, cfg: Cfg) cute.FinishError!void { + b.beginFunction(kernel_name, .cuda_kernel); + try kernel(b, cfg); + b.endFunction(.{ threads, 1, 1 }); + + b.beginFunction(Program.name, .host); + const h = try b.declareArgs(.{ + .q = .{ .ptr = cute.DType.bf16 }, + .wv = .{ .ptr = cute.DType.i8 }, + .ws = .{ .ptr = cute.DType.i8 }, + .cv = .{ .ptr = cute.DType.i8 }, + .cs = .{ .ptr = cute.DType.i8 }, + .wi = .{ .ptr = cute.DType.i32 }, + .ci = .{ .ptr = cute.DType.i32 }, + .lengths = .{ .ptr = cute.DType.i32 }, + .sink = .{ .ptr = cute.DType.f32 }, + .active = .{ .ptr = cute.DType.i32 }, + .out = .{ .ptr = cute.DType.bf16 }, + }); + launch(b, cfg, .{ h.q, h.wv, h.ws, h.cv, h.cs, h.wi, h.ci, h.lengths, h.sink, h.active, h.out }); + b.endFunction(null); +} + +fn launch(b: *B, cfg: Cfg, args: anytype) void { + const one = b.cst(.i32, 1); + const config = b.makeLaunchConfig(.{ + .grid = .{ b.cst(.i32, @as(i32, @intCast(cfg.gridCtas() * cfg.cluster))), one, one }, + .block = .{ b.cst(.i32, threads), one, one }, + .dynamic_smem = b.kernelSmemSize(kernel_name), + .stream = b.cudaStream(), + .cluster = .{ b.cst(.i32, @as(i32, @intCast(cfg.cluster))), one, one }, + .use_pdl = true, + }); + const status = b.launchEx(kernel_name, config, args); + b.returnHostStatus(b.cudaResultStatus(status)); +} + +fn kernel(b: *B, cfg: Cfg) cute.FinishError!void { + const g = try globalsOf(b, try b.declareArgs(.{ + .q = .{ .ptr = cute.DType.bf16 }, + .wv = .{ .ptr = cute.DType.i8 }, + .ws = .{ .ptr = cute.DType.i8 }, + .cv = .{ .ptr = cute.DType.i8 }, + .cs = .{ .ptr = cute.DType.i8 }, + .wi = .{ .ptr = cute.DType.i32 }, + .ci = .{ .ptr = cute.DType.i32 }, + .lengths = .{ .ptr = cute.DType.i32 }, + .sink = .{ .ptr = cute.DType.f32 }, + .active = .{ .ptr = cute.DType.i32 }, + .out = .{ .ptr = cute.DType.bf16 }, + })); + try kernelBody(b, cfg, g); +} + +fn globalsOf(b: *B, a: anytype) !Globals { + b.setFunctionAttribute("nvvm.minctasm", .int(b.ctx, .i32, 1)); + b.setFunctionAttribute("cu_attrs", b.parseAttribute("{max_dynamic_shared_size_bytes = #cuda.dev_max_shared_memory_optin, non_portable_cluster_size_allowed = 1 : i32}")); + + return .{ + .q = b.ptrToInt(a.q, .i64), + .wv = b.ptrToInt(a.wv, .i64), + .ws = b.ptrToInt(a.ws, .i64), + .cv = b.ptrToInt(a.cv, .i64), + .cs = b.ptrToInt(a.cs, .i64), + .wi = b.ptrToInt(a.wi, .i64), + .ci = b.ptrToInt(a.ci, .i64), + .lengths = b.ptrToInt(a.lengths, .i64), + .sink = b.ptrToInt(a.sink, .i64), + .active = b.ptrToInt(a.active, .i64), + .out = b.ptrToInt(a.out, .i64), + }; +} + +fn kernelBody(b: *B, cfg: Cfg, g: Globals) cute.FinishError!void { + const tid = b.threadIdx().x; + const warp = b.makeWarpUniform(tid.shrLogical(5)); + const lane = tid.bitAnd(31); + std.debug.assert(std.math.isPowerOfTwo(cfg.cluster) and cfg.cluster <= 16); + std.debug.assert(cfg.cluster == 1 or cfg.gridCtas() == cfg.queries); + const log_c: i32 = @intCast(std.math.log2_int(u64, @intCast(cfg.cluster))); + const cta = b.blockIdx().x.shrLogical(log_c); + const stride: i32 = @intCast(cfg.gridCtas()); + const queries: i32 = @intCast(cfg.queries); + const rank = if (cfg.cluster > 1) b.clusterCtaRank() else b.cst(.i32, 0); + const s = sharedStorage(b, smem.total); + + // Inputs come from the previous kernels of the layer. + b.waitForDependency(); + const c0: Ctx = .{ + .b = b, + .cfg = cfg, + .g = g, + .s = s, + .tid = tid, + .warp = warp, + .lane = lane, + .taddr = undefined, + .rank = rank, + .cta = cta, + .ntok = cta.sub(queries).neg().add(stride - 1).div(stride), + }; + // The front warp issues the first token's loads right away, overlapping the setup + // (other warps load harmless in-bounds addresses and discard them). + const first = frontLoad(c0, b.select(warp.eq(7), cta, b.cst(.i32, 0))); + { + var w7 = b.openIf(warp.eq(7)); + issueQ(c0, cta, b.cst(.i32, 0)); + w7.yieldThen(.{}); + } + { + var t0 = b.openIf(tid.eq(0)); + for (0..@intCast(rawSlots(cfg))) |i| { + b.mbarrierInit(bar(b, s, .raw_full, @as(i32, @intCast(i))), 64); + b.mbarrierInit(bar(b, s, .raw_empty, @as(i32, @intCast(i))), 128); + } + for (0..2) |i| { + b.mbarrierInit(bar(b, s, .tile_full, @as(i32, @intCast(i))), 128); + b.mbarrierInit(bar(b, s, .s_full, @as(i32, @intCast(i))), 1); + b.mbarrierInit(bar(b, s, .p_full, @as(i32, @intCast(i))), 128); + b.mbarrierInit(bar(b, s, .pv_done, @as(i32, @intCast(i))), 1); + b.mbarrierInit(bar(b, s, .idx_ready, @as(i32, @intCast(i))), 32); + b.mbarrierInit(bar(b, s, .idx_free, @as(i32, @intCast(i))), idx_consumers); + b.mbarrierInit(bar(b, s, .o_free, @as(i32, @intCast(i))), 128); + b.mbarrierInit(bar(b, s, .q_ready, @as(i32, @intCast(i))), 32); + b.mbarrierInit(bar(b, s, .q_free, @as(i32, @intCast(i))), 1); + } + b.fenceMbarrierInit(); + t0.yieldThen(.{}); + } + { + var w0 = b.openIf(warp.eq(0)); + b.tmemAllocAt(s.add(smem.holder), tmem_cols); + b.relinquishTmemAllocPermit(.{}); + w0.yieldThen(.{}); + } + + b.tcgen05Fence(.before); + b.barrierSync(0, threads); + b.tcgen05Fence(.after); + var c = c0; + c.taddr = b.loadWords(1, s.add(smem.holder), .shared, .{})[0]; + { + var role = b.openIf(warp.lt(4)); + softmaxRole(c); + role.yieldThen(.{}); + } + { + var role = b.openIf(warp.eq(4)); + mmaRole(c); + role.yieldThen(.{}); + } + { + var role = b.openIf(warp.eq(5).bitOr(warp.eq(6))); + loaderRole(c); + role.yieldThen(.{}); + } + { + var role = b.openIf(warp.eq(7)); + frontRole(c, first); + role.yieldThen(.{}); + } + { + var role = b.openIf(warp.ge(8)); + dequantRole(c); + role.yieldThen(.{}); + } + + b.tcgen05Fence(.before); + b.barrierSync(0, threads); + { + var w0 = b.openIf(warp.eq(0)); + b.tcgen05Fence(.after); + b.tmemDeallocAt(c.taddr, tmem_cols); + w0.yieldThen(.{}); + } + if (cfg.cluster > 1) { + // One token per cluster. + var cm = c; + cm.t = cta; + clusterMerge(cm); + } + b.launchDependents(); +} + +// ---- front (warp 7): indices, lengths and Q of each token --------------------------------- + +const FrontRegs = struct { + active: V, + lens: [2]V, + w: [4]V, // this lane's 4 window indices + c: [16]V, // this lane's 16 compressed indices +}; + +/// Issue the global loads of token t's lengths and indices (lane l holds +/// candidates [4 l, 4 l + 4) of the window and [16 l, 16 l + 16) of the compressed stream). +fn frontLoad(c: Ctx, t: V) FrontRegs { + const b = c.b; + var r: FrontRegs = undefined; + r.active = b.loadWords(1, c.g.active, .global, .{ .invariant = true })[0]; + r.lens = b.loadWords(2, c.g.lengths.add(t.to(.i64).mul(8)), .global, .{ .invariant = true }); + const wcap: i32 = @intCast(c.cfg.window_capacity); + const ccap: i32 = @intCast(c.cfg.compressed_capacity); + r.w = b.loadWords(4, c.g.wi.add(t.mul(wcap).add(c.lane.mul(@divExact(wcap, 32))).to(.i64).mul(4)), .global, .{ .invariant = true }); + for (&r.c) |*x| x.* = b.cst(.i32, -1); + if (ccap > 0) { + const per_lane = @divExact(ccap, 32); + const src = c.g.ci.add(t.mul(ccap).add(c.lane.mul(per_lane)).to(.i64).mul(4)); + for (0..@intCast(@divExact(per_lane, 4))) |v| { + const y = b.loadWords(4, src.add(@as(i64, @intCast(v * 16))), .global, .{ .invariant = true }); + for (0..4) |i| r.c[v * 4 + i] = y[i]; + } + } + return r; +} + +/// Validate token t's candidates (invalid ones become -1), store them and the +/// block counts in index buffer `kb`, and publish them. +fn frontStore(c: Ctx, t: V, kb: V, r: FrontRegs) void { + const b = c.b; + const zero = b.cst(.i32, 0); + // Padded (inactive) query rows get no key blocks: their output is zero. + const is_active = t.lt(r.active); + const wlen = b.select(is_active, r.lens[0].minimum(@as(i32, @intCast(c.cfg.window_capacity))).maximum(0), zero); + const clen = if (c.cfg.compressed_capacity > 0) b.select(is_active, r.lens[1].minimum(@as(i32, @intCast(c.cfg.compressed_capacity))).maximum(0), zero) else zero; + const ib = c.s.add(smem.idx).add(kb.mul(smem.idx_bytes)); + inline for (.{ Format.fp8, Format.fp4 }) |f| { + const capacity: i32 = @intCast(if (f == .fp8) c.cfg.window_capacity else c.cfg.compressed_capacity); + if (capacity > 0) { + const slots: i32 = @intCast(if (f == .fp8) c.cfg.window_slots else c.cfg.compressed_slots); + const length = if (f == .fp8) wlen else clen; + const per_lane = @divExact(capacity, 32); // 4 (window) or 16 (compressed) + for (0..@intCast(per_lane)) |i| { + const pos = c.lane.mul(per_lane).add(@as(i32, @intCast(i))); + const ix = if (f == .fp8) r.w[i] else r.c[i]; + const ok = pos.lt(length).bitAnd(ix.ge(0)).bitAnd(ix.lt(slots)); + b.storeWords(1, ib.add(idxBase(f)).add(pos.mul(4)), .shared, .{b.select(ok, ix, b.cst(.i32, -1))}); + } + } + } + { + var l0 = b.openIf(c.lane.eq(0)); + b.storeWords(2, ib.add(smem.counts), .shared, .{ wlen.add(BK - 1).shrLogical(6), clen.add(BK - 1).shrLogical(6) }); + l0.yieldThen(.{}); + } + b.mbarrierArrive(bar(b, c.s, .idx_ready, kb), 1); +} + +/// Q [16, 512] BF16 of token t -> SW128 K-major shared tile (the B operand of QK). +fn issueQ(c: Ctx, t: V, kb: V) void { + const b = c.b; + const q_row = c.g.q.add(t.to(.i64).mul(H * D * 2)); + const q_smem = c.s.add(b.select(kb.eq(0), b.cst(.i32, smem.q[0]), b.cst(.i32, smem.q[1]))); + for (0..32) |it| { + const chunk = c.lane.add(@as(i32, @intCast(it * 32))); + const row = chunk.shrLogical(6); + const col = chunk.bitAnd(63).mul(8); + b.cpAsync16(q_smem.add(sw128Offset(b, row, col, H)), q_row.add(row.mul(D).add(col).mul(2).to(.i64)), null); + } +} + +fn frontRole(c0: Ctx, first: FrontRegs) void { + const b = c0.b; + frontStore(c0, c0.cta, b.cst(.i32, 0), first); + var loop = b.openFor(0, c0.ntok, 1, .{}); + const k = loop.iv; + const c = forToken(c0, k, undefined, undefined, false); + { + // Q of token k (token 0: issued at start) once every QK of token k - 2 has completed. + var later = b.openIf(k.gt(0)); + { + var reuse = b.openIf(k.ge(2)); + b.mbarrierWait(bar(b, c.s, .q_free, c.kb), parity(k).bitXor(1)); + reuse.yieldThen(.{}); + } + issueQ(c, c.t, c.kb); + later.yieldThen(.{}); + } + // The MMA reads Q through the async proxy. + b.cpAsyncWaitAll(); + b.fenceProxyAsyncShared(); + b.mbarrierArrive(bar(b, c.s, .q_ready, c.kb), 1); + { + var more = b.openIf(k.add(1).lt(c.ntok)); + const next_t = c.t.add(@as(i32, @intCast(c.cfg.gridCtas()))); + // Warm L2 with the next token's Q (16 KiB: 4 lines per lane). + const q_next = c.g.q.add(next_t.to(.i64).mul(H * D * 2)); + for (0..4) |i| b.prefetchL2EvictLast(q_next.add(c.lane.add(@as(i32, @intCast(i * 32))).mul(128).to(.i64))); + const r = frontLoad(c, next_t); + const nkb = k.add(1).bitAnd(1); + { + // The buffer's previous token (k - 1) must be released by every consumer. + var reuse = b.openIf(k.ge(1)); + b.mbarrierWait(bar(b, c.s, .idx_free, nkb), parity(k.add(1)).bitXor(1)); + reuse.yieldThen(.{}); + } + frontStore(c, next_t, nkb, r); + more.yieldThen(.{}); + } + loop.yield(.{}); +} + +// ---- loaders (warps 5-6) ------------------------------------------------------------ + +fn loaderRole(c0: Ctx) void { + const b = c0.b; + var loop = b.openFor(0, c0.ntok, 1, .{b.cst(.i32, 0)}); + const c = forToken(c0, loop.iv, undefined, loop.carried[0], true); + { + var blocks = b.openFor(c.b0.minimum(c.nwb), c.b1.minimum(c.nwb), 1, .{}); + loadBlock(c, .fp8, blocks.iv.sub(c.b0), blocks.iv); + blocks.yield(.{}); + } + if (c.cfg.compressed_capacity > 0) { + var blocks = b.openFor(c.b0.maximum(c.nwb), c.b1, 1, .{}); + loadBlock(c, .fp4, blocks.iv.sub(c.b0), blocks.iv.sub(c.nwb)); + blocks.yield(.{}); + } + releaseIndices(c); + loop.yield(.{slotUse(c, c.n)}); +} + +fn idxBase(comptime f: Format) i32 { + return if (f == .fp8) 0 else 128 * 4; +} + +fn loadBlock(c: Ctx, comptime f: Format, j: V, local: V) void { + const b = c.b; + const use_a = slotUse(c, j); + const use_b = use_a.add(1); // second slot of an FP8 block + const slot_a = use_a.rem(rawSlots(c.cfg)); + const slot_b = use_b.rem(rawSlots(c.cfg)); + for (0..@as(usize, if (f == .fp8) 2 else 1)) |k| { + const use = if (k == 0) use_a else use_b; + var reuse = b.openIf(use.ge(rawSlots(c.cfg))); + b.mbarrierWait(bar(b, c.s, .raw_empty, use.rem(rawSlots(c.cfg))), slotParity(c, use).bitXor(1)); + reuse.yieldThen(.{}); + } + const values = if (f == .fp8) c.g.wv else c.g.cv; + const scales = if (f == .fp8) c.g.ws else c.g.cs; + // FP8 rows are split across two slots: lanes 0-15 copy bytes 0-255 into slot a, + // lanes 16-31 bytes 256-511 into slot b (both with the 288-byte FP4 row stride). + const raw = if (f == .fp8) b.select(c.lane.lt(16), slotAddr(c, use_a), slotAddr(c, use_b)) else slotAddr(c, use_a); + const sc = scaleAddr(c, if (f == .fp8) use_b else use_a); + const idx = c.ib.add(idxBase(f)).add(local.mul(BK * 4)); + // Row data in 16-byte chunks. FP4 (16 chunks per row): lanes 0-15 copy rows + // 0-31 and lanes 16-31 rows 32-63, two rows per warp instruction. FP8 (32 + // chunks per row): one row per instruction. Invalid rows (-1) are zero-filled. + const chunks_per_row = @divExact(f.rowBytes(), 16); + const chunk16 = c.lane.bitAnd(chunks_per_row - 1).mul(16); + const chunk_off = chunk16.to(.i64); + const dst_chunk = c.lane.bitAnd(15).mul(16); // within a 288-byte slot row + // Two loader warps: warp 5 takes the first half of this lane's rows, warp 6 the second. + // Four rows per iteration: unrolled for FP4 (4 iterations), a runtime loop for FP8 (small code). + const half_rows: i32 = comptime if (f == .fp4) 16 else 32; + const second = c.warp.eq(6).to(.i32).bitAnd(1); + const first_row = (if (f == .fp4) c.lane.shrLogical(4).mul(32) else b.cst(.i32, 0)).add(second.mul(half_rows)); + const src_base = values.add(chunk_off); + const dst_base = raw.add(first_row.mul(288)).add(dst_chunk); + const idx_base = idx.add(first_row.mul(4)); + const copyRows = struct { + fn f_(bb: *B, r: V, dst_base_: V, src_base_: V, idx_base_: V) void { + const ixs = ldSharedBatch(bb, 1, 4, idx_base_.add(r.mul(4)), .{0}); + for (0..4) |k| { + cpAsync16Row(bb, dst_base_.add(r.add(@as(i32, @intCast(k))).mul(288)), src_base_, ixs[k], @intCast(f.rowBytes()), bb.cst(.i64, 0)); + } + } + }.f_; + if (f == .fp4) { + for (0..half_rows / 4) |i| copyRows(b, b.cst(.i32, @as(i32, @intCast(4 * i))), dst_base, src_base, idx_base); + } else { + var rows = b.openFor(0, b.opaqueI32(half_rows), 4, .{}); + copyRows(b, rows.iv, dst_base, src_base, idx_base); + rows.yield(.{}); + } + // Scales: 16 (FP8) or 32 (FP4) bytes per row, one 16-byte chunk per lane and step. + const scale_chunks = comptime @divExact(f.scaleBytes(), 16); + const scale_steps = comptime @divExact(BK * scale_chunks, 32); + const sidx_offsets = comptime blk: { + var o: [scale_steps]i32 = undefined; + for (0..scale_steps) |k| o[k] = @as(i32, @intCast(k)) * @divExact(32, scale_chunks) * 4; + break :blk o; + }; + const scale_row0 = c.lane.shrLogical(std.math.log2_int(u32, @intCast(scale_chunks))); + const part16 = c.lane.bitAnd(scale_chunks - 1).mul(16); + const srow = ldSharedBatch(b, scale_steps, 1, idx.add(scale_row0.mul(4)), sidx_offsets); + const second_warp = c.warp.eq(6); + for (0..@intCast(scale_steps)) |k| { + // Warp 5 copies the even scale steps, warp 6 the odd ones. + var mine = b.openIf(if (k % 2 == 1) second_warp else c.warp.eq(5)); + const row = scale_row0.add(@as(i32, @intCast(k)) * @divExact(32, scale_chunks)); + cpAsync16Row(b, sc.add(row.mul(288)).add(part16), scales, srow[k], @intCast(f.scaleBytes()), part16.to(.i64)); + mine.yieldThen(.{}); + } + b.cpAsyncMbarrierArrive(bar(b, c.s, .raw_full, slot_a), true); + if (f == .fp8) b.cpAsyncMbarrierArrive(bar(b, c.s, .raw_full, slot_b), true); +} + +// ---- dequant (warps 8-11) ------------------------------------------------------------ + +fn dequantRole(c0: Ctx) void { + const b = c0.b; + const tid = c0.tid.sub(256); + var loop = b.openFor(0, c0.ntok, 1, .{ b.cst(.i32, 0), b.cst(.i32, 0) }); + const c = forToken(c0, loop.iv, loop.carried[0], loop.carried[1], true); + releaseIndices(c); + { + var blocks = b.openFor(0, c.nwl, 1, .{}); + dequantBlock(c, .fp8, blocks.iv, tid); + blocks.yield(.{}); + } + if (c.cfg.compressed_capacity > 0) { + var blocks = b.openFor(c.nwl, c.n, 1, .{}); + dequantBlock(c, .fp4, blocks.iv, tid); + blocks.yield(.{}); + } + loop.yield(.{ c.g0.add(c.n), slotUse(c, c.n) }); +} + +/// Thread (g = tid / 8, i = tid % 8) converts 8 consecutive values per step: +/// rows g + 16 * r (r < 4), columns 64 * step + 8 * i. The 8 threads of a row +/// write one 128-byte swizzle row, so each STS.128 is conflict free. The tile is +/// produced in four parts of 128 columns (steps 2p, 2p + 1): an FP8 block's first +/// raw slot (columns 0-255) is released after part 1, so the next block's copies +/// can start while parts 2-3 are converted. +fn dequantBlock(c: Ctx, comptime f: Format, j: V, tid: V) void { + const b = c.b; + const gj = c.g0.add(j); + const buf = gj.bitAnd(1); + const par = parity(gj); + const use_a = slotUse(c, j); + const use_b = use_a.add(1); + b.mbarrierWait(bar(b, c.s, .raw_full, use_a.rem(rawSlots(c.cfg))), slotParity(c, use_a)); + if (f == .fp8) b.mbarrierWait(bar(b, c.s, .raw_full, use_b.rem(rawSlots(c.cfg))), slotParity(c, use_b)); + { + var reuse = b.openIf(gj.ge(2)); + b.mbarrierWait(bar(b, c.s, .pv_done, buf), par.bitXor(1)); + reuse.yieldThen(.{}); + } + const g8 = tid.shrLogical(3); + const r8 = g8.bitAnd(7); + const in8 = tid.bitAnd(7); + const swz = in8.bitXor(r8).shl(4); + const raw = slotAddr(c, use_a).add(g8.mul(288)); + const raw_b = slotAddr(c, use_b).add(g8.mul(288)); + // FP8 scales follow the second slot's data (the first slot is released early). + const sc = scaleAddr(c, if (f == .fp8) use_b else use_a).add(g8.mul(288)); + const tile = c.s.add(b.select(buf.eq(0), b.cst(.i32, smem.tile[0]), b.cst(.i32, smem.tile[1]))); + const dst = tile.add(g8.shrLogical(3).mul(1024)).add(r8.mul(128)).add(swz); + // Raw words of part p for the thread's 4 rows (row stride 16 * 288 bytes): + // FP8: 2 x 2 words per row, then one scale word per row; + // FP4: 2 words per row, then 2 scale words per row. + const nw = if (f == .fp8) 20 else 16; + const loadPart = struct { + fn f_(cc: Ctx, comptime ff: Format, raw_: V, raw_b_: V, sc_: V, in8_: V, p: V) [nw]V { + const bb = cc.b; + var out: [nw]V = undefined; + if (ff == .fp8) { + // Steps 2p, 2p + 1: bytes [128 (p & 1), + 128) of the first (p < 2) or second slot. + const src = bb.select(p.lt(2), raw_, raw_b_).add(in8_.mul(8)).add(p.bitAnd(1).mul(128)); + const d = ldSharedBatch(bb, 8, 2, src, .{ 0, 64, 4608, 4672, 9216, 9280, 13824, 13888 }); + for (0..16) |i| out[i] = d[i]; + const sw = ldSharedBatch(bb, 4, 1, sc_.add(p.mul(4)), .{ 0, 4608, 9216, 13824 }); + for (0..4) |i| out[16 + i] = sw[i]; + } else { + const d = ldSharedBatch(bb, 8, 1, raw_.add(in8_.mul(4)).add(p.mul(64)), .{ 0, 32, 4608, 4640, 9216, 9248, 13824, 13856 }); + for (0..8) |i| out[i] = d[i]; + const sw = ldSharedBatch(bb, 4, 2, sc_.add(p.mul(8)), .{ 0, 4608, 9216, 13824 }); + for (0..8) |i| out[8 + i] = sw[i]; + } + return out; + } + }.f_; + const first = loadPart(c, f, raw, raw_b, sc, in8, b.cst(.i32, 0)); + if (f == .fp4) { + // Unrolled: the four parts of an FP4 block are short. + var cur = first; + for (0..4) |pi| { + const p = b.cst(.i32, @as(i32, @intCast(pi))); + const next = if (pi < 3) loadPart(c, f, raw, raw_b, sc, in8, p.add(1)) else cur; + dequantPart(c, f, cur, dst, in8, p); + cur = next; + } + } else { + var parts = b.openFor(0, b.opaqueI32(4), 1, tupleOf(nw, first)); + const p = parts.iv; + var cur: [nw]V = undefined; + for (0..nw) |i| cur[i] = parts.carried[i]; + // The last iteration re-loads its own part (harmless) to keep the loop uniform. + const next = loadPart(c, f, raw, raw_b, sc, in8, p.add(1).minimum(3)); + dequantPart(c, f, cur, dst, in8, p); + // Parts 0-1 were the first slot's last reads (part 2's loads read the second slot). + var first_done = b.openIf(p.eq(1)); + b.mbarrierArrive(bar(b, c.s, .raw_empty, use_a.rem(rawSlots(c.cfg))), 1); + first_done.yieldThen(.{}); + parts.yield(tupleOf(nw, next)); + } + b.fenceProxyAsyncShared(); + b.mbarrierArrive(bar(b, c.s, .tile_full, buf), 1); + b.mbarrierArrive(bar(b, c.s, .raw_empty, (if (f == .fp8) use_b else use_a).rem(rawSlots(c.cfg))), 1); +} + +/// Converts part `p` (raw words `cur`, see `dequantBlock`) into the bf16 tile at `dst`. +fn dequantPart(c: Ctx, comptime f: Format, cur: [if (f == .fp8) 20 else 16]V, dst: V, in8: V, p: V) void { + const b = c.b; + const dst_p = dst.add(p.mul(2 * BK * 128)); + // Invalid rows arrive zero-filled (raw bytes and scales), so they dequantize to 0. + for (0..4) |r| { + for (0..2) |k| { + const out = if (f == .fp8) blk: { + // Scale index 2 * step + (i >= 4) = 4 p + 2 k + (i >> 2): byte 2 k + (i >> 2) of word p. + const e = cur[16 + r].shrLogical(in8.shrLogical(2).add(@as(i32, @intCast(2 * k))).mul(8)).bitAnd(0xFF); + const scale = e.shl(7).bitOr(e.shl(23)); + break :blk fp8x8ToBf16x2x4Scaled(b, cur[r * 4 + k * 2], cur[r * 4 + k * 2 + 1], scale); + } else blk: { + // Scale index 4 * step + i / 2: byte i / 2 of the step's word. + const e = cur[8 + r * 2 + k].shrLogical(in8.shrLogical(1).mul(8)).bitAnd(0xFF); + const scale = b.e4m3x2ToBf16x2(e.bitOr(e.shl(8))); + break :blk fp4x8ToBf16x2x4Scaled(b, cur[r * 2 + k], scale); + }; + b.storeWords(4, dst_p.add(@as(i32, @intCast(r * 2048 + k * BK * 128))), .shared, out); + } + } +} + +// ---- MMA (warp 4) ------------------------------------------------------------------------ + +fn mmaRole(c0: Ctx) void { + const b = c0.b; + var leader = b.openIf(b.electSync()); + // Base descriptors; an MMA adds a constant (byte offset >> 4) to the start address field. + const q_descs = [2]V{ smemDescSw128(b, c0.s.add(smem.q[0]), 16, 1024), smemDescSw128(b, c0.s.add(smem.q[1]), 16, 1024) }; + const k_desc = [2]V{ smemDescSw128(b, c0.s.add(smem.tile[0]), 16, 1024), smemDescSw128(b, c0.s.add(smem.tile[1]), 16, 1024) }; + const v_desc = [2]V{ smemDescSw128(b, c0.s.add(smem.tile[0]), 8192, 1024), smemDescSw128(b, c0.s.add(smem.tile[1]), 8192, 1024) }; + const p_desc = [2]V{ smemDescSw128(b, c0.s.add(smem.p[0]), 16, 1024), smemDescSw128(b, c0.s.add(smem.p[1]), 16, 1024) }; + const qk_idesc = b.cst(.i32, @as(i32, @bitCast(instrDescBf16(64, H, .k, .k)))); + const pv_idesc = b.cst(.i32, @as(i32, @bitCast(instrDescBf16(128, H, .mn, .k)))); + const i32t = cute.DType.i32.toMlir(b.ctx); + + var tokens = b.openFor(0, c0.ntok, 1, .{b.cst(.i32, 0)}); + const c = forToken(c0, tokens.iv, tokens.carried[0], undefined, true); + // The MMA thread only needs the block count; the elected lane releases for the warp. + b.mbarrierArrive(bar(b, c.s, .idx_free, c.kb), 1); + b.mbarrierWait(bar(b, c.s, .q_ready, c.kb), parity(c.k)); + const q_desc = b.select(c.kb.eq(0), q_descs[0], q_descs[1]); + { + // O^T buffer kb: its previous token's epilogue must have read it. + var reuse = b.openIf(c.k.ge(2)); + b.mbarrierWait(bar(b, c.s, .o_free, c.kb), parity(c.k).bitXor(1)); + reuse.yieldThen(.{}); + } + b.tcgen05Fence(.after); + { + // No QK: Q is free right away. + var none = b.openIf(c.n.eq(0)); + b.tcgen05Commit(bar(b, c.s, .q_free, c.kb)); + none.yieldThen(.{}); + } + const o_taddr = c.taddr.add(c.kb.mul(4 * H)); + // Iteration i issues QK(i + 1) and PV(i) in whichever order their inputs + // become ready; i = -1 only issues QK(0). + var loop = b.openFor(-1, c.n, 1, .{}); + const i = loop.iv; + const next = i.add(1); + var poll = b.openWhile(.{ next.lt(c.n).to(.i32).bitAnd(1), i.ge(0).to(.i32).bitAnd(1) }, .{ i32t, i32t }); + poll.yieldBefore(poll.before_carried[0].bitOr(poll.before_carried[1]).ne(0), .{ poll.before_carried[0], poll.before_carried[1] }); + const qk_pending = poll.after_carried[0]; + const pv_pending = poll.after_carried[1]; + const gn = c.g0.add(next); + const nbuf = gn.bitAnd(1); + const qk_go = qk_pending.ne(0).bitAnd(b.mbarrierTestParity(bar(b, c.s, .tile_full, nbuf), parity(gn))); + { + var go = b.openIf(qk_go); + b.tcgen05Fence(.after); + const kd = b.select(nbuf.eq(0), k_desc[0], k_desc[1]); + const d = c.taddr.add(tmem_s).add(nbuf.mul(qk_parts * H)); + // K step s of part p covers latent dims 128 p + 16 s: consecutive MMAs go to + // different accumulators. + for (0..8) |step| { + for (0..qk_parts) |part| { + const ks = part * 8 + step; + const kb: i64 = @intCast(ks / 4); + const kin: i64 = @intCast(ks % 4); + const adesc = kd.add(@divExact(kb * 8192 + kin * 32, 16)); + const bdesc = q_desc.add(@divExact(kb * 2048 + kin * 32, 16)); + b.tcgen05MmaF16(d.add(@as(i32, @intCast(part * H))), adesc, bdesc, qk_idesc, b.cst(.i32, @intFromBool(step > 0)).ne(0)); + } + } + b.tcgen05Commit(bar(b, c.s, .s_full, nbuf)); + { + // The token's last QK: Q may be replaced once it completes. + var last = b.openIf(next.eq(c.n.sub(1))); + b.tcgen05Commit(bar(b, c.s, .q_free, c.kb)); + last.yieldThen(.{}); + } + go.yieldThen(.{}); + } + const gi = c.g0.add(i); + const buf = gi.bitAnd(1); + const pv_go = pv_pending.ne(0).bitAnd(b.mbarrierTestParity(bar(b, c.s, .p_full, buf), parity(gi))); + { + var go = b.openIf(pv_go); + b.tcgen05Fence(.after); + const vd = b.select(buf.eq(0), v_desc[0], v_desc[1]); + const pd = b.select(buf.eq(0), p_desc[0], p_desc[1]); + const first = b.select(i.eq(0), b.cst(.i32, 0), b.cst(.i32, 1)); + for (0..4) |ks| { + for (0..4) |chunk| { + const adesc = vd.add(@as(i64, @intCast(@divExact(2 * chunk * 8192 + ks * 2048, 16)))); + const bdesc = pd.add(@as(i64, @intCast(@divExact(ks * 32, 16)))); + const acc = if (ks > 0) b.cst(.i32, 1) else first; + b.tcgen05MmaF16(o_taddr.add(@as(i32, @intCast(chunk * H))), adesc, bdesc, pv_idesc, acc.ne(0)); + } + } + b.tcgen05Commit(bar(b, c.s, .pv_done, buf)); + go.yieldThen(.{}); + } + poll.yieldAfter(.{ b.select(qk_go, b.cst(.i32, 0), qk_pending), b.select(pv_go, b.cst(.i32, 0), pv_pending) }); + loop.yield(.{}); + tokens.yield(.{c.g0.add(c.n)}); + leader.yieldThen(.{}); +} + +// ---- softmax / epilogue (warps 0-3) ------------------------------------------------------ + +fn softmaxRole(c0: Ctx) void { + const b = c0.b; + const scale_log2: f32 = @floatCast(c0.cfg.scale * log2e); + const lane_tmem = c0.taddr.add(c0.warp.mul(32).shl(16)); + const row = c0.warp.mul(16).add(c0.lane); // S^T row held by this lane (lanes 0-15) + const has_row = c0.lane.lt(16); + + { + // Prefetch the sink logits for the epilogue. + var h16 = b.openIf(c0.tid.lt(H)); + b.storeWords(1, c0.s.add(smem.sink).add(c0.tid.mul(4)), .shared, .{b.loadWords(1, c0.g.sink.add(c0.tid.mul(4).to(.i64)), .global, .{ .invariant = true })[0].bitCast(.f32).bitCast(.i32)}); + h16.yieldThen(.{}); + } + var tokens = b.openFor(0, c0.ntok, 1, .{b.cst(.i32, 0)}); + const c = forToken(c0, tokens.iv, tokens.carried[0], undefined, true); + const o_tmem = lane_tmem.add(c.kb.mul(4 * H)); + var init: [2 * H]V = undefined; + for (0..H) |h| { + // The running max only guards exp2 against overflow: it need not be exact, + // and starting at 0 lets typical first blocks skip the cross-warp max. Logits + // far below it just give small P (BF16 / F32 range), as with a stale max. + init[h] = b.cst(.f32, 0); + init[H + h] = b.cst(.f32, 0); + } + var loop = b.openFor(0, c.n, 1, tupleOf(2 * H, init)); + const j = loop.iv; // local block; token block is b0 + j + var m: [H]V = undefined; + var l: [H]V = undefined; + for (0..H) |h| { + m[h] = loop.carried[h]; + l[h] = loop.carried[H + h]; + } + const gj = c.g0.add(j); + const buf = gj.bitAnd(1); + const par = parity(gj); + b.mbarrierWait(bar(b, c.s, .s_full, buf), par); + b.tcgen05Fence(.after); + const sparts = b.tmemLoad32x32b(qk_parts * H, lane_tmem.add(tmem_s).add(buf.mul(qk_parts * H))); + b.fenceTmemLoad(); + var sv: [H]V = undefined; + for (0..H) |h| { + var acc = sparts[h].bitCast(.f32); + for (1..qk_parts) |p_| acc = acc.add(sparts[p_ * H + h].bitCast(.f32)); + sv[h] = acc.bitCast(.i32); + } + // Candidate index of this row: window blocks first, then compressed blocks. + const tb = c.b0.add(j); + const is_window = tb.lt(c.nwb); + const cand = b.select(is_window, b.cst(.i32, idxBase(.fp8)).add(tb.mul(BK * 4)), b.cst(.i32, idxBase(.fp4)).add(tb.sub(c.nwb).mul(BK * 4))); + const row_ix = b.loadWords(1, c.ib.add(cand).add(row.bitAnd(BK - 1).mul(4)), .shared, .{})[0]; + const valid = has_row.bitAnd(row_ix.ge(0)); + + var sc: [H]V = undefined; + var exceed = b.cst(.i32, 0); + for (0..H) |h| { + sc[h] = b.select(valid, sv[h].bitCast(.f32).mul(scale_log2), b.cst(.f32, -std.math.inf(f32))); + exceed = exceed.bitOr(sc[h].gt(m[h].add(rescale_threshold)).to(.i32)); + } + // Usually no score exceeds the running max by the threshold: one barrier vote + // then skips the block max entirely (m stays, alpha = 1). + const f32t = cute.DType.f32.toMlir(b.ctx); + const i32t = cute.DType.i32.toMlir(b.ctx); + var upd = b.openIfElse(b.barrierReduceOr(1, 128, exceed.ne(0)), .{f32t} ** (2 * H) ++ .{i32t}); + { + // Block max per head: warp redux, then across the 4 warps through shared memory. + const red = c.s.add(b.select(buf.eq(0), b.cst(.i32, smem.red[0]), b.cst(.i32, smem.red[1]))); + for (0..H) |h| { + const wm = b.warpMaxF32(sc[h]); + var l0 = b.openIf(c.lane.eq(0)); + b.storeWords(1, red.add(c.warp.mul(H * 4)).add(@as(i32, @intCast(h * 4))), .shared, .{wm.bitCast(.i32)}); + l0.yieldThen(.{}); + } + b.barrierSync(1, 128); + const bms = blockReduce(c, red, .max); + var out: [2 * H + 1]V = undefined; + var changed = b.cst(.i32, 0); + for (0..H) |h| { + const bm = bms[h]; + const move = bm.gt(m[h].add(rescale_threshold)); + out[h] = b.select(move, bm, m[h]); + out[H + h] = b.exp2Approx(m[h].sub(out[h])); + changed = changed.bitOr(move.to(.i32)); + } + out[2 * H] = changed; + upd.yieldThen(tupleOf(2 * H + 1, out)); + } + { + var out: [2 * H + 1]V = undefined; + for (0..H) |h| { + out[h] = m[h]; + out[H + h] = b.cst(.f32, 1); + } + out[2 * H] = b.cst(.i32, 0); + upd.yieldElse(tupleOf(2 * H + 1, out)); + } + var new_m: [H]V = undefined; + var alpha: [H]V = undefined; + for (0..H) |h| { + new_m[h] = upd.results[h]; + alpha[h] = upd.results[H + h]; + } + const changed = upd.results[2 * H]; + // P^T [16 heads, 64 keys] BF16, SW128 K-major (one 128-byte row per head). + const p = c.s.add(b.select(buf.eq(0), b.cst(.i32, smem.p[0]), b.cst(.i32, smem.p[1]))); + var next_l: [H]V = undefined; + for (0..H) |h| { + const pr = b.select(valid, b.exp2Approx(sc[h].sub(new_m[h])), b.cst(.f32, 0)); + next_l[h] = l[h].mul(alpha[h]).add(pr); + const off = row.shrLogical(3).bitXor(@as(i32, @intCast(h & 7))).shl(4).add(row.bitAnd(7).shl(1)); + stSharedBf16If(b, p.add(@as(i32, @intCast(h * 128))).add(off), pr, has_row); + } + // Rescale O^T (TMEM) once the previous PV has landed. + { + var rescale = b.openIf(j.gt(0).bitAnd(changed.ne(0))); + const prev = gj.sub(1); + b.mbarrierWait(bar(b, c.s, .pv_done, prev.bitAnd(1)), parity(prev)); + b.tcgen05Fence(.after); + for (0..4) |chunk| { + const addr = o_tmem.add(@as(i32, @intCast(chunk * H))); + const o = b.tmemLoad32x32b(H, addr); + b.fenceTmemLoad(); + var scaled: [H]V = undefined; + for (0..H) |h| scaled[h] = o[h].bitCast(.f32).mul(alpha[h]).bitCast(.i32); + b.tmemStore32x32b(H, addr, scaled); + } + b.fenceTmemStore(); + rescale.yieldThen(.{}); + } + b.fenceProxyAsyncShared(); + b.tcgen05Fence(.before); + b.mbarrierArrive(bar(b, c.s, .p_full, buf), 1); + var carry: [2 * H]V = undefined; + for (0..H) |h| { + carry[h] = new_m[h]; + carry[H + h] = next_l[h]; + } + loop.yield(tupleOf(2 * H, carry)); + releaseIndices(c); + epilogue(c, o_tmem, loop.results[0..H].*, loop.results[H .. 2 * H].*); + tokens.yield(.{c.g0.add(c.n)}); +} + +/// Per head, the reduction over the 4 softmax warps of `buf` [4 warps][16 heads] f32. +fn blockReduce(c: Ctx, buf: V, comptime op: ReduceOp) [H]V { + const b = c.b; + var r: [H]V = undefined; + for (0..4) |w| { + for (0..4) |q4| { + const v = b.loadWords(4, buf.add(@as(i32, @intCast((w * H + q4 * 4) * 4))), .shared, .{}); + for (0..4) |i| { + const x = v[i].bitCast(.f32); + const h = q4 * 4 + i; + r[h] = if (w == 0) x else if (op == .max) r[h].maxnum(x) else r[h].add(x); + } + } + } + return r; +} + +/// O^T / (l + exp(sink - m)) of the token (or this CTA's partial result, for a cluster merge). +fn epilogue(c: Ctx, o_tmem: V, m: [H]V, l: [H]V) void { + const b = c.b; + const lane_tmem = c.taddr.add(c.warp.mul(32).shl(16)); + // The sums overlap the token's last PV. + const red_sum = c.s.add(b.select(c.kb.eq(0), b.cst(.i32, smem.red_sum[0]), b.cst(.i32, smem.red_sum[1]))); + { + // Lanes 16-31 hold no key rows (l = 0); lane h (< 16) gets head h's sum. + for (0..H) |h| { + const lsum = warpSumF32(b, l[h]); + var l0 = b.openIf(c.lane.eq(0)); + b.storeWords(1, red_sum.add(c.warp.mul(H * 4)).add(@as(i32, @intCast(h * 4))), .shared, .{lsum.bitCast(.i32)}); + l0.yieldThen(.{}); + } + } + b.barrierSync(1, 128); + const total = blockReduce(c, red_sum, .sum); + const has_blocks = c.n.gt(0); + { + var any = b.openIf(has_blocks); + const last = c.g0.add(c.n).sub(1); + b.mbarrierWait(bar(b, c.s, .pv_done, last.bitAnd(1)), parity(last)); + any.yieldThen(.{}); + } + b.tcgen05Fence(.after); + if (c.cfg.cluster > 1) { + // Partial result for the cluster merge: (m, l) per head and unnormalized O^T. + { + var t0 = b.openIf(c.tid.eq(0)); + for (0..H) |h| { + b.storeWords(1, c.s.add(smem.stats + @as(i32, @intCast(h * 4))), .shared, .{m[h].bitCast(.i32)}); + b.storeWords(1, c.s.add(smem.stats + @as(i32, @intCast((H + h) * 4))), .shared, .{total[h].bitCast(.i32)}); + } + t0.yieldThen(.{}); + } + // Normalized by this CTA's own sum (a convex combination of cache rows: bounded, + // so it travels as FP16); the owner re-weights with exp2(m - M) * l / L. + var inv_l: [H]V = undefined; + for (0..H) |h| inv_l[h] = b.select(total[h].gt(0).bitAnd(has_blocks), b.rcpApprox(total[h]), b.cst(.f32, 0)); + for (0..4) |chunk| { + const o = b.tmemLoad32x32b(H, o_tmem.add(@as(i32, @intCast(chunk * H)))); + b.fenceTmemLoad(); + const dim = c.warp.mul(32).add(c.lane).add(@as(i32, @intCast(chunk * 128))); + const dst = c.s.add(smem.obuf).add(dim.mul(H * 4)); + for (0..4) |q4| { + var v: [4]V = undefined; + // A rank without key blocks never wrote its O^T (TMEM holds stale data, possibly + // NaN/Inf): select, do not multiply by 0. + for (0..4) |i| v[i] = b.select(has_blocks, o[q4 * 4 + i].bitCast(.f32).mul(inv_l[q4 * 4 + i]), b.cst(.f32, 0)).bitCast(.i32); + b.storeWords(4, dst.add(@as(i32, @intCast(q4 * 16))), .shared, v); + } + } + _ = lane_tmem; + } else { + var inv: [H]V = undefined; + for (0..H) |h| { + const sink = b.loadWords(1, c.s.add(smem.sink + @as(i32, @intCast(h * 4))), .shared, .{})[0].bitCast(.f32); + const denom = total[h].add(b.exp2Approx(sink.mul(@as(f32, log2e)).sub(m[h]))); + // denom underflows only if every logit and the sink are ~2^126 below the running max. + inv[h] = b.select(has_blocks.bitAnd(denom.gt(0)), b.rcpApprox(denom), b.cst(.f32, 0)); + } + // The CTA's last token is staged in shared memory (Q is no longer needed) and + // written with 16-byte stores; earlier tokens store straight to global memory + // (a warp writes 64 contiguous bytes per head). + const last_token = c.k.eq(c.ntok.sub(1)); + const out_row = c.g.out.add(c.t.to(.i64).mul(H * D * 2)); + { + var direct = b.openIf(last_token.eq(false)); + var os: [4][H]V = undefined; + for (0..4) |chunk| os[chunk] = b.tmemLoad32x32b(H, o_tmem.add(@as(i32, @intCast(chunk * H)))); + b.fenceTmemLoad(); + // Lane pairs (d, d + 1) swap half their heads: the even lane stores heads 0-7, + // the odd lane heads 8-15, each as bf16 pairs of dims (d, d + 1). + const odd = c.lane.bitAnd(1).ne(0); + const d_even = c.warp.mul(32).add(c.lane.bitAnd(30)); + const h0 = b.select(odd, b.cst(.i32, 8), b.cst(.i32, 0)); + const base = out_row.add(h0.mul(D).add(d_even).mul(2).to(.i64)); + for (0..4) |chunk| { + for (0..8) |i| { + const lo = b.select(has_blocks, os[chunk][i].bitCast(.f32).mul(inv[i]), b.cst(.f32, 0)); + const hi = b.select(has_blocks, os[chunk][8 + i].bitCast(.f32).mul(inv[8 + i]), b.cst(.f32, 0)); + const send = b.select(odd, lo, hi); + const keep = b.select(odd, hi, lo); + const recv = b.shuffleXor(send, 1).bitCast(.f32); + // Even lane: (own dim, partner's dim); odd lane: (partner's dim, own dim). + const pair = b.packHalf2(b.select(odd, recv, keep), b.select(odd, keep, recv), .bf16); + b.storeWords(1, base.add(@as(i64, @intCast((i * D + chunk * 128) * 2))), .global, .{pair}); + } + } + direct.yieldThen(.{}); + } + { + var staged = b.openIf(last_token); + // The staging buffer is Q buffer 0: a token without key blocks never waited for + // its Q copy (cp.async into that buffer), which must land before it is overwritten. + b.mbarrierWait(bar(b, c.s, .q_ready, c.kb), parity(c.k)); + var os: [4][H]V = undefined; + for (0..4) |chunk| os[chunk] = b.tmemLoad32x32b(H, o_tmem.add(@as(i32, @intCast(chunk * H)))); + b.fenceTmemLoad(); + for (0..4) |chunk| { + const o = os[chunk]; + const dim = c.warp.mul(32).add(c.lane).add(@as(i32, @intCast(chunk * 128))); + for (0..H) |h| { + const v = b.select(has_blocks, o[h].bitCast(.f32).mul(inv[h]), b.cst(.f32, 0)); + b.storeHalf(c.s.add(smem.ostage).add(dim.add(@as(i32, @intCast(h * D))).mul(2)), .shared, b.f32ToBf16Bits(v)); + } + } + staged.yieldThen(.{}); + } + // O^T buffer kb may take the token after next. + b.tcgen05Fence(.before); + b.mbarrierArrive(bar(b, c.s, .o_free, c.kb), 1); + { + var staged = b.openIf(last_token); + b.barrierSync(1, 128); + for (0..8) |it| { + const off = c.tid.add(@as(i32, @intCast(it * 128))).mul(16); + b.storeWords(4, out_row.add(off.to(.i64)), .global, b.loadWords(4, c.s.add(smem.ostage).add(off), .shared, .{})); + } + staged.yieldThen(.{}); + } + } +} + +// ---- cluster merge (all 384 threads, cluster > 1) ---------------------------------------------- + +/// Combine the cluster's partial results. Rank r owns output dims +/// [r * 512 / C, (r + 1) * 512 / C): after every rank has finished its blocks, +/// each pushes its (m, l) and the owners' slices of its O^T into their shared +/// memory (remote stores do not stall), then every owner combines locally. +/// No global partials and no second kernel. +fn clusterMerge(c: Ctx) void { + const b = c.b; + const C: i32 = @intCast(c.cfg.cluster); + const dims = @divExact(D, C); + // Every rank is done with its tiles (the inbox reuses tile 0). + b.clusterSync(); + { + // (m, l) of head h to every rank's inbox, slot `rank`. + var heads_ = b.openIf(c.tid.lt(2 * H)); + const v = b.loadWords(1, c.s.add(smem.stats).add(c.tid.mul(4)), .shared, .{})[0].bitCast(.f32); + const dst = c.s.add(smem.stats_in).add(c.rank.mul(2 * H * 4)).add(c.tid.mul(4)); + for (0..@intCast(C)) |p| b.storeWordsPtr(1, b.mapaShared(dst, b.cst(.i32, @as(i32, @intCast(p)))), .{v.bitCast(.i32)}); + heads_.yieldThen(.{}); + } + { + // O^T rows of peer p's dims to its inbox slot `rank`: 16-byte chunks. + const chunks = D * @divExact(H, 4); + var loop = b.openFor(c.tid, chunks, threads, .{}); + const chunk = loop.iv; + const d = chunk.shrLogical(2); + const q4 = chunk.bitAnd(3); + const v = b.loadWords(4, c.s.add(smem.obuf).add(d.mul(H * 4)).add(q4.mul(16)), .shared, .{}); + const owner = d.div(dims); + // Normalized partials travel as FP16. + const packed_ = [2]V{ b.packHalf2(v[0].bitCast(.f32), v[1].bitCast(.f32), .f16), b.packHalf2(v[2].bitCast(.f32), v[3].bitCast(.f32), .f16) }; + const dst = c.s.add(smem.inbox).add(c.rank.mul(dims * H * 2)).add(d.rem(dims).mul(H * 2)).add(q4.mul(8)); + b.storeWordsPtr(2, b.mapaShared(dst, owner), packed_); + loop.yield(.{}); + } + b.clusterSync(); + // Per-head merge weights exp2(m_p - M) / L, with L = sum_p exp2(m_p - M) l_p + exp2(sink - M). + { + var heads_ = b.openIf(c.tid.lt(H)); + const h = c.tid; + var ms: [16]V = undefined; + var ls: [16]V = undefined; + var mx = b.cst(.f32, @as(f32, -1e30)); + for (0..@intCast(C)) |p| { + const slot = c.s.add(smem.stats_in + @as(i32, @intCast(p * 2 * H * 4))).add(h.mul(4)); + ms[p] = b.loadWords(1, slot, .shared, .{})[0].bitCast(.f32); + ls[p] = b.loadWords(1, slot.add(H * 4), .shared, .{})[0].bitCast(.f32); + mx = mx.maxnum(ms[p]); + } + const sink = b.loadWords(1, c.s.add(smem.sink).add(h.mul(4)), .shared, .{})[0].bitCast(.f32); + var denom = b.exp2Approx(sink.mul(@as(f32, log2e)).sub(mx)); + var w: [16]V = undefined; + for (0..@intCast(C)) |p| { + w[p] = b.exp2Approx(ms[p].sub(mx)); + denom = denom.add(w[p].mul(ls[p])); + } + const inv = b.rcpApprox(denom); + // Partials are O_p / l_p: weight exp2(m_p - M) * l_p / L. + for (0..@intCast(C)) |p| b.storeWords(1, c.s.add(smem.weights + @as(i32, @intCast(p * H * 4))).add(h.mul(4)), .shared, .{w[p].mul(ls[p]).mul(inv).bitCast(.i32)}); + heads_.yieldThen(.{}); + } + b.barrierSync(0, threads); + // Output: this rank's dims x 16 heads, 4 heads (one 16-byte load per rank) per item. + const items = dims * @divExact(H, 4); + const out_row = c.g.out.add(c.t.to(.i64).mul(H * D * 2)); + { + var loop = b.openFor(c.tid, items, threads, .{}); + const item = loop.iv; + // Consecutive threads take consecutive dims so the output stores coalesce. + const dl = item.rem(dims); + const hq = item.div(dims); // head quad + var acc = [4]V{ b.cst(.f32, 0), b.cst(.f32, 0), b.cst(.f32, 0), b.cst(.f32, 0) }; + for (0..@intCast(C)) |p| { + const o2 = b.loadWords(2, c.s.add(smem.inbox + @as(i32, @intCast(p)) * dims * H * 2).add(dl.mul(H * 2)).add(hq.mul(8)), .shared, .{}); + const lo = b.unpackHalf2(o2[0], .f16); + const hi = b.unpackHalf2(o2[1], .f16); + const o = [4]V{ lo[0], lo[1], hi[0], hi[1] }; + const wv = b.loadWords(4, c.s.add(smem.weights + @as(i32, @intCast(p * H * 4))).add(hq.mul(16)), .shared, .{}); + for (0..4) |i| acc[i] = acc[i].add(o[i].mul(wv[i].bitCast(.f32))); + } + for (0..4) |i| { + const head = hq.mul(4).add(@as(i32, @intCast(i))); + b.storeHalf(c.s.add(smem.ostage).add(head.mul(dims).add(dl).mul(2)), .shared, b.f32ToBf16Bits(acc[i])); + } + loop.yield(.{}); + } + b.barrierSync(0, threads); + { + // This rank's slice of every head row: dims / 8 16-byte chunks per head. + const per_head = @divExact(dims, 8); + var loop = b.openFor(c.tid, H * per_head, threads, .{}); + const h = loop.iv.div(per_head); + const k = loop.iv.rem(per_head); + const v = b.loadWords(4, c.s.add(smem.ostage).add(h.mul(dims).add(k.mul(8)).mul(2)), .shared, .{}); + b.storeWords(4, out_row.add(h.mul(D).add(c.rank.mul(dims)).add(k.mul(8)).to(.i64).mul(2)), .global, v); + loop.yield(.{}); + } +} + +// ---- helpers ------------------------------------------------------------------------------ +// Addresses are raw integers: shared `i32` (shared window), global `i64`, TMEM `i32` +// (lane << 16 | column). + +/// `bytes` of compiler-visible dynamic shared memory (1024-aligned), as a shared address. +fn sharedStorage(b: *B, comptime bytes: i64) V { + const storage = b.allocSmemStorage(.i8, 1024, b.layoutType(b.layoutSpec(.{bytes}, .{1})), 0, &.{std.fmt.comptimePrint("storage:{d}:0", .{bytes})}); + const base = b.getIterTyped(storage, b.ptrTy(.i8, .smem, 1024) catch @panic("bad smem ptr")).value(); + return b.ptrToInt(base, .i32); +} + +/// 16-byte cp.async of `base + row * row_bytes + offset`; when `row < 0` it zero-fills +/// the destination without reading memory. +fn cpAsync16Row(b: *B, dst: V, base: V, row: V, comptime row_bytes: u32, offset: V) void { + const src = base.add(row.maximum(0).to(.i64).mul(@as(i64, row_bytes))).add(offset); + b.cpAsync16(dst, src, row.ge(0)); +} + +/// `n` shared loads of `words` 32-bit words each (n * words results), from `base + offsets[i]`. +fn ldSharedBatch(b: *B, comptime n: usize, comptime words: usize, base: V, comptime offsets: [n]i32) [n * words]V { + var out: [n * words]V = undefined; + for (0..n) |i| { + const w = b.loadWords(words, base.add(offsets[i]), .shared, .{}); + for (0..words) |k| out[i * words + k] = w[k]; + } + return out; +} + +/// Eight packed FP8 e4m3 (two words) -> four bf16x2, each multiplied by the bf16x2 `scale`. +fn fp8x8ToBf16x2x4Scaled(b: *B, lo: V, hi: V, scale: V) [4]V { + const pairs = [4]V{ lo, lo.shrLogical(16), hi, hi.shrLogical(16) }; + var out: [4]V = undefined; + for (pairs, &out) |pair, *o| o.* = b.mulBf16x2(b.e4m3x2ToBf16x2(pair), scale); + return out; +} + +/// Eight packed FP4 e2m1 (one word, even element in the low nibble) -> four bf16x2, +/// each multiplied by the bf16x2 `scale`. +fn fp4x8ToBf16x2x4Scaled(b: *B, word: V, scale: V) [4]V { + var out = b.e2m1x8ToBf16x2x4(word); + for (&out) |*o| o.* = b.mulBf16x2(o.*, scale); + return out; +} + +/// 16-bit shared store of the bf16 rounding of `v` (f32) when `pred` (i1). +fn stSharedBf16If(b: *B, addr: V, v: V, pred: V) void { + var only = b.openIf(pred); + b.storeHalf(addr, .shared, b.f32ToBf16Bits(v)); + only.yieldThen(.{}); +} + +const Major = enum(u1) { k = 0, mn = 1 }; + +/// kind::f16 instruction descriptor: BF16 A/B, F32 accumulator. +fn instrDescBf16(comptime m: u32, comptime n: u32, comptime a_major: Major, comptime b_major: Major) u32 { + std.debug.assert((m == 64 or m == 128) and n % 8 == 0 and n >= 8 and n <= 256); + return (1 << 4) | // c_format = F32 + (1 << 7) | // a_format = BF16 + (1 << 10) | // b_format = BF16 + (@as(u32, @intFromEnum(a_major)) << 15) | + (@as(u32, @intFromEnum(b_major)) << 16) | + ((n >> 3) << 17) | + ((m >> 4) << 24); +} + +/// SM100 shared-memory matrix descriptor with 128-byte swizzle. `lbo`/`sbo` in bytes. +fn smemDescSw128(b: *B, addr: V, comptime lbo: u32, comptime sbo: u32) V { + const hi: u64 = (@as(u64, sbo >> 4) << 32) | (@as(u64, 1) << 46) | (@as(u64, 2) << 61); + const lo: u64 = @as(u64, lbo >> 4) << 16; + const start = addr.to(.i64).shrLogical(4).bitAnd(0x3FFF); + return start.bitOr(b.cst(.i64, @as(i64, @bitCast(hi | lo)))); +} + +/// Byte offset of element (row, col) of a bf16 tile stored as SW128 K-major atoms. +fn sw128Offset(b: *B, row: V, col: V, comptime rows: i32) V { + _ = b; + const slab = col.shrLogical(6).mul(rows * 128); + const chunk = col.bitAnd(63).shrLogical(3).bitXor(row.bitAnd(7)); + return slab.add(row.mul(128)).add(chunk.shl(4)).add(col.bitAnd(7).shl(1)); +} + +fn warpSumF32(b: *B, v: V) V { + var x = v; + inline for (.{ 16, 8, 4, 2, 1 }) |o| x = x.add(b.shuffleXor(x, o)); + return x; +} + +const ReduceOp = enum { max, sum }; diff --git a/zml/attention/cute_kernels/sparse_mla.zig b/zml/attention/cute_kernels/sparse_mla.zig new file mode 100644 index 00000000..5328eeb3 --- /dev/null +++ b/zml/attention/cute_kernels/sparse_mla.zig @@ -0,0 +1,266 @@ +//! CuTe backend of sparse MLA (`paged_attention.Mla`, backend `.cute`) on Blackwell +//! (SM100/SM103): in-place sparse attention over DeepSeek-V4.1-style quantized caches. +//! +//! Q is BF16 [queries, heads, 512]. Keys and values are the same 512-wide latent +//! rows, selected per query from two quantized paged caches: a sliding window +//! cache (FP8 e4m3, E8M0 scale per 32 values) and a compressed cache (packed +//! FP4 e2m1, E4M3 scale per 16 values). The selected rows are read and +//! dequantized on chip (`flashmla_sm100.zig`): nothing is gathered in global +//! memory and there is no split-KV reduction. +//! +//! `pagedAttention` is the backend entry of `paged_attention.Mla.pagedSparseAttention`: +//! it resolves the selected positions to physical rows of the paged caches and runs the +//! kernel once per group of 16 heads. +const std = @import("std"); + +const zml = @import("../../zml.zig"); +const Mla = @import("../paged_attention.zig").Mla; +const Tensor = zml.Tensor; + +pub const Config = struct { + queries: i64, + window_capacity: i64, + compressed_capacity: i64, + window_slots: i64, + compressed_slots: i64, + heads: i64 = 16, + scale: f64 = 1.0 / std.math.sqrt(512.0), + /// CTAs per query token (power of two); null picks one from the SM count. + cluster: ?i64 = null, + + /// Enough CTAs to cover the SMs in a single wave, but no more than there + /// are key blocks (64 candidates each) and at most 8 (portable cluster size). + /// Clusters only co-schedule within a GPC, so large clusters do not fill + /// every SM: on GB300 (152 SMs, one 226 KiB CTA per SM) at most 14 clusters + /// of 8 (112 CTAs) and 36 clusters of 4 run at once (measured 2026-09-29). + /// Persistent CTAs (one per SM: one 226 KiB CTA fits) when tokens are not split; + /// larger batches and prefill loop over tokens inside the kernel. + pub fn persistentCtas(c: Config, sm_count: usize) i64 { + if (c.clusterSize(sm_count) > 1) return c.queries; + return @min(c.queries, @as(i64, @intCast(sm_count))); + } + + pub fn clusterSize(c: Config, sm_count: usize) i64 { + if (c.cluster) |n| return n; + // Window-only layers have at most two blocks: splitting them never pays for the merge. + if (c.compressed_capacity == 0) return 1; + const blocks = @divTrunc(c.window_capacity + 63, 64) + @divTrunc(c.compressed_capacity + 63, 64); + const sms: i64 = @intCast(sm_count); + var size: i64 = 8; + while (size > 1) : (size = @divExact(size, 2)) { + const resident = switch (size) { + 8 => @divTrunc(sms * 112, 152), + 4 => @divTrunc(sms * 144, 152), + else => sms, + }; + if (size <= blocks and c.queries * size <= resident) break; + } + return size; + } +}; + +/// Kernel inputs: flat cache rows and per-query physical row indices. +pub const Inputs = struct { + q: zml.Tensor, // bf16 [queries, heads, 512] + window_values: zml.Tensor, // u8 [window_slots, 512] e4m3 + window_scales: zml.Tensor, // u8 [window_slots, 16] e8m0 + compressed_values: zml.Tensor, // u8 [compressed_slots, 256] e2m1 x2 (even value in the low nibble) + compressed_scales: zml.Tensor, // u8 [compressed_slots, 32] e4m3 + window_indices: zml.Tensor, // i32 [queries, window_capacity], physical rows, <0 invalid + compressed_indices: zml.Tensor, // i32 [queries, compressed_capacity] + lengths: zml.Tensor, // i32 [queries, 2] scan bounds per stream + sink: zml.Tensor, // f32 [heads] per-head logit with zero value + active: zml.Tensor, // i32 [1] leading active query rows +}; + +const kernel = @import("flashmla_sm100.zig"); + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + const cc = zml.platform.cuda.computeCapability(platform) orelse return false; + return cc.major == 10; +} + +/// Whether the kernel supports these inputs: Blackwell, BF16 Q with a 512-wide latent +/// and a multiple of 16 heads (at most 64; they run in groups of 16, one `kernelCall` +/// each), and the DeepSeek-V4.1 cache formats without a global scale: FP8 window rows +/// with E8M0 scales per 32 values, a window of 128 rows, and packed FP4 compressed rows +/// with E4M3 scales per 16 values, at most 512 (a multiple of 128) per query. +pub fn supports(args: Mla.SparseAttentionArgs) bool { + if (!isAvailable(zml.Compiler.current().platform)) return false; + const q = args.q.shape(); + const heads = q.dim(.h); + if (q.dtype() != .bf16 or q.dim(.hd) != 512 or @rem(heads, 16) != 0 or heads > 64) return false; + + const window = switch (args.kv.cache) { + .quantized => |input| input, + .latent => return false, + }; + if (window.global_scale != null) return false; + if (window.values.dtype() != .f8e4m3fn or lastDim(window.values) != 512) return false; + if (window.scales.dtype() != .f8e8m0 or lastDim(window.scales) != 16) return false; + if (args.kv.positions.dim(.topk) != 128) return false; + + const compressed = args.compressed orelse return true; + const input = switch (compressed.cache) { + .quantized => |input| input, + .latent => return false, + }; + if (input.global_scale != null) return false; + if (input.values.dtype() != .f4e2m1 or lastDim(input.values) != 512) return false; + if (input.scales.dtype() != .f8e4m3fn or lastDim(input.scales) != 32) return false; + const capacity = compressed.positions.dim(.topk); + return @rem(capacity, 128) == 0 and capacity <= 512; +} + +fn lastDim(t: Tensor) i64 { + return t.dim(t.rank() - 1); +} + +/// One kernel launch over 16 heads (see `Config` and `Inputs`). +pub fn kernelCall(c: Config, a: Inputs) zml.Tensor { + // The kernel is compiled for these shapes (see `supports`); anything else would + // read the caches with the wrong layout. + std.debug.assert(c.heads == 16); + std.debug.assert(c.window_capacity == 128); + std.debug.assert(@rem(c.compressed_capacity, 128) == 0 and c.compressed_capacity <= 512); + std.debug.assert(a.q.dtype() == .bf16 and a.q.dim(1) == 16 and a.q.dim(2) == 512); + std.debug.assert(a.window_values.dim(1) == 512 and a.window_scales.dim(1) == 16); + std.debug.assert(a.compressed_values.dim(1) == 256 and a.compressed_scales.dim(1) == 32); + const out = zml.Shape.init(.{ c.queries, c.heads, 512 }, .bf16); + return kernel.Program.call(.{ + .q = a.q, + .wv = a.window_values, + .ws = a.window_scales, + .cv = a.compressed_values, + .cs = a.compressed_scales, + .wi = a.window_indices, + .ci = a.compressed_indices, + .lengths = a.lengths, + .sink = a.sink, + .active = a.active, + }, .{ .out = out }, .{ .cfg = .{ + .queries = c.queries, + .window_capacity = c.window_capacity, + .compressed_capacity = c.compressed_capacity, + .window_slots = c.window_slots, + .compressed_slots = c.compressed_slots, + .scale = c.scale, + .cluster = c.clusterSize(zml.attention.triton.getCuCount()), + .ctas = c.persistentCtas(zml.attention.triton.getCuCount()), + } }).out; +} + +/// Backend `.cute` of `paged_attention.Mla.pagedSparseAttention`: the selected rows of a +/// quantized window cache and optionally of a quantized compressed cache, read in place. +/// Panics on inputs the kernel does not support (see `supports`). Each stream is scanned +/// up to its last valid entry. +pub fn pagedAttention(args: Mla.SparseAttentionArgs, opts: Mla.Options) Tensor { + if (!supports(args)) std.debug.panic("CuTe sparse MLA does not support q {f} over a {s} cache of {} rows per query and {s}: use another backend", .{ + args.q.shape(), + @tagName(args.kv.cache), + args.kv.positions.dim(.topk), + if (args.compressed) |c| @tagName(c.cache) else "no compressed cache", + }); + const sink = args.sink orelse std.debug.panic("CuTe sparse MLA requires an attention sink", .{}); + if (opts.value_rank != args.q.dim(.hd)) std.debug.panic("CuTe sparse MLA requires value_rank ({}) == head dim ({})", .{ opts.value_rank, args.q.dim(.hd) }); + // Physical rows of the selected positions, from each selection's page table. + const active_count = args.kv.activeCount(); + const rows = args.kv.physicalRows(args.tokens_pos, active_count); + const window = args.kv.cache.quantized; + const head_dim: f64 = @floatFromInt(args.q.dim(.hd)); + var context: ShardContext = .{ + .q = args.q.rename(.{ .q = .b }), + .window_values = window.values, + .window_scales = window.scales, + .window_indices = rows, + .compressed_values = null, + .compressed_scales = null, + // Window-only: a placeholder, not read (compressed capacity 0). + .compressed_indices = rows, + .compressed_capacity = 0, + .sink = sink, + .active_count = active_count, + .scale = if (opts.scale) |scale| scale else 1 / @sqrt(head_dim), + }; + if (args.compressed) |c| { + const compressed_rows = c.physicalRows(args.tokens_pos, active_count); + context.compressed_values = c.cache.quantized.values; + context.compressed_scales = c.cache.quantized.scales; + context.compressed_indices = compressed_rows; + context.compressed_capacity = compressed_rows.dim(.topk); + } + return zml.ops.manualComputation(ShardContext.run, context, args.q.shape().rename(.{ .q = .b })).rename(.{ .b = .q }); +} + +/// Per-device part: this shard's heads, run in groups of 16 (a TP4 shard has exactly 16). +const ShardContext = struct { + q: Tensor, + window_values: Tensor, + window_scales: Tensor, + window_indices: Tensor, + compressed_values: ?Tensor, // null for window-only layers + compressed_scales: ?Tensor, + compressed_indices: Tensor, + compressed_capacity: i64, + sink: Tensor, + active_count: Tensor, + scale: f64, + + fn run(self: ShardContext, _: zml.Shape) Tensor { + const q = self.q; + const heads = q.dim(.h); + const queries = q.dim(.b); + std.debug.assert(@rem(heads, 16) == 0); + const window_rows: i64 = @intCast(self.window_values.shape().count() / 512); + const compressed_rows: i64 = if (self.compressed_values) |v| @intCast(v.shape().count() / 512) else 1; + // Window-only layers: one zero row as a placeholder (compressed capacity 0, never read). + const compressed_values = if (self.compressed_values) |v| v.reshape(.{ compressed_rows, 256, 2 }).bitCast(.u8) else Tensor.zeroes(zml.Shape.init(.{ 1, 256 }, .u8)); + const compressed_scales = if (self.compressed_scales) |sc| sc.bitCast(.u8).reshape(.{ compressed_rows, 32 }) else Tensor.zeroes(zml.Shape.init(.{ 1, 32 }, .u8)); + const window_capacity = self.window_indices.dim(.topk); + const window_indices = self.window_indices.convert(.i32); + const compressed_indices = if (self.compressed_capacity > 0) + self.compressed_indices.convert(.i32) + else + Tensor.scalar(-1, .i32).broad(zml.Shape.init(.{ .q = queries, .topk = 0 }, .i32)); + const lengths = Tensor.concatenate(&.{ + scanLength(window_indices).insertAxes(.q, .{.stream}), + scanLength(compressed_indices).insertAxes(.q, .{.stream}), + }, .stream).transpose(.{ .q, .stream }).reshape(.{ queries, 2 }); + const cfg: Config = .{ + .queries = queries, + .window_capacity = window_capacity, + .compressed_capacity = self.compressed_capacity, + .window_slots = window_rows, + .compressed_slots = compressed_rows, + .scale = self.scale, + }; + const q_all = q.reshape(.{ queries, heads, 512 }); + const sink_all = self.sink.convert(.f32).reshape(.{heads}); + var outs: [4]Tensor = undefined; + const groups: usize = @intCast(@divExact(heads, 16)); + for (0..groups) |g| { + const start: i64 = @intCast(16 * g); + outs[g] = kernelCall(cfg, .{ + .q = if (groups == 1) q_all else q_all.slice(1, .{ .start = start, .end = start + 16 }), + .window_values = self.window_values.bitCast(.u8).reshape(.{ window_rows, 512 }), + .window_scales = self.window_scales.bitCast(.u8).reshape(.{ window_rows, 16 }), + .compressed_values = compressed_values, + .compressed_scales = compressed_scales, + .window_indices = window_indices.reshape(.{ queries, window_capacity }), + .compressed_indices = compressed_indices.reshape(.{ queries, self.compressed_capacity }), + .lengths = lengths, + .sink = if (groups == 1) sink_all else sink_all.slice(0, .{ .start = start, .end = start + 16 }), + .active = self.active_count.convert(.i32).reshape(.{1}), + }); + } + const out = if (groups == 1) outs[0] else Tensor.concatenate(outs[0..groups], 1); + return out.withTags(.{ .b, .h, .hd }).withTags(q.shape()); + } +}; + +/// Rows to scan per query: through the last valid (>= 0) entry ([.q] i32). +fn scanLength(indices: Tensor) Tensor { + return Tensor.iota(indices.shape(), .topk).addConstant(1) + .mask(indices.cmp(.GE, .scalar(0, .i32)), 0).max(.topk).squeeze(.topk); +} diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 818a19b3..eda324a9 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -5,6 +5,7 @@ const stdx = @import("stdx"); const zml = @import("../zml.zig"); const flashattn = @import("flashattn.zig"); const metal = @import("metal_attention.zig"); +const cute = @import("cute_kernels/sparse_mla.zig"); const sparse_mla = @import("sparse_mla.zig"); const tpu = @import("tpu_attention.zig"); const triton = @import("triton_attention.zig"); @@ -866,14 +867,111 @@ pub const Mla = struct { return attn_weights_non_sink.dot(selected_values, .kv).convert(q.dtype()); } - /// Computes sparse MLA scores over the complete cached key and returns - /// `opts.value_rank` output dimensions per head. - pub fn pagedSparseAttention(parameters: Parameters, q: zml.Tensor, kv_cache: KvCache, sink: ?zml.Tensor, topk: zml.Tensor, tokens_pos: zml.Tensor, opts: Mla.Options) zml.Tensor { - const latent_kv = switch (kv_cache) { - .latent => |latent_kv| latent_kv, - else => std.debug.panic("Sparse Multi-Latent Attention support only latent KV pages, got: {}", .{std.meta.activeTag(kv_cache)}), - }; + /// Rows of a paged sparse MLA cache, as stored + pub const Cache = union(enum) { + latent: zml.Tensor, + quantized: zml.quantization.QuantizedInput, + fn rowsPerPage(self: Cache) i64 { + return switch (self) { + .latent => |t| t.dim(.k_chunk), + .quantized => |input| input.values.dim(.k_chunk), + }; + } + }; + + /// The rows each query attends in a paged cache + pub const CacheSelection = struct { + cache: Cache, + /// Page table of this cache (block table, sequence lengths, query offsets) + parameters: Parameters, + /// The logical token positions to attend, -1 or a + /// position outside the sequence selects nothing + positions: zml.Tensor, + /// Ratio of tokens per row + compression_ratio: u32 = 1, + + /// Number of leading active query rows, from the page table + pub fn activeCount(self: CacheSelection) zml.Tensor { + return switch (self.parameters) { + .triton, .stablehlo => |p| p.query_start_len.slice(.b, .single(p.query_start_len.dim(.b) - 1)), + else => std.debug.panic("sparse MLA does not support {} parameters", .{std.meta.activeTag(self.parameters)}), + }; + } + + /// Flat physical row (page * rows per page + slot) of each selected position + /// -1 for none and for queries past `active_count` + pub fn physicalRows(self: CacheSelection, tokens_pos: zml.Tensor, active_count: zml.Tensor) zml.Tensor { + const page_tokens = self.cache.rowsPerPage() * self.compression_ratio; + const tokens = switch (self.parameters) { + .triton, .stablehlo => |parameters| triton.paged.topkToPhysical(parameters, self.positions, tokens_pos, page_tokens), + else => std.debug.panic("sparse MLA does not support {} parameters", .{std.meta.activeTag(self.parameters)}), + }; + const active = zml.Tensor.iota(tokens.shape(), .q).cmp(.LT, active_count); + const valid = tokens.cmp(.GE, .scalar(0, .i32)).logical(.AND, active); + return tokens.divByConst(self.compression_ratio).mask(valid, -1); + } + }; + + /// Inputs of `pagedSparseAttention`. + pub const SparseAttentionArgs = struct { + q: zml.Tensor, + kv: CacheSelection, + compressed: ?CacheSelection = null, + sink: ?zml.Tensor, + tokens_pos: zml.Tensor, + }; + + pub fn pagedSparseAttention(args: SparseAttentionArgs, opts: Mla.Options) zml.Tensor { + // Switch there because backends read input rows differently + return switch (opts.backend) { + .cute => cute.pagedAttention(args, opts), + .triton, .fly => latentAttention(args, opts), + }; + } + + fn latentAttention(args: SparseAttentionArgs, opts: Mla.Options) zml.Tensor { + const active_count = args.kv.activeCount(); + const rows = args.kv.physicalRows(args.tokens_pos, active_count); + const kv, const selected = if (args.compressed == null and args.kv.cache == .latent) + .{ args.kv.cache.latent, rows } + else blk: { + // These steps are made inside the cute kernel + var latent = gatherRows(args.kv.cache, rows, args.q.dtype()); + var valid = rows.cmp(.GE, .scalar(0, .i32)); + if (args.compressed) |c| { + const compressed_rows = c.physicalRows(args.tokens_pos, active_count); + latent = zml.Tensor.concatenate(&.{ latent, gatherRows(c.cache, compressed_rows, args.q.dtype()) }, .topk); + valid = zml.Tensor.concatenate(&.{ valid, compressed_rows.cmp(.GE, .scalar(0, .i32)) }, .topk); + } + // Query i reads its gathered rows i * topk + j. + const block_rows = zml.Tensor.iota(valid.shape().withDtype(.i32), .q).scale(valid.dim(.topk)) + .add(zml.Tensor.iota(valid.shape().withDtype(.i32), .topk)).mask(valid, -1); + break :blk .{ latent.rename(.{ .q = .page, .topk = .k_chunk }), block_rows }; + }; + const active = zml.Tensor.iota(.init(.{ .q = args.q.dim(.q) }, .i32), .q).cmp(.LT, active_count); + return latentSparseAttention(args.kv.parameters, args.q, kv, args.sink, selected, active_count, opts).mask(active, 0); + } + + /// Gathers the specified rows from the cache + fn gatherRows(cache: Cache, rows: zml.Tensor, dtype: zml.DataType) zml.Tensor { + const valid = rows.cmp(.GE, .scalar(0, .i32)); + const safe = rows.mask(valid, 0); + const values = switch (cache) { + .latent => |t| t.merge(.{ .slot = .{ .page, .k_chunk } }).gather(.{ .slot = safe }, .{}).convert(dtype), + .quantized => |input| blk: { + const v = input.values.merge(.{ .slot = .{ .page, .k_chunk } }).gather(.{ .slot = safe }, .{}); + const sc = input.scales.merge(.{ .slot = .{ .page, .k_chunk } }).gather(.{ .slot = safe }, .{}); + const grouped = v.convert(.f32).splitAxis(.hd, .{ .qb = sc.dim(.hd), .qi = .auto }); + var dq = grouped.mul(sc.convert(.f32).rename(.{ .hd = .qb }).broad(grouped.shape())).merge(.{ .hd = .{ .qb, .qi } }); + if (input.global_scale) |g| dq = dq.mul(g.convert(.f32).broad(dq.shape())); + break :blk dq.convert(dtype).insertAxes(.hd, .{.hkv}); + }, + }; + return values.mask(valid.insertAxes(.last, .{ .hkv, .hd }).broad(values.shape()), 0); + } + + fn latentSparseAttention(parameters: Parameters, q: zml.Tensor, latent_kv: zml.Tensor, sink: ?zml.Tensor, rows: zml.Tensor, active_count: zml.Tensor, opts: Mla.Options) zml.Tensor { stdx.debug.assert(q.shape().hasTags(.{ .q, .h, .hd }), "expected q to have tags .q, .h, .hd after flattening, got {f}", .{q.shape()}); stdx.debug.assert(q.dim(.hd) > opts.rope_rank, "expected q head dim ({}) to include a rope tail of {}", .{ q.dim(.hd), opts.rope_rank }); stdx.debug.assert(opts.value_rank > 0, "expected MLA value rank to be positive, got {}", .{opts.value_rank}); @@ -882,14 +980,8 @@ pub const Mla = struct { stdx.debug.assert(latent_kv.dim(.hd) == q.dim(.hd), "expected q and kv cache head dims to match, got q={} kv={}", .{ q.dim(.hd), latent_kv.dim(.hd) }); return switch (parameters) { - .triton => |triton_parameters| sparse_mla.pagedAttention(triton_parameters, q, latent_kv, sink, topk, tokens_pos, opts), - .stablehlo => |stablehlo_parameters| stablehlo_pagedSparseAttention( - q, - latent_kv, - sink, - triton.paged.topkToPhysical(stablehlo_parameters, topk, tokens_pos, latent_kv.dim(.k_chunk)), - opts, - ), + .triton => |p| sparse_mla.attention(q, latent_kv, sink, rows, active_count, !p.options_.is_prefill, opts), + .stablehlo => stablehlo_pagedSparseAttention(q, latent_kv, sink, rows, opts), else => @panic("NOPE"), }; } @@ -940,7 +1032,7 @@ test "Triton sparse MLA value ranks and padded queries" { const query_start_len: [2]i32 = .{ 0, 1 }; const q = zml.Tensor.init(q_shape, .f32); - const kv: KvCache = .{ .latent = zml.Tensor.init(kv_shape, .f32) }; + const kv = zml.Tensor.init(kv_shape, .f32); const sink = zml.Tensor.init(sink_shape, .f32); const topk = zml.Tensor.init(topk_shape, .i32); const tokens_pos = zml.Tensor.init(tokens_pos_shape, .i32); @@ -953,8 +1045,8 @@ test "Triton sparse MLA value ranks and padded queries" { defer zml.Buffer.deinitAll(Parameters, ¶meters_d); var q_d = try zml.Buffer.fromBytes(std.testing.io, platform, q_shape, .replicated, std.mem.sliceAsBytes(&q_data)); defer q_d.deinit(); - var kv_d: zml.Bufferized(KvCache) = .{ .latent = try .fromBytes(std.testing.io, platform, kv_shape, .replicated, std.mem.sliceAsBytes(&kv_data)) }; - defer zml.Buffer.deinitAll(KvCache, &kv_d); + var kv_d = try zml.Buffer.fromBytes(std.testing.io, platform, kv_shape, .replicated, std.mem.sliceAsBytes(&kv_data)); + defer kv_d.deinit(); var sink_d = try zml.Buffer.fromBytes(std.testing.io, platform, sink_shape, .replicated, std.mem.sliceAsBytes(&sink_data)); defer sink_d.deinit(); var topk_d = try zml.Buffer.fromBytes(std.testing.io, platform, topk_shape, .replicated, std.mem.sliceAsBytes(&topk_data)); @@ -974,7 +1066,7 @@ test "Triton sparse MLA value ranks and padded queries" { std.testing.allocator, std.testing.io, Mla.pagedSparseAttention, - .{ parameters, q, kv, sink, topk, tokens_pos, .{ + .{ Mla.SparseAttentionArgs{ .q = q, .kv = .{ .cache = .{ .latent = kv }, .parameters = parameters, .positions = topk }, .sink = sink, .tokens_pos = tokens_pos }, .{ .rope_rank = 64, .value_rank = case.value_rank, .scale = 1, @@ -989,7 +1081,7 @@ test "Triton sparse MLA value ranks and padded queries" { std.testing.io, &exe, Mla.pagedSparseAttention, - .{ parameters_d, q_d, kv_d, sink_d, topk_d, tokens_pos_d }, + .{.{ .q = q_d, .kv = .{ .cache = .{ .latent = kv_d }, .parameters = parameters_d, .positions = topk_d }, .compressed = null, .sink = sink_d, .tokens_pos = tokens_pos_d }}, ); defer output_d.deinit(); try std.testing.expect(output_d.shape().eql(q_shape.set(.hd, case.value_rank))); @@ -1056,7 +1148,7 @@ test "execute stablehlo mla kernel" { const query_start_len: [2]i32 = .{ 0, 1 }; const q = zml.Tensor.init(q_shape, .f32); - const kv: KvCache = .{ .latent = zml.Tensor.init(kv_shape, .f32) }; + const kv = zml.Tensor.init(kv_shape, .f32); const sink = zml.Tensor.init(sink_shape, .f32); const topk = zml.Tensor.init(topk_shape, .i32); const tokens_pos = zml.Tensor.init(tokens_pos_shape, .i32); @@ -1069,8 +1161,8 @@ test "execute stablehlo mla kernel" { defer zml.Buffer.deinitAll(Parameters, ¶meters_d); var q_d = try zml.Buffer.fromBytes(std.testing.io, platform, q_shape, .replicated, std.mem.sliceAsBytes(&q_data)); defer q_d.deinit(); - var kv_d: zml.Bufferized(KvCache) = .{ .latent = try .fromBytes(std.testing.io, platform, kv_shape, .replicated, std.mem.sliceAsBytes(&kv_data)) }; - defer zml.Buffer.deinitAll(KvCache, &kv_d); + var kv_d = try zml.Buffer.fromBytes(std.testing.io, platform, kv_shape, .replicated, std.mem.sliceAsBytes(&kv_data)); + defer kv_d.deinit(); var sink_d = try zml.Buffer.fromBytes(std.testing.io, platform, sink_shape, .replicated, std.mem.sliceAsBytes(&sink_data)); defer sink_d.deinit(); var topk_d = try zml.Buffer.fromBytes(std.testing.io, platform, topk_shape, .replicated, std.mem.sliceAsBytes(&topk_data)); @@ -1082,7 +1174,7 @@ test "execute stablehlo mla kernel" { std.testing.allocator, std.testing.io, Mla.pagedSparseAttention, - .{ parameters, q, kv, sink, topk, tokens_pos, .{ + .{ Mla.SparseAttentionArgs{ .q = q, .kv = .{ .cache = .{ .latent = kv }, .parameters = parameters, .positions = topk }, .sink = sink, .tokens_pos = tokens_pos }, .{ .rope_rank = 64, .value_rank = 64, .scale = 1, @@ -1096,7 +1188,7 @@ test "execute stablehlo mla kernel" { std.testing.io, &exe, Mla.pagedSparseAttention, - .{ parameters_d, q_d, kv_d, sink_d, topk_d, tokens_pos_d }, + .{.{ .q = q_d, .kv = .{ .cache = .{ .latent = kv_d }, .parameters = parameters_d, .positions = topk_d }, .compressed = null, .sink = sink_d, .tokens_pos = tokens_pos_d }}, ); defer zml.Buffer.deinitAll(zml.Tensor, &output_d); diff --git a/zml/attention/sparse_mla.zig b/zml/attention/sparse_mla.zig index 964bd571..fbb9573b 100644 --- a/zml/attention/sparse_mla.zig +++ b/zml/attention/sparse_mla.zig @@ -2,17 +2,22 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../zml.zig"); const triton = @import("triton_attention.zig"); +const cute = @import("cute_kernels/sparse_mla.zig"); const fly = @import("fly_kernels/sparse_mla.zig"); const MlaOptions = @import("paged_attention.zig").Mla.Options; +const SparseAttentionArgs = @import("paged_attention.zig").Mla.SparseAttentionArgs; pub const Backend = enum { triton, fly, + cute, - pub fn auto(platform: *const zml.Platform, dtype: zml.DataType) Backend { + pub fn auto(args: SparseAttentionArgs) Backend { + const platform = zml.Compiler.current().platform; return switch (platform.target) { + .cuda => if (cute.supports(args)) .cute else .triton, .rocm => switch (zml.platform.rocm.computeCapability(platform) orelse return .triton) { - .gfx942 => switch (dtype) { + .gfx942 => switch (args.q.dtype()) { .bf16 => .fly, else => .triton, }, @@ -23,13 +28,18 @@ pub const Backend = enum { } fn call(self: Backend, q: zml.Tensor, kv_cache: zml.Tensor, sink: ?zml.Tensor, topk: zml.Tensor, active_query_count: zml.Tensor, opts: Options) zml.Tensor { - const backend = switch (self) { - .fly => auto(zml.Compiler.current().platform, q.dtype()), + const backend: Backend = switch (self) { + .fly => blk: { + const platform = zml.Compiler.current().platform; + const gfx942 = platform.target == .rocm and zml.platform.rocm.computeCapability(platform) == .gfx942; + break :blk if (gfx942 and q.dtype() == .bf16) .fly else .triton; + }, .triton => .triton, + .cute => .cute, }; switch (backend) { .fly => if (fly.pagedAttention(q, kv_cache, sink, topk, active_query_count, opts, triton.getCuCount())) |output| return output, - .triton => {}, + .triton, .cute => {}, } return triton.paged.pagedSparseMlaKernel(q, kv_cache, sink, topk, active_query_count, opts); } @@ -133,71 +143,52 @@ pub fn launchConfig(paged_opts: Options, topk_count: usize, cu_count_: usize) Co return config; } +/// Sparse MLA over a paged latent cache, `topk` holding logical token positions. pub fn pagedAttention(parameters: triton.paged.Parameters, q: zml.Tensor, kv_cache: zml.Tensor, sink: ?zml.Tensor, topk: zml.Tensor, tokens_pos: zml.Tensor, opts: MlaOptions) zml.Tensor { + const rows = triton.paged.topkToPhysical(parameters, topk, tokens_pos, kv_cache.dim(.k_chunk)); + const active_count = parameters.query_start_len.slice(.b, .single(parameters.query_start_len.dim(.b) - 1)); + return attention(q, kv_cache, sink, rows, active_count, !parameters.options_.is_prefill, opts); +} + +/// Sparse MLA of `q` [.q, .h, .hd] over the latent cache `kv_cache` [.page, .k_chunk, .hkv, .hd] +/// with the Triton (or Fly) kernels: `rows` holds the physical row of each selected key +/// ([.q, .topk], -1 unused), `active_count` the number of leading active queries. +pub fn attention(q: zml.Tensor, kv_cache: zml.Tensor, sink: ?zml.Tensor, rows: zml.Tensor, active_count: zml.Tensor, all_decode: bool, opts: MlaOptions) zml.Tensor { const output_shape = q.shape().set(.hd, opts.value_rank); return zml.ops.manualComputation( (struct { q: zml.Tensor, kv_cache: zml.Tensor, sink: ?zml.Tensor, - topk: zml.Tensor, - tokens_pos: zml.Tensor, - block_table: zml.Tensor, - seq_lens: zml.Tensor, - query_start_len: zml.Tensor, + rows: zml.Tensor, + active_count: zml.Tensor, + all_decode: bool, opts: MlaOptions, - options: triton.paged.Options, fn body(self: @This(), _: zml.Shape) zml.Tensor { - const block_size = self.kv_cache.dim(.k_chunk); - - const parameters_: triton.paged.Parameters = .{ - .block_table = self.block_table, - .seq_lens = self.seq_lens, - .query_start_len = self.query_start_len, - .options_ = self.options, - }; - - const topk_final = triton.paged.topkToPhysical(parameters_, self.topk, self.tokens_pos, block_size); - const active_query_count = self.query_start_len - .slice(.b, .{ .start = self.query_start_len.dim(.b) - 1 }) - .squeeze(.b); - stdx.debug.assert(topk_final.dim(.q) == self.q.dim(.q), "expected topk q dim ({}) to match q dim ({})", .{ topk_final.dim(.q), self.q.dim(.q) }); - - const num_heads: usize = @intCast(self.q.dim(.h)); + stdx.debug.assert(self.rows.dim(.q) == self.q.dim(.q), "expected rows q dim ({}) to match q dim ({})", .{ self.rows.dim(.q), self.q.dim(.q) }); const paged_opts: Options = .{ .qk_rank = @intCast(self.q.dim(.hd)), .value_rank = @intCast(self.opts.value_rank), - .num_heads = num_heads, + .num_heads = @intCast(self.q.dim(.h)), .block_size = @intCast(self.kv_cache.dim(.k_chunk)), .rope_rank = @intCast(self.opts.rope_rank), .scale = self.opts.scale, .total_q_blocks = @intCast(self.q.dim(.q)), .num_kv_splits = self.opts.num_kv_splits, - .all_decode = !self.options.is_prefill, + .all_decode = self.all_decode, }; - - return self.opts.backend.call( - self.q, - self.kv_cache, - self.sink, - topk_final, - active_query_count, - paged_opts, - ); + return self.opts.backend.call(self.q, self.kv_cache, self.sink, self.rows, self.active_count, paged_opts); } }).body, .{ .q = q, .kv_cache = kv_cache, .sink = sink, - .topk = topk, - .tokens_pos = tokens_pos, - .block_table = parameters.block_table, - .seq_lens = parameters.seq_lens, - .query_start_len = parameters.query_start_len, + .rows = rows, + .active_count = active_count, + .all_decode = all_decode, .opts = opts, - .options = parameters.options_, }, output_shape, ); From 9e6793c6047395be770ef87c8042d9a1eb250c63 Mon Sep 17 00:00:00 2001 From: Louis Lechevalier Date: Fri, 2 Oct 2026 13:33:21 +0200 Subject: [PATCH 739/881] mlir/dialects: rename NVPTX specifics in LLVM dialect (#887) --- kernels/cute/builder.zig | 16 ++++++++-------- mlir/dialects/llvm_dialect.zig | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/kernels/cute/builder.zig b/kernels/cute/builder.zig index ce28b3f4..ebeae3e1 100644 --- a/kernels/cute/builder.zig +++ b/kernels/cute/builder.zig @@ -2567,7 +2567,7 @@ pub const Builder = struct { if (barrier.isInt()) return self.addressPtr(barrier, .shared); return self.emit(mlir.Operation.make(self.ctx, "builtin.unrealized_conversion_cast", .{ .operands = .{ .flat = &.{barrier.inner} }, - .results = .{ .flat = &.{llvm.pointerType(self.ctx, .shared)} }, + .results = .{ .flat = &.{llvm.nvptxPointerType(self.ctx, .shared)} }, .location = self.loc(), })); } @@ -2637,11 +2637,11 @@ pub const Builder = struct { // `i32` TMEM addresses (lane << 16 | column). These helpers emit the NVVM / // LLVM operations for them directly (no inline PTX). - pub const AddressSpace = llvm.AddressSpace; + pub const NVPTXAddressSpace = llvm.NVPTXAddressSpace; /// Typed LLVM pointer (`llvm.inttoptr`) for an integer address in `space`. - pub fn addressPtr(self: *Builder, address: Value, comptime space: AddressSpace) Value { - return self.emit(llvm.inttoptr(self.ctx, address.inner, llvm.pointerType(self.ctx, space), self.loc())); + pub fn addressPtr(self: *Builder, address: Value, comptime space: NVPTXAddressSpace) Value { + return self.emit(llvm.inttoptr(self.ctx, address.inner, llvm.nvptxPointerType(self.ctx, space), self.loc())); } fn wordsType(self: *Builder, comptime n: usize) *const mlir.Type { @@ -2678,7 +2678,7 @@ pub const Builder = struct { } /// Load `n` (1, 2 or 4) consecutive 32-bit words at an integer address. - pub fn loadWords(self: *Builder, comptime n: usize, address: Value, comptime space: AddressSpace, opts: LoadOptions) [n]Value { + pub fn loadWords(self: *Builder, comptime n: usize, address: Value, comptime space: NVPTXAddressSpace, opts: LoadOptions) [n]Value { return self.loadWordsPtr(n, self.addressPtr(address, space), opts); } @@ -2688,12 +2688,12 @@ pub const Builder = struct { } /// Store `n` (1, 2 or 4) consecutive 32-bit words at an integer address. - pub fn storeWords(self: *Builder, comptime n: usize, address: Value, comptime space: AddressSpace, words: [n]Value) void { + pub fn storeWords(self: *Builder, comptime n: usize, address: Value, comptime space: NVPTXAddressSpace, words: [n]Value) void { self.storeWordsPtr(n, self.addressPtr(address, space), words); } /// Store the low 16 bits of an `i32`. - pub fn storeHalf(self: *Builder, address: Value, comptime space: AddressSpace, word: Value) void { + pub fn storeHalf(self: *Builder, address: Value, comptime space: NVPTXAddressSpace, word: Value) void { self.emitVoid(llvm.store(self.ctx, word.to(.i16).inner, self.addressPtr(address, space).inner, .{ .alignment = 2 }, self.loc())); } @@ -2809,7 +2809,7 @@ pub const Builder = struct { /// The same shared-memory location (i32 address) in CTA `rank` of the cluster, as a /// `shared::cluster` pointer. pub fn mapaShared(self: *Builder, address: Value, rank: Value) Value { - return self.emit(nvvm.mapa(self.ctx, self.addressPtr(address, .shared).inner, rank.inner, llvm.pointerType(self.ctx, .shared_cluster), self.loc())); + return self.emit(nvvm.mapa(self.ctx, self.addressPtr(address, .shared).inner, rank.inner, llvm.nvptxPointerType(self.ctx, .shared_cluster), self.loc())); } /// Full cluster barrier (arrive.release + wait.acquire, aligned). diff --git a/mlir/dialects/llvm_dialect.zig b/mlir/dialects/llvm_dialect.zig index 9ca44ae2..50b3414c 100644 --- a/mlir/dialects/llvm_dialect.zig +++ b/mlir/dialects/llvm_dialect.zig @@ -7,7 +7,7 @@ const mlir = @import("mlir"); // ============================================================================= /// Address spaces of `!llvm.ptr` on NVPTX. -pub const AddressSpace = enum(u32) { +pub const NVPTXAddressSpace = enum(u32) { generic = 0, global = 1, shared = 3, @@ -25,7 +25,7 @@ pub const AddressSpace = enum(u32) { /// LLVM dialect registered in `ctx`, which may be another MLIR build than the C API's /// (e.g. in the CuTe compiler context), and loading the C API's dialect there would /// clash with it. -pub fn pointerType(ctx: *mlir.Context, space: AddressSpace) *const mlir.Type { +pub fn nvptxPointerType(ctx: *mlir.Context, space: NVPTXAddressSpace) *const mlir.Type { var buf: [32]u8 = undefined; const text = std.fmt.bufPrint(&buf, "!llvm.ptr<{d}>", .{@intFromEnum(space)}) catch unreachable; return mlir.Type.parse(ctx, text) catch std.debug.panic("failed to parse LLVM pointer type '{s}'", .{text}); From 75aa9878e1947f6f1b62665046e1983091727b31 Mon Sep 17 00:00:00 2001 From: Louis Lechevalier Date: Fri, 2 Oct 2026 15:45:47 +0200 Subject: [PATCH 740/881] mlir/dialects: keep pointerType general in LLVM dialect (#889) NVPTXAddressSpace is NVPTX specific. In LLVM, addressSpace is defined for each target (nvptx, amdgpu) The pointerType is general in llvm dialect by accepting an anytype. --- kernels/cute/builder.zig | 6 +++--- mlir/dialects/llvm_dialect.zig | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kernels/cute/builder.zig b/kernels/cute/builder.zig index ebeae3e1..9c777491 100644 --- a/kernels/cute/builder.zig +++ b/kernels/cute/builder.zig @@ -2567,7 +2567,7 @@ pub const Builder = struct { if (barrier.isInt()) return self.addressPtr(barrier, .shared); return self.emit(mlir.Operation.make(self.ctx, "builtin.unrealized_conversion_cast", .{ .operands = .{ .flat = &.{barrier.inner} }, - .results = .{ .flat = &.{llvm.nvptxPointerType(self.ctx, .shared)} }, + .results = .{ .flat = &.{llvm.pointerType(self.ctx, NVPTXAddressSpace.shared)} }, .location = self.loc(), })); } @@ -2641,7 +2641,7 @@ pub const Builder = struct { /// Typed LLVM pointer (`llvm.inttoptr`) for an integer address in `space`. pub fn addressPtr(self: *Builder, address: Value, comptime space: NVPTXAddressSpace) Value { - return self.emit(llvm.inttoptr(self.ctx, address.inner, llvm.nvptxPointerType(self.ctx, space), self.loc())); + return self.emit(llvm.inttoptr(self.ctx, address.inner, llvm.pointerType(self.ctx, space), self.loc())); } fn wordsType(self: *Builder, comptime n: usize) *const mlir.Type { @@ -2809,7 +2809,7 @@ pub const Builder = struct { /// The same shared-memory location (i32 address) in CTA `rank` of the cluster, as a /// `shared::cluster` pointer. pub fn mapaShared(self: *Builder, address: Value, rank: Value) Value { - return self.emit(nvvm.mapa(self.ctx, self.addressPtr(address, .shared).inner, rank.inner, llvm.nvptxPointerType(self.ctx, .shared_cluster), self.loc())); + return self.emit(nvvm.mapa(self.ctx, self.addressPtr(address, .shared).inner, rank.inner, llvm.pointerType(self.ctx, NVPTXAddressSpace.shared_cluster), self.loc())); } /// Full cluster barrier (arrive.release + wait.acquire, aligned). diff --git a/mlir/dialects/llvm_dialect.zig b/mlir/dialects/llvm_dialect.zig index 50b3414c..2f5d3091 100644 --- a/mlir/dialects/llvm_dialect.zig +++ b/mlir/dialects/llvm_dialect.zig @@ -25,7 +25,7 @@ pub const NVPTXAddressSpace = enum(u32) { /// LLVM dialect registered in `ctx`, which may be another MLIR build than the C API's /// (e.g. in the CuTe compiler context), and loading the C API's dialect there would /// clash with it. -pub fn nvptxPointerType(ctx: *mlir.Context, space: NVPTXAddressSpace) *const mlir.Type { +pub fn pointerType(ctx: *mlir.Context, space: anytype) *const mlir.Type { var buf: [32]u8 = undefined; const text = std.fmt.bufPrint(&buf, "!llvm.ptr<{d}>", .{@intFromEnum(space)}) catch unreachable; return mlir.Type.parse(ctx, text) catch std.debug.panic("failed to parse LLVM pointer type '{s}'", .{text}); From 773fb8f2f30453c8efedfcc8afa0b9880fb75bce Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Fri, 2 Oct 2026 22:30:34 +0200 Subject: [PATCH 741/881] platforms/oneapi: update latest pjrt artifact (#883) --- platforms/oneapi/oneapi.bzl | 12 +++--- platforms/oneapi/packages.lock.json | 66 ++++++++++++++--------------- platforms/oneapi/packages.yaml | 2 +- 3 files changed, 40 insertions(+), 40 deletions(-) diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index b6436a93..55075f0a 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -1,8 +1,8 @@ load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") -_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202609301700.135.1.f7a73483aa8a/zml-oneapi-linux-amd64.tar.zst" -_PLUGIN_ONEAPI_SHA256 = "2437fd3abc23913f1b3d38ea74c3ddd04a73961b56f89165ba4346cfd75eacde" +_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202610011939.137.1.0d59bb389ee2/zml-oneapi-linux-amd64.tar.zst" +_PLUGIN_ONEAPI_SHA256 = "9ca5e2c377d938b3d855c46c685d193e90424e44b1c1ea8d0a8e1c924b0d11f1" ONEAPI_VERSION = "2026.1" @@ -68,11 +68,11 @@ filegroup( "libigc2": """ filegroup( name = "libigc_so_2", - srcs = ["lib/x86_64-linux-gnu/libigc.so.2.36.3+0"], + srcs = ["lib/x86_64-linux-gnu/libigc.so.2.40.13+0"], ) filegroup( name = "libiga64_so_2", - srcs = ["lib/x86_64-linux-gnu/libiga64.so.2.36.3+0"], + srcs = ["lib/x86_64-linux-gnu/libiga64.so.2.40.13+0"], ) filegroup( name = "libigc2", @@ -84,11 +84,11 @@ filegroup( "libigdfcl2": """ filegroup( name = "libigdfcl_so_2", - srcs = ["lib/x86_64-linux-gnu/libigdfcl.so.2.36.3+0"], + srcs = ["lib/x86_64-linux-gnu/libigdfcl.so.2.40.13+0"], ) filegroup( name = "libopencl_clang", - srcs = ["lib/x86_64-linux-gnu/libopencl-clang.so.16"], + srcs = ["lib/x86_64-linux-gnu/libopencl-clang.so.17"], ) filegroup( name = "libigdfcl2", diff --git a/platforms/oneapi/packages.lock.json b/platforms/oneapi/packages.lock.json index 26a096ee..d8747a09 100644 --- a/platforms/oneapi/packages.lock.json +++ b/platforms/oneapi/packages.lock.json @@ -3,13 +3,13 @@ { "arch": "amd64", "dependencies": [], - "key": "intel-oneapi-ccl-2022.1_2022.1.0-142_amd64", + "key": "intel-oneapi-ccl-2022.1_2022.1.2-9_amd64", "name": "intel-oneapi-ccl-2022.1", - "sha256": "3569a089879b6b7358c5afb496ae782754715c8ace5137e00797e6b111fb4cc5", + "sha256": "432e62469978f60a9b35b55216cfd495b0adc76c334707d3bbc81f69b7cc2948", "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-ccl-2022.1-2022.1.0-142_amd64.deb" + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-ccl-2022.1-2022.1.2-9_amd64.deb" ], - "version": "2022.1.0-142" + "version": "2022.1.2-9" }, { "arch": "amd64", @@ -25,24 +25,24 @@ { "arch": "amd64", "dependencies": [], - "key": "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1_2026.1.0-235_amd64", + "key": "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1_2026.1.1-325_amd64", "name": "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1", - "sha256": "f557b04243b6bb2087646b9d9eee94026bf1abc80b188dcfa3e3b689e292d95e", + "sha256": "df0413d57cff5a9c67bd02c1e958d7e0512f77878bbcf976c9e08af7c0b1069c", "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1-2026.1.0-235_amd64.deb" + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1-2026.1.1-325_amd64.deb" ], - "version": "2026.1.0-235" + "version": "2026.1.1-325" }, { "arch": "amd64", "dependencies": [], - "key": "intel-oneapi-compiler-shared-runtime-2026.1_2026.1.0-235_amd64", + "key": "intel-oneapi-compiler-shared-runtime-2026.1_2026.1.1-325_amd64", "name": "intel-oneapi-compiler-shared-runtime-2026.1", - "sha256": "b5a077f78f192f93bba608f5497d7c138f77ec211a9d86356696a0ca68d3bf5d", + "sha256": "c0c77700243af0413f7fbefd26649943c4959171ead91b6ace36ef0c8dfb2630", "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-compiler-shared-runtime-2026.1-2026.1.0-235_amd64.deb" + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-compiler-shared-runtime-2026.1-2026.1.1-325_amd64.deb" ], - "version": "2026.1.0-235" + "version": "2026.1.1-325" }, { "arch": "amd64", @@ -201,57 +201,57 @@ { "arch": "amd64", "dependencies": [], - "key": "libigdgmm12_22.10.0-1_24.04_ppa1_amd64", + "key": "libigdgmm12_22.10.1-1_24.04_ppa1_amd64", "name": "libigdgmm12", - "sha256": "63fe9e124c3394193226c6cb8398dcabdc9306b5a10f63668118f7c799f5d792", + "sha256": "2c68fcb150cba1b679063266cdf0054f26f18af90de71d6d3b614eaf95c98f80", "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260715T000000Z/pool/main/i/intel-gmmlib/libigdgmm12_22.10.0-1~24.04~ppa1_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z/pool/main/i/intel-gmmlib/libigdgmm12_22.10.1-1~24.04~ppa1_amd64.deb" ], - "version": "22.10.0-1~24.04~ppa1" + "version": "22.10.1-1~24.04~ppa1" }, { "arch": "amd64", "dependencies": [], - "key": "libigc2_2.36.3-2_24.04_amd64", + "key": "libigc2_2.40.13-p-ds1-1_24.04_amd64", "name": "libigc2", - "sha256": "c11dd6e083d1010c50bbc33cabf5b614e64df53b61d349955e97efdababb85bf", + "sha256": "c57e4ca8c780dc85a309e35b4d3249fc866251fea7bf3ebe68a5ab5ef5f71362", "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260715T000000Z/pool/main/i/intel-graphics-compiler/libigc2_2.36.3-2~24.04_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z/pool/main/i/intel-graphics-compiler/libigc2_2.40.13+ds1-1~24.04_amd64.deb" ], - "version": "2.36.3-2~24.04" + "version": "2.40.13+ds1-1~24.04" }, { "arch": "amd64", "dependencies": [], - "key": "libigdfcl2_2.36.3-2_24.04_amd64", + "key": "libigdfcl2_2.40.13-p-ds1-1_24.04_amd64", "name": "libigdfcl2", - "sha256": "019c4d2802b03d5e1ddad888543e6b7b6003a46a14ccf2e16a2320e8923ad007", + "sha256": "e71f41b4dd54c9cda7024680528034204cd911a1c2f644ec2bc3c06e968f5afe", "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260715T000000Z/pool/main/i/intel-graphics-compiler/libigdfcl2_2.36.3-2~24.04_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z/pool/main/i/intel-graphics-compiler/libigdfcl2_2.40.13+ds1-1~24.04_amd64.deb" ], - "version": "2.36.3-2~24.04" + "version": "2.40.13+ds1-1~24.04" }, { "arch": "amd64", "dependencies": [], - "key": "libze-intel-gpu1_26.22.38646.6-1_24.04_ppa1_amd64", + "key": "libze-intel-gpu1_26.31.39395.14-1_24.04_ppa1_amd64", "name": "libze-intel-gpu1", - "sha256": "5d0903aae7ac4f94f43ae7d282a394bc988a0da12f813d92533dea1650eb91ff", + "sha256": "39f10740e40cf0de1015997d0de094a3c3dc6371c3b9ea1dcd15771d64559c14", "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260715T000000Z/pool/main/i/intel-compute-runtime/libze-intel-gpu1_26.22.38646.6-1~24.04~ppa1_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z/pool/main/i/intel-compute-runtime/libze-intel-gpu1_26.31.39395.14-1~24.04~ppa1_amd64.deb" ], - "version": "26.22.38646.6-1~24.04~ppa1" + "version": "26.31.39395.14-1~24.04~ppa1" }, { "arch": "amd64", "dependencies": [], - "key": "libze1_1.28.6-1_24.04_ppa1_amd64", + "key": "libze1_1.32.0-1_24.04_ppa1_amd64", "name": "libze1", - "sha256": "ca4aec5982d46f92875659e74e459372f24c9d8add2810556c1c7e71531e1ec0", + "sha256": "d4001dc474d3cb74ded50a449b9f646cfdfbb31e69e70b8af38f2d18273dfce1", "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260715T000000Z/pool/main/l/level-zero-loader/libze1_1.28.6-1~24.04~ppa1_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z/pool/main/l/level-zero-loader/libze1_1.32.0-1~24.04~ppa1_amd64.deb" ], - "version": "1.28.6-1~24.04~ppa1" + "version": "1.32.0-1~24.04~ppa1" }, { "arch": "amd64", @@ -266,4 +266,4 @@ } ], "version": 1 -} +} \ No newline at end of file diff --git a/platforms/oneapi/packages.yaml b/platforms/oneapi/packages.yaml index 6abf6e47..78f8b49e 100644 --- a/platforms/oneapi/packages.yaml +++ b/platforms/oneapi/packages.yaml @@ -13,7 +13,7 @@ sources: - channel: noble-updates main url: https://snapshot.ubuntu.com/ubuntu/20260715T000000Z - channel: noble main - url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260715T000000Z + url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z archs: - "amd64" From 8c2e4c6dc9ba2d54d15c4412bf7249e66d619bd5 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Mon, 5 Oct 2026 11:15:57 +0200 Subject: [PATCH 742/881] pjrt,zml/platform: expose devices' NUMA node (#891) Allows placing all the threads/memory on the right numa nodes. It's faster for a device to access memory on the same numa node and overall the it's better to have all threads on the same numa node if all devices are on the same one. --- pjrt/pjrt.zig | 34 ++++++++++++++++++++++++++++++++++ zml/platform.zig | 13 +++++++++++++ 2 files changed, 47 insertions(+) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 17339f66..bd58386f 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -680,6 +680,40 @@ pub const Client = opaque { pub const Device = opaque { const inner = InnerMixin(c.PJRT_Device).inner; + pub const Attributes = struct { + values: []const NamedValue, + state: *c.PJRT_Device_Attributes, + deleter: *const fn (?*c.PJRT_Device_Attributes) callconv(.c) void, + + pub fn deinit(self: Attributes) void { + self.deleter(self.state); + } + + /// String and list values remain valid until this owner is deinitialized. + pub fn get(self: Attributes, name: []const u8) ?NamedValue.Value { + for (self.values) |attribute| { + if (std.mem.eql(u8, attribute.name(), name)) return attribute.value(); + } + return null; + } + }; + + pub fn attributes(self: *const Device, api: *const Api) ApiError!Attributes { + const ret = try api.call(.PJRT_Device_GetAttributes, .{ .device = self.inner() }); + + const deleter = ret.attributes_deleter.?; + errdefer deleter(ret.device_attributes); + + const state = ret.device_attributes.?; + std.debug.assert((ret.attributes == null) == (ret.num_attributes == 0)); + + return .{ + .values = if (ret.attributes == null) &.{} else @ptrCast(ret.attributes[0..ret.num_attributes]), + .state = state, + .deleter = deleter, + }; + } + pub fn getDescription(self: *const Device, api: *const Api) *const DeviceDescription { const ret = api.call(.PJRT_Device_GetDescription, .{ .device = self.inner(), diff --git a/zml/platform.zig b/zml/platform.zig index 1706309e..826eeaee 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -196,6 +196,19 @@ pub const Device = struct { return self.pjrt_device.memoryStats(self.platform.pjrt_api) catch .zeroes; } + /// NUMA node closest to the device, when the plugin reports one. + pub fn numaNode(self: Device) ?u32 { + const attributes = self.pjrt_device.attributes(self.platform.pjrt_api) catch return null; + defer attributes.deinit(); + + const value = attributes.get("numa_node") orelse return null; + + return switch (value) { + .int64 => |node| std.math.cast(u32, node), + else => null, + }; + } + pub fn format(self: Device, writer: *std.Io.Writer) std.Io.Writer.Error!void { try writer.print("{s} ({s})", .{ self.pjrt_desc.kind(self.platform.pjrt_api), From 699add8fd28735609ae5afd6f858455db051db23 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Mon, 5 Oct 2026 11:16:39 +0200 Subject: [PATCH 743/881] zml/io: fix error handling when a direct shard writer fails to init (#894) DirectShardWriter.init treated retrieveBuffer errors as unreachable and leaked its transfer manager and buffer when a later step failed. DirectMemoryWriter.init counted a shard writer as initialized even when its init failed, so its errdefer deinitialized an undefined writer. --- zml/io.zig | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/zml/io.zig b/zml/io.zig index a6ff262d..b45bf619 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -754,8 +754,10 @@ const DirectShardWriter = struct { .memory = memory.pjrt_memory, }, ); + errdefer transfer_manager.deinit(memory.platform.pjrt_api); - const pjrt_buffer = transfer_manager.retrieveBuffer(memory.platform.pjrt_api, 0) catch unreachable; + const pjrt_buffer = try transfer_manager.retrieveBuffer(memory.platform.pjrt_api, 0); + errdefer pjrt_buffer.deinit(memory.platform.pjrt_api); const buf = try pool.get(allocator, io); @@ -1169,13 +1171,12 @@ pub const DirectMemoryWriter = struct { var pjrt_buffers: Buffer.Shards = .empty; const placement = try sharding.placement(shape); for (ordered_devices, 0..) |device, i| { - defer initialized += 1; - const pool = &pools[device.id]; const shard_dma_allocator = dma_allocators[device.id].allocator(); const pjrt_mem = platform.devices[device.id].memory(memory).?; shard_writers[i] = try .init(shard_dma_allocator, io, pjrt_mem, pool, placement.shape); + initialized += 1; pjrt_buffers.appendAssumeCapacity(shard_writers[i].pjrt_buffer); } From daac9c8833610c58e25a86a60b287e7652eed309 Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Mon, 5 Oct 2026 13:21:19 +0200 Subject: [PATCH 744/881] zml/model: [refactor] Change llama to a whole forward pass (#881) --- examples/llm/models/llama/inference.zig | 250 +++++++----------------- examples/llm/models/llama/session.zig | 17 +- 2 files changed, 76 insertions(+), 191 deletions(-) diff --git a/examples/llm/models/llama/inference.zig b/examples/llm/models/llama/inference.zig index 53a655ac..8a150e65 100644 --- a/examples/llm/models/llama/inference.zig +++ b/examples/llm/models/llama/inference.zig @@ -110,92 +110,88 @@ pub const CompiledModel = struct { pub const Inference = CompiledModel; -pub const KernelExe = struct { - embed: zml.FnExe(model.EmbedTokens.forward), - layer: zml.FnExe(model.TransformerLayer.forward), - sample: zml.FnExe(model.LmHead.forward), +const Forward = struct { + pub const Input = struct { + weights: model.Model, + tokens: zml.Tensor, + token_index: zml.Tensor, + kv_cache: model.KvCache, + rng: zml.Tensor.Rng, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, + }; - pub fn deinit(self: *const KernelExe) void { - self.embed.deinit(); - self.layer.deinit(); - self.sample.deinit(); + pub const Output = struct { + tokens: zml.Tensor, + kv_cache: model.KvCache, + rng: zml.Tensor.Rng, + }; + + pub fn forward(input: Input) Output { + var hidden = model.EmbedTokens.forward(.{ + .embedding = .{ .embed_tokens = input.weights.model.embed_tokens }, + .tokens = input.tokens, + }).hidden; + var kv_cache = input.kv_cache; + for (input.weights.model.layers, 0..) |layer, index| { + const result = model.TransformerLayer.forward(.{ + .layer = layer, + .hidden = hidden, + .token_index = input.token_index, + .kv_cache = kv_cache, + .kv_cache_index = .scalar(@as(u32, @intCast(index)), .u32), + .attention_metadata = input.attention_metadata, + .attention_parameters = input.attention_parameters, + }); + hidden = result.hidden; + kv_cache = result.kv_cache; + } + // LmHead owns final normalization; Llama.forward would apply it twice. + const result = model.LmHead.forward(.{ + .lm_head = .init(input.weights), + .hidden = hidden, + .tokens = input.tokens, + .rng = input.rng, + }); + return .{ .tokens = result.tokens, .kv_cache = kv_cache.reuseBuffer(input.kv_cache), .rng = result.rng }; } }; +pub const KernelExe = zml.FnExe(Forward.forward); + pub const KernelRunner = struct { - embed: zml.FnExe(model.EmbedTokens.forward).Runner(.{.embedding}), - layers: []zml.FnExe(model.TransformerLayer.forward).Runner(.{.layer}), - sample: zml.FnExe(model.LmHead.forward).Runner(.{.lm_head}), + forward: KernelExe.Runner(.{.weights}), pub fn init(allocator: std.mem.Allocator, exe: *const KernelExe, buffers: *const model.Buffers) !KernelRunner { - var embed = try zml.FnExe(model.EmbedTokens.forward).Runner(.{.embedding}).init(&exe.embed, allocator, .{ - .embedding = .{ .embed_tokens = buffers.model.embed_tokens }, - }); - errdefer embed.deinit(allocator); - - const layers = try allocator.alloc(zml.FnExe(model.TransformerLayer.forward).Runner(.{.layer}), buffers.model.layers.len); - errdefer allocator.free(layers); - var initialized_layers: usize = 0; - errdefer for (layers[0..initialized_layers]) |*layer| layer.deinit(allocator); - for (layers, buffers.model.layers) |*layer, layer_buffers| { - layer.* = try zml.FnExe(model.TransformerLayer.forward).Runner(.{.layer}).init(&exe.layer, allocator, .{ .layer = layer_buffers }); - initialized_layers += 1; - } - - var sample = try zml.FnExe(model.LmHead.forward).Runner(.{.lm_head}).init(&exe.sample, allocator, .{ - .lm_head = .{ - .lm_head = buffers.lm_head, - .embed_tokens = buffers.model.embed_tokens, - .norm = buffers.model.norm, - }, - }); - errdefer sample.deinit(allocator); - - return .{ .embed = embed, .layers = layers, .sample = sample }; + return .{ .forward = try .init(exe, allocator, .{ .weights = buffers.* }) }; } pub fn deinit(self: *KernelRunner, allocator: std.mem.Allocator) void { - self.embed.deinit(allocator); - for (self.layers) |*layer| layer.deinit(allocator); - allocator.free(self.layers); - self.sample.deinit(allocator); + self.forward.deinit(allocator); } }; -pub fn run(runner: *KernelRunner, args: Args, kv_cache_index_buffers: []const zml.Buffer) void { - var hidden_buffer: zml.Buffer = undefined; - runner.embed.run(args.io, .{ - .inputs = .{ - .tokens = args.tokens_buf.*, - }, - .outputs = .{ .hidden = &hidden_buffer }, - }); - defer hidden_buffer.deinit(); +pub fn run(runner: *KernelRunner, args: Args) void { + // Results replace handles even when PJRT declines optional donation. + // Pending execution retains its allocations after these inputs are released. + var tokens = args.tokens_buf.*; + var kv_cache = args.kv_cache_buffers.*; + var rng = args.rng_buffers.*; + defer tokens.deinit(); + defer model.KvCache.deinitBuffer(&kv_cache); + defer zml.Tensor.Rng.deinitBuffer(&rng); - for (runner.layers, kv_cache_index_buffers) |*layer, kv_cache_index_buffer| { - layer.run(args.io, .{ - .inputs = .{ - .hidden = hidden_buffer, - .token_index = args.token_index_buf.*, - .kv_cache = args.kv_cache_buffers.*, - .kv_cache_index = kv_cache_index_buffer, - .attention_metadata = args.attention_metadata_buffers.*, - }, - .outputs = .{ - .hidden = &hidden_buffer, - .kv_cache = args.kv_cache_buffers, - }, - }); - } - - runner.sample.run(args.io, .{ + runner.forward.run(args.io, .{ .inputs = .{ - .hidden = hidden_buffer, - .tokens = args.tokens_buf.*, - .rng = args.rng_buffers.*, + .tokens = tokens, + .token_index = args.token_index_buf.*, + .kv_cache = kv_cache, + .rng = rng, + .attention_metadata = args.attention_metadata_buffers.*, }, .outputs = .{ .tokens = args.tokens_buf, + .kv_cache = args.kv_cache_buffers, .rng = args.rng_buffers, }, }); @@ -212,121 +208,23 @@ fn compileKernel( phase: Phase, progress: *std.Progress.Node, ) !KernelExe { - const embed = try compileEmbed(allocator, io, platform, llama_model.model.embed_tokens, parameters, seqlen, phase, progress); - errdefer embed.deinit(); - const layer = try compileLayer(allocator, io, platform, llama_model, parameters, seqlen, attention_parameters, phase, progress); - errdefer layer.deinit(); - const sample = try compileSample(allocator, io, platform, llama_model, parameters, seqlen, phase, progress); - errdefer sample.deinit(); - return .{ .embed = embed, .layer = layer, .sample = sample }; -} - -fn compileEmbed( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - embed_tokens: zml.nn.TokenEmbedding, - parameters: CompilationOptions, - seqlen: usize, - phase: Phase, - progress: *std.Progress.Node, -) !zml.FnExe(model.EmbedTokens.forward) { progress.increaseEstimatedTotalItems(1); - var node = progress.start(phase.startMessage("embed_tokens"), 1); + var node = progress.start(phase.startMessage("forward"), 1); defer node.end(); const from: std.Io.Timestamp = .now(io, .awake); - defer phase.logCompileDone(log, "embed_tokens", io, from); + defer phase.logCompileDone(log, "forward", io, from); - const tokens: zml.Tensor = .init(.{ .s = seqlen }, .u32); - - return zml.FnExe(model.EmbedTokens.forward).compile(allocator, io, platform, .{ + return KernelExe.compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), - .program_name = phase.programName("llama", "embed_tokens"), + .program_name = phase.programName("llama", "forward"), }, .{.{ - .embedding = .{ .embed_tokens = embed_tokens }, - .tokens = tokens, - }}); -} - -fn compileLayer( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - llama_model: model.Model, - parameters: CompilationOptions, - seqlen: usize, - attention_parameters: zml.attention.Parameters, - phase: Phase, - progress: *std.Progress.Node, -) !zml.FnExe(model.TransformerLayer.forward) { - progress.increaseEstimatedTotalItems(1); - - var node = progress.start(phase.startMessage("transformer layer"), 1); - defer node.end(); - - const from: std.Io.Timestamp = .now(io, .awake); - defer phase.logCompileDone(log, "transformer layer", io, from); - - const hidden: zml.Tensor = .fromShape(zml.Shape.init( - .{ .s = seqlen, .d = llama_model.config.hidden_size }, - llama_model.model.embed_tokens.weight.dtype(), - ).withPartitioning(.{ .d = .replicated })); - - const kv_cache_index: zml.Tensor = .init(.{}, .u32); - - return zml.FnExe(model.TransformerLayer.forward).compile( - allocator, - io, - platform, - .{ - .shardings = ¶meters.shardings.all(), - .program_name = phase.programName("llama", "layer"), - }, - .{.{ - .layer = llama_model.model.layers[0], - .hidden = hidden, - .token_index = parameters.token_index, - .kv_cache = parameters.kv_cache, - .kv_cache_index = kv_cache_index, - .attention_metadata = parameters.attention_metadata, - .attention_parameters = attention_parameters, - }}, - ); -} - -fn compileSample( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - llama_model: model.Model, - parameters: CompilationOptions, - seqlen: usize, - phase: Phase, - progress: *std.Progress.Node, -) !zml.FnExe(model.LmHead.forward) { - progress.increaseEstimatedTotalItems(1); - - var node = progress.start(phase.startMessage("lm_head"), 1); - defer node.end(); - - const from: std.Io.Timestamp = .now(io, .awake); - defer phase.logCompileDone(log, "lm_head", io, from); - - const hidden: zml.Tensor = .fromShape(zml.Shape.init( - .{ .s = seqlen, .d = llama_model.config.hidden_size }, - llama_model.model.embed_tokens.weight.dtype(), - ).withPartitioning(.{ .d = .replicated })); - - const tokens: zml.Tensor = .init(.{ .s = seqlen }, .u32); - - return zml.FnExe(model.LmHead.forward).compile(allocator, io, platform, .{ - .shardings = ¶meters.shardings.all(), - .program_name = phase.programName("llama", "lm_head"), - }, .{.{ - .lm_head = model.LmHead.init(llama_model), - .hidden = hidden, - .tokens = tokens, + .weights = llama_model, + .tokens = zml.Tensor.init(.{ .s = seqlen }, .u32), + .token_index = parameters.token_index, + .kv_cache = parameters.kv_cache, .rng = parameters.rng, + .attention_metadata = parameters.attention_metadata, + .attention_parameters = attention_parameters, }}); } diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig index 98187a42..2269464f 100644 --- a/examples/llm/models/llama/session.zig +++ b/examples/llm/models/llama/session.zig @@ -14,7 +14,6 @@ pub const Session = struct { decode: inference.KernelRunner, kv_cache_buffers: zml.Bufferized(model.KvCache), token_index_buffers: []zml.Buffer, - kv_cache_index_buffers: []zml.Buffer, rng_buffers: zml.Bufferized(zml.Tensor.Rng), tokenizer: zml.tokenizer.Tokenizer, config: *const model.Config, @@ -53,15 +52,6 @@ pub const Session = struct { var rng_buffers = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); - const kv_cache_index_buffers = try allocator.alloc(zml.Buffer, model_buffers.model.layers.len); - errdefer allocator.free(kv_cache_index_buffers); - var initialized_kv_cache_index_buffers: usize = 0; - errdefer for (kv_cache_index_buffers[0..initialized_kv_cache_index_buffers]) |*buffer| buffer.deinit(); - for (kv_cache_index_buffers, 0..) |*buffer, i| { - buffer.* = try .scalar(io, platform, i, .u32); - initialized_kv_cache_index_buffers = i + 1; - } - var prefill = try inference.KernelRunner.init(allocator, &compiled_model.prefill, model_buffers); errdefer prefill.deinit(allocator); const decode = try inference.KernelRunner.init(allocator, &compiled_model.decode, model_buffers); @@ -75,7 +65,6 @@ pub const Session = struct { .decode = decode, .kv_cache_buffers = kv_cache_buffers, .token_index_buffers = token_index_buffers, - .kv_cache_index_buffers = kv_cache_index_buffers, .rng_buffers = rng_buffers, .tokenizer = tokenizer, .config = &compiled_model.loaded_model.parsed_config.value, @@ -92,8 +81,6 @@ pub const Session = struct { token_index_buffer.deinit(); } self.allocator.free(self.token_index_buffers); - for (self.kv_cache_index_buffers) |*buffer| buffer.deinit(); - self.allocator.free(self.kv_cache_index_buffers); zml.Tensor.Rng.deinitBuffer(&self.rng_buffers); } @@ -173,7 +160,7 @@ pub const Session = struct { .kv_cache_buffers = &self.kv_cache_buffers, .rng_buffers = &self.rng_buffers, .attention_metadata_buffers = &attention_metadata_buffers, - }, self.kv_cache_index_buffers); + }); try prefill_tokens_buffer.toSlice(self.io, prefill_tokens_slice); self.last_generated_token = prefill_tokens_slice.items(u32)[all_tokens.len - 1]; @@ -217,7 +204,7 @@ pub const Session = struct { .kv_cache_buffers = &self.kv_cache_buffers, .rng_buffers = &self.rng_buffers, .attention_metadata_buffers = &attention_metadata_buffers, - }, self.kv_cache_index_buffers); + }); last_token_id = try current_token_buffer.getValue(u32, self.io); } From af4e079f343993fd92ab38efd85caed0c5a8934a Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Mon, 5 Oct 2026 13:51:36 +0200 Subject: [PATCH 745/881] zml/exe: callOpts/zml.testing.autoCall replaced by eval + remove run|callOpts (#897) Removing callOpts & runOpts to simplify the API and changing `io` to be required for API clarity. I'm replacing `zml.testing.autoCall` with `Exe.eval` which solves most needs in a simpler way IMHO --- docs/tutorials/write_first_model.md | 8 +- examples/benchmark/main.zig | 4 +- examples/llm/models/lfm2/inference.zig | 6 +- examples/llm/models/lfm2_tests.zig | 6 +- examples/llm/models/llama/inference.zig | 2 +- examples/llm/models/llama_tests.zig | 2 +- examples/llm/models/qwen3_5/inference.zig | 8 +- examples/llm/models/qwen3_5_moe/inference.zig | 8 +- examples/mnist/mnist.zig | 2 +- examples/sharding/main.zig | 2 +- zml/attention.zig | 19 ++- zml/attention/paged_attention.zig | 28 ++-- zml/exe.zig | 76 +++++----- zml/io.zig | 2 +- zml/kernel.zig | 8 +- zml/moe/fused_experts.zig | 11 +- zml/nn.zig | 72 ++++++---- zml/ops.zig | 20 ++- zml/quantization.zig | 10 +- zml/tensor.zig | 132 +++++++++++------- zml/testing.zig | 21 +-- 21 files changed, 244 insertions(+), 203 deletions(-) diff --git a/docs/tutorials/write_first_model.md b/docs/tutorials/write_first_model.md index b4b1f652..4d577586 100644 --- a/docs/tutorials/write_first_model.md +++ b/docs/tutorials/write_first_model.md @@ -227,7 +227,7 @@ defer results.deinit(allocator); args.set(.{ layer_buffers, input_buffer }); // call our executable -executable.call(args, &results); +executable.call(io, args, &results, .{}); // Retrieve the resulting buffer var result = results.get(zml.Buffer); @@ -247,6 +247,10 @@ Note that the result of a computation is usually residing in the memory of the computation device, so with `.toSliceAlloc()` we bring it back to CPU memory in the form of a `Slice`. After that, we can print it. +`call(io, args, &results, .{})` enqueues execution and returns immediately. Use +`call(io, args, &results, .{ .wait = true })` to wait for execution to finish before returning. +Here, `toSliceAlloc()` waits for the result before reading it. + And that's it! Now, let's have a look at building and actually running this example! @@ -385,7 +389,7 @@ pub fn main(init: std.process.Init) !void { args.set(.{ layer_buffers, input_buffer }); // call our executable - executable.call(args, &results); + executable.call(io, args, &results, .{}); // Retrieve the resulting buffer var result = results.get(zml.Buffer); diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index c1e4086e..1bdaad7b 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -68,14 +68,14 @@ pub fn main(init: std.process.Init) !void { // Ignore first run { - exe.call(exe_args, &exe_results); + exe.call(io, exe_args, &exe_results, .{}); var result = exe_results.get(zml.Buffer); defer result.deinit(); } // call our executable module const run_start: std.Io.Timestamp = .now(io, .awake); - exe.call(exe_args, &exe_results); + exe.call(io, exe_args, &exe_results, .{}); var result = exe_results.get(zml.Buffer); _ = try result.await(io); defer result.deinit(); diff --git a/examples/llm/models/lfm2/inference.zig b/examples/llm/models/lfm2/inference.zig index a39aba0f..407df36e 100644 --- a/examples/llm/models/lfm2/inference.zig +++ b/examples/llm/models/lfm2/inference.zig @@ -161,7 +161,7 @@ pub fn run( .tokens = args.tokens_buf.*, }, .outputs = .{ .hidden = &hidden_buffer }, - }); + }, .{}); defer hidden_buffer.deinit(); for (runner.layers) |*layer| { @@ -181,7 +181,7 @@ pub fn run( .conv_cache_index = conv_cache_index_buffer, .kv_cache_index = kv_cache_index_buffer, }, - }); + }, .{}); } runner.sample.run(args.io, .{ @@ -194,7 +194,7 @@ pub fn run( .tokens = args.tokens_buf, .rng = args.rng_buf, }, - }); + }, .{}); } fn compileKernel( diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig index 1347403a..d3f8423d 100644 --- a/examples/llm/models/lfm2_tests.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -169,7 +169,7 @@ const TestContext = struct { var res = try exe.results(self.allocator); defer res.deinit(self.allocator); - exe.call(args, &res); + exe.call(self.io, args, &res, .{}); var out_result = res.get(zml.Buffer); defer out_result.deinit(); @@ -229,7 +229,7 @@ const TestContext = struct { var res = try exe.results(self.allocator); defer res.deinit(self.allocator); - exe.call(args, &res); + exe.call(self.io, args, &res, .{}); var out_result = res.get(zml.Buffer); defer out_result.deinit(); @@ -291,7 +291,7 @@ const TestContext = struct { var res = try exe.results(self.allocator); defer res.deinit(self.allocator); - exe.call(args, &res); + exe.call(self.io, args, &res, .{}); var out_result, var updated_kv = res.get(struct { zml.Buffer, zml.Bufferized(model.KvCache) }); defer out_result.deinit(); diff --git a/examples/llm/models/llama/inference.zig b/examples/llm/models/llama/inference.zig index 8a150e65..7513a3ad 100644 --- a/examples/llm/models/llama/inference.zig +++ b/examples/llm/models/llama/inference.zig @@ -194,7 +194,7 @@ pub fn run(runner: *KernelRunner, args: Args) void { .kv_cache = args.kv_cache_buffers, .rng = args.rng_buffers, }, - }); + }, .{}); } fn compileKernel( diff --git a/examples/llm/models/llama_tests.zig b/examples/llm/models/llama_tests.zig index 537c04a4..98798887 100644 --- a/examples/llm/models/llama_tests.zig +++ b/examples/llm/models/llama_tests.zig @@ -126,7 +126,7 @@ fn testLayer( var res = try exe.results(allocator); defer res.deinit(allocator); - exe.call(args, &res); + exe.call(io, args, &res, .{}); var out_result = res.get(zml.Buffer); defer out_result.deinit(); diff --git a/examples/llm/models/qwen3_5/inference.zig b/examples/llm/models/qwen3_5/inference.zig index 83825352..33a41086 100644 --- a/examples/llm/models/qwen3_5/inference.zig +++ b/examples/llm/models/qwen3_5/inference.zig @@ -158,7 +158,7 @@ pub fn run(runner: *KernelRunner, args: Args, layer_index_buffers: []const zml.B .tokens = args.tokens_buf.*, }, .outputs = .{ .hidden = &hidden_buffer }, - }); + }, .{}); defer hidden_buffer.deinit(); for (runner.layers, layer_index_buffers) |*layer_runner, layer_index_buffer| { @@ -176,7 +176,7 @@ pub fn run(runner: *KernelRunner, args: Args, layer_index_buffers: []const zml.B .cache = layer_cache, }, .outputs = .{ .hidden = &hidden_buffer, .cache = &layer_cache }, - }); + }, .{}); args.kv_cache_buffers.self_attn.k = layer_cache.k; args.kv_cache_buffers.self_attn.v = layer_cache.v; }, @@ -193,7 +193,7 @@ pub fn run(runner: *KernelRunner, args: Args, layer_index_buffers: []const zml.B .cache = layer_cache, }, .outputs = .{ .hidden = &hidden_buffer, .cache = &layer_cache }, - }); + }, .{}); args.kv_cache_buffers.gated_delta_net.conv_state = layer_cache.conv_state; args.kv_cache_buffers.gated_delta_net.recurrent_state = layer_cache.recurrent_state; }, @@ -211,7 +211,7 @@ pub fn run(runner: *KernelRunner, args: Args, layer_index_buffers: []const zml.B .rng = args.rng_buffers, .token_index = args.token_index_buf, }, - }); + }, .{}); } fn compileKernel( diff --git a/examples/llm/models/qwen3_5_moe/inference.zig b/examples/llm/models/qwen3_5_moe/inference.zig index a8c05bf4..c31f79cd 100644 --- a/examples/llm/models/qwen3_5_moe/inference.zig +++ b/examples/llm/models/qwen3_5_moe/inference.zig @@ -161,7 +161,7 @@ pub fn run(runner: *KernelRunner, args: RunArgs) void { .tokens = args.tokens_buffer.*, }, .outputs = .{ .hidden = &hidden_buffer }, - }); + }, .{}); defer hidden_buffer.deinit(); for (runner.layers, args.layer_index_buffers) |*layer_runner, layer_index_buffer| { @@ -183,7 +183,7 @@ pub fn run(runner: *KernelRunner, args: RunArgs) void { .cache = layer_cache, }, .outputs = .{ .hidden = &hidden_buffer, .cache = &layer_cache }, - }); + }, .{}); args.kv_cache_buffers.self_attn.k = layer_cache.k; args.kv_cache_buffers.self_attn.v = layer_cache.v; }, @@ -204,7 +204,7 @@ pub fn run(runner: *KernelRunner, args: RunArgs) void { .cache = layer_cache, }, .outputs = .{ .hidden = &hidden_buffer, .cache = &layer_cache }, - }); + }, .{}); args.kv_cache_buffers.gated_delta_net.conv_state = layer_cache.conv_state; args.kv_cache_buffers.gated_delta_net.recurrent_state = layer_cache.recurrent_state; }, @@ -222,7 +222,7 @@ pub fn run(runner: *KernelRunner, args: RunArgs) void { .rng = args.rng_buffers, .token_index = args.token_index_buffer, }, - }); + }, .{}); } fn compileKernel( diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 63529e1c..3b4349c4 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -143,7 +143,7 @@ pub fn main(init: std.process.Init) !void { printDigit(sample); args.set(.{ mnist_buffers, input_buffer }); - exe.call(args, &results); + exe.call(io, args, &results, .{}); var result: zml.Buffer = results.get(zml.Buffer); defer result.deinit(); diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index 37b8c68b..61a16318 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -243,7 +243,7 @@ pub fn main(init: std.process.Init) !void { defer exe_results.deinit(allocator); exe_args.set(.{ w_buf, b_buf, input_buf }); - exe.call(exe_args, &exe_results); + exe.call(io, exe_args, &exe_results, .{}); var out = exe_results.get(zml.Buffer); defer out.deinit(); diff --git a/zml/attention.zig b/zml/attention.zig index 8549ba89..b3701b48 100644 --- a/zml/attention.zig +++ b/zml/attention.zig @@ -2,8 +2,8 @@ const std = @import("std"); pub const attnd = @import("attention/attnd.zig"); pub const cute = @import("attention/cute_kernels/sparse_mla.zig"); -pub const fly = @import("attention/fly_kernels/sparse_mla.zig"); pub const flashattn = @import("attention/flashattn.zig"); +pub const fly = @import("attention/fly_kernels/sparse_mla.zig"); pub const metal = @import("attention/metal_attention.zig"); pub const nki = @import("attention/nki/attention.zig"); pub const paged_attention = @import("attention/paged_attention.zig"); @@ -264,10 +264,14 @@ pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []co const rng_k = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.k.shape(), .{ .mean = 0, .stddev = 1 } }, .{}); defer rng_k.deinit(); - const q = try zml.testing.autoCall(allocator, io, &rng_q, zml.Tensor.Rng.normal, {}); - const k = try zml.testing.autoCall(allocator, io, &rng_k, zml.Tensor.Rng.normal, {}); - const v = try zml.testing.autoCall(allocator, io, &rng_k, zml.Tensor.Rng.normal, {}); - const token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, .replicated, @ptrCast(token_index_h)); + var q = try rng_q.eval(allocator, io, {}); + defer q.deinit(); + var k = try rng_k.eval(allocator, io, {}); + defer k.deinit(); + var v = try rng_k.eval(allocator, io, {}); + defer v.deinit(); + var token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, .replicated, @ptrCast(token_index_h)); + defer token_index.deinit(); const shardings = platform.shardings.values(); const vanilla_exe = try platform.compileFn(allocator, io, attention, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, .vanilla, .vanilla }, .{ @@ -276,7 +280,8 @@ pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []co }); defer vanilla_exe.deinit(); - const vanilla_d = try zml.testing.autoCall(allocator, io, &vanilla_exe, attention, .{ q, k, v, token_index, .vanilla }); + var vanilla_d = try vanilla_exe.eval(allocator, io, .{ q, k, v, token_index, .vanilla }); + defer vanilla_d.deinit(); try vanilla_d.await(io); const vanilla_h: zml.Slice = try vanilla_d.toSliceAlloc(allocator, io); defer vanilla_h.free(allocator); @@ -304,7 +309,7 @@ pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []co var metadata_d = try metadata.initBuffer(io, platform, platform.shardings.get("model").?); defer Metadata.deinitBuffer(&metadata_d); - var output_d = try zml.testing.autoCall(allocator, io, &exe, attention, .{ q, k, v, token_index, metadata_d }); + var output_d = try exe.eval(allocator, io, .{ q, k, v, token_index, metadata_d }); defer output_d.deinit(); try output_d.await(io); const output_h = try output_d.toSliceAlloc(allocator, io); diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index eda324a9..26bf76ef 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -3,9 +3,9 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../zml.zig"); +const cute = @import("cute_kernels/sparse_mla.zig"); const flashattn = @import("flashattn.zig"); const metal = @import("metal_attention.zig"); -const cute = @import("cute_kernels/sparse_mla.zig"); const sparse_mla = @import("sparse_mla.zig"); const tpu = @import("tpu_attention.zig"); const triton = @import("triton_attention.zig"); @@ -402,18 +402,18 @@ test pagedAttention { .max_seqlen_q = 16 * 2, }; const triton_parameters: Parameters = .init(.fromBackend(triton_options_args)); - var q = try zml.testing.autoCall(allocator, io, &rng_q, zml.Tensor.Rng.normal, {}); + var q = try rng_q.eval(allocator, io, {}); defer q.deinit(); - var new_k = try zml.testing.autoCall(allocator, io, &rng_k, zml.Tensor.Rng.normal, {}); + var new_k = try rng_k.eval(allocator, io, {}); defer new_k.deinit(); - var new_v = try zml.testing.autoCall(allocator, io, &rng_k, zml.Tensor.Rng.normal, {}); + var new_v = try rng_k.eval(allocator, io, {}); defer new_v.deinit(); - var kv_cache_d: zml.Bufferized(KvCache) = .{ .split = .{ - .k = try zml.testing.autoCall(allocator, io, &rng_kv_cache, zml.Tensor.Rng.normal, {}), - .v = try zml.testing.autoCall(allocator, io, &rng_kv_cache, zml.Tensor.Rng.normal, {}), - } }; - defer zml.Buffer.deinitAll(KvCache, &kv_cache_d); + var kv_cache_d: zml.Bufferized(KvCache) = .{ .split = undefined }; + kv_cache_d.split.k = try rng_kv_cache.eval(allocator, io, .{}); + defer kv_cache_d.split.k.deinit(); + kv_cache_d.split.v = try rng_kv_cache.eval(allocator, io, .{}); + defer kv_cache_d.split.v.deinit(); const block_table: [batch_size][max_num_pages]i32 = .{ // prefilling pages 9-10 @@ -544,7 +544,7 @@ test pagedAttention { // cu_fa2 creates new buffers while other reuse triton buffers. defer if (backend == .cuda_fa2) zml.Buffer.deinitAll(Parameters, ¶meters_d); - var output_d = try zml.testing.autoCall(allocator, io, &exe, pagedAttention, .{ parameters_d, q, new_k, new_v, kv_cache_d, .{} }); + var output_d = try exe.eval(allocator, io, .{ parameters_d, q, new_k, new_v, kv_cache_d, .{} }); defer output_d.deinit(); results_per_backend.set(backend, try output_d.toSliceAlloc(allocator, io)); } @@ -1076,11 +1076,9 @@ test "Triton sparse MLA value ranks and padded queries" { ); defer exe.deinit(); - var output_d = try zml.testing.autoCall( + var output_d = try exe.eval( std.testing.allocator, std.testing.io, - &exe, - Mla.pagedSparseAttention, .{.{ .q = q_d, .kv = .{ .cache = .{ .latent = kv_d }, .parameters = parameters_d, .positions = topk_d }, .compressed = null, .sink = sink_d, .tokens_pos = tokens_pos_d }}, ); defer output_d.deinit(); @@ -1183,11 +1181,9 @@ test "execute stablehlo mla kernel" { ); defer exe.deinit(); - var output_d = try zml.testing.autoCall( + var output_d = try exe.eval( std.testing.allocator, std.testing.io, - &exe, - Mla.pagedSparseAttention, .{.{ .q = q_d, .kv = .{ .cache = .{ .latent = kv_d }, .parameters = parameters_d, .positions = topk_d }, .compressed = null, .sink = sink_d, .tokens_pos = tokens_pos_d }}, ); defer zml.Buffer.deinitAll(zml.Tensor, &output_d); diff --git a/zml/exe.zig b/zml/exe.zig index 890756b3..c4d39b04 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const builtin = @import("builtin"); const pjrt = @import("pjrt"); const stdx = @import("stdx"); @@ -262,6 +263,17 @@ pub const Exe = struct { } }; + /// Executes a single-output executable and waits for its result. + /// The caller owns the returned buffer. + pub fn eval(self: *const Exe, allocator: std.mem.Allocator, io: std.Io, input_values: anytype) !Buffer { + stdx.debug.assert(self.output_shapes.len == 1, "eval expects one output, got {d}; use a runner for multiple outputs", .{self.output_shapes.len}); + var result: Buffer = undefined; + var r = try self.runner(allocator); + defer r.deinit(allocator); + r.run(io, input_values, .{&result}, .{ .wait = true }); + return result; + } + pub fn runner(self: *const Exe, allocator: std.mem.Allocator) !Runner { return .init(self, allocator); } @@ -290,21 +302,26 @@ pub const Exe = struct { self.args.deinit(allocator); } - pub fn run(self: *Runner, input_values: anytype, output_values: anytype) void { + pub fn run(self: *Runner, io: std.Io, input_values: anytype, output_refs: anytype, opts: CallOpts) void { self.args.set(input_values); - self.exe.call(self.args, &self.results); - self.results.fill(output_values); - } - - pub fn runOpts(self: *Runner, io: std.Io, input_values: anytype, output_values: anytype, opts: CallOpts) void { - self.args.set(input_values); - self.exe.callOpts(io, self.args, &self.results, opts); - self.results.fill(output_values); + self.exe.call(io, self.args, &self.results, opts); + self.results.fill(output_refs); } }; - pub fn internalCall(self: *const Exe, io: ?std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { - stdx.debug.assert(opts.wait == false or io != null, "io should not be null when waiting for execution completion", .{}); + pub const CallOpts = struct { + /// Waits for execution to complete before returning. + wait: bool = false, + }; + + pub fn call(self: *const Exe, io: std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { + var span = tracer.span("zml.exe.call", .{ + .wait = opts.wait, + .arg_count = arguments.expected_shapes.len, + .result_count = results_.expected_shapes.len, + }); + defer span.end(); + var events: [Platform.MAX_NUM_DEVICES]?*pjrt.Event = @splat(null); const partition_events = events[0..@intCast(self.num_partitions)]; @@ -332,7 +349,7 @@ pub const Exe = struct { for (events_slice.?) |e| { if (e) |ev| { if (opts.wait) { - ev.await(self.platform.pjrt_api, io.?) catch |err| { + ev.await(self.platform.pjrt_api, io) catch |err| { std.debug.panic("PJRT execution failed with: {}", .{err}); }; } @@ -343,7 +360,7 @@ pub const Exe = struct { .cpu, .cuda, .rocm, .tpu, .oneapi, .metal => if (opts.wait) { for (events_slice.?) |e| { if (e) |ev| { - ev.await(self.platform.pjrt_api, io.?) catch |err| { + ev.await(self.platform.pjrt_api, io) catch |err| { std.debug.panic("PJRT execution failed with: {}", .{err}); }; ev.deinit(self.platform.pjrt_api); @@ -352,30 +369,6 @@ pub const Exe = struct { }, } } - - pub const CallOpts = struct { - wait: bool = false, - }; - - pub fn callOpts(self: *const Exe, io: std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { - var span = tracer.span("zml.exe.call", .{ - .wait = opts.wait, - .arg_count = arguments.expected_shapes.len, - .result_count = results_.expected_shapes.len, - }); - defer span.end(); - return self.internalCall(io, arguments, results_, opts); - } - - pub fn call(self: *const Exe, arguments: Arguments, results_: *Results) void { - var span = tracer.span("zml.exe.call", .{ - .wait = false, - .arg_count = arguments.expected_shapes.len, - .result_count = results_.expected_shapes.len, - }); - defer span.end(); - return self.internalCall(null, arguments, results_, .{}); - } }; /// A typed executable whose input and output buffer structures are derived from @@ -451,13 +444,14 @@ pub fn FnExe(comptime function_: anytype) type { self.args.deinit(allocator); } - pub fn run(self: *RunnerSelf, io: std.Io, call: struct { + pub const Call = struct { inputs: NonBakedInput, outputs: Output, - opts: Exe.CallOpts = .{}, - }) void { + }; + + pub fn run(self: *RunnerSelf, io: std.Io, call: Call, opts: Exe.CallOpts) void { self.args.set(call.inputs); - self.exe.callOpts(io, self.args, &self.results, call.opts); + self.exe.call(io, self.args, &self.results, opts); self.results.fill(call.outputs); } }; diff --git a/zml/io.zig b/zml/io.zig index b45bf619..7d29a352 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -544,7 +544,7 @@ pub const Loader = struct { var results = try exe.results(arena); args.set(.{buffers}); - exe.callOpts(io, args, &results, .{ .wait = true }); + exe.call(io, args, &results, .{ .wait = true }); buffer.* = results.get(Buffer); try self.delivered.put(self.allocator, tensor.id, {}); diff --git a/zml/kernel.zig b/zml/kernel.zig index 7c018cfb..c2114001 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -1,12 +1,12 @@ const std = @import("std"); -const stdx = @import("stdx"); -const dialects = @import("mlir/dialects"); -const mlir = @import("mlir"); const cuda_tile_builder = @import("kernels/cuda_tile/builder"); const cute_builder = @import("kernels/cute/builder"); +const dialects = @import("mlir/dialects"); const fly_builder = @import("kernels/fly/builder"); +const mlir = @import("mlir"); const mosaic_tpu_builder = @import("kernels/mosaic_tpu/builder"); +const stdx = @import("stdx"); const tpu_dialect = @import("mlir/dialects/mosaic_tpu"); const triton_builder = @import("kernels/triton/builder"); @@ -959,7 +959,7 @@ test "fly kernels run on rocm" { defer ba.deinit(); var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), .replicated, std.mem.sliceAsBytes(hb)); defer bb.deinit(); - var out = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ ba, bb }); + var out = try exe.eval(std.testing.allocator, std.testing.io, .{ ba, bb }); defer out.deinit(); return out.toSliceAlloc(std.testing.allocator, std.testing.io); } diff --git a/zml/moe/fused_experts.zig b/zml/moe/fused_experts.zig index 9d492e10..242e13e4 100644 --- a/zml/moe/fused_experts.zig +++ b/zml/moe/fused_experts.zig @@ -214,7 +214,7 @@ test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, threshold }, .{}); defer exe.deinit(); try zml.testing.expectEqualShapes(Shape.init(.{ .token = 1, .out = 3 }, .f32), exe.output_shapes[0]); - var output = try zml.testing.autoCall(allocator, io, &exe, Local.forward, .{input}); + var output = try exe.eval(allocator, io, .{input}); defer output.deinit(); var actual = try output.toSliceAlloc(allocator, io); defer actual.free(allocator); @@ -244,7 +244,7 @@ test "ReLU squared activation preserves width and applies threshold before squar var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, threshold }, .{}); defer exe.deinit(); try zml.testing.expectEqualShapes(x.shape(), exe.output_shapes[0]); - var output = try zml.testing.autoCall(allocator, io, &exe, Local.forward, .{input}); + var output = try exe.eval(allocator, io, .{input}); defer output.deinit(); var actual = try output.toSliceAlloc(allocator, io); defer actual.free(allocator); @@ -383,7 +383,10 @@ test "FP8 routed GEMM with bias matches dequantized weights" { buffers[i] = try zml.Buffer.fromSlice(io, platform, slice, .replicated); initialized += 1; } - var output = try zml.testing.autoCall(allocator, io, &exe, Local.forward, .{ buffers[0], buffers[1], buffers[2], buffers[3], buffers[4], buffers[5] }); + var output: zml.Bufferized(Local.Outputs) = undefined; + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + runner.run(io, .{ buffers[0], buffers[1], buffers[2], buffers[3], buffers[4], buffers[5] }, .{&output}, .{ .wait = true }); defer zml.Buffer.deinitAll(Local.Outputs, &output); zml.testing.expectClose(io, output.expected, output.actual, .{ .absolute_tolerance = 0.03125, .relative_tolerance = 0.01 }) catch |err| { log.warn("FP8 routed GEMM failed for scheme={s}, quantize_input={}, tokens={}", .{ @tagName(scheme), quantize_input, tokens }); @@ -461,7 +464,7 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { for ([_]RoutingWeightPlacement{ .before_down, .after_down }) |placement| { var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, placement, storage_dtype }, .{}); defer exe.deinit(); - var output = try zml.testing.autoCall(allocator, io, &exe, Local.forward, .{input}); + var output = try exe.eval(allocator, io, .{input}); defer output.deinit(); var actual = try output.toSliceAlloc(allocator, io); defer actual.free(allocator); diff --git a/zml/nn.zig b/zml/nn.zig index 9d46a5ba..a49865fa 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -293,7 +293,10 @@ test "block128 scaled dot layouts" { buffers[i] = try zml.Buffer.fromSlice(io, platform, slice, .replicated); initialized += 1; } - var output = try zml.testing.autoCall(allocator, io, &exe, Local.forward, .{ buffers[0], buffers[1], buffers[2] }); + var output: zml.Bufferized(Local.Outputs) = undefined; + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + runner.run(io, .{ buffers[0], buffers[1], buffers[2] }, .{&output}, .{ .wait = true }); defer zml.Buffer.deinitAll(Local.Outputs, &output); var expected = try output.expected.toSliceAlloc(allocator, io); defer expected.free(allocator); @@ -443,7 +446,7 @@ test normalizeL2 { var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&[_]f32{ -0.9686, -1.0058, -1.7808, 0.6698 })); defer input_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, normalizeL2, .{input_buffer}); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); defer res.deinit(); const expectation: Slice = .init(input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6937, -0.7203, -0.9360, 0.3520 })); @@ -963,7 +966,7 @@ test "real/img" { var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.interleaved}, .{}); defer exe.deinit(); - var d_interleaved = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitMergeIsId, {}); + var d_interleaved = try exe.eval(std.testing.allocator, std.testing.io, {}); defer d_interleaved.deinit(); try std.testing.expectEqual(20, try d_interleaved.getValue(i32, std.testing.io)); } @@ -971,7 +974,7 @@ test "real/img" { var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.real_im_pass}, .{}); defer exe.deinit(); - var d_sequential = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitMergeIsId, {}); + var d_sequential = try exe.eval(std.testing.allocator, std.testing.io, {}); defer d_sequential.deinit(); try std.testing.expectEqual(20, try d_sequential.getValue(i32, std.testing.io)); } @@ -981,7 +984,7 @@ test "real/img" { var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Fns.testSplitSeqVoid, .{{}}, .{}); defer exe.deinit(); - var d_split_seq_void = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitSeqVoid, {}); + var d_split_seq_void = try exe.eval(std.testing.allocator, std.testing.io, {}); defer d_split_seq_void.deinit(); try std.testing.expectEqual(20, try d_split_seq_void.getValue(i32, std.testing.io)); } @@ -991,7 +994,7 @@ test "real/img" { var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Fns.testSplitSeq, .{}, .{}); defer exe.deinit(); - var d_split_seq = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitSeq, {}); + var d_split_seq = try exe.eval(std.testing.allocator, std.testing.io, {}); defer d_split_seq.deinit(); try std.testing.expectEqual(20, try d_split_seq.getValue(i32, std.testing.io)); } @@ -1000,7 +1003,7 @@ test "real/img" { var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Fns.testSplitInterleaved, .{}, .{}); defer exe.deinit(); - var d_split_seq = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitInterleaved, {}); + var d_split_seq = try exe.eval(std.testing.allocator, std.testing.io, {}); defer d_split_seq.deinit(); try std.testing.expectEqual(20, try d_split_seq.getValue(i32, std.testing.io)); } @@ -1041,9 +1044,9 @@ test rope { var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&x_values)); defer x_buffer.deinit(); - var res1 = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_interleaved, Local._fwd, .{x_buffer}); + var res1 = try exe_interleaved.eval(std.testing.allocator, std.testing.io, .{x_buffer}); defer res1.deinit(); - var res2 = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_sequential, Local._fwd, .{x_buffer}); + var res2 = try exe_sequential.eval(std.testing.allocator, std.testing.io, .{x_buffer}); defer res2.deinit(); try zml.testing.expectClose(std.testing.io, res1, res2, .{}); @@ -1084,7 +1087,7 @@ test "rope: Proportional" { }; const expected: zml.Slice = .init(x.shape(), std.mem.sliceAsBytes(&expected_h)); - var res = try zml.testing.autoCall(allocator, io, &exe, rope, .{ x_buffer, null }); + var res = try exe.eval(allocator, io, .{ x_buffer, null }); defer res.deinit(); const res_h = try res.toSliceAlloc(allocator, io); defer res_h.free(allocator); @@ -1136,7 +1139,7 @@ test "rope: Yarn with partial_rotary_factor" { }; const expected: zml.Slice = .init(x.shape(), std.mem.sliceAsBytes(&expected_h)); - var res = try zml.testing.autoCall(allocator, io, &exe, rope, .{ x_buffer, null }); + var res = try exe.eval(allocator, io, .{ x_buffer, null }); defer res.deinit(); const res_h = try res.toSliceAlloc(allocator, io); defer res_h.free(allocator); @@ -1225,7 +1228,7 @@ test nearest { ); defer input_3d_basic_buffer.deinit(); - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_3d_basic_buffer}); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_3d_basic_buffer}); defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 1, 1, 6 }, result.shape().dims()); @@ -1251,7 +1254,7 @@ test nearest { })); defer input_3d_advanced_buffer.deinit(); - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_3d_advanced_buffer}); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_3d_advanced_buffer}); defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 2, 3, 8 }, result.shape().dims()); @@ -1285,7 +1288,7 @@ test nearest { var input_4d_basic_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_basic.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); defer input_4d_basic_buffer.deinit(); - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_4d_basic_buffer}); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_4d_basic_buffer}); defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 1, 1, 6, 6 }, result.shape().dims()); @@ -1320,7 +1323,7 @@ test nearest { } })); defer input_4d_advanced_buffer.deinit(); - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_4d_advanced_buffer}); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_4d_advanced_buffer}); defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 2, 2, 4, 4 }, result.shape().dims()); @@ -1371,7 +1374,7 @@ test nearest { var input_5d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_5d.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); defer input_5d_buffer.deinit(); - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_5d_buffer}); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_5d_buffer}); defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 1, 1, 2, 4, 4 }, result.shape().dims()); @@ -1883,12 +1886,14 @@ test "gated delta net" { var oversized_length_buffer = try zml.Buffer.scalar(std.testing.io, platform, @as(u32, 3), .u32); defer oversized_length_buffer.deinit(); - var result = try zml.testing.autoCall( - std.testing.allocator, + var result: zml.Bufferized(GatedDeltaNet.Output) = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run( std.testing.io, - &exe, - GatedDeltaNet.forward, .{ queries_buffer, keys_buffer, values_buffer, alphas_buffer, betas_buffer, .{ .s = initial_s_buffer }, oversized_length_buffer }, + .{&result}, + .{ .wait = true }, ); defer result.outputs.deinit(); defer result.state.s.deinit(); @@ -1919,12 +1924,12 @@ test "gated delta net" { var prefix_length_buffer = try zml.Buffer.scalar(std.testing.io, platform, @as(u32, 1), .u32); defer prefix_length_buffer.deinit(); - var prefix_result = try zml.testing.autoCall( - std.testing.allocator, + var prefix_result: zml.Bufferized(GatedDeltaNet.Output) = undefined; + runner.run( std.testing.io, - &exe, - GatedDeltaNet.forward, .{ queries_buffer, keys_buffer, values_buffer, alphas_buffer, betas_buffer, .{ .s = initial_s_buffer }, prefix_length_buffer }, + .{&prefix_result}, + .{ .wait = true }, ); defer prefix_result.outputs.deinit(); defer prefix_result.state.s.deinit(); @@ -2014,7 +2019,10 @@ test sampleTokens { var activations_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, activations.shape(), .replicated, std.mem.sliceAsBytes(&logits)); defer activations_buffer.deinit(); - var sampled, rng_buffer = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, sampleTokens, .{ activations_buffer, rng_buffer }); + var sampled: zml.Buffer = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{ activations_buffer, rng_buffer }, .{ &sampled, &rng_buffer }, .{ .wait = true }); defer sampled.deinit(); try zml.testing.expectEqual(expected, try sampled.getValue(i32, std.testing.io)); @@ -2162,7 +2170,11 @@ test sampleTokensDynamic { const args, const expected = args_expected; var dynamic_sampling_strategy_buffers = try DynamicSamplingStrategy.makeBuffers(std.testing.io, platform, .f32, args); defer DynamicSamplingStrategy.deinitBuffers(&dynamic_sampling_strategy_buffers); - var new_logits, var indices = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, fixupLogits, .{ logits_buffer, dynamic_sampling_strategy_buffers }); + var new_logits: zml.Buffer = undefined; + var indices: zml.Buffer = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{ logits_buffer, dynamic_sampling_strategy_buffers }, .{ &new_logits, &indices }, .{ .wait = true }); defer new_logits.deinit(); defer indices.deinit(); try std.testing.expectEqual(top_k_indices, try indices.getValue(@TypeOf(top_k_indices), std.testing.io)); @@ -2186,7 +2198,13 @@ test sampleTokensDynamic { var dynamic_sampling_strategy_bf16_buffers = try DynamicSamplingStrategy.makeBuffers(std.testing.io, platform, .bf16, .{ .top_k = 4, .top_p = 0.9, .min_p = 0.1 }); defer DynamicSamplingStrategy.deinitBuffers(&dynamic_sampling_strategy_bf16_buffers); - var new_logits, var indices = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_bf16, fixupLogits, .{ logits_bf16_buffer, dynamic_sampling_strategy_bf16_buffers }); + var new_logits: zml.Buffer = undefined; + var indices: zml.Buffer = undefined; + var runner = try exe_bf16.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{ logits_bf16_buffer, dynamic_sampling_strategy_bf16_buffers }, .{ &new_logits, &indices }, .{ .wait = true }); + defer new_logits.deinit(); + defer indices.deinit(); try std.testing.expectEqual([_]i32{ 0, 1, 2, 3 }, try indices.getValue([4]i32, std.testing.io)); try zml.testing.expectEqual([_]bf16{ boost, nerf, nerf, nerf }, try new_logits.getValue([4]bf16, std.testing.io)); diff --git a/zml/ops.zig b/zml/ops.zig index adaa3966..fd97ceb1 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -640,7 +640,10 @@ test @"while" { var sum_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_sum.shape(), .replicated, std.mem.sliceAsBytes(&[1]i64{0})); defer sum_buffer.deinit(); - var results = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, While.forward, .{ i_buffer, sum_buffer }); + var results: zml.Bufferized(While.State) = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{ i_buffer, sum_buffer }, .{&results}, .{ .wait = true }); defer results.i.deinit(); defer results.sum.deinit(); @@ -1054,7 +1057,7 @@ test "fly custom call, both entry ABIs" { defer exe.deinit(); var buf: zml.Buffer = try .fromBytes(std.testing.io, p, t.shape(), .replicated, std.mem.sliceAsBytes(h)); defer buf.deinit(); - var out = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{buf}); + var out = try exe.eval(std.testing.allocator, std.testing.io, .{buf}); defer out.deinit(); return out.toSliceAlloc(std.testing.allocator, std.testing.io); } @@ -1276,7 +1279,10 @@ test "triton" { var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), .replicated, std.mem.sliceAsBytes(&[1]f32{3})); defer b_buffer.deinit(); - var results = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, TritonMod.forward, .{ a_buffer, b_buffer }); + var results: [2]zml.Buffer = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{ a_buffer, b_buffer }, .{&results}, .{ .wait = true }); defer results[0].deinit(); defer results[1].deinit(); @@ -1494,7 +1500,7 @@ test "cute" { var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&input)); defer a_buffer.deinit(); - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Mod.forward, .{a_buffer}); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); defer result.deinit(); var host = try result.toSliceAlloc(std.testing.allocator, std.testing.io); @@ -1553,7 +1559,7 @@ test "cuda_tile" { var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&input)); defer a_buffer.deinit(); - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Mod.forward, .{a_buffer}); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); defer result.deinit(); var host = try result.toSliceAlloc(std.testing.allocator, std.testing.io); @@ -1620,7 +1626,7 @@ test "cuda_tile grid" { var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&input)); defer a_buffer.deinit(); - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Mod.forward, .{a_buffer}); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); defer result.deinit(); var host = try result.toSliceAlloc(std.testing.allocator, std.testing.io); @@ -1680,7 +1686,7 @@ test "cuda_tile zeroed_outputs" { var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&input)); defer a_buffer.deinit(); - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Mod.forward, .{a_buffer}); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); defer result.deinit(); var host = try result.toSliceAlloc(std.testing.allocator, std.testing.io); diff --git a/zml/quantization.zig b/zml/quantization.zig index b8beb352..1a058e0f 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -391,7 +391,10 @@ test "block FP8 quantization preserves axes and reconstructs constant blocks in } var buffer = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&host)); defer buffer.deinit(); - var output = try zml.testing.autoCall(allocator, io, &exe, Local.forward, .{buffer}); + var output: zml.Bufferized(Local.Outputs) = undefined; + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + runner.run(io, .{buffer}, .{&output}, .{ .wait = true }); defer zml.Buffer.deinitAll(Local.Outputs, &output); var values = try output.values.toSliceAlloc(allocator, io); defer values.free(allocator); @@ -448,7 +451,10 @@ test "block FP8 E8M0 scales preserve power-of-two boundaries and nonminor axes" } var buffer = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&host)); defer buffer.deinit(); - var output = try zml.testing.autoCall(allocator, io, &exe, Local.forward, .{buffer}); + var output: zml.Bufferized(Local.Outputs) = undefined; + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + runner.run(io, .{buffer}, .{&output}, .{ .wait = true }); defer zml.Buffer.deinitAll(Local.Outputs, &output); var values = try output.values.toSliceAlloc(allocator, io); defer values.free(allocator); diff --git a/zml/tensor.zig b/zml/tensor.zig index 38f9c548..3e5dea04 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -263,7 +263,7 @@ pub const Tensor = struct { const x_h_ptr: [*]f32 = @ptrCast(@alignCast(x_h.opaqueDevicePtr(0))); try std.testing.expectEqualSlices(f32, &inputs, x_h_ptr[0..8]); - var x_d = try zml.testing.autoCall(std.testing.allocator, io, &exe, Local.memcpyH2D, .{x_h}); + var x_d = try exe.eval(std.testing.allocator, io, .{x_h}); defer x_d.deinit(); try zml.testing.expectClose(std.testing.io, x_h, x_d, .exact_match); @@ -406,7 +406,13 @@ pub const Tensor = struct { y_memory[dev] = y_d.opaqueDevicePtr(dev); } - const y2_d, const z2_d = try zml.testing.autoCall(std.testing.allocator, io, &exe, Local.memcopy, .{ x_d, y_d, z_d }); + var y2_d: zml.Buffer = undefined; + var z2_d: zml.Buffer = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(io, .{ x_d, y_d, z_d }, .{ &y2_d, &z2_d }, .{ .wait = true }); + defer y2_d.deinit(); + defer z2_d.deinit(); var output_memory: [Platform.MAX_NUM_DEVICES]*anyopaque = undefined; for (0..y2_d.numShards()) |dev| { @@ -526,7 +532,8 @@ pub const Tensor = struct { var input_buffer = try zml.Buffer.fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&i)); defer input_buffer.deinit(); - const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.fmod, .{input_buffer}); + var output = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); + defer output.deinit(); try zml.testing.expectClose(std.testing.io, zml.Slice.init(zml.Shape.init(.{6}, .f32), std.mem.sliceAsBytes(&e)), output, .{}); } @@ -827,7 +834,11 @@ pub const Tensor = struct { var rng_buffer = try Rng.initBuffer(std.testing.io, platform, .replicated, 1234); defer rng_buffer._state.deinit(); - var rand, var stats = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Stats.uniformStats, .{rng_buffer}); + var rand: zml.Bufferized(Rng) = undefined; + var stats: zml.Bufferized(Stats) = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{rng_buffer}, .{ &rand, &stats }, .{ .wait = true }); defer rand._state.deinit(); defer stats.mean.deinit(); defer stats.variance.deinit(); @@ -937,7 +948,11 @@ pub const Tensor = struct { var tgt_dist_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, tgt_dist.shape(), .replicated, @ptrCast(&tgt_dist_data)); defer tgt_dist_buffer.deinit(); - var rand, var stats = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Stats.gumbelStats, .{ rng_buffer, tgt_dist_buffer }); + var rand: zml.Bufferized(Rng) = undefined; + var stats: zml.Bufferized(Stats) = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{ rng_buffer, tgt_dist_buffer }, .{ &rand, &stats }, .{ .wait = true }); defer rand._state.deinit(); defer stats.mean.deinit(); defer stats.variance.deinit(); @@ -1283,7 +1298,8 @@ pub const Tensor = struct { var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), .replicated, @ptrCast(&x_f32)); defer x_d_buffer.deinit(); - var x_f4_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); + var x_f4_xla_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_d_buffer}); + defer x_f4_xla_d.deinit(); const x_f4_xla = try x_f4_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); defer x_f4_xla.free(std.testing.allocator); const x_f4_items = x_f4_xla.items(floats.Float4E2M1.Packed); @@ -1314,7 +1330,8 @@ pub const Tensor = struct { var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), .replicated, std.mem.sliceAsBytes(&x)); defer x_d_buffer.deinit(); - var x_f8e3_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); + var x_f8e3_xla_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_d_buffer}); + defer x_f8e3_xla_d.deinit(); const x_f8e3_xla = try x_f8e3_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); defer x_f8e3_xla.free(std.testing.allocator); @@ -1343,7 +1360,7 @@ pub const Tensor = struct { var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_u2_t.shape(), .replicated, @ptrCast(&x_u2)); defer x_d_buffer.deinit(); - var x_u8_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); + var x_u8_xla_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_d_buffer}); defer x_u8_xla_d.deinit(); const x_u8_xla = try x_u8_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); defer x_u8_xla.free(std.testing.allocator); @@ -1371,7 +1388,8 @@ pub const Tensor = struct { var x_f4_d: zml.Buffer = try .fromBytes(std.testing.io, platform, x_t.shape(), .replicated, @ptrCast(&x_f4_packed)); defer x_f4_d.deinit(); - var x_f32_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_f4_d}); + var x_f32_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_f4_d}); + defer x_f32_d.deinit(); const x_f32_h = try x_f32_d.toSliceAlloc(std.testing.allocator, std.testing.io); defer x_f32_h.free(std.testing.allocator); @@ -1612,7 +1630,7 @@ pub const Tensor = struct { var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&[_]f32{ -0.6884, 1.6795 })); defer input_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.leakyReLU, .{input_buffer}); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); defer res.deinit(); const expectation: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[2]f32{ -0.0688, 1.6795 })); @@ -1676,7 +1694,7 @@ pub const Tensor = struct { var c_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, c.shape(), .replicated, std.mem.sliceAsBytes(&[3]f32{ 2, 2, 2 })); defer c_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.swiglu, .{ input_buffer, w_buffer, v_buffer, b_buffer, c_buffer }); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{ input_buffer, w_buffer, v_buffer, b_buffer, c_buffer }); defer res.deinit(); const output_shape = Shape.init(.{ .c = 3, .e = 3 }, .f32); @@ -1835,7 +1853,7 @@ pub const Tensor = struct { var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, @ptrCast(&[2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } })); defer x_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._cumsum, .{x_buffer}); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); defer res.deinit(); try std.testing.expectEqual([2][5]f32{ .{ 0, 1, 2, 2, 3 }, .{ 3, 4, 4, 6, 7 } }, try res.getValue([2][5]f32, std.testing.io)); @@ -2147,7 +2165,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._slice1dAxis, .{ x_buffer, .{} }); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{} }); defer res.deinit(); try std.testing.expectEqual(expectation, try res.getValue(@TypeOf(expectation), std.testing.io)); @@ -2180,7 +2198,7 @@ pub const Tensor = struct { var z_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, z.shape(), .replicated, std.mem.asBytes(&z_value)); defer z_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local.dynSliceResolved, .{ x_buffer, .{ .dyn_start = z_buffer } }); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ .dyn_start = z_buffer } }); defer res.deinit(); try std.testing.expectEqual(expectation, try res.getValue(@TypeOf(expectation), std.testing.io)); @@ -2304,7 +2322,8 @@ pub const Tensor = struct { var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&input_data)); defer input_buffer.deinit(); - const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local.repeat1d, .{input_buffer}); + var output = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); + defer output.deinit(); try std.testing.expectEqual(expectation, try output.getValue(@TypeOf(expectation), std.testing.io)); } } @@ -2367,7 +2386,8 @@ pub const Tensor = struct { var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&input_data)); defer input_buffer.deinit(); - const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local.stutter1d, .{input_buffer}); + var output = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); + defer output.deinit(); try std.testing.expectEqual(expectation, try output.getValue(@TypeOf(expectation), std.testing.io)); } } @@ -2472,7 +2492,8 @@ pub const Tensor = struct { ); defer exe.deinit(); - const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.arange, {}); + var output = try exe.eval(std.testing.allocator, std.testing.io, {}); + defer output.deinit(); try zml.testing.expectClose( std.testing.io, @@ -3065,11 +3086,11 @@ pub const Tensor = struct { var temp_exe = try zml.module.compile(std.testing.allocator, std.testing.io, temp_func, .{}, platform, .{}); defer temp_exe.deinit(); - const buffer = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &temp_exe, temp_func, {}); + const buffer = try temp_exe.eval(std.testing.allocator, std.testing.io, {}); break :b buffer; }; defer operand_buffer.deinit(); - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._gatherSlices, .{ operand_buffer, indices_buffer }); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{ operand_buffer, indices_buffer }); defer result.deinit(); const expected: zml.Slice = .init(Shape.init(.{ 2, 2, 2, 3 }, .u16), std.mem.sliceAsBytes(&[2][2][2][3]u16{ @@ -3278,8 +3299,7 @@ pub const Tensor = struct { defer scatter_indices_buffer.deinit(); var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), .replicated, std.mem.sliceAsBytes(&[2][3]i32{ .{ 10, 20, 30 }, .{ 70, 80, 90 } })); defer updates_buffer.deinit(); - - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatter, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); defer result.deinit(); const expected = [3][3]i32{ .{ 10, 21, 32 }, .{ 3, 4, 5 }, .{ 76, 87, 98 } }; @@ -3302,8 +3322,7 @@ pub const Tensor = struct { defer scatter_indices_buffer.deinit(); var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), .replicated, std.mem.sliceAsBytes(&[2]i32{ 20, 70 })); defer updates_buffer.deinit(); - - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatter, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); defer result.deinit(); const expected = [9]i32{ 0, 1, 22, 3, 4, 5, 6, 77, 8 }; @@ -3334,8 +3353,7 @@ pub const Tensor = struct { defer start_indices_buffer.deinit(); var values_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, values.shape(), .replicated, std.mem.sliceAsBytes(&@as([2 * 2 * 3 * 2 * 2]u16, @splat(1)))); defer values_buffer.deinit(); - - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatterCB, .{ operand_buffer, start_indices_buffer, values_buffer }); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{ operand_buffer, start_indices_buffer, values_buffer }); defer result.deinit(); const expected = [2][3][4][2]u16{ @@ -3441,7 +3459,10 @@ pub const Tensor = struct { var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, 4.1, 7.9, 0, 7.9 }})); defer x_buffer.deinit(); - var res = try zml.testing.autoCall(allocator, std.testing.io, &exe, ArgMaxTest._fwd, .{x_buffer}); + var res: zml.Bufferized(Tensor.ArgMaxRes) = undefined; + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + runner.run(std.testing.io, .{x_buffer}, .{&res}, .{ .wait = true }); defer res.values.deinit(); defer res.indices.deinit(); const max_ = res.values.getValue(f32, std.testing.io); @@ -3455,7 +3476,10 @@ pub const Tensor = struct { var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, std.math.nan(f32), 7.9, 0, 7.9 }})); defer x_buffer.deinit(); - var res = try zml.testing.autoCall(allocator, std.testing.io, &exe, ArgMaxTest._fwd, .{x_buffer}); + var res: zml.Bufferized(Tensor.ArgMaxRes) = undefined; + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + runner.run(std.testing.io, .{x_buffer}, .{&res}, .{ .wait = true }); defer res.values.deinit(); defer res.indices.deinit(); const max_ = try res.values.getValue(f32, std.testing.io); @@ -3509,7 +3533,7 @@ pub const Tensor = struct { var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 })); defer x_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); defer res.deinit(); const res_cpu = try res.toSliceAlloc(std.testing.allocator, std.testing.io); @@ -3534,7 +3558,7 @@ pub const Tensor = struct { })); defer x_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); defer res.deinit(); const res_cpu = try res.toSliceAlloc(std.testing.allocator, std.testing.io); @@ -3568,7 +3592,7 @@ pub const Tensor = struct { })); defer x_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); defer res.deinit(); const res_cpu = try res.toSliceAlloc(std.testing.allocator, std.testing.io); @@ -3896,17 +3920,11 @@ pub const Tensor = struct { var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&x_data)); defer x_buffer.deinit(); - var exe_args = try exe.args(std.testing.allocator); - defer exe_args.deinit(std.testing.allocator); - exe_args.set(.{x_buffer}); - - var results = try exe.results(std.testing.allocator); - defer results.deinit(std.testing.allocator); - exe.callOpts(std.testing.io, exe_args, &results, .{ .wait = true }); - const res = try std.testing.allocator.alloc(zml.Buffer, expectation.len); defer std.testing.allocator.free(res); - results.fill(res); + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{x_buffer}, res, .{ .wait = true }); defer for (res) |*buffer| buffer.deinit(); inline for (expectation, 0..) |expected, i| { @@ -4034,7 +4052,7 @@ pub const Tensor = struct { var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{4})); defer ids_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ .a = ids_buffer }, y_buffer }); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ .a = ids_buffer }, y_buffer }); defer res.deinit(); try std.testing.expectEqual([10]f32{ 0, 1, 2, 3, -1, -1, 6, 7, 8, 9 }, try res.getValue([10]f32, std.testing.io)); @@ -4060,7 +4078,7 @@ pub const Tensor = struct { var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{3})); defer ids_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, ids_buffer, y_buffer }); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, ids_buffer, y_buffer }); defer res.deinit(); try std.testing.expectEqualDeep( @@ -4089,7 +4107,7 @@ pub const Tensor = struct { var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{3})); defer ids_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, ids_buffer, y_buffer }); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, ids_buffer, y_buffer }); defer res.deinit(); try std.testing.expectEqualDeep( @@ -4121,7 +4139,7 @@ pub const Tensor = struct { var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{3})); defer idx_b_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ .a = idx_a_buffer, .b = idx_b_buffer }, y_buffer }); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ .a = idx_a_buffer, .b = idx_b_buffer }, y_buffer }); defer res.deinit(); try std.testing.expectEqualDeep( @@ -4153,7 +4171,7 @@ pub const Tensor = struct { var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{3})); defer idx_b_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ idx_a_buffer, idx_b_buffer }, y_buffer }); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ idx_a_buffer, idx_b_buffer }, y_buffer }); defer res.deinit(); try std.testing.expectEqualDeep( @@ -4227,7 +4245,7 @@ pub const Tensor = struct { var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[2][2]u8{ .{ 1, 2 }, .{ 3, 4 } })); defer x_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._toDiag, .{x_buffer}); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); defer res.deinit(); try std.testing.expectEqual( @@ -4283,7 +4301,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 0 }, platform, .{}); defer exe.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); defer res.deinit(); try std.testing.expectEqual( @@ -4299,7 +4317,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 1 }, platform, .{}); defer exe.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); defer res.deinit(); try std.testing.expectEqual( @@ -4315,7 +4333,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, -1 }, platform, .{}); defer exe.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); defer res.deinit(); try std.testing.expectEqual( @@ -4459,7 +4477,11 @@ pub const Tensor = struct { var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); defer y_buffer.deinit(); - var xs, var ys = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._cartesianProduct2, .{ x_buffer, y_buffer }); + var xs: zml.Buffer = undefined; + var ys: zml.Buffer = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{ x_buffer, y_buffer }, .{ &xs, &ys }, .{ .wait = true }); defer xs.deinit(); defer ys.deinit(); @@ -4532,7 +4554,7 @@ pub const Tensor = struct { var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); defer y_buffer.deinit(); - var z = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._fwd, .{ x_buffer, y_buffer }); + var z = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, y_buffer }); defer z.deinit(); try std.testing.expectEqualDeep( @@ -4682,7 +4704,10 @@ test "Tensor.maxPool1d" { var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&data)); defer x_buffer.deinit(); - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, MaxPool._fwd, .{x_buffer}); + var result: zml.Bufferized(Tensor.ArgMaxRes) = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{x_buffer}, .{&result}, .{ .wait = true }); defer result.values.deinit(); defer result.indices.deinit(); @@ -4721,7 +4746,10 @@ test "Tensor.maxPool2d" { var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&data)); defer x_buffer.deinit(); - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, MaxPool._fwd, .{x_buffer}); + var result: zml.Bufferized(Tensor.ArgMaxRes) = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{x_buffer}, .{&result}, .{ .wait = true }); defer result.values.deinit(); defer result.indices.deinit(); diff --git a/zml/testing.zig b/zml/testing.zig index 213eae9f..369cdc72 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -362,25 +362,6 @@ fn BufferizedWithArgs(comptime T: type) type { return zml.meta.MapType(zml.Tensor, zml.Buffer).map(T); } -/// Automatically calls the executable with the given arguments, taking care of arguments and results allocation. -/// This helper can be used in tests to make the code a bit less verbose. -/// It doesn't handle pointers in inputs/outputs structs. -pub fn autoCall(allocator: std.mem.Allocator, io: std.Io, exe: *const zml.exe.Exe, func: anytype, inputs: zml.Bufferized(std.meta.ArgsTuple(@TypeOf(func)))) !zml.Bufferized(stdx.meta.FnReturn(func)) { - var args = try exe.args(allocator); - defer args.deinit(allocator); - - var results = try exe.results(allocator); - defer results.deinit(allocator); - - args.set(inputs); - exe.callOpts(io, args, &results, .{ .wait = true }); - - var output: zml.Bufferized(stdx.meta.FnReturn(func)) = undefined; - results.fill(.{&output}); - - return output; -} - fn countWithPrefix(store: zml.io.TensorStore.View, prefix: []const u8) usize { var count: usize = 0; var it = store.store.registry.iterator(); @@ -474,7 +455,7 @@ pub fn testLayer( exe_args.set(.{ layer_weights, args_buffers }); - exe.callOpts(io, exe_args, &exe_results, .{ .wait = true }); + exe.call(io, exe_args, &exe_results, .{ .wait = true }); var results = try allocator.alloc(zml.Buffer, output_count); defer allocator.free(results); From 427499e81e2a221dc9bb96531f96e46daba21772 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Mon, 5 Oct 2026 13:52:13 +0200 Subject: [PATCH 746/881] zml/exe: Exe.Runner destroys donated input buffer handles (#896) Record donated input indices during compilation and destroy their PJRT_Buffer handles once execution is enqueued for `zml.Exe.Runner`. I'm not doing this automatically in `call()` because it's a bit invasive. Not all the code uses the pattern of overwriting an input buffer like `run(.{cache}, .{&cache})`, but most do. So I'm cleaning it up on easy path and leave `call` as is for finer control. --- docs/tutorials/write_first_model.md | 5 + examples/llm/models/lfm2_tests.zig | 41 +++---- examples/llm/models/llama/inference.zig | 21 ++-- zml/Compiler.zig | 1 + zml/exe.zig | 147 +++++++++++++++++++++++- zml/io.zig | 42 +++++++ zml/tensor.zig | 45 ++++---- zml/testing.zig | 4 +- 8 files changed, 241 insertions(+), 65 deletions(-) diff --git a/docs/tutorials/write_first_model.md b/docs/tutorials/write_first_model.md index 4d577586..da9113a9 100644 --- a/docs/tutorials/write_first_model.md +++ b/docs/tutorials/write_first_model.md @@ -251,6 +251,11 @@ the form of a `Slice`. After that, we can print it. `call(io, args, &results, .{ .wait = true })` to wait for execution to finish before returning. Here, `toSliceAlloc()` waits for the result before reading it. +Raw calls leave input buffer handles owned by the caller, including inputs donated +with `reuseBuffer()`. Deinit their original handles after calling. Runners and +`eval` destroy donated input handles automatically. With runners, use output +destinations pointing to those buffers to replace them with the returned outputs. + And that's it! Now, let's have a look at building and actually running this example! diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig index d3f8423d..cbc33811 100644 --- a/examples/llm/models/lfm2_tests.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -213,8 +213,6 @@ const TestContext = struct { const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, actual_seq_len_tensor, model.ConvCache{ .state = cache_tensor }, cache_index_tensor, model.ConvParameters{ .is_prefill = false } }, .{ .shardings = &.{self.sharding} }); defer exe.deinit(); - var args = try exe.args(self.allocator); - defer args.deinit(self.allocator); const conv_cache: zml.Bufferized(model.ConvCache) = .{ .state = cache_buffer }; const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{actual_seq_len})); @@ -225,16 +223,16 @@ const TestContext = struct { var cache_index_buf: zml.Buffer = try .fromSlice(self.io, self.platform, cache_index_slice, self.sharding); defer cache_index_buf.deinit(); - args.set(.{ layer_buffers, in_buffer, cache_pos_buffer, actual_seq_len_buf, conv_cache, cache_index_buf }); + var runner = try exe.runner(self.allocator); + defer runner.deinit(self.allocator); + runner.run( + self.io, + .{ layer_buffers, in_buffer, cache_pos_buffer, actual_seq_len_buf, conv_cache, cache_index_buf }, + .{ &in_buffer, &cache_buffer }, + .{}, + ); - var res = try exe.results(self.allocator); - defer res.deinit(self.allocator); - exe.call(self.io, args, &res, .{}); - - var out_result = res.get(zml.Buffer); - defer out_result.deinit(); - - try zml.testing.expectClose(self.io, out_result, out_buffer_expected, opts); + try zml.testing.expectClose(self.io, in_buffer, out_buffer_expected, opts); std.log.info("Layer {s} passed!", .{name}); } @@ -277,8 +275,6 @@ const TestContext = struct { const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, model.KvCache{ .k = key_cache_tensor, .v = value_cache_tensor }, cache_index_tensor, self.attention_metadata, self.attention_parameters }, .{ .shardings = &.{self.sharding} }); defer exe.deinit(); - var args = try exe.args(self.allocator); - defer args.deinit(self.allocator); const kv_cache: zml.Bufferized(model.KvCache) = .{ .k = key_cache_buffer, .v = value_cache_buffer }; var attention_metadata_buffers = try self.attention_metadata.initBuffer(self.io, self.platform, self.sharding); @@ -287,17 +283,16 @@ const TestContext = struct { var cache_index_buf: zml.Buffer = try .scalar(self.io, self.platform, cache_ix, .u32); defer cache_index_buf.deinit(); - args.set(.{ layer_buffers, in_buffer, cache_pos_buffer, kv_cache, cache_index_buf, attention_metadata_buffers }); + var runner = try exe.runner(self.allocator); + defer runner.deinit(self.allocator); + runner.run( + self.io, + .{ layer_buffers, in_buffer, cache_pos_buffer, kv_cache, cache_index_buf, attention_metadata_buffers }, + .{ &in_buffer, &key_cache_buffer, &value_cache_buffer }, + .{}, + ); - var res = try exe.results(self.allocator); - defer res.deinit(self.allocator); - exe.call(self.io, args, &res, .{}); - - var out_result, var updated_kv = res.get(struct { zml.Buffer, zml.Bufferized(model.KvCache) }); - defer out_result.deinit(); - defer model.KvCache.unloadBuffers(&updated_kv); - - try zml.testing.expectClose(self.io, out_result, out_buffer_expected, opts); + try zml.testing.expectClose(self.io, in_buffer, out_buffer_expected, opts); std.log.info("Layer {s} passed!", .{name}); } }; diff --git a/examples/llm/models/llama/inference.zig b/examples/llm/models/llama/inference.zig index 7513a3ad..4eafdcea 100644 --- a/examples/llm/models/llama/inference.zig +++ b/examples/llm/models/llama/inference.zig @@ -153,7 +153,11 @@ const Forward = struct { .tokens = input.tokens, .rng = input.rng, }); - return .{ .tokens = result.tokens, .kv_cache = kv_cache.reuseBuffer(input.kv_cache), .rng = result.rng }; + return .{ + .tokens = result.tokens, + .kv_cache = kv_cache.reuseBuffer(input.kv_cache), + .rng = .{ ._state = result.rng._state.reuseBuffer(input.rng._state) }, + }; } }; @@ -172,21 +176,12 @@ pub const KernelRunner = struct { }; pub fn run(runner: *KernelRunner, args: Args) void { - // Results replace handles even when PJRT declines optional donation. - // Pending execution retains its allocations after these inputs are released. - var tokens = args.tokens_buf.*; - var kv_cache = args.kv_cache_buffers.*; - var rng = args.rng_buffers.*; - defer tokens.deinit(); - defer model.KvCache.deinitBuffer(&kv_cache); - defer zml.Tensor.Rng.deinitBuffer(&rng); - runner.forward.run(args.io, .{ .inputs = .{ - .tokens = tokens, + .tokens = args.tokens_buf.*, .token_index = args.token_index_buf.*, - .kv_cache = kv_cache, - .rng = rng, + .kv_cache = args.kv_cache_buffers.*, + .rng = args.rng_buffers.*, .attention_metadata = args.attention_metadata_buffers.*, }, .outputs = .{ diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 57a357e4..23259b1d 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -443,6 +443,7 @@ pub fn compileInternal( result.output_info.items(.shape), result.input_info.items(.sharding), result.output_info.items(.sharding), + result.input_info.items(.aliasing_output), ); errdefer exe.deinit(); diff --git a/zml/exe.zig b/zml/exe.zig index c4d39b04..f97b8f7d 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -26,6 +26,10 @@ pub const Exe = struct { input_shardings: []const Sharding, output_shardings: []const Sharding, + /// Inputs whose buffers are donated to an output (`reuseBuffer`). + /// Runners destroy their PJRT handles after the raw call returns. + donated_input_indices: []const usize, + num_devices: usize, num_partitions: i32, @@ -41,6 +45,7 @@ pub const Exe = struct { output_shapes: []const Shape, input_shardings: []const Sharding, output_shardings: []const Sharding, + input_aliasing: []const ?u32, ) !Exe { var arena = std.heap.ArenaAllocator.init(allocator); errdefer arena.deinit(); @@ -52,6 +57,18 @@ pub const Exe = struct { const input_shardings_copy = try arena.allocator().dupe(Sharding, input_shardings); const output_shardings_copy = try arena.allocator().dupe(Sharding, output_shardings); + var donated_count: usize = 0; + for (input_aliasing) |aliasing| donated_count += @intFromBool(aliasing != null); + + const donated_input_indices = try arena.allocator().alloc(usize, donated_count); + + var donated_index: usize = 0; + for (input_aliasing, 0..) |aliasing, input_index| { + if (aliasing == null) continue; + donated_input_indices[donated_index] = input_index; + donated_index += 1; + } + return .{ .platform = platform, .exe = exe, @@ -59,6 +76,7 @@ pub const Exe = struct { .output_shapes = output_shapes_copy, .input_shardings = input_shardings_copy, .output_shardings = output_shardings_copy, + .donated_input_indices = donated_input_indices, .num_devices = num_devices, .num_partitions = num_partitions, .arena = arena, @@ -264,7 +282,7 @@ pub const Exe = struct { }; /// Executes a single-output executable and waits for its result. - /// The caller owns the returned buffer. + /// The caller owns the returned buffer. Donated inputs are consumed. pub fn eval(self: *const Exe, allocator: std.mem.Allocator, io: std.Io, input_values: anytype) !Buffer { stdx.debug.assert(self.output_shapes.len == 1, "eval expects one output, got {d}; use a runner for multiple outputs", .{self.output_shapes.len}); var result: Buffer = undefined; @@ -302,9 +320,13 @@ pub const Exe = struct { self.args.deinit(allocator); } + /// Destroys donated input handles and fills the output destinations. + /// Use the donated buffer's address as its output destination to replace it safely. + /// Non-donated inputs remain owned by the caller. pub fn run(self: *Runner, io: std.Io, input_values: anytype, output_refs: anytype, opts: CallOpts) void { self.args.set(input_values); self.exe.call(io, self.args, &self.results, opts); + self.exe.deinitDonatedInputs(self.args); self.results.fill(output_refs); } }; @@ -314,6 +336,19 @@ pub const Exe = struct { wait: bool = false, }; + fn deinitDonatedInputs(self: *const Exe, arguments: Arguments) void { + // Execution deletes donated inputs, but leaves their PJRT handles allocated. + // Arguments retain the old handles even when outputs replace the caller's buffers. + for (self.donated_input_indices) |input_index| { + for (arguments.flat_buffers.buffers[0..self.num_devices]) |device_buffers| { + device_buffers[input_index].deinit(self.platform.pjrt_api); + } + } + } + + /// Executes with optional waiting. The caller owns all input handles, including donated inputs. + /// Donated input contents cannot be used after the call; output buffers have their own handles. + /// Use a runner to destroy donated handles automatically and replace buffers through output destinations. pub fn call(self: *const Exe, io: std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { var span = tracer.span("zml.exe.call", .{ .wait = opts.wait, @@ -335,9 +370,7 @@ pub const Exe = struct { .num_args = arguments.expected_shapes.len, .results = results_.flat_buffers.buffers, .events = events_slice, - // this allows to tell a specific buffer shouldn't be donated, - // even if it has been marked as "can be donated" during compiler. - // TODO: expose it ? + // TODO: Set this flag for undonatable buffers. .non_donatable_input_indices = &.{}, .context = self.context, }) catch |err| { @@ -449,9 +482,11 @@ pub fn FnExe(comptime function_: anytype) type { outputs: Output, }; + /// Destroys donated input handles before filling outputs; non-donated inputs remain caller-owned. pub fn run(self: *RunnerSelf, io: std.Io, call: Call, opts: Exe.CallOpts) void { self.args.set(call.inputs); self.exe.call(io, self.args, &self.results, opts); + self.exe.deinitDonatedInputs(self.args); self.results.fill(call.outputs); } }; @@ -588,3 +623,107 @@ test "FnExe derives baked and runtime calls from a function" { ); try std.testing.expect(!@hasField(Model.Output, "metadata")); } + +test "Exe raw calls leave donated input handle cleanup to the caller" { + const platform = @import("testing.zig").env(); + const allocator = std.testing.allocator; + const io = std.testing.io; + const Functions = struct { + fn forward(value: Tensor, weight: Tensor) Tensor { + return value.add(weight).reuseBuffer(value); + } + }; + const value: Tensor = .init(.{2}, .i32); + const weight: Tensor = .init(.{2}, .i32); + const exe = try platform.compileFn(allocator, io, Functions.forward, .{ value, weight }, .{}); + defer exe.deinit(); + try std.testing.expectEqualSlices(usize, &.{0}, exe.donated_input_indices); + + var weight_buffer = try Buffer.fromBytes(io, platform, weight.shape(), .replicated, std.mem.asBytes(&[2]i32{ 10, 20 })); + defer weight_buffer.deinit(); + var args = try exe.args(allocator); + defer args.deinit(allocator); + var results = try exe.results(allocator); + defer results.deinit(allocator); + + for ([_]bool{ false, true }) |wait| { + var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), .replicated, std.mem.asBytes(&[2]i32{ 1, 2 })); + defer value_buffer.deinit(); + args.set(.{ value_buffer, weight_buffer }); + exe.call(io, args, &results, .{ .wait = wait }); + var output = results.get(Buffer); + defer output.deinit(); + + // The donated handle is still valid for cleanup even though its contents are deleted. + for (value_buffer._shards.constSlice()) |shard| { + try std.testing.expect(shard.isDeleted(platform.pjrt_api)); + } + value_buffer.deinit(); + try std.testing.expectEqual([2]i32{ 11, 22 }, try output.getValue([2]i32, io)); + try std.testing.expectEqual([2]i32{ 10, 20 }, try weight_buffer.getValue([2]i32, io)); + } +} + +test "FnExe runner replaces donated buffers with and without waiting" { + const platform = @import("testing.zig").env(); + const allocator = std.testing.allocator; + const io = std.testing.io; + const Functions = struct { + fn forward(inputs: struct { weight: Tensor, value: Tensor }) struct { value: Tensor } { + return .{ .value = inputs.value.add(inputs.weight).reuseBuffer(inputs.value) }; + } + }; + const Model = FnExe(Functions.forward); + const weight: Tensor = .init(.{2}, .i32); + const value: Tensor = .init(.{2}, .i32); + const exe = try Model.compile(allocator, io, platform, .{}, .{.{ .weight = weight, .value = value }}); + defer exe.deinit(); + + var weight_buffer = try Buffer.fromBytes(io, platform, weight.shape(), .replicated, std.mem.asBytes(&[2]i32{ 10, 20 })); + defer weight_buffer.deinit(); + var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), .replicated, std.mem.asBytes(&[2]i32{ 1, 2 })); + defer value_buffer.deinit(); + var runner = try Model.Runner(.{.weight}).init(&exe, allocator, .{ .weight = weight_buffer }); + defer runner.deinit(allocator); + + runner.run(io, .{ + .inputs = .{ .value = value_buffer }, + .outputs = .{ .value = &value_buffer }, + }, .{}); + try std.testing.expectEqual([2]i32{ 11, 22 }, try value_buffer.getValue([2]i32, io)); + + runner.run(io, .{ + .inputs = .{ .value = value_buffer }, + .outputs = .{ .value = &value_buffer }, + }, .{ .wait = true }); + try std.testing.expectEqual([2]i32{ 21, 42 }, try value_buffer.getValue([2]i32, io)); +} + +test "FnExe runner replaces unchanged donated buffers" { + const platform = @import("testing.zig").env(); + const allocator = std.testing.allocator; + const io = std.testing.io; + const Functions = struct { + fn forward(inputs: struct { value: Tensor }) struct { value: Tensor } { + return .{ .value = inputs.value.reuseBuffer(inputs.value) }; + } + }; + const Model = FnExe(Functions.forward); + const value: Tensor = .init(.{2}, .i32); + const exe = try Model.compile(allocator, io, platform, .{}, .{.{ .value = value }}); + defer exe.deinit(); + try std.testing.expectEqualSlices(usize, &.{0}, exe.raw.donated_input_indices); + + var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), .replicated, std.mem.asBytes(&[2]i32{ 1, 2 })); + defer value_buffer.deinit(); + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + + for ([_]bool{ false, true }) |wait| { + runner.run(io, .{ + .inputs = .{ .value = value_buffer }, + .outputs = .{ .value = &value_buffer }, + }, .{ .wait = wait }); + try std.testing.expectEqual([2]i32{ 1, 2 }, try value_buffer.getValue([2]i32, io)); + } +} diff --git a/zml/io.zig b/zml/io.zig index 7d29a352..589abd47 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -551,6 +551,48 @@ pub const Loader = struct { } }; +test "Loader.loadExecute cleans up donated and non-donated inputs" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = @import("testing.zig").env(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try tmp.dir.writeFile(io, .{ .sub_path = "inputs", .data = std.mem.asBytes(&[4]i32{ 1, 2, 10, 20 }) }); + const path = try tmp.dir.realPathFileAlloc(io, "inputs", allocator); + defer allocator.free(path); + + const shape: Shape = .init(.{2}, .i32); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try registry.registerTensor(.{ .name = "value", .file_uri = path, .shape = shape, .offset = 0 }); + try registry.registerTensor(.{ .name = "weight", .file_uri = path, .shape = shape, .offset = 2 * @sizeOf(i32) }); + var store: TensorStore = .fromRegistry(allocator, ®istry); + defer store.deinit(); + var loader = try Loader.init(allocator, platform, .{ .parallelism = 2, .dma_chunks = 2, .dma_chunk_size = 4096 }); + defer loader.deinit(); + + inline for (.{ false, true }) |donate| { + const Functions = struct { + fn forward(value: Tensor, weight: Tensor) Tensor { + const sum = value.add(weight); + return if (donate) sum.reuseBuffer(value) else sum; + } + }; + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + const tensor = store.view().maybeCreateBinding(&.{ "value", "weight" }, shape) orelse unreachable; + const exe = try platform.compileFn(allocator, io, Functions.forward, .{ Tensor.fromShape(shape), Tensor.fromShape(shape) }, .{}); + defer exe.deinit(); + try std.testing.expectEqual(@as(usize, @intFromBool(donate)), exe.donated_input_indices.len); + + var output: Buffer = undefined; + try loader.loadExecute(arena.allocator(), io, tensor, &output, &store, &.{}, &exe, .{}); + defer output.deinit(); + try std.testing.expectEqual([2]i32{ 11, 22 }, try output.getValue([2]i32, io)); + try std.testing.expect(loader.delivered.contains(tensor.id)); + } +} + pub const ProgressWriter = struct { inner: *std.Io.Writer, progress: *std.Progress.Node, diff --git a/zml/tensor.zig b/zml/tensor.zig index 3e5dea04..40aa2a80 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -356,7 +356,8 @@ pub const Tensor = struct { /// but this API allows to reuse buffer between input and output arguments /// of a given function. /// Note this is visible from the outside. The caller of a function with donations - /// is not allowed to reuse the donated input buffer after the call. + /// cannot use the donated input's contents after execution. Runners destroy its + /// handle automatically; raw `Exe.call` leaves it for the caller to deinit. /// For `reuseBuffer` to be effective, it needs to propagate all the way through the output. pub fn reuseBuffer(self: Tensor, origin: Tensor) Tensor { const compilation_context = Compiler.current(); @@ -406,25 +407,27 @@ pub const Tensor = struct { y_memory[dev] = y_d.opaqueDevicePtr(dev); } - var y2_d: zml.Buffer = undefined; - var z2_d: zml.Buffer = undefined; var runner = try exe.runner(std.testing.allocator); defer runner.deinit(std.testing.allocator); - runner.run(io, .{ x_d, y_d, z_d }, .{ &y2_d, &z2_d }, .{ .wait = true }); - defer y2_d.deinit(); - defer z2_d.deinit(); - var output_memory: [Platform.MAX_NUM_DEVICES]*anyopaque = undefined; - for (0..y2_d.numShards()) |dev| { - output_memory[dev] = y2_d.opaqueDevicePtr(dev); - } + for ([_]bool{ false, true }) |wait| { + var z2_d: zml.Buffer = undefined; + runner.run(io, .{ x_d, y_d, z_d }, .{ &y_d, &z2_d }, .{ .wait = wait }); + defer z2_d.deinit(); + try y_d.await(io); - // Check that y_d and y2_d point to the same addresses on the devices. - try std.testing.expectEqualSlices(*anyopaque, y_memory[0..y_d.numShards()], output_memory[0..y2_d.numShards()]); + var output_memory: [Platform.MAX_NUM_DEVICES]*anyopaque = undefined; + for (0..y_d.numShards()) |dev| { + output_memory[dev] = y_d.opaqueDevicePtr(dev); + } - // Check that z_d and z2_d DO NOT POINT to the same addresses - for (0..z2_d.numShards()) |dev| { - try std.testing.expect(z2_d.opaqueDevicePtr(dev) != z_d.opaqueDevicePtr(dev)); + // Check that the replacement y_d reuses the donated device memory. + try std.testing.expectEqualSlices(*anyopaque, y_memory[0..y_d.numShards()], output_memory[0..y_d.numShards()]); + + // Check that z_d and z2_d DO NOT POINT to the same addresses + for (0..z2_d.numShards()) |dev| { + try std.testing.expect(z2_d.opaqueDevicePtr(dev) != z_d.opaqueDevicePtr(dev)); + } } } @@ -834,19 +837,17 @@ pub const Tensor = struct { var rng_buffer = try Rng.initBuffer(std.testing.io, platform, .replicated, 1234); defer rng_buffer._state.deinit(); - var rand: zml.Bufferized(Rng) = undefined; var stats: zml.Bufferized(Stats) = undefined; var runner = try exe.runner(std.testing.allocator); defer runner.deinit(std.testing.allocator); - runner.run(std.testing.io, .{rng_buffer}, .{ &rand, &stats }, .{ .wait = true }); - defer rand._state.deinit(); + runner.run(std.testing.io, .{rng_buffer}, .{ &rng_buffer, &stats }, .{ .wait = true }); defer stats.mean.deinit(); defer stats.variance.deinit(); defer stats.min.deinit(); defer stats.max.deinit(); // Check the Rng state has been modified. - try std.testing.expect(try rand._state.getValue(u128, std.testing.io) != 1234); + try std.testing.expect(try rng_buffer._state.getValue(u128, std.testing.io) != 1234); // Check the mean and variance are close to theoritical values. const mean_ = try stats.mean.getValue(f32, std.testing.io); @@ -948,18 +949,16 @@ pub const Tensor = struct { var tgt_dist_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, tgt_dist.shape(), .replicated, @ptrCast(&tgt_dist_data)); defer tgt_dist_buffer.deinit(); - var rand: zml.Bufferized(Rng) = undefined; var stats: zml.Bufferized(Stats) = undefined; var runner = try exe.runner(std.testing.allocator); defer runner.deinit(std.testing.allocator); - runner.run(std.testing.io, .{ rng_buffer, tgt_dist_buffer }, .{ &rand, &stats }, .{ .wait = true }); - defer rand._state.deinit(); + runner.run(std.testing.io, .{ rng_buffer, tgt_dist_buffer }, .{ &rng_buffer, &stats }, .{ .wait = true }); defer stats.mean.deinit(); defer stats.variance.deinit(); defer stats.actual_dist.deinit(); // Check the Rng state has been modified. - try std.testing.expect(try rand._state.getValue(i128, std.testing.io) != 1234); + try std.testing.expect(try rng_buffer._state.getValue(i128, std.testing.io) != 1234); // Check the mean and variance are close to theoritical values. const mean_ = try stats.mean.getValue(f32, std.testing.io); diff --git a/zml/testing.zig b/zml/testing.zig index 369cdc72..e2e84502 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -455,12 +455,12 @@ pub fn testLayer( exe_args.set(.{ layer_weights, args_buffers }); - exe.call(io, exe_args, &exe_results, .{ .wait = true }); - var results = try allocator.alloc(zml.Buffer, output_count); defer allocator.free(results); + exe.call(io, exe_args, &exe_results, .{ .wait = true }); exe_results.fill(.{results}); + defer for (results) |*result| result.deinit(); var failed: bool = false; var reader_buffer: [4096]u8 = undefined; From daa0c0dca8cfbd7426acda5bb4a4aefc80ad7886 Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Mon, 5 Oct 2026 13:52:44 +0200 Subject: [PATCH 747/881] zml/quantization: MXFP8 and MXFP4 (#842) Introduce MXFP8 and MXFP4 quantization functions. Changes: - add `QuantizedInput.reuseBuffer` --- zml/quantization.zig | 58 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/zml/quantization.zig b/zml/quantization.zig index 1a058e0f..489162fb 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -3,6 +3,8 @@ const std = @import("std"); const stdx = @import("stdx"); +const zml = @import("zml.zig"); + const DataType = @import("dtype.zig").DataType; const platform_mod = @import("platform.zig"); const Platform = platform_mod.Platform; @@ -109,10 +111,19 @@ pub const QuantizedInput = struct { values: Tensor, scales: Tensor, global_scale: ?Tensor, + + pub fn reuseBuffer(self: QuantizedInput, other: QuantizedInput) QuantizedInput { + return .{ + .values = self.values.reuseBuffer(other.values), + .scales = self.scales.reuseBuffer(other.scales), + .global_scale = if (self.global_scale != null and other.global_scale != null) self.global_scale.?.reuseBuffer(other.global_scale.?) else self.global_scale, + }; + } }; pub fn quantizeInput(quantization: Quantization, input: Tensor, axis: Shape.Tag, platform: *const Platform) ?QuantizedInput { const global_scale: ?Tensor = if (quantization.input_scale) |scale| scale.asMultiplier() else null; + return switch (quantization.scheme) { .nvfp4 => if (supportsNvfp4InputQuantization(platform)) quantizeNvfp4(input.convert(.bf16), global_scale, axis) else null, .fp8_block32, .fp8_block128 => blk: { @@ -125,6 +136,7 @@ pub fn quantizeInput(quantization: Quantization, input: Tensor, axis: Shape.Tag, } else return null, else => return null, }; + break :blk switch (quantization.scheme) { .fp8_block32 => quantizeBlockFp8(input, axis, 32, dtype, .f8e8m0), .fp8_block128 => quantizeBlockFp8(input.convert(.bf16), axis, 128, dtype, .f32), @@ -135,6 +147,45 @@ pub fn quantizeInput(quantization: Quantization, input: Tensor, axis: Shape.Tag, }; } +pub fn quantizeMxfp4(x: Tensor, axis: anytype) QuantizedInput { + stdx.debug.assert(@mod(x.dim(axis), mx_block_size) == 0, "MXFP4 activation width must be divisible by {}, got {f}", .{ mx_block_size, x.shape() }); + + const grouped = x.convert(.f32).splitAxis(axis, .{ .mx_ks = -1, .mx_block = mx_block_size }); + const raw_scale = grouped.abs().max(.mx_block).maximum(.scalar(6 * 0x1p-126, .f32)).scale(1.0 / 6.0); + const scales = ceilPowerOfTwo(raw_scale); + + return .{ + .values = grouped.div(scales.broad(grouped.shape())) + .clamp(.scalar(-6, .f32), .scalar(6, .f32)) + .convert(.f4e2m1) + .reshape(x.shape().withDtype(.f4e2m1)), + .scales = scales.reshape(x.shape().setDim(axis, @divExact(x.dim(axis), mx_block_size)).withDtype(.f32)).convert(.f8e8m0), + .global_scale = null, + }; +} + +pub fn quantizeMxfp8(x: Tensor, axis: anytype) QuantizedInput { + const platform = zml.Compiler.current().platform; + + const dtype: ?DataType = switch (platform.target) { + .cuda, .cpu, .tpu => .f8e4m3fn, + .rocm => if (platform_mod.rocm.computeCapability(platform)) |capability| switch (capability.architecture()) { + .cdna4, .rdna4 => .f8e4m3fn, + .cdna3 => .f8e4m3fnuz, + .cdna1, .cdna2, .rdna2, .rdna3, .rdna3_5 => null, + } else null, + else => null, + }; + + if (dtype == null) std.debug.panic("MXFP8 is not supported on {}", .{platform.target}); + return quantizeBlockFp8(x, axis, mx_block_size, dtype.?, .f8e8m0); +} + +fn ceilPowerOfTwo(raw_scale: Tensor) Tensor { + const bits = raw_scale.bitCast(.u32); + return bits.addConstant(0x7fffff).logical(.AND, .scalar(0xff800000, .u32)).bitCast(.f32); +} + /// Quantize activation blocks, preserving the input's axes. /// E8M0 scales round upward to a power of two to avoid overflowing FP8 values. pub fn quantizeBlockFp8(x: Tensor, axis: anytype, block_size: i64, dtype: DataType, scale_dtype: DataType) QuantizedInput { @@ -155,10 +206,7 @@ pub fn quantizeBlockFp8(x: Tensor, axis: anytype, block_size: i64, dtype: DataTy .max(.fp8_block), .f8e8m0 => blk: { const raw_scale = grouped.abs().maximum(.scalar(1e-4, .f32)).scale(1.0 / fp8_max).max(.fp8_block); - // Round a positive F32 scale up to a power of two. Adding all - // mantissa bits carries into the exponent iff the mantissa is nonzero. - const bits = raw_scale.bitCast(.u32); - break :blk bits.addConstant(0x7fffff).logical(.AND, .scalar(0xff800000, .u32)).bitCast(.f32); + break :blk ceilPowerOfTwo(raw_scale); }, else => stdx.debug.panic("expected F32 or E8M0 scale dtype, got {s}", .{@tagName(scale_dtype)}), }; @@ -363,7 +411,6 @@ test "Quantization.Scheme.classify" { } test "block FP8 quantization preserves axes and reconstructs constant blocks in FN and FNUZ" { - const zml = @import("zml.zig"); const allocator = std.testing.allocator; const io = std.testing.io; const platform = zml.testing.env(); @@ -409,7 +456,6 @@ test "block FP8 quantization preserves axes and reconstructs constant blocks in } test "block FP8 E8M0 scales preserve power-of-two boundaries and nonminor axes" { - const zml = @import("zml.zig"); const allocator = std.testing.allocator; const io = std.testing.io; const platform = zml.testing.env(); From 98dc929608f987743cba774381c44481c71b16ef Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 5 Oct 2026 16:02:37 +0200 Subject: [PATCH 748/881] zml/moe: cleanup entrypoint (#865) Move logic out of the entry point while keeping everything the same --- zml/moe/cutlass_flashinfer.zig | 249 +++++++++++++++++++-- zml/moe/fused_experts.zig | 68 +++++- zml/moe/metal.zig | 42 ++++ zml/moe/moe.zig | 388 +-------------------------------- zml/moe/mosaic_tpu.zig | 113 +++++++++- 5 files changed, 454 insertions(+), 406 deletions(-) diff --git a/zml/moe/cutlass_flashinfer.zig b/zml/moe/cutlass_flashinfer.zig index d857e344..0735c3bc 100644 --- a/zml/moe/cutlass_flashinfer.zig +++ b/zml/moe/cutlass_flashinfer.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const stdx = @import("stdx"); const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); const platforms = @import("platforms"); @@ -544,9 +545,9 @@ pub fn fusedExpertsNvfp4( fc2_weight_block: zml.Tensor, fc2_global: zml.Tensor, options: Options, -) !zml.Tensor { - const runners = try currentRunners(); - var attributes = try validateInputs( +) zml.Tensor { + const runners = currentRunners() catch |e| stdx.debug.panic("Failed to get runners: {}", .{e}); + var attributes = validateInputs( hidden_states, fc1_weights, fc2_weights, @@ -559,20 +560,21 @@ pub fn fusedExpertsNvfp4( fc2_weight_block, fc2_global, options, - ); + ) catch |e| stdx.debug.panic("Invalid inputs: {}", .{e}); + attributes.runners = @intFromPtr(runners); - const deviceRunner = try runners.ensureRunner(options.workspace_query_device, .nvfp4xnvfp4); + const deviceRunner = runners.ensureRunner(options.workspace_query_device, .nvfp4xnvfp4) catch |e| stdx.debug.panic("Failed to ensure runner: {}", .{e}); const context = makeContext(attributes); var requirements = std.mem.zeroes(fi_cutlass_moe.WorkspaceRequirements); requirements.struct_size = @sizeOf(fi_cutlass_moe.WorkspaceRequirements); - try checkStatus( + checkStatus( deviceRunner.api, deviceRunner.api.getWorkspaceRequirements( deviceRunner.runner, &context, &requirements, ), - ); + ) catch |e| stdx.debug.panic("Failed to get workspace requirements: {}", .{e}); const result = routedNvfp4Call.call( .{ @@ -606,7 +608,7 @@ pub fn fusedExpertsBf16( topk_weights: zml.Tensor, topk_ids: zml.Tensor, options: Options, -) !zml.Tensor { +) zml.Tensor { if (hidden_states.dtype() != .bf16 or fc1_weights.dtype() != .bf16 or fc2_weights.dtype() != .bf16 or @@ -618,7 +620,8 @@ pub fn fusedExpertsBf16( topk_weights.rank() != 3 or topk_ids.rank() != 3) { - return error.InvalidInput; + // TODO(Corentin): Better error message + @panic("InvalidInput"); } const batch = hidden_states.dim(0); @@ -644,10 +647,11 @@ pub fn fusedExpertsBf16( topk_ids.dim(0) != batch or topk_ids.dim(1) != sequence) { - return error.InvalidShape; + // TODO(Corentin): Better error message + @panic("InvalidShape"); } - const runners = try currentRunners(); + const runners = currentRunners() catch |e| stdx.debug.panic("Failed to get runners: {}", .{e}); const attributes: Attributes = .{ .runners = @intFromPtr(runners), .num_tokens = batch * sequence, @@ -662,18 +666,18 @@ pub fn fusedExpertsBf16( .gemm1_tactic = options.gemm1_tactic, .gemm2_tactic = options.gemm2_tactic, }; - const deviceRunner = try runners.ensureRunner(options.workspace_query_device, .bf16xbf16); + const deviceRunner = runners.ensureRunner(options.workspace_query_device, .bf16xbf16) catch |e| stdx.debug.panic("Failed to ensure runner: {}", .{e}); const context = makeContext(attributes); var requirements = std.mem.zeroes(fi_cutlass_moe.WorkspaceRequirements); requirements.struct_size = @sizeOf(fi_cutlass_moe.WorkspaceRequirements); - try checkStatus( + checkStatus( deviceRunner.api, deviceRunner.api.getWorkspaceRequirements( deviceRunner.runner, &context, &requirements, ), - ); + ) catch |e| stdx.debug.panic("Failed to get workspace requirements: {}", .{e}); const result = routedBf16Call.call( .{ @@ -691,3 +695,220 @@ pub fn fusedExpertsBf16( ); return result.output; } + +pub fn fusedExperts( + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + opts: zml.moe.Options, + parameters: Parameters, +) zml.Tensor { + stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); + stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); + stdx.debug.assert(opts.activation_threshold == null, "Activation thresholds require the Triton MoE backend", .{}); + if (comptime !platforms.isEnabled(.cuda)) { + @panic("FlashInfer CUTLASS MoE is only supported on CUDA platforms"); + } + + if (gate_up.bias != null or down.bias != null) { + @panic("FlashInfer CUTLASS MoE does not support bias in gate_up or down linear layers"); + } + + const runner_options = try parameters.runnerOptions(); + const expert_partition = gate_up.weight.shape().partition(.expert); + + const quant_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; + + if (quant_scheme != null and quant_scheme == .nvfp4) { + const gate_up_weight_unpacked = zml.moe.unpackedWeight(gate_up); + const down_weight_unpacked = zml.moe.unpackedWeight(down); + + // TODO(Corentin): Do error checking on nvfp4 + // Also, maybe pass `zml.nn.Linear` directly + if (expert_partition.eql(.init(.experts))) { + return zml.ops.manualComputation( + (struct { + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + gate_up_weight_unpacked: zml.Tensor, + down_weight_unpacked: zml.Tensor, + gate_up_input_scale: zml.Tensor, + gate_up_scales: zml.Tensor, + gate_up_global_scale: zml.Tensor, + down_input_scale: zml.Tensor, + down_scales: zml.Tensor, + down_global_scale: zml.Tensor, + activation: Activation, + enable_pdl: bool, + gemm1_tactic: i32, + gemm2_tactic: i32, + workspace_query_device: i32, + + fn body( + self: @This(), + _: zml.Shape, + ) zml.Tensor { + const local_num_experts = self.gate_up_weight_unpacked.dim(.expert); + const partition_id = zml.ops.partitionId().convert(.i32); + const expert_start = partition_id.scale(local_num_experts).convert(.i32); + const expert_end = expert_start.addConstant(local_num_experts); + + const local_route_mask = self.topk_ids + .cmp(.GE, expert_start) + .logical(.AND, self.topk_ids.cmp(.LT, expert_end)); + const local_topk_ids = local_route_mask.select( + self.topk_ids.sub(expert_start), + zml.Tensor.scalar(-1, .i32), + ); + const local_topk_weights = local_route_mask.select( + self.topk_weights, + zml.Tensor.scalar(-1, self.topk_weights.dtype()), + ); + + const local_output = fusedExpertsNvfp4( + self.input, + self.gate_up_weight_unpacked, + self.down_weight_unpacked, + local_topk_weights, + local_topk_ids, + self.gate_up_input_scale, + self.gate_up_scales, + self.gate_up_global_scale, + self.down_input_scale, + self.down_scales, + self.down_global_scale, + .{ + .workspace_query_device = self.workspace_query_device, + .activation = self.activation, + .enable_pdl = self.enable_pdl, + .gemm1_tactic = self.gemm1_tactic, + .gemm2_tactic = self.gemm2_tactic, + }, + ); + const local_reshaped = local_output + .reshape(self.input.shape().dims()) + .withTags(.{ .b, .s, .d }); + return zml.ops.allReduce(local_reshaped, zml.Tensor.add); + } + }).body, + .{ + .input = input, + .topk_ids = topk_ids, + .topk_weights = topk_weights, + .gate_up_weight_unpacked = gate_up_weight_unpacked, + .down_weight_unpacked = down_weight_unpacked, + .gate_up_input_scale = gate_up.quantization.?.input_scale.?.asMultiplier(), + .gate_up_scales = gate_up.quantization.?.scales, + .gate_up_global_scale = gate_up.quantization.?.global_scale.?.asMultiplier(), + .down_input_scale = down.quantization.?.input_scale.?.asMultiplier(), + .down_scales = down.quantization.?.scales, + .down_global_scale = down.quantization.?.global_scale.?.asMultiplier(), + .activation = runner_options.activation, + .enable_pdl = runner_options.enable_pdl, + .gemm1_tactic = runner_options.gemm1_tactic, + .gemm2_tactic = runner_options.gemm2_tactic, + .workspace_query_device = runner_options.workspace_query_device, + }, + input.shape(), + ); + } + + return fusedExpertsNvfp4( + input, + gate_up_weight_unpacked, + down_weight_unpacked, + topk_weights, + topk_ids, + gate_up.quantization.?.input_scale.?.asMultiplier(), + gate_up.quantization.?.scales, + gate_up.quantization.?.global_scale.?.asMultiplier(), + down.quantization.?.input_scale.?.asMultiplier(), + down.quantization.?.scales, + down.quantization.?.global_scale.?.asMultiplier(), + runner_options, + ); + } + + if (expert_partition.eql(.init(.experts))) { + return zml.ops.manualComputation( + (struct { + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + weights_gate_up: zml.Tensor, + weights_down: zml.Tensor, + activation: Activation, + enable_pdl: bool, + gemm1_tactic: i32, + gemm2_tactic: i32, + workspace_query_device: i32, + + fn body( + self: @This(), + _: zml.Shape, + ) zml.Tensor { + const local_num_experts = self.weights_gate_up.dim(.expert); + const partition_id = zml.ops.partitionId().convert(.i32); + const expert_start = partition_id.scale(local_num_experts).convert(.i32); + const expert_end = expert_start.addConstant(local_num_experts); + + const local_route_mask = self.topk_ids + .cmp(.GE, expert_start) + .logical(.AND, self.topk_ids.cmp(.LT, expert_end)); + const local_topk_ids = local_route_mask.select( + self.topk_ids.sub(expert_start), + zml.Tensor.scalar(-1, .i32), + ); + const local_topk_weights = local_route_mask.select( + self.topk_weights, + zml.Tensor.scalar(-1, self.topk_weights.dtype()), + ); + + const local_output = fusedExpertsBf16( + self.input, + self.weights_gate_up, + self.weights_down, + local_topk_weights, + local_topk_ids, + .{ + .workspace_query_device = self.workspace_query_device, + .activation = self.activation, + .enable_pdl = self.enable_pdl, + .gemm1_tactic = self.gemm1_tactic, + .gemm2_tactic = self.gemm2_tactic, + }, + ); + const local_reshaped = local_output + .reshape(self.input.shape().dims()) + .withTags(.{ .b, .s, .d }); + return zml.ops.allReduce(local_reshaped, zml.Tensor.add); + } + }).body, + .{ + .input = input, + .topk_ids = topk_ids, + .topk_weights = topk_weights, + .weights_gate_up = gate_up.weight, + .weights_down = down.weight, + .activation = runner_options.activation, + .enable_pdl = runner_options.enable_pdl, + .gemm1_tactic = runner_options.gemm1_tactic, + .gemm2_tactic = runner_options.gemm2_tactic, + .workspace_query_device = runner_options.workspace_query_device, + }, + input.shape(), + ); + } + + return fusedExpertsBf16( + input, + gate_up.weight, + down.weight, + topk_weights, + topk_ids, + runner_options, + ); +} diff --git a/zml/moe/fused_experts.zig b/zml/moe/fused_experts.zig index 242e13e4..57a510d9 100644 --- a/zml/moe/fused_experts.zig +++ b/zml/moe/fused_experts.zig @@ -39,6 +39,68 @@ pub const Parameters = struct { } }; +pub fn fusedExperts( + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + opts: zml.moe.Options, + parameters: Parameters, + comptime backend: zml.moe.Backend, +) zml.Tensor { + const gate_up_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; + // TODO(Corentin): Better error message + const layout = backend.expertsLayout(gate_up_scheme) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); + const args: FusedExpertsArgs = .{ + .hidden_states = input, + .gate_up = gate_up, + .down = down, + .topk_weights = topk_weights, + .topk_ids = topk_ids, + .activation = parameters.activation, + .activation_threshold = opts.activation_threshold, + .quantize_input = opts.quantize_input, + .gate_up_layout = layout.gate_up, + .routing_weight_placement = opts.routing_weight_placement, + }; + const expert_partition = gate_up.weight.shape().partition(.expert); + + if (!expert_partition.eql(.init(.experts))) { + return fusedExpertsImpl(args, backend) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); + } + + return zml.ops.manualComputation( + (struct { + args: FusedExpertsArgs, + global_num_experts: i64, + + fn call(self: @This(), _: zml.Shape) zml.Tensor { + const local_args = self.args; + const local_num_experts = local_args.gate_up.weight.dim(.expert); + const partition_id = zml.ops.partitionId().convert(.i32); + const expert_start = partition_id.scale(local_num_experts).convert(.i32); + const global_expert_ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); + + // Map global expert ids to local ids, or -1 for experts outside this partition. + const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) + .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); + var mapped_args = local_args; + mapped_args.expert_map = local_expert_mask.select( + global_expert_ids.sub(expert_start), + zml.Tensor.scalar(-1, .i32), + ); + + const local_output = fusedExpertsImpl(mapped_args, backend) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); + const local_reshaped = local_output.reshape(local_args.hidden_states.shape().dims()).withTags(.{ .b, .s, .d }); + return zml.ops.allReduce(local_reshaped, zml.Tensor.add); + } + }).call, + .{ .args = args, .global_num_experts = gate_up.weight.dim(.expert) }, + input.shape(), + ); +} + pub const ProjectionLayout = enum { concatenated, interleaved }; pub const RoutingWeightPlacement = enum { before_down, after_down }; @@ -57,14 +119,14 @@ pub const FusedExpertsArgs = struct { routing_weight_placement: RoutingWeightPlacement, }; -pub fn fusedExperts(opts: FusedExpertsArgs, comptime backend: zml.moe.Backend) !Tensor { +pub fn fusedExpertsImpl(opts: FusedExpertsArgs, comptime backend: zml.moe.Backend) !Tensor { const Impl = switch (backend) { .fly => fly, .triton => triton, else => @compileError("unsupported fused-experts backend"), }; if (backend == .fly and (!backend.isAvailable(zml.Compiler.current().platform) or !fly.supports(opts))) { - return fusedExperts(opts, .triton); + return fusedExpertsImpl(opts, .triton); } const hidden_states = opts.hidden_states; @@ -440,7 +502,7 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { } const route = Tensor.arange(.{ .end = 2 }, .i32).reshape(.{ .b = 1, .s = 1, .top_expert = 2 }) .broad(Shape.init(.{ .b = 1, .s = x.dim(.s), .top_expert = 2 }, .i32)); - return fusedExperts(.{ + return fusedExpertsImpl(.{ .hidden_states = x, .gate_up = gate_up, .down = down, diff --git a/zml/moe/metal.zig b/zml/moe/metal.zig index f3b2f07a..2eb21b27 100644 --- a/zml/moe/metal.zig +++ b/zml/moe/metal.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const stdx = @import("stdx"); const zml = @import("../zml.zig"); const Tensor = zml.Tensor; @@ -110,6 +111,47 @@ fn applyDownGlobalScale(output: Tensor, global_scale: ?Tensor, expert_ids: Tenso return output.convert(.f32).mul(selected).convert(output.dtype()); } +pub fn fusedExperts( + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + opts: zml.moe.Options, + parameters: Parameters, +) zml.Tensor { + stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); + stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); + stdx.debug.assert(opts.activation_threshold == null, "Activation thresholds require the Triton MoE backend", .{}); + + const gate_up_scales: ?zml.Tensor = if (gate_up.quantization) |q| q.scales else null; + const gate_up_global_scale: ?zml.Tensor = if (gate_up.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; + + const down_scales: ?zml.Tensor = if (down.quantization) |q| q.scales else null; + const down_global_scale: ?zml.Tensor = if (down.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; + + const gate_up_weight_unpacked = zml.moe.unpackedWeight(gate_up); + const down_weight_unpacked = zml.moe.unpackedWeight(down); + + return fusedExpertsImpl( + input, + gate_up_weight_unpacked, + down_weight_unpacked, + topk_weights, + topk_ids, + .{ + .activation = parameters.activation, + .global_num_experts = gate_up_weight_unpacked.dim(.expert), + .w1_scale = gate_up_scales, + .w2_scale = down_scales, + .w1_global_scale = gate_up_global_scale, + .w2_global_scale = down_global_scale, + .w1_bias = gate_up.bias, + .w2_bias = down.bias, + }, + ) catch |e| stdx.debug.panic("fusedExpertsImpl failed: {}", .{e}); +} + pub fn fusedExpertsImpl( hidden_states: Tensor, gate_up: Tensor, diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index 7fa57104..82821b8e 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -225,395 +225,17 @@ pub fn forwardMoe( opts: Options, parameters: Parameters, ) !zml.Tensor { - switch (parameters) { - .triton, .fly, .cute_mxfp4, .triton_mxfp4 => {}, - .flashinfer_cutlass, .mosaic_tpu, .metal => { - stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); - stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); - stdx.debug.assert(opts.activation_threshold == null, "Activation thresholds require the Triton MoE backend", .{}); - }, - } - - const gate_up_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; - const down_scheme: ?zml.Quantization.Scheme = if (down.quantization) |q| q.scheme else null; - if (gate_up_scheme != down_scheme) return error.UnsupportedQuantization; - - const gate_up_scales: ?zml.Tensor = if (gate_up.quantization) |q| q.scales else null; - const down_scales: ?zml.Tensor = if (down.quantization) |q| q.scales else null; - const gate_up_global_scale: ?zml.Tensor = if (gate_up.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; - const down_global_scale: ?zml.Tensor = if (down.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; - const quant_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; - return switch (parameters) { .cute_mxfp4 => |p| cute_mxfp4.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts, p), .triton_mxfp4 => |p| triton_mxfp4.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts, p), - .flashinfer_cutlass => b: { - if (comptime !platforms.isEnabled(.cuda)) { - return error.UnsupportedPlatform; - } - if (gate_up.bias != null or down.bias != null) { - return error.UnsupportedBias; - } - - const runner_options = try parameters.flashinfer_cutlass.runnerOptions(); - const expert_partition = gate_up.weight.shape().partition(.expert); - - if (quant_scheme != null and quant_scheme == .nvfp4) { - const gate_up_weight_unpacked = unpackedWeight(gate_up); - const down_weight_unpacked = unpackedWeight(down); - - // TODO(Corentin): Do error checking on nvfp4 - // Also, maybe pass `zml.nn.Linear` directly - if (expert_partition.eql(.init(.experts))) { - break :b zml.ops.manualComputation( - (struct { - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - gate_up_weight_unpacked: zml.Tensor, - down_weight_unpacked: zml.Tensor, - gate_up_input_scale: zml.Tensor, - gate_up_scales: zml.Tensor, - gate_up_global_scale: zml.Tensor, - down_input_scale: zml.Tensor, - down_scales: zml.Tensor, - down_global_scale: zml.Tensor, - activation: cutlass_flashinfer.Activation, - enable_pdl: bool, - gemm1_tactic: i32, - gemm2_tactic: i32, - workspace_query_device: i32, - - fn body( - self: @This(), - _: zml.Shape, - ) zml.Tensor { - const local_num_experts = self.gate_up_weight_unpacked.dim(.expert); - const partition_id = zml.ops.partitionId().convert(.i32); - const expert_start = partition_id.scale(local_num_experts).convert(.i32); - const expert_end = expert_start.addConstant(local_num_experts); - - const local_route_mask = self.topk_ids - .cmp(.GE, expert_start) - .logical(.AND, self.topk_ids.cmp(.LT, expert_end)); - const local_topk_ids = local_route_mask.select( - self.topk_ids.sub(expert_start), - zml.Tensor.scalar(-1, .i32), - ); - const local_topk_weights = local_route_mask.select( - self.topk_weights, - zml.Tensor.scalar(-1, self.topk_weights.dtype()), - ); - - const local_output = cutlass_flashinfer.fusedExpertsNvfp4( - self.input, - self.gate_up_weight_unpacked, - self.down_weight_unpacked, - local_topk_weights, - local_topk_ids, - self.gate_up_input_scale, - self.gate_up_scales, - self.gate_up_global_scale, - self.down_input_scale, - self.down_scales, - self.down_global_scale, - .{ - .workspace_query_device = self.workspace_query_device, - .activation = self.activation, - .enable_pdl = self.enable_pdl, - .gemm1_tactic = self.gemm1_tactic, - .gemm2_tactic = self.gemm2_tactic, - }, - ) catch |err| stdx.debug.panic( - "FlashInfer CUTLASS NVFP4 MoE backend failed: {}", - .{err}, - ); - const local_reshaped = local_output - .reshape(self.input.shape().dims()) - .withTags(.{ .b, .s, .d }); - return zml.ops.allReduce(local_reshaped, zml.Tensor.add); - } - }).body, - .{ - .input = input, - .topk_ids = topk_ids, - .topk_weights = topk_weights, - .gate_up_weight_unpacked = gate_up_weight_unpacked, - .down_weight_unpacked = down_weight_unpacked, - .gate_up_input_scale = gate_up.quantization.?.input_scale.?.asMultiplier(), - .gate_up_scales = gate_up.quantization.?.scales, - .gate_up_global_scale = gate_up.quantization.?.global_scale.?.asMultiplier(), - .down_input_scale = down.quantization.?.input_scale.?.asMultiplier(), - .down_scales = down.quantization.?.scales, - .down_global_scale = down.quantization.?.global_scale.?.asMultiplier(), - .activation = runner_options.activation, - .enable_pdl = runner_options.enable_pdl, - .gemm1_tactic = runner_options.gemm1_tactic, - .gemm2_tactic = runner_options.gemm2_tactic, - .workspace_query_device = runner_options.workspace_query_device, - }, - input.shape(), - ); - } - - break :b try cutlass_flashinfer.fusedExpertsNvfp4( - input, - gate_up_weight_unpacked, - down_weight_unpacked, - topk_weights, - topk_ids, - gate_up.quantization.?.input_scale.?.asMultiplier(), - gate_up.quantization.?.scales, - gate_up.quantization.?.global_scale.?.asMultiplier(), - down.quantization.?.input_scale.?.asMultiplier(), - down.quantization.?.scales, - down.quantization.?.global_scale.?.asMultiplier(), - runner_options, - ); - } - - if (expert_partition.eql(.init(.experts))) { - break :b zml.ops.manualComputation( - (struct { - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - weights_gate_up: zml.Tensor, - weights_down: zml.Tensor, - activation: cutlass_flashinfer.Activation, - enable_pdl: bool, - gemm1_tactic: i32, - gemm2_tactic: i32, - workspace_query_device: i32, - - fn body( - self: @This(), - _: zml.Shape, - ) zml.Tensor { - const local_num_experts = self.weights_gate_up.dim(.expert); - const partition_id = zml.ops.partitionId().convert(.i32); - const expert_start = partition_id.scale(local_num_experts).convert(.i32); - const expert_end = expert_start.addConstant(local_num_experts); - - const local_route_mask = self.topk_ids - .cmp(.GE, expert_start) - .logical(.AND, self.topk_ids.cmp(.LT, expert_end)); - const local_topk_ids = local_route_mask.select( - self.topk_ids.sub(expert_start), - zml.Tensor.scalar(-1, .i32), - ); - const local_topk_weights = local_route_mask.select( - self.topk_weights, - zml.Tensor.scalar(-1, self.topk_weights.dtype()), - ); - - const local_output = cutlass_flashinfer.fusedExpertsBf16( - self.input, - self.weights_gate_up, - self.weights_down, - local_topk_weights, - local_topk_ids, - .{ - .workspace_query_device = self.workspace_query_device, - .activation = self.activation, - .enable_pdl = self.enable_pdl, - .gemm1_tactic = self.gemm1_tactic, - .gemm2_tactic = self.gemm2_tactic, - }, - ) catch |err| stdx.debug.panic( - "FlashInfer CUTLASS MoE backend failed: {}", - .{err}, - ); - const local_reshaped = local_output - .reshape(self.input.shape().dims()) - .withTags(.{ .b, .s, .d }); - return zml.ops.allReduce(local_reshaped, zml.Tensor.add); - } - }).body, - .{ - .input = input, - .topk_ids = topk_ids, - .topk_weights = topk_weights, - .weights_gate_up = gate_up.weight, - .weights_down = down.weight, - .activation = runner_options.activation, - .enable_pdl = runner_options.enable_pdl, - .gemm1_tactic = runner_options.gemm1_tactic, - .gemm2_tactic = runner_options.gemm2_tactic, - .workspace_query_device = runner_options.workspace_query_device, - }, - input.shape(), - ); - } - - break :b try cutlass_flashinfer.fusedExpertsBf16( - input, - gate_up.weight, - down.weight, - topk_weights, - topk_ids, - runner_options, - ); - }, - inline .triton, .fly => |p, backend| b: { - const layout = try backend.expertsLayout(gate_up_scheme); - const args: fused_experts.FusedExpertsArgs = .{ - .hidden_states = input, - .gate_up = gate_up, - .down = down, - .topk_weights = topk_weights, - .topk_ids = topk_ids, - .activation = p.activation, - .activation_threshold = opts.activation_threshold, - .quantize_input = opts.quantize_input, - .gate_up_layout = layout.gate_up, - .routing_weight_placement = opts.routing_weight_placement, - }; - const expert_partition = gate_up.weight.shape().partition(.expert); - - if (!expert_partition.eql(.init(.experts))) { - break :b try fused_experts.fusedExperts(args, backend); - } - - break :b zml.ops.manualComputation( - (struct { - args: fused_experts.FusedExpertsArgs, - global_num_experts: i64, - - fn call(self: @This(), _: zml.Shape) zml.Tensor { - const local_args = self.args; - const local_num_experts = local_args.gate_up.weight.dim(.expert); - const partition_id = zml.ops.partitionId().convert(.i32); - const expert_start = partition_id.scale(local_num_experts).convert(.i32); - const global_expert_ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); - - // Map global expert ids to local ids, or -1 for experts outside this partition. - const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) - .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); - var mapped_args = local_args; - mapped_args.expert_map = local_expert_mask.select( - global_expert_ids.sub(expert_start), - zml.Tensor.scalar(-1, .i32), - ); - - const local_output = fused_experts.fusedExperts(mapped_args, backend) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); - const local_reshaped = local_output.reshape(local_args.hidden_states.shape().dims()).withTags(.{ .b, .s, .d }); - return zml.ops.allReduce(local_reshaped, zml.Tensor.add); - } - }).call, - .{ .args = args, .global_num_experts = gate_up.weight.dim(.expert) }, - input.shape(), - ); - }, - .mosaic_tpu => b: { - const expert_partition = gate_up.weight.shape().partition(.expert); - - if (expert_partition.eql(.init(.experts))) { - const global_num_experts = down.weight.dim(.expert); - const partial_output = zml.ops.manualComputation( - (struct { - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - weights_gate_up: zml.Tensor, - weights_down: zml.Tensor, - activation: mosaic_tpu.ActivationMode, - global_num_experts: i64, - gate_up_scales: ?zml.Tensor, - bias_gate_up: ?zml.Tensor, - down_scales: ?zml.Tensor, - bias_down: ?zml.Tensor, - - fn body(self: @This(), _: zml.Shape) zml.Tensor { - const local_num_experts = self.weights_gate_up.dim(.expert); - const partition_id = zml.ops.partitionId().convert(.i32); - const expert_start = partition_id.scale(local_num_experts).convert(.i32); - const global_expert_ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); - - const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) - .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); - const expert_map = local_expert_mask.select( - global_expert_ids.sub(expert_start), - zml.Tensor.scalar(-1, .i32), - ); - const local_output = mosaic_tpu.fusedExpertsImpl( - self.input, - self.weights_gate_up, - self.weights_down, - self.topk_weights, - self.topk_ids, - .{ - .activation = self.activation, - .global_num_experts = self.global_num_experts, - .expert_map = expert_map, - .w1_scale = self.gate_up_scales, - .w2_scale = self.down_scales, - .w1_bias = self.bias_gate_up, - .w2_bias = self.bias_down, - }, - ) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); - return local_output.reshape(self.input.shape().dims()).withTags(.{ .b, .s, .d }); - } - }).body, - .{ - .input = input, - .topk_ids = topk_ids, - .topk_weights = topk_weights, - .weights_gate_up = gate_up.weight, - .weights_down = down.weight, - .activation = parameters.mosaic_tpu.activation, - .global_num_experts = global_num_experts, - .gate_up_scales = gate_up_scales, - .bias_gate_up = gate_up.bias, - .down_scales = down_scales, - .bias_down = down.bias, - }, - input.shape(), - ); - break :b zml.ops.allReduce(partial_output, zml.Tensor.add); - } - - break :b try mosaic_tpu.fusedExpertsImpl( - input, - gate_up.weight, - down.weight, - topk_weights, - topk_ids, - .{ - .activation = parameters.mosaic_tpu.activation, - .global_num_experts = gate_up.weight.dim(.expert), - .w1_scale = gate_up_scales, - .w2_scale = down_scales, - .w1_bias = gate_up.bias, - .w2_bias = down.bias, - }, - ); - }, - .metal => b: { - const gate_up_weight_unpacked = unpackedWeight(gate_up); - const down_weight_unpacked = unpackedWeight(down); - break :b try metal.fusedExpertsImpl( - input, - gate_up_weight_unpacked, - down_weight_unpacked, - topk_weights, - topk_ids, - .{ - .activation = parameters.metal.activation, - .global_num_experts = gate_up_weight_unpacked.dim(.expert), - .w1_scale = gate_up_scales, - .w2_scale = down_scales, - .w1_global_scale = gate_up_global_scale, - .w2_global_scale = down_global_scale, - .w1_bias = gate_up.bias, - .w2_bias = down.bias, - }, - ); - }, + .flashinfer_cutlass => |p| cutlass_flashinfer.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts, p), + inline .triton, .fly => |p, backend| fused_experts.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts, p, backend), + .mosaic_tpu => |p| mosaic_tpu.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts, p), + .metal => |p| metal.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts, p), }; } -fn unpackedWeight(linear: zml.nn.Linear) zml.Tensor { +pub fn unpackedWeight(linear: zml.nn.Linear) zml.Tensor { const quantization = linear.quantization orelse return linear.weight; return if (zml.nn.isPackedFp4(quantization.scheme, linear.weight.dtype())) zml.nn.unpackFp4(linear.weight, linear.tag, linear.tag) diff --git a/zml/moe/mosaic_tpu.zig b/zml/moe/mosaic_tpu.zig index 847033b0..41936baa 100644 --- a/zml/moe/mosaic_tpu.zig +++ b/zml/moe/mosaic_tpu.zig @@ -264,6 +264,106 @@ fn applyActivation(x: Tensor, mode: ActivationMode) Tensor { }; } +pub fn fusedExperts( + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + opts: zml.moe.Options, + parameters: Parameters, +) zml.Tensor { + stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); + stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); + stdx.debug.assert(opts.activation_threshold == null, "Activation thresholds require the Triton MoE backend", .{}); + + const gate_up_scales: ?zml.Tensor = if (gate_up.quantization) |q| q.scales else null; + const down_scales: ?zml.Tensor = if (down.quantization) |q| q.scales else null; + + const expert_partition = gate_up.weight.shape().partition(.expert); + + if (expert_partition.eql(.init(.experts))) { + const global_num_experts = down.weight.dim(.expert); + const partial_output = zml.ops.manualComputation( + (struct { + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + weights_gate_up: zml.Tensor, + weights_down: zml.Tensor, + activation: ActivationMode, + global_num_experts: i64, + gate_up_scales: ?zml.Tensor, + bias_gate_up: ?zml.Tensor, + down_scales: ?zml.Tensor, + bias_down: ?zml.Tensor, + + fn body(self: @This(), _: zml.Shape) zml.Tensor { + const local_num_experts = self.weights_gate_up.dim(.expert); + const partition_id = zml.ops.partitionId().convert(.i32); + const expert_start = partition_id.scale(local_num_experts).convert(.i32); + const global_expert_ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); + + const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) + .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); + const expert_map = local_expert_mask.select( + global_expert_ids.sub(expert_start), + zml.Tensor.scalar(-1, .i32), + ); + const local_output = fusedExpertsImpl( + self.input, + self.weights_gate_up, + self.weights_down, + self.topk_weights, + self.topk_ids, + .{ + .activation = self.activation, + .global_num_experts = self.global_num_experts, + .expert_map = expert_map, + .w1_scale = self.gate_up_scales, + .w2_scale = self.down_scales, + .w1_bias = self.bias_gate_up, + .w2_bias = self.bias_down, + }, + ); + return local_output.reshape(self.input.shape().dims()).withTags(.{ .b, .s, .d }); + } + }).body, + .{ + .input = input, + .topk_ids = topk_ids, + .topk_weights = topk_weights, + .weights_gate_up = gate_up.weight, + .weights_down = down.weight, + .activation = parameters.activation, + .global_num_experts = global_num_experts, + .gate_up_scales = gate_up_scales, + .bias_gate_up = gate_up.bias, + .down_scales = down_scales, + .bias_down = down.bias, + }, + input.shape(), + ); + return zml.ops.allReduce(partial_output, zml.Tensor.add); + } + + return fusedExpertsImpl( + input, + gate_up.weight, + down.weight, + topk_weights, + topk_ids, + .{ + .activation = parameters.activation, + .global_num_experts = gate_up.weight.dim(.expert), + .w1_scale = gate_up_scales, + .w2_scale = down_scales, + .w1_bias = gate_up.bias, + .w2_bias = down.bias, + }, + ); +} + pub fn fusedExpertsImpl( hidden_states: Tensor, w1: Tensor, @@ -271,21 +371,22 @@ pub fn fusedExpertsImpl( topk_weights: Tensor, topk_ids: Tensor, opts: Options, -) !Tensor { - try validateOptions(opts); +) Tensor { + validateOptions(opts) catch |e| stdx.debug.panic("Invalid options for fusedExpertsImpl: {}", .{e}); const b = hidden_states.dim(.b); const s = hidden_states.dim(.s); const hidden = hidden_states.reshape(.{ .token = b * s, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); - const gate_up = try canonicalizeGateUp(w1, hidden.dim(.in)); - const down = try canonicalizeDown(w2, hidden.dim(.in), gate_up.dim(.out)); + // TODO(Corentin): Better error message + const gate_up = canonicalizeGateUp(w1, hidden.dim(.in)) catch |e| stdx.debug.panic("Invalid gate_up weights for fusedExpertsImpl: {}", .{e}); + const down = canonicalizeDown(w2, hidden.dim(.in), gate_up.dim(.out)) catch |e| stdx.debug.panic("Invalid down weights for fusedExpertsImpl: {}", .{e}); const weights = topk_weights.reshape(.{ .token = b * s, .topk = topk_weights.dim(.top_expert) }).withTags(.{ .token, .topk }); const ids = topk_ids.reshape(.{ .token = b * s, .topk = topk_ids.dim(.top_expert) }).withTags(.{ .token, .topk }); const num_experts = if (opts.global_num_experts != -1) opts.global_num_experts else gate_up.dim(.expert); if (opts.expert_map) |expert_map| { - if (expert_map.dtype() != .i32) return error.UnsupportedType; - if (expert_map.rank() != 1 or expert_map.dim(.expert) != num_experts) return error.InvalidShape; + if (expert_map.dtype() != .i32) stdx.debug.panic("expert_map must be of dtype i32, got {}", .{expert_map.dtype()}); + if (expert_map.rank() != 1 or expert_map.dim(.expert) != num_experts) stdx.debug.panic("expert_map must be a 1D tensor of length num_experts, got shape {}", .{expert_map.shape()}); } const flat_ids_global = ids.transpose(.{ .topk, .token }).flatten().withTags(.{.route}); From 11aceb7430789c1e142b86e13580eedb99c2b852 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 5 Oct 2026 16:02:38 +0200 Subject: [PATCH 749/881] zml/moe: refactor MoE (#884) This PR removes a lot of slop in the MoE part of ZML, as well as some global improvements. Among the improvements: * Unify activation functions and parameters across backends * Unify activation computation across backends * Replace error bubbling with explicit asserts (because we have no meanignful way to handle them) * Add a StableHLO MoE backend --- examples/llm/models/qwen3_5_moe/inference.zig | 8 +- examples/llm/models/qwen3_5_moe/model.zig | 31 +-- zml/BUILD.bazel | 1 + zml/moe/cute_kernels/moe.zig | 7 +- zml/moe/cute_mxfp4.zig | 45 +++-- zml/moe/cutlass_flashinfer.zig | 97 +++------ zml/moe/fly_kernels/moe.zig | 2 +- zml/moe/fused_experts.zig | 185 ++++++++---------- zml/moe/metal.zig | 155 ++++++--------- zml/moe/moe.zig | 178 +++++++++-------- zml/moe/mosaic_tpu.zig | 76 ++----- zml/moe/stablehlo.zig | 182 +++++++++++++++++ zml/moe/triton.zig | 3 +- zml/moe/triton_kernels/triton_kernels.zig | 2 +- zml/moe/triton_mxfp4.zig | 51 ++--- zml/tensor.zig | 99 ++++++++++ 16 files changed, 635 insertions(+), 487 deletions(-) create mode 100644 zml/moe/stablehlo.zig diff --git a/examples/llm/models/qwen3_5_moe/inference.zig b/examples/llm/models/qwen3_5_moe/inference.zig index c31f79cd..80c6be65 100644 --- a/examples/llm/models/qwen3_5_moe/inference.zig +++ b/examples/llm/models/qwen3_5_moe/inference.zig @@ -10,7 +10,7 @@ const log = std.log.scoped(.qwen3_5_moe); pub const CompilationParameters = struct { kv_cache: model.KvCache, rng: zml.Tensor.Rng, - moe_parameters: zml.moe.Parameters, + moe_backend: zml.moe.Backend, seqlen: u32, shardings: common.Shardings, xla_dump_to: ?[]const u8, @@ -20,7 +20,7 @@ pub const CompilationParameters = struct { return .{ .kv_cache = .init(config, 1, seqlen, dtype, .f32, shardings.model), .rng = .init(), - .moe_parameters = .init(.fromBackend(moe_backend, config.text_config.num_experts_per_tok, zml.moe.ActivationMode.silu)), + .moe_backend = moe_backend, .seqlen = seqlen, .shardings = shardings, .xla_dump_to = "/home/ubuntu/xla_dump", @@ -282,7 +282,7 @@ fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *con .layer_index = zml.Tensor.init(.{}, .u32), }, .config = mdl.config, - .moe_parameters = parameters.moe_parameters, + .moe_backend = parameters.moe_backend, }}, parameters, progress, phase, "full-attention layer"); } @@ -297,7 +297,7 @@ fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *c .layer_index = zml.Tensor.init(.{}, .u32), }, .config = mdl.config, - .moe_parameters = parameters.moe_parameters, + .moe_backend = parameters.moe_backend, }}, parameters, progress, phase, "linear-attention layer"); } diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index f4096bfb..6b6f9bb0 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -233,10 +233,10 @@ pub const Model = struct { active_length: zml.Tensor, kv_cache: KvCache, rng: zml.Tensor.Rng, - moe_parameters: zml.moe.Parameters, + moe_backend: zml.moe.Backend, ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { const tokens = tokens_.withPartialTags(.{.s}); - const new_tokens, const updated_kv_cache, const new_rng = self.text_model.forward(tokens, token_index, active_length, kv_cache, self.config, rng, moe_parameters); + const new_tokens, const updated_kv_cache, const new_rng = self.text_model.forward(tokens, token_index, active_length, kv_cache, self.config, rng, moe_backend); return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; } }; @@ -355,7 +355,7 @@ pub const TextModel = struct { kv_cache: KvCache, config: Config, rng: zml.Tensor.Rng, - moe_parameters: zml.moe.Parameters, + moe_backend: zml.moe.Backend, ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { var hidden_states = EmbedTokens.forward(.{ .embedding = self.embed_tokens, @@ -364,7 +364,7 @@ pub const TextModel = struct { var updated_kv_cache = kv_cache; for (self.layers, 0..) |layer, i| { - hidden_states, updated_kv_cache = layer.forward(hidden_states, token_index, active_length, updated_kv_cache.atLayer(i), config, moe_parameters); + hidden_states, updated_kv_cache = layer.forward(hidden_states, token_index, active_length, updated_kv_cache.atLayer(i), config, moe_backend); } const result = Sampler.sampleTokens(.{ @@ -394,7 +394,7 @@ pub const TransformerLayer = struct { token_index: zml.Tensor, cache: KvCache.SelfAttnCache, config: Config, - moe_parameters: zml.moe.Parameters, + moe_backend: zml.moe.Backend, }; pub const SelfAttnOutput = struct { @@ -408,7 +408,7 @@ pub const TransformerLayer = struct { active_length: zml.Tensor, cache: KvCache.GatedDeltaNetCache, config: Config, - moe_parameters: zml.moe.Parameters, + moe_backend: zml.moe.Backend, }; pub const LinearAttnOutput = struct { @@ -459,7 +459,7 @@ pub const TransformerLayer = struct { const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); - const moe_output = self.moe.forward(normalized_hidden, input.moe_parameters); + const moe_output = self.moe.forward(normalized_hidden, input.moe_backend); return .{ .hidden = moe_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), @@ -483,7 +483,7 @@ pub const TransformerLayer = struct { const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); - const moe_output = self.moe.forward(normalized_hidden, input.moe_parameters); + const moe_output = self.moe.forward(normalized_hidden, input.moe_backend); return .{ .hidden = moe_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), @@ -498,7 +498,7 @@ pub const TransformerLayer = struct { active_length: zml.Tensor, kv_cache: KvCache.LayerView, config: Config, - moe_parameters: zml.moe.Parameters, + moe_backend: zml.moe.Backend, ) struct { zml.Tensor, KvCache } { _ = config; const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); @@ -522,7 +522,7 @@ pub const TransformerLayer = struct { const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); - const moe_output = self.moe.forward(normalized_hidden, moe_parameters); + const moe_output = self.moe.forward(normalized_hidden, moe_backend); return .{ moe_output.add(x1).withPartitioning(.{ .d = .replicated }), updated_kv_cache }; } @@ -722,9 +722,9 @@ const Router = struct { pub fn forward(self: Router, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { const router_logits = self.router.forward(x, .f32); - const routing = router_logits.topK(.{ .top_expert = .expert }, self.num_experts_per_tok, .{}); + const routing = router_logits.topK(.{ .topk = .expert }, self.num_experts_per_tok, .{}); const topk_ids = routing.indices.convert(.i32); - const router_scores = routing.values.softmax(.top_expert); + const router_scores = routing.values.softmax(.topk); return .{ router_scores, topk_ids }; } }; @@ -768,7 +768,7 @@ pub const Moe = struct { _ = allocator; } - pub fn forward(self: Moe, x: zml.Tensor, moe_parameters: zml.moe.Parameters) zml.Tensor { + pub fn forward(self: Moe, x: zml.Tensor, moe_backend: zml.moe.Backend) zml.Tensor { const routing_scores, const topk_ids = self.router.forward(x); const moe_output = zml.moe.forwardMoe( @@ -777,12 +777,13 @@ pub const Moe = struct { routing_scores, self.gate_up_proj, self.down_proj, + moe_backend, .{ + .activation = .{ .swiglu = .{} }, .quantize_input = false, .routing_weight_placement = .after_down, }, - moe_parameters, - ) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); + ); const shared_gate = self.shared_expert_gate.forward(x, x.dtype()).sigmoid().broad(x.shape()); const shared = self.shared_expert.forward(x).rename(.{ .dout = .d }).mul(shared_gate).withPartitioning(.{ diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 50b7254b..d1174cca 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -116,6 +116,7 @@ zig_library( "moe/moe.zig", "moe/mosaic_tpu.zig", "moe/mosaic_tpu_kernels/gmm_ep.zig", + "moe/stablehlo.zig", "moe/triton.zig", "moe/triton_kernels/mxfp4.zig", "moe/triton_kernels/triton_kernels.zig", diff --git a/zml/moe/cute_kernels/moe.zig b/zml/moe/cute_kernels/moe.zig index ca41474d..e5794dbb 100644 --- a/zml/moe/cute_kernels/moe.zig +++ b/zml/moe/cute_kernels/moe.zig @@ -4,6 +4,7 @@ //! and the top-k reduction. Routing weights are applied before the down //! projection, with the reference model's BF16 rounding. const std = @import("std"); +const stdx = @import("stdx"); const zml = @import("../../zml.zig"); const triton_mxfp4 = @import("../triton_mxfp4.zig"); @@ -27,8 +28,9 @@ pub const Inputs = struct { /// The kernels are tuned for the DeepSeek V4.1 expert geometry, shared by the /// main MoE and the D-Spark drafter. Expert count and top-k are parameters. -pub fn isSupported(hidden: i64, intermediate: i64) bool { - return hidden == 5120 and intermediate == 2304; +pub fn validateShapes(hidden: i64, intermediate: i64) void { + stdx.debug.assert(hidden == 5120, "cute_mxfp4 backend expects hidden dimension to be 5120, got {}", .{hidden}); + stdx.debug.assert(intermediate == 2304, "cute_mxfp4 backend expects intermediate dimension to be 2304, got {}", .{intermediate}); } /// Route each token/top-k pair to its own one-row group instead of grouping @@ -113,7 +115,6 @@ pub fn forward(tokens: i64, hidden: i64, intermediate: i64, experts: i64, topk: /// `forward` with an explicit GEMM tile N, for tuning. `null` selects it from /// the batch size. pub fn forwardWithTile(tokens: i64, hidden: i64, intermediate: i64, experts: i64, topk: i64, swiglu_limit: f32, a: Inputs, tile_n: ?i64) zml.Tensor { - if (!isSupported(hidden, intermediate)) @panic("unsupported SM100 CuTe MoE shape"); const n = tile_n orelse tileN(tokens, experts, topk); const direct = isDirect(tokens); const routing = route(tokens, hidden, intermediate, experts, topk, n, direct, a.ids); diff --git a/zml/moe/cute_mxfp4.zig b/zml/moe/cute_mxfp4.zig index 3e2d0edf..7cb235d0 100644 --- a/zml/moe/cute_mxfp4.zig +++ b/zml/moe/cute_mxfp4.zig @@ -1,4 +1,7 @@ //! Blackwell MXFP4 MoE backend with persistent CuTe GEMMs. + +const stdx = @import("stdx"); + const zml = @import("../zml.zig"); const triton_mxfp4 = @import("triton_mxfp4.zig"); pub const kernels = @import("cute_kernels/moe.zig"); @@ -17,6 +20,13 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major == 10; } +pub fn validateOptions(opts: zml.moe.Options) void { + stdx.debug.assert(opts.activation == .swiglu, "cute_mxfp4 backend only accepts swiglu activation, got {}", .{opts.activation}); + stdx.debug.assert(opts.activation.swiglu.limit != null, "cute_mxfp4 backend requires swiglu limit to be set", .{}); + stdx.debug.assert(opts.activation.swiglu.bias == null, "cute_mxfp4 backend requires swiglu bias to be null", .{}); + stdx.debug.assert(opts.activation.swiglu.scale == null, "cute_mxfp4 backend requires swiglu scale to be null", .{}); +} + pub fn fusedExperts( input: zml.Tensor, ids: zml.Tensor, @@ -24,31 +34,30 @@ pub fn fusedExperts( gate_up: zml.nn.Linear, down: zml.nn.Linear, options: zml.moe.Options, - parameters: Parameters, -) !zml.Tensor { - if (parameters.activation != .silu) return error.UnsupportedActivation; - if (input.dtype() != .bf16) return error.UnsupportedDataType; - if (gate_up.bias != null or down.bias != null) return error.UnsupportedBias; - const gq = gate_up.quantization orelse return error.UnsupportedQuantization; - const dq = down.quantization orelse return error.UnsupportedQuantization; - if (gq.scheme != .mxfp4 or dq.scheme != .mxfp4) return error.UnsupportedQuantization; +) zml.Tensor { + validateOptions(options); + stdx.debug.assert(input.dtype() == .bf16, "cute_mxfp4 backend only supports bf16 inputs, got {}", .{input.dtype()}); + stdx.debug.assert(gate_up.bias == null and down.bias == null, "cute_mxfp4 backend expects gate_up bias and down bias to be null", .{}); + + const gq = gate_up.quantization orelse @panic("cute_mxfp4 backend requires gate_up quantization to be set"); + const dq = down.quantization orelse @panic("cute_mxfp4 backend requires down quantization to be set"); + stdx.debug.assert(gq.scheme == .mxfp4 and dq.scheme == .mxfp4, "cute_mxfp4 expects gate_up and down quantization scheme to be mxfp4, got {} and {}", .{ gq.scheme, dq.scheme }); + // Weight storage for mxfp4 in HF is expressed as u8 or i8 - if ((gate_up.weight.dtype() != .u8 and gate_up.weight.dtype() != .i8) or - (down.weight.dtype() != .u8 and down.weight.dtype() != .i8)) - { - return error.UnsupportedWeightLayout; - } + stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "cute_mxfp4 expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); + stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "cute_mxfp4 expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); const expert_parallelism = gate_up.weight.shape().partition(.expert).eql(.init(.experts)); const hidden = down.weight.dim(1); const intermediate = down.weight.dim(2) * 2; + // The CuTe GEMMs are specialized on one set of dimensions - if (!kernels.isSupported(hidden, intermediate)) return error.UnsupportedShape; - const activation_threshold = options.activation_threshold orelse return error.UnsupportedActivation; + kernels.validateShapes(hidden, intermediate); + // The SwiGLU clamp is a kernel parameter, but the routing weight is // multiplied in the up epilogue, before the down projection: applying it // after would have to move into the down epilogue or the reduction. - if (options.routing_weight_placement != .before_down) return error.UnsupportedRoutingWeightPlacement; + stdx.debug.assert(options.routing_weight_placement == .before_down, "cute_mxfp4 backend only supports routing_weight_placement = .before_down, got {}", .{options.routing_weight_placement}); const context: Context = .{ .input = input, .ids = ids, @@ -57,8 +66,8 @@ pub fn fusedExperts( .s1 = gq.scales, .w2 = down.weight.bitCast(.u8), .s2 = dq.scales, - .topk = parameters.num_experts_per_tok, - .swiglu_limit = activation_threshold, + .topk = ids.dim(.topk), + .swiglu_limit = options.activation.swiglu.limit.?, .expert_parallel = expert_parallelism, }; return if (expert_parallelism) diff --git a/zml/moe/cutlass_flashinfer.zig b/zml/moe/cutlass_flashinfer.zig index 0735c3bc..18527910 100644 --- a/zml/moe/cutlass_flashinfer.zig +++ b/zml/moe/cutlass_flashinfer.zig @@ -5,72 +5,41 @@ const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); const platforms = @import("platforms"); const zml = @import("../zml.zig"); -const log = std.log.scoped(.moe_cutlass_flashinfer); +const log = std.log.scoped(.@"zml/moe/cutlass_flashinfer"); pub const auto_tactic: i32 = -1; pub const Options = struct { /// Device used at graph-construction time to select the architecture /// library and size the XLA-owned scratch buffer. - workspace_query_device: i32 = 0, - activation: Activation = .swiglu, - enable_pdl: bool = false, - gemm1_tactic: i32 = auto_tactic, + workspace_query_device: i32, + activation: zml.moe.Activation, + enable_pdl: bool, + gemm1_tactic: i32, /// GEMM2 uses FlashInfer's absolute tactic index. Query tacticCounts() to /// obtain the first valid GEMM2 index. - gemm2_tactic: i32 = auto_tactic, -}; + gemm2_tactic: i32, -pub const Activation = enum { - swiglu, - geglu, - geglu_tanh, - swiglu_step, - relu2, -}; - -pub const Parameters = struct { - num_experts_per_tok: u32, - activation: ActivationMode, - workspace_query_device: i32 = 0, - enable_pdl: bool = false, - gemm1_tactic: i32 = auto_tactic, - gemm2_tactic: i32 = auto_tactic, - - pub const ActivationMode = enum { - silu, - relu, - gelu, - }; - - pub const InitOptions = struct { - num_experts_per_tok: u32, - activation: ActivationMode, - }; - - pub fn init(opts: InitOptions) Parameters { + pub fn fromGlobalOptions(opts: zml.moe.Options) Options { return .{ - .num_experts_per_tok = opts.num_experts_per_tok, + .workspace_query_device = 0, .activation = opts.activation, - }; - } - - pub fn runnerOptions(self: Parameters) !Options { - return .{ - .workspace_query_device = self.workspace_query_device, - .activation = switch (self.activation) { - .silu => .swiglu, - // Tensor.gelu() is ZML's tanh GELU approximation. - .gelu => .geglu_tanh, - .relu => .relu2, - }, - .enable_pdl = self.enable_pdl, - .gemm1_tactic = self.gemm1_tactic, - .gemm2_tactic = self.gemm2_tactic, + .enable_pdl = false, + .gemm1_tactic = auto_tactic, + .gemm2_tactic = auto_tactic, }; } }; +pub fn validateOptions(opts: zml.moe.Options) void { + stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); + stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); + stdx.debug.assert(opts.activation == .swiglu, "cute_mxfp4 backend only accepts swiglu activation, got {}", .{opts.activation}); + stdx.debug.assert(opts.activation.swiglu.limit == null, "Activation thresholds require the Triton MoE backend", .{}); + stdx.debug.assert(opts.activation.swiglu.bias == null, "flashinfer_cutlass backend requires swiglu bias to be null", .{}); + stdx.debug.assert(opts.activation.swiglu.scale == null, "flashinfer_cutlass backend requires swiglu scale to be null", .{}); +} + const Input = struct { hidden_states: zml.Tensor, fc1_weights: zml.Tensor, @@ -202,13 +171,15 @@ fn currentRunners() !*Runners { return runners orelse error.RunnersNotLoaded; } -fn cutlassActivation(activation: Activation) i32 { +fn cutlassActivation(activation: zml.moe.Activation) i32 { return switch (activation) { + .gelu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GELU, + .relu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_RELU, + .silu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SILU, .swiglu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SWIGLU, + .swiglu_step => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SWIGLU_STEP, .geglu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GEGLU, .geglu_tanh => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GEGLU_TANH, - .swiglu_step => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SWIGLU_STEP, - .relu2 => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_RELU2, }; } @@ -405,11 +376,11 @@ pub fn fc1BlockScaleShape( num_experts: i64, hidden_size: i64, intermediate_size: i64, - activation: Activation, + activation: zml.moe.Activation, ) zml.Shape { const rows = switch (activation) { + .silu, .gelu, .relu => intermediate_size, .swiglu, .geglu, .geglu_tanh, .swiglu_step => 2 * intermediate_size, - .relu2 => intermediate_size, }; return .init( .{ num_experts, roundUp(rows, 128), roundUp(@divExact(hidden_size, 16), 4) }, @@ -474,8 +445,8 @@ fn validateInputs( const num_experts = fc1_weights.dim(0); const fc1_rows = fc1_weights.dim(1); const intermediate_size = switch (options.activation) { + .silu, .gelu, .relu => fc1_rows, .swiglu, .geglu, .geglu_tanh, .swiglu_step => @divExact(fc1_rows, 2), - .relu2 => fc1_rows, }; const top_k = topk_ids.dim(2); @@ -630,8 +601,8 @@ pub fn fusedExpertsBf16( const numExperts = fc1_weights.dim(0); const fc1Rows = fc1_weights.dim(1); const intermediateSize = switch (options.activation) { + .silu, .gelu, .relu => fc1Rows, .swiglu, .geglu, .geglu_tanh, .swiglu_step => @divExact(fc1Rows, 2), - .relu2 => fc1Rows, }; const topK = topk_ids.dim(2); if (batch <= 0 or sequence <= 0 or hiddenSize <= 0 or @@ -703,11 +674,7 @@ pub fn fusedExperts( gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: zml.moe.Options, - parameters: Parameters, ) zml.Tensor { - stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); - stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); - stdx.debug.assert(opts.activation_threshold == null, "Activation thresholds require the Triton MoE backend", .{}); if (comptime !platforms.isEnabled(.cuda)) { @panic("FlashInfer CUTLASS MoE is only supported on CUDA platforms"); } @@ -716,7 +683,7 @@ pub fn fusedExperts( @panic("FlashInfer CUTLASS MoE does not support bias in gate_up or down linear layers"); } - const runner_options = try parameters.runnerOptions(); + const runner_options: Options = .fromGlobalOptions(opts); const expert_partition = gate_up.weight.shape().partition(.expert); const quant_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; @@ -741,7 +708,7 @@ pub fn fusedExperts( down_input_scale: zml.Tensor, down_scales: zml.Tensor, down_global_scale: zml.Tensor, - activation: Activation, + activation: zml.moe.Activation, enable_pdl: bool, gemm1_tactic: i32, gemm2_tactic: i32, @@ -840,7 +807,7 @@ pub fn fusedExperts( topk_weights: zml.Tensor, weights_gate_up: zml.Tensor, weights_down: zml.Tensor, - activation: Activation, + activation: zml.moe.Activation, enable_pdl: bool, gemm1_tactic: i32, gemm2_tactic: i32, diff --git a/zml/moe/fly_kernels/moe.zig b/zml/moe/fly_kernels/moe.zig index 3a287fcb..201ef3ac 100644 --- a/zml/moe/fly_kernels/moe.zig +++ b/zml/moe/fly_kernels/moe.zig @@ -34,7 +34,7 @@ pub fn supports(opts: shared.FusedExpertsArgs) bool { else => return false, } } - const routes = opts.hidden_states.dim(.b) * opts.hidden_states.dim(.s) * opts.topk_ids.dim(.top_expert); + const routes = opts.hidden_states.dim(.b) * opts.hidden_states.dim(.s) * opts.topk_ids.dim(.topk); return routes > 0 and routes <= std.math.maxInt(i32) / 5120; } diff --git a/zml/moe/fused_experts.zig b/zml/moe/fused_experts.zig index 57a510d9..2c58baa8 100644 --- a/zml/moe/fused_experts.zig +++ b/zml/moe/fused_experts.zig @@ -10,35 +10,12 @@ const fly = @import("fly_kernels/moe.zig"); const triton = @import("triton.zig"); const callFusedMoe = triton.call; -const log = std.log.scoped(.moe); +const log = std.log.scoped(.@"zml/moe/fused_experts"); test { std.testing.refAllDecls(@This()); } -pub const Parameters = struct { - num_experts_per_tok: u32, - activation: ActivationMode, - - pub const ActivationMode = enum { - silu, - relu, - gelu, - }; - - pub const InitOptions = struct { - num_experts_per_tok: u32, - activation: ActivationMode, - }; - - pub fn init(opts: InitOptions) Parameters { - return .{ - .num_experts_per_tok = opts.num_experts_per_tok, - .activation = opts.activation, - }; - } -}; - pub fn fusedExperts( input: zml.Tensor, topk_ids: zml.Tensor, @@ -46,20 +23,23 @@ pub fn fusedExperts( gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: zml.moe.Options, - parameters: Parameters, comptime backend: zml.moe.Backend, ) zml.Tensor { + stdx.debug.assertComptime(backend == .triton or backend == .fly, "fusedExperts only supports the triton and fly backend", .{}); + const gate_up_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; - // TODO(Corentin): Better error message - const layout = backend.expertsLayout(gate_up_scheme) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); + const layout: zml.moe.ExpertsLayout = if (gate_up_scheme == .mxfp4) + .{ .gate_up = .interleaved, .packing = .plain } + else + .{ .gate_up = .concatenated, .packing = .plain }; + const args: FusedExpertsArgs = .{ .hidden_states = input, .gate_up = gate_up, .down = down, .topk_weights = topk_weights, .topk_ids = topk_ids, - .activation = parameters.activation, - .activation_threshold = opts.activation_threshold, + .activation = opts.activation, .quantize_input = opts.quantize_input, .gate_up_layout = layout.gate_up, .routing_weight_placement = opts.routing_weight_placement, @@ -67,7 +47,7 @@ pub fn fusedExperts( const expert_partition = gate_up.weight.shape().partition(.expert); if (!expert_partition.eql(.init(.experts))) { - return fusedExpertsImpl(args, backend) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); + return fusedExpertsImpl(args, backend); } return zml.ops.manualComputation( @@ -91,7 +71,7 @@ pub fn fusedExperts( zml.Tensor.scalar(-1, .i32), ); - const local_output = fusedExpertsImpl(mapped_args, backend) catch |err| stdx.debug.panic("moe backend failed: {}", .{err}); + const local_output = fusedExpertsImpl(mapped_args, backend); const local_reshaped = local_output.reshape(local_args.hidden_states.shape().dims()).withTags(.{ .b, .s, .d }); return zml.ops.allReduce(local_reshaped, zml.Tensor.add); } @@ -110,21 +90,21 @@ pub const FusedExpertsArgs = struct { down: zml.nn.Linear, topk_weights: Tensor, topk_ids: Tensor, - activation: Parameters.ActivationMode = .silu, + activation: zml.moe.Activation, expert_map: ?Tensor = null, - activation_threshold: ?f32 = null, /// Use FP8 activations for FP8 weights; false keeps BF16 activations. quantize_input: bool, gate_up_layout: ProjectionLayout, routing_weight_placement: RoutingWeightPlacement, }; -pub fn fusedExpertsImpl(opts: FusedExpertsArgs, comptime backend: zml.moe.Backend) !Tensor { +pub fn fusedExpertsImpl(opts: FusedExpertsArgs, comptime backend: zml.moe.Backend) Tensor { const Impl = switch (backend) { .fly => fly, .triton => triton, else => @compileError("unsupported fused-experts backend"), }; + const backend_name = @tagName(backend); if (backend == .fly and (!backend.isAvailable(zml.Compiler.current().platform) or !fly.supports(opts))) { return fusedExpertsImpl(opts, .triton); } @@ -140,7 +120,7 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs, comptime backend: zml.moe.Backen const gate_up_scheme = opts.gate_up.quantizationScheme(); const down_scheme = opts.down.quantizationScheme(); if (gate_up_scheme) |scheme| switch (scheme) { - .nvfp4 => return error.UnsupportedQuantization, + .nvfp4 => stdx.debug.panic("{s} backend doesn't support gate_up nvfp4 quantization scheme", .{backend_name}), .fp8_block128 => launch_config.block_size_k = 128, .fp8_block32 => launch_config.block_size_k = 32, .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, @@ -148,34 +128,46 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs, comptime backend: zml.moe.Backen var down_launch_config = launchConfigForTokens(num_tokens); if (down_scheme) |scheme| switch (scheme) { + .nvfp4 => stdx.debug.panic("{s} backend doesn't support down nvfp4 quantization scheme", .{backend_name}), .fp8_block128 => down_launch_config.block_size_k = 128, .fp8_block32 => down_launch_config.block_size_k = 32, .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, - .nvfp4 => return error.UnsupportedQuantization, }; const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); const gate_up = opts.gate_up.weight.withTags(.{ .expert, .out, .in }); const down = opts.down.weight.withTags(.{ .expert, .out, .mid }); - const routing_weights = topk_weights.reshape(.{ .token = num_tokens, .in = topk_weights.dim(.top_expert) }).withTags(.{ .token, .topk }); - const ids = topk_ids.reshape(.{ .token = num_tokens, .in = topk_ids.dim(.top_expert) }).withTags(.{ .token, .topk }); + const routing_weights = topk_weights.reshape(.{ .token = num_tokens, .in = topk_weights.dim(.topk) }).withTags(.{ .token, .topk }); + const ids = topk_ids.reshape(.{ .token = num_tokens, .in = topk_ids.dim(.topk) }).withTags(.{ .token, .topk }); - stdx.debug.assert(hidden.dtype() == .bf16, "expected BF16 hidden states, got {}", .{hidden.dtype()}); - stdx.debug.assert(if (gate_up_scheme == .mxfp4) gate_up.dtype() == .u8 or gate_up.dtype() == .i8 or gate_up.dtype() == .f4e2m1 else gate_up.dtype() == .bf16 or gate_up.dtype() == .f8e4m3fn or gate_up.dtype() == .f8e4m3fnuz, "unsupported gate/up weight dtype {}", .{gate_up.dtype()}); - stdx.debug.assert(if (down_scheme == .mxfp4) down.dtype() == .u8 or down.dtype() == .i8 or down.dtype() == .f4e2m1 else down.dtype() == .bf16 or down.dtype() == .f8e4m3fn or down.dtype() == .f8e4m3fnuz, "unsupported down weight dtype {}", .{down.dtype()}); - stdx.debug.assert(routing_weights.dtype() == .f32 or routing_weights.dtype() == .bf16, "expected FP32 or BF16 routing weights, got {}", .{routing_weights.dtype()}); - stdx.debug.assert(ids.dtype() == .i32, "expected I32 expert ids, got {}", .{ids.dtype()}); + stdx.debug.assert(hidden.dtype() == .bf16, "{s} backend expected BF16 hidden states, got {}", .{ backend_name, hidden.dtype() }); + if (gate_up_scheme == .mxfp4) { + stdx.debug.assert(gate_up.dtype() == .u8 or gate_up.dtype() == .i8 or gate_up.dtype() == .f4e2m1, "{s} backend expected u8, i8 or f4e2m1 gate_up data type when quantization scheme is mxfp4, got {}", .{ backend_name, gate_up.dtype() }); + } else { + stdx.debug.assert(gate_up.dtype() == .bf16 or gate_up.dtype() == .f8e4m3fn or gate_up.dtype() == .f8e4m3fnuz, "{s} backend expected gate_up data type to be bf16, fp8e4m3fn or f8e4m3fnuz, got {}", .{ backend_name, gate_up.dtype() }); + } + if (down_scheme == .mxfp4) { + stdx.debug.assert(down.dtype() == .u8 or down.dtype() == .i8 or down.dtype() == .f4e2m1, "{s} backend expected u8, i8 or f4e2m1 down data type when quantization scheme is mxfp4, got {}", .{ backend_name, down.dtype() }); + } else { + stdx.debug.assert(down.dtype() == .bf16 or down.dtype() == .f8e4m3fn or down.dtype() == .f8e4m3fnuz, "{s} backend expected down data type to be bf16, fp8e4m3fn or f8e4m3fnuz, got {}", .{ backend_name, down.dtype() }); + } + + stdx.debug.assert(routing_weights.dtype() == .f32 or routing_weights.dtype() == .bf16, "{s} backend expected f32 or bf16 routing weights, got {}", .{ backend_name, routing_weights.dtype() }); + stdx.debug.assert(ids.dtype() == .i32, "{s} backend expected i32 expert ids, got {}", .{ backend_name, ids.dtype() }); const gate_up_k = gate_up.dim(.in) * @as(i64, if (gate_up_scheme == .mxfp4 and gate_up.dtype() != .f4e2m1) 2 else 1); const down_k = down.dim(.mid) * @as(i64, if (down_scheme == .mxfp4 and down.dtype() != .f4e2m1) 2 else 1); - stdx.debug.assert(hidden.dim(.in) == gate_up_k, "hidden width {} must match gate/up input width {}", .{ hidden.dim(.in), gate_up_k }); - const activation_reduction: i64 = if (opts.activation == .relu) 1 else 2; - stdx.debug.assert(@rem(gate_up.dim(.out), activation_reduction) == 0, "gate/up output width {} must be divisible by {}", .{ gate_up.dim(.out), activation_reduction }); - stdx.debug.assert(down_k == @divFloor(gate_up.dim(.out), activation_reduction), "down input width {} must match activated width {}", .{ down_k, @divFloor(gate_up.dim(.out), activation_reduction) }); - stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "routing ids and weights must match hidden token count {}, got {} and {}", .{ hidden.dim(.token), ids.dim(.token), routing_weights.dim(.token) }); - stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "routing ids and weights must have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), routing_weights.dim(.topk) }); - stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "gate/up and down expert counts must match, got {} and {}", .{ gate_up.dim(.expert), down.dim(.expert) }); + stdx.debug.assert(hidden.dim(.in) == gate_up_k, "{s} backend expected hidden width to match gate/up input width, got {} and {}", .{ backend_name, hidden.dim(.in), gate_up_k }); + const activation_reduction: i64 = switch (opts.activation) { + .gelu, .relu, .silu => 1, + .swiglu, .swiglu_step, .geglu, .geglu_tanh => 2, + }; + stdx.debug.assert(@rem(gate_up.dim(.out), activation_reduction) == 0, "{s} backend expected gate/up output width to be divisible by {}, got {}", .{ backend_name, activation_reduction, gate_up.dim(.out) }); + stdx.debug.assert(down_k == @divFloor(gate_up.dim(.out), activation_reduction), "{s} backend expected down input width to match activated width, got {} and {}", .{ backend_name, down_k, @divFloor(gate_up.dim(.out), activation_reduction) }); + stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "{s} backend expected routing ids and weights to match hidden token count, got {}, {} and {}", .{ backend_name, ids.dim(.token), routing_weights.dim(.token), hidden.dim(.token) }); + stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "{s} backend expected routing ids and weights to have matching top-k dimensions, got {} and {}", .{ backend_name, ids.dim(.topk), routing_weights.dim(.topk) }); + stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "{s} backend expected gate/up and down expert counts to match, got {} and {}", .{ backend_name, gate_up.dim(.expert), down.dim(.expert) }); if (opts.expert_map) |expert_map| - stdx.debug.assert(expert_map.dtype() == .i32 and expert_map.rank() == 1, "expected a rank-1 I32 expert map, got rank {} and dtype {}", .{ expert_map.rank(), expert_map.dtype() }); + stdx.debug.assert(expert_map.dtype() == .i32 and expert_map.rank() == 1, "{s} backend expected a rank-1 i32 expert map, got rank {} and dtype {}", .{ backend_name, expert_map.rank(), expert_map.dtype() }); const num_experts = if (opts.expert_map) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); const routing = Impl.prepareRouting(ids, num_experts, @intCast(launch_config.block_size_m)); @@ -201,7 +193,7 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs, comptime backend: zml.moe.Backen .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), }); - var activated = applyExpertActivation(gate_up_out, opts.activation, opts.activation_threshold, opts.gate_up_layout); + var activated = zml.moe.applyActivation(gate_up_out, opts.activation, opts.gate_up_layout); if (opts.routing_weight_placement == .before_down) { const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); activated = activated.mul(weights.broad(activated.shape())); @@ -228,37 +220,13 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs, comptime backend: zml.moe.Backen return output.reshape(.{ .b = b, .token = s, .out = down.dim(.out) }); } -fn applyExpertActivation(input: Tensor, mode: Parameters.ActivationMode, activation_threshold: ?f32, layout: ProjectionLayout) Tensor { - const x = input.convert(.f32); - if (mode == .relu) { - const clipped = if (activation_threshold) |limit| x.minimum(Tensor.scalar(limit, x.dtype())) else x; - return clipped.relu().powByConst(2); - } - - const mid = @divFloor(x.dim(.out), 2); - var gate, var up = switch (layout) { - .concatenated => .{ x.slice(.out, .{ .end = mid }), x.slice(.out, .{ .start = mid }) }, - .interleaved => .{ x.slice(.out, .{ .start = 0, .step = 2 }), x.slice(.out, .{ .start = 1, .step = 2 }) }, - }; - if (activation_threshold) |limit_| { - const limit = Tensor.scalar(limit_, x.dtype()); - gate = gate.minimum(limit); - up = up.clamp(limit.negate(), limit); - } - return switch (mode) { - .silu => gate.silu().mul(up), - .relu => unreachable, - .gelu => gate.gelu().mul(up), - }; -} - test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { const allocator = std.testing.allocator; const io = std.testing.io; const platform = zml.testing.env(); const Local = struct { - fn forward(x: Tensor, layout: ProjectionLayout, threshold: ?f32) Tensor { - return applyExpertActivation(x, .silu, threshold, layout); + fn forward(x: Tensor, layout: ProjectionLayout, activation: zml.moe.Activation) Tensor { + return zml.moe.applyActivation(x, activation, layout); } }; const x: Tensor = .init(.{ .token = 1, .out = 6 }, .bf16); @@ -273,7 +241,8 @@ test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { var input = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&values)); defer input.deinit(); for ([_]?f32{ null, 2 }) |threshold| { - var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, threshold }, .{}); + const activation: zml.moe.Activation = .{ .swiglu = .{ .limit = threshold } }; + var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, activation }, .{}); defer exe.deinit(); try zml.testing.expectEqualShapes(Shape.init(.{ .token = 1, .out = 3 }, .f32), exe.output_shapes[0]); var output = try exe.eval(allocator, io, .{input}); @@ -289,31 +258,32 @@ test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { } } -test "ReLU squared activation preserves width and applies threshold before squaring" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = zml.testing.env(); - const Local = struct { - fn forward(x: Tensor, threshold: ?f32) Tensor { - return applyExpertActivation(x, .relu, threshold, .concatenated); - } - }; - const x: Tensor = .init(.{ .token = 1, .out = 5 }, .f32); - const values = [_]f32{ -3, 0, 1, 2, 4 }; - var input = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&values)); - defer input.deinit(); - for ([_]?f32{ null, 2 }) |threshold| { - var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, threshold }, .{}); - defer exe.deinit(); - try zml.testing.expectEqualShapes(x.shape(), exe.output_shapes[0]); - var output = try exe.eval(allocator, io, .{input}); - defer output.deinit(); - var actual = try output.toSliceAlloc(allocator, io); - defer actual.free(allocator); - const expected = [_]f32{ 0, 0, 1, 4, if (threshold != null) 4 else 16 }; - try std.testing.expectEqualSlices(f32, &expected, actual.constItems(f32)); - } -} +// TODO(Corentin): Add better tests +//test "ReLU squared activation preserves width and applies threshold before squaring" { +// const allocator = std.testing.allocator; +// const io = std.testing.io; +// const platform = zml.testing.env(); +// const Local = struct { +// fn forward(x: Tensor, threshold: ?f32) Tensor { +// return applyExpertActivation(x, .relu, threshold, .concatenated); +// } +// }; +// const x: Tensor = .init(.{ .token = 1, .out = 5 }, .f32); +// const values = [_]f32{ -3, 0, 1, 2, 4 }; +// var input = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&values)); +// defer input.deinit(); +// for ([_]?f32{ null, 2 }) |threshold| { +// var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, threshold }, .{}); +// defer exe.deinit(); +// try zml.testing.expectEqualShapes(x.shape(), exe.output_shapes[0]); +// var output = try exe.eval(allocator, io, .{input}); +// defer output.deinit(); +// var actual = try output.toSliceAlloc(allocator, io); +// defer actual.free(allocator); +// const expected = [_]f32{ 0, 0, 1, 4, if (threshold != null) 4 else 16 }; +// try std.testing.expectEqualSlices(f32, &expected, actual.constItems(f32)); +// } +//} /// Build the inputs tuple for FusedMoe and invoke it via `K.call(...)`. pub const GemmOptions = struct { @@ -500,8 +470,8 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { gate_up.quantization.?.scales = gate_up.quantization.?.scales.bitCast(.f8e8m0); down.quantization.?.scales = down.quantization.?.scales.bitCast(.f8e8m0); } - const route = Tensor.arange(.{ .end = 2 }, .i32).reshape(.{ .b = 1, .s = 1, .top_expert = 2 }) - .broad(Shape.init(.{ .b = 1, .s = x.dim(.s), .top_expert = 2 }, .i32)); + const route = Tensor.arange(.{ .end = 2 }, .i32).reshape(.{ .b = 1, .s = 1, .topk = 2 }) + .broad(Shape.init(.{ .b = 1, .s = x.dim(.s), .topk = 2 }, .i32)); return fusedExpertsImpl(.{ .hidden_states = x, .gate_up = gate_up, @@ -511,7 +481,8 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { .gate_up_layout = layout, .routing_weight_placement = placement, .quantize_input = false, - }, .triton) catch unreachable; + .activation = .{ .swiglu = .{} }, + }, .triton); } }; for ([_]DataType{ .bf16, .u8, .f4e2m1 }) |storage_dtype| { diff --git a/zml/moe/metal.zig b/zml/moe/metal.zig index 2eb21b27..e437b364 100644 --- a/zml/moe/metal.zig +++ b/zml/moe/metal.zig @@ -4,73 +4,33 @@ const stdx = @import("stdx"); const zml = @import("../zml.zig"); const Tensor = zml.Tensor; -pub const ActivationMode = enum { - silu, - relu, - gelu, -}; +const log = std.log.scoped(.@"zml/moe/metal"); pub const Options = struct { - activation: ActivationMode = .silu, + activation: zml.moe.Activation, global_num_experts: i64 = -1, - expert_map: ?Tensor = null, w1_scale: ?Tensor = null, w2_scale: ?Tensor = null, w1_global_scale: ?Tensor = null, w2_global_scale: ?Tensor = null, - w1_bias: ?Tensor = null, - w2_bias: ?Tensor = null, }; -pub const Parameters = struct { - num_experts_per_tok: u32, - activation: ActivationMode, - - pub const InitOptions = struct { - num_experts_per_tok: u32, - activation: ActivationMode = .silu, - }; - - pub fn init(opts: InitOptions) Parameters { - return .{ - .num_experts_per_tok = opts.num_experts_per_tok, - .activation = opts.activation, - }; - } -}; +fn validateOptions(opts: zml.moe.Options) void { + stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); + stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); +} const QuantMode = enum { none, fp8, nvfp4 }; -fn quantMode(gate_up: Tensor, down: Tensor) !QuantMode { - if (gate_up.dtype() != down.dtype()) return error.UnsupportedType; - return switch (gate_up.dtype()) { +fn quantMode(dtype: zml.DataType) QuantMode { + return switch (dtype) { .bf16, .f16, .f32 => .none, .f8e4m3fn => .fp8, .u8, .f4e2m1 => .nvfp4, - else => error.UnsupportedType, + else => stdx.debug.panic("unsupported data type: {}", .{dtype}), }; } -fn requireScalePair(opts: Options, dtype: zml.DataType) !void { - const s1 = opts.w1_scale orelse return error.UnsupportedQuantization; - const s2 = opts.w2_scale orelse return error.UnsupportedQuantization; - if (s1.dtype() != dtype or s2.dtype() != dtype) return error.UnsupportedType; -} - -fn applyActivation(x: Tensor, mode: ActivationMode) Tensor { - const mid = @divFloor(x.dim(.out), 2); - const gate = x.slice(.out, .{ .end = mid }); - const up = x.slice(.out, .{ .start = mid }); - - const act = switch (mode) { - .silu => gate.silu().mul(up), - .gelu => gate.gelu().mul(up), - .relu => gate.relu().mul(up), - }; - - return act.rename(.{ .out = .mid }); -} - fn moeGemm( x_rows: Tensor, w: Tensor, @@ -87,11 +47,10 @@ fn moeGemm( }; } -fn applyGateUpGlobalScale(output: Tensor, global_scale: ?Tensor, expert_ids: Tensor) !Tensor { +fn applyGateUpGlobalScale(output: Tensor, global_scale: ?Tensor, expert_ids: Tensor) Tensor { const scale = global_scale orelse return output; - if (scale.dtype() != .f32 or scale.rank() != 2 or scale.dim(.proj) != 2) { - return error.InvalidShape; - } + stdx.debug.assert(scale.dtype() == .f32, "metal backend expected gate_up scale data type to be f32, got {}", .{scale.dtype()}); + stdx.debug.assert(scale.rank() == 2 and scale.dim(.proj) == 2, "metal backend expected gate_up scale to be a 2D tensor with a .proj dim equal to 2, got {}", .{scale.shape()}); const mid = @divExact(output.dim(.out), 2); const selected = scale.gather(.{ .expert = expert_ids }, .{}); @@ -101,9 +60,10 @@ fn applyGateUpGlobalScale(output: Tensor, global_scale: ?Tensor, expert_ids: Ten return output.convert(.f32).mul(expanded).convert(output.dtype()); } -fn applyDownGlobalScale(output: Tensor, global_scale: ?Tensor, expert_ids: Tensor) !Tensor { +fn applyDownGlobalScale(output: Tensor, global_scale: ?Tensor, expert_ids: Tensor) Tensor { const scale = global_scale orelse return output; - if (scale.dtype() != .f32 or scale.rank() != 1) return error.InvalidShape; + stdx.debug.assert(scale.dtype() == .f32, "metal backend expected down scale data type to be f32, got {}", .{scale.dtype()}); + stdx.debug.assert(scale.rank() == 1, "metal backend expected down scale to be a 1D tensor, got {}", .{scale.shape()}); const selected = scale.gather(.{ .expert = expert_ids }, .{}) .appendAxes(.{.d}) @@ -118,11 +78,10 @@ pub fn fusedExperts( gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: zml.moe.Options, - parameters: Parameters, ) zml.Tensor { - stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); - stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); - stdx.debug.assert(opts.activation_threshold == null, "Activation thresholds require the Triton MoE backend", .{}); + validateOptions(opts); + stdx.debug.assert(gate_up.bias == null, "metal backend doesn't support gate_up bias", .{}); + stdx.debug.assert(down.bias == null, "metal backend doesn't support down bias", .{}); const gate_up_scales: ?zml.Tensor = if (gate_up.quantization) |q| q.scales else null; const gate_up_global_scale: ?zml.Tensor = if (gate_up.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; @@ -140,16 +99,14 @@ pub fn fusedExperts( topk_weights, topk_ids, .{ - .activation = parameters.activation, + .activation = opts.activation, .global_num_experts = gate_up_weight_unpacked.dim(.expert), .w1_scale = gate_up_scales, .w2_scale = down_scales, .w1_global_scale = gate_up_global_scale, .w2_global_scale = down_global_scale, - .w1_bias = gate_up.bias, - .w2_bias = down.bias, }, - ) catch |e| stdx.debug.panic("fusedExpertsImpl failed: {}", .{e}); + ); } pub fn fusedExpertsImpl( @@ -159,43 +116,45 @@ pub fn fusedExpertsImpl( topk_weights: Tensor, topk_ids: Tensor, opts: Options, -) !Tensor { - if (opts.expert_map != null) return error.InvalidShape; - if (opts.w1_bias != null or opts.w2_bias != null) return error.UnsupportedBias; - - const mode = try quantMode(gate_up, down); +) Tensor { + stdx.debug.assert(gate_up.dtype() == down.dtype(), "metal backend expected gate_up and down data type to match, got {} and {}", .{ gate_up.dtype(), down.dtype() }); + const mode = quantMode(gate_up.dtype()); switch (mode) { .none => { - if (opts.w1_scale != null or opts.w2_scale != null) return error.UnsupportedQuantization; - if (opts.w1_global_scale != null or opts.w2_global_scale != null) return error.UnsupportedQuantization; - switch (hidden_states.dtype()) { - .bf16, .f16, .f32 => {}, - else => return error.UnsupportedType, - } + stdx.debug.assert(opts.w1_scale == null and opts.w2_scale == null, "metal backend expected gate_up and down scale to be null when the weights are not quantized", .{}); + stdx.debug.assert(opts.w1_global_scale == null and opts.w2_global_scale == null, "metal backend expected gate_up and down global scale to be null when the weights are not quantized", .{}); + const is_hidden_states_valid_dtype = switch (hidden_states.dtype()) { + .bf16, .f16, .f32 => true, + else => false, + }; + stdx.debug.assert(is_hidden_states_valid_dtype, "metal backend expected hidden_states data type to be bf16, f16 or f32, got {}", .{hidden_states.dtype()}); }, .fp8 => { - try requireScalePair(opts, .bf16); - if (opts.w1_global_scale != null or opts.w2_global_scale != null) return error.UnsupportedQuantization; - if (hidden_states.dtype() != .bf16) return error.UnsupportedType; + stdx.debug.assert(opts.w1_scale != null and opts.w2_scale != null, "metal backend expected gate_up and down scale to be non null", .{}); + stdx.debug.assert(opts.w1_scale.?.dtype() == .bf16 and opts.w2_scale.?.dtype() == .bf16, "metal backend expected gate_up and down scale data type to be bf16, got {} and {}", .{ opts.w1_scale.?.dtype(), opts.w2_scale.?.dtype() }); + stdx.debug.assert(opts.w1_global_scale == null and opts.w2_global_scale == null, "metal backend expected gate_up and down global scale to be null", .{}); + stdx.debug.assert(hidden_states.dtype() == .bf16, "metal backend expected hidden_states data type to be bf16, got {}", .{hidden_states.dtype()}); }, .nvfp4 => { - try requireScalePair(opts, .f8e4m3fn); - if (hidden_states.dtype() != .bf16) return error.UnsupportedType; + stdx.debug.assert(opts.w1_scale != null and opts.w2_scale != null, "metal backend expected gate_up and down scale to be non null", .{}); + stdx.debug.assert(opts.w1_scale.?.dtype() == .f8e4m3fn and opts.w2_scale.?.dtype() == .f8e4m3fn, "metal backend expected gate_up and down scale data type to be f8e4m3fn, got {} and {}", .{ opts.w1_scale.?.dtype(), opts.w2_scale.?.dtype() }); + stdx.debug.assert(hidden_states.dtype() == .bf16, "metal backend expected hidden_states data type to be bf16, got {}", .{hidden_states.dtype()}); }, } - switch (topk_weights.dtype()) { - .bf16, .f16, .f32 => {}, - else => return error.UnsupportedType, - } - if (topk_ids.dtype() != .i32) return error.UnsupportedType; + const is_topk_weights_valid_dtype = switch (topk_weights.dtype()) { + .bf16, .f16, .f32 => true, + else => false, + }; + stdx.debug.assert(is_topk_weights_valid_dtype, "metal backend expected topk_weights data type to be bf16, f16 or f32, got {}", .{topk_weights.dtype()}); + stdx.debug.assert(topk_ids.dtype() == .i32, "metal backend expected topk_ids data type to be i32, got {}", .{topk_ids.dtype()}); const b = hidden_states.dim(.b); const s = hidden_states.dim(.s); const d = hidden_states.dim(.d); const num_tokens = b * s; - const top_expert = topk_ids.dim(.top_expert); - const num_routes = num_tokens * top_expert; + const topk = topk_ids.dim(.topk); + const num_routes = num_tokens * topk; const act_dtype: zml.DataType = switch (mode) { .none => hidden_states.dtype(), @@ -203,13 +162,13 @@ pub fn fusedExpertsImpl( }; const hidden = hidden_states.reshape(.{ .token = num_tokens, .d = d }).withTags(.{ .token, .d }); - const x_rows = hidden.insertAxes(.d, .{.top_expert}) - .broad(zml.Shape.init(.{ .token = num_tokens, .top_expert = top_expert, .d = d }, act_dtype)) - .merge(.{ .r = .{ .token, .top_expert } }); - const expert_ids = topk_ids.reshape(.{ .token = num_tokens, .top_expert = top_expert }).withTags(.{ .token, .top_expert }) - .merge(.{ .r = .{ .token, .top_expert } }).convert(.i32); + const x_rows = hidden.insertAxes(.d, .{.topk}) + .broad(zml.Shape.init(.{ .token = num_tokens, .topk = topk, .d = d }, act_dtype)) + .merge(.{ .r = .{ .token, .topk } }); + const expert_ids = topk_ids.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) + .merge(.{ .r = .{ .token, .topk } }).convert(.i32); - const gate_up_out = try applyGateUpGlobalScale(moeGemm( + const gate_up_out = applyGateUpGlobalScale(moeGemm( x_rows, gate_up, opts.w1_scale, @@ -217,9 +176,9 @@ pub fn fusedExpertsImpl( zml.Shape.init(.{ .r = num_routes, .out = gate_up.dim(.dout) }, act_dtype), mode, ), opts.w1_global_scale, expert_ids); - const activated = applyActivation(gate_up_out, opts.activation); + const activated = zml.moe.applyActivation(gate_up_out, opts.activation, .concatenated).convert(act_dtype); - const down_out = try applyDownGlobalScale(moeGemm( + const down_out = applyDownGlobalScale(moeGemm( activated, down, opts.w2_scale, @@ -228,11 +187,11 @@ pub fn fusedExpertsImpl( mode, ), opts.w2_global_scale, expert_ids); - const weights = topk_weights.reshape(.{ .token = num_tokens, .top_expert = top_expert }).withTags(.{ .token, .top_expert }) - .merge(.{ .r = .{ .token, .top_expert } }); + const weights = topk_weights.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) + .merge(.{ .r = .{ .token, .topk } }); const weighted = down_out.mul(weights.convert(down_out.dtype()).broad(down_out.shape())); - const combined = weighted.splitAxis(.r, .{ .token = num_tokens, .top_expert = top_expert }) - .sum(.top_expert).squeeze(.top_expert); + const combined = weighted.splitAxis(.r, .{ .token = num_tokens, .topk = topk }) + .sum(.topk).squeeze(.topk); return combined.reshape(.{ .b = b, .s = s, .d = down.dim(.d) }); } diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index 82821b8e..975aac94 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -4,6 +4,7 @@ const platforms = @import("platforms"); const zml = @import("../zml.zig"); const stdx = zml.stdx; +pub const stablehlo = @import("stablehlo.zig"); pub const triton_mxfp4 = @import("triton_mxfp4.zig"); pub const cute_mxfp4 = @import("cute_mxfp4.zig"); pub const cutlass_flashinfer = @import("cutlass_flashinfer.zig"); @@ -15,6 +16,8 @@ const fused_experts = @import("fused_experts.zig"); pub const triton_kernels = @import("triton_kernels/triton_kernels.zig"); pub const ProjectionLayout = fused_experts.ProjectionLayout; +const log = std.log.scoped(.@"zml/moe"); + test { std.testing.refAllDecls(@This()); } @@ -37,13 +40,41 @@ pub const ExpertsLayout = struct { }; }; -pub const ActivationMode = enum { - silu, - relu, +pub const ActivationKind = enum { + /// Gelu activation function gelu, + /// ReLU activation function + relu, + /// SiLU activation function + silu, + /// SwiGLU activation function (including clamped/clipped/scaled/biased variants) + swiglu, + /// SwiGLU activation function with step function + swiglu_step, + /// GeGlu activation function + geglu, + /// GeGlu activation function with tanh + geglu_tanh, +}; + +pub const Activation = union(ActivationKind) { + gelu: void, + relu: void, + silu: void, + swiglu: struct { + limit: ?f32 = null, + scale: ?f32 = null, + bias: ?f32 = null, + }, + swiglu_step: struct { + limit: ?f32 = null, + }, + geglu: void, + geglu_tanh: void, }; pub const Backend = enum { + stablehlo, cute_mxfp4, triton_mxfp4, flashinfer_cutlass, @@ -101,6 +132,7 @@ pub const Backend = enum { pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { return switch (backend) { + .stablehlo => true, .triton_mxfp4 => triton_mxfp4.isAvailable(platform), .cute_mxfp4 => cute_mxfp4.isAvailable(platform), .flashinfer_cutlass => cutlass_flashinfer.isAvailable(platform), @@ -120,6 +152,7 @@ pub const Backend = enum { /// The layout contract the expert weights must have for a given backend with scheme pub fn expertsLayout(backend: Backend, scheme: ?zml.Quantization.Scheme) !ExpertsLayout { return switch (backend) { + .stablehlo => .{ .gate_up = .concatenated, .packing = .plain }, .cute_mxfp4 => if (scheme == .mxfp4) .{ .gate_up = .interleaved, .packing = .swizzled_scales } else error.UnsupportedQuantization, .triton_mxfp4 => if (scheme == .mxfp4) .{ .gate_up = .interleaved, .packing = .plain } else error.UnsupportedQuantization, .flashinfer_cutlass => if (scheme == .nvfp4) .{ .gate_up = .concatenated, .packing = .flashinfer_nvfp4 } else error.UnsupportedQuantization, @@ -130,6 +163,7 @@ pub const Backend = enum { pub fn register(backend: Backend, platform: *zml.Platform) !void { return switch (backend) { + .stablehlo => {}, .cute_mxfp4, .triton_mxfp4 => {}, .flashinfer_cutlass => cutlass_flashinfer.register(platform), .triton, .fly => {}, @@ -139,99 +173,32 @@ pub const Backend = enum { } }; -pub const Parameters = union(Backend) { - cute_mxfp4: cute_mxfp4.Parameters, - triton_mxfp4: triton_mxfp4.Parameters, - flashinfer_cutlass: cutlass_flashinfer.Parameters, - triton: triton.Parameters, - fly: fly.Parameters, - mosaic_tpu: mosaic_tpu.Parameters, - metal: metal.Parameters, - - pub const InitOptions = union(Backend) { - cute_mxfp4: cute_mxfp4.Parameters.InitOptions, - triton_mxfp4: triton_mxfp4.Parameters.InitOptions, - flashinfer_cutlass: cutlass_flashinfer.Parameters.InitOptions, - triton: triton.Parameters.InitOptions, - fly: fly.Parameters.InitOptions, - mosaic_tpu: mosaic_tpu.Parameters.InitOptions, - metal: metal.Parameters.InitOptions, - - pub fn fromBackend(backend: Backend, num_experts_per_tok: u32, activation: ActivationMode) InitOptions { - return switch (backend) { - inline .cute_mxfp4, .triton_mxfp4 => |backend_tag| @unionInit(InitOptions, @tagName(backend_tag), .{ .num_experts_per_tok = num_experts_per_tok, .activation = activation }), - .flashinfer_cutlass => .{ .flashinfer_cutlass = .{ - .num_experts_per_tok = num_experts_per_tok, - .activation = switch (activation) { - .silu => .silu, - .relu => .relu, - .gelu => .gelu, - }, - } }, - inline .triton, .fly => |backend_tag| @unionInit(InitOptions, @tagName(backend_tag), .{ - .num_experts_per_tok = num_experts_per_tok, - .activation = switch (activation) { - .silu => .silu, - .relu => .relu, - .gelu => .gelu, - }, - }), - .mosaic_tpu => .{ .mosaic_tpu = .{ - .num_experts_per_tok = num_experts_per_tok, - .activation = switch (activation) { - .silu => .silu, - .relu => .relu, - .gelu => .gelu, - }, - } }, - .metal => .{ .metal = .{ - .num_experts_per_tok = num_experts_per_tok, - .activation = switch (activation) { - .silu => .silu, - .relu => .relu, - .gelu => .gelu, - }, - } }, - }; - } - }; - - pub fn init(opts: InitOptions) Parameters { - return switch (opts) { - .cute_mxfp4 => |v| .{ .cute_mxfp4 = cute_mxfp4.Parameters.init(v) }, - .triton_mxfp4 => |v| .{ .triton_mxfp4 = triton_mxfp4.Parameters.init(v) }, - .flashinfer_cutlass => |v| .{ .flashinfer_cutlass = cutlass_flashinfer.Parameters.init(v) }, - inline .triton, .fly => |v, backend_tag| @unionInit(Parameters, @tagName(backend_tag), fused_experts.Parameters.init(v)), - .mosaic_tpu => |v| .{ .mosaic_tpu = mosaic_tpu.Parameters.init(v) }, - .metal => |v| .{ .metal = metal.Parameters.init(v) }, - }; - } -}; - pub const Options = struct { - activation_threshold: ?f32 = null, + activation: Activation, /// Quantize activations for Triton FP8 GEMMs; false keeps BF16 activations. quantize_input: bool, /// Where routing weights are applied; FlashInfer, Mosaic and Metal require after_down. routing_weight_placement: fused_experts.RoutingWeightPlacement, }; +/// Routing IDs and weights have shape { b, s, topk }; input has shape { b, s, d }. pub fn forwardMoe( input: zml.Tensor, topk_ids: zml.Tensor, topk_weights: zml.Tensor, gate_up: zml.nn.Linear, down: zml.nn.Linear, + backend: Backend, opts: Options, - parameters: Parameters, -) !zml.Tensor { - return switch (parameters) { - .cute_mxfp4 => |p| cute_mxfp4.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts, p), - .triton_mxfp4 => |p| triton_mxfp4.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts, p), - .flashinfer_cutlass => |p| cutlass_flashinfer.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts, p), - inline .triton, .fly => |p, backend| fused_experts.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts, p, backend), - .mosaic_tpu => |p| mosaic_tpu.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts, p), - .metal => |p| metal.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts, p), +) zml.Tensor { + return switch (backend) { + .stablehlo => stablehlo.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + .cute_mxfp4 => cute_mxfp4.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + .triton_mxfp4 => triton_mxfp4.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + .flashinfer_cutlass => cutlass_flashinfer.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + inline .triton, .fly => |b| fused_experts.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts, b), + .mosaic_tpu => mosaic_tpu.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + .metal => metal.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), }; } @@ -242,3 +209,50 @@ pub fn unpackedWeight(linear: zml.nn.Linear) zml.Tensor { else linear.weight; } + +pub fn applyActivation(input: zml.Tensor, activation: zml.moe.Activation, layout: ProjectionLayout) zml.Tensor { + const x = input.convert(.f32); + return switch (activation) { + .gelu => x.gelu(), + .relu => x.relu(), + .silu => x.silu(), + .swiglu, .swiglu_step, .geglu, .geglu_tanh => b: { + const mid = @divFloor(x.dim(.out), 2); + var gate, var up = switch (layout) { + .concatenated => .{ x.slice(.out, .{ .end = mid }), x.slice(.out, .{ .start = mid }) }, + .interleaved => .{ x.slice(.out, .{ .start = 0, .step = 2 }), x.slice(.out, .{ .start = 1, .step = 2 }) }, + }; + + break :b switch (activation) { + .swiglu => |parameters| { + stdx.debug.assert(parameters.scale == null, "triton and fly moe backend don't support swiglu scale", .{}); + + const limit: ?zml.Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; + + // Apply limit on gate and clamp up + gate = if (limit) |l| gate.minimum(l) else gate; + up = if (limit) |l| up.clamp(l.negate(), l) else up; + + // Apply bias + up = if (parameters.bias) |bias| up.addConstant(bias) else up; + + break :b gate.silu().mul(up); + }, + .swiglu_step => |parameters| { + gate = gate.silu(); + + const limit: ?zml.Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; + gate = if (limit) |l| gate.minimum(l) else gate; + up = if (limit) |l| up.clamp(l.negate(), l) else up; + break :b gate.mul(up); + }, + .geglu_tanh => gate.gelu().mul(up), + .geglu => { + log.warn("The geglu activation function was requested but we only support the tanh approximation", .{}); + break :b gate.gelu().mul(up); + }, + else => unreachable, // already treated by the top-level switch + }; + }, + }; +} diff --git a/zml/moe/mosaic_tpu.zig b/zml/moe/mosaic_tpu.zig index 41936baa..6f6a7bc5 100644 --- a/zml/moe/mosaic_tpu.zig +++ b/zml/moe/mosaic_tpu.zig @@ -7,20 +7,13 @@ const zml = @import("../zml.zig"); const Tensor = zml.Tensor; pub const gmm_ep = @import("mosaic_tpu_kernels/gmm_ep.zig"); -const log = std.log.scoped(.moe_mosaic_tpu); +const log = std.log.scoped(.@"zml/moe/mosaic_tpu"); var gate_up_transpose: std.atomic.Value(bool) = .init(false); var down_transpose: std.atomic.Value(bool) = .init(false); -pub const ActivationMode = enum { - silu, - relu, - gelu, - quick_gelu_plus_one, -}; - pub const Options = struct { - activation: ActivationMode = .silu, + activation: zml.moe.Activation, global_num_experts: i64 = -1, expert_map: ?Tensor = null, w1_scale: ?Tensor = null, @@ -29,26 +22,12 @@ pub const Options = struct { w2_bias: ?Tensor = null, }; -pub const Parameters = struct { - num_experts_per_tok: u32, - activation: ActivationMode, - - pub const InitOptions = struct { - num_experts_per_tok: u32, - activation: ActivationMode = .silu, - }; - - pub fn init(opts: InitOptions) Parameters { - return .{ - .num_experts_per_tok = opts.num_experts_per_tok, - .activation = opts.activation, - }; - } -}; - -fn validateOptions(opts: Options) !void { - if (opts.expert_map != null and opts.global_num_experts == -1) return error.InvalidShape; - if (opts.w1_scale != null or opts.w2_scale != null) return error.UnsupportedQuantization; +fn validateOptions(opts: zml.moe.Options) void { + stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); + stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); + // TODO(Corentin): find where to put that + //if (opts.expert_map != null and opts.global_num_experts == -1) return error.InvalidShape; + //if (opts.w1_scale != null or opts.w2_scale != null) return error.UnsupportedQuantization; } fn validateInputs(hidden: Tensor, gate_up: Tensor, down: Tensor, weights: Tensor, ids: Tensor) !void { @@ -244,26 +223,6 @@ pub fn callGmmEp( return out.slice(.out, .{ .end = out_n }); } -fn applyActivation(x: Tensor, mode: ActivationMode) Tensor { - const mid = @divFloor(x.dim(.out), 2); - const gate = x.slice(.out, .{ .end = mid }); - const up = x.slice(.out, .{ .start = mid }); - - return switch (mode) { - .silu => gate.silu().mul(up), - .gelu => gate.gelu().mul(up), - .relu => x.relu().powByConst(2), - .quick_gelu_plus_one => blk: { - const gate_clamped = gate.minimum(Tensor.scalar(7, gate.dtype()).broad(gate.shape())); - const up_clamped = up.clamp( - Tensor.scalar(-7, up.dtype()).broad(up.shape()), - Tensor.scalar(7, up.dtype()).broad(up.shape()), - ); - break :blk gate_clamped.quickGelu().mul(up_clamped.addConstant(1)); - }, - }; -} - pub fn fusedExperts( input: zml.Tensor, topk_ids: zml.Tensor, @@ -271,11 +230,8 @@ pub fn fusedExperts( gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: zml.moe.Options, - parameters: Parameters, ) zml.Tensor { - stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); - stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); - stdx.debug.assert(opts.activation_threshold == null, "Activation thresholds require the Triton MoE backend", .{}); + validateOptions(opts); const gate_up_scales: ?zml.Tensor = if (gate_up.quantization) |q| q.scales else null; const down_scales: ?zml.Tensor = if (down.quantization) |q| q.scales else null; @@ -291,7 +247,7 @@ pub fn fusedExperts( topk_weights: zml.Tensor, weights_gate_up: zml.Tensor, weights_down: zml.Tensor, - activation: ActivationMode, + activation: zml.moe.Activation, global_num_experts: i64, gate_up_scales: ?zml.Tensor, bias_gate_up: ?zml.Tensor, @@ -335,7 +291,7 @@ pub fn fusedExperts( .topk_weights = topk_weights, .weights_gate_up = gate_up.weight, .weights_down = down.weight, - .activation = parameters.activation, + .activation = opts.activation, .global_num_experts = global_num_experts, .gate_up_scales = gate_up_scales, .bias_gate_up = gate_up.bias, @@ -354,7 +310,7 @@ pub fn fusedExperts( topk_weights, topk_ids, .{ - .activation = parameters.activation, + .activation = opts.activation, .global_num_experts = gate_up.weight.dim(.expert), .w1_scale = gate_up_scales, .w2_scale = down_scales, @@ -372,16 +328,14 @@ pub fn fusedExpertsImpl( topk_ids: Tensor, opts: Options, ) Tensor { - validateOptions(opts) catch |e| stdx.debug.panic("Invalid options for fusedExpertsImpl: {}", .{e}); - const b = hidden_states.dim(.b); const s = hidden_states.dim(.s); const hidden = hidden_states.reshape(.{ .token = b * s, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); // TODO(Corentin): Better error message const gate_up = canonicalizeGateUp(w1, hidden.dim(.in)) catch |e| stdx.debug.panic("Invalid gate_up weights for fusedExpertsImpl: {}", .{e}); const down = canonicalizeDown(w2, hidden.dim(.in), gate_up.dim(.out)) catch |e| stdx.debug.panic("Invalid down weights for fusedExpertsImpl: {}", .{e}); - const weights = topk_weights.reshape(.{ .token = b * s, .topk = topk_weights.dim(.top_expert) }).withTags(.{ .token, .topk }); - const ids = topk_ids.reshape(.{ .token = b * s, .topk = topk_ids.dim(.top_expert) }).withTags(.{ .token, .topk }); + const weights = topk_weights.reshape(.{ .token = b * s, .topk = topk_weights.dim(.topk) }).withTags(.{ .token, .topk }); + const ids = topk_ids.reshape(.{ .token = b * s, .topk = topk_ids.dim(.topk) }).withTags(.{ .token, .topk }); const num_experts = if (opts.global_num_experts != -1) opts.global_num_experts else gate_up.dim(.expert); if (opts.expert_map) |expert_map| { @@ -431,7 +385,7 @@ pub fn fusedExpertsImpl( gate_up_out = gate_up_out.add(bias_per_token); } - const activated = applyActivation(gate_up_out, opts.activation); + const activated = zml.moe.applyActivation(gate_up_out, opts.activation, .concatenated); const aligned_activated = alignSortedRowsByGroup(activated, expert_ids_sorted, group_sizes, tile_m); var down_out = callGmmEp(aligned_activated.rows, down, aligned_activated.group_sizes, hidden.dtype(), .none) diff --git a/zml/moe/stablehlo.zig b/zml/moe/stablehlo.zig new file mode 100644 index 00000000..1e05db5e --- /dev/null +++ b/zml/moe/stablehlo.zig @@ -0,0 +1,182 @@ +//! Deliberately simple MoE reference: no routed GEMMs, packing, or custom calls. +//! Weights use [expert, output, input] order with concatenated gate/up columns. +const std = @import("std"); +const stdx = @import("stdx"); + +const zml = @import("../zml.zig"); +const Tensor = zml.Tensor; +const Shape = zml.Shape; + +pub fn activate(x: Tensor, activation: zml.moe.Activation) Tensor { + return switch (activation) { + .silu => x.silu(), + .gelu => x.gelu(), + .relu => x.relu(), + .swiglu, .swiglu_step, .geglu, .geglu_tanh => blk: { + const mid = @divExact(x.dim(.out), 2); + var gate = x.slice(.out, .{ .end = mid }); + var up = x.slice(.out, .{ .start = mid }); + + switch (activation) { + .swiglu_step => { + gate = gate.div(gate.negate().exp().addConstant(1)); + if (activation.swiglu_step.limit) |limit| { + gate = gate.minimum(Tensor.scalar(limit, x.dtype())); + up = up.clamp(Tensor.scalar(-limit, x.dtype()), Tensor.scalar(limit, x.dtype())); + } + break :blk gate.mul(up); + }, + .swiglu => { + if (activation.swiglu.limit) |limit| { + gate = gate.minimum(Tensor.scalar(limit, x.dtype())); + up = up.clamp(Tensor.scalar(-limit, x.dtype()), Tensor.scalar(limit, x.dtype())); + } + if (activation.swiglu.bias) |bias| up = up.addConstant(bias); + break :blk gate.div(gate.negate().exp().addConstant(1)).mul(up); + }, + .geglu => break :blk gelu(gate).mul(up), + .geglu_tanh => break :blk gate.gelu().mul(up), + else => unreachable, // already handled above + } + }, + }; +} + +/// More precise GELU approximation than the default Zig implementation, which is +/// based on tanh. This is the same approximation used in JAX. +pub fn gelu(x: Tensor) Tensor { + const xf = x.convert(.f32); + const cdf = xf.scale(-1.0 / @sqrt(@as(f64, 2))) + .erfc() + .scale(0.5); + return xf.mul(cdf).convert(x.dtype()); +} + +fn expandScales(scales: Tensor, shape: Shape) Tensor { + if (scales.count() == 1) return scales.asScalar().convert(.f32).broad(shape.withDtype(.f32)); + stdx.debug.assert(scales.rank() == shape.rank(), "reference scale rank must match weights", .{}); + var result = scales.convert(.f32); + for (0..shape.rank()) |axis| { + const factor = std.math.divCeil(i64, shape.dim(axis), result.dim(axis)) catch unreachable; + const expanded = result.shape().insert(axis + 1, .{factor}); + result = result.broadcast(expanded, Shape.range(expanded.rank(), .i64).remove(axis + 1).dims()) + .reshape(result.shape().setDim(axis, result.dim(axis) * factor)) + .slice(axis, .{ .end = shape.dim(axis) }); + } + return result.withTags(shape.tags()); +} + +pub fn dequantize(linear: zml.nn.Linear) Tensor { + var weight = linear.weight; + const q = linear.quantization orelse return weight.convert(.f32); + stdx.debug.assert(!q.swizzled_scales, "StableHLO reference requires plain block scales", .{}); + if (zml.nn.isPackedFp4(q.scheme, weight.dtype())) { + // StableHLO bitcast expands the low nibble first along the input axis. + weight = weight.bitCast(.f4e2m1).merge(.{ .unpacked = .{ linear.tag, .bitcast } }).renameTag(.unpacked, linear.tag); + } + const scales = if (q.scheme.isMx() and q.scales.dtype() == .u8) q.scales.bitCast(.f8e8m0) else q.scales; + var result = weight.convert(.f32).mul(expandScales(scales, weight.shape())); + if (q.global_scale) |scale| result = result.mul(scale.asMultiplier().broad(result.shape())); + return result; +} + +// E2M1 round-to-nearest, ties-to-even expressed independently of native +// float-to-FP4 lowering. Even encoding indices win midpoint ties. +fn roundFp4(x: Tensor) Tensor { + const magnitude = x.abs(); + var rounded = Tensor.scalar(6, .f32).broad(x.shape()); + const thresholds = [_]f32{ 5, 3.5, 2.5, 1.75, 1.25, 0.75, 0.25 }; + const values = [_]f32{ 4, 3, 2, 1.5, 1, 0.5, 0 }; + inline for (thresholds, values, 0..) |threshold, value, i| { + rounded = magnitude.cmp(if (i % 2 == 0) .LE else .LT, Tensor.scalar(threshold, .f32)) + .select(Tensor.scalar(value, .f32).broad(x.shape()), rounded); + } + return x.cmp(.LT, Tensor.scalar(0, .f32)).select(rounded.negate(), rounded); +} + +// Quantize and dequantize activation groups using tensor math, independently of +// Triton's quantizer. The return value is the represented value in FP32. +fn roundedInput(x: Tensor, linear: zml.nn.Linear, quantize: bool, dtype: zml.DataType) Tensor { + const q = linear.quantization orelse return x.convert(dtype).convert(.f32); + if (q.scheme == .nvfp4) { + const global = if (q.input_scale) |scale| scale.asMultiplier() else Tensor.scalar(1, .f32); + stdx.debug.assert(global.count() == 1, "reference NVFP4 input scale must be global", .{}); + const grouped = x.convert(.bf16).convert(.f32).div(global.asScalar()) + .splitAxis(.in, .{ .group = @divExact(x.dim(.in), 16), .element = 16 }); + const scale = grouped.abs().max(.element).scale(1.0 / 6.0) + .minimum(Tensor.scalar(448, .f32)) + .convert(.f8e4m3fn).convert(.f32).broad(grouped.shape()); + const divisor = scale.cmp(.NE, Tensor.scalar(0, .f32)).select(scale, Tensor.scalar(1, .f32).broad(scale.shape())); + return roundFp4(grouped.mul(Tensor.scalar(1, .f32).div(divisor))).mul(scale) + .reshape(x.shape().withDtype(.f32)).mul(global.asScalar()); + } + if (!quantize or q.scheme == .mxfp4) return x.convert(dtype).convert(.f32); + stdx.debug.assert(q.scheme != .nvfp4, "reference FP8 input quantization does not accept NVFP4", .{}); + const group: i64 = switch (q.scheme) { + .mxfp8, .fp8_block32 => 32, + .fp8_block128 => 128, + .fp8_per_channel, .fp8_per_tensor => x.dim(.in), + else => unreachable, + }; + const grouped = x.convert(.f32).splitAxis(.in, .{ .group = @divExact(x.dim(.in), group), .element = group }); + const epsilon: f32 = if (q.scheme == .fp8_block32) 1e-4 else 1e-10; + const max_value: f32 = if (linear.weight.dtype() == .f8e4m3fnuz) 224 else 448; + var scale = grouped.abs().max(.element).maximum(Tensor.scalar(epsilon, .f32)).scale(1 / max_value); + if (q.scheme == .mxfp8 or q.scheme == .fp8_block32) scale = scale.log().scale(1 / @log(@as(f32, 2))).ceil().scale(@log(@as(f32, 2))).exp(); + const broad = scale.broad(grouped.shape()); + return grouped.div(broad).clamp(Tensor.scalar(-max_value, .f32), Tensor.scalar(max_value, .f32)) + .convert(linear.weight.dtype()).convert(.f32).mul(broad).reshape(x.shape().withDtype(.f32)); +} + +fn project(x: Tensor, linear: zml.nn.Linear, ids: Tensor, quantize: bool, dtype: zml.DataType) Tensor { + const weight = dequantize(linear).withTags(.{ .expert, .out, .in }); + const selected = weight.gather(.{ .expert = ids }, .{}); + var result = roundedInput(x, linear, quantize, dtype).dot(selected, .in); + if (linear.bias) |bias| { + const selected_bias = if (bias.rank() == 2) + bias.withTags(.{ .expert, .out }).gather(.{ .expert = ids }, .{}) + else + bias.withTags(.{.out}); + result = result.add(selected_bias.convert(.f32).broad(result.shape())); + } + return result; +} + +pub const Options = struct { + activation: zml.moe.Activation, + quantize_input: bool = false, + routing_weight_placement: @import("fused_experts.zig").RoutingWeightPlacement = .after_down, +}; + +pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: Options) Tensor { + stdx.debug.assert(input.rank() == 3 and topk_ids.rank() == 3 and topk_weights.rank() == 3, "reference expects rank-three input and routing tensors", .{}); + stdx.debug.assert(topk_ids.dtype() == .i32, "reference expects i32 expert ids", .{}); + stdx.debug.assert(gate_up.weight.rank() == 3 and down.weight.rank() == 3, "reference expects rank-three expert weights", .{}); + stdx.debug.assert(gate_up.weight.dim(.expert) == down.weight.dim(.expert), "reference expert counts must match", .{}); + const b = input.dim(.b); + const s = input.dim(.s); + const k = topk_ids.dim(.topk); + stdx.debug.assert(k > 0 and k <= gate_up.weight.dim(.expert), "reference top-k must fit the expert count", .{}); + stdx.debug.assert(topk_ids.dim(.b) == b and topk_ids.dim(.s) == s and topk_weights.dim(.b) == b and topk_weights.dim(.s) == s and topk_weights.dim(.topk) == k, "reference routing dimensions must match input and ids", .{}); + const ids = topk_ids.reshape(.{ .route = b * s * k }); + const weights = topk_weights.reshape(.{ .route = b * s * k }).convert(.f32); + const hidden = input.reshape(.{ .token = b * s, .in = input.dim(.d) }) + .insertAxes(.in, .{.topk}).broad(Shape.init(.{ .token = b * s, .topk = k, .in = input.dim(.d) }, input.dtype())) + .merge(.{ .route = .{ .token, .topk } }); + const projected = project(hidden, gate_up, ids, opts.quantize_input, input.dtype()).convert(input.dtype()).convert(.f32); + var activated = activate(projected, opts.activation); + if (opts.routing_weight_placement == .before_down) activated = activated.mul(weights.broad(activated.shape())); + var output = project(activated.rename(.{ .out = .in }), down, ids, opts.quantize_input, input.dtype()); + if (opts.routing_weight_placement == .after_down) output = output.mul(weights.broad(output.shape())); + return output.convert(input.dtype()).convert(.f32) + .reshape(.{ .b = b, .s = s, .topk = k, .d = output.dim(.out) }) + .sum(.topk).squeeze(.topk).convert(input.dtype()); +} + +pub fn fusedExperts(input: Tensor, ids: Tensor, weights: Tensor, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: zml.moe.Options) Tensor { + return reference(input, ids, weights, gate_up, down, .{ + .activation = opts.activation, + .quantize_input = opts.quantize_input, + .routing_weight_placement = opts.routing_weight_placement, + }); +} diff --git a/zml/moe/triton.zig b/zml/moe/triton.zig index 7bb0d2be..2394e1e5 100644 --- a/zml/moe/triton.zig +++ b/zml/moe/triton.zig @@ -14,7 +14,8 @@ pub const RoutingWeightPlacement = shared.RoutingWeightPlacement; pub const FusedExpertsArgs = shared.FusedExpertsArgs; const kernels = @import("triton_kernels/triton_kernels.zig"); -const log = std.log.scoped(.moe); +const log = std.log.scoped(.@"zml/moe/triton"); + pub fn call(opts: shared.GemmOptions) Tensor { // Native FP4 tensors expose logical K; the Triton operand is byte-packed. const weight = if (opts.quant_scheme == .mxfp4 and opts.weight.dtype() == .f4e2m1) diff --git a/zml/moe/triton_kernels/triton_kernels.zig b/zml/moe/triton_kernels/triton_kernels.zig index 2377929b..0561b074 100644 --- a/zml/moe/triton_kernels/triton_kernels.zig +++ b/zml/moe/triton_kernels/triton_kernels.zig @@ -13,7 +13,7 @@ const Builder = tri.Builder; const Value = tri.Value; const DType = tri.DType; -const log = std.log.scoped(.moe_triton); +const log = std.log.scoped(.@"zml/moe/triton_kernels"); /// Floor to a multiple of 16 — matches the Python `(v // 16) * 16` guards /// that keep dynamic strides aligned for tt.load/store. diff --git a/zml/moe/triton_mxfp4.zig b/zml/moe/triton_mxfp4.zig index 7ead08fd..bf6bbdf5 100644 --- a/zml/moe/triton_mxfp4.zig +++ b/zml/moe/triton_mxfp4.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const stdx = @import("stdx"); const zml = @import("../zml.zig"); pub const kernels = @import("triton_kernels/mxfp4.zig"); @@ -9,22 +10,12 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major == 10; } -pub const Parameters = struct { - pub const InitOptions = struct { - num_experts_per_tok: u32, - activation: zml.moe.ActivationMode, - }; - - num_experts_per_tok: u32, - activation: zml.moe.ActivationMode, - - pub fn init(opts: InitOptions) Parameters { - return .{ - .num_experts_per_tok = opts.num_experts_per_tok, - .activation = opts.activation, - }; - } -}; +pub fn validateOptions(opts: zml.moe.Options) void { + stdx.debug.assert(opts.activation == .swiglu, "cute_mxfp4 backend only accepts swiglu activation, got {}", .{opts.activation}); + stdx.debug.assert(opts.activation.swiglu.limit != null, "cute_mxfp4 backend requires swiglu limit to be set", .{}); + stdx.debug.assert(opts.activation.swiglu.bias == null, "cute_mxfp4 backend requires swiglu bias to be null", .{}); + stdx.debug.assert(opts.activation.swiglu.scale == null, "cute_mxfp4 backend requires swiglu scale to be null", .{}); +} /// Row-major MXFP4 E2M1 weights and linear E8M0 block32 scales. /// BF16 activations are quantized on the GPU to FP8 with per-32 E8M0 scales. @@ -35,20 +26,18 @@ pub fn fusedExperts( gate_up: zml.nn.Linear, down: zml.nn.Linear, options: zml.moe.Options, - parameters: Parameters, -) !zml.Tensor { - if (parameters.activation != .silu) return error.UnsupportedActivation; - if (input.dtype() != .bf16) return error.UnsupportedDataType; - if (gate_up.bias != null or down.bias != null) return error.UnsupportedBias; - const gq = gate_up.quantization orelse return error.UnsupportedQuantization; - const dq = down.quantization orelse return error.UnsupportedQuantization; - if (gq.scheme != .mxfp4 or dq.scheme != .mxfp4) return error.UnsupportedQuantization; +) zml.Tensor { + validateOptions(options); + stdx.debug.assert(input.dtype() == .bf16, "triton_mxfp4 backend only supports bf16 inputs, got {}", .{input.dtype()}); + stdx.debug.assert(gate_up.bias == null and down.bias == null, "triton_mxfp4 backend expects gate_up bias and down bias to be null", .{}); + + const gq = gate_up.quantization orelse @panic("triton_mxfp4 backend requires gate_up quantization to be set"); + const dq = down.quantization orelse @panic("triton_mxfp4 backend requires down quantization to be set"); + stdx.debug.assert(gq.scheme == .mxfp4 and dq.scheme == .mxfp4, "triton_mxfp4 expects gate_up and down quantization scheme to be mxfp4, got {} and {}", .{ gq.scheme, dq.scheme }); + // Weight storage for mxfp4 in HF is expressed as u8 or i8 - if ((gate_up.weight.dtype() != .u8 and gate_up.weight.dtype() != .i8) or - (down.weight.dtype() != .u8 and down.weight.dtype() != .i8)) - { - return error.UnsupportedWeightLayout; - } + stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "triton_mxfp4 expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); + stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "triton_mxfp4 expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); const expert_parallelism = gate_up.weight.shape().partition(.expert).eql(.init(.experts)); @@ -61,8 +50,8 @@ pub fn fusedExperts( .w2 = down.weight.bitCast(.u8), .s2 = dq.scales, .global_experts = gate_up.weight.dim(.expert), - .topk = parameters.num_experts_per_tok, - .limit = options.activation_threshold orelse 0, + .topk = ids.dim(.topk), + .limit = options.activation.swiglu.limit.?, .routing_weight_placement = options.routing_weight_placement, .expert_parallel = expert_parallelism, }; diff --git a/zml/tensor.zig b/zml/tensor.zig index 40aa2a80..83d4beba 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1706,6 +1706,105 @@ pub const Tensor = struct { try zml.testing.expectClose(std.testing.io, expectation, res, .{}); } + /// Returns an element-wise approximation of the error function, computed and returned in FP32. + /// Uses the Eigen rational approximation from CHLO's FP32 erf legalization: + /// https://github.com/openxla/stablehlo/blob/95ee4c70146864ed7d0a21dd135ccec37ece1b52/stablehlo/transforms/ChloLegalizeToStablehlo.cpp#L1038 + pub fn erf(input: Tensor) Tensor { + const x = input.convert(.f32).clamp(.scalar(-4, .f32), .scalar(4, .f32)); + const x2 = x.mul(x); + // Coefficients run from the highest degree to the constant term. + const alpha = [_]f32{ + -2.72614225801306e-10, + 2.77068142495902e-08, + -2.10102402082508e-06, + -5.69250639462346e-05, + -7.34990630326855e-04, + -2.95459980854025e-03, + -1.60960333262415e-02, + }; + const beta = [_]f32{ + -1.45660718464996e-05, + -2.13374055278905e-04, + -1.68282697438203e-03, + -7.37332916720468e-03, + -1.42647390514189e-02, + }; + var numerator = x2.scale(alpha[0]).addConstant(alpha[1]); + inline for (alpha[2..]) |coefficient| { + numerator = numerator.mul(x2).addConstant(coefficient); + } + var denominator = x2.scale(beta[0]).addConstant(beta[1]); + inline for (beta[2..]) |coefficient| { + denominator = denominator.mul(x2).addConstant(coefficient); + } + return x.mul(numerator).div(denominator).clamp(.scalar(-1, .f32), .scalar(1, .f32)); + } + + /// Returns an element-wise approximation of the complementary error function in FP32. + /// Uses CHLO's Cephes-based approximation to preserve the small positive tail: + /// https://github.com/openxla/stablehlo/blob/95ee4c70146864ed7d0a21dd135ccec37ece1b52/stablehlo/transforms/ChloLegalizeToStablehlo.cpp#L1018 + pub fn erfc(input: Tensor) Tensor { + const x = input.convert(.f32); + const abs_x = x.abs(); + const x2 = x.mul(x); + const one = Tensor.scalar(1, .f32); + + // For |x| < 1, erf(x) = x T(x^2). Coefficients run from highest degree to constant. + const t = [_]f32{ + 7.853861353153693e-5, + -8.010193625184903e-4, + 5.188327685732524e-3, + -2.685381193529856e-2, + 1.128358514861418e-1, + -3.761262582423300e-1, + 1.128379165726710, + }; + var small_poly = x2.scale(t[0]).addConstant(t[1]); + inline for (t[2..]) |coefficient| { + small_poly = small_poly.mul(x2).addConstant(coefficient); + } + const small = x.mul(small_poly).negate().addConstant(1); + + // For |x| >= 1, evaluate exp(-x^2) / |x| times P(1/x^2) or R(1/x^2). + const p = [_]f32{ + 2.326819970068386e-2, + -1.387039388740657e-1, + 3.687424674597105e-1, + -5.824733027278666e-1, + 6.210004621745983e-1, + -4.944515323274145e-1, + 3.404879937665872e-1, + -2.741127028184656e-1, + 5.638259427386472e-1, + }; + const r = [_]f32{ + -1.047766399936249e1, + 1.297719955372516e1, + -7.495518717768503, + 2.921019019210786, + -1.015265279202700, + 4.218463358204948e-1, + -2.820767439740514e-1, + 5.641895067754075e-1, + }; + const reciprocal_x2 = one.div(x2); + var middle_poly = reciprocal_x2.scale(p[0]).addConstant(p[1]); + inline for (p[2..]) |coefficient| { + middle_poly = middle_poly.mul(reciprocal_x2).addConstant(coefficient); + } + var tail_poly = reciprocal_x2.scale(r[0]).addConstant(r[1]); + inline for (r[2..]) |coefficient| { + tail_poly = tail_poly.mul(reciprocal_x2).addConstant(coefficient); + } + const poly = abs_x.cmp(.LT, .scalar(2, .f32)).select(middle_poly, tail_poly); + const z = x2.negate(); + const tail = z.exp().mul(one.div(abs_x)).mul(poly); + // Match CHLO's underflow cutoff, then use erfc(-x) = 2 - erfc(x). + const positive = z.cmp(.LT, .scalar(-88.72283905206835, .f32)).select(.scalar(0, .f32), tail); + const large = x.cmp(.LT, .scalar(0, .f32)).select(positive.negate().addConstant(2), positive); + return abs_x.cmp(.LT, one).select(small, large); + } + /// Returns a Tensor containing the Gaussian Error Linear Units (GeLU) activation function applied to each element of the input Tensor. /// /// We use an approximation of the erf function using tanh: From 177f81f21b0713acead79d10dd0d331f8eec85d9 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Mon, 5 Oct 2026 16:48:54 +0200 Subject: [PATCH 750/881] stdx: Add cpu.zig with cpu utilities (#902) --- stdx/BUILD.bazel | 9 +- stdx/cpu.zig | 411 +++++++++++++++++++++++++++++++++++++++++++++++ stdx/stdx.zig | 1 + 3 files changed, 420 insertions(+), 1 deletion(-) create mode 100644 stdx/cpu.zig diff --git a/stdx/BUILD.bazel b/stdx/BUILD.bazel index d4217b90..d7c94d82 100644 --- a/stdx/BUILD.bazel +++ b/stdx/BUILD.bazel @@ -2,7 +2,6 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "stdx", - tags = ["manual"], srcs = [ "Io.zig", "bounded_array.zig", @@ -10,6 +9,7 @@ zig_library( "debug.zig", "flags.zig", "fmt.zig", + "cpu.zig", "json.zig", "math.zig", "meta.zig", @@ -20,6 +20,7 @@ zig_library( "unicode.zig", ], main = "stdx.zig", + tags = ["manual"], visibility = ["//visibility:public"], zigopts = ["-lc"], ) @@ -29,3 +30,9 @@ zig_test( testonly = False, deps = [":stdx"], ) + +zig_test( + name = "cpu_test", + main = "cpu.zig", + zigopts = ["-lc"], +) diff --git a/stdx/cpu.zig b/stdx/cpu.zig new file mode 100644 index 00000000..c51b32d4 --- /dev/null +++ b/stdx/cpu.zig @@ -0,0 +1,411 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +/// Restricts the calling thread and existing threads to the CPUs of a NUMA +/// node within the caller's current affinity, preserving each thread's narrower +/// affinity. Threads with no overlap are left alone. Returns the number of +/// threads whose affinity changed. +/// Call during startup, before threads can concurrently create other threads. +/// On failure, previously changed affinities remain in effect. +pub fn pinProcessToNumaNode(io: std.Io, node: u32) !usize { + if (builtin.os.tag != .linux) return error.Unsupported; + + var path_buf: [64]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "/sys/devices/system/node/node{d}/cpulist", .{node}); + + var cpulist_buf: [4096]u8 = undefined; + const cpulist = try std.Io.Dir.cwd().readFile(io, path, &cpulist_buf); + + var set: std.os.linux.cpu_set_t = @splat(0); + try parseCpuList(cpulist, &set); + + return pinProcessToCpuSet(io, set); +} + +/// CPUs the calling thread may run on, ordered for worker pools. +pub const AllowedCores = struct { + /// Allowed CPU ids: one per physical core first, then the remaining SMT + /// siblings, round by round. + ids: []const usize, + /// `ids[0..physical_count]` holds exactly one CPU per physical core. Equals + /// `ids.len` when the topology is unknown. + physical_count: usize, + + /// One CPU per physical core, skipping SMT siblings. + pub fn physical(self: AllowedCores) []const usize { + return self.ids[0..self.physical_count]; + } + + pub fn deinit(self: AllowedCores, allocator: std.mem.Allocator) void { + allocator.free(self.ids); + } +}; + +/// Returns the CPUs the calling thread may run on. Linux uses sysfs topology so +/// that a second logical CPU of a physical core comes only after every allowed +/// physical core got one. +pub fn allowedCores(allocator: std.mem.Allocator, io: std.Io) !AllowedCores { + if (builtin.os.tag != .linux) return naturalOrder(allocator, try std.Thread.getCpuCount()); + + const cpus = linuxAllowedCoreIds(allocator) catch |err| switch (err) { + error.OutOfMemory => return err, + else => return naturalOrder(allocator, try std.Thread.getCpuCount()), + }; + errdefer allocator.free(cpus); + + const physical_count = linuxOrderBySiblings(allocator, io, cpus) catch |err| switch (err) { + error.OutOfMemory => return err, + else => cpus.len, + }; + return .{ .ids = cpus, .physical_count = physical_count }; +} + +fn pinProcessToCpuSet(io: std.Io, node_set: std.os.linux.cpu_set_t) !usize { + const linux = std.os.linux; + // Keep the caller's taskset/numactl choice as the process-wide limit, and + // intersect each thread's own affinity below so private bindings stay narrow. + const set = intersectCpuSets(node_set, try threadAffinity(0)); + if (std.mem.allEqual(usize, &set, 0)) return 0; + + var tasks = try std.Io.Dir.cwd().openDir(io, "/proc/self/task", .{ .iterate = true }); + defer tasks.close(io); + + // Pin the caller first so threads it creates inherit the target mask. + var pinned: usize = @intFromBool(try pinThreadToCpuSet(0, set)); + var it = tasks.iterate(); + while (try it.next(io)) |entry| { + const tid = std.fmt.parseInt(linux.pid_t, entry.name, 10) catch continue; + // Threads may exit while the list is being walked. + const changed = pinThreadToCpuSet(tid, set) catch |err| switch (err) { + error.ThreadGone => continue, + else => return err, + }; + pinned += @intFromBool(changed); + } + + return pinned; +} + +fn intersectCpuSets(a: std.os.linux.cpu_set_t, b: std.os.linux.cpu_set_t) std.os.linux.cpu_set_t { + var set = a; + for (&set, b) |*word, other| word.* &= other; + return set; +} + +fn threadAffinity(tid: std.os.linux.pid_t) error{ ThreadGone, GetAffinityFailed }!std.os.linux.cpu_set_t { + const linux = std.os.linux; + var set: linux.cpu_set_t = @splat(0); + switch (linux.errno(linux.sched_getaffinity(tid, @sizeOf(linux.cpu_set_t), &set))) { + .SUCCESS => return set, + .SRCH => return error.ThreadGone, + else => return error.GetAffinityFailed, + } +} + +fn setThreadAffinity(tid: std.os.linux.pid_t, set: *const std.os.linux.cpu_set_t) error{ ThreadGone, SetAffinityFailed }!void { + const linux = std.os.linux; + const rc = linux.syscall3(.sched_setaffinity, @as(usize, @bitCast(@as(isize, tid))), @sizeOf(linux.cpu_set_t), @intFromPtr(set)); + switch (linux.errno(rc)) { + .SUCCESS => {}, + .SRCH => return error.ThreadGone, + else => return error.SetAffinityFailed, + } +} + +fn pinThreadToCpuSet(tid: std.os.linux.pid_t, target: std.os.linux.cpu_set_t) !bool { + const current = try threadAffinity(tid); + const set = intersectCpuSets(target, current); + if (std.mem.allEqual(usize, &set, 0) or std.mem.eql(usize, &set, ¤t)) return false; + try setThreadAffinity(tid, &set); + return true; +} + +/// Parses Linux CPU lists, including singleton CPUs and inclusive ranges. +fn parseCpuList(cpulist: []const u8, set: *std.os.linux.cpu_set_t) !void { + const trimmed = std.mem.trim(u8, cpulist, " \t\r\n"); + if (trimmed.len == 0) return; + + const bits_per_word = @bitSizeOf(usize); + + var ranges = std.mem.splitScalar(u8, trimmed, ','); + while (ranges.next()) |raw_range| { + const range = std.mem.trim(u8, raw_range, " \t\r\n"); + var ends = std.mem.splitScalar(u8, range, '-'); + const first = std.fmt.parseInt(usize, ends.first(), 10) catch return error.InvalidCpuList; + + const last = if (ends.next()) |end| + std.fmt.parseInt(usize, end, 10) catch return error.InvalidCpuList + else + first; + + if (last < first or ends.next() != null or last >= set.len * bits_per_word) return error.InvalidCpuList; + for (first..last + 1) |cpu| { + set[cpu / bits_per_word] |= @as(usize, 1) << @intCast(cpu % bits_per_word); + } + } +} + +fn linuxAllowedCoreIds(allocator: std.mem.Allocator) ![]usize { + const set = try threadAffinity(0); + + var core_ids: std.ArrayList(usize) = .empty; + errdefer core_ids.deinit(allocator); + + var core_id: usize = 0; + while (core_id < @bitSizeOf(std.os.linux.cpu_set_t)) : (core_id += 1) { + if (cpuSetContains(set, core_id)) { + try core_ids.append(allocator, core_id); + } + } + + if (core_ids.items.len == 0) return error.MissingCpuTopology; + + return core_ids.toOwnedSlice(allocator); +} + +fn cpuSetContains(set: std.os.linux.cpu_set_t, core_id: usize) bool { + const word_bit_count = @bitSizeOf(usize); + const word_index = core_id / word_bit_count; + const bit_index: std.math.Log2Int(usize) = @intCast(core_id % word_bit_count); + return word_index < set.len and (set[word_index] & (@as(usize, 1) << bit_index)) != 0; +} + +fn linuxOrderBySiblings(allocator: std.mem.Allocator, io: std.Io, cpus: []usize) !usize { + const leaders = try allocator.alloc(usize, cpus.len); + defer allocator.free(leaders); + for (cpus, leaders) |cpu, *leader| leader.* = try linuxCoreLeader(io, cpu); + return orderBySiblings(allocator, cpus, leaders); +} + +/// Returns the lowest CPU id sharing a physical core with `cpu`. +fn linuxCoreLeader(io: std.Io, cpu: usize) !usize { + var cpulist_buf: [4096]u8 = undefined; + // `core_cpus_list` replaced `thread_siblings_list` in Linux 5.7. + const cpulist = linuxReadTopology(io, cpu, "core_cpus_list", &cpulist_buf) catch |err| switch (err) { + error.FileNotFound => try linuxReadTopology(io, cpu, "thread_siblings_list", &cpulist_buf), + else => return err, + }; + + var set: std.os.linux.cpu_set_t = @splat(0); + try parseCpuList(cpulist, &set); + for (set, 0..) |word, word_index| { + if (word != 0) return word_index * @bitSizeOf(usize) + @ctz(word); + } + return error.MissingCpuTopology; +} + +fn linuxReadTopology(io: std.Io, cpu: usize, name: []const u8, buf: []u8) ![]const u8 { + var path_buf: [96]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "/sys/devices/system/cpu/cpu{d}/topology/{s}", .{ cpu, name }); + return std.Io.Dir.cwd().readFile(io, path, buf); +} + +/// Reorders `cpus` so that each round holds at most one CPU per physical core, +/// in ascending order, before the next round of SMT siblings. `leaders[i]` +/// identifies the physical core of `cpus[i]`. Returns the number of physical +/// cores, which is also the size of the first round. +fn orderBySiblings(allocator: std.mem.Allocator, cpus: []usize, leaders: []const usize) !usize { + const Entry = struct { + round: usize, + cpu: usize, + + fn lessThan(_: void, lhs: @This(), rhs: @This()) bool { + return if (lhs.round != rhs.round) lhs.round < rhs.round else lhs.cpu < rhs.cpu; + } + }; + + const entries = try allocator.alloc(Entry, cpus.len); + defer allocator.free(entries); + + var physical_count: usize = 0; + for (entries, cpus, leaders, 0..) |*entry, cpu, leader, index| { + var round: usize = 0; + for (leaders[0..index]) |previous| round += @intFromBool(previous == leader); + entry.* = .{ .round = round, .cpu = cpu }; + physical_count += @intFromBool(round == 0); + } + + std.mem.sort(Entry, entries, {}, Entry.lessThan); + for (cpus, entries) |*cpu, entry| cpu.* = entry.cpu; + return physical_count; +} + +fn naturalOrder(allocator: std.mem.Allocator, cpu_count: usize) !AllowedCores { + const ids = try allocator.alloc(usize, cpu_count); + for (ids, 0..) |*id, index| id.* = index; + return .{ .ids = ids, .physical_count = ids.len }; +} + +test "CPU lists respect mask boundaries" { + var set: std.os.linux.cpu_set_t = @splat(0); + try parseCpuList("0-2,64,66-67\n", &set); + try std.testing.expectEqual(@as(usize, 0b111), set[0]); + try std.testing.expectEqual(@as(usize, 0b1101), set[1]); + const capacity = @bitSizeOf(std.os.linux.cpu_set_t); + var text: [32]u8 = undefined; + try parseCpuList(try std.fmt.bufPrint(&text, "{d}", .{capacity - 1}), &set); + try std.testing.expect((set[set.len - 1] & (@as(usize, 1) << (@bitSizeOf(usize) - 1))) != 0); + try std.testing.expectError(error.InvalidCpuList, parseCpuList(try std.fmt.bufPrint(&text, "{d}", .{capacity}), &set)); + try std.testing.expectError(error.InvalidCpuList, parseCpuList("3-1", &set)); + try std.testing.expectError(error.InvalidCpuList, parseCpuList("1-2-3", &set)); +} + +const AffinityTest = struct { + const CpuSet = std.os.linux.cpu_set_t; + + fn available(count: usize) !CpuSet { + if (builtin.os.tag != .linux) return error.SkipZigTest; + const current = try threadAffinity(0); + var selected: CpuSet = @splat(0); + var remaining = count; + for (0..@bitSizeOf(CpuSet)) |core_id| { + if (!cpuSetContains(current, core_id)) continue; + selected[core_id / @bitSizeOf(usize)] |= @as(usize, 1) << @intCast(core_id % @bitSizeOf(usize)); + remaining -= 1; + if (remaining == 0) break; + } + if (remaining != 0) return error.SkipZigTest; + return selected; + } + + fn check(caller: CpuSet, worker: CpuSet, node: CpuSet, expected: CpuSet) !void { + const io = std.testing.io; + const Worker = struct { + io: std.Io, + set: CpuSet, + tid: std.os.linux.pid_t = undefined, + ready: std.Io.Event = .unset, + stop: std.Io.Event = .unset, + failure: ?anyerror = null, + + fn run(self: *@This()) void { + self.tid = @intCast(std.Thread.getCurrentId()); + setThreadAffinity(0, &self.set) catch |err| { + self.failure = err; + }; + self.ready.set(self.io); + self.stop.waitUncancelable(self.io); + } + }; + var state: Worker = .{ .io = io, .set = worker }; + const thread = try std.Thread.spawn(.{}, Worker.run, .{&state}); + defer { + state.stop.set(io); + thread.join(); + } + try state.ready.wait(io); + if (state.failure) |err| return err; + + // Restore every thread touched by the process-wide helper, including + // any test runner IO workers, before the next test executes. + var saved: std.ArrayList(struct { tid: std.os.linux.pid_t, set: CpuSet }) = .empty; + defer { + for (saved.items) |entry| setThreadAffinity(entry.tid, &entry.set) catch |err| switch (err) { + error.ThreadGone => {}, + error.SetAffinityFailed => @panic("failed to restore test thread affinity"), + }; + saved.deinit(std.testing.allocator); + } + var tasks = try std.Io.Dir.cwd().openDir(io, "/proc/self/task", .{ .iterate = true }); + defer tasks.close(io); + var it = tasks.iterate(); + while (try it.next(io)) |entry| { + const tid = std.fmt.parseInt(std.os.linux.pid_t, entry.name, 10) catch continue; + const set = threadAffinity(tid) catch |err| switch (err) { + error.ThreadGone => continue, + else => return err, + }; + try saved.append(std.testing.allocator, .{ .tid = tid, .set = set }); + } + + try setThreadAffinity(0, &caller); + _ = try pinProcessToCpuSet(io, node); + try std.testing.expectEqualSlices(usize, &expected, &(try threadAffinity(state.tid))); + const expected_caller = intersectCpuSets(caller, node); + try std.testing.expectEqualSlices(usize, &expected_caller, &(try threadAffinity(0))); + } +}; + +test "process NUMA affinity preserves a narrower worker binding" { + const caller = try AffinityTest.available(3); + const node = try AffinityTest.available(2); + const worker = try AffinityTest.available(1); + try AffinityTest.check(caller, worker, node, worker); +} + +test "process NUMA affinity visits workers when the caller is already pinned" { + const caller = try AffinityTest.available(1); + const worker = try AffinityTest.available(2); + try AffinityTest.check(caller, worker, worker, caller); +} + +test "process NUMA affinity leaves a worker with no overlapping CPUs alone" { + const caller = try AffinityTest.available(2); + const node = try AffinityTest.available(1); + var worker = caller; + for (&worker, node) |*word, excluded| word.* &= ~excluded; + try AffinityTest.check(caller, worker, node, worker); +} + +test "CPU lists accept whitespace and empty node CPU sets" { + var set: std.os.linux.cpu_set_t = @splat(0); + try parseCpuList(" \t\r\n", &set); + try std.testing.expect(std.mem.allEqual(usize, &set, 0)); + try parseCpuList(" \t0-2, 64,66-67\r\n", &set); + try std.testing.expectEqual(@as(usize, 0b111), set[0]); + try std.testing.expectEqual(@as(usize, 0b1101), set[1]); +} + +test "CPU lists reject malformed ranges" { + for ([_][]const u8{ "3-1", "x", "-1", "1-", "1-2-3", "1,,2", "1,", ",1", "999999999999999999999999999999" }) |text| { + var set: std.os.linux.cpu_set_t = @splat(0); + try std.testing.expectError(error.InvalidCpuList, parseCpuList(text, &set)); + } +} + +test "natural order treats every CPU as physical" { + const cores = try naturalOrder(std.testing.allocator, 4); + defer cores.deinit(std.testing.allocator); + + try std.testing.expectEqualSlices(usize, &.{ 0, 1, 2, 3 }, cores.ids); + try std.testing.expectEqualSlices(usize, &.{ 0, 1, 2, 3 }, cores.physical()); +} + +test "sibling order puts one CPU per physical core first" { + const Case = struct { cpus: []const usize, leaders: []const usize, expected: []const usize, physical_count: usize }; + const cases = [_]Case{ + // Siblings numbered after all cores (typical Intel). + .{ .cpus = &.{ 0, 1, 2, 3 }, .leaders = &.{ 0, 1, 0, 1 }, .expected = &.{ 0, 1, 2, 3 }, .physical_count = 2 }, + // Adjacent siblings (typical AMD, some ARM). + .{ .cpus = &.{ 0, 1, 2, 3 }, .leaders = &.{ 0, 0, 2, 2 }, .expected = &.{ 0, 2, 1, 3 }, .physical_count = 2 }, + // Affinity allows only some siblings of cores {0,4}, {1,5}, {2,6}. + .{ .cpus = &.{ 0, 1, 2, 5 }, .leaders = &.{ 0, 1, 2, 1 }, .expected = &.{ 0, 1, 2, 5 }, .physical_count = 3 }, + // Affinity allows only the second sibling of some cores. + .{ .cpus = &.{ 1, 4, 5 }, .leaders = &.{ 1, 0, 1 }, .expected = &.{ 1, 4, 5 }, .physical_count = 2 }, + // No SMT. + .{ .cpus = &.{ 0, 1, 2 }, .leaders = &.{ 0, 1, 2 }, .expected = &.{ 0, 1, 2 }, .physical_count = 3 }, + }; + + for (cases) |case| { + const cpus = try std.testing.allocator.dupe(usize, case.cpus); + defer std.testing.allocator.free(cpus); + + const physical_count = try orderBySiblings(std.testing.allocator, cpus, case.leaders); + try std.testing.expectEqualSlices(usize, case.expected, cpus); + try std.testing.expectEqual(case.physical_count, physical_count); + } +} + +test "allowed cores cover the thread affinity" { + const cores = try allowedCores(std.testing.allocator, std.testing.io); + defer cores.deinit(std.testing.allocator); + + try std.testing.expect(cores.physical_count > 0 and cores.physical_count <= cores.ids.len); + if (builtin.os.tag != .linux) return; + + const affinity = try threadAffinity(0); + var expected_count: usize = 0; + for (affinity) |word| expected_count += @popCount(word); + try std.testing.expectEqual(expected_count, cores.ids.len); + for (cores.ids) |cpu| try std.testing.expect(cpuSetContains(affinity, cpu)); +} diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 40419397..8d1fb590 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -3,6 +3,7 @@ const builtin = @import("builtin"); pub const BoundedArray = @import("bounded_array.zig").BoundedArray; pub const BoundedArrayAligned = @import("bounded_array.zig").BoundedArrayAligned; +pub const cpu = @import("cpu.zig"); pub const crypto = @import("crypto.zig"); pub const debug = @import("debug.zig"); pub const flags = @import("flags.zig"); From 3b8b05b9d9b4b09ad3d951210f346ea58d471d53 Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Mon, 5 Oct 2026 17:23:16 +0200 Subject: [PATCH 751/881] zml/attention: unify masking options under AttentionMask (#888) Replace `is_causal` and `sliding_window` fields in `AttentionOptions` with a tagged union. This prepares the API for a subsequent commit that will introduce sparse attention mask. A tagged union makes these mutually exclusive behaviour explicit. --- zml/attention/flashattn.zig | 36 +++++++++++++++++------------- zml/attention/metal_attention.zig | 9 ++++++-- zml/attention/paged_attention.zig | 25 ++++++++++++++++----- zml/attention/tpu_attention.zig | 16 ++++++++++--- zml/attention/triton_attention.zig | 11 ++++++--- 5 files changed, 69 insertions(+), 28 deletions(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index 142bdbb3..f533b008 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -7,6 +7,7 @@ const stdx = @import("stdx"); const zml = @import("../zml.zig"); const ffi = zml.pjrt.ffi; const AttentionOptions = @import("paged_attention.zig").AttentionOptions; +const AttentionMask = @import("paged_attention.zig").AttentionMask; const log = std.log.scoped(.@"zml/attention/flashattn"); @@ -900,7 +901,8 @@ pub const paged_fa2 = struct { stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); const ctx = zml.Compiler.current(); - const window_size_left = windowSizeLeft(opts.sliding_window); + + const window_size_left = windowSizeLeft(opts.mask); const num_head_groups = q.dim(.hg); const num_kv_heads = q.dim(.hkv); @@ -911,7 +913,7 @@ pub const paged_fa2 = struct { const o = switch (parameters) { .decode => |decode_parameters| b: { - const seqlenq_ngroups_swapped = num_heads > num_kv_heads and @mod(head_dim, 8) == 0 and opts.sliding_window < 0; + const seqlenq_ngroups_swapped = num_heads > num_kv_heads and @mod(head_dim, 8) == 0 and opts.mask != .sliding_window; const block_table = decode_parameters.block_table.withPartitioning(.{ .b = .replicated }); const cu_seqlens_q = decode_parameters.cu_seqlens_q.withPartitioning(.{ .b = .replicated }); @@ -977,7 +979,7 @@ pub const paged_fa2 = struct { out_accum, }, .metadata = .{ - .is_causal = opts.is_causal, + .is_causal = opts.mask.isCausal(), .max_seqlen_k = decode_parameters.options.max_seqlen_k, .num_heads = num_heads_per_shard, .window_size_left = window_size_left, @@ -999,7 +1001,7 @@ pub const paged_fa2 = struct { break :b o; }, .mixed => |mixed_parameters| b: { - const seqlenq_ngroups_swapped = num_heads > num_kv_heads and @mod(head_dim, 8) == 0 and opts.sliding_window < 0; + const seqlenq_ngroups_swapped = num_heads > num_kv_heads and @mod(head_dim, 8) == 0 and opts.mask != .sliding_window; const block_table_prefill = mixed_parameters.block_table_prefill.withPartitioning(.{ .b = .replicated }); const cu_seqlens_q_prefill = mixed_parameters.cu_seqlens_q_prefill.withPartitioning(.{ .b = .replicated }); @@ -1063,7 +1065,7 @@ pub const paged_fa2 = struct { out_accum_prefill, }, .metadata = .{ - .is_causal = opts.is_causal, + .is_causal = opts.mask.isCausal(), .max_seqlen_k = mixed_parameters.options.max_seqlen_k, .max_seqlen_q = mixed_parameters.options.max_seqlen_q, .num_heads = num_heads_per_shard, @@ -1138,7 +1140,7 @@ pub const paged_fa2 = struct { out_accum_decode, }, .metadata = .{ - .is_causal = opts.is_causal, + .is_causal = opts.mask.isCausal(), .max_seqlen_k = mixed_parameters.options.max_seqlen_k, .num_heads = num_heads_per_shard, .window_size_left = window_size_left, @@ -1552,7 +1554,7 @@ pub const paged_fa3 = struct { stdx.debug.assert(q.shape().hasTags(.{ .b, .hg, .hkv, .hd }), "Expected q to have tags .b, .h, .hd", .{}); stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); - const window_size_left = windowSizeLeft(opts.sliding_window); + const window_size_left = windowSizeLeft(opts.mask); const num_head_groups = q.dim(.hg); const num_kv_heads = q.dim(.hkv); @@ -1616,7 +1618,7 @@ pub const paged_fa3 = struct { scheduler_metadata, }, .metadata = .{ - .is_causal = opts.is_causal, + .is_causal = opts.mask.isCausal(), .max_seqlen_k = decode_parameters.options.max_seqlen_k, .window_size_left = window_size_left, }, @@ -1693,7 +1695,7 @@ pub const paged_fa3 = struct { scheduler_metadata_prefill, }, .metadata = .{ - .is_causal = opts.is_causal, + .is_causal = opts.mask.isCausal(), .max_seqlen_k = mixed_parameters.options.max_seqlen_k, .max_seqlen_q = mixed_parameters.options.max_seqlen_q, .window_size_left = window_size_left, @@ -1758,7 +1760,7 @@ pub const paged_fa3 = struct { scheduler_metadata_decode, }, .metadata = .{ - .is_causal = opts.is_causal, + .is_causal = opts.mask.isCausal(), .max_seqlen_k = mixed_parameters.options.max_seqlen_k, .window_size_left = window_size_left, }, @@ -1777,12 +1779,16 @@ pub const paged_fa3 = struct { } }; -fn windowSizeLeft(sliding_window: i32) i32 { - return if (sliding_window > 0) sliding_window - 1 else sliding_window; +fn windowSizeLeft(mask: AttentionMask) i32 { + return switch (mask) { + .sliding_window => |w| @as(i32, @intCast(w)) - 1, + else => -1, + }; } test "FlashAttention sliding window uses an inclusive offset" { - try std.testing.expectEqual(@as(i32, 2047), windowSizeLeft(2048)); - try std.testing.expectEqual(@as(i32, 0), windowSizeLeft(1)); - try std.testing.expectEqual(@as(i32, -1), windowSizeLeft(-1)); + try std.testing.expectEqual(@as(i32, 2047), windowSizeLeft(.{ .sliding_window = 2048 })); + try std.testing.expectEqual(@as(i32, 0), windowSizeLeft(.{ .sliding_window = 1 })); + try std.testing.expectEqual(@as(i32, -1), windowSizeLeft(.causal)); + try std.testing.expectEqual(@as(i32, -1), windowSizeLeft(.none)); } diff --git a/zml/attention/metal_attention.zig b/zml/attention/metal_attention.zig index 667d8130..f5a3defa 100644 --- a/zml/attention/metal_attention.zig +++ b/zml/attention/metal_attention.zig @@ -92,13 +92,18 @@ pub const paged = struct { const kc = k_cache.transpose(.{ .page, .k_chunk, .hkv, .hd }); const vc = v_cache.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const sliding_window: i32 = switch (opts.mask) { + .sliding_window => |w| @intCast(w), + else => -1, + }; + const out = zml.ops.customCall( "zml$paged_attn", .{ qh, kc, vc, parameters.block_table, parameters.seq_lens, parameters.query_start_len }, qh.shape(), .{ - .is_causal = opts.is_causal, - .sliding_window = opts.sliding_window, + .is_causal = opts.mask.isCausal(), + .sliding_window = sliding_window, .scale = opts.scale, }, .{ .has_side_effect = false }, diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 26bf76ef..803bc70e 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -200,9 +200,24 @@ pub const Parameters = union(Backend) { } }; +pub const AttentionMask = union(enum) { + /// Attend to every valid key in the sequence. + none, + /// Attend to keys at or before the query position. + causal, + /// Causal window size, including the current token. + sliding_window: u32, + + pub fn isCausal(mask: AttentionMask) bool { + return switch (mask) { + .causal, .sliding_window => true, + .none => false, + }; + } +}; + pub const AttentionOptions = struct { - is_causal: bool = true, - sliding_window: i32 = -1, + mask: AttentionMask = .causal, scale: ?f32 = null, sink: ?zml.Tensor = null, }; @@ -458,17 +473,17 @@ test pagedAttention { }{ .{ .name = "unbounded", - .attention_options = .{ .is_causal = true }, + .attention_options = .{ .mask = .causal }, .backends = all_backends, }, .{ .name = "non_causal", - .attention_options = .{ .is_causal = false }, + .attention_options = .{ .mask = .none }, .backends = &option_sensitive_backends, }, .{ .name = "sliding_window", - .attention_options = .{ .is_causal = true, .sliding_window = page_size }, + .attention_options = .{ .mask = .{ .sliding_window = page_size } }, .backends = &option_sensitive_backends, }, }; diff --git a/zml/attention/tpu_attention.zig b/zml/attention/tpu_attention.zig index 537bb1b1..e5adb91c 100644 --- a/zml/attention/tpu_attention.zig +++ b/zml/attention/tpu_attention.zig @@ -200,6 +200,11 @@ pub const mosaic_tpu = struct { const num_queries_per_block = @max(@as(i64, 1), @min(q_token_count, max_num_seqs)); const logical_head_dim: f64 = @floatFromInt(parameters.opts.head_dim); + const sliding_window: ?i64 = switch (opts.mask) { + .sliding_window => |w| @intCast(w), + else => null, + }; + return .{ .num_q_tokens = q_token_count, .num_q_heads = num_q_heads, @@ -214,8 +219,8 @@ pub const mosaic_tpu = struct { .q_dtype = cfgDtype(q_shape.dtype()), .kv_dtype = cfgDtype(kv_pages_shape.dtype()), .sm_scale = opts.scale orelse @floatCast(1.0 / @sqrt(logical_head_dim)), - .is_causal = opts.is_causal, - .sliding_window = if (opts.sliding_window < 0) null else @intCast(opts.sliding_window), + .is_causal = opts.mask.isCausal(), + .sliding_window = sliding_window, }; } @@ -312,6 +317,11 @@ test "mosaic_tpu cfg uses kernel head dim and logical scale" { 0.000001, ); - const non_causal_cfg = mosaic_tpu.buildCfg(prepared_inputs, parameters, .{ .is_causal = false }); + const non_causal_cfg = mosaic_tpu.buildCfg(prepared_inputs, parameters, .{ .mask = .none }); try std.testing.expect(!non_causal_cfg.is_causal); + try std.testing.expectEqual(@as(?i64, null), non_causal_cfg.sliding_window); + + const window_cfg = mosaic_tpu.buildCfg(prepared_inputs, parameters, .{ .mask = .{ .sliding_window = 16 } }); + try std.testing.expect(window_cfg.is_causal); + try std.testing.expectEqual(@as(?i64, 16), window_cfg.sliding_window); } diff --git a/zml/attention/triton_attention.zig b/zml/attention/triton_attention.zig index 00876276..c1a35587 100644 --- a/zml/attention/triton_attention.zig +++ b/zml/attention/triton_attention.zig @@ -288,6 +288,11 @@ pub const paged = struct { const target_num_prgms: usize = cu_count * 4; const num_2d_prgms: usize = total_q_blocks * num_kv_heads; + const sliding_window: usize = switch (self.opts.mask) { + .sliding_window => |w| @intCast(w), + else => 0, + }; + const paged_attention_opts: PagedAttentionOptions = .{ .cu_count = getCuCount(), .all_decode = !self.options.is_prefill, @@ -299,7 +304,7 @@ pub const paged = struct { .block_size = @intCast(self.k_cache.dim(.k_chunk)), .num_blocks = @intCast(self.k_cache.dim(.page)), .max_num_block_per_seq = @intCast(parameters_.block_table.dim(.p)), - .sliding_window = if (self.opts.sliding_window < 0) 0 else @intCast(self.opts.sliding_window), + .sliding_window = sliding_window, .block_m = block_m, .block_q = block_q, .total_q_blocks = total_q_blocks, @@ -362,7 +367,7 @@ pub const paged = struct { .block_m = @intCast(config.block_m), .use_fp8 = false, .all_decode = paged_attention_opts.all_decode, - .is_causal = opts.is_causal, + .is_causal = opts.mask.isCausal(), }; log.debug("pagedAttention2d config: {any}", .{kernel_config}); @@ -442,7 +447,7 @@ pub const paged = struct { .block_m = @intCast(config.attention.block_m), .num_segments_per_seq = @intCast(config.attention.num_segments_per_seq), .all_decode = paged_attention_opts.all_decode, - .is_causal = opts.is_causal, + .is_causal = opts.mask.isCausal(), }; log.debug("pagedAttention3d attention config: {any}", .{attn_kernel_config}); From 108c2d429b66a759cae376ee9921cd079ed959ef Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Mon, 5 Oct 2026 17:23:18 +0200 Subject: [PATCH 752/881] zml/attention: support sparse attention (#892) --- zml/attention/metal_attention.zig | 2 + zml/attention/paged_attention.zig | 9 +- zml/attention/tpu_attention.zig | 2 + zml/attention/triton_attention.zig | 46 ++++++- .../triton_kernels/unified_attention.zig | 114 ++++++++++++++---- 5 files changed, 142 insertions(+), 31 deletions(-) diff --git a/zml/attention/metal_attention.zig b/zml/attention/metal_attention.zig index f5a3defa..11942afe 100644 --- a/zml/attention/metal_attention.zig +++ b/zml/attention/metal_attention.zig @@ -86,6 +86,8 @@ pub const paged = struct { }; pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { + if (opts.mask == .indices) @panic("Sparse Attention is not implemented on Metal"); + const num_kv_heads = q.dim(.hkv); const qh = q.merge(.{ .h = .{ .hkv, .hg } }).transpose(.{ .b, .h, .hd }); diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 803bc70e..97960240 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -207,11 +207,13 @@ pub const AttentionMask = union(enum) { causal, /// Causal window size, including the current token. sliding_window: u32, + /// Attend to keys at specific positions. + indices: zml.Tensor, pub fn isCausal(mask: AttentionMask) bool { return switch (mask) { .causal, .sliding_window => true, - .none => false, + .none, .indices => false, }; } }; @@ -601,6 +603,11 @@ fn stablehlo_pagedAttention( kv_cache: KvCache, opts: AttentionOptions, ) zml.Tensor { + switch (opts.mask) { + .sliding_window, .indices => @panic("Sliding window or sparse attention mask are not supported on StableHLO"), + else => {}, + } + const page_size = kv_cache.split.k.dim(.k_chunk); const final_state = zml.ops.@"while"( diff --git a/zml/attention/tpu_attention.zig b/zml/attention/tpu_attention.zig index e5adb91c..d1806730 100644 --- a/zml/attention/tpu_attention.zig +++ b/zml/attention/tpu_attention.zig @@ -225,6 +225,8 @@ pub const mosaic_tpu = struct { } pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, kv_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { + if (opts.mask == .indices) @panic("Sparse Attention is not supported on TPU"); + const prepared = prepareInputs(parameters, q, kv_cache); const q_out = zml.ops.manualComputation( diff --git a/zml/attention/triton_attention.zig b/zml/attention/triton_attention.zig index c1a35587..3887bc20 100644 --- a/zml/attention/triton_attention.zig +++ b/zml/attention/triton_attention.zig @@ -62,7 +62,26 @@ fn isCudaComputeCapability(expected: zml.platform.cuda.ComputeCapability) bool { /// Triton codegen bug is not diagnosed; this only keeps us out of it. const max_query_tile_elements: usize = 16 * 1024; +fn select2dSparseConfig(options: paged.PagedAttentionOptions) Config2D { + const tile_size: usize = switch (options.indices_count) { + 0...16 => 16, + 17...32 => 32, + else => if (options.head_dim > 128) 32 else 64, + }; + + return .{ + .block_m = @max(16, std.math.ceilPowerOfTwoAssert(usize, options.numQueriesPerKv())), + .block_q = 1, + .tile_size = tile_size, + .num_warps = 4, + .num_stages = 1, + .total_q_blocks = options.num_tokens + options.batch_size, + }; +} + fn select2dConfig(options: paged.PagedAttentionOptions) Config2D { + if (options.indices_count > 0) return select2dSparseConfig(options); + const max_num_stages_2d: usize = if (options.head_dim <= 128) 4 else 2; // Until we test on other platforms, gate the fix to GB300 @@ -129,7 +148,7 @@ fn select3dConfig(options: paged.PagedAttentionOptions) Config3D { var reduce_num_warps: usize = 2; // Intel decode needs more warps to spread the work and avoid register spill. const attn_warps: usize = if (options.all_decode and isOneapiTarget()) 8 else 2; - const tile_size = options.block_size; + const tile_size = if (options.indices_count > 0) 32 else options.block_size; //const MAX_SEGMENTS: usize = @min(128, std.math.divCeil(usize, max_seqlen_k, tile_size)); var num_segments = std.math.divCeil(usize, options.target_num_prgms, options.num_2d_prgms) catch unreachable; @@ -241,6 +260,8 @@ pub const paged = struct { num_2d_prgms: usize, max_seqlen_q: usize, scale: ?f32, + /// Number of indices for sparse attention + indices_count: usize, pub fn numQueriesPerKv(self: PagedAttentionOptions) usize { return self.num_heads / self.num_kv_heads; @@ -278,10 +299,12 @@ pub const paged = struct { // Intel decode: pack exactly one GQA group per tile (block_q == 1) so the // single decode query token doesn't carry masked-out fp32 acc lanes. // oneAPI decode keeps one GQA group per tile, padded to a power of two so tt.make_range emits legal Triton IR. - const block_m: usize = if (!self.options.is_prefill and isOneapiTarget()) + const block_m: usize = if (self.opts.mask == .indices) + @max(16, std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv)) + else if (!self.options.is_prefill and isOneapiTarget()) std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv) else if (num_queries_per_kv <= 16) 16 else std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv); - const block_q: usize = block_m / num_queries_per_kv; + const block_q: usize = if (self.opts.mask == .indices) 1 else block_m / num_queries_per_kv; const num_tokens: usize = @intCast(self.q.dim(.b)); const num_seqs: usize = @intCast(parameters_.block_table.dim(.b)); const total_q_blocks: usize = num_tokens / block_q + num_seqs; @@ -312,6 +335,10 @@ pub const paged = struct { .num_2d_prgms = num_2d_prgms, .max_seqlen_q = self.options.max_seqlen_q, .scale = self.opts.scale, + .indices_count = switch (self.opts.mask) { + .indices => |indices| @intCast(indices.dim(.topk)), + else => 0, + }, }; const use_2d_kernel = use2dKernel( @@ -321,7 +348,7 @@ pub const paged = struct { ); const output = if (use_2d_kernel) pagedAttention2d(parameters_, self.q, self.k_cache, self.v_cache, self.opts, paged_attention_opts) - else if (isOneapiTarget()) + else if (isOneapiTarget() and self.opts.mask != .indices) pagedAttention3dOneapi(parameters_, self.q, self.k_cache, self.v_cache, self.opts, paged_attention_opts) else pagedAttention3d(parameters_, self.q, self.k_cache, self.v_cache, self.opts, paged_attention_opts); @@ -368,6 +395,7 @@ pub const paged = struct { .use_fp8 = false, .all_decode = paged_attention_opts.all_decode, .is_causal = opts.mask.isCausal(), + .indices_count = @intCast(paged_attention_opts.indices_count), }; log.debug("pagedAttention2d config: {any}", .{kernel_config}); @@ -387,6 +415,10 @@ pub const paged = struct { const output = kernels.KernelUnifiedAttention2dPtr.Kernel.call( .{ .query_ptr = q, + .indices_ptr = switch (opts.mask) { + .indices => |indices| indices.transpose(.{ .b, .topk }), + else => dummy, + }, .key_cache_ptr = k_cache, .value_cache_ptr = v_cache, .sink_ptr = sink, @@ -448,6 +480,7 @@ pub const paged = struct { .num_segments_per_seq = @intCast(config.attention.num_segments_per_seq), .all_decode = paged_attention_opts.all_decode, .is_causal = opts.mask.isCausal(), + .indices_count = @intCast(paged_attention_opts.indices_count), }; log.debug("pagedAttention3d attention config: {any}", .{attn_kernel_config}); @@ -459,6 +492,7 @@ pub const paged = struct { .head_size_padded = head_size_padded, .block_q = @intCast(config.reduce.block_q), .num_segments_per_seq = @intCast(config.reduce.num_segments_per_seq), + .indices_count = @intCast(paged_attention_opts.indices_count), .use_fp8 = false, }; log.debug("pagedAttention3d reduce config: {any}", .{reduce_kernel_config}); @@ -485,6 +519,10 @@ pub const paged = struct { const attn_output = kernels.KernelUnifiedAttention3dPtr.Kernel.call( .{ .query_ptr = q, + .indices_ptr = switch (opts.mask) { + .indices => |indices| indices.transpose(.{ .b, .topk }), + else => dummy, + }, .key_cache_ptr = k_cache, .value_cache_ptr = v_cache, .sink_ptr = sink, diff --git a/zml/attention/triton_kernels/unified_attention.zig b/zml/attention/triton_kernels/unified_attention.zig index a94fae73..dd219a88 100644 --- a/zml/attention/triton_kernels/unified_attention.zig +++ b/zml/attention/triton_kernels/unified_attention.zig @@ -49,6 +49,7 @@ pub const KernelUnifiedAttention2dPtr = struct { fp8_max: f32 = FP8_E4M3_MAX, all_decode: bool = false, is_causal: bool = true, + indices_count: i64 = 0, }; pub const Kernel = tri.Kernel(Config, .{ @@ -60,7 +61,7 @@ pub const KernelUnifiedAttention2dPtr = struct { "softcap_ptr", "block_table_stride_ptr", "query_stride_0_ptr", "query_stride_1_ptr", "output_stride_0_ptr", "output_stride_1_ptr", "qq_bias_stride_0_ptr", "stride_k_cache_0_ptr", "stride_k_cache_1_ptr", "stride_k_cache_2_ptr", "stride_v_cache_0_ptr", "stride_v_cache_1_ptr", - "stride_v_cache_2_ptr", "query_start_len_ptr", "num_seqs_ptr", + "stride_v_cache_2_ptr", "query_start_len_ptr", "num_seqs_ptr", "indices_ptr", }, .outputs = &.{"output"}, .run = run, @@ -94,6 +95,7 @@ pub const KernelUnifiedAttention2dPtr = struct { .stride_v_cache_2_ptr = .{ .ptr = .i64 }, .query_start_len_ptr = .{ .ptr = .i32 }, .num_seqs_ptr = .{ .ptr = .i32 }, + .indices_ptr = .{ .ptr = .i32 }, .output_ptr = .{ .ptr = cfg.o_dtype }, }); @@ -120,6 +122,7 @@ pub const KernelUnifiedAttention2dPtr = struct { b, a.output_ptr, a.query_ptr, + a.indices_ptr, a.key_cache_ptr, a.value_cache_ptr, a.sink_ptr, @@ -180,6 +183,7 @@ pub const KernelUnifiedAttention3dPtr = struct { num_segments_per_seq: i64, all_decode: bool = false, is_causal: bool = true, + indices_count: i64 = 0, }; pub const Kernel = tri.Kernel(Config, .{ @@ -191,6 +195,7 @@ pub const KernelUnifiedAttention3dPtr = struct { "block_table_stride_ptr", "query_stride_0_ptr", "query_stride_1_ptr", "qq_bias_stride_0_ptr", "stride_k_cache_0_ptr", "stride_k_cache_1_ptr", "stride_k_cache_2_ptr", "stride_v_cache_0_ptr", "stride_v_cache_1_ptr", "stride_v_cache_2_ptr", "query_start_len_ptr", "num_seqs_ptr", + "indices_ptr", }, .outputs = &.{ "segm_output", "segm_max", "segm_expsum" }, .run = run, @@ -221,6 +226,7 @@ pub const KernelUnifiedAttention3dPtr = struct { .stride_v_cache_2_ptr = .{ .ptr = .i64 }, .query_start_len_ptr = .{ .ptr = .i32 }, .num_seqs_ptr = .{ .ptr = .i32 }, + .indices_ptr = .{ .ptr = .i32 }, .segm_output_ptr = .{ .ptr = .f32 }, .segm_max_ptr = .{ .ptr = .f32 }, .segm_expsum_ptr = .{ .ptr = .f32 }, @@ -248,6 +254,7 @@ pub const KernelUnifiedAttention3dPtr = struct { a.segm_max_ptr, a.segm_expsum_ptr, a.query_ptr, + a.indices_ptr, a.key_cache_ptr, a.value_cache_ptr, a.sink_ptr, @@ -290,6 +297,7 @@ pub const ReduceSegmentsPtr = struct { block_q: i64, num_segments_per_seq: i64, + indices_count: i64 = 0, use_fp8: bool, fp8_min: f32 = FP8_E4M3_MIN, fp8_max: f32 = FP8_E4M3_MAX, @@ -422,6 +430,7 @@ fn kernelUnifiedAttention2d( k: *Builder, output_ptr: Value, query_ptr: Value, + indices_ptr: Value, key_cache_ptr: Value, value_cache_ptr: Value, sink_ptr: Value, @@ -487,7 +496,11 @@ fn kernelUnifiedAttention2d( // query_pos = q_block_local_idx * BLOCK_Q + offs_m // NUM_QUERIES_PER_KV const q_local_bq = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))); - const query_pos = q_local_bq.add(offs_m.div(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); + // Sparse tiles contain one query token and its GQA heads, with padded head lanes. + const query_pos = if (config.indices_count > 0) + q_local_bq.splatTo(&.{BLOCK_M}) + else + q_local_bq.add(offs_m.div(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); const query_offset_0 = cur_batch_in_all_start_index.add(query_pos); const query_offset_1 = kv_head_idx.mul(@as(i32, @intCast(NUM_QUERIES_PER_KV))) @@ -502,7 +515,10 @@ fn kernelUnifiedAttention2d( else k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); const query_mask_0 = query_pos.lt(cur_batch_query_len); - const query_mask_1 = query_offset_1.lt(@as(i32, @intCast(NUM_QUERY_HEADS))); + const query_mask_1 = if (config.indices_count > 0) + offs_m.lt(@as(i32, @intCast(NUM_QUERIES_PER_KV))) + else + query_offset_1.lt(@as(i32, @intCast(NUM_QUERY_HEADS))); // Q : (BLOCK_M, HEAD_SIZE_PADDED) const q_mask_ab = k.mask2d(query_mask_0, dim_mask, BLOCK_M, HEAD_SIZE_PADDED); @@ -553,7 +569,7 @@ fn kernelUnifiedAttention2d( // represented in this block. In non-causal mode, scan the whole sequence // so draft rows can attend to all rows written for the same sequence. const pad_term: i32 = @intCast(@divTrunc(BLOCK_M - 1, NUM_QUERIES_PER_KV) + 1); - const max_seq_prefix_len = if (config.is_causal) b: { + const max_seq_prefix_len = if (config.indices_count > 0) k.liftAs(config.indices_count, .i32) else if (config.is_causal) b: { const max_prefix_raw = context_len.add(q_local_bq).add(pad_term); break :b max_prefix_raw.minimum(seq_len); } else seq_len; @@ -591,13 +607,22 @@ fn kernelUnifiedAttention2d( const L = loop.carried[1]; const acc = loop.carried[2]; - const seq_offset = j.mul(@as(i32, @intCast(TILE_SIZE))).add(offs_t); - - const tile_mask: Value = seq_offset.lt(max_seq_prefix_len); - - const physical_block_idx = k.load( - block_tables_ptr_shifted.addPtr(seq_offset.div(@as(i32, @intCast(BLOCK_SIZE)))), + const selection_offset = j.mul(@as(i32, @intCast(TILE_SIZE))).add(offs_t); + const selection_mask = selection_offset.lt(max_seq_prefix_len); + const seq_offset = if (config.indices_count > 0) k.loadOpts( + indices_ptr.addPtr(cur_batch_in_all_start_index.add(q_local_bq).to(.i64).mul(config.indices_count)).addPtr(selection_offset), + .{ .mask = selection_mask, .other = k.full(&.{TILE_SIZE}, -1, .i32) }, + ) else selection_offset; + var tile_mask = if (config.indices_count > 0) + selection_mask.bitAnd(seq_offset.ge(0)).bitAnd(seq_offset.lt(seq_len)) + else + selection_mask; + const safe_offset = k.where(tile_mask, seq_offset, k.zeros(&.{TILE_SIZE}, .i32)); + const physical_block_idx = k.loadOpts( + block_tables_ptr_shifted.addPtr(safe_offset.div(@as(i32, @intCast(BLOCK_SIZE)))), + .{ .mask = tile_mask, .other = k.full(&.{TILE_SIZE}, -1, .i32) }, ).to(.i64); + tile_mask = tile_mask.bitAnd(physical_block_idx.ge(0)); const v_offset = k.expandDims(physical_block_idx, 1).mul(stride_v_cache_0) .add(kv_head_idx.to(.i64).mul(stride_v_cache_2)) @@ -689,7 +714,12 @@ fn kernelUnifiedAttention2d( const alpha_2d = alpha.expandDims(1); const acc_scaled = acc.mul(alpha_2d); const new_L = L.mul(alpha).add(l_j); - const new_M = m_j; + // An empty sparse tile must not establish a maximum of zero: later + // selected keys can have very negative scores and still carry all mass. + const new_M = if (config.indices_count > 0) + k.where(new_L.gt(0.0), m_j, k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32)) + else + m_j; const P_cast = P.to(config.q_dtype); const new_acc = k.dot(P_cast, V, acc_scaled); @@ -700,7 +730,7 @@ fn kernelUnifiedAttention2d( var acc_final = loop.results[2]; const L_2d = k.expandDims(L_final, 1); - const one_over_L = k.full(&.{ BLOCK_M, 1 }, 1.0, .f32).div(L_2d); + const one_over_L = k.where(L_2d.gt(0.0), k.full(&.{ BLOCK_M, 1 }, 1.0, .f32).div(L_2d), k.zeros(&.{ BLOCK_M, 1 }, .f32)); acc_final = acc_final.mul(k.broadcastTo(one_over_L, &.{ BLOCK_M, HEAD_SIZE_PADDED })); if (config.use_fp8) { @@ -731,6 +761,7 @@ fn kernelUnifiedAttention3d( segm_max_ptr: Value, segm_expsum_ptr: Value, query_ptr: Value, + indices_ptr: Value, key_cache_ptr: Value, value_cache_ptr: Value, sink_ptr: Value, @@ -788,10 +819,11 @@ fn kernelUnifiedAttention3d( k.returnIf(q_out_of_range, .{}); const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); - const tiles_per_segment = cdivFn(seq_len, @as(i32, @intCast(NUM_SEGMENTS_PER_SEQ * TILE_SIZE))); + const attention_len = if (config.indices_count > 0) k.liftAs(config.indices_count, .i32) else seq_len; + const tiles_per_segment = cdivFn(attention_len, @as(i32, @intCast(NUM_SEGMENTS_PER_SEQ * TILE_SIZE))); const segm_lo = segm_idx.mul(tiles_per_segment).mul(@as(i32, @intCast(TILE_SIZE))); - const segm_out_of_range = segm_lo.ge(seq_len); + const segm_out_of_range = segm_lo.ge(attention_len); k.returnIf(segm_out_of_range, .{}); const offs_m = k.arange(0, BLOCK_M, .i32); @@ -799,7 +831,11 @@ fn kernelUnifiedAttention3d( const offs_t = k.arange(0, TILE_SIZE, .i32); const q_local_bq = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))); - const query_pos = q_local_bq.add(offs_m.div(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); + // Sparse tiles contain one query token and its GQA heads, with padded head lanes. + const query_pos = if (config.indices_count > 0) + q_local_bq.splatTo(&.{BLOCK_M}) + else + q_local_bq.add(offs_m.div(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); const query_offset_0 = cur_batch_in_all_start_index.add(query_pos); const query_offset_1 = kv_head_idx.mul(@as(i32, @intCast(NUM_QUERIES_PER_KV))) @@ -817,7 +853,10 @@ fn kernelUnifiedAttention3d( else k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); const query_mask_0 = query_pos.lt(cur_batch_query_len); - const query_mask_1 = query_offset_1.lt(@as(i32, @intCast(NUM_QUERY_HEADS))); + const query_mask_1 = if (config.indices_count > 0) + offs_m.lt(@as(i32, @intCast(NUM_QUERIES_PER_KV))) + else + query_offset_1.lt(@as(i32, @intCast(NUM_QUERY_HEADS))); const q_mask_pre = k.expandDims(query_mask_0, 1).bitAnd(k.expandDims(query_mask_1, 1)); const q_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) @@ -870,7 +909,7 @@ fn kernelUnifiedAttention3d( qq_bias_ptr.splatTo(&.{BLOCK_M}); const pad_term: i32 = @intCast(@divTrunc(BLOCK_M - 1, NUM_QUERIES_PER_KV) + 1); - const max_seq_prefix_len = if (config.is_causal) b: { + const max_seq_prefix_len = if (config.indices_count > 0) k.liftAs(config.indices_count, .i32) else if (config.is_causal) b: { const max_prefix_raw = context_len.add(q_local_bq).add(pad_term); break :b max_prefix_raw.minimum(seq_len); } else seq_len; @@ -889,12 +928,22 @@ fn kernelUnifiedAttention3d( const L = loop.carried[1]; const acc = loop.carried[2]; - const seq_offset = j.mul(@as(i32, @intCast(TILE_SIZE))).add(offs_t); - const tile_mask: Value = seq_offset.lt(max_seq_prefix_len); - - const physical_block_idx = k.load( - block_tables_ptr.addPtr(block_table_offset).addPtr(seq_offset.div(@as(i32, @intCast(BLOCK_SIZE)))), + const selection_offset = j.mul(@as(i32, @intCast(TILE_SIZE))).add(offs_t); + const selection_mask = selection_offset.lt(max_seq_prefix_len); + const seq_offset = if (config.indices_count > 0) k.loadOpts( + indices_ptr.addPtr(cur_batch_in_all_start_index.add(q_local_bq).to(.i64).mul(config.indices_count)).addPtr(selection_offset), + .{ .mask = selection_mask, .other = k.full(&.{TILE_SIZE}, -1, .i32) }, + ) else selection_offset; + var tile_mask = if (config.indices_count > 0) + selection_mask.bitAnd(seq_offset.ge(0)).bitAnd(seq_offset.lt(seq_len)) + else + selection_mask; + const safe_offset = k.where(tile_mask, seq_offset, k.zeros(&.{TILE_SIZE}, .i32)); + const physical_block_idx = k.loadOpts( + block_tables_ptr.addPtr(block_table_offset).addPtr(safe_offset.div(@as(i32, @intCast(BLOCK_SIZE)))), + .{ .mask = tile_mask, .other = k.full(&.{TILE_SIZE}, -1, .i32) }, ).to(.i64); + tile_mask = tile_mask.bitAnd(physical_block_idx.ge(0)); const v_offset = k.expandDims(physical_block_idx, 1).mul(stride_v_cache_0) .add(kv_head_idx.to(.i64).mul(stride_v_cache_2)) @@ -989,7 +1038,11 @@ fn kernelUnifiedAttention3d( const P_cast = P.to(config.q_dtype); const new_acc = k.dot(P_cast, V, acc_scaled); - loop.yield(.{ m_j, new_L, new_acc }); + const new_M = if (config.indices_count > 0) + k.where(new_L.gt(0.0), m_j, k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32)) + else + m_j; + loop.yield(.{ new_M, new_L, new_acc }); } const M_final = loop.results[0]; const L_final = loop.results[1]; @@ -1055,12 +1108,17 @@ fn reduceSegments( const query_token_idx = k.programId(.x); const query_head_idx = k.programId(.y); + if (config.indices_count > 0) { + const active_queries = k.load(query_start_len_ptr.addPtr(num_seqs)); + k.returnIf(query_token_idx.ge(active_queries), .{}); + } const seq_idx = findSeqIdx(k, query_start_len_ptr, query_token_idx, num_seqs, BLOCK_Q, false); const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); - const tiles_per_segment = cdivFn(seq_len, @as(i32, @intCast(NUM_SEGMENTS_PER_SEQ * TILE_SIZE))); - const act_num_segments = cdivFn(seq_len, tiles_per_segment.mul(@as(i32, @intCast(TILE_SIZE)))); + const attention_len = if (config.indices_count > 0) k.liftAs(config.indices_count, .i32) else seq_len; + const tiles_per_segment = cdivFn(attention_len, @as(i32, @intCast(NUM_SEGMENTS_PER_SEQ * TILE_SIZE))); + const act_num_segments = cdivFn(attention_len, tiles_per_segment.mul(@as(i32, @intCast(TILE_SIZE)))); const seg_range = k.arange(0, NUM_SEGMENTS_PER_SEQ, .i32); const segm_mask = seg_range.lt(act_num_segments); @@ -1082,7 +1140,11 @@ fn reduceSegments( .mask = segm_mask, .other = k.full(&.{NUM_SEGMENTS_PER_SEQ}, -std.math.inf(f32), .f32), }); - const overall_max = k.max(segm_max); + const max_value = k.max(segm_max); + const overall_max = if (config.indices_count > 0) + k.where(max_value.gt(-std.math.inf(f32)), max_value, k.liftAs(0.0, .f32)) + else + max_value; var segm_expsum = k.loadOpts(segm_expsum_ptr.addPtr(segm_offset), .{ .mask = segm_mask, From 3b70cc23a0286886155ff29b1e6923f923286749 Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Mon, 5 Oct 2026 17:51:01 +0200 Subject: [PATCH 753/881] zml/workspace: bump llvm to 0.8.24 (#901) Bump the hermetic LLVM Bazel dependency from 0.8.9 to 0.8.24 --- MODULE.bazel | 2 +- MODULE.bazel.lock | 33 +++++++++++++++++++++++---------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 71056c62..1e05f12a 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -51,7 +51,7 @@ bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "rules_swift", version = "3.6.1", repo_name = None) bazel_dep(name = "rules_zig", version = "0.15.1") bazel_dep(name = "tar.bzl", version = "0.10.4") -bazel_dep(name = "llvm", version = "0.8.9") +bazel_dep(name = "llvm", version = "0.8.24") bazel_dep(name = "zlib-ng", version = "2.3.3", repo_name = "llvm_zlib") bazel_dep(name = "zstd", version = "1.5.7.bcr.1", repo_name = "llvm_zstd") bazel_dep(name = "with_cfg.bzl", version = "0.14.4") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index a83ce7bb..838e2011 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -18,7 +18,8 @@ "https://bcr.bazel.build/modules/apple_support/1.24.2/MODULE.bazel": "0e62471818affb9f0b26f128831d5c40b074d32e6dda5a0d3852847215a41ca4", "https://bcr.bazel.build/modules/apple_support/2.3.0/MODULE.bazel": "d48f824ae8eeea5f837eb3038cef3615075d996d3e82eb9187192c2820605a81", "https://bcr.bazel.build/modules/apple_support/2.5.4/MODULE.bazel": "964e9268b37edb64e9a0fa525e5b5e535ba56bf63cadc68375f92e69fcb734f6", - "https://bcr.bazel.build/modules/apple_support/2.5.4/source.json": "df85e46410ff71c4006a13dbdeeeee488feba563be78f3a9ed8497c1fb93fb5d", + "https://bcr.bazel.build/modules/apple_support/2.8.0/MODULE.bazel": "c45f5176057092afba0be6c48a9ea02101666802f8be536947e099e94757f8ed", + "https://bcr.bazel.build/modules/apple_support/2.8.0/source.json": "b7f6d87669c206adaa7d82cc44733349c63e341b80a0e049496ebd91d78c3472", "https://bcr.bazel.build/modules/aspect_bazel_lib/1.31.2/MODULE.bazel": "7bee702b4862612f29333590f4b658a5832d433d6f8e4395f090e8f4e85d442f", "https://bcr.bazel.build/modules/aspect_bazel_lib/1.38.0/MODULE.bazel": "6307fec451ba9962c1c969eb516ebfe1e46528f7fa92e1c9ac8646bef4cdaa3f", "https://bcr.bazel.build/modules/aspect_bazel_lib/1.40.3/MODULE.bazel": "668e6bcb4d957fc0e284316dba546b705c8d43c857f87119619ee83c4555b859", @@ -56,6 +57,7 @@ "https://bcr.bazel.build/modules/bazel_features/1.32.0/MODULE.bazel": "095d67022a58cb20f7e20e1aefecfa65257a222c18a938e2914fd257b5f1ccdc", "https://bcr.bazel.build/modules/bazel_features/1.33.0/MODULE.bazel": "8b8dc9d2a4c88609409c3191165bccec0e4cb044cd7a72ccbe826583303459f6", "https://bcr.bazel.build/modules/bazel_features/1.34.0/MODULE.bazel": "e8475ad7c8965542e0c7aac8af68eb48c4af904be3d614b6aa6274c092c2ea1e", + "https://bcr.bazel.build/modules/bazel_features/1.36.0/MODULE.bazel": "596cb62090b039caf1cad1d52a8bc35cf188ca9a4e279a828005e7ee49a1bec3", "https://bcr.bazel.build/modules/bazel_features/1.38.0/MODULE.bazel": "f9b8a9c890ebd216b4049fd12a31d3c2602e3403c7af636b04fbbd7453edc9c9", "https://bcr.bazel.build/modules/bazel_features/1.39.0/MODULE.bazel": "28739425c1fc283c91931619749c832b555e60bcd1010b40d8441ce0a5cf726d", "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", @@ -63,7 +65,8 @@ "https://bcr.bazel.build/modules/bazel_features/1.42.1/MODULE.bazel": "275a59b5406ff18c01739860aa70ad7ccb3cfb474579411decca11c93b951080", "https://bcr.bazel.build/modules/bazel_features/1.43.0/MODULE.bazel": "defa2226f06ba20550d6548c3a2ea2a7929634437a52973869c20c225450eb91", "https://bcr.bazel.build/modules/bazel_features/1.47.0/MODULE.bazel": "e34df3cb35b1684cfa69923a61ae3803595babd3942cd306a488d51400886b30", - "https://bcr.bazel.build/modules/bazel_features/1.47.0/source.json": "4ba0b5138327f2d73352a51547a4e49a0a828ef400e046b15334d8905bf6b7ff", + "https://bcr.bazel.build/modules/bazel_features/1.50.0/MODULE.bazel": "2083ef9c7a469f520890483ccf8e0189d6e71e2117e7752e15e6554433d5ae3e", + "https://bcr.bazel.build/modules/bazel_features/1.50.0/source.json": "e0ee3debde2789ff56e4452e612d126925ba9ab64d4bde79c67f099d2902df9b", "https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b", "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", "https://bcr.bazel.build/modules/bazel_lib/3.0.0-beta.1/MODULE.bazel": "407729e232f611c3270005b016b437005daa7b1505826798ea584169a476e878", @@ -130,7 +133,8 @@ "https://bcr.bazel.build/modules/gazelle/0.39.1/MODULE.bazel": "1fa3fefad240e535066fd0e6950dfccd627d36dc699ee0034645e51dbde3980f", "https://bcr.bazel.build/modules/gazelle/0.45.0/MODULE.bazel": "ecd19ebe9f8e024e1ccffb6d997cc893a974bcc581f1ae08f386bdd448b10687", "https://bcr.bazel.build/modules/gazelle/0.47.0/MODULE.bazel": "b61bb007c4efad134aa30ee7f4a8e2a39b22aa5685f005edaa022fbd1de43ebc", - "https://bcr.bazel.build/modules/gazelle/0.47.0/source.json": "aeb2e5df14b7fb298625d75d08b9c65bdb0b56014c5eb89da9e5dd0572280ae6", + "https://bcr.bazel.build/modules/gazelle/0.51.3/MODULE.bazel": "618a729142f66de1e2cb776d026413763be5b80d5e3d29ffb9d3d90c5defde90", + "https://bcr.bazel.build/modules/gazelle/0.51.3/source.json": "fbe5312a01fb4a2a58caff4a0d40dcf384b6f0bda75e85546663360da1d7538a", "https://bcr.bazel.build/modules/google_benchmark/1.8.4/MODULE.bazel": "c6d54a11dcf64ee63545f42561eda3fd94c1b5f5ebe1357011de63ae33739d5e", "https://bcr.bazel.build/modules/google_benchmark/1.8.4/source.json": "84590f7bc5a1fd99e1ef274ee16bb41c214f705e62847b42e705010dfa81fe53", "https://bcr.bazel.build/modules/googleapis-cc/1.0.0/MODULE.bazel": "cf01757e7590c56140a4b81638ff2b3e7074769e6271720bbf738fcda25b6fc2", @@ -171,8 +175,8 @@ "https://bcr.bazel.build/modules/libbacktrace/1.0.0-20250926-7939218/source.json": "83239989d09748f63dee4bdd430d261e217a012670e066e2238925a303a65d89", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", "https://bcr.bazel.build/modules/libpfm/4.11.0/source.json": "caaffb3ac2b59b8aac456917a4ecf3167d40478ee79f15ab7a877ec9273937c9", - "https://bcr.bazel.build/modules/llvm/0.8.9/MODULE.bazel": "9e35ff5bcac996f9edc1b44b8f8baa58c7855e742c5608b07d925dcdbe642100", - "https://bcr.bazel.build/modules/llvm/0.8.9/source.json": "424b907b38875f1346b65946e5e482b202b842bd3a570abbe07047d2bc7d667b", + "https://bcr.bazel.build/modules/llvm/0.8.20/MODULE.bazel": "2c0fa2fb60ab1f3a8a7b46a6c60b67fd78bffcfaf7958477e74a971c0a1f1915", + "https://bcr.bazel.build/modules/llvm/0.8.20/source.json": "21ef6b5c124a1a39341752e8458de9acd6fb4abc9ad73e6e3147b374d95e5e8c", "https://bcr.bazel.build/modules/mbedtls/3.6.0/MODULE.bazel": "8e380e4698107c5f8766264d4df92e36766248447858db28187151d884995a09", "https://bcr.bazel.build/modules/mbedtls/3.6.0/source.json": "1dbe7eb5258050afcc3806b9d43050f71c6f539ce0175535c670df606790b30c", "https://bcr.bazel.build/modules/nlohmann_json/3.11.3/MODULE.bazel": "87023db2f55fc3a9949c7b08dc711fae4d4be339a80a99d04453c4bb3998eefc", @@ -196,7 +200,8 @@ "https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92", "https://bcr.bazel.build/modules/package_metadata/0.0.3/MODULE.bazel": "77890552ecea9e284b5424c9de827a58099348763a4359e975c359a83d4faa83", "https://bcr.bazel.build/modules/package_metadata/0.0.5/MODULE.bazel": "ef4f9439e3270fdd6b9fd4dbc3d2f29d13888e44c529a1b243f7a31dfbc2e8e4", - "https://bcr.bazel.build/modules/package_metadata/0.0.5/source.json": "2326db2f6592578177751c3e1f74786b79382cd6008834c9d01ec865b9126a85", + "https://bcr.bazel.build/modules/package_metadata/0.0.7/MODULE.bazel": "7adb03933fc8401f495800cf4eafcff0edc6da0ff55c7db223ef69d19f689486", + "https://bcr.bazel.build/modules/package_metadata/0.0.7/source.json": "50639625e937b56115012674c797cca7a05a96b4878c87d803c13dc2b31de8a0", "https://bcr.bazel.build/modules/patchelf/0.18.0.bcr.1/MODULE.bazel": "35320668481400503dea7c75313473b9ffd6dea4e7ba583b96a896199b0088d5", "https://bcr.bazel.build/modules/patchelf/0.18.0.bcr.1/source.json": "d877d271ef5d531407885322054c99c7f0617a9a85ff99c0a70f4ced7f13e13a", "https://bcr.bazel.build/modules/pcre2/10.45/MODULE.bazel": "9bdc838aa2e21be030d7181385e45a5c87b14575bb7ff35d10b5be3d56913bed", @@ -289,7 +294,9 @@ "https://bcr.bazel.build/modules/rules_cc/0.2.17/MODULE.bazel": "1849602c86cb60da8613d2de887f9566a6d354a6df6d7009f9d04a14402f9a84", "https://bcr.bazel.build/modules/rules_cc/0.2.18/MODULE.bazel": "4460ec36adc8f722a6a2a4ac9374cb91f2acebadaa93fc37966129afb3dece87", "https://bcr.bazel.build/modules/rules_cc/0.2.19/MODULE.bazel": "d5e0f05b63273281a16654eb6b1a8742a75ec153ac8b4f0419949d6e401e46f0", - "https://bcr.bazel.build/modules/rules_cc/0.2.19/source.json": "1ef48cdbd7aa6238015189b582d3d74ef0cbea3cb3e2cb259d782463f570c14a", + "https://bcr.bazel.build/modules/rules_cc/0.2.20/MODULE.bazel": "f5c07bce5ddcb99be21a0812ff5aadb439e688b7449c6542152363b2fd859c1a", + "https://bcr.bazel.build/modules/rules_cc/0.2.25/MODULE.bazel": "4a3d4f3606d3b3190be495dbc497acca552807d1d5540661ef0d1658472882e3", + "https://bcr.bazel.build/modules/rules_cc/0.2.25/source.json": "8df28a9a97b878fe45f0a4f4adf019869dda51a60ee2f6e15a0c79656a25058d", "https://bcr.bazel.build/modules/rules_cc/0.2.4/MODULE.bazel": "1ff1223dfd24f3ecf8f028446d4a27608aa43c3f41e346d22838a4223980b8cc", "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", "https://bcr.bazel.build/modules/rules_distroless/0.8.0/MODULE.bazel": "a3e473943b685190e20668f5149f62a116fdcbeba66c7b9778f71d2aed4533c1", @@ -311,7 +318,8 @@ "https://bcr.bazel.build/modules/rules_go/0.50.1/MODULE.bazel": "b91a308dc5782bb0a8021ad4330c81fea5bda77f96b9e4c117b9b9c8f6665ee0", "https://bcr.bazel.build/modules/rules_go/0.53.0/MODULE.bazel": "a4ed760d3ac0dbc0d7b967631a9a3fd9100d28f7d9fcf214b4df87d4bfff5f9a", "https://bcr.bazel.build/modules/rules_go/0.59.0/MODULE.bazel": "b7e43e7414a3139a7547d1b4909b29085fbe5182b6c58cbe1ed4c6272815aeae", - "https://bcr.bazel.build/modules/rules_go/0.59.0/source.json": "1df17bb7865cfc029492c30163cee891d0dd8658ea0d5bfdf252c4b6db5c1ef6", + "https://bcr.bazel.build/modules/rules_go/0.62.0/MODULE.bazel": "8ee616065c3d2b2f7ac0880108316ce8d0c332b3a30aad24e95c0bc124ec853e", + "https://bcr.bazel.build/modules/rules_go/0.62.0/source.json": "36d781c558eb7d3bd49dc5e190455714f174232372f15207ab200b4348bda3e6", "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", "https://bcr.bazel.build/modules/rules_java/5.3.5/MODULE.bazel": "a4ec4f2db570171e3e5eb753276ee4b389bae16b96207e9d3230895c99644b86", "https://bcr.bazel.build/modules/rules_java/5.5.0/MODULE.bazel": "486ad1aa15cdc881af632b4b1448b0136c76025a1fe1ad1b65c5899376b83a50", @@ -396,6 +404,7 @@ "https://bcr.bazel.build/modules/rules_python/1.6.0/MODULE.bazel": "7e04ad8f8d5bea40451cf80b1bd8262552aa73f841415d20db96b7241bd027d8", "https://bcr.bazel.build/modules/rules_python/1.6.3/MODULE.bazel": "a7b80c42cb3de5ee2a5fa1abc119684593704fcd2fec83165ebe615dec76574f", "https://bcr.bazel.build/modules/rules_python/1.7.0/MODULE.bazel": "d01f995ecd137abf30238ad9ce97f8fc3ac57289c8b24bd0bf53324d937a14f8", + "https://bcr.bazel.build/modules/rules_python/1.8.0/MODULE.bazel": "c151c025dbcc93d8f62ab68ecc313c9176a868a0e6386981bf2a12aec77cbe7b", "https://bcr.bazel.build/modules/rules_python/1.9.0/MODULE.bazel": "afc3a05f29f09f2d3ee95ad99a145250dab41a2b2d8d6f82cc91936b3213282c", "https://bcr.bazel.build/modules/rules_python/1.9.0/source.json": "3921ea0b65298d51aead5b9e4a82203d7be9b5918619b58b53f1c259f4e63169", "https://bcr.bazel.build/modules/rules_robolectric/4.14.1.2/MODULE.bazel": "d44fec647d0aeb67b9f3b980cf68ba634976f3ae7ccd6c07d790b59b87a4f251", @@ -408,7 +417,8 @@ "https://bcr.bazel.build/modules/rules_shell/0.4.0/MODULE.bazel": "0f8f11bb3cd11755f0b48c1de0bbcf62b4b34421023aa41a2fc74ef68d9584f0", "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", "https://bcr.bazel.build/modules/rules_shell/0.6.1/MODULE.bazel": "72e76b0eea4e81611ef5452aa82b3da34caca0c8b7b5c0c9584338aa93bae26b", - "https://bcr.bazel.build/modules/rules_shell/0.6.1/source.json": "20ec05cd5e592055e214b2da8ccb283c7f2a421ea0dc2acbf1aa792e11c03d0c", + "https://bcr.bazel.build/modules/rules_shell/0.8.0/MODULE.bazel": "f6a89f1d6a669a26f28fe814503857055d76306b79cfc11d12399af08d0b80ae", + "https://bcr.bazel.build/modules/rules_shell/0.8.0/source.json": "eb53cc815bc503c6683c5fe12d943f98883f81fc22f51403ec8a95610cba4195", "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", @@ -439,6 +449,8 @@ "https://bcr.bazel.build/modules/tar.bzl/0.6.0/MODULE.bazel": "a3584b4edcfafcabd9b0ef9819808f05b372957bbdff41601429d5fd0aac2e7c", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", "https://bcr.bazel.build/modules/upb/0.0.0-20230516-61a97ef/MODULE.bazel": "c0df5e35ad55e264160417fd0875932ee3c9dda63d9fccace35ac62f45e1b6f9", + "https://bcr.bazel.build/modules/windows_support/0.4.1/MODULE.bazel": "2287f95ac65c48ea05197f626a9b5711ea3c88cea61f2a5782882bdd55fdeb51", + "https://bcr.bazel.build/modules/windows_support/0.4.1/source.json": "bdb72e6717b00135e0a4a8512b3f6b626df0baea0bde74bf7312c7484f791287", "https://bcr.bazel.build/modules/with_cfg.bzl/0.12.0/MODULE.bazel": "b573395fe63aef4299ba095173e2f62ccfee5ad9bbf7acaa95dba73af9fc2b38", "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.1/MODULE.bazel": "aa0ef3f6c67dd35db7ac38c76785dc02f48b53f34e16b20487e70f51b5324d0a", "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.4/MODULE.bazel": "be2276f83878525d8f228b9ec777daee5eb7baba6100dad45081510d0cec35e3", @@ -449,8 +461,9 @@ "https://bcr.bazel.build/modules/yq.bzl/0.1.1/MODULE.bazel": "9039681f9bcb8958ee2c87ffc74bdafba9f4369096a2b5634b88abc0eaefa072", "https://bcr.bazel.build/modules/yq.bzl/0.3.1/MODULE.bazel": "9bcb7151b3cd4681b89d350530eaf7b45e32a44dda94843b8932b0cb1cd4594a", "https://bcr.bazel.build/modules/yq.bzl/0.3.1/source.json": "f0b0f204a2a6b0e34b4c9541efe8c04f2ef1af65948daa784eccea738b21dbd2", + "https://bcr.bazel.build/modules/zlib-ng/2.3.3.bcr.2/MODULE.bazel": "86130924115bf8dfc4aba286fb49a2ec8d32430a21771db9ccf5381acf0d10cc", + "https://bcr.bazel.build/modules/zlib-ng/2.3.3.bcr.2/source.json": "7aa4112eb2fbb99757e97cf0b7a4e2bdb3464b2b3cfb351bc807ffcb34bd349f", "https://bcr.bazel.build/modules/zlib-ng/2.3.3/MODULE.bazel": "0afa781a5f354b4fd811a5c9a086e111daf50e09d8e98ab1cf7c46eeacc33aac", - "https://bcr.bazel.build/modules/zlib-ng/2.3.3/source.json": "63314bf75a7683c75b7db365513e93dd1c13f865dacee676235310e8d92a7a4b", "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", "https://bcr.bazel.build/modules/zlib/1.2.12/MODULE.bazel": "3b1a8834ada2a883674be8cbd36ede1b6ec481477ada359cd2d3ddc562340b27", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.1/MODULE.bazel": "6a9fe6e3fc865715a7be9823ce694ceb01e364c35f7a846bf0d2b34762bc066b", From 3831492aa0cf514819f234f8ba34f9b121919aa8 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 5 Oct 2026 18:11:26 +0200 Subject: [PATCH 754/881] third_party/zio: bump zio=v0.19.0 --- bazel/zig_options.bzl | 30 +-- third_party/zio/progress-parent-file.patch | 17 -- third_party/zio/repo.bzl | 7 +- third_party/zio/runtime-page-size.patch | 297 --------------------- third_party/zio/zio.bazel | 30 ++- 5 files changed, 32 insertions(+), 349 deletions(-) delete mode 100644 third_party/zio/progress-parent-file.patch delete mode 100644 third_party/zio/runtime-page-size.patch diff --git a/bazel/zig_options.bzl b/bazel/zig_options.bzl index 88ab40bb..85938a0c 100644 --- a/bazel/zig_options.bzl +++ b/bazel/zig_options.bzl @@ -12,8 +12,8 @@ ZigOption = provider( "value": "The configured build setting or explicit value.", "type": "The Zig integer type, or None for other flag types.", "allowed_values": "Allowed values for string flags; an empty list allows any string. None for other flag types.", - "enum_name": "Optional exported enum type name for a constrained string flag.", - "nullable": "Whether an unconstrained string emits null for an empty value.", + "enum_name": "Exported enum type name for a constrained string flag; empty keeps the option as a string.", + "nullable": "Whether a string or enum emits null for an empty value.", }, ) @@ -41,8 +41,6 @@ def _zig_option_impl(ctx): fail("type is only supported for integer options") if ctx.attr.nullable and not types.is_string(value): fail("nullable is only supported for string options") - if ctx.attr.nullable and allowed_values: - fail("nullable strings cannot have enum values") if ctx.attr.enum_name and not allowed_values: fail("enum_name requires a string flag with nonempty values") return [ZigOption( @@ -81,15 +79,17 @@ def _zig_options_impl(ctx): elif types.is_int(value): zig_type = option.type literal = str(value) - elif option.allowed_values: - zig_type = "enum { " + ", ".join(["@" + _zig_string(v) for v in option.allowed_values]) + " }" - if option.enum_name: - if option.enum_name in names: - fail("Duplicate Zig declaration: {}".format(option.enum_name)) - names[option.enum_name] = True - lines.append("pub const @{} = {};".format(_zig_string(option.enum_name), zig_type)) - zig_type = "@" + _zig_string(option.enum_name) - literal = ".@" + _zig_string(value) + elif option.enum_name: + enum_values = [v for v in option.allowed_values if not option.nullable or v != ""] + zig_type = "enum { " + ", ".join(["@" + _zig_string(v) for v in enum_values]) + " }" + if option.enum_name in names: + fail("Duplicate Zig declaration: {}".format(option.enum_name)) + names[option.enum_name] = True + lines.append("pub const @{} = {};".format(_zig_string(option.enum_name), zig_type)) + zig_type = "@" + _zig_string(option.enum_name) + if option.nullable: + zig_type = "?" + zig_type + literal = "null" if option.nullable and not value else ".@" + _zig_string(value) else: zig_type = "?[]const u8" if option.nullable else "[]const u8" literal = "null" if option.nullable and not value else _zig_string(value) @@ -142,8 +142,8 @@ _zig_option = rule( "int": attr.int(), "value_source": attr.string(mandatory = True, values = ["flag", "bool", "string", "int"]), "type": attr.string(doc = "Zig integer type; defaults to usize. Only valid for integer options."), - "nullable": attr.bool(doc = "Emit an optional string; an empty value becomes null. Cannot be combined with values."), - "enum_name": attr.string(doc = "Optional exported Zig enum type name; requires nonempty values."), + "nullable": attr.bool(doc = "Emit an optional string or enum; an empty value becomes null. String flags with values must include an empty string to allow null; it is excluded from the enum."), + "enum_name": attr.string(doc = "Emit a named Zig enum instead of a string; requires nonempty values. Defaults to emitting a string."), }, doc = "Expose a build setting or explicit value as a Zig option, named after the flag or this target respectively.", ) diff --git a/third_party/zio/progress-parent-file.patch b/third_party/zio/progress-parent-file.patch deleted file mode 100644 index 2d92fd06..00000000 --- a/third_party/zio/progress-parent-file.patch +++ /dev/null @@ -1,17 +0,0 @@ -diff --git a/src/io.zig b/src/io.zig ---- a/src/io.zig -+++ b/src/io.zig -@@ -1853,7 +1853,12 @@ fn childKillImpl(_: ?*anyopaque, child: *std.process.Child) void { - } - - fn progressParentFileImpl(_: ?*anyopaque) std.Progress.ParentFileError!Io.File { -- @panic("progressParentFile: not supported"); -+ var environ: Io.Threaded.Environ = .{ .process_environ = processEnviron() }; -+ environ.scan(.failing); -+ var file = try environ.zig_progress_file; -+ // Inherited Windows handles are not associated with zio's IOCP port. -+ if (builtin.os.tag == .windows) flagsWritePollable(&file.flags, false); -+ return file; - } - - fn nowImpl(_: ?*anyopaque, clock: Io.Clock) Io.Timestamp { diff --git a/third_party/zio/repo.bzl b/third_party/zio/repo.bzl index 4f6e2d78..16b9e277 100644 --- a/third_party/zio/repo.bzl +++ b/third_party/zio/repo.bzl @@ -4,11 +4,6 @@ def repo(): git_repository( name = "zio", remote = "https://github.com/lalinsky/zio", - commit = "417757935ef6d0f0a2396d4023a44ca82ac1901c", + commit = "281d969fdefe04f2e8b28d1f45fac0011f881c73", # v0.19.0 build_file = Label("//third_party/zio:zio.bazel"), - patches = [ - Label("//third_party/zio:progress-parent-file.patch"), - Label("//third_party/zio:runtime-page-size.patch"), - ], - patch_args = ["-p1"], ) diff --git a/third_party/zio/runtime-page-size.patch b/third_party/zio/runtime-page-size.patch deleted file mode 100644 index 5a1db751..00000000 --- a/third_party/zio/runtime-page-size.patch +++ /dev/null @@ -1,297 +0,0 @@ -diff --git a/src/coro/stack.zig b/src/coro/stack.zig ---- a/src/coro/stack.zig -+++ b/src/coro/stack.zig -@@ -10,7 +10,12 @@ - - const log = @import("../common.zig").log; - --pub const page_size = if (builtin.os.tag == .freestanding) 1 else std.heap.page_size_min; -+// Pointer types need a comptime alignment; mapping geometry needs the OS page size. -+pub const page_alignment = if (builtin.os.tag == .freestanding) 1 else std.heap.page_size_min; -+ -+pub fn pageSize() usize { -+ return if (builtin.os.tag == .freestanding) 1 else std.heap.pageSize(); -+} - - // Signal type changed from c_int to enum in Zig 0.16 - const is_pre_016 = builtin.zig_version.major == 0 and builtin.zig_version.minor < 16; -@@ -24,7 +29,7 @@ - var old_sigbus_action: posix.Sigaction = undefined; - - pub const StackInfo = extern struct { -- allocation_ptr: [*]align(page_size) u8, // deallocation_stack on Windows (TEB offset 0x1478) -+ allocation_ptr: [*]align(page_alignment) u8, // deallocation_stack on Windows (TEB offset 0x1478) - base: usize, // stack_base on Windows (TEB offset 0x08) - limit: usize, // stack_limit on Windows (TEB offset 0x10) - allocation_len: usize, -@@ -45,6 +50,7 @@ - } - - fn stackAllocPosix(info: *StackInfo, maximum_size: usize, committed_size: usize) error{OutOfMemory}!void { -+ const page_size = pageSize(); - // Round maximum_size up to page boundary, then add guard page - const aligned_max = std.mem.alignForward(usize, maximum_size, page_size); - // Ensure we allocate at least 2 pages (guard + usable space) -@@ -102,7 +108,7 @@ - // Commit initial portion at top of stack - const stack_top = @intFromPtr(allocation.ptr) + size; - const initial_commit_start = stack_top - commit_size; -- const initial_region: [*]align(page_size) u8 = @ptrFromInt(initial_commit_start); -+ const initial_region: [*]align(page_alignment) u8 = @ptrFromInt(initial_commit_start); - posix.mprotect(initial_region[0..commit_size], posix.PROT.READ | posix.PROT.WRITE) catch |err| { - log.err("Failed to mprotect stack region (commit_size={d}): {}", .{ commit_size, err }); - return error.OutOfMemory; -@@ -160,6 +166,7 @@ - /// overflow faults on the guard page and crashes (no SIGSEGV growth handler is - /// installed, see setupStackGrowth). - fn stackAllocOpenBSD(info: *StackInfo, size: usize) error{OutOfMemory}!void { -+ const page_size = pageSize(); - const allocation = posix.mmap( - null, - size, -@@ -175,7 +182,7 @@ - - // Turn the lowest page into a PROT_NONE guard page. The rest of the mapping - // keeps its MAP_STACK flag, so the stack pointer stays valid for syscalls. -- const guard: [*]align(page_size) u8 = allocation.ptr; -+ const guard: [*]align(page_alignment) u8 = allocation.ptr; - posix.mprotect(guard[0..page_size], posix.PROT.NONE) catch |err| { - log.err("Failed to mprotect OpenBSD stack guard page: {}", .{err}); - return error.OutOfMemory; -@@ -197,7 +204,7 @@ - /// header. Pages between and below slots stay PROT_NONE, so guard pages cost - /// nothing extra. POSIX-only (the slab pool is comptime-disabled elsewhere); - /// OpenBSD is excluded because MAP_STACK cannot start as PROT_NONE. --pub fn slabReserve(len: usize) error{OutOfMemory}![]align(page_size) u8 { -+pub fn slabReserve(len: usize) error{OutOfMemory}![]align(page_alignment) u8 { - const prot_flags = posix.PROT.NONE | posix.PROT.MAX(posix.PROT.READ | posix.PROT.WRITE); - var map_flags = posix.MAP.PRIVATE | posix.MAP.ANONYMOUS; - if (builtin.os.tag == .linux or builtin.os.tag == .netbsd) { -@@ -216,7 +223,7 @@ - } - - // Commit the header page for the slab bookkeeping. -- posix.mprotect(allocation.ptr[0..page_size], posix.PROT.READ | posix.PROT.WRITE) catch |err| { -+ posix.mprotect(allocation.ptr[0..pageSize()], posix.PROT.READ | posix.PROT.WRITE) catch |err| { - log.err("Failed to commit stack slab header page: {}", .{err}); - return error.OutOfMemory; - }; -@@ -224,7 +231,7 @@ - return allocation; - } - --pub fn slabFree(mem: []align(page_size) u8) void { -+pub fn slabFree(mem: []align(page_alignment) u8) void { - posix.munmap(mem) catch {}; - } - -@@ -234,7 +241,8 @@ - /// exactly the layout stackAllocPosix produces, so growth, overflow - /// detection, and pooling treat both kinds identically. One mprotect per - /// cold slot; a recycled slot pays no syscalls at all. --pub fn stackInitSlot(info: *StackInfo, slot: []align(page_size) u8, committed_size: usize, owner_tag: usize) error{OutOfMemory}!void { -+pub fn stackInitSlot(info: *StackInfo, slot: []align(page_alignment) u8, committed_size: usize, owner_tag: usize) error{OutOfMemory}!void { -+ const page_size = pageSize(); - // Commit at least one page so the pool's FreeNode always fits. - const commit_size = @max(std.mem.alignForward(usize, committed_size, page_size), page_size); - if (commit_size > slot.len - page_size) { -@@ -244,7 +252,7 @@ - - const stack_top = @intFromPtr(slot.ptr) + slot.len; - const initial_commit_start = stack_top - commit_size; -- const initial_region: [*]align(page_size) u8 = @ptrFromInt(initial_commit_start); -+ const initial_region: [*]align(page_alignment) u8 = @ptrFromInt(initial_commit_start); - posix.mprotect(initial_region[0..commit_size], posix.PROT.READ | posix.PROT.WRITE) catch |err| { - log.err("Failed to commit slab stack slot (commit_size={d}): {}", .{ commit_size, err }); - return error.OutOfMemory; -@@ -299,7 +307,7 @@ - const committed_size = info.base - info.limit; - if (committed_size == 0) return; - -- const addr: [*]align(page_size) u8 = @ptrFromInt(committed_start); -+ const addr: [*]align(page_alignment) u8 = @ptrFromInt(committed_start); - - // MADV_FREE is available on Linux 4.5+, macOS, FreeBSD, NetBSD - // It allows lazy reclamation - physical pages are freed when system needs memory -@@ -332,6 +340,7 @@ - /// Mode .grow: Grow by 1.5x current size in 64KB chunks - /// Mode .full: Commit all remaining uncommitted stack - fn stackExtendPosix(info: *StackInfo, mode: StackExtendMode) error{StackOverflow}!void { -+ const page_size = pageSize(); - // OpenBSD stacks are fully committed at allocation time (MAP_STACK regions - // cannot be grown via mprotect), so there is nothing to extend. - if (builtin.os.tag == .openbsd) return; -@@ -341,7 +350,7 @@ - // Calculate new limit based on mode - const new_limit = switch (mode) { - .grow => blk: { -- const chunk_size = 64 * 1024; -+ const chunk_size = @max(64 * 1024, page_size); - const growth_factor_num = 3; - const growth_factor_den = 2; - -@@ -375,7 +384,7 @@ - // Commit the memory region - const commit_start = std.mem.alignBackward(usize, new_limit, page_size); - const commit_size = info.limit - commit_start; -- const addr: [*]align(page_size) u8 = @ptrFromInt(commit_start); -+ const addr: [*]align(page_alignment) u8 = @ptrFromInt(commit_start); - posix.mprotect(addr[0..commit_size], posix.PROT.READ | posix.PROT.WRITE) catch { - return error.StackOverflow; - }; -@@ -385,6 +394,7 @@ - } - - fn stackAllocWindows(info: *StackInfo, maximum_size: usize, committed_size: usize) error{OutOfMemory}!void { -+ const page_size = pageSize(); - // Round sizes up to page boundary - const commit_size = std.mem.alignForward(usize, committed_size, page_size); - const max_size = std.mem.alignForward(usize, maximum_size, page_size); -@@ -457,7 +467,7 @@ - - // Setup alternate stack for this thread if not already done - if (!altstack_installed) { -- const mem = try std.heap.page_allocator.alignedAlloc(u8, .fromByteUnits(page_size), altstack_size); -+ const mem = try std.heap.page_allocator.alignedAlloc(u8, .fromByteUnits(page_alignment), altstack_size); - errdefer std.heap.page_allocator.free(mem); - - var stack = posix.stack_t{ -@@ -599,7 +609,7 @@ - - // Stack layout: [guard_page][uncommitted][committed] - const stack_base = @intFromPtr(stack_info.allocation_ptr); -- const guard_page_end = stack_base + page_size; -+ const guard_page_end = stack_base + pageSize(); - const uncommitted_start = guard_page_end; - const uncommitted_end = stack_info.limit; - -@@ -631,7 +641,7 @@ - const stack_base = @intFromPtr(stack_info.allocation_ptr); - const stack_size = stack_info.allocation_len; - const committed = stack_info.base - stack_info.limit; -- const is_guard_page_fault = fault_addr >= stack_base and fault_addr < stack_base + page_size; -+ const is_guard_page_fault = fault_addr >= stack_base and fault_addr < stack_base + pageSize(); - - const msg = std.fmt.bufPrint( - &buf, -@@ -670,7 +680,7 @@ - // Verify at least the requested amount was committed (minus the owner - // tag reserve, which lives in committed memory above `base` on POSIX). - // Note: RtlCreateUserStack on Windows may commit more than requested -- const commit_size_rounded = std.mem.alignForward(usize, committed_size, page_size); -+ const commit_size_rounded = std.mem.alignForward(usize, committed_size, pageSize()); - const actual_committed = stack.base - stack.limit; - try std.testing.expect(actual_committed + owner_tag_reserve >= commit_size_rounded); - -diff --git a/src/coro/stack_pool.zig b/src/coro/stack_pool.zig ---- a/src/coro/stack_pool.zig -+++ b/src/coro/stack_pool.zig -@@ -33,7 +33,7 @@ - /// of a standalone stack mapping. - const Slab = struct { - next: ?*Slab, -- memory: []align(stack.page_size) u8, -+ memory: []align(stack.page_alignment) u8, - /// Slots handed out at least once. Only the newest slab still carves. - carved: usize, - /// Slots currently acquired by live coroutines. -@@ -114,9 +114,10 @@ - last_shrink: Timestamp, - - pub fn init(config: Config) StackPool { -+ const page_size = stack.pageSize(); - // Same slot layout as stackAllocPosix: usable size rounded to pages, - // plus the (never committed) guard page, at least two pages total. -- const aligned_max = std.mem.alignForward(usize, config.maximum_size, stack.page_size); -+ const aligned_max = std.mem.alignForward(usize, config.maximum_size, page_size); - return .{ - .config = config, - .mutex = .init(), -@@ -125,7 +126,7 @@ - .pool_size = 0, - .slabs = null, - .partial = null, -- .slot_size = @max(aligned_max + stack.page_size, stack.page_size * 2), -+ .slot_size = @max(aligned_max + page_size, page_size * 2), - .in_use = 0, - .epoch_peak = 0, - .retain_target = 0, -@@ -388,11 +389,12 @@ - /// The new slot is counted as in use on its slab; the caller accounts - /// the pool-wide acquire. - fn carveSlotLocked(self: *StackPool) ?StackInfo { -+ const page_size = stack.pageSize(); - const slab = blk: { - if (self.slabs) |s| { - if (s.carved < self.config.slab_slots) break :blk s; - } -- const len = stack.page_size + self.config.slab_slots * self.slot_size; -+ const len = page_size + self.config.slab_slots * self.slot_size; - const mem = stack.slabReserve(len) catch return null; - const s: *Slab = @ptrCast(@alignCast(mem.ptr)); - s.* = .{ .next = self.slabs, .memory = mem, .carved = 0, .in_use = 0, .free = null, .partial_prev = null, .partial_next = null }; -@@ -400,8 +402,8 @@ - break :blk s; - }; - -- const offset = stack.page_size + slab.carved * self.slot_size; -- const slot: []align(stack.page_size) u8 = @alignCast(slab.memory[offset .. offset + self.slot_size]); -+ const offset = page_size + slab.carved * self.slot_size; -+ const slot: []align(stack.page_alignment) u8 = @alignCast(slab.memory[offset .. offset + self.slot_size]); - var stack_info: StackInfo = undefined; - stack.stackInitSlot(&stack_info, slot, self.config.committed_size, @intFromPtr(slab)) catch return null; - slab.carved += 1; -@@ -428,7 +430,7 @@ - /// The slab this allocation was carved from, or null for an individually - /// mapped stack. Must run under mutex (the chain is mutated by carving - /// and shrinking). -- fn slabOfLocked(self: *StackPool, ptr: [*]align(stack.page_size) u8) ?*Slab { -+ fn slabOfLocked(self: *StackPool, ptr: [*]align(stack.page_alignment) u8) ?*Slab { - const addr = @intFromPtr(ptr); - var slab = self.slabs; - while (slab) |s| : (slab = s.next) { -@@ -771,3 +773,36 @@ - pool.shrink(.fromSeconds(30)); - try std.testing.expectEqual(0, pool.pool_size); - } -+ -+test "StackPool: runtime page alignment and full growth" { -+ if (!slab_supported) return error.SkipZigTest; -+ -+ for ([_]usize{ 0, 2 }) |slab_slots| { -+ var pool = StackPool.init(.{ -+ .maximum_size = 8 * 1024 * 1024, -+ .committed_size = 256 * 1024, -+ .slab_slots = slab_slots, -+ }); -+ defer pool.deinit(); -+ -+ // Keep slots live to exercise each offset and cross a slab boundary. -+ var infos: [3]StackInfo = undefined; -+ var acquired: usize = 0; -+ defer for (infos[0..acquired]) |info| pool.release(info); -+ for (&infos) |*info| { -+ info.* = try pool.acquire(); -+ acquired += 1; -+ if (slab_slots != 0) { -+ try std.testing.expect(pool.slabOfLocked(info.allocation_ptr) != null); -+ } -+ try std.testing.expectEqual(0, @intFromPtr(info.allocation_ptr) % stack.pageSize()); -+ try std.testing.expectEqual(0, info.limit % stack.pageSize()); -+ try stack.stackExtend(info, .grow); -+ try stack.stackExtend(info, .full); -+ try std.testing.expectEqual(@intFromPtr(info.allocation_ptr) + stack.pageSize(), info.limit); -+ const memory: [*]u8 = @ptrFromInt(info.limit); -+ memory[0] = 0xAA; -+ memory[info.base - info.limit - 1] = 0xBB; -+ } -+ } -+} -diff --git a/src/stack_test.zig b/src/stack_test.zig ---- /dev/null -+++ b/src/stack_test.zig -@@ -0,0 +1,4 @@ -+comptime { -+ _ = @import("coro/stack.zig"); -+ _ = @import("coro/stack_pool.zig"); -+} diff --git a/third_party/zio/zio.bazel b/third_party/zio/zio.bazel index 61cd6ef4..39805936 100644 --- a/third_party/zio/zio.bazel +++ b/third_party/zio/zio.bazel @@ -2,10 +2,16 @@ load("@bazel_skylib//rules:common_settings.bzl", "string_flag") load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") load("@zml//bazel:zig_options.bzl", "zig_option", "zig_options") -zig_option( +string_flag( name = "backend", - string = "", - nullable = True, + build_setting_default = "", + values = ["", "poll", "linux", "epoll", "kqueue", "io_uring", "iocp"], +) + +string_flag( + name = "scheduling", + build_setting_default = "work_stealing", + values = ["single_executor", "pinned", "work_stealing"], ) string_flag( @@ -14,23 +20,19 @@ string_flag( values = ["strict", "best_effort"], ) -zig_option( - name = "resolve_beneath_mode_option", - flag = ":resolve_beneath_mode", - enum_name = "ResolveBeneathMode", -) - +zig_option(name = "backend_option", flag = ":backend", nullable = True) +zig_option(name = "scheduling_option", flag = ":scheduling") +zig_option(name = "resolve_beneath_mode_option", flag = ":resolve_beneath_mode") zig_option(name = "no_hacks", bool = False) -zig_option(name = "task_migration", bool = True) zig_option(name = "scheduler_metrics", bool = False) zig_options( - name = "zio_options", + name = "zio_build_options", settings = [ - ":backend", + ":backend_option", + ":scheduling_option", ":resolve_beneath_mode_option", ":no_hacks", - ":task_migration", ":scheduler_metrics", ], ) @@ -39,6 +41,6 @@ zig_library( name = "zio", main = "src/zio.zig", srcs = glob(["src/**/*.zig"]), - deps = [":zio_options"], + deps = [":zio_build_options"], visibility = ["//visibility:public"], ) From b2f0e03043f6471ce6458a85aeefbca0c3fb151e Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 6 Oct 2026 15:37:22 +0200 Subject: [PATCH 755/881] zml/Tensor: explicit sharding in Tensor code (#868) In Buffer code we already have explicit sharding, but not for tensor. I'm adding a `zml.Sharding` to `.withPartitioning` to make it explicit which mesh we are working with. To simplify the migration, `.withPartitioning` in Tensor code can use `.model` syntax to lookup a sharding by name. This replace the previous logic of `selectSharding(shape)` which would look up a sharding that matches the axes of the given partitioning which could yield surprising behavior. This also modifies how sharding are stored in zml.Shape. Before it was storing a list of axis names + a tag (2 * 8 * 64bits), now I'm storing a zml.Sharding (64 bits) and a compact PartitionArray (32 bits) that references the logical axis of the sharding by offset instead of by name. There is an impact on the `TensorRegistry` because it also needs to be aware of the possible shardings, instead of delegating the matching partition spec -> sharding to loading. This means error will be detected sooner. `store.createTensor` accepts `@enumLiteral()` to lookup the relevant sharding. This also simplifies `zml.Buffer` because the full sharding is now stored in the shape, so it doesn't need to be passed independently. It also means that it will propagate from the tensor who are graph inputs, to the buffers that are eg inputs. `Placement` logic also got a bit simpler. --- README.md | 8 +- bin/zml-smi/BUILD.bazel | 18 +- bin/zml-smi/platforms/amdsmi/BUILD.bazel | 5 +- bin/zml-smi/platforms/nvml/BUILD.bazel | 5 +- docs/howtos/howto_torch2zml.md | 16 +- examples/benchmark/main.zig | 25 +- examples/io/main.zig | 27 +- examples/llm/main.zig | 11 +- examples/llm/models.zig | 17 +- examples/llm/models/common.zig | 18 +- examples/llm/models/lfm2/inference.zig | 16 +- examples/llm/models/lfm2/model.zig | 56 +- examples/llm/models/lfm2/session.zig | 20 +- examples/llm/models/lfm2_tests.zig | 57 +- examples/llm/models/llama/inference.zig | 12 +- examples/llm/models/llama/model.zig | 55 +- examples/llm/models/llama/session.zig | 13 +- examples/llm/models/llama_tests.zig | 40 +- examples/llm/models/qwen3_5/inference.zig | 26 +- examples/llm/models/qwen3_5/model.zig | 163 ++- examples/llm/models/qwen3_5/session.zig | 11 +- examples/llm/models/qwen3_5_moe/inference.zig | 20 +- examples/llm/models/qwen3_5_moe/model.zig | 195 ++-- examples/llm/models/qwen3_5_moe/session.zig | 9 +- examples/mnist/mnist.zig | 23 +- examples/sharding/main.zig | 37 +- platforms/cuda/BUILD.bazel | 2 + stdx/debug.zig | 3 +- stdx/fmt.zig | 38 + tools/BUILD.bazel | 3 +- zml/Compiler.zig | 189 ++-- zml/Sharding.zig | 941 +++++++++++------- zml/attention.zig | 47 +- zml/attention/attnd.zig | 9 +- zml/attention/flashattn.zig | 161 +-- zml/attention/metal_attention.zig | 3 +- zml/attention/nki/attention.zig | 9 +- zml/attention/paged_attention.zig | 79 +- zml/attention/sparse_mla.zig | 2 +- zml/attention/tpu_attention.zig | 14 +- zml/buffer.zig | 83 +- zml/exe.zig | 37 +- zml/io.zig | 224 ++--- zml/kernel.zig | 4 +- zml/mem.zig | 1 - zml/meta.zig | 13 +- zml/moe/cute_mxfp4.zig | 2 +- zml/moe/cutlass_flashinfer.zig | 6 +- zml/moe/fused_experts.zig | 10 +- zml/moe/mosaic_tpu.zig | 2 +- zml/moe/triton_mxfp4.zig | 2 +- zml/nn.zig | 35 +- zml/ops.zig | 121 ++- zml/platform.zig | 37 +- zml/quantization.zig | 4 +- zml/shape.zig | 566 +++++------ zml/tensor.zig | 278 ++++-- zml/testing.zig | 10 +- zml/zml.zig | 1 + 59 files changed, 2113 insertions(+), 1726 deletions(-) diff --git a/README.md b/README.md index f35453ff..d9e68b81 100644 --- a/README.md +++ b/README.md @@ -145,8 +145,8 @@ const Mnist = struct { pub fn init(store: zml.io.TensorStore.View) Layer { return .{ - .weight = store.createTensor("weight", .{ .d_out, .d }, null), - .bias = store.createTensor("bias", .{.d_out}, null), + .weight = store.createReplicatedTensor("weight", .{ .d_out, .d }), + .bias = store.createReplicatedTensor("bias", .{.d_out}), }; } @@ -168,10 +168,10 @@ const Mnist = struct { io: std.Io, platform: *const zml.Platform, store: *const zml.io.TensorStore, - shardings: []const zml.Sharding, + meshes: []const *const zml.Mesh, ) !zml.Bufferized(Mnist) { return zml.io.load(Mnist, self, allocator, io, platform, store, .{ - .shardings = shardings, + .meshes = meshes, .parallelism = 1, .dma_chunks = 1, .dma_chunk_size = 16 * 1024 * 1024, diff --git a/bin/zml-smi/BUILD.bazel b/bin/zml-smi/BUILD.bazel index d940a801..077f5b34 100644 --- a/bin/zml-smi/BUILD.bazel +++ b/bin/zml-smi/BUILD.bazel @@ -22,6 +22,7 @@ run_binary( "$(location libnrt.so.1)", "$(location @aws-neuronx-runtime-lib//:libnrt.patchelf)", ], + target_compatible_with = ["@platforms//os:linux"], tool = "@llvm//tools:llvm-strip", ) @@ -35,6 +36,7 @@ run_binary( "$(location libncfw.so.2)", "$(location @aws-neuronx-runtime-lib//:libncfw.patchelf)", ], + target_compatible_with = ["@platforms//os:linux"], tool = "@llvm//tools:llvm-strip", ) @@ -76,18 +78,18 @@ run_binary( zig_binary( name = "zml-smi", - visibility = ["//visibility:public"], data = [":sandbox"], main = "main.zig", + visibility = ["//visibility:public"], deps = [ + "//bin/zml-smi/api", "//bin/zml-smi/collector", "//bin/zml-smi/csv", - "//bin/zml-smi/json", - "//bin/zml-smi/api", - "//bin/zml-smi/prometheus", - "//bin/zml-smi/utils:sysfs", "//bin/zml-smi/info", + "//bin/zml-smi/json", + "//bin/zml-smi/prometheus", "//bin/zml-smi/tui", + "//bin/zml-smi/utils:sysfs", "//stdx", "@libvaxis//:vaxis", "@rules_zig//zig/runfiles", @@ -97,9 +99,9 @@ zig_binary( }) + select({ "@llvm//platforms/config:linux_x86_64": [ "//bin/zml-smi/platforms/amdsmi", - "//bin/zml-smi/platforms/oneapi", "//bin/zml-smi/platforms/neuron", "//bin/zml-smi/platforms/nvml", + "//bin/zml-smi/platforms/oneapi", "//bin/zml-smi/platforms/tpu", "//platforms/neuron:libnrt_headers", ], @@ -168,8 +170,8 @@ _RELEASE_PLATFORMS = [ [platform_transition_filegroup( name = "zml-smi-" + os + "-" + arch, srcs = [":archive_" + os + "_" + arch], - target_platform = "//platforms:" + os + "_" + arch, tags = ["manual"], + target_platform = "//platforms:" + os + "_" + arch, ) for os, arch in _RELEASE_PLATFORMS] [hashes( @@ -195,6 +197,7 @@ oci_image( name = "image_", base = "@distroless_cc_debian12", entrypoint = ["/zml-smi/zml-smi"], + tags = ["manual"], target_compatible_with = [ "@platforms//os:linux", "@platforms//cpu:x86_64", @@ -202,7 +205,6 @@ oci_image( tars = [ ":archive_linux_amd64", ], - tags = ["manual"], ) platform_transition_filegroup( diff --git a/bin/zml-smi/platforms/amdsmi/BUILD.bazel b/bin/zml-smi/platforms/amdsmi/BUILD.bazel index e3ac355d..dac6b935 100644 --- a/bin/zml-smi/platforms/amdsmi/BUILD.bazel +++ b/bin/zml-smi/platforms/amdsmi/BUILD.bazel @@ -2,12 +2,13 @@ load("@rules_zig//zig:defs.bzl", "zig_library") zig_library( name = "amdsmi", - import_name = "zml-smi/platforms/amdsmi", - main = "metrics.zig", srcs = [ "amdsmi.zig", "process.zig", ], + import_name = "zml-smi/platforms/amdsmi", + main = "metrics.zig", + target_compatible_with = ["@platforms//os:linux"], visibility = ["//visibility:public"], deps = [ "//bin/zml-smi/collector", diff --git a/bin/zml-smi/platforms/nvml/BUILD.bazel b/bin/zml-smi/platforms/nvml/BUILD.bazel index 3a9e54ed..0ab0fae1 100644 --- a/bin/zml-smi/platforms/nvml/BUILD.bazel +++ b/bin/zml-smi/platforms/nvml/BUILD.bazel @@ -2,12 +2,13 @@ load("@rules_zig//zig:defs.bzl", "zig_library") zig_library( name = "nvml", - import_name = "zml-smi/platforms/nvml", - main = "metrics.zig", srcs = [ "nvml.zig", "process.zig", ], + import_name = "zml-smi/platforms/nvml", + main = "metrics.zig", + target_compatible_with = ["@platforms//os:linux"], visibility = ["//visibility:public"], deps = [ "//bin/zml-smi/collector", diff --git a/docs/howtos/howto_torch2zml.md b/docs/howtos/howto_torch2zml.md index d05e2b6f..3d246f11 100644 --- a/docs/howtos/howto_torch2zml.md +++ b/docs/howtos/howto_torch2zml.md @@ -190,7 +190,7 @@ convert it to a `TensorStore`: ```zig pub fn main(init: std.process.Init) !void { ... - var model_store: zml.io.TensorStore = .fromRegistry(allocator, &model_registry); + var model_store: zml.io.TensorStore = .fromRegistry(allocator, &model_registry, &.{}); defer model_store.deinit(); } ``` @@ -205,18 +205,18 @@ pub fn main(init: std.process.Init) !void { const mlp: Mlp = .{ .up_proj = .init( - mlp_view.createTensor("up_proj.weight", .{ .dout, .d }, null), - mlp_view.maybeCreateTensor("up_proj.bias", .{.dout}, null), + mlp_view.createReplicatedTensor("up_proj.weight", .{ .dout, .d }), + mlp_view.maybeCreateReplicatedTensor("up_proj.bias", .{.dout}), .d, ), .gate_proj = .init( - mlp_view.createTensor("gate_proj.weight", .{ .dout, .d }, null), - mlp_view.maybeCreateTensor("gate_proj.bias", .{.dout}, null), + mlp_view.createReplicatedTensor("gate_proj.weight", .{ .dout, .d }), + mlp_view.maybeCreateReplicatedTensor("gate_proj.bias", .{.dout}), .d, ), .down_proj = .init( - mlp_view.createTensor("down_proj.weight", .{ .d, .dout }, null), - mlp_view.maybeCreateTensor("down_proj.bias", .{.d}, null), + mlp_view.createReplicatedTensor("down_proj.weight", .{ .d, .dout }), + mlp_view.maybeCreateReplicatedTensor("down_proj.bias", .{.d}), .dout, ), }; @@ -287,7 +287,7 @@ pub const Wrapper = struct { pub fn main(init: std.process.Init) !void { ... - var activations_store: zml.io.TensorStore = .fromRegistry(allocator, &activations_registry); + var activations_store: zml.io.TensorStore = .fromRegistry(allocator, &activations_registry, &.{}); defer activations_store.deinit(); try zml.testing.testLayer(allocator, io, platform, Wrapper{ .mlp = mlp }, .forward, activations_store.view(), "model.model.layers.0.mlp", .{ .mlp = mlp_weights }, &.{}, .{}); diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index 1bdaad7b..7ff70707 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -5,7 +5,7 @@ const zml = @import("zml"); const stdx = zml.stdx; pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { - return a.dot(b, .k).withPartitioning(.{ .m = .m, .n = .replicated }); + return a.dot(b, .k).withPartitioning(.benchmark_mesh, .{ .m = .m, .n = .replicated }); } pub fn main(init: std.process.Init) !void { @@ -26,34 +26,31 @@ pub fn main(init: std.process.Init) !void { log.info("\n{f}", .{platform.fmtVerbose()}); - const benchmark_sharding: zml.Sharding = try platform.registerSharding("benchmark_mesh", .mesh( + const mesh: *const zml.Mesh = try platform.registerMesh("mesh", .mesh( .{ .m = .low_bandwidth, .n = .high_bandwidth }, )); const cli_args: CliArgs = stdx.flags.parse(init.minimal.args, CliArgs); - const a_shape = zml.Shape.init(.{ .m = cli_args.size, .k = cli_args.size }, cli_args.dtype) - .withPartitioning(.{ .m = .m, .k = .replicated }); - const b_shape = zml.Shape.init(.{ .k = cli_args.size, .n = cli_args.size }, cli_args.dtype) - .withPartitioning(.{ .k = .replicated, .n = .n }); - - const a: zml.Tensor = .fromShape(a_shape); - const b: zml.Tensor = .fromShape(b_shape); + const a = zml.Tensor.init(.{ .m = cli_args.size, .k = cli_args.size }, cli_args.dtype) + .withPartitioning(mesh, .{ .m = .m, .k = .replicated }); + const b = zml.Tensor.init(.{ .k = cli_args.size, .n = cli_args.size }, cli_args.dtype) + .withPartitioning(mesh, .{ .k = .replicated, .n = .n }); var exe = blk: { log.info("⏱️ Compiling benchmark...", .{}); const now: std.Io.Timestamp = .now(io, .awake); defer log.info("✅ Compiled benchmark [{f}]", .{now.untilNow(io, .awake)}); - break :blk try platform.compileFn(allocator, io, benchmark, .{ a, b }, .{ .shardings = &.{benchmark_sharding} }); + break :blk try platform.compileFn(allocator, io, benchmark, .{ a, b }, .{ .meshes = &.{mesh} }); }; defer exe.deinit(); var rng = std.Random.DefaultPrng.init(0); const random = rng.random(); - var a_buffer = try createRandomBuffer(allocator, io, platform, a.shape(), benchmark_sharding, random); + var a_buffer = try createRandomBuffer(allocator, io, platform, a.shape(), random); defer a_buffer.deinit(); - var b_buffer = try createRandomBuffer(allocator, io, platform, b.shape(), benchmark_sharding, random); + var b_buffer = try createRandomBuffer(allocator, io, platform, b.shape(), random); defer b_buffer.deinit(); var exe_args = try exe.args(allocator); @@ -96,7 +93,7 @@ pub fn main(init: std.process.Init) !void { }); } -fn createRandomBuffer(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, shape: zml.Shape, sharding: zml.Sharding, random: std.Random) !zml.Buffer { +fn createRandomBuffer(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, shape: zml.Shape, random: std.Random) !zml.Buffer { const slice = try zml.Slice.alloc(allocator, shape); defer slice.free(allocator); @@ -122,5 +119,5 @@ fn createRandomBuffer(allocator: std.mem.Allocator, io: std.Io, platform: *const }, } - return .fromSlice(io, platform, slice, sharding); + return .fromSlice(io, platform, slice); } diff --git a/examples/io/main.zig b/examples/io/main.zig index 29215866..1cec23b0 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -155,9 +155,9 @@ pub fn main(init: std.process.Init) !void { try stdout_writer.interface.flush(); }, .load => { - const ShardingType = enum { replicated, sharded }; + const MeshType = enum { replicated, sharded }; - const sharding_type: ShardingType = std.meta.stringToEnum(ShardingType, it.next() orelse "sharded") orelse return error.InvalidShardingKind; + const mesh_type: MeshType = std.meta.stringToEnum(MeshType, it.next() orelse "sharded") orelse return error.InvalidMesheKind; const platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator, io); @@ -165,7 +165,13 @@ pub fn main(init: std.process.Init) !void { var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); defer registry.deinit(); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + const tensor_parallel: *const zml.Mesh = try platform.registerMesh( + "tp", + .mesh(.{ .model = .high_bandwidth }), + ); + const tp_partitioning: zml.Sharding.Partitioning = .init(&.{.sharded(@intCast(tensor_parallel.resolveLogicalAxis("model").?))}); + + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{tensor_parallel}); defer store.deinit(); const AllTensorsModel = struct { @@ -180,22 +186,17 @@ pub fn main(init: std.process.Init) !void { var registry_it = registry.iterator(); var load_count: usize = 0; while (registry_it.next()) |entry| : (load_count += 1) { - tensors[load_count] = switch (sharding_type) { - .replicated => store.view().createTensor(entry.key_ptr.*, null, .replicated), + tensors[load_count] = switch (mesh_type) { + .replicated => store.view().createReplicatedTensor(entry.key_ptr.*, null), .sharded => if (entry.value_ptr.shape.rank() > 0) - store.view().createTensor(entry.key_ptr.*, null, .{ ._0 = .model }) + store.view().createTensor(entry.key_ptr.*, null, .tp, tp_partitioning) else - store.view().createTensor(entry.key_ptr.*, null, .replicated), + store.view().createReplicatedTensor(entry.key_ptr.*, null), }; } const model: AllTensorsModel = .{ .tensors = tensors }; - const sharded_sharding: zml.Sharding = try platform.registerSharding( - "playground_model", - .mesh(.{ .model = .high_bandwidth }), - ); - var progress = std.Progress.start(io, .{ .root_name = "zml.examples.load" }); progress.increaseEstimatedTotalItems(load_count); defer progress.end(); @@ -215,7 +216,7 @@ pub fn main(init: std.process.Init) !void { }); defer loader.deinit(); - try loader.load(io, AllTensorsModel, &model, &buffers, &store, &.{sharded_sharding}, .{ .progress = &progress }); + try loader.load(io, AllTensorsModel, &model, &buffers, &store, .{ .progress = &progress }); try loader.await(io); const took = now.untilNow(io, .awake); diff --git a/examples/llm/main.zig b/examples/llm/main.zig index 4407aadb..e10db570 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -109,7 +109,9 @@ pub fn main(init: std.process.Init) !void { var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); defer registry.deinit(); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + // Defines how the model's tensors are sharded across the available devices. + const meshes: models.Meshes = try .init(platform); + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); defer store.deinit(); const generation: models.GenerationOptions = .{ @@ -121,9 +123,6 @@ pub fn main(init: std.process.Init) !void { var model = try models.LoadedModel.load(allocator, io, repo, store.view(), generation); defer model.deinit(allocator); - // Defines how the model's tensors are sharded across the available devices. - const shardings: models.Shardings = try .init(platform); - // // Load the model and compile it // @@ -135,11 +134,11 @@ pub fn main(init: std.process.Init) !void { var compiled_model = try allocator.create(models.CompiledModel); defer allocator.destroy(compiled_model); - compiled_model.* = try models.LoadedModel.compile(&model, allocator, io, platform, backend, shardings, args.seqlen, &progress); + compiled_model.* = try models.LoadedModel.compile(&model, allocator, io, platform, backend, meshes, args.seqlen, &progress); defer compiled_model.deinit(); // Load buffers after the model compilation to be sure to give enough room to the autotune. - var model_buffers = try models.LoadedModel.loadBuffers(&model, allocator, io, platform, &store, &progress, shardings); + var model_buffers = try models.LoadedModel.loadBuffers(&model, allocator, io, platform, &store, &progress); defer model.unloadBuffers(&model_buffers, allocator); progress.end(); diff --git a/examples/llm/models.zig b/examples/llm/models.zig index b88a855c..20aa60c7 100644 --- a/examples/llm/models.zig +++ b/examples/llm/models.zig @@ -3,7 +3,7 @@ const std = @import("std"); const zml = @import("zml"); const common = @import("models/common.zig"); -pub const Shardings = common.Shardings; +pub const Meshes = common.Meshes; pub const GenerationOptions = common.GenerationOptions; pub const parseConfig = common.parseConfig; pub const lfm2 = @import("models/lfm2.zig"); @@ -55,12 +55,19 @@ pub const LoadedModel = union(ModelType) { } } - pub fn loadBuffers(self: *LoadedModel, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, shardings: Shardings) !Buffers { + pub fn loadBuffers( + self: *LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + ) !Buffers { return switch (self.*) { inline else => |*m, t| @unionInit( Buffers, @tagName(t), - try m.loadBuffers(allocator, io, platform, store, progress, shardings), + try m.loadBuffers(allocator, io, platform, store, progress), ), }; } @@ -77,7 +84,7 @@ pub const LoadedModel = union(ModelType) { io: std.Io, platform: *const zml.Platform, backend: zml.attention.Backend, - shardings: Shardings, + meshes: Meshes, seqlen: usize, progress: *std.Progress.Node, ) !CompiledModel { @@ -85,7 +92,7 @@ pub const LoadedModel = union(ModelType) { inline else => |*m, t| @unionInit( CompiledModel.Inner, @tagName(t), - try m.compile(allocator, io, platform, backend, shardings, seqlen, progress), + try m.compile(allocator, io, platform, backend, meshes, seqlen, progress), ), }; return .{ diff --git a/examples/llm/models/common.zig b/examples/llm/models/common.zig index debeff00..62fc7560 100644 --- a/examples/llm/models/common.zig +++ b/examples/llm/models/common.zig @@ -42,11 +42,11 @@ pub const Phase = enum { } }; -pub const Shardings = struct { - model: zml.Sharding, - experts: zml.Sharding, +pub const Meshes = struct { + model: *const zml.Mesh, + experts: *const zml.Mesh, - pub fn init(platform: *zml.Platform) !Shardings { + pub fn init(platform: *zml.Platform) !Meshes { switch (platform.target) { .tpu => { const has_link_y = platform.physical_mesh.hasAxis(.link_y); @@ -59,18 +59,18 @@ pub const Shardings = struct { } return .{ - .model = try platform.registerShardingWithStrategy("model", .mesh(.{ .model = .high_bandwidth }), strategy_model), - .experts = try platform.registerShardingWithStrategy("experts", .mesh(.{ .experts = .high_bandwidth }), strategy_experts), + .model = try platform.registerMeshWithStrategy("model", .mesh(.{ .model = .high_bandwidth }), strategy_model), + .experts = try platform.registerMeshWithStrategy("experts", .mesh(.{ .experts = .high_bandwidth }), strategy_experts), }; }, .cuda, .rocm, .oneapi, .neuron, .metal, .cpu => return .{ - .model = try platform.registerSharding("model", .mesh(.{ .model = .high_bandwidth })), - .experts = try platform.registerSharding("experts", .mesh(.{ .experts = .high_bandwidth })), + .model = try platform.registerMesh("model", .mesh(.{ .model = .high_bandwidth })), + .experts = try platform.registerMesh("experts", .mesh(.{ .experts = .high_bandwidth })), }, } } - pub fn all(self: Shardings) [2]zml.Sharding { + pub fn all(self: Meshes) [2]*const zml.Mesh { return .{ self.model, self.experts }; } }; diff --git a/examples/llm/models/lfm2/inference.zig b/examples/llm/models/lfm2/inference.zig index 407df36e..92ab4462 100644 --- a/examples/llm/models/lfm2/inference.zig +++ b/examples/llm/models/lfm2/inference.zig @@ -17,9 +17,9 @@ pub const CompilationParameters = struct { attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, seqlen: u32, - shardings: common.Shardings, + meshes: common.Meshes, - pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: zml.attention.Backend, shardings: common.Shardings) CompilationParameters { + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: zml.attention.Backend, meshes: common.Meshes) CompilationParameters { stdx.debug.assert(seqlen >= config.conv_L_cache, "seqlen ({}) must be at least conv_L_cache ({})", .{ seqlen, config.conv_L_cache }); const cache: model.Cache = .{ .kv = .init(.init(.{ @@ -28,7 +28,7 @@ pub const CompilationParameters = struct { .k = seqlen, .h = config.num_key_value_heads, .hd = config.hidden_size / config.num_attention_heads, - }, mdl.embed_tokens.weight.dtype())), + }, mdl.embed_tokens.weight.dtype()), meshes.model), .conv = .init(.init(.{ .layer = mdl.num_conv_layers, .batch = 1, @@ -42,10 +42,10 @@ pub const CompilationParameters = struct { .batch_dim = 1, .rng = .init(), .cache = cache, - .attention_metadata = .init(.fromBackend(backend, seqlen, config.num_attention_heads)), + .attention_metadata = .init(.fromBackend(backend, seqlen, config.num_attention_heads), meshes.model), .attention_parameters = .init(.fromBackend(backend)), .seqlen = seqlen, - .shardings = shardings, + .meshes = meshes, }; } }; @@ -235,7 +235,7 @@ fn compileEmbed( defer phase.logCompileDone(log, "embed_tokens", io, from); return zml.FnExe(model.TokenEmbedding.forward).compile(allocator, io, platform, .{ - .shardings = &opts.shardings.all(), + .meshes = &opts.meshes.all(), .program_name = phase.programName("lfm2", "embed_tokens"), }, .{.{ .embedding = embed_tokens, @@ -273,7 +273,7 @@ fn compileLayer( } else unreachable; return zml.FnExe(model.DecoderLayer.forward).compile(allocator, io, platform, .{ - .shardings = &opts.shardings.all(), + .meshes = &opts.meshes.all(), .program_name = phase.programName("lfm2", if (kind == .conv) "conv_layer" else "attn_layer"), }, .{.{ .layer = layer, @@ -306,7 +306,7 @@ fn compileSample( defer phase.logCompileDone(log, "lm_head", io, from); return zml.FnExe(model.LmHead.forward).compile(allocator, io, platform, .{ - .shardings = &opts.shardings.all(), + .meshes = &opts.meshes.all(), .program_name = phase.programName("lfm2", "lm_head"), }, .{.{ .lm_head = mdl.lm_head, diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index 53f1071c..1d627025 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -79,7 +79,6 @@ pub const LoadedModel = struct { platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, - shardings: common.Shardings, ) !Buffers { progress.increaseEstimatedTotalItems(store.view().count()); const now: std.Io.Timestamp = .now(io, .awake); @@ -94,8 +93,7 @@ pub const LoadedModel = struct { }); defer loader.deinit(); - const all_shardings = shardings.all(); - try loader.load(io, Model, &self.inner, &buffers, store, &all_shardings, .{ .progress = progress }); + try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); try loader.await(io); const took = now.untilNow(io, .awake); @@ -117,11 +115,11 @@ pub const LoadedModel = struct { io: std.Io, platform: *const zml.Platform, backend: zml.attention.Backend, - shardings: common.Shardings, + meshes: common.Meshes, seqlen: usize, progress: *std.Progress.Node, ) !inference.CompiledModel { - const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), backend, shardings); + const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), backend, meshes); return inference.CompiledModel.init(allocator, io, @constCast(platform), self, self.inner, params, progress); } }; @@ -173,7 +171,6 @@ pub const Model = struct { platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, - shardings: common.Shardings, ) !zml.Bufferized(Model) { progress.increaseEstimatedTotalItems(store.view().count()); const now: std.Io.Timestamp = .now(io, .awake); @@ -188,8 +185,7 @@ pub const Model = struct { }); defer loader.deinit(); - const all_shardings = shardings.all(); - loader.load(io, Model, self, &buffers, store, &all_shardings); + loader.load(io, Model, self, &buffers, store); try loader.await(io); const took = now.untilNow(io, .awake); @@ -278,7 +274,7 @@ pub const TokenEmbedding = struct { }; pub fn init(store: zml.io.TensorStore.View) TokenEmbedding { - return .{ .weight = store.createTensor("weight", .{ .voc, .d }, .{ .voc = .replicated, .d = .model }) }; + return .{ .weight = store.createTensor("weight", .{ .voc, .d }, .model, .{ .voc = .replicated, .d = .model }) }; } pub fn forward(input: Input) Output { @@ -448,16 +444,16 @@ pub const ShortConv = struct { pub fn init(config: Config, store: zml.io.TensorStore.View) ShortConv { stdx.debug.assert(!config.conv_bias, "conv_bias is not supported.", .{}); return .{ - .in_proj = .init(store.withPrefix("in_proj").createTensor("weight", .{ .out, .d }, .{ .out = .model, .d = .replicated }), null, .d), - .out_proj = .init(store.withPrefix("out_proj").createTensor("weight", .{ .out, .d }, .{ .out = .replicated, .d = .model }), null, .d), - .kernel = store.createTensor("conv.weight", .{ .out, .in, .kernel_size }, .replicated), + .in_proj = .init(store.withPrefix("in_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + .out_proj = .init(store.withPrefix("out_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .replicated, .d = .model }), null, .d), + .kernel = store.createReplicatedTensor("conv.weight", .{ .out, .in, .kernel_size }), .config = config, }; } pub fn forward(self: ShortConv, input: zml.Tensor, tokens_position_offset: zml.Tensor, actual_seq_len: zml.Tensor, cache_: ConvCache, cache_index: zml.Tensor, parameters: ConvParameters) struct { zml.Tensor, ConvCache } { var cache = cache_; - const BCx = self.in_proj.forward(input.withPartitioning(.{ .d = .replicated })); + const BCx = self.in_proj.forward(input.withPartitioning(.model, .{ .d = .replicated })); const B, const C, const x = BCx.chunkExact(.d, 3); const Bx = B.mul(x); @@ -523,10 +519,10 @@ pub const Attention = struct { const head_dim = config.hidden_size / config.num_attention_heads; const num_key_value_groups = config.num_attention_heads / config.num_key_value_heads; return .{ - .q_proj = .init(store.withPrefix("q_proj").createTensor("weight", .{ .out, .d }, .{ .out = .model, .d = .replicated }), null, .d), - .k_proj = .init(store.withPrefix("k_proj").createTensor("weight", .{ .out, .d }, .{ .out = .model, .d = .replicated }), null, .d), - .v_proj = .init(store.withPrefix("v_proj").createTensor("weight", .{ .out, .d }, .{ .out = .model, .d = .replicated }), null, .d), - .out_proj = .init(store.withPrefix("out_proj").createTensor("weight", .{ .out, .d }, .{ .out = .replicated, .d = .model }), null, .d), + .q_proj = .init(store.withPrefix("q_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + .k_proj = .init(store.withPrefix("k_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + .v_proj = .init(store.withPrefix("v_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + .out_proj = .init(store.withPrefix("out_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .replicated, .d = .model }), null, .d), .q_layernorm = RmsNorm.init(store.withPrefix("q_layernorm"), config.norm_eps, .hd), .k_layernorm = RmsNorm.init(store.withPrefix("k_layernorm"), config.norm_eps, .hd), .head_dim = head_dim, @@ -544,7 +540,7 @@ pub const Attention = struct { attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, ) struct { zml.Tensor, KvCache } { - const x_qkv = x.withPartitioning(.{ .d = .replicated }); + const x_qkv = x.withPartitioning(.model, .{ .d = .replicated }); var q = self.q_proj.forward(x_qkv).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); var k = self.k_proj.forward(x_qkv).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); @@ -608,8 +604,8 @@ pub const Cache = struct { conv: ConvCache, kv: KvCache, - pub fn initBuffers(self: Cache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(Cache) { - return .{ .conv = try self.conv.initBuffers(allocator, io, platform, sharding), .kv = try self.kv.initBuffers(io, platform, sharding) }; + pub fn initBuffers(self: Cache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Cache) { + return .{ .conv = try self.conv.initBuffers(allocator, io, platform), .kv = try self.kv.initBuffers(io, platform) }; } pub fn unloadBuffers(self: *zml.Bufferized(Cache)) void { @@ -629,12 +625,12 @@ pub const ConvCache = struct { return .{ .state = .fromShape(shape) }; } - pub fn initBuffers(self: ConvCache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(ConvCache) { + pub fn initBuffers(self: ConvCache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(ConvCache) { const sh = self.state.shape(); const host = try allocator.alloc(u8, sh.byteSize()); defer allocator.free(host); @memset(host, 0); - return .{ .state = try zml.Buffer.fromBytes(io, platform, sh, sharding, host) }; + return .{ .state = try zml.Buffer.fromBytes(io, platform, sh, host) }; } pub fn unloadBuffers(self: *zml.Bufferized(ConvCache)) void { @@ -650,13 +646,13 @@ pub const KvCache = struct { k: zml.Tensor, v: zml.Tensor, - pub fn init(kv_shape: zml.Shape) KvCache { - const sharded_shape = kv_shape.withPartitioning(.{ .h = .model }); + pub fn init(kv_shape: zml.Shape, mesh: *const zml.Mesh) KvCache { + const sharded_shape = kv_shape.withPartitioning(mesh, .{ .h = .model }); return .{ .k = .fromShape(sharded_shape), .v = .fromShape(sharded_shape) }; } - pub fn initBuffers(self: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(KvCache) { - return .{ .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), sharding, .{}), .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), sharding, .{}) }; + pub fn initBuffers(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { + return .{ .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}) }; } pub fn unloadBuffers(self: *zml.Bufferized(KvCache)) void { @@ -717,9 +713,9 @@ const Mlp = struct { pub fn init(store: zml.io.TensorStore.View) Mlp { return .{ - .w1 = .init(store.withPrefix("w1").createTensor("weight", .{ .out, .d }, .{ .out = .model, .d = .replicated }), null, .d), - .w2 = .init(store.withPrefix("w2").createTensor("weight", .{ .out, .d }, .{ .out = .replicated, .d = .model }), null, .d), - .w3 = .init(store.withPrefix("w3").createTensor("weight", .{ .out, .d }, .{ .out = .model, .d = .replicated }), null, .d), + .w1 = .init(store.withPrefix("w1").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + .w2 = .init(store.withPrefix("w2").createTensor("weight", .{ .out, .d }, .model, .{ .out = .replicated, .d = .model }), null, .d), + .w3 = .init(store.withPrefix("w3").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), }; } @@ -740,7 +736,7 @@ const RmsNorm = struct { tag: zml.Shape.Tag, pub fn init(store: zml.io.TensorStore.View, eps: f32, tag: anytype) RmsNorm { - return .{ .weight = store.createTensor("weight", .{tag}, .replicated), .eps = eps, .tag = zml.Shape.toTag(tag) }; + return .{ .weight = store.createReplicatedTensor("weight", .{tag}), .eps = eps, .tag = zml.Shape.toTag(tag) }; } pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig index 67287861..b59a0416 100644 --- a/examples/llm/models/lfm2/session.zig +++ b/examples/llm/models/lfm2/session.zig @@ -30,9 +30,9 @@ pub const Session = struct { model_buffers: *model.Buffers, ) !Session { const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); - var cache_buffers = try compiled_model.params.cache.initBuffers(allocator, io, platform, compiled_model.params.shardings.model); + var cache_buffers = try compiled_model.params.cache.initBuffers(allocator, io, platform); errdefer model.Cache.unloadBuffers(&cache_buffers); - var rng_buf = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed); + var rng_buf = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buf); const generated_token_slice = try zml.Slice.alloc(allocator, zml.Shape.init(.{ .batch = 1, .seq = 1 }, .u32)); errdefer generated_token_slice.free(allocator); @@ -120,21 +120,21 @@ pub const Session = struct { @memset(tokens, self.config.pad_token_id); @memcpy(tokens[0..all_tokens.len], all_tokens); - var tokens_buf: zml.Buffer = try .fromSlice(self.io, self.platform, tokens_slice, .replicated); + var tokens_buf: zml.Buffer = try .fromSlice(self.io, self.platform, tokens_slice); defer tokens_buf.deinit(); const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{0})); - var tokens_pos_buf: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice, .replicated); + var tokens_pos_buf: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice); defer tokens_pos_buf.deinit(); const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(all_tokens.len)})); - var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice, .replicated); + var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice); defer actual_seq_len_buf.deinit(); const params = self.compiled_model.params; var attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = switch (params.attention_metadata) { .metal_fa => .{ .metal_fa = .{ .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32) } }, - else => try params.attention_metadata.initBuffer(self.io, self.platform, params.shardings.model), + else => try params.attention_metadata.initBuffer(self.io, self.platform), }; defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); @@ -161,18 +161,18 @@ pub const Session = struct { var decoder = try self.tokenizer.decoder(); defer decoder.deinit(); - var current_token_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, self.generated_token_slice, .replicated); + var current_token_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, self.generated_token_slice); defer current_token_buffer.deinit(); const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{0})); - var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice, .replicated); + var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice); defer actual_seq_len_buf.deinit(); const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); defer self.allocator.free(out_tokens_buffer); const params = self.compiled_model.params; - var attention_metadata_buffers = try params.attention_metadata.initBuffer(self.io, self.platform, params.shardings.model); + var attention_metadata_buffers = try params.attention_metadata.initBuffer(self.io, self.platform); defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); generation: while (true) { @@ -194,7 +194,7 @@ pub const Session = struct { if (all_tokens.items.len >= self.seqlen) break :generation; const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(all_tokens.items.len)})); - var token_pos_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice, .replicated); + var token_pos_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice); defer token_pos_buffer.deinit(); var conv_cache_index_buffer: zml.Buffer = try .scalar(self.io, self.platform, 0, .u32); diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig index cbc33811..2da991ec 100644 --- a/examples/llm/models/lfm2_tests.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -40,16 +40,16 @@ pub fn main(init: std.process.Init) !void { defer repo.close(io); var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); defer registry.deinit(); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + const meshes: common.Meshes = try .init(platform); + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); defer store.deinit(); var repo_model = try lfm2.LoadedModel.init(allocator, io, repo, store.view(), .{}); defer repo_model.deinit(allocator); var progress = std.Progress.start(io, .{ .root_name = args.model }); - const shardings: common.Shardings = try .init(platform); - var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress, shardings); + var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress); defer repo_model.unloadBuffers(&model_buffers, allocator); const backend = args.backend orelse zml.attention.Backend.auto(platform); @@ -58,11 +58,11 @@ pub fn main(init: std.process.Init) !void { repo_model.parsed_config.value, repo_model.parsed_config.value.max_position_embeddings, backend, - shardings, + meshes, ); progress.end(); - try run(allocator, io, platform, args.activations, repo_model.parsed_config.value, repo_model.inner, &model_buffers, params.attention_metadata, params.attention_parameters); + try run(allocator, io, platform, args.activations, repo_model.parsed_config.value, repo_model.inner, &model_buffers, params.attention_metadata, params.attention_parameters, meshes); } pub fn run( @@ -75,11 +75,12 @@ pub fn run( model_buffers: *lfm2.Buffers, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, + meshes: common.Meshes, ) !void { var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, activations_path); defer registry.deinit(); - var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); defer activation_store.deinit(); var ctx = TestContext{ @@ -89,7 +90,7 @@ pub fn run( .activations_store = &activation_store, .attention_metadata = attention_metadata, .attention_parameters = attention_parameters, - .sharding = platform.replicated_sharding, + .mesh = platform.replicated_mesh, }; try ctx.testLayer("embed_tokens", .{ .batch, .seq }, mdl.embed_tokens, model_buffers.embed_tokens, .{}); @@ -134,7 +135,7 @@ const TestContext = struct { activations_store: *zml.io.TensorStore, attention_metadata: zml.attention.Metadata, attention_parameters: zml.attention.Parameters, - sharding: zml.Sharding, + mesh: *const zml.Mesh, fn testLayerPrint(self: *TestContext, comptime name_fmt: []const u8, name_args: anytype, tagz: anytype, layer: anytype, layer_buffers: anytype, opts: zml.testing.CompareOpts) !void { const name = try std.fmt.allocPrint(self.allocator, name_fmt, name_args); @@ -147,19 +148,19 @@ const TestContext = struct { const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); defer self.allocator.free(in_key); - var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key, self.sharding); + var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); defer in_buffer.deinit(); const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(tagz); const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); defer self.allocator.free(out_key); - var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key, self.sharding); + var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); defer out_buffer_expected.deinit(); const exe = if (comptime @TypeOf(layer) == model.TokenEmbedding) - try self.platform.compileFn(self.allocator, self.io, model.TokenEmbedding.forward, .{.{ .embedding = layer, .tokens = in_tensor }}, .{ .shardings = &.{self.sharding} }) + try self.platform.compileFn(self.allocator, self.io, model.TokenEmbedding.forward, .{.{ .embedding = layer, .tokens = in_tensor }}, .{ .meshes = &.{self.mesh} }) else - try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor }, .{ .shardings = &.{self.sharding} }); + try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor }, .{ .meshes = &.{self.mesh} }); defer exe.deinit(); var args = try exe.args(self.allocator); @@ -185,42 +186,42 @@ const TestContext = struct { const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); defer self.allocator.free(in_key); - var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key, self.sharding); + var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); defer in_buffer.deinit(); const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(.{ .batch, .seq, .d }); const cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache", .{name}); defer self.allocator.free(cache_key); - var cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_key, self.sharding); + var cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_key); defer cache_buffer.deinit(); const cache_tensor = zml.Tensor.fromShape(cache_buffer.shape()).withTags(.{ .layer, .batch, .seq, .d }); const cache_pos_key = try std.fmt.allocPrint(self.allocator, "{s}.cache_position", .{name}); defer self.allocator.free(cache_pos_key); - var cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key, self.sharding); + var cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key); defer cache_pos_buffer.deinit(); const cache_pos_tensor = zml.Tensor.fromShape(cache_pos_buffer.shape()).withTags(.{.batch}); const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); defer self.allocator.free(out_key); - var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key, self.sharding); + var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); defer out_buffer_expected.deinit(); const actual_seq_len_tensor: zml.Tensor = .init(.{}, .u32); const actual_seq_len: u32 = @intCast(in_tensor.dim(.seq)); const cache_index_tensor: zml.Tensor = .init(.{}, .u32); - const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, actual_seq_len_tensor, model.ConvCache{ .state = cache_tensor }, cache_index_tensor, model.ConvParameters{ .is_prefill = false } }, .{ .shardings = &.{self.sharding} }); + const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, actual_seq_len_tensor, model.ConvCache{ .state = cache_tensor }, cache_index_tensor, model.ConvParameters{ .is_prefill = false } }, .{ .meshes = &.{self.mesh} }); defer exe.deinit(); const conv_cache: zml.Bufferized(model.ConvCache) = .{ .state = cache_buffer }; const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{actual_seq_len})); - var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice, self.sharding); + var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice); defer actual_seq_len_buf.deinit(); const cache_index_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(cache_ix)})); - var cache_index_buf: zml.Buffer = try .fromSlice(self.io, self.platform, cache_index_slice, self.sharding); + var cache_index_buf: zml.Buffer = try .fromSlice(self.io, self.platform, cache_index_slice); defer cache_index_buf.deinit(); var runner = try exe.runner(self.allocator); @@ -243,41 +244,41 @@ const TestContext = struct { const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); defer self.allocator.free(in_key); - var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key, self.sharding); + var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); defer in_buffer.deinit(); const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(.{ .batch, .seq, .d }); const key_cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache.key", .{name}); defer self.allocator.free(key_cache_key); - var key_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, key_cache_key, self.sharding); + var key_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, key_cache_key); defer key_cache_buffer.deinit(); const key_cache_tensor = zml.Tensor.fromShape(key_cache_buffer.shape()).withTags(.{ .layer, .batch, .h, .k, .hd }); const value_cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache.value", .{name}); defer self.allocator.free(value_cache_key); - var value_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, value_cache_key, self.sharding); + var value_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, value_cache_key); defer value_cache_buffer.deinit(); const value_cache_tensor = zml.Tensor.fromShape(value_cache_buffer.shape()).withTags(.{ .layer, .batch, .h, .k, .hd }); const cache_pos_key = try std.fmt.allocPrint(self.allocator, "{s}.cache_position", .{name}); defer self.allocator.free(cache_pos_key); - var cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key, self.sharding); + var cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key); defer cache_pos_buffer.deinit(); const cache_pos_tensor = zml.Tensor.fromShape(cache_pos_buffer.shape()).withTags(.{.batch}); const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); defer self.allocator.free(out_key); - var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key, self.sharding); + var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); defer out_buffer_expected.deinit(); const cache_index_tensor: zml.Tensor = .init(.{}, .u32); - const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, model.KvCache{ .k = key_cache_tensor, .v = value_cache_tensor }, cache_index_tensor, self.attention_metadata, self.attention_parameters }, .{ .shardings = &.{self.sharding} }); + const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, model.KvCache{ .k = key_cache_tensor, .v = value_cache_tensor }, cache_index_tensor, self.attention_metadata, self.attention_parameters }, .{ .meshes = &.{self.mesh} }); defer exe.deinit(); const kv_cache: zml.Bufferized(model.KvCache) = .{ .k = key_cache_buffer, .v = value_cache_buffer }; - var attention_metadata_buffers = try self.attention_metadata.initBuffer(self.io, self.platform, self.sharding); + var attention_metadata_buffers = try self.attention_metadata.initBuffer(self.io, self.platform); defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); var cache_index_buf: zml.Buffer = try .scalar(self.io, self.platform, cache_ix, .u32); @@ -297,7 +298,7 @@ const TestContext = struct { } }; -fn loadBufferFromStore(allocator: std.mem.Allocator, io: anytype, platform: *zml.Platform, store: *zml.io.TensorStore, key: []const u8, sharding: zml.Sharding) !zml.Buffer { +fn loadBufferFromStore(allocator: std.mem.Allocator, io: anytype, platform: *zml.Platform, store: *zml.io.TensorStore, key: []const u8) !zml.Buffer { const shape = store.view().getShape(key) orelse return error.NotFound; const host_bytes = try allocator.alloc(u8, shape.byteSize()); @@ -309,5 +310,5 @@ fn loadBufferFromStore(allocator: std.mem.Allocator, io: anytype, platform: *zml _ = try reader.interface.readSliceAll(host_bytes); - return zml.Buffer.fromBytes(io, platform, shape, sharding, host_bytes); + return zml.Buffer.fromBytes(io, platform, shape, host_bytes); } diff --git a/examples/llm/models/llama/inference.zig b/examples/llm/models/llama/inference.zig index 4eafdcea..331ee0dc 100644 --- a/examples/llm/models/llama/inference.zig +++ b/examples/llm/models/llama/inference.zig @@ -18,9 +18,9 @@ pub const CompilationParameters = struct { prefill_attention_parameters: zml.attention.Parameters, decode_attention_parameters: zml.attention.Parameters, seqlen: usize, - shardings: common.Shardings, + meshes: common.Meshes, - pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: zml.attention.Backend, shardings: common.Shardings) CompilationParameters { + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: zml.attention.Backend, meshes: common.Meshes) CompilationParameters { const head_dim = config.head_dim orelse @divExact(config.hidden_size, config.num_attention_heads); return .{ @@ -32,11 +32,11 @@ pub const CompilationParameters = struct { .k = seqlen, .h = config.num_key_value_heads, .hd = head_dim, - }, mdl.model.embed_tokens.weight.dtype())), + }, mdl.model.embed_tokens.weight.dtype()), meshes.model), .rng = .init(), .attention_metadata = switch (backend) { .attnd => .{ .attnd = .init() }, - else => .init(.fromBackend(backend, @intCast(seqlen), @intCast(config.num_attention_heads))), + else => .init(.fromBackend(backend, @intCast(seqlen), @intCast(config.num_attention_heads)), meshes.model), }, .prefill_attention_parameters = switch (backend) { .attnd => .{ .attnd = .init(.{ @@ -59,7 +59,7 @@ pub const CompilationParameters = struct { else => .init(.fromBackend(backend)), }, .seqlen = seqlen, - .shardings = shardings, + .meshes = meshes, }; } }; @@ -211,7 +211,7 @@ fn compileKernel( defer phase.logCompileDone(log, "forward", io, from); return KernelExe.compile(allocator, io, platform, .{ - .shardings = ¶meters.shardings.all(), + .meshes = ¶meters.meshes.all(), .program_name = phase.programName("llama", "forward"), }, .{.{ .weights = llama_model, diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index b86c6d6b..ae988b97 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -76,7 +76,6 @@ pub const LoadedModel = struct { platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, - shardings: common.Shardings, ) !Buffers { progress.increaseEstimatedTotalItems(store.view().count()); const now: std.Io.Timestamp = .now(io, .awake); @@ -91,8 +90,7 @@ pub const LoadedModel = struct { }); defer loader.deinit(); - const all_shardings = shardings.all(); - try loader.load(io, Model, &self.inner, &buffers, store, &all_shardings, .{ .progress = progress }); + try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); try loader.await(io); const took = now.untilNow(io, .awake); @@ -114,11 +112,11 @@ pub const LoadedModel = struct { io: std.Io, platform: *const zml.Platform, backend: zml.attention.Backend, - shardings: common.Shardings, + meshes: common.Meshes, seqlen: usize, progress: *std.Progress.Node, ) !inference.CompiledModel { - const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), backend, shardings); + const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), backend, meshes); return inference.CompiledModel.init(allocator, io, platform, self, self.inner, params, progress); } @@ -144,6 +142,7 @@ pub const Model = struct { const lm_head: ?zml.nn.Linear = if (store.withPrefix("lm_head").maybeCreateTensor( "weight", .{ .dout, .d }, + .model, .{ .dout = .model, .d = .replicated }, )) |weight| .init(weight, null, .d) @@ -168,7 +167,6 @@ pub const Model = struct { io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, - shardings: []const zml.Sharding, progress: *std.Progress.Node, ) !zml.Bufferized(Model) { progress.increaseEstimatedTotalItems(store.view().count()); @@ -184,7 +182,7 @@ pub const Model = struct { }); defer loader.deinit(); - loader.load(io, Model, self, &buffers, store, shardings); + loader.load(io, Model, self, &buffers, store); try loader.await(io); const took = now.untilNow(io, .awake); @@ -256,10 +254,11 @@ const Llama = struct { .embed_tokens = .{ .weight = store.createTensor( "embed_tokens.weight", .{ .voc, .d }, + .model, .{ .voc = .replicated, .d = .model }, ) }, .norm = .{ - .weight = store.withPrefix("norm").createTensor("weight", .{.d}, .{ .d = .replicated }), + .weight = store.withPrefix("norm").createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = config.rms_norm_eps, }, .layers = layers, @@ -329,7 +328,7 @@ pub const EmbedTokens = struct { const tokens = input.tokens.withPartialTags(.{.s}); return .{ .hidden = input.embedding.embed_tokens.forward(tokens) .withPartialTags(.{.d}) - .withPartitioning(.{ .d = .replicated }) }; + .withPartitioning(.model, .{ .d = .replicated }) }; } }; @@ -426,7 +425,7 @@ pub const TransformerLayer = struct { stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {f}", .{x0}); // Keep the residual stream replicated to avoid repeated gathers before q/k/v. - const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); const x0_normalized = self.input_layernorm.forward(x0_replicated); const delta0, const updated_kv_cache = self.self_attn.forward( x0_normalized, @@ -438,13 +437,13 @@ pub const TransformerLayer = struct { ); // Fully Connected - const x1 = x0_replicated.add(delta0).withPartitioning(.{ .d = .replicated }); + const x1 = x0_replicated.add(delta0).withPartitioning(.model, .{ .d = .replicated }); const x1_normalized = self.post_attention_layernorm.forward(x1); const x2 = self.mlp.forward(x1_normalized) .rename(.{ .dout = .d }) - .withPartitioning(.{ .d = .replicated }) + .withPartitioning(.model, .{ .d = .replicated }) .add(x1) - .withPartitioning(.{ .d = .replicated }); + .withPartitioning(.model, .{ .d = .replicated }); return .{ .hidden = x2.reuseBuffer(x0), .kv_cache = updated_kv_cache }; } @@ -456,7 +455,7 @@ const RmsNorm = struct { pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { return .{ - .weight = store.createTensor("weight", .{.d}, .{ .d = .replicated }), + .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps, }; } @@ -480,9 +479,9 @@ const Mlp = struct { pub fn init(store: zml.io.TensorStore.View) Mlp { return .{ - .up_proj = .init(store.createTensor("up_proj.weight", .{ .dout, .d }, .{ .dout = .model }), null, .d), - .gate_proj = .init(store.createTensor("gate_proj.weight", .{ .dout, .d }, .{ .dout = .model }), null, .d), - .down_proj = .init(store.createTensor("down_proj.weight", .{ .dout, .d }, .{ .d = .model }), null, .d), + .up_proj = .init(store.createTensor("up_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), + .gate_proj = .init(store.createTensor("gate_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), + .down_proj = .init(store.createTensor("down_proj.weight", .{ .dout, .d }, .model, .{ .d = .model }), null, .d), }; } @@ -515,10 +514,10 @@ const SelfAttn = struct { var rope_scaling = config.rope_scaling; rope_scaling.setRopeTheta(config.rope_theta); return .{ - .q_proj = .init(store.createTensor("q_proj.weight", .{ .dout, .d }, .{ .dout = .model }), null, .d), - .k_proj = .init(store.createTensor("k_proj.weight", .{ .dout, .d }, .{ .dout = .model }), null, .d), - .v_proj = .init(store.createTensor("v_proj.weight", .{ .dout, .d }, .{ .dout = .model }), null, .d), - .o_proj = .init(store.createTensor("o_proj.weight", .{ .dout, .d }, .{ .d = .model }), null, .d), + .q_proj = .init(store.createTensor("q_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), + .k_proj = .init(store.createTensor("k_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), + .v_proj = .init(store.createTensor("v_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), + .o_proj = .init(store.createTensor("o_proj.weight", .{ .dout, .d }, .model, .{ .d = .model }), null, .d), // TODO(Corentin): fix that .q_norm = null, .k_norm = null, @@ -555,7 +554,7 @@ const SelfAttn = struct { // Make hidden state replicated once and reuse it across q/k/v projections. // This avoids paying gather-style collectives independently for each projection. - const x_qkv = x.withPartitioning(.{ .d = .replicated }); + const x_qkv = x.withPartitioning(.model, .{ .d = .replicated }); var q = self.q_proj.forward(x_qkv, x_qkv.dtype()).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); var k = self.k_proj.forward(x_qkv, x_qkv.dtype()).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); @@ -605,7 +604,7 @@ const SelfAttn = struct { const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); const delta = self.o_proj.forward(attn, attn.dtype()) .rename(.{ .dout = .d }) - .withPartitioning(.{ .d = .replicated }); + .withPartitioning(.model, .{ .d = .replicated }); return .{ delta, new_kv_cache }; } }; @@ -616,8 +615,8 @@ pub const KvCache = struct { pub const Buffer = zml.Bufferized(KvCache); - pub fn init(kv_shape: zml.Shape) KvCache { - const sharded_shape = kv_shape.withPartitioning(.{ .h = .model }); + pub fn init(kv_shape: zml.Shape, mesh: *const zml.Mesh) KvCache { + const sharded_shape = kv_shape.withPartitioning(mesh, .{ .h = .model }); return .{ .k = .fromShape(sharded_shape), @@ -625,10 +624,10 @@ pub const KvCache = struct { }; } - pub fn initBuffer(kv: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !Buffer { + pub fn initBuffer(kv: KvCache, io: std.Io, platform: *const zml.Platform) !Buffer { return .{ - .k = try zml.Buffer.uninitialized(io, platform, kv.k.shape(), sharding, .{}), - .v = try zml.Buffer.uninitialized(io, platform, kv.v.shape(), sharding, .{}), + .k = try zml.Buffer.uninitialized(io, platform, kv.k.shape(), .{}), + .v = try zml.Buffer.uninitialized(io, platform, kv.v.shape(), .{}), }; } diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig index 2269464f..6a04cf11 100644 --- a/examples/llm/models/llama/session.zig +++ b/examples/llm/models/llama/session.zig @@ -29,8 +29,7 @@ pub const Session = struct { compiled_model: *inference.CompiledModel, model_buffers: *model.Buffers, ) !Session { - const shardings = &compiled_model.params.shardings; - var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform, shardings.model); + var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform); errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); const token_index_buffers = try allocator.alloc(zml.Buffer, compiled_model.params.seqlen); @@ -49,7 +48,7 @@ pub const Session = struct { const conversation_id: u64 = @bitCast(std.Io.Clock.now(.real, io).toMicroseconds()); const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); - var rng_buffers = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed); + var rng_buffers = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); var prefill = try inference.KernelRunner.init(allocator, &compiled_model.prefill, model_buffers); @@ -138,7 +137,7 @@ pub const Session = struct { @memset(prefill_tokens_slice.items(u32), 0); @memcpy(prefill_tokens_slice.items(u32)[0..all_tokens.len], all_tokens); - var prefill_tokens_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, prefill_tokens_slice, .replicated); + var prefill_tokens_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, prefill_tokens_slice); defer prefill_tokens_buffer.deinit(); const params = self.compiled_model.params; @@ -149,7 +148,7 @@ pub const Session = struct { .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32), } }, .metal_fa => .{ .metal_fa = .{ .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32) } }, - .vanilla, .cuda_fa2, .cuda_fa3, .nki => try params.attention_metadata.initBuffer(self.io, self.platform, params.shardings.model), + .vanilla, .cuda_fa2, .cuda_fa3, .nki => try params.attention_metadata.initBuffer(self.io, self.platform), }; defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); @@ -174,7 +173,7 @@ pub const Session = struct { defer self.allocator.free(decoder_out_buffer); var last_token_id: u32 = self.last_generated_token; - var current_token_buffer: zml.Buffer = try .fromBytes(self.io, self.platform, .init(.{ .s = 1 }, .u32), .replicated, @ptrCast(&last_token_id)); + var current_token_buffer: zml.Buffer = try .fromBytes(self.io, self.platform, .init(.{ .s = 1 }, .u32), @ptrCast(&last_token_id)); defer current_token_buffer.deinit(); const params = self.compiled_model.params; @@ -184,7 +183,7 @@ pub const Session = struct { .layer_id = try .scalar(self.io, self.platform, 0, .u16), .num_tokens = try .scalar(self.io, self.platform, 1, .u32), } }, - .vanilla, .cuda_fa2, .cuda_fa3, .nki, .metal_fa => try params.attention_metadata.initBuffer(self.io, self.platform, params.shardings.model), + .vanilla, .cuda_fa2, .cuda_fa3, .nki, .metal_fa => try params.attention_metadata.initBuffer(self.io, self.platform), }; defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); diff --git a/examples/llm/models/llama_tests.zig b/examples/llm/models/llama_tests.zig index 98798887..1b5894f1 100644 --- a/examples/llm/models/llama_tests.zig +++ b/examples/llm/models/llama_tests.zig @@ -38,20 +38,20 @@ pub fn main(init: std.process.Init) !void { defer repo.close(io); var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); defer registry.deinit(); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + const meshes: common.Meshes = try .init(platform); + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); defer store.deinit(); var repo_model = try llama.LoadedModel.init(allocator, io, repo, store.view(), .{}); defer repo_model.deinit(allocator); var progress = std.Progress.start(io, .{ .root_name = args.model }); - const shardings: common.Shardings = try .init(platform); - var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress, shardings); + var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress); defer repo_model.unloadBuffers(&model_buffers, allocator); progress.end(); - try run(allocator, io, platform, args.activations, repo_model.inner, &model_buffers, platform.replicated_sharding); + try run(allocator, io, platform, args.activations, repo_model.inner, &model_buffers, &meshes.all()); } fn run( @@ -61,28 +61,28 @@ fn run( activations_path: []const u8, mdl: model.Model, model_buffers: *model.Buffers, - sharding: zml.Sharding, + meshes: []const *const zml.Mesh, ) !void { var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, activations_path); defer registry.deinit(); - var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, meshes); defer activation_store.deinit(); - try testLayer(allocator, io, platform, activation_store.view(), "embed_tokens", mdl.model.embed_tokens, model_buffers.model.embed_tokens, sharding, .{ .absolute_tolerance = 1e-3 }); + try testLayer(allocator, io, platform, activation_store.view(), "embed_tokens", mdl.model.embed_tokens, model_buffers.model.embed_tokens, meshes, .{ .absolute_tolerance = 1e-3 }); if (mdl.model.layers.len == 0) return; const layer = mdl.model.layers[0]; const layer_buffers = model_buffers.model.layers[0]; - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.v_proj", layer.self_attn.v_proj, layer_buffers.self_attn.v_proj, sharding, .{ .absolute_tolerance = 1e-2 }); - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.q_proj", layer.self_attn.q_proj, layer_buffers.self_attn.q_proj, sharding, .{ .absolute_tolerance = 2e-2 }); - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.k_proj", layer.self_attn.k_proj, layer_buffers.self_attn.k_proj, sharding, .{ .absolute_tolerance = 2e-2 }); - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.o_proj", layer.self_attn.o_proj, layer_buffers.self_attn.o_proj, sharding, .{ .absolute_tolerance = 2e-2 }); - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.mlp", layer.mlp, layer_buffers.mlp, sharding, .{ .absolute_tolerance = 1e-2 }); - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.input_layernorm", layer.input_layernorm, layer_buffers.input_layernorm, sharding, .{ .absolute_tolerance = 1e-2 }); - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.post_attention_layernorm", layer.post_attention_layernorm, layer_buffers.post_attention_layernorm, sharding, .{ .absolute_tolerance = 1e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.v_proj", layer.self_attn.v_proj, layer_buffers.self_attn.v_proj, meshes, .{ .absolute_tolerance = 1e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.q_proj", layer.self_attn.q_proj, layer_buffers.self_attn.q_proj, meshes, .{ .absolute_tolerance = 2e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.k_proj", layer.self_attn.k_proj, layer_buffers.self_attn.k_proj, meshes, .{ .absolute_tolerance = 2e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.o_proj", layer.self_attn.o_proj, layer_buffers.self_attn.o_proj, meshes, .{ .absolute_tolerance = 2e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.mlp", layer.mlp, layer_buffers.mlp, meshes, .{ .absolute_tolerance = 1e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.input_layernorm", layer.input_layernorm, layer_buffers.input_layernorm, meshes, .{ .absolute_tolerance = 1e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.post_attention_layernorm", layer.post_attention_layernorm, layer_buffers.post_attention_layernorm, meshes, .{ .absolute_tolerance = 1e-2 }); } fn testLayer( @@ -93,19 +93,19 @@ fn testLayer( name: []const u8, layer: anytype, layer_weights: zml.Bufferized(@TypeOf(layer)), - sharding: zml.Sharding, + meshes: []const *const zml.Mesh, opts: zml.testing.CompareOpts, ) !void { const in_key = try std.fmt.allocPrint(allocator, "{s}.in", .{name}); defer allocator.free(in_key); const in_shape = activation_store.getShape(in_key) orelse return error.NotFound; - var in_buffer = try loadBufferFromStore(allocator, io, platform, activation_store, in_key, sharding); + var in_buffer = try loadBufferFromStore(allocator, io, platform, activation_store, in_key); defer in_buffer.deinit(); const in_tensor = zml.Tensor.fromShape(in_shape); const out_key = try std.fmt.allocPrint(allocator, "{s}.out", .{name}); defer allocator.free(out_key); - var out_buffer_expected = try loadBufferFromStore(allocator, io, platform, activation_store, out_key, sharding); + var out_buffer_expected = try loadBufferFromStore(allocator, io, platform, activation_store, out_key); defer out_buffer_expected.deinit(); // `zml.nn.Linear.forward` takes an explicit output dtype; every other layer here @@ -116,7 +116,7 @@ fn testLayer( return if (Layer == zml.nn.Linear) l.forward(x, x.dtype()) else Layer.forward(l, x); } }; - const exe = try platform.compileFn(allocator, io, Call.forward, .{ layer, in_tensor }, .{ .shardings = &.{sharding} }); + const exe = try platform.compileFn(allocator, io, Call.forward, .{ layer, in_tensor }, .{ .meshes = meshes }); defer exe.deinit(); var args = try exe.args(allocator); @@ -133,7 +133,7 @@ fn testLayer( try zml.testing.expectClose(io, out_result, out_buffer_expected, opts); } -fn loadBufferFromStore(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: zml.io.TensorStore.View, key: []const u8, sharding: zml.Sharding) !zml.Buffer { +fn loadBufferFromStore(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: zml.io.TensorStore.View, key: []const u8) !zml.Buffer { const shape = store.getShape(key) orelse return error.NotFound; const host_bytes = try allocator.alloc(u8, shape.byteSize()); @@ -145,5 +145,5 @@ fn loadBufferFromStore(allocator: std.mem.Allocator, io: std.Io, platform: *cons _ = try reader.interface.readSliceAll(host_bytes); - return zml.Buffer.fromBytes(io, platform, shape, sharding, host_bytes); + return zml.Buffer.fromBytes(io, platform, shape, host_bytes); } diff --git a/examples/llm/models/qwen3_5/inference.zig b/examples/llm/models/qwen3_5/inference.zig index 33a41086..6bfb71c5 100644 --- a/examples/llm/models/qwen3_5/inference.zig +++ b/examples/llm/models/qwen3_5/inference.zig @@ -15,18 +15,18 @@ pub const CompilationParameters = struct { kv_cache: model.KvCache, rng: zml.Tensor.Rng, seqlen: u32, - shardings: common.Shardings, + meshes: common.Meshes, - pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, shardings: common.Shardings) CompilationParameters { + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, meshes: common.Meshes) CompilationParameters { const dtype = mdl.text_model.embed_tokens.weight.dtype(); return .{ .prefill_tokens = .init(.{ .b = 1, .s = seqlen }, .u32), .decode_tokens = .init(.{ .b = 1, .s = 1 }, .u32), .token_index = .init(.{}, .u32), - .kv_cache = .init(config, 1, seqlen, dtype, .f32, shardings.model), + .kv_cache = .init(config, 1, seqlen, dtype, .f32, meshes.model), .rng = .init(), .seqlen = seqlen, - .shardings = shardings, + .meshes = meshes, }; } }; @@ -244,7 +244,7 @@ fn compileEmbed(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.P defer node.end(); const from: std.Io.Timestamp = .now(io, .awake); defer phase.logCompileDone(log, "embed tokens", io, from); - return zml.FnExe(model.EmbedTokens.forward).compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("qwen3_5", "embed_tokens") }, .{.{ + return zml.FnExe(model.EmbedTokens.forward).compile(allocator, io, platform, .{ .meshes = ¶meters.meshes.all(), .program_name = phase.programName("qwen3_5", "embed_tokens") }, .{.{ .embedding = .{ .embed_tokens = mdl.text_model.embed_tokens }, .tokens = zml.Tensor.init(.{ .b = 1, .s = seqlen }, .u32), }}); @@ -256,9 +256,9 @@ fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *con defer node.end(); const from: std.Io.Timestamp = .now(io, .awake); defer phase.logCompileDone(log, "full attention layer", io, from); - return zml.FnExe(model.TransformerLayer.forwardSelfAttn).compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("qwen3_5", "full_attention_layer") }, .{.{ + return zml.FnExe(model.TransformerLayer.forwardSelfAttn).compile(allocator, io, platform, .{ .meshes = ¶meters.meshes.all(), .program_name = phase.programName("qwen3_5", "full_attention_layer") }, .{.{ .layer = mdl.text_model.layers[layer_index], - .hidden = hiddenTensor(mdl, seqlen), + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), .token_index = parameters.token_index, .cache = .{ .k = parameters.kv_cache.self_attn.k, @@ -274,9 +274,9 @@ fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *c defer node.end(); const from: std.Io.Timestamp = .now(io, .awake); defer phase.logCompileDone(log, "linear attention layer", io, from); - return zml.FnExe(model.TransformerLayer.forwardLinearAttn).compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("qwen3_5", "linear_attention_layer") }, .{.{ + return zml.FnExe(model.TransformerLayer.forwardLinearAttn).compile(allocator, io, platform, .{ .meshes = ¶meters.meshes.all(), .program_name = phase.programName("qwen3_5", "linear_attention_layer") }, .{.{ .layer = mdl.text_model.layers[layer_index], - .hidden = hiddenTensor(mdl, seqlen), + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), .active_length = zml.Tensor.init(.{}, .u32), .cache = .{ .conv_state = parameters.kv_cache.gated_delta_net.conv_state, @@ -292,19 +292,19 @@ fn compileSample(allocator: std.mem.Allocator, io: std.Io, platform: *const zml. defer node.end(); const from: std.Io.Timestamp = .now(io, .awake); defer phase.logCompileDone(log, "sampler", io, from); - return zml.FnExe(model.Sampler.sampleTokens).compile(allocator, io, platform, .{ .shardings = ¶meters.shardings.all(), .program_name = phase.programName("qwen3_5", "sampler") }, .{.{ + return zml.FnExe(model.Sampler.sampleTokens).compile(allocator, io, platform, .{ .meshes = ¶meters.meshes.all(), .program_name = phase.programName("qwen3_5", "sampler") }, .{.{ .sampler = mdl.sampler(), - .hidden = hiddenTensor(mdl, seqlen), + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), .rng = parameters.rng, .token_index = parameters.token_index, }}); } -fn hiddenTensor(mdl: model.Model, seqlen: usize) zml.Tensor { +fn hiddenTensor(mdl: model.Model, seqlen: usize, mesh: *const zml.Mesh) zml.Tensor { return .fromShape(zml.Shape.init( .{ .b = 1, .s = seqlen, .d = mdl.config.text_config.hidden_size }, mdl.text_model.embed_tokens.weight.dtype(), - ).withPartitioning(.{ .b = .replicated, .s = .replicated, .d = .replicated })); + ).withPartitioning(mesh, .{ .b = .replicated, .s = .replicated, .d = .replicated })); } fn findFirstLayerIndex(layer_types: []const model.LayerType, target: model.LayerType) ?usize { diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 5a2a41ac..14edd022 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -81,7 +81,6 @@ pub const LoadedModel = struct { platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, - shardings: common.Shardings, ) !Buffers { progress.increaseEstimatedTotalItems(store.view().count()); const now: std.Io.Timestamp = .now(io, .awake); @@ -96,8 +95,7 @@ pub const LoadedModel = struct { }); defer loader.deinit(); - const all_shardings = shardings.all(); - try loader.load(io, Model, &self.inner, &buffers, store, &all_shardings, .{ .progress = progress }); + try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); try loader.await(io); const took = now.untilNow(io, .awake); @@ -120,12 +118,12 @@ pub const LoadedModel = struct { io: std.Io, platform: *const zml.Platform, backend: zml.attention.Backend, - shardings: common.Shardings, + meshes: common.Meshes, seqlen: usize, progress: *std.Progress.Node, ) !inference.CompiledModel { _ = backend; - const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), shardings); + const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), meshes); return inference.CompiledModel.init(allocator, io, platform, self, self.inner, params, progress); } }; @@ -161,7 +159,7 @@ pub const Model = struct { return .{ .text_model = try .init(allocator, store.withPrefix("model.language_model"), config), .lm_head = .init( - store.withPrefix(lm_head_prefix).createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), + store.withPrefix(lm_head_prefix).createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), null, .d, ), @@ -180,7 +178,7 @@ pub const Model = struct { io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, - shardings: []const zml.Sharding, + meshes: []const *const zml.Mesh, progress: *std.Progress.Node, ) !zml.Bufferized(Model) { progress.increaseEstimatedTotalItems(store.view().count()); @@ -196,7 +194,7 @@ pub const Model = struct { }); defer loader.deinit(); - loader.load(io, Model, self, &buffers, store, shardings); + loader.load(io, Model, self, &buffers, store, meshes); try loader.await(io); const took = now.untilNow(io, .awake); @@ -286,7 +284,7 @@ pub const EmbedTokens = struct { const tokens = input.tokens.withPartialTags(.{.s}); return .{ .hidden = input.embedding.embed_tokens.forward(tokens) .withPartialTags(.{.d}) - .withPartitioning(.{ .d = .replicated }) }; + .withPartitioning(.model, .{ .d = .replicated }) }; } }; @@ -308,7 +306,7 @@ pub const TextModel = struct { } return .{ - .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight", .{ .voc, .d }, .{ .voc = .replicated, .d = .model }) }, + .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight", .{ .voc, .d }, .model, .{ .voc = .replicated, .d = .model }) }, .layers = layers, .norm = RmsNorm.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), }; @@ -414,7 +412,7 @@ pub const TransformerLayer = struct { active_length: zml.Tensor, kv_cache: KvCache.LayerView, ) struct { zml.Tensor, KvCache } { - const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); var attention_output: zml.Tensor = undefined; @@ -440,18 +438,18 @@ pub const TransformerLayer = struct { }, } - const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); - const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.{ .d = .replicated }); + const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.model, .{ .d = .replicated }); - return .{ mlp_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), updated_kv_cache }; + return .{ mlp_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), updated_kv_cache }; } pub fn forwardSelfAttn(input: SelfAttnInput) SelfAttnOutput { const self = input.layer; const x0 = input.hidden; - const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); const self_attn = switch (self.attn) { @@ -460,12 +458,12 @@ pub const TransformerLayer = struct { }; const attention_output, const updated_kv_cache = self_attn.forward(normalized_x0, input.token_index, input.cache); - const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); - const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.{ .d = .replicated }); + const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.model, .{ .d = .replicated }); return .{ - .hidden = mlp_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), + .hidden = mlp_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), .cache = updated_kv_cache, }; } @@ -473,7 +471,7 @@ pub const TransformerLayer = struct { pub fn forwardLinearAttn(input: LinearAttnInput) LinearAttnOutput { const self = input.layer; const x0 = input.hidden; - const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); const linear_attn = switch (self.attn) { @@ -482,12 +480,12 @@ pub const TransformerLayer = struct { }; const attention_output, const updated_kv_cache = linear_attn.forward(normalized_x0, input.cache, input.active_length); - const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); - const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.{ .d = .replicated }); + const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.model, .{ .d = .replicated }); return .{ - .hidden = mlp_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), + .hidden = mlp_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), .cache = updated_kv_cache, }; } @@ -501,18 +499,18 @@ pub const Mlp = struct { pub fn init(store: zml.io.TensorStore.View) Mlp { return .{ .up_proj = .init( - store.withPrefix("up_proj").createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), - store.withPrefix("up_proj").maybeCreateTensor("bias", .{.dout}, .{ .dout = .model }), + store.withPrefix("up_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), + store.withPrefix("up_proj").maybeCreateTensor("bias", .{.dout}, .model, .{ .dout = .model }), .d, ), .gate_proj = .init( - store.withPrefix("gate_proj").createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), - store.withPrefix("gate_proj").maybeCreateTensor("bias", .{.dout}, .{ .dout = .model }), + store.withPrefix("gate_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), + store.withPrefix("gate_proj").maybeCreateTensor("bias", .{.dout}, .model, .{ .dout = .model }), .d, ), .down_proj = .init( - store.withPrefix("down_proj").createTensor("weight", .{ .d, .dout }, .{ .d = .replicated, .dout = .model }), - store.withPrefix("down_proj").maybeCreateTensor("bias", .{.d}, .{ .d = .replicated }), + store.withPrefix("down_proj").createTensor("weight", .{ .d, .dout }, .model, .{ .d = .replicated, .dout = .model }), + store.withPrefix("down_proj").maybeCreateTensor("bias", .{.d}, .model, .{ .d = .replicated }), .dout, ), }; @@ -549,8 +547,8 @@ pub const SelfAttn = struct { fn initProj(store: zml.io.TensorStore.View, partitions: anytype, bias_partitions: anytype) zml.nn.Linear { return .init( - store.createTensor("weight", .{ .dout, .d }, partitions), - store.maybeCreateTensor("bias", .{.dout}, bias_partitions), + store.createTensor("weight", .{ .dout, .d }, .model, partitions), + store.maybeCreateTensor("bias", .{.dout}, .model, bias_partitions), .d, ); } @@ -594,26 +592,26 @@ pub const SelfAttn = struct { return .{ k, v }; } - fn partitionProjectedKv(kv: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { - return switch (kv_head_sharding) { + fn partitionProjectedKv(kv: zml.Tensor, kv_head_meshe: zml.Sharding.DimMeshe) zml.Tensor { + return switch (kv_head_meshe) { .sharded => |heads| blk: { const sharded_kv = if (heads.factor != 1) kv.stutter1d(kv.axis(.h), heads.factor) else kv; - break :blk sharded_kv.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); + break :blk sharded_kv.withPartitioning(.model, .{ .s = .replicated, .h = .model, .hd = .replicated }); }, - .replicated => kv.withPartitioning(.{ .s = .replicated, .h = .replicated, .hd = .replicated }), + .replicated => kv.withPartitioning(.model, .{ .s = .replicated, .h = .replicated, .hd = .replicated }), }; } - fn partitionCachedKv(tensor: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { + fn partitionCachedKv(tensor: zml.Tensor, kv_head_meshe: zml.Sharding.DimMeshe) zml.Tensor { var cache_tensor = tensor.rename(.{ .s = .k }); - return switch (kv_head_sharding) { + return switch (kv_head_meshe) { .sharded => |heads| blk: { if (heads.factor != 1) { cache_tensor = cache_tensor.stutter1d(cache_tensor.axis(.h), heads.factor); } - break :blk cache_tensor.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); + break :blk cache_tensor.withPartitioning(.model, .{ .k = .replicated, .h = .model, .hd = .replicated }); }, - .replicated => cache_tensor.withPartitioning(.{ .k = .replicated, .h = .replicated, .hd = .replicated }), + .replicated => cache_tensor.withPartitioning(.model, .{ .k = .replicated, .h = .replicated, .hd = .replicated }), }; } @@ -623,18 +621,15 @@ pub const SelfAttn = struct { token_index: zml.Tensor, kv_cache: KvCache.SelfAttnCache, ) struct { zml.Tensor, KvCache.SelfAttnCache } { - const x_qkv = x.withPartitioning(.{ .d = .replicated }); + const tp = zml.Compiler.current().mesh(.model); + const x_qkv = x.withPartitioning(tp, .{ .d = .replicated }); var q, var gate = self.projectQAndGate(x_qkv); var k, var v = self.projectKV(x_qkv); - const kv_head_sharding = zml.Compiler.current().partitioning.shardableDim( - k.shape().withPartitioning(.{ .h = .model }), - .h, - q.dim(.h), - ) catch unreachable; + const kv_head_meshe = tp.shardableDim(k.dim(.h), .model, q.dim(.h)); - k = partitionProjectedKv(k, kv_head_sharding); - v = partitionProjectedKv(v, kv_head_sharding); + k = partitionProjectedKv(k, kv_head_meshe); + v = partitionProjectedKv(v, kv_head_meshe); q = self.q_norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); k = self.k_norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); @@ -646,22 +641,22 @@ pub const SelfAttn = struct { const cos, const sin = self.rotary_embed.getCosAndSin(position_ids, dtype); q = self.rotary_embed.applyRope(q, cos, sin); k = self.rotary_embed.applyRope(k, cos, sin); - k = partitionProjectedKv(k, kv_head_sharding); - v = partitionProjectedKv(v, kv_head_sharding); + k = partitionProjectedKv(k, kv_head_meshe); + v = partitionProjectedKv(v, kv_head_meshe); const new_kv_cache = kv_cache.update(k, v, token_index.convert(.u32)); k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); q = q.rename(.{ .s = .q }); - k = partitionCachedKv(k, kv_head_sharding); - v = partitionCachedKv(v, kv_head_sharding); + k = partitionCachedKv(k, kv_head_meshe); + v = partitionCachedKv(v, kv_head_meshe); const attn_output = zml.attention.attention( q, k, v, token_index, - zml.attention.Metadata.init(.fromBackend(.vanilla, x.dim(.s), self.num_heads)), + zml.attention.Metadata.init(.fromBackend(.vanilla, x.dim(.s), self.num_heads), tp), zml.attention.Parameters.init(.fromBackend(.vanilla)), ).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); @@ -669,7 +664,7 @@ pub const SelfAttn = struct { const projected_output = self.o_proj .forward(gated_output.rename(.{ .d_out_proj = .d }), gated_output.dtype()) .rename(.{ .dout = .d }) - .withPartitioning(.{ .d = .replicated }); + .withPartitioning(tp, .{ .d = .replicated }); return .{ projected_output, new_kv_cache }; } @@ -772,7 +767,7 @@ pub const GatedDeltaNet = struct { conv_kernel_size: i64, fn initProj(store: zml.io.TensorStore.View, partitions: anytype) zml.nn.Linear { - return .init(store.createTensor("weight", .{ .dout, .d }, partitions), null, .d); + return .init(store.createTensor("weight", .{ .dout, .d }, .model, partitions), null, .d); } pub fn init(store: zml.io.TensorStore.View, config: Config) GatedDeltaNet { @@ -784,9 +779,9 @@ pub const GatedDeltaNet = struct { .in_proj_b = initProj(store.withPrefix("in_proj_b"), .{ .dout = .model, .d = .replicated }), .in_proj_a = initProj(store.withPrefix("in_proj_a"), .{ .dout = .model, .d = .replicated }), .out_proj = initProj(store.withPrefix("out_proj"), .{ .dout = .replicated, .d = .model }), - .conv1d_weight = store.withPrefix("conv1d").createTensor("weight", .{ .out, .in, .kernel_size }, .{ .out = .model, .in = .replicated, .kernel_size = .replicated }), - .dt_bias = store.createTensor("dt_bias", .{.vh}, .{ .vh = .model }), - .aLog = store.createTensor("A_log", .{.vh}, .{ .vh = .model }), + .conv1d_weight = store.withPrefix("conv1d").createTensor("weight", .{ .out, .in, .kernel_size }, .model, .{ .out = .model, .in = .replicated, .kernel_size = .replicated }), + .dt_bias = store.createTensor("dt_bias", .{.vh}, .model, .{ .vh = .model }), + .aLog = store.createTensor("A_log", .{.vh}, .model, .{ .vh = .model }), .norm = RmsNormGated.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), .num_k_heads = config.text_config.linear_num_key_heads, .num_v_heads = config.text_config.linear_num_value_heads, @@ -873,10 +868,10 @@ pub const GatedDeltaNet = struct { const conv_dim = 2 * key_dim + value_dim; const left_pad = self.conv_kernel_size - 1; - const x_in = x.withPartitioning(.{ .d = .replicated }); + const x_in = x.withPartitioning(.model, .{ .d = .replicated }); const projected_qkv = self.in_proj_qkv.forward(x_in, x_in.dtype()) .rename(.{ .dout = .mix }) - .withPartitioning(.{ .s = .replicated, .mix = .model }); + .withPartitioning(.model, .{ .s = .replicated, .mix = .model }); const use_cached_state = x.dim(.s) == 1 and left_pad > 0; const conv_input = b: { if (use_cached_state) break :b zml.Tensor.concatenate(&.{ cache.convState(), projected_qkv }, .s); @@ -906,7 +901,7 @@ pub const GatedDeltaNet = struct { if (use_cached_state) { mixed_qkv = mixed_qkv.slice(.s, .{ .start = mixed_qkv.dim(.s) - 1, .end = mixed_qkv.dim(.s) }); } - mixed_qkv = mixed_qkv.withPartitioning(.{ .s = .replicated, .mix = .model }); + mixed_qkv = mixed_qkv.withPartitioning(.model, .{ .s = .replicated, .mix = .model }); const z = self.in_proj_z.forward(x_in, x_in.dtype()) .splitAxis(.dout, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }); @@ -959,7 +954,7 @@ pub const GatedDeltaNet = struct { const output = self.out_proj .forward(core_attn_out_normed.merge(.{ .d = .{ .vh, .vhd } }), core_attn_out_normed.dtype()) .rename(.{ .dout = .d }) - .withPartitioning(.{ .d = .replicated }); + .withPartitioning(.model, .{ .d = .replicated }); const updated_cache = cache.update( if (use_cached_state) buildUpdatedConvState(conv_input, left_pad) else buildUpdatedConvStateFromPrefix(projected_qkv, left_pad, active_length), last_recurrent_state, @@ -973,7 +968,7 @@ pub const RmsNorm = struct { eps: f32 = 1e-6, pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { - return .{ .weight = store.createTensor("weight", .{.d}, .{ .d = .replicated }), .eps = eps }; + return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { @@ -994,7 +989,7 @@ pub const RmsNormGated = struct { eps: f32 = 1e-6, pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNormGated { - return .{ .weight = store.createTensor("weight", .{.d}, .{ .d = .replicated }), .eps = eps }; + return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } pub fn unloadBuffers(self: *zml.Bufferized(RmsNormGated)) void { @@ -1026,7 +1021,7 @@ pub const KvCache = struct { pub const Buffers = zml.Bufferized(SelfAttnCache); - pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, model_sharding: zml.Sharding) SelfAttnCache { + pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, model_meshe: *const zml.Mesh) SelfAttnCache { const num_self_attn_layers = countLayers(config.text_config.layer_types, .full_attention); const kv_shape = zml.Shape.init(.{ .b = batch_dim, @@ -1035,10 +1030,10 @@ pub const KvCache = struct { .h = config.text_config.num_key_value_heads, .hd = config.text_config.head_dim, }, dtype); - const kv_head_sharding = model_sharding.shardableDim(kv_shape.dim(.h), .model, config.text_config.num_attention_heads); - const sharded_kv_shape = switch (kv_head_sharding) { - .sharded => |heads| kv_shape.setDim(.h, heads.dim).withPartitioning(.{ .h = .model }), - .replicated => kv_shape.withPartitioning(.{ .h = .replicated }), + const kv_head_meshe = model_meshe.shardableDim(kv_shape.dim(.h), .model, config.text_config.num_attention_heads); + const sharded_kv_shape = switch (kv_head_meshe) { + .sharded => |heads| kv_shape.setDim(.h, heads.dim).withPartitioning(model_meshe, .{ .h = .model }), + .replicated => kv_shape.withPartitioning(model_meshe, .{ .h = .replicated }), }; return .{ .k = .fromShape(sharded_kv_shape), @@ -1047,10 +1042,10 @@ pub const KvCache = struct { }; } - pub fn initBuffer(kv: SelfAttnCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !SelfAttnCache.Buffers { + pub fn initBuffer(kv: SelfAttnCache, io: std.Io, platform: *const zml.Platform) !SelfAttnCache.Buffers { return .{ - .k = try zml.Buffer.uninitialized(io, platform, kv.k.shape(), sharding, .{}), - .v = try zml.Buffer.uninitialized(io, platform, kv.v.shape(), sharding, .{}), + .k = try zml.Buffer.uninitialized(io, platform, kv.k.shape(), .{}), + .v = try zml.Buffer.uninitialized(io, platform, kv.v.shape(), .{}), .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), }; } @@ -1127,7 +1122,7 @@ pub const KvCache = struct { pub const Buffers = zml.Bufferized(GatedDeltaNetCache); - pub fn init(config: Config, batch_dim: i64, conv_dtype: zml.DataType, recurrent_dtype: zml.DataType) GatedDeltaNetCache { + pub fn init(config: Config, batch_dim: i64, conv_dtype: zml.DataType, recurrent_dtype: zml.DataType, model_meshe: *const zml.Mesh) GatedDeltaNetCache { const num_linear_attn_layers = countLayers(config.text_config.layer_types, .linear_attention); const conv_dim = 2 * config.text_config.linear_num_key_heads * config.text_config.linear_key_head_dim + @@ -1145,19 +1140,19 @@ pub const KvCache = struct { .khd = config.text_config.linear_key_head_dim, .vhd = config.text_config.linear_value_head_dim, }, recurrent_dtype); - const sharded_conv_state_shape = conv_state_shape.withPartitioning(.{ .mix = .model }); - const sharded_recurrent_state_shape = recurrent_state_shape.withPartitioning(.{ .vh = .model }); + const sharded_conv_state_shape = conv_state_shape.withPartitioning(model_meshe, .{ .mix = .model }); + const sharded_recurrent_state_shape = recurrent_state_shape.withPartitioning(model_meshe, .{ .vh = .model }); return .{ - .conv_state = .fromShape(sharded_conv_state_shape), - .recurrent_state = .fromShape(sharded_recurrent_state_shape), + .conv_state = zml.Tensor.fromShape(sharded_conv_state_shape), + .recurrent_state = zml.Tensor.fromShape(sharded_recurrent_state_shape), .layer_index = .init(.{}, .u32), }; } - pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !GatedDeltaNetCache.Buffers { + pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform) !GatedDeltaNetCache.Buffers { return .{ - .conv_state = try zml.Buffer.uninitialized(io, platform, self.conv_state.shape(), sharding, .{}), - .recurrent_state = try zml.Buffer.uninitialized(io, platform, self.recurrent_state.shape(), sharding, .{}), + .conv_state = try zml.Buffer.uninitialized(io, platform, self.conv_state.shape(), .{}), + .recurrent_state = try zml.Buffer.uninitialized(io, platform, self.recurrent_state.shape(), .{}), .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), }; } @@ -1231,19 +1226,19 @@ pub const KvCache = struct { max_seq_len: i64, cache_dtype: zml.DataType, recurrent_dtype: zml.DataType, - model_sharding: zml.Sharding, + model_meshe: *const zml.Mesh, ) KvCache { return .{ .layer_types = config.text_config.layer_types, - .self_attn = .init(config, batch_dim, max_seq_len, cache_dtype, model_sharding), - .gated_delta_net = .init(config, batch_dim, cache_dtype, recurrent_dtype), + .self_attn = .init(config, batch_dim, max_seq_len, cache_dtype, model_meshe), + .gated_delta_net = .init(config, batch_dim, cache_dtype, recurrent_dtype, model_meshe), }; } - pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(KvCache) { + pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { return .{ - .self_attn = try self.self_attn.initBuffer(io, platform, sharding), - .gated_delta_net = try self.gated_delta_net.initBuffer(io, platform, sharding), + .self_attn = try self.self_attn.initBuffer(io, platform), + .gated_delta_net = try self.gated_delta_net.initBuffer(io, platform), }; } diff --git a/examples/llm/models/qwen3_5/session.zig b/examples/llm/models/qwen3_5/session.zig index a49940cc..84963773 100644 --- a/examples/llm/models/qwen3_5/session.zig +++ b/examples/llm/models/qwen3_5/session.zig @@ -31,11 +31,11 @@ pub const Session = struct { compiled_model: *inference.CompiledModel, model_buffers: *model.Buffers, ) !Session { - var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform, compiled_model.params.shardings.model); + var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform); errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); - var rng_buffers = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed); + var rng_buffers = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); const layer_types = compiled_model.loaded_model.inner.config.text_config.layer_types; @@ -111,9 +111,7 @@ pub const Session = struct { @memset(prefill_tokens_slice.items(u32), 0); @memcpy(prefill_tokens_slice.items(u32)[0..all_tokens.len], all_tokens); - const replicated_sharding: zml.Sharding = .replicated; - - var prefill_tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, prefill_tokens_slice, replicated_sharding); + var prefill_tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, prefill_tokens_slice); defer prefill_tokens_buffer.deinit(); var prefill_token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 0), .u32); @@ -141,9 +139,8 @@ pub const Session = struct { const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); defer self.allocator.free(out_tokens_buffer); - const replicated_sharding: zml.Sharding = .replicated; - var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice, replicated_sharding); + var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice); defer current_token_buffer.deinit(); var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.items.len)), .u32); diff --git a/examples/llm/models/qwen3_5_moe/inference.zig b/examples/llm/models/qwen3_5_moe/inference.zig index 80c6be65..f29ecabd 100644 --- a/examples/llm/models/qwen3_5_moe/inference.zig +++ b/examples/llm/models/qwen3_5_moe/inference.zig @@ -12,17 +12,17 @@ pub const CompilationParameters = struct { rng: zml.Tensor.Rng, moe_backend: zml.moe.Backend, seqlen: u32, - shardings: common.Shardings, + meshes: common.Meshes, xla_dump_to: ?[]const u8, - pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, moe_backend: zml.moe.Backend, shardings: common.Shardings) CompilationParameters { + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, moe_backend: zml.moe.Backend, meshes: common.Meshes) CompilationParameters { const dtype = mdl.text_model.embed_tokens.weight.dtype(); return .{ - .kv_cache = .init(config, 1, seqlen, dtype, .f32, shardings.model), + .kv_cache = .init(config, 1, seqlen, dtype, .f32, meshes.model), .rng = .init(), .moe_backend = moe_backend, .seqlen = seqlen, - .shardings = shardings, + .meshes = meshes, .xla_dump_to = "/home/ubuntu/xla_dump", }; } @@ -274,7 +274,7 @@ fn compileEmbed(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.P fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, layer_index: usize, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardSelfAttn) { return compileExe(allocator, io, platform, model.TransformerLayer.forwardSelfAttn, .{.{ .layer = mdl.text_model.layers[layer_index], - .hidden = hiddenTensor(mdl, seqlen), + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), .token_index = zml.Tensor.init(.{}, .u32), .cache = .{ .k = parameters.kv_cache.self_attn.k, @@ -289,7 +289,7 @@ fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *con fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, layer_index: usize, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardLinearAttn) { return compileExe(allocator, io, platform, model.TransformerLayer.forwardLinearAttn, .{.{ .layer = mdl.text_model.layers[layer_index], - .hidden = hiddenTensor(mdl, seqlen), + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), .active_length = zml.Tensor.init(.{}, .u32), .cache = .{ .conv_state = parameters.kv_cache.gated_delta_net.conv_state, @@ -304,7 +304,7 @@ fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *c fn compileSample(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.Sampler.sampleTokens) { return compileExe(allocator, io, platform, model.Sampler.sampleTokens, .{.{ .sampler = mdl.text_model.sampler(), - .hidden = hiddenTensor(mdl, seqlen), + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), .rng = parameters.rng, .token_index = zml.Tensor.init(.{}, .u32), }}, parameters, progress, phase, "sampling"); @@ -329,16 +329,16 @@ fn compileExe( const now: std.Io.Timestamp = .now(io, .awake); defer log.info("Compiled {s} {s} [{f}]", .{ phase, component, now.untilNow(io, .awake) }); return zml.FnExe(function).compile(allocator, io, platform, .{ - .shardings = ¶meters.shardings.all(), + .meshes = ¶meters.meshes.all(), .xla_dump_to = parameters.xla_dump_to, }, args); } -fn hiddenTensor(mdl: model.Model, seqlen: usize) zml.Tensor { +fn hiddenTensor(mdl: model.Model, seqlen: usize, mesh: *const zml.Mesh) zml.Tensor { return .fromShape(zml.Shape.init( .{ .b = 1, .s = seqlen, .d = mdl.config.text_config.hidden_size }, mdl.text_model.embed_tokens.weight.dtype(), - ).withPartitioning(.{ .b = .replicated, .s = .replicated, .d = .replicated })); + ).withPartitioning(mesh, .{ .b = .replicated, .s = .replicated, .d = .replicated })); } fn findFirstLayerIndex(layer_types: []const model.LayerType, target: model.LayerType) ?usize { diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 6b6f9bb0..59d52815 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -47,26 +47,26 @@ pub const RopeParameters = struct { rope_theta: f32, }; -fn partitionProjectedKv(kv: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { - return switch (kv_head_sharding) { +fn partitionProjectedKv(kv: zml.Tensor, kv_head_meshe: zml.Sharding.DimMeshe) zml.Tensor { + return switch (kv_head_meshe) { .sharded => |heads| blk: { const sharded_kv = if (heads.factor != 1) kv.stutter1d(kv.axis(.h), heads.factor) else kv; - break :blk sharded_kv.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); + break :blk sharded_kv.withPartitioning(.model, .{ .s = .replicated, .h = .model, .hd = .replicated }); }, - .replicated => kv.withPartitioning(.{ .s = .replicated, .h = .replicated, .hd = .replicated }), + .replicated => kv.withPartitioning(.model, .{ .s = .replicated, .h = .replicated, .hd = .replicated }), }; } -fn partitionCachedKv(tensor: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { +fn partitionCachedKv(tensor: zml.Tensor, kv_head_meshe: zml.Sharding.DimMeshe) zml.Tensor { var kv = tensor.rename(.{ .s = .k }); - return switch (kv_head_sharding) { + return switch (kv_head_meshe) { .sharded => |heads| blk: { if (heads.factor != 1) { kv = kv.stutter1d(kv.axis(.h), heads.factor); } - break :blk kv.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); + break :blk kv.withPartitioning(.model, .{ .k = .replicated, .h = .model, .hd = .replicated }); }, - .replicated => kv.withPartitioning(.{ .k = .replicated, .h = .replicated, .hd = .replicated }), + .replicated => kv.withPartitioning(.model, .{ .k = .replicated, .h = .replicated, .hd = .replicated }), }; } @@ -107,7 +107,6 @@ pub const LoadedModel = struct { platform: *const zml.Platform, store: *zml.io.TensorStore, progress: *std.Progress.Node, - shardings: common.Shardings, ) !Buffers { progress.increaseEstimatedTotalItems(store.view().count()); const now: std.Io.Timestamp = .now(io, .awake); @@ -122,8 +121,7 @@ pub const LoadedModel = struct { }); defer loader.deinit(); - const all_shardings = shardings.all(); - try loader.load(io, Model, &self.inner, &buffers, store, &all_shardings, .{ .progress = progress }); + try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); try loader.await(io); const took = now.untilNow(io, .awake); @@ -145,7 +143,7 @@ pub const LoadedModel = struct { io: std.Io, platform: *const zml.Platform, backend: zml.attention.Backend, - shardings: common.Shardings, + meshes: common.Meshes, seqlen: usize, progress: *std.Progress.Node, ) !inference.CompiledModel { @@ -153,7 +151,7 @@ pub const LoadedModel = struct { const moe_gate_up = self.inner.text_model.layers[0].moe.gate_up_proj; log.info("Moe dtype : {}", .{moe_gate_up.weight.dtype()}); const moe_backend = try zml.moe.Backend.auto(platform, moe_gate_up.quantizationScheme(), moe_gate_up.weight.dtype()); - const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), moe_backend, shardings); + const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), moe_backend, meshes); return inference.CompiledModel.init(allocator, io, platform, self, self.inner, params, progress); } }; @@ -196,7 +194,7 @@ pub const Model = struct { io: std.Io, platform: *const zml.Platform, store: *zml.io.TensorStore, - shardings: []const zml.Sharding, + meshes: []const *const zml.Mesh, progress: *std.Progress.Node, ) !zml.Bufferized(Model) { progress.increaseEstimatedTotalItems(store.view().count()); @@ -212,7 +210,7 @@ pub const Model = struct { }); defer loader.deinit(); - loader.load(io, Model, self, &buffers, store, shardings); + loader.load(io, Model, self, &buffers, store, meshes); try loader.await(io); const took = now.untilNow(io, .awake); @@ -314,10 +312,10 @@ pub const TextModel = struct { } return .{ - .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight", .{ .voc, .d }, .{ .voc = .replicated, .d = .model }) }, + .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight", .{ .voc, .d }, .model, .{ .voc = .replicated, .d = .model }) }, .layers = layers, .norm = RmsNorm.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), - .lm_head = .init(store.root().withPrefix(lm_head_prefix).createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), null, .d), + .lm_head = .init(store.root().withPrefix(lm_head_prefix).createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), null, .d), .gen_options = gen_options, }; } @@ -447,7 +445,7 @@ pub const TransformerLayer = struct { const self = input.layer; const x0 = input.hidden; _ = input.config; - const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); const self_attn = switch (self.attn) { @@ -456,13 +454,13 @@ pub const TransformerLayer = struct { }; const attention_output, const updated_kv_cache = self_attn.forward(normalized_x0, input.token_index, input.cache); - const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); const moe_output = self.moe.forward(normalized_hidden, input.moe_backend); return .{ - .hidden = moe_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), + .hidden = moe_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), .cache = updated_kv_cache, }; } @@ -471,7 +469,7 @@ pub const TransformerLayer = struct { const self = input.layer; const x0 = input.hidden; _ = input.config; - const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); const linear_attn = switch (self.attn) { @@ -480,13 +478,13 @@ pub const TransformerLayer = struct { }; const attention_output, const updated_kv_cache = linear_attn.forward(normalized_x0, input.cache, input.active_length); - const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); const moe_output = self.moe.forward(normalized_hidden, input.moe_backend); return .{ - .hidden = moe_output.add(x1).withPartitioning(.{ .d = .replicated }).reuseBuffer(x0), + .hidden = moe_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), .cache = updated_kv_cache, }; } @@ -501,7 +499,7 @@ pub const TransformerLayer = struct { moe_backend: zml.moe.Backend, ) struct { zml.Tensor, KvCache } { _ = config; - const x0_replicated = x0.withPartitioning(.{ .d = .replicated }); + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); const normalized_x0 = self.input_layernorm.forward(x0_replicated); var attention_output: zml.Tensor = undefined; @@ -519,12 +517,12 @@ pub const TransformerLayer = struct { }, } - const x1 = attention_output.add(x0_replicated).withPartitioning(.{ .d = .replicated }); + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); const normalized_hidden = self.post_attention_layernorm.forward(x1); const moe_output = self.moe.forward(normalized_hidden, moe_backend); - return .{ moe_output.add(x1).withPartitioning(.{ .d = .replicated }), updated_kv_cache }; + return .{ moe_output.add(x1).withPartitioning(.model, .{ .d = .replicated }), updated_kv_cache }; } }; @@ -550,8 +548,8 @@ pub const SelfAttn = struct { fn initProj(store: zml.io.TensorStore.View, partitions: anytype, bias_partitions: anytype) zml.nn.Linear { return .init( - store.createTensor("weight", .{ .dout, .d }, partitions), - store.maybeCreateTensor("bias", .{.dout}, bias_partitions), + store.createTensor("weight", .{ .dout, .d }, .model, partitions), + store.maybeCreateTensor("bias", .{.dout}, .model, bias_partitions), .d, ); } @@ -605,19 +603,16 @@ pub const SelfAttn = struct { token_index: zml.Tensor, kv_cache: KvCache.SelfAttnCache, ) struct { zml.Tensor, KvCache.SelfAttnCache } { - const x_qkv = x.withPartitioning(.{ .d = .replicated }); + const tp = zml.Compiler.current().mesh(.model); + const x_qkv = x.withPartitioning(tp, .{ .d = .replicated }); var q, var gate = self.projectQAndGate(x_qkv); var k, var v = self.projectKV(x_qkv); - const kv_head_sharding = zml.Compiler.current().partitioning.shardableDim( - k.shape().withPartitioning(.{ .h = .model }), - .h, - q.dim(.h), - ) catch unreachable; + const kv_head_meshe = tp.shardableDim(k.dim(.h), .model, q.dim(.h)); - q = q.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); - gate = gate.withPartitioning(.{ .s = .replicated, .d_out_proj = .model }); - k = partitionProjectedKv(k, kv_head_sharding); - v = partitionProjectedKv(v, kv_head_sharding); + q = q.withPartitioning(tp, .{ .s = .replicated, .h = .model, .hd = .replicated }); + gate = gate.withPartitioning(tp, .{ .s = .replicated, .d_out_proj = .model }); + k = partitionProjectedKv(k, kv_head_meshe); + v = partitionProjectedKv(v, kv_head_meshe); q = self.q_norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); k = self.k_norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); @@ -629,28 +624,28 @@ pub const SelfAttn = struct { const cos, const sin = self.rotary_embed.getCosAndSin(position_ids, dtype); q = self.rotary_embed.applyRope(q, cos, sin); k = self.rotary_embed.applyRope(k, cos, sin); - q = q.withPartitioning(.{ .s = .replicated, .h = .model, .hd = .replicated }); - k = partitionProjectedKv(k, kv_head_sharding); - v = partitionProjectedKv(v, kv_head_sharding); + q = q.withPartitioning(tp, .{ .s = .replicated, .h = .model, .hd = .replicated }); + k = partitionProjectedKv(k, kv_head_meshe); + v = partitionProjectedKv(v, kv_head_meshe); const new_kv_cache = kv_cache.update(k, v, token_index.convert(.u32)); k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); - q = q.rename(.{ .s = .q }).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }); - k = partitionCachedKv(k, kv_head_sharding); - v = partitionCachedKv(v, kv_head_sharding); + q = q.rename(.{ .s = .q }).withPartitioning(tp, .{ .q = .replicated, .h = .model, .hd = .replicated }); + k = partitionCachedKv(k, kv_head_meshe); + v = partitionCachedKv(v, kv_head_meshe); const attn_output = zml.attention.attention( q, k, v, token_index, - zml.attention.Metadata.init(.fromBackend(.vanilla, x.dim(.s), self.num_heads)), + zml.attention.Metadata.init(.fromBackend(.vanilla, x.dim(.s), self.num_heads), tp), zml.attention.Parameters.init(.fromBackend(.vanilla)), - ).withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); + ).withPartitioning(tp, .{ .q = .replicated, .h = .model, .hd = .replicated }).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); const gated_output = attn_output.mul(gate.sigmoid()); - const projected_output = self.o_proj.forward(gated_output.rename(.{ .d_out_proj = .d }), gated_output.dtype()).rename(.{ .dout = .d }).withPartitioning(.{ .d = .replicated }); + const projected_output = self.o_proj.forward(gated_output.rename(.{ .d_out_proj = .d }), gated_output.dtype()).rename(.{ .dout = .d }).withPartitioning(tp, .{ .d = .replicated }); return .{ projected_output, new_kv_cache }; } @@ -667,20 +662,20 @@ pub const Mlp = struct { pub fn init(store: zml.io.TensorStore.View) Mlp { return .{ .up_proj = .init( - store.withPrefix("up_proj").createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), - store.withPrefix("up_proj").maybeCreateTensor("bias", .{.dout}, .{ .dout = .model }), + store.withPrefix("up_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), + store.withPrefix("up_proj").maybeCreateTensor("bias", .{.dout}, .model, .{ .dout = .model }), .d, ), .up_proj_scale = null, .gate_proj = .init( - store.withPrefix("gate_proj").createTensor("weight", .{ .dout, .d }, .{ .dout = .model, .d = .replicated }), - store.withPrefix("gate_proj").maybeCreateTensor("bias", .{.dout}, .{ .dout = .model }), + store.withPrefix("gate_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), + store.withPrefix("gate_proj").maybeCreateTensor("bias", .{.dout}, .model, .{ .dout = .model }), .d, ), .gate_proj_scale = null, .down_proj = .init( - store.withPrefix("down_proj").createTensor("weight", .{ .dout, .d }, .{ .dout = .replicated, .d = .model }), - store.withPrefix("down_proj").maybeCreateTensor("bias", .{.d}, .{ .d = .replicated }), + store.withPrefix("down_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .replicated, .d = .model }), + store.withPrefix("down_proj").maybeCreateTensor("bias", .{.d}, .model, .{ .d = .replicated }), .d, ), .down_proj_scale = null, @@ -708,8 +703,8 @@ const Router = struct { pub fn init(store: zml.io.TensorStore.View, num_experts_per_tok: u32) Router { return .{ .router = .init( - store.createTensor("weight", .{ .expert, .d }, .{ .expert = .replicated, .d = .replicated }), - store.maybeCreateTensor("bias", .{.expert}, .{ .expert = .replicated }), + store.createTensor("weight", .{ .expert, .d }, .replicated, .{ .expert = .replicated, .d = .replicated }), + store.maybeCreateTensor("bias", .{.expert}, .replicated, .{ .expert = .replicated }), .d, ), .num_experts_per_tok = num_experts_per_tok, @@ -742,19 +737,21 @@ pub const Moe = struct { const gate_up_proj_tensor = experts_store.createTensor( "gate_up_proj", .{ .expert, .dout, .d }, + .experts, .{ .expert = .experts, .dout = .replicated, .d = .replicated }, ); const down_proj_tensor = experts_store.createTensor( "down_proj", .{ .expert, .d, .dout }, + .experts, .{ .expert = .experts, .d = .replicated, .dout = .replicated }, ); return .{ .shared_expert = Mlp.init(store.withPrefix("shared_expert")), .shared_expert_gate = .init( - store.withPrefix("shared_expert_gate").createTensor("weight", .{ .dout, .d }, .{ .dout = .replicated, .d = .replicated }), - store.withPrefix("shared_expert_gate").maybeCreateTensor("bias", .{.dout}, .{ .dout = .replicated }), + store.withPrefix("shared_expert_gate").createTensor("weight", .{ .dout, .d }, .replicated, .{ .dout = .replicated, .d = .replicated }), + store.withPrefix("shared_expert_gate").maybeCreateTensor("bias", .{.dout}, .replicated, .{ .dout = .replicated }), .d, ), .gate_up_proj = .init(gate_up_proj_tensor, null, .d), @@ -786,7 +783,7 @@ pub const Moe = struct { ); const shared_gate = self.shared_expert_gate.forward(x, x.dtype()).sigmoid().broad(x.shape()); - const shared = self.shared_expert.forward(x).rename(.{ .dout = .d }).mul(shared_gate).withPartitioning(.{ + const shared = self.shared_expert.forward(x).rename(.{ .dout = .d }).mul(shared_gate).withPartitioning(.model, .{ .b = .replicated, .s = .replicated, .d = .replicated, @@ -896,7 +893,7 @@ pub const GatedDeltaNet = struct { conv_kernel_size: i64, fn initProj(store: zml.io.TensorStore.View, partitions: anytype) zml.nn.Linear { - return .init(store.createTensor("weight", .{ .dout, .d }, partitions), null, .d); + return .init(store.createTensor("weight", .{ .dout, .d }, .model, partitions), null, .d); } pub fn init(store: zml.io.TensorStore.View, config: Config) GatedDeltaNet { @@ -911,9 +908,9 @@ pub const GatedDeltaNet = struct { .in_proj_a = initProj(store.withPrefix("in_proj_a"), .{ .dout = .model, .d = .replicated }), .out_proj = initProj(store.withPrefix("out_proj"), .{ .dout = .replicated, .d = .model }), .out_proj_scale = null, - .conv1d_weight = store.withPrefix("conv1d").createTensor("weight", .{ .out, .in, .kernel_size }, .{ .out = .model, .in = .replicated, .kernel_size = .replicated }), - .dt_bias = store.createTensor("dt_bias", .{.vh}, .{ .vh = .model }), - .aLog = store.createTensor("A_log", .{.vh}, .{ .vh = .model }), + .conv1d_weight = store.withPrefix("conv1d").createTensor("weight", .{ .out, .in, .kernel_size }, .model, .{ .out = .model, .in = .replicated, .kernel_size = .replicated }), + .dt_bias = store.createTensor("dt_bias", .{.vh}, .model, .{ .vh = .model }), + .aLog = store.createTensor("A_log", .{.vh}, .model, .{ .vh = .model }), .norm = RmsNormGated.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), .num_k_heads = config.text_config.linear_num_key_heads, .num_v_heads = config.text_config.linear_num_value_heads, @@ -997,9 +994,9 @@ pub const GatedDeltaNet = struct { const conv_dim = 2 * key_dim + value_dim; const left_pad = self.conv_kernel_size - 1; - const x_in = x.withPartitioning(.{ .d = .replicated }); + const x_in = x.withPartitioning(.model, .{ .d = .replicated }); const projected_qkv = self.in_proj_qkv.forward(x_in, x_in.dtype()) - .rename(.{ .dout = .mix }).withPartitioning(.{ .s = .replicated, .mix = .model }); + .rename(.{ .dout = .mix }).withPartitioning(.model, .{ .s = .replicated, .mix = .model }); const use_cached_state = x.dim(.s) == 1 and left_pad > 0; const conv_input = if (use_cached_state) zml.Tensor.concatenate(&.{ cache.convState(), projected_qkv }, .s) @@ -1029,26 +1026,26 @@ pub const GatedDeltaNet = struct { if (use_cached_state) { mixed_qkv = mixed_qkv.slice(.s, .{ .start = mixed_qkv.dim(.s) - 1, .end = mixed_qkv.dim(.s) }); } - mixed_qkv = mixed_qkv.withPartitioning(.{ .s = .replicated, .mix = .model }); + mixed_qkv = mixed_qkv.withPartitioning(.model, .{ .s = .replicated, .mix = .model }); const z = self.in_proj_z.forward(x_in, x_in.dtype()) .splitAxis(.dout, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }) - .withPartitioning(.{ .s = .replicated, .vh = .model, .vhd = .replicated }); - const b = self.in_proj_b.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }).withPartitioning(.{ .s = .replicated, .vh = .model }); - const a = self.in_proj_a.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }).withPartitioning(.{ .s = .replicated, .vh = .model }); + .withPartitioning(.model, .{ .s = .replicated, .vh = .model, .vhd = .replicated }); + const b = self.in_proj_b.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }).withPartitioning(.model, .{ .s = .replicated, .vh = .model }); + const a = self.in_proj_a.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }).withPartitioning(.model, .{ .s = .replicated, .vh = .model }); const query = mixed_qkv .slice(.mix, .{ .start = 0, .end = key_dim }) .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }) - .withPartitioning(.{ .s = .replicated, .kh = .model, .khd = .replicated }); + .withPartitioning(.model, .{ .s = .replicated, .kh = .model, .khd = .replicated }); const key = mixed_qkv .slice(.mix, .{ .start = key_dim, .end = 2 * key_dim }) .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }) - .withPartitioning(.{ .s = .replicated, .kh = .model, .khd = .replicated }); + .withPartitioning(.model, .{ .s = .replicated, .kh = .model, .khd = .replicated }); const value = mixed_qkv .slice(.mix, .{ .start = 2 * key_dim, .end = 2 * key_dim + value_dim }) .splitAxis(.mix, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }) - .withPartitioning(.{ .s = .replicated, .vh = .model, .vhd = .replicated }); + .withPartitioning(.model, .{ .s = .replicated, .vh = .model, .vhd = .replicated }); const beta = b.sigmoid(); const aLog_type = self.aLog.dtype(); @@ -1073,10 +1070,10 @@ pub const GatedDeltaNet = struct { z.rename(.{ .vhd = .d }), ) .rename(.{ .d = .vhd }) - .withPartitioning(.{ .s = .replicated, .vh = .model, .vhd = .replicated }); + .withPartitioning(.model, .{ .s = .replicated, .vh = .model, .vhd = .replicated }); const output = self.out_proj.forward(core_attn_out_normed.merge(.{ .d = .{ .vh, .vhd } }), core_attn_out_normed.dtype()) - .rename(.{ .dout = .d }).withPartitioning(.{ .d = .replicated }); + .rename(.{ .dout = .d }).withPartitioning(.model, .{ .d = .replicated }); const updated_cache = cache.update( if (use_cached_state) buildUpdatedConvState(conv_input, left_pad) else buildUpdatedConvStateFromPrefix(projected_qkv, left_pad, active_length), last_recurrent_state, @@ -1090,7 +1087,7 @@ pub const RmsNorm = struct { eps: f32 = 1e-6, pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { - return .{ .weight = store.createTensor("weight", .{.d}, .{ .d = .replicated }), .eps = eps }; + return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { @@ -1111,7 +1108,7 @@ pub const RmsNormGated = struct { eps: f32 = 1e-6, pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNormGated { - return .{ .weight = store.createTensor("weight", .{.d}, .{ .d = .replicated }), .eps = eps }; + return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; } pub fn unloadBuffers(self: *zml.Bufferized(RmsNormGated)) void { @@ -1141,7 +1138,7 @@ pub const KvCache = struct { v: zml.Tensor, layer_index: zml.Tensor, - pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, model_sharding: zml.Sharding) SelfAttnCache { + pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, model_meshe: *const zml.Mesh) SelfAttnCache { const num_self_attn_layers = countLayers(config.text_config.layer_types, .full_attention); const kv_shape = zml.Shape.init(.{ .b = batch_dim, @@ -1150,22 +1147,22 @@ pub const KvCache = struct { .h = config.text_config.num_key_value_heads, .hd = config.text_config.head_dim, }, dtype); - const kv_head_sharding = model_sharding.shardableDim(kv_shape.dim(.h), .model, config.text_config.num_attention_heads); - const sharded_kv_shape = switch (kv_head_sharding) { - .sharded => |heads| kv_shape.setDim(.h, heads.dim).withPartitioning(.{ .h = .model }), - .replicated => kv_shape.withPartitioning(.{ .h = .replicated }), + const kv_head_meshe = model_meshe.shardableDim(kv_shape.dim(.h), .model, config.text_config.num_attention_heads); + const sharded_kv_shape = switch (kv_head_meshe) { + .sharded => |heads| kv_shape.setDim(.h, heads.dim).withPartitioning(model_meshe, .{ .h = .model }), + .replicated => kv_shape.withPartitioning(model_meshe, .{ .h = .replicated }), }; return .{ - .k = .fromShape(sharded_kv_shape), - .v = .fromShape(sharded_kv_shape), + .k = zml.Tensor.fromShape(sharded_kv_shape), + .v = zml.Tensor.fromShape(sharded_kv_shape), .layer_index = .init(.{}, .u32), }; } - pub fn initBuffer(self: SelfAttnCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(SelfAttnCache) { + pub fn initBuffer(self: SelfAttnCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(SelfAttnCache) { return .{ - .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), sharding, .{}), - .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), sharding, .{}), + .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), + .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}), .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), }; } @@ -1238,7 +1235,7 @@ pub const KvCache = struct { recurrent_state: zml.Tensor, layer_index: zml.Tensor, - pub fn init(config: Config, batch_dim: i64, conv_dtype: zml.DataType, recurrent_dtype: zml.DataType) GatedDeltaNetCache { + pub fn init(config: Config, batch_dim: i64, conv_dtype: zml.DataType, recurrent_dtype: zml.DataType, model_meshe: *const zml.Mesh) GatedDeltaNetCache { const num_linear_attn_layers = countLayers(config.text_config.layer_types, .linear_attention); const conv_dim = 2 * config.text_config.linear_num_key_heads * config.text_config.linear_key_head_dim + config.text_config.linear_num_value_heads * config.text_config.linear_value_head_dim; const conv_state_shape = zml.Shape.init(.{ @@ -1254,19 +1251,19 @@ pub const KvCache = struct { .khd = config.text_config.linear_key_head_dim, .vhd = config.text_config.linear_value_head_dim, }, recurrent_dtype); - const sharded_conv_state_shape = conv_state_shape.withPartitioning(.{ .mix = .model }); - const sharded_recurrent_state_shape = recurrent_state_shape.withPartitioning(.{ .vh = .model }); + const sharded_conv_state_shape = conv_state_shape.withPartitioning(model_meshe, .{ .mix = .model }); + const sharded_recurrent_state_shape = recurrent_state_shape.withPartitioning(model_meshe, .{ .vh = .model }); return .{ - .conv_state = .fromShape(sharded_conv_state_shape), - .recurrent_state = .fromShape(sharded_recurrent_state_shape), + .conv_state = zml.Tensor.fromShape(sharded_conv_state_shape), + .recurrent_state = zml.Tensor.fromShape(sharded_recurrent_state_shape), .layer_index = .init(.{}, .u32), }; } - pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(GatedDeltaNetCache) { + pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(GatedDeltaNetCache) { return .{ - .conv_state = try zml.Buffer.uninitialized(io, platform, self.conv_state.shape(), sharding, .{}), - .recurrent_state = try zml.Buffer.uninitialized(io, platform, self.recurrent_state.shape(), sharding, .{}), + .conv_state = try zml.Buffer.uninitialized(io, platform, self.conv_state.shape(), .{}), + .recurrent_state = try zml.Buffer.uninitialized(io, platform, self.recurrent_state.shape(), .{}), .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), }; } @@ -1334,19 +1331,19 @@ pub const KvCache = struct { max_seq_len: i64, cache_dtype: zml.DataType, recurrent_dtype: zml.DataType, - model_sharding: zml.Sharding, + model_meshe: *const zml.Mesh, ) KvCache { return .{ .layer_types = config.text_config.layer_types, - .self_attn = SelfAttnCache.init(config, batch_dim, max_seq_len, cache_dtype, model_sharding), - .gated_delta_net = GatedDeltaNetCache.init(config, batch_dim, cache_dtype, recurrent_dtype), + .self_attn = SelfAttnCache.init(config, batch_dim, max_seq_len, cache_dtype, model_meshe), + .gated_delta_net = GatedDeltaNetCache.init(config, batch_dim, cache_dtype, recurrent_dtype, model_meshe), }; } - pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(KvCache) { + pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { return .{ - .self_attn = try self.self_attn.initBuffer(io, platform, sharding), - .gated_delta_net = try self.gated_delta_net.initBuffer(io, platform, sharding), + .self_attn = try self.self_attn.initBuffer(io, platform), + .gated_delta_net = try self.gated_delta_net.initBuffer(io, platform), }; } diff --git a/examples/llm/models/qwen3_5_moe/session.zig b/examples/llm/models/qwen3_5_moe/session.zig index e1509df5..35998797 100644 --- a/examples/llm/models/qwen3_5_moe/session.zig +++ b/examples/llm/models/qwen3_5_moe/session.zig @@ -31,12 +31,11 @@ pub const Session = struct { compiled_model: *inference.CompiledModel, model_buffers: *model.Buffers, ) !Session { - const shardings = compiled_model.params.shardings; - var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform, shardings.model); + var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform); errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); - var rng_buffers = try zml.Tensor.Rng.initBuffer(io, platform, .replicated, seed); + var rng_buffers = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); const layer_types = compiled_model.loaded_model.inner.config.text_config.layer_types; @@ -122,7 +121,7 @@ pub const Session = struct { @memset(tokens_slice.items(u32), 0); @memcpy(tokens_slice.items(u32)[0..all_tokens.len], all_tokens); - var tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, tokens_slice, .replicated); + var tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, tokens_slice); defer tokens_buffer.deinit(); var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 0), .u32); defer token_index_buffer.deinit(); @@ -149,7 +148,7 @@ pub const Session = struct { const out_tokens_buffer = try self.allocator.alloc(u8, 1024); defer self.allocator.free(out_tokens_buffer); - var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice, .replicated); + var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice); defer current_token_buffer.deinit(); var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.items.len)), .u32); defer token_index_buffer.deinit(); diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 3b4349c4..d1291c1b 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -19,8 +19,8 @@ const Mnist = struct { pub fn init(store: zml.io.TensorStore.View) Layer { return .{ - .weight = store.createTensor("weight", .{ .d_out, .d }, .replicated), - .bias = store.createTensor("bias", .{.d_out}, .replicated), + .weight = store.createReplicatedTensor("weight", .{ .d_out, .d }), + .bias = store.createReplicatedTensor("bias", .{.d_out}), }; } @@ -49,7 +49,7 @@ const Mnist = struct { var loader: zml.io.Loader = try .init(allocator, platform, .default); errdefer loader.deinit(); - try loader.load(io, Mnist, self, &buffers, store, &.{}, .{}); + try loader.load(io, Mnist, self, &buffers, store, .{}); try loader.await(io); return buffers; @@ -66,7 +66,10 @@ const Mnist = struct { pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { var x = input.flatten().convert(.f32).withTags(.{.d}); const layers: []const Layer = &.{ self.fc1, self.fc2 }; - for (layers) |layer| { + const compiler = zml.Compiler.current(); + for (0.., layers) |i, layer| { + compiler.pushLocationFmt(@src(), "layer[{d}]", .{i}); + defer compiler.popLocation(); x = layer.forward(x); } return x.argMax(0).indices.convert(.u8); @@ -87,15 +90,15 @@ pub fn main(init: std.process.Init) !void { var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); defer registry.deinit(); - // Init model - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); - defer store.deinit(); - const mnist_model: Mnist = .init(store.view()); - // Auto-select platform const platform: *zml.Platform = try .auto(allocator, io, .{}); defer platform.deinit(allocator, io); + // Init model + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{platform.replicated_mesh}); + defer store.deinit(); + const mnist_model: Mnist = .init(store.view()); + // // Compile model const input: zml.Tensor = .init(.{ 28, 28 }, .u8); var exe = blk: { @@ -137,7 +140,7 @@ pub fn main(init: std.process.Init) !void { var sample: [28 * 28]u8 align(16) = undefined; _ = try dataset.readPositionalAll(io, &sample, 16 + (idx * 28 * 28)); - var input_buffer: zml.Buffer = try .fromSlice(io, platform, zml.Slice.init(input.shape(), &sample), .replicated); + var input_buffer: zml.Buffer = try .fromSlice(io, platform, zml.Slice.init(input.shape(), &sample)); defer input_buffer.deinit(); printDigit(sample); diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index 61a16318..cc46a1bd 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -2,7 +2,7 @@ const std = @import("std"); const zml = @import("zml"); -const log = std.log.scoped(.sharding); +const log = std.log.scoped(.mesh); pub const std_options: std.Options = .{ .log_level = .info, @@ -34,10 +34,10 @@ const DemoModel = struct { var y = x.dot(self.w, .feature); y = y.add(self.b.broad(y.shape())); - y = y.withPartitioning(.{ .batch = .data, .hidden = .model }); + y = y.withPartitioning(.dp_mp, .{ .batch = .data, .hidden = .model }); y.print("dense_out"); - const gate = y.scale(0.01).sigmoid(); + const gate = y.scale(0.01).sigmoid().withPartitioning(.dp_mp, .{ .batch = .data, .hidden = .model }); return zml.ops.manualComputation( (struct { y: zml.Tensor, @@ -142,7 +142,6 @@ fn createSequenceBuffer( io: std.Io, platform: *const zml.Platform, shape: zml.Shape, - sharding: zml.Sharding, start: f32, ) !zml.Buffer { const slice = try zml.Slice.alloc(allocator, shape); @@ -152,7 +151,7 @@ fn createSequenceBuffer( e.* = start + @as(f32, @floatFromInt(i)); } - return zml.Buffer.fromSlice(io, platform, slice, sharding); + return zml.Buffer.fromSlice(io, platform, slice); } pub fn main(init: std.process.Init) !void { @@ -193,24 +192,24 @@ pub fn main(init: std.process.Init) !void { } log.info("{f}", .{platform.physical_mesh}); - const sharding: zml.Sharding = try platform.registerSharding( - "demo_mesh", + const mesh: *const zml.Mesh = try platform.registerMesh( + "dp_mp", .mesh(.{ .data = .low_bandwidth, .model = .high_bandwidth }), ); - log.info("{f}", .{sharding.data.logical}); - log.info("{f}", .{sharding}); + log.info("{f}", .{mesh.logical}); + log.info("{f}", .{mesh}); const input_shape = zml.Shape.init(.{ .batch = 16, .feature = 32 }, .f32) - .withPartitioning(.{ .batch = .data, .feature = .replicated }); + .withPartitioning(mesh, .{ .batch = .data, .feature = .replicated }); const w_shape = zml.Shape.init(.{ .feature = 32, .hidden = 64 }, .f32) - .withPartitioning(.{ .feature = .replicated, .hidden = .model }); + .withPartitioning(mesh, .{ .feature = .replicated, .hidden = .model }); const b_shape = zml.Shape.init(.{ .hidden = 64 }, .f32) - .withPartitioning(.{ .hidden = .model }); + .withPartitioning(mesh, .{ .hidden = .model }); - const input: zml.Tensor = .fromShape(input_shape); - const w: zml.Tensor = .fromShape(w_shape); - const b: zml.Tensor = .fromShape(b_shape); + const input: zml.Tensor = zml.Tensor.fromShape(input_shape); + const w: zml.Tensor = zml.Tensor.fromShape(w_shape); + const b: zml.Tensor = zml.Tensor.fromShape(b_shape); const model: DemoModel = .init(w, b); var exe = try platform.compile( @@ -221,16 +220,16 @@ pub fn main(init: std.process.Init) !void { .{input}, .{ .partitioner = args.partitioner, - .shardings = &.{sharding}, + .meshes = &.{mesh}, }, ); defer exe.deinit(); - var w_buf = try createSequenceBuffer(allocator, io, platform, w_shape, sharding, 0.0); + var w_buf = try createSequenceBuffer(allocator, io, platform, w_shape, 0.0); defer w_buf.deinit(); - var b_buf = try createSequenceBuffer(allocator, io, platform, b_shape, sharding, 100.0); + var b_buf = try createSequenceBuffer(allocator, io, platform, b_shape, 100.0); defer b_buf.deinit(); - var input_buf = try createSequenceBuffer(allocator, io, platform, input_shape, sharding, 1000.0); + var input_buf = try createSequenceBuffer(allocator, io, platform, input_shape, 1000.0); defer input_buf.deinit(); log.info("input placement: {f}", .{input_buf}); diff --git a/platforms/cuda/BUILD.bazel b/platforms/cuda/BUILD.bazel index b337cc9a..7f7aed55 100644 --- a/platforms/cuda/BUILD.bazel +++ b/platforms/cuda/BUILD.bazel @@ -20,6 +20,7 @@ zig_shared_library( cc_library( name = "libzml_cuda_repo", + target_compatible_with = ["@platforms//os:linux"], deps = select({ "@llvm//platforms/config:linux_x86_64": ["@libzml_cuda_linux_amd64//:libzml_cuda"], "@llvm//platforms/config:linux_aarch64": ["@libzml_cuda_linux_arm64//:libzml_cuda"], @@ -28,6 +29,7 @@ cc_library( cc_library( name = "cuda_nvtx_headers", + target_compatible_with = ["@platforms//os:linux"], visibility = ["//visibility:public"], deps = select({ "@llvm//platforms/config:linux_x86_64": ["@cuda_nvtx_linux_x86_64//:headers"], diff --git a/stdx/debug.zig b/stdx/debug.zig index 4686b3b5..a217eca1 100644 --- a/stdx/debug.zig +++ b/stdx/debug.zig @@ -10,11 +10,12 @@ pub inline fn internalAssert(check: bool, comptime msg: []const u8, args: anytyp pub inline fn assert(check: bool, comptime msg: []const u8, args: anytype) void { if (!check) { + @branchHint(.cold); panic(msg, args); } } -pub inline fn panic(comptime format: []const u8, args: anytype) noreturn { +pub fn panic(comptime format: []const u8, args: anytype) noreturn { std.debug.panic(format, args); } diff --git a/stdx/fmt.zig b/stdx/fmt.zig index 47ec59aa..33351d93 100644 --- a/stdx/fmt.zig +++ b/stdx/fmt.zig @@ -5,6 +5,16 @@ pub fn slice(any_slice: anytype) FmtSlice(std.meta.Elem(@TypeOf(any_slice))) { return .{ .slice = any_slice }; } +/// Properly format a slice of []const u8. +pub fn strings(strs: []const []const u8) FmtStrings { + return .{ .strings = strs }; +} + +/// Properly format a slice of [*:0]const u8. +pub fn stringsZ(strs: []const [*:0]const u8) FmtZeroStrings { + return .{ .strings = strs }; +} + fn FmtSlice(T: type) type { return struct { slice: []const T, @@ -31,6 +41,26 @@ fn FmtSlice(T: type) type { }; } +pub const FmtStrings = struct { + strings: []const []const u8, + + pub fn format(f: FmtStrings, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return try formatSliceCustom(formatStr, f.strings, .{}, 1, writer); + } + + pub fn formatNumber(f: FmtStrings, writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + return try formatIntSlice(f.strings, n, 1, writer); + } +}; + +pub const FmtZeroStrings = struct { + strings: []const [*:0]const u8, + + pub fn format(f: FmtZeroStrings, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return try formatSliceCustom(formatStrZ, f.strings, .{}, 1, writer); + } +}; + pub fn formatFloat(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { const x = switch (@typeInfo(@TypeOf(value))) { .@"struct" => value.toF32(), @@ -77,6 +107,14 @@ pub fn formatBool(value: bool, spec: std.fmt.Number, writer: *std.Io.Writer) !vo try writer.alignBufferOptions(if (value) "1" else "0", .{ .alignment = spec.alignment, .fill = spec.fill }); } +pub fn formatStr(value: []const u8, _: std.fmt.Number, writer: *std.Io.Writer) !void { + try writer.writeAll(value); +} + +pub fn formatStrZ(value: [*:0]const u8, _: std.fmt.Number, writer: *std.Io.Writer) !void { + try writer.writeAll(std.mem.span(value)); +} + pub fn formatAny(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { var buf: [48]u8 = undefined; const T = @TypeOf(value); diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index e8c8519d..844778f5 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -16,10 +16,11 @@ sh_binary( sh_binary( name = "rocprofv3", + srcs = ["rocprofv3.sh"], data = [ "@amdrocm-profiler-base//:rocprofv3_tool", "@libzml_rocm//:sandbox", ], - srcs = ["rocprofv3.sh"], + target_compatible_with = ["@platforms//os:linux"], visibility = ["//visibility:public"], ) diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 23259b1d..41f175ae 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -20,7 +20,6 @@ const Platform = @import("platform.zig").Platform; const tracer = @import("profiling/tracer.zig"); const Shape = @import("shape.zig").Shape; const Sharding = @import("Sharding.zig"); -const Partitioning = Sharding.Partitioning; const Tensor = @import("tensor.zig").Tensor; const Compiler = @This(); @@ -35,7 +34,8 @@ mlir_ctx: *mlir.Context, mlir_pass_manager: *mlir.PassManager, module: *mlir.Module, platform: *const Platform, -partitioning: Sharding.Partitioning, +meshes: []const *const Sharding.Mesh, +partitioner: Sharding.Partitioner, mlir_known_types: std.enums.EnumArray(DataType, *const mlir.Type), @@ -74,7 +74,7 @@ fn mlirRegistry(io: std.Io) *mlir.DialectRegistry { pub const Options = struct { /// Incremented once after successful compilation. progress: std.Progress.Node = .none, - shardings: []const Sharding = &.{}, + meshes: []const *const Sharding.Mesh = &.{}, // If null, will be initialized from the target partitioner: ?Sharding.Partitioner = null, // Debugging options @@ -161,16 +161,13 @@ pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, } } - // Ensure replicated sharding is always included as a fallback option. - var shardings = std.ArrayList(Sharding).initCapacity(arena.allocator(), opts.shardings.len + 1) catch @panic("OOM"); - var needs_replicated: bool = true; - for (opts.shardings) |sharding| { - if (sharding.data == platform.replicated_sharding.data) needs_replicated = false; - shardings.appendAssumeCapacity(sharding.resolve(platform)); + var mesh_list: std.ArrayList(*const Sharding.Mesh) = .empty; + mesh_list.appendSlice(arena.allocator(), opts.meshes) catch @panic("OOM"); + if (std.mem.indexOfScalar(*const Sharding.Mesh, mesh_list.items, platform.replicated_mesh) == null) { + mesh_list.append(arena.allocator(), platform.replicated_mesh) catch @panic("OOM"); } - if (needs_replicated) shardings.appendAssumeCapacity(platform.replicated_sharding); - - const partitioning = Sharding.Partitioning.init(opts.partitioner orelse .fromTarget(platform.target), shardings.items) catch @panic("OOM"); + const meshes = mesh_list.items; + validateMeshes(meshes) catch |err| stdx.debug.panic("Incompatible meshes: {t}", .{err}); return .{ .allocator = allocator, @@ -182,7 +179,8 @@ pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, .mlir_known_types = mlir_known_types, .module = module, .platform = platform, - .partitioning = partitioning, + .partitioner = opts.partitioner orelse .fromTarget(platform.target), + .meshes = meshes, .location = unknown_location, .unknown_location = unknown_location, }; @@ -276,7 +274,8 @@ test pushLocation { .mlir_known_types = undefined, .module = undefined, .platform = undefined, - .partitioning = undefined, + .partitioner = undefined, + .meshes = undefined, .location = unknown_location, .unknown_location = unknown_location, }; @@ -328,6 +327,63 @@ pub fn allocPrint(self: *Compiler, comptime fmt: []const u8, args: anytype) []u8 return std.fmt.allocPrint(self.arena.allocator(), fmt, args) catch self.abortOOM(); } +/// Lookup a mesh by it's name +pub fn mesh(compiler: *const Compiler, name: @EnumLiteral()) *const Sharding.Mesh { + const name_slice = @tagName(name); + for (compiler.meshes) |m| { + if (std.mem.eql(u8, name_slice, m.name)) { + return m; + } + } + if (name == .replicated) return compiler.platform.replicated_mesh; + std.debug.panic( + \\Found no meshes named {s}. + \\Try passing more meshes to `zml.compile`. + \\Known meshes: {f} + , .{ name_slice, stdx.fmt.slice(compiler.meshes) }); +} + +/// Find a mesh that covers all the given axes +pub fn resolveMesh(compiler: *const Compiler, logical_axes: anytype) *const Sharding.Mesh { + if (@TypeOf(logical_axes) != []const Shape.Tag) { + const comp_tags = comptime Shape.parseTags(logical_axes); + const parsed_tags: []const Shape.Tag = comptime comp_tags.constSlice(); + return compiler.resolveMesh(parsed_tags); + } + + var ok_meshe: ?*const Sharding.Mesh = null; + for (compiler.meshes) |shd| { + var covers_all: bool = true; + for (logical_axes) |ax| { + const input_axis = Shape.toTag(ax); + var covers_this: bool = false; + for (shd.logical.axes.slice()) |existing_axis| { + if (std.mem.eql(u8, std.mem.span(existing_axis), std.mem.span(input_axis))) { + covers_this = true; + break; + } + } + covers_all = covers_all and covers_this; + } + if (covers_all) { + if (ok_meshe) |first_match| { + std.debug.panic( + \\Found two meshes covering axes: {any}, expected exacty one. + \\- First match: {f} + \\- Second match: {f} + , .{ logical_axes, first_match, shd }); + } + ok_meshe = shd; + } + } + + return ok_meshe orelse std.debug.panic( + \\Found no meshes covering axes: {any}, expected exacty one. + \\Try passing more meshes to `zml.compile`. + \\Known meshes: {f} + , .{ logical_axes, stdx.fmt.slice(compiler.meshes) }); +} + pub fn Typed(comptime func: anytype) type { return struct { pub fn compile( @@ -402,9 +458,9 @@ pub fn compileInternal( _ = result.func.appendTo(compiler.module.body()); - const num_partitions = compiler.partitioning.numPartitions(); - const num_replicas = compiler.partitioning.numReplicas(); - const num_devices = compiler.partitioning.numDevices(); + const num_partitions = compiler.meshes[0].numPartitions(); + const num_replicas = compiler.meshes[0].numReplicas(); + const num_devices = num_partitions * num_replicas; compiler.module.operation().setAttributeByName( "mhlo.num_partitions", @@ -422,10 +478,7 @@ pub fn compileInternal( }, }; - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - - const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), io, platform, compiler.module, compiler.partitioning, opts) catch |err| { + const loaded_executable = compiler.compileModuleToPjrtExecutable(opts) catch |err| { compiler.handleCompilationError(io, compiler.arena.allocator(), opts, err, "Pjrt failed to compile the following MLIR"); return err; }; @@ -441,8 +494,6 @@ pub fn compileInternal( // This will get copied into exe result.input_info.items(.shape), result.output_info.items(.shape), - result.input_info.items(.sharding), - result.output_info.items(.sharding), result.input_info.items(.aliasing_output), ); errdefer exe.deinit(); @@ -455,16 +506,15 @@ fn addPartitionerOperations(ctx: *Compiler) !void { const allocator = ctx.arena.allocator(); const mlir_ctx = ctx.mlir_ctx; const module = ctx.module; - const partitioning = ctx.partitioning; - switch (partitioning.partitioner) { + switch (ctx.partitioner) { .gspmd => {}, .shardy => { - for (partitioning.shardings) |sharding| { - const attr_str = try sharding.data.sdyMeshAttr(allocator); + for (ctx.meshes) |shd| { + const attr_str = try shd.sdyMeshAttr(allocator); defer allocator.free(attr_str); - const name = sharding.data.name; + const name = shd.name; const mesh_attr = try mlir.Attribute.parse(mlir_ctx, attr_str); const mesh_op = mlir.Operation.make(mlir_ctx, "sdy.mesh", .{ @@ -491,7 +541,6 @@ const EmitMlirResult = struct { pub const TensorInfo = struct { id: Tensor.Id, shape: Shape, - sharding: Sharding, value: *const mlir.Value, // Only used for input tensors, stores which output tensor ends up with their buffer @@ -501,12 +550,12 @@ pub const TensorInfo = struct { var attrs: AttributeList = .empty; const mlir_ctx = compiler.mlir_ctx; - const sharding_attr = try compiler.partitioning.tensorShardingAttr(arena, mlir_ctx, info.shape, info.sharding); - const name = switch (compiler.partitioning.partitioner) { + const meshe_attr = try compiler.tensorShardingAttr(arena, mlir_ctx, info.shape); + const name = switch (compiler.partitioner) { .gspmd => "mhlo.sharding", .shardy => "sdy.sharding", }; - attrs.appendAssumeCapacity(.named(mlir_ctx, name, sharding_attr)); + attrs.appendAssumeCapacity(.named(mlir_ctx, name, meshe_attr)); const memory = scope.id_to_memory.get(info.id) orelse .device; if (memory != .device) { @@ -583,16 +632,9 @@ fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{Ou defer ctx.current_argument_id += 1; - const input_sharding = ctx.compiler.partitioning.selectSharding(packed_shape) catch |err| switch (err) { - error.NoSuitableSharding => std.debug.panic( - "Failed to resolve sharding for input {f}({d}) because it's using unknown sharding. Pass more shardings to `platform.compile`. Known shardings: {f}", - .{ packed_shape, ctx.current_argument_id, stdx.fmt.slice(ctx.compiler.partitioning.shardings) }, - ), - }; try ctx.infos.append(ctx.compiler.allocator, .{ .id = tensor.id, .shape = og_shape, - .sharding = input_sharding, .value = value, }); } @@ -615,7 +657,6 @@ fn collectOutputInfo(compiler: *Compiler, scope: *Scope, v: anytype) error{OutOf fn cb(ctx: *@This(), tensor: *const Tensor) !void { const og_shape = tensor.shape(); - const packed_shape = og_shape.packedShape(); var value = ctx.scope.id_to_argument.get(tensor.id) orelse tensor._value orelse @panic("no value found for output tensor"); @@ -626,9 +667,9 @@ fn collectOutputInfo(compiler: *Compiler, scope: *Scope, v: anytype) error{OutOf try ctx.infos.append(ctx.compiler.allocator, .{ .id = tensor.id, + // const packed_shape = og_shape.packedShape(); + // TODO: clarify why this og_shape and not packedShape() .shape = og_shape, - // Note: the panic should have been triggered during createBlockArguments or emitMlir - .sharding = ctx.compiler.partitioning.selectSharding(packed_shape) catch @panic("failed to resolve output sharding"), .value = value, }); } @@ -666,14 +707,14 @@ fn finalizeMlirFunc(compiler: *Compiler, fn_scope: *Scope, input_info: std.Multi } } - // Input sharding/memory/aliasing attributes + // Input mesh/memory/aliasing attributes const input_attributes = try arena.alloc(*const mlir.Attribute, input_info.len); for (0.., input_attributes) |i, *input_attrs| { const attrs_list = try input_info.get(i).attributes(arena, compiler, fn_scope); input_attrs.* = .dict(mlir_ctx, attrs_list.constSlice()); } - // Output sharding/memory attributes + // Output mesh/memory attributes const output_attributes = try arena.alloc(*const mlir.Attribute, output_info.len); for (0.., output_attributes) |i, *output_attrs| { const attrs_list = try output_info.get(i).attributes(arena, compiler, fn_scope); @@ -748,6 +789,38 @@ fn repack(compiler: *Compiler, scope: *Scope, og_shape: Shape, value: *const mli return bit_cast_op.result(0); } +fn validateMeshes(meshes: []const *const Sharding.Mesh) !void { + stdx.debug.assert(meshes.len >= 1, "Waiting at leat 1 mesh strategy to be implemented", .{}); + + const first = meshes[0]; + const partitions = first.numPartitions(); + const replicas = first.numReplicas(); + + for (meshes[1..]) |s| { + if (s.numPartitions() != partitions or s.numReplicas() != replicas) { + // todo: deviceAssignments should also be checked for consistency here, but for simplicity we just check the cardinality numbers + return error.InconsistentMesheCardinality; + } + } +} + +pub fn tensorShardingAttr( + compiler: *const Compiler, + allocator: std.mem.Allocator, + mlir_ctx: *mlir.Context, + shape: Shape, +) error{OutOfMemory}!*const mlir.Attribute { + const mesh_ = shape._sharding.mesh orelse compiler.platform.replicated_mesh; + return switch (compiler.partitioner) { + .shardy => (try mesh_.sdyShardingAttrForShape(allocator, mlir_ctx, shape)).asAttr(), + .gspmd => mesh_.gspmdShardingAttrForShape(allocator, mlir_ctx, shape) catch |err| switch (err) { + error.WriteFailed => error.OutOfMemory, // We're writing to memory + error.OutOfMemory => error.OutOfMemory, + error.IncompatibleSharding => @panic("Shape dimensions must be divisible by their mesh partitions"), + }, + }; +} + fn setXlaOverrideFlag(map: *c.upb_Map, flag: []const u8, value: anytype, upb_arena: *c.upb_Arena) !void { const result = c.upb_Map_Set( map, @@ -770,21 +843,25 @@ fn setXlaOverrideFlag(map: *c.upb_Map, flag: []const u8, value: anytype, upb_are } } -fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: *const Platform, module: *const mlir.Module, partitioning: Partitioning, opts: Options) !*pjrt.LoadedExecutable { - var upb_alloc: upb.Allocator = .init(arena); - const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); - defer c.upb_Arena_Free(upb_arena); - - const use_shardy_partitioner = switch (partitioning.partitioner) { +fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.LoadedExecutable { + const use_shardy_partitioner = switch (compiler.partitioner) { .shardy => true, .gspmd => false, }; - const num_partitions = partitioning.numPartitions(); - const num_replicas = partitioning.numReplicas(); + const platform = compiler.platform; + const main_mesh = compiler.meshes[0]; + const num_partitions = main_mesh.numPartitions(); + const num_replicas = main_mesh.numReplicas(); - const device_assignment = try partitioning.deviceAssignment(arena); + const device_assignment = try main_mesh.deviceAssignment(compiler.allocator); + defer compiler.allocator.free(device_assignment); + var arena = std.heap.ArenaAllocator.init(compiler.allocator); + defer arena.deinit(); + var upb_alloc: upb.Allocator = .init(arena.allocator()); + const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); + defer c.upb_Arena_Free(upb_arena); const options = blk: { const options = try upb.new(c.xla_CompileOptionsProto, upb_arena); c.xla_CompileOptionsProto_set_executable_build_options(options, executable_build_options_blk: { @@ -918,9 +995,9 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: const loaded_executable = try pjrtx.Client.compile( platform.pjrt_client, platform.pjrt_api, - arena, - io, - module, + compiler.allocator, + compiler.io, + compiler.module, try upb.serialize(options, upb_arena), ); errdefer loaded_executable.deinit(); diff --git a/zml/Sharding.zig b/zml/Sharding.zig index b510a9c4..761721fa 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -16,33 +16,152 @@ const Target = @import("platform.zig").Target; const Sharding = @This(); -data: *const Data, - -pub const Error = error{ MissingLogicalBinding, IncompatibleSharding }; +mesh: ?*const Mesh, +partition: Partitioning, pub const MAX_MESH_RANK = 4; +pub const replicated: Sharding = .{ .mesh = null, .partition = .replicated(Shape.MAX_RANK) }; -var _replicated: [11]u8 align(@alignOf(Data)) = "_replicated".*; - -// special value to make public apis more fluent. -pub const replicated: Sharding = .{ .data = @ptrCast(&_replicated) }; - -pub fn resolve(sharding: Sharding, platform: *const Platform) Sharding { - return if (sharding.data == replicated.data) platform.replicated_sharding else sharding; +pub fn get(sharding: Sharding, ax: usize) PartitionSpec { + return if (sharding.mesh == null) .replicated else sharding.partition.get(ax); } -pub fn numPartitionsForLogicalAxis(sharding: Sharding, logical_axis: anytype) i64 { - return sharding.data.numPartitionsForLogicalAxis(logical_axis); +pub fn set(self: Sharding, ax: usize, spec: PartitionSpec) Sharding { + var res = self; + res.partition = res.partition.set(ax, spec); + return res; } -pub fn devicesInCanonicalOrder(sharding: Sharding) []const Device { - return sharding.data.physical.devices_in_canonical_order; +pub fn insert(self: Sharding, ax: usize, spec: PartitionSpec) Sharding { + var res = self; + res.partition = res.partition.insert(ax, spec); + return res; } -pub fn format(sharding: Sharding, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return sharding.data.format(writer); +pub fn orderedRemove(self: Sharding, ax: usize) Sharding { + var res = self; + res.partition = res.partition.orderedRemove(ax); + return res; } +pub fn toArray(self: Sharding) [Shape.MAX_RANK]PartitionSpec { + var result: [Shape.MAX_RANK]PartitionSpec = undefined; + for (&result, 0..) |*spec, ax| spec.* = self.get(ax); + return result; +} + +pub fn isFullyReplicated(self: Sharding) bool { + if (self.mesh == null) return true; + for (self.partition.toArray()) |spec| { + if (spec != .replicated and spec != .out_of_bound) return false; + } + return true; +} + +/// An unspecified mesh is compatible with any sharding. On the same mesh, +/// open axes accept any in-bounds partition, while closed axes must match. +pub fn isCompatible(self: Sharding, other: Sharding) bool { + if (self.mesh == null or other.mesh == null) return true; + if (self.mesh != other.mesh) return false; + for (self.partition.toArray(), other.partition.toArray()) |lhs, rhs| { + if (lhs == rhs) continue; + if (lhs == .out_of_bound or rhs == .out_of_bound) return false; + if (lhs != .open and rhs != .open) return false; + } + return true; +} + +test isCompatible { + const tp: Mesh = .initTest(.tp, .{.model}); + const dp: Mesh = .initTest(.dp, .{ .data, .model }); + + const sharded: Sharding = .{ .mesh = &tp, .partition = .init(&.{ .mesh_axis_0, .replicated }) }; + const cases = [_]struct { lhs: Sharding, rhs: Sharding, compatible: bool }{ + .{ .lhs = .replicated, .rhs = .replicated, .compatible = true }, + .{ .lhs = .replicated, .rhs = sharded, .compatible = true }, + .{ .lhs = sharded, .rhs = sharded, .compatible = true }, + .{ .lhs = sharded, .rhs = .{ .mesh = &tp, .partition = .open(2) }, .compatible = true }, + .{ .lhs = sharded.set(1, .open), .rhs = sharded.set(0, .open), .compatible = true }, + .{ .lhs = sharded, .rhs = sharded.set(0, .mesh_axis_1), .compatible = false }, + .{ .lhs = sharded, .rhs = sharded.set(0, .replicated), .compatible = false }, + .{ .lhs = sharded, .rhs = .{ .mesh = &dp, .partition = sharded.partition }, .compatible = false }, + .{ .lhs = sharded, .rhs = .{ .mesh = &dp, .partition = .open(2) }, .compatible = false }, + .{ .lhs = sharded, .rhs = sharded.set(1, .out_of_bound), .compatible = false }, + .{ .lhs = sharded, .rhs = sharded.set(2, .open), .compatible = false }, + }; + for (cases) |case| { + errdefer std.log.err("{f}.isCompatible({f}) expected to be {}, got {}", .{ case.lhs, case.rhs, case.compatible, !case.compatible }); + + try std.testing.expectEqual(case.compatible, case.lhs.isCompatible(case.rhs)); + try std.testing.expectEqual(case.compatible, case.rhs.isCompatible(case.lhs)); + } +} + +pub fn format(self: Sharding, writer: *std.Io.Writer) !void { + const mesh = self.mesh orelse return writer.writeAll("{replicated}"); + const specs = self.partition.toArray(); + var rank = specs.len; + while (rank > 0 and specs[rank - 1] == .out_of_bound) rank -= 1; + + try writer.writeByte('{'); + for (specs[0..rank], 0..) |spec, i| { + if (i > 0) try writer.writeByte(','); + switch (spec) { + .replicated => try writer.writeAll("replicated"), + .open => try writer.writeByte('?'), + .out_of_bound => try writer.writeByte('!'), + else => try writer.print("{s}", .{mesh.logical.axes.get(spec.meshAxis().?)}), + } + } + try writer.print("}}@{s}", .{mesh.name}); +} + +/// Describes how a given Shape axis behaves inside a mesh. +/// Is it replicated ? sharded along a specific logical axis ? or open to replication ? +pub const PartitionSpec = enum(u4) { + mesh_axis_0 = 0, + mesh_axis_1 = 1, + mesh_axis_2 = 2, + mesh_axis_3 = 3, + mesh_axis_4 = 4, + mesh_axis_5 = 5, + mesh_axis_6 = 6, + mesh_axis_7 = 7, + // an 8D mesh seems already a lot, the max we know about is 3D. + replicated = 8, + + // Partitioning don't encode the rank of corresponding tensor, so this is used to mark invalid positions. + out_of_bound = 0xa, + open = 15, + + pub fn sharded(mesh_axis: u3) PartitionSpec { + return @enumFromInt(mesh_axis); + } + + /// Extract the mesh axis along which we are sharded. Null if not sharded. + pub fn meshAxis(self: PartitionSpec) ?u3 { + const ax = @intFromEnum(self); + return if (ax < @intFromEnum(PartitionSpec.replicated)) @intCast(ax) else null; + } + + pub fn isSharded(self: PartitionSpec) bool { + return @intFromEnum(self) < @intFromEnum(PartitionSpec.replicated); + } + + pub fn isClosed(self: PartitionSpec) bool { + return @intFromEnum(self) <= @intFromEnum(PartitionSpec.replicated); + } + + test isClosed { + try std.testing.expect(PartitionSpec.mesh_axis_2.isClosed()); + try std.testing.expect(!PartitionSpec.open.isClosed()); + + try std.testing.expect(PartitionSpec.replicated.isClosed()); + + try std.testing.expect(!PartitionSpec.out_of_bound.isClosed()); + } +}; + pub const Partitioner = union(enum) { shardy, gspmd, @@ -54,121 +173,14 @@ pub const Partitioner = union(enum) { } }; -pub const Partitioning = struct { - partitioner: Partitioner, - shardings: []const Sharding, - - pub fn init(partitioner: Partitioner, shardings: []const Sharding) !Partitioning { - stdx.debug.assert(shardings.len >= 1, "Waiting at leat 1 sharding strategy to be implemented", .{}); - var plan: Partitioning = .{ .partitioner = partitioner, .shardings = shardings }; - - const first = plan.primarySharding(); - const partitions = first.data.numPartitions(); - const replicas = first.data.numReplicas(); - - for (shardings[1..]) |s| { - if (s.data.numPartitions() != partitions or s.data.numReplicas() != replicas) { - // todo: deviceAssignments should also be checked for consistency here, but for simplicity we just check the cardinality numbers - return error.InconsistentShardingCardinality; - } - } - - return plan; - } - - pub fn numPartitions(self: Partitioning) i32 { - const primary = self.primarySharding(); - return primary.data.numPartitions(); - } - - pub fn numReplicas(self: Partitioning) i32 { - const primary = self.primarySharding(); - return primary.data.numReplicas(); - } - - pub fn numDevices(self: Partitioning) i32 { - return self.numPartitions() * self.numReplicas(); - } - - pub fn deviceAssignment(self: Partitioning, allocator: std.mem.Allocator) ![]usize { - return switch (self.partitioner) { - .shardy, .gspmd => blk: { - const sharding = self.primarySharding(); - break :blk sharding.data.deviceAssignment(allocator); - }, - }; - } - - pub fn tensorShardingAttr( - self: Partitioning, - allocator: std.mem.Allocator, - ctx: *mlir.Context, - shape: Shape, - sharding: Sharding, - ) error{OutOfMemory}!*const mlir.Attribute { - return switch (self.partitioner) { - .shardy => (try sharding.data.sdyShardingAttrForShape(allocator, ctx, shape)).asAttr(), - .gspmd => sharding.data.gspmdShardingAttrForShape(allocator, ctx, shape) catch |err| switch (err) { - error.WriteFailed => error.OutOfMemory, // We're writing to memory - error.OutOfMemory => error.OutOfMemory, - // TODO(hugomano): clarify what can trigger this and consider moving the check to the Sharding creation - error.MissingDeviceInTile => @panic("MissingDeviceInTile"), - }, - }; - } - - pub fn localShapeForShape(self: Partitioning, shape: Shape) !Shape { - const sharding = try self.selectSharding(shape); - const ordered_devices = sharding.devicesInCanonicalOrder(); - if (ordered_devices.len == 0) return error.InvalidPhysicalMesh; - return sharding.shardedShape(shape); - } - - pub fn numPartitionsForLogicalAxis(self: Partitioning, shape: Shape, logical_axis: anytype) !i64 { - const sharding = try self.selectSharding(shape); - return sharding.data.numPartitionsForLogicalAxis(logical_axis); - } - - pub fn shardableDim(self: Partitioning, shape: Shape, axis: anytype, must_divide: i64) !DimSharding { - const ax = shape.axis(axis); - const spec = shape.partition(ax); - if (spec != .axis) return .replicated; - - const sharding = try self.selectSharding(shape); - return sharding.shardableDim(shape.dim(ax), spec.axis, must_divide); - } - - pub fn selectSharding(self: Partitioning, shape: Shape) error{NoSuitableSharding}!Sharding { - return pickSharding(self.shardings, shape, .any_covering) orelse error.NoSuitableSharding; - } - - fn primarySharding(self: Partitioning) Sharding { - return self.shardings[0]; - } -}; - -pub const SelectShardingMode = enum { - any_covering, - explicit_axis_binding, -}; - -pub fn pickSharding(shardings: []const Sharding, shape: Shape, mode: SelectShardingMode) ?Sharding { - if (mode == .explicit_axis_binding and !shapeHasAxisPartition(shape)) return null; - - for (shardings) |sharding| { - if (sharding.data.covers(shape)) return sharding; - } - return null; -} - fn shapeHasAxisPartition(shape: Shape) bool { for (0..shape.rank()) |ax| { - if (shape.partition(ax) == .axis) return true; + if (shape.partition(ax).meshAxis()) |_| return true; } return false; } -pub const DimSharding = union(enum) { +pub const DimMeshe = union(enum) { sharded: struct { dim: i64, factor: u32, @@ -176,43 +188,25 @@ pub const DimSharding = union(enum) { replicated, }; -pub fn shardableDim(sharding: Sharding, dim: i64, logical_axis: anytype, must_divide: i64) DimSharding { - const partitions = sharding.numPartitionsForLogicalAxis(logical_axis); - - if (@mod(dim, partitions) == 0) { - return if (@mod(must_divide, dim) == 0) .{ .sharded = .{ .dim = dim, .factor = 1 } } else .replicated; - } - - const gcd: i64 = @intCast(std.math.gcd(@as(u64, @intCast(dim)), @as(u64, @intCast(partitions)))); - const repeat_factor: u32 = @intCast(@divExact(partitions, gcd)); - const materialized_dim = std.math.mul(i64, dim, @as(i64, repeat_factor)) catch return .replicated; - if (@mod(must_divide, materialized_dim) == 0) { - return .{ .sharded = .{ .dim = materialized_dim, .factor = repeat_factor } }; - } else { - return .replicated; - } -} - pub fn sdyPerValueShardingAttr( allocator: std.mem.Allocator, ctx: *mlir.Context, shapes: []const Shape, - shardings: []const Sharding, + mesh: *const Mesh, ) error{OutOfMemory}!*const mlir.Attribute { - const sharding_attrs = try allocator.alloc(*const dialects.shardy.TensorShardingAttribute, shapes.len); - for (sharding_attrs, shapes, shardings) |*attr, shape, sharding| { - attr.* = try sharding.data.sdyShardingAttrForShape(allocator, ctx, shape); + const meshe_attrs = try allocator.alloc(*const dialects.shardy.TensorShardingAttribute, shapes.len); + for (meshe_attrs, shapes) |*attr, shape| { + attr.* = try mesh.sdyShardingAttrForShape(allocator, ctx, shape); } - return dialects.shardy.TensorShardingPerValueAttribute.init(ctx, sharding_attrs).asAttr(); + return dialects.shardy.TensorShardingPerValueAttribute.init(ctx, meshe_attrs).asAttr(); } pub fn sdyManualAxesAttr( allocator: std.mem.Allocator, ctx: *mlir.Context, in_shapes: []const Shape, - in_shardings: []const Sharding, out_shapes: []const Shape, - out_shardings: []const Sharding, + mesh: *const Mesh, ) error{OutOfMemory}!*const mlir.Attribute { var axis_names = std.ArrayList([]const u8).empty; defer axis_names.deinit(allocator); @@ -226,8 +220,8 @@ pub fn sdyManualAxesAttr( } }; - for (in_shapes, in_shardings) |shape, sharding| { - const attr = try sharding.data.sdyShardingAttrForShape(allocator, ctx, shape); + for (in_shapes) |shape| { + const attr = try mesh.sdyShardingAttrForShape(allocator, ctx, shape); for (0..attr.numReplicatedAxes()) |i| { Collect.appendUnique(&axis_names, allocator, attr.replicatedAxis(i).name()); } @@ -238,8 +232,8 @@ pub fn sdyManualAxesAttr( } } } - for (out_shapes, out_shardings) |shape, sharding| { - const attr = try sharding.data.sdyShardingAttrForShape(allocator, ctx, shape); + for (out_shapes) |shape| { + const attr = try mesh.sdyShardingAttrForShape(allocator, ctx, shape); for (0..attr.numReplicatedAxes()) |i| { Collect.appendUnique(&axis_names, allocator, attr.replicatedAxis(i).name()); } @@ -259,6 +253,192 @@ pub fn sdyManualAxesAttr( return dialects.shardy.ManualAxesAttribute.init(ctx, axes).asAttr(); } +pub const Partitioning = packed struct { + _0: PartitionSpec, + _1: PartitionSpec, + _2: PartitionSpec, + _3: PartitionSpec, + _4: PartitionSpec, + _5: PartitionSpec, + _6: PartitionSpec, + _7: PartitionSpec, + + pub const out_of_bound: Partitioning = splat(.out_of_bound); + + const MAX_RANK = Shape.MAX_RANK; + const Vec = @Vector(MAX_RANK, u4); + + pub fn init(specs: []const PartitionSpec) Partitioning { + var res: Partitioning = out_of_bound; + for (0.., specs) |i, spec| { + res = res.set(i, spec); + } + return res; + } + + pub fn replicated(rank_: usize) Partitioning { + return repeat(.replicated, rank_); + } + + pub fn open(rank_: usize) Partitioning { + return repeat(.open, rank_); + } + + pub fn splat(spec: PartitionSpec) Partitioning { + const vec: Vec = @splat(@intFromEnum(spec)); + return @bitCast(vec); + } + + pub fn repeat(spec: PartitionSpec, rank_: usize) Partitioning { + std.debug.assert(rank_ <= MAX_RANK); + const splatted: Vec = @splat(@intFromEnum(spec)); + const mask = std.simd.iota(u4, 8) < @as(Vec, @splat(@truncate(rank_))); + return @bitCast(@select(u4, mask, splatted, @as(Vec, @bitCast(out_of_bound)))); + } + + pub fn get(p: Partitioning, ax: usize) PartitionSpec { + std.debug.assert(ax < MAX_RANK); + const pack: u32 = @bitCast(p); + const shift: u5 = @intCast(4 * ax); + return @enumFromInt(@as(u4, @truncate(pack >> shift))); + } + + pub fn set(p: Partitioning, ax: usize, spec: PartitionSpec) Partitioning { + std.debug.assert(ax < MAX_RANK); + const pack: u32 = @bitCast(p); + const shift: u5 = @intCast(4 * ax); + const mask = @as(u32, 0xf) << shift; + return @bitCast((pack & ~mask) | (@as(u32, @intFromEnum(spec)) << shift)); + } + + pub fn rank(p: Partitioning) u8 { + for (0..MAX_RANK) |i_usize| { + const i: u8 = @truncate(i_usize); + if (p.get(i) == .out_of_bound) return i; + } + return MAX_RANK; + } + + /// Inserts a spec, shifting subsequent slots right and discarding the last slot. + pub fn insert(p: Partitioning, ax: usize, spec: PartitionSpec) Partitioning { + std.debug.assert(ax < MAX_RANK); + const pack: u32 = @bitCast(p); + const shift: u5 = @intCast(4 * ax); + const lower_mask = (@as(u32, 1) << shift) - 1; + return @bitCast((pack & lower_mask) | ((pack & ~lower_mask) << 4) | (@as(u32, @intFromEnum(spec)) << shift)); + } + + /// Removes a spec, shifting subsequent slots left and filling the last with unknown. + pub fn orderedRemove(p: Partitioning, ax: usize) Partitioning { + std.debug.assert(ax < MAX_RANK); + const pack: u32 = @bitCast(p); + const shift: u5 = @intCast(4 * ax); + const lower_mask = (@as(u32, 1) << shift) - 1; + return @bitCast((pack & lower_mask) | ((pack >> 4) & ~lower_mask) | (@as(u32, @intFromEnum(PartitionSpec.out_of_bound)) << 28)); + } + + pub fn toArray(p: Partitioning) [MAX_RANK]PartitionSpec { + var res: [MAX_RANK]PartitionSpec = undefined; + for (&res, 0..) |*spec, ax| spec.* = p.get(ax); + return res; + } + + pub fn hasUniqueAxes(p: Partitioning) bool { + var used_mesh_axes: std.StaticBitSet(MAX_RANK) = .initEmpty(); + for (0..MAX_RANK) |shape_ax| { + const spec = p.get(shape_ax); + if (spec.meshAxis()) |mesh_axis| { + if (used_mesh_axes.isSet(mesh_axis)) return false; + used_mesh_axes.set(mesh_axis); + } + } + return true; + } + + test hasUniqueAxes { + try std.testing.expect(hasUniqueAxes(.init(&.{ .mesh_axis_0, .mesh_axis_1, .replicated, .mesh_axis_2 }))); + try std.testing.expect(!hasUniqueAxes(.init(&.{ .mesh_axis_0, .mesh_axis_1, .replicated, .mesh_axis_0 }))); + } + + test "packed access and updates" { + try std.testing.expectEqual(32, @bitSizeOf(Partitioning)); + try std.testing.expectEqual(4, @sizeOf(Partitioning)); + inline for (std.meta.tags(PartitionSpec)) |spec| { + const filled = Partitioning.splat(spec); + for (0..MAX_RANK) |ax| { + try std.testing.expectEqual(spec, filled.get(ax)); + const updated = Partitioning.out_of_bound.set(ax, spec); + for (0..MAX_RANK) |i| { + try std.testing.expectEqual(if (i == ax) spec else .out_of_bound, updated.get(i)); + } + } + } + for (0..MAX_RANK + 1) |rank_| { + const parts = Partitioning.replicated(rank_); + for (0..MAX_RANK) |ax| { + try std.testing.expectEqual(if (ax < rank_) PartitionSpec.replicated else .out_of_bound, parts.get(ax)); + } + } + } + + test "insertion and removal at every slot" { + var parts: Partitioning = .out_of_bound; + for (0..MAX_RANK) |ax| parts = parts.set(ax, @enumFromInt(ax)); + const original = parts.toArray(); + for (0..MAX_RANK) |ax| { + var inserted = original; + std.mem.copyBackwards(PartitionSpec, inserted[ax + 1 ..], original[ax .. MAX_RANK - 1]); + inserted[ax] = .open; + try std.testing.expectEqualSlices(PartitionSpec, &inserted, &parts.insert(ax, .open).toArray()); + + var removed = original; + std.mem.copyForwards(PartitionSpec, removed[ax .. MAX_RANK - 1], original[ax + 1 ..]); + removed[MAX_RANK - 1] = .out_of_bound; + try std.testing.expectEqualSlices(PartitionSpec, &removed, &parts.orderedRemove(ax).toArray()); + try std.testing.expectEqual(parts, parts.orderedRemove(ax).insert(ax, parts.get(ax))); + } + try std.testing.expectEqual(Partitioning.out_of_bound, Partitioning.out_of_bound.orderedRemove(0)); + const comptime_parts = comptime Partitioning.out_of_bound.insert(7, .open).orderedRemove(0).set(0, .replicated); + try std.testing.expectEqual(PartitionSpec.replicated, comptime_parts.get(0)); + try std.testing.expectEqual(PartitionSpec.open, comptime_parts.get(6)); + try std.testing.expectEqual(PartitionSpec.out_of_bound, comptime_parts.get(7)); + } + + /// Parse a struct literal into a partitioning object. + /// The struct passed to partitioning must only have fields names present in `tags`. + /// The ordering of tags matter, the return struct will map names to axis as specified by `tags`. + /// The default value is important. + /// In Tensor code unspecified axis will be inferred as `.open` while in shape code, it's `.replicated`. + pub fn parse(tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec, partitioning: anytype) Partitioning { + var partition_: Partitioning = .repeat(default_value, tags.len); + + const T = @TypeOf(partitioning); + stdx.debug.assertComptime(stdx.meta.isStruct(T), "parsePartitioning expected a struct of enum literals eg {{ .b = .data, .d = .model }}, got: {any}", .{T}); + inline for (std.meta.fields(T)) |field| { + const shape_tag = Shape.toTag(field); + const shape_ax = Shape.axisFromTagMaybe(tags, shape_tag) orelse { + std.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), field.name }); + }; + + const value = @field(partitioning, field.name); + const spec: PartitionSpec = if (@TypeOf(value) == PartitionSpec) value else switch (value) { + .replicated => .replicated, + .out_of_bound => std.debug.panic("value out_of_bound not allowed", .{}), + .open => .open, + else => spec: { + const mesh_tag = Shape.toTag(value); + const mesh_ax = mesh.resolveLogicalAxis(mesh_tag) orelse { + stdx.debug.panic("{f} has no logical axis '{s}'", .{ mesh, mesh_tag }); + }; + break :spec .sharded(@intCast(mesh_ax)); + }, + }; + partition_ = partition_.set(shape_ax, spec); + } + return partition_; + } +}; + /// Device as part of a PhysicalMesh. pub const Device = struct { /// Unique device identifier in the mesh @@ -1127,7 +1307,7 @@ pub const PhysicalMesh = struct { }; pub const LogicalAxisIntent = enum { - /// Needs the fastest possible path (e.g. Weight sharding / All-Reduce). + /// Needs the fastest possible path (e.g. Weight mesh / All-Reduce). /// Requires high-bandwidth, low-latency interconnects (Cores/NVLink). high_bandwidth, @@ -1135,12 +1315,12 @@ pub const LogicalAxisIntent = enum { /// Suitable for chip-to-chip interconnects (NeuronLink/ICI). balanced, - /// Can tolerate lower bandwidth and higher latency (e.g. Batch sharding). + /// Can tolerate lower bandwidth and higher latency (e.g. Batch mesh). /// Suitable for system bus or network (PCIe/Ethernet). low_bandwidth, }; -// Defines the logical mesh used for sharding the model and data. +// Defines the logical mesh used for mesh the model and data. // E.g., batch, model, context, experts, etc. // Each axis represents a dimension along which the workload can be partitioned. The value is the intent of the axis. /// LogicalMesh defines *semantic* axes only (batch/model/context/...). @@ -1214,16 +1394,11 @@ pub const LogicalMesh = struct { } }; -pub const AxisList = stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK); -pub const Binding = struct { - logical: Shape.Tag, - physical: AxisList, -}; -pub const Bindings = stdx.BoundedArray(Binding, MAX_MESH_RANK); +pub const PhysicalList = stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK); pub const Fold = struct { target: PhysicalAxisTag, - sources: AxisList, + sources: PhysicalList, }; pub const Folds = stdx.BoundedArray(Fold, MAX_MESH_RANK); @@ -1260,38 +1435,73 @@ pub const PhysicalView = struct { } }; -pub const Data = struct { +pub const Mesh = struct { + pub const zml_no_meta_visit: void = {}; + name: []const u8, physical: *const PhysicalMesh, logical: LogicalMesh, /// Compact binding table: logical axis -> physical axes - bindings: Bindings, + bindings: stdx.BoundedArray(PhysicalList, MAX_MESH_RANK), /// Explicit folding rules: kept axis -> ordered source axes. folds: Folds, folds_consumed: std.EnumSet(PhysicalAxisTag), - pub fn binding(self: *const Data, tag: Shape.Tag) ?[]const PhysicalAxisTag { - for (self.bindings.slice()) |*b| { - if (b.logical == tag) return b.physical.slice(); + pub fn shardableDim(mesh: *const Mesh, dim: i64, logical_axis: anytype, must_divide: i64) DimMeshe { + const partitions = mesh.numPartitionsForLogicalAxis(logical_axis); + + if (@mod(dim, partitions) == 0) { + return if (@mod(must_divide, dim) == 0) .{ .sharded = .{ .dim = dim, .factor = 1 } } else .replicated; + } + + const gcd: i64 = @intCast(std.math.gcd(@as(u64, @intCast(dim)), @as(u64, @intCast(partitions)))); + const repeat_factor: u32 = @intCast(@divExact(partitions, gcd)); + const materialized_dim = std.math.mul(i64, dim, @as(i64, repeat_factor)) catch return .replicated; + if (@mod(must_divide, materialized_dim) == 0) { + return .{ .sharded = .{ .dim = materialized_dim, .factor = repeat_factor } }; + } else { + return .replicated; + } + } + + pub fn binding(self: *const Mesh, mesh_axis: usize) []const PhysicalAxisTag { + return self.bindings.constSlice()[mesh_axis].constSlice(); + } + + pub fn resolveLogicalAxis(self: *const Mesh, tag: Shape.Tag) ?u8 { + for (0.., self.logical.axes.slice()) |i, logical_axis| { + if (std.mem.eql(u8, std.mem.span(logical_axis), std.mem.span(tag))) return @truncate(i); } return null; } pub fn init( - name: []const u8, + name_: []const u8, physical: *const PhysicalMesh, logical: LogicalMesh, strategy: Strategy, - ) !Data { + ) !Mesh { const axis_order = physical.axisOrder().slice(); if (axis_order.len == 0) return error.InvalidPhysicalMesh; if (strategy.bindings.len == 0) return error.InvalidStrategy; - var bindings: Bindings = .empty; - for (strategy.bindings.slice()) |bind| { - bindings.appendAssumeCapacity(.{ .logical = bind.logical, .physical = bind.physical }); + // Create one binding per logical axis + var bindings: @FieldType(Mesh, "bindings") = .empty; + for (logical.axes.slice()) |_| { + bindings.appendAssumeCapacity(.empty); + } + + for (strategy.bindings.slice()) |strat_binding| { + for (logical.axes.slice(), bindings.slice()) |logical_axis, *binding_| { + if (std.mem.eql(u8, std.mem.span(logical_axis), std.mem.span(strat_binding.logical))) { + binding_.* = strat_binding.physical; + break; + } + } else { + std.debug.panic("{f} expects strategy to only list known logical axes, got unknown {s}", .{ logical, strat_binding.logical }); + } } var folds: Folds = .empty; @@ -1305,7 +1515,7 @@ pub const Data = struct { } return .{ - .name = name, + .name = name_, .logical = logical, .physical = physical, .bindings = bindings, @@ -1314,14 +1524,34 @@ pub const Data = struct { }; } - fn foldSources(self: *const Data, tag: PhysicalAxisTag) ?[]const PhysicalAxisTag { + pub fn initTest(name_: @EnumLiteral(), tagz: anytype) Mesh { + if (!builtin.is_test) @compileError("Cannot use zml.Sharding.Mesh.initTest outside of a test block"); + const axes_tags = Shape.parseTags(tagz); + return .{ + .name = @tagName(name_), + .logical = .{ + .axes = .init(axes_tags.slice()), + .intents = .{ .buffer = @splat(.high_bandwidth), .len = axes_tags.len }, + }, + .physical = undefined, + .bindings = .empty, + .folds = .empty, + .folds_consumed = .empty, + }; + } + + fn foldSources(self: *const Mesh, tag: PhysicalAxisTag) ?[]const PhysicalAxisTag { for (self.folds.slice()) |*f| { if (f.target == tag) return f.sources.slice(); } return null; } - pub fn physicalView(self: *const Data) PhysicalView { + pub fn devicesInCanonicalOrder(mesh: *const Mesh) []const Device { + return mesh.physical.devices_in_canonical_order; + } + + pub fn physicalView(self: *const Mesh) PhysicalView { const physical = self.physical; var view: PhysicalView = .{ .axes = .empty, @@ -1359,17 +1589,18 @@ pub const Data = struct { return view; } - pub fn logicalIndexFromCoords(self: *const Data, coords: []const u8) usize { + pub fn logicalIndexFromCoords(self: *const Mesh, coords: []const u8) usize { return self.physical.linearIndexFromCoords(coords); } - pub fn numPartitions(self: *const Data) i32 { + pub fn numPartitions(self: *const Mesh) i32 { return @intCast(self.physicalView().total_devices); } - pub fn numPartitionsForLogicalAxis(self: *const Data, logical_axis: anytype) i64 { + pub fn numPartitionsForLogicalAxis(self: *const Mesh, logical_axis: anytype) i64 { const logical_tag = Shape.toTag(logical_axis); - const bound_axes = self.binding(logical_tag) orelse return 1; + const logical_ax = self.resolveLogicalAxis(logical_tag) orelse std.debug.panic("mesh {f} has no axis {s}", .{ self, logical_tag }); + const bound_axes = self.binding(logical_ax); var physical_axes: std.EnumSet(PhysicalAxisTag) = .empty; for (bound_axes) |bound_axis| { @@ -1389,25 +1620,15 @@ pub const Data = struct { return partitions; } - pub fn numReplicas(_: *const Data) i32 { + pub fn numReplicas(_: *const Mesh) i32 { return 1; } - pub fn numDevices(self: *const Data) i32 { + pub fn numDevices(self: *const Mesh) i32 { return self.numPartitions() * self.numReplicas(); } - pub fn covers(sharding: *const Data, shape: Shape) bool { - for (shape._partitioning.slice()) |partitioning| { - switch (partitioning) { - .axis => |tag| if (sharding.binding(tag) == null) return false, - else => {}, - } - } - return true; - } - - pub fn sdyMeshAttr(self: *const Data, allocator: std.mem.Allocator) ![]const u8 { + pub fn sdyMeshAttr(self: *const Mesh, allocator: std.mem.Allocator) ![]const u8 { var out: std.Io.Writer.Allocating = .init(allocator); errdefer out.deinit(); @@ -1422,7 +1643,7 @@ pub const Data = struct { } const DimMapping = struct { - /// For each tensor dimension, the indices of physical axes (in PhysicalView) sharding it. + /// For each tensor dimension, the indices of physical axes (in PhysicalView) mesh it. axes_per_dim: stdx.BoundedArray(stdx.BoundedArray(usize, Shape.MAX_RANK), Shape.MAX_RANK), /// Indices of physical axes used for replication. replicated_axes: stdx.BoundedArray(usize, Shape.MAX_RANK), @@ -1431,7 +1652,7 @@ pub const Data = struct { }; /// Common logic to map tensor dimensions to physical mesh indices. - fn getDimMapping(self: *const Data, shape: Shape) DimMapping { + fn getDimMapping(self: *const Mesh, shape: Shape) DimMapping { const view = self.physicalView(); var axes_per_dim: stdx.BoundedArray(stdx.BoundedArray(usize, Shape.MAX_RANK), Shape.MAX_RANK) = .empty; var used_mask: [Shape.MAX_RANK]bool = @splat(false); @@ -1441,16 +1662,15 @@ pub const Data = struct { var dim_axes: stdx.BoundedArray(usize, Shape.MAX_RANK) = .empty; const spec = shape.partition(ax); - if (spec == .axis) { - if (self.binding(spec.axis)) |binding_| { - for (binding_) |p_tag| { - for (view.axes.slice(), 0..) |v_ax, i| { - // Only use the axis if it's bound and hasn't been consumed by a previous dimension - if (v_ax.contains(p_tag) and !globally_used.contains(v_ax.tag)) { - dim_axes.appendAssumeCapacity(i); - globally_used.insert(v_ax.tag); - used_mask[i] = true; - } + if (spec.meshAxis()) |mesh_ax| { + const binding_ = self.bindings.get(mesh_ax); + for (binding_.slice()) |p_tag| { + for (view.axes.slice(), 0..) |v_ax, i| { + // Only use the axis if it's bound and hasn't been consumed by a previous dimension + if (v_ax.contains(p_tag) and !globally_used.contains(v_ax.tag)) { + dim_axes.appendAssumeCapacity(i); + globally_used.insert(v_ax.tag); + used_mask[i] = true; } } } @@ -1471,7 +1691,7 @@ pub const Data = struct { } pub fn sdyShardingAttrForShape( - data: *const Data, + data: *const Mesh, parent_allocator: std.mem.Allocator, ctx: *mlir.Context, shape: Shape, @@ -1479,39 +1699,28 @@ pub const Data = struct { var arena = try stdx.arenaWithCapacity(parent_allocator, 1024); defer arena.deinit(); const allocator = arena.allocator(); - var any_explicit = false; - for (0..shape.rank()) |ax| { - if (shape.partition(ax) != .unknown) { - any_explicit = true; - break; - } - } - const all_replicated = !any_explicit; - const mapping = data.getDimMapping(shape); const dimensions = try allocator.alloc(*const dialects.shardy.DimensionShardingAttribute, shape.rank()); + // Note: here we map from the logical axis to physical. Is this a good idea ? + // why not stay in logical realm ? for (0.., dimensions) |ax, *d| { - const spec = shape.partition(ax); - d.* = switch (spec) { - .axis => |logical_tag| d: { - if (data.binding(logical_tag)) |_| { - const dim_phys_indices = mapping.axes_per_dim.get(ax); - if (dim_phys_indices.len == 0) { - break :d .replicated(ctx); - } else { - const axes = try allocator.alloc(*const dialects.shardy.AxisRefAttribute, dim_phys_indices.len); - for (dim_phys_indices.slice(), 0..) |p_idx, i| { - axes[i] = .named(ctx, @tagName(mapping.view.axes.get(p_idx).tag)); - } - break :d .closed(ctx, axes); - } + d.* = switch (shape.partition(ax)) { + .out_of_bound => @panic("undefined"), + .replicated => .replicated(ctx), + .open => .open(ctx, &.{}), + else => d: { + const dim_phys_indices = mapping.axes_per_dim.get(ax); + if (dim_phys_indices.len == 0) { + break :d .replicated(ctx); } else { - break :d .open(ctx, &.{}); + const axes = try allocator.alloc(*const dialects.shardy.AxisRefAttribute, dim_phys_indices.len); + for (dim_phys_indices.slice(), 0..) |p_idx, i| { + axes[i] = .named(ctx, @tagName(mapping.view.axes.get(p_idx).tag)); + } + break :d .closed(ctx, axes); } }, - .replicated => .replicated(ctx), - .open, .unknown => if (all_replicated) .replicated(ctx) else .open(ctx, &.{}), }; } @@ -1523,14 +1732,14 @@ pub const Data = struct { return .init(ctx, .{ .mesh = data.name, .dimensions = dimensions, .replicated_axes = replicated_axes }); } - pub fn gspmdShardingAttrForShape(self: *const Data, parent_allocator: std.mem.Allocator, ctx: *mlir.Context, shape: Shape) !*const mlir.Attribute { + pub fn gspmdShardingAttrForShape(self: *const Mesh, parent_allocator: std.mem.Allocator, ctx: *mlir.Context, shape: Shape) !*const mlir.Attribute { var arena = try stdx.arenaWithCapacity(parent_allocator, 1024); defer arena.deinit(); const allocator = arena.allocator(); var has_sharding = false; for (0..shape.rank()) |ax| { - if (shape.partition(ax) == .axis) { + if (shape.partition(ax).isSharded()) { has_sharding = true; break; } @@ -1562,9 +1771,7 @@ pub const Data = struct { var out: std.Io.Writer.Allocating = .init(allocator); errdefer out.deinit(); - // Emit explicit device assignment list to avoid XLA requiring - // a sharding attribute on the custom-call instruction. - const ids = try self.deviceAssignment(allocator); + const ids = try self.gspmdTileAssignment(allocator, shape); defer allocator.free(ids); try out.writer.writeAll("{devices=["); @@ -1591,12 +1798,36 @@ pub const Data = struct { return .string(ctx, try out.toOwnedSlice()); } - pub fn deviceAssignment(self: *const Data, allocator: std.mem.Allocator) ![]usize { - const view = self.physicalView(); - const count: usize = @intCast(view.total_devices); + fn gspmdTileAssignment(self: *const Mesh, allocator: std.mem.Allocator, shape: Shape) ![]u32 { + const placement = try Placement.init(shape); + const devices = self.devicesInCanonicalOrder(); + var tile_count: usize = 1; + for (placement.axis_plans.constSlice()) |plan| tile_count *= @intCast(plan.num_devices); + const replicas_per_tile = @divExact(devices.len, tile_count); + const replica_indices = try allocator.alloc(usize, tile_count); + defer allocator.free(replica_indices); + @memset(replica_indices, 0); + const ids = try allocator.alloc(u32, devices.len); - var ids = try allocator.alloc(usize, count); - @memset(ids, std.math.maxInt(usize)); + for (devices, 0..) |device, partition_index| { + var tile_index: usize = 0; + for (placement.axis_plans.constSlice()) |plan| { + tile_index = tile_index * @as(usize, @intCast(plan.num_devices)) + @as(usize, @intCast(plan.linearIndex(device.coords))); + } + // HLO shardings refer to logical partitions; executable device assignment + // separately maps those partitions to physical PJRT device IDs. + ids[tile_index * replicas_per_tile + replica_indices[tile_index]] = @intCast(partition_index); + replica_indices[tile_index] += 1; + } + return ids; + } + + pub fn deviceAssignment(self: *const Mesh, allocator: std.mem.Allocator) ![]u32 { + const view = self.physicalView(); + const count: u32 = @intCast(view.total_devices); + + var ids = try allocator.alloc(u32, count); + @memset(ids, std.math.maxInt(u32)); for (self.physical.devices_in_canonical_order) |d| { const coords = d.coords; @@ -1605,31 +1836,35 @@ pub const Data = struct { } for (ids) |id| { - if (id == std.math.maxInt(usize)) return error.MissingDeviceInTile; + if (id == std.math.maxInt(u32)) return error.MissingDeviceInTile; } return ids; } - pub fn format(self: Data, writer: *std.Io.Writer) !void { - try writer.print("Sharding(name={s})\n", .{self.name}); + /// Extract coordinate data from the physical mesh for a specific axis. + fn addPhysicalToSplit(mesh: *const Mesh, plan: *AxisSplit, tag: PhysicalAxisTag) void { + const physical = mesh.physical; + const info = physical.axisInfo(tag) orelse return; + const depth = physical.axis_traversal.depth(tag) orelse return; + plan.add(@intCast(info.size), depth); + } + + pub fn format(self: Mesh, writer: *std.Io.Writer) !void { + try writer.print("Mesh(name={s})\n", .{self.name}); try writer.writeAll("Bindings:\n"); - for (self.logical.axes.slice(), self.logical.intents.slice()) |l_tag, l_intent| { + for (self.logical.axes.slice(), self.logical.intents.slice(), self.bindings.slice()) |l_tag, l_intent, bind| { try writer.print(" - {s} ({s}) -> ", .{ l_tag, @tagName(l_intent) }); - if (self.binding(l_tag)) |axes| { - if (axes.len == 0) { - try writer.writeAll("replicated\n"); - } else { - for (axes, 0..) |p, i| { - if (i > 0) try writer.writeAll(", "); - try writer.writeAll(@tagName(p)); - } - try writer.writeAll("\n"); - } - } else { + if (bind.len == 0) { try writer.writeAll("unbound\n"); + } else { + for (bind.slice(), 0..) |physical, i| { + if (i > 0) try writer.writeAll(", "); + try writer.writeAll(@tagName(physical)); + } + try writer.writeAll("\n"); } } @@ -1665,19 +1900,19 @@ pub const Data = struct { }; pub const Strategy = struct { - pub const PhysicalList = stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK); - pub const Binding = struct { - logical: Shape.Tag, - physical: PhysicalList, - }; - pub const Bindings = stdx.BoundedArray(Strategy.Binding, MAX_MESH_RANK); pub const Fold = struct { target: PhysicalAxisTag, sources: PhysicalList, }; + pub const Binding = struct { + logical: Shape.Tag, + physical: PhysicalList, + }; + + pub const Bindings = stdx.BoundedArray(Binding, MAX_MESH_RANK); pub const Folding = stdx.BoundedArray(Strategy.Fold, MAX_MESH_RANK); - bindings: Strategy.Bindings, + bindings: Bindings, folding: Folding, // TODO: remove .init, promote .parseBindings to .init @@ -1784,16 +2019,19 @@ pub const Strategy = struct { }; /// For a given shape, compute the shape of the slice each shard will receive. -pub fn shardedShape(sharding: Sharding, shape: Shape) Error!Shape { - const pl = try sharding.placement(shape); - return pl.shape; +pub fn shardedShape(shape: Shape) error{IncompatibleSharding}!Shape { + if (shape._sharding.mesh != null) { + const pl: Placement = try .init(shape); + var local_shape = pl.shape; + local_shape._sharding = .replicated; + return local_shape; + } else { + // fully replicated + return shape; + } } -pub fn placement(sharding: Sharding, shape: Shape) Error!Placement { - return .init(sharding, shape); -} - -/// Precompute sharding information common to all devices for a given shape. +/// Precompute mesh information common to all devices for a given shape. pub const Placement = struct { pub const Slices = stdx.BoundedArray(Slice1d, Shape.MAX_RANK); pub const Slice1d = struct { @@ -1801,26 +2039,26 @@ pub const Placement = struct { size: i64, }; - sharding: Sharding, // Note this shape is mainly use to indicate the dims of the sharded buffer. // We may want to only store that. shape: Shape, global_shape: if (builtin.mode == .Debug) Shape else void, - axis_plans: stdx.BoundedArray(AxisSplit, Shape.MAX_RANK), - pub fn init(sharding: Sharding, shape: Shape) Error!Placement { + pub fn init(shape: Shape) error{IncompatibleSharding}!Placement { var pl: Placement = .{ - .sharding = sharding, .shape = shape, // modified below .global_shape = if (builtin.mode == .Debug) shape else {}, .axis_plans = .empty, // set below }; + // mesh == null => replicated => no axis plan + if (shape._sharding.mesh == null) return pl; + const mesh_ = shape._sharding.mesh.?; var used_axes: std.EnumSet(PhysicalAxisTag) = .empty; for (0..shape.rank()) |ax| { const axis_index: u8 = @intCast(ax); - pl.axis_plans.appendAssumeCapacity(try axisSplit(sharding, shape, &used_axes, axis_index)); + pl.axis_plans.appendAssumeCapacity(try axisSplit(mesh_, shape, &used_axes, axis_index)); } for (pl.shape._dims.slice(), shape.dims(), pl.axis_plans.slice()) |*shard_dim, dim, plan| { @@ -1829,8 +2067,16 @@ pub const Placement = struct { return pl; } + pub fn mesh(pl: *const Placement) ?*const Mesh { + return pl.shape._sharding.mesh; + } + pub fn slices(pl: *const Placement, device: Device.Coords) Slices { var res: Slices = .{ .buffer = undefined, .len = pl.shape.rank() }; + if (pl.mesh() == null) { + for (res.slice(), pl.shape.dims()) |*slice, dim| slice.* = .{ .start = 0, .size = dim }; + return res; + } for (res.slice(), pl.shape.dims(), pl.axis_plans.slice()) |*r, size, plan| { const start = plan.linearIndex(device) * size; r.* = .{ @@ -1848,6 +2094,8 @@ pub const Placement = struct { } var ptr: [*]const u8 = slice.constData().ptr; + // mesh = null => implies replicated => each device read from the start. + if (pl.shape._sharding.mesh == null) return ptr; for (pl.shape.dims(), pl.axis_plans.slice(), slice.byte_strides.slice()) |size, plan, stride| { const start = plan.linearIndex(device) * size; ptr = ptr[@intCast(start * stride)..]; @@ -1870,7 +2118,9 @@ pub const Placement = struct { } pub fn format(pl: Placement, writer: *std.Io.Writer) std.Io.Writer.Error!void { - const ordered_devices = pl.sharding.devicesInCanonicalOrder(); + const ordered_devices = if (pl.mesh()) |m| m.devicesInCanonicalOrder() else { + return try writer.print("Placement(shape={f}, replicated)\n", .{pl.shape}); + }; try pl.formatPlacementSummary(pl.shape, ordered_devices.len, writer); try writer.writeAll("Per-shard placement:\n"); @@ -1945,7 +2195,7 @@ const AxisSplit = struct { }; fn axisSplit( - sharding: Sharding, + mesh: *const Mesh, shape: Shape, used_axes: *std.EnumSet(PhysicalAxisTag), axis_index: u8, @@ -1953,37 +2203,34 @@ fn axisSplit( const dim = shape.dim(axis_index); const spec = shape.partition(axis_index); - switch (spec) { - .axis => |logical_tag| { - const binding = sharding.data.binding(logical_tag) orelse return error.MissingLogicalBinding; + return if (spec.meshAxis()) |mesh_axis| { + const binding = mesh.binding(mesh_axis); - // Calculate the split based on the physical coordinates of the current device. - return try calculateSplit(sharding, dim, binding, used_axes); - }, - else => return .empty, - } + // Calculate the split based on the physical coordinates of the current device. + return try calculateSplit(mesh, dim, binding, used_axes); + } else .empty; } fn calculateSplit( - sharding: Sharding, + mesh: *const Mesh, dim: i64, binding: []const PhysicalAxisTag, used_axes: *std.EnumSet(PhysicalAxisTag), -) !AxisSplit { +) error{IncompatibleSharding}!AxisSplit { var plan: AxisSplit = .empty; for (binding) |p_tag| { // Expand the physical tag: check if it's a target for folded source axes. - if (sharding.data.foldSources(p_tag)) |sources| { + if (mesh.foldSources(p_tag)) |sources| { for (sources) |src| { if (used_axes.contains(src)) continue; - addPhysicalToSplit(sharding, &plan, src); + mesh.addPhysicalToSplit(&plan, src); used_axes.insert(src); } } else { // Standard case: directly bound, not part of an explicit fold. if (used_axes.contains(p_tag)) continue; - addPhysicalToSplit(sharding, &plan, p_tag); + mesh.addPhysicalToSplit(&plan, p_tag); used_axes.insert(p_tag); } } @@ -1994,14 +2241,6 @@ fn calculateSplit( return plan; } -/// Extract coordinate data from the physical mesh for a specific axis. -fn addPhysicalToSplit(sharding: Sharding, plan: *AxisSplit, tag: PhysicalAxisTag) void { - const physical = sharding.data.physical; - const info = physical.axisInfo(tag) orelse return; - const depth = physical.axis_traversal.depth(tag) orelse return; - plan.add(@intCast(info.size), depth); -} - const ShardingTest = struct { pub const ExpectedShard = struct { // TODO: remove device_id from here, we always have ids from 0 to N @@ -2010,7 +2249,7 @@ const ShardingTest = struct { }; pub const Scenario = struct { - sharding: Sharding.Data, + mesh: *const Mesh, shape: Shape, expect_error: ?anyerror = null, @@ -2064,8 +2303,8 @@ const ShardingTest = struct { } pub fn run(self: ShardingTest, s: Scenario) !void { - const sharding = s.sharding; - errdefer std.log.warn("Failed sharding {f}", .{s}); + const mesh = s.mesh; + errdefer std.log.warn("Failed mesh {f}", .{s}); // Verify MLIR String if (s.expected_sdy) |expected_attr| { @@ -2079,7 +2318,7 @@ const ShardingTest = struct { defer ctx.deinit(); ctx.loadAllAvailableDialects(); - const actual_attr = try sharding.sdyShardingAttrForShape(self.allocator, ctx, s.shape); + const actual_attr = try mesh.sdyShardingAttrForShape(self.allocator, ctx, s.shape); var writer: std.Io.Writer.Allocating = .init(self.allocator); defer writer.deinit(); try actual_attr.asAttr().format(&writer.writer); @@ -2087,18 +2326,17 @@ const ShardingTest = struct { } // Verify Placement logic / Error - const sharding_ref: Sharding = .{ .data = &sharding }; - const ordered_devices = sharding_ref.devicesInCanonicalOrder(); + const ordered_devices = mesh.devicesInCanonicalOrder(); if (s.expect_error) |err| { errdefer std.log.warn("Expected error failed", .{}); try std.testing.expect(ordered_devices.len > 0); - try std.testing.expectError(err, sharding_ref.placement(s.shape)); + try std.testing.expectError(err, Sharding.Placement.init(s.shape)); return; } // Verify Shard Slices (Math) if (s.expected_shards.len > 0) { - const pl = try sharding_ref.placement(s.shape); + const pl: Placement = try .init(s.shape); errdefer std.log.warn("Expected shards failed. Got {f}", .{pl}); try std.testing.expectEqual(s.expected_shards.len, ordered_devices.len); for (s.expected_shards, ordered_devices) |expected, device| { @@ -2132,7 +2370,7 @@ const ShardingTest = struct { } }; -test "sharding: unknown partitioning implies replication" { +test "mesh: unknown partitioning implies replication" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const runner: ShardingTest = .init(arena.allocator()); @@ -2141,9 +2379,10 @@ test "sharding: unknown partitioning implies replication" { const logical: LogicalMesh = .mesh(.{ .batch = .low_bandwidth }); const strategy: Strategy = .parseBindings(.{ .batch = .link_x }); + const mesh: Sharding.Mesh = try .init("folded_mesh", &physical, logical, strategy); try runner.run(.{ - .sharding = try .init("folded_mesh", &physical, logical, strategy), + .mesh = &mesh, // No partitioning on shape .shape = Shape.init(.{ .batch = 8 }, .f32), .expected_sdy = "#sdy.sharding<@folded_mesh, [{}], replicated={\"link_x\", \"link_y\"}>", @@ -2156,7 +2395,7 @@ test "sharding: unknown partitioning implies replication" { }); } -test "sharding: suggest strategy realization" { +test "mesh: suggest strategy realization" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const runner: ShardingTest = .init(arena.allocator()); @@ -2169,11 +2408,12 @@ test "sharding: suggest strategy realization" { }); const strategy: Strategy = .suggest(logical, &physical); + const mesh: Mesh = try .init("suggested_mesh", &physical, logical, strategy); try runner.run(.{ - .sharding = try .init("suggested_mesh", &physical, logical, strategy), + .mesh = &mesh, .shape = Shape.init(.{ .batch = 4, .model = 4 }, .f32) - .withPartitioning(.{ .batch = .batch, .model = .model }), + .withPartitioning(&mesh, .{ .batch = .batch, .model = .model }), .expected_sdy = "#sdy.sharding<@suggested_mesh, [{\"link_z\"}, {\"link_x\"}], replicated={\"link_y\"}>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{ .{ 0, 2 }, .{ 0, 2 } } }, // x0, y0, z0 @@ -2188,7 +2428,7 @@ test "sharding: suggest strategy realization" { }); } -test "sharding: suggest folds logical axes with same intent" { +test "mesh: suggest folds logical axes with same intent" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const runner: ShardingTest = .init(arena.allocator()); @@ -2202,10 +2442,11 @@ test "sharding: suggest folds logical axes with same intent" { const strategy: Strategy = .suggest(logical, &physical); + const mesh: Mesh = try .init("fold_mesh", &physical, logical, strategy); try runner.run(.{ - .sharding = try .init("fold_mesh", &physical, logical, strategy), + .mesh = &mesh, .shape = Shape.init(.{ .model = 4, .experts = 4 }, .f32) - .withPartitioning(.{ .model = .model, .experts = .experts }), + .withPartitioning(&mesh, .{ .model = .model, .experts = .experts }), .expected_sdy = "#sdy.sharding<@fold_mesh, [{\"link_x\"}, {}]>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{ .{ 0, 1 }, .{ 0, 4 } } }, @@ -2216,7 +2457,7 @@ test "sharding: suggest folds logical axes with same intent" { }); } -test "sharding: multiple physical axes on one logical dimension (folding)" { +test "mesh: multiple physical axes on one logical dimension (folding)" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const runner: ShardingTest = .init(arena.allocator()); @@ -2226,9 +2467,10 @@ test "sharding: multiple physical axes on one logical dimension (folding)" { { const strategy: Strategy = .parseBindings(.{ .model = .{ .link_x, .link_y } }); + const mesh: Mesh = try .init("folded_mesh", &physical, logical, strategy); try runner.run(.{ - .sharding = try .init("folded_mesh", &physical, logical, strategy), - .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(.{ .model = .model }), + .mesh = &mesh, + .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(&mesh, .{ .model = .model }), .expected_sdy = "#sdy.sharding<@folded_mesh, [{\"link_x\", \"link_y\"}]>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, @@ -2243,9 +2485,10 @@ test "sharding: multiple physical axes on one logical dimension (folding)" { // Swap link_x and link_y order: // -> the roles of devices 1 (0,1) and 2 (1,0) are swapped const strategy: Strategy = .parseBindings(.{ .model = .{ .link_y, .link_x } }); + const mesh: Mesh = try .init("folded_mesh", &physical, logical, strategy); try runner.run(.{ - .sharding = try .init("folded_mesh", &physical, logical, strategy), - .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(.{ .model = .model }), + .mesh = &mesh, + .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(&mesh, .{ .model = .model }), .expected_sdy = "#sdy.sharding<@folded_mesh, [{\"link_y\", \"link_x\"}]>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, @@ -2257,7 +2500,7 @@ test "sharding: multiple physical axes on one logical dimension (folding)" { } } -test "sharding: explicit strategy folding" { +test "mesh: explicit strategy folding" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const runner: ShardingTest = .init(arena.allocator()); @@ -2268,9 +2511,10 @@ test "sharding: explicit strategy folding" { var strategy: Strategy = .parseBindings(.{ .model = .link_x }); strategy.addFold(.link_x, &.{ .link_x, .link_z }); + const mesh: Mesh = try .init("strategy_fold", &physical, logical, strategy); try runner.run(.{ - .sharding = try .init("strategy_fold", &physical, logical, strategy), - .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(.{ .model = .model }), + .mesh = &mesh, + .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(&mesh, .{ .model = .model }), .expected_sdy = "#sdy.sharding<@strategy_fold, [{\"link_x\"}], replicated={\"link_y\"}>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, @@ -2285,7 +2529,7 @@ test "sharding: explicit strategy folding" { }); } -test "sharding: open and replicated dimension mix" { +test "mesh: open and replicated dimension mix" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const runner: ShardingTest = .init(arena.allocator()); @@ -2295,9 +2539,10 @@ test "sharding: open and replicated dimension mix" { const strategy: Strategy = .parseBindings(.{ .batch = .link_x, .model = .link_y }); + const mesh: Mesh = try .init("mix_mesh", &physical, logical, strategy); try runner.run(.{ - .sharding = try .init("mix_mesh", &physical, logical, strategy), - .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(.{ .batch = .open, .model = .replicated }), + .mesh = &mesh, + .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(&mesh, .{ .batch = .open, .model = .replicated }), .expected_sdy = "#sdy.sharding<@mix_mesh, [{?}, {}], replicated={\"link_x\", \"link_y\"}>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, @@ -2308,7 +2553,7 @@ test "sharding: open and replicated dimension mix" { }); } -test "sharding: full 3D cluster sharding" { +test "mesh: full 3D cluster mesh" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const runner: ShardingTest = .init(arena.allocator()); @@ -2322,11 +2567,12 @@ test "sharding: full 3D cluster sharding" { .context = .link_z, }); + const mesh: Mesh = try .init("3d_mesh", &physical, logical, strategy); try runner.run(.{ - .sharding = try .init("3d_mesh", &physical, logical, strategy), + .mesh = &mesh, // Note: .context and .model are swapped compared to .link_x and .link_z .shape = Shape.init(.{ .batch = 4, .context = 4, .model = 4 }, .f32) - .withPartitioning(.{ .batch = .batch, .model = .model, .context = .context }), + .withPartitioning(&mesh, .{ .batch = .batch, .model = .model, .context = .context }), .expected_sdy = "#sdy.sharding<@\"3d_mesh\", [{\"link_x\"}, {\"link_z\"}, {\"link_y\"}]>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{ .{ 0, 2 }, .{ 0, 2 }, .{ 0, 2 } } }, @@ -2341,7 +2587,7 @@ test "sharding: full 3D cluster sharding" { }); } -test "sharding: num partitions for logical axis" { +test "mesh: num partitions for logical axis" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const runner: ShardingTest = .init(arena.allocator()); @@ -2355,8 +2601,35 @@ test "sharding: num partitions for logical axis" { var strategy: Strategy = .parseBindings(.{ .model = .link_x }); strategy.addFold(.link_x, &.{ .link_x, .link_z }); - const sharding: Sharding.Data = try .init("axis_parts_mesh", &physical, logical, strategy); + const mesh: Mesh = try .init("axis_parts_mesh", &physical, logical, strategy); - try std.testing.expectEqual(4, sharding.numPartitionsForLogicalAxis(.model)); - try std.testing.expectEqual(1, sharding.numPartitionsForLogicalAxis(.batch)); + try std.testing.expectEqual(4, mesh.numPartitionsForLogicalAxis(.model)); + try std.testing.expectEqual(1, mesh.numPartitionsForLogicalAxis(.batch)); +} + +test "localized shapes do not retain global sharding" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{2}, .{ .mesh = .torus }); + const mesh_ = try Mesh.init("local", &physical, .mesh(.{ .batch = .balanced }), .parseBindings(.{ .batch = .link_x })); + try std.testing.expectEqual(mesh_.bindings.constSlice()[0].constSlice().ptr, mesh_.binding(0).ptr); + const global = Shape.init(.{ .batch = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .batch }); + const local = try shardedShape(global); + try std.testing.expectEqual(4, local.dim(.batch)); + try std.testing.expectEqual(null, local._sharding.mesh); + try std.testing.expectEqual(8, global.dim(.batch)); + try std.testing.expectEqual(&mesh_, global._sharding.mesh); +} + +test "GSPMD tiles use tensor axis order and logical partition indices" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + for (physical.devices_in_canonical_order, 0..) |*device, i| device.id = @intCast(7 - i); + const mesh_ = try Mesh.init("tiles", &physical, .mesh(.{ .batch = .balanced, .model = .balanced }), .parseBindings(.{ .batch = .link_z, .model = .link_x })); + const shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .batch, .model = .model }); + const ids = try mesh_.gspmdTileAssignment(arena.allocator(), shape); + try std.testing.expectEqualSlices(u32, &.{ 0, 2, 4, 6, 1, 3, 5, 7 }, ids); } diff --git a/zml/attention.zig b/zml/attention.zig index b3701b48..f5399c5e 100644 --- a/zml/attention.zig +++ b/zml/attention.zig @@ -116,22 +116,22 @@ pub const Metadata = union(Backend) { } }; - pub fn init(opts: InitOptions) Metadata { + pub fn init(opts: InitOptions, mesh: *const zml.Sharding.Mesh) Metadata { return switch (opts) { .vanilla => .{ .vanilla = {} }, .attnd => @panic("Must be initialized manually"), .nki => .{ .nki = {} }, - .cuda_fa2 => |o| .{ .cuda_fa2 = flashattn.fa2.Metadata.init(o) }, - .cuda_fa3 => |o| .{ .cuda_fa3 = flashattn.fa3.Metadata.init(o) }, + .cuda_fa2 => |o| .{ .cuda_fa2 = .init(o, mesh) }, + .cuda_fa3 => |o| .{ .cuda_fa3 = .init(o, mesh) }, .metal_fa => .{ .metal_fa = .init() }, }; } - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(Metadata) { + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { return switch (self) { .vanilla => .{ .vanilla = {} }, .nki => .{ .nki = {} }, - inline else => |v, tag| @unionInit(zml.Bufferized(Metadata), @tagName(tag), try v.initBuffer(io, platform, sharding)), + inline else => |v, tag| @unionInit(zml.Bufferized(Metadata), @tagName(tag), try v.initBuffer(io, platform)), }; } @@ -179,6 +179,31 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T }; } +test "FlashAttention metadata initializes with explicit sharding outside compilation" { + try std.testing.expect(zml.Compiler.currentOrNull() == null); + const mesh: zml.Sharding.Mesh = .{ + .name = "metadata_mesh", + .physical = undefined, + .logical = .mesh(.{ .model = .high_bandwidth }), + .bindings = .init(&.{.init(&.{.link_x})}), + .folds = .empty, + .folds_consumed = .empty, + }; + inline for (.{ Backend.cuda_fa2, Backend.cuda_fa3 }) |backend| { + const metadata: Metadata = .init(.fromBackend(backend, 16, 8), &mesh); + const cuda_metadata = @field(metadata, @tagName(backend)); + inline for (std.meta.fields(@TypeOf(cuda_metadata))) |field| { + const shape = @field(cuda_metadata, field.name).shape(); + try std.testing.expectEqual(&mesh, shape._sharding.mesh); + if (shape.hasTag(.h)) |axis| { + try std.testing.expectEqual(zml.Sharding.PartitionSpec.sharded(0), shape.partition(axis)); + } else { + try std.testing.expectEqual(zml.Sharding.PartitionSpec.replicated, shape.partition(.meta)); + } + } + } +} + test "attention: q=1,qh=64,kh=8" { try testAttention( .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), @@ -270,13 +295,13 @@ pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []co defer k.deinit(); var v = try rng_k.eval(allocator, io, {}); defer v.deinit(); - var token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, .replicated, @ptrCast(token_index_h)); + var token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, @ptrCast(token_index_h)); defer token_index.deinit(); - const shardings = platform.shardings.values(); + const meshes = platform.meshes.values(); const vanilla_exe = try platform.compileFn(allocator, io, attention, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, .vanilla, .vanilla }, .{ .program_name = "attention_vanilla", - .shardings = shardings, + .meshes = meshes, }); defer vanilla_exe.deinit(); @@ -292,7 +317,7 @@ pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []co else => if (!backend.isAvailable(platform)) continue, } - const metadata: Metadata = .init(.fromBackend(backend, tensors.k.dim(.k), tensors.q.dim(.h))); + const metadata: Metadata = .init(.fromBackend(backend, tensors.k.dim(.k), tensors.q.dim(.h)), platform.meshes.get("model").?); const parameters: Parameters = .init(.fromBackend(backend)); const exe = try platform.compileFn( allocator, @@ -301,12 +326,12 @@ pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []co .{ tensors.q, tensors.k, tensors.v, tensors.token_index, metadata, parameters }, .{ .program_name = try std.fmt.allocPrint(arena, "attention_{t}", .{backend}), - .shardings = shardings, + .meshes = meshes, }, ); defer exe.deinit(); - var metadata_d = try metadata.initBuffer(io, platform, platform.shardings.get("model").?); + var metadata_d = try metadata.initBuffer(io, platform); defer Metadata.deinitBuffer(&metadata_d); var output_d = try exe.eval(allocator, io, .{ q, k, v, token_index, metadata_d }); diff --git a/zml/attention/attnd.zig b/zml/attention/attnd.zig index b1c39cd1..a986ab7b 100644 --- a/zml/attention/attnd.zig +++ b/zml/attention/attnd.zig @@ -74,12 +74,11 @@ pub const Metadata = struct { self: Metadata, io: std.Io, platform: *const zml.Platform, - sharding: zml.Sharding, ) !zml.Bufferized(Metadata) { return .{ .conversation_id = try zml.Buffer.scalar(io, platform, 749, .u64), - .layer_id = try zml.Buffer.uninitialized(io, platform, self.layer_id.shape(), sharding, .{}), - .num_tokens = try zml.Buffer.uninitialized(io, platform, self.num_tokens.shape(), sharding, .{}), + .layer_id = try zml.Buffer.uninitialized(io, platform, self.layer_id.shape(), .{}), + .num_tokens = try zml.Buffer.uninitialized(io, platform, self.num_tokens.shape(), .{}), }; } @@ -140,8 +139,8 @@ const Context = struct { }; pub fn causalAttention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_offset: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { - const ctx = zml.Compiler.current(); - const num_partitions = ctx.partitioning.numPartitionsForLogicalAxis(q.shape(), .model) catch unreachable; + const tp = zml.Compiler.current().mesh(.model); + const num_partitions = tp.numPartitionsForLogicalAxis(.model); const actual_k, const actual_v = if (parameters.is_prefill) .{ k, v } diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index f533b008..dd010217 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -236,27 +236,24 @@ pub const fa2 = struct { num_heads: i64, }; - pub fn init(opts: InitOptions) Metadata { + pub fn init(opts: InitOptions, mesh: *const zml.Sharding.Mesh) Metadata { return .{ - .softmax_lse = .fromShape(zml.Shape.init(.{ opts.seqlen, opts.num_heads, 1 }, .f32) - .withTags(.{ .s, .h, .dummy }) - .withPartitioning(.{ .h = .model })), + .softmax_lse = zml.Tensor.init(.{ .s = opts.seqlen, .h = opts.num_heads, .dummy = 1 }, .f32) + .withPartitioning(mesh, .{ .h = .model }), - .softmax_lse_accum = .fromShape(zml.Shape.init(.{ 1, opts.num_heads, 128 }, .f32) - .withTags(.{ .dummy, .h, .hd }) - .withPartitioning(.{ .h = .model })), + .softmax_lse_accum = zml.Tensor.init(.{ .dummy = 1, .h = opts.num_heads, .hd = 128 }, .f32) + .withPartitioning(mesh, .{ .h = .model }), - .out_accum = .fromShape(zml.Shape.init(.{ opts.seqlen, opts.num_heads, 128 }, .f32) - .withTags(.{ .s, .h, .hd }) - .withPartitioning(.{ .h = .model })), + .out_accum = zml.Tensor.init(.{ .s = opts.seqlen, .h = opts.num_heads, .hd = 128 }, .f32) + .withPartitioning(mesh, .{ .h = .model }), }; } - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(Metadata) { + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { return .{ - .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), sharding, .{}), - .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), sharding, .{}), - .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), sharding, .{}), + .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), .{}), + .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), .{}), + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), }; } @@ -313,20 +310,21 @@ pub const fa2 = struct { .merge(.{ .tot = .{ .tot, .ngroups } }); } - const q_sharded = q.withPartitioning(.{ .h = .model }); - const model_partitions: i32 = @intCast(ctx.partitioning.numPartitionsForLogicalAxis(q_sharded.shape(), .model) catch std.debug.panic("cu_fa2 attention backend requires a .model sharding", .{})); + const attn_sdy = ctx.resolveMesh(.{.model}); + const q_sharded = q.withPartitioning(attn_sdy, .{ .h = .model }); + const model_partitions: i32 = @intCast(attn_sdy.numPartitionsForLogicalAxis(.model)); const output = fa2_mha_varlen_fwd.call( .{ .q = q_sharded, - .k = k.withPartitioning(.{ .h = .model }), - .v = v.withPartitioning(.{ .h = .model }), + .k = k.withPartitioning(attn_sdy, .{ .h = .model }), + .v = v.withPartitioning(attn_sdy, .{ .h = .model }), .cu_seqlens_q = cu_seqlens_q, .cu_seqlens_k = cu_seqlens_k, .seqused_k = seqused_k, - .softmax_lse = metadata.softmax_lse.withPartitioning(.{ .h = .model }), - .softmax_lse_accum = metadata.softmax_lse_accum.withPartitioning(.{ .h = .model }), - .out_accum = metadata.out_accum.withPartitioning(.{ .h = .model }), + .softmax_lse = metadata.softmax_lse.withPartitioning(attn_sdy, .{ .h = .model }), + .softmax_lse_accum = metadata.softmax_lse_accum.withPartitioning(attn_sdy, .{ .h = .model }), + .out_accum = metadata.out_accum.withPartitioning(attn_sdy, .{ .h = .model }), }, .{ .o = q_sharded.shape(), @@ -451,25 +449,25 @@ pub const fa3 = struct { num_heads: i64, }; - pub fn init(opts: InitOptions) Metadata { + pub fn init(opts: InitOptions, mesh: *const zml.Sharding.Mesh) Metadata { return .{ - .softmax_lse = .fromShape(zml.Shape.init(.{opts.num_heads * opts.seqlen * 4}, .i8) - .withTags(.{.h}).withPartitioning(.{ .h = .model })), - .softmax_lse_accum = .fromShape(zml.Shape.init(.{opts.num_heads * 128 * 4}, .i8) - .withTags(.{.h}).withPartitioning(.{ .h = .model })), - .out_accum = .fromShape(zml.Shape.init(.{opts.num_heads * opts.seqlen * 128 * 4}, .i8) - .withTags(.{.h}).withPartitioning(.{ .h = .model })), - .scheduler_metadata = .fromShape(zml.Shape.init(.{2}, .i32) - .withTags(.{.meta}).withPartitioning(.{ .meta = .replicated })), + .softmax_lse = zml.Tensor.init(.{ .h = opts.num_heads * opts.seqlen * 4 }, .i8) + .withPartitioning(mesh, .{ .h = .model }), + .softmax_lse_accum = zml.Tensor.init(.{ .h = opts.num_heads * 128 * 4 }, .i8) + .withPartitioning(mesh, .{ .h = .model }), + .out_accum = zml.Tensor.init(.{ .h = opts.num_heads * opts.seqlen * 128 * 4 }, .i8) + .withPartitioning(mesh, .{ .h = .model }), + .scheduler_metadata = zml.Tensor.init(.{ .meta = 2 }, .i32) + .withPartitioning(mesh, .{ .meta = .replicated }), }; } - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(Metadata) { + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { return .{ - .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), sharding, .{}), - .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), sharding, .{}), - .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), sharding, .{}), - .scheduler_metadata = try zml.Buffer.uninitialized(io, platform, self.scheduler_metadata.shape(), sharding, .{}), + .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), .{}), + .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), .{}), + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + .scheduler_metadata = try zml.Buffer.uninitialized(io, platform, self.scheduler_metadata.shape(), .{}), }; } @@ -496,7 +494,7 @@ pub const fa3 = struct { const v = v_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); // TODO(Corendos): replace with cumsum const cu_seqlens_q = zml.Tensor.constantTensor(zml.Shape.init(.{2}, .i32), std.mem.sliceAsBytes(&[2]i32{ 0, max_seqlen_q })) - .withPartitioning(.{ ._0 = .replicated }); + .withPartitioning(.model, .{ ._0 = .replicated }); var o = zml.ops.customCall( custom_call_name, @@ -909,15 +907,18 @@ pub const paged_fa2 = struct { const head_dim = q.dim(.hd); const num_heads = num_head_groups * num_kv_heads; // FIXME: remove unreachable and propagate error correctly. - const num_heads_per_shard = @divExact(num_heads, ctx.partitioning.numPartitionsForLogicalAxis(q.shape(), .model) catch unreachable); + + const attn_sdy = ctx.resolveMesh(.{.model}); + const model_partitions: i32 = @intCast(attn_sdy.numPartitionsForLogicalAxis(.model)); + const num_heads_per_shard = @divExact(num_heads, model_partitions); const o = switch (parameters) { .decode => |decode_parameters| b: { const seqlenq_ngroups_swapped = num_heads > num_kv_heads and @mod(head_dim, 8) == 0 and opts.mask != .sliding_window; - const block_table = decode_parameters.block_table.withPartitioning(.{ .b = .replicated }); - const cu_seqlens_q = decode_parameters.cu_seqlens_q.withPartitioning(.{ .b = .replicated }); - const seqused_k = decode_parameters.seqused_k.withPartitioning(.{ .b = .replicated }); + const block_table = decode_parameters.block_table.withPartitioning(.model, .{ .b = .replicated }); + const cu_seqlens_q = decode_parameters.cu_seqlens_q.withPartitioning(.model, .{ .b = .replicated }); + const seqused_k = decode_parameters.seqused_k.withPartitioning(.model, .{ .b = .replicated }); const out_accum = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, @@ -925,27 +926,27 @@ pub const paged_fa2 = struct { .hg = num_head_groups, .b = q.dim(.b), .hd = head_dim, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse = zml.Tensor.uninitialized(.init(.{ .hkv = num_kv_heads, .hg = num_head_groups, .b = q.dim(.b), - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_accum = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, .hkv = num_kv_heads, .hg = num_head_groups, .b = q.dim(.b), - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const dummy_cu_seqlens_k: zml.Tensor = .zeroes(cu_seqlens_q.shape()); const batch_dim = q.dim(.b); var q2 = q; if (seqlenq_ngroups_swapped) { - q2 = q2.transpose(.{ .b, .hg, .hkv, .hd }).merge(.{ .b = .{ .b, .hg } }).withPartitioning(.{ .hkv = .model }); + q2 = q2.transpose(.{ .b, .hg, .hkv, .hd }).merge(.{ .b = .{ .b, .hg } }).withPartitioning(.model, .{ .hkv = .model }); } else { - q2 = q2.transpose(.{ .b, .hkv, .hg, .hd }).merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + q2 = q2.transpose(.{ .b, .hkv, .hg, .hd }).merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); } const output_shape = q2.shape(); @@ -1003,12 +1004,12 @@ pub const paged_fa2 = struct { .mixed => |mixed_parameters| b: { const seqlenq_ngroups_swapped = num_heads > num_kv_heads and @mod(head_dim, 8) == 0 and opts.mask != .sliding_window; - const block_table_prefill = mixed_parameters.block_table_prefill.withPartitioning(.{ .b = .replicated }); - const cu_seqlens_q_prefill = mixed_parameters.cu_seqlens_q_prefill.withPartitioning(.{ .b = .replicated }); - const seqused_k_prefill = mixed_parameters.seqused_k_prefill.withPartitioning(.{ .b = .replicated }); - const block_table_decode = mixed_parameters.block_table_decode.withPartitioning(.{ .b = .replicated }); - const cu_seqlens_q_decode = mixed_parameters.cu_seqlens_q_decode.withPartitioning(.{ .b = .replicated }); - const seqused_k_decode = mixed_parameters.seqused_k_decode.withPartitioning(.{ .b = .replicated }); + const block_table_prefill = mixed_parameters.block_table_prefill.withPartitioning(.model, .{ .b = .replicated }); + const cu_seqlens_q_prefill = mixed_parameters.cu_seqlens_q_prefill.withPartitioning(.model, .{ .b = .replicated }); + const seqused_k_prefill = mixed_parameters.seqused_k_prefill.withPartitioning(.model, .{ .b = .replicated }); + const block_table_decode = mixed_parameters.block_table_decode.withPartitioning(.model, .{ .b = .replicated }); + const cu_seqlens_q_decode = mixed_parameters.cu_seqlens_q_decode.withPartitioning(.model, .{ .b = .replicated }); + const seqused_k_decode = mixed_parameters.seqused_k_decode.withPartitioning(.model, .{ .b = .replicated }); const out_accum_prefill = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, @@ -1016,22 +1017,22 @@ pub const paged_fa2 = struct { .hg = num_head_groups, .b = q.dim(.b), .hd = head_dim, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_prefill = zml.Tensor.uninitialized(.init(.{ .hkv = num_kv_heads, .hg = num_head_groups, .b = q.dim(.b), - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_accum_prefill = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, .hkv = num_kv_heads, .hg = num_head_groups, .b = q.dim(.b), - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const dummy_cu_seqlens_k_prefill: zml.Tensor = .zeroes(cu_seqlens_q_prefill.shape()); var q2 = q; - q2 = q2.transpose(.{ .b, .hkv, .hg, .hd }).merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + q2 = q2.transpose(.{ .b, .hkv, .hg, .hd }).merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); const output_shape = q2.shape(); var o = zml.ops.manualComputation( @@ -1088,25 +1089,25 @@ pub const paged_fa2 = struct { .hg = num_head_groups, .b = batch_dim_decode, .hd = head_dim, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_decode = zml.Tensor.uninitialized(.init(.{ .hkv = num_kv_heads, .hg = num_head_groups, .b = batch_dim_decode, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_accum_decode = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, .hkv = num_kv_heads, .hg = num_head_groups, .b = batch_dim_decode, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const dummy_cu_seqlens_k_decode = zml.Tensor.zeroes(cu_seqlens_q_decode.shape()); var q_decode = q.slice(0, .dyn(mixed_parameters.metadata.decode_offset, batch_dim_decode)); if (seqlenq_ngroups_swapped) { - q_decode = q_decode.transpose(.{ .b, .hg, .hkv, .hd }).merge(.{ .b = .{ .b, .hg } }).withPartitioning(.{ .hkv = .model }); + q_decode = q_decode.transpose(.{ .b, .hg, .hkv, .hd }).merge(.{ .b = .{ .b, .hg } }).withPartitioning(.model, .{ .hkv = .model }); } else { - q_decode = q_decode.transpose(.{ .b, .hkv, .hg, .hd }).merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + q_decode = q_decode.transpose(.{ .b, .hkv, .hg, .hd }).merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); } const output_shape_decode = q_decode.shape(); @@ -1563,9 +1564,9 @@ pub const paged_fa3 = struct { .decode => |decode_parameters| b: { const batch_size = decode_parameters.block_table.dim(0); - const block_table = decode_parameters.block_table.withPartitioning(.{ .b = .replicated }); - const cu_seqlens_q = decode_parameters.cu_seqlens_q.withPartitioning(.{ .b = .replicated }); - const seqused_k = decode_parameters.seqused_k.withPartitioning(.{ .b = .replicated }); + const block_table = decode_parameters.block_table.withPartitioning(.model, .{ .b = .replicated }); + const cu_seqlens_q = decode_parameters.cu_seqlens_q.withPartitioning(.model, .{ .b = .replicated }); + const seqused_k = decode_parameters.seqused_k.withPartitioning(.model, .{ .b = .replicated }); const out_accum = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, @@ -1573,21 +1574,21 @@ pub const paged_fa3 = struct { .hg = num_head_groups, .b = q.dim(.b), .hd = head_dim, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse = zml.Tensor.uninitialized(.init(.{ .hkv = num_kv_heads, .hg = num_head_groups, .q = q.dim(.b), - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_accum = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, .q = q.dim(.b), .hkv = num_kv_heads, .hg = num_head_groups, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const scheduler_metadata = zml.Tensor.zeroes(.init(.{ .b = batch_size + 1 }, .i32)); - var q2 = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + var q2 = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); const output_shape = q2.shape(); var o = zml.ops.manualComputation( @@ -1636,12 +1637,12 @@ pub const paged_fa3 = struct { .mixed => |mixed_parameters| b: { const batch_size_prefill = mixed_parameters.block_table_prefill.dim(0); - const block_table_prefill = mixed_parameters.block_table_prefill.withPartitioning(.{ .b = .replicated }); - const cu_seqlens_q_prefill = mixed_parameters.cu_seqlens_q_prefill.withPartitioning(.{ .b = .replicated }); - const seqused_k_prefill = mixed_parameters.seqused_k_prefill.withPartitioning(.{ .b = .replicated }); - const block_table_decode = mixed_parameters.block_table_decode.withPartitioning(.{ .b = .replicated }); - const cu_seqlens_q_decode = mixed_parameters.cu_seqlens_q_decode.withPartitioning(.{ .b = .replicated }); - const seqused_k_decode = mixed_parameters.seqused_k_decode.withPartitioning(.{ .b = .replicated }); + const block_table_prefill = mixed_parameters.block_table_prefill.withPartitioning(.model, .{ .b = .replicated }); + const cu_seqlens_q_prefill = mixed_parameters.cu_seqlens_q_prefill.withPartitioning(.model, .{ .b = .replicated }); + const seqused_k_prefill = mixed_parameters.seqused_k_prefill.withPartitioning(.model, .{ .b = .replicated }); + const block_table_decode = mixed_parameters.block_table_decode.withPartitioning(.model, .{ .b = .replicated }); + const cu_seqlens_q_decode = mixed_parameters.cu_seqlens_q_decode.withPartitioning(.model, .{ .b = .replicated }); + const seqused_k_decode = mixed_parameters.seqused_k_decode.withPartitioning(.model, .{ .b = .replicated }); const out_accum_prefill = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, @@ -1649,21 +1650,21 @@ pub const paged_fa3 = struct { .hg = num_head_groups, .b = q.dim(.b), .hd = head_dim, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_prefill = zml.Tensor.uninitialized(.init(.{ .hkv = num_kv_heads, .hg = num_head_groups, .q = q.dim(.b), - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_accum_prefill = zml.Tensor.uninitialized(.init(.{ .splits = MAX_NUM_SPLITS, .q = q.dim(.b), .hkv = num_kv_heads, .hg = num_head_groups, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const scheduler_metadata_prefill = zml.Tensor.zeroes(.init(.{ .b = batch_size_prefill + 1 }, .i32)); - var q2 = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + var q2 = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); const output_shape = q2.shape(); var o = zml.ops.manualComputation( @@ -1714,7 +1715,7 @@ pub const paged_fa3 = struct { .hg = num_head_groups, .b = batch_size_decode, .hd = head_dim, - }, .f32)).withPartitioning(.{ .hkv = .model }); + }, .f32)).withPartitioning(.model, .{ .hkv = .model }); const softmax_lse_decode = zml.Tensor.uninitialized(.init(.{ .hkv = num_kv_heads, .hg = num_head_groups, @@ -1727,9 +1728,9 @@ pub const paged_fa3 = struct { .hg = num_head_groups, }, .f32)); const scheduler_metadata_decode = zml.Tensor.zeroes(.init(.{ .b = batch_size_decode + 1 }, .i32)); - var q_decode = q.slice(0, .dyn(mixed_parameters.metadata.decode_offset, batch_size_decode)).withPartitioning(.{ .hkv = .model }); + var q_decode = q.slice(0, .dyn(mixed_parameters.metadata.decode_offset, batch_size_decode)).withPartitioning(.model, .{ .hkv = .model }); - q_decode = q_decode.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + q_decode = q_decode.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); const decode_output_shape = q_decode.shape(); var o_decode = zml.ops.manualComputation( diff --git a/zml/attention/metal_attention.zig b/zml/attention/metal_attention.zig index 11942afe..42821a1f 100644 --- a/zml/attention/metal_attention.zig +++ b/zml/attention/metal_attention.zig @@ -14,9 +14,8 @@ pub const Metadata = struct { return .{ .num_tokens = .fromShape(.scalar(.u32)) }; } - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform, sharding: zml.Sharding) !zml.Bufferized(Metadata) { + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { _ = self; - _ = sharding; return .{ .num_tokens = try zml.Buffer.scalar(io, platform, 0, .u32) }; } diff --git a/zml/attention/nki/attention.zig b/zml/attention/nki/attention.zig index 6553b463..cdbad308 100644 --- a/zml/attention/nki/attention.zig +++ b/zml/attention/nki/attention.zig @@ -43,13 +43,14 @@ pub const Parameters = struct { /// Attention for Neuron. Both prefill and decode lower to NKI custom calls. pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, parameters: Parameters) zml.Tensor { - const q_sharded = q.withPartitioning(.{ .q = .replicated, .h = .model, .hd = .replicated }); - const k_sharded = k.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); - const v_sharded = v.withPartitioning(.{ .k = .replicated, .h = .model, .hd = .replicated }); + const tp_mesh = zml.Compiler.current().mesh(.model); + const q_sharded = q.withPartitioning(tp_mesh, .{ .q = .replicated, .h = .model, .hd = .replicated }); + const k_sharded = k.withPartitioning(tp_mesh, .{ .k = .replicated, .h = .model, .hd = .replicated }); + const v_sharded = v.withPartitioning(tp_mesh, .{ .k = .replicated, .h = .model, .hd = .replicated }); if (q.dim(.q) == 1) { const token_index_2d = token_index.broad(zml.Shape.init(.{ .row = 1, .col = 1 }, token_index.dtype())) - .withPartitioning(.{ .row = .replicated, .col = .replicated }); + .withPartitioning(tp_mesh, .{ .row = .replicated, .col = .replicated }); return zml.ops.manualComputation( (struct { diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 97960240..403b5a97 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -339,8 +339,8 @@ test "Backend.auto selects mosaic_tpu on TPU" { .devices = &.{}, .memories = &.{}, .physical_mesh = undefined, - .replicated_sharding = undefined, - .shardings = .empty, + .replicated_mesh = undefined, + .meshes = .empty, .io_impl = .threaded, }; @@ -357,8 +357,8 @@ test "Backend.auto selects triton on oneAPI" { .devices = &.{}, .memories = &.{}, .physical_mesh = undefined, - .replicated_sharding = undefined, - .shardings = .empty, + .replicated_mesh = undefined, + .meshes = .empty, .io_impl = .threaded, }; @@ -383,25 +383,26 @@ test pagedAttention { const batch_size = active_batch_size + 1; const query_token_count = prefill_token_count + num_decode; const dt: zml.DataType = .bf16; + const model_meshe = platform.meshes.get("model").?; const partition = .{ .hkv = .model }; const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache } = .{ - .q = .withPartitioning(.init(.{ .b = query_token_count, .hkv = 4, .hg = 4, .hd = 32 }, dt), partition), - .k = .withPartitioning(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt), partition), - .v = .withPartitioning(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt), partition), + .q = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hg = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), + .k = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), + .v = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), .kv_cache = .{ .split = .{ - .k = .withPartitioning(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt), partition), - .v = .withPartitioning(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt), partition), + .k = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), + .v = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), }, }, }; - const shardings: []const zml.Sharding = &.{ platform.replicated_sharding, platform.shardings.get("model").? }; - const rng_q = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.q.shape(), .{} }, .{ .shardings = shardings }); + const meshes: []const *const zml.Mesh = &.{ platform.replicated_mesh, platform.meshes.get("model").? }; + const rng_q = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.q.shape(), .{} }, .{ .meshes = meshes }); defer rng_q.deinit(); - const rng_k = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.k.shape(), .{} }, .{ .shardings = shardings }); + const rng_k = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.k.shape(), .{} }, .{ .meshes = meshes }); defer rng_k.deinit(); - const rng_kv_cache = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.kv_cache.split.k.shape(), .{} }, .{ .shardings = shardings }); + const rng_kv_cache = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.kv_cache.split.k.shape(), .{} }, .{ .meshes = meshes }); defer rng_kv_cache.deinit(); const triton_options_args: Options.Args = .{ @@ -458,9 +459,9 @@ test pagedAttention { const query_start_len: [batch_size + 1]i32 = .{ 0, 32, 33, 34, 35, 36, 37, 38, 38 }; var triton_parameters_d: zml.Bufferized(Parameters) = .{ .triton = .{ - .block_table = try .fromBytes(io, platform, triton_parameters.triton.block_table.shape(), .replicated, @ptrCast(&block_table)), - .seq_lens = try .fromBytes(io, platform, triton_parameters.triton.seq_lens.shape(), .replicated, @ptrCast(&seq_lens)), - .query_start_len = try .fromBytes(io, platform, triton_parameters.triton.query_start_len.shape(), .replicated, @ptrCast(&query_start_len)), + .block_table = try .fromBytes(io, platform, triton_parameters.triton.block_table.shape(), @ptrCast(&block_table)), + .seq_lens = try .fromBytes(io, platform, triton_parameters.triton.seq_lens.shape(), @ptrCast(&seq_lens)), + .query_start_len = try .fromBytes(io, platform, triton_parameters.triton.query_start_len.shape(), @ptrCast(&query_start_len)), } }; defer zml.Buffer.deinitAll(Parameters, &triton_parameters_d); @@ -514,7 +515,7 @@ test pagedAttention { io, pagedAttention, .{ parameters, tensors.q, tensors.k, tensors.v, tensors.kv_cache, test_case.attention_options }, - .{ .program_name = try std.fmt.allocPrint(arena, "paged_attention_{s}_{t}", .{ test_case.name, backend }), .shardings = shardings }, + .{ .program_name = try std.fmt.allocPrint(arena, "paged_attention_{s}_{t}", .{ test_case.name, backend }), .meshes = meshes }, ); defer exe.deinit(); @@ -540,13 +541,13 @@ test pagedAttention { @memcpy(&seqused_k_decode, seq_lens[num_prefill .. num_prefill + num_decode]); break :cuda_fa2 .{ .cuda_fa2 = .{ .mixed = .{ - .block_table_prefill = try .fromBytes(io, platform, params.mixed.block_table_prefill.shape(), .replicated, @ptrCast(&block_table_prefill)), - .cu_seqlens_q_prefill = try .fromBytes(io, platform, params.mixed.cu_seqlens_q_prefill.shape(), .replicated, @ptrCast(&cu_seqlens_q_prefill)), - .seqused_k_prefill = try .fromBytes(io, platform, params.mixed.seqused_k_prefill.shape(), .replicated, @ptrCast(&seqused_k_prefill)), + .block_table_prefill = try .fromBytes(io, platform, params.mixed.block_table_prefill.shape(), @ptrCast(&block_table_prefill)), + .cu_seqlens_q_prefill = try .fromBytes(io, platform, params.mixed.cu_seqlens_q_prefill.shape(), @ptrCast(&cu_seqlens_q_prefill)), + .seqused_k_prefill = try .fromBytes(io, platform, params.mixed.seqused_k_prefill.shape(), @ptrCast(&seqused_k_prefill)), - .block_table_decode = try .fromBytes(io, platform, params.mixed.block_table_decode.shape(), .replicated, @ptrCast(&block_table_decode)), - .cu_seqlens_q_decode = try .fromBytes(io, platform, params.mixed.cu_seqlens_q_decode.shape(), .replicated, @ptrCast(&cu_seqlens_q_decode)), - .seqused_k_decode = try .fromBytes(io, platform, params.mixed.seqused_k_decode.shape(), .replicated, @ptrCast(&seqused_k_decode)), + .block_table_decode = try .fromBytes(io, platform, params.mixed.block_table_decode.shape(), @ptrCast(&block_table_decode)), + .cu_seqlens_q_decode = try .fromBytes(io, platform, params.mixed.cu_seqlens_q_decode.shape(), @ptrCast(&cu_seqlens_q_decode)), + .seqused_k_decode = try .fromBytes(io, platform, params.mixed.seqused_k_decode.shape(), @ptrCast(&seqused_k_decode)), .metadata = .{ .decode_offset = try .scalar(io, platform, prefill_token_count, .i32) }, } } }; @@ -1060,20 +1061,20 @@ test "Triton sparse MLA value ranks and padded queries" { const tokens_pos = zml.Tensor.init(tokens_pos_shape, .i32); var parameters_d: zml.Bufferized(Parameters) = .{ .triton = .{ - .block_table = try .fromBytes(std.testing.io, platform, parameters.triton.block_table.shape(), .replicated, std.mem.sliceAsBytes(&block_table)), - .seq_lens = try .fromBytes(std.testing.io, platform, parameters.triton.seq_lens.shape(), .replicated, std.mem.sliceAsBytes(&seq_lens)), - .query_start_len = try .fromBytes(std.testing.io, platform, parameters.triton.query_start_len.shape(), .replicated, std.mem.sliceAsBytes(&query_start_len)), + .block_table = try .fromBytes(std.testing.io, platform, parameters.triton.block_table.shape(), std.mem.sliceAsBytes(&block_table)), + .seq_lens = try .fromBytes(std.testing.io, platform, parameters.triton.seq_lens.shape(), std.mem.sliceAsBytes(&seq_lens)), + .query_start_len = try .fromBytes(std.testing.io, platform, parameters.triton.query_start_len.shape(), std.mem.sliceAsBytes(&query_start_len)), } }; defer zml.Buffer.deinitAll(Parameters, ¶meters_d); - var q_d = try zml.Buffer.fromBytes(std.testing.io, platform, q_shape, .replicated, std.mem.sliceAsBytes(&q_data)); + var q_d = try zml.Buffer.fromBytes(std.testing.io, platform, q_shape, std.mem.sliceAsBytes(&q_data)); defer q_d.deinit(); - var kv_d = try zml.Buffer.fromBytes(std.testing.io, platform, kv_shape, .replicated, std.mem.sliceAsBytes(&kv_data)); + var kv_d = try zml.Buffer.fromBytes(std.testing.io, platform, kv_shape, std.mem.sliceAsBytes(&kv_data)); defer kv_d.deinit(); - var sink_d = try zml.Buffer.fromBytes(std.testing.io, platform, sink_shape, .replicated, std.mem.sliceAsBytes(&sink_data)); + var sink_d = try zml.Buffer.fromBytes(std.testing.io, platform, sink_shape, std.mem.sliceAsBytes(&sink_data)); defer sink_d.deinit(); - var topk_d = try zml.Buffer.fromBytes(std.testing.io, platform, topk_shape, .replicated, std.mem.sliceAsBytes(&topk_data)); + var topk_d = try zml.Buffer.fromBytes(std.testing.io, platform, topk_shape, std.mem.sliceAsBytes(&topk_data)); defer topk_d.deinit(); - var tokens_pos_d = try zml.Buffer.fromBytes(std.testing.io, platform, tokens_pos_shape, .replicated, std.mem.sliceAsBytes(&tokens_pos_data)); + var tokens_pos_d = try zml.Buffer.fromBytes(std.testing.io, platform, tokens_pos_shape, std.mem.sliceAsBytes(&tokens_pos_data)); defer tokens_pos_d.deinit(); const TestCase = struct { @@ -1174,20 +1175,20 @@ test "execute stablehlo mla kernel" { const tokens_pos = zml.Tensor.init(tokens_pos_shape, .i32); var parameters_d: zml.Bufferized(Parameters) = .{ .stablehlo = .{ - .block_table = try .fromBytes(std.testing.io, platform, parameters.stablehlo.block_table.shape(), .replicated, std.mem.sliceAsBytes(&block_table)), - .seq_lens = try .fromBytes(std.testing.io, platform, parameters.stablehlo.seq_lens.shape(), .replicated, std.mem.sliceAsBytes(&seq_lens)), - .query_start_len = try .fromBytes(std.testing.io, platform, parameters.stablehlo.query_start_len.shape(), .replicated, std.mem.sliceAsBytes(&query_start_len)), + .block_table = try .fromBytes(std.testing.io, platform, parameters.stablehlo.block_table.shape(), std.mem.sliceAsBytes(&block_table)), + .seq_lens = try .fromBytes(std.testing.io, platform, parameters.stablehlo.seq_lens.shape(), std.mem.sliceAsBytes(&seq_lens)), + .query_start_len = try .fromBytes(std.testing.io, platform, parameters.stablehlo.query_start_len.shape(), std.mem.sliceAsBytes(&query_start_len)), } }; defer zml.Buffer.deinitAll(Parameters, ¶meters_d); - var q_d = try zml.Buffer.fromBytes(std.testing.io, platform, q_shape, .replicated, std.mem.sliceAsBytes(&q_data)); + var q_d = try zml.Buffer.fromBytes(std.testing.io, platform, q_shape, std.mem.sliceAsBytes(&q_data)); defer q_d.deinit(); - var kv_d = try zml.Buffer.fromBytes(std.testing.io, platform, kv_shape, .replicated, std.mem.sliceAsBytes(&kv_data)); + var kv_d = try zml.Buffer.fromBytes(std.testing.io, platform, kv_shape, std.mem.sliceAsBytes(&kv_data)); defer kv_d.deinit(); - var sink_d = try zml.Buffer.fromBytes(std.testing.io, platform, sink_shape, .replicated, std.mem.sliceAsBytes(&sink_data)); + var sink_d = try zml.Buffer.fromBytes(std.testing.io, platform, sink_shape, std.mem.sliceAsBytes(&sink_data)); defer sink_d.deinit(); - var topk_d = try zml.Buffer.fromBytes(std.testing.io, platform, topk_shape, .replicated, std.mem.sliceAsBytes(&topk_data)); + var topk_d = try zml.Buffer.fromBytes(std.testing.io, platform, topk_shape, std.mem.sliceAsBytes(&topk_data)); defer topk_d.deinit(); - var tokens_pos_d = try zml.Buffer.fromBytes(std.testing.io, platform, tokens_pos_shape, .replicated, std.mem.sliceAsBytes(&tokens_pos_data)); + var tokens_pos_d = try zml.Buffer.fromBytes(std.testing.io, platform, tokens_pos_shape, std.mem.sliceAsBytes(&tokens_pos_data)); defer tokens_pos_d.deinit(); const exe = try platform.compileFn( diff --git a/zml/attention/sparse_mla.zig b/zml/attention/sparse_mla.zig index fbb9573b..3edbbe67 100644 --- a/zml/attention/sparse_mla.zig +++ b/zml/attention/sparse_mla.zig @@ -103,7 +103,7 @@ pub fn launchConfig(paged_opts: Options, topk_count: usize, cu_count_: usize) Co } // Full 256-query prefill chunks already expose enough query parallelism. // Retain BLOCK_M=16 and the generic split model, but process wider sparse - // tiles. The predicate uses the post-sharding flattened query shape. + // tiles. The predicate uses the post-mesh flattened query shape. if (is_sm103 and !paged_opts.all_decode and paged_opts.total_q_blocks >= 256 and diff --git a/zml/attention/tpu_attention.zig b/zml/attention/tpu_attention.zig index d1806730..82b39006 100644 --- a/zml/attention/tpu_attention.zig +++ b/zml/attention/tpu_attention.zig @@ -155,12 +155,12 @@ pub const mosaic_tpu = struct { .{ q.dim(.hd), target_head_dim }, ); return q.pad(0, .{ .hd = zml.Tensor.Pad{ .high = target_head_dim - q.dim(.hd) } }) - .withPartitioning(.{ .h = .model }); + .withPartitioning(.model, .{ .h = .model }); } inline fn prepareInputs(parameters: Parameters, q: zml.Tensor, kv_pages: zml.Tensor) PreparedInputs { - const kv_pages_partitioned = kv_pages.withPartitioning(.{ .hkv = .model }); - const q_merged = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.{ .h = .model }); + const kv_pages_partitioned = kv_pages.withPartitioning(.model, .{ .hkv = .model }); + const q_merged = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); const logical_head_dim: usize = @intCast(q_merged.dim(.hd)); const cache_head_dim: usize = @intCast(kv_pages_partitioned.dim(.hd)); stdx.debug.assert( @@ -169,15 +169,15 @@ pub const mosaic_tpu = struct { .{ cache_head_dim, logical_head_dim }, ); const q_ragged = alignQueryHeadDimForKernel(q_merged, kv_pages_partitioned.dim(.hd)); - const query_start_len = parameters.query_start_len.withPartitioning(.{ .b = .replicated }); + const query_start_len = parameters.query_start_len.withPartitioning(.model, .{ .b = .replicated }); return .{ .q = q_ragged, .kv_pages = kv_pages_partitioned, - .seq_lens = parameters.seq_lens.withPartitioning(.{ ._0 = .replicated }), - .block_table = parameters.block_table.withPartitioning(.{ ._0 = .replicated, ._1 = .replicated }), + .seq_lens = parameters.seq_lens.withPartitioning(.model, .{ ._0 = .replicated }), + .block_table = parameters.block_table.withPartitioning(.model, .{ ._0 = .replicated, ._1 = .replicated }), .query_start_len = query_start_len, - .num_seqs = activeSequenceCount(query_start_len).withPartitioning(.{ ._0 = .replicated }), + .num_seqs = activeSequenceCount(query_start_len).withPartitioning(.model, .{ ._0 = .replicated }), }; } diff --git a/zml/buffer.zig b/zml/buffer.zig index 5b86cae3..6f7acd8c 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -31,7 +31,6 @@ test { pub const Buffer = struct { _platform: *const Platform, _shape: Shape, - _sharding: Sharding, _shards: Shards, pub const MAX_NUM_SHARDS: u16 = Platform.MAX_NUM_DEVICES; @@ -115,10 +114,10 @@ pub const Buffer = struct { } pub fn format(self: Buffer, writer: *std.Io.Writer) !void { - const placement = self._sharding.placement(self._shape) catch { - return try writer.print("sharding error {} vs {}", .{ self._sharding, self._shape }); + const pl = Sharding.Placement.init(self._shape) catch |err| { + return try writer.print("Buffer({f}) with invalid sharding {} ({})", .{ self._shape, self._shape._sharding, err }); }; - try writer.print("{f}", .{placement}); + try writer.print("{f}", .{pl}); } /// Copies the content of the given buffer from host memory to the accelerator memory. @@ -126,7 +125,6 @@ pub const Buffer = struct { io: std.Io, platform: *const Platform, shape_: Shape, - sharding: Sharding, data_: []const u8, opts: FromOptions, ) !Buffer { @@ -135,7 +133,6 @@ pub const Buffer = struct { var res: Buffer = .{ ._platform = platform, ._shape = shape_, - ._sharding = sharding.resolve(platform), ._shards = .empty, }; errdefer for (res._shards.slice()) |shard| { @@ -146,7 +143,7 @@ pub const Buffer = struct { const slice = Slice.init(sh, data_); const buffer_type = pjrtx.bufferTypeFromDtype(sh.dtype()); - const placement = placementOrPanic(res._sharding, sh); + const placement = placementOrPanic(platform, sh); const shard_dims: []const i64 = placement.shape.dims(); const layout = platform.defaultMemoryLayout(shard_dims, sh.dtype()); @@ -179,28 +176,28 @@ pub const Buffer = struct { /// Copies the given Zig bytes to the accelerator memory and /// return a Buffer with the given dimensions. - pub fn fromBytes(io: std.Io, platform: *const Platform, sh: Shape, sharding: Sharding, data: []const u8) !Buffer { - return fromBytesOpts(io, platform, sh, sharding, data, .{}); + pub fn fromBytes(io: std.Io, platform: *const Platform, sh: Shape, data: []const u8) !Buffer { + return fromBytesOpts(io, platform, sh, data, .{}); } - pub fn fromBytesOpts(io: std.Io, platform: *const Platform, sh: Shape, sharding: Sharding, data: []const u8, opts: FromOptions) !Buffer { - return from(io, platform, sh, sharding, data, opts); + pub fn fromBytesOpts(io: std.Io, platform: *const Platform, sh: Shape, data: []const u8, opts: FromOptions) !Buffer { + return from(io, platform, sh, data, opts); } /// Copies the given zml.Slice to the accelerator memory and /// return a Buffer. - pub fn fromSlice(io: std.Io, platform: *const Platform, slice: Slice, sharding: Sharding) !Buffer { - return fromSliceOpts(io, platform, slice, sharding, .{}); + pub fn fromSlice(io: std.Io, platform: *const Platform, slice: Slice) !Buffer { + return fromSliceOpts(io, platform, slice, .{}); } - pub fn fromSliceOpts(io: std.Io, platform: *const Platform, slice: Slice, sharding: Sharding, opts: FromOptions) !Buffer { - return from(io, platform, slice.shape, sharding, std.mem.sliceAsBytes(slice.constData()), opts); + pub fn fromSliceOpts(io: std.Io, platform: *const Platform, slice: Slice, opts: FromOptions) !Buffer { + return from(io, platform, slice.shape, std.mem.sliceAsBytes(slice.constData()), opts); } /// Creates a Buffer with a single element. pub fn scalar(io: std.Io, platform: *const Platform, val: anytype, dtype_: DataType) !Buffer { const x = dtype_.constant(val); - return fromBytes(io, platform, .scalar(dtype_), .replicated, x.asBytes()); + return fromBytes(io, platform, .scalar(dtype_), x.asBytes()); } pub fn await(self: Buffer, io: std.Io) !void { @@ -217,7 +214,6 @@ pub const Buffer = struct { _: std.Io, platform: *const Platform, shape_: Shape, - sharding: Sharding, opts: UnitializedOptions, ) !Buffer { std.log.debug("uninitialized {f}", .{shape_}); @@ -225,7 +221,6 @@ pub const Buffer = struct { var res: Buffer = .{ ._platform = platform, ._shape = shape_, - ._sharding = sharding.resolve(platform), ._shards = .empty, }; errdefer for (res._shards.slice()) |shard| { @@ -234,7 +229,7 @@ pub const Buffer = struct { stdx.debug.assert(platform.devices[0].memory(opts.memory) != null, "Device doesn't have {} memory", .{opts.memory}); const element_type = pjrtx.bufferTypeFromDtype(sh.dtype()); - const placement = placementOrPanic(res._sharding, sh); + const placement = placementOrPanic(platform, sh); const shard_dims: []const i64 = placement.shape.dims(); const layout = platform.defaultMemoryLayout(shard_dims, sh.dtype()); @@ -257,14 +252,13 @@ pub const Buffer = struct { } /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. - pub fn fromPjrtBuffers(platform: *const Platform, sh: Shape, sharding: Sharding, pjrt_buffers: []const *pjrt.Buffer) Buffer { + pub fn fromPjrtBuffers(platform: *const Platform, sh: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); stdx.debug.assert(pjrt_buffers.len > 0, "fromPjrtBuffers expects at least one buffer, got 0.", .{}); return .{ ._platform = platform, ._shape = sh, - ._sharding = sharding, ._shards = Shards.fromSlice(pjrt_buffers) catch unreachable, }; } @@ -283,8 +277,8 @@ pub const Buffer = struct { pub fn toSlice(self: Buffer, io: std.Io, slice: Slice) !void { stdx.debug.assert(self._shape.eql(slice.shape), "Buffer shape {f} doesn't match destination slice {f}", .{ self._shape, slice.shape }); - const placement = placementOrPanic(self._sharding, self._shape); - for (self._sharding.devicesInCanonicalOrder(), 0..) |device, shard_index| { + const placement = placementOrPanic(self._platform, self._shape); + for (self._platform.physical_mesh.devices_in_canonical_order, 0..) |device, shard_index| { // TODO: handle replicated information, we shouldn't iterate over all the devices unless needed const sub_slice = placement.shardSlice(device.coords, slice); if (!sub_slice.isContiguous()) return error.NonContiguousShardRead; @@ -306,12 +300,12 @@ pub const Buffer = struct { const slice = try Slice.alloc(allocator, self.shape()); errdefer slice.free(allocator); - const placement = placementOrPanic(self._sharding, self._shape); + const placement = placementOrPanic(self._platform, self._shape); var shard_slice = try Slice.alloc(allocator, placement.shape); defer shard_slice.free(allocator); - for (self._sharding.devicesInCanonicalOrder(), 0..) |device, shard_index| { + for (self._platform.physical_mesh.devices_in_canonical_order, 0..) |device, shard_index| { const sub_slice = placement.shardSlice(device.coords, slice); const maybe_event = try self._shards.get(shard_index).toHostBuffer(self._platform.pjrt_api, shard_slice.data()); if (maybe_event) |event| { @@ -331,8 +325,8 @@ pub const Buffer = struct { /// ie: `num_devices * shard_byte_size` /// `shard_byte_size` can be up to `self.shape().byteSize()` when the buffer is fully replicated. pub fn byteSize(self: Buffer) usize { - const placement = placementOrPanic(self._sharding, self._shape); - return placement.shape.byteSize() * self._sharding.devicesInCanonicalOrder().len; + const placement = placementOrPanic(self._platform, self._shape); + return placement.shape.byteSize() * self._shards.len; } pub fn opaqueDevicePtr(self: Buffer, device_id: usize) *anyopaque { @@ -360,7 +354,7 @@ pub const Buffer = struct { view_shard_list.appendAssumeCapacity(view_shard); } - return fromPjrtBuffers(platform, self._shape, self._sharding, view_shard_list.items); + return fromPjrtBuffers(platform, self._shape, view_shard_list.items); } }; @@ -381,13 +375,14 @@ test "device round-trip" { .{ 56, 57, 58, 59, 60, 61, 62, 63 }, }; + const mp: *const zml.Mesh = platform.meshes.get("model").?; const x_h: zml.Slice = .init(.withPartitioning( .init(.{ .b = 8, .d = 8 }, .u32), + mp, .{ .b = .model }, ), std.mem.asBytes(&x)); // no free: x_h is stack allocated - const model_sharding: zml.Sharding = platform.shardings.get("model").?; - const x_d: zml.Buffer = try .fromSlice(io, platform, x_h, model_sharding); + const x_d: zml.Buffer = try .fromSlice(io, platform, x_h); try std.testing.expectEqual(platform.devices.len, x_d.numShards()); { @@ -409,29 +404,11 @@ test "device round-trip" { } } -fn placementOrPanic(sharding: Sharding, shape: Shape) Sharding.Placement { - return sharding.placement(shape) catch |err| { +fn placementOrPanic(platform: *const Platform, shape: Shape) Sharding.Placement { + var sh = shape; + if (sh._sharding.mesh == null) sh._sharding.mesh = platform.replicated_mesh; + return Sharding.Placement.init(sh) catch |err| { @branchHint(.cold); - switch (err) { - error.MissingLogicalBinding => { - log.err( - \\Failed to shard Buffer of shape {f}, with sharding: - \\{f} - \\ - \\The Buffer is probably inheriting a partitionned shape from a Tensor, - \\So Buffer creation must pass a Sharding, that maps the logical sharding of the Tensor to the physical mesh. - , .{ shape, sharding }); - @panic("Buffer shape and sharding should be consistent"); - }, - error.IncompatibleSharding => { - log.err( - \\Failed to shard Buffer of shape {f}, with sharding: - \\{f} - \\ - \\The Buffer dimension isn't properly divisible by the number of devices along the sharded axis. - , .{ shape, sharding }); - @panic("Buffer shape should be divisible by the number of devices along the sharded axis."); - }, - } + std.debug.panic("Buffer shape {f} is incompatible with its mesh: {t}", .{ shape, err }); }; } diff --git a/zml/exe.zig b/zml/exe.zig index f97b8f7d..1db04837 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -23,9 +23,6 @@ pub const Exe = struct { input_shapes: []const Shape, output_shapes: []const Shape, - input_shardings: []const Sharding, - output_shardings: []const Sharding, - /// Inputs whose buffers are donated to an output (`reuseBuffer`). /// Runners destroy their PJRT handles after the raw call returns. donated_input_indices: []const usize, @@ -43,8 +40,6 @@ pub const Exe = struct { num_partitions: i32, input_shapes: []const Shape, output_shapes: []const Shape, - input_shardings: []const Sharding, - output_shardings: []const Sharding, input_aliasing: []const ?u32, ) !Exe { var arena = std.heap.ArenaAllocator.init(allocator); @@ -53,10 +48,6 @@ pub const Exe = struct { const input_shapes_copy = try arena.allocator().dupe(Shape, input_shapes); const output_shapes_copy = try arena.allocator().dupe(Shape, output_shapes); - // Re-home sharding pointers into arena-owned values so exe doesn't depend on caller lifetimes. - const input_shardings_copy = try arena.allocator().dupe(Sharding, input_shardings); - const output_shardings_copy = try arena.allocator().dupe(Sharding, output_shardings); - var donated_count: usize = 0; for (input_aliasing) |aliasing| donated_count += @intFromBool(aliasing != null); @@ -74,8 +65,6 @@ pub const Exe = struct { .exe = exe, .input_shapes = input_shapes_copy, .output_shapes = output_shapes_copy, - .input_shardings = input_shardings_copy, - .output_shardings = output_shardings_copy, .donated_input_indices = donated_input_indices, .num_devices = num_devices, .num_partitions = num_partitions, @@ -90,11 +79,11 @@ pub const Exe = struct { } pub fn args(self: *const Exe, allocator: std.mem.Allocator) !Arguments { - return Arguments.init(allocator, self.input_shapes, self.input_shardings, self.num_devices); + return Arguments.init(allocator, self.input_shapes, self.num_devices); } pub fn results(self: *const Exe, allocator: std.mem.Allocator) !Results { - return Results.init(allocator, self.output_shapes, self.output_shardings, self.platform, self.num_devices); + return Results.init(allocator, self.output_shapes, self.platform, self.num_devices); } pub const FlatBuffers = struct { @@ -131,9 +120,8 @@ pub const Exe = struct { flat_buffers: FlatBuffers, expected_shapes: []const Shape, baked_count: usize = 0, - shardings: []const Sharding, - pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, shardings: []const Sharding, num_devices: usize) error{OutOfMemory}!Arguments { + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize) error{OutOfMemory}!Arguments { const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); errdefer flat_buffers.deinit(allocator); @@ -143,7 +131,6 @@ pub const Exe = struct { return .{ .flat_buffers = flat_buffers, .expected_shapes = expected_shapes, - .shardings = shardings, }; } @@ -219,9 +206,8 @@ pub const Exe = struct { flat_buffers: FlatBuffers, expected_shapes: []const Shape, - shardings: []const Sharding, - pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, shardings: []const Sharding, platform: *const Platform, num_devices: usize) !Results { + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, platform: *const Platform, num_devices: usize) !Results { const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); errdefer flat_buffers.deinit(allocator); @@ -232,7 +218,6 @@ pub const Exe = struct { .platform = platform, .flat_buffers = flat_buffers, .expected_shapes = expected_shapes, - .shardings = shardings, }; } @@ -254,7 +239,7 @@ pub const Exe = struct { for (0..context_.self.flat_buffers.num_devices) |device_index| { shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); } - buffer.* = Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], context_.self.shardings[context_.current_index], shards.constSlice()); + buffer.* = Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], shards.constSlice()); context_.current_index += 1; } }.cb, &context, &result); @@ -274,7 +259,7 @@ pub const Exe = struct { for (0..ctx.results.flat_buffers.num_devices) |device_index| { shards.appendAssumeCapacity(ctx.results.flat_buffers.buffers[device_index][ctx.current_index]); } - buffer.* = Buffer.fromPjrtBuffers(ctx.results.platform, ctx.results.expected_shapes[ctx.current_index], ctx.results.shardings[ctx.current_index], shards.constSlice()); + buffer.* = Buffer.fromPjrtBuffers(ctx.results.platform, ctx.results.expected_shapes[ctx.current_index], shards.constSlice()); ctx.current_index += 1; } }.cb, &context, &v); @@ -639,7 +624,7 @@ test "Exe raw calls leave donated input handle cleanup to the caller" { defer exe.deinit(); try std.testing.expectEqualSlices(usize, &.{0}, exe.donated_input_indices); - var weight_buffer = try Buffer.fromBytes(io, platform, weight.shape(), .replicated, std.mem.asBytes(&[2]i32{ 10, 20 })); + var weight_buffer = try Buffer.fromBytes(io, platform, weight.shape(), std.mem.asBytes(&[2]i32{ 10, 20 })); defer weight_buffer.deinit(); var args = try exe.args(allocator); defer args.deinit(allocator); @@ -647,7 +632,7 @@ test "Exe raw calls leave donated input handle cleanup to the caller" { defer results.deinit(allocator); for ([_]bool{ false, true }) |wait| { - var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), .replicated, std.mem.asBytes(&[2]i32{ 1, 2 })); + var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), std.mem.asBytes(&[2]i32{ 1, 2 })); defer value_buffer.deinit(); args.set(.{ value_buffer, weight_buffer }); exe.call(io, args, &results, .{ .wait = wait }); @@ -679,9 +664,9 @@ test "FnExe runner replaces donated buffers with and without waiting" { const exe = try Model.compile(allocator, io, platform, .{}, .{.{ .weight = weight, .value = value }}); defer exe.deinit(); - var weight_buffer = try Buffer.fromBytes(io, platform, weight.shape(), .replicated, std.mem.asBytes(&[2]i32{ 10, 20 })); + var weight_buffer = try Buffer.fromBytes(io, platform, weight.shape(), std.mem.asBytes(&[2]i32{ 10, 20 })); defer weight_buffer.deinit(); - var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), .replicated, std.mem.asBytes(&[2]i32{ 1, 2 })); + var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), std.mem.asBytes(&[2]i32{ 1, 2 })); defer value_buffer.deinit(); var runner = try Model.Runner(.{.weight}).init(&exe, allocator, .{ .weight = weight_buffer }); defer runner.deinit(allocator); @@ -714,7 +699,7 @@ test "FnExe runner replaces unchanged donated buffers" { defer exe.deinit(); try std.testing.expectEqualSlices(usize, &.{0}, exe.raw.donated_input_indices); - var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), .replicated, std.mem.asBytes(&[2]i32{ 1, 2 })); + var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), std.mem.asBytes(&[2]i32{ 1, 2 })); defer value_buffer.deinit(); var runner = try exe.runner(allocator); defer runner.deinit(allocator); diff --git a/zml/io.zig b/zml/io.zig index 589abd47..5fce0793 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -31,14 +31,16 @@ pub const TensorStore = struct { }; registry: *safetensors.TensorRegistry, + meshes: []const *const Sharding.Mesh, id_to_sources: std.AutoHashMapUnmanaged(Tensor.Id, Binding), allocator: std.mem.Allocator, arena: std.heap.ArenaAllocator, - pub fn fromRegistry(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry) TensorStore { - const arena: std.heap.ArenaAllocator = .init(allocator); + pub fn fromRegistry(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry, meshes: []const *const Sharding.Mesh) TensorStore { + var arena: std.heap.ArenaAllocator = .init(allocator); return .{ .registry = registry, + .meshes = arena.allocator().dupe(*const Sharding.Mesh, meshes) catch @panic("OOM"), .id_to_sources = .empty, .allocator = allocator, .arena = arena, @@ -104,6 +106,20 @@ pub const TensorStore = struct { return .{ .store = self }; } + pub fn mesh(store: *const TensorStore, name: @EnumLiteral()) *const Sharding.Mesh { + const name_slice = @tagName(name); + for (store.meshes) |m| { + if (std.mem.eql(u8, name_slice, m.name)) { + return m; + } + } + std.debug.panic( + \\Found no meshes named {s} in TensorStore. + \\Try passing more meshes to `zml.TensorStore.fromRegistry`. + \\Known meshes: {f} + , .{ name_slice, stdx.fmt.slice(store.meshes) }); + } + pub const View = struct { store: *TensorStore, @@ -162,19 +178,50 @@ pub const TensorStore = struct { } else false; } - pub const CreateTensorOpts = struct { - memory: Memory.Kind = .default, - }; - - pub fn maybeCreateTensor(self: View, subkey: []const u8, tagz: anytype, partitioning: anytype) ?Tensor { - return self.maybeCreateTensorInternal(subkey, tagz, partitioning, .{}); + /// Creates a zml.Tensor from a specific entry in the store. + pub fn createTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) Tensor { + return self.maybeCreateTensor(subkey, tags, mesh_, partitioning) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); } - pub fn maybeCreateHostPinnedTensor(self: View, subkey: []const u8, tagz: anytype, partitioning: anytype) ?Tensor { - return self.maybeCreateTensorInternal(subkey, tagz, partitioning, .{ .memory = .host_pinned }); + pub fn maybeCreateTensor(self: View, subkey: []const u8, comptime tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) ?Tensor { + const has_tags: bool = comptime @TypeOf(tags) != @TypeOf(null); + const parsed_tags: Shape.TagsArray = if (comptime has_tags) Shape.parseTags(tags) else undefined; + if (mesh_ == .replicated) { + return self.maybeCreateTensorInternal(subkey, if (has_tags) parsed_tags.constSlice() else null, .replicated); + } + const resolved_mesh = self.store.mesh(mesh_); + // Unspecified tensor axes are treated as replicated + const partition: Sharding.Partitioning = if (has_tags) + .parse(parsed_tags.constSlice(), resolved_mesh, .replicated, partitioning) + else + partitioning; + return self.maybeCreateTensorInternal(subkey, if (has_tags) parsed_tags.constSlice() else null, .{ .mesh = resolved_mesh, .partition = partition }); } - fn maybeCreateTensorInternal(self: View, subkey: []const u8, tagz: anytype, partitioning: anytype, opts: CreateTensorOpts) ?Tensor { + pub fn createHostPinnedTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) Tensor { + return self.maybeCreateHostPinnedTensor(subkey, tags, mesh_, partitioning) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); + } + + pub fn maybeCreateHostPinnedTensor(self: View, subkey: []const u8, comptime tags: anytype, mesh_: @EnumLiteral(), comptime partitioning: anytype) ?Tensor { + const tensor = self.maybeCreateTensor(subkey, tags, mesh_, partitioning); + if (tensor) |t| { + const storage = self.store.id_to_sources.getPtr(t.id); + storage.?.memory = .host_pinned; + } + return tensor; + } + + pub fn createReplicatedTensor(self: View, subkey: []const u8, tags: anytype) Tensor { + const has_tags: bool = comptime @TypeOf(tags) != @TypeOf(null); + const parsed_tags: Shape.TagsArray = if (comptime has_tags) Shape.parseTags(tags) else undefined; + + return self.maybeCreateTensorInternal(subkey, if (has_tags) parsed_tags.slice() else null, .replicated) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); + } + + fn maybeCreateTensorInternal(self: View, subkey: []const u8, tags: ?[]const Shape.Tag, sharding: Sharding) ?Tensor { var buffer: [256]u8 = undefined; const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); const source = self.store.dupeSource(key) orelse return null; @@ -184,57 +231,19 @@ pub const TensorStore = struct { sources[0] = source; var shape = source.shape; - shape = applyTags(shape, tagz); - shape = applyPartitioning(shape, partitioning); + if (tags) |user_tags| { + stdx.debug.assert(user_tags.len == shape.rank(), "tensor {s} from store has shape {f}, but `createTensor` got only {d} tags: {f}", .{ key, source.shape, user_tags.len, stdx.fmt.stringsZ(user_tags) }); + @memcpy(shape._tags.slice(), user_tags); + } + shape._sharding = sharding; + if (sharding.mesh) |mesh_| shape = shape.withPartitioning(mesh_, sharding.partition); const tensor: Tensor = .fromShape(shape); - self.store.putSourcesNoClobber(tensor.id, .{ .tensors = sources, .transformed = false, .memory = opts.memory }) catch |e| std.debug.panic("Not handling {} errors", .{e}); + self.store.putSourcesNoClobber(tensor.id, .{ .tensors = sources, .transformed = false, .memory = .default }) catch |e| std.debug.panic("Not handling {} errors", .{e}); return tensor; } - pub fn createTensor(self: View, subkey: []const u8, tagz: anytype, partitioning: anytype) Tensor { - return self.maybeCreateTensorInternal(subkey, tagz, partitioning, .{}) orelse - stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); - } - - pub fn createHostPinnedTensor(self: View, subkey: []const u8, tagz: anytype, partitioning: anytype) Tensor { - return self.maybeCreateTensorInternal(subkey, tagz, partitioning, .{ .memory = .host_pinned }) orelse - stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); - } - - fn applyTags(shape_: Shape, tagz: anytype) Shape { - var shape = shape_; - if (@TypeOf(tagz) != @TypeOf(null)) { - switch (@typeInfo(@TypeOf(tagz))) { - .optional => if (tagz) |t| { - shape = shape.withTags(t); - }, - else => shape = shape.withTags(tagz), - } - } - return shape; - } - - fn applyPartitioning(shape_: Shape, partitioning: anytype) Shape { - var shape = shape_; - - if (@TypeOf(partitioning) == @TypeOf(null)) { - @compileError("TensorStore.View.createTensor partitioning cannot be null; pass .replicated or an explicit partitioning"); - } - - switch (@typeInfo(@TypeOf(partitioning))) { - .optional => @compileError("TensorStore.View.createTensor partitioning cannot be optional; pass .replicated or an explicit partitioning"), - .enum_literal => switch (partitioning) { - .replicated => shape = shape.withReplicatedPartitioning(), - else => @compileError("Only .replicated is supported as a standalone partitioning enum literal"), - }, - else => shape = shape.withPartitioning(partitioning), - } - - return shape; - } - pub fn maybeCreateBinding(self: View, sources: []const []const u8, shape: Shape) ?Tensor { const arena = self.store.arena.allocator(); @@ -365,7 +374,7 @@ pub const Loader = struct { pub const LoadError = error{TransformedTensorNotDelivered}; - pub fn load(self: *Loader, io: std.Io, comptime T: type, model: *const T, buffers: *Bufferized(T), store: *const TensorStore, shardings: []const Sharding, opts: LoadOpts) LoadError!void { + pub fn load(self: *Loader, io: std.Io, comptime T: type, model: *const T, buffers: *Bufferized(T), store: *const TensorStore, opts: LoadOpts) LoadError!void { const tensor_count = meta.count(Tensor, model); var arena: std.heap.ArenaAllocator = .init(self.allocator); @@ -406,7 +415,7 @@ pub const Loader = struct { if (sources.transformed) continue; } - self.group.async(io, defaultCallback, .{ self, io, tensor, buffer, store, shardings, opts }); + self.group.async(io, defaultCallback, .{ self, io, tensor, buffer, store, opts }); } } @@ -424,25 +433,25 @@ pub const Loader = struct { log.err("Transformed tensor {} {f} was not delivered by loadExecute before load; sources: {s}", .{ tensor.id, tensor.shape(), names.written() }); } - pub fn loadBuffer(loader: *Loader, io: std.Io, buffer: *Buffer, tensor: Tensor, store: *const TensorStore, shardings: []const Sharding, opts: LoadOpts) void { - loader.group.async(io, defaultCallback, .{ loader, io, &tensor, buffer, store, shardings, opts }); + pub fn loadBuffer(loader: *Loader, io: std.Io, buffer: *Buffer, tensor: *const Tensor, store: *const TensorStore, opts: LoadOpts) void { + loader.group.async(io, defaultCallback, .{ loader, io, tensor, buffer, store, opts }); } - fn defaultCallback(self: *Loader, io: std.Io, tensor: *const Tensor, buffer: *Buffer, store: *const TensorStore, shardings: []const Sharding, opts: LoadOpts) void { + fn defaultCallback(self: *Loader, io: std.Io, tensor: *const Tensor, buffer: *Buffer, store: *const TensorStore, opts: LoadOpts) void { const sources = store.getSourcesById(tensor.id) orelse { std.log.debug("Failed to get sources for tensor with id: {}", .{tensor.id}); return; }; stdx.debug.assert(!sources.transformed and sources.tensors.len == 1, "Tensor {} is transformed or has {} sources; `load` only streams single-source tensors", .{ tensor.id, sources.tensors.len }); - self.loadSingleInner(io, sources.tensors[0], tensor.shape(), buffer, sources.memory, shardings, opts) catch |e| { + self.loadSingleInner(io, sources.tensors[0], tensor.shape(), buffer, sources.memory, opts) catch |e| { log.err("Errors are not handled in `defaultCallback`, got {}", .{e}); unreachable; }; } - fn loadSingle(self: *Loader, io: std.Io, source: *safetensors.Tensor, shape: Shape, buffer: *Buffer, loaded: *bool, shardings: []const Sharding, opts: LoadOpts) void { - self.loadSingleInner(io, source, shape, buffer, .device, shardings, opts) catch |e| { + fn loadSingle(self: *Loader, io: std.Io, source: *safetensors.Tensor, shape: Shape, buffer: *Buffer, loaded: *bool, opts: LoadOpts) void { + self.loadSingleInner(io, source, shape, buffer, .device, opts) catch |e| { log.err("Failed to load tensor {s}: {}", .{ source.name, e }); loaded.* = false; return; @@ -457,17 +466,11 @@ pub const Loader = struct { shape: Shape, buffer: *Buffer, memory: Memory.Kind, - shardings: []const Sharding, opts: LoadOpts, ) !void { var reader = try source.reader(io, &.{}, .{}); defer reader.deinit(); - const sharding = Sharding.pickSharding(shardings, shape, .explicit_axis_binding) orelse blk: { - log.debug("No sharding strategy found for tensor {s} with shape {f}, using replicated sharding", .{ reader.tensor.name, shape }); - break :blk self.platform.replicated_sharding; - }; - var writer = try MemoryWriter.init( self.allocator, io, @@ -476,7 +479,6 @@ pub const Loader = struct { self.dma_allocators, self.dma_chunk_size, shape, - sharding, buffer, memory, ); @@ -507,7 +509,6 @@ pub const Loader = struct { tensor: Tensor, buffer: *Buffer, store: *const TensorStore, - shardings: []const Sharding, exe: *const Exe, opts: LoadOpts, ) error{ NotFound, OutOfMemory, LoadFailed, Canceled }!void { @@ -530,7 +531,7 @@ pub const Loader = struct { defer if (node) |*n| n.end(); for (sources, 0..) |source, i| { - self.group.async(io, loadSingle, .{ self, io, source, source.shape, &buffers[i], &loaded[i], shardings, .{} }); + self.group.async(io, loadSingle, .{ self, io, source, source.shape, &buffers[i], &loaded[i], .{} }); } self.group.await(io) catch |err| switch (err) { error.Canceled => return error.Canceled, @@ -566,7 +567,7 @@ test "Loader.loadExecute cleans up donated and non-donated inputs" { defer registry.deinit(); try registry.registerTensor(.{ .name = "value", .file_uri = path, .shape = shape, .offset = 0 }); try registry.registerTensor(.{ .name = "weight", .file_uri = path, .shape = shape, .offset = 2 * @sizeOf(i32) }); - var store: TensorStore = .fromRegistry(allocator, ®istry); + var store: TensorStore = .fromRegistry(allocator, ®istry, &.{platform.replicated_mesh}); defer store.deinit(); var loader = try Loader.init(allocator, platform, .{ .parallelism = 2, .dma_chunks = 2, .dma_chunk_size = 4096 }); defer loader.deinit(); @@ -586,7 +587,7 @@ test "Loader.loadExecute cleans up donated and non-donated inputs" { try std.testing.expectEqual(@as(usize, @intFromBool(donate)), exe.donated_input_indices.len); var output: Buffer = undefined; - try loader.loadExecute(arena.allocator(), io, tensor, &output, &store, &.{}, &exe, .{}); + try loader.loadExecute(arena.allocator(), io, tensor, &output, &store, &exe, .{}); defer output.deinit(); try std.testing.expectEqual([2]i32{ 11, 22 }, try output.getValue([2]i32, io)); try std.testing.expect(loader.delivered.contains(tensor.id)); @@ -681,18 +682,17 @@ pub const MemoryWriter = union(enum) { dma_allocators: []const mem.DmaAllocator, dma_chunk_size: usize, shape: Shape, - sharding: Sharding, buffer: *Buffer, memory: Memory.Kind, ) !MemoryWriter { return switch (platform.target) { .cuda, .rocm, .oneapi => .{ - .direct = try .init(allocator, io, platform, pools, dma_allocators, dma_chunk_size, shape, sharding, buffer, memory), + .direct = try .init(allocator, io, platform, pools, dma_allocators, dma_chunk_size, shape, buffer, memory), }, .tpu, .neuron, .cpu, .metal => if (memory == .host_pinned) std.debug.panic("Host pinned memory is not supported on {}", .{platform.target}) else - .{ .buffered = try .init(allocator, io, platform, shape, sharding, buffer) }, + .{ .buffered = try .init(allocator, io, platform, shape, buffer) }, }; } @@ -722,16 +722,14 @@ pub const BufferedMemoryWriter = struct { io: std.Io, platform: *const Platform, shape: Shape, - sharding: Sharding, buffer: *Buffer, interface: std.Io.Writer, - pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, shape: Shape, sharding: Sharding, buffer: *Buffer) !BufferedMemoryWriter { + pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, shape: Shape, buffer: *Buffer) !BufferedMemoryWriter { return .{ .io = io, .platform = platform, .shape = shape, - .sharding = sharding, .buffer = buffer, .interface = .{ .buffer = try allocator.alloc(u8, shape.byteSize()), @@ -757,7 +755,6 @@ pub const BufferedMemoryWriter = struct { self.io, self.platform, self.shape, - self.sharding, @ptrCast(self.interface.buffer), .{ .wait = true }, ) catch return std.Io.Writer.Error.WriteFailed; @@ -982,9 +979,9 @@ const DispatchSpans = struct { spans: []DispatchSpan, mirror_writers: []usize, - fn init(allocator: std.mem.Allocator, shape: Shape, sharding: Sharding) !DispatchSpans { - const placement = try sharding.placement(shape); - const ordered_devices = sharding.devicesInCanonicalOrder(); + fn init(allocator: std.mem.Allocator, shape: Shape) !DispatchSpans { + const placement = try Sharding.Placement.init(shape); + const ordered_devices = placement.mesh().?.devicesInCanonicalOrder(); var placement_span_count: usize = 0; for (ordered_devices) |device| { @@ -1197,11 +1194,14 @@ pub const DirectMemoryWriter = struct { dma_allocators: []const mem.DmaAllocator, dma_chunk_size: usize, shape: Shape, - sharding: Sharding, buffer: *Buffer, memory: Memory.Kind, ) !DirectMemoryWriter { - const ordered_devices = sharding.devicesInCanonicalOrder(); + var placed_shape = shape; + if (placed_shape._sharding.mesh == null) placed_shape._sharding.mesh = platform.replicated_mesh; + const placement = try Sharding.Placement.init(placed_shape); + const mesh = placement.mesh().?; + const ordered_devices = mesh.devicesInCanonicalOrder(); var shard_writers = try allocator.alloc(DirectShardWriter, ordered_devices.len); errdefer allocator.free(shard_writers); @@ -1211,7 +1211,6 @@ pub const DirectMemoryWriter = struct { }; var pjrt_buffers: Buffer.Shards = .empty; - const placement = try sharding.placement(shape); for (ordered_devices, 0..) |device, i| { const pool = &pools[device.id]; const shard_dma_allocator = dma_allocators[device.id].allocator(); @@ -1223,9 +1222,9 @@ pub const DirectMemoryWriter = struct { pjrt_buffers.appendAssumeCapacity(shard_writers[i].pjrt_buffer); } - buffer.* = .fromPjrtBuffers(platform, shape, sharding, pjrt_buffers.constSlice()); + buffer.* = .fromPjrtBuffers(platform, shape, pjrt_buffers.constSlice()); - const dispatch_spans: DispatchSpans = try .init(allocator, shape, sharding); + const dispatch_spans: DispatchSpans = try .init(allocator, placed_shape); errdefer dispatch_spans.deinit(allocator); const first_span = dispatch_spans.spans[0]; @@ -1535,15 +1534,14 @@ const DirectMemoryWriterDeviceTest = struct { allocator: std.mem.Allocator, io: std.Io, - fn run(self: DirectMemoryWriterDeviceTest, scenario: Scenario) !void { + fn run(self: DirectMemoryWriterDeviceTest, scenario: Scenario, partitioning: anytype) !void { var platform = Platform.auto(self.allocator, self.io, scenario.create_options) catch return error.SkipZigTest; defer platform.deinit(self.allocator, self.io); - const sharding: Sharding.Data = try .init(scenario.name, &platform.physical_mesh, scenario.logical_mesh, scenario.strategy); + const mesh: Sharding.Mesh = try .init(scenario.name, &platform.physical_mesh, scenario.logical_mesh, scenario.strategy); try self.runDirectMemoryWriter( platform, - scenario.shape, - .{ .data = &sharding }, + scenario.shape.withPartitioning(&mesh, partitioning), scenario.write_mode, scenario.writable_slice_min_len, scenario.pool_chunks, @@ -1556,7 +1554,6 @@ const DirectMemoryWriterDeviceTest = struct { self: DirectMemoryWriterDeviceTest, platform: *const Platform, shape: Shape, - sharding: Sharding, write_mode: WriteMode, writable_slice_min_len: usize, pool_chunks: usize, @@ -1595,7 +1592,6 @@ const DirectMemoryWriterDeviceTest = struct { dma_allocators, pool_chunk_size, shape, - sharding, &written_buffer, memory, ); @@ -1643,11 +1639,10 @@ test "DirectMemoryWriter: replicated with auto topology" { .physical_mesh = .auto, .cpu = .{ .device_count = 4 }, }, - .shape = Shape.init(.{ .rows = 8, .cols = 128 }, .f32) - .withPartitioning(.{ .rows = .replicated, .cols = .replicated }), + .shape = Shape.init(.{ .rows = 8, .cols = 128 }, .f32), .logical_mesh = .mesh(.{ .x = .high_bandwidth }), .strategy = .parseBindings(.{ .x = .link_x }), - }); + }, .{ .rows = .replicated, .cols = .replicated }); } test "DirectMemoryWriter: 1D model split with 2x2 physical mesh" { @@ -1662,11 +1657,10 @@ test "DirectMemoryWriter: 1D model split with 2x2 physical mesh" { .physical_mesh = .{ .custom = buildMesh2x2 }, .cpu = .{ .device_count = 4 }, }, - .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32) - .withPartitioning(.{ .rows = .replicated, .cols = .model }), + .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32), .logical_mesh = .mesh(.{ .model = .high_bandwidth }), .strategy = .parseBindings(.{ .model = .link_x }), - }); + }, .{ .rows = .replicated, .cols = .model }); } test "DirectMemoryWriter: 2D batch/model split with 2x2 physical mesh" { @@ -1681,17 +1675,16 @@ test "DirectMemoryWriter: 2D batch/model split with 2x2 physical mesh" { .physical_mesh = .{ .custom = buildMesh2x2 }, .cpu = .{ .device_count = 4 }, }, - .shape = Shape.init(.{ .batch = 8, .model = 1024 }, .f32) - .withPartitioning(.{ .batch = .batch, .model = .model }), + .shape = Shape.init(.{ .batch = 8, .model = 1024 }, .f32), .logical_mesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth, }), .strategy = .parseBindings(.{ .batch = .link_x, .model = .link_y }), - }); + }, .{ .batch = .batch, .model = .model }); } -test "DirectMemoryWriter: folded model sharding with 2x2 physical mesh" { +test "DirectMemoryWriter: folded model mesh with 2x2 physical mesh" { const case: DirectMemoryWriterDeviceTest = .{ .allocator = std.testing.allocator, .io = std.testing.io, @@ -1703,14 +1696,14 @@ test "DirectMemoryWriter: folded model sharding with 2x2 physical mesh" { .physical_mesh = .{ .custom = buildMesh2x2 }, .cpu = .{ .device_count = 4 }, }, - .shape = Shape.init(.{ .model = 4096 }, .f32).withPartitioning(.{ .model = .model }), + .shape = Shape.init(.{ .model = 4096 }, .f32), .logical_mesh = .mesh(.{ .model = .high_bandwidth }), .strategy = blk: { var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); strategy.addFold(.link_x, &.{ .link_x, .link_y }); break :blk strategy; }, - }); + }, .{ .model = .model }); } test "DirectMemoryWriter: writableSliceGreedy with mirrored shards" { @@ -1725,14 +1718,13 @@ test "DirectMemoryWriter: writableSliceGreedy with mirrored shards" { .physical_mesh = .{ .custom = buildMesh2x2 }, .cpu = .{ .device_count = 4 }, }, - .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32) - .withPartitioning(.{ .rows = .replicated, .cols = .model }), + .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32), .logical_mesh = .mesh(.{ .model = .high_bandwidth }), .strategy = .parseBindings(.{ .model = .link_x }), .write_mode = .writable_slice_greedy, .writable_slice_min_len = 64, .pool_chunk_size = 1024, - }); + }, .{ .rows = .replicated, .cols = .model }); } test "DirectMemoryWriter: 3D topology folded model + replicated batch" { @@ -1747,8 +1739,7 @@ test "DirectMemoryWriter: 3D topology folded model + replicated batch" { .physical_mesh = .{ .custom = buildMesh2x2x2 }, .cpu = .{ .device_count = 8 }, }, - .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32) - .withPartitioning(.{ .batch = .replicated, .model = .model }), + .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), .logical_mesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth, @@ -1758,7 +1749,7 @@ test "DirectMemoryWriter: 3D topology folded model + replicated batch" { strategy.addFold(.link_x, &.{ .link_x, .link_z }); break :blk strategy; }, - }); + }, .{ .batch = .replicated, .model = .model }); } test "MemoryWriter can produce a host pinned buffer" { @@ -1772,10 +1763,9 @@ test "MemoryWriter can produce a host pinned buffer" { .create_options = .{ .physical_mesh = .{ .custom = buildMesh2 }, }, - .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32) - .withPartitioning(.{ .batch = .replicated, .model = .model }), + .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), .logical_mesh = .mesh(.{ .model = .high_bandwidth }), .strategy = .parseBindings(.{ .model = .link_x }), .memory = .host_pinned, - }); + }, .{ .batch = .replicated, .model = .model }); } diff --git a/zml/kernel.zig b/zml/kernel.zig index c2114001..2d8945e4 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -955,9 +955,9 @@ test "fly kernels run on rocm" { fn f(comptime forward: anytype, p: *const zml.Platform, ta: Tensor, ha: anytype, tb: Tensor, hb: anytype) !zml.Slice { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ ta, tb }, p, .{}); defer exe.deinit(); - var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), .replicated, std.mem.sliceAsBytes(ha)); + var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), std.mem.sliceAsBytes(ha)); defer ba.deinit(); - var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), .replicated, std.mem.sliceAsBytes(hb)); + var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), std.mem.sliceAsBytes(hb)); defer bb.deinit(); var out = try exe.eval(std.testing.allocator, std.testing.io, .{ ba, bb }); defer out.deinit(); diff --git a/zml/mem.zig b/zml/mem.zig index 5ba82c69..f471a83a 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -333,7 +333,6 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu // I'm not sure I like that. I'd rather set all fields but _shards than leaving most of the m undefined. ._shape = model._shape, ._shards = .empty, - ._sharding = undefined, ._platform = undefined, }; return; diff --git a/zml/meta.zig b/zml/meta.zig index 9ff4d39a..08738e11 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -456,6 +456,7 @@ const Visitor = struct { pub const Action = enum { callback, recurse, + exit, }; pub fn determineAction(comptime callback: anytype, comptime PtrTypeOfV: type) Action { @@ -473,6 +474,13 @@ const Visitor = struct { return .callback; } + switch (@typeInfo(ptr_info.child)) { + .@"struct", .@"union" => { + if (@hasDecl(ptr_info.child, "zml_no_meta_visit")) return .exit; + }, + else => {}, + } + return .recurse; } }; @@ -487,6 +495,7 @@ pub fn visit(comptime callback: anytype, ctx: FnParam(callback, 0), v: anytype) const can_error = stdx.meta.FnReturnErrorSet(callback) != null; return switch (action) { + .exit => {}, .callback => callback(ctx, v), .recurse => { const TargetType, const mutating_cb = switch (@typeInfo(FnParam(callback, 1))) { @@ -628,8 +637,8 @@ pub fn visitFlatStruct(comptime callback: anytype, ctx: FnParam(callback, 0), v: const err_args = .{ChildTypeV}; switch (action) { + .exit => {}, .callback => return callback(ctx, v), - .recurse => { if (comptime !Contains(ChildTypeV, TargetType)) return; @@ -929,6 +938,7 @@ pub fn Contains(Haystack: type, T: type) bool { return switch (@typeInfo(Haystack)) { .@"struct" => |info| { + if (@hasDecl(Haystack, "zml_no_meta_visit")) return false; inline for (info.fields) |field| { if (!field.is_comptime and Contains(field.type, T)) { return true; @@ -937,6 +947,7 @@ pub fn Contains(Haystack: type, T: type) bool { return false; }, .@"union" => |info| { + if (@hasDecl(Haystack, "zml_no_meta_visit")) return false; inline for (info.fields) |field| { if (Contains(field.type, T)) return true; diff --git a/zml/moe/cute_mxfp4.zig b/zml/moe/cute_mxfp4.zig index 7cb235d0..9928aeb4 100644 --- a/zml/moe/cute_mxfp4.zig +++ b/zml/moe/cute_mxfp4.zig @@ -47,7 +47,7 @@ pub fn fusedExperts( stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "cute_mxfp4 expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "cute_mxfp4 expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); - const expert_parallelism = gate_up.weight.shape().partition(.expert).eql(.init(.experts)); + const expert_parallelism = gate_up.weight.shape().partition(.expert).isSharded(); const hidden = down.weight.dim(1); const intermediate = down.weight.dim(2) * 2; diff --git a/zml/moe/cutlass_flashinfer.zig b/zml/moe/cutlass_flashinfer.zig index 18527910..f90a7882 100644 --- a/zml/moe/cutlass_flashinfer.zig +++ b/zml/moe/cutlass_flashinfer.zig @@ -252,7 +252,7 @@ fn ffiCallNvfp4( const routedNvfp4Call = zml.ops.CustomCall(Input, Output, Attributes, ffiCallNvfp4, .{ .name = "flashinfer_cutlass_nvfp4_routed_moe", - // Expert sharding is owned by forwardMoe outer manual computation. + // Expert mesh is owned by forwardMoe outer manual computation. .sharding_aware = false, .has_side_effect = false, }); @@ -694,7 +694,7 @@ pub fn fusedExperts( // TODO(Corentin): Do error checking on nvfp4 // Also, maybe pass `zml.nn.Linear` directly - if (expert_partition.eql(.init(.experts))) { + if (expert_partition.isSharded()) { return zml.ops.manualComputation( (struct { input: zml.Tensor, @@ -799,7 +799,7 @@ pub fn fusedExperts( ); } - if (expert_partition.eql(.init(.experts))) { + if (expert_partition.isSharded()) { return zml.ops.manualComputation( (struct { input: zml.Tensor, diff --git a/zml/moe/fused_experts.zig b/zml/moe/fused_experts.zig index 2c58baa8..b59e892b 100644 --- a/zml/moe/fused_experts.zig +++ b/zml/moe/fused_experts.zig @@ -46,7 +46,7 @@ pub fn fusedExperts( }; const expert_partition = gate_up.weight.shape().partition(.expert); - if (!expert_partition.eql(.init(.experts))) { + if (!expert_partition.isSharded()) { return fusedExpertsImpl(args, backend); } @@ -238,7 +238,7 @@ test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { values[if (layout == .concatenated) i else 2 * i] = .fromF32(gate); values[if (layout == .concatenated) i + 3 else 2 * i + 1] = .fromF32(up); } - var input = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&values)); + var input = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&values)); defer input.deinit(); for ([_]?f32{ null, 2 }) |threshold| { const activation: zml.moe.Activation = .{ .swiglu = .{ .limit = threshold } }; @@ -270,7 +270,7 @@ test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { // }; // const x: Tensor = .init(.{ .token = 1, .out = 5 }, .f32); // const values = [_]f32{ -3, 0, 1, 2, 4 }; -// var input = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&values)); +// var input = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&values)); // defer input.deinit(); // for ([_]?f32{ null, 2 }) |threshold| { // var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, threshold }, .{}); @@ -412,7 +412,7 @@ test "FP8 routed GEMM with bias matches dequantized weights" { @as(f32, @floatFromInt(@as(i32, @intCast((j * 7 + j / 256) % 13)) - 6)) / 4), } } - buffers[i] = try zml.Buffer.fromSlice(io, platform, slice, .replicated); + buffers[i] = try zml.Buffer.fromSlice(io, platform, slice); initialized += 1; } var output: zml.Bufferized(Local.Outputs) = undefined; @@ -491,7 +491,7 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { const host = try zml.Slice.alloc(allocator, x.shape()); defer host.free(allocator); @memset(host.items(zml.floats.BFloat16), .fromF32(1.0 / 128.0)); - var input = try zml.Buffer.fromSlice(io, platform, host, .replicated); + var input = try zml.Buffer.fromSlice(io, platform, host); defer input.deinit(); for ([_]ProjectionLayout{ .concatenated, .interleaved }) |layout| { for ([_]RoutingWeightPlacement{ .before_down, .after_down }) |placement| { diff --git a/zml/moe/mosaic_tpu.zig b/zml/moe/mosaic_tpu.zig index 6f6a7bc5..f43d1b84 100644 --- a/zml/moe/mosaic_tpu.zig +++ b/zml/moe/mosaic_tpu.zig @@ -238,7 +238,7 @@ pub fn fusedExperts( const expert_partition = gate_up.weight.shape().partition(.expert); - if (expert_partition.eql(.init(.experts))) { + if (expert_partition.isSharded()) { const global_num_experts = down.weight.dim(.expert); const partial_output = zml.ops.manualComputation( (struct { diff --git a/zml/moe/triton_mxfp4.zig b/zml/moe/triton_mxfp4.zig index bf6bbdf5..d20ffe8f 100644 --- a/zml/moe/triton_mxfp4.zig +++ b/zml/moe/triton_mxfp4.zig @@ -39,7 +39,7 @@ pub fn fusedExperts( stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "triton_mxfp4 expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "triton_mxfp4 expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); - const expert_parallelism = gate_up.weight.shape().partition(.expert).eql(.init(.experts)); + const expert_parallelism = gate_up.weight.shape().partition(.expert).isSharded(); const context: Context = .{ .input = input, diff --git a/zml/nn.zig b/zml/nn.zig index a49865fa..1403788f 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -141,7 +141,7 @@ pub fn scaledDot( const Axes = stdx.BoundedArray(i64, constants.MAX_RANK); - var res_shape: Shape = .{ ._dtype = out_dtype }; + var res_shape: Shape = .scalar(out_dtype); var lhs_batching_axes: Axes = .empty; var rhs_batching_axes: Axes = .empty; for (dot_axes.batching.constSlice()) |b_axes| { @@ -290,7 +290,7 @@ test "block128 scaled dot layouts" { slice.items(zml.floats.BFloat16)[j] = .fromF32(value); } } - buffers[i] = try zml.Buffer.fromSlice(io, platform, slice, .replicated); + buffers[i] = try zml.Buffer.fromSlice(io, platform, slice); initialized += 1; } var output: zml.Bufferized(Local.Outputs) = undefined; @@ -443,7 +443,7 @@ test normalizeL2 { var exe = try platform.compileFn(std.testing.allocator, std.testing.io, normalizeL2, .{ input, 1e-12 }, .{}); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&[_]f32{ -0.9686, -1.0058, -1.7808, 0.6698 })); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.9686, -1.0058, -1.7808, 0.6698 })); defer input_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); @@ -1041,7 +1041,7 @@ test rope { defer exe_sequential.deinit(); const x_values: [5][4]f32 = @splat(.{ 1.0, 0.1, -1.0, -0.5 }); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&x_values)); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&x_values)); defer x_buffer.deinit(); var res1 = try exe_interleaved.eval(std.testing.allocator, std.testing.io, .{x_buffer}); @@ -1075,7 +1075,7 @@ test "rope: Proportional" { defer exe.deinit(); const x_h: [5][16]f32 = @splat([_]f32{ 0.754, -0.146, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }); - var x_buffer: zml.Buffer = try .fromBytes(io, platform, x.shape(), .replicated, @ptrCast(&x_h)); + var x_buffer: zml.Buffer = try .fromBytes(io, platform, x.shape(), @ptrCast(&x_h)); defer x_buffer.deinit(); const expected_h: [5][16]f32 = .{ @@ -1127,7 +1127,7 @@ test "rope: Yarn with partial_rotary_factor" { defer exe.deinit(); const x_h: [5][16]f32 = @splat([_]f32{ 0.754, -0.146, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }); - var x_buffer: zml.Buffer = try .fromBytes(io, platform, x.shape(), .replicated, @ptrCast(&x_h)); + var x_buffer: zml.Buffer = try .fromBytes(io, platform, x.shape(), @ptrCast(&x_h)); defer x_buffer.deinit(); const expected_h: [5][16]f32 = .{ @@ -1223,7 +1223,6 @@ test nearest { std.testing.io, platform, input_3d_basic.shape(), - .replicated, std.mem.sliceAsBytes(&[1][1][2]i32{.{.{ 1, 2 }}}), ); defer input_3d_basic_buffer.deinit(); @@ -1248,7 +1247,7 @@ test nearest { ); defer exe.deinit(); - var input_3d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_3d_advanced.shape(), .replicated, std.mem.sliceAsBytes(&[2][3][4]i32{ + var input_3d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_3d_advanced.shape(), std.mem.sliceAsBytes(&[2][3][4]i32{ .{ .{ 1, 2, 3, 4 }, .{ 5, 6, 7, 8 }, .{ 9, 10, 11, 12 } }, .{ .{ 13, 14, 15, 16 }, .{ 17, 18, 19, 20 }, .{ 21, 22, 23, 24 } }, })); @@ -1285,7 +1284,7 @@ test nearest { ); defer exe.deinit(); - var input_4d_basic_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_basic.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); + var input_4d_basic_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_basic.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); defer input_4d_basic_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_4d_basic_buffer}); @@ -1314,7 +1313,7 @@ test nearest { ); defer exe.deinit(); - var input_4d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_advanced.shape(), .replicated, std.mem.sliceAsBytes(&[2][2][2][2]i32{ .{ + var input_4d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_advanced.shape(), std.mem.sliceAsBytes(&[2][2][2][2]i32{ .{ .{ .{ 1, 2 }, .{ 3, 4 } }, .{ .{ 5, 6 }, .{ 7, 8 } }, }, .{ @@ -1371,7 +1370,7 @@ test nearest { ); defer exe.deinit(); - var input_5d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_5d.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); + var input_5d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_5d.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); defer input_5d_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_5d_buffer}); @@ -1821,7 +1820,6 @@ test "gated delta net" { std.testing.io, platform, queries.shape(), - .replicated, std.mem.sliceAsBytes(&[2][2][2]f32{ .{ .{ 1.0, 0.0 }, .{ 0.0, 1.0 } }, .{ .{ 1.0, 1.0 }, .{ 1.0, -1.0 } }, @@ -1832,7 +1830,6 @@ test "gated delta net" { std.testing.io, platform, keys.shape(), - .replicated, std.mem.sliceAsBytes(&[2][2][2]f32{ .{ .{ 1.0, 2.0 }, .{ 0.0, 1.0 } }, .{ .{ 2.0, 1.0 }, .{ 1.0, 0.0 } }, @@ -1843,7 +1840,6 @@ test "gated delta net" { std.testing.io, platform, values.shape(), - .replicated, std.mem.sliceAsBytes(&[2][2][2]f32{ .{ .{ 3.0, 1.0 }, .{ 2.0, 4.0 } }, .{ .{ 1.0, 5.0 }, .{ 3.0, 0.0 } }, @@ -1854,7 +1850,6 @@ test "gated delta net" { std.testing.io, platform, alphas.shape(), - .replicated, std.mem.sliceAsBytes(&[2][2]f32{ .{ 0.5, 0.25 }, .{ 0.8, 0.6 }, @@ -1865,7 +1860,6 @@ test "gated delta net" { std.testing.io, platform, betas.shape(), - .replicated, std.mem.sliceAsBytes(&[2][2]f32{ .{ 1.0, 0.5 }, .{ 0.75, 1.0 }, @@ -1876,7 +1870,6 @@ test "gated delta net" { std.testing.io, platform, initial_s.shape(), - .replicated, std.mem.sliceAsBytes(&[2][2][2]f32{ .{ .{ 1.0, 0.0 }, .{ 0.0, 1.0 } }, .{ .{ 2.0, 1.0 }, .{ 1.0, 0.0 } }, @@ -2004,7 +1997,7 @@ test sampleTokens { var exe = try platform.compileFn(std.testing.allocator, std.testing.io, sampleTokens, .{ activations, .{ .topk = 4, .temperature = 2.0 }, rng }, .{}); defer exe.deinit(); - var rng_buffer = try zml.Tensor.Rng.initBuffer(std.testing.io, platform, .replicated, 0xdeadbeef); + var rng_buffer = try rng.initBuffer(std.testing.io, platform, 0xdeadbeef); defer rng_buffer._state.deinit(); const inf = std.math.inf(f32); @@ -2016,7 +2009,7 @@ test sampleTokens { }) |logits_expected| { const logits, const expected: i32 = logits_expected; - var activations_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, activations.shape(), .replicated, std.mem.sliceAsBytes(&logits)); + var activations_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, activations.shape(), std.mem.sliceAsBytes(&logits)); defer activations_buffer.deinit(); var sampled: zml.Buffer = undefined; @@ -2149,7 +2142,7 @@ test sampleTokensDynamic { var exe = try platform.compileFn(std.testing.allocator, std.testing.io, fixupLogits, .{ logits, dynamic_sampling_strategy }, .{}); defer exe.deinit(); - var logits_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits.shape(), .replicated, std.mem.sliceAsBytes(&logits_data)); + var logits_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits.shape(), std.mem.sliceAsBytes(&logits_data)); defer logits_buffer.deinit(); const Args = struct { DynamicSamplingStrategy.Opts, [4]f32 }; @@ -2193,7 +2186,7 @@ test sampleTokensDynamic { const boost = bf16.inf; const nerf = bf16.minus_inf; - var logits_bf16_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits_bf16.shape(), .replicated, std.mem.sliceAsBytes(&[4]bf16{ boost, boost, bf16.fromF32(2), nerf })); + var logits_bf16_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits_bf16.shape(), std.mem.sliceAsBytes(&[4]bf16{ boost, boost, bf16.fromF32(2), nerf })); defer logits_bf16_buffer.deinit(); var dynamic_sampling_strategy_bf16_buffers = try DynamicSamplingStrategy.makeBuffers(std.testing.io, platform, .bf16, .{ .top_k = 4, .top_p = 0.9, .min_p = 0.1 }); defer DynamicSamplingStrategy.deinitBuffers(&dynamic_sampling_strategy_bf16_buffers); diff --git a/zml/ops.zig b/zml/ops.zig index fd97ceb1..186805ea 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -21,6 +21,7 @@ const ShapeToCustomCallBuffer = @import("pjrtx.zig").ShapeToCustomCallBuffer; const Sharding = @import("Sharding.zig"); const Tensor = @import("tensor.zig").Tensor; const TensorToCustomCallBuffer = @import("pjrtx.zig").TensorToCustomCallBuffer; +const log = std.log.scoped(.@"zml/Compiler"); pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@TypeOf(inputs)) { const ctx = Compiler.current(); @@ -66,7 +67,7 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@T else => @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"), }; - const num_devices = ctx.partitioning.numPartitions(); + const num_devices = ctx.meshes[0].numPartitions(); if (num_devices <= 1) return inputs; const reducer_block = b: { @@ -635,9 +636,9 @@ test @"while" { ); defer exe.deinit(); - var i_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_i.shape(), .replicated, std.mem.sliceAsBytes(&[1]i64{1})); + var i_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_i.shape(), std.mem.sliceAsBytes(&[1]i64{1})); defer i_buffer.deinit(); - var sum_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_sum.shape(), .replicated, std.mem.sliceAsBytes(&[1]i64{0})); + var sum_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_sum.shape(), std.mem.sliceAsBytes(&[1]i64{0})); defer sum_buffer.deinit(); var results: zml.Bufferized(While.State) = undefined; @@ -865,10 +866,10 @@ fn fromMlirOperationWithTags(op: *const mlir.Operation, base: anytype) @TypeOf(b fn cb(inner_ctx: *LocalContext, tensor: *Tensor) void { var new = Tensor.fromMlirValue(inner_ctx.op.result(inner_ctx.index)); stdx.debug.internalAssert(new.rank() == tensor.rank(), "expected operand result to have rank {} but got {f}", .{ tensor.rank(), new }); - // copy tags and sharding info over + // copy tags and mesh info over // some ops can change dims eg reduceWindow, so we trust mlir here. new._shape._tags = tensor._shape._tags; - new._shape._partitioning = tensor._shape._partitioning; + new._shape._sharding = tensor._shape._sharding; tensor.* = new; inner_ctx.index += 1; } @@ -1055,7 +1056,7 @@ test "fly custom call, both entry ABIs" { fn f(comptime forward: anytype, p: *const zml.Platform, t: Tensor, h: []const f32) !zml.Slice { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{t}, p, .{}); defer exe.deinit(); - var buf: zml.Buffer = try .fromBytes(std.testing.io, p, t.shape(), .replicated, std.mem.sliceAsBytes(h)); + var buf: zml.Buffer = try .fromBytes(std.testing.io, p, t.shape(), std.mem.sliceAsBytes(h)); defer buf.deinit(); var out = try exe.eval(std.testing.allocator, std.testing.io, .{buf}); defer out.deinit(); @@ -1274,9 +1275,9 @@ test "triton" { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, TritonMod.forward, .{ a, b }, platform, .{}); defer exe.deinit(); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&[1]f32{1})); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[1]f32{1})); defer a_buffer.deinit(); - var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), .replicated, std.mem.sliceAsBytes(&[1]f32{3})); + var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), std.mem.sliceAsBytes(&[1]f32{3})); defer b_buffer.deinit(); var results: [2]zml.Buffer = undefined; @@ -1497,7 +1498,7 @@ test "cute" { var input: [128]f32 = undefined; for (&input, 0..) |*v, i| v.* = @floatFromInt(i); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&input)); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); defer a_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); @@ -1556,7 +1557,7 @@ test "cuda_tile" { var input: [128]f32 = undefined; for (&input, 0..) |*v, i| v.* = @floatFromInt(i); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&input)); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); defer a_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); @@ -1623,7 +1624,7 @@ test "cuda_tile grid" { defer exe.deinit(); const input = [_]f32{0} ** n; - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&input)); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); defer a_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); @@ -1683,7 +1684,7 @@ test "cuda_tile zeroed_outputs" { var input: [128]f32 = undefined; for (&input, 0..) |*v, i| v.* = @floatFromInt(i); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&input)); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); defer a_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); @@ -2109,7 +2110,7 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, if (indices_shape.count() == 1 and idx_axes.len == 1) { return self .slice(idx_axes[0], .dyn(indices_per_axis.get(0).asScalar(), 1)) - // Keep downstream resharding after the slice. SPMD engines like Neuron otherwise + // Keep downstream remeshe after the slice. SPMD engines like Neuron otherwise // may hoist an all-gather before the slice and materialize the full tensor. .optimizationBarrier() .reshape(res_shape); @@ -2522,6 +2523,7 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, }; } +/// Runs a per-shard body using one mesh for all input and output partition specs. pub fn manualComputation( comptime body_fn: anytype, inputs: stdx.meta.FnParam(body_fn, 0), @@ -2576,6 +2578,47 @@ fn manualComputationLocalizeInputs(allocator: std.mem.Allocator, inputs: anytype return local_inputs; } +fn manualComputationMesh(inputs: []const Shape, outputs: []const Shape, fallback: *const Sharding.Mesh) error{IncompatibleMeshes}!*const Sharding.Mesh { + var mesh: ?*const Sharding.Mesh = null; + for ([_][]const Shape{ inputs, outputs }) |shapes| { + for (shapes) |shape| { + if (shape._sharding.mesh) |shape_mesh| { + if (mesh) |m| { + if (m != shape_mesh) return error.IncompatibleMeshes; + } else mesh = shape_mesh; + } + } + } + return mesh orelse fallback; +} + +test "manualComputation infers one mesh from inputs and outputs" { + const fallback: Sharding.Mesh = .{ + .name = "fallback", + .physical = undefined, + .logical = .mesh(.{ .batch = .balanced }), + .bindings = .empty, + .folds = .empty, + .folds_consumed = .empty, + }; + var first = fallback; + first.name = "first"; + var second = fallback; + second.name = "second"; + const replicated: Shape = .init(.{8}, .f32); + var sharded = replicated; + sharded._sharding = .{ .mesh = &first, .partition = .init(&.{.mesh_axis_0}) }; + var other = sharded; + other._sharding.mesh = &second; + + try std.testing.expectEqual(&fallback, try manualComputationMesh(&.{replicated}, &.{replicated}, &fallback)); + try std.testing.expectEqual(&first, try manualComputationMesh(&.{ replicated, sharded, sharded }, &.{sharded}, &fallback)); + try std.testing.expectEqual(&first, try manualComputationMesh(&.{replicated}, &.{sharded}, &fallback)); + try std.testing.expectEqual(&first, try manualComputationMesh(&.{}, &.{sharded}, &fallback)); + try std.testing.expectError(error.IncompatibleMeshes, manualComputationMesh(&.{ sharded, other }, &.{}, &fallback)); + try std.testing.expectError(error.IncompatibleMeshes, manualComputationMesh(&.{sharded}, &.{other}, &fallback)); +} + fn manualComputationInternal( inputs: anytype, outputs: []const Shape, @@ -2596,35 +2639,21 @@ fn manualComputationInternal( const local_input_shapes = try arena.alloc(Shape, input_shapes.len); const local_output_shapes = try arena.alloc(Shape, outputs.len); - const input_shardings = try arena.alloc(Sharding, input_shapes.len); - const output_shardings = try arena.alloc(Sharding, outputs.len); + const manual_mesh = manualComputationMesh(input_shapes, outputs, ctx.platform.replicated_mesh) catch + @panic("manualComputation expects all inputs and outputs to use the same mesh"); for (input_shapes, 0..) |shape, i| { - const sharding = ctx.partitioning.selectSharding(shape) catch |err| switch (err) { - error.NoSuitableSharding => std.debug.panic( - "failed to shard manualComputation input {f}({d}) because it's using unknown sharding. Pass more shardings to `.compile`. Known shardings: {f}", - .{ shape, i, stdx.fmt.slice(ctx.partitioning.shardings) }, - ), - }; - input_shardings[i] = sharding; - local_input_shapes[i] = sharding.shardedShape(shape) catch std.debug.panic("can't shard {f} for {f}", .{ shape, sharding }); + local_input_shapes[i] = Sharding.shardedShape(shape) catch std.debug.panic("can't shard {f} for {f}", .{ shape, manual_mesh }); } for (outputs, 0..) |shape, i| { - const sharding = ctx.partitioning.selectSharding(shape) catch |err| switch (err) { - error.NoSuitableSharding => std.debug.panic( - "failed to shard manualComputation output {f}({d}) because it's using unknown sharding. Pass more shardings to `.compile`. Known shardings: {f}", - .{ shape, i, stdx.fmt.slice(ctx.partitioning.shardings) }, - ), - }; - output_shardings[i] = sharding; - local_output_shapes[i] = sharding.shardedShape(shape) catch std.debug.panic("can't shard {f} for {f}", .{ shape, sharding }); + local_output_shapes[i] = Sharding.shardedShape(shape) catch std.debug.panic("can't shard {f} for {f}", .{ shape, manual_mesh }); } - return switch (ctx.partitioning.partitioner) { + return switch (ctx.partitioner) { .shardy => { - const in_shardings_attr = try Sharding.sdyPerValueShardingAttr(arena, ctx.mlir_ctx, input_shapes, input_shardings); - const out_shardings_attr = try Sharding.sdyPerValueShardingAttr(arena, ctx.mlir_ctx, outputs, output_shardings); - const manual_axes_attr = try Sharding.sdyManualAxesAttr(arena, ctx.mlir_ctx, input_shapes, input_shardings, outputs, output_shardings); + const in_meshes_attr = try Sharding.sdyPerValueShardingAttr(arena, ctx.mlir_ctx, input_shapes, manual_mesh); + const out_meshes_attr = try Sharding.sdyPerValueShardingAttr(arena, ctx.mlir_ctx, outputs, manual_mesh); + const manual_axes_attr = try Sharding.sdyManualAxesAttr(arena, ctx.mlir_ctx, input_shapes, outputs, manual_mesh); const block_types = try arena.alloc(*const mlir.Type, input_shapes.len); for (local_input_shapes, 0..) |input_shape, i| { @@ -2675,8 +2704,8 @@ fn manualComputationInternal( .results = .{ .flat = global_result_types }, .blocks = &.{manual_block}, .attributes = &.{ - .named(ctx.mlir_ctx, "in_shardings", in_shardings_attr), - .named(ctx.mlir_ctx, "out_shardings", out_shardings_attr), + .named(ctx.mlir_ctx, "in_shardings", in_meshes_attr), + .named(ctx.mlir_ctx, "out_shardings", out_meshes_attr), .named(ctx.mlir_ctx, "manual_axes", manual_axes_attr), }, .verify = true, @@ -2686,7 +2715,7 @@ fn manualComputationInternal( // Use the compiler allocator to return memory to the parent const sharded_outputs = ctx.alloc(Tensor, outputs.len); for (outputs, 0..) |output, i| { - sharded_outputs[i] = Tensor._result(output, op.result(i)); + sharded_outputs[i] = Tensor._resultPropagateSharding(output, op.result(i)); } return sharded_outputs; }, @@ -2732,8 +2761,8 @@ fn manualComputationInternal( const global_values = try arena.alloc(*const mlir.Value, outputs.len); const global_types = try arena.alloc(*const mlir.Type, outputs.len); - for (outputs, output_shardings, 0..) |output_shape, output_sharding, i| { - const gspmd_attr = try ctx.partitioning.tensorShardingAttr(arena, ctx.mlir_ctx, output_shape, output_sharding); + for (outputs, 0..) |output_shape, i| { + const gspmd_attr = try ctx.tensorShardingAttr(arena, ctx.mlir_ctx, output_shape); global_types[i] = mlirx.Type.rankedTensor(ctx.mlir_ctx, output_shape); const shard_to_full = dialects.stablehlo.custom_call( @@ -2762,7 +2791,7 @@ fn manualComputationInternal( const sharded_outputs = ctx.alloc(Tensor, outputs.len); for (outputs, 0..) |output_shape, i| { - sharded_outputs[i] = Tensor._result(output_shape, barrier.result(i)); + sharded_outputs[i] = Tensor._resultPropagateSharding(output_shape, barrier.result(i)); } return sharded_outputs; }, @@ -3153,7 +3182,7 @@ pub fn typedCustomCall( } break :b &input_tensors; }, - // Extra case to support []const Tensor from shardingAwareTypedCustomCall + // Extra case to support []const Tensor from mesheAwareTypedCustomCall .pointer => |pointer_info| b: { if (pointer_info.size != .slice) @compileError("Expected input slice"); break :b input; @@ -3169,7 +3198,7 @@ pub fn typedCustomCall( } break :b &output_shapes; }, - // Extra case to support []const Shape from shardingAwareTypedCustomCall + // Extra case to support []const Shape from mesheAwareTypedCustomCall .pointer => |pointer_info| b: { if (pointer_info.size != .slice) @compileError("Expected input slice"); break :b output; @@ -3234,7 +3263,7 @@ pub fn typedCustomCall( ctx.location, ).appendTo(ctx.currentScope().block); - if (ctx.manual_computation_depth > 0 and ctx.partitioning.partitioner == .gspmd) { + if (ctx.manual_computation_depth > 0 and ctx.partitioner == .gspmd) { op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); } @@ -3246,7 +3275,7 @@ pub fn typedCustomCall( } return out; }, - // Extra case to support []const Shape from shardingAwareTypedCustomCall + // Extra case to support []const Shape from mesheAwareTypedCustomCall .pointer => |pointer_info| { if (pointer_info.size != .slice) @compileError("Expected input slice"); var out: []Tensor = allocator.alloc(Tensor, output_shapes.len) catch unreachable; @@ -3307,7 +3336,7 @@ test customCall { const scope = comp.pushBlock(block); defer scope.pop(); - const shape = zml.Shape.init(.{128}, .bf16).withPartitioning(.{ ._0 = .x }); + const shape = zml.Shape.init(.{128}, .bf16).withPartitioning(platform.meshes.get("model").?, .{ ._0 = .model }); const input = Tensor.constant(zml.DataType.bf16.constant(0)).broad(shape); const output = customCall("my_custom_call", .{input}, .{zml.Shape.init(.{128}, .bf16)}, .{}, .{ .has_side_effect = false, diff --git a/zml/platform.zig b/zml/platform.zig index 826eeaee..7a9ee710 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -296,8 +296,8 @@ pub const Platform = struct { devices: []const Device, memories: []const Memory, physical_mesh: zml.Sharding.PhysicalMesh, - replicated_sharding: zml.Sharding, - shardings: std.StringArrayHashMapUnmanaged(zml.Sharding), + meshes: std.StringArrayHashMapUnmanaged(*const zml.Mesh), + replicated_mesh: *const zml.Mesh, io_impl: IoImpl, pub const MAX_NUM_DEVICES: u16 = if (Target.tpu.isEnabled()) 64 else 32; @@ -337,12 +337,12 @@ pub const Platform = struct { .pjrt_api = api, .pjrt_client = pjrt_client, .state = State.init(target), - .shardings = .empty, + .meshes = .empty, + .replicated_mesh = undefined, // set below .devices = undefined, .memories = undefined, .physical_mesh = undefined, - .replicated_sharding = undefined, .io_impl = options.io_impl, }; break :platform platform; @@ -350,7 +350,7 @@ pub const Platform = struct { const arena = platform.arena.allocator(); errdefer platform.arena.deinit(); - try platform.shardings.ensureTotalCapacity(arena, 8); + try platform.meshes.ensureTotalCapacity(arena, 8); { const devices = try arena.alloc(Device, pjrt_devices.len); @@ -375,7 +375,7 @@ pub const Platform = struct { .auto => zml.Sharding.PhysicalMesh.auto(arena, target, devices), .custom => |builder| builder(arena, target, devices), }; - platform.replicated_sharding = try platform.registerSharding("replicated", .mesh(.{ .x = .high_bandwidth })); + platform.replicated_mesh = try platform.registerMesh("replicated", .mesh(.{ .all = .high_bandwidth })); } switch (target) { @@ -639,38 +639,37 @@ pub const Platform = struct { return try profiler_.profiler(self.pjrt_api, allocator, io, options); } - /// Create a Sharding based on the given logical mesh and the default strategy. + /// Create a Mesh based on the given logical mesh and the default strategy. /// Memory is owned by the platform, making it safe to copy around. - pub fn registerSharding(platform: *Platform, name: []const u8, logical: Sharding.LogicalMesh) error{OutOfMemory}!Sharding { - return platform.registerShardingWithStrategy( + pub fn registerMesh(platform: *Platform, name: []const u8, logical: Sharding.LogicalMesh) error{OutOfMemory}!*const Sharding.Mesh { + return platform.registerMeshWithStrategy( name, logical, .suggest(logical, &platform.physical_mesh), ) catch |err| switch (err) { error.InvalidPhysicalMesh, error.InvalidStrategy, error.InvalidPhysicalAxis => { - std.debug.panic("ZML failed to create a valid sharding for logical mesh: {f}\nand physical_mesh: {f}\nPlease report this bug.", .{ logical, platform.physical_mesh }); + std.debug.panic("ZML failed to create a valid mesh for logical mesh: {f}\nand physical_mesh: {f}\nPlease report this bug.", .{ logical, platform.physical_mesh }); }, error.OutOfMemory => |e| return e, }; } - /// Create a Sharding based on the given logical mesh and a strategy. + /// Create a Mesh based on the given logical mesh and a strategy. /// Memory is owned by the platform, making it safe to copy around. - pub fn registerShardingWithStrategy(platform: *Platform, name: []const u8, logical: Sharding.LogicalMesh, strategy: Sharding.Strategy) !Sharding { + pub fn registerMeshWithStrategy(platform: *Platform, name: []const u8, logical: Sharding.LogicalMesh, strategy: Sharding.Strategy) !*const Sharding.Mesh { const arena = platform.arena.allocator(); - const entry = try platform.shardings.getOrPut(arena, name); + const entry = try platform.meshes.getOrPut(arena, name); if (entry.found_existing) { - std.debug.panic("Another sharding already exists with this name: {s}", .{name}); + std.debug.panic("Another mesh already exists with this name: {s}", .{name}); } const owned_name = try arena.dupe(u8, name); - const owned_data = try arena.create(Sharding.Data); - owned_data.* = try .init(owned_name, &platform.physical_mesh, logical, strategy); - const sharding: Sharding = .{ .data = owned_data }; + const owned_mesh = try arena.create(Sharding.Mesh); + owned_mesh.* = try .init(owned_name, &platform.physical_mesh, logical, strategy); entry.key_ptr.* = owned_name; - entry.value_ptr.* = sharding; + entry.value_ptr.* = owned_mesh; - return sharding; + return owned_mesh; } fn memoryFromPjrt(self: *const Platform, pjrt_memory: *const pjrt.Memory) *const Memory { diff --git a/zml/quantization.zig b/zml/quantization.zig index 489162fb..b125513a 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -436,7 +436,7 @@ test "block FP8 quantization preserves axes and reconstructs constant blocks in else => -magnitude, }); } - var buffer = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&host)); + var buffer = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&host)); defer buffer.deinit(); var output: zml.Bufferized(Local.Outputs) = undefined; var runner = try exe.runner(allocator); @@ -495,7 +495,7 @@ test "block FP8 E8M0 scales preserve power-of-two boundaries and nonminor axes" const exponent = (bits >> 23) + @intFromBool(bits & 0x7fffff != 0); scale.* = @bitCast(exponent << 23); } - var buffer = try zml.Buffer.fromBytes(io, platform, x.shape(), .replicated, std.mem.asBytes(&host)); + var buffer = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&host)); defer buffer.deinit(); var output: zml.Bufferized(Local.Outputs) = undefined; var runner = try exe.runner(allocator); diff --git a/zml/shape.zig b/zml/shape.zig index 8afae990..11a3efac 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -6,141 +6,26 @@ const stdx = @import("stdx"); const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; +const Sharding = @import("Sharding.zig"); const log = std.log.scoped(.shape); +test { + std.testing.refAllDecls(Shape); +} + /// Represent the shape of a tensor. +/// Contains also the sharding information, +/// so it describes exactly how the corresponding tensor/buffer will be layed out +/// across all devices of the current Platform. +/// +/// By default all created Shape are replicated, meaning the corresponding tensor / buffer will be copied on each device. +/// `.withPartitioning` allows to specify how it should be split. pub const Shape = struct { - pub const MAX_RANK: u8 = 8; - - pub const PartitionSpec = union(enum) { - axis: Tag, - open, - replicated, - unknown, - - pub fn init(comptime value: anytype) PartitionSpec { - const T = @TypeOf(value); - - if (T == PartitionSpec) { - return value; - } - - if (comptime T == @EnumLiteral()) { - const name = @tagName(value); - inline for (std.meta.fields(PartitionSpec)) |field| { - if (field.type == void and std.mem.eql(u8, field.name, name)) { - return @field(PartitionSpec, field.name); - } - } - } - - if (comptime isTagConvertible(T)) { - const tag_ = Shape.toTag(value); - - return .{ .axis = tag_ }; - } - - stdx.debug.compileError("PartitionSpec.init expected a tag, got: {any}", .{T}); - } - - test "PartitionSpec.init" { - const spec_axis: PartitionSpec = .{ .axis = Shape.toTag(.a) }; - try testing.expectEqual(spec_axis, PartitionSpec.init(.a)); - - const spec_axis_open: PartitionSpec = .open; - try testing.expect(spec_axis_open.eql(.init(.open))); - } - - pub fn eql(self: PartitionSpec, other: PartitionSpec) bool { - if (@as(u4, @intFromEnum(self)) != @as(u4, @intFromEnum(other))) { - return false; - } - - switch (self) { - .axis => |t1| { - const t2 = other.axis; - return std.mem.eql(u8, std.mem.span(t1), std.mem.span(t2)); - }, - else => return true, - } - } - - test "PartitionSpec.eql" { - const spec_axis: PartitionSpec = .init(.a); - const spec_axis_b: PartitionSpec = .init(.b); - const spec_open: PartitionSpec = .open; - const spec_replicated: PartitionSpec = .replicated; - const spec_unknown: PartitionSpec = .unknown; - - try testing.expect(spec_axis.eql(.{ .axis = Shape.toTag(.a) })); - try testing.expect(!spec_axis.eql(spec_axis_b)); - try testing.expect(!spec_axis.eql(.open)); - try testing.expect(!spec_axis.eql(.replicated)); - try testing.expect(!spec_axis.eql(.unknown)); - - try testing.expect(spec_open.eql(.open)); - try testing.expect(!spec_open.eql(spec_axis)); - try testing.expect(!spec_open.eql(.replicated)); - try testing.expect(!spec_open.eql(.unknown)); - - try testing.expect(spec_replicated.eql(.replicated)); - try testing.expect(!spec_replicated.eql(spec_axis)); - try testing.expect(!spec_replicated.eql(.open)); - try testing.expect(!spec_replicated.eql(.unknown)); - - try testing.expect(spec_unknown.eql(.unknown)); - try testing.expect(!spec_unknown.eql(spec_axis)); - try testing.expect(!spec_unknown.eql(.open)); - try testing.expect(!spec_unknown.eql(.replicated)); - } - - pub fn isClosed(self: PartitionSpec) bool { - return switch (self) { - .axis => true, - .open => false, - .replicated => true, - .unknown => false, - }; - } - - test isClosed { - const spec_axis: PartitionSpec = .init(.a); - try testing.expect(spec_axis.isClosed()); - - const spec_open: PartitionSpec = .open; - try testing.expect(!spec_open.isClosed()); - - const spec_replicated: PartitionSpec = .replicated; - try testing.expect(spec_replicated.isClosed()); - - const spec_unknown: PartitionSpec = .unknown; - try testing.expect(!spec_unknown.isClosed()); - } - - pub fn toTag(self: PartitionSpec) Tag { - return switch (self) { - .axis => |t| return t, - .open => TagOpen, - .replicated => TagReplicated, - .unknown => TagUnknown, - }; - } - - test "PartitionSpec.toTag" { - const spec_axis: PartitionSpec = .init(.a); - try testing.expect(spec_axis.toTag() == Shape.toTag(.a)); - - const spec_open: PartitionSpec = .open; - try testing.expect(spec_open.toTag() == TagOpen); - - const spec_replicated: PartitionSpec = .replicated; - try testing.expect(spec_replicated.toTag() == TagReplicated); - - const spec_unknown: PartitionSpec = .unknown; - try testing.expect(spec_unknown.toTag() == TagUnknown); - } - }; + _dtype: DataType, + _dims: DimsArray, + _tags: TagsArray, + _sharding: Sharding, pub const Tag = [*:0]const u8; pub const TagUnknown = "_".ptr; @@ -148,17 +33,10 @@ pub const Shape = struct { const TagReplicated = "replicated".ptr; const TagOpen = "open".ptr; + pub const MAX_RANK: u8 = constants.MAX_RANK; pub const DimsArray = stdx.BoundedArray(i64, constants.MAX_RANK); pub const TagsArray = stdx.BoundedArray(Tag, constants.MAX_RANK); pub const AxesArray = stdx.BoundedArray(u3, constants.MAX_RANK); - pub const PartitionArray = stdx.BoundedArray(PartitionSpec, constants.MAX_RANK); - - const UnknownTags: TagsArray = .{ .len = 0, .buffer = @splat(TagUnknown) }; - - _dtype: DataType, - _dims: DimsArray = .empty, - _tags: TagsArray = UnknownTags, - _partitioning: PartitionArray = .empty, pub fn parseDimensions(v: anytype) struct { DimsArray, TagsArray } { const T = @TypeOf(v); @@ -252,28 +130,39 @@ pub const Shape = struct { /// Create a shape from a struct literal, eg: /// Shape.init(.{ .h = 1024, .w = 512, .c = 3 }); /// Shape.init(.{ 1024, 512, 3 }); + /// + /// The created shape is replicated. pub fn init(dimz: anytype, dt: DataType) Shape { - var res: Shape = .{ ._dtype = dt }; - res._dims, res._tags = parseDimensions(dimz); - res._partitioning.appendNTimes(.unknown, res._dims.len) catch @panic("Rank too large"); - - return res; + const _dims, const _tags = parseDimensions(dimz); + return .{ + ._dtype = dt, + ._dims = _dims, + ._tags = _tags, + ._sharding = .replicated, + }; } pub fn scalar(dt: DataType) Shape { - return .{ ._dtype = dt }; + return .{ + ._dtype = dt, + ._dims = .empty, + ._tags = .{ .len = 0, .buffer = @splat(TagUnknown) }, + ._sharding = .replicated, + }; } /// Creates a Shape with dims set to `.{0, 1, 2, ..., rank-1}`. pub fn range(rank_: usize, dt: DataType) Shape { - var res: Shape = .{ ._dtype = dt }; + stdx.debug.assert(rank_ <= MAX_RANK, "zml.Shape.range expects a maximum rank of {d}, got {}", .{ MAX_RANK, rank_ }); + var res: Shape = .{ + ._dtype = dt, + ._dims = .empty, + ._tags = .{ .buffer = @splat(TagUnknown), .len = rank_ }, + ._sharding = .replicated, + }; for (0..rank_) |i| { - res._dims.append(@intCast(i)) catch { - stdx.debug.panic("Too many dimensions! Max: {d}, passed: {d}", .{ res._dims.capacity(), rank_ }); - }; - res._tags.append(TagUnknown) catch unreachable; + res._dims.appendAssumeCapacity(@intCast(i)); } - res._partitioning.appendNTimes(.unknown, rank_) catch @panic("Rank too large"); return res; } @@ -319,7 +208,7 @@ pub const Shape = struct { fn ensureAttributesAreSync(self: Shape) void { if (builtin.mode == .Debug) { - stdx.debug.assert(self._dims.len == self._tags.len and self._dims.len == self._partitioning.len, "Tags, dims and partitioning have diverged! dims={d} tags={d} partitioning={d}", .{ self._dims.len, self._tags.len, self._partitioning.len }); + stdx.debug.assert(self._dims.len == self._tags.len, "Tags, dims and partitioning have diverged! dims={d} tags={d}", .{ self._dims.len, self._tags.len }); } } @@ -349,7 +238,7 @@ pub const Shape = struct { } pub fn hasTag(self: Shape, tag_: anytype) ?u3 { - return self.axisFromTagMaybe(toTag(tag_)); + return axisFromTagMaybe(self.tags(), toTag(tag_)); } pub fn hasTags(self: Shape, tagz: anytype) bool { @@ -443,7 +332,7 @@ pub const Shape = struct { @intCast(a); } - fn axisFromTagMaybe(self: Shape, t: Tag) ?u3 { + pub fn axisFromTagMaybe(tagz: []const Tag, t: Tag) ?u3 { if (t == TagUnknown) return null; if (axisFromLiteralInt(t)) |ax| return ax; @@ -451,7 +340,7 @@ pub const Shape = struct { if (@inComptime()) { // At comptime two duplicated strings may have two different representations const t_bytes: []const u8 = std.mem.span(t); - for (self.tags(), 0..) |self_tag, ax| { + for (tagz, 0..) |self_tag, ax| { if (std.mem.eql(u8, t_bytes, std.mem.span(self_tag))) { return @truncate(ax); } @@ -460,7 +349,7 @@ pub const Shape = struct { } // But at runtime the comptime strings have been deduplicated and ptr match is enough. - if (std.mem.indexOfScalar(Tag, self.tags(), t)) |ax| { + if (std.mem.indexOfScalar(Tag, tagz, t)) |ax| { return @truncate(ax); } return null; @@ -477,7 +366,7 @@ pub const Shape = struct { fn axisFromTag(self: Shape, d: Tag) u3 { stdx.debug.assert(d != TagUnknown, "The unknown tag .{s} can't be used to fetch axis in {f}", .{ d, self }); - return self.axisFromTagMaybe(d) orelse { + return axisFromTagMaybe(self.tags(), d) orelse { stdx.debug.panic("Tensor {f} doesn't have dimension with tag: {s}", .{ self, d }); }; } @@ -519,8 +408,7 @@ pub const Shape = struct { } /// Format the shape. - /// Default format: "Shape({.a=10, .b=20}, dtype=.f32)" - /// Bare format {_}: "{.a=10, .b=20}, dtype=.f32" + /// "Shape({.a=10, .b=20}, dtype=.f32)" pub fn format(self: Shape, writer: *std.Io.Writer) !void { _ = try writer.writeByte('{'); @@ -534,9 +422,16 @@ pub const Shape = struct { try writer.print("{d}", .{d}); } - const part = self._partitioning.get(i); - if (part.toTag() != TagUnknown) { - try writer.print("/{s}", .{part.toTag()}); + const partitioning = self._sharding.get(i); + switch (partitioning) { + .replicated => {}, + .open => try writer.writeAll("/?"), + // this state is invalid, but don't panic cause we may be called from a panic + .out_of_bound => try writer.writeAll("/!"), + else => { + const mesh_ax = partitioning.meshAxis().?; + try writer.print("/{s}", .{self._sharding.mesh.?.logical.axes.get(mesh_ax)}); + }, } need_comma = true; @@ -544,6 +439,10 @@ pub const Shape = struct { if (need_comma) try writer.writeByte(','); _ = try writer.write(@tagName(self.dtype())); _ = try writer.writeByte('}'); + if (self._sharding.mesh) |m| { + try writer.writeByte('@'); + try writer.writeAll(m.name); + } } /// Broadcasts a Tensor to the given shape, extending dimensions if needed. @@ -569,10 +468,17 @@ pub const Shape = struct { return true; } + /// Reshape a given shape but keep the same total number of elements. + /// Partitioning information is generally NOT preserved, + /// unless input was fully replicated. pub fn reshape(self: Shape, new_shape_: anytype) Shape { - var new_shape: Shape = .{ ._dtype = self.dtype() }; - new_shape._dims, new_shape._tags = parseDimensions(new_shape_); - new_shape._partitioning.appendNTimes(.unknown, new_shape._dims.len) catch @panic("Rank too large"); + const _dims, const _tags = parseDimensions(new_shape_); + var new_shape: Shape = .{ + ._dtype = self._dtype, + ._dims = _dims, + ._tags = _tags, + ._sharding = .{ .mesh = self._sharding.mesh, .partition = if (self.isFullyReplicated()) .replicated(_dims.len) else .open(_dims.len) }, + }; new_shape.inferMissingAxis(self.count()) catch |err| { std.debug.panic("Can't reshape {any} to {any}: {t}", .{ self.dims(), new_shape.dims(), err }); }; @@ -655,7 +561,7 @@ pub const Shape = struct { res._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; for (0..dims_.len) |_| { - res._partitioning.insert(ax, .unknown) catch unreachable; + res._sharding = res._sharding.insert(ax, .replicated); } return res; @@ -678,7 +584,7 @@ pub const Shape = struct { var res = self; res._dims.insert(ax, d) catch unreachable; res._tags.insert(ax, toTag(tag_)) catch unreachable; - res._partitioning.insert(ax, .unknown) catch unreachable; + res._sharding = res._sharding.insert(ax, .replicated); return res; } @@ -688,7 +594,7 @@ pub const Shape = struct { const dims_, const tags_ = parseDimensions(v); res._dims.appendSliceAssumeCapacity(dims_.constSlice()); res._tags.appendSliceAssumeCapacity(tags_.constSlice()); - res._partitioning.appendNTimesAssumeCapacity(.unknown, dims_.len); + for (self.rank()..res.rank()) |ax| res._sharding = res._sharding.set(ax, .replicated); return res; } @@ -710,7 +616,7 @@ pub const Shape = struct { var res = self; res._dims.appendAssumeCapacity(d); res._tags.appendAssumeCapacity(if (tag_) |t| t else TagUnknown); - res._partitioning.appendAssumeCapacity(.unknown); + res._sharding = res._sharding.set(self.rank(), .replicated); return res; } @@ -719,7 +625,7 @@ pub const Shape = struct { const a = self.axis(axis_); _ = res._dims.orderedRemove(a); _ = res._tags.orderedRemove(a); - _ = res._partitioning.orderedRemove(a); + res._sharding = res._sharding.orderedRemove(a); return res; } @@ -736,6 +642,7 @@ pub const Shape = struct { std.mem.sort(u3, to_remove.slice(), {}, std.sort.asc(u3)); var sh: Shape = self; + sh._sharding.partition = .out_of_bound; const rk = self.rank(); var res_ax: u32 = 0; for (0..rk) |ax| { @@ -745,12 +652,11 @@ pub const Shape = struct { sh._dims.buffer[res_ax] = self._dims.buffer[ax]; sh._tags.buffer[res_ax] = self._tags.buffer[ax]; - sh._partitioning.buffer[res_ax] = self._partitioning.buffer[ax]; + sh._sharding = sh._sharding.set(res_ax, self._sharding.get(ax)); res_ax += 1; } sh._dims.len = rk - to_remove.len; sh._tags.len = rk - to_remove.len; - sh._partitioning.len = rk - to_remove.len; return sh; } @@ -773,12 +679,12 @@ pub const Shape = struct { var res = self; const original_dims = self._dims; const original_tags = self._tags; - const original_parts = self._partitioning; + const original_parts = self._sharding; for (permutations_.constSlice(), 0..) |permutation, i| { res._dims.set(i, original_dims.get(permutation)); res._tags.set(i, original_tags.get(permutation)); - res._partitioning.set(i, original_parts.get(permutation)); + res._sharding = res._sharding.set(i, original_parts.get(permutation)); } return res; @@ -797,9 +703,9 @@ pub const Shape = struct { ), ); - const shape = Shape.init(.{ .a = 10, .c = 12, .b = 11 }, .f32).withPartitioning(.{ .a = .batch, .c = .colors, .b = .open }).transpose(.{ 0, 2, 1 }); + const shape = Shape.init(.{ .a = 10, .c = 12, .b = 11 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .c = .colors, .b = .open }).transpose(.{ 0, 2, 1 }); try testing.expect(shape.eqlWithTags(Shape.init(.{ .a = 10, .b = 11, .c = 12 }, .f32))); - try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .open, .init(.colors) }, shape._partitioning.constSlice()); + try testing.expectEqualDeep(Sharding.Partitioning.init(&.{ .mesh_axis_0, .open, .mesh_axis_2 }), shape._sharding.partition); } /// Tag each ax of this shape with tags from a tuple. @@ -908,154 +814,131 @@ pub const Shape = struct { return res; } - pub fn withPartitioning(self: Shape, specs: anytype) Shape { - const T = @TypeOf(specs); + // These tests only resolve logical names; placement does not use this fixture. + const test_mesh: Sharding.Mesh = .{ + .name = "test_mesh", + .physical = undefined, + .logical = .mesh(.{ .batch = .balanced, .feature = .balanced, .colors = .balanced }), + .bindings = .init(&.{ + .init(&.{.link_x}), + .init(&.{.link_y}), + .init(&.{.link_z}), + }), + .folds = .empty, + .folds_consumed = .empty, + }; - var res = self.withDefaultPartitioning(); // todo add test for this new change - - if (stdx.meta.isStruct(T)) { - inline for (std.meta.fields(T)) |field| { - const partition_axis = @field(specs, field.name); - const axis_ = res.axisFromTagMaybe(toTag(field)); - - if (axis_) |ax| { - res._partitioning.set(ax, PartitionSpec.init(partition_axis)); - } else { - stdx.debug.panic("Partitioning axis {s} not found", .{field.name}); - } - } - } else { - stdx.debug.panic("Expected a struct of enum literals, got: {any}", .{T}); + pub fn withPartitioning(self: Shape, mesh: *const Sharding.Mesh, partitioning: anytype) Shape { + if (@TypeOf(partitioning) != Sharding.Partitioning) { + return self.withPartitioning(mesh, Sharding.Partitioning.parse(self.tags(), mesh, .replicated, partitioning)); } - - // Check that no mesh axis is used to partition multiple tensor dimensions. - var used_mesh_axes: stdx.BoundedArray(Tag, constants.MAX_RANK) = .empty; - - for (res._partitioning.constSlice()) |spec| { - const axis_tag = spec.toTag(); - - if (std.mem.indexOfScalar(Tag, used_mesh_axes.constSlice(), axis_tag)) |_| { - stdx.debug.panic("Mesh axis '{s}' used to partition multiple tensor dimensions. This is not allowed.", .{axis_tag}); - } - - switch (spec) { - .unknown => {}, - .open => {}, - .replicated => {}, - .axis => used_mesh_axes.appendAssumeCapacity(axis_tag), - } + var res = self; + res._sharding = .{ .mesh = mesh, .partition = partitioning }; + for (0..self.rank()) |ax| { + if (res._sharding.get(ax) == .out_of_bound) res._sharding = res._sharding.set(ax, .replicated); } + for (self.rank()..MAX_RANK) |ax| res._sharding.partition = res._sharding.partition.set(ax, .out_of_bound); + + stdx.debug.assert(res._sharding.partition.hasUniqueAxes(), "{f}.withPartitioning({s}, ...) expects mesh axes to be used a most once, got {any}", .{ self, mesh.name, partitioning.toArray()[0..self.rank()] }); return res; } test withPartitioning { - var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(.{ .c = .feature, .a = .batch }); - try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .unknown, .init(.feature) }, shape._partitioning.constSlice()); + var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(&test_mesh, .{ .c = .feature, .a = .batch }); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .sharded(0), .replicated, .sharded(1) }, shape._sharding.toArray()[0..shape.rank()]); shape = shape.withDefaultPartitioning(); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); - shape = shape.withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); - try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .open, .init(.feature) }, shape._partitioning.constSlice()); + shape = shape.withPartitioning(&test_mesh, .{ .a = .batch, .b = .open, .c = .feature }); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .sharded(0), .open, .sharded(1) }, shape._sharding.toArray()[0..shape.rank()]); } - pub fn mapPartitioningAxes(self: Shape, mapping: anytype) Shape { + pub fn mapPartitioningAxes(self: Shape, mesh: *const Sharding.Mesh, mapping: anytype) Shape { const T = @TypeOf(mapping); stdx.debug.assertComptime(stdx.meta.isStruct(T), "Mapping must be a struct, e.g., .{ .old_axis = .new_axis }", .{}); - var res = self; - for (res._partitioning.slice()) |*spec| { - switch (spec.*) { - .unknown => {}, - .open => {}, - .replicated => {}, - .axis => |*old_axis_tag| { - inline for (std.meta.fields(T)) |field| { - const logical_axis_in_map = toTag(field); - - if (std.mem.eql(u8, std.mem.span(old_axis_tag.*), std.mem.span(logical_axis_in_map))) { - const physical_axis_from_map = @field(mapping, field.name); - spec.* = PartitionSpec.init(physical_axis_from_map); - break; - } - } - }, + for (0..self.rank()) |ax| { + const mesh_axis = self.partition(ax).meshAxis() orelse continue; + inline for (std.meta.fields(T)) |field| { + const old_axis = mesh.resolveLogicalAxis(toTag(field)) orelse + stdx.debug.panic("Mesh '{s}' has no logical axis '{s}'", .{ mesh.name, field.name }); + if (mesh_axis == old_axis) { + const new_tag = toTag(@field(mapping, field.name)); + const new_axis = mesh.resolveLogicalAxis(new_tag) orelse + stdx.debug.panic("Mesh '{s}' has no logical axis '{s}'", .{ mesh.name, new_tag }); + res._sharding = res._sharding.set(ax, .sharded(@intCast(new_axis))); + break; + } } } return res; } test mapPartitioningAxes { - const logical_shape = Shape.init(.{ .batch = 64, .seq = 128, .features = 512 }, .f32).withPartitioning(.{ - .batch = .data, - .seq = .replicated, - .features = .model, - }); - - const physical_shape = logical_shape.mapPartitioningAxes(.{ .data = .x, .model = .y }); - - try testing.expect(physical_shape.partition(.batch).eql(.init(.x))); - try testing.expect(physical_shape.partition(.features).eql(.init(.y))); - try testing.expect(physical_shape.partition(.seq).eql(.replicated)); - - const partial_mapped_shape = logical_shape.mapPartitioningAxes(.{ .data = .z }); - try testing.expect(partial_mapped_shape.partition(.batch).eql(.init(.z))); - try testing.expect(partial_mapped_shape.partition(.features).eql(.init(.model))); + const shape = Shape.init(.{ .batch = 64, .seq = 128, .features = 512 }, .f32) + .withPartitioning(&test_mesh, .{ .batch = .batch, .seq = .replicated, .features = .feature }); + const mapped = shape.mapPartitioningAxes(&test_mesh, .{ .batch = .feature, .feature = .batch }); + try testing.expectEqual(Sharding.PartitionSpec.sharded(1), mapped.partition(.batch)); + try testing.expectEqual(Sharding.PartitionSpec.sharded(0), mapped.partition(.features)); + try testing.expectEqual(Sharding.PartitionSpec.replicated, mapped.partition(.seq)); + const partial = shape.mapPartitioningAxes(&test_mesh, .{ .batch = .colors }); + try testing.expectEqual(Sharding.PartitionSpec.sharded(2), partial.partition(.batch)); + try testing.expectEqual(shape.partition(.features), partial.partition(.features)); } pub fn withDefaultPartitioning(self: Shape) Shape { var res = self; - res._partitioning.clear(); - res._partitioning.appendNTimes(.unknown, self._dims.len) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); + res._sharding.partition = .replicated(self.rank()); return res; } test withDefaultPartitioning { var shape = Shape.init(.{ 10, 20, 30 }, .f32); - try testing.expectEqual(3, shape._partitioning.len); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + try testing.expectEqual(3, shape.rank()); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); - shape = shape.withPartitioning(.{ ._1 = .batch }); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .init(.batch), .unknown }, shape._partitioning.constSlice()); + shape = shape.withPartitioning(&test_mesh, .{ ._1 = .batch }); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .sharded(0), .replicated }, shape._sharding.toArray()[0..shape.rank()]); shape = shape.withDefaultPartitioning(); - try testing.expectEqual(3, shape._partitioning.len); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + try testing.expectEqual(3, shape.rank()); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); } pub fn withReplicatedPartitioning(self: Shape) Shape { var res = self; - res._partitioning.clear(); - res._partitioning.appendNTimes(.replicated, self._dims.len) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); + res._sharding.partition = .replicated(self._dims.len); return res; } test withReplicatedPartitioning { var shape = Shape.init(.{ 10, 20, 30 }, .f32); - try testing.expectEqual(3, shape._partitioning.len); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + try testing.expectEqual(3, shape.rank()); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); - shape = shape.withPartitioning(.{ ._1 = .batch }); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .init(.batch), .unknown }, shape._partitioning.constSlice()); + shape = shape.withPartitioning(&test_mesh, .{ ._1 = .batch }); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .sharded(0), .replicated }, shape._sharding.toArray()[0..shape.rank()]); shape = shape.withReplicatedPartitioning(); - try testing.expectEqual(3, shape._partitioning.len); - try testing.expectEqualSlices(PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._partitioning.constSlice()); + try testing.expectEqual(3, shape.rank()); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); } pub fn hasAtLeastOnePartitionedAxis(self: Shape) bool { - for (self._partitioning.constSlice()) |spec| { - if (spec == .axis) return true; + for (0..self.rank()) |ax| { + const spec = self._sharding.get(ax); + if (spec.isSharded()) return true; } return false; } test hasAtLeastOnePartitionedAxis { - var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .x, .b = .replicated }); + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .b = .replicated }); try testing.expect(shape.hasAtLeastOnePartitionedAxis()); - shape = shape.withPartitioning(.{ .a = .replicated, .b = .replicated }); + shape = shape.withPartitioning(&test_mesh, .{ .a = .replicated, .b = .replicated }); try testing.expect(!shape.hasAtLeastOnePartitionedAxis()); shape = shape.withDefaultPartitioning(); @@ -1063,20 +946,18 @@ pub const Shape = struct { } pub fn isFullyPartitioned(self: Shape) bool { - for (self._partitioning.constSlice()) |spec| { - switch (spec) { - .replicated, .open, .unknown => return false, - .axis => {}, - } + for (0..self.rank()) |ax| { + const spec = self._sharding.get(ax); + if (!spec.isSharded()) return false; } return true; } test isFullyPartitioned { - var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .x, .b = .y }); + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .b = .feature }); try testing.expect(shape.isFullyPartitioned()); - shape = shape.withPartitioning(.{ .a = .x, .b = .replicated }); + shape = shape.withPartitioning(&test_mesh, .{ .a = .batch, .b = .replicated }); try testing.expect(!shape.isFullyPartitioned()); shape = shape.withDefaultPartitioning(); @@ -1084,54 +965,47 @@ pub const Shape = struct { } pub fn isFullyReplicated(self: Shape) bool { - for (self._partitioning.constSlice()) |spec| { - if (spec != .replicated and spec != .unknown) { - return false; - } - } - return true; + return self._sharding.isFullyReplicated(); } test isFullyReplicated { - var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .replicated, .b = .replicated }); + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(&test_mesh, .{ .a = .replicated, .b = .replicated }); try testing.expect(shape.isFullyReplicated()); - shape = shape.withPartitioning(.{ .a = .replicated, .b = .unknown }); - try testing.expect(shape.isFullyReplicated()); - - shape = shape.withPartitioning(.{ .a = .x, .b = .replicated }); + shape = shape.withPartitioning(&test_mesh, .{ .a = .batch, .b = .replicated }); try testing.expect(!shape.isFullyReplicated()); } - pub fn partition(self: Shape, ax: anytype) PartitionSpec { - return self._partitioning.get(self.axis(ax)); + pub fn partition(self: Shape, ax: anytype) Sharding.PartitionSpec { + return self._sharding.get(self.axis(ax)); } test partition { - const shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); - try testing.expectEqual(PartitionSpec.init(.batch), shape.partition(.a)); + const shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .b = .open, .c = .feature }); + try testing.expectEqual(Sharding.PartitionSpec.sharded(0), shape.partition(.a)); try testing.expectEqual(.open, shape.partition(.b)); - try testing.expectEqual(PartitionSpec.init(.feature), shape.partition(.c)); + try testing.expectEqual(Sharding.PartitionSpec.sharded(1), shape.partition(.c)); } - pub fn containsPartitionSpec(self: Shape, part: PartitionSpec) bool { - for (self._partitioning.constSlice()[0..self.rank()]) |dim_part| { - if (dim_part.eql(part)) return true; + pub fn containsPartitionSpec(self: Shape, part: Sharding.PartitionSpec) bool { + for (0..self.rank()) |ax| { + const dim_part = self._sharding.get(ax); + if (dim_part == part) return true; } return false; } test containsPartitionSpec { - var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30, .d = 40 }, .f32).withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); - try testing.expect(shape.containsPartitionSpec(.init(.batch))); + var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30, .d = 40 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .b = .open, .c = .feature }); + try testing.expect(shape.containsPartitionSpec(.sharded(0))); try testing.expect(shape.containsPartitionSpec(.open)); - try testing.expect(!shape.containsPartitionSpec(.replicated)); - try testing.expect(shape.containsPartitionSpec(.unknown)); + try testing.expect(shape.containsPartitionSpec(.replicated)); + try testing.expect(!shape.containsPartitionSpec(.out_of_bound)); shape = shape.withDefaultPartitioning(); - try testing.expect(!shape.containsPartitionSpec(.init(.batch))); - try testing.expect(shape.containsPartitionSpec(.unknown)); - try testing.expect(!shape.containsPartitionSpec(.replicated)); + try testing.expect(!shape.containsPartitionSpec(.sharded(0))); + try testing.expect(!shape.containsPartitionSpec(.out_of_bound)); + try testing.expect(shape.containsPartitionSpec(.replicated)); try testing.expect(!shape.containsPartitionSpec(.open)); } @@ -1329,10 +1203,10 @@ pub const Shape = struct { var new_shape = self; new_shape._dims.replaceRange(ax, 1, dims_.constSlice()) catch unreachable; new_shape._tags.replaceRange(ax, 1, tags_.constSlice()) catch unreachable; - _ = new_shape._partitioning.orderedRemove(ax); + new_shape._sharding = new_shape._sharding.orderedRemove(ax); for (0..dims_.len) |_| { - new_shape._partitioning.insert(ax, .unknown) catch unreachable; + new_shape._sharding = new_shape._sharding.insert(ax, .replicated); } new_shape.inferMissingAxis(self.count()) catch |err| { @@ -1383,39 +1257,34 @@ pub const Shape = struct { const axes_to_merge = self.axes(axes_); stdx.debug.assert(axes_to_merge.len > 1, "Must merge at least two axes, got {any}", .{axes_}); - var new_dim: i64 = 1; // Merging partitioned axes is complex. If ANY axis being merged is partitioned, - // the resulting sharding is ambiguous. A propagation pass must resolve this. + // the resulting mesh is ambiguous. A propagation pass must resolve this. // Marking it 'open' is the correct signal. If all are replicated/unknown, the // result is also replicated/unknown. - // rule: .replicated wins over .unknown. .open wins over everything. - var resulting_partition_spec: PartitionSpec = .unknown; - - for (0..axes_to_merge.len) |i| { - const current_axis = axes_to_merge.get(i); - new_dim *= self.dim(current_axis); - - if (i > 0) { - stdx.debug.assert(current_axis == axes_to_merge.get(i - 1) + 1, "Can't merge shape {f} along non-contiguous axes {any}", .{ self, axes_ }); - } - - switch (self.partition(current_axis)) { - .axis, .open => resulting_partition_spec = .open, - .replicated => if (resulting_partition_spec != .open) { - resulting_partition_spec = .replicated; - }, - .unknown => {}, - } - } + // rule: .replicated wins over .out_of_bound. .open wins over everything. const first_axis_to_merge = axes_to_merge.get(0); const num_axes_to_merge = axes_to_merge.len; - var new_shape = self; + var merged_dim: i64 = 1; + var merged_spec: Sharding.PartitionSpec = .out_of_bound; + for (axes_to_merge.constSlice()) |ax| { + merged_dim *= self.dim(ax); + const spec = self.partition(ax); + if (spec.isSharded() or spec == .open) { + merged_spec = .open; + } else if (spec == .replicated and merged_spec != .open) { + merged_spec = .replicated; + } + } - new_shape._dims.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{new_dim}) catch @panic("mergeAxis failed on dims"); + var new_shape = self; + for (1..num_axes_to_merge) |_| { + new_shape._sharding = new_shape._sharding.orderedRemove(first_axis_to_merge); + } + new_shape._sharding = new_shape._sharding.set(first_axis_to_merge, merged_spec); + new_shape._dims.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{merged_dim}) catch @panic("mergeAxis failed on dims"); new_shape._tags.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{toTag(axis_)}) catch @panic("mergeAxis failed on tags"); - new_shape._partitioning.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{resulting_partition_spec}) catch @panic("mergeAxis failed on partitioning"); return new_shape; } @@ -1451,23 +1320,23 @@ pub const Shape = struct { test "mergeAxis partitioning" { // Merging a partitioned axis with a replicated one -> open const shape1 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32) - .withPartitioning(.{ .a = .replicated, .b = .x, .c = .replicated }); + .withPartitioning(&test_mesh, .{ .a = .replicated, .b = .batch, .c = .replicated }); const merged1 = shape1.mergeAxis(.ab, .{ .a, .b }); - try testing.expect(merged1.partition(.ab).eql(.open)); + try testing.expect(merged1.partition(.ab) == .open); try testing.expect(merged1.dim(.ab) == 10); try testing.expect(merged1.rank() == 2); // Merging two replicated axes -> replicated const shape2 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32) - .withPartitioning(.{ .a = .replicated, .b = .replicated, .c = .y }); + .withPartitioning(&test_mesh, .{ .a = .replicated, .b = .replicated, .c = .feature }); const merged2 = shape2.mergeAxis(.ab, .{ .a, .b }); - try testing.expect(merged2.partition(.ab).eql(.replicated)); - try testing.expect(merged2.partition(.c).eql(.init(.y))); + try testing.expect(merged2.partition(.ab) == .replicated); + try testing.expect(merged2.partition(.c) == Sharding.PartitionSpec.sharded(1)); // Merging two unknown axes -> unknown - const shape3 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32); // default is .unknown + const shape3 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32).withDefaultPartitioning(); const merged3 = shape3.mergeAxis(.ab, .{ .a, .b }); - try testing.expect(merged3.partition(.ab).eql(.unknown)); + try testing.expect(merged3.partition(.ab) == .replicated); } pub fn mergeAxes(self: Shape, axes_: anytype) Shape { @@ -1641,6 +1510,37 @@ pub const Shape = struct { const x: Shape = .init(.{ 4, 8 }, .u2); try expectEqualShapes(.init(.{ 4, 2 }, .u8), x.packedShape()); } + + test "shape transformations preserve partition specs and clear unused slots" { + var shape = Shape.init(.{ 2, 3, 4 }, .f32); + shape._sharding = .{ .mesh = &test_mesh, .partition = Sharding.Partitioning.out_of_bound.set(0, .mesh_axis_0).set(1, .open).set(2, .mesh_axis_1) }; + for ([_]Shape{ + shape.insert(1, .{5}), + shape.append(.{5}), + shape.remove(1), + shape.removeMany(.{ 0, 1 }), + shape.transpose(.{ 2, 0, 1 }), + shape.reshape(.{24}), + shape.splitAxis(1, .{ 1, 3 }), + shape.mergeAxis(.merged, .{ 0, 1 }), + shape.withDefaultPartitioning(), + }) |transformed| { + try testing.expectEqual(&test_mesh, transformed._sharding.mesh); + } + const parts = shape._sharding.partition; + try testing.expectEqual(parts.insert(1, .replicated), shape.insert(1, .{5})._sharding.partition); + try testing.expectEqual(parts.insert(1, .replicated), shape.insertTag(1, 5, .new)._sharding.partition); + try testing.expectEqual(parts.set(3, .replicated), shape.append(.{5})._sharding.partition); + try testing.expectEqual(parts.set(3, .replicated), shape.appendDim(5, null)._sharding.partition); + try testing.expectEqual(parts.orderedRemove(1), shape.remove(1)._sharding.partition); + const removed = shape.removeMany(.{ 0, 1 }); + try testing.expectEqual(Sharding.Partitioning.out_of_bound.set(0, .mesh_axis_1), removed._sharding.partition); + try testing.expectEqual(Sharding.PartitionSpec.replicated, removed.append(.{5}).partition(1)); + try testing.expectEqual(Sharding.Partitioning.out_of_bound.set(0, .mesh_axis_1).set(1, .mesh_axis_0).set(2, .open), shape.transpose(.{ 2, 0, 1 })._sharding.partition); + try testing.expectEqual(Sharding.Partitioning.out_of_bound.set(0, .mesh_axis_0).set(1, .replicated).set(2, .replicated).set(3, .mesh_axis_1), shape.splitAxis(1, .{ 1, 3 })._sharding.partition); + try testing.expectEqual(Sharding.Partitioning.replicated(3), shape.withDefaultPartitioning()._sharding.partition); + try testing.expectEqual(Sharding.Partitioning.replicated(3), shape.withReplicatedPartitioning()._sharding.partition); + } }; /// Iterates over all coordinates of a shape, yielding the multi-dimensional diff --git a/zml/tensor.zig b/zml/tensor.zig index 83d4beba..e630af1f 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -77,8 +77,11 @@ pub const Tensor = struct { /// Internal use /// /// Creates a tensor from a Shape and an mlir.Value. + /// Note: sharding information is discarded, the output tensor is considered open for replication. + /// Use _resultPropagateSharding when the shape sharding should be propagated. pub fn _result(sh: Shape, val: *const mlir.Value) Tensor { - const res: Tensor = .{ ._shape = sh, ._value = val, .id = nextTensorId() }; + var res: Tensor = .{ ._shape = sh, ._value = val, .id = nextTensorId() }; + res._shape._sharding.partition = .open(sh.rank()); if (builtin.mode == .Debug) { // Check that the MLIR value actually have the same shape. @@ -91,9 +94,16 @@ pub const Tensor = struct { return res; } + pub fn _resultPropagateSharding(sh: Shape, val: *const mlir.Value) Tensor { + var res = _result(sh, val); + res._shape._sharding = sh._sharding; + return res; + } + /// Creates a Tensor from a mlir.Value /// /// The shape is derived from the type of the mlir.Value. + /// mesh is `unknown` pub fn fromMlirValue(val: *const mlir.Value) Tensor { const ctx = Compiler.current(); const ranked_tensor = val.type_().isA(mlir.RankedTensorType).?; @@ -101,13 +111,15 @@ pub const Tensor = struct { stdx.debug.assert(n <= constants.MAX_RANK, "Can't represent MLIR tensor of rank {}, max supported rank is {}.", .{ n, constants.MAX_RANK }); - var sh: Shape = .{ ._dtype = ctx.dtype(ranked_tensor.elementType()) }; + var sh: Shape = .{ + ._dtype = ctx.dtype(ranked_tensor.elementType()), + ._dims = .empty, + ._tags = .{ .buffer = @splat(Shape.TagUnknown), .len = n }, + ._sharding = .replicated, + }; for (0..n) |i| { sh._dims.appendAssumeCapacity(ranked_tensor.dimension(i)); } - sh._tags.appendNTimes(Shape.TagUnknown, n) catch unreachable; - sh._partitioning.appendNTimes(.unknown, n) catch unreachable; - return .{ ._shape = sh, ._value = val, .id = nextTensorId() }; } @@ -153,8 +165,24 @@ pub const Tensor = struct { return res; } - pub fn withPartitioning(self: Tensor, axes_: anytype) Tensor { - const partitioned_shape = self._shape.withPartitioning(axes_); + /// Specify the mesh of the input tensor. + /// * mesh: *const zml.Mesh, but during compilation the `.meshe_name` syntax can used to get + /// a known mesh from the compilation options. + /// * partition spec: a struct where the field names match the axis of the given mesh + /// + /// eg `x.withPartitioning(tp, .{ .h = .model }))` or `x.withPartitioning(.tp, .{ .h = .model }))` + pub fn withPartitioning(self: Tensor, mesh_: anytype, partitioning: anytype) Tensor { + if (@TypeOf(mesh_) == @EnumLiteral()) { + const compiler = Compiler.currentOrNull() orelse @panic("Out side of compilation, withPartitioning expects an explicit *const zml.Mesh object as input"); + return self.withPartitioning(compiler.mesh(mesh_), partitioning); + } + + if (@TypeOf(partitioning) != Sharding.Partitioning) { + const parsed = Sharding.Partitioning.parse(self._shape.tags(), mesh_, .open, partitioning); + return self.withPartitioning(mesh_, parsed); + } + + const partitioned_shape = self._shape.withPartitioning(mesh_, partitioning); const ctx = Compiler.currentOrNull() orelse { var res = self; @@ -162,15 +190,9 @@ pub const Tensor = struct { return res; }; - const sharding = ctx.partitioning.selectSharding(partitioned_shape) catch |err| switch (err) { - error.NoSuitableSharding => std.debug.panic( - "{f}.withPartitioning({f}) failed to resolve because it's using unknown sharding. Pass more shardings to `zml.compile`. Known shardings: {f}", - .{ self, partitioned_shape, stdx.fmt.slice(ctx.partitioning.shardings) }, - ), - }; - const attr = ctx.partitioning.tensorShardingAttr(ctx.allocator, ctx.mlir_ctx, partitioned_shape, sharding) catch @panic("OOM"); + const attr = ctx.tensorShardingAttr(ctx.allocator, ctx.mlir_ctx, partitioned_shape) catch @panic("OOM"); - const op_result = switch (ctx.partitioning.partitioner) { + const op_result = switch (ctx.partitioner) { .shardy => blk: { const op = mlir.Operation.make(ctx.mlir_ctx, "sdy.sharding_constraint", .{ .operands = .{ .flat = &.{self.value()} }, @@ -201,7 +223,52 @@ pub const Tensor = struct { }, }; - return _result(partitioned_shape, op_result); + std.log.warn("{f}.withPartitioning({f}) -> {f}", .{ self, partitioned_shape, attr }); + return _resultPropagateSharding(partitioned_shape, op_result); + } + + /// Force the input tensor to be fully replicated. + /// This can be detrimental for performance. + pub fn replicate(self: Tensor) Tensor { + const compiler = Compiler.current(); + const partitioning: Sharding.Partitioning = .repeat(.replicated, self.rank()); + return self.withPartitioning(compiler.platform.replicated_mesh, partitioning); + } + + test withPartitioning { + // Create two similar meshes, + // Then check that tensor.withPartitioning don't confuse the meshes + const dp_mp: Sharding.Mesh = .{ + .name = "dp_mp", + .physical = undefined, + .logical = .mesh(.{ .data = .low_bandwidth, .model = .high_bandwidth }), + .bindings = .init(&.{.init(&.{.link_x})}), + .folds = .empty, + .folds_consumed = .empty, + }; + + const mp_dp: Sharding.Mesh = .{ + .name = "mp_dp", + .physical = undefined, + .logical = .mesh(.{ .model = .low_bandwidth, .data = .high_bandwidth }), + .bindings = .init(&.{.init(&.{.link_x})}), + .folds = .empty, + .folds_consumed = .empty, + }; + + const x = Tensor.init(.{ .h = 8 }, .f32).withPartitioning(&dp_mp, .{ .h = .model }); + try std.testing.expectEqual(&dp_mp, x.shape()._sharding.mesh); + try std.testing.expectEqual(Sharding.PartitionSpec.sharded(1), x.shape().partition(.h)); + try std.testing.expectEqual(&dp_mp, x.shape().reshape(.{ 2, 4 })._sharding.mesh); + + const x2 = x.withPartitioning(&mp_dp, .{ .h = .model }); + try std.testing.expectEqual(&mp_dp, x2.shape()._sharding.mesh); + try std.testing.expectEqual(Sharding.PartitionSpec.sharded(0), x2.shape().partition(.h)); + try std.testing.expectEqual(&mp_dp, x2.shape().reshape(.{ 2, 4 })._sharding.mesh); + + const x2_replicated = x.withPartitioning(&mp_dp, .{ .h = .replicated }); + try std.testing.expectEqual(&mp_dp, x2_replicated.shape()._sharding.mesh); + try std.testing.expectEqual(Sharding.PartitionSpec.replicated, x2_replicated.shape().partition(.h)); } /// Copy the given tensor to the specified memory. @@ -257,7 +324,7 @@ pub const Tensor = struct { const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.memcpyH2D, .{x_t}, platform, .{}); defer exe.deinit(); - var x_h = try zml.Buffer.fromBytesOpts(io, platform, x_t.shape(), .replicated, @ptrCast(&inputs), .{ .memory = .host_pinned }); + var x_h = try zml.Buffer.fromBytesOpts(io, platform, x_t.shape(), @ptrCast(&inputs), .{ .memory = .host_pinned }); defer x_h.deinit(); const x_h_ptr: [*]f32 = @ptrCast(@alignCast(x_h.opaqueDevicePtr(0))); @@ -359,6 +426,8 @@ pub const Tensor = struct { /// cannot use the donated input's contents after execution. Runners destroy its /// handle automatically; raw `Exe.call` leaves it for the caller to deinit. /// For `reuseBuffer` to be effective, it needs to propagate all the way through the output. + /// + /// The sharding of the origin tensor is propagated to this one. pub fn reuseBuffer(self: Tensor, origin: Tensor) Tensor { const compilation_context = Compiler.current(); const scope = compilation_context.currentScope(); @@ -369,8 +438,30 @@ pub const Tensor = struct { .explicit => og_donation, }; scope.id_to_donation.put(scope.arena.allocator(), self.id, donation) catch @panic("OOM"); + + // The origin is mapped to a physical buffer either because it's an input or through `reuseBuffer` + // We can now look at its sharding. + if (origin._shape.isFullyReplicated()) { + stdx.debug.assert(self.byteSize() == origin.byteSize(), ".reuseBuffer expects two tensors with same byteSize, got {f} ({B}) and {f} ({B})", .{ self, self.byteSize(), origin, origin.byteSize() }); + } else { + // Origin is sharded, we can't just use the global byte size, we actually need the same byte size per shard. + // We are stricter here and require the same shape. + stdx.debug.assert(self._shape.eql(origin._shape), ".reuseBuffer expects two sharded tensors with same shape, got {f} and {f}", .{ self, origin }); + + // Normally self sharding is unspecified and we can just propagate origin sharding, + // but if the user called `.withPartitioning` explicitly it would be weird to override it silently, hence the check. + stdx.debug.assert(self._shape._sharding.isCompatible(origin._shape._sharding), ".reuseBuffer expects compatible shardings, got {f} and {f}", .{ self, origin }); + } + + var res = self; + res._shape._sharding = origin._shape._sharding; + return res; + } else { + // Origin tensor doesn't has any physical buffer associated so we can move on. + // This is allowed to allow a function like KVCache.update to call `reuseBuffer`, + // even though the caller may have constructed the inputs internally. + return self; } - return self; } test reuseBuffer { @@ -393,13 +484,13 @@ pub const Tensor = struct { const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.memcopy, .{ x_t, y_t, z_t }, platform, .{}); defer exe.deinit(); - var x_d = try zml.Buffer.fromBytes(io, platform, x_t.shape(), .replicated, @ptrCast(&inputs)); + var x_d = try zml.Buffer.fromBytes(io, platform, x_t.shape(), @ptrCast(&inputs)); defer x_d.deinit(); - var y_d = try zml.Buffer.uninitialized(io, platform, y_t.shape(), .replicated, .{}); + var y_d = try zml.Buffer.uninitialized(io, platform, y_t.shape(), .{}); defer y_d.deinit(); - var z_d = try zml.Buffer.uninitialized(io, platform, z_t.shape(), .replicated, .{}); + var z_d = try zml.Buffer.uninitialized(io, platform, z_t.shape(), .{}); defer z_d.deinit(); var y_memory: [Platform.MAX_NUM_DEVICES]*anyopaque = undefined; @@ -532,7 +623,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var input_buffer = try zml.Buffer.fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&i)); + var input_buffer = try zml.Buffer.fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&i)); defer input_buffer.deinit(); var output = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); @@ -729,9 +820,9 @@ pub const Tensor = struct { return .{ ._state = .init(.{2}, .u64) }; } - pub fn initBuffer(io: std.Io, platform: *const Platform, sharding: Sharding, seed: u128) !Buffer { + pub fn initBuffer(rng: Rng, io: std.Io, platform: *const Platform, seed: u128) !Buffer { return .{ - ._state = try .fromBytes(io, platform, Shape.init(.{2}, .u64), sharding, std.mem.asBytes(&seed)), + ._state = try .fromBytes(io, platform, rng._state._shape, std.mem.asBytes(&seed)), }; } @@ -753,12 +844,12 @@ pub const Tensor = struct { mlirx.Type.rankedTensor(mlirCtx(), sh), currentLoc(), ).appendTo(currentBlock()); - return .{ self.update(op.result(0)), _result(sh, op.result(1)) }; + return .{ self.update(op.result(0)), _resultPropagateSharding(sh, op.result(1)) }; } fn update(self: Rng, new_state: *const mlir.Value) Rng { return .{ - ._state = _result(self._state._shape, new_state).reuseBuffer(self._state), + ._state = _resultPropagateSharding(self._state._shape, new_state).reuseBuffer(self._state), .algorithm = self.algorithm, }; } @@ -799,7 +890,7 @@ pub const Tensor = struct { floats = floats.mul(scalar(opts.max - opts.min, dt)).addConstant(opts.min); // Convert back to integer if needed. - return .{ rng, floats.convert(shape_.dtype()) }; + return .{ rng, _resultPropagateSharding(shape_, floats.convert(shape_.dtype()).value()) }; } test uniform { @@ -834,7 +925,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.uniformStats, .{ Rng.init(), Shape.init(.{1024}, .f32), .{ .min = -2, .max = 10 } }, platform, .{}); defer exe.deinit(); - var rng_buffer = try Rng.initBuffer(std.testing.io, platform, .replicated, 1234); + var rng_buffer = try Rng.init().initBuffer(std.testing.io, platform, 1234); defer rng_buffer._state.deinit(); var stats: zml.Bufferized(Stats) = undefined; @@ -878,7 +969,7 @@ pub const Tensor = struct { const b = Tensor.constant(DataType.Value.init(sh.dtype(), opts.stddev)); const res_tensor_shape = Tensor.constantTensor(Shape.init(.{sh.rank()}, .i64), std.mem.sliceAsBytes(sh.dims())); const op = dialects.stablehlo.rng(mlirCtx(), a.value(), b.value(), res_tensor_shape.value(), .NORMAL, currentLoc()).appendTo(currentBlock()); - return _result(sh, op.result(0)); + return _resultPropagateSharding(sh, op.result(0)); } /// Returns a Tensor of the given shape, filled with floating point numbers sampled from a Gumbel distribution, and a new Rng state. @@ -896,7 +987,8 @@ pub const Tensor = struct { // We don't want 0 to be sampled otherwise `log` will return -inf. .{ .min = std.math.floatEps(f32), .max = 1 }, ); - return .{ rand, u.log().scale(-1).log().scale(-1).convert(shape_.dtype()) }; + const result = u.log().scale(-1).log().scale(-1).convert(shape_.dtype()); + return .{ rand, _resultPropagateSharding(shape_, result.value()) }; } test gumbel { @@ -944,9 +1036,9 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.gumbelStats, .{ Rng.init(), tgt_dist }, platform, .{}); defer exe.deinit(); - var rng_buffer = try Rng.initBuffer(std.testing.io, platform, .replicated, 1234); + var rng_buffer = try Rng.init().initBuffer(std.testing.io, platform, 1234); defer rng_buffer._state.deinit(); - var tgt_dist_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, tgt_dist.shape(), .replicated, @ptrCast(&tgt_dist_data)); + var tgt_dist_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, tgt_dist.shape(), @ptrCast(&tgt_dist_data)); defer tgt_dist_buffer.deinit(); var stats: zml.Bufferized(Stats) = undefined; @@ -1294,7 +1386,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), .replicated, @ptrCast(&x_f32)); + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), @ptrCast(&x_f32)); defer x_d_buffer.deinit(); var x_f4_xla_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_d_buffer}); @@ -1326,7 +1418,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), .replicated, std.mem.sliceAsBytes(&x)); + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), std.mem.sliceAsBytes(&x)); defer x_d_buffer.deinit(); var x_f8e3_xla_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_d_buffer}); @@ -1356,7 +1448,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_u2_t.shape(), .replicated, @ptrCast(&x_u2)); + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_u2_t.shape(), @ptrCast(&x_u2)); defer x_d_buffer.deinit(); var x_u8_xla_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_d_buffer}); @@ -1384,7 +1476,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var x_f4_d: zml.Buffer = try .fromBytes(std.testing.io, platform, x_t.shape(), .replicated, @ptrCast(&x_f4_packed)); + var x_f4_d: zml.Buffer = try .fromBytes(std.testing.io, platform, x_t.shape(), @ptrCast(&x_f4_packed)); defer x_f4_d.deinit(); var x_f32_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_f4_d}); @@ -1526,7 +1618,7 @@ pub const Tensor = struct { const Axes = stdx.BoundedArray(i64, constants.MAX_RANK); - var res_shape: Shape = .{ ._dtype = lhs.dtype() }; + var res_shape: Shape = .scalar(lhs.dtype()); // Validate batching axes var lhs_batching_axes: Axes = .empty; var rhs_batching_axes: Axes = .empty; @@ -1626,7 +1718,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&[_]f32{ -0.6884, 1.6795 })); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6884, 1.6795 })); defer input_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); @@ -1670,27 +1762,27 @@ pub const Tensor = struct { ); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&[4]f32{ 1, 0, 1, 2 })); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[4]f32{ 1, 0, 1, 2 })); defer input_buffer.deinit(); - var w_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, w.shape(), .replicated, std.mem.sliceAsBytes(&[3][4]f32{ + var w_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, w.shape(), std.mem.sliceAsBytes(&[3][4]f32{ .{ 1, 0, 2, 0 }, .{ 2, 1, 3, 3 }, .{ 1, 1, 0, 0 }, })); defer w_buffer.deinit(); - var v_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, v.shape(), .replicated, std.mem.sliceAsBytes(&[3][4]f32{ + var v_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, v.shape(), std.mem.sliceAsBytes(&[3][4]f32{ .{ 0, 0, 2, 1 }, .{ 1, 2, 1, 3 }, .{ 0, 0, 2, 0 }, })); defer v_buffer.deinit(); - var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), .replicated, std.mem.sliceAsBytes(&[3]f32{ 1, 1, 1 })); + var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), std.mem.sliceAsBytes(&[3]f32{ 1, 1, 1 })); defer b_buffer.deinit(); - var c_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, c.shape(), .replicated, std.mem.sliceAsBytes(&[3]f32{ 2, 2, 2 })); + var c_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, c.shape(), std.mem.sliceAsBytes(&[3]f32{ 2, 2, 2 })); defer c_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{ input_buffer, w_buffer, v_buffer, b_buffer, c_buffer }); @@ -1948,7 +2040,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, @ptrCast(&[2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), @ptrCast(&[2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } })); defer x_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); @@ -2236,7 +2328,7 @@ pub const Tensor = struct { const x: Tensor = .init(.{ 2, 5 }, .f32); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); defer x_buffer.deinit(); // Wrap slice to hide the anytype in the signature. @@ -2291,9 +2383,9 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.dynSliceResolved, .{ x, ax, .dyn(z, 2) }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&x_data)); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&x_data)); defer x_buffer.deinit(); - var z_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, z.shape(), .replicated, std.mem.asBytes(&z_value)); + var z_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, z.shape(), std.mem.asBytes(&z_value)); defer z_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ .dyn_start = z_buffer } }); @@ -2417,7 +2509,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&input_data)); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&input_data)); defer input_buffer.deinit(); var output = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); @@ -2481,7 +2573,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), .replicated, std.mem.sliceAsBytes(&input_data)); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&input_data)); defer input_buffer.deinit(); var output = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); @@ -3173,7 +3265,7 @@ pub const Tensor = struct { ); defer exe.deinit(); - var indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, indices.shape(), .replicated, std.mem.sliceAsBytes(&[2][2]i32{ .{ 2, 1 }, .{ 0, 3 } })); + var indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, indices.shape(), std.mem.sliceAsBytes(&[2][2]i32{ .{ 2, 1 }, .{ 0, 3 } })); defer indices_buffer.deinit(); var operand_buffer: zml.Buffer = b: { const temp_func = struct { @@ -3391,11 +3483,11 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform, .{}); defer exe.deinit(); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); defer a_buffer.deinit(); - var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), .replicated, @ptrCast(&[2]i32{ 0, 2 })); + var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), @ptrCast(&[2]i32{ 0, 2 })); defer scatter_indices_buffer.deinit(); - var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), .replicated, std.mem.sliceAsBytes(&[2][3]i32{ .{ 10, 20, 30 }, .{ 70, 80, 90 } })); + var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), std.mem.sliceAsBytes(&[2][3]i32{ .{ 10, 20, 30 }, .{ 70, 80, 90 } })); defer updates_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); defer result.deinit(); @@ -3414,11 +3506,11 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform, .{}); defer exe.deinit(); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), .replicated, std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); defer a_buffer.deinit(); - var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), .replicated, std.mem.sliceAsBytes(&[2]i32{ 2, 7 })); + var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), std.mem.sliceAsBytes(&[2]i32{ 2, 7 })); defer scatter_indices_buffer.deinit(); - var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), .replicated, std.mem.sliceAsBytes(&[2]i32{ 20, 70 })); + var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), std.mem.sliceAsBytes(&[2]i32{ 20, 70 })); defer updates_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); defer result.deinit(); @@ -3436,9 +3528,9 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatterCB, .{ operand, start_indices, values }, platform, .{}); defer exe.deinit(); - var operand_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, operand.shape(), .replicated, std.mem.sliceAsBytes(&@as([2 * 3 * 4 * 2]u16, @splat(0)))); + var operand_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, operand.shape(), std.mem.sliceAsBytes(&@as([2 * 3 * 4 * 2]u16, @splat(0)))); defer operand_buffer.deinit(); - var start_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, start_indices.shape(), .replicated, std.mem.sliceAsBytes(&[2][2][3][2]i32{ + var start_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, start_indices.shape(), std.mem.sliceAsBytes(&[2][2][3][2]i32{ .{ .{ .{ 0, 0 }, .{ 1, 0 }, .{ 2, 1 } }, .{ .{ 0, 1 }, .{ 1, 1 }, .{ 0, 9 } }, @@ -3449,7 +3541,7 @@ pub const Tensor = struct { }, })); defer start_indices_buffer.deinit(); - var values_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, values.shape(), .replicated, std.mem.sliceAsBytes(&@as([2 * 2 * 3 * 2 * 2]u16, @splat(1)))); + var values_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, values.shape(), std.mem.sliceAsBytes(&@as([2 * 2 * 3 * 2 * 2]u16, @splat(1)))); defer values_buffer.deinit(); var result = try exe.eval(std.testing.allocator, std.testing.io, .{ operand_buffer, start_indices_buffer, values_buffer }); defer result.deinit(); @@ -3554,7 +3646,7 @@ pub const Tensor = struct { defer exe.deinit(); { - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, 4.1, 7.9, 0, 7.9 }})); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, 4.1, 7.9, 0, 7.9 }})); defer x_buffer.deinit(); var res: zml.Bufferized(Tensor.ArgMaxRes) = undefined; @@ -3571,7 +3663,7 @@ pub const Tensor = struct { } { - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, std.math.nan(f32), 7.9, 0, 7.9 }})); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, std.math.nan(f32), 7.9, 0, 7.9 }})); defer x_buffer.deinit(); var res: zml.Bufferized(Tensor.ArgMaxRes) = undefined; @@ -3628,7 +3720,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{} }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 })); defer x_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); @@ -3647,7 +3739,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{ .descending = true } }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]f16{ + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f16{ -0.2505, 1.2520, -0.7041, 0.1066, 1.2773, -1.7246, 0.8389, 1.1094, 0.0601, 1.0684, 0.9619, 1.3916, 1.2246, -0.1406, 0.3674, -1.2480, -1.7051, -0.0934, 0.3435, 0.4373, 1.3809, 0.5444, -0.6079, 1.2031, -0.6880, 1.2979, -0.1869, 0.2991, 0.0156, 0.1847, @@ -3678,7 +3770,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 3, .{} }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 89, 31, 22, 42, 64, 39, 0, 30, 64, 71, 46, 31, @@ -4015,7 +4107,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._split, .{ x, ax, &args }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&x_data)); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&x_data)); defer x_buffer.deinit(); const res = try std.testing.allocator.alloc(zml.Buffer, expectation.len); @@ -4143,11 +4235,11 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = ids }, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); defer y_buffer.deinit(); - var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{4})); + var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{4})); defer ids_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ .a = ids_buffer }, y_buffer }); @@ -4169,11 +4261,11 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); defer y_buffer.deinit(); - var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{3})); + var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer ids_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, ids_buffer, y_buffer }); @@ -4198,11 +4290,11 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[2][1]f32{ .{-1}, .{-1} })); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[2][1]f32{ .{-1}, .{-1} })); defer y_buffer.deinit(); - var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{3})); + var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer ids_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, ids_buffer, y_buffer }); @@ -4228,13 +4320,13 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = idx_a, .b = idx_b }, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[1]f32{-1})); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[1]f32{-1})); defer y_buffer.deinit(); - var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{1})); + var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), std.mem.sliceAsBytes(&[1]i32{1})); defer idx_a_buffer.deinit(); - var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{3})); + var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer idx_b_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ .a = idx_a_buffer, .b = idx_b_buffer }, y_buffer }); @@ -4260,13 +4352,13 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ idx_a, idx_b }, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[1][1]f32{.{-1}})); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[1][1]f32{.{-1}})); defer y_buffer.deinit(); - var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{1})); + var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), std.mem.sliceAsBytes(&[1]i32{1})); defer idx_a_buffer.deinit(); - var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), .replicated, std.mem.sliceAsBytes(&[1]i32{3})); + var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer idx_b_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ idx_a_buffer, idx_b_buffer }, y_buffer }); @@ -4306,12 +4398,12 @@ pub const Tensor = struct { stdx.debug.assert(self.rank() < constants.MAX_RANK - 1, "toDiagonal expects input up to {d} rank, got {f}", .{ constants.MAX_RANK - 1, self }); const a = self.axis(axis_); const d = self.dim(a); - const p = self.shape()._partitioning.get(a); + const p = self.shape()._sharding.partition.get(a); var res_shape = self._shape; res_shape._dims.replaceRange(a, 1, &.{ d, d }) catch unreachable; res_shape._tags.replaceRange(a, 1, &.{ @tagName(new_tags[0]), @tagName(new_tags[1]) }) catch unreachable; - // TODO(Corentin): Not sure about that - res_shape._partitioning.replaceRange(a, 1, &.{ p, p }) catch unreachable; + + res_shape._sharding.partition = res_shape._sharding.partition.insert(a, p); const values = self.insertAxes(a + 1, .{new_tags[1]}).broad(res_shape); const zeros = Tensor.constant(self.dtype().zero()).broad(res_shape); @@ -4340,7 +4432,7 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._toDiag, .{x}, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[2][2]u8{ .{ 1, 2 }, .{ 3, 4 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][2]u8{ .{ 1, 2 }, .{ 3, 4 } })); defer x_buffer.deinit(); var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); @@ -4388,7 +4480,7 @@ pub const Tensor = struct { const x: Tensor = .init(.{ 3, 3 }, .u8); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[3][3]u8{ + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[3][3]u8{ .{ 1, 1, 1 }, .{ 1, 1, 1 }, .{ 1, 1, 1 }, @@ -4570,9 +4662,9 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cartesianProduct2, .{ x, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); defer y_buffer.deinit(); var xs: zml.Buffer = undefined; @@ -4647,9 +4739,9 @@ pub const Tensor = struct { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, y }, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); defer x_buffer.deinit(); - var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), .replicated, std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); defer y_buffer.deinit(); var z = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, y_buffer }); @@ -4799,7 +4891,7 @@ test "Tensor.maxPool1d" { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&data)); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&data)); defer x_buffer.deinit(); var result: zml.Bufferized(Tensor.ArgMaxRes) = undefined; @@ -4841,7 +4933,7 @@ test "Tensor.maxPool2d" { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform, .{}); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), .replicated, std.mem.sliceAsBytes(&data)); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&data)); defer x_buffer.deinit(); var result: zml.Bufferized(Tensor.ArgMaxRes) = undefined; diff --git a/zml/testing.zig b/zml/testing.zig index e2e84502..366351d7 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -26,8 +26,8 @@ pub fn env() *const Platform { }, ) catch @panic("Pjrt not available"); - // Register shardings required by flash attention. - _ = platform.registerSharding("model", .mesh(.{ .model = .high_bandwidth })) catch unreachable; + // Register meshes required by flash attention. + _ = platform.registerMesh("model", .mesh(.{ .model = .high_bandwidth })) catch unreachable; _platform = platform; } @@ -394,7 +394,7 @@ pub fn testLayer( activation_store: zml.io.TensorStore.View, name: []const u8, layer_weights: zml.Bufferized(@TypeOf(layer)), - shardings: []const *const zml.Sharding, + meshes: []const *const zml.Mesh, opts: CompareOpts, ) !void { var arena = std.heap.ArenaAllocator.init(allocator); @@ -424,12 +424,12 @@ pub fn testLayer( var buffer: [256]u8 = undefined; const subkey = std.fmt.bufPrint(&buffer, "{d}", .{ctx_.index}) catch unreachable; - tensor.* = ctx_.activation_store.createTensor(subkey, null, .replicated); + tensor.* = ctx_.activation_store.createReplicatedTensor(subkey, null); ctx_.index += 1; } }.cb, &ctx, &args); - const exe = try platform.compile(allocator, io, layer, func, args, .{ .shardings = if (shardings.len != 0) shardings else &.{} }); + const exe = try platform.compile(allocator, io, layer, func, args, .{ .meshes = if (meshes.len != 0) meshes else &.{} }); defer exe.deinit(); const output_name = try std.fmt.allocPrint(arena.allocator(), "{s}.out", .{name}); diff --git a/zml/zml.zig b/zml/zml.zig index 995318eb..2eb84a17 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -45,6 +45,7 @@ pub const tracer = @import("profiling/tracer.zig"); pub const safetensors = @import("safetensors.zig"); pub const shape = @import("shape.zig"); pub const Shape = shape.Shape; +pub const Mesh = Sharding.Mesh; pub const Sharding = @import("Sharding.zig"); pub const slice = @import("slice.zig"); pub const Slice = slice.Slice; From 857c59d331afd5f313a1a4de075fc11b0cdabc9f Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 6 Oct 2026 17:29:50 +0200 Subject: [PATCH 756/881] fix: remove debug log (#906) of course --- zml/tensor.zig | 1 - zml/zml.zig | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/zml/tensor.zig b/zml/tensor.zig index e630af1f..96b60c0c 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -223,7 +223,6 @@ pub const Tensor = struct { }, }; - std.log.warn("{f}.withPartitioning({f}) -> {f}", .{ self, partitioned_shape, attr }); return _resultPropagateSharding(partitioned_shape, op_result); } diff --git a/zml/zml.zig b/zml/zml.zig index 2eb84a17..7d457937 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -45,8 +45,8 @@ pub const tracer = @import("profiling/tracer.zig"); pub const safetensors = @import("safetensors.zig"); pub const shape = @import("shape.zig"); pub const Shape = shape.Shape; -pub const Mesh = Sharding.Mesh; pub const Sharding = @import("Sharding.zig"); +pub const Mesh = Sharding.Mesh; pub const slice = @import("slice.zig"); pub const Slice = slice.Slice; pub const tensor = @import("tensor.zig"); From 948f21c26c6fd111abf61a3df80a28534543bdd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20Gauti=C3=A9?= <51361266+tgautie@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:07:42 +0200 Subject: [PATCH 757/881] zml/attention: introduce new AttentionMask entry (#904) Follow-up to https://github.com/zml/zml/pull/888: We introduce a new `.standard` entry to `AttentionMask` enum, that accepts both flags `causal` and `sliding_window`. This makes the interface more flexible, to allow cases with both non-causal _and_ sliding_window options. --- zml/attention/flashattn.zig | 15 +++---- zml/attention/metal_attention.zig | 5 +-- zml/attention/paged_attention.zig | 39 ++++++++++++------- zml/attention/tpu_attention.zig | 9 ++--- zml/attention/triton_attention.zig | 6 +-- .../unified_attention_oneapi.zig | 9 +++-- 6 files changed, 43 insertions(+), 40 deletions(-) diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig index dd010217..1e583384 100644 --- a/zml/attention/flashattn.zig +++ b/zml/attention/flashattn.zig @@ -914,7 +914,7 @@ pub const paged_fa2 = struct { const o = switch (parameters) { .decode => |decode_parameters| b: { - const seqlenq_ngroups_swapped = num_heads > num_kv_heads and @mod(head_dim, 8) == 0 and opts.mask != .sliding_window; + const seqlenq_ngroups_swapped = num_heads > num_kv_heads and @mod(head_dim, 8) == 0 and opts.mask.slidingWindow() == null; const block_table = decode_parameters.block_table.withPartitioning(.model, .{ .b = .replicated }); const cu_seqlens_q = decode_parameters.cu_seqlens_q.withPartitioning(.model, .{ .b = .replicated }); @@ -1002,7 +1002,7 @@ pub const paged_fa2 = struct { break :b o; }, .mixed => |mixed_parameters| b: { - const seqlenq_ngroups_swapped = num_heads > num_kv_heads and @mod(head_dim, 8) == 0 and opts.mask != .sliding_window; + const seqlenq_ngroups_swapped = num_heads > num_kv_heads and @mod(head_dim, 8) == 0 and opts.mask.slidingWindow() == null; const block_table_prefill = mixed_parameters.block_table_prefill.withPartitioning(.model, .{ .b = .replicated }); const cu_seqlens_q_prefill = mixed_parameters.cu_seqlens_q_prefill.withPartitioning(.model, .{ .b = .replicated }); @@ -1781,15 +1781,12 @@ pub const paged_fa3 = struct { }; fn windowSizeLeft(mask: AttentionMask) i32 { - return switch (mask) { - .sliding_window => |w| @as(i32, @intCast(w)) - 1, - else => -1, - }; + return if (mask.slidingWindow()) |window| @as(i32, @intCast(window)) - 1 else -1; } test "FlashAttention sliding window uses an inclusive offset" { - try std.testing.expectEqual(@as(i32, 2047), windowSizeLeft(.{ .sliding_window = 2048 })); - try std.testing.expectEqual(@as(i32, 0), windowSizeLeft(.{ .sliding_window = 1 })); - try std.testing.expectEqual(@as(i32, -1), windowSizeLeft(.causal)); + try std.testing.expectEqual(@as(i32, 2047), windowSizeLeft(.{ .standard = .{ .sliding_window = 2048 } })); + try std.testing.expectEqual(@as(i32, 0), windowSizeLeft(.{ .standard = .{ .sliding_window = 1 } })); + try std.testing.expectEqual(@as(i32, -1), windowSizeLeft(.{ .standard = .{} })); try std.testing.expectEqual(@as(i32, -1), windowSizeLeft(.none)); } diff --git a/zml/attention/metal_attention.zig b/zml/attention/metal_attention.zig index 42821a1f..fa1e1929 100644 --- a/zml/attention/metal_attention.zig +++ b/zml/attention/metal_attention.zig @@ -93,10 +93,7 @@ pub const paged = struct { const kc = k_cache.transpose(.{ .page, .k_chunk, .hkv, .hd }); const vc = v_cache.transpose(.{ .page, .k_chunk, .hkv, .hd }); - const sliding_window: i32 = switch (opts.mask) { - .sliding_window => |w| @intCast(w), - else => -1, - }; + const sliding_window: i32 = if (opts.mask.slidingWindow()) |window| @intCast(window) else -1; const out = zml.ops.customCall( "zml$paged_attn", diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 403b5a97..7f83a10b 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -201,25 +201,36 @@ pub const Parameters = union(Backend) { }; pub const AttentionMask = union(enum) { - /// Attend to every valid key in the sequence. - none, - /// Attend to keys at or before the query position. - causal, - /// Causal window size, including the current token. - sliding_window: u32, + standard: struct { + /// Restrict attention to keys at or before the query position. + causal: bool = true, + /// Optional left window size, including the current token. + /// Non-causal attention also includes every later valid key. + sliding_window: ?u32 = null, + }, /// Attend to keys at specific positions. indices: zml.Tensor, + /// Attend to every valid key in the sequence. + pub const none: AttentionMask = .{ .standard = .{ .causal = false } }; + pub fn isCausal(mask: AttentionMask) bool { return switch (mask) { - .causal, .sliding_window => true, - .none, .indices => false, + .standard => |options| options.causal, + .indices => false, + }; + } + + pub fn slidingWindow(mask: AttentionMask) ?u32 { + return switch (mask) { + .standard => |options| options.sliding_window, + .indices => null, }; } }; pub const AttentionOptions = struct { - mask: AttentionMask = .causal, + mask: AttentionMask = .{ .standard = .{} }, scale: ?f32 = null, sink: ?zml.Tensor = null, }; @@ -476,17 +487,17 @@ test pagedAttention { }{ .{ .name = "unbounded", - .attention_options = .{ .mask = .causal }, + .attention_options = .{ .mask = .{ .standard = .{} } }, .backends = all_backends, }, .{ .name = "non_causal", - .attention_options = .{ .mask = .none }, + .attention_options = .{ .mask = .{ .standard = .{ .causal = false } } }, .backends = &option_sensitive_backends, }, .{ .name = "sliding_window", - .attention_options = .{ .mask = .{ .sliding_window = page_size } }, + .attention_options = .{ .mask = .{ .standard = .{ .sliding_window = page_size } } }, .backends = &option_sensitive_backends, }, }; @@ -605,8 +616,8 @@ fn stablehlo_pagedAttention( opts: AttentionOptions, ) zml.Tensor { switch (opts.mask) { - .sliding_window, .indices => @panic("Sliding window or sparse attention mask are not supported on StableHLO"), - else => {}, + .standard => |options| if (options.sliding_window != null) @panic("Sliding window attention mask is not supported on StableHLO"), + .indices => @panic("Sparse attention mask is not supported on StableHLO"), } const page_size = kv_cache.split.k.dim(.k_chunk); diff --git a/zml/attention/tpu_attention.zig b/zml/attention/tpu_attention.zig index 82b39006..f6e26f5a 100644 --- a/zml/attention/tpu_attention.zig +++ b/zml/attention/tpu_attention.zig @@ -200,10 +200,7 @@ pub const mosaic_tpu = struct { const num_queries_per_block = @max(@as(i64, 1), @min(q_token_count, max_num_seqs)); const logical_head_dim: f64 = @floatFromInt(parameters.opts.head_dim); - const sliding_window: ?i64 = switch (opts.mask) { - .sliding_window => |w| @intCast(w), - else => null, - }; + const sliding_window: ?i64 = if (opts.mask.slidingWindow()) |window| @intCast(window) else null; return .{ .num_q_tokens = q_token_count, @@ -319,11 +316,11 @@ test "mosaic_tpu cfg uses kernel head dim and logical scale" { 0.000001, ); - const non_causal_cfg = mosaic_tpu.buildCfg(prepared_inputs, parameters, .{ .mask = .none }); + const non_causal_cfg = mosaic_tpu.buildCfg(prepared_inputs, parameters, .{ .mask = .{ .standard = .{ .causal = false } } }); try std.testing.expect(!non_causal_cfg.is_causal); try std.testing.expectEqual(@as(?i64, null), non_causal_cfg.sliding_window); - const window_cfg = mosaic_tpu.buildCfg(prepared_inputs, parameters, .{ .mask = .{ .sliding_window = 16 } }); + const window_cfg = mosaic_tpu.buildCfg(prepared_inputs, parameters, .{ .mask = .{ .standard = .{ .sliding_window = 16 } } }); try std.testing.expect(window_cfg.is_causal); try std.testing.expectEqual(@as(?i64, 16), window_cfg.sliding_window); } diff --git a/zml/attention/triton_attention.zig b/zml/attention/triton_attention.zig index 3887bc20..9c2cf38c 100644 --- a/zml/attention/triton_attention.zig +++ b/zml/attention/triton_attention.zig @@ -311,10 +311,7 @@ pub const paged = struct { const target_num_prgms: usize = cu_count * 4; const num_2d_prgms: usize = total_q_blocks * num_kv_heads; - const sliding_window: usize = switch (self.opts.mask) { - .sliding_window => |w| @intCast(w), - else => 0, - }; + const sliding_window: usize = self.opts.mask.slidingWindow() orelse 0; const paged_attention_opts: PagedAttentionOptions = .{ .cu_count = getCuCount(), @@ -614,6 +611,7 @@ pub const paged = struct { .use_softcap = false, .use_sinks = (opts.sink != null), .sliding_window = @intCast(paged_attention_opts.sliding_window), + .is_causal = opts.mask.isCausal(), .block_q = @intCast(config.attention.block_q), .block_m = @intCast(config.attention.block_m), .num_segments_per_seq = @intCast(config.attention.num_segments_per_seq), diff --git a/zml/attention/triton_kernels/unified_attention_oneapi.zig b/zml/attention/triton_kernels/unified_attention_oneapi.zig index 66387db0..c1feea1c 100644 --- a/zml/attention/triton_kernels/unified_attention_oneapi.zig +++ b/zml/attention/triton_kernels/unified_attention_oneapi.zig @@ -100,6 +100,7 @@ pub const KernelUnifiedAttention3dPtr = struct { use_softcap: bool, use_sinks: bool, sliding_window: i64, + is_causal: bool = true, stride_k_cache_0: i64 = 0, stride_k_cache_1: i64 = 0, @@ -357,7 +358,7 @@ fn kernelUnifiedAttention3d( const pad_term: i32 = @intCast(@divTrunc(BLOCK_M - 1, NUM_QUERIES_PER_KV) + 1); const max_prefix_raw = context_len.add(q_local_bq).add(pad_term); - const max_seq_prefix_len = max_prefix_raw.minimum(seq_len); + const max_seq_prefix_len = if (config.is_causal) max_prefix_raw.minimum(seq_len) else seq_len; const num_tiles = cdivFn(max_seq_prefix_len, @as(i32, @intCast(TILE_SIZE))); const segm_tile_lo = segm_idx.mul(tiles_per_segment); @@ -437,8 +438,10 @@ fn kernelUnifiedAttention3d( // seq_mask = seq_offset[None, :] < context_len + query_pos[:, None] + 1 const ql_2d_i32 = query_pos.expandDims(1); const so_2d = seq_offset.expandDims(0); - const rhs = context_len.add(ql_2d_i32).add(1); - const seq_mask = so_2d.lt(rhs); + const seq_mask = if (config.is_causal) b: { + const rhs = context_len.add(ql_2d_i32).add(1); + break :b so_2d.lt(rhs); + } else k.broadcastTo(tile_mask.expandDims(0), &.{ BLOCK_M, TILE_SIZE }); const acc_zero = k.zeros(&.{ BLOCK_M, TILE_SIZE }, .f32); const qk = k.dot(Q, K, acc_zero); From 5a1875736849bd97cc85051a9c8a6de5ca94b954 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 7 Oct 2026 11:08:06 +0200 Subject: [PATCH 758/881] zig 0.17 (#693) Bump for zig 0.17 Deps: - uucode bumped to https://github.com/jacobsandlund/uucode/commit/1fb73433bba5d93366c57f23ff2e9d7939746500 - zio bumped to: https://github.com/lalinsky/zio/commit/417757935ef6d0f0a2396d4023a44ca82ac1901c - rules_zig upgraded to [version 0.17](https://github.com/hermeticbuild/rules_zig/releases/tag/v0.17.0) There is one Codex generated patch: - zigimg: minor fixes to allow zig 0.17 to compile it, until they officially support --- .bazelrc | 3 +- .github/workflows/ci.yaml | 2 +- MODULE.bazel | 7 +- MODULE.bazel.lock | 131 +- bazel/bazel.zig | 2 +- bazel/zig_index.json | 223 + bin/zml-smi/api/client.zig | 16 +- bin/zml-smi/csv/csv.zig | 15 +- bin/zml-smi/main.zig | 4 +- bin/zml-smi/platforms/neuron/nrt.zig | 12 +- bin/zml-smi/platforms/oneapi/monitor.zig | 2 +- bin/zml-smi/platforms/oneapi/oneapi.zig | 18 +- bin/zml-smi/prometheus/exposition.zig | 6 +- bin/zml-smi/tui/image_cache.zig | 2 +- bin/zml-smi/tui/print.zig | 13 +- bin/zml-smi/tui/top.zig | 5 +- bin/zml-smi/utils/dynlib.zig | 4 +- bin/zml-smi/utils/sandbox.zig | 2 +- kernels/common/args.zig | 6 +- kernels/common/control_flow.zig | 18 +- kernels/cuda_tile/builder.zig | 146 +- kernels/cuda_tile/control_flow.zig | 8 +- kernels/cuda_tile/dtype.zig | 4 +- kernels/cute/builder.zig | 56 +- kernels/cute/dtype.zig | 4 +- kernels/fly/builder.zig | 47 +- kernels/fly/dtype.zig | 6 +- kernels/fly/layout.zig | 8 +- kernels/mosaic_tpu/builder.zig | 64 +- kernels/mosaic_tpu/dtype.zig | 4 +- kernels/triton/builder.zig | 96 +- kernels/triton/dtype.zig | 4 +- mlir/dialects/arith.zig | 8 +- mlir/dialects/cuda_tile/cuda_tile.zig | 10 +- mlir/dialects/cute_ir/cute_ir.zig | 35 +- mlir/dialects/cute_ir/cute_nvgpu.zig | 442 +- mlir/dialects/fly/attributes.zig | 6 +- mlir/dialects/fly/fly.zig | 2 +- mlir/dialects/llvm_dialect.zig | 2 +- mlir/dialects/memref.zig | 2 +- mlir/dialects/stablehlo/stablehlo.zig | 4 +- mlir/dialects/ttir/ttir.zig | 24 +- mlir/mlir.zig | 10 +- pjrt/ffi.zig | 4 +- pjrt/pjrt.zig | 50 +- platforms/cuda/compat_probe.zig | 2 +- platforms/cuda/cuda.zig | 4 +- platforms/cuda/flashattn/flashattn.zig | 2 +- platforms/neuron/libzml_neuron.zig | 9 +- platforms/neuron/neuron.zig | 8 +- platforms/neuron/python_launcher.zig | 6 +- platforms/platforms.zig | 4 +- stdx/flags.zig | 197 +- stdx/fmt.zig | 6 +- stdx/json.zig | 50 +- stdx/meta.zig | 30 +- stdx/segmented_list.zig | 2 +- stdx/stdx.zig | 61 +- third_party/arocc/arocc.bazel | 12 +- third_party/arocc/repo.bzl | 5 +- third_party/libvaxis/bump-zig.patch | 2019 -------- third_party/libvaxis/fixes.patch | 274 +- third_party/libvaxis/repo.bzl | 4 +- third_party/translate-c/repo.bzl | 5 +- third_party/translate-c/translate-c.bazel | 9 + third_party/uucode/repo.bzl | 3 +- third_party/uucode/uucode.bazel | 2 + third_party/zigimg/bump-zig.patch | 4491 ----------------- third_party/zigimg/repo.bzl | 29 +- third_party/zigimg/zig-0.17.patch | 279 + third_party/zio/repo.bzl | 2 +- vfs/BUILD.bazel | 9 +- vfs/base.zig | 52 +- vfs/file.zig | 115 +- vfs/gcs.zig | 6 +- vfs/hf.zig | 4 +- vfs/http.zig | 4 +- vfs/s3.zig | 6 +- vfs/vfs.zig | 90 +- zml/Compiler.zig | 4 +- zml/Sharding.zig | 74 +- zml/attention.zig | 4 +- zml/attention/cute_kernels/flashmla_sm100.zig | 15 +- zml/dtype.zig | 7 +- zml/exe.zig | 49 +- zml/floats.zig | 14 +- zml/kernel.zig | 8 +- zml/mem.zig | 6 +- zml/meta.zig | 204 +- zml/moe/cute_kernels/persistent_mxfp4.zig | 4 +- zml/moe/cutlass_flashinfer.zig | 4 +- zml/moe/triton_kernels/mxfp4.zig | 4 +- zml/nn.zig | 11 +- zml/ops.zig | 114 +- zml/pjrtx.zig | 8 +- zml/platform.zig | 4 +- zml/profiling/profiler.zig | 2 +- zml/profiling/tracer.zig | 62 +- zml/quantization.zig | 2 +- zml/shape.zig | 72 +- zml/slice.zig | 10 +- zml/tensor.zig | 14 +- zml/tokenizer/iree/tokenizer.zig | 8 +- 103 files changed, 2097 insertions(+), 7945 deletions(-) delete mode 100644 third_party/libvaxis/bump-zig.patch delete mode 100644 third_party/zigimg/bump-zig.patch create mode 100644 third_party/zigimg/zig-0.17.patch diff --git a/.bazelrc b/.bazelrc index 81972716..1b47d230 100644 --- a/.bazelrc +++ b/.bazelrc @@ -61,8 +61,6 @@ common --incompatible_disallow_empty_glob=false # Use the C linker to link Zig code common --@rules_zig//zig/settings:use_cc_common_link=true -# Zig self hosted backend is too fragile for now -common --@rules_zig//zig/settings:zigopt=-fllvm common --@rules_zig//zig/settings:use_standalone_translate_c=True @@ -109,6 +107,7 @@ common:release --compilation_mode=opt common:release --copt=-Ofast common:release --strip=always common:release --@rules_zig//zig/settings:mode=release_safe +common:release --@rules_zig//zig/settings:zigopt=-fllvm # --config=native builds everything optimized for the local CPU common:native --copt=-march=native diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c996de04..13d153b3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -118,7 +118,7 @@ jobs: - name: Setup Zig uses: mlugg/setup-zig@v2.2.1 with: - version: 0.16.0 + version: 0.17.0 mirror: https://mirror.zml.ai/zig - name: Zig Format diff --git a/MODULE.bazel b/MODULE.bazel index 1e05f12a..2c35cdc0 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -49,7 +49,8 @@ bazel_dep(name = "rules_shell", version = "0.6.1") # repo_name = None prevents rules_swift from eagerly auto-registering toolchains. bazel_dep(name = "rules_swift", version = "3.6.1", repo_name = None) -bazel_dep(name = "rules_zig", version = "0.15.1") +bazel_dep(name = "rules_zig", version = "0.17.0") + bazel_dep(name = "tar.bzl", version = "0.10.4") bazel_dep(name = "llvm", version = "0.8.24") bazel_dep(name = "zlib-ng", version = "2.3.3", repo_name = "llvm_zlib") @@ -95,7 +96,7 @@ zig.index(file = "//bazel:zig_index.json") zig.toolchain( # Ensure Zig actions are run locally only extra_exec_compatible_with = ["//platforms:local_execution"], - zig_version = "0.16.0", + zig_version = "0.17.0", ) zig.mirrors(urls = [ "https://pkg.machengine.org/zig", @@ -133,7 +134,7 @@ register_execution_platforms("//platforms:local_execution_platform") zls = use_extension("@rules_zig//zig/zls:extensions.bzl", "zls") zls.index(file = "@rules_zig//zig/zls/private:versions.json") zls.toolchain( - zig_version = "0.16.0", + zig_version = "0.17.0", zls_version = "0.16.0", ) use_repo(zls, "zls_toolchains") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 838e2011..548aa0e3 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -175,8 +175,10 @@ "https://bcr.bazel.build/modules/libbacktrace/1.0.0-20250926-7939218/source.json": "83239989d09748f63dee4bdd430d261e217a012670e066e2238925a303a65d89", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", "https://bcr.bazel.build/modules/libpfm/4.11.0/source.json": "caaffb3ac2b59b8aac456917a4ecf3167d40478ee79f15ab7a877ec9273937c9", - "https://bcr.bazel.build/modules/llvm/0.8.20/MODULE.bazel": "2c0fa2fb60ab1f3a8a7b46a6c60b67fd78bffcfaf7958477e74a971c0a1f1915", - "https://bcr.bazel.build/modules/llvm/0.8.20/source.json": "21ef6b5c124a1a39341752e8458de9acd6fb4abc9ad73e6e3147b374d95e5e8c", + "https://bcr.bazel.build/modules/libxml2/2.15.3/MODULE.bazel": "a6bffafd8aa8b0c13eaa4bf25d55be188ec8548549395f03e98c39bab264adaa", + "https://bcr.bazel.build/modules/libxml2/2.15.3/source.json": "fd7c420cb14981416c9fadd0db555bd1e249355e0e5b63b2ce15b92996b5fbfb", + "https://bcr.bazel.build/modules/llvm/0.8.24/MODULE.bazel": "e03e030b1d0b44bb05c52665c2f43de9260af9152f666262e11e25f6c072aae3", + "https://bcr.bazel.build/modules/llvm/0.8.24/source.json": "924ed48858c1c081698f1992c7b489434375de901c47b7046e203459840cb544", "https://bcr.bazel.build/modules/mbedtls/3.6.0/MODULE.bazel": "8e380e4698107c5f8766264d4df92e36766248447858db28187151d884995a09", "https://bcr.bazel.build/modules/mbedtls/3.6.0/source.json": "1dbe7eb5258050afcc3806b9d43050f71c6f539ce0175535c670df606790b30c", "https://bcr.bazel.build/modules/nlohmann_json/3.11.3/MODULE.bazel": "87023db2f55fc3a9949c7b08dc711fae4d4be339a80a99d04453c4bb3998eefc", @@ -299,6 +301,7 @@ "https://bcr.bazel.build/modules/rules_cc/0.2.25/source.json": "8df28a9a97b878fe45f0a4f4adf019869dda51a60ee2f6e15a0c79656a25058d", "https://bcr.bazel.build/modules/rules_cc/0.2.4/MODULE.bazel": "1ff1223dfd24f3ecf8f028446d4a27608aa43c3f41e346d22838a4223980b8cc", "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", + "https://bcr.bazel.build/modules/rules_cc/0.2.9/MODULE.bazel": "34263f1dca62ea664265438cef714d7db124c03e1ed55ebb4f1dc860164308d1", "https://bcr.bazel.build/modules/rules_distroless/0.8.0/MODULE.bazel": "a3e473943b685190e20668f5149f62a116fdcbeba66c7b9778f71d2aed4533c1", "https://bcr.bazel.build/modules/rules_distroless/0.8.0/source.json": "a6931e20d97a13675693adb8b6cd3a0e6eb5218db43dfc83b75566ddc4158efc", "https://bcr.bazel.build/modules/rules_foreign_cc/0.10.1/MODULE.bazel": "b9527010e5fef060af92b6724edb3691970a5b1f76f74b21d39f7d433641be60", @@ -426,8 +429,8 @@ "https://bcr.bazel.build/modules/rules_swift/3.1.2/MODULE.bazel": "72c8f5cf9d26427cee6c76c8e3853eb46ce6b0412a081b2b6db6e8ad56267400", "https://bcr.bazel.build/modules/rules_swift/3.6.1/MODULE.bazel": "5a68bb05b6b8044f8a7bd7184d56150189e7eca05522ce525517c234fed50ed8", "https://bcr.bazel.build/modules/rules_swift/3.6.1/source.json": "9af2e521ffdb41961b6b9ac05293bd6519ee2fb52849a2be658a20b0221cb8db", - "https://bcr.bazel.build/modules/rules_zig/0.15.1/MODULE.bazel": "f0b813d9191c8d201e0b3ecbe5c5220ddedb717a15009fc711a78b3e1db8710d", - "https://bcr.bazel.build/modules/rules_zig/0.15.1/source.json": "0bd91d7459f504196dfdc90cd0fffbf1f0df2667c9cd5eb708feb3fd27578e6a", + "https://bcr.bazel.build/modules/rules_zig/0.17.0/MODULE.bazel": "2c1c08adc986cfdb5fee97e21982cac846f8a20a6a8054ae9cab8b02721c2011", + "https://bcr.bazel.build/modules/rules_zig/0.17.0/source.json": "488f71f57c596ed9b7635ebeb28066b23c6fc1802c30749d06bdad261d7b04a2", "https://bcr.bazel.build/modules/stardoc/0.5.0/MODULE.bazel": "f9f1f46ba8d9c3362648eea571c6f9100680efc44913618811b58cc9c02cd678", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", @@ -727,7 +730,7 @@ }, "@@rules_oci+//oci:extensions.bzl%oci": { "general": { - "bzlTransitiveDigest": "lRG1JBXWgJgDfEfl/34ZHgTFACz2I0bpfZWFtcoM+hc=", + "bzlTransitiveDigest": "u2chY5noKxJ9cum/JgI8hogPYhYniVKnJLd6f2ChPI4=", "usagesDigest": "JLbamplho20rviuebatMGqnQnaoj32ewCzLuRVusGoU=", "recordedInputs": [ "REPO_MAPPING:aspect_bazel_lib+,bazel_tools bazel_tools", @@ -1509,19 +1512,19 @@ }, "@@rules_zig+//zig:extensions.bzl%zig": { "general": { - "bzlTransitiveDigest": "VGLOg2kVw+STO9R5OrQnLFbv7Vl5TPdtIebIs324IpU=", - "usagesDigest": "4xBoe2eIScx2QI0jVieHWXdflezUUDIApwMqS+SWGbw=", + "bzlTransitiveDigest": "VY2CKW7wm6B3wCpZq7W6z3hCnHiiJlXtB4JpuB3OPD8=", + "usagesDigest": "FDLiNUluzC0WBatQ/5WFpxzyQP4ubA/DUgTwOzYLGw4=", "recordedInputs": [ "REPO_MAPPING:rules_zig+,bazel_skylib bazel_skylib+", "REPO_MAPPING:rules_zig+,bazel_tools bazel_tools", - "FILE:@@//bazel/zig_index.json 2e3b4b83f1add5d3ca12fd05797c04c38af21d86c7fc7c1a9b356560141a4797", - "FILE:@@rules_zig+//zig/private/versions.json 3e4e370dee9c3f3fdc8bfc7a600e76743278fde40815eead5d0bcda3aec3e2bd" + "FILE:@@//bazel/zig_index.json 387d1ab130896456fb092fb08bfaf0c57ccba854121955c752052600d1643d37", + "FILE:@@rules_zig+//zig/private/versions.json a33370258a702099b35362d4220fe8f02c869c2b845abcbf0aa78200abab097a" ], "generatedRepoSpecs": { - "zig_0.16.0_aarch64-linux": { + "zig_0.17.0_aarch64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.16.0/zig-aarch64-linux-0.16.0.tar.xz", + "url": "https://ziglang.org/download/0.17.0/zig-aarch64-linux-0.17.0.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1551,15 +1554,15 @@ "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds" ], - "sha256": "ea4b09bfb22ec6f6c6ceac57ab63efb6b46e17ab08d21f69f3a48b38e1534f17", - "zig_version": "0.16.0", + "sha256": "9e8d11661d4ae3bd57702a3832781e23ad151dde5798e16a5ccd503f65234ff8", + "zig_version": "0.17.0", "platform": "aarch64-linux" } }, - "zig_0.16.0_aarch64-macos": { + "zig_0.17.0_aarch64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.16.0/zig-aarch64-macos-0.16.0.tar.xz", + "url": "https://ziglang.org/download/0.17.0/zig-aarch64-macos-0.17.0.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1589,15 +1592,15 @@ "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds" ], - "sha256": "b23d70deaa879b5c2d486ed3316f7eaa53e84acf6fc9cc747de152450d401489", - "zig_version": "0.16.0", + "sha256": "b607e9b9234790a008116ae5bdb71c6243b84b9fb42a53a9e70fde41c06c536a", + "zig_version": "0.17.0", "platform": "aarch64-macos" } }, - "zig_0.16.0_aarch64-windows": { + "zig_0.17.0_aarch64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.16.0/zig-aarch64-windows-0.16.0.zip", + "url": "https://ziglang.org/download/0.17.0/zig-aarch64-windows-0.17.0.zip", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1627,15 +1630,15 @@ "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds" ], - "sha256": "aee38316ee4111717900f45dd3130145c39289e105541d737eb8c5ed653c78ef", - "zig_version": "0.16.0", + "sha256": "0a59d91fa1cb40cf068e9b0954434ce973500c7a2ea749f1e01af62cdab52d26", + "zig_version": "0.17.0", "platform": "aarch64-windows" } }, - "zig_0.16.0_x86_64-linux": { + "zig_0.17.0_x86_64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.16.0/zig-x86_64-linux-0.16.0.tar.xz", + "url": "https://ziglang.org/download/0.17.0/zig-x86_64-linux-0.17.0.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1665,15 +1668,15 @@ "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds" ], - "sha256": "70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00", - "zig_version": "0.16.0", + "sha256": "1cbe9df9f27e6b78d14ccbca43b6703a404ef79ef1c463de901d7f088d4e2026", + "zig_version": "0.17.0", "platform": "x86_64-linux" } }, - "zig_0.16.0_x86_64-macos": { + "zig_0.17.0_x86_64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.16.0/zig-x86_64-macos-0.16.0.tar.xz", + "url": "https://ziglang.org/download/0.17.0/zig-x86_64-macos-0.17.0.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1703,15 +1706,15 @@ "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds" ], - "sha256": "0387557ed1877bc6a2e1802c8391953baddba76081876301c522f52977b52ba7", - "zig_version": "0.16.0", + "sha256": "4f9a1c5269aa17ebda5e6d3c2b89d6cbf36f7d2b22a0306e9ab98f25f95529c6", + "zig_version": "0.17.0", "platform": "x86_64-macos" } }, - "zig_0.16.0_x86_64-windows": { + "zig_0.17.0_x86_64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.16.0/zig-x86_64-windows-0.16.0.zip", + "url": "https://ziglang.org/download/0.17.0/zig-x86_64-windows-0.17.0.zip", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1741,8 +1744,8 @@ "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds" ], - "sha256": "68659eb5f1e4eb1437a722f1dd889c5a322c9954607f5edcf337bc3684a75a7e", - "zig_version": "0.16.0", + "sha256": "b5663f69581dcf391293fbf16c06cb80d81d806545ce618b4d0bab7f0eb8c428", + "zig_version": "0.17.0", "platform": "x86_64-windows" } }, @@ -1750,28 +1753,28 @@ "repoRuleId": "@@rules_zig+//zig/private/repo:toolchains_repo.bzl%toolchains_repo", "attributes": { "names": [ - "zig_0.16.0_aarch64-linux", - "zig_0.16.0_aarch64-macos", - "zig_0.16.0_aarch64-windows", - "zig_0.16.0_x86_64-linux", - "zig_0.16.0_x86_64-macos", - "zig_0.16.0_x86_64-windows" + "zig_0.17.0_aarch64-linux", + "zig_0.17.0_aarch64-macos", + "zig_0.17.0_aarch64-windows", + "zig_0.17.0_x86_64-linux", + "zig_0.17.0_x86_64-macos", + "zig_0.17.0_x86_64-windows" ], "labels": [ - "@zig_0.16.0_aarch64-linux//:zig_toolchain", - "@zig_0.16.0_aarch64-macos//:zig_toolchain", - "@zig_0.16.0_aarch64-windows//:zig_toolchain", - "@zig_0.16.0_x86_64-linux//:zig_toolchain", - "@zig_0.16.0_x86_64-macos//:zig_toolchain", - "@zig_0.16.0_x86_64-windows//:zig_toolchain" + "@zig_0.17.0_aarch64-linux//:zig_toolchain", + "@zig_0.17.0_aarch64-macos//:zig_toolchain", + "@zig_0.17.0_aarch64-windows//:zig_toolchain", + "@zig_0.17.0_x86_64-linux//:zig_toolchain", + "@zig_0.17.0_x86_64-macos//:zig_toolchain", + "@zig_0.17.0_x86_64-windows//:zig_toolchain" ], "zig_versions": [ - "0.16.0", - "0.16.0", - "0.16.0", - "0.16.0", - "0.16.0", - "0.16.0" + "0.17.0", + "0.17.0", + "0.17.0", + "0.17.0", + "0.17.0", + "0.17.0" ], "exec_lengths": [ 3, @@ -1827,7 +1830,7 @@ "@@rules_zig+//zig/zls:extensions.bzl%zls": { "general": { "bzlTransitiveDigest": "ZuceGxE17jROQ+kQkaPqIcK7gMKGb56I97Aq1/Dubdc=", - "usagesDigest": "b/ERjMTWazfa8VFWuTc0PTtQQzLDuloWSHlzXj6Zb5w=", + "usagesDigest": "UswUnORpE1Loq1Ca5pqN4rYDdsd2ECF1KGnvjJxLhR8=", "recordedInputs": [ "REPO_MAPPING:,rules_zig rules_zig+", "REPO_MAPPING:rules_zig+,bazel_skylib bazel_skylib+", @@ -1899,12 +1902,12 @@ "repoRuleId": "@@rules_zig+//zig/zls/private/repo:toolchains_repo.bzl%toolchains_repo", "attributes": { "names": [ - "0.16.0_for_0.16.0_aarch64-linux", - "0.16.0_for_0.16.0_aarch64-macos", - "0.16.0_for_0.16.0_aarch64-windows", - "0.16.0_for_0.16.0_x86_64-linux", - "0.16.0_for_0.16.0_x86_64-macos", - "0.16.0_for_0.16.0_x86_64-windows" + "0.16.0_for_0.17.0_aarch64-linux", + "0.16.0_for_0.17.0_aarch64-macos", + "0.16.0_for_0.17.0_aarch64-windows", + "0.16.0_for_0.17.0_x86_64-linux", + "0.16.0_for_0.17.0_x86_64-macos", + "0.16.0_for_0.17.0_x86_64-windows" ], "labels": [ "@zls_0.16.0_aarch64-linux//:zls_toolchain", @@ -1915,12 +1918,12 @@ "@zls_0.16.0_x86_64-windows//:zls_toolchain" ], "zig_versions": [ - "0.16.0", - "0.16.0", - "0.16.0", - "0.16.0", - "0.16.0", - "0.16.0" + "0.17.0", + "0.17.0", + "0.17.0", + "0.17.0", + "0.17.0", + "0.17.0" ], "exec_lengths": [ 2, diff --git a/bazel/bazel.zig b/bazel/bazel.zig index 0cf23580..2ae3324a 100644 --- a/bazel/bazel.zig +++ b/bazel/bazel.zig @@ -12,7 +12,7 @@ const init_array_section = switch (builtin.object_format) { else => "", }; -export const _bazel_runfiles_init linksection(init_array_section) = &struct { +export const _bazel_runfiles_init align(@alignOf(usize)) linksection(init_array_section) = &struct { fn call(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { _ = argc; _ = envp; diff --git a/bazel/zig_index.json b/bazel/zig_index.json index 8b39b699..b8cf9b9e 100644 --- a/bazel/zig_index.json +++ b/bazel/zig_index.json @@ -1,4 +1,227 @@ { + "0.17.0": { + "version": "0.17.0", + "date": "2026-10-01", + "docs": "https://ziglang.org/documentation/0.17.0/", + "stdDocs": "https://ziglang.org/documentation/0.17.0/std/", + "notes": "https://ziglang.org/download/0.17.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.17.0/zig-0.17.0.tar.xz", + "shasum": "b6c7f1728f043700d6529bac980800792f824256a9d2f1839b3d62beed0b8abd", + "size": "23049960" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.17.0/zig-bootstrap-0.17.0.tar.xz", + "shasum": "1e9e9b8e3c753b35dfb1d9ea48f097579fadbd3e5c1e724ab3fc5bce05477b71", + "size": "57030388" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-macos-0.17.0.tar.xz", + "shasum": "4f9a1c5269aa17ebda5e6d3c2b89d6cbf36f7d2b22a0306e9ab98f25f95529c6", + "size": "59317572" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-macos-0.17.0.tar.xz", + "shasum": "b607e9b9234790a008116ae5bdb71c6243b84b9fb42a53a9e70fde41c06c536a", + "size": "53985220" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-linux-0.17.0.tar.xz", + "shasum": "1cbe9df9f27e6b78d14ccbca43b6703a404ef79ef1c463de901d7f088d4e2026", + "size": "57332648" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-linux-0.17.0.tar.xz", + "shasum": "9e8d11661d4ae3bd57702a3832781e23ad151dde5798e16a5ccd503f65234ff8", + "size": "52877280" + }, + "arm-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-arm-linux-0.17.0.tar.xz", + "shasum": "53f0045cdef7ba06da70a12b2ef654e4c5283b0139bc441aa7d84f364b46de43", + "size": "53841896" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-riscv64-linux-0.17.0.tar.xz", + "shasum": "18ff032ac6cecf746a84259f6463264f13160ff88bc3b80722ce06f9479b82ed", + "size": "57116976" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-powerpc64le-linux-0.17.0.tar.xz", + "shasum": "731e2887e66fff41fdc34f7fd9d8afa814d86d7426f5827805f8854b2977684b", + "size": "56990076" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86-linux-0.17.0.tar.xz", + "shasum": "55e39e175cd5b3098afc29ec47b2590134f5b874175369fb9ae7b8e836d7531a", + "size": "59823964" + }, + "loongarch64-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-loongarch64-linux-0.17.0.tar.xz", + "shasum": "5a6000b866201c3b3820b563fcfd29641185393e7d35039e50235db418d8367c", + "size": "54257324" + }, + "s390x-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-s390x-linux-0.17.0.tar.xz", + "shasum": "13905f11989b65566c544d191d4bc5bd5d6f858e303208cf1d2f090e5d91b512", + "size": "57136832" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-windows-0.17.0.zip", + "shasum": "b5663f69581dcf391293fbf16c06cb80d81d806545ce618b4d0bab7f0eb8c428", + "size": "100268137" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-windows-0.17.0.zip", + "shasum": "0a59d91fa1cb40cf068e9b0954434ce973500c7a2ea749f1e01af62cdab52d26", + "size": "95992853" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86-windows-0.17.0.zip", + "shasum": "c7b83507549c7bcd42cd58c4910b47037f4150ed7778e11b3684f130620969d1", + "size": "101891323" + }, + "aarch64-freebsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-freebsd-0.17.0.tar.xz", + "shasum": "3cf53d07b3587a26727b196a6eb7918704b907d951520daa68e4e1820fc6f48e", + "size": "52860244" + }, + "arm-freebsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-arm-freebsd-0.17.0.tar.xz", + "shasum": "e0c3ecf21c186a48d3b8e8d0b4459f9dc67ff8024445401506a7e970782da5f8", + "size": "54273908" + }, + "powerpc64le-freebsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-powerpc64le-freebsd-0.17.0.tar.xz", + "shasum": "94e127c2a4beb5151a8c58e8af4b82dc3649ed95f343ffbbfa381e2afba50234", + "size": "57027652" + }, + "riscv64-freebsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-riscv64-freebsd-0.17.0.tar.xz", + "shasum": "4ab73eb01a0cb910dfa23d8efc88d38dcdd5ebb2174537cf531738ba520354f1", + "size": "57199160" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-freebsd-0.17.0.tar.xz", + "shasum": "a314f1cc92d098bd39caf9e5607a0de2a8fd0d00064776cb3fd5b3bee38a67c9", + "size": "57468884" + }, + "aarch64-netbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-netbsd-0.17.0.tar.xz", + "shasum": "421fefe3910dedc8b8a6ae9a8905ea9200ecb0760499bcf0f56af7a5ba4930b5", + "size": "52769844" + }, + "arm-netbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-arm-netbsd-0.17.0.tar.xz", + "shasum": "2537d66f6dbd57438355d100180d09f19640c98895d4052e4e2994a02cac320a", + "size": "55345456" + }, + "riscv64-netbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-riscv64-netbsd-0.17.0.tar.xz", + "shasum": "687c92430ab3e7a5329e941e95f0ec4368d8d8ac0a4048f61c2dd714e675ca12", + "size": "57157568" + }, + "x86-netbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86-netbsd-0.17.0.tar.xz", + "shasum": "2025c508390cc78d9c469675d2a0602f2a975c92ec197af51faf09a565b27ee3", + "size": "60380332" + }, + "x86_64-netbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-netbsd-0.17.0.tar.xz", + "shasum": "10f71cc89efbafc6e22447f23c44047d53c5d2061c82b1c32efd1ef55ea2d9b6", + "size": "57363300" + }, + "aarch64-openbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-openbsd-0.17.0.tar.xz", + "shasum": "1cf6c33a0cdd9a58a0b40c7ee86d8de9c190ed7bd8d47ad61687908955ecac10", + "size": "53265892" + }, + "arm-openbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-arm-openbsd-0.17.0.tar.xz", + "shasum": "068d5200b027541c2b0a2e16efc3fb367f1fae78f9c3954e57512868c9a0ecc2", + "size": "54024240" + }, + "riscv64-openbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-riscv64-openbsd-0.17.0.tar.xz", + "shasum": "4858839a9e4d96fb50d6e4894d10270f5377a087b2929d238b1a12b8293e274a", + "size": "57490512" + }, + "x86_64-openbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-openbsd-0.17.0.tar.xz", + "shasum": "5e9606fc322ef4c346c199e162d26e4c293358a36ea3ca72e877d42ef71e24be", + "size": "58612444" + } + }, + "0.17.0-dev.1936+5a625d5f3": { + "version": "0.17.0-dev.1936+5a625d5f3", + "docs": "https://ziglang.org/documentation/master/", + "stdDocs": "https://ziglang.org/documentation/master/std/", + "x86_64-macos": { + "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.17.0-dev.1936+5a625d5f3.tar.xz", + "shasum": "8ab9f0d015abb75c0c9c039f7191711a7a2fc72c984458f553926725794791dd", + "size": "59162280" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.17.0-dev.1936+5a625d5f3.tar.xz", + "shasum": "e08d4485c96d2d31a8ce3ae80a58badec202920b675149ad6b0c3b974ec7c005", + "size": "53867184" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.17.0-dev.1936+5a625d5f3.tar.xz", + "shasum": "38620c06ca57b175f7dab8f65c8001f9a67b0ff1987c344d4e64ccb72b57174f", + "size": "57180268" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.17.0-dev.1936+5a625d5f3.tar.xz", + "shasum": "6c559ff4bca3e84fbbf7b6ad8521ce70bb3e460371b3d8803112353fb8ce70cc", + "size": "52741876" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.17.0-dev.1936+5a625d5f3.zip", + "shasum": "e0d62a00e067de41a0a34af2af73694e00692e045ca244748965a7b3729633a1", + "size": "100793891" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.17.0-dev.1936+5a625d5f3.zip", + "shasum": "e75f340afee0772fe4daed61d92b999534c8460601175b66fe890703cf752850", + "size": "96526396" + } + }, + "0.17.0-dev.1509+bb296ab9b": { + "version": "0.17.0-dev.1509+bb296ab9b", + "date": "2026-07-29", + "docs": "https://ziglang.org/documentation/master/", + "stdDocs": "https://ziglang.org/documentation/master/std/", + "x86_64-macos": { + "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.17.0-dev.1509+bb296ab9b.tar.xz", + "shasum": "087aad38fead6c0ba7a52b5db3c405e63708c4765eda643d4144b9e4933d45e5", + "size": "59064872" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.17.0-dev.1509+bb296ab9b.tar.xz", + "shasum": "24bd83c1d435b8ab6192f58afcc22f4e5252077a938ffbc35eaa4ca97c5be709", + "size": "53773824" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.17.0-dev.1509+bb296ab9b.tar.xz", + "shasum": "48cc865b8b410ec84eaa97e50c2bd7a657871802ce3aaaf04dd1da2294d4b28a", + "size": "57056988" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.17.0-dev.1509+bb296ab9b.tar.xz", + "shasum": "7f17e09b675cfe39805c5b551b251151f7750b9df8a168538007890c1c00dc8e", + "size": "52649416" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.17.0-dev.1509+bb296ab9b.zip", + "shasum": "a4f537761ca1f5274c9bb0e7da598c6e82205a4ea78f329690a26eade555d7e8", + "size": "100743514" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.17.0-dev.1509+bb296ab9b.zip", + "shasum": "1fa7a2b5f5442628287fead0f7451b8cb93ed2ca5dd6c79f8026fc8ca83580d2", + "size": "96467368" + } + }, "master": { "version": "0.16.0-dev.3153+d6f43caad", "date": "2026-04-12", diff --git a/bin/zml-smi/api/client.zig b/bin/zml-smi/api/client.zig index 17e49917..ddec6d2d 100644 --- a/bin/zml-smi/api/client.zig +++ b/bin/zml-smi/api/client.zig @@ -6,10 +6,10 @@ const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.Ar const Collector = @import("zml-smi/collector").Collector; fn setRemote(device: *DeviceInfo) void { - inline for (@typeInfo(DeviceInfo).@"union".fields) |field| { - if (device.* == @field(smi_info.device_info.Target, field.name)) { - @field(device, field.name).values[0].remote = true; - @field(device, field.name).values[1].remote = true; + inline for (@typeInfo(DeviceInfo).@"union".field_names) |field_name| { + if (device.* == @field(smi_info.device_info.Target, field_name)) { + @field(device, field_name).values[0].remote = true; + @field(device, field_name).values[1].remote = true; } } } @@ -86,10 +86,10 @@ fn pollOnce(poll_arena: *std.heap.ArenaAllocator, gpa: std.mem.Allocator, io: st var device = std.json.parseFromValueLeaky(DeviceInfo, poll_arena.allocator(), item, .{ .ignore_unknown_fields = true }) catch continue; setRemote(&device); - inline for (@typeInfo(DeviceInfo).@"union".fields) |field| { - if (devices[i].* == @field(smi_info.device_info.Target, field.name)) { - @field(devices[i], field.name).back().* = @field(device, field.name).front().*; - @field(devices[i], field.name).swap(); + inline for (@typeInfo(DeviceInfo).@"union".field_names) |field_name| { + if (devices[i].* == @field(smi_info.device_info.Target, field_name)) { + @field(devices[i], field_name).back().* = @field(device, field_name).front().*; + @field(devices[i], field_name).swap(); } } } diff --git a/bin/zml-smi/csv/csv.zig b/bin/zml-smi/csv/csv.zig index a05c5a4f..dbd80dd9 100644 --- a/bin/zml-smi/csv/csv.zig +++ b/bin/zml-smi/csv/csv.zig @@ -3,8 +3,9 @@ const smi_info = @import("zml-smi/info"); const DeviceInfo = smi_info.device_info.DeviceInfo; pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo) !void { - inline for (@typeInfo(DeviceInfo).@"union".fields) |tp| { - const tag = @field(smi_info.Target, tp.name); + const info = @typeInfo(DeviceInfo).@"union"; + inline for (info.field_names, info.field_types) |type_name, DeviceType| { + const tag = @field(smi_info.Target, type_name); var header_printed = false; for (devices, 0..) |dev, i| { @@ -12,8 +13,8 @@ pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo) !void { tag => |*db| { if (!header_printed) { try writer.writeAll("index,type"); - inline for (@typeInfo(tp.type.Value).@"struct".fields) |f| { - try writer.writeAll("," ++ f.name); + inline for (@typeInfo(DeviceType.Value).@"struct".field_names) |field_name| { + try writer.writeAll("," ++ field_name); } try writer.writeAll("\n"); @@ -22,10 +23,10 @@ pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo) !void { const val = db.front().*; - try writer.print("{d},{s}", .{ i, tp.name }); - inline for (@typeInfo(tp.type.Value).@"struct".fields) |f| { + try writer.print("{d},{s}", .{ i, type_name }); + inline for (@typeInfo(DeviceType.Value).@"struct".field_names) |field_name| { try writer.writeAll(","); - try writeValue(writer, @field(val, f.name)); + try writeValue(writer, @field(val, field_name)); } try writer.writeAll("\n"); }, diff --git a/bin/zml-smi/main.zig b/bin/zml-smi/main.zig index 47c861ff..f066dd34 100644 --- a/bin/zml-smi/main.zig +++ b/bin/zml-smi/main.zig @@ -163,9 +163,9 @@ pub fn main(init: std.process.Init) !void { try json.write(&json_writer.interface, collector.device_infos.items, procs.items); try json_writer.flush(); } else if (args.top) { - try tui.run(gpa, io, &state); + try tui.run(gpa, io, init.environ_map, &state); } else { - try static_print.run(arena, io, &state); + try static_print.run(arena, io, init.environ_map, &state); } } diff --git a/bin/zml-smi/platforms/neuron/nrt.zig b/bin/zml-smi/platforms/neuron/nrt.zig index 8fe21a27..cd77b3c0 100644 --- a/bin/zml-smi/platforms/neuron/nrt.zig +++ b/bin/zml-smi/platforms/neuron/nrt.zig @@ -151,11 +151,11 @@ pub fn hbmSize(dev: *c.ndl_device_t) usize { } pub fn deviceType(dev: *c.ndl_device_t) DeviceType { - return @enumFromInt(dev.device_type); + return @fromBackingInt(@intCast(dev.device_type)); } fn resolveFromElf(handle: *anyopaque, io: std.Io, comptime known_sym: [:0]const u8) !PrivateFns { - const fields = std.meta.fields(PrivateFns); + const field_names = comptime std.meta.fieldNames(PrivateFns); const sym_addr = std.c.dlsym(handle, known_sym) orelse return error.SymbolNotFound; @@ -181,7 +181,7 @@ fn resolveFromElf(handle: *anyopaque, io: std.Io, comptime known_sym: [:0]const defer std.posix.munmap(mmap); var result: PrivateFns = undefined; - var remaining: usize = fields.len; + var remaining: usize = field_names.len; const ehdr: *const elf.ElfN.Ehdr = @ptrCast(@alignCast(mmap)); const shdrs: []const elf.ElfN.Shdr = @as([*]const elf.ElfN.Shdr, @ptrCast(@alignCast(mmap[ehdr.shoff..])))[0..ehdr.shnum]; @@ -201,9 +201,9 @@ fn resolveFromElf(handle: *anyopaque, io: std.Io, comptime known_sym: [:0]const const sym_name = std.mem.span(@as([*c]const u8, @ptrCast(strtab[@as(usize, s.name)..]))); - inline for (fields) |field| { - if (std.mem.eql(u8, sym_name, field.name)) { - @field(result, field.name) = @ptrFromInt(elf_base + @as(usize, s.value)); + inline for (field_names) |field_name| { + if (std.mem.eql(u8, sym_name, field_name)) { + @field(result, field_name) = @ptrFromInt(elf_base + @as(usize, s.value)); remaining -= 1; } } diff --git a/bin/zml-smi/platforms/oneapi/monitor.zig b/bin/zml-smi/platforms/oneapi/monitor.zig index 503ae0df..dd5f97a9 100644 --- a/bin/zml-smi/platforms/oneapi/monitor.zig +++ b/bin/zml-smi/platforms/oneapi/monitor.zig @@ -149,7 +149,7 @@ inline fn openDevice(allocator: std.mem.Allocator, io: std.Io, render_idx: usize } pub fn device(self: *const Monitor, handle: Handle) !*const Device { - const idx = @intFromEnum(handle); + const idx = @backingInt(handle); if (idx >= self.devices.len) return error.NotFound; return &self.devices[idx]; } diff --git a/bin/zml-smi/platforms/oneapi/oneapi.zig b/bin/zml-smi/platforms/oneapi/oneapi.zig index 0e89a6a2..2e1e5deb 100644 --- a/bin/zml-smi/platforms/oneapi/oneapi.zig +++ b/bin/zml-smi/platforms/oneapi/oneapi.zig @@ -18,7 +18,7 @@ pub fn init(allocator: std.mem.Allocator, io: std.Io) !OneApi { pub fn handleByIndex(self: *const OneApi, device_id: usize) !Handle { if (device_id >= self.monitor.devices.len) return error.NotFound; - return @enumFromInt(@as(u32, @intCast(device_id))); + return @fromBackingInt(@intCast(@as(u32, @intCast(device_id)))); } pub fn name(self: *const OneApi, allocator: std.mem.Allocator, handle: Handle) ![]const u8 { @@ -111,7 +111,7 @@ pub fn driverVersion(self: *const OneApi, allocator: std.mem.Allocator) ![]const pub fn powerUsage(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { const dev = block: { - const idx = @intFromEnum(handle); + const idx = @backingInt(handle); if (idx >= self.monitor.devices.len) return error.NotFound; break :block &self.monitor.devices[idx]; }; @@ -144,7 +144,7 @@ pub fn fanSpeed(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u6 pub fn gpuUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { const dev = block: { - const idx = @intFromEnum(handle); + const idx = @backingInt(handle); if (idx >= self.monitor.devices.len) return error.NotFound; break :block &self.monitor.devices[idx]; }; @@ -155,7 +155,7 @@ pub fn gpuUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 }; defer usage.deinit(allocator); - const dev_idx: u16 = @intCast(@intFromEnum(handle)); + const dev_idx: u16 = @intCast(@backingInt(handle)); const current = if (usage.get(dev_idx)) |sample| sample.engine else null; const util = Monitor.processUtil(dev.activity_prev, current) orelse 0; dev.activity_prev = current; @@ -164,7 +164,7 @@ pub fn gpuUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 pub fn encoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { const dev = block: { - const idx = @intFromEnum(handle); + const idx = @backingInt(handle); if (idx >= self.monitor.devices.len) return error.NotFound; break :block &self.monitor.devices[idx]; }; @@ -175,7 +175,7 @@ pub fn encoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) }; defer usage.deinit(allocator); - const dev_idx: u16 = @intCast(@intFromEnum(handle)); + const dev_idx: u16 = @intCast(@backingInt(handle)); const current = if (usage.get(dev_idx)) |sample| sample.encoder else null; const util = Monitor.processUtil(dev.encoder_prev, current) orelse { dev.encoder_prev = current; @@ -187,7 +187,7 @@ pub fn encoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) pub fn decoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { const dev = block: { - const idx = @intFromEnum(handle); + const idx = @backingInt(handle); if (idx >= self.monitor.devices.len) return error.NotFound; break :block &self.monitor.devices[idx]; }; @@ -198,7 +198,7 @@ pub fn decoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) }; defer usage.deinit(allocator); - const dev_idx: u16 = @intCast(@intFromEnum(handle)); + const dev_idx: u16 = @intCast(@backingInt(handle)); const current = if (usage.get(dev_idx)) |sample| sample.decoder else null; const util = Monitor.processUtil(dev.decoder_prev, current) orelse { dev.decoder_prev = current; @@ -219,7 +219,7 @@ pub fn maxClockGraphics(self: *OneApi, allocator: std.mem.Allocator, handle: Han } pub fn memUsed(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev_idx: u16 = @intCast(@intFromEnum(handle)); + const dev_idx: u16 = @intCast(@backingInt(handle)); var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch return 0; defer usage.deinit(allocator); diff --git a/bin/zml-smi/prometheus/exposition.zig b/bin/zml-smi/prometheus/exposition.zig index c955226f..54d48cb6 100644 --- a/bin/zml-smi/prometheus/exposition.zig +++ b/bin/zml-smi/prometheus/exposition.zig @@ -48,14 +48,14 @@ pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo, host: *const try writer.writeAll("# HELP zml_device_info Device metadata\n"); try writer.writeAll("# TYPE zml_device_info gauge\n"); for (devices, 0..) |dev, i| { - inline for (@typeInfo(DeviceInfo).@"union".fields) |tp| { - const tag = @field(smi_info.device_info.Target, tp.name); + inline for (@typeInfo(DeviceInfo).@"union".field_names) |type_name| { + const tag = @field(smi_info.device_info.Target, type_name); switch (dev.*) { tag => |*db| { const val = db.front().*; - try writer.print("zml_device_info{{device=\"{d}\",type=\"{s}\"", .{ i, tp.name }); + try writer.print("zml_device_info{{device=\"{d}\",type=\"{s}\"", .{ i, type_name }); inline for (.{ .{ "name", "name" }, diff --git a/bin/zml-smi/tui/image_cache.zig b/bin/zml-smi/tui/image_cache.zig index cc841691..1aea4778 100644 --- a/bin/zml-smi/tui/image_cache.zig +++ b/bin/zml-smi/tui/image_cache.zig @@ -19,7 +19,7 @@ pub fn load( key: []const u8, data: []const u8, ) void { - const image = vx.loadImage(allocator, undefined, writer, .{ .mem = data }) catch return; + const image = vx.loadImage(allocator, writer, .{ .mem = data }) catch return; self.map.put(allocator, key, image) catch return; } diff --git a/bin/zml-smi/tui/print.zig b/bin/zml-smi/tui/print.zig index 7338d20e..238534cd 100644 --- a/bin/zml-smi/tui/print.zig +++ b/bin/zml-smi/tui/print.zig @@ -14,11 +14,12 @@ fn silentResize(vx: *vaxis.Vaxis, allocator: std.mem.Allocator, winsize: vaxis.W vx.screen_last = try vaxis.AllocatingScreen.init(allocator, winsize.cols, winsize.rows); } -pub fn run(allocator: std.mem.Allocator, io: std.Io, state: *data.SystemState) !void { - var tty = try vaxis.Tty.init(io); +pub fn run(allocator: std.mem.Allocator, io: std.Io, env_map: *std.process.Environ.Map, state: *data.SystemState) !void { + var tty_buffer: [4096]u8 = undefined; + var tty = try vaxis.Tty.init(io, &tty_buffer); defer tty.deinit(); - var vx = try vaxis.init(allocator, .{}); + var vx = try vaxis.init(io, allocator, env_map, .{}); defer { // Reset terminal modes without clearing screen content. vx.resetModes(tty.writer()) catch {}; @@ -27,12 +28,12 @@ pub fn run(allocator: std.mem.Allocator, io: std.Io, state: *data.SystemState) ! vx.deinit(allocator, &discarding.writer); } - const ws = try vaxis.Tty.getWinsize(tty.fd); + const ws = try tty.getWinsize(); try silentResize(&vx, allocator, ws); { const EventLoop = vaxis.Loop(vxfw.Event); - var loop: EventLoop = .{ .tty = &tty, .vaxis = &vx, .io = io, .queue = .{ .io = io } }; + var loop: EventLoop = .init(io, &tty, &vx); try loop.start(); defer loop.stop(); @@ -41,7 +42,7 @@ pub fn run(allocator: std.mem.Allocator, io: std.Io, state: *data.SystemState) ! const writer = tty.writer(); try writer.writeAll("\x1b7"); // DECSC: save cursor try writer.flush(); - try vx.queryTerminal(writer, io, 1 * std.time.ns_per_s); + try vx.queryTerminal(writer, .fromSeconds(1)); try writer.writeAll("\x1b8"); // DECRC: restore cursor try writer.flush(); } diff --git a/bin/zml-smi/tui/top.zig b/bin/zml-smi/tui/top.zig index 9a5b8624..74b3796b 100644 --- a/bin/zml-smi/tui/top.zig +++ b/bin/zml-smi/tui/top.zig @@ -151,8 +151,9 @@ const Model = struct { } }; -pub fn run(allocator: std.mem.Allocator, io: std.Io, state: *data.SystemState) !void { - var app = try vxfw.App.init(allocator, io); +pub fn run(allocator: std.mem.Allocator, io: std.Io, env_map: *std.process.Environ.Map, state: *data.SystemState) !void { + var tty_buffer: [4096]u8 = undefined; + var app = try vxfw.App.init(io, allocator, env_map, &tty_buffer); defer app.deinit(); var model: Model = .{ diff --git a/bin/zml-smi/utils/dynlib.zig b/bin/zml-smi/utils/dynlib.zig index 730b495d..4a478107 100644 --- a/bin/zml-smi/utils/dynlib.zig +++ b/bin/zml-smi/utils/dynlib.zig @@ -2,8 +2,8 @@ const std = @import("std"); pub fn lookupStruct(dynlib: *std.DynLib, comptime VTable: type) !VTable { var result: VTable = undefined; - inline for (std.meta.fields(VTable)) |field| { - @field(result, field.name) = dynlib.lookup(field.type, field.name) orelse return error.SymbolResolutionFailed; + inline for (comptime std.meta.fieldNames(VTable), comptime std.meta.fieldTypes(VTable)) |field_name, Field| { + @field(result, field_name) = dynlib.lookup(Field, field_name) orelse return error.SymbolResolutionFailed; } return result; diff --git a/bin/zml-smi/utils/sandbox.zig b/bin/zml-smi/utils/sandbox.zig index 1b53c9ea..d4e268bd 100644 --- a/bin/zml-smi/utils/sandbox.zig +++ b/bin/zml-smi/utils/sandbox.zig @@ -11,7 +11,7 @@ const init_array_section = switch (builtin.object_format) { else => "", }; -export const _ linksection(init_array_section) = &struct { +export const _ align(@alignOf(usize)) linksection(init_array_section) = &struct { fn call(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { _ = argc; _ = argv; diff --git a/kernels/common/args.zig b/kernels/common/args.zig index 6d89de34..84d57d49 100644 --- a/kernels/common/args.zig +++ b/kernels/common/args.zig @@ -1,6 +1,4 @@ pub fn NamedArgs(comptime Spec: type, comptime ValueT: type) type { - const in = @typeInfo(Spec).@"struct".fields; - comptime var names: [in.len][]const u8 = undefined; - for (in, 0..) |f, i| names[i] = f.name; - return @Struct(.auto, null, &names, &@splat(ValueT), &@splat(.{})); + const field_names = @typeInfo(Spec).@"struct".field_names; + return @Struct(.auto, null, field_names, &@splat(ValueT), &@splat(.{})); } diff --git a/kernels/common/control_flow.zig b/kernels/common/control_flow.zig index 252184f3..afd79e8f 100644 --- a/kernels/common/control_flow.zig +++ b/kernels/common/control_flow.zig @@ -13,7 +13,7 @@ pub fn tupleArity(comptime T: type, comptime what: []const u8) comptime_int { const info = @typeInfo(T); if (info != .@"struct" or !info.@"struct".is_tuple) @compileError(what ++ " must be a tuple literal like `.{ v1, v2 }`"); - return info.@"struct".fields.len; + return info.@"struct".field_names.len; } pub fn emitScfYield( @@ -27,13 +27,13 @@ pub fn emitScfYield( const info = @typeInfo(@TypeOf(values)); if (info != .@"struct" or !info.@"struct".is_tuple) @compileError(what ++ " expects a tuple literal"); - if (info.@"struct".fields.len != N) + if (info.@"struct".field_names.len != N) @compileError(what ++ ": yield arity must match the scope's declared arity"); var buf: [N]*const mlir.Value = undefined; - inline for (info.@"struct".fields, 0..) |f, i| { - if (f.type != ValueT) + inline for (info.@"struct".field_names, info.@"struct".field_types, 0..) |field_name, Field, i| { + if (Field != ValueT) @compileError(what ++ ": every tuple element must be a Value"); - buf[i] = @field(values, f.name).inner; + buf[i] = @field(values, field_name).inner; } _ = scf.yield(kernel.ctx, &buf, kernel.loc()).appendTo(block); } @@ -151,14 +151,14 @@ pub fn WhileScope(comptime BuilderT: type, comptime ValueT: type, comptime N: us const info = @typeInfo(@TypeOf(forwarded)); if (info != .@"struct" or !info.@"struct".is_tuple) @compileError("WhileScope.yieldBefore: forwarded must be a tuple literal"); - if (info.@"struct".fields.len != M) + if (info.@"struct".field_names.len != M) @compileError("WhileScope.yieldBefore: forwarded arity must match after_types arity"); const k = self.kernel; var buf: [M]*const mlir.Value = undefined; - inline for (info.@"struct".fields, 0..) |f, i| { - if (f.type != ValueT) + inline for (info.@"struct".field_names, info.@"struct".field_types, 0..) |field_name, Field, i| { + if (Field != ValueT) @compileError("WhileScope.yieldBefore: every forwarded element must be a Value"); - buf[i] = @field(forwarded, f.name).inner; + buf[i] = @field(forwarded, field_name).inner; } _ = scf.condition(k.ctx, cond.inner, &buf, k.loc()).appendTo(self.before_block); k.popBlock(); diff --git a/kernels/cuda_tile/builder.zig b/kernels/cuda_tile/builder.zig index d73742a1..6067ed4b 100644 --- a/kernels/cuda_tile/builder.zig +++ b/kernels/cuda_tile/builder.zig @@ -84,8 +84,8 @@ pub const Value = struct { pub fn isFloatElem(self: Value) bool { const et = self.elemType(); - inline for (std.meta.fields(mlir.FloatTypes)) |f| { - if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, f.name))) != null) return true; + inline for (comptime std.meta.fieldNames(mlir.FloatTypes)) |f_name| { + if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, f_name))) != null) return true; } return false; } @@ -509,13 +509,13 @@ pub const Builder = struct { pub fn declareArgsOpts(self: *Builder, spec: anytype, opts: Opts) !dsl.NamedArgs(@TypeOf(spec), Value) { const Spec = @TypeOf(spec); - const fields = @typeInfo(Spec).@"struct".fields; + const field_names = @typeInfo(Spec).@"struct".field_names; - var arg_specs: [fields.len]ArgSpec = undefined; - inline for (fields, 0..) |f, i| { - const raw = @field(spec, f.name); + var arg_specs: [field_names.len]ArgSpec = undefined; + inline for (0.., field_names) |i, field_name| { + const raw = @field(spec, field_name); const kind: ArgSpec.Kind = if (@TypeOf(raw) == ArgSpec.Kind) raw else blk: { - const variant = @typeInfo(@TypeOf(raw)).@"struct".fields[0].name; + const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; const tag = @field(std.meta.Tag(ArgSpec.Kind), variant); const inner = @field(raw, variant); break :blk switch (tag) { @@ -526,14 +526,14 @@ pub const Builder = struct { } }, }; }; - arg_specs[i] = .{ .name = f.name, .kind = kind }; + arg_specs[i] = .{ .name = field_name, .kind = kind }; } try self.declareArgsLowOpts(&arg_specs, opts); var named: dsl.NamedArgs(Spec, Value) = undefined; - inline for (fields, 0..) |f, i| { - @field(named, f.name) = self.arg(i); + inline for (0.., field_names) |i, field_name| { + @field(named, field_name) = self.arg(i); } return named; } @@ -643,12 +643,12 @@ pub const Builder = struct { const info = @typeInfo(T); if (info != .@"struct" or !info.@"struct".is_tuple) @compileError("Builder.yield expects a tuple literal like `.{ v1, v2 }`"); - const n = info.@"struct".fields.len; + const n = info.@"struct".field_names.len; const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.yield OOM"); - inline for (info.@"struct".fields, 0..) |f, i| { - if (f.type != Value) - @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(f.type)); - out[i] = @field(values, f.name); + inline for (info.@"struct".field_names, 0..) |f, i| { + if (@FieldType(T, f) != Value) + @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(@FieldType(T, f))); + out[i] = @field(values, f); } return out; } @@ -1633,7 +1633,7 @@ pub const Builder = struct { inits: anytype, ) ForScope(tupleArity(@TypeOf(inits), "openFor: inits")) { const N = comptime tupleArity(@TypeOf(inits), "openFor: inits"); - const fields = @typeInfo(@TypeOf(inits)).@"struct".fields; + const fields = @typeInfo(@TypeOf(inits)).@"struct".field_names; // Literals take the type of the first bound that is a Value; all // three must agree (`AllTypesMatch` on the op). @@ -1651,8 +1651,8 @@ pub const Builder = struct { block_locs[0] = self.loc(); var inits_inner: [N]*const mlir.Value = undefined; inline for (fields, 0..) |f, i| { - const raw = @field(inits, f.name); - const v: Value = if (f.type == Value) raw else self.lift(raw); + const raw = @field(inits, f); + const v: Value = if (@FieldType(@TypeOf(inits), f) == Value) raw else self.lift(raw); block_types[i + 1] = v.type_(); block_locs[i + 1] = self.loc(); inits_inner[i] = v.inner; @@ -1692,13 +1692,13 @@ pub const Builder = struct { result_types: anytype, ) IfScope(tupleArity(@TypeOf(result_types), "openIfElse: result_types")) { const N = comptime tupleArity(@TypeOf(result_types), "openIfElse: result_types"); - const fields = @typeInfo(@TypeOf(result_types)).@"struct".fields; + const fields = @typeInfo(@TypeOf(result_types)).@"struct".field_names; var types: [N]*const mlir.Type = undefined; inline for (fields, 0..) |f, i| { - if (f.type != *const mlir.Type) + if (@FieldType(@TypeOf(result_types), f) != *const mlir.Type) @compileError("openIfElse: every result_type must be *const mlir.Type (use b.scalarTy/tileTy)"); - types[i] = @field(result_types, f.name); + types[i] = @field(result_types, f); } const then_block = mlir.Block.init(&.{}, &.{}); @@ -1718,14 +1718,14 @@ pub const Builder = struct { /// inside an `if`, and close the body with `scope.yield`. pub fn openLoop(self: *Builder, inits: anytype) LoopScope(tupleArity(@TypeOf(inits), "openLoop: inits")) { const N = comptime tupleArity(@TypeOf(inits), "openLoop: inits"); - const fields = @typeInfo(@TypeOf(inits)).@"struct".fields; + const fields = @typeInfo(@TypeOf(inits)).@"struct".field_names; var block_types: [N]*const mlir.Type = undefined; var block_locs: [N]*const mlir.Location = undefined; var inits_inner: [N]*const mlir.Value = undefined; inline for (fields, 0..) |f, i| { - const raw = @field(inits, f.name); - const v: Value = if (f.type == Value) raw else self.lift(raw); + const raw = @field(inits, f); + const v: Value = if (@FieldType(@TypeOf(inits), f) == Value) raw else self.lift(raw); block_types[i] = v.type_(); block_locs[i] = self.loc(); inits_inner[i] = v.inner; @@ -1769,7 +1769,7 @@ pub const Builder = struct { try al.writer.print("{f}", .{self.module.operation()}); - return try self.allocator.dupeZ(u8, al.written()); + return try self.allocator.dupeSentinel(u8, al.written(), 0); } }; @@ -1799,12 +1799,12 @@ test "empty entry round-trips" { const ir = try b.finish(&.{}); defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.module @empty") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "entry @empty(") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "assume") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "div_by<16>") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.entry") == null); - try std.testing.expect(std.mem.indexOf(u8, ir, "return") != null); + try std.testing.expect(std.mem.find(u8, ir, "cuda_tile.module @empty") != null); + try std.testing.expect(std.mem.find(u8, ir, "entry @empty(") != null); + try std.testing.expect(std.mem.find(u8, ir, "assume") != null); + try std.testing.expect(std.mem.find(u8, ir, "div_by<16>") != null); + try std.testing.expect(std.mem.find(u8, ir, "cuda_tile.entry") == null); + try std.testing.expect(std.mem.find(u8, ir, "return") != null); try expectRoundTrip(ctx, ir); } @@ -1827,14 +1827,14 @@ test "add_one over pointer tiles matches the XLA milestone kernel" { const ir = try b.finish(&.{}); defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "iota : tile<128xi32>") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "reshape") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "broadcast") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "offset") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "load_ptr_tko weak") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "addf") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "store_ptr_tko weak") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "token = ") != null); + try std.testing.expect(std.mem.find(u8, ir, "iota : tile<128xi32>") != null); + try std.testing.expect(std.mem.find(u8, ir, "reshape") != null); + try std.testing.expect(std.mem.find(u8, ir, "broadcast") != null); + try std.testing.expect(std.mem.find(u8, ir, "offset") != null); + try std.testing.expect(std.mem.find(u8, ir, "load_ptr_tko weak") != null); + try std.testing.expect(std.mem.find(u8, ir, "addf") != null); + try std.testing.expect(std.mem.find(u8, ir, "store_ptr_tko weak") != null); + try std.testing.expect(std.mem.find(u8, ir, "token = ") != null); try expectRoundTrip(ctx, ir); } @@ -1866,13 +1866,13 @@ test "vector_add over partition views" { const ir = try b.finish(&.{}); defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "optimization_hints=") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "make_tensor_view") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "partition_view") != null or std.mem.indexOf(u8, ir, "addf") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "store_view_tko weak") != null); + try std.testing.expect(std.mem.find(u8, ir, "optimization_hints=") != null); + try std.testing.expect(std.mem.find(u8, ir, "make_tensor_view") != null); + try std.testing.expect(std.mem.find(u8, ir, "partition_view") != null or std.mem.find(u8, ir, "addf") != null); + try std.testing.expect(std.mem.find(u8, ir, "store_view_tko weak") != null); try expectRoundTrip(ctx, ir); } @@ -1907,10 +1907,10 @@ test "gemm with a for loop and mmaf" { const ir = try b.finish(&.{}); defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "for ") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "iter_values") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "mmaf") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "continue ") != null); + try std.testing.expect(std.mem.find(u8, ir, "for ") != null); + try std.testing.expect(std.mem.find(u8, ir, "iter_values") != null); + try std.testing.expect(std.mem.find(u8, ir, "mmaf") != null); + try std.testing.expect(std.mem.find(u8, ir, "continue ") != null); try expectRoundTrip(ctx, ir); } @@ -1997,14 +1997,14 @@ test "if, if-else, loop with break, reduce, scan, cast, select" { const ir = try b.finish(&.{}); defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "reduce ") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "scan ") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "ftoi") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "itof") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "select") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "loop ") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "break ") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "yield") != null); + try std.testing.expect(std.mem.find(u8, ir, "reduce ") != null); + try std.testing.expect(std.mem.find(u8, ir, "scan ") != null); + try std.testing.expect(std.mem.find(u8, ir, "ftoi") != null); + try std.testing.expect(std.mem.find(u8, ir, "itof") != null); + try std.testing.expect(std.mem.find(u8, ir, "select") != null); + try std.testing.expect(std.mem.find(u8, ir, "loop ") != null); + try std.testing.expect(std.mem.find(u8, ir, "break ") != null); + try std.testing.expect(std.mem.find(u8, ir, "yield") != null); try expectRoundTrip(ctx, ir); } @@ -2055,20 +2055,20 @@ test "atomics, print, globals, alloca, dynamic views, strided view, tokens" { const ir = try b.finish(&.{}); defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "global @lut") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "get_global @lut") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "tensor_view") != null); + try std.testing.expect(std.mem.find(u8, ir, "rounding") != null); try expectRoundTrip(ctx, ir); } diff --git a/kernels/cuda_tile/control_flow.zig b/kernels/cuda_tile/control_flow.zig index 2f971f1d..f51e8571 100644 --- a/kernels/cuda_tile/control_flow.zig +++ b/kernels/cuda_tile/control_flow.zig @@ -13,13 +13,13 @@ fn collect(comptime ValueT: type, comptime N: usize, values: anytype, comptime w const info = @typeInfo(@TypeOf(values)); if (info != .@"struct" or !info.@"struct".is_tuple) @compileError(what ++ " expects a tuple literal"); - if (info.@"struct".fields.len != N) + if (info.@"struct".field_names.len != N) @compileError(what ++ ": arity must match the scope's declared arity"); var buf: [N]*const mlir.Value = undefined; - inline for (info.@"struct".fields, 0..) |f, i| { - if (f.type != ValueT) + inline for (info.@"struct".field_names, 0..) |f, i| { + if (@FieldType(@TypeOf(values), f) != ValueT) @compileError(what ++ ": every tuple element must be a Value"); - buf[i] = @field(values, f.name).inner; + buf[i] = @field(values, f).inner; } return buf; } diff --git a/kernels/cuda_tile/dtype.zig b/kernels/cuda_tile/dtype.zig index 4552e662..a9193839 100644 --- a/kernels/cuda_tile/dtype.zig +++ b/kernels/cuda_tile/dtype.zig @@ -73,8 +73,8 @@ pub fn intBitwidth(dt: DType) u32 { } pub fn mlirElemToDType(ctx: *mlir.Context, elem: *const mlir.Type) DType { - inline for (std.meta.fields(DType)) |f| { - const dt = @field(DType, f.name); + inline for (comptime std.meta.fieldNames(DType)) |f_name| { + const dt = @field(DType, f_name); if (elem.eql(dt.toMlir(ctx))) return dt; } @panic("element type not a recognized CUDA Tile IR DType"); diff --git a/kernels/cute/builder.zig b/kernels/cute/builder.zig index 9c777491..77f5f824 100644 --- a/kernels/cute/builder.zig +++ b/kernels/cute/builder.zig @@ -97,8 +97,8 @@ pub const Value = struct { pub fn isFloat(self: Value) bool { const t = self.type_(); - inline for (std.meta.fields(mlir.FloatTypes)) |f| { - if (t.isA(mlir.FloatType(@field(mlir.FloatTypes, f.name))) != null) return true; + inline for (@typeInfo(mlir.FloatTypes).@"enum".field_names) |f| { + if (t.isA(mlir.FloatType(@field(mlir.FloatTypes, f))) != null) return true; } return false; } @@ -671,14 +671,14 @@ pub const Builder = struct { /// yields a `Tensor`, `.{ .p = .{ .ptr = .f32 } }` a pointer `Value`. pub fn declareArgs(self: *Builder, spec: anytype) FinishError!ArgsOf(@TypeOf(spec)) { const Spec = @TypeOf(spec); - const fields = @typeInfo(Spec).@"struct".fields; + const fields = @typeInfo(Spec).@"struct".field_names; const arg_specs = try self.arena.allocator().alloc(ArgSpec, fields.len); inline for (fields, 0..) |f, i| { - const raw = @field(spec, f.name); - const variant = @typeInfo(@TypeOf(raw)).@"struct".fields[0].name; + const raw = @field(spec, f); + const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; const inner = @field(raw, variant); - arg_specs[i] = .{ .name = f.name, .kind = switch (@field(std.meta.Tag(ArgSpec.Kind), variant)) { + arg_specs[i] = .{ .name = f, .kind = switch (@field(std.meta.Tag(ArgSpec.Kind), variant)) { .ptr => .{ .ptr = inner }, .tensor => .{ .tensor = .{ .dtype = inner.dtype, @@ -700,7 +700,7 @@ pub const Builder = struct { var named: ArgsOf(Spec) = undefined; inline for (fields, 0..) |f, i| { - @field(named, f.name) = switch (@FieldType(ArgsOf(Spec), f.name)) { + @field(named, f) = switch (@FieldType(ArgsOf(Spec), f)) { Tensor => self.tensors[i].?, else => self.arg(i), }; @@ -709,12 +709,12 @@ pub const Builder = struct { } fn ArgsOf(comptime Spec: type) type { - const in = @typeInfo(Spec).@"struct".fields; + const in = @typeInfo(Spec).@"struct".field_names; comptime var names: [in.len][]const u8 = undefined; comptime var types: [in.len]type = undefined; inline for (in, 0..) |f, i| { - names[i] = f.name; - types[i] = if (@hasField(f.type, "tensor")) Tensor else Value; + names[i] = f; + types[i] = if (@hasField(@FieldType(Spec, f), "tensor")) Tensor else Value; } return @Struct(.auto, null, &names, &types, &@splat(.{})); } @@ -1025,7 +1025,7 @@ pub const Builder = struct { rest = payload[end + 1 ..]; } try output.writer.writeAll(rest); - return try self.allocator.dupeZ(u8, output.written()); + return try self.allocator.dupeSentinel(u8, output.written(), 0); } // ==================== types ==================== @@ -1134,9 +1134,9 @@ pub const Builder = struct { .comptime_int, .int => writer.print("{d}", .{value}) catch @panic("OOM"), .@"struct" => |info| { writer.writeByte('(') catch @panic("OOM"); - inline for (info.fields, 0..) |field, i| { + inline for (info.field_names, 0..) |field, i| { if (i != 0) writer.writeByte(',') catch @panic("OOM"); - self.writeAlgebra(writer, @field(value, field.name)); + self.writeAlgebra(writer, @field(value, field)); } writer.writeByte(')') catch @panic("OOM"); }, @@ -1157,10 +1157,10 @@ pub const Builder = struct { pub fn basis(self: *Builder, numerator: i64, denominator: i64, modes: anytype) ScaledBasis { _ = self; if (denominator <= 0) @panic("CuTe scaled-basis denominator must be positive"); - const fields = @typeInfo(@TypeOf(modes)).@"struct".fields; + const fields = @typeInfo(@TypeOf(modes)).@"struct".field_names; if (fields.len == 0 or fields.len > MAX_RANK) @panic("CuTe scaled basis needs 1..MAX_RANK modes"); var result: ScaledBasis = .{ .numerator = numerator, .denominator = denominator, .mode_count = fields.len }; - inline for (fields, 0..) |field, i| result.modes[i] = @intCast(@field(modes, field.name)); + inline for (fields, 0..) |field, i| result.modes[i] = @intCast(@field(modes, field)); return result; } @@ -1349,13 +1349,13 @@ pub const Builder = struct { switch (@typeInfo(T)) { .comptime_int, .int => writer.print("{d}", .{coord}) catch @panic("OOM"), .@"struct" => |info| { - if (info.fields.len == 0) @compileError("makeCoord: empty tuple"); - if (info.fields.len > 1) writer.writeByte('(') catch @panic("OOM"); - inline for (info.fields, 0..) |field, i| { + if (info.field_names.len == 0) @compileError("makeCoord: empty tuple"); + if (info.field_names.len > 1) writer.writeByte('(') catch @panic("OOM"); + inline for (info.field_names, 0..) |field, i| { if (i != 0) writer.writeByte(',') catch @panic("OOM"); - self.writeCoord(writer, dyn, @field(coord, field.name)); + self.writeCoord(writer, dyn, @field(coord, field)); } - if (info.fields.len > 1) writer.writeByte(')') catch @panic("OOM"); + if (info.field_names.len > 1) writer.writeByte(')') catch @panic("OOM"); }, else => @compileError("makeCoord: unsupported coordinate leaf " ++ @typeName(T)), } @@ -3010,8 +3010,8 @@ pub const Builder = struct { pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(tupleArity(@TypeOf(result_types), "openIfElse: result_types")) { const N = comptime tupleArity(@TypeOf(result_types), "openIfElse: result_types"); var types: [N]*const mlir.Type = undefined; - inline for (@typeInfo(@TypeOf(result_types)).@"struct".fields, 0..) |f, i| { - types[i] = @field(result_types, f.name); + inline for (@typeInfo(@TypeOf(result_types)).@"struct".field_names, 0..) |f, i| { + types[i] = @field(result_types, f); } const then_block = mlir.Block.init(&.{}, &.{}); const else_block = mlir.Block.init(&.{}, &.{}); @@ -3031,8 +3031,8 @@ pub const Builder = struct { block_types[0] = lb.type_(); block_locs[0] = self.loc(); var inits_inner: [N]*const mlir.Value = undefined; - inline for (@typeInfo(@TypeOf(inits)).@"struct".fields, 0..) |f, i| { - const v = self.lift(@field(inits, f.name)); + inline for (@typeInfo(@TypeOf(inits)).@"struct".field_names, 0..) |f, i| { + const v = self.lift(@field(inits, f)); block_types[i + 1] = v.type_(); block_locs[i + 1] = self.loc(); inits_inner[i] = v.inner; @@ -3071,8 +3071,8 @@ pub const Builder = struct { var init_types: [N]*const mlir.Type = undefined; var init_locs: [N]*const mlir.Location = undefined; var inits_inner: [N]*const mlir.Value = undefined; - inline for (@typeInfo(@TypeOf(inits)).@"struct".fields, 0..) |f, i| { - const v = self.lift(@field(inits, f.name)); + inline for (@typeInfo(@TypeOf(inits)).@"struct".field_names, 0..) |f, i| { + const v = self.lift(@field(inits, f)); init_types[i] = v.type_(); init_locs[i] = self.loc(); inits_inner[i] = v.inner; @@ -3080,8 +3080,8 @@ pub const Builder = struct { var result_types: [M]*const mlir.Type = undefined; var result_locs: [M]*const mlir.Location = undefined; - inline for (@typeInfo(@TypeOf(after_types)).@"struct".fields, 0..) |f, i| { - result_types[i] = @field(after_types, f.name); + inline for (@typeInfo(@TypeOf(after_types)).@"struct".field_names, 0..) |f, i| { + result_types[i] = @field(after_types, f); result_locs[i] = self.loc(); } diff --git a/kernels/cute/dtype.zig b/kernels/cute/dtype.zig index 6dc9cc14..ae98cb08 100644 --- a/kernels/cute/dtype.zig +++ b/kernels/cute/dtype.zig @@ -55,8 +55,8 @@ pub fn dtypeBitwidth(dt: DType) usize { } pub fn mlirToDType(ctx: *mlir.Context, t: *const mlir.Type) DType { - inline for (std.meta.fields(DType)) |f| { - const dt = @field(DType, f.name); + inline for (@typeInfo(DType).@"enum".field_names) |f| { + const dt = @field(DType, f); if (t.eql(dt.toMlir(ctx))) return dt; } std.debug.panic("type {f} is not a CuTe DType", .{t}); diff --git a/kernels/fly/builder.zig b/kernels/fly/builder.zig index c560d2df..eee124ab 100644 --- a/kernels/fly/builder.zig +++ b/kernels/fly/builder.zig @@ -736,22 +736,22 @@ pub const Builder = struct { /// `.{ .tensor = .{ .dtype = ..., .dims = &.{ ... } } }`. pub fn declareArgs(self: *Builder, spec: anytype) !dsl.NamedArgs(@TypeOf(spec), Value) { const Spec = @TypeOf(spec); - const fields = @typeInfo(Spec).@"struct".fields; + const fields = @typeInfo(Spec).@"struct".field_names; var specs: [fields.len]ArgSpec = undefined; inline for (fields, 0..) |f, i| { - const raw = @field(spec, f.name); - const variant = @typeInfo(@TypeOf(raw)).@"struct".fields[0].name; + const raw = @field(spec, f); + const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; const inner = @field(raw, variant); specs[i] = if (comptime std.mem.eql(u8, variant, "ptr")) - .{ .name = f.name, .dtype = inner } + .{ .name = f, .dtype = inner } else if (comptime std.mem.eql(u8, variant, "tensor")) - .{ .name = f.name, .dtype = inner.dtype, .dims = inner.dims, .space = if (@hasField(@TypeOf(inner), "space")) inner.space else .global } + .{ .name = f, .dtype = inner.dtype, .dims = inner.dims, .space = if (@hasField(@TypeOf(inner), "space")) inner.space else .global } else - @compileError("fly.declareArgs: argument `" ++ f.name ++ "` must be `.{ .ptr = dtype }` or `.{ .tensor = .{ .dtype, .dims } }`"); + @compileError("fly.declareArgs: argument `" ++ f ++ "` must be `.{ .ptr = dtype }` or `.{ .tensor = .{ .dtype, .dims } }`"); } try self.declareArgsLow(&specs); var named: dsl.NamedArgs(Spec, Value) = undefined; - inline for (fields, 0..) |f, i| @field(named, f.name) = self.arg(i); + inline for (fields, 0..) |f, i| @field(named, f) = self.arg(i); return named; } @@ -969,8 +969,8 @@ pub const Builder = struct { .comptime_int, .int => return .static(@intCast(v)), .@"struct" => |s| { if (!s.is_tuple) @compileError("fly.intTuple: expected an integer, null, Value, IntTuple or tuple, got " ++ @typeName(T)); - const elems = self.alloc(IntTuple, s.fields.len); - inline for (s.fields, 0..) |f, i| elems[i] = self.intTupleSpec(operands, @field(v, f.name)); + const elems = self.alloc(IntTuple, s.field_names.len); + inline for (s.field_names, 0..) |f, i| elems[i] = self.intTupleSpec(operands, @field(v, f)); return .{ .tup = elems }; }, else => @compileError("fly.intTuple: expected an integer, null, Value, IntTuple or tuple, got " ++ @typeName(T)), @@ -1367,7 +1367,7 @@ pub const Builder = struct { inits: anytype, ) ForScope(tupleArity(@TypeOf(inits), "openFor: inits")) { const N = comptime tupleArity(@TypeOf(inits), "openFor: inits"); - const fields = @typeInfo(@TypeOf(inits)).@"struct".fields; + const fields = @typeInfo(@TypeOf(inits)).@"struct".field_names; const lb_v: Value = if (@TypeOf(lower) == Value) lower else self.constIndex(lower); const ub_v: Value = if (@TypeOf(upper) == Value) upper else self.constLike(upper, lb_v); @@ -1379,8 +1379,8 @@ pub const Builder = struct { block_locs[0] = self.loc(); var inits_inner: [N]*const mlir.Value = undefined; inline for (fields, 0..) |f, i| { - if (f.type != Value) @compileError("openFor: every init must be a Value"); - const v: Value = @field(inits, f.name); + if (@TypeOf(@field(inits, f)) != Value) @compileError("openFor: every init must be a Value"); + const v: Value = @field(inits, f); block_types[i + 1] = v.type_(); block_locs[i + 1] = self.loc(); inits_inner[i] = v.inner; @@ -1423,11 +1423,11 @@ pub const Builder = struct { pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(tupleArity(@TypeOf(result_types), "openIfElse: result_types")) { const N = comptime tupleArity(@TypeOf(result_types), "openIfElse: result_types"); - const fields = @typeInfo(@TypeOf(result_types)).@"struct".fields; + const fields = @typeInfo(@TypeOf(result_types)).@"struct".field_names; var types: [N]*const mlir.Type = undefined; inline for (fields, 0..) |f, i| { - if (f.type != *const mlir.Type) @compileError("openIfElse: every result_type must be *const mlir.Type"); - types[i] = @field(result_types, f.name); + if (@TypeOf(@field(result_types, f)) != *const mlir.Type) @compileError("openIfElse: every result_type must be *const mlir.Type"); + types[i] = @field(result_types, f); } const then_block = mlir.Block.init(&.{}, &.{}); const else_block = mlir.Block.init(&.{}, &.{}); @@ -1445,16 +1445,16 @@ pub const Builder = struct { ) { const N = comptime tupleArity(@TypeOf(inits), "openWhile: inits"); const M = comptime tupleArity(@TypeOf(after_types), "openWhile: after_types"); - const init_fields = @typeInfo(@TypeOf(inits)).@"struct".fields; - const type_fields = @typeInfo(@TypeOf(after_types)).@"struct".fields; + const init_fields = @typeInfo(@TypeOf(inits)).@"struct".field_names; + const type_fields = @typeInfo(@TypeOf(after_types)).@"struct".field_names; var before_types: [N]*const mlir.Type = undefined; var before_locs: [N]*const mlir.Location = undefined; var inits_inner: [N]*const mlir.Value = undefined; inline for (init_fields, 0..) |f, i| { - if (f.type != Value) + if (@TypeOf(@field(inits, f)) != Value) @compileError("openWhile: every init must be a Value"); - const v: Value = @field(inits, f.name); + const v: Value = @field(inits, f); before_types[i] = v.type_(); before_locs[i] = self.loc(); inits_inner[i] = v.inner; @@ -1464,9 +1464,9 @@ pub const Builder = struct { var after_tys: [M]*const mlir.Type = undefined; var after_locs: [M]*const mlir.Location = undefined; inline for (type_fields, 0..) |f, i| { - if (f.type != *const mlir.Type) + if (@TypeOf(@field(after_types, f)) != *const mlir.Type) @compileError("openWhile: every after_type must be *const mlir.Type"); - after_tys[i] = @field(after_types, f.name); + after_tys[i] = @field(after_types, f); after_locs[i] = self.loc(); } const after_block = mlir.Block.init(&after_tys, &after_locs); @@ -1500,7 +1500,7 @@ pub const Builder = struct { var al: std.Io.Writer.Allocating = .init(self.allocator); defer al.deinit(); try al.writer.print("{f}", .{self.module.operation().fmt(.{ .debug_info = false })}); - return try self.allocator.dupeZ(u8, al.written()); + return try self.allocator.dupeSentinel(u8, al.written(), 0); } }; @@ -1784,8 +1784,7 @@ test "tiledMma builds, verifies and re-parses" { test "MmaFlavor.atomThreads matches the dialect" { const ctx = try testContext(); defer ctx.deinit(); - inline for (std.meta.fields(MmaFlavor)) |field| { - const flavor: MmaFlavor = @enumFromInt(field.value); + inline for (comptime std.enums.values(MmaFlavor)) |flavor| { const atom = try fly.types.MmaAtomType.get(ctx, .{ .mmaOp = flavor.atomType(ctx) }); // `mma_atom.thr_layout` is the wavefront an atom occupies; the launch // geometry is derived from it, so a wrong table silently corrupts C. diff --git a/kernels/fly/dtype.zig b/kernels/fly/dtype.zig index 6962a898..ecc9f070 100644 --- a/kernels/fly/dtype.zig +++ b/kernels/fly/dtype.zig @@ -65,8 +65,7 @@ pub const DType = enum { /// Inverse of `toMlir`. pub fn fromMlir(ctx: *mlir.Context, ty: *const mlir.Type) ?DType { - inline for (std.meta.fields(DType)) |f| { - const dt: DType = @enumFromInt(f.value); + inline for (comptime std.enums.values(DType)) |dt| { if (ty.eql(dt.toMlir(ctx))) return dt; } return null; @@ -77,8 +76,7 @@ test "dtype types round-trip" { defer registry.deinit(); const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); defer ctx.deinit(); - inline for (std.meta.fields(DType)) |f| { - const dt: DType = @enumFromInt(f.value); + inline for (comptime std.enums.values(DType)) |dt| { try std.testing.expectEqual(dt, fromMlir(ctx, dt.toMlir(ctx)).?); } try std.testing.expectEqual(DType.i8, DType.i1.storageElem()); diff --git a/kernels/fly/layout.zig b/kernels/fly/layout.zig index 85495ed4..e70e0e00 100644 --- a/kernels/fly/layout.zig +++ b/kernels/fly/layout.zig @@ -189,8 +189,8 @@ fn itImpl(comptime v: anytype) IntTuple { .comptime_int, .int => return IntTuple.static(v), .@"struct" => |s| { if (!s.is_tuple) @compileError("fly.it: expected an integer, null, IntTuple or tuple, got " ++ @typeName(T)); - var elems: [s.fields.len]IntTuple = undefined; - for (s.fields, 0..) |f, i| elems[i] = itImpl(@field(v, f.name)); + var elems: [s.field_names.len]IntTuple = undefined; + for (s.field_names, 0..) |f, i| elems[i] = itImpl(@field(v, f)); const frozen = elems; return .{ .tup = &frozen }; }, @@ -413,8 +413,8 @@ fn tileImpl(comptime v: anytype) Tile { .comptime_int, .int => return .{ .leaf = .{ .s = v } }, .@"struct" => |s| { if (!s.is_tuple) @compileError("fly.tile: expected an integer, null, Layout, Tile or tuple, got " ++ @typeName(T)); - var elems: [s.fields.len]Tile = undefined; - for (s.fields, 0..) |f, i| elems[i] = tileImpl(@field(v, f.name)); + var elems: [s.field_names.len]Tile = undefined; + for (s.field_names, 0..) |f, i| elems[i] = tileImpl(@field(v, f)); const frozen = elems; return .{ .modes = &frozen }; }, diff --git a/kernels/mosaic_tpu/builder.zig b/kernels/mosaic_tpu/builder.zig index 157648df..77bc8f29 100644 --- a/kernels/mosaic_tpu/builder.zig +++ b/kernels/mosaic_tpu/builder.zig @@ -106,8 +106,8 @@ pub const Value = struct { /// True if the element type is a floating-point type (f16/bf16/f32/f64/fp8/…). pub fn isFloatElem(self: Value) bool { const et = self.elemType(); - inline for (std.meta.fields(mlir.FloatTypes)) |f| { - if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, f.name))) != null) return true; + inline for (comptime std.meta.fieldNames(mlir.FloatTypes)) |field_name| { + if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, field_name))) != null) return true; } return false; } @@ -600,13 +600,13 @@ pub const Builder = struct { opts: Opts, ) !dsl.NamedArgs(@TypeOf(spec), Value) { const Spec = @TypeOf(spec); - const fields = @typeInfo(Spec).@"struct".fields; + const field_names = @typeInfo(Spec).@"struct".field_names; - var arg_specs: [fields.len]ArgSpec = undefined; - inline for (fields, 0..) |f, i| { - const raw = @field(spec, f.name); + var arg_specs: [field_names.len]ArgSpec = undefined; + inline for (field_names, 0..) |field_name, i| { + const raw = @field(spec, field_name); const kind: ArgSpec.Kind = if (@TypeOf(raw) == ArgSpec.Kind) raw else blk: { - const variant = @typeInfo(@TypeOf(raw)).@"struct".fields[0].name; + const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; const tag = @field(std.meta.Tag(ArgSpec.Kind), variant); const inner = @field(raw, variant); break :blk switch (tag) { @@ -629,14 +629,14 @@ pub const Builder = struct { .sem => .{ .sem = inner }, }; }; - arg_specs[i] = .{ .name = f.name, .kind = kind }; + arg_specs[i] = .{ .name = field_name, .kind = kind }; } try self.declareArgsLowOpts(&arg_specs, result_types, opts); var named: dsl.NamedArgs(Spec, Value) = undefined; - inline for (fields, 0..) |f, i| { - @field(named, f.name) = self.arg(i); + inline for (field_names, 0..) |field_name, i| { + @field(named, field_name) = self.arg(i); } return named; } @@ -896,12 +896,12 @@ pub const Builder = struct { const info = @typeInfo(T); if (info != .@"struct" or !info.@"struct".is_tuple) @compileError("Builder.yield expects a tuple literal like `.{ v1, v2 }`"); - const n = info.@"struct".fields.len; + const n = info.@"struct".field_names.len; const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.yield OOM"); - inline for (info.@"struct".fields, 0..) |f, i| { - if (f.type != Value) - @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(f.type)); - out[i] = @field(values, f.name); + inline for (info.@"struct".field_names, info.@"struct".field_types, 0..) |field_name, Field, i| { + if (Field != Value) + @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(Field)); + out[i] = @field(values, field_name); } return out; } @@ -2048,7 +2048,7 @@ pub const Builder = struct { inits: anytype, ) dsl.ForScope(Builder, Value, dsl.tupleArity(@TypeOf(inits), "openFor: inits")) { const N = comptime dsl.tupleArity(@TypeOf(inits), "openFor: inits"); - const fields = @typeInfo(@TypeOf(inits)).@"struct".fields; + const info = @typeInfo(@TypeOf(inits)).@"struct"; // scf.for bounds default to `index`. const lb_v: Value = if (@TypeOf(lower) == Value) lower else self.cIndex(@intCast(lower)); @@ -2060,9 +2060,9 @@ pub const Builder = struct { block_types[0] = lb_v.type_(); block_locs[0] = self.loc(); var inits_inner: [N]*const mlir.Value = undefined; - inline for (fields, 0..) |f, i| { - const raw = @field(inits, f.name); - const v: Value = if (f.type == Value) raw else self.lift(raw); + inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { + const raw = @field(inits, field_name); + const v: Value = if (Field == Value) raw else self.lift(raw); block_types[i + 1] = v.type_(); block_locs[i + 1] = self.loc(); inits_inner[i] = v.inner; @@ -2102,13 +2102,13 @@ pub const Builder = struct { result_types: anytype, ) dsl.IfScope(Builder, Value, dsl.tupleArity(@TypeOf(result_types), "openIfElse: result_types")) { const N = comptime dsl.tupleArity(@TypeOf(result_types), "openIfElse: result_types"); - const fields = @typeInfo(@TypeOf(result_types)).@"struct".fields; + const info = @typeInfo(@TypeOf(result_types)).@"struct"; var types: [N]*const mlir.Type = undefined; - inline for (fields, 0..) |f, i| { - if (f.type != *const mlir.Type) + inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { + if (Field != *const mlir.Type) @compileError("openIfElse: every result_type must be *const mlir.Type (use k.scalarTy/vectorTy)"); - types[i] = @field(result_types, f.name); + types[i] = @field(result_types, field_name); } const then_block = mlir.Block.init(&.{}, &.{}); @@ -2136,16 +2136,16 @@ pub const Builder = struct { ) { const N = comptime dsl.tupleArity(@TypeOf(inits), "openWhile: inits"); const M = comptime dsl.tupleArity(@TypeOf(after_types), "openWhile: after_types"); - const init_fields = @typeInfo(@TypeOf(inits)).@"struct".fields; - const type_fields = @typeInfo(@TypeOf(after_types)).@"struct".fields; + const init_info = @typeInfo(@TypeOf(inits)).@"struct"; + const type_info = @typeInfo(@TypeOf(after_types)).@"struct"; var before_types: [N]*const mlir.Type = undefined; var before_locs: [N]*const mlir.Location = undefined; var inits_inner: [N]*const mlir.Value = undefined; - inline for (init_fields, 0..) |f, i| { - if (f.type != Value) + inline for (init_info.field_names, init_info.field_types, 0..) |field_name, Field, i| { + if (Field != Value) @compileError("openWhile: every init must be a Value"); - const v: Value = @field(inits, f.name); + const v: Value = @field(inits, field_name); before_types[i] = v.type_(); before_locs[i] = self.loc(); inits_inner[i] = v.inner; @@ -2154,10 +2154,10 @@ pub const Builder = struct { var after_tys: [M]*const mlir.Type = undefined; var after_locs: [M]*const mlir.Location = undefined; - inline for (type_fields, 0..) |f, i| { - if (f.type != *const mlir.Type) + inline for (type_info.field_names, type_info.field_types, 0..) |field_name, Field, i| { + if (Field != *const mlir.Type) @compileError("openWhile: every after_type must be *const mlir.Type"); - after_tys[i] = @field(after_types, f.name); + after_tys[i] = @field(after_types, field_name); after_locs[i] = self.loc(); } const after_block = mlir.Block.init(&after_tys, &after_locs); @@ -2223,7 +2223,7 @@ pub const Builder = struct { .debug_info = false, })}); - return try self.allocator.dupeZ(u8, al.written()); + return try self.allocator.dupeSentinel(u8, al.written(), 0); } /// Emit `transform_N` stub funcs — one per non-skipped i/o ref-arg. diff --git a/kernels/mosaic_tpu/dtype.zig b/kernels/mosaic_tpu/dtype.zig index d22370f6..549c69da 100644 --- a/kernels/mosaic_tpu/dtype.zig +++ b/kernels/mosaic_tpu/dtype.zig @@ -60,8 +60,8 @@ pub fn intBitwidth(dt: DType) u32 { } pub fn mlirElemToDType(ctx: *mlir.Context, elem: *const mlir.Type) DType { - inline for (std.meta.fields(DType)) |f| { - const dt = @field(DType, f.name); + inline for (comptime std.meta.fieldNames(DType)) |field_name| { + const dt = @field(DType, field_name); if (elem.eql(dt.toMlir(ctx))) return dt; } @panic("element type not a recognized Mosaic-TPU DType"); diff --git a/kernels/triton/builder.zig b/kernels/triton/builder.zig index 702fba0d..d585e8c1 100644 --- a/kernels/triton/builder.zig +++ b/kernels/triton/builder.zig @@ -66,8 +66,8 @@ pub const Value = struct { pub fn isFloatElem(self: Value) bool { const et = self.elemType(); - inline for (std.meta.fields(mlir.FloatTypes)) |f| { - if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, f.name))) != null) return true; + inline for (comptime std.meta.fieldNames(mlir.FloatTypes)) |field_name| { + if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, field_name))) != null) return true; } return false; } @@ -391,25 +391,25 @@ pub const ReturnIfScope = struct { pub fn yieldReturn(self: *ReturnIfScope, values: anytype) void { const k = self.kernel; - const fields = @typeInfo(@TypeOf(values)).@"struct".fields; + const info = @typeInfo(@TypeOf(values)).@"struct"; if (self.inline_return) { - var ret_operands: [fields.len]*const mlir.Value = undefined; - inline for (fields, 0..) |f, i| { - if (f.type != Value) + var ret_operands: [info.field_names.len]*const mlir.Value = undefined; + inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { + if (Field != Value) @compileError("ReturnIfScope.yieldReturn: every value must be a Value"); - ret_operands[i] = @field(values, f.name).inner; + ret_operands[i] = @field(values, field_name).inner; } _ = ttir.return_(k.ctx, &ret_operands, k.loc()).appendTo(self.ret_block); } else { if (k.exit_block == null) { k.exit_block = mlir.Block.init(&.{}, &.{}); - var ret_operands: [fields.len]*const mlir.Value = undefined; - inline for (fields, 0..) |f, i| { - if (f.type != Value) + var ret_operands: [info.field_names.len]*const mlir.Value = undefined; + inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { + if (Field != Value) @compileError("ReturnIfScope.yieldReturn: every value must be a Value"); - ret_operands[i] = @field(values, f.name).inner; + ret_operands[i] = @field(values, field_name).inner; } _ = ttir.return_(k.ctx, &ret_operands, k.loc()).appendTo(k.exit_block.?); } @@ -533,13 +533,13 @@ pub const Builder = struct { opts: Opts, ) !dsl.NamedArgs(@TypeOf(spec), Value) { const Spec = @TypeOf(spec); - const fields = @typeInfo(Spec).@"struct".fields; + const field_names = @typeInfo(Spec).@"struct".field_names; - var arg_specs: [fields.len]ArgSpec = undefined; - inline for (fields, 0..) |f, i| { - const raw = @field(spec, f.name); + var arg_specs: [field_names.len]ArgSpec = undefined; + inline for (field_names, 0..) |field_name, i| { + const raw = @field(spec, field_name); const kind: ArgSpec.Kind = if (@TypeOf(raw) == ArgSpec.Kind) raw else blk: { - const variant = @typeInfo(@TypeOf(raw)).@"struct".fields[0].name; + const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; const tag = @field(std.meta.Tag(ArgSpec.Kind), variant); const inner = @field(raw, variant); break :blk switch (tag) { @@ -557,14 +557,14 @@ pub const Builder = struct { } }, }; }; - arg_specs[i] = .{ .name = f.name, .kind = kind }; + arg_specs[i] = .{ .name = field_name, .kind = kind }; } try self.declareArgsLowOpts(&arg_specs, result_types, opts); var named: dsl.NamedArgs(Spec, Value) = undefined; - inline for (fields, 0..) |f, i| { - @field(named, f.name) = self.arg(i); + inline for (field_names, 0..) |field_name, i| { + @field(named, field_name) = self.arg(i); } return named; } @@ -679,12 +679,12 @@ pub const Builder = struct { const info = @typeInfo(T); if (info != .@"struct" or !info.@"struct".is_tuple) @compileError("Builder.yield expects a tuple literal like `.{ v1, v2 }`"); - const n = info.@"struct".fields.len; + const n = info.@"struct".field_names.len; const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.yield OOM"); - inline for (info.@"struct".fields, 0..) |f, i| { - if (f.type != Value) - @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(f.type)); - out[i] = @field(values, f.name); + inline for (info.@"struct".field_names, info.@"struct".field_types, 0..) |field_name, Field, i| { + if (Field != Value) + @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(Field)); + out[i] = @field(values, field_name); } return out; } @@ -898,8 +898,8 @@ pub const Builder = struct { } fn mlirElemToDType(self: *const Builder, elem: *const mlir.Type) DType { - inline for (std.meta.fields(DType)) |f| { - const dt = @field(DType, f.name); + inline for (comptime std.meta.fieldNames(DType)) |field_name| { + const dt = @field(DType, field_name); if (elem.eql(dt.toMlir(self.ctx))) return dt; } @panic("element type not a recognized DSL DType"); @@ -2012,7 +2012,7 @@ pub const Builder = struct { inits: anytype, ) ForScope(tupleArity(@TypeOf(inits), "openFor: inits")) { const N = comptime tupleArity(@TypeOf(inits), "openFor: inits"); - const fields = @typeInfo(@TypeOf(inits)).@"struct".fields; + const info = @typeInfo(@TypeOf(inits)).@"struct"; const lb_v: Value = if (@TypeOf(lower) == Value) lower else self.lift(lower); const ub_v: Value = if (@TypeOf(upper) == Value) upper else self.constMatching(upper, lb_v.type_()); @@ -2023,9 +2023,9 @@ pub const Builder = struct { block_types[0] = lb_v.type_(); block_locs[0] = self.loc(); var inits_inner: [N]*const mlir.Value = undefined; - inline for (fields, 0..) |f, i| { - const raw = @field(inits, f.name); - const v: Value = if (f.type == Value) raw else self.lift(raw); + inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { + const raw = @field(inits, field_name); + const v: Value = if (Field == Value) raw else self.lift(raw); block_types[i + 1] = v.type_(); block_locs[i + 1] = self.loc(); inits_inner[i] = v.inner; @@ -2058,12 +2058,12 @@ pub const Builder = struct { const region = current.parentRegion(); const cont_block = mlir.Block.init(&.{}, &.{}); - const fields = @typeInfo(@TypeOf(values)).@"struct".fields; - var ret_operands: [fields.len]*const mlir.Value = undefined; - inline for (fields, 0..) |f, i| { - if (f.type != Value) + const info = @typeInfo(@TypeOf(values)).@"struct"; + var ret_operands: [info.field_names.len]*const mlir.Value = undefined; + inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { + if (Field != Value) @compileError("returnIfOpts: every value must be a Value"); - ret_operands[i] = @field(values, f.name).inner; + ret_operands[i] = @field(values, field_name).inner; } const ret_target: *mlir.Block = if (opts.inline_return) blk: { @@ -2144,13 +2144,13 @@ pub const Builder = struct { result_types: anytype, ) IfScope(tupleArity(@TypeOf(result_types), "openIfElse: result_types")) { const N = comptime tupleArity(@TypeOf(result_types), "openIfElse: result_types"); - const fields = @typeInfo(@TypeOf(result_types)).@"struct".fields; + const info = @typeInfo(@TypeOf(result_types)).@"struct"; var types: [N]*const mlir.Type = undefined; - inline for (fields, 0..) |f, i| { - if (f.type != *const mlir.Type) + inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { + if (Field != *const mlir.Type) @compileError("openIfElse: every result_type must be *const mlir.Type (use k.scalarTy/tensorTy)"); - types[i] = @field(result_types, f.name); + types[i] = @field(result_types, field_name); } const then_block = mlir.Block.init(&.{}, &.{}); @@ -2176,16 +2176,16 @@ pub const Builder = struct { ) { const N = comptime tupleArity(@TypeOf(inits), "openWhile: inits"); const M = comptime tupleArity(@TypeOf(after_types), "openWhile: after_types"); - const init_fields = @typeInfo(@TypeOf(inits)).@"struct".fields; - const type_fields = @typeInfo(@TypeOf(after_types)).@"struct".fields; + const init_info = @typeInfo(@TypeOf(inits)).@"struct"; + const type_info = @typeInfo(@TypeOf(after_types)).@"struct"; var before_types: [N]*const mlir.Type = undefined; var before_locs: [N]*const mlir.Location = undefined; var inits_inner: [N]*const mlir.Value = undefined; - inline for (init_fields, 0..) |f, i| { - if (f.type != Value) + inline for (init_info.field_names, init_info.field_types, 0..) |field_name, Field, i| { + if (Field != Value) @compileError("openWhile: every init must be a Value"); - const v: Value = @field(inits, f.name); + const v: Value = @field(inits, field_name); before_types[i] = v.type_(); before_locs[i] = self.loc(); inits_inner[i] = v.inner; @@ -2194,10 +2194,10 @@ pub const Builder = struct { var after_tys: [M]*const mlir.Type = undefined; var after_locs: [M]*const mlir.Location = undefined; - inline for (type_fields, 0..) |f, i| { - if (f.type != *const mlir.Type) + inline for (type_info.field_names, type_info.field_types, 0..) |field_name, Field, i| { + if (Field != *const mlir.Type) @compileError("openWhile: every after_type must be *const mlir.Type"); - after_tys[i] = @field(after_types, f.name); + after_tys[i] = @field(after_types, field_name); after_locs[i] = self.loc(); } const after_block = mlir.Block.init(&after_tys, &after_locs); @@ -2249,6 +2249,6 @@ pub const Builder = struct { .print_name_loc_as_prefix = true, })}); - return try self.allocator.dupeZ(u8, al.written()); + return try self.allocator.dupeSentinel(u8, al.written(), 0); } }; diff --git a/kernels/triton/dtype.zig b/kernels/triton/dtype.zig index c8c1d623..45018e67 100644 --- a/kernels/triton/dtype.zig +++ b/kernels/triton/dtype.zig @@ -62,8 +62,8 @@ pub fn intBitwidth(dt: DType) u32 { } pub fn mlirElemToDType(ctx: *mlir.Context, elem: *const mlir.Type) DType { - inline for (std.meta.fields(DType)) |f| { - const dt = @field(DType, f.name); + inline for (comptime std.meta.fieldNames(DType)) |field_name| { + const dt = @field(DType, field_name); if (elem.eql(dt.toMlir(ctx))) return dt; } @panic("element type not a recognized Triton DType"); diff --git a/mlir/dialects/arith.zig b/mlir/dialects/arith.zig index 14e2bf4b..23e75c10 100644 --- a/mlir/dialects/arith.zig +++ b/mlir/dialects/arith.zig @@ -314,7 +314,7 @@ pub fn convertf( ) *mlir.Operation { var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; if (opts.rounding) |rm| { - attrs.appendAssumeCapacity(.named(ctx, "roundingmode", .int(ctx, .i32, @intFromEnum(rm)))); + attrs.appendAssumeCapacity(.named(ctx, "roundingmode", .int(ctx, .i32, @backingInt(rm)))); } return mlir.Operation.make(ctx, "arith.convertf", .{ .operands = .{ .flat = &.{src} }, @@ -335,7 +335,7 @@ pub fn scaling_truncf( ) *mlir.Operation { var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; if (opts.rounding) |rm| { - attrs.appendAssumeCapacity(.named(ctx, "roundingmode", .int(ctx, .i32, @intFromEnum(rm)))); + attrs.appendAssumeCapacity(.named(ctx, "roundingmode", .int(ctx, .i32, @backingInt(rm)))); } return mlir.Operation.make(ctx, "arith.scaling_truncf", .{ .operands = .{ .flat = &.{ src, scale } }, @@ -360,7 +360,7 @@ pub fn cmpi( .operands = .{ .flat = &.{ lhs, rhs } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "predicate", .int(ctx, .i64, @intFromEnum(predicate))), + .named(ctx, "predicate", .int(ctx, .i64, @backingInt(predicate))), }, .location = location, }); @@ -377,7 +377,7 @@ pub fn cmpf( .operands = .{ .flat = &.{ lhs, rhs } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "predicate", .int(ctx, .i64, @intFromEnum(predicate))), + .named(ctx, "predicate", .int(ctx, .i64, @backingInt(predicate))), }, .location = location, }); diff --git a/mlir/dialects/cuda_tile/cuda_tile.zig b/mlir/dialects/cuda_tile/cuda_tile.zig index ff86836a..09a314c0 100644 --- a/mlir/dialects/cuda_tile/cuda_tile.zig +++ b/mlir/dialects/cuda_tile/cuda_tile.zig @@ -433,14 +433,14 @@ pub const PtrLoadStoreHint = struct { fn writeHintFields(w: *std.Io.Writer, hint: anytype) !void { var first = true; - inline for (@typeInfo(@TypeOf(hint)).@"struct".fields) |f| { - if (comptime std.mem.eql(u8, f.name, "arch")) continue; - if (@field(hint, f.name)) |v| { + inline for (@typeInfo(@TypeOf(hint)).@"struct".field_names) |f_name| { + if (comptime std.mem.eql(u8, f_name, "arch")) continue; + if (@field(hint, f_name)) |v| { if (!first) try w.writeAll(", "); first = false; switch (@TypeOf(v)) { - bool => try w.print("{s} = {}", .{ f.name, v }), - else => try w.print("{s} = {d}", .{ f.name, v }), + bool => try w.print("{s} = {}", .{ f_name, v }), + else => try w.print("{s} = {d}", .{ f_name, v }), } } } diff --git a/mlir/dialects/cute_ir/cute_ir.zig b/mlir/dialects/cute_ir/cute_ir.zig index 3e9ff08f..91e242d5 100644 --- a/mlir/dialects/cute_ir/cute_ir.zig +++ b/mlir/dialects/cute_ir/cute_ir.zig @@ -634,14 +634,14 @@ pub const PtrType = opaque { bitlayout: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCutePtrTypeGet(ctx.ptr(), if (args.valueType) |v| v.ptr() else c.MlirType{ .ptr = null }, @intFromEnum(args.addressSpace), args.alignment, if (args.swizzle) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, if (args.bitlayout) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCutePtrTypeGet(ctx.ptr(), if (args.valueType) |v| v.ptr() else c.MlirType{ .ptr = null }, @backingInt(args.addressSpace), args.alignment, if (args.swizzle) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, if (args.bitlayout) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValueType(self: *const Self) ?*const mlir.Type { return @ptrCast(c.mlirCutePtrTypeGetValueType(self.ptr()).ptr); } pub fn getAddressSpace(self: *const Self) AddressSpace { - return @enumFromInt(c.mlirCutePtrTypeGetAddressSpace(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCutePtrTypeGetAddressSpace(self.ptr()))); } pub fn getAlignment(self: *const Self) u64 { return c.mlirCutePtrTypeGetAlignment(self.ptr()); @@ -1218,11 +1218,11 @@ pub const ReductionOpAttr = opaque { value: ReductionOp, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteReductionOpAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteReductionOpAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) ReductionOp { - return @enumFromInt(c.mlirCuteReductionOpAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteReductionOpAttrGetValue(self.ptr()))); } }; @@ -2850,11 +2850,11 @@ test { std.testing.refAllDecls(@This()); std.testing.refAllDecls(nvgpu); inline for (comptime std.meta.declarations(@This())) |decl| { - const T = @field(@This(), decl.name); + const T = @field(@This(), decl); if (@TypeOf(T) == type) std.testing.refAllDecls(T); } inline for (comptime std.meta.declarations(nvgpu)) |decl| { - const T = @field(nvgpu, decl.name); + const T = @field(nvgpu, decl); if (@TypeOf(T) == type) std.testing.refAllDecls(T); } } @@ -2976,13 +2976,14 @@ test "generated types and attributes rebuild from their getters" { // Every case of every enum attribute. inline for (.{ cute, nvgpu }) |namespace| { inline for (comptime std.meta.declarations(namespace)) |decl| { - const T = @field(namespace, decl.name); + const T = @field(namespace, decl); if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque" and @hasDecl(T, "InitArgs")) { - const fields = @typeInfo(T.InitArgs).@"struct".fields; - if (fields.len == 1 and comptime std.mem.eql(u8, fields[0].name, "value")) { - inline for (@typeInfo(fields[0].type).@"enum".fields) |case| { - const attr = try T.get(ctx, .{ .value = @enumFromInt(case.value) }); - try std.testing.expectEqual(case.value, @intFromEnum(attr.getValue())); + const fields = @typeInfo(T.InitArgs).@"struct".field_names; + if (fields.len == 1 and comptime std.mem.eql(u8, fields[0], "value")) { + inline for (@typeInfo(@FieldType(T.InitArgs, fields[0])).@"enum".field_names) |case| { + const case_value = @field(@FieldType(T.InitArgs, fields[0]), case); + const attr = try T.get(ctx, .{ .value = case_value }); + try std.testing.expectEqual(@backingInt(case_value), @backingInt(attr.getValue())); try std.testing.expect((try rebuild(T, ctx, attr)).eql(attr)); } } @@ -2995,14 +2996,14 @@ test "generated types and attributes rebuild from their getters" { fn rebuild(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T { var args: T.InitArgs = undefined; var elements: [8]*const mlir.Type = undefined; - inline for (@typeInfo(T.InitArgs).@"struct".fields) |field| { - const name = [_]u8{comptime std.ascii.toUpper(field.name[0])} ++ field.name[1..]; - if (field.type == []const *const mlir.Type) { + inline for (@typeInfo(T.InitArgs).@"struct".field_names) |field| { + const name = [_]u8{comptime std.ascii.toUpper(field[0])} ++ field[1..]; + if (@FieldType(T.InitArgs, field) == []const *const mlir.Type) { const count = @field(T, "getNum" ++ name)(value); for (elements[0..count], 0..) |*element, i| element.* = @field(T, "get" ++ name[0 .. name.len - 1])(value, i); - @field(args, field.name) = elements[0..count]; + @field(args, field) = elements[0..count]; } else { - @field(args, field.name) = @field(T, "get" ++ name)(value); + @field(args, field) = @field(T, "get" ++ name)(value); } } return T.get(ctx, args); diff --git a/mlir/dialects/cute_ir/cute_nvgpu.zig b/mlir/dialects/cute_ir/cute_nvgpu.zig index 7d7ac63d..652ff9e1 100644 --- a/mlir/dialects/cute_ir/cute_nvgpu.zig +++ b/mlir/dialects/cute_ir/cute_nvgpu.zig @@ -564,7 +564,7 @@ pub const CopyAtomG2RType = opaque { invariant: bool = false, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomG2RTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.memOrder), @intFromEnum(args.memScope), @intFromEnum(args.l2PrefetchSize), @intFromEnum(args.l1CacheEvictPriority), @intFromEnum(args.loadCacheMode), @intFromEnum(args.sharedSpace), args.invariant); + const result = c.mlirCuteNVGPUCopyAtomG2RTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.l2PrefetchSize), @backingInt(args.l1CacheEvictPriority), @backingInt(args.loadCacheMode), @backingInt(args.sharedSpace), args.invariant); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -574,22 +574,22 @@ pub const CopyAtomG2RType = opaque { return c.mlirCuteNVGPUCopyAtomG2RTypeGetCopyBits(self.ptr()); } pub fn getMemOrder(self: *const Self) MemOrderKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomG2RTypeGetMemOrder(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetMemOrder(self.ptr()))); } pub fn getMemScope(self: *const Self) MemScopeKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomG2RTypeGetMemScope(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetMemScope(self.ptr()))); } pub fn getL2PrefetchSize(self: *const Self) L2PrefetchSize { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomG2RTypeGetL2PrefetchSize(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetL2PrefetchSize(self.ptr()))); } pub fn getL1CacheEvictPriority(self: *const Self) CacheEvictionPriority { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomG2RTypeGetL1CacheEvictPriority(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetL1CacheEvictPriority(self.ptr()))); } pub fn getLoadCacheMode(self: *const Self) LoadCacheMode { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomG2RTypeGetLoadCacheMode(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetLoadCacheMode(self.ptr()))); } pub fn getSharedSpace(self: *const Self) SharedSpace { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomG2RTypeGetSharedSpace(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetSharedSpace(self.ptr()))); } pub fn getInvariant(self: *const Self) bool { return c.mlirCuteNVGPUCopyAtomG2RTypeGetInvariant(self.ptr()); @@ -705,7 +705,7 @@ pub const CopyAtomLdsmType = opaque { transpose: bool = false, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomLdsmTypeGet(ctx.ptr(), args.valType.ptr(), args.mode.ptr(), @intFromEnum(args.szPattern), args.numMatrices, args.transpose); + const result = c.mlirCuteNVGPUCopyAtomLdsmTypeGet(ctx.ptr(), args.valType.ptr(), args.mode.ptr(), @backingInt(args.szPattern), args.numMatrices, args.transpose); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -715,7 +715,7 @@ pub const CopyAtomLdsmType = opaque { return @ptrCast(c.mlirCuteNVGPUCopyAtomLdsmTypeGetMode(self.ptr()).ptr.?); } pub fn getSzPattern(self: *const Self) LdsmSzPattern { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomLdsmTypeGetSzPattern(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomLdsmTypeGetSzPattern(self.ptr()))); } pub fn getNumMatrices(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomLdsmTypeGetNumMatrices(self.ptr()); @@ -748,11 +748,11 @@ pub const CopyAtomNonExec2DGather4TmaLoadType = opaque { tmaFormat: ?TmaDataFormat = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGet(ctx.ptr(), @intFromEnum(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); + const result = c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getKind(self: *const Self) GatherScatterTmaLoad { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetKind(self.ptr()))); } pub fn getValType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetValType(self.ptr()).ptr.?); @@ -765,7 +765,7 @@ pub const CopyAtomNonExec2DGather4TmaLoadType = opaque { } pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @enumFromInt(value); + return if (value < 0) null else @fromBackingInt(@intCast(value)); } }; @@ -791,7 +791,7 @@ pub const CopyAtomNonExec2DScatter4TmaStoreType = opaque { tmaFormat: ?TmaDataFormat = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); + const result = c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -805,7 +805,7 @@ pub const CopyAtomNonExec2DScatter4TmaStoreType = opaque { } pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @enumFromInt(value); + return if (value < 0) null else @fromBackingInt(@intCast(value)); } }; @@ -832,11 +832,11 @@ pub const CopyAtomNonExecIm2ColTmaLoadType = opaque { tmaFormat: ?TmaDataFormat = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGet(ctx.ptr(), @intFromEnum(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); + const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getKind(self: *const Self) Im2ColTmaLoad { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetKind(self.ptr()))); } pub fn getValType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetValType(self.ptr()).ptr.?); @@ -849,7 +849,7 @@ pub const CopyAtomNonExecIm2ColTmaLoadType = opaque { } pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @enumFromInt(value); + return if (value < 0) null else @fromBackingInt(@intCast(value)); } }; @@ -875,7 +875,7 @@ pub const CopyAtomNonExecIm2ColTmaStoreType = opaque { tmaFormat: ?TmaDataFormat = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); + const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -889,7 +889,7 @@ pub const CopyAtomNonExecIm2ColTmaStoreType = opaque { } pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @enumFromInt(value); + return if (value < 0) null else @fromBackingInt(@intCast(value)); } }; @@ -916,11 +916,11 @@ pub const CopyAtomNonExecTiledTmaLoadType = opaque { tmaFormat: ?TmaDataFormat = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGet(ctx.ptr(), @intFromEnum(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); + const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getKind(self: *const Self) TiledTmaLoad { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetKind(self.ptr()))); } pub fn getValType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetValType(self.ptr()).ptr.?); @@ -933,7 +933,7 @@ pub const CopyAtomNonExecTiledTmaLoadType = opaque { } pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @enumFromInt(value); + return if (value < 0) null else @fromBackingInt(@intCast(value)); } }; @@ -960,11 +960,11 @@ pub const CopyAtomNonExecTiledTmaReduceType = opaque { tmaFormat: ?TmaDataFormat = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGet(ctx.ptr(), @intFromEnum(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); + const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getKind(self: *const Self) ReductionKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetKind(self.ptr()))); } pub fn getValType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetValType(self.ptr()).ptr.?); @@ -977,7 +977,7 @@ pub const CopyAtomNonExecTiledTmaReduceType = opaque { } pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @enumFromInt(value); + return if (value < 0) null else @fromBackingInt(@intCast(value)); } }; @@ -1003,7 +1003,7 @@ pub const CopyAtomNonExecTiledTmaStoreType = opaque { tmaFormat: ?TmaDataFormat = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); + const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1017,7 +1017,7 @@ pub const CopyAtomNonExecTiledTmaStoreType = opaque { } pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @enumFromInt(value); + return if (value < 0) null else @fromBackingInt(@intCast(value)); } }; @@ -1046,7 +1046,7 @@ pub const CopyAtomR2GType = opaque { sharedSpace: SharedSpace = .CTA, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomR2GTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.memOrder), @intFromEnum(args.memScope), @intFromEnum(args.l1CacheEvictPriority), @intFromEnum(args.storeCacheMode), @intFromEnum(args.sharedSpace)); + const result = c.mlirCuteNVGPUCopyAtomR2GTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.l1CacheEvictPriority), @backingInt(args.storeCacheMode), @backingInt(args.sharedSpace)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1056,19 +1056,19 @@ pub const CopyAtomR2GType = opaque { return c.mlirCuteNVGPUCopyAtomR2GTypeGetCopyBits(self.ptr()); } pub fn getMemOrder(self: *const Self) MemOrderKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2GTypeGetMemOrder(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetMemOrder(self.ptr()))); } pub fn getMemScope(self: *const Self) MemScopeKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2GTypeGetMemScope(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetMemScope(self.ptr()))); } pub fn getL1CacheEvictPriority(self: *const Self) CacheEvictionPriority { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2GTypeGetL1CacheEvictPriority(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetL1CacheEvictPriority(self.ptr()))); } pub fn getStoreCacheMode(self: *const Self) StoreCacheMode { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2GTypeGetStoreCacheMode(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetStoreCacheMode(self.ptr()))); } pub fn getSharedSpace(self: *const Self) SharedSpace { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2GTypeGetSharedSpace(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetSharedSpace(self.ptr()))); } }; @@ -1095,7 +1095,7 @@ pub const CopyAtomR2SType = opaque { sharedSpace: SharedSpace = .CTA, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomR2STypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.memOrder), @intFromEnum(args.memScope), @intFromEnum(args.sharedSpace)); + const result = c.mlirCuteNVGPUCopyAtomR2STypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.sharedSpace)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1105,13 +1105,13 @@ pub const CopyAtomR2SType = opaque { return c.mlirCuteNVGPUCopyAtomR2STypeGetCopyBits(self.ptr()); } pub fn getMemOrder(self: *const Self) MemOrderKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2STypeGetMemOrder(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2STypeGetMemOrder(self.ptr()))); } pub fn getMemScope(self: *const Self) MemScopeKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2STypeGetMemScope(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2STypeGetMemScope(self.ptr()))); } pub fn getSharedSpace(self: *const Self) SharedSpace { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2STypeGetSharedSpace(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2STypeGetSharedSpace(self.ptr()))); } }; @@ -1138,7 +1138,7 @@ pub const CopyAtomS2RType = opaque { sharedSpace: SharedSpace = .CTA, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomS2RTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.memOrder), @intFromEnum(args.memScope), @intFromEnum(args.sharedSpace)); + const result = c.mlirCuteNVGPUCopyAtomS2RTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.sharedSpace)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1148,13 +1148,13 @@ pub const CopyAtomS2RType = opaque { return c.mlirCuteNVGPUCopyAtomS2RTypeGetCopyBits(self.ptr()); } pub fn getMemOrder(self: *const Self) MemOrderKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomS2RTypeGetMemOrder(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetMemOrder(self.ptr()))); } pub fn getMemScope(self: *const Self) MemScopeKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomS2RTypeGetMemScope(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetMemScope(self.ptr()))); } pub fn getSharedSpace(self: *const Self) SharedSpace { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomS2RTypeGetSharedSpace(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetSharedSpace(self.ptr()))); } }; @@ -1179,14 +1179,14 @@ pub const CopyAtomSIMTAsyncCopyType = opaque { copyBits: c_int, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGet(ctx.ptr(), args.valType.ptr(), @intFromEnum(args.cache), args.copyBits); + const result = c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGet(ctx.ptr(), args.valType.ptr(), @backingInt(args.cache), args.copyBits); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetValType(self.ptr()).ptr.?); } pub fn getCache(self: *const Self) LoadCacheMode { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCache(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCache(self.ptr()))); } pub fn getCopyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCopyBits(self.ptr()); @@ -1217,7 +1217,7 @@ pub const CopyAtomSIMTMultimemLdReduceType = opaque { memScope: MemScopeKind = .SYS, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.reduction), if (args.ldReduceAccPrecision) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, @intFromEnum(args.memOrder), @intFromEnum(args.memScope)); + const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.reduction), if (args.ldReduceAccPrecision) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, @backingInt(args.memOrder), @backingInt(args.memScope)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1227,16 +1227,16 @@ pub const CopyAtomSIMTMultimemLdReduceType = opaque { return c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetCopyBits(self.ptr()); } pub fn getReduction(self: *const Self) ReductionKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetReduction(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetReduction(self.ptr()))); } pub fn getLdReduceAccPrecision(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetLdReduceAccPrecision(self.ptr()).ptr); } pub fn getMemOrder(self: *const Self) MemOrderKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemOrder(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemOrder(self.ptr()))); } pub fn getMemScope(self: *const Self) MemScopeKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemScope(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemScope(self.ptr()))); } }; @@ -1263,7 +1263,7 @@ pub const CopyAtomSIMTMultimemRedType = opaque { memScope: MemScopeKind = .SYS, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.reduction), @intFromEnum(args.memOrder), @intFromEnum(args.memScope)); + const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.reduction), @backingInt(args.memOrder), @backingInt(args.memScope)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1273,13 +1273,13 @@ pub const CopyAtomSIMTMultimemRedType = opaque { return c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetCopyBits(self.ptr()); } pub fn getReduction(self: *const Self) ReductionKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetReduction(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetReduction(self.ptr()))); } pub fn getMemOrder(self: *const Self) MemOrderKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemOrder(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemOrder(self.ptr()))); } pub fn getMemScope(self: *const Self) MemScopeKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemScope(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemScope(self.ptr()))); } }; @@ -1305,7 +1305,7 @@ pub const CopyAtomSIMTMultimemStType = opaque { memScope: MemScopeKind = .SYS, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.memOrder), @intFromEnum(args.memScope)); + const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1315,10 +1315,10 @@ pub const CopyAtomSIMTMultimemStType = opaque { return c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetCopyBits(self.ptr()); } pub fn getMemOrder(self: *const Self) MemOrderKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemOrder(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemOrder(self.ptr()))); } pub fn getMemScope(self: *const Self) MemScopeKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemScope(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemScope(self.ptr()))); } }; @@ -1344,7 +1344,7 @@ pub const CopyAtomSIMTSyncCopyType = opaque { dstSpace: cute.AddressSpace = .generic, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.srcSpace), @intFromEnum(args.dstSpace)); + const result = c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.srcSpace), @backingInt(args.dstSpace)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1354,10 +1354,10 @@ pub const CopyAtomSIMTSyncCopyType = opaque { return c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetCopyBits(self.ptr()); } pub fn getSrcSpace(self: *const Self) cute.AddressSpace { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetSrcSpace(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetSrcSpace(self.ptr()))); } pub fn getDstSpace(self: *const Self) cute.AddressSpace { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetDstSpace(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetDstSpace(self.ptr()))); } }; @@ -1384,7 +1384,7 @@ pub const CopyAtomSM100CopyS2TType = opaque { broadcast: CopyS2TBroadcast = .none, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numCta, @intFromEnum(args.broadcast)); + const result = c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numCta, @backingInt(args.broadcast)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1400,7 +1400,7 @@ pub const CopyAtomSM100CopyS2TType = opaque { return c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumCta(self.ptr()); } pub fn getBroadcast(self: *const Self) CopyS2TBroadcast { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetBroadcast(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetBroadcast(self.ptr()))); } }; @@ -1428,7 +1428,7 @@ pub const CopyAtomSM100S2TCopyV2Type = opaque { broadcast: CopyS2TBroadcast = .none, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numCta, @intFromEnum(args.smemMajor), @intFromEnum(args.broadcast)); + const result = c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numCta, @backingInt(args.smemMajor), @backingInt(args.broadcast)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1444,10 +1444,10 @@ pub const CopyAtomSM100S2TCopyV2Type = opaque { return c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumCta(self.ptr()); } pub fn getSmemMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetSmemMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetSmemMajor(self.ptr()))); } pub fn getBroadcast(self: *const Self) CopyS2TBroadcast { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetBroadcast(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetBroadcast(self.ptr()))); } }; @@ -1562,7 +1562,7 @@ pub const CopyAtomSM107TmemLoadSPCompressType = opaque { nan: bool = false, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, @intFromEnum(args.redOp), args.red, args.nan); + const result = c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, @backingInt(args.redOp), args.red, args.nan); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1578,7 +1578,7 @@ pub const CopyAtomSM107TmemLoadSPCompressType = opaque { return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumRep(self.ptr()); } pub fn getRedOp(self: *const Self) TmemLoadRedOp { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRedOp(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRedOp(self.ptr()))); } pub fn getRed(self: *const Self) bool { return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRed(self.ptr()); @@ -1613,7 +1613,7 @@ pub const CopyAtomSM10xTmemLoadRedType = opaque { halfSplitOff: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, @intFromEnum(args.redOp), args.nan, if (args.halfSplitOff) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, @backingInt(args.redOp), args.nan, if (args.halfSplitOff) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1629,7 +1629,7 @@ pub const CopyAtomSM10xTmemLoadRedType = opaque { return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumRep(self.ptr()); } pub fn getRedOp(self: *const Self) TmemLoadRedOp { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetRedOp(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetRedOp(self.ptr()))); } pub fn getNan(self: *const Self) bool { return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNan(self.ptr()); @@ -1706,7 +1706,7 @@ pub const CopyAtomTmaLoadType = opaque { noFullyOobTile: bool = false, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomTmaLoadTypeGet(ctx.ptr(), args.valType.ptr(), args.sparsity, args.copyBits, @intFromEnum(args.mode), args.numCta, args.gStride.ptr(), args.mcast, if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }, args.override, args.noFullyOobTile); + const result = c.mlirCuteNVGPUCopyAtomTmaLoadTypeGet(ctx.ptr(), args.valType.ptr(), args.sparsity, args.copyBits, @backingInt(args.mode), args.numCta, args.gStride.ptr(), args.mcast, if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }, args.override, args.noFullyOobTile); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1719,7 +1719,7 @@ pub const CopyAtomTmaLoadType = opaque { return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetCopyBits(self.ptr()); } pub fn getMode(self: *const Self) TmaLoadMode { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetMode(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetMode(self.ptr()))); } pub fn getNumCta(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetNumCta(self.ptr()); @@ -1765,7 +1765,7 @@ pub const CopyAtomTmaReduceType = opaque { tmaGbasis: ?*const mlir.Type = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomTmaReduceTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.mode), @intFromEnum(args.kind), args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }); + const result = c.mlirCuteNVGPUCopyAtomTmaReduceTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.mode), @backingInt(args.kind), args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1775,10 +1775,10 @@ pub const CopyAtomTmaReduceType = opaque { return c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetCopyBits(self.ptr()); } pub fn getMode(self: *const Self) TmaStoreMode { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetMode(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetMode(self.ptr()))); } pub fn getKind(self: *const Self) ReductionKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetKind(self.ptr()))); } pub fn getGStride(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetGStride(self.ptr()).ptr.?); @@ -1814,7 +1814,7 @@ pub const CopyAtomTmaStoreType = opaque { noFullyOobTile: bool = false, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.sparsity, args.copyBits, @intFromEnum(args.mode), args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }, args.override, args.noFullyOobTile); + const result = c.mlirCuteNVGPUCopyAtomTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.sparsity, args.copyBits, @backingInt(args.mode), args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }, args.override, args.noFullyOobTile); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1827,7 +1827,7 @@ pub const CopyAtomTmaStoreType = opaque { return c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetCopyBits(self.ptr()); } pub fn getMode(self: *const Self) TmaStoreMode { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetMode(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetMode(self.ptr()))); } pub fn getGStride(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetGStride(self.ptr()).ptr.?); @@ -1874,7 +1874,7 @@ pub const MmaAtomSM100UMMABlockScaledSparseType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @intFromEnum(args.sparseMetadataFormat), @intFromEnum(args.aFragKind), args.vecSize, @intFromEnum(args.archPromote), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.archPromote), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -1884,10 +1884,10 @@ pub const MmaAtomSM100UMMABlockScaledSparseType = opaque { return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetNumCta(self.ptr()); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAMajor(self.ptr()))); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBMajor(self.ptr()))); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAType(self.ptr()).ptr.?); @@ -1902,16 +1902,16 @@ pub const MmaAtomSM100UMMABlockScaledSparseType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSfType(self.ptr()).ptr.?); } pub fn getSparseMetadataFormat(self: *const Self) SparseMetadataFormat { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSparseMetadataFormat(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSparseMetadataFormat(self.ptr()))); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAFragKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAFragKind(self.ptr()))); } pub fn getVecSize(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetVecSize(self.ptr()); } pub fn getArchPromote(self: *const Self) Arch { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetArchPromote(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetArchPromote(self.ptr()))); } pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetIntOverflow(self.ptr()).ptr); @@ -1948,7 +1948,7 @@ pub const MmaAtomSM100UMMABlockScaledType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @intFromEnum(args.aFragKind), args.vecSize, @intFromEnum(args.archPromote), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.archPromote), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -1958,10 +1958,10 @@ pub const MmaAtomSM100UMMABlockScaledType = opaque { return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetNumCta(self.ptr()); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAMajor(self.ptr()))); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBMajor(self.ptr()))); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAType(self.ptr()).ptr.?); @@ -1976,13 +1976,13 @@ pub const MmaAtomSM100UMMABlockScaledType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetSfType(self.ptr()).ptr.?); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAFragKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAFragKind(self.ptr()))); } pub fn getVecSize(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetVecSize(self.ptr()); } pub fn getArchPromote(self: *const Self) Arch { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetArchPromote(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetArchPromote(self.ptr()))); } pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetIntOverflow(self.ptr()).ptr); @@ -2019,7 +2019,7 @@ pub const MmaAtomSM100UMMASparseType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.eType.ptr(), @intFromEnum(args.sparseMetadataFormat), @intFromEnum(args.aFragKind), args.cScaleExp, if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.eType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.cScaleExp, if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -2029,10 +2029,10 @@ pub const MmaAtomSM100UMMASparseType = opaque { return c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetNumCta(self.ptr()); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAMajor(self.ptr()))); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBMajor(self.ptr()))); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAType(self.ptr()).ptr.?); @@ -2047,10 +2047,10 @@ pub const MmaAtomSM100UMMASparseType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetEType(self.ptr()).ptr.?); } pub fn getSparseMetadataFormat(self: *const Self) SparseMetadataFormat { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetSparseMetadataFormat(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetSparseMetadataFormat(self.ptr()))); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAFragKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAFragKind(self.ptr()))); } pub fn getCScaleExp(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCScaleExp(self.ptr()); @@ -2088,7 +2088,7 @@ pub const MmaAtomSM100UMMAType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM100UMMATypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @intFromEnum(args.aFragKind), args.cScaleExp, if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM100UMMATypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.aFragKind), args.cScaleExp, if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -2098,10 +2098,10 @@ pub const MmaAtomSM100UMMAType = opaque { return c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetNumCta(self.ptr()); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAMajor(self.ptr()))); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetBMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetBMajor(self.ptr()))); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAType(self.ptr()).ptr.?); @@ -2113,7 +2113,7 @@ pub const MmaAtomSM100UMMAType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetCType(self.ptr()).ptr.?); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAFragKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAFragKind(self.ptr()))); } pub fn getCScaleExp(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetCScaleExp(self.ptr()); @@ -2155,7 +2155,7 @@ pub const MmaAtomSM107UMMABlockScaledSparseType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @intFromEnum(args.sparseMetadataFormat), @intFromEnum(args.aFragKind), args.vecSize, @intFromEnum(args.aCollectorOp), @intFromEnum(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -2165,10 +2165,10 @@ pub const MmaAtomSM107UMMABlockScaledSparseType = opaque { return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetNumCta(self.ptr()); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAMajor(self.ptr()))); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBMajor(self.ptr()))); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAType(self.ptr()).ptr.?); @@ -2183,19 +2183,19 @@ pub const MmaAtomSM107UMMABlockScaledSparseType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSfType(self.ptr()).ptr.?); } pub fn getSparseMetadataFormat(self: *const Self) SparseMetadataFormat { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSparseMetadataFormat(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSparseMetadataFormat(self.ptr()))); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAFragKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAFragKind(self.ptr()))); } pub fn getVecSize(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetVecSize(self.ptr()); } pub fn getACollectorOp(self: *const Self) MmaCollectorOp { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetACollectorOp(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetACollectorOp(self.ptr()))); } pub fn getBCollectorOp(self: *const Self) MmaCollectorOp { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBCollectorOp(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBCollectorOp(self.ptr()))); } pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetIntOverflow(self.ptr()).ptr); @@ -2233,7 +2233,7 @@ pub const MmaAtomSM107UMMABlockScaledType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @intFromEnum(args.aFragKind), args.vecSize, @intFromEnum(args.aCollectorOp), @intFromEnum(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -2243,10 +2243,10 @@ pub const MmaAtomSM107UMMABlockScaledType = opaque { return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetNumCta(self.ptr()); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAMajor(self.ptr()))); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBMajor(self.ptr()))); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAType(self.ptr()).ptr.?); @@ -2261,16 +2261,16 @@ pub const MmaAtomSM107UMMABlockScaledType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetSfType(self.ptr()).ptr.?); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAFragKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAFragKind(self.ptr()))); } pub fn getVecSize(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetVecSize(self.ptr()); } pub fn getACollectorOp(self: *const Self) MmaCollectorOp { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetACollectorOp(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetACollectorOp(self.ptr()))); } pub fn getBCollectorOp(self: *const Self) MmaCollectorOp { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBCollectorOp(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBCollectorOp(self.ptr()))); } pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetIntOverflow(self.ptr()).ptr); @@ -2309,7 +2309,7 @@ pub const MmaAtomSM107UMMASparseType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.eType.ptr(), @intFromEnum(args.sparseMetadataFormat), @intFromEnum(args.aFragKind), args.cScaleExp, @intFromEnum(args.aCollectorOp), @intFromEnum(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.eType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.cScaleExp, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -2319,10 +2319,10 @@ pub const MmaAtomSM107UMMASparseType = opaque { return c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetNumCta(self.ptr()); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAMajor(self.ptr()))); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBMajor(self.ptr()))); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAType(self.ptr()).ptr.?); @@ -2337,19 +2337,19 @@ pub const MmaAtomSM107UMMASparseType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetEType(self.ptr()).ptr.?); } pub fn getSparseMetadataFormat(self: *const Self) SparseMetadataFormat { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetSparseMetadataFormat(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetSparseMetadataFormat(self.ptr()))); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAFragKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAFragKind(self.ptr()))); } pub fn getCScaleExp(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCScaleExp(self.ptr()); } pub fn getACollectorOp(self: *const Self) MmaCollectorOp { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetACollectorOp(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetACollectorOp(self.ptr()))); } pub fn getBCollectorOp(self: *const Self) MmaCollectorOp { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBCollectorOp(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBCollectorOp(self.ptr()))); } pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetIntOverflow(self.ptr()).ptr); @@ -2386,7 +2386,7 @@ pub const MmaAtomSM107UMMAType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM107UMMATypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @intFromEnum(args.aFragKind), args.cScaleExp, @intFromEnum(args.aCollectorOp), @intFromEnum(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM107UMMATypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.aFragKind), args.cScaleExp, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -2396,10 +2396,10 @@ pub const MmaAtomSM107UMMAType = opaque { return c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetNumCta(self.ptr()); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAMajor(self.ptr()))); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBMajor(self.ptr()))); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAType(self.ptr()).ptr.?); @@ -2411,16 +2411,16 @@ pub const MmaAtomSM107UMMAType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetCType(self.ptr()).ptr.?); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAFragKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAFragKind(self.ptr()))); } pub fn getCScaleExp(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetCScaleExp(self.ptr()); } pub fn getACollectorOp(self: *const Self) MmaCollectorOp { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetACollectorOp(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetACollectorOp(self.ptr()))); } pub fn getBCollectorOp(self: *const Self) MmaCollectorOp { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBCollectorOp(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBCollectorOp(self.ptr()))); } pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetIntOverflow(self.ptr()).ptr); @@ -2502,7 +2502,7 @@ pub const MmaAtomSM80SparseType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM80SparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @intFromEnum(args.sparseMetadataFormat), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM80SparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.sparseMetadataFormat), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -2518,7 +2518,7 @@ pub const MmaAtomSM80SparseType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetCType(self.ptr()).ptr.?); } pub fn getSparseMetadataFormat(self: *const Self) SparseMetadataFormat { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetSparseMetadataFormat(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetSparseMetadataFormat(self.ptr()))); } pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetIntOverflow(self.ptr()).ptr); @@ -2639,17 +2639,17 @@ pub const MmaAtomSM90Type = opaque { bNeg: bool = false, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM90TypeGet(ctx.ptr(), args.shapeMnk.ptr(), @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @intFromEnum(args.aFragKind), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, args.aNeg, args.bNeg); + const result = c.mlirCuteNVGPUMmaAtomSM90TypeGet(ctx.ptr(), args.shapeMnk.ptr(), @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.aFragKind), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, args.aNeg, args.bNeg); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetShapeMnk(self.ptr()).ptr.?); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM90TypeGetAMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetAMajor(self.ptr()))); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM90TypeGetBMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetBMajor(self.ptr()))); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetAType(self.ptr()).ptr.?); @@ -2661,7 +2661,7 @@ pub const MmaAtomSM90Type = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetCType(self.ptr()).ptr.?); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM90TypeGetAFragKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetAFragKind(self.ptr()))); } pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetIntOverflow(self.ptr()).ptr); @@ -2932,11 +2932,11 @@ pub const AtomCopyFieldBulkCopyG2SAttr = opaque { value: AtomCopyFieldBulkCopyG2S, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldBulkCopyG2S { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGetValue(self.ptr()))); } }; @@ -2959,11 +2959,11 @@ pub const AtomCopyFieldBulkCopyS2GAttr = opaque { value: AtomCopyFieldBulkCopyS2G, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldBulkCopyS2G { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGetValue(self.ptr()))); } }; @@ -2986,11 +2986,11 @@ pub const AtomCopyFieldBulkCopyS2SAttr = opaque { value: AtomCopyFieldBulkCopyS2S, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldBulkCopyS2S { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGetValue(self.ptr()))); } }; @@ -3013,11 +3013,11 @@ pub const AtomCopyFieldDsmemStoreAttr = opaque { value: AtomCopyFieldDsmemStore, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldDsmemStore { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGetValue(self.ptr()))); } }; @@ -3040,11 +3040,11 @@ pub const AtomCopyFieldLoadGlobalAttr = opaque { value: AtomCopyFieldLoadGlobal, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldLoadGlobal { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGetValue(self.ptr()))); } }; @@ -3067,11 +3067,11 @@ pub const AtomCopyFieldNonExec2DGather4TmaLoadAttr = opaque { value: AtomCopyFieldNonExec2DGather4TmaLoad, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldNonExec2DGather4TmaLoad { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGetValue(self.ptr()))); } }; @@ -3094,11 +3094,11 @@ pub const AtomCopyFieldNonExec2DScatter4TmaStoreAttr = opaque { value: AtomCopyFieldNonExec2DScatter4TmaStore, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldNonExec2DScatter4TmaStore { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGetValue(self.ptr()))); } }; @@ -3121,11 +3121,11 @@ pub const AtomCopyFieldNonExecIm2ColTmaLoadAttr = opaque { value: AtomCopyFieldNonExecIm2ColTmaLoad, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldNonExecIm2ColTmaLoad { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGetValue(self.ptr()))); } }; @@ -3148,11 +3148,11 @@ pub const AtomCopyFieldNonExecIm2ColTmaStoreAttr = opaque { value: AtomCopyFieldNonExecIm2ColTmaStore, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldNonExecIm2ColTmaStore { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGetValue(self.ptr()))); } }; @@ -3175,11 +3175,11 @@ pub const AtomCopyFieldNonExecTiledTmaLoadAttr = opaque { value: AtomCopyFieldNonExecTiledTmaLoad, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldNonExecTiledTmaLoad { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGetValue(self.ptr()))); } }; @@ -3202,11 +3202,11 @@ pub const AtomCopyFieldNonExecTiledTmaReduceAttr = opaque { value: AtomCopyFieldNonExecTiledTmaReduce, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldNonExecTiledTmaReduce { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGetValue(self.ptr()))); } }; @@ -3229,11 +3229,11 @@ pub const AtomCopyFieldNonExecTiledTmaStoreAttr = opaque { value: AtomCopyFieldNonExecTiledTmaStore, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldNonExecTiledTmaStore { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGetValue(self.ptr()))); } }; @@ -3256,11 +3256,11 @@ pub const AtomCopyFieldStoreGlobalAttr = opaque { value: AtomCopyFieldStoreGlobal, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldStoreGlobal { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGetValue(self.ptr()))); } }; @@ -3283,11 +3283,11 @@ pub const AtomCopyFieldTmaLoadAttr = opaque { value: AtomCopyFieldTmaLoad, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldTmaLoad { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGetValue(self.ptr()))); } }; @@ -3310,11 +3310,11 @@ pub const AtomCopyFieldTmaReduceAttr = opaque { value: AtomCopyFieldTmaReduce, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldTmaReduce { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGetValue(self.ptr()))); } }; @@ -3337,11 +3337,11 @@ pub const AtomCopyFieldTmaStoreAttr = opaque { value: AtomCopyFieldTmaStore, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldTmaStore { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGetValue(self.ptr()))); } }; @@ -3364,11 +3364,11 @@ pub const AtomMmaFieldSM100Attr = opaque { value: AtomMmaFieldSM100, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM100AttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomMmaFieldSM100AttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomMmaFieldSM100 { - return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM100AttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100AttrGetValue(self.ptr()))); } }; @@ -3391,11 +3391,11 @@ pub const AtomMmaFieldSM100BlockScaledAttr = opaque { value: AtomMmaFieldSM100BlockScaled, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomMmaFieldSM100BlockScaled { - return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGetValue(self.ptr()))); } }; @@ -3418,11 +3418,11 @@ pub const AtomMmaFieldSM100BlockScaledSparseAttr = opaque { value: AtomMmaFieldSM100BlockScaledSparse, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomMmaFieldSM100BlockScaledSparse { - return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGetValue(self.ptr()))); } }; @@ -3445,11 +3445,11 @@ pub const AtomMmaFieldSM100SparseAttr = opaque { value: AtomMmaFieldSM100Sparse, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomMmaFieldSM100Sparse { - return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGetValue(self.ptr()))); } }; @@ -3472,11 +3472,11 @@ pub const AtomMmaFieldSM120BlockScaledAttr = opaque { value: AtomMmaFieldSM120BlockScaled, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomMmaFieldSM120BlockScaled { - return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGetValue(self.ptr()))); } }; @@ -3499,11 +3499,11 @@ pub const AtomMmaFieldSM80SparseAttr = opaque { value: AtomMmaFieldSM80Sparse, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomMmaFieldSM80Sparse { - return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGetValue(self.ptr()))); } }; @@ -3526,11 +3526,11 @@ pub const AtomMmaFieldSM90Attr = opaque { value: AtomMmaFieldSM90, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM90AttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomMmaFieldSM90AttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomMmaFieldSM90 { - return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM90AttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM90AttrGetValue(self.ptr()))); } }; @@ -3553,11 +3553,11 @@ pub const BinaryOpAttr = opaque { value: BinaryOp, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUBinaryOpAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUBinaryOpAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) BinaryOp { - return @enumFromInt(c.mlirCuteNVGPUBinaryOpAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUBinaryOpAttrGetValue(self.ptr()))); } }; @@ -3580,11 +3580,11 @@ pub const CopyS2TBroadcastAttr = opaque { value: CopyS2TBroadcast, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyS2TBroadcastAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUCopyS2TBroadcastAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) CopyS2TBroadcast { - return @enumFromInt(c.mlirCuteNVGPUCopyS2TBroadcastAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyS2TBroadcastAttrGetValue(self.ptr()))); } }; @@ -3607,11 +3607,11 @@ pub const GatherScatterTmaLoadAttr = opaque { value: GatherScatterTmaLoad, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUGatherScatterTmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUGatherScatterTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) GatherScatterTmaLoad { - return @enumFromInt(c.mlirCuteNVGPUGatherScatterTmaLoadAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUGatherScatterTmaLoadAttrGetValue(self.ptr()))); } }; @@ -3634,11 +3634,11 @@ pub const Im2ColTmaLoadAttr = opaque { value: Im2ColTmaLoad, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUIm2ColTmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUIm2ColTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) Im2ColTmaLoad { - return @enumFromInt(c.mlirCuteNVGPUIm2ColTmaLoadAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUIm2ColTmaLoadAttrGetValue(self.ptr()))); } }; @@ -3661,11 +3661,11 @@ pub const LdReduceAccPrecisionKindAttr = opaque { value: LdReduceAccPrecisionKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPULdReduceAccPrecisionKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPULdReduceAccPrecisionKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) LdReduceAccPrecisionKind { - return @enumFromInt(c.mlirCuteNVGPULdReduceAccPrecisionKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPULdReduceAccPrecisionKindAttrGetValue(self.ptr()))); } }; @@ -3688,11 +3688,11 @@ pub const LdsmSzPatternAttr = opaque { value: LdsmSzPattern, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPULdsmSzPatternAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPULdsmSzPatternAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) LdsmSzPattern { - return @enumFromInt(c.mlirCuteNVGPULdsmSzPatternAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPULdsmSzPatternAttrGetValue(self.ptr()))); } }; @@ -3715,11 +3715,11 @@ pub const LoadCacheModeAttr = opaque { value: LoadCacheMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPULoadCacheModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPULoadCacheModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) LoadCacheMode { - return @enumFromInt(c.mlirCuteNVGPULoadCacheModeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPULoadCacheModeAttrGetValue(self.ptr()))); } }; @@ -3742,11 +3742,11 @@ pub const MMAIntOverflowAttr = opaque { value: MMAIntOverflow, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMMAIntOverflowAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUMMAIntOverflowAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MMAIntOverflow { - return @enumFromInt(c.mlirCuteNVGPUMMAIntOverflowAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMMAIntOverflowAttrGetValue(self.ptr()))); } }; @@ -3769,11 +3769,11 @@ pub const MajorModeAttr = opaque { value: cute.MajorMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMajorModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUMajorModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMajorModeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMajorModeAttrGetValue(self.ptr()))); } }; @@ -3796,11 +3796,11 @@ pub const MmaCollectorOpAttr = opaque { value: MmaCollectorOp, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaCollectorOpAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUMmaCollectorOpAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MmaCollectorOp { - return @enumFromInt(c.mlirCuteNVGPUMmaCollectorOpAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaCollectorOpAttrGetValue(self.ptr()))); } }; @@ -3823,11 +3823,11 @@ pub const MmaFragKindAttr = opaque { value: MmaFragKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaFragKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUMmaFragKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MmaFragKind { - return @enumFromInt(c.mlirCuteNVGPUMmaFragKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaFragKindAttrGetValue(self.ptr()))); } }; @@ -3873,11 +3873,11 @@ pub const ReductionKindAttr = opaque { value: ReductionKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUReductionKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUReductionKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) ReductionKind { - return @enumFromInt(c.mlirCuteNVGPUReductionKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUReductionKindAttrGetValue(self.ptr()))); } }; @@ -3902,14 +3902,14 @@ pub const RmemFrgAttr = opaque { deriveElemType: bool = false, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPURmemFrgAttrGet(ctx.ptr(), args.valueType.ptr(), @intFromEnum(args.operand), args.deriveElemType); + const result = c.mlirCuteNVGPURmemFrgAttrGet(ctx.ptr(), args.valueType.ptr(), @backingInt(args.operand), args.deriveElemType); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValueType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPURmemFrgAttrGetValueType(self.ptr()).ptr.?); } pub fn getOperand(self: *const Self) cute.MmaOperand { - return @enumFromInt(c.mlirCuteNVGPURmemFrgAttrGetOperand(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPURmemFrgAttrGetOperand(self.ptr()))); } pub fn getDeriveElemType(self: *const Self) bool { return c.mlirCuteNVGPURmemFrgAttrGetDeriveElemType(self.ptr()); @@ -3935,11 +3935,11 @@ pub const SM100CircularSmemFrgAttr = opaque { majorMode: cute.MajorMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSM100CircularSmemFrgAttrGet(ctx.ptr(), @intFromEnum(args.majorMode)); + const result = c.mlirCuteNVGPUSM100CircularSmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getMajorMode(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUSM100CircularSmemFrgAttrGetMajorMode(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100CircularSmemFrgAttrGetMajorMode(self.ptr()))); } }; @@ -3962,11 +3962,11 @@ pub const SM100SmemFrgAttr = opaque { majorMode: cute.MajorMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSM100SmemFrgAttrGet(ctx.ptr(), @intFromEnum(args.majorMode)); + const result = c.mlirCuteNVGPUSM100SmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getMajorMode(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUSM100SmemFrgAttrGetMajorMode(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100SmemFrgAttrGetMajorMode(self.ptr()))); } }; @@ -4023,7 +4023,7 @@ pub const SM100TmemFrgAttr = opaque { tmemAllocMode: TmemAllocMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSM100TmemFrgAttrGet(ctx.ptr(), args.dataType.ptr(), args.storageType.ptr(), args.ctaGroup, @intFromEnum(args.tmemAllocMode)); + const result = c.mlirCuteNVGPUSM100TmemFrgAttrGet(ctx.ptr(), args.dataType.ptr(), args.storageType.ptr(), args.ctaGroup, @backingInt(args.tmemAllocMode)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getDataType(self: *const Self) *const mlir.Type { @@ -4036,7 +4036,7 @@ pub const SM100TmemFrgAttr = opaque { return c.mlirCuteNVGPUSM100TmemFrgAttrGetCtaGroup(self.ptr()); } pub fn getTmemAllocMode(self: *const Self) TmemAllocMode { - return @enumFromInt(c.mlirCuteNVGPUSM100TmemFrgAttrGetTmemAllocMode(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100TmemFrgAttrGetTmemAllocMode(self.ptr()))); } }; @@ -4063,7 +4063,7 @@ pub const SM100TmemSfFrgAttr = opaque { tmemAllocMode: TmemAllocMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSM100TmemSfFrgAttrGet(ctx.ptr(), args.sfType.ptr(), args.sfVecSize, args.ctaGroup, args.isSfa, @intFromEnum(args.tmemAllocMode)); + const result = c.mlirCuteNVGPUSM100TmemSfFrgAttrGet(ctx.ptr(), args.sfType.ptr(), args.sfVecSize, args.ctaGroup, args.isSfa, @backingInt(args.tmemAllocMode)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getSfType(self: *const Self) *const mlir.Type { @@ -4079,7 +4079,7 @@ pub const SM100TmemSfFrgAttr = opaque { return c.mlirCuteNVGPUSM100TmemSfFrgAttrGetIsSfa(self.ptr()); } pub fn getTmemAllocMode(self: *const Self) TmemAllocMode { - return @enumFromInt(c.mlirCuteNVGPUSM100TmemSfFrgAttrGetTmemAllocMode(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100TmemSfFrgAttrGetTmemAllocMode(self.ptr()))); } }; @@ -4102,11 +4102,11 @@ pub const SM107SmemFrgAttr = opaque { majorMode: cute.MajorMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSM107SmemFrgAttrGet(ctx.ptr(), @intFromEnum(args.majorMode)); + const result = c.mlirCuteNVGPUSM107SmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getMajorMode(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUSM107SmemFrgAttrGetMajorMode(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM107SmemFrgAttrGetMajorMode(self.ptr()))); } }; @@ -4129,11 +4129,11 @@ pub const SM90SmemFrgAttr = opaque { majorMode: cute.MajorMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSM90SmemFrgAttrGet(ctx.ptr(), @intFromEnum(args.majorMode)); + const result = c.mlirCuteNVGPUSM90SmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getMajorMode(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUSM90SmemFrgAttrGetMajorMode(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM90SmemFrgAttrGetMajorMode(self.ptr()))); } }; @@ -4156,11 +4156,11 @@ pub const TiledTmaLoadAttr = opaque { value: TiledTmaLoad, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTiledTmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUTiledTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TiledTmaLoad { - return @enumFromInt(c.mlirCuteNVGPUTiledTmaLoadAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUTiledTmaLoadAttrGetValue(self.ptr()))); } }; @@ -4183,11 +4183,11 @@ pub const TmaDataFormatAttr = opaque { value: TmaDataFormat, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTmaDataFormatAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUTmaDataFormatAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TmaDataFormat { - return @enumFromInt(c.mlirCuteNVGPUTmaDataFormatAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmaDataFormatAttrGetValue(self.ptr()))); } }; @@ -4210,11 +4210,11 @@ pub const TmaLoadModeAttr = opaque { value: TmaLoadMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTmaLoadModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUTmaLoadModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TmaLoadMode { - return @enumFromInt(c.mlirCuteNVGPUTmaLoadModeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmaLoadModeAttrGetValue(self.ptr()))); } }; @@ -4237,11 +4237,11 @@ pub const TmaStoreModeAttr = opaque { value: TmaStoreMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTmaStoreModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUTmaStoreModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TmaStoreMode { - return @enumFromInt(c.mlirCuteNVGPUTmaStoreModeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmaStoreModeAttrGetValue(self.ptr()))); } }; @@ -4264,11 +4264,11 @@ pub const TmemAllocModeAttr = opaque { value: TmemAllocMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTmemAllocModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUTmemAllocModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TmemAllocMode { - return @enumFromInt(c.mlirCuteNVGPUTmemAllocModeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmemAllocModeAttrGetValue(self.ptr()))); } }; @@ -4291,11 +4291,11 @@ pub const TmemLoadRedOpAttr = opaque { value: TmemLoadRedOp, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTmemLoadRedOpAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUTmemLoadRedOpAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TmemLoadRedOp { - return @enumFromInt(c.mlirCuteNVGPUTmemLoadRedOpAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmemLoadRedOpAttrGetValue(self.ptr()))); } }; diff --git a/mlir/dialects/fly/attributes.zig b/mlir/dialects/fly/attributes.zig index eceb8b0f..a30336fc 100644 --- a/mlir/dialects/fly/attributes.zig +++ b/mlir/dialects/fly/attributes.zig @@ -206,7 +206,7 @@ pub const AddressSpaceAttr = opaque { }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirFlyAddressSpaceAttrGet(ctx.ptr(), @intFromEnum(args.addressSpace)); + const result = c.mlirFlyAddressSpaceAttrGet(ctx.ptr(), @backingInt(args.addressSpace)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } }; @@ -214,7 +214,7 @@ pub const AddressSpaceAttr = opaque { pub const MmaOperand = enum(i32) { a = 0, b = 1, c = 2, d = 3 }; pub fn mmaOperandAttr(ctx: *mlir.Context, operand: MmaOperand) mlir.Error!*const mlir.Attribute { - const result = c.mlirFlyMmaOperandAttrGet(ctx.ptr(), @intFromEnum(operand)); + const result = c.mlirFlyMmaOperandAttrGet(ctx.ptr(), @backingInt(operand)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } @@ -234,6 +234,6 @@ pub const GemmTraversalOrder = enum(i32) { }; pub fn gemmTraversalOrderAttr(ctx: *mlir.Context, order: GemmTraversalOrder) mlir.Error!*const mlir.Attribute { - const result = c.mlirFlyGemmTraversalOrderAttrGet(ctx.ptr(), @intFromEnum(order)); + const result = c.mlirFlyGemmTraversalOrderAttrGet(ctx.ptr(), @backingInt(order)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } diff --git a/mlir/dialects/fly/fly.zig b/mlir/dialects/fly/fly.zig index 6083f308..63e24077 100644 --- a/mlir/dialects/fly/fly.zig +++ b/mlir/dialects/fly/fly.zig @@ -252,7 +252,7 @@ test { inline for (.{ types, attributes, rocdl }) |module| { std.testing.refAllDecls(module); inline for (comptime std.meta.declarations(module)) |decl| { - const value = @field(module, decl.name); + const value = @field(module, decl); if (@TypeOf(value) == type and @typeInfo(value) == .@"opaque") std.testing.refAllDecls(value); } } diff --git a/mlir/dialects/llvm_dialect.zig b/mlir/dialects/llvm_dialect.zig index 2f5d3091..8845ccbd 100644 --- a/mlir/dialects/llvm_dialect.zig +++ b/mlir/dialects/llvm_dialect.zig @@ -27,7 +27,7 @@ pub const NVPTXAddressSpace = enum(u32) { /// clash with it. pub fn pointerType(ctx: *mlir.Context, space: anytype) *const mlir.Type { var buf: [32]u8 = undefined; - const text = std.fmt.bufPrint(&buf, "!llvm.ptr<{d}>", .{@intFromEnum(space)}) catch unreachable; + const text = std.fmt.bufPrint(&buf, "!llvm.ptr<{d}>", .{@backingInt(space)}) catch unreachable; return mlir.Type.parse(ctx, text) catch std.debug.panic("failed to parse LLVM pointer type '{s}'", .{text}); } diff --git a/mlir/dialects/memref.zig b/mlir/dialects/memref.zig index 77acbc5d..338d1732 100644 --- a/mlir/dialects/memref.zig +++ b/mlir/dialects/memref.zig @@ -216,7 +216,7 @@ pub fn atomic_rmw( .operands = .{ .flat = buf.constSlice() }, .results = .{ .flat = &.{value.type_()} }, .attributes = &.{ - .named(ctx, "kind", .int(ctx, .i64, @intFromEnum(kind))), + .named(ctx, "kind", .int(ctx, .i64, @backingInt(kind))), }, .location = location, }); diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index efae3018..f9ea1e7e 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -805,7 +805,7 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result const backend_config_api_version: ?CustomCallOpts.ApiVersion = if (opts.backend_config) |bc| bc else null; if (opts.api_version orelse backend_config_api_version) |v| { attrs.appendAssumeCapacity( - .named(ctx, "api_version", .int(ctx, .i32, @intFromEnum(v))), + .named(ctx, "api_version", .int(ctx, .i32, @backingInt(v))), ); } if (opts.backend_config) |backend_config| { @@ -1463,7 +1463,7 @@ pub fn channelHandle( return @ptrCast(c.stablehloChannelHandleGet( ctx.ptr(), handle, - @intFromEnum(channel_type), + @backingInt(channel_type), ).ptr); } diff --git a/mlir/dialects/ttir/ttir.zig b/mlir/dialects/ttir/ttir.zig index 81d926dc..ee302f92 100644 --- a/mlir/dialects/ttir/ttir.zig +++ b/mlir/dialects/ttir/ttir.zig @@ -142,51 +142,51 @@ pub const DescriptorReduceKind = enum(i32) { }; pub fn programDim(ctx: *mlir.Context, v: ProgramIDDim) *const mlir.Attribute { - return @ptrCast(c.mlirTritonProgramDimGet(ctx.ptr(), @intFromEnum(v)).ptr); + return @ptrCast(c.mlirTritonProgramDimGet(ctx.ptr(), @backingInt(v)).ptr); } pub fn cacheModifier(ctx: *mlir.Context, v: CacheModifier) *const mlir.Attribute { - return @ptrCast(c.mlirTritonCacheModifierGet(ctx.ptr(), @intFromEnum(v)).ptr); + return @ptrCast(c.mlirTritonCacheModifierGet(ctx.ptr(), @backingInt(v)).ptr); } pub fn evictionPolicy(ctx: *mlir.Context, v: EvictionPolicy) *const mlir.Attribute { - return @ptrCast(c.mlirTritonEvictionPolicyGet(ctx.ptr(), @intFromEnum(v)).ptr); + return @ptrCast(c.mlirTritonEvictionPolicyGet(ctx.ptr(), @backingInt(v)).ptr); } pub fn paddingOption(ctx: *mlir.Context, v: PaddingOption) *const mlir.Attribute { - return @ptrCast(c.mlirTritonPaddingOptionGet(ctx.ptr(), @intFromEnum(v)).ptr); + return @ptrCast(c.mlirTritonPaddingOptionGet(ctx.ptr(), @backingInt(v)).ptr); } pub fn roundingMode(ctx: *mlir.Context, v: RoundingMode) *const mlir.Attribute { - return @ptrCast(c.mlirTritonRoundingModeGet(ctx.ptr(), @intFromEnum(v)).ptr); + return @ptrCast(c.mlirTritonRoundingModeGet(ctx.ptr(), @backingInt(v)).ptr); } pub fn inputPrecision(ctx: *mlir.Context, v: InputPrecision) *const mlir.Attribute { - return @ptrCast(c.mlirTritonInputPrecisionGet(ctx.ptr(), @intFromEnum(v)).ptr); + return @ptrCast(c.mlirTritonInputPrecisionGet(ctx.ptr(), @backingInt(v)).ptr); } pub fn rmwOp(ctx: *mlir.Context, v: RMWOp) *const mlir.Attribute { - return @ptrCast(c.mlirTritonRMWOpGet(ctx.ptr(), @intFromEnum(v)).ptr); + return @ptrCast(c.mlirTritonRMWOpGet(ctx.ptr(), @backingInt(v)).ptr); } pub fn memSemantic(ctx: *mlir.Context, v: MemSemantic) *const mlir.Attribute { - return @ptrCast(c.mlirTritonMemSemanticGet(ctx.ptr(), @intFromEnum(v)).ptr); + return @ptrCast(c.mlirTritonMemSemanticGet(ctx.ptr(), @backingInt(v)).ptr); } pub fn memSyncScope(ctx: *mlir.Context, v: MemSyncScope) *const mlir.Attribute { - return @ptrCast(c.mlirTritonMemSyncScopeGet(ctx.ptr(), @intFromEnum(v)).ptr); + return @ptrCast(c.mlirTritonMemSyncScopeGet(ctx.ptr(), @backingInt(v)).ptr); } pub fn propagateNan(ctx: *mlir.Context, v: PropagateNan) *const mlir.Attribute { - return @ptrCast(c.mlirTritonPropagateNanGet(ctx.ptr(), @intFromEnum(v)).ptr); + return @ptrCast(c.mlirTritonPropagateNanGet(ctx.ptr(), @backingInt(v)).ptr); } pub fn scaleDotElemType(ctx: *mlir.Context, v: ScaleDotElemType) *const mlir.Attribute { - return @ptrCast(c.mlirTritonScaleDotElemTypeGet(ctx.ptr(), @intFromEnum(v)).ptr); + return @ptrCast(c.mlirTritonScaleDotElemTypeGet(ctx.ptr(), @backingInt(v)).ptr); } pub fn descriptorReduceKind(ctx: *mlir.Context, v: DescriptorReduceKind) *const mlir.Attribute { - return @ptrCast(c.mlirTritonDescriptorReduceKindGet(ctx.ptr(), @intFromEnum(v)).ptr); + return @ptrCast(c.mlirTritonDescriptorReduceKindGet(ctx.ptr(), @backingInt(v)).ptr); } // ============================================================================= diff --git a/mlir/mlir.zig b/mlir/mlir.zig index d759dbfa..83bec2f5 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -88,16 +88,16 @@ pub fn registerFuncExtensions(registry: *DialectRegistry) void { } fn isPtrConst(comptime T: type) bool { - return @typeInfo(T).pointer.is_const; + return @typeInfo(T).pointer.attrs.@"const"; } fn CastedPtr(comptime T1: type, comptime T2: type) type { - return if (@typeInfo(T1).pointer.is_const) *const T2 else *T2; + return if (@typeInfo(T1).pointer.attrs.@"const") *const T2 else *T2; } pub fn Methods(comptime T: type, comptime NativeT: type) type { return struct { - pub const IsPtrConst = @typeInfo(@typeInfo(@FieldType(NativeT, "ptr")).optional.child).pointer.is_const; + pub const IsPtrConst = @typeInfo(@typeInfo(@FieldType(NativeT, "ptr")).optional.child).pointer.attrs.@"const"; pub const Ptr = if (IsPtrConst) *const T else *T; pub fn ptr(self: anytype) NativeT { @@ -1467,11 +1467,11 @@ pub const Operation = opaque { (struct { pub fn callback(op: c.MlirOperation, ctx_: ?*anyopaque) callconv(.c) c.MlirWalkResult { const inner_ctx_: *ContextType = @ptrCast(@alignCast(ctx_)); - return @intFromEnum(walkfn(inner_ctx_.ctx, @ptrCast(op.ptr))); + return @backingInt(walkfn(inner_ctx_.ctx, @ptrCast(op.ptr))); } }).callback, &inner_ctx, - @intFromEnum(order), + @backingInt(order), ); } diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index 0c160321..aa05bbc9 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -11,7 +11,7 @@ const Stream = @import("pjrt.zig").Stream; const log = std.log.scoped(.pjrt); comptime { - if (@typeInfo(TypeId).@"struct".fields.len != 1) @compileError("TypeId has changed"); + if (@typeInfo(TypeId).@"struct".field_names.len != 1) @compileError("TypeId has changed"); } /// The signature of a generic custom call. @@ -450,7 +450,7 @@ pub const Error = opaque { pub fn create(api: *const Api, error_code: ErrorCode, message: []const u8) *Error { var ret: pjrt.meta.Struct(c.XLA_FFI_Error_Create_Args) = .{ .message = message.ptr, - .errc = @intFromEnum(error_code), + .errc = @backingInt(error_code), }; return fromInner(api.inner().XLA_FFI_Error_Create.?(@ptrCast(&ret)).?); } diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index bd58386f..b223188c 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -31,18 +31,18 @@ pub const meta = struct { } pub fn Struct(comptime T: type) type { - const fields = std.meta.fields(T); - var names: [fields.len][]const u8 = undefined; - var types: [fields.len]type = undefined; - var attributes: [fields.len]std.builtin.Type.StructField.Attributes = undefined; - for (fields, &names, &types, &attributes) |field, *name, *type_, *attr| { - name.* = field.name; - type_.* = field.type; + const info = @typeInfo(T).@"struct"; + var names: [info.field_names.len][]const u8 = undefined; + var types: [info.field_names.len]type = undefined; + var attributes: [info.field_names.len]std.builtin.Type.Struct.FieldAttributes = undefined; + for (info.field_names, info.field_types, &names, &types, &attributes) |field_name, Field, *name, *type_, *attr| { + name.* = field_name; + type_.* = Field; attr.* = .{ - .default_value_ptr = @ptrCast(if (std.mem.eql(u8, field.name, "struct_size")) + .default_value_ptr = @ptrCast(if (std.mem.eql(u8, field_name, "struct_size")) &structSize(T) else - &std.mem.zeroes(field.type)), + &std.mem.zeroes(Field)), }; } return @Struct( @@ -153,7 +153,7 @@ pub const Api = struct { const fti = @typeInfo(@FieldType(c.PJRT_Api, @tagName(func))); const fn_ptr = @typeInfo(fti.optional.child); const fn_type_info = @typeInfo(fn_ptr.pointer.child); - const arg_array_type_info = @typeInfo(fn_type_info.@"fn".params[0].type.?); + const arg_array_type_info = @typeInfo(fn_type_info.@"fn".param_types[0].?); return arg_array_type_info.pointer.child; } @@ -257,7 +257,7 @@ pub const Api = struct { pub fn stablehloCurrentVersion(self: *const Api) ?[]const u8 { const state = struct { var buf: [32]u8 = undefined; - var str: ?[:0]const u8 = null; + var str: ?[]const u8 = null; }; if (state.str) |str| { return str; @@ -265,7 +265,7 @@ pub const Api = struct { if (self.pluginAttribute("stablehlo_current_version")) |nv| { switch (nv.value()) { .int64list => |v| { - state.str = std.fmt.bufPrintZ(&state.buf, "{d}.{d}.{d}", .{ v[0], v[1], v[2] }) catch unreachable; + state.str = std.mem.print(&state.buf, "{d}.{d}.{d}", .{ v[0], v[1], v[2] }) catch unreachable; }, else => unreachable, } @@ -367,7 +367,7 @@ pub const Error = opaque { const ret = api.call(.PJRT_Error_GetCode, .{ .@"error" = self.inner(), }) catch unreachable; - return @enumFromInt(ret.code); + return @fromBackingInt(@intCast(ret.code)); } pub fn getMessage(self: *Error, api: *const Api) []const u8 { @@ -392,7 +392,7 @@ pub const ShapeSpec = extern struct { .inner = .{ .dims = @ptrCast(@constCast(dims_)), .num_dims = dims_.len, - .element_type = @intFromEnum(bt), + .element_type = @backingInt(bt), }, }; } @@ -402,7 +402,7 @@ pub const ShapeSpec = extern struct { } pub fn bufferType(self: ShapeSpec) BufferType { - return @enumFromInt(self.inner.element_type); + return @fromBackingInt(@intCast(self.inner.element_type)); } }; @@ -496,13 +496,13 @@ pub const Client = opaque { const ret = try api.call(.PJRT_Client_BufferFromHostBuffer, .{ .client = self.inner(), .data = @constCast(args.data), - .type = @intFromEnum(args.buffer_type), + .type = @backingInt(args.buffer_type), .dims = @ptrCast(@constCast(args.dims)), .num_dims = args.dims.len, .byte_strides = if (args.byte_strides) |bs| @ptrCast(@constCast(bs)) else null, .num_byte_strides = if (args.byte_strides) |bs| bs.len else 0, .device_layout = @ptrCast(@constCast(&args.layout.toCStruct())), - .host_buffer_semantics = @intFromEnum(args.host_buffer_semantics), + .host_buffer_semantics = @backingInt(args.host_buffer_semantics), .device = if (args.dst == .device) @ptrCast(@constCast(args.dst.device)) else null, .memory = if (args.dst == .memory) args.dst.memory.inner() else null, }); @@ -542,7 +542,7 @@ pub const Client = opaque { .device_buffer_ptr = @constCast(args.device_buffer_ptr), .dims = args.dims.ptr, .num_dims = args.dims.len, - .element_type = @intFromEnum(args.element_type), + .element_type = @backingInt(args.element_type), .layout = @ptrCast(@constCast(&layout)), .device = @ptrCast(@constCast(args.device)), .on_delete_callback = args.on_delete_callback, @@ -607,7 +607,7 @@ pub const Client = opaque { .struct_size = c.PJRT_Layouts_PJRT_Client_GetDefaultLayout_Args_STRUCT_SIZE, .extension_start = null, .client = self.inner(), - .type = @intFromEnum(element_type), + .type = @backingInt(element_type), .dims = dims.ptr, .num_dims = dims.len, .layout = null, @@ -668,7 +668,7 @@ pub const Client = opaque { .client = self.inner(), .shape_dims = args.dims.ptr, .shape_num_dims = args.dims.len, - .shape_element_type = @intFromEnum(args.element_type), + .shape_element_type = @backingInt(args.element_type), .shape_layout = @ptrCast(&layout), .device = if (args.dst == .device) @ptrCast(@constCast(args.dst.device)) else null, .memory = if (args.dst == .memory) args.dst.memory.inner() else null, @@ -806,7 +806,7 @@ pub const Device = opaque { const ret = try api.call(.PJRT_Device_MemoryStats, .{ .device = self.inner(), }); - return .fromCStruct(@bitCast(ret)); + return .fromCStruct(ret); } }; @@ -1285,7 +1285,7 @@ pub const Buffer = opaque { const ret = api.call(.PJRT_Buffer_ElementType, .{ .buffer = self.inner(), }) catch unreachable; - return @enumFromInt(ret.type); + return @fromBackingInt(@intCast(ret.type)); } pub fn dimensions(self: *const Buffer, api: *const Api) []const i64 { @@ -1615,7 +1615,7 @@ pub const NamedValue = extern struct { }; pub fn kind(self: NamedValue) Kind { - return @enumFromInt(self.inner.type); + return @fromBackingInt(@intCast(self.inner.type)); } pub fn name(self: NamedValue) []const u8 { @@ -1632,14 +1632,14 @@ pub const NamedValue = extern struct { }; } - pub fn init(comptime kind_: Kind, name_: []const u8, value_: std.meta.fieldInfo(Value, kind_).type) NamedValue { + pub fn init(comptime kind_: Kind, name_: []const u8, value_: @FieldType(Value, @tagName(kind_))) NamedValue { return .{ .inner = .{ .struct_size = c.PJRT_NamedValue_STRUCT_SIZE, .extension_start = null, .name = @ptrCast(name_), .name_size = name_.len, - .type = @intFromEnum(kind_), + .type = @backingInt(kind_), .unnamed_0 = switch (kind_) { .string => .{ .string_value = @ptrCast(@constCast(value_)) }, .int64 => .{ .int64_value = value_ }, diff --git a/platforms/cuda/compat_probe.zig b/platforms/cuda/compat_probe.zig index 9e962577..3f890c6a 100644 --- a/platforms/cuda/compat_probe.zig +++ b/platforms/cuda/compat_probe.zig @@ -22,7 +22,7 @@ pub fn main(init: std.process.Init) !u8 { const cuInit = libcuda.lookup(*const fn (c_uint) callconv(.c) c_int, "cuInit").?; const cuGetErrorString = libcuda.lookup(*const fn (c_int, *[*c]const u8) callconv(.c) c_int, "cuGetErrorString").?; - return @intFromEnum(switch (cuInit(0)) { + return @backingInt(switch (cuInit(0)) { CUDA_SUCCESS => ExitCode.Success, CUDA_ERROR_SYSTEM_DRIVER_MISMATCH => ExitCode.SystemDriverMismatch, CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE => ExitCode.CompatNotSupportedOnDevice, diff --git a/platforms/cuda/cuda.zig b/platforms/cuda/cuda.zig index 562fdc56..692ea226 100644 --- a/platforms/cuda/cuda.zig +++ b/platforms/cuda/cuda.zig @@ -47,7 +47,7 @@ pub fn needsCudaCompat(io: std.Io, sandbox_path: []const u8) !bool { log.err("Failed to run CUDA compatibility probe: {any}", .{err}); return err; }; - const result: compat_probe.ExitCode = @enumFromInt(res.exited); + const result: compat_probe.ExitCode = @fromBackingInt(@intCast(res.exited)); return switch (result) { .Success => true, @@ -69,7 +69,7 @@ fn hasNvidiaDevice(io: std.Io) bool { fn hasCudaPathInLDPath() bool { const ldLibraryPath = std.c.getenv("LD_LIBRARY_PATH") orelse return false; - return std.ascii.indexOfIgnoreCase(std.mem.span(ldLibraryPath), nvidiaLibsPath) != null; + return std.ascii.findIgnoreCase(std.mem.span(ldLibraryPath), nvidiaLibsPath) != null; } fn setupXlaGpuCudaDirFlag(allocator: std.mem.Allocator, sandbox: []const u8) !void { diff --git a/platforms/cuda/flashattn/flashattn.zig b/platforms/cuda/flashattn/flashattn.zig index b4014c25..7d9c72b0 100644 --- a/platforms/cuda/flashattn/flashattn.zig +++ b/platforms/cuda/flashattn/flashattn.zig @@ -20,7 +20,7 @@ pub const Tensor = struct { .rank = @intCast(rank), .dims = undefined, .strides = undefined, - .dtype = @intCast(@intFromEnum(dtype)), + .dtype = @intCast(@backingInt(dtype)), }, }; @memcpy(ret.inner.dims[0..rank], dims); diff --git a/platforms/neuron/libzml_neuron.zig b/platforms/neuron/libzml_neuron.zig index 968f1584..404d8cc0 100644 --- a/platforms/neuron/libzml_neuron.zig +++ b/platforms/neuron/libzml_neuron.zig @@ -46,7 +46,7 @@ fn setupNeuronEnv(io: std.Io, sandbox_path: []const u8) !void { var root_comm_buf: [256]u8 = undefined; _ = setenv( "NEURON_RT_ROOT_COMM_ID", - try std.fmt.bufPrintZ(&root_comm_buf, "127.0.0.1:{d}", .{try findFreeTcpPort(io)}), + try std.mem.printSentinel(&root_comm_buf, "127.0.0.1:{d}", .{try findFreeTcpPort(io)}, 0), 1, ); _ = setenv( @@ -69,9 +69,9 @@ fn setupNeuronEnv(io: std.Io, sandbox_path: []const u8) !void { const old_path = if (std.c.getenv("PATH")) |path| std.mem.span(path) else ""; var new_path_buf: [std.posix.PATH_MAX:0]u8 = undefined; const new_path = if (old_path.len == 0) - try std.fmt.bufPrintZ(&new_path_buf, "{s}", .{bin_path}) + try std.mem.printSentinel(&new_path_buf, "{s}", .{bin_path}, 0) else - try std.fmt.bufPrintZ(&new_path_buf, "{s}:{s}", .{ bin_path, old_path }); + try std.mem.printSentinel(&new_path_buf, "{s}:{s}", .{ bin_path, old_path }, 0); if (setenv("PATH", new_path.ptr, 1) != 0) return error.SetEnvFailed; if (std.c.getenv("NEURON_CC_FLAGS") != null) return; @@ -81,7 +81,7 @@ fn setupNeuronEnv(io: std.Io, sandbox_path: []const u8) !void { var flags_buf: [512:0]u8 = undefined; const instance = try neuron.instance(); const target = @tagName(instance.compilerTarget()); - const flags = try std.fmt.bufPrintZ( + const flags = try std.mem.printSentinel( &flags_buf, "--target={s} " ++ "--optlevel=1 " ++ @@ -98,6 +98,7 @@ fn setupNeuronEnv(io: std.Io, sandbox_path: []const u8) !void { log_level, log_level, }, + 0, ); _ = setenv("NEURON_CC_FLAGS", flags.ptr, 0); diff --git a/platforms/neuron/neuron.zig b/platforms/neuron/neuron.zig index 039cc566..e0365fe9 100644 --- a/platforms/neuron/neuron.zig +++ b/platforms/neuron/neuron.zig @@ -82,16 +82,16 @@ pub const Instance = struct { pub fn instance() !Instance { var info = std.mem.zeroes(c.struct_nrt_instance_info); - info.family = @intFromEnum(Family.unknown); - info.size = @intFromEnum(Size.unknown); + info.family = @backingInt(Family.unknown); + info.size = @backingInt(Size.unknown); if (c.nrt_get_instance_info(&info, @sizeOf(c.struct_nrt_instance_info)) != c.NRT_SUCCESS) { return error.Unavailable; } return .{ - .family = @enumFromInt(info.family), - .size = @enumFromInt(info.size), + .family = @fromBackingInt(@intCast(info.family)), + .size = @fromBackingInt(@intCast(info.size)), }; } diff --git a/platforms/neuron/python_launcher.zig b/platforms/neuron/python_launcher.zig index 26ca9766..075034ab 100644 --- a/platforms/neuron/python_launcher.zig +++ b/platforms/neuron/python_launcher.zig @@ -113,7 +113,7 @@ fn configureInterpreter(init: std.process.Init, config: *c.PyConfig) ![]const u8 const args = try init.minimal.args.toSlice(arena.allocator()); const c_args = try arena.allocator().alloc([*:0]const u8, args.len); for (args, 0..) |arg, i| { - c_args[i] = (try arena.allocator().dupeZ(u8, arg)).ptr; + c_args[i] = (try arena.allocator().dupeSentinel(u8, arg, 0)).ptr; } pyStatusCheck(c.PyConfig_SetBytesArgv(config, @intCast(c_args.len), @ptrCast(c_args.ptr))); @@ -125,7 +125,7 @@ fn configureInterpreter(init: std.process.Init, config: *c.PyConfig) ![]const u8 { var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const home = try std.fmt.bufPrintZ(&buf, "{f}{d}.{d}", .{ + const home = try std.mem.printSentinel(&buf, "{f}{d}.{d}", .{ std.Io.Dir.path.fmtJoin(&.{ self_exe_dir, "..", @@ -134,7 +134,7 @@ fn configureInterpreter(init: std.process.Init, config: *c.PyConfig) ![]const u8 }), c.PY_MAJOR_VERSION, c.PY_MINOR_VERSION, - }); + }, 0); pyStatusCheck(c.PyConfig_SetBytesString(config, &config.home, home)); pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(home))); } diff --git a/platforms/platforms.zig b/platforms/platforms.zig index 3f0c56fd..f1320f3e 100644 --- a/platforms/platforms.zig +++ b/platforms/platforms.zig @@ -22,8 +22,8 @@ pub const Platform = enum { metal, pub fn selectFirstAcceleratorEnabled() error{Unavailable}!Platform { - inline for (@typeInfo(Platform).@"enum".fields) |field| { - const target: Platform = @enumFromInt(field.value); + inline for (@typeInfo(Platform).@"enum".field_names) |field| { + const target: Platform = @field(Platform, field); if (comptime (target != .cpu and target.isEnabled())) return target; } diff --git a/stdx/flags.zig b/stdx/flags.zig index 6ee06eaf..52a9e6a4 100644 --- a/stdx/flags.zig +++ b/stdx/flags.zig @@ -94,7 +94,8 @@ pub fn parseProcessArgs(init_: std.process.Init.Minimal, comptime CliArgs: type) fn parse_commands(args: *std.process.Args.Iterator, comptime Commands: type) Commands { comptime assert(@typeInfo(Commands) == .@"union"); - comptime assert(std.meta.fields(Commands).len >= 2); + const field_names = std.meta.fieldNames(Commands); + comptime assert(field_names.len >= 2); const first_arg = args.next() orelse fatal( "subcommand required, expected {s}", @@ -109,10 +110,10 @@ fn parse_commands(args: *std.process.Args.Iterator, comptime Commands: type) Com } } - inline for (comptime std.meta.fields(Commands)) |field| { - comptime assert(std.mem.indexOf(u8, field.name, "_") == null); - if (std.mem.eql(u8, first_arg, field.name)) { - return @unionInit(Commands, field.name, parse_flags(args, field.type)); + inline for (field_names, std.meta.fieldTypes(Commands)) |field_name, FieldType| { + comptime assert(std.mem.indexOf(u8, field_name, "_") == null); + if (std.mem.eql(u8, first_arg, field_name)) { + return @unionInit(Commands, field_name, parse_flags(args, FieldType)); } } fatal("unknown subcommand: '{s}'", .{first_arg}); @@ -128,52 +129,61 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { return {}; } - assert(@typeInfo(Flags) == .@"struct"); + const flags_info = @typeInfo(Flags).@"struct"; - comptime var fields: [std.meta.fields(Flags).len]std.builtin.Type.StructField = undefined; + const total_num_fields = flags_info.field_names.len; + comptime var field_names: [total_num_fields][]const u8 = undefined; + comptime var field_types: [total_num_fields]type = undefined; + comptime var field_defaults: [total_num_fields]?*const anyopaque = undefined; comptime var field_count = 0; - comptime var positional_fields: []const std.builtin.Type.StructField = &.{}; + const positional_infos: ?std.lang.Type.Struct = if (@hasField(Flags, "positional")) + @typeInfo(@FieldType(Flags, "positional")).@"struct" + else + null; - comptime for (std.meta.fields(Flags)) |field| { - if (std.mem.eql(u8, field.name, "positional")) { - assert(@typeInfo(field.type) == .@"struct"); - positional_fields = std.meta.fields(field.type); + comptime for (flags_info.field_names, flags_info.field_types, flags_info.field_attrs) |field_name, FieldType, field_attr| { + if (std.mem.eql(u8, field_name, "positional")) { var optional_tail = false; - for (positional_fields) |positional_field| { - if (default_value(positional_field) == null) { + for (positional_infos.?.field_types, positional_infos.?.field_attrs) |PosType, pos_attr| { + const default: ?PosType = pos_attr.defaultValue(PosType); + if (default == null) { if (optional_tail) @panic("optional positional arguments must be last"); } else { optional_tail = true; } - switch (@typeInfo(positional_field.type)) { + switch (@typeInfo(PosType)) { .optional => |optional| { // optional flags should have a default - assert(default_value(positional_field) != null); - assert(default_value(positional_field).? == null); + assert(default != null); + assert(default.? == null); assert_valid_value_type(optional.child); }, else => { - assert_valid_value_type(positional_field.type); + assert_valid_value_type(PosType); }, } } } else { - fields[field_count] = field; + field_names[field_count] = field_name; + field_types[field_count] = FieldType; + field_defaults[field_count] = field_attr.default_value_ptr; field_count += 1; - switch (@typeInfo(field.type)) { + const default: ?FieldType = field_attr.defaultValue(FieldType); + + switch (@typeInfo(FieldType)) { .bool => { // boolean flags should have a default - debug.assertComptime(default_value(field) != null and default_value(field).? == false, "boolean flag --{s} should default to false", .{field.name}); + debug.assertComptime(default != null and default.? == false, "boolean flag --{s} should default to false", .{field_name}); }, .optional => |optional| { // optional flags should have a default - debug.assertComptime(default_value(field) != null and default_value(field).? == null, "optional flag --{s} should have a null default value", .{field.name}); + debug.assertComptime(default != null and default.? == null, "optional flag --{s} should have a null default value", .{field_name}); assert_valid_value_type(optional.child); }, else => { - assert_valid_value_type(field.type); + assert_valid_value_type(FieldType); }, } } @@ -182,29 +192,33 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { var result: Flags = undefined; // Would use std.enums.EnumFieldStruct(Flags, u32, 0) here but Flags is a struct not an Enum. var counts = comptime blk: { - const f = std.meta.fields(Flags); - var count_field_names: [fields.len][]const u8 = undefined; - var count_field_types: [fields.len]type = undefined; - var count_field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; - for (f, &count_field_names, &count_field_types, &count_field_attrs) |field, *field_name, *field_type, *field_attr| { - field_name.* = field.name; - field_type.* = u32; - field_attr.* = .{ - .@"align" = @alignOf(u32), - .default_value_ptr = @ptrCast(&@as(u32, 0)), - }; - } - break :blk @Struct(.auto, null, &count_field_names, &count_field_types, &count_field_attrs){}; + const count_field_types: [total_num_fields]type = @splat(u32); + const count_field_attrs: [total_num_fields]std.builtin.Type.Struct.FieldAttributes = @splat(.{ + .@"align" = @alignOf(u32), + .default_value_ptr = @ptrCast(&@as(u32, 0)), + }); + break :blk @Struct(.auto, null, flags_info.field_names, &count_field_types, &count_field_attrs){}; }; // When parsing arguments, we must consider longer arguments first, such that `--foo-bar=92` is // not confused for a misspelled `--foo=92`. Using `std.sort` for comptime-only values does not // work, so open-code insertion sort, and comptime assert order during the actual parsing. comptime { - for (fields[0..field_count], 0..) |*field_right, i| { - for (fields[0..i]) |*field_left| { - if (field_left.name.len < field_right.name.len) { - std.mem.swap(std.builtin.Type.StructField, field_left, field_right); + for ( + 0.., + field_names[0..field_count], + field_types[0..field_count], + field_defaults[0..field_count], + ) |i, *name_right, *type_right, *default_right| { + for ( + field_names[0..i], + field_types[0..i], + field_defaults[0..i], + ) |*name_left, *type_left, *default_left| { + if (name_left.len < name_right.len) { + std.mem.swap([]const u8, name_left, name_right); + std.mem.swap(type, type_left, type_right); + std.mem.swap(?*const anyopaque, default_left, default_right); } } } @@ -221,29 +235,29 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { } comptime var field_len_prev = std.math.maxInt(usize); - inline for (fields[0..field_count]) |field| { - const flag = comptime flag_name(field); + inline for (field_names[0..field_count], field_types[0..field_count]) |field_name, FieldType| { + const flag = comptime flag_name(field_name); - comptime assert(field_len_prev >= field.name.len); - field_len_prev = field.name.len; + comptime assert(field_len_prev >= field_name.len); + field_len_prev = field_name.len; if (std.mem.startsWith(u8, arg, flag)) { if (parsed_positional) { fatal("unexpected trailing option: '{s}'", .{arg}); } - @field(counts, field.name) += 1; - const flag_value = parse_flag(field.type, flag, arg); - @field(result, field.name) = flag_value; + @field(counts, field_name) += 1; + const flag_value = parse_flag(FieldType, flag, arg); + @field(result, field_name) = flag_value; continue :next_arg; } } - if (@hasField(Flags, "positional")) { + if (positional_infos) |pos_infos| { counts.positional += 1; switch (counts.positional - 1) { - inline 0...positional_fields.len - 1 => |positional_index| { - const positional_field = positional_fields[positional_index]; - const flag = comptime flag_name_positional(positional_field); + inline 0...pos_infos.field_names.len - 1 => |positional_index| { + const positional_field_name = pos_infos.field_names[positional_index]; + const flag = comptime flag_name_positional(positional_field_name); if (arg.len == 0) fatal("{s}: empty argument", .{flag}); // Prevent ambiguity between a flag and positional argument value. We could add @@ -252,8 +266,8 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { if (arg[0] == '-') fatal("unexpected argument: '{s}'", .{arg}); parsed_positional = true; - @field(result.positional, positional_field.name) = - parse_value(positional_field.type, flag, arg); + @field(result.positional, positional_field_name) = + parse_value(pos_infos.field_types[positional_index], flag, arg); continue :next_arg; }, else => {}, // Fall-through to the unexpected argument error. @@ -263,11 +277,15 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { fatal("unexpected argument: '{s}'", .{arg}); } - inline for (fields[0..field_count]) |field| { - const flag = flag_name(field); - switch (@field(counts, field.name)) { - 0 => if (default_value(field)) |default| { - @field(result, field.name) = default; + inline for ( + field_names[0..field_count], + field_types[0..field_count], + field_defaults[0..field_count], + ) |field_name, FieldType, default_ptr| { + const flag = flag_name(field_name); + switch (@field(counts, field_name)) { + 0 => if (defaultValue(FieldType, default_ptr)) |default| { + @field(result, field_name) = default; } else { fatal("{s}: argument is required", .{flag}); }, @@ -276,15 +294,19 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { } } - if (@hasField(Flags, "positional")) { - assert(counts.positional <= positional_fields.len); - inline for (positional_fields, 0..) |positional_field, positional_index| { + if (positional_infos) |pos_infos| { + assert(counts.positional <= pos_infos.field_names.len); + inline for ( + 0.., + pos_infos.field_names, + pos_infos.field_types, + pos_infos.field_attrs, + ) |positional_index, name, PosFieldType, attr| { if (positional_index >= counts.positional) { - const flag = comptime flag_name_positional(positional_field); - if (default_value(positional_field)) |default| { - @field(result.positional, positional_field.name) = default; + if (attr.defaultValue(PosFieldType)) |default| { + @field(result.positional, name) = default; } else { - fatal("{s}: argument is required", .{flag}); + fatal("<{s}>: argument is required", .{name}); } } } @@ -299,8 +321,8 @@ fn assert_valid_value_type(comptime T: type) void { if (@typeInfo(T) == .@"enum") { const info = @typeInfo(T).@"enum"; - assert(info.is_exhaustive); - assert(info.fields.len >= 2); + assert(info.mode == .exhaustive); + assert(info.field_names.len >= 2); return; } @@ -449,7 +471,7 @@ pub const ByteSize = struct { else ByteUnit.bytes; - _ = std.math.mul(u64, amount, @intFromEnum(unit)) catch { + _ = std.math.mul(u64, amount, @backingInt(unit)) catch { return ByteSizeParseError.BytesOverflow; }; @@ -460,7 +482,7 @@ pub const ByteSize = struct { return std.math.mul( u64, size.value, - @intFromEnum(size.unit), + @backingInt(size.unit), ) catch unreachable; } @@ -542,7 +564,7 @@ fn parse_value_float(comptime T: type, flag: []const u8, value: [:0]const u8) T fn parse_value_enum(comptime E: type, flag: []const u8, value: [:0]const u8) E { assert((flag[0] == '-' and flag[1] == '-') or flag[0] == '<'); - comptime assert(@typeInfo(E).@"enum".is_exhaustive); + comptime assert(@typeInfo(E).@"enum".mode == .exhaustive); return std.meta.stringToEnum(E, value) orelse fatal( "{s}: expected one of {s}, but found '{s}'", @@ -552,53 +574,50 @@ fn parse_value_enum(comptime E: type, flag: []const u8, value: [:0]const u8) E { fn fields_to_comma_list(comptime E: type) []const u8 { comptime { - const field_count = std.meta.fields(E).len; + const field_names = std.meta.fieldNames(E); + const field_count = field_names.len; assert(field_count >= 2); var result: []const u8 = ""; - for (std.meta.fields(E), 0..) |field, field_index| { + for (field_names, 0..) |field_name, field_index| { const separator = switch (field_index) { 0 => "", else => ", ", field_count - 1 => if (field_count == 2) " or " else ", or ", }; - result = result ++ separator ++ "'" ++ field.name ++ "'"; + result = result ++ separator ++ "'" ++ field_name ++ "'"; } return result; } } -pub fn flag_name(comptime field: std.builtin.Type.StructField) []const u8 { +pub fn flag_name(comptime field_name: []const u8) []const u8 { // TODO(Zig): Cleanup when this is fixed after Zig 0.11. // Without comptime blk, the compiler thinks the result is a runtime slice returning a UAF. return comptime blk: { - assert(!std.mem.eql(u8, field.name, "positional")); + assert(!std.mem.eql(u8, field_name, "positional")); var result: []const u8 = "--"; var index = 0; - while (std.mem.indexOf(u8, field.name[index..], "_")) |i| { - result = result ++ field.name[index..][0..i] ++ "-"; + while (std.mem.indexOf(u8, field_name[index..], "_")) |i| { + result = result ++ field_name[index..][0..i] ++ "-"; index = index + i + 1; } - result = result ++ field.name[index..]; + result = result ++ field_name[index..]; break :blk result; }; } test flag_name { - const field = @typeInfo(struct { statsd: bool }).@"struct".fields[0]; - try std.testing.expectEqualStrings(flag_name(field), "--statsd"); + try std.testing.expectEqualStrings(flag_name("statsd"), "--statsd"); } -fn flag_name_positional(comptime field: std.builtin.Type.StructField) []const u8 { - comptime assert(std.mem.indexOf(u8, field.name, "_") == null); - return "<" ++ field.name ++ ">"; +fn flag_name_positional(comptime field_name: []const u8) []const u8 { + comptime assert(std.mem.indexOf(u8, field_name, "_") == null); + return "<" ++ field_name ++ ">"; } -/// This is essentially `field.default_value`, but with a useful type instead of `?*anyopaque`. -pub fn default_value(comptime field: std.builtin.Type.StructField) ?field.type { - return if (field.default_value_ptr) |default_opaque| - @as(*const field.type, @ptrCast(@alignCast(default_opaque))).* - else - null; +pub fn defaultValue(comptime FieldType: type, default_value_ptr: ?*const anyopaque) ?FieldType { + const dp: *const FieldType = @ptrCast(@alignCast(default_value_ptr orelse return null)); + return dp.*; } diff --git a/stdx/fmt.zig b/stdx/fmt.zig index 33351d93..a62d4f7d 100644 --- a/stdx/fmt.zig +++ b/stdx/fmt.zig @@ -222,15 +222,15 @@ fn printValue( switch (@typeInfo(T)) { .@"struct" => |info| { try w.writeAll(".{ "); - inline for (info.fields, 0..) |f, i| { + inline for (info.field_names, 0..) |field_name, i| { if (i > 0) try w.writeAll(", "); if (!info.is_tuple) { try w.writeByte('.'); - try w.writeAll(f.name); + try w.writeAll(field_name); try w.writeAll(" = "); } - try printValue(w, options, @field(value, f.name), max_depth - 1); + try printValue(w, options, @field(value, field_name), max_depth - 1); } try w.writeAll(" }"); }, diff --git a/stdx/json.zig b/stdx/json.zig index d1143efc..c1f71a4a 100644 --- a/stdx/json.zig +++ b/stdx/json.zig @@ -42,22 +42,22 @@ pub fn UnionHelpers(comptime T: type) type { } pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!T { - inline for (std.meta.fields(T)) |field| { - switch (field.type) { - bool => if (source == .bool) return @unionInit(T, field.name, source.bool), + inline for (comptime std.meta.fieldNames(T), comptime std.meta.fieldTypes(T)) |field_name, Field| { + switch (Field) { + bool => if (source == .bool) return @unionInit(T, field_name, source.bool), []const u8 => switch (source) { .string, .number_string => |v| { const dupe = try allocator.dupe(u8, v); - return @unionInit(T, field.name, dupe); + return @unionInit(T, field_name, dupe); }, else => {}, }, - else => switch (@typeInfo(field.type)) { - .int => if (source == .integer) return @unionInit(T, field.name, @intCast(source.integer)), - .float => if (source == .float) return @unionInit(T, field.name, @floatCast(source.float)), - .@"struct" => if (source == .object) return @unionInit(T, field.name, try std.json.innerParseFromValue(field.type, allocator, source, options)), + else => switch (@typeInfo(Field)) { + .int => if (source == .integer) return @unionInit(T, field_name, @intCast(source.integer)), + .float => if (source == .float) return @unionInit(T, field_name, @floatCast(source.float)), + .@"struct" => if (source == .object) return @unionInit(T, field_name, try std.json.innerParseFromValue(Field, allocator, source, options)), inline else => switch (source) { - .number_string, .array => return @unionInit(T, field.name, try std.json.innerParseFromValue(field.type, allocator, source, options)), + .number_string, .array => return @unionInit(T, field_name, try std.json.innerParseFromValue(Field, allocator, source, options)), else => {}, }, }, @@ -130,9 +130,9 @@ pub fn TaggedUnionHelpers(comptime T: type, comptime tag_name: [:0]const u8) typ if (@typeInfo(T) != .@"union") { @compileError("TaggedUnion expects a union type, found " ++ @typeName(T)); } - for (std.meta.fields(T)) |field| { - if (@typeInfo(field.type) != .@"struct") { - @compileError("TaggedUnion member '" ++ field.name ++ "' must be a struct, found " ++ @typeName(field.type)); + for (std.meta.fieldNames(T), std.meta.fieldTypes(T)) |field_name, Field| { + if (@typeInfo(Field) != .@"struct") { + @compileError("TaggedUnion member '" ++ field_name ++ "' must be a struct, found " ++ @typeName(Field)); } } } @@ -152,13 +152,13 @@ pub fn TaggedUnionHelpers(comptime T: type, comptime tag_name: [:0]const u8) typ var o = source.object; const tag = (o.fetchSwapRemove(tag_name) orelse return error.MissingField).value; if (tag != .string) return error.LengthMismatch; - inline for (std.meta.fields(T)) |field| { - if (std.mem.eql(u8, field.name, tag.string)) { - const inner: field.type = std.json.parseFromValueLeaky(field.type, allocator, .{ .object = o }, options) catch |err| { - std.log.warn("failed to interpret {s} as a {s}: {}", .{ tag.string, @typeName(field.type), err }); + inline for (comptime std.meta.fieldNames(T), comptime std.meta.fieldTypes(T)) |field_name, Field| { + if (std.mem.eql(u8, field_name, tag.string)) { + const inner: Field = std.json.parseFromValueLeaky(Field, allocator, .{ .object = o }, options) catch |err| { + std.log.warn("failed to interpret {s} as a {s}: {}", .{ tag.string, @typeName(Field), err }); return err; }; - return @unionInit(T, field.name, inner); + return @unionInit(T, field_name, inner); } } return error.InvalidEnumTag; @@ -172,9 +172,9 @@ pub fn TaggedUnionHelpers(comptime T: type, comptime tag_name: [:0]const u8) typ try jw.objectField(tag_name); try jw.write(@tagName(tag)); switch (@typeInfo(@TypeOf(v))) { - .@"struct" => inline for (std.meta.fields(@TypeOf(v))) |field| { - try jw.objectField(field.name); - try jw.write(@field(v, field.name)); + .@"struct" => inline for (comptime std.meta.fieldNames(@TypeOf(v))) |name| { + try jw.objectField(name); + try jw.write(@field(v, name)); }, else => unreachable, // Would have failed with comptime check. } @@ -230,11 +230,11 @@ pub fn NeverNull(comptime T: type, comptime default_value: T) type { } pub fn fillDefaultStructValues(comptime T: type, r: *T) !void { - inline for (@typeInfo(T).Struct.fields) |field| { - if (field.default_value) |default_ptr| { - if (@field(r, field.name) == null) { - const default = @as(*align(1) const field.type, @ptrCast(default_ptr)).*; - @field(r, field.name) = default; + const info = @typeInfo(T).@"struct"; + inline for (info.field_names, info.field_types, info.field_attrs) |field_name, Field, attrs| { + if (attrs.defaultValue(Field)) |default| { + if (@field(r, field_name) == null) { + @field(r, field_name) = default; } } } diff --git a/stdx/meta.zig b/stdx/meta.zig index cb9aaeba..9b1c05af 100644 --- a/stdx/meta.zig +++ b/stdx/meta.zig @@ -20,8 +20,8 @@ pub fn isTuple(comptime T: type) bool { pub fn isStructOf(comptime T: type, comptime Elem: type) bool { return switch (@typeInfo(T)) { .@"struct" => |info| blk: { - inline for (info.fields) |field| { - if (field.type != Elem) { + inline for (info.field_types) |FieldType| { + if (FieldType != Elem) { break :blk false; } } @@ -34,8 +34,8 @@ pub fn isStructOf(comptime T: type, comptime Elem: type) bool { pub fn isStructOfAny(comptime T: type, comptime f: fn (comptime type) bool) bool { return switch (@typeInfo(T)) { .@"struct" => |info| blk: { - inline for (info.fields) |field| { - if (f(field.type) == false) { + inline for (info.field_types) |FieldType| { + if (f(FieldType) == false) { break :blk false; } } @@ -121,9 +121,9 @@ pub fn asSlice(comptime T: type) type { } pub fn TupleRange(comptime T: type, comptime start: ?usize, comptime end: ?usize) type { - const fields = std.meta.fields(T); + const all_field_types = std.meta.fieldTypes(T); const start_ = start orelse 0; - const end_ = end orelse fields.len; + const end_ = end orelse all_field_types.len; if (start_ == end_) { return @Tuple(&.{}); @@ -131,17 +131,17 @@ pub fn TupleRange(comptime T: type, comptime start: ?usize, comptime end: ?usize var field_types: [end_ - start_]type = undefined; inline for (start_..end_, 0..) |i, j| { - field_types[j] = fields[i].type; + field_types[j] = all_field_types[i]; } return @Tuple(&field_types); } pub fn FnParam(comptime func: anytype, comptime n: comptime_int) type { - const params = @typeInfo(@TypeOf(func)).@"fn".params; + const params = @typeInfo(@TypeOf(func)).@"fn".param_types; if (n >= params.len) { @compileError("param doesn't exist in func"); } - return params[n].type orelse @compileError("anytype is not supported"); + return params[n] orelse @compileError("anytype is not supported"); } pub fn FnReturn(comptime func: anytype) type { @@ -167,8 +167,8 @@ pub fn FnReturnErrorSet(comptime func: anytype) ?type { pub fn Head(comptime Tuple: type) type { return switch (@typeInfo(Tuple)) { .@"struct" => |struct_info| { - if (struct_info.fields.len == 0) @compileError("Can't tail empty tuple"); - return struct_info.fields[0].type; + if (struct_info.field_types.len == 0) @compileError("Can't tail empty tuple"); + return struct_info.field_types[0]; }, else => @compileError("Head works on tuple type"), }; @@ -177,10 +177,10 @@ pub fn Head(comptime Tuple: type) type { pub fn Tail(comptime Tuple: type) type { return switch (@typeInfo(Tuple)) { .@"struct" => |struct_info| { - if (struct_info.fields.len == 0) @compileError("Can't tail empty tuple"); - var types: [struct_info.fields.len - 1]type = undefined; - for (struct_info.fields[1..], &types) |field, *type_| { - type_.* = field.type; + if (struct_info.field_types.len == 0) @compileError("Can't tail empty tuple"); + var types: [struct_info.field_types.len - 1]type = undefined; + for (struct_info.field_types[1..], &types) |FieldType, *type_| { + type_.* = FieldType; } return @Tuple(&types); }, diff --git a/stdx/segmented_list.zig b/stdx/segmented_list.zig index ee115ce2..567534be 100644 --- a/stdx/segmented_list.zig +++ b/stdx/segmented_list.zig @@ -107,7 +107,7 @@ pub fn SegmentedList(comptime T: type, comptime prealloc_item_count: usize) type pub const prealloc_count = prealloc_item_count; fn AtType(comptime SelfType: type) type { - if (@typeInfo(SelfType).pointer.is_const) { + if (@typeInfo(SelfType).pointer.attrs.@"const") { return *const T; } else { return *T; diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 8d1fb590..1b19e638 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -28,7 +28,7 @@ pub inline fn stackSlice(comptime max_len: usize, T: type, len: usize) []T { return storage[0..len]; } -pub const noalloc: std.mem.Allocator = if (builtin.mode == .ReleaseFast) undefined else std.testing.failing_allocator; +pub const noalloc: std.mem.Allocator = if (builtin.mode == .fast) undefined else std.testing.failing_allocator; pub fn arenaWithCapacity(parent: std.mem.Allocator, initial_capacity: usize) std.mem.Allocator.Error!std.heap.ArenaAllocator { var a: std.heap.ArenaAllocator = .init(parent); @@ -43,7 +43,7 @@ pub fn pinToCore(core_id: usize) void { if (builtin.os.tag == .linux) { const CPUSet = std.bit_set.ArrayBitSet(usize, std.os.linux.CPU_SETSIZE * @sizeOf(usize)); - var set: CPUSet = .initEmpty(); + var set: CPUSet = .empty; set.set(core_id); std.os.linux.sched_setaffinity(0, @ptrCast(&set.masks)) catch {}; } @@ -86,3 +86,60 @@ pub fn Once(comptime f: anytype) type { } }; } + +/// Packed storage of sub-byte values +/// +/// Note: before Zig 0.17, @Vector(u2, 4) was always one byte and could be used as a storage type, +/// But that's not true anymore. +pub fn Packed(T: type) type { + return switch (@bitSizeOf(T)) { + 2 => packed struct(u8) { + x: T, + y: T, + z: T, + w: T, + + const P = @This(); + pub fn pack(x: T, y: T, z: T, w: T) P { + return .{ .x = x, .y = y, .z = z, .w = w }; + } + + pub fn unpack(p: P) [4]T { + return .{ p.x, p.y, p.z, p.w }; + } + + pub fn formatNumber(p: P, w: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + try formatPackedNumber(T, &p.unpack(), w, n); + } + }, + 4 => packed struct(u8) { + x: T, + y: T, + + const P = @This(); + pub fn pack(x: T, y: T) P { + return .{ .x = x, .y = y }; + } + + pub fn unpack(p: P) [2]T { + return .{ p.x, p.y }; + } + + pub fn formatNumber(p: P, w: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + try formatPackedNumber(T, &p.unpack(), w, n); + } + }, + else => debug.compileError("stdx.Packed expects a bitsize of 2 or 4, got type {} with bitsize {d}", .{ T, @bitSizeOf(T) }), + }; +} + +fn formatPackedNumber(T: type, values: []const T, w: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + for (0.., values) |i, val| { + if (i > 0) try w.writeByte(','); + switch (@typeInfo(T)) { + .@"struct" => try val.formatNumber(w, n), + .int => try fmt.formatInt(val, n, w), + else => try w.print("{}", .{val}), + } + } +} diff --git a/third_party/arocc/arocc.bazel b/third_party/arocc/arocc.bazel index 853399f5..a5de1d98 100644 --- a/third_party/arocc/arocc.bazel +++ b/third_party/arocc/arocc.bazel @@ -80,19 +80,10 @@ zig_options( ], ) -zig_library( - name = "zig", - srcs = glob(["deps/zig/**/*.zig"]), - main = "deps/zig/lib.zig", -) - zig_library( name = "backend", main = "src/backend.zig", - deps = [ - ":build_options", - ":zig", - ], + deps = [":build_options"], ) zig_library( @@ -127,7 +118,6 @@ zig_binary( ":assembly_backend", ":build_options", ], - zigopts = ["-fm"], ) zig_binary( diff --git a/third_party/arocc/repo.bzl b/third_party/arocc/repo.bzl index 98a10e38..15ff6d7f 100644 --- a/third_party/arocc/repo.bzl +++ b/third_party/arocc/repo.bzl @@ -1,9 +1,10 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") def repo(): + # https://github.com/zml/arocc/releases/tag/0.17.0 git_repository( name = "arocc", - remote = "https://github.com/Vexu/arocc.git", - commit = "5f5a050569a95ecc40a426f0c3666ae7ef987ede", + remote = "https://github.com/zml/arocc", + commit = "d0c8c4d9c55daa7ef6e40cf0f630a5b5e900989b", build_file = Label("//third_party/arocc:arocc.bazel"), ) diff --git a/third_party/libvaxis/bump-zig.patch b/third_party/libvaxis/bump-zig.patch deleted file mode 100644 index 45e221b7..00000000 --- a/third_party/libvaxis/bump-zig.patch +++ /dev/null @@ -1,2019 +0,0 @@ -diff --git a/build.zig.zon b/build.zig.zon -index 6192c1a..4ca04f8 100644 ---- a/build.zig.zon -+++ b/build.zig.zon -@@ -2,15 +2,15 @@ - .name = .vaxis, - .fingerprint = 0x14fbbb94fc556305, - .version = "0.5.1", -- .minimum_zig_version = "0.15.1", -+ .minimum_zig_version = "0.16.0-dev.2261+d6b3dd25a", - .dependencies = .{ - .zigimg = .{ -- .url = "git+https://github.com/zigimg/zigimg#eab2522c023b9259db8b13f2f90d609b7437e5f6", -- .hash = "zigimg-0.1.0-8_eo2vUZFgAAtN1c6dAO5DdqL0d4cEWHtn6iR5ucZJti", -+ .url = "https://github.com/unexge/zigimg/archive/refs/heads/master.zip", -+ .hash = "zigimg-0.1.0-8_eo2tGrFwD-VzWa_E8HuIuNenM2US7PRXBj5EJdGfrw", - }, - .uucode = .{ -- .url = "git+https://github.com/jacobsandlund/uucode#5f05f8f83a75caea201f12cc8ea32a2d82ea9732", -- .hash = "uucode-0.1.0-ZZjBPj96QADXyt5sqwBJUnhaDYs_qBeeKijZvlRa0eqM", -+ .url = "https://github.com/jacobsandlund/uucode/archive/refs/heads/zig-0.16.zip", -+ .hash = "uucode-0.2.0-ZZjBPkJiVABQGPdyw9BOtU2sflHv3vovBzmYtPRj2hvm", - }, - }, - .paths = .{ -diff --git a/examples/cli.zig b/examples/cli.zig -index 3b44676..2b9a288 100644 ---- a/examples/cli.zig -+++ b/examples/cli.zig -@@ -4,30 +4,23 @@ const Cell = vaxis.Cell; - const TextInput = vaxis.widgets.TextInput; - - const log = std.log.scoped(.main); --pub fn main() !void { -- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -- defer { -- const deinit_status = gpa.deinit(); -- if (deinit_status == .leak) { -- log.err("memory leak", .{}); -- } -- } -- const alloc = gpa.allocator(); -+pub fn main(init: std.process.Init) !void { -+ const alloc = init.gpa; - - var buffer: [1024]u8 = undefined; -- var tty = try vaxis.Tty.init(&buffer); -+ var tty = try vaxis.Tty.init(init.io, &buffer); - defer tty.deinit(); - - var vx = try vaxis.init(alloc, .{}); - defer vx.deinit(alloc, tty.writer()); - -- var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx }; -+ var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx, .io = init.io, .queue = .{ .io = init.io } }; - try loop.init(); - - try loop.start(); - defer loop.stop(); - -- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_s); -+ try vx.queryTerminal(tty.writer(), init.io, 1 * std.time.ns_per_s); - - var text_input = TextInput.init(alloc); - defer text_input.deinit(); -diff --git a/examples/counter.zig b/examples/counter.zig -index d19af42..e68420b 100644 ---- a/examples/counter.zig -+++ b/examples/counter.zig -@@ -111,13 +111,10 @@ const Model = struct { - } - }; - --pub fn main() !void { -- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -- defer _ = gpa.deinit(); -+pub fn main(init: std.process.Init) !void { -+ const allocator = init.gpa; - -- const allocator = gpa.allocator(); -- -- var app = try vxfw.App.init(allocator); -+ var app = try vxfw.App.init(allocator, init.io); - defer app.deinit(); - - // We heap allocate our model because we will require a stable pointer to it in our Button -diff --git a/examples/fuzzy.zig b/examples/fuzzy.zig -index 6ffbe6a..7fef980 100644 ---- a/examples/fuzzy.zig -+++ b/examples/fuzzy.zig -@@ -11,7 +11,6 @@ const Model = struct { - - /// Used for filtered RichText Spans and result - arena: std.heap.ArenaAllocator, -- filtered: std.ArrayList(vxfw.RichText), - result: []const u8, - - pub fn init(gpa: std.mem.Allocator) !*Model { -@@ -201,31 +200,24 @@ fn toLower(arena: std.mem.Allocator, src: []const u8) std.mem.Allocator.Error![] - return lower; - } - --pub fn main() !void { -- var debug_allocator = std.heap.GeneralPurposeAllocator(.{}){}; -- defer _ = debug_allocator.deinit(); -- -- const gpa = debug_allocator.allocator(); -+pub fn main(init: std.process.Init) !void { -+ const gpa = init.gpa; - -- var app = try vxfw.App.init(gpa); -+ var app = try vxfw.App.init(gpa, init.io); - errdefer app.deinit(); - - const model = try Model.init(gpa); - defer model.deinit(gpa); - - // Run the command -- var fd = std.process.Child.init(&.{"fd"}, gpa); -- fd.stdout_behavior = .Pipe; -- fd.stderr_behavior = .Pipe; -- var stdout = std.ArrayList(u8).empty; -- var stderr = std.ArrayList(u8).empty; -- defer stdout.deinit(gpa); -- defer stderr.deinit(gpa); -- try fd.spawn(); -- try fd.collectOutput(gpa, &stdout, &stderr, 10_000_000); -- _ = try fd.wait(); -- -- var iter = std.mem.splitScalar(u8, stdout.items, '\n'); -+ const result = try std.process.run(gpa, init.io, .{ -+ .argv = &.{"fd"}, -+ .stdout_limit = .limited(10_000_000), -+ }); -+ defer gpa.free(result.stdout); -+ defer gpa.free(result.stderr); -+ -+ var iter = std.mem.splitScalar(u8, result.stdout, '\n'); - while (iter.next()) |line| { - if (line.len == 0) continue; - try model.list.append(gpa, .{ .text = line }); -@@ -235,8 +227,8 @@ pub fn main() !void { - app.deinit(); - - if (model.result.len > 0) { -- _ = try std.posix.write(std.posix.STDOUT_FILENO, model.result); -- _ = try std.posix.write(std.posix.STDOUT_FILENO, "\n"); -+ try std.Io.File.stdout().writeStreamingAll(init.io, model.result); -+ try std.Io.File.stdout().writeStreamingAll(init.io, "\n"); - } else { - std.process.exit(130); - } -diff --git a/examples/image.zig b/examples/image.zig -index c14bbdb..af62862 100644 ---- a/examples/image.zig -+++ b/examples/image.zig -@@ -8,41 +8,34 @@ const Event = union(enum) { - winsize: vaxis.Winsize, - }; - --pub fn main() !void { -- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -- defer { -- const deinit_status = gpa.deinit(); -- if (deinit_status == .leak) { -- log.err("memory leak", .{}); -- } -- } -- const alloc = gpa.allocator(); -+pub fn main(init: std.process.Init) !void { -+ const alloc = init.gpa; - - var buffer: [1024]u8 = undefined; -- var tty = try vaxis.Tty.init(&buffer); -+ var tty = try vaxis.Tty.init(init.io, &buffer); - defer tty.deinit(); - - var vx = try vaxis.init(alloc, .{}); - defer vx.deinit(alloc, tty.writer()); - -- var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx }; -+ var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx, .io = init.io, .queue = .{ .io = init.io } }; - try loop.init(); - - try loop.start(); - defer loop.stop(); - - try vx.enterAltScreen(tty.writer()); -- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_s); -+ try vx.queryTerminal(tty.writer(), init.io, 1 * std.time.ns_per_s); - - var read_buffer: [1024 * 1024]u8 = undefined; // 1MB buffer -- var img1 = try vaxis.zigimg.Image.fromFilePath(alloc, "examples/zig.png", &read_buffer); -+ var img1 = try vaxis.zigimg.Image.fromFilePath(alloc, init.io, "examples/zig.png", &read_buffer); - defer img1.deinit(alloc); - - const imgs = [_]vaxis.Image{ - try vx.transmitImage(alloc, tty.writer(), &img1, .rgba), - // var img1 = try vaxis.zigimg.Image.fromFilePath(alloc, "examples/zig.png"); -- // try vx.loadImage(alloc, tty.writer(), .{ .path = "examples/zig.png" }), -- try vx.loadImage(alloc, tty.writer(), .{ .path = "examples/vaxis.png" }), -+ // try vx.loadImage(alloc, init.io, tty.writer(), .{ .path = "examples/zig.png" }), -+ try vx.loadImage(alloc, init.io, tty.writer(), .{ .path = "examples/vaxis.png" }), - }; - defer vx.freeImage(tty.writer(), imgs[0].id); - defer vx.freeImage(tty.writer(), imgs[1].id); -diff --git a/examples/list_view.zig b/examples/list_view.zig -index 24cbf8d..5b961e9 100644 ---- a/examples/list_view.zig -+++ b/examples/list_view.zig -@@ -52,13 +52,10 @@ const Model = struct { - } - }; - --pub fn main() !void { -- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -- defer _ = gpa.deinit(); -+pub fn main(init: std.process.Init) !void { -+ const allocator = init.gpa; - -- const allocator = gpa.allocator(); -- -- var app = try vxfw.App.init(allocator); -+ var app = try vxfw.App.init(allocator, init.io); - defer app.deinit(); - - const model = try allocator.create(Model); -diff --git a/examples/main.zig b/examples/main.zig -index 2a4b96b..3caa7d9 100644 ---- a/examples/main.zig -+++ b/examples/main.zig -@@ -3,25 +3,17 @@ const vaxis = @import("vaxis"); - const Cell = vaxis.Cell; - - const log = std.log.scoped(.main); --pub fn main() !void { -- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -- defer { -- const deinit_status = gpa.deinit(); -- //fail test; can't try in defer as defer is executed after we return -- if (deinit_status == .leak) { -- log.err("memory leak", .{}); -- } -- } -- const alloc = gpa.allocator(); -+pub fn main(init: std.process.Init) !void { -+ const alloc = init.gpa; - - var buffer: [1024]u8 = undefined; -- var tty = try vaxis.Tty.init(&buffer); -+ var tty = try vaxis.Tty.init(init.io, &buffer); - defer tty.deinit(); - - var vx = try vaxis.init(alloc, .{}); - defer vx.deinit(alloc, tty.writer()); - -- var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx }; -+ var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx, .io = init.io, .queue = .{ .io = init.io } }; - try loop.init(); - - try loop.start(); -@@ -29,7 +21,7 @@ pub fn main() !void { - - // Optionally enter the alternate screen - try vx.enterAltScreen(tty.writer()); -- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_s); -+ try vx.queryTerminal(tty.writer(), init.io, 1 * std.time.ns_per_s); - - // We'll adjust the color index every keypress - var color_idx: u8 = 0; -diff --git a/examples/scroll.zig b/examples/scroll.zig -index 0f92573..adf1f44 100644 ---- a/examples/scroll.zig -+++ b/examples/scroll.zig -@@ -151,13 +151,10 @@ const Model = struct { - } - }; - --pub fn main() !void { -- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -- defer _ = gpa.deinit(); -+pub fn main(init: std.process.Init) !void { -+ const allocator = init.gpa; - -- const allocator = gpa.allocator(); -- -- var app = try vxfw.App.init(allocator); -+ var app = try vxfw.App.init(allocator, init.io); - errdefer app.deinit(); - - var arena = std.heap.ArenaAllocator.init(allocator); -diff --git a/examples/split_view.zig b/examples/split_view.zig -index 78d4e6f..5c48432 100644 ---- a/examples/split_view.zig -+++ b/examples/split_view.zig -@@ -49,13 +49,10 @@ const Model = struct { - } - }; - --pub fn main() !void { -- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -- defer _ = gpa.deinit(); -+pub fn main(init: std.process.Init) !void { -+ const allocator = init.gpa; - -- const allocator = gpa.allocator(); -- -- var app = try vxfw.App.init(allocator); -+ var app = try vxfw.App.init(allocator, init.io); - defer app.deinit(); - - const model = try allocator.create(Model); -diff --git a/examples/table.zig b/examples/table.zig -index 430c3d5..c489828 100644 ---- a/examples/table.zig -+++ b/examples/table.zig -@@ -14,16 +14,14 @@ const ActiveSection = enum { - btm, - }; - --pub fn main() !void { -- var gpa = heap.GeneralPurposeAllocator(.{}){}; -- defer if (gpa.detectLeaks()) log.err("Memory leak detected!", .{}); -- const alloc = gpa.allocator(); -+pub fn main(init: std.process.Init) !void { -+ const alloc = init.gpa; - - // Users set up below the main function - const users_buf = try alloc.dupe(User, users[0..]); - - var buffer: [1024]u8 = undefined; -- var tty = try vaxis.Tty.init(&buffer); -+ var tty = try vaxis.Tty.init(init.io, &buffer); - defer tty.deinit(); - const tty_writer = tty.writer(); - var vx = try vaxis.init(alloc, .{ -@@ -35,12 +33,12 @@ pub fn main() !void { - key_press: vaxis.Key, - winsize: vaxis.Winsize, - table_upd, -- }) = .{ .tty = &tty, .vaxis = &vx }; -+ }) = .{ .tty = &tty, .vaxis = &vx, .io = init.io, .queue = .{ .io = init.io } }; - try loop.init(); - try loop.start(); - defer loop.stop(); - try vx.enterAltScreen(tty.writer()); -- try vx.queryTerminal(tty.writer(), 250 * std.time.ns_per_ms); -+ try vx.queryTerminal(tty.writer(), init.io, 250 * std.time.ns_per_ms); - - const logo = - \\░█░█░█▀█░█░█░▀█▀░█▀▀░░░▀█▀░█▀█░█▀▄░█░░░█▀▀░ -diff --git a/examples/text_input.zig b/examples/text_input.zig -index cea0466..5c3c6c6 100644 ---- a/examples/text_input.zig -+++ b/examples/text_input.zig -@@ -18,20 +18,12 @@ const Event = union(enum) { - foo: u8, - }; - --pub fn main() !void { -- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -- defer { -- const deinit_status = gpa.deinit(); -- //fail test; can't try in defer as defer is executed after we return -- if (deinit_status == .leak) { -- log.err("memory leak", .{}); -- } -- } -- const alloc = gpa.allocator(); -+pub fn main(init: std.process.Init) !void { -+ const alloc = init.gpa; - - // Initalize a tty - var buffer: [1024]u8 = undefined; -- var tty = try vaxis.Tty.init(&buffer); -+ var tty = try vaxis.Tty.init(init.io, &buffer); - defer tty.deinit(); - - // Use a buffered writer for better performance. There are a lot of writes -@@ -47,6 +39,8 @@ pub fn main() !void { - var loop: vaxis.Loop(Event) = .{ - .vaxis = &vx, - .tty = &tty, -+ .io = init.io, -+ .queue = .{ .io = init.io }, - }; - try loop.init(); - -@@ -71,7 +65,7 @@ pub fn main() !void { - try writer.flush(); - // Sends queries to terminal to detect certain features. This should - // _always_ be called, but is left to the application to decide when -- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_s); -+ try vx.queryTerminal(tty.writer(), init.io, 1 * std.time.ns_per_s); - - // The main event loop. Vaxis provides a thread safe, blocking, buffered - // queue which can serve as the primary event queue for an application -@@ -94,8 +88,10 @@ pub fn main() !void { - } else if (key.matches('n', .{ .ctrl = true })) { - try vx.notify(tty.writer(), "vaxis", "hello from vaxis"); - loop.stop(); -- var child = std.process.Child.init(&.{"nvim"}, alloc); -- _ = try child.spawnAndWait(); -+ var child = try std.process.spawn(init.io, .{ -+ .argv = &.{"nvim"}, -+ }); -+ _ = try child.wait(init.io); - try loop.start(); - try vx.enterAltScreen(tty.writer()); - vx.queueRefresh(); -diff --git a/examples/text_view.zig b/examples/text_view.zig -index 73e8e54..d49cb5b 100644 ---- a/examples/text_view.zig -+++ b/examples/text_view.zig -@@ -9,31 +9,24 @@ const Event = union(enum) { - winsize: vaxis.Winsize, - }; - --pub fn main() !void { -- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -- -- defer { -- const deinit_status = gpa.deinit(); -- if (deinit_status == .leak) { -- log.err("memory leak", .{}); -- } -- } -- -- const alloc = gpa.allocator(); -+pub fn main(init: std.process.Init) !void { -+ const alloc = init.gpa; - var buffer: [1024]u8 = undefined; -- var tty = try vaxis.Tty.init(&buffer); -+ var tty = try vaxis.Tty.init(init.io, &buffer); - defer tty.deinit(); - var vx = try vaxis.init(alloc, .{}); - defer vx.deinit(alloc, tty.writer()); - var loop: vaxis.Loop(Event) = .{ - .vaxis = &vx, - .tty = &tty, -+ .io = init.io, -+ .queue = .{ .io = init.io }, - }; - try loop.init(); - try loop.start(); - defer loop.stop(); - try vx.enterAltScreen(tty.writer()); -- try vx.queryTerminal(tty.writer(), 20 * std.time.ns_per_s); -+ try vx.queryTerminal(tty.writer(), init.io, 20 * std.time.ns_per_s); - var text_view = TextView{}; - var text_view_buffer = TextView.Buffer{}; - defer text_view_buffer.deinit(alloc); -@@ -61,6 +54,6 @@ pub fn main() !void { - win.clear(); - text_view.draw(win, text_view_buffer); - try vx.render(tty.writer()); -- try tty.writer.flush(); -+ try tty.writer().flush(); - } - } -diff --git a/examples/vaxis.zig b/examples/vaxis.zig -index 5ca566c..8a12a66 100644 ---- a/examples/vaxis.zig -+++ b/examples/vaxis.zig -@@ -9,32 +9,24 @@ const Event = union(enum) { - - pub const panic = vaxis.panic_handler; - --pub fn main() !void { -- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -- defer { -- const deinit_status = gpa.deinit(); -- //fail test; can't try in defer as defer is executed after we return -- if (deinit_status == .leak) { -- std.log.err("memory leak", .{}); -- } -- } -- const alloc = gpa.allocator(); -+pub fn main(init: std.process.Init) !void { -+ const alloc = init.gpa; - - var buffer: [1024]u8 = undefined; -- var tty = try vaxis.Tty.init(&buffer); -+ var tty = try vaxis.Tty.init(init.io, &buffer); - defer tty.deinit(); - - var vx = try vaxis.init(alloc, .{}); - defer vx.deinit(alloc, tty.writer()); - -- var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx }; -+ var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx, .io = init.io, .queue = .{ .io = init.io } }; - try loop.init(); - - try loop.start(); - defer loop.stop(); - - try vx.enterAltScreen(tty.writer()); -- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_s); -+ try vx.queryTerminal(tty.writer(), init.io, 1 * std.time.ns_per_s); - - try vx.queryColor(tty.writer(), .fg); - try vx.queryColor(tty.writer(), .bg); -@@ -84,7 +76,7 @@ pub fn main() !void { - // try vx.render(bw.writer().any()); - // try bw.flush(); - try vx.render(tty.writer()); -- std.Thread.sleep(16 * std.time.ns_per_ms); -+ try std.Io.sleep(init.io, std.Io.Duration.fromMilliseconds(16), .real); - switch (dir) { - .up => { - pct += 1; -diff --git a/examples/view.zig b/examples/view.zig -index 1b01dfd..3f5108a 100644 ---- a/examples/view.zig -+++ b/examples/view.zig -@@ -13,15 +13,8 @@ const Event = union(enum) { - winsize: vaxis.Winsize, - }; - --pub fn main() !void { -- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -- defer { -- const deinit_status = gpa.deinit(); -- if (deinit_status == .leak) { -- log.err("memory leak", .{}); -- } -- } -- const alloc = gpa.allocator(); -+pub fn main(init: std.process.Init) !void { -+ const alloc = init.gpa; - - var world_map: []const u8 = lg_world_map; - var map_width = lg_map_width; -@@ -45,7 +38,7 @@ pub fn main() !void { - }); - - var buffer: [1024]u8 = undefined; -- var tty = try vaxis.Tty.init(&buffer); -+ var tty = try vaxis.Tty.init(init.io, &buffer); - defer tty.deinit(); - - const writer = tty.writer(); -@@ -58,13 +51,15 @@ pub fn main() !void { - var loop: vaxis.Loop(Event) = .{ - .vaxis = &vx, - .tty = &tty, -+ .io = init.io, -+ .queue = .{ .io = init.io }, - }; - try loop.init(); - try loop.start(); - defer loop.stop(); - try vx.enterAltScreen(writer); - try writer.flush(); -- try vx.queryTerminal(tty.writer(), 20 * std.time.ns_per_s); -+ try vx.queryTerminal(tty.writer(), init.io, 20 * std.time.ns_per_s); - - // Initialize Views - // - Large Map -diff --git a/examples/vt.zig b/examples/vt.zig -index fc13149..6d7f647 100644 ---- a/examples/vt.zig -+++ b/examples/vt.zig -@@ -9,33 +9,24 @@ const Event = union(enum) { - - pub const panic = vaxis.panic_handler; - --pub fn main() !void { -- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -- defer { -- const deinit_status = gpa.deinit(); -- //fail test; can't try in defer as defer is executed after we return -- if (deinit_status == .leak) { -- std.log.err("memory leak", .{}); -- } -- } -- const alloc = gpa.allocator(); -+pub fn main(init: std.process.Init) !void { -+ const alloc = init.gpa; - - var buffer: [1024]u8 = undefined; -- var tty = try vaxis.Tty.init(&buffer); -+ var tty = try vaxis.Tty.init(init.io, &buffer); - const writer = tty.writer(); - var vx = try vaxis.init(alloc, .{}); - defer vx.deinit(alloc, writer); - -- var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx }; -+ var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx, .io = init.io, .queue = .{ .io = init.io } }; - try loop.init(); - - try loop.start(); - defer loop.stop(); - - try vx.enterAltScreen(writer); -- try vx.queryTerminal(writer, 1 * std.time.ns_per_s); -- var env = try std.process.getEnvMap(alloc); -- defer env.deinit(); -+ try vx.queryTerminal(writer, init.io, 1 * std.time.ns_per_s); -+ const env = init.environ_map; - - const vt_opts: vaxis.widgets.Terminal.Options = .{ - .winsize = .{ -@@ -52,8 +43,9 @@ pub fn main() !void { - var write_buf: [4096]u8 = undefined; - var vt = try vaxis.widgets.Terminal.init( - alloc, -+ init.io, - &argv, -- &env, -+ env, - vt_opts, - &write_buf, - ); -@@ -62,7 +54,7 @@ pub fn main() !void { - - var redraw: bool = false; - while (true) { -- std.Thread.sleep(8 * std.time.ns_per_ms); -+ std.Io.sleep(init.io, .fromMilliseconds(8), .awake) catch {}; - // try vt events first - while (vt.tryEvent()) |event| { - redraw = true; -diff --git a/src/Loop.zig b/src/Loop.zig -index a11e2f7..cd7e9d9 100644 ---- a/src/Loop.zig -+++ b/src/Loop.zig -@@ -18,8 +18,9 @@ pub fn Loop(comptime T: type) type { - - tty: *Tty, - vaxis: *Vaxis, -+ io: std.Io, - -- queue: Queue(T, 512) = .{}, -+ queue: Queue(T, 512), - thread: ?std.Thread = null, - should_quit: bool = false, - -@@ -214,7 +215,7 @@ pub fn handleEventGeneric(self: anytype, vx: *Vaxis, cache: *GraphemeCache, Even - } - }, - .cap_da1 => { -- std.Thread.Futex.wake(&vx.query_futex, 10); -+ std.Io.futexWake(self.io, u32, &vx.query_futex.raw, 10); - vx.queries_done.store(true, .unordered); - }, - .mouse => |mouse| { -@@ -359,7 +360,7 @@ pub fn handleEventGeneric(self: anytype, vx: *Vaxis, cache: *GraphemeCache, Even - vx.caps.multi_cursor = true; - }, - .cap_da1 => { -- std.Thread.Futex.wake(&vx.query_futex, 10); -+ std.Io.futexWake(self.io, u32, &vx.query_futex.raw, 10); - vx.queries_done.store(true, .unordered); - }, - .winsize => |winsize| { -@@ -386,13 +387,13 @@ test Loop { - foo: u8, - }; - -- var tty = try vaxis.Tty.init(&.{}); -+ var tty = try vaxis.Tty.init(std.testing.io); - defer tty.deinit(); - - var vx = try vaxis.init(std.testing.allocator, .{}); - defer vx.deinit(std.testing.allocator, tty.writer()); - -- var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx }; -+ var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx, .io = std.testing.io, .queue = .{ .io = std.testing.io } }; - try loop.init(); - - try loop.start(); -@@ -400,5 +401,5 @@ test Loop { - - // Optionally enter the alternate screen - try vx.enterAltScreen(tty.writer()); -- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_ms); -+ try vx.queryTerminal(tty.writer(), std.testing.io, 1 * std.time.ns_per_ms); - } -diff --git a/src/Parser.zig b/src/Parser.zig -index 6358269..1b125dd 100644 ---- a/src/Parser.zig -+++ b/src/Parser.zig -@@ -118,7 +118,7 @@ inline fn parseGround(input: []const u8) !Result { - var grapheme_len: usize = 0; - var cp_count: usize = 0; - -- while (grapheme_iter.next()) |result| { -+ while (grapheme_iter.nextCodePoint()) |result| { - cp_count += 1; - if (result.is_break) { - // Found the first grapheme boundary -diff --git a/src/Vaxis.zig b/src/Vaxis.zig -index a6a3830..2e24e9b 100644 ---- a/src/Vaxis.zig -+++ b/src/Vaxis.zig -@@ -3,7 +3,7 @@ const builtin = @import("builtin"); - const atomic = std.atomic; - const base64Encoder = std.base64.standard.Encoder; - const zigimg = @import("zigimg"); --const IoWriter = std.io.Writer; -+const IoWriter = std.Io.Writer; - - const Cell = @import("Cell.zig"); - const Image = @import("Image.zig"); -@@ -250,10 +250,10 @@ pub fn exitAltScreen(self: *Vaxis, tty: *IoWriter) !void { - /// - /// This call will block until Vaxis.query_futex is woken up, or the timeout. - /// Event loops can wake up this futex when cap_da1 is received --pub fn queryTerminal(self: *Vaxis, tty: *IoWriter, timeout_ns: u64) !void { -+pub fn queryTerminal(self: *Vaxis, tty: *IoWriter, io: std.Io, timeout_ns: u64) !void { - try self.queryTerminalSend(tty); - // 1 second timeout -- std.Thread.Futex.timedWait(&self.query_futex, 0, timeout_ns) catch {}; -+ std.Io.futexWaitTimeout(io, u32, &self.query_futex.raw, 0, .{ .duration = .{ .raw = .{ .nanoseconds = @intCast(timeout_ns) }, .clock = .awake } }) catch {}; - self.queries_done.store(true, .unordered); - try self.enableDetectedFeatures(tty); - } -@@ -265,7 +265,7 @@ pub fn queryTerminalSend(vx: *Vaxis, tty: *IoWriter) !void { - vx.queries_done.store(false, .unordered); - - // TODO: re-enable this -- // const colorterm = std.posix.getenv("COLORTERM") orelse ""; -+ // const colorterm = std.c.getenv("COLORTERM") orelse ""; - // if (std.mem.eql(u8, colorterm, "truecolor") or - // std.mem.eql(u8, colorterm, "24bit")) - // { -@@ -320,22 +320,22 @@ pub fn enableDetectedFeatures(self: *Vaxis, tty: *IoWriter) !void { - }, - else => { - // Apply any environment variables -- if (std.posix.getenv("TERMUX_VERSION")) |_| -+ if (std.c.getenv("TERMUX_VERSION")) |_| - self.sgr = .legacy; -- if (std.posix.getenv("VHS_RECORD")) |_| { -+ if (std.c.getenv("VHS_RECORD")) |_| { - self.caps.unicode = .wcwidth; - self.caps.kitty_keyboard = false; - self.sgr = .legacy; - } -- if (std.posix.getenv("TERM_PROGRAM")) |prg| { -- if (std.mem.eql(u8, prg, "vscode")) -+ if (std.c.getenv("TERM_PROGRAM")) |prg| { -+ if (std.mem.eql(u8, std.mem.span(prg), "vscode")) - self.sgr = .legacy; - } -- if (std.posix.getenv("VAXIS_FORCE_LEGACY_SGR")) |_| -+ if (std.c.getenv("VAXIS_FORCE_LEGACY_SGR")) |_| - self.sgr = .legacy; -- if (std.posix.getenv("VAXIS_FORCE_WCWIDTH")) |_| -+ if (std.c.getenv("VAXIS_FORCE_WCWIDTH")) |_| - self.caps.unicode = .wcwidth; -- if (std.posix.getenv("VAXIS_FORCE_UNICODE")) |_| -+ if (std.c.getenv("VAXIS_FORCE_UNICODE")) |_| - self.caps.unicode = .unicode; - - // enable detected features -@@ -1067,6 +1067,7 @@ pub fn transmitImage( - pub fn loadImage( - self: *Vaxis, - alloc: std.mem.Allocator, -+ io: std.Io, - tty: *IoWriter, - src: Image.Source, - ) !Image { -@@ -1074,7 +1075,7 @@ pub fn loadImage( - - var read_buffer: [1024 * 1024]u8 = undefined; // 1MB buffer - var img = switch (src) { -- .path => |path| try zigimg.Image.fromFilePath(alloc, path, &read_buffer), -+ .path => |path| try zigimg.Image.fromFilePath(alloc, io, path, &read_buffer), - .mem => |bytes| try zigimg.Image.fromMemory(alloc, bytes), - }; - defer img.deinit(alloc); -@@ -1481,7 +1482,7 @@ pub fn setTerminalWorkingDirectory(_: *Vaxis, tty: *IoWriter, path: []const u8) - return error.InvalidAbsolutePath; - const hostname = switch (builtin.os.tag) { - .windows => null, -- else => std.posix.getenv("HOSTNAME"), -+ else => std.c.getenv("HOSTNAME"), - } orelse "localhost"; - - const uri: std.Uri = .{ -@@ -1495,11 +1496,11 @@ pub fn setTerminalWorkingDirectory(_: *Vaxis, tty: *IoWriter, path: []const u8) - - test "render: no output when no changes" { - var vx = try Vaxis.init(std.testing.allocator, .{}); -- var deinit_writer = std.io.Writer.Allocating.init(std.testing.allocator); -+ var deinit_writer: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer deinit_writer.deinit(); - defer vx.deinit(std.testing.allocator, &deinit_writer.writer); - -- var render_writer = std.io.Writer.Allocating.init(std.testing.allocator); -+ var render_writer: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer render_writer.deinit(); - try vx.render(&render_writer.writer); - const output = try render_writer.toOwnedSlice(); -diff --git a/src/gwidth.zig b/src/gwidth.zig -index 194139f..ecffaf7 100644 ---- a/src/gwidth.zig -+++ b/src/gwidth.zig -@@ -55,10 +55,10 @@ pub fn gwidth(str: []const u8, method: Method) u16 { - var grapheme_start: usize = 0; - var prev_break: bool = true; - -- while (grapheme_iter.next()) |result| { -+ while (grapheme_iter.nextCodePoint()) |result| { - if (prev_break and !result.is_break) { - // Start of a new grapheme -- const cp_len: usize = std.unicode.utf8CodepointSequenceLength(result.cp) catch 1; -+ const cp_len: usize = std.unicode.utf8CodepointSequenceLength(result.code_point) catch 1; - grapheme_start = grapheme_iter.i - cp_len; - } - -diff --git a/src/queue.zig b/src/queue.zig -index 6d847ea..908cf70 100644 ---- a/src/queue.zig -+++ b/src/queue.zig -@@ -1,7 +1,7 @@ - const std = @import("std"); - const assert = std.debug.assert; - const atomic = std.atomic; --const Condition = std.Thread.Condition; -+const Condition = std.Io.Condition; - - /// Thread safe. Fixed size. Blocking push and pop. - pub fn Queue( -@@ -11,23 +11,25 @@ pub fn Queue( - return struct { - buf: [size]T = undefined, - -+ io: std.Io, -+ - read_index: usize = 0, - write_index: usize = 0, - -- mutex: std.Thread.Mutex = .{}, -+ mutex: std.Io.Mutex = .init, - // blocks when the buffer is full -- not_full: Condition = .{}, -+ not_full: Condition = .init, - // ...or empty -- not_empty: Condition = .{}, -+ not_empty: Condition = .init, - - const Self = @This(); - - /// Pop an item from the queue. Blocks until an item is available. - pub fn pop(self: *Self) T { -- self.mutex.lock(); -- defer self.mutex.unlock(); -+ self.mutex.lockUncancelable(self.io); -+ defer self.mutex.unlock(self.io); - while (self.isEmptyLH()) { -- self.not_empty.wait(&self.mutex); -+ self.not_empty.waitUncancelable(self.io, &self.mutex); - } - std.debug.assert(!self.isEmptyLH()); - return self.popAndSignalLH(); -@@ -36,10 +38,10 @@ pub fn Queue( - /// Push an item into the queue. Blocks until an item has been - /// put in the queue. - pub fn push(self: *Self, item: T) void { -- self.mutex.lock(); -- defer self.mutex.unlock(); -+ self.mutex.lockUncancelable(self.io); -+ defer self.mutex.unlock(self.io); - while (self.isFullLH()) { -- self.not_full.wait(&self.mutex); -+ self.not_full.waitUncancelable(self.io, &self.mutex); - } - std.debug.assert(!self.isFullLH()); - self.pushAndSignalLH(item); -@@ -49,8 +51,8 @@ pub fn Queue( - /// was successfully placed in the queue, false if the queue - /// was full. - pub fn tryPush(self: *Self, item: T) bool { -- self.mutex.lock(); -- defer self.mutex.unlock(); -+ self.mutex.lockUncancelable(self.io); -+ defer self.mutex.unlock(self.io); - if (self.isFullLH()) return false; - self.pushAndSignalLH(item); - return true; -@@ -59,28 +61,28 @@ pub fn Queue( - /// Pop an item from the queue. Returns null when no item is - /// available. - pub fn tryPop(self: *Self) ?T { -- self.mutex.lock(); -- defer self.mutex.unlock(); -+ self.mutex.lockUncancelable(self.io); -+ defer self.mutex.unlock(self.io); - if (self.isEmptyLH()) return null; - return self.popAndSignalLH(); - } - - /// Poll the queue. This call blocks until events are in the queue - pub fn poll(self: *Self) void { -- self.mutex.lock(); -- defer self.mutex.unlock(); -+ self.mutex.lockUncancelable(self.io); -+ defer self.mutex.unlock(self.io); - while (self.isEmptyLH()) { -- self.not_empty.wait(&self.mutex); -+ self.not_empty.waitUncancelable(self.io, &self.mutex); - } - std.debug.assert(!self.isEmptyLH()); - } - - pub fn lock(self: *Self) void { -- self.mutex.lock(); -+ self.mutex.lockUncancelable(self.io); - } - - pub fn unlock(self: *Self) void { -- self.mutex.unlock(); -+ self.mutex.unlock(self.io); - } - - /// Used to efficiently drain the queue while the lock is externally held -@@ -92,7 +94,7 @@ pub fn Queue( - const was_full = self.isFullLH(); - const item = self.popLH(); - if (was_full) { -- self.not_full.signal(); -+ self.not_full.signal(self.io); - } - return item; - } -@@ -108,15 +110,15 @@ pub fn Queue( - - /// Returns `true` if the queue is empty and `false` otherwise. - pub fn isEmpty(self: *Self) bool { -- self.mutex.lock(); -- defer self.mutex.unlock(); -+ self.mutex.lockUncancelable(self.io); -+ defer self.mutex.unlock(self.io); - return self.isEmptyLH(); - } - - /// Returns `true` if the queue is full and `false` otherwise. - pub fn isFull(self: *Self) bool { -- self.mutex.lock(); -- defer self.mutex.unlock(); -+ self.mutex.lockUncancelable(self.io); -+ defer self.mutex.unlock(self.io); - return self.isFullLH(); - } - -@@ -143,7 +145,7 @@ pub fn Queue( - self.buf[self.mask(self.write_index)] = item; - self.write_index = self.mask2(self.write_index + 1); - if (was_empty) { -- self.not_empty.signal(); -+ self.not_empty.signal(self.io); - } - } - -@@ -151,7 +153,7 @@ pub fn Queue( - const was_full = self.isFullLH(); - const result = self.popLH(); - if (was_full) { -- self.not_full.signal(); -+ self.not_full.signal(self.io); - } - return result; - } -@@ -167,7 +169,7 @@ pub fn Queue( - const testing = std.testing; - const cfg = Thread.SpawnConfig{ .allocator = testing.allocator }; - test "Queue: simple push / pop" { -- var queue: Queue(u8, 16) = .{}; -+ var queue: Queue(u8, 16) = .{ .io = std.testing.io }; - queue.push(1); - queue.push(2); - const pop = queue.pop(); -@@ -182,7 +184,7 @@ fn testPushPop(q: *Queue(u8, 2)) !void { - } - - test "Fill, wait to push, pop once in another thread" { -- var queue: Queue(u8, 2) = .{}; -+ var queue: Queue(u8, 2) = .{ .io = std.testing.io }; - queue.push(1); - queue.push(2); - const t = try Thread.spawn(cfg, testPushPop, .{&queue}); -@@ -202,7 +204,7 @@ fn testPush(q: *Queue(u8, 2)) void { - } - - test "Try to pop, fill from another thread" { -- var queue: Queue(u8, 2) = .{}; -+ var queue: Queue(u8, 2) = .{ .io = std.testing.io }; - const thread = try Thread.spawn(cfg, testPush, .{&queue}); - for (0..5) |idx| { - try testing.expectEqual(@as(u8, @intCast(idx)), queue.pop()); -@@ -217,8 +219,8 @@ fn sleepyPop(q: *Queue(u8, 2), state: *atomic.Value(u8)) !void { - - // Then we spuriously wake it up, because that's a thing that can - // happen. -- q.not_full.signal(); -- q.not_empty.signal(); -+ q.not_full.signal(q.io); -+ q.not_empty.signal(q.io); - - // Then give the other thread a good chance of waking up. It's not - // clear that yield guarantees the other thread will be scheduled, -@@ -226,7 +228,7 @@ fn sleepyPop(q: *Queue(u8, 2), state: *atomic.Value(u8)) !void { - // still full and the push in the other thread is still blocked - // waiting for space. - try Thread.yield(); -- std.Thread.sleep(10 * std.time.ns_per_ms); -+ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; - // Finally, let that other thread go. - try std.testing.expectEqual(1, q.pop()); - -@@ -235,15 +237,15 @@ fn sleepyPop(q: *Queue(u8, 2), state: *atomic.Value(u8)) !void { - try Thread.yield(); - // But we want to ensure that there's a second push waiting, so - // here's another sleep. -- std.Thread.sleep(10 * std.time.ns_per_ms); -+ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; - - // Another spurious wake... -- q.not_full.signal(); -- q.not_empty.signal(); -+ q.not_full.signal(q.io); -+ q.not_empty.signal(q.io); - // And another chance for the other thread to see that it's - // spurious and go back to sleep. - try Thread.yield(); -- std.Thread.sleep(10 * std.time.ns_per_ms); -+ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; - - // Pop that thing and we're done. - try std.testing.expectEqual(2, q.pop()); -@@ -256,19 +258,20 @@ test "Fill, block, fill, block" { - // that too (after some time) then drain the queue. This test - // fails if the while loop in `push` is turned into an `if`. - -- var queue: Queue(u8, 2) = .{}; -+ var queue: Queue(u8, 2) = .{ .io = std.testing.io }; - var state = atomic.Value(u8).init(0); - const thread = try Thread.spawn(cfg, sleepyPop, .{ &queue, &state }); - queue.push(1); - queue.push(2); - state.store(1, .release); -- const now = std.time.milliTimestamp(); -+ const start = std.Io.Clock.Timestamp.now(std.testing.io, .awake); - queue.push(3); // This one should block. -- const then = std.time.milliTimestamp(); -+ const end = std.Io.Clock.Timestamp.now(std.testing.io, .awake); - - // Just to make sure the sleeps are yielding to this thread, make - // sure it took at least 5ms to do the push. -- try std.testing.expect(then - now > 5); -+ const elapsed = start.durationTo(end); -+ try std.testing.expect(elapsed.raw.toMilliseconds() > 5); - - state.store(2, .release); - // This should block again, waiting for the other thread. -@@ -283,14 +286,14 @@ test "Fill, block, fill, block" { - fn sleepyPush(q: *Queue(u8, 1), state: *atomic.Value(u8)) !void { - // Try to ensure the other thread has already started trying to pop. - try Thread.yield(); -- std.Thread.sleep(10 * std.time.ns_per_ms); -+ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; - - // Spurious wake -- q.not_full.signal(); -- q.not_empty.signal(); -+ q.not_full.signal(q.io); -+ q.not_empty.signal(q.io); - - try Thread.yield(); -- std.Thread.sleep(10 * std.time.ns_per_ms); -+ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; - - // Stick something in the queue so it can be popped. - q.push(1); -@@ -299,11 +302,11 @@ fn sleepyPush(q: *Queue(u8, 1), state: *atomic.Value(u8)) !void { - try Thread.yield(); - // Give the other thread time to block again. - try Thread.yield(); -- std.Thread.sleep(10 * std.time.ns_per_ms); -+ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; - - // Spurious wake -- q.not_full.signal(); -- q.not_empty.signal(); -+ q.not_full.signal(q.io); -+ q.not_empty.signal(q.io); - - q.push(2); - } -@@ -313,7 +316,7 @@ test "Drain, block, drain, block" { - // test should fail if the `while` loop in `pop` is turned into an - // `if`. - -- var queue: Queue(u8, 1) = .{}; -+ var queue: Queue(u8, 1) = .{ .io = std.testing.io }; - var state = atomic.Value(u8).init(0); - const thread = try Thread.spawn(cfg, sleepyPush, .{ &queue, &state }); - try std.testing.expectEqual(1, queue.pop()); -@@ -328,11 +331,11 @@ fn readerThread(q: *Queue(u8, 1)) !void { - - test "2 readers" { - // 2 threads read, one thread writes -- var queue: Queue(u8, 1) = .{}; -+ var queue: Queue(u8, 1) = .{ .io = std.testing.io }; - const t1 = try Thread.spawn(cfg, readerThread, .{&queue}); - const t2 = try Thread.spawn(cfg, readerThread, .{&queue}); - try Thread.yield(); -- std.Thread.sleep(10 * std.time.ns_per_ms); -+ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; - queue.push(1); - queue.push(1); - t1.join(); -@@ -344,7 +347,7 @@ fn writerThread(q: *Queue(u8, 1)) !void { - } - - test "2 writers" { -- var queue: Queue(u8, 1) = .{}; -+ var queue: Queue(u8, 1) = .{ .io = std.testing.io }; - const t1 = try Thread.spawn(cfg, writerThread, .{&queue}); - const t2 = try Thread.spawn(cfg, writerThread, .{&queue}); - -diff --git a/src/tty.zig b/src/tty.zig -index 4a3c142..c1da795 100644 ---- a/src/tty.zig -+++ b/src/tty.zig -@@ -33,8 +33,11 @@ pub const PosixTty = struct { - /// The file descriptor of the tty - fd: posix.fd_t, - -+ io: std.Io, -+ - /// File.Writer for efficient buffered writing -- tty_writer: std.fs.File.Writer, -+ tty_buffer: [1024]u8 = undefined, -+ tty_writer: ?std.Io.File.Writer = null, - - pub const SignalHandler = struct { - context: *anyopaque, -@@ -43,7 +46,8 @@ pub const PosixTty = struct { - - /// global signal handlers - var handlers: [8]SignalHandler = undefined; -- var handler_mutex: std.Thread.Mutex = .{}; -+ var handler_mutex: std.Io.Mutex = .init; -+ var handler_io: std.Io = undefined; - var handler_idx: usize = 0; - - var handler_installed: bool = false; -@@ -51,9 +55,9 @@ pub const PosixTty = struct { - /// initializes a Tty instance by opening /dev/tty and "making it raw". A - /// signal handler is installed for SIGWINCH. No callbacks are installed, be - /// sure to register a callback when initializing the event loop -- pub fn init(buffer: []u8) !PosixTty { -+ pub fn init(io: std.Io) !PosixTty { - // Open our tty -- const fd = try posix.open("/dev/tty", .{ .ACCMODE = .RDWR }, 0); -+ const fd = try posix.openat(posix.AT.FDCWD, "/dev/tty", .{ .ACCMODE = .RDWR }, 0); - - // Set the termios of the tty - const termios = try makeRaw(fd); -@@ -69,13 +73,8 @@ pub const PosixTty = struct { - posix.sigaction(posix.SIG.WINCH, &act, null); - handler_installed = true; - -- const file = std.fs.File{ .handle = fd }; -- -- const self: PosixTty = .{ -- .fd = fd, -- .termios = termios, -- .tty_writer = .initStreaming(file, buffer), -- }; -+ handler_io = io; -+ const self: PosixTty = .{ .fd = fd, .io = io, .termios = termios }; - - global_tty = self; - -@@ -88,7 +87,7 @@ pub const PosixTty = struct { - std.log.err("couldn't restore terminal: {}", .{err}); - }; - if (builtin.os.tag != .macos) // closing /dev/tty may block indefinitely on macos -- posix.close(self.fd); -+ (std.Io.File{ .handle = self.fd, .flags = .{ .nonblocking = false } }).close(handler_io); - } - - /// Resets the signal handler to it's default -@@ -107,7 +106,12 @@ pub const PosixTty = struct { - } - - pub fn writer(self: *PosixTty) *std.Io.Writer { -- return &self.tty_writer.interface; -+ if (self.tty_writer == null) { -+ const file: std.Io.File = .{ .handle = self.fd, .flags = .{ .nonblocking = false } }; -+ self.tty_writer = .initStreaming(file, self.io, &self.tty_buffer); -+ } -+ -+ return &self.tty_writer.?.interface; - } - - pub fn read(self: *const PosixTty, buf: []u8) !usize { -@@ -117,16 +121,16 @@ pub const PosixTty = struct { - /// Install a signal handler for winsize. A maximum of 8 handlers may be - /// installed - pub fn notifyWinsize(handler: SignalHandler) !void { -- handler_mutex.lock(); -- defer handler_mutex.unlock(); -+ handler_mutex.lockUncancelable(handler_io); -+ defer handler_mutex.unlock(handler_io); - if (handler_idx == handlers.len) return error.OutOfMemory; - handlers[handler_idx] = handler; - handler_idx += 1; - } - -- fn handleWinch(_: c_int) callconv(.c) void { -- handler_mutex.lock(); -- defer handler_mutex.unlock(); -+ fn handleWinch(_: posix.SIG) callconv(.c) void { -+ if (!handler_mutex.tryLock()) return; -+ defer handler_mutex.unlock(handler_io); - var i: usize = 0; - while (i < handler_idx) : (i += 1) { - const handler = handlers[i]; -@@ -189,6 +193,7 @@ pub const PosixTty = struct { - pub const WindowsTty = struct { - stdin: windows.HANDLE, - stdout: windows.HANDLE, -+ io: std.Io, - - initial_codepage: c_uint, - initial_input_mode: CONSOLE_MODE_INPUT, -@@ -198,7 +203,8 @@ pub const WindowsTty = struct { - buf: [4]u8 = undefined, - - /// File.Writer for efficient buffered writing -- tty_writer: std.fs.File.Writer, -+ tty_buffer: [1024]u8 = undefined, -+ tty_writer: ?std.Io.File.Writer = null, - - /// The last mouse button that was pressed. We store the previous state of button presses on each - /// mouse event so we can detect which button was released -@@ -221,7 +227,7 @@ pub const WindowsTty = struct { - .ENABLE_LVB_GRID_WORLDWIDE = 1, // enables reverse video and underline - }; - -- pub fn init(buffer: []u8) !Tty { -+ pub fn init(io: std.Io) !Tty { - const stdin: std.fs.File = .stdin(); - const stdout: std.fs.File = .stdout(); - -@@ -237,12 +243,12 @@ pub const WindowsTty = struct { - return windows.unexpectedError(windows.kernel32.GetLastError()); - - const self: Tty = .{ -+ .io = io, - .stdin = stdin.handle, - .stdout = stdout.handle, - .initial_codepage = initial_output_codepage, - .initial_input_mode = initial_input_mode, - .initial_output_mode = initial_output_mode, -- .tty_writer = .initStreaming(stdout, buffer), - }; - - // save a copy of this tty as the global_tty for panic handling -@@ -298,7 +304,12 @@ pub const WindowsTty = struct { - } - - pub fn writer(self: *Tty) *std.Io.Writer { -- return &self.tty_writer.interface; -+ if (self.tty_writer == null) { -+ const stdout: std.fs.File = .stdout(); -+ self.tty_writer = .initStreaming(stdout, &self.tty_buffer); -+ } -+ -+ return &self.tty_writer.?.interface; - } - - pub fn read(self: *const Tty, buf: []u8) !usize { -@@ -694,20 +705,20 @@ pub const WindowsTty = struct { - pub const TestTty = struct { - /// Used for API compat - fd: posix.fd_t, -+ io: std.Io, - pipe_read: posix.fd_t, - pipe_write: posix.fd_t, - tty_writer: *std.Io.Writer.Allocating, - - /// Initializes a TestTty. -- pub fn init(buffer: []u8) !TestTty { -- _ = buffer; -- -+ pub fn init(io: std.Io) !TestTty { - if (builtin.os.tag == .windows) return error.SkipZigTest; - const list = try std.testing.allocator.create(std.Io.Writer.Allocating); - list.* = .init(std.testing.allocator); -- const r, const w = try posix.pipe(); -+ const r, const w = try std.Io.Threaded.pipe2(.{}); - return .{ - .fd = r, -+ .io = io, - .pipe_read = r, - .pipe_write = w, - .tty_writer = list, -@@ -715,8 +726,9 @@ pub const TestTty = struct { - } - - pub fn deinit(self: TestTty) void { -- std.posix.close(self.pipe_read); -- std.posix.close(self.pipe_write); -+ const io = std.testing.io; -+ (std.Io.File{ .handle = self.pipe_read, .flags = .{ .nonblocking = false } }).close(io); -+ (std.Io.File{ .handle = self.pipe_write, .flags = .{ .nonblocking = false } }).close(io); - self.tty_writer.deinit(); - std.testing.allocator.destroy(self.tty_writer); - } -diff --git a/src/unicode.zig b/src/unicode.zig -index a5253ac..4ff2e4c 100644 ---- a/src/unicode.zig -+++ b/src/unicode.zig -@@ -26,10 +26,10 @@ pub const GraphemeIterator = struct { - } - - pub fn next(self: *GraphemeIterator) ?Grapheme { -- while (self.inner.next()) |res| { -+ while (self.inner.nextCodePoint()) |res| { - // When leaving a break and entering a non-break, set the start of a cluster - if (self.prev_break and !res.is_break) { -- const cp_len: usize = std.unicode.utf8CodepointSequenceLength(res.cp) catch 1; -+ const cp_len: usize = std.unicode.utf8CodepointSequenceLength(res.code_point) catch 1; - self.start = self.inner.i - cp_len; - } - -diff --git a/src/vxfw/App.zig b/src/vxfw/App.zig -index 6d394a9..fc3962f 100644 ---- a/src/vxfw/App.zig -+++ b/src/vxfw/App.zig -@@ -16,7 +16,7 @@ tty: vaxis.Tty, - vx: vaxis.Vaxis, - timers: std.ArrayList(vxfw.Tick), - wants_focus: ?vxfw.Widget, --buffer: [1024]u8, -+io: std.Io, - - /// Runtime options - pub const Options = struct { -@@ -27,10 +27,10 @@ pub const Options = struct { - /// Create an application. We require stable pointers to do the set up, so this will create an App - /// object on the heap. Call destroy when the app is complete to reset terminal state and release - /// resources --pub fn init(allocator: Allocator) !App { -- var app: App = .{ -+pub fn init(allocator: Allocator, io: std.Io) !App { -+ return .{ - .allocator = allocator, -- .tty = undefined, -+ .tty = try vaxis.Tty.init(io), - .vx = try vaxis.init(allocator, .{ - .system_clipboard_allocator = allocator, - .kitty_keyboard_flags = .{ -@@ -39,10 +39,8 @@ pub fn init(allocator: Allocator) !App { - }), - .timers = std.ArrayList(vxfw.Tick){}, - .wants_focus = null, -- .buffer = undefined, -+ .io = io, - }; -- app.tty = try vaxis.Tty.init(&app.buffer); -- return app; - } - - pub fn deinit(self: *App) void { -@@ -55,7 +53,8 @@ pub fn run(self: *App, widget: vxfw.Widget, opts: Options) anyerror!void { - const tty = &self.tty; - const vx = &self.vx; - -- var loop: EventLoop = .{ .tty = tty, .vaxis = vx }; -+ const io = self.io; -+ var loop: EventLoop = .{ .tty = tty, .vaxis = vx, .io = io, .queue = .{ .io = io } }; - try loop.start(); - defer loop.stop(); - -@@ -65,7 +64,7 @@ pub fn run(self: *App, widget: vxfw.Widget, opts: Options) anyerror!void { - loop.postEvent(.focus_in); - - try vx.enterAltScreen(tty.writer()); -- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_s); -+ try vx.queryTerminal(tty.writer(), io, 1 * std.time.ns_per_s); - try vx.setBracketedPaste(tty.writer(), true); - try vx.subscribeToColorSchemeUpdates(tty.writer()); - -@@ -97,13 +96,14 @@ pub fn run(self: *App, widget: vxfw.Widget, opts: Options) anyerror!void { - defer focus_handler.deinit(self.allocator); - - // Timestamp of our next frame -- var next_frame_ms: u64 = @intCast(std.time.milliTimestamp()); -+ var next_frame_ms: u64 = nowMs(io); - - // Create our event context - var ctx: vxfw.EventContext = .{ - .alloc = self.allocator, - .phase = .capturing, - .cmds = vxfw.CommandList{}, -+ .io = io, - .consume_event = false, - .redraw = false, - .quit = false, -@@ -111,13 +111,13 @@ pub fn run(self: *App, widget: vxfw.Widget, opts: Options) anyerror!void { - defer ctx.cmds.deinit(self.allocator); - - while (true) { -- const now_ms: u64 = @intCast(std.time.milliTimestamp()); -+ const now_ms: u64 = nowMs(io); - if (now_ms >= next_frame_ms) { - // Deadline exceeded. Schedule the next frame - next_frame_ms = now_ms + tick_ms; - } else { - // Sleep until the deadline -- std.Thread.sleep((next_frame_ms - now_ms) * std.time.ns_per_ms); -+ try std.Io.sleep(io, .fromMilliseconds(@intCast(next_frame_ms - now_ms)), .real); - next_frame_ms += tick_ms; - } - -@@ -296,7 +296,7 @@ fn handleCommand(self: *App, cmds: *vxfw.CommandList) Allocator.Error!void { - } - - fn checkTimers(self: *App, ctx: *vxfw.EventContext) anyerror!void { -- const now_ms = std.time.milliTimestamp(); -+ const now_ms = nowMs(ctx.io); - - // timers are always sorted descending - while (self.timers.pop()) |tick| { -@@ -349,12 +349,16 @@ const MouseHandler = struct { - .surface = last_frame, - .z_index = 0, - }; -- const mouse_point: vxfw.Point = .{ -- .row = @intCast(mouse.row), -- .col = @intCast(mouse.col), -- }; -- if (sub.containsPoint(mouse_point)) { -- try last_frame.hitTest(app.allocator, &hits, mouse_point); -+ -+ if (mouse.row >= 0 and mouse.col >= 0) { -+ const mouse_point: vxfw.Point = .{ -+ .row = @intCast(mouse.row), -+ .col = @intCast(mouse.col), -+ }; -+ -+ if (sub.containsPoint(mouse_point)) { -+ try last_frame.hitTest(app.allocator, &hits, mouse_point); -+ } - } - - // We store the hit list from the last mouse event to determine mouse_enter and mouse_leave -@@ -408,12 +412,16 @@ const MouseHandler = struct { - .surface = last_frame, - .z_index = 0, - }; -- const mouse_point: vxfw.Point = .{ -- .row = @intCast(mouse.row), -- .col = @intCast(mouse.col), -- }; -- if (sub.containsPoint(mouse_point)) { -- try last_frame.hitTest(app.allocator, &hits, mouse_point); -+ -+ if (mouse.row >= 0 and mouse.col >= 0) { -+ const mouse_point: vxfw.Point = .{ -+ .row = @intCast(mouse.row), -+ .col = @intCast(mouse.col), -+ }; -+ -+ if (sub.containsPoint(mouse_point)) { -+ try last_frame.hitTest(app.allocator, &hits, mouse_point); -+ } - } - - // Handle mouse_enter and mouse_leave events -@@ -602,3 +610,8 @@ const FocusHandler = struct { - } - } - }; -+ -+fn nowMs(io: std.Io) u64 { -+ const now = std.Io.Clock.now(.real, io); -+ return @intCast(now.toMilliseconds()); -+} -diff --git a/src/vxfw/Button.zig b/src/vxfw/Button.zig -index d4b2a9c..075627e 100644 ---- a/src/vxfw/Button.zig -+++ b/src/vxfw/Button.zig -@@ -142,6 +142,7 @@ test Button { - var ctx: vxfw.EventContext = .{ - .alloc = std.testing.allocator, - .cmds = .empty, -+ .io = std.testing.io, - }; - defer ctx.cmds.deinit(ctx.alloc); - -diff --git a/src/vxfw/ListView.zig b/src/vxfw/ListView.zig -index 8ff9551..de93d5d 100644 ---- a/src/vxfw/ListView.zig -+++ b/src/vxfw/ListView.zig -@@ -564,6 +564,7 @@ test ListView { - var ctx: vxfw.EventContext = .{ - .alloc = std.testing.allocator, - .cmds = .empty, -+ .io = std.testing.io, - }; - defer ctx.cmds.deinit(ctx.alloc); - -@@ -731,6 +732,7 @@ test "ListView: uneven scroll" { - var ctx: vxfw.EventContext = .{ - .alloc = std.testing.allocator, - .cmds = .empty, -+ .io = std.testing.io, - }; - defer ctx.cmds.deinit(ctx.alloc); - -diff --git a/src/vxfw/ScrollView.zig b/src/vxfw/ScrollView.zig -index 36319a4..38859ea 100644 ---- a/src/vxfw/ScrollView.zig -+++ b/src/vxfw/ScrollView.zig -@@ -646,6 +646,7 @@ test ScrollView { - var ctx: vxfw.EventContext = .{ - .alloc = std.testing.allocator, - .cmds = .empty, -+ .io = std.testing.io, - }; - defer ctx.cmds.deinit(ctx.alloc); - -@@ -1044,6 +1045,7 @@ test "ScrollView: uneven scroll" { - var ctx: vxfw.EventContext = .{ - .alloc = std.testing.allocator, - .cmds = .empty, -+ .io = std.testing.io, - }; - defer ctx.cmds.deinit(ctx.alloc); - -diff --git a/src/vxfw/Spinner.zig b/src/vxfw/Spinner.zig -index 6619522..eb3d8ba 100644 ---- a/src/vxfw/Spinner.zig -+++ b/src/vxfw/Spinner.zig -@@ -20,11 +20,11 @@ frame: u4 = 0, - was_spinning: std.atomic.Value(bool) = .{ .raw = false }, - - /// Start, or add one, to the spinner counter. Thread safe. --pub fn start(self: *Spinner) ?vxfw.Command { -+pub fn start(self: *Spinner, io: std.Io) ?vxfw.Command { - self.was_spinning.store(true, .unordered); - const count = self.count.fetchAdd(1, .monotonic); - if (count == 0) { -- return vxfw.Tick.in(time_lapse, self.widget()); -+ return vxfw.Tick.in(io, time_lapse, self.widget()); - } - return null; - } -@@ -111,13 +111,13 @@ test Spinner { - // Start the spinner. This (maybe) returns a Tick command to schedule the next frame. If the - // spinner is already running, no command is returned. Calling start is thread safe. The - // returned command can be added to an EventContext to schedule the frame -- const maybe_cmd = spinner.start(); -+ const maybe_cmd = spinner.start(std.testing.io); - try std.testing.expect(maybe_cmd != null); - try std.testing.expect(maybe_cmd.? == .tick); - try std.testing.expectEqual(1, spinner.count.load(.unordered)); - - // If we call start again, we won't get another command but our counter will go up -- const maybe_cmd2 = spinner.start(); -+ const maybe_cmd2 = spinner.start(std.testing.io); - try std.testing.expect(maybe_cmd2 == null); - try std.testing.expectEqual(2, spinner.count.load(.unordered)); - -@@ -126,6 +126,7 @@ test Spinner { - var ctx: vxfw.EventContext = .{ - .alloc = arena.allocator(), - .cmds = .empty, -+ .io = std.testing.io, - }; - - // The event loop handles the tick event and calls us back with a .tick event. If we should keep -diff --git a/src/vxfw/SplitView.zig b/src/vxfw/SplitView.zig -index 9a355a0..591e5e9 100644 ---- a/src/vxfw/SplitView.zig -+++ b/src/vxfw/SplitView.zig -@@ -229,6 +229,7 @@ test SplitView { - var ctx: vxfw.EventContext = .{ - .alloc = arena.allocator(), - .cmds = .empty, -+ .io = std.testing.io, - }; - try split_widget.handleEvent(&ctx, .{ .mouse = mouse }); - // We should get a command to change the mouse shape -diff --git a/src/vxfw/Text.zig b/src/vxfw/Text.zig -index 1f76cba..a7b71bb 100644 ---- a/src/vxfw/Text.zig -+++ b/src/vxfw/Text.zig -@@ -223,7 +223,7 @@ pub const SoftwrapIterator = struct { - // Advance the hard iterator - if (self.index == self.line.len) { - self.line = self.hard_iter.next() orelse return null; -- self.line = std.mem.trimRight(u8, self.line, " \t"); -+ self.line = std.mem.trimEnd(u8, self.line, " \t"); - self.index = 0; - } - -@@ -237,7 +237,7 @@ pub const SoftwrapIterator = struct { - if (self.ctx.max.width) |max| { - if (cur_width + next_width > max) { - // Trim the word to see if it can fit on a line by itself -- const trimmed = std.mem.trimLeft(u8, word, " \t"); -+ const trimmed = std.mem.trimStart(u8, word, " \t"); - const trimmed_bytes = word.len - trimmed.len; - // The number of bytes we trimmed is equal to the reduction in length - const trimmed_width = next_width - trimmed_bytes; -diff --git a/src/vxfw/TextField.zig b/src/vxfw/TextField.zig -index 6ad108b..e5a4d5b 100644 ---- a/src/vxfw/TextField.zig -+++ b/src/vxfw/TextField.zig -@@ -346,7 +346,7 @@ pub fn deleteAfterCursor(self: *TextField) void { - /// Moves the cursor backward by words. If the character before the cursor is a space, the cursor is - /// positioned just after the next previous space - pub fn moveBackwardWordwise(self: *TextField) void { -- const trimmed = std.mem.trimRight(u8, self.buf.firstHalf(), " "); -+ const trimmed = std.mem.trimEnd(u8, self.buf.firstHalf(), " "); - const idx = if (std.mem.lastIndexOfScalar(u8, trimmed, ' ')) |last| - last + 1 - else -@@ -562,6 +562,7 @@ test TextField { - var ctx: vxfw.EventContext = .{ - .alloc = arena.allocator(), - .cmds = .empty, -+ .io = std.testing.io, - }; - - // Enough boiler plate...Create the text field -diff --git a/src/vxfw/vxfw.zig b/src/vxfw/vxfw.zig -index 1510e71..f768d56 100644 ---- a/src/vxfw/vxfw.zig -+++ b/src/vxfw/vxfw.zig -@@ -61,8 +61,8 @@ pub const Tick = struct { - return lhs.deadline_ms > rhs.deadline_ms; - } - -- pub fn in(ms: u32, widget: Widget) Command { -- const now = std.time.milliTimestamp(); -+ pub fn in(io: std.Io, ms: u32, widget: Widget) Command { -+ const now = std.Io.Clock.now(.real, io).toMilliseconds(); - return .{ .tick = .{ - .deadline_ms = now + ms, - .widget = widget, -@@ -101,6 +101,7 @@ pub const EventContext = struct { - phase: Phase = .at_target, - alloc: Allocator, - cmds: CommandList, -+ io: std.Io, - - /// The event was handled, do not pass it on - consume_event: bool = false, -@@ -120,7 +121,7 @@ pub const EventContext = struct { - } - - pub fn tick(self: *EventContext, ms: u32, widget: Widget) Allocator.Error!void { -- try self.addCmd(Tick.in(ms, widget)); -+ try self.addCmd(Tick.in(self.io, ms, widget)); - } - - pub fn consumeAndRedraw(self: *EventContext) void { -@@ -541,10 +542,10 @@ test "All widgets have a doctest and refAllDecls test" { - // it easy to fail CI early, or spot bad tests vs non-existant tests - const excludes = &[_][]const u8{ "vxfw.zig", "App.zig" }; - -- var cwd = try std.fs.cwd().openDir("./src/vxfw", .{ .iterate = true }); -+ var cwd = try std.Io.Dir.cwd().openDir(std.testing.io, "./src/vxfw", .{ .iterate = true }); - var iter = cwd.iterate(); -- defer cwd.close(); -- outer: while (try iter.next()) |file| { -+ defer cwd.close(std.testing.io); -+ outer: while (try iter.next(std.testing.io)) |file| { - if (file.kind != .file) continue; - for (excludes) |ex| if (std.mem.eql(u8, ex, file.name)) continue :outer; - -@@ -552,7 +553,7 @@ test "All widgets have a doctest and refAllDecls test" { - file.name[0..idx] - else - continue; -- const data = try cwd.readFileAllocOptions(std.testing.allocator, file.name, 10_000_000, null, .of(u8), 0x00); -+ const data = try cwd.readFileAllocOptions(std.testing.io, file.name, std.testing.allocator, std.Io.Limit.limited(10_000_000), .of(u8), 0x00); - defer std.testing.allocator.free(data); - var ast = try std.zig.Ast.parse(std.testing.allocator, data, .zig); - defer ast.deinit(std.testing.allocator); -diff --git a/src/widgets/TextInput.zig b/src/widgets/TextInput.zig -index a6236a4..6b8871b 100644 ---- a/src/widgets/TextInput.zig -+++ b/src/widgets/TextInput.zig -@@ -266,7 +266,7 @@ pub fn deleteAfterCursor(self: *TextInput) void { - /// Moves the cursor backward by words. If the character before the cursor is a space, the cursor is - /// positioned just after the next previous space - pub fn moveBackwardWordwise(self: *TextInput) void { -- const trimmed = std.mem.trimRight(u8, self.buf.firstHalf(), " "); -+ const trimmed = std.mem.trimEnd(u8, self.buf.firstHalf(), " "); - const idx = if (std.mem.lastIndexOfScalar(u8, trimmed, ' ')) |last| - last + 1 - else -diff --git a/src/widgets/TextView.zig b/src/widgets/TextView.zig -index 45f9e44..ee326cf 100644 ---- a/src/widgets/TextView.zig -+++ b/src/widgets/TextView.zig -@@ -79,52 +79,22 @@ pub const Buffer = struct { - var cols: usize = self.last_cols; - var iter = uucode.grapheme.Iterator(uucode.utf8.Iterator).init(.init(content.bytes)); - -- var grapheme_start: usize = 0; -- var prev_break: bool = true; -- -- while (iter.next()) |result| { -- if (prev_break and !result.is_break) { -- // Start of a new grapheme -- const cp_len: usize = std.unicode.utf8CodepointSequenceLength(result.cp) catch 1; -- grapheme_start = iter.i - cp_len; -- } -- -- if (result.is_break) { -- // End of a grapheme -- const grapheme_end = iter.i; -- const grapheme_len = grapheme_end - grapheme_start; -- -- try self.grapheme.append(allocator, .{ -- .len = @intCast(grapheme_len), -- .offset = @intCast(self.content.items.len + grapheme_start), -- }); -- -- const cluster = content.bytes[grapheme_start..grapheme_end]; -- if (std.mem.eql(u8, cluster, "\n")) { -- self.cols = @max(self.cols, cols); -- cols = 0; -- } else { -- // Calculate width using gwidth -- const w = vaxis.gwidth.gwidth(cluster, .unicode); -- cols +|= w; -- } -- -- grapheme_start = grapheme_end; -- } -- prev_break = result.is_break; -- } -- -- // Flush the last grapheme if we ended mid-cluster -- if (!prev_break and grapheme_start < content.bytes.len) { -- const grapheme_len = content.bytes.len - grapheme_start; -+ while (iter.nextGrapheme()) |grapheme| { -+ const grapheme_start = grapheme.start; -+ const grapheme_end = grapheme.end; -+ const grapheme_len = grapheme_end - grapheme_start; - - try self.grapheme.append(allocator, .{ - .len = @intCast(grapheme_len), - .offset = @intCast(self.content.items.len + grapheme_start), - }); - -- const cluster = content.bytes[grapheme_start..]; -- if (!std.mem.eql(u8, cluster, "\n")) { -+ const cluster = content.bytes[grapheme_start..grapheme_end]; -+ if (std.mem.eql(u8, cluster, "\n")) { -+ self.cols = @max(self.cols, cols); -+ cols = 0; -+ } else { -+ // Calculate width using gwidth - const w = vaxis.gwidth.gwidth(cluster, .unicode); - cols +|= w; - } -diff --git a/src/widgets/terminal/Command.zig b/src/widgets/terminal/Command.zig -index 0cd24f7..ed41553 100644 ---- a/src/widgets/terminal/Command.zig -+++ b/src/widgets/terminal/Command.zig -@@ -14,7 +14,7 @@ working_directory: ?[]const u8, - // Set after spawn() - pid: ?std.posix.pid_t = null, - --env_map: *const std.process.EnvMap, -+env_map: *const std.process.Environ.Map, - - pty: Pty, - -@@ -43,8 +43,8 @@ pub fn spawn(self: *Command, allocator: std.mem.Allocator) !void { - try posix.dup2(self.pty.tty.handle, std.posix.STDOUT_FILENO); - try posix.dup2(self.pty.tty.handle, std.posix.STDERR_FILENO); - -- self.pty.tty.close(); -- if (self.pty.pty.handle > 2) self.pty.pty.close(); -+ _ = std.os.linux.close(self.pty.tty.handle); -+ if (self.pty.pty.handle > 2) _ = std.os.linux.close(self.pty.pty.handle); - - if (self.working_directory) |wd| { - try std.posix.chdir(wd); -@@ -78,8 +78,8 @@ pub fn spawn(self: *Command, allocator: std.mem.Allocator) !void { - fn handleSigChild(_: c_int) callconv(.c) void { - const result = std.posix.waitpid(-1, 0); - -- Terminal.global_vt_mutex.lock(); -- defer Terminal.global_vt_mutex.unlock(); -+ if (!Terminal.global_vt_mutex.tryLock()) return; -+ defer Terminal.global_vt_mutex.unlock(Terminal.global_vt_io); - if (Terminal.global_vts) |vts| { - var vt = vts.get(result.pid) orelse return; - vt.event_queue.push(.exited); -@@ -97,7 +97,7 @@ pub fn kill(self: *Command) void { - /// hash map plus options. - fn createEnvironFromMap( - arena: std.mem.Allocator, -- map: *const std.process.EnvMap, -+ map: *const std.process.Environ.Map, - ) ![:null]?[*:0]u8 { - const envp_count: usize = map.count(); - -diff --git a/src/widgets/terminal/Pty.zig b/src/widgets/terminal/Pty.zig -index d9f4138..4c67328 100644 ---- a/src/widgets/terminal/Pty.zig -+++ b/src/widgets/terminal/Pty.zig -@@ -7,8 +7,8 @@ const Winsize = @import("../../main.zig").Winsize; - - const posix = std.posix; - --pty: std.fs.File, --tty: std.fs.File, -+pty: std.Io.File, -+tty: std.Io.File, - - /// opens a new tty/pty pair - pub fn init() !Pty { -@@ -19,9 +19,9 @@ pub fn init() !Pty { - } - - /// closes the tty and pty --pub fn deinit(self: Pty) void { -- self.pty.close(); -- self.tty.close(); -+pub fn deinit(self: Pty, io: std.Io) void { -+ self.pty.close(io); -+ self.tty.close(io); - } - - /// sets the size of the pty -@@ -37,7 +37,7 @@ pub fn setSize(self: Pty, ws: Winsize) !void { - } - - fn openPtyLinux() !Pty { -- const p = try posix.open("/dev/ptmx", .{ .ACCMODE = .RDWR, .NOCTTY = true }, 0); -+ const p = try posix.openat(posix.AT.FDCWD, "/dev/ptmx", .{ .ACCMODE = .RDWR, .NOCTTY = true }, 0); - errdefer posix.close(p); - - // unlockpt -@@ -50,7 +50,7 @@ fn openPtyLinux() !Pty { - const sname = try std.fmt.bufPrint(&buf, "/dev/pts/{d}", .{n}); - std.log.debug("pts: {s}", .{sname}); - -- const t = try posix.open(sname, .{ .ACCMODE = .RDWR, .NOCTTY = true }, 0); -+ const t = try posix.openat(posix.AT.FDCWD, sname, .{ .ACCMODE = .RDWR, .NOCTTY = true }, 0); - - return .{ - .pty = .{ .handle = p }, -diff --git a/src/widgets/terminal/Terminal.zig b/src/widgets/terminal/Terminal.zig -index 36d61d7..786ca85 100644 ---- a/src/widgets/terminal/Terminal.zig -+++ b/src/widgets/terminal/Terminal.zig -@@ -43,21 +43,23 @@ pub const InputEvent = union(enum) { - key_press: vaxis.Key, - }; - --pub var global_vt_mutex: std.Thread.Mutex = .{}; -+pub var global_vt_mutex: std.Io.Mutex = .init; -+var global_vt_io: std.Io = undefined; - pub var global_vts: ?std.AutoHashMap(i32, *Terminal) = null; - pub var global_sigchild_installed: bool = false; - - allocator: std.mem.Allocator, -+io: std.Io, - scrollback_size: u16, - - pty: Pty, --pty_writer: std.fs.File.Writer, -+pty_writer: std.Io.File.Writer, - cmd: Command, - thread: ?std.Thread = null, - - /// the screen we draw from - front_screen: Screen, --front_mutex: std.Thread.Mutex = .{}, -+front_mutex: std.Io.Mutex = .init, - - /// the back screens - back_screen: *Screen = undefined, -@@ -65,7 +67,7 @@ back_screen_pri: Screen, - back_screen_alt: Screen, - // only applies to primary screen - scroll_offset: usize = 0, --back_mutex: std.Thread.Mutex = .{}, -+back_mutex: std.Io.Mutex = .init, - // dirty is protected by back_mutex. Only access this field when you hold that mutex - dirty: bool = false, - -@@ -79,14 +81,15 @@ working_directory: std.ArrayList(u8) = .empty, - - last_printed: []const u8 = "", - --event_queue: Queue = .{}, -+event_queue: Queue, - - /// initialize a Terminal. This sets the size of the underlying pty and allocates the sizes of the - /// screen - pub fn init( - allocator: std.mem.Allocator, -+ io: std.Io, - argv: []const []const u8, -- env: *const std.process.EnvMap, -+ env: *const std.process.Environ.Map, - opts: Options, - write_buf: []u8, - ) !Terminal { -@@ -107,8 +110,10 @@ pub fn init( - while (col < opts.winsize.cols) : (col += 8) { - try tabs.append(allocator, col); - } -+ global_vt_io = io; - return .{ - .allocator = allocator, -+ .io = io, - .pty = pty, - .pty_writer = pty.pty.writerStreaming(write_buf), - .cmd = cmd, -@@ -117,6 +122,7 @@ pub fn init( - .back_screen_pri = try Screen.init(allocator, opts.winsize.cols, opts.winsize.rows + opts.scrollback_size), - .back_screen_alt = try Screen.init(allocator, opts.winsize.cols, opts.winsize.rows), - .tab_stops = tabs, -+ .event_queue = .{ .io = io }, - }; - } - -@@ -125,8 +131,8 @@ pub fn deinit(self: *Terminal) void { - self.should_quit = true; - - pid: { -- global_vt_mutex.lock(); -- defer global_vt_mutex.unlock(); -+ global_vt_mutex.lockUncancelable(global_vt_io); -+ defer global_vt_mutex.unlock(global_vt_io); - var vts = global_vts orelse break :pid; - if (self.cmd.pid) |pid| - _ = vts.remove(pid); -@@ -143,7 +149,7 @@ pub fn deinit(self: *Terminal) void { - thread.join(); - self.thread = null; - } -- self.pty.deinit(); -+ self.pty.deinit(self.io); - self.front_screen.deinit(self.allocator); - self.back_screen_pri.deinit(self.allocator); - self.back_screen_alt.deinit(self.allocator); -@@ -162,16 +168,15 @@ pub fn spawn(self: *Terminal) !void { - if (self.cmd.working_directory) |pwd| { - try self.working_directory.appendSlice(self.allocator, pwd); - } else { -- const pwd = std.fs.cwd(); - var buffer: [std.fs.max_path_bytes]u8 = undefined; -- const out_path = try std.os.getFdPath(pwd.fd, &buffer); -+ const out_path = try std.posix.readlinkZ("/proc/self/cwd", &buffer); - try self.working_directory.appendSlice(self.allocator, out_path); - } - - { - // add to our global list -- global_vt_mutex.lock(); -- defer global_vt_mutex.unlock(); -+ global_vt_mutex.lockUncancelable(global_vt_io); -+ defer global_vt_mutex.unlock(global_vt_io); - if (global_vts == null) - global_vts = std.AutoHashMap(i32, *Terminal).init(self.allocator); - if (self.cmd.pid) |pid| -@@ -190,8 +195,8 @@ pub fn resize(self: *Terminal, ws: Winsize) !void { - ws.rows == self.front_screen.height) - return; - -- self.back_mutex.lock(); -- defer self.back_mutex.unlock(); -+ self.back_mutex.lockUncancelable(self.io); -+ defer self.back_mutex.unlock(self.io); - - self.front_screen.deinit(self.allocator); - self.front_screen = try Screen.init(self.allocator, ws.cols, ws.rows); -@@ -206,7 +211,7 @@ pub fn resize(self: *Terminal, ws: Winsize) !void { - - pub fn draw(self: *Terminal, allocator: std.mem.Allocator, win: vaxis.Window) !void { - if (self.back_mutex.tryLock()) { -- defer self.back_mutex.unlock(); -+ defer self.back_mutex.unlock(self.io); - // We keep this as a separate condition so we don't deadlock by obtaining the lock but not - // having sync - if (!self.mode.sync) { -@@ -249,7 +254,7 @@ pub fn get_pty_writer(self: *Terminal) *std.Io.Writer { - return &self.pty_writer.interface; - } - --fn reader(self: *const Terminal, buf: []u8) std.fs.File.Reader { -+fn reader(self: *const Terminal, buf: []u8) std.Io.File.Reader { - return self.pty.pty.readerStreaming(buf); - } - -@@ -265,8 +270,8 @@ fn run(self: *Terminal) !void { - - while (!self.should_quit) { - const event = try parser.parseReader(&reader_.interface); -- self.back_mutex.lock(); -- defer self.back_mutex.unlock(); -+ self.back_mutex.lockUncancelable(self.io); -+ defer self.back_mutex.unlock(self.io); - - if (!self.dirty and self.event_queue.tryPush(.redraw)) - self.dirty = true; diff --git a/third_party/libvaxis/fixes.patch b/third_party/libvaxis/fixes.patch index aca9c136..6a33c47e 100644 --- a/third_party/libvaxis/fixes.patch +++ b/third_party/libvaxis/fixes.patch @@ -1,33 +1,38 @@ -diff --git a/examples/table.zig b/examples/table.zig -index c489828..16aefdc 100644 ---- a/examples/table.zig -+++ b/examples/table.zig -@@ -20,8 +20,8 @@ pub fn main(init: std.process.Init) !void { - // Users set up below the main function - const users_buf = try alloc.dupe(User, users[0..]); - -- var buffer: [1024]u8 = undefined; -- var tty = try vaxis.Tty.init(init.io, &buffer); -+ // var buffer: [1024]u8 = undefined; -+ var tty = try vaxis.Tty.init(init.io); - defer tty.deinit(); - const tty_writer = tty.writer(); - var vx = try vaxis.init(alloc, .{ +diff --git a/src/Loop.zig b/src/Loop.zig +--- a/src/Loop.zig ++++ b/src/Loop.zig +@@ -352,7 +352,7 @@ + } + }, + .cap_da1 => { +- std.Io.futexWake(vx.io, std.atomic.Value(u32), &vx.query_futex, 10); ++ std.Io.futexWake(vx.io, u32, &vx.query_futex.raw, 10); + vx.queries_done.store(true, .unordered); + }, + .mouse => |mouse| { +@@ -480,7 +480,7 @@ + vx.caps.multi_cursor = true; + }, + .cap_da1 => { +- std.Io.futexWake(vx.io, std.atomic.Value(u32), &vx.query_futex, 10); ++ std.Io.futexWake(vx.io, u32, &vx.query_futex.raw, 10); + vx.queries_done.store(true, .unordered); + }, + .winsize => |winsize| { diff --git a/src/Vaxis.zig b/src/Vaxis.zig -index 2e24e9b..2b2ee49 100644 --- a/src/Vaxis.zig +++ b/src/Vaxis.zig -@@ -127,14 +127,13 @@ pub fn deinit(self: *Vaxis, alloc: ?std.mem.Allocator, tty: *IoWriter) void { +@@ -137,14 +137,13 @@ } } -/// resets enabled features, sends cursor to home and clears below cursor --pub fn resetState(self: *Vaxis, tty: *IoWriter) !void { +-pub fn resetState(self: *Vaxis, tty: *std.Io.Writer) !void { - // always show the cursor on state reset +/// Resets enabled terminal features without clearing screen content. +/// Use this for non-interactive / single-shot rendering where the +/// output should remain visible after exit. -+pub fn resetModes(self: *Vaxis, tty: *IoWriter) !void { ++pub fn resetModes(self: *Vaxis, tty: *std.Io.Writer) !void { tty.writeAll(ctlseqs.show_cursor) catch {}; tty.writeAll(ctlseqs.sgr_reset) catch {}; if (self.screen.cursor_shape != .default) { @@ -36,51 +41,81 @@ index 2e24e9b..2b2ee49 100644 tty.print(ctlseqs.cursor_shape, .{@intFromEnum(Cell.CursorShape.default)}) catch {}; } if (self.state.kitty_keyboard) { -@@ -147,18 +146,6 @@ pub fn resetState(self: *Vaxis, tty: *IoWriter) !void { +@@ -157,6 +156,32 @@ if (self.state.bracketed_paste) { try self.setBracketedPaste(tty, false); } -- if (self.state.alt_screen) { -- try tty.writeAll(ctlseqs.home); -- try tty.writeAll(ctlseqs.erase_below_cursor); -- try self.exitAltScreen(tty); -- } else { -- try tty.writeByte('\r'); -- var i: u16 = 0; -- while (i < self.state.cursor.row) : (i += 1) { -- try tty.writeAll(ctlseqs.ri); -- } -- try tty.writeAll(ctlseqs.erase_below_cursor); -- } - if (self.state.color_scheme_updates) { - try tty.writeAll(ctlseqs.color_scheme_reset); - self.state.color_scheme_updates = false; -@@ -179,7 +166,24 @@ pub fn resetState(self: *Vaxis, tty: *IoWriter) !void { - try tty.writeAll(ctlseqs.osc12_reset); - self.state.changed_cursor_color = false; - } ++ if (self.state.color_scheme_updates) { ++ try tty.writeAll(ctlseqs.color_scheme_reset); ++ self.state.color_scheme_updates = false; ++ } ++ if (self.state.in_band_resize) { ++ try tty.writeAll(ctlseqs.in_band_resize_reset); ++ self.state.in_band_resize = false; ++ } ++ if (self.state.changed_default_fg) { ++ try tty.writeAll(ctlseqs.osc10_reset); ++ self.state.changed_default_fg = false; ++ } ++ if (self.state.changed_default_bg) { ++ try tty.writeAll(ctlseqs.osc11_reset); ++ self.state.changed_default_bg = false; ++ } ++ if (self.state.changed_cursor_color) { ++ try tty.writeAll(ctlseqs.osc12_reset); ++ self.state.changed_cursor_color = false; ++ } + try tty.flush(); +} - ++ +/// resets enabled features, sends cursor to home and clears below cursor -+pub fn resetState(self: *Vaxis, tty: *IoWriter) !void { ++pub fn resetState(self: *Vaxis, tty: *std.Io.Writer) !void { + self.resetModes(tty) catch {}; -+ if (self.state.alt_screen) { -+ try tty.writeAll(ctlseqs.home); -+ try tty.writeAll(ctlseqs.erase_below_cursor); -+ try self.exitAltScreen(tty); -+ } else { -+ try tty.writeByte('\r'); -+ var i: u16 = 0; -+ while (i < self.state.cursor.row) : (i += 1) { -+ try tty.writeAll(ctlseqs.ri); -+ } -+ try tty.writeAll(ctlseqs.erase_below_cursor); -+ } + if (self.state.alt_screen) { + try tty.writeAll(ctlseqs.home); + try tty.writeAll(ctlseqs.erase_below_cursor); +@@ -169,27 +194,6 @@ + } + try tty.writeAll(ctlseqs.erase_below_cursor); + } +- if (self.state.color_scheme_updates) { +- try tty.writeAll(ctlseqs.color_scheme_reset); +- self.state.color_scheme_updates = false; +- } +- if (self.state.in_band_resize) { +- try tty.writeAll(ctlseqs.in_band_resize_reset); +- self.state.in_band_resize = false; +- } +- if (self.state.changed_default_fg) { +- try tty.writeAll(ctlseqs.osc10_reset); +- self.state.changed_default_fg = false; +- } +- if (self.state.changed_default_bg) { +- try tty.writeAll(ctlseqs.osc11_reset); +- self.state.changed_default_bg = false; +- } +- if (self.state.changed_cursor_color) { +- try tty.writeAll(ctlseqs.osc12_reset); +- self.state.changed_cursor_color = false; +- } +- try tty.flush(); } -@@ -943,19 +947,19 @@ pub fn transmitLocalImagePath( +@@ -272,9 +276,9 @@ + try self.queryTerminalSend(tty); + try std.Io.futexWaitTimeout( + self.io, +- atomic.Value(u32), +- &self.query_futex, +- .init(0), ++ u32, ++ &self.query_futex.raw, ++ 0, + .{ + .duration = .{ + .clock = .real, +@@ -980,19 +984,19 @@ switch (format) { .rgb => { try tty.print( @@ -103,7 +138,7 @@ index 2e24e9b..2b2ee49 100644 .{ id, medium_char, encoded }, ); }, -@@ -991,7 +995,7 @@ pub fn transmitPreEncodedImage( +@@ -1028,7 +1032,7 @@ if (bytes.len < 4096) { try tty.print( @@ -112,7 +147,7 @@ index 2e24e9b..2b2ee49 100644 .{ fmt, width, -@@ -1004,14 +1008,14 @@ pub fn transmitPreEncodedImage( +@@ -1041,14 +1045,14 @@ var n: usize = 4096; try tty.print( @@ -129,11 +164,22 @@ index 2e24e9b..2b2ee49 100644 .{ m, bytes[n..end], +diff --git a/src/c_api.zig b/src/c_api.zig +--- a/src/c_api.zig ++++ b/src/c_api.zig +@@ -951,7 +951,7 @@ + .cap_color_scheme_updates => vx.caps.color_scheme_updates = true, + .cap_multi_cursor => vx.caps.multi_cursor = true, + .cap_da1 => { +- std.Io.futexWake(vx.io, std.atomic.Value(u32), &vx.query_futex, 10); ++ std.Io.futexWake(vx.io, u32, &vx.query_futex.raw, 10); + vx.queries_done.store(true, .unordered); + }, + .winsize => { diff --git a/src/ctlseqs.zig b/src/ctlseqs.zig -index ab415c6..4baa496 100644 --- a/src/ctlseqs.zig +++ b/src/ctlseqs.zig -@@ -138,8 +138,8 @@ pub const osc52_clipboard_copy = "\x1b]52;c;{s}\x1b\\"; +@@ -138,8 +138,8 @@ pub const osc52_clipboard_request = "\x1b]52;c;?\x1b\\"; // Kitty graphics @@ -144,118 +190,10 @@ index ab415c6..4baa496 100644 pub const kitty_graphics_closing = ",C=1\x1b\\"; // Color control sequences -diff --git a/src/tty.zig b/src/tty.zig -index c1da795..7f6a303 100644 ---- a/src/tty.zig -+++ b/src/tty.zig -@@ -52,6 +52,10 @@ pub const PosixTty = struct { - - var handler_installed: bool = false; - -+ /// Atomic flag set by the SIGWINCH signal handler. Checked and cleared -+ /// by pollWinch() from a non-signal context to avoid deadlocks. -+ var winch_pending: std.atomic.Value(bool) = .{ .raw = false }; -+ - /// initializes a Tty instance by opening /dev/tty and "making it raw". A - /// signal handler is installed for SIGWINCH. No callbacks are installed, be - /// sure to register a callback when initializing the event loop -@@ -129,7 +133,20 @@ pub const PosixTty = struct { - } - - fn handleWinch(_: posix.SIG) callconv(.c) void { -- if (!handler_mutex.tryLock()) return; -+ // Only set an atomic flag. Calling mutex.lock() or queue.push() -+ // from a signal handler is not async-signal-safe and causes -+ // deadlocks when the signal is delivered to a thread that already -+ // holds one of those locks. Registered handlers are dispatched -+ // from pollWinch() instead. -+ winch_pending.store(true, .release); -+ } -+ -+ /// Check if a SIGWINCH was received and dispatch registered handlers. -+ /// Must be called from a non-signal context (e.g., the main event loop) -+ /// to avoid the async-signal-safety issues with the old approach. -+ pub fn pollWinch() void { -+ if (!winch_pending.swap(false, .acquire)) return; -+ handler_mutex.lockUncancelable(handler_io); - defer handler_mutex.unlock(handler_io); - var i: usize = 0; - while (i < handler_idx) : (i += 1) { -diff --git a/src/vxfw/App.zig b/src/vxfw/App.zig -index fc3962f..2661cf4 100644 ---- a/src/vxfw/App.zig -+++ b/src/vxfw/App.zig -@@ -1,4 +1,5 @@ - const std = @import("std"); -+const builtin = @import("builtin"); - const vaxis = @import("../main.zig"); - const vxfw = @import("vxfw.zig"); - -@@ -36,8 +37,8 @@ fn init(allocator: Allocator, io: std.Io) !App { - .kitty_keyboard_flags = .{ - .report_events = true, - }, - }), -- .timers = std.ArrayList(vxfw.Tick){}, -+ .timers = .empty, - .wants_focus = null, - .io = io, - }; -@@ -102,7 +103,7 @@ - var ctx: vxfw.EventContext = .{ - .alloc = self.allocator, - .phase = .capturing, -- .cmds = vxfw.CommandList{}, -+ .cmds = .empty, - .io = io, - .consume_event = false, - .redraw = false, -@@ -123,6 +124,13 @@ pub fn run(self: *App, widget: vxfw.Widget, opts: Options) anyerror!void { - - try self.checkTimers(&ctx); - -+ // Poll for pending SIGWINCH outside of queue lock. -+ // The signal handler only sets an atomic flag; we dispatch -+ // the registered callbacks here where mutex operations are safe. -+ if (builtin.os.tag != .windows and !builtin.is_test) { -+ if (!vx.state.in_band_resize) vaxis.Tty.pollWinch(); -+ } -+ - { - loop.queue.lock(); - defer loop.queue.unlock(); -@@ -342,7 +350,7 @@ - // For mouse events we store the last frame and use that for hit testing - const last_frame = surface; - -- var hits = std.ArrayList(vxfw.HitResult){}; -+ var hits: std.ArrayList(vxfw.HitResult) = .empty; - defer hits.deinit(app.allocator); - const sub: vxfw.SubSurface = .{ - .origin = .{ .row = 0, .col = 0 }, -@@ -405,7 +413,7 @@ - const last_frame = self.last_frame; - self.mouse = mouse; - -- var hits = std.ArrayList(vxfw.HitResult){}; -+ var hits: std.ArrayList(vxfw.HitResult) = .empty; - defer hits.deinit(app.allocator); - const sub: vxfw.SubSurface = .{ - .origin = .{ .row = 0, .col = 0 }, -@@ -524,7 +532,7 @@ - return .{ - .root = root, - .focused_widget = root, -- .path_to_focused = std.ArrayList(Widget){}, -+ .path_to_focused = .empty, - }; - } - diff --git a/src/vxfw/Padding.zig b/src/vxfw/Padding.zig -index e60162f..6e5f40b 100644 --- a/src/vxfw/Padding.zig +++ b/src/vxfw/Padding.zig -@@ -6,7 +6,7 @@ const Allocator = std.mem.Allocator; +@@ -6,7 +6,7 @@ const vxfw = @import("vxfw.zig"); const Padding = @This(); @@ -264,7 +202,7 @@ index e60162f..6e5f40b 100644 left: u16 = 0, right: u16 = 0, top: u16 = 0, -@@ -54,10 +54,6 @@ fn typeErasedDrawFn(ptr: *anyopaque, ctx: vxfw.DrawContext) Allocator.Error!vxfw +@@ -54,10 +54,6 @@ pub fn draw(self: *const Padding, ctx: vxfw.DrawContext) Allocator.Error!vxfw.Surface { const pad = self.padding; diff --git a/third_party/libvaxis/repo.bzl b/third_party/libvaxis/repo.bzl index 8305d1c0..7473e880 100644 --- a/third_party/libvaxis/repo.bzl +++ b/third_party/libvaxis/repo.bzl @@ -4,8 +4,8 @@ def repo(): git_repository( name = "libvaxis", remote = "https://github.com/rockorager/libvaxis.git", - commit = "a3ae1d53feeeeaeb6218de3d38837559811acae4", + commit = "173a890d1394946b5d7623c66cd34bcd36d8eeb8", build_file = "//third_party/libvaxis:libvaxis.bazel", - patches = ["//third_party/libvaxis:bump-zig.patch", "//third_party/libvaxis:fixes.patch"], + patches = ["//third_party/libvaxis:fixes.patch"], patch_args = ["-p1"], ) diff --git a/third_party/translate-c/repo.bzl b/third_party/translate-c/repo.bzl index 8315b2af..dfd8835d 100644 --- a/third_party/translate-c/repo.bzl +++ b/third_party/translate-c/repo.bzl @@ -3,8 +3,9 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") def repo(): git_repository( name = "translate-c", - # Use ZML Github mirror of translate-c because Codeber is too unstable + # Use the ZML GitHub mirror because Codeberg is unreliable. remote = "https://github.com/zml/translate-c", - commit = "46b5609b5ac4c0a896217d1d984f3ae50e4810b5", + # 2.0.0 + commit = "0da7a16c3235b935b82421646076e0657cda21f6", build_file = Label("//third_party/translate-c:translate-c.bazel"), ) diff --git a/third_party/translate-c/translate-c.bazel b/third_party/translate-c/translate-c.bazel index b2ff9c98..7de2a5eb 100644 --- a/third_party/translate-c/translate-c.bazel +++ b/third_party/translate-c/translate-c.bazel @@ -1,4 +1,12 @@ load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_configure_binary", "zig_library") +load("@zml//bazel:zig_options.bzl", "zig_option", "zig_options") + +zig_option(name = "debug_allocations", bool = False) + +zig_options( + name = "build_options", + settings = [":debug_allocations"], +) zig_library( name = "c_builtins", @@ -19,6 +27,7 @@ zig_binary( main = "src/main.zig", linkopts = ["-lc"], deps = [ + ":build_options", ":c_builtins", ":helpers", "@arocc//:aro", diff --git a/third_party/uucode/repo.bzl b/third_party/uucode/repo.bzl index 20a2f080..5f7b1cbf 100644 --- a/third_party/uucode/repo.bzl +++ b/third_party/uucode/repo.bzl @@ -21,7 +21,8 @@ _uucode_repo = repository_rule( def repo(): _uucode_repo( name = "uucode", - commit = "f1c687f09174f9f11eda14d3e3ce497d68ab09a6", + # main branch + commit = "1fb73433bba5d93366c57f23ff2e9d7939746500", build_config = "//third_party/uucode:build_config.zig", build_file = "//third_party/uucode:uucode.bazel", config_storage_patch = "//third_party/uucode:config-storage-module.patch", diff --git a/third_party/uucode/uucode.bazel b/third_party/uucode/uucode.bazel index f2b5634d..4a3710f2 100644 --- a/third_party/uucode/uucode.bazel +++ b/third_party/uucode/uucode.bazel @@ -8,6 +8,7 @@ zig_library( main = "src/config.zig", srcs = [ "src/components.zig", + "src/ucd_inputs.zig", "src/fields.zig", "src/multi_slice.zig", "src/quirks.zig", @@ -51,6 +52,7 @@ zig_library( main = "src/config.zig", srcs = [ "src/components.zig", + "src/ucd_inputs.zig", "src/fields.zig", "src/multi_slice.zig", "src/quirks.zig", diff --git a/third_party/zigimg/bump-zig.patch b/third_party/zigimg/bump-zig.patch deleted file mode 100644 index b668dadc..00000000 --- a/third_party/zigimg/bump-zig.patch +++ /dev/null @@ -1,4491 +0,0 @@ -diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml -index 400aee8..72ad6f9 100644 ---- a/.github/workflows/main.yml -+++ b/.github/workflows/main.yml -@@ -25,7 +25,7 @@ jobs: - - name: Setup Zig - uses: mlugg/setup-zig@v2 - with: -- version: 0.15.2 -+ version: 0.16.0-dev.2261+d6b3dd25a - - name: AST Verify - run: zig fmt --check --ast-check src/ - working-directory: zigimg -diff --git a/build.zig.zon b/build.zig.zon -index 3ff4816..536d37b 100644 ---- a/build.zig.zon -+++ b/build.zig.zon -@@ -1,7 +1,7 @@ - .{ - .name = .zigimg, - .fingerprint = 0x6d2b443cdaa8f7f3, -- .minimum_zig_version = "0.15.2", -+ .minimum_zig_version = "0.16.0-dev.2261+d6b3dd25a", - .version = "0.1.0", - .paths = .{ - "build.zig", -diff --git a/src/Image.zig b/src/Image.zig -index d67cd61..6472129 100644 ---- a/src/Image.zig -+++ b/src/Image.zig -@@ -71,7 +71,7 @@ pub const WriteError = Error || - std.mem.Allocator.Error || - io.ReadStream.Error || - io.WriteStream.Error || -- std.fs.File.OpenError || -+ std.Io.File.OpenError || - error{ EndOfStream, InvalidData, UnfinishedBits }; - - pub const ConvertError = Error || -@@ -101,7 +101,7 @@ - }; - - pub const Animation = struct { -- frames: FrameList = .{}, -+ frames: FrameList = .empty, - loop_count: i32 = AnimationLoopInfinite, - - pub const FrameList = std.ArrayList(AnimationFrame); -@@ -150,16 +150,16 @@ pub fn deinit(self: *Image, allocator: std.mem.Allocator) void { - } - - /// Detect which image format is used by the file path --pub fn detectFormatFromFilePath(file_path: []const u8, read_buffer: []u8) !Format { -- var file = try std.fs.cwd().openFile(file_path, .{}); -- defer file.close(); -+pub fn detectFormatFromFilePath(std_io: std.Io, file_path: []const u8, read_buffer: []u8) !Format { -+ var file = try std.Io.Dir.openFile(std.Io.Dir.cwd(), std_io, file_path, .{}); -+ defer file.close(std_io); - -- return detectFormatFromFile(file, read_buffer); -+ return detectFormatFromFile(std_io, file, read_buffer); - } - - /// Detect which image format is used by the file --pub fn detectFormatFromFile(file: std.fs.File, read_buffer: []u8) !Format { -- var read_stream = io.ReadStream.initFile(file, read_buffer); -+pub fn detectFormatFromFile(std_io: std.Io, file: std.Io.File, read_buffer: []u8) !Format { -+ var read_stream = io.ReadStream.initFile(std_io, file, read_buffer); - return internalDetectFormat(&read_stream); - } - -@@ -170,16 +170,16 @@ pub fn detectFormatFromMemory(buffer: []const u8) !Format { - } - - /// Load an image from a file path --pub fn fromFilePath(allocator: std.mem.Allocator, file_path: []const u8, read_buffer: []u8) !Image { -- var file = try std.fs.cwd().openFile(file_path, .{}); -- defer file.close(); -+pub fn fromFilePath(allocator: std.mem.Allocator, std_io: std.Io, file_path: []const u8, read_buffer: []u8) !Image { -+ var file = try std.Io.Dir.openFile(std.Io.Dir.cwd(), std_io, file_path, .{}); -+ defer file.close(std_io); - -- return fromFile(allocator, file, read_buffer); -+ return fromFile(allocator, std_io, file, read_buffer); - } - --/// Load an image from a standard library std.fs.File --pub fn fromFile(allocator: std.mem.Allocator, file: std.fs.File, read_buffer: []u8) !Image { -- var read_stream = io.ReadStream.initFile(file, read_buffer); -+/// Load an image from a standard library std.Io.File -+pub fn fromFile(allocator: std.mem.Allocator, std_io: std.Io, file: std.Io.File, read_buffer: []u8) !Image { -+ var read_stream = io.ReadStream.initFile(std_io, file, read_buffer); - return internalRead(allocator, &read_stream); - } - -@@ -260,16 +260,16 @@ pub fn toManaged(self: Image, allocator: std.mem.Allocator) Managed { - } - - /// Write the image to an image format to the specified path --pub fn writeToFilePath(self: Image, allocator: std.mem.Allocator, file_path: []const u8, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { -- var file = try std.fs.cwd().createFile(file_path, .{}); -- defer file.close(); -+pub fn writeToFilePath(self: Image, allocator: std.mem.Allocator, std_io: std.Io, file_path: []const u8, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { -+ var file = try std.Io.Dir.createFile(std.Io.Dir.cwd(), std_io, file_path, .{}); -+ defer file.close(std_io); - -- try self.writeToFile(allocator, file, write_buffer, encoder_options); -+ try self.writeToFile(allocator, std_io, file, write_buffer, encoder_options); - } - --/// Write the image to an image format to the specified std.fs.File --pub fn writeToFile(self: Image, allocator: std.mem.Allocator, file: std.fs.File, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { -- var write_stream = io.WriteStream.initFile(file, write_buffer); -+/// Write the image to an image format to the specified std.Io.File -+pub fn writeToFile(self: Image, allocator: std.mem.Allocator, std_io: std.Io, file: std.Io.File, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { -+ var write_stream = io.WriteStream.initFile(std_io, file, write_buffer); - - try self.internalWrite(allocator, &write_stream, encoder_options); - } -diff --git a/src/Image/Managed.zig b/src/Image/Managed.zig -index 608b1d6..1517e7c 100644 ---- a/src/Image/Managed.zig -+++ b/src/Image/Managed.zig -@@ -20,14 +20,16 @@ width: usize = 0, - height: usize = 0, - pixels: color.PixelStorage = .{ .invalid = void{} }, - animation: Animation = .{}, -+io: std.Io = undefined, - allocator: std.mem.Allocator = undefined, // Allocator needs to be last in order to be able to ptrCast to Image - - const Managed = @This(); - - /// Init an empty image with no pixel data --pub fn init(allocator: std.mem.Allocator) Managed { -+pub fn init(allocator: std.mem.Allocator, io: std.Io) Managed { - return Managed{ - .allocator = allocator, -+ .io = io, - }; - } - -@@ -37,13 +39,13 @@ pub fn deinit(self: *Managed) void { - } - - /// Detect which image format is used by the file path --pub fn detectFormatFromFilePath(file_path: []const u8, read_buffer: []u8) !Format { -- return Image.detectFormatFromFilePath(file_path, read_buffer); -+pub fn detectFormatFromFilePath(io: std.Io, file_path: []const u8, read_buffer: []u8) !Format { -+ return Image.detectFormatFromFilePath(io, file_path, read_buffer); - } - - /// Detect which image format is used by the file --pub fn detectFormatFromFile(file: std.fs.File, read_buffer: []u8) !Format { -- return Image.detectFormatFromFile(file, read_buffer); -+pub fn detectFormatFromFile(io: std.Io, file: std.Io.File, read_buffer: []u8) !Format { -+ return Image.detectFormatFromFile(io, file, read_buffer); - } - - /// Detect which image format is used by the memory buffer -@@ -52,13 +54,13 @@ pub fn detectFormatFromMemory(buffer: []const u8) !Format { - } - - /// Load an image from a file path --pub fn fromFilePath(allocator: std.mem.Allocator, file_path: []const u8, read_buffer: []u8) !Managed { -- return (try Image.fromFilePath(allocator, file_path, read_buffer)).toManaged(allocator); -+pub fn fromFilePath(allocator: std.mem.Allocator, io: std.Io, file_path: []const u8, read_buffer: []u8) !Managed { -+ return (try Image.fromFilePath(allocator, io, file_path, read_buffer)).toManaged(allocator); - } - --/// Load an image from a standard library std.fs.File --pub fn fromFile(allocator: std.mem.Allocator, file: std.fs.File, read_buffer: []u8) !Managed { -- return (try Image.fromFile(allocator, file, read_buffer)).toManaged(allocator); -+/// Load an image from a standard library std.Io.File -+pub fn fromFile(allocator: std.mem.Allocator, io: std.Io, file: std.Io.File, read_buffer: []u8) !Managed { -+ return (try Image.fromFile(allocator, io, file, read_buffer)).toManaged(allocator); - } - - /// Load an image from a memory buffer -@@ -116,12 +118,12 @@ pub fn isAnimation(self: Managed) bool { - - /// Write the image to an image format to the specified path - pub fn writeToFilePath(self: Managed, file_path: []const u8, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { -- return Image.writeToFilePath(self.toUnmanaged(), self.allocator, file_path, write_buffer, encoder_options); -+ return Image.writeToFilePath(self.toUnmanaged(), self.allocator, self.io, file_path, write_buffer, encoder_options); - } - --/// Write the image to an image format to the specified std.fs.File --pub fn writeToFile(self: Managed, file: std.fs.File, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { -- return Image.writeToFile(self.toUnmanaged(), self.allocator, file, write_buffer, encoder_options); -+/// Write the image to an image format to the specified std.Io.File -+pub fn writeToFile(self: Managed, file: std.Io.File, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { -+ return Image.writeToFile(self.toUnmanaged(), self.allocator, file, self.io, write_buffer, encoder_options); - } - - /// Write the image to an image format in a memory buffer. The memory buffer is not grown -diff --git a/src/color.zig b/src/color.zig -index 4591bce..8a1a6ac 100644 ---- a/src/color.zig -+++ b/src/color.zig -@@ -55,10 +55,7 @@ pub fn ScaleValue(comptime T: type) fn (anytype) T { - // In order to fit `value * out_max` we need an - // int of size `in_bits + out_bits` in the worst - // case scenario. -- const FitInt = @Type(.{ .int = .{ -- .bits = in_bits + out_bits, -- .signedness = .unsigned, -- } }); -+ const FitInt = @Int(.unsigned, in_bits + out_bits); - - return @truncate((@as(FitInt, value) * out_max + in_max / 2) / in_max); - } else return value; -@@ -395,7 +392,7 @@ fn ToMethods( - return packed struct(u0) { - const To = @This(); - -- fn getSelf(to: *const To) *align(1) const T { -+ fn getSelf(to: anytype) *align(1) const T { - // @fieldParentPtr is broken for packed structs. - // See: https://github.com/ziglang/zig/issues/20458 - if (@typeInfo(T).@"struct".layout == .@"packed") { -@@ -408,47 +405,47 @@ fn ToMethods( - } - - /// Assumes the target color type has `FromMethods` for it. -- pub fn color(to: *const To, ColorT: type) ColorT { -- return ColorT.from.color(to.getSelf().*); -+ pub fn color(to: anytype, ColorT: type) ColorT { -+ return ColorT.from.color(getSelf(to).*); - } - -- pub fn u32Rgba(to: *const To) u32 { -- const self = to.getSelf(); -+ pub fn u32Rgba(to: anytype) u32 { -+ const self = getSelf(to); - return @as(u32, toU8(self.r)) << 24 | - @as(u32, toU8(self.g)) << 16 | - @as(u32, toU8(self.b)) << 8 | - if (has_alpha) toU8(self.a) else 0xff; - } - -- pub fn u32Rgb(to: *const To) u32 { -- const self = to.getSelf(); -+ pub fn u32Rgb(to: anytype) u32 { -+ const self = getSelf(to); - return @as(u32, toU8(self.r)) << 16 | - @as(u32, toU8(self.g)) << 8 | - toU8(self.b); - } - -- pub fn u64Rgba(to: *const To) u64 { -- const self = to.getSelf(); -+ pub fn u64Rgba(to: anytype) u64 { -+ const self = getSelf(to); - return @as(u64, toU16(self.r)) << 48 | - @as(u64, toU16(self.g)) << 32 | - @as(u64, toU16(self.b)) << 16 | - if (has_alpha) toU16(self.a) else 0xffff; - } - -- pub fn u64Rgb(to: *const To) u64 { -- const self = to.getSelf(); -+ pub fn u64Rgb(to: anytype) u64 { -+ const self = getSelf(to); - return @as(u64, toU16(self.r)) << 32 | - @as(u64, toU16(self.g)) << 16 | - toU16(self.b); - } - - /// Only valid for color types where all channels are the same type. -- pub fn array(to: *const To) [4]RedT { -+ pub fn array(to: anytype) [4]RedT { - if (comptime multiple_channel_types) { - @compileError("Color.to.array may only be used when all channels in the color are the same type."); - } - -- const self = to.getSelf(); -+ const self = getSelf(to); - - return .{ - self.r, -@@ -461,8 +458,8 @@ fn ToMethods( - }; - } - -- pub fn float4(to: *const To) math.float4 { -- const self = to.getSelf(); -+ pub fn float4(to: anytype) math.float4 { -+ const self = getSelf(to); - - return .{ - toF32(self.r), -@@ -477,8 +474,8 @@ fn ToMethods( - - /// For int channels, premultiplication - /// is done with a round-trip to f32. -- pub fn premultipliedAlpha(to: *const To) T { -- const self = to.getSelf(); -+ pub fn premultipliedAlpha(to: anytype) T { -+ const self = getSelf(to); - var res = self.*; - if (!has_alpha) return res; - -diff --git a/src/formats/pam.zig b/src/formats/pam.zig -index f366d95..7ef9833 100644 ---- a/src/formats/pam.zig -+++ b/src/formats/pam.zig -@@ -70,7 +70,7 @@ - var maybe_maxval: ?u16 = null; - var maybe_tuple_type: ?TupleType = null; -- var comments = std.ArrayListUnmanaged([]const u8){}; -+ var comments: std.ArrayListUnmanaged([]const u8) = .empty; - defer { - for (comments.items) |comment| allocator.free(comment); - comments.deinit(allocator); -@@ -77,7 +77,7 @@ const Header = struct { - } - - { -- var line_buffer_stream = try std.io.Writer.Allocating.initCapacity(allocator, 32); -+ var line_buffer_stream = try std.Io.Writer.Allocating.initCapacity(allocator, 32); - defer line_buffer_stream.deinit(); - - while (true) { -diff --git a/src/formats/pcx.zig b/src/formats/pcx.zig -index b694762..52c2bb0 100644 ---- a/src/formats/pcx.zig -+++ b/src/formats/pcx.zig -@@ -218,7 +218,7 @@ test "PCX RLE Fast encoder" { - const uncompressed_data = [_]u8{ 1, 1, 1, 1, 1, 1, 1, 1, 1, 64, 64, 2, 2, 2, 2, 2, 215, 215, 215, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 200, 200, 200, 200, 210, 210 }; - const compressed_data = [_]u8{ 0xC9, 0x01, 0xC2, 0x40, 0xC5, 0x02, 0xC3, 0xD7, 0xCA, 0x03, 0xC4, 0xC8, 0xC2, 0xD2 }; - -- var writer_alloc = std.io.Writer.Allocating.init(std.testing.allocator); -+ var writer_alloc = std.Io.Writer.Allocating.init(std.testing.allocator); - defer writer_alloc.deinit(); - - try RLEFastEncoder.encode(uncompressed_data[0..], &writer_alloc.writer); -@@ -233,7 +233,7 @@ test "PCX RLE Fast encoder should encore more than 63 bytes similar" { - - const compressed_data = [_]u8{ 0xFF, 0x45, 0x45, 0x45, 0xC4, 0x1 }; - -- var writer_alloc = std.io.Writer.Allocating.init(std.testing.allocator); -+ var writer_alloc = std.Io.Writer.Allocating.init(std.testing.allocator); - defer writer_alloc.deinit(); - - try RLEFastEncoder.encode(uncompressed_data[0..], &writer_alloc.writer); -diff --git a/src/formats/tiff/types.zig b/src/formats/tiff/types.zig -index 8bcce86..067480a 100644 ---- a/src/formats/tiff/types.zig -+++ b/src/formats/tiff/types.zig -@@ -335,7 +335,7 @@ pub const IFD = struct { - // Skip unknown tags instead of panicking on invalid enum values. - // Many TIFF files contain vendor/proprietary tags (Make, Model, ICC profile, etc.) - // that we don't need to handle for image decoding. -- const maybe_tag_id = std.meta.intToEnum(TagId, tag_id_raw) catch continue; -+ const maybe_tag_id = std.enums.fromInt(TagId, tag_id_raw) orelse continue; - - try self.tags_map.put(maybe_tag_id, TagField{ - .data_type = std.mem.toNative(u16, tag.data_type, endianess), -diff --git a/src/formats/xbm.zig b/src/formats/xbm.zig -index d72e4ec..19ac9c0 100644 ---- a/src/formats/xbm.zig -+++ b/src/formats/xbm.zig -@@ -100,7 +100,7 @@ pub const XBM = struct { - - var line_full = try reader.takeDelimiterInclusive('\n'); - while (line_full.len > 0) { -- const line = std.mem.trimRight(u8, line_full, " \r\n"); -+ const line = std.mem.trimEnd(u8, line_full, " \r\n"); - if (isDefineLine(line)) { - if (parseDefineValue(line)) |value| { - if (width == null) { -diff --git a/src/io.zig b/src/io.zig -index 2c1363d..8d463a1 100644 ---- a/src/io.zig -+++ b/src/io.zig -@@ -5,11 +5,11 @@ pub const DEFAULT_BUFFER_SIZE = 4096; - - pub const ReadStream = union(enum) { - memory: std.Io.Reader, -- file: std.fs.File.Reader, -+ file: std.Io.File.Reader, - - pub const Error = SeekError || EndPosError || std.Io.Reader.Error || std.Io.Reader.StreamError; -- pub const SeekError = std.fs.File.Reader.SeekError; -- pub const EndPosError = std.fs.File.Reader.SizeError; -+ pub const SeekError = std.Io.File.Reader.SeekError; -+ pub const EndPosError = std.Io.File.Reader.SizeError; - - pub fn initMemory(buffer: []const u8) ReadStream { - return .{ -@@ -17,9 +17,9 @@ pub const ReadStream = union(enum) { - }; - } - -- pub fn initFile(file: std.fs.File, buffer: []u8) ReadStream { -+ pub fn initFile(io: std.Io, file: std.Io.File, buffer: []u8) ReadStream { - return .{ -- .file = file.reader(buffer), -+ .file = file.reader(io, buffer), - }; - } - -@@ -58,7 +58,7 @@ pub const ReadStream = union(enum) { - .memory => |*memory| { - const new_pos: i64 = @as(i64, @intCast(memory.seek)) + offset; - if (new_pos < 0 or new_pos >= memory.end) { -- return std.fs.File.SeekError.Unseekable; -+ return std.Io.File.SeekError.Unseekable; - } - - memory.seek = @intCast(new_pos); -@@ -96,10 +96,10 @@ pub const ReadStream = union(enum) { - - pub const WriteStream = union(enum) { - memory: std.Io.Writer, -- file: std.fs.File.Writer, -+ file: std.Io.File.Writer, - - pub const Error = SeekError || std.Io.Writer.Error; -- pub const SeekError = std.fs.File.SeekError; -+ pub const SeekError = std.Io.File.SeekError || std.Io.Writer.Error; - - pub fn initMemory(buffer: []u8) WriteStream { - return .{ -@@ -107,9 +107,9 @@ pub const WriteStream = union(enum) { - }; - } - -- pub fn initFile(file: std.fs.File, buffer: []u8) WriteStream { -+ pub fn initFile(io: std.Io, file: std.Io.File, buffer: []u8) WriteStream { - return .{ -- .file = file.writer(buffer), -+ .file = file.writer(io, buffer), - }; - } - -diff --git a/src/math.zig b/src/math.zig -index c4e5910..bad9687 100644 ---- a/src/math.zig -+++ b/src/math.zig -@@ -24,11 +24,11 @@ pub fn Matrix(comptime T: type) type { - var temp = self; - - // To find the determinant, use a Gaussian elimination to transform the temp matrix into the row echelon form -- for (0..(ComponentSize - 1)) |row| { -- for ((row + 1)..ComponentSize) |next_row| { -+ inline for (0..(ComponentSize - 1)) |row| { -+ inline for ((row + 1)..ComponentSize) |next_row| { - const factor = temp.matrix[next_row][row] / temp.matrix[row][row]; - -- for (0..ComponentSize) |column| { -+ inline for (0..ComponentSize) |column| { - temp.matrix[next_row][column] = temp.matrix[next_row][column] - factor * temp.matrix[row][column]; - } - } -@@ -37,7 +37,7 @@ pub fn Matrix(comptime T: type) type { - // Once the temp matrix is in row echelon form, multiply the diagonal to find the determinant - var result: f32 = temp.matrix[0][0]; - -- for (1..ComponentSize) |diagonal| { -+ inline for (1..ComponentSize) |diagonal| { - result *= temp.matrix[diagonal][diagonal]; - } - -@@ -48,9 +48,9 @@ pub fn Matrix(comptime T: type) type { - pub fn fromArray(array: [ComponentSize * ComponentSize]f32) Self { - var result: Self = undefined; - -- for (0..ComponentSize) |row| { -+ inline for (0..ComponentSize) |row| { - const stride = row * ComponentSize; -- for (0..ComponentSize) |column| { -+ inline for (0..ComponentSize) |column| { - result.matrix[row][column] = array[stride + column]; - } - } -@@ -72,20 +72,20 @@ pub fn Matrix(comptime T: type) type { - var temp = self; - var result: Self = identity(); - -- for (0..ComponentSize) |row| { -+ inline for (0..ComponentSize) |row| { - // Transform the pivot to 1 by multiplying the row by its inverse - const inverse_pivot = 1.0 / temp.matrix[row][row]; - -- for (0..ComponentSize) |column| { -+ inline for (0..ComponentSize) |column| { - temp.matrix[row][column] *= inverse_pivot; - result.matrix[row][column] *= inverse_pivot; - } - - // Then do Gaussian elimination from current row to bottom -- for ((row + 1)..ComponentSize) |next_row| { -+ inline for ((row + 1)..ComponentSize) |next_row| { - const factor = temp.matrix[next_row][row] / temp.matrix[row][row]; - -- for (0..ComponentSize) |column| { -+ inline for (0..ComponentSize) |column| { - temp.matrix[next_row][column] = temp.matrix[next_row][column] - factor * temp.matrix[row][column]; - result.matrix[next_row][column] = result.matrix[next_row][column] - factor * result.matrix[row][column]; - } -@@ -94,14 +94,12 @@ pub fn Matrix(comptime T: type) type { - - // Then do a Gaussian elimination bottom-up to transform the temp matrix into an identity matrix. - // The result will be the inverse of the matrix -- var row: usize = ComponentSize -% 1; -- while (row >= 1 and row < ComponentSize) : (row -%= 1) { -- var previous_row = row -% 1; -- -- while (previous_row >= 0 and previous_row < ComponentSize) : (previous_row -%= 1) { -+ inline for (1..ComponentSize) |i| { -+ const row = ComponentSize - i; -+ inline for (0..row) |previous_row| { - const factor = temp.matrix[previous_row][row] / temp.matrix[row][row]; - -- for (0..ComponentSize) |column| { -+ inline for (0..ComponentSize) |column| { - temp.matrix[previous_row][column] = temp.matrix[previous_row][column] - factor * temp.matrix[row][column]; - result.matrix[previous_row][column] = result.matrix[previous_row][column] - factor * result.matrix[row][column]; - } -@@ -126,8 +124,8 @@ pub fn Matrix(comptime T: type) type { - - const transposed_right = right.transpose(); - -- for (0..ComponentSize) |row| { -- for (0..ComponentSize) |column| { -+ inline for (0..ComponentSize) |row| { -+ inline for (0..ComponentSize) |column| { - result.matrix[row][column] = @reduce(.Add, self.matrix[row] * transposed_right.matrix[column]); - } - } -@@ -138,8 +136,8 @@ pub fn Matrix(comptime T: type) type { - pub fn transpose(self: Self) Self { - var result = std.mem.zeroes(Self); - -- for (0..ComponentSize) |row| { -- for (0..ComponentSize) |column| { -+ inline for (0..ComponentSize) |row| { -+ inline for (0..ComponentSize) |column| { - result.matrix[row][column] = self.matrix[column][row]; - } - } -diff --git a/src/simd.zig b/src/simd.zig -index 20c5dcb..cb115c9 100644 ---- a/src/simd.zig -+++ b/src/simd.zig -@@ -78,8 +78,10 @@ pub fn vpermd(v: @Vector(8, i32), mask: @Vector(8, i32)) @Vector(8, i32) { - [v] "x" (v), - ); - } else { -- var res: @Vector(8, i32) = undefined; -- inline for (0..8) |i| res[i] = v[@as(u32, @bitCast(mask[i]))]; -+ const v_arr: [8]i32 = v; -+ const mask_arr: [8]i32 = mask; -+ var res: [8]i32 = undefined; -+ for (0..8) |i| res[i] = v_arr[@intCast(mask_arr[i] & 7)]; - return res; - } - } -diff --git a/src/utils.zig b/src/utils.zig -index f55c12c..a12733e 100644 ---- a/src/utils.zig -+++ b/src/utils.zig -@@ -4,7 +4,7 @@ const std = @import("std"); - const native_endian = builtin.target.cpu.arch.endian(); - - pub const StructReadError = error{InvalidData} || std.Io.Reader.Error; --pub const StructWriteError = std.io.StreamSource.WriteError; -+pub const StructWriteError = std.Io.Writer.Error; - - pub fn FixedStorage(comptime T: type, comptime storage_size: usize) type { - return struct { -diff --git a/tests/formats/bmp_test.zig b/tests/formats/bmp_test.zig -index cc80b74..635dc04 100644 ---- a/tests/formats/bmp_test.zig -+++ b/tests/formats/bmp_test.zig -@@ -87,12 +87,12 @@ fn verifyBitmapRGBAV5(the_bitmap: bmp.BMP, pixels: zigimg.color.PixelStorage) !v - - test "Read simple version 4 24-bit RGB bitmap" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_bitmap = bmp.BMP{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_bitmap.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -145,10 +145,10 @@ test "Read simple version 4 24-bit RGB bitmap" { - - test "Read a valid version 5 RGBA bitmap from file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/windows_rgba_v5.bmp"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var the_bitmap = bmp.BMP{}; - -@@ -174,10 +174,10 @@ test "Read a valid version 5 RGBA bitmap from memory" { - - test "Should error when reading an invalid file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/notbmp.png"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var the_bitmap = bmp.BMP{}; - -@@ -215,17 +215,17 @@ test "Write a v4 bitmap when writing an image with bgr24 pixel format" { - pixels.bgr24[7] = zigimg.color.Bgr24.from.rgb(255, 255, 0); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .bmp = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .bmp = .{} }); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var read_bmp = bmp.BMP{}; - -@@ -277,17 +277,17 @@ test "Write a v5 bitmap when writing an image with bgra32 pixel format" { - pixels.bgra32[8] = zigimg.color.Bgra32.from.rgba(0, 0, 0, 0); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .bmp = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .bmp = .{} }); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var read_bmp = bmp.BMP{}; - -diff --git a/tests/formats/farbfeld_test.zig b/tests/formats/farbfeld_test.zig -index a48a7fb..00a73a0 100644 ---- a/tests/formats/farbfeld_test.zig -+++ b/tests/formats/farbfeld_test.zig -@@ -5,10 +5,10 @@ const farbfeld = zigimg.formats.farbfeld; - - test "Farbfeld: Read yellow file" { - const yellow_file = try helpers.testOpenFile(helpers.fixtures_path ++ "farbfeld/yellow-1x1-semitransparent.png.ff"); -- defer yellow_file.close(); -+ defer yellow_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var yellow_read_stream = zigimg.io.ReadStream.initFile(yellow_file, read_buffer[0..]); -+ var yellow_read_stream = zigimg.io.ReadStream.initFile(std.testing.io, yellow_file, read_buffer[0..]); - - var yellow_image = farbfeld.Farbfeld{}; - const yellow_pixels = try yellow_image.read(helpers.zigimg_test_allocator, &yellow_read_stream); -@@ -21,10 +21,10 @@ test "Farbfeld: Read yellow file" { - - test "Farbfeld: read dragon file" { - const dragon_file = try helpers.testOpenFile(helpers.fixtures_path ++ "farbfeld/dragon.ff"); -- defer dragon_file.close(); -+ defer dragon_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var dragon_read_stream = zigimg.io.ReadStream.initFile(dragon_file, read_buffer[0..]); -+ var dragon_read_stream = zigimg.io.ReadStream.initFile(std.testing.io, dragon_file, read_buffer[0..]); - - var dragon_image = farbfeld.Farbfeld{}; - const dragon_pixels = try dragon_image.read(helpers.zigimg_test_allocator, &dragon_read_stream); -@@ -37,10 +37,10 @@ test "Farbfeld: read dragon file" { - - test "Farbfeld: invalid file format" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "farbfeld/dragon.png"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var farbfeld_image = farbfeld.Farbfeld{}; - const image_error = farbfeld_image.read(helpers.zigimg_test_allocator, &read_stream); -@@ -50,10 +50,10 @@ test "Farbfeld: invalid file format" { - - test "Farbfeld: read writeImage output" { - const source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "farbfeld/yellow-1x1-semitransparent.png.ff"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var source_read_stream = zigimg.io.ReadStream.initFile(source_file, read_buffer[0..]); -+ var source_read_stream = zigimg.io.ReadStream.initFile(std.testing.io, source_file, read_buffer[0..]); - - var source_image: farbfeld.Farbfeld = .{}; - const source_pixels = try source_image.read(helpers.zigimg_test_allocator, &source_read_stream); -diff --git a/tests/formats/gif_test.zig b/tests/formats/gif_test.zig -index 496e14f..bdd84f6 100644 ---- a/tests/formats/gif_test.zig -+++ b/tests/formats/gif_test.zig -@@ -325,10 +325,10 @@ test "GIF writer roundtrip read existing GIF, write, read again" { - - // Read an existing GIF file using low-level reader - const gif_input_file = try helpers.testOpenFile(helpers.fixtures_path ++ "gif/rotating_earth.gif"); -- defer gif_input_file.close(); -+ defer gif_input_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(gif_input_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, gif_input_file, read_buffer[0..]); - - var original_gif = gif.GIF.init(allocator); - defer original_gif.deinit(); -@@ -489,10 +489,10 @@ test "GIF writer rejects non-indexed data without auto convert" { - - test "Should error on non GIF images" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var gif_file = gif.GIF.init(helpers.zigimg_test_allocator); - defer gif_file.deinit(); -@@ -563,7 +563,7 @@ const SINGLE_GIF_FILE_TEST = false; - fn loadRotatingEarthImage(allocator: std.mem.Allocator) !zigimg.Image { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_path = helpers.fixtures_path ++ "gif/rotating_earth.gif"; -- return zigimg.Image.fromFilePath(allocator, image_path, read_buffer[0..]); -+ return zigimg.Image.fromFilePath(allocator, std.testing.io, image_path, read_buffer[0..]); - } - - test "GIF test suite" { -@@ -575,10 +575,10 @@ test "GIF test suite" { - defer test_list.deinit(helpers.zigimg_test_allocator); - - const test_list_file = try helpers.testOpenFile(helpers.fixtures_path ++ "gif/TESTS"); -- defer test_list_file.close(); -+ defer test_list_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(test_list_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, test_list_file, read_buffer[0..]); - - var reader = read_stream.reader(); - -@@ -598,11 +598,11 @@ test "GIF test suite" { - for (test_list.items) |entry| { - doGifTest(entry) catch |err| { - has_failed_file = true; -- std.debug.print("Error: {}\n", .{err}); -+ std.log.info("Error: {}\n", .{err}); - continue; - }; - -- std.debug.print("OK\n", .{}); -+ std.log.info("OK\n", .{}); - } - - if (has_failed_file) { -@@ -612,10 +612,10 @@ test "GIF test suite" { - - test "Rotating Earth GIF" { - const gif_input_file = try helpers.testOpenFile(helpers.fixtures_path ++ "gif/rotating_earth.gif"); -- defer gif_input_file.close(); -+ defer gif_input_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(gif_input_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, gif_input_file, read_buffer[0..]); - - var gif_file = gif.GIF.init(helpers.zigimg_test_allocator); - defer gif_file.deinit(); -@@ -710,8 +710,8 @@ const IniFile = struct { - const equals_sign_position_opt = std.mem.indexOf(u8, read_line[0..], "="); - - if (equals_sign_position_opt) |equals_sign_position| { -- const key_name = std.mem.trimRight(u8, read_line[0..(equals_sign_position - 1)], " "); -- const string_value = std.mem.trimLeft(u8, read_line[(equals_sign_position + 1)..], " "); -+ const key_name = std.mem.trimEnd(u8, read_line[0..(equals_sign_position - 1)], " "); -+ const string_value = std.mem.trimStart(u8, read_line[(equals_sign_position + 1)..], " "); - - if (self.sections.getPtr(current_section)) |section_entry| { - const value = blk: { -@@ -760,7 +760,7 @@ const IniFile = struct { - }; - - fn doGifTest(entry_name: []const u8) !void { -- std.debug.print("GIF test {s}... ", .{entry_name}); -+ std.log.info("GIF test {s}... ", .{entry_name}); - - var area_alloc = std.heap.ArenaAllocator.init(helpers.zigimg_test_allocator); - const area_allocator = area_alloc.allocator(); -@@ -770,13 +770,13 @@ fn doGifTest(entry_name: []const u8) !void { - const config_filepath = try std.fs.path.resolve(area_allocator, &[_][]const u8{ helpers.fixtures_path, "gif", config_filename }); - - const config_file = try helpers.testOpenFile(config_filepath); -- defer config_file.close(); -+ defer config_file.close(std.testing.io); - - var config_ini = IniFile.init(helpers.zigimg_test_allocator); - defer config_ini.deinit(); - - var ini_read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var ini_read_stream = zigimg.io.ReadStream.initFile(config_file, ini_read_buffer[0..]); -+ var ini_read_stream = zigimg.io.ReadStream.initFile(std.testing.io, config_file, ini_read_buffer[0..]); - - try config_ini.parse(ini_read_stream.reader()); - -@@ -804,10 +804,10 @@ fn doGifTest(entry_name: []const u8) !void { - - const gif_input_filepath = try std.fs.path.resolve(area_allocator, &[_][]const u8{ helpers.fixtures_path, "gif", input_filename.string }); - const gif_input_file = try helpers.testOpenFile(gif_input_filepath); -- defer gif_input_file.close(); -+ defer gif_input_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(gif_input_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, gif_input_file, read_buffer[0..]); - - var gif_file = gif.GIF.init(helpers.zigimg_test_allocator); - defer gif_file.deinit(); -@@ -856,10 +856,10 @@ fn doGifTest(entry_name: []const u8) !void { - const pixels_filename = frame_section.getValue("pixels") orelse return error.InvalidGifConfigFile; - const pixels_filepath = try std.fs.path.resolve(area_allocator, &[_][]const u8{ helpers.fixtures_path, "gif", pixels_filename.string }); - const pixels_file = try helpers.testOpenFile(pixels_filepath); -- defer pixels_file.close(); -+ defer pixels_file.close(std.testing.io); - - var pixels_read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var pixels_read_stream = zigimg.io.ReadStream.initFile(pixels_file, pixels_read_buffer[0..]); -+ var pixels_read_stream = zigimg.io.ReadStream.initFile(std.testing.io, pixels_file, pixels_read_buffer[0..]); - - var pixels_reader = pixels_read_stream.reader(); - -@@ -894,7 +894,7 @@ fn doGifTest(entry_name: []const u8) !void { - try helpers.expectEq(actual_color.to.color(zigimg.color.Rgba32), gif_background_color); - } else { - helpers.expectEq(actual_color.to.color(zigimg.color.Rgba32), expected_color) catch |err| { -- std.debug.print("Pixel #{} does not match: expected={}, actual={}\n", .{ index, expected_color, actual_color.to.color(zigimg.color.Rgba32) }); -+ std.log.info("Pixel #{} does not match: expected={}, actual={}\n", .{ index, expected_color, actual_color.to.color(zigimg.color.Rgba32) }); - return err; - }; - } -@@ -929,27 +929,27 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { - - // Build path to GIF file - const gif_filename = std.fmt.allocPrint(area_allocator, "{s}.gif", .{entry_name}) catch { -- std.debug.print("FAIL (allocPrint failed)\n", .{}); -+ std.log.info("FAIL (allocPrint failed)\n", .{}); - return .failed; - }; - const gif_filepath = std.fs.path.resolve(area_allocator, &[_][]const u8{ helpers.fixtures_path, "gif", gif_filename }) catch { -- std.debug.print("FAIL (path resolve failed)\n", .{}); -+ std.log.info("FAIL (path resolve failed)\n", .{}); - return .failed; - }; - - // Read original GIF -- const gif_input_file = try std.fs.cwd().openFile(gif_filepath, .{}); -- defer gif_input_file.close(); -+ const gif_input_file = try std.Io.Dir.openFile(std.Io.Dir.cwd(), std.testing.io, gif_filepath, .{}); -+ defer gif_input_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(gif_input_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, gif_input_file, read_buffer[0..]); - - var original_gif = gif.GIF.init(helpers.zigimg_test_allocator); - defer original_gif.deinit(); - - var original_frames = original_gif.read(&read_stream) catch { - // Reader correctly rejected bad data -- std.debug.print("OK (reader correctly rejected bad data)\n", .{}); -+ std.log.info("OK (reader correctly rejected bad data)\n", .{}); - return .passed; - }; - defer { -@@ -961,7 +961,7 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { - - if (original_frames.items.len == 0) { - // No frames means nothing to write -- std.debug.print("OK (no frames to write)\n", .{}); -+ std.log.info("OK (no frames to write)\n", .{}); - return .passed; - } - -@@ -974,13 +974,13 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { - - // Non-indexed formats can't be written without auto_convert - if (!image.pixelFormat().isIndexed()) { -- std.debug.print("OK (non-indexed format correctly not written: {s})\n", .{@tagName(image.pixels)}); -+ std.log.info("OK (non-indexed format correctly not written: {s})\n", .{@tagName(image.pixels)}); - return .passed; - } - - // Zero dimensions can't be written - if (image.width == 0 or image.height == 0) { -- std.debug.print("OK (zero dimensions correctly not written)\n", .{}); -+ std.log.info("OK (zero dimensions correctly not written)\n", .{}); - return .passed; - } - -@@ -991,16 +991,16 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { - gif.GIF.writeImage(helpers.zigimg_test_allocator, &write_stream, image, .{ .gif = .{} }) catch |err| { - // For very large images, running out of buffer space is expected - if (err == error.WriteFailed and (image.width > 1000 or image.height > 1000)) { -- std.debug.print("OK (large image exceeded buffer)\n", .{}); -+ std.log.info("OK (large image exceeded buffer)\n", .{}); - return .passed; - } -- std.debug.print("FAIL (write error: {})\n", .{err}); -+ std.log.info("FAIL (write error: {})\n", .{err}); - return .failed; - }; - - const written_len = write_stream.writer().end; - if (written_len == 0) { -- std.debug.print("FAIL (wrote 0 bytes)\n", .{}); -+ std.log.info("FAIL (wrote 0 bytes)\n", .{}); - return .failed; - } - -@@ -1011,7 +1011,7 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { - defer decoded_gif.deinit(); - - var decoded_frames = decoded_gif.read(&read_back_stream) catch |err| { -- std.debug.print("FAIL (read-back error: {})\n", .{err}); -+ std.log.info("FAIL (read-back error: {})\n", .{err}); - return .failed; - }; - defer { -@@ -1025,7 +1025,7 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { - if (original_gif.header.width != decoded_gif.header.width or - original_gif.header.height != decoded_gif.header.height) - { -- std.debug.print("FAIL (dimension mismatch: {}x{} vs {}x{})\n", .{ -+ std.log.info("FAIL (dimension mismatch: {}x{} vs {}x{})\n", .{ - original_gif.header.width, - original_gif.header.height, - decoded_gif.header.width, -@@ -1036,7 +1036,7 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { - - // Verify we got at least one frame - if (decoded_frames.items.len == 0) { -- std.debug.print("FAIL (no frames in output)\n", .{}); -+ std.log.info("FAIL (no frames in output)\n", .{}); - return .failed; - } - -@@ -1048,14 +1048,14 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { - while (original_iter.next()) |original_color| { - const decoded_color_opt = decoded_iter.next(); - if (decoded_color_opt == null) { -- std.debug.print("FAIL (missing pixel at index {})\n", .{pixel_index}); -+ std.log.info("FAIL (missing pixel at index {})\n", .{pixel_index}); - return .failed; - } - const original = original_color.to.color(zigimg.color.Rgba32); - const decoded = decoded_color_opt.?.to.color(zigimg.color.Rgba32); - - if (original.r != decoded.r or original.g != decoded.g or original.b != decoded.b) { -- std.debug.print("FAIL (pixel {} mismatch: ({},{},{}) vs ({},{},{}))\n", .{ -+ std.log.info("FAIL (pixel {} mismatch: ({},{},{}) vs ({},{},{}))\n", .{ - pixel_index, original.r, original.g, original.b, decoded.r, decoded.g, decoded.b, - }); - return .failed; -@@ -1071,10 +1071,10 @@ test "GIF writer roundtrip test suite" { - defer test_list.deinit(helpers.zigimg_test_allocator); - - const test_list_file = try helpers.testOpenFile(helpers.fixtures_path ++ "gif/TESTS"); -- defer test_list_file.close(); -+ defer test_list_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(test_list_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, test_list_file, read_buffer[0..]); - - var reader = read_stream.reader(); - -diff --git a/tests/formats/iff_test.zig b/tests/formats/iff_test.zig -index ab2dd6b..6a3594d 100644 ---- a/tests/formats/iff_test.zig -+++ b/tests/formats/iff_test.zig -@@ -5,12 +5,12 @@ const zigimg = @import("zigimg"); - - test "IFF-PBM indexed8 (chunky Deluxe Paint DOS file)" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-pbm.iff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_iff = iff.IFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -37,12 +37,12 @@ test "IFF-PBM indexed8 (chunky Deluxe Paint DOS file)" { - - test "IFF-ILBM indexed8 8 bitplanes" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-ilbm-8bit-compressed.iff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_iff = iff.IFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -63,12 +63,12 @@ test "IFF-ILBM indexed8 8 bitplanes" { - - test "IFF-ILBM indexed8 8 bitplanes uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-ilbm-8bit-uncompressed.iff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_iff = iff.IFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -89,12 +89,12 @@ test "IFF-ILBM indexed8 8 bitplanes uncompressed" { - - test "IFF-ILBM indexed8 6 bitplanes EHB" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-ehb.iff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_iff = iff.IFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -121,12 +121,12 @@ test "IFF-ILBM indexed8 6 bitplanes EHB" { - - test "IFF-ILBM indexed8 4 bitplanes HAM" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-ham.iff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_iff = iff.IFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -150,12 +150,12 @@ test "IFF-ILBM indexed8 4 bitplanes HAM" { - - test "IFF-ILBM indexed8 6 bitplanes HAM8" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-ham8.iff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_iff = iff.IFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -179,12 +179,12 @@ test "IFF-ILBM indexed8 6 bitplanes HAM8" { - - test "IFF-ILBM 24bit" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-24bit.iff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_iff = iff.IFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -208,12 +208,12 @@ test "IFF-ILBM 24bit" { - - test "IFF-ILBM indexed8 4 bitplanes Atari ST" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-ilbm-4bit-compressed-atari.iff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_iff = iff.IFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -240,12 +240,12 @@ test "IFF-ILBM indexed8 4 bitplanes Atari ST" { - - test "IFF-ACBM indexed8 3 bitplanes uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-8bit.acbm"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_iff = iff.IFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); - defer pixels.deinit(helpers.zigimg_test_allocator); -diff --git a/tests/formats/jpeg_test.zig b/tests/formats/jpeg_test.zig -index 8288b62..0f1254f 100644 ---- a/tests/formats/jpeg_test.zig -+++ b/tests/formats/jpeg_test.zig -@@ -7,10 +7,10 @@ const color = zigimg.color; - - test "Should error on non JPEG images" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); - defer jpeg_file.deinit(); -@@ -28,10 +28,10 @@ test "Should error on non JPEG images" { - - test "Read JFIF header properly and decode simple Huffman stream" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "jpeg/huff_simple0.jpg"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); - defer jpeg_file.deinit(); -@@ -59,10 +59,10 @@ test "Read JFIF header properly and decode simple Huffman stream" { - - test "Read the tuba properly" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "jpeg/tuba.jpg"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); - defer jpeg_file.deinit(); -@@ -95,10 +95,10 @@ test "Read the tuba properly" { - - test "Read grayscale images" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "jpeg/grayscale_sample0.jpg"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); - defer jpeg_file.deinit(); -@@ -130,19 +130,19 @@ test "Read grayscale images" { - } - - test "Read subsampling images" { -- var testdir = std.fs.cwd().openDir(helpers.fixtures_path ++ "jpeg/", .{ .access_sub_paths = false, .no_follow = true, .iterate = true }) catch null; -+ var testdir = std.Io.Dir.openDir(std.Io.Dir.cwd(), std.testing.io, helpers.fixtures_path ++ "jpeg/", .{ .access_sub_paths = false, .iterate = true }) catch null; - if (testdir) |*idir| { -- defer idir.close(); -+ defer idir.close(std.testing.io); - - var it = idir.iterate(); -- while (try it.next()) |entry| { -+ while (try it.next(std.testing.io)) |entry| { - if (entry.kind != .file or !std.mem.endsWith(u8, entry.name, ".jpg") or !std.mem.startsWith(u8, entry.name, "subsampling_")) continue; - -- var test_file = try idir.openFile(entry.name, .{ .mode = .read_only }); -- defer test_file.close(); -+ var test_file = try idir.openFile(std.testing.io, entry.name, .{ .mode = .read_only }); -+ defer test_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(test_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, test_file, read_buffer[0..]); - - var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); - defer jpeg_file.deinit(); -@@ -182,10 +182,10 @@ test "Read subsampling images" { - - test "Read progressive jpeg with restart intervals" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "jpeg/tuba_restart_prog.jpg"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); - defer jpeg_file.deinit(); -@@ -403,10 +403,10 @@ test "JPEG writer round-trip with all test fixtures" { - for (test_fixtures) |fixture_path| { - // Skip if file doesn't exist - const file = helpers.testOpenFile(fixture_path) catch continue; -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [4096]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); - defer jpeg_file.deinit(); - -diff --git a/tests/formats/netpbm_test.zig b/tests/formats/netpbm_test.zig -index ee06575..e0ffe05 100644 ---- a/tests/formats/netpbm_test.zig -+++ b/tests/formats/netpbm_test.zig -@@ -5,10 +5,10 @@ const zigimg = @import("zigimg"); - - test "Load ASCII PBM image" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/pbm_ascii.pbm"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var pbm_file = netpbm.PBM{}; - -@@ -27,10 +27,10 @@ test "Load ASCII PBM image" { - - test "Load binary PBM image" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/pbm_binary.pbm"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var pbm_file = netpbm.PBM{}; - -@@ -49,10 +49,10 @@ test "Load binary PBM image" { - - test "Load ASCII PGM 8-bit grayscale image" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/pgm_ascii_grayscale8.pgm"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var pgm_file = netpbm.PGM{}; - -@@ -71,10 +71,10 @@ test "Load ASCII PGM 8-bit grayscale image" { - - test "Load Binary PGM 8-bit grayscale image" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/pgm_binary_grayscale8.pgm"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var pgm_file = netpbm.PGM{}; - -@@ -93,10 +93,10 @@ test "Load Binary PGM 8-bit grayscale image" { - - test "Load ASCII PGM 16-bit grayscale image" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/pgm_ascii_grayscale16.pgm"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var pgm_file = netpbm.PGM{}; - -@@ -115,10 +115,10 @@ test "Load ASCII PGM 16-bit grayscale image" { - - test "Load Binary PGM 16-bit grayscale image" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/pgm_binary_grayscale16.pgm"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var pgm_file = netpbm.PGM{}; - -@@ -137,10 +137,10 @@ test "Load Binary PGM 16-bit grayscale image" { - - test "Load ASCII PPM image" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/ppm_ascii_rgb24.ppm"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var ppm_file = netpbm.PPM{}; - -@@ -176,10 +176,10 @@ test "Load ASCII PPM image" { - - test "Load binary PPM image" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/ppm_binary_rgb24.ppm"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var ppm_file = netpbm.PPM{}; - -@@ -234,16 +234,16 @@ test "Write bitmap(grayscale1) ASCII PBM file" { - } - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .pbm = .{ .binary = false }, - }); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch unreachable; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; - } - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, image_file_name, read_buffer[0..]); -+ var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, read_buffer[0..]); - defer read_image.deinit(helpers.zigimg_test_allocator); - - try helpers.expectEq(read_image.width, width); -@@ -281,16 +281,16 @@ test "Write bitmap(Grayscale1) binary PBM file" { - } - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .pbm = .{ .binary = true }, - }); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch unreachable; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; - } - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, image_file_name, read_buffer[0..]); -+ var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, read_buffer[0..]); - defer read_image.deinit(helpers.zigimg_test_allocator); - - try helpers.expectEq(read_image.width, width); -@@ -324,16 +324,16 @@ test "Write grayscale8 ASCII PGM file" { - } - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .pgm = .{ .binary = false }, - }); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch unreachable; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; - } - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, image_file_name, read_buffer[0..]); -+ var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, read_buffer[0..]); - defer read_image.deinit(helpers.zigimg_test_allocator); - - try helpers.expectEq(read_image.width, width); -@@ -367,16 +367,16 @@ test "Write grayscale8 binary PGM file" { - } - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .pgm = .{ .binary = true }, - }); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch unreachable; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; - } - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, image_file_name, read_buffer[0..]); -+ var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, read_buffer[0..]); - defer read_image.deinit(helpers.zigimg_test_allocator); - - try helpers.expectEq(read_image.width, width); -@@ -420,16 +420,16 @@ test "Writing Rgb24 ASCII PPM format" { - pixels.rgb24[7] = zigimg.color.Rgb24.from.rgb(255, 255, 0); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .ppm = .{ .binary = false }, - }); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch unreachable; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; - } - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, image_file_name, read_buffer[0..]); -+ var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, read_buffer[0..]); - defer read_image.deinit(helpers.zigimg_test_allocator); - - try helpers.expectEq(read_image.width, width); -@@ -474,16 +474,16 @@ test "Writing Rgb24 binary PPM format" { - pixels.rgb24[7] = zigimg.color.Rgb24.from.rgb(255, 255, 0); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .ppm = .{ .binary = true }, - }); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch unreachable; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; - } - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, image_file_name, read_buffer[0..]); -+ var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, read_buffer[0..]); - defer read_image.deinit(helpers.zigimg_test_allocator); - - try helpers.expectEq(read_image.width, width); -@@ -505,7 +505,7 @@ test "Trying to write a bitmap or grayscale Netpbm with an true color pixel form - defer source_image.deinit(helpers.zigimg_test_allocator); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch unreachable; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; - } - - const pixels = source_image.pixels; -@@ -527,12 +527,12 @@ test "Trying to write a bitmap or grayscale Netpbm with an true color pixel form - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - - { -- const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .pbm = .{} }); -+ const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .pbm = .{} }); - try std.testing.expectError(zigimg.Image.WriteError.Unsupported, write_error); - } - - { -- const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .pgm = .{} }); -+ const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .pgm = .{} }); - try std.testing.expectError(zigimg.Image.WriteError.Unsupported, write_error); - } - } -@@ -551,7 +551,7 @@ test "Trying to write a bitmap or true color Netpbm with a 8-bit grayscale pixel - defer source_image.deinit(helpers.zigimg_test_allocator); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch unreachable; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; - } - - const source = source_image.pixels; -@@ -562,12 +562,12 @@ test "Trying to write a bitmap or true color Netpbm with a 8-bit grayscale pixel - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - - { -- const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .pbm = .{} }); -+ const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .pbm = .{} }); - try std.testing.expectError(zigimg.Image.WriteError.Unsupported, write_error); - } - - { -- const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .ppm = .{} }); -+ const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .ppm = .{} }); - try std.testing.expectError(zigimg.Image.WriteError.Unsupported, write_error); - } - } -@@ -589,7 +589,7 @@ test "Trying to write a grayscale or true color Netbpm with a 1-bit grayscale pi - defer source_image.deinit(helpers.zigimg_test_allocator); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch unreachable; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; - } - - const source = source_image.pixels; -@@ -601,12 +601,12 @@ test "Trying to write a grayscale or true color Netbpm with a 1-bit grayscale pi - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - - { -- const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .pgm = .{} }); -+ const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .pgm = .{} }); - try std.testing.expectError(zigimg.Image.WriteError.Unsupported, write_error); - } - - { -- const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .ppm = .{} }); -+ const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .ppm = .{} }); - try std.testing.expectError(zigimg.Image.WriteError.Unsupported, write_error); - } - } -diff --git a/tests/formats/pam_test.zig b/tests/formats/pam_test.zig -index 2d58d45..5742f77 100644 ---- a/tests/formats/pam_test.zig -+++ b/tests/formats/pam_test.zig -@@ -5,10 +5,10 @@ const zigimg = @import("zigimg"); - - test "rejects non-PAM images" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - -@@ -17,10 +17,10 @@ test "rejects non-PAM images" { - - test "rejects PAM images with unsupported depth" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/unsupported_depth.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - -@@ -29,10 +29,10 @@ test "rejects PAM images with unsupported depth" { - - test "rejects PAM images with invalid maxval" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/invalid_maxval.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - -@@ -41,10 +41,10 @@ test "rejects PAM images with invalid maxval" { - - test "rejects PAM images with component values greater than maxval" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/value_greater_than_maxval.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - try helpers.expectError(invalid, zigimg.Image.ReadError.InvalidData); -@@ -52,10 +52,10 @@ test "rejects PAM images with component values greater than maxval" { - - test "rejects PAM images with unknown tuple type" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/unknown_tupletype.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - -@@ -64,10 +64,10 @@ test "rejects PAM images with unknown tuple type" { - - test "rejects PAM images with invalid first token" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/invalid_first_token.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - -@@ -76,10 +76,10 @@ test "rejects PAM images with invalid first token" { - - test "rejects PAM images with tuple type not matching other parameters" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/non_matching_tuple_type.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - -@@ -88,10 +88,10 @@ test "rejects PAM images with tuple type not matching other parameters" { - - test "accepts comments" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_blackandwhite_comments.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - defer image.deinit(helpers.zigimg_test_allocator); -@@ -110,10 +110,10 @@ test "accepts comments" { - - test "reads blackandwhite pam" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_blackandwhite.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - defer image.deinit(helpers.zigimg_test_allocator); -@@ -132,10 +132,10 @@ test "reads blackandwhite pam" { - - test "reads blackandwhite_alpha pam" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_blackandwhite_alpha.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - defer image.deinit(helpers.zigimg_test_allocator); -@@ -154,10 +154,10 @@ test "reads blackandwhite_alpha pam" { - - test "reads grayscale pam with maxval 255" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_grayscale_maxval_255.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - defer image.deinit(helpers.zigimg_test_allocator); -@@ -176,10 +176,10 @@ test "reads grayscale pam with maxval 255" { - - test "reads grayscale alpha pam with maxval 255" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_grayscale_alpha_maxval_255.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - defer image.deinit(helpers.zigimg_test_allocator); -@@ -198,10 +198,10 @@ test "reads grayscale alpha pam with maxval 255" { - - test "read of rgb pam with maxval 255" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/horse.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - defer image.deinit(helpers.zigimg_test_allocator); -@@ -211,10 +211,10 @@ test "read of rgb pam with maxval 255" { - - test "basic read-write-read produces same result" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_grayscale_alpha_maxval_255.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - defer image.deinit(helpers.zigimg_test_allocator); -@@ -239,7 +239,7 @@ test "basic read-write-read produces same result" { - - var decoded_image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_back_stream); - defer decoded_image.deinit(helpers.zigimg_test_allocator); -- -+ - try std.testing.expectEqual(@as(usize, 4), decoded_image.height); - try std.testing.expectEqual(@as(usize, 4), decoded_image.width); - try helpers.expectEqSlice(zigimg.color.Grayscale8Alpha, decoded_image.pixels.grayscale8Alpha, &[16]zigimg.color.Grayscale8Alpha{ -@@ -254,10 +254,10 @@ test "basic read-write-read produces same result" { - - test "reads rgba pam with maxval 255" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_rgba_maxval_255.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - defer image.deinit(helpers.zigimg_test_allocator); -@@ -273,10 +273,10 @@ test "reads rgba pam with maxval 255" { - - test "reads rgba pam with maxval 65535" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_rgba_maxval_65535.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - defer image.deinit(helpers.zigimg_test_allocator); -diff --git a/tests/formats/pcx_test.zig b/tests/formats/pcx_test.zig -index db5d556..ee51851 100644 ---- a/tests/formats/pcx_test.zig -+++ b/tests/formats/pcx_test.zig -@@ -7,10 +7,10 @@ const toU8 = zigimg.color.ScaleValue(u8); - - test "PCX indexed1 (linear)" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp1.pcx"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var pcx_file = pcx.PCX{}; - -@@ -45,10 +45,10 @@ test "PCX indexed1 (linear)" { - - test "PCX indexed4 (linear)" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp4.pcx"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var pcx_file = pcx.PCX{}; - -@@ -84,10 +84,10 @@ test "PCX indexed4 (linear)" { - - test "PCX indexed8 (linear)" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp8.pcx"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var pcx_file = pcx.PCX{}; - -@@ -125,10 +125,10 @@ test "PCX indexed8 (linear)" { - - test "PCX indexed24 (planar)" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp24.pcx"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var pcx_file = pcx.PCX{}; - -@@ -169,25 +169,25 @@ test "Write PCX indexed1 (odd width)" { - const image_file_name = "zigimg_pcx_indexed1_odd.pcx"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp1.pcx"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .pcx = .{}, - }); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var pcx_file = pcx.PCX{}; - -@@ -242,18 +242,18 @@ test "Write PCX indexed 1 (even width)" { - const image_file_name = "zigimg_pcx_indexed1_even.pcx"; - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try image_pattern.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try image_pattern.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .pcx = .{}, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var pcx_file = pcx.PCX{}; - -@@ -285,25 +285,25 @@ test "Write PCX indexed4 (odd width)" { - const image_file_name = "zigimg_pcx_indexed4_odd.pcx"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp4.pcx"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .pcx = .{}, - }); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var pcx_file = pcx.PCX{}; - -@@ -359,18 +359,18 @@ test "Write PCX indexed 4 (even width)" { - const image_file_name = "zigimg_pcx_indexed4_even.pcx"; - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try rainbow_test.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try rainbow_test.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .pcx = .{}, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var pcx_file = pcx.PCX{}; - -@@ -400,25 +400,25 @@ test "Write PCX indexed8 (odd width)" { - const image_file_name = "zigimg_pcx_indexed8_odd.pcx"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp8.pcx"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .pcx = .{}, - }); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var pcx_file = pcx.PCX{}; - -@@ -472,18 +472,18 @@ test "Write PCX indexed 8 (even width)" { - const image_file_name = "zigimg_pcx_indexed8_even.pcx"; - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try rainbow_test.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try rainbow_test.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .pcx = .{}, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var pcx_file = pcx.PCX{}; - -@@ -513,25 +513,25 @@ test "Write PCX rgb24 (odd width)" { - const image_file_name = "zigimg_pcx_bpp24_odd.pcx"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp24.pcx"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .pcx = .{}, - }); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var pcx_file = pcx.PCX{}; - -@@ -594,18 +594,18 @@ test "Write PCX rgb24 (even width)" { - const image_file_name = "zigimg_pcx_rgb24_even.pcx"; - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try rainbow_test.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try rainbow_test.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .pcx = .{}, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var pcx_file = pcx.PCX{}; - -diff --git a/tests/formats/png_test.zig b/tests/formats/png_test.zig -index ec39855..def25c7 100644 ---- a/tests/formats/png_test.zig -+++ b/tests/formats/png_test.zig -@@ -8,10 +8,10 @@ const valid_header_data = png.magic_header ++ "\x00\x00\x00\x0d" ++ png.Chunks.I - - test "png: Should error on non PNG images" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const invalidFile = png.PNG.readImage(helpers.zigimg_test_allocator, &read_stream); - -@@ -111,10 +111,10 @@ test "png: Indexed PNG with transparency (Aseprite output)" { - // mlarouche: While the full test suite already test this image, I like having a smaller test that I can verify - // some specific info myself - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "png/aseprite_indexed_transparent.png"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var png_image = try png.PNG.readImage(helpers.zigimg_test_allocator, &read_stream); - defer png_image.deinit(helpers.zigimg_test_allocator); -@@ -180,18 +180,18 @@ test "png: Don't write tRNS chunk in indexed format when there is no alpha" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_indexed_no_trns.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .png = .{}, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var check_trns_processor: CheckTrnsPresentProcessor = .{}; - -@@ -220,18 +220,18 @@ test "png: Write tRNS chunk in indexed format only when alpha is present" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_indexed_trns.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .png = .{}, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var check_trns_processor: CheckTrnsPresentProcessor = .{}; - -@@ -268,16 +268,16 @@ test "png: Write indexed1 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_indexed1.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -322,16 +322,16 @@ test "png: Write indexed2 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_indexed2.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -376,16 +376,16 @@ test "png: Write indexed4 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_indexed4.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -430,16 +430,16 @@ test "png: Write indexed8 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_indexed8.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -475,16 +475,16 @@ test "png: Write grayscale1 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_grayscale1.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -513,16 +513,16 @@ test "png: Write grayscale2 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_grayscale2.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -551,16 +551,16 @@ test "png: Write grayscale4 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_grayscale4.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -589,16 +589,16 @@ test "png: Write grayscale8 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_grayscale8.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -628,16 +628,16 @@ test "png: Write grayscale8Alpha format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_grayscale8Alpha.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -667,16 +667,16 @@ test "png: Write grayscale16 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_grayscale16.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -706,16 +706,16 @@ test "png: Write grayscale16Alpha format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_grayscale16Alpha.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -747,16 +747,16 @@ test "png: Write rgb24 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_rgb24.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -790,16 +790,16 @@ test "png: Write rgba32 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_rgba32.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -833,16 +833,16 @@ test "png: Write rgb48 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_rgb48.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -876,16 +876,16 @@ test "png: Write rgba64 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_rgba64.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -910,32 +910,32 @@ test "png: Official Test Suite" { - - // Useful to quickly test everything on full dir of images - pub fn testWithDir(directory: []const u8, test_md5_signature: bool) !void { -- var test_dir_opt = std.fs.cwd().openDir(directory, .{ .access_sub_paths = false, .no_follow = true, .iterate = true }) catch null; -+ var test_dir_opt = std.Io.Dir.openDir(std.Io.Dir.cwd(), std.testing.io, directory, .{ .access_sub_paths = false, .follow_symlinks = false, .iterate = true }) catch null; - if (test_dir_opt) |*test_dir| { -- defer test_dir.close(); -+ defer test_dir.close(std.testing.io); - var it = test_dir.iterate(); - if (test_md5_signature) { -- std.debug.print("\n", .{}); -+ std.log.info("\n", .{}); - } -- while (try it.next()) |entry| { -+ while (try it.next(std.testing.io)) |entry| { - if (entry.kind != .file or !std.mem.eql(u8, std.fs.path.extension(entry.name), ".png")) { - continue; - } - - if (test_md5_signature) { -- std.debug.print("Testing file {s} ... ", .{entry.name}); -+ std.log.info("Testing file {s} ... ", .{entry.name}); - } - -- var test_file = try test_dir.openFile(entry.name, .{ .mode = .read_only }); -- defer test_file.close(); -+ var test_file = try test_dir.openFile(std.testing.io, entry.name, .{ .mode = .read_only }); -+ defer test_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(test_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, test_file, read_buffer[0..]); - - if (entry.name[0] == 'x' and entry.name[2] != 't' and entry.name[2] != 's') { - try std.testing.expectError(zigimg.Image.ReadError.InvalidData, png.loadHeader(&read_stream)); - if (test_md5_signature) { -- std.debug.print("OK\n", .{}); -+ std.log.info("OK\n", .{}); - } - continue; - } -@@ -946,7 +946,7 @@ pub fn testWithDir(directory: []const u8, test_md5_signature: bool) !void { - const error_result = png.loadWithHeader(&read_stream, &header, std.testing.allocator, default_options.get()); - try std.testing.expectError(zigimg.Image.ReadError.InvalidData, error_result); - if (test_md5_signature) { -- std.debug.print("OK\n", .{}); -+ std.log.info("OK\n", .{}); - } - continue; - } -@@ -968,11 +968,11 @@ pub fn testWithDir(directory: []const u8, test_md5_signature: bool) !void { - @memcpy(test_data_name[len - 3 .. len], "tsd"); - - // Read test data and check with it -- if (test_dir.openFile(test_data_name[0..len], .{ .mode = .read_only })) |test_data_file| { -- defer test_data_file.close(); -+ if (test_dir.openFile(std.testing.io, test_data_name[0..len], .{ .mode = .read_only })) |test_data_file| { -+ defer test_data_file.close(std.testing.io); - - var test_read_buffer: [512]u8 = undefined; -- var test_file_reader = test_data_file.reader(test_read_buffer[0..]); -+ var test_file_reader = test_data_file.reader(std.testing.io, test_read_buffer[0..]); - var test_reader = &test_file_reader.interface; - - var expected_md5: [16]u8 = undefined; -@@ -983,31 +983,31 @@ pub fn testWithDir(directory: []const u8, test_md5_signature: bool) !void { - const str_md5 = try test_reader.takeDelimiterExclusive('\n'); - _ = try std.fmt.hexToBytes(expected_md5[0..], str_md5); - try std.testing.expectEqual(expected_pixel_format, std.meta.activeTag(result)); -- try std.testing.expectEqualSlices(u8, expected_md5[0..], md5_val[0..]); // catch std.debug.print("MD5 Expected: {s} Got {s}\n", .{std.fmt.fmtSliceHexUpper(expected_md5[0..]), std.fmt.fmtSliceHexUpper(md5_val[0..])}); -+ try std.testing.expectEqualSlices(u8, expected_md5[0..], md5_val[0..]); // catch std.log.info("MD5 Expected: {s} Got {s}\n", .{std.fmt.fmtSliceHexUpper(expected_md5[0..]), std.fmt.fmtSliceHexUpper(md5_val[0..])}); - } else |_| { - // If there is no test data assume test is correct and write it out - try writeTestData(test_dir, test_data_name[0..len], &result, md5_val[0..]); - } - - if (test_md5_signature) { -- std.debug.print("OK\n", .{}); -+ std.log.info("OK\n", .{}); - } - - // Write Raw bytes - // std.mem.copyForwards(u8, tst_data_name[len - 3 .. len + 1], "data"); - // var rawoutput = try idir.createFile(tst_data_name[0 .. len + 1], .{}); -- // defer rawoutput.close(); -+ // defer rawoutput.close(std.testing.io); - // try rawoutput.writeAll(result_bytes); - } - } - } - --fn writeTestData(dir: *std.fs.Dir, test_data_name: []const u8, result: *zigimg.color.PixelStorage, md5_value: []const u8) !void { -- var test_output = try dir.createFile(test_data_name, .{}); -- defer test_output.close(); -+fn writeTestData(dir: *std.Io.Dir, test_data_name: []const u8, result: *zigimg.color.PixelStorage, md5_value: []const u8) !void { -+ var test_output = try dir.createFile(std.testing.io, test_data_name, .{}); -+ defer test_output.close(std.testing.io); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var file_writer = test_output.writer(write_buffer[0..]); -+ var file_writer = test_output.writer(std.testing.io, write_buffer[0..]); - var writer = &file_writer.interface; - - try writer.print("{s}\n{X}", .{ @tagName(result.*), md5_value }); -@@ -1018,14 +1018,14 @@ fn writeTestData(dir: *std.fs.Dir, test_data_name: []const u8, result: *zigimg.c - test "png: InfoProcessor on Png Test suite" { - const directory = helpers.fixtures_path ++ "png/"; - -- var test_dir_opt = std.fs.cwd().openDir(directory, .{ .access_sub_paths = false, .no_follow = true, .iterate = true }) catch null; -+ var test_dir_opt = std.Io.Dir.openDir(std.Io.Dir.cwd(), std.testing.io, directory, .{ .access_sub_paths = false, .iterate = true }) catch null; - if (test_dir_opt) |*test_dir| { -- defer test_dir.close(); -+ defer test_dir.close(std.testing.io); - var it = test_dir.iterate(); - - var info_buffer: [16384]u8 = undefined; - -- while (try it.next()) |entry| { -+ while (try it.next(std.testing.io)) |entry| { - if (entry.kind != .file or !std.mem.eql(u8, std.fs.path.extension(entry.name), ".png")) { - continue; - } -@@ -1033,11 +1033,11 @@ test "png: InfoProcessor on Png Test suite" { - var info_stream = zigimg.io.WriteStream.initMemory(info_buffer[0..]); - var options = png.InfoProcessor.PngInfoOptions.init(png.InfoProcessor.init(info_stream.writer())); - -- var test_file = try test_dir.openFile(entry.name, .{ .mode = .read_only }); -- defer test_file.close(); -+ var test_file = try test_dir.openFile(std.testing.io, entry.name, .{ .mode = .read_only }); -+ defer test_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(test_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, test_file, read_buffer[0..]); - if (entry.name[0] == 'x') { - continue; - } -@@ -1051,13 +1051,13 @@ test "png: InfoProcessor on Png Test suite" { - @memcpy(test_data_name[len - 4 .. len], "info"); - - // Read test data and check with it -- if (test_dir.openFile(test_data_name[0..len], .{ .mode = .read_only })) |test_data_file| { -- defer test_data_file.close(); -+ if (test_dir.openFile(std.testing.io, test_data_name[0..len], .{ .mode = .read_only })) |test_data_file| { -+ defer test_data_file.close(std.testing.io); - - var expected_data_buffer: [16384]u8 = undefined; - - var test_read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var file_reader = test_data_file.reader(test_read_buffer[0..]); -+ var file_reader = test_data_file.reader(std.testing.io, test_read_buffer[0..]); - var reader = &file_reader.interface; - - const loaded = try reader.readSliceShort(expected_data_buffer[0..]); -@@ -1065,11 +1065,11 @@ test "png: InfoProcessor on Png Test suite" { - try std.testing.expectEqualSlices(u8, expected_data_buffer[0..loaded], info_buffer[0..loaded]); - } else |_| { - // If there is no test data assume test is correct and write it out -- var test_output_file = try test_dir.createFile(test_data_name[0..len], .{}); -- defer test_output_file.close(); -+ var test_output_file = try test_dir.createFile(std.testing.io, test_data_name[0..len], .{}); -+ defer test_output_file.close(std.testing.io); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var file_writer = test_output_file.writer(write_buffer[0..]); -+ var file_writer = test_output_file.writer(std.testing.io, write_buffer[0..]); - var writer = &file_writer.interface; - - try writer.writeAll(info_buffer[0..@intCast(info_stream.getPos())]); -diff --git a/tests/formats/qoi_test.zig b/tests/formats/qoi_test.zig -index 96256e3..5a9f985 100644 ---- a/tests/formats/qoi_test.zig -+++ b/tests/formats/qoi_test.zig -@@ -8,10 +8,10 @@ const zero_qoi_file = helpers.fixtures_path ++ "qoi/zero.qoi"; - - test "Should error on non QOI images" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var qoi_file = qoi.QOI{}; - -@@ -22,10 +22,10 @@ test "Should error on non QOI images" { - - test "Read zero.qoi file" { - const file = try helpers.testOpenFile(zero_qoi_file); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var qoi_file = qoi.QOI{}; - -diff --git a/tests/formats/ras_test.zig b/tests/formats/ras_test.zig -index d7240a1..5452098 100644 ---- a/tests/formats/ras_test.zig -+++ b/tests/formats/ras_test.zig -@@ -5,10 +5,10 @@ const helpers = @import("../helpers.zig"); - - test "Should error on non RAS images" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var ras_file = ras.RAS{}; - -@@ -18,12 +18,12 @@ test "Should error on non RAS images" { - - test "Sun-Raster 24-bit RGB24 uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-rgb24.ras"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var ras_file = ras.RAS{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -46,12 +46,12 @@ test "Sun-Raster 24-bit RGB24 uncompressed" { - - test "Sun-Raster 24-bit BGR24 uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-bgr24.ras"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var ras_file = ras.RAS{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -74,12 +74,12 @@ test "Sun-Raster 24-bit BGR24 uncompressed" { - - test "Sun-Raster 32-bit xRGB uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-xrgb32.ras"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var ras_file = ras.RAS{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -98,12 +98,12 @@ test "Sun-Raster 32-bit xRGB uncompressed" { - - test "Sun-Raster 8-bit with palette uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-8bit.ras"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var ras_file = ras.RAS{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -124,12 +124,12 @@ test "Sun-Raster 8-bit with palette uncompressed" { - - test "Sun-Raster 8-bit grayscale uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-8bit-grayscale.ras"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var ras_file = ras.RAS{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -144,12 +144,12 @@ test "Sun-Raster 8-bit grayscale uncompressed" { - - test "Sun-Raster 1-bit black & white uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-blackwhite.ras"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var ras_file = ras.RAS{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -164,12 +164,12 @@ test "Sun-Raster 1-bit black & white uncompressed" { - - test "Sun-Raster bgr24 rle compressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-24bit-bgr-rle.ras"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var ras_file = ras.RAS{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -diff --git a/tests/formats/sgi_test.zig b/tests/formats/sgi_test.zig -index d098ef7..ed23769 100644 ---- a/tests/formats/sgi_test.zig -+++ b/tests/formats/sgi_test.zig -@@ -5,10 +5,10 @@ const zigimg = @import("zigimg"); - - test "Should error on non SGI images" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var sgi_file = sgi.SGI{}; - -@@ -18,12 +18,12 @@ test "Should error on non SGI images" { - - test "SGI 24-bit uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-rgb24.sgi"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var sgi_file = sgi.SGI{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -46,12 +46,12 @@ test "SGI 24-bit uncompressed" { - - test "SGI grayscale uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-blackwhite.sgi"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var sgi_file = sgi.SGI{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -66,12 +66,12 @@ test "SGI grayscale uncompressed" { - - test "SGI 32-bit RGBA uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-rgba.sgi"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var sgi_file = sgi.SGI{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -94,12 +94,12 @@ test "SGI 32-bit RGBA uncompressed" { - - test "SGI RGB48be uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-rgb48be.sgi"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var sgi_file = sgi.SGI{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -122,12 +122,12 @@ test "SGI RGB48be uncompressed" { - - test "SGI grayscale rle compressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-gray-rle.sgi"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var sgi_file = sgi.SGI{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -142,12 +142,12 @@ test "SGI grayscale rle compressed" { - - test "SGI 24-bit rle compressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-24bit-rle.sgi"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var sgi_file = sgi.SGI{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -170,12 +170,12 @@ test "SGI 24-bit rle compressed" { - - test "SGI RGB48be rle uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-rgb48be-rle.sgi"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var sgi_file = sgi.SGI{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -198,12 +198,12 @@ test "SGI RGB48be rle uncompressed" { - - test "SGI 32-bit RGBA rle compressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-rgba-rle.sgi"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var sgi_file = sgi.SGI{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -diff --git a/tests/formats/tga_test.zig b/tests/formats/tga_test.zig -index 83f2d76..c3182d4 100644 ---- a/tests/formats/tga_test.zig -+++ b/tests/formats/tga_test.zig -@@ -5,10 +5,10 @@ const zigimg = @import("zigimg"); - - test "Should error on non TGA images" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -19,10 +19,10 @@ test "Should error on non TGA images" { - - test "Read ubw8 TGA file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ubw8.tga"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -56,10 +56,10 @@ test "Read ubw8 TGA file" { - - test "Read ucm8 TGA file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ucm8.tga"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -101,10 +101,10 @@ test "Read ucm8 TGA file" { - - test "Read utc16 TGA file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc16.tga"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -140,10 +140,10 @@ test "Read utc16 TGA file" { - - test "Read utc24 TGA file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc24.tga"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -179,10 +179,10 @@ test "Read utc24 TGA file" { - - test "Read utc32 TGA file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc32.tga"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -218,10 +218,10 @@ test "Read utc32 TGA file" { - - test "Read cbw8 TGA file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/cbw8.tga"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -255,10 +255,10 @@ test "Read cbw8 TGA file" { - - test "Read ccm8 TGA file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ccm8.tga"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -300,10 +300,10 @@ test "Read ccm8 TGA file" { - - test "Read ctc24 TGA file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ctc24.tga"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -339,10 +339,10 @@ test "Read ctc24 TGA file" { - - test "Read matte-01 TGA file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/matte-01.tga"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -384,10 +384,10 @@ test "Read matte-01 TGA file" { - - test "Read font TGA file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/font.tga"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -411,10 +411,10 @@ test "Read font TGA file" { - - test "Read stopsignsmall TGA v1 file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/stopsignsmall.tga"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -436,10 +436,10 @@ test "Read stopsignsmall TGA v1 file" { - - test "Read stopsignsmallcompressed TGA v1 file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/stopsignsmallcompressed.tga"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -463,11 +463,11 @@ test "Write TGA uncompressed grayscale8" { - const image_file_name = "zigimg_tga_uncompressed_grayscale8.tga"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ubw8.tga"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - const encoder_options = zigimg.Image.EncoderOptions{ -@@ -500,15 +500,15 @@ test "Write TGA uncompressed grayscale8" { - }; - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], encoder_options); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], encoder_options); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -558,15 +558,15 @@ test "Write TGA compressed grayscale8" { - const image_file_name = "zigimg_tga_compressed_grayscale8.tga"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/cbw8.tga"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = true, - .color_map_depth = 16, -@@ -575,13 +575,13 @@ test "Write TGA compressed grayscale8" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -617,15 +617,15 @@ test "Write uncompressed indexed8 (color map 16-bit)" { - const image_file_name = "zigimg_tga_uncompressed_indexed8.tga"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ucm8.tga"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = false, - .color_map_depth = 16, -@@ -634,13 +634,13 @@ test "Write uncompressed indexed8 (color map 16-bit)" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -684,15 +684,15 @@ test "Write uncompressed indexed8 (color map 24-bit)" { - const image_file_name = "zigimg_tga_uncompressed_indexed8.tga"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ucm8.tga"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = false, - .color_map_depth = 24, -@@ -701,13 +701,13 @@ test "Write uncompressed indexed8 (color map 24-bit)" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -751,15 +751,15 @@ test "Write compressed indexed8 (color map 16-bit)" { - const image_file_name = "zigimg_tga_compressed_indexed8.tga"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ccm8.tga"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = true, - .color_map_depth = 16, -@@ -768,13 +768,13 @@ test "Write compressed indexed8 (color map 16-bit)" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -818,15 +818,15 @@ test "Write compressed indexed8 (color map 24-bit)" { - const image_file_name = "zigimg_tga_compressed_indexed8.tga"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ccm8.tga"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = true, - .color_map_depth = 24, -@@ -835,13 +835,13 @@ test "Write compressed indexed8 (color map 24-bit)" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -885,15 +885,15 @@ test "Write uncompressed 16-bit true color TGA" { - const image_file_name = "zigimg_tga_uncompressed_true_color_16.tga"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc16.tga"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = false, - .color_map_depth = 16, -@@ -902,13 +902,13 @@ test "Write uncompressed 16-bit true color TGA" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -946,15 +946,15 @@ test "Write compressed 16-bit true color TGA" { - const image_file_name = "zigimg_tga_compressed_true_color_16.tga"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc16.tga"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = true, - .color_map_depth = 16, -@@ -963,13 +963,13 @@ test "Write compressed 16-bit true color TGA" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -1007,15 +1007,15 @@ test "Write uncompressed 24-bit true color TGA" { - const image_file_name = "zigimg_tga_uncompressed_true_color_24.tga"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc24.tga"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = false, - .color_map_depth = 24, -@@ -1024,13 +1024,13 @@ test "Write uncompressed 24-bit true color TGA" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -1068,15 +1068,15 @@ test "Write compressed 24-bit true color TGA" { - const image_file_name = "zigimg_tga_compressed_true_color_24.tga"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ctc24.tga"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = true, - .color_map_depth = 24, -@@ -1085,13 +1085,13 @@ test "Write compressed 24-bit true color TGA" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -1129,15 +1129,15 @@ test "Write uncompressed 32-bit true color TGA" { - const image_file_name = "zigimg_tga_uncompressed_true_color_32.tga"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc32.tga"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = false, - .color_map_depth = 24, -@@ -1146,13 +1146,13 @@ test "Write uncompressed 32-bit true color TGA" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -1190,15 +1190,15 @@ test "Write compressed 32-bit true color TGA" { - const image_file_name = "zigimg_tga_compressed_true_color_32.tga"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ctc32.tga"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = true, - .color_map_depth = 24, -@@ -1207,13 +1207,13 @@ test "Write compressed 32-bit true color TGA" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -1270,7 +1270,7 @@ test "Write uncompressed Rgb24 to TGA" { - @memcpy(source_image.pixels.rgb24[0..], uncompressed_source[0..]); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = false, - .color_map_depth = 24, -@@ -1279,14 +1279,14 @@ test "Write uncompressed Rgb24 to TGA" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -1329,7 +1329,7 @@ test "Write compressed Rgb24 to TGA" { - @memcpy(source_image.pixels.rgb24[0..], uncompressed_source[0..]); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = true, - .color_map_depth = 24, -@@ -1338,14 +1338,14 @@ test "Write compressed Rgb24 to TGA" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -1403,7 +1403,7 @@ test "Write uncompressed Rgba32 to TGA" { - @memcpy(source_image.pixels.rgba32[0..], uncompressed_source[0..]); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = false, - .color_map_depth = 24, -@@ -1412,14 +1412,14 @@ test "Write uncompressed Rgba32 to TGA" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -1478,7 +1478,7 @@ test "Write compressed Rgba32 to TGA" { - @memcpy(source_image.pixels.rgba32[0..], uncompressed_source[0..]); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = true, - .color_map_depth = 24, -@@ -1487,14 +1487,14 @@ test "Write compressed Rgba32 to TGA" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -diff --git a/tests/formats/tiff_test.zig b/tests/formats/tiff_test.zig -index cce4cfa..2a8a042 100644 ---- a/tests/formats/tiff_test.zig -+++ b/tests/formats/tiff_test.zig -@@ -5,10 +5,10 @@ const zigimg = @import("zigimg"); - - test "Should error on non TIFF images" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var sgi_file = tiff.TIFF{}; - -@@ -18,12 +18,12 @@ test "Should error on non TIFF images" { - - test "TIFF/LE monochrome black uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-monob-raw.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -39,12 +39,12 @@ test "TIFF/LE monochrome black uncompressed" { - - test "TIFF/LE grayscale8 uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-grayscale8-raw.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -61,12 +61,12 @@ test "TIFF/LE grayscale8 uncompressed" { - - test "TIFF/LE 8-bit with colormap uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-pal8-raw.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -87,12 +87,12 @@ test "TIFF/LE 8-bit with colormap uncompressed" { - - test "TIFF/LE 24-bit uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgb24-raw.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -115,12 +115,12 @@ test "TIFF/LE 24-bit uncompressed" { - - test "TIFF/BE rgb24 gray single strip uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/big-endian/sample-rgb24-single-strip.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -143,12 +143,12 @@ test "TIFF/BE rgb24 gray single strip uncompressed" { - - test "TIFF/BE rgb24 color single strip uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/big-endian/sample-pal8-raw.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -171,12 +171,12 @@ test "TIFF/BE rgb24 color single strip uncompressed" { - - test "TIFF/BE 24-bit uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/big-endian/sample-rgb24-raw.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -199,12 +199,12 @@ test "TIFF/BE 24-bit uncompressed" { - - test "TIFF/LE RGBA uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgba-raw.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -223,12 +223,12 @@ test "TIFF/LE RGBA uncompressed" { - - test "TIFF/LE monochrome black packbits" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-monob-packbits.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -244,12 +244,12 @@ test "TIFF/LE monochrome black packbits" { - - test "TIFF/LE grayscale8 packbits" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-grayscale8-packbits.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -266,12 +266,12 @@ test "TIFF/LE grayscale8 packbits" { - - test "TIFF/LE 8-bit with colormap packbits" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-pal8-packbits.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -292,12 +292,12 @@ test "TIFF/LE 8-bit with colormap packbits" { - - test "TIFF/LE 24-bit packbits" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgb24-packbits.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -320,12 +320,12 @@ test "TIFF/LE 24-bit packbits" { - - test "TIFF/LE RGBA packbits" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgba-packbits.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -344,12 +344,12 @@ test "TIFF/LE RGBA packbits" { - - test "TIFF/LE monochrome black CCITT" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/ccitt_rle.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -364,12 +364,12 @@ test "TIFF/LE monochrome black CCITT" { - - test "TIFF/LE monochrome black LZW" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-monob-lzw.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -385,12 +385,12 @@ test "TIFF/LE monochrome black LZW" { - - test "TIFF/LE grayscale8 LZW" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-grayscale8-lzw.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -407,12 +407,12 @@ test "TIFF/LE grayscale8 LZW" { - - test "TIFF/LE 8-bit with colormap LZW" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-pal8-lzw.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -433,12 +433,12 @@ test "TIFF/LE 8-bit with colormap LZW" { - - test "TIFF/LE 24-bit LZW" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgb24-lzw.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -461,12 +461,12 @@ test "TIFF/LE 24-bit LZW" { - - test "TIFF/LE RGBA LZW" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgba-lzw.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -485,12 +485,12 @@ test "TIFF/LE RGBA LZW" { - - test "TIFF/LE monochrome black Deflate" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-monob-deflate.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_bitmap = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_bitmap.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -506,12 +506,12 @@ test "TIFF/LE monochrome black Deflate" { - - test "TIFF/LE grayscale8 Deflate" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-grayscale8-deflate.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -528,12 +528,12 @@ test "TIFF/LE grayscale8 Deflate" { - - test "TIFF/LE 8-bit with colormap Deflate" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-pal8-deflate.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -554,12 +554,12 @@ test "TIFF/LE 8-bit with colormap Deflate" { - - test "TIFF/LE 24-bit Deflate" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgb24-deflate.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -582,12 +582,12 @@ test "TIFF/LE 24-bit Deflate" { - - test "TIFF/LE RGBA Deflate" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgba-deflate.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -diff --git a/tests/formats/xbm_test.zig b/tests/formats/xbm_test.zig -index 52eb29c..053e41e 100644 ---- a/tests/formats/xbm_test.zig -+++ b/tests/formats/xbm_test.zig -@@ -30,12 +30,12 @@ const simple_4x4_xbm = - test "XBM: invalid file format" { - { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "xbm/bad_missing_dim.xbm"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_xbm = xbm.XBM{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const actual_error = the_xbm.read(helpers.zigimg_test_allocator, &read_stream); - try helpers.expectError(actual_error, zigimg.Image.ReadError.InvalidData); -@@ -43,12 +43,12 @@ test "XBM: invalid file format" { - - { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "xbm/bad_missing_pixels.xbm"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_xbm = xbm.XBM{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const actual_error = the_xbm.read(helpers.zigimg_test_allocator, &read_stream); - try helpers.expectError(actual_error, zigimg.Image.ReadError.InvalidData); -diff --git a/tests/helpers.zig b/tests/helpers.zig -index 9a4ef29..b9f06b1 100644 ---- a/tests/helpers.zig -+++ b/tests/helpers.zig -@@ -30,17 +30,17 @@ pub inline fn expectApproxEqRel(actual: anytype, expected: anytype, tolerance: a - return try std.testing.expectApproxEqRel(expected, actual, tolerance); - } - --pub fn testOpenFile(file_path: []const u8) !std.fs.File { -- return std.fs.cwd().openFile(file_path, .{}) catch |err| -+pub fn testOpenFile(file_path: []const u8) !std.Io.File { -+ return std.Io.Dir.openFile(std.Io.Dir.cwd(), std.testing.io, file_path, .{}) catch |err| - if (err == error.FileNotFound) return error.SkipZigTest else return err; - } - - pub fn testImageFromFile(image_path: []const u8, buffer: []u8) !zigimg.Image { -- return zigimg.Image.fromFilePath(zigimg_test_allocator, image_path, buffer) catch |err| -+ return zigimg.Image.fromFilePath(zigimg_test_allocator, std.testing.io, image_path, buffer) catch |err| - if (err == error.FileNotFound) return error.SkipZigTest else return err; - } - - pub fn testReadFile(file_path: []const u8, buffer: []u8) ![]u8 { -- return std.fs.cwd().readFile(file_path, buffer) catch |err| -+ return std.Io.Dir.readFile(std.Io.Dir.cwd(), std.testing.io, file_path, buffer) catch |err| - if (err == error.FileNotFound) return error.SkipZigTest else return err; - } -diff --git a/tests/image_test.zig b/tests/image_test.zig -index ccd2d11..529be41 100644 ---- a/tests/image_test.zig -+++ b/tests/image_test.zig -@@ -187,7 +187,7 @@ test "Should detect BMP properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .bmp); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -213,7 +213,7 @@ test "Should detect GIF properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .gif); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -231,7 +231,7 @@ test "Should detect PCX properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .pcx); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -247,7 +247,7 @@ test "Should detect PBM properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .pbm); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -265,7 +265,7 @@ test "Should detect PGM properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .pgm); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -281,7 +281,7 @@ test "Should detect PPM properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .ppm); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -297,7 +297,7 @@ test "Should detect PNG properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .png); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -321,7 +321,7 @@ test "Should detect TGA properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .tga); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -334,7 +334,7 @@ test "Should detect QOI properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .qoi); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -350,7 +350,7 @@ test "Should detect JPEG properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .jpeg); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -366,7 +366,7 @@ test "Should detect Farbfeld properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .farbfeld); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -381,7 +381,7 @@ test "Should detect IFF/PBM properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .iff); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -396,7 +396,7 @@ test "Should detect RAS properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .ras); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -411,7 +411,7 @@ test "Should detect SGI properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .sgi); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -426,7 +426,7 @@ test "Should detect TIFF properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .tiff); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -449,7 +449,7 @@ test "Should detect XBM properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .xbm); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -diff --git a/tests/io_test.zig b/tests/io_test.zig -index 2dc6f7a..1ec5d60 100644 ---- a/tests/io_test.zig -+++ b/tests/io_test.zig -@@ -55,12 +55,12 @@ test "io.ReadStream: should read and seek properly within a unbuffered file" { - var temp_folder = std.testing.tmpDir(.{}); - defer temp_folder.cleanup(); - -- try temp_folder.dir.writeFile(.{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); -+ try temp_folder.dir.writeFile(std.testing.io, .{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); - -- const read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); -- defer read_file.close(); -+ const read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, &.{}); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, &.{}); - - var reader = read_stream.reader(); - -@@ -99,12 +99,12 @@ test "io.ReadStream: should error on invalid seek within unbuffered file" { - var temp_folder = std.testing.tmpDir(.{}); - defer temp_folder.cleanup(); - -- try temp_folder.dir.writeFile(.{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); -+ try temp_folder.dir.writeFile(std.testing.io, .{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); - -- const read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); -- defer read_file.close(); -+ const read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, &.{}); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, &.{}); - - const seek_to_error = read_stream.seekTo(123456); - try helpers.expectError(seek_to_error, zigimg.io.ReadStream.SeekError.Unseekable); -@@ -122,13 +122,13 @@ test "io.ReadStream: should read and seek properly within a file with a small bu - var temp_folder = std.testing.tmpDir(.{}); - defer temp_folder.cleanup(); - -- try temp_folder.dir.writeFile(.{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); -+ try temp_folder.dir.writeFile(std.testing.io, .{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); - -- const read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); -- defer read_file.close(); -+ const read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); -+ defer read_file.close(std.testing.io); - - var small_buffer: [30]u8 = @splat(0); -- var read_stream = zigimg.io.ReadStream.initFile(read_file, small_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, small_buffer[0..]); - - var reader = read_stream.reader(); - -@@ -164,13 +164,13 @@ test "io.ReadStream: should read and seek properly within a file with a large bu - var temp_folder = std.testing.tmpDir(.{}); - defer temp_folder.cleanup(); - -- try temp_folder.dir.writeFile(.{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); -+ try temp_folder.dir.writeFile(std.testing.io, .{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); - -- const read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); -- defer read_file.close(); -+ const read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var reader = read_stream.reader(); - -@@ -206,13 +206,13 @@ test "io.ReadStream: should error on invalid seek within a file with a large buf - var temp_folder = std.testing.tmpDir(.{}); - defer temp_folder.cleanup(); - -- try temp_folder.dir.writeFile(.{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); -+ try temp_folder.dir.writeFile(std.testing.io, .{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); - -- const read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); -- defer read_file.close(); -+ const read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - const seek_to_error = read_stream.seekTo(123456); - try helpers.expectError(seek_to_error, zigimg.io.ReadStream.SeekError.Unseekable); -@@ -261,10 +261,10 @@ test "io.WriteStream: should write property within a file with a small buffer" { - defer temp_folder.cleanup(); - - { -- const write_file = try temp_folder.dir.createFile(TEST_FILENAME, .{}); -- defer write_file.close(); -+ const write_file = try temp_folder.dir.createFile(std.testing.io, TEST_FILENAME, .{}); -+ defer write_file.close(std.testing.io); - -- var write_stream = zigimg.io.WriteStream.initFile(write_file, buffer[0..]); -+ var write_stream = zigimg.io.WriteStream.initFile(std.testing.io, write_file, buffer[0..]); - - var writer = write_stream.writer(); - try writer.writeAll(TEST_FILE_CONTENTS[0..3]); -@@ -283,11 +283,11 @@ test "io.WriteStream: should write property within a file with a small buffer" { - try writer.flush(); - } - -- var read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); -- defer read_file.close(); -+ var read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); -+ defer read_file.close(std.testing.io); - - var read_buffer: [1024]u8 = @splat(0); -- var file_reader = read_file.reader(read_buffer[0..]); -+ var file_reader = read_file.reader(std.testing.io, read_buffer[0..]); - - var reader = &file_reader.interface; - -@@ -305,11 +305,11 @@ test "io.WriteStream: should write property within a file with a large buffer" { - defer temp_folder.cleanup(); - - { -- const write_file = try temp_folder.dir.createFile(TEST_FILENAME, .{}); -- defer write_file.close(); -+ const write_file = try temp_folder.dir.createFile(std.testing.io, TEST_FILENAME, .{}); -+ defer write_file.close(std.testing.io); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var write_stream = zigimg.io.WriteStream.initFile(write_file, write_buffer[0..]); -+ var write_stream = zigimg.io.WriteStream.initFile(std.testing.io, write_file, write_buffer[0..]); - - var writer = write_stream.writer(); - try writer.writeAll(TEST_FILE_CONTENTS[0..3]); -@@ -328,11 +328,11 @@ test "io.WriteStream: should write property within a file with a large buffer" { - try writer.flush(); - } - -- var read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); -- defer read_file.close(); -+ var read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); -+ defer read_file.close(std.testing.io); - - var read_buffer: [1024]u8 = @splat(0); -- var file_reader = read_file.reader(read_buffer[0..]); -+ var file_reader = read_file.reader(std.testing.io, read_buffer[0..]); - - var reader = &file_reader.interface; - -diff --git a/zigimg.zig b/zigimg.zig -index 9f6b1a8..a53b23c 100644 ---- a/zigimg.zig -+++ b/zigimg.zig -@@ -38,5 +38,5 @@ test { - @import("tests/octree_quantizer_test.zig"), - @import("tests/pixel_format_converter_test.zig"), - @import("tests/pixel_format_test.zig"), -- }) |source_file| std.testing.refAllDeclsRecursive(source_file); -+ }) |source_file| std.testing.refAllDecls(source_file); - } -diff --git a/src/formats/gif.zig b/src/formats/gif.zig ---- a/src/formats/gif.zig -+++ b/src/formats/gif.zig -@@ -8,7 +8,7 @@ - const std = @import("std"); - const utils = @import("../utils.zig"); - --pub const HeaderFlags = packed struct { -+pub const HeaderFlags = packed struct(u8) { - global_color_table_size: u3 = 0, - sorted: bool = false, - color_resolution: u3 = 0, -@@ -129,9 +129,9 @@ - pub const GIF = struct { - header: Header = .{}, - global_color_table: utils.FixedStorage(color.Rgb24, 256) = .{}, -- frames: std.ArrayList(FrameData) = .{}, -- comments: std.ArrayList(CommentExtension) = .{}, -- application_infos: std.ArrayList(ApplicationExtension) = .{}, -+ frames: std.ArrayList(FrameData) = .empty, -+ comments: std.ArrayList(CommentExtension) = .empty, -+ application_infos: std.ArrayList(ApplicationExtension) = .empty, - arena_allocator: std.heap.ArenaAllocator = undefined, - - pub const SubImage = struct { -@@ -147,6 +147,6 @@ - pub const FrameData = struct { - graphics_control: ?GraphicControlExtension = null, -- sub_images: std.ArrayList(*SubImage) = .{}, -+ sub_images: std.ArrayList(*SubImage) = .empty, - - pub fn deinit(self: *FrameData, allocator: std.mem.Allocator) void { - for (self.sub_images.items) |sub_image| { -@@ -683,7 +683,7 @@ - const final_pixel_format = self.findBestPixelFormat(); - - const frame_list_allocator = self.arena_allocator.child_allocator; - -- var frame_list = Image.Animation.FrameList{}; -+ var frame_list: Image.Animation.FrameList = .empty; - - if (self.frames.items.len == 0) { -diff --git a/src/compressions/lzw.zig b/src/compressions/lzw.zig ---- a/src/compressions/lzw.zig -+++ b/src/compressions/lzw.zig -@@ -11,9 +11,10 @@ - next_code: u13 = 0, - previous_code: ?u13 = null, -- dictionary: std.AutoArrayHashMap(u13, []const u8), -+ dictionary: std.AutoArrayHashMapUnmanaged(u13, []const u8) = .empty, -+ dict_allocator: std.mem.Allocator, - - remaining_data: ?u13 = null, - remaining_bits: u4 = 0, -@@ -27,7 +28,7 @@ - var result = Self{ - .area_allocator = std.heap.ArenaAllocator.init(allocator), - .code_size = initial_code_size, -- .dictionary = std.AutoArrayHashMap(u13, []const u8).init(allocator), -+ .dict_allocator = allocator, - .initial_code_size = initial_code_size, - .clear_code = @as(u13, 1) << @intCast(initial_code_size), - .end_information_code = (@as(u13, 1) << @intCast(initial_code_size)) + 1, -@@ -43,7 +44,7 @@ - - pub fn deinit(self: *Self) void { - self.area_allocator.deinit(); -- self.dictionary.deinit(); -+ self.dictionary.deinit(self.dict_allocator); - } - - pub fn decode(self: *Self, reader: *std.Io.Reader, writer: *std.Io.Writer) !void { -@@ -85,7 +86,7 @@ - var new_value = try allocator.alloc(u8, previous_value.len + 1); - std.mem.copyForwards(u8, new_value, previous_value); - new_value[previous_value.len] = value[0]; -- try self.dictionary.put(self.next_code, new_value); -+ try self.dictionary.put(self.dict_allocator, self.next_code, new_value); - - self.next_code += 1; - -@@ -109,7 +110,7 @@ - var new_value = try allocator.alloc(u8, previous_value.len + 1); - std.mem.copyForwards(u8, new_value, previous_value); - new_value[previous_value.len] = previous_value[0]; -- try self.dictionary.put(self.next_code, new_value); -+ try self.dictionary.put(self.dict_allocator, self.next_code, new_value); - - _ = try writer.write(new_value); - -@@ -154,7 +155,7 @@ - var data = try allocator.alloc(u8, 1); - data[0] = @as(u8, @truncate(index)); - -- try self.dictionary.put(index, data); -+ try self.dictionary.put(self.dict_allocator, index, data); - } - } - }; -diff --git a/src/formats/jpeg/huffman.zig b/src/formats/jpeg/huffman.zig ---- a/src/formats/jpeg/huffman.zig -+++ b/src/formats/jpeg/huffman.zig -@@ -9,7 +9,7 @@ - - const HuffmanCode = struct { length_minus_one: u4, code: u16 }; --const HuffmanCodeMap = std.AutoArrayHashMap(HuffmanCode, u8); -+const HuffmanCodeMap = std.AutoArrayHashMapUnmanaged(HuffmanCode, u8); - - const JPEG_DEBUG = false; - const JPEG_VERY_DEBUG = false; -@@ -40,8 +40,8 @@ - for (code_counts) |count| total_huffman_codes += count; - -- var huffman_code_map = HuffmanCodeMap.init(allocator); -- errdefer huffman_code_map.deinit(); -+ var huffman_code_map: HuffmanCodeMap = .empty; -+ errdefer huffman_code_map.deinit(allocator); - - var fast_table: [1 << fast_bits]u8 = @splat(255); - var fast_size: [1 << fast_bits]u5 = @splat(0); -@@ -65,7 +65,7 @@ - - const byte = try reader.takeByte(); -- try huffman_code_map.put(.{ .length_minus_one = @as(u4, @intCast(i)), .code = code }, byte); -+ try huffman_code_map.put(allocator, .{ .length_minus_one = @as(u4, @intCast(i)), .code = code }, byte); - - // construct accelaration structure see stb_image - if (i + 1 <= fast_bits) { -@@ -96,5 +96,5 @@ - - pub fn deinit(self: *Table) void { -- self.code_map.deinit(); -+ self.code_map.deinit(self.allocator); - } - }; - diff --git a/third_party/zigimg/repo.bzl b/third_party/zigimg/repo.bzl index aa375cfc..282f0fb5 100644 --- a/third_party/zigimg/repo.bzl +++ b/third_party/zigimg/repo.bzl @@ -1,25 +1,12 @@ -def _zigimg_repo_impl(ctx): - ctx.download_and_extract( - url = "https://github.com/zigimg/zigimg/archive/{}.tar.gz".format(ctx.attr.commit), - stripPrefix = "zigimg-{}".format(ctx.attr.commit), - ) - - ctx.patch(ctx.attr.patch, strip = 1) - ctx.symlink(ctx.attr.build_file, "BUILD.bazel") - -_zigimg_repo = repository_rule( - implementation = _zigimg_repo_impl, - attrs = { - "commit": attr.string(mandatory = True), - "patch": attr.label(mandatory = True, allow_single_file = True), - "build_file": attr.label(mandatory = True, allow_single_file = True), - }, -) +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") def repo(): - _zigimg_repo( + git_repository( name = "zigimg", - commit = "7b98e82621fe302a9edc147df1191f4d1b7ff7a5", - patch = "//third_party/zigimg:bump-zig.patch", - build_file = "//third_party/zigimg:zigimg.bazel", + remote = "https://github.com/zigimg/zigimg.git", + # Match libvaxis build.zig.zon. + commit = "d695acd97c02e57bb151e8f659d1280f5cd6ca70", + build_file = Label("//third_party/zigimg:zigimg.bazel"), + patches = ["//third_party/zigimg:zig-0.17.patch"], + patch_args = ["-p1"], ) diff --git a/third_party/zigimg/zig-0.17.patch b/third_party/zigimg/zig-0.17.patch new file mode 100644 index 00000000..99a60b5d --- /dev/null +++ b/third_party/zigimg/zig-0.17.patch @@ -0,0 +1,279 @@ +diff --git a/src/Image/Managed.zig b/src/Image/Managed.zig +--- a/src/Image/Managed.zig ++++ b/src/Image/Managed.zig +@@ -18,7 +18,7 @@ + // This layout must match the one in Image + width: usize = 0, + height: usize = 0, +-pixels: color.PixelStorage = .{ .invalid = void{} }, ++pixels: color.PixelStorage = .{ .invalid = {} }, + animation: Animation = .{}, + allocator: std.mem.Allocator = undefined, // Allocator needs to be last in order to be able to ptrCast to Image + +diff --git a/src/Image.zig b/src/Image.zig +--- a/src/Image.zig ++++ b/src/Image.zig +@@ -9,7 +9,7 @@ + + width: usize = 0, + height: usize = 0, +-pixels: color.PixelStorage = .{ .invalid = void{} }, ++pixels: color.PixelStorage = .{ .invalid = {} }, + animation: Animation = .{}, + + const Image = @This(); +@@ -124,15 +124,15 @@ + const all_formats_delcs = std.meta.declarations(SupportedFormats); + var result: []const FormatInteraceFnType = &[0]FormatInteraceFnType{}; + for (all_formats_delcs) |decl| { +- const decl_value = @field(SupportedFormats, decl.name); ++ const decl_value = @field(SupportedFormats, decl); + const entry_type = @TypeOf(decl_value); + if (entry_type == type) { + const entry_type_info = @typeInfo(decl_value); + if (entry_type_info == .@"struct") { +- for (entry_type_info.@"struct".decls) |struct_entry| { +- if (std.mem.eql(u8, struct_entry.name, "formatInterface")) { ++ for (entry_type_info.@"struct".decl_names) |struct_entry| { ++ if (std.mem.eql(u8, struct_entry, "formatInterface")) { + result = result ++ [_]FormatInteraceFnType{ +- @field(decl_value, struct_entry.name), ++ @field(decl_value, struct_entry), + }; + break; + } +diff --git a/src/PixelFormatConverter.zig b/src/PixelFormatConverter.zig +--- a/src/PixelFormatConverter.zig ++++ b/src/PixelFormatConverter.zig +@@ -752,14 +752,7 @@ + } + + fn getFieldNameFromPixelFormat(comptime source_format: PixelFormat) []const u8 { +- const enum_fields = std.meta.fields(PixelFormat); +- inline for (enum_fields) |field| { +- if (field.value == @intFromEnum(source_format)) { +- return field.name; +- } +- } +- +- return ""; ++ return @tagName(source_format); + } + + // ======================== +@@ -782,19 +775,19 @@ + } + + fn grayscaleToGrayscale(comptime T: type, gray: anytype) T { +- const scaleValue = color.ScaleValue(std.meta.fieldInfo(T, .value).type); ++ const scaleValue = color.ScaleValue(@FieldType(T, "value")); + return .{ .value = scaleValue(gray.value) }; + } + + fn grayscaleAlphaToGrayscale(comptime T: type, gray: anytype) T { + const toF32 = color.ScaleValue(f32); +- const scaleValue = color.ScaleValue(std.meta.fieldInfo(T, .value).type); ++ const scaleValue = color.ScaleValue(@FieldType(T, "value")); + return .{ .value = scaleValue(toF32(gray.value) * toF32(gray.alpha)) }; + } + + fn grayscaleAlphaToGrayscaleAlpha(comptime T: type, gray: anytype) T { +- const scaleValue = color.ScaleValue(std.meta.fieldInfo(T, .value).type); +- const scaleAlpha = color.ScaleValue(std.meta.fieldInfo(T, .alpha).type); ++ const scaleValue = color.ScaleValue(@FieldType(T, "value")); ++ const scaleAlpha = color.ScaleValue(@FieldType(T, "alpha")); + return .{ + .value = scaleValue(gray.value), + .alpha = scaleAlpha(gray.alpha), +@@ -1101,7 +1094,7 @@ + var destination_pixels = @field(destination, getFieldNameFromPixelFormat(destination_format)); + const DestinationType = @TypeOf(destination_pixels[0]); + +- const scaleValue = color.ScaleValue(std.meta.fieldInfo(DestinationType, .value).type); ++ const scaleValue = color.ScaleValue(@FieldType(DestinationType, "value")); + + for (0..source_rgb.len) |index| { + const source_float4 = source_rgb[index].to.float4(); +@@ -1125,8 +1118,8 @@ + var destination_pixels = @field(destination, getFieldNameFromPixelFormat(destination_format)); + const DestinationType = @TypeOf(destination_pixels[0]); + +- const scaleValue = color.ScaleValue(std.meta.fieldInfo(DestinationType, .value).type); +- const scaleAlpha = color.ScaleValue(std.meta.fieldInfo(DestinationType, .alpha).type); ++ const scaleValue = color.ScaleValue(@FieldType(DestinationType, "value")); ++ const scaleAlpha = color.ScaleValue(@FieldType(DestinationType, "alpha")); + + for (0..source_rgb.len) |index| { + const source_float4 = source_rgb[index].to.float4(); +@@ -1482,7 +1475,7 @@ + var destination_pixels = @field(destination, getFieldNameFromPixelFormat(destination_format)); + const DestinationType = @TypeOf(destination_pixels[0]); + +- const scaleValue = color.ScaleValue(std.meta.fieldInfo(DestinationType, .value).type); ++ const scaleValue = color.ScaleValue(@FieldType(DestinationType, "value")); + + for (0..source_pixels.len) |index| { + const source_float4 = source_pixels[index].to.float4(); +@@ -1503,8 +1496,8 @@ + var destination_pixels = @field(destination, getFieldNameFromPixelFormat(destination_format)); + const DestinationType = @TypeOf(destination_pixels[0]); + +- const scaleValue = color.ScaleValue(std.meta.fieldInfo(DestinationType, .value).type); +- const scaleAlpha = color.ScaleValue(std.meta.fieldInfo(DestinationType, .alpha).type); ++ const scaleValue = color.ScaleValue(@FieldType(DestinationType, "value")); ++ const scaleAlpha = color.ScaleValue(@FieldType(DestinationType, "alpha")); + + for (0..source_pixels.len) |index| { + const source_float4 = source_pixels[index].to.float4(); +diff --git a/src/color.zig b/src/color.zig +--- a/src/color.zig ++++ b/src/color.zig +@@ -830,7 +830,7 @@ + return switch (format) { + .invalid => { + return .{ +- .invalid = void{}, ++ .invalid = {}, + }; + }, + .indexed1 => { +diff --git a/src/compressions/deflate/Lookup.zig b/src/compressions/deflate/Lookup.zig +--- a/src/compressions/deflate/Lookup.zig ++++ b/src/compressions/deflate/Lookup.zig +@@ -13,9 +13,9 @@ + const chain_len = 2 * consts.history.len; + + // Maps hash => first position +-head: [consts.lookup.len]u16 = [_]u16{0} ** consts.lookup.len, ++head: [consts.lookup.len]u16 = @splat(0), + // Maps position => previous positions for the same hash value +-chain: [chain_len]u16 = [_]u16{0} ** (chain_len), ++chain: [chain_len]u16 = @splat(0), + + // Calculates hash of the 4 bytes from data. + // Inserts `pos` position of that hash in the lookup tables. +diff --git a/src/compressions/deflate/huffman_encoder.zig b/src/compressions/deflate/huffman_encoder.zig +--- a/src/compressions/deflate/huffman_encoder.zig ++++ b/src/compressions/deflate/huffman_encoder.zig +@@ -338,7 +338,7 @@ + } + + pub fn huffmanDistanceEncoder() DistanceEncoder { +- var distance_freq = [1]u16{0} ** consts.distance_code_count; ++ var distance_freq = @as([consts.distance_code_count]u16, @splat(0)); + distance_freq[0] = 1; + // huff_distance is a static distance encoder used for huffman only encoding. + // It can be reused since we will not be encoding distance values. +diff --git a/src/formats/jpeg/writer.zig b/src/formats/jpeg/writer.zig +--- a/src/formats/jpeg/writer.zig ++++ b/src/formats/jpeg/writer.zig +@@ -20,7 +20,7 @@ + luminance, + chrominance, + }; +-const quant_index_size = @typeInfo(QuantIndex).@"enum".fields.len; ++const quant_index_size = @typeInfo(QuantIndex).@"enum".field_names.len; + + const n_huff_index = 4; + const HuffmanIndex = enum(usize) { +@@ -156,7 +156,7 @@ + + // Initialize Huffman LUT from Huffman specification + fn initHuffmanLUT(spec: HuffmanSpec) [256]u32 { +- var lut: [256]u32 = [_]u32{0} ** 256; ++ var lut: [256]u32 = @splat(0); + + var code: u32 = 0; + var value_index: usize = 0; +diff --git a/src/formats/pam.zig b/src/formats/pam.zig +--- a/src/formats/pam.zig ++++ b/src/formats/pam.zig +@@ -382,7 +382,7 @@ + + if (src_maxval == dst_maxval) return val; + +- const W = std.meta.Int(.unsigned, @bitSizeOf(T) * 2); ++ const W = @Int(.unsigned, @bitSizeOf(T) * 2); + return @intCast(@min(std.math.maxInt(T), @as(W, dst_maxval) * @as(W, val) / @as(W, src_maxval))); + } + +diff --git a/src/formats/tga.zig b/src/formats/tga.zig +--- a/src/formats/tga.zig ++++ b/src/formats/tga.zig +@@ -374,7 +374,7 @@ + const VectorType = @Vector(VectorLength, IntType); + const BytesPerPixels = (@typeInfo(IntType).int.bits + 7) / 8; + const IndexStep = VectorLength * BytesPerPixels; +- const MaskType = std.meta.Int(.unsigned, VectorLength); ++ const MaskType = @Int(.unsigned, VectorLength); + + comptime { + if (!std.math.isPowerOfTwo(@typeInfo(IntType).int.bits)) { +@@ -466,7 +466,7 @@ + if (std.mem.eql(u8, std.mem.asBytes(&rle_value), std.mem.asBytes(&value))) { + self.length += 1; + } else { +- try RunLengthEncoderCommon.flush(IntType, writer, @as(IntType, @bitCast(rle_value)), self.length); ++ try RunLengthEncoderCommon.flush(IntType, writer, std.mem.readInt(IntType, std.mem.asBytes(&rle_value), .little), self.length); + + self.length = 1; + self.rle_value = value; +@@ -480,7 +480,7 @@ + } + + if (self.rle_value) |rle_value| { +- try RunLengthEncoderCommon.flush(IntType, writer, @as(IntType, @bitCast(rle_value)), self.length); ++ try RunLengthEncoderCommon.flush(IntType, writer, std.mem.readInt(IntType, std.mem.asBytes(&rle_value), .little), self.length); + } + } + }; +@@ -1242,7 +1242,7 @@ + if (self.header.image_type.run_length) { + // The TGA spec recommend that the RLE compression should be done on scanline per scanline basis + inline for (1..(4 + 1)) |bpp| { +- const IntType = std.meta.Int(.unsigned, bpp * 8); ++ const IntType = @Int(.unsigned, bpp * 8); + + if (bytes_per_pixel == bpp) { + if (comptime std.math.isPowerOfTwo(bpp)) { +diff --git a/src/io.zig b/src/io.zig +--- a/src/io.zig ++++ b/src/io.zig +@@ -193,7 +193,7 @@ + } + + fn initBits(comptime T: type, out: anytype, num: u16) Bits(T) { +- const UT = std.meta.Int(.unsigned, @bitSizeOf(T)); ++ const UT = @Int(.unsigned, @bitSizeOf(T)); + return .{ + @bitCast(@as(UT, @intCast(out))), + num, +@@ -222,7 +222,7 @@ + /// containing them in the least significant end, and the number of bits successfully + /// read. Reaching the end of the stream is not an error. + pub fn readBitsTuple(self: *@This(), comptime T: type, num: u16) !Bits(T) { +- const UT = std.meta.Int(.unsigned, @bitSizeOf(T)); ++ const UT = @Int(.unsigned, @bitSizeOf(T)); + const U = if (@bitSizeOf(T) < 8) u8 else UT; //it is a pain to work with return result, + .INTR => continue, + else => return DirectIoError.UnexpectedFcntlResult, + } + } +} + +fn setStatusFlags(file: std.Io.File, flags: c_int) DirectIoError!void { + while (true) { + switch (std.posix.errno(std.posix.system.fcntl(file.handle, std.posix.F.SETFL, flags))) { + .SUCCESS => return, + .INTR => continue, + else => return DirectIoError.UnexpectedFcntlResult, + } + } +} + fn useDirectIO(file: std.Io.File) DirectIoError!bool { if (comptime canUseDirectIO()) { - const flags = std.posix.system.fcntl(file.handle, std.posix.F.GETFL); + const flags = try getStatusFlags(file); const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); return (flags & direct_flag) != 0; } else { @@ -33,12 +54,11 @@ fn useDirectIO(file: std.Io.File) DirectIoError!bool { fn switchToBufferedIO(file: std.fs.File) DirectIoError!void { if (comptime canUseDirectIO()) { - const flags = std.posix.system.fcntl(file.handle, std.posix.F.GETFL); + const flags = try getStatusFlags(file); const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); if ((flags & direct_flag) == 0) return; - const result = try std.posix.fcntl(file.handle, std.posix.F.SETFL, flags & ~@as(c_uint, @bitCast(@as(u32, @intCast(direct_flag))))); - if (result != 0) return DirectIoError.UnexpectedFcntlResult; + try setStatusFlags(file, flags & ~direct_flag); } else { return DirectIoError.UnsupportedPlatform; } @@ -46,11 +66,9 @@ fn switchToBufferedIO(file: std.fs.File) DirectIoError!void { fn switchToDirectIO(file: std.Io.File) DirectIoError!void { if (builtin.os.tag == .linux and canUseDirectIO()) { - const flags = std.posix.system.fcntl(file.handle, std.posix.F.GETFL); + const flags = try getStatusFlags(file); const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); - - const result = std.posix.system.fcntl(file.handle, std.posix.F.SETFL, flags | direct_flag); - if (result != 0) return DirectIoError.UnexpectedFcntlResult; + try setStatusFlags(file, flags | direct_flag); } else { return DirectIoError.UnsupportedPlatform; } @@ -111,6 +129,7 @@ pub const File = struct { .dirStat = dirStat, .dirStatFile = dirStatFile, .dirAccess = dirAccess, + .dirCreateFile = dirCreateFile, .dirOpenFile = dirOpenFile, .dirClose = dirClose, .dirRead = dirRead, @@ -119,6 +138,9 @@ pub const File = struct { .fileStat = fileStat, .fileLength = fileLength, .fileClose = fileClose, + .fileWritePositional = fileWritePositional, + .fileWriteFileStreaming = fileWriteFileStreaming, + .fileWriteFilePositional = fileWriteFilePositional, .fileReadPositional = fileReadPositional, .fileSeekBy = fileSeekBy, .fileSeekTo = fileSeekTo, @@ -151,7 +173,7 @@ pub const File = struct { return self.handles.at(@intCast(file.handle)); } - fn canOpenWithDirectIO(self: *File, file: std.Io.File, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!bool { + fn canOpenWithDirectIO(self: *File, file: std.Io.File, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!bool { const file_stat = self.base.inner.vtable.fileStat(self.base.inner.userdata, file) catch |err| switch (err) { else => return std.Io.File.OpenError.Unexpected, }; @@ -201,7 +223,18 @@ pub const File = struct { }, }); }, - .file_write_streaming, .device_io_control, .net_receive => { + .file_write_streaming => |o| { + const handle = self.getFileHandle(o.file); + return self.base.inner.vtable.operate(self.base.inner.userdata, .{ + .file_write_streaming = .{ + .file = handle.innerFile(), + .header = o.header, + .data = o.data, + .splat = o.splat, + }, + }); + }, + .device_io_control, .net_receive, .net_read, .net_send, .net_write => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -227,7 +260,21 @@ pub const File = struct { return self.base.inner.vtable.dirAccess(self.base.inner.userdata, dir, sub_path, options); } - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.CreateFileOptions) std.Io.File.OpenError!std.Io.File { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const inner_file = try self.base.inner.vtable.dirCreateFile(self.base.inner.userdata, dir, sub_path, flags); + errdefer inner_file.close(self.base.inner); + + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; + handle.* = .{ + .inner_handle = inner_file.handle, + .inner_flags = inner_file.flags, + .direct_io_handle = null, + }; + return .{ .handle = @intCast(idx), .flags = inner_file.flags }; + } + + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); const inner_file = try self.base.inner.vtable.dirOpenFile(self.base.inner.userdata, dir, sub_path, flags); errdefer inner_file.close(self.base.inner); @@ -311,6 +358,52 @@ pub const File = struct { } } + fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) std.Io.File.WritePositionalError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return self.base.inner.vtable.fileWritePositional(self.base.inner.userdata, handle.innerFile(), header, data, splat, offset); + } + + fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const own_io = self.io(); + if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; + + const destination = self.getFileHandle(file); + const source = self.getFileHandle(reader.file); + + const original_source_io = reader.io; + const original_source_file = reader.file; + reader.io = self.base.inner; + reader.file = source.innerFile(); + defer { + reader.io = original_source_io; + reader.file = original_source_file; + } + + return self.base.inner.vtable.fileWriteFileStreaming(self.base.inner.userdata, destination.innerFile(), header, reader, limit); + } + + fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const own_io = self.io(); + if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; + + const destination = self.getFileHandle(file); + const source = self.getFileHandle(reader.file); + + const original_source_io = reader.io; + const original_source_file = reader.file; + reader.io = self.base.inner; + reader.file = source.innerFile(); + defer { + reader.io = original_source_io; + reader.file = original_source_file; + } + + return self.base.inner.vtable.fileWriteFilePositional(self.base.inner.userdata, destination.innerFile(), header, reader, limit, offset); + } + fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); const inner_file = self.innerFile(file, data, offset); diff --git a/vfs/gcs.zig b/vfs/gcs.zig index 649e9958..acc7f634 100644 --- a/vfs/gcs.zig +++ b/vfs/gcs.zig @@ -652,7 +652,7 @@ pub const GCS = struct { handle.pos += @intCast(total); return .{ .file_read_streaming = total }; }, - .file_write_streaming, .device_io_control, .net_receive => { + .file_write_streaming, .device_io_control, .net_receive, .net_read, .net_send, .net_write => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -709,7 +709,7 @@ pub const GCS = struct { fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { error.FileNotFound => return std.Io.File.OpenError.FileNotFound, @@ -875,7 +875,7 @@ pub const GCS = struct { return try std.fmt.bufPrint(buf, "{d:0>4}{d:0>2}{d:0>2}T{d:0>2}{d:0>2}{d:0>2}Z", .{ year_day.year, - @intFromEnum(month_day.month), + @backingInt(month_day.month), month_day.day_index + 1, day_seconds.getHoursIntoDay(), day_seconds.getMinutesIntoHour(), diff --git a/vfs/hf.zig b/vfs/hf.zig index 2a1e9249..8770fe18 100644 --- a/vfs/hf.zig +++ b/vfs/hf.zig @@ -652,7 +652,7 @@ pub const HF = struct { handle.pos += @intCast(total); return .{ .file_read_streaming = total }; }, - .file_write_streaming, .device_io_control, .net_receive => { + .file_write_streaming, .device_io_control, .net_receive, .net_read, .net_send, .net_write => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -732,7 +732,7 @@ pub const HF = struct { if (node == null) return std.Io.Dir.AccessError.FileNotFound; } - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); var path_buffer: [8 * 1024]u8 = undefined; diff --git a/vfs/http.zig b/vfs/http.zig index ed8c5ee4..9054983e 100644 --- a/vfs/http.zig +++ b/vfs/http.zig @@ -156,7 +156,7 @@ pub const HTTP = struct { handle.pos += @intCast(total); return .{ .file_read_streaming = total }; }, - .file_write_streaming, .device_io_control, .net_receive => { + .file_write_streaming, .device_io_control, .net_receive, .net_read, .net_send, .net_write => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -210,7 +210,7 @@ pub const HTTP = struct { fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); const size = self.fetchSize(dir, sub_path) catch return std.Io.File.OpenError.Unexpected; diff --git a/vfs/s3.zig b/vfs/s3.zig index 51adcc7c..eb849e21 100644 --- a/vfs/s3.zig +++ b/vfs/s3.zig @@ -544,7 +544,7 @@ pub const S3 = struct { handle.pos += @intCast(total); return .{ .file_read_streaming = total }; }, - .file_write_streaming, .device_io_control, .net_receive => { + .file_write_streaming, .device_io_control, .net_receive, .net_read, .net_send, .net_write => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -601,7 +601,7 @@ pub const S3 = struct { fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { error.FileNotFound => return std.Io.File.OpenError.FileNotFound, @@ -767,7 +767,7 @@ pub const S3 = struct { return try std.fmt.bufPrint(buf, "{d:0>4}{d:0>2}{d:0>2}T{d:0>2}{d:0>2}{d:0>2}Z", .{ year_day.year, - @intFromEnum(month_day.month), + @backingInt(month_day.month), month_day.day_index + 1, day_seconds.getHoursIntoDay(), day_seconds.getMinutesIntoHour(), diff --git a/vfs/vfs.zig b/vfs/vfs.zig index b5e5c580..fe626ff9 100644 --- a/vfs/vfs.zig +++ b/vfs/vfs.zig @@ -187,7 +187,16 @@ fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable .data = o.data, } }); }, - .device_io_control, .file_write_streaming, .net_receive => { + .file_write_streaming => |o| { + const handle, const backend = self.getFileHandle(o.file); + return backend.vtable.operate(backend.userdata, .{ .file_write_streaming = .{ + .file = .{ .handle = @intCast(handle.handle), .flags = handle.flags }, + .header = o.header, + .data = o.data, + .splat = o.splat, + } }); + }, + .device_io_control, .net_receive, .net_read, .net_send, .net_write => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -235,7 +244,7 @@ pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, o return backend.vtable.dirAccess(backend.userdata, dir_, stripScheme(sub_path), options); } -pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.CreateFlags) std.Io.File.OpenError!std.Io.File { +pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.CreateFileOptions) std.Io.File.OpenError!std.Io.File { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { log.err("Failed to lookup backend for dir create file '{s}' : {any}", .{ sub_path, err }); @@ -243,25 +252,29 @@ pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u }; const file = try backend.vtable.dirCreateFile(backend.userdata, dir_, stripScheme(sub_path), flags); + errdefer file.close(backend); const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; handle.* = .{ .handle = @intCast(file.handle), .backend_idx = backend_idx, + .flags = file.flags, }; - return .{ .handle = @intCast(idx), .flags = .{ .nonblocking = false } }; + return .{ .handle = @intCast(idx), .flags = file.flags }; } -fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { +fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { log.err("Failed to lookup backend for opening file '{s}' : {any}", .{ sub_path, err }); return std.Io.File.OpenError.Unexpected; }; const file = try backend.vtable.dirOpenFile(backend.userdata, dir_, stripScheme(sub_path), flags); + errdefer file.close(backend); const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; handle.* = .{ .handle = @intCast(file.handle), .backend_idx = backend_idx, + .flags = file.flags, }; return .{ .handle = @intCast(idx), .flags = handle.flags }; } @@ -353,26 +366,42 @@ fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const own_io = self.io(); + if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; const dst_handle, const dst_backend = self.getFileHandle(file); - const src_handle, _ = self.getFileHandle(reader.file); + const src_handle, const src_backend = self.getFileHandle(reader.file); + if (src_backend.userdata != dst_backend.userdata or src_backend.vtable != dst_backend.vtable) return error.Unimplemented; + const original_src_io = reader.io; const original_src_handle = reader.file; + reader.io = src_backend; reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; - defer reader.file = original_src_handle; + defer { + reader.io = original_src_io; + reader.file = original_src_handle; + } return dst_backend.vtable.fileWriteFileStreaming(dst_backend.userdata, .{ .handle = @intCast(dst_handle.handle), .flags = dst_handle.flags }, header, reader, limit); } fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const own_io = self.io(); + if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; const dst_handle, const dst_backend = self.getFileHandle(file); - const src_handle, _ = self.getFileHandle(reader.file); + const src_handle, const src_backend = self.getFileHandle(reader.file); + if (src_backend.userdata != dst_backend.userdata or src_backend.vtable != dst_backend.vtable) return error.Unimplemented; + const original_src_io = reader.io; const original_src_handle = reader.file; + reader.io = src_backend; reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; - defer reader.file = original_src_handle; + defer { + reader.io = original_src_io; + reader.file = original_src_handle; + } return dst_backend.vtable.fileWriteFilePositional(dst_backend.userdata, .{ .handle = @intCast(dst_handle.handle), .flags = dst_handle.flags }, header, reader, limit, offset); } @@ -407,3 +436,48 @@ fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std. return try backend.vtable.fileRealPath(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, out_buffer); } } + +test "copy a local file through the VFS" { + const allocator = std.testing.allocator; + const process_io = std.testing.io; + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + const source_data: [8192]u8 = @splat('x'); + try tmp.dir.writeFile(process_io, .{ + .sub_path = "source", + .data = &source_data, + }); + + var path_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path_len = try tmp.dir.realPath(process_io, &path_buffer); + const source_uri = try std.fmt.allocPrint(allocator, "file://{s}/source", .{path_buffer[0..path_len]}); + defer allocator.free(source_uri); + const destination_uri = try std.fmt.allocPrint(allocator, "file://{s}/destination", .{path_buffer[0..path_len]}); + defer allocator.free(destination_uri); + + var file_backend: File = .init(allocator, process_io, .{}); + defer file_backend.deinit(); + var vfs: VFS = try .init(allocator, process_io); + defer vfs.deinit(); + try vfs.register("file", file_backend.io()); + + const vfs_io = vfs.io(); + { + var source = try std.Io.Dir.openFile(.cwd(), vfs_io, source_uri, .{}); + defer source.close(vfs_io); + var destination = try std.Io.Dir.createFile(.cwd(), vfs_io, destination_uri, .{}); + defer destination.close(vfs_io); + + var reader: std.Io.File.Reader = .initStreaming(source, vfs_io, &.{}); + var copy_buffer: [4096]u8 align(4096) = undefined; + var writer = destination.writer(vfs_io, ©_buffer); + try std.testing.expectEqual(source_data.len, try reader.interface.streamRemaining(&writer.interface)); + try writer.interface.flush(); + } + + const copied = try tmp.dir.readFileAlloc(process_io, "destination", allocator, .limited(source_data.len + 1)); + defer allocator.free(copied); + try std.testing.expectEqualSlices(u8, &source_data, copied); +} diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 41f175ae..89bc2bb6 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -157,7 +157,7 @@ pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, var mlir_known_types: std.enums.EnumArray(DataType, *const mlir.Type) = .initUndefined(); { for (0.., &mlir_known_types.values) |i, *mlir_type| { - mlir_type.* = mlirx.Type.fromDType(mlir_ctx, @enumFromInt(i)); + mlir_type.* = mlirx.Type.fromDType(mlir_ctx, @fromBackingInt(@intCast(i))); } } @@ -309,7 +309,7 @@ pub fn mlirType(self: *const Compiler, dt: DataType) *const mlir.Type { pub fn dtype(self: *const Compiler, mlir_type: *const mlir.Type) DataType { @setRuntimeSafety(false); for (0.., &self.mlir_known_types.values) |i, known_type| { - if (known_type == mlir_type) return @enumFromInt(i); + if (known_type == mlir_type) return @fromBackingInt(@intCast(i)); } std.debug.panic("Can't convert unknown mlir type to dtype: {f}", .{mlir_type}); } diff --git a/zml/Sharding.zig b/zml/Sharding.zig index 761721fa..b8005144 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -135,21 +135,21 @@ pub const PartitionSpec = enum(u4) { open = 15, pub fn sharded(mesh_axis: u3) PartitionSpec { - return @enumFromInt(mesh_axis); + return @fromBackingInt(@intCast(mesh_axis)); } /// Extract the mesh axis along which we are sharded. Null if not sharded. pub fn meshAxis(self: PartitionSpec) ?u3 { - const ax = @intFromEnum(self); - return if (ax < @intFromEnum(PartitionSpec.replicated)) @intCast(ax) else null; + const ax = @backingInt(self); + return if (ax < @backingInt(PartitionSpec.replicated)) @intCast(ax) else null; } pub fn isSharded(self: PartitionSpec) bool { - return @intFromEnum(self) < @intFromEnum(PartitionSpec.replicated); + return @backingInt(self) < @backingInt(PartitionSpec.replicated); } pub fn isClosed(self: PartitionSpec) bool { - return @intFromEnum(self) <= @intFromEnum(PartitionSpec.replicated); + return @backingInt(self) <= @backingInt(PartitionSpec.replicated); } test isClosed { @@ -285,13 +285,13 @@ pub const Partitioning = packed struct { } pub fn splat(spec: PartitionSpec) Partitioning { - const vec: Vec = @splat(@intFromEnum(spec)); + const vec: Vec = @splat(@backingInt(spec)); return @bitCast(vec); } pub fn repeat(spec: PartitionSpec, rank_: usize) Partitioning { std.debug.assert(rank_ <= MAX_RANK); - const splatted: Vec = @splat(@intFromEnum(spec)); + const splatted: Vec = @splat(@backingInt(spec)); const mask = std.simd.iota(u4, 8) < @as(Vec, @splat(@truncate(rank_))); return @bitCast(@select(u4, mask, splatted, @as(Vec, @bitCast(out_of_bound)))); } @@ -300,7 +300,7 @@ pub const Partitioning = packed struct { std.debug.assert(ax < MAX_RANK); const pack: u32 = @bitCast(p); const shift: u5 = @intCast(4 * ax); - return @enumFromInt(@as(u4, @truncate(pack >> shift))); + return @fromBackingInt(@truncate(pack >> shift)); } pub fn set(p: Partitioning, ax: usize, spec: PartitionSpec) Partitioning { @@ -308,7 +308,7 @@ pub const Partitioning = packed struct { const pack: u32 = @bitCast(p); const shift: u5 = @intCast(4 * ax); const mask = @as(u32, 0xf) << shift; - return @bitCast((pack & ~mask) | (@as(u32, @intFromEnum(spec)) << shift)); + return @bitCast((pack & ~mask) | (@as(u32, @backingInt(spec)) << shift)); } pub fn rank(p: Partitioning) u8 { @@ -325,7 +325,7 @@ pub const Partitioning = packed struct { const pack: u32 = @bitCast(p); const shift: u5 = @intCast(4 * ax); const lower_mask = (@as(u32, 1) << shift) - 1; - return @bitCast((pack & lower_mask) | ((pack & ~lower_mask) << 4) | (@as(u32, @intFromEnum(spec)) << shift)); + return @bitCast((pack & lower_mask) | ((pack & ~lower_mask) << 4) | (@as(u32, @backingInt(spec)) << shift)); } /// Removes a spec, shifting subsequent slots left and filling the last with unknown. @@ -334,7 +334,7 @@ pub const Partitioning = packed struct { const pack: u32 = @bitCast(p); const shift: u5 = @intCast(4 * ax); const lower_mask = (@as(u32, 1) << shift) - 1; - return @bitCast((pack & lower_mask) | ((pack >> 4) & ~lower_mask) | (@as(u32, @intFromEnum(PartitionSpec.out_of_bound)) << 28)); + return @bitCast((pack & lower_mask) | ((pack >> 4) & ~lower_mask) | (@as(u32, @backingInt(PartitionSpec.out_of_bound)) << 28)); } pub fn toArray(p: Partitioning) [MAX_RANK]PartitionSpec { @@ -344,7 +344,7 @@ pub const Partitioning = packed struct { } pub fn hasUniqueAxes(p: Partitioning) bool { - var used_mesh_axes: std.StaticBitSet(MAX_RANK) = .initEmpty(); + var used_mesh_axes: std.StaticBitSet(MAX_RANK) = .empty; for (0..MAX_RANK) |shape_ax| { const spec = p.get(shape_ax); if (spec.meshAxis()) |mesh_axis| { @@ -383,7 +383,7 @@ pub const Partitioning = packed struct { test "insertion and removal at every slot" { var parts: Partitioning = .out_of_bound; - for (0..MAX_RANK) |ax| parts = parts.set(ax, @enumFromInt(ax)); + for (0..MAX_RANK) |ax| parts = parts.set(ax, @fromBackingInt(@intCast(ax))); const original = parts.toArray(); for (0..MAX_RANK) |ax| { var inserted = original; @@ -414,13 +414,13 @@ pub const Partitioning = packed struct { const T = @TypeOf(partitioning); stdx.debug.assertComptime(stdx.meta.isStruct(T), "parsePartitioning expected a struct of enum literals eg {{ .b = .data, .d = .model }}, got: {any}", .{T}); - inline for (std.meta.fields(T)) |field| { - const shape_tag = Shape.toTag(field); + inline for (comptime std.meta.fieldNames(T)) |field_name| { + const shape_tag = Shape.toTag(field_name); const shape_ax = Shape.axisFromTagMaybe(tags, shape_tag) orelse { - std.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), field.name }); + std.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), field_name }); }; - const value = @field(partitioning, field.name); + const value = @field(partitioning, field_name); const spec: PartitionSpec = if (@TypeOf(value) == PartitionSpec) value else switch (value) { .replicated => .replicated, .out_of_bound => std.debug.panic("value out_of_bound not allowed", .{}), @@ -603,11 +603,11 @@ pub const PhysicalMesh = struct { try writer.writeAll("], depth_by_tag={"); - const fields = std.meta.fields(PhysicalAxisTag); - inline for (fields, 0..) |field, i| { + const field_names = comptime std.meta.fieldNames(PhysicalAxisTag); + inline for (field_names, 0..) |field_name, i| { if (i > 0) try writer.writeAll(", "); - const tag = @field(PhysicalAxisTag, field.name); - try writer.writeAll(field.name); + const tag = @field(PhysicalAxisTag, field_name); + try writer.writeAll(field_name); try writer.writeAll("="); if (self.depth_by_tag.get(tag)) |d| { try writer.print("{d}", .{d}); @@ -1349,9 +1349,9 @@ pub const LogicalMesh = struct { var axes: Axes = .empty; var intents: Intents = .empty; - inline for (std.meta.fields(T)) |field| { - const value = @field(axes_, field.name); - axes.appendAssumeCapacity(Shape.toTag(field)); + inline for (comptime std.meta.fieldNames(T)) |field_name| { + const value = @field(axes_, field_name); + axes.appendAssumeCapacity(Shape.toTag(field_name)); intents.appendAssumeCapacity(intentFromValue(value)); } @@ -1922,15 +1922,15 @@ pub const Strategy = struct { pub fn parseBindings(bindings: anytype) Strategy { const err_msg = "Strategy.parseBindings excepts fields to be PhysicalAxisTag or tuple of PhysicalAxisTag, got {}"; var res: Strategy = .{ .bindings = .empty, .folding = .empty }; - const fields = @typeInfo(@TypeOf(bindings)).@"struct".fields; - if (fields.len == 0) @compileError("Strategy.parseBindings requires at least one binding"); - inline for (fields) |field_info| { - switch (@typeInfo(field_info.type)) { - .enum_literal, .@"enum" => res.addBinding(field_info, @field(bindings, field_info.name)), + const info = @typeInfo(@TypeOf(bindings)).@"struct"; + if (info.field_names.len == 0) @compileError("Strategy.parseBindings requires at least one binding"); + inline for (info.field_names, info.field_types) |field_name, Field| { + switch (@typeInfo(Field)) { + .enum_literal, .@"enum" => res.addBinding(field_name, @field(bindings, field_name)), .@"struct" => |struct_info| { stdx.debug.assertComptime(struct_info.is_tuple, err_msg, .{@TypeOf(bindings)}); - inline for (@field(bindings, field_info.name)) |axis_tag| { - res.addBinding(field_info, axis_tag); + inline for (@field(bindings, field_name)) |axis_tag| { + res.addBinding(field_name, axis_tag); } }, else => stdx.debug.compileError(err_msg, .{@TypeOf(bindings)}), @@ -2042,13 +2042,13 @@ pub const Placement = struct { // Note this shape is mainly use to indicate the dims of the sharded buffer. // We may want to only store that. shape: Shape, - global_shape: if (builtin.mode == .Debug) Shape else void, + global_shape: if (builtin.mode == .debug) Shape else void, axis_plans: stdx.BoundedArray(AxisSplit, Shape.MAX_RANK), pub fn init(shape: Shape) error{IncompatibleSharding}!Placement { var pl: Placement = .{ .shape = shape, // modified below - .global_shape = if (builtin.mode == .Debug) shape else {}, + .global_shape = if (builtin.mode == .debug) shape else {}, .axis_plans = .empty, // set below }; // mesh == null => replicated => no axis plan @@ -2088,7 +2088,7 @@ pub const Placement = struct { } pub fn shardPtr(pl: *const Placement, device: Device.Coords, slice: Slice) [*]const u8 { - if (builtin.mode == .Debug) { + if (builtin.mode == .debug) { // there is a bug in caller code that used a placement for a different shape std.debug.assert(pl.global_shape.eql(slice.shape)); } @@ -2276,10 +2276,10 @@ const ShardingTest = struct { /// Creates a 1D-3D PhysicalMesh from simple dimensions. pub fn physical(self: ShardingTest, dims: anytype, geometry: AxisGeometry) !PhysicalMesh { const info = @typeInfo(@TypeOf(dims)).@"struct"; - const N = info.fields.len; + const N = info.field_names.len; var sizes: [N]usize = undefined; - inline for (info.fields, sizes[0..]) |field, *s| { - s.* = @intCast(@field(dims, field.name)); + inline for (info.field_names, sizes[0..]) |field_name, *s| { + s.* = @intCast(@field(dims, field_name)); } const tags: [3]PhysicalAxisTag = .{ .link_x, .link_y, .link_z }; diff --git a/zml/attention.zig b/zml/attention.zig index f5399c5e..cb2b1966 100644 --- a/zml/attention.zig +++ b/zml/attention.zig @@ -192,8 +192,8 @@ test "FlashAttention metadata initializes with explicit sharding outside compila inline for (.{ Backend.cuda_fa2, Backend.cuda_fa3 }) |backend| { const metadata: Metadata = .init(.fromBackend(backend, 16, 8), &mesh); const cuda_metadata = @field(metadata, @tagName(backend)); - inline for (std.meta.fields(@TypeOf(cuda_metadata))) |field| { - const shape = @field(cuda_metadata, field.name).shape(); + inline for (comptime std.meta.fieldNames(@TypeOf(cuda_metadata))) |field_name| { + const shape = @field(cuda_metadata, field_name).shape(); try std.testing.expectEqual(&mesh, shape._sharding.mesh); if (shape.hasTag(.h)) |axis| { try std.testing.expectEqual(zml.Sharding.PartitionSpec.sharded(0), shape.partition(axis)); diff --git a/zml/attention/cute_kernels/flashmla_sm100.zig b/zml/attention/cute_kernels/flashmla_sm100.zig index ed03b394..529594c0 100644 --- a/zml/attention/cute_kernels/flashmla_sm100.zig +++ b/zml/attention/cute_kernels/flashmla_sm100.zig @@ -120,7 +120,7 @@ const Bar = enum(i32) { const idx_consumers = 11; fn bar(b: *B, base: V, comptime which: Bar, buf: anytype) V { - return base.add(smem.bars + 8 * @intFromEnum(which)).add(b.lift(buf).mul(8)); + return base.add(smem.bars + 8 * @backingInt(which)).add(b.lift(buf).mul(8)); } // TMEM columns: O^T of token parity ob in 64 * ob (4 chunks of 128 dims x 16 @@ -252,8 +252,8 @@ fn parity(j: V) V { return j.shrLogical(1).bitAnd(1); } -fn tupleOf(comptime n: usize, arr: [n]V) std.meta.Tuple(&([_]type{V} ** n)) { - var t: std.meta.Tuple(&([_]type{V} ** n)) = undefined; +fn tupleOf(comptime n: usize, arr: [n]V) @Tuple(&@as([n]type, @splat(V))) { + var t: @Tuple(&@as([n]type, @splat(V))) = undefined; inline for (0..n) |i| t[i] = arr[i]; return t; } @@ -944,7 +944,10 @@ fn softmaxRole(c0: Ctx) void { // then skips the block max entirely (m stays, alpha = 1). const f32t = cute.DType.f32.toMlir(b.ctx); const i32t = cute.DType.i32.toMlir(b.ctx); - var upd = b.openIfElse(b.barrierReduceOr(1, 128, exceed.ne(0)), .{f32t} ** (2 * H) ++ .{i32t}); + var update_types: @Tuple(&@as([2 * H + 1]type, @splat(@TypeOf(f32t)))) = undefined; + inline for (0..2 * H) |h| update_types[h] = f32t; + update_types[2 * H] = i32t; + var upd = b.openIfElse(b.barrierReduceOr(1, 128, exceed.ne(0)), update_types); { // Block max per head: warp redux, then across the 4 warps through shared memory. const red = c.s.add(b.select(buf.eq(0), b.cst(.i32, smem.red[0]), b.cst(.i32, smem.red[1]))); @@ -1324,8 +1327,8 @@ fn instrDescBf16(comptime m: u32, comptime n: u32, comptime a_major: Major, comp return (1 << 4) | // c_format = F32 (1 << 7) | // a_format = BF16 (1 << 10) | // b_format = BF16 - (@as(u32, @intFromEnum(a_major)) << 15) | - (@as(u32, @intFromEnum(b_major)) << 16) | + (@as(u32, @backingInt(a_major)) << 15) | + (@as(u32, @backingInt(b_major)) << 16) | ((n >> 3) << 17) | ((m >> 4) << 24); } diff --git a/zml/dtype.zig b/zml/dtype.zig index c56fdd5b..f8d24089 100644 --- a/zml/dtype.zig +++ b/zml/dtype.zig @@ -1,5 +1,7 @@ const std = @import("std"); +const stdx = @import("stdx"); + const floats = @import("floats.zig"); const C64 = std.math.Complex(f32); @@ -139,8 +141,9 @@ pub const DataType = enum(u8) { pub fn toPackedZigType(comptime dtype: DataType) type { return switch (dtype) { .bool => bool, - .i2, .i4, .u2, .u4 => |dt| @Vector(8 / dt.bitSizeOf(), @FieldType(Value, @tagName(dtype))), + .i2, .i4, .u2, .u4 => |dt| stdx.Packed(@FieldType(Value, @tagName(dt))), .f4e2m1 => floats.Float4E2M1.Packed, + inline .c64, .c128 => |dt| @FieldType(Value, @tagName(dt)), else => { const T = @FieldType(Value, @tagName(dtype)); if (@bitSizeOf(T) < 8) @compileLog("Forgot type !", T); @@ -165,6 +168,8 @@ pub const DataType = enum(u8) { pub fn bitSizeOf(self: DataType) u16 { return switch (self) { .bool => 8, // In PJRT/stablehlo bool always occupy a full byte + .c64 => 64, + .c128 => 128, inline else => |tag| @bitSizeOf(tag.toZigType()), }; } diff --git a/zml/exe.zig b/zml/exe.zig index 1db04837..8b60dcb9 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -396,7 +396,7 @@ pub fn FnExe(comptime function_: anytype) type { .@"fn" => |info| info, else => @compileError("FnExe expects a function, got " ++ @typeName(@TypeOf(function_))), }; - if (function_info.is_var_args or function_info.params.len != 1) { + if (function_info.attrs.varargs or function_info.param_types.len != 1) { @compileError("FnExe function must accept exactly one input struct"); } @@ -446,7 +446,7 @@ pub fn FnExe(comptime function_: anytype) type { const RunnerSelf = @This(); pub const BakedInput = structFieldRange(Input, 0, count); - pub const NonBakedInput = structFieldRange(Input, count, @typeInfo(Input).@"struct".fields.len); + pub const NonBakedInput = structFieldRange(Input, count, @typeInfo(Input).@"struct".field_names.len); pub fn init(exe: *const Self, allocator: std.mem.Allocator, baked: BakedInput) !RunnerSelf { var arguments = try exe.raw.args(allocator); @@ -488,42 +488,30 @@ fn countBackedFields(comptime Input: type, comptime baked_fields: anytype) usize @compileError("FnExe baked fields must be a tuple of enum literals"); } - const input_fields = @typeInfo(Input).@"struct".fields; - if (baked_info.fields.len > input_fields.len) { + const input_info = @typeInfo(Input).@"struct"; + if (baked_info.field_names.len > input_info.field_names.len) { @compileError("FnExe baked fields must be a prefix of its bufferized input fields"); } - inline for (baked_info.fields, 0..) |tuple_field, i| { - const baked_field = @field(baked_fields, tuple_field.name); + inline for (baked_info.field_names, 0..) |tuple_field_name, i| { + const baked_field = @field(baked_fields, tuple_field_name); if (@TypeOf(baked_field) != @EnumLiteral()) { @compileError("FnExe baked fields must be enum literals"); } - if (!std.mem.eql(u8, @tagName(baked_field), input_fields[i].name)) { + if (!std.mem.eql(u8, @tagName(baked_field), input_info.field_names[i])) { @compileError("FnExe baked fields must be an ordered prefix of its bufferized input fields"); } } - return baked_info.fields.len; + return baked_info.field_names.len; } fn structFieldRange(comptime Struct: type, comptime start: usize, comptime end: usize) type { - const fields = @typeInfo(Struct).@"struct".fields[start..end]; - var field_names: [fields.len][]const u8 = undefined; - var field_types: [fields.len]type = undefined; - var field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; - for (&field_names, &field_types, &field_attrs, fields) |*name, *T, *attrs, field| { - name.* = field.name; - T.* = field.type; - attrs.* = .{ - .@"comptime" = field.is_comptime, - .@"align" = field.alignment, - .default_value_ptr = field.default_value_ptr, - }; - } - return @Struct(.auto, null, &field_names, &field_types, &field_attrs); + const info = @typeInfo(Struct).@"struct"; + return @Struct(.auto, null, info.field_names[start..end], info.field_types[start..end], info.field_attrs[start..end]); } fn typedFunctionInput(comptime function_: anytype) type { - const Input = @typeInfo(@TypeOf(function_)).@"fn".params[0].type orelse + const Input = @typeInfo(@TypeOf(function_)).@"fn".param_types[0] orelse @compileError("FnExe function must have a concrete input type"); validateFnExeStruct("input", Input); return Input; @@ -547,16 +535,13 @@ fn validateFnExeStruct(comptime role: []const u8, comptime T: type) void { fn typedOutputDestinations(comptime BufferizedOutput: type) type { const output_info = @typeInfo(BufferizedOutput).@"struct"; - const fields = output_info.fields; - var field_names: [fields.len][]const u8 = undefined; - var field_types: [fields.len]type = undefined; - var field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; - for (&field_names, &field_types, &field_attrs, fields) |*name, *T, *attrs, field| { - name.* = field.name; - T.* = *field.type; - attrs.* = .{ .@"align" = @alignOf(*field.type) }; + var field_types: [output_info.field_names.len]type = undefined; + var field_attrs: [output_info.field_names.len]std.builtin.Type.Struct.FieldAttributes = undefined; + for (&field_types, &field_attrs, output_info.field_types) |*T, *attrs, Field| { + T.* = *Field; + attrs.* = .{ .@"align" = @alignOf(*Field) }; } - return @Struct(.auto, null, &field_names, &field_types, &field_attrs); + return @Struct(.auto, null, output_info.field_names, &field_types, &field_attrs); } test "FnExe derives baked and runtime calls from a function" { diff --git a/zml/floats.zig b/zml/floats.zig index 7bf08824..689c22a8 100644 --- a/zml/floats.zig +++ b/zml/floats.zig @@ -8,8 +8,8 @@ fn FloatConversionHelpers(Float: type) type { @compileError(err_msg); } comptime { - for (info.@"struct".fields, &.{ "mantissa", "exponent", "sign" }) |field, expected_name| { - if (!std.mem.eql(u8, field.name, expected_name)) + for (info.@"struct".field_names, &.{ "mantissa", "exponent", "sign" }) |field_name, expected_name| { + if (!std.mem.eql(u8, field_name, expected_name)) @compileError(err_msg); } } @@ -20,7 +20,7 @@ fn FloatConversionHelpers(Float: type) type { const mantissa_bits: u8 = @typeInfo(@FieldType(Float, "mantissa")).int.bits; const exponent_bits: u8 = @typeInfo(@FieldType(Float, "exponent")).int.bits; const f32_mantissa_bits: u8 = @typeInfo(@FieldType(Float32, "mantissa")).int.bits; - const exp_bias: i16 = std.math.maxInt(std.meta.Int(.unsigned, exponent_bits - 1)); + const exp_bias: i16 = std.math.maxInt(@Int(.unsigned, exponent_bits - 1)); const exp_off: u8 = FloatConversionHelpers(Float32).exp_bias - exp_bias; /// Lossy conversion from f32, similar to @floatCast @@ -106,8 +106,8 @@ fn FloatHelpers(Float: type) type { @compileError(err_msg); } comptime { - for (info.@"struct".fields, &.{ "mantissa", "exponent", "sign" }) |field, expected_name| { - if (!std.mem.eql(u8, field.name, expected_name)) + for (info.@"struct".field_names, &.{ "mantissa", "exponent", "sign" }) |field_name, expected_name| { + if (!std.mem.eql(u8, field_name, expected_name)) @compileError(err_msg); } } @@ -517,7 +517,7 @@ pub const Float4E2M1 = packed struct(u4) { pub fn formatNumber(xy: Packed, w: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { try xy.x.formatNumber(w, n); - w.writeByte(','); + try w.writeByte(','); try xy.y.formatNumber(w, n); } }; @@ -544,7 +544,7 @@ pub fn isInf(x: anytype) bool { } if (!@hasDecl(Float, "inf")) return false; - const FBits = std.meta.Int(.unsigned, @bitSizeOf(Float)); + const FBits = @Int(.unsigned, @bitSizeOf(Float)); const remove_sign = ~@as(FBits, 0) >> 1; return @as(FBits, @bitCast(x)) & remove_sign == @as(FBits, @bitCast(Float.inf)); } diff --git a/zml/kernel.zig b/zml/kernel.zig index 2d8945e4..b934c7db 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -50,10 +50,10 @@ fn resolveOutputOperandAliases( var out: stdx.BoundedArray(dialects.stablehlo.CustomCallOpts.OutputOperandAlias, dialects.stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".fields, 0..) |field, output_index| { - if (@field(a, field.name)) |operand| out.appendAssumeCapacity(.{ + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@intFromEnum(operand))) + operand_offset, + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, }); } } @@ -832,7 +832,7 @@ pub const fly = struct { /// `zeroed_args` index of an output. pub fn zeroedOutput(comptime f: std.meta.FieldEnum(Outputs)) i32 { - return @intCast(spec.inputs.len + @intFromEnum(f)); + return @intCast(spec.inputs.len + @backingInt(f)); } /// `shapes` is inputs, then outputs. diff --git a/zml/mem.zig b/zml/mem.zig index f471a83a..87a824ef 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -341,8 +341,8 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu const type_info = @typeInfo(ModelBufferized); switch (type_info) { .@"struct" => |struct_type_info| { - inline for (struct_type_info.fields) |field| { - try bufferizeInner(allocator, @field(model, field.name), &@field(bufferized_, field.name)); + inline for (struct_type_info.field_names) |field_name| { + try bufferizeInner(allocator, @field(model, field_name), &@field(bufferized_, field_name)); } }, .@"union" => { @@ -364,7 +364,7 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu .pointer => |p| { switch (p.size) { .slice => { - bufferized_.* = try allocator.alignedAlloc(p.child, .fromByteUnits(p.alignment orelse @alignOf(p.child)), model.len); + bufferized_.* = try allocator.alignedAlloc(p.child, .fromByteUnits(p.attrs.@"align" orelse @alignOf(p.child)), model.len); for (model, bufferized_.*) |src, *dst| { try bufferizeInner(allocator, src, dst); } diff --git a/zml/meta.zig b/zml/meta.zig index 08738e11..c9a43f25 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -20,27 +20,30 @@ pub fn MapType(From: type, To: type) type { return switch (@typeInfo(T)) { .@"struct" => |struct_infos| { - const fields = struct_infos.fields; + const field_names = struct_infos.field_names; var same: bool = true; - var struct_field_names: [fields.len][]const u8 = undefined; - var struct_field_types: [fields.len]type = undefined; - var struct_field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; - for (struct_field_names[0..], struct_field_types[0..], struct_field_attrs[0..], fields) |*field_name, *field_type, *field_attr, field| { - if (!field.is_comptime) { - const R = map(field.type); - if (R == field.type) { - field_name.* = field.name; - field_type.* = field.type; - field_attr.* = .{ - .@"comptime" = field.is_comptime, - .@"align" = field.alignment, - .default_value_ptr = field.default_value_ptr, - }; + var struct_field_names: [field_names.len][]const u8 = undefined; + var struct_field_types: [field_names.len]type = undefined; + var struct_field_attrs: [field_names.len]std.builtin.Type.Struct.FieldAttributes = undefined; + for ( + struct_field_names[0..], + struct_field_types[0..], + struct_field_attrs[0..], + struct_infos.field_names, + struct_infos.field_types, + struct_infos.field_attrs, + ) |*field_name, *field_type, *field_attr, source_name, SourceField, source_attrs| { + if (!source_attrs.@"comptime") { + const R = map(SourceField); + if (R == SourceField) { + field_name.* = source_name; + field_type.* = SourceField; + field_attr.* = source_attrs; } else { - field_name.* = field.name; + field_name.* = source_name; field_type.* = R; field_attr.* = .{ - .@"comptime" = field.is_comptime, + .@"comptime" = source_attrs.@"comptime", .@"align" = @alignOf(R), }; same = false; @@ -52,34 +55,35 @@ pub fn MapType(From: type, To: type) type { } } } else { - field_name.* = field.name; - field_type.* = field.type; - field_attr.* = .{ - .@"comptime" = field.is_comptime, - .@"align" = field.alignment, - .default_value_ptr = field.default_value_ptr, - }; + field_name.* = source_name; + field_type.* = SourceField; + field_attr.* = source_attrs; } } if (same) return T; return if (struct_infos.is_tuple) @Tuple(&struct_field_types) else @Struct(.auto, null, &struct_field_names, &struct_field_types, &struct_field_attrs); }, .@"union" => |union_info| { - const fields = union_info.fields; + const field_names = union_info.field_names; var same: bool = true; - var union_field_names: [fields.len][]const u8 = undefined; - var union_field_types: [fields.len]type = undefined; - var union_field_attrs: [fields.len]std.builtin.Type.UnionField.Attributes = undefined; - for (union_field_names[0..], union_field_types[0..], union_field_attrs[0..], fields) |*field_name, *field_type, *field_attr, field| { - const R = map(field.type); - if (R == field.type) { - field_name.* = field.name; - field_type.* = field.type; - field_attr.* = .{ - .@"align" = field.alignment, - }; + var union_field_names: [field_names.len][]const u8 = undefined; + var union_field_types: [field_names.len]type = undefined; + var union_field_attrs: [field_names.len]std.builtin.Type.Union.FieldAttributes = undefined; + for ( + union_field_names[0..], + union_field_types[0..], + union_field_attrs[0..], + union_info.field_names, + union_info.field_types, + union_info.field_attrs, + ) |*field_name, *field_type, *field_attr, source_name, SourceField, source_attrs| { + const R = map(SourceField); + if (R == SourceField) { + field_name.* = source_name; + field_type.* = SourceField; + field_attr.* = source_attrs; } else { - field_name.* = field.name; + field_name.* = source_name; field_type.* = R; field_attr.* = .{ .@"align" = @alignOf(R), @@ -92,11 +96,11 @@ pub fn MapType(From: type, To: type) type { }, .array => |arr_info| [arr_info.len]map(arr_info.child), .pointer => |ptr_info| switch (ptr_info.size) { - .slice => if (ptr_info.is_const) + .slice => if (ptr_info.attrs.@"const") []const map(ptr_info.child) else []map(ptr_info.child), - .one => if (ptr_info.is_const) + .one => if (ptr_info.attrs.@"const") *const map(ptr_info.child) else *map(ptr_info.child), @@ -179,26 +183,26 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam if (@sizeOf(ToStruct) == 0) return; switch (type_info_to) { - .@"struct" => |info| inline for (info.fields) |field| { - if (field.is_comptime or @sizeOf(field.type) == 0) continue; - const field_type_info = @typeInfo(field.type); + .@"struct" => |info| inline for (info.field_names, info.field_types, info.field_attrs) |field_name, Field, field_attrs| { + if (field_attrs.@"comptime" or @sizeOf(Field) == 0) continue; + const field_type_info = @typeInfo(Field); // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. switch (field_type_info) { // .pointer => try convertType(From, To, allocator, @field(from, field.name), @field(to, field.name), Ctx, ctx, cb), - .array, .optional, .@"union", .@"struct", .pointer => if (@hasField(FromStruct, field.name)) { + .array, .optional, .@"union", .@"struct", .pointer => if (@hasField(FromStruct, field_name)) { try mapAlloc( cb, allocator, ctx, - @field(from, field.name), - &@field(to, field.name), + @field(from, field_name), + &@field(to, field_name), ); - } else if (field.default_value_ptr) |_| { - @field(to, field.name) = null; + } else if (field_attrs.default_value_ptr) |_| { + @field(to, field_name) = null; } else { - stdx.debug.compileError("Mapping {} -> {} inside {} failed. Missing field {s} in {}", .{ From, To, FromStruct, field.name, ToStruct }); + stdx.debug.compileError("Mapping {} -> {} inside {} failed. Missing field {s} in {}", .{ From, To, FromStruct, field_name, ToStruct }); }, - else => @field(to, field.name) = @field(from, field.name), + else => @field(to, field_name) = @field(from, field_name), } }, .@"union" => { @@ -332,25 +336,21 @@ pub fn MapRestrict(From: type, To: type) type { .@"struct" => |struct_infos| { // We know that at least one of the struct field contains a From. // We map each field individually. Fields without From and comptime fields are removed. - const fields = struct_infos.fields; + const field_names = struct_infos.field_names; var num_fields: usize = 0; - var struct_field_names: [fields.len][]const u8 = undefined; - var struct_field_types: [fields.len]type = undefined; - var struct_field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; - for (fields) |field| { - if (!field.is_comptime and Contains(field.type, From)) { - const R = map(field.type); - if (R == field.type) { - struct_field_names[num_fields] = field.name; - struct_field_types[num_fields] = field.type; - struct_field_attrs[num_fields] = .{ - .@"comptime" = field.is_comptime, - .@"align" = field.alignment, - .default_value_ptr = field.default_value_ptr, - }; + var struct_field_names: [field_names.len][]const u8 = undefined; + var struct_field_types: [field_names.len]type = undefined; + var struct_field_attrs: [field_names.len]std.builtin.Type.Struct.FieldAttributes = undefined; + for (struct_infos.field_names, struct_infos.field_types, struct_infos.field_attrs) |field_name, Field, field_attrs| { + if (!field_attrs.@"comptime" and Contains(Field, From)) { + const R = map(Field); + if (R == Field) { + struct_field_names[num_fields] = field_name; + struct_field_types[num_fields] = Field; + struct_field_attrs[num_fields] = field_attrs; } else { - const name = if (struct_infos.is_tuple) struct_infos.fields[num_fields].name else field.name; + const name = if (struct_infos.is_tuple) struct_infos.field_names[num_fields] else field_name; struct_field_names[num_fields] = name; struct_field_types[num_fields] = R; struct_field_attrs[num_fields] = .{ @@ -377,13 +377,13 @@ pub fn MapRestrict(From: type, To: type) type { .@"union" => |union_info| { // We know that at least one of the union field contains a From. // We map each field individually. Fields without From, are replaced by "void". - const fields = union_info.fields; - var union_field_names: [fields.len][]const u8 = undefined; - var union_field_types: [fields.len]type = undefined; - var union_field_attrs: [fields.len]std.builtin.Type.UnionField.Attributes = undefined; - for (0.., fields) |i, field| { - const FT = map(field.type); - union_field_names[i] = field.name; + const field_names = union_info.field_names; + var union_field_names: [field_names.len][]const u8 = undefined; + var union_field_types: [field_names.len]type = undefined; + var union_field_attrs: [field_names.len]std.builtin.Type.Union.FieldAttributes = undefined; + for (0.., union_info.field_names, union_info.field_types) |i, field_name, Field| { + const FT = map(Field); + union_field_names[i] = field_name; union_field_types[i] = FT; union_field_attrs[i] = .{ .@"align" = @alignOf(FT), @@ -393,19 +393,19 @@ pub fn MapRestrict(From: type, To: type) type { }, .array => |arr_info| [arr_info.len]map(arr_info.child), .pointer => |ptr_info| switch (ptr_info.size) { - .slice => if (ptr_info.is_const) + .slice => if (ptr_info.attrs.@"const") []const map(ptr_info.child) else []map(ptr_info.child), - .one => if (ptr_info.is_const) + .one => if (ptr_info.attrs.@"const") *const map(ptr_info.child) else *map(ptr_info.child), - .many => if (ptr_info.is_const) + .many => if (ptr_info.attrs.@"const") [*]const map(ptr_info.child) else [*]map(ptr_info.child), - .c => if (ptr_info.is_const) + .c => if (ptr_info.attrs.@"const") [*c]map(ptr_info.child) else [*c]map(ptr_info.child), @@ -499,7 +499,7 @@ pub fn visit(comptime callback: anytype, ctx: FnParam(callback, 0), v: anytype) .callback => callback(ctx, v), .recurse => { const TargetType, const mutating_cb = switch (@typeInfo(FnParam(callback, 1))) { - .pointer => |info| .{ info.child, !info.is_const }, + .pointer => |info| .{ info.child, !info.attrs.@"const" }, else => stdx.debug.compileError("zml.meta.visit({any}) is expecting a callback with a pointer as second argument but found {any}", .{ @TypeOf(callback), FnParam(callback, 1) }), }; @@ -511,12 +511,12 @@ pub fn visit(comptime callback: anytype, ctx: FnParam(callback, 0), v: anytype) switch (ptr_info_v.size) { .one => switch (@typeInfo(ChildTypeV)) { - .@"struct" => |s| inline for (s.fields) |field| { - if (field.is_comptime or comptime !Contains(field.type, TargetType)) continue; - if (@typeInfo(field.type) == .pointer) { - if (can_error) try visit(callback, ctx, @field(v, field.name)) else visit(callback, ctx, @field(v, field.name)); + .@"struct" => |s| inline for (s.field_names, s.field_types, s.field_attrs) |field_name, Field, field_attrs| { + if (field_attrs.@"comptime" or comptime !Contains(Field, TargetType)) continue; + if (@typeInfo(Field) == .pointer) { + if (can_error) try visit(callback, ctx, @field(v, field_name)) else visit(callback, ctx, @field(v, field_name)); } else { - if (can_error) try visit(callback, ctx, &@field(v, field.name)) else visit(callback, ctx, &@field(v, field.name)); + if (can_error) try visit(callback, ctx, &@field(v, field_name)) else visit(callback, ctx, &@field(v, field_name)); } }, .array => for (v[0..]) |*elem| { @@ -654,11 +654,11 @@ pub fn visitFlatStruct(comptime callback: anytype, ctx: FnParam(callback, 0), v: return switch (ptr_info_v.size) { .one => switch (@typeInfo(ChildTypeV)) { - .@"struct" => |s| inline for (s.fields) |field| { - if (field.is_comptime or comptime !Contains(field.type, TargetType)) continue; - stdx.debug.assertComptime(@typeInfo(field.type) != .pointer, err_msg, err_args); + .@"struct" => |s| inline for (s.field_names, s.field_types, s.field_attrs) |field_name, Field, field_attrs| { + if (field_attrs.@"comptime" or comptime !Contains(Field, TargetType)) continue; + stdx.debug.assertComptime(@typeInfo(Field) != .pointer, err_msg, err_args); - if (can_error) try visitFlatStruct(callback, ctx, &@field(v, field.name)) else visitFlatStruct(callback, ctx, &@field(v, field.name)); + if (can_error) try visitFlatStruct(callback, ctx, &@field(v, field_name)) else visitFlatStruct(callback, ctx, &@field(v, field_name)); }, .array => for (v[0..]) |*elem| { if (can_error) try visitFlatStruct(callback, ctx, elem) else visitFlatStruct(callback, ctx, elem); @@ -786,17 +786,17 @@ pub fn zip(comptime func: anytype, allocator: std.mem.Allocator, values: anytype .pointer => stdx.debug.compileError("zip only accept by value arguments. Received: {}", .{V}), .@"struct" => |struct_info| { var out: V = values[0]; - inline for (struct_info.fields) |f| { - if (f.is_comptime) continue; - if (@typeInfo(f.type) == .pointer) { + inline for (struct_info.field_names, struct_info.field_types, struct_info.field_attrs) |field_name, Field, field_attrs| { + if (field_attrs.@"comptime") continue; + if (@typeInfo(Field) == .pointer) { stdx.debug.compileError("zip doesn't follow pointers and don't accept struct containing them. Received: {}", .{V}); } - var fields = try allocator.alloc(f.type, values.len); + var fields = try allocator.alloc(Field, values.len); defer allocator.free(fields); for (values, 0..) |val, i| { - fields[i] = @field(val, f.name); + fields[i] = @field(val, field_name); } - @field(out, f.name) = try zip(func, allocator, fields, args); + @field(out, field_name) = try zip(func, allocator, fields, args); } return out; }, @@ -844,7 +844,7 @@ pub fn collectAlloc( allocator: std.mem.Allocator, obj: anytype, ) std.mem.Allocator.Error![]stdx.meta.FnReturn(func) { - stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".params.len <= 2, "zml.meta.collect expects a func with two arguments, got: {}", .{@TypeOf(func)}); + stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".param_types.len <= 2, "zml.meta.collect expects a func with two arguments, got: {}", .{@TypeOf(func)}); const CollectAllocCtx = struct { func_ctx: _CollectCtx(func), @@ -871,7 +871,7 @@ pub fn collectAlloc( /// finds all X in the given object, and write the result of func(X) into a given slice. /// Asserts that the number of X found is equal to the slice len. pub fn collectBuf(func: anytype, func_ctx: _CollectCtx(func), obj: anytype, out: []stdx.meta.FnReturn(func)) void { - stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".params.len <= 2, "zml.meta.collectBuf expects a func with one or two arguments, got: {}", .{@TypeOf(func)}); + stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".param_types.len <= 2, "zml.meta.collectBuf expects a func with one or two arguments, got: {}", .{@TypeOf(func)}); const LocalContext = struct { func_ctx: _CollectCtx(func), out: @TypeOf(out), @@ -916,14 +916,14 @@ pub fn collectPtrs( } fn _CollectCtx(func: anytype) type { - const params = @typeInfo(@TypeOf(func)).@"fn".params; + const params = @typeInfo(@TypeOf(func)).@"fn".param_types; if (params.len == 1) return void; - return params[0].type orelse @compileError("anytype not supported in collect"); + return params[0] orelse @compileError("anytype not supported in collect"); } fn _CollectArg(func: anytype) type { - const params = @typeInfo(@TypeOf(func)).@"fn".params; - return params[params.len - 1].type orelse @compileError("anytype not supported in collect"); + const params = @typeInfo(@TypeOf(func)).@"fn".param_types; + return params[params.len - 1] orelse @compileError("anytype not supported in collect"); } pub fn Contains(Haystack: type, T: type) bool { @@ -939,8 +939,8 @@ pub fn Contains(Haystack: type, T: type) bool { return switch (@typeInfo(Haystack)) { .@"struct" => |info| { if (@hasDecl(Haystack, "zml_no_meta_visit")) return false; - inline for (info.fields) |field| { - if (!field.is_comptime and Contains(field.type, T)) { + inline for (info.field_types, info.field_attrs) |Field, field_attrs| { + if (!field_attrs.@"comptime" and Contains(Field, T)) { return true; } } @@ -948,8 +948,8 @@ pub fn Contains(Haystack: type, T: type) bool { }, .@"union" => |info| { if (@hasDecl(Haystack, "zml_no_meta_visit")) return false; - inline for (info.fields) |field| { - if (Contains(field.type, T)) + inline for (info.field_types) |Field| { + if (Contains(Field, T)) return true; } return false; diff --git a/zml/moe/cute_kernels/persistent_mxfp4.zig b/zml/moe/cute_kernels/persistent_mxfp4.zig index 1f4b25cb..7208e6ce 100644 --- a/zml/moe/cute_kernels/persistent_mxfp4.zig +++ b/zml/moe/cute_kernels/persistent_mxfp4.zig @@ -1032,11 +1032,11 @@ fn runTmaProducer( } fn Tuple(comptime T: type, comptime len: usize) type { - return std.meta.Tuple(&([_]type{T} ** len)); + return @Tuple(&@as([len]type, @splat(T))); } fn arity(comptime T: type) usize { - return @typeInfo(T).@"struct".fields.len; + return @typeInfo(T).@"struct".field_names.len; } /// The MLIR types of a tuple of values, as `openWhile` expects them. diff --git a/zml/moe/cutlass_flashinfer.zig b/zml/moe/cutlass_flashinfer.zig index f90a7882..e2aa61a9 100644 --- a/zml/moe/cutlass_flashinfer.zig +++ b/zml/moe/cutlass_flashinfer.zig @@ -95,7 +95,7 @@ pub const Variant = enum { const max_num_devices = zml.platform.Platform.MAX_NUM_DEVICES; pub const Runners = struct { - runners: [std.meta.fields(Variant).len][max_num_devices]?DeviceRunner = + runners: [std.meta.fieldNames(Variant).len][max_num_devices]?DeviceRunner = @splat(@splat(null)), pub fn deinit(self: *Runners) void { @@ -112,7 +112,7 @@ pub const Runners = struct { if (device < 0 or device >= max_num_devices) return error.UnsupportedDevice; const index: usize = @intCast(device); - const variant_index: usize = @intFromEnum(variant); + const variant_index: usize = @backingInt(variant); if (self.runners[variant_index][index]) |runner| return runner; const api = try fi_cutlass_moe.apiForDevice(device); diff --git a/zml/moe/triton_kernels/mxfp4.zig b/zml/moe/triton_kernels/mxfp4.zig index 4da51a5a..74170966 100644 --- a/zml/moe/triton_kernels/mxfp4.zig +++ b/zml/moe/triton_kernels/mxfp4.zig @@ -64,8 +64,8 @@ pub fn validateInputs(c: Config, a: Inputs) !void { .w2 = zml.Shape.init(.{ c.experts, c.hidden, @divTrunc(c.intermediate, 2) }, .u8), .s2 = zml.Shape.init(.{ c.experts, c.hidden, a.s2.dim(2) }, .f8e8m0), }; - inline for (std.meta.fields(Inputs)) |field| { - if (!@field(a, field.name).shape().eql(@field(shapes, field.name))) return error.InvalidInputShape; + inline for (comptime std.meta.fieldNames(Inputs)) |field| { + if (!@field(a, field).shape().eql(@field(shapes, field))) return error.InvalidInputShape; } } diff --git a/zml/nn.zig b/zml/nn.zig index 1403788f..7519a435 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -2259,8 +2259,13 @@ pub fn collectDims( return context.res; } -fn shapeToDims(shape: anytype) [@divExact(@sizeOf(@TypeOf(shape)), @sizeOf(i64))]i64 { - return @bitCast(shape); +fn shapeToDims(shape: anytype) [std.meta.fieldNames(@TypeOf(shape)).len]i64 { + const field_names = comptime std.meta.fieldNames(@TypeOf(shape)); + var result: [field_names.len]i64 = undefined; + inline for (field_names, 0..) |field_name, i| { + result[i] = @field(shape, field_name); + } + return result; } test collectDims { @@ -2315,7 +2320,7 @@ fn ShapeStruct(comptime dims: anytype) type { @setEvalBranchQuota(rank + 5); var struct_field_names: [rank][]const u8 = undefined; var struct_field_types: [rank]type = undefined; - var struct_field_attrs: [rank]std.builtin.Type.StructField.Attributes = undefined; + var struct_field_attrs: [rank]std.builtin.Type.Struct.FieldAttributes = undefined; const default: i64 = NOT_SET; for (&struct_field_names, &struct_field_types, &struct_field_attrs, dims) |*field_name, *field_type, *field_attr, axis| { field_name.* = @tagName(axis); diff --git a/zml/ops.zig b/zml/ops.zig index 186805ea..ba7172a4 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -34,7 +34,7 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@T if (!struct_info.is_tuple) { @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"); } - break :b struct_info.fields.len; + break :b struct_info.field_names.len; }, .array => |array_info| b: { if (array_info.child != Tensor) { @@ -72,7 +72,7 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@T const reducer_block = b: { const ArgsTypes: [input_tensors.len]type = @splat(ReduceArgs); - var args: std.meta.Tuple(&ArgsTypes) = undefined; + var args: @Tuple(&ArgsTypes) = undefined; var block_types: [2 * input_tensors.len]*const mlir.Type = undefined; inline for (0..input_tensors.len) |i| { @@ -158,8 +158,8 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@T } var out: @TypeOf(inputs) = undefined; - inline for (struct_info.fields, 0..) |field, i| { - @field(out, field.name) = Tensor._result(input_tensors[i].shape(), op.result(i)); + inline for (struct_info.field_names, 0..) |field_name, i| { + @field(out, field_name) = Tensor._result(input_tensors[i].shape(), op.result(i)); } break :b out; }, @@ -216,7 +216,7 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func // Note: inputs and inits are infered to be tuple of ReduceArgs const reduce_block, var result = b: { const ArgsTypes: [inits.len]type = @splat(ReduceArgs); - const ArgsType = std.meta.Tuple(&ArgsTypes); + const ArgsType = @Tuple(&ArgsTypes); var args: ArgsType = undefined; var block_types: [2 * inits.len]*const mlir.Type = undefined; @@ -315,7 +315,7 @@ pub const ReduceWindowOpts = struct { pub fn ReduceWindowFn(N: comptime_int) type { return @Fn( &@as([N]type, @splat(ReduceArgs)), - &@as([N]std.builtin.Type.Fn.Param.Attributes, @splat(.{})), + &@as([N]std.builtin.Type.Fn.ParamAttributes, @splat(.{})), [N]Tensor, .{}, ); @@ -405,7 +405,7 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytyp const sort_block = b: { const ArgsTypes: [inputs.len]type = @splat(SortArgs); - const ArgsType = std.meta.Tuple(&ArgsTypes); + const ArgsType = @Tuple(&ArgsTypes); var args: ArgsType = undefined; var block_types: [2 * inputs.len]*const mlir.Type = undefined; @@ -547,7 +547,7 @@ pub fn @"while"( const result: While.State = captured_context.body(initial_state); - if (builtin.mode == .Debug) { + if (builtin.mode == .debug) { const result_tensors = meta.collectPtrs(Tensor, allocator, &result) catch @panic("OOM"); stdx.debug.assert(flat_operands.len == result_tensors.len, "zml.ops.while({s}) expects body to return {d} Tensor values, got {d} values in ", .{ @typeName(While), flat_operands.len, result_tensors.len }); for (0.., flat_operands, result_tensors) |i, in, out| { @@ -1623,7 +1623,7 @@ test "cuda_tile grid" { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Mod.forward, .{a}, platform, .{}); defer exe.deinit(); - const input = [_]f32{0} ** n; + const input: [n]f32 = @splat(0); var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); defer a_buffer.deinit(); @@ -1729,7 +1729,7 @@ pub fn scatter( const update_block, var result = b: { const ArgsTypes: [inputs.len]type = @splat(ScatterArgs); - const ArgsType = std.meta.Tuple(&ArgsTypes); + const ArgsType = @Tuple(&ArgsTypes); var args: ArgsType = undefined; var block_types: [2 * inputs.len]*const mlir.Type = undefined; @@ -2063,7 +2063,7 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, var self_kind: stdx.BoundedArray(GatherAxisKind, constants.MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; for (self._shape.tags(), 0..self.rank()) |t, self_ax| { - const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, 1, @intCast(self_ax)); + const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, @intCast(self_ax), 1); if (indices_shape.hasTag(t)) |id_ax| { // tag is both in self and indices -> it's a batching dim // Note: tags are required for batching. @@ -2087,7 +2087,7 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, if (ax == idx_axes[0]) { // The first val_ax is special cause this is the place where we insert indices axes. for (0.., indices_shape.tags(), indices_shape.dims()) |id_axis_order, id_axis, id_inserted_dim| { - const is_batching_axis = std.mem.containsAtLeastScalar(i64, idx_batch_axes.constSlice(), 1, @intCast(id_axis_order)); + const is_batching_axis = std.mem.containsAtLeastScalar(i64, idx_batch_axes.constSlice(), @intCast(id_axis_order), 1); // Batching axis is already in self. if (is_batching_axis) continue; @@ -2278,10 +2278,10 @@ fn CustomCallResultTypeFromOutputSpec(comptime OutputSpecT: type) type { .@"struct" => |struct_info| b: { if (OutputSpecT == Shape) break :b Tensor; if (!struct_info.is_tuple) @compileError("Expected tuple output shape spec"); - break :b if (struct_info.fields.len == 1) + break :b if (struct_info.field_names.len == 1) Tensor else - [struct_info.fields.len]Tensor; + [struct_info.field_names.len]Tensor; }, .array => |array_info| b: { break :b if (array_info.len == 1) @@ -2322,20 +2322,20 @@ pub fn customCallOutputOperandAliases( ) ?[]const dialects.stablehlo.CustomCallOpts.OutputOperandAlias { const aliases_ = aliases orelse return null; - const output_fields = @typeInfo(O).@"struct".fields; + const output_field_names = @typeInfo(O).@"struct".field_names; comptime var num_aliases = 0; - inline for (output_fields) |field| { - if (@field(aliases_, field.name) != null) num_aliases += 1; + inline for (output_field_names) |field_name| { + if (@field(aliases_, field_name) != null) num_aliases += 1; } const output_operand_aliases = comptime blk: { var aliases_buffer: [num_aliases]dialects.stablehlo.CustomCallOpts.OutputOperandAlias = undefined; var i = 0; - for (output_fields, 0..) |field, output_index| { - if (@field(aliases_, field.name)) |operand| { + for (output_field_names, 0..) |field_name, output_index| { + if (@field(aliases_, field_name)) |operand| { aliases_buffer[i] = .{ .output_index = @intCast(output_index), - .operand_index = @intCast(@intFromEnum(operand)), + .operand_index = @intCast(@backingInt(operand)), }; i += 1; } @@ -2447,7 +2447,7 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, break :b &[1]Tensor{inputs}; } if (!struct_info.is_tuple) @compileError("Expected tuple"); - var inputs_: [struct_info.fields.len]Tensor = undefined; + var inputs_: [struct_info.field_names.len]Tensor = undefined; meta.collectBuf((struct { pub fn func(t: Tensor) Tensor { return t; @@ -2471,7 +2471,7 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, break :b &[1]Shape{outputs}; } if (!struct_info.is_tuple) @compileError("Expected tuple"); - var output_shapes: [struct_info.fields.len]Shape = undefined; + var output_shapes: [struct_info.field_names.len]Shape = undefined; meta.collectBuf((struct { pub fn func(t: Shape) Shape { return t; @@ -2504,14 +2504,14 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, .@"struct" => |struct_info| b: { if (@TypeOf(outputs) == Shape) break :b outputs_flat[0]; if (!struct_info.is_tuple) @compileError("Expected tuple"); - if (struct_info.fields.len == 1) break :b outputs_flat[0]; - var outputs_: [struct_info.fields.len]Tensor = undefined; + if (struct_info.field_names.len == 1) break :b outputs_flat[0]; + var outputs_: [struct_info.field_names.len]Tensor = undefined; @memcpy(&outputs_, outputs_flat); break :b outputs_; }, .array => |array_info| b: { if (array_info.len == 1) break :b outputs_flat[0]; - var outputs_: [array_info.fields.len]Tensor = undefined; + var outputs_: [array_info.len]Tensor = undefined; @memcpy(&outputs_, outputs_flat); break :b outputs_; }, @@ -2536,7 +2536,7 @@ pub fn manualComputation( break :b &[1]Shape{outputs}; } if (!struct_info.is_tuple) @compileError("Expected tuple output shapes"); - var output_shapes_flat: [struct_info.fields.len]Shape = undefined; + var output_shapes_flat: [struct_info.field_names.len]Shape = undefined; meta.collectBuf((struct { pub fn func(t: Shape) Shape { return t; @@ -2945,7 +2945,7 @@ fn manualComputationOutputShapesArg(comptime OutputShapesT: type, local_output_s if (pointer_info.size != .slice or pointer_info.child != Shape) { @compileError("manualComputation body output shapes must be void, Shape, []Shape, or []const Shape; got " ++ @typeName(OutputShapesT)); } - break :b if (pointer_info.is_const) + break :b if (pointer_info.attrs.@"const") local_output_shapes else @constCast(local_output_shapes); @@ -2999,7 +2999,7 @@ fn metadataFieldToMlirAttribute(mlir_ctx: *mlir.Context, comptime T: type, value return switch (type_info) { .comptime_int => .int(mlir_ctx, .u64, @as(u64, value)), .@"enum" => |enum_field| switch (@typeInfo(enum_field.tag_type)) { - .int => |int_tag| metadataIntegerFieldToMlirAttribute(mlir_ctx, int_tag, @intFromEnum(value)), + .int => |int_tag| metadataIntegerFieldToMlirAttribute(mlir_ctx, int_tag, @backingInt(value)), else => @compileError("Unsupported tag type for enum metadata: " ++ @typeName(enum_field.tag_type)), }, .int => |int_field| metadataIntegerFieldToMlirAttribute(mlir_ctx, int_field, value), @@ -3106,15 +3106,11 @@ pub fn ShapeToTensor(T: type) type { } pub fn CustomCallOutputOperandAliases(I: type, O: type) type { - const output_fields = @typeInfo(O).@"struct".fields; - var field_names: [output_fields.len][]const u8 = undefined; - inline for (output_fields, 0..) |field, i| { - field_names[i] = field.name; - } + const field_names = @typeInfo(O).@"struct".field_names; return @Struct( .auto, null, - &field_names, + field_names, &@splat(?std.meta.FieldEnum(I)), &@splat(.{ .default_value_ptr = &@as(?std.meta.FieldEnum(I), null) }), ); @@ -3131,9 +3127,9 @@ pub fn shardingAwareTypedCustomCall( const Output = @TypeOf(output); const Attributes = @TypeOf(attributes); - var output_shapes: [@typeInfo(Output).@"struct".fields.len]Shape = undefined; - inline for (@typeInfo(Output).@"struct".fields, 0..) |field, i| { - output_shapes[i] = @field(output, field.name); + var output_shapes: [@typeInfo(Output).@"struct".field_names.len]Shape = undefined; + inline for (@typeInfo(Output).@"struct".field_names, 0..) |field_name, i| { + output_shapes[i] = @field(output, field_name); } const Handler = struct { @@ -3152,8 +3148,8 @@ pub fn shardingAwareTypedCustomCall( // Convert the slice back to a struct var out: ShapeToTensor(Output) = undefined; - inline for (@typeInfo(Output).@"struct".fields, 0..) |field, i| { - @field(out, field.name) = output_tensors[i]; + inline for (@typeInfo(Output).@"struct".field_names, 0..) |field_name, i| { + @field(out, field_name) = output_tensors[i]; } return out; } @@ -3176,9 +3172,9 @@ pub fn typedCustomCall( const input_tensors: []const Tensor = switch (@typeInfo(Input)) { .@"struct" => |struct_info| b: { - var input_tensors: [struct_info.fields.len]Tensor = undefined; - inline for (struct_info.fields, 0..) |field, i| { - input_tensors[i] = @field(input, field.name); + var input_tensors: [struct_info.field_names.len]Tensor = undefined; + inline for (struct_info.field_names, 0..) |field_name, i| { + input_tensors[i] = @field(input, field_name); } break :b &input_tensors; }, @@ -3192,9 +3188,9 @@ pub fn typedCustomCall( const output_shapes: []const Shape = switch (@typeInfo(Output)) { .@"struct" => |struct_info| b: { - var output_shapes: [struct_info.fields.len]Shape = undefined; - inline for (struct_info.fields, 0..) |field, i| { - output_shapes[i] = @field(output, field.name); + var output_shapes: [struct_info.field_names.len]Shape = undefined; + inline for (struct_info.field_names, 0..) |field_name, i| { + output_shapes[i] = @field(output, field_name); } break :b &output_shapes; }, @@ -3208,13 +3204,13 @@ pub fn typedCustomCall( var metadata_attribute_list = switch (@typeInfo(Attributes)) { .void => std.ArrayList(mlir.NamedAttribute).initCapacity(allocator, 2) catch unreachable, - .@"struct" => |struct_info| std.ArrayList(mlir.NamedAttribute).initCapacity(allocator, struct_info.fields.len + 2) catch unreachable, + .@"struct" => |struct_info| std.ArrayList(mlir.NamedAttribute).initCapacity(allocator, struct_info.field_names.len + 2) catch unreachable, else => @compileError("Unsupported type: " ++ @typeName(Attributes)), }; if (@typeInfo(Attributes) == .@"struct") { - inline for (@typeInfo(Attributes).@"struct".fields) |field| { - if (metadataFieldToMlirAttribute(ctx.mlir_ctx, field.type, @field(attributes, field.name))) |attribute| { - metadata_attribute_list.appendAssumeCapacity(mlir.NamedAttribute.named(ctx.mlir_ctx, field.name, attribute)); + inline for (@typeInfo(Attributes).@"struct".field_names, @typeInfo(Attributes).@"struct".field_types) |field_name, Field| { + if (metadataFieldToMlirAttribute(ctx.mlir_ctx, Field, @field(attributes, field_name))) |attribute| { + metadata_attribute_list.appendAssumeCapacity(mlir.NamedAttribute.named(ctx.mlir_ctx, field_name, attribute)); } } } @@ -3270,8 +3266,8 @@ pub fn typedCustomCall( switch (@typeInfo(Output)) { .@"struct" => |struct_info| { var out: ShapeToTensor(Output) = undefined; - inline for (output_shapes, struct_info.fields, 0..) |output_shape, field, i| { - @field(out, field.name) = Tensor._result(output_shape, op.result(i)); + inline for (output_shapes, struct_info.field_names, 0..) |output_shape, field_name, i| { + @field(out, field_name) = Tensor._result(output_shape, op.result(i)); } return out; }, @@ -3294,26 +3290,26 @@ fn customCallArgsFromPjrtCallFrame(I: type, O: type, A: type, call_frame: *pjrt. A, } { var input: TensorToCustomCallBuffer(I) = undefined; - inline for (@typeInfo(I).@"struct".fields, 0..) |field, i| { + inline for (@typeInfo(I).@"struct".field_names, 0..) |field_name, i| { const buf = call_frame.args.buffers()[i]; - @field(input, field.name) = .fromPjrt(buf); + @field(input, field_name) = .fromPjrt(buf); } var output: ShapeToCustomCallBuffer(O) = undefined; - inline for (@typeInfo(O).@"struct".fields, 0..) |field, i| { + inline for (@typeInfo(O).@"struct".field_names, 0..) |field_name, i| { const shape_buf = call_frame.results.buffers()[i]; - @field(output, field.name) = .fromPjrt(shape_buf); + @field(output, field_name) = .fromPjrt(shape_buf); } const attributes: A = switch (@typeInfo(A)) { .void => {}, .@"struct" => blk: { var attrs: A = undefined; - inline for (@typeInfo(A).@"struct".fields) |field| { + inline for (@typeInfo(A).@"struct".field_names, @typeInfo(A).@"struct".field_types) |field_name, Field| { // TODO: Use getByIndex instead? - const attribute = call_frame.attrs.getByName(.scalar, field.name) orelse - @panic("Attribute not found: " ++ field.name); - @field(attrs, field.name) = attribute.get(field.type); + const attribute = call_frame.attrs.getByName(.scalar, field_name) orelse + @panic("Attribute not found: " ++ field_name); + @field(attrs, field_name) = attribute.get(Field); } break :blk attrs; }, diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index f492b120..8543b591 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -117,13 +117,13 @@ pub fn dtypeFromBufferType(pjrt_type: pjrt.BufferType) DataType { } test bufferTypeFromDtype { - inline for (@typeInfo(DataType).@"enum".fields) |field| { - const dt: DataType = @enumFromInt(field.value); + inline for (@typeInfo(DataType).@"enum".field_values) |field_value| { + const dt: DataType = @fromBackingInt(@intCast(field_value)); try std.testing.expectEqual(dt, dtypeFromBufferType(bufferTypeFromDtype(dt))); } - inline for (@typeInfo(pjrt.BufferType).@"enum".fields) |field| { - const dt: pjrt.BufferType = @enumFromInt(field.value); + inline for (@typeInfo(pjrt.BufferType).@"enum".field_values) |field_value| { + const dt: pjrt.BufferType = @fromBackingInt(@intCast(field_value)); if (dt == .invalid) continue; try std.testing.expectEqual(dt, bufferTypeFromDtype(dtypeFromBufferType(dt))); } diff --git a/zml/platform.zig b/zml/platform.zig index 7a9ee710..16ffe6c8 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -217,7 +217,7 @@ pub const Device = struct { } pub fn memory(self: *const Device, memory_kind: Memory.Kind) ?*const Memory { - return self.memory_by_kind.values[@intFromEnum(memory_kind)]; + return self.memory_by_kind.values[@backingInt(memory_kind)]; } }; @@ -239,7 +239,7 @@ fn sortDevicesById(target: Target, devices: []Device) void { std.mem.sort(Device, devices, Context{ .target = target }, Context.lessThan); - if (builtin.mode == .Debug) { + if (builtin.mode == .debug) { for (devices, 0..) |device, expected_id| { std.debug.assert(platformDeviceSortId(target, device) == expected_id); } diff --git a/zml/profiling/profiler.zig b/zml/profiling/profiler.zig index 3f3c8409..d46a0c4c 100644 --- a/zml/profiling/profiler.zig +++ b/zml/profiling/profiler.zig @@ -92,7 +92,7 @@ pub const ProfilerOptions = struct { c.tensorflow_ProfileOptions_set_repository_path(proto, upb.stringView(self.repository_path)); if (self.device_type) |value| { - c.tensorflow_ProfileOptions_set_device_type(proto, @intFromEnum(value)); + c.tensorflow_ProfileOptions_set_device_type(proto, @backingInt(value)); } if (self.include_dataset_ops) |value| { c.tensorflow_ProfileOptions_set_include_dataset_ops(proto, value); diff --git a/zml/profiling/tracer.zig b/zml/profiling/tracer.zig index e3157d10..5aed01df 100644 --- a/zml/profiling/tracer.zig +++ b/zml/profiling/tracer.zig @@ -3,6 +3,7 @@ const builtin = @import("builtin"); const c = @import("c"); const platforms = @import("platforms"); +const stdx = @import("stdx"); const zffi = @import("ffi"); pub const Span = struct { @@ -44,14 +45,14 @@ pub fn span(comptime name: []const u8, metadata: anytype) Span { return .start(name); }, .@"struct" => |info| { - if (info.fields.len == 0) { + if (info.field_names.len == 0) { return .start(name); } var buffer: [computeEncodedNameLen(name, @TypeOf(metadata))]u8 = undefined; var encoded: std.ArrayList(u8) = .initBuffer(&buffer); - appendEncodedName(&encoded, .failing, name, info, metadata) catch unreachable; + appendEncodedName(&encoded, .failing, name, metadata) catch unreachable; return .start(encoded.items); }, else => @compileError("Unsupported metadata."), @@ -62,34 +63,34 @@ pub fn span(comptime name: []const u8, metadata: anytype) Span { /// bounded by `span` at comptime. The caller owns the returned slice. pub fn formatSpanName(allocator: std.mem.Allocator, name: []const u8, metadata: anytype) ![]const u8 { switch (@typeInfo(@TypeOf(metadata))) { - .@"struct" => |info| { + .@"struct" => { var encoded: std.ArrayList(u8) = .empty; errdefer encoded.deinit(allocator); - try appendEncodedName(&encoded, allocator, name, info, metadata); + try appendEncodedName(&encoded, allocator, name, metadata); return try encoded.toOwnedSlice(allocator); }, else => @compileError("trace metadata must be a struct literal like .{ .step_num = 42 }"), } } -fn appendEncodedName(buffer: *std.ArrayList(u8), allocator: std.mem.Allocator, name: []const u8, info: std.builtin.Type.Struct, metadata: anytype) !void { +fn appendEncodedName(buffer: *std.ArrayList(u8), allocator: std.mem.Allocator, name: []const u8, metadata: anytype) !void { try buffer.appendSlice(allocator, name); var field_count: usize = 0; try buffer.append(allocator, '#'); - inline for (info.fields) |field| { - if (shouldEncodeField(field, @field(metadata, field.name))) { + inline for (comptime std.meta.fieldNames(@TypeOf(metadata))) |field_name| { + if (shouldEncodeField(field_name, @field(metadata, field_name))) { if (field_count != 0) { try buffer.append(allocator, ','); } field_count += 1; - const key = comptime metadataKey(field.name); + const key = comptime metadataKey(field_name); validateMetadataToken(key); try buffer.appendSlice(allocator, key); try buffer.append(allocator, '='); - try appendMetadataValue(buffer, allocator, @field(metadata, field.name), field.name); + try appendMetadataValue(buffer, allocator, @field(metadata, field_name), field_name); } } @@ -114,33 +115,38 @@ fn metadataKey(comptime field_name: []const u8) []const u8 { fn computeEncodedNameLen(comptime name: []const u8, comptime Metadata: type) usize { switch (@typeInfo(Metadata)) { .@"struct" => |info| { - if (info.fields.len == 0) return name.len; + if (info.field_names.len == 0) return name.len; // name#[attrs]# var max_len = name.len + 2; - inline for (info.fields) |field| { - const key = comptime metadataKey(field.name); + inline for ( + info.field_names, + info.field_types, + info.field_attrs, + ) |field_name, FieldType, field_attr| { + const key = comptime metadataKey(field_name); validateMetadataToken(key); + const default: ?FieldType = field_attr.defaultValue(FieldType); // [key]=[value], max_len += key.len + 2; - max_len += switch (@typeInfo(field.type)) { + max_len += switch (@typeInfo(FieldType)) { .bool => 1, .int => @max( - std.fmt.comptimePrint("{}", .{std.math.minInt(field.type)}).len, - std.fmt.comptimePrint("{}", .{std.math.maxInt(field.type)}).len, + std.fmt.comptimePrint("{}", .{std.math.minInt(FieldType)}).len, + std.fmt.comptimePrint("{}", .{std.math.maxInt(FieldType)}).len, ), .@"enum" => |ty| b: { var max_enum_len: usize = 0; - for (ty.fields) |enum_field| { - max_enum_len = @max(max_enum_len, enum_field.name.len); + for (ty.field_names) |enum_field_name| { + max_enum_len = @max(max_enum_len, enum_field_name.len); } break :b max_enum_len; }, - .comptime_int => std.fmt.comptimePrint("{}", .{defaultMetadataValue(field).?}).len, - .comptime_float => std.fmt.comptimePrint("{d}", .{defaultMetadataValue(field).?}).len, - .enum_literal => @tagName(defaultMetadataValue(field).?).len, - else => @compileError("trace metadata field '" ++ field.name ++ "' is not statically bounded; use formatspanName"), + .comptime_int => std.fmt.comptimePrint("{}", .{default.?}).len, + .comptime_float => std.fmt.comptimePrint("{d}", .{default.?}).len, + .enum_literal => @tagName(default.?).len, + else => @compileError("trace metadata field '" ++ field_name ++ "' is not statically bounded; use formatspanName"), }; } return max_len; @@ -149,17 +155,11 @@ fn computeEncodedNameLen(comptime name: []const u8, comptime Metadata: type) usi } } -fn defaultMetadataValue(comptime field: std.builtin.Type.StructField) ?field.type { - return if (field.default_value_ptr) |default_opaque| - @as(*const field.type, @ptrCast(@alignCast(default_opaque))).* - else - null; -} - -fn shouldEncodeField(field: std.builtin.Type.StructField, value: anytype) bool { - return switch (@typeInfo(field.type)) { +fn shouldEncodeField(comptime field_name: []const u8, value: anytype) bool { + const FieldType = @TypeOf(value); + return switch (@typeInfo(FieldType)) { .optional => value != null, - .bool => if (comptime std.mem.eql(u8, field.name, "root")) value else true, + .bool => if (comptime std.mem.eql(u8, field_name, "root")) value else true, else => true, }; } diff --git a/zml/quantization.zig b/zml/quantization.zig index b125513a..89e1b210 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -474,7 +474,7 @@ test "block FP8 E8M0 scales preserve power-of-two boundaries and nonminor axes" try zml.testing.expectEqualShapes(x.shape().setDim(.k, 8).withDtype(.f32), exe.output_shapes[1]); const fp8_max: f32 = if (dtype == .f8e4m3fn) 448.0 else 224.0; var host: [2 * 256 * 3]f32 = undefined; - var maxima = [_]f32{0} ** (2 * 8 * 3); + var maxima: [2 * 8 * 3]f32 = @splat(0); for (&host, 0..) |*value, i| { const block = (i / 3) % 256 / 32; const scale_index = (i / (256 * 3)) * 24 + block * 3 + i % 3; diff --git a/zml/shape.zig b/zml/shape.zig index 11a3efac..3e3fb56b 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -58,19 +58,19 @@ pub const Shape = struct { if (comptime stdx.meta.isStruct(T)) { var dims_: DimsArray = .empty; var tags_: TagsArray = .empty; - inline for (std.meta.fields(T)) |field| { - const fv = @field(v, field.name); - if (comptime stdx.meta.isInteger(field.type)) { + inline for (comptime std.meta.fieldNames(T), comptime std.meta.fieldTypes(T)) |field_name, Field| { + const fv = @field(v, field_name); + if (comptime stdx.meta.isInteger(Field)) { dims_.appendAssumeCapacity(@intCast(fv)); } else if (@TypeOf(fv) == @EnumLiteral() and comptime isAutoDim(fv)) { dims_.appendAssumeCapacity(-1); } else { - stdx.debug.compileError("Field {s} should be an integer or an auto dimension, got {}", .{ field.name, field.type }); + stdx.debug.compileError("Field {s} should be an integer or an auto dimension, got {}", .{ field_name, Field }); } if (comptime stdx.meta.isTuple(T)) { tags_.appendAssumeCapacity(TagUnknown); } else { - tags_.appendAssumeCapacity(toTag(field)); + tags_.appendAssumeCapacity(toTag(field_name)); } } @@ -107,9 +107,9 @@ pub const Shape = struct { } if (comptime stdx.meta.isTupleOfAny(T, isAxisConvertible)) { - inline for (std.meta.fields(T)) |field| { - axes_.appendAssumeCapacity(self.axis(@field(v, field.name))); - tags_.appendAssumeCapacity(self.tag(@field(v, field.name))); + inline for (comptime std.meta.fieldNames(T)) |field_name| { + axes_.appendAssumeCapacity(self.axis(@field(v, field_name))); + tags_.appendAssumeCapacity(self.tag(@field(v, field_name))); } return .{ axes_, tags_ }; } @@ -191,7 +191,7 @@ pub const Shape = struct { fn isTagConvertible(comptime T: type) bool { return switch (T) { - @EnumLiteral(), std.builtin.Type.StructField, Tag => true, + @EnumLiteral(), [:0]const u8, Tag => true, else => false, }; } @@ -200,14 +200,14 @@ pub const Shape = struct { const T = @TypeOf(v); return switch (T) { @EnumLiteral() => @tagName(v).ptr, - std.builtin.Type.StructField => v.name.ptr, + [:0]const u8 => v.ptr, Tag => v, - else => stdx.debug.compileError("Shape tag should be an @EnumLiteral(), a Shape.Tag or a StructField, got {}", .{T}), + else => stdx.debug.compileError("Shape tag should be an @EnumLiteral(), a Shape.Tag or a field name, got {}", .{T}), }; } fn ensureAttributesAreSync(self: Shape) void { - if (builtin.mode == .Debug) { + if (builtin.mode == .debug) { stdx.debug.assert(self._dims.len == self._tags.len, "Tags, dims and partitioning have diverged! dims={d} tags={d}", .{ self._dims.len, self._tags.len }); } } @@ -312,8 +312,8 @@ pub const Shape = struct { } if (comptime stdx.meta.isStruct(T)) { - inline for (std.meta.fields(T)) |field| { - res.appendAssumeCapacity(self.axis(@field(axes_, field.name))); + inline for (comptime std.meta.fieldNames(T)) |field_name| { + res.appendAssumeCapacity(self.axis(@field(axes_, field_name))); } return res; } @@ -861,11 +861,11 @@ pub const Shape = struct { var res = self; for (0..self.rank()) |ax| { const mesh_axis = self.partition(ax).meshAxis() orelse continue; - inline for (std.meta.fields(T)) |field| { - const old_axis = mesh.resolveLogicalAxis(toTag(field)) orelse - stdx.debug.panic("Mesh '{s}' has no logical axis '{s}'", .{ mesh.name, field.name }); + inline for (comptime std.meta.fieldNames(T)) |field_name| { + const old_axis = mesh.resolveLogicalAxis(toTag(field_name)) orelse + stdx.debug.panic("Mesh '{s}' has no logical axis '{s}'", .{ mesh.name, field_name }); if (mesh_axis == old_axis) { - const new_tag = toTag(@field(mapping, field.name)); + const new_tag = toTag(@field(mapping, field_name)); const new_axis = mesh.resolveLogicalAxis(new_tag) orelse stdx.debug.panic("Mesh '{s}' has no logical axis '{s}'", .{ mesh.name, new_tag }); res._sharding = res._sharding.set(ax, .sharded(@intCast(new_axis))); @@ -1015,10 +1015,10 @@ pub const Shape = struct { const T = @TypeOf(renames); stdx.debug.assertComptime(stdx.meta.isStructOfAny(T, isAxisConvertible), "Must pass a struct of enum literals. Passed: {any}", .{T}); var res = self; - inline for (std.meta.fields(T)) |field| { - const new_field = @field(renames, field.name); + inline for (comptime std.meta.fieldNames(T)) |field_name| { + const new_field = @field(renames, field_name); stdx.debug.assert(self.hasTag(new_field) == null, "{f}.rename({any}) failed because of duplicated axis {any}", .{ self, renames, new_field }); - res._tags.set(self.axis(field), toTag(new_field)); + res._tags.set(self.axis(field_name), toTag(new_field)); } return res; } @@ -1232,8 +1232,8 @@ pub const Shape = struct { stdx.debug.assertComptime(stdx.meta.isStruct(T), "Must pass struct of enum literals like .{ .a = .{ .a1, .a2 } }. Passed: {any}", .{T}); var res = self; - inline for (std.meta.fields(T)) |field| { - res = res.splitAxis(field, @field(axes_, field.name)); + inline for (comptime std.meta.fieldNames(T)) |field_name| { + res = res.splitAxis(field_name, @field(axes_, field_name)); } return res; } @@ -1344,9 +1344,9 @@ pub const Shape = struct { stdx.debug.assertComptime(stdx.meta.isStruct(T), "Must pass struct of enum literals like .{ .a = .{ .a1, .a2 } }. Passed: {any}", .{T}); var res = self; - inline for (std.meta.fields(T)) |field| { - stdx.debug.assertComptime(stdx.meta.isTupleOfAny(field.type, isAxisConvertible) or stdx.meta.isSliceOfAny(field.type, isAxisConvertible), "Must pass struct of axes. Passed: {any}", .{field.type}); - res = res.mergeAxis(field, @field(axes_, field.name)); + inline for (comptime std.meta.fieldNames(T), comptime std.meta.fieldTypes(T)) |field_name, Field| { + stdx.debug.assertComptime(stdx.meta.isTupleOfAny(Field, isAxisConvertible) or stdx.meta.isSliceOfAny(Field, isAxisConvertible), "Must pass struct of axes. Passed: {any}", .{Field}); + res = res.mergeAxis(field_name, @field(axes_, field_name)); } return res; } @@ -1398,14 +1398,14 @@ pub const Shape = struct { } if (comptime stdx.meta.isStruct(V)) { - const fields = std.meta.fields(V); - stdx.debug.assertComptime(fields.len <= constants.MAX_RANK, "Too many fields in struct {} ({d}). Max supported is {d}.", .{ V, fields.len, constants.MAX_RANK }); - inline for (fields) |field| { - const fv = @field(v, field.name); + const field_names = comptime std.meta.fieldNames(V); + stdx.debug.assertComptime(field_names.len <= constants.MAX_RANK, "Too many fields in struct {} ({d}). Max supported is {d}.", .{ V, field_names.len, constants.MAX_RANK }); + inline for (field_names) |field_name| { + const fv = @field(v, field_name); vals_.appendAssumeCapacity(fv); if (!comptime stdx.meta.isTuple(V)) { - tags_.appendAssumeCapacity(toTag(field)); + tags_.appendAssumeCapacity(toTag(field_name)); } } return .{ vals_, tags_ }; @@ -1435,11 +1435,11 @@ pub const Shape = struct { if (comptime stdx.meta.isStruct(V)) { for (0..self.rank()) |_| res.appendAssumeCapacity(default); - const fields = std.meta.fields(V); - stdx.debug.assertComptime(fields.len <= constants.MAX_RANK, "expects up to {} options struct literal, got {}", .{ V, constants.MAX_RANK, fields.len }); - inline for (fields) |field| { - const a = self.axis(field); - res.buffer[a] = @field(options, field.name); + const field_names = comptime std.meta.fieldNames(V); + stdx.debug.assertComptime(field_names.len <= constants.MAX_RANK, "expects up to {} options struct literal, got {}", .{ V, constants.MAX_RANK, field_names.len }); + inline for (field_names) |field_name| { + const a = self.axis(field_name); + res.buffer[a] = @field(options, field_name); } return res; } diff --git a/zml/slice.zig b/zml/slice.zig index f096608b..2deb96d2 100644 --- a/zml/slice.zig +++ b/zml/slice.zig @@ -31,7 +31,7 @@ pub const Slice = struct { const byte_strides = shape.computeByteStrides(); return .{ .bytes = bytes, - .mutable = !type_info.pointer.is_const, + .mutable = !type_info.pointer.attrs.@"const", .shape = shape, .offset_bytes = 0, .byte_strides = byte_strides, @@ -147,12 +147,12 @@ pub const Slice = struct { } pub fn items(slice: Slice, comptime T: type) []T { - stdx.debug.assertComptime(T == bool or @bitSizeOf(T) >= 8, "zml.Slice stores packed sub-bytes type so you need to pass a packed type here like @Vector({}, {}). Got: {}", .{ @divFloor(8, @bitSizeOf(T)), T, T }); + stdx.debug.assertComptime(T == bool or @bitSizeOf(T) >= 8, "zml.Slice stores packed sub-bytes type so you need to pass a packed struct here like stdx.Packed({0}). Got: {0}", .{T}); return @ptrCast(@alignCast(slice.data())); } pub fn constItems(slice: Slice, comptime T: type) []const T { - stdx.debug.assertComptime(T == bool or @bitSizeOf(T) >= 8, "zml.Slice stores packed sub-bytes type so you need to pass a packed type here like @Vector({}, {}). Got: {}", .{ @divFloor(8, @bitSizeOf(T)), T, T }); + stdx.debug.assertComptime(T == bool or @bitSizeOf(T) >= 8, "zml.Slice stores packed sub-bytes type so you need to pass a packed struct here like zml.Float4E2M1.Packed. Got: {}", .{T}); return @ptrCast(@alignCast(slice.constData())); } @@ -289,6 +289,10 @@ pub const Slice = struct { const raw_values: []const T = @ptrCast(@alignCast(slice.bytes[slice.offset_bytes..])); const values = raw_values[0..needed_len]; + if (comptime dt.bitSizeOf() < 8) { + return try stdx.fmt.formatSliceAny(values, options, stride, writer); + } + switch (comptime dt.class()) { .float => try stdx.fmt.formatFloatSlice(values, options, stride, writer), .integer => try stdx.fmt.formatIntSlice(values, options, stride, writer), diff --git a/zml/tensor.zig b/zml/tensor.zig index 96b60c0c..0a3c530f 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -38,7 +38,7 @@ pub const Tensor = struct { } fn nextTensorId() Id { - return @enumFromInt(Tensor.current_id.fetchAdd(1, .seq_cst)); + return @fromBackingInt(@intCast(Tensor.current_id.fetchAdd(1, .seq_cst))); } pub fn fromShape(shape_: Shape) Tensor { @@ -83,7 +83,7 @@ pub const Tensor = struct { var res: Tensor = .{ ._shape = sh, ._value = val, .id = nextTensorId() }; res._shape._sharding.partition = .open(sh.rank()); - if (builtin.mode == .Debug) { + if (builtin.mode == .debug) { // Check that the MLIR value actually have the same shape. const other = fromMlirValue(val); const eql_dtype = sh.dtype() == other.dtype(); @@ -468,7 +468,7 @@ pub const Tensor = struct { const platform = zml.testing.env(); const io = std.testing.io; - const inputs: [6]@Vector(2, i4) = .{ .{ -3.0, -2 }, .{ -1, 1 }, .{ 2, 3 }, .{ 1, 2 }, .{ 3, 4 }, .{ 5, -5 } }; + const inputs: [6]stdx.Packed(i4) = .{ .pack(-3.0, -2), .pack(-1, 1), .pack(2, 3), .pack(1, 2), .pack(3, 4), .pack(5, -5) }; const x_t = Tensor.init(.{ 6, 2 }, .i4); const y_t = Tensor.init(.{ 6, 2 }, .i4); const z_t = Tensor.init(.{ 6, 2 }, .i4); @@ -1432,8 +1432,8 @@ pub const Tensor = struct { test "convert u2 -> u8" { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const x_u2: [2]@Vector(4, u2) = .{ .{ 0, 1, 2, 3 }, .{ 3, 2, 1, 0 } }; - std.debug.assert(@sizeOf(@TypeOf(x_u2)) == 2); + const x_u2: [2]stdx.Packed(u2) = .{ .pack(0, 1, 2, 3), .pack(3, 2, 1, 0) }; + comptime std.debug.assert(@sizeOf(@TypeOf(x_u2)) == 2); const x_u8: [8]u8 = .{ 0, 1, 2, 3, 3, 2, 1, 0 }; const x_u2_t: Tensor = .init(.{8}, .u2); @@ -3809,8 +3809,8 @@ pub const Tensor = struct { const has_name: ?[:0]const u8, const a = switch (@typeInfo(@TypeOf(named_axis_))) { .int, .comptime_int => .{ null, self.axis(@as(i64, @intCast(named_axis_))) }, .@"struct" => |info| blk: { - stdx.debug.assertComptime(info.fields.len == 1, err_msg, .{}); - break :blk .{ info.fields[0].name, self.axis(@field(named_axis_, info.fields[0].name)) }; + stdx.debug.assertComptime(info.field_names.len == 1, err_msg, .{}); + break :blk .{ info.field_names[0], self.axis(@field(named_axis_, info.field_names[0])) }; }, else => stdx.debug.compileError(err_msg, .{}), }; diff --git a/zml/tokenizer/iree/tokenizer.zig b/zml/tokenizer/iree/tokenizer.zig index bc986d4b..0ae25c42 100644 --- a/zml/tokenizer/iree/tokenizer.zig +++ b/zml/tokenizer/iree/tokenizer.zig @@ -577,8 +577,8 @@ pub const Normalizer = struct { ), }, .Strip => |direction| b: { - const strip_left = @intFromEnum(direction) & 0b10 != 0; - const strip_right = @intFromEnum(direction) & 0b01 != 0; + const strip_left = @backingInt(direction) & 0b10 != 0; + const strip_right = @backingInt(direction) & 0b01 != 0; break :b c.iree_tokenizer_normalizer_strip_allocate(strip_left, strip_right, iree_allocator, &norm); }, }; @@ -785,10 +785,10 @@ test "normalizer sequence preserves composition across tiles" { const allocator = std.testing.allocator; var normalizer = try Normalizer.init(allocator, &.{ .NFC, .Lowercase }); defer normalizer.deinit(); - const input = "a" ** 63 ++ "e\u{301}b"; + const input = @as([63]u8, @splat('a')) ++ "e\u{301}b"; const output = try normalizer.normalize(allocator, input); defer allocator.free(output); - try std.testing.expectEqualSlices(u8, "a" ** 63 ++ "éb", output); + try std.testing.expectEqualSlices(u8, @as([63]u8, @splat('a')) ++ "éb", output); } test "normalizer sequence allocation failure retains child ownership" { From 6c279b469e00267fa956c978be8a1aa9e62747d4 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 7 Oct 2026 11:18:10 +0200 Subject: [PATCH 759/881] zml/Compiler: add extra_device_memory_bytes (#843) Adding `extra_device_memory_bytes` to the compilation options to have XLA compute the accurate amount of memory left for the temporaries with our engram hack. --- zml/Compiler.zig | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 89bc2bb6..591e3fcb 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -77,6 +77,13 @@ pub const Options = struct { meshes: []const *const Sharding.Mesh = &.{}, // If null, will be initialized from the target partitioner: ?Sharding.Partitioner = null, + /// HACK: Extra bytes added to the device memory size reported to XLA. + /// Some buffers (e.g. engram tables) live in host-pinned memory but are declared to XLA as + /// device memory, because XLA only supports a few GPU operations reading host-pinned memory. + /// XLA then counts them against device memory, leaves no budget for temporary buffers and + /// rematerializes aggressively. Callers set this to the per-device size of such buffers to + /// compensate. Remove once XLA properly supports host-pinned inputs. + extra_device_memory_bytes: u64 = 0, // Debugging options program_name: []const u8 = "zml", xla_dump_to: ?[]const u8 = null, @@ -886,7 +893,8 @@ fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.Load device_memory_size = @min(device_memory_size orelse bytes_limit, bytes_limit); } if (device_memory_size) |bytes_limit| { - c.xla_ExecutableBuildOptionsProto_set_device_memory_size(exec_build_options, @intCast(bytes_limit)); + // HACK: See `Options.extra_device_memory_bytes`. + c.xla_ExecutableBuildOptionsProto_set_device_memory_size(exec_build_options, @intCast(bytes_limit + opts.extra_device_memory_bytes)); } c.xla_ExecutableBuildOptionsProto_set_device_assignment(exec_build_options, device_assignment_blk: { From 303ff3e6a4daad72021968e2116da12fa33c25ba Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 7 Oct 2026 11:34:51 +0200 Subject: [PATCH 760/881] Revert "zig 0.17 (#693)" (#909) This reverts commit 5a1875736849bd97cc85051a9c8a6de5ca94b954. Our internal repos has a few zig patches, I need to also upgrade them before we ugprade ZML --- .bazelrc | 3 +- .github/workflows/ci.yaml | 2 +- MODULE.bazel | 7 +- MODULE.bazel.lock | 131 +- bazel/bazel.zig | 2 +- bazel/zig_index.json | 223 - bin/zml-smi/api/client.zig | 16 +- bin/zml-smi/csv/csv.zig | 15 +- bin/zml-smi/main.zig | 4 +- bin/zml-smi/platforms/neuron/nrt.zig | 12 +- bin/zml-smi/platforms/oneapi/monitor.zig | 2 +- bin/zml-smi/platforms/oneapi/oneapi.zig | 18 +- bin/zml-smi/prometheus/exposition.zig | 6 +- bin/zml-smi/tui/image_cache.zig | 2 +- bin/zml-smi/tui/print.zig | 13 +- bin/zml-smi/tui/top.zig | 5 +- bin/zml-smi/utils/dynlib.zig | 4 +- bin/zml-smi/utils/sandbox.zig | 2 +- kernels/common/args.zig | 6 +- kernels/common/control_flow.zig | 18 +- kernels/cuda_tile/builder.zig | 146 +- kernels/cuda_tile/control_flow.zig | 8 +- kernels/cuda_tile/dtype.zig | 4 +- kernels/cute/builder.zig | 56 +- kernels/cute/dtype.zig | 4 +- kernels/fly/builder.zig | 47 +- kernels/fly/dtype.zig | 6 +- kernels/fly/layout.zig | 8 +- kernels/mosaic_tpu/builder.zig | 64 +- kernels/mosaic_tpu/dtype.zig | 4 +- kernels/triton/builder.zig | 96 +- kernels/triton/dtype.zig | 4 +- mlir/dialects/arith.zig | 8 +- mlir/dialects/cuda_tile/cuda_tile.zig | 10 +- mlir/dialects/cute_ir/cute_ir.zig | 35 +- mlir/dialects/cute_ir/cute_nvgpu.zig | 442 +- mlir/dialects/fly/attributes.zig | 6 +- mlir/dialects/fly/fly.zig | 2 +- mlir/dialects/llvm_dialect.zig | 2 +- mlir/dialects/memref.zig | 2 +- mlir/dialects/stablehlo/stablehlo.zig | 4 +- mlir/dialects/ttir/ttir.zig | 24 +- mlir/mlir.zig | 10 +- pjrt/ffi.zig | 4 +- pjrt/pjrt.zig | 50 +- platforms/cuda/compat_probe.zig | 2 +- platforms/cuda/cuda.zig | 4 +- platforms/cuda/flashattn/flashattn.zig | 2 +- platforms/neuron/libzml_neuron.zig | 9 +- platforms/neuron/neuron.zig | 8 +- platforms/neuron/python_launcher.zig | 6 +- platforms/platforms.zig | 4 +- stdx/flags.zig | 197 +- stdx/fmt.zig | 6 +- stdx/json.zig | 50 +- stdx/meta.zig | 30 +- stdx/segmented_list.zig | 2 +- stdx/stdx.zig | 61 +- third_party/arocc/arocc.bazel | 12 +- third_party/arocc/repo.bzl | 5 +- third_party/libvaxis/bump-zig.patch | 2019 ++++++++ third_party/libvaxis/fixes.patch | 274 +- third_party/libvaxis/repo.bzl | 4 +- third_party/translate-c/repo.bzl | 5 +- third_party/translate-c/translate-c.bazel | 9 - third_party/uucode/repo.bzl | 3 +- third_party/uucode/uucode.bazel | 2 - third_party/zigimg/bump-zig.patch | 4491 +++++++++++++++++ third_party/zigimg/repo.bzl | 29 +- third_party/zigimg/zig-0.17.patch | 279 - third_party/zio/repo.bzl | 2 +- vfs/BUILD.bazel | 9 +- vfs/base.zig | 52 +- vfs/file.zig | 115 +- vfs/gcs.zig | 6 +- vfs/hf.zig | 4 +- vfs/http.zig | 4 +- vfs/s3.zig | 6 +- vfs/vfs.zig | 90 +- zml/Compiler.zig | 4 +- zml/Sharding.zig | 74 +- zml/attention.zig | 4 +- zml/attention/cute_kernels/flashmla_sm100.zig | 15 +- zml/dtype.zig | 7 +- zml/exe.zig | 49 +- zml/floats.zig | 14 +- zml/kernel.zig | 8 +- zml/mem.zig | 6 +- zml/meta.zig | 204 +- zml/moe/cute_kernels/persistent_mxfp4.zig | 4 +- zml/moe/cutlass_flashinfer.zig | 4 +- zml/moe/triton_kernels/mxfp4.zig | 4 +- zml/nn.zig | 11 +- zml/ops.zig | 114 +- zml/pjrtx.zig | 8 +- zml/platform.zig | 4 +- zml/profiling/profiler.zig | 2 +- zml/profiling/tracer.zig | 62 +- zml/quantization.zig | 2 +- zml/shape.zig | 72 +- zml/slice.zig | 10 +- zml/tensor.zig | 14 +- zml/tokenizer/iree/tokenizer.zig | 8 +- 103 files changed, 7945 insertions(+), 2097 deletions(-) create mode 100644 third_party/libvaxis/bump-zig.patch create mode 100644 third_party/zigimg/bump-zig.patch delete mode 100644 third_party/zigimg/zig-0.17.patch diff --git a/.bazelrc b/.bazelrc index 1b47d230..81972716 100644 --- a/.bazelrc +++ b/.bazelrc @@ -61,6 +61,8 @@ common --incompatible_disallow_empty_glob=false # Use the C linker to link Zig code common --@rules_zig//zig/settings:use_cc_common_link=true +# Zig self hosted backend is too fragile for now +common --@rules_zig//zig/settings:zigopt=-fllvm common --@rules_zig//zig/settings:use_standalone_translate_c=True @@ -107,7 +109,6 @@ common:release --compilation_mode=opt common:release --copt=-Ofast common:release --strip=always common:release --@rules_zig//zig/settings:mode=release_safe -common:release --@rules_zig//zig/settings:zigopt=-fllvm # --config=native builds everything optimized for the local CPU common:native --copt=-march=native diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 13d153b3..c996de04 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -118,7 +118,7 @@ jobs: - name: Setup Zig uses: mlugg/setup-zig@v2.2.1 with: - version: 0.17.0 + version: 0.16.0 mirror: https://mirror.zml.ai/zig - name: Zig Format diff --git a/MODULE.bazel b/MODULE.bazel index 2c35cdc0..1e05f12a 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -49,8 +49,7 @@ bazel_dep(name = "rules_shell", version = "0.6.1") # repo_name = None prevents rules_swift from eagerly auto-registering toolchains. bazel_dep(name = "rules_swift", version = "3.6.1", repo_name = None) -bazel_dep(name = "rules_zig", version = "0.17.0") - +bazel_dep(name = "rules_zig", version = "0.15.1") bazel_dep(name = "tar.bzl", version = "0.10.4") bazel_dep(name = "llvm", version = "0.8.24") bazel_dep(name = "zlib-ng", version = "2.3.3", repo_name = "llvm_zlib") @@ -96,7 +95,7 @@ zig.index(file = "//bazel:zig_index.json") zig.toolchain( # Ensure Zig actions are run locally only extra_exec_compatible_with = ["//platforms:local_execution"], - zig_version = "0.17.0", + zig_version = "0.16.0", ) zig.mirrors(urls = [ "https://pkg.machengine.org/zig", @@ -134,7 +133,7 @@ register_execution_platforms("//platforms:local_execution_platform") zls = use_extension("@rules_zig//zig/zls:extensions.bzl", "zls") zls.index(file = "@rules_zig//zig/zls/private:versions.json") zls.toolchain( - zig_version = "0.17.0", + zig_version = "0.16.0", zls_version = "0.16.0", ) use_repo(zls, "zls_toolchains") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 548aa0e3..838e2011 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -175,10 +175,8 @@ "https://bcr.bazel.build/modules/libbacktrace/1.0.0-20250926-7939218/source.json": "83239989d09748f63dee4bdd430d261e217a012670e066e2238925a303a65d89", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", "https://bcr.bazel.build/modules/libpfm/4.11.0/source.json": "caaffb3ac2b59b8aac456917a4ecf3167d40478ee79f15ab7a877ec9273937c9", - "https://bcr.bazel.build/modules/libxml2/2.15.3/MODULE.bazel": "a6bffafd8aa8b0c13eaa4bf25d55be188ec8548549395f03e98c39bab264adaa", - "https://bcr.bazel.build/modules/libxml2/2.15.3/source.json": "fd7c420cb14981416c9fadd0db555bd1e249355e0e5b63b2ce15b92996b5fbfb", - "https://bcr.bazel.build/modules/llvm/0.8.24/MODULE.bazel": "e03e030b1d0b44bb05c52665c2f43de9260af9152f666262e11e25f6c072aae3", - "https://bcr.bazel.build/modules/llvm/0.8.24/source.json": "924ed48858c1c081698f1992c7b489434375de901c47b7046e203459840cb544", + "https://bcr.bazel.build/modules/llvm/0.8.20/MODULE.bazel": "2c0fa2fb60ab1f3a8a7b46a6c60b67fd78bffcfaf7958477e74a971c0a1f1915", + "https://bcr.bazel.build/modules/llvm/0.8.20/source.json": "21ef6b5c124a1a39341752e8458de9acd6fb4abc9ad73e6e3147b374d95e5e8c", "https://bcr.bazel.build/modules/mbedtls/3.6.0/MODULE.bazel": "8e380e4698107c5f8766264d4df92e36766248447858db28187151d884995a09", "https://bcr.bazel.build/modules/mbedtls/3.6.0/source.json": "1dbe7eb5258050afcc3806b9d43050f71c6f539ce0175535c670df606790b30c", "https://bcr.bazel.build/modules/nlohmann_json/3.11.3/MODULE.bazel": "87023db2f55fc3a9949c7b08dc711fae4d4be339a80a99d04453c4bb3998eefc", @@ -301,7 +299,6 @@ "https://bcr.bazel.build/modules/rules_cc/0.2.25/source.json": "8df28a9a97b878fe45f0a4f4adf019869dda51a60ee2f6e15a0c79656a25058d", "https://bcr.bazel.build/modules/rules_cc/0.2.4/MODULE.bazel": "1ff1223dfd24f3ecf8f028446d4a27608aa43c3f41e346d22838a4223980b8cc", "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", - "https://bcr.bazel.build/modules/rules_cc/0.2.9/MODULE.bazel": "34263f1dca62ea664265438cef714d7db124c03e1ed55ebb4f1dc860164308d1", "https://bcr.bazel.build/modules/rules_distroless/0.8.0/MODULE.bazel": "a3e473943b685190e20668f5149f62a116fdcbeba66c7b9778f71d2aed4533c1", "https://bcr.bazel.build/modules/rules_distroless/0.8.0/source.json": "a6931e20d97a13675693adb8b6cd3a0e6eb5218db43dfc83b75566ddc4158efc", "https://bcr.bazel.build/modules/rules_foreign_cc/0.10.1/MODULE.bazel": "b9527010e5fef060af92b6724edb3691970a5b1f76f74b21d39f7d433641be60", @@ -429,8 +426,8 @@ "https://bcr.bazel.build/modules/rules_swift/3.1.2/MODULE.bazel": "72c8f5cf9d26427cee6c76c8e3853eb46ce6b0412a081b2b6db6e8ad56267400", "https://bcr.bazel.build/modules/rules_swift/3.6.1/MODULE.bazel": "5a68bb05b6b8044f8a7bd7184d56150189e7eca05522ce525517c234fed50ed8", "https://bcr.bazel.build/modules/rules_swift/3.6.1/source.json": "9af2e521ffdb41961b6b9ac05293bd6519ee2fb52849a2be658a20b0221cb8db", - "https://bcr.bazel.build/modules/rules_zig/0.17.0/MODULE.bazel": "2c1c08adc986cfdb5fee97e21982cac846f8a20a6a8054ae9cab8b02721c2011", - "https://bcr.bazel.build/modules/rules_zig/0.17.0/source.json": "488f71f57c596ed9b7635ebeb28066b23c6fc1802c30749d06bdad261d7b04a2", + "https://bcr.bazel.build/modules/rules_zig/0.15.1/MODULE.bazel": "f0b813d9191c8d201e0b3ecbe5c5220ddedb717a15009fc711a78b3e1db8710d", + "https://bcr.bazel.build/modules/rules_zig/0.15.1/source.json": "0bd91d7459f504196dfdc90cd0fffbf1f0df2667c9cd5eb708feb3fd27578e6a", "https://bcr.bazel.build/modules/stardoc/0.5.0/MODULE.bazel": "f9f1f46ba8d9c3362648eea571c6f9100680efc44913618811b58cc9c02cd678", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", @@ -730,7 +727,7 @@ }, "@@rules_oci+//oci:extensions.bzl%oci": { "general": { - "bzlTransitiveDigest": "u2chY5noKxJ9cum/JgI8hogPYhYniVKnJLd6f2ChPI4=", + "bzlTransitiveDigest": "lRG1JBXWgJgDfEfl/34ZHgTFACz2I0bpfZWFtcoM+hc=", "usagesDigest": "JLbamplho20rviuebatMGqnQnaoj32ewCzLuRVusGoU=", "recordedInputs": [ "REPO_MAPPING:aspect_bazel_lib+,bazel_tools bazel_tools", @@ -1512,19 +1509,19 @@ }, "@@rules_zig+//zig:extensions.bzl%zig": { "general": { - "bzlTransitiveDigest": "VY2CKW7wm6B3wCpZq7W6z3hCnHiiJlXtB4JpuB3OPD8=", - "usagesDigest": "FDLiNUluzC0WBatQ/5WFpxzyQP4ubA/DUgTwOzYLGw4=", + "bzlTransitiveDigest": "VGLOg2kVw+STO9R5OrQnLFbv7Vl5TPdtIebIs324IpU=", + "usagesDigest": "4xBoe2eIScx2QI0jVieHWXdflezUUDIApwMqS+SWGbw=", "recordedInputs": [ "REPO_MAPPING:rules_zig+,bazel_skylib bazel_skylib+", "REPO_MAPPING:rules_zig+,bazel_tools bazel_tools", - "FILE:@@//bazel/zig_index.json 387d1ab130896456fb092fb08bfaf0c57ccba854121955c752052600d1643d37", - "FILE:@@rules_zig+//zig/private/versions.json a33370258a702099b35362d4220fe8f02c869c2b845abcbf0aa78200abab097a" + "FILE:@@//bazel/zig_index.json 2e3b4b83f1add5d3ca12fd05797c04c38af21d86c7fc7c1a9b356560141a4797", + "FILE:@@rules_zig+//zig/private/versions.json 3e4e370dee9c3f3fdc8bfc7a600e76743278fde40815eead5d0bcda3aec3e2bd" ], "generatedRepoSpecs": { - "zig_0.17.0_aarch64-linux": { + "zig_0.16.0_aarch64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.17.0/zig-aarch64-linux-0.17.0.tar.xz", + "url": "https://ziglang.org/download/0.16.0/zig-aarch64-linux-0.16.0.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1554,15 +1551,15 @@ "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds" ], - "sha256": "9e8d11661d4ae3bd57702a3832781e23ad151dde5798e16a5ccd503f65234ff8", - "zig_version": "0.17.0", + "sha256": "ea4b09bfb22ec6f6c6ceac57ab63efb6b46e17ab08d21f69f3a48b38e1534f17", + "zig_version": "0.16.0", "platform": "aarch64-linux" } }, - "zig_0.17.0_aarch64-macos": { + "zig_0.16.0_aarch64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.17.0/zig-aarch64-macos-0.17.0.tar.xz", + "url": "https://ziglang.org/download/0.16.0/zig-aarch64-macos-0.16.0.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1592,15 +1589,15 @@ "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds" ], - "sha256": "b607e9b9234790a008116ae5bdb71c6243b84b9fb42a53a9e70fde41c06c536a", - "zig_version": "0.17.0", + "sha256": "b23d70deaa879b5c2d486ed3316f7eaa53e84acf6fc9cc747de152450d401489", + "zig_version": "0.16.0", "platform": "aarch64-macos" } }, - "zig_0.17.0_aarch64-windows": { + "zig_0.16.0_aarch64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.17.0/zig-aarch64-windows-0.17.0.zip", + "url": "https://ziglang.org/download/0.16.0/zig-aarch64-windows-0.16.0.zip", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1630,15 +1627,15 @@ "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds" ], - "sha256": "0a59d91fa1cb40cf068e9b0954434ce973500c7a2ea749f1e01af62cdab52d26", - "zig_version": "0.17.0", + "sha256": "aee38316ee4111717900f45dd3130145c39289e105541d737eb8c5ed653c78ef", + "zig_version": "0.16.0", "platform": "aarch64-windows" } }, - "zig_0.17.0_x86_64-linux": { + "zig_0.16.0_x86_64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.17.0/zig-x86_64-linux-0.17.0.tar.xz", + "url": "https://ziglang.org/download/0.16.0/zig-x86_64-linux-0.16.0.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1668,15 +1665,15 @@ "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds" ], - "sha256": "1cbe9df9f27e6b78d14ccbca43b6703a404ef79ef1c463de901d7f088d4e2026", - "zig_version": "0.17.0", + "sha256": "70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00", + "zig_version": "0.16.0", "platform": "x86_64-linux" } }, - "zig_0.17.0_x86_64-macos": { + "zig_0.16.0_x86_64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.17.0/zig-x86_64-macos-0.17.0.tar.xz", + "url": "https://ziglang.org/download/0.16.0/zig-x86_64-macos-0.16.0.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1706,15 +1703,15 @@ "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds" ], - "sha256": "4f9a1c5269aa17ebda5e6d3c2b89d6cbf36f7d2b22a0306e9ab98f25f95529c6", - "zig_version": "0.17.0", + "sha256": "0387557ed1877bc6a2e1802c8391953baddba76081876301c522f52977b52ba7", + "zig_version": "0.16.0", "platform": "x86_64-macos" } }, - "zig_0.17.0_x86_64-windows": { + "zig_0.16.0_x86_64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.17.0/zig-x86_64-windows-0.17.0.zip", + "url": "https://ziglang.org/download/0.16.0/zig-x86_64-windows-0.16.0.zip", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1744,8 +1741,8 @@ "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds" ], - "sha256": "b5663f69581dcf391293fbf16c06cb80d81d806545ce618b4d0bab7f0eb8c428", - "zig_version": "0.17.0", + "sha256": "68659eb5f1e4eb1437a722f1dd889c5a322c9954607f5edcf337bc3684a75a7e", + "zig_version": "0.16.0", "platform": "x86_64-windows" } }, @@ -1753,28 +1750,28 @@ "repoRuleId": "@@rules_zig+//zig/private/repo:toolchains_repo.bzl%toolchains_repo", "attributes": { "names": [ - "zig_0.17.0_aarch64-linux", - "zig_0.17.0_aarch64-macos", - "zig_0.17.0_aarch64-windows", - "zig_0.17.0_x86_64-linux", - "zig_0.17.0_x86_64-macos", - "zig_0.17.0_x86_64-windows" + "zig_0.16.0_aarch64-linux", + "zig_0.16.0_aarch64-macos", + "zig_0.16.0_aarch64-windows", + "zig_0.16.0_x86_64-linux", + "zig_0.16.0_x86_64-macos", + "zig_0.16.0_x86_64-windows" ], "labels": [ - "@zig_0.17.0_aarch64-linux//:zig_toolchain", - "@zig_0.17.0_aarch64-macos//:zig_toolchain", - "@zig_0.17.0_aarch64-windows//:zig_toolchain", - "@zig_0.17.0_x86_64-linux//:zig_toolchain", - "@zig_0.17.0_x86_64-macos//:zig_toolchain", - "@zig_0.17.0_x86_64-windows//:zig_toolchain" + "@zig_0.16.0_aarch64-linux//:zig_toolchain", + "@zig_0.16.0_aarch64-macos//:zig_toolchain", + "@zig_0.16.0_aarch64-windows//:zig_toolchain", + "@zig_0.16.0_x86_64-linux//:zig_toolchain", + "@zig_0.16.0_x86_64-macos//:zig_toolchain", + "@zig_0.16.0_x86_64-windows//:zig_toolchain" ], "zig_versions": [ - "0.17.0", - "0.17.0", - "0.17.0", - "0.17.0", - "0.17.0", - "0.17.0" + "0.16.0", + "0.16.0", + "0.16.0", + "0.16.0", + "0.16.0", + "0.16.0" ], "exec_lengths": [ 3, @@ -1830,7 +1827,7 @@ "@@rules_zig+//zig/zls:extensions.bzl%zls": { "general": { "bzlTransitiveDigest": "ZuceGxE17jROQ+kQkaPqIcK7gMKGb56I97Aq1/Dubdc=", - "usagesDigest": "UswUnORpE1Loq1Ca5pqN4rYDdsd2ECF1KGnvjJxLhR8=", + "usagesDigest": "b/ERjMTWazfa8VFWuTc0PTtQQzLDuloWSHlzXj6Zb5w=", "recordedInputs": [ "REPO_MAPPING:,rules_zig rules_zig+", "REPO_MAPPING:rules_zig+,bazel_skylib bazel_skylib+", @@ -1902,12 +1899,12 @@ "repoRuleId": "@@rules_zig+//zig/zls/private/repo:toolchains_repo.bzl%toolchains_repo", "attributes": { "names": [ - "0.16.0_for_0.17.0_aarch64-linux", - "0.16.0_for_0.17.0_aarch64-macos", - "0.16.0_for_0.17.0_aarch64-windows", - "0.16.0_for_0.17.0_x86_64-linux", - "0.16.0_for_0.17.0_x86_64-macos", - "0.16.0_for_0.17.0_x86_64-windows" + "0.16.0_for_0.16.0_aarch64-linux", + "0.16.0_for_0.16.0_aarch64-macos", + "0.16.0_for_0.16.0_aarch64-windows", + "0.16.0_for_0.16.0_x86_64-linux", + "0.16.0_for_0.16.0_x86_64-macos", + "0.16.0_for_0.16.0_x86_64-windows" ], "labels": [ "@zls_0.16.0_aarch64-linux//:zls_toolchain", @@ -1918,12 +1915,12 @@ "@zls_0.16.0_x86_64-windows//:zls_toolchain" ], "zig_versions": [ - "0.17.0", - "0.17.0", - "0.17.0", - "0.17.0", - "0.17.0", - "0.17.0" + "0.16.0", + "0.16.0", + "0.16.0", + "0.16.0", + "0.16.0", + "0.16.0" ], "exec_lengths": [ 2, diff --git a/bazel/bazel.zig b/bazel/bazel.zig index 2ae3324a..0cf23580 100644 --- a/bazel/bazel.zig +++ b/bazel/bazel.zig @@ -12,7 +12,7 @@ const init_array_section = switch (builtin.object_format) { else => "", }; -export const _bazel_runfiles_init align(@alignOf(usize)) linksection(init_array_section) = &struct { +export const _bazel_runfiles_init linksection(init_array_section) = &struct { fn call(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { _ = argc; _ = envp; diff --git a/bazel/zig_index.json b/bazel/zig_index.json index b8cf9b9e..8b39b699 100644 --- a/bazel/zig_index.json +++ b/bazel/zig_index.json @@ -1,227 +1,4 @@ { - "0.17.0": { - "version": "0.17.0", - "date": "2026-10-01", - "docs": "https://ziglang.org/documentation/0.17.0/", - "stdDocs": "https://ziglang.org/documentation/0.17.0/std/", - "notes": "https://ziglang.org/download/0.17.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.17.0/zig-0.17.0.tar.xz", - "shasum": "b6c7f1728f043700d6529bac980800792f824256a9d2f1839b3d62beed0b8abd", - "size": "23049960" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.17.0/zig-bootstrap-0.17.0.tar.xz", - "shasum": "1e9e9b8e3c753b35dfb1d9ea48f097579fadbd3e5c1e724ab3fc5bce05477b71", - "size": "57030388" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-macos-0.17.0.tar.xz", - "shasum": "4f9a1c5269aa17ebda5e6d3c2b89d6cbf36f7d2b22a0306e9ab98f25f95529c6", - "size": "59317572" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-macos-0.17.0.tar.xz", - "shasum": "b607e9b9234790a008116ae5bdb71c6243b84b9fb42a53a9e70fde41c06c536a", - "size": "53985220" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-linux-0.17.0.tar.xz", - "shasum": "1cbe9df9f27e6b78d14ccbca43b6703a404ef79ef1c463de901d7f088d4e2026", - "size": "57332648" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-linux-0.17.0.tar.xz", - "shasum": "9e8d11661d4ae3bd57702a3832781e23ad151dde5798e16a5ccd503f65234ff8", - "size": "52877280" - }, - "arm-linux": { - "tarball": "https://ziglang.org/download/0.17.0/zig-arm-linux-0.17.0.tar.xz", - "shasum": "53f0045cdef7ba06da70a12b2ef654e4c5283b0139bc441aa7d84f364b46de43", - "size": "53841896" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.17.0/zig-riscv64-linux-0.17.0.tar.xz", - "shasum": "18ff032ac6cecf746a84259f6463264f13160ff88bc3b80722ce06f9479b82ed", - "size": "57116976" - }, - "powerpc64le-linux": { - "tarball": "https://ziglang.org/download/0.17.0/zig-powerpc64le-linux-0.17.0.tar.xz", - "shasum": "731e2887e66fff41fdc34f7fd9d8afa814d86d7426f5827805f8854b2977684b", - "size": "56990076" - }, - "x86-linux": { - "tarball": "https://ziglang.org/download/0.17.0/zig-x86-linux-0.17.0.tar.xz", - "shasum": "55e39e175cd5b3098afc29ec47b2590134f5b874175369fb9ae7b8e836d7531a", - "size": "59823964" - }, - "loongarch64-linux": { - "tarball": "https://ziglang.org/download/0.17.0/zig-loongarch64-linux-0.17.0.tar.xz", - "shasum": "5a6000b866201c3b3820b563fcfd29641185393e7d35039e50235db418d8367c", - "size": "54257324" - }, - "s390x-linux": { - "tarball": "https://ziglang.org/download/0.17.0/zig-s390x-linux-0.17.0.tar.xz", - "shasum": "13905f11989b65566c544d191d4bc5bd5d6f858e303208cf1d2f090e5d91b512", - "size": "57136832" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-windows-0.17.0.zip", - "shasum": "b5663f69581dcf391293fbf16c06cb80d81d806545ce618b4d0bab7f0eb8c428", - "size": "100268137" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-windows-0.17.0.zip", - "shasum": "0a59d91fa1cb40cf068e9b0954434ce973500c7a2ea749f1e01af62cdab52d26", - "size": "95992853" - }, - "x86-windows": { - "tarball": "https://ziglang.org/download/0.17.0/zig-x86-windows-0.17.0.zip", - "shasum": "c7b83507549c7bcd42cd58c4910b47037f4150ed7778e11b3684f130620969d1", - "size": "101891323" - }, - "aarch64-freebsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-freebsd-0.17.0.tar.xz", - "shasum": "3cf53d07b3587a26727b196a6eb7918704b907d951520daa68e4e1820fc6f48e", - "size": "52860244" - }, - "arm-freebsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-arm-freebsd-0.17.0.tar.xz", - "shasum": "e0c3ecf21c186a48d3b8e8d0b4459f9dc67ff8024445401506a7e970782da5f8", - "size": "54273908" - }, - "powerpc64le-freebsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-powerpc64le-freebsd-0.17.0.tar.xz", - "shasum": "94e127c2a4beb5151a8c58e8af4b82dc3649ed95f343ffbbfa381e2afba50234", - "size": "57027652" - }, - "riscv64-freebsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-riscv64-freebsd-0.17.0.tar.xz", - "shasum": "4ab73eb01a0cb910dfa23d8efc88d38dcdd5ebb2174537cf531738ba520354f1", - "size": "57199160" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-freebsd-0.17.0.tar.xz", - "shasum": "a314f1cc92d098bd39caf9e5607a0de2a8fd0d00064776cb3fd5b3bee38a67c9", - "size": "57468884" - }, - "aarch64-netbsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-netbsd-0.17.0.tar.xz", - "shasum": "421fefe3910dedc8b8a6ae9a8905ea9200ecb0760499bcf0f56af7a5ba4930b5", - "size": "52769844" - }, - "arm-netbsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-arm-netbsd-0.17.0.tar.xz", - "shasum": "2537d66f6dbd57438355d100180d09f19640c98895d4052e4e2994a02cac320a", - "size": "55345456" - }, - "riscv64-netbsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-riscv64-netbsd-0.17.0.tar.xz", - "shasum": "687c92430ab3e7a5329e941e95f0ec4368d8d8ac0a4048f61c2dd714e675ca12", - "size": "57157568" - }, - "x86-netbsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-x86-netbsd-0.17.0.tar.xz", - "shasum": "2025c508390cc78d9c469675d2a0602f2a975c92ec197af51faf09a565b27ee3", - "size": "60380332" - }, - "x86_64-netbsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-netbsd-0.17.0.tar.xz", - "shasum": "10f71cc89efbafc6e22447f23c44047d53c5d2061c82b1c32efd1ef55ea2d9b6", - "size": "57363300" - }, - "aarch64-openbsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-openbsd-0.17.0.tar.xz", - "shasum": "1cf6c33a0cdd9a58a0b40c7ee86d8de9c190ed7bd8d47ad61687908955ecac10", - "size": "53265892" - }, - "arm-openbsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-arm-openbsd-0.17.0.tar.xz", - "shasum": "068d5200b027541c2b0a2e16efc3fb367f1fae78f9c3954e57512868c9a0ecc2", - "size": "54024240" - }, - "riscv64-openbsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-riscv64-openbsd-0.17.0.tar.xz", - "shasum": "4858839a9e4d96fb50d6e4894d10270f5377a087b2929d238b1a12b8293e274a", - "size": "57490512" - }, - "x86_64-openbsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-openbsd-0.17.0.tar.xz", - "shasum": "5e9606fc322ef4c346c199e162d26e4c293358a36ea3ca72e877d42ef71e24be", - "size": "58612444" - } - }, - "0.17.0-dev.1936+5a625d5f3": { - "version": "0.17.0-dev.1936+5a625d5f3", - "docs": "https://ziglang.org/documentation/master/", - "stdDocs": "https://ziglang.org/documentation/master/std/", - "x86_64-macos": { - "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.17.0-dev.1936+5a625d5f3.tar.xz", - "shasum": "8ab9f0d015abb75c0c9c039f7191711a7a2fc72c984458f553926725794791dd", - "size": "59162280" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.17.0-dev.1936+5a625d5f3.tar.xz", - "shasum": "e08d4485c96d2d31a8ce3ae80a58badec202920b675149ad6b0c3b974ec7c005", - "size": "53867184" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.17.0-dev.1936+5a625d5f3.tar.xz", - "shasum": "38620c06ca57b175f7dab8f65c8001f9a67b0ff1987c344d4e64ccb72b57174f", - "size": "57180268" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.17.0-dev.1936+5a625d5f3.tar.xz", - "shasum": "6c559ff4bca3e84fbbf7b6ad8521ce70bb3e460371b3d8803112353fb8ce70cc", - "size": "52741876" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.17.0-dev.1936+5a625d5f3.zip", - "shasum": "e0d62a00e067de41a0a34af2af73694e00692e045ca244748965a7b3729633a1", - "size": "100793891" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.17.0-dev.1936+5a625d5f3.zip", - "shasum": "e75f340afee0772fe4daed61d92b999534c8460601175b66fe890703cf752850", - "size": "96526396" - } - }, - "0.17.0-dev.1509+bb296ab9b": { - "version": "0.17.0-dev.1509+bb296ab9b", - "date": "2026-07-29", - "docs": "https://ziglang.org/documentation/master/", - "stdDocs": "https://ziglang.org/documentation/master/std/", - "x86_64-macos": { - "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.17.0-dev.1509+bb296ab9b.tar.xz", - "shasum": "087aad38fead6c0ba7a52b5db3c405e63708c4765eda643d4144b9e4933d45e5", - "size": "59064872" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.17.0-dev.1509+bb296ab9b.tar.xz", - "shasum": "24bd83c1d435b8ab6192f58afcc22f4e5252077a938ffbc35eaa4ca97c5be709", - "size": "53773824" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.17.0-dev.1509+bb296ab9b.tar.xz", - "shasum": "48cc865b8b410ec84eaa97e50c2bd7a657871802ce3aaaf04dd1da2294d4b28a", - "size": "57056988" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.17.0-dev.1509+bb296ab9b.tar.xz", - "shasum": "7f17e09b675cfe39805c5b551b251151f7750b9df8a168538007890c1c00dc8e", - "size": "52649416" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.17.0-dev.1509+bb296ab9b.zip", - "shasum": "a4f537761ca1f5274c9bb0e7da598c6e82205a4ea78f329690a26eade555d7e8", - "size": "100743514" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.17.0-dev.1509+bb296ab9b.zip", - "shasum": "1fa7a2b5f5442628287fead0f7451b8cb93ed2ca5dd6c79f8026fc8ca83580d2", - "size": "96467368" - } - }, "master": { "version": "0.16.0-dev.3153+d6f43caad", "date": "2026-04-12", diff --git a/bin/zml-smi/api/client.zig b/bin/zml-smi/api/client.zig index ddec6d2d..17e49917 100644 --- a/bin/zml-smi/api/client.zig +++ b/bin/zml-smi/api/client.zig @@ -6,10 +6,10 @@ const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.Ar const Collector = @import("zml-smi/collector").Collector; fn setRemote(device: *DeviceInfo) void { - inline for (@typeInfo(DeviceInfo).@"union".field_names) |field_name| { - if (device.* == @field(smi_info.device_info.Target, field_name)) { - @field(device, field_name).values[0].remote = true; - @field(device, field_name).values[1].remote = true; + inline for (@typeInfo(DeviceInfo).@"union".fields) |field| { + if (device.* == @field(smi_info.device_info.Target, field.name)) { + @field(device, field.name).values[0].remote = true; + @field(device, field.name).values[1].remote = true; } } } @@ -86,10 +86,10 @@ fn pollOnce(poll_arena: *std.heap.ArenaAllocator, gpa: std.mem.Allocator, io: st var device = std.json.parseFromValueLeaky(DeviceInfo, poll_arena.allocator(), item, .{ .ignore_unknown_fields = true }) catch continue; setRemote(&device); - inline for (@typeInfo(DeviceInfo).@"union".field_names) |field_name| { - if (devices[i].* == @field(smi_info.device_info.Target, field_name)) { - @field(devices[i], field_name).back().* = @field(device, field_name).front().*; - @field(devices[i], field_name).swap(); + inline for (@typeInfo(DeviceInfo).@"union".fields) |field| { + if (devices[i].* == @field(smi_info.device_info.Target, field.name)) { + @field(devices[i], field.name).back().* = @field(device, field.name).front().*; + @field(devices[i], field.name).swap(); } } } diff --git a/bin/zml-smi/csv/csv.zig b/bin/zml-smi/csv/csv.zig index dbd80dd9..a05c5a4f 100644 --- a/bin/zml-smi/csv/csv.zig +++ b/bin/zml-smi/csv/csv.zig @@ -3,9 +3,8 @@ const smi_info = @import("zml-smi/info"); const DeviceInfo = smi_info.device_info.DeviceInfo; pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo) !void { - const info = @typeInfo(DeviceInfo).@"union"; - inline for (info.field_names, info.field_types) |type_name, DeviceType| { - const tag = @field(smi_info.Target, type_name); + inline for (@typeInfo(DeviceInfo).@"union".fields) |tp| { + const tag = @field(smi_info.Target, tp.name); var header_printed = false; for (devices, 0..) |dev, i| { @@ -13,8 +12,8 @@ pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo) !void { tag => |*db| { if (!header_printed) { try writer.writeAll("index,type"); - inline for (@typeInfo(DeviceType.Value).@"struct".field_names) |field_name| { - try writer.writeAll("," ++ field_name); + inline for (@typeInfo(tp.type.Value).@"struct".fields) |f| { + try writer.writeAll("," ++ f.name); } try writer.writeAll("\n"); @@ -23,10 +22,10 @@ pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo) !void { const val = db.front().*; - try writer.print("{d},{s}", .{ i, type_name }); - inline for (@typeInfo(DeviceType.Value).@"struct".field_names) |field_name| { + try writer.print("{d},{s}", .{ i, tp.name }); + inline for (@typeInfo(tp.type.Value).@"struct".fields) |f| { try writer.writeAll(","); - try writeValue(writer, @field(val, field_name)); + try writeValue(writer, @field(val, f.name)); } try writer.writeAll("\n"); }, diff --git a/bin/zml-smi/main.zig b/bin/zml-smi/main.zig index f066dd34..47c861ff 100644 --- a/bin/zml-smi/main.zig +++ b/bin/zml-smi/main.zig @@ -163,9 +163,9 @@ pub fn main(init: std.process.Init) !void { try json.write(&json_writer.interface, collector.device_infos.items, procs.items); try json_writer.flush(); } else if (args.top) { - try tui.run(gpa, io, init.environ_map, &state); + try tui.run(gpa, io, &state); } else { - try static_print.run(arena, io, init.environ_map, &state); + try static_print.run(arena, io, &state); } } diff --git a/bin/zml-smi/platforms/neuron/nrt.zig b/bin/zml-smi/platforms/neuron/nrt.zig index cd77b3c0..8fe21a27 100644 --- a/bin/zml-smi/platforms/neuron/nrt.zig +++ b/bin/zml-smi/platforms/neuron/nrt.zig @@ -151,11 +151,11 @@ pub fn hbmSize(dev: *c.ndl_device_t) usize { } pub fn deviceType(dev: *c.ndl_device_t) DeviceType { - return @fromBackingInt(@intCast(dev.device_type)); + return @enumFromInt(dev.device_type); } fn resolveFromElf(handle: *anyopaque, io: std.Io, comptime known_sym: [:0]const u8) !PrivateFns { - const field_names = comptime std.meta.fieldNames(PrivateFns); + const fields = std.meta.fields(PrivateFns); const sym_addr = std.c.dlsym(handle, known_sym) orelse return error.SymbolNotFound; @@ -181,7 +181,7 @@ fn resolveFromElf(handle: *anyopaque, io: std.Io, comptime known_sym: [:0]const defer std.posix.munmap(mmap); var result: PrivateFns = undefined; - var remaining: usize = field_names.len; + var remaining: usize = fields.len; const ehdr: *const elf.ElfN.Ehdr = @ptrCast(@alignCast(mmap)); const shdrs: []const elf.ElfN.Shdr = @as([*]const elf.ElfN.Shdr, @ptrCast(@alignCast(mmap[ehdr.shoff..])))[0..ehdr.shnum]; @@ -201,9 +201,9 @@ fn resolveFromElf(handle: *anyopaque, io: std.Io, comptime known_sym: [:0]const const sym_name = std.mem.span(@as([*c]const u8, @ptrCast(strtab[@as(usize, s.name)..]))); - inline for (field_names) |field_name| { - if (std.mem.eql(u8, sym_name, field_name)) { - @field(result, field_name) = @ptrFromInt(elf_base + @as(usize, s.value)); + inline for (fields) |field| { + if (std.mem.eql(u8, sym_name, field.name)) { + @field(result, field.name) = @ptrFromInt(elf_base + @as(usize, s.value)); remaining -= 1; } } diff --git a/bin/zml-smi/platforms/oneapi/monitor.zig b/bin/zml-smi/platforms/oneapi/monitor.zig index dd5f97a9..503ae0df 100644 --- a/bin/zml-smi/platforms/oneapi/monitor.zig +++ b/bin/zml-smi/platforms/oneapi/monitor.zig @@ -149,7 +149,7 @@ inline fn openDevice(allocator: std.mem.Allocator, io: std.Io, render_idx: usize } pub fn device(self: *const Monitor, handle: Handle) !*const Device { - const idx = @backingInt(handle); + const idx = @intFromEnum(handle); if (idx >= self.devices.len) return error.NotFound; return &self.devices[idx]; } diff --git a/bin/zml-smi/platforms/oneapi/oneapi.zig b/bin/zml-smi/platforms/oneapi/oneapi.zig index 2e1e5deb..0e89a6a2 100644 --- a/bin/zml-smi/platforms/oneapi/oneapi.zig +++ b/bin/zml-smi/platforms/oneapi/oneapi.zig @@ -18,7 +18,7 @@ pub fn init(allocator: std.mem.Allocator, io: std.Io) !OneApi { pub fn handleByIndex(self: *const OneApi, device_id: usize) !Handle { if (device_id >= self.monitor.devices.len) return error.NotFound; - return @fromBackingInt(@intCast(@as(u32, @intCast(device_id)))); + return @enumFromInt(@as(u32, @intCast(device_id))); } pub fn name(self: *const OneApi, allocator: std.mem.Allocator, handle: Handle) ![]const u8 { @@ -111,7 +111,7 @@ pub fn driverVersion(self: *const OneApi, allocator: std.mem.Allocator) ![]const pub fn powerUsage(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { const dev = block: { - const idx = @backingInt(handle); + const idx = @intFromEnum(handle); if (idx >= self.monitor.devices.len) return error.NotFound; break :block &self.monitor.devices[idx]; }; @@ -144,7 +144,7 @@ pub fn fanSpeed(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u6 pub fn gpuUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { const dev = block: { - const idx = @backingInt(handle); + const idx = @intFromEnum(handle); if (idx >= self.monitor.devices.len) return error.NotFound; break :block &self.monitor.devices[idx]; }; @@ -155,7 +155,7 @@ pub fn gpuUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 }; defer usage.deinit(allocator); - const dev_idx: u16 = @intCast(@backingInt(handle)); + const dev_idx: u16 = @intCast(@intFromEnum(handle)); const current = if (usage.get(dev_idx)) |sample| sample.engine else null; const util = Monitor.processUtil(dev.activity_prev, current) orelse 0; dev.activity_prev = current; @@ -164,7 +164,7 @@ pub fn gpuUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 pub fn encoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { const dev = block: { - const idx = @backingInt(handle); + const idx = @intFromEnum(handle); if (idx >= self.monitor.devices.len) return error.NotFound; break :block &self.monitor.devices[idx]; }; @@ -175,7 +175,7 @@ pub fn encoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) }; defer usage.deinit(allocator); - const dev_idx: u16 = @intCast(@backingInt(handle)); + const dev_idx: u16 = @intCast(@intFromEnum(handle)); const current = if (usage.get(dev_idx)) |sample| sample.encoder else null; const util = Monitor.processUtil(dev.encoder_prev, current) orelse { dev.encoder_prev = current; @@ -187,7 +187,7 @@ pub fn encoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) pub fn decoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { const dev = block: { - const idx = @backingInt(handle); + const idx = @intFromEnum(handle); if (idx >= self.monitor.devices.len) return error.NotFound; break :block &self.monitor.devices[idx]; }; @@ -198,7 +198,7 @@ pub fn decoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) }; defer usage.deinit(allocator); - const dev_idx: u16 = @intCast(@backingInt(handle)); + const dev_idx: u16 = @intCast(@intFromEnum(handle)); const current = if (usage.get(dev_idx)) |sample| sample.decoder else null; const util = Monitor.processUtil(dev.decoder_prev, current) orelse { dev.decoder_prev = current; @@ -219,7 +219,7 @@ pub fn maxClockGraphics(self: *OneApi, allocator: std.mem.Allocator, handle: Han } pub fn memUsed(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev_idx: u16 = @intCast(@backingInt(handle)); + const dev_idx: u16 = @intCast(@intFromEnum(handle)); var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch return 0; defer usage.deinit(allocator); diff --git a/bin/zml-smi/prometheus/exposition.zig b/bin/zml-smi/prometheus/exposition.zig index 54d48cb6..c955226f 100644 --- a/bin/zml-smi/prometheus/exposition.zig +++ b/bin/zml-smi/prometheus/exposition.zig @@ -48,14 +48,14 @@ pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo, host: *const try writer.writeAll("# HELP zml_device_info Device metadata\n"); try writer.writeAll("# TYPE zml_device_info gauge\n"); for (devices, 0..) |dev, i| { - inline for (@typeInfo(DeviceInfo).@"union".field_names) |type_name| { - const tag = @field(smi_info.device_info.Target, type_name); + inline for (@typeInfo(DeviceInfo).@"union".fields) |tp| { + const tag = @field(smi_info.device_info.Target, tp.name); switch (dev.*) { tag => |*db| { const val = db.front().*; - try writer.print("zml_device_info{{device=\"{d}\",type=\"{s}\"", .{ i, type_name }); + try writer.print("zml_device_info{{device=\"{d}\",type=\"{s}\"", .{ i, tp.name }); inline for (.{ .{ "name", "name" }, diff --git a/bin/zml-smi/tui/image_cache.zig b/bin/zml-smi/tui/image_cache.zig index 1aea4778..cc841691 100644 --- a/bin/zml-smi/tui/image_cache.zig +++ b/bin/zml-smi/tui/image_cache.zig @@ -19,7 +19,7 @@ pub fn load( key: []const u8, data: []const u8, ) void { - const image = vx.loadImage(allocator, writer, .{ .mem = data }) catch return; + const image = vx.loadImage(allocator, undefined, writer, .{ .mem = data }) catch return; self.map.put(allocator, key, image) catch return; } diff --git a/bin/zml-smi/tui/print.zig b/bin/zml-smi/tui/print.zig index 238534cd..7338d20e 100644 --- a/bin/zml-smi/tui/print.zig +++ b/bin/zml-smi/tui/print.zig @@ -14,12 +14,11 @@ fn silentResize(vx: *vaxis.Vaxis, allocator: std.mem.Allocator, winsize: vaxis.W vx.screen_last = try vaxis.AllocatingScreen.init(allocator, winsize.cols, winsize.rows); } -pub fn run(allocator: std.mem.Allocator, io: std.Io, env_map: *std.process.Environ.Map, state: *data.SystemState) !void { - var tty_buffer: [4096]u8 = undefined; - var tty = try vaxis.Tty.init(io, &tty_buffer); +pub fn run(allocator: std.mem.Allocator, io: std.Io, state: *data.SystemState) !void { + var tty = try vaxis.Tty.init(io); defer tty.deinit(); - var vx = try vaxis.init(io, allocator, env_map, .{}); + var vx = try vaxis.init(allocator, .{}); defer { // Reset terminal modes without clearing screen content. vx.resetModes(tty.writer()) catch {}; @@ -28,12 +27,12 @@ pub fn run(allocator: std.mem.Allocator, io: std.Io, env_map: *std.process.Envir vx.deinit(allocator, &discarding.writer); } - const ws = try tty.getWinsize(); + const ws = try vaxis.Tty.getWinsize(tty.fd); try silentResize(&vx, allocator, ws); { const EventLoop = vaxis.Loop(vxfw.Event); - var loop: EventLoop = .init(io, &tty, &vx); + var loop: EventLoop = .{ .tty = &tty, .vaxis = &vx, .io = io, .queue = .{ .io = io } }; try loop.start(); defer loop.stop(); @@ -42,7 +41,7 @@ pub fn run(allocator: std.mem.Allocator, io: std.Io, env_map: *std.process.Envir const writer = tty.writer(); try writer.writeAll("\x1b7"); // DECSC: save cursor try writer.flush(); - try vx.queryTerminal(writer, .fromSeconds(1)); + try vx.queryTerminal(writer, io, 1 * std.time.ns_per_s); try writer.writeAll("\x1b8"); // DECRC: restore cursor try writer.flush(); } diff --git a/bin/zml-smi/tui/top.zig b/bin/zml-smi/tui/top.zig index 74b3796b..9a5b8624 100644 --- a/bin/zml-smi/tui/top.zig +++ b/bin/zml-smi/tui/top.zig @@ -151,9 +151,8 @@ const Model = struct { } }; -pub fn run(allocator: std.mem.Allocator, io: std.Io, env_map: *std.process.Environ.Map, state: *data.SystemState) !void { - var tty_buffer: [4096]u8 = undefined; - var app = try vxfw.App.init(io, allocator, env_map, &tty_buffer); +pub fn run(allocator: std.mem.Allocator, io: std.Io, state: *data.SystemState) !void { + var app = try vxfw.App.init(allocator, io); defer app.deinit(); var model: Model = .{ diff --git a/bin/zml-smi/utils/dynlib.zig b/bin/zml-smi/utils/dynlib.zig index 4a478107..730b495d 100644 --- a/bin/zml-smi/utils/dynlib.zig +++ b/bin/zml-smi/utils/dynlib.zig @@ -2,8 +2,8 @@ const std = @import("std"); pub fn lookupStruct(dynlib: *std.DynLib, comptime VTable: type) !VTable { var result: VTable = undefined; - inline for (comptime std.meta.fieldNames(VTable), comptime std.meta.fieldTypes(VTable)) |field_name, Field| { - @field(result, field_name) = dynlib.lookup(Field, field_name) orelse return error.SymbolResolutionFailed; + inline for (std.meta.fields(VTable)) |field| { + @field(result, field.name) = dynlib.lookup(field.type, field.name) orelse return error.SymbolResolutionFailed; } return result; diff --git a/bin/zml-smi/utils/sandbox.zig b/bin/zml-smi/utils/sandbox.zig index d4e268bd..1b53c9ea 100644 --- a/bin/zml-smi/utils/sandbox.zig +++ b/bin/zml-smi/utils/sandbox.zig @@ -11,7 +11,7 @@ const init_array_section = switch (builtin.object_format) { else => "", }; -export const _ align(@alignOf(usize)) linksection(init_array_section) = &struct { +export const _ linksection(init_array_section) = &struct { fn call(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { _ = argc; _ = argv; diff --git a/kernels/common/args.zig b/kernels/common/args.zig index 84d57d49..6d89de34 100644 --- a/kernels/common/args.zig +++ b/kernels/common/args.zig @@ -1,4 +1,6 @@ pub fn NamedArgs(comptime Spec: type, comptime ValueT: type) type { - const field_names = @typeInfo(Spec).@"struct".field_names; - return @Struct(.auto, null, field_names, &@splat(ValueT), &@splat(.{})); + const in = @typeInfo(Spec).@"struct".fields; + comptime var names: [in.len][]const u8 = undefined; + for (in, 0..) |f, i| names[i] = f.name; + return @Struct(.auto, null, &names, &@splat(ValueT), &@splat(.{})); } diff --git a/kernels/common/control_flow.zig b/kernels/common/control_flow.zig index afd79e8f..252184f3 100644 --- a/kernels/common/control_flow.zig +++ b/kernels/common/control_flow.zig @@ -13,7 +13,7 @@ pub fn tupleArity(comptime T: type, comptime what: []const u8) comptime_int { const info = @typeInfo(T); if (info != .@"struct" or !info.@"struct".is_tuple) @compileError(what ++ " must be a tuple literal like `.{ v1, v2 }`"); - return info.@"struct".field_names.len; + return info.@"struct".fields.len; } pub fn emitScfYield( @@ -27,13 +27,13 @@ pub fn emitScfYield( const info = @typeInfo(@TypeOf(values)); if (info != .@"struct" or !info.@"struct".is_tuple) @compileError(what ++ " expects a tuple literal"); - if (info.@"struct".field_names.len != N) + if (info.@"struct".fields.len != N) @compileError(what ++ ": yield arity must match the scope's declared arity"); var buf: [N]*const mlir.Value = undefined; - inline for (info.@"struct".field_names, info.@"struct".field_types, 0..) |field_name, Field, i| { - if (Field != ValueT) + inline for (info.@"struct".fields, 0..) |f, i| { + if (f.type != ValueT) @compileError(what ++ ": every tuple element must be a Value"); - buf[i] = @field(values, field_name).inner; + buf[i] = @field(values, f.name).inner; } _ = scf.yield(kernel.ctx, &buf, kernel.loc()).appendTo(block); } @@ -151,14 +151,14 @@ pub fn WhileScope(comptime BuilderT: type, comptime ValueT: type, comptime N: us const info = @typeInfo(@TypeOf(forwarded)); if (info != .@"struct" or !info.@"struct".is_tuple) @compileError("WhileScope.yieldBefore: forwarded must be a tuple literal"); - if (info.@"struct".field_names.len != M) + if (info.@"struct".fields.len != M) @compileError("WhileScope.yieldBefore: forwarded arity must match after_types arity"); const k = self.kernel; var buf: [M]*const mlir.Value = undefined; - inline for (info.@"struct".field_names, info.@"struct".field_types, 0..) |field_name, Field, i| { - if (Field != ValueT) + inline for (info.@"struct".fields, 0..) |f, i| { + if (f.type != ValueT) @compileError("WhileScope.yieldBefore: every forwarded element must be a Value"); - buf[i] = @field(forwarded, field_name).inner; + buf[i] = @field(forwarded, f.name).inner; } _ = scf.condition(k.ctx, cond.inner, &buf, k.loc()).appendTo(self.before_block); k.popBlock(); diff --git a/kernels/cuda_tile/builder.zig b/kernels/cuda_tile/builder.zig index 6067ed4b..d73742a1 100644 --- a/kernels/cuda_tile/builder.zig +++ b/kernels/cuda_tile/builder.zig @@ -84,8 +84,8 @@ pub const Value = struct { pub fn isFloatElem(self: Value) bool { const et = self.elemType(); - inline for (comptime std.meta.fieldNames(mlir.FloatTypes)) |f_name| { - if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, f_name))) != null) return true; + inline for (std.meta.fields(mlir.FloatTypes)) |f| { + if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, f.name))) != null) return true; } return false; } @@ -509,13 +509,13 @@ pub const Builder = struct { pub fn declareArgsOpts(self: *Builder, spec: anytype, opts: Opts) !dsl.NamedArgs(@TypeOf(spec), Value) { const Spec = @TypeOf(spec); - const field_names = @typeInfo(Spec).@"struct".field_names; + const fields = @typeInfo(Spec).@"struct".fields; - var arg_specs: [field_names.len]ArgSpec = undefined; - inline for (0.., field_names) |i, field_name| { - const raw = @field(spec, field_name); + var arg_specs: [fields.len]ArgSpec = undefined; + inline for (fields, 0..) |f, i| { + const raw = @field(spec, f.name); const kind: ArgSpec.Kind = if (@TypeOf(raw) == ArgSpec.Kind) raw else blk: { - const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; + const variant = @typeInfo(@TypeOf(raw)).@"struct".fields[0].name; const tag = @field(std.meta.Tag(ArgSpec.Kind), variant); const inner = @field(raw, variant); break :blk switch (tag) { @@ -526,14 +526,14 @@ pub const Builder = struct { } }, }; }; - arg_specs[i] = .{ .name = field_name, .kind = kind }; + arg_specs[i] = .{ .name = f.name, .kind = kind }; } try self.declareArgsLowOpts(&arg_specs, opts); var named: dsl.NamedArgs(Spec, Value) = undefined; - inline for (0.., field_names) |i, field_name| { - @field(named, field_name) = self.arg(i); + inline for (fields, 0..) |f, i| { + @field(named, f.name) = self.arg(i); } return named; } @@ -643,12 +643,12 @@ pub const Builder = struct { const info = @typeInfo(T); if (info != .@"struct" or !info.@"struct".is_tuple) @compileError("Builder.yield expects a tuple literal like `.{ v1, v2 }`"); - const n = info.@"struct".field_names.len; + const n = info.@"struct".fields.len; const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.yield OOM"); - inline for (info.@"struct".field_names, 0..) |f, i| { - if (@FieldType(T, f) != Value) - @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(@FieldType(T, f))); - out[i] = @field(values, f); + inline for (info.@"struct".fields, 0..) |f, i| { + if (f.type != Value) + @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(f.type)); + out[i] = @field(values, f.name); } return out; } @@ -1633,7 +1633,7 @@ pub const Builder = struct { inits: anytype, ) ForScope(tupleArity(@TypeOf(inits), "openFor: inits")) { const N = comptime tupleArity(@TypeOf(inits), "openFor: inits"); - const fields = @typeInfo(@TypeOf(inits)).@"struct".field_names; + const fields = @typeInfo(@TypeOf(inits)).@"struct".fields; // Literals take the type of the first bound that is a Value; all // three must agree (`AllTypesMatch` on the op). @@ -1651,8 +1651,8 @@ pub const Builder = struct { block_locs[0] = self.loc(); var inits_inner: [N]*const mlir.Value = undefined; inline for (fields, 0..) |f, i| { - const raw = @field(inits, f); - const v: Value = if (@FieldType(@TypeOf(inits), f) == Value) raw else self.lift(raw); + const raw = @field(inits, f.name); + const v: Value = if (f.type == Value) raw else self.lift(raw); block_types[i + 1] = v.type_(); block_locs[i + 1] = self.loc(); inits_inner[i] = v.inner; @@ -1692,13 +1692,13 @@ pub const Builder = struct { result_types: anytype, ) IfScope(tupleArity(@TypeOf(result_types), "openIfElse: result_types")) { const N = comptime tupleArity(@TypeOf(result_types), "openIfElse: result_types"); - const fields = @typeInfo(@TypeOf(result_types)).@"struct".field_names; + const fields = @typeInfo(@TypeOf(result_types)).@"struct".fields; var types: [N]*const mlir.Type = undefined; inline for (fields, 0..) |f, i| { - if (@FieldType(@TypeOf(result_types), f) != *const mlir.Type) + if (f.type != *const mlir.Type) @compileError("openIfElse: every result_type must be *const mlir.Type (use b.scalarTy/tileTy)"); - types[i] = @field(result_types, f); + types[i] = @field(result_types, f.name); } const then_block = mlir.Block.init(&.{}, &.{}); @@ -1718,14 +1718,14 @@ pub const Builder = struct { /// inside an `if`, and close the body with `scope.yield`. pub fn openLoop(self: *Builder, inits: anytype) LoopScope(tupleArity(@TypeOf(inits), "openLoop: inits")) { const N = comptime tupleArity(@TypeOf(inits), "openLoop: inits"); - const fields = @typeInfo(@TypeOf(inits)).@"struct".field_names; + const fields = @typeInfo(@TypeOf(inits)).@"struct".fields; var block_types: [N]*const mlir.Type = undefined; var block_locs: [N]*const mlir.Location = undefined; var inits_inner: [N]*const mlir.Value = undefined; inline for (fields, 0..) |f, i| { - const raw = @field(inits, f); - const v: Value = if (@FieldType(@TypeOf(inits), f) == Value) raw else self.lift(raw); + const raw = @field(inits, f.name); + const v: Value = if (f.type == Value) raw else self.lift(raw); block_types[i] = v.type_(); block_locs[i] = self.loc(); inits_inner[i] = v.inner; @@ -1769,7 +1769,7 @@ pub const Builder = struct { try al.writer.print("{f}", .{self.module.operation()}); - return try self.allocator.dupeSentinel(u8, al.written(), 0); + return try self.allocator.dupeZ(u8, al.written()); } }; @@ -1799,12 +1799,12 @@ test "empty entry round-trips" { const ir = try b.finish(&.{}); defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.find(u8, ir, "cuda_tile.module @empty") != null); - try std.testing.expect(std.mem.find(u8, ir, "entry @empty(") != null); - try std.testing.expect(std.mem.find(u8, ir, "assume") != null); - try std.testing.expect(std.mem.find(u8, ir, "div_by<16>") != null); - try std.testing.expect(std.mem.find(u8, ir, "cuda_tile.entry") == null); - try std.testing.expect(std.mem.find(u8, ir, "return") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.module @empty") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "entry @empty(") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "assume") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "div_by<16>") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.entry") == null); + try std.testing.expect(std.mem.indexOf(u8, ir, "return") != null); try expectRoundTrip(ctx, ir); } @@ -1827,14 +1827,14 @@ test "add_one over pointer tiles matches the XLA milestone kernel" { const ir = try b.finish(&.{}); defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.find(u8, ir, "iota : tile<128xi32>") != null); - try std.testing.expect(std.mem.find(u8, ir, "reshape") != null); - try std.testing.expect(std.mem.find(u8, ir, "broadcast") != null); - try std.testing.expect(std.mem.find(u8, ir, "offset") != null); - try std.testing.expect(std.mem.find(u8, ir, "load_ptr_tko weak") != null); - try std.testing.expect(std.mem.find(u8, ir, "addf") != null); - try std.testing.expect(std.mem.find(u8, ir, "store_ptr_tko weak") != null); - try std.testing.expect(std.mem.find(u8, ir, "token = ") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "iota : tile<128xi32>") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "reshape") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "broadcast") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "offset") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "load_ptr_tko weak") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "addf") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "store_ptr_tko weak") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "token = ") != null); try expectRoundTrip(ctx, ir); } @@ -1866,13 +1866,13 @@ test "vector_add over partition views" { const ir = try b.finish(&.{}); defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.find(u8, ir, "optimization_hints=") != null); - try std.testing.expect(std.mem.find(u8, ir, "make_tensor_view") != null); - try std.testing.expect(std.mem.find(u8, ir, "partition_view") != null or std.mem.find(u8, ir, "addf") != null); - try std.testing.expect(std.mem.find(u8, ir, "store_view_tko weak") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "optimization_hints=") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "make_tensor_view") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "partition_view") != null or std.mem.indexOf(u8, ir, "addf") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "store_view_tko weak") != null); try expectRoundTrip(ctx, ir); } @@ -1907,10 +1907,10 @@ test "gemm with a for loop and mmaf" { const ir = try b.finish(&.{}); defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.find(u8, ir, "for ") != null); - try std.testing.expect(std.mem.find(u8, ir, "iter_values") != null); - try std.testing.expect(std.mem.find(u8, ir, "mmaf") != null); - try std.testing.expect(std.mem.find(u8, ir, "continue ") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "for ") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "iter_values") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "mmaf") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "continue ") != null); try expectRoundTrip(ctx, ir); } @@ -1997,14 +1997,14 @@ test "if, if-else, loop with break, reduce, scan, cast, select" { const ir = try b.finish(&.{}); defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.find(u8, ir, "reduce ") != null); - try std.testing.expect(std.mem.find(u8, ir, "scan ") != null); - try std.testing.expect(std.mem.find(u8, ir, "ftoi") != null); - try std.testing.expect(std.mem.find(u8, ir, "itof") != null); - try std.testing.expect(std.mem.find(u8, ir, "select") != null); - try std.testing.expect(std.mem.find(u8, ir, "loop ") != null); - try std.testing.expect(std.mem.find(u8, ir, "break ") != null); - try std.testing.expect(std.mem.find(u8, ir, "yield") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "reduce ") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "scan ") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "ftoi") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "itof") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "select") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "loop ") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "break ") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "yield") != null); try expectRoundTrip(ctx, ir); } @@ -2055,20 +2055,20 @@ test "atomics, print, globals, alloca, dynamic views, strided view, tokens" { const ir = try b.finish(&.{}); defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.find(u8, ir, "global @lut") != null); - try std.testing.expect(std.mem.find(u8, ir, "get_global @lut") != null); - try std.testing.expect(std.mem.find(u8, ir, "tensor_view") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "rounding") != null); try expectRoundTrip(ctx, ir); } diff --git a/kernels/cuda_tile/control_flow.zig b/kernels/cuda_tile/control_flow.zig index f51e8571..2f971f1d 100644 --- a/kernels/cuda_tile/control_flow.zig +++ b/kernels/cuda_tile/control_flow.zig @@ -13,13 +13,13 @@ fn collect(comptime ValueT: type, comptime N: usize, values: anytype, comptime w const info = @typeInfo(@TypeOf(values)); if (info != .@"struct" or !info.@"struct".is_tuple) @compileError(what ++ " expects a tuple literal"); - if (info.@"struct".field_names.len != N) + if (info.@"struct".fields.len != N) @compileError(what ++ ": arity must match the scope's declared arity"); var buf: [N]*const mlir.Value = undefined; - inline for (info.@"struct".field_names, 0..) |f, i| { - if (@FieldType(@TypeOf(values), f) != ValueT) + inline for (info.@"struct".fields, 0..) |f, i| { + if (f.type != ValueT) @compileError(what ++ ": every tuple element must be a Value"); - buf[i] = @field(values, f).inner; + buf[i] = @field(values, f.name).inner; } return buf; } diff --git a/kernels/cuda_tile/dtype.zig b/kernels/cuda_tile/dtype.zig index a9193839..4552e662 100644 --- a/kernels/cuda_tile/dtype.zig +++ b/kernels/cuda_tile/dtype.zig @@ -73,8 +73,8 @@ pub fn intBitwidth(dt: DType) u32 { } pub fn mlirElemToDType(ctx: *mlir.Context, elem: *const mlir.Type) DType { - inline for (comptime std.meta.fieldNames(DType)) |f_name| { - const dt = @field(DType, f_name); + inline for (std.meta.fields(DType)) |f| { + const dt = @field(DType, f.name); if (elem.eql(dt.toMlir(ctx))) return dt; } @panic("element type not a recognized CUDA Tile IR DType"); diff --git a/kernels/cute/builder.zig b/kernels/cute/builder.zig index 77f5f824..9c777491 100644 --- a/kernels/cute/builder.zig +++ b/kernels/cute/builder.zig @@ -97,8 +97,8 @@ pub const Value = struct { pub fn isFloat(self: Value) bool { const t = self.type_(); - inline for (@typeInfo(mlir.FloatTypes).@"enum".field_names) |f| { - if (t.isA(mlir.FloatType(@field(mlir.FloatTypes, f))) != null) return true; + inline for (std.meta.fields(mlir.FloatTypes)) |f| { + if (t.isA(mlir.FloatType(@field(mlir.FloatTypes, f.name))) != null) return true; } return false; } @@ -671,14 +671,14 @@ pub const Builder = struct { /// yields a `Tensor`, `.{ .p = .{ .ptr = .f32 } }` a pointer `Value`. pub fn declareArgs(self: *Builder, spec: anytype) FinishError!ArgsOf(@TypeOf(spec)) { const Spec = @TypeOf(spec); - const fields = @typeInfo(Spec).@"struct".field_names; + const fields = @typeInfo(Spec).@"struct".fields; const arg_specs = try self.arena.allocator().alloc(ArgSpec, fields.len); inline for (fields, 0..) |f, i| { - const raw = @field(spec, f); - const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; + const raw = @field(spec, f.name); + const variant = @typeInfo(@TypeOf(raw)).@"struct".fields[0].name; const inner = @field(raw, variant); - arg_specs[i] = .{ .name = f, .kind = switch (@field(std.meta.Tag(ArgSpec.Kind), variant)) { + arg_specs[i] = .{ .name = f.name, .kind = switch (@field(std.meta.Tag(ArgSpec.Kind), variant)) { .ptr => .{ .ptr = inner }, .tensor => .{ .tensor = .{ .dtype = inner.dtype, @@ -700,7 +700,7 @@ pub const Builder = struct { var named: ArgsOf(Spec) = undefined; inline for (fields, 0..) |f, i| { - @field(named, f) = switch (@FieldType(ArgsOf(Spec), f)) { + @field(named, f.name) = switch (@FieldType(ArgsOf(Spec), f.name)) { Tensor => self.tensors[i].?, else => self.arg(i), }; @@ -709,12 +709,12 @@ pub const Builder = struct { } fn ArgsOf(comptime Spec: type) type { - const in = @typeInfo(Spec).@"struct".field_names; + const in = @typeInfo(Spec).@"struct".fields; comptime var names: [in.len][]const u8 = undefined; comptime var types: [in.len]type = undefined; inline for (in, 0..) |f, i| { - names[i] = f; - types[i] = if (@hasField(@FieldType(Spec, f), "tensor")) Tensor else Value; + names[i] = f.name; + types[i] = if (@hasField(f.type, "tensor")) Tensor else Value; } return @Struct(.auto, null, &names, &types, &@splat(.{})); } @@ -1025,7 +1025,7 @@ pub const Builder = struct { rest = payload[end + 1 ..]; } try output.writer.writeAll(rest); - return try self.allocator.dupeSentinel(u8, output.written(), 0); + return try self.allocator.dupeZ(u8, output.written()); } // ==================== types ==================== @@ -1134,9 +1134,9 @@ pub const Builder = struct { .comptime_int, .int => writer.print("{d}", .{value}) catch @panic("OOM"), .@"struct" => |info| { writer.writeByte('(') catch @panic("OOM"); - inline for (info.field_names, 0..) |field, i| { + inline for (info.fields, 0..) |field, i| { if (i != 0) writer.writeByte(',') catch @panic("OOM"); - self.writeAlgebra(writer, @field(value, field)); + self.writeAlgebra(writer, @field(value, field.name)); } writer.writeByte(')') catch @panic("OOM"); }, @@ -1157,10 +1157,10 @@ pub const Builder = struct { pub fn basis(self: *Builder, numerator: i64, denominator: i64, modes: anytype) ScaledBasis { _ = self; if (denominator <= 0) @panic("CuTe scaled-basis denominator must be positive"); - const fields = @typeInfo(@TypeOf(modes)).@"struct".field_names; + const fields = @typeInfo(@TypeOf(modes)).@"struct".fields; if (fields.len == 0 or fields.len > MAX_RANK) @panic("CuTe scaled basis needs 1..MAX_RANK modes"); var result: ScaledBasis = .{ .numerator = numerator, .denominator = denominator, .mode_count = fields.len }; - inline for (fields, 0..) |field, i| result.modes[i] = @intCast(@field(modes, field)); + inline for (fields, 0..) |field, i| result.modes[i] = @intCast(@field(modes, field.name)); return result; } @@ -1349,13 +1349,13 @@ pub const Builder = struct { switch (@typeInfo(T)) { .comptime_int, .int => writer.print("{d}", .{coord}) catch @panic("OOM"), .@"struct" => |info| { - if (info.field_names.len == 0) @compileError("makeCoord: empty tuple"); - if (info.field_names.len > 1) writer.writeByte('(') catch @panic("OOM"); - inline for (info.field_names, 0..) |field, i| { + if (info.fields.len == 0) @compileError("makeCoord: empty tuple"); + if (info.fields.len > 1) writer.writeByte('(') catch @panic("OOM"); + inline for (info.fields, 0..) |field, i| { if (i != 0) writer.writeByte(',') catch @panic("OOM"); - self.writeCoord(writer, dyn, @field(coord, field)); + self.writeCoord(writer, dyn, @field(coord, field.name)); } - if (info.field_names.len > 1) writer.writeByte(')') catch @panic("OOM"); + if (info.fields.len > 1) writer.writeByte(')') catch @panic("OOM"); }, else => @compileError("makeCoord: unsupported coordinate leaf " ++ @typeName(T)), } @@ -3010,8 +3010,8 @@ pub const Builder = struct { pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(tupleArity(@TypeOf(result_types), "openIfElse: result_types")) { const N = comptime tupleArity(@TypeOf(result_types), "openIfElse: result_types"); var types: [N]*const mlir.Type = undefined; - inline for (@typeInfo(@TypeOf(result_types)).@"struct".field_names, 0..) |f, i| { - types[i] = @field(result_types, f); + inline for (@typeInfo(@TypeOf(result_types)).@"struct".fields, 0..) |f, i| { + types[i] = @field(result_types, f.name); } const then_block = mlir.Block.init(&.{}, &.{}); const else_block = mlir.Block.init(&.{}, &.{}); @@ -3031,8 +3031,8 @@ pub const Builder = struct { block_types[0] = lb.type_(); block_locs[0] = self.loc(); var inits_inner: [N]*const mlir.Value = undefined; - inline for (@typeInfo(@TypeOf(inits)).@"struct".field_names, 0..) |f, i| { - const v = self.lift(@field(inits, f)); + inline for (@typeInfo(@TypeOf(inits)).@"struct".fields, 0..) |f, i| { + const v = self.lift(@field(inits, f.name)); block_types[i + 1] = v.type_(); block_locs[i + 1] = self.loc(); inits_inner[i] = v.inner; @@ -3071,8 +3071,8 @@ pub const Builder = struct { var init_types: [N]*const mlir.Type = undefined; var init_locs: [N]*const mlir.Location = undefined; var inits_inner: [N]*const mlir.Value = undefined; - inline for (@typeInfo(@TypeOf(inits)).@"struct".field_names, 0..) |f, i| { - const v = self.lift(@field(inits, f)); + inline for (@typeInfo(@TypeOf(inits)).@"struct".fields, 0..) |f, i| { + const v = self.lift(@field(inits, f.name)); init_types[i] = v.type_(); init_locs[i] = self.loc(); inits_inner[i] = v.inner; @@ -3080,8 +3080,8 @@ pub const Builder = struct { var result_types: [M]*const mlir.Type = undefined; var result_locs: [M]*const mlir.Location = undefined; - inline for (@typeInfo(@TypeOf(after_types)).@"struct".field_names, 0..) |f, i| { - result_types[i] = @field(after_types, f); + inline for (@typeInfo(@TypeOf(after_types)).@"struct".fields, 0..) |f, i| { + result_types[i] = @field(after_types, f.name); result_locs[i] = self.loc(); } diff --git a/kernels/cute/dtype.zig b/kernels/cute/dtype.zig index ae98cb08..6dc9cc14 100644 --- a/kernels/cute/dtype.zig +++ b/kernels/cute/dtype.zig @@ -55,8 +55,8 @@ pub fn dtypeBitwidth(dt: DType) usize { } pub fn mlirToDType(ctx: *mlir.Context, t: *const mlir.Type) DType { - inline for (@typeInfo(DType).@"enum".field_names) |f| { - const dt = @field(DType, f); + inline for (std.meta.fields(DType)) |f| { + const dt = @field(DType, f.name); if (t.eql(dt.toMlir(ctx))) return dt; } std.debug.panic("type {f} is not a CuTe DType", .{t}); diff --git a/kernels/fly/builder.zig b/kernels/fly/builder.zig index eee124ab..c560d2df 100644 --- a/kernels/fly/builder.zig +++ b/kernels/fly/builder.zig @@ -736,22 +736,22 @@ pub const Builder = struct { /// `.{ .tensor = .{ .dtype = ..., .dims = &.{ ... } } }`. pub fn declareArgs(self: *Builder, spec: anytype) !dsl.NamedArgs(@TypeOf(spec), Value) { const Spec = @TypeOf(spec); - const fields = @typeInfo(Spec).@"struct".field_names; + const fields = @typeInfo(Spec).@"struct".fields; var specs: [fields.len]ArgSpec = undefined; inline for (fields, 0..) |f, i| { - const raw = @field(spec, f); - const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; + const raw = @field(spec, f.name); + const variant = @typeInfo(@TypeOf(raw)).@"struct".fields[0].name; const inner = @field(raw, variant); specs[i] = if (comptime std.mem.eql(u8, variant, "ptr")) - .{ .name = f, .dtype = inner } + .{ .name = f.name, .dtype = inner } else if (comptime std.mem.eql(u8, variant, "tensor")) - .{ .name = f, .dtype = inner.dtype, .dims = inner.dims, .space = if (@hasField(@TypeOf(inner), "space")) inner.space else .global } + .{ .name = f.name, .dtype = inner.dtype, .dims = inner.dims, .space = if (@hasField(@TypeOf(inner), "space")) inner.space else .global } else - @compileError("fly.declareArgs: argument `" ++ f ++ "` must be `.{ .ptr = dtype }` or `.{ .tensor = .{ .dtype, .dims } }`"); + @compileError("fly.declareArgs: argument `" ++ f.name ++ "` must be `.{ .ptr = dtype }` or `.{ .tensor = .{ .dtype, .dims } }`"); } try self.declareArgsLow(&specs); var named: dsl.NamedArgs(Spec, Value) = undefined; - inline for (fields, 0..) |f, i| @field(named, f) = self.arg(i); + inline for (fields, 0..) |f, i| @field(named, f.name) = self.arg(i); return named; } @@ -969,8 +969,8 @@ pub const Builder = struct { .comptime_int, .int => return .static(@intCast(v)), .@"struct" => |s| { if (!s.is_tuple) @compileError("fly.intTuple: expected an integer, null, Value, IntTuple or tuple, got " ++ @typeName(T)); - const elems = self.alloc(IntTuple, s.field_names.len); - inline for (s.field_names, 0..) |f, i| elems[i] = self.intTupleSpec(operands, @field(v, f)); + const elems = self.alloc(IntTuple, s.fields.len); + inline for (s.fields, 0..) |f, i| elems[i] = self.intTupleSpec(operands, @field(v, f.name)); return .{ .tup = elems }; }, else => @compileError("fly.intTuple: expected an integer, null, Value, IntTuple or tuple, got " ++ @typeName(T)), @@ -1367,7 +1367,7 @@ pub const Builder = struct { inits: anytype, ) ForScope(tupleArity(@TypeOf(inits), "openFor: inits")) { const N = comptime tupleArity(@TypeOf(inits), "openFor: inits"); - const fields = @typeInfo(@TypeOf(inits)).@"struct".field_names; + const fields = @typeInfo(@TypeOf(inits)).@"struct".fields; const lb_v: Value = if (@TypeOf(lower) == Value) lower else self.constIndex(lower); const ub_v: Value = if (@TypeOf(upper) == Value) upper else self.constLike(upper, lb_v); @@ -1379,8 +1379,8 @@ pub const Builder = struct { block_locs[0] = self.loc(); var inits_inner: [N]*const mlir.Value = undefined; inline for (fields, 0..) |f, i| { - if (@TypeOf(@field(inits, f)) != Value) @compileError("openFor: every init must be a Value"); - const v: Value = @field(inits, f); + if (f.type != Value) @compileError("openFor: every init must be a Value"); + const v: Value = @field(inits, f.name); block_types[i + 1] = v.type_(); block_locs[i + 1] = self.loc(); inits_inner[i] = v.inner; @@ -1423,11 +1423,11 @@ pub const Builder = struct { pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(tupleArity(@TypeOf(result_types), "openIfElse: result_types")) { const N = comptime tupleArity(@TypeOf(result_types), "openIfElse: result_types"); - const fields = @typeInfo(@TypeOf(result_types)).@"struct".field_names; + const fields = @typeInfo(@TypeOf(result_types)).@"struct".fields; var types: [N]*const mlir.Type = undefined; inline for (fields, 0..) |f, i| { - if (@TypeOf(@field(result_types, f)) != *const mlir.Type) @compileError("openIfElse: every result_type must be *const mlir.Type"); - types[i] = @field(result_types, f); + if (f.type != *const mlir.Type) @compileError("openIfElse: every result_type must be *const mlir.Type"); + types[i] = @field(result_types, f.name); } const then_block = mlir.Block.init(&.{}, &.{}); const else_block = mlir.Block.init(&.{}, &.{}); @@ -1445,16 +1445,16 @@ pub const Builder = struct { ) { const N = comptime tupleArity(@TypeOf(inits), "openWhile: inits"); const M = comptime tupleArity(@TypeOf(after_types), "openWhile: after_types"); - const init_fields = @typeInfo(@TypeOf(inits)).@"struct".field_names; - const type_fields = @typeInfo(@TypeOf(after_types)).@"struct".field_names; + const init_fields = @typeInfo(@TypeOf(inits)).@"struct".fields; + const type_fields = @typeInfo(@TypeOf(after_types)).@"struct".fields; var before_types: [N]*const mlir.Type = undefined; var before_locs: [N]*const mlir.Location = undefined; var inits_inner: [N]*const mlir.Value = undefined; inline for (init_fields, 0..) |f, i| { - if (@TypeOf(@field(inits, f)) != Value) + if (f.type != Value) @compileError("openWhile: every init must be a Value"); - const v: Value = @field(inits, f); + const v: Value = @field(inits, f.name); before_types[i] = v.type_(); before_locs[i] = self.loc(); inits_inner[i] = v.inner; @@ -1464,9 +1464,9 @@ pub const Builder = struct { var after_tys: [M]*const mlir.Type = undefined; var after_locs: [M]*const mlir.Location = undefined; inline for (type_fields, 0..) |f, i| { - if (@TypeOf(@field(after_types, f)) != *const mlir.Type) + if (f.type != *const mlir.Type) @compileError("openWhile: every after_type must be *const mlir.Type"); - after_tys[i] = @field(after_types, f); + after_tys[i] = @field(after_types, f.name); after_locs[i] = self.loc(); } const after_block = mlir.Block.init(&after_tys, &after_locs); @@ -1500,7 +1500,7 @@ pub const Builder = struct { var al: std.Io.Writer.Allocating = .init(self.allocator); defer al.deinit(); try al.writer.print("{f}", .{self.module.operation().fmt(.{ .debug_info = false })}); - return try self.allocator.dupeSentinel(u8, al.written(), 0); + return try self.allocator.dupeZ(u8, al.written()); } }; @@ -1784,7 +1784,8 @@ test "tiledMma builds, verifies and re-parses" { test "MmaFlavor.atomThreads matches the dialect" { const ctx = try testContext(); defer ctx.deinit(); - inline for (comptime std.enums.values(MmaFlavor)) |flavor| { + inline for (std.meta.fields(MmaFlavor)) |field| { + const flavor: MmaFlavor = @enumFromInt(field.value); const atom = try fly.types.MmaAtomType.get(ctx, .{ .mmaOp = flavor.atomType(ctx) }); // `mma_atom.thr_layout` is the wavefront an atom occupies; the launch // geometry is derived from it, so a wrong table silently corrupts C. diff --git a/kernels/fly/dtype.zig b/kernels/fly/dtype.zig index ecc9f070..6962a898 100644 --- a/kernels/fly/dtype.zig +++ b/kernels/fly/dtype.zig @@ -65,7 +65,8 @@ pub const DType = enum { /// Inverse of `toMlir`. pub fn fromMlir(ctx: *mlir.Context, ty: *const mlir.Type) ?DType { - inline for (comptime std.enums.values(DType)) |dt| { + inline for (std.meta.fields(DType)) |f| { + const dt: DType = @enumFromInt(f.value); if (ty.eql(dt.toMlir(ctx))) return dt; } return null; @@ -76,7 +77,8 @@ test "dtype types round-trip" { defer registry.deinit(); const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); defer ctx.deinit(); - inline for (comptime std.enums.values(DType)) |dt| { + inline for (std.meta.fields(DType)) |f| { + const dt: DType = @enumFromInt(f.value); try std.testing.expectEqual(dt, fromMlir(ctx, dt.toMlir(ctx)).?); } try std.testing.expectEqual(DType.i8, DType.i1.storageElem()); diff --git a/kernels/fly/layout.zig b/kernels/fly/layout.zig index e70e0e00..85495ed4 100644 --- a/kernels/fly/layout.zig +++ b/kernels/fly/layout.zig @@ -189,8 +189,8 @@ fn itImpl(comptime v: anytype) IntTuple { .comptime_int, .int => return IntTuple.static(v), .@"struct" => |s| { if (!s.is_tuple) @compileError("fly.it: expected an integer, null, IntTuple or tuple, got " ++ @typeName(T)); - var elems: [s.field_names.len]IntTuple = undefined; - for (s.field_names, 0..) |f, i| elems[i] = itImpl(@field(v, f)); + var elems: [s.fields.len]IntTuple = undefined; + for (s.fields, 0..) |f, i| elems[i] = itImpl(@field(v, f.name)); const frozen = elems; return .{ .tup = &frozen }; }, @@ -413,8 +413,8 @@ fn tileImpl(comptime v: anytype) Tile { .comptime_int, .int => return .{ .leaf = .{ .s = v } }, .@"struct" => |s| { if (!s.is_tuple) @compileError("fly.tile: expected an integer, null, Layout, Tile or tuple, got " ++ @typeName(T)); - var elems: [s.field_names.len]Tile = undefined; - for (s.field_names, 0..) |f, i| elems[i] = tileImpl(@field(v, f)); + var elems: [s.fields.len]Tile = undefined; + for (s.fields, 0..) |f, i| elems[i] = tileImpl(@field(v, f.name)); const frozen = elems; return .{ .modes = &frozen }; }, diff --git a/kernels/mosaic_tpu/builder.zig b/kernels/mosaic_tpu/builder.zig index 77bc8f29..157648df 100644 --- a/kernels/mosaic_tpu/builder.zig +++ b/kernels/mosaic_tpu/builder.zig @@ -106,8 +106,8 @@ pub const Value = struct { /// True if the element type is a floating-point type (f16/bf16/f32/f64/fp8/…). pub fn isFloatElem(self: Value) bool { const et = self.elemType(); - inline for (comptime std.meta.fieldNames(mlir.FloatTypes)) |field_name| { - if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, field_name))) != null) return true; + inline for (std.meta.fields(mlir.FloatTypes)) |f| { + if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, f.name))) != null) return true; } return false; } @@ -600,13 +600,13 @@ pub const Builder = struct { opts: Opts, ) !dsl.NamedArgs(@TypeOf(spec), Value) { const Spec = @TypeOf(spec); - const field_names = @typeInfo(Spec).@"struct".field_names; + const fields = @typeInfo(Spec).@"struct".fields; - var arg_specs: [field_names.len]ArgSpec = undefined; - inline for (field_names, 0..) |field_name, i| { - const raw = @field(spec, field_name); + var arg_specs: [fields.len]ArgSpec = undefined; + inline for (fields, 0..) |f, i| { + const raw = @field(spec, f.name); const kind: ArgSpec.Kind = if (@TypeOf(raw) == ArgSpec.Kind) raw else blk: { - const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; + const variant = @typeInfo(@TypeOf(raw)).@"struct".fields[0].name; const tag = @field(std.meta.Tag(ArgSpec.Kind), variant); const inner = @field(raw, variant); break :blk switch (tag) { @@ -629,14 +629,14 @@ pub const Builder = struct { .sem => .{ .sem = inner }, }; }; - arg_specs[i] = .{ .name = field_name, .kind = kind }; + arg_specs[i] = .{ .name = f.name, .kind = kind }; } try self.declareArgsLowOpts(&arg_specs, result_types, opts); var named: dsl.NamedArgs(Spec, Value) = undefined; - inline for (field_names, 0..) |field_name, i| { - @field(named, field_name) = self.arg(i); + inline for (fields, 0..) |f, i| { + @field(named, f.name) = self.arg(i); } return named; } @@ -896,12 +896,12 @@ pub const Builder = struct { const info = @typeInfo(T); if (info != .@"struct" or !info.@"struct".is_tuple) @compileError("Builder.yield expects a tuple literal like `.{ v1, v2 }`"); - const n = info.@"struct".field_names.len; + const n = info.@"struct".fields.len; const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.yield OOM"); - inline for (info.@"struct".field_names, info.@"struct".field_types, 0..) |field_name, Field, i| { - if (Field != Value) - @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(Field)); - out[i] = @field(values, field_name); + inline for (info.@"struct".fields, 0..) |f, i| { + if (f.type != Value) + @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(f.type)); + out[i] = @field(values, f.name); } return out; } @@ -2048,7 +2048,7 @@ pub const Builder = struct { inits: anytype, ) dsl.ForScope(Builder, Value, dsl.tupleArity(@TypeOf(inits), "openFor: inits")) { const N = comptime dsl.tupleArity(@TypeOf(inits), "openFor: inits"); - const info = @typeInfo(@TypeOf(inits)).@"struct"; + const fields = @typeInfo(@TypeOf(inits)).@"struct".fields; // scf.for bounds default to `index`. const lb_v: Value = if (@TypeOf(lower) == Value) lower else self.cIndex(@intCast(lower)); @@ -2060,9 +2060,9 @@ pub const Builder = struct { block_types[0] = lb_v.type_(); block_locs[0] = self.loc(); var inits_inner: [N]*const mlir.Value = undefined; - inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { - const raw = @field(inits, field_name); - const v: Value = if (Field == Value) raw else self.lift(raw); + inline for (fields, 0..) |f, i| { + const raw = @field(inits, f.name); + const v: Value = if (f.type == Value) raw else self.lift(raw); block_types[i + 1] = v.type_(); block_locs[i + 1] = self.loc(); inits_inner[i] = v.inner; @@ -2102,13 +2102,13 @@ pub const Builder = struct { result_types: anytype, ) dsl.IfScope(Builder, Value, dsl.tupleArity(@TypeOf(result_types), "openIfElse: result_types")) { const N = comptime dsl.tupleArity(@TypeOf(result_types), "openIfElse: result_types"); - const info = @typeInfo(@TypeOf(result_types)).@"struct"; + const fields = @typeInfo(@TypeOf(result_types)).@"struct".fields; var types: [N]*const mlir.Type = undefined; - inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { - if (Field != *const mlir.Type) + inline for (fields, 0..) |f, i| { + if (f.type != *const mlir.Type) @compileError("openIfElse: every result_type must be *const mlir.Type (use k.scalarTy/vectorTy)"); - types[i] = @field(result_types, field_name); + types[i] = @field(result_types, f.name); } const then_block = mlir.Block.init(&.{}, &.{}); @@ -2136,16 +2136,16 @@ pub const Builder = struct { ) { const N = comptime dsl.tupleArity(@TypeOf(inits), "openWhile: inits"); const M = comptime dsl.tupleArity(@TypeOf(after_types), "openWhile: after_types"); - const init_info = @typeInfo(@TypeOf(inits)).@"struct"; - const type_info = @typeInfo(@TypeOf(after_types)).@"struct"; + const init_fields = @typeInfo(@TypeOf(inits)).@"struct".fields; + const type_fields = @typeInfo(@TypeOf(after_types)).@"struct".fields; var before_types: [N]*const mlir.Type = undefined; var before_locs: [N]*const mlir.Location = undefined; var inits_inner: [N]*const mlir.Value = undefined; - inline for (init_info.field_names, init_info.field_types, 0..) |field_name, Field, i| { - if (Field != Value) + inline for (init_fields, 0..) |f, i| { + if (f.type != Value) @compileError("openWhile: every init must be a Value"); - const v: Value = @field(inits, field_name); + const v: Value = @field(inits, f.name); before_types[i] = v.type_(); before_locs[i] = self.loc(); inits_inner[i] = v.inner; @@ -2154,10 +2154,10 @@ pub const Builder = struct { var after_tys: [M]*const mlir.Type = undefined; var after_locs: [M]*const mlir.Location = undefined; - inline for (type_info.field_names, type_info.field_types, 0..) |field_name, Field, i| { - if (Field != *const mlir.Type) + inline for (type_fields, 0..) |f, i| { + if (f.type != *const mlir.Type) @compileError("openWhile: every after_type must be *const mlir.Type"); - after_tys[i] = @field(after_types, field_name); + after_tys[i] = @field(after_types, f.name); after_locs[i] = self.loc(); } const after_block = mlir.Block.init(&after_tys, &after_locs); @@ -2223,7 +2223,7 @@ pub const Builder = struct { .debug_info = false, })}); - return try self.allocator.dupeSentinel(u8, al.written(), 0); + return try self.allocator.dupeZ(u8, al.written()); } /// Emit `transform_N` stub funcs — one per non-skipped i/o ref-arg. diff --git a/kernels/mosaic_tpu/dtype.zig b/kernels/mosaic_tpu/dtype.zig index 549c69da..d22370f6 100644 --- a/kernels/mosaic_tpu/dtype.zig +++ b/kernels/mosaic_tpu/dtype.zig @@ -60,8 +60,8 @@ pub fn intBitwidth(dt: DType) u32 { } pub fn mlirElemToDType(ctx: *mlir.Context, elem: *const mlir.Type) DType { - inline for (comptime std.meta.fieldNames(DType)) |field_name| { - const dt = @field(DType, field_name); + inline for (std.meta.fields(DType)) |f| { + const dt = @field(DType, f.name); if (elem.eql(dt.toMlir(ctx))) return dt; } @panic("element type not a recognized Mosaic-TPU DType"); diff --git a/kernels/triton/builder.zig b/kernels/triton/builder.zig index d585e8c1..702fba0d 100644 --- a/kernels/triton/builder.zig +++ b/kernels/triton/builder.zig @@ -66,8 +66,8 @@ pub const Value = struct { pub fn isFloatElem(self: Value) bool { const et = self.elemType(); - inline for (comptime std.meta.fieldNames(mlir.FloatTypes)) |field_name| { - if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, field_name))) != null) return true; + inline for (std.meta.fields(mlir.FloatTypes)) |f| { + if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, f.name))) != null) return true; } return false; } @@ -391,25 +391,25 @@ pub const ReturnIfScope = struct { pub fn yieldReturn(self: *ReturnIfScope, values: anytype) void { const k = self.kernel; - const info = @typeInfo(@TypeOf(values)).@"struct"; + const fields = @typeInfo(@TypeOf(values)).@"struct".fields; if (self.inline_return) { - var ret_operands: [info.field_names.len]*const mlir.Value = undefined; - inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { - if (Field != Value) + var ret_operands: [fields.len]*const mlir.Value = undefined; + inline for (fields, 0..) |f, i| { + if (f.type != Value) @compileError("ReturnIfScope.yieldReturn: every value must be a Value"); - ret_operands[i] = @field(values, field_name).inner; + ret_operands[i] = @field(values, f.name).inner; } _ = ttir.return_(k.ctx, &ret_operands, k.loc()).appendTo(self.ret_block); } else { if (k.exit_block == null) { k.exit_block = mlir.Block.init(&.{}, &.{}); - var ret_operands: [info.field_names.len]*const mlir.Value = undefined; - inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { - if (Field != Value) + var ret_operands: [fields.len]*const mlir.Value = undefined; + inline for (fields, 0..) |f, i| { + if (f.type != Value) @compileError("ReturnIfScope.yieldReturn: every value must be a Value"); - ret_operands[i] = @field(values, field_name).inner; + ret_operands[i] = @field(values, f.name).inner; } _ = ttir.return_(k.ctx, &ret_operands, k.loc()).appendTo(k.exit_block.?); } @@ -533,13 +533,13 @@ pub const Builder = struct { opts: Opts, ) !dsl.NamedArgs(@TypeOf(spec), Value) { const Spec = @TypeOf(spec); - const field_names = @typeInfo(Spec).@"struct".field_names; + const fields = @typeInfo(Spec).@"struct".fields; - var arg_specs: [field_names.len]ArgSpec = undefined; - inline for (field_names, 0..) |field_name, i| { - const raw = @field(spec, field_name); + var arg_specs: [fields.len]ArgSpec = undefined; + inline for (fields, 0..) |f, i| { + const raw = @field(spec, f.name); const kind: ArgSpec.Kind = if (@TypeOf(raw) == ArgSpec.Kind) raw else blk: { - const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; + const variant = @typeInfo(@TypeOf(raw)).@"struct".fields[0].name; const tag = @field(std.meta.Tag(ArgSpec.Kind), variant); const inner = @field(raw, variant); break :blk switch (tag) { @@ -557,14 +557,14 @@ pub const Builder = struct { } }, }; }; - arg_specs[i] = .{ .name = field_name, .kind = kind }; + arg_specs[i] = .{ .name = f.name, .kind = kind }; } try self.declareArgsLowOpts(&arg_specs, result_types, opts); var named: dsl.NamedArgs(Spec, Value) = undefined; - inline for (field_names, 0..) |field_name, i| { - @field(named, field_name) = self.arg(i); + inline for (fields, 0..) |f, i| { + @field(named, f.name) = self.arg(i); } return named; } @@ -679,12 +679,12 @@ pub const Builder = struct { const info = @typeInfo(T); if (info != .@"struct" or !info.@"struct".is_tuple) @compileError("Builder.yield expects a tuple literal like `.{ v1, v2 }`"); - const n = info.@"struct".field_names.len; + const n = info.@"struct".fields.len; const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.yield OOM"); - inline for (info.@"struct".field_names, info.@"struct".field_types, 0..) |field_name, Field, i| { - if (Field != Value) - @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(Field)); - out[i] = @field(values, field_name); + inline for (info.@"struct".fields, 0..) |f, i| { + if (f.type != Value) + @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(f.type)); + out[i] = @field(values, f.name); } return out; } @@ -898,8 +898,8 @@ pub const Builder = struct { } fn mlirElemToDType(self: *const Builder, elem: *const mlir.Type) DType { - inline for (comptime std.meta.fieldNames(DType)) |field_name| { - const dt = @field(DType, field_name); + inline for (std.meta.fields(DType)) |f| { + const dt = @field(DType, f.name); if (elem.eql(dt.toMlir(self.ctx))) return dt; } @panic("element type not a recognized DSL DType"); @@ -2012,7 +2012,7 @@ pub const Builder = struct { inits: anytype, ) ForScope(tupleArity(@TypeOf(inits), "openFor: inits")) { const N = comptime tupleArity(@TypeOf(inits), "openFor: inits"); - const info = @typeInfo(@TypeOf(inits)).@"struct"; + const fields = @typeInfo(@TypeOf(inits)).@"struct".fields; const lb_v: Value = if (@TypeOf(lower) == Value) lower else self.lift(lower); const ub_v: Value = if (@TypeOf(upper) == Value) upper else self.constMatching(upper, lb_v.type_()); @@ -2023,9 +2023,9 @@ pub const Builder = struct { block_types[0] = lb_v.type_(); block_locs[0] = self.loc(); var inits_inner: [N]*const mlir.Value = undefined; - inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { - const raw = @field(inits, field_name); - const v: Value = if (Field == Value) raw else self.lift(raw); + inline for (fields, 0..) |f, i| { + const raw = @field(inits, f.name); + const v: Value = if (f.type == Value) raw else self.lift(raw); block_types[i + 1] = v.type_(); block_locs[i + 1] = self.loc(); inits_inner[i] = v.inner; @@ -2058,12 +2058,12 @@ pub const Builder = struct { const region = current.parentRegion(); const cont_block = mlir.Block.init(&.{}, &.{}); - const info = @typeInfo(@TypeOf(values)).@"struct"; - var ret_operands: [info.field_names.len]*const mlir.Value = undefined; - inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { - if (Field != Value) + const fields = @typeInfo(@TypeOf(values)).@"struct".fields; + var ret_operands: [fields.len]*const mlir.Value = undefined; + inline for (fields, 0..) |f, i| { + if (f.type != Value) @compileError("returnIfOpts: every value must be a Value"); - ret_operands[i] = @field(values, field_name).inner; + ret_operands[i] = @field(values, f.name).inner; } const ret_target: *mlir.Block = if (opts.inline_return) blk: { @@ -2144,13 +2144,13 @@ pub const Builder = struct { result_types: anytype, ) IfScope(tupleArity(@TypeOf(result_types), "openIfElse: result_types")) { const N = comptime tupleArity(@TypeOf(result_types), "openIfElse: result_types"); - const info = @typeInfo(@TypeOf(result_types)).@"struct"; + const fields = @typeInfo(@TypeOf(result_types)).@"struct".fields; var types: [N]*const mlir.Type = undefined; - inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { - if (Field != *const mlir.Type) + inline for (fields, 0..) |f, i| { + if (f.type != *const mlir.Type) @compileError("openIfElse: every result_type must be *const mlir.Type (use k.scalarTy/tensorTy)"); - types[i] = @field(result_types, field_name); + types[i] = @field(result_types, f.name); } const then_block = mlir.Block.init(&.{}, &.{}); @@ -2176,16 +2176,16 @@ pub const Builder = struct { ) { const N = comptime tupleArity(@TypeOf(inits), "openWhile: inits"); const M = comptime tupleArity(@TypeOf(after_types), "openWhile: after_types"); - const init_info = @typeInfo(@TypeOf(inits)).@"struct"; - const type_info = @typeInfo(@TypeOf(after_types)).@"struct"; + const init_fields = @typeInfo(@TypeOf(inits)).@"struct".fields; + const type_fields = @typeInfo(@TypeOf(after_types)).@"struct".fields; var before_types: [N]*const mlir.Type = undefined; var before_locs: [N]*const mlir.Location = undefined; var inits_inner: [N]*const mlir.Value = undefined; - inline for (init_info.field_names, init_info.field_types, 0..) |field_name, Field, i| { - if (Field != Value) + inline for (init_fields, 0..) |f, i| { + if (f.type != Value) @compileError("openWhile: every init must be a Value"); - const v: Value = @field(inits, field_name); + const v: Value = @field(inits, f.name); before_types[i] = v.type_(); before_locs[i] = self.loc(); inits_inner[i] = v.inner; @@ -2194,10 +2194,10 @@ pub const Builder = struct { var after_tys: [M]*const mlir.Type = undefined; var after_locs: [M]*const mlir.Location = undefined; - inline for (type_info.field_names, type_info.field_types, 0..) |field_name, Field, i| { - if (Field != *const mlir.Type) + inline for (type_fields, 0..) |f, i| { + if (f.type != *const mlir.Type) @compileError("openWhile: every after_type must be *const mlir.Type"); - after_tys[i] = @field(after_types, field_name); + after_tys[i] = @field(after_types, f.name); after_locs[i] = self.loc(); } const after_block = mlir.Block.init(&after_tys, &after_locs); @@ -2249,6 +2249,6 @@ pub const Builder = struct { .print_name_loc_as_prefix = true, })}); - return try self.allocator.dupeSentinel(u8, al.written(), 0); + return try self.allocator.dupeZ(u8, al.written()); } }; diff --git a/kernels/triton/dtype.zig b/kernels/triton/dtype.zig index 45018e67..c8c1d623 100644 --- a/kernels/triton/dtype.zig +++ b/kernels/triton/dtype.zig @@ -62,8 +62,8 @@ pub fn intBitwidth(dt: DType) u32 { } pub fn mlirElemToDType(ctx: *mlir.Context, elem: *const mlir.Type) DType { - inline for (comptime std.meta.fieldNames(DType)) |field_name| { - const dt = @field(DType, field_name); + inline for (std.meta.fields(DType)) |f| { + const dt = @field(DType, f.name); if (elem.eql(dt.toMlir(ctx))) return dt; } @panic("element type not a recognized Triton DType"); diff --git a/mlir/dialects/arith.zig b/mlir/dialects/arith.zig index 23e75c10..14e2bf4b 100644 --- a/mlir/dialects/arith.zig +++ b/mlir/dialects/arith.zig @@ -314,7 +314,7 @@ pub fn convertf( ) *mlir.Operation { var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; if (opts.rounding) |rm| { - attrs.appendAssumeCapacity(.named(ctx, "roundingmode", .int(ctx, .i32, @backingInt(rm)))); + attrs.appendAssumeCapacity(.named(ctx, "roundingmode", .int(ctx, .i32, @intFromEnum(rm)))); } return mlir.Operation.make(ctx, "arith.convertf", .{ .operands = .{ .flat = &.{src} }, @@ -335,7 +335,7 @@ pub fn scaling_truncf( ) *mlir.Operation { var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; if (opts.rounding) |rm| { - attrs.appendAssumeCapacity(.named(ctx, "roundingmode", .int(ctx, .i32, @backingInt(rm)))); + attrs.appendAssumeCapacity(.named(ctx, "roundingmode", .int(ctx, .i32, @intFromEnum(rm)))); } return mlir.Operation.make(ctx, "arith.scaling_truncf", .{ .operands = .{ .flat = &.{ src, scale } }, @@ -360,7 +360,7 @@ pub fn cmpi( .operands = .{ .flat = &.{ lhs, rhs } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "predicate", .int(ctx, .i64, @backingInt(predicate))), + .named(ctx, "predicate", .int(ctx, .i64, @intFromEnum(predicate))), }, .location = location, }); @@ -377,7 +377,7 @@ pub fn cmpf( .operands = .{ .flat = &.{ lhs, rhs } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "predicate", .int(ctx, .i64, @backingInt(predicate))), + .named(ctx, "predicate", .int(ctx, .i64, @intFromEnum(predicate))), }, .location = location, }); diff --git a/mlir/dialects/cuda_tile/cuda_tile.zig b/mlir/dialects/cuda_tile/cuda_tile.zig index 09a314c0..ff86836a 100644 --- a/mlir/dialects/cuda_tile/cuda_tile.zig +++ b/mlir/dialects/cuda_tile/cuda_tile.zig @@ -433,14 +433,14 @@ pub const PtrLoadStoreHint = struct { fn writeHintFields(w: *std.Io.Writer, hint: anytype) !void { var first = true; - inline for (@typeInfo(@TypeOf(hint)).@"struct".field_names) |f_name| { - if (comptime std.mem.eql(u8, f_name, "arch")) continue; - if (@field(hint, f_name)) |v| { + inline for (@typeInfo(@TypeOf(hint)).@"struct".fields) |f| { + if (comptime std.mem.eql(u8, f.name, "arch")) continue; + if (@field(hint, f.name)) |v| { if (!first) try w.writeAll(", "); first = false; switch (@TypeOf(v)) { - bool => try w.print("{s} = {}", .{ f_name, v }), - else => try w.print("{s} = {d}", .{ f_name, v }), + bool => try w.print("{s} = {}", .{ f.name, v }), + else => try w.print("{s} = {d}", .{ f.name, v }), } } } diff --git a/mlir/dialects/cute_ir/cute_ir.zig b/mlir/dialects/cute_ir/cute_ir.zig index 91e242d5..3e9ff08f 100644 --- a/mlir/dialects/cute_ir/cute_ir.zig +++ b/mlir/dialects/cute_ir/cute_ir.zig @@ -634,14 +634,14 @@ pub const PtrType = opaque { bitlayout: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCutePtrTypeGet(ctx.ptr(), if (args.valueType) |v| v.ptr() else c.MlirType{ .ptr = null }, @backingInt(args.addressSpace), args.alignment, if (args.swizzle) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, if (args.bitlayout) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCutePtrTypeGet(ctx.ptr(), if (args.valueType) |v| v.ptr() else c.MlirType{ .ptr = null }, @intFromEnum(args.addressSpace), args.alignment, if (args.swizzle) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, if (args.bitlayout) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValueType(self: *const Self) ?*const mlir.Type { return @ptrCast(c.mlirCutePtrTypeGetValueType(self.ptr()).ptr); } pub fn getAddressSpace(self: *const Self) AddressSpace { - return @fromBackingInt(@intCast(c.mlirCutePtrTypeGetAddressSpace(self.ptr()))); + return @enumFromInt(c.mlirCutePtrTypeGetAddressSpace(self.ptr())); } pub fn getAlignment(self: *const Self) u64 { return c.mlirCutePtrTypeGetAlignment(self.ptr()); @@ -1218,11 +1218,11 @@ pub const ReductionOpAttr = opaque { value: ReductionOp, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteReductionOpAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteReductionOpAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) ReductionOp { - return @fromBackingInt(@intCast(c.mlirCuteReductionOpAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteReductionOpAttrGetValue(self.ptr())); } }; @@ -2850,11 +2850,11 @@ test { std.testing.refAllDecls(@This()); std.testing.refAllDecls(nvgpu); inline for (comptime std.meta.declarations(@This())) |decl| { - const T = @field(@This(), decl); + const T = @field(@This(), decl.name); if (@TypeOf(T) == type) std.testing.refAllDecls(T); } inline for (comptime std.meta.declarations(nvgpu)) |decl| { - const T = @field(nvgpu, decl); + const T = @field(nvgpu, decl.name); if (@TypeOf(T) == type) std.testing.refAllDecls(T); } } @@ -2976,14 +2976,13 @@ test "generated types and attributes rebuild from their getters" { // Every case of every enum attribute. inline for (.{ cute, nvgpu }) |namespace| { inline for (comptime std.meta.declarations(namespace)) |decl| { - const T = @field(namespace, decl); + const T = @field(namespace, decl.name); if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque" and @hasDecl(T, "InitArgs")) { - const fields = @typeInfo(T.InitArgs).@"struct".field_names; - if (fields.len == 1 and comptime std.mem.eql(u8, fields[0], "value")) { - inline for (@typeInfo(@FieldType(T.InitArgs, fields[0])).@"enum".field_names) |case| { - const case_value = @field(@FieldType(T.InitArgs, fields[0]), case); - const attr = try T.get(ctx, .{ .value = case_value }); - try std.testing.expectEqual(@backingInt(case_value), @backingInt(attr.getValue())); + const fields = @typeInfo(T.InitArgs).@"struct".fields; + if (fields.len == 1 and comptime std.mem.eql(u8, fields[0].name, "value")) { + inline for (@typeInfo(fields[0].type).@"enum".fields) |case| { + const attr = try T.get(ctx, .{ .value = @enumFromInt(case.value) }); + try std.testing.expectEqual(case.value, @intFromEnum(attr.getValue())); try std.testing.expect((try rebuild(T, ctx, attr)).eql(attr)); } } @@ -2996,14 +2995,14 @@ test "generated types and attributes rebuild from their getters" { fn rebuild(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T { var args: T.InitArgs = undefined; var elements: [8]*const mlir.Type = undefined; - inline for (@typeInfo(T.InitArgs).@"struct".field_names) |field| { - const name = [_]u8{comptime std.ascii.toUpper(field[0])} ++ field[1..]; - if (@FieldType(T.InitArgs, field) == []const *const mlir.Type) { + inline for (@typeInfo(T.InitArgs).@"struct".fields) |field| { + const name = [_]u8{comptime std.ascii.toUpper(field.name[0])} ++ field.name[1..]; + if (field.type == []const *const mlir.Type) { const count = @field(T, "getNum" ++ name)(value); for (elements[0..count], 0..) |*element, i| element.* = @field(T, "get" ++ name[0 .. name.len - 1])(value, i); - @field(args, field) = elements[0..count]; + @field(args, field.name) = elements[0..count]; } else { - @field(args, field) = @field(T, "get" ++ name)(value); + @field(args, field.name) = @field(T, "get" ++ name)(value); } } return T.get(ctx, args); diff --git a/mlir/dialects/cute_ir/cute_nvgpu.zig b/mlir/dialects/cute_ir/cute_nvgpu.zig index 652ff9e1..7d7ac63d 100644 --- a/mlir/dialects/cute_ir/cute_nvgpu.zig +++ b/mlir/dialects/cute_ir/cute_nvgpu.zig @@ -564,7 +564,7 @@ pub const CopyAtomG2RType = opaque { invariant: bool = false, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomG2RTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.l2PrefetchSize), @backingInt(args.l1CacheEvictPriority), @backingInt(args.loadCacheMode), @backingInt(args.sharedSpace), args.invariant); + const result = c.mlirCuteNVGPUCopyAtomG2RTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.memOrder), @intFromEnum(args.memScope), @intFromEnum(args.l2PrefetchSize), @intFromEnum(args.l1CacheEvictPriority), @intFromEnum(args.loadCacheMode), @intFromEnum(args.sharedSpace), args.invariant); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -574,22 +574,22 @@ pub const CopyAtomG2RType = opaque { return c.mlirCuteNVGPUCopyAtomG2RTypeGetCopyBits(self.ptr()); } pub fn getMemOrder(self: *const Self) MemOrderKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetMemOrder(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomG2RTypeGetMemOrder(self.ptr())); } pub fn getMemScope(self: *const Self) MemScopeKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetMemScope(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomG2RTypeGetMemScope(self.ptr())); } pub fn getL2PrefetchSize(self: *const Self) L2PrefetchSize { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetL2PrefetchSize(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomG2RTypeGetL2PrefetchSize(self.ptr())); } pub fn getL1CacheEvictPriority(self: *const Self) CacheEvictionPriority { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetL1CacheEvictPriority(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomG2RTypeGetL1CacheEvictPriority(self.ptr())); } pub fn getLoadCacheMode(self: *const Self) LoadCacheMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetLoadCacheMode(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomG2RTypeGetLoadCacheMode(self.ptr())); } pub fn getSharedSpace(self: *const Self) SharedSpace { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetSharedSpace(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomG2RTypeGetSharedSpace(self.ptr())); } pub fn getInvariant(self: *const Self) bool { return c.mlirCuteNVGPUCopyAtomG2RTypeGetInvariant(self.ptr()); @@ -705,7 +705,7 @@ pub const CopyAtomLdsmType = opaque { transpose: bool = false, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomLdsmTypeGet(ctx.ptr(), args.valType.ptr(), args.mode.ptr(), @backingInt(args.szPattern), args.numMatrices, args.transpose); + const result = c.mlirCuteNVGPUCopyAtomLdsmTypeGet(ctx.ptr(), args.valType.ptr(), args.mode.ptr(), @intFromEnum(args.szPattern), args.numMatrices, args.transpose); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -715,7 +715,7 @@ pub const CopyAtomLdsmType = opaque { return @ptrCast(c.mlirCuteNVGPUCopyAtomLdsmTypeGetMode(self.ptr()).ptr.?); } pub fn getSzPattern(self: *const Self) LdsmSzPattern { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomLdsmTypeGetSzPattern(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomLdsmTypeGetSzPattern(self.ptr())); } pub fn getNumMatrices(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomLdsmTypeGetNumMatrices(self.ptr()); @@ -748,11 +748,11 @@ pub const CopyAtomNonExec2DGather4TmaLoadType = opaque { tmaFormat: ?TmaDataFormat = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); + const result = c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGet(ctx.ptr(), @intFromEnum(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getKind(self: *const Self) GatherScatterTmaLoad { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetKind(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetKind(self.ptr())); } pub fn getValType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetValType(self.ptr()).ptr.?); @@ -765,7 +765,7 @@ pub const CopyAtomNonExec2DGather4TmaLoadType = opaque { } pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @fromBackingInt(@intCast(value)); + return if (value < 0) null else @enumFromInt(value); } }; @@ -791,7 +791,7 @@ pub const CopyAtomNonExec2DScatter4TmaStoreType = opaque { tmaFormat: ?TmaDataFormat = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); + const result = c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -805,7 +805,7 @@ pub const CopyAtomNonExec2DScatter4TmaStoreType = opaque { } pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @fromBackingInt(@intCast(value)); + return if (value < 0) null else @enumFromInt(value); } }; @@ -832,11 +832,11 @@ pub const CopyAtomNonExecIm2ColTmaLoadType = opaque { tmaFormat: ?TmaDataFormat = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); + const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGet(ctx.ptr(), @intFromEnum(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getKind(self: *const Self) Im2ColTmaLoad { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetKind(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetKind(self.ptr())); } pub fn getValType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetValType(self.ptr()).ptr.?); @@ -849,7 +849,7 @@ pub const CopyAtomNonExecIm2ColTmaLoadType = opaque { } pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @fromBackingInt(@intCast(value)); + return if (value < 0) null else @enumFromInt(value); } }; @@ -875,7 +875,7 @@ pub const CopyAtomNonExecIm2ColTmaStoreType = opaque { tmaFormat: ?TmaDataFormat = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); + const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -889,7 +889,7 @@ pub const CopyAtomNonExecIm2ColTmaStoreType = opaque { } pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @fromBackingInt(@intCast(value)); + return if (value < 0) null else @enumFromInt(value); } }; @@ -916,11 +916,11 @@ pub const CopyAtomNonExecTiledTmaLoadType = opaque { tmaFormat: ?TmaDataFormat = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); + const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGet(ctx.ptr(), @intFromEnum(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getKind(self: *const Self) TiledTmaLoad { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetKind(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetKind(self.ptr())); } pub fn getValType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetValType(self.ptr()).ptr.?); @@ -933,7 +933,7 @@ pub const CopyAtomNonExecTiledTmaLoadType = opaque { } pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @fromBackingInt(@intCast(value)); + return if (value < 0) null else @enumFromInt(value); } }; @@ -960,11 +960,11 @@ pub const CopyAtomNonExecTiledTmaReduceType = opaque { tmaFormat: ?TmaDataFormat = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); + const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGet(ctx.ptr(), @intFromEnum(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getKind(self: *const Self) ReductionKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetKind(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetKind(self.ptr())); } pub fn getValType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetValType(self.ptr()).ptr.?); @@ -977,7 +977,7 @@ pub const CopyAtomNonExecTiledTmaReduceType = opaque { } pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @fromBackingInt(@intCast(value)); + return if (value < 0) null else @enumFromInt(value); } }; @@ -1003,7 +1003,7 @@ pub const CopyAtomNonExecTiledTmaStoreType = opaque { tmaFormat: ?TmaDataFormat = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); + const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1017,7 +1017,7 @@ pub const CopyAtomNonExecTiledTmaStoreType = opaque { } pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @fromBackingInt(@intCast(value)); + return if (value < 0) null else @enumFromInt(value); } }; @@ -1046,7 +1046,7 @@ pub const CopyAtomR2GType = opaque { sharedSpace: SharedSpace = .CTA, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomR2GTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.l1CacheEvictPriority), @backingInt(args.storeCacheMode), @backingInt(args.sharedSpace)); + const result = c.mlirCuteNVGPUCopyAtomR2GTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.memOrder), @intFromEnum(args.memScope), @intFromEnum(args.l1CacheEvictPriority), @intFromEnum(args.storeCacheMode), @intFromEnum(args.sharedSpace)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1056,19 +1056,19 @@ pub const CopyAtomR2GType = opaque { return c.mlirCuteNVGPUCopyAtomR2GTypeGetCopyBits(self.ptr()); } pub fn getMemOrder(self: *const Self) MemOrderKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetMemOrder(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2GTypeGetMemOrder(self.ptr())); } pub fn getMemScope(self: *const Self) MemScopeKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetMemScope(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2GTypeGetMemScope(self.ptr())); } pub fn getL1CacheEvictPriority(self: *const Self) CacheEvictionPriority { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetL1CacheEvictPriority(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2GTypeGetL1CacheEvictPriority(self.ptr())); } pub fn getStoreCacheMode(self: *const Self) StoreCacheMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetStoreCacheMode(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2GTypeGetStoreCacheMode(self.ptr())); } pub fn getSharedSpace(self: *const Self) SharedSpace { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetSharedSpace(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2GTypeGetSharedSpace(self.ptr())); } }; @@ -1095,7 +1095,7 @@ pub const CopyAtomR2SType = opaque { sharedSpace: SharedSpace = .CTA, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomR2STypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.sharedSpace)); + const result = c.mlirCuteNVGPUCopyAtomR2STypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.memOrder), @intFromEnum(args.memScope), @intFromEnum(args.sharedSpace)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1105,13 +1105,13 @@ pub const CopyAtomR2SType = opaque { return c.mlirCuteNVGPUCopyAtomR2STypeGetCopyBits(self.ptr()); } pub fn getMemOrder(self: *const Self) MemOrderKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2STypeGetMemOrder(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2STypeGetMemOrder(self.ptr())); } pub fn getMemScope(self: *const Self) MemScopeKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2STypeGetMemScope(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2STypeGetMemScope(self.ptr())); } pub fn getSharedSpace(self: *const Self) SharedSpace { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2STypeGetSharedSpace(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2STypeGetSharedSpace(self.ptr())); } }; @@ -1138,7 +1138,7 @@ pub const CopyAtomS2RType = opaque { sharedSpace: SharedSpace = .CTA, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomS2RTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.sharedSpace)); + const result = c.mlirCuteNVGPUCopyAtomS2RTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.memOrder), @intFromEnum(args.memScope), @intFromEnum(args.sharedSpace)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1148,13 +1148,13 @@ pub const CopyAtomS2RType = opaque { return c.mlirCuteNVGPUCopyAtomS2RTypeGetCopyBits(self.ptr()); } pub fn getMemOrder(self: *const Self) MemOrderKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetMemOrder(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomS2RTypeGetMemOrder(self.ptr())); } pub fn getMemScope(self: *const Self) MemScopeKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetMemScope(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomS2RTypeGetMemScope(self.ptr())); } pub fn getSharedSpace(self: *const Self) SharedSpace { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetSharedSpace(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomS2RTypeGetSharedSpace(self.ptr())); } }; @@ -1179,14 +1179,14 @@ pub const CopyAtomSIMTAsyncCopyType = opaque { copyBits: c_int, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGet(ctx.ptr(), args.valType.ptr(), @backingInt(args.cache), args.copyBits); + const result = c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGet(ctx.ptr(), args.valType.ptr(), @intFromEnum(args.cache), args.copyBits); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetValType(self.ptr()).ptr.?); } pub fn getCache(self: *const Self) LoadCacheMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCache(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCache(self.ptr())); } pub fn getCopyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCopyBits(self.ptr()); @@ -1217,7 +1217,7 @@ pub const CopyAtomSIMTMultimemLdReduceType = opaque { memScope: MemScopeKind = .SYS, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.reduction), if (args.ldReduceAccPrecision) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, @backingInt(args.memOrder), @backingInt(args.memScope)); + const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.reduction), if (args.ldReduceAccPrecision) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, @intFromEnum(args.memOrder), @intFromEnum(args.memScope)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1227,16 +1227,16 @@ pub const CopyAtomSIMTMultimemLdReduceType = opaque { return c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetCopyBits(self.ptr()); } pub fn getReduction(self: *const Self) ReductionKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetReduction(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetReduction(self.ptr())); } pub fn getLdReduceAccPrecision(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetLdReduceAccPrecision(self.ptr()).ptr); } pub fn getMemOrder(self: *const Self) MemOrderKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemOrder(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemOrder(self.ptr())); } pub fn getMemScope(self: *const Self) MemScopeKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemScope(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemScope(self.ptr())); } }; @@ -1263,7 +1263,7 @@ pub const CopyAtomSIMTMultimemRedType = opaque { memScope: MemScopeKind = .SYS, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.reduction), @backingInt(args.memOrder), @backingInt(args.memScope)); + const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.reduction), @intFromEnum(args.memOrder), @intFromEnum(args.memScope)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1273,13 +1273,13 @@ pub const CopyAtomSIMTMultimemRedType = opaque { return c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetCopyBits(self.ptr()); } pub fn getReduction(self: *const Self) ReductionKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetReduction(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetReduction(self.ptr())); } pub fn getMemOrder(self: *const Self) MemOrderKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemOrder(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemOrder(self.ptr())); } pub fn getMemScope(self: *const Self) MemScopeKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemScope(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemScope(self.ptr())); } }; @@ -1305,7 +1305,7 @@ pub const CopyAtomSIMTMultimemStType = opaque { memScope: MemScopeKind = .SYS, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope)); + const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.memOrder), @intFromEnum(args.memScope)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1315,10 +1315,10 @@ pub const CopyAtomSIMTMultimemStType = opaque { return c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetCopyBits(self.ptr()); } pub fn getMemOrder(self: *const Self) MemOrderKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemOrder(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemOrder(self.ptr())); } pub fn getMemScope(self: *const Self) MemScopeKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemScope(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemScope(self.ptr())); } }; @@ -1344,7 +1344,7 @@ pub const CopyAtomSIMTSyncCopyType = opaque { dstSpace: cute.AddressSpace = .generic, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.srcSpace), @backingInt(args.dstSpace)); + const result = c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.srcSpace), @intFromEnum(args.dstSpace)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1354,10 +1354,10 @@ pub const CopyAtomSIMTSyncCopyType = opaque { return c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetCopyBits(self.ptr()); } pub fn getSrcSpace(self: *const Self) cute.AddressSpace { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetSrcSpace(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetSrcSpace(self.ptr())); } pub fn getDstSpace(self: *const Self) cute.AddressSpace { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetDstSpace(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetDstSpace(self.ptr())); } }; @@ -1384,7 +1384,7 @@ pub const CopyAtomSM100CopyS2TType = opaque { broadcast: CopyS2TBroadcast = .none, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numCta, @backingInt(args.broadcast)); + const result = c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numCta, @intFromEnum(args.broadcast)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1400,7 +1400,7 @@ pub const CopyAtomSM100CopyS2TType = opaque { return c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumCta(self.ptr()); } pub fn getBroadcast(self: *const Self) CopyS2TBroadcast { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetBroadcast(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetBroadcast(self.ptr())); } }; @@ -1428,7 +1428,7 @@ pub const CopyAtomSM100S2TCopyV2Type = opaque { broadcast: CopyS2TBroadcast = .none, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numCta, @backingInt(args.smemMajor), @backingInt(args.broadcast)); + const result = c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numCta, @intFromEnum(args.smemMajor), @intFromEnum(args.broadcast)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1444,10 +1444,10 @@ pub const CopyAtomSM100S2TCopyV2Type = opaque { return c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumCta(self.ptr()); } pub fn getSmemMajor(self: *const Self) cute.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetSmemMajor(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetSmemMajor(self.ptr())); } pub fn getBroadcast(self: *const Self) CopyS2TBroadcast { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetBroadcast(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetBroadcast(self.ptr())); } }; @@ -1562,7 +1562,7 @@ pub const CopyAtomSM107TmemLoadSPCompressType = opaque { nan: bool = false, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, @backingInt(args.redOp), args.red, args.nan); + const result = c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, @intFromEnum(args.redOp), args.red, args.nan); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1578,7 +1578,7 @@ pub const CopyAtomSM107TmemLoadSPCompressType = opaque { return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumRep(self.ptr()); } pub fn getRedOp(self: *const Self) TmemLoadRedOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRedOp(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRedOp(self.ptr())); } pub fn getRed(self: *const Self) bool { return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRed(self.ptr()); @@ -1613,7 +1613,7 @@ pub const CopyAtomSM10xTmemLoadRedType = opaque { halfSplitOff: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, @backingInt(args.redOp), args.nan, if (args.halfSplitOff) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, @intFromEnum(args.redOp), args.nan, if (args.halfSplitOff) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1629,7 +1629,7 @@ pub const CopyAtomSM10xTmemLoadRedType = opaque { return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumRep(self.ptr()); } pub fn getRedOp(self: *const Self) TmemLoadRedOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetRedOp(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetRedOp(self.ptr())); } pub fn getNan(self: *const Self) bool { return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNan(self.ptr()); @@ -1706,7 +1706,7 @@ pub const CopyAtomTmaLoadType = opaque { noFullyOobTile: bool = false, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomTmaLoadTypeGet(ctx.ptr(), args.valType.ptr(), args.sparsity, args.copyBits, @backingInt(args.mode), args.numCta, args.gStride.ptr(), args.mcast, if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }, args.override, args.noFullyOobTile); + const result = c.mlirCuteNVGPUCopyAtomTmaLoadTypeGet(ctx.ptr(), args.valType.ptr(), args.sparsity, args.copyBits, @intFromEnum(args.mode), args.numCta, args.gStride.ptr(), args.mcast, if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }, args.override, args.noFullyOobTile); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1719,7 +1719,7 @@ pub const CopyAtomTmaLoadType = opaque { return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetCopyBits(self.ptr()); } pub fn getMode(self: *const Self) TmaLoadMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetMode(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetMode(self.ptr())); } pub fn getNumCta(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetNumCta(self.ptr()); @@ -1765,7 +1765,7 @@ pub const CopyAtomTmaReduceType = opaque { tmaGbasis: ?*const mlir.Type = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomTmaReduceTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.mode), @backingInt(args.kind), args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }); + const result = c.mlirCuteNVGPUCopyAtomTmaReduceTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.mode), @intFromEnum(args.kind), args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1775,10 +1775,10 @@ pub const CopyAtomTmaReduceType = opaque { return c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetCopyBits(self.ptr()); } pub fn getMode(self: *const Self) TmaStoreMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetMode(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetMode(self.ptr())); } pub fn getKind(self: *const Self) ReductionKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetKind(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetKind(self.ptr())); } pub fn getGStride(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetGStride(self.ptr()).ptr.?); @@ -1814,7 +1814,7 @@ pub const CopyAtomTmaStoreType = opaque { noFullyOobTile: bool = false, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.sparsity, args.copyBits, @backingInt(args.mode), args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }, args.override, args.noFullyOobTile); + const result = c.mlirCuteNVGPUCopyAtomTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.sparsity, args.copyBits, @intFromEnum(args.mode), args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }, args.override, args.noFullyOobTile); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1827,7 +1827,7 @@ pub const CopyAtomTmaStoreType = opaque { return c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetCopyBits(self.ptr()); } pub fn getMode(self: *const Self) TmaStoreMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetMode(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetMode(self.ptr())); } pub fn getGStride(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetGStride(self.ptr()).ptr.?); @@ -1874,7 +1874,7 @@ pub const MmaAtomSM100UMMABlockScaledSparseType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.archPromote), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @intFromEnum(args.sparseMetadataFormat), @intFromEnum(args.aFragKind), args.vecSize, @intFromEnum(args.archPromote), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -1884,10 +1884,10 @@ pub const MmaAtomSM100UMMABlockScaledSparseType = opaque { return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetNumCta(self.ptr()); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAMajor(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAMajor(self.ptr())); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBMajor(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBMajor(self.ptr())); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAType(self.ptr()).ptr.?); @@ -1902,16 +1902,16 @@ pub const MmaAtomSM100UMMABlockScaledSparseType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSfType(self.ptr()).ptr.?); } pub fn getSparseMetadataFormat(self: *const Self) SparseMetadataFormat { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSparseMetadataFormat(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSparseMetadataFormat(self.ptr())); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAFragKind(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAFragKind(self.ptr())); } pub fn getVecSize(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetVecSize(self.ptr()); } pub fn getArchPromote(self: *const Self) Arch { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetArchPromote(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetArchPromote(self.ptr())); } pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetIntOverflow(self.ptr()).ptr); @@ -1948,7 +1948,7 @@ pub const MmaAtomSM100UMMABlockScaledType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.archPromote), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @intFromEnum(args.aFragKind), args.vecSize, @intFromEnum(args.archPromote), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -1958,10 +1958,10 @@ pub const MmaAtomSM100UMMABlockScaledType = opaque { return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetNumCta(self.ptr()); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAMajor(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAMajor(self.ptr())); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBMajor(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBMajor(self.ptr())); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAType(self.ptr()).ptr.?); @@ -1976,13 +1976,13 @@ pub const MmaAtomSM100UMMABlockScaledType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetSfType(self.ptr()).ptr.?); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAFragKind(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAFragKind(self.ptr())); } pub fn getVecSize(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetVecSize(self.ptr()); } pub fn getArchPromote(self: *const Self) Arch { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetArchPromote(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetArchPromote(self.ptr())); } pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetIntOverflow(self.ptr()).ptr); @@ -2019,7 +2019,7 @@ pub const MmaAtomSM100UMMASparseType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.eType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.cScaleExp, if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.eType.ptr(), @intFromEnum(args.sparseMetadataFormat), @intFromEnum(args.aFragKind), args.cScaleExp, if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -2029,10 +2029,10 @@ pub const MmaAtomSM100UMMASparseType = opaque { return c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetNumCta(self.ptr()); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAMajor(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAMajor(self.ptr())); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBMajor(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBMajor(self.ptr())); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAType(self.ptr()).ptr.?); @@ -2047,10 +2047,10 @@ pub const MmaAtomSM100UMMASparseType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetEType(self.ptr()).ptr.?); } pub fn getSparseMetadataFormat(self: *const Self) SparseMetadataFormat { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetSparseMetadataFormat(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetSparseMetadataFormat(self.ptr())); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAFragKind(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAFragKind(self.ptr())); } pub fn getCScaleExp(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCScaleExp(self.ptr()); @@ -2088,7 +2088,7 @@ pub const MmaAtomSM100UMMAType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM100UMMATypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.aFragKind), args.cScaleExp, if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM100UMMATypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @intFromEnum(args.aFragKind), args.cScaleExp, if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -2098,10 +2098,10 @@ pub const MmaAtomSM100UMMAType = opaque { return c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetNumCta(self.ptr()); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAMajor(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAMajor(self.ptr())); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetBMajor(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetBMajor(self.ptr())); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAType(self.ptr()).ptr.?); @@ -2113,7 +2113,7 @@ pub const MmaAtomSM100UMMAType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetCType(self.ptr()).ptr.?); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAFragKind(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAFragKind(self.ptr())); } pub fn getCScaleExp(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetCScaleExp(self.ptr()); @@ -2155,7 +2155,7 @@ pub const MmaAtomSM107UMMABlockScaledSparseType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @intFromEnum(args.sparseMetadataFormat), @intFromEnum(args.aFragKind), args.vecSize, @intFromEnum(args.aCollectorOp), @intFromEnum(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -2165,10 +2165,10 @@ pub const MmaAtomSM107UMMABlockScaledSparseType = opaque { return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetNumCta(self.ptr()); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAMajor(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAMajor(self.ptr())); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBMajor(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBMajor(self.ptr())); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAType(self.ptr()).ptr.?); @@ -2183,19 +2183,19 @@ pub const MmaAtomSM107UMMABlockScaledSparseType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSfType(self.ptr()).ptr.?); } pub fn getSparseMetadataFormat(self: *const Self) SparseMetadataFormat { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSparseMetadataFormat(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSparseMetadataFormat(self.ptr())); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAFragKind(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAFragKind(self.ptr())); } pub fn getVecSize(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetVecSize(self.ptr()); } pub fn getACollectorOp(self: *const Self) MmaCollectorOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetACollectorOp(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetACollectorOp(self.ptr())); } pub fn getBCollectorOp(self: *const Self) MmaCollectorOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBCollectorOp(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBCollectorOp(self.ptr())); } pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetIntOverflow(self.ptr()).ptr); @@ -2233,7 +2233,7 @@ pub const MmaAtomSM107UMMABlockScaledType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @intFromEnum(args.aFragKind), args.vecSize, @intFromEnum(args.aCollectorOp), @intFromEnum(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -2243,10 +2243,10 @@ pub const MmaAtomSM107UMMABlockScaledType = opaque { return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetNumCta(self.ptr()); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAMajor(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAMajor(self.ptr())); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBMajor(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBMajor(self.ptr())); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAType(self.ptr()).ptr.?); @@ -2261,16 +2261,16 @@ pub const MmaAtomSM107UMMABlockScaledType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetSfType(self.ptr()).ptr.?); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAFragKind(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAFragKind(self.ptr())); } pub fn getVecSize(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetVecSize(self.ptr()); } pub fn getACollectorOp(self: *const Self) MmaCollectorOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetACollectorOp(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetACollectorOp(self.ptr())); } pub fn getBCollectorOp(self: *const Self) MmaCollectorOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBCollectorOp(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBCollectorOp(self.ptr())); } pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetIntOverflow(self.ptr()).ptr); @@ -2309,7 +2309,7 @@ pub const MmaAtomSM107UMMASparseType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.eType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.cScaleExp, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.eType.ptr(), @intFromEnum(args.sparseMetadataFormat), @intFromEnum(args.aFragKind), args.cScaleExp, @intFromEnum(args.aCollectorOp), @intFromEnum(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -2319,10 +2319,10 @@ pub const MmaAtomSM107UMMASparseType = opaque { return c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetNumCta(self.ptr()); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAMajor(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAMajor(self.ptr())); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBMajor(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBMajor(self.ptr())); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAType(self.ptr()).ptr.?); @@ -2337,19 +2337,19 @@ pub const MmaAtomSM107UMMASparseType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetEType(self.ptr()).ptr.?); } pub fn getSparseMetadataFormat(self: *const Self) SparseMetadataFormat { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetSparseMetadataFormat(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetSparseMetadataFormat(self.ptr())); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAFragKind(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAFragKind(self.ptr())); } pub fn getCScaleExp(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCScaleExp(self.ptr()); } pub fn getACollectorOp(self: *const Self) MmaCollectorOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetACollectorOp(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetACollectorOp(self.ptr())); } pub fn getBCollectorOp(self: *const Self) MmaCollectorOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBCollectorOp(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBCollectorOp(self.ptr())); } pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetIntOverflow(self.ptr()).ptr); @@ -2386,7 +2386,7 @@ pub const MmaAtomSM107UMMAType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM107UMMATypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.aFragKind), args.cScaleExp, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM107UMMATypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @intFromEnum(args.aFragKind), args.cScaleExp, @intFromEnum(args.aCollectorOp), @intFromEnum(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -2396,10 +2396,10 @@ pub const MmaAtomSM107UMMAType = opaque { return c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetNumCta(self.ptr()); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAMajor(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAMajor(self.ptr())); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBMajor(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBMajor(self.ptr())); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAType(self.ptr()).ptr.?); @@ -2411,16 +2411,16 @@ pub const MmaAtomSM107UMMAType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetCType(self.ptr()).ptr.?); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAFragKind(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAFragKind(self.ptr())); } pub fn getCScaleExp(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetCScaleExp(self.ptr()); } pub fn getACollectorOp(self: *const Self) MmaCollectorOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetACollectorOp(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetACollectorOp(self.ptr())); } pub fn getBCollectorOp(self: *const Self) MmaCollectorOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBCollectorOp(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBCollectorOp(self.ptr())); } pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetIntOverflow(self.ptr()).ptr); @@ -2502,7 +2502,7 @@ pub const MmaAtomSM80SparseType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM80SparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.sparseMetadataFormat), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM80SparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @intFromEnum(args.sparseMetadataFormat), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -2518,7 +2518,7 @@ pub const MmaAtomSM80SparseType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetCType(self.ptr()).ptr.?); } pub fn getSparseMetadataFormat(self: *const Self) SparseMetadataFormat { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetSparseMetadataFormat(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetSparseMetadataFormat(self.ptr())); } pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetIntOverflow(self.ptr()).ptr); @@ -2639,17 +2639,17 @@ pub const MmaAtomSM90Type = opaque { bNeg: bool = false, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM90TypeGet(ctx.ptr(), args.shapeMnk.ptr(), @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.aFragKind), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, args.aNeg, args.bNeg); + const result = c.mlirCuteNVGPUMmaAtomSM90TypeGet(ctx.ptr(), args.shapeMnk.ptr(), @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @intFromEnum(args.aFragKind), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, args.aNeg, args.bNeg); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetShapeMnk(self.ptr()).ptr.?); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetAMajor(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM90TypeGetAMajor(self.ptr())); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetBMajor(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM90TypeGetBMajor(self.ptr())); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetAType(self.ptr()).ptr.?); @@ -2661,7 +2661,7 @@ pub const MmaAtomSM90Type = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetCType(self.ptr()).ptr.?); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetAFragKind(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM90TypeGetAFragKind(self.ptr())); } pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetIntOverflow(self.ptr()).ptr); @@ -2932,11 +2932,11 @@ pub const AtomCopyFieldBulkCopyG2SAttr = opaque { value: AtomCopyFieldBulkCopyG2S, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldBulkCopyG2S { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGetValue(self.ptr())); } }; @@ -2959,11 +2959,11 @@ pub const AtomCopyFieldBulkCopyS2GAttr = opaque { value: AtomCopyFieldBulkCopyS2G, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldBulkCopyS2G { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGetValue(self.ptr())); } }; @@ -2986,11 +2986,11 @@ pub const AtomCopyFieldBulkCopyS2SAttr = opaque { value: AtomCopyFieldBulkCopyS2S, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldBulkCopyS2S { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGetValue(self.ptr())); } }; @@ -3013,11 +3013,11 @@ pub const AtomCopyFieldDsmemStoreAttr = opaque { value: AtomCopyFieldDsmemStore, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldDsmemStore { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGetValue(self.ptr())); } }; @@ -3040,11 +3040,11 @@ pub const AtomCopyFieldLoadGlobalAttr = opaque { value: AtomCopyFieldLoadGlobal, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldLoadGlobal { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGetValue(self.ptr())); } }; @@ -3067,11 +3067,11 @@ pub const AtomCopyFieldNonExec2DGather4TmaLoadAttr = opaque { value: AtomCopyFieldNonExec2DGather4TmaLoad, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldNonExec2DGather4TmaLoad { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGetValue(self.ptr())); } }; @@ -3094,11 +3094,11 @@ pub const AtomCopyFieldNonExec2DScatter4TmaStoreAttr = opaque { value: AtomCopyFieldNonExec2DScatter4TmaStore, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldNonExec2DScatter4TmaStore { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGetValue(self.ptr())); } }; @@ -3121,11 +3121,11 @@ pub const AtomCopyFieldNonExecIm2ColTmaLoadAttr = opaque { value: AtomCopyFieldNonExecIm2ColTmaLoad, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldNonExecIm2ColTmaLoad { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGetValue(self.ptr())); } }; @@ -3148,11 +3148,11 @@ pub const AtomCopyFieldNonExecIm2ColTmaStoreAttr = opaque { value: AtomCopyFieldNonExecIm2ColTmaStore, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldNonExecIm2ColTmaStore { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGetValue(self.ptr())); } }; @@ -3175,11 +3175,11 @@ pub const AtomCopyFieldNonExecTiledTmaLoadAttr = opaque { value: AtomCopyFieldNonExecTiledTmaLoad, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldNonExecTiledTmaLoad { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGetValue(self.ptr())); } }; @@ -3202,11 +3202,11 @@ pub const AtomCopyFieldNonExecTiledTmaReduceAttr = opaque { value: AtomCopyFieldNonExecTiledTmaReduce, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldNonExecTiledTmaReduce { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGetValue(self.ptr())); } }; @@ -3229,11 +3229,11 @@ pub const AtomCopyFieldNonExecTiledTmaStoreAttr = opaque { value: AtomCopyFieldNonExecTiledTmaStore, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldNonExecTiledTmaStore { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGetValue(self.ptr())); } }; @@ -3256,11 +3256,11 @@ pub const AtomCopyFieldStoreGlobalAttr = opaque { value: AtomCopyFieldStoreGlobal, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldStoreGlobal { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGetValue(self.ptr())); } }; @@ -3283,11 +3283,11 @@ pub const AtomCopyFieldTmaLoadAttr = opaque { value: AtomCopyFieldTmaLoad, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldTmaLoad { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGetValue(self.ptr())); } }; @@ -3310,11 +3310,11 @@ pub const AtomCopyFieldTmaReduceAttr = opaque { value: AtomCopyFieldTmaReduce, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldTmaReduce { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGetValue(self.ptr())); } }; @@ -3337,11 +3337,11 @@ pub const AtomCopyFieldTmaStoreAttr = opaque { value: AtomCopyFieldTmaStore, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldTmaStore { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGetValue(self.ptr())); } }; @@ -3364,11 +3364,11 @@ pub const AtomMmaFieldSM100Attr = opaque { value: AtomMmaFieldSM100, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM100AttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUAtomMmaFieldSM100AttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomMmaFieldSM100 { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100AttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM100AttrGetValue(self.ptr())); } }; @@ -3391,11 +3391,11 @@ pub const AtomMmaFieldSM100BlockScaledAttr = opaque { value: AtomMmaFieldSM100BlockScaled, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomMmaFieldSM100BlockScaled { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGetValue(self.ptr())); } }; @@ -3418,11 +3418,11 @@ pub const AtomMmaFieldSM100BlockScaledSparseAttr = opaque { value: AtomMmaFieldSM100BlockScaledSparse, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomMmaFieldSM100BlockScaledSparse { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGetValue(self.ptr())); } }; @@ -3445,11 +3445,11 @@ pub const AtomMmaFieldSM100SparseAttr = opaque { value: AtomMmaFieldSM100Sparse, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomMmaFieldSM100Sparse { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGetValue(self.ptr())); } }; @@ -3472,11 +3472,11 @@ pub const AtomMmaFieldSM120BlockScaledAttr = opaque { value: AtomMmaFieldSM120BlockScaled, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomMmaFieldSM120BlockScaled { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGetValue(self.ptr())); } }; @@ -3499,11 +3499,11 @@ pub const AtomMmaFieldSM80SparseAttr = opaque { value: AtomMmaFieldSM80Sparse, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomMmaFieldSM80Sparse { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGetValue(self.ptr())); } }; @@ -3526,11 +3526,11 @@ pub const AtomMmaFieldSM90Attr = opaque { value: AtomMmaFieldSM90, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM90AttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUAtomMmaFieldSM90AttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomMmaFieldSM90 { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM90AttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM90AttrGetValue(self.ptr())); } }; @@ -3553,11 +3553,11 @@ pub const BinaryOpAttr = opaque { value: BinaryOp, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUBinaryOpAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUBinaryOpAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) BinaryOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUBinaryOpAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUBinaryOpAttrGetValue(self.ptr())); } }; @@ -3580,11 +3580,11 @@ pub const CopyS2TBroadcastAttr = opaque { value: CopyS2TBroadcast, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyS2TBroadcastAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUCopyS2TBroadcastAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) CopyS2TBroadcast { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyS2TBroadcastAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUCopyS2TBroadcastAttrGetValue(self.ptr())); } }; @@ -3607,11 +3607,11 @@ pub const GatherScatterTmaLoadAttr = opaque { value: GatherScatterTmaLoad, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUGatherScatterTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUGatherScatterTmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) GatherScatterTmaLoad { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUGatherScatterTmaLoadAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUGatherScatterTmaLoadAttrGetValue(self.ptr())); } }; @@ -3634,11 +3634,11 @@ pub const Im2ColTmaLoadAttr = opaque { value: Im2ColTmaLoad, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUIm2ColTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUIm2ColTmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) Im2ColTmaLoad { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUIm2ColTmaLoadAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUIm2ColTmaLoadAttrGetValue(self.ptr())); } }; @@ -3661,11 +3661,11 @@ pub const LdReduceAccPrecisionKindAttr = opaque { value: LdReduceAccPrecisionKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPULdReduceAccPrecisionKindAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPULdReduceAccPrecisionKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) LdReduceAccPrecisionKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPULdReduceAccPrecisionKindAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPULdReduceAccPrecisionKindAttrGetValue(self.ptr())); } }; @@ -3688,11 +3688,11 @@ pub const LdsmSzPatternAttr = opaque { value: LdsmSzPattern, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPULdsmSzPatternAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPULdsmSzPatternAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) LdsmSzPattern { - return @fromBackingInt(@intCast(c.mlirCuteNVGPULdsmSzPatternAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPULdsmSzPatternAttrGetValue(self.ptr())); } }; @@ -3715,11 +3715,11 @@ pub const LoadCacheModeAttr = opaque { value: LoadCacheMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPULoadCacheModeAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPULoadCacheModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) LoadCacheMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPULoadCacheModeAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPULoadCacheModeAttrGetValue(self.ptr())); } }; @@ -3742,11 +3742,11 @@ pub const MMAIntOverflowAttr = opaque { value: MMAIntOverflow, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMMAIntOverflowAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUMMAIntOverflowAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MMAIntOverflow { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMMAIntOverflowAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMMAIntOverflowAttrGetValue(self.ptr())); } }; @@ -3769,11 +3769,11 @@ pub const MajorModeAttr = opaque { value: cute.MajorMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMajorModeAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUMajorModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) cute.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMajorModeAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMajorModeAttrGetValue(self.ptr())); } }; @@ -3796,11 +3796,11 @@ pub const MmaCollectorOpAttr = opaque { value: MmaCollectorOp, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaCollectorOpAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUMmaCollectorOpAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MmaCollectorOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaCollectorOpAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaCollectorOpAttrGetValue(self.ptr())); } }; @@ -3823,11 +3823,11 @@ pub const MmaFragKindAttr = opaque { value: MmaFragKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaFragKindAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUMmaFragKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MmaFragKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaFragKindAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUMmaFragKindAttrGetValue(self.ptr())); } }; @@ -3873,11 +3873,11 @@ pub const ReductionKindAttr = opaque { value: ReductionKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUReductionKindAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUReductionKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) ReductionKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUReductionKindAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUReductionKindAttrGetValue(self.ptr())); } }; @@ -3902,14 +3902,14 @@ pub const RmemFrgAttr = opaque { deriveElemType: bool = false, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPURmemFrgAttrGet(ctx.ptr(), args.valueType.ptr(), @backingInt(args.operand), args.deriveElemType); + const result = c.mlirCuteNVGPURmemFrgAttrGet(ctx.ptr(), args.valueType.ptr(), @intFromEnum(args.operand), args.deriveElemType); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValueType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPURmemFrgAttrGetValueType(self.ptr()).ptr.?); } pub fn getOperand(self: *const Self) cute.MmaOperand { - return @fromBackingInt(@intCast(c.mlirCuteNVGPURmemFrgAttrGetOperand(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPURmemFrgAttrGetOperand(self.ptr())); } pub fn getDeriveElemType(self: *const Self) bool { return c.mlirCuteNVGPURmemFrgAttrGetDeriveElemType(self.ptr()); @@ -3935,11 +3935,11 @@ pub const SM100CircularSmemFrgAttr = opaque { majorMode: cute.MajorMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSM100CircularSmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); + const result = c.mlirCuteNVGPUSM100CircularSmemFrgAttrGet(ctx.ptr(), @intFromEnum(args.majorMode)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getMajorMode(self: *const Self) cute.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100CircularSmemFrgAttrGetMajorMode(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUSM100CircularSmemFrgAttrGetMajorMode(self.ptr())); } }; @@ -3962,11 +3962,11 @@ pub const SM100SmemFrgAttr = opaque { majorMode: cute.MajorMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSM100SmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); + const result = c.mlirCuteNVGPUSM100SmemFrgAttrGet(ctx.ptr(), @intFromEnum(args.majorMode)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getMajorMode(self: *const Self) cute.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100SmemFrgAttrGetMajorMode(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUSM100SmemFrgAttrGetMajorMode(self.ptr())); } }; @@ -4023,7 +4023,7 @@ pub const SM100TmemFrgAttr = opaque { tmemAllocMode: TmemAllocMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSM100TmemFrgAttrGet(ctx.ptr(), args.dataType.ptr(), args.storageType.ptr(), args.ctaGroup, @backingInt(args.tmemAllocMode)); + const result = c.mlirCuteNVGPUSM100TmemFrgAttrGet(ctx.ptr(), args.dataType.ptr(), args.storageType.ptr(), args.ctaGroup, @intFromEnum(args.tmemAllocMode)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getDataType(self: *const Self) *const mlir.Type { @@ -4036,7 +4036,7 @@ pub const SM100TmemFrgAttr = opaque { return c.mlirCuteNVGPUSM100TmemFrgAttrGetCtaGroup(self.ptr()); } pub fn getTmemAllocMode(self: *const Self) TmemAllocMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100TmemFrgAttrGetTmemAllocMode(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUSM100TmemFrgAttrGetTmemAllocMode(self.ptr())); } }; @@ -4063,7 +4063,7 @@ pub const SM100TmemSfFrgAttr = opaque { tmemAllocMode: TmemAllocMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSM100TmemSfFrgAttrGet(ctx.ptr(), args.sfType.ptr(), args.sfVecSize, args.ctaGroup, args.isSfa, @backingInt(args.tmemAllocMode)); + const result = c.mlirCuteNVGPUSM100TmemSfFrgAttrGet(ctx.ptr(), args.sfType.ptr(), args.sfVecSize, args.ctaGroup, args.isSfa, @intFromEnum(args.tmemAllocMode)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getSfType(self: *const Self) *const mlir.Type { @@ -4079,7 +4079,7 @@ pub const SM100TmemSfFrgAttr = opaque { return c.mlirCuteNVGPUSM100TmemSfFrgAttrGetIsSfa(self.ptr()); } pub fn getTmemAllocMode(self: *const Self) TmemAllocMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100TmemSfFrgAttrGetTmemAllocMode(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUSM100TmemSfFrgAttrGetTmemAllocMode(self.ptr())); } }; @@ -4102,11 +4102,11 @@ pub const SM107SmemFrgAttr = opaque { majorMode: cute.MajorMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSM107SmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); + const result = c.mlirCuteNVGPUSM107SmemFrgAttrGet(ctx.ptr(), @intFromEnum(args.majorMode)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getMajorMode(self: *const Self) cute.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM107SmemFrgAttrGetMajorMode(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUSM107SmemFrgAttrGetMajorMode(self.ptr())); } }; @@ -4129,11 +4129,11 @@ pub const SM90SmemFrgAttr = opaque { majorMode: cute.MajorMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSM90SmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); + const result = c.mlirCuteNVGPUSM90SmemFrgAttrGet(ctx.ptr(), @intFromEnum(args.majorMode)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getMajorMode(self: *const Self) cute.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM90SmemFrgAttrGetMajorMode(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUSM90SmemFrgAttrGetMajorMode(self.ptr())); } }; @@ -4156,11 +4156,11 @@ pub const TiledTmaLoadAttr = opaque { value: TiledTmaLoad, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTiledTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUTiledTmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TiledTmaLoad { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUTiledTmaLoadAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUTiledTmaLoadAttrGetValue(self.ptr())); } }; @@ -4183,11 +4183,11 @@ pub const TmaDataFormatAttr = opaque { value: TmaDataFormat, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTmaDataFormatAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUTmaDataFormatAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TmaDataFormat { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmaDataFormatAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUTmaDataFormatAttrGetValue(self.ptr())); } }; @@ -4210,11 +4210,11 @@ pub const TmaLoadModeAttr = opaque { value: TmaLoadMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTmaLoadModeAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUTmaLoadModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TmaLoadMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmaLoadModeAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUTmaLoadModeAttrGetValue(self.ptr())); } }; @@ -4237,11 +4237,11 @@ pub const TmaStoreModeAttr = opaque { value: TmaStoreMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTmaStoreModeAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUTmaStoreModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TmaStoreMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmaStoreModeAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUTmaStoreModeAttrGetValue(self.ptr())); } }; @@ -4264,11 +4264,11 @@ pub const TmemAllocModeAttr = opaque { value: TmemAllocMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTmemAllocModeAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUTmemAllocModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TmemAllocMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmemAllocModeAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUTmemAllocModeAttrGetValue(self.ptr())); } }; @@ -4291,11 +4291,11 @@ pub const TmemLoadRedOpAttr = opaque { value: TmemLoadRedOp, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTmemLoadRedOpAttrGet(ctx.ptr(), @backingInt(args.value)); + const result = c.mlirCuteNVGPUTmemLoadRedOpAttrGet(ctx.ptr(), @intFromEnum(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TmemLoadRedOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmemLoadRedOpAttrGetValue(self.ptr()))); + return @enumFromInt(c.mlirCuteNVGPUTmemLoadRedOpAttrGetValue(self.ptr())); } }; diff --git a/mlir/dialects/fly/attributes.zig b/mlir/dialects/fly/attributes.zig index a30336fc..eceb8b0f 100644 --- a/mlir/dialects/fly/attributes.zig +++ b/mlir/dialects/fly/attributes.zig @@ -206,7 +206,7 @@ pub const AddressSpaceAttr = opaque { }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirFlyAddressSpaceAttrGet(ctx.ptr(), @backingInt(args.addressSpace)); + const result = c.mlirFlyAddressSpaceAttrGet(ctx.ptr(), @intFromEnum(args.addressSpace)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } }; @@ -214,7 +214,7 @@ pub const AddressSpaceAttr = opaque { pub const MmaOperand = enum(i32) { a = 0, b = 1, c = 2, d = 3 }; pub fn mmaOperandAttr(ctx: *mlir.Context, operand: MmaOperand) mlir.Error!*const mlir.Attribute { - const result = c.mlirFlyMmaOperandAttrGet(ctx.ptr(), @backingInt(operand)); + const result = c.mlirFlyMmaOperandAttrGet(ctx.ptr(), @intFromEnum(operand)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } @@ -234,6 +234,6 @@ pub const GemmTraversalOrder = enum(i32) { }; pub fn gemmTraversalOrderAttr(ctx: *mlir.Context, order: GemmTraversalOrder) mlir.Error!*const mlir.Attribute { - const result = c.mlirFlyGemmTraversalOrderAttrGet(ctx.ptr(), @backingInt(order)); + const result = c.mlirFlyGemmTraversalOrderAttrGet(ctx.ptr(), @intFromEnum(order)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } diff --git a/mlir/dialects/fly/fly.zig b/mlir/dialects/fly/fly.zig index 63e24077..6083f308 100644 --- a/mlir/dialects/fly/fly.zig +++ b/mlir/dialects/fly/fly.zig @@ -252,7 +252,7 @@ test { inline for (.{ types, attributes, rocdl }) |module| { std.testing.refAllDecls(module); inline for (comptime std.meta.declarations(module)) |decl| { - const value = @field(module, decl); + const value = @field(module, decl.name); if (@TypeOf(value) == type and @typeInfo(value) == .@"opaque") std.testing.refAllDecls(value); } } diff --git a/mlir/dialects/llvm_dialect.zig b/mlir/dialects/llvm_dialect.zig index 8845ccbd..2f5d3091 100644 --- a/mlir/dialects/llvm_dialect.zig +++ b/mlir/dialects/llvm_dialect.zig @@ -27,7 +27,7 @@ pub const NVPTXAddressSpace = enum(u32) { /// clash with it. pub fn pointerType(ctx: *mlir.Context, space: anytype) *const mlir.Type { var buf: [32]u8 = undefined; - const text = std.fmt.bufPrint(&buf, "!llvm.ptr<{d}>", .{@backingInt(space)}) catch unreachable; + const text = std.fmt.bufPrint(&buf, "!llvm.ptr<{d}>", .{@intFromEnum(space)}) catch unreachable; return mlir.Type.parse(ctx, text) catch std.debug.panic("failed to parse LLVM pointer type '{s}'", .{text}); } diff --git a/mlir/dialects/memref.zig b/mlir/dialects/memref.zig index 338d1732..77acbc5d 100644 --- a/mlir/dialects/memref.zig +++ b/mlir/dialects/memref.zig @@ -216,7 +216,7 @@ pub fn atomic_rmw( .operands = .{ .flat = buf.constSlice() }, .results = .{ .flat = &.{value.type_()} }, .attributes = &.{ - .named(ctx, "kind", .int(ctx, .i64, @backingInt(kind))), + .named(ctx, "kind", .int(ctx, .i64, @intFromEnum(kind))), }, .location = location, }); diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index f9ea1e7e..efae3018 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -805,7 +805,7 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result const backend_config_api_version: ?CustomCallOpts.ApiVersion = if (opts.backend_config) |bc| bc else null; if (opts.api_version orelse backend_config_api_version) |v| { attrs.appendAssumeCapacity( - .named(ctx, "api_version", .int(ctx, .i32, @backingInt(v))), + .named(ctx, "api_version", .int(ctx, .i32, @intFromEnum(v))), ); } if (opts.backend_config) |backend_config| { @@ -1463,7 +1463,7 @@ pub fn channelHandle( return @ptrCast(c.stablehloChannelHandleGet( ctx.ptr(), handle, - @backingInt(channel_type), + @intFromEnum(channel_type), ).ptr); } diff --git a/mlir/dialects/ttir/ttir.zig b/mlir/dialects/ttir/ttir.zig index ee302f92..81d926dc 100644 --- a/mlir/dialects/ttir/ttir.zig +++ b/mlir/dialects/ttir/ttir.zig @@ -142,51 +142,51 @@ pub const DescriptorReduceKind = enum(i32) { }; pub fn programDim(ctx: *mlir.Context, v: ProgramIDDim) *const mlir.Attribute { - return @ptrCast(c.mlirTritonProgramDimGet(ctx.ptr(), @backingInt(v)).ptr); + return @ptrCast(c.mlirTritonProgramDimGet(ctx.ptr(), @intFromEnum(v)).ptr); } pub fn cacheModifier(ctx: *mlir.Context, v: CacheModifier) *const mlir.Attribute { - return @ptrCast(c.mlirTritonCacheModifierGet(ctx.ptr(), @backingInt(v)).ptr); + return @ptrCast(c.mlirTritonCacheModifierGet(ctx.ptr(), @intFromEnum(v)).ptr); } pub fn evictionPolicy(ctx: *mlir.Context, v: EvictionPolicy) *const mlir.Attribute { - return @ptrCast(c.mlirTritonEvictionPolicyGet(ctx.ptr(), @backingInt(v)).ptr); + return @ptrCast(c.mlirTritonEvictionPolicyGet(ctx.ptr(), @intFromEnum(v)).ptr); } pub fn paddingOption(ctx: *mlir.Context, v: PaddingOption) *const mlir.Attribute { - return @ptrCast(c.mlirTritonPaddingOptionGet(ctx.ptr(), @backingInt(v)).ptr); + return @ptrCast(c.mlirTritonPaddingOptionGet(ctx.ptr(), @intFromEnum(v)).ptr); } pub fn roundingMode(ctx: *mlir.Context, v: RoundingMode) *const mlir.Attribute { - return @ptrCast(c.mlirTritonRoundingModeGet(ctx.ptr(), @backingInt(v)).ptr); + return @ptrCast(c.mlirTritonRoundingModeGet(ctx.ptr(), @intFromEnum(v)).ptr); } pub fn inputPrecision(ctx: *mlir.Context, v: InputPrecision) *const mlir.Attribute { - return @ptrCast(c.mlirTritonInputPrecisionGet(ctx.ptr(), @backingInt(v)).ptr); + return @ptrCast(c.mlirTritonInputPrecisionGet(ctx.ptr(), @intFromEnum(v)).ptr); } pub fn rmwOp(ctx: *mlir.Context, v: RMWOp) *const mlir.Attribute { - return @ptrCast(c.mlirTritonRMWOpGet(ctx.ptr(), @backingInt(v)).ptr); + return @ptrCast(c.mlirTritonRMWOpGet(ctx.ptr(), @intFromEnum(v)).ptr); } pub fn memSemantic(ctx: *mlir.Context, v: MemSemantic) *const mlir.Attribute { - return @ptrCast(c.mlirTritonMemSemanticGet(ctx.ptr(), @backingInt(v)).ptr); + return @ptrCast(c.mlirTritonMemSemanticGet(ctx.ptr(), @intFromEnum(v)).ptr); } pub fn memSyncScope(ctx: *mlir.Context, v: MemSyncScope) *const mlir.Attribute { - return @ptrCast(c.mlirTritonMemSyncScopeGet(ctx.ptr(), @backingInt(v)).ptr); + return @ptrCast(c.mlirTritonMemSyncScopeGet(ctx.ptr(), @intFromEnum(v)).ptr); } pub fn propagateNan(ctx: *mlir.Context, v: PropagateNan) *const mlir.Attribute { - return @ptrCast(c.mlirTritonPropagateNanGet(ctx.ptr(), @backingInt(v)).ptr); + return @ptrCast(c.mlirTritonPropagateNanGet(ctx.ptr(), @intFromEnum(v)).ptr); } pub fn scaleDotElemType(ctx: *mlir.Context, v: ScaleDotElemType) *const mlir.Attribute { - return @ptrCast(c.mlirTritonScaleDotElemTypeGet(ctx.ptr(), @backingInt(v)).ptr); + return @ptrCast(c.mlirTritonScaleDotElemTypeGet(ctx.ptr(), @intFromEnum(v)).ptr); } pub fn descriptorReduceKind(ctx: *mlir.Context, v: DescriptorReduceKind) *const mlir.Attribute { - return @ptrCast(c.mlirTritonDescriptorReduceKindGet(ctx.ptr(), @backingInt(v)).ptr); + return @ptrCast(c.mlirTritonDescriptorReduceKindGet(ctx.ptr(), @intFromEnum(v)).ptr); } // ============================================================================= diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 83bec2f5..d759dbfa 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -88,16 +88,16 @@ pub fn registerFuncExtensions(registry: *DialectRegistry) void { } fn isPtrConst(comptime T: type) bool { - return @typeInfo(T).pointer.attrs.@"const"; + return @typeInfo(T).pointer.is_const; } fn CastedPtr(comptime T1: type, comptime T2: type) type { - return if (@typeInfo(T1).pointer.attrs.@"const") *const T2 else *T2; + return if (@typeInfo(T1).pointer.is_const) *const T2 else *T2; } pub fn Methods(comptime T: type, comptime NativeT: type) type { return struct { - pub const IsPtrConst = @typeInfo(@typeInfo(@FieldType(NativeT, "ptr")).optional.child).pointer.attrs.@"const"; + pub const IsPtrConst = @typeInfo(@typeInfo(@FieldType(NativeT, "ptr")).optional.child).pointer.is_const; pub const Ptr = if (IsPtrConst) *const T else *T; pub fn ptr(self: anytype) NativeT { @@ -1467,11 +1467,11 @@ pub const Operation = opaque { (struct { pub fn callback(op: c.MlirOperation, ctx_: ?*anyopaque) callconv(.c) c.MlirWalkResult { const inner_ctx_: *ContextType = @ptrCast(@alignCast(ctx_)); - return @backingInt(walkfn(inner_ctx_.ctx, @ptrCast(op.ptr))); + return @intFromEnum(walkfn(inner_ctx_.ctx, @ptrCast(op.ptr))); } }).callback, &inner_ctx, - @backingInt(order), + @intFromEnum(order), ); } diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index aa05bbc9..0c160321 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -11,7 +11,7 @@ const Stream = @import("pjrt.zig").Stream; const log = std.log.scoped(.pjrt); comptime { - if (@typeInfo(TypeId).@"struct".field_names.len != 1) @compileError("TypeId has changed"); + if (@typeInfo(TypeId).@"struct".fields.len != 1) @compileError("TypeId has changed"); } /// The signature of a generic custom call. @@ -450,7 +450,7 @@ pub const Error = opaque { pub fn create(api: *const Api, error_code: ErrorCode, message: []const u8) *Error { var ret: pjrt.meta.Struct(c.XLA_FFI_Error_Create_Args) = .{ .message = message.ptr, - .errc = @backingInt(error_code), + .errc = @intFromEnum(error_code), }; return fromInner(api.inner().XLA_FFI_Error_Create.?(@ptrCast(&ret)).?); } diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index b223188c..bd58386f 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -31,18 +31,18 @@ pub const meta = struct { } pub fn Struct(comptime T: type) type { - const info = @typeInfo(T).@"struct"; - var names: [info.field_names.len][]const u8 = undefined; - var types: [info.field_names.len]type = undefined; - var attributes: [info.field_names.len]std.builtin.Type.Struct.FieldAttributes = undefined; - for (info.field_names, info.field_types, &names, &types, &attributes) |field_name, Field, *name, *type_, *attr| { - name.* = field_name; - type_.* = Field; + const fields = std.meta.fields(T); + var names: [fields.len][]const u8 = undefined; + var types: [fields.len]type = undefined; + var attributes: [fields.len]std.builtin.Type.StructField.Attributes = undefined; + for (fields, &names, &types, &attributes) |field, *name, *type_, *attr| { + name.* = field.name; + type_.* = field.type; attr.* = .{ - .default_value_ptr = @ptrCast(if (std.mem.eql(u8, field_name, "struct_size")) + .default_value_ptr = @ptrCast(if (std.mem.eql(u8, field.name, "struct_size")) &structSize(T) else - &std.mem.zeroes(Field)), + &std.mem.zeroes(field.type)), }; } return @Struct( @@ -153,7 +153,7 @@ pub const Api = struct { const fti = @typeInfo(@FieldType(c.PJRT_Api, @tagName(func))); const fn_ptr = @typeInfo(fti.optional.child); const fn_type_info = @typeInfo(fn_ptr.pointer.child); - const arg_array_type_info = @typeInfo(fn_type_info.@"fn".param_types[0].?); + const arg_array_type_info = @typeInfo(fn_type_info.@"fn".params[0].type.?); return arg_array_type_info.pointer.child; } @@ -257,7 +257,7 @@ pub const Api = struct { pub fn stablehloCurrentVersion(self: *const Api) ?[]const u8 { const state = struct { var buf: [32]u8 = undefined; - var str: ?[]const u8 = null; + var str: ?[:0]const u8 = null; }; if (state.str) |str| { return str; @@ -265,7 +265,7 @@ pub const Api = struct { if (self.pluginAttribute("stablehlo_current_version")) |nv| { switch (nv.value()) { .int64list => |v| { - state.str = std.mem.print(&state.buf, "{d}.{d}.{d}", .{ v[0], v[1], v[2] }) catch unreachable; + state.str = std.fmt.bufPrintZ(&state.buf, "{d}.{d}.{d}", .{ v[0], v[1], v[2] }) catch unreachable; }, else => unreachable, } @@ -367,7 +367,7 @@ pub const Error = opaque { const ret = api.call(.PJRT_Error_GetCode, .{ .@"error" = self.inner(), }) catch unreachable; - return @fromBackingInt(@intCast(ret.code)); + return @enumFromInt(ret.code); } pub fn getMessage(self: *Error, api: *const Api) []const u8 { @@ -392,7 +392,7 @@ pub const ShapeSpec = extern struct { .inner = .{ .dims = @ptrCast(@constCast(dims_)), .num_dims = dims_.len, - .element_type = @backingInt(bt), + .element_type = @intFromEnum(bt), }, }; } @@ -402,7 +402,7 @@ pub const ShapeSpec = extern struct { } pub fn bufferType(self: ShapeSpec) BufferType { - return @fromBackingInt(@intCast(self.inner.element_type)); + return @enumFromInt(self.inner.element_type); } }; @@ -496,13 +496,13 @@ pub const Client = opaque { const ret = try api.call(.PJRT_Client_BufferFromHostBuffer, .{ .client = self.inner(), .data = @constCast(args.data), - .type = @backingInt(args.buffer_type), + .type = @intFromEnum(args.buffer_type), .dims = @ptrCast(@constCast(args.dims)), .num_dims = args.dims.len, .byte_strides = if (args.byte_strides) |bs| @ptrCast(@constCast(bs)) else null, .num_byte_strides = if (args.byte_strides) |bs| bs.len else 0, .device_layout = @ptrCast(@constCast(&args.layout.toCStruct())), - .host_buffer_semantics = @backingInt(args.host_buffer_semantics), + .host_buffer_semantics = @intFromEnum(args.host_buffer_semantics), .device = if (args.dst == .device) @ptrCast(@constCast(args.dst.device)) else null, .memory = if (args.dst == .memory) args.dst.memory.inner() else null, }); @@ -542,7 +542,7 @@ pub const Client = opaque { .device_buffer_ptr = @constCast(args.device_buffer_ptr), .dims = args.dims.ptr, .num_dims = args.dims.len, - .element_type = @backingInt(args.element_type), + .element_type = @intFromEnum(args.element_type), .layout = @ptrCast(@constCast(&layout)), .device = @ptrCast(@constCast(args.device)), .on_delete_callback = args.on_delete_callback, @@ -607,7 +607,7 @@ pub const Client = opaque { .struct_size = c.PJRT_Layouts_PJRT_Client_GetDefaultLayout_Args_STRUCT_SIZE, .extension_start = null, .client = self.inner(), - .type = @backingInt(element_type), + .type = @intFromEnum(element_type), .dims = dims.ptr, .num_dims = dims.len, .layout = null, @@ -668,7 +668,7 @@ pub const Client = opaque { .client = self.inner(), .shape_dims = args.dims.ptr, .shape_num_dims = args.dims.len, - .shape_element_type = @backingInt(args.element_type), + .shape_element_type = @intFromEnum(args.element_type), .shape_layout = @ptrCast(&layout), .device = if (args.dst == .device) @ptrCast(@constCast(args.dst.device)) else null, .memory = if (args.dst == .memory) args.dst.memory.inner() else null, @@ -806,7 +806,7 @@ pub const Device = opaque { const ret = try api.call(.PJRT_Device_MemoryStats, .{ .device = self.inner(), }); - return .fromCStruct(ret); + return .fromCStruct(@bitCast(ret)); } }; @@ -1285,7 +1285,7 @@ pub const Buffer = opaque { const ret = api.call(.PJRT_Buffer_ElementType, .{ .buffer = self.inner(), }) catch unreachable; - return @fromBackingInt(@intCast(ret.type)); + return @enumFromInt(ret.type); } pub fn dimensions(self: *const Buffer, api: *const Api) []const i64 { @@ -1615,7 +1615,7 @@ pub const NamedValue = extern struct { }; pub fn kind(self: NamedValue) Kind { - return @fromBackingInt(@intCast(self.inner.type)); + return @enumFromInt(self.inner.type); } pub fn name(self: NamedValue) []const u8 { @@ -1632,14 +1632,14 @@ pub const NamedValue = extern struct { }; } - pub fn init(comptime kind_: Kind, name_: []const u8, value_: @FieldType(Value, @tagName(kind_))) NamedValue { + pub fn init(comptime kind_: Kind, name_: []const u8, value_: std.meta.fieldInfo(Value, kind_).type) NamedValue { return .{ .inner = .{ .struct_size = c.PJRT_NamedValue_STRUCT_SIZE, .extension_start = null, .name = @ptrCast(name_), .name_size = name_.len, - .type = @backingInt(kind_), + .type = @intFromEnum(kind_), .unnamed_0 = switch (kind_) { .string => .{ .string_value = @ptrCast(@constCast(value_)) }, .int64 => .{ .int64_value = value_ }, diff --git a/platforms/cuda/compat_probe.zig b/platforms/cuda/compat_probe.zig index 3f890c6a..9e962577 100644 --- a/platforms/cuda/compat_probe.zig +++ b/platforms/cuda/compat_probe.zig @@ -22,7 +22,7 @@ pub fn main(init: std.process.Init) !u8 { const cuInit = libcuda.lookup(*const fn (c_uint) callconv(.c) c_int, "cuInit").?; const cuGetErrorString = libcuda.lookup(*const fn (c_int, *[*c]const u8) callconv(.c) c_int, "cuGetErrorString").?; - return @backingInt(switch (cuInit(0)) { + return @intFromEnum(switch (cuInit(0)) { CUDA_SUCCESS => ExitCode.Success, CUDA_ERROR_SYSTEM_DRIVER_MISMATCH => ExitCode.SystemDriverMismatch, CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE => ExitCode.CompatNotSupportedOnDevice, diff --git a/platforms/cuda/cuda.zig b/platforms/cuda/cuda.zig index 692ea226..562fdc56 100644 --- a/platforms/cuda/cuda.zig +++ b/platforms/cuda/cuda.zig @@ -47,7 +47,7 @@ pub fn needsCudaCompat(io: std.Io, sandbox_path: []const u8) !bool { log.err("Failed to run CUDA compatibility probe: {any}", .{err}); return err; }; - const result: compat_probe.ExitCode = @fromBackingInt(@intCast(res.exited)); + const result: compat_probe.ExitCode = @enumFromInt(res.exited); return switch (result) { .Success => true, @@ -69,7 +69,7 @@ fn hasNvidiaDevice(io: std.Io) bool { fn hasCudaPathInLDPath() bool { const ldLibraryPath = std.c.getenv("LD_LIBRARY_PATH") orelse return false; - return std.ascii.findIgnoreCase(std.mem.span(ldLibraryPath), nvidiaLibsPath) != null; + return std.ascii.indexOfIgnoreCase(std.mem.span(ldLibraryPath), nvidiaLibsPath) != null; } fn setupXlaGpuCudaDirFlag(allocator: std.mem.Allocator, sandbox: []const u8) !void { diff --git a/platforms/cuda/flashattn/flashattn.zig b/platforms/cuda/flashattn/flashattn.zig index 7d9c72b0..b4014c25 100644 --- a/platforms/cuda/flashattn/flashattn.zig +++ b/platforms/cuda/flashattn/flashattn.zig @@ -20,7 +20,7 @@ pub const Tensor = struct { .rank = @intCast(rank), .dims = undefined, .strides = undefined, - .dtype = @intCast(@backingInt(dtype)), + .dtype = @intCast(@intFromEnum(dtype)), }, }; @memcpy(ret.inner.dims[0..rank], dims); diff --git a/platforms/neuron/libzml_neuron.zig b/platforms/neuron/libzml_neuron.zig index 404d8cc0..968f1584 100644 --- a/platforms/neuron/libzml_neuron.zig +++ b/platforms/neuron/libzml_neuron.zig @@ -46,7 +46,7 @@ fn setupNeuronEnv(io: std.Io, sandbox_path: []const u8) !void { var root_comm_buf: [256]u8 = undefined; _ = setenv( "NEURON_RT_ROOT_COMM_ID", - try std.mem.printSentinel(&root_comm_buf, "127.0.0.1:{d}", .{try findFreeTcpPort(io)}, 0), + try std.fmt.bufPrintZ(&root_comm_buf, "127.0.0.1:{d}", .{try findFreeTcpPort(io)}), 1, ); _ = setenv( @@ -69,9 +69,9 @@ fn setupNeuronEnv(io: std.Io, sandbox_path: []const u8) !void { const old_path = if (std.c.getenv("PATH")) |path| std.mem.span(path) else ""; var new_path_buf: [std.posix.PATH_MAX:0]u8 = undefined; const new_path = if (old_path.len == 0) - try std.mem.printSentinel(&new_path_buf, "{s}", .{bin_path}, 0) + try std.fmt.bufPrintZ(&new_path_buf, "{s}", .{bin_path}) else - try std.mem.printSentinel(&new_path_buf, "{s}:{s}", .{ bin_path, old_path }, 0); + try std.fmt.bufPrintZ(&new_path_buf, "{s}:{s}", .{ bin_path, old_path }); if (setenv("PATH", new_path.ptr, 1) != 0) return error.SetEnvFailed; if (std.c.getenv("NEURON_CC_FLAGS") != null) return; @@ -81,7 +81,7 @@ fn setupNeuronEnv(io: std.Io, sandbox_path: []const u8) !void { var flags_buf: [512:0]u8 = undefined; const instance = try neuron.instance(); const target = @tagName(instance.compilerTarget()); - const flags = try std.mem.printSentinel( + const flags = try std.fmt.bufPrintZ( &flags_buf, "--target={s} " ++ "--optlevel=1 " ++ @@ -98,7 +98,6 @@ fn setupNeuronEnv(io: std.Io, sandbox_path: []const u8) !void { log_level, log_level, }, - 0, ); _ = setenv("NEURON_CC_FLAGS", flags.ptr, 0); diff --git a/platforms/neuron/neuron.zig b/platforms/neuron/neuron.zig index e0365fe9..039cc566 100644 --- a/platforms/neuron/neuron.zig +++ b/platforms/neuron/neuron.zig @@ -82,16 +82,16 @@ pub const Instance = struct { pub fn instance() !Instance { var info = std.mem.zeroes(c.struct_nrt_instance_info); - info.family = @backingInt(Family.unknown); - info.size = @backingInt(Size.unknown); + info.family = @intFromEnum(Family.unknown); + info.size = @intFromEnum(Size.unknown); if (c.nrt_get_instance_info(&info, @sizeOf(c.struct_nrt_instance_info)) != c.NRT_SUCCESS) { return error.Unavailable; } return .{ - .family = @fromBackingInt(@intCast(info.family)), - .size = @fromBackingInt(@intCast(info.size)), + .family = @enumFromInt(info.family), + .size = @enumFromInt(info.size), }; } diff --git a/platforms/neuron/python_launcher.zig b/platforms/neuron/python_launcher.zig index 075034ab..26ca9766 100644 --- a/platforms/neuron/python_launcher.zig +++ b/platforms/neuron/python_launcher.zig @@ -113,7 +113,7 @@ fn configureInterpreter(init: std.process.Init, config: *c.PyConfig) ![]const u8 const args = try init.minimal.args.toSlice(arena.allocator()); const c_args = try arena.allocator().alloc([*:0]const u8, args.len); for (args, 0..) |arg, i| { - c_args[i] = (try arena.allocator().dupeSentinel(u8, arg, 0)).ptr; + c_args[i] = (try arena.allocator().dupeZ(u8, arg)).ptr; } pyStatusCheck(c.PyConfig_SetBytesArgv(config, @intCast(c_args.len), @ptrCast(c_args.ptr))); @@ -125,7 +125,7 @@ fn configureInterpreter(init: std.process.Init, config: *c.PyConfig) ![]const u8 { var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const home = try std.mem.printSentinel(&buf, "{f}{d}.{d}", .{ + const home = try std.fmt.bufPrintZ(&buf, "{f}{d}.{d}", .{ std.Io.Dir.path.fmtJoin(&.{ self_exe_dir, "..", @@ -134,7 +134,7 @@ fn configureInterpreter(init: std.process.Init, config: *c.PyConfig) ![]const u8 }), c.PY_MAJOR_VERSION, c.PY_MINOR_VERSION, - }, 0); + }); pyStatusCheck(c.PyConfig_SetBytesString(config, &config.home, home)); pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(home))); } diff --git a/platforms/platforms.zig b/platforms/platforms.zig index f1320f3e..3f0c56fd 100644 --- a/platforms/platforms.zig +++ b/platforms/platforms.zig @@ -22,8 +22,8 @@ pub const Platform = enum { metal, pub fn selectFirstAcceleratorEnabled() error{Unavailable}!Platform { - inline for (@typeInfo(Platform).@"enum".field_names) |field| { - const target: Platform = @field(Platform, field); + inline for (@typeInfo(Platform).@"enum".fields) |field| { + const target: Platform = @enumFromInt(field.value); if (comptime (target != .cpu and target.isEnabled())) return target; } diff --git a/stdx/flags.zig b/stdx/flags.zig index 52a9e6a4..6ee06eaf 100644 --- a/stdx/flags.zig +++ b/stdx/flags.zig @@ -94,8 +94,7 @@ pub fn parseProcessArgs(init_: std.process.Init.Minimal, comptime CliArgs: type) fn parse_commands(args: *std.process.Args.Iterator, comptime Commands: type) Commands { comptime assert(@typeInfo(Commands) == .@"union"); - const field_names = std.meta.fieldNames(Commands); - comptime assert(field_names.len >= 2); + comptime assert(std.meta.fields(Commands).len >= 2); const first_arg = args.next() orelse fatal( "subcommand required, expected {s}", @@ -110,10 +109,10 @@ fn parse_commands(args: *std.process.Args.Iterator, comptime Commands: type) Com } } - inline for (field_names, std.meta.fieldTypes(Commands)) |field_name, FieldType| { - comptime assert(std.mem.indexOf(u8, field_name, "_") == null); - if (std.mem.eql(u8, first_arg, field_name)) { - return @unionInit(Commands, field_name, parse_flags(args, FieldType)); + inline for (comptime std.meta.fields(Commands)) |field| { + comptime assert(std.mem.indexOf(u8, field.name, "_") == null); + if (std.mem.eql(u8, first_arg, field.name)) { + return @unionInit(Commands, field.name, parse_flags(args, field.type)); } } fatal("unknown subcommand: '{s}'", .{first_arg}); @@ -129,61 +128,52 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { return {}; } - const flags_info = @typeInfo(Flags).@"struct"; + assert(@typeInfo(Flags) == .@"struct"); - const total_num_fields = flags_info.field_names.len; - comptime var field_names: [total_num_fields][]const u8 = undefined; - comptime var field_types: [total_num_fields]type = undefined; - comptime var field_defaults: [total_num_fields]?*const anyopaque = undefined; + comptime var fields: [std.meta.fields(Flags).len]std.builtin.Type.StructField = undefined; comptime var field_count = 0; - const positional_infos: ?std.lang.Type.Struct = if (@hasField(Flags, "positional")) - @typeInfo(@FieldType(Flags, "positional")).@"struct" - else - null; + comptime var positional_fields: []const std.builtin.Type.StructField = &.{}; - comptime for (flags_info.field_names, flags_info.field_types, flags_info.field_attrs) |field_name, FieldType, field_attr| { - if (std.mem.eql(u8, field_name, "positional")) { + comptime for (std.meta.fields(Flags)) |field| { + if (std.mem.eql(u8, field.name, "positional")) { + assert(@typeInfo(field.type) == .@"struct"); + positional_fields = std.meta.fields(field.type); var optional_tail = false; - for (positional_infos.?.field_types, positional_infos.?.field_attrs) |PosType, pos_attr| { - const default: ?PosType = pos_attr.defaultValue(PosType); - if (default == null) { + for (positional_fields) |positional_field| { + if (default_value(positional_field) == null) { if (optional_tail) @panic("optional positional arguments must be last"); } else { optional_tail = true; } - switch (@typeInfo(PosType)) { + switch (@typeInfo(positional_field.type)) { .optional => |optional| { // optional flags should have a default - assert(default != null); - assert(default.? == null); + assert(default_value(positional_field) != null); + assert(default_value(positional_field).? == null); assert_valid_value_type(optional.child); }, else => { - assert_valid_value_type(PosType); + assert_valid_value_type(positional_field.type); }, } } } else { - field_names[field_count] = field_name; - field_types[field_count] = FieldType; - field_defaults[field_count] = field_attr.default_value_ptr; + fields[field_count] = field; field_count += 1; - const default: ?FieldType = field_attr.defaultValue(FieldType); - - switch (@typeInfo(FieldType)) { + switch (@typeInfo(field.type)) { .bool => { // boolean flags should have a default - debug.assertComptime(default != null and default.? == false, "boolean flag --{s} should default to false", .{field_name}); + debug.assertComptime(default_value(field) != null and default_value(field).? == false, "boolean flag --{s} should default to false", .{field.name}); }, .optional => |optional| { // optional flags should have a default - debug.assertComptime(default != null and default.? == null, "optional flag --{s} should have a null default value", .{field_name}); + debug.assertComptime(default_value(field) != null and default_value(field).? == null, "optional flag --{s} should have a null default value", .{field.name}); assert_valid_value_type(optional.child); }, else => { - assert_valid_value_type(FieldType); + assert_valid_value_type(field.type); }, } } @@ -192,33 +182,29 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { var result: Flags = undefined; // Would use std.enums.EnumFieldStruct(Flags, u32, 0) here but Flags is a struct not an Enum. var counts = comptime blk: { - const count_field_types: [total_num_fields]type = @splat(u32); - const count_field_attrs: [total_num_fields]std.builtin.Type.Struct.FieldAttributes = @splat(.{ - .@"align" = @alignOf(u32), - .default_value_ptr = @ptrCast(&@as(u32, 0)), - }); - break :blk @Struct(.auto, null, flags_info.field_names, &count_field_types, &count_field_attrs){}; + const f = std.meta.fields(Flags); + var count_field_names: [fields.len][]const u8 = undefined; + var count_field_types: [fields.len]type = undefined; + var count_field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; + for (f, &count_field_names, &count_field_types, &count_field_attrs) |field, *field_name, *field_type, *field_attr| { + field_name.* = field.name; + field_type.* = u32; + field_attr.* = .{ + .@"align" = @alignOf(u32), + .default_value_ptr = @ptrCast(&@as(u32, 0)), + }; + } + break :blk @Struct(.auto, null, &count_field_names, &count_field_types, &count_field_attrs){}; }; // When parsing arguments, we must consider longer arguments first, such that `--foo-bar=92` is // not confused for a misspelled `--foo=92`. Using `std.sort` for comptime-only values does not // work, so open-code insertion sort, and comptime assert order during the actual parsing. comptime { - for ( - 0.., - field_names[0..field_count], - field_types[0..field_count], - field_defaults[0..field_count], - ) |i, *name_right, *type_right, *default_right| { - for ( - field_names[0..i], - field_types[0..i], - field_defaults[0..i], - ) |*name_left, *type_left, *default_left| { - if (name_left.len < name_right.len) { - std.mem.swap([]const u8, name_left, name_right); - std.mem.swap(type, type_left, type_right); - std.mem.swap(?*const anyopaque, default_left, default_right); + for (fields[0..field_count], 0..) |*field_right, i| { + for (fields[0..i]) |*field_left| { + if (field_left.name.len < field_right.name.len) { + std.mem.swap(std.builtin.Type.StructField, field_left, field_right); } } } @@ -235,29 +221,29 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { } comptime var field_len_prev = std.math.maxInt(usize); - inline for (field_names[0..field_count], field_types[0..field_count]) |field_name, FieldType| { - const flag = comptime flag_name(field_name); + inline for (fields[0..field_count]) |field| { + const flag = comptime flag_name(field); - comptime assert(field_len_prev >= field_name.len); - field_len_prev = field_name.len; + comptime assert(field_len_prev >= field.name.len); + field_len_prev = field.name.len; if (std.mem.startsWith(u8, arg, flag)) { if (parsed_positional) { fatal("unexpected trailing option: '{s}'", .{arg}); } - @field(counts, field_name) += 1; - const flag_value = parse_flag(FieldType, flag, arg); - @field(result, field_name) = flag_value; + @field(counts, field.name) += 1; + const flag_value = parse_flag(field.type, flag, arg); + @field(result, field.name) = flag_value; continue :next_arg; } } - if (positional_infos) |pos_infos| { + if (@hasField(Flags, "positional")) { counts.positional += 1; switch (counts.positional - 1) { - inline 0...pos_infos.field_names.len - 1 => |positional_index| { - const positional_field_name = pos_infos.field_names[positional_index]; - const flag = comptime flag_name_positional(positional_field_name); + inline 0...positional_fields.len - 1 => |positional_index| { + const positional_field = positional_fields[positional_index]; + const flag = comptime flag_name_positional(positional_field); if (arg.len == 0) fatal("{s}: empty argument", .{flag}); // Prevent ambiguity between a flag and positional argument value. We could add @@ -266,8 +252,8 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { if (arg[0] == '-') fatal("unexpected argument: '{s}'", .{arg}); parsed_positional = true; - @field(result.positional, positional_field_name) = - parse_value(pos_infos.field_types[positional_index], flag, arg); + @field(result.positional, positional_field.name) = + parse_value(positional_field.type, flag, arg); continue :next_arg; }, else => {}, // Fall-through to the unexpected argument error. @@ -277,15 +263,11 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { fatal("unexpected argument: '{s}'", .{arg}); } - inline for ( - field_names[0..field_count], - field_types[0..field_count], - field_defaults[0..field_count], - ) |field_name, FieldType, default_ptr| { - const flag = flag_name(field_name); - switch (@field(counts, field_name)) { - 0 => if (defaultValue(FieldType, default_ptr)) |default| { - @field(result, field_name) = default; + inline for (fields[0..field_count]) |field| { + const flag = flag_name(field); + switch (@field(counts, field.name)) { + 0 => if (default_value(field)) |default| { + @field(result, field.name) = default; } else { fatal("{s}: argument is required", .{flag}); }, @@ -294,19 +276,15 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { } } - if (positional_infos) |pos_infos| { - assert(counts.positional <= pos_infos.field_names.len); - inline for ( - 0.., - pos_infos.field_names, - pos_infos.field_types, - pos_infos.field_attrs, - ) |positional_index, name, PosFieldType, attr| { + if (@hasField(Flags, "positional")) { + assert(counts.positional <= positional_fields.len); + inline for (positional_fields, 0..) |positional_field, positional_index| { if (positional_index >= counts.positional) { - if (attr.defaultValue(PosFieldType)) |default| { - @field(result.positional, name) = default; + const flag = comptime flag_name_positional(positional_field); + if (default_value(positional_field)) |default| { + @field(result.positional, positional_field.name) = default; } else { - fatal("<{s}>: argument is required", .{name}); + fatal("{s}: argument is required", .{flag}); } } } @@ -321,8 +299,8 @@ fn assert_valid_value_type(comptime T: type) void { if (@typeInfo(T) == .@"enum") { const info = @typeInfo(T).@"enum"; - assert(info.mode == .exhaustive); - assert(info.field_names.len >= 2); + assert(info.is_exhaustive); + assert(info.fields.len >= 2); return; } @@ -471,7 +449,7 @@ pub const ByteSize = struct { else ByteUnit.bytes; - _ = std.math.mul(u64, amount, @backingInt(unit)) catch { + _ = std.math.mul(u64, amount, @intFromEnum(unit)) catch { return ByteSizeParseError.BytesOverflow; }; @@ -482,7 +460,7 @@ pub const ByteSize = struct { return std.math.mul( u64, size.value, - @backingInt(size.unit), + @intFromEnum(size.unit), ) catch unreachable; } @@ -564,7 +542,7 @@ fn parse_value_float(comptime T: type, flag: []const u8, value: [:0]const u8) T fn parse_value_enum(comptime E: type, flag: []const u8, value: [:0]const u8) E { assert((flag[0] == '-' and flag[1] == '-') or flag[0] == '<'); - comptime assert(@typeInfo(E).@"enum".mode == .exhaustive); + comptime assert(@typeInfo(E).@"enum".is_exhaustive); return std.meta.stringToEnum(E, value) orelse fatal( "{s}: expected one of {s}, but found '{s}'", @@ -574,50 +552,53 @@ fn parse_value_enum(comptime E: type, flag: []const u8, value: [:0]const u8) E { fn fields_to_comma_list(comptime E: type) []const u8 { comptime { - const field_names = std.meta.fieldNames(E); - const field_count = field_names.len; + const field_count = std.meta.fields(E).len; assert(field_count >= 2); var result: []const u8 = ""; - for (field_names, 0..) |field_name, field_index| { + for (std.meta.fields(E), 0..) |field, field_index| { const separator = switch (field_index) { 0 => "", else => ", ", field_count - 1 => if (field_count == 2) " or " else ", or ", }; - result = result ++ separator ++ "'" ++ field_name ++ "'"; + result = result ++ separator ++ "'" ++ field.name ++ "'"; } return result; } } -pub fn flag_name(comptime field_name: []const u8) []const u8 { +pub fn flag_name(comptime field: std.builtin.Type.StructField) []const u8 { // TODO(Zig): Cleanup when this is fixed after Zig 0.11. // Without comptime blk, the compiler thinks the result is a runtime slice returning a UAF. return comptime blk: { - assert(!std.mem.eql(u8, field_name, "positional")); + assert(!std.mem.eql(u8, field.name, "positional")); var result: []const u8 = "--"; var index = 0; - while (std.mem.indexOf(u8, field_name[index..], "_")) |i| { - result = result ++ field_name[index..][0..i] ++ "-"; + while (std.mem.indexOf(u8, field.name[index..], "_")) |i| { + result = result ++ field.name[index..][0..i] ++ "-"; index = index + i + 1; } - result = result ++ field_name[index..]; + result = result ++ field.name[index..]; break :blk result; }; } test flag_name { - try std.testing.expectEqualStrings(flag_name("statsd"), "--statsd"); + const field = @typeInfo(struct { statsd: bool }).@"struct".fields[0]; + try std.testing.expectEqualStrings(flag_name(field), "--statsd"); } -fn flag_name_positional(comptime field_name: []const u8) []const u8 { - comptime assert(std.mem.indexOf(u8, field_name, "_") == null); - return "<" ++ field_name ++ ">"; +fn flag_name_positional(comptime field: std.builtin.Type.StructField) []const u8 { + comptime assert(std.mem.indexOf(u8, field.name, "_") == null); + return "<" ++ field.name ++ ">"; } -pub fn defaultValue(comptime FieldType: type, default_value_ptr: ?*const anyopaque) ?FieldType { - const dp: *const FieldType = @ptrCast(@alignCast(default_value_ptr orelse return null)); - return dp.*; +/// This is essentially `field.default_value`, but with a useful type instead of `?*anyopaque`. +pub fn default_value(comptime field: std.builtin.Type.StructField) ?field.type { + return if (field.default_value_ptr) |default_opaque| + @as(*const field.type, @ptrCast(@alignCast(default_opaque))).* + else + null; } diff --git a/stdx/fmt.zig b/stdx/fmt.zig index a62d4f7d..33351d93 100644 --- a/stdx/fmt.zig +++ b/stdx/fmt.zig @@ -222,15 +222,15 @@ fn printValue( switch (@typeInfo(T)) { .@"struct" => |info| { try w.writeAll(".{ "); - inline for (info.field_names, 0..) |field_name, i| { + inline for (info.fields, 0..) |f, i| { if (i > 0) try w.writeAll(", "); if (!info.is_tuple) { try w.writeByte('.'); - try w.writeAll(field_name); + try w.writeAll(f.name); try w.writeAll(" = "); } - try printValue(w, options, @field(value, field_name), max_depth - 1); + try printValue(w, options, @field(value, f.name), max_depth - 1); } try w.writeAll(" }"); }, diff --git a/stdx/json.zig b/stdx/json.zig index c1f71a4a..d1143efc 100644 --- a/stdx/json.zig +++ b/stdx/json.zig @@ -42,22 +42,22 @@ pub fn UnionHelpers(comptime T: type) type { } pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!T { - inline for (comptime std.meta.fieldNames(T), comptime std.meta.fieldTypes(T)) |field_name, Field| { - switch (Field) { - bool => if (source == .bool) return @unionInit(T, field_name, source.bool), + inline for (std.meta.fields(T)) |field| { + switch (field.type) { + bool => if (source == .bool) return @unionInit(T, field.name, source.bool), []const u8 => switch (source) { .string, .number_string => |v| { const dupe = try allocator.dupe(u8, v); - return @unionInit(T, field_name, dupe); + return @unionInit(T, field.name, dupe); }, else => {}, }, - else => switch (@typeInfo(Field)) { - .int => if (source == .integer) return @unionInit(T, field_name, @intCast(source.integer)), - .float => if (source == .float) return @unionInit(T, field_name, @floatCast(source.float)), - .@"struct" => if (source == .object) return @unionInit(T, field_name, try std.json.innerParseFromValue(Field, allocator, source, options)), + else => switch (@typeInfo(field.type)) { + .int => if (source == .integer) return @unionInit(T, field.name, @intCast(source.integer)), + .float => if (source == .float) return @unionInit(T, field.name, @floatCast(source.float)), + .@"struct" => if (source == .object) return @unionInit(T, field.name, try std.json.innerParseFromValue(field.type, allocator, source, options)), inline else => switch (source) { - .number_string, .array => return @unionInit(T, field_name, try std.json.innerParseFromValue(Field, allocator, source, options)), + .number_string, .array => return @unionInit(T, field.name, try std.json.innerParseFromValue(field.type, allocator, source, options)), else => {}, }, }, @@ -130,9 +130,9 @@ pub fn TaggedUnionHelpers(comptime T: type, comptime tag_name: [:0]const u8) typ if (@typeInfo(T) != .@"union") { @compileError("TaggedUnion expects a union type, found " ++ @typeName(T)); } - for (std.meta.fieldNames(T), std.meta.fieldTypes(T)) |field_name, Field| { - if (@typeInfo(Field) != .@"struct") { - @compileError("TaggedUnion member '" ++ field_name ++ "' must be a struct, found " ++ @typeName(Field)); + for (std.meta.fields(T)) |field| { + if (@typeInfo(field.type) != .@"struct") { + @compileError("TaggedUnion member '" ++ field.name ++ "' must be a struct, found " ++ @typeName(field.type)); } } } @@ -152,13 +152,13 @@ pub fn TaggedUnionHelpers(comptime T: type, comptime tag_name: [:0]const u8) typ var o = source.object; const tag = (o.fetchSwapRemove(tag_name) orelse return error.MissingField).value; if (tag != .string) return error.LengthMismatch; - inline for (comptime std.meta.fieldNames(T), comptime std.meta.fieldTypes(T)) |field_name, Field| { - if (std.mem.eql(u8, field_name, tag.string)) { - const inner: Field = std.json.parseFromValueLeaky(Field, allocator, .{ .object = o }, options) catch |err| { - std.log.warn("failed to interpret {s} as a {s}: {}", .{ tag.string, @typeName(Field), err }); + inline for (std.meta.fields(T)) |field| { + if (std.mem.eql(u8, field.name, tag.string)) { + const inner: field.type = std.json.parseFromValueLeaky(field.type, allocator, .{ .object = o }, options) catch |err| { + std.log.warn("failed to interpret {s} as a {s}: {}", .{ tag.string, @typeName(field.type), err }); return err; }; - return @unionInit(T, field_name, inner); + return @unionInit(T, field.name, inner); } } return error.InvalidEnumTag; @@ -172,9 +172,9 @@ pub fn TaggedUnionHelpers(comptime T: type, comptime tag_name: [:0]const u8) typ try jw.objectField(tag_name); try jw.write(@tagName(tag)); switch (@typeInfo(@TypeOf(v))) { - .@"struct" => inline for (comptime std.meta.fieldNames(@TypeOf(v))) |name| { - try jw.objectField(name); - try jw.write(@field(v, name)); + .@"struct" => inline for (std.meta.fields(@TypeOf(v))) |field| { + try jw.objectField(field.name); + try jw.write(@field(v, field.name)); }, else => unreachable, // Would have failed with comptime check. } @@ -230,11 +230,11 @@ pub fn NeverNull(comptime T: type, comptime default_value: T) type { } pub fn fillDefaultStructValues(comptime T: type, r: *T) !void { - const info = @typeInfo(T).@"struct"; - inline for (info.field_names, info.field_types, info.field_attrs) |field_name, Field, attrs| { - if (attrs.defaultValue(Field)) |default| { - if (@field(r, field_name) == null) { - @field(r, field_name) = default; + inline for (@typeInfo(T).Struct.fields) |field| { + if (field.default_value) |default_ptr| { + if (@field(r, field.name) == null) { + const default = @as(*align(1) const field.type, @ptrCast(default_ptr)).*; + @field(r, field.name) = default; } } } diff --git a/stdx/meta.zig b/stdx/meta.zig index 9b1c05af..cb9aaeba 100644 --- a/stdx/meta.zig +++ b/stdx/meta.zig @@ -20,8 +20,8 @@ pub fn isTuple(comptime T: type) bool { pub fn isStructOf(comptime T: type, comptime Elem: type) bool { return switch (@typeInfo(T)) { .@"struct" => |info| blk: { - inline for (info.field_types) |FieldType| { - if (FieldType != Elem) { + inline for (info.fields) |field| { + if (field.type != Elem) { break :blk false; } } @@ -34,8 +34,8 @@ pub fn isStructOf(comptime T: type, comptime Elem: type) bool { pub fn isStructOfAny(comptime T: type, comptime f: fn (comptime type) bool) bool { return switch (@typeInfo(T)) { .@"struct" => |info| blk: { - inline for (info.field_types) |FieldType| { - if (f(FieldType) == false) { + inline for (info.fields) |field| { + if (f(field.type) == false) { break :blk false; } } @@ -121,9 +121,9 @@ pub fn asSlice(comptime T: type) type { } pub fn TupleRange(comptime T: type, comptime start: ?usize, comptime end: ?usize) type { - const all_field_types = std.meta.fieldTypes(T); + const fields = std.meta.fields(T); const start_ = start orelse 0; - const end_ = end orelse all_field_types.len; + const end_ = end orelse fields.len; if (start_ == end_) { return @Tuple(&.{}); @@ -131,17 +131,17 @@ pub fn TupleRange(comptime T: type, comptime start: ?usize, comptime end: ?usize var field_types: [end_ - start_]type = undefined; inline for (start_..end_, 0..) |i, j| { - field_types[j] = all_field_types[i]; + field_types[j] = fields[i].type; } return @Tuple(&field_types); } pub fn FnParam(comptime func: anytype, comptime n: comptime_int) type { - const params = @typeInfo(@TypeOf(func)).@"fn".param_types; + const params = @typeInfo(@TypeOf(func)).@"fn".params; if (n >= params.len) { @compileError("param doesn't exist in func"); } - return params[n] orelse @compileError("anytype is not supported"); + return params[n].type orelse @compileError("anytype is not supported"); } pub fn FnReturn(comptime func: anytype) type { @@ -167,8 +167,8 @@ pub fn FnReturnErrorSet(comptime func: anytype) ?type { pub fn Head(comptime Tuple: type) type { return switch (@typeInfo(Tuple)) { .@"struct" => |struct_info| { - if (struct_info.field_types.len == 0) @compileError("Can't tail empty tuple"); - return struct_info.field_types[0]; + if (struct_info.fields.len == 0) @compileError("Can't tail empty tuple"); + return struct_info.fields[0].type; }, else => @compileError("Head works on tuple type"), }; @@ -177,10 +177,10 @@ pub fn Head(comptime Tuple: type) type { pub fn Tail(comptime Tuple: type) type { return switch (@typeInfo(Tuple)) { .@"struct" => |struct_info| { - if (struct_info.field_types.len == 0) @compileError("Can't tail empty tuple"); - var types: [struct_info.field_types.len - 1]type = undefined; - for (struct_info.field_types[1..], &types) |FieldType, *type_| { - type_.* = FieldType; + if (struct_info.fields.len == 0) @compileError("Can't tail empty tuple"); + var types: [struct_info.fields.len - 1]type = undefined; + for (struct_info.fields[1..], &types) |field, *type_| { + type_.* = field.type; } return @Tuple(&types); }, diff --git a/stdx/segmented_list.zig b/stdx/segmented_list.zig index 567534be..ee115ce2 100644 --- a/stdx/segmented_list.zig +++ b/stdx/segmented_list.zig @@ -107,7 +107,7 @@ pub fn SegmentedList(comptime T: type, comptime prealloc_item_count: usize) type pub const prealloc_count = prealloc_item_count; fn AtType(comptime SelfType: type) type { - if (@typeInfo(SelfType).pointer.attrs.@"const") { + if (@typeInfo(SelfType).pointer.is_const) { return *const T; } else { return *T; diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 1b19e638..8d1fb590 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -28,7 +28,7 @@ pub inline fn stackSlice(comptime max_len: usize, T: type, len: usize) []T { return storage[0..len]; } -pub const noalloc: std.mem.Allocator = if (builtin.mode == .fast) undefined else std.testing.failing_allocator; +pub const noalloc: std.mem.Allocator = if (builtin.mode == .ReleaseFast) undefined else std.testing.failing_allocator; pub fn arenaWithCapacity(parent: std.mem.Allocator, initial_capacity: usize) std.mem.Allocator.Error!std.heap.ArenaAllocator { var a: std.heap.ArenaAllocator = .init(parent); @@ -43,7 +43,7 @@ pub fn pinToCore(core_id: usize) void { if (builtin.os.tag == .linux) { const CPUSet = std.bit_set.ArrayBitSet(usize, std.os.linux.CPU_SETSIZE * @sizeOf(usize)); - var set: CPUSet = .empty; + var set: CPUSet = .initEmpty(); set.set(core_id); std.os.linux.sched_setaffinity(0, @ptrCast(&set.masks)) catch {}; } @@ -86,60 +86,3 @@ pub fn Once(comptime f: anytype) type { } }; } - -/// Packed storage of sub-byte values -/// -/// Note: before Zig 0.17, @Vector(u2, 4) was always one byte and could be used as a storage type, -/// But that's not true anymore. -pub fn Packed(T: type) type { - return switch (@bitSizeOf(T)) { - 2 => packed struct(u8) { - x: T, - y: T, - z: T, - w: T, - - const P = @This(); - pub fn pack(x: T, y: T, z: T, w: T) P { - return .{ .x = x, .y = y, .z = z, .w = w }; - } - - pub fn unpack(p: P) [4]T { - return .{ p.x, p.y, p.z, p.w }; - } - - pub fn formatNumber(p: P, w: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { - try formatPackedNumber(T, &p.unpack(), w, n); - } - }, - 4 => packed struct(u8) { - x: T, - y: T, - - const P = @This(); - pub fn pack(x: T, y: T) P { - return .{ .x = x, .y = y }; - } - - pub fn unpack(p: P) [2]T { - return .{ p.x, p.y }; - } - - pub fn formatNumber(p: P, w: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { - try formatPackedNumber(T, &p.unpack(), w, n); - } - }, - else => debug.compileError("stdx.Packed expects a bitsize of 2 or 4, got type {} with bitsize {d}", .{ T, @bitSizeOf(T) }), - }; -} - -fn formatPackedNumber(T: type, values: []const T, w: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { - for (0.., values) |i, val| { - if (i > 0) try w.writeByte(','); - switch (@typeInfo(T)) { - .@"struct" => try val.formatNumber(w, n), - .int => try fmt.formatInt(val, n, w), - else => try w.print("{}", .{val}), - } - } -} diff --git a/third_party/arocc/arocc.bazel b/third_party/arocc/arocc.bazel index a5de1d98..853399f5 100644 --- a/third_party/arocc/arocc.bazel +++ b/third_party/arocc/arocc.bazel @@ -80,10 +80,19 @@ zig_options( ], ) +zig_library( + name = "zig", + srcs = glob(["deps/zig/**/*.zig"]), + main = "deps/zig/lib.zig", +) + zig_library( name = "backend", main = "src/backend.zig", - deps = [":build_options"], + deps = [ + ":build_options", + ":zig", + ], ) zig_library( @@ -118,6 +127,7 @@ zig_binary( ":assembly_backend", ":build_options", ], + zigopts = ["-fm"], ) zig_binary( diff --git a/third_party/arocc/repo.bzl b/third_party/arocc/repo.bzl index 15ff6d7f..98a10e38 100644 --- a/third_party/arocc/repo.bzl +++ b/third_party/arocc/repo.bzl @@ -1,10 +1,9 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") def repo(): - # https://github.com/zml/arocc/releases/tag/0.17.0 git_repository( name = "arocc", - remote = "https://github.com/zml/arocc", - commit = "d0c8c4d9c55daa7ef6e40cf0f630a5b5e900989b", + remote = "https://github.com/Vexu/arocc.git", + commit = "5f5a050569a95ecc40a426f0c3666ae7ef987ede", build_file = Label("//third_party/arocc:arocc.bazel"), ) diff --git a/third_party/libvaxis/bump-zig.patch b/third_party/libvaxis/bump-zig.patch new file mode 100644 index 00000000..45e221b7 --- /dev/null +++ b/third_party/libvaxis/bump-zig.patch @@ -0,0 +1,2019 @@ +diff --git a/build.zig.zon b/build.zig.zon +index 6192c1a..4ca04f8 100644 +--- a/build.zig.zon ++++ b/build.zig.zon +@@ -2,15 +2,15 @@ + .name = .vaxis, + .fingerprint = 0x14fbbb94fc556305, + .version = "0.5.1", +- .minimum_zig_version = "0.15.1", ++ .minimum_zig_version = "0.16.0-dev.2261+d6b3dd25a", + .dependencies = .{ + .zigimg = .{ +- .url = "git+https://github.com/zigimg/zigimg#eab2522c023b9259db8b13f2f90d609b7437e5f6", +- .hash = "zigimg-0.1.0-8_eo2vUZFgAAtN1c6dAO5DdqL0d4cEWHtn6iR5ucZJti", ++ .url = "https://github.com/unexge/zigimg/archive/refs/heads/master.zip", ++ .hash = "zigimg-0.1.0-8_eo2tGrFwD-VzWa_E8HuIuNenM2US7PRXBj5EJdGfrw", + }, + .uucode = .{ +- .url = "git+https://github.com/jacobsandlund/uucode#5f05f8f83a75caea201f12cc8ea32a2d82ea9732", +- .hash = "uucode-0.1.0-ZZjBPj96QADXyt5sqwBJUnhaDYs_qBeeKijZvlRa0eqM", ++ .url = "https://github.com/jacobsandlund/uucode/archive/refs/heads/zig-0.16.zip", ++ .hash = "uucode-0.2.0-ZZjBPkJiVABQGPdyw9BOtU2sflHv3vovBzmYtPRj2hvm", + }, + }, + .paths = .{ +diff --git a/examples/cli.zig b/examples/cli.zig +index 3b44676..2b9a288 100644 +--- a/examples/cli.zig ++++ b/examples/cli.zig +@@ -4,30 +4,23 @@ const Cell = vaxis.Cell; + const TextInput = vaxis.widgets.TextInput; + + const log = std.log.scoped(.main); +-pub fn main() !void { +- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +- defer { +- const deinit_status = gpa.deinit(); +- if (deinit_status == .leak) { +- log.err("memory leak", .{}); +- } +- } +- const alloc = gpa.allocator(); ++pub fn main(init: std.process.Init) !void { ++ const alloc = init.gpa; + + var buffer: [1024]u8 = undefined; +- var tty = try vaxis.Tty.init(&buffer); ++ var tty = try vaxis.Tty.init(init.io, &buffer); + defer tty.deinit(); + + var vx = try vaxis.init(alloc, .{}); + defer vx.deinit(alloc, tty.writer()); + +- var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx }; ++ var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx, .io = init.io, .queue = .{ .io = init.io } }; + try loop.init(); + + try loop.start(); + defer loop.stop(); + +- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_s); ++ try vx.queryTerminal(tty.writer(), init.io, 1 * std.time.ns_per_s); + + var text_input = TextInput.init(alloc); + defer text_input.deinit(); +diff --git a/examples/counter.zig b/examples/counter.zig +index d19af42..e68420b 100644 +--- a/examples/counter.zig ++++ b/examples/counter.zig +@@ -111,13 +111,10 @@ const Model = struct { + } + }; + +-pub fn main() !void { +- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +- defer _ = gpa.deinit(); ++pub fn main(init: std.process.Init) !void { ++ const allocator = init.gpa; + +- const allocator = gpa.allocator(); +- +- var app = try vxfw.App.init(allocator); ++ var app = try vxfw.App.init(allocator, init.io); + defer app.deinit(); + + // We heap allocate our model because we will require a stable pointer to it in our Button +diff --git a/examples/fuzzy.zig b/examples/fuzzy.zig +index 6ffbe6a..7fef980 100644 +--- a/examples/fuzzy.zig ++++ b/examples/fuzzy.zig +@@ -11,7 +11,6 @@ const Model = struct { + + /// Used for filtered RichText Spans and result + arena: std.heap.ArenaAllocator, +- filtered: std.ArrayList(vxfw.RichText), + result: []const u8, + + pub fn init(gpa: std.mem.Allocator) !*Model { +@@ -201,31 +200,24 @@ fn toLower(arena: std.mem.Allocator, src: []const u8) std.mem.Allocator.Error![] + return lower; + } + +-pub fn main() !void { +- var debug_allocator = std.heap.GeneralPurposeAllocator(.{}){}; +- defer _ = debug_allocator.deinit(); +- +- const gpa = debug_allocator.allocator(); ++pub fn main(init: std.process.Init) !void { ++ const gpa = init.gpa; + +- var app = try vxfw.App.init(gpa); ++ var app = try vxfw.App.init(gpa, init.io); + errdefer app.deinit(); + + const model = try Model.init(gpa); + defer model.deinit(gpa); + + // Run the command +- var fd = std.process.Child.init(&.{"fd"}, gpa); +- fd.stdout_behavior = .Pipe; +- fd.stderr_behavior = .Pipe; +- var stdout = std.ArrayList(u8).empty; +- var stderr = std.ArrayList(u8).empty; +- defer stdout.deinit(gpa); +- defer stderr.deinit(gpa); +- try fd.spawn(); +- try fd.collectOutput(gpa, &stdout, &stderr, 10_000_000); +- _ = try fd.wait(); +- +- var iter = std.mem.splitScalar(u8, stdout.items, '\n'); ++ const result = try std.process.run(gpa, init.io, .{ ++ .argv = &.{"fd"}, ++ .stdout_limit = .limited(10_000_000), ++ }); ++ defer gpa.free(result.stdout); ++ defer gpa.free(result.stderr); ++ ++ var iter = std.mem.splitScalar(u8, result.stdout, '\n'); + while (iter.next()) |line| { + if (line.len == 0) continue; + try model.list.append(gpa, .{ .text = line }); +@@ -235,8 +227,8 @@ pub fn main() !void { + app.deinit(); + + if (model.result.len > 0) { +- _ = try std.posix.write(std.posix.STDOUT_FILENO, model.result); +- _ = try std.posix.write(std.posix.STDOUT_FILENO, "\n"); ++ try std.Io.File.stdout().writeStreamingAll(init.io, model.result); ++ try std.Io.File.stdout().writeStreamingAll(init.io, "\n"); + } else { + std.process.exit(130); + } +diff --git a/examples/image.zig b/examples/image.zig +index c14bbdb..af62862 100644 +--- a/examples/image.zig ++++ b/examples/image.zig +@@ -8,41 +8,34 @@ const Event = union(enum) { + winsize: vaxis.Winsize, + }; + +-pub fn main() !void { +- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +- defer { +- const deinit_status = gpa.deinit(); +- if (deinit_status == .leak) { +- log.err("memory leak", .{}); +- } +- } +- const alloc = gpa.allocator(); ++pub fn main(init: std.process.Init) !void { ++ const alloc = init.gpa; + + var buffer: [1024]u8 = undefined; +- var tty = try vaxis.Tty.init(&buffer); ++ var tty = try vaxis.Tty.init(init.io, &buffer); + defer tty.deinit(); + + var vx = try vaxis.init(alloc, .{}); + defer vx.deinit(alloc, tty.writer()); + +- var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx }; ++ var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx, .io = init.io, .queue = .{ .io = init.io } }; + try loop.init(); + + try loop.start(); + defer loop.stop(); + + try vx.enterAltScreen(tty.writer()); +- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_s); ++ try vx.queryTerminal(tty.writer(), init.io, 1 * std.time.ns_per_s); + + var read_buffer: [1024 * 1024]u8 = undefined; // 1MB buffer +- var img1 = try vaxis.zigimg.Image.fromFilePath(alloc, "examples/zig.png", &read_buffer); ++ var img1 = try vaxis.zigimg.Image.fromFilePath(alloc, init.io, "examples/zig.png", &read_buffer); + defer img1.deinit(alloc); + + const imgs = [_]vaxis.Image{ + try vx.transmitImage(alloc, tty.writer(), &img1, .rgba), + // var img1 = try vaxis.zigimg.Image.fromFilePath(alloc, "examples/zig.png"); +- // try vx.loadImage(alloc, tty.writer(), .{ .path = "examples/zig.png" }), +- try vx.loadImage(alloc, tty.writer(), .{ .path = "examples/vaxis.png" }), ++ // try vx.loadImage(alloc, init.io, tty.writer(), .{ .path = "examples/zig.png" }), ++ try vx.loadImage(alloc, init.io, tty.writer(), .{ .path = "examples/vaxis.png" }), + }; + defer vx.freeImage(tty.writer(), imgs[0].id); + defer vx.freeImage(tty.writer(), imgs[1].id); +diff --git a/examples/list_view.zig b/examples/list_view.zig +index 24cbf8d..5b961e9 100644 +--- a/examples/list_view.zig ++++ b/examples/list_view.zig +@@ -52,13 +52,10 @@ const Model = struct { + } + }; + +-pub fn main() !void { +- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +- defer _ = gpa.deinit(); ++pub fn main(init: std.process.Init) !void { ++ const allocator = init.gpa; + +- const allocator = gpa.allocator(); +- +- var app = try vxfw.App.init(allocator); ++ var app = try vxfw.App.init(allocator, init.io); + defer app.deinit(); + + const model = try allocator.create(Model); +diff --git a/examples/main.zig b/examples/main.zig +index 2a4b96b..3caa7d9 100644 +--- a/examples/main.zig ++++ b/examples/main.zig +@@ -3,25 +3,17 @@ const vaxis = @import("vaxis"); + const Cell = vaxis.Cell; + + const log = std.log.scoped(.main); +-pub fn main() !void { +- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +- defer { +- const deinit_status = gpa.deinit(); +- //fail test; can't try in defer as defer is executed after we return +- if (deinit_status == .leak) { +- log.err("memory leak", .{}); +- } +- } +- const alloc = gpa.allocator(); ++pub fn main(init: std.process.Init) !void { ++ const alloc = init.gpa; + + var buffer: [1024]u8 = undefined; +- var tty = try vaxis.Tty.init(&buffer); ++ var tty = try vaxis.Tty.init(init.io, &buffer); + defer tty.deinit(); + + var vx = try vaxis.init(alloc, .{}); + defer vx.deinit(alloc, tty.writer()); + +- var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx }; ++ var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx, .io = init.io, .queue = .{ .io = init.io } }; + try loop.init(); + + try loop.start(); +@@ -29,7 +21,7 @@ pub fn main() !void { + + // Optionally enter the alternate screen + try vx.enterAltScreen(tty.writer()); +- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_s); ++ try vx.queryTerminal(tty.writer(), init.io, 1 * std.time.ns_per_s); + + // We'll adjust the color index every keypress + var color_idx: u8 = 0; +diff --git a/examples/scroll.zig b/examples/scroll.zig +index 0f92573..adf1f44 100644 +--- a/examples/scroll.zig ++++ b/examples/scroll.zig +@@ -151,13 +151,10 @@ const Model = struct { + } + }; + +-pub fn main() !void { +- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +- defer _ = gpa.deinit(); ++pub fn main(init: std.process.Init) !void { ++ const allocator = init.gpa; + +- const allocator = gpa.allocator(); +- +- var app = try vxfw.App.init(allocator); ++ var app = try vxfw.App.init(allocator, init.io); + errdefer app.deinit(); + + var arena = std.heap.ArenaAllocator.init(allocator); +diff --git a/examples/split_view.zig b/examples/split_view.zig +index 78d4e6f..5c48432 100644 +--- a/examples/split_view.zig ++++ b/examples/split_view.zig +@@ -49,13 +49,10 @@ const Model = struct { + } + }; + +-pub fn main() !void { +- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +- defer _ = gpa.deinit(); ++pub fn main(init: std.process.Init) !void { ++ const allocator = init.gpa; + +- const allocator = gpa.allocator(); +- +- var app = try vxfw.App.init(allocator); ++ var app = try vxfw.App.init(allocator, init.io); + defer app.deinit(); + + const model = try allocator.create(Model); +diff --git a/examples/table.zig b/examples/table.zig +index 430c3d5..c489828 100644 +--- a/examples/table.zig ++++ b/examples/table.zig +@@ -14,16 +14,14 @@ const ActiveSection = enum { + btm, + }; + +-pub fn main() !void { +- var gpa = heap.GeneralPurposeAllocator(.{}){}; +- defer if (gpa.detectLeaks()) log.err("Memory leak detected!", .{}); +- const alloc = gpa.allocator(); ++pub fn main(init: std.process.Init) !void { ++ const alloc = init.gpa; + + // Users set up below the main function + const users_buf = try alloc.dupe(User, users[0..]); + + var buffer: [1024]u8 = undefined; +- var tty = try vaxis.Tty.init(&buffer); ++ var tty = try vaxis.Tty.init(init.io, &buffer); + defer tty.deinit(); + const tty_writer = tty.writer(); + var vx = try vaxis.init(alloc, .{ +@@ -35,12 +33,12 @@ pub fn main() !void { + key_press: vaxis.Key, + winsize: vaxis.Winsize, + table_upd, +- }) = .{ .tty = &tty, .vaxis = &vx }; ++ }) = .{ .tty = &tty, .vaxis = &vx, .io = init.io, .queue = .{ .io = init.io } }; + try loop.init(); + try loop.start(); + defer loop.stop(); + try vx.enterAltScreen(tty.writer()); +- try vx.queryTerminal(tty.writer(), 250 * std.time.ns_per_ms); ++ try vx.queryTerminal(tty.writer(), init.io, 250 * std.time.ns_per_ms); + + const logo = + \\░█░█░█▀█░█░█░▀█▀░█▀▀░░░▀█▀░█▀█░█▀▄░█░░░█▀▀░ +diff --git a/examples/text_input.zig b/examples/text_input.zig +index cea0466..5c3c6c6 100644 +--- a/examples/text_input.zig ++++ b/examples/text_input.zig +@@ -18,20 +18,12 @@ const Event = union(enum) { + foo: u8, + }; + +-pub fn main() !void { +- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +- defer { +- const deinit_status = gpa.deinit(); +- //fail test; can't try in defer as defer is executed after we return +- if (deinit_status == .leak) { +- log.err("memory leak", .{}); +- } +- } +- const alloc = gpa.allocator(); ++pub fn main(init: std.process.Init) !void { ++ const alloc = init.gpa; + + // Initalize a tty + var buffer: [1024]u8 = undefined; +- var tty = try vaxis.Tty.init(&buffer); ++ var tty = try vaxis.Tty.init(init.io, &buffer); + defer tty.deinit(); + + // Use a buffered writer for better performance. There are a lot of writes +@@ -47,6 +39,8 @@ pub fn main() !void { + var loop: vaxis.Loop(Event) = .{ + .vaxis = &vx, + .tty = &tty, ++ .io = init.io, ++ .queue = .{ .io = init.io }, + }; + try loop.init(); + +@@ -71,7 +65,7 @@ pub fn main() !void { + try writer.flush(); + // Sends queries to terminal to detect certain features. This should + // _always_ be called, but is left to the application to decide when +- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_s); ++ try vx.queryTerminal(tty.writer(), init.io, 1 * std.time.ns_per_s); + + // The main event loop. Vaxis provides a thread safe, blocking, buffered + // queue which can serve as the primary event queue for an application +@@ -94,8 +88,10 @@ pub fn main() !void { + } else if (key.matches('n', .{ .ctrl = true })) { + try vx.notify(tty.writer(), "vaxis", "hello from vaxis"); + loop.stop(); +- var child = std.process.Child.init(&.{"nvim"}, alloc); +- _ = try child.spawnAndWait(); ++ var child = try std.process.spawn(init.io, .{ ++ .argv = &.{"nvim"}, ++ }); ++ _ = try child.wait(init.io); + try loop.start(); + try vx.enterAltScreen(tty.writer()); + vx.queueRefresh(); +diff --git a/examples/text_view.zig b/examples/text_view.zig +index 73e8e54..d49cb5b 100644 +--- a/examples/text_view.zig ++++ b/examples/text_view.zig +@@ -9,31 +9,24 @@ const Event = union(enum) { + winsize: vaxis.Winsize, + }; + +-pub fn main() !void { +- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +- +- defer { +- const deinit_status = gpa.deinit(); +- if (deinit_status == .leak) { +- log.err("memory leak", .{}); +- } +- } +- +- const alloc = gpa.allocator(); ++pub fn main(init: std.process.Init) !void { ++ const alloc = init.gpa; + var buffer: [1024]u8 = undefined; +- var tty = try vaxis.Tty.init(&buffer); ++ var tty = try vaxis.Tty.init(init.io, &buffer); + defer tty.deinit(); + var vx = try vaxis.init(alloc, .{}); + defer vx.deinit(alloc, tty.writer()); + var loop: vaxis.Loop(Event) = .{ + .vaxis = &vx, + .tty = &tty, ++ .io = init.io, ++ .queue = .{ .io = init.io }, + }; + try loop.init(); + try loop.start(); + defer loop.stop(); + try vx.enterAltScreen(tty.writer()); +- try vx.queryTerminal(tty.writer(), 20 * std.time.ns_per_s); ++ try vx.queryTerminal(tty.writer(), init.io, 20 * std.time.ns_per_s); + var text_view = TextView{}; + var text_view_buffer = TextView.Buffer{}; + defer text_view_buffer.deinit(alloc); +@@ -61,6 +54,6 @@ pub fn main() !void { + win.clear(); + text_view.draw(win, text_view_buffer); + try vx.render(tty.writer()); +- try tty.writer.flush(); ++ try tty.writer().flush(); + } + } +diff --git a/examples/vaxis.zig b/examples/vaxis.zig +index 5ca566c..8a12a66 100644 +--- a/examples/vaxis.zig ++++ b/examples/vaxis.zig +@@ -9,32 +9,24 @@ const Event = union(enum) { + + pub const panic = vaxis.panic_handler; + +-pub fn main() !void { +- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +- defer { +- const deinit_status = gpa.deinit(); +- //fail test; can't try in defer as defer is executed after we return +- if (deinit_status == .leak) { +- std.log.err("memory leak", .{}); +- } +- } +- const alloc = gpa.allocator(); ++pub fn main(init: std.process.Init) !void { ++ const alloc = init.gpa; + + var buffer: [1024]u8 = undefined; +- var tty = try vaxis.Tty.init(&buffer); ++ var tty = try vaxis.Tty.init(init.io, &buffer); + defer tty.deinit(); + + var vx = try vaxis.init(alloc, .{}); + defer vx.deinit(alloc, tty.writer()); + +- var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx }; ++ var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx, .io = init.io, .queue = .{ .io = init.io } }; + try loop.init(); + + try loop.start(); + defer loop.stop(); + + try vx.enterAltScreen(tty.writer()); +- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_s); ++ try vx.queryTerminal(tty.writer(), init.io, 1 * std.time.ns_per_s); + + try vx.queryColor(tty.writer(), .fg); + try vx.queryColor(tty.writer(), .bg); +@@ -84,7 +76,7 @@ pub fn main() !void { + // try vx.render(bw.writer().any()); + // try bw.flush(); + try vx.render(tty.writer()); +- std.Thread.sleep(16 * std.time.ns_per_ms); ++ try std.Io.sleep(init.io, std.Io.Duration.fromMilliseconds(16), .real); + switch (dir) { + .up => { + pct += 1; +diff --git a/examples/view.zig b/examples/view.zig +index 1b01dfd..3f5108a 100644 +--- a/examples/view.zig ++++ b/examples/view.zig +@@ -13,15 +13,8 @@ const Event = union(enum) { + winsize: vaxis.Winsize, + }; + +-pub fn main() !void { +- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +- defer { +- const deinit_status = gpa.deinit(); +- if (deinit_status == .leak) { +- log.err("memory leak", .{}); +- } +- } +- const alloc = gpa.allocator(); ++pub fn main(init: std.process.Init) !void { ++ const alloc = init.gpa; + + var world_map: []const u8 = lg_world_map; + var map_width = lg_map_width; +@@ -45,7 +38,7 @@ pub fn main() !void { + }); + + var buffer: [1024]u8 = undefined; +- var tty = try vaxis.Tty.init(&buffer); ++ var tty = try vaxis.Tty.init(init.io, &buffer); + defer tty.deinit(); + + const writer = tty.writer(); +@@ -58,13 +51,15 @@ pub fn main() !void { + var loop: vaxis.Loop(Event) = .{ + .vaxis = &vx, + .tty = &tty, ++ .io = init.io, ++ .queue = .{ .io = init.io }, + }; + try loop.init(); + try loop.start(); + defer loop.stop(); + try vx.enterAltScreen(writer); + try writer.flush(); +- try vx.queryTerminal(tty.writer(), 20 * std.time.ns_per_s); ++ try vx.queryTerminal(tty.writer(), init.io, 20 * std.time.ns_per_s); + + // Initialize Views + // - Large Map +diff --git a/examples/vt.zig b/examples/vt.zig +index fc13149..6d7f647 100644 +--- a/examples/vt.zig ++++ b/examples/vt.zig +@@ -9,33 +9,24 @@ const Event = union(enum) { + + pub const panic = vaxis.panic_handler; + +-pub fn main() !void { +- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +- defer { +- const deinit_status = gpa.deinit(); +- //fail test; can't try in defer as defer is executed after we return +- if (deinit_status == .leak) { +- std.log.err("memory leak", .{}); +- } +- } +- const alloc = gpa.allocator(); ++pub fn main(init: std.process.Init) !void { ++ const alloc = init.gpa; + + var buffer: [1024]u8 = undefined; +- var tty = try vaxis.Tty.init(&buffer); ++ var tty = try vaxis.Tty.init(init.io, &buffer); + const writer = tty.writer(); + var vx = try vaxis.init(alloc, .{}); + defer vx.deinit(alloc, writer); + +- var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx }; ++ var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx, .io = init.io, .queue = .{ .io = init.io } }; + try loop.init(); + + try loop.start(); + defer loop.stop(); + + try vx.enterAltScreen(writer); +- try vx.queryTerminal(writer, 1 * std.time.ns_per_s); +- var env = try std.process.getEnvMap(alloc); +- defer env.deinit(); ++ try vx.queryTerminal(writer, init.io, 1 * std.time.ns_per_s); ++ const env = init.environ_map; + + const vt_opts: vaxis.widgets.Terminal.Options = .{ + .winsize = .{ +@@ -52,8 +43,9 @@ pub fn main() !void { + var write_buf: [4096]u8 = undefined; + var vt = try vaxis.widgets.Terminal.init( + alloc, ++ init.io, + &argv, +- &env, ++ env, + vt_opts, + &write_buf, + ); +@@ -62,7 +54,7 @@ pub fn main() !void { + + var redraw: bool = false; + while (true) { +- std.Thread.sleep(8 * std.time.ns_per_ms); ++ std.Io.sleep(init.io, .fromMilliseconds(8), .awake) catch {}; + // try vt events first + while (vt.tryEvent()) |event| { + redraw = true; +diff --git a/src/Loop.zig b/src/Loop.zig +index a11e2f7..cd7e9d9 100644 +--- a/src/Loop.zig ++++ b/src/Loop.zig +@@ -18,8 +18,9 @@ pub fn Loop(comptime T: type) type { + + tty: *Tty, + vaxis: *Vaxis, ++ io: std.Io, + +- queue: Queue(T, 512) = .{}, ++ queue: Queue(T, 512), + thread: ?std.Thread = null, + should_quit: bool = false, + +@@ -214,7 +215,7 @@ pub fn handleEventGeneric(self: anytype, vx: *Vaxis, cache: *GraphemeCache, Even + } + }, + .cap_da1 => { +- std.Thread.Futex.wake(&vx.query_futex, 10); ++ std.Io.futexWake(self.io, u32, &vx.query_futex.raw, 10); + vx.queries_done.store(true, .unordered); + }, + .mouse => |mouse| { +@@ -359,7 +360,7 @@ pub fn handleEventGeneric(self: anytype, vx: *Vaxis, cache: *GraphemeCache, Even + vx.caps.multi_cursor = true; + }, + .cap_da1 => { +- std.Thread.Futex.wake(&vx.query_futex, 10); ++ std.Io.futexWake(self.io, u32, &vx.query_futex.raw, 10); + vx.queries_done.store(true, .unordered); + }, + .winsize => |winsize| { +@@ -386,13 +387,13 @@ test Loop { + foo: u8, + }; + +- var tty = try vaxis.Tty.init(&.{}); ++ var tty = try vaxis.Tty.init(std.testing.io); + defer tty.deinit(); + + var vx = try vaxis.init(std.testing.allocator, .{}); + defer vx.deinit(std.testing.allocator, tty.writer()); + +- var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx }; ++ var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx, .io = std.testing.io, .queue = .{ .io = std.testing.io } }; + try loop.init(); + + try loop.start(); +@@ -400,5 +401,5 @@ test Loop { + + // Optionally enter the alternate screen + try vx.enterAltScreen(tty.writer()); +- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_ms); ++ try vx.queryTerminal(tty.writer(), std.testing.io, 1 * std.time.ns_per_ms); + } +diff --git a/src/Parser.zig b/src/Parser.zig +index 6358269..1b125dd 100644 +--- a/src/Parser.zig ++++ b/src/Parser.zig +@@ -118,7 +118,7 @@ inline fn parseGround(input: []const u8) !Result { + var grapheme_len: usize = 0; + var cp_count: usize = 0; + +- while (grapheme_iter.next()) |result| { ++ while (grapheme_iter.nextCodePoint()) |result| { + cp_count += 1; + if (result.is_break) { + // Found the first grapheme boundary +diff --git a/src/Vaxis.zig b/src/Vaxis.zig +index a6a3830..2e24e9b 100644 +--- a/src/Vaxis.zig ++++ b/src/Vaxis.zig +@@ -3,7 +3,7 @@ const builtin = @import("builtin"); + const atomic = std.atomic; + const base64Encoder = std.base64.standard.Encoder; + const zigimg = @import("zigimg"); +-const IoWriter = std.io.Writer; ++const IoWriter = std.Io.Writer; + + const Cell = @import("Cell.zig"); + const Image = @import("Image.zig"); +@@ -250,10 +250,10 @@ pub fn exitAltScreen(self: *Vaxis, tty: *IoWriter) !void { + /// + /// This call will block until Vaxis.query_futex is woken up, or the timeout. + /// Event loops can wake up this futex when cap_da1 is received +-pub fn queryTerminal(self: *Vaxis, tty: *IoWriter, timeout_ns: u64) !void { ++pub fn queryTerminal(self: *Vaxis, tty: *IoWriter, io: std.Io, timeout_ns: u64) !void { + try self.queryTerminalSend(tty); + // 1 second timeout +- std.Thread.Futex.timedWait(&self.query_futex, 0, timeout_ns) catch {}; ++ std.Io.futexWaitTimeout(io, u32, &self.query_futex.raw, 0, .{ .duration = .{ .raw = .{ .nanoseconds = @intCast(timeout_ns) }, .clock = .awake } }) catch {}; + self.queries_done.store(true, .unordered); + try self.enableDetectedFeatures(tty); + } +@@ -265,7 +265,7 @@ pub fn queryTerminalSend(vx: *Vaxis, tty: *IoWriter) !void { + vx.queries_done.store(false, .unordered); + + // TODO: re-enable this +- // const colorterm = std.posix.getenv("COLORTERM") orelse ""; ++ // const colorterm = std.c.getenv("COLORTERM") orelse ""; + // if (std.mem.eql(u8, colorterm, "truecolor") or + // std.mem.eql(u8, colorterm, "24bit")) + // { +@@ -320,22 +320,22 @@ pub fn enableDetectedFeatures(self: *Vaxis, tty: *IoWriter) !void { + }, + else => { + // Apply any environment variables +- if (std.posix.getenv("TERMUX_VERSION")) |_| ++ if (std.c.getenv("TERMUX_VERSION")) |_| + self.sgr = .legacy; +- if (std.posix.getenv("VHS_RECORD")) |_| { ++ if (std.c.getenv("VHS_RECORD")) |_| { + self.caps.unicode = .wcwidth; + self.caps.kitty_keyboard = false; + self.sgr = .legacy; + } +- if (std.posix.getenv("TERM_PROGRAM")) |prg| { +- if (std.mem.eql(u8, prg, "vscode")) ++ if (std.c.getenv("TERM_PROGRAM")) |prg| { ++ if (std.mem.eql(u8, std.mem.span(prg), "vscode")) + self.sgr = .legacy; + } +- if (std.posix.getenv("VAXIS_FORCE_LEGACY_SGR")) |_| ++ if (std.c.getenv("VAXIS_FORCE_LEGACY_SGR")) |_| + self.sgr = .legacy; +- if (std.posix.getenv("VAXIS_FORCE_WCWIDTH")) |_| ++ if (std.c.getenv("VAXIS_FORCE_WCWIDTH")) |_| + self.caps.unicode = .wcwidth; +- if (std.posix.getenv("VAXIS_FORCE_UNICODE")) |_| ++ if (std.c.getenv("VAXIS_FORCE_UNICODE")) |_| + self.caps.unicode = .unicode; + + // enable detected features +@@ -1067,6 +1067,7 @@ pub fn transmitImage( + pub fn loadImage( + self: *Vaxis, + alloc: std.mem.Allocator, ++ io: std.Io, + tty: *IoWriter, + src: Image.Source, + ) !Image { +@@ -1074,7 +1075,7 @@ pub fn loadImage( + + var read_buffer: [1024 * 1024]u8 = undefined; // 1MB buffer + var img = switch (src) { +- .path => |path| try zigimg.Image.fromFilePath(alloc, path, &read_buffer), ++ .path => |path| try zigimg.Image.fromFilePath(alloc, io, path, &read_buffer), + .mem => |bytes| try zigimg.Image.fromMemory(alloc, bytes), + }; + defer img.deinit(alloc); +@@ -1481,7 +1482,7 @@ pub fn setTerminalWorkingDirectory(_: *Vaxis, tty: *IoWriter, path: []const u8) + return error.InvalidAbsolutePath; + const hostname = switch (builtin.os.tag) { + .windows => null, +- else => std.posix.getenv("HOSTNAME"), ++ else => std.c.getenv("HOSTNAME"), + } orelse "localhost"; + + const uri: std.Uri = .{ +@@ -1495,11 +1496,11 @@ pub fn setTerminalWorkingDirectory(_: *Vaxis, tty: *IoWriter, path: []const u8) + + test "render: no output when no changes" { + var vx = try Vaxis.init(std.testing.allocator, .{}); +- var deinit_writer = std.io.Writer.Allocating.init(std.testing.allocator); ++ var deinit_writer: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer deinit_writer.deinit(); + defer vx.deinit(std.testing.allocator, &deinit_writer.writer); + +- var render_writer = std.io.Writer.Allocating.init(std.testing.allocator); ++ var render_writer: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer render_writer.deinit(); + try vx.render(&render_writer.writer); + const output = try render_writer.toOwnedSlice(); +diff --git a/src/gwidth.zig b/src/gwidth.zig +index 194139f..ecffaf7 100644 +--- a/src/gwidth.zig ++++ b/src/gwidth.zig +@@ -55,10 +55,10 @@ pub fn gwidth(str: []const u8, method: Method) u16 { + var grapheme_start: usize = 0; + var prev_break: bool = true; + +- while (grapheme_iter.next()) |result| { ++ while (grapheme_iter.nextCodePoint()) |result| { + if (prev_break and !result.is_break) { + // Start of a new grapheme +- const cp_len: usize = std.unicode.utf8CodepointSequenceLength(result.cp) catch 1; ++ const cp_len: usize = std.unicode.utf8CodepointSequenceLength(result.code_point) catch 1; + grapheme_start = grapheme_iter.i - cp_len; + } + +diff --git a/src/queue.zig b/src/queue.zig +index 6d847ea..908cf70 100644 +--- a/src/queue.zig ++++ b/src/queue.zig +@@ -1,7 +1,7 @@ + const std = @import("std"); + const assert = std.debug.assert; + const atomic = std.atomic; +-const Condition = std.Thread.Condition; ++const Condition = std.Io.Condition; + + /// Thread safe. Fixed size. Blocking push and pop. + pub fn Queue( +@@ -11,23 +11,25 @@ pub fn Queue( + return struct { + buf: [size]T = undefined, + ++ io: std.Io, ++ + read_index: usize = 0, + write_index: usize = 0, + +- mutex: std.Thread.Mutex = .{}, ++ mutex: std.Io.Mutex = .init, + // blocks when the buffer is full +- not_full: Condition = .{}, ++ not_full: Condition = .init, + // ...or empty +- not_empty: Condition = .{}, ++ not_empty: Condition = .init, + + const Self = @This(); + + /// Pop an item from the queue. Blocks until an item is available. + pub fn pop(self: *Self) T { +- self.mutex.lock(); +- defer self.mutex.unlock(); ++ self.mutex.lockUncancelable(self.io); ++ defer self.mutex.unlock(self.io); + while (self.isEmptyLH()) { +- self.not_empty.wait(&self.mutex); ++ self.not_empty.waitUncancelable(self.io, &self.mutex); + } + std.debug.assert(!self.isEmptyLH()); + return self.popAndSignalLH(); +@@ -36,10 +38,10 @@ pub fn Queue( + /// Push an item into the queue. Blocks until an item has been + /// put in the queue. + pub fn push(self: *Self, item: T) void { +- self.mutex.lock(); +- defer self.mutex.unlock(); ++ self.mutex.lockUncancelable(self.io); ++ defer self.mutex.unlock(self.io); + while (self.isFullLH()) { +- self.not_full.wait(&self.mutex); ++ self.not_full.waitUncancelable(self.io, &self.mutex); + } + std.debug.assert(!self.isFullLH()); + self.pushAndSignalLH(item); +@@ -49,8 +51,8 @@ pub fn Queue( + /// was successfully placed in the queue, false if the queue + /// was full. + pub fn tryPush(self: *Self, item: T) bool { +- self.mutex.lock(); +- defer self.mutex.unlock(); ++ self.mutex.lockUncancelable(self.io); ++ defer self.mutex.unlock(self.io); + if (self.isFullLH()) return false; + self.pushAndSignalLH(item); + return true; +@@ -59,28 +61,28 @@ pub fn Queue( + /// Pop an item from the queue. Returns null when no item is + /// available. + pub fn tryPop(self: *Self) ?T { +- self.mutex.lock(); +- defer self.mutex.unlock(); ++ self.mutex.lockUncancelable(self.io); ++ defer self.mutex.unlock(self.io); + if (self.isEmptyLH()) return null; + return self.popAndSignalLH(); + } + + /// Poll the queue. This call blocks until events are in the queue + pub fn poll(self: *Self) void { +- self.mutex.lock(); +- defer self.mutex.unlock(); ++ self.mutex.lockUncancelable(self.io); ++ defer self.mutex.unlock(self.io); + while (self.isEmptyLH()) { +- self.not_empty.wait(&self.mutex); ++ self.not_empty.waitUncancelable(self.io, &self.mutex); + } + std.debug.assert(!self.isEmptyLH()); + } + + pub fn lock(self: *Self) void { +- self.mutex.lock(); ++ self.mutex.lockUncancelable(self.io); + } + + pub fn unlock(self: *Self) void { +- self.mutex.unlock(); ++ self.mutex.unlock(self.io); + } + + /// Used to efficiently drain the queue while the lock is externally held +@@ -92,7 +94,7 @@ pub fn Queue( + const was_full = self.isFullLH(); + const item = self.popLH(); + if (was_full) { +- self.not_full.signal(); ++ self.not_full.signal(self.io); + } + return item; + } +@@ -108,15 +110,15 @@ pub fn Queue( + + /// Returns `true` if the queue is empty and `false` otherwise. + pub fn isEmpty(self: *Self) bool { +- self.mutex.lock(); +- defer self.mutex.unlock(); ++ self.mutex.lockUncancelable(self.io); ++ defer self.mutex.unlock(self.io); + return self.isEmptyLH(); + } + + /// Returns `true` if the queue is full and `false` otherwise. + pub fn isFull(self: *Self) bool { +- self.mutex.lock(); +- defer self.mutex.unlock(); ++ self.mutex.lockUncancelable(self.io); ++ defer self.mutex.unlock(self.io); + return self.isFullLH(); + } + +@@ -143,7 +145,7 @@ pub fn Queue( + self.buf[self.mask(self.write_index)] = item; + self.write_index = self.mask2(self.write_index + 1); + if (was_empty) { +- self.not_empty.signal(); ++ self.not_empty.signal(self.io); + } + } + +@@ -151,7 +153,7 @@ pub fn Queue( + const was_full = self.isFullLH(); + const result = self.popLH(); + if (was_full) { +- self.not_full.signal(); ++ self.not_full.signal(self.io); + } + return result; + } +@@ -167,7 +169,7 @@ pub fn Queue( + const testing = std.testing; + const cfg = Thread.SpawnConfig{ .allocator = testing.allocator }; + test "Queue: simple push / pop" { +- var queue: Queue(u8, 16) = .{}; ++ var queue: Queue(u8, 16) = .{ .io = std.testing.io }; + queue.push(1); + queue.push(2); + const pop = queue.pop(); +@@ -182,7 +184,7 @@ fn testPushPop(q: *Queue(u8, 2)) !void { + } + + test "Fill, wait to push, pop once in another thread" { +- var queue: Queue(u8, 2) = .{}; ++ var queue: Queue(u8, 2) = .{ .io = std.testing.io }; + queue.push(1); + queue.push(2); + const t = try Thread.spawn(cfg, testPushPop, .{&queue}); +@@ -202,7 +204,7 @@ fn testPush(q: *Queue(u8, 2)) void { + } + + test "Try to pop, fill from another thread" { +- var queue: Queue(u8, 2) = .{}; ++ var queue: Queue(u8, 2) = .{ .io = std.testing.io }; + const thread = try Thread.spawn(cfg, testPush, .{&queue}); + for (0..5) |idx| { + try testing.expectEqual(@as(u8, @intCast(idx)), queue.pop()); +@@ -217,8 +219,8 @@ fn sleepyPop(q: *Queue(u8, 2), state: *atomic.Value(u8)) !void { + + // Then we spuriously wake it up, because that's a thing that can + // happen. +- q.not_full.signal(); +- q.not_empty.signal(); ++ q.not_full.signal(q.io); ++ q.not_empty.signal(q.io); + + // Then give the other thread a good chance of waking up. It's not + // clear that yield guarantees the other thread will be scheduled, +@@ -226,7 +228,7 @@ fn sleepyPop(q: *Queue(u8, 2), state: *atomic.Value(u8)) !void { + // still full and the push in the other thread is still blocked + // waiting for space. + try Thread.yield(); +- std.Thread.sleep(10 * std.time.ns_per_ms); ++ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; + // Finally, let that other thread go. + try std.testing.expectEqual(1, q.pop()); + +@@ -235,15 +237,15 @@ fn sleepyPop(q: *Queue(u8, 2), state: *atomic.Value(u8)) !void { + try Thread.yield(); + // But we want to ensure that there's a second push waiting, so + // here's another sleep. +- std.Thread.sleep(10 * std.time.ns_per_ms); ++ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; + + // Another spurious wake... +- q.not_full.signal(); +- q.not_empty.signal(); ++ q.not_full.signal(q.io); ++ q.not_empty.signal(q.io); + // And another chance for the other thread to see that it's + // spurious and go back to sleep. + try Thread.yield(); +- std.Thread.sleep(10 * std.time.ns_per_ms); ++ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; + + // Pop that thing and we're done. + try std.testing.expectEqual(2, q.pop()); +@@ -256,19 +258,20 @@ test "Fill, block, fill, block" { + // that too (after some time) then drain the queue. This test + // fails if the while loop in `push` is turned into an `if`. + +- var queue: Queue(u8, 2) = .{}; ++ var queue: Queue(u8, 2) = .{ .io = std.testing.io }; + var state = atomic.Value(u8).init(0); + const thread = try Thread.spawn(cfg, sleepyPop, .{ &queue, &state }); + queue.push(1); + queue.push(2); + state.store(1, .release); +- const now = std.time.milliTimestamp(); ++ const start = std.Io.Clock.Timestamp.now(std.testing.io, .awake); + queue.push(3); // This one should block. +- const then = std.time.milliTimestamp(); ++ const end = std.Io.Clock.Timestamp.now(std.testing.io, .awake); + + // Just to make sure the sleeps are yielding to this thread, make + // sure it took at least 5ms to do the push. +- try std.testing.expect(then - now > 5); ++ const elapsed = start.durationTo(end); ++ try std.testing.expect(elapsed.raw.toMilliseconds() > 5); + + state.store(2, .release); + // This should block again, waiting for the other thread. +@@ -283,14 +286,14 @@ test "Fill, block, fill, block" { + fn sleepyPush(q: *Queue(u8, 1), state: *atomic.Value(u8)) !void { + // Try to ensure the other thread has already started trying to pop. + try Thread.yield(); +- std.Thread.sleep(10 * std.time.ns_per_ms); ++ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; + + // Spurious wake +- q.not_full.signal(); +- q.not_empty.signal(); ++ q.not_full.signal(q.io); ++ q.not_empty.signal(q.io); + + try Thread.yield(); +- std.Thread.sleep(10 * std.time.ns_per_ms); ++ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; + + // Stick something in the queue so it can be popped. + q.push(1); +@@ -299,11 +302,11 @@ fn sleepyPush(q: *Queue(u8, 1), state: *atomic.Value(u8)) !void { + try Thread.yield(); + // Give the other thread time to block again. + try Thread.yield(); +- std.Thread.sleep(10 * std.time.ns_per_ms); ++ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; + + // Spurious wake +- q.not_full.signal(); +- q.not_empty.signal(); ++ q.not_full.signal(q.io); ++ q.not_empty.signal(q.io); + + q.push(2); + } +@@ -313,7 +316,7 @@ test "Drain, block, drain, block" { + // test should fail if the `while` loop in `pop` is turned into an + // `if`. + +- var queue: Queue(u8, 1) = .{}; ++ var queue: Queue(u8, 1) = .{ .io = std.testing.io }; + var state = atomic.Value(u8).init(0); + const thread = try Thread.spawn(cfg, sleepyPush, .{ &queue, &state }); + try std.testing.expectEqual(1, queue.pop()); +@@ -328,11 +331,11 @@ fn readerThread(q: *Queue(u8, 1)) !void { + + test "2 readers" { + // 2 threads read, one thread writes +- var queue: Queue(u8, 1) = .{}; ++ var queue: Queue(u8, 1) = .{ .io = std.testing.io }; + const t1 = try Thread.spawn(cfg, readerThread, .{&queue}); + const t2 = try Thread.spawn(cfg, readerThread, .{&queue}); + try Thread.yield(); +- std.Thread.sleep(10 * std.time.ns_per_ms); ++ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; + queue.push(1); + queue.push(1); + t1.join(); +@@ -344,7 +347,7 @@ fn writerThread(q: *Queue(u8, 1)) !void { + } + + test "2 writers" { +- var queue: Queue(u8, 1) = .{}; ++ var queue: Queue(u8, 1) = .{ .io = std.testing.io }; + const t1 = try Thread.spawn(cfg, writerThread, .{&queue}); + const t2 = try Thread.spawn(cfg, writerThread, .{&queue}); + +diff --git a/src/tty.zig b/src/tty.zig +index 4a3c142..c1da795 100644 +--- a/src/tty.zig ++++ b/src/tty.zig +@@ -33,8 +33,11 @@ pub const PosixTty = struct { + /// The file descriptor of the tty + fd: posix.fd_t, + ++ io: std.Io, ++ + /// File.Writer for efficient buffered writing +- tty_writer: std.fs.File.Writer, ++ tty_buffer: [1024]u8 = undefined, ++ tty_writer: ?std.Io.File.Writer = null, + + pub const SignalHandler = struct { + context: *anyopaque, +@@ -43,7 +46,8 @@ pub const PosixTty = struct { + + /// global signal handlers + var handlers: [8]SignalHandler = undefined; +- var handler_mutex: std.Thread.Mutex = .{}; ++ var handler_mutex: std.Io.Mutex = .init; ++ var handler_io: std.Io = undefined; + var handler_idx: usize = 0; + + var handler_installed: bool = false; +@@ -51,9 +55,9 @@ pub const PosixTty = struct { + /// initializes a Tty instance by opening /dev/tty and "making it raw". A + /// signal handler is installed for SIGWINCH. No callbacks are installed, be + /// sure to register a callback when initializing the event loop +- pub fn init(buffer: []u8) !PosixTty { ++ pub fn init(io: std.Io) !PosixTty { + // Open our tty +- const fd = try posix.open("/dev/tty", .{ .ACCMODE = .RDWR }, 0); ++ const fd = try posix.openat(posix.AT.FDCWD, "/dev/tty", .{ .ACCMODE = .RDWR }, 0); + + // Set the termios of the tty + const termios = try makeRaw(fd); +@@ -69,13 +73,8 @@ pub const PosixTty = struct { + posix.sigaction(posix.SIG.WINCH, &act, null); + handler_installed = true; + +- const file = std.fs.File{ .handle = fd }; +- +- const self: PosixTty = .{ +- .fd = fd, +- .termios = termios, +- .tty_writer = .initStreaming(file, buffer), +- }; ++ handler_io = io; ++ const self: PosixTty = .{ .fd = fd, .io = io, .termios = termios }; + + global_tty = self; + +@@ -88,7 +87,7 @@ pub const PosixTty = struct { + std.log.err("couldn't restore terminal: {}", .{err}); + }; + if (builtin.os.tag != .macos) // closing /dev/tty may block indefinitely on macos +- posix.close(self.fd); ++ (std.Io.File{ .handle = self.fd, .flags = .{ .nonblocking = false } }).close(handler_io); + } + + /// Resets the signal handler to it's default +@@ -107,7 +106,12 @@ pub const PosixTty = struct { + } + + pub fn writer(self: *PosixTty) *std.Io.Writer { +- return &self.tty_writer.interface; ++ if (self.tty_writer == null) { ++ const file: std.Io.File = .{ .handle = self.fd, .flags = .{ .nonblocking = false } }; ++ self.tty_writer = .initStreaming(file, self.io, &self.tty_buffer); ++ } ++ ++ return &self.tty_writer.?.interface; + } + + pub fn read(self: *const PosixTty, buf: []u8) !usize { +@@ -117,16 +121,16 @@ pub const PosixTty = struct { + /// Install a signal handler for winsize. A maximum of 8 handlers may be + /// installed + pub fn notifyWinsize(handler: SignalHandler) !void { +- handler_mutex.lock(); +- defer handler_mutex.unlock(); ++ handler_mutex.lockUncancelable(handler_io); ++ defer handler_mutex.unlock(handler_io); + if (handler_idx == handlers.len) return error.OutOfMemory; + handlers[handler_idx] = handler; + handler_idx += 1; + } + +- fn handleWinch(_: c_int) callconv(.c) void { +- handler_mutex.lock(); +- defer handler_mutex.unlock(); ++ fn handleWinch(_: posix.SIG) callconv(.c) void { ++ if (!handler_mutex.tryLock()) return; ++ defer handler_mutex.unlock(handler_io); + var i: usize = 0; + while (i < handler_idx) : (i += 1) { + const handler = handlers[i]; +@@ -189,6 +193,7 @@ pub const PosixTty = struct { + pub const WindowsTty = struct { + stdin: windows.HANDLE, + stdout: windows.HANDLE, ++ io: std.Io, + + initial_codepage: c_uint, + initial_input_mode: CONSOLE_MODE_INPUT, +@@ -198,7 +203,8 @@ pub const WindowsTty = struct { + buf: [4]u8 = undefined, + + /// File.Writer for efficient buffered writing +- tty_writer: std.fs.File.Writer, ++ tty_buffer: [1024]u8 = undefined, ++ tty_writer: ?std.Io.File.Writer = null, + + /// The last mouse button that was pressed. We store the previous state of button presses on each + /// mouse event so we can detect which button was released +@@ -221,7 +227,7 @@ pub const WindowsTty = struct { + .ENABLE_LVB_GRID_WORLDWIDE = 1, // enables reverse video and underline + }; + +- pub fn init(buffer: []u8) !Tty { ++ pub fn init(io: std.Io) !Tty { + const stdin: std.fs.File = .stdin(); + const stdout: std.fs.File = .stdout(); + +@@ -237,12 +243,12 @@ pub const WindowsTty = struct { + return windows.unexpectedError(windows.kernel32.GetLastError()); + + const self: Tty = .{ ++ .io = io, + .stdin = stdin.handle, + .stdout = stdout.handle, + .initial_codepage = initial_output_codepage, + .initial_input_mode = initial_input_mode, + .initial_output_mode = initial_output_mode, +- .tty_writer = .initStreaming(stdout, buffer), + }; + + // save a copy of this tty as the global_tty for panic handling +@@ -298,7 +304,12 @@ pub const WindowsTty = struct { + } + + pub fn writer(self: *Tty) *std.Io.Writer { +- return &self.tty_writer.interface; ++ if (self.tty_writer == null) { ++ const stdout: std.fs.File = .stdout(); ++ self.tty_writer = .initStreaming(stdout, &self.tty_buffer); ++ } ++ ++ return &self.tty_writer.?.interface; + } + + pub fn read(self: *const Tty, buf: []u8) !usize { +@@ -694,20 +705,20 @@ pub const WindowsTty = struct { + pub const TestTty = struct { + /// Used for API compat + fd: posix.fd_t, ++ io: std.Io, + pipe_read: posix.fd_t, + pipe_write: posix.fd_t, + tty_writer: *std.Io.Writer.Allocating, + + /// Initializes a TestTty. +- pub fn init(buffer: []u8) !TestTty { +- _ = buffer; +- ++ pub fn init(io: std.Io) !TestTty { + if (builtin.os.tag == .windows) return error.SkipZigTest; + const list = try std.testing.allocator.create(std.Io.Writer.Allocating); + list.* = .init(std.testing.allocator); +- const r, const w = try posix.pipe(); ++ const r, const w = try std.Io.Threaded.pipe2(.{}); + return .{ + .fd = r, ++ .io = io, + .pipe_read = r, + .pipe_write = w, + .tty_writer = list, +@@ -715,8 +726,9 @@ pub const TestTty = struct { + } + + pub fn deinit(self: TestTty) void { +- std.posix.close(self.pipe_read); +- std.posix.close(self.pipe_write); ++ const io = std.testing.io; ++ (std.Io.File{ .handle = self.pipe_read, .flags = .{ .nonblocking = false } }).close(io); ++ (std.Io.File{ .handle = self.pipe_write, .flags = .{ .nonblocking = false } }).close(io); + self.tty_writer.deinit(); + std.testing.allocator.destroy(self.tty_writer); + } +diff --git a/src/unicode.zig b/src/unicode.zig +index a5253ac..4ff2e4c 100644 +--- a/src/unicode.zig ++++ b/src/unicode.zig +@@ -26,10 +26,10 @@ pub const GraphemeIterator = struct { + } + + pub fn next(self: *GraphemeIterator) ?Grapheme { +- while (self.inner.next()) |res| { ++ while (self.inner.nextCodePoint()) |res| { + // When leaving a break and entering a non-break, set the start of a cluster + if (self.prev_break and !res.is_break) { +- const cp_len: usize = std.unicode.utf8CodepointSequenceLength(res.cp) catch 1; ++ const cp_len: usize = std.unicode.utf8CodepointSequenceLength(res.code_point) catch 1; + self.start = self.inner.i - cp_len; + } + +diff --git a/src/vxfw/App.zig b/src/vxfw/App.zig +index 6d394a9..fc3962f 100644 +--- a/src/vxfw/App.zig ++++ b/src/vxfw/App.zig +@@ -16,7 +16,7 @@ tty: vaxis.Tty, + vx: vaxis.Vaxis, + timers: std.ArrayList(vxfw.Tick), + wants_focus: ?vxfw.Widget, +-buffer: [1024]u8, ++io: std.Io, + + /// Runtime options + pub const Options = struct { +@@ -27,10 +27,10 @@ pub const Options = struct { + /// Create an application. We require stable pointers to do the set up, so this will create an App + /// object on the heap. Call destroy when the app is complete to reset terminal state and release + /// resources +-pub fn init(allocator: Allocator) !App { +- var app: App = .{ ++pub fn init(allocator: Allocator, io: std.Io) !App { ++ return .{ + .allocator = allocator, +- .tty = undefined, ++ .tty = try vaxis.Tty.init(io), + .vx = try vaxis.init(allocator, .{ + .system_clipboard_allocator = allocator, + .kitty_keyboard_flags = .{ +@@ -39,10 +39,8 @@ pub fn init(allocator: Allocator) !App { + }), + .timers = std.ArrayList(vxfw.Tick){}, + .wants_focus = null, +- .buffer = undefined, ++ .io = io, + }; +- app.tty = try vaxis.Tty.init(&app.buffer); +- return app; + } + + pub fn deinit(self: *App) void { +@@ -55,7 +53,8 @@ pub fn run(self: *App, widget: vxfw.Widget, opts: Options) anyerror!void { + const tty = &self.tty; + const vx = &self.vx; + +- var loop: EventLoop = .{ .tty = tty, .vaxis = vx }; ++ const io = self.io; ++ var loop: EventLoop = .{ .tty = tty, .vaxis = vx, .io = io, .queue = .{ .io = io } }; + try loop.start(); + defer loop.stop(); + +@@ -65,7 +64,7 @@ pub fn run(self: *App, widget: vxfw.Widget, opts: Options) anyerror!void { + loop.postEvent(.focus_in); + + try vx.enterAltScreen(tty.writer()); +- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_s); ++ try vx.queryTerminal(tty.writer(), io, 1 * std.time.ns_per_s); + try vx.setBracketedPaste(tty.writer(), true); + try vx.subscribeToColorSchemeUpdates(tty.writer()); + +@@ -97,13 +96,14 @@ pub fn run(self: *App, widget: vxfw.Widget, opts: Options) anyerror!void { + defer focus_handler.deinit(self.allocator); + + // Timestamp of our next frame +- var next_frame_ms: u64 = @intCast(std.time.milliTimestamp()); ++ var next_frame_ms: u64 = nowMs(io); + + // Create our event context + var ctx: vxfw.EventContext = .{ + .alloc = self.allocator, + .phase = .capturing, + .cmds = vxfw.CommandList{}, ++ .io = io, + .consume_event = false, + .redraw = false, + .quit = false, +@@ -111,13 +111,13 @@ pub fn run(self: *App, widget: vxfw.Widget, opts: Options) anyerror!void { + defer ctx.cmds.deinit(self.allocator); + + while (true) { +- const now_ms: u64 = @intCast(std.time.milliTimestamp()); ++ const now_ms: u64 = nowMs(io); + if (now_ms >= next_frame_ms) { + // Deadline exceeded. Schedule the next frame + next_frame_ms = now_ms + tick_ms; + } else { + // Sleep until the deadline +- std.Thread.sleep((next_frame_ms - now_ms) * std.time.ns_per_ms); ++ try std.Io.sleep(io, .fromMilliseconds(@intCast(next_frame_ms - now_ms)), .real); + next_frame_ms += tick_ms; + } + +@@ -296,7 +296,7 @@ fn handleCommand(self: *App, cmds: *vxfw.CommandList) Allocator.Error!void { + } + + fn checkTimers(self: *App, ctx: *vxfw.EventContext) anyerror!void { +- const now_ms = std.time.milliTimestamp(); ++ const now_ms = nowMs(ctx.io); + + // timers are always sorted descending + while (self.timers.pop()) |tick| { +@@ -349,12 +349,16 @@ const MouseHandler = struct { + .surface = last_frame, + .z_index = 0, + }; +- const mouse_point: vxfw.Point = .{ +- .row = @intCast(mouse.row), +- .col = @intCast(mouse.col), +- }; +- if (sub.containsPoint(mouse_point)) { +- try last_frame.hitTest(app.allocator, &hits, mouse_point); ++ ++ if (mouse.row >= 0 and mouse.col >= 0) { ++ const mouse_point: vxfw.Point = .{ ++ .row = @intCast(mouse.row), ++ .col = @intCast(mouse.col), ++ }; ++ ++ if (sub.containsPoint(mouse_point)) { ++ try last_frame.hitTest(app.allocator, &hits, mouse_point); ++ } + } + + // We store the hit list from the last mouse event to determine mouse_enter and mouse_leave +@@ -408,12 +412,16 @@ const MouseHandler = struct { + .surface = last_frame, + .z_index = 0, + }; +- const mouse_point: vxfw.Point = .{ +- .row = @intCast(mouse.row), +- .col = @intCast(mouse.col), +- }; +- if (sub.containsPoint(mouse_point)) { +- try last_frame.hitTest(app.allocator, &hits, mouse_point); ++ ++ if (mouse.row >= 0 and mouse.col >= 0) { ++ const mouse_point: vxfw.Point = .{ ++ .row = @intCast(mouse.row), ++ .col = @intCast(mouse.col), ++ }; ++ ++ if (sub.containsPoint(mouse_point)) { ++ try last_frame.hitTest(app.allocator, &hits, mouse_point); ++ } + } + + // Handle mouse_enter and mouse_leave events +@@ -602,3 +610,8 @@ const FocusHandler = struct { + } + } + }; ++ ++fn nowMs(io: std.Io) u64 { ++ const now = std.Io.Clock.now(.real, io); ++ return @intCast(now.toMilliseconds()); ++} +diff --git a/src/vxfw/Button.zig b/src/vxfw/Button.zig +index d4b2a9c..075627e 100644 +--- a/src/vxfw/Button.zig ++++ b/src/vxfw/Button.zig +@@ -142,6 +142,7 @@ test Button { + var ctx: vxfw.EventContext = .{ + .alloc = std.testing.allocator, + .cmds = .empty, ++ .io = std.testing.io, + }; + defer ctx.cmds.deinit(ctx.alloc); + +diff --git a/src/vxfw/ListView.zig b/src/vxfw/ListView.zig +index 8ff9551..de93d5d 100644 +--- a/src/vxfw/ListView.zig ++++ b/src/vxfw/ListView.zig +@@ -564,6 +564,7 @@ test ListView { + var ctx: vxfw.EventContext = .{ + .alloc = std.testing.allocator, + .cmds = .empty, ++ .io = std.testing.io, + }; + defer ctx.cmds.deinit(ctx.alloc); + +@@ -731,6 +732,7 @@ test "ListView: uneven scroll" { + var ctx: vxfw.EventContext = .{ + .alloc = std.testing.allocator, + .cmds = .empty, ++ .io = std.testing.io, + }; + defer ctx.cmds.deinit(ctx.alloc); + +diff --git a/src/vxfw/ScrollView.zig b/src/vxfw/ScrollView.zig +index 36319a4..38859ea 100644 +--- a/src/vxfw/ScrollView.zig ++++ b/src/vxfw/ScrollView.zig +@@ -646,6 +646,7 @@ test ScrollView { + var ctx: vxfw.EventContext = .{ + .alloc = std.testing.allocator, + .cmds = .empty, ++ .io = std.testing.io, + }; + defer ctx.cmds.deinit(ctx.alloc); + +@@ -1044,6 +1045,7 @@ test "ScrollView: uneven scroll" { + var ctx: vxfw.EventContext = .{ + .alloc = std.testing.allocator, + .cmds = .empty, ++ .io = std.testing.io, + }; + defer ctx.cmds.deinit(ctx.alloc); + +diff --git a/src/vxfw/Spinner.zig b/src/vxfw/Spinner.zig +index 6619522..eb3d8ba 100644 +--- a/src/vxfw/Spinner.zig ++++ b/src/vxfw/Spinner.zig +@@ -20,11 +20,11 @@ frame: u4 = 0, + was_spinning: std.atomic.Value(bool) = .{ .raw = false }, + + /// Start, or add one, to the spinner counter. Thread safe. +-pub fn start(self: *Spinner) ?vxfw.Command { ++pub fn start(self: *Spinner, io: std.Io) ?vxfw.Command { + self.was_spinning.store(true, .unordered); + const count = self.count.fetchAdd(1, .monotonic); + if (count == 0) { +- return vxfw.Tick.in(time_lapse, self.widget()); ++ return vxfw.Tick.in(io, time_lapse, self.widget()); + } + return null; + } +@@ -111,13 +111,13 @@ test Spinner { + // Start the spinner. This (maybe) returns a Tick command to schedule the next frame. If the + // spinner is already running, no command is returned. Calling start is thread safe. The + // returned command can be added to an EventContext to schedule the frame +- const maybe_cmd = spinner.start(); ++ const maybe_cmd = spinner.start(std.testing.io); + try std.testing.expect(maybe_cmd != null); + try std.testing.expect(maybe_cmd.? == .tick); + try std.testing.expectEqual(1, spinner.count.load(.unordered)); + + // If we call start again, we won't get another command but our counter will go up +- const maybe_cmd2 = spinner.start(); ++ const maybe_cmd2 = spinner.start(std.testing.io); + try std.testing.expect(maybe_cmd2 == null); + try std.testing.expectEqual(2, spinner.count.load(.unordered)); + +@@ -126,6 +126,7 @@ test Spinner { + var ctx: vxfw.EventContext = .{ + .alloc = arena.allocator(), + .cmds = .empty, ++ .io = std.testing.io, + }; + + // The event loop handles the tick event and calls us back with a .tick event. If we should keep +diff --git a/src/vxfw/SplitView.zig b/src/vxfw/SplitView.zig +index 9a355a0..591e5e9 100644 +--- a/src/vxfw/SplitView.zig ++++ b/src/vxfw/SplitView.zig +@@ -229,6 +229,7 @@ test SplitView { + var ctx: vxfw.EventContext = .{ + .alloc = arena.allocator(), + .cmds = .empty, ++ .io = std.testing.io, + }; + try split_widget.handleEvent(&ctx, .{ .mouse = mouse }); + // We should get a command to change the mouse shape +diff --git a/src/vxfw/Text.zig b/src/vxfw/Text.zig +index 1f76cba..a7b71bb 100644 +--- a/src/vxfw/Text.zig ++++ b/src/vxfw/Text.zig +@@ -223,7 +223,7 @@ pub const SoftwrapIterator = struct { + // Advance the hard iterator + if (self.index == self.line.len) { + self.line = self.hard_iter.next() orelse return null; +- self.line = std.mem.trimRight(u8, self.line, " \t"); ++ self.line = std.mem.trimEnd(u8, self.line, " \t"); + self.index = 0; + } + +@@ -237,7 +237,7 @@ pub const SoftwrapIterator = struct { + if (self.ctx.max.width) |max| { + if (cur_width + next_width > max) { + // Trim the word to see if it can fit on a line by itself +- const trimmed = std.mem.trimLeft(u8, word, " \t"); ++ const trimmed = std.mem.trimStart(u8, word, " \t"); + const trimmed_bytes = word.len - trimmed.len; + // The number of bytes we trimmed is equal to the reduction in length + const trimmed_width = next_width - trimmed_bytes; +diff --git a/src/vxfw/TextField.zig b/src/vxfw/TextField.zig +index 6ad108b..e5a4d5b 100644 +--- a/src/vxfw/TextField.zig ++++ b/src/vxfw/TextField.zig +@@ -346,7 +346,7 @@ pub fn deleteAfterCursor(self: *TextField) void { + /// Moves the cursor backward by words. If the character before the cursor is a space, the cursor is + /// positioned just after the next previous space + pub fn moveBackwardWordwise(self: *TextField) void { +- const trimmed = std.mem.trimRight(u8, self.buf.firstHalf(), " "); ++ const trimmed = std.mem.trimEnd(u8, self.buf.firstHalf(), " "); + const idx = if (std.mem.lastIndexOfScalar(u8, trimmed, ' ')) |last| + last + 1 + else +@@ -562,6 +562,7 @@ test TextField { + var ctx: vxfw.EventContext = .{ + .alloc = arena.allocator(), + .cmds = .empty, ++ .io = std.testing.io, + }; + + // Enough boiler plate...Create the text field +diff --git a/src/vxfw/vxfw.zig b/src/vxfw/vxfw.zig +index 1510e71..f768d56 100644 +--- a/src/vxfw/vxfw.zig ++++ b/src/vxfw/vxfw.zig +@@ -61,8 +61,8 @@ pub const Tick = struct { + return lhs.deadline_ms > rhs.deadline_ms; + } + +- pub fn in(ms: u32, widget: Widget) Command { +- const now = std.time.milliTimestamp(); ++ pub fn in(io: std.Io, ms: u32, widget: Widget) Command { ++ const now = std.Io.Clock.now(.real, io).toMilliseconds(); + return .{ .tick = .{ + .deadline_ms = now + ms, + .widget = widget, +@@ -101,6 +101,7 @@ pub const EventContext = struct { + phase: Phase = .at_target, + alloc: Allocator, + cmds: CommandList, ++ io: std.Io, + + /// The event was handled, do not pass it on + consume_event: bool = false, +@@ -120,7 +121,7 @@ pub const EventContext = struct { + } + + pub fn tick(self: *EventContext, ms: u32, widget: Widget) Allocator.Error!void { +- try self.addCmd(Tick.in(ms, widget)); ++ try self.addCmd(Tick.in(self.io, ms, widget)); + } + + pub fn consumeAndRedraw(self: *EventContext) void { +@@ -541,10 +542,10 @@ test "All widgets have a doctest and refAllDecls test" { + // it easy to fail CI early, or spot bad tests vs non-existant tests + const excludes = &[_][]const u8{ "vxfw.zig", "App.zig" }; + +- var cwd = try std.fs.cwd().openDir("./src/vxfw", .{ .iterate = true }); ++ var cwd = try std.Io.Dir.cwd().openDir(std.testing.io, "./src/vxfw", .{ .iterate = true }); + var iter = cwd.iterate(); +- defer cwd.close(); +- outer: while (try iter.next()) |file| { ++ defer cwd.close(std.testing.io); ++ outer: while (try iter.next(std.testing.io)) |file| { + if (file.kind != .file) continue; + for (excludes) |ex| if (std.mem.eql(u8, ex, file.name)) continue :outer; + +@@ -552,7 +553,7 @@ test "All widgets have a doctest and refAllDecls test" { + file.name[0..idx] + else + continue; +- const data = try cwd.readFileAllocOptions(std.testing.allocator, file.name, 10_000_000, null, .of(u8), 0x00); ++ const data = try cwd.readFileAllocOptions(std.testing.io, file.name, std.testing.allocator, std.Io.Limit.limited(10_000_000), .of(u8), 0x00); + defer std.testing.allocator.free(data); + var ast = try std.zig.Ast.parse(std.testing.allocator, data, .zig); + defer ast.deinit(std.testing.allocator); +diff --git a/src/widgets/TextInput.zig b/src/widgets/TextInput.zig +index a6236a4..6b8871b 100644 +--- a/src/widgets/TextInput.zig ++++ b/src/widgets/TextInput.zig +@@ -266,7 +266,7 @@ pub fn deleteAfterCursor(self: *TextInput) void { + /// Moves the cursor backward by words. If the character before the cursor is a space, the cursor is + /// positioned just after the next previous space + pub fn moveBackwardWordwise(self: *TextInput) void { +- const trimmed = std.mem.trimRight(u8, self.buf.firstHalf(), " "); ++ const trimmed = std.mem.trimEnd(u8, self.buf.firstHalf(), " "); + const idx = if (std.mem.lastIndexOfScalar(u8, trimmed, ' ')) |last| + last + 1 + else +diff --git a/src/widgets/TextView.zig b/src/widgets/TextView.zig +index 45f9e44..ee326cf 100644 +--- a/src/widgets/TextView.zig ++++ b/src/widgets/TextView.zig +@@ -79,52 +79,22 @@ pub const Buffer = struct { + var cols: usize = self.last_cols; + var iter = uucode.grapheme.Iterator(uucode.utf8.Iterator).init(.init(content.bytes)); + +- var grapheme_start: usize = 0; +- var prev_break: bool = true; +- +- while (iter.next()) |result| { +- if (prev_break and !result.is_break) { +- // Start of a new grapheme +- const cp_len: usize = std.unicode.utf8CodepointSequenceLength(result.cp) catch 1; +- grapheme_start = iter.i - cp_len; +- } +- +- if (result.is_break) { +- // End of a grapheme +- const grapheme_end = iter.i; +- const grapheme_len = grapheme_end - grapheme_start; +- +- try self.grapheme.append(allocator, .{ +- .len = @intCast(grapheme_len), +- .offset = @intCast(self.content.items.len + grapheme_start), +- }); +- +- const cluster = content.bytes[grapheme_start..grapheme_end]; +- if (std.mem.eql(u8, cluster, "\n")) { +- self.cols = @max(self.cols, cols); +- cols = 0; +- } else { +- // Calculate width using gwidth +- const w = vaxis.gwidth.gwidth(cluster, .unicode); +- cols +|= w; +- } +- +- grapheme_start = grapheme_end; +- } +- prev_break = result.is_break; +- } +- +- // Flush the last grapheme if we ended mid-cluster +- if (!prev_break and grapheme_start < content.bytes.len) { +- const grapheme_len = content.bytes.len - grapheme_start; ++ while (iter.nextGrapheme()) |grapheme| { ++ const grapheme_start = grapheme.start; ++ const grapheme_end = grapheme.end; ++ const grapheme_len = grapheme_end - grapheme_start; + + try self.grapheme.append(allocator, .{ + .len = @intCast(grapheme_len), + .offset = @intCast(self.content.items.len + grapheme_start), + }); + +- const cluster = content.bytes[grapheme_start..]; +- if (!std.mem.eql(u8, cluster, "\n")) { ++ const cluster = content.bytes[grapheme_start..grapheme_end]; ++ if (std.mem.eql(u8, cluster, "\n")) { ++ self.cols = @max(self.cols, cols); ++ cols = 0; ++ } else { ++ // Calculate width using gwidth + const w = vaxis.gwidth.gwidth(cluster, .unicode); + cols +|= w; + } +diff --git a/src/widgets/terminal/Command.zig b/src/widgets/terminal/Command.zig +index 0cd24f7..ed41553 100644 +--- a/src/widgets/terminal/Command.zig ++++ b/src/widgets/terminal/Command.zig +@@ -14,7 +14,7 @@ working_directory: ?[]const u8, + // Set after spawn() + pid: ?std.posix.pid_t = null, + +-env_map: *const std.process.EnvMap, ++env_map: *const std.process.Environ.Map, + + pty: Pty, + +@@ -43,8 +43,8 @@ pub fn spawn(self: *Command, allocator: std.mem.Allocator) !void { + try posix.dup2(self.pty.tty.handle, std.posix.STDOUT_FILENO); + try posix.dup2(self.pty.tty.handle, std.posix.STDERR_FILENO); + +- self.pty.tty.close(); +- if (self.pty.pty.handle > 2) self.pty.pty.close(); ++ _ = std.os.linux.close(self.pty.tty.handle); ++ if (self.pty.pty.handle > 2) _ = std.os.linux.close(self.pty.pty.handle); + + if (self.working_directory) |wd| { + try std.posix.chdir(wd); +@@ -78,8 +78,8 @@ pub fn spawn(self: *Command, allocator: std.mem.Allocator) !void { + fn handleSigChild(_: c_int) callconv(.c) void { + const result = std.posix.waitpid(-1, 0); + +- Terminal.global_vt_mutex.lock(); +- defer Terminal.global_vt_mutex.unlock(); ++ if (!Terminal.global_vt_mutex.tryLock()) return; ++ defer Terminal.global_vt_mutex.unlock(Terminal.global_vt_io); + if (Terminal.global_vts) |vts| { + var vt = vts.get(result.pid) orelse return; + vt.event_queue.push(.exited); +@@ -97,7 +97,7 @@ pub fn kill(self: *Command) void { + /// hash map plus options. + fn createEnvironFromMap( + arena: std.mem.Allocator, +- map: *const std.process.EnvMap, ++ map: *const std.process.Environ.Map, + ) ![:null]?[*:0]u8 { + const envp_count: usize = map.count(); + +diff --git a/src/widgets/terminal/Pty.zig b/src/widgets/terminal/Pty.zig +index d9f4138..4c67328 100644 +--- a/src/widgets/terminal/Pty.zig ++++ b/src/widgets/terminal/Pty.zig +@@ -7,8 +7,8 @@ const Winsize = @import("../../main.zig").Winsize; + + const posix = std.posix; + +-pty: std.fs.File, +-tty: std.fs.File, ++pty: std.Io.File, ++tty: std.Io.File, + + /// opens a new tty/pty pair + pub fn init() !Pty { +@@ -19,9 +19,9 @@ pub fn init() !Pty { + } + + /// closes the tty and pty +-pub fn deinit(self: Pty) void { +- self.pty.close(); +- self.tty.close(); ++pub fn deinit(self: Pty, io: std.Io) void { ++ self.pty.close(io); ++ self.tty.close(io); + } + + /// sets the size of the pty +@@ -37,7 +37,7 @@ pub fn setSize(self: Pty, ws: Winsize) !void { + } + + fn openPtyLinux() !Pty { +- const p = try posix.open("/dev/ptmx", .{ .ACCMODE = .RDWR, .NOCTTY = true }, 0); ++ const p = try posix.openat(posix.AT.FDCWD, "/dev/ptmx", .{ .ACCMODE = .RDWR, .NOCTTY = true }, 0); + errdefer posix.close(p); + + // unlockpt +@@ -50,7 +50,7 @@ fn openPtyLinux() !Pty { + const sname = try std.fmt.bufPrint(&buf, "/dev/pts/{d}", .{n}); + std.log.debug("pts: {s}", .{sname}); + +- const t = try posix.open(sname, .{ .ACCMODE = .RDWR, .NOCTTY = true }, 0); ++ const t = try posix.openat(posix.AT.FDCWD, sname, .{ .ACCMODE = .RDWR, .NOCTTY = true }, 0); + + return .{ + .pty = .{ .handle = p }, +diff --git a/src/widgets/terminal/Terminal.zig b/src/widgets/terminal/Terminal.zig +index 36d61d7..786ca85 100644 +--- a/src/widgets/terminal/Terminal.zig ++++ b/src/widgets/terminal/Terminal.zig +@@ -43,21 +43,23 @@ pub const InputEvent = union(enum) { + key_press: vaxis.Key, + }; + +-pub var global_vt_mutex: std.Thread.Mutex = .{}; ++pub var global_vt_mutex: std.Io.Mutex = .init; ++var global_vt_io: std.Io = undefined; + pub var global_vts: ?std.AutoHashMap(i32, *Terminal) = null; + pub var global_sigchild_installed: bool = false; + + allocator: std.mem.Allocator, ++io: std.Io, + scrollback_size: u16, + + pty: Pty, +-pty_writer: std.fs.File.Writer, ++pty_writer: std.Io.File.Writer, + cmd: Command, + thread: ?std.Thread = null, + + /// the screen we draw from + front_screen: Screen, +-front_mutex: std.Thread.Mutex = .{}, ++front_mutex: std.Io.Mutex = .init, + + /// the back screens + back_screen: *Screen = undefined, +@@ -65,7 +67,7 @@ back_screen_pri: Screen, + back_screen_alt: Screen, + // only applies to primary screen + scroll_offset: usize = 0, +-back_mutex: std.Thread.Mutex = .{}, ++back_mutex: std.Io.Mutex = .init, + // dirty is protected by back_mutex. Only access this field when you hold that mutex + dirty: bool = false, + +@@ -79,14 +81,15 @@ working_directory: std.ArrayList(u8) = .empty, + + last_printed: []const u8 = "", + +-event_queue: Queue = .{}, ++event_queue: Queue, + + /// initialize a Terminal. This sets the size of the underlying pty and allocates the sizes of the + /// screen + pub fn init( + allocator: std.mem.Allocator, ++ io: std.Io, + argv: []const []const u8, +- env: *const std.process.EnvMap, ++ env: *const std.process.Environ.Map, + opts: Options, + write_buf: []u8, + ) !Terminal { +@@ -107,8 +110,10 @@ pub fn init( + while (col < opts.winsize.cols) : (col += 8) { + try tabs.append(allocator, col); + } ++ global_vt_io = io; + return .{ + .allocator = allocator, ++ .io = io, + .pty = pty, + .pty_writer = pty.pty.writerStreaming(write_buf), + .cmd = cmd, +@@ -117,6 +122,7 @@ pub fn init( + .back_screen_pri = try Screen.init(allocator, opts.winsize.cols, opts.winsize.rows + opts.scrollback_size), + .back_screen_alt = try Screen.init(allocator, opts.winsize.cols, opts.winsize.rows), + .tab_stops = tabs, ++ .event_queue = .{ .io = io }, + }; + } + +@@ -125,8 +131,8 @@ pub fn deinit(self: *Terminal) void { + self.should_quit = true; + + pid: { +- global_vt_mutex.lock(); +- defer global_vt_mutex.unlock(); ++ global_vt_mutex.lockUncancelable(global_vt_io); ++ defer global_vt_mutex.unlock(global_vt_io); + var vts = global_vts orelse break :pid; + if (self.cmd.pid) |pid| + _ = vts.remove(pid); +@@ -143,7 +149,7 @@ pub fn deinit(self: *Terminal) void { + thread.join(); + self.thread = null; + } +- self.pty.deinit(); ++ self.pty.deinit(self.io); + self.front_screen.deinit(self.allocator); + self.back_screen_pri.deinit(self.allocator); + self.back_screen_alt.deinit(self.allocator); +@@ -162,16 +168,15 @@ pub fn spawn(self: *Terminal) !void { + if (self.cmd.working_directory) |pwd| { + try self.working_directory.appendSlice(self.allocator, pwd); + } else { +- const pwd = std.fs.cwd(); + var buffer: [std.fs.max_path_bytes]u8 = undefined; +- const out_path = try std.os.getFdPath(pwd.fd, &buffer); ++ const out_path = try std.posix.readlinkZ("/proc/self/cwd", &buffer); + try self.working_directory.appendSlice(self.allocator, out_path); + } + + { + // add to our global list +- global_vt_mutex.lock(); +- defer global_vt_mutex.unlock(); ++ global_vt_mutex.lockUncancelable(global_vt_io); ++ defer global_vt_mutex.unlock(global_vt_io); + if (global_vts == null) + global_vts = std.AutoHashMap(i32, *Terminal).init(self.allocator); + if (self.cmd.pid) |pid| +@@ -190,8 +195,8 @@ pub fn resize(self: *Terminal, ws: Winsize) !void { + ws.rows == self.front_screen.height) + return; + +- self.back_mutex.lock(); +- defer self.back_mutex.unlock(); ++ self.back_mutex.lockUncancelable(self.io); ++ defer self.back_mutex.unlock(self.io); + + self.front_screen.deinit(self.allocator); + self.front_screen = try Screen.init(self.allocator, ws.cols, ws.rows); +@@ -206,7 +211,7 @@ pub fn resize(self: *Terminal, ws: Winsize) !void { + + pub fn draw(self: *Terminal, allocator: std.mem.Allocator, win: vaxis.Window) !void { + if (self.back_mutex.tryLock()) { +- defer self.back_mutex.unlock(); ++ defer self.back_mutex.unlock(self.io); + // We keep this as a separate condition so we don't deadlock by obtaining the lock but not + // having sync + if (!self.mode.sync) { +@@ -249,7 +254,7 @@ pub fn get_pty_writer(self: *Terminal) *std.Io.Writer { + return &self.pty_writer.interface; + } + +-fn reader(self: *const Terminal, buf: []u8) std.fs.File.Reader { ++fn reader(self: *const Terminal, buf: []u8) std.Io.File.Reader { + return self.pty.pty.readerStreaming(buf); + } + +@@ -265,8 +270,8 @@ fn run(self: *Terminal) !void { + + while (!self.should_quit) { + const event = try parser.parseReader(&reader_.interface); +- self.back_mutex.lock(); +- defer self.back_mutex.unlock(); ++ self.back_mutex.lockUncancelable(self.io); ++ defer self.back_mutex.unlock(self.io); + + if (!self.dirty and self.event_queue.tryPush(.redraw)) + self.dirty = true; diff --git a/third_party/libvaxis/fixes.patch b/third_party/libvaxis/fixes.patch index 6a33c47e..aca9c136 100644 --- a/third_party/libvaxis/fixes.patch +++ b/third_party/libvaxis/fixes.patch @@ -1,38 +1,33 @@ -diff --git a/src/Loop.zig b/src/Loop.zig ---- a/src/Loop.zig -+++ b/src/Loop.zig -@@ -352,7 +352,7 @@ - } - }, - .cap_da1 => { -- std.Io.futexWake(vx.io, std.atomic.Value(u32), &vx.query_futex, 10); -+ std.Io.futexWake(vx.io, u32, &vx.query_futex.raw, 10); - vx.queries_done.store(true, .unordered); - }, - .mouse => |mouse| { -@@ -480,7 +480,7 @@ - vx.caps.multi_cursor = true; - }, - .cap_da1 => { -- std.Io.futexWake(vx.io, std.atomic.Value(u32), &vx.query_futex, 10); -+ std.Io.futexWake(vx.io, u32, &vx.query_futex.raw, 10); - vx.queries_done.store(true, .unordered); - }, - .winsize => |winsize| { +diff --git a/examples/table.zig b/examples/table.zig +index c489828..16aefdc 100644 +--- a/examples/table.zig ++++ b/examples/table.zig +@@ -20,8 +20,8 @@ pub fn main(init: std.process.Init) !void { + // Users set up below the main function + const users_buf = try alloc.dupe(User, users[0..]); + +- var buffer: [1024]u8 = undefined; +- var tty = try vaxis.Tty.init(init.io, &buffer); ++ // var buffer: [1024]u8 = undefined; ++ var tty = try vaxis.Tty.init(init.io); + defer tty.deinit(); + const tty_writer = tty.writer(); + var vx = try vaxis.init(alloc, .{ diff --git a/src/Vaxis.zig b/src/Vaxis.zig +index 2e24e9b..2b2ee49 100644 --- a/src/Vaxis.zig +++ b/src/Vaxis.zig -@@ -137,14 +137,13 @@ +@@ -127,14 +127,13 @@ pub fn deinit(self: *Vaxis, alloc: ?std.mem.Allocator, tty: *IoWriter) void { } } -/// resets enabled features, sends cursor to home and clears below cursor --pub fn resetState(self: *Vaxis, tty: *std.Io.Writer) !void { +-pub fn resetState(self: *Vaxis, tty: *IoWriter) !void { - // always show the cursor on state reset +/// Resets enabled terminal features without clearing screen content. +/// Use this for non-interactive / single-shot rendering where the +/// output should remain visible after exit. -+pub fn resetModes(self: *Vaxis, tty: *std.Io.Writer) !void { ++pub fn resetModes(self: *Vaxis, tty: *IoWriter) !void { tty.writeAll(ctlseqs.show_cursor) catch {}; tty.writeAll(ctlseqs.sgr_reset) catch {}; if (self.screen.cursor_shape != .default) { @@ -41,81 +36,51 @@ diff --git a/src/Vaxis.zig b/src/Vaxis.zig tty.print(ctlseqs.cursor_shape, .{@intFromEnum(Cell.CursorShape.default)}) catch {}; } if (self.state.kitty_keyboard) { -@@ -157,6 +156,32 @@ +@@ -147,18 +146,6 @@ pub fn resetState(self: *Vaxis, tty: *IoWriter) !void { if (self.state.bracketed_paste) { try self.setBracketedPaste(tty, false); } -+ if (self.state.color_scheme_updates) { -+ try tty.writeAll(ctlseqs.color_scheme_reset); -+ self.state.color_scheme_updates = false; -+ } -+ if (self.state.in_band_resize) { -+ try tty.writeAll(ctlseqs.in_band_resize_reset); -+ self.state.in_band_resize = false; -+ } -+ if (self.state.changed_default_fg) { -+ try tty.writeAll(ctlseqs.osc10_reset); -+ self.state.changed_default_fg = false; -+ } -+ if (self.state.changed_default_bg) { -+ try tty.writeAll(ctlseqs.osc11_reset); -+ self.state.changed_default_bg = false; -+ } -+ if (self.state.changed_cursor_color) { -+ try tty.writeAll(ctlseqs.osc12_reset); -+ self.state.changed_cursor_color = false; -+ } +- if (self.state.alt_screen) { +- try tty.writeAll(ctlseqs.home); +- try tty.writeAll(ctlseqs.erase_below_cursor); +- try self.exitAltScreen(tty); +- } else { +- try tty.writeByte('\r'); +- var i: u16 = 0; +- while (i < self.state.cursor.row) : (i += 1) { +- try tty.writeAll(ctlseqs.ri); +- } +- try tty.writeAll(ctlseqs.erase_below_cursor); +- } + if (self.state.color_scheme_updates) { + try tty.writeAll(ctlseqs.color_scheme_reset); + self.state.color_scheme_updates = false; +@@ -179,7 +166,24 @@ pub fn resetState(self: *Vaxis, tty: *IoWriter) !void { + try tty.writeAll(ctlseqs.osc12_reset); + self.state.changed_cursor_color = false; + } + try tty.flush(); +} -+ + +/// resets enabled features, sends cursor to home and clears below cursor -+pub fn resetState(self: *Vaxis, tty: *std.Io.Writer) !void { ++pub fn resetState(self: *Vaxis, tty: *IoWriter) !void { + self.resetModes(tty) catch {}; - if (self.state.alt_screen) { - try tty.writeAll(ctlseqs.home); - try tty.writeAll(ctlseqs.erase_below_cursor); -@@ -169,27 +194,6 @@ - } - try tty.writeAll(ctlseqs.erase_below_cursor); - } -- if (self.state.color_scheme_updates) { -- try tty.writeAll(ctlseqs.color_scheme_reset); -- self.state.color_scheme_updates = false; -- } -- if (self.state.in_band_resize) { -- try tty.writeAll(ctlseqs.in_band_resize_reset); -- self.state.in_band_resize = false; -- } -- if (self.state.changed_default_fg) { -- try tty.writeAll(ctlseqs.osc10_reset); -- self.state.changed_default_fg = false; -- } -- if (self.state.changed_default_bg) { -- try tty.writeAll(ctlseqs.osc11_reset); -- self.state.changed_default_bg = false; -- } -- if (self.state.changed_cursor_color) { -- try tty.writeAll(ctlseqs.osc12_reset); -- self.state.changed_cursor_color = false; -- } -- ++ if (self.state.alt_screen) { ++ try tty.writeAll(ctlseqs.home); ++ try tty.writeAll(ctlseqs.erase_below_cursor); ++ try self.exitAltScreen(tty); ++ } else { ++ try tty.writeByte('\r'); ++ var i: u16 = 0; ++ while (i < self.state.cursor.row) : (i += 1) { ++ try tty.writeAll(ctlseqs.ri); ++ } ++ try tty.writeAll(ctlseqs.erase_below_cursor); ++ } try tty.flush(); } -@@ -272,9 +276,9 @@ - try self.queryTerminalSend(tty); - try std.Io.futexWaitTimeout( - self.io, -- atomic.Value(u32), -- &self.query_futex, -- .init(0), -+ u32, -+ &self.query_futex.raw, -+ 0, - .{ - .duration = .{ - .clock = .real, -@@ -980,19 +984,19 @@ +@@ -943,19 +947,19 @@ pub fn transmitLocalImagePath( switch (format) { .rgb => { try tty.print( @@ -138,7 +103,7 @@ diff --git a/src/Vaxis.zig b/src/Vaxis.zig .{ id, medium_char, encoded }, ); }, -@@ -1028,7 +1032,7 @@ +@@ -991,7 +995,7 @@ pub fn transmitPreEncodedImage( if (bytes.len < 4096) { try tty.print( @@ -147,7 +112,7 @@ diff --git a/src/Vaxis.zig b/src/Vaxis.zig .{ fmt, width, -@@ -1041,14 +1045,14 @@ +@@ -1004,14 +1008,14 @@ pub fn transmitPreEncodedImage( var n: usize = 4096; try tty.print( @@ -164,22 +129,11 @@ diff --git a/src/Vaxis.zig b/src/Vaxis.zig .{ m, bytes[n..end], -diff --git a/src/c_api.zig b/src/c_api.zig ---- a/src/c_api.zig -+++ b/src/c_api.zig -@@ -951,7 +951,7 @@ - .cap_color_scheme_updates => vx.caps.color_scheme_updates = true, - .cap_multi_cursor => vx.caps.multi_cursor = true, - .cap_da1 => { -- std.Io.futexWake(vx.io, std.atomic.Value(u32), &vx.query_futex, 10); -+ std.Io.futexWake(vx.io, u32, &vx.query_futex.raw, 10); - vx.queries_done.store(true, .unordered); - }, - .winsize => { diff --git a/src/ctlseqs.zig b/src/ctlseqs.zig +index ab415c6..4baa496 100644 --- a/src/ctlseqs.zig +++ b/src/ctlseqs.zig -@@ -138,8 +138,8 @@ +@@ -138,8 +138,8 @@ pub const osc52_clipboard_copy = "\x1b]52;c;{s}\x1b\\"; pub const osc52_clipboard_request = "\x1b]52;c;?\x1b\\"; // Kitty graphics @@ -190,10 +144,118 @@ diff --git a/src/ctlseqs.zig b/src/ctlseqs.zig pub const kitty_graphics_closing = ",C=1\x1b\\"; // Color control sequences +diff --git a/src/tty.zig b/src/tty.zig +index c1da795..7f6a303 100644 +--- a/src/tty.zig ++++ b/src/tty.zig +@@ -52,6 +52,10 @@ pub const PosixTty = struct { + + var handler_installed: bool = false; + ++ /// Atomic flag set by the SIGWINCH signal handler. Checked and cleared ++ /// by pollWinch() from a non-signal context to avoid deadlocks. ++ var winch_pending: std.atomic.Value(bool) = .{ .raw = false }; ++ + /// initializes a Tty instance by opening /dev/tty and "making it raw". A + /// signal handler is installed for SIGWINCH. No callbacks are installed, be + /// sure to register a callback when initializing the event loop +@@ -129,7 +133,20 @@ pub const PosixTty = struct { + } + + fn handleWinch(_: posix.SIG) callconv(.c) void { +- if (!handler_mutex.tryLock()) return; ++ // Only set an atomic flag. Calling mutex.lock() or queue.push() ++ // from a signal handler is not async-signal-safe and causes ++ // deadlocks when the signal is delivered to a thread that already ++ // holds one of those locks. Registered handlers are dispatched ++ // from pollWinch() instead. ++ winch_pending.store(true, .release); ++ } ++ ++ /// Check if a SIGWINCH was received and dispatch registered handlers. ++ /// Must be called from a non-signal context (e.g., the main event loop) ++ /// to avoid the async-signal-safety issues with the old approach. ++ pub fn pollWinch() void { ++ if (!winch_pending.swap(false, .acquire)) return; ++ handler_mutex.lockUncancelable(handler_io); + defer handler_mutex.unlock(handler_io); + var i: usize = 0; + while (i < handler_idx) : (i += 1) { +diff --git a/src/vxfw/App.zig b/src/vxfw/App.zig +index fc3962f..2661cf4 100644 +--- a/src/vxfw/App.zig ++++ b/src/vxfw/App.zig +@@ -1,4 +1,5 @@ + const std = @import("std"); ++const builtin = @import("builtin"); + const vaxis = @import("../main.zig"); + const vxfw = @import("vxfw.zig"); + +@@ -36,8 +37,8 @@ fn init(allocator: Allocator, io: std.Io) !App { + .kitty_keyboard_flags = .{ + .report_events = true, + }, + }), +- .timers = std.ArrayList(vxfw.Tick){}, ++ .timers = .empty, + .wants_focus = null, + .io = io, + }; +@@ -102,7 +103,7 @@ + var ctx: vxfw.EventContext = .{ + .alloc = self.allocator, + .phase = .capturing, +- .cmds = vxfw.CommandList{}, ++ .cmds = .empty, + .io = io, + .consume_event = false, + .redraw = false, +@@ -123,6 +124,13 @@ pub fn run(self: *App, widget: vxfw.Widget, opts: Options) anyerror!void { + + try self.checkTimers(&ctx); + ++ // Poll for pending SIGWINCH outside of queue lock. ++ // The signal handler only sets an atomic flag; we dispatch ++ // the registered callbacks here where mutex operations are safe. ++ if (builtin.os.tag != .windows and !builtin.is_test) { ++ if (!vx.state.in_band_resize) vaxis.Tty.pollWinch(); ++ } ++ + { + loop.queue.lock(); + defer loop.queue.unlock(); +@@ -342,7 +350,7 @@ + // For mouse events we store the last frame and use that for hit testing + const last_frame = surface; + +- var hits = std.ArrayList(vxfw.HitResult){}; ++ var hits: std.ArrayList(vxfw.HitResult) = .empty; + defer hits.deinit(app.allocator); + const sub: vxfw.SubSurface = .{ + .origin = .{ .row = 0, .col = 0 }, +@@ -405,7 +413,7 @@ + const last_frame = self.last_frame; + self.mouse = mouse; + +- var hits = std.ArrayList(vxfw.HitResult){}; ++ var hits: std.ArrayList(vxfw.HitResult) = .empty; + defer hits.deinit(app.allocator); + const sub: vxfw.SubSurface = .{ + .origin = .{ .row = 0, .col = 0 }, +@@ -524,7 +532,7 @@ + return .{ + .root = root, + .focused_widget = root, +- .path_to_focused = std.ArrayList(Widget){}, ++ .path_to_focused = .empty, + }; + } + diff --git a/src/vxfw/Padding.zig b/src/vxfw/Padding.zig +index e60162f..6e5f40b 100644 --- a/src/vxfw/Padding.zig +++ b/src/vxfw/Padding.zig -@@ -6,7 +6,7 @@ +@@ -6,7 +6,7 @@ const Allocator = std.mem.Allocator; const vxfw = @import("vxfw.zig"); const Padding = @This(); @@ -202,7 +264,7 @@ diff --git a/src/vxfw/Padding.zig b/src/vxfw/Padding.zig left: u16 = 0, right: u16 = 0, top: u16 = 0, -@@ -54,10 +54,6 @@ +@@ -54,10 +54,6 @@ fn typeErasedDrawFn(ptr: *anyopaque, ctx: vxfw.DrawContext) Allocator.Error!vxfw pub fn draw(self: *const Padding, ctx: vxfw.DrawContext) Allocator.Error!vxfw.Surface { const pad = self.padding; diff --git a/third_party/libvaxis/repo.bzl b/third_party/libvaxis/repo.bzl index 7473e880..8305d1c0 100644 --- a/third_party/libvaxis/repo.bzl +++ b/third_party/libvaxis/repo.bzl @@ -4,8 +4,8 @@ def repo(): git_repository( name = "libvaxis", remote = "https://github.com/rockorager/libvaxis.git", - commit = "173a890d1394946b5d7623c66cd34bcd36d8eeb8", + commit = "a3ae1d53feeeeaeb6218de3d38837559811acae4", build_file = "//third_party/libvaxis:libvaxis.bazel", - patches = ["//third_party/libvaxis:fixes.patch"], + patches = ["//third_party/libvaxis:bump-zig.patch", "//third_party/libvaxis:fixes.patch"], patch_args = ["-p1"], ) diff --git a/third_party/translate-c/repo.bzl b/third_party/translate-c/repo.bzl index dfd8835d..8315b2af 100644 --- a/third_party/translate-c/repo.bzl +++ b/third_party/translate-c/repo.bzl @@ -3,9 +3,8 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") def repo(): git_repository( name = "translate-c", - # Use the ZML GitHub mirror because Codeberg is unreliable. + # Use ZML Github mirror of translate-c because Codeber is too unstable remote = "https://github.com/zml/translate-c", - # 2.0.0 - commit = "0da7a16c3235b935b82421646076e0657cda21f6", + commit = "46b5609b5ac4c0a896217d1d984f3ae50e4810b5", build_file = Label("//third_party/translate-c:translate-c.bazel"), ) diff --git a/third_party/translate-c/translate-c.bazel b/third_party/translate-c/translate-c.bazel index 7de2a5eb..b2ff9c98 100644 --- a/third_party/translate-c/translate-c.bazel +++ b/third_party/translate-c/translate-c.bazel @@ -1,12 +1,4 @@ load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_configure_binary", "zig_library") -load("@zml//bazel:zig_options.bzl", "zig_option", "zig_options") - -zig_option(name = "debug_allocations", bool = False) - -zig_options( - name = "build_options", - settings = [":debug_allocations"], -) zig_library( name = "c_builtins", @@ -27,7 +19,6 @@ zig_binary( main = "src/main.zig", linkopts = ["-lc"], deps = [ - ":build_options", ":c_builtins", ":helpers", "@arocc//:aro", diff --git a/third_party/uucode/repo.bzl b/third_party/uucode/repo.bzl index 5f7b1cbf..20a2f080 100644 --- a/third_party/uucode/repo.bzl +++ b/third_party/uucode/repo.bzl @@ -21,8 +21,7 @@ _uucode_repo = repository_rule( def repo(): _uucode_repo( name = "uucode", - # main branch - commit = "1fb73433bba5d93366c57f23ff2e9d7939746500", + commit = "f1c687f09174f9f11eda14d3e3ce497d68ab09a6", build_config = "//third_party/uucode:build_config.zig", build_file = "//third_party/uucode:uucode.bazel", config_storage_patch = "//third_party/uucode:config-storage-module.patch", diff --git a/third_party/uucode/uucode.bazel b/third_party/uucode/uucode.bazel index 4a3710f2..f2b5634d 100644 --- a/third_party/uucode/uucode.bazel +++ b/third_party/uucode/uucode.bazel @@ -8,7 +8,6 @@ zig_library( main = "src/config.zig", srcs = [ "src/components.zig", - "src/ucd_inputs.zig", "src/fields.zig", "src/multi_slice.zig", "src/quirks.zig", @@ -52,7 +51,6 @@ zig_library( main = "src/config.zig", srcs = [ "src/components.zig", - "src/ucd_inputs.zig", "src/fields.zig", "src/multi_slice.zig", "src/quirks.zig", diff --git a/third_party/zigimg/bump-zig.patch b/third_party/zigimg/bump-zig.patch new file mode 100644 index 00000000..b668dadc --- /dev/null +++ b/third_party/zigimg/bump-zig.patch @@ -0,0 +1,4491 @@ +diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml +index 400aee8..72ad6f9 100644 +--- a/.github/workflows/main.yml ++++ b/.github/workflows/main.yml +@@ -25,7 +25,7 @@ jobs: + - name: Setup Zig + uses: mlugg/setup-zig@v2 + with: +- version: 0.15.2 ++ version: 0.16.0-dev.2261+d6b3dd25a + - name: AST Verify + run: zig fmt --check --ast-check src/ + working-directory: zigimg +diff --git a/build.zig.zon b/build.zig.zon +index 3ff4816..536d37b 100644 +--- a/build.zig.zon ++++ b/build.zig.zon +@@ -1,7 +1,7 @@ + .{ + .name = .zigimg, + .fingerprint = 0x6d2b443cdaa8f7f3, +- .minimum_zig_version = "0.15.2", ++ .minimum_zig_version = "0.16.0-dev.2261+d6b3dd25a", + .version = "0.1.0", + .paths = .{ + "build.zig", +diff --git a/src/Image.zig b/src/Image.zig +index d67cd61..6472129 100644 +--- a/src/Image.zig ++++ b/src/Image.zig +@@ -71,7 +71,7 @@ pub const WriteError = Error || + std.mem.Allocator.Error || + io.ReadStream.Error || + io.WriteStream.Error || +- std.fs.File.OpenError || ++ std.Io.File.OpenError || + error{ EndOfStream, InvalidData, UnfinishedBits }; + + pub const ConvertError = Error || +@@ -101,7 +101,7 @@ + }; + + pub const Animation = struct { +- frames: FrameList = .{}, ++ frames: FrameList = .empty, + loop_count: i32 = AnimationLoopInfinite, + + pub const FrameList = std.ArrayList(AnimationFrame); +@@ -150,16 +150,16 @@ pub fn deinit(self: *Image, allocator: std.mem.Allocator) void { + } + + /// Detect which image format is used by the file path +-pub fn detectFormatFromFilePath(file_path: []const u8, read_buffer: []u8) !Format { +- var file = try std.fs.cwd().openFile(file_path, .{}); +- defer file.close(); ++pub fn detectFormatFromFilePath(std_io: std.Io, file_path: []const u8, read_buffer: []u8) !Format { ++ var file = try std.Io.Dir.openFile(std.Io.Dir.cwd(), std_io, file_path, .{}); ++ defer file.close(std_io); + +- return detectFormatFromFile(file, read_buffer); ++ return detectFormatFromFile(std_io, file, read_buffer); + } + + /// Detect which image format is used by the file +-pub fn detectFormatFromFile(file: std.fs.File, read_buffer: []u8) !Format { +- var read_stream = io.ReadStream.initFile(file, read_buffer); ++pub fn detectFormatFromFile(std_io: std.Io, file: std.Io.File, read_buffer: []u8) !Format { ++ var read_stream = io.ReadStream.initFile(std_io, file, read_buffer); + return internalDetectFormat(&read_stream); + } + +@@ -170,16 +170,16 @@ pub fn detectFormatFromMemory(buffer: []const u8) !Format { + } + + /// Load an image from a file path +-pub fn fromFilePath(allocator: std.mem.Allocator, file_path: []const u8, read_buffer: []u8) !Image { +- var file = try std.fs.cwd().openFile(file_path, .{}); +- defer file.close(); ++pub fn fromFilePath(allocator: std.mem.Allocator, std_io: std.Io, file_path: []const u8, read_buffer: []u8) !Image { ++ var file = try std.Io.Dir.openFile(std.Io.Dir.cwd(), std_io, file_path, .{}); ++ defer file.close(std_io); + +- return fromFile(allocator, file, read_buffer); ++ return fromFile(allocator, std_io, file, read_buffer); + } + +-/// Load an image from a standard library std.fs.File +-pub fn fromFile(allocator: std.mem.Allocator, file: std.fs.File, read_buffer: []u8) !Image { +- var read_stream = io.ReadStream.initFile(file, read_buffer); ++/// Load an image from a standard library std.Io.File ++pub fn fromFile(allocator: std.mem.Allocator, std_io: std.Io, file: std.Io.File, read_buffer: []u8) !Image { ++ var read_stream = io.ReadStream.initFile(std_io, file, read_buffer); + return internalRead(allocator, &read_stream); + } + +@@ -260,16 +260,16 @@ pub fn toManaged(self: Image, allocator: std.mem.Allocator) Managed { + } + + /// Write the image to an image format to the specified path +-pub fn writeToFilePath(self: Image, allocator: std.mem.Allocator, file_path: []const u8, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { +- var file = try std.fs.cwd().createFile(file_path, .{}); +- defer file.close(); ++pub fn writeToFilePath(self: Image, allocator: std.mem.Allocator, std_io: std.Io, file_path: []const u8, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { ++ var file = try std.Io.Dir.createFile(std.Io.Dir.cwd(), std_io, file_path, .{}); ++ defer file.close(std_io); + +- try self.writeToFile(allocator, file, write_buffer, encoder_options); ++ try self.writeToFile(allocator, std_io, file, write_buffer, encoder_options); + } + +-/// Write the image to an image format to the specified std.fs.File +-pub fn writeToFile(self: Image, allocator: std.mem.Allocator, file: std.fs.File, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { +- var write_stream = io.WriteStream.initFile(file, write_buffer); ++/// Write the image to an image format to the specified std.Io.File ++pub fn writeToFile(self: Image, allocator: std.mem.Allocator, std_io: std.Io, file: std.Io.File, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { ++ var write_stream = io.WriteStream.initFile(std_io, file, write_buffer); + + try self.internalWrite(allocator, &write_stream, encoder_options); + } +diff --git a/src/Image/Managed.zig b/src/Image/Managed.zig +index 608b1d6..1517e7c 100644 +--- a/src/Image/Managed.zig ++++ b/src/Image/Managed.zig +@@ -20,14 +20,16 @@ width: usize = 0, + height: usize = 0, + pixels: color.PixelStorage = .{ .invalid = void{} }, + animation: Animation = .{}, ++io: std.Io = undefined, + allocator: std.mem.Allocator = undefined, // Allocator needs to be last in order to be able to ptrCast to Image + + const Managed = @This(); + + /// Init an empty image with no pixel data +-pub fn init(allocator: std.mem.Allocator) Managed { ++pub fn init(allocator: std.mem.Allocator, io: std.Io) Managed { + return Managed{ + .allocator = allocator, ++ .io = io, + }; + } + +@@ -37,13 +39,13 @@ pub fn deinit(self: *Managed) void { + } + + /// Detect which image format is used by the file path +-pub fn detectFormatFromFilePath(file_path: []const u8, read_buffer: []u8) !Format { +- return Image.detectFormatFromFilePath(file_path, read_buffer); ++pub fn detectFormatFromFilePath(io: std.Io, file_path: []const u8, read_buffer: []u8) !Format { ++ return Image.detectFormatFromFilePath(io, file_path, read_buffer); + } + + /// Detect which image format is used by the file +-pub fn detectFormatFromFile(file: std.fs.File, read_buffer: []u8) !Format { +- return Image.detectFormatFromFile(file, read_buffer); ++pub fn detectFormatFromFile(io: std.Io, file: std.Io.File, read_buffer: []u8) !Format { ++ return Image.detectFormatFromFile(io, file, read_buffer); + } + + /// Detect which image format is used by the memory buffer +@@ -52,13 +54,13 @@ pub fn detectFormatFromMemory(buffer: []const u8) !Format { + } + + /// Load an image from a file path +-pub fn fromFilePath(allocator: std.mem.Allocator, file_path: []const u8, read_buffer: []u8) !Managed { +- return (try Image.fromFilePath(allocator, file_path, read_buffer)).toManaged(allocator); ++pub fn fromFilePath(allocator: std.mem.Allocator, io: std.Io, file_path: []const u8, read_buffer: []u8) !Managed { ++ return (try Image.fromFilePath(allocator, io, file_path, read_buffer)).toManaged(allocator); + } + +-/// Load an image from a standard library std.fs.File +-pub fn fromFile(allocator: std.mem.Allocator, file: std.fs.File, read_buffer: []u8) !Managed { +- return (try Image.fromFile(allocator, file, read_buffer)).toManaged(allocator); ++/// Load an image from a standard library std.Io.File ++pub fn fromFile(allocator: std.mem.Allocator, io: std.Io, file: std.Io.File, read_buffer: []u8) !Managed { ++ return (try Image.fromFile(allocator, io, file, read_buffer)).toManaged(allocator); + } + + /// Load an image from a memory buffer +@@ -116,12 +118,12 @@ pub fn isAnimation(self: Managed) bool { + + /// Write the image to an image format to the specified path + pub fn writeToFilePath(self: Managed, file_path: []const u8, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { +- return Image.writeToFilePath(self.toUnmanaged(), self.allocator, file_path, write_buffer, encoder_options); ++ return Image.writeToFilePath(self.toUnmanaged(), self.allocator, self.io, file_path, write_buffer, encoder_options); + } + +-/// Write the image to an image format to the specified std.fs.File +-pub fn writeToFile(self: Managed, file: std.fs.File, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { +- return Image.writeToFile(self.toUnmanaged(), self.allocator, file, write_buffer, encoder_options); ++/// Write the image to an image format to the specified std.Io.File ++pub fn writeToFile(self: Managed, file: std.Io.File, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { ++ return Image.writeToFile(self.toUnmanaged(), self.allocator, file, self.io, write_buffer, encoder_options); + } + + /// Write the image to an image format in a memory buffer. The memory buffer is not grown +diff --git a/src/color.zig b/src/color.zig +index 4591bce..8a1a6ac 100644 +--- a/src/color.zig ++++ b/src/color.zig +@@ -55,10 +55,7 @@ pub fn ScaleValue(comptime T: type) fn (anytype) T { + // In order to fit `value * out_max` we need an + // int of size `in_bits + out_bits` in the worst + // case scenario. +- const FitInt = @Type(.{ .int = .{ +- .bits = in_bits + out_bits, +- .signedness = .unsigned, +- } }); ++ const FitInt = @Int(.unsigned, in_bits + out_bits); + + return @truncate((@as(FitInt, value) * out_max + in_max / 2) / in_max); + } else return value; +@@ -395,7 +392,7 @@ fn ToMethods( + return packed struct(u0) { + const To = @This(); + +- fn getSelf(to: *const To) *align(1) const T { ++ fn getSelf(to: anytype) *align(1) const T { + // @fieldParentPtr is broken for packed structs. + // See: https://github.com/ziglang/zig/issues/20458 + if (@typeInfo(T).@"struct".layout == .@"packed") { +@@ -408,47 +405,47 @@ fn ToMethods( + } + + /// Assumes the target color type has `FromMethods` for it. +- pub fn color(to: *const To, ColorT: type) ColorT { +- return ColorT.from.color(to.getSelf().*); ++ pub fn color(to: anytype, ColorT: type) ColorT { ++ return ColorT.from.color(getSelf(to).*); + } + +- pub fn u32Rgba(to: *const To) u32 { +- const self = to.getSelf(); ++ pub fn u32Rgba(to: anytype) u32 { ++ const self = getSelf(to); + return @as(u32, toU8(self.r)) << 24 | + @as(u32, toU8(self.g)) << 16 | + @as(u32, toU8(self.b)) << 8 | + if (has_alpha) toU8(self.a) else 0xff; + } + +- pub fn u32Rgb(to: *const To) u32 { +- const self = to.getSelf(); ++ pub fn u32Rgb(to: anytype) u32 { ++ const self = getSelf(to); + return @as(u32, toU8(self.r)) << 16 | + @as(u32, toU8(self.g)) << 8 | + toU8(self.b); + } + +- pub fn u64Rgba(to: *const To) u64 { +- const self = to.getSelf(); ++ pub fn u64Rgba(to: anytype) u64 { ++ const self = getSelf(to); + return @as(u64, toU16(self.r)) << 48 | + @as(u64, toU16(self.g)) << 32 | + @as(u64, toU16(self.b)) << 16 | + if (has_alpha) toU16(self.a) else 0xffff; + } + +- pub fn u64Rgb(to: *const To) u64 { +- const self = to.getSelf(); ++ pub fn u64Rgb(to: anytype) u64 { ++ const self = getSelf(to); + return @as(u64, toU16(self.r)) << 32 | + @as(u64, toU16(self.g)) << 16 | + toU16(self.b); + } + + /// Only valid for color types where all channels are the same type. +- pub fn array(to: *const To) [4]RedT { ++ pub fn array(to: anytype) [4]RedT { + if (comptime multiple_channel_types) { + @compileError("Color.to.array may only be used when all channels in the color are the same type."); + } + +- const self = to.getSelf(); ++ const self = getSelf(to); + + return .{ + self.r, +@@ -461,8 +458,8 @@ fn ToMethods( + }; + } + +- pub fn float4(to: *const To) math.float4 { +- const self = to.getSelf(); ++ pub fn float4(to: anytype) math.float4 { ++ const self = getSelf(to); + + return .{ + toF32(self.r), +@@ -477,8 +474,8 @@ fn ToMethods( + + /// For int channels, premultiplication + /// is done with a round-trip to f32. +- pub fn premultipliedAlpha(to: *const To) T { +- const self = to.getSelf(); ++ pub fn premultipliedAlpha(to: anytype) T { ++ const self = getSelf(to); + var res = self.*; + if (!has_alpha) return res; + +diff --git a/src/formats/pam.zig b/src/formats/pam.zig +index f366d95..7ef9833 100644 +--- a/src/formats/pam.zig ++++ b/src/formats/pam.zig +@@ -70,7 +70,7 @@ + var maybe_maxval: ?u16 = null; + var maybe_tuple_type: ?TupleType = null; +- var comments = std.ArrayListUnmanaged([]const u8){}; ++ var comments: std.ArrayListUnmanaged([]const u8) = .empty; + defer { + for (comments.items) |comment| allocator.free(comment); + comments.deinit(allocator); +@@ -77,7 +77,7 @@ const Header = struct { + } + + { +- var line_buffer_stream = try std.io.Writer.Allocating.initCapacity(allocator, 32); ++ var line_buffer_stream = try std.Io.Writer.Allocating.initCapacity(allocator, 32); + defer line_buffer_stream.deinit(); + + while (true) { +diff --git a/src/formats/pcx.zig b/src/formats/pcx.zig +index b694762..52c2bb0 100644 +--- a/src/formats/pcx.zig ++++ b/src/formats/pcx.zig +@@ -218,7 +218,7 @@ test "PCX RLE Fast encoder" { + const uncompressed_data = [_]u8{ 1, 1, 1, 1, 1, 1, 1, 1, 1, 64, 64, 2, 2, 2, 2, 2, 215, 215, 215, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 200, 200, 200, 200, 210, 210 }; + const compressed_data = [_]u8{ 0xC9, 0x01, 0xC2, 0x40, 0xC5, 0x02, 0xC3, 0xD7, 0xCA, 0x03, 0xC4, 0xC8, 0xC2, 0xD2 }; + +- var writer_alloc = std.io.Writer.Allocating.init(std.testing.allocator); ++ var writer_alloc = std.Io.Writer.Allocating.init(std.testing.allocator); + defer writer_alloc.deinit(); + + try RLEFastEncoder.encode(uncompressed_data[0..], &writer_alloc.writer); +@@ -233,7 +233,7 @@ test "PCX RLE Fast encoder should encore more than 63 bytes similar" { + + const compressed_data = [_]u8{ 0xFF, 0x45, 0x45, 0x45, 0xC4, 0x1 }; + +- var writer_alloc = std.io.Writer.Allocating.init(std.testing.allocator); ++ var writer_alloc = std.Io.Writer.Allocating.init(std.testing.allocator); + defer writer_alloc.deinit(); + + try RLEFastEncoder.encode(uncompressed_data[0..], &writer_alloc.writer); +diff --git a/src/formats/tiff/types.zig b/src/formats/tiff/types.zig +index 8bcce86..067480a 100644 +--- a/src/formats/tiff/types.zig ++++ b/src/formats/tiff/types.zig +@@ -335,7 +335,7 @@ pub const IFD = struct { + // Skip unknown tags instead of panicking on invalid enum values. + // Many TIFF files contain vendor/proprietary tags (Make, Model, ICC profile, etc.) + // that we don't need to handle for image decoding. +- const maybe_tag_id = std.meta.intToEnum(TagId, tag_id_raw) catch continue; ++ const maybe_tag_id = std.enums.fromInt(TagId, tag_id_raw) orelse continue; + + try self.tags_map.put(maybe_tag_id, TagField{ + .data_type = std.mem.toNative(u16, tag.data_type, endianess), +diff --git a/src/formats/xbm.zig b/src/formats/xbm.zig +index d72e4ec..19ac9c0 100644 +--- a/src/formats/xbm.zig ++++ b/src/formats/xbm.zig +@@ -100,7 +100,7 @@ pub const XBM = struct { + + var line_full = try reader.takeDelimiterInclusive('\n'); + while (line_full.len > 0) { +- const line = std.mem.trimRight(u8, line_full, " \r\n"); ++ const line = std.mem.trimEnd(u8, line_full, " \r\n"); + if (isDefineLine(line)) { + if (parseDefineValue(line)) |value| { + if (width == null) { +diff --git a/src/io.zig b/src/io.zig +index 2c1363d..8d463a1 100644 +--- a/src/io.zig ++++ b/src/io.zig +@@ -5,11 +5,11 @@ pub const DEFAULT_BUFFER_SIZE = 4096; + + pub const ReadStream = union(enum) { + memory: std.Io.Reader, +- file: std.fs.File.Reader, ++ file: std.Io.File.Reader, + + pub const Error = SeekError || EndPosError || std.Io.Reader.Error || std.Io.Reader.StreamError; +- pub const SeekError = std.fs.File.Reader.SeekError; +- pub const EndPosError = std.fs.File.Reader.SizeError; ++ pub const SeekError = std.Io.File.Reader.SeekError; ++ pub const EndPosError = std.Io.File.Reader.SizeError; + + pub fn initMemory(buffer: []const u8) ReadStream { + return .{ +@@ -17,9 +17,9 @@ pub const ReadStream = union(enum) { + }; + } + +- pub fn initFile(file: std.fs.File, buffer: []u8) ReadStream { ++ pub fn initFile(io: std.Io, file: std.Io.File, buffer: []u8) ReadStream { + return .{ +- .file = file.reader(buffer), ++ .file = file.reader(io, buffer), + }; + } + +@@ -58,7 +58,7 @@ pub const ReadStream = union(enum) { + .memory => |*memory| { + const new_pos: i64 = @as(i64, @intCast(memory.seek)) + offset; + if (new_pos < 0 or new_pos >= memory.end) { +- return std.fs.File.SeekError.Unseekable; ++ return std.Io.File.SeekError.Unseekable; + } + + memory.seek = @intCast(new_pos); +@@ -96,10 +96,10 @@ pub const ReadStream = union(enum) { + + pub const WriteStream = union(enum) { + memory: std.Io.Writer, +- file: std.fs.File.Writer, ++ file: std.Io.File.Writer, + + pub const Error = SeekError || std.Io.Writer.Error; +- pub const SeekError = std.fs.File.SeekError; ++ pub const SeekError = std.Io.File.SeekError || std.Io.Writer.Error; + + pub fn initMemory(buffer: []u8) WriteStream { + return .{ +@@ -107,9 +107,9 @@ pub const WriteStream = union(enum) { + }; + } + +- pub fn initFile(file: std.fs.File, buffer: []u8) WriteStream { ++ pub fn initFile(io: std.Io, file: std.Io.File, buffer: []u8) WriteStream { + return .{ +- .file = file.writer(buffer), ++ .file = file.writer(io, buffer), + }; + } + +diff --git a/src/math.zig b/src/math.zig +index c4e5910..bad9687 100644 +--- a/src/math.zig ++++ b/src/math.zig +@@ -24,11 +24,11 @@ pub fn Matrix(comptime T: type) type { + var temp = self; + + // To find the determinant, use a Gaussian elimination to transform the temp matrix into the row echelon form +- for (0..(ComponentSize - 1)) |row| { +- for ((row + 1)..ComponentSize) |next_row| { ++ inline for (0..(ComponentSize - 1)) |row| { ++ inline for ((row + 1)..ComponentSize) |next_row| { + const factor = temp.matrix[next_row][row] / temp.matrix[row][row]; + +- for (0..ComponentSize) |column| { ++ inline for (0..ComponentSize) |column| { + temp.matrix[next_row][column] = temp.matrix[next_row][column] - factor * temp.matrix[row][column]; + } + } +@@ -37,7 +37,7 @@ pub fn Matrix(comptime T: type) type { + // Once the temp matrix is in row echelon form, multiply the diagonal to find the determinant + var result: f32 = temp.matrix[0][0]; + +- for (1..ComponentSize) |diagonal| { ++ inline for (1..ComponentSize) |diagonal| { + result *= temp.matrix[diagonal][diagonal]; + } + +@@ -48,9 +48,9 @@ pub fn Matrix(comptime T: type) type { + pub fn fromArray(array: [ComponentSize * ComponentSize]f32) Self { + var result: Self = undefined; + +- for (0..ComponentSize) |row| { ++ inline for (0..ComponentSize) |row| { + const stride = row * ComponentSize; +- for (0..ComponentSize) |column| { ++ inline for (0..ComponentSize) |column| { + result.matrix[row][column] = array[stride + column]; + } + } +@@ -72,20 +72,20 @@ pub fn Matrix(comptime T: type) type { + var temp = self; + var result: Self = identity(); + +- for (0..ComponentSize) |row| { ++ inline for (0..ComponentSize) |row| { + // Transform the pivot to 1 by multiplying the row by its inverse + const inverse_pivot = 1.0 / temp.matrix[row][row]; + +- for (0..ComponentSize) |column| { ++ inline for (0..ComponentSize) |column| { + temp.matrix[row][column] *= inverse_pivot; + result.matrix[row][column] *= inverse_pivot; + } + + // Then do Gaussian elimination from current row to bottom +- for ((row + 1)..ComponentSize) |next_row| { ++ inline for ((row + 1)..ComponentSize) |next_row| { + const factor = temp.matrix[next_row][row] / temp.matrix[row][row]; + +- for (0..ComponentSize) |column| { ++ inline for (0..ComponentSize) |column| { + temp.matrix[next_row][column] = temp.matrix[next_row][column] - factor * temp.matrix[row][column]; + result.matrix[next_row][column] = result.matrix[next_row][column] - factor * result.matrix[row][column]; + } +@@ -94,14 +94,12 @@ pub fn Matrix(comptime T: type) type { + + // Then do a Gaussian elimination bottom-up to transform the temp matrix into an identity matrix. + // The result will be the inverse of the matrix +- var row: usize = ComponentSize -% 1; +- while (row >= 1 and row < ComponentSize) : (row -%= 1) { +- var previous_row = row -% 1; +- +- while (previous_row >= 0 and previous_row < ComponentSize) : (previous_row -%= 1) { ++ inline for (1..ComponentSize) |i| { ++ const row = ComponentSize - i; ++ inline for (0..row) |previous_row| { + const factor = temp.matrix[previous_row][row] / temp.matrix[row][row]; + +- for (0..ComponentSize) |column| { ++ inline for (0..ComponentSize) |column| { + temp.matrix[previous_row][column] = temp.matrix[previous_row][column] - factor * temp.matrix[row][column]; + result.matrix[previous_row][column] = result.matrix[previous_row][column] - factor * result.matrix[row][column]; + } +@@ -126,8 +124,8 @@ pub fn Matrix(comptime T: type) type { + + const transposed_right = right.transpose(); + +- for (0..ComponentSize) |row| { +- for (0..ComponentSize) |column| { ++ inline for (0..ComponentSize) |row| { ++ inline for (0..ComponentSize) |column| { + result.matrix[row][column] = @reduce(.Add, self.matrix[row] * transposed_right.matrix[column]); + } + } +@@ -138,8 +136,8 @@ pub fn Matrix(comptime T: type) type { + pub fn transpose(self: Self) Self { + var result = std.mem.zeroes(Self); + +- for (0..ComponentSize) |row| { +- for (0..ComponentSize) |column| { ++ inline for (0..ComponentSize) |row| { ++ inline for (0..ComponentSize) |column| { + result.matrix[row][column] = self.matrix[column][row]; + } + } +diff --git a/src/simd.zig b/src/simd.zig +index 20c5dcb..cb115c9 100644 +--- a/src/simd.zig ++++ b/src/simd.zig +@@ -78,8 +78,10 @@ pub fn vpermd(v: @Vector(8, i32), mask: @Vector(8, i32)) @Vector(8, i32) { + [v] "x" (v), + ); + } else { +- var res: @Vector(8, i32) = undefined; +- inline for (0..8) |i| res[i] = v[@as(u32, @bitCast(mask[i]))]; ++ const v_arr: [8]i32 = v; ++ const mask_arr: [8]i32 = mask; ++ var res: [8]i32 = undefined; ++ for (0..8) |i| res[i] = v_arr[@intCast(mask_arr[i] & 7)]; + return res; + } + } +diff --git a/src/utils.zig b/src/utils.zig +index f55c12c..a12733e 100644 +--- a/src/utils.zig ++++ b/src/utils.zig +@@ -4,7 +4,7 @@ const std = @import("std"); + const native_endian = builtin.target.cpu.arch.endian(); + + pub const StructReadError = error{InvalidData} || std.Io.Reader.Error; +-pub const StructWriteError = std.io.StreamSource.WriteError; ++pub const StructWriteError = std.Io.Writer.Error; + + pub fn FixedStorage(comptime T: type, comptime storage_size: usize) type { + return struct { +diff --git a/tests/formats/bmp_test.zig b/tests/formats/bmp_test.zig +index cc80b74..635dc04 100644 +--- a/tests/formats/bmp_test.zig ++++ b/tests/formats/bmp_test.zig +@@ -87,12 +87,12 @@ fn verifyBitmapRGBAV5(the_bitmap: bmp.BMP, pixels: zigimg.color.PixelStorage) !v + + test "Read simple version 4 24-bit RGB bitmap" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_bitmap = bmp.BMP{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_bitmap.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -145,10 +145,10 @@ test "Read simple version 4 24-bit RGB bitmap" { + + test "Read a valid version 5 RGBA bitmap from file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/windows_rgba_v5.bmp"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var the_bitmap = bmp.BMP{}; + +@@ -174,10 +174,10 @@ test "Read a valid version 5 RGBA bitmap from memory" { + + test "Should error when reading an invalid file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/notbmp.png"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var the_bitmap = bmp.BMP{}; + +@@ -215,17 +215,17 @@ test "Write a v4 bitmap when writing an image with bgr24 pixel format" { + pixels.bgr24[7] = zigimg.color.Bgr24.from.rgb(255, 255, 0); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .bmp = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .bmp = .{} }); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var read_bmp = bmp.BMP{}; + +@@ -277,17 +277,17 @@ test "Write a v5 bitmap when writing an image with bgra32 pixel format" { + pixels.bgra32[8] = zigimg.color.Bgra32.from.rgba(0, 0, 0, 0); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .bmp = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .bmp = .{} }); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var read_bmp = bmp.BMP{}; + +diff --git a/tests/formats/farbfeld_test.zig b/tests/formats/farbfeld_test.zig +index a48a7fb..00a73a0 100644 +--- a/tests/formats/farbfeld_test.zig ++++ b/tests/formats/farbfeld_test.zig +@@ -5,10 +5,10 @@ const farbfeld = zigimg.formats.farbfeld; + + test "Farbfeld: Read yellow file" { + const yellow_file = try helpers.testOpenFile(helpers.fixtures_path ++ "farbfeld/yellow-1x1-semitransparent.png.ff"); +- defer yellow_file.close(); ++ defer yellow_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var yellow_read_stream = zigimg.io.ReadStream.initFile(yellow_file, read_buffer[0..]); ++ var yellow_read_stream = zigimg.io.ReadStream.initFile(std.testing.io, yellow_file, read_buffer[0..]); + + var yellow_image = farbfeld.Farbfeld{}; + const yellow_pixels = try yellow_image.read(helpers.zigimg_test_allocator, &yellow_read_stream); +@@ -21,10 +21,10 @@ test "Farbfeld: Read yellow file" { + + test "Farbfeld: read dragon file" { + const dragon_file = try helpers.testOpenFile(helpers.fixtures_path ++ "farbfeld/dragon.ff"); +- defer dragon_file.close(); ++ defer dragon_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var dragon_read_stream = zigimg.io.ReadStream.initFile(dragon_file, read_buffer[0..]); ++ var dragon_read_stream = zigimg.io.ReadStream.initFile(std.testing.io, dragon_file, read_buffer[0..]); + + var dragon_image = farbfeld.Farbfeld{}; + const dragon_pixels = try dragon_image.read(helpers.zigimg_test_allocator, &dragon_read_stream); +@@ -37,10 +37,10 @@ test "Farbfeld: read dragon file" { + + test "Farbfeld: invalid file format" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "farbfeld/dragon.png"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var farbfeld_image = farbfeld.Farbfeld{}; + const image_error = farbfeld_image.read(helpers.zigimg_test_allocator, &read_stream); +@@ -50,10 +50,10 @@ test "Farbfeld: invalid file format" { + + test "Farbfeld: read writeImage output" { + const source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "farbfeld/yellow-1x1-semitransparent.png.ff"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var source_read_stream = zigimg.io.ReadStream.initFile(source_file, read_buffer[0..]); ++ var source_read_stream = zigimg.io.ReadStream.initFile(std.testing.io, source_file, read_buffer[0..]); + + var source_image: farbfeld.Farbfeld = .{}; + const source_pixels = try source_image.read(helpers.zigimg_test_allocator, &source_read_stream); +diff --git a/tests/formats/gif_test.zig b/tests/formats/gif_test.zig +index 496e14f..bdd84f6 100644 +--- a/tests/formats/gif_test.zig ++++ b/tests/formats/gif_test.zig +@@ -325,10 +325,10 @@ test "GIF writer roundtrip read existing GIF, write, read again" { + + // Read an existing GIF file using low-level reader + const gif_input_file = try helpers.testOpenFile(helpers.fixtures_path ++ "gif/rotating_earth.gif"); +- defer gif_input_file.close(); ++ defer gif_input_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(gif_input_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, gif_input_file, read_buffer[0..]); + + var original_gif = gif.GIF.init(allocator); + defer original_gif.deinit(); +@@ -489,10 +489,10 @@ test "GIF writer rejects non-indexed data without auto convert" { + + test "Should error on non GIF images" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var gif_file = gif.GIF.init(helpers.zigimg_test_allocator); + defer gif_file.deinit(); +@@ -563,7 +563,7 @@ const SINGLE_GIF_FILE_TEST = false; + fn loadRotatingEarthImage(allocator: std.mem.Allocator) !zigimg.Image { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_path = helpers.fixtures_path ++ "gif/rotating_earth.gif"; +- return zigimg.Image.fromFilePath(allocator, image_path, read_buffer[0..]); ++ return zigimg.Image.fromFilePath(allocator, std.testing.io, image_path, read_buffer[0..]); + } + + test "GIF test suite" { +@@ -575,10 +575,10 @@ test "GIF test suite" { + defer test_list.deinit(helpers.zigimg_test_allocator); + + const test_list_file = try helpers.testOpenFile(helpers.fixtures_path ++ "gif/TESTS"); +- defer test_list_file.close(); ++ defer test_list_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(test_list_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, test_list_file, read_buffer[0..]); + + var reader = read_stream.reader(); + +@@ -598,11 +598,11 @@ test "GIF test suite" { + for (test_list.items) |entry| { + doGifTest(entry) catch |err| { + has_failed_file = true; +- std.debug.print("Error: {}\n", .{err}); ++ std.log.info("Error: {}\n", .{err}); + continue; + }; + +- std.debug.print("OK\n", .{}); ++ std.log.info("OK\n", .{}); + } + + if (has_failed_file) { +@@ -612,10 +612,10 @@ test "GIF test suite" { + + test "Rotating Earth GIF" { + const gif_input_file = try helpers.testOpenFile(helpers.fixtures_path ++ "gif/rotating_earth.gif"); +- defer gif_input_file.close(); ++ defer gif_input_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(gif_input_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, gif_input_file, read_buffer[0..]); + + var gif_file = gif.GIF.init(helpers.zigimg_test_allocator); + defer gif_file.deinit(); +@@ -710,8 +710,8 @@ const IniFile = struct { + const equals_sign_position_opt = std.mem.indexOf(u8, read_line[0..], "="); + + if (equals_sign_position_opt) |equals_sign_position| { +- const key_name = std.mem.trimRight(u8, read_line[0..(equals_sign_position - 1)], " "); +- const string_value = std.mem.trimLeft(u8, read_line[(equals_sign_position + 1)..], " "); ++ const key_name = std.mem.trimEnd(u8, read_line[0..(equals_sign_position - 1)], " "); ++ const string_value = std.mem.trimStart(u8, read_line[(equals_sign_position + 1)..], " "); + + if (self.sections.getPtr(current_section)) |section_entry| { + const value = blk: { +@@ -760,7 +760,7 @@ const IniFile = struct { + }; + + fn doGifTest(entry_name: []const u8) !void { +- std.debug.print("GIF test {s}... ", .{entry_name}); ++ std.log.info("GIF test {s}... ", .{entry_name}); + + var area_alloc = std.heap.ArenaAllocator.init(helpers.zigimg_test_allocator); + const area_allocator = area_alloc.allocator(); +@@ -770,13 +770,13 @@ fn doGifTest(entry_name: []const u8) !void { + const config_filepath = try std.fs.path.resolve(area_allocator, &[_][]const u8{ helpers.fixtures_path, "gif", config_filename }); + + const config_file = try helpers.testOpenFile(config_filepath); +- defer config_file.close(); ++ defer config_file.close(std.testing.io); + + var config_ini = IniFile.init(helpers.zigimg_test_allocator); + defer config_ini.deinit(); + + var ini_read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var ini_read_stream = zigimg.io.ReadStream.initFile(config_file, ini_read_buffer[0..]); ++ var ini_read_stream = zigimg.io.ReadStream.initFile(std.testing.io, config_file, ini_read_buffer[0..]); + + try config_ini.parse(ini_read_stream.reader()); + +@@ -804,10 +804,10 @@ fn doGifTest(entry_name: []const u8) !void { + + const gif_input_filepath = try std.fs.path.resolve(area_allocator, &[_][]const u8{ helpers.fixtures_path, "gif", input_filename.string }); + const gif_input_file = try helpers.testOpenFile(gif_input_filepath); +- defer gif_input_file.close(); ++ defer gif_input_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(gif_input_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, gif_input_file, read_buffer[0..]); + + var gif_file = gif.GIF.init(helpers.zigimg_test_allocator); + defer gif_file.deinit(); +@@ -856,10 +856,10 @@ fn doGifTest(entry_name: []const u8) !void { + const pixels_filename = frame_section.getValue("pixels") orelse return error.InvalidGifConfigFile; + const pixels_filepath = try std.fs.path.resolve(area_allocator, &[_][]const u8{ helpers.fixtures_path, "gif", pixels_filename.string }); + const pixels_file = try helpers.testOpenFile(pixels_filepath); +- defer pixels_file.close(); ++ defer pixels_file.close(std.testing.io); + + var pixels_read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var pixels_read_stream = zigimg.io.ReadStream.initFile(pixels_file, pixels_read_buffer[0..]); ++ var pixels_read_stream = zigimg.io.ReadStream.initFile(std.testing.io, pixels_file, pixels_read_buffer[0..]); + + var pixels_reader = pixels_read_stream.reader(); + +@@ -894,7 +894,7 @@ fn doGifTest(entry_name: []const u8) !void { + try helpers.expectEq(actual_color.to.color(zigimg.color.Rgba32), gif_background_color); + } else { + helpers.expectEq(actual_color.to.color(zigimg.color.Rgba32), expected_color) catch |err| { +- std.debug.print("Pixel #{} does not match: expected={}, actual={}\n", .{ index, expected_color, actual_color.to.color(zigimg.color.Rgba32) }); ++ std.log.info("Pixel #{} does not match: expected={}, actual={}\n", .{ index, expected_color, actual_color.to.color(zigimg.color.Rgba32) }); + return err; + }; + } +@@ -929,27 +929,27 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { + + // Build path to GIF file + const gif_filename = std.fmt.allocPrint(area_allocator, "{s}.gif", .{entry_name}) catch { +- std.debug.print("FAIL (allocPrint failed)\n", .{}); ++ std.log.info("FAIL (allocPrint failed)\n", .{}); + return .failed; + }; + const gif_filepath = std.fs.path.resolve(area_allocator, &[_][]const u8{ helpers.fixtures_path, "gif", gif_filename }) catch { +- std.debug.print("FAIL (path resolve failed)\n", .{}); ++ std.log.info("FAIL (path resolve failed)\n", .{}); + return .failed; + }; + + // Read original GIF +- const gif_input_file = try std.fs.cwd().openFile(gif_filepath, .{}); +- defer gif_input_file.close(); ++ const gif_input_file = try std.Io.Dir.openFile(std.Io.Dir.cwd(), std.testing.io, gif_filepath, .{}); ++ defer gif_input_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(gif_input_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, gif_input_file, read_buffer[0..]); + + var original_gif = gif.GIF.init(helpers.zigimg_test_allocator); + defer original_gif.deinit(); + + var original_frames = original_gif.read(&read_stream) catch { + // Reader correctly rejected bad data +- std.debug.print("OK (reader correctly rejected bad data)\n", .{}); ++ std.log.info("OK (reader correctly rejected bad data)\n", .{}); + return .passed; + }; + defer { +@@ -961,7 +961,7 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { + + if (original_frames.items.len == 0) { + // No frames means nothing to write +- std.debug.print("OK (no frames to write)\n", .{}); ++ std.log.info("OK (no frames to write)\n", .{}); + return .passed; + } + +@@ -974,13 +974,13 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { + + // Non-indexed formats can't be written without auto_convert + if (!image.pixelFormat().isIndexed()) { +- std.debug.print("OK (non-indexed format correctly not written: {s})\n", .{@tagName(image.pixels)}); ++ std.log.info("OK (non-indexed format correctly not written: {s})\n", .{@tagName(image.pixels)}); + return .passed; + } + + // Zero dimensions can't be written + if (image.width == 0 or image.height == 0) { +- std.debug.print("OK (zero dimensions correctly not written)\n", .{}); ++ std.log.info("OK (zero dimensions correctly not written)\n", .{}); + return .passed; + } + +@@ -991,16 +991,16 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { + gif.GIF.writeImage(helpers.zigimg_test_allocator, &write_stream, image, .{ .gif = .{} }) catch |err| { + // For very large images, running out of buffer space is expected + if (err == error.WriteFailed and (image.width > 1000 or image.height > 1000)) { +- std.debug.print("OK (large image exceeded buffer)\n", .{}); ++ std.log.info("OK (large image exceeded buffer)\n", .{}); + return .passed; + } +- std.debug.print("FAIL (write error: {})\n", .{err}); ++ std.log.info("FAIL (write error: {})\n", .{err}); + return .failed; + }; + + const written_len = write_stream.writer().end; + if (written_len == 0) { +- std.debug.print("FAIL (wrote 0 bytes)\n", .{}); ++ std.log.info("FAIL (wrote 0 bytes)\n", .{}); + return .failed; + } + +@@ -1011,7 +1011,7 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { + defer decoded_gif.deinit(); + + var decoded_frames = decoded_gif.read(&read_back_stream) catch |err| { +- std.debug.print("FAIL (read-back error: {})\n", .{err}); ++ std.log.info("FAIL (read-back error: {})\n", .{err}); + return .failed; + }; + defer { +@@ -1025,7 +1025,7 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { + if (original_gif.header.width != decoded_gif.header.width or + original_gif.header.height != decoded_gif.header.height) + { +- std.debug.print("FAIL (dimension mismatch: {}x{} vs {}x{})\n", .{ ++ std.log.info("FAIL (dimension mismatch: {}x{} vs {}x{})\n", .{ + original_gif.header.width, + original_gif.header.height, + decoded_gif.header.width, +@@ -1036,7 +1036,7 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { + + // Verify we got at least one frame + if (decoded_frames.items.len == 0) { +- std.debug.print("FAIL (no frames in output)\n", .{}); ++ std.log.info("FAIL (no frames in output)\n", .{}); + return .failed; + } + +@@ -1048,14 +1048,14 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { + while (original_iter.next()) |original_color| { + const decoded_color_opt = decoded_iter.next(); + if (decoded_color_opt == null) { +- std.debug.print("FAIL (missing pixel at index {})\n", .{pixel_index}); ++ std.log.info("FAIL (missing pixel at index {})\n", .{pixel_index}); + return .failed; + } + const original = original_color.to.color(zigimg.color.Rgba32); + const decoded = decoded_color_opt.?.to.color(zigimg.color.Rgba32); + + if (original.r != decoded.r or original.g != decoded.g or original.b != decoded.b) { +- std.debug.print("FAIL (pixel {} mismatch: ({},{},{}) vs ({},{},{}))\n", .{ ++ std.log.info("FAIL (pixel {} mismatch: ({},{},{}) vs ({},{},{}))\n", .{ + pixel_index, original.r, original.g, original.b, decoded.r, decoded.g, decoded.b, + }); + return .failed; +@@ -1071,10 +1071,10 @@ test "GIF writer roundtrip test suite" { + defer test_list.deinit(helpers.zigimg_test_allocator); + + const test_list_file = try helpers.testOpenFile(helpers.fixtures_path ++ "gif/TESTS"); +- defer test_list_file.close(); ++ defer test_list_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(test_list_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, test_list_file, read_buffer[0..]); + + var reader = read_stream.reader(); + +diff --git a/tests/formats/iff_test.zig b/tests/formats/iff_test.zig +index ab2dd6b..6a3594d 100644 +--- a/tests/formats/iff_test.zig ++++ b/tests/formats/iff_test.zig +@@ -5,12 +5,12 @@ const zigimg = @import("zigimg"); + + test "IFF-PBM indexed8 (chunky Deluxe Paint DOS file)" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-pbm.iff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_iff = iff.IFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -37,12 +37,12 @@ test "IFF-PBM indexed8 (chunky Deluxe Paint DOS file)" { + + test "IFF-ILBM indexed8 8 bitplanes" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-ilbm-8bit-compressed.iff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_iff = iff.IFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -63,12 +63,12 @@ test "IFF-ILBM indexed8 8 bitplanes" { + + test "IFF-ILBM indexed8 8 bitplanes uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-ilbm-8bit-uncompressed.iff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_iff = iff.IFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -89,12 +89,12 @@ test "IFF-ILBM indexed8 8 bitplanes uncompressed" { + + test "IFF-ILBM indexed8 6 bitplanes EHB" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-ehb.iff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_iff = iff.IFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -121,12 +121,12 @@ test "IFF-ILBM indexed8 6 bitplanes EHB" { + + test "IFF-ILBM indexed8 4 bitplanes HAM" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-ham.iff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_iff = iff.IFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -150,12 +150,12 @@ test "IFF-ILBM indexed8 4 bitplanes HAM" { + + test "IFF-ILBM indexed8 6 bitplanes HAM8" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-ham8.iff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_iff = iff.IFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -179,12 +179,12 @@ test "IFF-ILBM indexed8 6 bitplanes HAM8" { + + test "IFF-ILBM 24bit" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-24bit.iff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_iff = iff.IFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -208,12 +208,12 @@ test "IFF-ILBM 24bit" { + + test "IFF-ILBM indexed8 4 bitplanes Atari ST" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-ilbm-4bit-compressed-atari.iff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_iff = iff.IFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -240,12 +240,12 @@ test "IFF-ILBM indexed8 4 bitplanes Atari ST" { + + test "IFF-ACBM indexed8 3 bitplanes uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-8bit.acbm"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_iff = iff.IFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); + defer pixels.deinit(helpers.zigimg_test_allocator); +diff --git a/tests/formats/jpeg_test.zig b/tests/formats/jpeg_test.zig +index 8288b62..0f1254f 100644 +--- a/tests/formats/jpeg_test.zig ++++ b/tests/formats/jpeg_test.zig +@@ -7,10 +7,10 @@ const color = zigimg.color; + + test "Should error on non JPEG images" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); + defer jpeg_file.deinit(); +@@ -28,10 +28,10 @@ test "Should error on non JPEG images" { + + test "Read JFIF header properly and decode simple Huffman stream" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "jpeg/huff_simple0.jpg"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); + defer jpeg_file.deinit(); +@@ -59,10 +59,10 @@ test "Read JFIF header properly and decode simple Huffman stream" { + + test "Read the tuba properly" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "jpeg/tuba.jpg"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); + defer jpeg_file.deinit(); +@@ -95,10 +95,10 @@ test "Read the tuba properly" { + + test "Read grayscale images" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "jpeg/grayscale_sample0.jpg"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); + defer jpeg_file.deinit(); +@@ -130,19 +130,19 @@ test "Read grayscale images" { + } + + test "Read subsampling images" { +- var testdir = std.fs.cwd().openDir(helpers.fixtures_path ++ "jpeg/", .{ .access_sub_paths = false, .no_follow = true, .iterate = true }) catch null; ++ var testdir = std.Io.Dir.openDir(std.Io.Dir.cwd(), std.testing.io, helpers.fixtures_path ++ "jpeg/", .{ .access_sub_paths = false, .iterate = true }) catch null; + if (testdir) |*idir| { +- defer idir.close(); ++ defer idir.close(std.testing.io); + + var it = idir.iterate(); +- while (try it.next()) |entry| { ++ while (try it.next(std.testing.io)) |entry| { + if (entry.kind != .file or !std.mem.endsWith(u8, entry.name, ".jpg") or !std.mem.startsWith(u8, entry.name, "subsampling_")) continue; + +- var test_file = try idir.openFile(entry.name, .{ .mode = .read_only }); +- defer test_file.close(); ++ var test_file = try idir.openFile(std.testing.io, entry.name, .{ .mode = .read_only }); ++ defer test_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(test_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, test_file, read_buffer[0..]); + + var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); + defer jpeg_file.deinit(); +@@ -182,10 +182,10 @@ test "Read subsampling images" { + + test "Read progressive jpeg with restart intervals" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "jpeg/tuba_restart_prog.jpg"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); + defer jpeg_file.deinit(); +@@ -403,10 +403,10 @@ test "JPEG writer round-trip with all test fixtures" { + for (test_fixtures) |fixture_path| { + // Skip if file doesn't exist + const file = helpers.testOpenFile(fixture_path) catch continue; +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [4096]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); + defer jpeg_file.deinit(); + +diff --git a/tests/formats/netpbm_test.zig b/tests/formats/netpbm_test.zig +index ee06575..e0ffe05 100644 +--- a/tests/formats/netpbm_test.zig ++++ b/tests/formats/netpbm_test.zig +@@ -5,10 +5,10 @@ const zigimg = @import("zigimg"); + + test "Load ASCII PBM image" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/pbm_ascii.pbm"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var pbm_file = netpbm.PBM{}; + +@@ -27,10 +27,10 @@ test "Load ASCII PBM image" { + + test "Load binary PBM image" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/pbm_binary.pbm"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var pbm_file = netpbm.PBM{}; + +@@ -49,10 +49,10 @@ test "Load binary PBM image" { + + test "Load ASCII PGM 8-bit grayscale image" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/pgm_ascii_grayscale8.pgm"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var pgm_file = netpbm.PGM{}; + +@@ -71,10 +71,10 @@ test "Load ASCII PGM 8-bit grayscale image" { + + test "Load Binary PGM 8-bit grayscale image" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/pgm_binary_grayscale8.pgm"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var pgm_file = netpbm.PGM{}; + +@@ -93,10 +93,10 @@ test "Load Binary PGM 8-bit grayscale image" { + + test "Load ASCII PGM 16-bit grayscale image" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/pgm_ascii_grayscale16.pgm"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var pgm_file = netpbm.PGM{}; + +@@ -115,10 +115,10 @@ test "Load ASCII PGM 16-bit grayscale image" { + + test "Load Binary PGM 16-bit grayscale image" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/pgm_binary_grayscale16.pgm"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var pgm_file = netpbm.PGM{}; + +@@ -137,10 +137,10 @@ test "Load Binary PGM 16-bit grayscale image" { + + test "Load ASCII PPM image" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/ppm_ascii_rgb24.ppm"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var ppm_file = netpbm.PPM{}; + +@@ -176,10 +176,10 @@ test "Load ASCII PPM image" { + + test "Load binary PPM image" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/ppm_binary_rgb24.ppm"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var ppm_file = netpbm.PPM{}; + +@@ -234,16 +234,16 @@ test "Write bitmap(grayscale1) ASCII PBM file" { + } + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .pbm = .{ .binary = false }, + }); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch unreachable; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; + } + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, image_file_name, read_buffer[0..]); ++ var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, read_buffer[0..]); + defer read_image.deinit(helpers.zigimg_test_allocator); + + try helpers.expectEq(read_image.width, width); +@@ -281,16 +281,16 @@ test "Write bitmap(Grayscale1) binary PBM file" { + } + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .pbm = .{ .binary = true }, + }); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch unreachable; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; + } + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, image_file_name, read_buffer[0..]); ++ var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, read_buffer[0..]); + defer read_image.deinit(helpers.zigimg_test_allocator); + + try helpers.expectEq(read_image.width, width); +@@ -324,16 +324,16 @@ test "Write grayscale8 ASCII PGM file" { + } + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .pgm = .{ .binary = false }, + }); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch unreachable; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; + } + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, image_file_name, read_buffer[0..]); ++ var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, read_buffer[0..]); + defer read_image.deinit(helpers.zigimg_test_allocator); + + try helpers.expectEq(read_image.width, width); +@@ -367,16 +367,16 @@ test "Write grayscale8 binary PGM file" { + } + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .pgm = .{ .binary = true }, + }); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch unreachable; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; + } + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, image_file_name, read_buffer[0..]); ++ var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, read_buffer[0..]); + defer read_image.deinit(helpers.zigimg_test_allocator); + + try helpers.expectEq(read_image.width, width); +@@ -420,16 +420,16 @@ test "Writing Rgb24 ASCII PPM format" { + pixels.rgb24[7] = zigimg.color.Rgb24.from.rgb(255, 255, 0); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .ppm = .{ .binary = false }, + }); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch unreachable; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; + } + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, image_file_name, read_buffer[0..]); ++ var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, read_buffer[0..]); + defer read_image.deinit(helpers.zigimg_test_allocator); + + try helpers.expectEq(read_image.width, width); +@@ -474,16 +474,16 @@ test "Writing Rgb24 binary PPM format" { + pixels.rgb24[7] = zigimg.color.Rgb24.from.rgb(255, 255, 0); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .ppm = .{ .binary = true }, + }); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch unreachable; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; + } + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, image_file_name, read_buffer[0..]); ++ var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, read_buffer[0..]); + defer read_image.deinit(helpers.zigimg_test_allocator); + + try helpers.expectEq(read_image.width, width); +@@ -505,7 +505,7 @@ test "Trying to write a bitmap or grayscale Netpbm with an true color pixel form + defer source_image.deinit(helpers.zigimg_test_allocator); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch unreachable; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; + } + + const pixels = source_image.pixels; +@@ -527,12 +527,12 @@ test "Trying to write a bitmap or grayscale Netpbm with an true color pixel form + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + + { +- const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .pbm = .{} }); ++ const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .pbm = .{} }); + try std.testing.expectError(zigimg.Image.WriteError.Unsupported, write_error); + } + + { +- const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .pgm = .{} }); ++ const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .pgm = .{} }); + try std.testing.expectError(zigimg.Image.WriteError.Unsupported, write_error); + } + } +@@ -551,7 +551,7 @@ test "Trying to write a bitmap or true color Netpbm with a 8-bit grayscale pixel + defer source_image.deinit(helpers.zigimg_test_allocator); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch unreachable; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; + } + + const source = source_image.pixels; +@@ -562,12 +562,12 @@ test "Trying to write a bitmap or true color Netpbm with a 8-bit grayscale pixel + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + + { +- const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .pbm = .{} }); ++ const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .pbm = .{} }); + try std.testing.expectError(zigimg.Image.WriteError.Unsupported, write_error); + } + + { +- const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .ppm = .{} }); ++ const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .ppm = .{} }); + try std.testing.expectError(zigimg.Image.WriteError.Unsupported, write_error); + } + } +@@ -589,7 +589,7 @@ test "Trying to write a grayscale or true color Netbpm with a 1-bit grayscale pi + defer source_image.deinit(helpers.zigimg_test_allocator); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch unreachable; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; + } + + const source = source_image.pixels; +@@ -601,12 +601,12 @@ test "Trying to write a grayscale or true color Netbpm with a 1-bit grayscale pi + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + + { +- const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .pgm = .{} }); ++ const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .pgm = .{} }); + try std.testing.expectError(zigimg.Image.WriteError.Unsupported, write_error); + } + + { +- const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .ppm = .{} }); ++ const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .ppm = .{} }); + try std.testing.expectError(zigimg.Image.WriteError.Unsupported, write_error); + } + } +diff --git a/tests/formats/pam_test.zig b/tests/formats/pam_test.zig +index 2d58d45..5742f77 100644 +--- a/tests/formats/pam_test.zig ++++ b/tests/formats/pam_test.zig +@@ -5,10 +5,10 @@ const zigimg = @import("zigimg"); + + test "rejects non-PAM images" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + +@@ -17,10 +17,10 @@ test "rejects non-PAM images" { + + test "rejects PAM images with unsupported depth" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/unsupported_depth.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + +@@ -29,10 +29,10 @@ test "rejects PAM images with unsupported depth" { + + test "rejects PAM images with invalid maxval" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/invalid_maxval.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + +@@ -41,10 +41,10 @@ test "rejects PAM images with invalid maxval" { + + test "rejects PAM images with component values greater than maxval" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/value_greater_than_maxval.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + try helpers.expectError(invalid, zigimg.Image.ReadError.InvalidData); +@@ -52,10 +52,10 @@ test "rejects PAM images with component values greater than maxval" { + + test "rejects PAM images with unknown tuple type" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/unknown_tupletype.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + +@@ -64,10 +64,10 @@ test "rejects PAM images with unknown tuple type" { + + test "rejects PAM images with invalid first token" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/invalid_first_token.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + +@@ -76,10 +76,10 @@ test "rejects PAM images with invalid first token" { + + test "rejects PAM images with tuple type not matching other parameters" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/non_matching_tuple_type.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + +@@ -88,10 +88,10 @@ test "rejects PAM images with tuple type not matching other parameters" { + + test "accepts comments" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_blackandwhite_comments.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + defer image.deinit(helpers.zigimg_test_allocator); +@@ -110,10 +110,10 @@ test "accepts comments" { + + test "reads blackandwhite pam" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_blackandwhite.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + defer image.deinit(helpers.zigimg_test_allocator); +@@ -132,10 +132,10 @@ test "reads blackandwhite pam" { + + test "reads blackandwhite_alpha pam" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_blackandwhite_alpha.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + defer image.deinit(helpers.zigimg_test_allocator); +@@ -154,10 +154,10 @@ test "reads blackandwhite_alpha pam" { + + test "reads grayscale pam with maxval 255" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_grayscale_maxval_255.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + defer image.deinit(helpers.zigimg_test_allocator); +@@ -176,10 +176,10 @@ test "reads grayscale pam with maxval 255" { + + test "reads grayscale alpha pam with maxval 255" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_grayscale_alpha_maxval_255.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + defer image.deinit(helpers.zigimg_test_allocator); +@@ -198,10 +198,10 @@ test "reads grayscale alpha pam with maxval 255" { + + test "read of rgb pam with maxval 255" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/horse.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + defer image.deinit(helpers.zigimg_test_allocator); +@@ -211,10 +211,10 @@ test "read of rgb pam with maxval 255" { + + test "basic read-write-read produces same result" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_grayscale_alpha_maxval_255.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + defer image.deinit(helpers.zigimg_test_allocator); +@@ -239,7 +239,7 @@ test "basic read-write-read produces same result" { + + var decoded_image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_back_stream); + defer decoded_image.deinit(helpers.zigimg_test_allocator); +- ++ + try std.testing.expectEqual(@as(usize, 4), decoded_image.height); + try std.testing.expectEqual(@as(usize, 4), decoded_image.width); + try helpers.expectEqSlice(zigimg.color.Grayscale8Alpha, decoded_image.pixels.grayscale8Alpha, &[16]zigimg.color.Grayscale8Alpha{ +@@ -254,10 +254,10 @@ test "basic read-write-read produces same result" { + + test "reads rgba pam with maxval 255" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_rgba_maxval_255.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + defer image.deinit(helpers.zigimg_test_allocator); +@@ -273,10 +273,10 @@ test "reads rgba pam with maxval 255" { + + test "reads rgba pam with maxval 65535" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_rgba_maxval_65535.pam"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); + defer image.deinit(helpers.zigimg_test_allocator); +diff --git a/tests/formats/pcx_test.zig b/tests/formats/pcx_test.zig +index db5d556..ee51851 100644 +--- a/tests/formats/pcx_test.zig ++++ b/tests/formats/pcx_test.zig +@@ -7,10 +7,10 @@ const toU8 = zigimg.color.ScaleValue(u8); + + test "PCX indexed1 (linear)" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp1.pcx"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var pcx_file = pcx.PCX{}; + +@@ -45,10 +45,10 @@ test "PCX indexed1 (linear)" { + + test "PCX indexed4 (linear)" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp4.pcx"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var pcx_file = pcx.PCX{}; + +@@ -84,10 +84,10 @@ test "PCX indexed4 (linear)" { + + test "PCX indexed8 (linear)" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp8.pcx"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var pcx_file = pcx.PCX{}; + +@@ -125,10 +125,10 @@ test "PCX indexed8 (linear)" { + + test "PCX indexed24 (planar)" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp24.pcx"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var pcx_file = pcx.PCX{}; + +@@ -169,25 +169,25 @@ test "Write PCX indexed1 (odd width)" { + const image_file_name = "zigimg_pcx_indexed1_odd.pcx"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp1.pcx"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .pcx = .{}, + }); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var pcx_file = pcx.PCX{}; + +@@ -242,18 +242,18 @@ test "Write PCX indexed 1 (even width)" { + const image_file_name = "zigimg_pcx_indexed1_even.pcx"; + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try image_pattern.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try image_pattern.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .pcx = .{}, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var pcx_file = pcx.PCX{}; + +@@ -285,25 +285,25 @@ test "Write PCX indexed4 (odd width)" { + const image_file_name = "zigimg_pcx_indexed4_odd.pcx"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp4.pcx"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .pcx = .{}, + }); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var pcx_file = pcx.PCX{}; + +@@ -359,18 +359,18 @@ test "Write PCX indexed 4 (even width)" { + const image_file_name = "zigimg_pcx_indexed4_even.pcx"; + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try rainbow_test.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try rainbow_test.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .pcx = .{}, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var pcx_file = pcx.PCX{}; + +@@ -400,25 +400,25 @@ test "Write PCX indexed8 (odd width)" { + const image_file_name = "zigimg_pcx_indexed8_odd.pcx"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp8.pcx"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .pcx = .{}, + }); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var pcx_file = pcx.PCX{}; + +@@ -472,18 +472,18 @@ test "Write PCX indexed 8 (even width)" { + const image_file_name = "zigimg_pcx_indexed8_even.pcx"; + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try rainbow_test.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try rainbow_test.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .pcx = .{}, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var pcx_file = pcx.PCX{}; + +@@ -513,25 +513,25 @@ test "Write PCX rgb24 (odd width)" { + const image_file_name = "zigimg_pcx_bpp24_odd.pcx"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp24.pcx"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .pcx = .{}, + }); + + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var pcx_file = pcx.PCX{}; + +@@ -594,18 +594,18 @@ test "Write PCX rgb24 (even width)" { + const image_file_name = "zigimg_pcx_rgb24_even.pcx"; + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try rainbow_test.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ ++ try rainbow_test.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ + .pcx = .{}, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var pcx_file = pcx.PCX{}; + +diff --git a/tests/formats/png_test.zig b/tests/formats/png_test.zig +index ec39855..def25c7 100644 +--- a/tests/formats/png_test.zig ++++ b/tests/formats/png_test.zig +@@ -8,10 +8,10 @@ const valid_header_data = png.magic_header ++ "\x00\x00\x00\x0d" ++ png.Chunks.I + + test "png: Should error on non PNG images" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const invalidFile = png.PNG.readImage(helpers.zigimg_test_allocator, &read_stream); + +@@ -111,10 +111,10 @@ test "png: Indexed PNG with transparency (Aseprite output)" { + // mlarouche: While the full test suite already test this image, I like having a smaller test that I can verify + // some specific info myself + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "png/aseprite_indexed_transparent.png"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var png_image = try png.PNG.readImage(helpers.zigimg_test_allocator, &read_stream); + defer png_image.deinit(helpers.zigimg_test_allocator); +@@ -180,18 +180,18 @@ test "png: Don't write tRNS chunk in indexed format when there is no alpha" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_indexed_no_trns.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .png = .{}, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var check_trns_processor: CheckTrnsPresentProcessor = .{}; + +@@ -220,18 +220,18 @@ test "png: Write tRNS chunk in indexed format only when alpha is present" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_indexed_trns.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .png = .{}, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var check_trns_processor: CheckTrnsPresentProcessor = .{}; + +@@ -268,16 +268,16 @@ test "png: Write indexed1 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_indexed1.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -322,16 +322,16 @@ test "png: Write indexed2 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_indexed2.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -376,16 +376,16 @@ test "png: Write indexed4 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_indexed4.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -430,16 +430,16 @@ test "png: Write indexed8 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_indexed8.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -475,16 +475,16 @@ test "png: Write grayscale1 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_grayscale1.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -513,16 +513,16 @@ test "png: Write grayscale2 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_grayscale2.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -551,16 +551,16 @@ test "png: Write grayscale4 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_grayscale4.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -589,16 +589,16 @@ test "png: Write grayscale8 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_grayscale8.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -628,16 +628,16 @@ test "png: Write grayscale8Alpha format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_grayscale8Alpha.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -667,16 +667,16 @@ test "png: Write grayscale16 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_grayscale16.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -706,16 +706,16 @@ test "png: Write grayscale16Alpha format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_grayscale16Alpha.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -747,16 +747,16 @@ test "png: Write rgb24 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_rgb24.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -790,16 +790,16 @@ test "png: Write rgba32 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_rgba32.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -833,16 +833,16 @@ test "png: Write rgb48 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_rgb48.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -876,16 +876,16 @@ test "png: Write rgba64 format" { + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + const image_file_name = "zigimg_png_rgba64.png"; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var options = png.DefaultOptions.init(.{}); + +@@ -910,32 +910,32 @@ test "png: Official Test Suite" { + + // Useful to quickly test everything on full dir of images + pub fn testWithDir(directory: []const u8, test_md5_signature: bool) !void { +- var test_dir_opt = std.fs.cwd().openDir(directory, .{ .access_sub_paths = false, .no_follow = true, .iterate = true }) catch null; ++ var test_dir_opt = std.Io.Dir.openDir(std.Io.Dir.cwd(), std.testing.io, directory, .{ .access_sub_paths = false, .follow_symlinks = false, .iterate = true }) catch null; + if (test_dir_opt) |*test_dir| { +- defer test_dir.close(); ++ defer test_dir.close(std.testing.io); + var it = test_dir.iterate(); + if (test_md5_signature) { +- std.debug.print("\n", .{}); ++ std.log.info("\n", .{}); + } +- while (try it.next()) |entry| { ++ while (try it.next(std.testing.io)) |entry| { + if (entry.kind != .file or !std.mem.eql(u8, std.fs.path.extension(entry.name), ".png")) { + continue; + } + + if (test_md5_signature) { +- std.debug.print("Testing file {s} ... ", .{entry.name}); ++ std.log.info("Testing file {s} ... ", .{entry.name}); + } + +- var test_file = try test_dir.openFile(entry.name, .{ .mode = .read_only }); +- defer test_file.close(); ++ var test_file = try test_dir.openFile(std.testing.io, entry.name, .{ .mode = .read_only }); ++ defer test_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(test_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, test_file, read_buffer[0..]); + + if (entry.name[0] == 'x' and entry.name[2] != 't' and entry.name[2] != 's') { + try std.testing.expectError(zigimg.Image.ReadError.InvalidData, png.loadHeader(&read_stream)); + if (test_md5_signature) { +- std.debug.print("OK\n", .{}); ++ std.log.info("OK\n", .{}); + } + continue; + } +@@ -946,7 +946,7 @@ pub fn testWithDir(directory: []const u8, test_md5_signature: bool) !void { + const error_result = png.loadWithHeader(&read_stream, &header, std.testing.allocator, default_options.get()); + try std.testing.expectError(zigimg.Image.ReadError.InvalidData, error_result); + if (test_md5_signature) { +- std.debug.print("OK\n", .{}); ++ std.log.info("OK\n", .{}); + } + continue; + } +@@ -968,11 +968,11 @@ pub fn testWithDir(directory: []const u8, test_md5_signature: bool) !void { + @memcpy(test_data_name[len - 3 .. len], "tsd"); + + // Read test data and check with it +- if (test_dir.openFile(test_data_name[0..len], .{ .mode = .read_only })) |test_data_file| { +- defer test_data_file.close(); ++ if (test_dir.openFile(std.testing.io, test_data_name[0..len], .{ .mode = .read_only })) |test_data_file| { ++ defer test_data_file.close(std.testing.io); + + var test_read_buffer: [512]u8 = undefined; +- var test_file_reader = test_data_file.reader(test_read_buffer[0..]); ++ var test_file_reader = test_data_file.reader(std.testing.io, test_read_buffer[0..]); + var test_reader = &test_file_reader.interface; + + var expected_md5: [16]u8 = undefined; +@@ -983,31 +983,31 @@ pub fn testWithDir(directory: []const u8, test_md5_signature: bool) !void { + const str_md5 = try test_reader.takeDelimiterExclusive('\n'); + _ = try std.fmt.hexToBytes(expected_md5[0..], str_md5); + try std.testing.expectEqual(expected_pixel_format, std.meta.activeTag(result)); +- try std.testing.expectEqualSlices(u8, expected_md5[0..], md5_val[0..]); // catch std.debug.print("MD5 Expected: {s} Got {s}\n", .{std.fmt.fmtSliceHexUpper(expected_md5[0..]), std.fmt.fmtSliceHexUpper(md5_val[0..])}); ++ try std.testing.expectEqualSlices(u8, expected_md5[0..], md5_val[0..]); // catch std.log.info("MD5 Expected: {s} Got {s}\n", .{std.fmt.fmtSliceHexUpper(expected_md5[0..]), std.fmt.fmtSliceHexUpper(md5_val[0..])}); + } else |_| { + // If there is no test data assume test is correct and write it out + try writeTestData(test_dir, test_data_name[0..len], &result, md5_val[0..]); + } + + if (test_md5_signature) { +- std.debug.print("OK\n", .{}); ++ std.log.info("OK\n", .{}); + } + + // Write Raw bytes + // std.mem.copyForwards(u8, tst_data_name[len - 3 .. len + 1], "data"); + // var rawoutput = try idir.createFile(tst_data_name[0 .. len + 1], .{}); +- // defer rawoutput.close(); ++ // defer rawoutput.close(std.testing.io); + // try rawoutput.writeAll(result_bytes); + } + } + } + +-fn writeTestData(dir: *std.fs.Dir, test_data_name: []const u8, result: *zigimg.color.PixelStorage, md5_value: []const u8) !void { +- var test_output = try dir.createFile(test_data_name, .{}); +- defer test_output.close(); ++fn writeTestData(dir: *std.Io.Dir, test_data_name: []const u8, result: *zigimg.color.PixelStorage, md5_value: []const u8) !void { ++ var test_output = try dir.createFile(std.testing.io, test_data_name, .{}); ++ defer test_output.close(std.testing.io); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var file_writer = test_output.writer(write_buffer[0..]); ++ var file_writer = test_output.writer(std.testing.io, write_buffer[0..]); + var writer = &file_writer.interface; + + try writer.print("{s}\n{X}", .{ @tagName(result.*), md5_value }); +@@ -1018,14 +1018,14 @@ fn writeTestData(dir: *std.fs.Dir, test_data_name: []const u8, result: *zigimg.c + test "png: InfoProcessor on Png Test suite" { + const directory = helpers.fixtures_path ++ "png/"; + +- var test_dir_opt = std.fs.cwd().openDir(directory, .{ .access_sub_paths = false, .no_follow = true, .iterate = true }) catch null; ++ var test_dir_opt = std.Io.Dir.openDir(std.Io.Dir.cwd(), std.testing.io, directory, .{ .access_sub_paths = false, .iterate = true }) catch null; + if (test_dir_opt) |*test_dir| { +- defer test_dir.close(); ++ defer test_dir.close(std.testing.io); + var it = test_dir.iterate(); + + var info_buffer: [16384]u8 = undefined; + +- while (try it.next()) |entry| { ++ while (try it.next(std.testing.io)) |entry| { + if (entry.kind != .file or !std.mem.eql(u8, std.fs.path.extension(entry.name), ".png")) { + continue; + } +@@ -1033,11 +1033,11 @@ test "png: InfoProcessor on Png Test suite" { + var info_stream = zigimg.io.WriteStream.initMemory(info_buffer[0..]); + var options = png.InfoProcessor.PngInfoOptions.init(png.InfoProcessor.init(info_stream.writer())); + +- var test_file = try test_dir.openFile(entry.name, .{ .mode = .read_only }); +- defer test_file.close(); ++ var test_file = try test_dir.openFile(std.testing.io, entry.name, .{ .mode = .read_only }); ++ defer test_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(test_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, test_file, read_buffer[0..]); + if (entry.name[0] == 'x') { + continue; + } +@@ -1051,13 +1051,13 @@ test "png: InfoProcessor on Png Test suite" { + @memcpy(test_data_name[len - 4 .. len], "info"); + + // Read test data and check with it +- if (test_dir.openFile(test_data_name[0..len], .{ .mode = .read_only })) |test_data_file| { +- defer test_data_file.close(); ++ if (test_dir.openFile(std.testing.io, test_data_name[0..len], .{ .mode = .read_only })) |test_data_file| { ++ defer test_data_file.close(std.testing.io); + + var expected_data_buffer: [16384]u8 = undefined; + + var test_read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var file_reader = test_data_file.reader(test_read_buffer[0..]); ++ var file_reader = test_data_file.reader(std.testing.io, test_read_buffer[0..]); + var reader = &file_reader.interface; + + const loaded = try reader.readSliceShort(expected_data_buffer[0..]); +@@ -1065,11 +1065,11 @@ test "png: InfoProcessor on Png Test suite" { + try std.testing.expectEqualSlices(u8, expected_data_buffer[0..loaded], info_buffer[0..loaded]); + } else |_| { + // If there is no test data assume test is correct and write it out +- var test_output_file = try test_dir.createFile(test_data_name[0..len], .{}); +- defer test_output_file.close(); ++ var test_output_file = try test_dir.createFile(std.testing.io, test_data_name[0..len], .{}); ++ defer test_output_file.close(std.testing.io); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var file_writer = test_output_file.writer(write_buffer[0..]); ++ var file_writer = test_output_file.writer(std.testing.io, write_buffer[0..]); + var writer = &file_writer.interface; + + try writer.writeAll(info_buffer[0..@intCast(info_stream.getPos())]); +diff --git a/tests/formats/qoi_test.zig b/tests/formats/qoi_test.zig +index 96256e3..5a9f985 100644 +--- a/tests/formats/qoi_test.zig ++++ b/tests/formats/qoi_test.zig +@@ -8,10 +8,10 @@ const zero_qoi_file = helpers.fixtures_path ++ "qoi/zero.qoi"; + + test "Should error on non QOI images" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var qoi_file = qoi.QOI{}; + +@@ -22,10 +22,10 @@ test "Should error on non QOI images" { + + test "Read zero.qoi file" { + const file = try helpers.testOpenFile(zero_qoi_file); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var qoi_file = qoi.QOI{}; + +diff --git a/tests/formats/ras_test.zig b/tests/formats/ras_test.zig +index d7240a1..5452098 100644 +--- a/tests/formats/ras_test.zig ++++ b/tests/formats/ras_test.zig +@@ -5,10 +5,10 @@ const helpers = @import("../helpers.zig"); + + test "Should error on non RAS images" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var ras_file = ras.RAS{}; + +@@ -18,12 +18,12 @@ test "Should error on non RAS images" { + + test "Sun-Raster 24-bit RGB24 uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-rgb24.ras"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var ras_file = ras.RAS{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -46,12 +46,12 @@ test "Sun-Raster 24-bit RGB24 uncompressed" { + + test "Sun-Raster 24-bit BGR24 uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-bgr24.ras"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var ras_file = ras.RAS{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -74,12 +74,12 @@ test "Sun-Raster 24-bit BGR24 uncompressed" { + + test "Sun-Raster 32-bit xRGB uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-xrgb32.ras"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var ras_file = ras.RAS{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -98,12 +98,12 @@ test "Sun-Raster 32-bit xRGB uncompressed" { + + test "Sun-Raster 8-bit with palette uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-8bit.ras"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var ras_file = ras.RAS{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -124,12 +124,12 @@ test "Sun-Raster 8-bit with palette uncompressed" { + + test "Sun-Raster 8-bit grayscale uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-8bit-grayscale.ras"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var ras_file = ras.RAS{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -144,12 +144,12 @@ test "Sun-Raster 8-bit grayscale uncompressed" { + + test "Sun-Raster 1-bit black & white uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-blackwhite.ras"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var ras_file = ras.RAS{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -164,12 +164,12 @@ test "Sun-Raster 1-bit black & white uncompressed" { + + test "Sun-Raster bgr24 rle compressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-24bit-bgr-rle.ras"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var ras_file = ras.RAS{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +diff --git a/tests/formats/sgi_test.zig b/tests/formats/sgi_test.zig +index d098ef7..ed23769 100644 +--- a/tests/formats/sgi_test.zig ++++ b/tests/formats/sgi_test.zig +@@ -5,10 +5,10 @@ const zigimg = @import("zigimg"); + + test "Should error on non SGI images" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var sgi_file = sgi.SGI{}; + +@@ -18,12 +18,12 @@ test "Should error on non SGI images" { + + test "SGI 24-bit uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-rgb24.sgi"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var sgi_file = sgi.SGI{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -46,12 +46,12 @@ test "SGI 24-bit uncompressed" { + + test "SGI grayscale uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-blackwhite.sgi"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var sgi_file = sgi.SGI{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -66,12 +66,12 @@ test "SGI grayscale uncompressed" { + + test "SGI 32-bit RGBA uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-rgba.sgi"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var sgi_file = sgi.SGI{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -94,12 +94,12 @@ test "SGI 32-bit RGBA uncompressed" { + + test "SGI RGB48be uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-rgb48be.sgi"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var sgi_file = sgi.SGI{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -122,12 +122,12 @@ test "SGI RGB48be uncompressed" { + + test "SGI grayscale rle compressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-gray-rle.sgi"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var sgi_file = sgi.SGI{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -142,12 +142,12 @@ test "SGI grayscale rle compressed" { + + test "SGI 24-bit rle compressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-24bit-rle.sgi"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var sgi_file = sgi.SGI{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -170,12 +170,12 @@ test "SGI 24-bit rle compressed" { + + test "SGI RGB48be rle uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-rgb48be-rle.sgi"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var sgi_file = sgi.SGI{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -198,12 +198,12 @@ test "SGI RGB48be rle uncompressed" { + + test "SGI 32-bit RGBA rle compressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-rgba-rle.sgi"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var sgi_file = sgi.SGI{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +diff --git a/tests/formats/tga_test.zig b/tests/formats/tga_test.zig +index 83f2d76..c3182d4 100644 +--- a/tests/formats/tga_test.zig ++++ b/tests/formats/tga_test.zig +@@ -5,10 +5,10 @@ const zigimg = @import("zigimg"); + + test "Should error on non TGA images" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -19,10 +19,10 @@ test "Should error on non TGA images" { + + test "Read ubw8 TGA file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ubw8.tga"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -56,10 +56,10 @@ test "Read ubw8 TGA file" { + + test "Read ucm8 TGA file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ucm8.tga"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -101,10 +101,10 @@ test "Read ucm8 TGA file" { + + test "Read utc16 TGA file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc16.tga"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -140,10 +140,10 @@ test "Read utc16 TGA file" { + + test "Read utc24 TGA file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc24.tga"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -179,10 +179,10 @@ test "Read utc24 TGA file" { + + test "Read utc32 TGA file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc32.tga"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -218,10 +218,10 @@ test "Read utc32 TGA file" { + + test "Read cbw8 TGA file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/cbw8.tga"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -255,10 +255,10 @@ test "Read cbw8 TGA file" { + + test "Read ccm8 TGA file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ccm8.tga"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -300,10 +300,10 @@ test "Read ccm8 TGA file" { + + test "Read ctc24 TGA file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ctc24.tga"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -339,10 +339,10 @@ test "Read ctc24 TGA file" { + + test "Read matte-01 TGA file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/matte-01.tga"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -384,10 +384,10 @@ test "Read matte-01 TGA file" { + + test "Read font TGA file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/font.tga"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -411,10 +411,10 @@ test "Read font TGA file" { + + test "Read stopsignsmall TGA v1 file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/stopsignsmall.tga"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -436,10 +436,10 @@ test "Read stopsignsmall TGA v1 file" { + + test "Read stopsignsmallcompressed TGA v1 file" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/stopsignsmallcompressed.tga"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -463,11 +463,11 @@ test "Write TGA uncompressed grayscale8" { + const image_file_name = "zigimg_tga_uncompressed_grayscale8.tga"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ubw8.tga"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + const encoder_options = zigimg.Image.EncoderOptions{ +@@ -500,15 +500,15 @@ test "Write TGA uncompressed grayscale8" { + }; + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], encoder_options); ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], encoder_options); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -558,15 +558,15 @@ test "Write TGA compressed grayscale8" { + const image_file_name = "zigimg_tga_compressed_grayscale8.tga"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/cbw8.tga"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = true, + .color_map_depth = 16, +@@ -575,13 +575,13 @@ test "Write TGA compressed grayscale8" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -617,15 +617,15 @@ test "Write uncompressed indexed8 (color map 16-bit)" { + const image_file_name = "zigimg_tga_uncompressed_indexed8.tga"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ucm8.tga"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = false, + .color_map_depth = 16, +@@ -634,13 +634,13 @@ test "Write uncompressed indexed8 (color map 16-bit)" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -684,15 +684,15 @@ test "Write uncompressed indexed8 (color map 24-bit)" { + const image_file_name = "zigimg_tga_uncompressed_indexed8.tga"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ucm8.tga"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = false, + .color_map_depth = 24, +@@ -701,13 +701,13 @@ test "Write uncompressed indexed8 (color map 24-bit)" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -751,15 +751,15 @@ test "Write compressed indexed8 (color map 16-bit)" { + const image_file_name = "zigimg_tga_compressed_indexed8.tga"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ccm8.tga"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = true, + .color_map_depth = 16, +@@ -768,13 +768,13 @@ test "Write compressed indexed8 (color map 16-bit)" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -818,15 +818,15 @@ test "Write compressed indexed8 (color map 24-bit)" { + const image_file_name = "zigimg_tga_compressed_indexed8.tga"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ccm8.tga"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = true, + .color_map_depth = 24, +@@ -835,13 +835,13 @@ test "Write compressed indexed8 (color map 24-bit)" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -885,15 +885,15 @@ test "Write uncompressed 16-bit true color TGA" { + const image_file_name = "zigimg_tga_uncompressed_true_color_16.tga"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc16.tga"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = false, + .color_map_depth = 16, +@@ -902,13 +902,13 @@ test "Write uncompressed 16-bit true color TGA" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -946,15 +946,15 @@ test "Write compressed 16-bit true color TGA" { + const image_file_name = "zigimg_tga_compressed_true_color_16.tga"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc16.tga"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = true, + .color_map_depth = 16, +@@ -963,13 +963,13 @@ test "Write compressed 16-bit true color TGA" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -1007,15 +1007,15 @@ test "Write uncompressed 24-bit true color TGA" { + const image_file_name = "zigimg_tga_uncompressed_true_color_24.tga"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc24.tga"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = false, + .color_map_depth = 24, +@@ -1024,13 +1024,13 @@ test "Write uncompressed 24-bit true color TGA" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -1068,15 +1068,15 @@ test "Write compressed 24-bit true color TGA" { + const image_file_name = "zigimg_tga_compressed_true_color_24.tga"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ctc24.tga"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = true, + .color_map_depth = 24, +@@ -1085,13 +1085,13 @@ test "Write compressed 24-bit true color TGA" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -1129,15 +1129,15 @@ test "Write uncompressed 32-bit true color TGA" { + const image_file_name = "zigimg_tga_uncompressed_true_color_32.tga"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc32.tga"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = false, + .color_map_depth = 24, +@@ -1146,13 +1146,13 @@ test "Write uncompressed 32-bit true color TGA" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -1190,15 +1190,15 @@ test "Write compressed 32-bit true color TGA" { + const image_file_name = "zigimg_tga_compressed_true_color_32.tga"; + + var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ctc32.tga"); +- defer source_file.close(); ++ defer source_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; + +- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); ++ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); + defer source_image.deinit(helpers.zigimg_test_allocator); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = true, + .color_map_depth = 24, +@@ -1207,13 +1207,13 @@ test "Write compressed 32-bit true color TGA" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -1270,7 +1270,7 @@ test "Write uncompressed Rgb24 to TGA" { + @memcpy(source_image.pixels.rgb24[0..], uncompressed_source[0..]); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = false, + .color_map_depth = 24, +@@ -1279,14 +1279,14 @@ test "Write uncompressed Rgb24 to TGA" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -1329,7 +1329,7 @@ test "Write compressed Rgb24 to TGA" { + @memcpy(source_image.pixels.rgb24[0..], uncompressed_source[0..]); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = true, + .color_map_depth = 24, +@@ -1338,14 +1338,14 @@ test "Write compressed Rgb24 to TGA" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -1403,7 +1403,7 @@ test "Write uncompressed Rgba32 to TGA" { + @memcpy(source_image.pixels.rgba32[0..], uncompressed_source[0..]); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = false, + .color_map_depth = 24, +@@ -1412,14 +1412,14 @@ test "Write uncompressed Rgba32 to TGA" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +@@ -1478,7 +1478,7 @@ test "Write compressed Rgba32 to TGA" { + @memcpy(source_image.pixels.rgba32[0..], uncompressed_source[0..]); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ ++ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ + .tga = .{ + .rle_compressed = true, + .color_map_depth = 24, +@@ -1487,14 +1487,14 @@ test "Write compressed Rgba32 to TGA" { + }, + }); + defer { +- std.fs.cwd().deleteFile(image_file_name) catch {}; ++ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; + } + + const read_file = try helpers.testOpenFile(image_file_name); +- defer read_file.close(); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var tga_file = tga.TGA{}; + +diff --git a/tests/formats/tiff_test.zig b/tests/formats/tiff_test.zig +index cce4cfa..2a8a042 100644 +--- a/tests/formats/tiff_test.zig ++++ b/tests/formats/tiff_test.zig +@@ -5,10 +5,10 @@ const zigimg = @import("zigimg"); + + test "Should error on non TIFF images" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + var sgi_file = tiff.TIFF{}; + +@@ -18,12 +18,12 @@ test "Should error on non TIFF images" { + + test "TIFF/LE monochrome black uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-monob-raw.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -39,12 +39,12 @@ test "TIFF/LE monochrome black uncompressed" { + + test "TIFF/LE grayscale8 uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-grayscale8-raw.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -61,12 +61,12 @@ test "TIFF/LE grayscale8 uncompressed" { + + test "TIFF/LE 8-bit with colormap uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-pal8-raw.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -87,12 +87,12 @@ test "TIFF/LE 8-bit with colormap uncompressed" { + + test "TIFF/LE 24-bit uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgb24-raw.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -115,12 +115,12 @@ test "TIFF/LE 24-bit uncompressed" { + + test "TIFF/BE rgb24 gray single strip uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/big-endian/sample-rgb24-single-strip.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -143,12 +143,12 @@ test "TIFF/BE rgb24 gray single strip uncompressed" { + + test "TIFF/BE rgb24 color single strip uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/big-endian/sample-pal8-raw.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -171,12 +171,12 @@ test "TIFF/BE rgb24 color single strip uncompressed" { + + test "TIFF/BE 24-bit uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/big-endian/sample-rgb24-raw.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -199,12 +199,12 @@ test "TIFF/BE 24-bit uncompressed" { + + test "TIFF/LE RGBA uncompressed" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgba-raw.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -223,12 +223,12 @@ test "TIFF/LE RGBA uncompressed" { + + test "TIFF/LE monochrome black packbits" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-monob-packbits.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -244,12 +244,12 @@ test "TIFF/LE monochrome black packbits" { + + test "TIFF/LE grayscale8 packbits" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-grayscale8-packbits.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -266,12 +266,12 @@ test "TIFF/LE grayscale8 packbits" { + + test "TIFF/LE 8-bit with colormap packbits" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-pal8-packbits.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -292,12 +292,12 @@ test "TIFF/LE 8-bit with colormap packbits" { + + test "TIFF/LE 24-bit packbits" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgb24-packbits.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -320,12 +320,12 @@ test "TIFF/LE 24-bit packbits" { + + test "TIFF/LE RGBA packbits" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgba-packbits.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -344,12 +344,12 @@ test "TIFF/LE RGBA packbits" { + + test "TIFF/LE monochrome black CCITT" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/ccitt_rle.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -364,12 +364,12 @@ test "TIFF/LE monochrome black CCITT" { + + test "TIFF/LE monochrome black LZW" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-monob-lzw.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -385,12 +385,12 @@ test "TIFF/LE monochrome black LZW" { + + test "TIFF/LE grayscale8 LZW" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-grayscale8-lzw.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -407,12 +407,12 @@ test "TIFF/LE grayscale8 LZW" { + + test "TIFF/LE 8-bit with colormap LZW" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-pal8-lzw.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -433,12 +433,12 @@ test "TIFF/LE 8-bit with colormap LZW" { + + test "TIFF/LE 24-bit LZW" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgb24-lzw.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -461,12 +461,12 @@ test "TIFF/LE 24-bit LZW" { + + test "TIFF/LE RGBA LZW" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgba-lzw.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -485,12 +485,12 @@ test "TIFF/LE RGBA LZW" { + + test "TIFF/LE monochrome black Deflate" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-monob-deflate.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_bitmap = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_bitmap.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -506,12 +506,12 @@ test "TIFF/LE monochrome black Deflate" { + + test "TIFF/LE grayscale8 Deflate" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-grayscale8-deflate.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -528,12 +528,12 @@ test "TIFF/LE grayscale8 Deflate" { + + test "TIFF/LE 8-bit with colormap Deflate" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-pal8-deflate.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -554,12 +554,12 @@ test "TIFF/LE 8-bit with colormap Deflate" { + + test "TIFF/LE 24-bit Deflate" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgb24-deflate.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +@@ -582,12 +582,12 @@ test "TIFF/LE 24-bit Deflate" { + + test "TIFF/LE RGBA Deflate" { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgba-deflate.tiff"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_tiff = tiff.TIFF{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); + defer pixels.deinit(helpers.zigimg_test_allocator); +diff --git a/tests/formats/xbm_test.zig b/tests/formats/xbm_test.zig +index 52eb29c..053e41e 100644 +--- a/tests/formats/xbm_test.zig ++++ b/tests/formats/xbm_test.zig +@@ -30,12 +30,12 @@ const simple_4x4_xbm = + test "XBM: invalid file format" { + { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "xbm/bad_missing_dim.xbm"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_xbm = xbm.XBM{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const actual_error = the_xbm.read(helpers.zigimg_test_allocator, &read_stream); + try helpers.expectError(actual_error, zigimg.Image.ReadError.InvalidData); +@@ -43,12 +43,12 @@ test "XBM: invalid file format" { + + { + const file = try helpers.testOpenFile(helpers.fixtures_path ++ "xbm/bad_missing_pixels.xbm"); +- defer file.close(); ++ defer file.close(std.testing.io); + + var the_xbm = xbm.XBM{}; + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); + + const actual_error = the_xbm.read(helpers.zigimg_test_allocator, &read_stream); + try helpers.expectError(actual_error, zigimg.Image.ReadError.InvalidData); +diff --git a/tests/helpers.zig b/tests/helpers.zig +index 9a4ef29..b9f06b1 100644 +--- a/tests/helpers.zig ++++ b/tests/helpers.zig +@@ -30,17 +30,17 @@ pub inline fn expectApproxEqRel(actual: anytype, expected: anytype, tolerance: a + return try std.testing.expectApproxEqRel(expected, actual, tolerance); + } + +-pub fn testOpenFile(file_path: []const u8) !std.fs.File { +- return std.fs.cwd().openFile(file_path, .{}) catch |err| ++pub fn testOpenFile(file_path: []const u8) !std.Io.File { ++ return std.Io.Dir.openFile(std.Io.Dir.cwd(), std.testing.io, file_path, .{}) catch |err| + if (err == error.FileNotFound) return error.SkipZigTest else return err; + } + + pub fn testImageFromFile(image_path: []const u8, buffer: []u8) !zigimg.Image { +- return zigimg.Image.fromFilePath(zigimg_test_allocator, image_path, buffer) catch |err| ++ return zigimg.Image.fromFilePath(zigimg_test_allocator, std.testing.io, image_path, buffer) catch |err| + if (err == error.FileNotFound) return error.SkipZigTest else return err; + } + + pub fn testReadFile(file_path: []const u8, buffer: []u8) ![]u8 { +- return std.fs.cwd().readFile(file_path, buffer) catch |err| ++ return std.Io.Dir.readFile(std.Io.Dir.cwd(), std.testing.io, file_path, buffer) catch |err| + if (err == error.FileNotFound) return error.SkipZigTest else return err; + } +diff --git a/tests/image_test.zig b/tests/image_test.zig +index ccd2d11..529be41 100644 +--- a/tests/image_test.zig ++++ b/tests/image_test.zig +@@ -187,7 +187,7 @@ test "Should detect BMP properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .bmp); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -213,7 +213,7 @@ test "Should detect GIF properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .gif); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -231,7 +231,7 @@ test "Should detect PCX properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .pcx); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -247,7 +247,7 @@ test "Should detect PBM properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .pbm); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -265,7 +265,7 @@ test "Should detect PGM properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .pgm); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -281,7 +281,7 @@ test "Should detect PPM properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .ppm); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -297,7 +297,7 @@ test "Should detect PNG properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .png); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -321,7 +321,7 @@ test "Should detect TGA properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .tga); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -334,7 +334,7 @@ test "Should detect QOI properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .qoi); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -350,7 +350,7 @@ test "Should detect JPEG properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .jpeg); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -366,7 +366,7 @@ test "Should detect Farbfeld properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .farbfeld); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -381,7 +381,7 @@ test "Should detect IFF/PBM properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .iff); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -396,7 +396,7 @@ test "Should detect RAS properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .ras); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -411,7 +411,7 @@ test "Should detect SGI properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .sgi); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -426,7 +426,7 @@ test "Should detect TIFF properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .tiff); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +@@ -449,7 +449,7 @@ test "Should detect XBM properly" { + + for (image_tests) |image_path| { + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); ++ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); + try std.testing.expect(format == .xbm); + + var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); +diff --git a/tests/io_test.zig b/tests/io_test.zig +index 2dc6f7a..1ec5d60 100644 +--- a/tests/io_test.zig ++++ b/tests/io_test.zig +@@ -55,12 +55,12 @@ test "io.ReadStream: should read and seek properly within a unbuffered file" { + var temp_folder = std.testing.tmpDir(.{}); + defer temp_folder.cleanup(); + +- try temp_folder.dir.writeFile(.{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); ++ try temp_folder.dir.writeFile(std.testing.io, .{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); + +- const read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); +- defer read_file.close(); ++ const read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, &.{}); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, &.{}); + + var reader = read_stream.reader(); + +@@ -99,12 +99,12 @@ test "io.ReadStream: should error on invalid seek within unbuffered file" { + var temp_folder = std.testing.tmpDir(.{}); + defer temp_folder.cleanup(); + +- try temp_folder.dir.writeFile(.{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); ++ try temp_folder.dir.writeFile(std.testing.io, .{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); + +- const read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); +- defer read_file.close(); ++ const read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); ++ defer read_file.close(std.testing.io); + +- var read_stream = zigimg.io.ReadStream.initFile(read_file, &.{}); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, &.{}); + + const seek_to_error = read_stream.seekTo(123456); + try helpers.expectError(seek_to_error, zigimg.io.ReadStream.SeekError.Unseekable); +@@ -122,13 +122,13 @@ test "io.ReadStream: should read and seek properly within a file with a small bu + var temp_folder = std.testing.tmpDir(.{}); + defer temp_folder.cleanup(); + +- try temp_folder.dir.writeFile(.{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); ++ try temp_folder.dir.writeFile(std.testing.io, .{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); + +- const read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); +- defer read_file.close(); ++ const read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); ++ defer read_file.close(std.testing.io); + + var small_buffer: [30]u8 = @splat(0); +- var read_stream = zigimg.io.ReadStream.initFile(read_file, small_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, small_buffer[0..]); + + var reader = read_stream.reader(); + +@@ -164,13 +164,13 @@ test "io.ReadStream: should read and seek properly within a file with a large bu + var temp_folder = std.testing.tmpDir(.{}); + defer temp_folder.cleanup(); + +- try temp_folder.dir.writeFile(.{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); ++ try temp_folder.dir.writeFile(std.testing.io, .{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); + +- const read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); +- defer read_file.close(); ++ const read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + var reader = read_stream.reader(); + +@@ -206,13 +206,13 @@ test "io.ReadStream: should error on invalid seek within a file with a large buf + var temp_folder = std.testing.tmpDir(.{}); + defer temp_folder.cleanup(); + +- try temp_folder.dir.writeFile(.{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); ++ try temp_folder.dir.writeFile(std.testing.io, .{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); + +- const read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); +- defer read_file.close(); ++ const read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); ++ defer read_file.close(std.testing.io); + + var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); ++ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); + + const seek_to_error = read_stream.seekTo(123456); + try helpers.expectError(seek_to_error, zigimg.io.ReadStream.SeekError.Unseekable); +@@ -261,10 +261,10 @@ test "io.WriteStream: should write property within a file with a small buffer" { + defer temp_folder.cleanup(); + + { +- const write_file = try temp_folder.dir.createFile(TEST_FILENAME, .{}); +- defer write_file.close(); ++ const write_file = try temp_folder.dir.createFile(std.testing.io, TEST_FILENAME, .{}); ++ defer write_file.close(std.testing.io); + +- var write_stream = zigimg.io.WriteStream.initFile(write_file, buffer[0..]); ++ var write_stream = zigimg.io.WriteStream.initFile(std.testing.io, write_file, buffer[0..]); + + var writer = write_stream.writer(); + try writer.writeAll(TEST_FILE_CONTENTS[0..3]); +@@ -283,11 +283,11 @@ test "io.WriteStream: should write property within a file with a small buffer" { + try writer.flush(); + } + +- var read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); +- defer read_file.close(); ++ var read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); ++ defer read_file.close(std.testing.io); + + var read_buffer: [1024]u8 = @splat(0); +- var file_reader = read_file.reader(read_buffer[0..]); ++ var file_reader = read_file.reader(std.testing.io, read_buffer[0..]); + + var reader = &file_reader.interface; + +@@ -305,11 +305,11 @@ test "io.WriteStream: should write property within a file with a large buffer" { + defer temp_folder.cleanup(); + + { +- const write_file = try temp_folder.dir.createFile(TEST_FILENAME, .{}); +- defer write_file.close(); ++ const write_file = try temp_folder.dir.createFile(std.testing.io, TEST_FILENAME, .{}); ++ defer write_file.close(std.testing.io); + + var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; +- var write_stream = zigimg.io.WriteStream.initFile(write_file, write_buffer[0..]); ++ var write_stream = zigimg.io.WriteStream.initFile(std.testing.io, write_file, write_buffer[0..]); + + var writer = write_stream.writer(); + try writer.writeAll(TEST_FILE_CONTENTS[0..3]); +@@ -328,11 +328,11 @@ test "io.WriteStream: should write property within a file with a large buffer" { + try writer.flush(); + } + +- var read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); +- defer read_file.close(); ++ var read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); ++ defer read_file.close(std.testing.io); + + var read_buffer: [1024]u8 = @splat(0); +- var file_reader = read_file.reader(read_buffer[0..]); ++ var file_reader = read_file.reader(std.testing.io, read_buffer[0..]); + + var reader = &file_reader.interface; + +diff --git a/zigimg.zig b/zigimg.zig +index 9f6b1a8..a53b23c 100644 +--- a/zigimg.zig ++++ b/zigimg.zig +@@ -38,5 +38,5 @@ test { + @import("tests/octree_quantizer_test.zig"), + @import("tests/pixel_format_converter_test.zig"), + @import("tests/pixel_format_test.zig"), +- }) |source_file| std.testing.refAllDeclsRecursive(source_file); ++ }) |source_file| std.testing.refAllDecls(source_file); + } +diff --git a/src/formats/gif.zig b/src/formats/gif.zig +--- a/src/formats/gif.zig ++++ b/src/formats/gif.zig +@@ -8,7 +8,7 @@ + const std = @import("std"); + const utils = @import("../utils.zig"); + +-pub const HeaderFlags = packed struct { ++pub const HeaderFlags = packed struct(u8) { + global_color_table_size: u3 = 0, + sorted: bool = false, + color_resolution: u3 = 0, +@@ -129,9 +129,9 @@ + pub const GIF = struct { + header: Header = .{}, + global_color_table: utils.FixedStorage(color.Rgb24, 256) = .{}, +- frames: std.ArrayList(FrameData) = .{}, +- comments: std.ArrayList(CommentExtension) = .{}, +- application_infos: std.ArrayList(ApplicationExtension) = .{}, ++ frames: std.ArrayList(FrameData) = .empty, ++ comments: std.ArrayList(CommentExtension) = .empty, ++ application_infos: std.ArrayList(ApplicationExtension) = .empty, + arena_allocator: std.heap.ArenaAllocator = undefined, + + pub const SubImage = struct { +@@ -147,6 +147,6 @@ + pub const FrameData = struct { + graphics_control: ?GraphicControlExtension = null, +- sub_images: std.ArrayList(*SubImage) = .{}, ++ sub_images: std.ArrayList(*SubImage) = .empty, + + pub fn deinit(self: *FrameData, allocator: std.mem.Allocator) void { + for (self.sub_images.items) |sub_image| { +@@ -683,7 +683,7 @@ + const final_pixel_format = self.findBestPixelFormat(); + + const frame_list_allocator = self.arena_allocator.child_allocator; + +- var frame_list = Image.Animation.FrameList{}; ++ var frame_list: Image.Animation.FrameList = .empty; + + if (self.frames.items.len == 0) { +diff --git a/src/compressions/lzw.zig b/src/compressions/lzw.zig +--- a/src/compressions/lzw.zig ++++ b/src/compressions/lzw.zig +@@ -11,9 +11,10 @@ + next_code: u13 = 0, + previous_code: ?u13 = null, +- dictionary: std.AutoArrayHashMap(u13, []const u8), ++ dictionary: std.AutoArrayHashMapUnmanaged(u13, []const u8) = .empty, ++ dict_allocator: std.mem.Allocator, + + remaining_data: ?u13 = null, + remaining_bits: u4 = 0, +@@ -27,7 +28,7 @@ + var result = Self{ + .area_allocator = std.heap.ArenaAllocator.init(allocator), + .code_size = initial_code_size, +- .dictionary = std.AutoArrayHashMap(u13, []const u8).init(allocator), ++ .dict_allocator = allocator, + .initial_code_size = initial_code_size, + .clear_code = @as(u13, 1) << @intCast(initial_code_size), + .end_information_code = (@as(u13, 1) << @intCast(initial_code_size)) + 1, +@@ -43,7 +44,7 @@ + + pub fn deinit(self: *Self) void { + self.area_allocator.deinit(); +- self.dictionary.deinit(); ++ self.dictionary.deinit(self.dict_allocator); + } + + pub fn decode(self: *Self, reader: *std.Io.Reader, writer: *std.Io.Writer) !void { +@@ -85,7 +86,7 @@ + var new_value = try allocator.alloc(u8, previous_value.len + 1); + std.mem.copyForwards(u8, new_value, previous_value); + new_value[previous_value.len] = value[0]; +- try self.dictionary.put(self.next_code, new_value); ++ try self.dictionary.put(self.dict_allocator, self.next_code, new_value); + + self.next_code += 1; + +@@ -109,7 +110,7 @@ + var new_value = try allocator.alloc(u8, previous_value.len + 1); + std.mem.copyForwards(u8, new_value, previous_value); + new_value[previous_value.len] = previous_value[0]; +- try self.dictionary.put(self.next_code, new_value); ++ try self.dictionary.put(self.dict_allocator, self.next_code, new_value); + + _ = try writer.write(new_value); + +@@ -154,7 +155,7 @@ + var data = try allocator.alloc(u8, 1); + data[0] = @as(u8, @truncate(index)); + +- try self.dictionary.put(index, data); ++ try self.dictionary.put(self.dict_allocator, index, data); + } + } + }; +diff --git a/src/formats/jpeg/huffman.zig b/src/formats/jpeg/huffman.zig +--- a/src/formats/jpeg/huffman.zig ++++ b/src/formats/jpeg/huffman.zig +@@ -9,7 +9,7 @@ + + const HuffmanCode = struct { length_minus_one: u4, code: u16 }; +-const HuffmanCodeMap = std.AutoArrayHashMap(HuffmanCode, u8); ++const HuffmanCodeMap = std.AutoArrayHashMapUnmanaged(HuffmanCode, u8); + + const JPEG_DEBUG = false; + const JPEG_VERY_DEBUG = false; +@@ -40,8 +40,8 @@ + for (code_counts) |count| total_huffman_codes += count; + +- var huffman_code_map = HuffmanCodeMap.init(allocator); +- errdefer huffman_code_map.deinit(); ++ var huffman_code_map: HuffmanCodeMap = .empty; ++ errdefer huffman_code_map.deinit(allocator); + + var fast_table: [1 << fast_bits]u8 = @splat(255); + var fast_size: [1 << fast_bits]u5 = @splat(0); +@@ -65,7 +65,7 @@ + + const byte = try reader.takeByte(); +- try huffman_code_map.put(.{ .length_minus_one = @as(u4, @intCast(i)), .code = code }, byte); ++ try huffman_code_map.put(allocator, .{ .length_minus_one = @as(u4, @intCast(i)), .code = code }, byte); + + // construct accelaration structure see stb_image + if (i + 1 <= fast_bits) { +@@ -96,5 +96,5 @@ + + pub fn deinit(self: *Table) void { +- self.code_map.deinit(); ++ self.code_map.deinit(self.allocator); + } + }; + diff --git a/third_party/zigimg/repo.bzl b/third_party/zigimg/repo.bzl index 282f0fb5..aa375cfc 100644 --- a/third_party/zigimg/repo.bzl +++ b/third_party/zigimg/repo.bzl @@ -1,12 +1,25 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") +def _zigimg_repo_impl(ctx): + ctx.download_and_extract( + url = "https://github.com/zigimg/zigimg/archive/{}.tar.gz".format(ctx.attr.commit), + stripPrefix = "zigimg-{}".format(ctx.attr.commit), + ) + + ctx.patch(ctx.attr.patch, strip = 1) + ctx.symlink(ctx.attr.build_file, "BUILD.bazel") + +_zigimg_repo = repository_rule( + implementation = _zigimg_repo_impl, + attrs = { + "commit": attr.string(mandatory = True), + "patch": attr.label(mandatory = True, allow_single_file = True), + "build_file": attr.label(mandatory = True, allow_single_file = True), + }, +) def repo(): - git_repository( + _zigimg_repo( name = "zigimg", - remote = "https://github.com/zigimg/zigimg.git", - # Match libvaxis build.zig.zon. - commit = "d695acd97c02e57bb151e8f659d1280f5cd6ca70", - build_file = Label("//third_party/zigimg:zigimg.bazel"), - patches = ["//third_party/zigimg:zig-0.17.patch"], - patch_args = ["-p1"], + commit = "7b98e82621fe302a9edc147df1191f4d1b7ff7a5", + patch = "//third_party/zigimg:bump-zig.patch", + build_file = "//third_party/zigimg:zigimg.bazel", ) diff --git a/third_party/zigimg/zig-0.17.patch b/third_party/zigimg/zig-0.17.patch deleted file mode 100644 index 99a60b5d..00000000 --- a/third_party/zigimg/zig-0.17.patch +++ /dev/null @@ -1,279 +0,0 @@ -diff --git a/src/Image/Managed.zig b/src/Image/Managed.zig ---- a/src/Image/Managed.zig -+++ b/src/Image/Managed.zig -@@ -18,7 +18,7 @@ - // This layout must match the one in Image - width: usize = 0, - height: usize = 0, --pixels: color.PixelStorage = .{ .invalid = void{} }, -+pixels: color.PixelStorage = .{ .invalid = {} }, - animation: Animation = .{}, - allocator: std.mem.Allocator = undefined, // Allocator needs to be last in order to be able to ptrCast to Image - -diff --git a/src/Image.zig b/src/Image.zig ---- a/src/Image.zig -+++ b/src/Image.zig -@@ -9,7 +9,7 @@ - - width: usize = 0, - height: usize = 0, --pixels: color.PixelStorage = .{ .invalid = void{} }, -+pixels: color.PixelStorage = .{ .invalid = {} }, - animation: Animation = .{}, - - const Image = @This(); -@@ -124,15 +124,15 @@ - const all_formats_delcs = std.meta.declarations(SupportedFormats); - var result: []const FormatInteraceFnType = &[0]FormatInteraceFnType{}; - for (all_formats_delcs) |decl| { -- const decl_value = @field(SupportedFormats, decl.name); -+ const decl_value = @field(SupportedFormats, decl); - const entry_type = @TypeOf(decl_value); - if (entry_type == type) { - const entry_type_info = @typeInfo(decl_value); - if (entry_type_info == .@"struct") { -- for (entry_type_info.@"struct".decls) |struct_entry| { -- if (std.mem.eql(u8, struct_entry.name, "formatInterface")) { -+ for (entry_type_info.@"struct".decl_names) |struct_entry| { -+ if (std.mem.eql(u8, struct_entry, "formatInterface")) { - result = result ++ [_]FormatInteraceFnType{ -- @field(decl_value, struct_entry.name), -+ @field(decl_value, struct_entry), - }; - break; - } -diff --git a/src/PixelFormatConverter.zig b/src/PixelFormatConverter.zig ---- a/src/PixelFormatConverter.zig -+++ b/src/PixelFormatConverter.zig -@@ -752,14 +752,7 @@ - } - - fn getFieldNameFromPixelFormat(comptime source_format: PixelFormat) []const u8 { -- const enum_fields = std.meta.fields(PixelFormat); -- inline for (enum_fields) |field| { -- if (field.value == @intFromEnum(source_format)) { -- return field.name; -- } -- } -- -- return ""; -+ return @tagName(source_format); - } - - // ======================== -@@ -782,19 +775,19 @@ - } - - fn grayscaleToGrayscale(comptime T: type, gray: anytype) T { -- const scaleValue = color.ScaleValue(std.meta.fieldInfo(T, .value).type); -+ const scaleValue = color.ScaleValue(@FieldType(T, "value")); - return .{ .value = scaleValue(gray.value) }; - } - - fn grayscaleAlphaToGrayscale(comptime T: type, gray: anytype) T { - const toF32 = color.ScaleValue(f32); -- const scaleValue = color.ScaleValue(std.meta.fieldInfo(T, .value).type); -+ const scaleValue = color.ScaleValue(@FieldType(T, "value")); - return .{ .value = scaleValue(toF32(gray.value) * toF32(gray.alpha)) }; - } - - fn grayscaleAlphaToGrayscaleAlpha(comptime T: type, gray: anytype) T { -- const scaleValue = color.ScaleValue(std.meta.fieldInfo(T, .value).type); -- const scaleAlpha = color.ScaleValue(std.meta.fieldInfo(T, .alpha).type); -+ const scaleValue = color.ScaleValue(@FieldType(T, "value")); -+ const scaleAlpha = color.ScaleValue(@FieldType(T, "alpha")); - return .{ - .value = scaleValue(gray.value), - .alpha = scaleAlpha(gray.alpha), -@@ -1101,7 +1094,7 @@ - var destination_pixels = @field(destination, getFieldNameFromPixelFormat(destination_format)); - const DestinationType = @TypeOf(destination_pixels[0]); - -- const scaleValue = color.ScaleValue(std.meta.fieldInfo(DestinationType, .value).type); -+ const scaleValue = color.ScaleValue(@FieldType(DestinationType, "value")); - - for (0..source_rgb.len) |index| { - const source_float4 = source_rgb[index].to.float4(); -@@ -1125,8 +1118,8 @@ - var destination_pixels = @field(destination, getFieldNameFromPixelFormat(destination_format)); - const DestinationType = @TypeOf(destination_pixels[0]); - -- const scaleValue = color.ScaleValue(std.meta.fieldInfo(DestinationType, .value).type); -- const scaleAlpha = color.ScaleValue(std.meta.fieldInfo(DestinationType, .alpha).type); -+ const scaleValue = color.ScaleValue(@FieldType(DestinationType, "value")); -+ const scaleAlpha = color.ScaleValue(@FieldType(DestinationType, "alpha")); - - for (0..source_rgb.len) |index| { - const source_float4 = source_rgb[index].to.float4(); -@@ -1482,7 +1475,7 @@ - var destination_pixels = @field(destination, getFieldNameFromPixelFormat(destination_format)); - const DestinationType = @TypeOf(destination_pixels[0]); - -- const scaleValue = color.ScaleValue(std.meta.fieldInfo(DestinationType, .value).type); -+ const scaleValue = color.ScaleValue(@FieldType(DestinationType, "value")); - - for (0..source_pixels.len) |index| { - const source_float4 = source_pixels[index].to.float4(); -@@ -1503,8 +1496,8 @@ - var destination_pixels = @field(destination, getFieldNameFromPixelFormat(destination_format)); - const DestinationType = @TypeOf(destination_pixels[0]); - -- const scaleValue = color.ScaleValue(std.meta.fieldInfo(DestinationType, .value).type); -- const scaleAlpha = color.ScaleValue(std.meta.fieldInfo(DestinationType, .alpha).type); -+ const scaleValue = color.ScaleValue(@FieldType(DestinationType, "value")); -+ const scaleAlpha = color.ScaleValue(@FieldType(DestinationType, "alpha")); - - for (0..source_pixels.len) |index| { - const source_float4 = source_pixels[index].to.float4(); -diff --git a/src/color.zig b/src/color.zig ---- a/src/color.zig -+++ b/src/color.zig -@@ -830,7 +830,7 @@ - return switch (format) { - .invalid => { - return .{ -- .invalid = void{}, -+ .invalid = {}, - }; - }, - .indexed1 => { -diff --git a/src/compressions/deflate/Lookup.zig b/src/compressions/deflate/Lookup.zig ---- a/src/compressions/deflate/Lookup.zig -+++ b/src/compressions/deflate/Lookup.zig -@@ -13,9 +13,9 @@ - const chain_len = 2 * consts.history.len; - - // Maps hash => first position --head: [consts.lookup.len]u16 = [_]u16{0} ** consts.lookup.len, -+head: [consts.lookup.len]u16 = @splat(0), - // Maps position => previous positions for the same hash value --chain: [chain_len]u16 = [_]u16{0} ** (chain_len), -+chain: [chain_len]u16 = @splat(0), - - // Calculates hash of the 4 bytes from data. - // Inserts `pos` position of that hash in the lookup tables. -diff --git a/src/compressions/deflate/huffman_encoder.zig b/src/compressions/deflate/huffman_encoder.zig ---- a/src/compressions/deflate/huffman_encoder.zig -+++ b/src/compressions/deflate/huffman_encoder.zig -@@ -338,7 +338,7 @@ - } - - pub fn huffmanDistanceEncoder() DistanceEncoder { -- var distance_freq = [1]u16{0} ** consts.distance_code_count; -+ var distance_freq = @as([consts.distance_code_count]u16, @splat(0)); - distance_freq[0] = 1; - // huff_distance is a static distance encoder used for huffman only encoding. - // It can be reused since we will not be encoding distance values. -diff --git a/src/formats/jpeg/writer.zig b/src/formats/jpeg/writer.zig ---- a/src/formats/jpeg/writer.zig -+++ b/src/formats/jpeg/writer.zig -@@ -20,7 +20,7 @@ - luminance, - chrominance, - }; --const quant_index_size = @typeInfo(QuantIndex).@"enum".fields.len; -+const quant_index_size = @typeInfo(QuantIndex).@"enum".field_names.len; - - const n_huff_index = 4; - const HuffmanIndex = enum(usize) { -@@ -156,7 +156,7 @@ - - // Initialize Huffman LUT from Huffman specification - fn initHuffmanLUT(spec: HuffmanSpec) [256]u32 { -- var lut: [256]u32 = [_]u32{0} ** 256; -+ var lut: [256]u32 = @splat(0); - - var code: u32 = 0; - var value_index: usize = 0; -diff --git a/src/formats/pam.zig b/src/formats/pam.zig ---- a/src/formats/pam.zig -+++ b/src/formats/pam.zig -@@ -382,7 +382,7 @@ - - if (src_maxval == dst_maxval) return val; - -- const W = std.meta.Int(.unsigned, @bitSizeOf(T) * 2); -+ const W = @Int(.unsigned, @bitSizeOf(T) * 2); - return @intCast(@min(std.math.maxInt(T), @as(W, dst_maxval) * @as(W, val) / @as(W, src_maxval))); - } - -diff --git a/src/formats/tga.zig b/src/formats/tga.zig ---- a/src/formats/tga.zig -+++ b/src/formats/tga.zig -@@ -374,7 +374,7 @@ - const VectorType = @Vector(VectorLength, IntType); - const BytesPerPixels = (@typeInfo(IntType).int.bits + 7) / 8; - const IndexStep = VectorLength * BytesPerPixels; -- const MaskType = std.meta.Int(.unsigned, VectorLength); -+ const MaskType = @Int(.unsigned, VectorLength); - - comptime { - if (!std.math.isPowerOfTwo(@typeInfo(IntType).int.bits)) { -@@ -466,7 +466,7 @@ - if (std.mem.eql(u8, std.mem.asBytes(&rle_value), std.mem.asBytes(&value))) { - self.length += 1; - } else { -- try RunLengthEncoderCommon.flush(IntType, writer, @as(IntType, @bitCast(rle_value)), self.length); -+ try RunLengthEncoderCommon.flush(IntType, writer, std.mem.readInt(IntType, std.mem.asBytes(&rle_value), .little), self.length); - - self.length = 1; - self.rle_value = value; -@@ -480,7 +480,7 @@ - } - - if (self.rle_value) |rle_value| { -- try RunLengthEncoderCommon.flush(IntType, writer, @as(IntType, @bitCast(rle_value)), self.length); -+ try RunLengthEncoderCommon.flush(IntType, writer, std.mem.readInt(IntType, std.mem.asBytes(&rle_value), .little), self.length); - } - } - }; -@@ -1242,7 +1242,7 @@ - if (self.header.image_type.run_length) { - // The TGA spec recommend that the RLE compression should be done on scanline per scanline basis - inline for (1..(4 + 1)) |bpp| { -- const IntType = std.meta.Int(.unsigned, bpp * 8); -+ const IntType = @Int(.unsigned, bpp * 8); - - if (bytes_per_pixel == bpp) { - if (comptime std.math.isPowerOfTwo(bpp)) { -diff --git a/src/io.zig b/src/io.zig ---- a/src/io.zig -+++ b/src/io.zig -@@ -193,7 +193,7 @@ - } - - fn initBits(comptime T: type, out: anytype, num: u16) Bits(T) { -- const UT = std.meta.Int(.unsigned, @bitSizeOf(T)); -+ const UT = @Int(.unsigned, @bitSizeOf(T)); - return .{ - @bitCast(@as(UT, @intCast(out))), - num, -@@ -222,7 +222,7 @@ - /// containing them in the least significant end, and the number of bits successfully - /// read. Reaching the end of the stream is not an error. - pub fn readBitsTuple(self: *@This(), comptime T: type, num: u16) !Bits(T) { -- const UT = std.meta.Int(.unsigned, @bitSizeOf(T)); -+ const UT = @Int(.unsigned, @bitSizeOf(T)); - const U = if (@bitSizeOf(T) < 8) u8 else UT; //it is a pain to work with return result, - .INTR => continue, - else => return DirectIoError.UnexpectedFcntlResult, - } - } -} - -fn setStatusFlags(file: std.Io.File, flags: c_int) DirectIoError!void { - while (true) { - switch (std.posix.errno(std.posix.system.fcntl(file.handle, std.posix.F.SETFL, flags))) { - .SUCCESS => return, - .INTR => continue, - else => return DirectIoError.UnexpectedFcntlResult, - } - } -} - fn useDirectIO(file: std.Io.File) DirectIoError!bool { if (comptime canUseDirectIO()) { - const flags = try getStatusFlags(file); + const flags = std.posix.system.fcntl(file.handle, std.posix.F.GETFL); const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); return (flags & direct_flag) != 0; } else { @@ -54,11 +33,12 @@ fn useDirectIO(file: std.Io.File) DirectIoError!bool { fn switchToBufferedIO(file: std.fs.File) DirectIoError!void { if (comptime canUseDirectIO()) { - const flags = try getStatusFlags(file); + const flags = std.posix.system.fcntl(file.handle, std.posix.F.GETFL); const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); if ((flags & direct_flag) == 0) return; - try setStatusFlags(file, flags & ~direct_flag); + const result = try std.posix.fcntl(file.handle, std.posix.F.SETFL, flags & ~@as(c_uint, @bitCast(@as(u32, @intCast(direct_flag))))); + if (result != 0) return DirectIoError.UnexpectedFcntlResult; } else { return DirectIoError.UnsupportedPlatform; } @@ -66,9 +46,11 @@ fn switchToBufferedIO(file: std.fs.File) DirectIoError!void { fn switchToDirectIO(file: std.Io.File) DirectIoError!void { if (builtin.os.tag == .linux and canUseDirectIO()) { - const flags = try getStatusFlags(file); + const flags = std.posix.system.fcntl(file.handle, std.posix.F.GETFL); const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); - try setStatusFlags(file, flags | direct_flag); + + const result = std.posix.system.fcntl(file.handle, std.posix.F.SETFL, flags | direct_flag); + if (result != 0) return DirectIoError.UnexpectedFcntlResult; } else { return DirectIoError.UnsupportedPlatform; } @@ -129,7 +111,6 @@ pub const File = struct { .dirStat = dirStat, .dirStatFile = dirStatFile, .dirAccess = dirAccess, - .dirCreateFile = dirCreateFile, .dirOpenFile = dirOpenFile, .dirClose = dirClose, .dirRead = dirRead, @@ -138,9 +119,6 @@ pub const File = struct { .fileStat = fileStat, .fileLength = fileLength, .fileClose = fileClose, - .fileWritePositional = fileWritePositional, - .fileWriteFileStreaming = fileWriteFileStreaming, - .fileWriteFilePositional = fileWriteFilePositional, .fileReadPositional = fileReadPositional, .fileSeekBy = fileSeekBy, .fileSeekTo = fileSeekTo, @@ -173,7 +151,7 @@ pub const File = struct { return self.handles.at(@intCast(file.handle)); } - fn canOpenWithDirectIO(self: *File, file: std.Io.File, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!bool { + fn canOpenWithDirectIO(self: *File, file: std.Io.File, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!bool { const file_stat = self.base.inner.vtable.fileStat(self.base.inner.userdata, file) catch |err| switch (err) { else => return std.Io.File.OpenError.Unexpected, }; @@ -223,18 +201,7 @@ pub const File = struct { }, }); }, - .file_write_streaming => |o| { - const handle = self.getFileHandle(o.file); - return self.base.inner.vtable.operate(self.base.inner.userdata, .{ - .file_write_streaming = .{ - .file = handle.innerFile(), - .header = o.header, - .data = o.data, - .splat = o.splat, - }, - }); - }, - .device_io_control, .net_receive, .net_read, .net_send, .net_write => { + .file_write_streaming, .device_io_control, .net_receive => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -260,21 +227,7 @@ pub const File = struct { return self.base.inner.vtable.dirAccess(self.base.inner.userdata, dir, sub_path, options); } - fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.CreateFileOptions) std.Io.File.OpenError!std.Io.File { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const inner_file = try self.base.inner.vtable.dirCreateFile(self.base.inner.userdata, dir, sub_path, flags); - errdefer inner_file.close(self.base.inner); - - const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; - handle.* = .{ - .inner_handle = inner_file.handle, - .inner_flags = inner_file.flags, - .direct_io_handle = null, - }; - return .{ .handle = @intCast(idx), .flags = inner_file.flags }; - } - - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); const inner_file = try self.base.inner.vtable.dirOpenFile(self.base.inner.userdata, dir, sub_path, flags); errdefer inner_file.close(self.base.inner); @@ -358,52 +311,6 @@ pub const File = struct { } } - fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) std.Io.File.WritePositionalError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - return self.base.inner.vtable.fileWritePositional(self.base.inner.userdata, handle.innerFile(), header, data, splat, offset); - } - - fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const own_io = self.io(); - if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; - - const destination = self.getFileHandle(file); - const source = self.getFileHandle(reader.file); - - const original_source_io = reader.io; - const original_source_file = reader.file; - reader.io = self.base.inner; - reader.file = source.innerFile(); - defer { - reader.io = original_source_io; - reader.file = original_source_file; - } - - return self.base.inner.vtable.fileWriteFileStreaming(self.base.inner.userdata, destination.innerFile(), header, reader, limit); - } - - fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const own_io = self.io(); - if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; - - const destination = self.getFileHandle(file); - const source = self.getFileHandle(reader.file); - - const original_source_io = reader.io; - const original_source_file = reader.file; - reader.io = self.base.inner; - reader.file = source.innerFile(); - defer { - reader.io = original_source_io; - reader.file = original_source_file; - } - - return self.base.inner.vtable.fileWriteFilePositional(self.base.inner.userdata, destination.innerFile(), header, reader, limit, offset); - } - fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); const inner_file = self.innerFile(file, data, offset); diff --git a/vfs/gcs.zig b/vfs/gcs.zig index acc7f634..649e9958 100644 --- a/vfs/gcs.zig +++ b/vfs/gcs.zig @@ -652,7 +652,7 @@ pub const GCS = struct { handle.pos += @intCast(total); return .{ .file_read_streaming = total }; }, - .file_write_streaming, .device_io_control, .net_receive, .net_read, .net_send, .net_write => { + .file_write_streaming, .device_io_control, .net_receive => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -709,7 +709,7 @@ pub const GCS = struct { fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { error.FileNotFound => return std.Io.File.OpenError.FileNotFound, @@ -875,7 +875,7 @@ pub const GCS = struct { return try std.fmt.bufPrint(buf, "{d:0>4}{d:0>2}{d:0>2}T{d:0>2}{d:0>2}{d:0>2}Z", .{ year_day.year, - @backingInt(month_day.month), + @intFromEnum(month_day.month), month_day.day_index + 1, day_seconds.getHoursIntoDay(), day_seconds.getMinutesIntoHour(), diff --git a/vfs/hf.zig b/vfs/hf.zig index 8770fe18..2a1e9249 100644 --- a/vfs/hf.zig +++ b/vfs/hf.zig @@ -652,7 +652,7 @@ pub const HF = struct { handle.pos += @intCast(total); return .{ .file_read_streaming = total }; }, - .file_write_streaming, .device_io_control, .net_receive, .net_read, .net_send, .net_write => { + .file_write_streaming, .device_io_control, .net_receive => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -732,7 +732,7 @@ pub const HF = struct { if (node == null) return std.Io.Dir.AccessError.FileNotFound; } - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); var path_buffer: [8 * 1024]u8 = undefined; diff --git a/vfs/http.zig b/vfs/http.zig index 9054983e..ed8c5ee4 100644 --- a/vfs/http.zig +++ b/vfs/http.zig @@ -156,7 +156,7 @@ pub const HTTP = struct { handle.pos += @intCast(total); return .{ .file_read_streaming = total }; }, - .file_write_streaming, .device_io_control, .net_receive, .net_read, .net_send, .net_write => { + .file_write_streaming, .device_io_control, .net_receive => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -210,7 +210,7 @@ pub const HTTP = struct { fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); const size = self.fetchSize(dir, sub_path) catch return std.Io.File.OpenError.Unexpected; diff --git a/vfs/s3.zig b/vfs/s3.zig index eb849e21..51adcc7c 100644 --- a/vfs/s3.zig +++ b/vfs/s3.zig @@ -544,7 +544,7 @@ pub const S3 = struct { handle.pos += @intCast(total); return .{ .file_read_streaming = total }; }, - .file_write_streaming, .device_io_control, .net_receive, .net_read, .net_send, .net_write => { + .file_write_streaming, .device_io_control, .net_receive => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -601,7 +601,7 @@ pub const S3 = struct { fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { error.FileNotFound => return std.Io.File.OpenError.FileNotFound, @@ -767,7 +767,7 @@ pub const S3 = struct { return try std.fmt.bufPrint(buf, "{d:0>4}{d:0>2}{d:0>2}T{d:0>2}{d:0>2}{d:0>2}Z", .{ year_day.year, - @backingInt(month_day.month), + @intFromEnum(month_day.month), month_day.day_index + 1, day_seconds.getHoursIntoDay(), day_seconds.getMinutesIntoHour(), diff --git a/vfs/vfs.zig b/vfs/vfs.zig index fe626ff9..b5e5c580 100644 --- a/vfs/vfs.zig +++ b/vfs/vfs.zig @@ -187,16 +187,7 @@ fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable .data = o.data, } }); }, - .file_write_streaming => |o| { - const handle, const backend = self.getFileHandle(o.file); - return backend.vtable.operate(backend.userdata, .{ .file_write_streaming = .{ - .file = .{ .handle = @intCast(handle.handle), .flags = handle.flags }, - .header = o.header, - .data = o.data, - .splat = o.splat, - } }); - }, - .device_io_control, .net_receive, .net_read, .net_send, .net_write => { + .device_io_control, .file_write_streaming, .net_receive => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -244,7 +235,7 @@ pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, o return backend.vtable.dirAccess(backend.userdata, dir_, stripScheme(sub_path), options); } -pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.CreateFileOptions) std.Io.File.OpenError!std.Io.File { +pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.CreateFlags) std.Io.File.OpenError!std.Io.File { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { log.err("Failed to lookup backend for dir create file '{s}' : {any}", .{ sub_path, err }); @@ -252,29 +243,25 @@ pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u }; const file = try backend.vtable.dirCreateFile(backend.userdata, dir_, stripScheme(sub_path), flags); - errdefer file.close(backend); const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; handle.* = .{ .handle = @intCast(file.handle), .backend_idx = backend_idx, - .flags = file.flags, }; - return .{ .handle = @intCast(idx), .flags = file.flags }; + return .{ .handle = @intCast(idx), .flags = .{ .nonblocking = false } }; } -fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { +fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { log.err("Failed to lookup backend for opening file '{s}' : {any}", .{ sub_path, err }); return std.Io.File.OpenError.Unexpected; }; const file = try backend.vtable.dirOpenFile(backend.userdata, dir_, stripScheme(sub_path), flags); - errdefer file.close(backend); const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; handle.* = .{ .handle = @intCast(file.handle), .backend_idx = backend_idx, - .flags = file.flags, }; return .{ .handle = @intCast(idx), .flags = handle.flags }; } @@ -366,42 +353,26 @@ fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const own_io = self.io(); - if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; const dst_handle, const dst_backend = self.getFileHandle(file); - const src_handle, const src_backend = self.getFileHandle(reader.file); - if (src_backend.userdata != dst_backend.userdata or src_backend.vtable != dst_backend.vtable) return error.Unimplemented; + const src_handle, _ = self.getFileHandle(reader.file); - const original_src_io = reader.io; const original_src_handle = reader.file; - reader.io = src_backend; reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; - defer { - reader.io = original_src_io; - reader.file = original_src_handle; - } + defer reader.file = original_src_handle; return dst_backend.vtable.fileWriteFileStreaming(dst_backend.userdata, .{ .handle = @intCast(dst_handle.handle), .flags = dst_handle.flags }, header, reader, limit); } fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const own_io = self.io(); - if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; const dst_handle, const dst_backend = self.getFileHandle(file); - const src_handle, const src_backend = self.getFileHandle(reader.file); - if (src_backend.userdata != dst_backend.userdata or src_backend.vtable != dst_backend.vtable) return error.Unimplemented; + const src_handle, _ = self.getFileHandle(reader.file); - const original_src_io = reader.io; const original_src_handle = reader.file; - reader.io = src_backend; reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; - defer { - reader.io = original_src_io; - reader.file = original_src_handle; - } + defer reader.file = original_src_handle; return dst_backend.vtable.fileWriteFilePositional(dst_backend.userdata, .{ .handle = @intCast(dst_handle.handle), .flags = dst_handle.flags }, header, reader, limit, offset); } @@ -436,48 +407,3 @@ fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std. return try backend.vtable.fileRealPath(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, out_buffer); } } - -test "copy a local file through the VFS" { - const allocator = std.testing.allocator; - const process_io = std.testing.io; - - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - - const source_data: [8192]u8 = @splat('x'); - try tmp.dir.writeFile(process_io, .{ - .sub_path = "source", - .data = &source_data, - }); - - var path_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path_len = try tmp.dir.realPath(process_io, &path_buffer); - const source_uri = try std.fmt.allocPrint(allocator, "file://{s}/source", .{path_buffer[0..path_len]}); - defer allocator.free(source_uri); - const destination_uri = try std.fmt.allocPrint(allocator, "file://{s}/destination", .{path_buffer[0..path_len]}); - defer allocator.free(destination_uri); - - var file_backend: File = .init(allocator, process_io, .{}); - defer file_backend.deinit(); - var vfs: VFS = try .init(allocator, process_io); - defer vfs.deinit(); - try vfs.register("file", file_backend.io()); - - const vfs_io = vfs.io(); - { - var source = try std.Io.Dir.openFile(.cwd(), vfs_io, source_uri, .{}); - defer source.close(vfs_io); - var destination = try std.Io.Dir.createFile(.cwd(), vfs_io, destination_uri, .{}); - defer destination.close(vfs_io); - - var reader: std.Io.File.Reader = .initStreaming(source, vfs_io, &.{}); - var copy_buffer: [4096]u8 align(4096) = undefined; - var writer = destination.writer(vfs_io, ©_buffer); - try std.testing.expectEqual(source_data.len, try reader.interface.streamRemaining(&writer.interface)); - try writer.interface.flush(); - } - - const copied = try tmp.dir.readFileAlloc(process_io, "destination", allocator, .limited(source_data.len + 1)); - defer allocator.free(copied); - try std.testing.expectEqualSlices(u8, &source_data, copied); -} diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 591e3fcb..b6cd15b7 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -164,7 +164,7 @@ pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, var mlir_known_types: std.enums.EnumArray(DataType, *const mlir.Type) = .initUndefined(); { for (0.., &mlir_known_types.values) |i, *mlir_type| { - mlir_type.* = mlirx.Type.fromDType(mlir_ctx, @fromBackingInt(@intCast(i))); + mlir_type.* = mlirx.Type.fromDType(mlir_ctx, @enumFromInt(i)); } } @@ -316,7 +316,7 @@ pub fn mlirType(self: *const Compiler, dt: DataType) *const mlir.Type { pub fn dtype(self: *const Compiler, mlir_type: *const mlir.Type) DataType { @setRuntimeSafety(false); for (0.., &self.mlir_known_types.values) |i, known_type| { - if (known_type == mlir_type) return @fromBackingInt(@intCast(i)); + if (known_type == mlir_type) return @enumFromInt(i); } std.debug.panic("Can't convert unknown mlir type to dtype: {f}", .{mlir_type}); } diff --git a/zml/Sharding.zig b/zml/Sharding.zig index b8005144..761721fa 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -135,21 +135,21 @@ pub const PartitionSpec = enum(u4) { open = 15, pub fn sharded(mesh_axis: u3) PartitionSpec { - return @fromBackingInt(@intCast(mesh_axis)); + return @enumFromInt(mesh_axis); } /// Extract the mesh axis along which we are sharded. Null if not sharded. pub fn meshAxis(self: PartitionSpec) ?u3 { - const ax = @backingInt(self); - return if (ax < @backingInt(PartitionSpec.replicated)) @intCast(ax) else null; + const ax = @intFromEnum(self); + return if (ax < @intFromEnum(PartitionSpec.replicated)) @intCast(ax) else null; } pub fn isSharded(self: PartitionSpec) bool { - return @backingInt(self) < @backingInt(PartitionSpec.replicated); + return @intFromEnum(self) < @intFromEnum(PartitionSpec.replicated); } pub fn isClosed(self: PartitionSpec) bool { - return @backingInt(self) <= @backingInt(PartitionSpec.replicated); + return @intFromEnum(self) <= @intFromEnum(PartitionSpec.replicated); } test isClosed { @@ -285,13 +285,13 @@ pub const Partitioning = packed struct { } pub fn splat(spec: PartitionSpec) Partitioning { - const vec: Vec = @splat(@backingInt(spec)); + const vec: Vec = @splat(@intFromEnum(spec)); return @bitCast(vec); } pub fn repeat(spec: PartitionSpec, rank_: usize) Partitioning { std.debug.assert(rank_ <= MAX_RANK); - const splatted: Vec = @splat(@backingInt(spec)); + const splatted: Vec = @splat(@intFromEnum(spec)); const mask = std.simd.iota(u4, 8) < @as(Vec, @splat(@truncate(rank_))); return @bitCast(@select(u4, mask, splatted, @as(Vec, @bitCast(out_of_bound)))); } @@ -300,7 +300,7 @@ pub const Partitioning = packed struct { std.debug.assert(ax < MAX_RANK); const pack: u32 = @bitCast(p); const shift: u5 = @intCast(4 * ax); - return @fromBackingInt(@truncate(pack >> shift)); + return @enumFromInt(@as(u4, @truncate(pack >> shift))); } pub fn set(p: Partitioning, ax: usize, spec: PartitionSpec) Partitioning { @@ -308,7 +308,7 @@ pub const Partitioning = packed struct { const pack: u32 = @bitCast(p); const shift: u5 = @intCast(4 * ax); const mask = @as(u32, 0xf) << shift; - return @bitCast((pack & ~mask) | (@as(u32, @backingInt(spec)) << shift)); + return @bitCast((pack & ~mask) | (@as(u32, @intFromEnum(spec)) << shift)); } pub fn rank(p: Partitioning) u8 { @@ -325,7 +325,7 @@ pub const Partitioning = packed struct { const pack: u32 = @bitCast(p); const shift: u5 = @intCast(4 * ax); const lower_mask = (@as(u32, 1) << shift) - 1; - return @bitCast((pack & lower_mask) | ((pack & ~lower_mask) << 4) | (@as(u32, @backingInt(spec)) << shift)); + return @bitCast((pack & lower_mask) | ((pack & ~lower_mask) << 4) | (@as(u32, @intFromEnum(spec)) << shift)); } /// Removes a spec, shifting subsequent slots left and filling the last with unknown. @@ -334,7 +334,7 @@ pub const Partitioning = packed struct { const pack: u32 = @bitCast(p); const shift: u5 = @intCast(4 * ax); const lower_mask = (@as(u32, 1) << shift) - 1; - return @bitCast((pack & lower_mask) | ((pack >> 4) & ~lower_mask) | (@as(u32, @backingInt(PartitionSpec.out_of_bound)) << 28)); + return @bitCast((pack & lower_mask) | ((pack >> 4) & ~lower_mask) | (@as(u32, @intFromEnum(PartitionSpec.out_of_bound)) << 28)); } pub fn toArray(p: Partitioning) [MAX_RANK]PartitionSpec { @@ -344,7 +344,7 @@ pub const Partitioning = packed struct { } pub fn hasUniqueAxes(p: Partitioning) bool { - var used_mesh_axes: std.StaticBitSet(MAX_RANK) = .empty; + var used_mesh_axes: std.StaticBitSet(MAX_RANK) = .initEmpty(); for (0..MAX_RANK) |shape_ax| { const spec = p.get(shape_ax); if (spec.meshAxis()) |mesh_axis| { @@ -383,7 +383,7 @@ pub const Partitioning = packed struct { test "insertion and removal at every slot" { var parts: Partitioning = .out_of_bound; - for (0..MAX_RANK) |ax| parts = parts.set(ax, @fromBackingInt(@intCast(ax))); + for (0..MAX_RANK) |ax| parts = parts.set(ax, @enumFromInt(ax)); const original = parts.toArray(); for (0..MAX_RANK) |ax| { var inserted = original; @@ -414,13 +414,13 @@ pub const Partitioning = packed struct { const T = @TypeOf(partitioning); stdx.debug.assertComptime(stdx.meta.isStruct(T), "parsePartitioning expected a struct of enum literals eg {{ .b = .data, .d = .model }}, got: {any}", .{T}); - inline for (comptime std.meta.fieldNames(T)) |field_name| { - const shape_tag = Shape.toTag(field_name); + inline for (std.meta.fields(T)) |field| { + const shape_tag = Shape.toTag(field); const shape_ax = Shape.axisFromTagMaybe(tags, shape_tag) orelse { - std.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), field_name }); + std.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), field.name }); }; - const value = @field(partitioning, field_name); + const value = @field(partitioning, field.name); const spec: PartitionSpec = if (@TypeOf(value) == PartitionSpec) value else switch (value) { .replicated => .replicated, .out_of_bound => std.debug.panic("value out_of_bound not allowed", .{}), @@ -603,11 +603,11 @@ pub const PhysicalMesh = struct { try writer.writeAll("], depth_by_tag={"); - const field_names = comptime std.meta.fieldNames(PhysicalAxisTag); - inline for (field_names, 0..) |field_name, i| { + const fields = std.meta.fields(PhysicalAxisTag); + inline for (fields, 0..) |field, i| { if (i > 0) try writer.writeAll(", "); - const tag = @field(PhysicalAxisTag, field_name); - try writer.writeAll(field_name); + const tag = @field(PhysicalAxisTag, field.name); + try writer.writeAll(field.name); try writer.writeAll("="); if (self.depth_by_tag.get(tag)) |d| { try writer.print("{d}", .{d}); @@ -1349,9 +1349,9 @@ pub const LogicalMesh = struct { var axes: Axes = .empty; var intents: Intents = .empty; - inline for (comptime std.meta.fieldNames(T)) |field_name| { - const value = @field(axes_, field_name); - axes.appendAssumeCapacity(Shape.toTag(field_name)); + inline for (std.meta.fields(T)) |field| { + const value = @field(axes_, field.name); + axes.appendAssumeCapacity(Shape.toTag(field)); intents.appendAssumeCapacity(intentFromValue(value)); } @@ -1922,15 +1922,15 @@ pub const Strategy = struct { pub fn parseBindings(bindings: anytype) Strategy { const err_msg = "Strategy.parseBindings excepts fields to be PhysicalAxisTag or tuple of PhysicalAxisTag, got {}"; var res: Strategy = .{ .bindings = .empty, .folding = .empty }; - const info = @typeInfo(@TypeOf(bindings)).@"struct"; - if (info.field_names.len == 0) @compileError("Strategy.parseBindings requires at least one binding"); - inline for (info.field_names, info.field_types) |field_name, Field| { - switch (@typeInfo(Field)) { - .enum_literal, .@"enum" => res.addBinding(field_name, @field(bindings, field_name)), + const fields = @typeInfo(@TypeOf(bindings)).@"struct".fields; + if (fields.len == 0) @compileError("Strategy.parseBindings requires at least one binding"); + inline for (fields) |field_info| { + switch (@typeInfo(field_info.type)) { + .enum_literal, .@"enum" => res.addBinding(field_info, @field(bindings, field_info.name)), .@"struct" => |struct_info| { stdx.debug.assertComptime(struct_info.is_tuple, err_msg, .{@TypeOf(bindings)}); - inline for (@field(bindings, field_name)) |axis_tag| { - res.addBinding(field_name, axis_tag); + inline for (@field(bindings, field_info.name)) |axis_tag| { + res.addBinding(field_info, axis_tag); } }, else => stdx.debug.compileError(err_msg, .{@TypeOf(bindings)}), @@ -2042,13 +2042,13 @@ pub const Placement = struct { // Note this shape is mainly use to indicate the dims of the sharded buffer. // We may want to only store that. shape: Shape, - global_shape: if (builtin.mode == .debug) Shape else void, + global_shape: if (builtin.mode == .Debug) Shape else void, axis_plans: stdx.BoundedArray(AxisSplit, Shape.MAX_RANK), pub fn init(shape: Shape) error{IncompatibleSharding}!Placement { var pl: Placement = .{ .shape = shape, // modified below - .global_shape = if (builtin.mode == .debug) shape else {}, + .global_shape = if (builtin.mode == .Debug) shape else {}, .axis_plans = .empty, // set below }; // mesh == null => replicated => no axis plan @@ -2088,7 +2088,7 @@ pub const Placement = struct { } pub fn shardPtr(pl: *const Placement, device: Device.Coords, slice: Slice) [*]const u8 { - if (builtin.mode == .debug) { + if (builtin.mode == .Debug) { // there is a bug in caller code that used a placement for a different shape std.debug.assert(pl.global_shape.eql(slice.shape)); } @@ -2276,10 +2276,10 @@ const ShardingTest = struct { /// Creates a 1D-3D PhysicalMesh from simple dimensions. pub fn physical(self: ShardingTest, dims: anytype, geometry: AxisGeometry) !PhysicalMesh { const info = @typeInfo(@TypeOf(dims)).@"struct"; - const N = info.field_names.len; + const N = info.fields.len; var sizes: [N]usize = undefined; - inline for (info.field_names, sizes[0..]) |field_name, *s| { - s.* = @intCast(@field(dims, field_name)); + inline for (info.fields, sizes[0..]) |field, *s| { + s.* = @intCast(@field(dims, field.name)); } const tags: [3]PhysicalAxisTag = .{ .link_x, .link_y, .link_z }; diff --git a/zml/attention.zig b/zml/attention.zig index cb2b1966..f5399c5e 100644 --- a/zml/attention.zig +++ b/zml/attention.zig @@ -192,8 +192,8 @@ test "FlashAttention metadata initializes with explicit sharding outside compila inline for (.{ Backend.cuda_fa2, Backend.cuda_fa3 }) |backend| { const metadata: Metadata = .init(.fromBackend(backend, 16, 8), &mesh); const cuda_metadata = @field(metadata, @tagName(backend)); - inline for (comptime std.meta.fieldNames(@TypeOf(cuda_metadata))) |field_name| { - const shape = @field(cuda_metadata, field_name).shape(); + inline for (std.meta.fields(@TypeOf(cuda_metadata))) |field| { + const shape = @field(cuda_metadata, field.name).shape(); try std.testing.expectEqual(&mesh, shape._sharding.mesh); if (shape.hasTag(.h)) |axis| { try std.testing.expectEqual(zml.Sharding.PartitionSpec.sharded(0), shape.partition(axis)); diff --git a/zml/attention/cute_kernels/flashmla_sm100.zig b/zml/attention/cute_kernels/flashmla_sm100.zig index 529594c0..ed03b394 100644 --- a/zml/attention/cute_kernels/flashmla_sm100.zig +++ b/zml/attention/cute_kernels/flashmla_sm100.zig @@ -120,7 +120,7 @@ const Bar = enum(i32) { const idx_consumers = 11; fn bar(b: *B, base: V, comptime which: Bar, buf: anytype) V { - return base.add(smem.bars + 8 * @backingInt(which)).add(b.lift(buf).mul(8)); + return base.add(smem.bars + 8 * @intFromEnum(which)).add(b.lift(buf).mul(8)); } // TMEM columns: O^T of token parity ob in 64 * ob (4 chunks of 128 dims x 16 @@ -252,8 +252,8 @@ fn parity(j: V) V { return j.shrLogical(1).bitAnd(1); } -fn tupleOf(comptime n: usize, arr: [n]V) @Tuple(&@as([n]type, @splat(V))) { - var t: @Tuple(&@as([n]type, @splat(V))) = undefined; +fn tupleOf(comptime n: usize, arr: [n]V) std.meta.Tuple(&([_]type{V} ** n)) { + var t: std.meta.Tuple(&([_]type{V} ** n)) = undefined; inline for (0..n) |i| t[i] = arr[i]; return t; } @@ -944,10 +944,7 @@ fn softmaxRole(c0: Ctx) void { // then skips the block max entirely (m stays, alpha = 1). const f32t = cute.DType.f32.toMlir(b.ctx); const i32t = cute.DType.i32.toMlir(b.ctx); - var update_types: @Tuple(&@as([2 * H + 1]type, @splat(@TypeOf(f32t)))) = undefined; - inline for (0..2 * H) |h| update_types[h] = f32t; - update_types[2 * H] = i32t; - var upd = b.openIfElse(b.barrierReduceOr(1, 128, exceed.ne(0)), update_types); + var upd = b.openIfElse(b.barrierReduceOr(1, 128, exceed.ne(0)), .{f32t} ** (2 * H) ++ .{i32t}); { // Block max per head: warp redux, then across the 4 warps through shared memory. const red = c.s.add(b.select(buf.eq(0), b.cst(.i32, smem.red[0]), b.cst(.i32, smem.red[1]))); @@ -1327,8 +1324,8 @@ fn instrDescBf16(comptime m: u32, comptime n: u32, comptime a_major: Major, comp return (1 << 4) | // c_format = F32 (1 << 7) | // a_format = BF16 (1 << 10) | // b_format = BF16 - (@as(u32, @backingInt(a_major)) << 15) | - (@as(u32, @backingInt(b_major)) << 16) | + (@as(u32, @intFromEnum(a_major)) << 15) | + (@as(u32, @intFromEnum(b_major)) << 16) | ((n >> 3) << 17) | ((m >> 4) << 24); } diff --git a/zml/dtype.zig b/zml/dtype.zig index f8d24089..c56fdd5b 100644 --- a/zml/dtype.zig +++ b/zml/dtype.zig @@ -1,7 +1,5 @@ const std = @import("std"); -const stdx = @import("stdx"); - const floats = @import("floats.zig"); const C64 = std.math.Complex(f32); @@ -141,9 +139,8 @@ pub const DataType = enum(u8) { pub fn toPackedZigType(comptime dtype: DataType) type { return switch (dtype) { .bool => bool, - .i2, .i4, .u2, .u4 => |dt| stdx.Packed(@FieldType(Value, @tagName(dt))), + .i2, .i4, .u2, .u4 => |dt| @Vector(8 / dt.bitSizeOf(), @FieldType(Value, @tagName(dtype))), .f4e2m1 => floats.Float4E2M1.Packed, - inline .c64, .c128 => |dt| @FieldType(Value, @tagName(dt)), else => { const T = @FieldType(Value, @tagName(dtype)); if (@bitSizeOf(T) < 8) @compileLog("Forgot type !", T); @@ -168,8 +165,6 @@ pub const DataType = enum(u8) { pub fn bitSizeOf(self: DataType) u16 { return switch (self) { .bool => 8, // In PJRT/stablehlo bool always occupy a full byte - .c64 => 64, - .c128 => 128, inline else => |tag| @bitSizeOf(tag.toZigType()), }; } diff --git a/zml/exe.zig b/zml/exe.zig index 8b60dcb9..1db04837 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -396,7 +396,7 @@ pub fn FnExe(comptime function_: anytype) type { .@"fn" => |info| info, else => @compileError("FnExe expects a function, got " ++ @typeName(@TypeOf(function_))), }; - if (function_info.attrs.varargs or function_info.param_types.len != 1) { + if (function_info.is_var_args or function_info.params.len != 1) { @compileError("FnExe function must accept exactly one input struct"); } @@ -446,7 +446,7 @@ pub fn FnExe(comptime function_: anytype) type { const RunnerSelf = @This(); pub const BakedInput = structFieldRange(Input, 0, count); - pub const NonBakedInput = structFieldRange(Input, count, @typeInfo(Input).@"struct".field_names.len); + pub const NonBakedInput = structFieldRange(Input, count, @typeInfo(Input).@"struct".fields.len); pub fn init(exe: *const Self, allocator: std.mem.Allocator, baked: BakedInput) !RunnerSelf { var arguments = try exe.raw.args(allocator); @@ -488,30 +488,42 @@ fn countBackedFields(comptime Input: type, comptime baked_fields: anytype) usize @compileError("FnExe baked fields must be a tuple of enum literals"); } - const input_info = @typeInfo(Input).@"struct"; - if (baked_info.field_names.len > input_info.field_names.len) { + const input_fields = @typeInfo(Input).@"struct".fields; + if (baked_info.fields.len > input_fields.len) { @compileError("FnExe baked fields must be a prefix of its bufferized input fields"); } - inline for (baked_info.field_names, 0..) |tuple_field_name, i| { - const baked_field = @field(baked_fields, tuple_field_name); + inline for (baked_info.fields, 0..) |tuple_field, i| { + const baked_field = @field(baked_fields, tuple_field.name); if (@TypeOf(baked_field) != @EnumLiteral()) { @compileError("FnExe baked fields must be enum literals"); } - if (!std.mem.eql(u8, @tagName(baked_field), input_info.field_names[i])) { + if (!std.mem.eql(u8, @tagName(baked_field), input_fields[i].name)) { @compileError("FnExe baked fields must be an ordered prefix of its bufferized input fields"); } } - return baked_info.field_names.len; + return baked_info.fields.len; } fn structFieldRange(comptime Struct: type, comptime start: usize, comptime end: usize) type { - const info = @typeInfo(Struct).@"struct"; - return @Struct(.auto, null, info.field_names[start..end], info.field_types[start..end], info.field_attrs[start..end]); + const fields = @typeInfo(Struct).@"struct".fields[start..end]; + var field_names: [fields.len][]const u8 = undefined; + var field_types: [fields.len]type = undefined; + var field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; + for (&field_names, &field_types, &field_attrs, fields) |*name, *T, *attrs, field| { + name.* = field.name; + T.* = field.type; + attrs.* = .{ + .@"comptime" = field.is_comptime, + .@"align" = field.alignment, + .default_value_ptr = field.default_value_ptr, + }; + } + return @Struct(.auto, null, &field_names, &field_types, &field_attrs); } fn typedFunctionInput(comptime function_: anytype) type { - const Input = @typeInfo(@TypeOf(function_)).@"fn".param_types[0] orelse + const Input = @typeInfo(@TypeOf(function_)).@"fn".params[0].type orelse @compileError("FnExe function must have a concrete input type"); validateFnExeStruct("input", Input); return Input; @@ -535,13 +547,16 @@ fn validateFnExeStruct(comptime role: []const u8, comptime T: type) void { fn typedOutputDestinations(comptime BufferizedOutput: type) type { const output_info = @typeInfo(BufferizedOutput).@"struct"; - var field_types: [output_info.field_names.len]type = undefined; - var field_attrs: [output_info.field_names.len]std.builtin.Type.Struct.FieldAttributes = undefined; - for (&field_types, &field_attrs, output_info.field_types) |*T, *attrs, Field| { - T.* = *Field; - attrs.* = .{ .@"align" = @alignOf(*Field) }; + const fields = output_info.fields; + var field_names: [fields.len][]const u8 = undefined; + var field_types: [fields.len]type = undefined; + var field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; + for (&field_names, &field_types, &field_attrs, fields) |*name, *T, *attrs, field| { + name.* = field.name; + T.* = *field.type; + attrs.* = .{ .@"align" = @alignOf(*field.type) }; } - return @Struct(.auto, null, output_info.field_names, &field_types, &field_attrs); + return @Struct(.auto, null, &field_names, &field_types, &field_attrs); } test "FnExe derives baked and runtime calls from a function" { diff --git a/zml/floats.zig b/zml/floats.zig index 689c22a8..7bf08824 100644 --- a/zml/floats.zig +++ b/zml/floats.zig @@ -8,8 +8,8 @@ fn FloatConversionHelpers(Float: type) type { @compileError(err_msg); } comptime { - for (info.@"struct".field_names, &.{ "mantissa", "exponent", "sign" }) |field_name, expected_name| { - if (!std.mem.eql(u8, field_name, expected_name)) + for (info.@"struct".fields, &.{ "mantissa", "exponent", "sign" }) |field, expected_name| { + if (!std.mem.eql(u8, field.name, expected_name)) @compileError(err_msg); } } @@ -20,7 +20,7 @@ fn FloatConversionHelpers(Float: type) type { const mantissa_bits: u8 = @typeInfo(@FieldType(Float, "mantissa")).int.bits; const exponent_bits: u8 = @typeInfo(@FieldType(Float, "exponent")).int.bits; const f32_mantissa_bits: u8 = @typeInfo(@FieldType(Float32, "mantissa")).int.bits; - const exp_bias: i16 = std.math.maxInt(@Int(.unsigned, exponent_bits - 1)); + const exp_bias: i16 = std.math.maxInt(std.meta.Int(.unsigned, exponent_bits - 1)); const exp_off: u8 = FloatConversionHelpers(Float32).exp_bias - exp_bias; /// Lossy conversion from f32, similar to @floatCast @@ -106,8 +106,8 @@ fn FloatHelpers(Float: type) type { @compileError(err_msg); } comptime { - for (info.@"struct".field_names, &.{ "mantissa", "exponent", "sign" }) |field_name, expected_name| { - if (!std.mem.eql(u8, field_name, expected_name)) + for (info.@"struct".fields, &.{ "mantissa", "exponent", "sign" }) |field, expected_name| { + if (!std.mem.eql(u8, field.name, expected_name)) @compileError(err_msg); } } @@ -517,7 +517,7 @@ pub const Float4E2M1 = packed struct(u4) { pub fn formatNumber(xy: Packed, w: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { try xy.x.formatNumber(w, n); - try w.writeByte(','); + w.writeByte(','); try xy.y.formatNumber(w, n); } }; @@ -544,7 +544,7 @@ pub fn isInf(x: anytype) bool { } if (!@hasDecl(Float, "inf")) return false; - const FBits = @Int(.unsigned, @bitSizeOf(Float)); + const FBits = std.meta.Int(.unsigned, @bitSizeOf(Float)); const remove_sign = ~@as(FBits, 0) >> 1; return @as(FBits, @bitCast(x)) & remove_sign == @as(FBits, @bitCast(Float.inf)); } diff --git a/zml/kernel.zig b/zml/kernel.zig index b934c7db..2d8945e4 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -50,10 +50,10 @@ fn resolveOutputOperandAliases( var out: stdx.BoundedArray(dialects.stablehlo.CustomCallOpts.OutputOperandAlias, dialects.stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + inline for (@typeInfo(@TypeOf(a)).@"struct".fields, 0..) |field, output_index| { + if (@field(a, field.name)) |operand| out.appendAssumeCapacity(.{ .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + .operand_index = @as(i64, @intCast(@intFromEnum(operand))) + operand_offset, }); } } @@ -832,7 +832,7 @@ pub const fly = struct { /// `zeroed_args` index of an output. pub fn zeroedOutput(comptime f: std.meta.FieldEnum(Outputs)) i32 { - return @intCast(spec.inputs.len + @backingInt(f)); + return @intCast(spec.inputs.len + @intFromEnum(f)); } /// `shapes` is inputs, then outputs. diff --git a/zml/mem.zig b/zml/mem.zig index 87a824ef..f471a83a 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -341,8 +341,8 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu const type_info = @typeInfo(ModelBufferized); switch (type_info) { .@"struct" => |struct_type_info| { - inline for (struct_type_info.field_names) |field_name| { - try bufferizeInner(allocator, @field(model, field_name), &@field(bufferized_, field_name)); + inline for (struct_type_info.fields) |field| { + try bufferizeInner(allocator, @field(model, field.name), &@field(bufferized_, field.name)); } }, .@"union" => { @@ -364,7 +364,7 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu .pointer => |p| { switch (p.size) { .slice => { - bufferized_.* = try allocator.alignedAlloc(p.child, .fromByteUnits(p.attrs.@"align" orelse @alignOf(p.child)), model.len); + bufferized_.* = try allocator.alignedAlloc(p.child, .fromByteUnits(p.alignment orelse @alignOf(p.child)), model.len); for (model, bufferized_.*) |src, *dst| { try bufferizeInner(allocator, src, dst); } diff --git a/zml/meta.zig b/zml/meta.zig index c9a43f25..08738e11 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -20,30 +20,27 @@ pub fn MapType(From: type, To: type) type { return switch (@typeInfo(T)) { .@"struct" => |struct_infos| { - const field_names = struct_infos.field_names; + const fields = struct_infos.fields; var same: bool = true; - var struct_field_names: [field_names.len][]const u8 = undefined; - var struct_field_types: [field_names.len]type = undefined; - var struct_field_attrs: [field_names.len]std.builtin.Type.Struct.FieldAttributes = undefined; - for ( - struct_field_names[0..], - struct_field_types[0..], - struct_field_attrs[0..], - struct_infos.field_names, - struct_infos.field_types, - struct_infos.field_attrs, - ) |*field_name, *field_type, *field_attr, source_name, SourceField, source_attrs| { - if (!source_attrs.@"comptime") { - const R = map(SourceField); - if (R == SourceField) { - field_name.* = source_name; - field_type.* = SourceField; - field_attr.* = source_attrs; + var struct_field_names: [fields.len][]const u8 = undefined; + var struct_field_types: [fields.len]type = undefined; + var struct_field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; + for (struct_field_names[0..], struct_field_types[0..], struct_field_attrs[0..], fields) |*field_name, *field_type, *field_attr, field| { + if (!field.is_comptime) { + const R = map(field.type); + if (R == field.type) { + field_name.* = field.name; + field_type.* = field.type; + field_attr.* = .{ + .@"comptime" = field.is_comptime, + .@"align" = field.alignment, + .default_value_ptr = field.default_value_ptr, + }; } else { - field_name.* = source_name; + field_name.* = field.name; field_type.* = R; field_attr.* = .{ - .@"comptime" = source_attrs.@"comptime", + .@"comptime" = field.is_comptime, .@"align" = @alignOf(R), }; same = false; @@ -55,35 +52,34 @@ pub fn MapType(From: type, To: type) type { } } } else { - field_name.* = source_name; - field_type.* = SourceField; - field_attr.* = source_attrs; + field_name.* = field.name; + field_type.* = field.type; + field_attr.* = .{ + .@"comptime" = field.is_comptime, + .@"align" = field.alignment, + .default_value_ptr = field.default_value_ptr, + }; } } if (same) return T; return if (struct_infos.is_tuple) @Tuple(&struct_field_types) else @Struct(.auto, null, &struct_field_names, &struct_field_types, &struct_field_attrs); }, .@"union" => |union_info| { - const field_names = union_info.field_names; + const fields = union_info.fields; var same: bool = true; - var union_field_names: [field_names.len][]const u8 = undefined; - var union_field_types: [field_names.len]type = undefined; - var union_field_attrs: [field_names.len]std.builtin.Type.Union.FieldAttributes = undefined; - for ( - union_field_names[0..], - union_field_types[0..], - union_field_attrs[0..], - union_info.field_names, - union_info.field_types, - union_info.field_attrs, - ) |*field_name, *field_type, *field_attr, source_name, SourceField, source_attrs| { - const R = map(SourceField); - if (R == SourceField) { - field_name.* = source_name; - field_type.* = SourceField; - field_attr.* = source_attrs; + var union_field_names: [fields.len][]const u8 = undefined; + var union_field_types: [fields.len]type = undefined; + var union_field_attrs: [fields.len]std.builtin.Type.UnionField.Attributes = undefined; + for (union_field_names[0..], union_field_types[0..], union_field_attrs[0..], fields) |*field_name, *field_type, *field_attr, field| { + const R = map(field.type); + if (R == field.type) { + field_name.* = field.name; + field_type.* = field.type; + field_attr.* = .{ + .@"align" = field.alignment, + }; } else { - field_name.* = source_name; + field_name.* = field.name; field_type.* = R; field_attr.* = .{ .@"align" = @alignOf(R), @@ -96,11 +92,11 @@ pub fn MapType(From: type, To: type) type { }, .array => |arr_info| [arr_info.len]map(arr_info.child), .pointer => |ptr_info| switch (ptr_info.size) { - .slice => if (ptr_info.attrs.@"const") + .slice => if (ptr_info.is_const) []const map(ptr_info.child) else []map(ptr_info.child), - .one => if (ptr_info.attrs.@"const") + .one => if (ptr_info.is_const) *const map(ptr_info.child) else *map(ptr_info.child), @@ -183,26 +179,26 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam if (@sizeOf(ToStruct) == 0) return; switch (type_info_to) { - .@"struct" => |info| inline for (info.field_names, info.field_types, info.field_attrs) |field_name, Field, field_attrs| { - if (field_attrs.@"comptime" or @sizeOf(Field) == 0) continue; - const field_type_info = @typeInfo(Field); + .@"struct" => |info| inline for (info.fields) |field| { + if (field.is_comptime or @sizeOf(field.type) == 0) continue; + const field_type_info = @typeInfo(field.type); // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. switch (field_type_info) { // .pointer => try convertType(From, To, allocator, @field(from, field.name), @field(to, field.name), Ctx, ctx, cb), - .array, .optional, .@"union", .@"struct", .pointer => if (@hasField(FromStruct, field_name)) { + .array, .optional, .@"union", .@"struct", .pointer => if (@hasField(FromStruct, field.name)) { try mapAlloc( cb, allocator, ctx, - @field(from, field_name), - &@field(to, field_name), + @field(from, field.name), + &@field(to, field.name), ); - } else if (field_attrs.default_value_ptr) |_| { - @field(to, field_name) = null; + } else if (field.default_value_ptr) |_| { + @field(to, field.name) = null; } else { - stdx.debug.compileError("Mapping {} -> {} inside {} failed. Missing field {s} in {}", .{ From, To, FromStruct, field_name, ToStruct }); + stdx.debug.compileError("Mapping {} -> {} inside {} failed. Missing field {s} in {}", .{ From, To, FromStruct, field.name, ToStruct }); }, - else => @field(to, field_name) = @field(from, field_name), + else => @field(to, field.name) = @field(from, field.name), } }, .@"union" => { @@ -336,21 +332,25 @@ pub fn MapRestrict(From: type, To: type) type { .@"struct" => |struct_infos| { // We know that at least one of the struct field contains a From. // We map each field individually. Fields without From and comptime fields are removed. - const field_names = struct_infos.field_names; + const fields = struct_infos.fields; var num_fields: usize = 0; - var struct_field_names: [field_names.len][]const u8 = undefined; - var struct_field_types: [field_names.len]type = undefined; - var struct_field_attrs: [field_names.len]std.builtin.Type.Struct.FieldAttributes = undefined; - for (struct_infos.field_names, struct_infos.field_types, struct_infos.field_attrs) |field_name, Field, field_attrs| { - if (!field_attrs.@"comptime" and Contains(Field, From)) { - const R = map(Field); - if (R == Field) { - struct_field_names[num_fields] = field_name; - struct_field_types[num_fields] = Field; - struct_field_attrs[num_fields] = field_attrs; + var struct_field_names: [fields.len][]const u8 = undefined; + var struct_field_types: [fields.len]type = undefined; + var struct_field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; + for (fields) |field| { + if (!field.is_comptime and Contains(field.type, From)) { + const R = map(field.type); + if (R == field.type) { + struct_field_names[num_fields] = field.name; + struct_field_types[num_fields] = field.type; + struct_field_attrs[num_fields] = .{ + .@"comptime" = field.is_comptime, + .@"align" = field.alignment, + .default_value_ptr = field.default_value_ptr, + }; } else { - const name = if (struct_infos.is_tuple) struct_infos.field_names[num_fields] else field_name; + const name = if (struct_infos.is_tuple) struct_infos.fields[num_fields].name else field.name; struct_field_names[num_fields] = name; struct_field_types[num_fields] = R; struct_field_attrs[num_fields] = .{ @@ -377,13 +377,13 @@ pub fn MapRestrict(From: type, To: type) type { .@"union" => |union_info| { // We know that at least one of the union field contains a From. // We map each field individually. Fields without From, are replaced by "void". - const field_names = union_info.field_names; - var union_field_names: [field_names.len][]const u8 = undefined; - var union_field_types: [field_names.len]type = undefined; - var union_field_attrs: [field_names.len]std.builtin.Type.Union.FieldAttributes = undefined; - for (0.., union_info.field_names, union_info.field_types) |i, field_name, Field| { - const FT = map(Field); - union_field_names[i] = field_name; + const fields = union_info.fields; + var union_field_names: [fields.len][]const u8 = undefined; + var union_field_types: [fields.len]type = undefined; + var union_field_attrs: [fields.len]std.builtin.Type.UnionField.Attributes = undefined; + for (0.., fields) |i, field| { + const FT = map(field.type); + union_field_names[i] = field.name; union_field_types[i] = FT; union_field_attrs[i] = .{ .@"align" = @alignOf(FT), @@ -393,19 +393,19 @@ pub fn MapRestrict(From: type, To: type) type { }, .array => |arr_info| [arr_info.len]map(arr_info.child), .pointer => |ptr_info| switch (ptr_info.size) { - .slice => if (ptr_info.attrs.@"const") + .slice => if (ptr_info.is_const) []const map(ptr_info.child) else []map(ptr_info.child), - .one => if (ptr_info.attrs.@"const") + .one => if (ptr_info.is_const) *const map(ptr_info.child) else *map(ptr_info.child), - .many => if (ptr_info.attrs.@"const") + .many => if (ptr_info.is_const) [*]const map(ptr_info.child) else [*]map(ptr_info.child), - .c => if (ptr_info.attrs.@"const") + .c => if (ptr_info.is_const) [*c]map(ptr_info.child) else [*c]map(ptr_info.child), @@ -499,7 +499,7 @@ pub fn visit(comptime callback: anytype, ctx: FnParam(callback, 0), v: anytype) .callback => callback(ctx, v), .recurse => { const TargetType, const mutating_cb = switch (@typeInfo(FnParam(callback, 1))) { - .pointer => |info| .{ info.child, !info.attrs.@"const" }, + .pointer => |info| .{ info.child, !info.is_const }, else => stdx.debug.compileError("zml.meta.visit({any}) is expecting a callback with a pointer as second argument but found {any}", .{ @TypeOf(callback), FnParam(callback, 1) }), }; @@ -511,12 +511,12 @@ pub fn visit(comptime callback: anytype, ctx: FnParam(callback, 0), v: anytype) switch (ptr_info_v.size) { .one => switch (@typeInfo(ChildTypeV)) { - .@"struct" => |s| inline for (s.field_names, s.field_types, s.field_attrs) |field_name, Field, field_attrs| { - if (field_attrs.@"comptime" or comptime !Contains(Field, TargetType)) continue; - if (@typeInfo(Field) == .pointer) { - if (can_error) try visit(callback, ctx, @field(v, field_name)) else visit(callback, ctx, @field(v, field_name)); + .@"struct" => |s| inline for (s.fields) |field| { + if (field.is_comptime or comptime !Contains(field.type, TargetType)) continue; + if (@typeInfo(field.type) == .pointer) { + if (can_error) try visit(callback, ctx, @field(v, field.name)) else visit(callback, ctx, @field(v, field.name)); } else { - if (can_error) try visit(callback, ctx, &@field(v, field_name)) else visit(callback, ctx, &@field(v, field_name)); + if (can_error) try visit(callback, ctx, &@field(v, field.name)) else visit(callback, ctx, &@field(v, field.name)); } }, .array => for (v[0..]) |*elem| { @@ -654,11 +654,11 @@ pub fn visitFlatStruct(comptime callback: anytype, ctx: FnParam(callback, 0), v: return switch (ptr_info_v.size) { .one => switch (@typeInfo(ChildTypeV)) { - .@"struct" => |s| inline for (s.field_names, s.field_types, s.field_attrs) |field_name, Field, field_attrs| { - if (field_attrs.@"comptime" or comptime !Contains(Field, TargetType)) continue; - stdx.debug.assertComptime(@typeInfo(Field) != .pointer, err_msg, err_args); + .@"struct" => |s| inline for (s.fields) |field| { + if (field.is_comptime or comptime !Contains(field.type, TargetType)) continue; + stdx.debug.assertComptime(@typeInfo(field.type) != .pointer, err_msg, err_args); - if (can_error) try visitFlatStruct(callback, ctx, &@field(v, field_name)) else visitFlatStruct(callback, ctx, &@field(v, field_name)); + if (can_error) try visitFlatStruct(callback, ctx, &@field(v, field.name)) else visitFlatStruct(callback, ctx, &@field(v, field.name)); }, .array => for (v[0..]) |*elem| { if (can_error) try visitFlatStruct(callback, ctx, elem) else visitFlatStruct(callback, ctx, elem); @@ -786,17 +786,17 @@ pub fn zip(comptime func: anytype, allocator: std.mem.Allocator, values: anytype .pointer => stdx.debug.compileError("zip only accept by value arguments. Received: {}", .{V}), .@"struct" => |struct_info| { var out: V = values[0]; - inline for (struct_info.field_names, struct_info.field_types, struct_info.field_attrs) |field_name, Field, field_attrs| { - if (field_attrs.@"comptime") continue; - if (@typeInfo(Field) == .pointer) { + inline for (struct_info.fields) |f| { + if (f.is_comptime) continue; + if (@typeInfo(f.type) == .pointer) { stdx.debug.compileError("zip doesn't follow pointers and don't accept struct containing them. Received: {}", .{V}); } - var fields = try allocator.alloc(Field, values.len); + var fields = try allocator.alloc(f.type, values.len); defer allocator.free(fields); for (values, 0..) |val, i| { - fields[i] = @field(val, field_name); + fields[i] = @field(val, f.name); } - @field(out, field_name) = try zip(func, allocator, fields, args); + @field(out, f.name) = try zip(func, allocator, fields, args); } return out; }, @@ -844,7 +844,7 @@ pub fn collectAlloc( allocator: std.mem.Allocator, obj: anytype, ) std.mem.Allocator.Error![]stdx.meta.FnReturn(func) { - stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".param_types.len <= 2, "zml.meta.collect expects a func with two arguments, got: {}", .{@TypeOf(func)}); + stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".params.len <= 2, "zml.meta.collect expects a func with two arguments, got: {}", .{@TypeOf(func)}); const CollectAllocCtx = struct { func_ctx: _CollectCtx(func), @@ -871,7 +871,7 @@ pub fn collectAlloc( /// finds all X in the given object, and write the result of func(X) into a given slice. /// Asserts that the number of X found is equal to the slice len. pub fn collectBuf(func: anytype, func_ctx: _CollectCtx(func), obj: anytype, out: []stdx.meta.FnReturn(func)) void { - stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".param_types.len <= 2, "zml.meta.collectBuf expects a func with one or two arguments, got: {}", .{@TypeOf(func)}); + stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".params.len <= 2, "zml.meta.collectBuf expects a func with one or two arguments, got: {}", .{@TypeOf(func)}); const LocalContext = struct { func_ctx: _CollectCtx(func), out: @TypeOf(out), @@ -916,14 +916,14 @@ pub fn collectPtrs( } fn _CollectCtx(func: anytype) type { - const params = @typeInfo(@TypeOf(func)).@"fn".param_types; + const params = @typeInfo(@TypeOf(func)).@"fn".params; if (params.len == 1) return void; - return params[0] orelse @compileError("anytype not supported in collect"); + return params[0].type orelse @compileError("anytype not supported in collect"); } fn _CollectArg(func: anytype) type { - const params = @typeInfo(@TypeOf(func)).@"fn".param_types; - return params[params.len - 1] orelse @compileError("anytype not supported in collect"); + const params = @typeInfo(@TypeOf(func)).@"fn".params; + return params[params.len - 1].type orelse @compileError("anytype not supported in collect"); } pub fn Contains(Haystack: type, T: type) bool { @@ -939,8 +939,8 @@ pub fn Contains(Haystack: type, T: type) bool { return switch (@typeInfo(Haystack)) { .@"struct" => |info| { if (@hasDecl(Haystack, "zml_no_meta_visit")) return false; - inline for (info.field_types, info.field_attrs) |Field, field_attrs| { - if (!field_attrs.@"comptime" and Contains(Field, T)) { + inline for (info.fields) |field| { + if (!field.is_comptime and Contains(field.type, T)) { return true; } } @@ -948,8 +948,8 @@ pub fn Contains(Haystack: type, T: type) bool { }, .@"union" => |info| { if (@hasDecl(Haystack, "zml_no_meta_visit")) return false; - inline for (info.field_types) |Field| { - if (Contains(Field, T)) + inline for (info.fields) |field| { + if (Contains(field.type, T)) return true; } return false; diff --git a/zml/moe/cute_kernels/persistent_mxfp4.zig b/zml/moe/cute_kernels/persistent_mxfp4.zig index 7208e6ce..1f4b25cb 100644 --- a/zml/moe/cute_kernels/persistent_mxfp4.zig +++ b/zml/moe/cute_kernels/persistent_mxfp4.zig @@ -1032,11 +1032,11 @@ fn runTmaProducer( } fn Tuple(comptime T: type, comptime len: usize) type { - return @Tuple(&@as([len]type, @splat(T))); + return std.meta.Tuple(&([_]type{T} ** len)); } fn arity(comptime T: type) usize { - return @typeInfo(T).@"struct".field_names.len; + return @typeInfo(T).@"struct".fields.len; } /// The MLIR types of a tuple of values, as `openWhile` expects them. diff --git a/zml/moe/cutlass_flashinfer.zig b/zml/moe/cutlass_flashinfer.zig index e2aa61a9..f90a7882 100644 --- a/zml/moe/cutlass_flashinfer.zig +++ b/zml/moe/cutlass_flashinfer.zig @@ -95,7 +95,7 @@ pub const Variant = enum { const max_num_devices = zml.platform.Platform.MAX_NUM_DEVICES; pub const Runners = struct { - runners: [std.meta.fieldNames(Variant).len][max_num_devices]?DeviceRunner = + runners: [std.meta.fields(Variant).len][max_num_devices]?DeviceRunner = @splat(@splat(null)), pub fn deinit(self: *Runners) void { @@ -112,7 +112,7 @@ pub const Runners = struct { if (device < 0 or device >= max_num_devices) return error.UnsupportedDevice; const index: usize = @intCast(device); - const variant_index: usize = @backingInt(variant); + const variant_index: usize = @intFromEnum(variant); if (self.runners[variant_index][index]) |runner| return runner; const api = try fi_cutlass_moe.apiForDevice(device); diff --git a/zml/moe/triton_kernels/mxfp4.zig b/zml/moe/triton_kernels/mxfp4.zig index 74170966..4da51a5a 100644 --- a/zml/moe/triton_kernels/mxfp4.zig +++ b/zml/moe/triton_kernels/mxfp4.zig @@ -64,8 +64,8 @@ pub fn validateInputs(c: Config, a: Inputs) !void { .w2 = zml.Shape.init(.{ c.experts, c.hidden, @divTrunc(c.intermediate, 2) }, .u8), .s2 = zml.Shape.init(.{ c.experts, c.hidden, a.s2.dim(2) }, .f8e8m0), }; - inline for (comptime std.meta.fieldNames(Inputs)) |field| { - if (!@field(a, field).shape().eql(@field(shapes, field))) return error.InvalidInputShape; + inline for (std.meta.fields(Inputs)) |field| { + if (!@field(a, field.name).shape().eql(@field(shapes, field.name))) return error.InvalidInputShape; } } diff --git a/zml/nn.zig b/zml/nn.zig index 7519a435..1403788f 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -2259,13 +2259,8 @@ pub fn collectDims( return context.res; } -fn shapeToDims(shape: anytype) [std.meta.fieldNames(@TypeOf(shape)).len]i64 { - const field_names = comptime std.meta.fieldNames(@TypeOf(shape)); - var result: [field_names.len]i64 = undefined; - inline for (field_names, 0..) |field_name, i| { - result[i] = @field(shape, field_name); - } - return result; +fn shapeToDims(shape: anytype) [@divExact(@sizeOf(@TypeOf(shape)), @sizeOf(i64))]i64 { + return @bitCast(shape); } test collectDims { @@ -2320,7 +2315,7 @@ fn ShapeStruct(comptime dims: anytype) type { @setEvalBranchQuota(rank + 5); var struct_field_names: [rank][]const u8 = undefined; var struct_field_types: [rank]type = undefined; - var struct_field_attrs: [rank]std.builtin.Type.Struct.FieldAttributes = undefined; + var struct_field_attrs: [rank]std.builtin.Type.StructField.Attributes = undefined; const default: i64 = NOT_SET; for (&struct_field_names, &struct_field_types, &struct_field_attrs, dims) |*field_name, *field_type, *field_attr, axis| { field_name.* = @tagName(axis); diff --git a/zml/ops.zig b/zml/ops.zig index ba7172a4..186805ea 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -34,7 +34,7 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@T if (!struct_info.is_tuple) { @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"); } - break :b struct_info.field_names.len; + break :b struct_info.fields.len; }, .array => |array_info| b: { if (array_info.child != Tensor) { @@ -72,7 +72,7 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@T const reducer_block = b: { const ArgsTypes: [input_tensors.len]type = @splat(ReduceArgs); - var args: @Tuple(&ArgsTypes) = undefined; + var args: std.meta.Tuple(&ArgsTypes) = undefined; var block_types: [2 * input_tensors.len]*const mlir.Type = undefined; inline for (0..input_tensors.len) |i| { @@ -158,8 +158,8 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@T } var out: @TypeOf(inputs) = undefined; - inline for (struct_info.field_names, 0..) |field_name, i| { - @field(out, field_name) = Tensor._result(input_tensors[i].shape(), op.result(i)); + inline for (struct_info.fields, 0..) |field, i| { + @field(out, field.name) = Tensor._result(input_tensors[i].shape(), op.result(i)); } break :b out; }, @@ -216,7 +216,7 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func // Note: inputs and inits are infered to be tuple of ReduceArgs const reduce_block, var result = b: { const ArgsTypes: [inits.len]type = @splat(ReduceArgs); - const ArgsType = @Tuple(&ArgsTypes); + const ArgsType = std.meta.Tuple(&ArgsTypes); var args: ArgsType = undefined; var block_types: [2 * inits.len]*const mlir.Type = undefined; @@ -315,7 +315,7 @@ pub const ReduceWindowOpts = struct { pub fn ReduceWindowFn(N: comptime_int) type { return @Fn( &@as([N]type, @splat(ReduceArgs)), - &@as([N]std.builtin.Type.Fn.ParamAttributes, @splat(.{})), + &@as([N]std.builtin.Type.Fn.Param.Attributes, @splat(.{})), [N]Tensor, .{}, ); @@ -405,7 +405,7 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytyp const sort_block = b: { const ArgsTypes: [inputs.len]type = @splat(SortArgs); - const ArgsType = @Tuple(&ArgsTypes); + const ArgsType = std.meta.Tuple(&ArgsTypes); var args: ArgsType = undefined; var block_types: [2 * inputs.len]*const mlir.Type = undefined; @@ -547,7 +547,7 @@ pub fn @"while"( const result: While.State = captured_context.body(initial_state); - if (builtin.mode == .debug) { + if (builtin.mode == .Debug) { const result_tensors = meta.collectPtrs(Tensor, allocator, &result) catch @panic("OOM"); stdx.debug.assert(flat_operands.len == result_tensors.len, "zml.ops.while({s}) expects body to return {d} Tensor values, got {d} values in ", .{ @typeName(While), flat_operands.len, result_tensors.len }); for (0.., flat_operands, result_tensors) |i, in, out| { @@ -1623,7 +1623,7 @@ test "cuda_tile grid" { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Mod.forward, .{a}, platform, .{}); defer exe.deinit(); - const input: [n]f32 = @splat(0); + const input = [_]f32{0} ** n; var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); defer a_buffer.deinit(); @@ -1729,7 +1729,7 @@ pub fn scatter( const update_block, var result = b: { const ArgsTypes: [inputs.len]type = @splat(ScatterArgs); - const ArgsType = @Tuple(&ArgsTypes); + const ArgsType = std.meta.Tuple(&ArgsTypes); var args: ArgsType = undefined; var block_types: [2 * inputs.len]*const mlir.Type = undefined; @@ -2063,7 +2063,7 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, var self_kind: stdx.BoundedArray(GatherAxisKind, constants.MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; for (self._shape.tags(), 0..self.rank()) |t, self_ax| { - const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, @intCast(self_ax), 1); + const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, 1, @intCast(self_ax)); if (indices_shape.hasTag(t)) |id_ax| { // tag is both in self and indices -> it's a batching dim // Note: tags are required for batching. @@ -2087,7 +2087,7 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, if (ax == idx_axes[0]) { // The first val_ax is special cause this is the place where we insert indices axes. for (0.., indices_shape.tags(), indices_shape.dims()) |id_axis_order, id_axis, id_inserted_dim| { - const is_batching_axis = std.mem.containsAtLeastScalar(i64, idx_batch_axes.constSlice(), @intCast(id_axis_order), 1); + const is_batching_axis = std.mem.containsAtLeastScalar(i64, idx_batch_axes.constSlice(), 1, @intCast(id_axis_order)); // Batching axis is already in self. if (is_batching_axis) continue; @@ -2278,10 +2278,10 @@ fn CustomCallResultTypeFromOutputSpec(comptime OutputSpecT: type) type { .@"struct" => |struct_info| b: { if (OutputSpecT == Shape) break :b Tensor; if (!struct_info.is_tuple) @compileError("Expected tuple output shape spec"); - break :b if (struct_info.field_names.len == 1) + break :b if (struct_info.fields.len == 1) Tensor else - [struct_info.field_names.len]Tensor; + [struct_info.fields.len]Tensor; }, .array => |array_info| b: { break :b if (array_info.len == 1) @@ -2322,20 +2322,20 @@ pub fn customCallOutputOperandAliases( ) ?[]const dialects.stablehlo.CustomCallOpts.OutputOperandAlias { const aliases_ = aliases orelse return null; - const output_field_names = @typeInfo(O).@"struct".field_names; + const output_fields = @typeInfo(O).@"struct".fields; comptime var num_aliases = 0; - inline for (output_field_names) |field_name| { - if (@field(aliases_, field_name) != null) num_aliases += 1; + inline for (output_fields) |field| { + if (@field(aliases_, field.name) != null) num_aliases += 1; } const output_operand_aliases = comptime blk: { var aliases_buffer: [num_aliases]dialects.stablehlo.CustomCallOpts.OutputOperandAlias = undefined; var i = 0; - for (output_field_names, 0..) |field_name, output_index| { - if (@field(aliases_, field_name)) |operand| { + for (output_fields, 0..) |field, output_index| { + if (@field(aliases_, field.name)) |operand| { aliases_buffer[i] = .{ .output_index = @intCast(output_index), - .operand_index = @intCast(@backingInt(operand)), + .operand_index = @intCast(@intFromEnum(operand)), }; i += 1; } @@ -2447,7 +2447,7 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, break :b &[1]Tensor{inputs}; } if (!struct_info.is_tuple) @compileError("Expected tuple"); - var inputs_: [struct_info.field_names.len]Tensor = undefined; + var inputs_: [struct_info.fields.len]Tensor = undefined; meta.collectBuf((struct { pub fn func(t: Tensor) Tensor { return t; @@ -2471,7 +2471,7 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, break :b &[1]Shape{outputs}; } if (!struct_info.is_tuple) @compileError("Expected tuple"); - var output_shapes: [struct_info.field_names.len]Shape = undefined; + var output_shapes: [struct_info.fields.len]Shape = undefined; meta.collectBuf((struct { pub fn func(t: Shape) Shape { return t; @@ -2504,14 +2504,14 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, .@"struct" => |struct_info| b: { if (@TypeOf(outputs) == Shape) break :b outputs_flat[0]; if (!struct_info.is_tuple) @compileError("Expected tuple"); - if (struct_info.field_names.len == 1) break :b outputs_flat[0]; - var outputs_: [struct_info.field_names.len]Tensor = undefined; + if (struct_info.fields.len == 1) break :b outputs_flat[0]; + var outputs_: [struct_info.fields.len]Tensor = undefined; @memcpy(&outputs_, outputs_flat); break :b outputs_; }, .array => |array_info| b: { if (array_info.len == 1) break :b outputs_flat[0]; - var outputs_: [array_info.len]Tensor = undefined; + var outputs_: [array_info.fields.len]Tensor = undefined; @memcpy(&outputs_, outputs_flat); break :b outputs_; }, @@ -2536,7 +2536,7 @@ pub fn manualComputation( break :b &[1]Shape{outputs}; } if (!struct_info.is_tuple) @compileError("Expected tuple output shapes"); - var output_shapes_flat: [struct_info.field_names.len]Shape = undefined; + var output_shapes_flat: [struct_info.fields.len]Shape = undefined; meta.collectBuf((struct { pub fn func(t: Shape) Shape { return t; @@ -2945,7 +2945,7 @@ fn manualComputationOutputShapesArg(comptime OutputShapesT: type, local_output_s if (pointer_info.size != .slice or pointer_info.child != Shape) { @compileError("manualComputation body output shapes must be void, Shape, []Shape, or []const Shape; got " ++ @typeName(OutputShapesT)); } - break :b if (pointer_info.attrs.@"const") + break :b if (pointer_info.is_const) local_output_shapes else @constCast(local_output_shapes); @@ -2999,7 +2999,7 @@ fn metadataFieldToMlirAttribute(mlir_ctx: *mlir.Context, comptime T: type, value return switch (type_info) { .comptime_int => .int(mlir_ctx, .u64, @as(u64, value)), .@"enum" => |enum_field| switch (@typeInfo(enum_field.tag_type)) { - .int => |int_tag| metadataIntegerFieldToMlirAttribute(mlir_ctx, int_tag, @backingInt(value)), + .int => |int_tag| metadataIntegerFieldToMlirAttribute(mlir_ctx, int_tag, @intFromEnum(value)), else => @compileError("Unsupported tag type for enum metadata: " ++ @typeName(enum_field.tag_type)), }, .int => |int_field| metadataIntegerFieldToMlirAttribute(mlir_ctx, int_field, value), @@ -3106,11 +3106,15 @@ pub fn ShapeToTensor(T: type) type { } pub fn CustomCallOutputOperandAliases(I: type, O: type) type { - const field_names = @typeInfo(O).@"struct".field_names; + const output_fields = @typeInfo(O).@"struct".fields; + var field_names: [output_fields.len][]const u8 = undefined; + inline for (output_fields, 0..) |field, i| { + field_names[i] = field.name; + } return @Struct( .auto, null, - field_names, + &field_names, &@splat(?std.meta.FieldEnum(I)), &@splat(.{ .default_value_ptr = &@as(?std.meta.FieldEnum(I), null) }), ); @@ -3127,9 +3131,9 @@ pub fn shardingAwareTypedCustomCall( const Output = @TypeOf(output); const Attributes = @TypeOf(attributes); - var output_shapes: [@typeInfo(Output).@"struct".field_names.len]Shape = undefined; - inline for (@typeInfo(Output).@"struct".field_names, 0..) |field_name, i| { - output_shapes[i] = @field(output, field_name); + var output_shapes: [@typeInfo(Output).@"struct".fields.len]Shape = undefined; + inline for (@typeInfo(Output).@"struct".fields, 0..) |field, i| { + output_shapes[i] = @field(output, field.name); } const Handler = struct { @@ -3148,8 +3152,8 @@ pub fn shardingAwareTypedCustomCall( // Convert the slice back to a struct var out: ShapeToTensor(Output) = undefined; - inline for (@typeInfo(Output).@"struct".field_names, 0..) |field_name, i| { - @field(out, field_name) = output_tensors[i]; + inline for (@typeInfo(Output).@"struct".fields, 0..) |field, i| { + @field(out, field.name) = output_tensors[i]; } return out; } @@ -3172,9 +3176,9 @@ pub fn typedCustomCall( const input_tensors: []const Tensor = switch (@typeInfo(Input)) { .@"struct" => |struct_info| b: { - var input_tensors: [struct_info.field_names.len]Tensor = undefined; - inline for (struct_info.field_names, 0..) |field_name, i| { - input_tensors[i] = @field(input, field_name); + var input_tensors: [struct_info.fields.len]Tensor = undefined; + inline for (struct_info.fields, 0..) |field, i| { + input_tensors[i] = @field(input, field.name); } break :b &input_tensors; }, @@ -3188,9 +3192,9 @@ pub fn typedCustomCall( const output_shapes: []const Shape = switch (@typeInfo(Output)) { .@"struct" => |struct_info| b: { - var output_shapes: [struct_info.field_names.len]Shape = undefined; - inline for (struct_info.field_names, 0..) |field_name, i| { - output_shapes[i] = @field(output, field_name); + var output_shapes: [struct_info.fields.len]Shape = undefined; + inline for (struct_info.fields, 0..) |field, i| { + output_shapes[i] = @field(output, field.name); } break :b &output_shapes; }, @@ -3204,13 +3208,13 @@ pub fn typedCustomCall( var metadata_attribute_list = switch (@typeInfo(Attributes)) { .void => std.ArrayList(mlir.NamedAttribute).initCapacity(allocator, 2) catch unreachable, - .@"struct" => |struct_info| std.ArrayList(mlir.NamedAttribute).initCapacity(allocator, struct_info.field_names.len + 2) catch unreachable, + .@"struct" => |struct_info| std.ArrayList(mlir.NamedAttribute).initCapacity(allocator, struct_info.fields.len + 2) catch unreachable, else => @compileError("Unsupported type: " ++ @typeName(Attributes)), }; if (@typeInfo(Attributes) == .@"struct") { - inline for (@typeInfo(Attributes).@"struct".field_names, @typeInfo(Attributes).@"struct".field_types) |field_name, Field| { - if (metadataFieldToMlirAttribute(ctx.mlir_ctx, Field, @field(attributes, field_name))) |attribute| { - metadata_attribute_list.appendAssumeCapacity(mlir.NamedAttribute.named(ctx.mlir_ctx, field_name, attribute)); + inline for (@typeInfo(Attributes).@"struct".fields) |field| { + if (metadataFieldToMlirAttribute(ctx.mlir_ctx, field.type, @field(attributes, field.name))) |attribute| { + metadata_attribute_list.appendAssumeCapacity(mlir.NamedAttribute.named(ctx.mlir_ctx, field.name, attribute)); } } } @@ -3266,8 +3270,8 @@ pub fn typedCustomCall( switch (@typeInfo(Output)) { .@"struct" => |struct_info| { var out: ShapeToTensor(Output) = undefined; - inline for (output_shapes, struct_info.field_names, 0..) |output_shape, field_name, i| { - @field(out, field_name) = Tensor._result(output_shape, op.result(i)); + inline for (output_shapes, struct_info.fields, 0..) |output_shape, field, i| { + @field(out, field.name) = Tensor._result(output_shape, op.result(i)); } return out; }, @@ -3290,26 +3294,26 @@ fn customCallArgsFromPjrtCallFrame(I: type, O: type, A: type, call_frame: *pjrt. A, } { var input: TensorToCustomCallBuffer(I) = undefined; - inline for (@typeInfo(I).@"struct".field_names, 0..) |field_name, i| { + inline for (@typeInfo(I).@"struct".fields, 0..) |field, i| { const buf = call_frame.args.buffers()[i]; - @field(input, field_name) = .fromPjrt(buf); + @field(input, field.name) = .fromPjrt(buf); } var output: ShapeToCustomCallBuffer(O) = undefined; - inline for (@typeInfo(O).@"struct".field_names, 0..) |field_name, i| { + inline for (@typeInfo(O).@"struct".fields, 0..) |field, i| { const shape_buf = call_frame.results.buffers()[i]; - @field(output, field_name) = .fromPjrt(shape_buf); + @field(output, field.name) = .fromPjrt(shape_buf); } const attributes: A = switch (@typeInfo(A)) { .void => {}, .@"struct" => blk: { var attrs: A = undefined; - inline for (@typeInfo(A).@"struct".field_names, @typeInfo(A).@"struct".field_types) |field_name, Field| { + inline for (@typeInfo(A).@"struct".fields) |field| { // TODO: Use getByIndex instead? - const attribute = call_frame.attrs.getByName(.scalar, field_name) orelse - @panic("Attribute not found: " ++ field_name); - @field(attrs, field_name) = attribute.get(Field); + const attribute = call_frame.attrs.getByName(.scalar, field.name) orelse + @panic("Attribute not found: " ++ field.name); + @field(attrs, field.name) = attribute.get(field.type); } break :blk attrs; }, diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 8543b591..f492b120 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -117,13 +117,13 @@ pub fn dtypeFromBufferType(pjrt_type: pjrt.BufferType) DataType { } test bufferTypeFromDtype { - inline for (@typeInfo(DataType).@"enum".field_values) |field_value| { - const dt: DataType = @fromBackingInt(@intCast(field_value)); + inline for (@typeInfo(DataType).@"enum".fields) |field| { + const dt: DataType = @enumFromInt(field.value); try std.testing.expectEqual(dt, dtypeFromBufferType(bufferTypeFromDtype(dt))); } - inline for (@typeInfo(pjrt.BufferType).@"enum".field_values) |field_value| { - const dt: pjrt.BufferType = @fromBackingInt(@intCast(field_value)); + inline for (@typeInfo(pjrt.BufferType).@"enum".fields) |field| { + const dt: pjrt.BufferType = @enumFromInt(field.value); if (dt == .invalid) continue; try std.testing.expectEqual(dt, bufferTypeFromDtype(dtypeFromBufferType(dt))); } diff --git a/zml/platform.zig b/zml/platform.zig index 16ffe6c8..7a9ee710 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -217,7 +217,7 @@ pub const Device = struct { } pub fn memory(self: *const Device, memory_kind: Memory.Kind) ?*const Memory { - return self.memory_by_kind.values[@backingInt(memory_kind)]; + return self.memory_by_kind.values[@intFromEnum(memory_kind)]; } }; @@ -239,7 +239,7 @@ fn sortDevicesById(target: Target, devices: []Device) void { std.mem.sort(Device, devices, Context{ .target = target }, Context.lessThan); - if (builtin.mode == .debug) { + if (builtin.mode == .Debug) { for (devices, 0..) |device, expected_id| { std.debug.assert(platformDeviceSortId(target, device) == expected_id); } diff --git a/zml/profiling/profiler.zig b/zml/profiling/profiler.zig index d46a0c4c..3f3c8409 100644 --- a/zml/profiling/profiler.zig +++ b/zml/profiling/profiler.zig @@ -92,7 +92,7 @@ pub const ProfilerOptions = struct { c.tensorflow_ProfileOptions_set_repository_path(proto, upb.stringView(self.repository_path)); if (self.device_type) |value| { - c.tensorflow_ProfileOptions_set_device_type(proto, @backingInt(value)); + c.tensorflow_ProfileOptions_set_device_type(proto, @intFromEnum(value)); } if (self.include_dataset_ops) |value| { c.tensorflow_ProfileOptions_set_include_dataset_ops(proto, value); diff --git a/zml/profiling/tracer.zig b/zml/profiling/tracer.zig index 5aed01df..e3157d10 100644 --- a/zml/profiling/tracer.zig +++ b/zml/profiling/tracer.zig @@ -3,7 +3,6 @@ const builtin = @import("builtin"); const c = @import("c"); const platforms = @import("platforms"); -const stdx = @import("stdx"); const zffi = @import("ffi"); pub const Span = struct { @@ -45,14 +44,14 @@ pub fn span(comptime name: []const u8, metadata: anytype) Span { return .start(name); }, .@"struct" => |info| { - if (info.field_names.len == 0) { + if (info.fields.len == 0) { return .start(name); } var buffer: [computeEncodedNameLen(name, @TypeOf(metadata))]u8 = undefined; var encoded: std.ArrayList(u8) = .initBuffer(&buffer); - appendEncodedName(&encoded, .failing, name, metadata) catch unreachable; + appendEncodedName(&encoded, .failing, name, info, metadata) catch unreachable; return .start(encoded.items); }, else => @compileError("Unsupported metadata."), @@ -63,34 +62,34 @@ pub fn span(comptime name: []const u8, metadata: anytype) Span { /// bounded by `span` at comptime. The caller owns the returned slice. pub fn formatSpanName(allocator: std.mem.Allocator, name: []const u8, metadata: anytype) ![]const u8 { switch (@typeInfo(@TypeOf(metadata))) { - .@"struct" => { + .@"struct" => |info| { var encoded: std.ArrayList(u8) = .empty; errdefer encoded.deinit(allocator); - try appendEncodedName(&encoded, allocator, name, metadata); + try appendEncodedName(&encoded, allocator, name, info, metadata); return try encoded.toOwnedSlice(allocator); }, else => @compileError("trace metadata must be a struct literal like .{ .step_num = 42 }"), } } -fn appendEncodedName(buffer: *std.ArrayList(u8), allocator: std.mem.Allocator, name: []const u8, metadata: anytype) !void { +fn appendEncodedName(buffer: *std.ArrayList(u8), allocator: std.mem.Allocator, name: []const u8, info: std.builtin.Type.Struct, metadata: anytype) !void { try buffer.appendSlice(allocator, name); var field_count: usize = 0; try buffer.append(allocator, '#'); - inline for (comptime std.meta.fieldNames(@TypeOf(metadata))) |field_name| { - if (shouldEncodeField(field_name, @field(metadata, field_name))) { + inline for (info.fields) |field| { + if (shouldEncodeField(field, @field(metadata, field.name))) { if (field_count != 0) { try buffer.append(allocator, ','); } field_count += 1; - const key = comptime metadataKey(field_name); + const key = comptime metadataKey(field.name); validateMetadataToken(key); try buffer.appendSlice(allocator, key); try buffer.append(allocator, '='); - try appendMetadataValue(buffer, allocator, @field(metadata, field_name), field_name); + try appendMetadataValue(buffer, allocator, @field(metadata, field.name), field.name); } } @@ -115,38 +114,33 @@ fn metadataKey(comptime field_name: []const u8) []const u8 { fn computeEncodedNameLen(comptime name: []const u8, comptime Metadata: type) usize { switch (@typeInfo(Metadata)) { .@"struct" => |info| { - if (info.field_names.len == 0) return name.len; + if (info.fields.len == 0) return name.len; // name#[attrs]# var max_len = name.len + 2; - inline for ( - info.field_names, - info.field_types, - info.field_attrs, - ) |field_name, FieldType, field_attr| { - const key = comptime metadataKey(field_name); + inline for (info.fields) |field| { + const key = comptime metadataKey(field.name); validateMetadataToken(key); - const default: ?FieldType = field_attr.defaultValue(FieldType); // [key]=[value], max_len += key.len + 2; - max_len += switch (@typeInfo(FieldType)) { + max_len += switch (@typeInfo(field.type)) { .bool => 1, .int => @max( - std.fmt.comptimePrint("{}", .{std.math.minInt(FieldType)}).len, - std.fmt.comptimePrint("{}", .{std.math.maxInt(FieldType)}).len, + std.fmt.comptimePrint("{}", .{std.math.minInt(field.type)}).len, + std.fmt.comptimePrint("{}", .{std.math.maxInt(field.type)}).len, ), .@"enum" => |ty| b: { var max_enum_len: usize = 0; - for (ty.field_names) |enum_field_name| { - max_enum_len = @max(max_enum_len, enum_field_name.len); + for (ty.fields) |enum_field| { + max_enum_len = @max(max_enum_len, enum_field.name.len); } break :b max_enum_len; }, - .comptime_int => std.fmt.comptimePrint("{}", .{default.?}).len, - .comptime_float => std.fmt.comptimePrint("{d}", .{default.?}).len, - .enum_literal => @tagName(default.?).len, - else => @compileError("trace metadata field '" ++ field_name ++ "' is not statically bounded; use formatspanName"), + .comptime_int => std.fmt.comptimePrint("{}", .{defaultMetadataValue(field).?}).len, + .comptime_float => std.fmt.comptimePrint("{d}", .{defaultMetadataValue(field).?}).len, + .enum_literal => @tagName(defaultMetadataValue(field).?).len, + else => @compileError("trace metadata field '" ++ field.name ++ "' is not statically bounded; use formatspanName"), }; } return max_len; @@ -155,11 +149,17 @@ fn computeEncodedNameLen(comptime name: []const u8, comptime Metadata: type) usi } } -fn shouldEncodeField(comptime field_name: []const u8, value: anytype) bool { - const FieldType = @TypeOf(value); - return switch (@typeInfo(FieldType)) { +fn defaultMetadataValue(comptime field: std.builtin.Type.StructField) ?field.type { + return if (field.default_value_ptr) |default_opaque| + @as(*const field.type, @ptrCast(@alignCast(default_opaque))).* + else + null; +} + +fn shouldEncodeField(field: std.builtin.Type.StructField, value: anytype) bool { + return switch (@typeInfo(field.type)) { .optional => value != null, - .bool => if (comptime std.mem.eql(u8, field_name, "root")) value else true, + .bool => if (comptime std.mem.eql(u8, field.name, "root")) value else true, else => true, }; } diff --git a/zml/quantization.zig b/zml/quantization.zig index 89e1b210..b125513a 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -474,7 +474,7 @@ test "block FP8 E8M0 scales preserve power-of-two boundaries and nonminor axes" try zml.testing.expectEqualShapes(x.shape().setDim(.k, 8).withDtype(.f32), exe.output_shapes[1]); const fp8_max: f32 = if (dtype == .f8e4m3fn) 448.0 else 224.0; var host: [2 * 256 * 3]f32 = undefined; - var maxima: [2 * 8 * 3]f32 = @splat(0); + var maxima = [_]f32{0} ** (2 * 8 * 3); for (&host, 0..) |*value, i| { const block = (i / 3) % 256 / 32; const scale_index = (i / (256 * 3)) * 24 + block * 3 + i % 3; diff --git a/zml/shape.zig b/zml/shape.zig index 3e3fb56b..11a3efac 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -58,19 +58,19 @@ pub const Shape = struct { if (comptime stdx.meta.isStruct(T)) { var dims_: DimsArray = .empty; var tags_: TagsArray = .empty; - inline for (comptime std.meta.fieldNames(T), comptime std.meta.fieldTypes(T)) |field_name, Field| { - const fv = @field(v, field_name); - if (comptime stdx.meta.isInteger(Field)) { + inline for (std.meta.fields(T)) |field| { + const fv = @field(v, field.name); + if (comptime stdx.meta.isInteger(field.type)) { dims_.appendAssumeCapacity(@intCast(fv)); } else if (@TypeOf(fv) == @EnumLiteral() and comptime isAutoDim(fv)) { dims_.appendAssumeCapacity(-1); } else { - stdx.debug.compileError("Field {s} should be an integer or an auto dimension, got {}", .{ field_name, Field }); + stdx.debug.compileError("Field {s} should be an integer or an auto dimension, got {}", .{ field.name, field.type }); } if (comptime stdx.meta.isTuple(T)) { tags_.appendAssumeCapacity(TagUnknown); } else { - tags_.appendAssumeCapacity(toTag(field_name)); + tags_.appendAssumeCapacity(toTag(field)); } } @@ -107,9 +107,9 @@ pub const Shape = struct { } if (comptime stdx.meta.isTupleOfAny(T, isAxisConvertible)) { - inline for (comptime std.meta.fieldNames(T)) |field_name| { - axes_.appendAssumeCapacity(self.axis(@field(v, field_name))); - tags_.appendAssumeCapacity(self.tag(@field(v, field_name))); + inline for (std.meta.fields(T)) |field| { + axes_.appendAssumeCapacity(self.axis(@field(v, field.name))); + tags_.appendAssumeCapacity(self.tag(@field(v, field.name))); } return .{ axes_, tags_ }; } @@ -191,7 +191,7 @@ pub const Shape = struct { fn isTagConvertible(comptime T: type) bool { return switch (T) { - @EnumLiteral(), [:0]const u8, Tag => true, + @EnumLiteral(), std.builtin.Type.StructField, Tag => true, else => false, }; } @@ -200,14 +200,14 @@ pub const Shape = struct { const T = @TypeOf(v); return switch (T) { @EnumLiteral() => @tagName(v).ptr, - [:0]const u8 => v.ptr, + std.builtin.Type.StructField => v.name.ptr, Tag => v, - else => stdx.debug.compileError("Shape tag should be an @EnumLiteral(), a Shape.Tag or a field name, got {}", .{T}), + else => stdx.debug.compileError("Shape tag should be an @EnumLiteral(), a Shape.Tag or a StructField, got {}", .{T}), }; } fn ensureAttributesAreSync(self: Shape) void { - if (builtin.mode == .debug) { + if (builtin.mode == .Debug) { stdx.debug.assert(self._dims.len == self._tags.len, "Tags, dims and partitioning have diverged! dims={d} tags={d}", .{ self._dims.len, self._tags.len }); } } @@ -312,8 +312,8 @@ pub const Shape = struct { } if (comptime stdx.meta.isStruct(T)) { - inline for (comptime std.meta.fieldNames(T)) |field_name| { - res.appendAssumeCapacity(self.axis(@field(axes_, field_name))); + inline for (std.meta.fields(T)) |field| { + res.appendAssumeCapacity(self.axis(@field(axes_, field.name))); } return res; } @@ -861,11 +861,11 @@ pub const Shape = struct { var res = self; for (0..self.rank()) |ax| { const mesh_axis = self.partition(ax).meshAxis() orelse continue; - inline for (comptime std.meta.fieldNames(T)) |field_name| { - const old_axis = mesh.resolveLogicalAxis(toTag(field_name)) orelse - stdx.debug.panic("Mesh '{s}' has no logical axis '{s}'", .{ mesh.name, field_name }); + inline for (std.meta.fields(T)) |field| { + const old_axis = mesh.resolveLogicalAxis(toTag(field)) orelse + stdx.debug.panic("Mesh '{s}' has no logical axis '{s}'", .{ mesh.name, field.name }); if (mesh_axis == old_axis) { - const new_tag = toTag(@field(mapping, field_name)); + const new_tag = toTag(@field(mapping, field.name)); const new_axis = mesh.resolveLogicalAxis(new_tag) orelse stdx.debug.panic("Mesh '{s}' has no logical axis '{s}'", .{ mesh.name, new_tag }); res._sharding = res._sharding.set(ax, .sharded(@intCast(new_axis))); @@ -1015,10 +1015,10 @@ pub const Shape = struct { const T = @TypeOf(renames); stdx.debug.assertComptime(stdx.meta.isStructOfAny(T, isAxisConvertible), "Must pass a struct of enum literals. Passed: {any}", .{T}); var res = self; - inline for (comptime std.meta.fieldNames(T)) |field_name| { - const new_field = @field(renames, field_name); + inline for (std.meta.fields(T)) |field| { + const new_field = @field(renames, field.name); stdx.debug.assert(self.hasTag(new_field) == null, "{f}.rename({any}) failed because of duplicated axis {any}", .{ self, renames, new_field }); - res._tags.set(self.axis(field_name), toTag(new_field)); + res._tags.set(self.axis(field), toTag(new_field)); } return res; } @@ -1232,8 +1232,8 @@ pub const Shape = struct { stdx.debug.assertComptime(stdx.meta.isStruct(T), "Must pass struct of enum literals like .{ .a = .{ .a1, .a2 } }. Passed: {any}", .{T}); var res = self; - inline for (comptime std.meta.fieldNames(T)) |field_name| { - res = res.splitAxis(field_name, @field(axes_, field_name)); + inline for (std.meta.fields(T)) |field| { + res = res.splitAxis(field, @field(axes_, field.name)); } return res; } @@ -1344,9 +1344,9 @@ pub const Shape = struct { stdx.debug.assertComptime(stdx.meta.isStruct(T), "Must pass struct of enum literals like .{ .a = .{ .a1, .a2 } }. Passed: {any}", .{T}); var res = self; - inline for (comptime std.meta.fieldNames(T), comptime std.meta.fieldTypes(T)) |field_name, Field| { - stdx.debug.assertComptime(stdx.meta.isTupleOfAny(Field, isAxisConvertible) or stdx.meta.isSliceOfAny(Field, isAxisConvertible), "Must pass struct of axes. Passed: {any}", .{Field}); - res = res.mergeAxis(field_name, @field(axes_, field_name)); + inline for (std.meta.fields(T)) |field| { + stdx.debug.assertComptime(stdx.meta.isTupleOfAny(field.type, isAxisConvertible) or stdx.meta.isSliceOfAny(field.type, isAxisConvertible), "Must pass struct of axes. Passed: {any}", .{field.type}); + res = res.mergeAxis(field, @field(axes_, field.name)); } return res; } @@ -1398,14 +1398,14 @@ pub const Shape = struct { } if (comptime stdx.meta.isStruct(V)) { - const field_names = comptime std.meta.fieldNames(V); - stdx.debug.assertComptime(field_names.len <= constants.MAX_RANK, "Too many fields in struct {} ({d}). Max supported is {d}.", .{ V, field_names.len, constants.MAX_RANK }); - inline for (field_names) |field_name| { - const fv = @field(v, field_name); + const fields = std.meta.fields(V); + stdx.debug.assertComptime(fields.len <= constants.MAX_RANK, "Too many fields in struct {} ({d}). Max supported is {d}.", .{ V, fields.len, constants.MAX_RANK }); + inline for (fields) |field| { + const fv = @field(v, field.name); vals_.appendAssumeCapacity(fv); if (!comptime stdx.meta.isTuple(V)) { - tags_.appendAssumeCapacity(toTag(field_name)); + tags_.appendAssumeCapacity(toTag(field)); } } return .{ vals_, tags_ }; @@ -1435,11 +1435,11 @@ pub const Shape = struct { if (comptime stdx.meta.isStruct(V)) { for (0..self.rank()) |_| res.appendAssumeCapacity(default); - const field_names = comptime std.meta.fieldNames(V); - stdx.debug.assertComptime(field_names.len <= constants.MAX_RANK, "expects up to {} options struct literal, got {}", .{ V, constants.MAX_RANK, field_names.len }); - inline for (field_names) |field_name| { - const a = self.axis(field_name); - res.buffer[a] = @field(options, field_name); + const fields = std.meta.fields(V); + stdx.debug.assertComptime(fields.len <= constants.MAX_RANK, "expects up to {} options struct literal, got {}", .{ V, constants.MAX_RANK, fields.len }); + inline for (fields) |field| { + const a = self.axis(field); + res.buffer[a] = @field(options, field.name); } return res; } diff --git a/zml/slice.zig b/zml/slice.zig index 2deb96d2..f096608b 100644 --- a/zml/slice.zig +++ b/zml/slice.zig @@ -31,7 +31,7 @@ pub const Slice = struct { const byte_strides = shape.computeByteStrides(); return .{ .bytes = bytes, - .mutable = !type_info.pointer.attrs.@"const", + .mutable = !type_info.pointer.is_const, .shape = shape, .offset_bytes = 0, .byte_strides = byte_strides, @@ -147,12 +147,12 @@ pub const Slice = struct { } pub fn items(slice: Slice, comptime T: type) []T { - stdx.debug.assertComptime(T == bool or @bitSizeOf(T) >= 8, "zml.Slice stores packed sub-bytes type so you need to pass a packed struct here like stdx.Packed({0}). Got: {0}", .{T}); + stdx.debug.assertComptime(T == bool or @bitSizeOf(T) >= 8, "zml.Slice stores packed sub-bytes type so you need to pass a packed type here like @Vector({}, {}). Got: {}", .{ @divFloor(8, @bitSizeOf(T)), T, T }); return @ptrCast(@alignCast(slice.data())); } pub fn constItems(slice: Slice, comptime T: type) []const T { - stdx.debug.assertComptime(T == bool or @bitSizeOf(T) >= 8, "zml.Slice stores packed sub-bytes type so you need to pass a packed struct here like zml.Float4E2M1.Packed. Got: {}", .{T}); + stdx.debug.assertComptime(T == bool or @bitSizeOf(T) >= 8, "zml.Slice stores packed sub-bytes type so you need to pass a packed type here like @Vector({}, {}). Got: {}", .{ @divFloor(8, @bitSizeOf(T)), T, T }); return @ptrCast(@alignCast(slice.constData())); } @@ -289,10 +289,6 @@ pub const Slice = struct { const raw_values: []const T = @ptrCast(@alignCast(slice.bytes[slice.offset_bytes..])); const values = raw_values[0..needed_len]; - if (comptime dt.bitSizeOf() < 8) { - return try stdx.fmt.formatSliceAny(values, options, stride, writer); - } - switch (comptime dt.class()) { .float => try stdx.fmt.formatFloatSlice(values, options, stride, writer), .integer => try stdx.fmt.formatIntSlice(values, options, stride, writer), diff --git a/zml/tensor.zig b/zml/tensor.zig index 0a3c530f..96b60c0c 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -38,7 +38,7 @@ pub const Tensor = struct { } fn nextTensorId() Id { - return @fromBackingInt(@intCast(Tensor.current_id.fetchAdd(1, .seq_cst))); + return @enumFromInt(Tensor.current_id.fetchAdd(1, .seq_cst)); } pub fn fromShape(shape_: Shape) Tensor { @@ -83,7 +83,7 @@ pub const Tensor = struct { var res: Tensor = .{ ._shape = sh, ._value = val, .id = nextTensorId() }; res._shape._sharding.partition = .open(sh.rank()); - if (builtin.mode == .debug) { + if (builtin.mode == .Debug) { // Check that the MLIR value actually have the same shape. const other = fromMlirValue(val); const eql_dtype = sh.dtype() == other.dtype(); @@ -468,7 +468,7 @@ pub const Tensor = struct { const platform = zml.testing.env(); const io = std.testing.io; - const inputs: [6]stdx.Packed(i4) = .{ .pack(-3.0, -2), .pack(-1, 1), .pack(2, 3), .pack(1, 2), .pack(3, 4), .pack(5, -5) }; + const inputs: [6]@Vector(2, i4) = .{ .{ -3.0, -2 }, .{ -1, 1 }, .{ 2, 3 }, .{ 1, 2 }, .{ 3, 4 }, .{ 5, -5 } }; const x_t = Tensor.init(.{ 6, 2 }, .i4); const y_t = Tensor.init(.{ 6, 2 }, .i4); const z_t = Tensor.init(.{ 6, 2 }, .i4); @@ -1432,8 +1432,8 @@ pub const Tensor = struct { test "convert u2 -> u8" { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const x_u2: [2]stdx.Packed(u2) = .{ .pack(0, 1, 2, 3), .pack(3, 2, 1, 0) }; - comptime std.debug.assert(@sizeOf(@TypeOf(x_u2)) == 2); + const x_u2: [2]@Vector(4, u2) = .{ .{ 0, 1, 2, 3 }, .{ 3, 2, 1, 0 } }; + std.debug.assert(@sizeOf(@TypeOf(x_u2)) == 2); const x_u8: [8]u8 = .{ 0, 1, 2, 3, 3, 2, 1, 0 }; const x_u2_t: Tensor = .init(.{8}, .u2); @@ -3809,8 +3809,8 @@ pub const Tensor = struct { const has_name: ?[:0]const u8, const a = switch (@typeInfo(@TypeOf(named_axis_))) { .int, .comptime_int => .{ null, self.axis(@as(i64, @intCast(named_axis_))) }, .@"struct" => |info| blk: { - stdx.debug.assertComptime(info.field_names.len == 1, err_msg, .{}); - break :blk .{ info.field_names[0], self.axis(@field(named_axis_, info.field_names[0])) }; + stdx.debug.assertComptime(info.fields.len == 1, err_msg, .{}); + break :blk .{ info.fields[0].name, self.axis(@field(named_axis_, info.fields[0].name)) }; }, else => stdx.debug.compileError(err_msg, .{}), }; diff --git a/zml/tokenizer/iree/tokenizer.zig b/zml/tokenizer/iree/tokenizer.zig index 0ae25c42..bc986d4b 100644 --- a/zml/tokenizer/iree/tokenizer.zig +++ b/zml/tokenizer/iree/tokenizer.zig @@ -577,8 +577,8 @@ pub const Normalizer = struct { ), }, .Strip => |direction| b: { - const strip_left = @backingInt(direction) & 0b10 != 0; - const strip_right = @backingInt(direction) & 0b01 != 0; + const strip_left = @intFromEnum(direction) & 0b10 != 0; + const strip_right = @intFromEnum(direction) & 0b01 != 0; break :b c.iree_tokenizer_normalizer_strip_allocate(strip_left, strip_right, iree_allocator, &norm); }, }; @@ -785,10 +785,10 @@ test "normalizer sequence preserves composition across tiles" { const allocator = std.testing.allocator; var normalizer = try Normalizer.init(allocator, &.{ .NFC, .Lowercase }); defer normalizer.deinit(); - const input = @as([63]u8, @splat('a')) ++ "e\u{301}b"; + const input = "a" ** 63 ++ "e\u{301}b"; const output = try normalizer.normalize(allocator, input); defer allocator.free(output); - try std.testing.expectEqualSlices(u8, @as([63]u8, @splat('a')) ++ "éb", output); + try std.testing.expectEqualSlices(u8, "a" ** 63 ++ "éb", output); } test "normalizer sequence allocation failure retains child ownership" { From b148f0b919d73ed0f936473c35fc7c495aa7e90e Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 7 Oct 2026 16:33:44 +0200 Subject: [PATCH 761/881] workspace: bump XLA + LLVM + IRs (#903) --- MODULE.bazel | 5 +- MODULE.bazel.lock | 13 +- kernels/fly/builder.zig | 63 +++- kernels/mosaic_tpu/README.md | 18 +- kernels/mosaic_tpu/builder.zig | 36 ++- mlir/dialects/cuda_tile/cuda_tile.zig | 11 +- mlir/dialects/fly/fly.zig | 6 +- mlir/dialects/gpu.zig | 2 +- mlir/dialects/mosaic_tpu/BUILD.bazel | 2 +- mlir/dialects/mosaic_tpu/mosaic_tpu.zig | 189 +++++++++-- mlir/dialects/shardy.zig | 1 + mlir/dialects/ttir/ttir_capi.cc | 6 +- pjrt/BUILD.bazel | 3 +- pjrt/ffi.zig | 22 +- pjrt/pjrt.zig | 2 - platforms/cpu/cpu.bzl | 12 +- platforms/cuda/cuda.bzl | 8 +- platforms/metal/metal.bzl | 4 +- platforms/rocm/rocm.bzl | 4 +- platforms/tpu/tpu.bzl | 4 +- third_party/cuda_tile/cuda_tile.bazel | 35 ++ third_party/cuda_tile/repo.bzl | 4 +- third_party/flydsl/flydsl.bazel | 29 +- third_party/flydsl/mlir_legacy_assembly.patch | 39 +++ third_party/flydsl/mlir_property_ref.patch | 298 ++---------------- third_party/flydsl/repo.bzl | 3 +- third_party/grpc/plugin-flags.patch | 19 ++ third_party/mosaic_tpu/BUILD.bazel | 1 - third_party/mosaic_tpu/mosaic_tpu.bazel | 275 ---------------- third_party/mosaic_tpu/repo.bzl | 11 - third_party/non_module_deps.bzl | 2 - third_party/xla/repo.bzl | 2 +- zml/Compiler.zig | 5 + zml/ops.zig | 2 +- 34 files changed, 484 insertions(+), 652 deletions(-) create mode 100644 third_party/flydsl/mlir_legacy_assembly.patch create mode 100644 third_party/grpc/plugin-flags.patch delete mode 100644 third_party/mosaic_tpu/BUILD.bazel delete mode 100644 third_party/mosaic_tpu/mosaic_tpu.bazel delete mode 100644 third_party/mosaic_tpu/repo.bzl diff --git a/MODULE.bazel b/MODULE.bazel index 1e05f12a..5e5d9c17 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -52,9 +52,12 @@ bazel_dep(name = "rules_swift", version = "3.6.1", repo_name = None) bazel_dep(name = "rules_zig", version = "0.15.1") bazel_dep(name = "tar.bzl", version = "0.10.4") bazel_dep(name = "llvm", version = "0.8.24") +bazel_dep(name = "libxml2", version = "2.15.3") +bazel_dep(name = "xz", version = "5.4.5.bcr.9", repo_name = "llvm_xz") bazel_dep(name = "zlib-ng", version = "2.3.3", repo_name = "llvm_zlib") bazel_dep(name = "zstd", version = "1.5.7.bcr.1", repo_name = "llvm_zstd") bazel_dep(name = "with_cfg.bzl", version = "0.14.4") +bazel_dep(name = "highway", version = "1.4.0") bazel_dep(name = "grpc", version = "1.80.0") single_version_override( module_name = "grpc", @@ -62,6 +65,7 @@ single_version_override( patches = [ "//:third_party/grpc/protobuf-35-upb-load.patch", "//:third_party/grpc/grpc++_unsecure-visibility.patch", + "//:third_party/grpc/plugin-flags.patch", ], version = "1.80.0", ) @@ -314,7 +318,6 @@ use_repo( "libvaxis", "linenoise", "mnist", - "mosaic_tpu", "org_swig_swig", "stb", "translate-c", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 838e2011..b019f47e 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -165,6 +165,8 @@ "https://bcr.bazel.build/modules/grpc-proto/0.0.0-20240627-ec30f58/MODULE.bazel": "88de79051e668a04726e9ea94a481ec6f1692086735fd6f488ab908b3b909238", "https://bcr.bazel.build/modules/grpc/1.80.0/MODULE.bazel": "35ef253a0a31055c8a2495640516224647545b51673b0ccb956b116ee92ef554", "https://bcr.bazel.build/modules/grpc/1.80.0/source.json": "55bf4cdcf278a450185ff93bef843a0d7db0a83d0b9761d9c1dee91faa9cdf46", + "https://bcr.bazel.build/modules/highway/1.4.0/MODULE.bazel": "22d48d947bda0c685d6174e62f9b6690089fefb87dfff4943cd420410ae82f68", + "https://bcr.bazel.build/modules/highway/1.4.0/source.json": "ae87bf2499d4902a15d760e14b7a1e94dc5dd0564bec256b9b0ed46c0e835f89", "https://bcr.bazel.build/modules/jq.bzl/0.1.0/MODULE.bazel": "2ce69b1af49952cd4121a9c3055faa679e748ce774c7f1fda9657f936cae902f", "https://bcr.bazel.build/modules/jq.bzl/0.1.0/source.json": "746bf13cac0860f091df5e4911d0c593971cd8796b5ad4e809b2f8e133eee3d5", "https://bcr.bazel.build/modules/jsoncpp/1.9.5/MODULE.bazel": "31271aedc59e815656f5736f282bb7509a97c7ecb43e927ac1a37966e0578075", @@ -175,8 +177,10 @@ "https://bcr.bazel.build/modules/libbacktrace/1.0.0-20250926-7939218/source.json": "83239989d09748f63dee4bdd430d261e217a012670e066e2238925a303a65d89", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", "https://bcr.bazel.build/modules/libpfm/4.11.0/source.json": "caaffb3ac2b59b8aac456917a4ecf3167d40478ee79f15ab7a877ec9273937c9", - "https://bcr.bazel.build/modules/llvm/0.8.20/MODULE.bazel": "2c0fa2fb60ab1f3a8a7b46a6c60b67fd78bffcfaf7958477e74a971c0a1f1915", - "https://bcr.bazel.build/modules/llvm/0.8.20/source.json": "21ef6b5c124a1a39341752e8458de9acd6fb4abc9ad73e6e3147b374d95e5e8c", + "https://bcr.bazel.build/modules/libxml2/2.15.3/MODULE.bazel": "a6bffafd8aa8b0c13eaa4bf25d55be188ec8548549395f03e98c39bab264adaa", + "https://bcr.bazel.build/modules/libxml2/2.15.3/source.json": "fd7c420cb14981416c9fadd0db555bd1e249355e0e5b63b2ce15b92996b5fbfb", + "https://bcr.bazel.build/modules/llvm/0.8.24/MODULE.bazel": "e03e030b1d0b44bb05c52665c2f43de9260af9152f666262e11e25f6c072aae3", + "https://bcr.bazel.build/modules/llvm/0.8.24/source.json": "924ed48858c1c081698f1992c7b489434375de901c47b7046e203459840cb544", "https://bcr.bazel.build/modules/mbedtls/3.6.0/MODULE.bazel": "8e380e4698107c5f8766264d4df92e36766248447858db28187151d884995a09", "https://bcr.bazel.build/modules/mbedtls/3.6.0/source.json": "1dbe7eb5258050afcc3806b9d43050f71c6f539ce0175535c670df606790b30c", "https://bcr.bazel.build/modules/nlohmann_json/3.11.3/MODULE.bazel": "87023db2f55fc3a9949c7b08dc711fae4d4be339a80a99d04453c4bb3998eefc", @@ -299,6 +303,7 @@ "https://bcr.bazel.build/modules/rules_cc/0.2.25/source.json": "8df28a9a97b878fe45f0a4f4adf019869dda51a60ee2f6e15a0c79656a25058d", "https://bcr.bazel.build/modules/rules_cc/0.2.4/MODULE.bazel": "1ff1223dfd24f3ecf8f028446d4a27608aa43c3f41e346d22838a4223980b8cc", "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", + "https://bcr.bazel.build/modules/rules_cc/0.2.9/MODULE.bazel": "34263f1dca62ea664265438cef714d7db124c03e1ed55ebb4f1dc860164308d1", "https://bcr.bazel.build/modules/rules_distroless/0.8.0/MODULE.bazel": "a3e473943b685190e20668f5149f62a116fdcbeba66c7b9778f71d2aed4533c1", "https://bcr.bazel.build/modules/rules_distroless/0.8.0/source.json": "a6931e20d97a13675693adb8b6cd3a0e6eb5218db43dfc83b75566ddc4158efc", "https://bcr.bazel.build/modules/rules_foreign_cc/0.10.1/MODULE.bazel": "b9527010e5fef060af92b6724edb3691970a5b1f76f74b21d39f7d433641be60", @@ -458,6 +463,8 @@ "https://bcr.bazel.build/modules/xds/0.0.0-20240423-555b57e/MODULE.bazel": "cea509976a77e34131411684ef05a1d6ad194dd71a8d5816643bc5b0af16dc0f", "https://bcr.bazel.build/modules/xds/0.0.0-20251210-ee656c7/MODULE.bazel": "03e1c50b16fc2403de5dd98ff8e2dc0b404865740434b8705c7c0853241f943d", "https://bcr.bazel.build/modules/xds/0.0.0-20251210-ee656c7/source.json": "69b87f3bff38ba29399c012f3e3fc0f239ac4e5c1ae66ebd9481be6d8461da55", + "https://bcr.bazel.build/modules/xz/5.4.5.bcr.9/MODULE.bazel": "ff7cf8a5334a96b7b9ffcbe73418d45929ce125efebac39c1ee7d14bd2ae7a9b", + "https://bcr.bazel.build/modules/xz/5.4.5.bcr.9/source.json": "abf61f3c1630eab7b81722f710060b6f31e7c160511bcbfaad74bb4385883051", "https://bcr.bazel.build/modules/yq.bzl/0.1.1/MODULE.bazel": "9039681f9bcb8958ee2c87ffc74bdafba9f4369096a2b5634b88abc0eaefa072", "https://bcr.bazel.build/modules/yq.bzl/0.3.1/MODULE.bazel": "9bcb7151b3cd4681b89d350530eaf7b45e32a44dda94843b8932b0cb1cd4594a", "https://bcr.bazel.build/modules/yq.bzl/0.3.1/source.json": "f0b0f204a2a6b0e34b4c9541efe8c04f2ef1af65948daa784eccea738b21dbd2", @@ -727,7 +734,7 @@ }, "@@rules_oci+//oci:extensions.bzl%oci": { "general": { - "bzlTransitiveDigest": "lRG1JBXWgJgDfEfl/34ZHgTFACz2I0bpfZWFtcoM+hc=", + "bzlTransitiveDigest": "u2chY5noKxJ9cum/JgI8hogPYhYniVKnJLd6f2ChPI4=", "usagesDigest": "JLbamplho20rviuebatMGqnQnaoj32ewCzLuRVusGoU=", "recordedInputs": [ "REPO_MAPPING:aspect_bazel_lib+,bazel_tools bazel_tools", diff --git a/kernels/fly/builder.zig b/kernels/fly/builder.zig index c560d2df..bb6e68f0 100644 --- a/kernels/fly/builder.zig +++ b/kernels/fly/builder.zig @@ -536,7 +536,7 @@ pub const Value = struct { const off: Value = if (@TypeOf(offset) == Value) offset else k.constant(.i32, offset); const wid = k.constant(.i32, width); const mode_attr = switch (mode) { - inline else => |m| fly.parseAttr(k.ctx, "#gpu"), + inline else => |m| fly.parseAttr(k.ctx, "#gpu>"), }; const op = fly.make(k.ctx, "gpu.shuffle", .{ .operands = .{ .flat = &.{ self.inner, off.inner, wid.inner } }, @@ -1063,7 +1063,9 @@ pub const Builder = struct { /// A matrix atom with SSA operands; unlike `gemm`, this needs no register /// allocas and can carry its accumulator through control flow. pub fn mmaAtomCall(self: *Builder, atom: Value, a: Value, b: Value, c: Value) Value { - return self.emit(fly.typed(self.ctx, "mma_atom_call_ssa", &.{ atom.inner, a.inner, b.inner, c.inner }, &.{c.type_()}, .empty, self.loc())); + return self.emit(fly.typed(self.ctx, "mma_atom_call_ssa", &.{ atom.inner, a.inner, b.inner, c.inner }, &.{c.type_()}, fly.attrs(&.{ + .named(self.ctx, "operandSegmentSizes", .denseArray(self.ctx, .i32, &.{ 1, 0, 1, 1, 1 })), + }), self.loc())); } /// AMD byte permutation. ROCDL has no corresponding operation at our pin. @@ -1893,3 +1895,60 @@ test "static queries read inferred types" { try std.testing.expectEqual(@as(i64, 64 * 32), tiled.sizeStatic()); try std.testing.expectEqual(@as(usize, 4), tiled.shapeStatic().rank()); } + +test "atomic addition builds, verifies and re-parses" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "atomic_add"); + defer b.deinit(); + _ = try b.declareArgs(.{}); + const counter = b.sharedArray(.i32, 1, 4); + _ = b.ptrAtomicAdd(counter.emitIter(), b.constant(.i32, 1), .workgroup); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + + // LLVM's current custom assembly stores alignment in the property dictionary. + for ([_][]const u8{ + "llvm.atomicrmw add", + "syncscope(\"workgroup\") monotonic ", + ": !llvm.ptr<3>, i32", + }) |needle| { + if (std.mem.indexOf(u8, ir, needle) == null) { + std.debug.print("IR missing `{s}`:\n{s}\n", .{ needle, ir }); + return error.TestUnexpectedResult; + } + } + const reparsed = try mlir.Module.parse(ctx, ir); + defer reparsed.deinit(); + try std.testing.expect(reparsed.operation().verify()); +} + +test "SSA matrix atom builds, verifies and re-parses" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "mma_atom_ssa"); + defer b.deinit(); + _ = try b.declareArgs(.{}); + const atom_type = try fly.rocdl.MmaOpCDNA3MFMAType.get(ctx, .{ + .m = 16, + .n = 16, + .k = 16, + .elemTyA = .float(ctx, .bf16), + .elemTyB = .float(ctx, .bf16), + .elemTyAcc = .float(ctx, .f32), + }); + const atom = b.mmaAtom(atom_type.type_()); + const lhs = b.constant(.bf16, 0).splat(4); + const rhs = b.constant(.bf16, 0).splat(4); + const acc = b.constant(.f32, 0).splat(4); + const result = b.mmaAtomCall(atom, lhs, rhs, acc); + try std.testing.expect(result.type_().eql(acc.type_())); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + + const reparsed = try mlir.Module.parse(ctx, ir); + defer reparsed.deinit(); + try std.testing.expect(reparsed.operation().verify()); +} diff --git a/kernels/mosaic_tpu/README.md b/kernels/mosaic_tpu/README.md index 2b0d7004..f6893259 100644 --- a/kernels/mosaic_tpu/README.md +++ b/kernels/mosaic_tpu/README.md @@ -1,6 +1,6 @@ # ZML Mosaic TPU DSL — Author's Guide -A Zig builder for Mosaic TPU IR (the `tpu` dialect shipped by JAX) that tries +A Zig builder for Mosaic TPU IR (the `tpu` dialect in XLA) that tries to read like `pallas.tpu` Python while staying in pure Zig. Used by `examples/mosaic_gdn` and by anything else that wants to emit a TPU kernel op-for-op against what `pallas_call` produces. @@ -9,8 +9,8 @@ The DSL talks to the `tpu` dialect through `mlir.Operation.make(ctx, "tpu.", A Mosaic kernel is a pure IR generator: it builds its MLIR in a throwaway context, canonicalizes it, serializes it to a string, and hands that to a `stablehlo.custom_call` targeting `tpu_custom_call` — it never touches ZML's -compilation context (which carries only `func`/`stablehlo`/`sdy`). Until the -JAX repo is wired into `MODULE.bazel`, the whole stack is `manual`-tagged in Bazel. +compilation context (which carries only `func`/`stablehlo`/`sdy`). The dialect and +its C API come from the pinned XLA repository. Two layers define the surface: @@ -479,7 +479,7 @@ Use these for SMEM args (e.g. `cu_seqlens`) where Pallas emits scalar | `b.concatenate(sources, dimension)` | `tpu.concatenate`. | | `b.transpose(src, permutation)` | `tpu.transpose`. | | `b.broadcastInSublanes(src, lane)` | `tpu.broadcast_in_sublanes`. | -| `b.rotate(src, .{ .amount, .dimension, .stride, .stride_dimension })` | `tpu.rotate`. | +| `b.rotate(src, .{ .amount, .dimension, .stride, .stride_dimension })` | `tpu.dynamic_rotate` with a constant i32 amount. | | `b.vectorExtract(src, position)` | `vector.extract` — scalar / lower-rank slice. | | `b.iota(shape, dtype, dimensions)` | `tpu.iota` over a chosen dim. | | `b.arange(n, dtype)` | Pallas-style 1-D iota: `<1xN>` `tpu.iota` + `vector.shape_cast` to ``. Mirrors `_iota_lowering_rule`. | @@ -536,7 +536,7 @@ through `vector.broadcast`) or `b.full / b.splat` for comptime constants. |-----------------------------------------------------|----------------------------------------------------------| | `b.allReduce(input, kind, dim)` | `tpu.all_reduce` — keeps `dim` as size 1. | | `b.reduceIndex(input, kind, axis)` | `tpu.reduce_index` — drop axis, replace with `index`. | -| `b.scan(input, kind, mask)` | `tpu.scan`. | +| `b.scan(input, kind, mask)` | `tpu.scan` along the last dimension; optional rank-one mask. | | `b.sort(keys, values, mask, .{ .descending })` | `tpu.sort` — returns `[3]Value` (sorted_keys, indices, sorted_values). | --- @@ -619,9 +619,9 @@ For multi-core / async DMA orchestration: | `b.semBarrier()` | `tpu.sem_barrier`. | | `b.semRead(sem)` | `tpu.sem_read`. | | `b.semWait(sem, amount)` | `tpu.sem_wait`. | -| `b.semSignal(sem, amount)` / `b.semSignalOpts(sem, amount, .{ .device_id, .core_id })` | `tpu.sem_signal`. | +| `b.semSignal(sem, amount)` / `b.semSignalOpts(sem, amount, .{ .device_id, .core_id, .subcore_id })` | `tpu.sem_signal`. | | `b.barrier(barrier_id)` | `tpu.barrier`. | -| `b.enqueueDma(src, dst, target_sem, .{ .source_semaphore, .device_id, .core_id, .priority, .strict_ordering })` | `tpu.enqueue_dma`. | +| `b.enqueueDma(src, dst, target_sem, .{ .source_semaphore, .device_id, .core_id, .subcore_id, .priority, .strict_ordering })` | `tpu.enqueue_dma`. | | `b.waitDma2(sem, src, dst, .{ .device_id, .core_id, .strict_ordering })` | `tpu.wait_dma2`. | | `b.deviceId()` | `tpu.device_id`. | | `b.delay(cycles)` | `tpu.delay`. | @@ -830,3 +830,7 @@ is the manual form. always means an operand-type mismatch or a missing block terminator — `scf.yield` operand count must match the loop's result types, and `func.return` arity must match the declared result types. + +The removed low-level `tpu.relayout` helper has no replacement: upstream deleted +the unused operation and its vector-layout attribute. `Builder.rotate` retains +its static-amount API and emits `tpu.dynamic_rotate` with an i32 constant. diff --git a/kernels/mosaic_tpu/builder.zig b/kernels/mosaic_tpu/builder.zig index 157648df..5da2c899 100644 --- a/kernels/mosaic_tpu/builder.zig +++ b/kernels/mosaic_tpu/builder.zig @@ -3,7 +3,7 @@ //! Args are memrefs (`!memref`) plus optional scalars, //! iteration-index, and semaphore args. The dialect must be registered into //! the MLIR context — see `mlir/dialects/mosaic_tpu`. The entire Mosaic stack is -//! `manual`-tagged in Bazel until the JAX repo is wired into `MODULE.bazel`. +//! backed by the Mosaic dialect in the pinned XLA repository. const std = @import("std"); @@ -380,6 +380,7 @@ pub const VectorStoreOpts = struct { pub const MatmulOpts = struct { transpose_lhs: bool = false, transpose_rhs: bool = false, + transpose_lhs_hint: bool = false, precision: ?ContractPrecision = null, /// Optional `#tpu.dot_dimension_numbers<...>` attribute. When omitted the /// canonicalizer derives one. Build with `mlir.Attribute.parse`. @@ -398,12 +399,14 @@ pub const SortOpts = struct { pub const SemSignalOpts = struct { device_id: ?Value = null, core_id: ?Value = null, + subcore_id: ?Value = null, }; pub const EnqueueDmaOpts = struct { source_semaphore: ?Value = null, device_id: ?Value = null, core_id: ?Value = null, + subcore_id: ?Value = null, priority: i32 = 0, strict_ordering: bool = false, }; @@ -1594,6 +1597,7 @@ pub const Builder = struct { .{ .transpose_lhs = opts.transpose_lhs, .transpose_rhs = opts.transpose_rhs, + .transpose_lhs_hint = opts.transpose_lhs_hint, .precision = opts.precision, .dimension_numbers = opts.dimension_numbers, }, @@ -1763,8 +1767,8 @@ pub const Builder = struct { } pub fn rotate(self: *Builder, value: Value, opts: RotateOpts) Value { - return self.emit(tpu.rotate(self.ctx, value.inner, .{ - .amount = opts.amount, + const amount = self.lift(@as(i32, opts.amount)); + return self.emit(tpu.dynamic_rotate(self.ctx, value.inner, amount.inner, .{ .dimension = opts.dimension, .stride = opts.stride, .stride_dimension = opts.stride_dimension, @@ -1794,7 +1798,8 @@ pub const Builder = struct { return self.emit(tpu.reduce_index(self.ctx, input.inner, axis, kind, ty, self.loc())); } - /// `tpu.scan(input, kind, mask?)` — same shape as input. + /// Scan the last dimension; a mask must be rank one with that dimension + /// as its length. The result has the same shape as the input. pub fn scan(self: *Builder, input: Value, kind: ReductionKind, mask: ?Value) Value { const m_inner: ?*const mlir.Value = if (mask) |m| m.inner else null; return self.emit(tpu.scan(self.ctx, input.inner, kind, m_inner, input.type_(), self.loc())); @@ -1945,9 +1950,11 @@ pub const Builder = struct { const a = if (@TypeOf(amount) == Value) amount else self.lift(amount); const dev: ?*const mlir.Value = if (opts.device_id) |d| d.inner else null; const core: ?*const mlir.Value = if (opts.core_id) |c_| c_.inner else null; + const subcore: ?*const mlir.Value = if (opts.subcore_id) |sc| sc.inner else null; _ = tpu.sem_signal(self.ctx, semaphore.inner, a.inner, .{ .device_id = dev, .core_id = core, + .subcore_id = subcore, }, self.loc()).appendTo(self.currentBlock()); } @@ -1959,10 +1966,12 @@ pub const Builder = struct { const src_sem: ?*const mlir.Value = if (opts.source_semaphore) |s| s.inner else null; const dev: ?*const mlir.Value = if (opts.device_id) |d| d.inner else null; const core: ?*const mlir.Value = if (opts.core_id) |c_| c_.inner else null; + const subcore: ?*const mlir.Value = if (opts.subcore_id) |sc| sc.inner else null; _ = tpu.enqueue_dma(self.ctx, source.inner, target.inner, target_semaphore.inner, .{ .source_semaphore = src_sem, .device_id = dev, .core_id = core, + .subcore_id = subcore, .priority = opts.priority, .strict_ordering = opts.strict_ordering, }, self.loc()).appendTo(self.currentBlock()); @@ -2330,3 +2339,22 @@ pub fn WhileScope(comptime N: usize, comptime M: usize) type { test { std.testing.refAllDecls(@This()); } + +test "static rotate uses the current dynamic rotate operation" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + inline for (dialects_needed) |dialect| registry.registerDialect(dialect); + mlir.registerFuncExtensions(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + var b = try Builder.init(std.testing.allocator, ctx, "rotate", &.{}, &.{}); + defer b.deinit(); + const input = b.zeros(&.{ 8, 128 }, .f32); + const rotated = b.rotate(input, .{ .amount = 3, .dimension = 1 }); + try std.testing.expectEqualStrings("tpu.dynamic_rotate", rotated.inner.owner().name()); + const amount: Value = .{ .inner = rotated.inner.owner().operand(1), .kernel = &b }; + try std.testing.expectEqual(@as(?i64, 3), amount.asConstantInt()); + const ir = try b.finish(&.{}); + defer std.testing.allocator.free(ir); +} diff --git a/mlir/dialects/cuda_tile/cuda_tile.zig b/mlir/dialects/cuda_tile/cuda_tile.zig index ff86836a..8e011aa8 100644 --- a/mlir/dialects/cuda_tile/cuda_tile.zig +++ b/mlir/dialects/cuda_tile/cuda_tile.zig @@ -647,7 +647,7 @@ pub fn minf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, } pub fn pow(ctx: *mlir.Context, source: *const mlir.Value, exponent: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return binary(ctx, "pow", source, exponent, .empty, location); + return binary(ctx, "fpowf", source, exponent, .empty, location); } pub fn atan2(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { @@ -1278,6 +1278,9 @@ pub fn load_view_tko(ctx: *mlir.Context, view: *const mlir.Value, index: []const segs.add(index); segs.add(tok.slice()); var attrs = memAttrs(ctx, opts); + // Preserve bounds checks unless the caller has proved a coordinate in bounds. + const inbounds = [_]i32{0} ** mlir.ShapedType.MAX_RANK; + attrs.appendAssumeCapacity(.named(ctx, "inbounds", .denseArray(ctx, .bool, inbounds[0..index.len]))); attrs.appendAssumeCapacity(segs.attr(ctx)); return mlir.Operation.make(ctx, opName("load_view_tko"), .{ .operands = .{ .flat = segs.values.constSlice() }, @@ -1296,6 +1299,8 @@ pub fn store_view_tko(ctx: *mlir.Context, tile: *const mlir.Value, view: *const segs.add(index); segs.add(tok.slice()); var attrs = memAttrs(ctx, opts); + const inbounds = [_]i32{0} ** mlir.ShapedType.MAX_RANK; + attrs.appendAssumeCapacity(.named(ctx, "inbounds", .denseArray(ctx, .bool, inbounds[0..index.len]))); attrs.appendAssumeCapacity(segs.attr(ctx)); return mlir.Operation.make(ctx, opName("store_view_tko"), .{ .operands = .{ .flat = segs.values.constSlice() }, @@ -1637,6 +1642,7 @@ test "views, control flow and mma verify" { const body = mlir.Block.init(&.{ i32_tile, acc_ty }, &.{ loc, loc }); const at = load_view_tko(ctx, va.result(0), &.{ bid.result(0), body.argument(0) }, .{}, loc).appendTo(body); const bt = load_view_tko(ctx, vb.result(0), &.{ body.argument(0), bid.result(1) }, .{}, loc).appendTo(body); + try std.testing.expect(at.attributeByName("inbounds").?.eql(.denseArray(ctx, .bool, &.{ 0, 0 }))); const acc = mmaf(ctx, at.result(0), bt.result(0), body.argument(1), false, loc).appendTo(body); _ = continue_(ctx, &.{acc.result(0)}, loc).appendTo(body); const loop_op = for_(ctx, zero.result(0), one.result(0), one.result(0), &.{acc0.result(0)}, body, false, loc).appendTo(block); @@ -1649,7 +1655,8 @@ test "views, control flow and mma verify" { _ = yield(ctx, &.{acc0.result(0)}, loc).appendTo(else_block); const if_op = if_(ctx, cond.result(0), &.{acc_ty}, then_block, else_block, loc).appendTo(block); - _ = store_view_tko(ctx, if_op.result(0), vc.result(0), &.{ bid.result(0), bid.result(1) }, .{}, loc).appendTo(block); + const store = store_view_tko(ctx, if_op.result(0), vc.result(0), &.{ bid.result(0), bid.result(1) }, .{}, loc).appendTo(block); + try std.testing.expect(store.attributeByName("inbounds").?.eql(.denseArray(ctx, .bool, &.{ 0, 0 }))); _ = return_(ctx, &.{}, loc).appendTo(block); const ct_body = mlir.Block.init(&.{}, &.{}); diff --git a/mlir/dialects/fly/fly.zig b/mlir/dialects/fly/fly.zig index 6083f308..4f4e9c07 100644 --- a/mlir/dialects/fly/fly.zig +++ b/mlir/dialects/fly/fly.zig @@ -216,7 +216,9 @@ pub const GemmOpts = struct { }; pub fn gemm(ctx: *mlir.Context, atom: *const mlir.Value, d: *const mlir.Value, a: *const mlir.Value, b: *const mlir.Value, cc: *const mlir.Value, opts: GemmOpts, location: *const mlir.Location) *mlir.Operation { - var at: Attrs = .empty; + var at = attrs(&.{.named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ + 1, 1, 1, 1, 1, @intFromBool(opts.traversal_layout != null), + }))}); if (opts.traversal_order) |o| { const attr = attributes.gemmTraversalOrderAttr(ctx, o) catch @panic("fly.gemm: invalid traversal order"); at.appendAssumeCapacity(.named(ctx, "traversalOrder", attr)); @@ -378,7 +380,7 @@ test "fly enum and gpu attributes" { defer ctx.deinit(); _ = try attributes.mmaOperandAttr(ctx, .a); _ = try attributes.gemmTraversalOrderAttr(ctx, .kmn); - _ = parseAttr(ctx, "#gpu"); + _ = parseAttr(ctx, "#gpu>"); } test "inferred fly ops compute layout algebra" { diff --git a/mlir/dialects/gpu.zig b/mlir/dialects/gpu.zig index 35538ab2..3d690a83 100644 --- a/mlir/dialects/gpu.zig +++ b/mlir/dialects/gpu.zig @@ -52,7 +52,7 @@ pub fn return_(ctx: *mlir.Context, values: []const *const mlir.Value, location: fn dimAttr(ctx: *mlir.Context, dim: Dim) *const mlir.Attribute { return switch (dim) { - inline else => |d| mlir.Attribute.parse(ctx, "#gpu") catch unreachable, + inline else => |d| mlir.Attribute.parse(ctx, "#gpu>") catch unreachable, }; } diff --git a/mlir/dialects/mosaic_tpu/BUILD.bazel b/mlir/dialects/mosaic_tpu/BUILD.bazel index 5ae15776..ac3a12b0 100644 --- a/mlir/dialects/mosaic_tpu/BUILD.bazel +++ b/mlir/dialects/mosaic_tpu/BUILD.bazel @@ -9,7 +9,7 @@ zig_library( "//mlir", "//mlir:c", "//stdx", - "@mosaic_tpu//:tpu_dialect_capi", + "@xla//xla/mosaic:tpu_dialect_capi", ], ) diff --git a/mlir/dialects/mosaic_tpu/mosaic_tpu.zig b/mlir/dialects/mosaic_tpu/mosaic_tpu.zig index 5895a548..03496d9e 100644 --- a/mlir/dialects/mosaic_tpu/mosaic_tpu.zig +++ b/mlir/dialects/mosaic_tpu/mosaic_tpu.zig @@ -4,10 +4,10 @@ const c = @import("c"); const mlir = @import("mlir"); const stdx = @import("stdx"); -/// Latest Mosaic-serde format version. Pallas's `tpu_custom_call.py:CustomCallBackendConfig.to_json` -/// surfaces this through `serialization_format`. Bumped in lockstep with -/// `jaxlib/mosaic/dialect/tpu/transforms/serde.h:kVersion`. -pub const SERDE_VERSION: i32 = 11; +/// Mosaic IR version stored as `stable_mosaic.version`, matching +/// `xla/mosaic/dialect/tpu/transforms/serde.h:kVersion`. This is distinct from +/// the custom-call JSON `serialization_format`, which remains 1. +pub const SERDE_VERSION: i32 = 18; /// Register the `mosaic-serde` pass so `PassManager.parse` can pick it up by /// name. Idempotent — safe to call repeatedly. Required before running the @@ -65,6 +65,15 @@ pub const ReductionKind = enum { sum, max, min, + maxf, + minf, + maxsi, + minsi, + maxui, + minui, + arg_max, + arg_min, + find_first_set, @"and", @"or", xor, @@ -75,6 +84,18 @@ pub const ReductionKind = enum { return mlir.Attribute.parse(ctx, text) catch std.debug.panic("failed to parse tpu.reduction_kind '{s}'", .{text}); } + + fn forInput(self: ReductionKind, input: *const mlir.Value, unsigned_integer: bool) ReductionKind { + if (self != .max and self != .min) return self; + const element_type = input.type_().isA(mlir.ShapedType).?.elementType(); + if (element_type.isFloat()) return if (self == .max) .maxf else .minf; + // Match Mosaic's serde upgrade: scans used unsigned integer extrema, + // while all-reduce used signed integer extrema. + return if (unsigned_integer) + (if (self == .max) .maxui else .minui) + else + (if (self == .max) .maxsi else .minsi); + } }; /// `tpu.contract_precision` — matmul precision setting. @@ -220,7 +241,7 @@ pub fn all_reduce( .results = .{ .flat = &.{result_type} }, .attributes = &.{ .named(ctx, "dim", .int(ctx, .i64, dim)), - .named(ctx, "kind", kind.attribute(ctx)), + .named(ctx, "kind", kind.forInput(input, false).attribute(ctx)), }, .location = location, }); @@ -245,6 +266,7 @@ pub fn reduce_index( }); } +/// Scan the last dimension; an optional rank-one mask spans that dimension. pub fn scan( ctx: *mlir.Context, input: *const mlir.Value, @@ -261,7 +283,8 @@ pub fn scan( .operands = .{ .flat = operands_buf.constSlice() }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "kind", kind.attribute(ctx)), + .named(ctx, "kind", kind.forInput(input, true).attribute(ctx)), + .named(ctx, "dimension", .int(ctx, .i64, @as(i64, @intCast(input.type_().isA(mlir.ShapedType).?.rank())) - 1)), }, .location = location, }); @@ -486,6 +509,7 @@ pub const MatmulOpts = struct { /// Deprecated when `dimension_numbers` is provided. transpose_lhs: bool = false, transpose_rhs: bool = false, + transpose_lhs_hint: bool = false, /// Optional precision; pass null to omit. precision: ?ContractPrecision = null, /// Optional `#tpu.dot_dimension_numbers<...>` attribute. When omitted the @@ -502,10 +526,11 @@ pub fn matmul( result_type: *const mlir.Type, location: *const mlir.Location, ) *mlir.Operation { - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; attrs.appendSliceAssumeCapacity(&.{ .named(ctx, "transpose_lhs", .boolean(ctx, opts.transpose_lhs)), .named(ctx, "transpose_rhs", .boolean(ctx, opts.transpose_rhs)), + .named(ctx, "transpose_lhs_hint", .boolean(ctx, opts.transpose_lhs_hint)), }); if (opts.precision) |p| { attrs.appendAssumeCapacity(.named(ctx, "precision", p.attribute(ctx))); @@ -588,7 +613,6 @@ pub const truncf = castOp("tpu.truncf").call; pub const bitcast = castOp("tpu.bitcast").call; pub const bitcast_vreg = castOp("tpu.bitcast_vreg").call; pub const mask_cast = castOp("tpu.mask_cast").call; -pub const relayout = castOp("tpu.relayout").call; // ============================================================================= // Shape — reshape / repeat / concatenate / transpose / broadcast_in_sublanes @@ -677,29 +701,28 @@ pub fn broadcast_in_sublanes( }); } -pub const RotateOpts = struct { - amount: i32, +pub const DynamicRotateOpts = struct { dimension: i32, stride: ?i32 = null, stride_dimension: ?i32 = null, }; -pub fn rotate( +pub fn dynamic_rotate( ctx: *mlir.Context, value: *const mlir.Value, - opts: RotateOpts, + amount: *const mlir.Value, + opts: DynamicRotateOpts, location: *const mlir.Location, ) *mlir.Operation { - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; attrs.appendSliceAssumeCapacity(&.{ - .named(ctx, "amount", .int(ctx, .i32, opts.amount)), - .named(ctx, "dimension", .int(ctx, .i32, opts.dimension)), + .named(ctx, "dimension", .int(ctx, .si32, opts.dimension)), }); - if (opts.stride) |s| attrs.appendAssumeCapacity(.named(ctx, "stride", .int(ctx, .i32, s))); - if (opts.stride_dimension) |sd| attrs.appendAssumeCapacity(.named(ctx, "stride_dimension", .int(ctx, .i32, sd))); + if (opts.stride) |s| attrs.appendAssumeCapacity(.named(ctx, "stride", .int(ctx, .si32, s))); + if (opts.stride_dimension) |sd| attrs.appendAssumeCapacity(.named(ctx, "stride_dimension", .int(ctx, .si32, sd))); - return mlir.Operation.make(ctx, "tpu.rotate", .{ - .operands = .{ .flat = &.{value} }, + return mlir.Operation.make(ctx, "tpu.dynamic_rotate", .{ + .operands = .{ .flat = &.{ value, amount } }, .results = .{ .flat = &.{value.type_()} }, .attributes = attrs.constSlice(), .location = location, @@ -788,6 +811,7 @@ pub fn reinterpret_cast( return mlir.Operation.make(ctx, "tpu.reinterpret_cast", .{ .operands = .{ .flat = &.{mem_ref} }, .results = .{ .flat = &.{result_type} }, + .attributes = &.{.named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ 1, 0, 0, 0 }))}, .location = location, }); } @@ -895,6 +919,7 @@ pub fn sem_wait( pub const SemSignalOpts = struct { device_id: ?*const mlir.Value = null, core_id: ?*const mlir.Value = null, + subcore_id: ?*const mlir.Value = null, }; pub fn sem_signal( @@ -904,7 +929,7 @@ pub fn sem_signal( opts: SemSignalOpts, location: *const mlir.Location, ) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 4) = .empty; + var operands_buf: stdx.BoundedArray(*const mlir.Value, 5) = .empty; operands_buf.appendSliceAssumeCapacity(&.{ semaphore, amount }); const dev_len: i32 = if (opts.device_id) |d| blk: { operands_buf.appendAssumeCapacity(d); @@ -915,7 +940,12 @@ pub fn sem_signal( break :blk 1; } else 0; - const seg_sizes = [4]i32{ 1, 1, dev_len, core_len }; + const subcore_len: i32 = if (opts.subcore_id) |subcore| blk: { + operands_buf.appendAssumeCapacity(subcore); + break :blk 1; + } else 0; + + const seg_sizes = [5]i32{ 1, 1, dev_len, core_len, subcore_len }; return mlir.Operation.make(ctx, "tpu.sem_signal", .{ .operands = .{ .flat = operands_buf.constSlice() }, .attributes = &.{ @@ -940,6 +970,7 @@ pub const EnqueueDmaOpts = struct { source_semaphore: ?*const mlir.Value = null, device_id: ?*const mlir.Value = null, core_id: ?*const mlir.Value = null, + subcore_id: ?*const mlir.Value = null, priority: i32 = 0, strict_ordering: bool = false, }; @@ -952,7 +983,7 @@ pub fn enqueue_dma( opts: EnqueueDmaOpts, location: *const mlir.Location, ) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 6) = .empty; + var operands_buf: stdx.BoundedArray(*const mlir.Value, 7) = .empty; operands_buf.appendAssumeCapacity(source); const src_sem_len: i32 = if (opts.source_semaphore) |s| blk: { operands_buf.appendAssumeCapacity(s); @@ -969,8 +1000,13 @@ pub fn enqueue_dma( break :blk 1; } else 0; - // Source / source_semaphore? / target / target_semaphore / device_id? / core_id? - const seg_sizes = [6]i32{ 1, src_sem_len, 1, 1, dev_len, core_len }; + // Source / source_semaphore? / target / target_semaphore / device_id? / core_id? / subcore_id? + const subcore_len: i32 = if (opts.subcore_id) |subcore| blk: { + operands_buf.appendAssumeCapacity(subcore); + break :blk 1; + } else 0; + + const seg_sizes = [7]i32{ 1, src_sem_len, 1, 1, dev_len, core_len, subcore_len }; return mlir.Operation.make(ctx, "tpu.enqueue_dma", .{ .operands = .{ .flat = operands_buf.constSlice() }, .attributes = &.{ @@ -1176,3 +1212,108 @@ fn denseBoolArrayAttribute(ctx: *mlir.Context, values: []const bool) *const mlir test { std.testing.refAllDecls(@This()); } + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + registry.registerDialect("tpu"); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectSegments(op: *const mlir.Operation, expected: []const i32) !void { + const segments = op.attributeByName("operandSegmentSizes").?.isA(mlir.DenseArrayAttribute(.i32)).?; + try std.testing.expectEqual(expected.len, segments.numElements()); + for (expected, 0..) |size, i| try std.testing.expectEqual(size, segments.element(i)); +} + +test "DMA and semaphore wrappers satisfy current operand groups" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc = mlir.Location.unknown(ctx); + const args = mlir.Block.init(&.{}, &.{}); + defer args.deinit(); + const source = args.addArgument(try mlir.Type.parse(ctx, "memref<8x128xbf16, #tpu.memory_space>"), loc); + const target = args.addArgument(try mlir.Type.parse(ctx, "memref<8x128xbf16, #tpu.memory_space>"), loc); + const dma_sem = args.addArgument(try mlir.Type.parse(ctx, "memref>"), loc); + const sem = args.addArgument(try mlir.Type.parse(ctx, "memref>"), loc); + const amount = args.addArgument(.int(ctx, .i32), loc); + + const dma = enqueue_dma(ctx, source, target, dma_sem, .{}, loc); + defer dma.deinit(); + try expectSegments(dma, &.{ 1, 0, 1, 1, 0, 0, 0 }); + try std.testing.expect(dma.verify()); + const signal = sem_signal(ctx, sem, amount, .{}, loc); + defer signal.deinit(); + try expectSegments(signal, &.{ 1, 1, 0, 0, 0 }); + try std.testing.expect(signal.verify()); + + const sc_sem = args.addArgument(try mlir.Type.parse(ctx, "memref>"), loc); + const sc_dma_sem = args.addArgument(try mlir.Type.parse(ctx, "memref>"), loc); + const core = args.addArgument(.int(ctx, .i32), loc); + const subcore = args.addArgument(.int(ctx, .i32), loc); + const remote_signal = sem_signal(ctx, sc_sem, amount, .{ .core_id = core, .subcore_id = subcore }, loc); + defer remote_signal.deinit(); + try expectSegments(remote_signal, &.{ 1, 1, 0, 1, 1 }); + try std.testing.expect(remote_signal.operand(3).eql(subcore)); + const remote_dma = enqueue_dma(ctx, source, target, sc_dma_sem, .{ + .source_semaphore = dma_sem, + .core_id = core, + .subcore_id = subcore, + }, loc); + defer remote_dma.deinit(); + try expectSegments(remote_dma, &.{ 1, 1, 1, 1, 0, 1, 1 }); + try std.testing.expect(remote_dma.operand(5).eql(subcore)); +} + +test "matmul and reinterpret cast satisfy current required properties" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc = mlir.Location.unknown(ctx); + const args = mlir.Block.init(&.{}, &.{}); + defer args.deinit(); + const vector_type = mlir.Type.vector(&.{ 8, 8 }, .float(ctx, .f32)); + const lhs = args.addArgument(vector_type, loc); + const rhs = args.addArgument(vector_type, loc); + const acc = args.addArgument(vector_type, loc); + const product = matmul(ctx, lhs, rhs, acc, .{}, vector_type, loc); + defer product.deinit(); + try std.testing.expect(product.verify()); + try std.testing.expect(!product.attributeByName("transpose_lhs_hint").?.isA(mlir.BoolAttribute).?.value()); + + const ref_type = try mlir.Type.parse(ctx, "memref<8x8xf32, #tpu.memory_space>"); + const ref = args.addArgument(ref_type, loc); + const cast = reinterpret_cast(ctx, ref, ref_type, loc); + defer cast.deinit(); + try expectSegments(cast, &.{ 1, 0, 0, 0 }); + try std.testing.expect(cast.verify()); +} + +test "legacy reduction extrema and scan axis preserve their semantics" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc = mlir.Location.unknown(ctx); + const args = mlir.Block.init(&.{}, &.{}); + defer args.deinit(); + const float_type = mlir.Type.vector(&.{ 2, 8 }, .float(ctx, .f32)); + const int_type = mlir.Type.vector(&.{ 2, 8 }, .int(ctx, .i32)); + const floats = args.addArgument(float_type, loc); + const integers = args.addArgument(int_type, loc); + const mask = args.addArgument(mlir.Type.vector(&.{8}, .int(ctx, .i1)), loc); + const float_min = all_reduce(ctx, floats, 1, .min, float_type, loc); + defer float_min.deinit(); + const int_max = all_reduce(ctx, integers, 1, .max, int_type, loc); + defer int_max.deinit(); + try std.testing.expect(float_min.attributeByName("kind").?.eql(try mlir.Attribute.parse(ctx, "#tpu.reduction_kind"))); + try std.testing.expect(int_max.attributeByName("kind").?.eql(try mlir.Attribute.parse(ctx, "#tpu.reduction_kind"))); + + const int_scan = scan(ctx, integers, .max, mask, int_type, loc); + defer int_scan.deinit(); + try std.testing.expect(int_scan.verify()); + try std.testing.expect(int_scan.attributeByName("kind").?.eql(try mlir.Attribute.parse(ctx, "#tpu.reduction_kind"))); + try std.testing.expectEqual(@as(i64, 1), int_scan.attributeByName("dimension").?.isA(mlir.IntegerAttribute).?.value(i64)); + const float_scan = scan(ctx, floats, .min, null, float_type, loc); + defer float_scan.deinit(); + try std.testing.expect(float_scan.attributeByName("kind").?.eql(try mlir.Attribute.parse(ctx, "#tpu.reduction_kind"))); +} diff --git a/mlir/dialects/shardy.zig b/mlir/dialects/shardy.zig index 376820a5..1e2307fa 100644 --- a/mlir/dialects/shardy.zig +++ b/mlir/dialects/shardy.zig @@ -212,6 +212,7 @@ pub const TensorShardingAttribute = opaque { replicated.ptr, @intCast(unreduced.len), unreduced.ptr, + 0, ).ptr); } diff --git a/mlir/dialects/ttir/ttir_capi.cc b/mlir/dialects/ttir/ttir_capi.cc index ed7d533e..ffd92990 100644 --- a/mlir/dialects/ttir/ttir_capi.cc +++ b/mlir/dialects/ttir/ttir_capi.cc @@ -17,7 +17,8 @@ using mlir::isa; extern "C" { MlirType mlirTritonPointerTypeGet(MlirType pointee, int32_t address_space) { - return wrap(mlir::triton::PointerType::get(unwrap(pointee), address_space)); + return wrap(mlir::triton::PointerType::get( + unwrap(pointee), static_cast(address_space))); } MlirType mlirTritonPointerTypeGetPointee(MlirType ptr) { @@ -25,7 +26,8 @@ MlirType mlirTritonPointerTypeGetPointee(MlirType ptr) { } int32_t mlirTritonPointerTypeGetAddressSpace(MlirType ptr) { - return cast(unwrap(ptr)).getAddressSpace(); + return static_cast( + cast(unwrap(ptr)).getAddressSpace()); } bool mlirTritonTypeIsAPointer(MlirType t) { diff --git a/pjrt/BUILD.bazel b/pjrt/BUILD.bazel index 3d00cde7..5370afdc 100644 --- a/pjrt/BUILD.bazel +++ b/pjrt/BUILD.bazel @@ -2,9 +2,9 @@ load("@rules_zig//zig:defs.bzl", "zig_library") zig_library( name = "pjrt", - tags = ["manual"], srcs = ["ffi.zig"], main = "pjrt.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ "@xla//xla/ffi/api:c_api", @@ -20,6 +20,5 @@ zig_library( "@xla//xla/pjrt/c:pjrt_c_api_profiler_extension_hdrs", "@xla//xla/pjrt/c:pjrt_c_api_raw_buffer_extension_hdrs", "@xla//xla/pjrt/c:pjrt_c_api_stream_extension_hdrs", - "@xla//xla/pjrt/c:pjrt_c_api_triton_extension_hdrs", ], ) diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index 0c160321..7c7ed2f6 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -27,7 +27,7 @@ pub const ApiVersion = extern struct { minor_version: i32, }; -pub const ExtensionType = enum(c.XLA_FFI_Extension_Type) { +pub const ExtensionType = enum(c.XLA_FFI_InternalExtension_Type) { metadata = c.XLA_FFI_Extension_Metadata, }; @@ -152,17 +152,17 @@ pub const ExecutionStage = enum(c.XLA_FFI_ExecutionStage) { }; pub const ExecutionContext = opaque { - pub const inner = TransmuteMixin(ExecutionContext, c.XLA_FFI_ExecutionContext).to; + pub const inner = TransmuteMixin(ExecutionContext, c.XLA_FFI_InvokeContext).to; pub fn getContext(self: *const ExecutionContext, type_id: TypeId, api: *const Api) pjrt.ApiError!*anyopaque { - var ret: c.XLA_FFI_ExecutionContext_Get_Args = .{ - .struct_size = pjrt.pjrtStructSize(c.XLA_FFI_ExecutionContext_Get_Args), + var ret: c.XLA_FFI_InvokeContext_Get_Args = .{ + .struct_size = pjrt.meta.structSize(c.XLA_FFI_InvokeContext_Get_Args), .extension_start = api.inner().extension_start, .ctx = @ptrCast(@constCast(self)), .type_id = @constCast(&type_id), - .data = undefined, // set by XLA_FFI_ExecutionContext_Get. + .data = undefined, // set by XLA_FFI_InvokeContext_Get. }; - const maybe_err = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); + const maybe_err = api.inner().XLA_FFI_InvokeContext_Get.?(&ret); if (maybe_err) |ffi_error| { const err = Error.fromInner(ffi_error); @@ -257,7 +257,7 @@ pub const DataType = enum(c.XLA_FFI_DataType) { pub const Buffer = extern struct { struct_size: usize, - extension_start: ?*c.XLA_FFI_Extension_Base, + extension_start: ?*c.XLA_FFI_InternalExtension, dtype: DataType, data: [*]u8, rank: u64, @@ -274,7 +274,7 @@ pub const Buffer = extern struct { pub const Args = extern struct { struct_size: usize, - extension_start: ?*const c.XLA_FFI_Extension_Base, + extension_start: ?*const c.XLA_FFI_InternalExtension, len: u64, types: [*]const Type, ptr: [*]*const Buffer, @@ -290,7 +290,7 @@ pub const Args = extern struct { pub const Rets = extern struct { struct_size: usize, - extension_start: ?*const c.XLA_FFI_Extension_Base, + extension_start: ?*const c.XLA_FFI_InternalExtension, len: u64, types: [*]const Type, ptr: [*]*const Buffer, @@ -461,10 +461,10 @@ pub const Error = opaque { } pub fn getMessage(err: *Error, api: *const Api) [:0]const u8 { - var ret: pjrt.meta.Struct(c.XLA_FFI_Error_GetMessage_Args) = .{ + var ret: pjrt.meta.Struct(c.XLA_FFI_Error_GetDetails_Args) = .{ .@"error" = err.inner(), }; - api.inner().XLA_FFI_Error_GetMessage.?(@ptrCast(&ret)); + api.inner().XLA_FFI_Error_GetDetails.?(@ptrCast(&ret)); return std.mem.span(ret.message); } }; diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index bd58386f..e6ae0acf 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -194,7 +194,6 @@ pub const Api = struct { layouts = c.PJRT_Extension_Type_Layouts, ffi = c.PJRT_Extension_Type_FFI, memory_descriptions = c.PJRT_Extension_Type_MemoryDescriptions, - triton = c.PJRT_Extension_Type_Triton, raw_buffer = c.PJRT_Extension_Type_RawBuffer, phase_compile = c.PJRT_Extension_Type_PhaseCompile, unknown = c.PJRT_Extension_Type_Unknown, @@ -207,7 +206,6 @@ pub const Api = struct { layouts: *const c.PJRT_Layouts_Extension, ffi: *const c.PJRT_FFI_Extension, memory_descriptions: *const c.PJRT_MemoryDescriptions_Extension, - triton: *const c.PJRT_Triton_Extension, raw_buffer: *const c.PJRT_RawBuffer_Extension, phase_compile: *const c.PJRT_PhaseCompile_Extension, unknown: *const c.PJRT_Extension_Base, diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index b3c63138..94670793 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -42,22 +42,22 @@ def _cpu_plugin_impl(mctx): http_archive( name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "32432b7e4b9894a1de2e70e2f064093df0996679587adee2ebb1c930682d4251", - url = "https://mirror.zml.ai/plugins/202609301537.129.1.ac4bc67c46bf/zml-cpu-linux-amd64.tar.zst", + sha256 = "5051c6719809f1dd86a83665b6bb65a4c0c8ec3be4613c8652eecf4856ab1d27", + url = "https://mirror.zml.ai/plugins/202610071230.162.1.1e1e094fa75c/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "be60fe17befb5809f943346490f10c171c039c1893ca87ec624bc8649ea4740d", - url = "https://mirror.zml.ai/plugins/202609301537.129.1.ac4bc67c46bf/zml-cpu-darwin-amd64.tar.zst", + sha256 = "d2ea2bce16aba06cc3f75725b1cd8c2fc72169a8ec8cb72c21637161dc79a1c0", + url = "https://mirror.zml.ai/plugins/202610071230.162.1.1e1e094fa75c/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "79e5212f139a8214e3ea88fa31c85d24401af21441a3324f06269e66ef15b7be", - url = "https://mirror.zml.ai/plugins/202609301537.129.1.ac4bc67c46bf/zml-cpu-darwin-arm64.tar.zst", + sha256 = "7e614948453f9692ca76d7014ffae9f25d9fe80836088ca92a5f764b26a6fe43", + url = "https://mirror.zml.ai/plugins/202610071230.162.1.1e1e094fa75c/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 449d2bc4..64e733d7 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -224,12 +224,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "fd84f28d7697ff7c364f1a67c406f7436d123cf6ebb4039d968548d1f13ed70c", - "url": "https://mirror.zml.ai/plugins/202609301537.129.1.ac4bc67c46bf/zml-cuda-linux-amd64.tar.zst", + "sha256": "b0bc93cd61025fe4b1da8cba1655c0e4e731a96b3fdb8d1dcd04513b969af4b7", + "url": "https://mirror.zml.ai/plugins/202610071230.162.1.1e1e094fa75c/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "df853190c31465b67bd1eedb916b955d4395880fd5752b616bc8885799bd5122", - "url": "https://mirror.zml.ai/plugins/202609301537.129.1.ac4bc67c46bf/zml-cuda-linux-arm64.tar.zst", + "sha256": "a2683ab471fc349cf6315085bae60bb5b89c9abba3f043e54b62b48361f03cb4", + "url": "https://mirror.zml.ai/plugins/202610071230.162.1.1e1e094fa75c/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index aefcbe71..0d205070 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "d3aa3df41f14622624a9ed53b9574d5c8a7947543003d1b6c8625ec6ce683c53", - url = "https://mirror.zml.ai/plugins/202609301537.129.1.ac4bc67c46bf/zml-metal-darwin-arm64.tar.zst", + sha256 = "a880a201b274fae35596b1b4cea1a79862dfea29a95eb22439b3b5c8f1f86767", + url = "https://mirror.zml.ai/plugins/202610071230.162.1.1e1e094fa75c/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 77cdf80b..9b7cc4d1 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202609301537.129.1.ac4bc67c46bf/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "483a34bdb6bb1844af0afaaca87b326d8c7f8bf85352c73ccdca5e06ea40a534" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610071230.162.1.1e1e094fa75c/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "6afe9ca497e04ecb07a3f56645f0c64170770c39c9216316d51de3afe13103ae" def _rocm_package_name(name): return name + _ROCM_VERSION diff --git a/platforms/tpu/tpu.bzl b/platforms/tpu/tpu.bzl index 55e2719f..2fbc2a9b 100644 --- a/platforms/tpu/tpu.bzl +++ b/platforms/tpu/tpu.bzl @@ -4,9 +4,9 @@ def _tpu_impl(mctx): # https://storage.googleapis.com/jax-releases/libtpu_releases.html http_archive( name = "libzml_tpu", - url = "https://storage.googleapis.com/libtpu-nightly-releases/wheels/libtpu/libtpu-0.0.42.dev20260613+nightly-cp314-cp314t-manylinux_2_31_x86_64.whl", + url = "https://storage.googleapis.com/libtpu-nightly-releases/wheels/libtpu/libtpu-0.0.50.dev20261005+nightly-cp314-cp314t-manylinux_2_31_x86_64.whl", type = "zip", - sha256 = "1d0bbb1608154bda6944902abd6f56f956638539d7503db4894927e21a41baed", + sha256 = "a1e8615a33f60e1bfce15ec9d570642f882a8bec1d46763531e3b831edb2bb42", build_file = "libzml_tpu.BUILD.bazel", ) return mctx.extension_metadata( diff --git a/third_party/cuda_tile/cuda_tile.bazel b/third_party/cuda_tile/cuda_tile.bazel index b19b7a35..bfa768e9 100644 --- a/third_party/cuda_tile/cuda_tile.bazel +++ b/third_party/cuda_tile/cuda_tile.bazel @@ -158,6 +158,35 @@ gentbl_cc_library( deps = [":CudaTileTdFiles"], ) +gentbl_cc_library( + name = "CudaTileAttrCodeGenIncGen", + tbl_outs = [ + ( + ["-gen-enum-attr-list"], + "include/cuda_tile/Dialect/CudaTile/IR/CudaTileEnumAttrs.inc", + ), + ( + ["-gen-gpu-arch-def"], + "include/cuda_tile/Dialect/CudaTile/IR/GpuArchitectures.inc", + ), + ( + ["-gen-gpu-arch-constants"], + "include/cuda_tile/Dialect/CudaTile/IR/GpuArchConstants.inc", + ), + ( + ["-gen-hint-key-accessors"], + "include/cuda_tile/Dialect/CudaTile/IR/HintKeyAccessors.inc", + ), + ( + ["-gen-hint-key-impl"], + "include/cuda_tile/Dialect/CudaTile/IR/HintKeyImpl.inc", + ), + ], + tblgen = ":cuda-tile-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/AttrDefs.td", + deps = [":CudaTileTdFiles"], +) + gentbl_cc_library( name = "CudaTileRemarksIncGen", tbl_outs = [ @@ -221,12 +250,15 @@ cc_library( ], visibility = ["//visibility:public"], deps = [ + ":CudaTileAttrCodeGenIncGen", ":CudaTileAttrDefsIncGen", ":CudaTileDialectIncGen", ":CudaTileInterfacesIncGen", ":CudaTileOpsCanonicalizationIncGen", ":CudaTileOpsIncGen", + ":CudaTileRemarksIncGen", ":CudaTileTypesIncGen", + "@llvm-project//llvm:Core", "@llvm-project//llvm:Support", "@llvm-project//mlir:ArithDialect", "@llvm-project//mlir:BytecodeOpInterface", @@ -278,6 +310,7 @@ cc_library( "@llvm-project//llvm:Support", "@llvm-project//mlir:IR", "@llvm-project//mlir:Pass", + "@llvm-project//mlir:SideEffectInterfaces", "@llvm-project//mlir:TransformUtils", ], ) @@ -365,7 +398,9 @@ cc_library( ":CudaTileBytecodeTypeIncGen", ":CudaTileDialect", "@llvm-project//llvm:Support", + "@llvm-project//mlir:FunctionInterfaces", "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", "@llvm-project//mlir:TranslateLib", ], ) diff --git a/third_party/cuda_tile/repo.bzl b/third_party/cuda_tile/repo.bzl index 59e8d6ab..79076c51 100644 --- a/third_party/cuda_tile/repo.bzl +++ b/third_party/cuda_tile/repo.bzl @@ -1,10 +1,10 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") -# v13.3.3 +# v13.4.0 with upstream LLVM compatibility fixes def repo(): git_repository( name = "cuda_tile", remote = "https://github.com/NVIDIA/cuda-tile.git", - commit = "af2417041cc939b87ef56d92cfdcf61737c5457e", + commit = "5b3fa4c0dd04d90edeb935a1887c20bf14a86ebe", build_file = "//third_party/cuda_tile:cuda_tile.bazel", ) diff --git a/third_party/flydsl/flydsl.bazel b/third_party/flydsl/flydsl.bazel index 0c06878c..7c8cb7e9 100644 --- a/third_party/flydsl/flydsl.bazel +++ b/third_party/flydsl/flydsl.bazel @@ -250,6 +250,19 @@ gentbl_cc_library( deps = [":td_files"], ) +gentbl_cc_library( + name = "fly_rocdl_passes_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/FlyROCDL/Transforms/Passes.h.inc": [ + "--gen-pass-decls", + "--name=FlyROCDL", + ], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/FlyROCDL/Transforms/Passes.td", + deps = [":td_files"], +) + cc_library( name = "FlyDialect", srcs = [ @@ -327,6 +340,9 @@ cc_library( "lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp", "lib/Dialect/FlyROCDL/CDNA4/CopyAtom.cpp", "lib/Dialect/FlyROCDL/CDNA4/MmaAtom.cpp", + "lib/Dialect/FlyROCDL/CDNA5/CopyAtom.cpp", + "lib/Dialect/FlyROCDL/CDNA5/TdmAtomBuilder.cpp", + "lib/Dialect/FlyROCDL/CDNA5/TdmGeometry.cpp", "lib/Dialect/FlyROCDL/Dialect.cpp", "lib/Dialect/FlyROCDL/GFX11/MmaAtom.cpp", "lib/Dialect/FlyROCDL/GFX120X/MmaAtom.cpp", @@ -334,6 +350,8 @@ cc_library( "lib/Dialect/FlyROCDL/GFX1250/MmaAtom.cpp", "lib/Dialect/FlyROCDL/GFX1250/MmaAtomScale.cpp", "lib/Dialect/FlyROCDL/Ops.cpp", + "lib/Dialect/FlyROCDL/Transforms/ClusterAttr.cpp", + "lib/Dialect/FlyROCDL/Transforms/ExpandOps.cpp", ], hdrs = glob([ "include/flydsl/Dialect/FlyROCDL/**/*.h", @@ -346,10 +364,12 @@ cc_library( ":fly_rocdl_attrs_inc_gen", ":fly_rocdl_dialect_inc_gen", ":fly_rocdl_ops_inc_gen", + ":fly_rocdl_passes_inc_gen", "@llvm-project//llvm:Support", "@llvm-project//mlir:ArithDialect", "@llvm-project//mlir:IR", "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:Pass", "@llvm-project//mlir:ROCDLDialect", "@llvm-project//mlir:SCFDialect", "@llvm-project//mlir:VectorDialect", @@ -358,7 +378,10 @@ cc_library( cc_library( name = "FlyToROCDL", - srcs = ["lib/Conversion/FlyToROCDL/FlyToROCDL.cpp"], + srcs = [ + "lib/Conversion/FlyToROCDL/FastMathOps.cpp", + "lib/Conversion/FlyToROCDL/FlyToROCDL.cpp", + ], hdrs = glob(["include/flydsl/Conversion/**/*.h"]), includes = ["include"], deps = [ @@ -368,13 +391,17 @@ cc_library( "@llvm-project//llvm:Support", "@llvm-project//mlir:ArithDialect", "@llvm-project//mlir:ArithUtils", + "@llvm-project//mlir:ControlFlowTransforms", "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:FuncTransforms", "@llvm-project//mlir:GPUDialect", "@llvm-project//mlir:IR", "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:MathDialect", "@llvm-project//mlir:Pass", "@llvm-project//mlir:ROCDLDialect", "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:SCFTransforms", "@llvm-project//mlir:Transforms", "@llvm-project//mlir:VectorDialect", ], diff --git a/third_party/flydsl/mlir_legacy_assembly.patch b/third_party/flydsl/mlir_legacy_assembly.patch new file mode 100644 index 00000000..0739e489 --- /dev/null +++ b/third_party/flydsl/mlir_legacy_assembly.patch @@ -0,0 +1,39 @@ +--- a/include/flydsl/Dialect/Fly/IR/FlyDialect.td ++++ b/include/flydsl/Dialect/Fly/IR/FlyDialect.td +@@ -19,6 +19,11 @@ + + let useDefaultTypePrinterParser = 1; + let useDefaultAttributePrinterParser = 1; ++ ++ // ZML custom-call inputs use Fly assembly printed by a separately pinned ++ // MLIR/Fly build. Keep accepting its inherent attributes in attr-dict until ++ // the producer and consumer can migrate to prop-dict together. ++ let useStrictPropertiesInAssemblyFormat = 0; + } + + class Fly_Type traits = []> +@@ -38,7 +43,10 @@ + } + + class Fly_EnumAttr traits = []> +- : EnumAttr; ++ : EnumAttr { ++ // Keep the Fly text ABI independent of MLIR's default enum delimiters. ++ let assemblyFormat = "$value"; ++} + + class Fly_Op traits = []> + : Op { +diff --git a/include/flydsl/Dialect/FlyROCDL/IR/Ops.td b/include/flydsl/Dialect/FlyROCDL/IR/Ops.td +index 2b745065..875d8a69 100644 +--- a/include/flydsl/Dialect/FlyROCDL/IR/Ops.td ++++ b/include/flydsl/Dialect/FlyROCDL/IR/Ops.td +@@ -44,7 +44,7 @@ class FlyROCDL_MakeTiledTdmAtomOp + let results = (outs AnyType:$atom, AnyType:$coordTensor); + + let assemblyFormat = [{ +- $tensor `,` $smemLayout `,` $tiler attr-dict `:` type(operands) ++ $tensor `,` $smemLayout `,` $tiler prop-dict attr-dict `:` type(operands) + }]; + } + diff --git a/third_party/flydsl/mlir_property_ref.patch b/third_party/flydsl/mlir_property_ref.patch index 3833fb78..145bcc1b 100644 --- a/third_party/flydsl/mlir_property_ref.patch +++ b/third_party/flydsl/mlir_property_ref.patch @@ -1,27 +1,16 @@ -diff --git a/lib/Dialect/Fly/IR/FlyOps.cpp b/lib/Dialect/Fly/IR/FlyOps.cpp ---- a/lib/Dialect/Fly/IR/FlyOps.cpp -+++ b/lib/Dialect/Fly/IR/FlyOps.cpp -@@ -192,7 +192,7 @@ - llvm::LogicalResult OP::inferReturnTypes( \ - mlir::MLIRContext *context, std::optional<::mlir::Location> location, \ - mlir::ValueRange operands, mlir::DictionaryAttr attributes, \ -- mlir::OpaqueProperties properties, mlir::RegionRange regions, \ -+ mlir::PropertyRef properties, mlir::RegionRange regions, \ - llvm::SmallVectorImpl &inferredReturnTypes) - - //===----------------------------------------------------------------------===// diff --git a/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp b/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp +index 80d2fb3b..7981077c 100644 --- a/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp +++ b/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp -@@ -10,6 +10,7 @@ - #include "mlir/Dialect/SCF/IR/SCF.h" +@@ -13,6 +13,7 @@ + #include "mlir/Dialect/SCF/Transforms/Patterns.h" #include "mlir/Dialect/Vector/IR/VectorOps.h" #include "mlir/IR/BuiltinAttributes.h" +#include "mlir/IR/BuiltinOps.h" #include "mlir/IR/BuiltinTypes.h" #include "mlir/Pass/Pass.h" #include "mlir/Transforms/DialectConversion.h" -@@ -63,10 +64,10 @@ +@@ -85,10 +86,10 @@ static unsigned getLLVMAlignment(fly::PointerType ptrTy) { // chosen to avoid collisions with existing globals in the module. Shared by // the static-LDS `make_ptr` and dynamic-LDS `get_dyn_shared` lowerings. static LLVM::GlobalOp createLDSGlobal(ConversionPatternRewriter &rewriter, @@ -34,7 +23,7 @@ diff --git a/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp b/lib/Conversion/FlyToROCD existingNames.insert(globalOp.getSymName()); unsigned counter = 0; -@@ -74,7 +75,7 @@ +@@ -96,7 +97,7 @@ static LLVM::GlobalOp createLDSGlobal(ConversionPatternRewriter &rewriter, prefix, [&](StringRef candidate) { return existingNames.contains(candidate); }, counter); OpBuilder::InsertionGuard guard(rewriter); @@ -43,7 +32,7 @@ diff --git a/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp b/lib/Conversion/FlyToROCD auto arrayTy = LLVM::LLVMArrayType::get(IntegerType::get(rewriter.getContext(), 8), nbytes); auto globalOp = LLVM::GlobalOp::create(rewriter, loc, arrayTy, -@@ -127,12 +128,17 @@ +@@ -149,12 +150,17 @@ public: return rewriter.notifyMatchFailure( op, "shared make_ptr requires allocBytes, allocAlign in dictAttrs"); @@ -64,7 +53,7 @@ diff --git a/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp b/lib/Conversion/FlyToROCD allocAlignAttr.getInt(), /*addrSpace=*/3); rewriter.replaceOpWithNewOp(op, global); return success(); -@@ -169,15 +175,20 @@ +@@ -191,15 +197,20 @@ public: auto flyPtrTy = cast(op.getResult().getType()); unsigned addrSpace = mapAttrToLLVMAddressSpace(flyPtrTy.getAddressSpace()); @@ -88,7 +77,7 @@ diff --git a/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp b/lib/Conversion/FlyToROCD if (auto arrayType = dyn_cast(globalOp.getType())) { if (globalOp.getAddrSpace() == addrSpace && arrayType.getNumElements() == 0 && globalOp.getAlignment().value_or(0) == 1024) { -@@ -187,7 +198,7 @@ +@@ -209,7 +220,7 @@ public: } } if (!sharedGlobal) { @@ -97,32 +86,7 @@ diff --git a/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp b/lib/Conversion/FlyToROCD /*nbytes=*/0, /*align=*/1024, addrSpace); } -@@ -402,10 +413,11 @@ - if (isTargetAddressSpace(flyPtrTy.getAddressSpace())) { - BufferFatPtr bp(flyPtrTy, ptr); - Value zero = arith::ConstantIntOp::create(rewriter, loc, 0, 32); -+ Attribute aux = rewriter.getI32IntegerAttr(0); - ArrayAttr noAttrs; - Value loaded = ROCDL::RawPtrBufferLoadOp::create( - rewriter, loc, loadTy, bp.bufferRsrc(rewriter, loc), bp.swizzleByteOffset(rewriter, loc), -- zero, zero, noAttrs, noAttrs, noAttrs); -+ zero, aux, noAttrs, noAttrs, noAttrs); - rewriter.replaceOp(op, loaded); - return success(); - } else { -@@ -447,9 +459,10 @@ - if (isTargetAddressSpace(flyPtrTy.getAddressSpace())) { - BufferFatPtr bp(flyPtrTy, ptr); - Value zero = arith::ConstantIntOp::create(rewriter, loc, 0, 32); -+ Attribute aux = rewriter.getI32IntegerAttr(0); - ArrayAttr noAttrs; - ROCDL::RawPtrBufferStoreOp::create(rewriter, loc, value, bp.bufferRsrc(rewriter, loc), -- bp.swizzleByteOffset(rewriter, loc), zero, zero, noAttrs, -+ bp.swizzleByteOffset(rewriter, loc), zero, aux, noAttrs, - noAttrs, noAttrs); - rewriter.eraseOp(op); - return success(); -@@ -747,10 +760,6 @@ +@@ -810,10 +821,6 @@ public: if (Value tok = op.getAsyncToken()) asyncTokenType = tok.getType(); @@ -133,7 +97,7 @@ diff --git a/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp b/lib/Conversion/FlyToROCD if (Value asyncObj = adaptor.getAsyncObject()) { if (!adaptor.getAsyncDependencies().empty()) return rewriter.notifyMatchFailure( -@@ -758,13 +767,14 @@ +@@ -821,13 +828,14 @@ public: rewriter.replaceOpWithNewOp( op, kernelRef, grid, block, adaptor.getDynamicSharedMemorySize(), @@ -150,235 +114,15 @@ diff --git a/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp b/lib/Conversion/FlyToROCD return success(); } }; -diff --git a/lib/Dialect/Fly/Transforms/PromoteRegMemToVectorSSA.cpp b/lib/Dialect/Fly/Transforms/PromoteRegMemToVectorSSA.cpp ---- a/lib/Dialect/Fly/Transforms/PromoteRegMemToVectorSSA.cpp -+++ b/lib/Dialect/Fly/Transforms/PromoteRegMemToVectorSSA.cpp -@@ -30,11 +30,6 @@ - template <> struct DenseMapInfo : DenseMapInfo { - using Base = DenseMapInfo; +diff --git a/lib/Dialect/Fly/Transforms/LayoutLowering.cpp b/lib/Dialect/Fly/Transforms/LayoutLowering.cpp +--- a/lib/Dialect/Fly/Transforms/LayoutLowering.cpp ++++ b/lib/Dialect/Fly/Transforms/LayoutLowering.cpp +@@ -2994,7 +2994,7 @@ + // A value reached twice on one walk (cyclic chain) cannot be bounded here. + if (!visited.insert(def).second) + return std::nullopt; +- auto pop = llvm::make_scope_exit([&] { visited.erase(def); }); ++ auto pop = llvm::scope_exit([&] { visited.erase(def); }); -- static mlir::fly::MakePtrOp getEmptyKey() { return mlir::fly::MakePtrOp(Base::getEmptyKey()); } -- -- static mlir::fly::MakePtrOp getTombstoneKey() { -- return mlir::fly::MakePtrOp(Base::getTombstoneKey()); -- } - }; - - } // namespace llvm -diff --git a/lib/Dialect/FlyROCDL/CDNA3/CopyAtom.cpp b/lib/Dialect/FlyROCDL/CDNA3/CopyAtom.cpp ---- a/lib/Dialect/FlyROCDL/CDNA3/CopyAtom.cpp -+++ b/lib/Dialect/FlyROCDL/CDNA3/CopyAtom.cpp -@@ -89,7 +89,7 @@ - }; - - // raw buffer load/store cachepolicy (0=cached, 2=nt) -- Value aux = arith::ConstantIntOp::create(builder, loc, getCacheModifier(), 32); -+ Attribute aux = builder.getI32IntegerAttr(getCacheModifier()); - ArrayAttr noAttrs; - - auto srcMemTy = srcTyArg ? dyn_cast(srcTyArg) : fly::MemRefType(); -@@ -313,7 +313,7 @@ - soffset = arith::DivUIOp::create(builder, loc, bits, eight); - } - -- Value zero = arith::ConstantIntOp::create(builder, loc, 0, 32); -+ Attribute aux = builder.getI32IntegerAttr(0); - Value size = arith::ConstantIntOp::create(builder, loc, sizeBytes, 32); - - BufferFatPtr bp(srcMemTy.getPointerType(), src); -@@ -322,7 +322,7 @@ - - ArrayAttr noAttrs; - ROCDL::RawPtrBufferLoadLdsOp::create(builder, loc, srcRsrc, dst, size, srcOff, soffset, immOffset, -- zero, noAttrs, noAttrs, noAttrs); -+ aux, noAttrs, noAttrs, noAttrs); - return success(); - } - -@@ -428,7 +428,7 @@ - soffset = arith::DivUIOp::create(builder, loc, bits, eight); - } - -- Value zero = arith::ConstantIntOp::create(builder, loc, 0, 32); -+ Attribute aux = builder.getI32IntegerAttr(0); - ArrayAttr noAttrs; - - AtomicOp op = getAtomicOp().getValue(); -@@ -437,21 +437,21 @@ - case AtomicOp::Add: - if (!isFloat) - return failure(); -- ROCDL::RawPtrBufferAtomicFaddOp::create(builder, loc, src, dstRsrc, dstOff, soffset, zero, -+ ROCDL::RawPtrBufferAtomicFaddOp::create(builder, loc, src.getType(), src, dstRsrc, dstOff, soffset, aux, - noAttrs, noAttrs, noAttrs); - break; - case AtomicOp::Max: - if (isFloat) -- ROCDL::RawPtrBufferAtomicFmaxOp::create(builder, loc, src, dstRsrc, dstOff, soffset, zero, -+ ROCDL::RawPtrBufferAtomicFmaxOp::create(builder, loc, src.getType(), src, dstRsrc, dstOff, soffset, aux, - noAttrs, noAttrs, noAttrs); - else -- ROCDL::RawPtrBufferAtomicSmaxOp::create(builder, loc, src, dstRsrc, dstOff, soffset, zero, -+ ROCDL::RawPtrBufferAtomicSmaxOp::create(builder, loc, src.getType(), src, dstRsrc, dstOff, soffset, aux, - noAttrs, noAttrs, noAttrs); - break; - case AtomicOp::Min: - if (isFloat) - return failure(); -- ROCDL::RawPtrBufferAtomicUminOp::create(builder, loc, src, dstRsrc, dstOff, soffset, zero, -+ ROCDL::RawPtrBufferAtomicUminOp::create(builder, loc, src.getType(), src, dstRsrc, dstOff, soffset, aux, - noAttrs, noAttrs, noAttrs); - break; - default: -diff --git a/lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp b/lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp ---- a/lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp -+++ b/lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp -@@ -169,8 +169,8 @@ - - #define DISPATCH_MFMA_SSA(M_, K_, PRED, OP) \ - if (m == M_ && n == M_ && k == K_ && (PRED)) { \ -- auto zeroAttr = builder.getI32IntegerAttr(0); \ -- return ROCDL::OP::create(builder, loc, accTy, a, b, c, zeroAttr, zeroAttr, zeroAttr) \ -+ return ROCDL::OP::create(builder, loc, accTy, a, b, c, 0, 0, \ -+ ROCDL::MFMAPermB::none) \ - .getResult(); \ - } - -diff --git a/lib/Dialect/FlyROCDL/Ops.cpp b/lib/Dialect/FlyROCDL/Ops.cpp ---- a/lib/Dialect/FlyROCDL/Ops.cpp -+++ b/lib/Dialect/FlyROCDL/Ops.cpp -@@ -15,7 +15,7 @@ - LogicalResult GetBufferRsrcOp::inferReturnTypes(MLIRContext *context, - std::optional location, - ValueRange operands, DictionaryAttr attributes, -- OpaqueProperties properties, RegionRange regions, -+ PropertyRef properties, RegionRange regions, - SmallVectorImpl &inferredReturnTypes) { - auto ptrTy = dyn_cast(operands[0].getType()); - if (!ptrTy) -diff --git a/lib/Dialect/FlyROCDL/GFX1250/MmaAtom.cpp b/lib/Dialect/FlyROCDL/GFX1250/MmaAtom.cpp ---- a/lib/Dialect/FlyROCDL/GFX1250/MmaAtom.cpp -+++ b/lib/Dialect/FlyROCDL/GFX1250/MmaAtom.cpp -@@ -239,14 +239,12 @@ - Value res; - if constexpr (Variant == WmmaVariant::ModsAllReuse) { - // Float path: no sign/clamp operands. -- res = WmmaOp::create(builder, loc, accTy, -- /*signA=*/false, a, /*signB=*/false, b, -- /*modC=*/(uint16_t)0, c) -+ res = WmmaOp::create(builder, loc, accTy, a, b, ROCDL::WMMACModifier::none, c, -+ /*reuseA=*/false, /*reuseB=*/false) - .getResult(); - } else if constexpr (Variant == WmmaVariant::ModsC) { - // fp8 path: no sign/clamp operands. -- res = WmmaOp::create(builder, loc, accTy, a, b, -- /*modC=*/(uint16_t)0, c, -+ res = WmmaOp::create(builder, loc, accTy, a, b, ROCDL::WMMACModifier::none, c, - /*reuseA=*/false, /*reuseB=*/false) - .getResult(); - } else if constexpr (Variant == WmmaVariant::ModsABClamp) { -diff --git a/lib/Dialect/FlyROCDL/GFX1250/MmaAtomScale.cpp b/lib/Dialect/FlyROCDL/GFX1250/MmaAtomScale.cpp ---- a/lib/Dialect/FlyROCDL/GFX1250/MmaAtomScale.cpp -+++ b/lib/Dialect/FlyROCDL/GFX1250/MmaAtomScale.cpp -@@ -244,15 +244,20 @@ - // fp4-only form; no fmtA/fmtB operands. - if (block16) - return ROCDL::wmma_scale16_f32_32x16x128_f4::create( -- builder, loc, accTy, a, b, /*modC=*/(uint16_t)getModC(), c, -- /*scaleAType=*/(uint32_t)getOpselA(), /*fmtScaleA=*/(uint32_t)0, scaleA, -- /*scaleBType=*/(uint32_t)getOpselB(), /*fmtScaleB=*/(uint32_t)0, scaleB, -+ builder, loc, accTy, a, b, -+ static_cast(getModC()), c, -+ static_cast(getOpselA()), -+ ROCDL::WMMAMatrixScaleFormat::e8, scaleA, -+ static_cast(getOpselB()), -+ ROCDL::WMMAMatrixScaleFormat::e8, scaleB, - /*reuseA=*/getReuseA(), /*reuseB=*/getReuseB()) - .getResult(); - return ROCDL::wmma_scale_f32_32x16x128_f4::create( -- builder, loc, accTy, a, b, /*modC=*/(uint16_t)getModC(), c, -- /*scaleAType=*/(uint32_t)getOpselA(), /*fmtScaleA=*/(uint32_t)0, scaleA, -- /*scaleBType=*/(uint32_t)getOpselB(), /*fmtScaleB=*/(uint32_t)0, scaleB, -+ builder, loc, accTy, a, b, static_cast(getModC()), c, -+ static_cast(getOpselA()), -+ ROCDL::WMMAMatrixScaleFormat::e8, scaleA, -+ static_cast(getOpselB()), -+ ROCDL::WMMAMatrixScaleFormat::e8, scaleB, - /*reuseA=*/getReuseA(), /*reuseB=*/getReuseB()) - .getResult(); - } -@@ -264,17 +269,23 @@ - - if (block16) - return ROCDL::wmma_scale16_f32_16x16x128_f8f6f4::create( -- builder, loc, accTy, /*fmtA=*/*aFmt, a, /*fmtB=*/*bFmt, b, -- /*modC=*/(uint16_t)getModC(), c, -- /*scaleAType=*/(uint32_t)getOpselA(), /*fmtScaleA=*/(uint32_t)0, scaleA, -- /*scaleBType=*/(uint32_t)getOpselB(), /*fmtScaleB=*/(uint32_t)0, scaleB, -+ builder, loc, accTy, static_cast(*aFmt), a, -+ static_cast(*bFmt), b, -+ static_cast(getModC()), c, -+ static_cast(getOpselA()), -+ ROCDL::WMMAMatrixScaleFormat::e8, scaleA, -+ static_cast(getOpselB()), -+ ROCDL::WMMAMatrixScaleFormat::e8, scaleB, - /*reuseA=*/getReuseA(), /*reuseB=*/getReuseB()) - .getResult(); - return ROCDL::wmma_scale_f32_16x16x128_f8f6f4::create( -- builder, loc, accTy, /*fmtA=*/*aFmt, a, /*fmtB=*/*bFmt, b, -- /*modC=*/(uint16_t)getModC(), c, -- /*scaleAType=*/(uint32_t)getOpselA(), /*fmtScaleA=*/(uint32_t)0, scaleA, -- /*scaleBType=*/(uint32_t)getOpselB(), /*fmtScaleB=*/(uint32_t)0, scaleB, -+ builder, loc, accTy, static_cast(*aFmt), a, -+ static_cast(*bFmt), b, -+ static_cast(getModC()), c, -+ static_cast(getOpselA()), -+ ROCDL::WMMAMatrixScaleFormat::e8, scaleA, -+ static_cast(getOpselB()), -+ ROCDL::WMMAMatrixScaleFormat::e8, scaleB, - /*reuseA=*/getReuseA(), /*reuseB=*/getReuseB()) - .getResult(); - } -diff --git a/lib/Dialect/FlyROCDL/CDNA4/MmaAtom.cpp b/lib/Dialect/FlyROCDL/CDNA4/MmaAtom.cpp ---- a/lib/Dialect/FlyROCDL/CDNA4/MmaAtom.cpp -+++ b/lib/Dialect/FlyROCDL/CDNA4/MmaAtom.cpp -@@ -237,21 +237,17 @@ - Value scaleB = LLVM::ExtractValueOp::create( - builder, loc, atomVal, ArrayRef{*getFieldIndex(AtomStateField::ScaleB)}); - -- auto cbszAttr = builder.getI32IntegerAttr(*aTypeCode); -- auto blgpAttr = builder.getI32IntegerAttr(*bTypeCode); -- auto opselAAttr = builder.getI32IntegerAttr(getOpselA()); -- auto opselBAttr = builder.getI32IntegerAttr(getOpselB()); -+ auto cbsz = static_cast(*aTypeCode); -+ auto blgp = static_cast(*bTypeCode); - - if (m == 16 && n == 16 && k == 128) { -- return ROCDL::mfma_scale_f32_16x16x128_f8f6f4::create(builder, loc, accTy, a, b, c, cbszAttr, -- blgpAttr, opselAAttr, scaleA, opselBAttr, -- scaleB) -+ return ROCDL::mfma_scale_f32_16x16x128_f8f6f4::create( -+ builder, loc, accTy, a, b, c, cbsz, blgp, getOpselA(), scaleA, getOpselB(), scaleB) - .getResult(); - } - if (m == 32 && n == 32 && k == 64) { -- return ROCDL::mfma_scale_f32_32x32x64_f8f6f4::create(builder, loc, accTy, a, b, c, cbszAttr, -- blgpAttr, opselAAttr, scaleA, opselBAttr, -- scaleB) -+ return ROCDL::mfma_scale_f32_32x32x64_f8f6f4::create( -+ builder, loc, accTy, a, b, c, cbsz, blgp, getOpselA(), scaleA, getOpselB(), scaleB) - .getResult(); - } - -diff --git a/lib/Dialect/FlyROCDL/GFX1250/CopyAtom.cpp b/lib/Dialect/FlyROCDL/GFX1250/CopyAtom.cpp ---- a/lib/Dialect/FlyROCDL/GFX1250/CopyAtom.cpp -+++ b/lib/Dialect/FlyROCDL/GFX1250/CopyAtom.cpp -@@ -503,7 +503,7 @@ - vector::FromElementsOp::create(builder, loc, VectorType::get({8}, builder.getI32Type()), - ValueRange{zero, zero, zero, zero, zero, zero, zero, zero}); - -- uint32_t cachePolicy = static_cast(getCacheModifier()); -+ Attribute cachePolicy = builder.getI32IntegerAttr(getCacheModifier()); - ArrayAttr noAliasScopes; - if (isLoad) - ROCDL::TensorLoadToLDSOp::create(builder, loc, dgroup0, dgroup1, dg2, dg3, dg4, cachePolicy, + // Launch coordinates: ask the op for its own range. Every gpu index op + // implements InferIntRangeInterface, and that implementation already prefers diff --git a/third_party/flydsl/repo.bzl b/third_party/flydsl/repo.bzl index 8a8c3be2..144e5af5 100644 --- a/third_party/flydsl/repo.bzl +++ b/third_party/flydsl/repo.bzl @@ -4,11 +4,12 @@ def repo(): git_repository( name = "flydsl", remote = "https://github.com/ROCm/FlyDSL.git", - commit = "c62159d0c18e232794a1903f192fc094148f4a43", + commit = "1941889400621f1fc3d8f03bada1a4e7380cdf7e", build_file = "//third_party/flydsl:flydsl.bazel", patches = [ "//third_party/flydsl:atomic_exchange.patch", "//third_party/flydsl:mlir_property_ref.patch", + "//third_party/flydsl:mlir_legacy_assembly.patch", "//third_party/flydsl:vector_atomic.patch", "//third_party/flydsl:xf32_mfma.patch", ], diff --git a/third_party/grpc/plugin-flags.patch b/third_party/grpc/plugin-flags.patch new file mode 100644 index 00000000..b69a77f5 --- /dev/null +++ b/third_party/grpc/plugin-flags.patch @@ -0,0 +1,19 @@ +# Match XLA's cc_grpc_library extension for services_namespace. +--- a/bazel/cc_grpc_library.bzl ++++ b/bazel/cc_grpc_library.bzl +@@ -29,6 +29,7 @@ + allow_deprecated = False, + use_external = False, # @unused + grpc_only = False, ++ plugin_flags = [], + **kwargs): + """Generates C++ grpc classes for services defined in a proto file. + +@@ -110,6 +111,7 @@ + well_known_protos = well_known_protos, + generate_mocks = generate_mocks, + allow_deprecated = allow_deprecated, ++ flags = plugin_flags, + **kwargs + ) + diff --git a/third_party/mosaic_tpu/BUILD.bazel b/third_party/mosaic_tpu/BUILD.bazel deleted file mode 100644 index 8b137891..00000000 --- a/third_party/mosaic_tpu/BUILD.bazel +++ /dev/null @@ -1 +0,0 @@ - diff --git a/third_party/mosaic_tpu/mosaic_tpu.bazel b/third_party/mosaic_tpu/mosaic_tpu.bazel deleted file mode 100644 index ad2d1eb9..00000000 --- a/third_party/mosaic_tpu/mosaic_tpu.bazel +++ /dev/null @@ -1,275 +0,0 @@ -load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") - -# Copyright 2023 The JAX Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -load("@rules_cc//cc:cc_library.bzl", "cc_library") - -licenses(["notice"]) - -package( - default_applicable_licenses = [], - default_visibility = [ - "//visibility:public", - ], -) - -################################################################################ -# TPU dialect - -cc_library( - name = "tpu_dialect", - srcs = [ - "jaxlib/mosaic/dialect/tpu/array_util.cc", - "jaxlib/mosaic/dialect/tpu/layout.cc", - "jaxlib/mosaic/dialect/tpu/tpu_dialect.cc", - "jaxlib/mosaic/dialect/tpu/tpu_ops.cc", - "jaxlib/mosaic/dialect/tpu/util.cc", - "jaxlib/mosaic/dialect/tpu/vreg_util.cc", - ], - hdrs = [ - "jaxlib/mosaic/dialect/tpu/array_util.h", - "jaxlib/mosaic/dialect/tpu/layout.h", - "jaxlib/mosaic/dialect/tpu/tpu_dialect.h", - "jaxlib/mosaic/dialect/tpu/util.h", - "jaxlib/mosaic/dialect/tpu/vreg_util.h", - ], - # compatible with libtpu - deps = [ - ":stringify_util", - ":tpu_inc_gen", - "@com_google_absl//absl/algorithm:container", - "@com_google_absl//absl/hash", - "@com_google_absl//absl/log:check", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/strings:str_format", - "@com_google_absl//absl/types:span", - "@llvm-project//llvm:Support", - "@llvm-project//mlir:ArithDialect", - "@llvm-project//mlir:CommonFolders", - "@llvm-project//mlir:DialectUtils", - "@llvm-project//mlir:FuncDialect", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:MathDialect", - "@llvm-project//mlir:MemRefDialect", - "@llvm-project//mlir:Pass", - "@llvm-project//mlir:SCFDialect", - "@llvm-project//mlir:Support", - "@llvm-project//mlir:VectorDialect", - "@xla//xla:array", - "@xla//xla:shape_util", - "@xla//xla/tsl/platform:statusor", - ], -) - -cc_library( - name = "tpu_serde_pass", - srcs = ["jaxlib/mosaic/dialect/tpu/transforms/serde.cc"], - hdrs = ["jaxlib/mosaic/dialect/tpu/transforms/serde.h"], - # compatible with libtpu - deps = [ - ":pass_boilerplate", - ":serde", - ":tpu_dialect", - "@llvm-project//llvm:Support", - "@llvm-project//mlir:ArithDialect", - "@llvm-project//mlir:DataLayoutInterfaces", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:Pass", - "@llvm-project//mlir:Support", - "@llvm-project//mlir:VectorDialect", - ], -) - -gentbl_cc_library( - name = "tpu_inc_gen", - # compatible with libtpu - tbl_outs = { - "jaxlib/mosaic/dialect/tpu/tpu_ops.h.inc": [ - "-gen-op-decls", - "-dialect=tpu", - ], - "jaxlib/mosaic/dialect/tpu/tpu_ops.cc.inc": [ - "-gen-op-defs", - "-dialect=tpu", - ], - "jaxlib/mosaic/dialect/tpu/tpu_dialect.h.inc": [ - "-gen-dialect-decls", - "-dialect=tpu", - ], - "jaxlib/mosaic/dialect/tpu/tpu_dialect.cc.inc": [ - "-gen-dialect-defs", - "-dialect=tpu", - ], - "jaxlib/mosaic/dialect/tpu/tpu_enums.h.inc": [ - "-gen-enum-decls", - "-dialect=tpu", - ], - "jaxlib/mosaic/dialect/tpu/tpu_enums.cc.inc": [ - "-gen-enum-defs", - "-dialect=tpu", - ], - "jaxlib/mosaic/dialect/tpu/tpu_attr_defs.h.inc": [ - "-gen-attrdef-decls", - "-dialect=tpu", - "--attrdefs-dialect=tpu", - ], - "jaxlib/mosaic/dialect/tpu/tpu_attr_defs.cc.inc": [ - "-gen-attrdef-defs", - "-dialect=tpu", - "--attrdefs-dialect=tpu", - ], - "jaxlib/mosaic/dialect/tpu/tpu_type_defs.h.inc": [ - "-gen-typedef-decls", - "-dialect=tpu", - "--typedefs-dialect=tpu", - ], - "jaxlib/mosaic/dialect/tpu/tpu_type_defs.cc.inc": [ - "-gen-typedef-defs", - "-dialect=tpu", - "--typedefs-dialect=tpu", - ], - "jaxlib/mosaic/dialect/tpu/tpu_passes.h.inc": [ - "-gen-pass-decls", - "-name=TPU", - ], - "jaxlib/mosaic/dialect/tpu/integrations/c/tpu_passes.capi.h.inc": [ - "-gen-pass-capi-header", - "--prefix=TPU", - ], - "jaxlib/mosaic/dialect/tpu/integrations/c/tpu_passes.capi.cc.inc": [ - "-gen-pass-capi-impl", - "--prefix=TPU", - ], - }, - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "jaxlib/mosaic/dialect/tpu/tpu_ops.td", - deps = [":tpu_ops_td_files"], -) - -td_library( - name = "tpu_td_files", - srcs = [ - "jaxlib/mosaic/dialect/tpu/tpu.td", - ], - # compatible with libtpu - deps = [ - "@llvm-project//mlir:BuiltinDialectTdFiles", - ], -) - -td_library( - name = "tpu_ops_td_files", - srcs = [ - "jaxlib/mosaic/dialect/tpu/tpu_ops.td", - ], - # compatible with libtpu - deps = [ - ":tpu_td_files", - "@llvm-project//mlir:BuiltinDialectTdFiles", - "@llvm-project//mlir:ControlFlowInterfacesTdFiles", - "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", - "@llvm-project//mlir:OpBaseTdFiles", - "@llvm-project//mlir:PassBaseTdFiles", - "@llvm-project//mlir:SideEffectInterfacesTdFiles", - ], -) - -# C API targets - -TPU_CAPI_SOURCES = [ - "jaxlib/mosaic/dialect/tpu/integrations/c/tpu_dialect.cc", - "jaxlib/mosaic/dialect/tpu/integrations/c/tpu_passes.capi.cc.inc", -] - -TPU_CAPI_HEADERS = [ - "jaxlib/mosaic/dialect/tpu/integrations/c/tpu_dialect.h", - "jaxlib/mosaic/dialect/tpu/integrations/c/tpu_passes.capi.h.inc", -] - -cc_library( - name = "tpu_dialect_capi", - srcs = TPU_CAPI_SOURCES, - hdrs = TPU_CAPI_HEADERS, - deps = [ - ":tpu_dialect", - ":tpu_inc_gen", - ":tpu_serde_pass", - "@llvm-project//llvm:Support", - "@llvm-project//mlir:CAPIIR", - "@llvm-project//mlir:FuncDialect", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:Support", - ], -) - -# Header-only target, used when using the C API from a separate shared library. -cc_library( - name = "tpu_dialect_capi_headers", - hdrs = TPU_CAPI_HEADERS, - deps = [ - ":tpu_inc_gen", - "@llvm-project//mlir:CAPIIRHeaders", - ], -) - -# Alwayslink target, used when exporting the C API from a shared library. -cc_library( - name = "tpu_dialect_capi_objects", - srcs = TPU_CAPI_SOURCES, - hdrs = TPU_CAPI_HEADERS, - deps = [ - ":tpu_dialect", - ":tpu_inc_gen", - ":tpu_serde_pass", - "@llvm-project//llvm:Support", - "@llvm-project//mlir:CAPIIR", - "@llvm-project//mlir:FuncDialect", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:Support", - ], - alwayslink = True, -) - -cc_library( - name = "pass_boilerplate", - hdrs = ["jaxlib/mosaic/pass_boilerplate.h"], - # compatible with libtpu - deps = [ - "@llvm-project//mlir:IR", - "@llvm-project//mlir:Pass", - "@llvm-project//mlir:Support", - ], -) - -cc_library( - name = "serde", - srcs = ["jaxlib/mosaic/serde.cc"], - hdrs = ["jaxlib/mosaic/serde.h"], - # compatible with libtpu - deps = [ - "@llvm-project//llvm:Support", - "@llvm-project//mlir:DataLayoutInterfaces", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:Support", - ], -) - -cc_library( - name = "stringify_util", - hdrs = ["jaxlib/mosaic/dialect/tpu/stringify_util.h"], - # compatible with libtpu - deps = ["@llvm-project//llvm:Support"], -) diff --git a/third_party/mosaic_tpu/repo.bzl b/third_party/mosaic_tpu/repo.bzl deleted file mode 100644 index e3fc240e..00000000 --- a/third_party/mosaic_tpu/repo.bzl +++ /dev/null @@ -1,11 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") - -def repo(): - git_repository( - name = "mosaic_tpu", - remote = "https://github.com/jax-ml/jax.git", - commit = "92348e8631f89bc9e4071f918524b432bcb73f97", - build_file = "//third_party/mosaic_tpu:mosaic_tpu.bazel", - sparse_checkout_patterns = ["jaxlib/mosaic/**"], - patch_cmds = ["find . -name BUILD -delete"], - ) diff --git a/third_party/non_module_deps.bzl b/third_party/non_module_deps.bzl index 4e5f49a3..44632c4d 100644 --- a/third_party/non_module_deps.bzl +++ b/third_party/non_module_deps.bzl @@ -10,7 +10,6 @@ load("//third_party/iree:repo.bzl", iree = "repo") load("//third_party/libvaxis:repo.bzl", libvaxis = "repo") load("//third_party/linenoise:repo.bzl", linenoise = "repo") load("//third_party/mnist:repo.bzl", mnist = "repo") -load("//third_party/mosaic_tpu:repo.bzl", mosaic_tpu = "repo") load("//third_party/org_swig_swig:repo.bzl", org_swig_swig = "repo") load("//third_party/stb:repo.bzl", stb = "repo") load("//third_party/translate-c:repo.bzl", translate_c = "repo") @@ -34,7 +33,6 @@ def _non_module_deps_impl(mctx): zigimg() uucode() libvaxis() - mosaic_tpu() iree() cuda_tile() flydsl() diff --git a/third_party/xla/repo.bzl b/third_party/xla/repo.bzl index 3e153172..8db1747f 100644 --- a/third_party/xla/repo.bzl +++ b/third_party/xla/repo.bzl @@ -4,7 +4,7 @@ def repo(): git_repository( name = "xla", remote = "https://github.com/openxla/xla.git", - commit = "41370d1124c74d7b93a207136a636d8c631cbed9", + commit = "f3dee9ae87745e151837b2ff1b0485928f8e5a5d", patches = [ "//third_party/xla:cuda-root-path-local-defines.patch", ], diff --git a/zml/Compiler.zig b/zml/Compiler.zig index b6cd15b7..a76a828e 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -945,6 +945,8 @@ fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.Load try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_nccl_user_buffers", true, upb_arena); // Enable for both Blackwell+ and Ampere+ // try setXlaOverrideFlag(overrides_map, "xla_gpu_cudnn_gemm_fusion_level", 2, upb_arena); + // Need to adapt XLA to support that (for other platforms too) + try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_enable_tiling_propagation", false, upb_arena); }, .rocm => { try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_scheduling_mode", "CONCURRENT", upb_arena); @@ -959,9 +961,11 @@ fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.Load // Enable user buffers to assign color 1 and receive the reclaim exemption: // https://github.com/openxla/xla/pull/46029 try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_nccl_user_buffers", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_enable_tiling_propagation", false, upb_arena); }, .metal => { try setXlaOverrideFlag(overrides_map, "xla_gpu_metal_fast_math", false, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_enable_tiling_propagation", false, upb_arena); }, .oneapi => { // More efficient for the allgather/broadcast implementation of the collective permute. @@ -972,6 +976,7 @@ fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.Load // Grouped GEMM still needs a oneDNN lowering. try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_ragged_dot_grouped_gemm", false, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_enable_tiling_propagation", false, upb_arena); }, else => {}, } diff --git a/zml/ops.zig b/zml/ops.zig index 186805ea..84ff27a1 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1243,7 +1243,7 @@ test "triton" { const ir = \\ module { - \\ tt.func public @add_one(%arg0: !tt.ptr {tt.divisibility = 32 : i32}, %arg1: !tt.ptr {tt.divisibility = 32 : i32}, %arg2: !tt.ptr {tt.divisibility = 32 : i32}, %arg3: !tt.ptr {tt.divisibility = 32 : i32}) { + \\ tt.func public @add_one(%arg0: !tt.ptr {tt.divisibility = 32 : i32}, %arg1: !tt.ptr {tt.divisibility = 32 : i32}, %arg2: !tt.ptr {tt.divisibility = 32 : i32}, %arg3: !tt.ptr {tt.divisibility = 32 : i32}) { \\ %0 = tt.get_program_id x : i32 \\ %1 = tt.load %arg0 {cache = 1 : i32, evict = 1 : i32, isVolatile = false} : !tt.ptr \\ %2 = tt.load %arg1 {cache = 1 : i32, evict = 1 : i32, isVolatile = false} : !tt.ptr From daab01572f1cb51b89f366a23eed79b5aa6b2c98 Mon Sep 17 00:00:00 2001 From: Olivier Boulant Date: Wed, 7 Oct 2026 19:13:33 +0200 Subject: [PATCH 762/881] platforms: switch LLVM libunwind packages from 23 to 22 (#907) `apt.llvm.org` removed some links, breaking downloads from our lockfiles and resulting in 404. Switch CPU/CUDA to `llvm-toolchain-bookworm-22` and ROCm to `llvm-toolchain-jammy-22`, and regenerate the lockfiles. All test passed on: - macOS - 9985wx-5090x4 - gb300-2* - mi300 * See reverted commit [f7eb1cb](https://github.com/zml/zml/pull/907/commits/f7eb1cbe3b715da465bd4e9cba8a2b071499b045) @hugomano --- It is a follow up from https://github.com/zml/zml/pull/875 --- platforms/cpu/packages.lock.json | 8 ++++---- platforms/cpu/packages.yaml | 4 ++-- platforms/cuda/packages.lock.json | 16 ++++++++-------- platforms/cuda/packages.yaml | 4 ++-- platforms/rocm/packages.lock.json | 8 ++++---- platforms/rocm/packages.yaml | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/platforms/cpu/packages.lock.json b/platforms/cpu/packages.lock.json index 776c49b1..7e9963e3 100644 --- a/platforms/cpu/packages.lock.json +++ b/platforms/cpu/packages.lock.json @@ -19,13 +19,13 @@ "version": "12.2.0-14+deb12u1" } ], - "key": "llvm-libunwind1_1-23.1.2_-p--p-20260916013112-p-21a77e7bb6bf-1_exp1_20260916133123.74_amd64", + "key": "llvm-libunwind1_1-22.1.8_-p--p-20260714014842-p-ca7933e47d3a-1_exp1_20260714134950.86_amd64", "name": "llvm-libunwind1", - "sha256": "7fe1e2f61f6db688b53a4015a2d391f4474e0c0aec76b4dc262fe91a43854e84", + "sha256": "04cb76d70da569e5bcf2395f28dd7e9b9e9f0bf4521cf3b008b0f62ae9d2b712", "urls": [ - "https://apt.llvm.org/bookworm/pool/main/l/llvm-toolchain-23/llvm-libunwind1_23.1.2~++20260916013112+21a77e7bb6bf-1~exp1~20260916133123.74_amd64.deb" + "https://apt.llvm.org/bookworm/pool/main/l/llvm-toolchain-22/llvm-libunwind1_22.1.8~++20260714014842+ca7933e47d3a-1~exp1~20260714134950.86_amd64.deb" ], - "version": "1:23.1.2~++20260916013112+21a77e7bb6bf-1~exp1~20260916133123.74" + "version": "1:22.1.8~++20260714014842+ca7933e47d3a-1~exp1~20260714134950.86" }, { "arch": "amd64", diff --git a/platforms/cpu/packages.yaml b/platforms/cpu/packages.yaml index b8ce6251..d6f97d05 100644 --- a/platforms/cpu/packages.yaml +++ b/platforms/cpu/packages.yaml @@ -6,11 +6,11 @@ version: 1 sources: - channel: bookworm main url: https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z - - channel: llvm-toolchain-bookworm-23 main + - channel: llvm-toolchain-bookworm-22 main url: https://apt.llvm.org/bookworm archs: - "amd64" packages: - - "llvm-libunwind1" # LLVM 23 runtime from llvm-toolchain-bookworm-23. + - "llvm-libunwind1" # LLVM 22 runtime from llvm-toolchain-bookworm-22. diff --git a/platforms/cuda/packages.lock.json b/platforms/cuda/packages.lock.json index 65ab6ca1..f582fa45 100755 --- a/platforms/cuda/packages.lock.json +++ b/platforms/cuda/packages.lock.json @@ -19,13 +19,13 @@ "version": "12.2.0-14+deb12u1" } ], - "key": "llvm-libunwind1_1-23.1.2_-p--p-20260920033443-p-85ac56026243-1_exp1_20260920033605.79_amd64", + "key": "llvm-libunwind1_1-22.1.8_-p--p-20260714014842-p-ca7933e47d3a-1_exp1_20260714134950.86_amd64", "name": "llvm-libunwind1", - "sha256": "aa690e56d52dc2798491167a06a879cb047b4f294cc953bd4c9f7267b0b0cecd", + "sha256": "04cb76d70da569e5bcf2395f28dd7e9b9e9f0bf4521cf3b008b0f62ae9d2b712", "urls": [ - "https://apt.llvm.org/bookworm/pool/main/l/llvm-toolchain-23/llvm-libunwind1_23.1.2~++20260920033443+85ac56026243-1~exp1~20260920033605.79_amd64.deb" + "https://apt.llvm.org/bookworm/pool/main/l/llvm-toolchain-22/llvm-libunwind1_22.1.8~++20260714014842+ca7933e47d3a-1~exp1~20260714134950.86_amd64.deb" ], - "version": "1:23.1.2~++20260920033443+85ac56026243-1~exp1~20260920033605.79" + "version": "1:22.1.8~++20260714014842+ca7933e47d3a-1~exp1~20260714134950.86" }, { "arch": "amd64", @@ -106,13 +106,13 @@ "version": "12.2.0-14+deb12u1" } ], - "key": "llvm-libunwind1_1-23.1.2_-p--p-20260920033443-p-85ac56026243-1_exp1_20260920033605.79_arm64", + "key": "llvm-libunwind1_1-22.1.8_-p--p-20260714014842-p-ca7933e47d3a-1_exp1_20260714134950.86_arm64", "name": "llvm-libunwind1", - "sha256": "a34362f952c5c815e715ab9b9b9c4776f6690683cb2037f04cf71322f1822e3b", + "sha256": "05890076ab166d0fe3eab3ab08f9076bca1edc7322d123df8970760ae26c25e8", "urls": [ - "https://apt.llvm.org/bookworm/pool/main/l/llvm-toolchain-23/llvm-libunwind1_23.1.2~++20260920033443+85ac56026243-1~exp1~20260920033605.79_arm64.deb" + "https://apt.llvm.org/bookworm/pool/main/l/llvm-toolchain-22/llvm-libunwind1_22.1.8~++20260714014842+ca7933e47d3a-1~exp1~20260714134950.86_arm64.deb" ], - "version": "1:23.1.2~++20260920033443+85ac56026243-1~exp1~20260920033605.79" + "version": "1:22.1.8~++20260714014842+ca7933e47d3a-1~exp1~20260714134950.86" }, { "arch": "arm64", diff --git a/platforms/cuda/packages.yaml b/platforms/cuda/packages.yaml index 6828e7b5..9f6ac8b9 100644 --- a/platforms/cuda/packages.yaml +++ b/platforms/cuda/packages.yaml @@ -6,7 +6,7 @@ version: 1 sources: - channel: bookworm main url: https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z - - channel: llvm-toolchain-bookworm-23 main + - channel: llvm-toolchain-bookworm-22 main url: https://apt.llvm.org/bookworm archs: @@ -14,5 +14,5 @@ archs: - "arm64" packages: - - "llvm-libunwind1" # LLVM 23 runtime from llvm-toolchain-bookworm-23. + - "llvm-libunwind1" # LLVM 22 runtime from llvm-toolchain-bookworm-22. - "zlib1g" diff --git a/platforms/rocm/packages.lock.json b/platforms/rocm/packages.lock.json index 8eb071d7..e22d3b1e 100644 --- a/platforms/rocm/packages.lock.json +++ b/platforms/rocm/packages.lock.json @@ -24,13 +24,13 @@ "version": "12.3.0-1ubuntu1~22.04" } ], - "key": "llvm-libunwind1_1-23.1.2_-p--p-20260916113350-p-ba019b7aafce-1_exp1_20260916233400.76_amd64", + "key": "llvm-libunwind1_1-22.1.8_-p--p-20260613092327-p-e80beda6e255-1_exp1_20260613092437.81_amd64", "name": "llvm-libunwind1", - "sha256": "72f83f34f22e01e1219c9600f9e8562505afc1eabfb5b0cfae5f4603d0694720", + "sha256": "0da2b5596faee4e53456b7bdf51458e7bd5ec06852af93f2538e74c122032df2", "urls": [ - "https://apt.llvm.org/jammy/pool/main/l/llvm-toolchain-23/llvm-libunwind1_23.1.2~++20260916113350+ba019b7aafce-1~exp1~20260916233400.76_amd64.deb" + "https://apt.llvm.org/jammy/pool/main/l/llvm-toolchain-22/llvm-libunwind1_22.1.8~++20260613092327+e80beda6e255-1~exp1~20260613092437.81_amd64.deb" ], - "version": "1:23.1.2~++20260916113350+ba019b7aafce-1~exp1~20260916233400.76" + "version": "1:22.1.8~++20260613092327+e80beda6e255-1~exp1~20260613092437.81" }, { "arch": "amd64", diff --git a/platforms/rocm/packages.yaml b/platforms/rocm/packages.yaml index d3c8bbcd..40b74c93 100644 --- a/platforms/rocm/packages.yaml +++ b/platforms/rocm/packages.yaml @@ -4,7 +4,7 @@ version: 1 sources: - - channel: llvm-toolchain-jammy-23 main + - channel: llvm-toolchain-jammy-22 main url: https://apt.llvm.org/jammy - channel: stable main url: https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391 @@ -19,7 +19,7 @@ archs: - "amd64" packages: - - "llvm-libunwind1" # LLVM 23 runtime from llvm-toolchain-jammy-23. + - "llvm-libunwind1" # LLVM 22 runtime from llvm-toolchain-jammy-22. - "libatomic1" - "libdrm-common" - "amdrocm-amdsmi10.1" From edd3e0b4e13b2ef81f7655c904c7daa522cb2ba9 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 7 Oct 2026 22:13:30 +0200 Subject: [PATCH 763/881] zml/moe: avoid unecessary bitcast (#910) let the partitioning information propagate down to manual computation --- zml/moe/cute_mxfp4.zig | 4 ++-- zml/ops.zig | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/zml/moe/cute_mxfp4.zig b/zml/moe/cute_mxfp4.zig index 9928aeb4..650378ef 100644 --- a/zml/moe/cute_mxfp4.zig +++ b/zml/moe/cute_mxfp4.zig @@ -62,9 +62,9 @@ pub fn fusedExperts( .input = input, .ids = ids, .weights = weights, - .w1 = gate_up.weight.bitCast(.u8), + .w1 = gate_up.weight, .s1 = gq.scales, - .w2 = down.weight.bitCast(.u8), + .w2 = down.weight, .s2 = dq.scales, .topk = ids.dim(.topk), .swiglu_limit = options.activation.swiglu.limit.?, diff --git a/zml/ops.zig b/zml/ops.zig index 84ff27a1..20853e95 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -2641,9 +2641,11 @@ fn manualComputationInternal( const local_output_shapes = try arena.alloc(Shape, outputs.len); const manual_mesh = manualComputationMesh(input_shapes, outputs, ctx.platform.replicated_mesh) catch @panic("manualComputation expects all inputs and outputs to use the same mesh"); + log.debug("manualComputation({s}) will use mesh {s}", .{ @typeName(@TypeOf(body_fn)), manual_mesh.name }); for (input_shapes, 0..) |shape, i| { local_input_shapes[i] = Sharding.shardedShape(shape) catch std.debug.panic("can't shard {f} for {f}", .{ shape, manual_mesh }); + log.debug("\t- {f} -> {f}", .{ shape, local_input_shapes[i] }); } for (outputs, 0..) |shape, i| { local_output_shapes[i] = Sharding.shardedShape(shape) catch std.debug.panic("can't shard {f} for {f}", .{ shape, manual_mesh }); From b774b7ad16eed79a9e479a4f359b0b116eb543b7 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Thu, 8 Oct 2026 10:39:26 +0200 Subject: [PATCH 764/881] mlir/dialects: bind missing attributes and types through the C API (#905) Some dialects (i.e. `gpu`) were missing the C bindings, this is now done. --- kernels/cute/README.md | 20 +- kernels/cute/builder.zig | 78 +- kernels/fly/builder.zig | 224 +- kernels/mosaic_tpu/README.md | 3 +- kernels/mosaic_tpu/builder.zig | 2 +- mlir/dialects/BUILD.bazel | 27 + mlir/dialects/arith.zig | 73 + mlir/dialects/arith_capi.cc | 20 + mlir/dialects/arith_capi.h | 25 + mlir/dialects/cuda.zig | 115 +- mlir/dialects/cuda_tile/cuda_tile.zig | 366 +- mlir/dialects/cuda_tile/cuda_tile_capi.cc | 193 +- mlir/dialects/cuda_tile/cuda_tile_capi.h | 63 +- mlir/dialects/cute_ir/BUILD.bazel | 10 +- mlir/dialects/cute_ir/README.md | 35 +- mlir/dialects/cute_ir/REGENERATING.md | 124 +- mlir/dialects/cute_ir/cuda.zig | 3550 ++++++++ mlir/dialects/cute_ir/cute_ir.zig | 96 +- .../include/cute_ir-c/Dialect/BUILD.bazel | 21 + .../cute_ir/include/cute_ir-c/Dialect/Cuda.h | 22 + .../cute_ir-c/Dialect/CudaAttributes.h | 52 + .../include/cute_ir-c/Dialect/CudaTypes.h | 149 + .../cute_ir/include/cute_ir-c/Dialect/NVVM.h | 21 + .../cute_ir-c/Dialect/NVVMAttributes.h | 509 ++ .../cute_ir/Dialect/Cuda/IR/BUILD.bazel | 142 + .../cute_ir/Dialect/Cuda/IR/CudaAttrs.td | 68 + .../cute_ir/Dialect/Cuda/IR/CudaDialect.h | 30 + .../cute_ir/Dialect/Cuda/IR/CudaDialect.td | 38 + .../cute_ir/Dialect/Cuda/IR/CudaEnums.td | 336 + .../cute_ir/Dialect/Cuda/IR/CudaOps.td | 3072 +++++++ .../cute_ir/Dialect/Cuda/IR/CudaTypes.td | 212 + .../cute_ir/Dialect/NVVM/IR/BUILD.bazel | 117 + .../cute_ir/Dialect/NVVM/IR/NVVMAttrs.td | 52 + .../cute_ir/Dialect/NVVM/IR/NVVMDialect.h | 27 + .../cute_ir/Dialect/NVVM/IR/NVVMDialect.td | 33 + .../cute_ir/Dialect/NVVM/IR/NVVMEnums.td | 1386 ++++ .../cute_ir/Dialect/NVVM/IR/NVVMOps.td | 6445 +++++++++++++++ mlir/dialects/cute_ir/lib/CAPI/BUILD.bazel | 29 + mlir/dialects/cute_ir/lib/CAPI/Cuda.cpp | 10 + .../cute_ir/lib/CAPI/CudaAttributes.cpp | 136 + mlir/dialects/cute_ir/lib/CAPI/CudaTypes.cpp | 378 + mlir/dialects/cute_ir/lib/CAPI/NVVM.cpp | 10 + .../cute_ir/lib/CAPI/NVVMAttributes.cpp | 1681 ++++ .../cute_ir/lib/Dialect/Cuda/IR/BUILD.bazel | 18 + .../lib/Dialect/Cuda/IR/CudaDialect.cpp | 38 + .../lib/Dialect/Cuda/IR/CudaInference.cpp | 17 + .../cute_ir/lib/Dialect/NVVM/IR/BUILD.bazel | 18 + .../lib/Dialect/NVVM/IR/NVVMDialect.cpp | 46 + .../lib/Dialect/NVVM/IR/NVVMInference.cpp | 227 + mlir/dialects/cute_ir/nvvm.zig | 7129 +++++++++++++++++ mlir/dialects/fly/BUILD.bazel | 6 +- mlir/dialects/fly/REGENERATING.md | 84 + mlir/dialects/fly/fly.zig | 161 +- mlir/dialects/fly/fly_capi.cc | 2 - mlir/dialects/fly/fly_capi.h | 1 - mlir/dialects/fly/ops.zig | 1146 +++ mlir/dialects/fly/rocdl.zig | 2 + mlir/dialects/fly/rocdl_ops.zig | 69 + mlir/dialects/gpu.zig | 117 +- mlir/dialects/gpu_capi.cc | 47 + mlir/dialects/gpu_capi.h | 35 + mlir/dialects/llvm_dialect.zig | 13 +- mlir/dialects/mosaic_tpu/BUILD.bazel | 18 + mlir/dialects/mosaic_tpu/mosaic_tpu.zig | 709 +- mlir/dialects/mosaic_tpu/mosaic_tpu_capi.cc | 225 + mlir/dialects/mosaic_tpu/mosaic_tpu_capi.h | 105 + mlir/dialects/nvvm.zig | 359 +- mlir/dialects/vector.zig | 134 +- mlir/dialects/vector_capi.cc | 44 + mlir/dialects/vector_capi.h | 32 + zml/kernel.zig | 3 +- 71 files changed, 29863 insertions(+), 942 deletions(-) create mode 100644 mlir/dialects/arith_capi.cc create mode 100644 mlir/dialects/arith_capi.h create mode 100644 mlir/dialects/cute_ir/cuda.zig create mode 100644 mlir/dialects/cute_ir/include/cute_ir-c/Dialect/Cuda.h create mode 100644 mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CudaAttributes.h create mode 100644 mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CudaTypes.h create mode 100644 mlir/dialects/cute_ir/include/cute_ir-c/Dialect/NVVM.h create mode 100644 mlir/dialects/cute_ir/include/cute_ir-c/Dialect/NVVMAttributes.h create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/BUILD.bazel create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaAttrs.td create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaDialect.h create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaDialect.td create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaEnums.td create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaOps.td create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaTypes.td create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/BUILD.bazel create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMAttrs.td create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMDialect.h create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMDialect.td create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMEnums.td create mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMOps.td create mode 100644 mlir/dialects/cute_ir/lib/CAPI/Cuda.cpp create mode 100644 mlir/dialects/cute_ir/lib/CAPI/CudaAttributes.cpp create mode 100644 mlir/dialects/cute_ir/lib/CAPI/CudaTypes.cpp create mode 100644 mlir/dialects/cute_ir/lib/CAPI/NVVM.cpp create mode 100644 mlir/dialects/cute_ir/lib/CAPI/NVVMAttributes.cpp create mode 100644 mlir/dialects/cute_ir/lib/Dialect/Cuda/IR/BUILD.bazel create mode 100644 mlir/dialects/cute_ir/lib/Dialect/Cuda/IR/CudaDialect.cpp create mode 100644 mlir/dialects/cute_ir/lib/Dialect/Cuda/IR/CudaInference.cpp create mode 100644 mlir/dialects/cute_ir/lib/Dialect/NVVM/IR/BUILD.bazel create mode 100644 mlir/dialects/cute_ir/lib/Dialect/NVVM/IR/NVVMDialect.cpp create mode 100644 mlir/dialects/cute_ir/lib/Dialect/NVVM/IR/NVVMInference.cpp create mode 100644 mlir/dialects/cute_ir/nvvm.zig create mode 100644 mlir/dialects/fly/REGENERATING.md create mode 100644 mlir/dialects/fly/ops.zig create mode 100644 mlir/dialects/fly/rocdl_ops.zig create mode 100644 mlir/dialects/gpu_capi.cc create mode 100644 mlir/dialects/gpu_capi.h create mode 100644 mlir/dialects/mosaic_tpu/mosaic_tpu_capi.cc create mode 100644 mlir/dialects/mosaic_tpu/mosaic_tpu_capi.h create mode 100644 mlir/dialects/vector_capi.cc create mode 100644 mlir/dialects/vector_capi.h diff --git a/kernels/cute/README.md b/kernels/cute/README.md index 2c968a96..52aae51c 100644 --- a/kernels/cute/README.md +++ b/kernels/cute/README.md @@ -9,8 +9,9 @@ ops to a block stack, a `Value` with fluent scalar arithmetic, and a `zml.kernel.cute.Kernel` wrapper that turns a `run` function into a `stablehlo.custom_call` XLA compiles in process with `libcute_compiler.so`. -The dialect binding lives in `mlir/dialects/cute_ir` (`cute` and -`cute_nvgpu`, types through the dialect's C API). +The dialect binding lives in `mlir/dialects/cute_ir` (`cute`, `cute_nvgpu`, +and NVIDIA's `nvvm` and `cuda`, types and attributes through the dialects' +C API). ## A kernel @@ -94,11 +95,16 @@ const c = NaiveElementwiseAdd.call(.{ .gA = a, .gB = b }, .{ .gC = a.shape() }, * **Scalars are `i32` and `f32` by default**, like `Int32`/`Float32`; `cst(dtype, v)` picks another type and `coerce` lifts literals to the other operand's type. -* **`nvvm` is not linked into ZML.** Thread indices and barriers are emitted - as unregistered ops in generic form; the compiler has the dialect. -* **The module is just the kernel.** `finish` prints `module { func.func - @name(...) {...} }`; the CuTe compiler makes every public function a kernel - entry, so there is no `gpu.module` or host launch function to write. +* **`nvvm` and `cuda` are the compiler's.** The context registers NVIDIA's + `nvvm` and `cuda` dialects (`cute.nvvm`, `cute.cuda`, carried in + `mlir/dialects/cute_ir`) and upstream's `llvm`, so every operation is + verified as it is built; nothing is left unregistered. +* **The module is just the kernel.** `finish` prints a module holding only + `func.func @name(...)`; the CuTe compiler makes every public function a + kernel entry, so there is no `gpu.module` or host launch function to write. +* **Generic form.** `finish` and `finishProgram` print the generic operation + form: NVIDIA's compiler parses with its own MLIR revision, whose custom + assembly differs from ours (`llvm.load`/`llvm.store` alignment, for one). * **Explicit result types:** Zig covers local tiling, TMA partitioning, tiled copies, tensor-memory fragments, and SM100 block-scaled MMA. CuTe derives their layouts in the compiler, so callers spell out the resulting CuTe type diff --git a/kernels/cute/builder.zig b/kernels/cute/builder.zig index 9c777491..d012b222 100644 --- a/kernels/cute/builder.zig +++ b/kernels/cute/builder.zig @@ -30,9 +30,10 @@ test { std.testing.refAllDecls(Atom); } -/// `nvvm` is not linked into ZML: its ops are emitted unregistered, in -/// generic form; the CuTe compiler has the dialect. -pub const dialects_needed = [_][]const u8{ "func", "gpu", "cute", "cute_nvgpu", "arith", "scf", "math", "cf", "vector" }; +/// `nvvm` and `cuda` are NVIDIA's (`cute.nvvm`, `cute.cuda`, handles `cute_nvvm` +/// and `cuda`); `llvm` is upstream's, whose types and operations NVIDIA's +/// compiler reads as its own. +pub const dialects_needed = [_][]const u8{ "func", "gpu", "cute", "cute_nvgpu", cute.nvvm.dialect_handle, cute.cuda.dialect_handle, "llvm", "arith", "scf", "math", "cf", "vector" }; pub const FinishError = error{InvalidMlir} || std.mem.Allocator.Error || std.Io.Writer.Error; @@ -626,8 +627,6 @@ pub const Builder = struct { tensors: []?Tensor = &.{}, pub fn open(allocator: std.mem.Allocator, ctx: *mlir.Context, name: []const u8) !Builder { - // `nvvm.*` is built unregistered. - ctx.setAllowUnregisteredDialects(true); const module: *mlir.Module = .init(.unknown(ctx)); errdefer module.deinit(); return .{ @@ -923,9 +922,6 @@ pub const Builder = struct { }; /// Build the mutable CUDA launch configuration used by `cuda.launch_ex`. - /// The segment attribute is written explicitly because the CUDA dialect is - /// supplied by the runtime CuTe compiler and is intentionally unregistered - /// in ZML's lightweight construction context. pub fn makeLaunchConfig(self: *Builder, opts: LaunchOptions) LaunchConfig { const config = self.emit(cuda.launch_cfg_create(self.ctx, .{ .max_attrs = 17, @@ -953,7 +949,7 @@ pub const Builder = struct { .config = config.inner, .inputs = values, .callee = self.parseAttribute(symbol), - .assume_kernel_attr = self.targetAttribute("#cuda.assume_kernel_attr"), + .assume_kernel_attr = cuda.assumeKernelAttr(self.ctx, true), .location = self.loc(), })); } @@ -991,41 +987,18 @@ pub const Builder = struct { pub fn finishProgram(self: *Builder) FinishError![:0]const u8 { std.debug.assert(self.entry_block == null and self.func_op == null); if (!self.module.operation().verify()) return error.InvalidMlir; - return self.renderModule(true); + return self.renderModule(); } - const target_attr_marker = "__zml_cute_target_attr__"; - - fn renderModule(self: *Builder, generic: bool) FinishError![:0]const u8 { + fn renderModule(self: *Builder) FinishError![:0]const u8 { var printed: std.Io.Writer.Allocating = .init(self.allocator); defer printed.deinit(); - // Generic form is stable across the lightweight construction dialect - // and NVIDIA's runtime compiler dialect. In particular, the two - // versions currently use different custom assembly printers for - // `cute.make_composed_layout`. - if (generic) { - try printed.writer.print("{f}", .{self.module.operation().fmt(.{ .print_generic_op_form = true })}); - } else { - try printed.writer.print("{f}", .{self.module.operation()}); - } - - // A target attribute is initially a quoted StringAttr so the local - // verifier can carry it through an AnyAttr field. Replace the whole - // quoted string with its CuTe assembly before handing the module to - // the real compiler, which owns the attribute parser. - var output: std.Io.Writer.Allocating = .init(self.allocator); - defer output.deinit(); - var rest = printed.written(); - while (std.mem.indexOf(u8, rest, target_attr_marker)) |marker| { - if (marker == 0 or rest[marker - 1] != '"') return error.InvalidMlir; - try output.writer.writeAll(rest[0 .. marker - 1]); - const payload = rest[marker + target_attr_marker.len ..]; - const end = std.mem.indexOfScalar(u8, payload, '"') orelse return error.InvalidMlir; - try output.writer.writeAll(payload[0..end]); - rest = payload[end + 1 ..]; - } - try output.writer.writeAll(rest); - return try self.allocator.dupeZ(u8, output.written()); + // Always the generic form: NVIDIA's compiler parses the module with its + // own MLIR revision, whose custom assembly differs from ours (e.g. + // `cute.make_composed_layout`, or `llvm.load`/`llvm.store` printing + // their alignment as a `` property). + try printed.writer.print("{f}", .{self.module.operation().fmt(.{ .print_generic_op_form = true })}); + return try self.allocator.dupeZ(u8, printed.written()); } // ==================== types ==================== @@ -1081,16 +1054,6 @@ pub const Builder = struct { return mlir.Attribute.parse(self.ctx, text) catch std.debug.panic("invalid MLIR attribute: {s}", .{text}); } - /// Preserve an attribute owned by the runtime CuTe compiler when the - /// lightweight ZML MLIR context has no parser hook for that attribute. - /// It is represented as a verifier-safe string while constructing the - /// module and restored to target assembly by `finish`/`finishProgram`. - pub fn targetAttribute(self: *Builder, text: []const u8) *const mlir.Attribute { - if (std.mem.indexOfScalar(u8, text, '"') != null) @panic("targetAttribute cannot contain a quote"); - const encoded = std.fmt.allocPrint(self.arena.allocator(), "__zml_cute_target_attr__{s}", .{text}) catch @panic("OOM"); - return .string(self.ctx, encoded); - } - fn algebra(self: *Builder, dims: []const i64) []const u8 { var out: std.Io.Writer.Allocating = .init(self.arena.allocator()); if (dims.len == 1) { @@ -3115,7 +3078,7 @@ pub const Builder = struct { func_op.setAttributeByName("nvvm.reqntid", .denseArray(self.ctx, .i32, &block)); if (!self.module.operation().verify()) return error.InvalidMlir; - return self.renderModule(false); + return self.renderModule(); } }; @@ -3160,15 +3123,15 @@ test "naive elementwise add matches the Python notebook kernel" { // The module round-trips through the local dialects. const ir = try b.finish(.{ 128, 1, 1 }); defer std.testing.allocator.free(ir); - try expectContains(ir, "module {\n func.func @naive_elementwise_add_kernel(%arg0: !cute.ptr>, %arg1: !cute.ptr>, %arg2: !cute.ptr>)"); - try expectContains(ir, "attributes {cute.kernel, gpu.kernel, nvvm.reqntid = array}"); + try expectContains(ir, "\"func.func\"() <{function_type = (!cute.ptr>, !cute.ptr>, !cute.ptr>) -> (), sym_name = \"naive_elementwise_add_kernel\"}>"); + try expectContains(ir, "}) {cute.kernel, gpu.kernel, nvvm.reqntid = array} : () -> ()"); try expectContains(ir, "\"nvvm.read.ptx.sreg.tid.x\"() : () -> i32"); try expectContains(ir, "!cute.layout<\"(16,8):(8,1)\">"); try expectContains(ir, "!cute.coord<\"(?,?)\">"); try expectContains(ir, "cute.memref.load"); try expectContains(ir, "cute.memref.store"); try expectContains(ir, "arith.addf"); - try expectContains(ir, " return\n }\n}\n"); + try expectContains(ir, "\"func.return\"() : () -> ()\n })"); try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); const parsed = try mlir.Module.parse(ctx, ir); defer parsed.deinit(); @@ -3206,7 +3169,7 @@ test "guard, shared memory, sync, layouts, for loop" { defer std.testing.allocator.free(kernel); try expectContains(kernel, "cute_nvgpu.arch.alloc_smem"); try expectContains(kernel, "!cute.ptr>"); - try expectContains(kernel, "\"nvvm.barrier\"() {operandSegmentSizes = array} : () -> ()"); + try expectContains(kernel, "\"nvvm.barrier\"() <{operandSegmentSizes = array}> : () -> ()"); try expectContains(kernel, "bar.sync 2, 128;"); try expectContains(kernel, "griddepcontrol.launch_dependents;"); try expectContains(kernel, "griddepcontrol.wait;"); @@ -3273,7 +3236,7 @@ test "casts, select, if-else, integer widening" { const kernel = try b.finish(.{ 1, 1, 1 }); defer std.testing.allocator.free(kernel); try expectContains(kernel, "arith.extsi"); - try expectContains(kernel, "arith.constant 3000000000 : i64"); + try expectContains(kernel, "\"arith.constant\"() <{value = 3000000000 : i64}> : () -> i64"); try expectContains(kernel, "arith.sitofp"); try expectContains(kernel, "arith.truncf"); try expectContains(kernel, "arith.extf"); @@ -3283,7 +3246,8 @@ test "casts, select, if-else, integer widening" { try expectContains(kernel, "arith.extui"); try expectContains(kernel, "arith.select"); try expectContains(kernel, "shfl.sync.bfly.b32"); - try expectContains(kernel, "} else {"); + try expectContains(kernel, "\"scf.if\""); + try expectContains(kernel, "}, {"); const parsed = try mlir.Module.parse(ctx, kernel); defer parsed.deinit(); try std.testing.expect(parsed.operation().verify()); diff --git a/kernels/fly/builder.zig b/kernels/fly/builder.zig index bb6e68f0..c7c087dc 100644 --- a/kernels/fly/builder.zig +++ b/kernels/fly/builder.zig @@ -107,61 +107,63 @@ pub const Value = struct { // Every op below takes a Value or a comptime Layout/Tile/tuple literal, // and FlyDSL infers the result type. - fn unary(self: Value, comptime mnemonic: []const u8) Value { + fn unary(self: Value, comptime build: anytype) Value { const k = self.kern(); - return k.emit(fly.inferred(k.ctx, mnemonic, &.{self.inner}, .empty, k.loc())); + return k.emit(build(k.ctx, self.inner, k.loc())); } - fn binary(self: Value, comptime mnemonic: []const u8, rhs: anytype) Value { + fn binary(self: Value, comptime build: anytype, rhs: anytype) Value { const k = self.kern(); const r = k.lift(rhs); - return k.emit(fly.inferred(k.ctx, mnemonic, &.{ self.inner, r.inner }, .empty, k.loc())); + return k.emit(build(k.ctx, self.inner, r.inner, k.loc())); } pub fn flatDivide(self: Value, tiler: anytype) Value { - return self.binary("flat_divide", tiler); + return self.binary(fly.ops.flat_divide, tiler); } pub fn logicalDivide(self: Value, tiler: anytype) Value { - return self.binary("logical_divide", tiler); + return self.binary(fly.ops.logical_divide, tiler); } pub fn zippedDivide(self: Value, tiler: anytype) Value { - return self.binary("zipped_divide", tiler); + return self.binary(fly.ops.zipped_divide, tiler); } pub fn tiledDivide(self: Value, tiler: anytype) Value { - return self.binary("tiled_divide", tiler); + return self.binary(fly.ops.tiled_divide, tiler); } pub fn logicalProduct(self: Value, tiler: anytype) Value { - return self.binary("logical_product", tiler); + return self.binary(fly.ops.logical_product, tiler); } pub fn zippedProduct(self: Value, tiler: anytype) Value { - return self.binary("zipped_product", tiler); + return self.binary(fly.ops.zipped_product, tiler); } pub fn tiledProduct(self: Value, tiler: anytype) Value { - return self.binary("tiled_product", tiler); + return self.binary(fly.ops.tiled_product, tiler); } pub fn flatProduct(self: Value, tiler: anytype) Value { - return self.binary("flat_product", tiler); + return self.binary(fly.ops.flat_product, tiler); } pub fn blockedProduct(self: Value, tiler: anytype) Value { - return self.binary("blocked_product", tiler); + return self.binary(fly.ops.blocked_product, tiler); } pub fn rakedProduct(self: Value, tiler: anytype) Value { - return self.binary("raked_product", tiler); + return self.binary(fly.ops.raked_product, tiler); } pub fn composition(self: Value, inner: anytype) Value { - return self.binary("composition", inner); + return self.binary(fly.ops.composition, inner); } pub fn rightInverse(self: Value) Value { - return self.unary("right_inverse"); + return self.unary(fly.ops.right_inverse); } pub fn leftInverse(self: Value) Value { - return self.unary("left_inverse"); + return self.unary(fly.ops.left_inverse); } pub fn coalesce(self: Value) Value { - return self.unary("coalesce"); + const k = self.kern(); + return k.emit(fly.ops.coalesce(k.ctx, self.inner, null, k.loc())); } pub fn complement(self: Value) Value { - return self.unary("complement"); + const k = self.kern(); + return k.emit(fly.ops.complement(k.ctx, self.inner, null, k.loc())); } /// `t[coord]` with a `null` somewhere; `coord` is a tuple literal @@ -169,66 +171,66 @@ pub const Value = struct { pub fn slice(self: Value, coord: anytype) Value { const k = self.kern(); const c = k.intTuple(coord); - return k.emit(fly.inferred(k.ctx, "slice", &.{ self.inner, c.inner }, .empty, k.loc())); + return k.emit(fly.ops.slice(k.ctx, self.inner, c.inner, k.loc())); } pub fn dice(self: Value, coord: anytype) Value { const k = self.kern(); const c = k.intTuple(coord); - return k.emit(fly.inferred(k.ctx, "dice", &.{ self.inner, c.inner }, .empty, k.loc())); + return k.emit(fly.ops.dice(k.ctx, self.inner, c.inner, k.loc())); } /// The sub-tuple / sub-layout at a mode path. pub fn get(self: Value, mode: []const i32) Value { const k = self.kern(); - return k.emit(fly.get(k.ctx, self.inner, mode, k.loc())); + return k.emit(fly.ops.get(k.ctx, self.inner, .denseArray(k.ctx, .i32, mode), k.loc())); } pub fn selectModes(self: Value, indices: []const i32) Value { const k = self.kern(); - return k.emit(fly.select(k.ctx, self.inner, indices, k.loc())); + return k.emit(fly.ops.select(k.ctx, self.inner, .denseArray(k.ctx, .i32, indices), k.loc())); } pub fn take(self: Value, begin: i32, end: i32) Value { const k = self.kern(); - return k.emit(fly.takeOrGroup(k.ctx, "take", self.inner, begin, end, k.loc())); + return k.emit(fly.ops.take(k.ctx, self.inner, .int(k.ctx, .i32, begin), .int(k.ctx, .i32, end), k.loc())); } pub fn group(self: Value, begin: i32, end: i32) Value { const k = self.kern(); - return k.emit(fly.takeOrGroup(k.ctx, "group", self.inner, begin, end, k.loc())); + return k.emit(fly.ops.group(k.ctx, self.inner, .int(k.ctx, .i32, begin), .int(k.ctx, .i32, end), k.loc())); } /// Emits an op; `shapeStatic` reads the type instead. pub fn emitShape(self: Value) Value { - return self.unary("get_shape"); + return self.unary(fly.ops.get_shape); } pub fn emitStride(self: Value) Value { - return self.unary("get_stride"); + return self.unary(fly.ops.get_stride); } pub fn emitLayout(self: Value) Value { - return self.unary("get_layout"); + return self.unary(fly.ops.get_layout); } pub fn emitIter(self: Value) Value { - return self.unary("get_iter"); + return self.unary(fly.ops.get_iter); } pub fn emitSize(self: Value) Value { - return self.unary("size"); + return self.unary(fly.ops.size); } pub fn cosize(self: Value) Value { - return self.unary("cosize"); + return self.unary(fly.ops.cosize); } pub fn productEach(self: Value) Value { - return self.unary("int_tuple_product_each"); + return self.unary(fly.ops.int_tuple_product_each); } /// A register tensor shaped like `self`. pub fn makeFragmentLike(self: Value, dtype: ?DType) Value { const k = self.kern(); - const t: ?*const mlir.Type = if (dtype) |d| d.toMlir(k.ctx) else null; - return k.emit(fly.makeFragmentLike(k.ctx, self.inner, t, k.loc())); + const dtype_attr: ?*const mlir.Attribute = if (dtype) |d| .typeAttr(d.toMlir(k.ctx)) else null; + return k.emit(fly.ops.make_fragment_like(k.ctx, self.inner, dtype_attr, k.loc())); } pub fn elemLess(self: Value, rhs: anytype) Value { const k = self.kern(); const r = k.intTuple(rhs); - return k.emit(fly.typed(k.ctx, "elem_less", &.{ self.inner, r.inner }, &.{.int(k.ctx, .i1)}, .empty, k.loc())); + return k.emit(fly.ops.elem_less(k.ctx, self.inner, r.inner, .int(k.ctx, .i1), k.loc())); } // ---------------------------------------------------------------- tensor access @@ -237,23 +239,23 @@ pub const Value = struct { pub fn at(self: Value, coord: anytype) Value { const k = self.kern(); const c = k.intTuple(coord); - return k.emit(fly.inferred(k.ctx, "memref.load", &.{ self.inner, c.inner }, .empty, k.loc())); + return k.emit(fly.ops.memref_load(k.ctx, self.inner, c.inner, k.loc())); } pub fn set(self: Value, coord: anytype, value: Value) void { const k = self.kern(); const c = k.intTuple(coord); - k.emitNone(fly.effect(k.ctx, "memref.store", &.{ value.inner, self.inner, c.inner }, .empty, k.loc())); + k.emitNone(fly.ops.memref_store(k.ctx, value.inner, self.inner, c.inner, k.loc())); } /// The whole register tensor as a `vector`. pub fn load(self: Value) Value { - return self.unary("memref.load_vec"); + return self.unary(fly.ops.memref_load_vec); } pub fn store(self: Value, vector: Value) void { const k = self.kern(); - k.emitNone(fly.effect(k.ctx, "memref.store_vec", &.{ vector.inner, self.inner }, .empty, k.loc())); + k.emitNone(fly.ops.memref_store_vec(k.ctx, vector.inner, self.inner, k.loc())); } /// Store a splat vector of the tensor's static size. @@ -272,24 +274,24 @@ pub const Value = struct { pub fn addOffset(self: Value, offset: anytype) Value { const k = self.kern(); const o = k.intTuple(offset); - return k.emit(fly.inferred(k.ctx, "add_offset", &.{ self.inner, o.inner }, .empty, k.loc())); + return k.emit(fly.ops.add_offset(k.ctx, self.inner, o.inner, k.loc())); } /// Scalar load. pub fn ptrLoad(self: Value) Value { const k = self.kern(); const elem = self.elemDType().toMlir(k.ctx); - return k.emit(fly.typed(k.ctx, "ptr.load", &.{self.inner}, &.{elem}, .empty, k.loc())); + return k.emit(fly.ops.ptr_load(k.ctx, self.inner, elem, k.loc())); } pub fn ptrStore(self: Value, value: Value) void { const k = self.kern(); - k.emitNone(fly.effect(k.ctx, "ptr.store", &.{ value.inner, self.inner }, .empty, k.loc())); + k.emitNone(fly.ops.ptr_store(k.ctx, value.inner, self.inner, k.loc())); } /// A tensor over this pointer. pub fn view(self: Value, lay: anytype) Value { - return self.binary("make_view", lay); + return self.binary(fly.ops.make_view, lay); } // ---------------------------------------------------------------- arithmetic @@ -529,22 +531,13 @@ pub const Value = struct { return self.shuffle(.xor, offset, width); } - pub const ShuffleMode = enum { xor, up, down, idx }; + pub const ShuffleMode = gpu.ShuffleMode; pub fn shuffle(self: Value, mode: ShuffleMode, offset: anytype, width: i32) Value { const k = self.kern(); const off: Value = if (@TypeOf(offset) == Value) offset else k.constant(.i32, offset); const wid = k.constant(.i32, width); - const mode_attr = switch (mode) { - inline else => |m| fly.parseAttr(k.ctx, "#gpu>"), - }; - const op = fly.make(k.ctx, "gpu.shuffle", .{ - .operands = .{ .flat = &.{ self.inner, off.inner, wid.inner } }, - .results = .{ .flat = &.{ self.type_(), .int(k.ctx, .i1) } }, - .attributes = &.{.named(k.ctx, "mode", mode_attr)}, - .location = k.loc(), - }); - return k.emit(op); + return k.emit(gpu.shuffle(k.ctx, self.inner, off.inner, wid.inner, mode, k.loc())); } pub const ReduceKind = enum { add, mul, minf, maxf, minsi, maxsi, and_, or_, xor }; @@ -600,17 +593,17 @@ pub const ThrCopy = struct { pub fn partitionS(self: ThrCopy, src: Value) Value { const k = self.tiled_copy.kern(); - return k.emit(fly.inferred(k.ctx, "tiled_copy.partition_src", &.{ self.tiled_copy.inner, src.inner, self.thr.inner }, .empty, k.loc())); + return k.emit(fly.ops.tiled_copy_partition_src(k.ctx, self.tiled_copy.inner, src.inner, self.thr.inner, k.loc())); } pub fn partitionD(self: ThrCopy, dst: Value) Value { const k = self.tiled_copy.kern(); - return k.emit(fly.inferred(k.ctx, "tiled_copy.partition_dst", &.{ self.tiled_copy.inner, dst.inner, self.thr.inner }, .empty, k.loc())); + return k.emit(fly.ops.tiled_copy_partition_dst(k.ctx, self.tiled_copy.inner, dst.inner, self.thr.inner, k.loc())); } pub fn retile(self: ThrCopy, t: Value) Value { const k = self.tiled_copy.kern(); - return k.emit(fly.inferred(k.ctx, "tiled_copy.retile", &.{ self.tiled_copy.inner, t.inner }, .empty, k.loc())); + return k.emit(fly.ops.tiled_copy_retile(k.ctx, self.tiled_copy.inner, t.inner, k.loc())); } }; @@ -627,10 +620,9 @@ pub const TiledMma = struct { fn makeFragment(self: TiledMma, operand: fly.MmaOperand, input: Value, opts: FragmentOpts) Value { const k = self.value.kern(); - var a: fly.Attrs = .empty; - a.appendAssumeCapacity(.named(k.ctx, "operand_id", Builder.must(fly.attributes.mmaOperandAttr(k.ctx, operand)))); - if (opts.stages) |st| a.appendAssumeCapacity(.named(k.ctx, "stages", .int(k.ctx, .i32, st))); - return k.emit(fly.inferred(k.ctx, "mma.make_fragment", &.{ self.value.inner, input.inner }, a, k.loc())); + const operand_id = Builder.must(fly.attributes.mmaOperandAttr(k.ctx, operand)); + const stages: ?*const mlir.Attribute = if (opts.stages) |st| .int(k.ctx, .i32, st) else null; + return k.emit(fly.ops.mma_make_fragment(k.ctx, self.value.inner, input.inner, operand_id, stages, k.loc())); } /// Per-thread register fragment for the A operand. @@ -657,9 +649,8 @@ pub const ThrMma = struct { fn partition(self: ThrMma, operand: fly.MmaOperand, t: Value) Value { const k = self.tiled_mma.kern(); - return k.emit(fly.inferred(k.ctx, "tiled_mma.partition", &.{ self.tiled_mma.inner, t.inner, self.thr.inner }, fly.attrs(&.{ - .named(k.ctx, "operand_id", fly.mmaOperandAttr(k.ctx, operand)), - }), k.loc())); + const operand_id = Builder.must(fly.attributes.mmaOperandAttr(k.ctx, operand)); + return k.emit(fly.ops.tiled_mma_partition(k.ctx, self.tiled_mma.inner, t.inner, self.thr.inner, operand_id, k.loc())); } pub fn partitionA(self: ThrMma, a: Value) Value { @@ -900,7 +891,7 @@ pub const Builder = struct { /// `fly.static` of an already-built type. pub fn staticTy(self: *Builder, t: *const mlir.Type) Value { - return self.emit(fly.static(self.ctx, t, self.loc())); + return self.emit(fly.ops.static(self.ctx, t, self.loc())); } pub fn static(self: *Builder, v: anytype) Value { @@ -947,7 +938,7 @@ pub const Builder = struct { var operands: stdx.BoundedArray(*const mlir.Value, 32) = .empty; const spec = self.intTupleSpec(&operands, elems); const t = self.intTupleType(spec); - return self.emit(fly.makeIntTuple(self.ctx, operands.constSlice(), t, self.loc())); + return self.emit(fly.ops.make_int_tuple(self.ctx, operands.constSlice(), t, self.loc())); } /// `intTuple`, spelled for coordinates. @@ -1038,7 +1029,8 @@ pub const Builder = struct { pub fn emitFast(self: *Builder, op: *mlir.Operation) Value { if (self.fast_math and op.numResults() > 0 and (Value{ .inner = op.result(0), .kernel = self }).isFloatElem()) { - op.setAttributeByName("fastmath", fly.parseAttr(self.ctx, "#arith.fastmath")); + const fast = arith.FastMathAttr.get(self.ctx, .{ .value = .fast }) catch unreachable; + op.setAttributeByName("fastmath", fast.attribute()); } return self.emit(op); } @@ -1063,22 +1055,14 @@ pub const Builder = struct { /// A matrix atom with SSA operands; unlike `gemm`, this needs no register /// allocas and can carry its accumulator through control flow. pub fn mmaAtomCall(self: *Builder, atom: Value, a: Value, b: Value, c: Value) Value { - return self.emit(fly.typed(self.ctx, "mma_atom_call_ssa", &.{ atom.inner, a.inner, b.inner, c.inner }, &.{c.type_()}, fly.attrs(&.{ - .named(self.ctx, "operandSegmentSizes", .denseArray(self.ctx, .i32, &.{ 1, 0, 1, 1, 1 })), - }), self.loc())); + return self.emit(fly.ops.mma_atom_call_ssa(self.ctx, atom.inner, null, &.{a.inner}, &.{b.inner}, c.inner, &.{c.type_()}, self.loc())); } /// AMD byte permutation. ROCDL has no corresponding operation at our pin. pub fn perm(self: *Builder, hi: Value, lo: Value, selector: Value) Value { - return self.emit(fly.make(self.ctx, "llvm.inline_asm", .{ - .operands = .{ .flat = &.{ hi.inner, lo.inner, selector.inner } }, - .results = .{ .flat = &.{.int(self.ctx, .i32)} }, - .attributes = &.{ - .named(self.ctx, "asm_string", .string(self.ctx, "v_perm_b32 $0, $1, $2, $3")), - .named(self.ctx, "constraints", .string(self.ctx, "=v,v,v,v")), - }, - .location = self.loc(), - })); + return self.emit(dialects.llvm.inline_asm(self.ctx, "v_perm_b32 $0, $1, $2, $3", &.{ hi.inner, lo.inner, selector.inner }, .int(self.ctx, .i32), .{ + .constraints = "=v,v,v,v", + }, self.loc())); } /// CDNA3 packed FP8 conversion, selecting two bytes from the low/high word. @@ -1101,9 +1085,7 @@ pub const Builder = struct { .workgroup => 3, .agent => 1, }; - const llvm_ptr = self.emit(fly.inferred(self.ctx, "to_llvm_ptr", &.{ptr.inner}, fly.attrs(&.{ - .named(self.ctx, "llvm_address_space", .int(self.ctx, .i32, space)), - }), self.loc())); + const llvm_ptr = self.emit(fly.ops.to_llvm_ptr(self.ctx, ptr.inner, .int(self.ctx, .i32, space), self.loc())); return self.emit(fly.make(self.ctx, "llvm.atomicrmw", .{ .operands = .{ .flat = &.{ llvm_ptr.inner, value.inner } }, .results = .{ .flat = &.{value.type_()} }, @@ -1131,14 +1113,14 @@ pub const Builder = struct { .named(self.ctx, "allocBytes", .int(self.ctx, .i64, bytes)), .named(self.ctx, "allocAlign", .int(self.ctx, .i64, alignment)), }); - const raw = self.emit(fly.makePtr(self.ctx, &.{}, raw_ty, dict, self.loc())); + const raw = self.emit(fly.ops.make_ptr(self.ctx, &.{}, raw_ty, dict, self.loc())); const typed = self.recast(raw, dtype); return typed.view(self.static(Layout{ .shape = .static(n), .stride = .static(1) })); } /// A single atom-sized copy. pub fn copyAtomCall(self: *Builder, atom: Value, src: Value, dst: Value) void { - self.emitNone(fly.effect(self.ctx, "copy_atom_call", &.{ atom.inner, src.inner, dst.inner }, .empty, self.loc())); + self.emitNone(fly.ops.copy_atom_call(self.ctx, atom.inner, src.inner, dst.inner, null, self.loc())); } /// One atom-sized load straight into a `vector`. @@ -1150,23 +1132,12 @@ pub const Builder = struct { const n = src.sizeStatic(); const dt = src.elemDType(); const vec_ty = mlir.Type.vector(&.{n}, dt.toMlir(self.ctx)); - const op = fly.make(self.ctx, "fly.copy_atom_call_ssa", .{ - .operands = .{ .flat = &.{ atom.inner, src.inner } }, - .results = .{ .flat = &.{vec_ty} }, - .attributes = &.{.named(self.ctx, "operandSegmentSizes", .denseArray(self.ctx, .i32, &.{ 1, 1, 0, 0 }))}, - .location = self.loc(), - }); - return self.emit(op); + return self.emit(fly.ops.copy_atom_call_ssa(self.ctx, atom.inner, src.inner, null, null, &.{vec_ty}, self.loc())); } /// One atom-sized store of a `vector`; see `copyAtomLoad`. pub fn copyAtomStore(self: *Builder, atom: Value, vec: Value, dst: Value) void { - const op = fly.make(self.ctx, "fly.copy_atom_call_ssa", .{ - .operands = .{ .flat = &.{ atom.inner, vec.inner, dst.inner } }, - .attributes = &.{.named(self.ctx, "operandSegmentSizes", .denseArray(self.ctx, .i32, &.{ 1, 1, 1, 0 }))}, - .location = self.loc(), - }); - self.emitNone(op); + self.emitNone(fly.ops.copy_atom_call_ssa(self.ctx, atom.inner, vec.inner, dst.inner, null, &.{}, self.loc())); } // ---------------------------------------------------------------- tensors @@ -1179,7 +1150,7 @@ pub const Builder = struct { /// A coordinate tensor: value == logical coord. pub fn identity(self: *Builder, shape: anytype) Value { const shp = self.intTuple(shape); - const lay = self.emit(fly.inferred(self.ctx, "make_identity_layout", &.{shp.inner}, .empty, self.loc())); + const lay = self.emit(fly.ops.make_identity_layout(self.ctx, shp.inner, self.loc())); const rank_ = shp.intTupleStatic().rank(); const zero: IntTuple = if (rank_ == 1) .static(0) else blk: { const zeros = self.alloc(IntTuple, rank_); @@ -1187,7 +1158,7 @@ pub const Builder = struct { break :blk .{ .tup = zeros }; }; const base_ty = self.intTupleType(zero); - const base = self.emit(fly.makeIntTuple(self.ctx, &.{}, base_ty, self.loc())); + const base = self.emit(fly.ops.make_int_tuple(self.ctx, &.{}, base_ty, self.loc())); return base.view(lay); } @@ -1196,21 +1167,21 @@ pub const Builder = struct { pub fn rmemTensor(self: *Builder, comptime lay: Layout, dtype: DType) Value { const lay_v = self.static(lay); const t = self.memRefType(dtype, .register, lay); - return self.emit(fly.typed(self.ctx, "memref.alloca", &.{lay_v.inner}, &.{t}, .empty, self.loc())); + return self.emit(fly.ops.memref_alloca(self.ctx, lay_v.inner, t, self.loc())); } /// `rmemTensor` with a runtime `Layout`. pub fn rmemTensorRuntime(self: *Builder, lay: Layout, dtype: DType) Value { const lay_v = self.static(lay); const t = self.memRefType(dtype, .register, lay); - return self.emit(fly.typed(self.ctx, "memref.alloca", &.{lay_v.inner}, &.{t}, .empty, self.loc())); + return self.emit(fly.ops.memref_alloca(self.ctx, lay_v.inner, t, self.loc())); } /// Reinterpret a pointer as another element type, keeping the address /// space, alignment and swizzle. pub fn recast(self: *Builder, ptr: Value, dtype: DType) Value { const t = must(fly.ptrWithElem(self.ctx, ptr.type_(), dtype.toMlir(self.ctx))).type_(); - return self.emit(fly.typed(self.ctx, "recast_iter", &.{ptr.inner}, &.{t}, .empty, self.loc())); + return self.emit(fly.ops.recast_iter(self.ctx, ptr.inner, t, self.loc())); } // ---------------------------------------------------------------- atoms @@ -1233,19 +1204,19 @@ pub const Builder = struct { pub fn copyAtom(self: *Builder, op: CopyOp, dtype: DType) Value { const bits: i32 = @intCast(dtype.bitWidth()); const t = must(fly.types.CopyAtomType.get(self.ctx, .{ .copyOp = op.type_(self.ctx), .valBits = bits })).type_(); - return self.emit(fly.makeCopyAtom(self.ctx, t, bits, self.loc())); + return self.emit(fly.ops.make_copy_atom(self.ctx, &.{}, t, .int(self.ctx, .i32, bits), self.loc())); } /// `op` is an MMA op type, such as a `fly.rocdl.MmaOpCDNA3MFMAType`. pub fn mmaAtom(self: *Builder, op: *const mlir.Type) Value { const t = must(fly.types.MmaAtomType.get(self.ctx, .{ .mmaOp = op })).type_(); - return self.emit(fly.makeMmaAtom(self.ctx, t, self.loc())); + return self.emit(fly.ops.make_mma_atom(self.ctx, t, self.loc())); } pub fn tiledCopy(self: *Builder, atom: Value, layout_tv: anytype, tile_mn: anytype) TiledCopy { const tv = self.lift(layout_tv); const tl = self.lift(tile_mn); - const v = self.emit(fly.inferred(self.ctx, "make_tiled_copy", &.{ atom.inner, tv.inner, tl.inner }, .empty, self.loc())); + const v = self.emit(fly.ops.make_tiled_copy(self.ctx, atom.inner, tv.inner, tl.inner, self.loc())); return .{ .value = v, .tile = tl }; } @@ -1291,7 +1262,7 @@ pub const Builder = struct { const one = self.intTuple(1); const tile_modes = self.alloc(Tile, modes.len); for (modes, 0..) |m, i| { - const lay = self.emit(fly.inferred(self.ctx, "make_layout", &.{ tile_size.selectModes(&.{m}).inner, one.inner }, .empty, self.loc())); + const lay = self.emit(fly.ops.make_layout(self.ctx, tile_size.selectModes(&.{m}).inner, one.inner, self.loc())); tile_modes[i] = .{ .layout = self.readLayout(lay.type_()) }; } return self.tiledCopy(copy_atom, layout_tv, Tile{ .modes = tile_modes }); @@ -1319,16 +1290,13 @@ pub const Builder = struct { const flags = self.constant(.i32, 0x27000); const elem = t.elemDType(); const buf_ty = self.ptrType(elem, (must(fly.rocdl.BufferDescAddressAttr.get(self.ctx))).attribute()); - const buf_ptr = self.emit(fly.makePtr(self.ctx, &.{ ptr.inner, c0.inner, nrec.inner, flags.inner }, buf_ty, null, self.loc())); + const buf_ptr = self.emit(fly.ops.make_ptr(self.ctx, &.{ ptr.inner, c0.inner, nrec.inner, flags.inner }, buf_ty, null, self.loc())); return buf_ptr.view(lay); } pub fn tiledMma(self: *Builder, atom: Value, atom_layout: anytype, permutation: ?Value) TiledMma { const al = self.lift(atom_layout); - const v = if (permutation) |p| - self.emit(fly.inferred(self.ctx, "make_tiled_mma", &.{ atom.inner, al.inner, p.inner }, .empty, self.loc())) - else - self.emit(fly.inferred(self.ctx, "make_tiled_mma", &.{ atom.inner, al.inner }, .empty, self.loc())); + const v = self.emit(fly.ops.make_tiled_mma(self.ctx, atom.inner, al.inner, if (permutation) |p| p.inner else null, self.loc())); return .{ .value = v }; } @@ -1337,13 +1305,11 @@ pub const Builder = struct { }; pub fn copy(self: *Builder, atom: Value, src: Value, dst: Value, opts: CopyOpts) void { - self.emitNone(fly.copy(self.ctx, atom.inner, src.inner, dst.inner, .{ - .pred = if (opts.pred) |p| p.inner else null, - }, self.loc())); + self.emitNone(fly.ops.copy(self.ctx, atom.inner, src.inner, dst.inner, if (opts.pred) |p| p.inner else null, self.loc())); } pub fn gemm(self: *Builder, atom: Value, d: Value, a: Value, b: Value, c: Value) void { - self.emitNone(fly.gemm(self.ctx, atom.inner, d.inner, a.inner, b.inner, c.inner, .{}, self.loc())); + self.emitNone(fly.ops.gemm(self.ctx, atom.inner, d.inner, &.{a.inner}, &.{b.inner}, c.inner, null, null, self.loc())); } // ---------------------------------------------------------------- control flow @@ -1881,6 +1847,34 @@ test "scalar conversions and constants" { } } +test "shuffles and fast math carry their attributes" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "shuffle"); + defer b.deinit(); + _ = try b.declareArgs(.{ + .x = .{ .tensor = .{ .dtype = .f32, .dims = &.{16} } }, + }); + + b.setFastMath(true); + const x = b.constant(.f32, 1.0); + _ = x.shuffleXor(1, 64).add(x.shuffle(.idx, 3, 64)); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + for ([_][]const u8{ + "gpu.shuffle xor", + "gpu.shuffle idx", + "arith.addf %shuffleResult, %shuffleResult_1 fastmath", + }) |needle| { + if (std.mem.indexOf(u8, ir, needle) == null) { + std.debug.print("missing `{s}` in:\n{s}\n", .{ needle, ir }); + return error.TestUnexpectedResult; + } + } +} + test "static queries read inferred types" { const ctx = try testContext(); defer ctx.deinit(); diff --git a/kernels/mosaic_tpu/README.md b/kernels/mosaic_tpu/README.md index f6893259..a6b01a00 100644 --- a/kernels/mosaic_tpu/README.md +++ b/kernels/mosaic_tpu/README.md @@ -200,7 +200,8 @@ Pallas's `len(grid_mapping.block_mappings)`), and re-emits the The default `memory_space` for a `.ref` is `.vmem`. Override with `.smem`, `.cmem`, `.hbm`, `.semaphore_mem`, `.vmem_shared`, or `.any` per the `MemorySpace` enum. Iteration semantics (`.parallel`, `.arbitrary`, -`.sequential`) attach `tpu.dimension_semantics` to the iter arg. +`.core_parallel`, `.subcore_parallel`) attach `tpu.dimension_semantics` to +the iter arg. **Runtime payloads:** the variant tags are fixed per kernel, but shape / dtype / memory_space values can be runtime — typical for kernels diff --git a/kernels/mosaic_tpu/builder.zig b/kernels/mosaic_tpu/builder.zig index 5da2c899..52307bca 100644 --- a/kernels/mosaic_tpu/builder.zig +++ b/kernels/mosaic_tpu/builder.zig @@ -383,7 +383,7 @@ pub const MatmulOpts = struct { transpose_lhs_hint: bool = false, precision: ?ContractPrecision = null, /// Optional `#tpu.dot_dimension_numbers<...>` attribute. When omitted the - /// canonicalizer derives one. Build with `mlir.Attribute.parse`. + /// canonicalizer derives one. Build with `tpu.dotDimensionNumbers`. dimension_numbers: ?*const mlir.Attribute = null, }; diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/BUILD.bazel index 9938dbaa..220394e8 100644 --- a/mlir/dialects/BUILD.bazel +++ b/mlir/dialects/BUILD.bazel @@ -1,5 +1,31 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") +# What upstream's C API leaves out of the dialects bound here. +cc_library( + name = "capi", + srcs = [ + "arith_capi.cc", + "gpu_capi.cc", + "vector_capi.cc", + ], + hdrs = [ + "arith_capi.h", + "gpu_capi.h", + "vector_capi.h", + ], + strip_include_prefix = ".", + tags = ["manual"], + visibility = ["//mlir/dialects:__subpackages__"], + deps = [ + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:CAPIIR", + "@llvm-project//mlir:GPUDialect", + "@llvm-project//mlir:VectorDialect", + ], + alwayslink = True, +) + zig_library( name = "dialects", srcs = [ @@ -22,6 +48,7 @@ zig_library( tags = ["manual"], visibility = ["//visibility:public"], deps = [ + ":capi", "//mlir", "//mlir:c", "//mlir/dialects/cuda_tile", diff --git a/mlir/dialects/arith.zig b/mlir/dialects/arith.zig index 14e2bf4b..aee1c869 100644 --- a/mlir/dialects/arith.zig +++ b/mlir/dialects/arith.zig @@ -42,6 +42,52 @@ pub const CmpFPredicate = enum(i64) { always_true = 15, }; +// ============================================================================= +// Flag attributes +// ============================================================================= + +/// `mlir::arith::FastMathFlags`, one bit per flag. +pub const FastMathFlags = packed struct(u32) { + reassoc: bool = false, + nnan: bool = false, + ninf: bool = false, + nsz: bool = false, + arcp: bool = false, + contract: bool = false, + afn: bool = false, + _padding: u25 = 0, + + pub const none: FastMathFlags = .{}; + pub const fast: FastMathFlags = .{ .reassoc = true, .nnan = true, .ninf = true, .nsz = true, .arcp = true, .contract = true, .afn = true }; +}; + +/// `#arith.fastmath<...>`. +pub const FastMathAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAArithFastMath; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "fastmath"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: FastMathFlags, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirArithFastMathAttrGet(ctx.ptr(), @bitCast(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) FastMathFlags { + return @bitCast(c.mlirArithFastMathAttrGetValue(self.ptr())); + } +}; + // ============================================================================= // Constants // ============================================================================= @@ -405,3 +451,30 @@ pub fn select( test { std.testing.refAllDecls(@This()); } + +fn expectPrints(expected: []const u8, value: anytype) !void { + var buf: [128]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(expected, w.buffered()); +} + +test "arith fastmath attribute is built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + const cases = [_]struct { FastMathFlags, []const u8 }{ + .{ .none, "#arith.fastmath" }, + .{ .fast, "#arith.fastmath" }, + .{ .{ .reassoc = true }, "#arith.fastmath" }, + .{ .{ .nnan = true, .ninf = true, .afn = true }, "#arith.fastmath" }, + .{ .{ .nsz = true, .arcp = true, .contract = true }, "#arith.fastmath" }, + }; + for (cases) |case| { + const attr = try FastMathAttr.get(ctx, .{ .value = case[0] }); + try std.testing.expectEqual(case[0], attr.getValue()); + try std.testing.expect(attr.attribute().isA(FastMathAttr) != null); + try expectPrints(case[1], attr); + } + try std.testing.expectError(error.InvalidMlir, FastMathAttr.get(ctx, .{ .value = .{ ._padding = 1 } })); +} diff --git a/mlir/dialects/arith_capi.cc b/mlir/dialects/arith_capi.cc new file mode 100644 index 00000000..7fb07c15 --- /dev/null +++ b/mlir/dialects/arith_capi.cc @@ -0,0 +1,20 @@ +#include "arith_capi.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/Dialect/Arith/IR/Arith.h" + +bool mlirAttributeIsAArithFastMath(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirArithFastMathAttrGet(MlirContext ctx, uint32_t value) { + mlir::MLIRContext *context = unwrap(ctx); + std::optional flags = mlir::arith::symbolizeFastMathFlags(value); + if (!context || !flags) return {nullptr}; + context->getOrLoadDialect(); + return wrap(mlir::arith::FastMathFlagsAttr::get(context, *flags)); +} + +uint32_t mlirArithFastMathAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} diff --git a/mlir/dialects/arith_capi.h b/mlir/dialects/arith_capi.h new file mode 100644 index 00000000..6012e6ed --- /dev/null +++ b/mlir/dialects/arith_capi.h @@ -0,0 +1,25 @@ +// C entry points for the `arith` dialect's flag attributes, which upstream's +// C API leaves out. +#ifndef ZML_MLIR_DIALECTS_ARITH_CAPI_H +#define ZML_MLIR_DIALECTS_ARITH_CAPI_H + +#include +#include + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// #arith.fastmath<...>: `value` is the `FastMathFlags` bit set; `Get` returns +// a null attribute when it has bits outside the enum. +MLIR_CAPI_EXPORTED bool mlirAttributeIsAArithFastMath(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirArithFastMathAttrGet(MlirContext ctx, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirArithFastMathAttrGetValue(MlirAttribute attr); + +#ifdef __cplusplus +} +#endif + +#endif // ZML_MLIR_DIALECTS_ARITH_CAPI_H diff --git a/mlir/dialects/cuda.zig b/mlir/dialects/cuda.zig index 1f75d39b..d610e623 100644 --- a/mlir/dialects/cuda.zig +++ b/mlir/dialects/cuda.zig @@ -1,27 +1,34 @@ -//! Builders for the NVIDIA CUDA host-program dialect used by CuTe DSL. -//! -//! The dialect itself is registered by NVIDIA's runtime compiler. ZML builds -//! these operations in an allow-unregistered context, so constructors mirror -//! the ODS schemas from NVIDIA CUTLASS DSL's generated `_cuda_ops_gen.py`. +//! Builders for the CUDA host-program operations CuTe kernels emit, over the +//! bindings of NVIDIA's `cuda` dialect (`mlir/dialects/cute_ir`, `cute.cuda`), +//! which the CuTe-DSL compiler registers. Each builder fixes the attributes the +//! kernels do not choose. + const std = @import("std"); +const cute = @import("mlir/dialects/cute_ir"); const mlir = @import("mlir"); const stdx = @import("stdx"); -pub const dialect_namespace = "cuda"; +pub const ops = cute.cuda; + +pub const dialect_namespace = ops.dialect_namespace; pub fn streamType(ctx: *mlir.Context) *const mlir.Type { - return mlir.Type.parse(ctx, "!cuda.stream") catch @panic("CUDA stream type is unavailable"); + return (ops.StreamType.get(ctx, .{}) catch unreachable).type_(); } pub fn resultType(ctx: *mlir.Context) *const mlir.Type { - return mlir.Type.parse(ctx, "!cuda.result") catch @panic("CUDA result type is unavailable"); + return (ops.ResultType.get(ctx, .{}) catch unreachable).type_(); } pub fn launchConfigType(ctx: *mlir.Context, max_attrs: u32) *const mlir.Type { - var buffer: [64]u8 = undefined; - const text = std.fmt.bufPrint(&buffer, "!cuda.launch_cfg", .{max_attrs}) catch @panic("CUDA launch-config type overflow"); - return mlir.Type.parse(ctx, text) catch @panic("CUDA launch-config type is unavailable"); + return (ops.LaunchConfigType.get(ctx, .{ .maxNumAttrs = max_attrs }) catch unreachable).type_(); +} + +/// `#cuda.assume_kernel_attr`, whether `cuda.launch_ex` may assume its +/// callee is a kernel. +pub fn assumeKernelAttr(ctx: *mlir.Context, value: bool) *const mlir.Attribute { + return (ops.AssumeKernelAttr.get(ctx, .{ .value = value }) catch unreachable).attribute(); } pub const KernelArgs = struct { @@ -33,37 +40,19 @@ pub const KernelArgs = struct { location: *const mlir.Location, }; +/// `cuda.kernel`; `body` is filled afterwards. pub fn kernel(ctx: *mlir.Context, args: KernelArgs) *mlir.Operation { - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 16) = .empty; - attrs.appendSliceAssumeCapacity(&.{ - .named(ctx, "sym_name", .string(ctx, args.name)), - .named(ctx, "function_type", .typeAttr(args.function_type)), - }); - if (args.arg_attrs) |attr| attrs.appendAssumeCapacity(.named(ctx, "arg_attrs", attr)); - attrs.appendSliceAssumeCapacity(args.extra_attributes); - return mlir.Operation.make(ctx, "cuda.kernel", .{ - .blocks = &.{args.body}, - .attributes = attrs.constSlice(), - .location = args.location, - .verify = false, - }); + const op = ops.kernel(ctx, .string(ctx, args.name), .typeAttr(args.function_type), args.arg_attrs, null, null, .{args.body}, args.location); + for (args.extra_attributes) |attr| op.setAttributeByName(attr.name().str(), attr.attribute()); + return op; } pub fn return_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.return", .{ - .operands = .{ .flat = values }, - .location = location, - .verify = false, - }); + return ops.@"return"(ctx, values, location); } pub fn cast(ctx: *mlir.Context, dst: *const mlir.Type, src: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.cast", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{dst} }, - .location = location, - .verify = false, - }); + return ops.cast(ctx, src, dst, location); } pub const LaunchConfigCreateArgs = struct { @@ -76,40 +65,32 @@ pub const LaunchConfigCreateArgs = struct { }; pub fn launch_cfg_create(ctx: *mlir.Context, args: LaunchConfigCreateArgs) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.launch_cfg.create", .{ - .operands = .{ .flat = &.{ - args.block[0], args.block[1], args.block[2], args.dynamic_smem, - args.grid[0], args.grid[1], args.grid[2], args.stream, - } }, - .results = .{ .flat = &.{launchConfigType(ctx, args.max_attrs)} }, - .attributes = &.{.named(ctx, "maxNumAttrs", .int(ctx, .i32, args.max_attrs))}, - .location = args.location, - .verify = false, - }); + return ops.launch_cfg_create( + ctx, + args.block[0], + args.block[1], + args.block[2], + args.dynamic_smem, + args.grid[0], + args.grid[1], + args.grid[2], + args.stream, + launchConfigType(ctx, args.max_attrs), + .int(ctx, .i32, args.max_attrs), + args.location, + ); } pub fn launch_cfg_cluster_dim(ctx: *mlir.Context, config: *const mlir.Value, dims: [3]*const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.launch_cfg.cluster_dim", .{ - .operands = .{ .flat = &.{ config, dims[0], dims[1], dims[2] } }, - .location = location, - .verify = false, - }); -} - -fn launchCfgFlag(ctx: *mlir.Context, comptime name: []const u8, config: *const mlir.Value, enabled: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, name, .{ - .operands = .{ .flat = &.{ config, enabled } }, - .location = location, - .verify = false, - }); + return ops.launch_cfg_cluster_dim(ctx, config, dims[0], dims[1], dims[2], location); } pub fn launch_cfg_cooperative(ctx: *mlir.Context, config: *const mlir.Value, enabled: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return launchCfgFlag(ctx, "cuda.launch_cfg.cooperative", config, enabled, location); + return ops.launch_cfg_cooperative(ctx, config, enabled, location); } pub fn launch_cfg_programmatic_stream_serialization_allowed(ctx: *mlir.Context, config: *const mlir.Value, enabled: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return launchCfgFlag(ctx, "cuda.launch_cfg.programmatic_stream_serialization_allowed", config, enabled, location); + return ops.launch_cfg_programmatic_stream_serialization_allowed(ctx, config, enabled, location); } pub const LaunchExArgs = struct { @@ -121,19 +102,7 @@ pub const LaunchExArgs = struct { }; pub fn launch_ex(ctx: *mlir.Context, args: LaunchExArgs) *mlir.Operation { - var operands: stdx.BoundedArray(*const mlir.Value, 128) = .empty; - operands.appendAssumeCapacity(args.config); - operands.appendSliceAssumeCapacity(args.inputs); - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - attrs.appendAssumeCapacity(.named(ctx, "callee", args.callee)); - if (args.assume_kernel_attr) |attr| attrs.appendAssumeCapacity(.named(ctx, "assume_kernel_attr", attr)); - return mlir.Operation.make(ctx, "cuda.launch_ex", .{ - .operands = .{ .flat = operands.constSlice() }, - .results = .{ .flat = &.{resultType(ctx)} }, - .attributes = attrs.constSlice(), - .location = args.location, - .verify = false, - }); + return ops.launch_ex(ctx, args.config, args.inputs, resultType(ctx), args.callee, null, null, args.assume_kernel_attr, args.location); } test { diff --git a/mlir/dialects/cuda_tile/cuda_tile.zig b/mlir/dialects/cuda_tile/cuda_tile.zig index 8e011aa8..e32ddc26 100644 --- a/mlir/dialects/cuda_tile/cuda_tile.zig +++ b/mlir/dialects/cuda_tile/cuda_tile.zig @@ -1,6 +1,6 @@ //! Zig bindings for NVIDIA's `cuda_tile` MLIR dialect (CUDA Tile IR): one -//! function per op in `Ops.td` order. Types and enums go through the dialect's -//! C API; what it has no getter for is parsed from text. +//! function per op in `Ops.td` order. Types and attributes go through the +//! dialect's C API, completed by `cuda_tile_capi.h` where it has no getter. const std = @import("std"); @@ -245,7 +245,55 @@ pub const StridedViewType = opaque { }; /// `!cuda_tile.gather_scatter_view, sparse_dim=N>` -/// (13.3). The C API has no getter for it, so it is parsed from text. +/// (13.3): gathers or scatters along `sparseDim`, one tile per index. +pub const GatherScatterViewType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaTileGatherScatterView; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "gather_scatter_view"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + tileShape: []const i32, + tensorView: *const TensorViewType, + sparseDim: u32, + paddingValue: ?PaddingValue = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaTileGatherScatterViewTypeGet( + ctx.ptr(), + @intCast(args.tileShape.len), + args.tileShape.ptr, + args.tensorView.ptr(), + args.sparseDim, + if (args.paddingValue) |p| p.attribute(ctx).ptr() else null_attr, + ); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getTileShape(self: *const Self) *const mlir.DenseArrayAttribute(.i32) { + return @ptrCast(attrFromC(c.mlirCudaTileGatherScatterViewTypeGetTileShape(self.ptr()))); + } + pub fn getTensorView(self: *const Self) *const TensorViewType { + return @ptrCast(typeFromC(c.mlirCudaTileGatherScatterViewTypeGetTensorView(self.ptr()))); + } + pub fn getSparseDim(self: *const Self) u32 { + return c.mlirCudaTileGatherScatterViewTypeGetSparseDim(self.ptr()); + } + pub fn getPaddingValue(self: *const Self) ?PaddingValue { + const attr = c.mlirCudaTileGatherScatterViewTypeGetPaddingValue(self.ptr()); + if (attr.ptr == null) return null; + const name = c.mlirCudaTilePaddingValueAttrGetValue(attr); + return std.meta.stringToEnum(PaddingValue, name.data[0..name.length]).?; + } +}; + pub fn gatherScatterViewType( ctx: *mlir.Context, tile_shape: []const i64, @@ -253,31 +301,28 @@ pub fn gatherScatterViewType( sparse_dim: u32, padding: ?PaddingValue, ) *const mlir.Type { - var buf: [1024]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - w.writeAll("!cuda_tile.gather_scatter_view 0) w.writeAll("x") catch unreachable; - w.print("{d}", .{d}) catch unreachable; - } - w.writeAll("), ") catch unreachable; - if (padding) |p| w.print("padding_value={s}, ", .{@tagName(p)}) catch unreachable; - w.print("{f}, sparse_dim={d}>", .{ tensor_view, sparse_dim }) catch unreachable; - const text = w.buffered(); - return mlir.Type.parse(ctx, text) catch std.debug.panic("cuda_tile: cannot parse '{s}'", .{text}); + var shape: stdx.BoundedArray(i32, mlir.ShapedType.MAX_RANK) = .empty; + for (tile_shape) |d| shape.appendAssumeCapacity(@intCast(d)); + const view = GatherScatterViewType.get(ctx, .{ + .tileShape = shape.constSlice(), + .tensorView = tensor_view.isA(TensorViewType) orelse std.debug.panic("cuda_tile: {f} is not a tensor_view", .{tensor_view}), + .sparseDim = sparse_dim, + .paddingValue = padding, + }) catch std.debug.panic("cuda_tile: invalid gather_scatter_view over {f}", .{tensor_view}); + return view.type_(); } /// The `tile<...>` type a load through `view` yields, for any view kind — /// through the dialect's `TileView` type interface (`cuda_tile_capi.cc`), so /// the gather/scatter view, which has no getters in NVIDIA's C API, works too. pub fn tileTypeOfView(view: *const mlir.Type) *const mlir.Type { - if (!c.zmlCudaTileTypeIsATileView(view.ptr())) std.debug.panic("cuda_tile: {f} is not a tiled view", .{view}); - return typeFromC(c.zmlCudaTileTileViewGetViewTileType(view.ptr())); + if (!c.mlirTypeIsACudaTileTileView(view.ptr())) std.debug.panic("cuda_tile: {f} is not a tiled view", .{view}); + return typeFromC(c.mlirCudaTileTileViewTypeGetViewTileType(view.ptr())); } pub fn indexRankOfView(view: *const mlir.Type) usize { - if (!c.zmlCudaTileTypeIsATileView(view.ptr())) std.debug.panic("cuda_tile: {f} is not a tiled view", .{view}); - return @intCast(c.zmlCudaTileTileViewGetViewIndexRank(view.ptr())); + if (!c.mlirTypeIsACudaTileTileView(view.ptr())) std.debug.panic("cuda_tile: {f} is not a tiled view", .{view}); + return @intCast(c.mlirCudaTileTileViewTypeGetViewIndexRank(view.ptr())); } // Enum attributes: built through the C getter from the tag name, so a typo is @@ -403,8 +448,7 @@ pub const SymbolVisibility = enum { } }; -// optimization_hints: per-architecture dictionaries, built from text so -// several architectures can be listed. +// optimization_hints: one dictionary of hints per architecture. pub const Arch = enum { default, sm_80, sm_86, sm_87, sm_88, sm_89, sm_90, sm_100, sm_103, sm_110, sm_120, sm_121 }; @@ -431,78 +475,185 @@ pub const PtrLoadStoreHint = struct { latency: ?i32 = null, }; -fn writeHintFields(w: *std.Io.Writer, hint: anytype) !void { - var first = true; - inline for (@typeInfo(@TypeOf(hint)).@"struct".fields) |f| { +/// `#cuda_tile.optimization_hints`: a +/// dictionary from architecture name to a dictionary of hints. +pub const OptimizationHintsAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirCudaTileAttributeIsAOptimizationHintsAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "optimization_hints"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: *const mlir.DictionaryAttribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaTileOptimizationHintsAttrGet(ctx.ptr(), args.value.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) *const mlir.DictionaryAttribute { + return @ptrCast(attrFromC(c.mlirCudaTileOptimizationHintsAttrGetValue(self.ptr()))); + } +}; + +/// The hint dictionary of one architecture: the non-null fields of `hint` +/// but `arch`, integers as `i64` like the textual form. +fn hintDictionary(ctx: *mlir.Context, hint: anytype) *const mlir.Attribute { + const fields = @typeInfo(@TypeOf(hint)).@"struct".fields; + var entries: stdx.BoundedArray(mlir.NamedAttribute, fields.len) = .empty; + inline for (fields) |f| { if (comptime std.mem.eql(u8, f.name, "arch")) continue; if (@field(hint, f.name)) |v| { - if (!first) try w.writeAll(", "); - first = false; - switch (@TypeOf(v)) { - bool => try w.print("{s} = {}", .{ f.name, v }), - else => try w.print("{s} = {d}", .{ f.name, v }), - } + const value: *const mlir.Attribute = switch (@TypeOf(v)) { + bool => .boolean(ctx, v), + else => .int(ctx, .i64, v), + }; + entries.appendAssumeCapacity(.named(ctx, f.name, value)); } } + return .dict(ctx, entries.constSlice()); } -/// `#cuda_tile.optimization_hints` from a -/// slice of `EntryHint` or `LoadStoreHint`. +/// `#cuda_tile.optimization_hints` from a slice of `EntryHint`, +/// `LoadStoreHint` or `PtrLoadStoreHint`. pub fn optimizationHints(ctx: *mlir.Context, hints: anytype) *const mlir.Attribute { - var buf: [2048]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - w.writeAll("#cuda_tile.optimization_hints<") catch unreachable; - for (hints, 0..) |h, i| { - if (i > 0) w.writeAll(", ") catch unreachable; - w.print("{s} = {{", .{@tagName(h.arch)}) catch unreachable; - writeHintFields(&w, h) catch unreachable; - w.writeAll("}") catch unreachable; + var entries: stdx.BoundedArray(mlir.NamedAttribute, @typeInfo(Arch).@"enum".fields.len) = .empty; + for (hints) |h| entries.appendAssumeCapacity(.named(ctx, @tagName(h.arch), hintDictionary(ctx, h))); + const value: *const mlir.DictionaryAttribute = @ptrCast(mlir.Attribute.dict(ctx, entries.constSlice())); + const attr = OptimizationHintsAttr.get(ctx, .{ .value = value }) catch + std.debug.panic("cuda_tile: invalid optimization_hints {f}", .{value}); + return attr.attribute(); +} + +// assume predicates. + +/// `#cuda_tile.div_by`, or `` for a +/// per-group claim; `every` and `along` are both set or both null. +pub const DivByAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaTileDivBy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "div_by"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); } - w.writeAll(">") catch unreachable; - const text = w.buffered(); - return mlir.Attribute.parse(ctx, text) catch std.debug.panic("cuda_tile: cannot parse '{s}'", .{text}); -} + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + divisor: u64, + every: ?i64 = null, + along: ?i64 = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaTileDivByAttrGet( + ctx.ptr(), + args.divisor, + args.every != null, + args.every orelse 0, + args.along != null, + args.along orelse 0, + ); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getDivisor(self: *const Self) u64 { + return c.mlirCudaTileDivByAttrGetDivisor(self.ptr()); + } + pub fn getEvery(self: *const Self) ?i64 { + return if (c.mlirCudaTileDivByAttrHasEvery(self.ptr())) c.mlirCudaTileDivByAttrGetEvery(self.ptr()) else null; + } + pub fn getAlong(self: *const Self) ?i64 { + return if (c.mlirCudaTileDivByAttrHasAlong(self.ptr())) c.mlirCudaTileDivByAttrGetAlong(self.ptr()) else null; + } +}; -// assume predicates: no C getters; parsed from their documented spellings. - -fn parseAttr(ctx: *mlir.Context, comptime fmt: []const u8, args: anytype) *const mlir.Attribute { - var buf: [256]u8 = undefined; - const text = std.fmt.bufPrint(&buf, fmt, args) catch unreachable; - return mlir.Attribute.parse(ctx, text) catch std.debug.panic("cuda_tile: cannot parse '{s}'", .{text}); -} - -/// `#cuda_tile.div_by`, or `` for a per-group claim. +/// `along` defaults to 0 when only `every` is given. pub fn divBy(ctx: *mlir.Context, divisor: u64, every: ?i64, along: ?i64) *const mlir.Attribute { - if (every) |e| { - return parseAttr(ctx, "#cuda_tile.div_by<{d}, every {d} along {d}>", .{ divisor, e, along orelse 0 }); - } - return parseAttr(ctx, "#cuda_tile.div_by<{d}>", .{divisor}); + const attr = DivByAttr.get(ctx, .{ + .divisor = divisor, + .every = every, + .along = if (every != null) along orelse 0 else null, + }) catch unreachable; + return attr.attribute(); } +/// `#cuda_tile.same_elements<[...]>`: per dimension, the size of the groups +/// of equal elements. +pub const SameElementsAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaTileSameElements; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "same_elements"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + values: []const i64, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaTileSameElementsAttrGet(ctx.ptr(), @intCast(args.values.len), args.values.ptr); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValues(self: *const Self) *const mlir.DenseArrayAttribute(.i64) { + return @ptrCast(attrFromC(c.mlirCudaTileSameElementsAttrGetValues(self.ptr()))); + } +}; + pub fn sameElements(ctx: *mlir.Context, values: []const i64) *const mlir.Attribute { - var buf: [256]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - w.writeAll("#cuda_tile.same_elements<[") catch unreachable; - for (values, 0..) |v, i| { - if (i > 0) w.writeAll(", ") catch unreachable; - w.print("{d}", .{v}) catch unreachable; - } - w.writeAll("]>") catch unreachable; - const text = w.buffered(); - return mlir.Attribute.parse(ctx, text) catch std.debug.panic("cuda_tile: cannot parse '{s}'", .{text}); + const attr = SameElementsAttr.get(ctx, .{ .values = values }) catch unreachable; + return attr.attribute(); } /// `#cuda_tile.bounded`, either side `?` when null. +pub const BoundedAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaTileBounded; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "bounded"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + lb: ?i64 = null, + ub: ?i64 = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaTileBoundedAttrGet(ctx.ptr(), args.lb != null, args.lb orelse 0, args.ub != null, args.ub orelse 0); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getLb(self: *const Self) ?i64 { + return if (c.mlirCudaTileBoundedAttrHasLb(self.ptr())) c.mlirCudaTileBoundedAttrGetLb(self.ptr()) else null; + } + pub fn getUb(self: *const Self) ?i64 { + return if (c.mlirCudaTileBoundedAttrHasUb(self.ptr())) c.mlirCudaTileBoundedAttrGetUb(self.ptr()) else null; + } +}; + pub fn bounded(ctx: *mlir.Context, lb: ?i64, ub: ?i64) *const mlir.Attribute { - var buf: [128]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - w.writeAll("#cuda_tile.bounded<") catch unreachable; - if (lb) |v| w.print("{d}", .{v}) catch unreachable else w.writeAll("?") catch unreachable; - w.writeAll(", ") catch unreachable; - if (ub) |v| w.print("{d}", .{v}) catch unreachable else w.writeAll("?") catch unreachable; - w.writeAll(">") catch unreachable; - const text = w.buffered(); - return mlir.Attribute.parse(ctx, text) catch std.debug.panic("cuda_tile: cannot parse '{s}'", .{text}); + const attr = BoundedAttr.get(ctx, .{ .lb = lb, .ub = ub }) catch unreachable; + return attr.attribute(); } // Constant values: DenseElementsAttr over a tile type, which is a ShapedType. @@ -1534,6 +1685,33 @@ test "dialect registers and types round-trip" { try std.testing.expectEqual(@as(usize, 1), indexRankOfView(pv)); try std.testing.expect(tokenType(ctx).eql(try mlir.Type.parse(ctx, "!cuda_tile.token"))); + + const gather_tv = tensorViewType(ctx, f32_ty, &.{ 64, 32 }, &.{ 32, 1 }); + const gsv = gatherScatterViewType(ctx, &.{ 8, 16 }, gather_tv, 0, .zero); + try std.testing.expect(gsv.eql(try mlir.Type.parse(ctx, "!cuda_tile.gather_scatter_view, sparse_dim=0>"))); + const typed_gsv = gsv.isA(GatherScatterViewType).?; + const gsv_tile = typed_gsv.getTileShape(); + var gsv_shape: [2]i32 = undefined; + try std.testing.expectEqual(gsv_shape.len, gsv_tile.numElements()); + for (&gsv_shape, 0..) |*d, i| d.* = gsv_tile.element(i); + const rebuilt_gsv = try GatherScatterViewType.get(ctx, .{ + .tileShape = &gsv_shape, + .tensorView = typed_gsv.getTensorView(), + .sparseDim = typed_gsv.getSparseDim(), + .paddingValue = typed_gsv.getPaddingValue(), + }); + try std.testing.expect(rebuilt_gsv.eql(typed_gsv)); + try std.testing.expect(tileTypeOfView(gsv).eql(tileType(ctx, &.{ 8, 16 }, f32_ty))); + try std.testing.expect((try GatherScatterViewType.get(ctx, .{ + .tileShape = &.{ 8, 16 }, + .tensorView = gather_tv.isA(TensorViewType).?, + .sparseDim = 1, + })).getPaddingValue() == null); + try std.testing.expectError(error.InvalidMlir, GatherScatterViewType.get(ctx, .{ + .tileShape = &.{8}, + .tensorView = gather_tv.isA(TensorViewType).?, + .sparseDim = 0, + })); } test "enum and hint attributes" { @@ -1550,10 +1728,40 @@ test "enum and hint attributes" { .{ .arch = .sm_120, .num_cta_in_cga = 2, .occupancy = 2 }, }); try std.testing.expect(hints.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.optimization_hints"))); + const typed_hints = hints.isA(OptimizationHintsAttr).?; + try std.testing.expect((try OptimizationHintsAttr.get(ctx, .{ .value = typed_hints.getValue() })).eql(typed_hints)); + const ls_hints = optimizationHints(ctx, &[_]LoadStoreHint{.{ .arch = .sm_100, .allow_tma = false, .latency = 3 }}); + try std.testing.expect(ls_hints.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.optimization_hints"))); + const not_a_dict: *const mlir.DictionaryAttribute = @ptrCast(mlir.Attribute.dict(ctx, &.{.named(ctx, "sm_100", .int(ctx, .i64, 1))})); + try std.testing.expectError(error.InvalidMlir, OptimizationHintsAttr.get(ctx, .{ .value = not_a_dict })); - _ = divBy(ctx, 16, null, null); - _ = sameElements(ctx, &.{ 2, 4 }); - _ = bounded(ctx, 0, null); + const div = divBy(ctx, 16, null, null); + try std.testing.expect(div.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.div_by<16>"))); + const div_every = divBy(ctx, 4, 4, 1); + try std.testing.expect(div_every.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.div_by<4, every 4 along 1>"))); + try std.testing.expect(divBy(ctx, 32, 4, null).eql(try mlir.Attribute.parse(ctx, "#cuda_tile.div_by<32, every 4 along 0>"))); + const typed_div = div_every.isA(DivByAttr).?; + try std.testing.expect((try DivByAttr.get(ctx, .{ + .divisor = typed_div.getDivisor(), + .every = typed_div.getEvery(), + .along = typed_div.getAlong(), + })).eql(typed_div)); + try std.testing.expect(div.isA(DivByAttr).?.getEvery() == null); + try std.testing.expectError(error.InvalidMlir, DivByAttr.get(ctx, .{ .divisor = 4, .every = 4 })); + + const same = sameElements(ctx, &.{ 2, 4 }); + try std.testing.expect(same.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.same_elements<[2, 4]>"))); + const same_values = same.isA(SameElementsAttr).?.getValues(); + try std.testing.expectEqual(@as(usize, 2), same_values.numElements()); + try std.testing.expectEqual(@as(i64, 4), same_values.element(1)); + + const lower = bounded(ctx, 0, null); + try std.testing.expect(lower.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.bounded<0, ?>"))); + try std.testing.expect(bounded(ctx, null, 7).eql(try mlir.Attribute.parse(ctx, "#cuda_tile.bounded"))); + try std.testing.expect(bounded(ctx, -3, 7).eql(try mlir.Attribute.parse(ctx, "#cuda_tile.bounded<-3, 7>"))); + const typed_lower = lower.isA(BoundedAttr).?; + try std.testing.expectEqual(@as(?i64, 0), typed_lower.getLb()); + try std.testing.expect(typed_lower.getUb() == null); } test "entry round-trip through print and parse" { diff --git a/mlir/dialects/cuda_tile/cuda_tile_capi.cc b/mlir/dialects/cuda_tile/cuda_tile_capi.cc index 755be788..de37f42f 100644 --- a/mlir/dialects/cuda_tile/cuda_tile_capi.cc +++ b/mlir/dialects/cuda_tile/cuda_tile_capi.cc @@ -1,17 +1,198 @@ #include "cuda_tile_capi.h" +#include + +#include "cuda_tile/Dialect/CudaTile/IR/Attributes.h" +#include "cuda_tile/Dialect/CudaTile/IR/Dialect.h" #include "cuda_tile/Dialect/CudaTile/IR/Types.h" #include "mlir/CAPI/IR.h" +#include "mlir/IR/BuiltinAttributes.h" +#include "mlir/IR/Diagnostics.h" -bool zmlCudaTileTypeIsATileView(MlirType type) { - return llvm::isa(unwrap(type)); +using namespace mlir::cuda_tile; + +namespace { + +mlir::MLIRContext *loadCudaTile(MlirContext context) { + mlir::MLIRContext *ctx = unwrap(context); + if (ctx) ctx->getOrLoadDialect(); + return ctx; } -MlirType zmlCudaTileTileViewGetViewTileType(MlirType type) { - return wrap(llvm::cast(unwrap(type)).getViewTileType()); +std::optional optional(bool has, int64_t value) { + return has ? std::optional(value) : std::nullopt; } -intptr_t zmlCudaTileTileViewGetViewIndexRank(MlirType type) { +} // namespace + +//===----------------------------------------------------------------------===// +// TileView +//===----------------------------------------------------------------------===// + +bool mlirTypeIsACudaTileTileView(MlirType type) { + return llvm::isa(unwrap(type)); +} + +MlirType mlirCudaTileTileViewTypeGetViewTileType(MlirType type) { + return wrap(llvm::cast(unwrap(type)).getViewTileType()); +} + +intptr_t mlirCudaTileTileViewTypeGetViewIndexRank(MlirType type) { return static_cast( - llvm::cast(unwrap(type)).getViewIndexRank()); + llvm::cast(unwrap(type)).getViewIndexRank()); +} + +//===----------------------------------------------------------------------===// +// GatherScatterViewType +//===----------------------------------------------------------------------===// + +bool mlirTypeIsACudaTileGatherScatterView(MlirType type) { + return llvm::isa(unwrap(type)); +} + +MlirType mlirCudaTileGatherScatterViewTypeGet(MlirContext context, + intptr_t rank, + const int32_t *tileShape, + MlirType tensorView, + uint32_t sparseDim, + MlirAttribute paddingValue) { + mlir::MLIRContext *ctx = loadCudaTile(context); + if (!ctx) return {nullptr}; + auto view = llvm::dyn_cast_or_null(unwrap(tensorView)); + if (!view) return {nullptr}; + PaddingValueAttr padding; + if (!mlirAttributeIsNull(paddingValue)) { + padding = llvm::dyn_cast(unwrap(paddingValue)); + if (!padding) return {nullptr}; + } + auto shape = mlir::DenseI32ArrayAttr::get( + ctx, llvm::ArrayRef(tileShape, rank)); + return wrap(GatherScatterViewType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, shape, + view, sparseDim, padding)); +} + +MlirAttribute mlirCudaTileGatherScatterViewTypeGetTileShape(MlirType type) { + return wrap(llvm::cast(unwrap(type)).getTileShape()); +} + +MlirType mlirCudaTileGatherScatterViewTypeGetTensorView(MlirType type) { + return wrap(llvm::cast(unwrap(type)).getTensorView()); +} + +uint32_t mlirCudaTileGatherScatterViewTypeGetSparseDim(MlirType type) { + return llvm::cast(unwrap(type)).getSparseDim(); +} + +MlirAttribute mlirCudaTileGatherScatterViewTypeGetPaddingValue(MlirType type) { + return wrap(llvm::cast(unwrap(type)).getPaddingValue()); +} + +//===----------------------------------------------------------------------===// +// OptimizationHintsAttr +//===----------------------------------------------------------------------===// + +MlirAttribute mlirCudaTileOptimizationHintsAttrGet(MlirContext context, + MlirAttribute value) { + mlir::MLIRContext *ctx = loadCudaTile(context); + if (!ctx) return {nullptr}; + auto dict = llvm::dyn_cast_or_null(unwrap(value)); + if (!dict) return {nullptr}; + return wrap(OptimizationHintsAttr::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, dict)); +} + +MlirAttribute mlirCudaTileOptimizationHintsAttrGetValue(MlirAttribute attr) { + return wrap(llvm::cast(unwrap(attr)).getValue()); +} + +//===----------------------------------------------------------------------===// +// DivByAttr +//===----------------------------------------------------------------------===// + +bool mlirAttributeIsACudaTileDivBy(MlirAttribute attr) { + return llvm::isa(unwrap(attr)); +} + +MlirAttribute mlirCudaTileDivByAttrGet(MlirContext context, uint64_t divisor, + bool hasEvery, int64_t every, + bool hasAlong, int64_t along) { + mlir::MLIRContext *ctx = loadCudaTile(context); + // The printer writes `every` and `along` together. + if (!ctx || hasEvery != hasAlong) return {nullptr}; + return wrap(DivByAttr::get(ctx, divisor, optional(hasEvery, every), + optional(hasAlong, along))); +} + +uint64_t mlirCudaTileDivByAttrGetDivisor(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getDivisor(); +} + +bool mlirCudaTileDivByAttrHasEvery(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getEvery().has_value(); +} + +int64_t mlirCudaTileDivByAttrGetEvery(MlirAttribute attr) { + return *llvm::cast(unwrap(attr)).getEvery(); +} + +bool mlirCudaTileDivByAttrHasAlong(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getAlong().has_value(); +} + +int64_t mlirCudaTileDivByAttrGetAlong(MlirAttribute attr) { + return *llvm::cast(unwrap(attr)).getAlong(); +} + +//===----------------------------------------------------------------------===// +// SameElementsAttr +//===----------------------------------------------------------------------===// + +bool mlirAttributeIsACudaTileSameElements(MlirAttribute attr) { + return llvm::isa(unwrap(attr)); +} + +MlirAttribute mlirCudaTileSameElementsAttrGet(MlirContext context, + intptr_t rank, + const int64_t *values) { + mlir::MLIRContext *ctx = loadCudaTile(context); + if (!ctx) return {nullptr}; + return wrap(SameElementsAttr::get( + ctx, mlir::DenseI64ArrayAttr::get(ctx, + llvm::ArrayRef(values, rank)))); +} + +MlirAttribute mlirCudaTileSameElementsAttrGetValues(MlirAttribute attr) { + return wrap(llvm::cast(unwrap(attr)).getValues()); +} + +//===----------------------------------------------------------------------===// +// BoundedAttr +//===----------------------------------------------------------------------===// + +bool mlirAttributeIsACudaTileBounded(MlirAttribute attr) { + return llvm::isa(unwrap(attr)); +} + +MlirAttribute mlirCudaTileBoundedAttrGet(MlirContext context, bool hasLb, + int64_t lb, bool hasUb, int64_t ub) { + mlir::MLIRContext *ctx = loadCudaTile(context); + if (!ctx) return {nullptr}; + return wrap(BoundedAttr::get(ctx, optional(hasLb, lb), optional(hasUb, ub))); +} + +bool mlirCudaTileBoundedAttrHasLb(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getLb().has_value(); +} + +int64_t mlirCudaTileBoundedAttrGetLb(MlirAttribute attr) { + return *llvm::cast(unwrap(attr)).getLb(); +} + +bool mlirCudaTileBoundedAttrHasUb(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getUb().has_value(); +} + +int64_t mlirCudaTileBoundedAttrGetUb(MlirAttribute attr) { + return *llvm::cast(unwrap(attr)).getUb(); } diff --git a/mlir/dialects/cuda_tile/cuda_tile_capi.h b/mlir/dialects/cuda_tile/cuda_tile_capi.h index 4f0f5c70..fcce31cb 100644 --- a/mlir/dialects/cuda_tile/cuda_tile_capi.h +++ b/mlir/dialects/cuda_tile/cuda_tile_capi.h @@ -1,6 +1,7 @@ // C entry points for what NVIDIA's cuda_tile C API leaves out: the // `TileView` type interface, shared by the partition, strided and -// gather/scatter views. +// gather/scatter views, the gather/scatter view type, the general +// `optimization_hints` constructor and the `assume` predicates. #ifndef ZML_MLIR_DIALECTS_CUDA_TILE_CAPI_H_ #define ZML_MLIR_DIALECTS_CUDA_TILE_CAPI_H_ @@ -13,9 +14,63 @@ extern "C" { #endif -bool zmlCudaTileTypeIsATileView(MlirType type); -MlirType zmlCudaTileTileViewGetViewTileType(MlirType type); -intptr_t zmlCudaTileTileViewGetViewIndexRank(MlirType type); +// Constructors return null for invalid parameters; optional attribute +// parameters take null handles, and std::optional integers a `has` flag. +// Getters require a value accepted by the matching IsA. + +bool mlirTypeIsACudaTileTileView(MlirType type); +MlirType mlirCudaTileTileViewTypeGetViewTileType(MlirType type); +intptr_t mlirCudaTileTileViewTypeGetViewIndexRank(MlirType type); + +// `!cuda_tile.gather_scatter_view`. `paddingValue` is a +// `#cuda_tile.padding_value` attribute, or null for none. +bool mlirTypeIsACudaTileGatherScatterView(MlirType type); +MlirType mlirCudaTileGatherScatterViewTypeGet(MlirContext context, + intptr_t rank, + const int32_t *tileShape, + MlirType tensorView, + uint32_t sparseDim, + MlirAttribute paddingValue); +// A DenseI32ArrayAttr. +MlirAttribute mlirCudaTileGatherScatterViewTypeGetTileShape(MlirType type); +MlirType mlirCudaTileGatherScatterViewTypeGetTensorView(MlirType type); +uint32_t mlirCudaTileGatherScatterViewTypeGetSparseDim(MlirType type); +// Null when the view has no padding value. +MlirAttribute mlirCudaTileGatherScatterViewTypeGetPaddingValue(MlirType type); + +// `#cuda_tile.optimization_hints` over its dictionary of per-architecture +// dictionaries (IsA: NVIDIA's mlirCudaTileAttributeIsAOptimizationHintsAttr). +MlirAttribute mlirCudaTileOptimizationHintsAttrGet(MlirContext context, + MlirAttribute value); +MlirAttribute mlirCudaTileOptimizationHintsAttrGetValue(MlirAttribute attr); + +// `#cuda_tile.div_by`. `every` and `along` are both present or both absent. +bool mlirAttributeIsACudaTileDivBy(MlirAttribute attr); +MlirAttribute mlirCudaTileDivByAttrGet(MlirContext context, uint64_t divisor, + bool hasEvery, int64_t every, + bool hasAlong, int64_t along); +uint64_t mlirCudaTileDivByAttrGetDivisor(MlirAttribute attr); +bool mlirCudaTileDivByAttrHasEvery(MlirAttribute attr); +int64_t mlirCudaTileDivByAttrGetEvery(MlirAttribute attr); +bool mlirCudaTileDivByAttrHasAlong(MlirAttribute attr); +int64_t mlirCudaTileDivByAttrGetAlong(MlirAttribute attr); + +// `#cuda_tile.same_elements`. +bool mlirAttributeIsACudaTileSameElements(MlirAttribute attr); +MlirAttribute mlirCudaTileSameElementsAttrGet(MlirContext context, + intptr_t rank, + const int64_t *values); +// A DenseI64ArrayAttr. +MlirAttribute mlirCudaTileSameElementsAttrGetValues(MlirAttribute attr); + +// `#cuda_tile.bounded`, each bound optional. +bool mlirAttributeIsACudaTileBounded(MlirAttribute attr); +MlirAttribute mlirCudaTileBoundedAttrGet(MlirContext context, bool hasLb, + int64_t lb, bool hasUb, int64_t ub); +bool mlirCudaTileBoundedAttrHasLb(MlirAttribute attr); +int64_t mlirCudaTileBoundedAttrGetLb(MlirAttribute attr); +bool mlirCudaTileBoundedAttrHasUb(MlirAttribute attr); +int64_t mlirCudaTileBoundedAttrGetUb(MlirAttribute attr); #ifdef __cplusplus } diff --git a/mlir/dialects/cute_ir/BUILD.bazel b/mlir/dialects/cute_ir/BUILD.bazel index 3b310559..e5354d89 100644 --- a/mlir/dialects/cute_ir/BUILD.bazel +++ b/mlir/dialects/cute_ir/BUILD.bazel @@ -2,17 +2,25 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "cute_ir", - srcs = ["cute_nvgpu.zig"], + srcs = [ + "cuda.zig", + "cute_nvgpu.zig", + "nvvm.zig", + ], import_name = "mlir/dialects/cute_ir", main = "cute_ir.zig", visibility = ["//visibility:public"], deps = [ "//mlir", "//mlir:c", + "//mlir/dialects/cute_ir/include/cute_ir-c/Dialect:cuda", "//mlir/dialects/cute_ir/include/cute_ir-c/Dialect:cute", "//mlir/dialects/cute_ir/include/cute_ir-c/Dialect:cute_nvgpu", + "//mlir/dialects/cute_ir/include/cute_ir-c/Dialect:nvvm", + "//mlir/dialects/cute_ir/lib/CAPI:cuda", "//mlir/dialects/cute_ir/lib/CAPI:cute", "//mlir/dialects/cute_ir/lib/CAPI:cute_nvgpu", + "//mlir/dialects/cute_ir/lib/CAPI:nvvm", "//stdx", ], ) diff --git a/mlir/dialects/cute_ir/README.md b/mlir/dialects/cute_ir/README.md index d75d0950..6a67ad3d 100644 --- a/mlir/dialects/cute_ir/README.md +++ b/mlir/dialects/cute_ir/README.md @@ -11,6 +11,13 @@ allowed, the compiler's attributes, types and enums beyond the release `CuteNVGPU*.td` files), and every operation of the compiler's registry in `CuteOps.td` and `CuteNVGPUOps.td`. +It also carries NVIDIA's `nvvm` and `cuda` dialects as the same compiler +registers them (`include/cute_ir/Dialect/{NVVM,Cuda}`): every operation, +attribute, type and enum, rebuilt from the release's bindings and the live +compiler, since no `.td` ships for them and the compiler's `nvvm` is not +upstream's. The CuTe builder registers them, with the upstream `llvm` +dialect, so its kernels are built without parsing any of their text. + Updating to a new DSL release means refreshing the compiler entry-point offsets and regenerating the `.td` files from the release's Python bindings and compiler. The C API (except the algebra attributes, `CuteAttributes.*`) @@ -22,9 +29,12 @@ from those `.td` files and follow them. Depend on `//mlir/dialects/cute_ir` and import `mlir/dialects/cute_ir`, or use `@import("mlir/dialects").cute` through the dialect aggregator. -`cute.registerDialects(registry)` registers both `cute` and `cute_nvgpu`. -Cute builders, types, and attributes live directly in `cute`; CuteNVGPU builders -and types live in `cute.nvgpu`. +`cute.registerDialects(registry)` registers `cute`, `cute_nvgpu`, and NVIDIA's +`nvvm` and `cuda` (handles `cute_nvvm` and `cuda`: this `nvvm` and upstream's +cannot share a registry). Cute builders, types, and attributes live directly in +`cute`; CuteNVGPU builders and types live in `cute.nvgpu`, NVVM's in +`cute.nvvm` and CUDA's in `cute.cuda`. `mlir/dialects/nvvm.zig` and +`mlir/dialects/cuda.zig` are convenience builders over the last two. ```zig const cute = @import("mlir/dialects/cute_ir"); @@ -66,15 +76,24 @@ through `Operation.make`'s `.variadic` API, which writes the segment-size attributes. Builders verify the created operations; to defer verification, use `mlir.Operation.make` directly with `.verify = false`. -The C API is also available independently of Zig from `lib/CAPI:cute` and -`lib/CAPI:cute_nvgpu`, via `cute_ir-c/Dialect/Cute.h` and `CuteNVGPU.h`; -constructors return a null handle for invalid parameters. +Builders of operations with regions (`cuda.kernel`) and of terminators +(`cuda.return`) do not verify on creation, since their blocks are completed +afterwards; the module is verified instead. + +The C API is also available independently of Zig from `lib/CAPI:cute`, +`lib/CAPI:cute_nvgpu`, `lib/CAPI:nvvm` and `lib/CAPI:cuda`, via +`cute_ir-c/Dialect/Cute.h`, `CuteNVGPU.h`, `NVVM.h` (names prefixed +`CuteNVVM`) and `Cuda.h`; constructors return a null handle for invalid +parameters. ```sh bazel test //mlir/dialects/cute_ir:test ``` -These tests only check the bindings: every bound operation is registered and -every type and attribute rebuilds from its getters. The dialect itself is +These tests only check the bindings: every bound operation is registered, +every type and attribute rebuilds from its getters, and a sample of the nvvm +and cuda attributes and types print the compiler's own text byte for byte +(`#cuda` and `#cuda` print with a +space the compiler omits; it parses both). The dialect itself is tested where it is maintained. The dialects here do not implement the compiler's semantic verification or lowering. diff --git a/mlir/dialects/cute_ir/REGENERATING.md b/mlir/dialects/cute_ir/REGENERATING.md index 949ed51c..823973ac 100644 --- a/mlir/dialects/cute_ir/REGENERATING.md +++ b/mlir/dialects/cute_ir/REGENERATING.md @@ -10,8 +10,8 @@ Paths below are relative to this directory (the one holding this file). ## Goal -Every dialect this directory carries (`cute`, `cute_nvgpu`, and `cuda` when -`include/cute_ir/Dialect/Cuda` exists) must describe the DSL compiler's +Every dialect this directory carries (`cute`, `cute_nvgpu`, and NVIDIA's +`nvvm` and `cuda`, see below) must describe the DSL compiler's registry as completely and as precisely as the release allows: every operation, type, attribute and enum the compiler registers, with every fact that can be established about each of them. Nothing is opaque unless it cannot @@ -36,14 +36,18 @@ without it, and so that anything built here is accepted by it unchanged. the parsers, printers and verifiers of every cute type, fixed wherever the release's parsing diverges from the compiler's. `CuteInference.{h,cpp}` holds result type inference; the other dialect `.cpp` files and the `BUILD.bazel` - files complete them. Edit them as the new definitions require. + files complete them. Edit them as the new definitions require. For `nvvm` + and `cuda` these are `NVVMDialect.{h,cpp}` (with the `#nvvm.target` + verifier) and `CudaDialect.{h,cpp}` under `include/` and `lib/Dialect/`. - **Generated: what you produce.** Every `.td` file under `include/` that is not a dialect declaration: the operations (`CuteOps.td`, `CuteNVGPUOps.td`, - `CudaOps.td`), the compiler's types, attributes and enums beyond the release - (`CuteTypesCompiler.td`, `CuteAttrsCompiler.td`, `CuteEnums.td`, - `CuteNVGPUTypes.td`, `CuteNVGPUAttrs.td`, `CuteNVGPUEnums.td`, `CudaTypes.td`, - `CudaAttrs.td`, `CudaEnums.td`), and any new `.td` you need to split them - sensibly. The dialect declarations (`*Dialect.td`) change only as far as the + `NVVMOps.td`, `CudaOps.td`), the compiler's types, attributes and enums + beyond the release (`CuteTypesCompiler.td`, `CuteAttrsCompiler.td`, + `CuteEnums.td`, `CuteNVGPUTypes.td`, `CuteNVGPUAttrs.td`, `CuteNVGPUEnums.td`, + `NVVMAttrs.td`, `NVVMEnums.td`, `CudaTypes.td`, `CudaAttrs.td`, + `CudaEnums.td`), the inference of `nvvm` and `cuda` operations whose results + are not one buildable type (`NVVMInference.cpp`, `CudaInference.cpp`), and + any new `.td` you need to split them sensibly. The dialect declarations (`*Dialect.td`) change only as far as the generated files need. - **Language bindings, if present.** When the tree carries bindings derived from these definitions (a C API under `lib/CAPI`, builder files next to this @@ -55,12 +59,98 @@ without it, and so that anything built here is accepted by it unchanged. Consumers outside this directory use the dialect classes and must keep compiling: search the repository for the dialect namespaces -(`cutlass_compiler::cute`, `cute_nvgpu`, `cuda`) and for the headers under -`include/cute_ir`. When a definition changes shape, update those users in the +(`cutlass_compiler::cute`, `cute_nvgpu`, `nvvm`, `cuda`), for the Zig modules +(`cute.nvgpu`, `cute.nvvm`, `cute.cuda`, and the convenience layers over them +in `mlir/dialects/nvvm.zig` and `mlir/dialects/cuda.zig`) and for the headers +under `include/cute_ir`. When a definition changes shape, update those users in the same change. Tighter operand constraints also break consumers that built IR with loose types, and so does an operation that stops inferring its results: fix the consumer, not the constraint. +## NVIDIA's `nvvm` and `cuda` dialects + +The kernels the CuTe builder emits (`kernels/cute`) use two more dialects of +the compiler: `nvvm` (special registers, mbarriers, tcgen05, ...) and `cuda` +(kernels, launch configurations, host runtime calls). Both are carried here +and must be regenerated with every release, like `cute` and `cute_nvgpu`. + +- **Neither is upstream's.** The compiler is built on NVIDIA's own LLVM + revision (the `.so` names it: `LLVM version 23.0.0git ()`, a hash + upstream does not have), and its `nvvm` differs from upstream NVVM: it has + operations, operands and enums upstream lacks (`nvvm.mbarrier.wait.parity`, + `#nvvm.mbar_wait`) and lacks some upstream has. `cuda` exists only in the + compiler. No `.td` ships with the release for either: rebuild both from the + bindings and the live compiler, as for the cute dialects. Upstream's NVVM + `.td` is neither a source nor a substitute. +- **Where they live.** `include/cute_ir/Dialect/{NVVM,Cuda}/IR` (declarations + and generated `.td`), `lib/Dialect/{NVVM,Cuda}/IR` (dialect and inference + `.cpp`), `include/cute_ir-c/Dialect/{NVVM,Cuda}*.h` and + `lib/CAPI/{NVVM,Cuda}*.cpp` (C API), and `nvvm.zig`, `cuda.zig` (Zig, + `cute.nvvm` and `cute.cuda`). +- **Names.** The C++ namespaces are `mlir::cutlass_compiler::nvvm` and + `mlir::cutlass_compiler::cuda`. The dialects are named `nvvm` and `cuda` as + in the compiler, so a registry holds this `nvvm` or upstream's, never both; + upstream's NVVM C++ is linked into the tree and must stay unregistered where + this one is. The C dialect handles are `cute_nvvm` and `cuda`, and the C API + of `nvvm` is prefixed `CuteNVVM` (`mlirAttributeIsACuteNVVM`, + `mlirCuteNVVMAttrGet`) so that neither collides with upstream's + `mlirGetDialectHandle__nvvm__` or a future upstream NVVM C API; `cuda`'s is + prefixed `Cuda`. +- **`llvm` is upstream's.** NVVM operands are `!llvm.ptr` and LLVM + structs, and the builder emits `llvm.*` operations. The CuTe context + registers the upstream LLVM dialect, bound by `mlir/dialects/llvm_dialect.zig` + (its builders, and `mlirLLVMPointerTypeGet` for `!llvm.ptr`); the nvvm + and cuda constraints name `::mlir::LLVM` types. NVIDIA's `llvm` is not + vendored: its types print and parse the same, and the compiler reads the + `llvm.*` operations upstream prints (custom and generic forms, with the + properties upstream fills in) into the same module. This holds only while + the compatibility check below passes; when it stops holding, vendor NVIDIA's + `llvm` the same way (its bindings are `_llvm_ops_gen.py` and + `_llvm_enum_gen.py`) rather than patching around it. + +Sources specific to them, besides those in the table below: + +- `_nvvm_ops_gen.py`, `_cuda_ops_gen.py`: the operations. Operand kinds come + from how `__init__` appends them (and `_ODS_OPERAND_SEGMENTS`), result names + and kinds from the result properties, inference from whether `__init__` + builds a `results` list or passes its `results=` argument through. +- `_nvvm_enum_gen.py`, `_cuda_enum_gen.py`: the enums. A builder registered + under the enum's own name builds the plain `I32EnumAttr` (it follows its + enum class in the file); one registered as `nvvm.Attr` builds the + dialect attribute and gives its mnemonic and form: `#nvvm.` + or `#nvvm`. Some enums only have the latter. Parsing an + invalid case names the C++ enum (`expected ::mlir::NVVM:: to be one + of: ...`) or its summary, which maps each attribute to its enum. +- `cutlass._mlir._mlir_libs._cutlass_ir._mlirDialectsCuda`: the typed cuda + type and attribute classes and their `get()` parameters. The other cuda + types and attributes, and the three non-enum nvvm attributes (`shape`, + `ld_st_matrix_shape`, `target`), are found by trying the `.so`'s + identifiers as mnemonics (`!cuda.`, `#nvvm.`); `nvvm` has no + types. +- Attribute constraints the bindings name but no builder covers + (`nvvm.FPArithRoundingMode`, the `nvvm.Multimem*` orders, anonymous + `TypeAttr` constraints) are recovered by probing values: the verifier's + `failed to satisfy constraint: ...` names the rule, and the values it lets + through give the predicate. +- The compiler aborts on some malformed probes (`nvvm.mma.sync`, + `nvvm.mma.sp.sync`): run those checks one process each. +- Results the compiler infers but that are not one buildable type get an + inference rule only where varying the operand types shows the result is one + operand's type; elsewhere inference fails and the types written stand + (`_InferredOp`, as in cute). Vary the attributes too: a rule must + hold for every attribute value (`nvvm.mul` with `mode = wide` doubles the + width, `nvvm.shfl.sync` with `return_value_and_is_valid` returns a struct), + or inference must fail. + +The bindings follow the cute ones: the C API and the Zig enums, typed +attribute and type wrappers and operation builders are generated from +`llvm-tblgen --dump-json` of the generated `.td` files, then formatted with +`zig fmt`. Builders of operations with regions, and of terminators, do not +verify on creation (their blocks are filled, or their block is completed, +afterwards). `mlir/dialects/nvvm.zig` and `mlir/dialects/cuda.zig` are the +hand-written convenience layer the builder calls; they fix the attributes the +kernels do not choose and must keep building no text. + ## Sources Pin everything to one release version, read from the repository rule that @@ -222,7 +312,19 @@ A change is done when all of these hold: 4. **The repository's tests pass**, including the dialect's own tests and the tests of every consumer found above (kernel emitters, host tools, language bindings), built with the tree's usual Bazel configuration. -5. **Both copies agree.** When the same dialect is carried by more than one +5. **NVIDIA's compiler reads what the kernels emit.** Dump the IR of every + kernel the repository's tests emit through `kernels/cute`: the builder's + own tests, the MXFP4 emission tests under `zml/moe/cute_kernels`, and the + FlashMLA program (`zml/attention/cute_kernels/flashmla_sm100.zig`, whose + `Program.emit` needs a temporary test with a few configurations, single-CTA, + clustered and persistent). Dump both with the previous definitions and with + the new ones. In the release's venv, `import cutlass`, then parse and + verify each module with `cutlass._mlir.ir.Module.parse`, and compare the + generic print (`get_asm(print_generic_op_form=True)`) of each module before + and after: it must be identical, and a module the compiler rejected before + must be rejected the same way after. This is also what keeps upstream + `llvm` acceptable (see above). +6. **Both copies agree.** When the same dialect is carried by more than one repository, the shared files are identical in each, and only repository-specific files (`BUILD.bazel`, bindings) differ. diff --git a/mlir/dialects/cute_ir/cuda.zig b/mlir/dialects/cute_ir/cuda.zig new file mode 100644 index 00000000..6c546113 --- /dev/null +++ b/mlir/dialects/cute_ir/cuda.zig @@ -0,0 +1,3550 @@ +//! Zig bindings of NVIDIA's CUDA host dialect of the CuTe DSL (nvidia-cutlass-dsl 4.8.0), generated +//! from the dialect's .td files; do not edit. + +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub const dialect_namespace = "cuda"; +/// The C dialect handle, `mlirGetDialectHandle__cuda__`. +pub const dialect_handle = "cuda"; + +// Enums, generated from the dialect's .td files; values are the compiler's. + +/// Enumerated type describing portability. +pub const ArchPortability = enum(u32) { + portable = 0, + conditional = 1, + family = 2, +}; + +/// Enumerated type describing CUDA device attributes can be set through Cuda Dialect. +pub const CUDeviceAttr = enum(u32) { + maxThreadsPerBlock = 1, + maxBlockDimX = 2, + maxBlockDimY = 3, + maxBlockDimZ = 4, + maxGridDimX = 5, + maxGridDimY = 6, + maxGridDimZ = 7, + maxSharedMemoryPerBlock = 8, + totalConstantMemory = 9, + warpSize = 10, + maxPitch = 11, + maxRegistersPerBlock = 12, + clockRate = 13, + textureAlignment = 14, + gpuOverlap = 15, + multiProcessorCount = 16, + kernelExecTimeout = 17, + integrated = 18, + canMapHostMemory = 19, + computeMode = 20, + maxTexture1DWidth = 21, + maxTexture2DWidth = 22, + maxTexture2DHeight = 23, + maxTexture3DWidth = 24, + maxTexture3DHeight = 25, + maxTexture3DDepth = 26, + maxTexture2DLayeredWidth = 27, + maxTexture2DLayeredHeight = 28, + maxTexture2DLayeredLayers = 29, + surfaceAlignment = 30, + concurrentKernels = 31, + eccEnabled = 32, + pciBusId = 33, + pciDeviceId = 34, + tccDriver = 35, + memoryClockRate = 36, + globalMemoryBusWidth = 37, + l2CacheSize = 38, + maxThreadsPerMultiProcessor = 39, + asyncEngineCount = 40, + unifiedAddressing = 41, + maxTexture1DLayeredWidth = 42, + maxTexture1DLayeredLayers = 43, + maxTexture2DGatherWidth = 45, + maxTexture2DGatherHeight = 46, + maxTexture3DWidthAlt = 47, + maxTexture3DHeightAlt = 48, + maxTexture3DDepthAlt = 49, + pciDomainId = 50, + texturePitchAlignment = 51, + maxTextureCubemapWidth = 52, + maxTextureCubemapLayeredWidth = 53, + maxTextureCubemapLayeredLayers = 54, + maxSurface1DWidth = 55, + maxSurface2DWidth = 56, + maxSurface2DHeight = 57, + maxSurface3DWidth = 58, + maxSurface3DHeight = 59, + maxSurface3DDepth = 60, + maxSurface1DLayeredWidth = 61, + maxSurface1DLayeredLayers = 62, + maxSurface2DLayeredWidth = 63, + maxSurface2DLayeredHeight = 64, + maxSurface2DLayeredLayers = 65, + maxSurfaceCubemapWidth = 66, + maxSurfaceCubemapLayeredWidth = 67, + maxSurfaceCubemapLayeredLayers = 68, + maxTexture1DLinearWidth = 69, + maxTexture2DLinearWidth = 70, + maxTexture2DLinearHeight = 71, + maxTexture2DLinearPitch = 72, + maxTexture2DMipmappedWidth = 73, + maxTexture2DMipmappedHeight = 74, + computeCapabilityMajor = 75, + computeCapabilityMinor = 76, + maxTexture1DMipmappedWidth = 77, + streamPrioritiesSupported = 78, + globalL1CacheSupported = 79, + localL1CacheSupported = 80, + maxSharedMemoryPerMultiprocessor = 81, + maxRegistersPerMultiprocessor = 82, + managedMemory = 83, + isMultiGpuBoard = 84, + multiGpuBoardGroupID = 85, + hostNativeAtomicSupported = 86, + singleToDoublePrecisionPerfRatio = 87, + pageableMemoryAccess = 88, + concurrentManagedAccess = 89, + computePreemptionSupported = 90, + canUseHostPointerForRegisteredMem = 91, + reserved92 = 92, + reserved93 = 93, + reserved94 = 94, + cooperativeLaunch = 95, + reserved96 = 96, + maxSharedMemoryPerBlockOptin = 97, + canFlushRemoteWrites = 98, + hostRegisterSupported = 99, + pageableMemoryAccessUsesHostPageTables = 100, + directManagedMemAccessFromHost = 101, + maxBlocksPerMultiprocessor = 106, + maxPersistingL2CacheSize = 108, + maxAccessPolicyWindowSize = 109, + reservedSharedMemoryPerBlock = 111, + sparseCudaArraySupported = 112, + hostRegisterReadOnlySupported = 113, + timelineSemaphoreInteropSupported = 114, + memoryPoolsSupported = 115, + gpuDirectRdmaSupported = 116, + gpuDirectRdmaFlushWritesOptions = 117, + gpuDirectRdmaWritesOrdering = 118, + memoryPoolSupportedHandleTypes = 119, + clusterLaunch = 120, + deferredMappingCudaArraySupported = 121, + reserved122 = 122, + reserved123 = 123, + reserved124 = 124, + ipcEventSupport = 125, + memSyncDomainCount = 126, + reserved127 = 127, + reserved128 = 128, + reserved129 = 129, + numaConfig = 130, + numaId = 131, + reserved132 = 132, + mpsEnabled = 133, + hostNumaId = 134, + d3d12CigSupported = 135, + vulkanCigSupported = 138, + gpuPciDeviceId = 139, + gpuPciSubsystemId = 140, + reserved141 = 141, + hostNumaMemoryPoolsSupported = 142, + hostNumaMultinodeIpcSupported = 143, + hostMemoryPoolsSupported = 144, + reserved145 = 145, + onlyPartialHostNativeAtomicSupported = 147, + cudaDevAttrOversizedSharedMemoryPerBlock = 150, +}; + +/// Enumerated type describing CUDA function attributes can be set through Cuda Dialect. +pub const CUFunctionAttribute = enum(u32) { + max_threads_per_block = 0, + shared_size_bytes = 1, + const_size_bytes = 2, + local_size_bytes = 3, + num_regs = 4, + ptx_version = 5, + binary_version = 6, + cache_mode_ca = 7, + max_dynamic_shared_size_bytes = 8, + preferred_shared_memory_carveout = 9, + cluster_size_must_be_set = 10, + required_cluster_width = 11, + required_cluster_height = 12, + required_cluster_depth = 13, + non_portable_cluster_size_allowed = 14, + cluster_scheduling_policy_preference = 15, +}; + +/// Enumerated of the host library ABI. +pub const CudaABI = enum(u32) { + LP64 = 0, + ILP64 = 1, + LLP64 = 2, +}; + +/// Specifies performance hint with cudaAccessPolicyWindow for hitProp and missProp members. +pub const CudaAccessProperty = enum(u32) { + cudaAccessPropertyNormal = 0, + cudaAccessPropertyStreaming = 1, + cudaAccessPropertyPersisting = 2, +}; + +/// Enumerated type describing CUDA cluster scheduling policy. +pub const CudaClusterSchedulingPolicy = enum(u32) { + cudaClusterSchedulingPolicyDefault = 0, + cudaClusterSchedulingPolicySpread = 1, + cudaClusterSchedulingPolicyLoadBalancing = 2, +}; + +/// Enumerated type describing CUDA function attributes can be set through Cuda Dialect. +pub const CudaFuncAttribute = enum(u32) { + cudaFuncAttributeSharedMemoryMode = 16, +}; + +/// Enumerated type mirroring CudaLaunchAttributeID from CUDA Runtime API. +pub const CudaLaunchAttributeID = enum(u32) { + cudaLaunchAttributeIgnore = 0, + cudaLaunchAttributeAccessPolicyWindow = 1, + cudaLaunchAttributeCooperative = 2, + cudaLaunchAttributeSynchronizationPolicy = 3, + cudaLaunchAttributeClusterDimension = 4, + cudaLaunchAttributeClusterSchedulingPolicyPreference = 5, + cudaLaunchAttributeProgrammaticStreamSerialization = 6, + cudaLaunchAttributeProgrammaticEvent = 7, + cudaLaunchAttributePriority = 8, + cudaLaunchAttributeMemSyncDomainMap = 9, + cudaLaunchAttributeMemSyncDomain = 10, + cudaLaunchAttributePreferredClusterDimension = 11, + cudaLaunchAttributeLaunchCompletionEvent = 12, + cudaLaunchAttributeDeviceUpdatableKernelNode = 13, + cudaLaunchAttributePreferredSharedMemoryCarveout = 14, + cudaLaunchAttributeNvlinkUtilCentricScheduling = 16, +}; + +/// Enumerated type describing CUDA memory sync domain. +pub const CudaLaunchMemSyncDomain = enum(u32) { + cudaLaunchMemSyncDomainDefault = 0, + cudaLaunchMemSyncDomainRemote = 1, +}; + +/// Enumerated type describing CUDA shared memory modes. +pub const CudaSharedMemoryMode = enum(u32) { + cudaSharedMemoryModeDefault = 0, + cudaSharedMemoryModeRequirePortable = 1, + cudaSharedMemoryModeAllowNonPortable = 2, + cudaSharedMemoryModeAllowOversizedSharedMemory = 3, + cudaSharedMemoryModePreferOversizedSharedMemory = 4, +}; + +/// Enumerated type describing CUDA synchronization policy. +pub const CudaSynchronizationPolicy = enum(u32) { + cudaSyncPolicyAuto = 0, + cudaSyncPolicySpin = 1, + cudaSyncPolicyYield = 2, + cudaSyncPolicyBlockingSync = 3, +}; + +/// Enumerated type describing the format of an executable. +pub const ExecutableFormat = enum(u32) { + text = 0, + elf = 1, + bitcode = 2, + fatbin = 3, +}; + +/// Enumerated type describing the representation of an executable. +pub const ExecutableRepresentation = enum(u32) { + sass = 0, + ptx = 2, + nvvm = 3, + ltoir = 4, + mlir = 5, + llvm = 6, + hostobj = 7, +}; + +/// Enumerated type denoting a GPU Architecture. +pub const GpuArchitecture = enum(u32) { + sm_80 = 80, + sm_86 = 86, + sm_87 = 87, + sm_88 = 88, + sm_89 = 89, + sm_90 = 90, + sm_100 = 100, + sm_101 = 101, + sm_103 = 103, + sm_107 = 107, + sm_110 = 110, + sm_120 = 120, + sm_121 = 121, +}; + +/// Enumerated type describing memcpy directions. +pub const MemcpyKind = enum(u32) { + HostToHost = 0, + HostToDevice = 1, + DeviceToHost = 2, + DeviceToDevice = 3, + Default = 4, +}; + +/// Enumerated type describing CUDA stream capture modes. +pub const StreamCaptureModeEnum = enum(u32) { + Global = 0, + ThreadLocal = 1, + Relaxed = 2, +}; + +// Types and attributes, generated from the dialect's .td files. + +/// `#cuda.assume_kernel_attr`: Whether `cuda.launch` may assume its callee is a kernel +pub const AssumeKernelAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaAssumeKernel; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "assume_kernel_attr"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: bool, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaAssumeKernelAttrGet(ctx.ptr(), args.value); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) bool { + return c.mlirCudaAssumeKernelAttrGetValue(self.ptr()); + } +}; + +/// `#cuda.compute_target`: Representation, portability and architectures of a compiled executable +pub const ComputeTargetAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaComputeTarget; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "compute_target"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + representation: ExecutableRepresentation, + portability: ArchPortability, + archs: []const GpuArchitecture, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaComputeTargetAttrGet(ctx.ptr(), @intFromEnum(args.representation), @intFromEnum(args.portability), @intCast(args.archs.len), @ptrCast(args.archs.ptr)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getRepresentation(self: *const Self) ExecutableRepresentation { + return @enumFromInt(c.mlirCudaComputeTargetAttrGetRepresentation(self.ptr())); + } + pub fn getPortability(self: *const Self) ArchPortability { + return @enumFromInt(c.mlirCudaComputeTargetAttrGetPortability(self.ptr())); + } + pub fn getNumArchs(self: *const Self) usize { + return @intCast(c.mlirCudaComputeTargetAttrGetNumArchs(self.ptr())); + } + pub fn getArch(self: *const Self, pos: usize) GpuArchitecture { + return @enumFromInt(c.mlirCudaComputeTargetAttrGetArchs(self.ptr(), @intCast(pos))); + } +}; + +/// `#cuda.dev_max_shared_memory_optin`: The device's opt-in maximum of shared memory per block +pub const DevMaxSharedMemoryOptinAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaDevMaxSharedMemoryOptin; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "dev_max_shared_memory_optin"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaDevMaxSharedMemoryOptinAttrGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `#cuda.device_attributes`: CUDA device attributes +pub const DeviceAttributesAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaDeviceAttributes; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "device_attributes"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + values: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaDeviceAttributesAttrGet(ctx.ptr(), args.values.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValues(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCudaDeviceAttributesAttrGetValues(self.ptr()).ptr.?); + } +}; + +/// `#cuda.executable`: Format of a compiled executable +pub const ExecutableAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaExecutable; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "executable"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + format: ExecutableFormat, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaExecutableAttrGet(ctx.ptr(), @intFromEnum(args.format)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getFormat(self: *const Self) ExecutableFormat { + return @enumFromInt(c.mlirCudaExecutableAttrGetFormat(self.ptr())); + } +}; + +/// `#cuda.func_attributes`: CUDA function attributes of a kernel +pub const FuncAttributesAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaFuncAttributes; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "func_attributes"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + values: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaFuncAttributesAttrGet(ctx.ptr(), args.values.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValues(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCudaFuncAttributesAttrGetValues(self.ptr()).ptr.?); + } +}; + +/// `!cuda.arch_set`: `!cuda.arch_set` +pub const ArchSetType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaArchSet; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "arch_set"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaArchSetTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.event`: `Event` represents CUDA event +pub const EventType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaEvent; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "event"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaEventTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.graph_device_node`: `GraphDeviceNode` represents CUDA device node handle for device-side node update +pub const GraphDeviceNodeType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaGraphDeviceNode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "graph_device_node"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaGraphDeviceNodeTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.graph_exec`: CUDA Graph Execution +pub const GraphExecType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaGraphExec; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "graph_exec"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaGraphExecTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.graph_node`: CUDA Graph Node handle +pub const GraphNodeType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaGraphNode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "graph_node"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaGraphNodeTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.graph`: CUDA Graph +pub const GraphType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaGraph; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "graph"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaGraphTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.int`: `cuda.int` is a integer type with a bitwidth matching the underlying ABI integer bitwidth +pub const IntType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaInt; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "int"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaIntTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.kernel`: CUDA Kernel handle +pub const KernelType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaKernel; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "kernel"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaKernelTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.launch_cfg`: Representation of CUDA's extensible launch configuration mirroring cuda runtime API +pub const LaunchConfigType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaLaunchConfig; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "launch_cfg"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + maxNumAttrs: u32, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaLaunchConfigTypeGet(ctx.ptr(), args.maxNumAttrs); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getMaxNumAttrs(self: *const Self) u32 { + return c.mlirCudaLaunchConfigTypeGetMaxNumAttrs(self.ptr()); + } +}; + +/// `!cuda.library`: CUDA Library handle +pub const LibraryType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaLibrary; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "library"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaLibraryTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.module`: CUDA Module handle +pub const ModuleType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaModule; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "module"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaModuleTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.result`: CUDA Result type +pub const ResultType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaResult; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "result"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaResultTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.device_prop`: CUDA cudaDeviceProp struct type (value-typed). +pub const RuntimeDevicePropType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeDeviceProp; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "runtime.device_prop"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeDevicePropTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.dim_3`: CUDA dim3 struct type (value-typed). +pub const RuntimeDim3Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeDim3; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "runtime.dim_3"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeDim3TypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.extent`: CUDA cudaExtent struct type (value-typed). +pub const RuntimeExtentType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeExtent; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "runtime.extent"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeExtentTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.kernel_node_params`: CUDA cudaKernelNodeParams struct type (value-typed). +pub const RuntimeKernelNodeParamsType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeKernelNodeParams; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "runtime.kernel_node_params"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeKernelNodeParamsTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference`: `!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference` +pub const RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreference; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.cluster_scheduling_policy_preference"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.cooperative`: `!cuda.runtime.launch_attribute_value.cooperative` +pub const RuntimeLaunchAttributeValueCooperativeType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueCooperative; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.cooperative"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValueCooperativeTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.mem_sync_domain`: `!cuda.runtime.launch_attribute_value.mem_sync_domain` +pub const RuntimeLaunchAttributeValueMemSyncDomainType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueMemSyncDomain; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.mem_sync_domain"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValueMemSyncDomainTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling`: `!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling` +pub const RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueNvlinkUtilCentricScheduling; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.nvlink_util_centric_scheduling"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.portable_cluster_size_mode`: `!cuda.runtime.launch_attribute_value.portable_cluster_size_mode` +pub const RuntimeLaunchAttributeValuePortableClusterSizeModeType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValuePortableClusterSizeMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.portable_cluster_size_mode"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValuePortableClusterSizeModeTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.priority`: `!cuda.runtime.launch_attribute_value.priority` +pub const RuntimeLaunchAttributeValuePriorityType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValuePriority; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.priority"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValuePriorityTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed`: `!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed` +pub const RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowed; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.programmatic_stream_serialization_allowed"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.shared_mem_carveout`: `!cuda.runtime.launch_attribute_value.shared_mem_carveout` +pub const RuntimeLaunchAttributeValueSharedMemCarveoutType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemCarveout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.shared_mem_carveout"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValueSharedMemCarveoutTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.shared_memory_mode`: `!cuda.runtime.launch_attribute_value.shared_memory_mode` +pub const RuntimeLaunchAttributeValueSharedMemoryModeType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemoryMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.shared_memory_mode"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValueSharedMemoryModeTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.sync_policy`: `!cuda.runtime.launch_attribute_value.sync_policy` +pub const RuntimeLaunchAttributeValueSyncPolicyType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueSyncPolicy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.sync_policy"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValueSyncPolicyTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.memcpy_3d_params`: CUDA cudaMemcpy3DParms struct type (value-typed). +pub const RuntimeMemcpy3DParamsType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeMemcpy3DParams; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "runtime.memcpy_3d_params"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeMemcpy3DParamsTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.pitched_ptr`: CUDA cudaPitchedPtr struct type (value-typed). +pub const RuntimePitchedPtrType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimePitchedPtr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "runtime.pitched_ptr"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimePitchedPtrTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.pos`: CUDA cudaPos struct type (value-typed). +pub const RuntimePosType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimePos; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "runtime.pos"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimePosTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.uuid`: CUDA cudaUUID_t struct type (value-typed). +pub const RuntimeUuidType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeUuid; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "runtime.uuid"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeUuidTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.stream`: CUDA Stream +pub const StreamType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaStream; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "stream"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaStreamTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.tensor_map`: CUDA Tensor Map type +pub const TensorMapType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaTensorMap; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "tensor_map"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaTensorMapTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +// Operation builders, generated from the operations' .td. +/// `cuda.addressof`. Result types are explicit; attributes use mlir.Attribute. +pub fn addressof(ctx: *mlir.Context, result_type: *const mlir.Type, symname: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.addressof", .{ + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "symname", symname), + }, + .location = location, + }); +} + +/// `cuda.binary`. Result types are explicit; attributes use mlir.Attribute. Not verified on creation: the blocks are usually filled afterwards. +pub fn binary(ctx: *mlir.Context, sym_name: *const mlir.Attribute, binary_: *const mlir.Attribute, blocks: [1]*mlir.Block, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.binary", .{ + .attributes = &.{ + .named(ctx, "sym_name", sym_name), + .named(ctx, "binary", binary_), + }, + .blocks = &blocks, + .verify = false, + .location = location, + }); +} + +/// `cuda.cast`. Result types are explicit; attributes use mlir.Attribute. +pub fn cast(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.cast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cuda.compose`. Result types are explicit; attributes use mlir.Attribute. +pub fn compose(ctx: *mlir.Context, fields: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.compose", .{ + .operands = .{ .flat = fields }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.decompose`. Result types are explicit; attributes use mlir.Attribute. +pub fn decompose(ctx: *mlir.Context, source: *const mlir.Value, fields_type: ?*const mlir.Type, location: *const mlir.Location) *mlir.Operation { + var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; + if (fields_type) |value| results.appendAssumeCapacity(value); + return mlir.Operation.make(ctx, "cuda.decompose", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = results.constSlice() }, + .location = location, + }); +} + +/// `cuda.device.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn device_sync(ctx: *mlir.Context, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.device.sync", .{ + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.driver.event.create`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_event_create(ctx: *mlir.Context, Flags: *const mlir.Value, cuda_result_type: *const mlir.Type, phEvent_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.event.create", .{ + .operands = .{ .flat = &.{Flags} }, + .results = .{ .flat = &.{ cuda_result_type, phEvent_type } }, + .location = location, + }); +} + +/// `cuda.driver.event.destroy`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_event_destroy(ctx: *mlir.Context, hEvent: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.event.destroy", .{ + .operands = .{ .flat = &.{hEvent} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.driver.event.elapsed_time`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_event_elapsed_time(ctx: *mlir.Context, hStart: *const mlir.Value, hEnd: *const mlir.Value, cuda_result_type: *const mlir.Type, pMilliseconds_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.event.elapsed_time", .{ + .operands = .{ .flat = &.{ hStart, hEnd } }, + .results = .{ .flat = &.{ cuda_result_type, pMilliseconds_type } }, + .location = location, + }); +} + +/// `cuda.driver.event.query`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_event_query(ctx: *mlir.Context, hEvent: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.event.query", .{ + .operands = .{ .flat = &.{hEvent} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.driver.event.record`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_event_record(ctx: *mlir.Context, hEvent: *const mlir.Value, hStream: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.event.record", .{ + .operands = .{ .flat = &.{ hEvent, hStream } }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.driver.event.record_with_flags`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_event_record_with_flags(ctx: *mlir.Context, hEvent: *const mlir.Value, hStream: *const mlir.Value, flags: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.event.record_with_flags", .{ + .operands = .{ .flat = &.{ hEvent, hStream, flags } }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.driver.event.synchronize`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_event_synchronize(ctx: *mlir.Context, hEvent: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.event.synchronize", .{ + .operands = .{ .flat = &.{hEvent} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.driver.library.get_global`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_get_global(ctx: *mlir.Context, library: *const mlir.Value, name_arg: *const mlir.Value, cuda_result_type: *const mlir.Type, dptr_type: *const mlir.Type, bytes_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.get_global", .{ + .operands = .{ .flat = &.{ library, name_arg } }, + .results = .{ .flat = &.{ cuda_result_type, dptr_type, bytes_type } }, + .location = location, + }); +} + +/// `cuda.driver.library.get_kernel`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_get_kernel(ctx: *mlir.Context, library: *const mlir.Value, name_arg: *const mlir.Value, cuda_result_type: *const mlir.Type, pKernel_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.get_kernel", .{ + .operands = .{ .flat = &.{ library, name_arg } }, + .results = .{ .flat = &.{ cuda_result_type, pKernel_type } }, + .location = location, + }); +} + +/// `cuda.driver.library.get_kernel_count`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_get_kernel_count(ctx: *mlir.Context, lib: *const mlir.Value, cuda_result_type: *const mlir.Type, count_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.get_kernel_count", .{ + .operands = .{ .flat = &.{lib} }, + .results = .{ .flat = &.{ cuda_result_type, count_type } }, + .location = location, + }); +} + +/// `cuda.driver.library.get_managed`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_get_managed(ctx: *mlir.Context, library: *const mlir.Value, name_arg: *const mlir.Value, cuda_result_type: *const mlir.Type, dptr_type: *const mlir.Type, bytes_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.get_managed", .{ + .operands = .{ .flat = &.{ library, name_arg } }, + .results = .{ .flat = &.{ cuda_result_type, dptr_type, bytes_type } }, + .location = location, + }); +} + +/// `cuda.driver.library.get_module`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_get_module(ctx: *mlir.Context, library: *const mlir.Value, cuda_result_type: *const mlir.Type, pMod_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.get_module", .{ + .operands = .{ .flat = &.{library} }, + .results = .{ .flat = &.{ cuda_result_type, pMod_type } }, + .location = location, + }); +} + +/// `cuda.driver.library.get_unified_function`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_get_unified_function(ctx: *mlir.Context, library: *const mlir.Value, symbol: *const mlir.Value, cuda_result_type: *const mlir.Type, fptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.get_unified_function", .{ + .operands = .{ .flat = &.{ library, symbol } }, + .results = .{ .flat = &.{ cuda_result_type, fptr_type } }, + .location = location, + }); +} + +/// `cuda.driver.library.load_data`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_load_data(ctx: *mlir.Context, code_arg: *const mlir.Value, jitOptions: *const mlir.Value, jitOptionsValues: *const mlir.Value, numJitOptions: *const mlir.Value, libraryOptions: *const mlir.Value, libraryOptionValues: *const mlir.Value, numLibraryOptions: *const mlir.Value, cuda_result_type: *const mlir.Type, library_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.load_data", .{ + .operands = .{ .flat = &.{ code_arg, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions } }, + .results = .{ .flat = &.{ cuda_result_type, library_type } }, + .location = location, + }); +} + +/// `cuda.driver.library.load_from_file`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_load_from_file(ctx: *mlir.Context, fileName: *const mlir.Value, jitOptions: *const mlir.Value, jitOptionsValues: *const mlir.Value, numJitOptions: *const mlir.Value, libraryOptions: *const mlir.Value, libraryOptionValues: *const mlir.Value, numLibraryOptions: *const mlir.Value, cuda_result_type: *const mlir.Type, library_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.load_from_file", .{ + .operands = .{ .flat = &.{ fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions } }, + .results = .{ .flat = &.{ cuda_result_type, library_type } }, + .location = location, + }); +} + +/// `cuda.driver.library.unload`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_unload(ctx: *mlir.Context, library: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.unload", .{ + .operands = .{ .flat = &.{library} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.driver.tensor_map.encode_im2col`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_tensor_map_encode_im2col(ctx: *mlir.Context, tensorDataType: *const mlir.Value, tensorRank: *const mlir.Value, globalAddress: *const mlir.Value, globalDim: *const mlir.Value, globalStrides: *const mlir.Value, pixelBoxLowerCorner: *const mlir.Value, pixelBoxUpperCorner: *const mlir.Value, channelsPerPixel: *const mlir.Value, pixelsPerColumn: *const mlir.Value, elementStrides: *const mlir.Value, interleave: *const mlir.Value, swizzle: *const mlir.Value, l2Promotion: *const mlir.Value, oobFill: *const mlir.Value, cuda_result_type: *const mlir.Type, tensorMap_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.tensor_map.encode_im2col", .{ + .operands = .{ .flat = &.{ tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, pixelBoxLowerCorner, pixelBoxUpperCorner, channelsPerPixel, pixelsPerColumn, elementStrides, interleave, swizzle, l2Promotion, oobFill } }, + .results = .{ .flat = &.{ cuda_result_type, tensorMap_type } }, + .location = location, + }); +} + +/// `cuda.driver.tensor_map.encode_im2col_wide`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_tensor_map_encode_im2col_wide(ctx: *mlir.Context, tensorDataType: *const mlir.Value, tensorRank: *const mlir.Value, globalAddress: *const mlir.Value, globalDim: *const mlir.Value, globalStrides: *const mlir.Value, pixelBoxLowerCornerWidth: *const mlir.Value, pixelBoxUpperCornerWidth: *const mlir.Value, channelsPerPixel: *const mlir.Value, pixelsPerColumn: *const mlir.Value, elementStrides: *const mlir.Value, interleave: *const mlir.Value, mode: *const mlir.Value, swizzle: *const mlir.Value, l2Promotion: *const mlir.Value, oobFill: *const mlir.Value, cuda_result_type: *const mlir.Type, tensorMap_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.tensor_map.encode_im2col_wide", .{ + .operands = .{ .flat = &.{ tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, pixelBoxLowerCornerWidth, pixelBoxUpperCornerWidth, channelsPerPixel, pixelsPerColumn, elementStrides, interleave, mode, swizzle, l2Promotion, oobFill } }, + .results = .{ .flat = &.{ cuda_result_type, tensorMap_type } }, + .location = location, + }); +} + +/// `cuda.driver.tensor_map.replace_address`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_tensor_map_replace_address(ctx: *mlir.Context, globalAddress: *const mlir.Value, cuda_result_type: *const mlir.Type, tensorMap_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.tensor_map.replace_address", .{ + .operands = .{ .flat = &.{globalAddress} }, + .results = .{ .flat = &.{ cuda_result_type, tensorMap_type } }, + .location = location, + }); +} + +/// `cuda.event.create`. Result types are explicit; attributes use mlir.Attribute. +pub fn event_create(ctx: *mlir.Context, result_type: *const mlir.Type, event_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.event.create", .{ + .results = .{ .flat = &.{ result_type, event_type } }, + .location = location, + }); +} + +/// `cuda.event.destroy`. Result types are explicit; attributes use mlir.Attribute. +pub fn event_destroy(ctx: *mlir.Context, event: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.event.destroy", .{ + .operands = .{ .flat = &.{event} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.event.elapsed_time`. Result types are explicit; attributes use mlir.Attribute. +pub fn event_elapsed_time(ctx: *mlir.Context, start_event: *const mlir.Value, end_event: *const mlir.Value, cuda_result_type: *const mlir.Type, elapsed_time_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.event.elapsed_time", .{ + .operands = .{ .flat = &.{ start_event, end_event } }, + .results = .{ .flat = &.{ cuda_result_type, elapsed_time_type } }, + .location = location, + }); +} + +/// `cuda.event.record`. Result types are explicit; attributes use mlir.Attribute. +pub fn event_record(ctx: *mlir.Context, event: *const mlir.Value, stream: ?*const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.event.record", .{ + .operands = .{ .flat = if (stream) |value| &.{ event, value } else &.{event} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.func`. Result types are explicit; attributes use mlir.Attribute. Not verified on creation: the blocks are usually filled afterwards. +pub fn func(ctx: *mlir.Context, sym_name: *const mlir.Attribute, function_type: *const mlir.Attribute, arg_attrs: ?*const mlir.Attribute, res_attrs: ?*const mlir.Attribute, blocks: [1]*mlir.Block, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "sym_name", sym_name)); + attributes.appendAssumeCapacity(.named(ctx, "function_type", function_type)); + if (arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); + if (res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); + return mlir.Operation.make(ctx, "cuda.func", .{ + .attributes = attributes.constSlice(), + .blocks = &blocks, + .verify = false, + .location = location, + }); +} + +/// `cuda.func.get_name`. Result types are explicit; attributes use mlir.Attribute. +pub fn func_get_name(ctx: *mlir.Context, kernel_: *const mlir.Value, result_type: *const mlir.Type, name_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.func.get_name", .{ + .operands = .{ .flat = &.{kernel_} }, + .results = .{ .flat = &.{ result_type, name_type } }, + .location = location, + }); +} + +/// `cuda.graph.destroy`. Result types are explicit; attributes use mlir.Attribute. +pub fn graph_destroy(ctx: *mlir.Context, graph: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.graph.destroy", .{ + .operands = .{ .flat = &.{graph} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.graph.instantiate`. Result types are explicit; attributes use mlir.Attribute. +pub fn graph_instantiate(ctx: *mlir.Context, graph: *const mlir.Value, result_type: *const mlir.Type, graph_exec_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.graph.instantiate", .{ + .operands = .{ .flat = &.{graph} }, + .results = .{ .flat = &.{ result_type, graph_exec_type } }, + .location = location, + }); +} + +/// `cuda.graph.launch`. Result types are explicit; attributes use mlir.Attribute. +pub fn graph_launch(ctx: *mlir.Context, graph_exec: *const mlir.Value, stream: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.graph.launch", .{ + .operands = .{ .flat = &.{ graph_exec, stream } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.graph_exec.destroy`. Result types are explicit; attributes use mlir.Attribute. +pub fn graph_exec_destroy(ctx: *mlir.Context, graph_exec: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.graph_exec.destroy", .{ + .operands = .{ .flat = &.{graph_exec} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.kernel`. Result types are explicit; attributes use mlir.Attribute. Not verified on creation: the blocks are usually filled afterwards. +pub fn kernel(ctx: *mlir.Context, sym_name: *const mlir.Attribute, function_type: *const mlir.Attribute, arg_attrs: ?*const mlir.Attribute, res_attrs: ?*const mlir.Attribute, cu_func_attrs: ?*const mlir.Attribute, blocks: [1]*mlir.Block, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "sym_name", sym_name)); + attributes.appendAssumeCapacity(.named(ctx, "function_type", function_type)); + if (arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); + if (res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); + if (cu_func_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "cu_func_attrs", value)); + return mlir.Operation.make(ctx, "cuda.kernel", .{ + .attributes = attributes.constSlice(), + .blocks = &blocks, + .verify = false, + .location = location, + }); +} + +/// `cuda.launch`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch(ctx: *mlir.Context, grid: []const *const mlir.Value, block: []const *const mlir.Value, dynamicSmem: *const mlir.Value, cudaStream: *const mlir.Value, inputs: []const *const mlir.Value, cuda_result_type: *const mlir.Type, callee: *const mlir.Attribute, arg_attrs: ?*const mlir.Attribute, res_attrs: ?*const mlir.Attribute, assume_kernel_attr: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "callee", callee)); + if (arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); + if (res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); + if (assume_kernel_attr) |value| attributes.appendAssumeCapacity(.named(ctx, "assume_kernel_attr", value)); + return mlir.Operation.make(ctx, "cuda.launch", .{ + .operands = .{ .variadic = &.{ + grid, + block, + &.{dynamicSmem}, + &.{cudaStream}, + inputs, + } }, + .results = .{ .flat = &.{cuda_result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cuda.launch_cfg.access_policy_window`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_access_policy_window(ctx: *mlir.Context, launch_cfg: *const mlir.Value, base_ptr: *const mlir.Value, hitRatio: *const mlir.Value, num_bytes: *const mlir.Value, hitProp: *const mlir.Attribute, missProp: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.access_policy_window", .{ + .operands = .{ .flat = &.{ launch_cfg, base_ptr, hitRatio, num_bytes } }, + .attributes = &.{ + .named(ctx, "hitProp", hitProp), + .named(ctx, "missProp", missProp), + }, + .location = location, + }); +} + +/// `cuda.launch_cfg.cluster_dim`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_cluster_dim(ctx: *mlir.Context, launch_cfg: *const mlir.Value, clusterDimX: *const mlir.Value, clusterDimY: *const mlir.Value, clusterDimZ: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.cluster_dim", .{ + .operands = .{ .flat = &.{ launch_cfg, clusterDimX, clusterDimY, clusterDimZ } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.cluster_scheduling_policy`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_cluster_scheduling_policy(ctx: *mlir.Context, launch_cfg: *const mlir.Value, policy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.cluster_scheduling_policy", .{ + .operands = .{ .flat = &.{launch_cfg} }, + .attributes = &.{ + .named(ctx, "policy", policy), + }, + .location = location, + }); +} + +/// `cuda.launch_cfg.cooperative`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_cooperative(ctx: *mlir.Context, launch_cfg: *const mlir.Value, cooperative: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.cooperative", .{ + .operands = .{ .flat = &.{ launch_cfg, cooperative } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.create`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_create(ctx: *mlir.Context, blockDimX: *const mlir.Value, blockDimY: *const mlir.Value, blockDimZ: *const mlir.Value, dynamicSmemBytes: *const mlir.Value, gridDimX: *const mlir.Value, gridDimY: *const mlir.Value, gridDimZ: *const mlir.Value, stream: *const mlir.Value, result_type: *const mlir.Type, maxNumAttrs: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.create", .{ + .operands = .{ .flat = &.{ blockDimX, blockDimY, blockDimZ, dynamicSmemBytes, gridDimX, gridDimY, gridDimZ, stream } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "maxNumAttrs", maxNumAttrs), + }, + .location = location, + }); +} + +/// `cuda.launch_cfg.create_from_stream`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_create_from_stream(ctx: *mlir.Context, stream: *const mlir.Value, result_type: *const mlir.Type, maxNumAttrs: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.create_from_stream", .{ + .operands = .{ .flat = &.{stream} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "maxNumAttrs", maxNumAttrs), + }, + .location = location, + }); +} + +/// `cuda.launch_cfg.device_updatable_kernel_node`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_device_updatable_kernel_node(ctx: *mlir.Context, launch_cfg: *const mlir.Value, devNode: *const mlir.Value, deviceUpdatable: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.device_updatable_kernel_node", .{ + .operands = .{ .flat = &.{ launch_cfg, devNode, deviceUpdatable } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.grid_dimensions`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_grid_dimensions(ctx: *mlir.Context, launch_cfg: *const mlir.Value, gridDimX: *const mlir.Value, gridDimY: *const mlir.Value, gridDimZ: *const mlir.Value, blockDimX: *const mlir.Value, blockDimY: *const mlir.Value, blockDimZ: *const mlir.Value, dynamicSmemBytes: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.grid_dimensions", .{ + .operands = .{ .flat = &.{ launch_cfg, gridDimX, gridDimY, gridDimZ, blockDimX, blockDimY, blockDimZ, dynamicSmemBytes } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.launch_completion_event`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_launch_completion_event(ctx: *mlir.Context, launch_cfg: *const mlir.Value, event: *const mlir.Value, flags: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.launch_completion_event", .{ + .operands = .{ .flat = &.{ launch_cfg, event, flags } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.mem_sync_domain`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_mem_sync_domain(ctx: *mlir.Context, launch_cfg: *const mlir.Value, memSyncDomain: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.mem_sync_domain", .{ + .operands = .{ .flat = &.{launch_cfg} }, + .attributes = &.{ + .named(ctx, "memSyncDomain", memSyncDomain), + }, + .location = location, + }); +} + +/// `cuda.launch_cfg.mem_sync_domain_map`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_mem_sync_domain_map(ctx: *mlir.Context, launch_cfg: *const mlir.Value, domain_default: *const mlir.Value, domain_remote: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.mem_sync_domain_map", .{ + .operands = .{ .flat = &.{ launch_cfg, domain_default, domain_remote } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.nvlink_util_centric_scheduling`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_nvlink_util_centric_scheduling(ctx: *mlir.Context, launch_cfg: *const mlir.Value, nvlinkSchedulingEnabled: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.nvlink_util_centric_scheduling", .{ + .operands = .{ .flat = &.{ launch_cfg, nvlinkSchedulingEnabled } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.preferred_cluster_dim`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_preferred_cluster_dim(ctx: *mlir.Context, launch_cfg: *const mlir.Value, preferredClusterDimX: *const mlir.Value, preferredClusterDimY: *const mlir.Value, preferredClusterDimZ: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.preferred_cluster_dim", .{ + .operands = .{ .flat = &.{ launch_cfg, preferredClusterDimX, preferredClusterDimY, preferredClusterDimZ } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.print`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_print(ctx: *mlir.Context, launch_cfg: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.print", .{ + .operands = .{ .flat = &.{launch_cfg} }, + .location = location, + }); +} + +/// `cuda.launch_cfg.priority`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_priority(ctx: *mlir.Context, launch_cfg: *const mlir.Value, priority: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.priority", .{ + .operands = .{ .flat = &.{ launch_cfg, priority } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.programmatic_event`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_programmatic_event(ctx: *mlir.Context, launch_cfg: *const mlir.Value, event: *const mlir.Value, flags: *const mlir.Value, triggerAtBlockStart: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.programmatic_event", .{ + .operands = .{ .flat = &.{ launch_cfg, event, flags, triggerAtBlockStart } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.programmatic_stream_serialization_allowed`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_programmatic_stream_serialization_allowed(ctx: *mlir.Context, launch_cfg: *const mlir.Value, programmaticStreamSerializationAllowed: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.programmatic_stream_serialization_allowed", .{ + .operands = .{ .flat = &.{ launch_cfg, programmaticStreamSerializationAllowed } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.shared_mem_carveout`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_shared_mem_carveout(ctx: *mlir.Context, launch_cfg: *const mlir.Value, sharedMemCarveout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.shared_mem_carveout", .{ + .operands = .{ .flat = &.{ launch_cfg, sharedMemCarveout } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.sync_policy`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_sync_policy(ctx: *mlir.Context, launch_cfg: *const mlir.Value, syncPolicy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.sync_policy", .{ + .operands = .{ .flat = &.{launch_cfg} }, + .attributes = &.{ + .named(ctx, "syncPolicy", syncPolicy), + }, + .location = location, + }); +} + +/// `cuda.launch_ex`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_ex(ctx: *mlir.Context, launch_cfg: *const mlir.Value, inputs: []const *const mlir.Value, cuda_result_type: *const mlir.Type, callee: *const mlir.Attribute, arg_attrs: ?*const mlir.Attribute, res_attrs: ?*const mlir.Attribute, assume_kernel_attr: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 256) = .empty; + operands.appendAssumeCapacity(launch_cfg); + operands.appendSliceAssumeCapacity(inputs); + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "callee", callee)); + if (arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); + if (res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); + if (assume_kernel_attr) |value| attributes.appendAssumeCapacity(.named(ctx, "assume_kernel_attr", value)); + return mlir.Operation.make(ctx, "cuda.launch_ex", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = &.{cuda_result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cuda.launch_kernel`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_kernel(ctx: *mlir.Context, kernel_: *const mlir.Value, grid: []const *const mlir.Value, block: []const *const mlir.Value, dynamicSmem: *const mlir.Value, cudaStream: *const mlir.Value, kernel_args: *const mlir.Value, cuda_result_type: *const mlir.Type, arg_attrs: ?*const mlir.Attribute, res_attrs: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); + if (res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); + return mlir.Operation.make(ctx, "cuda.launch_kernel", .{ + .operands = .{ .variadic = &.{ + &.{kernel_}, + grid, + block, + &.{dynamicSmem}, + &.{cudaStream}, + &.{kernel_args}, + } }, + .results = .{ .flat = &.{cuda_result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cuda.library.get_kernel`. Result types are explicit; attributes use mlir.Attribute. +pub fn library_get_kernel(ctx: *mlir.Context, library: *const mlir.Value, kernel_name: *const mlir.Value, result_type: *const mlir.Type, kernel_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.library.get_kernel", .{ + .operands = .{ .flat = &.{ library, kernel_name } }, + .results = .{ .flat = &.{ result_type, kernel_type } }, + .location = location, + }); +} + +/// `cuda.library.load_data`. Result types are explicit; attributes use mlir.Attribute. +pub fn library_load_data(ctx: *mlir.Context, data: *const mlir.Value, result_type: *const mlir.Type, library_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.library.load_data", .{ + .operands = .{ .flat = &.{data} }, + .results = .{ .flat = &.{ result_type, library_type } }, + .location = location, + }); +} + +/// `cuda.library.unload`. Result types are explicit; attributes use mlir.Attribute. +pub fn library_unload(ctx: *mlir.Context, library: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.library.unload", .{ + .operands = .{ .flat = &.{library} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.memory.alloc`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_alloc(ctx: *mlir.Context, bytes: *const mlir.Value, cuda_result_type: *const mlir.Type, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.alloc", .{ + .operands = .{ .flat = &.{bytes} }, + .results = .{ .flat = &.{ cuda_result_type, ptr_type } }, + .location = location, + }); +} + +/// `cuda.memory.alloc_async`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_alloc_async(ctx: *mlir.Context, bytes: *const mlir.Value, stream: *const mlir.Value, cuda_result_type: *const mlir.Type, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.alloc_async", .{ + .operands = .{ .flat = &.{ bytes, stream } }, + .results = .{ .flat = &.{ cuda_result_type, ptr_type } }, + .location = location, + }); +} + +/// `cuda.memory.alloc_host`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_alloc_host(ctx: *mlir.Context, bytes: *const mlir.Value, cuda_result_type: *const mlir.Type, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.alloc_host", .{ + .operands = .{ .flat = &.{bytes} }, + .results = .{ .flat = &.{ cuda_result_type, ptr_type } }, + .location = location, + }); +} + +/// `cuda.memory.alloc_managed`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_alloc_managed(ctx: *mlir.Context, bytes: *const mlir.Value, cuda_result_type: *const mlir.Type, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.alloc_managed", .{ + .operands = .{ .flat = &.{bytes} }, + .results = .{ .flat = &.{ cuda_result_type, ptr_type } }, + .location = location, + }); +} + +/// `cuda.memory.copy`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_copy(ctx: *mlir.Context, src: *const mlir.Value, dst: *const mlir.Value, bytes: *const mlir.Value, cuda_result_type: *const mlir.Type, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.copy", .{ + .operands = .{ .flat = &.{ src, dst, bytes } }, + .results = .{ .flat = &.{cuda_result_type} }, + .attributes = &.{ + .named(ctx, "kind", kind), + }, + .location = location, + }); +} + +/// `cuda.memory.copy_async`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_copy_async(ctx: *mlir.Context, src: *const mlir.Value, dst: *const mlir.Value, bytes: *const mlir.Value, stream: ?*const mlir.Value, cuda_result_type: *const mlir.Type, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.copy_async", .{ + .operands = .{ .flat = if (stream) |value| &.{ src, dst, bytes, value } else &.{ src, dst, bytes } }, + .results = .{ .flat = &.{cuda_result_type} }, + .attributes = &.{ + .named(ctx, "kind", kind), + }, + .location = location, + }); +} + +/// `cuda.memory.free`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_free(ctx: *mlir.Context, ptr: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.free", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.memory.free_async`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_free_async(ctx: *mlir.Context, ptr: *const mlir.Value, stream: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.free_async", .{ + .operands = .{ .flat = &.{ ptr, stream } }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.memory.free_host`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_free_host(ctx: *mlir.Context, ptr: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.free_host", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.memory.set`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_set(ctx: *mlir.Context, ptr: *const mlir.Value, value_: *const mlir.Value, bytes: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.set", .{ + .operands = .{ .flat = &.{ ptr, value_, bytes } }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.memory.set_async`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_set_async(ctx: *mlir.Context, ptr: *const mlir.Value, value_: *const mlir.Value, bytes: *const mlir.Value, stream: ?*const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.set_async", .{ + .operands = .{ .flat = if (stream) |value| &.{ ptr, value_, bytes, value } else &.{ ptr, value_, bytes } }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.occupancy_max_active_blocks_per_multiprocessor`. Result types are explicit; attributes use mlir.Attribute. +pub fn occupancy_max_active_blocks_per_multiprocessor(ctx: *mlir.Context, block_size: *const mlir.Value, dynamic_smem_size: *const mlir.Value, res_type: *const mlir.Type, num_blocks_type: *const mlir.Type, func_: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.occupancy_max_active_blocks_per_multiprocessor", .{ + .operands = .{ .flat = &.{ block_size, dynamic_smem_size } }, + .results = .{ .flat = &.{ res_type, num_blocks_type } }, + .attributes = &.{ + .named(ctx, "func", func_), + }, + .location = location, + }); +} + +/// `cuda.profiler.start`. Result types are explicit; attributes use mlir.Attribute. +pub fn profiler_start(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.profiler.start", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.profiler.stop`. Result types are explicit; attributes use mlir.Attribute. +pub fn profiler_stop(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.profiler.stop", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.result.assert_success`. Result types are explicit; attributes use mlir.Attribute. +pub fn result_assert_success(ctx: *mlir.Context, cuda_result: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.result.assert_success", .{ + .operands = .{ .flat = &.{cuda_result} }, + .location = location, + }); +} + +/// `cuda.result.failed`. Result types are explicit; attributes use mlir.Attribute. +pub fn result_failed(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.result.failed", .{ + .location = location, + }); +} + +/// `cuda.result.print_on_error`. Result types are explicit; attributes use mlir.Attribute. +pub fn result_print_on_error(ctx: *mlir.Context, cuda_result: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.result.print_on_error", .{ + .operands = .{ .flat = &.{cuda_result} }, + .location = location, + }); +} + +/// `cuda.result.succeeded`. Result types are explicit; attributes use mlir.Attribute. +pub fn result_succeeded(ctx: *mlir.Context, cuda_result: *const mlir.Value, success_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.result.succeeded", .{ + .operands = .{ .flat = &.{cuda_result} }, + .results = .{ .flat = &.{success_type} }, + .location = location, + }); +} + +/// `cuda.return`. Result types are explicit; attributes use mlir.Attribute. Not verified on creation: a terminator is only one in its block. +pub fn @"return"(ctx: *mlir.Context, inputs: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.return", .{ + .operands = .{ .flat = inputs }, + .verify = false, + .location = location, + }); +} + +/// `cuda.return_if_error`. Result types are explicit; attributes use mlir.Attribute. +pub fn return_if_error(ctx: *mlir.Context, inputs: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.return_if_error", .{ + .operands = .{ .flat = inputs }, + .location = location, + }); +} + +/// `cuda.runtime.device.properties`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_properties(ctx: *mlir.Context, device: *const mlir.Value, cuda_result_type: *const mlir.Type, prop_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device.properties", .{ + .operands = .{ .flat = &.{device} }, + .results = .{ .flat = &.{ cuda_result_type, prop_type } }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_access_policy_max_window_size`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_access_policy_max_window_size(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_access_policy_max_window_size", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_async_engine_count`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_async_engine_count(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_async_engine_count", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_can_map_host_memory`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_can_map_host_memory(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_can_map_host_memory", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_can_use_host_pointer_for_registered_mem`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_can_use_host_pointer_for_registered_mem(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_can_use_host_pointer_for_registered_mem", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_cluster_launch`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_cluster_launch(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_cluster_launch", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_compute_preemption_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_compute_preemption_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_compute_preemption_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_concurrent_kernels`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_concurrent_kernels(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_concurrent_kernels", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_concurrent_managed_access`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_concurrent_managed_access(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_concurrent_managed_access", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_cooperative_launch`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_cooperative_launch(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_cooperative_launch", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_deferred_mapping_cuda_array_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_deferred_mapping_cuda_array_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_deferred_mapping_cuda_array_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_device_numa_config`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_device_numa_config(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_device_numa_config", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_device_numa_id`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_device_numa_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_device_numa_id", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_direct_managed_mem_access_from_host`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_direct_managed_mem_access_from_host(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_direct_managed_mem_access_from_host", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_ecc_enabled`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_ecc_enabled(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_ecc_enabled", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_global_l1cache_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_global_l1cache_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_global_l1cache_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_gpu_direct_rdma_flush_writes_options`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_gpu_direct_rdma_flush_writes_options(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_gpu_direct_rdma_flush_writes_options", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_gpu_direct_rdma_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_gpu_direct_rdma_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_gpu_direct_rdma_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_gpu_direct_rdma_writes_ordering`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_gpu_direct_rdma_writes_ordering(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_gpu_direct_rdma_writes_ordering", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_gpu_pci_device_id`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_gpu_pci_device_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_gpu_pci_device_id", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_gpu_pci_subsystem_id`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_gpu_pci_subsystem_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_gpu_pci_subsystem_id", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_host_native_atomic_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_host_native_atomic_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_host_native_atomic_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_host_numa_id`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_host_numa_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_host_numa_id", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_host_numa_multinode_ipc_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_host_numa_multinode_ipc_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_host_numa_multinode_ipc_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_host_register_read_only_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_host_register_read_only_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_host_register_read_only_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_host_register_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_host_register_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_host_register_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_integrated`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_integrated(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_integrated", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_ipc_event_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_ipc_event_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_ipc_event_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_is_multi_gpu_board`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_is_multi_gpu_board(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_is_multi_gpu_board", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_l_2cache_size`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_l_2cache_size(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_l_2cache_size", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_local_l1cache_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_local_l1cache_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_local_l1cache_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_luid`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_luid(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_luid", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_luid_device_node_mask`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_luid_device_node_mask(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_luid_device_node_mask", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_major`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_major(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_major", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_managed_memory`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_managed_memory(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_managed_memory", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_blocks_per_multi_processor`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_blocks_per_multi_processor(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_blocks_per_multi_processor", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_grid_size`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_grid_size(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_grid_size", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_surface_1d`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_surface_1d(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_1d", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_surface_1d_layered`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_surface_1d_layered(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_1d_layered", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_surface_2d`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_surface_2d(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_2d", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_surface_2d_layered`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_surface_2d_layered(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_2d_layered", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_surface_3d`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_surface_3d(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_3d", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_surface_cubemap`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_surface_cubemap(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_cubemap", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_surface_cubemap_layered`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_surface_cubemap_layered(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_cubemap_layered", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_1d`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_1d(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_1d", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_1d_layered`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_1d_layered(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_1d_layered", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_1d_mipmap`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_1d_mipmap(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_1d_mipmap", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_2d`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_2d(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_2d", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_2d_gather`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_2d_gather(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_2d_gather", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_2d_layered`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_2d_layered(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_2d_layered", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_2d_linear`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_2d_linear(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_2d_linear", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_2d_mipmap`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_2d_mipmap(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_2d_mipmap", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_3d`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_3d(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_3d", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_3d_alt`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_3d_alt(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_3d_alt", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_cubemap`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_cubemap(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_cubemap", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_cubemap_layered`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_cubemap_layered(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_cubemap_layered", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_threads_dim`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_threads_dim(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_threads_dim", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_threads_per_block`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_threads_per_block(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_threads_per_block", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_threads_per_multi_processor`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_threads_per_multi_processor(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_threads_per_multi_processor", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_mem_pitch`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_mem_pitch(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_mem_pitch", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_memory_bus_width`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_memory_bus_width(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_memory_bus_width", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_memory_pool_supported_handle_types`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_memory_pool_supported_handle_types(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_memory_pool_supported_handle_types", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_memory_pools_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_memory_pools_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_memory_pools_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_minor`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_minor(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_minor", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_mps_enabled`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_mps_enabled(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_mps_enabled", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_multi_gpu_board_group_id`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_multi_gpu_board_group_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_multi_gpu_board_group_id", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_multi_processor_count`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_multi_processor_count(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_multi_processor_count", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_name`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_name(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_name", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_pageable_memory_access`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_pageable_memory_access(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_pageable_memory_access", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_pageable_memory_access_uses_host_page_tables`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_pageable_memory_access_uses_host_page_tables(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_pageable_memory_access_uses_host_page_tables", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_pci_bus_id`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_pci_bus_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_pci_bus_id", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_pci_device_id`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_pci_device_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_pci_device_id", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_pci_domain_id`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_pci_domain_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_pci_domain_id", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_persisting_l2cache_max_size`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_persisting_l2cache_max_size(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_persisting_l2cache_max_size", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_regs_per_block`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_regs_per_block(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_regs_per_block", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_regs_per_multiprocessor`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_regs_per_multiprocessor(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_regs_per_multiprocessor", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_reserved_shared_mem_per_block`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_reserved_shared_mem_per_block(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_reserved_shared_mem_per_block", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_shared_mem_per_block`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_shared_mem_per_block(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_shared_mem_per_block", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_shared_mem_per_block_optin`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_shared_mem_per_block_optin(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_shared_mem_per_block_optin", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_shared_mem_per_multiprocessor`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_shared_mem_per_multiprocessor(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_shared_mem_per_multiprocessor", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_sparse_cuda_array_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_sparse_cuda_array_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_sparse_cuda_array_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_stream_priorities_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_stream_priorities_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_stream_priorities_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_surface_alignment`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_surface_alignment(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_surface_alignment", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_tcc_driver`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_tcc_driver(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_tcc_driver", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_texture_alignment`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_texture_alignment(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_texture_alignment", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_texture_pitch_alignment`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_texture_pitch_alignment(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_texture_pitch_alignment", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_timeline_semaphore_interop_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_timeline_semaphore_interop_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_timeline_semaphore_interop_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_total_const_mem`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_total_const_mem(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_total_const_mem", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_total_global_mem`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_total_global_mem(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_total_global_mem", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_unified_addressing`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_unified_addressing(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_unified_addressing", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_unified_function_pointers`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_unified_function_pointers(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_unified_function_pointers", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_uuid`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_uuid(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_uuid", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_warp_size`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_warp_size(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_warp_size", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.dim_3.get_x`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_dim_3_get_x(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.dim_3.get_x", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.dim_3.get_y`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_dim_3_get_y(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.dim_3.get_y", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.dim_3.get_z`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_dim_3_get_z(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.dim_3.get_z", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.event.create_with_flags`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_event_create_with_flags(ctx: *mlir.Context, flags: *const mlir.Value, cuda_result_type: *const mlir.Type, event_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.event.create_with_flags", .{ + .operands = .{ .flat = &.{flags} }, + .results = .{ .flat = &.{ cuda_result_type, event_type } }, + .location = location, + }); +} + +/// `cuda.runtime.event.query`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_event_query(ctx: *mlir.Context, event: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.event.query", .{ + .operands = .{ .flat = &.{event} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.event.record_with_flags`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_event_record_with_flags(ctx: *mlir.Context, event: *const mlir.Value, stream: *const mlir.Value, flags: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.event.record_with_flags", .{ + .operands = .{ .flat = &.{ event, stream, flags } }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.event.synchronize`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_event_synchronize(ctx: *mlir.Context, event: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.event.synchronize", .{ + .operands = .{ .flat = &.{event} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.extent.get_depth`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_extent_get_depth(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.extent.get_depth", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.extent.get_height`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_extent_get_height(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.extent.get_height", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.extent.get_width`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_extent_get_width(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.extent.get_width", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.graph.add_kernel_node`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_graph_add_kernel_node(ctx: *mlir.Context, graph: *const mlir.Value, pDependencies: *const mlir.Value, numDependencies: *const mlir.Value, pNodeParams: *const mlir.Value, cuda_result_type: *const mlir.Type, pGraphNode_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.graph.add_kernel_node", .{ + .operands = .{ .flat = &.{ graph, pDependencies, numDependencies, pNodeParams } }, + .results = .{ .flat = &.{ cuda_result_type, pGraphNode_type } }, + .location = location, + }); +} + +/// `cuda.runtime.graph.create`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_graph_create(ctx: *mlir.Context, flags: *const mlir.Value, cuda_result_type: *const mlir.Type, pGraph_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.graph.create", .{ + .operands = .{ .flat = &.{flags} }, + .results = .{ .flat = &.{ cuda_result_type, pGraph_type } }, + .location = location, + }); +} + +/// `cuda.runtime.graph.kernel_node_get_attribute`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_graph_kernel_node_get_attribute(ctx: *mlir.Context, hNode: *const mlir.Value, attr: *const mlir.Value, cuda_result_type: *const mlir.Type, value_out_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.graph.kernel_node_get_attribute", .{ + .operands = .{ .flat = &.{ hNode, attr } }, + .results = .{ .flat = &.{ cuda_result_type, value_out_type } }, + .location = location, + }); +} + +/// `cuda.runtime.graph.kernel_node_set_attribute`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_graph_kernel_node_set_attribute(ctx: *mlir.Context, hNode: *const mlir.Value, attr: *const mlir.Value, value_: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.graph.kernel_node_set_attribute", .{ + .operands = .{ .flat = &.{ hNode, attr, value_ } }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.kernel_node_params.get_block_dim`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_kernel_node_params_get_block_dim(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.kernel_node_params.get_block_dim", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.kernel_node_params.get_extra`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_kernel_node_params_get_extra(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.kernel_node_params.get_extra", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.kernel_node_params.get_func`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_kernel_node_params_get_func(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.kernel_node_params.get_func", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.kernel_node_params.get_grid_dim`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_kernel_node_params_get_grid_dim(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.kernel_node_params.get_grid_dim", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.kernel_node_params.get_kernel_params`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_kernel_node_params_get_kernel_params(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.kernel_node_params.get_kernel_params", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.kernel_node_params.get_shared_mem_bytes`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_kernel_node_params_get_shared_mem_bytes(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.kernel_node_params.get_shared_mem_bytes", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.library.get_global`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_library_get_global(ctx: *mlir.Context, library: *const mlir.Value, name_arg: *const mlir.Value, cuda_result_type: *const mlir.Type, dptr_type: *const mlir.Type, bytes_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.library.get_global", .{ + .operands = .{ .flat = &.{ library, name_arg } }, + .results = .{ .flat = &.{ cuda_result_type, dptr_type, bytes_type } }, + .location = location, + }); +} + +/// `cuda.runtime.library.get_kernel`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_library_get_kernel(ctx: *mlir.Context, library: *const mlir.Value, name_arg: *const mlir.Value, cuda_result_type: *const mlir.Type, pKernel_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.library.get_kernel", .{ + .operands = .{ .flat = &.{ library, name_arg } }, + .results = .{ .flat = &.{ cuda_result_type, pKernel_type } }, + .location = location, + }); +} + +/// `cuda.runtime.library.get_kernel_count`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_library_get_kernel_count(ctx: *mlir.Context, lib: *const mlir.Value, cuda_result_type: *const mlir.Type, count_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.library.get_kernel_count", .{ + .operands = .{ .flat = &.{lib} }, + .results = .{ .flat = &.{ cuda_result_type, count_type } }, + .location = location, + }); +} + +/// `cuda.runtime.library.get_managed`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_library_get_managed(ctx: *mlir.Context, library: *const mlir.Value, name_arg: *const mlir.Value, cuda_result_type: *const mlir.Type, dptr_type: *const mlir.Type, bytes_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.library.get_managed", .{ + .operands = .{ .flat = &.{ library, name_arg } }, + .results = .{ .flat = &.{ cuda_result_type, dptr_type, bytes_type } }, + .location = location, + }); +} + +/// `cuda.runtime.library.get_unified_function`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_library_get_unified_function(ctx: *mlir.Context, library: *const mlir.Value, symbol: *const mlir.Value, cuda_result_type: *const mlir.Type, fptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.library.get_unified_function", .{ + .operands = .{ .flat = &.{ library, symbol } }, + .results = .{ .flat = &.{ cuda_result_type, fptr_type } }, + .location = location, + }); +} + +/// `cuda.runtime.library.load_data`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_library_load_data(ctx: *mlir.Context, code_arg: *const mlir.Value, jitOptions: *const mlir.Value, jitOptionsValues: *const mlir.Value, numJitOptions: *const mlir.Value, libraryOptions: *const mlir.Value, libraryOptionValues: *const mlir.Value, numLibraryOptions: *const mlir.Value, cuda_result_type: *const mlir.Type, library_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.library.load_data", .{ + .operands = .{ .flat = &.{ code_arg, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions } }, + .results = .{ .flat = &.{ cuda_result_type, library_type } }, + .location = location, + }); +} + +/// `cuda.runtime.library.load_from_file`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_library_load_from_file(ctx: *mlir.Context, fileName: *const mlir.Value, jitOptions: *const mlir.Value, jitOptionsValues: *const mlir.Value, numJitOptions: *const mlir.Value, libraryOptions: *const mlir.Value, libraryOptionValues: *const mlir.Value, numLibraryOptions: *const mlir.Value, cuda_result_type: *const mlir.Type, library_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.library.load_from_file", .{ + .operands = .{ .flat = &.{ fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions } }, + .results = .{ .flat = &.{ cuda_result_type, library_type } }, + .location = location, + }); +} + +/// `cuda.runtime.library.unload`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_library_unload(ctx: *mlir.Context, library: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.library.unload", .{ + .operands = .{ .flat = &.{library} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memcpy_3d_params.get_dst_array`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memcpy_3d_params_get_dst_array(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_dst_array", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memcpy_3d_params.get_dst_pos`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memcpy_3d_params_get_dst_pos(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_dst_pos", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memcpy_3d_params.get_dst_ptr`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memcpy_3d_params_get_dst_ptr(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_dst_ptr", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memcpy_3d_params.get_extent`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memcpy_3d_params_get_extent(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_extent", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memcpy_3d_params.get_kind`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memcpy_3d_params_get_kind(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_kind", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memcpy_3d_params.get_src_array`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memcpy_3d_params_get_src_array(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_src_array", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memcpy_3d_params.get_src_pos`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memcpy_3d_params_get_src_pos(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_src_pos", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memcpy_3d_params.get_src_ptr`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memcpy_3d_params_get_src_ptr(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_src_ptr", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memory.cpy3d`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memory_cpy3d(ctx: *mlir.Context, p: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memory.cpy3d", .{ + .operands = .{ .flat = &.{p} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memory.cpy3d_async`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memory_cpy3d_async(ctx: *mlir.Context, p: *const mlir.Value, stream: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memory.cpy3d_async", .{ + .operands = .{ .flat = &.{ p, stream } }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.pitched_ptr.get_pitch`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_pitched_ptr_get_pitch(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.pitched_ptr.get_pitch", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.pitched_ptr.get_ptr`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_pitched_ptr_get_ptr(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.pitched_ptr.get_ptr", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.pitched_ptr.get_xsize`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_pitched_ptr_get_xsize(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.pitched_ptr.get_xsize", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.pitched_ptr.get_ysize`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_pitched_ptr_get_ysize(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.pitched_ptr.get_ysize", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.pos.get_x`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_pos_get_x(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.pos.get_x", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.pos.get_y`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_pos_get_y(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.pos.get_y", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.pos.get_z`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_pos_get_z(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.pos.get_z", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.uuid.get_bytes`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_uuid_get_bytes(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.uuid.get_bytes", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.stream.add_callback`. Result types are explicit; attributes use mlir.Attribute. +pub fn stream_add_callback(ctx: *mlir.Context, stream: *const mlir.Value, user_data: *const mlir.Value, result_type: *const mlir.Type, callback: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.stream.add_callback", .{ + .operands = .{ .flat = &.{ stream, user_data } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "callback", callback), + }, + .location = location, + }); +} + +/// `cuda.stream.begin_capture`. Result types are explicit; attributes use mlir.Attribute. +pub fn stream_begin_capture(ctx: *mlir.Context, stream: *const mlir.Value, result_type: *const mlir.Type, mode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.stream.begin_capture", .{ + .operands = .{ .flat = &.{stream} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "mode", mode), + }, + .location = location, + }); +} + +/// `cuda.stream.create`. Result types are explicit; attributes use mlir.Attribute. +pub fn stream_create(ctx: *mlir.Context, cuda_result_type: *const mlir.Type, stream_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.stream.create", .{ + .results = .{ .flat = &.{ cuda_result_type, stream_type } }, + .location = location, + }); +} + +/// `cuda.stream.destroy`. Result types are explicit; attributes use mlir.Attribute. +pub fn stream_destroy(ctx: *mlir.Context, stream: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.stream.destroy", .{ + .operands = .{ .flat = &.{stream} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.stream.end_capture`. Result types are explicit; attributes use mlir.Attribute. +pub fn stream_end_capture(ctx: *mlir.Context, stream: *const mlir.Value, result_type: *const mlir.Type, graph_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.stream.end_capture", .{ + .operands = .{ .flat = &.{stream} }, + .results = .{ .flat = &.{ result_type, graph_type } }, + .location = location, + }); +} + +/// `cuda.stream.synchronize`. Result types are explicit; attributes use mlir.Attribute. +pub fn stream_synchronize(ctx: *mlir.Context, stream: *const mlir.Value, event: ?*const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.stream.synchronize", .{ + .operands = .{ .flat = if (event) |value| &.{ stream, value } else &.{stream} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.tensor_map_encode_tiled`. Result types are explicit; attributes use mlir.Attribute. +pub fn tensor_map_encode_tiled(ctx: *mlir.Context, tensorDataType: *const mlir.Value, tensorRank: *const mlir.Value, globalAddress: *const mlir.Value, globalDims: *const mlir.Value, globalStrides: *const mlir.Value, boxDims: *const mlir.Value, elementStrides: *const mlir.Value, interleave: *const mlir.Value, swizzle: *const mlir.Value, l2Promotion: *const mlir.Value, oobFill: *const mlir.Value, cuda_result_type: *const mlir.Type, cuTensorMap_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.tensor_map_encode_tiled", .{ + .operands = .{ .flat = &.{ tensorDataType, tensorRank, globalAddress, globalDims, globalStrides, boxDims, elementStrides, interleave, swizzle, l2Promotion, oobFill } }, + .results = .{ .flat = &.{ cuda_result_type, cuTensorMap_type } }, + .location = location, + }); +} + +/// Every operation bound here. +pub const operation_names: []const []const u8 = &.{ + "cuda.addressof", + "cuda.binary", + "cuda.cast", + "cuda.compose", + "cuda.decompose", + "cuda.device.sync", + "cuda.driver.event.create", + "cuda.driver.event.destroy", + "cuda.driver.event.elapsed_time", + "cuda.driver.event.query", + "cuda.driver.event.record", + "cuda.driver.event.record_with_flags", + "cuda.driver.event.synchronize", + "cuda.driver.library.get_global", + "cuda.driver.library.get_kernel", + "cuda.driver.library.get_kernel_count", + "cuda.driver.library.get_managed", + "cuda.driver.library.get_module", + "cuda.driver.library.get_unified_function", + "cuda.driver.library.load_data", + "cuda.driver.library.load_from_file", + "cuda.driver.library.unload", + "cuda.driver.tensor_map.encode_im2col", + "cuda.driver.tensor_map.encode_im2col_wide", + "cuda.driver.tensor_map.replace_address", + "cuda.event.create", + "cuda.event.destroy", + "cuda.event.elapsed_time", + "cuda.event.record", + "cuda.func", + "cuda.func.get_name", + "cuda.graph.destroy", + "cuda.graph.instantiate", + "cuda.graph.launch", + "cuda.graph_exec.destroy", + "cuda.kernel", + "cuda.launch", + "cuda.launch_cfg.access_policy_window", + "cuda.launch_cfg.cluster_dim", + "cuda.launch_cfg.cluster_scheduling_policy", + "cuda.launch_cfg.cooperative", + "cuda.launch_cfg.create", + "cuda.launch_cfg.create_from_stream", + "cuda.launch_cfg.device_updatable_kernel_node", + "cuda.launch_cfg.grid_dimensions", + "cuda.launch_cfg.launch_completion_event", + "cuda.launch_cfg.mem_sync_domain", + "cuda.launch_cfg.mem_sync_domain_map", + "cuda.launch_cfg.nvlink_util_centric_scheduling", + "cuda.launch_cfg.preferred_cluster_dim", + "cuda.launch_cfg.print", + "cuda.launch_cfg.priority", + "cuda.launch_cfg.programmatic_event", + "cuda.launch_cfg.programmatic_stream_serialization_allowed", + "cuda.launch_cfg.shared_mem_carveout", + "cuda.launch_cfg.sync_policy", + "cuda.launch_ex", + "cuda.launch_kernel", + "cuda.library.get_kernel", + "cuda.library.load_data", + "cuda.library.unload", + "cuda.memory.alloc", + "cuda.memory.alloc_async", + "cuda.memory.alloc_host", + "cuda.memory.alloc_managed", + "cuda.memory.copy", + "cuda.memory.copy_async", + "cuda.memory.free", + "cuda.memory.free_async", + "cuda.memory.free_host", + "cuda.memory.set", + "cuda.memory.set_async", + "cuda.occupancy_max_active_blocks_per_multiprocessor", + "cuda.profiler.start", + "cuda.profiler.stop", + "cuda.result.assert_success", + "cuda.result.failed", + "cuda.result.print_on_error", + "cuda.result.succeeded", + "cuda.return", + "cuda.return_if_error", + "cuda.runtime.device.properties", + "cuda.runtime.device_prop.get_access_policy_max_window_size", + "cuda.runtime.device_prop.get_async_engine_count", + "cuda.runtime.device_prop.get_can_map_host_memory", + "cuda.runtime.device_prop.get_can_use_host_pointer_for_registered_mem", + "cuda.runtime.device_prop.get_cluster_launch", + "cuda.runtime.device_prop.get_compute_preemption_supported", + "cuda.runtime.device_prop.get_concurrent_kernels", + "cuda.runtime.device_prop.get_concurrent_managed_access", + "cuda.runtime.device_prop.get_cooperative_launch", + "cuda.runtime.device_prop.get_deferred_mapping_cuda_array_supported", + "cuda.runtime.device_prop.get_device_numa_config", + "cuda.runtime.device_prop.get_device_numa_id", + "cuda.runtime.device_prop.get_direct_managed_mem_access_from_host", + "cuda.runtime.device_prop.get_ecc_enabled", + "cuda.runtime.device_prop.get_global_l1cache_supported", + "cuda.runtime.device_prop.get_gpu_direct_rdma_flush_writes_options", + "cuda.runtime.device_prop.get_gpu_direct_rdma_supported", + "cuda.runtime.device_prop.get_gpu_direct_rdma_writes_ordering", + "cuda.runtime.device_prop.get_gpu_pci_device_id", + "cuda.runtime.device_prop.get_gpu_pci_subsystem_id", + "cuda.runtime.device_prop.get_host_native_atomic_supported", + "cuda.runtime.device_prop.get_host_numa_id", + "cuda.runtime.device_prop.get_host_numa_multinode_ipc_supported", + "cuda.runtime.device_prop.get_host_register_read_only_supported", + "cuda.runtime.device_prop.get_host_register_supported", + "cuda.runtime.device_prop.get_integrated", + "cuda.runtime.device_prop.get_ipc_event_supported", + "cuda.runtime.device_prop.get_is_multi_gpu_board", + "cuda.runtime.device_prop.get_l_2cache_size", + "cuda.runtime.device_prop.get_local_l1cache_supported", + "cuda.runtime.device_prop.get_luid", + "cuda.runtime.device_prop.get_luid_device_node_mask", + "cuda.runtime.device_prop.get_major", + "cuda.runtime.device_prop.get_managed_memory", + "cuda.runtime.device_prop.get_max_blocks_per_multi_processor", + "cuda.runtime.device_prop.get_max_grid_size", + "cuda.runtime.device_prop.get_max_surface_1d", + "cuda.runtime.device_prop.get_max_surface_1d_layered", + "cuda.runtime.device_prop.get_max_surface_2d", + "cuda.runtime.device_prop.get_max_surface_2d_layered", + "cuda.runtime.device_prop.get_max_surface_3d", + "cuda.runtime.device_prop.get_max_surface_cubemap", + "cuda.runtime.device_prop.get_max_surface_cubemap_layered", + "cuda.runtime.device_prop.get_max_texture_1d", + "cuda.runtime.device_prop.get_max_texture_1d_layered", + "cuda.runtime.device_prop.get_max_texture_1d_mipmap", + "cuda.runtime.device_prop.get_max_texture_2d", + "cuda.runtime.device_prop.get_max_texture_2d_gather", + "cuda.runtime.device_prop.get_max_texture_2d_layered", + "cuda.runtime.device_prop.get_max_texture_2d_linear", + "cuda.runtime.device_prop.get_max_texture_2d_mipmap", + "cuda.runtime.device_prop.get_max_texture_3d", + "cuda.runtime.device_prop.get_max_texture_3d_alt", + "cuda.runtime.device_prop.get_max_texture_cubemap", + "cuda.runtime.device_prop.get_max_texture_cubemap_layered", + "cuda.runtime.device_prop.get_max_threads_dim", + "cuda.runtime.device_prop.get_max_threads_per_block", + "cuda.runtime.device_prop.get_max_threads_per_multi_processor", + "cuda.runtime.device_prop.get_mem_pitch", + "cuda.runtime.device_prop.get_memory_bus_width", + "cuda.runtime.device_prop.get_memory_pool_supported_handle_types", + "cuda.runtime.device_prop.get_memory_pools_supported", + "cuda.runtime.device_prop.get_minor", + "cuda.runtime.device_prop.get_mps_enabled", + "cuda.runtime.device_prop.get_multi_gpu_board_group_id", + "cuda.runtime.device_prop.get_multi_processor_count", + "cuda.runtime.device_prop.get_name", + "cuda.runtime.device_prop.get_pageable_memory_access", + "cuda.runtime.device_prop.get_pageable_memory_access_uses_host_page_tables", + "cuda.runtime.device_prop.get_pci_bus_id", + "cuda.runtime.device_prop.get_pci_device_id", + "cuda.runtime.device_prop.get_pci_domain_id", + "cuda.runtime.device_prop.get_persisting_l2cache_max_size", + "cuda.runtime.device_prop.get_regs_per_block", + "cuda.runtime.device_prop.get_regs_per_multiprocessor", + "cuda.runtime.device_prop.get_reserved_shared_mem_per_block", + "cuda.runtime.device_prop.get_shared_mem_per_block", + "cuda.runtime.device_prop.get_shared_mem_per_block_optin", + "cuda.runtime.device_prop.get_shared_mem_per_multiprocessor", + "cuda.runtime.device_prop.get_sparse_cuda_array_supported", + "cuda.runtime.device_prop.get_stream_priorities_supported", + "cuda.runtime.device_prop.get_surface_alignment", + "cuda.runtime.device_prop.get_tcc_driver", + "cuda.runtime.device_prop.get_texture_alignment", + "cuda.runtime.device_prop.get_texture_pitch_alignment", + "cuda.runtime.device_prop.get_timeline_semaphore_interop_supported", + "cuda.runtime.device_prop.get_total_const_mem", + "cuda.runtime.device_prop.get_total_global_mem", + "cuda.runtime.device_prop.get_unified_addressing", + "cuda.runtime.device_prop.get_unified_function_pointers", + "cuda.runtime.device_prop.get_uuid", + "cuda.runtime.device_prop.get_warp_size", + "cuda.runtime.dim_3.get_x", + "cuda.runtime.dim_3.get_y", + "cuda.runtime.dim_3.get_z", + "cuda.runtime.event.create_with_flags", + "cuda.runtime.event.query", + "cuda.runtime.event.record_with_flags", + "cuda.runtime.event.synchronize", + "cuda.runtime.extent.get_depth", + "cuda.runtime.extent.get_height", + "cuda.runtime.extent.get_width", + "cuda.runtime.graph.add_kernel_node", + "cuda.runtime.graph.create", + "cuda.runtime.graph.kernel_node_get_attribute", + "cuda.runtime.graph.kernel_node_set_attribute", + "cuda.runtime.kernel_node_params.get_block_dim", + "cuda.runtime.kernel_node_params.get_extra", + "cuda.runtime.kernel_node_params.get_func", + "cuda.runtime.kernel_node_params.get_grid_dim", + "cuda.runtime.kernel_node_params.get_kernel_params", + "cuda.runtime.kernel_node_params.get_shared_mem_bytes", + "cuda.runtime.library.get_global", + "cuda.runtime.library.get_kernel", + "cuda.runtime.library.get_kernel_count", + "cuda.runtime.library.get_managed", + "cuda.runtime.library.get_unified_function", + "cuda.runtime.library.load_data", + "cuda.runtime.library.load_from_file", + "cuda.runtime.library.unload", + "cuda.runtime.memcpy_3d_params.get_dst_array", + "cuda.runtime.memcpy_3d_params.get_dst_pos", + "cuda.runtime.memcpy_3d_params.get_dst_ptr", + "cuda.runtime.memcpy_3d_params.get_extent", + "cuda.runtime.memcpy_3d_params.get_kind", + "cuda.runtime.memcpy_3d_params.get_src_array", + "cuda.runtime.memcpy_3d_params.get_src_pos", + "cuda.runtime.memcpy_3d_params.get_src_ptr", + "cuda.runtime.memory.cpy3d", + "cuda.runtime.memory.cpy3d_async", + "cuda.runtime.pitched_ptr.get_pitch", + "cuda.runtime.pitched_ptr.get_ptr", + "cuda.runtime.pitched_ptr.get_xsize", + "cuda.runtime.pitched_ptr.get_ysize", + "cuda.runtime.pos.get_x", + "cuda.runtime.pos.get_y", + "cuda.runtime.pos.get_z", + "cuda.runtime.uuid.get_bytes", + "cuda.stream.add_callback", + "cuda.stream.begin_capture", + "cuda.stream.create", + "cuda.stream.destroy", + "cuda.stream.end_capture", + "cuda.stream.synchronize", + "cuda.tensor_map_encode_tiled", +}; diff --git a/mlir/dialects/cute_ir/cute_ir.zig b/mlir/dialects/cute_ir/cute_ir.zig index 3e9ff08f..8420bcbf 100644 --- a/mlir/dialects/cute_ir/cute_ir.zig +++ b/mlir/dialects/cute_ir/cute_ir.zig @@ -7,6 +7,9 @@ const cute = @This(); pub const dialect_namespace = "cute"; pub const nvgpu = @import("cute_nvgpu.zig"); +/// NVIDIA's `nvvm` and `cuda` dialects, as the DSL compiler registers them. +pub const nvvm = @import("nvvm.zig"); +pub const cuda = @import("cuda.zig"); pub const AttributeKind = enum { int_tuple, coord, shape, stride, layout, tile, composed_layout, swizzle }; @@ -24,10 +27,14 @@ pub const CuteType = opaque { } }; -/// Registers both dialects in the existing MLIR registry. +/// Registers `cute`, `cute_nvgpu`, and NVIDIA's `nvvm` and `cuda` in the +/// existing MLIR registry. The `nvvm` here is the compiler's, not upstream's: +/// a registry cannot hold both. pub fn registerDialects(registry: *mlir.DialectRegistry) void { mlir.DialectHandle.fromString(dialect_namespace).insertDialect(registry); mlir.DialectHandle.fromString(nvgpu.dialect_namespace).insertDialect(registry); + mlir.DialectHandle.fromString(nvvm.dialect_handle).insertDialect(registry); + mlir.DialectHandle.fromString(cuda.dialect_handle).insertDialect(registry); } pub const AlgebraKind = AttributeKind; @@ -2847,8 +2854,17 @@ fn testContext() !*mlir.Context { } test { + @setEvalBranchQuota(100_000); std.testing.refAllDecls(@This()); std.testing.refAllDecls(nvgpu); + std.testing.refAllDecls(nvvm); + std.testing.refAllDecls(cuda); + inline for (.{ nvvm, cuda }) |namespace| { + inline for (comptime std.meta.declarations(namespace)) |decl| { + const T = @field(namespace, decl.name); + if (@TypeOf(T) == type) std.testing.refAllDecls(T); + } + } inline for (comptime std.meta.declarations(@This())) |decl| { const T = @field(@This(), decl.name); if (@TypeOf(T) == type) std.testing.refAllDecls(T); @@ -2864,6 +2880,8 @@ test "every bound operation is registered" { defer ctx.deinit(); for (cute.operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); for (nvgpu.operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + for (nvvm.operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + for (cuda.operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); try std.testing.expect(!ctx.allowUnregisteredDialects()); } @@ -2992,6 +3010,82 @@ test "generated types and attributes rebuild from their getters" { try std.testing.expect((try nvgpu.SmemDescType.get(ctx, .{})).type_().isA(nvgpu.NVGPUType) != null); } +test "nvvm and cuda types and attributes print as the compiler does" { + @setEvalBranchQuota(100_000); + const ctx = try testContext(); + defer ctx.deinit(); + // The compiler's printed forms, which these must print byte for byte. + inline for (.{ + .{ nvvm.MMAShapeAttr, "#nvvm.shape" }, + .{ nvvm.LdStMatrixShapeAttr, "#nvvm.ld_st_matrix_shape" }, + .{ nvvm.TargetAttr, "#nvvm.target" }, + .{ nvvm.TargetAttr, "#nvvm.target" }, + .{ nvvm.TargetAttr, "#nvvm.target" }, + .{ cuda.AssumeKernelAttr, "#cuda.assume_kernel_attr" }, + .{ cuda.ComputeTargetAttr, "#cuda.compute_target" }, + .{ cuda.DevMaxSharedMemoryOptinAttr, "#cuda.dev_max_shared_memory_optin" }, + .{ cuda.ExecutableAttr, "#cuda.executable" }, + .{ cuda.LaunchConfigType, "!cuda.launch_cfg" }, + .{ cuda.StreamType, "!cuda.stream" }, + .{ cuda.ResultType, "!cuda.result" }, + .{ cuda.IntType, "!cuda.int" }, + .{ cuda.RuntimeDim3Type, "!cuda.runtime.dim_3" }, + .{ cuda.RuntimeLaunchAttributeValueSyncPolicyType, "!cuda.runtime.launch_attribute_value.sync_policy" }, + .{ cuda.RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType, "!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference" }, + .{ cuda.RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType, "!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling" }, + .{ cuda.RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType, "!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed" }, + }) |example| { + const T = example[0]; + const is_type = @hasDecl(T, "type_"); + const parsed = if (is_type) try mlir.Type.parse(ctx, example[1]) else try mlir.Attribute.parse(ctx, example[1]); + const value = parsed.isA(T) orelse return error.TestUnexpectedResult; + try std.testing.expect((try rebuildEnums(T, ctx, value)).eql(value)); + var buf: [512]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(example[1], w.buffered()); + } + // The compiler prints these without the space before the dictionary; it + // parses both. + inline for (.{ + .{ cuda.DeviceAttributesAttr, "#cuda" }, + .{ cuda.FuncAttributesAttr, "#cuda" }, + }) |example| { + const parsed = try mlir.Attribute.parse(ctx, example[1]); + const value = parsed.isA(example[0]) orelse return error.TestUnexpectedResult; + try std.testing.expect((try rebuildEnums(example[0], ctx, value)).eql(value)); + } +} + +test "nvvm target rejects what the compiler rejects" { + const ctx = try testContext(); + defer ctx.deinit(); + try std.testing.expectError(error.InvalidMlir, nvvm.TargetAttr.get(ctx, .{ .O = 4 })); + try std.testing.expectError(error.InvalidMlir, nvvm.TargetAttr.get(ctx, .{ .chip = "" })); + const not_strings = try mlir.Attribute.parse(ctx, "[\"a\", 2]"); + try std.testing.expectError(error.InvalidMlir, nvvm.TargetAttr.get(ctx, .{ .link = not_strings })); +} + +/// `rebuild` for parameters that are slices of enums. +fn rebuildEnums(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T { + var args: T.InitArgs = undefined; + var buffers: [4][16]u32 = undefined; + inline for (@typeInfo(T.InitArgs).@"struct".fields, 0..) |field, i| { + const name = [_]u8{comptime std.ascii.toUpper(field.name[0])} ++ field.name[1..]; + const info = @typeInfo(field.type); + if (info == .pointer and info.pointer.size == .slice and @typeInfo(info.pointer.child) == .@"enum") { + const E = info.pointer.child; + const count = @field(T, "getNum" ++ name)(value); + const elements: []E = @ptrCast(buffers[i][0..count]); + for (elements, 0..) |*element, pos| element.* = @field(T, "get" ++ name[0 .. name.len - 1])(value, pos); + @field(args, field.name) = elements; + } else { + @field(args, field.name) = @field(T, "get" ++ name)(value); + } + } + return T.get(ctx, args); +} + fn rebuild(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T { var args: T.InitArgs = undefined; var elements: [8]*const mlir.Type = undefined; diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/BUILD.bazel b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/BUILD.bazel index 3833369f..e30f2462 100644 --- a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/BUILD.bazel +++ b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/BUILD.bazel @@ -24,3 +24,24 @@ cc_library( includes = ["../.."], deps = ["@llvm-project//mlir:CAPIIR"], ) + +cc_library( + name = "cuda", + hdrs = [ + "Cuda.h", + "CudaAttributes.h", + "CudaTypes.h", + ], + includes = ["../.."], + deps = ["@llvm-project//mlir:CAPIIR"], +) + +cc_library( + name = "nvvm", + hdrs = [ + "NVVM.h", + "NVVMAttributes.h", + ], + includes = ["../.."], + deps = ["@llvm-project//mlir:CAPIIR"], +) diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/Cuda.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/Cuda.h new file mode 100644 index 00000000..fbe099a4 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/Cuda.h @@ -0,0 +1,22 @@ +//===- Cuda.h - C API for NVIDIA's CUDA host dialect of the CuTe DSL -*- C -*-===// +// +// The dialect handle is `cuda`. + +#ifndef CUTE_IR_C_DIALECT_CUDA_H +#define CUTE_IR_C_DIALECT_CUDA_H + +#include "cute_ir-c/Dialect/CudaAttributes.h" +#include "cute_ir-c/Dialect/CudaTypes.h" +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Cuda, cuda); + +#ifdef __cplusplus +} +#endif + +#endif // CUTE_IR_C_DIALECT_CUDA_H diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CudaAttributes.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CudaAttributes.h new file mode 100644 index 00000000..b0d55d2e --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CudaAttributes.h @@ -0,0 +1,52 @@ +// C API for the cuda attributes, generated from the dialect's .td files. +#ifndef CUTE_IR_C_DIALECT_CUDA_ATTRIBUTES_H +#define CUTE_IR_C_DIALECT_CUDA_ATTRIBUTES_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles. Enums are their integer +// values (the dialect's *Enums.td). Getters require a value accepted by the +// matching IsA. + +// `#cuda.assume_kernel_attr`: Whether `cuda.launch` may assume its callee is a kernel +MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaAssumeKernel(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCudaAssumeKernelAttrGet(MlirContext context, bool value); +MLIR_CAPI_EXPORTED bool mlirCudaAssumeKernelAttrGetValue(MlirAttribute attr); + +// `#cuda.compute_target`: Representation, portability and architectures of a compiled executable +MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaComputeTarget(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCudaComputeTargetAttrGet(MlirContext context, uint32_t representation, uint32_t portability, intptr_t numArchs, const uint32_t *archs); +MLIR_CAPI_EXPORTED uint32_t mlirCudaComputeTargetAttrGetRepresentation(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t mlirCudaComputeTargetAttrGetPortability(MlirAttribute attr); +MLIR_CAPI_EXPORTED intptr_t mlirCudaComputeTargetAttrGetNumArchs(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t mlirCudaComputeTargetAttrGetArchs(MlirAttribute attr, intptr_t pos); + +// `#cuda.dev_max_shared_memory_optin`: The device's opt-in maximum of shared memory per block +MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaDevMaxSharedMemoryOptin(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCudaDevMaxSharedMemoryOptinAttrGet(MlirContext context); + +// `#cuda.device_attributes`: CUDA device attributes +MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaDeviceAttributes(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCudaDeviceAttributesAttrGet(MlirContext context, MlirAttribute values); +MLIR_CAPI_EXPORTED MlirAttribute mlirCudaDeviceAttributesAttrGetValues(MlirAttribute attr); + +// `#cuda.executable`: Format of a compiled executable +MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaExecutable(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCudaExecutableAttrGet(MlirContext context, uint32_t format); +MLIR_CAPI_EXPORTED uint32_t mlirCudaExecutableAttrGetFormat(MlirAttribute attr); + +// `#cuda.func_attributes`: CUDA function attributes of a kernel +MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaFuncAttributes(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCudaFuncAttributesAttrGet(MlirContext context, MlirAttribute values); +MLIR_CAPI_EXPORTED MlirAttribute mlirCudaFuncAttributesAttrGetValues(MlirAttribute attr); + +#ifdef __cplusplus +} +#endif + +#endif // CUTE_IR_C_DIALECT_CUDA_ATTRIBUTES_H diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CudaTypes.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CudaTypes.h new file mode 100644 index 00000000..c788e0f8 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CudaTypes.h @@ -0,0 +1,149 @@ +// C API for the cuda types, generated from the dialect's .td files. +#ifndef CUTE_IR_C_DIALECT_CUDA_TYPES_H +#define CUTE_IR_C_DIALECT_CUDA_TYPES_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles. Enums are their integer +// values (the dialect's *Enums.td). Getters require a value accepted by the +// matching IsA. + +// `!cuda.arch_set`: `!cuda.arch_set` +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaArchSet(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaArchSetTypeGet(MlirContext context); + +// `!cuda.event`: `Event` represents CUDA event +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaEvent(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaEventTypeGet(MlirContext context); + +// `!cuda.graph_device_node`: `GraphDeviceNode` represents CUDA device node handle for device-side node update +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaGraphDeviceNode(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaGraphDeviceNodeTypeGet(MlirContext context); + +// `!cuda.graph_exec`: CUDA Graph Execution +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaGraphExec(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaGraphExecTypeGet(MlirContext context); + +// `!cuda.graph_node`: CUDA Graph Node handle +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaGraphNode(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaGraphNodeTypeGet(MlirContext context); + +// `!cuda.graph`: CUDA Graph +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaGraph(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaGraphTypeGet(MlirContext context); + +// `!cuda.int`: `cuda.int` is a integer type with a bitwidth matching the underlying ABI integer bitwidth +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaInt(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaIntTypeGet(MlirContext context); + +// `!cuda.kernel`: CUDA Kernel handle +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaKernel(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaKernelTypeGet(MlirContext context); + +// `!cuda.launch_cfg`: Representation of CUDA's extensible launch configuration mirroring cuda runtime API +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaLaunchConfig(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaLaunchConfigTypeGet(MlirContext context, uint32_t maxNumAttrs); +MLIR_CAPI_EXPORTED uint32_t mlirCudaLaunchConfigTypeGetMaxNumAttrs(MlirType type); + +// `!cuda.library`: CUDA Library handle +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaLibrary(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaLibraryTypeGet(MlirContext context); + +// `!cuda.module`: CUDA Module handle +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaModule(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaModuleTypeGet(MlirContext context); + +// `!cuda.result`: CUDA Result type +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaResult(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaResultTypeGet(MlirContext context); + +// `!cuda.runtime.device_prop`: CUDA cudaDeviceProp struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeDeviceProp(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeDevicePropTypeGet(MlirContext context); + +// `!cuda.runtime.dim_3`: CUDA dim3 struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeDim3(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeDim3TypeGet(MlirContext context); + +// `!cuda.runtime.extent`: CUDA cudaExtent struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeExtent(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeExtentTypeGet(MlirContext context); + +// `!cuda.runtime.kernel_node_params`: CUDA cudaKernelNodeParams struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeKernelNodeParams(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeKernelNodeParamsTypeGet(MlirContext context); + +// `!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference`: `!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference` +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreference(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceTypeGet(MlirContext context); + +// `!cuda.runtime.launch_attribute_value.cooperative`: `!cuda.runtime.launch_attribute_value.cooperative` +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeLaunchAttributeValueCooperative(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeLaunchAttributeValueCooperativeTypeGet(MlirContext context); + +// `!cuda.runtime.launch_attribute_value.mem_sync_domain`: `!cuda.runtime.launch_attribute_value.mem_sync_domain` +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeLaunchAttributeValueMemSyncDomain(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeLaunchAttributeValueMemSyncDomainTypeGet(MlirContext context); + +// `!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling`: `!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling` +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeLaunchAttributeValueNvlinkUtilCentricScheduling(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingTypeGet(MlirContext context); + +// `!cuda.runtime.launch_attribute_value.portable_cluster_size_mode`: `!cuda.runtime.launch_attribute_value.portable_cluster_size_mode` +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeLaunchAttributeValuePortableClusterSizeMode(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeLaunchAttributeValuePortableClusterSizeModeTypeGet(MlirContext context); + +// `!cuda.runtime.launch_attribute_value.priority`: `!cuda.runtime.launch_attribute_value.priority` +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeLaunchAttributeValuePriority(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeLaunchAttributeValuePriorityTypeGet(MlirContext context); + +// `!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed`: `!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed` +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowed(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedTypeGet(MlirContext context); + +// `!cuda.runtime.launch_attribute_value.shared_mem_carveout`: `!cuda.runtime.launch_attribute_value.shared_mem_carveout` +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemCarveout(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeLaunchAttributeValueSharedMemCarveoutTypeGet(MlirContext context); + +// `!cuda.runtime.launch_attribute_value.shared_memory_mode`: `!cuda.runtime.launch_attribute_value.shared_memory_mode` +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemoryMode(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeLaunchAttributeValueSharedMemoryModeTypeGet(MlirContext context); + +// `!cuda.runtime.launch_attribute_value.sync_policy`: `!cuda.runtime.launch_attribute_value.sync_policy` +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeLaunchAttributeValueSyncPolicy(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeLaunchAttributeValueSyncPolicyTypeGet(MlirContext context); + +// `!cuda.runtime.memcpy_3d_params`: CUDA cudaMemcpy3DParms struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeMemcpy3DParams(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeMemcpy3DParamsTypeGet(MlirContext context); + +// `!cuda.runtime.pitched_ptr`: CUDA cudaPitchedPtr struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimePitchedPtr(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimePitchedPtrTypeGet(MlirContext context); + +// `!cuda.runtime.pos`: CUDA cudaPos struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimePos(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimePosTypeGet(MlirContext context); + +// `!cuda.runtime.uuid`: CUDA cudaUUID_t struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeUuid(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeUuidTypeGet(MlirContext context); + +// `!cuda.stream`: CUDA Stream +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaStream(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaStreamTypeGet(MlirContext context); + +// `!cuda.tensor_map`: CUDA Tensor Map type +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaTensorMap(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaTensorMapTypeGet(MlirContext context); + +#ifdef __cplusplus +} +#endif + +#endif // CUTE_IR_C_DIALECT_CUDA_TYPES_H diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/NVVM.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/NVVM.h new file mode 100644 index 00000000..94d00310 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/NVVM.h @@ -0,0 +1,21 @@ +//===- NVVM.h - C API for NVIDIA's NVVM dialect of the CuTe DSL -*- C -*-===// +// +// The dialect handle is `cute_nvvm`, not the upstream dialect's `nvvm`: both register a dialect named `nvvm`. + +#ifndef CUTE_IR_C_DIALECT_NVVM_H +#define CUTE_IR_C_DIALECT_NVVM_H + +#include "cute_ir-c/Dialect/NVVMAttributes.h" +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(CuteNVVM, cute_nvvm); + +#ifdef __cplusplus +} +#endif + +#endif // CUTE_IR_C_DIALECT_NVVM_H diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/NVVMAttributes.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/NVVMAttributes.h new file mode 100644 index 00000000..e6eac4d4 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/NVVMAttributes.h @@ -0,0 +1,509 @@ +// C API for the nvvm attributes, generated from the dialect's .td files. +#ifndef CUTE_IR_C_DIALECT_NVVM_ATTRIBUTES_H +#define CUTE_IR_C_DIALECT_NVVM_ATTRIBUTES_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles. Enums are their integer +// values (the dialect's *Enums.td). Getters require a value accepted by the +// matching IsA. + +// `#nvvm.atomic_op`: operations supported by atom instruction +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMAtomicOpKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMAtomicOpKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMAtomicOpKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.reduction`: NVVM barrier reduction operation +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMBarrierReduction(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMBarrierReductionAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMBarrierReductionAttrGetValue(MlirAttribute attr); + +// `#nvvm.barrier_redux_kind`: NVVM barrier redux kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMBarrierReduxKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMBarrierReduxKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMBarrierReduxKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.block_scale_format`: MMA Block Scale Format +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMBlockScaleFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMBlockScaleFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMBlockScaleFormatAttrGetValue(MlirAttribute attr); + +// `#nvvm.cta_group`: NVVM CTA group kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMCTAGroupKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMCTAGroupKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMCTAGroupKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.packfloat_type`: NVVM CVT Pack Float kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMCVTPackFloatKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMCVTPackFloatKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMCVTPackFloatKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.cache_eviction_priority`: NVVM Cache Eviction Priority +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMCacheEvictionPriority(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMCacheEvictionPriorityAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMCacheEvictionPriorityAttrGetValue(MlirAttribute attr); + +// `#nvvm.cluster_launch_control_query_type`: NVVM ClusterLaunchControlQueryType +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMClusterLaunchControlQueryType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMClusterLaunchControlQueryTypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMClusterLaunchControlQueryTypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.op`: Comparison operator encoding +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMCompareOpKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMCompareOpKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMCompareOpKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.convert_fp4_type`: NVVM ConvertFP4Type kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMConvertFP4Type(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMConvertFP4TypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMConvertFP4TypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.convert_fp8_type`: NVVM ConvertFP8Type kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMConvertFP8Type(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMConvertFP8TypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMConvertFP8TypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.convert_scale_kind`: NVVM ConvertScale kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMConvertScaleKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMConvertScaleKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMConvertScaleKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.dot_accumulate_type`: NVVM DotAccumulateType +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMDotAccumulateType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMDotAccumulateTypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMDotAccumulateTypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.evict_kind`: NVVM L2 Prefetch Size +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMEvictKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMEvictKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMEvictKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.fp_rnd_mode`: NVVM FPRoundingMode kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMFPRoundingMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMFPRoundingModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMFPRoundingModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.grid_dep_action`: Action kind for grid dependency control +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMGridDepActionKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMGridDepActionKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMGridDepActionKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.int_rnd_mode`: NVVM IntegerRoundingMode kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMIntegerRoundingMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMIntegerRoundingModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMIntegerRoundingModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.l2_prefetch`: NVVM L2 Prefetch Size +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVML2PrefetchSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVML2PrefetchSizeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVML2PrefetchSizeAttrGetValue(MlirAttribute attr); + +// `#nvvm.ld_st_matrix_elt_type`: Element type for ldmatrix and stmatrix +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMLdStMatrixEltType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMLdStMatrixEltTypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMLdStMatrixEltTypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.load_cache_modifier_ext`: NVVM load cache modifier kind(Ext) +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMLoadCacheModifierExtKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMLoadCacheModifierExtKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMLoadCacheModifierExtKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.load_cache_modifier`: NVVM load cache modifier kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMLoadCacheModifierKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMLoadCacheModifierKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMLoadCacheModifierKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.load_shape`: shape attribute for ldmatrix +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMLoadShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMLoadShapeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMLoadShapeAttrGetValue(MlirAttribute attr); + +// `#nvvm.load_src_format`: source format for ldmatrix +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMLoadSrcFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMLoadSrcFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMLoadSrcFormatAttrGetValue(MlirAttribute attr); + +// `#nvvm.mbarrier_layout`: NVVM MBarrier Layout +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMBarrierLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMBarrierLayoutAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMBarrierLayoutAttrGetValue(MlirAttribute attr); + +// `#nvvm.mbarrier_phase`: NVVM mbarrier phase type +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMBarrierPhase(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMBarrierPhaseAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMBarrierPhaseAttrGetValue(MlirAttribute attr); + +// `#nvvm.mbar_scope`: NVVM MBarrier scope kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMBarrierScopeKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMBarrierScopeKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMBarrierScopeKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mbar_space`: NVVM MBarrier space kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMBarrierSpaceKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMBarrierSpaceKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMBarrierSpaceKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mbar_txn_kind`: NVVM MBarrier Transaction kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMBarrierTxnKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMBarrierTxnKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMBarrierTxnKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mbar_wait`: NVVM MBarrier wait kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMBarrierWaitKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMBarrierWaitKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMBarrierWaitKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_b1op`: MMA binary operations +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAB1Op(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMAB1OpAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMAB1OpAttrGetValue(MlirAttribute attr); + +// `#nvvm.block_scale_kind`: Block Scale Kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMABlockScaleKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMABlockScaleKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMABlockScaleKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_cta_count`: MMA CTA count +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMACtaCount(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMACtaCountAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMACtaCountAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_frag`: NVVM MMA frag type +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAFrag(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMAFragAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMAFragAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_int_overflow`: MMA overflow options +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAIntOverflow(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMAIntOverflowAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMAIntOverflowAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_kind`: MMA operation kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMAKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMAKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_layout`: NVVM MMA layout +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMALayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMALayoutAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMALayoutAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_type`: NVVM MMA types +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMATypes(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMATypesAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMATypesAttrGetValue(MlirAttribute attr); + +// `#nvvm.match_sync_kind`: NVVM match sync kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMatchSyncKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMatchSyncKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMatchSyncKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mem_order`: NVVM Memory Ordering kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMemOrderKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMemOrderKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMemOrderKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mem_scope`: NVVM Memory Scope kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMemScopeKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMemScopeKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMemScopeKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mul_mode`: multiply mode attribute +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMulMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMulModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMulModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.memory_space`: NVVM Memory Space +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMNVVMMemorySpace(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMNVVMMemorySpaceAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMNVVMMemorySpaceAttrGetValue(MlirAttribute attr); + +// `#nvvm.permute_mode`: NVVM permute mode +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMPermuteMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMPermuteModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMPermuteModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.prefetch_cache_level`: NVVM Prefetch Cache Level +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMPrefetchCacheLevel(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMPrefetchCacheLevelAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMPrefetchCacheLevelAttrGetValue(MlirAttribute attr); + +// `#nvvm.proxy_kind`: Proxy kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMProxyKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMProxyKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMProxyKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.reduction_kind`: NVVM Reduction Kind attribute +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMReductionKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMReductionKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMReductionKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.red_op`: Ops supported by red instruction +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMReductionOp(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMReductionOpAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMReductionOpAttrGetValue(MlirAttribute attr); + +// `#nvvm.red_type`: types supported by red instruction +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMReductionType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMReductionTypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMReductionTypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spcomp_elem_size`: Sparse tensor compression element size +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSPCompressElemSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSPCompressElemSizeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSPCompressElemSizeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spcomp_factor`: Sparse tensor compression factor type +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSPCompressFactorType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSPCompressFactorTypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSPCompressFactorTypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spcomp_index_size`: Sparse tensor compression index size +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSPCompressIndexSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSPCompressIndexSizeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSPCompressIndexSizeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spcomp_op_kind`: spcompress operation kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSPCompressOpKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSPCompressOpKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSPCompressOpKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.spcomp_rep_factor`: Sparse tensor compression repetition factor +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSPCompressRepFactor(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSPCompressRepFactorAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSPCompressRepFactorAttrGetValue(MlirAttribute attr); + +// `#nvvm.spdecomp_elem_size`: Sparse tensor decompression element size +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSPDecompressElemSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSPDecompressElemSizeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSPDecompressElemSizeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spdecomp_factor`: Sparse tensor decompression factor type +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSPDecompressFactorType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSPDecompressFactorTypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSPDecompressFactorTypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spdecomp_index_size`: Sparse tensor decompression index size +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSPDecompressIndexSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSPDecompressIndexSizeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSPDecompressIndexSizeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spdecomp_rep_factor`: Sparse tensor decompression repetition factor +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSPDecompressRepFactor(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSPDecompressRepFactorAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSPDecompressRepFactorAttrGetValue(MlirAttribute attr); + +// `#nvvm.sat_mode`: NVVM SaturationMode kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSaturationMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSaturationModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSaturationModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.sat`: NVVM SaturationMode kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSaturationModeKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSaturationModeKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSaturationModeKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.scale_vec_size`: MMA Scale Vector Sizes +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMScaleVecSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMScaleVecSizeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMScaleVecSizeAttrGetValue(MlirAttribute attr); + +// `#nvvm.action`: NVVM set max register action +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSetMaxRegisterAction(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSetMaxRegisterActionAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSetMaxRegisterActionAttrGetValue(MlirAttribute attr); + +// `#nvvm.shared_space`: Shared memory space +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSharedSpace(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSharedSpaceAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSharedSpaceAttrGetValue(MlirAttribute attr); + +// `#nvvm.shfl_kind`: NVVM shuffle kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMShflKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMShflKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMShflKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.sparsity_format`: MMA Sparsity Format +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSparsityFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSparsityFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSparsityFormatAttrGetValue(MlirAttribute attr); + +// `#nvvm.state_space`: NVVM State Space +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMStateSpace(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMStateSpaceAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMStateSpaceAttrGetValue(MlirAttribute attr); + +// `#nvvm.store_cache_modifier`: NVVM store cache modifier kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMStoreCacheModifierKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMStoreCacheModifierKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMStoreCacheModifierKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.store_shape`: shape attribute for stmatrix +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMStoreShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMStoreShapeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMStoreShapeAttrGetValue(MlirAttribute attr); + +// `#nvvm.TCBarParam`: Cluster MMA Barrier Parameter Type +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTCBarParam(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTCBarParamAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTCBarParamAttrGetValue(MlirAttribute attr); + +// `#nvvm.tma_load_mode`: NVVM TMA Load Mode +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTMALoadMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTMALoadModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTMALoadModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.tma_redux_kind`: NVVM TMA redux kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTMAReduxKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTMAReduxKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTMAReduxKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.tma_store_mode`: NVVM TMA Store Mode +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTMAStoreMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTMAStoreModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTMAStoreModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_cp_multicast`: tcgen05 cp multicast +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTcgen05CpMulticast(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTcgen05CpMulticastAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTcgen05CpMulticastAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_cp_shape`: tcgen05 cp shapes +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTcgen05CpShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTcgen05CpShapeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTcgen05CpShapeAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_cp_src_fmt`: tcgen05 cp source format +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTcgen05CpSrcFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTcgen05CpSrcFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTcgen05CpSrcFormatAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_fence`: NVVM Tcgen05 fence kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTcgen05FenceKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTcgen05FenceKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTcgen05FenceKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_ldst_shape`: allowed 32-bit signless integer cases: 0, 1, 2, 3, 4 +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTcgen05LdStShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTcgen05LdStShapeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTcgen05LdStShapeAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_mma_block_scale`: tcgen05.mma block scale attribute +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTcgen05MMABlockScale(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTcgen05MMABlockScaleAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTcgen05MMABlockScaleAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_mma_collectorb`: tcgen05 MMA Collector Buffer B Attribute +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTcgen05MMACollectorBBuffer(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTcgen05MMACollectorBBufferAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTcgen05MMACollectorBBufferAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_mma_collectorop`: tcgen05.mma Collector Buffer Operation +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTcgen05MMACollectorOp(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTcgen05MMACollectorOpAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTcgen05MMACollectorOpAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_mma_kind`: tcgen05 MMA Supported Types +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTcgen05MMAKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTcgen05MMAKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTcgen05MMAKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_wait`: NVVM Tcgen05 wait kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTcgen05WaitKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTcgen05WaitKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTcgen05WaitKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.tensormap_elemtype`: NVVM Tensormap Elemtype +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTensormapElemtype(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTensormapElemtypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTensormapElemtypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.tensormap_field`: NVVM Tensormap Field Kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTensormapField(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTensormapFieldAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTensormapFieldAttrGetValue(MlirAttribute attr); + +// `#nvvm.tensormap_fill_mode`: NVVM Tensormap Fill Mode +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTensormapFillMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTensormapFillModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTensormapFillModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.tensormap_interleave_layout`: NVVM Tensormap Interleave Layout +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTensormapInterleaveLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTensormapInterleaveLayoutAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTensormapInterleaveLayoutAttrGetValue(MlirAttribute attr); + +// `#nvvm.tensormap_swizzle_atomicity`: NVVM Tensormap Swizzle Atomicity +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTensormapSwizzleAtomicity(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTensormapSwizzleAtomicityAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTensormapSwizzleAtomicityAttrGetValue(MlirAttribute attr); + +// `#nvvm.tensormap_swizzle_mode`: NVVM Tensormap Swizzle Mode +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTensormapSwizzleMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTensormapSwizzleModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTensormapSwizzleModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.TmemLayout`: Tensor Memory Layout Enumerated Type +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTmemLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTmemLayoutAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTmemLayoutAttrGetValue(MlirAttribute attr); + +// `#nvvm.validate_pattern`: NVVM validate data pattern +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMValidatePattern(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMValidatePatternAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMValidatePatternAttrGetValue(MlirAttribute attr); + +// `#nvvm.vote_sync_kind`: NVVM vote sync kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMVoteSyncKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMVoteSyncKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMVoteSyncKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.wgmma_scale_in`: WGMMA overflow options +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMWGMMAScaleIn(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMWGMMAScaleInAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMWGMMAScaleInAttrGetValue(MlirAttribute attr); + +// `#nvvm.wgmma_scale_out`: WGMMA input predicate +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMWGMMAScaleOut(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMWGMMAScaleOutAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMWGMMAScaleOutAttrGetValue(MlirAttribute attr); + +// `#nvvm.wgmma_type`: NVVM WGMMA types +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMWGMMATypes(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMWGMMATypesAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMWGMMATypesAttrGetValue(MlirAttribute attr); + +// `#nvvm.ld_st_matrix_shape`: Matrix shape of ldmatrix, stmatrix and movmatrix +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMLdStMatrixShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMLdStMatrixShapeAttrGet(MlirContext context, int m, int n); +MLIR_CAPI_EXPORTED int mlirCuteNVVMLdStMatrixShapeAttrGetM(MlirAttribute attr); +MLIR_CAPI_EXPORTED int mlirCuteNVVMLdStMatrixShapeAttrGetN(MlirAttribute attr); + +// `#nvvm.shape`: Shape of an MMA operation +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMAShapeAttrGet(MlirContext context, int m, int n, int k); +MLIR_CAPI_EXPORTED int mlirCuteNVVMMMAShapeAttrGetM(MlirAttribute attr); +MLIR_CAPI_EXPORTED int mlirCuteNVVMMMAShapeAttrGetN(MlirAttribute attr); +MLIR_CAPI_EXPORTED int mlirCuteNVVMMMAShapeAttrGetK(MlirAttribute attr); + +// `#nvvm.target`: GPU target of an NVVM module +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTarget(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTargetAttrGet(MlirContext context, int O, MlirStringRef triple, MlirStringRef chip, MlirStringRef features, MlirAttribute flags, MlirAttribute link, bool verifyTarget); +MLIR_CAPI_EXPORTED int mlirCuteNVVMTargetAttrGetO(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirCuteNVVMTargetAttrGetTriple(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirCuteNVVMTargetAttrGetChip(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirCuteNVVMTargetAttrGetFeatures(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTargetAttrGetFlags(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTargetAttrGetLink(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirCuteNVVMTargetAttrGetVerifyTarget(MlirAttribute attr); + +#ifdef __cplusplus +} +#endif + +#endif // CUTE_IR_C_DIALECT_NVVM_ATTRIBUTES_H diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/BUILD.bazel b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/BUILD.bazel new file mode 100644 index 00000000..615d9b29 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/BUILD.bazel @@ -0,0 +1,142 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:defs.bzl", "cc_library") + +package(default_visibility = ["//visibility:public"]) + +td_library( + name = "td_files", + srcs = glob(["*.td"]), + includes = ["/mlir/dialects/cute_ir/include"], + deps = [ + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "dialect_inc_gen", + includes = ["../../../.."], + tbl_outs = [ + ( + [ + "-gen-dialect-decls", + "-dialect=cuda", + ], + "CudaDialect.h.inc", + ), + ( + [ + "-gen-dialect-defs", + "-dialect=cuda", + ], + "CudaDialect.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CudaDialect.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "enums_inc_gen", + includes = ["../../../.."], + tbl_outs = [ + ( + ["-gen-enum-decls"], + "CudaEnums.h.inc", + ), + ( + ["-gen-enum-defs"], + "CudaEnums.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CudaEnums.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "attrs_inc_gen", + includes = ["../../../.."], + tbl_outs = [ + ( + [ + "-gen-attrdef-decls", + "-attrdefs-dialect=cuda", + ], + "CudaAttrs.h.inc", + ), + ( + [ + "-gen-attrdef-defs", + "-attrdefs-dialect=cuda", + ], + "CudaAttrs.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CudaAttrs.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "types_inc_gen", + includes = ["../../../.."], + tbl_outs = [ + ( + [ + "-gen-typedef-decls", + "-typedefs-dialect=cuda", + ], + "CudaTypes.h.inc", + ), + ( + [ + "-gen-typedef-defs", + "-typedefs-dialect=cuda", + ], + "CudaTypes.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CudaTypes.td", + deps = [":td_files"], +) + +# Every cuda operation of the compiler's registry. +gentbl_cc_library( + name = "ops_inc_gen", + includes = ["../../../.."], + tbl_outs = [ + ( + ["-gen-op-decls"], + "CudaOps.h.inc", + ), + ( + ["-gen-op-defs"], + "CudaOps.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CudaOps.td", + deps = [":td_files"], +) + +cc_library( + name = "headers", + hdrs = ["CudaDialect.h"], + includes = ["../../../.."], + deps = [ + ":attrs_inc_gen", + ":dialect_inc_gen", + ":enums_inc_gen", + ":ops_inc_gen", + ":types_inc_gen", + "@llvm-project//mlir:BytecodeOpInterface", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:SideEffectInterfaces", + ], +) diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaAttrs.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaAttrs.td new file mode 100644 index 00000000..abfbb5ee --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaAttrs.td @@ -0,0 +1,68 @@ +//===- CudaAttrs.td - CUDA host attributes of the CuTe DSL --*- tablegen -*-===// +// +// Generated for nvidia-cutlass-dsl 4.8.0 from the DSL compiler +// (`_cutlass_ir.cu13` parser messages, its mnemonics, and probing its +// attribute parser and printer); do not edit by hand. +// +// Every cuda attribute the compiler's parser knows. `device_attributes` and +// `func_attributes` print as `#cuda`: the compiler +// parses any dictionary there and prints back only the entries it keeps, +// whose keys were not recovered; these keep the dictionary as written, and +// print a space before it, which the compiler parses too. +// +//===----------------------------------------------------------------------===// + +#ifndef CUTE_IR_DIALECT_CUDA_IR_CUDA_ATTRS_TD +#define CUTE_IR_DIALECT_CUDA_IR_CUDA_ATTRS_TD + +include "cute_ir/Dialect/Cuda/IR/CudaDialect.td" +include "cute_ir/Dialect/Cuda/IR/CudaEnums.td" + +defvar Cuda_Ns = "::mlir::cutlass_compiler::cuda"; + +def Cuda_AssumeKernelAttr : Cuda_Attr<"AssumeKernel", "assume_kernel_attr"> { + let summary = "Whether `cuda.launch` may assume its callee is a kernel"; + let description = "`#cuda.assume_kernel_attr`"; + let parameters = (ins "bool":$value); + let assemblyFormat = "`<` $value `>`"; +} + +def Cuda_ComputeTargetAttr : Cuda_Attr<"ComputeTarget", "compute_target"> { + let summary = "Representation, portability and architectures of a compiled executable"; + let description = "`#cuda.compute_target`"; + let parameters = (ins + AttrParameter:$representation, + AttrParameter:$portability, + ArrayRefParameter:$archs + ); + let assemblyFormat = "`<` $representation `,` $portability `,` `[` $archs `]` `>`"; +} + +def Cuda_DevMaxSharedMemoryOptinAttr + : Cuda_Attr<"DevMaxSharedMemoryOptin", "dev_max_shared_memory_optin"> { + let summary = "The device's opt-in maximum of shared memory per block"; + let description = "`#cuda.dev_max_shared_memory_optin`"; +} + +def Cuda_DeviceAttributesAttr : Cuda_Attr<"DeviceAttributes", "device_attributes"> { + let summary = "CUDA device attributes"; + let description = "`#cuda`"; + let parameters = (ins "::mlir::DictionaryAttr":$values); + let assemblyFormat = "$values"; +} + +def Cuda_ExecutableAttr : Cuda_Attr<"Executable", "executable"> { + let summary = "Format of a compiled executable"; + let description = "`#cuda.executable`"; + let parameters = (ins AttrParameter:$format); + let assemblyFormat = "`<` $format `>`"; +} + +def Cuda_FuncAttributesAttr : Cuda_Attr<"FuncAttributes", "func_attributes"> { + let summary = "CUDA function attributes of a kernel"; + let description = "`#cuda`"; + let parameters = (ins "::mlir::DictionaryAttr":$values); + let assemblyFormat = "$values"; +} + +#endif // CUTE_IR_DIALECT_CUDA_IR_CUDA_ATTRS_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaDialect.h b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaDialect.h new file mode 100644 index 00000000..fac57ef2 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaDialect.h @@ -0,0 +1,30 @@ +//===- CudaDialect.h - NVIDIA's cuda dialect of the CuTe DSL ------*- C++ -*-===// + +#ifndef CUTE_IR_DIALECT_CUDA_IR_CUDA_DIALECT_H +#define CUTE_IR_DIALECT_CUDA_IR_CUDA_DIALECT_H + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/Dialect/LLVMIR/LLVMAttrs.h" +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/Dialect/LLVMIR/LLVMTypes.h" +#include "mlir/IR/BuiltinAttributes.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/DialectImplementation.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/InferTypeOpInterface.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "cute_ir/Dialect/Cuda/IR/CudaDialect.h.inc" + +#include "cute_ir/Dialect/Cuda/IR/CudaEnums.h.inc" + +#define GET_ATTRDEF_CLASSES +#include "cute_ir/Dialect/Cuda/IR/CudaAttrs.h.inc" + +#define GET_TYPEDEF_CLASSES +#include "cute_ir/Dialect/Cuda/IR/CudaTypes.h.inc" + +#define GET_OP_CLASSES +#include "cute_ir/Dialect/Cuda/IR/CudaOps.h.inc" + +#endif // CUTE_IR_DIALECT_CUDA_IR_CUDA_DIALECT_H diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaDialect.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaDialect.td new file mode 100644 index 00000000..d8c9d551 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaDialect.td @@ -0,0 +1,38 @@ +//===- CudaDialect.td - NVIDIA's CUDA host dialect of the CuTe DSL -*- tablegen -*-===// + +#ifndef CUTE_IR_DIALECT_CUDA_IR_CUDA_DIALECT_TD +#define CUTE_IR_DIALECT_CUDA_IR_CUDA_DIALECT_TD + +include "mlir/IR/AttrTypeBase.td" +include "mlir/IR/DialectBase.td" +include "mlir/IR/OpBase.td" + +def Cuda_Dialect : Dialect { + let name = "cuda"; + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let summary = "NVIDIA's CUDA host-program dialect used by CuTe-DSL"; + let description = [{ + This dialect mirrors the operation, type, attribute and enum registry of + the proprietary `cuda` dialect embedded in the nvidia-cutlass-dsl 4.8.0 + compiler: kernels, launch configurations and the CUDA runtime and driver + calls a host function makes. + }]; + let dependentDialects = ["::mlir::LLVM::LLVMDialect"]; + let useDefaultTypePrinterParser = 1; + let useDefaultAttributePrinterParser = 1; +} + +class Cuda_Op traits = []> + : Op; + +class Cuda_Type + : TypeDef { + let mnemonic = typeMnemonic; +} + +class Cuda_Attr + : AttrDef { + let mnemonic = attrMnemonic; +} + +#endif // CUTE_IR_DIALECT_CUDA_IR_CUDA_DIALECT_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaEnums.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaEnums.td new file mode 100644 index 00000000..cc415457 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaEnums.td @@ -0,0 +1,336 @@ +// The cuda dialect's enums, generated from the nvidia-cutlass-dsl 4.8.0 enum +// bindings (`_cuda_enum_gen.py`); do not edit. +// +// Case values and spellings are the compiler's. An enum the bindings build +// as a dialect attribute also has its EnumAttr, printed `#` +// or, where the compiler prints it so, `#nvvm`. + +#ifndef CUTE_IR_GENERATED_CUDA_ENUMS_TD +#define CUTE_IR_GENERATED_CUDA_ENUMS_TD + +include "cute_ir/Dialect/Cuda/IR/CudaDialect.td" +include "mlir/IR/EnumAttr.td" + +def Cuda_ArchPortabilityEnum : I32EnumAttr<"ArchPortability", "Enumerated type describing portability", [ + I32EnumAttrCase<"portable", 0, "portable">, + I32EnumAttrCase<"conditional", 1, "conditional">, + I32EnumAttrCase<"family", 2, "family"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CUDeviceAttrEnum : I32EnumAttr<"CUDeviceAttr", "Enumerated type describing CUDA device attributes can be set through Cuda Dialect", [ + I32EnumAttrCase<"maxThreadsPerBlock", 1, "maxThreadsPerBlock">, + I32EnumAttrCase<"maxBlockDimX", 2, "maxBlockDimX">, + I32EnumAttrCase<"maxBlockDimY", 3, "maxBlockDimY">, + I32EnumAttrCase<"maxBlockDimZ", 4, "maxBlockDimZ">, + I32EnumAttrCase<"maxGridDimX", 5, "maxGridDimX">, + I32EnumAttrCase<"maxGridDimY", 6, "maxGridDimY">, + I32EnumAttrCase<"maxGridDimZ", 7, "maxGridDimZ">, + I32EnumAttrCase<"maxSharedMemoryPerBlock", 8, "maxSharedMemoryPerBlock">, + I32EnumAttrCase<"totalConstantMemory", 9, "totalConstantMemory">, + I32EnumAttrCase<"warpSize", 10, "warpSize">, + I32EnumAttrCase<"maxPitch", 11, "maxPitch">, + I32EnumAttrCase<"maxRegistersPerBlock", 12, "maxRegistersPerBlock">, + I32EnumAttrCase<"clockRate", 13, "clockRate">, + I32EnumAttrCase<"textureAlignment", 14, "textureAlignment">, + I32EnumAttrCase<"gpuOverlap", 15, "gpuOverlap">, + I32EnumAttrCase<"multiProcessorCount", 16, "multiProcessorCount">, + I32EnumAttrCase<"kernelExecTimeout", 17, "kernelExecTimeout">, + I32EnumAttrCase<"integrated", 18, "integrated">, + I32EnumAttrCase<"canMapHostMemory", 19, "canMapHostMemory">, + I32EnumAttrCase<"computeMode", 20, "computeMode">, + I32EnumAttrCase<"maxTexture1DWidth", 21, "maxTexture1DWidth">, + I32EnumAttrCase<"maxTexture2DWidth", 22, "maxTexture2DWidth">, + I32EnumAttrCase<"maxTexture2DHeight", 23, "maxTexture2DHeight">, + I32EnumAttrCase<"maxTexture3DWidth", 24, "maxTexture3DWidth">, + I32EnumAttrCase<"maxTexture3DHeight", 25, "maxTexture3DHeight">, + I32EnumAttrCase<"maxTexture3DDepth", 26, "maxTexture3DDepth">, + I32EnumAttrCase<"maxTexture2DLayeredWidth", 27, "maxTexture2DLayeredWidth">, + I32EnumAttrCase<"maxTexture2DLayeredHeight", 28, "maxTexture2DLayeredHeight">, + I32EnumAttrCase<"maxTexture2DLayeredLayers", 29, "maxTexture2DLayeredLayers">, + I32EnumAttrCase<"surfaceAlignment", 30, "surfaceAlignment">, + I32EnumAttrCase<"concurrentKernels", 31, "concurrentKernels">, + I32EnumAttrCase<"eccEnabled", 32, "eccEnabled">, + I32EnumAttrCase<"pciBusId", 33, "pciBusId">, + I32EnumAttrCase<"pciDeviceId", 34, "pciDeviceId">, + I32EnumAttrCase<"tccDriver", 35, "tccDriver">, + I32EnumAttrCase<"memoryClockRate", 36, "memoryClockRate">, + I32EnumAttrCase<"globalMemoryBusWidth", 37, "globalMemoryBusWidth">, + I32EnumAttrCase<"l2CacheSize", 38, "l2CacheSize">, + I32EnumAttrCase<"maxThreadsPerMultiProcessor", 39, "maxThreadsPerMultiProcessor">, + I32EnumAttrCase<"asyncEngineCount", 40, "asyncEngineCount">, + I32EnumAttrCase<"unifiedAddressing", 41, "unifiedAddressing">, + I32EnumAttrCase<"maxTexture1DLayeredWidth", 42, "maxTexture1DLayeredWidth">, + I32EnumAttrCase<"maxTexture1DLayeredLayers", 43, "maxTexture1DLayeredLayers">, + I32EnumAttrCase<"maxTexture2DGatherWidth", 45, "maxTexture2DGatherWidth">, + I32EnumAttrCase<"maxTexture2DGatherHeight", 46, "maxTexture2DGatherHeight">, + I32EnumAttrCase<"maxTexture3DWidthAlt", 47, "maxTexture3DWidthAlt">, + I32EnumAttrCase<"maxTexture3DHeightAlt", 48, "maxTexture3DHeightAlt">, + I32EnumAttrCase<"maxTexture3DDepthAlt", 49, "maxTexture3DDepthAlt">, + I32EnumAttrCase<"pciDomainId", 50, "pciDomainId">, + I32EnumAttrCase<"texturePitchAlignment", 51, "texturePitchAlignment">, + I32EnumAttrCase<"maxTextureCubemapWidth", 52, "maxTextureCubemapWidth">, + I32EnumAttrCase<"maxTextureCubemapLayeredWidth", 53, "maxTextureCubemapLayeredWidth">, + I32EnumAttrCase<"maxTextureCubemapLayeredLayers", 54, "maxTextureCubemapLayeredLayers">, + I32EnumAttrCase<"maxSurface1DWidth", 55, "maxSurface1DWidth">, + I32EnumAttrCase<"maxSurface2DWidth", 56, "maxSurface2DWidth">, + I32EnumAttrCase<"maxSurface2DHeight", 57, "maxSurface2DHeight">, + I32EnumAttrCase<"maxSurface3DWidth", 58, "maxSurface3DWidth">, + I32EnumAttrCase<"maxSurface3DHeight", 59, "maxSurface3DHeight">, + I32EnumAttrCase<"maxSurface3DDepth", 60, "maxSurface3DDepth">, + I32EnumAttrCase<"maxSurface1DLayeredWidth", 61, "maxSurface1DLayeredWidth">, + I32EnumAttrCase<"maxSurface1DLayeredLayers", 62, "maxSurface1DLayeredLayers">, + I32EnumAttrCase<"maxSurface2DLayeredWidth", 63, "maxSurface2DLayeredWidth">, + I32EnumAttrCase<"maxSurface2DLayeredHeight", 64, "maxSurface2DLayeredHeight">, + I32EnumAttrCase<"maxSurface2DLayeredLayers", 65, "maxSurface2DLayeredLayers">, + I32EnumAttrCase<"maxSurfaceCubemapWidth", 66, "maxSurfaceCubemapWidth">, + I32EnumAttrCase<"maxSurfaceCubemapLayeredWidth", 67, "maxSurfaceCubemapLayeredWidth">, + I32EnumAttrCase<"maxSurfaceCubemapLayeredLayers", 68, "maxSurfaceCubemapLayeredLayers">, + I32EnumAttrCase<"maxTexture1DLinearWidth", 69, "maxTexture1DLinearWidth">, + I32EnumAttrCase<"maxTexture2DLinearWidth", 70, "maxTexture2DLinearWidth">, + I32EnumAttrCase<"maxTexture2DLinearHeight", 71, "maxTexture2DLinearHeight">, + I32EnumAttrCase<"maxTexture2DLinearPitch", 72, "maxTexture2DLinearPitch">, + I32EnumAttrCase<"maxTexture2DMipmappedWidth", 73, "maxTexture2DMipmappedWidth">, + I32EnumAttrCase<"maxTexture2DMipmappedHeight", 74, "maxTexture2DMipmappedHeight">, + I32EnumAttrCase<"computeCapabilityMajor", 75, "computeCapabilityMajor">, + I32EnumAttrCase<"computeCapabilityMinor", 76, "computeCapabilityMinor">, + I32EnumAttrCase<"maxTexture1DMipmappedWidth", 77, "maxTexture1DMipmappedWidth">, + I32EnumAttrCase<"streamPrioritiesSupported", 78, "streamPrioritiesSupported">, + I32EnumAttrCase<"globalL1CacheSupported", 79, "globalL1CacheSupported">, + I32EnumAttrCase<"localL1CacheSupported", 80, "localL1CacheSupported">, + I32EnumAttrCase<"maxSharedMemoryPerMultiprocessor", 81, "maxSharedMemoryPerMultiprocessor">, + I32EnumAttrCase<"maxRegistersPerMultiprocessor", 82, "maxRegistersPerMultiprocessor">, + I32EnumAttrCase<"managedMemory", 83, "managedMemory">, + I32EnumAttrCase<"isMultiGpuBoard", 84, "isMultiGpuBoard">, + I32EnumAttrCase<"multiGpuBoardGroupID", 85, "multiGpuBoardGroupID">, + I32EnumAttrCase<"hostNativeAtomicSupported", 86, "hostNativeAtomicSupported">, + I32EnumAttrCase<"singleToDoublePrecisionPerfRatio", 87, "singleToDoublePrecisionPerfRatio">, + I32EnumAttrCase<"pageableMemoryAccess", 88, "pageableMemoryAccess">, + I32EnumAttrCase<"concurrentManagedAccess", 89, "concurrentManagedAccess">, + I32EnumAttrCase<"computePreemptionSupported", 90, "computePreemptionSupported">, + I32EnumAttrCase<"canUseHostPointerForRegisteredMem", 91, "canUseHostPointerForRegisteredMem">, + I32EnumAttrCase<"reserved92", 92, "reserved92">, + I32EnumAttrCase<"reserved93", 93, "reserved93">, + I32EnumAttrCase<"reserved94", 94, "reserved94">, + I32EnumAttrCase<"cooperativeLaunch", 95, "cooperativeLaunch">, + I32EnumAttrCase<"reserved96", 96, "reserved96">, + I32EnumAttrCase<"maxSharedMemoryPerBlockOptin", 97, "maxSharedMemoryPerBlockOptin">, + I32EnumAttrCase<"canFlushRemoteWrites", 98, "canFlushRemoteWrites">, + I32EnumAttrCase<"hostRegisterSupported", 99, "hostRegisterSupported">, + I32EnumAttrCase<"pageableMemoryAccessUsesHostPageTables", 100, "pageableMemoryAccessUsesHostPageTables">, + I32EnumAttrCase<"directManagedMemAccessFromHost", 101, "directManagedMemAccessFromHost">, + I32EnumAttrCase<"maxBlocksPerMultiprocessor", 106, "maxBlocksPerMultiprocessor">, + I32EnumAttrCase<"maxPersistingL2CacheSize", 108, "maxPersistingL2CacheSize">, + I32EnumAttrCase<"maxAccessPolicyWindowSize", 109, "maxAccessPolicyWindowSize">, + I32EnumAttrCase<"reservedSharedMemoryPerBlock", 111, "reservedSharedMemoryPerBlock">, + I32EnumAttrCase<"sparseCudaArraySupported", 112, "sparseCudaArraySupported">, + I32EnumAttrCase<"hostRegisterReadOnlySupported", 113, "hostRegisterReadOnlySupported">, + I32EnumAttrCase<"timelineSemaphoreInteropSupported", 114, "timelineSemaphoreInteropSupported">, + I32EnumAttrCase<"memoryPoolsSupported", 115, "memoryPoolsSupported">, + I32EnumAttrCase<"gpuDirectRdmaSupported", 116, "gpuDirectRdmaSupported">, + I32EnumAttrCase<"gpuDirectRdmaFlushWritesOptions", 117, "gpuDirectRdmaFlushWritesOptions">, + I32EnumAttrCase<"gpuDirectRdmaWritesOrdering", 118, "gpuDirectRdmaWritesOrdering">, + I32EnumAttrCase<"memoryPoolSupportedHandleTypes", 119, "memoryPoolSupportedHandleTypes">, + I32EnumAttrCase<"clusterLaunch", 120, "clusterLaunch">, + I32EnumAttrCase<"deferredMappingCudaArraySupported", 121, "deferredMappingCudaArraySupported">, + I32EnumAttrCase<"reserved122", 122, "reserved122">, + I32EnumAttrCase<"reserved123", 123, "reserved123">, + I32EnumAttrCase<"reserved124", 124, "reserved124">, + I32EnumAttrCase<"ipcEventSupport", 125, "ipcEventSupport">, + I32EnumAttrCase<"memSyncDomainCount", 126, "memSyncDomainCount">, + I32EnumAttrCase<"reserved127", 127, "reserved127">, + I32EnumAttrCase<"reserved128", 128, "reserved128">, + I32EnumAttrCase<"reserved129", 129, "reserved129">, + I32EnumAttrCase<"numaConfig", 130, "numaConfig">, + I32EnumAttrCase<"numaId", 131, "numaId">, + I32EnumAttrCase<"reserved132", 132, "reserved132">, + I32EnumAttrCase<"mpsEnabled", 133, "mpsEnabled">, + I32EnumAttrCase<"hostNumaId", 134, "hostNumaId">, + I32EnumAttrCase<"d3d12CigSupported", 135, "d3d12CigSupported">, + I32EnumAttrCase<"vulkanCigSupported", 138, "vulkanCigSupported">, + I32EnumAttrCase<"gpuPciDeviceId", 139, "gpuPciDeviceId">, + I32EnumAttrCase<"gpuPciSubsystemId", 140, "gpuPciSubsystemId">, + I32EnumAttrCase<"reserved141", 141, "reserved141">, + I32EnumAttrCase<"hostNumaMemoryPoolsSupported", 142, "hostNumaMemoryPoolsSupported">, + I32EnumAttrCase<"hostNumaMultinodeIpcSupported", 143, "hostNumaMultinodeIpcSupported">, + I32EnumAttrCase<"hostMemoryPoolsSupported", 144, "hostMemoryPoolsSupported">, + I32EnumAttrCase<"reserved145", 145, "reserved145">, + I32EnumAttrCase<"onlyPartialHostNativeAtomicSupported", 147, "onlyPartialHostNativeAtomicSupported">, + I32EnumAttrCase<"cudaDevAttrOversizedSharedMemoryPerBlock", 150, "cudaDevAttrOversizedSharedMemoryPerBlock"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CUFunctionAttributeEnum : I32EnumAttr<"CUFunctionAttribute", "Enumerated type describing CUDA function attributes can be set through Cuda Dialect", [ + I32EnumAttrCase<"max_threads_per_block", 0, "max_threads_per_block">, + I32EnumAttrCase<"shared_size_bytes", 1, "shared_size_bytes">, + I32EnumAttrCase<"const_size_bytes", 2, "const_size_bytes">, + I32EnumAttrCase<"local_size_bytes", 3, "local_size_bytes">, + I32EnumAttrCase<"num_regs", 4, "num_regs">, + I32EnumAttrCase<"ptx_version", 5, "ptx_version">, + I32EnumAttrCase<"binary_version", 6, "binary_version">, + I32EnumAttrCase<"cache_mode_ca", 7, "cache_mode_ca">, + I32EnumAttrCase<"max_dynamic_shared_size_bytes", 8, "max_dynamic_shared_size_bytes">, + I32EnumAttrCase<"preferred_shared_memory_carveout", 9, "preferred_shared_memory_carveout">, + I32EnumAttrCase<"cluster_size_must_be_set", 10, "cluster_size_must_be_set">, + I32EnumAttrCase<"required_cluster_width", 11, "required_cluster_width">, + I32EnumAttrCase<"required_cluster_height", 12, "required_cluster_height">, + I32EnumAttrCase<"required_cluster_depth", 13, "required_cluster_depth">, + I32EnumAttrCase<"non_portable_cluster_size_allowed", 14, "non_portable_cluster_size_allowed">, + I32EnumAttrCase<"cluster_scheduling_policy_preference", 15, "cluster_scheduling_policy_preference"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CudaABIEnum : I32EnumAttr<"CudaABI", "Enumerated of the host library ABI", [ + I32EnumAttrCase<"LP64", 0, "LP64">, + I32EnumAttrCase<"ILP64", 1, "ILP64">, + I32EnumAttrCase<"LLP64", 2, "LLP64"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CudaAccessPropertyEnum : I32EnumAttr<"CudaAccessProperty", "Specifies performance hint with cudaAccessPolicyWindow for hitProp and missProp members", [ + I32EnumAttrCase<"cudaAccessPropertyNormal", 0, "cudaAccessPropertyNormal">, + I32EnumAttrCase<"cudaAccessPropertyStreaming", 1, "cudaAccessPropertyStreaming">, + I32EnumAttrCase<"cudaAccessPropertyPersisting", 2, "cudaAccessPropertyPersisting"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CudaClusterSchedulingPolicyEnum : I32EnumAttr<"CudaClusterSchedulingPolicy", "Enumerated type describing CUDA cluster scheduling policy", [ + I32EnumAttrCase<"cudaClusterSchedulingPolicyDefault", 0, "cudaClusterSchedulingPolicyDefault">, + I32EnumAttrCase<"cudaClusterSchedulingPolicySpread", 1, "cudaClusterSchedulingPolicySpread">, + I32EnumAttrCase<"cudaClusterSchedulingPolicyLoadBalancing", 2, "cudaClusterSchedulingPolicyLoadBalancing"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CudaFuncAttributeEnum : I32EnumAttr<"CudaFuncAttribute", "Enumerated type describing CUDA function attributes can be set through Cuda Dialect", [ + I32EnumAttrCase<"cudaFuncAttributeSharedMemoryMode", 16, "cudaFuncAttributeSharedMemoryMode"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CudaLaunchAttributeIDEnum : I32EnumAttr<"CudaLaunchAttributeID", "Enumerated type mirroring CudaLaunchAttributeID from CUDA Runtime API", [ + I32EnumAttrCase<"cudaLaunchAttributeIgnore", 0, "cudaLaunchAttributeIgnore">, + I32EnumAttrCase<"cudaLaunchAttributeAccessPolicyWindow", 1, "cudaLaunchAttributeAccessPolicyWindow">, + I32EnumAttrCase<"cudaLaunchAttributeCooperative", 2, "cudaLaunchAttributeCooperative">, + I32EnumAttrCase<"cudaLaunchAttributeSynchronizationPolicy", 3, "cudaLaunchAttributeSynchronizationPolicy">, + I32EnumAttrCase<"cudaLaunchAttributeClusterDimension", 4, "cudaLaunchAttributeClusterDimension">, + I32EnumAttrCase<"cudaLaunchAttributeClusterSchedulingPolicyPreference", 5, "cudaLaunchAttributeClusterSchedulingPolicyPreference">, + I32EnumAttrCase<"cudaLaunchAttributeProgrammaticStreamSerialization", 6, "cudaLaunchAttributeProgrammaticStreamSerialization">, + I32EnumAttrCase<"cudaLaunchAttributeProgrammaticEvent", 7, "cudaLaunchAttributeProgrammaticEvent">, + I32EnumAttrCase<"cudaLaunchAttributePriority", 8, "cudaLaunchAttributePriority">, + I32EnumAttrCase<"cudaLaunchAttributeMemSyncDomainMap", 9, "cudaLaunchAttributeMemSyncDomainMap">, + I32EnumAttrCase<"cudaLaunchAttributeMemSyncDomain", 10, "cudaLaunchAttributeMemSyncDomain">, + I32EnumAttrCase<"cudaLaunchAttributePreferredClusterDimension", 11, "cudaLaunchAttributePreferredClusterDimension">, + I32EnumAttrCase<"cudaLaunchAttributeLaunchCompletionEvent", 12, "cudaLaunchAttributeLaunchCompletionEvent">, + I32EnumAttrCase<"cudaLaunchAttributeDeviceUpdatableKernelNode", 13, "cudaLaunchAttributeDeviceUpdatableKernelNode">, + I32EnumAttrCase<"cudaLaunchAttributePreferredSharedMemoryCarveout", 14, "cudaLaunchAttributePreferredSharedMemoryCarveout">, + I32EnumAttrCase<"cudaLaunchAttributeNvlinkUtilCentricScheduling", 16, "cudaLaunchAttributeNvlinkUtilCentricScheduling"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CudaLaunchMemSyncDomainEnum : I32EnumAttr<"CudaLaunchMemSyncDomain", "Enumerated type describing CUDA memory sync domain", [ + I32EnumAttrCase<"cudaLaunchMemSyncDomainDefault", 0, "cudaLaunchMemSyncDomainDefault">, + I32EnumAttrCase<"cudaLaunchMemSyncDomainRemote", 1, "cudaLaunchMemSyncDomainRemote"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CudaSharedMemoryModeEnum : I32EnumAttr<"CudaSharedMemoryMode", "Enumerated type describing CUDA shared memory modes", [ + I32EnumAttrCase<"cudaSharedMemoryModeDefault", 0, "cudaSharedMemoryModeDefault">, + I32EnumAttrCase<"cudaSharedMemoryModeRequirePortable", 1, "cudaSharedMemoryModeRequirePortable">, + I32EnumAttrCase<"cudaSharedMemoryModeAllowNonPortable", 2, "cudaSharedMemoryModeAllowNonPortable">, + I32EnumAttrCase<"cudaSharedMemoryModeAllowOversizedSharedMemory", 3, "cudaSharedMemoryModeAllowOversizedSharedMemory">, + I32EnumAttrCase<"cudaSharedMemoryModePreferOversizedSharedMemory", 4, "cudaSharedMemoryModePreferOversizedSharedMemory"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CudaSynchronizationPolicyEnum : I32EnumAttr<"CudaSynchronizationPolicy", "Enumerated type describing CUDA synchronization policy", [ + I32EnumAttrCase<"cudaSyncPolicyAuto", 0, "cudaSyncPolicyAuto">, + I32EnumAttrCase<"cudaSyncPolicySpin", 1, "cudaSyncPolicySpin">, + I32EnumAttrCase<"cudaSyncPolicyYield", 2, "cudaSyncPolicyYield">, + I32EnumAttrCase<"cudaSyncPolicyBlockingSync", 3, "cudaSyncPolicyBlockingSync"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_ExecutableFormatEnum : I32EnumAttr<"ExecutableFormat", "Enumerated type describing the format of an executable", [ + I32EnumAttrCase<"text", 0, "text">, + I32EnumAttrCase<"elf", 1, "elf">, + I32EnumAttrCase<"bitcode", 2, "bitcode">, + I32EnumAttrCase<"fatbin", 3, "fatbin"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_ExecutableRepresentationEnum : I32EnumAttr<"ExecutableRepresentation", "Enumerated type describing the representation of an executable", [ + I32EnumAttrCase<"sass", 0, "sass">, + I32EnumAttrCase<"ptx", 2, "ptx">, + I32EnumAttrCase<"nvvm", 3, "nvvm">, + I32EnumAttrCase<"ltoir", 4, "ltoir">, + I32EnumAttrCase<"mlir", 5, "mlir">, + I32EnumAttrCase<"llvm", 6, "llvm">, + I32EnumAttrCase<"hostobj", 7, "hostobj"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_GpuArchitectureEnum : I32EnumAttr<"GpuArchitecture", "Enumerated type denoting a GPU Architecture", [ + I32EnumAttrCase<"sm_80", 80, "sm_80">, + I32EnumAttrCase<"sm_86", 86, "sm_86">, + I32EnumAttrCase<"sm_87", 87, "sm_87">, + I32EnumAttrCase<"sm_88", 88, "sm_88">, + I32EnumAttrCase<"sm_89", 89, "sm_89">, + I32EnumAttrCase<"sm_90", 90, "sm_90">, + I32EnumAttrCase<"sm_100", 100, "sm_100">, + I32EnumAttrCase<"sm_101", 101, "sm_101">, + I32EnumAttrCase<"sm_103", 103, "sm_103">, + I32EnumAttrCase<"sm_107", 107, "sm_107">, + I32EnumAttrCase<"sm_110", 110, "sm_110">, + I32EnumAttrCase<"sm_120", 120, "sm_120">, + I32EnumAttrCase<"sm_121", 121, "sm_121"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_MemcpyKindEnum : I32EnumAttr<"MemcpyKind", "Enumerated type describing memcpy directions", [ + I32EnumAttrCase<"HostToHost", 0, "HostToHost">, + I32EnumAttrCase<"HostToDevice", 1, "HostToDevice">, + I32EnumAttrCase<"DeviceToHost", 2, "DeviceToHost">, + I32EnumAttrCase<"DeviceToDevice", 3, "DeviceToDevice">, + I32EnumAttrCase<"Default", 4, "Default"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_StreamCaptureModeEnumEnum : I32EnumAttr<"StreamCaptureModeEnum", "Enumerated type describing CUDA stream capture modes", [ + I32EnumAttrCase<"Global", 0, "Global">, + I32EnumAttrCase<"ThreadLocal", 1, "ThreadLocal">, + I32EnumAttrCase<"Relaxed", 2, "Relaxed"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +#endif // CUTE_IR_GENERATED_CUDA_ENUMS_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaOps.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaOps.td new file mode 100644 index 00000000..ee468f71 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaOps.td @@ -0,0 +1,3072 @@ +// The cuda dialect's operations, generated from the nvidia-cutlass-dsl 4.8.0 release; do not edit. +// +// Sources: the operation bindings (`_cuda_ops_gen.py`: names, classes, +// docstrings, operand/result/attribute structure, inference) and the live +// compiler (`_cutlass_ir` loaded from Python): each operand and result +// constraint carries the verifier's wording and accepts every type class the +// verifier let through when probed with one instance of every type; traits +// and interfaces are the registered operation's; memory effects come from +// loop-invariant code motion (hoisted: Pure) and CSE (merged: at most +// reads). Operations whose effects include writes or could not be told +// apart carry none (unknown effects). The compiler's own verifiers (operand +// combinations, shapes, instruction legality) are left to it. +// +// Operations keep the generic assembly form. +// +// Result types the compiler infers without a recovered rule (written types stand): +// cuda.launch_cfg.create_from_stream +// +// Constraints not recovered: + +#ifndef CUTE_IR_GENERATED_CUDA_OPS_TD +#define CUTE_IR_GENERATED_CUDA_OPS_TD + +include "cute_ir/Dialect/Cuda/IR/CudaAttrs.td" +include "cute_ir/Dialect/Cuda/IR/CudaEnums.td" +include "cute_ir/Dialect/Cuda/IR/CudaTypes.td" + +include "mlir/Interfaces/InferTypeOpInterface.td" +include "mlir/Interfaces/SideEffectInterfaces.td" + +// Operand and result constraints, in the compiler's words. +def Cuda_AnyCUDAStructType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeDim3Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeExtentType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeKernelNodeParamsType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeMemcpy3DParamsType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimePitchedPtrType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimePosType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeUuidType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueCooperativeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueMemSyncDomainType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValuePortableClusterSizeModeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValuePriorityType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueSharedMemCarveoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueSharedMemoryModeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueSyncPolicyType>($_self)">, "any CUDA struct type">; +def Cuda_AnyType : Type, "any type">; +def Cuda_CUDACudaDevicePropStructTypeValueTyped : Type($_self)">, "CUDA cudaDeviceProp struct type (value-typed).">, BuildableType<"::mlir::cutlass_compiler::cuda::RuntimeDevicePropType::get($_builder.getContext())">; +def Cuda_CUDACudaDevicePropStructTypeValueTyped2 : Type($_self)">, "CUDA cudaDeviceProp struct type (value-typed).">; +def Cuda_CUDACudaExtentStructTypeValueTyped : Type($_self)">, "CUDA cudaExtent struct type (value-typed).">; +def Cuda_CUDACudaExtentStructTypeValueTyped2 : Type($_self)">, "CUDA cudaExtent struct type (value-typed).">, BuildableType<"::mlir::cutlass_compiler::cuda::RuntimeExtentType::get($_builder.getContext())">; +def Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped : Type($_self)">, "CUDA cudaKernelNodeParams struct type (value-typed).">; +def Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped : Type($_self)">, "CUDA cudaMemcpy3DParms struct type (value-typed).">; +def Cuda_CUDACudaPitchedPtrStructTypeValueTyped : Type($_self)">, "CUDA cudaPitchedPtr struct type (value-typed).">, BuildableType<"::mlir::cutlass_compiler::cuda::RuntimePitchedPtrType::get($_builder.getContext())">; +def Cuda_CUDACudaPitchedPtrStructTypeValueTyped2 : Type($_self)">, "CUDA cudaPitchedPtr struct type (value-typed).">; +def Cuda_CUDACudaPosStructTypeValueTyped : Type($_self)">, "CUDA cudaPos struct type (value-typed).">, BuildableType<"::mlir::cutlass_compiler::cuda::RuntimePosType::get($_builder.getContext())">; +def Cuda_CUDACudaPosStructTypeValueTyped2 : Type($_self)">, "CUDA cudaPos struct type (value-typed).">; +def Cuda_CUDACudaUUIDTStructTypeValue : Type($_self)">, "CUDA cudaUUID_t struct type (value-typed).">, BuildableType<"::mlir::cutlass_compiler::cuda::RuntimeUuidType::get($_builder.getContext())">; +def Cuda_CUDACudaUUIDTStructTypeValue2 : Type($_self)">, "CUDA cudaUUID_t struct type (value-typed).">; +def Cuda_CUDADim3StructTypeValueTyped : Type($_self)">, "CUDA dim3 struct type (value-typed).">; +def Cuda_CUDADim3StructTypeValueTyped2 : Type($_self)">, "CUDA dim3 struct type (value-typed).">, BuildableType<"::mlir::cutlass_compiler::cuda::RuntimeDim3Type::get($_builder.getContext())">; +def Cuda_CUDAGraph : Type($_self)">, "CUDA Graph">; +def Cuda_CUDAGraph2 : Type($_self)">, "CUDA Graph">, BuildableType<"::mlir::cutlass_compiler::cuda::GraphType::get($_builder.getContext())">; +def Cuda_CUDAGraphExecution : Type($_self)">, "CUDA Graph Execution">, BuildableType<"::mlir::cutlass_compiler::cuda::GraphExecType::get($_builder.getContext())">; +def Cuda_CUDAGraphExecution2 : Type($_self)">, "CUDA Graph Execution">; +def Cuda_CUDAGraphNodeHandle : Type($_self)">, "CUDA Graph Node handle">, BuildableType<"::mlir::cutlass_compiler::cuda::GraphNodeType::get($_builder.getContext())">; +def Cuda_CUDAGraphNodeHandle2 : Type($_self)">, "CUDA Graph Node handle">; +def Cuda_CUDAKernelHandle : Type($_self)">, "CUDA Kernel handle">, BuildableType<"::mlir::cutlass_compiler::cuda::KernelType::get($_builder.getContext())">; +def Cuda_CUDAKernelHandle2 : Type($_self)">, "CUDA Kernel handle">; +def Cuda_CUDALibraryHandle : Type($_self)">, "CUDA Library handle">; +def Cuda_CUDALibraryHandle2 : Type($_self)">, "CUDA Library handle">, BuildableType<"::mlir::cutlass_compiler::cuda::LibraryType::get($_builder.getContext())">; +def Cuda_CUDAModuleHandle : Type($_self)">, "CUDA Module handle">, BuildableType<"::mlir::cutlass_compiler::cuda::ModuleType::get($_builder.getContext())">; +def Cuda_CUDAResultType : Type($_self)">, "CUDA Result type">, BuildableType<"::mlir::cutlass_compiler::cuda::ResultType::get($_builder.getContext())">; +def Cuda_CUDAResultType2 : Type($_self)">, "CUDA Result type">; +def Cuda_CUDAResultTypeOrCUDAStream : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::StreamType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::ResultType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::EventType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::GraphDeviceNodeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::TensorMapType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::IntType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::LaunchConfigType>($_self) || ($_self.getDialect().getNamespace() == \"ptr\" || (::llvm::isa<::mlir::OpaqueType>($_self) && ::llvm::cast<::mlir::OpaqueType>($_self).getDialectNamespace() == \"ptr\"))">, "CUDA Result type or CUDA Stream or `Event` represents CUDA event or CUDA Tensor Map type or Representation of CUDA's extensible launch configuration mirroring cuda runtime API or `GraphDeviceNode` represents CUDA device node handle for device-side node update or `cuda.int` is a integer type with a bitwidth matching the underlying ABI integer bitwidth or 32-bit signless integer or 64-bit signless integer or pointer type or LLVM pointer type">; +def Cuda_CUDAStream : Type($_self)">, "CUDA Stream">; +def Cuda_CUDAStream2 : Type($_self)">, "CUDA Stream">, BuildableType<"::mlir::cutlass_compiler::cuda::StreamType::get($_builder.getContext())">; +def Cuda_CUDATensorMapType : Type($_self)">, "CUDA Tensor Map type">, BuildableType<"::mlir::cutlass_compiler::cuda::TensorMapType::get($_builder.getContext())">; +def Cuda_CudaIntIsAIntegerType : Type($_self)">, "`cuda.int` is a integer type with a bitwidth matching the underlying ABI integer bitwidth">; +def Cuda_EventRepresentsCUDAEvent : Type($_self)">, "`Event` represents CUDA event">, BuildableType<"::mlir::cutlass_compiler::cuda::EventType::get($_builder.getContext())">; +def Cuda_EventRepresentsCUDAEvent2 : Type($_self)">, "`Event` represents CUDA event">; +def Cuda_FixedLengthVectorOf32Bit : Type($_self)">, "fixed-length vector of 32-bit signless integer values of length 3">; +def Cuda_FixedLengthVectorOf32Bit2 : Type($_self)">, "fixed-length vector of 32-bit signless integer values of length 2">; +def Cuda_FixedLengthVectorOf8Bit : Type($_self)">, "fixed-length vector of 8-bit signless integer values of length 8">; +def Cuda_FixedLengthVectorOf8Bit2 : Type($_self)">, "fixed-length vector of 8-bit signless integer values of length 256">; +def Cuda_FixedLengthVectorOf8Bit3 : Type($_self)">, "fixed-length vector of 8-bit signless integer values of length 16">; +def Cuda_GraphDeviceNodeRepresentsCUDADeviceNodeHandle : Type($_self)">, "`GraphDeviceNode` represents CUDA device node handle for device-side node update">; +def Cuda_LLVMPointerInAddressSpace0 : Type($_self)">, "LLVM pointer in address space 0">; +def Cuda_LLVMPointerType : Type($_self)">, "LLVM pointer type">; +def Cuda_PointerType : Type($_self) && ::llvm::cast<::mlir::OpaqueType>($_self).getDialectNamespace() == \"ptr\"))">, "pointer type">; +def Cuda_RepresentationOfCUDASExtensibleLaunch : Type($_self)">, "Representation of CUDA's extensible launch configuration mirroring cuda runtime API">; +def Cuda_T1BitSignlessInteger : Type, "1-bit signless integer">, BuildableType<"$_builder.getIntegerType(1)">; +def Cuda_T32BitFloat : Type($_self)">, "32-bit float">, BuildableType<"::mlir::Float32Type::get($_builder.getContext())">; +def Cuda_T32BitFloat2 : Type($_self)">, "32-bit float">; +def Cuda_T32BitSignlessInteger : Type, "32-bit signless integer">; +def Cuda_T32BitSignlessInteger2 : Type, "32-bit signless integer">, BuildableType<"$_builder.getIntegerType(32)">; +def Cuda_T64BitSignlessInteger : Type, "64-bit signless integer">; +def Cuda_T64BitSignlessInteger2 : Type, "64-bit signless integer">, BuildableType<"$_builder.getIntegerType(64)">; +def Cuda_VariadicOf32BitSignlessInteger : Type, "variadic of 32-bit signless integer">; +def Cuda_VariantCooperativeOfCUDACudaLaunchAttributeValueUnion : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueMemSyncDomainType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValuePortableClusterSizeModeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValuePriorityType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueSharedMemCarveoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueSharedMemoryModeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueSyncPolicyType>($_self)">, "Variant `cooperative` of CUDA `cudaLaunchAttributeValue` union (payload: int). or Variant `syncPolicy` of CUDA `cudaLaunchAttributeValue` union (payload: enum cudaSynchronizationPolicy). or Variant `clusterSchedulingPolicyPreference` of CUDA `cudaLaunchAttributeValue` union (payload: enum cudaClusterSchedulingPolicy). or Variant `programmaticStreamSerializationAllowed` of CUDA `cudaLaunchAttributeValue` union (payload: int). or Variant `priority` of CUDA `cudaLaunchAttributeValue` union (payload: int). or Variant `memSyncDomain` of CUDA `cudaLaunchAttributeValue` union (payload: cudaLaunchMemSyncDomain). or Variant `sharedMemCarveout` of CUDA `cudaLaunchAttributeValue` union (payload: unsigned int). or Variant `nvlinkUtilCentricScheduling` of CUDA `cudaLaunchAttributeValue` union (payload: unsigned int). or Variant `portableClusterSizeMode` of CUDA `cudaLaunchAttributeValue` union (payload: cudaLaunchAttributePortableClusterMode). or Variant `sharedMemoryMode` of CUDA `cudaLaunchAttributeValue` union (payload: cudaSharedMemoryMode).">; + +// Attribute constraints the bindings name but do not describe. +def Cuda_DenseResourceElementsAttr : Attr($_self)">, "dense resource elements attribute"> { + let storageType = "::mlir::DenseResourceElementsAttr"; + let returnType = "::mlir::DenseResourceElementsAttr"; + let convertFromStorage = "$_self"; +} + +def Cuda_FuncFunction_typeAttr : Attr($_self) && (::llvm::isa<::mlir::FunctionType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of function type"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} + +def Cuda_KernelFunction_typeAttr : Attr($_self) && (::llvm::isa<::mlir::FunctionType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of function type"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} + +def Cuda_IntegerAttr : Attr($_self)">, "integer attribute"> { + let storageType = "::mlir::IntegerAttr"; + let returnType = "::mlir::IntegerAttr"; + let convertFromStorage = "$_self"; +} + +// Operations whose result types are inferred in C++ (CudaInference.cpp). +// Verification compares them with the inferred ones where the rule is +// recovered; elsewhere the types written stand, as the compiler accepted them. +class Cuda_InferredOp traits = []> + : Cuda_Op]> { + let extraClassDefinition = [{ + ::llvm::LogicalResult $cppClass::refineReturnTypes( + ::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, + ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, + ::mlir::PropertyRef properties, ::mlir::RegionRange regions, + ::llvm::SmallVectorImpl<::mlir::Type> &returnTypes) { + ::llvm::SmallVector<::mlir::Type, 4> inferred; + if (::mlir::failed(inferReturnTypes(context, ::std::nullopt, operands, + attributes, properties, regions, + inferred))) + return ::mlir::success(); + if (!isCompatibleReturnTypes(inferred, returnTypes)) + return ::mlir::emitOptionalError( + location, "'", getOperationName(), "' op inferred type(s) ", + inferred, " are incompatible with return type(s) of operation ", + returnTypes); + return ::mlir::success(); + } + }]; +} + +def Cuda_AddressOfOp : Cuda_Op<"addressof"> { + let arguments = (ins + SymbolRefAttr:$symname + ); + let results = (outs + Cuda_PointerType:$result + ); +} +def Cuda_BinaryOp : Cuda_Op<"binary", [NoTerminator]> { + let description = [{ + This operation represents a CUDA binary, which contains a number of functions and + a serialized representation of the binary. + + Examples: + + ```mlir + + cuda.binary @binary_name <...> { + cuda.kernel @function_name function_type + } + ``` + }]; + let arguments = (ins + SymbolNameAttr:$sym_name, + Cuda_DenseResourceElementsAttr:$binary + ); + let results = (outs); + let regions = (region AnyRegion:$body); +} +def Cuda_CastOp : Cuda_Op<"cast", [Pure]> { + let description = [{ + Casts between CUDA types and standard dialect or built-in types. + The source/destination types are statically checked and the IR + is invalid if the cast is not possible. + + Examples: + ```mlir + + %res_i32 = cuda.cast %res : cuda.result -> i32 + %res1 = cuda.cast %res_i32 : i32 -> cuda.result + + %s_ptr = cuda.cast %hstream : cuda.stream -> ptr.ptr + %s_i64 = cuda.cast %hstream : cuda.stream -> i64 + ``` + }]; + let arguments = (ins + Cuda_CUDAResultTypeOrCUDAStream:$src + ); + let results = (outs + Cuda_CUDAResultTypeOrCUDAStream:$dst + ); +} +def Cuda_ComposeOp : Cuda_Op<"compose"> { + let description = [{ + Generic constructor for any auto-generated CUDA struct type. The result + type identifies which struct is being built; operands are the struct's + non-padding fields in declaration order. The verifier checks operand + count and per-operand type against the result type's + `CudaStructTypeInterface::getOperandFieldTypes()`. + + Lowers to `llvm.mlir.undef` + a chain of `llvm.insertvalue` (one per + operand-mapped slot from `getOperandFieldIndices()`). Pure value SSA; + no memory. + + Example: + ```mlir + %pos = cuda.compose %x, %y, %z + : i64, i64, i64 -> !cuda.runtime.pos + ``` + }]; + let arguments = (ins + Variadic:$fields + ); + let results = (outs + Cuda_AnyCUDAStructType:$result + ); +} +def Cuda_DecomposeOp : Cuda_Op<"decompose"> { + let description = [{ + Generic accessor: returns each non-padding field of a CUDA struct value + as an SSA value, in declaration order. The verifier checks result count + and per-result type against the source type's + `CudaStructTypeInterface::getOperandFieldTypes()`. + + Lowers to a chain of `llvm.extractvalue` (one per operand-mapped slot + from `getOperandFieldIndices()`). Pure value SSA; no memory. + + Example: + ```mlir + %x, %y, %z = cuda.decompose %pos + : !cuda.runtime.pos -> i64, i64, i64 + ``` + }]; + let arguments = (ins + Cuda_AnyCUDAStructType:$source + ); + let results = (outs + Optional:$fields + ); +} +def Cuda_DeviceSyncOp : Cuda_Op<"device.sync"> { + let description = [{ + Operation to synchronize with the device. See cudaDeviceSynchronize() + }]; + let arguments = (ins); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_DriverEventCreateOp : Cuda_Op<"driver.event.create"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_T32BitSignlessInteger:$Flags + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_EventRepresentsCUDAEvent:$phEvent + ); +} +def Cuda_DriverEventDestroyOp : Cuda_Op<"driver.event.destroy"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$hEvent + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_DriverEventElapsedTimeOp : Cuda_Op<"driver.event.elapsed_time"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$hStart, + Cuda_EventRepresentsCUDAEvent2:$hEnd + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_T32BitFloat:$pMilliseconds + ); +} +def Cuda_DriverEventQueryOp : Cuda_Op<"driver.event.query", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$hEvent + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_DriverEventRecordOp : Cuda_Op<"driver.event.record"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$hEvent, + Cuda_CUDAStream:$hStream + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_DriverEventRecordWithFlagsOp : Cuda_Op<"driver.event.record_with_flags"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$hEvent, + Cuda_CUDAStream:$hStream, + Cuda_T32BitSignlessInteger:$flags + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_DriverEventSynchronizeOp : Cuda_Op<"driver.event.synchronize"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$hEvent + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_DriverLibraryGetGlobalOp : Cuda_Op<"driver.library.get_global", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$name_arg + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$dptr, + Cuda_T64BitSignlessInteger:$bytes + ); +} +def Cuda_DriverLibraryGetKernelOp : Cuda_Op<"driver.library.get_kernel", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$name_arg + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDAKernelHandle:$pKernel + ); +} +def Cuda_DriverLibraryGetKernelCountOp : Cuda_Op<"driver.library.get_kernel_count", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$lib + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_T32BitSignlessInteger2:$count + ); +} +def Cuda_DriverLibraryGetManagedOp : Cuda_Op<"driver.library.get_managed", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$name_arg + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$dptr, + Cuda_T64BitSignlessInteger:$bytes + ); +} +def Cuda_DriverLibraryGetModuleOp : Cuda_Op<"driver.library.get_module", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDAModuleHandle:$pMod + ); +} +def Cuda_DriverLibraryGetUnifiedFunctionOp : Cuda_Op<"driver.library.get_unified_function", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$symbol + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$fptr + ); +} +def Cuda_DriverLibraryLoadDataOp : Cuda_Op<"driver.library.load_data"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_PointerType:$code_arg, + Cuda_PointerType:$jitOptions, + Cuda_PointerType:$jitOptionsValues, + Cuda_T32BitSignlessInteger:$numJitOptions, + Cuda_PointerType:$libraryOptions, + Cuda_PointerType:$libraryOptionValues, + Cuda_T32BitSignlessInteger:$numLibraryOptions + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDALibraryHandle2:$library + ); +} +def Cuda_DriverLibraryLoadFromFileOp : Cuda_Op<"driver.library.load_from_file"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_PointerType:$fileName, + Cuda_PointerType:$jitOptions, + Cuda_PointerType:$jitOptionsValues, + Cuda_T32BitSignlessInteger:$numJitOptions, + Cuda_PointerType:$libraryOptions, + Cuda_PointerType:$libraryOptionValues, + Cuda_T32BitSignlessInteger:$numLibraryOptions + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDALibraryHandle2:$library + ); +} +def Cuda_DriverLibraryUnloadOp : Cuda_Op<"driver.library.unload"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_DriverTensorMapEncodeIm2colOp : Cuda_Op<"driver.tensor_map.encode_im2col"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_T32BitSignlessInteger:$tensorDataType, + Cuda_T32BitSignlessInteger:$tensorRank, + Cuda_PointerType:$globalAddress, + Cuda_PointerType:$globalDim, + Cuda_PointerType:$globalStrides, + Cuda_PointerType:$pixelBoxLowerCorner, + Cuda_PointerType:$pixelBoxUpperCorner, + Cuda_T32BitSignlessInteger:$channelsPerPixel, + Cuda_T32BitSignlessInteger:$pixelsPerColumn, + Cuda_PointerType:$elementStrides, + Cuda_T32BitSignlessInteger:$interleave, + Cuda_T32BitSignlessInteger:$swizzle, + Cuda_T32BitSignlessInteger:$l2Promotion, + Cuda_T32BitSignlessInteger:$oobFill + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDATensorMapType:$tensorMap + ); +} +def Cuda_DriverTensorMapEncodeIm2colWideOp : Cuda_Op<"driver.tensor_map.encode_im2col_wide"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_T32BitSignlessInteger:$tensorDataType, + Cuda_T32BitSignlessInteger:$tensorRank, + Cuda_PointerType:$globalAddress, + Cuda_PointerType:$globalDim, + Cuda_PointerType:$globalStrides, + Cuda_T32BitSignlessInteger:$pixelBoxLowerCornerWidth, + Cuda_T32BitSignlessInteger:$pixelBoxUpperCornerWidth, + Cuda_T32BitSignlessInteger:$channelsPerPixel, + Cuda_T32BitSignlessInteger:$pixelsPerColumn, + Cuda_PointerType:$elementStrides, + Cuda_T32BitSignlessInteger:$interleave, + Cuda_T32BitSignlessInteger:$mode, + Cuda_T32BitSignlessInteger:$swizzle, + Cuda_T32BitSignlessInteger:$l2Promotion, + Cuda_T32BitSignlessInteger:$oobFill + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDATensorMapType:$tensorMap + ); +} +def Cuda_DriverTensorMapReplaceAddressOp : Cuda_Op<"driver.tensor_map.replace_address"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_PointerType:$globalAddress + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDATensorMapType:$tensorMap + ); +} +def Cuda_EventCreateOp : Cuda_Op<"event.create"> { + let description = [{ + Operation to create a CUDA event. + }]; + let arguments = (ins); + let results = (outs + Cuda_CUDAResultType:$result, + Cuda_EventRepresentsCUDAEvent:$event + ); +} +def Cuda_EventDestroyOp : Cuda_Op<"event.destroy"> { + let description = [{ + Operation to destroy a CUDA event. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$event + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_EventElapsedTimeOp : Cuda_Op<"event.elapsed_time"> { + let description = [{ + Operation to compute the elapsed time between events + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$start_event, + Cuda_EventRepresentsCUDAEvent2:$end_event + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_T32BitFloat:$elapsed_time + ); +} +def Cuda_EventRecordOp : Cuda_Op<"event.record"> { + let description = [{ + Operation to record an event in a CUDA stream. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$event, + Optional:$stream + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_FuncOp : Cuda_Op<"func"> { + let description = [{ + This operation represents a CUDA device function which can be called from + another `cuda.func` or a `cuda.kernel`. It cannot be launched from the host. + + Examples: + + ```mlir + + cuda.module @module_name { + + cuda.func @device_func(%arg0 : i32) -> i32 { + %0 = arith.constant 0 : i32 + cuda.return %0 : i32 + } + + } + ``` + }]; + let arguments = (ins + SymbolNameAttr:$sym_name, + Cuda_FuncFunction_typeAttr:$function_type, + OptionalAttr:$arg_attrs, + OptionalAttr:$res_attrs + ); + let results = (outs); + let regions = (region AnyRegion:$body); +} +def Cuda_FuncGetNameOp : Cuda_Op<"func.get_name"> { + let arguments = (ins + Cuda_CUDAKernelHandle2:$kernel + ); + let results = (outs + Cuda_CUDAResultType2:$result, + Cuda_PointerType:$name + ); +} +def Cuda_GraphDestroyOp : Cuda_Op<"graph.destroy"> { + let description = [{ + Operation to destroy a CUDA graph. + }]; + let arguments = (ins + Cuda_CUDAGraph:$graph + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_GraphInstantiateOp : Cuda_Op<"graph.instantiate"> { + let description = [{ + Operation to instantiate a CUDA graph. + }]; + let arguments = (ins + Cuda_CUDAGraph:$graph + ); + let results = (outs + Cuda_CUDAResultType:$result, + Cuda_CUDAGraphExecution:$graph_exec + ); +} +def Cuda_GraphLaunchOp : Cuda_Op<"graph.launch"> { + let description = [{ + Operation to launch a CUDA graph. + }]; + let arguments = (ins + Cuda_CUDAGraphExecution2:$graph_exec, + Cuda_CUDAStream:$stream + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_GraphExecDestroyOp : Cuda_Op<"graph_exec.destroy"> { + let description = [{ + Operation to destroy a CUDA graph execution. + }]; + let arguments = (ins + Cuda_CUDAGraphExecution2:$graph_exec + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_KernelOp : Cuda_Op<"kernel"> { + let description = [{ + This operation represents a CUDA kernel which may be launched as a grid. It + differs from a `cuda.func`. + + Examples: + + ```mlir + + cuda.module @module_name { + + cuda.func @kernel(%arg0 : i32) + attributes { + cu_attrs = { + num_regs = 128, + max_dynamic_shared_size_bytes = 2048 + // ... add more cuda function attributes here + } + } + } + + ... + + %res = cuda.launch @module_name::@device_func <<< %grid, %block, %cluster, %smem, %stream >>>(%0) + + %0 = cuda.result.succeeded(%res) + scf.if %0 { + + } + else { + + } + ``` + }]; + let arguments = (ins + SymbolNameAttr:$sym_name, + Cuda_KernelFunction_typeAttr:$function_type, + OptionalAttr:$arg_attrs, + OptionalAttr:$res_attrs, + OptionalAttr:$cu_func_attrs + ); + let results = (outs); + let regions = (region AnyRegion:$body); +} +def Cuda_LaunchOp : Cuda_Op<"launch", [AttrSizedOperandSegments]> { + let description = [{ + Operation to launch a `cuda.kernel` on the device. + + Examples: + + ```mlir + + cuda.module @module_name { + + cuda.func @kernel(%arg0 : i32) { + + } + + } + + ... + + %0 = ... // kernel arguments + + %res = cuda.launch @module_name::@device_func<<< %grid, %block, %smem, %stream >>>(%0) : !cuda.launch_cfg, ... -> !cuda.result + + // If calling with an Op that implements a FunctionOpInterface + %res = cuda.launch @module_name::@device_func<<< %grid, %block, %smem, %stream >>>(%0) : !cuda.launch_cfg, ... -> !cuda.result + + %res2 = cuda.result.succeeded(%res) + scf.if %res2 { + + } + else { + + } + ``` + }]; + let arguments = (ins + Variadic:$grid, + Variadic:$block, + Cuda_T64BitSignlessInteger:$dynamicSmem, + Cuda_CUDAStream:$cudaStream, + Variadic:$inputs, + SymbolRefAttr:$callee, + OptionalAttr:$arg_attrs, + OptionalAttr:$res_attrs, + OptionalAttr:$assume_kernel_attr + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_LaunchCfgAccessPolicyWindowOp : Cuda_Op<"launch_cfg.access_policy_window"> { + let description = [{ + Sets accessPolicyWindow for a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create(...) + cuda.launch_cfg.access_policy_window [%cfg0] ( + base_ptr=%basePtr, + hitProp=%hitProp, + hitRatio=%hitRatio, + missProp=%missProp, + numBytes=%numBytes + ) : !cuda.launch_cfg, !ptr.ptr<#ptr.generic_space>, f32, i64 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_PointerType:$base_ptr, + Cuda_T32BitFloat2:$hitRatio, + Cuda_T64BitSignlessInteger:$num_bytes, + Cuda_CudaAccessPropertyEnum:$hitProp, + Cuda_CudaAccessPropertyEnum:$missProp + ); + let results = (outs); +} +def Cuda_LaunchCfgClusterDimOp : Cuda_Op<"launch_cfg.cluster_dim"> { + let description = [{ + Sets clusterDim for a launch_config + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create(...) + cuda.launch_cfg.cluster_dim [%cfg0] (1, 2, 8) : !cuda.launch_cfg, i32, i32, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$clusterDimX, + Cuda_T32BitSignlessInteger:$clusterDimY, + Cuda_T32BitSignlessInteger:$clusterDimZ + ); + let results = (outs); +} +def Cuda_LaunchCfgClusterSchedulingPolicyOp : Cuda_Op<"launch_cfg.cluster_scheduling_policy"> { + let description = [{ + Sets clusterSchedulingPolicyPreference for a launch_config + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.cluster_scheduling_policy [%cfg0] cudaClusterSchedulingPolicyLoadBalancing + : !cuda.launch_cfg + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_CudaClusterSchedulingPolicyEnum:$policy + ); + let results = (outs); +} +def Cuda_LaunchCfgCooperativeOp : Cuda_Op<"launch_cfg.cooperative"> { + let description = [{ + Sets cooperative attribute to a launch_config. Nonzero indicates a cooperative kernel. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.cooperative [%cfg0] 1 + : !cuda.launch_cfg, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$cooperative + ); + let results = (outs); +} +def Cuda_LaunchCfgCreateOp : Cuda_Op<"launch_cfg.create"> { + let description = [{ + Creates a launch_config object with the specified runtime values. + The result is a Cuda_LaunchConfig type that contains all the runtime data. + + Example: + ```mlir + %cfg = cuda.launch_cfg.create( + blockDim=(%block_x, %block_y, %block_z), + dynamicSmemBytes=%dynamic_smem_bytes, + gridDim=(%grid_x, %grid_y, %grid_z), + stream=%hstream + ) : i32, i32, i32, i64, i32, i32, i32, !cuda.stream -> !cuda.launch_cfg + ``` + }]; + let arguments = (ins + Cuda_T32BitSignlessInteger:$blockDimX, + Cuda_T32BitSignlessInteger:$blockDimY, + Cuda_T32BitSignlessInteger:$blockDimZ, + Cuda_T64BitSignlessInteger:$dynamicSmemBytes, + Cuda_T32BitSignlessInteger:$gridDimX, + Cuda_T32BitSignlessInteger:$gridDimY, + Cuda_T32BitSignlessInteger:$gridDimZ, + Cuda_CUDAStream:$stream, + Cuda_IntegerAttr:$maxNumAttrs + ); + let results = (outs + Cuda_RepresentationOfCUDASExtensibleLaunch:$result + ); +} +def Cuda_LaunchCfgCreateFromStreamOp : Cuda_InferredOp<"launch_cfg.create_from_stream"> { + let description = [{ + Creates a launch_config object initialized with a CUDA stream and space for + the specified number of launch attributes. Grid dimensions, block + dimensions, and dynamic shared memory bytes can be supplied later with + `cuda.launch_cfg.grid_dimensions`. + + Example: + ```mlir + %cfg = cuda.launch_cfg.create_from_stream( + stream=%hstream + ) : !cuda.stream + ``` + }]; + let arguments = (ins + Cuda_CUDAStream:$stream, + Cuda_IntegerAttr:$maxNumAttrs + ); + let results = (outs + Cuda_RepresentationOfCUDASExtensibleLaunch:$result + ); +} +def Cuda_LaunchCfgDeviceUpdatableKernelNodeOp : Cuda_Op<"launch_cfg.device_updatable_kernel_node"> { + let description = [{ + Sets deviceUpdatableKernelNode attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.device_updatable_kernel_node [%cfg0] (devNode=%devNode, deviceUpdatable=%deviceUpdatable) + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_GraphDeviceNodeRepresentsCUDADeviceNodeHandle:$devNode, + Cuda_T32BitSignlessInteger:$deviceUpdatable + ); + let results = (outs); +} +def Cuda_LaunchCfgGridDimensionsOp : Cuda_Op<"launch_cfg.grid_dimensions"> { + let description = [{ + Sets the fixed launch dimensions for a launch_config object. This updates + the direct `cudaLaunchConfig_t` fields, not the launch attribute array. + + Example: + ```mlir + cuda.launch_cfg.grid_dimensions [%cfg0] + grid(%gx, %gy, %gz), block(%bx, %by, %bz), dynamicSmem(%smem) + : !cuda.launch_cfg, i32, i32, i32, i32, i32, i32, i64 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$gridDimX, + Cuda_T32BitSignlessInteger:$gridDimY, + Cuda_T32BitSignlessInteger:$gridDimZ, + Cuda_T32BitSignlessInteger:$blockDimX, + Cuda_T32BitSignlessInteger:$blockDimY, + Cuda_T32BitSignlessInteger:$blockDimZ, + Cuda_T64BitSignlessInteger:$dynamicSmemBytes + ); + let results = (outs); +} +def Cuda_LaunchCfgLaunchCompletionEventOp : Cuda_Op<"launch_cfg.launch_completion_event"> { + let description = [{ + Sets launchCompletionEvent attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.launch_completion_event [%cfg0] (event=%event, flags=%flags) + : !cuda.launch_cfg, !cuda.event, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_EventRepresentsCUDAEvent2:$event, + Cuda_T32BitSignlessInteger:$flags + ); + let results = (outs); +} +def Cuda_LaunchCfgMemSyncDomainOp : Cuda_Op<"launch_cfg.mem_sync_domain"> { + let description = [{ + Sets memSyncDomain attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.mem_sync_domain [%cfg0] cudaMemSyncDomainRemote + : !cuda.launch_cfg + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_CudaLaunchMemSyncDomainEnum:$memSyncDomain + ); + let results = (outs); +} +def Cuda_LaunchCfgMemSyncDomainMapOp : Cuda_Op<"launch_cfg.mem_sync_domain_map"> { + let description = [{ + Sets memSyncDomainMap attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.mem_sync_domain_map [%cfg0] (domain_default=0, domain_remote=1) + : !cuda.launch_cfg, i32, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$domain_default, + Cuda_T32BitSignlessInteger:$domain_remote + ); + let results = (outs); +} +def Cuda_LaunchCfgNvlinkUtilCentricSchedulingOp : Cuda_Op<"launch_cfg.nvlink_util_centric_scheduling"> { + let description = [{ + Sets nvlinkUtilCentricScheduling attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.nvlink_util_centric_scheduling [%cfg0] 1 : !cuda.launch_cfg, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$nvlinkSchedulingEnabled + ); + let results = (outs); +} +def Cuda_LaunchCfgPreferredClusterDimOp : Cuda_Op<"launch_cfg.preferred_cluster_dim"> { + let description = [{ + Sets preferredClusterDim attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.preferred_cluster_dim [%cfg0] (1, 1, 8) + : !cuda.launch_cfg, i32, i32, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$preferredClusterDimX, + Cuda_T32BitSignlessInteger:$preferredClusterDimY, + Cuda_T32BitSignlessInteger:$preferredClusterDimZ + ); + let results = (outs); +} +def Cuda_LaunchCfgPrintOp : Cuda_Op<"launch_cfg.print"> { + let description = [{ + Prints a cuda launch config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create(...) + cuda.launch_cfg.print [%cfg0] : !cuda.launch_cfg + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg + ); + let results = (outs); +} +def Cuda_LaunchCfgPriorityOp : Cuda_Op<"launch_cfg.priority"> { + let description = [{ + Sets priority attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.priority [%cfg0] 1 + : !cuda.launch_cfg, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$priority + ); + let results = (outs); +} +def Cuda_LaunchCfgProgrammaticEventOp : Cuda_Op<"launch_cfg.programmatic_event"> { + let description = [{ + Sets programmaticEvent attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.programmatic_event [%cfg0] ( + event=%event, + flags=%flags, + triggerAtBlockStart=%triggerAtBlockStart) + : !cuda.launch_cfg, !cuda.event, i32, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_EventRepresentsCUDAEvent2:$event, + Cuda_T32BitSignlessInteger:$flags, + Cuda_T32BitSignlessInteger:$triggerAtBlockStart + ); + let results = (outs); +} +def Cuda_LaunchCfgProgrammaticStreamSerializationAllowedOp : Cuda_Op<"launch_cfg.programmatic_stream_serialization_allowed"> { + let description = [{ + Sets programmaticStreamSerializationAllowed attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.programmatic_stream_serialization_allowed [%cfg0] 1 + : !cuda.launch_cfg, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$programmaticStreamSerializationAllowed + ); + let results = (outs); +} +def Cuda_LaunchCfgSharedMemCarveoutOp : Cuda_Op<"launch_cfg.shared_mem_carveout"> { + let description = [{ + Sets sharedMemCarveout attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.shared_mem_carveout [%cfg0] 1024 + : !cuda.launch_cfg, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$sharedMemCarveout + ); + let results = (outs); +} +def Cuda_LaunchCfgSyncPolicyOp : Cuda_Op<"launch_cfg.sync_policy"> { + let description = [{ + Sets syncPolicy for a launch_config + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.sync_policy [%cfg0] cudaSynchronizationPolicyDefault + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_CudaSynchronizationPolicyEnum:$syncPolicy + ); + let results = (outs); +} +def Cuda_LaunchExOp : Cuda_Op<"launch_ex"> { + let description = [{ + Operation to launch a `cuda.kernel` on the device using extensible launch API. + + Examples: + + ```mlir + + cuda.module @module_name { + + cuda.func @kernel(%arg0 : i32) { + + } + + } + + ... + %0 = ... // kernel arguments + + %res = cuda.launch_ex @module_name::@device_func<%cfg>(%0) : !cuda.launch_cfg, ... -> !cuda.result + + // If calling with an Op that implements a FunctionOpInterface + %res = cuda.launch_ex @module_name::@device_func<%cfg>(%0) : !cuda.launch_cfg, ... -> !cuda.result + + %res2 = cuda.result.succeeded(%res) + scf.if %res2 { + + } + else { + + } + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Variadic:$inputs, + SymbolRefAttr:$callee, + OptionalAttr:$arg_attrs, + OptionalAttr:$res_attrs, + OptionalAttr:$assume_kernel_attr + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_LaunchKernelOp : Cuda_Op<"launch_kernel", [AttrSizedOperandSegments]> { + let description = [{ + Operation to launch a kernel from a kernel pointer on the device. This launch API + directly follows the CUDA C++ runtime API, passing all kernel arguments as a pointer + to a struct. + + Example: + + ```mlir + %res2 = cuda.launch_kernel %kernel, (%one32, %one32, %one32), (%one32, %one32, %one32), %zero64, %stream, %zeroPtr : !ptr.ptr<#ptr.generic_space> -> !cuda.result + + ``` + }]; + let arguments = (ins + Cuda_CUDAKernelHandle2:$kernel, + Variadic:$grid, + Variadic:$block, + Cuda_T64BitSignlessInteger:$dynamicSmem, + Cuda_CUDAStream:$cudaStream, + Cuda_PointerType:$kernel_args, + OptionalAttr:$arg_attrs, + OptionalAttr:$res_attrs + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_LibraryGetKernelOp : Cuda_Op<"library.get_kernel"> { + let description = [{ + Return a kernel handle object if found in the given library. + + ## Example + + ``` + %result_0, %kernel = cuda.library.get_kernel %l1, %name : !cuda.library, !ptr.ptr<#ptr.generic_space> -> !cuda.result, !cuda.kernel + ``` + + ## Behavior + + This op executes the CUDA API function cudaGetKernel. If the kernel is not found the cuda result will be `cudaErrorSymbolNotFound`. If the `kernel_name` is not a valid, null-terminated ASCII string, the behavior is UNDEFINED. Upon a result of `cudaSuccess`, `kernel` contains a valid kernel handle object. Is the result is not `cudaSuccess`, and use of the `kernel_name` handle is UNDEFINED. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$kernel_name + ); + let results = (outs + Cuda_CUDAResultType:$result, + Cuda_CUDAKernelHandle:$kernel + ); +} +def Cuda_LibraryLoadDataOp : Cuda_Op<"library.load_data"> { + let description = [{ + Load a library from a pointer. + + ## Example + + ``` + %result,%l1 = cuda.library.load_data %modptr : !ptr.ptr<#ptr.generic_space> -> !cuda.result, !cuda.library + ``` + ## Behavior + + This op execute the CUDA API function cudaLibraryLoadData. Whether the library is loaded immediately depends on the module load environment. + }]; + let arguments = (ins + Cuda_PointerType:$data + ); + let results = (outs + Cuda_CUDAResultType:$result, + Cuda_CUDALibraryHandle2:$library + ); +} +def Cuda_LibraryUnloadOp : Cuda_Op<"library.unload"> { + let arguments = (ins + Cuda_CUDALibraryHandle:$library + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_MemoryAllocOp : Cuda_Op<"memory.alloc"> { + let description = [{ + Operation to allocate device memory. Calls `cuMemAlloc()`. + + Examples: + ```mlir + + %bytes = arith.constant 0 : i64 + %res, %device_mem = cuda.memory.alloc(%bytes) : i64 -> cuda.result, cuda.ptr + + %res0 = cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%device_mem) + + %res1 = cuda.memory.free(%device_mem) : i64 -> cuda.result + ``` + }]; + let arguments = (ins + Cuda_T64BitSignlessInteger:$bytes + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$ptr + ); +} +def Cuda_MemoryAllocAsyncOp : Cuda_Op<"memory.alloc_async"> { + let description = [{ + Operation to allocate device memory asynchronously. Calls `cuMemAllocAsync()`. + + Examples: + ```mlir + + %res, %device_mem = cuda.memory.alloc_async(%bytes, %stream) : i64, cuda.ptr -> cuda.result + + cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%device_mem) + + $res1 = cuda.memory.free_async(%device_mem, %stream) : i64, !cuda.stream -> cuda.result + ``` + }]; + let arguments = (ins + Cuda_T64BitSignlessInteger:$bytes, + Cuda_CUDAStream:$stream + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$ptr + ); +} +def Cuda_MemoryAllocHostOp : Cuda_Op<"memory.alloc_host"> { + let description = [{ + Operation to allocate host memory. Calls `cuMemAllocHost()`. + + Examples: + ```mlir + + %bytes = arith.constant 0 : i64 + %host_mem, %res = cuda.memory.alloc_host(%bytes) : (i64) -> (cuda.ptr, cuda.result) + + %res0 = cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%host_mem) + + %res1 = cuda.memory.free_host(%host_mem) + ``` + }]; + let arguments = (ins + Cuda_T64BitSignlessInteger:$bytes + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$ptr + ); +} +def Cuda_MemoryAllocManagedOp : Cuda_Op<"memory.alloc_managed"> { + let description = [{ + Operation to allocate managed memory. + + Examples: + ```mlir + + %bytes = arith.constant 0 : i64 + %managed_mem, %res = cuda.memory.alloc_managed(%bytes) : (i64) -> (cuda.ptr, cuda.result) + + %res0 = cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%managed_mem) + + %res1 = cuda.memory.free(%managed_mem) + ``` + }]; + let arguments = (ins + Cuda_T64BitSignlessInteger:$bytes + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$ptr + ); +} +def Cuda_MemoryCopyOp : Cuda_Op<"memory.copy"> { + let description = [{ + Operation to copy memory between host and device. + + Examples: + ```mlir + + %bytes = arith.constant 1024 : i64 + + %device_mem, %res = cuda.memory.alloc %bytes : i64 -> (!cuda.result, !ptr.ptr) + + %res = cuda.memory.copy %device_mem, %host_mem, %bytes [HostToDevice] + + %res = cuda.launch @module::@kernel<<< (%gridX, %gridY, %gridZ), (%blockX, %blockY, %blockZ), %smem, %stream >>>(%device_mem) + + %res = cuda.memory.copy %host_mem, %device_mem, %bytes [DeviceToHost] + + %res = cuda.memory.free %device_mem : !ptr.ptr -> !cuda.result + ``` + }]; + let arguments = (ins + Cuda_PointerType:$src, + Cuda_PointerType:$dst, + Cuda_T64BitSignlessInteger:$bytes, + Cuda_MemcpyKindEnum:$kind + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_MemoryCopyAsyncOp : Cuda_Op<"memory.copy_async"> { + let description = [{ + Operation to copy memory between host and device asynchronously. + + Examples: + ```mlir + + %bytes = arith.constant 1024 : i64 + + %device_mem, %res = cuda.memory.alloc %bytes : i64 -> (!cuda.result, !ptr.ptr) + + %res = cuda.memory.copy_async %device_mem, %host_mem, %bytes [HostToDevice] + + %res = cuda.launch @module::@kernel<<< (%gridX, %gridY, %gridZ), (%blockX, %blockY, %blockZ), %smem, %stream >>>(%device_mem) + + %res = cuda.memory.copy_async %host_mem, %device_mem, %bytes [DeviceToHost] + + %res = cuda.memory.free %device_mem : !ptr.ptr -> !cuda.result + ``` + }]; + let arguments = (ins + Cuda_PointerType:$src, + Cuda_PointerType:$dst, + Cuda_T64BitSignlessInteger:$bytes, + Optional:$stream, + Cuda_MemcpyKindEnum:$kind + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_MemoryFreeOp : Cuda_Op<"memory.free"> { + let description = [{ + Operation to free allocated device memory. + + Examples: + ```mlir + + %bytes = arith.constant 0 : i64 + %res, %device_mem = cuda.memory.alloc(%bytes) : i64 -> cuda.result, cuda.ptr + + %res0 = cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%device_mem) + + %res1 = cuda.memory.free(%device_mem) : i64 -> cuda.result + ``` + }]; + let arguments = (ins + Cuda_PointerType:$ptr + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_MemoryFreeAsyncOp : Cuda_Op<"memory.free_async"> { + let description = [{ + Operation to free allocated device memory asynchronously. + + Examples: + ```mlir + + %res, %device_mem = cuda.memory.alloc_async(%bytes, %stream) : i64, cuda.ptr -> cuda.result + + cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%device_mem) + + $res1 = cuda.memory.free_async(%device_mem, %stream) : i64, !cuda.stream -> cuda.result + ``` + + Preserves semantics of [cuMemAllocAsync](https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MALLOC__ASYNC.html#group__CUDA__MALLOC__ASYNC_1g13413273e84a641bce1929eae9e6501f) + }]; + let arguments = (ins + Cuda_PointerType:$ptr, + Cuda_CUDAStream:$stream + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_MemoryFreeHostOp : Cuda_Op<"memory.free_host"> { + let description = [{ + Operation to free allocated host memory. + + Examples: + ```mlir + + %bytes = arith.constant 0 : i64 + %host_mem, %res = cuda.memory.alloc_host(%bytes) : (i64) -> (cuda.ptr, cuda.result) + + %res0 = cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%host_mem) + + %res1 = cuda.memory.free_host(%host_mem) + ``` + }]; + let arguments = (ins + Cuda_PointerType:$ptr + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_MemorySetOp : Cuda_Op<"memory.set"> { + let description = [{ + Operation to fill memory with a uniform value. + }]; + let arguments = (ins + Cuda_PointerType:$ptr, + Cuda_T32BitSignlessInteger:$value, + Cuda_T64BitSignlessInteger:$bytes + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_MemorySetAsyncOp : Cuda_Op<"memory.set_async"> { + let description = [{ + Operation to fill memory with a uniform value asynchronously. + }]; + let arguments = (ins + Cuda_PointerType:$ptr, + Cuda_T32BitSignlessInteger:$value, + Cuda_T64BitSignlessInteger:$bytes, + Optional:$stream + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_OccupancyMaxActiveBlocks : Cuda_Op<"occupancy_max_active_blocks_per_multiprocessor"> { + let description = [{ + Returns in `num_blocks` the maximum number of active blocks per streaming multiplrocessor for the device function. + + Example: + + ```mlir + %one64 = arith.constant 1 : i64 + %one32 = arith.constant 1 : i32 + %res, %num_blocks = cuda.occupancy_max_active_blocks_per_multiprocessor @funcname, %one32, %one64 : (i32, i64) -> (!cuda.result, i32) + ``` + }]; + let arguments = (ins + Cuda_CudaIntIsAIntegerType:$block_size, + Cuda_T64BitSignlessInteger:$dynamic_smem_size, + SymbolRefAttr:$func + ); + let results = (outs + Cuda_CUDAResultType:$res, + Cuda_T32BitSignlessInteger2:$num_blocks + ); +} +def Cuda_ProfilerStart : Cuda_Op<"profiler.start"> { + let description = [{ + Starts the CUDA profiler. + }]; + let arguments = (ins); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_ProfilerStop : Cuda_Op<"profiler.stop"> { + let description = [{ + Stops the CUDA profiler. + }]; + let arguments = (ins); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_ResultAssertSuccessOp : Cuda_Op<"result.assert_success"> { + let description = [{ + Asserts that a `cuda.result` indicates success. + }]; + let arguments = (ins + Cuda_CUDAResultType2:$cuda_result + ); + let results = (outs); +} +def Cuda_ResultFailedOp : Cuda_Op<"result.failed"> { + let description = [{ + Determines if a `cuda.result` indicates failure. + }]; + let arguments = (ins); + let results = (outs); +} +def Cuda_ResultPrintOnErrorOp : Cuda_Op<"result.print_on_error"> { + let description = [{ + Prints a message on error. + }]; + let arguments = (ins + Cuda_CUDAResultType2:$cuda_result + ); + let results = (outs); +} +def Cuda_ResultSucceededOp : Cuda_Op<"result.succeeded"> { + let description = [{ + Determines if a `cuda.result` indicates success. + }]; + let arguments = (ins + Cuda_CUDAResultType2:$cuda_result + ); + let results = (outs + Cuda_T1BitSignlessInteger:$success + ); +} +def Cuda_ReturnOp : Cuda_Op<"return", [Terminator]> { + let description = [{ + Returns a value from a `cuda.kernel` or `cuda.func` region. + + Note, the emitter seems to omit the dialect and simply prints `return` in the + custom assembly syntax. + + Examples: + + ```mlir + + cuda.module @module_name { + + cuda.func @device_func(%arg0 : i32) -> i32 { + %0 = arith.constant 0 : i32 + cuda.return %0 : i32 + } + + } + ``` + }]; + let arguments = (ins + Variadic:$inputs + ); + let results = (outs); +} +def Cuda_ReturnIfErrorOp : Cuda_Op<"return_if_error"> { + let description = [{ + The first operand must be the !cuda.result to check. + The subsequent operands are the values to be returned if an error exists. + This op is first lowered to an intermediate op `cuda.return_with_error`, + before final conversion to LLVM in a subsequent pass `cuda-finalize-error-handling`. + + Examples: + ```mlir + func.func @host_func() -> i32 { + %ten = arith.constant 10 : i32 + %cuda_result = cuda.launch @module::@kernel... + // Create an undefined/poison value for the error return path. + // Users can also input any arbitrary value for the error return path. + %poison = llvm.mlir.poison : i32 + // return poison if cuda_result is not cudaSuccess + %res_i32 = cuda.cast %cuda_result : !cuda.result -> i32 + cuda.return_if_error %res_i32, %poison : i32, i32 + + // return 10 if execution reaches here + return %ten : i32 + } + ``` + }]; + let arguments = (ins + Variadic:$inputs + ); + let results = (outs); +} +def Cuda_RuntimeGetDevicePropertiesOp : Cuda_Op<"runtime.device.properties"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_T32BitSignlessInteger:$device + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDACudaDevicePropStructTypeValueTyped:$prop + ); +} +def Cuda_RuntimeDevicePropGetAccessPolicyMaxWindowSizeOp : Cuda_Op<"runtime.device_prop.get_access_policy_max_window_size", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetAsyncEngineCountOp : Cuda_Op<"runtime.device_prop.get_async_engine_count", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetCanMapHostMemoryOp : Cuda_Op<"runtime.device_prop.get_can_map_host_memory", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetCanUseHostPointerForRegisteredMemOp : Cuda_Op<"runtime.device_prop.get_can_use_host_pointer_for_registered_mem", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetClusterLaunchOp : Cuda_Op<"runtime.device_prop.get_cluster_launch", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetComputePreemptionSupportedOp : Cuda_Op<"runtime.device_prop.get_compute_preemption_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetConcurrentKernelsOp : Cuda_Op<"runtime.device_prop.get_concurrent_kernels", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetConcurrentManagedAccessOp : Cuda_Op<"runtime.device_prop.get_concurrent_managed_access", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetCooperativeLaunchOp : Cuda_Op<"runtime.device_prop.get_cooperative_launch", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetDeferredMappingCudaArraySupportedOp : Cuda_Op<"runtime.device_prop.get_deferred_mapping_cuda_array_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetDeviceNumaConfigOp : Cuda_Op<"runtime.device_prop.get_device_numa_config", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetDeviceNumaIdOp : Cuda_Op<"runtime.device_prop.get_device_numa_id", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetDirectManagedMemAccessFromHostOp : Cuda_Op<"runtime.device_prop.get_direct_managed_mem_access_from_host", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetECCEnabledOp : Cuda_Op<"runtime.device_prop.get_ecc_enabled", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetGlobalL1CacheSupportedOp : Cuda_Op<"runtime.device_prop.get_global_l1cache_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetGpuDirectRDMAFlushWritesOptionsOp : Cuda_Op<"runtime.device_prop.get_gpu_direct_rdma_flush_writes_options", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetGpuDirectRDMASupportedOp : Cuda_Op<"runtime.device_prop.get_gpu_direct_rdma_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetGpuDirectRDMAWritesOrderingOp : Cuda_Op<"runtime.device_prop.get_gpu_direct_rdma_writes_ordering", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetGpuPciDeviceIDOp : Cuda_Op<"runtime.device_prop.get_gpu_pci_device_id", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetGpuPciSubsystemIDOp : Cuda_Op<"runtime.device_prop.get_gpu_pci_subsystem_id", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetHostNativeAtomicSupportedOp : Cuda_Op<"runtime.device_prop.get_host_native_atomic_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetHostNumaIdOp : Cuda_Op<"runtime.device_prop.get_host_numa_id", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetHostNumaMultinodeIpcSupportedOp : Cuda_Op<"runtime.device_prop.get_host_numa_multinode_ipc_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetHostRegisterReadOnlySupportedOp : Cuda_Op<"runtime.device_prop.get_host_register_read_only_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetHostRegisterSupportedOp : Cuda_Op<"runtime.device_prop.get_host_register_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetIntegratedOp : Cuda_Op<"runtime.device_prop.get_integrated", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetIpcEventSupportedOp : Cuda_Op<"runtime.device_prop.get_ipc_event_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetIsMultiGpuBoardOp : Cuda_Op<"runtime.device_prop.get_is_multi_gpu_board", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetL2CacheSizeOp : Cuda_Op<"runtime.device_prop.get_l_2cache_size", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetLocalL1CacheSupportedOp : Cuda_Op<"runtime.device_prop.get_local_l1cache_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetLuidOp : Cuda_Op<"runtime.device_prop.get_luid", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf8Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetLuidDeviceNodeMaskOp : Cuda_Op<"runtime.device_prop.get_luid_device_node_mask", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMajorOp : Cuda_Op<"runtime.device_prop.get_major", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetManagedMemoryOp : Cuda_Op<"runtime.device_prop.get_managed_memory", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxBlocksPerMultiProcessorOp : Cuda_Op<"runtime.device_prop.get_max_blocks_per_multi_processor", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxGridSizeOp : Cuda_Op<"runtime.device_prop.get_max_grid_size", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxSurface1DOp : Cuda_Op<"runtime.device_prop.get_max_surface_1d", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxSurface1DLayeredOp : Cuda_Op<"runtime.device_prop.get_max_surface_1d_layered", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxSurface2DOp : Cuda_Op<"runtime.device_prop.get_max_surface_2d", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxSurface2DLayeredOp : Cuda_Op<"runtime.device_prop.get_max_surface_2d_layered", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxSurface3DOp : Cuda_Op<"runtime.device_prop.get_max_surface_3d", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxSurfaceCubemapOp : Cuda_Op<"runtime.device_prop.get_max_surface_cubemap", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxSurfaceCubemapLayeredOp : Cuda_Op<"runtime.device_prop.get_max_surface_cubemap_layered", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture1DOp : Cuda_Op<"runtime.device_prop.get_max_texture_1d", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture1DLayeredOp : Cuda_Op<"runtime.device_prop.get_max_texture_1d_layered", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture1DMipmapOp : Cuda_Op<"runtime.device_prop.get_max_texture_1d_mipmap", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture2DOp : Cuda_Op<"runtime.device_prop.get_max_texture_2d", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture2DGatherOp : Cuda_Op<"runtime.device_prop.get_max_texture_2d_gather", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture2DLayeredOp : Cuda_Op<"runtime.device_prop.get_max_texture_2d_layered", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture2DLinearOp : Cuda_Op<"runtime.device_prop.get_max_texture_2d_linear", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture2DMipmapOp : Cuda_Op<"runtime.device_prop.get_max_texture_2d_mipmap", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture3DOp : Cuda_Op<"runtime.device_prop.get_max_texture_3d", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture3DAltOp : Cuda_Op<"runtime.device_prop.get_max_texture_3d_alt", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTextureCubemapOp : Cuda_Op<"runtime.device_prop.get_max_texture_cubemap", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTextureCubemapLayeredOp : Cuda_Op<"runtime.device_prop.get_max_texture_cubemap_layered", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxThreadsDimOp : Cuda_Op<"runtime.device_prop.get_max_threads_dim", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxThreadsPerBlockOp : Cuda_Op<"runtime.device_prop.get_max_threads_per_block", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxThreadsPerMultiProcessorOp : Cuda_Op<"runtime.device_prop.get_max_threads_per_multi_processor", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMemPitchOp : Cuda_Op<"runtime.device_prop.get_mem_pitch", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMemoryBusWidthOp : Cuda_Op<"runtime.device_prop.get_memory_bus_width", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMemoryPoolSupportedHandleTypesOp : Cuda_Op<"runtime.device_prop.get_memory_pool_supported_handle_types", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMemoryPoolsSupportedOp : Cuda_Op<"runtime.device_prop.get_memory_pools_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMinorOp : Cuda_Op<"runtime.device_prop.get_minor", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMpsEnabledOp : Cuda_Op<"runtime.device_prop.get_mps_enabled", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMultiGpuBoardGroupIDOp : Cuda_Op<"runtime.device_prop.get_multi_gpu_board_group_id", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMultiProcessorCountOp : Cuda_Op<"runtime.device_prop.get_multi_processor_count", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetNameOp : Cuda_Op<"runtime.device_prop.get_name", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf8Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetPageableMemoryAccessOp : Cuda_Op<"runtime.device_prop.get_pageable_memory_access", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetPageableMemoryAccessUsesHostPageTablesOp : Cuda_Op<"runtime.device_prop.get_pageable_memory_access_uses_host_page_tables", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetPciBusIDOp : Cuda_Op<"runtime.device_prop.get_pci_bus_id", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetPciDeviceIDOp : Cuda_Op<"runtime.device_prop.get_pci_device_id", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetPciDomainIDOp : Cuda_Op<"runtime.device_prop.get_pci_domain_id", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetPersistingL2CacheMaxSizeOp : Cuda_Op<"runtime.device_prop.get_persisting_l2cache_max_size", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetRegsPerBlockOp : Cuda_Op<"runtime.device_prop.get_regs_per_block", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetRegsPerMultiprocessorOp : Cuda_Op<"runtime.device_prop.get_regs_per_multiprocessor", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetReservedSharedMemPerBlockOp : Cuda_Op<"runtime.device_prop.get_reserved_shared_mem_per_block", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetSharedMemPerBlockOp : Cuda_Op<"runtime.device_prop.get_shared_mem_per_block", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetSharedMemPerBlockOptinOp : Cuda_Op<"runtime.device_prop.get_shared_mem_per_block_optin", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetSharedMemPerMultiprocessorOp : Cuda_Op<"runtime.device_prop.get_shared_mem_per_multiprocessor", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetSparseCudaArraySupportedOp : Cuda_Op<"runtime.device_prop.get_sparse_cuda_array_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetStreamPrioritiesSupportedOp : Cuda_Op<"runtime.device_prop.get_stream_priorities_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetSurfaceAlignmentOp : Cuda_Op<"runtime.device_prop.get_surface_alignment", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetTccDriverOp : Cuda_Op<"runtime.device_prop.get_tcc_driver", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetTextureAlignmentOp : Cuda_Op<"runtime.device_prop.get_texture_alignment", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetTexturePitchAlignmentOp : Cuda_Op<"runtime.device_prop.get_texture_pitch_alignment", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetTimelineSemaphoreInteropSupportedOp : Cuda_Op<"runtime.device_prop.get_timeline_semaphore_interop_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetTotalConstMemOp : Cuda_Op<"runtime.device_prop.get_total_const_mem", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetTotalGlobalMemOp : Cuda_Op<"runtime.device_prop.get_total_global_mem", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetUnifiedAddressingOp : Cuda_Op<"runtime.device_prop.get_unified_addressing", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetUnifiedFunctionPointersOp : Cuda_Op<"runtime.device_prop.get_unified_function_pointers", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetUuidOp : Cuda_Op<"runtime.device_prop.get_uuid", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_CUDACudaUUIDTStructTypeValue:$result + ); +} +def Cuda_RuntimeDevicePropGetWarpSizeOp : Cuda_Op<"runtime.device_prop.get_warp_size", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDim3GetXOp : Cuda_Op<"runtime.dim_3.get_x", [Pure]> { + let arguments = (ins + Cuda_CUDADim3StructTypeValueTyped:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDim3GetYOp : Cuda_Op<"runtime.dim_3.get_y", [Pure]> { + let arguments = (ins + Cuda_CUDADim3StructTypeValueTyped:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDim3GetZOp : Cuda_Op<"runtime.dim_3.get_z", [Pure]> { + let arguments = (ins + Cuda_CUDADim3StructTypeValueTyped:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeEventCreateWithFlagsOp : Cuda_Op<"runtime.event.create_with_flags"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_T32BitSignlessInteger:$flags + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_EventRepresentsCUDAEvent:$event + ); +} +def Cuda_RuntimeEventQueryOp : Cuda_Op<"runtime.event.query", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$event + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_RuntimeEventRecordWithFlagsOp : Cuda_Op<"runtime.event.record_with_flags"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$event, + Cuda_CUDAStream:$stream, + Cuda_T32BitSignlessInteger:$flags + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_RuntimeEventSynchronizeOp : Cuda_Op<"runtime.event.synchronize"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$event + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_RuntimeExtentGetDepthOp : Cuda_Op<"runtime.extent.get_depth", [Pure]> { + let arguments = (ins + Cuda_CUDACudaExtentStructTypeValueTyped:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeExtentGetHeightOp : Cuda_Op<"runtime.extent.get_height", [Pure]> { + let arguments = (ins + Cuda_CUDACudaExtentStructTypeValueTyped:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeExtentGetWidthOp : Cuda_Op<"runtime.extent.get_width", [Pure]> { + let arguments = (ins + Cuda_CUDACudaExtentStructTypeValueTyped:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeGraphAddKernelNodeOp : Cuda_Op<"runtime.graph.add_kernel_node"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDAGraph:$graph, + Cuda_PointerType:$pDependencies, + Cuda_T64BitSignlessInteger:$numDependencies, + Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$pNodeParams + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDAGraphNodeHandle:$pGraphNode + ); +} +def Cuda_RuntimeGraphCreateOp : Cuda_Op<"runtime.graph.create"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_T32BitSignlessInteger:$flags + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDAGraph2:$pGraph + ); +} +def Cuda_RuntimeGraphKernelNodeGetAttributeOp : Cuda_Op<"runtime.graph.kernel_node_get_attribute", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDAGraphNodeHandle2:$hNode, + Cuda_T32BitSignlessInteger:$attr + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_VariantCooperativeOfCUDACudaLaunchAttributeValueUnion:$value_out + ); +} +def Cuda_RuntimeGraphKernelNodeSetAttributeOp : Cuda_Op<"runtime.graph.kernel_node_set_attribute"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDAGraphNodeHandle2:$hNode, + Cuda_T32BitSignlessInteger:$attr, + Cuda_VariantCooperativeOfCUDACudaLaunchAttributeValueUnion:$value + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_RuntimeKernelNodeParamsGetBlockDimOp : Cuda_Op<"runtime.kernel_node_params.get_block_dim", [Pure]> { + let arguments = (ins + Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_CUDADim3StructTypeValueTyped2:$result + ); +} +def Cuda_RuntimeKernelNodeParamsGetExtraOp : Cuda_Op<"runtime.kernel_node_params.get_extra", [Pure]> { + let arguments = (ins + Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_LLVMPointerType:$result + ); +} +def Cuda_RuntimeKernelNodeParamsGetFuncOp : Cuda_Op<"runtime.kernel_node_params.get_func", [Pure]> { + let arguments = (ins + Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_LLVMPointerType:$result + ); +} +def Cuda_RuntimeKernelNodeParamsGetGridDimOp : Cuda_Op<"runtime.kernel_node_params.get_grid_dim", [Pure]> { + let arguments = (ins + Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_CUDADim3StructTypeValueTyped2:$result + ); +} +def Cuda_RuntimeKernelNodeParamsGetKernelParamsOp : Cuda_Op<"runtime.kernel_node_params.get_kernel_params", [Pure]> { + let arguments = (ins + Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_LLVMPointerType:$result + ); +} +def Cuda_RuntimeKernelNodeParamsGetSharedMemBytesOp : Cuda_Op<"runtime.kernel_node_params.get_shared_mem_bytes", [Pure]> { + let arguments = (ins + Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeLibraryGetGlobalOp : Cuda_Op<"runtime.library.get_global", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$name_arg + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$dptr, + Cuda_T64BitSignlessInteger:$bytes + ); +} +def Cuda_RuntimeLibraryGetKernelOp : Cuda_Op<"runtime.library.get_kernel", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$name_arg + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDAKernelHandle:$pKernel + ); +} +def Cuda_RuntimeLibraryGetKernelCountOp : Cuda_Op<"runtime.library.get_kernel_count", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$lib + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_T32BitSignlessInteger2:$count + ); +} +def Cuda_RuntimeLibraryGetManagedOp : Cuda_Op<"runtime.library.get_managed", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$name_arg + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$dptr, + Cuda_T64BitSignlessInteger:$bytes + ); +} +def Cuda_RuntimeLibraryGetUnifiedFunctionOp : Cuda_Op<"runtime.library.get_unified_function", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$symbol + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$fptr + ); +} +def Cuda_RuntimeLibraryLoadDataOp : Cuda_Op<"runtime.library.load_data"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_PointerType:$code_arg, + Cuda_PointerType:$jitOptions, + Cuda_PointerType:$jitOptionsValues, + Cuda_T32BitSignlessInteger:$numJitOptions, + Cuda_PointerType:$libraryOptions, + Cuda_PointerType:$libraryOptionValues, + Cuda_T32BitSignlessInteger:$numLibraryOptions + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDALibraryHandle2:$library + ); +} +def Cuda_RuntimeLibraryLoadFromFileOp : Cuda_Op<"runtime.library.load_from_file"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_PointerType:$fileName, + Cuda_PointerType:$jitOptions, + Cuda_PointerType:$jitOptionsValues, + Cuda_T32BitSignlessInteger:$numJitOptions, + Cuda_PointerType:$libraryOptions, + Cuda_PointerType:$libraryOptionValues, + Cuda_T32BitSignlessInteger:$numLibraryOptions + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDALibraryHandle2:$library + ); +} +def Cuda_RuntimeLibraryUnloadOp : Cuda_Op<"runtime.library.unload"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_RuntimeMemcpy3DParamsGetDstArrayOp : Cuda_Op<"runtime.memcpy_3d_params.get_dst_array", [Pure]> { + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_PointerType:$result + ); +} +def Cuda_RuntimeMemcpy3DParamsGetDstPosOp : Cuda_Op<"runtime.memcpy_3d_params.get_dst_pos", [Pure]> { + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_CUDACudaPosStructTypeValueTyped:$result + ); +} +def Cuda_RuntimeMemcpy3DParamsGetDstPtrOp : Cuda_Op<"runtime.memcpy_3d_params.get_dst_ptr", [Pure]> { + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_CUDACudaPitchedPtrStructTypeValueTyped:$result + ); +} +def Cuda_RuntimeMemcpy3DParamsGetExtentOp : Cuda_Op<"runtime.memcpy_3d_params.get_extent", [Pure]> { + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_CUDACudaExtentStructTypeValueTyped2:$result + ); +} +def Cuda_RuntimeMemcpy3DParamsGetKindOp : Cuda_Op<"runtime.memcpy_3d_params.get_kind", [Pure]> { + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeMemcpy3DParamsGetSrcArrayOp : Cuda_Op<"runtime.memcpy_3d_params.get_src_array", [Pure]> { + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_PointerType:$result + ); +} +def Cuda_RuntimeMemcpy3DParamsGetSrcPosOp : Cuda_Op<"runtime.memcpy_3d_params.get_src_pos", [Pure]> { + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_CUDACudaPosStructTypeValueTyped:$result + ); +} +def Cuda_RuntimeMemcpy3DParamsGetSrcPtrOp : Cuda_Op<"runtime.memcpy_3d_params.get_src_ptr", [Pure]> { + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_CUDACudaPitchedPtrStructTypeValueTyped:$result + ); +} +def Cuda_RuntimeMemcpy3DOp : Cuda_Op<"runtime.memory.cpy3d"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$p + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_RuntimeMemcpy3DAsyncOp : Cuda_Op<"runtime.memory.cpy3d_async"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$p, + Cuda_CUDAStream:$stream + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_RuntimePitchedPtrGetPitchOp : Cuda_Op<"runtime.pitched_ptr.get_pitch", [Pure]> { + let arguments = (ins + Cuda_CUDACudaPitchedPtrStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimePitchedPtrGetPtrOp : Cuda_Op<"runtime.pitched_ptr.get_ptr", [Pure]> { + let arguments = (ins + Cuda_CUDACudaPitchedPtrStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_LLVMPointerType:$result + ); +} +def Cuda_RuntimePitchedPtrGetXsizeOp : Cuda_Op<"runtime.pitched_ptr.get_xsize", [Pure]> { + let arguments = (ins + Cuda_CUDACudaPitchedPtrStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimePitchedPtrGetYsizeOp : Cuda_Op<"runtime.pitched_ptr.get_ysize", [Pure]> { + let arguments = (ins + Cuda_CUDACudaPitchedPtrStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimePosGetXOp : Cuda_Op<"runtime.pos.get_x", [Pure]> { + let arguments = (ins + Cuda_CUDACudaPosStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimePosGetYOp : Cuda_Op<"runtime.pos.get_y", [Pure]> { + let arguments = (ins + Cuda_CUDACudaPosStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimePosGetZOp : Cuda_Op<"runtime.pos.get_z", [Pure]> { + let arguments = (ins + Cuda_CUDACudaPosStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeUuidGetBytesOp : Cuda_Op<"runtime.uuid.get_bytes", [Pure]> { + let arguments = (ins + Cuda_CUDACudaUUIDTStructTypeValue2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf8Bit3:$result + ); +} +def Cuda_StreamAddCallbackOp : Cuda_Op<"stream.add_callback"> { + let description = [{ + Enqueues a host-side callback on the given CUDA stream. The callback is provided + as a symbol to a `func.func` with ABI: + `func @cb(!ptr.ptr<#ptr.generic_space>, i32, !ptr.ptr<#ptr.generic_space>) -> ()` + i.e., (stream, status, userData) -> void. + }]; + let arguments = (ins + Cuda_CUDAStream:$stream, + Cuda_PointerType:$user_data, + FlatSymbolRefAttr:$callback + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_StreamBeginCaptureOp : Cuda_Op<"stream.begin_capture"> { + let description = [{ + Operation to begin capturing a CUDA stream. + }]; + let arguments = (ins + Cuda_CUDAStream:$stream, + Cuda_StreamCaptureModeEnumEnum:$mode + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_StreamCreateOp : Cuda_Op<"stream.create"> { + let description = [{ + Operation to create a CUDA stream. + }]; + let arguments = (ins); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDAStream2:$stream + ); +} +def Cuda_StreamDestroyOp : Cuda_Op<"stream.destroy"> { + let description = [{ + Operation to destroy a CUDA stream. + }]; + let arguments = (ins + Cuda_CUDAStream:$stream + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_StreamEndCaptureOp : Cuda_Op<"stream.end_capture"> { + let description = [{ + Operation to end capturing a CUDA stream. + }]; + let arguments = (ins + Cuda_CUDAStream:$stream + ); + let results = (outs + Cuda_CUDAResultType:$result, + Cuda_CUDAGraph2:$graph + ); +} +def Cuda_StreamSynchronizeOp : Cuda_Op<"stream.synchronize", [MemoryEffects<[MemRead]>]> { + let description = [{ + Operation to synchronize with a CUDA Stream. + }]; + let arguments = (ins + Cuda_CUDAStream:$stream, + Optional:$event + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_TensorMapEncodeTiledOp : Cuda_Op<"tensor_map_encode_tiled"> { + let description = [{ + Creates a descriptor for Tensor Memory Access (TMA) object specified by the parameters + describing a tiled region and returns it in tensorMap. + + This operation corresponds to the CUDA Driver API function: + `cuTensorMapEncodeTiled` + + Tensor map objects are only supported on devices of compute capability 9.0 or higher. + The tensorMap address must be aligned to 64 bytes. + + All parameters are provided as runtime SSA values. + }]; + let arguments = (ins + Cuda_T32BitSignlessInteger:$tensorDataType, + Cuda_T32BitSignlessInteger:$tensorRank, + Cuda_PointerType:$globalAddress, + Cuda_LLVMPointerInAddressSpace0:$globalDims, + Cuda_LLVMPointerInAddressSpace0:$globalStrides, + Cuda_LLVMPointerInAddressSpace0:$boxDims, + Cuda_LLVMPointerInAddressSpace0:$elementStrides, + Cuda_T32BitSignlessInteger:$interleave, + Cuda_T32BitSignlessInteger:$swizzle, + Cuda_T32BitSignlessInteger:$l2Promotion, + Cuda_T32BitSignlessInteger:$oobFill + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDATensorMapType:$cuTensorMap + ); +} + +#endif // CUTE_IR_GENERATED_CUDA_OPS_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaTypes.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaTypes.td new file mode 100644 index 00000000..6dc97d51 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaTypes.td @@ -0,0 +1,212 @@ +//===- CudaTypes.td - CUDA host types of the CuTe DSL -----*- tablegen -*-===// +// +// Generated for nvidia-cutlass-dsl 4.8.0 from the DSL compiler (its typed +// Python type classes, its mnemonics, `_cutlass_ir.cu13` parser messages and +// the verifier's wording for each type); do not edit by hand. +// +// Every cuda type the compiler's parser knows. Only `launch_cfg` has a +// parameter; the C++ class names not given by the typed Python classes +// (`!cuda.int`, the graph handles, `arch_set`, `kernel`, `module` and the +// launch attribute values) follow the mnemonic. +// +//===----------------------------------------------------------------------===// + +#ifndef CUTE_IR_DIALECT_CUDA_IR_CUDA_TYPES_TD +#define CUTE_IR_DIALECT_CUDA_IR_CUDA_TYPES_TD + +include "cute_ir/Dialect/Cuda/IR/CudaDialect.td" + +def Cuda_ArchSetType : Cuda_Type<"ArchSet", "arch_set"> { + let summary = "`!cuda.arch_set`"; + let description = "`!cuda.arch_set`"; + let assemblyFormat = ""; +} + +def Cuda_EventType : Cuda_Type<"Event", "event"> { + let summary = "`Event` represents CUDA event"; + let description = "`!cuda.event`"; + let assemblyFormat = ""; +} + +def Cuda_GraphType : Cuda_Type<"Graph", "graph"> { + let summary = "CUDA Graph"; + let description = "`!cuda.graph`"; + let assemblyFormat = ""; +} + +def Cuda_GraphDeviceNodeType : Cuda_Type<"GraphDeviceNode", "graph_device_node"> { + let summary = "`GraphDeviceNode` represents CUDA device node handle for device-side node update"; + let description = "`!cuda.graph_device_node`"; + let assemblyFormat = ""; +} + +def Cuda_GraphExecType : Cuda_Type<"GraphExec", "graph_exec"> { + let summary = "CUDA Graph Execution"; + let description = "`!cuda.graph_exec`"; + let assemblyFormat = ""; +} + +def Cuda_GraphNodeType : Cuda_Type<"GraphNode", "graph_node"> { + let summary = "CUDA Graph Node handle"; + let description = "`!cuda.graph_node`"; + let assemblyFormat = ""; +} + +def Cuda_IntType : Cuda_Type<"Int", "int"> { + let summary = "`cuda.int` is a integer type with a bitwidth matching the underlying ABI integer bitwidth"; + let description = "`!cuda.int`"; + let assemblyFormat = ""; +} + +def Cuda_KernelType : Cuda_Type<"Kernel", "kernel"> { + let summary = "CUDA Kernel handle"; + let description = "`!cuda.kernel`"; + let assemblyFormat = ""; +} + +def Cuda_LaunchConfigType : Cuda_Type<"LaunchConfig", "launch_cfg"> { + let summary = "Representation of CUDA's extensible launch configuration mirroring cuda runtime API"; + let description = "`!cuda.launch_cfg`"; + let parameters = (ins "uint32_t":$maxNumAttrs); + let assemblyFormat = "`<` `max_attrs` `=` $maxNumAttrs `>`"; +} + +def Cuda_LibraryType : Cuda_Type<"Library", "library"> { + let summary = "CUDA Library handle"; + let description = "`!cuda.library`"; + let assemblyFormat = ""; +} + +def Cuda_ModuleType : Cuda_Type<"Module", "module"> { + let summary = "CUDA Module handle"; + let description = "`!cuda.module`"; + let assemblyFormat = ""; +} + +def Cuda_ResultType : Cuda_Type<"Result", "result"> { + let summary = "CUDA Result type"; + let description = "`!cuda.result`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeDevicePropType : Cuda_Type<"RuntimeDeviceProp", "runtime.device_prop"> { + let summary = "CUDA cudaDeviceProp struct type (value-typed)."; + let description = "`!cuda.runtime.device_prop`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeDim3Type : Cuda_Type<"RuntimeDim3", "runtime.dim_3"> { + let summary = "CUDA dim3 struct type (value-typed)."; + let description = "`!cuda.runtime.dim_3`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeExtentType : Cuda_Type<"RuntimeExtent", "runtime.extent"> { + let summary = "CUDA cudaExtent struct type (value-typed)."; + let description = "`!cuda.runtime.extent`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeKernelNodeParamsType : Cuda_Type<"RuntimeKernelNodeParams", "runtime.kernel_node_params"> { + let summary = "CUDA cudaKernelNodeParams struct type (value-typed)."; + let description = "`!cuda.runtime.kernel_node_params`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType : Cuda_Type<"RuntimeLaunchAttributeValueClusterSchedulingPolicyPreference", "runtime.launch_attribute_value.cluster_scheduling_policy_preference"> { + let summary = "`!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference`"; + let description = "`!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValueCooperativeType : Cuda_Type<"RuntimeLaunchAttributeValueCooperative", "runtime.launch_attribute_value.cooperative"> { + let summary = "`!cuda.runtime.launch_attribute_value.cooperative`"; + let description = "`!cuda.runtime.launch_attribute_value.cooperative`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValueMemSyncDomainType : Cuda_Type<"RuntimeLaunchAttributeValueMemSyncDomain", "runtime.launch_attribute_value.mem_sync_domain"> { + let summary = "`!cuda.runtime.launch_attribute_value.mem_sync_domain`"; + let description = "`!cuda.runtime.launch_attribute_value.mem_sync_domain`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType : Cuda_Type<"RuntimeLaunchAttributeValueNvlinkUtilCentricScheduling", "runtime.launch_attribute_value.nvlink_util_centric_scheduling"> { + let summary = "`!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling`"; + let description = "`!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValuePortableClusterSizeModeType : Cuda_Type<"RuntimeLaunchAttributeValuePortableClusterSizeMode", "runtime.launch_attribute_value.portable_cluster_size_mode"> { + let summary = "`!cuda.runtime.launch_attribute_value.portable_cluster_size_mode`"; + let description = "`!cuda.runtime.launch_attribute_value.portable_cluster_size_mode`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValuePriorityType : Cuda_Type<"RuntimeLaunchAttributeValuePriority", "runtime.launch_attribute_value.priority"> { + let summary = "`!cuda.runtime.launch_attribute_value.priority`"; + let description = "`!cuda.runtime.launch_attribute_value.priority`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType : Cuda_Type<"RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowed", "runtime.launch_attribute_value.programmatic_stream_serialization_allowed"> { + let summary = "`!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed`"; + let description = "`!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValueSharedMemCarveoutType : Cuda_Type<"RuntimeLaunchAttributeValueSharedMemCarveout", "runtime.launch_attribute_value.shared_mem_carveout"> { + let summary = "`!cuda.runtime.launch_attribute_value.shared_mem_carveout`"; + let description = "`!cuda.runtime.launch_attribute_value.shared_mem_carveout`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValueSharedMemoryModeType : Cuda_Type<"RuntimeLaunchAttributeValueSharedMemoryMode", "runtime.launch_attribute_value.shared_memory_mode"> { + let summary = "`!cuda.runtime.launch_attribute_value.shared_memory_mode`"; + let description = "`!cuda.runtime.launch_attribute_value.shared_memory_mode`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValueSyncPolicyType : Cuda_Type<"RuntimeLaunchAttributeValueSyncPolicy", "runtime.launch_attribute_value.sync_policy"> { + let summary = "`!cuda.runtime.launch_attribute_value.sync_policy`"; + let description = "`!cuda.runtime.launch_attribute_value.sync_policy`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeMemcpy3DParamsType : Cuda_Type<"RuntimeMemcpy3DParams", "runtime.memcpy_3d_params"> { + let summary = "CUDA cudaMemcpy3DParms struct type (value-typed)."; + let description = "`!cuda.runtime.memcpy_3d_params`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimePitchedPtrType : Cuda_Type<"RuntimePitchedPtr", "runtime.pitched_ptr"> { + let summary = "CUDA cudaPitchedPtr struct type (value-typed)."; + let description = "`!cuda.runtime.pitched_ptr`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimePosType : Cuda_Type<"RuntimePos", "runtime.pos"> { + let summary = "CUDA cudaPos struct type (value-typed)."; + let description = "`!cuda.runtime.pos`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeUuidType : Cuda_Type<"RuntimeUuid", "runtime.uuid"> { + let summary = "CUDA cudaUUID_t struct type (value-typed)."; + let description = "`!cuda.runtime.uuid`"; + let assemblyFormat = ""; +} + +def Cuda_StreamType : Cuda_Type<"Stream", "stream"> { + let summary = "CUDA Stream"; + let description = "`!cuda.stream`"; + let assemblyFormat = ""; +} + +def Cuda_TensorMapType : Cuda_Type<"TensorMap", "tensor_map"> { + let summary = "CUDA Tensor Map type"; + let description = "`!cuda.tensor_map`"; + let assemblyFormat = ""; +} + +#endif // CUTE_IR_DIALECT_CUDA_IR_CUDA_TYPES_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/BUILD.bazel b/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/BUILD.bazel new file mode 100644 index 00000000..bcb3ad97 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/BUILD.bazel @@ -0,0 +1,117 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:defs.bzl", "cc_library") + +package(default_visibility = ["//visibility:public"]) + +td_library( + name = "td_files", + srcs = glob(["*.td"]), + includes = ["/mlir/dialects/cute_ir/include"], + deps = [ + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "dialect_inc_gen", + includes = ["../../../.."], + tbl_outs = [ + ( + [ + "-gen-dialect-decls", + "-dialect=nvvm", + ], + "NVVMDialect.h.inc", + ), + ( + [ + "-gen-dialect-defs", + "-dialect=nvvm", + ], + "NVVMDialect.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "NVVMDialect.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "enums_inc_gen", + includes = ["../../../.."], + tbl_outs = [ + ( + ["-gen-enum-decls"], + "NVVMEnums.h.inc", + ), + ( + ["-gen-enum-defs"], + "NVVMEnums.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "NVVMEnums.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "attrs_inc_gen", + includes = ["../../../.."], + tbl_outs = [ + ( + [ + "-gen-attrdef-decls", + "-attrdefs-dialect=nvvm", + ], + "NVVMAttrs.h.inc", + ), + ( + [ + "-gen-attrdef-defs", + "-attrdefs-dialect=nvvm", + ], + "NVVMAttrs.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "NVVMAttrs.td", + deps = [":td_files"], +) + +# Every nvvm operation of the compiler's registry. +gentbl_cc_library( + name = "ops_inc_gen", + includes = ["../../../.."], + tbl_outs = [ + ( + ["-gen-op-decls"], + "NVVMOps.h.inc", + ), + ( + ["-gen-op-defs"], + "NVVMOps.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "NVVMOps.td", + deps = [":td_files"], +) + +cc_library( + name = "headers", + hdrs = ["NVVMDialect.h"], + includes = ["../../../.."], + deps = [ + ":attrs_inc_gen", + ":dialect_inc_gen", + ":enums_inc_gen", + ":ops_inc_gen", + "@llvm-project//mlir:BytecodeOpInterface", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:SideEffectInterfaces", + ], +) diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMAttrs.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMAttrs.td new file mode 100644 index 00000000..f6ee1f61 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMAttrs.td @@ -0,0 +1,52 @@ +//===- NVVMAttrs.td - NVVM attributes of the CuTe DSL ------*- tablegen -*-===// +// +// Generated for nvidia-cutlass-dsl 4.8.0 from the DSL compiler +// (`_cutlass_ir.cu13` parser messages, its mnemonics, and probing its +// attribute parser and printer); do not edit by hand. +// +// The enum attributes are NVVMEnums.td. These are the compiler's other nvvm +// attributes: every mnemonic its parser knows is one of the two. +// +//===----------------------------------------------------------------------===// + +#ifndef CUTE_IR_DIALECT_NVVM_IR_NVVM_ATTRS_TD +#define CUTE_IR_DIALECT_NVVM_IR_NVVM_ATTRS_TD + +include "cute_ir/Dialect/NVVM/IR/NVVMDialect.td" +include "cute_ir/Dialect/NVVM/IR/NVVMEnums.td" + +def NVVM_MMAShapeAttr : NVVM_Attr<"MMAShape", "shape"> { + let summary = "Shape of an MMA operation"; + let description = "`#nvvm.shape`"; + let parameters = (ins "int":$m, "int":$n, "int":$k); + let assemblyFormat = "`<` struct(params) `>`"; +} + +def NVVM_LdStMatrixShapeAttr : NVVM_Attr<"LdStMatrixShape", "ld_st_matrix_shape"> { + let summary = "Matrix shape of ldmatrix, stmatrix and movmatrix"; + let description = "`#nvvm.ld_st_matrix_shape`"; + let parameters = (ins "int":$m, "int":$n); + let assemblyFormat = "`<` struct(params) `>`"; +} + +// Defaulted fields print only when set; the defaults and the verifier's rules +// and messages are the compiler's. +def NVVM_TargetAttr : NVVM_Attr<"Target", "target"> { + let summary = "GPU target of an NVVM module"; + let description = [{ + `#nvvm.target` + }]; + let parameters = (ins + DefaultValuedParameter<"int", "2">:$O, + StringRefParameter<"target triple", "\"nvptx64-nvidia-cuda\"">:$triple, + StringRefParameter<"target chip", "\"sm_75\"">:$chip, + StringRefParameter<"target features", "\"\"">:$features, + OptionalParameter<"::mlir::DictionaryAttr">:$flags, + OptionalParameter<"::mlir::ArrayAttr">:$link, + DefaultValuedParameter<"bool", "false">:$verifyTarget + ); + let assemblyFormat = "(`<` struct(params)^ `>`)?"; + let genVerifyDecl = 1; +} + +#endif // CUTE_IR_DIALECT_NVVM_IR_NVVM_ATTRS_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMDialect.h b/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMDialect.h new file mode 100644 index 00000000..db64d49b --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMDialect.h @@ -0,0 +1,27 @@ +//===- NVVMDialect.h - NVIDIA's nvvm dialect of the CuTe DSL ------*- C++ -*-===// + +#ifndef CUTE_IR_DIALECT_NVVM_IR_NVVM_DIALECT_H +#define CUTE_IR_DIALECT_NVVM_IR_NVVM_DIALECT_H + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/Dialect/LLVMIR/LLVMAttrs.h" +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/Dialect/LLVMIR/LLVMTypes.h" +#include "mlir/IR/BuiltinAttributes.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/DialectImplementation.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/InferTypeOpInterface.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "cute_ir/Dialect/NVVM/IR/NVVMDialect.h.inc" + +#include "cute_ir/Dialect/NVVM/IR/NVVMEnums.h.inc" + +#define GET_ATTRDEF_CLASSES +#include "cute_ir/Dialect/NVVM/IR/NVVMAttrs.h.inc" + +#define GET_OP_CLASSES +#include "cute_ir/Dialect/NVVM/IR/NVVMOps.h.inc" + +#endif // CUTE_IR_DIALECT_NVVM_IR_NVVM_DIALECT_H diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMDialect.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMDialect.td new file mode 100644 index 00000000..59f5dfaa --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMDialect.td @@ -0,0 +1,33 @@ +//===- NVVMDialect.td - NVIDIA's NVVM dialect of the CuTe DSL -*- tablegen -*-===// + +#ifndef CUTE_IR_DIALECT_NVVM_IR_NVVM_DIALECT_TD +#define CUTE_IR_DIALECT_NVVM_IR_NVVM_DIALECT_TD + +include "mlir/IR/AttrTypeBase.td" +include "mlir/IR/DialectBase.td" +include "mlir/IR/OpBase.td" + +def NVVM_Dialect : Dialect { + let name = "nvvm"; + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let summary = "NVIDIA's NVVM dialect as the CuTe-DSL compiler registers it"; + let description = [{ + This dialect mirrors the operation, attribute and enum registry of the + `nvvm` dialect embedded in the nvidia-cutlass-dsl 4.8.0 compiler, built on + NVIDIA's own LLVM revision (23.0.0git 271b151610907). It differs from the + upstream NVVM dialect (`mlir::NVVM`) and is not meant to be registered + next to it: both are named `nvvm`. + }]; + let dependentDialects = ["::mlir::LLVM::LLVMDialect"]; + let useDefaultAttributePrinterParser = 1; +} + +class NVVM_Op traits = []> + : Op; + +class NVVM_Attr + : AttrDef { + let mnemonic = attrMnemonic; +} + +#endif // CUTE_IR_DIALECT_NVVM_IR_NVVM_DIALECT_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMEnums.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMEnums.td new file mode 100644 index 00000000..e5487b64 --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMEnums.td @@ -0,0 +1,1386 @@ +// The nvvm dialect's enums, generated from the nvidia-cutlass-dsl 4.8.0 enum +// bindings (`_nvvm_enum_gen.py`); do not edit. +// +// Case values and spellings are the compiler's. An enum the bindings build +// as a dialect attribute also has its EnumAttr, printed `#` +// or, where the compiler prints it so, `#nvvm`. + +#ifndef CUTE_IR_GENERATED_NVVM_ENUMS_TD +#define CUTE_IR_GENERATED_NVVM_ENUMS_TD + +include "cute_ir/Dialect/NVVM/IR/NVVMDialect.td" +include "mlir/IR/EnumAttr.td" + +def NVVM_AtomicOpKindEnum : I32EnumAttr<"AtomicOpKind", "operations supported by atom instruction", [ + I32EnumAttrCase<"AND", 0, "and">, + I32EnumAttrCase<"OR", 1, "or">, + I32EnumAttrCase<"XOR", 2, "xor">, + I32EnumAttrCase<"CAS", 3, "cas">, + I32EnumAttrCase<"EXCH", 4, "exch">, + I32EnumAttrCase<"ADD", 5, "add">, + I32EnumAttrCase<"INC", 6, "inc">, + I32EnumAttrCase<"DEC", 7, "dec">, + I32EnumAttrCase<"MIN", 8, "min">, + I32EnumAttrCase<"MAX", 9, "max">, + I32EnumAttrCase<"UMIN", 10, "umin">, + I32EnumAttrCase<"UMAX", 11, "umax">, + I32EnumAttrCase<"FADD", 12, "fadd">, + I32EnumAttrCase<"SUB", 13, "sub"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_BarrierReductionEnum : I32EnumAttr<"BarrierReduction", "NVVM barrier reduction operation", [ + I32EnumAttrCase<"POPC", 0, "popc">, + I32EnumAttrCase<"AND", 1, "and">, + I32EnumAttrCase<"OR", 2, "or"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_BarrierReduxKindEnum : I32EnumAttr<"BarrierReduxKind", "NVVM barrier redux kind", [ + I32EnumAttrCase<"AND", 0, "and">, + I32EnumAttrCase<"OR", 1, "or">, + I32EnumAttrCase<"POPC", 2, "popc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_BlockScaleFormatEnum : I32EnumAttr<"BlockScaleFormat", "MMA Block Scale Format", [ + I32EnumAttrCase<"UE8M0", 0, "ue8m0">, + I32EnumAttrCase<"UE4M3", 1, "ue4m3"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_CTAGroupKindEnum : I32EnumAttr<"CTAGroupKind", "NVVM CTA group kind", [ + I32EnumAttrCase<"CTA_1", 0, "cta_1">, + I32EnumAttrCase<"CTA_2", 1, "cta_2"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_CVTPackFloatKindEnum : I32EnumAttr<"CVTPackFloatKind", "NVVM CVT Pack Float kind", [ + I32EnumAttrCase<"F32", 0, "f32">, + I32EnumAttrCase<"F16x2", 1, "f16x2">, + I32EnumAttrCase<"E4M3x2", 2, "e4m3x2">, + I32EnumAttrCase<"E5M2x2", 3, "e5m2x2">, + I32EnumAttrCase<"BF16x2", 4, "bf16x2">, + I32EnumAttrCase<"E2M1x2", 5, "e2m1x2">, + I32EnumAttrCase<"E2M3x2", 6, "e2m3x2">, + I32EnumAttrCase<"E3M2x2", 7, "e3m2x2">, + I32EnumAttrCase<"UE8M0x2", 8, "ue8m0x2"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_CacheEvictionPriorityEnum : I32EnumAttr<"CacheEvictionPriority", "NVVM Cache Eviction Priority", [ + I32EnumAttrCase<"EvictNormal", 0, "evict_normal">, + I32EnumAttrCase<"EvictFirst", 1, "evict_first">, + I32EnumAttrCase<"EvictLast", 2, "evict_last">, + I32EnumAttrCase<"EvictUnchanged", 3, "evict_unchanged">, + I32EnumAttrCase<"NoAllocate", 4, "no_allocate"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ClusterLaunchControlQueryTypeEnum : I32EnumAttr<"ClusterLaunchControlQueryType", "NVVM ClusterLaunchControlQueryType", [ + I32EnumAttrCase<"IS_CANCELED", 0, "is_canceled">, + I32EnumAttrCase<"GET_FIRST_CTA_ID_X", 1, "get_first_cta_id_x">, + I32EnumAttrCase<"GET_FIRST_CTA_ID_Y", 2, "get_first_cta_id_y">, + I32EnumAttrCase<"GET_FIRST_CTA_ID_Z", 3, "get_first_cta_id_z"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_CompareOpKindEnum : I32EnumAttr<"CompareOpKind", "Comparison operator encoding", [ + I32EnumAttrCase<"EQ", 0, "eq">, + I32EnumAttrCase<"NE", 1, "ne">, + I32EnumAttrCase<"UGT", 2, "ugt">, + I32EnumAttrCase<"UGE", 3, "uge">, + I32EnumAttrCase<"ULT", 4, "ult">, + I32EnumAttrCase<"ULE", 5, "ule">, + I32EnumAttrCase<"SGT", 6, "sgt">, + I32EnumAttrCase<"SGE", 7, "sge">, + I32EnumAttrCase<"SLT", 8, "slt">, + I32EnumAttrCase<"SLE", 9, "sle"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ConvertFP4TypeEnum : I32EnumAttr<"ConvertFP4Type", "NVVM ConvertFP4Type kind", [ + I32EnumAttrCase<"E2M1", 0, "e2m1"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ConvertFP8TypeEnum : I32EnumAttr<"ConvertFP8Type", "NVVM ConvertFP8Type kind", [ + I32EnumAttrCase<"E4M3", 0, "e4m3">, + I32EnumAttrCase<"E5M2", 1, "e5m2">, + I32EnumAttrCase<"UE8M0", 2, "ue8m0">, + I32EnumAttrCase<"UE5M3", 3, "ue5m3"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ConvertScaleKindEnum : I32EnumAttr<"ConvertScaleKind", "NVVM ConvertScale kind", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"UE8M0", 1, "ue8m0"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_DotAccumulateTypeEnum : I32EnumAttr<"DotAccumulateType", "NVVM DotAccumulateType", [ + I32EnumAttrCase<"SIGNED", 1, "signed">, + I32EnumAttrCase<"UNSIGNED", 0, "unsigned"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_EvictKindEnum : I32EnumAttr<"EvictKind", "NVVM L2 Prefetch Size", [ + I32EnumAttrCase<"NORMAL", 0, "evict_normal">, + I32EnumAttrCase<"FIRST", 1, "evict_first">, + I32EnumAttrCase<"LAST", 2, "evict_last">, + I32EnumAttrCase<"UNCHANGED", 3, "evict_unchanged">, + I32EnumAttrCase<"NOALLOCATE", 4, "evict_no_allocate">, + I32EnumAttrCase<"NONE", 5, "none"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_FPRoundingModeEnum : I32EnumAttr<"FPRoundingMode", "NVVM FPRoundingMode kind", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"RN", 1, "rn">, + I32EnumAttrCase<"RM", 2, "rm">, + I32EnumAttrCase<"RP", 3, "rp">, + I32EnumAttrCase<"RZ", 4, "rz">, + I32EnumAttrCase<"RNA", 5, "rna">, + I32EnumAttrCase<"RS", 6, "rs"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_GridDepActionKindEnum : I32EnumAttr<"GridDepActionKind", "Action kind for grid dependency control", [ + I32EnumAttrCase<"wait", 0, "wait">, + I32EnumAttrCase<"launch_dependents", 1, "launch_dependents"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_IntegerRoundingModeEnum : I32EnumAttr<"IntegerRoundingMode", "NVVM IntegerRoundingMode kind", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"RNI", 1, "rni">, + I32EnumAttrCase<"RZI", 2, "rzi">, + I32EnumAttrCase<"RMI", 3, "rmi">, + I32EnumAttrCase<"RPI", 4, "rpi"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_L2PrefetchSizeEnum : I32EnumAttr<"L2PrefetchSize", "NVVM L2 Prefetch Size", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"RESERVED", 1, "reserved">, + I32EnumAttrCase<"SIZE_64B", 2, "size_64b">, + I32EnumAttrCase<"SIZE_128B", 3, "size_128b">, + I32EnumAttrCase<"SIZE_256B", 4, "size_256b"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_LdStMatrixEltTypeEnum : I32EnumAttr<"LdStMatrixEltType", "Element type for ldmatrix and stmatrix", [ + I32EnumAttrCase<"B16", 0, "b16">, + I32EnumAttrCase<"B8", 1, "b8">, + I32EnumAttrCase<"B8X16_B6X16_P32", 2, "b8x16.b6x16_p32">, + I32EnumAttrCase<"B8X16_B4X16_P64", 3, "b8x16.b4x16_p64"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_LoadCacheModifierExtKindEnum : I32EnumAttr<"LoadCacheModifierExtKind", "NVVM load cache modifier kind(Ext)", [ + I32EnumAttrCase<"CA", 0, "ca">, + I32EnumAttrCase<"CG", 1, "cg">, + I32EnumAttrCase<"CS", 2, "cs">, + I32EnumAttrCase<"LU", 3, "lu">, + I32EnumAttrCase<"CV", 4, "cv">, + I32EnumAttrCase<"NONE", 5, "none"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_LoadCacheModifierKindEnum : I32EnumAttr<"LoadCacheModifierKind", "NVVM load cache modifier kind", [ + I32EnumAttrCase<"CA", 0, "ca">, + I32EnumAttrCase<"CG", 1, "cg">, + I32EnumAttrCase<"CS", 2, "cs">, + I32EnumAttrCase<"LU", 3, "lu">, + I32EnumAttrCase<"CV", 4, "cv"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_LoadShapeEnum : I32EnumAttr<"LoadShape", "shape attribute for ldmatrix", [ + I32EnumAttrCase<"M8N8", 0, "m8n8">, + I32EnumAttrCase<"M8N16", 1, "m8n16">, + I32EnumAttrCase<"M16N16", 3, "m16n16"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_LoadSrcFormatEnum : I32EnumAttr<"LoadSrcFormat", "source format for ldmatrix", [ + I32EnumAttrCase<"B6x16_P32", 0, "b6x16_p32">, + I32EnumAttrCase<"B4x16_P64", 1, "b4x16_p64">, + I32EnumAttrCase<"B8", 2, "b8"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MBarrierLayoutEnum : I32EnumAttr<"MBarrierLayout", "NVVM MBarrier Layout", [ + I32EnumAttrCase<"V0", 0, "v0">, + I32EnumAttrCase<"V1", 1, "v1"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MBarrierPhaseEnum : I32EnumAttr<"MBarrierPhase", "NVVM mbarrier phase type", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"PRIMARY", 1, "primary">, + I32EnumAttrCase<"CONDITIONAL", 2, "conditional"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MBarrierScopeKindEnum : I32EnumAttr<"MBarrierScopeKind", "NVVM MBarrier scope kind", [ + I32EnumAttrCase<"CTA", 0, "cta">, + I32EnumAttrCase<"CLUSTER", 1, "cluster"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MBarrierSpaceKindEnum : I32EnumAttr<"MBarrierSpaceKind", "NVVM MBarrier space kind", [ + I32EnumAttrCase<"CTA", 0, "cta">, + I32EnumAttrCase<"CLUSTER", 1, "cluster"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MBarrierTxnKindEnum : I32EnumAttr<"MBarrierTxnKind", "NVVM MBarrier Transaction kind", [ + I32EnumAttrCase<"ARRIVE", 0, "arrive">, + I32EnumAttrCase<"ARRIVE_DROP", 1, "arrive_drop">, + I32EnumAttrCase<"ARRIVE_EXPECT_TX", 2, "arrive_expect_tx">, + I32EnumAttrCase<"ARRIVE_DROP_EXPECT_TX", 3, "arrive_drop_expect_tx">, + I32EnumAttrCase<"EXPECT_TX", 4, "expect_tx">, + I32EnumAttrCase<"COMPLETE_TX", 5, "complete_tx"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MBarrierWaitKindEnum : I32EnumAttr<"MBarrierWaitKind", "NVVM MBarrier wait kind", [ + I32EnumAttrCase<"TEST", 0, "test">, + I32EnumAttrCase<"TRY", 1, "try"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MMAB1OpEnum : I32EnumAttr<"MMAB1Op", "MMA binary operations", [ + I32EnumAttrCase<"none", 0, "none">, + I32EnumAttrCase<"xor_popc", 1, "xor_popc">, + I32EnumAttrCase<"and_popc", 2, "and_popc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MMABlockScaleKindEnum : I32EnumAttr<"MMABlockScaleKind", "Block Scale Kind", [ + I32EnumAttrCase<"MXF8F6F4", 0, "mxf8f6f4">, + I32EnumAttrCase<"MXF4", 1, "mxf4">, + I32EnumAttrCase<"MXF4NVF4", 2, "mxf4nvf4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MMACtaCountEnum : I32EnumAttr<"MMACtaCount", "MMA CTA count", [ + I32EnumAttrCase<"cta1", 1, "cta1">, + I32EnumAttrCase<"cta2", 2, "cta2"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MMAFragEnum : I32EnumAttr<"MMAFrag", "NVVM MMA frag type", [ + I32EnumAttrCase<"a", 0, "a">, + I32EnumAttrCase<"b", 1, "b">, + I32EnumAttrCase<"c", 2, "c"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MMAIntOverflowEnum : I32EnumAttr<"MMAIntOverflow", "MMA overflow options", [ + I32EnumAttrCase<"satfinite", 1, "satfinite">, + I32EnumAttrCase<"wrapped", 0, "wrapped"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MMAKindEnum : I32EnumAttr<"MMAKind", "MMA operation kind", [ + I32EnumAttrCase<"f8f6f4", 0, "f8f6f4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MMALayoutEnum : I32EnumAttr<"MMALayout", "NVVM MMA layout", [ + I32EnumAttrCase<"row", 0, "row">, + I32EnumAttrCase<"col", 1, "col"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MMATypesEnum : I32EnumAttr<"MMATypes", "NVVM MMA types", [ + I32EnumAttrCase<"f16", 0, "f16">, + I32EnumAttrCase<"f32", 1, "f32">, + I32EnumAttrCase<"tf32", 2, "tf32">, + I32EnumAttrCase<"bf16", 9, "bf16">, + I32EnumAttrCase<"s8", 4, "s8">, + I32EnumAttrCase<"u8", 3, "u8">, + I32EnumAttrCase<"s32", 5, "s32">, + I32EnumAttrCase<"s4", 8, "s4">, + I32EnumAttrCase<"u4", 7, "u4">, + I32EnumAttrCase<"b1", 6, "b1">, + I32EnumAttrCase<"f64", 10, "f64">, + I32EnumAttrCase<"e4m3", 11, "e4m3">, + I32EnumAttrCase<"e5m2", 12, "e5m2">, + I32EnumAttrCase<"e3m2", 13, "e3m2">, + I32EnumAttrCase<"e2m3", 14, "e2m3">, + I32EnumAttrCase<"e2m1", 15, "e2m1"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MatchSyncKindEnum : I32EnumAttr<"MatchSyncKind", "NVVM match sync kind", [ + I32EnumAttrCase<"any", 0, "any">, + I32EnumAttrCase<"all", 1, "all"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MemOrderKindEnum : I32EnumAttr<"MemOrderKind", "NVVM Memory Ordering kind", [ + I32EnumAttrCase<"WEAK", 0, "weak">, + I32EnumAttrCase<"RELAXED", 1, "relaxed">, + I32EnumAttrCase<"ACQUIRE", 2, "acquire">, + I32EnumAttrCase<"RELEASE", 3, "release">, + I32EnumAttrCase<"ACQ_REL", 4, "acq_rel">, + I32EnumAttrCase<"SC", 5, "sc">, + I32EnumAttrCase<"MMIO", 6, "mmio">, + I32EnumAttrCase<"VOLATILE", 8, "volatile">, + I32EnumAttrCase<"CONSTANT", 7, "constant"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MemScopeKindEnum : I32EnumAttr<"MemScopeKind", "NVVM Memory Scope kind", [ + I32EnumAttrCase<"CTA", 0, "cta">, + I32EnumAttrCase<"CLUSTER", 1, "cluster">, + I32EnumAttrCase<"GPU", 2, "gpu">, + I32EnumAttrCase<"SYS", 3, "sys"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MulModeEnum : I32EnumAttr<"MulMode", "multiply mode attribute", [ + I32EnumAttrCase<"HI", 0, "hi">, + I32EnumAttrCase<"LO", 1, "lo">, + I32EnumAttrCase<"WIDE", 2, "wide"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_NVVMMemorySpaceEnum : I32EnumAttr<"NVVMMemorySpace", "NVVM Memory Space", [ + I32EnumAttrCase<"Generic", 0, "generic">, + I32EnumAttrCase<"Global", 1, "global">, + I32EnumAttrCase<"Shared", 3, "shared">, + I32EnumAttrCase<"Constant", 4, "constant">, + I32EnumAttrCase<"Local", 5, "local">, + I32EnumAttrCase<"Tensor", 6, "tensor">, + I32EnumAttrCase<"SharedCluster", 7, "shared_cluster"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_PermuteModeEnum : I32EnumAttr<"PermuteMode", "NVVM permute mode", [ + I32EnumAttrCase<"DEFAULT", 0, "default">, + I32EnumAttrCase<"F4E", 1, "f4e">, + I32EnumAttrCase<"B4E", 2, "b4e">, + I32EnumAttrCase<"RC8", 3, "rc8">, + I32EnumAttrCase<"ECL", 4, "ecl">, + I32EnumAttrCase<"ECR", 5, "ecr">, + I32EnumAttrCase<"RC16", 6, "rc16"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_PrefetchCacheLevelEnum : I32EnumAttr<"PrefetchCacheLevel", "NVVM Prefetch Cache Level", [ + I32EnumAttrCase<"L1", 0, "L1">, + I32EnumAttrCase<"L2", 1, "L2"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ProxyKindEnum : I32EnumAttr<"ProxyKind", "Proxy kind", [ + I32EnumAttrCase<"alias", 0, "alias">, + I32EnumAttrCase<"async_", 1, "async">, + I32EnumAttrCase<"async_global", 2, "async.global">, + I32EnumAttrCase<"async_shared", 3, "async.shared">, + I32EnumAttrCase<"TENSORMAP", 4, "tensormap">, + I32EnumAttrCase<"GENERIC", 5, "generic"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ReductionKindEnum : I32EnumAttr<"ReductionKind", "NVVM Reduction Kind attribute", [ + I32EnumAttrCase<"ADD", 1, "add">, + I32EnumAttrCase<"AND", 2, "and">, + I32EnumAttrCase<"MAX", 3, "max">, + I32EnumAttrCase<"MIN", 4, "min">, + I32EnumAttrCase<"OR", 5, "or">, + I32EnumAttrCase<"UMAX", 6, "umax">, + I32EnumAttrCase<"UMIN", 7, "umin">, + I32EnumAttrCase<"XOR", 8, "xor">, + I32EnumAttrCase<"FMIN", 9, "fmin">, + I32EnumAttrCase<"FMAX", 10, "fmax"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ReductionOpEnum : I32EnumAttr<"ReductionOp", "Ops supported by red instruction", [ + I32EnumAttrCase<"AND", 0, "and">, + I32EnumAttrCase<"OR", 1, "or">, + I32EnumAttrCase<"XOR", 2, "xor">, + I32EnumAttrCase<"ADD", 3, "add">, + I32EnumAttrCase<"INC", 4, "inc">, + I32EnumAttrCase<"DEC", 5, "dec">, + I32EnumAttrCase<"MIN", 6, "min">, + I32EnumAttrCase<"MAX", 7, "max"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ReductionTypeEnum : I32EnumAttr<"ReductionType", "types supported by red instruction", [ + I32EnumAttrCase<"B32", 0, "b32">, + I32EnumAttrCase<"B64", 1, "b64">, + I32EnumAttrCase<"U32", 2, "u32">, + I32EnumAttrCase<"U64", 3, "u64">, + I32EnumAttrCase<"S32", 4, "s32">, + I32EnumAttrCase<"S64", 5, "s64">, + I32EnumAttrCase<"F32", 6, "f32">, + I32EnumAttrCase<"F64", 7, "f64">, + I32EnumAttrCase<"F16", 8, "f16">, + I32EnumAttrCase<"F16X2", 9, "f16x2">, + I32EnumAttrCase<"BF16", 10, "bf16">, + I32EnumAttrCase<"BF16X2", 11, "bf16x2"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPCompressElemSizeEnum : I32EnumAttr<"SPCompressElemSize", "Sparse tensor compression element size", [ + I32EnumAttrCase<"B4", 4, "b4">, + I32EnumAttrCase<"B8", 8, "b8">, + I32EnumAttrCase<"B16", 16, "b16"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPCompressFactorTypeEnum : I32EnumAttr<"SPCompressFactorType", "Sparse tensor compression factor type", [ + I32EnumAttrCase<"SP2TO4", 0, "sp2to4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPCompressIndexSizeEnum : I32EnumAttr<"SPCompressIndexSize", "Sparse tensor compression index size", [ + I32EnumAttrCase<"B2", 2, "b2">, + I32EnumAttrCase<"B4", 4, "b4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPCompressOpKindEnum : I32EnumAttr<"SPCompressOpKind", "spcompress operation kind", [ + I32EnumAttrCase<"MIN", 0, "min">, + I32EnumAttrCase<"MAX", 1, "max"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPCompressRepFactorEnum : I32EnumAttr<"SPCompressRepFactor", "Sparse tensor compression repetition factor", [ + I32EnumAttrCase<"X1", 1, "x1">, + I32EnumAttrCase<"X2", 2, "x2">, + I32EnumAttrCase<"X4", 4, "x4">, + I32EnumAttrCase<"X8", 8, "x8">, + I32EnumAttrCase<"X16", 16, "x16">, + I32EnumAttrCase<"X32", 32, "x32">, + I32EnumAttrCase<"X64", 64, "x64">, + I32EnumAttrCase<"X128", 128, "x128"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPDecompressElemSizeEnum : I32EnumAttr<"SPDecompressElemSize", "Sparse tensor decompression element size", [ + I32EnumAttrCase<"B4", 4, "b4">, + I32EnumAttrCase<"B8", 8, "b8">, + I32EnumAttrCase<"B16", 16, "b16"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPDecompressFactorTypeEnum : I32EnumAttr<"SPDecompressFactorType", "Sparse tensor decompression factor type", [ + I32EnumAttrCase<"SP1TO2", 0, "sp1to2">, + I32EnumAttrCase<"SP1TO4", 1, "sp1to4">, + I32EnumAttrCase<"SP1TO8", 2, "sp1to8">, + I32EnumAttrCase<"SP1TO16", 3, "sp1to16">, + I32EnumAttrCase<"SP2TO4", 4, "sp2to4">, + I32EnumAttrCase<"SP2TO8", 5, "sp2to8">, + I32EnumAttrCase<"SP2TO16", 6, "sp2to16">, + I32EnumAttrCase<"SP4TO8", 7, "sp4to8">, + I32EnumAttrCase<"SP4TO16", 8, "sp4to16">, + I32EnumAttrCase<"SP8TO16", 9, "sp8to16"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPDecompressIndexSizeEnum : I32EnumAttr<"SPDecompressIndexSize", "Sparse tensor decompression index size", [ + I32EnumAttrCase<"B2", 2, "b2">, + I32EnumAttrCase<"B4", 4, "b4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPDecompressRepFactorEnum : I32EnumAttr<"SPDecompressRepFactor", "Sparse tensor decompression repetition factor", [ + I32EnumAttrCase<"X1", 1, "x1">, + I32EnumAttrCase<"X2", 2, "x2">, + I32EnumAttrCase<"X4", 4, "x4">, + I32EnumAttrCase<"X8", 8, "x8">, + I32EnumAttrCase<"X16", 16, "x16">, + I32EnumAttrCase<"X32", 32, "x32">, + I32EnumAttrCase<"X64", 64, "x64"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SaturationModeEnum : I32EnumAttr<"SaturationMode", "NVVM SaturationMode kind", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"SATFINITE", 1, "satfinite">, + I32EnumAttrCase<"SAT", 2, "sat"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SaturationModeKindEnum : I32EnumAttr<"SaturationModeKind", "NVVM SaturationMode kind", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"SATFINITE", 1, "satfinite"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ScaleVecSizeEnum : I32EnumAttr<"ScaleVecSize", "MMA Scale Vector Sizes", [ + I32EnumAttrCase<"X1", 0, "x1">, + I32EnumAttrCase<"X2", 1, "x2">, + I32EnumAttrCase<"X4", 2, "x4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SetMaxRegisterActionEnum : I32EnumAttr<"SetMaxRegisterAction", "NVVM set max register action", [ + I32EnumAttrCase<"decrease", 1, "decrease">, + I32EnumAttrCase<"increase", 0, "increase"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SharedSpaceEnum : I32EnumAttr<"SharedSpace", "Shared memory space", [ + I32EnumAttrCase<"shared_cta", 0, "cta">, + I32EnumAttrCase<"shared_cluster", 1, "cluster"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ShflKindEnum : I32EnumAttr<"ShflKind", "NVVM shuffle kind", [ + I32EnumAttrCase<"bfly", 0, "bfly">, + I32EnumAttrCase<"up", 1, "up">, + I32EnumAttrCase<"down", 2, "down">, + I32EnumAttrCase<"idx", 3, "idx"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SparsityFormatEnum : I32EnumAttr<"SparsityFormat", "MMA Sparsity Format", [ + I32EnumAttrCase<"THREAD", 0, "thread"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_StateSpaceEnum : I32EnumAttr<"StateSpace", "NVVM State Space", [ + I32EnumAttrCase<"GENERIC", 0, "generic">, + I32EnumAttrCase<"GLOBAL", 1, "global">, + I32EnumAttrCase<"SHARED_CTA", 2, "shared_cta">, + I32EnumAttrCase<"SHARED_CLUSTER", 3, "shared_cluster">, + I32EnumAttrCase<"CONSTANT", 4, "constant">, + I32EnumAttrCase<"LOCAL", 5, "local">, + I32EnumAttrCase<"TENSOR", 6, "tensor"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_StoreCacheModifierKindEnum : I32EnumAttr<"StoreCacheModifierKind", "NVVM store cache modifier kind", [ + I32EnumAttrCase<"WB", 0, "wb">, + I32EnumAttrCase<"CG", 1, "cg">, + I32EnumAttrCase<"CS", 2, "cs">, + I32EnumAttrCase<"WT", 3, "wt">, + I32EnumAttrCase<"NONE", 4, "none"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_StoreShapeEnum : I32EnumAttr<"StoreShape", "shape attribute for stmatrix", [ + I32EnumAttrCase<"M8N8", 0, "m8n8">, + I32EnumAttrCase<"M16N8", 2, "m16n8"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TCBarParamEnum : I32EnumAttr<"TCBarParam", "Cluster MMA Barrier Parameter Type", [ + I32EnumAttrCase<"a1t0", 0, "a1t0">, + I32EnumAttrCase<"a0tx", 1, "a0tx"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TMALoadModeEnum : I32EnumAttr<"TMALoadMode", "NVVM TMA Load Mode", [ + I32EnumAttrCase<"TILE", 0, "tile">, + I32EnumAttrCase<"IM2COL", 1, "im2col">, + I32EnumAttrCase<"IM2COL_W", 2, "im2col_w">, + I32EnumAttrCase<"IM2COL_W_128", 3, "im2col_w_128">, + I32EnumAttrCase<"TILE_GATHER4", 4, "tile_gather4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TMAReduxKindEnum : I32EnumAttr<"TMAReduxKind", "NVVM TMA redux kind", [ + I32EnumAttrCase<"ADD", 0, "add">, + I32EnumAttrCase<"MAX", 2, "max">, + I32EnumAttrCase<"MIN", 1, "min">, + I32EnumAttrCase<"INC", 3, "inc">, + I32EnumAttrCase<"DEC", 4, "dec">, + I32EnumAttrCase<"AND", 5, "and">, + I32EnumAttrCase<"OR", 6, "or">, + I32EnumAttrCase<"XOR", 7, "xor"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TMAStoreModeEnum : I32EnumAttr<"TMAStoreMode", "NVVM TMA Store Mode", [ + I32EnumAttrCase<"TILE", 0, "tile">, + I32EnumAttrCase<"IM2COL", 1, "im2col">, + I32EnumAttrCase<"TILE_SCATTER4", 2, "tile_scatter4">, + I32EnumAttrCase<"IM2COL_W", 3, "im2col_w"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05CpMulticastEnum : I32EnumAttr<"Tcgen05CpMulticast", "tcgen05 cp multicast", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"WARPX2_02_13", 1, "warpx2_02_13">, + I32EnumAttrCase<"WARPX2_01_23", 2, "warpx2_01_23">, + I32EnumAttrCase<"WARPX4", 3, "warpx4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05CpShapeEnum : I32EnumAttr<"Tcgen05CpShape", "tcgen05 cp shapes", [ + I32EnumAttrCase<"SHAPE_128x256b", 0, "shape_128x256b">, + I32EnumAttrCase<"SHAPE_4x256b", 1, "shape_4x256b">, + I32EnumAttrCase<"SHAPE_128x128b", 2, "shape_128x128b">, + I32EnumAttrCase<"SHAPE_64x128b", 3, "shape_64x128b">, + I32EnumAttrCase<"SHAPE_32x128b", 4, "shape_32x128b"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05CpSrcFormatEnum : I32EnumAttr<"Tcgen05CpSrcFormat", "tcgen05 cp source format", [ + I32EnumAttrCase<"B6x16_P32", 0, "b6x16_p32">, + I32EnumAttrCase<"B4x16_P64", 1, "b4x16_p64"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05FenceKindEnum : I32EnumAttr<"Tcgen05FenceKind", "NVVM Tcgen05 fence kind", [ + I32EnumAttrCase<"BEFORE_THREAD_SYNC", 0, "before">, + I32EnumAttrCase<"AFTER_THREAD_SYNC", 1, "after"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05LdStShapeEnum : I32EnumAttr<"Tcgen05LdStShape", "allowed 32-bit signless integer cases: 0, 1, 2, 3, 4", [ + I32EnumAttrCase<"SHAPE_16X64B", 0, "shape_16x64b">, + I32EnumAttrCase<"SHAPE_16X128B", 1, "shape_16x128b">, + I32EnumAttrCase<"SHAPE_16X256B", 2, "shape_16x256b">, + I32EnumAttrCase<"SHAPE_32X32B", 3, "shape_32x32b">, + I32EnumAttrCase<"SHAPE_16X32BX2", 4, "shape_16x32bx2"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05MMABlockScaleEnum : I32EnumAttr<"Tcgen05MMABlockScale", "tcgen05.mma block scale attribute", [ + I32EnumAttrCase<"DEFAULT", 0, "default">, + I32EnumAttrCase<"BLOCK16", 1, "block16">, + I32EnumAttrCase<"BLOCK32", 2, "block32"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05MMACollectorBBufferEnum : I32EnumAttr<"Tcgen05MMACollectorBBuffer", "tcgen05 MMA Collector Buffer B Attribute", [ + I32EnumAttrCase<"B0", 0, "b0">, + I32EnumAttrCase<"B1", 1, "b1">, + I32EnumAttrCase<"B2", 2, "b2">, + I32EnumAttrCase<"B3", 3, "b3"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05MMACollectorOpEnum : I32EnumAttr<"Tcgen05MMACollectorOp", "tcgen05.mma Collector Buffer Operation", [ + I32EnumAttrCase<"DISCARD", 0, "discard">, + I32EnumAttrCase<"LASTUSE", 1, "lastuse">, + I32EnumAttrCase<"FILL", 2, "fill">, + I32EnumAttrCase<"USE", 3, "use"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05MMAKindEnum : I32EnumAttr<"Tcgen05MMAKind", "tcgen05 MMA Supported Types", [ + I32EnumAttrCase<"F16", 0, "f16">, + I32EnumAttrCase<"TF32", 1, "tf32">, + I32EnumAttrCase<"F8F6F4", 2, "f8f6f4">, + I32EnumAttrCase<"I8", 3, "i8">, + I32EnumAttrCase<"MXF8F6F4", 4, "mxf8f6f4">, + I32EnumAttrCase<"MXF4", 5, "mxf4">, + I32EnumAttrCase<"MXF4NVF4", 6, "mxf4nvf4">, + I32EnumAttrCase<"TI16", 7, "ti16"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05WaitKindEnum : I32EnumAttr<"Tcgen05WaitKind", "NVVM Tcgen05 wait kind", [ + I32EnumAttrCase<"LOAD", 0, "load">, + I32EnumAttrCase<"STORE", 1, "store"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TensormapElemtypeEnum : I32EnumAttr<"TensormapElemtype", "NVVM Tensormap Elemtype", [ + I32EnumAttrCase<"U8", 0, "u8">, + I32EnumAttrCase<"U16", 1, "u16">, + I32EnumAttrCase<"U32", 2, "u32">, + I32EnumAttrCase<"S32", 3, "s32">, + I32EnumAttrCase<"U64", 4, "u64">, + I32EnumAttrCase<"S64", 5, "s64">, + I32EnumAttrCase<"F16", 6, "f16">, + I32EnumAttrCase<"F32", 7, "f32">, + I32EnumAttrCase<"F32_FTZ", 8, "f32.ftz">, + I32EnumAttrCase<"F64", 9, "f64">, + I32EnumAttrCase<"BF16", 10, "bf16">, + I32EnumAttrCase<"TF32", 11, "tf32">, + I32EnumAttrCase<"TF32_FTZ", 12, "tf32.ftz">, + I32EnumAttrCase<"B4x16", 13, "b4x16">, + I32EnumAttrCase<"B4x16P64", 14, "b4x16_p64">, + I32EnumAttrCase<"B6x16P32", 15, "b6x16_p32"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TensormapFieldEnum : I32EnumAttr<"TensormapField", "NVVM Tensormap Field Kind", [ + I32EnumAttrCase<"GLOBAL_ADDRESS", 0, "global_address">, + I32EnumAttrCase<"RANK", 1, "rank">, + I32EnumAttrCase<"BOX_DIM", 2, "box_dim">, + I32EnumAttrCase<"GLOBAL_DIM", 3, "global_dim">, + I32EnumAttrCase<"GLOBAL_STRIDE", 4, "global_stride">, + I32EnumAttrCase<"ELEMENT_STRIDE", 5, "element_stride">, + I32EnumAttrCase<"ELEMTYPE", 6, "elemtype">, + I32EnumAttrCase<"INTERLEAVE_LAYOUT", 7, "interleave_layout">, + I32EnumAttrCase<"SWIZZLE_MODE", 8, "swizzle_mode">, + I32EnumAttrCase<"SWIZZLE_ATOMICITY", 9, "swizzle_atomicity">, + I32EnumAttrCase<"FILL_MODE", 10, "fill_mode"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TensormapFillModeEnum : I32EnumAttr<"TensormapFillMode", "NVVM Tensormap Fill Mode", [ + I32EnumAttrCase<"ZERO", 0, "zero">, + I32EnumAttrCase<"OOB_NAN", 1, "oob_nan"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TensormapInterleaveLayoutEnum : I32EnumAttr<"TensormapInterleaveLayout", "NVVM Tensormap Interleave Layout", [ + I32EnumAttrCase<"NO_INTERLEAVE", 0, "no_interleave">, + I32EnumAttrCase<"B16", 1, "b16">, + I32EnumAttrCase<"B32", 2, "b32"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TensormapSwizzleAtomicityEnum : I32EnumAttr<"TensormapSwizzleAtomicity", "NVVM Tensormap Swizzle Atomicity", [ + I32EnumAttrCase<"B16", 0, "b16">, + I32EnumAttrCase<"B32", 1, "b32">, + I32EnumAttrCase<"B32_FLIP_8B", 2, "b32_flip_b8">, + I32EnumAttrCase<"B64", 3, "b64"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TensormapSwizzleModeEnum : I32EnumAttr<"TensormapSwizzleMode", "NVVM Tensormap Swizzle Mode", [ + I32EnumAttrCase<"NO_SWIZZLING", 0, "no_swizzling">, + I32EnumAttrCase<"B32", 1, "b32">, + I32EnumAttrCase<"B64", 2, "b64">, + I32EnumAttrCase<"B128", 3, "b128">, + I32EnumAttrCase<"B96", 4, "b96"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TmemLayoutEnum : I32EnumAttr<"TmemLayout", "Tensor Memory Layout Enumerated Type", [ + I32EnumAttrCase<"tmem_16dp_128bit", 0, "tmem_16dp_128bit">, + I32EnumAttrCase<"tmem_16dp_256bit", 1, "tmem_16dp_256bit">, + I32EnumAttrCase<"tmem_32dp_32bit", 2, "tmem_32dp_32bit">, + I32EnumAttrCase<"tmem_16dp_64bit", 3, "tmem_16dp_64bit">, + I32EnumAttrCase<"tmem_16dp_32bit_t0_t15", 4, "tmem_16dp_32bit_t0_t15">, + I32EnumAttrCase<"tmem_16dp_32bit_t16_t31", 5, "tmem_16dp_32bit_t16_t31"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ValidatePatternEnum : I32EnumAttr<"ValidatePattern", "NVVM validate data pattern", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"PER_16BYTES_80000000", 1, "per_16bytes_80000000">, + I32EnumAttrCase<"PER_16BYTES_8000", 2, "per_16bytes_8000">, + I32EnumAttrCase<"PER_16BYTES_80", 3, "per_16bytes_80">, + I32EnumAttrCase<"PER_16BYTES_8", 4, "per_16bytes_8">, + I32EnumAttrCase<"PER_ELEMENT_FF", 5, "per_element_ff">, + I32EnumAttrCase<"PER_ELEMENT_80000000", 6, "per_element_80000000">, + I32EnumAttrCase<"PER_ELEMENT_8000", 7, "per_element_8000">, + I32EnumAttrCase<"PER_ELEMENT_80", 8, "per_element_80">, + I32EnumAttrCase<"PER_ELEMENT_8", 9, "per_element_8">, + I32EnumAttrCase<"PER_16BYTES_80000000_PAIRED", 10, "per_16bytes_80000000_paired"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_VoteSyncKindEnum : I32EnumAttr<"VoteSyncKind", "NVVM vote sync kind", [ + I32EnumAttrCase<"any", 0, "any">, + I32EnumAttrCase<"all", 1, "all">, + I32EnumAttrCase<"ballot", 2, "ballot">, + I32EnumAttrCase<"uni", 3, "uni"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_WGMMAScaleInEnum : I32EnumAttr<"WGMMAScaleIn", "WGMMA overflow options", [ + I32EnumAttrCase<"one", 1, "one">, + I32EnumAttrCase<"neg", 2, "neg"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_WGMMAScaleOutEnum : I32EnumAttr<"WGMMAScaleOut", "WGMMA input predicate", [ + I32EnumAttrCase<"zero", 0, "zero">, + I32EnumAttrCase<"one", 1, "one"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_WGMMATypesEnum : I32EnumAttr<"WGMMATypes", "NVVM WGMMA types", [ + I32EnumAttrCase<"f16", 0, "f16">, + I32EnumAttrCase<"tf32", 1, "tf32">, + I32EnumAttrCase<"u8", 2, "u8">, + I32EnumAttrCase<"s8", 3, "s8">, + I32EnumAttrCase<"b1", 4, "b1">, + I32EnumAttrCase<"bf16", 5, "bf16">, + I32EnumAttrCase<"e4m3", 6, "e4m3">, + I32EnumAttrCase<"e5m2", 7, "e5m2">, + I32EnumAttrCase<"f32", 8, "f32">, + I32EnumAttrCase<"s32", 9, "s32"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_AtomicOpKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_BarrierReductionAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_BarrierReduxKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_BlockScaleFormatAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_CTAGroupKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_CVTPackFloatKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_CacheEvictionPriorityAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_ClusterLaunchControlQueryTypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_CompareOpKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_ConvertFP4TypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_ConvertFP8TypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_ConvertScaleKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_DotAccumulateTypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_EvictKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_FPRoundingModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_GridDepActionAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_IntegerRoundingModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_L2PrefetchSizeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_LdStMatrixEltTypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_LoadCacheModifierAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_LoadCacheModifierAttrExt : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_LoadShapeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_LoadSrcFormatAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MBarrierLayoutAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MBarrierScopeKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MBarrierSpaceKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MBarrierTxnKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MBarrierWaitKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MMAB1OpAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MMABlockScaleKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MMACtaCountAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MMAFragAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MMAIntOverflowAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MMAKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MMALayoutAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MMATypesAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MatchSyncKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_MemOrderKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MemScopeKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MulModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_NVVMMemorySpaceAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_NVVM_MBarrierPhaseAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_PermuteModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_PrefetchCacheLevelAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_ProxyKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_ReductionKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_ReductionOpAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_ReductionTypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPCompressElemSizeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPCompressFactorTypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPCompressIndexSizeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPCompressOpKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPCompressRepFactorAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPDecompressElemSizeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPDecompressFactorTypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPDecompressIndexSizeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPDecompressRepFactorAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SaturationModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SaturationModeKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_ScaleVecSizeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SetMaxRegisterActionAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_SharedSpaceAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_ShflKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_SparsityFormatAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_StateSpaceAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_StoreCacheModifierAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_StoreShapeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TCBarParamEnumAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TMALoadModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TMAReduxKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TMAStoreModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05CpMulticastAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05CpShapeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05CpSrcFormatAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05FenceKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05LdStShapeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05MMABlockScaleAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05MMACollectorBBufferAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05MMACollectorOpAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05MMAKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05WaitKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TensormapElemtypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TensormapFieldAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_TensormapFillModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TensormapInterleaveLayoutAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TensormapSwizzleAtomicityAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TensormapSwizzleModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TmemLayoutEnumAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_ValidatePatternAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_VoteSyncKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_WGMMAScaleInAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_WGMMAScaleOutAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_WGMMATypesAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +#endif // CUTE_IR_GENERATED_NVVM_ENUMS_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMOps.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMOps.td new file mode 100644 index 00000000..908a2dff --- /dev/null +++ b/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMOps.td @@ -0,0 +1,6445 @@ +// The nvvm dialect's operations, generated from the nvidia-cutlass-dsl 4.8.0 release; do not edit. +// +// Sources: the operation bindings (`_nvvm_ops_gen.py`: names, classes, +// docstrings, operand/result/attribute structure, inference) and the live +// compiler (`_cutlass_ir` loaded from Python): each operand and result +// constraint carries the verifier's wording and accepts every type class the +// verifier let through when probed with one instance of every type; traits +// and interfaces are the registered operation's; memory effects come from +// loop-invariant code motion (hoisted: Pure) and CSE (merged: at most +// reads). Operations whose effects include writes or could not be told +// apart carry none (unknown effects). The compiler's own verifiers (operand +// combinations, shapes, instruction legality) are left to it. +// +// Operations keep the generic assembly form. +// +// Result types the compiler infers without a recovered rule (written types stand): +// nvvm.add.packed.f32x2 +// nvvm.atomicrmw +// nvvm.barrier +// nvvm.clusterlaunchcontrol.query.cancel +// nvvm.fma +// nvvm.fma.packed.f32x2 +// nvvm.ldmatrix +// nvvm.match.sync +// nvvm.mbarrier.arrive +// nvvm.mbarrier.arrive.expect_tx +// nvvm.mbarrier.arrive_drop +// nvvm.mbarrier.arrive_drop.expect_tx +// nvvm.mbarrier.test.wait +// nvvm.mbarrier.try_wait +// nvvm.mul (`mode = wide` doubles the operand width) +// nvvm.mul.packed.f32x2 +// nvvm.redux.sync +// nvvm.shfl.sync (`return_value_and_is_valid` returns `!llvm.struct<(T, i1)>`) +// nvvm.sub.packed.f32x2 +// nvvm.vote.sync +// +// Constraints not recovered: +// nvvm.convert.bf16x2.to.f8x2 `dstTy`: `fnv8E5M3FNU` has no type here and is not accepted + +#ifndef CUTE_IR_GENERATED_NVVM_OPS_TD +#define CUTE_IR_GENERATED_NVVM_OPS_TD + +include "cute_ir/Dialect/NVVM/IR/NVVMAttrs.td" +include "cute_ir/Dialect/NVVM/IR/NVVMEnums.td" + +include "mlir/Interfaces/InferTypeOpInterface.td" +include "mlir/Interfaces/SideEffectInterfaces.td" + +// Operand and result constraints, in the compiler's words. +def NVVM_AnyType : Type, "any type">; +def NVVM_FixedLengthVectorOf32Bit : Type($_self)">, "fixed-length vector of 32-bit signless integer values of length 4/8">; +def NVVM_FixedLengthVectorOf8Bit : Type($_self)">, "fixed-length vector of 8-bit signless integer values of length 4 or fixed-length vector of 16-bit signless integer values of length 2">; +def NVVM_FloatingPointOrFixedLengthVector : Type($_self) || ::llvm::isa<::mlir::VectorType>($_self)">, "floating-point or fixed-length vector of floating-point values of length 2">; +def NVVM_LLVMDialectCompatibleType : Type($_self) || ::llvm::isa<::mlir::Float16Type>($_self) || ::llvm::isa<::mlir::BFloat16Type>($_self) || ::llvm::isa<::mlir::Float32Type>($_self) || ::llvm::isa<::mlir::Float64Type>($_self) || ::llvm::isa<::mlir::VectorType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMPointerType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMStructType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMArrayType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMFunctionType>($_self)">, "LLVM dialect-compatible type">; +def NVVM_LLVMPointerInAddressSpace0 : Type($_self)">, "LLVM pointer in address space 0">; +def NVVM_LLVMPointerInAddressSpace02 : Type($_self)">, "LLVM pointer in address space 0 or LLVM pointer in address space 3">; +def NVVM_LLVMPointerInAddressSpace03 : Type($_self)">, "LLVM pointer in address space 0 or LLVM pointer in address space 7">; +def NVVM_LLVMPointerInAddressSpace04 : Type($_self)">, "LLVM pointer in address space 0 or LLVM pointer in address space 3 or LLVM pointer in address space 7">; +def NVVM_LLVMPointerInAddressSpace05 : Type($_self)">, "LLVM pointer in address space 0 or LLVM pointer in address space 3 or LLVM pointer in address space 1 or LLVM pointer in address space 7">; +def NVVM_LLVMPointerInAddressSpace1 : Type($_self)">, "LLVM pointer in address space 1">; +def NVVM_LLVMPointerInAddressSpace12 : Type($_self)">, "LLVM pointer in address space 1 or LLVM pointer in address space 0 or LLVM pointer in address space 3 or LLVM pointer in address space 7">; +def NVVM_LLVMPointerInAddressSpace13 : Type($_self)">, "LLVM pointer in address space 1 or LLVM pointer in address space 5 or LLVM pointer in address space 0 or LLVM pointer in address space 4">; +def NVVM_LLVMPointerInAddressSpace14 : Type($_self)">, "LLVM pointer in address space 1 or LLVM pointer in address space 3">; +def NVVM_LLVMPointerInAddressSpace3 : Type($_self)">, "LLVM pointer in address space 3">; +def NVVM_LLVMPointerInAddressSpace32 : Type($_self)">, "LLVM pointer in address space 3 or LLVM pointer in address space 7">; +def NVVM_LLVMPointerInAddressSpace6 : Type($_self)">, "LLVM pointer in address space 6">; +def NVVM_LLVMPointerInAddressSpace62 : Type($_self)">, "LLVM pointer in address space 6 or 64-bit signless integer">; +def NVVM_LLVMPointerInAddressSpace7 : Type($_self)">, "LLVM pointer in address space 7">; +def NVVM_LLVMPointerInAddressSpace72 : Type($_self)">, "LLVM pointer in address space 7 or LLVM pointer in address space 3">; +def NVVM_LLVMPointerType : Type($_self)">, "LLVM pointer type">; +def NVVM_LLVMPointerTypeOrLLVMPointer : Type($_self)">, "LLVM pointer type or LLVM pointer in address space 3">; +def NVVM_LLVMStructureType : Type($_self)">, "LLVM structure type">; +def NVVM_LLVMStructureTypeOr64Bit : Type($_self) || ::llvm::isa<::mlir::LLVM::LLVMStructType>($_self)">, "LLVM structure type or 64-bit float">; +def NVVM_LLVMTypeWithSize : Type($_self) || ::llvm::isa<::mlir::Float16Type>($_self) || ::llvm::isa<::mlir::BFloat16Type>($_self) || ::llvm::isa<::mlir::Float32Type>($_self) || ::llvm::isa<::mlir::Float64Type>($_self) || ::llvm::isa<::mlir::VectorType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMPointerType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMStructType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMArrayType>($_self)">, "LLVM type with size">; +def NVVM_T128BitSignlessInteger : Type, "128-bit signless integer">; +def NVVM_T16BitFloatOrBfloat16Type : Type($_self) || ::llvm::isa<::mlir::BFloat16Type>($_self) || ::llvm::isa<::mlir::Float32Type>($_self) || ::llvm::isa<::mlir::Float64Type>($_self) || ::llvm::isa<::mlir::VectorType>($_self)">, "16-bit float or bfloat16 type or 32-bit float or 64-bit float or vector of 16-bit float or bfloat16 type or 32-bit float or 64-bit float values of length 2">; +def NVVM_T16BitSignlessInteger : Type, "16-bit signless integer">; +def NVVM_T16BitSignlessInteger2 : Type, "16-bit signless integer">, BuildableType<"$_builder.getIntegerType(16)">; +def NVVM_T16BitSignlessIntegerOr32 : Type($_self) || ::llvm::isa<::mlir::BFloat16Type>($_self) || ::llvm::isa<::mlir::Float32Type>($_self) || ::llvm::isa<::mlir::Float64Type>($_self) || ::llvm::isa<::mlir::VectorType>($_self)">, "16-bit signless integer or 32-bit signless integer or 64-bit signless integer or 128-bit signless integer or 16-bit float or bfloat16 type or 32-bit float or 64-bit float or 128-bit float or vector of 32-bit float values of length 2/4 or vector of 16-bit float or bfloat16 type values of length 2/4/8">; +def NVVM_T16BitSignlessIntegerOr322 : Type, "16-bit signless integer or 32-bit signless integer">; +def NVVM_T16BitSignlessIntegerOr323 : Type, "16-bit signless integer or 32-bit signless integer or 64-bit signless integer">; +def NVVM_T16BitSignlessIntegerOrVector : Type($_self)">, "16-bit signless integer or vector of 8-bit signless integer values of length 2">; +def NVVM_T1BitSignlessInteger : Type, "1-bit signless integer">; +def NVVM_T1BitSignlessInteger2 : Type, "1-bit signless integer">, BuildableType<"$_builder.getIntegerType(1)">; +def NVVM_T1BitSignlessIntegerOr32 : Type, "1-bit signless integer or 32-bit signless integer">; +def NVVM_T1BitSignlessIntegerOrLLVM : Type($_self)">, "1-bit signless integer or LLVM structure type">; +def NVVM_T32BitFloat : Type($_self)">, "32-bit float">; +def NVVM_T32BitFloat2 : Type($_self)">, "32-bit float">, BuildableType<"::mlir::Float32Type::get($_builder.getContext())">; +def NVVM_T32BitFloatOr64Bit : Type($_self) || ::llvm::isa<::mlir::Float64Type>($_self)">, "32-bit float or 64-bit float">; +def NVVM_T32BitSignlessInteger : Type, "32-bit signless integer">; +def NVVM_T32BitSignlessInteger2 : Type, "32-bit signless integer">, BuildableType<"$_builder.getIntegerType(32)">; +def NVVM_T32BitSignlessIntegerOr1 : Type, "32-bit signless integer or 1-bit signless integer">; +def NVVM_T32BitSignlessIntegerOr32 : Type($_self)">, "32-bit signless integer or 32-bit float">; +def NVVM_T32BitSignlessIntegerOr322 : Type($_self) || ::llvm::isa<::mlir::LLVM::LLVMStructType>($_self)">, "32-bit signless integer or 32-bit float or LLVM struct type">; +def NVVM_T32BitSignlessIntegerOr64 : Type, "32-bit signless integer or 64-bit signless integer">; +def NVVM_T32BitSignlessIntegerOr642 : Type($_self) || ::llvm::isa<::mlir::BFloat16Type>($_self) || ::llvm::isa<::mlir::Float32Type>($_self) || ::llvm::isa<::mlir::Float64Type>($_self) || ::llvm::isa<::mlir::VectorType>($_self)">, "32-bit signless integer or 64-bit signless integer or 64-bit float or bfloat16 type or 16-bit float or 32-bit float or vector of 32-bit float or 32-bit signless integer values of length 2/4 or vector of 16-bit float or bfloat16 type values of length 2/4/8">; +def NVVM_T32BitSignlessIntegerOr643 : Type($_self)">, "32-bit signless integer or 64-bit signless integer or vector of 32-bit signless integer or 64-bit signless integer values of length 2/4">; +def NVVM_T32BitSignlessIntegerOrLLVM : Type($_self)">, "32-bit signless integer or LLVM struct type">; +def NVVM_T32BitSignlessIntegerOrVector : Type($_self)">, "32-bit signless integer or vector of 32-bit signless integer values of length 2/4/8/16/32/64">; +def NVVM_T32BitSignlessIntegerOrVector2 : Type($_self)">, "32-bit signless integer or vector of 32-bit signless integer values of length 2/4/8/16/32/64/128">; +def NVVM_T32BitSignlessIntegerOrVector3 : Type($_self)">, "32-bit signless integer or vector of 32-bit signless integer values of length 2/4">; +def NVVM_T64BitSignlessInteger : Type, "64-bit signless integer">; +def NVVM_T64BitSignlessInteger2 : Type, "64-bit signless integer">, BuildableType<"$_builder.getIntegerType(64)">; +def NVVM_T64BitSignlessIntegerOr32 : Type, "64-bit signless integer or 32-bit signless integer">; +def NVVM_T8BitSignlessInteger : Type, "8-bit signless integer">, BuildableType<"$_builder.getIntegerType(8)">; +def NVVM_T8BitSignlessInteger2 : Type, "8-bit signless integer">; +def NVVM_T8BitSignlessIntegerOr32 : Type, "8-bit signless integer or 32-bit signless integer">; +def NVVM_VariadicOf16BitSignlessInteger : Type, "variadic of 16-bit signless integer">; +def NVVM_VariadicOf32BitSignlessInteger : Type, "variadic of 32-bit signless integer">; +def NVVM_VariadicOfLLVMDialectCompatibleType : Type($_self) || ::llvm::isa<::mlir::Float16Type>($_self) || ::llvm::isa<::mlir::BFloat16Type>($_self) || ::llvm::isa<::mlir::Float32Type>($_self) || ::llvm::isa<::mlir::Float64Type>($_self) || ::llvm::isa<::mlir::VectorType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMPointerType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMStructType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMArrayType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMFunctionType>($_self)">, "variadic of LLVM dialect-compatible type">; +def NVVM_VectorOf16BitFloatValues : Type($_self)">, "vector of 16-bit float values of length 2">; +def NVVM_VectorOf16BitSignlessInteger : Type($_self)">, "vector of 16-bit signless integer values of length 2">; +def NVVM_VectorOf32BitFloatValues : Type($_self)">, "vector of 32-bit float values of length 2">; +def NVVM_VectorOf32BitFloatValues2 : Type($_self)">, "vector of 32-bit float values">; +def NVVM_VectorOf32BitFloatValues3 : Type($_self)">, "vector of 32-bit float values of length 4">; +def NVVM_VectorOf32BitSignlessInteger : Type($_self)">, "vector of 32-bit signless integer values of length 2/4/8/16/32/64/128">; +def NVVM_VectorOf32BitSignlessInteger2 : Type($_self)">, "vector of 32-bit signless integer values of length 2/4/8/16/32/64/128 or 32-bit signless integer">; +def NVVM_VectorOf32BitSignlessInteger3 : Type($_self)">, "vector of 32-bit signless integer or 32-bit float values of length 2/4/8/16/32/64/128">; +def NVVM_VectorOf32BitSignlessInteger4 : Type($_self)">, "vector of 32-bit signless integer values of length 2/4/8/16/32/64">; +def NVVM_VectorOf8BitSignlessInteger : Type($_self)">, "vector of 8-bit signless integer values of length 4">; +def NVVM_VectorOf8BitSignlessInteger2 : Type($_self)">, "vector of 8-bit signless integer values of length 2">; +def NVVM_VectorOfAnyTypeValues : Type($_self)">, "vector of any type values">; +def NVVM_VectorOfBfloat16TypeValuesOf : Type($_self)">, "vector of bfloat16 type values of length 2">; + +// Attribute constraints the bindings name but do not describe. +def NVVM_ConvertBF16x2ToF4x2DstTyAttr : Attr($_self) && (::llvm::isa<::mlir::Float4E2M1FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of f4E2M1FN type"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_ConvertBF16x2ToF6x2DstTyAttr : Attr($_self) && (::llvm::isa<::mlir::Float6E2M3FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float6E3M2FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of f6E2M3FN type or f6E3M2FN type"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_ConvertBF16x2ToF8x2DstTyAttr : Attr($_self) && (::llvm::isa<::mlir::Float8E8M0FNUType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float8E4M3FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float8E5M2Type>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of f8E8M0FNU type or f8E4M3FN type or f8E5M2 type or fnv8E5M3FNU type"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_ConvertF16x2ToF4x2DstTyAttr : Attr($_self) && (::llvm::isa<::mlir::Float4E2M1FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of f4E2M1FN type"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_ConvertF16x2ToF6x2DstTyAttr : Attr($_self) && (::llvm::isa<::mlir::Float6E2M3FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float6E3M2FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of f6E2M3FN type or f6E3M2FN type"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_FmaRndAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef<::mlir::cutlass_compiler::nvvm::FPRoundingMode>{::mlir::cutlass_compiler::nvvm::FPRoundingMode::NONE, ::mlir::cutlass_compiler::nvvm::FPRoundingMode::RN, ::mlir::cutlass_compiler::nvvm::FPRoundingMode::RM, ::mlir::cutlass_compiler::nvvm::FPRoundingMode::RP, ::mlir::cutlass_compiler::nvvm::FPRoundingMode::RZ}, ::llvm::cast<::mlir::cutlass_compiler::nvvm::FPRoundingModeAttr>($_self).getValue())">, "NVVM FPRoundingMode kind whose value is one of {none, rm, rn, rp, rz}"> { + let storageType = "::mlir::cutlass_compiler::nvvm::FPRoundingModeAttr"; + let returnType = "::mlir::cutlass_compiler::nvvm::FPRoundingModeAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_MultimemLdReduceOrderingAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef<::mlir::cutlass_compiler::nvvm::MemOrderKind>{::mlir::cutlass_compiler::nvvm::MemOrderKind::WEAK, ::mlir::cutlass_compiler::nvvm::MemOrderKind::RELAXED, ::mlir::cutlass_compiler::nvvm::MemOrderKind::ACQUIRE}, ::llvm::cast<::mlir::cutlass_compiler::nvvm::MemOrderKindAttr>($_self).getValue())">, "NVVM Memory Ordering kind whose value is one of {weak, relaxed, acquire}"> { + let storageType = "::mlir::cutlass_compiler::nvvm::MemOrderKindAttr"; + let returnType = "::mlir::cutlass_compiler::nvvm::MemOrderKindAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_MultimemLdReduceOpAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef<::mlir::cutlass_compiler::nvvm::ReductionOp>{::mlir::cutlass_compiler::nvvm::ReductionOp::AND, ::mlir::cutlass_compiler::nvvm::ReductionOp::OR, ::mlir::cutlass_compiler::nvvm::ReductionOp::XOR, ::mlir::cutlass_compiler::nvvm::ReductionOp::ADD, ::mlir::cutlass_compiler::nvvm::ReductionOp::MIN, ::mlir::cutlass_compiler::nvvm::ReductionOp::MAX}, ::llvm::cast<::mlir::cutlass_compiler::nvvm::ReductionOpAttr>($_self).getValue())">, "Ops supported by red instruction whose value is none of {inc, dec}"> { + let storageType = "::mlir::cutlass_compiler::nvvm::ReductionOpAttr"; + let returnType = "::mlir::cutlass_compiler::nvvm::ReductionOpAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_MultimemRedOrderingAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef<::mlir::cutlass_compiler::nvvm::MemOrderKind>{::mlir::cutlass_compiler::nvvm::MemOrderKind::RELAXED, ::mlir::cutlass_compiler::nvvm::MemOrderKind::RELEASE}, ::llvm::cast<::mlir::cutlass_compiler::nvvm::MemOrderKindAttr>($_self).getValue())">, "NVVM Memory Ordering kind whose value is one of {relaxed, release}"> { + let storageType = "::mlir::cutlass_compiler::nvvm::MemOrderKindAttr"; + let returnType = "::mlir::cutlass_compiler::nvvm::MemOrderKindAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_MultimemRedOpAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef<::mlir::cutlass_compiler::nvvm::ReductionOp>{::mlir::cutlass_compiler::nvvm::ReductionOp::AND, ::mlir::cutlass_compiler::nvvm::ReductionOp::OR, ::mlir::cutlass_compiler::nvvm::ReductionOp::XOR, ::mlir::cutlass_compiler::nvvm::ReductionOp::ADD, ::mlir::cutlass_compiler::nvvm::ReductionOp::INC, ::mlir::cutlass_compiler::nvvm::ReductionOp::DEC, ::mlir::cutlass_compiler::nvvm::ReductionOp::MIN, ::mlir::cutlass_compiler::nvvm::ReductionOp::MAX}, ::llvm::cast<::mlir::cutlass_compiler::nvvm::ReductionOpAttr>($_self).getValue())">, "ReductionOp attribute"> { + let storageType = "::mlir::cutlass_compiler::nvvm::ReductionOpAttr"; + let returnType = "::mlir::cutlass_compiler::nvvm::ReductionOpAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_MultimemStOrderingAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef<::mlir::cutlass_compiler::nvvm::MemOrderKind>{::mlir::cutlass_compiler::nvvm::MemOrderKind::WEAK, ::mlir::cutlass_compiler::nvvm::MemOrderKind::RELAXED, ::mlir::cutlass_compiler::nvvm::MemOrderKind::RELEASE}, ::llvm::cast<::mlir::cutlass_compiler::nvvm::MemOrderKindAttr>($_self).getValue())">, "NVVM Memory Ordering kind whose value is one of {weak, relaxed, release}"> { + let storageType = "::mlir::cutlass_compiler::nvvm::MemOrderKindAttr"; + let returnType = "::mlir::cutlass_compiler::nvvm::MemOrderKindAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_LLVMConstantRangeAttr : Attr($_self)">, "A range of two integers, corresponding to LLVM's ConstantRange"> { + let storageType = "::mlir::LLVM::ConstantRangeAttr"; + let returnType = "::mlir::LLVM::ConstantRangeAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_TensormapFieldValueAttr : Attr($_self)">, "NVVM Tensormap Elemtype or NVVM Tensormap Interleave Layout or NVVM Tensormap Swizzle Mode or NVVM Tensormap Swizzle Atomicity or NVVM Tensormap Fill Mode"> { + let storageType = "::mlir::Attribute"; + let returnType = "::mlir::Attribute"; + let convertFromStorage = "$_self"; +} + +// Operations whose result types are inferred in C++ (NVVMInference.cpp). +// Verification compares them with the inferred ones where the rule is +// recovered; elsewhere the types written stand, as the compiler accepted them. +class NVVM_InferredOp traits = []> + : NVVM_Op]> { + let extraClassDefinition = [{ + ::llvm::LogicalResult $cppClass::refineReturnTypes( + ::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, + ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, + ::mlir::PropertyRef properties, ::mlir::RegionRange regions, + ::llvm::SmallVectorImpl<::mlir::Type> &returnTypes) { + ::llvm::SmallVector<::mlir::Type, 4> inferred; + if (::mlir::failed(inferReturnTypes(context, ::std::nullopt, operands, + attributes, properties, regions, + inferred))) + return ::mlir::success(); + if (!isCompatibleReturnTypes(inferred, returnTypes)) + return ::mlir::emitOptionalError( + location, "'", getOperationName(), "' op inferred type(s) ", + inferred, " are incompatible with return type(s) of operation ", + returnTypes); + return ::mlir::success(); + } + }]; +} + +def NVVM_AddPacked_BF16x2_F32x2_F32x2Op : NVVM_Op<"add.packed.bf16x2.f32x2.f32x2"> { + let description = [{ + This Op performs the add operation on a pair of f32x2 vectors. + The inputs and output are of the form v2f32 and v2bf16 respectively. + The inputs are converted to a scalar of packed i64 where the 0th element of the + vector forms the low-32 bits and the 1st element occupies the high-32 bits. The round to zero + is the only rounding mode supported for this operation. + + Examples: + ```mlir + // Round to zero (only rounding mode supported) + %0 = nvvm.add .packed.bf16x2.f32x2.f32x2 %srcA, %srcB : vector<2xf32>, vector<2xf32> -> vector<2xbf16> + ``` + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB + ); + let results = (outs + NVVM_VectorOfBfloat16TypeValuesOf:$res + ); +} +def NVVM_AddPacked_F16x2_F32x2_F32x2Op : NVVM_Op<"add.packed.f16x2.f32x2.f32x2"> { + let description = [{ + This Op performs the add operation on a pair of f32x2 vectors. + The inputs and output are of the form v2f32 and v2f16 respectively. + The inputs are converted to a scalar of packed i64 where the 0th element of the + vector forms the low-32 bits and the 1st element occupies the high-32 bits. The round to zero + is the only rounding mode supported for this operation. + + Examples: + ```mlir + // Round to zero (only rounding mode supported) + %0 = nvvm.add .packed.f16x2.f32x2.f32x2 %srcA, %srcB : vector<2xf32>, vector<2xf32> -> vector<2xf16> + ``` + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB + ); + let results = (outs + NVVM_VectorOf16BitFloatValues:$res + ); +} +def NVVM_AddPackedF32x2Op : NVVM_InferredOp<"add.packed.f32x2"> { + let description = [{ + This Op performs addition of floating-point vectors. + The inputs and outputs are of the form . The `rnd` attribute + specifies the rounding mode. The `ftz` attribute, when set, flushes the + subnormal inputs and results to sign-preserving zero. + This Op is supported from SM100 and onwards. + [For more information, see PTX ISA] + (TODO: Add public PTX link when available) + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues2:$srcA, + NVVM_VectorOf32BitFloatValues2:$srcB, + OptionalAttr:$rnd, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_VectorOf32BitFloatValues2:$res + ); +} +def NVVM_AddPacked_F32x2_BF16x2_F32x2Op : NVVM_Op<"add.packed.f32x2.bf16x2.f32x2"> { + let description = [{ + This Op performs the add operation on a pair of floating-point values. + The inputs are of the form v2bf16 and v2f32. The first input `srcA` is + converted to a scalar of packed i32 where the 0th element of the vector forms + the low-16 bits and the 1st element occupies the high-16 bits. Similarly, + the second input `srcB` is converted to a scalar of packed i64 where the 0th + element of the vector forms the low-32 bits and the 1st element occupies the + high-32 bits. The `rnd` attribute specifies the rounding mode. + + The rounding mode can be one of the following: + - rn: round to nearest even + - rz: round toward zero + - rm: round toward negative infinity + - rp: round toward positive infinity + + Examples: + ```mlir + // Round to nearest even (default) + %0 = nvvm.add .packed.f32x2.bf16x2.f32x2 %srcA, %srcB : vector<2xbf16>, vector<2xf32> -> vector<2xf32> + + // Round toward negative infinity + %1 = nvvm.add .packed.f32x2.bf16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xbf16>, vector<2xf32> -> vector<2xf32> + + // Round toward zero with flush-to-zero + %2 = nvvm.add .packed.f32x2.bf16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xbf16>, vector<2xf32> -> vector<2xf32> + ``` + }]; + let arguments = (ins + NVVM_VectorOfBfloat16TypeValuesOf:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB, + OptionalAttr:$rnd + ); + let results = (outs + NVVM_VectorOf32BitFloatValues:$res + ); +} +def NVVM_AddPacked_F32x2_F16x2_F32x2Op : NVVM_Op<"add.packed.f32x2.f16x2.f32x2"> { + let description = [{ + This Op performs the add operation on a pair of floating-point values. + The inputs are of the form v2f16 and v2f32. The first input `srcA` is + converted to a scalar of packed i32 where the 0th element of the vector forms + the low-16 bits and the 1st element occupies the high-16 bits. Similarly, + the second input `srcB` is converted to a scalar of packed i64 where the 0th + element of the vector forms the low-32 bits and the 1st element occupies the + high-32 bits. The `rnd` attribute specifies the rounding mode. + + The rounding mode can be one of the following: + - rn: round to nearest even + - rz: round toward zero + - rm: round toward negative infinity + - rp: round toward positive infinity + + Examples: + ```mlir + // Round to nearest even (default) + %0 = nvvm.add .packed.f32x2.f16x2.f32x2 %srcA, %srcB : vector<2xf16>, vector<2xf32> -> vector<2xf32> + + // Round toward negative infinity + %1 = nvvm.add .packed.f32x2.f16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xf16>, vector<2xf32> -> vector<2xf32> + + // Round toward zero with flush-to-zero + %2 = nvvm.add .packed.f32x2.f16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xf16>, vector<2xf32> -> vector<2xf32> + ``` + }]; + let arguments = (ins + NVVM_VectorOf16BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB, + OptionalAttr:$rnd + ); + let results = (outs + NVVM_VectorOf32BitFloatValues:$res + ); +} +def NVVM_AddFOp : NVVM_InferredOp<"addf", [Pure]> { + let description = [{ + The `nvvm.addf` operation performs floating point addition of two floating + point operands of the same type. + + The rounding mode is specified by the `rnd` attribute, saturation mode by + the `sat` attribute, and flush-to-zero by the `ftz` attribute. + + For more information, see PTX ISA: + - [floating point addition](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-add) + - [half-precision floating point addition](https://docs.nvidia.com/cuda/parallel-thread-execution/#half-precision-floating-point-instructions-add) + }]; + let arguments = (ins + NVVM_T16BitFloatOrBfloat16Type:$lhs, + NVVM_T16BitFloatOrBfloat16Type:$rhs, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_T16BitFloatOrBfloat16Type:$res + ); +} +def NVVM_ApplyPriorityAsyncBulkOp : NVVM_Op<"applypriority.async.bulk"> { + let description = [{ + Sets the eviction priority to evict normal for cache lines within the specified + region of L2 cache that are previously present and marked as evict last. + Other L2 cache lines in the specified region are not affected. + + Arguments: + - `ptr`: Global memory pointer. + - `size`: Address range of length. + - `evict`: Eviction priority. + + Example: + ```mlir + nvvm.applypriority.async.bulk %ptr, %size evict_priority = evict_normal + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace1:$ptr, + NVVM_T32BitSignlessInteger:$size, + OptionalAttr:$evict + ); + let results = (outs); +} +def NVVM_ApplyPriorityAsyncBulkTensorOp : NVVM_Op<"applypriority.async.bulk.tensor", [AttrSizedOperandSegments]> { + let description = [{ + Sets the eviction priority to evict normal for cache lines within the + specified region of L2 cache that are previously present and marked as + evict last. Other L2 cache lines in the specified region are not affected. + + Arguments: + - `tmaDesc`: The TMA descriptor. + - `box/coordinates`: The coordinates of the tensor. + - `im2col`: The im2col arguments. + - `mode`: Tensor load mode. + - `evict`: Eviction priority. + + Example: + ```mlir + nvvm.applypriority.async.bulk.tensor %tma_desc, box [ %d0] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr + nvvm.applypriority.async.bulk.tensor %tma_desc, box [ %d0, %d1, %d2] im2col [ %im2col0] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr + nvvm.applypriority.async.bulk.tensor %tma_desc, box [ %d0, %d1, %d2] im2col [ %im2w_halo, %im2w_offset] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerType:$tmaDesc, + Variadic:$coordinates, + Variadic:$im2colArgs, + OptionalAttr:$mode, + OptionalAttr:$evict + ); + let results = (outs); +} +def NVVM_ApplyPriorityAsyncBulkTensorOverrideOp : NVVM_Op<"applypriority.async.bulk.tensor.override", [AttrSizedOperandSegments]> { + let description = [{ + Sets the eviction priority to evict normal for cache lines within the + specified region of L2 cache that are previously present and marked as + evict last. Other L2 cache lines in the specified region are not affected. + It supports a way to override the global address of the tensor. + + Arguments: + - `tmaDesc`: The TMA descriptor. + - `overrideAdrr`: The global address to override. + - `box/coordinates`: The coordinates of the tensor. + - `tensorSize`: The size of the tensor. + - `lowerStride`: The lower stride of the tensor. + - `upperStride`: The upper stride of the tensor. + - `im2col`: The im2col arguments. + - `mode`: Tensor load mode. + - `evict`: Eviction priority. + + Example: + ```mlir + nvvm.applypriority.async.bulk.tensor.override %tma_desc, %gbl_addr, box [ %d0] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr + nvvm.applypriority.async.bulk.tensor.override %tma_desc, %gbl_addr, box [ %d0, %d1, %d2] im2col [ %im2col0] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr + nvvm.applypriority.async.bulk.tensor.override %tma_desc, %gbl_addr, box [ %d0] tensor_size [ %ts0] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr + nvvm.applypriority.async.bulk.tensor.override %tma_desc, %gbl_addr, box [ %d0, %d1] tensor_size [ %ts0, %ts1] lower_stride [ %lstrd0] upper_stride [ %ustrd] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerType:$tmaDesc, + NVVM_LLVMPointerInAddressSpace1:$overrideAdrr, + Variadic:$coordinates, + Variadic:$tensorSize, + Variadic:$lowerStride, + Optional:$upperStride, + Variadic:$im2colArgs, + OptionalAttr:$mode, + OptionalAttr:$evict + ); + let results = (outs); +} +def NVVM_AtomicRMWOp : NVVM_InferredOp<"atomicrmw"> { + let description = [{ + Atomically loads the original value at location `a` into destination register `res`, + performs a reduction operation with operand `b` and the value in location `a`, and + stores the result of the specified operation at location `a`, overwriting the + original value. Operand `a` specifies a location in the specified state space. + If no state space is given, perform the memory accesses using `Generic Addressing`. + For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-atom) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace12:$ptr, + NVVM_T16BitSignlessIntegerOr32:$a, + Optional:$b, + NVVM_AtomicOpKindAttr:$op, + OptionalAttr:$memOrder, + OptionalAttr:$syncscope + ); + let results = (outs + NVVM_T16BitSignlessIntegerOr32:$res + ); +} +def NVVM_SyncWarpOp : NVVM_Op<"bar.warp.sync"> { + let description = [{ + The `nvvm.bar.warp.sync` operation performs barrier synchronization for threads + within a warp. + + This operation causes the executing thread to wait until all threads corresponding + to the `mask` operand have executed a `bar.warp.sync` with the same mask value + before resuming execution. + + The `mask` operand specifies the threads participating in the barrier, where each + bit position corresponds to the thread's lane ID within the warp. Only threads with + their corresponding bit set in the mask participate in the barrier synchronization. + + **Important constraints**: + - The behavior is undefined if the executing thread is not included in the mask + (i.e., the bit corresponding to the thread's lane ID is not set) + - For compute capability sm_6x or below, all threads in the mask must execute + the same `bar.warp.sync` instruction in convergence + + This operation also guarantees memory ordering among participating threads. + Threads within the warp that wish to communicate via memory can store to memory, + execute `bar.warp.sync`, and then safely read values stored by other threads + in the warp. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar-warp-sync) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$mask + ); + let results = (outs); +} +def NVVM_BarrierOp : NVVM_InferredOp<"barrier", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.barrier` operation performs barrier synchronization and communication + within a CTA (Cooperative Thread Array). It causes executing threads to wait for + all non-exited threads participating in the barrier to arrive. + + The operation takes two optional operands: + + - `barrierId`: Specifies a logical barrier resource with value 0 through 15. + Each CTA instance has sixteen barriers numbered 0..15. Defaults to 0 if not specified. + - `numberOfThreads`: Specifies the number of threads participating in the barrier. + When specified, the value must be a multiple of the warp size. If not specified, + all threads in the CTA participate in the barrier. + - `reductionOp`: specifies the reduction operation (`popc`, `and`, `or`). + - `reductionPredicate`: specifies the predicate to be used with the + `reductionOp`. + + The barrier operation guarantees that when the barrier completes, prior memory + accesses requested by participating threads are performed relative to all threads + participating in the barrier. It also ensures that no new memory access is + requested by participating threads before the barrier completes. + + When a barrier completes, the waiting threads are restarted without delay, and + the barrier is reinitialized so that it can be immediately reused. + + This operation generates an aligned barrier, indicating that all threads in the CTA + will execute the same barrier instruction. Behavior is undefined if all threads in the + CTA do not reach this instruction. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar) + }]; + let arguments = (ins + Optional:$barrierId, + Optional:$numberOfThreads, + Optional:$reductionPredicate, + OptionalAttr:$reductionOp + ); + let results = (outs + Optional:$res + ); +} +def NVVM_BarrierArriveOp : NVVM_Op<"barrier.arrive"> { + let description = [{ + Thread that executes this op announces their arrival at the barrier with + given id and continue their execution. + + The default barrier id is 0 that is similar to `nvvm.barrier` Op. When + `barrierId` is not present, the default barrier id is used. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar) + }]; + let arguments = (ins + Optional:$barrierId, + NVVM_T32BitSignlessInteger:$numberOfThreads + ); + let results = (outs); +} +def NVVM_BarrierCTAArriveOp : NVVM_Op<"barrier.cta.arrive"> { + let description = [{ + The `nvvm.barrier.cta.arrive` is used by the threads within the CTA for synchronization and communication. + The `barrierId` specifies a logical barrier resource and the `threadCount` specifies the number of threads + participating in the barrier. The `barrier.cta.arrive` does not cause the executing thread to wait for + threads of other participating warps. The `aligned` attribute, when set to 1, generates the .aligned + version of the PTX instruction. + [For more information, see PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar-barrier) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$barrierId, + NVVM_T32BitSignlessInteger:$threadCount, + OptionalAttr:$aligned + ); + let results = (outs); +} +def NVVM_BarrierCTAReduxOp : NVVM_Op<"barrier.cta.red"> { + let description = [{ + The `nvvm.barrier.cta.red` is used to perform a reduction operation across threads in the CTA. + The `pred` from all the participating threads are combined using the specified reduction operator. + AND, OR, POPC (population-count) are available as reduction operators. The `barrierId` specifies + a logical barrier resource and the `threadCount` specifies the number of threads participating + in the barrier. When the `threadCount` is not specified, all threads in the CTA participate in + the reduction operation. The `aligned` attribute, when set to 1, generates the .aligned version + of the PTX instruction. + [For more information, see PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar-barrier) + }]; + let arguments = (ins + NVVM_T1BitSignlessInteger:$pred, + NVVM_T32BitSignlessInteger:$barrierId, + Optional:$threadCount, + NVVM_BarrierReduxKindAttr:$kind, + OptionalAttr:$aligned + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$res + ); +} +def NVVM_BarrierCTASyncOp : NVVM_Op<"barrier.cta.sync"> { + let description = [{ + The `nvvm.barrier.cta.sync` is used by the threads within the CTA for synchronization and communication. + The `barrierId` specifies a logical barrier resource and the `threadCount` specifies the number of + threads participating in the barrier. When the `threadCount` is not specified, all threads in the CTA + participate in the barrier. The `aligned` attribute, when set to 1, generates the .aligned version of + the PTX instruction. + [For more information, see PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar-barrier) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$barrierId, + Optional:$threadCount, + OptionalAttr:$aligned + ); + let results = (outs); +} +def NVVM_Barrier0Op : NVVM_Op<"barrier0"> { + let description = [{ + The `nvvm.barrier0` operation is a convenience operation that performs barrier + synchronization and communication within a CTA (Cooperative Thread Array) using + barrier ID 0. It is functionally equivalent to `nvvm.barrier` or `nvvm.barrier id=0`. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar) + }]; + let arguments = (ins); + let results = (outs); +} +def NVVM_Breakpoint : NVVM_Op<"breakpoint"> { + let description = [{ + Breakpoint suspends execution of the program for debugging. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#miscellaneous-instructions-brkpt) + }]; + let arguments = (ins); + let results = (outs); +} +def NVVM_ClmadOp : NVVM_Op<"clmad", [Pure]> { + let description = [{ + Performs a carryless multiplication of `a` and `b`, followed by a carryless + addition of `c`. + When `hi` is true, the result is the upper 64 bits of the product, with the + addition of `c`. Otherwise, the result is the lower 64 bits of the product, + with the addition of `c`. + }]; + let arguments = (ins + NVVM_T64BitSignlessInteger:$a, + NVVM_T64BitSignlessInteger:$b, + NVVM_T64BitSignlessInteger:$c, + OptionalAttr:$high + ); + let results = (outs + NVVM_T64BitSignlessInteger2:$res + ); +} +def NVVM_ClusterArriveOp : NVVM_Op<"cluster.arrive"> { + let description = [{ + The `cluster.arrive` can be used by the threads within the cluster for synchronization and + communication. The `cluster.arrive` instruction marks the warps' arrival at the barrier + without causing the executing thread to wait for other participating threads. + + The `aligned` attribute, when provided, generates the .aligned version of the PTX instruction. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-barrier-cluster) + }]; + let arguments = (ins + UnitAttr:$aligned + ); + let results = (outs); +} +def NVVM_ClusterArriveRelaxedOp : NVVM_Op<"cluster.arrive.relaxed"> { + let description = [{ + The `cluster.arrive` can be used by the threads within the cluster for synchronization and + communication. The `cluster.arrive` instruction marks the warps' arrival at the barrier + without causing the executing thread to wait for other participating threads. + + The `aligned` attribute, when provided, generates the .aligned version of the PTX instruction. + The .relaxed qualifier on `cluster.arrive` specifies that there are no memory + ordering and visibility guarantees provided for the memory accesses performed prior to + `cluster.arrive`. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-barrier-cluster) + }]; + let arguments = (ins + UnitAttr:$aligned + ); + let results = (outs); +} +def NVVM_ClusterWaitOp : NVVM_Op<"cluster.wait"> { + let description = [{ + The `cluster.wait` causes the executing thread to wait for all non-exited threads + of the cluster to perform `cluster.arrive`. The `aligned` attribute, when provided, + generates the .aligned version of the PTX instruction. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-barrier-cluster) + }]; + let arguments = (ins + UnitAttr:$aligned + ); + let results = (outs); +} +def NVVM_ClusterLaunchControlQueryCancelOp : NVVM_InferredOp<"clusterlaunchcontrol.query.cancel"> { + let description = [{ + `clusterlaunchcontrol.query.cancel` queries the response of a + `clusterlaunchcontrol.try.cancel` operation specified by operand + `try_cancel_response`. + + Operand `query_type` specifies the type of query to perform and can be one + of the following: + - `is_canceled` : Returns true if the try cancel request succeeded, + and false otherwise. + - `get_first_cta_id_{x/y/z}` : Returns the x, y, or z coordinate of the + first CTA in the canceled cluster. Behaviour is defined only if the try + cancel request succeeded. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-query-cancel) + }]; + let arguments = (ins + NVVM_T128BitSignlessInteger:$try_cancel_response, + NVVM_ClusterLaunchControlQueryTypeAttr:$query_type + ); + let results = (outs + NVVM_T1BitSignlessIntegerOr32:$res + ); +} +def NVVM_ClusterLaunchControlTryCancelOp : NVVM_Op<"clusterlaunchcontrol.try.cancel"> { + let description = [{ + `clusterlaunchcontrol.try.cancel` requests atomically canceling the launch + of a cluster that has not started running yet. It asynchronously writes an + opaque response to shared memory indicating whether the operation succeeded + or failed. + + Operand `smemAddress` specifies the naturally aligned address of the + 16-byte wide shared memory location where the request's response is written. + + Operand `mbarrier` specifies the mbarrier object used to track the + completion of the asynchronous operation. + + If `multicast` is specified, the response is asynchronously written to the + corresponding local shared memory location (specifed by `addr`) of each CTA + in the requesting cluster. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace3:$smemAddress, + NVVM_LLVMPointerInAddressSpace3:$mbarrier, + UnitAttr:$multicast + ); + let results = (outs); +} +def NVVM_CompareAndSetOp : NVVM_InferredOp<"compare.and.set"> { + let description = [{ + Compare two values and return boolean mask of 1's or 0's. Return value + will be of the same type as the operands (vector<4xi8> or vector<2xi16>). + + Arguments: + - `a`: The first value. + - `b`: The second value. + - `op`: The comparison operator. (eq, ne, ugt, uge, ult, ule, sgt, sge, slt, sle) + + Example: + ```mlir + %res = nvvm.compare.and.set %a, %b : vector<4xi8> + %res = nvvm.compare.and.set %a, %b : vector<2xi16> + ``` + }]; + let arguments = (ins + NVVM_FixedLengthVectorOf8Bit:$a, + NVVM_FixedLengthVectorOf8Bit:$b, + NVVM_CompareOpKindAttr:$op + ); + let results = (outs + NVVM_FixedLengthVectorOf8Bit:$result + ); +} +def NVVM_ConvertPackIntegerOp : NVVM_Op<"convert.and.pack.integer"> { + let description = [{ + This Op converts the given integer inputs to the specified integer type + with saturation and packs the result into the lower bits of the result. + + The result `dst` is represented as an i32 type. + + The optional operand `srcC` must be specified when the converted types add + up to less than 32 bits. The remaining bits of the result are filled with + the lower bits of `srcC`. + + The `convert_type` attribute specifies the integer type to convert to + before packing. The `signed` attribute can be specified to indicate that + the `convert_type` is signed. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cvt-pack) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$srcA, + NVVM_T32BitSignlessInteger:$srcB, + Optional:$srcC, + UnitAttr:$is_signed, + TypeAttr:$convert_type + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$dst + ); +} +def NVVM_ConvertBF16x2ToF4x2Op : NVVM_Op<"convert.bf16x2.to.f4x2"> { + let description = [{ + This Op converts each of the given BF16 inputs in an bf16x2 vector to the specified fp4 type. + The result `dst` is returned as an i8 type where the converted values are + packed such that the value converted from the first element of `a` is + stored in the lower 4 bits of `dst` and the value converted from the second + element of `a` is stored in the upper 4 bits of `dst`. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. + The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + }]; + let arguments = (ins + NVVM_VectorOfBfloat16TypeValuesOf:$src, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$relu, + NVVM_ConvertBF16x2ToF4x2DstTyAttr:$dstTy + ); + let results = (outs + NVVM_T8BitSignlessInteger:$dst + ); +} +def NVVM_ConvertBF16x2ToF6x2Op : NVVM_Op<"convert.bf16x2.to.f6x2"> { + let description = [{ + This Op converts each of the given BF16 inputs in an bf16x2 vector to the specified fp6 type. The result `dst` is + represented either as an i16 type or as a vector of two i8 types. + If `dst` is returned as an i16 type, the converted values are packed such + that the value converted from the first element of `a` is stored in the + lower 8 bits of `dst` with 2 MSB bits padded with zeros and the value + converted from the second element of `a` is stored in the upper 8 bits of + `dst` with 2 MSB bits padded with zeros. + If `dst` is returned as a vector type, each converted value is stored as an + i8 element in the vector with 2 MSB bits padded with zeros. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + }]; + let arguments = (ins + NVVM_VectorOfBfloat16TypeValuesOf:$src, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$relu, + NVVM_ConvertBF16x2ToF6x2DstTyAttr:$dstTy + ); + let results = (outs + NVVM_T16BitSignlessIntegerOrVector:$dst + ); +} +def NVVM_ConvertBF16x2ToF8x2Op : NVVM_Op<"convert.bf16x2.to.f8x2"> { + let description = [{ + This Op converts the given bf16 inputs in a bf16x2 vector to the specified + f8 type. The result `dst` is represented either as a packed i16 type or as + a vector of two i8 types. + If `dst` is returned as an i16 type, the converted values are packed such + that the value converted from the first element of `a` is stored in the + lower 8 bits of `dst` and the value converted from the second element of + `a` is stored in the upper 8 bits of `dst`. + If `dst` is returned as a vector type, each converted value is stored as an + i8 element in the vector. + The `rnd` and `sat` attributes specify the rounding and saturation modes + respectively. + + The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOfBfloat16TypeValuesOf:$src, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$relu, + NVVM_ConvertBF16x2ToF8x2DstTyAttr:$dstTy + ); + let results = (outs + NVVM_T16BitSignlessIntegerOrVector:$dst + ); +} +def NVVM_ConvertBF16x2ToS2F6x2Op : NVVM_Op<"convert.bf16x2.to.s2f6x2"> { + let description = [{ + This Op converts each of the given BF16 inputs in a bf16x2 vector to the + S2F6x2 type. The result `dst` can be either a packed i16 type or a vector + of two i8 types. + If `dst` is returned as an i16 type, the converted values are packed such + that the value converted from the first element of `a` is stored in the + lower 8 bits of `dst` and the value converted from the second element of + `a` is stored in the upper 8 bits of `dst`. + If `dst` is returned as a vector type, each converted value is stored as an + i8 element in the vector. + The `relu` attribute, when set, lowers to the '.relu' variant + of the cvt instruction. + The optional scaling-factors for each of the inputs are provided through + the operand `scaleFactor` as a packed i16 type. Only `ue8m0` is supported + as the type of the scale-factor currently. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOfBfloat16TypeValuesOf:$src, + Optional:$scaleFactor, + OptionalAttr:$relu + ); + let results = (outs + NVVM_T16BitSignlessIntegerOrVector:$dst + ); +} +def NVVM_ConvertF16x2ToF4x2Op : NVVM_Op<"convert.f16x2.to.f4x2"> { + let description = [{ + This Op converts each of the given F16 inputs in an f16x2 vector to the specified fp4 type. + The result `dst` is returned as an i8 type where the converted values are + packed such that the value converted from the first element of `a` is + stored in the lower 4 bits of `dst` and the value converted from the second + element of `a` is stored in the upper 4 bits of `dst`. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. + The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + }]; + let arguments = (ins + NVVM_VectorOf16BitFloatValues:$src, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$relu, + NVVM_ConvertF16x2ToF4x2DstTyAttr:$dstTy + ); + let results = (outs + NVVM_T8BitSignlessInteger:$dst + ); +} +def NVVM_ConvertF16x2ToF6x2Op : NVVM_Op<"convert.f16x2.to.f6x2"> { + let description = [{ + This Op converts each of the given F16 inputs in an f16x2 vector to the specified fp6 type. The result `dst` is + represented either as an i16 type or as a vector of two i8 types. + If `dst` is returned as an i16 type, the converted values are packed such + that the value converted from the first element of `a` is stored in the + lower 8 bits of `dst` with 2 MSB bits padded with zeros and the value + converted from the second element of `a` is stored in the upper 8 bits of + `dst` with 2 MSB bits padded with zeros. + If `dst` is returned as a vector type, each converted value is stored as an + i8 element in the vector with 2 MSB bits padded with zeros. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + }]; + let arguments = (ins + NVVM_VectorOf16BitFloatValues:$src, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$relu, + NVVM_ConvertF16x2ToF6x2DstTyAttr:$dstTy + ); + let results = (outs + NVVM_T16BitSignlessIntegerOrVector:$dst + ); +} +def NVVM_ConvertF16x2ToF8x2Op : NVVM_Op<"convert.f16x2.to.f8x2"> { + let description = [{ + This Op converts the given f16 inputs in an f16x2 vector to the specified + f8 type. + The result `dst` is represented as an i16 type or as a vector + of two i8 types. + If `dst` is returned as an i16 type, the converted values from `a` + are packed such that the value converted from the first element of `a` + is stored in the upper 8 bits of `dst` and the value converted from the + second element of `a` is stored in the lower 8 bits of `dst`. + If `dst` is returned as a vector type, each converted value is stored as an + i8 element in the vector. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. + The `rnd` and `sat` attributes specify the rounding and saturation modes + respectively. + The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOf16BitFloatValues:$a, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$relu, + TypeAttr:$dstTy + ); + let results = (outs + NVVM_T16BitSignlessIntegerOrVector:$dst + ); +} +def NVVM_ConvertF32x2ToBF16x2Op : NVVM_Op<"convert.f32x2.to.bf16x2"> { + let description = [{ + Converts two F32 values to packed bf16x2 format with + the specified rounding mode. The `src_hi` and `src_lo` parameters + correspond to operands `a` and `b` in the PTX ISA, respectively. + + The `random_bits` parameter is required for stochastic rounding and + provides the [random bits](https://docs.nvidia.com/cuda/parallel-thread-execution/#cvt-rs-rbits-layout-bf16) to be used for the conversion. + + The `relu` attribute clamps negative results to 0. + + The `sat` attribute determines saturation behavior. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$src_hi, + NVVM_T32BitFloat:$src_lo, + Optional:$random_bits, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$relu + ); + let results = (outs + NVVM_VectorOfBfloat16TypeValuesOf:$dst + ); +} +def NVVM_ConvertF32x2ToF16x2Op : NVVM_Op<"convert.f32x2.to.f16x2"> { + let description = [{ + Converts two F32 values to packed f16x2 format with + the specified rounding mode. The `src_hi` and `src_lo` parameters + correspond to operands `a` and `b` in the PTX ISA, respectively. + + The `random_bits` parameter is required for stochastic rounding and + provides the [random bits](https://docs.nvidia.com/cuda/parallel-thread-execution/#cvt-rs-rbits-layout-f16) to be used for the conversion. + + The `relu` attribute clamps negative results to 0. + + The `sat` attribute determines saturation behavior. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$src_hi, + NVVM_T32BitFloat:$src_lo, + Optional:$random_bits, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$relu + ); + let results = (outs + NVVM_VectorOf16BitFloatValues:$dst + ); +} +def NVVM_ConvertF32x2ToF4x2Op : NVVM_Op<"convert.f32x2.to.f4x2"> { + let description = [{ + This Op converts each of the given float inputs to the specified fp4 type. + + The result `dst` is returned as an i8 type where the converted values are + packed such that the value converted from `a` is stored in the upper 4 bits + of `dst` and the value converted from `b` is stored in the lower 4 bits of + `dst`. + + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. + + The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$a, + NVVM_T32BitFloat:$b, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$relu, + TypeAttr:$dstType + ); + let results = (outs + NVVM_T8BitSignlessInteger:$dst + ); +} +def NVVM_ConvertF32x2ToF6x2Op : NVVM_Op<"convert.f32x2.to.f6x2"> { + let description = [{ + This Op converts each of the given float inputs to the specified fp6 type. + The result `dst` is represented either as an i16 type or as a vector + of two i8 types. + If `dst` is returned as an i16 type, the converted values are packed such + that the value converted from `a` is stored in the upper 8 bits of `dst` + with 2 MSB bits padded with zeros and the value converted from `b` is + stored in the lower 8 bits of `dst` with 2 MSB bits padded with zeros. + If `dst` is returned as a vector type, each converted value is stored as an + i8 element in the vector. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. + + The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$a, + NVVM_T32BitFloat:$b, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$relu, + TypeAttr:$dstTy + ); + let results = (outs + NVVM_T16BitSignlessIntegerOrVector:$dst + ); +} +def NVVM_ConvertF32x2ToF8x2Op : NVVM_Op<"convert.f32x2.to.f8x2"> { + let description = [{ + This Op converts each of the given float inputs to the specified fp8 type. + The result `dst` is represented as an i16 type or as a vector + of two i8 types. + If `dst` is returned as an i16 type, the converted values are packed such + that the value converted from `a` is stored in the upper 8 bits of `dst` + and the value converted from `b` is stored in the lower 8 bits of `dst`. + If `dst` is returned as a vector type, each converted value is stored as an + i8 element in the vector. + The `rnd` and `sat` attributes specify the rounding and saturation modes respectively. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. + + The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$a, + NVVM_T32BitFloat:$b, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$relu, + TypeAttr:$dstTy + ); + let results = (outs + NVVM_T16BitSignlessIntegerOrVector:$dst + ); +} +def NVVM_ConvertF32x2ToS2F6x2Op : NVVM_Op<"convert.f32x2.to.s2f6x2"> { + let description = [{ + This Op converts each of the given f32 inputs to the + S2F6x2 type. The result `dst` can be either a packed i16 type or a vector + of two i8 types. + If `dst` is returned as an i16 type, the converted values are packed such + that the value converted from `a` is stored in the upper 8 bits of `dst` + and the value converted from `b` is stored in the lower 8 bits of `dst`. + If `dst` is returned as a vector type, each converted value is stored as an + i8 element in the vector. + The `relu` attribute, when set, lowers to the '.relu' variant + of the cvt instruction. + The optional scaling-factors for each of the inputs are provided through + the operand `scaleFactor` as a packed i16 type. Only `ue8m0` is supported + as the type of the scale-factor currently. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$a, + NVVM_T32BitFloat:$b, + Optional:$scaleFactor, + OptionalAttr:$relu + ); + let results = (outs + NVVM_T16BitSignlessIntegerOrVector:$dst + ); +} +def NVVM_ConvertF32x4ToF4x4Op : NVVM_Op<"convert.f32x4.to.f4x4"> { + let description = [{ + Converts a vector<4xf32> to packed f4x4 format using + stochastic rounding (.rs) mode with SATFINITE saturation. Randomness is + provided by the `rbits` parameter. The `dstTy` attribute specifies the + target floating-point format. The `relu` attribute clamps negative results to 0. + + Note: These operations always use RS rounding mode and SATFINITE saturation mode. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues3:$src, + NVVM_T32BitSignlessInteger:$rbits, + OptionalAttr:$relu, + TypeAttr:$dstTy + ); + let results = (outs + NVVM_T16BitSignlessInteger2:$dst + ); +} +def NVVM_ConvertF32x4ToF6x4Op : NVVM_Op<"convert.f32x4.to.f6x4"> { + let description = [{ + Converts a vector<4xf32> to packed f6x4 format using + stochastic rounding (.rs) mode with SATFINITE saturation. Randomness is + provided by the `rbits` parameter. The `dstTy` attribute specifies the + target floating-point format. The `relu` attribute clamps negative results to 0. + + Note: These operations always use RS rounding mode and SATFINITE saturation mode. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues3:$src, + NVVM_T32BitSignlessInteger:$rbits, + OptionalAttr:$relu, + TypeAttr:$dstTy + ); + let results = (outs + NVVM_VectorOf8BitSignlessInteger:$dst + ); +} +def NVVM_ConvertF32x4ToF8x4Op : NVVM_Op<"convert.f32x4.to.f8x4"> { + let description = [{ + Converts a vector<4xf32> to packed f8x4 format using + stochastic rounding (.rs) mode with SATFINITE saturation. Randomness is + provided by the `rbits` parameter. The `dstTy` attribute specifies the + target floating-point format. The `relu` attribute clamps negative results to 0. + + Note: These operations always use RS rounding mode and SATFINITE saturation mode. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues3:$src, + NVVM_T32BitSignlessInteger:$rbits, + OptionalAttr:$relu, + TypeAttr:$dstTy + ); + let results = (outs + NVVM_VectorOf8BitSignlessInteger:$dst + ); +} +def NVVM_ConvertF4x2ToF16x2Op : NVVM_Op<"convert.f4x2.to.f16x2"> { + let description = [{ + This Op converts the given f4 inputs in a packed i8 to f16. + + The result `dst` is represented as a vector of f16 elements. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction." + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T8BitSignlessInteger2:$src, + OptionalAttr:$relu, + TypeAttr:$srcType + ); + let results = (outs + NVVM_VectorOf16BitFloatValues:$dst + ); +} +def NVVM_ConvertF6x2ToF16x2Op : NVVM_Op<"convert.f6x2.to.f16x2"> { + let description = [{ + This Op converts the given f6 inputs in a i8x2 vector to f16. + + The result `dst` is represented as a vector of f16 elements. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction." + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOf8BitSignlessInteger2:$src, + OptionalAttr:$relu, + TypeAttr:$srcType + ); + let results = (outs + NVVM_VectorOf16BitFloatValues:$dst + ); +} +def NVVM_ConvertF8x2ToBF16x2Op : NVVM_Op<"convert.f8x2.to.bf16x2"> { + let description = [{ + This Op converts the given f8 inputs in a i8x2 vector to bf16. + + The result `dst` is represented as a vector of bf16 elements. + The `sat` attribute specifies the saturation mode. + + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOf8BitSignlessInteger2:$src, + Optional:$scaleFactor, + OptionalAttr:$sat, + TypeAttr:$srcType + ); + let results = (outs + NVVM_VectorOfBfloat16TypeValuesOf:$dst + ); +} +def NVVM_ConvertF8x2ToF16x2Op : NVVM_Op<"convert.f8x2.to.f16x2"> { + let description = [{ + This Op converts the given f8 inputs in a i8x2 vector to f16. + + The result `dst` is represented as a vector of f16 elements. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction." + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOf8BitSignlessInteger2:$src, + OptionalAttr:$relu, + TypeAttr:$srcType + ); + let results = (outs + NVVM_VectorOf16BitFloatValues:$dst + ); +} +def NVVM_ConvertFloatToIntegerOp : NVVM_Op<"convert.float.to.integer"> { + let description = [{ + This Op converts the given float input to the specified integer type. + + The `rnd` attribute specifies the rounding mode. + The `sat` atttribute, when set, limits the result to the range of the + integer type. + The `ftz` attribute, when set, converts denormal inputs to zero. + The `signed` attribute, when set, converts the input to a signed integer. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$src, + OptionalAttr:$rnd, + UnitAttr:$sat, + UnitAttr:$ftz, + UnitAttr:$is_signed + ); + let results = (outs + NVVM_T8BitSignlessIntegerOr32:$dst + ); +} +def NVVM_ConvertFloatToTF32Op : NVVM_Op<"convert.float.to.tf32"> { + let description = [{ + This Op converts the given f32 input to tf32. + The result `res` is represented as an i32 type. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. The `rnd` and `sat` attributes specify the + the rounding and saturation modes respectively. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$src, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$relu + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$res + ); +} +def NVVM_ConvertS2F6x2ToBF16x2Op : NVVM_Op<"convert.s2f6x2.to.bf16x2"> { + let description = [{ + This Op converts a pair of s2f6x2 inputs to bf16x2 type. The result `dst` + is represented as a vector of two bf16 elements. + + The `relu` attribute, when set, lowers to the '.relu' variant + of the cvt instruction. + + The optional scaling-factors for each of the inputs are provided through + the operand `scaleFactor` as a packed i16 type. Only `ue8m0` is supported + as the type of the scale-factor currently. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOf8BitSignlessInteger2:$src, + Optional:$scaleFactor, + OptionalAttr:$sat, + OptionalAttr:$relu + ); + let results = (outs + NVVM_VectorOfBfloat16TypeValuesOf:$dst + ); +} +def NVVM_CosOp : NVVM_Op<"cos", [Pure]> { + let description = [{ + Computes a fast approximation of the cosine of the input value (in + radians). The `ftz` attribute, when set, flushes subnormal inputs + and results to sign-preserving zero. + }]; + let arguments = (ins + NVVM_T32BitFloat:$src, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_T32BitFloat2:$res + ); +} +def NVVM_CpAsyncBulkCommitGroupOp : NVVM_Op<"cp.async.bulk.commit.group"> { + let description = [{ + This Op commits all prior initiated but uncommitted cp.async.bulk + instructions into a cp.async.bulk-group. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-commit-group) + }]; + let arguments = (ins); + let results = (outs); +} +def NVVM_CpAsyncBulkSharedCTAToGlobalOp : NVVM_Op<"cp.async.bulk.global.shared.cta", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous copy operation from Shared CTA memory to + global memory. The 32-bit operand `size` specifies the amount of + memory to be copied, in terms of number of bytes. `size` must be a + multiple of 16. The `l2CacheHint` operand is optional, and it is used + to specify cache eviction policy that may be used during the memory + access. The `byteMask` operand is optional. The i-th bit in the 16-bit + wide `byteMask` specifies whether the i-th byte of each 16-byte wide + chunk of source data is copied to the destination. If the bit is set, + the byte is copied. + + Example: + ```mlir + nvvm.cp.async.bulk.global.shared.cta %dst, %src, %size + : !llvm.ptr<1>, !llvm.ptr<3> + + // with l2_cache_hint + nvvm.cp.async.bulk.global.shared.cta %dst, %src, %size l2_cache_hint = %ch + : !llvm.ptr<1>, !llvm.ptr<3> + + // with byte_mask + nvvm.cp.async.bulk.global.shared.cta %dst, %src, %size byte_mask = %mask + : !llvm.ptr<1>, !llvm.ptr<3> + + // with both l2_cache_hint and byte_mask + nvvm.cp.async.bulk.global.shared.cta %dst, %src, %size l2_cache_hint = %ch byte_mask = %mask + : !llvm.ptr<1>, !llvm.ptr<3> + ``` + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace1:$dstMem, + NVVM_LLVMPointerInAddressSpace3:$srcMem, + NVVM_T32BitSignlessInteger:$size, + Optional:$l2CacheHint, + Optional:$byteMask + ); + let results = (outs); +} +def NVVM_CpAsyncBulkPrefetchOp : NVVM_Op<"cp.async.bulk.prefetch"> { + let description = [{ + Initiates an asynchronous prefetch of data from the location + specified by `srcMem` to the L2 cache. + + The `l2CacheHint` operand is optional, and it is used to specify cache + eviction policy that may be used during the memory access. + The `evict` operand is optional, and it is used to specify cache eviction + priority. The `evict` operand is mutually exclusive with the `l2CacheHint` + operand. + + Example: + ```mlir + nvvm.cp.async.bulk.prefetch %src, %size : !llvm.ptr<1> + + // with l2_cache_hint + nvvm.cp.async.bulk.prefetch %src, %size l2_cache_hint = %ch : !llvm.ptr<1> + ``` + Example: + ```mlir + // with evict_priority + nvvm.cp.async.bulk.prefetch %src, %size evict_priority = evict_last : !llvm.ptr<1> + ``` + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-prefetch) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace1:$srcMem, + NVVM_T32BitSignlessInteger:$size, + Optional:$l2CacheHint, + OptionalAttr:$evict + ); + let results = (outs); +} +def NVVM_CpAsyncBulkGlobalToSharedClusterOp : NVVM_Op<"cp.async.bulk.shared.cluster.global", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous copy operation from global memory to shared + memory or shared_cluster memory. + + The `multicastMask` operand is optional and can be used only when the + destination is shared::cluster memory. When it is present, this Op copies + data from global memory to shared memory of multiple CTAs in the cluster. + Operand `multicastMask` specifies the destination CTAs in the cluster such + that each bit position in the 16-bit `multicastMask` operand corresponds to + the `nvvm.read.ptx.sreg.ctaid` of the destination CTA. + + The `l2CacheHint` operand is optional, and it is used to specify cache + eviction policy that may be used during the memory access. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace32:$dstMem, + NVVM_LLVMPointerInAddressSpace1:$srcMem, + NVVM_LLVMPointerInAddressSpace3:$mbar, + NVVM_T32BitSignlessInteger:$size, + Optional:$multicastMask, + Optional:$l2CacheHint, + OptionalAttr:$validatePattern + ); + let results = (outs); +} +def NVVM_CpAsyncBulkSharedCTAToSharedClusterOp : NVVM_Op<"cp.async.bulk.shared.cluster.shared.cta"> { + let description = [{ + Initiates an asynchronous copy operation from Shared CTA memory to Shared + cluster memory. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace7:$dstMem, + NVVM_LLVMPointerInAddressSpace3:$srcMem, + NVVM_LLVMPointerInAddressSpace3:$mbar, + NVVM_T32BitSignlessInteger:$size + ); + let results = (outs); +} +def NVVM_CpAsyncBulkTensorSharedCTAToGlobalOp : NVVM_Op<"cp.async.bulk.tensor.global.shared.cta", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous copy of the tensor data from shared::cta + memory to global memory. This Op supports all the store modes specified in + `TMAStoreMode`. + + The `l2CacheHint` operand is optional, and it is used to specify cache + eviction policy that may be used during the memory access. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-tensor) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace0:$tmaDescriptor, + NVVM_LLVMPointerInAddressSpace3:$srcMem, + Variadic:$coordinates, + Optional:$l2CacheHint, + Optional:$predicate, + OptionalAttr:$mode + ); + let results = (outs); +} +def NVVM_CpAsyncBulkTensorSharedCTAToGlobalOverrideAddrOp : NVVM_Op<"cp.async.bulk.tensor.global.shared.cta.override", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous bulk tensor operation that transfers data from shared + memory to global memory overriding one or many of the fields: addr, size, and stride. + + The operation: + - Reads data from shared memory location specified by `srcMem` + - Uses a TMA descriptor `tmaDesc` to control the data access pattern + - Overrides the default destination address with `overrideAdrr` + - Specifies box coordinates via `coordinates` array + - Can specify tensor size via `tensorSize` (optional) + - Controls memory access patterns with `lowerStride` and `upperStride` (optional) + - Supports an optional L2 cache hint with `l2CacheHint` to support cache eviction policy (optional) + - Supports different store modes (`TILE`, `IM2COL`, `IM2COL_W`, and `TILE_SCATTER4`) via the `mode` attribute + + Examples: + + // Basic usage with 1D box coordinates (TILE mode) + ```mlir + nvvm.cp.async.bulk.tensor.global.shared.cta.override %tma_desc, %src, %override_addr, box[%d0] { + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + + // With L2 cache hint + ```mlir + nvvm.cp.async.bulk.tensor.global.shared.cta.override %tma_desc, %src, %override_addr, box[%d0, %d1, %d2] + l2_cache_hint = %ch { + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + + // IM2COL mode + ```mlir + nvvm.cp.async.bulk.tensor.global.shared.cta.override %tma_desc, %src, %override_addr, box[%d0, %d1, %d2] { + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + + // With tensor size + ```mlir + nvvm.cp.async.bulk.tensor.global.shared.cta.override %tma_desc, %src, %override_addr, + box[%d0] tensor_size[%ts0] { + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + + // With stride + ```mlir + nvvm.cp.async.bulk.tensor.global.shared.cta.override %tma_desc, %src, %override_addr, + box[%d0, %d1] tensor_size[%ts0, %ts1] lower_stride[%lstrd0] upper_stride[%ustrd] { + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerType:$tmaDesc, + NVVM_LLVMPointerInAddressSpace3:$srcMem, + NVVM_LLVMPointerInAddressSpace1:$overrideAdrr, + Variadic:$coordinates, + Variadic:$tensorSize, + Variadic:$lowerStride, + Optional:$upperStride, + Optional:$l2CacheHint, + OptionalAttr:$mode + ); + let results = (outs); +} +def NVVM_CpAsyncBulkTensorPrefetchOp : NVVM_Op<"cp.async.bulk.tensor.prefetch", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous prefetch operation on the tensor data from global + memory to L2 cache. This Op supports all the load modes specified in + `TMALoadMode`. + + The `l2CacheHint` operand is optional, and it is used to specify cache + eviction policy that may be used during the memory access. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace0:$tmaDescriptor, + Variadic:$coordinates, + Variadic:$im2colOffsets, + Optional:$l2CacheHint, + OptionalAttr:$mode, + OptionalAttr:$evict + ); + let results = (outs); +} +def NVVM_CpAsyncBulkTensorPrefetchOverrideAddrOp : NVVM_Op<"cp.async.bulk.tensor.prefetch.override", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous bulk tensor operation that prefetches data from + global memory to L2 cache overriding one or many of the fields: addr, size, and stride. + + The operation: + - Reads data from global memory location specified by `overrideAdrr` + - Uses a TMA descriptor `tmaDesc` to control the data access pattern + - Specifies box coordinates via `coordinates` array + - Can specify tensor size via `tensorSize` (optional) + - Controls memory access patterns with `lowerStride` and `upperStride` (optional) + - Can specify im2col offsets via `im2colOffsets` (optional) + - Supports an optional L2 cache hint with `l2CacheHint` to support cache eviction policy (optional) + - Supports an optional cache eviction priority with `evict` (optional) + - Supports different load modes (`TILE`/`IM2COL`/`IM2COL_W`/`IM2COL_W_128`/`TILE_GATHER4`) via the `mode` attribute (default = `TILE`) + + Examples: + + // Basic prefetch operations + ```mlir + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0] : !llvm.ptr, !llvm.ptr<1> + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1, %d2] : !llvm.ptr, !llvm.ptr<1> + ``` + + // With L2 cache hint + ```mlir + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0] l2_cache_hint = %ch : !llvm.ptr, !llvm.ptr<1> + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1, %d2] l2_cache_hint = %ch : !llvm.ptr, !llvm.ptr<1> + ``` + + // With evict_priority + ```mlir + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0] evict_priority = evict_last : !llvm.ptr, !llvm.ptr<1> + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1, %d2] evict_priority = evict_last : !llvm.ptr, !llvm.ptr<1> + ``` + + // With tensor size specification + ```mlir + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0] tensor_size[%ts0] : !llvm.ptr, !llvm.ptr<1> + ``` + + // With tensor size and stride specifications + ```mlir + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1] + tensor_size[%ts0, %ts1] lower_stride[%ls0] upper_stride[%us0] : !llvm.ptr, !llvm.ptr<1> + ``` + + // With im2col offsets + ```mlir + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1, %d2] + im2col[%off0] : !llvm.ptr, !llvm.ptr<1> + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1, %d2, %d3] + im2col[%off0, %off1] : !llvm.ptr, !llvm.ptr<1> + ``` + + // Dim stride with cache hint + ```mlir + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1, %d2, %d3, %d4] + tensor_size[%ts0, %ts1, %ts2, %ts3, %ts4] + lower_stride[%ls0, %ls1, %ls2, %ls3] + upper_stride[%us0] + l2_cache_hint = %ch : !llvm.ptr, !llvm.ptr<1> + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerType:$tmaDesc, + NVVM_LLVMPointerInAddressSpace1:$overrideAdrr, + Variadic:$coordinates, + Variadic:$tensorSize, + Variadic:$lowerStride, + Optional:$upperStride, + Variadic:$im2colOffsets, + Optional:$l2CacheHint, + OptionalAttr:$evict, + OptionalAttr:$mode + ); + let results = (outs); +} +def NVVM_CpAsyncBulkTensorReduceOp : NVVM_Op<"cp.async.bulk.tensor.reduce", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous reduction operation of tensor data in + global memory with tensor data in shared memory. + + The `mode` attribute indicates whether the copy mode is tile or im2col. + The `redOp` attribute specifies the reduction operations applied. + The supported reduction operations are: + {add, min, max, inc, dec, and, or, xor} + + The `l2CacheHint` operand is optional, and it is used to specify cache + eviction policy that may be used during the memory access. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-reduce-async-bulk-tensor) + }]; + let arguments = (ins + NVVM_LLVMPointerType:$tmaDescriptor, + NVVM_LLVMPointerInAddressSpace3:$srcMem, + Variadic:$coordinates, + Optional:$l2CacheHint, + NVVM_TMAReduxKindAttr:$redKind, + OptionalAttr:$mode + ); + let results = (outs); +} +def NVVM_CpAsyncBulkTensorReduceOverrideAddrOp : NVVM_Op<"cp.async.bulk.tensor.reduce.override", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous bulk tensor operation that performs a reduction operation + overriding one or many of the fields: addr, size, and stride. + + The operation: + - Reads data from shared memory location specified by `srcMem` + - Uses a TMA descriptor `tmaDesc` to control the data access pattern + - Overrides the default destination address with `overrideAdrr` + - Specifies box coordinates via `coordinates` array + - Can specify tensor size via `tensorSize` (optional) + - Controls memory access patterns with `lowerStride` and `upperStride` (optional) + - Supports an optional L2 cache hint with `l2CacheHint` to support cache eviction policy (optional) + - Supports different store modes (`TILE` or `IM2COL`) via the `mode` attribute + - Performs a reduction operation specified by the `redKind` attribute (supports ADD, MIN, MAX, + INC, DEC, AND, OR, XOR) that combines the source data stored at the destination location + + Examples: + + // Override address with ADD reduction (TILE mode) + ```mlir + nvvm.cp.async.bulk.tensor.reduce.override %tma_desc, %src, %override_addr, box[%d0] { + redKind = #nvvm.tma_redux_kind, + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + + // With L2 cache hint + ```mlir + nvvm.cp.async.bulk.tensor.reduce.override %tma_desc, %src, %override_addr, box[%d0, %d1] + l2_cache_hint = %ch { + redKind = #nvvm.tma_redux_kind, + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + + // IM2COL mode with XOR reduction + ```mlir + nvvm.cp.async.bulk.tensor.reduce.override %tma_desc, %src, %override_addr, box[%d0, %d1, %d2] { + redKind = #nvvm.tma_redux_kind, + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + + // With tensor size (dim variant) + ```mlir + nvvm.cp.async.bulk.tensor.reduce.override %tma_desc, %src, %override_addr, + box[%d0] tensor_size[%ts0] { + redKind = #nvvm.tma_redux_kind, + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + + // With stride information (dim_stride variant) + ```mlir + nvvm.cp.async.bulk.tensor.reduce.override %tma_desc, %src, %override_addr, + box[%d0, %d1] tensor_size[%ts0, %ts1] lower_stride[%lstrd0] upper_stride[%ustrd] { + redKind = #nvvm.tma_redux_kind, + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerType:$tmaDesc, + NVVM_LLVMPointerInAddressSpace3:$srcMem, + NVVM_LLVMPointerInAddressSpace1:$overrideAdrr, + Variadic:$coordinates, + Variadic:$tensorSize, + Variadic:$lowerStride, + Optional:$upperStride, + Optional:$l2CacheHint, + NVVM_TMAReduxKindAttr:$redKind, + OptionalAttr:$mode + ); + let results = (outs); +} +def NVVM_CpAsyncBulkTensorGlobalToSharedClusterOp : NVVM_Op<"cp.async.bulk.tensor.shared.cluster.global", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous copy operation on the tensor data from global + memory to shared::cluster (or) shared::cta memory. This Op supports all + the load modes specified in `TMALoadMode`. + + The `multicastMask` operand is optional. When it is present, the Op copies + data from global memory to shared memory of multiple CTAs in the cluster. + Operand `multicastMask` specifies the destination CTAs in the cluster such + that each bit position in the 16-bit `multicastMask` operand corresponds to + the `nvvm.read.ptx.sreg.ctaid` of the destination CTA. + + The `l2CacheHint` operand is optional, and it is used to specify cache + eviction policy that may be used during the memory access. + + When the `isCTAOnly` attribute is set to true, the destination is + shared::cta only. Hence, `multicastMask` and `CTAGroup` are not applicable + when `isCTAOnly` is true. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace32:$dstMem, + NVVM_LLVMPointerInAddressSpace0:$tmaDescriptor, + Variadic:$coordinates, + NVVM_LLVMPointerInAddressSpace3:$mbar, + Variadic:$im2colOffsets, + Optional:$multicastMask, + Optional:$l2CacheHint, + Optional:$predicate, + OptionalAttr:$validatePattern, + OptionalAttr:$mode, + OptionalAttr:$isCTAOnly, + OptionalAttr:$group + ); + let results = (outs); +} +def NVVM_CpAsyncBulkTensorGlobalToSharedClusterOverrideAddrOp : NVVM_Op<"cp.async.bulk.tensor.shared.cluster.global.override", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous bulk tensor operation that transfers data from global + memory to shared::cluster or shared::cta memory overriding one or many of the fields: addr, size, and stride. + + The operation: + - Writes data to shared::cluster or shared::cta memory location specified by `dstMem` + - Uses a TMA descriptor `tmaDesc` to specify the properties of the tensor copy operation + - Uses `mbar` to specify the memory barrier for the operation + - Overrides the default source address with `overrideAddr` + - Specifies box coordinates via `coordinates` array + - Specifies im2col arguments via `im2colArgs` (optional) + - Can specify tensor size via `tensorSize` (optional) + - Controls memory access patterns with `lowerStride` and `upperStride` (optional) + - Supports multicast mask via `multicastMask` (optional) + - Supports an optional L2 cache hint with `l2CacheHint` to support cache eviction policy (optional) + - Supports different load modes (`TILE`/`IM2COL`/`IM2COL_W`/`IM2COL_W_128`/`TILE_GATHER4`) via the `mode` attribute (default = `TILE`) + - Supports different group kinds (`CTA_1` or `CTA_2`) via the `group` attribute (optional) + - Supports different validate load data patterns via the `validatePattern` attribute (optional) + + Examples: + // Basic usage with box coordinates + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0, %d1] { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + // With L2 cache hint + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] + l2_cache_hint = %ch { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + // With multicast mask (i16) + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] + multicast_mask = %mc { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1>, i16 + ``` + + // With multicast mask (i32) + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] + multicast_mask = %mc32 { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1>, i32 + ``` + + // With both multicast mask and L2 cache hint + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] + multicast_mask = %mc + l2_cache_hint = %ch { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1>, i16 + ``` + + // With CTA_2 group + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] + l2_cache_hint = %ch { + group = #nvvm.cta_group, + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + // With tensor size and strides + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] + tensor_size[%ts0] { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0, %d1] + tensor_size[%ts0, %ts1] + lower_stride[%lstrd0] + upper_stride[%ustrd] { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + // Using IM2COL mode with im2col arguments + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0, %d1, %d2] + im2col[%im2col0] { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + // Using IM2COL_W mode + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0, %d1, %d2] + im2col[%wHalo, %wOffset] { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + // Using IM2COL_W_128 mode + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0, %d1, %d2] + im2col[%wHalo, %wOffset] { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + // Comprehensive example with all optional parameters + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0, %d1] + tensor_size[%ts0, %ts1] + lower_stride[%lstrd0] + upper_stride[%ustrd] + multicast_mask = %mc32 + l2_cache_hint = %ch { + group = #nvvm.cta_group, + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1>, i32 + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace72:$dstMem, + NVVM_LLVMPointerType:$tmaDesc, + NVVM_LLVMPointerInAddressSpace3:$mbar, + NVVM_LLVMPointerInAddressSpace1:$overrideAddr, + Variadic:$coordinates, + Variadic:$im2colArgs, + Variadic:$tensorSize, + Variadic:$lowerStride, + Optional:$upperStride, + Optional:$multicastMask, + Optional:$l2CacheHint, + OptionalAttr:$group, + OptionalAttr:$validatePattern, + OptionalAttr:$mode + ); + let results = (outs); +} +def NVVM_CpAsyncBulkWaitGroupOp : NVVM_Op<"cp.async.bulk.wait_group"> { + let description = [{ + Op waits for completion of the most recent bulk async-groups. + + The `$group` operand tells waiting has to be done until for $group or fewer + of the most recent bulk async-groups. If `$group` is 0, the op wait until + all the most recent bulk async-groups have completed. + + The `$read` indicates that the waiting has to be done until all the bulk + async operations in the specified bulk async-group have completed reading + from their source locations. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-wait-group) + }]; + let arguments = (ins + I32Attr:$group, + UnitAttr:$read + ); + let results = (outs); +} +def NVVM_CpAsyncCommitGroupOp : NVVM_Op<"cp.async.commit.group"> { + let arguments = (ins); + let results = (outs); +} +def NVVM_CpAsyncMBarrierArriveOp : NVVM_Op<"cp.async.mbarrier.arrive"> { + let description = [{ + The `cp.async.mbarrier.arrive` Op makes the *mbarrier object* track + all prior cp.async operations initiated by the executing thread. + The `addr` operand specifies the address of the *mbarrier object* + in generic or shared::cta address space. When it is generic, the + underlying memory should fall within the shared::cta space; + otherwise the behavior is undefined. The `noinc` attr impacts + how the mbarrier's state is updated. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-cp-async-mbarrier-arrive) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr, + OptionalAttr:$noinc + ); + let results = (outs); +} +def NVVM_CpAsyncOp : NVVM_Op<"cp.async.shared.global"> { + let arguments = (ins + NVVM_LLVMPointerInAddressSpace3:$dst, + NVVM_LLVMPointerInAddressSpace1:$src, + Optional:$cpSize, + I32Attr:$size, + NVVM_LoadCacheModifierAttr:$modifier + ); + let results = (outs); +} +def NVVM_CpAsyncWaitGroupOp : NVVM_Op<"cp.async.wait.group"> { + let arguments = (ins + I32Attr:$n + ); + let results = (outs); +} +def NVVM_CvtPackFloatOp : NVVM_Op<"cvt.packfloat"> { + let description = [{ + This Op converts the given floating point input (`srcA`) to another + packed floating point type (`res`). The operand `from` represents the + source type and `to` represents the destination type. + The result is merged with the given `srcC`. Both the input and result + are represented as 'i32' type. + + Below table describes the format of the result: + ``` + |--------------|-------------------------------------------------------| + | Dst Types | Description | + |--------------|-------------------------------------------------------| + | f16x2 | The result contains two fp16 values, in the low | + | bf16x2 | and high bits. | + |--------------|-------------------------------------------------------| + | e4m3x2 | The result contains two fp8 values in the low or | + | e5m2x2 | high 16-bits of `srcC` depending on `ExtractHi`. | + | ue8m0x2 | When `ExtractHi` is 0, the result is packed in | + | | bits `[7:0]` and `[15:8]` of `srcC` and the high | + | | 16-bits of srcC remain unchanged. | + | | When `ExtractHi` is 1, the result is packed in | + | | bits `[23:16]` and `[31:24]` of `srcC` and the low | + | | 16-bits of srcC remain unchanged. | + |--------------|-------------------------------------------------------| + | e2m3x2 | The result contains two 6-bit floating point values | + | e3m2x2 | with the two MSB's padded with zeroes. The 16-bits | + | | of the result are then packed in the same format | + | | described above. | + |--------------|-------------------------------------------------------| + | e2m1x2 | The result contains two 4-bit floating point values | + | | packed in bits `[7:0]` or `[23:16]` of `srcC` | + | | depending on `ExtractHi`. The other bits of `srcC` | + | | remain unchanged. | + |--------------|-------------------------------------------------------| + ``` + + Below table describes the format of the source operand: + ``` + |--------------|-------------------------------------------------------| + | Src Types | Description | + |--------------|-------------------------------------------------------| + | f16x2 | The two fp16 values are taken from the low and high | + | bf16x2 | high bits of operand `srcA`. | + |--------------|-------------------------------------------------------| + | e4m3x2 | The two fp8 values are taken from the low or high | + | e5m2x2 | 16-bits of `srcA` depending on `ExtractHi`. | + | ue8m0x2 | When `ExtractHi` is 0, the inputs are taken from | + | | bits `[7:0]` and `[15:8]` of `srcA`. | + | | When `ExtractHi` is 1, the inputs are sourced from | + | | bits `[23:16]` and `[31:24]` of `srcA`. | + |--------------|-------------------------------------------------------| + | e2m3x2 | Similar to the row above. | + | e3m2x2 | | + |--------------|-------------------------------------------------------| + | e2m1x2 | The two 4-bit floating point input values are taken | + | | from bits `[7:0]` or `[23:16]` of `srcA` depending | + | | on `ExtractHi`. | + |--------------|-------------------------------------------------------| + ``` + The `relu` attribute, when set, lowers to the '.relu' variant of the + cvt instruction. The `rnd` and `sat` attributes specify the rounding + and saturation respectively. The ue8m0x2, e2m3x2, e3m2x2 and e2m1x2 + types, either as source or destination types, are supported only in SM100a. + [For more information, see PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$srcA, + NVVM_T32BitSignlessInteger:$srcC, + NVVM_CVTPackFloatKindAttr:$from, + NVVM_CVTPackFloatKindAttr:$to, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$relu, + OptionalAttr:$extractHi + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$res + ); +} +def NVVM_CvtPackFloatF32Op : NVVM_Op<"cvt.packfloat.f32"> { + let description = [{ + This Op converts two given F32 inputs (in `srcA` and `srcB`) to a + packed floating point type requested in `to`, and the + result is merged with the given `srcC`. + `extractHi`= 0: the result is packed into the low 16-bits of `srcC`. + (and the high-16 bits of `srcC` remain as is) + `extractHi`= 1: the result is packed into the high 16-bits of `srcC`. + (and the low-16 bits of `srcC` remain as is) + The result `res` is represented as an i32 type. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. The `rnd` and `sat` attributes specify the + the rounding and saturation modes respectively. The ue8m0x2, + e2m3x2, e3m2x2 and e2m1x2 types are supported only in SM100a. + [For more information, see PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$srcA, + NVVM_T32BitFloat:$srcB, + NVVM_T32BitSignlessInteger:$srcC, + NVVM_CVTPackFloatKindAttr:$to, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$relu, + OptionalAttr:$extractHi + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$res + ); +} +def NVVM_CvtToF4x8PackedOp : NVVM_Op<"cvt.to.f4x8.packed"> { + let description = [{ + Fused conversion + packing: performs 4 pairwise conversions to f4 and + packs the 4 byte-sized results into a single i32 using inline PTX + `{ .reg .b8; 4x cvt; mov.b32 }`. + + This avoids the NVPTX backend's lack of a B8 register class, which + causes 3x `prmt.b32` instructions instead of a single `mov.b32` when + packing 4x i8 values via `BUILD_VECTOR v4i8`. + + For f32 source: takes 8 f32 values (4 high-low pairs). + For f16 source: takes 4 i32 values (packed f16x2). + For bf16 source: takes 4 i32 values (packed bf16x2). + + The `srcType` attribute specifies the source element type (f32, f16, + or bf16) and is used to select the correct PTX cvt instruction suffix. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + Variadic:$srcs, + OptionalAttr:$rnd, + TypeAttr:$srcType, + TypeAttr:$dstType + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$dst + ); +} +def NVVM_DotAccumulate2WayOp : NVVM_Op<"dot.accumulate.2way"> { + let description = [{ + Performs a two-way 16-bit to 8-bit dot-product which is accumulated in a + 32-bit result. + Operand `a` is a vector of two 16-bit elements and operand `b` a vector + of four 8-bit elements between which the dot product is computed. + + The `a_type` and `b_type` attributes specify the type of the elements in `a` + and `b` respectively. + If `a_type` or `b_type` is `s`, then the elements in the corresponding + vector are sign-extended to 32-bit before the dot product is computed. + If `a_type` or `b_type` is `u`, then the elements in the corresponding + vector are zero-extended to 32-bit instead. + + The `b_hi` boolean attribute specifies which two bytes of `b` are used for + the dot product. If `b_hi` is true, then the dot product is computed + between `a` and elements at indices 2 and 3 of `b`. If `b_hi` is false, + then the dot product is computed between `a` and elements at indices 0 and + 1 of `b`. + + Operand `c` is a 32-bit integer to which the result is accumulated. It is + treated as holding a signed integer if any of `a_type` or `b_type` is + signed. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#integer-arithmetic-instructions-dp2a) + }]; + let arguments = (ins + NVVM_VectorOf16BitSignlessInteger:$a, + NVVM_VectorOf8BitSignlessInteger:$b, + NVVM_T32BitSignlessInteger:$c, + NVVM_DotAccumulateTypeAttr:$a_type, + NVVM_DotAccumulateTypeAttr:$b_type, + BoolAttr:$b_hi + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$res + ); +} +def NVVM_DotAccumulate4WayOp : NVVM_Op<"dot.accumulate.4way"> { + let description = [{ + Performs a four-way byte dot-product which is accumulated in a 32-bit + result. + Operand `a` and `b` are vectors of 4 bytes between which the dot product is + computed. + + The `a_type` and `b_type` attributes specify the type of the elements in `a` + and `b` respectively. + If `a_type` or `b_type` is `signed`, then the elements in the corresponding + vector are sign-extended to 32-bit before the dot product is computed. + If `a_type` or `b_type` is `unsigned`, then the elements in the + corresponding vector are zero-extended to 32-bit instead. + + Operand `c` is a 32-bit integer to which the result is accumulated. It is + treated as holding a signed integer if any of `a_type` or `b_type` is `s8`. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#integer-arithmetic-instructions-dp4a) + }]; + let arguments = (ins + NVVM_VectorOf8BitSignlessInteger:$a, + NVVM_VectorOf8BitSignlessInteger:$b, + NVVM_T32BitSignlessInteger:$c, + NVVM_DotAccumulateTypeAttr:$a_type, + NVVM_DotAccumulateTypeAttr:$b_type + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$res + ); +} +def NVVM_ElectSyncOp : NVVM_Op<"elect.sync"> { + let description = [{ + The `elect.sync` instruction elects one predicated active leader + thread from among a set of threads specified in the `membermask`. + When the `membermask` is not provided explicitly, a default value + of `0xFFFFFFFF` is used. The predicate result is set to `True` for + the leader thread, and `False` for all other threads. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-elect-sync) + }]; + let arguments = (ins + Optional:$membermask + ); + let results = (outs + NVVM_T1BitSignlessInteger2:$pred + ); +} +def NVVM_Ex2Op : NVVM_Op<"ex2", [Pure]> { + let description = [{ + Computes a fast approximation of 2 raised to the power of the input + value. The `ftz` attribute, when set, flushes subnormal inputs and + results to sign-preserving zero. + }]; + let arguments = (ins + NVVM_T32BitFloat:$src, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_T32BitFloat2:$res + ); +} +def NVVM_Exit : NVVM_Op<"exit"> { + let description = [{ + Ends execution of a thread. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#control-flow-instructions-exit) + }]; + let arguments = (ins); + let results = (outs); +} +def NVVM_FAbsOp : NVVM_InferredOp<"fabs"> { + let description = [{ + Returns the absolute value of the floating point operand. The Op also has an + optional `ftz` attribute which flushes subnormal inputs and returns to + sign-preserving zero + + [For more information, see PTX ISA] + (http://sw-mobile-docs/CUDA/GPGPU/parallel-thread-execution/index.html#floating-point-instructions-abs) + }]; + let arguments = (ins + NVVM_FloatingPointOrFixedLengthVector:$value, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_FloatingPointOrFixedLengthVector:$res + ); +} +def NVVM_FenceMbarrierInitOp : NVVM_Op<"fence.mbarrier.init"> { + let description = [{ + Fence operation that applies on the prior nvvm.mbarrier.init + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar) + }]; + let arguments = (ins); + let results = (outs); +} +def NVVM_FenceProxyOp : NVVM_Op<"fence.proxy"> { + let description = [{ + Fence operation with proxy to establish an ordering between memory accesses + that may happen through different proxies. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar) + }]; + let arguments = (ins + NVVM_ProxyKindAttr:$kind, + OptionalAttr:$space + ); + let results = (outs); +} +def NVVM_FenceProxyAcquireOp : NVVM_Op<"fence.proxy.acquire"> { + let description = [{ + `fence.proxy.acquire` is a uni-directional fence used to establish ordering + between a prior memory access performed via the generic proxy and a + subsequent memory access performed via the tensormap proxy + + The address operand `addr` and the operand `size` together specify the + memory range `[addr, addr+size)` on which the ordering guarantees on the + memory accesses across the proxies is to be provided. The only supported + value for the `size` operand is 128 and must be an immediate. Generic Addressing + is used unconditionally, and the address specified by the operand `addr` must + fall within the `.global` state space. Otherwise, the behavior is undefined + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace0:$addr, + NVVM_T32BitSignlessInteger:$size, + NVVM_MemScopeKindAttr:$scope, + OptionalAttr:$fromProxy, + OptionalAttr:$toProxy + ); + let results = (outs); +} +def NVVM_FenceProxyReleaseOp : NVVM_Op<"fence.proxy.release"> { + let description = [{ + `fence.proxy.release` is a uni-directional fence used to establish ordering + between a prior memory access performed via the generic proxy and a + subsequent memory access performed via the tensormap proxy. `fence.proxy.release` + operation can form a release sequence that synchronizes with an acquire + sequence that contains the fence.proxy.acquire proxy fence operation + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar) + }]; + let arguments = (ins + NVVM_MemScopeKindAttr:$scope, + OptionalAttr:$fromProxy, + OptionalAttr:$toProxy + ); + let results = (outs); +} +def NVVM_FenceProxySyncRestrictOp : NVVM_Op<"fence.proxy.sync_restrict"> { + let description = [{ + The `nvvm.fence.proxy.sync_restrict` Op used to establish + ordering between a prior memory access performed between proxies. Currently, + the ordering is only supported between async and generic proxies. `sync_restrict` + restricts `acquire` memory semantics to `shared_cluster` and `release` memory + semantics to `shared_cta` with cluster scope. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar) + }]; + let arguments = (ins + NVVM_MemOrderKindAttr:$order, + OptionalAttr:$fromProxy, + OptionalAttr:$toProxy + ); + let results = (outs); +} +def NVVM_FenceScClusterOp : NVVM_Op<"fence.sc.cluster"> { + let arguments = (ins); + let results = (outs); +} +def NVVM_FenceSyncRestrictOp : NVVM_Op<"fence.sync_restrict"> { + let description = [{ + The `nvvm.fence.sync_restrict` Op restricts the class of memory + operations for which the fence instruction provides the memory ordering guarantees. + `sync_restrict` restricts `acquire` memory semantics to `shared_cluster` and + `release` memory semantics to `shared_cta` with cluster scope. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar) + }]; + let arguments = (ins + NVVM_MemOrderKindAttr:$order + ); + let results = (outs); +} +def NVVM_FmaOp : NVVM_InferredOp<"fma"> { + let description = [{ + The `nvvm.fma` operation performs floating point fused multiply-add of + three operands of the same type. + + The rounding mode is specified by the `rnd` attribute, saturation mode by + the `sat` attribute, flush-to-zero by the `ftz` attribute, and ReLU by the + `relu` attribute. + + Out-of-bounds (OOB) behavior is controlled by the `oob` attribute. `oob` + clamps the result to 0 if either of the operands is `OOB NaN` (see [Tensors](https://docs.nvidia.com/cuda/parallel-thread-execution/#tensors)). + + For more information, see PTX ISA: + - [floating point fused multiply-add](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-fma) + - [half-precision floating point fused multiply-add](https://docs.nvidia.com/cuda/parallel-thread-execution/#half-precision-floating-point-instructions-fma) + }]; + let arguments = (ins + NVVM_T16BitFloatOrBfloat16Type:$a, + NVVM_T16BitFloatOrBfloat16Type:$b, + NVVM_T16BitFloatOrBfloat16Type:$c, + NVVM_FmaRndAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$ftz, + OptionalAttr:$relu, + OptionalAttr:$oob + ); + let results = (outs + NVVM_T16BitFloatOrBfloat16Type:$res + ); +} +def NVVM_FMAPackedF32x2Op : NVVM_InferredOp<"fma.packed.f32x2"> { + let description = [{ + This Op performs FMA of floating-point vectors. + The inputs and outputs are of the form . The `rnd` attribute + specifies the rounding mode. The `ftz` attribute, when set, flushes the + subnormal inputs and results to sign-preserving zero. + This Op is supported from SM100 and onwards. + [For more information, see PTX ISA] + (TODO: Add public PTX link when available) + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues2:$srcA, + NVVM_VectorOf32BitFloatValues2:$srcB, + NVVM_VectorOf32BitFloatValues2:$srcC, + OptionalAttr:$rnd, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_VectorOf32BitFloatValues2:$res + ); +} +def NVVM_FMAPacked_F32x2_BF16x2_F32x2_F32x2Op : NVVM_Op<"fma.packed.f32x2.bf16x2.f32x2.f32x2"> { + let description = [{ + This Op performs FMA of a pair of floating-point values. + The first operand is a vector<2xbf16>, and the second and third operands + are vector<2xf32>. The result is a vector<2xf32>. The first input is converted to + a scalar of packed i32 where the 0th element of the vector forms the low-16 bits + and the 1st element occupies the high-16 bits. The second and third inputs are + converted to a scalar of packed i64 where the 0th element of the vector forms the + low-32 bits and the 1st element occupies the high-32 bits. + + The rounding mode can be one of the following: + - rn: round to nearest even + - rz: round toward zero + - rm: round toward negative infinity + - rp: round toward positive infinity + + Examples: + ```mlir + // Round to nearest even (default) + %0 = nvvm.fma.packed.f32x2.bf16x2.f32x2.f32x2 %srcA, %srcB, %srcC : vector<2xbf16>, vector<2xf32>, vector<2xf32> -> vector<2xf32> + + // Round toward negative infinity + %1 = nvvm.fma.packed.f32x2.bf16x2.f32x2.f32x2 %srcA, %srcB, %srcC {rnd = #nvvm.fp_rnd_mode} : vector<2xbf16>, vector<2xf32>, vector<2xf32> -> vector<2xf32> + ``` + }]; + let arguments = (ins + NVVM_VectorOfBfloat16TypeValuesOf:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB, + NVVM_VectorOf32BitFloatValues:$srcC, + OptionalAttr:$rnd + ); + let results = (outs + NVVM_VectorOf32BitFloatValues:$res + ); +} +def NVVM_FMAPacked_F32x2_F16x2_F32x2_F32x2Op : NVVM_Op<"fma.packed.f32x2.f16x2.f32x2.f32x2"> { + let description = [{ + This Op performs FMA of a pair of floating-point values. + The first operand is a vector<2xf16>, and the second and third operands + are vector<2xf32>. The result is a vector<2xf32>. The first input is converted to + a scalar of packed i32 where the 0th element of the vector forms the low-16 bits + and the 1st element occupies the high-16 bits. The second and third inputs are + converted to a scalar of packed i64 where the 0th element of the vector forms the + low-32 bits and the 1st element occupies the high-32 bits. + + The rounding mode can be one of the following: + - rn: round to nearest even + - rz: round toward zero + - rm: round toward negative infinity + - rp: round toward positive infinity + + Examples: + ```mlir + // Round to nearest even (default) + %0 = nvvm.fma.packed.f32x2.f16x2.f32x2.f32x2 %srcA, %srcB, %srcC : vector<2xf16>, vector<2xf32>, vector<2xf32> -> vector<2xf32> + + // Round toward negative infinity + %1 = nvvm.fma.packed.f32x2.f16x2.f32x2.f32x2 %srcA, %srcB, %srcC {rnd = #nvvm.fp_rnd_mode} : vector<2xf16>, vector<2xf32>, vector<2xf32> -> vector<2xf32> + ``` + }]; + let arguments = (ins + NVVM_VectorOf16BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB, + NVVM_VectorOf32BitFloatValues:$srcC, + OptionalAttr:$rnd + ); + let results = (outs + NVVM_VectorOf32BitFloatValues:$res + ); +} +def NVVM_MaxOp : NVVM_Op<"fmax"> { + let description = [{ + `nvvm.fmax` Op calculates maximum of given values. if `nan` attributes is + used, then the result is a canonical nan if any of the input is nan. + if `abs` attribute is used, the magnitude of destination operation d is + the maximum of absolute values of input arguments and for two operand + version, `xorsign` is enabled by default which sets the sign bit of the + result by XOR'ing the sign bits of the inputs. `ftz` modifiers flushes + the subnormal inputs and the results to sign preserving zero + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-max) + }]; + let arguments = (ins + NVVM_T32BitFloat:$a, + NVVM_T32BitFloat:$b, + Optional:$c, + UnitAttr:$ftz, + UnitAttr:$nan, + UnitAttr:$abs + ); + let results = (outs + NVVM_T32BitFloat2:$res + ); +} +def NVVM_MinOp : NVVM_Op<"fmin"> { + let description = [{ + `nvvm.fmin` Op calculates minimum of given values. if `nan` attributes is + used, then the result is a canonical nan if any of the input is nan. + if `abs` attribute is used, the magnitude of destination operation d is + the minimum of absolute values of input arguments and for two operand + version, `xorsign` is enabled by default which sets the sign bit of the + result by XOR'ing the sign bits of the inputs. `ftz` modifiers flushes + the subnormal inputs and the results to sign preserving zero. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-min) + }]; + let arguments = (ins + NVVM_T32BitFloat:$a, + NVVM_T32BitFloat:$b, + Optional:$c, + UnitAttr:$ftz, + UnitAttr:$nan, + UnitAttr:$abs + ); + let results = (outs + NVVM_T32BitFloat2:$res + ); +} +def NVVM_GriddepcontrolOp : NVVM_Op<"griddepcontrol"> { + let description = [{ + If the $kind attribute is set to `wait`, it causes the + executing thread to wait until all prerequisite grids in flight + have completed and all the memory operations from the prerequisite grids + are performed and made visible to the current grid. + + When the $kind is launch_dependents, it signals that specific dependents + the runtime system designated to react to this instruction can be scheduled + as soon as all other CTAs in the grid issue the same instruction or have + completed. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-griddepcontrol) + }]; + let arguments = (ins + NVVM_GridDepActionAttr:$kind + ); + let results = (outs); +} +def NVVM_InlinePtxOp : NVVM_Op<"inline_ptx", [AttrSizedOperandSegments]> { + let description = [{ + This op allows using PTX directly within the NVVM + dialect, while greatly simplifying llvm.inline_asm generation. It + automatically handles register size selection and sets the correct + read/write access for each operand. The operation leverages the + `BasicPtxBuilderInterface` to abstract away low-level details of + PTX assembly formatting. + + The `predicate` attribute is used to specify a predicate for the + PTX instruction. + + Example 1: Read-only Parameters + ```mlir + nvvm.inline_ptx "mbarrier.init.b64 [$0], $1;" (%barrier_gen, %count) : !llvm.ptr, i32 + + // Lowers to: + llvm.inline_asm has_side_effects asm_dialect = att + "mbarrier.init.b64 [$0], $1;", "l,r" %arg0, %arg2 : (!llvm.ptr, i32) -> () + ``` + + Example 2: Read-only and Write-only Parameters + ```mlir + %0 = nvvm.inline_ptx "ex2.approx.ftz.f32 $0, $1;" (%input) : f32 -> f32 + + // Lowers to: + %0 = llvm.inline_asm has_side_effects asm_dialect = att + "ex2.approx.ftz.f32 $0, $1;", "=f,f" %arg0 : (f32) -> f32 + ``` + + Example 3: Predicate Usage + ```mlir + nvvm.inline_ptx "mbarrier.init.b64 [$0], $1;" (%barrier_gen, %count), + predicate = %pred : !llvm.ptr, i32, i1 + + // Lowers to: + llvm.inline_asm has_side_effects asm_dialect = att + "@$2 mbarrier.init.b64 [$0], $1;", "l,r,b" %arg0, %arg2, %arg3 + : (!llvm.ptr, i32, i1) -> () + ``` + }]; + let arguments = (ins + Variadic:$readOnlyArgs, + Variadic:$readWriteArgs, + Optional:$predicate, + StrAttr:$ptxCode + ); + let results = (outs + Optional:$writeOnlyArgs + ); +} +def NVVM_LdMatrixOp : NVVM_InferredOp<"ldmatrix"> { + let arguments = (ins + NVVM_LLVMPointerType:$ptr, + I32Attr:$num, + NVVM_MMALayoutAttr:$layout, + OptionalAttr:$shape, + OptionalAttr:$srcFormat + ); + let results = (outs + NVVM_AnyType:$res + ); +} +def NVVM_ExtLoadOp : NVVM_Op<"load.ext"> { + let description = [{ + The `load.ext` operation is used to read from memory in generic, + global or shared address space. This Op explicitly supports + various cache properties and memory orderings associated with + the load operation. + + The `l2CacheHint` operand is optional, and it is used to specify cache + eviction policy that may be used during the memory access. + [For more information, see PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld) + }]; + let arguments = (ins + NVVM_LLVMPointerType:$addr, + Optional:$l2CacheHint, + OptionalAttr:$order, + OptionalAttr:$scope, + OptionalAttr:$prefetch, + OptionalAttr:$evict, + OptionalAttr:$cacheModifier, + OptionalAttr:$sharedSpace, + OptionalAttr:$unified + ); + let results = (outs + NVVM_LLVMTypeWithSize:$res + ); +} +def NVVM_Log2Op : NVVM_Op<"log2", [Pure]> { + let description = [{ + Computes a fast approximation of the base-2 logarithm of the input + value. The `ftz` attribute, when set, flushes subnormal inputs and + results to sign-preserving zero. + }]; + let arguments = (ins + NVVM_T32BitFloat:$src, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_T32BitFloat2:$res + ); +} +def NVVM_MapaOp : NVVM_Op<"mapa"> { + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$a, + NVVM_T32BitSignlessInteger:$b + ); + let results = (outs + NVVM_LLVMPointerInAddressSpace03:$res + ); +} +def NVVM_MatchSyncOp : NVVM_InferredOp<"match.sync"> { + let description = [{ + The `match.sync` op performs broadcast and compare of operand `val` across + all non-exited threads in `thread_mask` and returns a mask depending on the + kind and an optional predicate. + + The matching operation kinds are: + - `any`: Returns a mask corresponding to the non-exited threads in the + `thread_mask` that have the same value of operand `val`. + - `all`: Returns a mask and a predicate. If all non-exited threads in the + `thread_mask` have the same value of operand `val`, the predicate is set to + true and the mask corresponds to the non-exited threads in the + `thread_mask`. Otherwise, the predicate is set to false and the mask is 0. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-match-sync) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$thread_mask, + NVVM_T32BitSignlessIntegerOr64:$val, + NVVM_MatchSyncKindAttr:$kind + ); + let results = (outs + NVVM_T32BitSignlessIntegerOrLLVM:$res + ); +} +def NVVM_MBarrierArriveOp : NVVM_InferredOp<"mbarrier.arrive", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.mbarrier.arrive` operation performs an arrive-on operation on the + *mbarrier object* at the specified address. Uses the default `.release.cta` semantics. + This release pattern establishes memory ordering for operations occurring in program + order before this arrive instruction by making operations from the current thread + visible to subsequent operations in other threads within the CTA. When other threads + perform corresponding acquire operations (like 'mbarrier.test.wait'), they synchronize + with this release pattern. + + This operation causes the executing thread to signal its arrival at the barrier. + + - `res`: When the `space` is not shared_cluster, this operation returns an + opaque 64-bit value capturing the phase of the *mbarrier object* prior to + the arrive-on operation. The contents of this return value are + implementation-specific. An *mbarrier object* located in the shared_cluster + space cannot return a value. + + The operation takes the following operands: + - `addr`: A pointer to the memory location of the *mbarrier object*. The `addr` + must be a pointer to generic or shared_cta or shared_cluster memory. When it + is generic, the underlying address must be within the shared_cta memory space; + otherwise the behavior is undefined. + - `count`: This specifies the amount by which the pending arrival count is + decremented. If the `count` argument is not specified, the pending arrival + count is decremented by 1. + - `scope`: This specifies the set of threads that directly observe the memory + synchronizing effect of the `mbarrier.arrive` operation. + - `space`: This indicates the memory space where the mbarrier object resides. + - `relaxed`: When set to true, the `arrive` operation has relaxed memory semantics + and does not provide any ordering or visibility guarantees. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace04:$addr, + Optional:$count, + Optional:$multicastMask, + OptionalAttr:$scope, + OptionalAttr:$relaxed + ); + let results = (outs + Optional:$res + ); +} +def NVVM_MBarrierArriveExpectTxOp : NVVM_InferredOp<"mbarrier.arrive.expect_tx", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.mbarrier.arrive.expect_tx` operation performs an expect-tx operation + followed by an arrive-on operation on the *mbarrier object*. Uses the default + `.release.cta` semantics. This release pattern establishes memory ordering for + operations occurring in program order before this arrive instruction by making + operations from the current thread visible to subsequent operations in other + threads within the CTA. When other threads perform corresponding acquire operations + (like 'mbarrier.test.wait'), they synchronize with this release pattern. + + This operation first performs an expect-tx operation with the specified transaction + count, then performs an arrive-on operation with an implicit count of 1. The + expect-tx operation increases the expect-count of the *mbarrier object* by the + specified value (i.e. `txcount`), setting the current phase to expect and track + the completion of additional asynchronous transactions. + + The operation takes the following operands: + - `addr`: A pointer to the memory location of the *mbarrier object*. Uses generic + addressing, but the address must still be in the shared memory space. + - `txcount`: An unsigned integer specifying the expected transaction count + for the expect-tx operation. This represents the number of asynchronous transactions + expected to complete before the barrier phase completes. + - `scope`: This specifies the set of threads that directly observe the memory + synchronizing effect of the `mbarrier.test.wait` operation. + - `relaxed`: When set to true, the `arrive` operation has relaxed memory semantics + and does not provide any ordering or visibility guarantees. + - `predicate`: Optional predicate for conditional execution used only when lowering to + inline-ptx. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive-drop) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace04:$addr, + NVVM_T32BitSignlessInteger:$txcount, + Optional:$multicastMask, + Optional:$predicate, + OptionalAttr:$scope, + OptionalAttr:$relaxed + ); + let results = (outs + Optional:$res + ); +} +def NVVM_MBarrierArriveNocompleteOp : NVVM_Op<"mbarrier.arrive.nocomplete"> { + let description = [{ + The `nvvm.mbarrier.arrive.nocomplete` operation performs an arrive-on operation + on the *mbarrier object* with the guarantee that it will not cause the barrier to + complete its current phase. Uses the default `.release.cta` semantics. This release + pattern establishes memory ordering for operations occurring in program order before + this arrive instruction by making operations from the current thread visible to + subsequent operations in other threads within the CTA. When other threads perform + corresponding acquire operations (like 'mbarrier.test.wait'), they synchronize with + this release pattern. + + This operation causes the executing thread to signal its arrival at the barrier + with a specified count, but ensures that the barrier phase will not complete as + a result of this operation. The operation returns an opaque value that + captures the phase of the *mbarrier object* prior to the arrive-on operation. + + The operation takes the following operands: + - `addr`: A pointer to the memory location of the *mbarrier object*. The `addr` + must be a pointer to generic or shared::cta memory. When it is generic, the + underlying address must be within the shared::cta memory space; otherwise + the behavior is undefined. + - `count`: Integer specifying the count argument to the arrive-on operation. + Must be in the valid range as specified in the *mbarrier object* contents. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr, + NVVM_T32BitSignlessInteger:$count + ); + let results = (outs + NVVM_T64BitSignlessInteger2:$res + ); +} +def NVVM_MBarrierArriveDropOp : NVVM_InferredOp<"mbarrier.arrive_drop", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.mbarrier.arrive_drop` operation decrements the expected arrival + count of the *mbarrier object* by `count` and then performs an arrive-on + operation. When `count` is not specified, it defaults to 1. The decrement + of the expected arrival count applies to all the subsequent phases of the + *mbarrier object*. The remaining semantics are identical to those of the + `nvvm.mbarrier.arrive` operation. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive-drop) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace04:$addr, + Optional:$count, + Optional:$multicastMask, + OptionalAttr:$scope, + OptionalAttr:$relaxed + ); + let results = (outs + Optional:$res + ); +} +def NVVM_MBarrierArriveDropExpectTxOp : NVVM_InferredOp<"mbarrier.arrive_drop.expect_tx"> { + let description = [{ + The `nvvm.mbarrier.arrive_drop.expect_tx` operation is similar to the + `nvvm.mbarrier.arrive.expect_tx` operation except that it performs an + `arrive_drop` operation instead of only an `arrive` operation. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive-drop) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace04:$addr, + NVVM_T32BitSignlessInteger:$txcount, + Optional:$multicastMask, + OptionalAttr:$scope, + OptionalAttr:$relaxed + ); + let results = (outs + Optional:$res + ); +} +def NVVM_MBarrierArriveDropNocompleteOp : NVVM_Op<"mbarrier.arrive_drop.nocomplete"> { + let description = [{ + The `nvvm.mbarrier.arrive_drop.nocomplete` operation decrements the expected + arrival count of the *mbarrier object* by the amount `count` and then performs + an arrive-on operation on the *mbarrier object* with the guarantee that it + will not cause the barrier to complete its current phase. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive-drop) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr, + NVVM_T32BitSignlessInteger:$count + ); + let results = (outs + NVVM_T64BitSignlessInteger2:$res + ); +} +def NVVM_MBarrierCheckLayoutOp : NVVM_Op<"mbarrier.check.layout"> { + let description = [{ + The `nvvm.mbarrier.check.layout` operation returns `true` if the *mbarrier object* at `addr` + uses the given `layout`; otherwise it returns `false`. + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace3:$addr, + NVVM_MBarrierLayoutAttr:$layout + ); + let results = (outs + NVVM_T1BitSignlessInteger2:$res + ); +} +def NVVM_MBarrierCompleteTxOp : NVVM_Op<"mbarrier.complete_tx", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.mbarrier.complete_tx` operation decrements the transaction + count of the *mbarrier object* at `addr` by `txcount`. It also signals + the completion of asynchronous transactions that were tracked by the + current phase. The `scope` specifies the set of threads that can directly + observe the memory synchronizing effect of the `mbarrier.complete_tx` + operation. `CTA` and `CLUSTER` are the only allowed values for `scope`. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-complete-tx) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace32:$addr, + NVVM_T32BitSignlessInteger:$txcount, + Optional:$report, + Optional:$multicastMask, + OptionalAttr:$scope + ); + let results = (outs); +} +def NVVM_MBarrierExpectTxOp : NVVM_Op<"mbarrier.expect_tx"> { + let description = [{ + The `nvvm.mbarrier.expect_tx` operation increases the transaction count + of the mbarrier located at `addr` by `txcount` amount. The `scope` + specifies the set of threads that can directly observe the memory + synchronizing effect of the `mbarrier.expect_tx` operation. `CTA` + and `CLUSTER` are the only allowed values for `scope`. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-expect-tx) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace32:$addr, + NVVM_T32BitSignlessInteger:$txcount, + Optional:$multicastMask, + OptionalAttr:$scope + ); + let results = (outs); +} +def NVVM_MBarrierInitOp : NVVM_Op<"mbarrier.init"> { + let description = [{ + The `nvvm.mbarrier.init` operation initializes an *mbarrier object* at the specified + memory location. + + This operation initializes the *mbarrier object* with the following state: + - Current phase: 0 + - Expected arrival count: `count` + - Pending arrival count: `count` + - Transaction count (tx-count): 0 + + The operation takes the following operands: + - `addr`: A pointer to the memory location of the *mbarrier object*. The `addr` + must be a pointer to generic or shared::cta memory. When it is generic, the + underlying address must be within the shared::cta memory space; otherwise + the behavior is undefined. + - `count`: Integer specifying the number of threads that will participate in barrier + synchronization. Must be in the range [1, 2²⁰ - 1]. + - `predicate`: Optional predicate for conditional execution. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-init) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr, + NVVM_T32BitSignlessInteger:$count, + Optional:$predicate, + OptionalAttr:$layout + ); + let results = (outs); +} +def NVVM_MBarrierInvalOp : NVVM_Op<"mbarrier.inval"> { + let description = [{ + The `nvvm.mbarrier.inval` operation invalidates an *mbarrier object* at the + specified memory location. + + This operation marks the *mbarrier object* as invalid, making it safe to repurpose + the memory location for other uses or to reinitialize it as a new *mbarrier object*. + It is undefined behavior if the *mbarrier object* is already invalid. + + The operation takes the following operand: + - `addr`: A pointer to the memory location of the *mbarrier object*. The `addr` + must be a pointer to generic or shared::cta memory. When it is generic, the + underlying address must be within the shared::cta memory space; otherwise + the behavior is undefined. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-inval) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr + ); + let results = (outs); +} +def NVVM_MBarrierTestWaitOp : NVVM_InferredOp<"mbarrier.test.wait"> { + let description = [{ + The `nvvm.mbarrier.test.wait` operation performs a non-blocking test for the + completion of a specific phase of an *mbarrier object*. It uses the default + `.acquire.cta` semantics. This acquire pattern establishes memory ordering for + operations occurring in program order after this wait instruction by making + operations from other threads in the CTA visible to subsequent operations in the current + thread. When this wait completes, it synchronizes with the corresponding release + pattern from the `mbarrier.arrive` operation, establishing memory ordering within + the CTA. + + This operation tests whether the mbarrier phase specified by the state operand + has completed. It is a non-blocking instruction that immediately returns the + completion status without suspending the executing thread. + + The operation takes the following operands: + - `addr`: A pointer to the memory location of the *mbarrier object*. Uses generic + addressing, but the address must still be in the shared memory space. + - `stateOrPhase`: This argument represents a `state` when it is a 64-bit value + and represents a `phase` when it is a 32-bit value. The `state` is an opaque + value returned by a previous `mbarrier.arrive` operation on the same + *mbarrier object* during the current or immediately preceding phase. + The `phase` is an integer specifying the phase parity (0 or 1). + Even phases have parity 0, odd phases have parity 1. + - `scope`: This specifies the set of threads that directly observe the memory + synchronizing effect of the `mbarrier.test.wait` operation. + - `relaxed`: When set to true, the `arrive` operation has relaxed memory semantics + and does not provide any ordering or visibility guarantees. + + The operation returns a boolean value indicating whether the specified phase + has completed: + - `true`: The immediately preceding phase has completed + - `false`: The phase is still incomplete (current phase) + + The primary-phase variants the result is an LLVM struct whose first + field (`res`) is the `i1` completion flag. An optional second field + (`isReportSeen`, `i1`) indicates whether a report was observed, and an + optional third field (`report`, `i32`) carries the report payload. + + **Memory ordering guarantees**: When this wait returns true, the following + ordering guarantees hold: + + 1. All memory accesses (except async operations) requested prior to + `mbarrier.arrive` having release semantics by participating CTA threads + are visible to the executing thread. + 2. All `cp.async` operations requested prior to `cp.async.mbarrier.arrive` + by participating CTA threads are visible to the executing thread. + 3. All `cp.async.bulk` operations using the same *mbarrier object* requested + prior to `mbarrier.arrive` having release semantics by participating CTA + threads are visible to the executing thread. + 4. Memory accesses requested after this wait are not visible to memory + accesses performed prior to `mbarrier.arrive` by other participating + threads. + 5. No ordering guarantee exists for memory accesses by the same thread + between `mbarrier.arrive` and this wait. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr, + NVVM_T64BitSignlessIntegerOr32:$stateOrPhase, + OptionalAttr:$scope, + OptionalAttr:$relaxed, + OptionalAttr:$mbarrierPhase + ); + let results = (outs + NVVM_T1BitSignlessIntegerOrLLVM:$res + ); +} +def NVVM_MBarrierTryWaitOp : NVVM_InferredOp<"mbarrier.try_wait"> { + let description = [{ + The `nvvm.mbarrier.try_wait` operation checks whether the specified + *mbarrier object* at `addr` has completed the given phase. Note that + unlike the `nvvm.mbarrier.test.wait` operation, the try_wait operation + is a potentially-blocking one. If the phase is not yet complete, the + calling thread may be suspended. A suspended thread resumes execution + once the phase completes or when a system-defined timeout occurs. + Optionally, the `ticks` operand can be used to provide a custom timeout + (in nanoseconds), overriding the system-defined one. The semantics of + this operation and its operands are otherwise similar to those of the + `nvvm.mbarrier.test.wait` Op. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait) + + The primary-phase variants the result is an LLVM struct whose first + field (`res`) is the `i1` completion flag. An optional second field + (`isReportSeen`, `i1`) indicates whether a report was observed, and an + optional third field (`report`, `i32`) carries the report payload. + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr, + NVVM_T64BitSignlessIntegerOr32:$stateOrPhase, + Optional:$ticks, + OptionalAttr:$scope, + OptionalAttr:$relaxed, + OptionalAttr:$mbarrierPhase + ); + let results = (outs + NVVM_T1BitSignlessIntegerOrLLVM:$res + ); +} +def NVVM_MBarrierTryWaitParityOp : NVVM_Op<"mbarrier.try_wait.parity"> { + let description = [{ + The `nvvm.mbarrier.try_wait.parity` operation performs a potentially-blocking + test for the completion of a specific phase of an *mbarrier object* using phase + parity. It uses the default `.acquire.cta` semantics. This acquire pattern + establishes memory ordering for operations occurring in program order after this + wait instruction by making operations from other threads in the CTA visible to subsequent + operations in the current thread. When this wait completes, it synchronizes with + the corresponding release pattern from the `mbarrier.arrive` operation, establishing + memory ordering within the CTA. + + This operation waits for the completion of the mbarrier phase indicated by the + phase parity. While it uses the underlying PTX `mbarrier.try_wait.parity` + instruction, this MLIR operation generates a loop that enforces the test to + complete before continuing execution, ensuring the barrier phase is actually + completed rather than potentially timing out. + + The operation takes the following operands: + - `addr`: A pointer to the memory location of the *mbarrier object*. Uses generic + addressing, but the address must still be in the shared memory space. + - `phase`: An integer specifying the phase parity (0 or 1). Even phases + have parity 0, odd phases have parity 1. + - `ticks`: An unsigned integer specifying the suspend time hint in + nanoseconds. This may be used instead of the system-dependent time limit. + + **Memory ordering guarantees**: When this wait returns true, the following + ordering guarantees hold: + + 1. All memory accesses (except async operations) requested prior to + `mbarrier.arrive` having release semantics by participating CTA threads + are visible to the executing thread. + 2. All `cp.async` operations requested prior to `cp.async.mbarrier.arrive` + by participating CTA threads are visible to the executing thread. + 3. All `cp.async.bulk` operations using the same *mbarrier object* requested + prior to `mbarrier.arrive` having release semantics by participating CTA + threads are visible to the executing thread. + 4. Memory accesses requested after this wait are not visible to memory + accesses performed prior to `mbarrier.arrive` by other participating + threads. + 5. No ordering guarantee exists for memory accesses by the same thread + between `mbarrier.arrive` and this wait. + + **Implementation behavior**: + This operation generates a PTX loop that repeatedly calls the underlying + `mbarrier.try_wait.parity` instruction until the barrier phase completes. + Unlike the raw PTX instruction which may return without completion after a + timeout, this MLIR operation guarantees completion by continuing to loop until + the specified phase is reached. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr, + NVVM_T32BitSignlessInteger:$phase, + NVVM_T32BitSignlessInteger:$ticks, + OptionalAttr:$useIntrinsic + ); + let results = (outs); +} +def NVVM_MBarrierTryWaitParityTimeLimitOp : NVVM_Op<"mbarrier.try_wait.parity.timelimit"> { + let description = [{ + The `mbarrier.try_wait.parity.timelimit` is a potentially blocking + instruction which tests for the completion of the phase. If + the phase is not complete, the executing thread may be suspended. + Suspended thread resumes execution when the specified phase completes + or a system-dependent time limit elapses. The 32-bit `timeLimit` operand + specifies the time limit, in nanoseconds, that may be used instead of + the system-dependent limit. The `addr` operand specifies the address + of the mbarrier in generic or shared memory space. The `phase` operand + is the integer parity of either the current phase or the immediately + preceding phase of the mbarrier object. The `order` denotes the + ordering semantics for the memory consistency model. This Op only + supports 'acquire' or 'relaxed' ordering. + [For more information refer PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-mbarrier-try-wait) + }]; + let arguments = (ins + NVVM_LLVMPointerTypeOrLLVMPointer:$addr, + NVVM_T32BitSignlessInteger:$phase, + NVVM_T32BitSignlessInteger:$timeLimit, + OptionalAttr:$scope, + OptionalAttr:$order + ); + let results = (outs + NVVM_T1BitSignlessInteger2:$res + ); +} +def NVVM_MBarrierTryWaitTimeLimitOp : NVVM_Op<"mbarrier.try_wait.timelimit"> { + let description = [{ + The `mbarrier.try_wait.timelimit` is a potentially blocking + instruction which tests for the completion of the phase. If + the phase is not complete, the executing thread may be suspended. + Suspended thread resumes execution when the specified phase completes + or a system-dependent time limit elapses. The 32-bit `timeLimit` operand + specifies the time limit, in nanoseconds, that may be used instead of + the system-dependent limit. The `addr` operand specifies the address + of the mbarrier in generic or shared memory space. The `state` operand + is the captured state of the mbarrier object from prior operations. + The `order` denotes the ordering semantics for the memory consistency + model. This Op only supports 'acquire' or 'relaxed' ordering. + [For more information refer PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-mbarrier-try-wait) + }]; + let arguments = (ins + NVVM_LLVMPointerTypeOrLLVMPointer:$addr, + NVVM_T64BitSignlessInteger:$state, + NVVM_T32BitSignlessInteger:$timeLimit, + OptionalAttr:$scope, + OptionalAttr:$order + ); + let results = (outs + NVVM_T1BitSignlessInteger2:$res + ); +} +def NVVM_MBarrierTransactionOp : NVVM_Op<"mbarrier.txn"> { + let description = [{ + The `mbarrier.txn` Op exposes the transactions that can be done on an MBarrier object. + + Operands: + - `addr`: Address of the mbarrier in generic or shared memory space + - `count`: Transaction count for the given kind + + - `multicast`: (Optional) Multicast mask for cluster space operations + + + Attributes: + - `kind`: One of the transaction kinds from MBarrierTxnKind + - `space`: Memory space (CTA or CLUSTER) where MBarrier object is located + - `scope`: Scope of synchronization (CTA or CLUSTER) + - `order`: Memory ordering semantics (release or relaxed only) + + This Op supports MBarrier objects in both shared::CTA and shared::CLUSTER spaces. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-mbarrier) + }]; + let arguments = (ins + NVVM_LLVMPointerTypeOrLLVMPointer:$addr, + NVVM_T32BitSignlessInteger:$count, + Optional:$multicast, + NVVM_MBarrierTxnKindAttr:$kind, + OptionalAttr:$space, + OptionalAttr:$scope, + OptionalAttr:$order + ); + let results = (outs); +} +def NVVM_MBarrierTransactionCTASpaceOp : NVVM_Op<"mbarrier.txn.cta"> { + let description = [{ + The `mbarrier.txn.cta` Op exposes the transactions that can be done + using an MBarrier object. The `addr` operand specifies the address of + the mbarrier in generic or shared memory space. The `kind` is one of + the various transaction kinds supported in `MBarrierTxnKind`. The + `count` argument is the transaction count for a given `kind`. The + `order` denotes the ordering semantics for the memory consistency + model. This Op only supports 'release' or 'relaxed' ordering. This + Op supports only MBarrier objects located in shared::CTA space (but + not in shared::CLUSTER space) and hence the 'CTASpace' suffix in the + naming. This Op returns an opaque 64-bit value capturing the phase of + the mbarrier object prior to the transaction. + [For more information on 'order/space/scope/kind-of-transactions' refer PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier) + }]; + let arguments = (ins + NVVM_LLVMPointerTypeOrLLVMPointer:$addr, + NVVM_T32BitSignlessInteger:$count, + NVVM_MBarrierTxnKindAttr:$kind, + OptionalAttr:$scope, + OptionalAttr:$order + ); + let results = (outs + NVVM_T64BitSignlessInteger2:$res + ); +} +def NVVM_MBarrierWaitOp : NVVM_Op<"mbarrier.wait"> { + let description = [{ + The `mbarrier.wait` Op exposes the try/test wait operations that + can be performed on MBarrier object. The `addr` operand specifies + the address of the mbarrier in generic or shared memory space. + The `state` operand is the captured state of the mbarrier object + from prior operations. The `kind` is one of TEST_WAIT or TRY_WAIT. + The `order` denotes the ordering semantics for the memory consistency + model. This Op only supports 'acquire' or 'relaxed' ordering. + [For more information refer PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-mbarrier-try-wait) + }]; + let arguments = (ins + NVVM_LLVMPointerTypeOrLLVMPointer:$addr, + NVVM_T64BitSignlessInteger:$state, + NVVM_MBarrierWaitKindAttr:$kind, + OptionalAttr:$scope, + OptionalAttr:$order + ); + let results = (outs + NVVM_T1BitSignlessInteger2:$res + ); +} +def NVVM_MBarrierWaitParityOp : NVVM_Op<"mbarrier.wait.parity"> { + let description = [{ + The `mbarrier.wait` Op exposes the try/test wait operations that + can be performed on MBarrier object. The `addr` operand specifies + the address of the mbarrier in generic or shared memory space. + The `phase` operand is the integer parity of either the current + phase or the immediately preceding phase of the mbarrier object. + The `kind` is one of TEST_WAIT or TRY_WAIT. + The `order` denotes the ordering semantics for the memory consistency + model. This Op only supports 'acquire' or 'relaxed' ordering. + [For more information refer PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-mbarrier-try-wait) + }]; + let arguments = (ins + NVVM_LLVMPointerTypeOrLLVMPointer:$addr, + NVVM_T32BitSignlessInteger:$phase, + NVVM_MBarrierWaitKindAttr:$kind, + OptionalAttr:$scope, + OptionalAttr:$order + ); + let results = (outs + NVVM_T1BitSignlessInteger2:$res + ); +} +def NVVM_MembarOp : NVVM_Op<"memory.barrier"> { + let description = [{ + `membar` operation guarantees that prior memory accesses requested by this + thread are performed at the specified `scope`, before later memory + operations requested by this thread following the membar instruction. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-membar) + }]; + let arguments = (ins + NVVM_MemScopeKindAttr:$scope + ); + let results = (outs); +} +def NVVM_MmaBlockScaleOp : NVVM_Op<"mma.block_scale", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.mma.block_scale` operation collectively performs the operation + `D = matmul(A * SF_A, B * SF_B) + C` using all threads in a warp. + + A, B, C and D are dense matrices and SF_A and SF_B are scaling factors. + Dimensions of SF_A and SF_B are based on scale vector sizes (x1, x2, x4), + and the data type must be either ue8m0 or ue4m3. + + All the threads in the warp must execute the same `mma.block_scale` operation. + + This operation follows the same design pattern as `nvvm.mma.sync`, with additional + scaling operands for both A and B matrices. + + Example: + ```mlir + %d = nvvm.mma.block_scale A[%a0, %a1] B[%b0, %b1] C[%c0, %c1] + scaleA[%scaleAData, %byteIdA, %threadIdA] + scaleB[%scaleBData, %byteIdB, %threadIdB] + {shape = #nvvm.shape, + multiplicandAPtxType = #nvvm.mma_type, + multiplicandBPtxType = #nvvm.mma_type, + scaleVecSize = #nvvm.scale_vec_size, + blockScaleFormat = #nvvm.block_scale_format, + kind = #nvvm.block_scale_kind} + : (vector<4xf16>, vector<2xf16>, vector<2xf32>) -> !llvm.struct<(f32, f32)> + ``` + }]; + let arguments = (ins + Variadic:$operandA, + Variadic:$operandB, + Variadic:$operandC, + NVVM_T32BitSignlessInteger:$scaleAData, + NVVM_T16BitSignlessInteger:$byteIdA, + NVVM_T16BitSignlessInteger:$threadIdA, + NVVM_T32BitSignlessInteger:$scaleBData, + NVVM_T16BitSignlessInteger:$byteIdB, + NVVM_T16BitSignlessInteger:$threadIdB, + NVVM_MMAShapeAttr:$shape, + OptionalAttr:$multiplicandAPtxType, + OptionalAttr:$multiplicandBPtxType, + NVVM_ScaleVecSizeAttr:$scaleVecSize, + NVVM_BlockScaleFormatAttr:$blockScaleFormat, + NVVM_MMABlockScaleKindAttr:$kind + ); + let results = (outs + NVVM_LLVMStructureType:$res + ); +} +def NVVM_InternalMmaBlockScaleOp : NVVM_InferredOp<"mma.block_scale.internal"> { + let description = [{ + The `nvvm.mma.blockscale.internal` operation collectively performs the operation + `D = matmul(A * SF_A, B * SF_B) + C` using all threads in a warp. + + A, B, C and D are dense matrices and SF_A and SF_B are scaling factor. + Dimensions of SF_A and SF_B are based on scale vector sizes (X1, X2, X4), + the data type of must be either ue8m0 or ue4m3 + + Example: + ```mlir + + %0 = nvvm.mma.block_scale.internal %operandA, %scaleAData, %byteIdA, %threadIdA, + %operandB, %scaleBData, %byteIdB, %threadIdB, + %operandC + { + layoutA = #nvvm.mma_layout, + layoutB = #nvvm.mma_layout, + shape = #nvvm.shape, + aType = #nvvm.mma_type, + bType = #nvvm.mma_type, + cType = #nvvm.mma_type, + scaleVecSize = #nvvm.scale_vec_size, + blockScaleFormat = #nvvm.block_scale_format + kind = #nvvm.block_scale_kind + } + ``` + }]; + let arguments = (ins + NVVM_VectorOfAnyTypeValues:$operandA, + NVVM_T32BitSignlessInteger:$scaleAData, + NVVM_T16BitSignlessInteger:$byteIdA, + NVVM_T16BitSignlessInteger:$threadIdA, + NVVM_VectorOfAnyTypeValues:$operandB, + NVVM_T32BitSignlessInteger:$scaleBData, + NVVM_T16BitSignlessInteger:$byteIdB, + NVVM_T16BitSignlessInteger:$threadIdB, + NVVM_VectorOfAnyTypeValues:$operandC, + NVVM_MMAShapeAttr:$shape, + NVVM_MMALayoutAttr:$layoutA, + NVVM_MMALayoutAttr:$layoutB, + NVVM_MMATypesAttr:$aType, + NVVM_MMATypesAttr:$bType, + NVVM_MMATypesAttr:$cType, + NVVM_ScaleVecSizeAttr:$scaleVecSize, + NVVM_BlockScaleFormatAttr:$blockScaleFormat, + OptionalAttr:$kind + ); + let results = (outs + NVVM_VectorOfAnyTypeValues:$res + ); +} +def NVVM_MmaSpBlockScaleOp : NVVM_Op<"mma.sp.block_scale", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.mma.sp.block_scale` operation collectively performs the operation + `D = matmul(A_sparse * SF_A, B * SF_B) + C` using all threads in a warp. + + A is a sparse matrix, and B, C and D are dense matrices. + SF_A and SF_B are scaling factors. + Dimensions of SF_A and SF_B are based on scale vector sizes (x1, x2, x4), + and the data type must be either ue8m0 or ue4m3. + + This operation is similar to `nvvm.mma.block_scale` but with structured sparsity + in the A operand. The sparsity follows the 2:4 structured sparse pattern + where 2 out of every 4 elements are non-zero. + + All the threads in the warp must execute the same `mma.sp.block_scale` operation. + + The `sparseMetadata` operand provides the sparsity indices that indicate + which elements in the A operand are non-zero. The `sparsitySelector` + controls how the indices are distributed among threads in the warp and + should typically be 0 or 1. + + This operation follows the same design pattern as `nvvm.mma.sp.sync`, with additional + scaling operands for both A and B matrices. Note that sparse block scale operations + always use ordered metadata (sm_90+). + + Example: + ```mlir + %d = nvvm.mma.sp.block_scale A[%a0, %a1] B[%b0, %b1] C[%c0, %c1] + sparseMetadata[%meta] selector[%sel] + scaleA[%scaleAData, %byteIdA, %threadIdA] + scaleB[%scaleBData, %byteIdB, %threadIdB] + {shape = #nvvm.shape, + multiplicandAPtxType = #nvvm.mma_type, + multiplicandBPtxType = #nvvm.mma_type, + scaleVecSize = #nvvm.scale_vec_size, + blockScaleFormat = #nvvm.block_scale_format, + kind = #nvvm.block_scale_kind} + : (vector<2xf16>, vector<2xf16>, vector<2xf32>) -> !llvm.struct<(f32, f32)> + ``` + }]; + let arguments = (ins + Variadic:$operandA, + Variadic:$operandB, + Variadic:$operandC, + NVVM_T32BitSignlessInteger:$sparseMetadata, + NVVM_T32BitSignlessInteger:$sparsitySelector, + NVVM_T32BitSignlessInteger:$scaleAData, + NVVM_T16BitSignlessInteger:$byteIdA, + NVVM_T16BitSignlessInteger:$threadIdA, + NVVM_T32BitSignlessInteger:$scaleBData, + NVVM_T16BitSignlessInteger:$byteIdB, + NVVM_T16BitSignlessInteger:$threadIdB, + NVVM_MMAShapeAttr:$shape, + OptionalAttr:$multiplicandAPtxType, + OptionalAttr:$multiplicandBPtxType, + NVVM_ScaleVecSizeAttr:$scaleVecSize, + NVVM_BlockScaleFormatAttr:$blockScaleFormat, + NVVM_MMABlockScaleKindAttr:$kind, + UnitAttr:$orderedMetadata + ); + let results = (outs + NVVM_LLVMStructureType:$res + ); +} +def NVVM_InternalMmaSparseBlockScaleOp : NVVM_InferredOp<"mma.sp.block_scale.internal"> { + let description = [{ + The `nvvm.mma.sparse.blockscale` operation collectively performs the + operation `D = matmul(A * SF_A, B * SF_B) + C` using all threads in a warp. + + Where A is structued sparse with atleast 50% zeros and B, C and D are + dense and SF_A and SF_B are scaling factor. Dimensions of SF_A and SF_B + are based on scale vector sizes (X1, X2, X4), the data type of + must be either ue8m0 or ue4m3 + + Example: + ```mlir + + %selector = llvm.mlir.constant(0) + %0 = nvvm.mma.sparse.block_scale + %operandA, %scaleAData, %byteIdA, %threadIdA, + %operandB, %scaleBData, %byteIdB, %threadIdB, + %operandC, %spMetadata, %selector + { + layoutA = #nvvm.mma_layout, + layoutB = #nvvm.mma_layout, + shape = #nvvm.shape, + aType = #nvvm.mma_type, + bType = #nvvm.mma_type, + cType = #nvvm.mma_type, + scaleVecSize = #nvvm.scale_vec_size, + blockScaleFormat = #nvvm.block_scale_format, + sparityFormat = #nvvm.sparsity_format + } + ``` + }]; + let arguments = (ins + NVVM_VectorOfAnyTypeValues:$operandA, + NVVM_T32BitSignlessInteger:$scaleAData, + NVVM_T16BitSignlessInteger:$byteIdA, + NVVM_T16BitSignlessInteger:$threadIdA, + NVVM_VectorOfAnyTypeValues:$operandB, + NVVM_T32BitSignlessInteger:$scaleBData, + NVVM_T16BitSignlessInteger:$byteIdB, + NVVM_T16BitSignlessInteger:$threadIdB, + NVVM_VectorOfAnyTypeValues:$operandC, + NVVM_T32BitSignlessInteger:$sp_metadata, + NVVM_T32BitSignlessInteger:$selector, + NVVM_MMAShapeAttr:$shape, + OptionalAttr:$b1Op, + OptionalAttr:$intOverflowBehavior, + NVVM_MMALayoutAttr:$layoutA, + NVVM_MMALayoutAttr:$layoutB, + NVVM_MMATypesAttr:$aType, + NVVM_MMATypesAttr:$bType, + NVVM_MMATypesAttr:$cType, + NVVM_ScaleVecSizeAttr:$scaleVecSize, + NVVM_BlockScaleFormatAttr:$blockScaleFormat, + NVVM_SparsityFormatAttr:$sparsityFormat + ); + let results = (outs + NVVM_VectorOfAnyTypeValues:$res + ); +} +def NVVM_MmaSpOp : NVVM_Op<"mma.sp.sync", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.mma.sp.sync` operation collectively performs the sparse operation + `D = matmul(A_sparse, B) + C` using all threads in a warp. + + This operation is similar to `nvvm.mma.sync` but with structured sparsity + in the A operand. The sparsity follows the 2:4 structured sparse pattern + where 2 out of every 4 elements are non-zero. + + All the threads in the warp must execute the same `mma.sp.sync` operation. + + The `sparseMetadata` operand provides the sparsity indices that indicate + which elements in the A operand are non-zero. The `sparsitySelector` + controls how the indices are distributed among threads in the warp and + should typically be 0 or 1. + + The optional `orderedMetadata` attribute specifies the metadata ordering: + - Absence (default): Uses standard sparse metadata ordering + - Presence: Uses ordered metadata (PTX ISA 8.5+, sm_90+) + + The optional `kind` attribute specifies mixed-precision modes for FP8 operations: + - `f8f6f4`: Enables e3m2, e2m3, e2m1 FP8 types and f16 accumulator (PTX ISA 8.7+, sm_90+) + - Only valid with ordered metadata and m16n8k64 shape + + The shapes, layouts, and data types follow the same constraints as the + regular `nvvm.mma.sync` operation, but the A operand contains only the + non-zero elements in compressed format. + + Example: + ```mlir + %d = nvvm.mma.sp.sync A[%a0, %a1] B[%b0, %b1] C[%c0, %c1] + sparseMetadata[%meta] selector[%sel] + {shape = {k = 32 : i32, m = 16 : i32, n = 8 : i32}} + : (vector<2xf16>, vector<2xf16>, vector<2xf16>) -> !llvm.struct<(vector<2xf16>, vector<2xf16>)> + + // With ordered metadata: + %d = nvvm.mma.sp.sync A[%a0, %a1] B[%b0, %b1] C[%c0, %c1] + sparseMetadata[%meta] selector[%sel] + {orderedMetadata, shape = {k = 32 : i32, m = 16 : i32, n = 8 : i32}} + : (vector<2xf16>, vector<2xf16>, vector<2xf16>) -> !llvm.struct<(vector<2xf16>, vector<2xf16>)> + ``` + }]; + let arguments = (ins + Variadic:$operandA, + Variadic:$operandB, + Variadic:$operandC, + NVVM_T32BitSignlessInteger:$sparseMetadata, + NVVM_T32BitSignlessInteger:$sparsitySelector, + NVVM_MMAShapeAttr:$shape, + OptionalAttr:$intOverflowBehavior, + OptionalAttr:$multiplicandAPtxType, + OptionalAttr:$multiplicandBPtxType, + UnitAttr:$orderedMetadata, + OptionalAttr:$kind + ); + let results = (outs + NVVM_LLVMStructureType:$res + ); +} +def NVVM_MmaOp : NVVM_Op<"mma.sync", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.mma.sync` operation collectively performs the operation + `D = matmul(A, B) + C` using all threads in a warp. + + All the threads in the warp must execute the same `mma.sync` operation. + + For each possible multiplicand PTX data type, there are one or more possible + instruction shapes given as "mMnNkK". The below table describes the posssibilities + as well as the types required for the operands. Note that the data type for + C (the accumulator) and D (the result) can vary independently when there are + multiple possibilities in the "C/D Type" column. + + When an optional attribute cannot be immediately inferred from the types of + the operands and the result during parsing or validation, an error will be + raised. + + `b1Op` is only relevant when the binary (b1) type is given to + `multiplicandDataType`. It specifies how the multiply-and-acumulate is + performed and is either `xor_popc` or `and_poc`. The default is `xor_popc`. + + `intOverflowBehavior` is only relevant when the `multiplicandType` attribute + is one of `u8, s8, u4, s4`, this attribute describes how overflow is handled + in the accumulator. When the attribute is `satfinite`, the accumulator values + are clamped in the int32 range on overflow. This is the default behavior. + Alternatively, accumulator behavior `wrapped` can also be specified, in + which case overflow wraps from one end of the range to the other. + + `layoutA` and `layoutB` are required and should generally be set to + `#nvvm.mma_layout` and `#nvvm.mma_layout` respectively, but other + combinations are possible for certain layouts according to the table below. + + ``` + | A/B Type | Shape | ALayout | BLayout | A Type | B Type | C/D Type | + |----------|-----------|---------|---------|----------|----------|-------------------| + | f64 | .m8n8k4 | row | col | 1x f64 | 1x f64 | 2x f64 | + | f16 | .m8n8k4 | row/col | row/col | 2x f16x2 | 2x f16x2 | 4x f16x2 or 8xf32 | + | | .m16n8k8 | row | col | 2x f16x2 | 1x f16x2 | 2x f16x2 or 4 f32 | + | | .m16n8k16 | row | col | 4x f16x2 | 2x f16x2 | 2x f16x2 or 4 f32 | + | bf16 | .m16n8k8 | row | col | 2x i32 | 1x i32 | 4x f32 | + | | .m16n8k16 | row | col | 4x i32 | 2x i32 | 4x f32 | + | tf32 | .m16n8k4 | row | col | 2x i32 | 1x i32 | 4x f32 | + | | .m16n8k8 | row | col | 4x i32 | 2x i32 | 2x f16x2 or 4 f32 | + | u8/s8 | .m8n8k16 | row | col | 1x i32 | 1x i32 | 2x i32 | + | | .m16n8k16 | row | col | 2x i32 | 1x i32 | 4x i32 | + | | .m16n8k32 | row | col | 4x i32 | 2x i32 | 4x i32 | + | u4/s4 | .m8n8k32 | row | col | 1x i32 | 1x i32 | 2x i32 | + | | m16n8k32 | row | col | 2x i32 | 1x i32 | 4x i32 | + | | m16n8k64 | row | col | 4x i32 | 2x i32 | 4x i32 | + | b1 | m8n8k128 | row | col | 1x i32 | 1x i32 | 2x i32 | + | | m16n8k128 | row | col | 2x i32 | 1x i32 | 4x i32 | + ``` + + Example: + ```mlir + + %128 = nvvm.mma.sync A[%120, %121, %122, %123] + B[%124, %125] + C[%126, %127] + {layoutA = #nvvm.mma_layout, + layoutB = #nvvm.mma_layout, + shape = {k = 16 : i32, m = 16 : i32, n = 8 : i32}} + : (vector<2xf16>, vector<2xf16>, vector<2xf16>) + -> !llvm.struct<(vector<2xf16>, vector<2xf16>)> + ``` + }]; + let arguments = (ins + Variadic:$operandA, + Variadic:$operandB, + Variadic:$operandC, + NVVM_MMAShapeAttr:$shape, + OptionalAttr:$b1Op, + OptionalAttr:$intOverflowBehavior, + NVVM_MMALayoutAttr:$layoutA, + NVVM_MMALayoutAttr:$layoutB, + OptionalAttr:$multiplicandAPtxType, + OptionalAttr:$multiplicandBPtxType + ); + let results = (outs + NVVM_LLVMStructureType:$res + ); +} +def NVVM_MmaSmemDescOp : NVVM_Op<"mma_smem_desc", [MemoryEffects<[MemRead]>]> { + let description = [{ + The `nvvm.mma_smem_desc` constructs a Shared Memory descriptor for use as the MMA Shared Memory operand. + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$pointer, + NVVM_T32BitSignlessInteger:$ldm, + NVVM_T32BitSignlessInteger:$stride, + NVVM_T8BitSignlessInteger2:$baseOffset, + NVVM_T8BitSignlessInteger2:$swizzle, + OptionalAttr:$mmaDescVersion + ); + let results = (outs + NVVM_T64BitSignlessInteger2:$res + ); +} +def NVVM_MovMatrixOp : NVVM_Op<"movmatrix"> { + let description = [{ + Moves a row-major matrix across all threads in a warp, reading elements + from source `$src`, and writing the transposed elements to destination + `$dst`. + + The `shape` attribute indicates the dimensions of the matrix being + transposed. Each matrix element holds 16-bit data as indicated by the + `eltType` attribute. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#warp-level-matrix-movmatrix-instruction) + + Example: + ```mlir + %dst = nvvm.movmatrix %src {shape = #nvvm.ld_st_matrix_shape, + eltType = #nvvm.ld_st_matrix_elt_type} : i32 + ``` + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$src, + NVVM_LdStMatrixShapeAttr:$shape, + OptionalAttr:$layout, + NVVM_LdStMatrixEltTypeAttr:$eltType + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$dst + ); +} +def NVVM_MulOp : NVVM_InferredOp<"mul"> { + let description = [{ + nvvm.mul Op multiplies two values with a given `type` and `mode`. + The type of the operation represents the types of the `a` and `b` operands. + If `.hi` or `.lo` is specified, then `res` is the same size as `a` and `b`, + and either the upper or lower half of the result is written to the + destination register. If `.wide` is specified, then `res` is twice as + wide as `a` and `b` to receive the full result of the multiplication. The + `.wide` suffix is supported only for 16- and 32-bit integer types. + For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#integer-arithmetic-instructions-mul) + }]; + let arguments = (ins + NVVM_T16BitSignlessIntegerOr323:$a, + NVVM_T16BitSignlessIntegerOr323:$b, + NVVM_MulModeAttr:$mode, + OptionalAttr:$isSigned + ); + let results = (outs + NVVM_T16BitSignlessIntegerOr323:$res + ); +} +def NVVM_MulPacked_BF16x2_BF16x2_F16x2Op : NVVM_Op<"mul.packed.bf16x2.bf16x2.f16x2"> { + let description = [{ + This Op performs the multiplication operation on a pair of "bf16"x2 and "f16"x2 vectors. + The inputs are of the form v2bf16 and v2f16, and the output is of the form v2bf16. + The inputs are converted to a scalar of packed i32 where the 0th element of the + vector forms the low-16 bits and the 1st element occupies the high-16 bits. + + Examples: + ```mlir + %0 = nvvm.mul.packed.bf16x2.bf16x2.f16x2 %srcA, %srcB : vector<2xbf16>, vector<2xf16> -> vector<2xbf16> + ``` + }]; + let arguments = (ins + NVVM_VectorOfBfloat16TypeValuesOf:$srcA, + NVVM_VectorOf16BitFloatValues:$srcB + ); + let results = (outs + NVVM_VectorOfBfloat16TypeValuesOf:$res + ); +} +def NVVM_MulPacked_BF16x2_F32x2_F32x2Op : NVVM_Op<"mul.packed.bf16x2.f32x2.f32x2"> { + let description = [{ + This Op performs the multiplication operation on a pair of f32x2 vectors. + The inputs are of the form v2f32 and the output is of the form v2bf16. + The inputs are converted to a scalar of packed i64 where the 0th element of the + vector forms the low-32 bits and the 1st element occupies the high-32 bits. The round to zero + is the only rounding mode supported for this operation. + + Examples: + ```mlir + // Round to zero (only rounding mode supported) + %0 = nvvm.mul.packed.bf16x2.f32x2.f32x2 %srcA, %srcB : vector<2xf32>, vector<2xf32> -> vector<2xbf16> + ``` + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB + ); + let results = (outs + NVVM_VectorOfBfloat16TypeValuesOf:$res + ); +} +def NVVM_MulPacked_F16x2_F16x2_BF16x2Op : NVVM_Op<"mul.packed.f16x2.f16x2.bf16x2"> { + let description = [{ + This Op performs the multiplication operation on a pair of "f16"x2 and "bf16"x2 vectors. + The inputs are of the form v2f16 and v2bf16, and the output is of the form v2f16. + The inputs are converted to a scalar of packed i32 where the 0th element of the + vector forms the low-16 bits and the 1st element occupies the high-16 bits. + + Examples: + ```mlir + %0 = nvvm.mul.packed.f16x2.f16x2.bf16x2 %srcA, %srcB : vector<2xf16>, vector<2xbf16> -> vector<2xf16> + ``` + }]; + let arguments = (ins + NVVM_VectorOf16BitFloatValues:$srcA, + NVVM_VectorOfBfloat16TypeValuesOf:$srcB + ); + let results = (outs + NVVM_VectorOf16BitFloatValues:$res + ); +} +def NVVM_MulPacked_F16x2_F32x2_F32x2Op : NVVM_Op<"mul.packed.f16x2.f32x2.f32x2"> { + let description = [{ + This Op performs the multiplication operation on a pair of f32x2 vectors. + The inputs are of the form v2f32 and the output is of the form v2f16. + The inputs are converted to a scalar of packed i64 where the 0th element of the + vector forms the low-32 bits and the 1st element occupies the high-32 bits. The round to zero + is the only rounding mode supported for this operation. + + Examples: + ```mlir + // Round to zero (only rounding mode supported) + %0 = nvvm.mul.packed.f16x2.f32x2.f32x2 %srcA, %srcB : vector<2xf32>, vector<2xf32> -> vector<2xf16> + ``` + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB + ); + let results = (outs + NVVM_VectorOf16BitFloatValues:$res + ); +} +def NVVM_MulPackedF32x2Op : NVVM_InferredOp<"mul.packed.f32x2"> { + let description = [{ + This Op multiplies floating-point vectors. + The inputs and outputs are of the form . The `rnd` attribute + specifies the rounding mode. The `ftz` attribute, when set, flushes the + subnormal inputs and results to sign-preserving zero. + This Op is supported from SM100 and onwards. + [For more information, see PTX ISA] + (TODO: Add public PTX link when available) + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues2:$srcA, + NVVM_VectorOf32BitFloatValues2:$srcB, + OptionalAttr:$rnd, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_VectorOf32BitFloatValues2:$res + ); +} +def NVVM_MultimemLdReduceOp : NVVM_Op<"multimem.ld.reduce"> { + let description = [{ + The `nvvm.multimem.ld.reduce` operation performs a multimem load-reduce + instruction. It reads from a multimem address and reduces partial + results across participating threads. + + The `ordering` attribute specifies the memory ordering semantics (weak, + relaxed, acquire). + + The `op` attribute specifies the reduction operation (and, or, + xor, add, min, max). + + The `dataType` attribute specifies the PTX data type: + - Integer: b32, s32, u32, b64, s64, u64 + - Floating-point: f16, bf16, f32, f64 + - FP8: e5m2, e4m3 + + The optional `accPrec` attribute specifies the accumulator precision type + for sub-single-precision types (f16/bf16/f8E5M2/f8E4M3FN): f16 or f32. + + Example: + ```mlir + %res = nvvm.multimem.ld.reduce %addr + {ordering = #nvvm.mem_order, + op = #nvvm.red_op, + dataType = f32} + : !llvm.ptr<1> -> f32 + ``` + + See PTX ISA: + https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-multimem + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace1:$addr, + NVVM_MultimemLdReduceOrderingAttr:$ordering, + NVVM_MultimemLdReduceOpAttr:$op, + TypeAttr:$dataType, + OptionalAttr:$accPrec + ); + let results = (outs + NVVM_AnyType:$res + ); +} +def NVVM_MultimemRedOp : NVVM_Op<"multimem.red"> { + let description = [{ + The `nvvm.multimem.red` operation performs a multimem atomic reduction, + applying a reduction operation between a source value and the value at a + multimem address. + + The `ordering` attribute specifies the memory ordering semantics. + The `scope` attribute specifies the scope of the operation. + The `op` attribute specifies the reduction operation. + The `dataType` attribute specifies the PTX data type (b32/s32/u32/b64/s64/ + u64 for integer types, f16/bf16/f32/f64 for floating-point types). + + Example: + ```mlir + nvvm.multimem.red %val, %addr + {ordering = #nvvm.mem_order, + scope = #nvvm.mem_scope, + op = #nvvm.red_op, + dataType = f32} + : f32, !llvm.ptr<1> + ``` + + See PTX ISA: + https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-multimem + }]; + let arguments = (ins + NVVM_AnyType:$val, + NVVM_AnyType:$addr, + NVVM_MultimemRedOrderingAttr:$ordering, + NVVM_MemScopeKindAttr:$scope, + NVVM_MultimemRedOpAttr:$op, + TypeAttr:$dataType + ); + let results = (outs); +} +def NVVM_MultimemStOp : NVVM_Op<"multimem.st"> { + let description = [{ + The `nvvm.multimem.st` operation performs a multimem store instruction, + broadcasting a value to a multimem address. + + The `ordering` attribute specifies the memory ordering semantics. + The `scope` attribute specifies the scope of the operation. + The `dataType` attribute specifies the PTX data type (b32/s32/u32/b64/s64/ + u64 for integer types, f16/bf16/f32/f64 for floating-point, e5m2/e4m3 for + FP8 types). + + Example: + ```mlir + nvvm.multimem.st %val, %addr + {ordering = #nvvm.mem_order, + scope = #nvvm.mem_scope, + dataType = f32} + : f32, !llvm.ptr<1> + ``` + + See PTX ISA: + https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-multimem + }]; + let arguments = (ins + NVVM_AnyType:$val, + NVVM_LLVMPointerInAddressSpace1:$addr, + NVVM_MultimemStOrderingAttr:$ordering, + NVVM_MemScopeKindAttr:$scope, + TypeAttr:$dataType + ); + let results = (outs); +} +def NVVM_NanosleepOp : NVVM_Op<"nanosleep"> { + let description = [{ + The op suspends the thread for a sleep duration approximately close to the + delay `$duration`, specified in nanoseconds. + + The sleep duration is approximated, but guaranteed to be in the + interval [0, 2*t]. The maximum sleep duration is 1 millisecond. + The implementation may reduce the sleep duration for individual threads + within a warp such that all sleeping threads in the warp wake up together. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#miscellaneous-instructions-nanosleep) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$duration + ); + let results = (outs); +} +def NVVM_PMEventOp : NVVM_Op<"pmevent"> { + let description = [{ + Triggers one or more of a fixed number of performance monitor events, with + event index or mask specified by immediate operand. + + Without `mask` it triggers a single performance monitor event indexed by + immediate operand a, in the range 0..15. + + With `mask` it triggers one or more of the performance monitor events. Each + bit in the 16-bit immediate operand controls an event. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#miscellaneous-instructions-pmevent) + }]; + let arguments = (ins + OptionalAttr:$maskedEventId, + OptionalAttr:$eventId + ); + let results = (outs); +} +def NVVM_PrefetchOp : NVVM_Op<"prefetch"> { + let description = [{ + Prefetches the cache line containing the address given by `addr`. The + operand may be a global, local, or generic pointer. When `tensormap` is + specified, the operand may instead be a constant or generic pointer. If the + address maps to shared memory, the operation has no effect. + + At most one of `cacheLevel` or `tensormap` may be present. The `cacheLevel` + attribute selects the target cache level. When combined with `uniform`, the + prefetch is performed to the uniform cache, in which case `addr` must be a + generic pointer. + + When `tensormap` is used, the line containing `addr` is brought from the + constant or parameter state space for later use by `cp.async.bulk.tensor`. + If `in_param_space` is specified, the generic pointer is interpreted as + referring to the parameter state space. + + `uniform` can be specified after the `cacheLevel` to indicate that the + prefetch is performed to the specified uniform cache level. If `uniform` is + specified, `addr` must be a generic address pointer and no operation is + performed if `addr` maps to a `const`, `local`, or `shared` memory location. + + The `evictPriority` attribute is optional and specifies the cache eviction + priority when `cacheLevel` is L2. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-prefetch-prefetchu) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace13:$addr, + Optional:$predicate, + OptionalAttr:$cacheLevel, + OptionalAttr:$evictPriority, + UnitAttr:$tensormap, + UnitAttr:$uniform, + UnitAttr:$in_param_space + ); + let results = (outs); +} +def NVVM_PermuteOp : NVVM_Op<"prmt", [Pure]> { + let description = [{ + The `nvvm.prmt` operation constructs a permutation of the + bytes of the first one or two operands, selecting based on + the 2 least significant bits of the final operand. + + The bytes in the first one or two source operands are numbered. + The first source operand (%lo) is numbered {b3, b2, b1, b0}, + in the case of the '``default``', '``f4e``' and '``b4e``' variants, + the second source operand (%hi) is numbered {b7, b6, b5, b4}. + + Modes: + - `default`: Index mode - each nibble in `selector` selects a byte from the 8-byte pool + - `f4e` : Forward 4 extract - extracts 4 contiguous bytes starting from position in `selector` + - `b4e` : Backward 4 extract - extracts 4 contiguous bytes in reverse order + - `rc8` : Replicate 8 - replicates the lower 8 bits across the 32-bit result + - `ecl` : Edge clamp left - clamps out-of-range indices to the leftmost valid byte + - `ecr` : Edge clamp right - clamps out-of-range indices to the rightmost valid byte + - `rc16` : Replicate 16 - replicates the lower 16 bits across the 32-bit result + + Depending on the 2 least significant bits of the %selector operand, the result + of the permutation is defined as follows: + + +------------+----------------+--------------+ + | Mode | %selector[1:0] | Output | + +------------+----------------+--------------+ + | '``f4e``' | 0 | {3, 2, 1, 0} | + | +----------------+--------------+ + | | 1 | {4, 3, 2, 1} | + | +----------------+--------------+ + | | 2 | {5, 4, 3, 2} | + | +----------------+--------------+ + | | 3 | {6, 5, 4, 3} | + +------------+----------------+--------------+ + | '``b4e``' | 0 | {5, 6, 7, 0} | + | +----------------+--------------+ + | | 1 | {6, 7, 0, 1} | + | +----------------+--------------+ + | | 2 | {7, 0, 1, 2} | + | +----------------+--------------+ + | | 3 | {0, 1, 2, 3} | + +------------+----------------+--------------+ + | '``rc8``' | 0 | {0, 0, 0, 0} | + | +----------------+--------------+ + | | 1 | {1, 1, 1, 1} | + | +----------------+--------------+ + | | 2 | {2, 2, 2, 2} | + | +----------------+--------------+ + | | 3 | {3, 3, 3, 3} | + +------------+----------------+--------------+ + | '``ecl``' | 0 | {3, 2, 1, 0} | + | +----------------+--------------+ + | | 1 | {3, 2, 1, 1} | + | +----------------+--------------+ + | | 2 | {3, 2, 2, 2} | + | +----------------+--------------+ + | | 3 | {3, 3, 3, 3} | + +------------+----------------+--------------+ + | '``ecr``' | 0 | {0, 0, 0, 0} | + | +----------------+--------------+ + | | 1 | {1, 1, 1, 0} | + | +----------------+--------------+ + | | 2 | {2, 2, 1, 0} | + | +----------------+--------------+ + | | 3 | {3, 2, 1, 0} | + +------------+----------------+--------------+ + | '``rc16``' | 0 | {1, 0, 1, 0} | + | +----------------+--------------+ + | | 1 | {3, 2, 3, 2} | + | +----------------+--------------+ + | | 2 | {1, 0, 1, 0} | + | +----------------+--------------+ + | | 3 | {3, 2, 3, 2} | + +------------+----------------+--------------+ + + [For more information, see PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-prmt) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$lo, + Optional:$hi, + NVVM_T32BitSignlessInteger:$selector, + NVVM_PermuteModeAttr:$mode + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$res + ); +} +def NVVM_RcpApproxFtzF32Op : NVVM_Op<"rcp.approx.ftz.f", [Pure]> { + let arguments = (ins + NVVM_T32BitFloat:$arg + ); + let results = (outs + NVVM_T32BitFloat2:$res + ); +} +def NVVM_AggrSmemSize : NVVM_Op<"read.ptx.sreg.aggr.smem.size", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClockOp : NVVM_Op<"read.ptx.sreg.clock"> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_Clock64Op : NVVM_Op<"read.ptx.sreg.clock64"> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockInClusterIdXOp : NVVM_Op<"read.ptx.sreg.cluster.ctaid.x", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockInClusterIdYOp : NVVM_Op<"read.ptx.sreg.cluster.ctaid.y", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockInClusterIdZOp : NVVM_Op<"read.ptx.sreg.cluster.ctaid.z", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterId : NVVM_Op<"read.ptx.sreg.cluster.ctarank", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterDimBlocksXOp : NVVM_Op<"read.ptx.sreg.cluster.nctaid.x", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterDimBlocksYOp : NVVM_Op<"read.ptx.sreg.cluster.nctaid.y", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterDimBlocksZOp : NVVM_Op<"read.ptx.sreg.cluster.nctaid.z", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterDim : NVVM_Op<"read.ptx.sreg.cluster.nctarank", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterIdXOp : NVVM_Op<"read.ptx.sreg.clusterid.x", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterIdYOp : NVVM_Op<"read.ptx.sreg.clusterid.y", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterIdZOp : NVVM_Op<"read.ptx.sreg.clusterid.z", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockIdXOp : NVVM_Op<"read.ptx.sreg.ctaid.x", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockIdYOp : NVVM_Op<"read.ptx.sreg.ctaid.y", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockIdZOp : NVVM_Op<"read.ptx.sreg.ctaid.z", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_DynamicSmemSize : NVVM_Op<"read.ptx.sreg.dynamic.smem.size", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg0Op : NVVM_Op<"read.ptx.sreg.envreg0", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg1Op : NVVM_Op<"read.ptx.sreg.envreg1", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg10Op : NVVM_Op<"read.ptx.sreg.envreg10", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg11Op : NVVM_Op<"read.ptx.sreg.envreg11", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg12Op : NVVM_Op<"read.ptx.sreg.envreg12", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg13Op : NVVM_Op<"read.ptx.sreg.envreg13", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg14Op : NVVM_Op<"read.ptx.sreg.envreg14", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg15Op : NVVM_Op<"read.ptx.sreg.envreg15", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg16Op : NVVM_Op<"read.ptx.sreg.envreg16", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg17Op : NVVM_Op<"read.ptx.sreg.envreg17", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg18Op : NVVM_Op<"read.ptx.sreg.envreg18", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg19Op : NVVM_Op<"read.ptx.sreg.envreg19", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg2Op : NVVM_Op<"read.ptx.sreg.envreg2", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg20Op : NVVM_Op<"read.ptx.sreg.envreg20", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg21Op : NVVM_Op<"read.ptx.sreg.envreg21", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg22Op : NVVM_Op<"read.ptx.sreg.envreg22", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg23Op : NVVM_Op<"read.ptx.sreg.envreg23", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg24Op : NVVM_Op<"read.ptx.sreg.envreg24", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg25Op : NVVM_Op<"read.ptx.sreg.envreg25", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg26Op : NVVM_Op<"read.ptx.sreg.envreg26", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg27Op : NVVM_Op<"read.ptx.sreg.envreg27", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg28Op : NVVM_Op<"read.ptx.sreg.envreg28", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg29Op : NVVM_Op<"read.ptx.sreg.envreg29", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg3Op : NVVM_Op<"read.ptx.sreg.envreg3", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg30Op : NVVM_Op<"read.ptx.sreg.envreg30", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg31Op : NVVM_Op<"read.ptx.sreg.envreg31", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg4Op : NVVM_Op<"read.ptx.sreg.envreg4", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg5Op : NVVM_Op<"read.ptx.sreg.envreg5", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg6Op : NVVM_Op<"read.ptx.sreg.envreg6", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg7Op : NVVM_Op<"read.ptx.sreg.envreg7", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg8Op : NVVM_Op<"read.ptx.sreg.envreg8", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg9Op : NVVM_Op<"read.ptx.sreg.envreg9", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_GlobalTimerOp : NVVM_Op<"read.ptx.sreg.globaltimer"> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_GlobalTimerLoOp : NVVM_Op<"read.ptx.sreg.globaltimer.lo"> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_GridIdOp : NVVM_Op<"read.ptx.sreg.gridid", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_LaneIdOp : NVVM_Op<"read.ptx.sreg.laneid", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_LaneMaskEqOp : NVVM_Op<"read.ptx.sreg.lanemask.eq", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_LaneMaskGeOp : NVVM_Op<"read.ptx.sreg.lanemask.ge", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_LaneMaskGtOp : NVVM_Op<"read.ptx.sreg.lanemask.gt", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_LaneMaskLeOp : NVVM_Op<"read.ptx.sreg.lanemask.le", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_LaneMaskLtOp : NVVM_Op<"read.ptx.sreg.lanemask.lt", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterDimXOp : NVVM_Op<"read.ptx.sreg.nclusterid.x", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterDimYOp : NVVM_Op<"read.ptx.sreg.nclusterid.y", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterDimZOp : NVVM_Op<"read.ptx.sreg.nclusterid.z", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_GridDimXOp : NVVM_Op<"read.ptx.sreg.nctaid.x", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_GridDimYOp : NVVM_Op<"read.ptx.sreg.nctaid.y", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_GridDimZOp : NVVM_Op<"read.ptx.sreg.nctaid.z", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_SmDimOp : NVVM_Op<"read.ptx.sreg.nsmid", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockDimXOp : NVVM_Op<"read.ptx.sreg.ntid.x", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockDimYOp : NVVM_Op<"read.ptx.sreg.ntid.y", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockDimZOp : NVVM_Op<"read.ptx.sreg.ntid.z", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_WarpDimOp : NVVM_Op<"read.ptx.sreg.nwarpid", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_SmIdOp : NVVM_Op<"read.ptx.sreg.smid", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ThreadIdXOp : NVVM_Op<"read.ptx.sreg.tid.x", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ThreadIdYOp : NVVM_Op<"read.ptx.sreg.tid.y", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ThreadIdZOp : NVVM_Op<"read.ptx.sreg.tid.z", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_TotalSmemSize : NVVM_Op<"read.ptx.sreg.total.smem.size", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_WarpIdOp : NVVM_Op<"read.ptx.sreg.warpid", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_WarpSizeOp : NVVM_Op<"read.ptx.sreg.warpsize", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_RedOp : NVVM_Op<"red"> { + let description = [{ + Performs a reduction operation with operand `b` and the value in + location `a`, and stores the result of the specified operation at + location `a`. Operand `a` specifies a location in the specified state space. + If no state space is given, perform the memory accesses + using Generic Addressing + + [For more information, see PTX ISA] + (http://sw-mobile-docs/CUDA/GPGPU/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-red) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace05:$a, + NVVM_T32BitSignlessIntegerOr642:$b, + Optional:$cacheHint, + OptionalAttr:$memOrder, + OptionalAttr:$memScope, + NVVM_ReductionOpAttr:$op, + NVVM_ReductionTypeAttr:$type + ); + let results = (outs); +} +def NVVM_ReduxOp : NVVM_InferredOp<"redux.sync"> { + let description = [{ + `redux.sync` performs a reduction operation `kind` of the 32 bit source + register across all non-exited threads in the membermask. + + The `abs` and `nan` attributes can be used in the case of f32 input type, + where the `abs` attribute causes the absolute value of the input to be used + in the reduction operation, and the `nan` attribute causes the reduction + operation to return NaN if any of the inputs to participating threads are + NaN. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-redux-sync) + }]; + let arguments = (ins + NVVM_T32BitSignlessIntegerOr32:$val, + NVVM_T32BitSignlessInteger:$mask_and_clamp, + NVVM_ReductionKindAttr:$kind, + OptionalAttr:$abs, + OptionalAttr:$nan + ); + let results = (outs + NVVM_T32BitSignlessIntegerOr32:$res + ); +} +def NVVM_RsqrtOp : NVVM_InferredOp<"rsqrt", [Pure]> { + let description = [{ + Computes an approximation of the reciprocal of the square root of the + input value: `d = 1 / sqrt(a)`. Supports both f32 and f64. The maximum + relative error for the f32 form over the entire positive finite range + is 2^-22.9. + + The `ftz` attribute, when set, flushes subnormal inputs and results to + sign-preserving zero. For f64 inputs, `ftz=true` selects a coarser + approximation that uses only the upper 32 bits of the input (the lower + 32 bits of the result are zeroed). + + For more information, see PTX ISA: + [rsqrt](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-rsqrt) + }]; + let arguments = (ins + NVVM_T32BitFloatOr64Bit:$src, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_T32BitFloatOr64Bit:$res + ); +} +def NVVM_SetMaxRegisterOp : NVVM_Op<"setmaxregister"> { + let arguments = (ins + I32Attr:$regCount, + NVVM_SetMaxRegisterActionAttr:$action + ); + let results = (outs); +} +def NVVM_ShflOp : NVVM_InferredOp<"shfl.sync"> { + let description = [{ + The `shfl.sync` Op implements data shuffle within threads of a warp. + The `thread_mask` denotes the threads participating in the Op where + the bit position corresponds to a particular thread's laneid. + The `offset` specifies a source lane or source lane offset + (depending on `kind`). The `val` is the input value to be copied from + the source. The `mask_and_clamp` contains two packed values specifying + a mask for logically splitting warps into sub-segments and an upper bound + for clamping the source lane index. + + The `return_value_and_is_valid` unit attribute can be specified to indicate + that the return value is a two-element struct, where the first element is + the result value and the second element is a predicate indicating if the + computed source lane index is valid. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-shfl-sync) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$thread_mask, + NVVM_T32BitSignlessIntegerOr32:$val, + NVVM_T32BitSignlessInteger:$offset, + NVVM_T32BitSignlessInteger:$mask_and_clamp, + NVVM_ShflKindAttr:$kind, + UnitAttr:$return_value_and_is_valid + ); + let results = (outs + NVVM_T32BitSignlessIntegerOr322:$res + ); +} +def NVVM_SinOp : NVVM_Op<"sin", [Pure]> { + let description = [{ + Computes a fast approximation of the sine of the input value (in radians). + The `ftz` attribute, when set, flushes subnormal inputs and results to + sign-preserving zero. + + For more information, see PTX ISA: + [sin](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-sin) + }]; + let arguments = (ins + NVVM_T32BitFloat:$src, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_T32BitFloat2:$res + ); +} +def NVVM_SPCompressOp : NVVM_Op<"spcompress"> { + let description = [{ + Compresses elements from a dense vector `data` into a structured sparse vector `compressed_data`. + The metadata information is stored in `metadata`. + + Arguments: + - `spDesc`: Sparsity descriptor specifying the selection criteria for elements + - `data`: Input dense vector containing elements to be compressed + - `compFactor`: Compression factor attribute (only `sp2to4` is allowed, resulting in 2:4 compression) + - `indexSize`: Size of metadata index for each selected element (can only be `b2` or `b4` given 2:4 compression) + - `elemSize`: Size in bits for each element from input vector `data` + - `repFactor`: Repeat factor for the operation + + Results: + - `metadata`: Vector storing indices for chosen elements + - `compressed_data`: Output compressed vector containing selected elements + + Examples: + // Example 1: Basic compression with 4-bit elements, 2-bit indices, repeat factor of 1 + ```mlir + %metadata, %compressed_data = nvvm.spcompress %spdesc, %data { + compFactor = #nvvm.spcomp_factor, + indexSize = #nvvm.spcomp_index_size, + elemSize = #nvvm.spcomp_elem_size, + repFactor = #nvvm.spcomp_rep_factor + } : i32, vector<2xi32> -> i32, i32 + ``` + + // Example 2: Compression with 8-bit elements, 4-bit indices, repeat factor of 2 + ```mlir + %metadata, %compressed_data = nvvm.spcompress %spdesc, %data { + compFactor = #nvvm.spcomp_factor, + indexSize = #nvvm.spcomp_index_size, + elemSize = #nvvm.spcomp_elem_size, + repFactor = #nvvm.spcomp_rep_factor + } : i32, vector<4xi32> -> i32, vector<2xi32> + ``` + + // Example 3: Compression with 16-bit elements, 4-bit indices, larger repeat factor + ```mlir + %metadata, %compressed_data = nvvm.spcompress %spdesc, %data { + compFactor = #nvvm.spcomp_factor, + indexSize = #nvvm.spcomp_index_size, + elemSize = #nvvm.spcomp_elem_size, + repFactor = #nvvm.spcomp_rep_factor + } : i32, vector<16xi32> -> vector<2xi32>, vector<8xi32> + ``` + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$spDesc, + NVVM_VectorOf32BitSignlessInteger:$data, + NVVM_SPCompressFactorTypeAttr:$compFactor, + NVVM_SPCompressIndexSizeAttr:$indexSize, + NVVM_SPCompressElemSizeAttr:$elemSize, + NVVM_SPCompressRepFactorAttr:$repFactor + ); + let results = (outs + NVVM_T32BitSignlessIntegerOrVector:$metadata, + NVVM_T32BitSignlessIntegerOrVector:$compressed_data + ); +} +def NVVM_SPDecompressOp : NVVM_Op<"spdecompress"> { + let description = [{ + Decompresses elements from a structured sparse vector `compressed_data` into a dense vector `data`. + The metadata information is stored in `metadata`. + + Arguments: + - `metadata`: Vector storing indices for chosen elements + - `compressed_data`: Compressed vector containing selected elements + - `compFactor`: Decompression factor attribute (sp1to2, sp1to4, sp1to8, sp1to16, sp2to4, sp2to8, sp2to16, sp4to8, sp4to16, sp8to16) + - `indexSize`: Size of metadata index for each selected element (b2 or b4) + - `elemSize`: Size in bits for each element from output vector `data` (b4, b8, b16) + - `repFactor`: Repeat factor for the operation (x1, x2, x4, x8, x16, x32, x64) + + Results: + - `data`: Output dense vector containing decompressed elements + + Examples: + // Example 1: Decompression with 4-bit elements, 2-bit indices, repeat factor of 2 + ```mlir + %res = nvvm.spdecompress %metadata, %compdata { + compFactor = #nvvm.spdecomp_factor, + indexSize = #nvvm.spdecomp_index_size, + elemSize = #nvvm.spdecomp_elem_size, + repFactor = #nvvm.spdecomp_rep_factor + } : i32, i32 -> i32 + ``` + + // Example 2: Decompression with 8-bit elements, 4-bit indices, repeat factor of 2 + ```mlir + %res = nvvm.spdecompress %metadata, %compdata { + compFactor = #nvvm.spdecomp_factor, + indexSize = #nvvm.spdecomp_index_size, + elemSize = #nvvm.spdecomp_elem_size, + repFactor = #nvvm.spdecomp_rep_factor + } : i32, i32 -> vector<2 x i32> + ``` + + // Example 3: Decompression with 16-bit elements, 4-bit indices, repeat factor of 64 + ```mlir + %res = nvvm.spdecompress %metadata, %compdata { + compFactor = #nvvm.spdecomp_factor, + indexSize = #nvvm.spdecomp_index_size, + elemSize = #nvvm.spdecomp_elem_size, + repFactor = #nvvm.spdecomp_rep_factor + } : vector<16 x i32>, vector<64 x i32> -> vector<128 x i32> + ``` + }]; + let arguments = (ins + NVVM_T32BitSignlessIntegerOrVector:$metadata, + NVVM_T32BitSignlessIntegerOrVector:$compressed_data, + NVVM_SPDecompressFactorTypeAttr:$compFactor, + NVVM_SPDecompressIndexSizeAttr:$indexSize, + NVVM_SPDecompressElemSizeAttr:$elemSize, + NVVM_SPDecompressRepFactorAttr:$repFactor + ); + let results = (outs + NVVM_VectorOf32BitSignlessInteger2:$data + ); +} +def NVVM_BulkStoreOp : NVVM_Op<"st.bulk"> { + let description = [{ + Initializes a region of shared memory at the address given by `addr`. + The `size` operand specifies the number of bytes to initialize and must be + a multiple of 8. + The `initVal` operand specifies the value to initialize the memory to. The + only supported value is 0. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-st-bulk) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr, + NVVM_T64BitSignlessInteger:$size, + OptionalAttr:$initVal + ); + let results = (outs); +} +def NVVM_StMatrixOp : NVVM_Op<"stmatrix"> { + let description = [{ + Collectively store one or more matrices across all threads in a warp to the + location indicated by the address operand $ptr in shared memory. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#warp-level-matrix-store-instruction-stmatrix) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace3:$ptr, + Variadic:$sources, + NVVM_MMALayoutAttr:$layout, + OptionalAttr:$shape + ); + let results = (outs); +} +def NVVM_AsyncStoreOp : NVVM_Op<"store.async"> { + let arguments = (ins + NVVM_LLVMPointerInAddressSpace7:$addr, + NVVM_T32BitSignlessIntegerOr643:$value, + NVVM_LLVMPointerInAddressSpace7:$mbarrier + ); + let results = (outs); +} +def NVVM_ExtStoreOp : NVVM_Op<"store.ext"> { + let description = [{ + The `store.ext` operation is used to write to memory in generic, + global or shared address space. This Op explicitly supports various + cache properties and memory orderings associated with the store operation. + + The `l2CacheHint` operand is optional, and it is used to specify cache + eviction policy that may be used during the memory access. + [For more information, see PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-st) + }]; + let arguments = (ins + NVVM_LLVMTypeWithSize:$value, + NVVM_LLVMPointerType:$addr, + Optional:$l2CacheHint, + OptionalAttr:$order, + OptionalAttr:$scope, + OptionalAttr:$evict, + OptionalAttr:$cacheModifier, + OptionalAttr:$sharedSpace + ); + let results = (outs); +} +def NVVM_SubPacked_BF16x2_F32x2_F32x2Op : NVVM_Op<"sub.packed.bf16x2.f32x2.f32x2"> { + let description = [{ + This Op performs the sub operation on a pair of f32x2 vectors. + The inputs and output are of the form v2f32 and v2bf16 respectively. + The inputs are converted to a scalar of packed i64 where the 0th element of the + vector forms the low-32 bits and the 1st element occupies the high-32 bits. The round to zero + is the only rounding mode supported for this operation. + + Examples: + ```mlir + // Round to zero (only rounding mode supported) + %0 = nvvm.sub .packed.bf16x2.f32x2.f32x2 %srcA, %srcB : vector<2xf32>, vector<2xf32> -> vector<2xbf16> + ``` + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB + ); + let results = (outs + NVVM_VectorOfBfloat16TypeValuesOf:$res + ); +} +def NVVM_SubPacked_F16x2_F32x2_F32x2Op : NVVM_Op<"sub.packed.f16x2.f32x2.f32x2"> { + let description = [{ + This Op performs the sub operation on a pair of f32x2 vectors. + The inputs and output are of the form v2f32 and v2f16 respectively. + The inputs are converted to a scalar of packed i64 where the 0th element of the + vector forms the low-32 bits and the 1st element occupies the high-32 bits. The round to zero + is the only rounding mode supported for this operation. + + Examples: + ```mlir + // Round to zero (only rounding mode supported) + %0 = nvvm.sub .packed.f16x2.f32x2.f32x2 %srcA, %srcB : vector<2xf32>, vector<2xf32> -> vector<2xf16> + ``` + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB + ); + let results = (outs + NVVM_VectorOf16BitFloatValues:$res + ); +} +def NVVM_SubPackedF32x2Op : NVVM_InferredOp<"sub.packed.f32x2"> { + let description = [{ + This Op performs subtraction of floating-point vectors. + The inputs and outputs are of the form . The `rnd` attribute + specifies the rounding mode. The `ftz` attribute, when set, flushes the + subnormal inputs and results to sign-preserving zero. + This Op is supported from SM100 and onwards. + [For more information, see PTX ISA] + (TODO: Add public PTX link when available) + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues2:$srcA, + NVVM_VectorOf32BitFloatValues2:$srcB, + OptionalAttr:$rnd, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_VectorOf32BitFloatValues2:$res + ); +} +def NVVM_SubPacked_F32x2_BF16x2_F32x2Op : NVVM_Op<"sub.packed.f32x2.bf16x2.f32x2"> { + let description = [{ + This Op performs the sub operation on a pair of floating-point values. + The inputs are of the form v2bf16 and v2f32. The first input `srcA` is + converted to a scalar of packed i32 where the 0th element of the vector forms + the low-16 bits and the 1st element occupies the high-16 bits. Similarly, + the second input `srcB` is converted to a scalar of packed i64 where the 0th + element of the vector forms the low-32 bits and the 1st element occupies the + high-32 bits. The `rnd` attribute specifies the rounding mode. + + The rounding mode can be one of the following: + - rn: round to nearest even + - rz: round toward zero + - rm: round toward negative infinity + - rp: round toward positive infinity + + Examples: + ```mlir + // Round to nearest even (default) + %0 = nvvm.sub .packed.f32x2.bf16x2.f32x2 %srcA, %srcB : vector<2xbf16>, vector<2xf32> -> vector<2xf32> + + // Round toward negative infinity + %1 = nvvm.sub .packed.f32x2.bf16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xbf16>, vector<2xf32> -> vector<2xf32> + + // Round toward zero with flush-to-zero + %2 = nvvm.sub .packed.f32x2.bf16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xbf16>, vector<2xf32> -> vector<2xf32> + ``` + }]; + let arguments = (ins + NVVM_VectorOfBfloat16TypeValuesOf:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB, + OptionalAttr:$rnd + ); + let results = (outs + NVVM_VectorOf32BitFloatValues:$res + ); +} +def NVVM_SubPacked_F32x2_F16x2_F32x2Op : NVVM_Op<"sub.packed.f32x2.f16x2.f32x2"> { + let description = [{ + This Op performs the sub operation on a pair of floating-point values. + The inputs are of the form v2f16 and v2f32. The first input `srcA` is + converted to a scalar of packed i32 where the 0th element of the vector forms + the low-16 bits and the 1st element occupies the high-16 bits. Similarly, + the second input `srcB` is converted to a scalar of packed i64 where the 0th + element of the vector forms the low-32 bits and the 1st element occupies the + high-32 bits. The `rnd` attribute specifies the rounding mode. + + The rounding mode can be one of the following: + - rn: round to nearest even + - rz: round toward zero + - rm: round toward negative infinity + - rp: round toward positive infinity + + Examples: + ```mlir + // Round to nearest even (default) + %0 = nvvm.sub .packed.f32x2.f16x2.f32x2 %srcA, %srcB : vector<2xf16>, vector<2xf32> -> vector<2xf32> + + // Round toward negative infinity + %1 = nvvm.sub .packed.f32x2.f16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xf16>, vector<2xf32> -> vector<2xf32> + + // Round toward zero with flush-to-zero + %2 = nvvm.sub .packed.f32x2.f16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xf16>, vector<2xf32> -> vector<2xf32> + ``` + }]; + let arguments = (ins + NVVM_VectorOf16BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB, + OptionalAttr:$rnd + ); + let results = (outs + NVVM_VectorOf32BitFloatValues:$res + ); +} +def NVVM_SubFOp : NVVM_InferredOp<"subf", [Pure]> { + let description = [{ + The `nvvm.subf` operation performs floating point subtraction of two + operands. + + It supports the same type combinations and modifiers as `nvvm.addf`. + This is equivalent to `nvvm.addf(lhs, -rhs)`. + + For more information, see PTX ISA: + - [floating point subtraction](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-sub) + - [half-precision floating point subtraction](https://docs.nvidia.com/cuda/parallel-thread-execution/#half-precision-floating-point-instructions-sub) + }]; + let arguments = (ins + NVVM_T16BitFloatOrBfloat16Type:$lhs, + NVVM_T16BitFloatOrBfloat16Type:$rhs, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_T16BitFloatOrBfloat16Type:$res + ); +} +def NVVM_Tcgen05AllocOp : NVVM_Op<"tcgen05.alloc"> { + let description = [{ + The `tcgen05.alloc` Op allocates tensor core memory for + the amount specified by `nCols` and writes the destination + address to the `addr` argument. The `nCols` operand specifies the + number of columns to be allocated and it must be a power-of-two. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-memory-alloc-manage-instructions) + }]; + let arguments = (ins + NVVM_LLVMPointerTypeOrLLVMPointer:$addr, + NVVM_T32BitSignlessInteger:$nCols, + UnitAttr:$isExclusive, + OptionalAttr:$group + ); + let results = (outs); +} +def NVVM_Tcgen05CommitOp : NVVM_Op<"tcgen05.commit"> { + let description = [{ + The `tcgen05.commit` makes the *mbarrier object*, specified by + the operand `addr`, track the completion of all the prior + async-tcgen05 operations initiated by the executing thread. + The multicast variants allow signaling on the *mbarrier objects* + of multiple CTAs within the cluster. Operand `multicastMask`, + when present, specifies the destination CTAs in the cluster such + that each bit position in the 16-bit or 32-bit `multicastMask` operand + corresponds to the `nvvm.read.ptx.sreg.ctaid` of the destination CTA. + Operand `smem_a_read` when present, specifies track only shared memory + read of "Matrix A for the prior tcgen05.mma Ops" + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen-async-sync-operations-commit) + }]; + let arguments = (ins + NVVM_LLVMPointerTypeOrLLVMPointer:$addr, + Optional:$multicastMask, + UnitAttr:$smem_a_read, + OptionalAttr:$group + ); + let results = (outs); +} +def NVVM_Tcgen05CpOp : NVVM_Op<"tcgen05.cp"> { + let description = [{ + Instruction tcgen05.cp initiates an asynchronous copy operation from + shared memory to the location specified by the address operand `taddr` + in the Tensor Memory. The 64-bit register operand `smem_desc` specifies + the matrix descriptor representing the source matrix in the shared memory + that needs to be copied. + + Example: + ```mlir + nvvm.tcgen05.cp %taddr, %smem_desc { + group = #nvvm.cta_group, + shape = #nvvm.tcgen05_cp_shape, + multicast = #nvvm.tcgen05_cp_multicast, + srcFormat = #nvvm.tcgen05_cp_src_fmt + } + ``` + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tensorcore-5th-generation-instructions-tcgen05-cp) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$taddr, + NVVM_T64BitSignlessInteger:$smem_desc, + NVVM_Tcgen05CpShapeAttr:$shape, + OptionalAttr:$group, + OptionalAttr:$multicast, + OptionalAttr:$srcFormat + ); + let results = (outs); +} +def NVVM_Tcgen05DeallocOp : NVVM_Op<"tcgen05.dealloc"> { + let description = [{ + The `tcgen05.dealloc` Op de-allocates the tensor core memory + specified by `tmemAddr`, which must be from a previous tensor + memory allocation. The `nCols` operand specifies the number + of columns to be de-allocated, and it must be a power-of-two. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-memory-alloc-manage-instructions) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$taddr, + NVVM_T32BitSignlessInteger:$nCols, + UnitAttr:$isExclusive, + OptionalAttr:$group + ); + let results = (outs); +} +def NVVM_Tcgen05FenceOp : NVVM_Op<"tcgen05.fence"> { + let description = [{ + The `tcgen05.fence` orders all prior async tcgen05 operations + with respect to the subsequent tcgen05 and execution ordering operations. + The `tcgen05.fence` orders all subsequent async tcgen05 operations + with respect to the prior tcgen05 and execution ordering operations. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tensorcore-5th-generation-instructions-tcgen05-fence) + }]; + let arguments = (ins + NVVM_Tcgen05FenceKindAttr:$kind + ); + let results = (outs); +} +def NVVM_Tcgen05LdOp : NVVM_Op<"tcgen05.ld"> { + let description = [{ + Instruction `tcgen05.ld` asynchronously loads data from the Tensor Memory at + the location specified by the 32-bit address operand `tmemAddr` into the + destination register `res`, collectively across all threads of the warps. + + The `shape` and the `num` attribute together determines the total + dimension of the data which is loaded from the Tensor Memory. The `shape` + attribute indicates the base dimension of data to be accessed as described + in the Data Movement Shape. The `num` attribute indicates the repeat + factor on the base dimension resulting in the total dimension of the data + that is accessed. + + The shape `16x32bx2` performs two accesses into Tensor Memory of the shape + `16x32b`. The base address of the first access is specified by `tmemAddr` + and the base address of the second access is specified by + `tmemAddr + offset`, where `offset` is an immediate argument. + + The unit attribute `pack` can be used to pack two 16-bit + elements from adjacent columns into a single 32-bit element during the load. + + The following table describes the size of the vector for various combinations + of `num` and `shape` attributes: + ``` + |=====================================================================| + | num/shape | 16x32bx2/16x64b/32x32b | 16x128b | 16x256b | + |=====================================================================| + | x1 | 1 | 2 | 4 | + | x2 | 2 | 4 | 8 | + | x4 | 4 | 8 | 16 | + | x8 | 8 | 16 | 32 | + | x16 | 16 | 32 | 64 | + | x32 | 32 | 64 | 128 | + | x64 | 64 | 128 | NA | + | x128 | 128 | NA | NA | + |=====================================================================| + ``` + + Example: + ```mlir + nvvm.tcgen05.ld %tmemAddr, %offset pack { + shape = #nvvm.tcgen05_ldst_shape, + } : <2xi32> + ``` + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-st) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$tmemAddr, + Optional:$offset, + UnitAttr:$pack, + NVVM_Tcgen05LdStShapeAttr:$shape + ); + let results = (outs + NVVM_T32BitSignlessIntegerOrVector2:$res + ); +} +def NVVM_Tcgen05LdRedOp : NVVM_Op<"tcgen05.ld.red"> { + let description = [{ + Instruction `tcgen05.ld.red` asynchronously loads data from the Tensor + Memory at the location specified by the 32-bit address operand `addr` into + the destination register `data`, collectively across all threads of the + warp. The operation also performs reduction operation specified by `op` on + the loaded data across columns in each lane and stored into `redVal` + + The `shape` and the `num` attribute together determines the total + dimension of the data which is loaded from the Tensor Memory. The `shape` + attribute indicates the base dimension of data to be accessed as described + in the Data Movement Shape. The `num` attribute indicates the repeat + factor on the base dimension resulting in the total dimension of the data + that is accessed. + + The shape `16x32bx2` performs two accesses into Tensor Memory of the shape + `16x32b`. The base address of the first access is specified by `addr` + and the base address of the second access is specified by + `addr + offset`, where `offset` is an immediate argument. + + `isSigned` attribute is used to specify whether the integer reduction value is is signed, it is to false by default + + The following table describes the size of the vector for various combinations + of `num` and `shape` attributes: + ``` + |=============================================| + | num/shape | 16x32bx2/32x32b | + |=============================================| + | x2 | 2 | + | x4 | 4 | + | x8 | 8 | + | x16 | 16 | + | x32 | 32 | + | x64 | 64 | + | x128 | 128 | + |=============================================| + ``` + + Example: + ```mlir + // u32 + %data, %redval = nvvm.tcgen05.ld,red %addr, %offset { + shape = #nvvm.tcgen05_ldst_shape, + } : <2xi32>, i32 + + // s32 + %data, %redval = nvvm.tcgen05.ld,red %addr, %offset { + shape = #nvvm.tcgen05_ldst_shape, + isSigned = true + } : <2xi32>, i32 + + // f32 + %data, %redval = nvvm.tcgen05.ld,red %addr { + shape = #nvvm.tcgen05_ldst_shape, + } : <2xf32>, f32 + ``` + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-ld) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$addr, + Optional:$offset, + NVVM_Tcgen05LdStShapeAttr:$shape, + NVVM_ReductionKindAttr:$op, + UnitAttr:$abs, + UnitAttr:$nan, + OptionalAttr:$isSigned + ); + let results = (outs + NVVM_VectorOf32BitSignlessInteger3:$data, + NVVM_T32BitSignlessIntegerOr32:$redVal + ); +} +def NVVM_Tcgen05LdRedSpcompressOp : NVVM_Op<"tcgen05.ld.red.spcompress"> { + let description = [{ + The `tcgen05.ld.red.spcompress` performs in-line sparse compression when loading data from Tensor Memory. + The operation uses a combination of `abs` and `opKind` parameters to select values for compression. + The NaN detection mode is always used for the compression operation, while the .NAN flag only applies + to the reduction portion. + + Arguments: + - `tmemAddr`: Tensor memory address to load from + - `abs`: Whether to use absolute value comparison (used with opKind for value selection) + - `opKind`: Comparison operator kind (min/max) for selecting values (used with abs for value selection) + - `shape`: Pattern used to construct the Matrix (used with repFactor for Matrix construction) + - `repFactor`: Repeat factor, number of times the pattern has to be repeated to form the Matrix (used with shape for Matrix construction) + - `compFactor`: Compression factor (2:4 sparsity, choose 2 pair of elems from every 4 pairs of elems) + - `nan`: Whether to use NaN detection mode + + Results: + - `metadata`: Vector storing sparse metadata + - `compressedData`: Vector storing compressed data + - `redVal`: Reduction value + + Examples: + ```mlir + // MIN operation with x4 repetition factor + %metadata, %compressed_data, %red_val = nvvm.tcgen05.ld.red.spcompress %taddr { + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : i32, vector<2xi32> + + // Max operation with absolute value comparison + %metadata, %compressed_data, %red_val = nvvm.tcgen05.ld.red.spcompress %taddr { + abs, + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : i32, vector<4xi32> + + // MIN operation with NaN detection for reduction + %metadata, %compressed_data, %red_val = nvvm.tcgen05.ld.red.spcompress %taddr { + nan, + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : i32, vector<8xi32> + + // MAX operation with abs and nan options + %metadata, %compressed_data, %red_val = nvvm.tcgen05.ld.red.spcompress %taddr { + abs, + nan, + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : vector<2xi32>, vector<32xi32> + + // MIN operation with maximum repetition factor + %metadata, %compressed_data, %red_val = nvvm.tcgen05.ld.red.spcompress %taddr { + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : vector<4xi32>, vector<64xi32> + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$tmemAddr, + UnitAttr:$abs, + UnitAttr:$nan, + NVVM_Tcgen05LdStShapeAttr:$shape, + NVVM_SPCompressOpKindAttr:$opKind, + NVVM_SPCompressFactorTypeAttr:$compFactor, + NVVM_SPCompressRepFactorAttr:$repFactor + ); + let results = (outs + NVVM_T32BitSignlessIntegerOrVector3:$metadata, + NVVM_VectorOf32BitSignlessInteger4:$compressedData, + NVVM_T32BitSignlessInteger:$redVal + ); +} +def NVVM_Tcgen05LdSpcompressOp : NVVM_Op<"tcgen05.ld.spcompress"> { + let description = [{ + The `tcgen05.ld.spcompress` performs an in-line sparse compression when loading data from Tensor Memory. + The operation uses a combination of `abs` and `opKind` parameters to select values for compression. + + With a 2-to-4 sparsity factor (specified by `compFactor`), half as much data is written to the output + as is read from Tensor Memory. The sequence of sparse metadata is written to the `metadata` result, + while the compressed data is written to `compressedData`. + + Arguments: + - `tmemAddr`: Tensor memory address to load from + - `abs`: Whether to use absolute value comparison (used with opKind for value selection) + - `opKind`: Comparison operator kind (min/max) for selecting values (used with abs for value selection) + - `shape`: Pattern used to construct the Matrix (used with repFactor for Matrix construction) + - `repFactor`: Repeat factor, number of times the pattern has to be repeated to form the Matrix (used with shape for Matrix construction) + - `compFactor`: Compression factor (2:4 sparsity, choose 2 pair of elems from every 4 pairs of elems) + + Results: + - `metadata`: Vector storing sparse metadata + - `compressedData`: Vector storing compressed data + + Example: + ```mlir + // MIN operation with x4 repetition factor + %metadata, %compressed_data = nvvm.tcgen05.ld.spcompress %taddr { + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : i32, vector<2xi32> + + // MAX operation with abs option + %metadata, %compressed_data = nvvm.tcgen05.ld.spcompress %taddr { + abs, + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : i32, vector<4xi32> + + // MIN operation with x64 repetition factor + %metadata, %compressed_data = nvvm.tcgen05.ld.spcompress %taddr { + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : vector<2xi32>, vector<32xi32> + + // MAX operation with abs and x128 repetition factor + %metadata, %compressed_data = nvvm.tcgen05.ld.spcompress %taddr { + abs, + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : vector<4xi32>, vector<64xi32> + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$tmemAddr, + UnitAttr:$abs, + NVVM_Tcgen05LdStShapeAttr:$shape, + NVVM_SPCompressOpKindAttr:$opKind, + NVVM_SPCompressFactorTypeAttr:$compFactor, + NVVM_SPCompressRepFactorAttr:$repFactor + ); + let results = (outs + NVVM_T32BitSignlessIntegerOrVector3:$metadata, + NVVM_VectorOf32BitSignlessInteger4:$compressedData + ); +} +def NVVM_Tcgen05MMAOp : NVVM_Op<"tcgen05.mma", [AttrSizedOperandSegments]> { + let description = [{ + The `tcgen05.mma` operation is an asynchronous tensor core instruction that + performs matrix multiplication, accumulation in a single fused operation. It + targets 5th-generation tensor cores, providing developers with fine-grained + control over execution and scheduling. + + ``` + D = A * B + (D * 2^ -scaleInputD) // if `scaleInputD` is provided + D = A * B // if `enableInputD` is false + D = A * B + D // otherwise + ``` + + where: + - A is an `M x K` matrix in tensor memory or described using shared memory descriptor + - B is a `K x N` matrix described using shared memory descriptor + - D is an `M x N` accumulator matrix in tensor memory + + The `shared memory descriptor` can be generated using `tcgen05.mma_smem_desc` Op + + - idesc is a 32-bit value representing the [Instruction Descriptor](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instruction-descriptor) + + Optional Operands: + - `scaleInputD` is an Immediate value operand used for scaling D matrix by 2 ^ (-scaleInputD). The valid range is [0, 15] + + - `disableOutputLane` is a vector mask for selective output + * vector<4 x i32> when ctaGroup is CTA_1 + * vector<8 x i32> when ctaGroup is CTA_2 + + Required Attributes: + - `kind` is a Tcgen05MMAKind attribute + + - `ctaGroup` specifies CTA group configuration + * cta_1: MMA will be performed on the current thread's CTA + * cta_2: MMA will be performed on the current thread and it's peer CTA + + Default Attributes: + - collectorOp is a Tcgen05MMACollectorOp attribute with matrix A as the collector buffer + + - `aShift` shifts the rows of the A matrix down by one row and can only be + applied if A is in tensor memory + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$matrixD, + NVVM_LLVMPointerInAddressSpace62:$matrixA, + NVVM_T64BitSignlessInteger:$matrixB, + NVVM_T32BitSignlessInteger:$idesc, + NVVM_T1BitSignlessInteger:$enableInputD, + Optional:$scaleInputD, + Optional:$disableOutputLane, + NVVM_Tcgen05MMAKindAttr:$kind, + NVVM_CTAGroupKindAttr:$ctaGroup, + OptionalAttr:$collectorOp, + OptionalAttr:$collectorOpB, + UnitAttr:$aShift + ); + let results = (outs); +} +def NVVM_Tcgen05MMABlockScaleOp : NVVM_Op<"tcgen05.mma.block_scale"> { + let description = [{ + The `tcgen05.mma.block_scale` operation is an asynchronous tensor core instruction + that performs matrix multiplication, accumulation with block scaling in a + single fused operation. It targets 5th-generation tensor cores, providing + developers with fine-grained control over execution and scheduling. + + ``` + D = (A * scale_a) * (B * scale_b)` // if `enableInputD` is false + D = (A * scale_a) * (B * scale_b) + D` + ``` + + where: + - A is an M x (K / 2) matrix in tensor memory or described using shared memory descriptor + - B is a K x N matrix described using shared memory descriptor + - D is an M x N accumulator matrix in tensor memory + - `scale_a` and `scale_b` are matrices in tensor memory used to scale `A` and `B` respectively + + The `shared memory descriptor` can be generated using `tcgen05.mma_smem_desc` Op + + - `idesc` is a 32 bit value representing the [Instruction Descriptor](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instruction-descriptor) + + Required Attributes: + - `kind` is a Tcgen05MMAKind attribute restricted to mxf8f6f4, mxf4, or mxf4nvf4 + + - `ctaGroup` specifies CTA group configuration + * cta_1: MMA will be performed on the current thread's CTA + * cta_2: MMA will be performed on the current thread and it's peer CTA + + Default Attributes: + - collectorOp is a Tcgen05MMACollectorOp attribute with matrix A as the collector buffer + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma) + }]; + let arguments = (ins + NVVM_AnyType:$matrixD, + NVVM_AnyType:$matrixA, + NVVM_AnyType:$matrixB, + NVVM_AnyType:$idesc, + NVVM_AnyType:$enableInputD, + NVVM_AnyType:$scaleA, + NVVM_AnyType:$scaleB, + NVVM_Tcgen05MMAKindAttr:$kind, + NVVM_CTAGroupKindAttr:$ctaGroup, + OptionalAttr:$blockScale, + OptionalAttr:$collectorOp, + OptionalAttr:$collectorOpB + ); + let results = (outs); +} +def NVVM_Tcgen05MMABlockScaleDecompressBOp : NVVM_Op<"tcgen05.mma.block_scale.decompress_b"> { + let description = [{ + The `tcgen05.mma.block_scale.decompress_b` operation is an asynchronous tensor core + instruction that decompresses the B matrix and performs matrix + multiplication, accumulation with block scaling in a single fused operation. + It targets 5th-generation tensor cores, providing developers with + fine-grained control over execution and scheduling. + + ``` + D = (A * scale_a) * (B * scale_b)` // if `enableInputD` is false + D = (A * scale_a) * (B * scale_b) + D` + ``` + + where: + - A is an M x (K / 2) matrix in tensor memory or described using shared memory descriptor + - B is a `K x N` matrix described using shared memory descriptor + - D is an `M x N` accumulator matrix in tensor memory + - `scale_a` and `scale_b` are matrices in tensor memory used to scale `A` and `B` respectively + + The `shared memory descriptor` can be generated using `tcgen05.mma_smem_desc` Op + + - `idesc` is a 32-bit value representing the [Instruction Descriptor](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instruction-descriptor) + + - `decompressBMetadata` is a decompress metadata of B matrix + + Required Attributes: + - `ctaGroup` specifies CTA group configuration + * cta_1: MMA will be performed on the current thread's CTA + * cta_2: MMA will be performed on the current thread and it's peer CTA + + Default Attributes: + - collectorOpA is a Tcgen05MMACollectorOp attribute with matrix A as the collector buffer + - collectorOpB is a Tcgen05MMACollectorOp attribute with matrix B as the collector buffer + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma) + }]; + let arguments = (ins + NVVM_AnyType:$matrixD, + NVVM_AnyType:$matrixA, + NVVM_AnyType:$matrixB, + NVVM_AnyType:$idesc, + NVVM_AnyType:$enableInputD, + NVVM_AnyType:$scaleA, + NVVM_AnyType:$scaleB, + NVVM_AnyType:$decompressBMetadata, + NVVM_Tcgen05MMAKindAttr:$kind, + NVVM_CTAGroupKindAttr:$ctaGroup, + OptionalAttr:$blockScale, + OptionalAttr:$collectorOpA, + OptionalAttr:$collectorOpB + ); + let results = (outs); +} +def NVVM_Tcgen05MMADecompressBOp : NVVM_Op<"tcgen05.mma.decompress_b"> { + let description = [{ + The `tcgen05.mma.decompress_b` operation is an asynchronous tensor core + instruction that decompresses the B matrix and performs matrix + multiplication, accumulation in a single fused operation. It targets + 5th-generation tensor cores, providing developers with fine-grained + control over execution and scheduling. + + ``` + D = A * B // if `enableInputD` is false + D = A * B + D // otherwise + ``` + + where: + - A is an `M x K` matrix in tensor memory or described using shared memory descriptor + - B is a `K x N` matrix described using shared memory descriptor + - D is an `M x N` accumulator matrix in tensor memory + + The `shared memory descriptor` can be generated using `tcgen05.mma_smem_desc` Op + + - `idesc` is a 32-bit value representing the [Instruction Descriptor](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instruction-descriptor) + + - `decompressBMetadata` is a decompress metadata of B matrix + + Optional Operands: + - `disableOutputLane` is a vector mask for selective output + * vector<4 x i32> when ctaGroup is CTA_1 + * vector<8 x i32> when ctaGroup is CTA_2 + + Required Attributes: + - `ctaGroup` specifies CTA group configuration + * cta_1: MMA will be performed on the current thread's CTA + * cta_2: MMA will be performed on the current thread and it's peer CTA + + Default Attributes: + - collectorOAp is a Tcgen05MMACollectorOp attribute with matrix A as the collector buffer + - collectorOpB is a Tcgen05MMACollectorOp attribute with matrix B as the collector buffer + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma) + }]; + let arguments = (ins + NVVM_AnyType:$matrixD, + NVVM_AnyType:$matrixA, + NVVM_AnyType:$matrixB, + NVVM_AnyType:$idesc, + NVVM_AnyType:$enableInputD, + NVVM_AnyType:$decompressBMetadata, + Optional:$disableOutputLane, + NVVM_Tcgen05MMAKindAttr:$kind, + NVVM_CTAGroupKindAttr:$ctaGroup, + OptionalAttr:$collectorOpA, + OptionalAttr:$collectorOpB + ); + let results = (outs); +} +def NVVM_Tcgen05MMASparseOp : NVVM_Op<"tcgen05.mma.sp", [AttrSizedOperandSegments]> { + let description = [{ + The `tcgen05.mma.sp` operation is an asynchronous tensor core instruction + that performs matrix multiplication, accumulation with sparse `A` matrix in + a single fused operation. It targets 5th-generation tensor cores, providing + developers with fine-grained control over execution and scheduling. + + ``` + D = A * B + (D * 2^ -scaleInputD) // if `scaleInputD` is provided + D = A * B // if `enableInputD` is false + D = A * B + D // otherwise + ``` + + where: + - A is an `M x (K / 2)` matrix in tensor memory or described using shared memory descriptor + - B is a `K x N` matrix described using shared memory descriptor + - D is an `M x N` accumulator matrix in tensor memory + - sparseMetadata located in tensor memory specifies the mapping of the `K / 2` + non-zero elements to the K elements before performing the MMA operation + + Other attributes and operands are similar to that of tcgen05.mma Op + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma-sp) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$matrixD, + NVVM_LLVMPointerInAddressSpace62:$matrixA, + NVVM_T64BitSignlessInteger:$matrixB, + NVVM_T32BitSignlessInteger:$idesc, + NVVM_T1BitSignlessInteger:$enableInputD, + NVVM_LLVMPointerInAddressSpace6:$sparseMetadata, + Optional:$scaleInputD, + Optional:$disableOutputLane, + NVVM_Tcgen05MMAKindAttr:$kind, + NVVM_CTAGroupKindAttr:$ctaGroup, + OptionalAttr:$collectorOp, + OptionalAttr:$collectorOpB, + UnitAttr:$aShift + ); + let results = (outs); +} +def NVVM_Tcgen05MMASparseBlockScaleOp : NVVM_Op<"tcgen05.mma.sp.block_scale"> { + let description = [{ + The `tcgen05.mma.sp.block_scale` operation is an asynchronous tensor core + instruction that performs matrix multiplication, accumulation with block + scaling, and sparse `A` matrix in a single fused operation. It targets + 5th-generation tensor cores, providing developers with fine-grained control + over execution, and scheduling. + + ``` + D = (A * scale_a) * (B * scale_b) // if `enableInputD` is specified + D = (A * scale_a) * (B * scale_b) + D // otherwise + ``` + + where: + - A is an M x (K / 2) matrix in tensor memory or described using shared memory descriptor + - B is a K x N matrix described using shared memory descriptor + - D is an M x N accumulator matrix in tensor memory + - `scale_a` and `scale_b` are matrices in tensor memory used to scale `A` and `B` respectively + + Other attributes and operands are similar to that of tcgen05.mma.block_scale Op + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma-sp) + }]; + let arguments = (ins + NVVM_AnyType:$matrixD, + NVVM_AnyType:$matrixA, + NVVM_AnyType:$matrixB, + NVVM_AnyType:$idesc, + NVVM_AnyType:$enableInputD, + NVVM_AnyType:$sparseMetadata, + NVVM_AnyType:$scaleA, + NVVM_AnyType:$scaleB, + NVVM_Tcgen05MMAKindAttr:$kind, + NVVM_CTAGroupKindAttr:$ctaGroup, + OptionalAttr:$blockScale, + OptionalAttr:$collectorOp, + OptionalAttr:$collectorOpB + ); + let results = (outs); +} +def NVVM_Tcgen05MMAWsOp : NVVM_Op<"tcgen05.mma.ws"> { + let description = [{ + The `tcgen05.mma.ws` operation is an asynchronous tensor core instruction + that performs weight stationary convolution matrix multiplication, accumulation + in a single fused operation. It targets 5th-generation tensor cores, providing + developers with fine-grained control over execution, and scheduling. + + ``` + D = A * B` // if `enableInputD` is false + D = A * B + D` // otherwise + ``` + + where: + - A is an `M x K` matrix in tensor memory or described using shared memory descriptor + - B is a `K x N` matrix described using shared memory descriptor + - D is an `M x N` accumulator matrix in tensor memory + + The `shared memory descriptor` can be generated using `tcgen05.mma_smem_desc` Op + + - idesc is a 32-bit value representing the [Instruction Descriptor](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instruction-descriptor) + + Optional Operands: + - zeroColMask is a 64 bit value representing the [Zero-column mask descriptor](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-zero-column-mask-descriptor) + + Required Attributes: + - `kind` is a Tcgen05MMAKind attribute + + Default Valued Attributes: + - collectorBBuffer specifies collector buffer for matrix B: b0 (default), b1, b2, b3 + + - collectorOp is a Tcgen05MMACollectorOp attribute with matrix B as the collector buffer + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma-ws) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$matrixD, + NVVM_LLVMPointerInAddressSpace62:$matrixA, + NVVM_T64BitSignlessInteger:$matrixB, + NVVM_T32BitSignlessInteger:$idesc, + NVVM_T1BitSignlessInteger:$enableInputD, + Optional:$zeroColMask, + NVVM_Tcgen05MMAKindAttr:$kind, + OptionalAttr:$collectorBBuffer, + OptionalAttr:$collectorOp + ); + let results = (outs); +} +def NVVM_Tcgen05MMAWsSparseOp : NVVM_Op<"tcgen05.mma.ws.sp"> { + let description = [{ + The `tcgen05.mma.ws.sp` operation is an asynchronous tensor core instruction + that performs weight stationary convolution matrix multiplication, accumulation + with sparse `A` matrix in a single fused operation. It targets 5th-generation + tensor cores, providing developers with fine-grained control over execution, + and scheduling. + + ``` + D = A * B` // if `enableInputD` is false + D = A * B + D` // otherwise + ``` + + where: + - A is an M x (K / 2) matrix in memory or descriptor format + - B is a K x N matrix + - D is an M x N accumulator matrix + - sparseMetadata located in tensor memory specifies the mapping of the `K / 2` + non-zero elements to the K elements before performing the MMA operation + + Other attributes and operands are similar to that of tcgen05.mma.ws Op + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma-ws-sp) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$matrixD, + NVVM_LLVMPointerInAddressSpace62:$matrixA, + NVVM_T64BitSignlessInteger:$matrixB, + NVVM_T32BitSignlessInteger:$idesc, + NVVM_T1BitSignlessInteger:$enableInputD, + NVVM_LLVMPointerInAddressSpace6:$sparseMetadata, + Optional:$zeroColMask, + NVVM_Tcgen05MMAKindAttr:$kind, + OptionalAttr:$collectorBBuffer, + OptionalAttr:$collectorOp + ); + let results = (outs); +} +def NVVM_Tcgen05MmaSmemDescOp : NVVM_Op<"tcgen05.mma_smem_desc", [MemoryEffects<[MemRead]>]> { + let description = [{ + The `nvvm.tcgen05_mma_smem_desc` constructs a Shared Memory descriptor + for tcgen05.mma. This descriptor is a 64-bit value which describes the + properties of multiplicand matrix in shared memory including its location + in the shared memory of the current CTA. + + ``` + +-----------+------+------------------------------------------------------+ + | Bit-field | Size | Description | + +-----------+------+------------------------------------------------------+ + | 0-13 | 14 | Matrix start address | + | 14-15 | 2 | Reserved | + | 16-29 | 14 | Leading dim relative-offset (or) absolute-address | + | 30-31 | 2 | Reserved | + | 32-45 | 14 | Stride dimension byte offset | + | 46-48 | 3 | Fixed constant value of 0b001 | + | 49-51 | 3 | Matrix base offset | + | 52 | 1 | Leading dimension stride mode: | + | | | 0: byte offset relative | + | | | 1: byte address absolute | + | 53-60 | 8 | Fixed constant value of 0xb00000000 | + | 61-63 | 3 | Swizzling mode: | + | | | 0: No swizzling | + | | | 1: 128-Byte with 32B atomic swizzling | + | | | 2: 128-Byte swizzling | + | | | 4: 64-Byte swizzling | + | | | 6: 32-Byte swizzling | + | | | (Values 3, 5 and 7 are invalid) | + +-----------+------+------------------------------------------------------+ + ``` + + Example: + ```mlir + %desc = nvvm.tcgen05.mma_smem_desc (%startAddr, %leadingDimOffset, %strideDimOffset, + %baseOffset, %leadingDimMode, %swizzleMode) : (i32, i32, i32, i8, i1, i8) -> i64 + ``` + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-shared-memory-descriptor) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$startAddr, + NVVM_T32BitSignlessInteger:$leadingDimOffset, + NVVM_T32BitSignlessInteger:$strideDimOffset, + NVVM_T8BitSignlessInteger2:$baseOffset, + NVVM_T1BitSignlessInteger:$leadingDimMode, + NVVM_T8BitSignlessInteger2:$swizzleMode + ); + let results = (outs + NVVM_T64BitSignlessInteger2:$res + ); +} +def NVVM_Tcgen05RelinquishAllocPermitOp : NVVM_Op<"tcgen05.relinquish_alloc_permit"> { + let description = [{ + The `tcgen05.relinquish_alloc_permit` Op specifies that the CTA + of the executing thread is relinquishing the right to allocate + Tensor Memory. So, it is illegal for a CTA to perform `tcgen05.alloc` + after any of its constituent threads execute `tcgen05.relinquish_alloc_permit`. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-memory-alloc-manage-instructions) + }]; + let arguments = (ins + OptionalAttr:$group + ); + let results = (outs); +} +def NVVM_Tcgen05ShiftOp : NVVM_Op<"tcgen05.shift"> { + let description = [{ + The `tcgen05.shift` is an asynchronous instruction which initiates + the shifting of 32-byte elements downwards across all the rows, + except the last, by one row. The operand `taddr` specifies the base + address of the matrix in Tensor Memory whose rows must be down shifted. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-shift) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$taddr, + OptionalAttr:$group + ); + let results = (outs); +} +def NVVM_Tcgen05StOp : NVVM_Op<"tcgen05.st"> { + let description = [{ + Instruction `tcgen05.st` asynchronously stores data from the source register `r` + into the Tensor Memory at the location specified by the 32-bit address operand + `tmemAddr`, collectively across all threads of the warps. + + The `shape` and the `num` attribute together determines the total dimension of + the data which is stored to the Tensor Memory. The `shape` indicates the base + dimension of data to be accessed. The `num` attribute indicates the repeat + factor on the base dimension resulting in the total dimension of the data that + is accessed. + + The shape `16x32bx2` performs two accesses into Tensor Memory of the shape + `16x32b`. The base address of the first access is specified by `tmemAddr` + and the base address of the second access is specified by + `tmemAddr + offset`, where `offset` is an immediate argument. + + The unit attribute `unpack` can be used to unpack a 32-bit element + in the register into two 16-bit elements and store them in adjacent columns. + + The following table describes the size of the vector for various combinations + of `num` and `shape` attributes: + ``` + |=====================================================================| + | num/shape | 16x32bx2/16x64b/32x32b | 16x128b | 16x256b | + |=====================================================================| + | x1 | 1 | 2 | 4 | + | x2 | 2 | 4 | 8 | + | x4 | 4 | 8 | 16 | + | x8 | 8 | 16 | 32 | + | x16 | 16 | 32 | 64 | + | x32 | 32 | 64 | 128 | + | x64 | 64 | 128 | NA | + | x128 | 128 | NA | NA | + |=====================================================================| + ``` + + Example: + ```mlir + nvvm.tcgen05.st %tmemAddr, %val, %offset unpack { + shape = #nvvm.tcgen05_ldst_shape, + } : <2xi32> + ``` + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-st) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$tmemAddr, + NVVM_T32BitSignlessIntegerOrVector2:$val, + Optional:$offset, + UnitAttr:$unpack, + NVVM_Tcgen05LdStShapeAttr:$shape + ); + let results = (outs); +} +def NVVM_Tcgen05WaitOp : NVVM_Op<"tcgen05.wait"> { + let description = [{ + The `tcgen05.wait` causes the executing thread to block until + all prior `tcgen05.ld` operations issued by the executing thread + have completed. Similarly, the `tcgen05.wait` causes the executing + thread to block until all prior `tcgen05.st` operations issued by the + executing thread have completed. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-wait) + }]; + let arguments = (ins + NVVM_Tcgen05WaitKindAttr:$kind + ); + let results = (outs); +} +def NVVM_Tcgen05MmaSmemDescOpV2 : NVVM_Op<"tcgen05_mma_smem_desc_v2", [MemoryEffects<[MemRead]>]> { + let description = [{ + The `nvvm.tcgen05_mma_smem_desc_v2` constructs a Shared Memory descriptor for use as the MMA + Shared Memory operand. It configures the memory layout according to the provided parameters. + + The descriptor contains the following bitfields: + - Bits 63-61 (3 bits): Swizzle descriptor type + 0 = SWIZZLE_NONE matrix descriptor + 1 = SWIZZLE_128B_ATOM_32B + 2 = SWIZZLE_128B matrix descriptor + 3 = RESERVED + 4 = SWIZZLE_64B descriptor + 5 = RESERVED + 6 = SWIZZLE_32B descriptor + 7 = RESERVED + - Bits 60-54 (7 bits): Reserved + - Bit 53 (1 bit): UTCQMMA.LUTB Matrix B K_Segment start offset in K=128 (48B) block + 0: K[0:63] = GDESC.start_address + 0B + 1: K[64:127] = GDESC.start_address + 24B + - Bit 52 (1 bit): Matrix A or B leading dimension mode + 0 = BYTE_OFFSET (legacy) + 1 = NEXT_BUFFER_ADDR (circular buffer) + - Bits 51-49 (3 bits): Matrix A or B base offset (valid only for matrix descriptor 1, 2 or 3, 4) + - Bits 48-46 (3 bits): Matrix A or B descriptor version, needs to be set to 1 + - Bits 45-32 (14 bits): Matrix A or B stride dimension byte offset, 16 byte aligned, 4 LSBs not included + - Bit 31 (1 bit): Reserved + - Bits 30-16 (15 bits): Matrix A or B leading dimension byte offset or next buffer start address + If LEADING_DIM_MODE == BYTE_OFFSET: Matrix A or B leading dimension byte offset + If LEADING_DIM_MODE == NEXT_BUFFER_ADDR: Matrix A or B next buffer start address + Offset or address is 16 byte aligned, 4 LSBs not included + - Bit 15 (1 bit): Reserved + - Bits 14-0 (15 bits): Matrix A or B start address, 16 byte aligned, 4 LSBs not included + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$startAddress, + NVVM_T32BitSignlessInteger:$leadingDimOffset, + NVVM_T32BitSignlessInteger:$strideDimOffset, + NVVM_T8BitSignlessInteger2:$descriptorVersion, + NVVM_T8BitSignlessInteger2:$baseOffset, + NVVM_T1BitSignlessInteger:$leadingDimMode, + NVVM_T1BitSignlessInteger:$kSegmentOffset, + NVVM_T8BitSignlessInteger2:$swizzleType + ); + let results = (outs + NVVM_T64BitSignlessInteger2:$res + ); +} +def NVVM_TensormapCpFenceProxyOp : NVVM_Op<"tensormap.cp_fenceproxy"> { + let description = [{ + The `tensormap.cp_fenceproxy` is a fused copy and fence operation. + Instructions perform the following operations in order: + + Copies data of size specified by the `size` argument, in bytes, from the + location specified by the address operand `src` in shared memory to the + location specified by the address operand `dst` in the global memory, in + the generic proxy. + + Establishes a uni-directional proxy release pattern on the ordering from + the copy operation to the subsequent access performed in the tensormap + proxy on the address `dst` + + The valid value of `size` operand is 128 and must be an immediate. + + The operands `src` and `dst` specify non-generic addresses in shared::cta and + global state space respectively + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace1:$dst, + NVVM_LLVMPointerInAddressSpace3:$src, + NVVM_T32BitSignlessInteger:$size, + NVVM_MemScopeKindAttr:$scope + ); + let results = (outs); +} +def NVVM_TensormapReplaceOp : NVVM_Op<"tensormap.replace"> { + let description = [{ + The `nvvm.tensormap.replace` replaces the specified field of the tensor-map + object at the location specified by `addr` with a new value (specified by + `new_value` or `new_value_attr`). + + The `field` argument specifies the field of the tensor-map object to + replace. + + `new_value` is an `i32`/`i64` argument that specifies the new value to + replace the `field` with for the `global_address`, `rank`, `box_dim`, + `global_dim`, `global_stride`, and `element_stride` fields. It must be an + `i64` for the `global_address` and `global_stride` fields and `i32` for the + remaining fields. + + For `rank`, `new_value` must be one less than the desired tensor rank as + this field uses zero-based numbering. + + `new_value_attr` is an attribute that specifies the new value to replace + the `field` with for the `elemtype`, `interleave_layout`, `swizzle_mode`, + `swizzle_atomicity`, and `fill_mode` fields. It takes the place of + `new_value` for these fields. It must be a valid attribute corresponding to + the `field` type. + + The ordinal `ord` is an immediate integer argument that specifies the + ordinal of the `field` across the tensor which needs to be replaced and is + required only for the `box_dim`, `global_dim`, `global_stride`, and + `element_stride` fields. + + [For more information, see PTX ISA.](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-tensormap-replace) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace14:$addr, + Optional:$new_value, + NVVM_TensormapFieldAttr:$field, + OptionalAttr:$ord, + OptionalAttr:$new_value_attr + ); + let results = (outs); +} +def NVVM_TraceMarkOp : NVVM_Op<"trace.mark", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.trace.mark` operation introduces trace events for profiling + programs. + + It takes `event_type`, a 12-bit value in the range 0-4095, which denotes + the type of the event. Additionally, it takes `domain` and `event`, which + are strings that identify the domain and event name of the trace event. + Finally, it takes an optional 32-bit or 64-bit payload, an optional + payload descriptor (required when payload is provided), and a predicate to + conditionally execute the trace event. + + Examples: + + ```mlir + // Basic trace event + nvvm.trace.mark event_type = 42, domain = "myDomain", event = "myEvent" + + // With 32-bit payload + nvvm.trace.mark event_type = 42, payload = %p, + domain = "myDomain", event = "myEvent", payload_descriptor = "counter" + : i32 + + // Conditional trace event + nvvm.trace.mark event_type = 42, predicate = %pred, + domain = "myDomain", event = "myEvent" + ``` + }]; + let arguments = (ins + Optional:$payload, + Optional:$predicate, + I32Attr:$eventType, + StrAttr:$domain, + StrAttr:$event, + OptionalAttr:$payloadDescriptor + ); + let results = (outs); +} +def NVVM_VoteSyncOp : NVVM_InferredOp<"vote.sync"> { + let description = [{ + The `vote.sync` op will cause executing thread to wait until all non-exited + threads corresponding to membermask have executed `vote.sync` with the same + qualifiers and same membermask value before resuming execution. + + The vote operation kinds are: + - `any`: True if source predicate is True for some thread in membermask. + - `all`: True if source predicate is True for all non-exited threads in + membermask. + - `uni`: True if source predicate has the same value in all non-exited + threads in membermask. + - `ballot`: In the ballot form, the destination result is a 32 bit integer. + In this form, the predicate from each thread in membermask are copied into + the corresponding bit position of the result, where the bit position + corresponds to the thread's lane id. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-vote-sync) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$mask, + NVVM_T1BitSignlessInteger:$pred, + NVVM_VoteSyncKindAttr:$kind + ); + let results = (outs + NVVM_T32BitSignlessIntegerOr1:$res + ); +} +def NVVM_WgmmaGroupSyncAlignedOp : NVVM_Op<"wgmma.commit.group.sync.aligned"> { + let description = [{ + Commits all prior uncommitted warpgroup level matrix multiplication operations. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-commit-group) + }]; + let arguments = (ins); + let results = (outs); +} +def NVVM_WgmmaFenceAlignedOp : NVVM_Op<"wgmma.fence.aligned"> { + let description = [{ + Enforce an ordering of register accesses between warpgroup level matrix + multiplication and other operations. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-fence) + }]; + let arguments = (ins); + let results = (outs); +} +def NVVM_WgmmaMmaAsyncOp : NVVM_Op<"wgmma.mma_async"> { + let description = [{ + The warpgroup (128 threads) level matrix multiply and accumulate operation + has either of the following forms, where matrix D is called accumulator: + D = A * B + D + D = A * B, where the input from accumulator D is disabled. + + Supported shapes: + ``` + |--------------|--------------|------------|--------------|---------------| + | | | | |f16+=e4m3*e4m3 | + | | | | |f16+=e5m2*e5m2 | + |f32+=tf32*tf32|f16+=f16 *f16 | s32+=s8*s8 |s32 += b1 * b1|f16+=e5m2*e4m3 | + | |f32+=f16 *f16 | s32+=u8*u8 | |f16+=e4m3*e5m2 | + | |f32+=bf16*bf16| s32+=u8*u8 | |f16+=e4m3*e5m2 | + | |f32+=bf16*bf16| s32+=s8*u8 | |f32+=e4m3*e4m3 | + | | | s32+=u8*s8 | |f32+=e5m2*e5m2 | + | | | | |f32+=e4m3*e5m2 | + | | | | |f32+=e4m3*e5m2 | + |--------------|--------------|------------|--------------|---------------| + | .m64n8k8 | .m64n8k16 | .m64n8k32 | .m64n8k256 | .m64n8k32 | + | .m64n16k8 | .m64n16k16 | .m64n16k32 | .m64n16k256 | .m64n16k32 | + | .m64n24k8 | .m64n24k16 | .m64n24k32 | .m64n24k256 | .m64n24k32 | + | .m64n32k8 | .m64n32k16 | .m64n32k32 | .m64n32k256 | .m64n32k32 | + | .m64n40k8 | .m64n40k16 | .m64n48k32 | .m64n48k256 | .m64n40k32 | + | .m64n48k8 | .m64n48k16 | .m64n64k32 | .m64n64k256 | .m64n48k32 | + | .m64n56k8 | .m64n56k16 | .m64n80k32 | .m64n80k256 | .m64n56k32 | + | .m64n64k8 | .m64n64k16 | .m64n96k32 | .m64n96k256 | .m64n64k32 | + | .m64n72k8 | .m64n72k16 | .m64n112k32| .m64n112k256 | .m64n72k32 | + | .m64n80k8 | .m64n80k16 | .m64n128k32| .m64n128k256 | .m64n80k32 | + | .m64n88k8 | .m64n88k16 | .m64n144k32| .m64n144k256 | .m64n88k32 | + | .m64n96k8 | .m64n96k16 | .m64n160k32| .m64n160k256 | .m64n96k32 | + | .m64n104k8 | .m64n104k16 | .m64n176k32| .m64n176k256 | .m64n104k32 | + | .m64n112k8 | .m64n112k16 | .m64n192k32| .m64n192k256 | .m64n112k32 | + | .m64n120k8 | .m64n120k16 | .m64n208k32| .m64n208k256 | .m64n120k32 | + | .m64n128k8 | .m64n128k16 | .m64n224k32| .m64n224k256 | .m64n128k32 | + | .m64n136k8 | .m64n136k16 | .m64n240k32| .m64n240k256 | .m64n136k32 | + | .m64n144k8 | .m64n144k16 | .m64n256k32| .m64n256k256 | .m64n144k32 | + | .m64n152k8 | .m64n152k16 | | | .m64n152k32 | + | .m64n160k8 | .m64n160k16 | | | .m64n160k32 | + | .m64n168k8 | .m64n168k16 | | | .m64n168k32 | + | .m64n176k8 | .m64n176k16 | | | .m64n176k32 | + | .m64n184k8 | .m64n184k16 | | | .m64n184k32 | + | .m64n192k8 | .m64n192k16 | | | .m64n192k32 | + | .m64n200k8 | .m64n200k16 | | | .m64n200k32 | + | .m64n208k8 | .m64n208k16 | | | .m64n208k32 | + | .m64n216k8 | .m64n216k16 | | | .m64n216k32 | + | .m64n224k8 | .m64n224k16 | | | .m64n224k32 | + | .m64n232k8 | .m64n232k16 | | | .m64n232k32 | + | .m64n240k8 | .m64n240k16 | | | .m64n240k32 | + | .m64n248k8 | .m64n248k16 | | | .m64n248k32 | + | .m64n256k8 | .m64n256k16 | | | .m64n256k32 | + |--------------|--------------|------------|--------------|---------------| + ``` + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions) + }]; + let arguments = (ins + NVVM_LLVMStructureType:$inouts, + NVVM_T64BitSignlessInteger:$descriptorA, + NVVM_T64BitSignlessInteger:$descriptorB, + NVVM_MMAShapeAttr:$shape, + NVVM_WGMMATypesAttr:$typeA, + NVVM_WGMMATypesAttr:$typeB, + NVVM_WGMMATypesAttr:$typeD, + NVVM_WGMMAScaleOutAttr:$scaleD, + NVVM_WGMMAScaleInAttr:$scaleA, + NVVM_WGMMAScaleInAttr:$scaleB, + NVVM_MMALayoutAttr:$layoutA, + NVVM_MMALayoutAttr:$layoutB, + OptionalAttr:$satfinite + ); + let results = (outs + NVVM_LLVMStructureType:$results_ + ); +} +def NVVM_WgmmaWaitGroupSyncOp : NVVM_Op<"wgmma.wait.group.sync.aligned"> { + let description = [{ + Signal the completion of a preceding warpgroup operation. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-wait-group) + }]; + let arguments = (ins + I64Attr:$group + ); + let results = (outs); +} +def NVVM_WMMALoadOp : NVVM_Op<"wmma.load"> { + let arguments = (ins + NVVM_LLVMPointerType:$ptr, + NVVM_T32BitSignlessInteger:$stride, + I32Attr:$m, + I32Attr:$n, + I32Attr:$k, + NVVM_MMALayoutAttr:$layout, + NVVM_MMATypesAttr:$eltype, + NVVM_MMAFragAttr:$frag + ); + let results = (outs + NVVM_LLVMStructureTypeOr64Bit:$res + ); +} +def NVVM_WMMAMmaOp : NVVM_Op<"wmma.mma"> { + let arguments = (ins + Variadic:$args, + I32Attr:$m, + I32Attr:$n, + I32Attr:$k, + NVVM_MMALayoutAttr:$layoutA, + NVVM_MMALayoutAttr:$layoutB, + NVVM_MMATypesAttr:$eltypeA, + NVVM_MMATypesAttr:$eltypeB + ); + let results = (outs + NVVM_LLVMStructureType:$res + ); +} +def NVVM_WMMAStoreOp : NVVM_Op<"wmma.store"> { + let arguments = (ins + NVVM_LLVMPointerType:$ptr, + Variadic:$args, + NVVM_T32BitSignlessInteger:$stride, + I32Attr:$m, + I32Attr:$n, + I32Attr:$k, + NVVM_MMALayoutAttr:$layout, + NVVM_MMATypesAttr:$eltype + ); + let results = (outs); +} + +#endif // CUTE_IR_GENERATED_NVVM_OPS_TD diff --git a/mlir/dialects/cute_ir/lib/CAPI/BUILD.bazel b/mlir/dialects/cute_ir/lib/CAPI/BUILD.bazel index 213c3f0c..b471d7f4 100644 --- a/mlir/dialects/cute_ir/lib/CAPI/BUILD.bazel +++ b/mlir/dialects/cute_ir/lib/CAPI/BUILD.bazel @@ -32,3 +32,32 @@ cc_library( ], alwayslink = True, ) + +cc_library( + name = "cuda", + srcs = [ + "Cuda.cpp", + "CudaAttributes.cpp", + "CudaTypes.cpp", + ], + deps = [ + "//mlir/dialects/cute_ir/include/cute_ir-c/Dialect:cuda", + "//mlir/dialects/cute_ir/lib/Dialect/Cuda/IR:dialect", + "@llvm-project//mlir:CAPIIR", + ], + alwayslink = True, +) + +cc_library( + name = "nvvm", + srcs = [ + "NVVM.cpp", + "NVVMAttributes.cpp", + ], + deps = [ + "//mlir/dialects/cute_ir/include/cute_ir-c/Dialect:nvvm", + "//mlir/dialects/cute_ir/lib/Dialect/NVVM/IR:dialect", + "@llvm-project//mlir:CAPIIR", + ], + alwayslink = True, +) diff --git a/mlir/dialects/cute_ir/lib/CAPI/Cuda.cpp b/mlir/dialects/cute_ir/lib/CAPI/Cuda.cpp new file mode 100644 index 00000000..eef03db0 --- /dev/null +++ b/mlir/dialects/cute_ir/lib/CAPI/Cuda.cpp @@ -0,0 +1,10 @@ +//===- Cuda.cpp - C API for NVIDIA's CUDA host dialect of the CuTe DSL ------===// + +#include "cute_ir-c/Dialect/Cuda.h" + +#include "cute_ir/Dialect/Cuda/IR/CudaDialect.h" +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION( + Cuda, cuda, mlir::cutlass_compiler::cuda::CudaDialect) diff --git a/mlir/dialects/cute_ir/lib/CAPI/CudaAttributes.cpp b/mlir/dialects/cute_ir/lib/CAPI/CudaAttributes.cpp new file mode 100644 index 00000000..33bbac33 --- /dev/null +++ b/mlir/dialects/cute_ir/lib/CAPI/CudaAttributes.cpp @@ -0,0 +1,136 @@ +// C API for the cuda attributes, generated from the dialect's .td +// files. +#include "cute_ir-c/Dialect/CudaAttributes.h" + +#include "cute_ir/Dialect/Cuda/IR/CudaDialect.h" +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Support.h" + +namespace cuda = mlir::cutlass_compiler::cuda; + +namespace { +// Unwraps `handle` into `out`: false when it is of another context or not a +// T, or when it is null and not optional. +template +bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { + auto value = unwrap(handle); + if (!value) + return optional; + if (value.getContext() != ctx) + return false; + out = llvm::dyn_cast(value); + return static_cast(out); +} +} // namespace + +bool mlirAttributeIsACudaAssumeKernel(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCudaAssumeKernelAttrGet(MlirContext context, bool value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::AssumeKernelAttr::get(ctx, value)); +} +bool mlirCudaAssumeKernelAttrGetValue(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getValue(); +} + +bool mlirAttributeIsACudaComputeTarget(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCudaComputeTargetAttrGet(MlirContext context, uint32_t representation, uint32_t portability, intptr_t numArchs, const uint32_t *archs) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto representationValue = ::mlir::cutlass_compiler::cuda::symbolizeExecutableRepresentation(representation); + if (!representationValue) + return {nullptr}; + auto portabilityValue = ::mlir::cutlass_compiler::cuda::symbolizeArchPortability(portability); + if (!portabilityValue) + return {nullptr}; + llvm::SmallVector<::mlir::cutlass_compiler::cuda::GpuArchitecture> archsValue; + for (intptr_t i = 0; i < numArchs; ++i) { + auto value = ::mlir::cutlass_compiler::cuda::symbolizeGpuArchitecture(archs[i]); + if (!value) + return {nullptr}; + archsValue.push_back(*value); + } + return wrap(cuda::ComputeTargetAttr::get(ctx, *representationValue, *portabilityValue, archsValue)); +} +uint32_t mlirCudaComputeTargetAttrGetRepresentation(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getRepresentation()); +} +uint32_t mlirCudaComputeTargetAttrGetPortability(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getPortability()); +} +intptr_t mlirCudaComputeTargetAttrGetNumArchs(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getArchs().size(); +} +uint32_t mlirCudaComputeTargetAttrGetArchs(MlirAttribute attr, intptr_t pos) { + return static_cast(llvm::cast(unwrap(attr)).getArchs()[pos]); +} + +bool mlirAttributeIsACudaDevMaxSharedMemoryOptin(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCudaDevMaxSharedMemoryOptinAttrGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::DevMaxSharedMemoryOptinAttr::get(ctx)); +} + +bool mlirAttributeIsACudaDeviceAttributes(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCudaDeviceAttributesAttrGet(MlirContext context, MlirAttribute values) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::DictionaryAttr valuesValue; + if (!unwrapAs(ctx, values, false, valuesValue)) + return {nullptr}; + return wrap(cuda::DeviceAttributesAttr::get(ctx, valuesValue)); +} +MlirAttribute mlirCudaDeviceAttributesAttrGetValues(MlirAttribute attr) { + return wrap(static_cast(llvm::cast(unwrap(attr)).getValues())); +} + +bool mlirAttributeIsACudaExecutable(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCudaExecutableAttrGet(MlirContext context, uint32_t format) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto formatValue = ::mlir::cutlass_compiler::cuda::symbolizeExecutableFormat(format); + if (!formatValue) + return {nullptr}; + return wrap(cuda::ExecutableAttr::get(ctx, *formatValue)); +} +uint32_t mlirCudaExecutableAttrGetFormat(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getFormat()); +} + +bool mlirAttributeIsACudaFuncAttributes(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCudaFuncAttributesAttrGet(MlirContext context, MlirAttribute values) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::DictionaryAttr valuesValue; + if (!unwrapAs(ctx, values, false, valuesValue)) + return {nullptr}; + return wrap(cuda::FuncAttributesAttr::get(ctx, valuesValue)); +} +MlirAttribute mlirCudaFuncAttributesAttrGetValues(MlirAttribute attr) { + return wrap(static_cast(llvm::cast(unwrap(attr)).getValues())); +} diff --git a/mlir/dialects/cute_ir/lib/CAPI/CudaTypes.cpp b/mlir/dialects/cute_ir/lib/CAPI/CudaTypes.cpp new file mode 100644 index 00000000..751f7ed5 --- /dev/null +++ b/mlir/dialects/cute_ir/lib/CAPI/CudaTypes.cpp @@ -0,0 +1,378 @@ +// C API for the cuda types, generated from the dialect's .td files. +#include "cute_ir-c/Dialect/CudaTypes.h" + +#include "cute_ir/Dialect/Cuda/IR/CudaDialect.h" +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Support.h" + +namespace cuda = mlir::cutlass_compiler::cuda; + +namespace { +// Unwraps `handle` into `out`: false when it is of another context or not a +// T, or when it is null and not optional. +template +bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { + auto value = unwrap(handle); + if (!value) + return optional; + if (value.getContext() != ctx) + return false; + out = llvm::dyn_cast(value); + return static_cast(out); +} +} // namespace + +bool mlirTypeIsACudaArchSet(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaArchSetTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::ArchSetType::get(ctx)); +} + +bool mlirTypeIsACudaEvent(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaEventTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::EventType::get(ctx)); +} + +bool mlirTypeIsACudaGraphDeviceNode(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaGraphDeviceNodeTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::GraphDeviceNodeType::get(ctx)); +} + +bool mlirTypeIsACudaGraphExec(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaGraphExecTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::GraphExecType::get(ctx)); +} + +bool mlirTypeIsACudaGraphNode(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaGraphNodeTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::GraphNodeType::get(ctx)); +} + +bool mlirTypeIsACudaGraph(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaGraphTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::GraphType::get(ctx)); +} + +bool mlirTypeIsACudaInt(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaIntTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::IntType::get(ctx)); +} + +bool mlirTypeIsACudaKernel(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaKernelTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::KernelType::get(ctx)); +} + +bool mlirTypeIsACudaLaunchConfig(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaLaunchConfigTypeGet(MlirContext context, uint32_t maxNumAttrs) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::LaunchConfigType::get(ctx, maxNumAttrs)); +} +uint32_t mlirCudaLaunchConfigTypeGetMaxNumAttrs(MlirType type) { + return llvm::cast(unwrap(type)).getMaxNumAttrs(); +} + +bool mlirTypeIsACudaLibrary(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaLibraryTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::LibraryType::get(ctx)); +} + +bool mlirTypeIsACudaModule(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaModuleTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::ModuleType::get(ctx)); +} + +bool mlirTypeIsACudaResult(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaResultTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::ResultType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeDeviceProp(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeDevicePropTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeDevicePropType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeDim3(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeDim3TypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeDim3Type::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeExtent(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeExtentTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeExtentType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeKernelNodeParams(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeKernelNodeParamsTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeKernelNodeParamsType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreference(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValueCooperative(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeLaunchAttributeValueCooperativeTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeLaunchAttributeValueCooperativeType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValueMemSyncDomain(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeLaunchAttributeValueMemSyncDomainTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeLaunchAttributeValueMemSyncDomainType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValueNvlinkUtilCentricScheduling(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValuePortableClusterSizeMode(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeLaunchAttributeValuePortableClusterSizeModeTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeLaunchAttributeValuePortableClusterSizeModeType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValuePriority(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeLaunchAttributeValuePriorityTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeLaunchAttributeValuePriorityType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowed(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemCarveout(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeLaunchAttributeValueSharedMemCarveoutTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeLaunchAttributeValueSharedMemCarveoutType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemoryMode(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeLaunchAttributeValueSharedMemoryModeTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeLaunchAttributeValueSharedMemoryModeType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValueSyncPolicy(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeLaunchAttributeValueSyncPolicyTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeLaunchAttributeValueSyncPolicyType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeMemcpy3DParams(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeMemcpy3DParamsTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeMemcpy3DParamsType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimePitchedPtr(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimePitchedPtrTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimePitchedPtrType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimePos(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimePosTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimePosType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeUuid(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeUuidTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeUuidType::get(ctx)); +} + +bool mlirTypeIsACudaStream(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaStreamTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::StreamType::get(ctx)); +} + +bool mlirTypeIsACudaTensorMap(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaTensorMapTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::TensorMapType::get(ctx)); +} diff --git a/mlir/dialects/cute_ir/lib/CAPI/NVVM.cpp b/mlir/dialects/cute_ir/lib/CAPI/NVVM.cpp new file mode 100644 index 00000000..e290c183 --- /dev/null +++ b/mlir/dialects/cute_ir/lib/CAPI/NVVM.cpp @@ -0,0 +1,10 @@ +//===- NVVM.cpp - C API for NVIDIA's NVVM dialect of the CuTe DSL ------===// + +#include "cute_ir-c/Dialect/NVVM.h" + +#include "cute_ir/Dialect/NVVM/IR/NVVMDialect.h" +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION( + CuteNVVM, cute_nvvm, mlir::cutlass_compiler::nvvm::NVVMDialect) diff --git a/mlir/dialects/cute_ir/lib/CAPI/NVVMAttributes.cpp b/mlir/dialects/cute_ir/lib/CAPI/NVVMAttributes.cpp new file mode 100644 index 00000000..c3dc0042 --- /dev/null +++ b/mlir/dialects/cute_ir/lib/CAPI/NVVMAttributes.cpp @@ -0,0 +1,1681 @@ +// C API for the nvvm attributes, generated from the dialect's .td +// files. +#include "cute_ir-c/Dialect/NVVMAttributes.h" + +#include "cute_ir/Dialect/NVVM/IR/NVVMDialect.h" +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Support.h" + +namespace nvvm = mlir::cutlass_compiler::nvvm; + +namespace { +// Unwraps `handle` into `out`: false when it is of another context or not a +// T, or when it is null and not optional. +template +bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { + auto value = unwrap(handle); + if (!value) + return optional; + if (value.getContext() != ctx) + return false; + out = llvm::dyn_cast(value); + return static_cast(out); +} +} // namespace + +bool mlirAttributeIsACuteNVVMAtomicOpKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMAtomicOpKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeAtomicOpKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::AtomicOpKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMAtomicOpKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMBarrierReduction(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMBarrierReductionAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeBarrierReduction(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::BarrierReductionAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMBarrierReductionAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMBarrierReduxKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMBarrierReduxKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeBarrierReduxKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::BarrierReduxKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMBarrierReduxKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMBlockScaleFormat(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMBlockScaleFormatAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeBlockScaleFormat(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::BlockScaleFormatAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMBlockScaleFormatAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMCTAGroupKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMCTAGroupKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeCTAGroupKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::CTAGroupKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMCTAGroupKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMCVTPackFloatKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMCVTPackFloatKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeCVTPackFloatKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::CVTPackFloatKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMCVTPackFloatKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMCacheEvictionPriority(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMCacheEvictionPriorityAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeCacheEvictionPriority(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::CacheEvictionPriorityAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMCacheEvictionPriorityAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMClusterLaunchControlQueryType(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMClusterLaunchControlQueryTypeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeClusterLaunchControlQueryType(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ClusterLaunchControlQueryTypeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMClusterLaunchControlQueryTypeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMCompareOpKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMCompareOpKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeCompareOpKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::CompareOpKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMCompareOpKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMConvertFP4Type(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMConvertFP4TypeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeConvertFP4Type(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ConvertFP4TypeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMConvertFP4TypeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMConvertFP8Type(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMConvertFP8TypeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeConvertFP8Type(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ConvertFP8TypeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMConvertFP8TypeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMConvertScaleKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMConvertScaleKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeConvertScaleKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ConvertScaleKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMConvertScaleKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMDotAccumulateType(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMDotAccumulateTypeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeDotAccumulateType(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::DotAccumulateTypeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMDotAccumulateTypeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMEvictKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMEvictKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeEvictKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::EvictKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMEvictKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMFPRoundingMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMFPRoundingModeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeFPRoundingMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::FPRoundingModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMFPRoundingModeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMGridDepActionKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMGridDepActionKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeGridDepActionKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::GridDepActionKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMGridDepActionKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMIntegerRoundingMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMIntegerRoundingModeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeIntegerRoundingMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::IntegerRoundingModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMIntegerRoundingModeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVML2PrefetchSize(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVML2PrefetchSizeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeL2PrefetchSize(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::L2PrefetchSizeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVML2PrefetchSizeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMLdStMatrixEltType(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMLdStMatrixEltTypeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeLdStMatrixEltType(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::LdStMatrixEltTypeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMLdStMatrixEltTypeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMLoadCacheModifierExtKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMLoadCacheModifierExtKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeLoadCacheModifierExtKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::LoadCacheModifierExtKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMLoadCacheModifierExtKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMLoadCacheModifierKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMLoadCacheModifierKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeLoadCacheModifierKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::LoadCacheModifierKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMLoadCacheModifierKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMLoadShape(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMLoadShapeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeLoadShape(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::LoadShapeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMLoadShapeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMLoadSrcFormat(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMLoadSrcFormatAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeLoadSrcFormat(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::LoadSrcFormatAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMLoadSrcFormatAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMBarrierLayout(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMBarrierLayoutAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierLayout(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MBarrierLayoutAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMBarrierLayoutAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMBarrierPhase(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMBarrierPhaseAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierPhase(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MBarrierPhaseAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMBarrierPhaseAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMBarrierScopeKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMBarrierScopeKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierScopeKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MBarrierScopeKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMBarrierScopeKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMBarrierSpaceKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMBarrierSpaceKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierSpaceKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MBarrierSpaceKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMBarrierSpaceKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMBarrierTxnKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMBarrierTxnKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierTxnKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MBarrierTxnKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMBarrierTxnKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMBarrierWaitKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMBarrierWaitKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierWaitKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MBarrierWaitKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMBarrierWaitKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMMAB1Op(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMAB1OpAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMAB1Op(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MMAB1OpAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMMAB1OpAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMMABlockScaleKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMABlockScaleKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMABlockScaleKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MMABlockScaleKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMMABlockScaleKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMMACtaCount(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMACtaCountAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMACtaCount(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MMACtaCountAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMMACtaCountAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMMAFrag(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMAFragAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMAFrag(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MMAFragAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMMAFragAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMMAIntOverflow(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMAIntOverflowAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMAIntOverflow(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MMAIntOverflowAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMMAIntOverflowAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMMAKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMAKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMAKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MMAKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMMAKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMMALayout(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMALayoutAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMALayout(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MMALayoutAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMMALayoutAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMMATypes(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMATypesAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMATypes(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MMATypesAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMMATypesAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMatchSyncKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMatchSyncKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMatchSyncKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MatchSyncKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMatchSyncKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMemOrderKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMemOrderKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMemOrderKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MemOrderKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMemOrderKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMemScopeKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMemScopeKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMemScopeKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MemScopeKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMemScopeKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMulMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMulModeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMulMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MulModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMulModeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMNVVMMemorySpace(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMNVVMMemorySpaceAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeNVVMMemorySpace(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::NVVMMemorySpaceAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMNVVMMemorySpaceAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMPermuteMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMPermuteModeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizePermuteMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::PermuteModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMPermuteModeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMPrefetchCacheLevel(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMPrefetchCacheLevelAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizePrefetchCacheLevel(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::PrefetchCacheLevelAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMPrefetchCacheLevelAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMProxyKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMProxyKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeProxyKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ProxyKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMProxyKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMReductionKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMReductionKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeReductionKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ReductionKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMReductionKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMReductionOp(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMReductionOpAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeReductionOp(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ReductionOpAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMReductionOpAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMReductionType(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMReductionTypeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeReductionType(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ReductionTypeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMReductionTypeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPCompressElemSize(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPCompressElemSizeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressElemSize(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPCompressElemSizeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPCompressElemSizeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPCompressFactorType(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPCompressFactorTypeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressFactorType(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPCompressFactorTypeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPCompressFactorTypeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPCompressIndexSize(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPCompressIndexSizeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressIndexSize(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPCompressIndexSizeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPCompressIndexSizeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPCompressOpKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPCompressOpKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressOpKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPCompressOpKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPCompressOpKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPCompressRepFactor(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPCompressRepFactorAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressRepFactor(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPCompressRepFactorAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPCompressRepFactorAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPDecompressElemSize(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPDecompressElemSizeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSPDecompressElemSize(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPDecompressElemSizeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPDecompressElemSizeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPDecompressFactorType(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPDecompressFactorTypeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSPDecompressFactorType(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPDecompressFactorTypeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPDecompressFactorTypeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPDecompressIndexSize(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPDecompressIndexSizeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSPDecompressIndexSize(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPDecompressIndexSizeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPDecompressIndexSizeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPDecompressRepFactor(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPDecompressRepFactorAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSPDecompressRepFactor(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPDecompressRepFactorAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPDecompressRepFactorAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSaturationMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSaturationModeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSaturationMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SaturationModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSaturationModeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSaturationModeKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSaturationModeKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSaturationModeKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SaturationModeKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSaturationModeKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMScaleVecSize(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMScaleVecSizeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeScaleVecSize(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ScaleVecSizeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMScaleVecSizeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSetMaxRegisterAction(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSetMaxRegisterActionAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSetMaxRegisterAction(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SetMaxRegisterActionAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSetMaxRegisterActionAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSharedSpace(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSharedSpaceAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSharedSpace(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SharedSpaceAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSharedSpaceAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMShflKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMShflKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeShflKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ShflKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMShflKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSparsityFormat(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSparsityFormatAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSparsityFormat(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SparsityFormatAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSparsityFormatAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMStateSpace(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMStateSpaceAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeStateSpace(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::StateSpaceAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMStateSpaceAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMStoreCacheModifierKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMStoreCacheModifierKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeStoreCacheModifierKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::StoreCacheModifierKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMStoreCacheModifierKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMStoreShape(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMStoreShapeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeStoreShape(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::StoreShapeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMStoreShapeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTCBarParam(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTCBarParamAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTCBarParam(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TCBarParamAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTCBarParamAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTMALoadMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTMALoadModeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTMALoadMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TMALoadModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTMALoadModeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTMAReduxKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTMAReduxKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTMAReduxKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TMAReduxKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTMAReduxKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTMAStoreMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTMAStoreModeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTMAStoreMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TMAStoreModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTMAStoreModeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05CpMulticast(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05CpMulticastAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05CpMulticast(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05CpMulticastAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05CpMulticastAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05CpShape(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05CpShapeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05CpShape(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05CpShapeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05CpShapeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05CpSrcFormat(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05CpSrcFormatAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05CpSrcFormat(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05CpSrcFormatAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05CpSrcFormatAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05FenceKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05FenceKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05FenceKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05FenceKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05FenceKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05LdStShape(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05LdStShapeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05LdStShape(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05LdStShapeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05LdStShapeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05MMABlockScale(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05MMABlockScaleAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05MMABlockScale(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05MMABlockScaleAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05MMABlockScaleAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05MMACollectorBBuffer(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05MMACollectorBBufferAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05MMACollectorBBuffer(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05MMACollectorBBufferAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05MMACollectorBBufferAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05MMACollectorOp(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05MMACollectorOpAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05MMACollectorOp(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05MMACollectorOpAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05MMACollectorOpAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05MMAKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05MMAKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05MMAKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05MMAKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05MMAKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05WaitKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05WaitKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05WaitKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05WaitKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05WaitKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTensormapElemtype(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTensormapElemtypeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTensormapElemtype(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TensormapElemtypeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTensormapElemtypeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTensormapField(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTensormapFieldAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTensormapField(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TensormapFieldAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTensormapFieldAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTensormapFillMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTensormapFillModeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTensormapFillMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TensormapFillModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTensormapFillModeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTensormapInterleaveLayout(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTensormapInterleaveLayoutAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTensormapInterleaveLayout(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TensormapInterleaveLayoutAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTensormapInterleaveLayoutAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTensormapSwizzleAtomicity(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTensormapSwizzleAtomicityAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTensormapSwizzleAtomicity(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TensormapSwizzleAtomicityAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTensormapSwizzleAtomicityAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTensormapSwizzleMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTensormapSwizzleModeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTensormapSwizzleMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TensormapSwizzleModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTensormapSwizzleModeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTmemLayout(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTmemLayoutAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTmemLayout(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TmemLayoutAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTmemLayoutAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMValidatePattern(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMValidatePatternAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeValidatePattern(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ValidatePatternAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMValidatePatternAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMVoteSyncKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMVoteSyncKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeVoteSyncKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::VoteSyncKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMVoteSyncKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMWGMMAScaleIn(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMWGMMAScaleInAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeWGMMAScaleIn(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::WGMMAScaleInAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMWGMMAScaleInAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMWGMMAScaleOut(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMWGMMAScaleOutAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeWGMMAScaleOut(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::WGMMAScaleOutAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMWGMMAScaleOutAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMWGMMATypes(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMWGMMATypesAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeWGMMATypes(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::WGMMATypesAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMWGMMATypesAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMLdStMatrixShape(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMLdStMatrixShapeAttrGet(MlirContext context, int m, int n) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(nvvm::LdStMatrixShapeAttr::get(ctx, m, n)); +} +int mlirCuteNVVMLdStMatrixShapeAttrGetM(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getM(); +} +int mlirCuteNVVMLdStMatrixShapeAttrGetN(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getN(); +} + +bool mlirAttributeIsACuteNVVMMMAShape(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMAShapeAttrGet(MlirContext context, int m, int n, int k) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(nvvm::MMAShapeAttr::get(ctx, m, n, k)); +} +int mlirCuteNVVMMMAShapeAttrGetM(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getM(); +} +int mlirCuteNVVMMMAShapeAttrGetN(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getN(); +} +int mlirCuteNVVMMMAShapeAttrGetK(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getK(); +} + +bool mlirAttributeIsACuteNVVMTarget(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTargetAttrGet(MlirContext context, int O, MlirStringRef triple, MlirStringRef chip, MlirStringRef features, MlirAttribute flags, MlirAttribute link, bool verifyTarget) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::DictionaryAttr flagsValue; + if (!unwrapAs(ctx, flags, true, flagsValue)) + return {nullptr}; + ::mlir::ArrayAttr linkValue; + if (!unwrapAs(ctx, link, true, linkValue)) + return {nullptr}; + return wrap(nvvm::TargetAttr::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, O, unwrap(triple), unwrap(chip), unwrap(features), flagsValue, linkValue, verifyTarget)); +} +int mlirCuteNVVMTargetAttrGetO(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getO(); +} +MlirStringRef mlirCuteNVVMTargetAttrGetTriple(MlirAttribute attr) { + return wrap(llvm::cast(unwrap(attr)).getTriple()); +} +MlirStringRef mlirCuteNVVMTargetAttrGetChip(MlirAttribute attr) { + return wrap(llvm::cast(unwrap(attr)).getChip()); +} +MlirStringRef mlirCuteNVVMTargetAttrGetFeatures(MlirAttribute attr) { + return wrap(llvm::cast(unwrap(attr)).getFeatures()); +} +MlirAttribute mlirCuteNVVMTargetAttrGetFlags(MlirAttribute attr) { + return wrap(static_cast(llvm::cast(unwrap(attr)).getFlags())); +} +MlirAttribute mlirCuteNVVMTargetAttrGetLink(MlirAttribute attr) { + return wrap(static_cast(llvm::cast(unwrap(attr)).getLink())); +} +bool mlirCuteNVVMTargetAttrGetVerifyTarget(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getVerifyTarget(); +} diff --git a/mlir/dialects/cute_ir/lib/Dialect/Cuda/IR/BUILD.bazel b/mlir/dialects/cute_ir/lib/Dialect/Cuda/IR/BUILD.bazel new file mode 100644 index 00000000..946266c0 --- /dev/null +++ b/mlir/dialects/cute_ir/lib/Dialect/Cuda/IR/BUILD.bazel @@ -0,0 +1,18 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "dialect", + srcs = [ + "CudaDialect.cpp", + "CudaInference.cpp", + ], + deps = [ + "//mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR:headers", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:LLVMDialect", + ], +) diff --git a/mlir/dialects/cute_ir/lib/Dialect/Cuda/IR/CudaDialect.cpp b/mlir/dialects/cute_ir/lib/Dialect/Cuda/IR/CudaDialect.cpp new file mode 100644 index 00000000..53353381 --- /dev/null +++ b/mlir/dialects/cute_ir/lib/Dialect/Cuda/IR/CudaDialect.cpp @@ -0,0 +1,38 @@ +//===- CudaDialect.cpp - NVIDIA's cuda dialect of the CuTe DSL -----------===// + +#include "cute_ir/Dialect/Cuda/IR/CudaDialect.h" + +#include "llvm/ADT/TypeSwitch.h" +#include "mlir/IR/Builders.h" +#include "mlir/IR/DialectImplementation.h" + +using namespace mlir; +using namespace mlir::cutlass_compiler::cuda; + +#include "cute_ir/Dialect/Cuda/IR/CudaDialect.cpp.inc" + +#include "cute_ir/Dialect/Cuda/IR/CudaEnums.cpp.inc" + +#define GET_ATTRDEF_CLASSES +#include "cute_ir/Dialect/Cuda/IR/CudaAttrs.cpp.inc" + +#define GET_TYPEDEF_CLASSES +#include "cute_ir/Dialect/Cuda/IR/CudaTypes.cpp.inc" + +#define GET_OP_CLASSES +#include "cute_ir/Dialect/Cuda/IR/CudaOps.cpp.inc" + +void CudaDialect::initialize() { + addAttributes< +#define GET_ATTRDEF_LIST +#include "cute_ir/Dialect/Cuda/IR/CudaAttrs.cpp.inc" + >(); + addTypes< +#define GET_TYPEDEF_LIST +#include "cute_ir/Dialect/Cuda/IR/CudaTypes.cpp.inc" + >(); + addOperations< +#define GET_OP_LIST +#include "cute_ir/Dialect/Cuda/IR/CudaOps.cpp.inc" + >(); +} diff --git a/mlir/dialects/cute_ir/lib/Dialect/Cuda/IR/CudaInference.cpp b/mlir/dialects/cute_ir/lib/Dialect/Cuda/IR/CudaInference.cpp new file mode 100644 index 00000000..b13ce484 --- /dev/null +++ b/mlir/dialects/cute_ir/lib/Dialect/Cuda/IR/CudaInference.cpp @@ -0,0 +1,17 @@ +//===- CudaInference.cpp - result types the compiler infers ------------===// +// +// Generated from the nvidia-cutlass-dsl 4.8.0 compiler (its inference +// probed by varying operand types); do not edit. An operation without a +// recovered rule fails inference, so the result types written stand. + +#include "cute_ir/Dialect/Cuda/IR/CudaDialect.h" + +using namespace mlir; +using namespace ::mlir::cutlass_compiler::cuda; + +LogicalResult LaunchCfgCreateFromStreamOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} diff --git a/mlir/dialects/cute_ir/lib/Dialect/NVVM/IR/BUILD.bazel b/mlir/dialects/cute_ir/lib/Dialect/NVVM/IR/BUILD.bazel new file mode 100644 index 00000000..dddca500 --- /dev/null +++ b/mlir/dialects/cute_ir/lib/Dialect/NVVM/IR/BUILD.bazel @@ -0,0 +1,18 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "dialect", + srcs = [ + "NVVMDialect.cpp", + "NVVMInference.cpp", + ], + deps = [ + "//mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR:headers", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:LLVMDialect", + ], +) diff --git a/mlir/dialects/cute_ir/lib/Dialect/NVVM/IR/NVVMDialect.cpp b/mlir/dialects/cute_ir/lib/Dialect/NVVM/IR/NVVMDialect.cpp new file mode 100644 index 00000000..28cc6bfd --- /dev/null +++ b/mlir/dialects/cute_ir/lib/Dialect/NVVM/IR/NVVMDialect.cpp @@ -0,0 +1,46 @@ +//===- NVVMDialect.cpp - NVIDIA's nvvm dialect of the CuTe DSL -----------===// + +#include "cute_ir/Dialect/NVVM/IR/NVVMDialect.h" + +#include "llvm/ADT/TypeSwitch.h" +#include "mlir/IR/Builders.h" +#include "mlir/IR/DialectImplementation.h" + +using namespace mlir; +using namespace mlir::cutlass_compiler::nvvm; + +#include "cute_ir/Dialect/NVVM/IR/NVVMDialect.cpp.inc" + +#include "cute_ir/Dialect/NVVM/IR/NVVMEnums.cpp.inc" + +#define GET_ATTRDEF_CLASSES +#include "cute_ir/Dialect/NVVM/IR/NVVMAttrs.cpp.inc" + +#define GET_OP_CLASSES +#include "cute_ir/Dialect/NVVM/IR/NVVMOps.cpp.inc" + +void NVVMDialect::initialize() { + addAttributes< +#define GET_ATTRDEF_LIST +#include "cute_ir/Dialect/NVVM/IR/NVVMAttrs.cpp.inc" + >(); + addOperations< +#define GET_OP_LIST +#include "cute_ir/Dialect/NVVM/IR/NVVMOps.cpp.inc" + >(); +} + +// The compiler's rules and messages. +LogicalResult TargetAttr::verify(function_ref emitError, + int O, StringRef triple, StringRef chip, + StringRef, DictionaryAttr, ArrayAttr link, bool) { + if (O < 0 || O > 3) + return emitError() << "The optimization level must be a number between 0 and 3."; + if (triple.empty()) + return emitError() << "The target triple cannot be empty."; + if (chip.empty()) + return emitError() << "The target chip cannot be empty."; + if (link && !llvm::all_of(link, [](Attribute attr) { return attr && isa(attr); })) + return emitError() << "All the elements in the `link` array must be strings."; + return success(); +} diff --git a/mlir/dialects/cute_ir/lib/Dialect/NVVM/IR/NVVMInference.cpp b/mlir/dialects/cute_ir/lib/Dialect/NVVM/IR/NVVMInference.cpp new file mode 100644 index 00000000..b0ae6e5a --- /dev/null +++ b/mlir/dialects/cute_ir/lib/Dialect/NVVM/IR/NVVMInference.cpp @@ -0,0 +1,227 @@ +//===- NVVMInference.cpp - result types the compiler infers ------------===// +// +// Generated from the nvidia-cutlass-dsl 4.8.0 compiler (its inference +// probed by varying operand types); do not edit. An operation without a +// recovered rule fails inference, so the result types written stand. + +#include "cute_ir/Dialect/NVVM/IR/NVVMDialect.h" + +using namespace mlir; +using namespace ::mlir::cutlass_compiler::nvvm; + +LogicalResult AddFOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(0); + if (group.empty()) return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult AddPackedF32x2Op::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult AtomicRMWOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult BarrierOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult ClusterLaunchControlQueryCancelOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult CompareAndSetOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(0); + if (group.empty()) return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult FAbsOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(0); + if (group.empty()) return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult FMAPackedF32x2Op::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult FmaOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult InternalMmaBlockScaleOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(8); + if (group.empty()) return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult InternalMmaSparseBlockScaleOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(8); + if (group.empty()) return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult LdMatrixOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult MBarrierArriveDropExpectTxOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult MBarrierArriveDropOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult MBarrierArriveExpectTxOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult MBarrierArriveOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult MBarrierTestWaitOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult MBarrierTryWaitOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult MatchSyncOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult MulOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult MulPackedF32x2Op::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult ReduxOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult RsqrtOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(0); + if (group.empty()) return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult ShflOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult SubFOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(0); + if (group.empty()) return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult SubPackedF32x2Op::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult VoteSyncOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} diff --git a/mlir/dialects/cute_ir/nvvm.zig b/mlir/dialects/cute_ir/nvvm.zig new file mode 100644 index 00000000..75cfcae3 --- /dev/null +++ b/mlir/dialects/cute_ir/nvvm.zig @@ -0,0 +1,7129 @@ +//! Zig bindings of NVIDIA's NVVM dialect of the CuTe DSL (nvidia-cutlass-dsl 4.8.0), generated +//! from the dialect's .td files; do not edit. + +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub const dialect_namespace = "nvvm"; +/// The C dialect handle, `mlirGetDialectHandle__cute_nvvm__`. +pub const dialect_handle = "cute_nvvm"; + +// Enums, generated from the dialect's .td files; values are the compiler's. + +/// operations supported by atom instruction. +pub const AtomicOpKind = enum(u32) { + @"and" = 0, + @"or" = 1, + xor = 2, + cas = 3, + exch = 4, + add = 5, + inc = 6, + dec = 7, + min = 8, + max = 9, + umin = 10, + umax = 11, + fadd = 12, + sub = 13, +}; + +/// NVVM barrier reduction operation. +pub const BarrierReduction = enum(u32) { + popc = 0, + @"and" = 1, + @"or" = 2, +}; + +/// NVVM barrier redux kind. +pub const BarrierReduxKind = enum(u32) { + @"and" = 0, + @"or" = 1, + popc = 2, +}; + +/// MMA Block Scale Format. +pub const BlockScaleFormat = enum(u32) { + ue8m0 = 0, + ue4m3 = 1, +}; + +/// NVVM CTA group kind. +pub const CTAGroupKind = enum(u32) { + cta_1 = 0, + cta_2 = 1, +}; + +/// NVVM CVT Pack Float kind. +pub const CVTPackFloatKind = enum(u32) { + f32 = 0, + f16x2 = 1, + e4m3x2 = 2, + e5m2x2 = 3, + bf16x2 = 4, + e2m1x2 = 5, + e2m3x2 = 6, + e3m2x2 = 7, + ue8m0x2 = 8, +}; + +/// NVVM Cache Eviction Priority. +pub const CacheEvictionPriority = enum(u32) { + evict_normal = 0, + evict_first = 1, + evict_last = 2, + evict_unchanged = 3, + no_allocate = 4, +}; + +/// NVVM ClusterLaunchControlQueryType. +pub const ClusterLaunchControlQueryType = enum(u32) { + is_canceled = 0, + get_first_cta_id_x = 1, + get_first_cta_id_y = 2, + get_first_cta_id_z = 3, +}; + +/// Comparison operator encoding. +pub const CompareOpKind = enum(u32) { + eq = 0, + ne = 1, + ugt = 2, + uge = 3, + ult = 4, + ule = 5, + sgt = 6, + sge = 7, + slt = 8, + sle = 9, +}; + +/// NVVM ConvertFP4Type kind. +pub const ConvertFP4Type = enum(u32) { + e2m1 = 0, +}; + +/// NVVM ConvertFP8Type kind. +pub const ConvertFP8Type = enum(u32) { + e4m3 = 0, + e5m2 = 1, + ue8m0 = 2, + ue5m3 = 3, +}; + +/// NVVM ConvertScale kind. +pub const ConvertScaleKind = enum(u32) { + none = 0, + ue8m0 = 1, +}; + +/// NVVM DotAccumulateType. +pub const DotAccumulateType = enum(u32) { + signed = 1, + unsigned = 0, +}; + +/// NVVM L2 Prefetch Size. +pub const EvictKind = enum(u32) { + evict_normal = 0, + evict_first = 1, + evict_last = 2, + evict_unchanged = 3, + evict_no_allocate = 4, + none = 5, +}; + +/// NVVM FPRoundingMode kind. +pub const FPRoundingMode = enum(u32) { + none = 0, + rn = 1, + rm = 2, + rp = 3, + rz = 4, + rna = 5, + rs = 6, +}; + +/// Action kind for grid dependency control. +pub const GridDepActionKind = enum(u32) { + wait = 0, + launch_dependents = 1, +}; + +/// NVVM IntegerRoundingMode kind. +pub const IntegerRoundingMode = enum(u32) { + none = 0, + rni = 1, + rzi = 2, + rmi = 3, + rpi = 4, +}; + +/// NVVM L2 Prefetch Size. +pub const L2PrefetchSize = enum(u32) { + none = 0, + reserved = 1, + size_64b = 2, + size_128b = 3, + size_256b = 4, +}; + +/// Element type for ldmatrix and stmatrix. +pub const LdStMatrixEltType = enum(u32) { + b16 = 0, + b8 = 1, + @"b8x16.b6x16_p32" = 2, + @"b8x16.b4x16_p64" = 3, +}; + +/// NVVM load cache modifier kind(Ext). +pub const LoadCacheModifierExtKind = enum(u32) { + ca = 0, + cg = 1, + cs = 2, + lu = 3, + cv = 4, + none = 5, +}; + +/// NVVM load cache modifier kind. +pub const LoadCacheModifierKind = enum(u32) { + ca = 0, + cg = 1, + cs = 2, + lu = 3, + cv = 4, +}; + +/// shape attribute for ldmatrix. +pub const LoadShape = enum(u32) { + m8n8 = 0, + m8n16 = 1, + m16n16 = 3, +}; + +/// source format for ldmatrix. +pub const LoadSrcFormat = enum(u32) { + b6x16_p32 = 0, + b4x16_p64 = 1, + b8 = 2, +}; + +/// NVVM MBarrier Layout. +pub const MBarrierLayout = enum(u32) { + v0 = 0, + v1 = 1, +}; + +/// NVVM mbarrier phase type. +pub const MBarrierPhase = enum(u32) { + none = 0, + primary = 1, + conditional = 2, +}; + +/// NVVM MBarrier scope kind. +pub const MBarrierScopeKind = enum(u32) { + cta = 0, + cluster = 1, +}; + +/// NVVM MBarrier space kind. +pub const MBarrierSpaceKind = enum(u32) { + cta = 0, + cluster = 1, +}; + +/// NVVM MBarrier Transaction kind. +pub const MBarrierTxnKind = enum(u32) { + arrive = 0, + arrive_drop = 1, + arrive_expect_tx = 2, + arrive_drop_expect_tx = 3, + expect_tx = 4, + complete_tx = 5, +}; + +/// NVVM MBarrier wait kind. +pub const MBarrierWaitKind = enum(u32) { + @"test" = 0, + @"try" = 1, +}; + +/// MMA binary operations. +pub const MMAB1Op = enum(u32) { + none = 0, + xor_popc = 1, + and_popc = 2, +}; + +/// Block Scale Kind. +pub const MMABlockScaleKind = enum(u32) { + mxf8f6f4 = 0, + mxf4 = 1, + mxf4nvf4 = 2, +}; + +/// MMA CTA count. +pub const MMACtaCount = enum(u32) { + cta1 = 1, + cta2 = 2, +}; + +/// NVVM MMA frag type. +pub const MMAFrag = enum(u32) { + a = 0, + b = 1, + c = 2, +}; + +/// MMA overflow options. +pub const MMAIntOverflow = enum(u32) { + satfinite = 1, + wrapped = 0, +}; + +/// MMA operation kind. +pub const MMAKind = enum(u32) { + f8f6f4 = 0, +}; + +/// NVVM MMA layout. +pub const MMALayout = enum(u32) { + row = 0, + col = 1, +}; + +/// NVVM MMA types. +pub const MMATypes = enum(u32) { + f16 = 0, + f32 = 1, + tf32 = 2, + bf16 = 9, + s8 = 4, + u8 = 3, + s32 = 5, + s4 = 8, + u4 = 7, + b1 = 6, + f64 = 10, + e4m3 = 11, + e5m2 = 12, + e3m2 = 13, + e2m3 = 14, + e2m1 = 15, +}; + +/// NVVM match sync kind. +pub const MatchSyncKind = enum(u32) { + any = 0, + all = 1, +}; + +/// NVVM Memory Ordering kind. +pub const MemOrderKind = enum(u32) { + weak = 0, + relaxed = 1, + acquire = 2, + release = 3, + acq_rel = 4, + sc = 5, + mmio = 6, + @"volatile" = 8, + constant = 7, +}; + +/// NVVM Memory Scope kind. +pub const MemScopeKind = enum(u32) { + cta = 0, + cluster = 1, + gpu = 2, + sys = 3, +}; + +/// multiply mode attribute. +pub const MulMode = enum(u32) { + hi = 0, + lo = 1, + wide = 2, +}; + +/// NVVM Memory Space. +pub const NVVMMemorySpace = enum(u32) { + generic = 0, + global = 1, + shared = 3, + constant = 4, + local = 5, + tensor = 6, + shared_cluster = 7, +}; + +/// NVVM permute mode. +pub const PermuteMode = enum(u32) { + default = 0, + f4e = 1, + b4e = 2, + rc8 = 3, + ecl = 4, + ecr = 5, + rc16 = 6, +}; + +/// NVVM Prefetch Cache Level. +pub const PrefetchCacheLevel = enum(u32) { + L1 = 0, + L2 = 1, +}; + +/// Proxy kind. +pub const ProxyKind = enum(u32) { + alias = 0, + async = 1, + @"async.global" = 2, + @"async.shared" = 3, + tensormap = 4, + generic = 5, +}; + +/// NVVM Reduction Kind attribute. +pub const ReductionKind = enum(u32) { + add = 1, + @"and" = 2, + max = 3, + min = 4, + @"or" = 5, + umax = 6, + umin = 7, + xor = 8, + fmin = 9, + fmax = 10, +}; + +/// Ops supported by red instruction. +pub const ReductionOp = enum(u32) { + @"and" = 0, + @"or" = 1, + xor = 2, + add = 3, + inc = 4, + dec = 5, + min = 6, + max = 7, +}; + +/// types supported by red instruction. +pub const ReductionType = enum(u32) { + b32 = 0, + b64 = 1, + u32 = 2, + u64 = 3, + s32 = 4, + s64 = 5, + f32 = 6, + f64 = 7, + f16 = 8, + f16x2 = 9, + bf16 = 10, + bf16x2 = 11, +}; + +/// Sparse tensor compression element size. +pub const SPCompressElemSize = enum(u32) { + b4 = 4, + b8 = 8, + b16 = 16, +}; + +/// Sparse tensor compression factor type. +pub const SPCompressFactorType = enum(u32) { + sp2to4 = 0, +}; + +/// Sparse tensor compression index size. +pub const SPCompressIndexSize = enum(u32) { + b2 = 2, + b4 = 4, +}; + +/// spcompress operation kind. +pub const SPCompressOpKind = enum(u32) { + min = 0, + max = 1, +}; + +/// Sparse tensor compression repetition factor. +pub const SPCompressRepFactor = enum(u32) { + x1 = 1, + x2 = 2, + x4 = 4, + x8 = 8, + x16 = 16, + x32 = 32, + x64 = 64, + x128 = 128, +}; + +/// Sparse tensor decompression element size. +pub const SPDecompressElemSize = enum(u32) { + b4 = 4, + b8 = 8, + b16 = 16, +}; + +/// Sparse tensor decompression factor type. +pub const SPDecompressFactorType = enum(u32) { + sp1to2 = 0, + sp1to4 = 1, + sp1to8 = 2, + sp1to16 = 3, + sp2to4 = 4, + sp2to8 = 5, + sp2to16 = 6, + sp4to8 = 7, + sp4to16 = 8, + sp8to16 = 9, +}; + +/// Sparse tensor decompression index size. +pub const SPDecompressIndexSize = enum(u32) { + b2 = 2, + b4 = 4, +}; + +/// Sparse tensor decompression repetition factor. +pub const SPDecompressRepFactor = enum(u32) { + x1 = 1, + x2 = 2, + x4 = 4, + x8 = 8, + x16 = 16, + x32 = 32, + x64 = 64, +}; + +/// NVVM SaturationMode kind. +pub const SaturationMode = enum(u32) { + none = 0, + satfinite = 1, + sat = 2, +}; + +/// NVVM SaturationMode kind. +pub const SaturationModeKind = enum(u32) { + none = 0, + satfinite = 1, +}; + +/// MMA Scale Vector Sizes. +pub const ScaleVecSize = enum(u32) { + x1 = 0, + x2 = 1, + x4 = 2, +}; + +/// NVVM set max register action. +pub const SetMaxRegisterAction = enum(u32) { + decrease = 1, + increase = 0, +}; + +/// Shared memory space. +pub const SharedSpace = enum(u32) { + cta = 0, + cluster = 1, +}; + +/// NVVM shuffle kind. +pub const ShflKind = enum(u32) { + bfly = 0, + up = 1, + down = 2, + idx = 3, +}; + +/// MMA Sparsity Format. +pub const SparsityFormat = enum(u32) { + thread = 0, +}; + +/// NVVM State Space. +pub const StateSpace = enum(u32) { + generic = 0, + global = 1, + shared_cta = 2, + shared_cluster = 3, + constant = 4, + local = 5, + tensor = 6, +}; + +/// NVVM store cache modifier kind. +pub const StoreCacheModifierKind = enum(u32) { + wb = 0, + cg = 1, + cs = 2, + wt = 3, + none = 4, +}; + +/// shape attribute for stmatrix. +pub const StoreShape = enum(u32) { + m8n8 = 0, + m16n8 = 2, +}; + +/// Cluster MMA Barrier Parameter Type. +pub const TCBarParam = enum(u32) { + a1t0 = 0, + a0tx = 1, +}; + +/// NVVM TMA Load Mode. +pub const TMALoadMode = enum(u32) { + tile = 0, + im2col = 1, + im2col_w = 2, + im2col_w_128 = 3, + tile_gather4 = 4, +}; + +/// NVVM TMA redux kind. +pub const TMAReduxKind = enum(u32) { + add = 0, + max = 2, + min = 1, + inc = 3, + dec = 4, + @"and" = 5, + @"or" = 6, + xor = 7, +}; + +/// NVVM TMA Store Mode. +pub const TMAStoreMode = enum(u32) { + tile = 0, + im2col = 1, + tile_scatter4 = 2, + im2col_w = 3, +}; + +/// tcgen05 cp multicast. +pub const Tcgen05CpMulticast = enum(u32) { + none = 0, + warpx2_02_13 = 1, + warpx2_01_23 = 2, + warpx4 = 3, +}; + +/// tcgen05 cp shapes. +pub const Tcgen05CpShape = enum(u32) { + shape_128x256b = 0, + shape_4x256b = 1, + shape_128x128b = 2, + shape_64x128b = 3, + shape_32x128b = 4, +}; + +/// tcgen05 cp source format. +pub const Tcgen05CpSrcFormat = enum(u32) { + b6x16_p32 = 0, + b4x16_p64 = 1, +}; + +/// NVVM Tcgen05 fence kind. +pub const Tcgen05FenceKind = enum(u32) { + before = 0, + after = 1, +}; + +/// allowed 32-bit signless integer cases: 0, 1, 2, 3, 4. +pub const Tcgen05LdStShape = enum(u32) { + shape_16x64b = 0, + shape_16x128b = 1, + shape_16x256b = 2, + shape_32x32b = 3, + shape_16x32bx2 = 4, +}; + +/// tcgen05.mma block scale attribute. +pub const Tcgen05MMABlockScale = enum(u32) { + default = 0, + block16 = 1, + block32 = 2, +}; + +/// tcgen05 MMA Collector Buffer B Attribute. +pub const Tcgen05MMACollectorBBuffer = enum(u32) { + b0 = 0, + b1 = 1, + b2 = 2, + b3 = 3, +}; + +/// tcgen05.mma Collector Buffer Operation. +pub const Tcgen05MMACollectorOp = enum(u32) { + discard = 0, + lastuse = 1, + fill = 2, + use = 3, +}; + +/// tcgen05 MMA Supported Types. +pub const Tcgen05MMAKind = enum(u32) { + f16 = 0, + tf32 = 1, + f8f6f4 = 2, + i8 = 3, + mxf8f6f4 = 4, + mxf4 = 5, + mxf4nvf4 = 6, + ti16 = 7, +}; + +/// NVVM Tcgen05 wait kind. +pub const Tcgen05WaitKind = enum(u32) { + load = 0, + store = 1, +}; + +/// NVVM Tensormap Elemtype. +pub const TensormapElemtype = enum(u32) { + u8 = 0, + u16 = 1, + u32 = 2, + s32 = 3, + u64 = 4, + s64 = 5, + f16 = 6, + f32 = 7, + @"f32.ftz" = 8, + f64 = 9, + bf16 = 10, + tf32 = 11, + @"tf32.ftz" = 12, + b4x16 = 13, + b4x16_p64 = 14, + b6x16_p32 = 15, +}; + +/// NVVM Tensormap Field Kind. +pub const TensormapField = enum(u32) { + global_address = 0, + rank = 1, + box_dim = 2, + global_dim = 3, + global_stride = 4, + element_stride = 5, + elemtype = 6, + interleave_layout = 7, + swizzle_mode = 8, + swizzle_atomicity = 9, + fill_mode = 10, +}; + +/// NVVM Tensormap Fill Mode. +pub const TensormapFillMode = enum(u32) { + zero = 0, + oob_nan = 1, +}; + +/// NVVM Tensormap Interleave Layout. +pub const TensormapInterleaveLayout = enum(u32) { + no_interleave = 0, + b16 = 1, + b32 = 2, +}; + +/// NVVM Tensormap Swizzle Atomicity. +pub const TensormapSwizzleAtomicity = enum(u32) { + b16 = 0, + b32 = 1, + b32_flip_b8 = 2, + b64 = 3, +}; + +/// NVVM Tensormap Swizzle Mode. +pub const TensormapSwizzleMode = enum(u32) { + no_swizzling = 0, + b32 = 1, + b64 = 2, + b128 = 3, + b96 = 4, +}; + +/// Tensor Memory Layout Enumerated Type. +pub const TmemLayout = enum(u32) { + tmem_16dp_128bit = 0, + tmem_16dp_256bit = 1, + tmem_32dp_32bit = 2, + tmem_16dp_64bit = 3, + tmem_16dp_32bit_t0_t15 = 4, + tmem_16dp_32bit_t16_t31 = 5, +}; + +/// NVVM validate data pattern. +pub const ValidatePattern = enum(u32) { + none = 0, + per_16bytes_80000000 = 1, + per_16bytes_8000 = 2, + per_16bytes_80 = 3, + per_16bytes_8 = 4, + per_element_ff = 5, + per_element_80000000 = 6, + per_element_8000 = 7, + per_element_80 = 8, + per_element_8 = 9, + per_16bytes_80000000_paired = 10, +}; + +/// NVVM vote sync kind. +pub const VoteSyncKind = enum(u32) { + any = 0, + all = 1, + ballot = 2, + uni = 3, +}; + +/// WGMMA overflow options. +pub const WGMMAScaleIn = enum(u32) { + one = 1, + neg = 2, +}; + +/// WGMMA input predicate. +pub const WGMMAScaleOut = enum(u32) { + zero = 0, + one = 1, +}; + +/// NVVM WGMMA types. +pub const WGMMATypes = enum(u32) { + f16 = 0, + tf32 = 1, + u8 = 2, + s8 = 3, + b1 = 4, + bf16 = 5, + e4m3 = 6, + e5m2 = 7, + f32 = 8, + s32 = 9, +}; + +// Types and attributes, generated from the dialect's .td files. + +/// `#nvvm.atomic_op`: operations supported by atom instruction +pub const AtomicOpKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMAtomicOpKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "atomic_op"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomicOpKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMAtomicOpKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) AtomicOpKind { + return @enumFromInt(c.mlirCuteNVVMAtomicOpKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.reduction`: NVVM barrier reduction operation +pub const BarrierReductionAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMBarrierReduction; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "reduction"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: BarrierReduction, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMBarrierReductionAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) BarrierReduction { + return @enumFromInt(c.mlirCuteNVVMBarrierReductionAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.barrier_redux_kind`: NVVM barrier redux kind +pub const BarrierReduxKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMBarrierReduxKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "barrier_redux_kind"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: BarrierReduxKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMBarrierReduxKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) BarrierReduxKind { + return @enumFromInt(c.mlirCuteNVVMBarrierReduxKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.block_scale_format`: MMA Block Scale Format +pub const BlockScaleFormatAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMBlockScaleFormat; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "block_scale_format"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: BlockScaleFormat, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMBlockScaleFormatAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) BlockScaleFormat { + return @enumFromInt(c.mlirCuteNVVMBlockScaleFormatAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.cta_group`: NVVM CTA group kind +pub const CTAGroupKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMCTAGroupKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "cta_group"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: CTAGroupKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMCTAGroupKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) CTAGroupKind { + return @enumFromInt(c.mlirCuteNVVMCTAGroupKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.packfloat_type`: NVVM CVT Pack Float kind +pub const CVTPackFloatKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMCVTPackFloatKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "packfloat_type"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: CVTPackFloatKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMCVTPackFloatKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) CVTPackFloatKind { + return @enumFromInt(c.mlirCuteNVVMCVTPackFloatKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.cache_eviction_priority`: NVVM Cache Eviction Priority +pub const CacheEvictionPriorityAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMCacheEvictionPriority; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "cache_eviction_priority"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: CacheEvictionPriority, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMCacheEvictionPriorityAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) CacheEvictionPriority { + return @enumFromInt(c.mlirCuteNVVMCacheEvictionPriorityAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.cluster_launch_control_query_type`: NVVM ClusterLaunchControlQueryType +pub const ClusterLaunchControlQueryTypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMClusterLaunchControlQueryType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "cluster_launch_control_query_type"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ClusterLaunchControlQueryType, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMClusterLaunchControlQueryTypeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) ClusterLaunchControlQueryType { + return @enumFromInt(c.mlirCuteNVVMClusterLaunchControlQueryTypeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.op`: Comparison operator encoding +pub const CompareOpKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMCompareOpKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "op"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: CompareOpKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMCompareOpKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) CompareOpKind { + return @enumFromInt(c.mlirCuteNVVMCompareOpKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.convert_fp4_type`: NVVM ConvertFP4Type kind +pub const ConvertFP4TypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMConvertFP4Type; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "convert_fp4_type"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ConvertFP4Type, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMConvertFP4TypeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) ConvertFP4Type { + return @enumFromInt(c.mlirCuteNVVMConvertFP4TypeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.convert_fp8_type`: NVVM ConvertFP8Type kind +pub const ConvertFP8TypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMConvertFP8Type; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "convert_fp8_type"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ConvertFP8Type, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMConvertFP8TypeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) ConvertFP8Type { + return @enumFromInt(c.mlirCuteNVVMConvertFP8TypeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.convert_scale_kind`: NVVM ConvertScale kind +pub const ConvertScaleKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMConvertScaleKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "convert_scale_kind"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ConvertScaleKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMConvertScaleKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) ConvertScaleKind { + return @enumFromInt(c.mlirCuteNVVMConvertScaleKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.dot_accumulate_type`: NVVM DotAccumulateType +pub const DotAccumulateTypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMDotAccumulateType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "dot_accumulate_type"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: DotAccumulateType, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMDotAccumulateTypeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) DotAccumulateType { + return @enumFromInt(c.mlirCuteNVVMDotAccumulateTypeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.evict_kind`: NVVM L2 Prefetch Size +pub const EvictKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMEvictKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "evict_kind"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: EvictKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMEvictKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) EvictKind { + return @enumFromInt(c.mlirCuteNVVMEvictKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.fp_rnd_mode`: NVVM FPRoundingMode kind +pub const FPRoundingModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMFPRoundingMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "fp_rnd_mode"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: FPRoundingMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMFPRoundingModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) FPRoundingMode { + return @enumFromInt(c.mlirCuteNVVMFPRoundingModeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.grid_dep_action`: Action kind for grid dependency control +pub const GridDepActionKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMGridDepActionKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "grid_dep_action"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: GridDepActionKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMGridDepActionKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) GridDepActionKind { + return @enumFromInt(c.mlirCuteNVVMGridDepActionKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.int_rnd_mode`: NVVM IntegerRoundingMode kind +pub const IntegerRoundingModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMIntegerRoundingMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "int_rnd_mode"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: IntegerRoundingMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMIntegerRoundingModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) IntegerRoundingMode { + return @enumFromInt(c.mlirCuteNVVMIntegerRoundingModeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.l2_prefetch`: NVVM L2 Prefetch Size +pub const L2PrefetchSizeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVML2PrefetchSize; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "l2_prefetch"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: L2PrefetchSize, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVML2PrefetchSizeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) L2PrefetchSize { + return @enumFromInt(c.mlirCuteNVVML2PrefetchSizeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.ld_st_matrix_elt_type`: Element type for ldmatrix and stmatrix +pub const LdStMatrixEltTypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMLdStMatrixEltType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "ld_st_matrix_elt_type"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: LdStMatrixEltType, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMLdStMatrixEltTypeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) LdStMatrixEltType { + return @enumFromInt(c.mlirCuteNVVMLdStMatrixEltTypeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.load_cache_modifier_ext`: NVVM load cache modifier kind(Ext) +pub const LoadCacheModifierExtKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMLoadCacheModifierExtKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "load_cache_modifier_ext"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: LoadCacheModifierExtKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMLoadCacheModifierExtKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) LoadCacheModifierExtKind { + return @enumFromInt(c.mlirCuteNVVMLoadCacheModifierExtKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.load_cache_modifier`: NVVM load cache modifier kind +pub const LoadCacheModifierKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMLoadCacheModifierKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "load_cache_modifier"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: LoadCacheModifierKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMLoadCacheModifierKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) LoadCacheModifierKind { + return @enumFromInt(c.mlirCuteNVVMLoadCacheModifierKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.load_shape`: shape attribute for ldmatrix +pub const LoadShapeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMLoadShape; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "load_shape"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: LoadShape, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMLoadShapeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) LoadShape { + return @enumFromInt(c.mlirCuteNVVMLoadShapeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.load_src_format`: source format for ldmatrix +pub const LoadSrcFormatAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMLoadSrcFormat; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "load_src_format"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: LoadSrcFormat, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMLoadSrcFormatAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) LoadSrcFormat { + return @enumFromInt(c.mlirCuteNVVMLoadSrcFormatAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.mbarrier_layout`: NVVM MBarrier Layout +pub const MBarrierLayoutAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "mbarrier_layout"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MBarrierLayout, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMBarrierLayoutAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) MBarrierLayout { + return @enumFromInt(c.mlirCuteNVVMMBarrierLayoutAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.mbarrier_phase`: NVVM mbarrier phase type +pub const MBarrierPhaseAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierPhase; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "mbarrier_phase"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MBarrierPhase, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMBarrierPhaseAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) MBarrierPhase { + return @enumFromInt(c.mlirCuteNVVMMBarrierPhaseAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.mbar_scope`: NVVM MBarrier scope kind +pub const MBarrierScopeKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierScopeKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "mbar_scope"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MBarrierScopeKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMBarrierScopeKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) MBarrierScopeKind { + return @enumFromInt(c.mlirCuteNVVMMBarrierScopeKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.mbar_space`: NVVM MBarrier space kind +pub const MBarrierSpaceKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierSpaceKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "mbar_space"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MBarrierSpaceKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMBarrierSpaceKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) MBarrierSpaceKind { + return @enumFromInt(c.mlirCuteNVVMMBarrierSpaceKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.mbar_txn_kind`: NVVM MBarrier Transaction kind +pub const MBarrierTxnKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierTxnKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "mbar_txn_kind"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MBarrierTxnKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMBarrierTxnKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) MBarrierTxnKind { + return @enumFromInt(c.mlirCuteNVVMMBarrierTxnKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.mbar_wait`: NVVM MBarrier wait kind +pub const MBarrierWaitKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierWaitKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "mbar_wait"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MBarrierWaitKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMBarrierWaitKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) MBarrierWaitKind { + return @enumFromInt(c.mlirCuteNVVMMBarrierWaitKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.mma_b1op`: MMA binary operations +pub const MMAB1OpAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMAB1Op; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "mma_b1op"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMAB1Op, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMAB1OpAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) MMAB1Op { + return @enumFromInt(c.mlirCuteNVVMMMAB1OpAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.block_scale_kind`: Block Scale Kind +pub const MMABlockScaleKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMABlockScaleKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "block_scale_kind"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMABlockScaleKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMABlockScaleKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) MMABlockScaleKind { + return @enumFromInt(c.mlirCuteNVVMMMABlockScaleKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.mma_cta_count`: MMA CTA count +pub const MMACtaCountAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMACtaCount; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "mma_cta_count"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMACtaCount, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMACtaCountAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) MMACtaCount { + return @enumFromInt(c.mlirCuteNVVMMMACtaCountAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.mma_frag`: NVVM MMA frag type +pub const MMAFragAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMAFrag; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "mma_frag"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMAFrag, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMAFragAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) MMAFrag { + return @enumFromInt(c.mlirCuteNVVMMMAFragAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.mma_int_overflow`: MMA overflow options +pub const MMAIntOverflowAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMAIntOverflow; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "mma_int_overflow"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMAIntOverflow, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMAIntOverflowAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) MMAIntOverflow { + return @enumFromInt(c.mlirCuteNVVMMMAIntOverflowAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.mma_kind`: MMA operation kind +pub const MMAKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMAKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "mma_kind"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMAKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMAKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) MMAKind { + return @enumFromInt(c.mlirCuteNVVMMMAKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.mma_layout`: NVVM MMA layout +pub const MMALayoutAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMALayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "mma_layout"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMALayout, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMALayoutAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) MMALayout { + return @enumFromInt(c.mlirCuteNVVMMMALayoutAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.mma_type`: NVVM MMA types +pub const MMATypesAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMATypes; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "mma_type"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMATypes, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMATypesAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) MMATypes { + return @enumFromInt(c.mlirCuteNVVMMMATypesAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.match_sync_kind`: NVVM match sync kind +pub const MatchSyncKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMatchSyncKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "match_sync_kind"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MatchSyncKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMatchSyncKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) MatchSyncKind { + return @enumFromInt(c.mlirCuteNVVMMatchSyncKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.mem_order`: NVVM Memory Ordering kind +pub const MemOrderKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMemOrderKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "mem_order"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MemOrderKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMemOrderKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) MemOrderKind { + return @enumFromInt(c.mlirCuteNVVMMemOrderKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.mem_scope`: NVVM Memory Scope kind +pub const MemScopeKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMemScopeKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "mem_scope"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MemScopeKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMemScopeKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) MemScopeKind { + return @enumFromInt(c.mlirCuteNVVMMemScopeKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.mul_mode`: multiply mode attribute +pub const MulModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMulMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "mul_mode"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MulMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMulModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) MulMode { + return @enumFromInt(c.mlirCuteNVVMMulModeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.memory_space`: NVVM Memory Space +pub const NVVMMemorySpaceAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMNVVMMemorySpace; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "memory_space"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: NVVMMemorySpace, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMNVVMMemorySpaceAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) NVVMMemorySpace { + return @enumFromInt(c.mlirCuteNVVMNVVMMemorySpaceAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.permute_mode`: NVVM permute mode +pub const PermuteModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMPermuteMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "permute_mode"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: PermuteMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMPermuteModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) PermuteMode { + return @enumFromInt(c.mlirCuteNVVMPermuteModeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.prefetch_cache_level`: NVVM Prefetch Cache Level +pub const PrefetchCacheLevelAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMPrefetchCacheLevel; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "prefetch_cache_level"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: PrefetchCacheLevel, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMPrefetchCacheLevelAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) PrefetchCacheLevel { + return @enumFromInt(c.mlirCuteNVVMPrefetchCacheLevelAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.proxy_kind`: Proxy kind +pub const ProxyKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMProxyKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "proxy_kind"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ProxyKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMProxyKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) ProxyKind { + return @enumFromInt(c.mlirCuteNVVMProxyKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.reduction_kind`: NVVM Reduction Kind attribute +pub const ReductionKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMReductionKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "reduction_kind"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ReductionKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMReductionKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) ReductionKind { + return @enumFromInt(c.mlirCuteNVVMReductionKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.red_op`: Ops supported by red instruction +pub const ReductionOpAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMReductionOp; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "red_op"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ReductionOp, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMReductionOpAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) ReductionOp { + return @enumFromInt(c.mlirCuteNVVMReductionOpAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.red_type`: types supported by red instruction +pub const ReductionTypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMReductionType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "red_type"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ReductionType, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMReductionTypeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) ReductionType { + return @enumFromInt(c.mlirCuteNVVMReductionTypeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.spcomp_elem_size`: Sparse tensor compression element size +pub const SPCompressElemSizeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPCompressElemSize; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "spcomp_elem_size"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPCompressElemSize, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPCompressElemSizeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) SPCompressElemSize { + return @enumFromInt(c.mlirCuteNVVMSPCompressElemSizeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.spcomp_factor`: Sparse tensor compression factor type +pub const SPCompressFactorTypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPCompressFactorType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "spcomp_factor"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPCompressFactorType, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPCompressFactorTypeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) SPCompressFactorType { + return @enumFromInt(c.mlirCuteNVVMSPCompressFactorTypeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.spcomp_index_size`: Sparse tensor compression index size +pub const SPCompressIndexSizeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPCompressIndexSize; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "spcomp_index_size"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPCompressIndexSize, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPCompressIndexSizeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) SPCompressIndexSize { + return @enumFromInt(c.mlirCuteNVVMSPCompressIndexSizeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.spcomp_op_kind`: spcompress operation kind +pub const SPCompressOpKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPCompressOpKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "spcomp_op_kind"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPCompressOpKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPCompressOpKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) SPCompressOpKind { + return @enumFromInt(c.mlirCuteNVVMSPCompressOpKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.spcomp_rep_factor`: Sparse tensor compression repetition factor +pub const SPCompressRepFactorAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPCompressRepFactor; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "spcomp_rep_factor"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPCompressRepFactor, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPCompressRepFactorAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) SPCompressRepFactor { + return @enumFromInt(c.mlirCuteNVVMSPCompressRepFactorAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.spdecomp_elem_size`: Sparse tensor decompression element size +pub const SPDecompressElemSizeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPDecompressElemSize; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "spdecomp_elem_size"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPDecompressElemSize, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPDecompressElemSizeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) SPDecompressElemSize { + return @enumFromInt(c.mlirCuteNVVMSPDecompressElemSizeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.spdecomp_factor`: Sparse tensor decompression factor type +pub const SPDecompressFactorTypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPDecompressFactorType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "spdecomp_factor"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPDecompressFactorType, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPDecompressFactorTypeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) SPDecompressFactorType { + return @enumFromInt(c.mlirCuteNVVMSPDecompressFactorTypeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.spdecomp_index_size`: Sparse tensor decompression index size +pub const SPDecompressIndexSizeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPDecompressIndexSize; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "spdecomp_index_size"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPDecompressIndexSize, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPDecompressIndexSizeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) SPDecompressIndexSize { + return @enumFromInt(c.mlirCuteNVVMSPDecompressIndexSizeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.spdecomp_rep_factor`: Sparse tensor decompression repetition factor +pub const SPDecompressRepFactorAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPDecompressRepFactor; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "spdecomp_rep_factor"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPDecompressRepFactor, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPDecompressRepFactorAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) SPDecompressRepFactor { + return @enumFromInt(c.mlirCuteNVVMSPDecompressRepFactorAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.sat_mode`: NVVM SaturationMode kind +pub const SaturationModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSaturationMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "sat_mode"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SaturationMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSaturationModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) SaturationMode { + return @enumFromInt(c.mlirCuteNVVMSaturationModeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.sat`: NVVM SaturationMode kind +pub const SaturationModeKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSaturationModeKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "sat"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SaturationModeKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSaturationModeKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) SaturationModeKind { + return @enumFromInt(c.mlirCuteNVVMSaturationModeKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.scale_vec_size`: MMA Scale Vector Sizes +pub const ScaleVecSizeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMScaleVecSize; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "scale_vec_size"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ScaleVecSize, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMScaleVecSizeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) ScaleVecSize { + return @enumFromInt(c.mlirCuteNVVMScaleVecSizeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.action`: NVVM set max register action +pub const SetMaxRegisterActionAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSetMaxRegisterAction; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "action"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SetMaxRegisterAction, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSetMaxRegisterActionAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) SetMaxRegisterAction { + return @enumFromInt(c.mlirCuteNVVMSetMaxRegisterActionAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.shared_space`: Shared memory space +pub const SharedSpaceAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSharedSpace; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "shared_space"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SharedSpace, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSharedSpaceAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) SharedSpace { + return @enumFromInt(c.mlirCuteNVVMSharedSpaceAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.shfl_kind`: NVVM shuffle kind +pub const ShflKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMShflKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "shfl_kind"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ShflKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMShflKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) ShflKind { + return @enumFromInt(c.mlirCuteNVVMShflKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.sparsity_format`: MMA Sparsity Format +pub const SparsityFormatAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSparsityFormat; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "sparsity_format"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SparsityFormat, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSparsityFormatAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) SparsityFormat { + return @enumFromInt(c.mlirCuteNVVMSparsityFormatAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.state_space`: NVVM State Space +pub const StateSpaceAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMStateSpace; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "state_space"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: StateSpace, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMStateSpaceAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) StateSpace { + return @enumFromInt(c.mlirCuteNVVMStateSpaceAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.store_cache_modifier`: NVVM store cache modifier kind +pub const StoreCacheModifierKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMStoreCacheModifierKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "store_cache_modifier"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: StoreCacheModifierKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMStoreCacheModifierKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) StoreCacheModifierKind { + return @enumFromInt(c.mlirCuteNVVMStoreCacheModifierKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.store_shape`: shape attribute for stmatrix +pub const StoreShapeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMStoreShape; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "store_shape"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: StoreShape, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMStoreShapeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) StoreShape { + return @enumFromInt(c.mlirCuteNVVMStoreShapeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.TCBarParam`: Cluster MMA Barrier Parameter Type +pub const TCBarParamAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTCBarParam; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "TCBarParam"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TCBarParam, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTCBarParamAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) TCBarParam { + return @enumFromInt(c.mlirCuteNVVMTCBarParamAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.tma_load_mode`: NVVM TMA Load Mode +pub const TMALoadModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTMALoadMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tma_load_mode"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TMALoadMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTMALoadModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) TMALoadMode { + return @enumFromInt(c.mlirCuteNVVMTMALoadModeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.tma_redux_kind`: NVVM TMA redux kind +pub const TMAReduxKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTMAReduxKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tma_redux_kind"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TMAReduxKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTMAReduxKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) TMAReduxKind { + return @enumFromInt(c.mlirCuteNVVMTMAReduxKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.tma_store_mode`: NVVM TMA Store Mode +pub const TMAStoreModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTMAStoreMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tma_store_mode"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TMAStoreMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTMAStoreModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) TMAStoreMode { + return @enumFromInt(c.mlirCuteNVVMTMAStoreModeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.tcgen05_cp_multicast`: tcgen05 cp multicast +pub const Tcgen05CpMulticastAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05CpMulticast; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_cp_multicast"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05CpMulticast, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05CpMulticastAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) Tcgen05CpMulticast { + return @enumFromInt(c.mlirCuteNVVMTcgen05CpMulticastAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.tcgen05_cp_shape`: tcgen05 cp shapes +pub const Tcgen05CpShapeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05CpShape; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_cp_shape"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05CpShape, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05CpShapeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) Tcgen05CpShape { + return @enumFromInt(c.mlirCuteNVVMTcgen05CpShapeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.tcgen05_cp_src_fmt`: tcgen05 cp source format +pub const Tcgen05CpSrcFormatAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05CpSrcFormat; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_cp_src_fmt"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05CpSrcFormat, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05CpSrcFormatAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) Tcgen05CpSrcFormat { + return @enumFromInt(c.mlirCuteNVVMTcgen05CpSrcFormatAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.tcgen05_fence`: NVVM Tcgen05 fence kind +pub const Tcgen05FenceKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05FenceKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_fence"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05FenceKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05FenceKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) Tcgen05FenceKind { + return @enumFromInt(c.mlirCuteNVVMTcgen05FenceKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.tcgen05_ldst_shape`: allowed 32-bit signless integer cases: 0, 1, 2, 3, 4 +pub const Tcgen05LdStShapeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05LdStShape; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_ldst_shape"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05LdStShape, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05LdStShapeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) Tcgen05LdStShape { + return @enumFromInt(c.mlirCuteNVVMTcgen05LdStShapeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.tcgen05_mma_block_scale`: tcgen05.mma block scale attribute +pub const Tcgen05MMABlockScaleAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05MMABlockScale; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_mma_block_scale"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05MMABlockScale, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05MMABlockScaleAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) Tcgen05MMABlockScale { + return @enumFromInt(c.mlirCuteNVVMTcgen05MMABlockScaleAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.tcgen05_mma_collectorb`: tcgen05 MMA Collector Buffer B Attribute +pub const Tcgen05MMACollectorBBufferAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05MMACollectorBBuffer; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_mma_collectorb"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05MMACollectorBBuffer, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05MMACollectorBBufferAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) Tcgen05MMACollectorBBuffer { + return @enumFromInt(c.mlirCuteNVVMTcgen05MMACollectorBBufferAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.tcgen05_mma_collectorop`: tcgen05.mma Collector Buffer Operation +pub const Tcgen05MMACollectorOpAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05MMACollectorOp; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_mma_collectorop"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05MMACollectorOp, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05MMACollectorOpAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) Tcgen05MMACollectorOp { + return @enumFromInt(c.mlirCuteNVVMTcgen05MMACollectorOpAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.tcgen05_mma_kind`: tcgen05 MMA Supported Types +pub const Tcgen05MMAKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05MMAKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_mma_kind"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05MMAKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05MMAKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) Tcgen05MMAKind { + return @enumFromInt(c.mlirCuteNVVMTcgen05MMAKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.tcgen05_wait`: NVVM Tcgen05 wait kind +pub const Tcgen05WaitKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05WaitKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_wait"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05WaitKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05WaitKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) Tcgen05WaitKind { + return @enumFromInt(c.mlirCuteNVVMTcgen05WaitKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.tensormap_elemtype`: NVVM Tensormap Elemtype +pub const TensormapElemtypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapElemtype; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tensormap_elemtype"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TensormapElemtype, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTensormapElemtypeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) TensormapElemtype { + return @enumFromInt(c.mlirCuteNVVMTensormapElemtypeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.tensormap_field`: NVVM Tensormap Field Kind +pub const TensormapFieldAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapField; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tensormap_field"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TensormapField, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTensormapFieldAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) TensormapField { + return @enumFromInt(c.mlirCuteNVVMTensormapFieldAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.tensormap_fill_mode`: NVVM Tensormap Fill Mode +pub const TensormapFillModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapFillMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tensormap_fill_mode"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TensormapFillMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTensormapFillModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) TensormapFillMode { + return @enumFromInt(c.mlirCuteNVVMTensormapFillModeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.tensormap_interleave_layout`: NVVM Tensormap Interleave Layout +pub const TensormapInterleaveLayoutAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapInterleaveLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tensormap_interleave_layout"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TensormapInterleaveLayout, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTensormapInterleaveLayoutAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) TensormapInterleaveLayout { + return @enumFromInt(c.mlirCuteNVVMTensormapInterleaveLayoutAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.tensormap_swizzle_atomicity`: NVVM Tensormap Swizzle Atomicity +pub const TensormapSwizzleAtomicityAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapSwizzleAtomicity; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tensormap_swizzle_atomicity"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TensormapSwizzleAtomicity, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTensormapSwizzleAtomicityAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) TensormapSwizzleAtomicity { + return @enumFromInt(c.mlirCuteNVVMTensormapSwizzleAtomicityAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.tensormap_swizzle_mode`: NVVM Tensormap Swizzle Mode +pub const TensormapSwizzleModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapSwizzleMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "tensormap_swizzle_mode"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TensormapSwizzleMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTensormapSwizzleModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) TensormapSwizzleMode { + return @enumFromInt(c.mlirCuteNVVMTensormapSwizzleModeAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.TmemLayout`: Tensor Memory Layout Enumerated Type +pub const TmemLayoutAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTmemLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "TmemLayout"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TmemLayout, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTmemLayoutAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) TmemLayout { + return @enumFromInt(c.mlirCuteNVVMTmemLayoutAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.validate_pattern`: NVVM validate data pattern +pub const ValidatePatternAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMValidatePattern; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "validate_pattern"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ValidatePattern, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMValidatePatternAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) ValidatePattern { + return @enumFromInt(c.mlirCuteNVVMValidatePatternAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.vote_sync_kind`: NVVM vote sync kind +pub const VoteSyncKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMVoteSyncKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "vote_sync_kind"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: VoteSyncKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMVoteSyncKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) VoteSyncKind { + return @enumFromInt(c.mlirCuteNVVMVoteSyncKindAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.wgmma_scale_in`: WGMMA overflow options +pub const WGMMAScaleInAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMWGMMAScaleIn; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "wgmma_scale_in"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: WGMMAScaleIn, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMWGMMAScaleInAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) WGMMAScaleIn { + return @enumFromInt(c.mlirCuteNVVMWGMMAScaleInAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.wgmma_scale_out`: WGMMA input predicate +pub const WGMMAScaleOutAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMWGMMAScaleOut; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "wgmma_scale_out"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: WGMMAScaleOut, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMWGMMAScaleOutAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) WGMMAScaleOut { + return @enumFromInt(c.mlirCuteNVVMWGMMAScaleOutAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.wgmma_type`: NVVM WGMMA types +pub const WGMMATypesAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMWGMMATypes; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "wgmma_type"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: WGMMATypes, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMWGMMATypesAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) WGMMATypes { + return @enumFromInt(c.mlirCuteNVVMWGMMATypesAttrGetValue(self.ptr())); + } +}; + +/// `#nvvm.ld_st_matrix_shape`: Matrix shape of ldmatrix, stmatrix and movmatrix +pub const LdStMatrixShapeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMLdStMatrixShape; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "ld_st_matrix_shape"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + m: c_int, + n: c_int, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMLdStMatrixShapeAttrGet(ctx.ptr(), args.m, args.n); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getM(self: *const Self) c_int { + return c.mlirCuteNVVMLdStMatrixShapeAttrGetM(self.ptr()); + } + pub fn getN(self: *const Self) c_int { + return c.mlirCuteNVVMLdStMatrixShapeAttrGetN(self.ptr()); + } +}; + +/// `#nvvm.shape`: Shape of an MMA operation +pub const MMAShapeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMAShape; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "shape"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + m: c_int, + n: c_int, + k: c_int, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMAShapeAttrGet(ctx.ptr(), args.m, args.n, args.k); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getM(self: *const Self) c_int { + return c.mlirCuteNVVMMMAShapeAttrGetM(self.ptr()); + } + pub fn getN(self: *const Self) c_int { + return c.mlirCuteNVVMMMAShapeAttrGetN(self.ptr()); + } + pub fn getK(self: *const Self) c_int { + return c.mlirCuteNVVMMMAShapeAttrGetK(self.ptr()); + } +}; + +/// `#nvvm.target`: GPU target of an NVVM module +pub const TargetAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTarget; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "target"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + O: c_int = 2, + triple: []const u8 = "nvptx64-nvidia-cuda", + chip: []const u8 = "sm_75", + features: []const u8 = "", + flags: ?*const mlir.Attribute = null, + link: ?*const mlir.Attribute = null, + verifyTarget: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTargetAttrGet(ctx.ptr(), args.O, mlir.stringRef(args.triple), mlir.stringRef(args.chip), mlir.stringRef(args.features), if (args.flags) |v| v.ptr() else .{ .ptr = null }, if (args.link) |v| v.ptr() else .{ .ptr = null }, args.verifyTarget); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getO(self: *const Self) c_int { + return c.mlirCuteNVVMTargetAttrGetO(self.ptr()); + } + pub fn getTriple(self: *const Self) []const u8 { + const s = c.mlirCuteNVVMTargetAttrGetTriple(self.ptr()); + return if (s.length == 0) "" else mlir.string(s); + } + pub fn getChip(self: *const Self) []const u8 { + const s = c.mlirCuteNVVMTargetAttrGetChip(self.ptr()); + return if (s.length == 0) "" else mlir.string(s); + } + pub fn getFeatures(self: *const Self) []const u8 { + const s = c.mlirCuteNVVMTargetAttrGetFeatures(self.ptr()); + return if (s.length == 0) "" else mlir.string(s); + } + pub fn getFlags(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVVMTargetAttrGetFlags(self.ptr()).ptr); + } + pub fn getLink(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVVMTargetAttrGetLink(self.ptr()).ptr); + } + pub fn getVerifyTarget(self: *const Self) bool { + return c.mlirCuteNVVMTargetAttrGetVerifyTarget(self.ptr()); + } +}; + +// Operation builders, generated from the operations' .td. +/// `nvvm.add.packed.bf16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn add_packed_bf16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.add.packed.bf16x2.f32x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.add.packed.f16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn add_packed_f16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.add.packed.f16x2.f32x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.add.packed.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn add_packed_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, ftz: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.add.packed.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.add.packed.f32x2.bf16x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn add_packed_f32x2_bf16x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + return mlir.Operation.make(ctx, "nvvm.add.packed.f32x2.bf16x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.add.packed.f32x2.f16x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn add_packed_f32x2_f16x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + return mlir.Operation.make(ctx, "nvvm.add.packed.f32x2.f16x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.addf`. Result types are explicit; attributes use mlir.Attribute. +pub fn addf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, sat: ?*const mlir.Attribute, ftz: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.addf", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.applypriority.async.bulk`. Result types are explicit; attributes use mlir.Attribute. +pub fn applypriority_async_bulk(ctx: *mlir.Context, ptr: *const mlir.Value, size: *const mlir.Value, evict: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + return mlir.Operation.make(ctx, "nvvm.applypriority.async.bulk", .{ + .operands = .{ .flat = &.{ ptr, size } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.applypriority.async.bulk.tensor`. Result types are explicit; attributes use mlir.Attribute. +pub fn applypriority_async_bulk_tensor(ctx: *mlir.Context, tmaDesc: *const mlir.Value, coordinates: []const *const mlir.Value, im2colArgs: []const *const mlir.Value, mode: ?*const mlir.Attribute, evict: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + if (evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + return mlir.Operation.make(ctx, "nvvm.applypriority.async.bulk.tensor", .{ + .operands = .{ .variadic = &.{ + &.{tmaDesc}, + coordinates, + im2colArgs, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.applypriority.async.bulk.tensor.override`. Result types are explicit; attributes use mlir.Attribute. +pub fn applypriority_async_bulk_tensor_override(ctx: *mlir.Context, tmaDesc: *const mlir.Value, overrideAdrr: *const mlir.Value, coordinates: []const *const mlir.Value, tensorSize: []const *const mlir.Value, lowerStride: []const *const mlir.Value, upperStride: ?*const mlir.Value, im2colArgs: []const *const mlir.Value, mode: ?*const mlir.Attribute, evict: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + if (evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + return mlir.Operation.make(ctx, "nvvm.applypriority.async.bulk.tensor.override", .{ + .operands = .{ .variadic = &.{ + &.{tmaDesc}, + &.{overrideAdrr}, + coordinates, + tensorSize, + lowerStride, + if (upperStride) |value| &.{value} else &.{}, + im2colArgs, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.atomicrmw`. Result types are explicit; attributes use mlir.Attribute. +pub fn atomicrmw(ctx: *mlir.Context, ptr: *const mlir.Value, a: *const mlir.Value, b: ?*const mlir.Value, res_type: *const mlir.Type, op: *const mlir.Attribute, memOrder: ?*const mlir.Attribute, syncscope: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "op", op)); + if (memOrder) |value| attributes.appendAssumeCapacity(.named(ctx, "memOrder", value)); + if (syncscope) |value| attributes.appendAssumeCapacity(.named(ctx, "syncscope", value)); + return mlir.Operation.make(ctx, "nvvm.atomicrmw", .{ + .operands = .{ .flat = if (b) |value| &.{ ptr, a, value } else &.{ ptr, a } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.bar.warp.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn bar_warp_sync(ctx: *mlir.Context, mask: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.bar.warp.sync", .{ + .operands = .{ .flat = &.{mask} }, + .location = location, + }); +} + +/// `nvvm.barrier`. Result types are explicit; attributes use mlir.Attribute. +pub fn barrier(ctx: *mlir.Context, barrierId: ?*const mlir.Value, numberOfThreads: ?*const mlir.Value, reductionPredicate: ?*const mlir.Value, res_type: ?*const mlir.Type, reductionOp: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; + if (res_type) |value| results.appendAssumeCapacity(value); + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (reductionOp) |value| attributes.appendAssumeCapacity(.named(ctx, "reductionOp", value)); + return mlir.Operation.make(ctx, "nvvm.barrier", .{ + .operands = .{ .variadic = &.{ + if (barrierId) |value| &.{value} else &.{}, + if (numberOfThreads) |value| &.{value} else &.{}, + if (reductionPredicate) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = results.constSlice() }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.barrier.arrive`. Result types are explicit; attributes use mlir.Attribute. +pub fn barrier_arrive(ctx: *mlir.Context, barrierId: ?*const mlir.Value, numberOfThreads: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.barrier.arrive", .{ + .operands = .{ .flat = if (barrierId) |value| &.{ value, numberOfThreads } else &.{numberOfThreads} }, + .location = location, + }); +} + +/// `nvvm.barrier.cta.arrive`. Result types are explicit; attributes use mlir.Attribute. +pub fn barrier_cta_arrive(ctx: *mlir.Context, barrierId: *const mlir.Value, threadCount: *const mlir.Value, aligned: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); + return mlir.Operation.make(ctx, "nvvm.barrier.cta.arrive", .{ + .operands = .{ .flat = &.{ barrierId, threadCount } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.barrier.cta.red`. Result types are explicit; attributes use mlir.Attribute. +pub fn barrier_cta_red(ctx: *mlir.Context, pred: *const mlir.Value, barrierId: *const mlir.Value, threadCount: ?*const mlir.Value, res_type: *const mlir.Type, kind: *const mlir.Attribute, aligned: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); + return mlir.Operation.make(ctx, "nvvm.barrier.cta.red", .{ + .operands = .{ .flat = if (threadCount) |value| &.{ pred, barrierId, value } else &.{ pred, barrierId } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.barrier.cta.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn barrier_cta_sync(ctx: *mlir.Context, barrierId: *const mlir.Value, threadCount: ?*const mlir.Value, aligned: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); + return mlir.Operation.make(ctx, "nvvm.barrier.cta.sync", .{ + .operands = .{ .flat = if (threadCount) |value| &.{ barrierId, value } else &.{barrierId} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.barrier0`. Result types are explicit; attributes use mlir.Attribute. +pub fn barrier0(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.barrier0", .{ + .location = location, + }); +} + +/// `nvvm.breakpoint`. Result types are explicit; attributes use mlir.Attribute. +pub fn breakpoint(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.breakpoint", .{ + .location = location, + }); +} + +/// `nvvm.clmad`. Result types are explicit; attributes use mlir.Attribute. +pub fn clmad(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, c_: *const mlir.Value, res_type: *const mlir.Type, high: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (high) |value| attributes.appendAssumeCapacity(.named(ctx, "high", value)); + return mlir.Operation.make(ctx, "nvvm.clmad", .{ + .operands = .{ .flat = &.{ a, b, c_ } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cluster.arrive`. Result types are explicit; attributes use mlir.Attribute. +pub fn cluster_arrive(ctx: *mlir.Context, aligned: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); + return mlir.Operation.make(ctx, "nvvm.cluster.arrive", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cluster.arrive.relaxed`. Result types are explicit; attributes use mlir.Attribute. +pub fn cluster_arrive_relaxed(ctx: *mlir.Context, aligned: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); + return mlir.Operation.make(ctx, "nvvm.cluster.arrive.relaxed", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cluster.wait`. Result types are explicit; attributes use mlir.Attribute. +pub fn cluster_wait(ctx: *mlir.Context, aligned: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); + return mlir.Operation.make(ctx, "nvvm.cluster.wait", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.clusterlaunchcontrol.query.cancel`. Result types are explicit; attributes use mlir.Attribute. +pub fn clusterlaunchcontrol_query_cancel(ctx: *mlir.Context, try_cancel_response: *const mlir.Value, res_type: *const mlir.Type, query_type: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.clusterlaunchcontrol.query.cancel", .{ + .operands = .{ .flat = &.{try_cancel_response} }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "query_type", query_type), + }, + .location = location, + }); +} + +/// `nvvm.clusterlaunchcontrol.try.cancel`. Result types are explicit; attributes use mlir.Attribute. +pub fn clusterlaunchcontrol_try_cancel(ctx: *mlir.Context, smemAddress: *const mlir.Value, mbarrier: *const mlir.Value, multicast: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "multicast", value)); + return mlir.Operation.make(ctx, "nvvm.clusterlaunchcontrol.try.cancel", .{ + .operands = .{ .flat = &.{ smemAddress, mbarrier } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.compare.and.set`. Result types are explicit; attributes use mlir.Attribute. +pub fn compare_and_set(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, result_type: *const mlir.Type, op: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.compare.and.set", .{ + .operands = .{ .flat = &.{ a, b } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "op", op), + }, + .location = location, + }); +} + +/// `nvvm.convert.and.pack.integer`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_and_pack_integer(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, srcC: ?*const mlir.Value, dst_type: *const mlir.Type, is_signed: ?*const mlir.Attribute, convert_type: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (is_signed) |value| attributes.appendAssumeCapacity(.named(ctx, "is_signed", value)); + attributes.appendAssumeCapacity(.named(ctx, "convert_type", convert_type)); + return mlir.Operation.make(ctx, "nvvm.convert.and.pack.integer", .{ + .operands = .{ .flat = if (srcC) |value| &.{ srcA, srcB, value } else &.{ srcA, srcB } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.bf16x2.to.f4x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_bf16x2_to_f4x2(ctx: *mlir.Context, src: *const mlir.Value, scale_factor: ?*const mlir.Value, dst_type: *const mlir.Type, isPZO: ?*const mlir.Attribute, scale_factor_kind: ?*const mlir.Attribute, rnd: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, dstTy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.bf16x2.to.f4x2", .{ + .operands = .{ .flat = if (scale_factor) |value| &.{ src, value } else &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.bf16x2.to.f6x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_bf16x2_to_f6x2(ctx: *mlir.Context, src: *const mlir.Value, scale_factor: ?*const mlir.Value, dst_type: *const mlir.Type, isPZO: ?*const mlir.Attribute, scale_factor_kind: ?*const mlir.Attribute, rnd: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, dstTy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.bf16x2.to.f6x2", .{ + .operands = .{ .flat = if (scale_factor) |value| &.{ src, value } else &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.bf16x2.to.f8x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_bf16x2_to_f8x2(ctx: *mlir.Context, src: *const mlir.Value, scale_factor: ?*const mlir.Value, dst_type: *const mlir.Type, isPZO: ?*const mlir.Attribute, scale_factor_kind: ?*const mlir.Attribute, rnd: ?*const mlir.Attribute, sat: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, dstTy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + if (isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.bf16x2.to.f8x2", .{ + .operands = .{ .flat = if (scale_factor) |value| &.{ src, value } else &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.bf16x2.to.s2f6x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_bf16x2_to_s2f6x2(ctx: *mlir.Context, src: *const mlir.Value, scaleFactor: ?*const mlir.Value, dst_type: *const mlir.Type, relu: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + return mlir.Operation.make(ctx, "nvvm.convert.bf16x2.to.s2f6x2", .{ + .operands = .{ .flat = if (scaleFactor) |value| &.{ src, value } else &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f16x2.to.f4x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f16x2_to_f4x2(ctx: *mlir.Context, src: *const mlir.Value, scale_factor: ?*const mlir.Value, dst_type: *const mlir.Type, isPZO: ?*const mlir.Attribute, scale_factor_kind: ?*const mlir.Attribute, rnd: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, dstTy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.f16x2.to.f4x2", .{ + .operands = .{ .flat = if (scale_factor) |value| &.{ src, value } else &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f16x2.to.f6x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f16x2_to_f6x2(ctx: *mlir.Context, src: *const mlir.Value, scale_factor: ?*const mlir.Value, dst_type: *const mlir.Type, isPZO: ?*const mlir.Attribute, scale_factor_kind: ?*const mlir.Attribute, rnd: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, dstTy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.f16x2.to.f6x2", .{ + .operands = .{ .flat = if (scale_factor) |value| &.{ src, value } else &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f16x2.to.f8x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f16x2_to_f8x2(ctx: *mlir.Context, a: *const mlir.Value, scale_factor: ?*const mlir.Value, dst_type: *const mlir.Type, isPZO: ?*const mlir.Attribute, scale_factor_kind: ?*const mlir.Attribute, rnd: ?*const mlir.Attribute, sat: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, dstTy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + if (isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.f16x2.to.f8x2", .{ + .operands = .{ .flat = if (scale_factor) |value| &.{ a, value } else &.{a} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x2.to.bf16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x2_to_bf16x2(ctx: *mlir.Context, src_hi: *const mlir.Value, src_lo: *const mlir.Value, random_bits: ?*const mlir.Value, dst_type: *const mlir.Type, rnd: ?*const mlir.Attribute, sat: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.bf16x2", .{ + .operands = .{ .flat = if (random_bits) |value| &.{ src_hi, src_lo, value } else &.{ src_hi, src_lo } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x2.to.f16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x2_to_f16x2(ctx: *mlir.Context, src_hi: *const mlir.Value, src_lo: *const mlir.Value, random_bits: ?*const mlir.Value, dst_type: *const mlir.Type, rnd: ?*const mlir.Attribute, sat: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.f16x2", .{ + .operands = .{ .flat = if (random_bits) |value| &.{ src_hi, src_lo, value } else &.{ src_hi, src_lo } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x2.to.f4x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x2_to_f4x2(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, scale_factor: ?*const mlir.Value, dst_type: *const mlir.Type, isPZO: ?*const mlir.Attribute, scale_factor_kind: ?*const mlir.Attribute, rnd: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, dstType: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstType", dstType)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.f4x2", .{ + .operands = .{ .flat = if (scale_factor) |value| &.{ a, b, value } else &.{ a, b } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x2.to.f6x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x2_to_f6x2(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, scale_factor: ?*const mlir.Value, dst_type: *const mlir.Type, isPZO: ?*const mlir.Attribute, scale_factor_kind: ?*const mlir.Attribute, rnd: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, dstTy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.f6x2", .{ + .operands = .{ .flat = if (scale_factor) |value| &.{ a, b, value } else &.{ a, b } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x2.to.f8x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x2_to_f8x2(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, scale_factor: ?*const mlir.Value, dst_type: *const mlir.Type, isPZO: ?*const mlir.Attribute, scale_factor_kind: ?*const mlir.Attribute, rnd: ?*const mlir.Attribute, sat: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, dstTy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + if (isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.f8x2", .{ + .operands = .{ .flat = if (scale_factor) |value| &.{ a, b, value } else &.{ a, b } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x2.to.s2f6x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x2_to_s2f6x2(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, scaleFactor: ?*const mlir.Value, dst_type: *const mlir.Type, relu: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.s2f6x2", .{ + .operands = .{ .flat = if (scaleFactor) |value| &.{ a, b, value } else &.{ a, b } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x4.to.f4x4`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x4_to_f4x4(ctx: *mlir.Context, src: *const mlir.Value, rbits: *const mlir.Value, dst_type: *const mlir.Type, relu: ?*const mlir.Attribute, dstTy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x4.to.f4x4", .{ + .operands = .{ .flat = &.{ src, rbits } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x4.to.f6x4`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x4_to_f6x4(ctx: *mlir.Context, src: *const mlir.Value, rbits: *const mlir.Value, dst_type: *const mlir.Type, relu: ?*const mlir.Attribute, dstTy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x4.to.f6x4", .{ + .operands = .{ .flat = &.{ src, rbits } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x4.to.f8x4`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x4_to_f8x4(ctx: *mlir.Context, src: *const mlir.Value, rbits: *const mlir.Value, dst_type: *const mlir.Type, relu: ?*const mlir.Attribute, dstTy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x4.to.f8x4", .{ + .operands = .{ .flat = &.{ src, rbits } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f4x2.to.f16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f4x2_to_f16x2(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, relu: ?*const mlir.Attribute, srcType: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "srcType", srcType)); + return mlir.Operation.make(ctx, "nvvm.convert.f4x2.to.f16x2", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f6x2.to.f16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f6x2_to_f16x2(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, relu: ?*const mlir.Attribute, srcType: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "srcType", srcType)); + return mlir.Operation.make(ctx, "nvvm.convert.f6x2.to.f16x2", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f8x2.to.bf16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f8x2_to_bf16x2(ctx: *mlir.Context, src: *const mlir.Value, scaleFactor: ?*const mlir.Value, dst_type: *const mlir.Type, sat: ?*const mlir.Attribute, srcType: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + attributes.appendAssumeCapacity(.named(ctx, "srcType", srcType)); + return mlir.Operation.make(ctx, "nvvm.convert.f8x2.to.bf16x2", .{ + .operands = .{ .flat = if (scaleFactor) |value| &.{ src, value } else &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f8x2.to.f16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f8x2_to_f16x2(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, relu: ?*const mlir.Attribute, srcType: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "srcType", srcType)); + return mlir.Operation.make(ctx, "nvvm.convert.f8x2.to.f16x2", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.float.to.integer`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_float_to_integer(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, rnd: ?*const mlir.Attribute, sat: ?*const mlir.Attribute, ftz: ?*const mlir.Attribute, is_signed: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (is_signed) |value| attributes.appendAssumeCapacity(.named(ctx, "is_signed", value)); + return mlir.Operation.make(ctx, "nvvm.convert.float.to.integer", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.float.to.tf32`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_float_to_tf32(ctx: *mlir.Context, src: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, sat: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + return mlir.Operation.make(ctx, "nvvm.convert.float.to.tf32", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.s2f6x2.to.bf16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_s2f6x2_to_bf16x2(ctx: *mlir.Context, src: *const mlir.Value, scaleFactor: ?*const mlir.Value, dst_type: *const mlir.Type, sat: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + return mlir.Operation.make(ctx, "nvvm.convert.s2f6x2.to.bf16x2", .{ + .operands = .{ .flat = if (scaleFactor) |value| &.{ src, value } else &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cos`. Result types are explicit; attributes use mlir.Attribute. +pub fn cos(ctx: *mlir.Context, src: *const mlir.Value, res_type: *const mlir.Type, ftz: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.cos", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.commit.group`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_commit_group(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.commit.group", .{ + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.global.shared.cta`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_global_shared_cta(ctx: *mlir.Context, dstMem: *const mlir.Value, srcMem: *const mlir.Value, size: *const mlir.Value, l2CacheHint: ?*const mlir.Value, byteMask: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.global.shared.cta", .{ + .operands = .{ .variadic = &.{ + &.{dstMem}, + &.{srcMem}, + &.{size}, + if (l2CacheHint) |value| &.{value} else &.{}, + if (byteMask) |value| &.{value} else &.{}, + } }, + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.prefetch`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_prefetch(ctx: *mlir.Context, srcMem: *const mlir.Value, size: *const mlir.Value, l2CacheHint: ?*const mlir.Value, evict: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.prefetch", .{ + .operands = .{ .flat = if (l2CacheHint) |value| &.{ srcMem, size, value } else &.{ srcMem, size } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.shared.cluster.global`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_shared_cluster_global(ctx: *mlir.Context, dstMem: *const mlir.Value, srcMem: *const mlir.Value, mbar: *const mlir.Value, size: *const mlir.Value, multicastMask: ?*const mlir.Value, l2CacheHint: ?*const mlir.Value, validatePattern: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (validatePattern) |value| attributes.appendAssumeCapacity(.named(ctx, "validatePattern", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.shared.cluster.global", .{ + .operands = .{ .variadic = &.{ + &.{dstMem}, + &.{srcMem}, + &.{mbar}, + &.{size}, + if (multicastMask) |value| &.{value} else &.{}, + if (l2CacheHint) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.shared.cluster.shared.cta`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_shared_cluster_shared_cta(ctx: *mlir.Context, dstMem: *const mlir.Value, srcMem: *const mlir.Value, mbar: *const mlir.Value, size: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.shared.cluster.shared.cta", .{ + .operands = .{ .flat = &.{ dstMem, srcMem, mbar, size } }, + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.tensor.global.shared.cta`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_tensor_global_shared_cta(ctx: *mlir.Context, tmaDescriptor: *const mlir.Value, srcMem: *const mlir.Value, coordinates: []const *const mlir.Value, l2CacheHint: ?*const mlir.Value, predicate: ?*const mlir.Value, mode: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.global.shared.cta", .{ + .operands = .{ .variadic = &.{ + &.{tmaDescriptor}, + &.{srcMem}, + coordinates, + if (l2CacheHint) |value| &.{value} else &.{}, + if (predicate) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.tensor.global.shared.cta.override`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_tensor_global_shared_cta_override(ctx: *mlir.Context, tmaDesc: *const mlir.Value, srcMem: *const mlir.Value, overrideAdrr: *const mlir.Value, coordinates: []const *const mlir.Value, tensorSize: []const *const mlir.Value, lowerStride: []const *const mlir.Value, upperStride: ?*const mlir.Value, l2CacheHint: ?*const mlir.Value, mode: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.global.shared.cta.override", .{ + .operands = .{ .variadic = &.{ + &.{tmaDesc}, + &.{srcMem}, + &.{overrideAdrr}, + coordinates, + tensorSize, + lowerStride, + if (upperStride) |value| &.{value} else &.{}, + if (l2CacheHint) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.tensor.prefetch`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_tensor_prefetch(ctx: *mlir.Context, tmaDescriptor: *const mlir.Value, coordinates: []const *const mlir.Value, im2colOffsets: []const *const mlir.Value, l2CacheHint: ?*const mlir.Value, mode: ?*const mlir.Attribute, evict: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + if (evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.prefetch", .{ + .operands = .{ .variadic = &.{ + &.{tmaDescriptor}, + coordinates, + im2colOffsets, + if (l2CacheHint) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.tensor.prefetch.override`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_tensor_prefetch_override(ctx: *mlir.Context, tmaDesc: *const mlir.Value, overrideAdrr: *const mlir.Value, coordinates: []const *const mlir.Value, tensorSize: []const *const mlir.Value, lowerStride: []const *const mlir.Value, upperStride: ?*const mlir.Value, im2colOffsets: []const *const mlir.Value, l2CacheHint: ?*const mlir.Value, evict: ?*const mlir.Attribute, mode: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.prefetch.override", .{ + .operands = .{ .variadic = &.{ + &.{tmaDesc}, + &.{overrideAdrr}, + coordinates, + tensorSize, + lowerStride, + if (upperStride) |value| &.{value} else &.{}, + im2colOffsets, + if (l2CacheHint) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.tensor.reduce`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_tensor_reduce(ctx: *mlir.Context, tmaDescriptor: *const mlir.Value, srcMem: *const mlir.Value, coordinates: []const *const mlir.Value, l2CacheHint: ?*const mlir.Value, redKind: *const mlir.Attribute, mode: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "redKind", redKind)); + if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.reduce", .{ + .operands = .{ .variadic = &.{ + &.{tmaDescriptor}, + &.{srcMem}, + coordinates, + if (l2CacheHint) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.tensor.reduce.override`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_tensor_reduce_override(ctx: *mlir.Context, tmaDesc: *const mlir.Value, srcMem: *const mlir.Value, overrideAdrr: *const mlir.Value, coordinates: []const *const mlir.Value, tensorSize: []const *const mlir.Value, lowerStride: []const *const mlir.Value, upperStride: ?*const mlir.Value, l2CacheHint: ?*const mlir.Value, redKind: *const mlir.Attribute, mode: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "redKind", redKind)); + if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.reduce.override", .{ + .operands = .{ .variadic = &.{ + &.{tmaDesc}, + &.{srcMem}, + &.{overrideAdrr}, + coordinates, + tensorSize, + lowerStride, + if (upperStride) |value| &.{value} else &.{}, + if (l2CacheHint) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.tensor.shared.cluster.global`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_tensor_shared_cluster_global(ctx: *mlir.Context, dstMem: *const mlir.Value, tmaDescriptor: *const mlir.Value, coordinates: []const *const mlir.Value, mbar: *const mlir.Value, im2colOffsets: []const *const mlir.Value, multicastMask: ?*const mlir.Value, l2CacheHint: ?*const mlir.Value, predicate: ?*const mlir.Value, validatePattern: ?*const mlir.Attribute, mode: ?*const mlir.Attribute, isCTAOnly: ?*const mlir.Attribute, group: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (validatePattern) |value| attributes.appendAssumeCapacity(.named(ctx, "validatePattern", value)); + if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + if (isCTAOnly) |value| attributes.appendAssumeCapacity(.named(ctx, "isCTAOnly", value)); + if (group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.shared.cluster.global", .{ + .operands = .{ .variadic = &.{ + &.{dstMem}, + &.{tmaDescriptor}, + coordinates, + &.{mbar}, + im2colOffsets, + if (multicastMask) |value| &.{value} else &.{}, + if (l2CacheHint) |value| &.{value} else &.{}, + if (predicate) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.tensor.shared.cluster.global.override`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_tensor_shared_cluster_global_override(ctx: *mlir.Context, dstMem: *const mlir.Value, tmaDesc: *const mlir.Value, mbar: *const mlir.Value, overrideAddr: *const mlir.Value, coordinates: []const *const mlir.Value, im2colArgs: []const *const mlir.Value, tensorSize: []const *const mlir.Value, lowerStride: []const *const mlir.Value, upperStride: ?*const mlir.Value, multicastMask: ?*const mlir.Value, l2CacheHint: ?*const mlir.Value, group: ?*const mlir.Attribute, validatePattern: ?*const mlir.Attribute, mode: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + if (validatePattern) |value| attributes.appendAssumeCapacity(.named(ctx, "validatePattern", value)); + if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.shared.cluster.global.override", .{ + .operands = .{ .variadic = &.{ + &.{dstMem}, + &.{tmaDesc}, + &.{mbar}, + &.{overrideAddr}, + coordinates, + im2colArgs, + tensorSize, + lowerStride, + if (upperStride) |value| &.{value} else &.{}, + if (multicastMask) |value| &.{value} else &.{}, + if (l2CacheHint) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.wait_group`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_wait_group(ctx: *mlir.Context, group: *const mlir.Attribute, read: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "group", group)); + if (read) |value| attributes.appendAssumeCapacity(.named(ctx, "read", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.wait_group", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.commit.group`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_commit_group(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.cp.async.commit.group", .{ + .location = location, + }); +} + +/// `nvvm.cp.async.mbarrier.arrive`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_mbarrier_arrive(ctx: *mlir.Context, addr: *const mlir.Value, noinc: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (noinc) |value| attributes.appendAssumeCapacity(.named(ctx, "noinc", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.mbarrier.arrive", .{ + .operands = .{ .flat = &.{addr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.shared.global`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_shared_global(ctx: *mlir.Context, dst: *const mlir.Value, src: *const mlir.Value, cpSize: ?*const mlir.Value, size: *const mlir.Attribute, modifier: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.cp.async.shared.global", .{ + .operands = .{ .flat = if (cpSize) |value| &.{ dst, src, value } else &.{ dst, src } }, + .attributes = &.{ + .named(ctx, "size", size), + .named(ctx, "modifier", modifier), + }, + .location = location, + }); +} + +/// `nvvm.cp.async.wait.group`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_wait_group(ctx: *mlir.Context, n: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.cp.async.wait.group", .{ + .attributes = &.{ + .named(ctx, "n", n), + }, + .location = location, + }); +} + +/// `nvvm.cvt.packfloat`. Result types are explicit; attributes use mlir.Attribute. +pub fn cvt_packfloat(ctx: *mlir.Context, srcA: *const mlir.Value, srcC: *const mlir.Value, res_type: *const mlir.Type, from: *const mlir.Attribute, to: *const mlir.Attribute, rnd: ?*const mlir.Attribute, sat: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, extractHi: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "from", from)); + attributes.appendAssumeCapacity(.named(ctx, "to", to)); + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (extractHi) |value| attributes.appendAssumeCapacity(.named(ctx, "extractHi", value)); + return mlir.Operation.make(ctx, "nvvm.cvt.packfloat", .{ + .operands = .{ .flat = &.{ srcA, srcC } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cvt.packfloat.f32`. Result types are explicit; attributes use mlir.Attribute. +pub fn cvt_packfloat_f32(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, srcC: *const mlir.Value, res_type: *const mlir.Type, to: *const mlir.Attribute, rnd: ?*const mlir.Attribute, sat: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, extractHi: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "to", to)); + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (extractHi) |value| attributes.appendAssumeCapacity(.named(ctx, "extractHi", value)); + return mlir.Operation.make(ctx, "nvvm.cvt.packfloat.f32", .{ + .operands = .{ .flat = &.{ srcA, srcB, srcC } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cvt.to.f4x8.packed`. Result types are explicit; attributes use mlir.Attribute. +pub fn cvt_to_f4x8_packed(ctx: *mlir.Context, srcs: []const *const mlir.Value, dst_type: *const mlir.Type, rnd: ?*const mlir.Attribute, srcType: *const mlir.Attribute, dstType: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + attributes.appendAssumeCapacity(.named(ctx, "srcType", srcType)); + attributes.appendAssumeCapacity(.named(ctx, "dstType", dstType)); + return mlir.Operation.make(ctx, "nvvm.cvt.to.f4x8.packed", .{ + .operands = .{ .flat = srcs }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.dot.accumulate.2way`. Result types are explicit; attributes use mlir.Attribute. +pub fn dot_accumulate_2way(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, c_: *const mlir.Value, res_type: *const mlir.Type, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, b_hi: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.dot.accumulate.2way", .{ + .operands = .{ .flat = &.{ a, b, c_ } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "a_type", a_type), + .named(ctx, "b_type", b_type), + .named(ctx, "b_hi", b_hi), + }, + .location = location, + }); +} + +/// `nvvm.dot.accumulate.4way`. Result types are explicit; attributes use mlir.Attribute. +pub fn dot_accumulate_4way(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, c_: *const mlir.Value, res_type: *const mlir.Type, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.dot.accumulate.4way", .{ + .operands = .{ .flat = &.{ a, b, c_ } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "a_type", a_type), + .named(ctx, "b_type", b_type), + }, + .location = location, + }); +} + +/// `nvvm.elect.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn elect_sync(ctx: *mlir.Context, membermask: ?*const mlir.Value, pred_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.elect.sync", .{ + .operands = .{ .flat = if (membermask) |value| &.{value} else &.{} }, + .results = .{ .flat = &.{pred_type} }, + .location = location, + }); +} + +/// `nvvm.ex2`. Result types are explicit; attributes use mlir.Attribute. +pub fn ex2(ctx: *mlir.Context, src: *const mlir.Value, res_type: *const mlir.Type, ftz: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.ex2", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.exit`. Result types are explicit; attributes use mlir.Attribute. +pub fn exit(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.exit", .{ + .location = location, + }); +} + +/// `nvvm.fabs`. Result types are explicit; attributes use mlir.Attribute. +pub fn fabs(ctx: *mlir.Context, value_: *const mlir.Value, res_type: *const mlir.Type, ftz: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.fabs", .{ + .operands = .{ .flat = &.{value_} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fence.mbarrier.init`. Result types are explicit; attributes use mlir.Attribute. +pub fn fence_mbarrier_init(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.fence.mbarrier.init", .{ + .location = location, + }); +} + +/// `nvvm.fence.proxy`. Result types are explicit; attributes use mlir.Attribute. +pub fn fence_proxy(ctx: *mlir.Context, kind: *const mlir.Attribute, space: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (space) |value| attributes.appendAssumeCapacity(.named(ctx, "space", value)); + return mlir.Operation.make(ctx, "nvvm.fence.proxy", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fence.proxy.acquire`. Result types are explicit; attributes use mlir.Attribute. +pub fn fence_proxy_acquire(ctx: *mlir.Context, addr: *const mlir.Value, size: *const mlir.Value, scope: *const mlir.Attribute, fromProxy: ?*const mlir.Attribute, toProxy: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scope", scope)); + if (fromProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "fromProxy", value)); + if (toProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "toProxy", value)); + return mlir.Operation.make(ctx, "nvvm.fence.proxy.acquire", .{ + .operands = .{ .flat = &.{ addr, size } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fence.proxy.release`. Result types are explicit; attributes use mlir.Attribute. +pub fn fence_proxy_release(ctx: *mlir.Context, scope: *const mlir.Attribute, fromProxy: ?*const mlir.Attribute, toProxy: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scope", scope)); + if (fromProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "fromProxy", value)); + if (toProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "toProxy", value)); + return mlir.Operation.make(ctx, "nvvm.fence.proxy.release", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fence.proxy.sync_restrict`. Result types are explicit; attributes use mlir.Attribute. +pub fn fence_proxy_sync_restrict(ctx: *mlir.Context, order: *const mlir.Attribute, fromProxy: ?*const mlir.Attribute, toProxy: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "order", order)); + if (fromProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "fromProxy", value)); + if (toProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "toProxy", value)); + return mlir.Operation.make(ctx, "nvvm.fence.proxy.sync_restrict", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fence.sc.cluster`. Result types are explicit; attributes use mlir.Attribute. +pub fn fence_sc_cluster(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.fence.sc.cluster", .{ + .location = location, + }); +} + +/// `nvvm.fence.sync_restrict`. Result types are explicit; attributes use mlir.Attribute. +pub fn fence_sync_restrict(ctx: *mlir.Context, order: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.fence.sync_restrict", .{ + .attributes = &.{ + .named(ctx, "order", order), + }, + .location = location, + }); +} + +/// `nvvm.fma`. Result types are explicit; attributes use mlir.Attribute. +pub fn fma(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, c_: *const mlir.Value, res_type: *const mlir.Type, rnd: *const mlir.Attribute, sat: ?*const mlir.Attribute, ftz: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, oob: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "rnd", rnd)); + if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (oob) |value| attributes.appendAssumeCapacity(.named(ctx, "oob", value)); + return mlir.Operation.make(ctx, "nvvm.fma", .{ + .operands = .{ .flat = &.{ a, b, c_ } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fma.packed.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn fma_packed_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, srcC: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, ftz: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.fma.packed.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB, srcC } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fma.packed.f32x2.bf16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn fma_packed_f32x2_bf16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, srcC: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + return mlir.Operation.make(ctx, "nvvm.fma.packed.f32x2.bf16x2.f32x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB, srcC } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fma.packed.f32x2.f16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn fma_packed_f32x2_f16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, srcC: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + return mlir.Operation.make(ctx, "nvvm.fma.packed.f32x2.f16x2.f32x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB, srcC } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fmax`. Result types are explicit; attributes use mlir.Attribute. +pub fn fmax(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, c_: ?*const mlir.Value, res_type: *const mlir.Type, ftz: ?*const mlir.Attribute, nan: ?*const mlir.Attribute, abs: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); + if (abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); + return mlir.Operation.make(ctx, "nvvm.fmax", .{ + .operands = .{ .flat = if (c_) |value| &.{ a, b, value } else &.{ a, b } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fmin`. Result types are explicit; attributes use mlir.Attribute. +pub fn fmin(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, c_: ?*const mlir.Value, res_type: *const mlir.Type, ftz: ?*const mlir.Attribute, nan: ?*const mlir.Attribute, abs: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); + if (abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); + return mlir.Operation.make(ctx, "nvvm.fmin", .{ + .operands = .{ .flat = if (c_) |value| &.{ a, b, value } else &.{ a, b } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.griddepcontrol`. Result types are explicit; attributes use mlir.Attribute. +pub fn griddepcontrol(ctx: *mlir.Context, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.griddepcontrol", .{ + .attributes = &.{ + .named(ctx, "kind", kind), + }, + .location = location, + }); +} + +/// `nvvm.inline_ptx`. Result types are explicit; attributes use mlir.Attribute. +pub fn inline_ptx(ctx: *mlir.Context, readOnlyArgs: []const *const mlir.Value, readWriteArgs: []const *const mlir.Value, predicate: ?*const mlir.Value, writeOnlyArgs_type: ?*const mlir.Type, ptxCode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; + if (writeOnlyArgs_type) |value| results.appendAssumeCapacity(value); + return mlir.Operation.make(ctx, "nvvm.inline_ptx", .{ + .operands = .{ .variadic = &.{ + readOnlyArgs, + readWriteArgs, + if (predicate) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = results.constSlice() }, + .attributes = &.{ + .named(ctx, "ptxCode", ptxCode), + }, + .location = location, + }); +} + +/// `nvvm.ldmatrix`. Result types are explicit; attributes use mlir.Attribute. +pub fn ldmatrix(ctx: *mlir.Context, ptr: *const mlir.Value, res_type: *const mlir.Type, num: *const mlir.Attribute, layout: *const mlir.Attribute, shape: ?*const mlir.Attribute, srcFormat: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "num", num)); + attributes.appendAssumeCapacity(.named(ctx, "layout", layout)); + if (shape) |value| attributes.appendAssumeCapacity(.named(ctx, "shape", value)); + if (srcFormat) |value| attributes.appendAssumeCapacity(.named(ctx, "srcFormat", value)); + return mlir.Operation.make(ctx, "nvvm.ldmatrix", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.load.ext`. Result types are explicit; attributes use mlir.Attribute. +pub fn load_ext(ctx: *mlir.Context, addr: *const mlir.Value, l2CacheHint: ?*const mlir.Value, res_type: *const mlir.Type, order: ?*const mlir.Attribute, scope: ?*const mlir.Attribute, prefetch_: ?*const mlir.Attribute, evict: ?*const mlir.Attribute, cacheModifier: ?*const mlir.Attribute, sharedSpace: ?*const mlir.Attribute, unified: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; + if (order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (prefetch_) |value| attributes.appendAssumeCapacity(.named(ctx, "prefetch", value)); + if (evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + if (cacheModifier) |value| attributes.appendAssumeCapacity(.named(ctx, "cacheModifier", value)); + if (sharedSpace) |value| attributes.appendAssumeCapacity(.named(ctx, "sharedSpace", value)); + if (unified) |value| attributes.appendAssumeCapacity(.named(ctx, "unified", value)); + return mlir.Operation.make(ctx, "nvvm.load.ext", .{ + .operands = .{ .flat = if (l2CacheHint) |value| &.{ addr, value } else &.{addr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.log2`. Result types are explicit; attributes use mlir.Attribute. +pub fn log2(ctx: *mlir.Context, src: *const mlir.Value, res_type: *const mlir.Type, ftz: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.log2", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mapa`. Result types are explicit; attributes use mlir.Attribute. +pub fn mapa(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mapa", .{ + .operands = .{ .flat = &.{ a, b } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.match.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn match_sync(ctx: *mlir.Context, thread_mask: *const mlir.Value, val: *const mlir.Value, res_type: *const mlir.Type, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.match.sync", .{ + .operands = .{ .flat = &.{ thread_mask, val } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "kind", kind), + }, + .location = location, + }); +} + +/// `nvvm.mbarrier.arrive`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_arrive(ctx: *mlir.Context, addr: *const mlir.Value, count: ?*const mlir.Value, multicastMask: ?*const mlir.Value, res_type: ?*const mlir.Type, scope: ?*const mlir.Attribute, relaxed: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; + if (res_type) |value| results.appendAssumeCapacity(value); + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive", .{ + .operands = .{ .variadic = &.{ + &.{addr}, + if (count) |value| &.{value} else &.{}, + if (multicastMask) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = results.constSlice() }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.arrive.expect_tx`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_arrive_expect_tx(ctx: *mlir.Context, addr: *const mlir.Value, txcount: *const mlir.Value, multicastMask: ?*const mlir.Value, predicate: ?*const mlir.Value, res_type: ?*const mlir.Type, scope: ?*const mlir.Attribute, relaxed: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; + if (res_type) |value| results.appendAssumeCapacity(value); + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive.expect_tx", .{ + .operands = .{ .variadic = &.{ + &.{addr}, + &.{txcount}, + if (multicastMask) |value| &.{value} else &.{}, + if (predicate) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = results.constSlice() }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.arrive.nocomplete`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_arrive_nocomplete(ctx: *mlir.Context, addr: *const mlir.Value, count: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive.nocomplete", .{ + .operands = .{ .flat = &.{ addr, count } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.mbarrier.arrive_drop`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_arrive_drop(ctx: *mlir.Context, addr: *const mlir.Value, count: ?*const mlir.Value, multicastMask: ?*const mlir.Value, res_type: ?*const mlir.Type, scope: ?*const mlir.Attribute, relaxed: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; + if (res_type) |value| results.appendAssumeCapacity(value); + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive_drop", .{ + .operands = .{ .variadic = &.{ + &.{addr}, + if (count) |value| &.{value} else &.{}, + if (multicastMask) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = results.constSlice() }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.arrive_drop.expect_tx`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_arrive_drop_expect_tx(ctx: *mlir.Context, addr: *const mlir.Value, txcount: *const mlir.Value, multicastMask: ?*const mlir.Value, res_type: ?*const mlir.Type, scope: ?*const mlir.Attribute, relaxed: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; + if (res_type) |value| results.appendAssumeCapacity(value); + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive_drop.expect_tx", .{ + .operands = .{ .flat = if (multicastMask) |value| &.{ addr, txcount, value } else &.{ addr, txcount } }, + .results = .{ .flat = results.constSlice() }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.arrive_drop.nocomplete`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_arrive_drop_nocomplete(ctx: *mlir.Context, addr: *const mlir.Value, count: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive_drop.nocomplete", .{ + .operands = .{ .flat = &.{ addr, count } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.mbarrier.check.layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_check_layout(ctx: *mlir.Context, addr: *const mlir.Value, res_type: *const mlir.Type, layout: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mbarrier.check.layout", .{ + .operands = .{ .flat = &.{addr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "layout", layout), + }, + .location = location, + }); +} + +/// `nvvm.mbarrier.complete_tx`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_complete_tx(ctx: *mlir.Context, addr: *const mlir.Value, txcount: *const mlir.Value, report: ?*const mlir.Value, multicastMask: ?*const mlir.Value, scope: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.complete_tx", .{ + .operands = .{ .variadic = &.{ + &.{addr}, + &.{txcount}, + if (report) |value| &.{value} else &.{}, + if (multicastMask) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.expect_tx`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_expect_tx(ctx: *mlir.Context, addr: *const mlir.Value, txcount: *const mlir.Value, multicastMask: ?*const mlir.Value, scope: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.expect_tx", .{ + .operands = .{ .flat = if (multicastMask) |value| &.{ addr, txcount, value } else &.{ addr, txcount } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.init`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_init(ctx: *mlir.Context, addr: *const mlir.Value, count: *const mlir.Value, predicate: ?*const mlir.Value, layout: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (layout) |value| attributes.appendAssumeCapacity(.named(ctx, "layout", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.init", .{ + .operands = .{ .flat = if (predicate) |value| &.{ addr, count, value } else &.{ addr, count } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.inval`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_inval(ctx: *mlir.Context, addr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mbarrier.inval", .{ + .operands = .{ .flat = &.{addr} }, + .location = location, + }); +} + +/// `nvvm.mbarrier.test.wait`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_test_wait(ctx: *mlir.Context, addr: *const mlir.Value, stateOrPhase: *const mlir.Value, res_type: *const mlir.Type, scope: ?*const mlir.Attribute, relaxed: ?*const mlir.Attribute, mbarrierPhase: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); + if (mbarrierPhase) |value| attributes.appendAssumeCapacity(.named(ctx, "mbarrierPhase", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.test.wait", .{ + .operands = .{ .flat = &.{ addr, stateOrPhase } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.try_wait`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_try_wait(ctx: *mlir.Context, addr: *const mlir.Value, stateOrPhase: *const mlir.Value, ticks: ?*const mlir.Value, res_type: *const mlir.Type, scope: ?*const mlir.Attribute, relaxed: ?*const mlir.Attribute, mbarrierPhase: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); + if (mbarrierPhase) |value| attributes.appendAssumeCapacity(.named(ctx, "mbarrierPhase", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.try_wait", .{ + .operands = .{ .flat = if (ticks) |value| &.{ addr, stateOrPhase, value } else &.{ addr, stateOrPhase } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.try_wait.parity`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_try_wait_parity(ctx: *mlir.Context, addr: *const mlir.Value, phase: *const mlir.Value, ticks: *const mlir.Value, useIntrinsic: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (useIntrinsic) |value| attributes.appendAssumeCapacity(.named(ctx, "useIntrinsic", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.try_wait.parity", .{ + .operands = .{ .flat = &.{ addr, phase, ticks } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.try_wait.parity.timelimit`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_try_wait_parity_timelimit(ctx: *mlir.Context, addr: *const mlir.Value, phase: *const mlir.Value, timeLimit: *const mlir.Value, res_type: *const mlir.Type, scope: ?*const mlir.Attribute, order: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.try_wait.parity.timelimit", .{ + .operands = .{ .flat = &.{ addr, phase, timeLimit } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.try_wait.timelimit`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_try_wait_timelimit(ctx: *mlir.Context, addr: *const mlir.Value, state: *const mlir.Value, timeLimit: *const mlir.Value, res_type: *const mlir.Type, scope: ?*const mlir.Attribute, order: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.try_wait.timelimit", .{ + .operands = .{ .flat = &.{ addr, state, timeLimit } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.txn`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_txn(ctx: *mlir.Context, addr: *const mlir.Value, count: *const mlir.Value, multicast: ?*const mlir.Value, kind: *const mlir.Attribute, space: ?*const mlir.Attribute, scope: ?*const mlir.Attribute, order: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (space) |value| attributes.appendAssumeCapacity(.named(ctx, "space", value)); + if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.txn", .{ + .operands = .{ .flat = if (multicast) |value| &.{ addr, count, value } else &.{ addr, count } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.txn.cta`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_txn_cta(ctx: *mlir.Context, addr: *const mlir.Value, count: *const mlir.Value, res_type: *const mlir.Type, kind: *const mlir.Attribute, scope: ?*const mlir.Attribute, order: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.txn.cta", .{ + .operands = .{ .flat = &.{ addr, count } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.wait`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_wait(ctx: *mlir.Context, addr: *const mlir.Value, state: *const mlir.Value, res_type: *const mlir.Type, kind: *const mlir.Attribute, scope: ?*const mlir.Attribute, order: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.wait", .{ + .operands = .{ .flat = &.{ addr, state } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.wait.parity`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_wait_parity(ctx: *mlir.Context, addr: *const mlir.Value, phase: *const mlir.Value, res_type: *const mlir.Type, kind: *const mlir.Attribute, scope: ?*const mlir.Attribute, order: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.wait.parity", .{ + .operands = .{ .flat = &.{ addr, phase } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.memory.barrier`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_barrier(ctx: *mlir.Context, scope: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.memory.barrier", .{ + .attributes = &.{ + .named(ctx, "scope", scope), + }, + .location = location, + }); +} + +/// `nvvm.mma.block_scale`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_block_scale(ctx: *mlir.Context, operandA: []const *const mlir.Value, operandB: []const *const mlir.Value, operandC: []const *const mlir.Value, scaleAData: *const mlir.Value, byteIdA: *const mlir.Value, threadIdA: *const mlir.Value, scaleBData: *const mlir.Value, byteIdB: *const mlir.Value, threadIdB: *const mlir.Value, res_type: *const mlir.Type, shape: *const mlir.Attribute, multiplicandAPtxType: ?*const mlir.Attribute, multiplicandBPtxType: ?*const mlir.Attribute, scaleVecSize: *const mlir.Attribute, blockScaleFormat: *const mlir.Attribute, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + if (multiplicandAPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandAPtxType", value)); + if (multiplicandBPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandBPtxType", value)); + attributes.appendAssumeCapacity(.named(ctx, "scaleVecSize", scaleVecSize)); + attributes.appendAssumeCapacity(.named(ctx, "blockScaleFormat", blockScaleFormat)); + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + return mlir.Operation.make(ctx, "nvvm.mma.block_scale", .{ + .operands = .{ .variadic = &.{ + operandA, + operandB, + operandC, + &.{scaleAData}, + &.{byteIdA}, + &.{threadIdA}, + &.{scaleBData}, + &.{byteIdB}, + &.{threadIdB}, + } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mma.block_scale.internal`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_block_scale_internal(ctx: *mlir.Context, operandA: *const mlir.Value, scaleAData: *const mlir.Value, byteIdA: *const mlir.Value, threadIdA: *const mlir.Value, operandB: *const mlir.Value, scaleBData: *const mlir.Value, byteIdB: *const mlir.Value, threadIdB: *const mlir.Value, operandC: *const mlir.Value, res_type: *const mlir.Type, shape: *const mlir.Attribute, layoutA: *const mlir.Attribute, layoutB: *const mlir.Attribute, aType: *const mlir.Attribute, bType: *const mlir.Attribute, cType: *const mlir.Attribute, scaleVecSize: *const mlir.Attribute, blockScaleFormat: *const mlir.Attribute, kind: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 9) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + attributes.appendAssumeCapacity(.named(ctx, "layoutA", layoutA)); + attributes.appendAssumeCapacity(.named(ctx, "layoutB", layoutB)); + attributes.appendAssumeCapacity(.named(ctx, "aType", aType)); + attributes.appendAssumeCapacity(.named(ctx, "bType", bType)); + attributes.appendAssumeCapacity(.named(ctx, "cType", cType)); + attributes.appendAssumeCapacity(.named(ctx, "scaleVecSize", scaleVecSize)); + attributes.appendAssumeCapacity(.named(ctx, "blockScaleFormat", blockScaleFormat)); + if (kind) |value| attributes.appendAssumeCapacity(.named(ctx, "kind", value)); + return mlir.Operation.make(ctx, "nvvm.mma.block_scale.internal", .{ + .operands = .{ .flat = &.{ operandA, scaleAData, byteIdA, threadIdA, operandB, scaleBData, byteIdB, threadIdB, operandC } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mma.sp.block_scale`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_sp_block_scale(ctx: *mlir.Context, operandA: []const *const mlir.Value, operandB: []const *const mlir.Value, operandC: []const *const mlir.Value, sparseMetadata: *const mlir.Value, sparsitySelector: *const mlir.Value, scaleAData: *const mlir.Value, byteIdA: *const mlir.Value, threadIdA: *const mlir.Value, scaleBData: *const mlir.Value, byteIdB: *const mlir.Value, threadIdB: *const mlir.Value, res_type: *const mlir.Type, shape: *const mlir.Attribute, multiplicandAPtxType: ?*const mlir.Attribute, multiplicandBPtxType: ?*const mlir.Attribute, scaleVecSize: *const mlir.Attribute, blockScaleFormat: *const mlir.Attribute, kind: *const mlir.Attribute, orderedMetadata: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + if (multiplicandAPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandAPtxType", value)); + if (multiplicandBPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandBPtxType", value)); + attributes.appendAssumeCapacity(.named(ctx, "scaleVecSize", scaleVecSize)); + attributes.appendAssumeCapacity(.named(ctx, "blockScaleFormat", blockScaleFormat)); + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (orderedMetadata) |value| attributes.appendAssumeCapacity(.named(ctx, "orderedMetadata", value)); + return mlir.Operation.make(ctx, "nvvm.mma.sp.block_scale", .{ + .operands = .{ .variadic = &.{ + operandA, + operandB, + operandC, + &.{sparseMetadata}, + &.{sparsitySelector}, + &.{scaleAData}, + &.{byteIdA}, + &.{threadIdA}, + &.{scaleBData}, + &.{byteIdB}, + &.{threadIdB}, + } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mma.sp.block_scale.internal`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_sp_block_scale_internal(ctx: *mlir.Context, operandA: *const mlir.Value, scaleAData: *const mlir.Value, byteIdA: *const mlir.Value, threadIdA: *const mlir.Value, operandB: *const mlir.Value, scaleBData: *const mlir.Value, byteIdB: *const mlir.Value, threadIdB: *const mlir.Value, operandC: *const mlir.Value, sp_metadata: *const mlir.Value, selector: *const mlir.Value, res_type: *const mlir.Type, shape: *const mlir.Attribute, b1Op: ?*const mlir.Attribute, intOverflowBehavior: ?*const mlir.Attribute, layoutA: *const mlir.Attribute, layoutB: *const mlir.Attribute, aType: *const mlir.Attribute, bType: *const mlir.Attribute, cType: *const mlir.Attribute, scaleVecSize: *const mlir.Attribute, blockScaleFormat: *const mlir.Attribute, sparsityFormat: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 11) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + if (b1Op) |value| attributes.appendAssumeCapacity(.named(ctx, "b1Op", value)); + if (intOverflowBehavior) |value| attributes.appendAssumeCapacity(.named(ctx, "intOverflowBehavior", value)); + attributes.appendAssumeCapacity(.named(ctx, "layoutA", layoutA)); + attributes.appendAssumeCapacity(.named(ctx, "layoutB", layoutB)); + attributes.appendAssumeCapacity(.named(ctx, "aType", aType)); + attributes.appendAssumeCapacity(.named(ctx, "bType", bType)); + attributes.appendAssumeCapacity(.named(ctx, "cType", cType)); + attributes.appendAssumeCapacity(.named(ctx, "scaleVecSize", scaleVecSize)); + attributes.appendAssumeCapacity(.named(ctx, "blockScaleFormat", blockScaleFormat)); + attributes.appendAssumeCapacity(.named(ctx, "sparsityFormat", sparsityFormat)); + return mlir.Operation.make(ctx, "nvvm.mma.sp.block_scale.internal", .{ + .operands = .{ .flat = &.{ operandA, scaleAData, byteIdA, threadIdA, operandB, scaleBData, byteIdB, threadIdB, operandC, sp_metadata, selector } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mma.sp.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_sp_sync(ctx: *mlir.Context, operandA: []const *const mlir.Value, operandB: []const *const mlir.Value, operandC: []const *const mlir.Value, sparseMetadata: *const mlir.Value, sparsitySelector: *const mlir.Value, res_type: *const mlir.Type, shape: *const mlir.Attribute, intOverflowBehavior: ?*const mlir.Attribute, multiplicandAPtxType: ?*const mlir.Attribute, multiplicandBPtxType: ?*const mlir.Attribute, orderedMetadata: ?*const mlir.Attribute, kind: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + if (intOverflowBehavior) |value| attributes.appendAssumeCapacity(.named(ctx, "intOverflowBehavior", value)); + if (multiplicandAPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandAPtxType", value)); + if (multiplicandBPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandBPtxType", value)); + if (orderedMetadata) |value| attributes.appendAssumeCapacity(.named(ctx, "orderedMetadata", value)); + if (kind) |value| attributes.appendAssumeCapacity(.named(ctx, "kind", value)); + return mlir.Operation.make(ctx, "nvvm.mma.sp.sync", .{ + .operands = .{ .variadic = &.{ + operandA, + operandB, + operandC, + &.{sparseMetadata}, + &.{sparsitySelector}, + } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mma.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_sync(ctx: *mlir.Context, operandA: []const *const mlir.Value, operandB: []const *const mlir.Value, operandC: []const *const mlir.Value, res_type: *const mlir.Type, shape: *const mlir.Attribute, b1Op: ?*const mlir.Attribute, intOverflowBehavior: ?*const mlir.Attribute, layoutA: *const mlir.Attribute, layoutB: *const mlir.Attribute, multiplicandAPtxType: ?*const mlir.Attribute, multiplicandBPtxType: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + if (b1Op) |value| attributes.appendAssumeCapacity(.named(ctx, "b1Op", value)); + if (intOverflowBehavior) |value| attributes.appendAssumeCapacity(.named(ctx, "intOverflowBehavior", value)); + attributes.appendAssumeCapacity(.named(ctx, "layoutA", layoutA)); + attributes.appendAssumeCapacity(.named(ctx, "layoutB", layoutB)); + if (multiplicandAPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandAPtxType", value)); + if (multiplicandBPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandBPtxType", value)); + return mlir.Operation.make(ctx, "nvvm.mma.sync", .{ + .operands = .{ .variadic = &.{ + operandA, + operandB, + operandC, + } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mma_smem_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_smem_desc(ctx: *mlir.Context, pointer: *const mlir.Value, ldm: *const mlir.Value, stride: *const mlir.Value, baseOffset: *const mlir.Value, swizzle: *const mlir.Value, res_type: *const mlir.Type, mmaDescVersion: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (mmaDescVersion) |value| attributes.appendAssumeCapacity(.named(ctx, "mmaDescVersion", value)); + return mlir.Operation.make(ctx, "nvvm.mma_smem_desc", .{ + .operands = .{ .flat = &.{ pointer, ldm, stride, baseOffset, swizzle } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.movmatrix`. Result types are explicit; attributes use mlir.Attribute. +pub fn movmatrix(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, shape: *const mlir.Attribute, layout: ?*const mlir.Attribute, eltType: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + if (layout) |value| attributes.appendAssumeCapacity(.named(ctx, "layout", value)); + attributes.appendAssumeCapacity(.named(ctx, "eltType", eltType)); + return mlir.Operation.make(ctx, "nvvm.movmatrix", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mul`. Result types are explicit; attributes use mlir.Attribute. +pub fn mul(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, res_type: *const mlir.Type, mode: *const mlir.Attribute, isSigned: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "mode", mode)); + if (isSigned) |value| attributes.appendAssumeCapacity(.named(ctx, "isSigned", value)); + return mlir.Operation.make(ctx, "nvvm.mul", .{ + .operands = .{ .flat = &.{ a, b } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mul.packed.bf16x2.bf16x2.f16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn mul_packed_bf16x2_bf16x2_f16x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mul.packed.bf16x2.bf16x2.f16x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.mul.packed.bf16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn mul_packed_bf16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mul.packed.bf16x2.f32x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.mul.packed.f16x2.f16x2.bf16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn mul_packed_f16x2_f16x2_bf16x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mul.packed.f16x2.f16x2.bf16x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.mul.packed.f16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn mul_packed_f16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mul.packed.f16x2.f32x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.mul.packed.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn mul_packed_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, ftz: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.mul.packed.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.multimem.ld.reduce`. Result types are explicit; attributes use mlir.Attribute. +pub fn multimem_ld_reduce(ctx: *mlir.Context, addr: *const mlir.Value, res_type: *const mlir.Type, ordering: *const mlir.Attribute, op: *const mlir.Attribute, dataType: *const mlir.Attribute, accPrec: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "ordering", ordering)); + attributes.appendAssumeCapacity(.named(ctx, "op", op)); + attributes.appendAssumeCapacity(.named(ctx, "dataType", dataType)); + if (accPrec) |value| attributes.appendAssumeCapacity(.named(ctx, "accPrec", value)); + return mlir.Operation.make(ctx, "nvvm.multimem.ld.reduce", .{ + .operands = .{ .flat = &.{addr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.multimem.red`. Result types are explicit; attributes use mlir.Attribute. +pub fn multimem_red(ctx: *mlir.Context, val: *const mlir.Value, addr: *const mlir.Value, ordering: *const mlir.Attribute, scope: *const mlir.Attribute, op: *const mlir.Attribute, dataType: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.multimem.red", .{ + .operands = .{ .flat = &.{ val, addr } }, + .attributes = &.{ + .named(ctx, "ordering", ordering), + .named(ctx, "scope", scope), + .named(ctx, "op", op), + .named(ctx, "dataType", dataType), + }, + .location = location, + }); +} + +/// `nvvm.multimem.st`. Result types are explicit; attributes use mlir.Attribute. +pub fn multimem_st(ctx: *mlir.Context, val: *const mlir.Value, addr: *const mlir.Value, ordering: *const mlir.Attribute, scope: *const mlir.Attribute, dataType: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.multimem.st", .{ + .operands = .{ .flat = &.{ val, addr } }, + .attributes = &.{ + .named(ctx, "ordering", ordering), + .named(ctx, "scope", scope), + .named(ctx, "dataType", dataType), + }, + .location = location, + }); +} + +/// `nvvm.nanosleep`. Result types are explicit; attributes use mlir.Attribute. +pub fn nanosleep(ctx: *mlir.Context, duration: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.nanosleep", .{ + .operands = .{ .flat = &.{duration} }, + .location = location, + }); +} + +/// `nvvm.pmevent`. Result types are explicit; attributes use mlir.Attribute. +pub fn pmevent(ctx: *mlir.Context, maskedEventId: ?*const mlir.Attribute, eventId: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (maskedEventId) |value| attributes.appendAssumeCapacity(.named(ctx, "maskedEventId", value)); + if (eventId) |value| attributes.appendAssumeCapacity(.named(ctx, "eventId", value)); + return mlir.Operation.make(ctx, "nvvm.pmevent", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.prefetch`. Result types are explicit; attributes use mlir.Attribute. +pub fn prefetch(ctx: *mlir.Context, addr: *const mlir.Value, predicate: ?*const mlir.Value, cacheLevel: ?*const mlir.Attribute, evictPriority: ?*const mlir.Attribute, tensormap: ?*const mlir.Attribute, uniform: ?*const mlir.Attribute, in_param_space: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (cacheLevel) |value| attributes.appendAssumeCapacity(.named(ctx, "cacheLevel", value)); + if (evictPriority) |value| attributes.appendAssumeCapacity(.named(ctx, "evictPriority", value)); + if (tensormap) |value| attributes.appendAssumeCapacity(.named(ctx, "tensormap", value)); + if (uniform) |value| attributes.appendAssumeCapacity(.named(ctx, "uniform", value)); + if (in_param_space) |value| attributes.appendAssumeCapacity(.named(ctx, "in_param_space", value)); + return mlir.Operation.make(ctx, "nvvm.prefetch", .{ + .operands = .{ .flat = if (predicate) |value| &.{ addr, value } else &.{addr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.prmt`. Result types are explicit; attributes use mlir.Attribute. +pub fn prmt(ctx: *mlir.Context, lo: *const mlir.Value, hi: ?*const mlir.Value, selector: *const mlir.Value, res_type: *const mlir.Type, mode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.prmt", .{ + .operands = .{ .flat = if (hi) |value| &.{ lo, value, selector } else &.{ lo, selector } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "mode", mode), + }, + .location = location, + }); +} + +/// `nvvm.rcp.approx.ftz.f`. Result types are explicit; attributes use mlir.Attribute. +pub fn rcp_approx_ftz_f(ctx: *mlir.Context, arg: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.rcp.approx.ftz.f", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.aggr.smem.size`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_aggr_smem_size(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.aggr.smem.size", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.clock`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_clock(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.clock", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.clock64`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_clock64(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.clock64", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.cluster.ctaid.x`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_cluster_ctaid_x(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.ctaid.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.cluster.ctaid.y`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_cluster_ctaid_y(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.ctaid.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.cluster.ctaid.z`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_cluster_ctaid_z(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.ctaid.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.cluster.ctarank`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_cluster_ctarank(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.ctarank", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.cluster.nctaid.x`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_cluster_nctaid_x(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.nctaid.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.cluster.nctaid.y`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_cluster_nctaid_y(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.nctaid.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.cluster.nctaid.z`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_cluster_nctaid_z(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.nctaid.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.cluster.nctarank`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_cluster_nctarank(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.nctarank", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.clusterid.x`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_clusterid_x(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.clusterid.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.clusterid.y`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_clusterid_y(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.clusterid.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.clusterid.z`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_clusterid_z(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.clusterid.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.ctaid.x`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_ctaid_x(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ctaid.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.ctaid.y`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_ctaid_y(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ctaid.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.ctaid.z`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_ctaid_z(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ctaid.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.dynamic.smem.size`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_dynamic_smem_size(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.dynamic.smem.size", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg0`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg0(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg0", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg1`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg1(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg1", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg10`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg10(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg10", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg11`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg11(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg11", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg12`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg12(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg12", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg13`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg13(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg13", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg14`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg14(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg14", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg15`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg15(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg15", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg16`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg16(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg16", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg17`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg17(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg17", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg18`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg18(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg18", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg19`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg19(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg19", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg2`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg2(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg2", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg20`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg20(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg20", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg21`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg21(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg21", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg22`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg22(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg22", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg23`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg23(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg23", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg24`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg24(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg24", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg25`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg25(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg25", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg26`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg26(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg26", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg27`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg27(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg27", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg28`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg28(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg28", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg29`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg29(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg29", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg3`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg3(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg3", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg30`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg30(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg30", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg31`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg31(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg31", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg4`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg4(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg4", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg5`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg5(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg5", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg6`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg6(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg6", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg7`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg7(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg7", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg8`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg8(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg8", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg9`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg9(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg9", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.globaltimer`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_globaltimer(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.globaltimer", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.globaltimer.lo`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_globaltimer_lo(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.globaltimer.lo", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.gridid`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_gridid(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.gridid", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.laneid`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_laneid(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.laneid", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.lanemask.eq`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_lanemask_eq(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.lanemask.eq", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.lanemask.ge`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_lanemask_ge(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.lanemask.ge", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.lanemask.gt`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_lanemask_gt(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.lanemask.gt", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.lanemask.le`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_lanemask_le(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.lanemask.le", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.lanemask.lt`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_lanemask_lt(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.lanemask.lt", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.nclusterid.x`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_nclusterid_x(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nclusterid.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.nclusterid.y`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_nclusterid_y(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nclusterid.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.nclusterid.z`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_nclusterid_z(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nclusterid.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.nctaid.x`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_nctaid_x(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nctaid.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.nctaid.y`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_nctaid_y(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nctaid.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.nctaid.z`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_nctaid_z(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nctaid.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.nsmid`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_nsmid(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nsmid", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.ntid.x`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_ntid_x(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ntid.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.ntid.y`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_ntid_y(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ntid.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.ntid.z`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_ntid_z(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ntid.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.nwarpid`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_nwarpid(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nwarpid", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.smid`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_smid(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.smid", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.tid.x`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_tid_x(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.tid.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.tid.y`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_tid_y(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.tid.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.tid.z`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_tid_z(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.tid.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.total.smem.size`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_total_smem_size(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.total.smem.size", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.warpid`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_warpid(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.warpid", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.warpsize`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_warpsize(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.warpsize", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.red`. Result types are explicit; attributes use mlir.Attribute. +pub fn red(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, cacheHint: ?*const mlir.Value, memOrder: ?*const mlir.Attribute, memScope: ?*const mlir.Attribute, op: *const mlir.Attribute, type_: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (memOrder) |value| attributes.appendAssumeCapacity(.named(ctx, "memOrder", value)); + if (memScope) |value| attributes.appendAssumeCapacity(.named(ctx, "memScope", value)); + attributes.appendAssumeCapacity(.named(ctx, "op", op)); + attributes.appendAssumeCapacity(.named(ctx, "type", type_)); + return mlir.Operation.make(ctx, "nvvm.red", .{ + .operands = .{ .flat = if (cacheHint) |value| &.{ a, b, value } else &.{ a, b } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.redux.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn redux_sync(ctx: *mlir.Context, val: *const mlir.Value, mask_and_clamp: *const mlir.Value, res_type: *const mlir.Type, kind: *const mlir.Attribute, abs: ?*const mlir.Attribute, nan: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); + if (nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); + return mlir.Operation.make(ctx, "nvvm.redux.sync", .{ + .operands = .{ .flat = &.{ val, mask_and_clamp } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.rsqrt`. Result types are explicit; attributes use mlir.Attribute. +pub fn rsqrt(ctx: *mlir.Context, src: *const mlir.Value, res_type: *const mlir.Type, ftz: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.rsqrt", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.setmaxregister`. Result types are explicit; attributes use mlir.Attribute. +pub fn setmaxregister(ctx: *mlir.Context, regCount: *const mlir.Attribute, action: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.setmaxregister", .{ + .attributes = &.{ + .named(ctx, "regCount", regCount), + .named(ctx, "action", action), + }, + .location = location, + }); +} + +/// `nvvm.shfl.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn shfl_sync(ctx: *mlir.Context, thread_mask: *const mlir.Value, val: *const mlir.Value, offset: *const mlir.Value, mask_and_clamp: *const mlir.Value, res_type: *const mlir.Type, kind: *const mlir.Attribute, return_value_and_is_valid: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (return_value_and_is_valid) |value| attributes.appendAssumeCapacity(.named(ctx, "return_value_and_is_valid", value)); + return mlir.Operation.make(ctx, "nvvm.shfl.sync", .{ + .operands = .{ .flat = &.{ thread_mask, val, offset, mask_and_clamp } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.sin`. Result types are explicit; attributes use mlir.Attribute. +pub fn sin(ctx: *mlir.Context, src: *const mlir.Value, res_type: *const mlir.Type, ftz: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.sin", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.spcompress`. Result types are explicit; attributes use mlir.Attribute. +pub fn spcompress(ctx: *mlir.Context, spDesc: *const mlir.Value, data: *const mlir.Value, metadata_type: *const mlir.Type, compressed_data_type: *const mlir.Type, compFactor: *const mlir.Attribute, indexSize: *const mlir.Attribute, elemSize: *const mlir.Attribute, repFactor: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.spcompress", .{ + .operands = .{ .flat = &.{ spDesc, data } }, + .results = .{ .flat = &.{ metadata_type, compressed_data_type } }, + .attributes = &.{ + .named(ctx, "compFactor", compFactor), + .named(ctx, "indexSize", indexSize), + .named(ctx, "elemSize", elemSize), + .named(ctx, "repFactor", repFactor), + }, + .location = location, + }); +} + +/// `nvvm.spdecompress`. Result types are explicit; attributes use mlir.Attribute. +pub fn spdecompress(ctx: *mlir.Context, metadata: *const mlir.Value, compressed_data: *const mlir.Value, data_type: *const mlir.Type, compFactor: *const mlir.Attribute, indexSize: *const mlir.Attribute, elemSize: *const mlir.Attribute, repFactor: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.spdecompress", .{ + .operands = .{ .flat = &.{ metadata, compressed_data } }, + .results = .{ .flat = &.{data_type} }, + .attributes = &.{ + .named(ctx, "compFactor", compFactor), + .named(ctx, "indexSize", indexSize), + .named(ctx, "elemSize", elemSize), + .named(ctx, "repFactor", repFactor), + }, + .location = location, + }); +} + +/// `nvvm.st.bulk`. Result types are explicit; attributes use mlir.Attribute. +pub fn st_bulk(ctx: *mlir.Context, addr: *const mlir.Value, size: *const mlir.Value, initVal: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (initVal) |value| attributes.appendAssumeCapacity(.named(ctx, "initVal", value)); + return mlir.Operation.make(ctx, "nvvm.st.bulk", .{ + .operands = .{ .flat = &.{ addr, size } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.stmatrix`. Result types are explicit; attributes use mlir.Attribute. +pub fn stmatrix(ctx: *mlir.Context, ptr: *const mlir.Value, sources: []const *const mlir.Value, layout: *const mlir.Attribute, shape: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 256) = .empty; + operands.appendAssumeCapacity(ptr); + operands.appendSliceAssumeCapacity(sources); + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "layout", layout)); + if (shape) |value| attributes.appendAssumeCapacity(.named(ctx, "shape", value)); + return mlir.Operation.make(ctx, "nvvm.stmatrix", .{ + .operands = .{ .flat = operands.constSlice() }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.store.async`. Result types are explicit; attributes use mlir.Attribute. +pub fn store_async(ctx: *mlir.Context, addr: *const mlir.Value, value_: *const mlir.Value, mbarrier: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.store.async", .{ + .operands = .{ .flat = &.{ addr, value_, mbarrier } }, + .location = location, + }); +} + +/// `nvvm.store.ext`. Result types are explicit; attributes use mlir.Attribute. +pub fn store_ext(ctx: *mlir.Context, value_: *const mlir.Value, addr: *const mlir.Value, l2CacheHint: ?*const mlir.Value, order: ?*const mlir.Attribute, scope: ?*const mlir.Attribute, evict: ?*const mlir.Attribute, cacheModifier: ?*const mlir.Attribute, sharedSpace: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + if (cacheModifier) |value| attributes.appendAssumeCapacity(.named(ctx, "cacheModifier", value)); + if (sharedSpace) |value| attributes.appendAssumeCapacity(.named(ctx, "sharedSpace", value)); + return mlir.Operation.make(ctx, "nvvm.store.ext", .{ + .operands = .{ .flat = if (l2CacheHint) |value| &.{ value_, addr, value } else &.{ value_, addr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.sub.packed.bf16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn sub_packed_bf16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.sub.packed.bf16x2.f32x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.sub.packed.f16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn sub_packed_f16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.sub.packed.f16x2.f32x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.sub.packed.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn sub_packed_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, ftz: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.sub.packed.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.sub.packed.f32x2.bf16x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn sub_packed_f32x2_bf16x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + return mlir.Operation.make(ctx, "nvvm.sub.packed.f32x2.bf16x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.sub.packed.f32x2.f16x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn sub_packed_f32x2_f16x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + return mlir.Operation.make(ctx, "nvvm.sub.packed.f32x2.f16x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.subf`. Result types are explicit; attributes use mlir.Attribute. +pub fn subf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, sat: ?*const mlir.Attribute, ftz: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.subf", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.alloc`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_alloc(ctx: *mlir.Context, addr: *const mlir.Value, nCols: *const mlir.Value, isExclusive: ?*const mlir.Attribute, group: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (isExclusive) |value| attributes.appendAssumeCapacity(.named(ctx, "isExclusive", value)); + if (group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.alloc", .{ + .operands = .{ .flat = &.{ addr, nCols } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.commit`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_commit(ctx: *mlir.Context, addr: *const mlir.Value, multicastMask: ?*const mlir.Value, smem_a_read: ?*const mlir.Attribute, group: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (smem_a_read) |value| attributes.appendAssumeCapacity(.named(ctx, "smem_a_read", value)); + if (group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.commit", .{ + .operands = .{ .flat = if (multicastMask) |value| &.{ addr, value } else &.{addr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.cp`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_cp(ctx: *mlir.Context, taddr: *const mlir.Value, smem_desc: *const mlir.Value, shape: *const mlir.Attribute, group: ?*const mlir.Attribute, multicast: ?*const mlir.Attribute, srcFormat: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + if (group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + if (multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "multicast", value)); + if (srcFormat) |value| attributes.appendAssumeCapacity(.named(ctx, "srcFormat", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.cp", .{ + .operands = .{ .flat = &.{ taddr, smem_desc } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.dealloc`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_dealloc(ctx: *mlir.Context, taddr: *const mlir.Value, nCols: *const mlir.Value, isExclusive: ?*const mlir.Attribute, group: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (isExclusive) |value| attributes.appendAssumeCapacity(.named(ctx, "isExclusive", value)); + if (group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.dealloc", .{ + .operands = .{ .flat = &.{ taddr, nCols } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.fence`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_fence(ctx: *mlir.Context, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.tcgen05.fence", .{ + .attributes = &.{ + .named(ctx, "kind", kind), + }, + .location = location, + }); +} + +/// `nvvm.tcgen05.ld`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_ld(ctx: *mlir.Context, tmemAddr: *const mlir.Value, offset: ?*const mlir.Value, res_type: *const mlir.Type, pack: ?*const mlir.Attribute, shape: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (pack) |value| attributes.appendAssumeCapacity(.named(ctx, "pack", value)); + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.ld", .{ + .operands = .{ .flat = if (offset) |value| &.{ tmemAddr, value } else &.{tmemAddr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.ld.red`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_ld_red(ctx: *mlir.Context, addr: *const mlir.Value, offset: ?*const mlir.Value, data_type: *const mlir.Type, redVal_type: *const mlir.Type, shape: *const mlir.Attribute, op: *const mlir.Attribute, abs: ?*const mlir.Attribute, nan: ?*const mlir.Attribute, isSigned: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + attributes.appendAssumeCapacity(.named(ctx, "op", op)); + if (abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); + if (nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); + if (isSigned) |value| attributes.appendAssumeCapacity(.named(ctx, "isSigned", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.ld.red", .{ + .operands = .{ .flat = if (offset) |value| &.{ addr, value } else &.{addr} }, + .results = .{ .flat = &.{ data_type, redVal_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.ld.red.spcompress`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_ld_red_spcompress(ctx: *mlir.Context, tmemAddr: *const mlir.Value, metadata_type: *const mlir.Type, compressedData_type: *const mlir.Type, redVal_type: *const mlir.Type, abs: ?*const mlir.Attribute, nan: ?*const mlir.Attribute, shape: *const mlir.Attribute, opKind: *const mlir.Attribute, compFactor: *const mlir.Attribute, repFactor: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + if (abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); + if (nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + attributes.appendAssumeCapacity(.named(ctx, "opKind", opKind)); + attributes.appendAssumeCapacity(.named(ctx, "compFactor", compFactor)); + attributes.appendAssumeCapacity(.named(ctx, "repFactor", repFactor)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.ld.red.spcompress", .{ + .operands = .{ .flat = &.{tmemAddr} }, + .results = .{ .flat = &.{ metadata_type, compressedData_type, redVal_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.ld.spcompress`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_ld_spcompress(ctx: *mlir.Context, tmemAddr: *const mlir.Value, metadata_type: *const mlir.Type, compressedData_type: *const mlir.Type, abs: ?*const mlir.Attribute, shape: *const mlir.Attribute, opKind: *const mlir.Attribute, compFactor: *const mlir.Attribute, repFactor: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + attributes.appendAssumeCapacity(.named(ctx, "opKind", opKind)); + attributes.appendAssumeCapacity(.named(ctx, "compFactor", compFactor)); + attributes.appendAssumeCapacity(.named(ctx, "repFactor", repFactor)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.ld.spcompress", .{ + .operands = .{ .flat = &.{tmemAddr} }, + .results = .{ .flat = &.{ metadata_type, compressedData_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma(ctx: *mlir.Context, matrixD: *const mlir.Value, matrixA: *const mlir.Value, matrixB: *const mlir.Value, idesc: *const mlir.Value, enableInputD: *const mlir.Value, scaleInputD: ?*const mlir.Value, disableOutputLane: ?*const mlir.Value, kind: *const mlir.Attribute, ctaGroup: *const mlir.Attribute, collectorOp: ?*const mlir.Attribute, collectorOpB: ?*const mlir.Attribute, aShift: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); + if (collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); + if (collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); + if (aShift) |value| attributes.appendAssumeCapacity(.named(ctx, "aShift", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma", .{ + .operands = .{ .variadic = &.{ + &.{matrixD}, + &.{matrixA}, + &.{matrixB}, + &.{idesc}, + &.{enableInputD}, + if (scaleInputD) |value| &.{value} else &.{}, + if (disableOutputLane) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma.block_scale`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_block_scale(ctx: *mlir.Context, matrixD: *const mlir.Value, matrixA: *const mlir.Value, matrixB: *const mlir.Value, idesc: *const mlir.Value, enableInputD: *const mlir.Value, scaleA: *const mlir.Value, scaleB: *const mlir.Value, kind: *const mlir.Attribute, ctaGroup: *const mlir.Attribute, blockScale: ?*const mlir.Attribute, collectorOp: ?*const mlir.Attribute, collectorOpB: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); + if (blockScale) |value| attributes.appendAssumeCapacity(.named(ctx, "blockScale", value)); + if (collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); + if (collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.block_scale", .{ + .operands = .{ .flat = &.{ matrixD, matrixA, matrixB, idesc, enableInputD, scaleA, scaleB } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma.block_scale.decompress_b`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_block_scale_decompress_b(ctx: *mlir.Context, matrixD: *const mlir.Value, matrixA: *const mlir.Value, matrixB: *const mlir.Value, idesc: *const mlir.Value, enableInputD: *const mlir.Value, scaleA: *const mlir.Value, scaleB: *const mlir.Value, decompressBMetadata: *const mlir.Value, kind: *const mlir.Attribute, ctaGroup: *const mlir.Attribute, blockScale: ?*const mlir.Attribute, collectorOpA: ?*const mlir.Attribute, collectorOpB: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); + if (blockScale) |value| attributes.appendAssumeCapacity(.named(ctx, "blockScale", value)); + if (collectorOpA) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpA", value)); + if (collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.block_scale.decompress_b", .{ + .operands = .{ .flat = &.{ matrixD, matrixA, matrixB, idesc, enableInputD, scaleA, scaleB, decompressBMetadata } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma.decompress_b`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_decompress_b(ctx: *mlir.Context, matrixD: *const mlir.Value, matrixA: *const mlir.Value, matrixB: *const mlir.Value, idesc: *const mlir.Value, enableInputD: *const mlir.Value, decompressBMetadata: *const mlir.Value, disableOutputLane: ?*const mlir.Value, kind: *const mlir.Attribute, ctaGroup: *const mlir.Attribute, collectorOpA: ?*const mlir.Attribute, collectorOpB: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); + if (collectorOpA) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpA", value)); + if (collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.decompress_b", .{ + .operands = .{ .flat = if (disableOutputLane) |value| &.{ matrixD, matrixA, matrixB, idesc, enableInputD, decompressBMetadata, value } else &.{ matrixD, matrixA, matrixB, idesc, enableInputD, decompressBMetadata } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma.sp`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_sp(ctx: *mlir.Context, matrixD: *const mlir.Value, matrixA: *const mlir.Value, matrixB: *const mlir.Value, idesc: *const mlir.Value, enableInputD: *const mlir.Value, sparseMetadata: *const mlir.Value, scaleInputD: ?*const mlir.Value, disableOutputLane: ?*const mlir.Value, kind: *const mlir.Attribute, ctaGroup: *const mlir.Attribute, collectorOp: ?*const mlir.Attribute, collectorOpB: ?*const mlir.Attribute, aShift: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); + if (collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); + if (collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); + if (aShift) |value| attributes.appendAssumeCapacity(.named(ctx, "aShift", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.sp", .{ + .operands = .{ .variadic = &.{ + &.{matrixD}, + &.{matrixA}, + &.{matrixB}, + &.{idesc}, + &.{enableInputD}, + &.{sparseMetadata}, + if (scaleInputD) |value| &.{value} else &.{}, + if (disableOutputLane) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma.sp.block_scale`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_sp_block_scale(ctx: *mlir.Context, matrixD: *const mlir.Value, matrixA: *const mlir.Value, matrixB: *const mlir.Value, idesc: *const mlir.Value, enableInputD: *const mlir.Value, sparseMetadata: *const mlir.Value, scaleA: *const mlir.Value, scaleB: *const mlir.Value, kind: *const mlir.Attribute, ctaGroup: *const mlir.Attribute, blockScale: ?*const mlir.Attribute, collectorOp: ?*const mlir.Attribute, collectorOpB: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); + if (blockScale) |value| attributes.appendAssumeCapacity(.named(ctx, "blockScale", value)); + if (collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); + if (collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.sp.block_scale", .{ + .operands = .{ .flat = &.{ matrixD, matrixA, matrixB, idesc, enableInputD, sparseMetadata, scaleA, scaleB } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma.ws`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_ws(ctx: *mlir.Context, matrixD: *const mlir.Value, matrixA: *const mlir.Value, matrixB: *const mlir.Value, idesc: *const mlir.Value, enableInputD: *const mlir.Value, zeroColMask: ?*const mlir.Value, kind: *const mlir.Attribute, collectorBBuffer: ?*const mlir.Attribute, collectorOp: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (collectorBBuffer) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorBBuffer", value)); + if (collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.ws", .{ + .operands = .{ .flat = if (zeroColMask) |value| &.{ matrixD, matrixA, matrixB, idesc, enableInputD, value } else &.{ matrixD, matrixA, matrixB, idesc, enableInputD } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma.ws.sp`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_ws_sp(ctx: *mlir.Context, matrixD: *const mlir.Value, matrixA: *const mlir.Value, matrixB: *const mlir.Value, idesc: *const mlir.Value, enableInputD: *const mlir.Value, sparseMetadata: *const mlir.Value, zeroColMask: ?*const mlir.Value, kind: *const mlir.Attribute, collectorBBuffer: ?*const mlir.Attribute, collectorOp: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (collectorBBuffer) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorBBuffer", value)); + if (collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.ws.sp", .{ + .operands = .{ .flat = if (zeroColMask) |value| &.{ matrixD, matrixA, matrixB, idesc, enableInputD, sparseMetadata, value } else &.{ matrixD, matrixA, matrixB, idesc, enableInputD, sparseMetadata } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma_smem_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_smem_desc(ctx: *mlir.Context, startAddr: *const mlir.Value, leadingDimOffset: *const mlir.Value, strideDimOffset: *const mlir.Value, baseOffset: *const mlir.Value, leadingDimMode: *const mlir.Value, swizzleMode: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma_smem_desc", .{ + .operands = .{ .flat = &.{ startAddr, leadingDimOffset, strideDimOffset, baseOffset, leadingDimMode, swizzleMode } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.tcgen05.relinquish_alloc_permit`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_relinquish_alloc_permit(ctx: *mlir.Context, group: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.relinquish_alloc_permit", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.shift`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_shift(ctx: *mlir.Context, taddr: *const mlir.Value, group: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.shift", .{ + .operands = .{ .flat = &.{taddr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.st`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_st(ctx: *mlir.Context, tmemAddr: *const mlir.Value, val: *const mlir.Value, offset: ?*const mlir.Value, unpack: ?*const mlir.Attribute, shape: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (unpack) |value| attributes.appendAssumeCapacity(.named(ctx, "unpack", value)); + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.st", .{ + .operands = .{ .flat = if (offset) |value| &.{ tmemAddr, val, value } else &.{ tmemAddr, val } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.wait`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_wait(ctx: *mlir.Context, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.tcgen05.wait", .{ + .attributes = &.{ + .named(ctx, "kind", kind), + }, + .location = location, + }); +} + +/// `nvvm.tcgen05_mma_smem_desc_v2`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_smem_desc_v2(ctx: *mlir.Context, startAddress: *const mlir.Value, leadingDimOffset: *const mlir.Value, strideDimOffset: *const mlir.Value, descriptorVersion: *const mlir.Value, baseOffset: *const mlir.Value, leadingDimMode: *const mlir.Value, kSegmentOffset: *const mlir.Value, swizzleType: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.tcgen05_mma_smem_desc_v2", .{ + .operands = .{ .flat = &.{ startAddress, leadingDimOffset, strideDimOffset, descriptorVersion, baseOffset, leadingDimMode, kSegmentOffset, swizzleType } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.tensormap.cp_fenceproxy`. Result types are explicit; attributes use mlir.Attribute. +pub fn tensormap_cp_fenceproxy(ctx: *mlir.Context, dst: *const mlir.Value, src: *const mlir.Value, size: *const mlir.Value, scope: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.tensormap.cp_fenceproxy", .{ + .operands = .{ .flat = &.{ dst, src, size } }, + .attributes = &.{ + .named(ctx, "scope", scope), + }, + .location = location, + }); +} + +/// `nvvm.tensormap.replace`. Result types are explicit; attributes use mlir.Attribute. +pub fn tensormap_replace(ctx: *mlir.Context, addr: *const mlir.Value, new_value: ?*const mlir.Value, field: *const mlir.Attribute, ord: ?*const mlir.Attribute, new_value_attr: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "field", field)); + if (ord) |value| attributes.appendAssumeCapacity(.named(ctx, "ord", value)); + if (new_value_attr) |value| attributes.appendAssumeCapacity(.named(ctx, "new_value_attr", value)); + return mlir.Operation.make(ctx, "nvvm.tensormap.replace", .{ + .operands = .{ .flat = if (new_value) |value| &.{ addr, value } else &.{addr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.trace.mark`. Result types are explicit; attributes use mlir.Attribute. +pub fn trace_mark(ctx: *mlir.Context, payload: ?*const mlir.Value, predicate: ?*const mlir.Value, eventType: *const mlir.Attribute, domain: *const mlir.Attribute, event: *const mlir.Attribute, payloadDescriptor: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "eventType", eventType)); + attributes.appendAssumeCapacity(.named(ctx, "domain", domain)); + attributes.appendAssumeCapacity(.named(ctx, "event", event)); + if (payloadDescriptor) |value| attributes.appendAssumeCapacity(.named(ctx, "payloadDescriptor", value)); + return mlir.Operation.make(ctx, "nvvm.trace.mark", .{ + .operands = .{ .variadic = &.{ + if (payload) |value| &.{value} else &.{}, + if (predicate) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.vote.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn vote_sync(ctx: *mlir.Context, mask: *const mlir.Value, pred: *const mlir.Value, res_type: *const mlir.Type, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.vote.sync", .{ + .operands = .{ .flat = &.{ mask, pred } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "kind", kind), + }, + .location = location, + }); +} + +/// `nvvm.wgmma.commit.group.sync.aligned`. Result types are explicit; attributes use mlir.Attribute. +pub fn wgmma_commit_group_sync_aligned(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.wgmma.commit.group.sync.aligned", .{ + .location = location, + }); +} + +/// `nvvm.wgmma.fence.aligned`. Result types are explicit; attributes use mlir.Attribute. +pub fn wgmma_fence_aligned(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.wgmma.fence.aligned", .{ + .location = location, + }); +} + +/// `nvvm.wgmma.mma_async`. Result types are explicit; attributes use mlir.Attribute. +pub fn wgmma_mma_async(ctx: *mlir.Context, inouts: *const mlir.Value, descriptorA: *const mlir.Value, descriptorB: *const mlir.Value, results__type: *const mlir.Type, shape: *const mlir.Attribute, typeA: *const mlir.Attribute, typeB: *const mlir.Attribute, typeD: *const mlir.Attribute, scaleD: *const mlir.Attribute, scaleA: *const mlir.Attribute, scaleB: *const mlir.Attribute, layoutA: *const mlir.Attribute, layoutB: *const mlir.Attribute, satfinite: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 10) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + attributes.appendAssumeCapacity(.named(ctx, "typeA", typeA)); + attributes.appendAssumeCapacity(.named(ctx, "typeB", typeB)); + attributes.appendAssumeCapacity(.named(ctx, "typeD", typeD)); + attributes.appendAssumeCapacity(.named(ctx, "scaleD", scaleD)); + attributes.appendAssumeCapacity(.named(ctx, "scaleA", scaleA)); + attributes.appendAssumeCapacity(.named(ctx, "scaleB", scaleB)); + attributes.appendAssumeCapacity(.named(ctx, "layoutA", layoutA)); + attributes.appendAssumeCapacity(.named(ctx, "layoutB", layoutB)); + if (satfinite) |value| attributes.appendAssumeCapacity(.named(ctx, "satfinite", value)); + return mlir.Operation.make(ctx, "nvvm.wgmma.mma_async", .{ + .operands = .{ .flat = &.{ inouts, descriptorA, descriptorB } }, + .results = .{ .flat = &.{results__type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.wgmma.wait.group.sync.aligned`. Result types are explicit; attributes use mlir.Attribute. +pub fn wgmma_wait_group_sync_aligned(ctx: *mlir.Context, group: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.wgmma.wait.group.sync.aligned", .{ + .attributes = &.{ + .named(ctx, "group", group), + }, + .location = location, + }); +} + +/// `nvvm.wmma.load`. Result types are explicit; attributes use mlir.Attribute. +pub fn wmma_load(ctx: *mlir.Context, ptr: *const mlir.Value, stride: *const mlir.Value, res_type: *const mlir.Type, m: *const mlir.Attribute, n: *const mlir.Attribute, k: *const mlir.Attribute, layout: *const mlir.Attribute, eltype: *const mlir.Attribute, frag: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.wmma.load", .{ + .operands = .{ .flat = &.{ ptr, stride } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "m", m), + .named(ctx, "n", n), + .named(ctx, "k", k), + .named(ctx, "layout", layout), + .named(ctx, "eltype", eltype), + .named(ctx, "frag", frag), + }, + .location = location, + }); +} + +/// `nvvm.wmma.mma`. Result types are explicit; attributes use mlir.Attribute. +pub fn wmma_mma(ctx: *mlir.Context, args: []const *const mlir.Value, res_type: *const mlir.Type, m: *const mlir.Attribute, n: *const mlir.Attribute, k: *const mlir.Attribute, layoutA: *const mlir.Attribute, layoutB: *const mlir.Attribute, eltypeA: *const mlir.Attribute, eltypeB: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.wmma.mma", .{ + .operands = .{ .flat = args }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "m", m), + .named(ctx, "n", n), + .named(ctx, "k", k), + .named(ctx, "layoutA", layoutA), + .named(ctx, "layoutB", layoutB), + .named(ctx, "eltypeA", eltypeA), + .named(ctx, "eltypeB", eltypeB), + }, + .location = location, + }); +} + +/// `nvvm.wmma.store`. Result types are explicit; attributes use mlir.Attribute. +pub fn wmma_store(ctx: *mlir.Context, ptr: *const mlir.Value, args: []const *const mlir.Value, stride: *const mlir.Value, m: *const mlir.Attribute, n: *const mlir.Attribute, k: *const mlir.Attribute, layout: *const mlir.Attribute, eltype: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 256) = .empty; + operands.appendAssumeCapacity(ptr); + operands.appendSliceAssumeCapacity(args); + operands.appendAssumeCapacity(stride); + return mlir.Operation.make(ctx, "nvvm.wmma.store", .{ + .operands = .{ .flat = operands.constSlice() }, + .attributes = &.{ + .named(ctx, "m", m), + .named(ctx, "n", n), + .named(ctx, "k", k), + .named(ctx, "layout", layout), + .named(ctx, "eltype", eltype), + }, + .location = location, + }); +} + +test "every enum attribute prints as the compiler does" { + @setEvalBranchQuota(1_000_000); + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + mlir.DialectHandle.fromString(dialect_handle).insertDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + inline for (.{ + .{ AtomicOpKindAttr, "#nvvm" }, + .{ BarrierReductionAttr, "#nvvm.reduction<{s}>" }, + .{ BarrierReduxKindAttr, "#nvvm.barrier_redux_kind<{s}>" }, + .{ BlockScaleFormatAttr, "#nvvm.block_scale_format<{s}>" }, + .{ CTAGroupKindAttr, "#nvvm.cta_group<{s}>" }, + .{ CVTPackFloatKindAttr, "#nvvm.packfloat_type<{s}>" }, + .{ CacheEvictionPriorityAttr, "#nvvm" }, + .{ ClusterLaunchControlQueryTypeAttr, "#nvvm" }, + .{ CompareOpKindAttr, "#nvvm.op<{s}>" }, + .{ ConvertFP4TypeAttr, "#nvvm.convert_fp4_type<{s}>" }, + .{ ConvertFP8TypeAttr, "#nvvm.convert_fp8_type<{s}>" }, + .{ ConvertScaleKindAttr, "#nvvm.convert_scale_kind<{s}>" }, + .{ DotAccumulateTypeAttr, "#nvvm.dot_accumulate_type<{s}>" }, + .{ EvictKindAttr, "#nvvm.evict_kind<{s}>" }, + .{ FPRoundingModeAttr, "#nvvm.fp_rnd_mode<{s}>" }, + .{ GridDepActionKindAttr, "#nvvm" }, + .{ IntegerRoundingModeAttr, "#nvvm.int_rnd_mode<{s}>" }, + .{ L2PrefetchSizeAttr, "#nvvm.l2_prefetch<{s}>" }, + .{ LdStMatrixEltTypeAttr, "#nvvm.ld_st_matrix_elt_type<{s}>" }, + .{ LoadCacheModifierExtKindAttr, "#nvvm.load_cache_modifier_ext<{s}>" }, + .{ LoadCacheModifierKindAttr, "#nvvm" }, + .{ LoadShapeAttr, "#nvvm.load_shape<{s}>" }, + .{ LoadSrcFormatAttr, "#nvvm.load_src_format<{s}>" }, + .{ MBarrierLayoutAttr, "#nvvm.mbarrier_layout<{s}>" }, + .{ MBarrierPhaseAttr, "#nvvm.mbarrier_phase<{s}>" }, + .{ MBarrierScopeKindAttr, "#nvvm.mbar_scope<{s}>" }, + .{ MBarrierSpaceKindAttr, "#nvvm.mbar_space<{s}>" }, + .{ MBarrierTxnKindAttr, "#nvvm.mbar_txn_kind<{s}>" }, + .{ MBarrierWaitKindAttr, "#nvvm.mbar_wait<{s}>" }, + .{ MMAB1OpAttr, "#nvvm.mma_b1op<{s}>" }, + .{ MMABlockScaleKindAttr, "#nvvm.block_scale_kind<{s}>" }, + .{ MMACtaCountAttr, "#nvvm.mma_cta_count<{s}>" }, + .{ MMAFragAttr, "#nvvm.mma_frag<{s}>" }, + .{ MMAIntOverflowAttr, "#nvvm.mma_int_overflow<{s}>" }, + .{ MMAKindAttr, "#nvvm.mma_kind<{s}>" }, + .{ MMALayoutAttr, "#nvvm.mma_layout<{s}>" }, + .{ MMATypesAttr, "#nvvm.mma_type<{s}>" }, + .{ MatchSyncKindAttr, "#nvvm" }, + .{ MemOrderKindAttr, "#nvvm.mem_order<{s}>" }, + .{ MemScopeKindAttr, "#nvvm.mem_scope<{s}>" }, + .{ MulModeAttr, "#nvvm" }, + .{ NVVMMemorySpaceAttr, "#nvvm.memory_space<{s}>" }, + .{ PermuteModeAttr, "#nvvm.permute_mode<{s}>" }, + .{ PrefetchCacheLevelAttr, "#nvvm" }, + .{ ProxyKindAttr, "#nvvm.proxy_kind<{s}>" }, + .{ ReductionKindAttr, "#nvvm" }, + .{ ReductionOpAttr, "#nvvm.red_op<{s}>" }, + .{ ReductionTypeAttr, "#nvvm.red_type<{s}>" }, + .{ SPCompressElemSizeAttr, "#nvvm.spcomp_elem_size<{s}>" }, + .{ SPCompressFactorTypeAttr, "#nvvm.spcomp_factor<{s}>" }, + .{ SPCompressIndexSizeAttr, "#nvvm.spcomp_index_size<{s}>" }, + .{ SPCompressOpKindAttr, "#nvvm.spcomp_op_kind<{s}>" }, + .{ SPCompressRepFactorAttr, "#nvvm.spcomp_rep_factor<{s}>" }, + .{ SPDecompressElemSizeAttr, "#nvvm.spdecomp_elem_size<{s}>" }, + .{ SPDecompressFactorTypeAttr, "#nvvm.spdecomp_factor<{s}>" }, + .{ SPDecompressIndexSizeAttr, "#nvvm.spdecomp_index_size<{s}>" }, + .{ SPDecompressRepFactorAttr, "#nvvm.spdecomp_rep_factor<{s}>" }, + .{ SaturationModeAttr, "#nvvm.sat_mode<{s}>" }, + .{ SaturationModeKindAttr, "#nvvm.sat<{s}>" }, + .{ ScaleVecSizeAttr, "#nvvm.scale_vec_size<{s}>" }, + .{ SetMaxRegisterActionAttr, "#nvvm" }, + .{ SharedSpaceAttr, "#nvvm.shared_space<{s}>" }, + .{ ShflKindAttr, "#nvvm" }, + .{ SparsityFormatAttr, "#nvvm.sparsity_format<{s}>" }, + .{ StateSpaceAttr, "#nvvm.state_space<{s}>" }, + .{ StoreCacheModifierKindAttr, "#nvvm.store_cache_modifier<{s}>" }, + .{ StoreShapeAttr, "#nvvm.store_shape<{s}>" }, + .{ TCBarParamAttr, "#nvvm.TCBarParam<{s}>" }, + .{ TMALoadModeAttr, "#nvvm.tma_load_mode<{s}>" }, + .{ TMAReduxKindAttr, "#nvvm.tma_redux_kind<{s}>" }, + .{ TMAStoreModeAttr, "#nvvm.tma_store_mode<{s}>" }, + .{ Tcgen05CpMulticastAttr, "#nvvm.tcgen05_cp_multicast<{s}>" }, + .{ Tcgen05CpShapeAttr, "#nvvm.tcgen05_cp_shape<{s}>" }, + .{ Tcgen05CpSrcFormatAttr, "#nvvm.tcgen05_cp_src_fmt<{s}>" }, + .{ Tcgen05FenceKindAttr, "#nvvm.tcgen05_fence<{s}>" }, + .{ Tcgen05LdStShapeAttr, "#nvvm.tcgen05_ldst_shape<{s}>" }, + .{ Tcgen05MMABlockScaleAttr, "#nvvm.tcgen05_mma_block_scale<{s}>" }, + .{ Tcgen05MMACollectorBBufferAttr, "#nvvm.tcgen05_mma_collectorb<{s}>" }, + .{ Tcgen05MMACollectorOpAttr, "#nvvm.tcgen05_mma_collectorop<{s}>" }, + .{ Tcgen05MMAKindAttr, "#nvvm.tcgen05_mma_kind<{s}>" }, + .{ Tcgen05WaitKindAttr, "#nvvm.tcgen05_wait<{s}>" }, + .{ TensormapElemtypeAttr, "#nvvm.tensormap_elemtype<{s}>" }, + .{ TensormapFieldAttr, "#nvvm" }, + .{ TensormapFillModeAttr, "#nvvm.tensormap_fill_mode<{s}>" }, + .{ TensormapInterleaveLayoutAttr, "#nvvm.tensormap_interleave_layout<{s}>" }, + .{ TensormapSwizzleAtomicityAttr, "#nvvm.tensormap_swizzle_atomicity<{s}>" }, + .{ TensormapSwizzleModeAttr, "#nvvm.tensormap_swizzle_mode<{s}>" }, + .{ TmemLayoutAttr, "#nvvm.TmemLayout<{s}>" }, + .{ ValidatePatternAttr, "#nvvm.validate_pattern<{s}>" }, + .{ VoteSyncKindAttr, "#nvvm" }, + .{ WGMMAScaleInAttr, "#nvvm.wgmma_scale_in<{s}>" }, + .{ WGMMAScaleOutAttr, "#nvvm.wgmma_scale_out<{s}>" }, + .{ WGMMATypesAttr, "#nvvm.wgmma_type<{s}>" }, + }) |example| { + const T = example[0]; + inline for (@typeInfo(@FieldType(T.InitArgs, "value")).@"enum".fields) |case| { + const attr = try T.get(ctx, .{ .value = @enumFromInt(case.value) }); + try std.testing.expectEqual(case.value, @intFromEnum(attr.getValue())); + var buf: [256]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{attr}); + try std.testing.expectEqualStrings(std.fmt.comptimePrint(example[1], .{case.name}), w.buffered()); + } + } +} + +/// Every operation bound here. +pub const operation_names: []const []const u8 = &.{ + "nvvm.add.packed.bf16x2.f32x2.f32x2", + "nvvm.add.packed.f16x2.f32x2.f32x2", + "nvvm.add.packed.f32x2", + "nvvm.add.packed.f32x2.bf16x2.f32x2", + "nvvm.add.packed.f32x2.f16x2.f32x2", + "nvvm.addf", + "nvvm.applypriority.async.bulk", + "nvvm.applypriority.async.bulk.tensor", + "nvvm.applypriority.async.bulk.tensor.override", + "nvvm.atomicrmw", + "nvvm.bar.warp.sync", + "nvvm.barrier", + "nvvm.barrier.arrive", + "nvvm.barrier.cta.arrive", + "nvvm.barrier.cta.red", + "nvvm.barrier.cta.sync", + "nvvm.barrier0", + "nvvm.breakpoint", + "nvvm.clmad", + "nvvm.cluster.arrive", + "nvvm.cluster.arrive.relaxed", + "nvvm.cluster.wait", + "nvvm.clusterlaunchcontrol.query.cancel", + "nvvm.clusterlaunchcontrol.try.cancel", + "nvvm.compare.and.set", + "nvvm.convert.and.pack.integer", + "nvvm.convert.bf16x2.to.f4x2", + "nvvm.convert.bf16x2.to.f6x2", + "nvvm.convert.bf16x2.to.f8x2", + "nvvm.convert.bf16x2.to.s2f6x2", + "nvvm.convert.f16x2.to.f4x2", + "nvvm.convert.f16x2.to.f6x2", + "nvvm.convert.f16x2.to.f8x2", + "nvvm.convert.f32x2.to.bf16x2", + "nvvm.convert.f32x2.to.f16x2", + "nvvm.convert.f32x2.to.f4x2", + "nvvm.convert.f32x2.to.f6x2", + "nvvm.convert.f32x2.to.f8x2", + "nvvm.convert.f32x2.to.s2f6x2", + "nvvm.convert.f32x4.to.f4x4", + "nvvm.convert.f32x4.to.f6x4", + "nvvm.convert.f32x4.to.f8x4", + "nvvm.convert.f4x2.to.f16x2", + "nvvm.convert.f6x2.to.f16x2", + "nvvm.convert.f8x2.to.bf16x2", + "nvvm.convert.f8x2.to.f16x2", + "nvvm.convert.float.to.integer", + "nvvm.convert.float.to.tf32", + "nvvm.convert.s2f6x2.to.bf16x2", + "nvvm.cos", + "nvvm.cp.async.bulk.commit.group", + "nvvm.cp.async.bulk.global.shared.cta", + "nvvm.cp.async.bulk.prefetch", + "nvvm.cp.async.bulk.shared.cluster.global", + "nvvm.cp.async.bulk.shared.cluster.shared.cta", + "nvvm.cp.async.bulk.tensor.global.shared.cta", + "nvvm.cp.async.bulk.tensor.global.shared.cta.override", + "nvvm.cp.async.bulk.tensor.prefetch", + "nvvm.cp.async.bulk.tensor.prefetch.override", + "nvvm.cp.async.bulk.tensor.reduce", + "nvvm.cp.async.bulk.tensor.reduce.override", + "nvvm.cp.async.bulk.tensor.shared.cluster.global", + "nvvm.cp.async.bulk.tensor.shared.cluster.global.override", + "nvvm.cp.async.bulk.wait_group", + "nvvm.cp.async.commit.group", + "nvvm.cp.async.mbarrier.arrive", + "nvvm.cp.async.shared.global", + "nvvm.cp.async.wait.group", + "nvvm.cvt.packfloat", + "nvvm.cvt.packfloat.f32", + "nvvm.cvt.to.f4x8.packed", + "nvvm.dot.accumulate.2way", + "nvvm.dot.accumulate.4way", + "nvvm.elect.sync", + "nvvm.ex2", + "nvvm.exit", + "nvvm.fabs", + "nvvm.fence.mbarrier.init", + "nvvm.fence.proxy", + "nvvm.fence.proxy.acquire", + "nvvm.fence.proxy.release", + "nvvm.fence.proxy.sync_restrict", + "nvvm.fence.sc.cluster", + "nvvm.fence.sync_restrict", + "nvvm.fma", + "nvvm.fma.packed.f32x2", + "nvvm.fma.packed.f32x2.bf16x2.f32x2.f32x2", + "nvvm.fma.packed.f32x2.f16x2.f32x2.f32x2", + "nvvm.fmax", + "nvvm.fmin", + "nvvm.griddepcontrol", + "nvvm.inline_ptx", + "nvvm.ldmatrix", + "nvvm.load.ext", + "nvvm.log2", + "nvvm.mapa", + "nvvm.match.sync", + "nvvm.mbarrier.arrive", + "nvvm.mbarrier.arrive.expect_tx", + "nvvm.mbarrier.arrive.nocomplete", + "nvvm.mbarrier.arrive_drop", + "nvvm.mbarrier.arrive_drop.expect_tx", + "nvvm.mbarrier.arrive_drop.nocomplete", + "nvvm.mbarrier.check.layout", + "nvvm.mbarrier.complete_tx", + "nvvm.mbarrier.expect_tx", + "nvvm.mbarrier.init", + "nvvm.mbarrier.inval", + "nvvm.mbarrier.test.wait", + "nvvm.mbarrier.try_wait", + "nvvm.mbarrier.try_wait.parity", + "nvvm.mbarrier.try_wait.parity.timelimit", + "nvvm.mbarrier.try_wait.timelimit", + "nvvm.mbarrier.txn", + "nvvm.mbarrier.txn.cta", + "nvvm.mbarrier.wait", + "nvvm.mbarrier.wait.parity", + "nvvm.memory.barrier", + "nvvm.mma.block_scale", + "nvvm.mma.block_scale.internal", + "nvvm.mma.sp.block_scale", + "nvvm.mma.sp.block_scale.internal", + "nvvm.mma.sp.sync", + "nvvm.mma.sync", + "nvvm.mma_smem_desc", + "nvvm.movmatrix", + "nvvm.mul", + "nvvm.mul.packed.bf16x2.bf16x2.f16x2", + "nvvm.mul.packed.bf16x2.f32x2.f32x2", + "nvvm.mul.packed.f16x2.f16x2.bf16x2", + "nvvm.mul.packed.f16x2.f32x2.f32x2", + "nvvm.mul.packed.f32x2", + "nvvm.multimem.ld.reduce", + "nvvm.multimem.red", + "nvvm.multimem.st", + "nvvm.nanosleep", + "nvvm.pmevent", + "nvvm.prefetch", + "nvvm.prmt", + "nvvm.rcp.approx.ftz.f", + "nvvm.read.ptx.sreg.aggr.smem.size", + "nvvm.read.ptx.sreg.clock", + "nvvm.read.ptx.sreg.clock64", + "nvvm.read.ptx.sreg.cluster.ctaid.x", + "nvvm.read.ptx.sreg.cluster.ctaid.y", + "nvvm.read.ptx.sreg.cluster.ctaid.z", + "nvvm.read.ptx.sreg.cluster.ctarank", + "nvvm.read.ptx.sreg.cluster.nctaid.x", + "nvvm.read.ptx.sreg.cluster.nctaid.y", + "nvvm.read.ptx.sreg.cluster.nctaid.z", + "nvvm.read.ptx.sreg.cluster.nctarank", + "nvvm.read.ptx.sreg.clusterid.x", + "nvvm.read.ptx.sreg.clusterid.y", + "nvvm.read.ptx.sreg.clusterid.z", + "nvvm.read.ptx.sreg.ctaid.x", + "nvvm.read.ptx.sreg.ctaid.y", + "nvvm.read.ptx.sreg.ctaid.z", + "nvvm.read.ptx.sreg.dynamic.smem.size", + "nvvm.read.ptx.sreg.envreg0", + "nvvm.read.ptx.sreg.envreg1", + "nvvm.read.ptx.sreg.envreg10", + "nvvm.read.ptx.sreg.envreg11", + "nvvm.read.ptx.sreg.envreg12", + "nvvm.read.ptx.sreg.envreg13", + "nvvm.read.ptx.sreg.envreg14", + "nvvm.read.ptx.sreg.envreg15", + "nvvm.read.ptx.sreg.envreg16", + "nvvm.read.ptx.sreg.envreg17", + "nvvm.read.ptx.sreg.envreg18", + "nvvm.read.ptx.sreg.envreg19", + "nvvm.read.ptx.sreg.envreg2", + "nvvm.read.ptx.sreg.envreg20", + "nvvm.read.ptx.sreg.envreg21", + "nvvm.read.ptx.sreg.envreg22", + "nvvm.read.ptx.sreg.envreg23", + "nvvm.read.ptx.sreg.envreg24", + "nvvm.read.ptx.sreg.envreg25", + "nvvm.read.ptx.sreg.envreg26", + "nvvm.read.ptx.sreg.envreg27", + "nvvm.read.ptx.sreg.envreg28", + "nvvm.read.ptx.sreg.envreg29", + "nvvm.read.ptx.sreg.envreg3", + "nvvm.read.ptx.sreg.envreg30", + "nvvm.read.ptx.sreg.envreg31", + "nvvm.read.ptx.sreg.envreg4", + "nvvm.read.ptx.sreg.envreg5", + "nvvm.read.ptx.sreg.envreg6", + "nvvm.read.ptx.sreg.envreg7", + "nvvm.read.ptx.sreg.envreg8", + "nvvm.read.ptx.sreg.envreg9", + "nvvm.read.ptx.sreg.globaltimer", + "nvvm.read.ptx.sreg.globaltimer.lo", + "nvvm.read.ptx.sreg.gridid", + "nvvm.read.ptx.sreg.laneid", + "nvvm.read.ptx.sreg.lanemask.eq", + "nvvm.read.ptx.sreg.lanemask.ge", + "nvvm.read.ptx.sreg.lanemask.gt", + "nvvm.read.ptx.sreg.lanemask.le", + "nvvm.read.ptx.sreg.lanemask.lt", + "nvvm.read.ptx.sreg.nclusterid.x", + "nvvm.read.ptx.sreg.nclusterid.y", + "nvvm.read.ptx.sreg.nclusterid.z", + "nvvm.read.ptx.sreg.nctaid.x", + "nvvm.read.ptx.sreg.nctaid.y", + "nvvm.read.ptx.sreg.nctaid.z", + "nvvm.read.ptx.sreg.nsmid", + "nvvm.read.ptx.sreg.ntid.x", + "nvvm.read.ptx.sreg.ntid.y", + "nvvm.read.ptx.sreg.ntid.z", + "nvvm.read.ptx.sreg.nwarpid", + "nvvm.read.ptx.sreg.smid", + "nvvm.read.ptx.sreg.tid.x", + "nvvm.read.ptx.sreg.tid.y", + "nvvm.read.ptx.sreg.tid.z", + "nvvm.read.ptx.sreg.total.smem.size", + "nvvm.read.ptx.sreg.warpid", + "nvvm.read.ptx.sreg.warpsize", + "nvvm.red", + "nvvm.redux.sync", + "nvvm.rsqrt", + "nvvm.setmaxregister", + "nvvm.shfl.sync", + "nvvm.sin", + "nvvm.spcompress", + "nvvm.spdecompress", + "nvvm.st.bulk", + "nvvm.stmatrix", + "nvvm.store.async", + "nvvm.store.ext", + "nvvm.sub.packed.bf16x2.f32x2.f32x2", + "nvvm.sub.packed.f16x2.f32x2.f32x2", + "nvvm.sub.packed.f32x2", + "nvvm.sub.packed.f32x2.bf16x2.f32x2", + "nvvm.sub.packed.f32x2.f16x2.f32x2", + "nvvm.subf", + "nvvm.tcgen05.alloc", + "nvvm.tcgen05.commit", + "nvvm.tcgen05.cp", + "nvvm.tcgen05.dealloc", + "nvvm.tcgen05.fence", + "nvvm.tcgen05.ld", + "nvvm.tcgen05.ld.red", + "nvvm.tcgen05.ld.red.spcompress", + "nvvm.tcgen05.ld.spcompress", + "nvvm.tcgen05.mma", + "nvvm.tcgen05.mma.block_scale", + "nvvm.tcgen05.mma.block_scale.decompress_b", + "nvvm.tcgen05.mma.decompress_b", + "nvvm.tcgen05.mma.sp", + "nvvm.tcgen05.mma.sp.block_scale", + "nvvm.tcgen05.mma.ws", + "nvvm.tcgen05.mma.ws.sp", + "nvvm.tcgen05.mma_smem_desc", + "nvvm.tcgen05.relinquish_alloc_permit", + "nvvm.tcgen05.shift", + "nvvm.tcgen05.st", + "nvvm.tcgen05.wait", + "nvvm.tcgen05_mma_smem_desc_v2", + "nvvm.tensormap.cp_fenceproxy", + "nvvm.tensormap.replace", + "nvvm.trace.mark", + "nvvm.vote.sync", + "nvvm.wgmma.commit.group.sync.aligned", + "nvvm.wgmma.fence.aligned", + "nvvm.wgmma.mma_async", + "nvvm.wgmma.wait.group.sync.aligned", + "nvvm.wmma.load", + "nvvm.wmma.mma", + "nvvm.wmma.store", +}; diff --git a/mlir/dialects/fly/BUILD.bazel b/mlir/dialects/fly/BUILD.bazel index 8f9963df..24adcc45 100644 --- a/mlir/dialects/fly/BUILD.bazel +++ b/mlir/dialects/fly/BUILD.bazel @@ -13,7 +13,6 @@ cc_library( "@flydsl//:FlyROCDLDialect", "@flydsl//:TiledOpTraits", "@llvm-project//mlir:CAPIIR", - "@llvm-project//mlir:GPUDialect", "@llvm-project//mlir:UBDialect", ], alwayslink = True, @@ -21,19 +20,22 @@ cc_library( zig_library( name = "fly", - tags = ["manual"], srcs = [ "attributes.zig", + "ops.zig", "rocdl.zig", + "rocdl_ops.zig", "types.zig", ], import_name = "mlir/dialects/fly", main = "fly.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ ":fly_capi", "//mlir", "//mlir:c", + "//mlir/dialects:capi", "//stdx", ], ) diff --git a/mlir/dialects/fly/REGENERATING.md b/mlir/dialects/fly/REGENERATING.md new file mode 100644 index 00000000..163842d6 --- /dev/null +++ b/mlir/dialects/fly/REGENERATING.md @@ -0,0 +1,84 @@ +# Regenerating the operation builders + +These are instructions for an agent. They describe what `ops.zig` and +`rocdl_ops.zig` must contain after a FlyDSL bump, where to derive them from, +and how to check the result. Scripts and intermediate files are not +committed; only the generated builders are. + +Paths below are relative to this directory. + +## Goal + +`ops.zig` binds every operation of the `fly` dialect and `rocdl_ops.zig` every +operation of the `fly_rocdl` dialect, one builder per operation, so that no +code builds a FlyDSL operation from its name. The attributes and types +(`attributes.zig`, `types.zig`, `rocdl.zig`, `fly_capi.*`) are maintained by +hand and are not covered here. + +## Sources + +The FlyDSL commit is the one `third_party/flydsl/repo.bzl` pins, with its +patches applied: read the `.td` files from the fetched repository +(`$(bazel info output_base)/external/+non_module_deps+flydsl`), never from an +unpatched checkout. + +- `include/flydsl/Dialect/Fly/IR/FlyOps.td` for `ops.zig`. +- `include/flydsl/Dialect/FlyROCDL/IR/Ops.td` for `rocdl_ops.zig`. + +Dump their records with the LLVM the workspace builds: + +```sh +bazel build @llvm-project//llvm:llvm-tblgen +llvm-tblgen --dump-json \ + -I /include -I /external/+llvm+llvm-project/mlir/include \ + /include/flydsl/Dialect/Fly/IR/FlyOps.td -o FlyOps.json +``` + +Every record in `!instanceof.Op` whose `opDialect` names the dialect is an +operation. Do not regex the `.td` files. + +## What each builder must be + +- One `pub fn` per operation, named by its mnemonic with dots replaced by + underscores (`fly.tiled_copy.partition_src` → `tiled_copy_partition_src`), + sorted by mnemonic, with a doc comment naming the operation, its `summary`, + and whether result types are inferred, explicit, or absent. +- Parameters, in order: `ctx`, the operands, the result types (only when the + operation does not implement `InferTypeOpInterface`), the attributes, then + `location`. Names are the ODS names in snake case, with a trailing + underscore when they collide with a Zig keyword, a builder name or a + reserved local (`ctx`, `location`, `make`, `operands`, `attributes`, + `present`). +- Operands: single → `*const mlir.Value`, `Optional` → `?*const mlir.Value`, + `Variadic` → `[]const *const mlir.Value`. Operations with + `AttrSizedOperandSegments` pass every operand group through + `Operation.make`'s `.variadic` operands, which writes `operandSegmentSizes`; + others pass a flat list (an optional operand may only be last). +- Results: single → `*const mlir.Type`, `Variadic` → + `[]const *const mlir.Type` named `_types`. Inferred operations set + `.result_type_inference = true` and take no result types. Inference comes + from the flattened trait list (trait lists such as `Pure` expand to their + members); an `OpInterface` record whose `cppInterfaceName` is + `InferTypeOpInterface` marks it. +- Attributes: `*const mlir.Attribute`, optional (`?`) when the attribute is + `OptionalAttr`, `UnitAttr` or has a default value; set under its ODS name. +- Builders call `make` from `fly.zig`, which verifies the operation and names + the operand types when creation fails. +- Each file starts with a header naming the FlyDSL commit and the `.td` it + was generated from, and exports `names`, the list of operation names, for + the registration test. + +Regions, successors and `VariadicOfVariadic` operands have no use in either +dialect today; when one appears, extend the generator and this file. + +## Checks + +- `zig fmt` the generated files. +- `bazel test //mlir/dialects/fly:test //kernels/fly:test`: every name in + `ops.names` and `rocdl.ops.names` must be a registered operation. +- Callers that no longer compile are the point of generating: an operation + whose signature changed in FlyDSL now fails at the call site. Fix them in + the same change (`kernels/fly/builder.zig`, `zml/*/fly_kernels`). +- Emit the IR of the fly kernels before and after (for instance + `sparse_mla.Main2D.emit` and its siblings with representative configs) and + diff it: a regeneration without an upstream change must be byte-identical. diff --git a/mlir/dialects/fly/fly.zig b/mlir/dialects/fly/fly.zig index 4f4e9c07..97571450 100644 --- a/mlir/dialects/fly/fly.zig +++ b/mlir/dialects/fly/fly.zig @@ -5,6 +5,7 @@ const mlir = @import("mlir"); const stdx = @import("stdx"); pub const attributes = @import("attributes.zig"); +pub const ops = @import("ops.zig"); pub const rocdl = @import("rocdl.zig"); pub const types = @import("types.zig"); @@ -22,19 +23,6 @@ pub fn insertDialects(registry: *mlir.DialectRegistry) void { } } -pub fn opName(comptime mnemonic: []const u8) []const u8 { - return "fly." ++ mnemonic; -} - -/// By value, so a helper's temporary outlives the slice `Operation.make` gets. -pub const Attrs = stdx.BoundedArray(mlir.NamedAttribute, 8); - -pub fn attrs(list: []const mlir.NamedAttribute) Attrs { - var out: Attrs = .empty; - out.appendSliceAssumeCapacity(list); - return out; -} - /// `Operation.make` that names the op and its operand types on failure: a /// rejected `inferReturnTypes` otherwise panics with nothing to go on. pub fn make(ctx: *mlir.Context, name: []const u8, args: mlir.Operation.MakeArgs) *mlir.Operation { @@ -59,44 +47,6 @@ pub fn make(ctx: *mlir.Context, name: []const u8, args: mlir.Operation.MakeArgs) }; } -/// Result type inferred by FlyDSL. -pub fn inferred(ctx: *mlir.Context, comptime mnemonic: []const u8, operands: []const *const mlir.Value, attributes_: Attrs, location: *const mlir.Location) *mlir.Operation { - return make(ctx, opName(mnemonic), .{ - .operands = .{ .flat = operands }, - .result_type_inference = true, - .attributes = attributes_.constSlice(), - .location = location, - }); -} - -/// Explicit result types. -pub fn typed(ctx: *mlir.Context, comptime mnemonic: []const u8, operands: []const *const mlir.Value, results: []const *const mlir.Type, attributes_: Attrs, location: *const mlir.Location) *mlir.Operation { - return make(ctx, opName(mnemonic), .{ - .operands = .{ .flat = operands }, - .results = .{ .flat = results }, - .attributes = attributes_.constSlice(), - .location = location, - }); -} - -/// No results. -pub fn effect(ctx: *mlir.Context, comptime mnemonic: []const u8, operands: []const *const mlir.Value, attributes_: Attrs, location: *const mlir.Location) *mlir.Operation { - return make(ctx, opName(mnemonic), .{ - .operands = .{ .flat = operands }, - .attributes = attributes_.constSlice(), - .location = location, - }); -} - -/// Panics with the text on failure. -pub fn parseType(ctx: *mlir.Context, text: []const u8) *const mlir.Type { - return mlir.Type.parse(ctx, text) catch std.debug.panic("fly: cannot parse type `{s}`", .{text}); -} - -pub fn parseAttr(ctx: *mlir.Context, text: []const u8) *const mlir.Attribute { - return mlir.Attribute.parse(ctx, text) catch std.debug.panic("fly: cannot parse attribute `{s}`", .{text}); -} - pub const TypeKind = enum { memref, coord_tensor, @@ -177,81 +127,9 @@ pub fn ptrWithElem(ctx: *mlir.Context, ptr: *const mlir.Type, elem: *const mlir. }); } -/// A value whose whole content lives in its type. -pub fn static(ctx: *mlir.Context, ty: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return typed(ctx, "static", &.{}, &.{ty}, .empty, location); -} - -/// `ty` names the tuple with a `?` per dynamic operand, in order. -pub fn makeIntTuple(ctx: *mlir.Context, dynamic: []const *const mlir.Value, ty: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return typed(ctx, "make_int_tuple", dynamic, &.{ty}, .empty, location); -} - -pub fn makeCopyAtom(ctx: *mlir.Context, ty: *const mlir.Type, val_bits: i32, location: *const mlir.Location) *mlir.Operation { - return typed(ctx, "make_copy_atom", &.{}, &.{ty}, attrs(&.{.named(ctx, "valBits", .int(ctx, .i32, val_bits))}), location); -} - -pub fn makeMmaAtom(ctx: *mlir.Context, ty: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return typed(ctx, "make_mma_atom", &.{}, &.{ty}, .empty, location); -} - -pub fn makeFragmentLike(ctx: *mlir.Context, src: *const mlir.Value, dtype: ?*const mlir.Type, location: *const mlir.Location) *mlir.Operation { - var a: Attrs = .empty; - if (dtype) |t| a.appendAssumeCapacity(.named(ctx, "dtype", .typeAttr(t))); - return inferred(ctx, "make_fragment_like", &.{src}, a, location); -} - -pub const CopyOpts = struct { - pred: ?*const mlir.Value = null, -}; - -pub fn copy(ctx: *mlir.Context, atom: *const mlir.Value, src: *const mlir.Value, dst: *const mlir.Value, opts: CopyOpts, location: *const mlir.Location) *mlir.Operation { - if (opts.pred) |p| return effect(ctx, "copy", &.{ atom, src, dst, p }, .empty, location); - return effect(ctx, "copy", &.{ atom, src, dst }, .empty, location); -} - -pub const GemmOpts = struct { - traversal_order: ?GemmTraversalOrder = null, - traversal_layout: ?*const mlir.Value = null, -}; - -pub fn gemm(ctx: *mlir.Context, atom: *const mlir.Value, d: *const mlir.Value, a: *const mlir.Value, b: *const mlir.Value, cc: *const mlir.Value, opts: GemmOpts, location: *const mlir.Location) *mlir.Operation { - var at = attrs(&.{.named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ - 1, 1, 1, 1, 1, @intFromBool(opts.traversal_layout != null), - }))}); - if (opts.traversal_order) |o| { - const attr = attributes.gemmTraversalOrderAttr(ctx, o) catch @panic("fly.gemm: invalid traversal order"); - at.appendAssumeCapacity(.named(ctx, "traversalOrder", attr)); - } - if (opts.traversal_layout) |l| return effect(ctx, "gemm", &.{ atom, d, a, b, cc, l }, at, location); - return effect(ctx, "gemm", &.{ atom, d, a, b, cc }, at, location); -} - -pub fn get(ctx: *mlir.Context, input: *const mlir.Value, mode: []const i32, location: *const mlir.Location) *mlir.Operation { - return inferred(ctx, "get", &.{input}, attrs(&.{.named(ctx, "mode", .denseArray(ctx, .i32, mode))}), location); -} - -pub fn select(ctx: *mlir.Context, input: *const mlir.Value, indices: []const i32, location: *const mlir.Location) *mlir.Operation { - return inferred(ctx, "select", &.{input}, attrs(&.{.named(ctx, "indices", .denseArray(ctx, .i32, indices))}), location); -} - -pub fn takeOrGroup(ctx: *mlir.Context, comptime mnemonic: []const u8, input: *const mlir.Value, begin: i32, end: i32, location: *const mlir.Location) *mlir.Operation { - return inferred(ctx, mnemonic, &.{input}, attrs(&.{ - .named(ctx, "begin", .int(ctx, .i32, begin)), - .named(ctx, "end", .int(ctx, .i32, end)), - }), location); -} - -/// Explicit pointer type; `fly.make_ptr` infers nothing. -pub fn makePtr(ctx: *mlir.Context, operands: []const *const mlir.Value, ty: *const mlir.Type, dict_attrs: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - var a: Attrs = .empty; - if (dict_attrs) |d| a.appendAssumeCapacity(.named(ctx, "dictAttrs", d)); - return typed(ctx, "make_ptr", operands, &.{ty}, a, location); -} - test { std.testing.refAllDecls(@This()); - inline for (.{ types, attributes, rocdl }) |module| { + inline for (.{ types, attributes, ops, rocdl, rocdl.ops }) |module| { std.testing.refAllDecls(module); inline for (comptime std.meta.declarations(module)) |decl| { const value = @field(module, decl.name); @@ -289,6 +167,18 @@ test "fly dialects register" { try std.testing.expect(ctx.isRegisteredOperation("ub.poison")); } +test "every generated operation builder names a registered operation" { + const ctx = try testContext(); + defer ctx.deinit(); + + for (ops.names ++ rocdl.ops.names) |name| { + if (!ctx.isRegisteredOperation(name)) { + std.debug.print("`{s}` is not registered\n", .{name}); + return error.TestUnexpectedResult; + } + } +} + test "fly types parse and print round-trip" { const ctx = try testContext(); defer ctx.deinit(); @@ -375,12 +265,11 @@ test "types and attributes are built through the C API" { })); } -test "fly enum and gpu attributes" { +test "fly enum attributes" { const ctx = try testContext(); defer ctx.deinit(); _ = try attributes.mmaOperandAttr(ctx, .a); _ = try attributes.gemmTraversalOrderAttr(ctx, .kmn); - _ = parseAttr(ctx, "#gpu>"); } test "inferred fly ops compute layout algebra" { @@ -392,11 +281,11 @@ test "inferred fly ops compute layout algebra" { const block = module.body(); // raked_product((8,16):(16,1), (1,4):(1,1)) — the vectorAdd TV layout. - const thr = static(ctx, parseType(ctx, "!fly.layout<(8,16):(16,1)>"), loc).appendTo(block); - const val = static(ctx, parseType(ctx, "!fly.layout<(1,4):(1,1)>"), loc).appendTo(block); - const mn = inferred(ctx, "raked_product", &.{ thr.result(0), val.result(0) }, .empty, loc).appendTo(block); - const shape = inferred(ctx, "get_shape", &.{mn.result(0)}, .empty, loc).appendTo(block); - const tiler = inferred(ctx, "int_tuple_product_each", &.{shape.result(0)}, .empty, loc).appendTo(block); + const thr = ops.static(ctx, try mlir.Type.parse(ctx, "!fly.layout<(8,16):(16,1)>"), loc).appendTo(block); + const val = ops.static(ctx, try mlir.Type.parse(ctx, "!fly.layout<(1,4):(1,1)>"), loc).appendTo(block); + const mn = ops.raked_product(ctx, thr.result(0), val.result(0), loc).appendTo(block); + const shape = ops.get_shape(ctx, mn.result(0), loc).appendTo(block); + const tiler = ops.int_tuple_product_each(ctx, shape.result(0), loc).appendTo(block); try expectPrints("!fly.int_tuple<(8,64)>", tiler.result(0).type_()); try std.testing.expect(module.operation().verify()); @@ -407,7 +296,7 @@ test "atom traits" { defer ctx.deinit(); // MFMA 16x16x4 f32 tiled over a (2,2,1) atom layout. - const tm = expect(parseType(ctx, "!fly.tiled_mma f32>>, <(2,2,1):(1,2,0)>>"), .tiled_mma); + const tm = expect(try mlir.Type.parse(ctx, "!fly.tiled_mma f32>>, <(2,2,1):(1,2,0)>>"), .tiled_mma); try expectPrints("!fly.int_tuple<(32,32,4)>", tm.getTileSizeMNK()); _ = tm.getTiledThrValLayoutA(); _ = tm.getThrLayoutVMNK(); @@ -434,7 +323,7 @@ test "atom traits" { })).type_() }); try expectPrints("!fly.layout<32:1>", wmma.getThrLayout()); - const ca = expect(parseType(ctx, "!fly.copy_atom, 32>"), .copy_atom); + const ca = expect(try mlir.Type.parse(ctx, "!fly.copy_atom, 32>"), .copy_atom); _ = ca.getThrValLayoutSrc(); } @@ -451,7 +340,7 @@ test "type kinds" { .{ "!fly.mma_atom f32>>", TypeKind.mma_atom }, }; inline for (cases) |case| { - try std.testing.expectEqual(case[1], typeKind(parseType(ctx, case[0]))); + try std.testing.expectEqual(case[1], typeKind(try mlir.Type.parse(ctx, case[0]))); } try std.testing.expectEqual(TypeKind.other, typeKind(.float(ctx, .f32))); } @@ -459,7 +348,7 @@ test "type kinds" { test "structural readers" { const ctx = try testContext(); defer ctx.deinit(); - const it = expect(parseType(ctx, "!fly.int_tuple<((2,4),?)>"), .int_tuple); + const it = expect(try mlir.Type.parse(ctx, "!fly.int_tuple<((2,4),?)>"), .int_tuple); try std.testing.expectEqual(@as(usize, 2), it.getNumElements()); try std.testing.expect(!it.isLeaf()); try std.testing.expect(!it.isStatic()); @@ -468,7 +357,7 @@ test "structural readers" { try std.testing.expectEqual(types.IntTupleType.Leaf{ .static = 4 }, inner.getElement(1).getLeaf()); try std.testing.expectEqual(types.IntTupleType.Leaf.dynamic, it.getElement(1).getLeaf()); - const m = parseType(ctx, "!fly.memref o 0 o (64,32):(32,1), align<16>>"); + const m = try mlir.Type.parse(ctx, "!fly.memref o 0 o (64,32):(32,1), align<16>>"); try expectPrints("!fly.int_tuple<(64,32)>", layoutLikeShape(m)); try expectPrints("bf16", elemType(m)); } diff --git a/mlir/dialects/fly/fly_capi.cc b/mlir/dialects/fly/fly_capi.cc index fe28ff98..0f2f3c11 100644 --- a/mlir/dialects/fly/fly_capi.cc +++ b/mlir/dialects/fly/fly_capi.cc @@ -5,14 +5,12 @@ #include "flydsl/Dialect/FlyROCDL/IR/Dialect.h" #include "mlir/CAPI/IR.h" #include "mlir/CAPI/Registration.h" -#include "mlir/Dialect/GPU/IR/GPUDialect.h" #include "mlir/Dialect/UB/IR/UBOps.h" MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Fly, fly, mlir::fly::FlyDialect) MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(FlyROCDL, fly_rocdl, mlir::fly_rocdl::FlyROCDLDialect) MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(UB, ub, mlir::ub::UBDialect) -MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(GPU, gpu, mlir::gpu::GPUDialect) using mlir::cast; using mlir::isa; diff --git a/mlir/dialects/fly/fly_capi.h b/mlir/dialects/fly/fly_capi.h index bf92ed47..6a0e3f39 100644 --- a/mlir/dialects/fly/fly_capi.h +++ b/mlir/dialects/fly/fly_capi.h @@ -14,7 +14,6 @@ extern "C" { MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Fly, fly); MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(FlyROCDL, fly_rocdl); MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(UB, ub); -MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(GPU, gpu); MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyInt(MlirAttribute attr); MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntAttrGetStatic(MlirContext ctx, int32_t value); diff --git a/mlir/dialects/fly/ops.zig b/mlir/dialects/fly/ops.zig new file mode 100644 index 00000000..7ac4adba --- /dev/null +++ b/mlir/dialects/fly/ops.zig @@ -0,0 +1,1146 @@ +//! `fly` operation builders, generated from FlyDSL's `include/flydsl/Dialect/Fly/IR/FlyOps.td` at +//! 1941889400621f1fc3d8f03bada1a4e7380cdf7e; do not edit, see REGENERATING.md. +//! +//! Builders take the operands, then the result types (only for operations +//! that do not infer them), then the attributes, then the location. +//! Parameters are the ODS names in snake case, with a trailing underscore +//! where needed. Pass `null` for absent optional operands or attributes and +//! `&.{}` for empty variadic ones; variable-length operand groups of +//! `AttrSizedOperandSegments` operations get their segment sizes from +//! `Operation.make`. Attributes use `mlir.Attribute`. + +const std = @import("std"); + +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +const make = @import("fly.zig").make; + +/// Every operation bound here, for the registration test. +pub const names = [_][]const u8{ + "fly.add_offset", + "fly.append", + "fly.apply_swizzle", + "fly.assume", + "fly.atom.set_value", + "fly.blocked_product", + "fly.ceil_div", + "fly.coalesce", + "fly.complement", + "fly.composed_get_inner", + "fly.composed_get_offset", + "fly.composed_get_outer", + "fly.composition", + "fly.coprofile", + "fly.copy", + "fly.copy_atom_call", + "fly.copy_atom_call_ssa", + "fly.coshape", + "fly.cosize", + "fly.crd2idx", + "fly.decomposition", + "fly.dice", + "fly.elem_less", + "fly.equal", + "fly.extract_aligned_pointer_as_index", + "fly.flat_divide", + "fly.flat_product", + "fly.gemm", + "fly.get", + "fly.get_1d_coord", + "fly.get_copy_atom", + "fly.get_dyn_shared", + "fly.get_flat_coord", + "fly.get_iter", + "fly.get_layout", + "fly.get_leaves", + "fly.get_mma_atom", + "fly.get_scalar", + "fly.get_shape", + "fly.get_stride", + "fly.group", + "fly.idx2crd", + "fly.int_tuple_add", + "fly.int_tuple_div", + "fly.int_tuple_mod", + "fly.int_tuple_mul", + "fly.int_tuple_product", + "fly.int_tuple_product_each", + "fly.int_tuple_product_like", + "fly.int_tuple_sub", + "fly.inttoptr", + "fly.left_inverse", + "fly.logical_divide", + "fly.logical_product", + "fly.make_composed_layout", + "fly.make_coord", + "fly.make_copy_atom", + "fly.make_fragment_layout_like", + "fly.make_fragment_like", + "fly.make_identity_layout", + "fly.make_int_tuple", + "fly.make_layout", + "fly.make_layout_like", + "fly.make_mma_atom", + "fly.make_ordered_layout", + "fly.make_ptr", + "fly.make_shape", + "fly.make_stride", + "fly.make_tiled_copy", + "fly.make_tiled_mma", + "fly.make_view", + "fly.memref.alloca", + "fly.memref.load", + "fly.memref.load_vec", + "fly.memref.store", + "fly.memref.store_vec", + "fly.mma.make_fragment", + "fly.mma_atom_call", + "fly.mma_atom_call_ssa", + "fly.prepend", + "fly.print", + "fly.ptr.load", + "fly.ptr.store", + "fly.ptrtoint", + "fly.raked_product", + "fly.recast_iter", + "fly.recast_layout", + "fly.right_inverse", + "fly.select", + "fly.shape_div", + "fly.size", + "fly.slice", + "fly.static", + "fly.take", + "fly.tile_to_shape", + "fly.tiled_copy.partition_dst", + "fly.tiled_copy.partition_src", + "fly.tiled_copy.retile", + "fly.tiled_divide", + "fly.tiled_mma.partition", + "fly.tiled_mma.partition_shape", + "fly.tiled_product", + "fly.to_llvm_ptr", + "fly.zipped_divide", + "fly.zipped_product", +}; + +/// `fly.add_offset`. Result types are inferred. +pub fn add_offset(ctx: *mlir.Context, ptr: *const mlir.Value, offset: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.add_offset", .{ + .operands = .{ .flat = &.{ ptr, offset } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.append`. Result types are inferred. +pub fn append(ctx: *mlir.Context, tuple: *const mlir.Value, elem: *const mlir.Value, n: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (n) |present| attributes.appendAssumeCapacity(.named(ctx, "n", present)); + return make(ctx, "fly.append", .{ + .operands = .{ .flat = &.{ tuple, elem } }, + .attributes = attributes.constSlice(), + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.apply_swizzle`. Result types are inferred. +pub fn apply_swizzle(ctx: *mlir.Context, ptr: *const mlir.Value, swizzle: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.apply_swizzle", .{ + .operands = .{ .flat = &.{ ptr, swizzle } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.assume`. Result types are explicit. +pub fn assume(ctx: *mlir.Context, dst: *const mlir.Value, src: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.assume", .{ + .operands = .{ .flat = &.{ dst, src } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.atom.set_value`. Result types are inferred. +pub fn atom_set_value(ctx: *mlir.Context, atom: *const mlir.Value, value: *const mlir.Value, field: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.atom.set_value", .{ + .operands = .{ .flat = &.{ atom, value } }, + .attributes = &.{ + .named(ctx, "field", field), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.blocked_product`. Result types are inferred. +pub fn blocked_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.blocked_product", .{ + .operands = .{ .flat = &.{ layout, tile } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.ceil_div`. Result types are inferred. +pub fn ceil_div(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.ceil_div", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.coalesce`. Result types are inferred. +pub fn coalesce(ctx: *mlir.Context, layout: *const mlir.Value, pattern: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.coalesce", .{ + .operands = .{ .flat = if (pattern) |present| &.{ layout, present } else &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.complement`. Result types are inferred. +pub fn complement(ctx: *mlir.Context, layout: *const mlir.Value, codomain_size: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.complement", .{ + .operands = .{ .flat = if (codomain_size) |present| &.{ layout, present } else &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.composed_get_inner`. Result types are inferred. +pub fn composed_get_inner(ctx: *mlir.Context, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.composed_get_inner", .{ + .operands = .{ .flat = &.{input} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.composed_get_offset`. Result types are inferred. +pub fn composed_get_offset(ctx: *mlir.Context, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.composed_get_offset", .{ + .operands = .{ .flat = &.{input} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.composed_get_outer`. Result types are inferred. +pub fn composed_get_outer(ctx: *mlir.Context, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.composed_get_outer", .{ + .operands = .{ .flat = &.{input} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.composition`. Result types are inferred. +pub fn composition(ctx: *mlir.Context, outer: *const mlir.Value, inner: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.composition", .{ + .operands = .{ .flat = &.{ outer, inner } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.coprofile`. Result types are inferred. +pub fn coprofile(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.coprofile", .{ + .operands = .{ .flat = &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.copy`. No results. +pub fn copy(ctx: *mlir.Context, copy_atom: *const mlir.Value, src: *const mlir.Value, dst: *const mlir.Value, pred: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.copy", .{ + .operands = .{ .flat = if (pred) |present| &.{ copy_atom, src, dst, present } else &.{ copy_atom, src, dst } }, + .location = location, + }); +} + +/// `fly.copy_atom_call`. No results. +pub fn copy_atom_call(ctx: *mlir.Context, copy_atom: *const mlir.Value, src: *const mlir.Value, dst: *const mlir.Value, pred: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.copy_atom_call", .{ + .operands = .{ .flat = if (pred) |present| &.{ copy_atom, src, dst, present } else &.{ copy_atom, src, dst } }, + .location = location, + }); +} + +/// `fly.copy_atom_call_ssa`. Result types are explicit. +pub fn copy_atom_call_ssa(ctx: *mlir.Context, copy_atom: *const mlir.Value, src: *const mlir.Value, dst: ?*const mlir.Value, pred: ?*const mlir.Value, results_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.copy_atom_call_ssa", .{ + .operands = .{ .variadic = &.{ + &.{copy_atom}, + &.{src}, + if (dst) |present| &.{present} else &.{}, + if (pred) |present| &.{present} else &.{}, + } }, + .results = .{ .flat = results_types }, + .location = location, + }); +} + +/// `fly.coshape`. Result types are inferred. +pub fn coshape(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.coshape", .{ + .operands = .{ .flat = &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.cosize`. Result types are inferred. +pub fn cosize(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.cosize", .{ + .operands = .{ .flat = &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.crd2idx`. Result types are inferred. +pub fn crd2idx(ctx: *mlir.Context, coord: *const mlir.Value, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.crd2idx", .{ + .operands = .{ .flat = &.{ coord, layout } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.decomposition`. Result types are inferred. +pub fn decomposition(ctx: *mlir.Context, tensor: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.decomposition", .{ + .operands = .{ .flat = &.{tensor} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.dice`. Result types are inferred. +pub fn dice(ctx: *mlir.Context, src: *const mlir.Value, coord: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.dice", .{ + .operands = .{ .flat = &.{ src, coord } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.elem_less`. Result types are explicit. +pub fn elem_less(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.elem_less", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.equal`. Result types are explicit. +pub fn equal(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.equal", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.extract_aligned_pointer_as_index`. Extract the raw pointer from a fly.memref. Result types are explicit. +pub fn extract_aligned_pointer_as_index(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.extract_aligned_pointer_as_index", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.flat_divide`. Result types are inferred. +pub fn flat_divide(ctx: *mlir.Context, layout: *const mlir.Value, divisor: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.flat_divide", .{ + .operands = .{ .flat = &.{ layout, divisor } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.flat_product`. Result types are inferred. +pub fn flat_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.flat_product", .{ + .operands = .{ .flat = &.{ layout, tile } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.gemm`. Fully expand tiled GEMM with variadic A and B operand groups. No results. +pub fn gemm(ctx: *mlir.Context, mma_atom: *const mlir.Value, d: *const mlir.Value, a: []const *const mlir.Value, b: []const *const mlir.Value, c: *const mlir.Value, traversal_layout: ?*const mlir.Value, traversal_order: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (traversal_order) |present| attributes.appendAssumeCapacity(.named(ctx, "traversalOrder", present)); + return make(ctx, "fly.gemm", .{ + .operands = .{ .variadic = &.{ + &.{mma_atom}, + &.{d}, + a, + b, + &.{c}, + if (traversal_layout) |present| &.{present} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `fly.get`. Result types are inferred. +pub fn get(ctx: *mlir.Context, input: *const mlir.Value, mode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get", .{ + .operands = .{ .flat = &.{input} }, + .attributes = &.{ + .named(ctx, "mode", mode), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_1d_coord`. Result types are inferred. +pub fn get_1d_coord(ctx: *mlir.Context, index: *const mlir.Value, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_1d_coord", .{ + .operands = .{ .flat = &.{ index, layout } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_copy_atom`. Extract the copy atom and its runtime state from a tiled copy. Result types are inferred. +pub fn get_copy_atom(ctx: *mlir.Context, tiled_copy: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_copy_atom", .{ + .operands = .{ .flat = &.{tiled_copy} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_dyn_shared`. Result types are inferred. +pub fn get_dyn_shared(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_dyn_shared", .{ + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_flat_coord`. Result types are inferred. +pub fn get_flat_coord(ctx: *mlir.Context, index: *const mlir.Value, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_flat_coord", .{ + .operands = .{ .flat = &.{ index, layout } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_iter`. Result types are inferred. +pub fn get_iter(ctx: *mlir.Context, memref: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_iter", .{ + .operands = .{ .flat = &.{memref} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_layout`. Result types are inferred. +pub fn get_layout(ctx: *mlir.Context, memref: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_layout", .{ + .operands = .{ .flat = &.{memref} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_leaves`. Result types are inferred. +pub fn get_leaves(ctx: *mlir.Context, input: *const mlir.Value, dynamic_only: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (dynamic_only) |present| attributes.appendAssumeCapacity(.named(ctx, "dynamicOnly", present)); + return make(ctx, "fly.get_leaves", .{ + .operands = .{ .flat = &.{input} }, + .attributes = attributes.constSlice(), + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_mma_atom`. Extract the MMA atom and its runtime state from a tiled MMA. Result types are inferred. +pub fn get_mma_atom(ctx: *mlir.Context, tiled_mma: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_mma_atom", .{ + .operands = .{ .flat = &.{tiled_mma} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_scalar`. Result types are inferred. +pub fn get_scalar(ctx: *mlir.Context, int_tuple: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_scalar", .{ + .operands = .{ .flat = &.{int_tuple} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_shape`. Result types are inferred. +pub fn get_shape(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_shape", .{ + .operands = .{ .flat = &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_stride`. Result types are inferred. +pub fn get_stride(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_stride", .{ + .operands = .{ .flat = &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.group`. Result types are inferred. +pub fn group(ctx: *mlir.Context, tuple: *const mlir.Value, begin: *const mlir.Attribute, end: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.group", .{ + .operands = .{ .flat = &.{tuple} }, + .attributes = &.{ + .named(ctx, "begin", begin), + .named(ctx, "end", end), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.idx2crd`. Result types are inferred. +pub fn idx2crd(ctx: *mlir.Context, coord: *const mlir.Value, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.idx2crd", .{ + .operands = .{ .flat = &.{ coord, layout } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.int_tuple_add`. Result types are inferred. +pub fn int_tuple_add(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.int_tuple_add", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.int_tuple_div`. Result types are inferred. +pub fn int_tuple_div(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.int_tuple_div", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.int_tuple_mod`. Result types are inferred. +pub fn int_tuple_mod(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.int_tuple_mod", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.int_tuple_mul`. Result types are inferred. +pub fn int_tuple_mul(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.int_tuple_mul", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.int_tuple_product`. Result types are inferred. +pub fn int_tuple_product(ctx: *mlir.Context, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.int_tuple_product", .{ + .operands = .{ .flat = &.{input} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.int_tuple_product_each`. Result types are inferred. +pub fn int_tuple_product_each(ctx: *mlir.Context, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.int_tuple_product_each", .{ + .operands = .{ .flat = &.{input} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.int_tuple_product_like`. Result types are inferred. +pub fn int_tuple_product_like(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.int_tuple_product_like", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.int_tuple_sub`. Result types are inferred. +pub fn int_tuple_sub(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.int_tuple_sub", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.inttoptr`. Result types are explicit. +pub fn inttoptr(ctx: *mlir.Context, src: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.inttoptr", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.left_inverse`. Result types are inferred. +pub fn left_inverse(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.left_inverse", .{ + .operands = .{ .flat = &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.logical_divide`. Result types are inferred. +pub fn logical_divide(ctx: *mlir.Context, layout: *const mlir.Value, divisor: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.logical_divide", .{ + .operands = .{ .flat = &.{ layout, divisor } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.logical_product`. Result types are inferred. +pub fn logical_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.logical_product", .{ + .operands = .{ .flat = &.{ layout, tile } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_composed_layout`. Result types are inferred. +pub fn make_composed_layout(ctx: *mlir.Context, inner: *const mlir.Value, offset: *const mlir.Value, outer: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_composed_layout", .{ + .operands = .{ .flat = &.{ inner, offset, outer } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_coord`. Result types are explicit. +pub fn make_coord(ctx: *mlir.Context, dync_elems: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_coord", .{ + .operands = .{ .flat = dync_elems }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.make_copy_atom`. Result types are explicit. +pub fn make_copy_atom(ctx: *mlir.Context, args: []const *const mlir.Value, result_type: *const mlir.Type, val_bits: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_copy_atom", .{ + .operands = .{ .flat = args }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "valBits", val_bits), + }, + .location = location, + }); +} + +/// `fly.make_fragment_layout_like`. Result types are inferred. +pub fn make_fragment_layout_like(ctx: *mlir.Context, src: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_fragment_layout_like", .{ + .operands = .{ .flat = &.{src} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_fragment_like`. Result types are inferred. +pub fn make_fragment_like(ctx: *mlir.Context, src: *const mlir.Value, dtype: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (dtype) |present| attributes.appendAssumeCapacity(.named(ctx, "dtype", present)); + return make(ctx, "fly.make_fragment_like", .{ + .operands = .{ .flat = &.{src} }, + .attributes = attributes.constSlice(), + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_identity_layout`. Result types are inferred. +pub fn make_identity_layout(ctx: *mlir.Context, shape: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_identity_layout", .{ + .operands = .{ .flat = &.{shape} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_int_tuple`. Result types are explicit. +pub fn make_int_tuple(ctx: *mlir.Context, dync_elems: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_int_tuple", .{ + .operands = .{ .flat = dync_elems }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.make_layout`. Result types are inferred. +pub fn make_layout(ctx: *mlir.Context, shape: *const mlir.Value, stride: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_layout", .{ + .operands = .{ .flat = &.{ shape, stride } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_layout_like`. Result types are inferred. +pub fn make_layout_like(ctx: *mlir.Context, ref: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_layout_like", .{ + .operands = .{ .flat = &.{ref} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_mma_atom`. Result types are explicit. +pub fn make_mma_atom(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_mma_atom", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.make_ordered_layout`. Result types are inferred. +pub fn make_ordered_layout(ctx: *mlir.Context, shape: *const mlir.Value, order: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_ordered_layout", .{ + .operands = .{ .flat = &.{ shape, order } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_ptr`. Result types are explicit. +pub fn make_ptr(ctx: *mlir.Context, args: []const *const mlir.Value, result_type: *const mlir.Type, dict_attrs: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (dict_attrs) |present| attributes.appendAssumeCapacity(.named(ctx, "dictAttrs", present)); + return make(ctx, "fly.make_ptr", .{ + .operands = .{ .flat = args }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `fly.make_shape`. Result types are explicit. +pub fn make_shape(ctx: *mlir.Context, dync_elems: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_shape", .{ + .operands = .{ .flat = dync_elems }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.make_stride`. Result types are explicit. +pub fn make_stride(ctx: *mlir.Context, dync_elems: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_stride", .{ + .operands = .{ .flat = dync_elems }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.make_tiled_copy`. Result types are inferred. +pub fn make_tiled_copy(ctx: *mlir.Context, copy_atom: *const mlir.Value, layout_thr_val: *const mlir.Value, tile_mn: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_tiled_copy", .{ + .operands = .{ .flat = &.{ copy_atom, layout_thr_val, tile_mn } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_tiled_mma`. Result types are inferred. +pub fn make_tiled_mma(ctx: *mlir.Context, mma_atom: *const mlir.Value, atom_layout: *const mlir.Value, permutation: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_tiled_mma", .{ + .operands = .{ .flat = if (permutation) |present| &.{ mma_atom, atom_layout, present } else &.{ mma_atom, atom_layout } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_view`. Result types are inferred. +pub fn make_view(ctx: *mlir.Context, iter: *const mlir.Value, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_view", .{ + .operands = .{ .flat = &.{ iter, layout } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.memref.alloca`. Result types are explicit. +pub fn memref_alloca(ctx: *mlir.Context, layout: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.memref.alloca", .{ + .operands = .{ .flat = &.{layout} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.memref.load`. Result types are inferred. +pub fn memref_load(ctx: *mlir.Context, memref: *const mlir.Value, indices: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.memref.load", .{ + .operands = .{ .flat = &.{ memref, indices } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.memref.load_vec`. Result types are inferred. +pub fn memref_load_vec(ctx: *mlir.Context, memref: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.memref.load_vec", .{ + .operands = .{ .flat = &.{memref} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.memref.store`. No results. +pub fn memref_store(ctx: *mlir.Context, value: *const mlir.Value, memref: *const mlir.Value, indices: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.memref.store", .{ + .operands = .{ .flat = &.{ value, memref, indices } }, + .location = location, + }); +} + +/// `fly.memref.store_vec`. No results. +pub fn memref_store_vec(ctx: *mlir.Context, vector: *const mlir.Value, memref: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.memref.store_vec", .{ + .operands = .{ .flat = &.{ vector, memref } }, + .location = location, + }); +} + +/// `fly.mma.make_fragment`. Result types are inferred. +pub fn mma_make_fragment(ctx: *mlir.Context, tiled_mma: *const mlir.Value, input: *const mlir.Value, operand_id: *const mlir.Attribute, stages: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "operand_id", operand_id)); + if (stages) |present| attributes.appendAssumeCapacity(.named(ctx, "stages", present)); + return make(ctx, "fly.mma.make_fragment", .{ + .operands = .{ .flat = &.{ tiled_mma, input } }, + .attributes = attributes.constSlice(), + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.mma_atom_call`. No results. +pub fn mma_atom_call(ctx: *mlir.Context, mma_atom: *const mlir.Value, d: *const mlir.Value, a: []const *const mlir.Value, b: []const *const mlir.Value, c: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.mma_atom_call", .{ + .operands = .{ .variadic = &.{ + &.{mma_atom}, + &.{d}, + a, + b, + &.{c}, + } }, + .location = location, + }); +} + +/// `fly.mma_atom_call_ssa`. Result types are explicit. +pub fn mma_atom_call_ssa(ctx: *mlir.Context, mma_atom: *const mlir.Value, d: ?*const mlir.Value, a: []const *const mlir.Value, b: []const *const mlir.Value, c: *const mlir.Value, results_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.mma_atom_call_ssa", .{ + .operands = .{ .variadic = &.{ + &.{mma_atom}, + if (d) |present| &.{present} else &.{}, + a, + b, + &.{c}, + } }, + .results = .{ .flat = results_types }, + .location = location, + }); +} + +/// `fly.prepend`. Result types are inferred. +pub fn prepend(ctx: *mlir.Context, tuple: *const mlir.Value, elem: *const mlir.Value, n: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (n) |present| attributes.appendAssumeCapacity(.named(ctx, "n", present)); + return make(ctx, "fly.prepend", .{ + .operands = .{ .flat = &.{ tuple, elem } }, + .attributes = attributes.constSlice(), + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.print`. No results. +pub fn print(ctx: *mlir.Context, values: []const *const mlir.Value, format: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.print", .{ + .operands = .{ .flat = values }, + .attributes = &.{ + .named(ctx, "format", format), + }, + .location = location, + }); +} + +/// `fly.ptr.load`. Result types are explicit. +pub fn ptr_load(ctx: *mlir.Context, ptr: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.ptr.load", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.ptr.store`. No results. +pub fn ptr_store(ctx: *mlir.Context, value: *const mlir.Value, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.ptr.store", .{ + .operands = .{ .flat = &.{ value, ptr } }, + .location = location, + }); +} + +/// `fly.ptrtoint`. Result types are inferred. +pub fn ptrtoint(ctx: *mlir.Context, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.ptrtoint", .{ + .operands = .{ .flat = &.{ptr} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.raked_product`. Result types are inferred. +pub fn raked_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.raked_product", .{ + .operands = .{ .flat = &.{ layout, tile } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.recast_iter`. Result types are explicit. +pub fn recast_iter(ctx: *mlir.Context, src: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.recast_iter", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.recast_layout`. Result types are inferred. +pub fn recast_layout(ctx: *mlir.Context, src: *const mlir.Value, new_type_bits: *const mlir.Attribute, old_type_bits: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.recast_layout", .{ + .operands = .{ .flat = &.{src} }, + .attributes = &.{ + .named(ctx, "new_type_bits", new_type_bits), + .named(ctx, "old_type_bits", old_type_bits), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.right_inverse`. Result types are inferred. +pub fn right_inverse(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.right_inverse", .{ + .operands = .{ .flat = &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.select`. Result types are inferred. +pub fn select(ctx: *mlir.Context, tuple: *const mlir.Value, indices: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.select", .{ + .operands = .{ .flat = &.{tuple} }, + .attributes = &.{ + .named(ctx, "indices", indices), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.shape_div`. Result types are inferred. +pub fn shape_div(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.shape_div", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.size`. Result types are inferred. +pub fn size(ctx: *mlir.Context, int_tuple: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.size", .{ + .operands = .{ .flat = &.{int_tuple} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.slice`. Result types are inferred. +pub fn slice(ctx: *mlir.Context, src: *const mlir.Value, coord: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.slice", .{ + .operands = .{ .flat = &.{ src, coord } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.static`. Result types are explicit. +pub fn static(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.static", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.take`. Result types are inferred. +pub fn take(ctx: *mlir.Context, tuple: *const mlir.Value, begin: *const mlir.Attribute, end: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.take", .{ + .operands = .{ .flat = &.{tuple} }, + .attributes = &.{ + .named(ctx, "begin", begin), + .named(ctx, "end", end), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.tile_to_shape`. Result types are inferred. +pub fn tile_to_shape(ctx: *mlir.Context, block: *const mlir.Value, trg_shape: *const mlir.Value, ord_shape: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.tile_to_shape", .{ + .operands = .{ .flat = &.{ block, trg_shape, ord_shape } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.tiled_copy.partition_dst`. Result types are inferred. +pub fn tiled_copy_partition_dst(ctx: *mlir.Context, tiled_copy: *const mlir.Value, dst: *const mlir.Value, coord: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.tiled_copy.partition_dst", .{ + .operands = .{ .flat = &.{ tiled_copy, dst, coord } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.tiled_copy.partition_src`. Result types are inferred. +pub fn tiled_copy_partition_src(ctx: *mlir.Context, tiled_copy: *const mlir.Value, src: *const mlir.Value, coord: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.tiled_copy.partition_src", .{ + .operands = .{ .flat = &.{ tiled_copy, src, coord } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.tiled_copy.retile`. Result types are inferred. +pub fn tiled_copy_retile(ctx: *mlir.Context, tiled_copy: *const mlir.Value, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.tiled_copy.retile", .{ + .operands = .{ .flat = &.{ tiled_copy, input } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.tiled_divide`. Result types are inferred. +pub fn tiled_divide(ctx: *mlir.Context, layout: *const mlir.Value, divisor: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.tiled_divide", .{ + .operands = .{ .flat = &.{ layout, divisor } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.tiled_mma.partition`. Result types are inferred. +pub fn tiled_mma_partition(ctx: *mlir.Context, tiled_mma: *const mlir.Value, input: *const mlir.Value, coord: *const mlir.Value, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.tiled_mma.partition", .{ + .operands = .{ .flat = &.{ tiled_mma, input, coord } }, + .attributes = &.{ + .named(ctx, "operand_id", operand_id), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.tiled_mma.partition_shape`. Result types are inferred. +pub fn tiled_mma_partition_shape(ctx: *mlir.Context, tiled_mma: *const mlir.Value, shape: *const mlir.Value, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.tiled_mma.partition_shape", .{ + .operands = .{ .flat = &.{ tiled_mma, shape } }, + .attributes = &.{ + .named(ctx, "operand_id", operand_id), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.tiled_product`. Result types are inferred. +pub fn tiled_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.tiled_product", .{ + .operands = .{ .flat = &.{ layout, tile } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.to_llvm_ptr`. Result types are inferred. +pub fn to_llvm_ptr(ctx: *mlir.Context, ptr: *const mlir.Value, llvm_address_space: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.to_llvm_ptr", .{ + .operands = .{ .flat = &.{ptr} }, + .attributes = &.{ + .named(ctx, "llvm_address_space", llvm_address_space), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.zipped_divide`. Result types are inferred. +pub fn zipped_divide(ctx: *mlir.Context, layout: *const mlir.Value, divisor: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.zipped_divide", .{ + .operands = .{ .flat = &.{ layout, divisor } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.zipped_product`. Result types are inferred. +pub fn zipped_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.zipped_product", .{ + .operands = .{ .flat = &.{ layout, tile } }, + .result_type_inference = true, + .location = location, + }); +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/mlir/dialects/fly/rocdl.zig b/mlir/dialects/fly/rocdl.zig index a1a19f23..fa8a8a56 100644 --- a/mlir/dialects/fly/rocdl.zig +++ b/mlir/dialects/fly/rocdl.zig @@ -3,6 +3,8 @@ const std = @import("std"); const c = @import("c"); const mlir = @import("mlir"); +pub const ops = @import("rocdl_ops.zig"); + /// `#fly_rocdl.buffer_desc`, an address space for buffer-descriptor pointers. pub const BufferDescAddressAttr = opaque { const Self = @This(); diff --git a/mlir/dialects/fly/rocdl_ops.zig b/mlir/dialects/fly/rocdl_ops.zig new file mode 100644 index 00000000..4a8a53cb --- /dev/null +++ b/mlir/dialects/fly/rocdl_ops.zig @@ -0,0 +1,69 @@ +//! `fly_rocdl` operation builders, generated from FlyDSL's `include/flydsl/Dialect/FlyROCDL/IR/Ops.td` at +//! 1941889400621f1fc3d8f03bada1a4e7380cdf7e; do not edit, see REGENERATING.md. +//! +//! Builders take the operands, then the result types (only for operations +//! that do not infer them), then the attributes, then the location. +//! Parameters are the ODS names in snake case, with a trailing underscore +//! where needed. Pass `null` for absent optional operands or attributes and +//! `&.{}` for empty variadic ones; variable-length operand groups of +//! `AttrSizedOperandSegments` operations get their segment sizes from +//! `Operation.make`. Attributes use `mlir.Attribute`. + +const std = @import("std"); + +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +const make = @import("fly.zig").make; + +/// Every operation bound here, for the registration test. +pub const names = [_][]const u8{ + "fly_rocdl.get_buffer_rsrc", + "fly_rocdl.make_tiled_tdm_load_atom", + "fly_rocdl.make_tiled_tdm_store_atom", +}; + +/// `fly_rocdl.get_buffer_rsrc`. Extract the raw ROCDL buffer resource from a buffer-descriptor pointer. Result types are inferred. +pub fn get_buffer_rsrc(ctx: *mlir.Context, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly_rocdl.get_buffer_rsrc", .{ + .operands = .{ .flat = &.{ptr} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly_rocdl.make_tiled_tdm_load_atom`. Build a CDNA5 TDM load atom and the coordinate tensor that addresses it. Result types are inferred. +pub fn make_tiled_tdm_load_atom(ctx: *mlir.Context, tensor: *const mlir.Value, smem_layout: *const mlir.Value, tiler: *const mlir.Value, num_warps: ?*const mlir.Attribute, init_boundary_check: ?*const mlir.Attribute, cache_modifier: ?*const mlir.Attribute, atomic_barrier: ?*const mlir.Attribute, internal_type: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (num_warps) |present| attributes.appendAssumeCapacity(.named(ctx, "numWarps", present)); + if (init_boundary_check) |present| attributes.appendAssumeCapacity(.named(ctx, "initBoundaryCheck", present)); + if (cache_modifier) |present| attributes.appendAssumeCapacity(.named(ctx, "cacheModifier", present)); + if (atomic_barrier) |present| attributes.appendAssumeCapacity(.named(ctx, "atomicBarrier", present)); + if (internal_type) |present| attributes.appendAssumeCapacity(.named(ctx, "internalType", present)); + return make(ctx, "fly_rocdl.make_tiled_tdm_load_atom", .{ + .operands = .{ .flat = &.{ tensor, smem_layout, tiler } }, + .attributes = attributes.constSlice(), + .result_type_inference = true, + .location = location, + }); +} + +/// `fly_rocdl.make_tiled_tdm_store_atom`. Build a CDNA5 TDM store atom and the coordinate tensor that addresses it. Result types are inferred. +pub fn make_tiled_tdm_store_atom(ctx: *mlir.Context, tensor: *const mlir.Value, smem_layout: *const mlir.Value, tiler: *const mlir.Value, num_warps: ?*const mlir.Attribute, init_boundary_check: ?*const mlir.Attribute, cache_modifier: ?*const mlir.Attribute, atomic_barrier: ?*const mlir.Attribute, internal_type: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (num_warps) |present| attributes.appendAssumeCapacity(.named(ctx, "numWarps", present)); + if (init_boundary_check) |present| attributes.appendAssumeCapacity(.named(ctx, "initBoundaryCheck", present)); + if (cache_modifier) |present| attributes.appendAssumeCapacity(.named(ctx, "cacheModifier", present)); + if (atomic_barrier) |present| attributes.appendAssumeCapacity(.named(ctx, "atomicBarrier", present)); + if (internal_type) |present| attributes.appendAssumeCapacity(.named(ctx, "internalType", present)); + return make(ctx, "fly_rocdl.make_tiled_tdm_store_atom", .{ + .operands = .{ .flat = &.{ tensor, smem_layout, tiler } }, + .attributes = attributes.constSlice(), + .result_type_inference = true, + .location = location, + }); +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/mlir/dialects/gpu.zig b/mlir/dialects/gpu.zig index 3d690a83..d43687fc 100644 --- a/mlir/dialects/gpu.zig +++ b/mlir/dialects/gpu.zig @@ -1,9 +1,77 @@ const std = @import("std"); +const c = @import("c"); const mlir = @import("mlir"); const stdx = @import("stdx"); -pub const Dim = enum { x, y, z }; +/// A dimension, either `x`, `y` or `z`. +pub const Dim = enum(u32) { + x = 0, + y = 1, + z = 2, +}; + +/// Indexing modes supported by `gpu.shuffle`. +pub const ShuffleMode = enum(u32) { + xor = 0, + down = 1, + up = 2, + idx = 3, +}; + +/// `#gpu`. +pub const DimensionAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAGPUDimension; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "dim"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Dim, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirGPUDimensionAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) Dim { + return @enumFromInt(c.mlirGPUDimensionAttrGetValue(self.ptr())); + } +}; + +/// `#gpu`. +pub const ShuffleModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAGPUShuffleMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "shuffle_mode"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ShuffleMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirGPUShuffleModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) ShuffleMode { + return @enumFromInt(c.mlirGPUShuffleModeAttrGetValue(self.ptr())); + } +}; pub fn module(ctx: *mlir.Context, name: []const u8, body: *mlir.Block, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "gpu.module", .{ @@ -50,16 +118,11 @@ pub fn return_(ctx: *mlir.Context, values: []const *const mlir.Value, location: }); } -fn dimAttr(ctx: *mlir.Context, dim: Dim) *const mlir.Attribute { - return switch (dim) { - inline else => |d| mlir.Attribute.parse(ctx, "#gpu>") catch unreachable, - }; -} - fn idOp(ctx: *mlir.Context, comptime name: []const u8, dim: Dim, location: *const mlir.Location) *mlir.Operation { + const dimension = DimensionAttr.get(ctx, .{ .value = dim }) catch unreachable; return mlir.Operation.make(ctx, name, .{ .results = .{ .flat = &.{.index(ctx)} }, - .attributes = &.{.named(ctx, "dimension", dimAttr(ctx, dim))}, + .attributes = &.{.named(ctx, "dimension", dimension.attribute())}, .location = location, }); } @@ -80,6 +143,18 @@ pub fn grid_dim(ctx: *mlir.Context, dim: Dim, location: *const mlir.Location) *m return idOp(ctx, "gpu.grid_dim", dim, location); } +/// `gpu.shuffle`: `value` from lane `offset` (per `mode`) within `width` lanes, +/// and whether that lane was valid (i1). +pub fn shuffle(ctx: *mlir.Context, value: *const mlir.Value, offset: *const mlir.Value, width: *const mlir.Value, mode: ShuffleMode, location: *const mlir.Location) *mlir.Operation { + const mode_attr = ShuffleModeAttr.get(ctx, .{ .value = mode }) catch unreachable; + return mlir.Operation.make(ctx, "gpu.shuffle", .{ + .operands = .{ .flat = &.{ value, offset, width } }, + .results = .{ .flat = &.{ value.type_(), .int(ctx, .i1) } }, + .attributes = &.{.named(ctx, "mode", mode_attr.attribute())}, + .location = location, + }); +} + pub fn barrier(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "gpu.barrier", .{ .location = location }); } @@ -87,3 +162,29 @@ pub fn barrier(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operati test { std.testing.refAllDecls(@This()); } + +fn expectPrints(expected: []const u8, value: anytype) !void { + var buf: [128]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(expected, w.buffered()); +} + +test "gpu enum attributes are built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + inline for (std.meta.fields(Dim)) |field| { + const value: Dim = @enumFromInt(field.value); + const attr = try DimensionAttr.get(ctx, .{ .value = value }); + try std.testing.expectEqual(value, attr.getValue()); + try std.testing.expect(attr.attribute().isA(DimensionAttr) != null); + try expectPrints("#gpu.dim<" ++ field.name ++ ">", attr); + } + inline for (std.meta.fields(ShuffleMode)) |field| { + const value: ShuffleMode = @enumFromInt(field.value); + const attr = try ShuffleModeAttr.get(ctx, .{ .value = value }); + try std.testing.expectEqual(value, attr.getValue()); + try expectPrints("#gpu.shuffle_mode<" ++ field.name ++ ">", attr); + } +} diff --git a/mlir/dialects/gpu_capi.cc b/mlir/dialects/gpu_capi.cc new file mode 100644 index 00000000..2e8bd4a9 --- /dev/null +++ b/mlir/dialects/gpu_capi.cc @@ -0,0 +1,47 @@ +#include "gpu_capi.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" +#include "mlir/Dialect/GPU/IR/GPUDialect.h" + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(GPU, gpu, mlir::gpu::GPUDialect) + +namespace { + +mlir::MLIRContext *loadGPU(MlirContext ctx) { + mlir::MLIRContext *context = unwrap(ctx); + if (context) context->getOrLoadDialect(); + return context; +} + +} // namespace + +bool mlirAttributeIsAGPUDimension(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirGPUDimensionAttrGet(MlirContext ctx, uint32_t value) { + mlir::MLIRContext *context = loadGPU(ctx); + std::optional dimension = mlir::gpu::symbolizeDimension(value); + if (!context || !dimension) return {nullptr}; + return wrap(mlir::gpu::DimensionAttr::get(context, *dimension)); +} + +uint32_t mlirGPUDimensionAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAGPUShuffleMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirGPUShuffleModeAttrGet(MlirContext ctx, uint32_t value) { + mlir::MLIRContext *context = loadGPU(ctx); + std::optional mode = mlir::gpu::symbolizeShuffleMode(value); + if (!context || !mode) return {nullptr}; + return wrap(mlir::gpu::ShuffleModeAttr::get(context, *mode)); +} + +uint32_t mlirGPUShuffleModeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} diff --git a/mlir/dialects/gpu_capi.h b/mlir/dialects/gpu_capi.h new file mode 100644 index 00000000..033042e0 --- /dev/null +++ b/mlir/dialects/gpu_capi.h @@ -0,0 +1,35 @@ +// C entry points for the `gpu` dialect that upstream's C API leaves out: its +// registration (outside `CAPIGPU`, which also brings the GPU passes) and the +// enum attributes kernels build. +#ifndef ZML_MLIR_DIALECTS_GPU_CAPI_H +#define ZML_MLIR_DIALECTS_GPU_CAPI_H + +#include +#include + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(GPU, gpu); + +// Enum attributes take and return the C++ enum's integer value; `Get` returns +// a null attribute for a value outside the enum. + +// #gpu +MLIR_CAPI_EXPORTED bool mlirAttributeIsAGPUDimension(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirGPUDimensionAttrGet(MlirContext ctx, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirGPUDimensionAttrGetValue(MlirAttribute attr); + +// #gpu +MLIR_CAPI_EXPORTED bool mlirAttributeIsAGPUShuffleMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirGPUShuffleModeAttrGet(MlirContext ctx, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirGPUShuffleModeAttrGetValue(MlirAttribute attr); + +#ifdef __cplusplus +} +#endif + +#endif // ZML_MLIR_DIALECTS_GPU_CAPI_H diff --git a/mlir/dialects/llvm_dialect.zig b/mlir/dialects/llvm_dialect.zig index 2f5d3091..2b7e904d 100644 --- a/mlir/dialects/llvm_dialect.zig +++ b/mlir/dialects/llvm_dialect.zig @@ -1,5 +1,6 @@ const std = @import("std"); +const c = @import("c"); const mlir = @import("mlir"); // ============================================================================= @@ -19,16 +20,10 @@ pub const NVPTXAddressSpace = enum(u32) { shared_cluster = 7, }; -/// `!llvm.ptr`, an opaque pointer. -/// -/// Parsed rather than built with `mlirLLVMPointerTypeGet`: the type must come from the -/// LLVM dialect registered in `ctx`, which may be another MLIR build than the C API's -/// (e.g. in the CuTe compiler context), and loading the C API's dialect there would -/// clash with it. +/// `!llvm.ptr`, an opaque pointer. Loads the LLVM dialect into `ctx`. pub fn pointerType(ctx: *mlir.Context, space: anytype) *const mlir.Type { - var buf: [32]u8 = undefined; - const text = std.fmt.bufPrint(&buf, "!llvm.ptr<{d}>", .{@intFromEnum(space)}) catch unreachable; - return mlir.Type.parse(ctx, text) catch std.debug.panic("failed to parse LLVM pointer type '{s}'", .{text}); + _ = c.mlirContextGetOrLoadDialect(ctx.ptr(), mlir.stringRef("llvm")); + return @ptrCast(c.mlirLLVMPointerTypeGet(ctx.ptr(), @intCast(@intFromEnum(space))).ptr); } // ============================================================================= diff --git a/mlir/dialects/mosaic_tpu/BUILD.bazel b/mlir/dialects/mosaic_tpu/BUILD.bazel index ac3a12b0..37933e22 100644 --- a/mlir/dialects/mosaic_tpu/BUILD.bazel +++ b/mlir/dialects/mosaic_tpu/BUILD.bazel @@ -1,11 +1,29 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") +# The types and attributes Mosaic's C API does not expose. +cc_library( + name = "mosaic_tpu_capi", + srcs = ["mosaic_tpu_capi.cc"], + hdrs = ["mosaic_tpu_capi.h"], + strip_include_prefix = ".", + tags = ["manual"], + visibility = ["//visibility:private"], + deps = [ + "@llvm-project//llvm:Support", + "@llvm-project//mlir:CAPIIR", + "@xla//xla/mosaic:tpu_dialect", + ], + alwayslink = True, +) + zig_library( name = "mosaic_tpu", import_name = "mlir/dialects/mosaic_tpu", main = "mosaic_tpu.zig", visibility = ["//visibility:public"], deps = [ + ":mosaic_tpu_capi", "//mlir", "//mlir:c", "//stdx", diff --git a/mlir/dialects/mosaic_tpu/mosaic_tpu.zig b/mlir/dialects/mosaic_tpu/mosaic_tpu.zig index 03496d9e..3cd164c0 100644 --- a/mlir/dialects/mosaic_tpu/mosaic_tpu.zig +++ b/mlir/dialects/mosaic_tpu/mosaic_tpu.zig @@ -21,16 +21,62 @@ pub fn registerMosaicSerdePass() void { // Types — !tpu.semaphore, !tpu.dma_semaphore, !tpu.float8_exmy<...> // ============================================================================= +/// `!tpu.semaphore`. +pub const SemaphoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsATpuSemaphore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "semaphore"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirTpuSemaphoreTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + /// `!tpu.semaphore` element type. pub fn semaphoreType(ctx: *mlir.Context) *const mlir.Type { - return mlir.Type.parse(ctx, "!tpu.semaphore") catch - @panic("failed to parse !tpu.semaphore — is the tpu dialect registered?"); + const ty = SemaphoreType.get(ctx, .{}) catch unreachable; + return ty.type_(); } +/// `!tpu.dma_semaphore`. +pub const DMASemaphoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsATpuDMASemaphore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const isA = M.isA; + pub const mnemonic = "dma_semaphore"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.type_().format(writer); + } + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirTpuDMASemaphoreTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + /// Build `!tpu.dma_semaphore`. pub fn dmaSemaphoreType(ctx: *mlir.Context) *const mlir.Type { - return mlir.Type.parse(ctx, "!tpu.dma_semaphore") catch - @panic("failed to parse !tpu.dma_semaphore — is the tpu dialect registered?"); + const ty = DMASemaphoreType.get(ctx, .{}) catch unreachable; + return ty.type_(); } /// `!tpu.float8_exmy` — Mosaic Float8 type. @@ -57,32 +103,27 @@ pub fn float8ExmyType(ctx: *mlir.Context, underlying: *const mlir.Type) *const m } // ============================================================================= -// Enum / dialect-attribute helpers — parsed from their textual form. +// Attributes // ============================================================================= /// Reduction kind for `tpu.all_reduce`, `tpu.reduce_index`, `tpu.scan`. -pub const ReductionKind = enum { - sum, - max, - min, - maxf, - minf, - maxsi, - minsi, - maxui, - minui, - arg_max, - arg_min, - find_first_set, - @"and", - @"or", - xor, +pub const ReductionKind = enum(u32) { + sum = 0, + max = 1, + min = 2, + arg_max = 3, + arg_min = 4, + find_first_set = 5, + maxf = 6, + minf = 7, + maxsi = 8, + minsi = 9, + maxui = 10, + minui = 11, pub fn attribute(self: ReductionKind, ctx: *mlir.Context) *const mlir.Attribute { - var buf: [64]u8 = undefined; - const text = std.fmt.bufPrint(&buf, "#tpu.reduction_kind<{s}>", .{@tagName(self)}) catch unreachable; - return mlir.Attribute.parse(ctx, text) catch - std.debug.panic("failed to parse tpu.reduction_kind '{s}'", .{text}); + const attr = ReductionKindAttr.get(ctx, .{ .value = self }) catch unreachable; + return attr.attribute(); } fn forInput(self: ReductionKind, input: *const mlir.Value, unsigned_integer: bool) ReductionKind { @@ -98,105 +139,355 @@ pub const ReductionKind = enum { } }; -/// `tpu.contract_precision` — matmul precision setting. -pub const ContractPrecision = enum { - bf16, - fp32, +/// `#tpu.reduction_kind<...>`. +pub const ReductionKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuReductionKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "reduction_kind"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ReductionKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuReductionKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) ReductionKind { + return @enumFromInt(c.mlirTpuReductionKindAttrGetValue(self.ptr())); + } +}; + +/// Matmul precision setting of `tpu.matmul`. +pub const ContractPrecision = enum(u32) { + bf16 = 0, + fp32 = 1, + bf16x3 = 2, pub fn attribute(self: ContractPrecision, ctx: *mlir.Context) *const mlir.Attribute { - const text = switch (self) { - .bf16 => "#tpu.contract_precision", - .fp32 => "#tpu.contract_precision", - }; - return mlir.Attribute.parse(ctx, text) catch @panic("contract_precision parse"); + const attr = ContractPrecisionAttr.get(ctx, .{ .value = self }) catch unreachable; + return attr.attribute(); } }; -/// `tpu.rounding_mode` — used by tpu.fptosi / tpu.fptoui / etc. -pub const RoundingMode = enum { - to_nearest_even, - to_nearest_away, - upward, - downward, - toward_zero, +/// `#tpu.contract_precision<...>`. +pub const ContractPrecisionAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuContractPrecision; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "contract_precision"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ContractPrecision, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuContractPrecisionAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) ContractPrecision { + return @enumFromInt(c.mlirTpuContractPrecisionAttrGetValue(self.ptr())); + } +}; + +/// Rounding mode of `tpu.fptosi` / `tpu.fptoui` / etc. +pub const RoundingMode = enum(u32) { + towards_zero = 0, + to_nearest_even = 1, pub fn attribute(self: RoundingMode, ctx: *mlir.Context) *const mlir.Attribute { - var buf: [64]u8 = undefined; - const text = std.fmt.bufPrint(&buf, "#tpu.rounding_mode<{s}>", .{@tagName(self)}) catch unreachable; - return mlir.Attribute.parse(ctx, text) catch - std.debug.panic("failed to parse tpu.rounding_mode '{s}'", .{text}); + const attr = RoundingModeAttr.get(ctx, .{ .value = self }) catch unreachable; + return attr.attribute(); } }; -/// `#tpu.memory_space<...>` — passed as the `memory_space` of a memref's type. -pub const MemorySpace = enum { - vmem, - smem, - cmem, - hbm, - semaphore_mem, - vmem_shared, - any, - - pub fn attribute(self: MemorySpace, ctx: *mlir.Context) *const mlir.Attribute { - var buf: [64]u8 = undefined; - const text = std.fmt.bufPrint(&buf, "#tpu.memory_space<{s}>", .{@tagName(self)}) catch unreachable; - return mlir.Attribute.parse(ctx, text) catch - std.debug.panic("failed to parse tpu.memory_space '{s}'", .{text}); +/// `#tpu.rounding_mode<...>`. +pub const RoundingModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuRoundingMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "rounding_mode"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: RoundingMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuRoundingModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) RoundingMode { + return @enumFromInt(c.mlirTpuRoundingModeAttrGetValue(self.ptr())); } }; -/// `#tpu.core_type<...>` — used by remote DMAs / sem_signal target hints. -pub const CoreType = enum { - tc, - sc_scalar_subcore, - sc_vector_subcore, +/// Target core of a kernel, remote DMAs and `tpu.sem_signal`. +pub const CoreType = enum(u32) { + tc = 0, + sc_scalar_subcore = 1, + sc_vector_subcore = 2, pub fn attribute(self: CoreType, ctx: *mlir.Context) *const mlir.Attribute { - var buf: [64]u8 = undefined; - const text = std.fmt.bufPrint(&buf, "#tpu.core_type<{s}>", .{@tagName(self)}) catch unreachable; - return mlir.Attribute.parse(ctx, text) catch - std.debug.panic("failed to parse tpu.core_type '{s}'", .{text}); + const attr = CoreTypeAttr.get(ctx, .{ .value = self }) catch unreachable; + return attr.attribute(); } }; -/// `#tpu.dimension_semantics<...>` — grid dimension semantics annotation. -pub const DimensionSemantics = enum { - parallel, - arbitrary, - sequential, +/// `#tpu.core_type<...>`. +pub const CoreTypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuCoreType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "core_type"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: CoreType, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuCoreTypeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) CoreType { + return @enumFromInt(c.mlirTpuCoreTypeAttrGetValue(self.ptr())); + } +}; + +/// Grid dimension semantics annotation. +pub const DimensionSemantics = enum(u32) { + parallel = 0, + arbitrary = 1, + core_parallel = 2, + subcore_parallel = 3, pub fn attribute(self: DimensionSemantics, ctx: *mlir.Context) *const mlir.Attribute { - var buf: [64]u8 = undefined; - const text = std.fmt.bufPrint(&buf, "#tpu.dimension_semantics<{s}>", .{@tagName(self)}) catch unreachable; - return mlir.Attribute.parse(ctx, text) catch - std.debug.panic("failed to parse tpu.dimension_semantics '{s}'", .{text}); + const attr = DimensionSemanticsAttr.get(ctx, .{ .value = self }) catch unreachable; + return attr.attribute(); } }; -/// `#tpu.pipeline_mode<...>` — double-buffering mode in `window_params`. -pub const PipelineMode = enum { - synchronous, - double_buffered, +/// `#tpu.dimension_semantics<...>`. +pub const DimensionSemanticsAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuDimensionSemantics; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "dimension_semantics"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: DimensionSemantics, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuDimensionSemanticsAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) DimensionSemantics { + return @enumFromInt(c.mlirTpuDimensionSemanticsAttrGetValue(self.ptr())); + } +}; + +/// Buffering mode of a `window_params` entry. +pub const PipelineMode = enum(u32) { + synchronous = 1, + double_buffered = 2, pub fn attribute(self: PipelineMode, ctx: *mlir.Context) *const mlir.Attribute { - var buf: [64]u8 = undefined; - const text = std.fmt.bufPrint(&buf, "#tpu.pipeline_mode<{s}>", .{@tagName(self)}) catch unreachable; - return mlir.Attribute.parse(ctx, text) catch - std.debug.panic("failed to parse tpu.pipeline_mode '{s}'", .{text}); + const attr = PipelineModeAttr.get(ctx, .{ .value = self }) catch unreachable; + return attr.attribute(); } }; -/// `#tpu.revisit_mode<...>` — controls block revisit for multi-pass outputs. -pub const RevisitMode = enum { - immediate, - any, +/// `#tpu.pipeline_mode<...>`. +pub const PipelineModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuPipelineMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "pipeline_mode"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: PipelineMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuPipelineModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) PipelineMode { + return @enumFromInt(c.mlirTpuPipelineModeAttrGetValue(self.ptr())); + } +}; + +/// Block revisit mode of a `window_params` entry. +pub const RevisitMode = enum(u32) { + immediate = 0, + any = 1, pub fn attribute(self: RevisitMode, ctx: *mlir.Context) *const mlir.Attribute { - var buf: [64]u8 = undefined; - const text = std.fmt.bufPrint(&buf, "#tpu.revisit_mode<{s}>", .{@tagName(self)}) catch unreachable; - return mlir.Attribute.parse(ctx, text) catch - std.debug.panic("failed to parse tpu.revisit_mode '{s}'", .{text}); + const attr = RevisitModeAttr.get(ctx, .{ .value = self }) catch unreachable; + return attr.attribute(); + } +}; + +/// `#tpu.revisit_mode<...>`. +pub const RevisitModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuRevisitMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "revisit_mode"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: RevisitMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuRevisitModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) RevisitMode { + return @enumFromInt(c.mlirTpuRevisitModeAttrGetValue(self.ptr())); + } +}; + +/// Memory space of a memref, `#tpu.memory_space<...>`. +pub const MemorySpace = enum(u32) { + any = 0xffff_ffff, + vmem = 0, + smem = 1, + hbm = 2, + cmem = 3, + semaphore_mem = 4, + vmem_shared = 5, + host = 6, + + pub fn attribute(self: MemorySpace, ctx: *mlir.Context) *const mlir.Attribute { + const attr = MemorySpaceAttr.get(ctx, .{ .value = self }) catch unreachable; + return attr.attribute(); + } +}; + +/// `#tpu.memory_space` — passed as the `memory_space` of a memref's type. +pub const MemorySpaceAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuMemorySpace; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "memory_space"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MemorySpace, + coreType: ?CoreType = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const core_type: i64 = if (args.coreType) |ct| @intFromEnum(ct) else -1; + const result = c.mlirTpuMemorySpaceAttrGet(ctx.ptr(), @intFromEnum(args.value), core_type); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) MemorySpace { + return @enumFromInt(c.mlirTpuMemorySpaceAttrGetValue(self.ptr())); + } + pub fn getCoreType(self: *const Self) ?CoreType { + const core_type = c.mlirTpuMemorySpaceAttrGetCoreType(self.ptr()); + return if (core_type < 0) null else @enumFromInt(core_type); + } +}; + +/// `#tpu.element_window<[pad_low], [pad_high]>` — per-arg window padding. +pub const ElementWindowAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuElementWindow; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "element_window"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + padLow: []const i64, + padHigh: []const i64, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuElementWindowAttrGet( + ctx.ptr(), + @intCast(args.padLow.len), + args.padLow.ptr, + @intCast(args.padHigh.len), + args.padHigh.ptr, + ); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getNumPadLow(self: *const Self) usize { + return @intCast(c.mlirTpuElementWindowAttrGetNumPadLow(self.ptr())); + } + pub fn getPadLow(self: *const Self, pos: usize) i64 { + return c.mlirTpuElementWindowAttrGetPadLow(self.ptr(), @intCast(pos)); + } + pub fn getNumPadHigh(self: *const Self) usize { + return @intCast(c.mlirTpuElementWindowAttrGetNumPadHigh(self.ptr())); + } + pub fn getPadHigh(self: *const Self, pos: usize) i64 { + return c.mlirTpuElementWindowAttrGetPadHigh(self.ptr(), @intCast(pos)); } }; @@ -206,24 +497,100 @@ pub fn elementWindowAttribute( pad_low: []const i64, pad_high: []const i64, ) *const mlir.Attribute { - var buf: [256]u8 = undefined; - var len: usize = 0; - len += (std.fmt.bufPrint(buf[len..], "#tpu.element_window<[", .{}) catch unreachable).len; - for (pad_low, 0..) |p, i| { - const sep = if (i == 0) "" else ", "; - len += (std.fmt.bufPrint(buf[len..], "{s}{d}", .{ sep, p }) catch unreachable).len; - } - len += (std.fmt.bufPrint(buf[len..], "], [", .{}) catch unreachable).len; - for (pad_high, 0..) |p, i| { - const sep = if (i == 0) "" else ", "; - len += (std.fmt.bufPrint(buf[len..], "{s}{d}", .{ sep, p }) catch unreachable).len; - } - len += (std.fmt.bufPrint(buf[len..], "]>", .{}) catch unreachable).len; - const text = buf[0..len]; - return mlir.Attribute.parse(ctx, text) catch - std.debug.panic("failed to parse tpu.element_window '{s}'", .{text}); + const attr = ElementWindowAttr.get(ctx, .{ .padLow = pad_low, .padHigh = pad_high }) catch unreachable; + return attr.attribute(); } +/// `#tpu.dot_dimension_numbers<...>` — the dimension mapping of `tpu.matmul`. +pub const DotDimensionNumbersAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuDotDimensionNumbers; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "dot_dimension_numbers"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + lhsContractingDims: []const i64, + rhsContractingDims: []const i64, + lhsNonContractingDims: []const i64, + /// Empty when rhs is a 1-D vector. + rhsNonContractingDims: []const i64 = &.{}, + /// Flattened (operand, dim) pairs: operand 0 is lhs, 1 is rhs. + outputDimOrder: []const i64, + lhsBatchDims: []const i64 = &.{}, + rhsBatchDims: []const i64 = &.{}, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuDotDimensionNumbersAttrGet( + ctx.ptr(), + @intCast(args.lhsContractingDims.len), + args.lhsContractingDims.ptr, + @intCast(args.rhsContractingDims.len), + args.rhsContractingDims.ptr, + @intCast(args.lhsNonContractingDims.len), + args.lhsNonContractingDims.ptr, + @intCast(args.rhsNonContractingDims.len), + args.rhsNonContractingDims.ptr, + @intCast(args.outputDimOrder.len), + args.outputDimOrder.ptr, + @intCast(args.lhsBatchDims.len), + args.lhsBatchDims.ptr, + @intCast(args.rhsBatchDims.len), + args.rhsBatchDims.ptr, + ); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getNumLhsContractingDims(self: *const Self) usize { + return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumLhsContractingDims(self.ptr())); + } + pub fn getLhsContractingDims(self: *const Self, pos: usize) i64 { + return c.mlirTpuDotDimensionNumbersAttrGetLhsContractingDims(self.ptr(), @intCast(pos)); + } + pub fn getNumRhsContractingDims(self: *const Self) usize { + return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumRhsContractingDims(self.ptr())); + } + pub fn getRhsContractingDims(self: *const Self, pos: usize) i64 { + return c.mlirTpuDotDimensionNumbersAttrGetRhsContractingDims(self.ptr(), @intCast(pos)); + } + pub fn getNumLhsNonContractingDims(self: *const Self) usize { + return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumLhsNonContractingDims(self.ptr())); + } + pub fn getLhsNonContractingDims(self: *const Self, pos: usize) i64 { + return c.mlirTpuDotDimensionNumbersAttrGetLhsNonContractingDims(self.ptr(), @intCast(pos)); + } + pub fn getNumRhsNonContractingDims(self: *const Self) usize { + return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumRhsNonContractingDims(self.ptr())); + } + pub fn getRhsNonContractingDims(self: *const Self, pos: usize) i64 { + return c.mlirTpuDotDimensionNumbersAttrGetRhsNonContractingDims(self.ptr(), @intCast(pos)); + } + pub fn getNumOutputDimOrder(self: *const Self) usize { + return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumOutputDimOrder(self.ptr())); + } + pub fn getOutputDimOrder(self: *const Self, pos: usize) i64 { + return c.mlirTpuDotDimensionNumbersAttrGetOutputDimOrder(self.ptr(), @intCast(pos)); + } + pub fn getNumLhsBatchDims(self: *const Self) usize { + return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumLhsBatchDims(self.ptr())); + } + pub fn getLhsBatchDims(self: *const Self, pos: usize) i64 { + return c.mlirTpuDotDimensionNumbersAttrGetLhsBatchDims(self.ptr(), @intCast(pos)); + } + pub fn getNumRhsBatchDims(self: *const Self) usize { + return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumRhsBatchDims(self.ptr())); + } + pub fn getRhsBatchDims(self: *const Self, pos: usize) i64 { + return c.mlirTpuDotDimensionNumbersAttrGetRhsBatchDims(self.ptr(), @intCast(pos)); + } +}; + // ============================================================================= // Reductions / scan / sort // ============================================================================= @@ -513,7 +880,8 @@ pub const MatmulOpts = struct { /// Optional precision; pass null to omit. precision: ?ContractPrecision = null, /// Optional `#tpu.dot_dimension_numbers<...>` attribute. When omitted the - /// canonicalizer derives one. Build with `mlir.Attribute.parse`. + /// canonicalizer derives one. Build with `DotDimensionNumbersAttr` or + /// `dotDimensionNumbers`. dimension_numbers: ?*const mlir.Attribute = null, }; @@ -1135,29 +1503,16 @@ pub fn dotDimensionNumbers( lhs_batch: []const i64, rhs_batch: []const i64, ) *const mlir.Attribute { - var buf: [512]u8 = undefined; - var len: usize = 0; - len += (std.fmt.bufPrint(buf[len..], "#tpu.dot_dimension_numbers<", .{}) catch unreachable).len; - inline for (.{ - lhs_contracting, - rhs_contracting, - lhs_non_contracting, - rhs_non_contracting, - output_dim_order, - lhs_batch, - rhs_batch, - }, 0..) |arr, i| { - if (i != 0) len += (std.fmt.bufPrint(buf[len..], ", ", .{}) catch unreachable).len; - len += (std.fmt.bufPrint(buf[len..], "[", .{}) catch unreachable).len; - for (arr, 0..) |v, j| { - const sep = if (j == 0) "" else ", "; - len += (std.fmt.bufPrint(buf[len..], "{s}{d}", .{ sep, v }) catch unreachable).len; - } - len += (std.fmt.bufPrint(buf[len..], "]", .{}) catch unreachable).len; - } - len += (std.fmt.bufPrint(buf[len..], ">", .{}) catch unreachable).len; - return mlir.Attribute.parse(ctx, buf[0..len]) catch - std.debug.panic("failed to parse tpu.dot_dimension_numbers '{s}'", .{buf[0..len]}); + const attr = DotDimensionNumbersAttr.get(ctx, .{ + .lhsContractingDims = lhs_contracting, + .rhsContractingDims = rhs_contracting, + .lhsNonContractingDims = lhs_non_contracting, + .rhsNonContractingDims = rhs_non_contracting, + .outputDimOrder = output_dim_order, + .lhsBatchDims = lhs_batch, + .rhsBatchDims = rhs_batch, + }) catch unreachable; + return attr.attribute(); } // ============================================================================= @@ -1317,3 +1672,89 @@ test "legacy reduction extrema and scan axis preserve their semantics" { defer float_scan.deinit(); try std.testing.expect(float_scan.attributeByName("kind").?.eql(try mlir.Attribute.parse(ctx, "#tpu.reduction_kind"))); } + +fn expectPrints(expected: []const u8, value: anytype) !void { + var buf: [256]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(expected, w.buffered()); +} + +fn expectEnumAttr(comptime Attr: type, ctx: *mlir.Context) !void { + const Enum = @FieldType(Attr.InitArgs, "value"); + inline for (std.meta.fields(Enum)) |field| { + const value: Enum = @enumFromInt(field.value); + const attr = try Attr.get(ctx, .{ .value = value }); + try std.testing.expectEqual(value, attr.getValue()); + try std.testing.expect(attr.attribute().isA(Attr) != null); + try std.testing.expect(value.attribute(ctx).eql(attr.attribute())); + try expectPrints("#tpu." ++ Attr.mnemonic ++ "<" ++ field.name ++ ">", attr); + } +} + +test "tpu types are built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + const semaphore = try SemaphoreType.get(ctx, .{}); + try std.testing.expect(semaphoreType(ctx).isA(SemaphoreType) != null); + try std.testing.expect(semaphoreType(ctx).isA(DMASemaphoreType) == null); + try expectPrints("!tpu.semaphore", semaphore); + + const dma_semaphore = try DMASemaphoreType.get(ctx, .{}); + try std.testing.expect(dmaSemaphoreType(ctx).isA(DMASemaphoreType) != null); + try expectPrints("!tpu.dma_semaphore", dma_semaphore); +} + +test "tpu enum attributes are built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + try expectEnumAttr(ReductionKindAttr, ctx); + try expectEnumAttr(ContractPrecisionAttr, ctx); + try expectEnumAttr(RoundingModeAttr, ctx); + try expectEnumAttr(CoreTypeAttr, ctx); + try expectEnumAttr(DimensionSemanticsAttr, ctx); + try expectEnumAttr(PipelineModeAttr, ctx); + try expectEnumAttr(RevisitModeAttr, ctx); + try expectEnumAttr(MemorySpaceAttr, ctx); + + const smem_tc = try MemorySpaceAttr.get(ctx, .{ .value = .smem, .coreType = .sc_scalar_subcore }); + try std.testing.expectEqual(MemorySpace.smem, smem_tc.getValue()); + try std.testing.expectEqual(CoreType.sc_scalar_subcore, smem_tc.getCoreType().?); + try expectPrints("#tpu.memory_space", smem_tc); + try std.testing.expectEqual(null, (try MemorySpaceAttr.get(ctx, .{ .value = .hbm })).getCoreType()); + // The dialect verifier's pairing rules: no core type on hbm, no scalar subcore on vmem. + try std.testing.expectError(error.InvalidMlir, MemorySpaceAttr.get(ctx, .{ .value = .hbm, .coreType = .tc })); + try std.testing.expectError(error.InvalidMlir, MemorySpaceAttr.get(ctx, .{ .value = .vmem, .coreType = .sc_scalar_subcore })); +} + +test "tpu array attributes are built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + const window = try ElementWindowAttr.get(ctx, .{ .padLow = &.{ 0, 2 }, .padHigh = &.{ 1, 3 } }); + try std.testing.expectEqual(2, window.getNumPadLow()); + try std.testing.expectEqual(2, window.getPadLow(1)); + try std.testing.expectEqual(2, window.getNumPadHigh()); + try std.testing.expectEqual(3, window.getPadHigh(1)); + try expectPrints("#tpu.element_window<[0, 2], [1, 3]>", window); + try std.testing.expect(elementWindowAttribute(ctx, &.{ 0, 2 }, &.{ 1, 3 }).eql(window.attribute())); + + const dims = dotDimensionNumbers(ctx, &.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}); + try expectPrints("#tpu.dot_dimension_numbers<[1], [0], [0], [1], [0, 0, 1, 1], [], []>", dims); + + const batched = try DotDimensionNumbersAttr.get(ctx, .{ + .lhsContractingDims = &.{2}, + .rhsContractingDims = &.{1}, + .lhsNonContractingDims = &.{1}, + .outputDimOrder = &.{ 0, 0, 0, 1 }, + .lhsBatchDims = &.{0}, + .rhsBatchDims = &.{0}, + }); + try std.testing.expectEqual(0, batched.getNumRhsNonContractingDims()); + try std.testing.expectEqual(4, batched.getNumOutputDimOrder()); + try std.testing.expectEqual(1, batched.getOutputDimOrder(3)); + try std.testing.expectEqual(0, batched.getRhsBatchDims(0)); + try expectPrints("#tpu.dot_dimension_numbers<[2], [1], [1], [], [0, 0, 0, 1], [0], [0]>", batched); +} diff --git a/mlir/dialects/mosaic_tpu/mosaic_tpu_capi.cc b/mlir/dialects/mosaic_tpu/mosaic_tpu_capi.cc new file mode 100644 index 00000000..5b9e7e20 --- /dev/null +++ b/mlir/dialects/mosaic_tpu/mosaic_tpu_capi.cc @@ -0,0 +1,225 @@ +#include "mosaic_tpu_capi.h" + +#include +#include + +#include "xla/mosaic/dialect/tpu/tpu_dialect.h" +#include "llvm/ADT/ArrayRef.h" +#include "mlir/CAPI/IR.h" +#include "mlir/IR/Diagnostics.h" +#include "mlir/IR/Location.h" + +namespace { + +mlir::MLIRContext *loadTPU(MlirContext context) { + mlir::MLIRContext *ctx = unwrap(context); + if (ctx) ctx->getOrLoadDialect(); + return ctx; +} + +template +MlirAttribute enumAttrGet(MlirContext context, std::optional value) { + mlir::MLIRContext *ctx = loadTPU(context); + if (!ctx || !value) return {nullptr}; + return wrap(AttrT::get(ctx, *value)); +} + +template +uint32_t enumAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +llvm::ArrayRef arrayRef(intptr_t size, const int64_t *data) { + return llvm::ArrayRef(data, static_cast(size)); +} + +} // namespace + +bool mlirTypeIsATpuSemaphore(MlirType type) { return llvm::isa(unwrap(type)); } + +MlirType mlirTpuSemaphoreTypeGet(MlirContext context) { + mlir::MLIRContext *ctx = loadTPU(context); + if (!ctx) return {nullptr}; + return wrap(mlir::tpu::SemaphoreType::get(ctx)); +} + +bool mlirTypeIsATpuDMASemaphore(MlirType type) { return llvm::isa(unwrap(type)); } + +MlirType mlirTpuDMASemaphoreTypeGet(MlirContext context) { + mlir::MLIRContext *ctx = loadTPU(context); + if (!ctx) return {nullptr}; + return wrap(mlir::tpu::DMASemaphoreType::get(ctx)); +} + +bool mlirAttributeIsATpuReductionKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuReductionKindAttrGet(MlirContext context, uint32_t value) { + return enumAttrGet(context, mlir::tpu::symbolizeReductionKind(value)); +} + +uint32_t mlirTpuReductionKindAttrGetValue(MlirAttribute attr) { + return enumAttrGetValue(attr); +} + +bool mlirAttributeIsATpuContractPrecision(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuContractPrecisionAttrGet(MlirContext context, uint32_t value) { + return enumAttrGet(context, mlir::tpu::symbolizeContractPrecision(value)); +} + +uint32_t mlirTpuContractPrecisionAttrGetValue(MlirAttribute attr) { + return enumAttrGetValue(attr); +} + +bool mlirAttributeIsATpuRoundingMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuRoundingModeAttrGet(MlirContext context, uint32_t value) { + return enumAttrGet(context, mlir::tpu::symbolizeRoundingMode(value)); +} + +uint32_t mlirTpuRoundingModeAttrGetValue(MlirAttribute attr) { + return enumAttrGetValue(attr); +} + +bool mlirAttributeIsATpuCoreType(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuCoreTypeAttrGet(MlirContext context, uint32_t value) { + return enumAttrGet(context, mlir::tpu::symbolizeCoreType(value)); +} + +uint32_t mlirTpuCoreTypeAttrGetValue(MlirAttribute attr) { return enumAttrGetValue(attr); } + +bool mlirAttributeIsATpuDimensionSemantics(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuDimensionSemanticsAttrGet(MlirContext context, uint32_t value) { + return enumAttrGet(context, mlir::tpu::symbolizeDimensionSemantics(value)); +} + +uint32_t mlirTpuDimensionSemanticsAttrGetValue(MlirAttribute attr) { + return enumAttrGetValue(attr); +} + +bool mlirAttributeIsATpuPipelineMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuPipelineModeAttrGet(MlirContext context, uint32_t value) { + return enumAttrGet(context, mlir::tpu::symbolizePipelineMode(value)); +} + +uint32_t mlirTpuPipelineModeAttrGetValue(MlirAttribute attr) { + return enumAttrGetValue(attr); +} + +bool mlirAttributeIsATpuRevisitMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuRevisitModeAttrGet(MlirContext context, uint32_t value) { + return enumAttrGet(context, mlir::tpu::symbolizeRevisitMode(value)); +} + +uint32_t mlirTpuRevisitModeAttrGetValue(MlirAttribute attr) { + return enumAttrGetValue(attr); +} + +bool mlirAttributeIsATpuMemorySpace(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuMemorySpaceAttrGet(MlirContext context, uint32_t value, int64_t coreType) { + mlir::MLIRContext *ctx = loadTPU(context); + std::optional memorySpace = mlir::tpu::symbolizeMemorySpace(value); + if (!ctx || !memorySpace) return {nullptr}; + std::optional core; + if (coreType >= 0) { + if (coreType > UINT32_MAX) return {nullptr}; + core = mlir::tpu::symbolizeCoreType(static_cast(coreType)); + if (!core) return {nullptr}; + } + return wrap(mlir::tpu::MemorySpaceAttr::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, *memorySpace, core)); +} + +uint32_t mlirTpuMemorySpaceAttrGetValue(MlirAttribute attr) { + return enumAttrGetValue(attr); +} + +int64_t mlirTpuMemorySpaceAttrGetCoreType(MlirAttribute attr) { + std::optional core = llvm::cast(unwrap(attr)).getCoreType(); + return core ? static_cast(*core) : -1; +} + +bool mlirAttributeIsATpuElementWindow(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuElementWindowAttrGet(MlirContext context, intptr_t numPadLow, const int64_t *padLow, + intptr_t numPadHigh, const int64_t *padHigh) { + mlir::MLIRContext *ctx = loadTPU(context); + if (!ctx) return {nullptr}; + return wrap(mlir::tpu::ElementWindowAttr::get(ctx, arrayRef(numPadLow, padLow), arrayRef(numPadHigh, padHigh))); +} + +intptr_t mlirTpuElementWindowAttrGetNumPadLow(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getPadLow().size(); +} + +int64_t mlirTpuElementWindowAttrGetPadLow(MlirAttribute attr, intptr_t pos) { + return llvm::cast(unwrap(attr)).getPadLow()[pos]; +} + +intptr_t mlirTpuElementWindowAttrGetNumPadHigh(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getPadHigh().size(); +} + +int64_t mlirTpuElementWindowAttrGetPadHigh(MlirAttribute attr, intptr_t pos) { + return llvm::cast(unwrap(attr)).getPadHigh()[pos]; +} + +bool mlirAttributeIsATpuDotDimensionNumbers(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuDotDimensionNumbersAttrGet( + MlirContext context, intptr_t numLhsContractingDims, const int64_t *lhsContractingDims, + intptr_t numRhsContractingDims, const int64_t *rhsContractingDims, intptr_t numLhsNonContractingDims, + const int64_t *lhsNonContractingDims, intptr_t numRhsNonContractingDims, const int64_t *rhsNonContractingDims, + intptr_t numOutputDimOrder, const int64_t *outputDimOrder, intptr_t numLhsBatchDims, const int64_t *lhsBatchDims, + intptr_t numRhsBatchDims, const int64_t *rhsBatchDims) { + mlir::MLIRContext *ctx = loadTPU(context); + if (!ctx) return {nullptr}; + return wrap(mlir::tpu::DotDimensionNumbersAttr::get( + ctx, arrayRef(numLhsContractingDims, lhsContractingDims), + arrayRef(numRhsContractingDims, rhsContractingDims), arrayRef(numLhsNonContractingDims, lhsNonContractingDims), + arrayRef(numRhsNonContractingDims, rhsNonContractingDims), arrayRef(numOutputDimOrder, outputDimOrder), + arrayRef(numLhsBatchDims, lhsBatchDims), arrayRef(numRhsBatchDims, rhsBatchDims))); +} + +#define ZML_TPU_DOT_DIMS_GETTERS(Name) \ + intptr_t mlirTpuDotDimensionNumbersAttrGetNum##Name(MlirAttribute attr) { \ + return llvm::cast(unwrap(attr)).get##Name().size(); \ + } \ + int64_t mlirTpuDotDimensionNumbersAttrGet##Name(MlirAttribute attr, intptr_t pos) { \ + return llvm::cast(unwrap(attr)).get##Name()[pos]; \ + } + +ZML_TPU_DOT_DIMS_GETTERS(LhsContractingDims) +ZML_TPU_DOT_DIMS_GETTERS(RhsContractingDims) +ZML_TPU_DOT_DIMS_GETTERS(LhsNonContractingDims) +ZML_TPU_DOT_DIMS_GETTERS(RhsNonContractingDims) +ZML_TPU_DOT_DIMS_GETTERS(OutputDimOrder) +ZML_TPU_DOT_DIMS_GETTERS(LhsBatchDims) +ZML_TPU_DOT_DIMS_GETTERS(RhsBatchDims) + +#undef ZML_TPU_DOT_DIMS_GETTERS diff --git a/mlir/dialects/mosaic_tpu/mosaic_tpu_capi.h b/mlir/dialects/mosaic_tpu/mosaic_tpu_capi.h new file mode 100644 index 00000000..816d3aa6 --- /dev/null +++ b/mlir/dialects/mosaic_tpu/mosaic_tpu_capi.h @@ -0,0 +1,105 @@ +// C entry points for the `tpu` dialect that Mosaic's C API +// (`tpu_dialect.h`) leaves out: the semaphore types and the attributes +// kernels build. +#ifndef ZML_MLIR_DIALECTS_MOSAIC_TPU_CAPI_H +#define ZML_MLIR_DIALECTS_MOSAIC_TPU_CAPI_H + +#include +#include + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// !tpu.semaphore +MLIR_CAPI_EXPORTED bool mlirTypeIsATpuSemaphore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirTpuSemaphoreTypeGet(MlirContext context); + +// !tpu.dma_semaphore +MLIR_CAPI_EXPORTED bool mlirTypeIsATpuDMASemaphore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirTpuDMASemaphoreTypeGet(MlirContext context); + +// Enum attributes take and return the C++ enum's integer value (tpu_ops.td); +// `Get` returns a null attribute for a value outside the enum. + +// #tpu.reduction_kind<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuReductionKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuReductionKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirTpuReductionKindAttrGetValue(MlirAttribute attr); + +// #tpu.contract_precision<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuContractPrecision(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuContractPrecisionAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirTpuContractPrecisionAttrGetValue(MlirAttribute attr); + +// #tpu.rounding_mode<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuRoundingMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuRoundingModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirTpuRoundingModeAttrGetValue(MlirAttribute attr); + +// #tpu.core_type<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuCoreType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuCoreTypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirTpuCoreTypeAttrGetValue(MlirAttribute attr); + +// #tpu.dimension_semantics<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuDimensionSemantics(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuDimensionSemanticsAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirTpuDimensionSemanticsAttrGetValue(MlirAttribute attr); + +// #tpu.pipeline_mode<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuPipelineMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuPipelineModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirTpuPipelineModeAttrGetValue(MlirAttribute attr); + +// #tpu.revisit_mode<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuRevisitMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuRevisitModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirTpuRevisitModeAttrGetValue(MlirAttribute attr); + +// #tpu.memory_space: a negative `coreType` leaves the +// optional core type out, and `GetCoreType` returns -1 when it is absent. +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuMemorySpace(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuMemorySpaceAttrGet(MlirContext context, uint32_t value, int64_t coreType); +MLIR_CAPI_EXPORTED uint32_t mlirTpuMemorySpaceAttrGetValue(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirTpuMemorySpaceAttrGetCoreType(MlirAttribute attr); + +// #tpu.element_window<[pad_low], [pad_high]> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuElementWindow(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuElementWindowAttrGet(MlirContext context, intptr_t numPadLow, const int64_t *padLow, + intptr_t numPadHigh, const int64_t *padHigh); +MLIR_CAPI_EXPORTED intptr_t mlirTpuElementWindowAttrGetNumPadLow(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirTpuElementWindowAttrGetPadLow(MlirAttribute attr, intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t mlirTpuElementWindowAttrGetNumPadHigh(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirTpuElementWindowAttrGetPadHigh(MlirAttribute attr, intptr_t pos); + +// #tpu.dot_dimension_numbers<...>: the dimension lists in ODS order. +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuDotDimensionNumbers(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuDotDimensionNumbersAttrGet( + MlirContext context, intptr_t numLhsContractingDims, const int64_t *lhsContractingDims, + intptr_t numRhsContractingDims, const int64_t *rhsContractingDims, intptr_t numLhsNonContractingDims, + const int64_t *lhsNonContractingDims, intptr_t numRhsNonContractingDims, const int64_t *rhsNonContractingDims, + intptr_t numOutputDimOrder, const int64_t *outputDimOrder, intptr_t numLhsBatchDims, const int64_t *lhsBatchDims, + intptr_t numRhsBatchDims, const int64_t *rhsBatchDims); +MLIR_CAPI_EXPORTED intptr_t mlirTpuDotDimensionNumbersAttrGetNumLhsContractingDims(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetLhsContractingDims(MlirAttribute attr, intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t mlirTpuDotDimensionNumbersAttrGetNumRhsContractingDims(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetRhsContractingDims(MlirAttribute attr, intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t mlirTpuDotDimensionNumbersAttrGetNumLhsNonContractingDims(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetLhsNonContractingDims(MlirAttribute attr, intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t mlirTpuDotDimensionNumbersAttrGetNumRhsNonContractingDims(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetRhsNonContractingDims(MlirAttribute attr, intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t mlirTpuDotDimensionNumbersAttrGetNumOutputDimOrder(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetOutputDimOrder(MlirAttribute attr, intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t mlirTpuDotDimensionNumbersAttrGetNumLhsBatchDims(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetLhsBatchDims(MlirAttribute attr, intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t mlirTpuDotDimensionNumbersAttrGetNumRhsBatchDims(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetRhsBatchDims(MlirAttribute attr, intptr_t pos); + +#ifdef __cplusplus +} +#endif + +#endif // ZML_MLIR_DIALECTS_MOSAIC_TPU_CAPI_H diff --git a/mlir/dialects/nvvm.zig b/mlir/dialects/nvvm.zig index 6cf5d71d..3a986dd4 100644 --- a/mlir/dialects/nvvm.zig +++ b/mlir/dialects/nvvm.zig @@ -1,103 +1,39 @@ +//! Builders for the NVVM operations CuTe kernels emit, over the bindings of +//! NVIDIA's `nvvm` dialect (`mlir/dialects/cute_ir`, `cute.nvvm`): the CuTe-DSL +//! compiler's own NVVM, which the upstream dialect differs from. Each builder +//! fixes the attributes the kernels do not choose. + const std = @import("std"); +const cute = @import("mlir/dialects/cute_ir"); const mlir = @import("mlir"); -// ============================================================================= -// Enum attributes — parsed from their textual `#nvvm.` form. -// ============================================================================= +pub const ops = cute.nvvm; -fn enumAttribute(ctx: *mlir.Context, comptime mnemonic: []const u8, value: []const u8) *const mlir.Attribute { - var buf: [96]u8 = undefined; - const text = std.fmt.bufPrint(&buf, "#nvvm." ++ mnemonic ++ "<{s}>", .{value}) catch unreachable; - return mlir.Attribute.parse(ctx, text) catch std.debug.panic("failed to parse NVVM attribute '{s}'", .{text}); -} - -fn EnumAttribute(comptime mnemonic: []const u8, comptime Tag: type) type { - return struct { - pub fn attribute(self: Tag, ctx: *mlir.Context) *const mlir.Attribute { - return enumAttribute(ctx, mnemonic, @tagName(self)); - } - }; +fn enumAttribute(comptime T: type, ctx: *mlir.Context, value: @FieldType(T.InitArgs, "value")) *const mlir.Attribute { + return (T.get(ctx, .{ .value = value }) catch unreachable).attribute(); } /// `#nvvm.cta_group<...>`: CTAs cooperating on a tcgen05 operation. -pub const CtaGroup = enum { - cta_1, - cta_2, - pub const attribute = EnumAttribute("cta_group", CtaGroup).attribute; -}; - +pub const CtaGroup = ops.CTAGroupKind; /// `#nvvm.tcgen05_fence<...>`: `tcgen05.fence::{before,after}_thread_sync`. -pub const Tcgen05FenceKind = enum { - before, - after, - pub const attribute = EnumAttribute("tcgen05_fence", Tcgen05FenceKind).attribute; -}; - +pub const Tcgen05FenceKind = ops.Tcgen05FenceKind; /// `#nvvm.tcgen05_wait<...>`: `tcgen05.wait::{ld,st}`. -pub const Tcgen05WaitKind = enum { - load, - store, - pub const attribute = EnumAttribute("tcgen05_wait", Tcgen05WaitKind).attribute; -}; - +pub const Tcgen05WaitKind = ops.Tcgen05WaitKind; /// `#nvvm.tcgen05_mma_kind<...>`: input types of `tcgen05.mma`. -pub const Tcgen05MmaKind = enum { - f16, - tf32, - f8f6f4, - i8, - pub const attribute = EnumAttribute("tcgen05_mma_kind", Tcgen05MmaKind).attribute; -}; - +pub const Tcgen05MmaKind = ops.Tcgen05MMAKind; /// `#nvvm.tcgen05_ldst_shape<...>`: lane/bit shape of `tcgen05.ld` / `tcgen05.st`. -pub const Tcgen05LdStShape = enum { - shape_16x64b, - shape_16x128b, - shape_16x256b, - shape_32x32b, - shape_16x32bx2, - pub const attribute = EnumAttribute("tcgen05_ldst_shape", Tcgen05LdStShape).attribute; -}; - +pub const Tcgen05LdStShape = ops.Tcgen05LdStShape; /// `#nvvm.mem_scope<...>`: scope of a memory operation. -pub const MemScope = enum { - cta, - cluster, - gpu, - sys, - pub const attribute = EnumAttribute("mem_scope", MemScope).attribute; -}; - +pub const MemScope = ops.MemScopeKind; /// `#nvvm.mbar_wait<...>`: `test` (non-blocking) or `try` (may suspend) wait. -pub const MBarrierWaitKind = enum { - @"test", - @"try", - pub const attribute = EnumAttribute("mbar_wait", MBarrierWaitKind).attribute; -}; - +pub const MBarrierWaitKind = ops.MBarrierWaitKind; /// `#nvvm.mbar_scope<...>`: scope of an mbarrier wait. -pub const MBarrierScope = enum { - cta, - cluster, - pub const attribute = EnumAttribute("mbar_scope", MBarrierScope).attribute; -}; - +pub const MBarrierScope = ops.MBarrierScopeKind; /// `#nvvm.proxy_kind<...>`: memory proxy of `fence.proxy`. -pub const ProxyKind = enum { - alias, - async, - @"async.global", - @"async.shared", - pub const attribute = EnumAttribute("proxy_kind", ProxyKind).attribute; -}; - +pub const ProxyKind = ops.ProxyKind; /// `#nvvm.shared_space<...>`: `shared::cta` or `shared::cluster`. -pub const SharedSpace = enum { - cta, - cluster, - pub const attribute = EnumAttribute("shared_space", SharedSpace).attribute; -}; +pub const SharedSpace = ops.SharedSpace; // ============================================================================= // Special registers and thread synchronization @@ -106,75 +42,48 @@ pub const SharedSpace = enum { /// nvvm.read.ptx.sreg. — an i32 special register, e.g. `tid.x`, `ctaid.y`, /// `cluster.ctarank`. pub fn read_sreg(ctx: *mlir.Context, comptime name: []const u8, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg." ++ name, .{ - .results = .{ .flat = &.{.int(ctx, .i32)} }, - .location = location, - }); + const builder = comptime blk: { + var fn_name: [name.len]u8 = name[0..name.len].*; + std.mem.replaceScalar(u8, &fn_name, '.', '_'); + break :blk "read_ptx_sreg_" ++ fn_name; + }; + return @field(ops, builder)(ctx, .int(ctx, .i32), null, location); } /// nvvm.barrier — `bar.sync 0` over the whole CTA. pub fn barrier(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.barrier", .{ - .attributes = &.{.named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ 0, 0, 0 }))}, - .location = location, - }); + return ops.barrier(ctx, null, null, null, null, null, location); } /// nvvm.bar.warp.sync — `bar.warp.sync mask` (i32 lane mask, -1 for the full warp). pub fn bar_warp_sync(ctx: *mlir.Context, mask: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.bar.warp.sync", .{ - .operands = .{ .flat = &.{mask} }, - .location = location, - }); + return ops.bar_warp_sync(ctx, mask, location); } /// nvvm.elect.sync — i1, true in one elected lane of the warp. pub fn elect_sync(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.elect.sync", .{ - .results = .{ .flat = &.{.int(ctx, .i1)} }, - .location = location, - }); + return ops.elect_sync(ctx, null, .int(ctx, .i1), location); } // ============================================================================= // mbarrier // ============================================================================= -/// nvvm.mbarrier.init — initialize the shared mbarrier at `ptr` (`!llvm.ptr<3>`) -/// expecting `count` (i32) arrivals per phase. +/// nvvm.mbarrier.init — initialize the mbarrier at `ptr` (`!llvm.ptr<3>`) to expect +/// `count` (i32) arrivals per phase. pub fn mbarrier_init(ctx: *mlir.Context, ptr: *const mlir.Value, count: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.mbarrier.init", .{ - .operands = .{ .flat = &.{ ptr, count } }, - .attributes = &.{.named(ctx, "layout", enumAttribute(ctx, "mbarrier_layout", "v0"))}, - .location = location, - }); + return ops.mbarrier_init(ctx, ptr, count, null, enumAttribute(ops.MBarrierLayoutAttr, ctx, .v0), location); } /// nvvm.mbarrier.arrive — arrive `count` (i32) times (release semantics). pub fn mbarrier_arrive(ctx: *mlir.Context, ptr: *const mlir.Value, count: *const mlir.Value, scope: MemScope, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive", .{ - .operands = .{ .flat = &.{ ptr, count } }, - .attributes = &.{ - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ 1, 1, 0 })), - .named(ctx, "relaxed", .boolean(ctx, false)), - .named(ctx, "scope", scope.attribute(ctx)), - }, - .location = location, - }); + return ops.mbarrier_arrive(ctx, ptr, count, null, null, enumAttribute(ops.MemScopeKindAttr, ctx, scope), .boolean(ctx, false), location); } /// nvvm.mbarrier.arrive.expect_tx — arrive and expect `bytes` (i32) of asynchronous /// transactions (SM90+). pub fn mbarrier_arrive_expect_tx(ctx: *mlir.Context, ptr: *const mlir.Value, bytes: *const mlir.Value, scope: MemScope, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive.expect_tx", .{ - .operands = .{ .flat = &.{ ptr, bytes } }, - .attributes = &.{ - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ 1, 1, 0, 0 })), - .named(ctx, "relaxed", .boolean(ctx, false)), - .named(ctx, "scope", scope.attribute(ctx)), - }, - .location = location, - }); + return ops.mbarrier_arrive_expect_tx(ctx, ptr, bytes, null, null, null, enumAttribute(ops.MemScopeKindAttr, ctx, scope), .boolean(ctx, false), location); } /// nvvm.mbarrier.wait.parity — i1, whether phase `parity` (i32) has completed @@ -187,19 +96,20 @@ pub fn mbarrier_wait_parity( scope: MBarrierScope, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.mbarrier.wait.parity", .{ - .operands = .{ .flat = &.{ ptr, parity } }, - .results = .{ .flat = &.{.int(ctx, .i1)} }, - .attributes = &.{ - .named(ctx, "kind", kind.attribute(ctx)), - .named(ctx, "scope", scope.attribute(ctx)), - }, - .location = location, - }); + return ops.mbarrier_wait_parity( + ctx, + ptr, + parity, + .int(ctx, .i1), + enumAttribute(ops.MBarrierWaitKindAttr, ctx, kind), + enumAttribute(ops.MBarrierScopeKindAttr, ctx, scope), + null, + location, + ); } -/// nvvm.mbarrier.try_wait.parity — loop until phase `parity` (i32) has completed, -/// suspending up to `suspend_time` (i32, cycles) per attempt. +/// nvvm.mbarrier.try_wait.parity — wait (looping in PTX) until phase `parity` (i32) +/// of the mbarrier completes, suspending up to `suspend_time` (i32) cycles per try. pub fn mbarrier_try_wait_parity( ctx: *mlir.Context, ptr: *const mlir.Value, @@ -207,117 +117,79 @@ pub fn mbarrier_try_wait_parity( suspend_time: *const mlir.Value, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.mbarrier.try_wait.parity", .{ - .operands = .{ .flat = &.{ ptr, parity, suspend_time } }, - .attributes = &.{.named(ctx, "useIntrinsic", .boolean(ctx, false))}, - .location = location, - }); + return ops.mbarrier_try_wait_parity(ctx, ptr, parity, suspend_time, .boolean(ctx, false), location); } -/// nvvm.fence.mbarrier.init — make mbarrier initialization visible to the async proxy. +/// nvvm.fence.mbarrier.init — make mbarrier initialization visible to the cluster. pub fn fence_mbarrier_init(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.fence.mbarrier.init", .{ .location = location }); + return ops.fence_mbarrier_init(ctx, location); } -/// nvvm.fence.proxy — order memory accesses across proxies (e.g. generic writes -/// before asynchronous TMA / tcgen05 reads of shared memory). +/// nvvm.fence.proxy — order memory accesses across proxies. pub fn fence_proxy(ctx: *mlir.Context, kind: ProxyKind, space: SharedSpace, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.fence.proxy", .{ - .attributes = &.{ - .named(ctx, "kind", kind.attribute(ctx)), - .named(ctx, "space", space.attribute(ctx)), - }, - .location = location, - }); + return ops.fence_proxy(ctx, enumAttribute(ops.ProxyKindAttr, ctx, kind), enumAttribute(ops.SharedSpaceAttr, ctx, space), location); } // ============================================================================= // Asynchronous copies // ============================================================================= -/// nvvm.cp.async.commit.group — close the current group of cp.async copies. +/// nvvm.cp.async.commit.group — commit the pending `cp.async` copies as a group. pub fn cp_async_commit_group(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.cp.async.commit.group", .{ .location = location }); + return ops.cp_async_commit_group(ctx, location); } -/// nvvm.cp.async.wait.group — wait until at most `pending` cp.async groups are in flight. +/// nvvm.cp.async.wait.group — wait until at most `pending` `cp.async` groups are in flight. pub fn cp_async_wait_group(ctx: *mlir.Context, pending: u32, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.cp.async.wait.group", .{ - .attributes = &.{.named(ctx, "n", .int(ctx, .i32, pending))}, - .location = location, - }); + return ops.cp_async_wait_group(ctx, .int(ctx, .i32, pending), location); } -/// nvvm.cp.async.bulk.commit.group — close the current group of bulk copies (SM90+). +/// nvvm.cp.async.bulk.commit.group — commit the pending bulk (TMA) copies as a group. pub fn cp_async_bulk_commit_group(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.commit.group", .{ .location = location }); + return ops.cp_async_bulk_commit_group(ctx, location); } /// nvvm.cp.async.bulk.wait_group — wait until at most `pending` bulk groups are in /// flight; with `read`, only until their sources have been read. pub fn cp_async_bulk_wait_group(ctx: *mlir.Context, pending: u32, read: bool, location: *const mlir.Location) *mlir.Operation { - var attrs: [2]mlir.NamedAttribute = undefined; - attrs[0] = .named(ctx, "group", .int(ctx, .i32, pending)); - var len: usize = 1; - if (read) { - attrs[len] = .named(ctx, "read", .unit(ctx)); - len += 1; - } - return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.wait_group", .{ - .attributes = attrs[0..len], - .location = location, - }); + return ops.cp_async_bulk_wait_group(ctx, .int(ctx, .i32, pending), if (read) .unit(ctx) else null, location); } // ============================================================================= // tcgen05 (SM100 tensor cores and tensor memory) // ============================================================================= -/// nvvm.tcgen05.alloc — allocate `columns` (i32) TMEM columns for the warp; the base -/// address is written to `holder` (`!llvm.ptr<3>`). +/// nvvm.tcgen05.alloc — allocate `columns` (i32) tensor-memory columns and write the +/// address to `holder` (`!llvm.ptr<3>`). pub fn tcgen05_alloc(ctx: *mlir.Context, holder: *const mlir.Value, columns: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.tcgen05.alloc", .{ - .operands = .{ .flat = &.{ holder, columns } }, - .location = location, - }); + return ops.tcgen05_alloc(ctx, holder, columns, null, null, location); } -/// nvvm.tcgen05.dealloc — free `columns` (i32) TMEM columns at `taddr` (`!llvm.ptr<6>`). +/// nvvm.tcgen05.dealloc — free `columns` (i32) tensor-memory columns at `taddr` +/// (`!llvm.ptr<6>`). pub fn tcgen05_dealloc(ctx: *mlir.Context, taddr: *const mlir.Value, columns: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.tcgen05.dealloc", .{ - .operands = .{ .flat = &.{ taddr, columns } }, - .location = location, - }); + return ops.tcgen05_dealloc(ctx, taddr, columns, null, null, location); } -/// nvvm.tcgen05.fence — `tcgen05.fence::{before,after}_thread_sync`. +/// nvvm.tcgen05.fence — order tcgen05 operations around a thread synchronization. pub fn tcgen05_fence(ctx: *mlir.Context, kind: Tcgen05FenceKind, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.tcgen05.fence", .{ - .attributes = &.{.named(ctx, "kind", kind.attribute(ctx))}, - .location = location, - }); + return ops.tcgen05_fence(ctx, enumAttribute(ops.Tcgen05FenceKindAttr, ctx, kind), location); } -/// nvvm.tcgen05.wait — wait for this thread's tcgen05 loads or stores. +/// nvvm.tcgen05.wait — wait for the issued tensor-memory loads or stores. pub fn tcgen05_wait(ctx: *mlir.Context, kind: Tcgen05WaitKind, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.tcgen05.wait", .{ - .attributes = &.{.named(ctx, "kind", kind.attribute(ctx))}, - .location = location, - }); + return ops.tcgen05_wait(ctx, enumAttribute(ops.Tcgen05WaitKindAttr, ctx, kind), location); } -/// nvvm.tcgen05.commit — arrive on the mbarrier at `barrier` (`!llvm.ptr<3>`) once the -/// preceding tcgen05 MMAs complete. +/// nvvm.tcgen05.commit — arrive on the mbarrier at `barrier_ptr` when the issued +/// tcgen05 operations complete. pub fn tcgen05_commit(ctx: *mlir.Context, barrier_ptr: *const mlir.Value, group: CtaGroup, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.tcgen05.commit", .{ - .operands = .{ .flat = &.{barrier_ptr} }, - .attributes = &.{.named(ctx, "group", group.attribute(ctx))}, - .location = location, - }); + return ops.tcgen05_commit(ctx, barrier_ptr, null, null, enumAttribute(ops.CTAGroupKindAttr, ctx, group), location); } -/// nvvm.tcgen05.mma — D (TMEM, `!llvm.ptr<6>`) = A * B (+ D when `accumulate`, i1), -/// A and B given by shared-memory descriptors (i64), `idesc` the instruction descriptor (i32). +/// nvvm.tcgen05.mma — D (tensor memory, `!llvm.ptr<6>`) = A (shared-memory +/// descriptor, i64) * B (descriptor, i64) [+ D when `accumulate` (i1)], with the +/// instruction descriptor `idesc` (i32). pub fn tcgen05_mma( ctx: *mlir.Context, d: *const mlir.Value, @@ -329,89 +201,66 @@ pub fn tcgen05_mma( cta_group: CtaGroup, location: *const mlir.Location, ) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.tcgen05.mma", .{ - .operands = .{ .flat = &.{ d, a_desc, b_desc, idesc, accumulate } }, - .attributes = &.{ - .named(ctx, "kind", kind.attribute(ctx)), - .named(ctx, "ctaGroup", cta_group.attribute(ctx)), - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ 1, 1, 1, 1, 1, 0, 0 })), - }, - .location = location, - }); + return ops.tcgen05_mma( + ctx, + d, + a_desc, + b_desc, + idesc, + accumulate, + null, + null, + enumAttribute(ops.Tcgen05MMAKindAttr, ctx, kind), + enumAttribute(ops.CTAGroupKindAttr, ctx, cta_group), + null, + null, + null, + location, + ); } -/// nvvm.tcgen05.ld — load from TMEM at `taddr` (`!llvm.ptr<6>`) into `result_type` -/// (i32 or `vector`). +/// nvvm.tcgen05.ld — load `result_type` (an i32 or vector of i32) from tensor memory +/// at `taddr` (`!llvm.ptr<6>`). pub fn tcgen05_ld(ctx: *mlir.Context, taddr: *const mlir.Value, result_type: *const mlir.Type, shape: Tcgen05LdStShape, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.tcgen05.ld", .{ - .operands = .{ .flat = &.{taddr} }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{.named(ctx, "shape", shape.attribute(ctx))}, - .location = location, - }); + return ops.tcgen05_ld(ctx, taddr, null, result_type, null, enumAttribute(ops.Tcgen05LdStShapeAttr, ctx, shape), location); } -/// nvvm.tcgen05.st — store `value` (i32 or `vector`) to TMEM at `taddr`. +/// nvvm.tcgen05.st — store `value` (an i32 or vector of i32) to tensor memory at `taddr`. pub fn tcgen05_st(ctx: *mlir.Context, taddr: *const mlir.Value, value: *const mlir.Value, shape: Tcgen05LdStShape, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.tcgen05.st", .{ - .operands = .{ .flat = &.{ taddr, value } }, - .attributes = &.{.named(ctx, "shape", shape.attribute(ctx))}, - .location = location, - }); + return ops.tcgen05_st(ctx, taddr, value, null, null, enumAttribute(ops.Tcgen05LdStShapeAttr, ctx, shape), location); } // ============================================================================= // Clusters // ============================================================================= -/// nvvm.mapa — the shared-memory location `ptr` (`!llvm.ptr<3>`) in CTA `rank` (i32) -/// of the cluster, as a `result_type` (`!llvm.ptr<7>`) pointer. +/// nvvm.mapa — the address of `ptr` in the shared memory of cluster CTA `rank` (i32). pub fn mapa(ctx: *mlir.Context, ptr: *const mlir.Value, rank: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.mapa", .{ - .operands = .{ .flat = &.{ ptr, rank } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); + return ops.mapa(ctx, ptr, rank, result_type, location); } -/// nvvm.cluster.arrive — `barrier.cluster.arrive` (release), `.aligned` when every -/// thread of the warp executes it. +/// nvvm.cluster.arrive — arrive on the cluster barrier. pub fn cluster_arrive(ctx: *mlir.Context, aligned: bool, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.cluster.arrive", .{ - .attributes = if (aligned) &.{.named(ctx, "aligned", .unit(ctx))} else &.{}, - .location = location, - }); + return ops.cluster_arrive(ctx, if (aligned) .unit(ctx) else null, location); } -/// nvvm.cluster.wait — `barrier.cluster.wait` (acquire). +/// nvvm.cluster.wait — wait on the cluster barrier. pub fn cluster_wait(ctx: *mlir.Context, aligned: bool, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.cluster.wait", .{ - .attributes = if (aligned) &.{.named(ctx, "aligned", .unit(ctx))} else &.{}, - .location = location, - }); + return ops.cluster_wait(ctx, if (aligned) .unit(ctx) else null, location); } // ============================================================================= // Math // ============================================================================= -/// nvvm.ex2 — approximate 2^x (f32), flushing denormals when `ftz`. +/// nvvm.ex2 — `ex2.approx[.ftz].f32`. pub fn ex2(ctx: *mlir.Context, value: *const mlir.Value, ftz: bool, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.ex2", .{ - .operands = .{ .flat = &.{value} }, - .results = .{ .flat = &.{mlir.Type.float(ctx, .f32)} }, - .attributes = &.{.named(ctx, "ftz", .boolean(ctx, ftz))}, - .location = location, - }); + return ops.ex2(ctx, value, .float(ctx, .f32), .boolean(ctx, ftz), location); } -/// nvvm.rcp.approx.ftz.f — approximate 1/x (f32), flushing denormals. +/// nvvm.rcp.approx.ftz.f — `rcp.approx.ftz.f32`. pub fn rcp_approx_ftz(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.rcp.approx.ftz.f", .{ - .operands = .{ .flat = &.{value} }, - .results = .{ .flat = &.{mlir.Type.float(ctx, .f32)} }, - .location = location, - }); + return ops.rcp_approx_ftz_f(ctx, value, .float(ctx, .f32), location); } test { diff --git a/mlir/dialects/vector.zig b/mlir/dialects/vector.zig index 13cb6351..04e2384f 100644 --- a/mlir/dialects/vector.zig +++ b/mlir/dialects/vector.zig @@ -5,45 +5,93 @@ const mlir = @import("mlir"); const stdx = @import("stdx"); // ============================================================================= -// Enum attributes — parsed from their textual `#vector.<...>` form. +// Enum attributes // ============================================================================= -/// Matches `mlir::vector::CombiningKind` (I32EnumAttr in VectorAttributes.td). -pub const CombiningKind = enum { - add, - mul, - minui, - minsi, - minnumf, - maxui, - maxsi, - maxnumf, - @"and", - @"or", - xor, - minimumf, - maximumf, +/// Kind of combining function for contractions and reductions. +pub const CombiningKind = enum(u32) { + add = 0, + mul = 1, + minui = 2, + minsi = 3, + minnumf = 4, + maxui = 5, + maxsi = 6, + maxnumf = 7, + @"and" = 8, + @"or" = 9, + xor = 10, + minimumf = 11, + maximumf = 12, + minimumnumf = 13, + maximumnumf = 14, pub fn attribute(self: CombiningKind, ctx: *mlir.Context) *const mlir.Attribute { - var buf: [64]u8 = undefined; - const text = std.fmt.bufPrint(&buf, "#vector.kind<{s}>", .{@tagName(self)}) catch unreachable; - return mlir.Attribute.parse(ctx, text) catch - std.debug.panic("failed to parse CombiningKind '{s}'", .{text}); + return (CombiningKindAttr.get(ctx, .{ .value = self }) catch unreachable).attribute(); } }; -/// Matches `mlir::vector::IteratorType` — used by `vector.contract`. -pub const IteratorType = enum { - parallel, - reduction, +/// Iterator type of a `vector.contract` dimension. +pub const IteratorType = enum(u32) { + parallel = 0, + reduction = 1, pub fn attribute(self: IteratorType, ctx: *mlir.Context) *const mlir.Attribute { - const text = switch (self) { - .parallel => "#vector.iterator_type", - .reduction => "#vector.iterator_type", - }; - return mlir.Attribute.parse(ctx, text) catch - std.debug.panic("failed to parse IteratorType", .{}); + return (IteratorTypeAttr.get(ctx, .{ .value = self }) catch unreachable).attribute(); + } +}; + +/// `#vector.kind<...>`. +pub const CombiningKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAVectorCombiningKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "kind"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: CombiningKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirVectorCombiningKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) CombiningKind { + return @enumFromInt(c.mlirVectorCombiningKindAttrGetValue(self.ptr())); + } +}; + +/// `#vector.iterator_type<...>`. +pub const IteratorTypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAVectorIteratorType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const isA = M.isA; + pub const mnemonic = "iterator_type"; + pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return self.attribute().format(writer); + } + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: IteratorType, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirVectorIteratorTypeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn getValue(self: *const Self) IteratorType { + return @enumFromInt(c.mlirVectorIteratorTypeAttrGetValue(self.ptr())); } }; @@ -774,3 +822,29 @@ pub fn create_mask( test { std.testing.refAllDecls(@This()); } + +fn expectPrints(expected: []const u8, value: anytype) !void { + var buf: [128]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(expected, w.buffered()); +} + +test "vector enum attributes are built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + inline for (std.meta.fields(CombiningKind)) |field| { + const value: CombiningKind = @enumFromInt(field.value); + const attr = try CombiningKindAttr.get(ctx, .{ .value = value }); + try std.testing.expectEqual(value, attr.getValue()); + try std.testing.expect(value.attribute(ctx).isA(CombiningKindAttr) != null); + try expectPrints("#vector.kind<" ++ field.name ++ ">", attr); + } + inline for (std.meta.fields(IteratorType)) |field| { + const value: IteratorType = @enumFromInt(field.value); + const attr = try IteratorTypeAttr.get(ctx, .{ .value = value }); + try std.testing.expectEqual(value, attr.getValue()); + try expectPrints("#vector.iterator_type<" ++ field.name ++ ">", attr); + } +} diff --git a/mlir/dialects/vector_capi.cc b/mlir/dialects/vector_capi.cc new file mode 100644 index 00000000..0b412f66 --- /dev/null +++ b/mlir/dialects/vector_capi.cc @@ -0,0 +1,44 @@ +#include "vector_capi.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/Dialect/Vector/IR/VectorOps.h" + +namespace { + +mlir::MLIRContext *loadVector(MlirContext ctx) { + mlir::MLIRContext *context = unwrap(ctx); + if (context) context->getOrLoadDialect(); + return context; +} + +} // namespace + +bool mlirAttributeIsAVectorCombiningKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirVectorCombiningKindAttrGet(MlirContext ctx, uint32_t value) { + mlir::MLIRContext *context = loadVector(ctx); + std::optional kind = mlir::vector::symbolizeCombiningKind(value); + if (!context || !kind) return {nullptr}; + return wrap(mlir::vector::CombiningKindAttr::get(context, *kind)); +} + +uint32_t mlirVectorCombiningKindAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAVectorIteratorType(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirVectorIteratorTypeAttrGet(MlirContext ctx, uint32_t value) { + mlir::MLIRContext *context = loadVector(ctx); + std::optional type = mlir::vector::symbolizeIteratorType(value); + if (!context || !type) return {nullptr}; + return wrap(mlir::vector::IteratorTypeAttr::get(context, *type)); +} + +uint32_t mlirVectorIteratorTypeAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} diff --git a/mlir/dialects/vector_capi.h b/mlir/dialects/vector_capi.h new file mode 100644 index 00000000..7fae8e6f --- /dev/null +++ b/mlir/dialects/vector_capi.h @@ -0,0 +1,32 @@ +// C entry points for the `vector` dialect's enum attributes, which upstream's +// C API leaves out. +#ifndef ZML_MLIR_DIALECTS_VECTOR_CAPI_H +#define ZML_MLIR_DIALECTS_VECTOR_CAPI_H + +#include +#include + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Enum attributes take and return the C++ enum's integer value; `Get` returns +// a null attribute for a value outside the enum. + +// #vector.kind<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsAVectorCombiningKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirVectorCombiningKindAttrGet(MlirContext ctx, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirVectorCombiningKindAttrGetValue(MlirAttribute attr); + +// #vector.iterator_type<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsAVectorIteratorType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirVectorIteratorTypeAttrGet(MlirContext ctx, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirVectorIteratorTypeAttrGetValue(MlirAttribute attr); + +#ifdef __cplusplus +} +#endif + +#endif // ZML_MLIR_DIALECTS_VECTOR_CAPI_H diff --git a/zml/kernel.zig b/zml/kernel.zig index 2d8945e4..33c685cc 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -1064,7 +1064,8 @@ test "cute kernel emits the module the CuTe compiler takes" { const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1000 }, .{ 128, 1, 1 }); defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "module {\n func.func @add_one(") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "\"func.func\"() <{function_type = (") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "sym_name = \"add_one\"") != null); try std.testing.expect(std.mem.indexOf(u8, ir, "nvvm.reqntid = array") != null); try std.testing.expect(std.mem.indexOf(u8, ir, "cute.memref.load") != null); try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); From 08391af750dc6be08146ced32ef523b944017485 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 8 Oct 2026 12:33:34 +0200 Subject: [PATCH 765/881] zml/Compiler: remove conditionals from command buffer (#913) Due to recent changes in XLA (introduced in https://github.com/openxla/xla/commit/3cf18a01c7), the way child are counted to decide if a conditional is expressed as a graph conditional or kept outside has been changed. With the new corrected way, the body of the conditional used in the paged attention StableHLO backend was recorded as a child graph. The problem is that when we are sharded, there are collectives inside this child graph and XLA adds event wait/record around the actual collective kernel. This is explicitly forbidden because https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cuda-graphs.html#conditional-node-body-graph-requirements states: > The graph can only contain kernel nodes, empty nodes, memcpy nodes, memset nodes, child graph nodes, and conditional nodes. The temporary solution is to remove `CONDITIONAL` from the `xla_gpu_enable_command_buffer` XLA flag. In practice this reverts to the old behavior we had when the counting was done differently. Long term, I don't have a solution yet and. There is an environment variable that solves it [NCCL_GRAPH_MIXING_SUPPORT](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/env.html#nccl-graph-mixing-support) but I don't know if it's safe to disable it. Maybe we could also check if the events introduced by NCCL are actually required. --- AGENTS.md | 4 ++++ zml/Compiler.zig | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e3f5ef1e..fe0f5d9c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,3 +29,7 @@ Tests are generally inline Zig `test "..." {}` blocks near the code they cover. ## Commit & Pull Request Guidelines Recent history uses scoped, imperative commit subjects such as `zml/tensor: add Tensor.onMemory()` or `workspace: use latest version of upstreamable rules_zig`. Keep subjects concise and mention the affected area first. Pull requests should include a short description, relevant linked issue, platform impact if any, and the exact `bazel build` or `bazel test` commands run. Include screenshots only for docs or UI-visible changes. + +## Quirks + +If during build, you encounter an `AccessDenied` error, the solution is not to use the `--repo_env` bazel argument to override the zig cache. The solution is to run `sudo chmod -R 777`on `/tmp/zig-cache` and `/var/tmp/zig-cache`. This is due to the fact that all users share the same zig cache directory, and bazel runs as a different user than the one who created the cache. diff --git a/zml/Compiler.zig b/zml/Compiler.zig index a76a828e..bb0d21a7 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -937,7 +937,7 @@ fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.Load try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_update_mode", "SKIP_TEMP", upb_arena); try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_collective_kernels", "COLLECTIVE_KERNEL_ALL_REDUCE", upb_arena); // Add collectives to the default list - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES,CONDITIONAL,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); // With SKIP_TEMP, captured NCCL collectives can retain stale physical mappings // if VMM reclaims their backing under memory pressure, causing silent corruption. // Enable user buffers to assign color 1 and receive the reclaim exemption: @@ -955,7 +955,7 @@ fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.Load // Only capture all-reduce and all-gather collectives in HIP graphs; other collective types execute outside command buffers since they are unsupported in HIP graphs. try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_collectives_command_buffer_filter", "ALLREDUCE,ALLGATHER", upb_arena); // Add collectives to the default list - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES,CONDITIONAL,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); // With SKIP_TEMP, captured NCCL collectives can retain stale physical mappings // if VMM reclaims their backing under memory pressure, causing silent corruption. // Enable user buffers to assign color 1 and receive the reclaim exemption: From 728eedfbb383b27f96cdb5288e01522b3edd77cd Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Thu, 8 Oct 2026 13:40:37 +0200 Subject: [PATCH 766/881] zml/Compiler: Add COLLECTIVES_KERNEL to command buffers (#912) We don't have a single command buffer in Deepseek since `xla_gpu_experimental_use_collective_kernels=COLLECTIVE_KERNEL_ALL_REDUCE` was added --- zml/Compiler.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/Compiler.zig b/zml/Compiler.zig index bb0d21a7..200f2265 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -937,7 +937,7 @@ fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.Load try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_update_mode", "SKIP_TEMP", upb_arena); try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_collective_kernels", "COLLECTIVE_KERNEL_ALL_REDUCE", upb_arena); // Add collectives to the default list - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES_KERNEL,COLLECTIVES,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); // With SKIP_TEMP, captured NCCL collectives can retain stale physical mappings // if VMM reclaims their backing under memory pressure, causing silent corruption. // Enable user buffers to assign color 1 and receive the reclaim exemption: From 734ec462301e060c6100978eecf5ede32594f823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20Gauti=C3=A9?= <51361266+tgautie@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:22:40 +0200 Subject: [PATCH 767/881] workspace: bump ZML plugins to 202610081055.168.1.56e99704ada4 (#915) --- platforms/cpu/cpu.bzl | 6 +++--- platforms/cuda/cuda.bzl | 8 ++++---- platforms/metal/metal.bzl | 4 ++-- platforms/rocm/rocm.bzl | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 94670793..7fc024dc 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -43,21 +43,21 @@ def _cpu_plugin_impl(mctx): name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, sha256 = "5051c6719809f1dd86a83665b6bb65a4c0c8ec3be4613c8652eecf4856ab1d27", - url = "https://mirror.zml.ai/plugins/202610071230.162.1.1e1e094fa75c/zml-cpu-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610081055.168.1.56e99704ada4/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "d2ea2bce16aba06cc3f75725b1cd8c2fc72169a8ec8cb72c21637161dc79a1c0", - url = "https://mirror.zml.ai/plugins/202610071230.162.1.1e1e094fa75c/zml-cpu-darwin-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610081055.168.1.56e99704ada4/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "7e614948453f9692ca76d7014ffae9f25d9fe80836088ca92a5f764b26a6fe43", - url = "https://mirror.zml.ai/plugins/202610071230.162.1.1e1e094fa75c/zml-cpu-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610081055.168.1.56e99704ada4/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 64e733d7..451dd3c3 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -224,12 +224,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "b0bc93cd61025fe4b1da8cba1655c0e4e731a96b3fdb8d1dcd04513b969af4b7", - "url": "https://mirror.zml.ai/plugins/202610071230.162.1.1e1e094fa75c/zml-cuda-linux-amd64.tar.zst", + "sha256": "128fd48dba2db29f3ffe1b8d57282cd171f0e2b95c9739260da068b703cd9a59", + "url": "https://mirror.zml.ai/plugins/202610081055.168.1.56e99704ada4/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "a2683ab471fc349cf6315085bae60bb5b89c9abba3f043e54b62b48361f03cb4", - "url": "https://mirror.zml.ai/plugins/202610071230.162.1.1e1e094fa75c/zml-cuda-linux-arm64.tar.zst", + "sha256": "5c86f09731ec75a7779be53b195ae7e8e0bbf7b4a20c7b71cccdcf47fe83378b", + "url": "https://mirror.zml.ai/plugins/202610081055.168.1.56e99704ada4/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 0d205070..b7a0bf3f 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "a880a201b274fae35596b1b4cea1a79862dfea29a95eb22439b3b5c8f1f86767", - url = "https://mirror.zml.ai/plugins/202610071230.162.1.1e1e094fa75c/zml-metal-darwin-arm64.tar.zst", + sha256 = "bde54cd2d67d28c7ed91fa953c3b13074d40e7d73c1157e4ad4a84aa90394988", + url = "https://mirror.zml.ai/plugins/202610081055.168.1.56e99704ada4/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 9b7cc4d1..69bcf2d4 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610071230.162.1.1e1e094fa75c/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "6afe9ca497e04ecb07a3f56645f0c64170770c39c9216316d51de3afe13103ae" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610081055.168.1.56e99704ada4/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "e8d24524364816e53b3deb55ca138a152b5c6c006a853bf410e930c681c2b087" def _rocm_package_name(name): return name + _ROCM_VERSION From 3972d4a0117f965b654cc49999714699b1c887a4 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Thu, 8 Oct 2026 14:52:10 +0200 Subject: [PATCH 768/881] zml/buffer: add Buffer.HostAccessible (#899) `Buffer` placed in `.host_pinned` memory can be read by executables, but the host can't address it. `Buffer.HostAccessible` gives the host typed views over its allocations, to write inputs and read outputs in place between executions, without staging copies. Like `Buffer`, it is a value: executions read its `buffer` field or donate it through `&buffer`, and `await` then picks up the handles they replaced. - `init` takes a `Buffer.Donation`. On CPU, CUDA and ROCm, the devices holding a shard of an `.undonatable` buffer share one allocation: the first allocates it and the others read it through `createViewOfDeviceBuffer`, which now takes the memory the view belongs to; an external reference keeps the allocation alive. Elsewhere each device owns an allocation, and writes reach all of them. Each device owns its allocation of a `.donatable` buffer, so that no device reads another's output. - `View(T)` addresses a row-major range of the array: `slice`, `get`, `set`, `fill`, `fillIota`, `copyFrom`, `copyTo`, `copyFromView`, `indexOfScalar`, `indexOfAny`, and `blocks`/`reverseBlocks`, which visit runs contiguous both in memory and in global order. Reads use the first replica of a region, and writes reach every replica. `constSlice` and `edit` expose a range as one slice: in place when every shard holds the whole array densely, through a scratch slice otherwise. Addressing follows PJRT's layouts: row-major, strided, and tiled as XLA's LayoutUtil::LinearIndexForNestedTiling expands them, including TPU's `T(8,128)(2,1)` packing of bf16 and 8-bit arrays. The test reference is a direct port of that function, checked against both figures of https://openxla.org/xla/tiled_layout. Buffers whose shards all hold the whole array densely, the case on every platform but TPU, index elements directly. The buffer is built for llmd's decode loop, which touches it thousands of times per step: - Reading a host pointer from PJRT takes and releases a hold on the buffer and allocates an external reference, about 0.9 us. Pointers are read at creation, and by `await` only when execution replaced the handles; `view` panics before that `await` instead of writing to a released allocation. - The layout metadata and allocations take several KB. They are prepared once in a heap-allocated mapping, which views (24 B) and blocks borrow rather than copy. On Qwen3.8-27B in llmd (TP=2 on GB300, batch 256, 8k context), this takes host time between graph completion and the next Execute from 2.07 ms, when views copied the layout and donated buffers read their pointers in every view, to 0.27-0.30 ms per step (bpftrace probes). `initForTests` builds a buffer over host memory, for the tiled layouts and shardings that no available platform produces. `Buffer.from` and `Buffer.uninitialized` share the new `emptyShell` with `HostAccessible.init` to compute the PJRT shape, placement and layout. --- pjrt/pjrt.zig | 4 + zml/BUILD.bazel | 3 +- zml/buffer.zig | 101 +- zml/buffer/HostAccessible.zig | 2216 +++++++++++++++++++++++++++++++++ zml/zml.zig | 4 +- 5 files changed, 2288 insertions(+), 40 deletions(-) create mode 100644 zml/buffer/HostAccessible.zig diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index e6ae0acf..44a72aca 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -526,6 +526,9 @@ pub const Client = opaque { element_type: BufferType, layout: MemoryLayout, device: ?*const Device = null, + /// The memory holding `device_buffer_ptr`. Takes precedence over `device`, + /// which designates its default memory. + memory: ?*const Memory = null, on_delete_callback: *const fn (device_buffer_ptr: ?*anyopaque, ctx: ?*anyopaque) callconv(.c) void = &struct { fn call(_: ?*anyopaque, _: ?*anyopaque) callconv(.c) void {} }.call, @@ -546,6 +549,7 @@ pub const Client = opaque { .on_delete_callback = args.on_delete_callback, .on_delete_callback_arg = args.on_delete_callback_arg, .stream = @bitCast(@intFromPtr(args.stream)), + .memory = if (args.memory) |memory| memory.inner() else null, }); return @ptrCast(ret.buffer.?); } diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index d1174cca..46ed4aea 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -80,9 +80,9 @@ zig_library( "Sharding.zig", "attention.zig", "attention/attnd.zig", - "attention/flashattn.zig", "attention/cute_kernels/flashmla_sm100.zig", "attention/cute_kernels/sparse_mla.zig", + "attention/flashattn.zig", "attention/fly_kernels/sparse_mla.zig", "attention/metal_attention.zig", "attention/mosaic_tpu_kernels/ragged_attention.zig", @@ -94,6 +94,7 @@ zig_library( "attention/triton_kernels/unified_attention_oneapi.zig", "attention/triton_kernels/unified_sparse_mla.zig", "buffer.zig", + "buffer/HostAccessible.zig", "constants.zig", "dtype.zig", "exe.zig", diff --git a/zml/buffer.zig b/zml/buffer.zig index 6f7acd8c..10f0f8d1 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -21,6 +21,7 @@ const log = std.log.scoped(.zml); test { std.testing.refAllDecls(Buffer); + std.testing.refAllDecls(Buffer.HostAccessible); } /// Buffer is a multi-dimension array, whose memory is allocated on an accelerator. @@ -33,6 +34,18 @@ pub const Buffer = struct { _shape: Shape, _shards: Shards, + /// Whether executions may donate a buffer, writing their outputs into it. + /// `HostAccessible.init` uses it to decide whether devices share allocations. + pub const Donation = enum { + /// Executions only read the buffer. Unlike XLA's undonatable buffers, the + /// host may still rewrite a HostAccessible one between executions. + undonatable, + /// Executions may donate the buffer, replacing its handles with their outputs. + donatable, + }; + + pub const HostAccessible = @import("buffer/HostAccessible.zig"); + pub const MAX_NUM_SHARDS: u16 = Platform.MAX_NUM_DEVICES; pub const Shards = stdx.BoundedArray(*pjrt.Buffer, MAX_NUM_SHARDS); @@ -128,35 +141,25 @@ pub const Buffer = struct { data_: []const u8, opts: FromOptions, ) !Buffer { - // Use the PJRT shape for everything - const sh = shape_.packedShape(); - var res: Buffer = .{ - ._platform = platform, - ._shape = shape_, - ._shards = .empty, - }; - errdefer for (res._shards.slice()) |shard| { + var buffer, const metadata = emptyShell(platform, shape_); + + errdefer for (buffer._shards.slice()) |shard| { shard.deinit(platform.pjrt_api); }; stdx.debug.assert(platform.devices[0].memory(opts.memory) != null, "Device doesn't have {} memory", .{opts.memory}); - const slice = Slice.init(sh, data_); - const buffer_type = pjrtx.bufferTypeFromDtype(sh.dtype()); - - const placement = placementOrPanic(platform, sh); - const shard_dims: []const i64 = placement.shape.dims(); - const layout = platform.defaultMemoryLayout(shard_dims, sh.dtype()); + const slice = Slice.init(metadata.shape, data_); for (platform.physical_mesh.devices_in_canonical_order) |device| { const memory = platform.devices[device.id].memory(opts.memory).?; const args: pjrt.Client.BufferFromHostBufferArgs = .{ // Change for each device - .data = placement.shardPtr(device.coords, slice), + .data = metadata.placement.shardPtr(device.coords, slice), .dst = .{ .memory = memory.pjrt_memory }, // Constant across devices - .layout = layout, - .dims = shard_dims, - .buffer_type = buffer_type, + .layout = metadata.layout, + .dims = metadata.placement.shape.dims(), + .buffer_type = metadata.ty, .byte_strides = slice.byte_strides.constSlice(), .host_buffer_semantics = .ImmutableUntilTransferCompletes, }; @@ -164,14 +167,14 @@ pub const Buffer = struct { const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); if (event) |ev| ev.deinit(platform.pjrt_api); - res._shards.appendAssumeCapacity(pjrt_buffer); + buffer._shards.appendAssumeCapacity(pjrt_buffer); } if (opts.wait) { - try res.await(io); + try buffer.await(io); } - return res; + return buffer; } /// Copies the given Zig bytes to the accelerator memory and @@ -217,21 +220,12 @@ pub const Buffer = struct { opts: UnitializedOptions, ) !Buffer { std.log.debug("uninitialized {f}", .{shape_}); - const sh = shape_.packedShape(); - var res: Buffer = .{ - ._platform = platform, - ._shape = shape_, - ._shards = .empty, - }; - errdefer for (res._shards.slice()) |shard| { + var buffer, const metadata = emptyShell(platform, shape_); + errdefer for (buffer._shards.slice()) |shard| { shard.deinit(platform.pjrt_api); }; stdx.debug.assert(platform.devices[0].memory(opts.memory) != null, "Device doesn't have {} memory", .{opts.memory}); - const element_type = pjrtx.bufferTypeFromDtype(sh.dtype()); - const placement = placementOrPanic(platform, sh); - const shard_dims: []const i64 = placement.shape.dims(); - const layout = platform.defaultMemoryLayout(shard_dims, sh.dtype()); for (platform.physical_mesh.devices_in_canonical_order) |device| { const memory = platform.devices[device.id].memory(opts.memory).?; @@ -239,16 +233,16 @@ pub const Buffer = struct { // Change for each device .dst = .{ .memory = memory.pjrt_memory }, // Constant across devices - .layout = layout, - .dims = shard_dims, - .element_type = element_type, + .layout = metadata.layout, + .dims = metadata.placement.shape.dims(), + .element_type = metadata.ty, }; const shard_buffer = try platform.pjrt_client.createUninitializedBuffer(platform.pjrt_api, args); - res._shards.appendAssumeCapacity(shard_buffer); + buffer._shards.appendAssumeCapacity(shard_buffer); } - return res; + return buffer; } /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. @@ -404,6 +398,39 @@ test "device round-trip" { } } +const PjrtBufferMetadata = struct { + shape: Shape, + ty: pjrt.BufferType, + placement: Sharding.Placement, + layout: pjrt.MemoryLayout, +}; + +/// Inline because the layout may point to stack memory, as with +/// `Platform.defaultMemoryLayout`. Outside `Buffer` so that +/// `Buffer.HostAccessible` can use it without it being part of the API: +/// zml.zig only exports `Buffer` from this file. +pub inline fn emptyShell(platform: *const Platform, shape_: Shape) struct { Buffer, PjrtBufferMetadata } { + const buf: Buffer = .{ + ._platform = platform, + ._shape = shape_, + ._shards = .empty, + }; + + // Shards use the PJRT shape, which packs sub-byte elements. + const packed_shape = shape_.packedShape(); + const placement = placementOrPanic(platform, packed_shape); + + return .{ + buf, + .{ + .shape = packed_shape, + .ty = pjrtx.bufferTypeFromDtype(packed_shape.dtype()), + .placement = placement, + .layout = platform.defaultMemoryLayout(placement.shape.dims(), packed_shape.dtype()), + }, + }; +} + fn placementOrPanic(platform: *const Platform, shape: Shape) Sharding.Placement { var sh = shape; if (sh._sharding.mesh == null) sh._sharding.mesh = platform.replicated_mesh; diff --git a/zml/buffer/HostAccessible.zig b/zml/buffer/HostAccessible.zig new file mode 100644 index 00000000..212c460d --- /dev/null +++ b/zml/buffer/HostAccessible.zig @@ -0,0 +1,2216 @@ +//! A `Buffer` in `.host_pinned` memory that the host reads and writes in place +//! through views, which a plain `Buffer` placed there doesn't offer. Views +//! address elements as PJRT stores them, with sub-byte elements packed in bytes. +//! The host may only access the buffer between executions. Host pointers are +//! read at creation, and by `await` when an execution replaced the handles. +//! Executions must preserve shape, sharding and layout. +//! +//! Like `Buffer`, it is a value. Pass `buffer` to an execution to read it as +//! an input, or `&buffer` to donate it as an output, then `await` the +//! HostAccessible whose `buffer` the execution replaced. Copies share the +//! mapping, and a copy whose handles the execution did not replace panics on +//! `view` rather than write to an allocation the execution may have released. +const std = @import("std"); +const builtin = @import("builtin"); + +const pjrt = @import("pjrt"); +const stdx = @import("stdx"); + +const Buffer = @import("../buffer.zig").Buffer; +const emptyShell = @import("../buffer.zig").emptyShell; +const Platform = @import("../platform.zig").Platform; +const Shape = @import("../shape.zig").Shape; +const Sharding = @import("../Sharding.zig"); +const testing = @import("../testing.zig"); + +const HostAccessible = @This(); + +/// The device side, with one PJRT buffer per device, for executions to read +/// or donate. +buffer: Buffer, +/// What views read, heap allocated so views can borrow it while the +/// HostAccessible moves. +mapping: *Mapping, + +/// The host side of the buffer. Views and blocks copy nothing from it: the +/// layout and shard placement are prepared once at creation and take several +/// KB, and copying them into every view, slice and block made memcpy most of +/// llmd's decode thread. +const Mapping = struct { + /// The PJRT shape, which packs sub-byte elements in bytes, unlike + /// `buffer._shape`. Views address its elements. + packed_shape: Shape, + shard_shape: Shape, + layout_metadata: LayoutMetadata, + allocations: HostPinnedAllocations, + /// Every shard holds the whole array densely, so element `i` sits at + /// `i * element_size` in each of them, as in replicated buffers on every + /// platform but TPU. `get`, `set` and blocks then skip locating elements + /// through the origins and the layout. + contiguous: bool, + + /// Takes ownership of the allocations, on success only. + fn create( + allocator: std.mem.Allocator, + shape_: Shape, + shard_shape: Shape, + layout: pjrt.MemoryLayout, + allocations: HostPinnedAllocations, + ) !*Mapping { + const mapping = try allocator.create(Mapping); + errdefer allocator.destroy(mapping); + + const layout_metadata: LayoutMetadata = .init(shape_, shard_shape, layout); + + mapping.* = .{ + .packed_shape = shape_, + .shard_shape = shard_shape, + .layout_metadata = layout_metadata, + .allocations = allocations, + .contiguous = layout_metadata.addressing == .row_major and + allocations.regions.len == 1 and + shard_shape.count() == shape_.count(), + }; + + return mapping; + } +}; + +/// The host allocations behind the buffer's shards. Devices that share an +/// allocation map to one entry, and entries holding the same region of the +/// array are its replicas. Entries stay in region order, so a region's +/// replicas are a range of every field. +const HostPinnedAllocations = struct { + entries: std.MultiArrayList(Allocation) = .empty, + /// The regions the entries hold, in order of first appearance. + regions: stdx.BoundedArray(Region, Platform.MAX_NUM_DEVICES) = .empty, + + const empty: HostPinnedAllocations = .{}; + + const Allocation = struct { + /// Global coordinates of the allocation's first element. + origin: Coordinates, + /// Read from `handle`. Blocks borrow their region's range. + ptr: [*]u8, + /// The shard owning the allocation, which must destroy it: an index + /// into `buffer._shards`, whose handles execution may replace. + shard_index: usize, + /// The handle `ptr` was read from. Execution replaces the handles of + /// donated buffers with its outputs, which may not reuse the allocation. + handle: *pjrt.Buffer, + /// Whether `handle` holds an external reference, which keeps the + /// allocation alive for the device views of other devices. + shared: bool = false, + }; + + /// `entries[start..][0..len]` hold the region's replicas, in the order they + /// were added. Reads use the first. + const Region = struct { start: u8, len: u8 }; + + fn deinit(self: *HostPinnedAllocations, allocator: std.mem.Allocator) void { + self.entries.deinit(allocator); + } + + /// Add an allocation to the replicas of its region, within the + /// reserved capacity. + fn add(self: *HostPinnedAllocations, shard_shape: Shape, allocation: Allocation) void { + const region_index = self.findRegion(allocation.origin, shard_shape) orelse { + self.regions.appendAssumeCapacity(.{ .start = @intCast(self.entries.len), .len = 1 }); + self.entries.appendAssumeCapacity(allocation); + return; + }; + const region = &self.regions.slice()[region_index]; + // Keep each region's entries together by shifting the regions after it. + self.entries.insertAssumeCapacity(region.start + region.len, allocation); + region.len += 1; + for (self.regions.slice()[region_index + 1 ..]) |*next| next.start += 1; + } + + /// The index of the region holding the shards that start at `origin`. + fn findRegion(self: *const HostPinnedAllocations, origin: Coordinates, shard_shape: Shape) ?usize { + const origins = self.entries.items(.origin); + for (self.regions.constSlice(), 0..) |region, i| { + if (origin.eqlWithin(origins[region.start], shard_shape)) return i; + } + return null; + } + + /// The host pointers of a region's replicas. + fn replicas(self: *const HostPinnedAllocations, region: Region) []const [*]u8 { + return self.entries.items(.ptr)[region.start..][0..region.len]; + } + + /// Read the host pointers of the allocations from their current handles. + /// Each read takes and releases a hold on the PJRT buffer and allocates an + /// external reference, about 0.9 us: llmd would make thousands per step if + /// views read them, so they are read at creation and by `await` only. + fn refreshPointers(self: *HostPinnedAllocations, api: *const pjrt.Api, handles: []const *pjrt.Buffer) !void { + const shard_indices = self.entries.items(.shard_index); + // Read every pointer before replacing any, so a failure keeps the previous ones. + var ptrs: [Platform.MAX_NUM_DEVICES][*]u8 = undefined; + for (shard_indices, ptrs[0..shard_indices.len]) |shard_index, *ptr| { + ptr.* = @ptrCast(try handles[shard_index].opaqueDeviceMemoryDataPointer(api)); + } + @memcpy(self.entries.items(.ptr), ptrs[0..shard_indices.len]); + for (self.entries.items(.handle), shard_indices) |*handle, shard_index| handle.* = handles[shard_index]; + } + + /// Whether the host pointers were read from the current handles, which + /// execution replaces for donated buffers. + fn pointersAreCurrent(self: *const HostPinnedAllocations, handles: []const *pjrt.Buffer) bool { + // Host memory in tests has no handles: its pointers never change. + if (handles.len == 0) return true; + for (self.entries.items(.handle), self.entries.items(.shard_index)) |handle, shard_index| { + if (handle != handles[shard_index]) return false; + } + return true; + } + + /// Destroy every handle, the device views before the allocations they borrow. + fn deinitHandles(self: *const HostPinnedAllocations, api: *const pjrt.Api, handles: []const *pjrt.Buffer) void { + const shard_indices = self.entries.items(.shard_index); + for (handles, 0..) |handle, index| { + if (std.mem.indexOfScalar(usize, shard_indices, index) == null) handle.deinit(api); + } + for (shard_indices, self.entries.items(.shared)) |shard_index, shared| { + const handle = handles[shard_index]; + // `shared` is set by the one successful increase, and only undonatable + // buffers share allocations, whose handles execution never replaces. + if (shared) handle.decreaseExternalReferenceCount(api) catch |err| { + std.debug.panic("shared HostAccessible allocation has no reference to release, was it donated? {}", .{err}); + }; + handle.deinit(api); + } + } +}; + +/// Host memory standing in for PJRT buffers, to test layouts and shardings +/// that no available platform produces: every platform but TPU uses dense +/// row-major layouts, and CI has few devices. `origins` holds the row-major +/// index in `shape_` of each allocation's first element, and `ptrs` one pointer +/// per origin. Copies the layout, the origins and the pointers, and borrows the +/// storage. +pub fn initForTests( + allocator: std.mem.Allocator, + shape_: Shape, + shard_shape: Shape, + layout: pjrt.MemoryLayout, + origins: []const usize, + ptrs: []const [*]u8, +) !HostAccessible { + if (!builtin.is_test) @compileError("HostAccessible.initForTests is only available in tests"); + + var allocations: HostPinnedAllocations = .empty; + errdefer allocations.deinit(allocator); + + try allocations.entries.ensureTotalCapacity(allocator, origins.len); + // Without PJRT buffers, there are no handles to read pointers from or destroy. + for (origins, ptrs) |origin, ptr| { + allocations.add( + shard_shape, + .{ + // An empty array has no element to locate the origin of. + .origin = if (shape_.count() == 0) .zero else .unflatten(shape_, origin), + .ptr = ptr, + .shard_index = undefined, + .handle = undefined, + }, + ); + } + + return .{ + .buffer = .{ ._platform = undefined, ._shape = shape_, ._shards = .empty }, + .mapping = try .create(allocator, shape_, shard_shape, layout, allocations), + }; +} + +/// With `.undonatable`, devices holding the same shard share one allocation on +/// CPU, CUDA and ROCm: the first allocates it, and the others read it through +/// device views. Elsewhere each device owns an allocation, and writes reach +/// every one of them. Execution never replaces the handles of an undonatable +/// buffer, so `await` never reads its host pointers again. +/// With `.donatable`, each device owns an allocation: writing a shared one +/// would expose the output of a device to the reads of another. +pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + shape_: Shape, + donation: Buffer.Donation, +) !HostAccessible { + const can_share_allocations = switch (platform.target) { + .cpu, .cuda, .rocm => donation == .undonatable, + else => false, + }; + const api = platform.pjrt_api; + + var buffer, const metadata = emptyShell(platform, shape_); + const devices = platform.physical_mesh.devices_in_canonical_order; + + var allocations: HostPinnedAllocations = .empty; + errdefer allocations.deinit(allocator); + + try allocations.entries.ensureTotalCapacity(allocator, devices.len); + errdefer allocations.deinitHandles(api, buffer._shards.constSlice()); + + for (devices, 0..) |device, shard_index| { + const memory = platform.devices[device.id].memory(.host_pinned).?.pjrt_memory; + const origin: Coordinates = .ofShard(&metadata.placement, device); + const shared_region = if (can_share_allocations) allocations.findRegion(origin, metadata.placement.shape) else null; + + if (shared_region) |region_index| { + // Devices sharing allocations add no replicas: the region has a single entry. + const entry = allocations.regions.get(region_index).start; + const existing = allocations.entries.get(entry); + if (!existing.shared) { + try existing.handle.increaseExternalReferenceCount(api); + allocations.entries.items(.shared)[entry] = true; + } + buffer._shards.appendAssumeCapacity(try platform.pjrt_client.createViewOfDeviceBuffer(api, .{ + .device_buffer_ptr = try existing.handle.opaqueDeviceMemoryDataPointer(api), + .dims = metadata.placement.shape.dims(), + .element_type = metadata.ty, + .layout = try existing.handle.memoryLayout(api), + // The backing host allocation is accessible across devices + // (portable pinned memory on CUDA/ROCm). This memory space + // associates the view with the current device rather than + // the device that originally allocated the memory. + .memory = memory, + })); + } else { + const handle = try platform.pjrt_client.createUninitializedBuffer(api, .{ + .dims = metadata.placement.shape.dims(), + .element_type = metadata.ty, + .layout = metadata.layout, + .dst = .{ .memory = memory }, + }); + buffer._shards.appendAssumeCapacity(handle); + // The pointer is read once the creation completes, after `await` below. + allocations.add(metadata.placement.shape, .{ + .origin = origin, + .ptr = undefined, + .shard_index = shard_index, + .handle = handle, + }); + } + } + + try buffer.await(io); + try allocations.refreshPointers(api, buffer._shards.constSlice()); + + return .{ + .buffer = buffer, + .mapping = try .create(allocator, metadata.shape, metadata.placement.shape, metadata.layout, allocations), + }; +} + +/// Execution must be done with the buffer. Copies must not be used after. +pub fn deinit(self: *const HostAccessible, allocator: std.mem.Allocator) void { + // Host memory in tests has neither handles nor a platform. + if (self.buffer._shards.len != 0) self.mapping.allocations.deinitHandles(self.buffer._platform.pjrt_api, self.buffer._shards.constSlice()); + self.mapping.allocations.deinit(allocator); + allocator.destroy(self.mapping); +} + +/// Wait for execution to finish, and read the host pointers of handles that +/// execution replaced. Take new views to access the output. +pub fn await(self: *HostAccessible, io: std.Io) !void { + try self.buffer.await(io); + const handles = self.buffer._shards.constSlice(); + const allocations = &self.mapping.allocations; + if (!allocations.pointersAreCurrent(handles)) try allocations.refreshPointers(self.buffer._platform.pjrt_api, handles); +} + +/// Borrows the mapping. Executions using the buffer must have completed, and +/// `await` must follow any that replaced the handles: `view` panics before +/// then rather than write to an allocation the execution may have released. +/// The view and its blocks are valid until the next execution starts or the +/// HostAccessible is deinitialized. +pub fn view(self: *const HostAccessible, comptime T: type) View(T) { + const mapping = self.mapping; + std.debug.assert(@sizeOf(T) == mapping.shard_shape.dtype().sizeOf()); + stdx.debug.assert(mapping.allocations.pointersAreCurrent(self.buffer._shards.constSlice()), "await the HostAccessible after execution replaces its handles", .{}); + return .{ .mapping = mapping, .len = mapping.packed_shape.count() }; +} + +/// Prints every element, read through a view: the same rules as `view` apply. +pub fn format(self: *const HostAccessible, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("HostAccessible({f}) {{", .{self.mapping.packed_shape}); + switch (self.mapping.shard_shape.dtype()) { + inline else => |dt| { + if (comptime dt.bitSizeOf() < 8) unreachable; + const values = self.view(dt.toZigType()); + var blocks = values.blocks(); + while (blocks.next()) |block| { + for (block.firstReplica()) |value| try writer.print(" {any}", .{value}); + } + }, + } + try writer.writeAll(" }"); +} + +/// A range of the array's elements in row-major order, borrowed from a +/// HostAccessible. Reads use the first replica, and writes reach every replica. +pub fn View(comptime T: type) type { + return struct { + mapping: *const Mapping, + start: usize = 0, + len: usize, + + const Self = @This(); + + /// A view without elements, which never accesses the buffer. + pub const empty: Self = .{ + .mapping = undefined, + .len = 0, + }; + + /// Select `len_` elements starting at `start`, or the rest of the view for null. + pub fn slice(self: Self, start: usize, len_: ?usize) Self { + std.debug.assert(start <= self.len); + const len = len_ orelse self.len - start; + std.debug.assert(len <= self.len - start); + var result = self; + result.start += start; + result.len = len; + return result; + } + + // Single elements are addressed directly, without slicing and iterating. + pub fn get(self: Self, index: usize) T { + const element = self.locateElement(index); + return itemAt(self.mapping.allocations.replicas(element.region)[0], element.byte_offset).*; + } + + pub fn set(self: Self, index: usize, value: T) void { + const element = self.locateElement(index); + for (self.mapping.allocations.replicas(element.region)) |ptr| itemAt(ptr, element.byte_offset).* = value; + } + + fn locateElement(self: *const Self, index: usize) struct { region: HostPinnedAllocations.Region, byte_offset: usize } { + std.debug.assert(index < self.len); + if (self.mapping.contiguous) { + return .{ .region = self.mapping.allocations.regions.get(0), .byte_offset = (self.start + index) * @sizeOf(T) }; + } + const location = self.locate(.unflatten(self.mapping.packed_shape, self.start + index)); + return .{ .region = location.region, .byte_offset = self.mapping.layout_metadata.byteOffset(location.offset, location.local) }; + } + + pub fn fill(self: Self, value: T) void { + var iterator = self.blocks(); + while (iterator.next()) |block| { + for (0..block.replicas.len) |i| @memset(block.replica(i), value); + } + } + + /// Fill with `first + i * step`, where `i` is relative to this view. + /// `T` must be an integer type holding every `i` and value; safe builds check it. + pub fn fillIota(self: Self, first: T, step: T) void { + var iterator = self.blocks(); + while (iterator.next()) |block| { + const items = block.firstReplica(); + // TODO: SIMD + for (items, block.offset..) |*value, i| value.* = first + @as(T, @intCast(i)) * step; + for (1..block.replicas.len) |i| @memcpy(block.replica(i), items); + } + } + + pub fn copyFrom(self: Self, values: []const T) void { + std.debug.assert(values.len == self.len); + var iterator = self.blocks(); + while (iterator.next()) |block| { + for (0..block.replicas.len) |i| @memcpy(block.replica(i), values[block.offset..][0..block.len]); + } + } + + pub fn copyTo(self: Self, values: []T) void { + std.debug.assert(values.len == self.len); + var iterator = self.blocks(); + while (iterator.next()) |block| { + @memcpy(values[block.offset..][0..block.len], block.firstReplica()); + } + } + + /// The index of the first element equal to `value`, searching each + /// block with `std.mem.indexOfScalar`. + pub fn indexOfScalar(self: Self, value: T) ?usize { + var iterator = self.blocks(); + while (iterator.next()) |block| { + if (std.mem.indexOfScalar(T, block.firstReplica(), value)) |index| return block.offset + index; + } + return null; + } + + /// The index of the first element equal to any of `values`, searching + /// each block with `std.mem.indexOfAny`. + pub fn indexOfAny(self: Self, values: []const T) ?usize { + var iterator = self.blocks(); + while (iterator.next()) |block| { + if (std.mem.indexOfAny(T, block.firstReplica(), values)) |index| return block.offset + index; + } + return null; + } + + /// Source and destination must not overlap. + pub fn copyFromView(self: Self, src: Self) void { + std.debug.assert(src.len == self.len); + var iterator = self.blocks(); + while (iterator.next()) |block| { + const items = block.firstReplica(); + src.slice(block.offset, block.len).copyTo(items); + for (1..block.replicas.len) |i| @memcpy(block.replica(i), items); + } + } + + /// The elements as one read-only slice, for reads that index or zip + /// several views. Contiguous buffers, such as replicated ones on every + /// platform but TPU, return their first replica in place; others + /// copy into `scratch`, which must hold `len` elements on every + /// platform. The slice is valid until the next execution, and + /// writes must go through the view. + pub fn constSlice(self: Self, scratch: []T) []const T { + std.debug.assert(scratch.len >= self.len); + if (self.inPlace()) |block| return block.firstReplica(); + self.copyTo(scratch[0..self.len]); + return scratch[0..self.len]; + } + + /// A slice to write the elements into, then `commit`, for writes + /// that compute elements in place. Contiguous buffers hand out their + /// first replica; others hand out `scratch`, which must hold `len` + /// elements on every platform. + pub fn edit(self: Self, scratch: []T) Edit { + std.debug.assert(scratch.len >= self.len); + return .{ + .view = self, + .items = if (self.inPlace()) |block| block.firstReplica() else scratch[0..self.len], + }; + } + + pub const Edit = struct { + view: Self, + /// Holds the previous values or garbage: write every element. + items: []T, + + /// Make the writes reach every replica: copy the first replica to + /// the others, or `items` into the view. + pub fn commit(self: Edit) void { + const block = self.view.inPlace() orelse return self.view.copyFrom(self.items); + for (1..block.replicas.len) |i| @memcpy(block.replica(i), self.items); + } + }; + + /// The view as a single block, when the buffer is contiguous. + fn inPlace(self: Self) ?Block { + if (self.len == 0 or !self.mapping.contiguous) return null; + var iterator = self.blocks(); + return iterator.next().?; + } + + /// Visit runs of consecutive logical elements in global row-major + /// order, without padding, grouping replicas of each logical range. + /// Shards must cover the view with disjoint regions or exact replicas. + pub fn blocks(self: Self) BlockIterator { + return .init(self); + } + + /// Visit the same blocks in reverse order. Elements within each + /// replica slice remain in ascending logical order. + pub fn reverseBlocks(self: Self) ReverseBlockIterator { + return .init(self); + } + + /// A run of elements that are consecutive in the view's row-major order and + /// contiguous in memory, so each replica holds them as a plain slice. + /// Writes must reach every replica; reads use the first. Borrows the buffer's + /// host pointers, like the view it comes from. + pub const Block = struct { + /// Logical position of the first element, relative to this view. + offset: usize, + len: usize, + /// Byte offset of the first element in the allocation of every replica. + byte_offset: usize, + /// Host pointer of each replica's allocation. + replicas: []const [*]u8, + + /// The block's elements in its `i`-th replica. + pub fn replica(self: Block, i: usize) []T { + const ptr: [*]T = @ptrCast(@alignCast(self.replicas[i] + self.byte_offset)); + return ptr[0..self.len]; + } + + /// The elements in the first replica, which reads use. + pub fn firstReplica(self: Block) []T { + return self.replica(0); + } + }; + + pub const BlockIterator = OrderedBlockIterator(false); + pub const ReverseBlockIterator = OrderedBlockIterator(true); + + fn OrderedBlockIterator(comptime reverse: bool) type { + return struct { + view: Self, + /// Next logical offset, or one past it for reverse traversal. + offset: usize, + /// Coordinates of the next block's first (or reverse last) element. + global: Coordinates, + + const Iterator = @This(); + + fn init(v: Self) Iterator { + return .{ + .view = v, + .offset = if (reverse) v.len else 0, + .global = if (v.len == 0 or v.mapping.contiguous) .zero else .unflatten( + v.mapping.packed_shape, + v.start + (if (reverse) v.len - 1 else 0), + ), + }; + } + + pub fn next(self: *Iterator) ?Block { + const v = &self.view; + if (if (reverse) self.offset == 0 else self.offset == v.len) return null; + if (v.mapping.contiguous) { + // The whole view is one run in every replica. + self.offset = if (reverse) 0 else v.len; + return v.blockAt(v.mapping.allocations.regions.get(0), 0, v.len, v.start * @sizeOf(T)); + } + const location = v.locate(self.global); + const bounds = v.blockBounds(location.offset); + const len = if (reverse) + @min(location.offset + 1 - bounds.start, self.offset) + else + @min(bounds.end - location.offset, v.len - self.offset); + // A block is physically contiguous, so its last element's + // address also gives its start without decoding coordinates again. + const byte_offset = v.mapping.layout_metadata.byteOffset(location.offset, location.local) - + (if (reverse) (len - 1) * v.mapping.layout_metadata.element_size else 0); + const offset = if (reverse) self.offset - len else self.offset; + self.offset = if (reverse) offset else offset + len; + if (if (reverse) self.offset != 0 else self.offset != v.len) { + self.global.move(reverse, v.mapping.packed_shape, len); + } + return v.blockAt(location.region, offset, len, byte_offset); + } + }; + } + + const Bounds = struct { start: usize, end: usize }; + + /// The block containing a shard-local element: elements contiguous in both + /// physical memory and global row-major order. Bounds are shard-local + /// row-major indices, not physical memory offsets. + /// + /// Example: an unsharded [2,5] array with row-major 2x2 tiles. + /// + /// 1. Logical indices, with tile boundaries. `_` denotes padding: + /// ```text + /// tile 0 tile 1 tile 2 + /// +-----+ +-----+ +-----+ + /// | 0 1 | | 2 3 | | 4 _ | + /// | 5 6 | | 7 8 | | 9 _ | + /// +-----+ +-----+ +-----+ + /// ``` + /// 2. Tiles are stored consecutively, each in row-major order: + /// ```text + /// [0 1 5 6] [2 3 7 8] [4 _ 9 _] + /// ``` + /// Thus 0 and 1 are adjacent in memory, but 1 and 2 are not. + /// `block_size = 2` is the maximum contiguous logical run. + /// + /// 3. Align runs within each logical row, clipping before padding: + /// ```text + /// last_dim_size = 5 + /// alignment_size = max(5, 2) = 5 + /// + /// alignment_start blocks of logical elements row end + /// 0 [0 1] [2 3] [4] 5 + /// 5 [5 6] [7 8] [9] 10 + /// actual block sizes: 2 2 1 + /// + /// local alignment_start start end returned bounds + /// 4 0 4 5 [4,5) + /// 5 5 5 7 [5,7) + /// 6 5 5 7 [5,7) + /// 9 5 9 10 [9,10) + /// ``` + /// For local=6, start = 6 - (6-5)%2 = 5, end = min(5+2, 5+5) = 7. + /// For local=4, end = min(4+2, 0+5) = 5: the final run is shortened. + /// + /// Without tiles, the same dense row-major array has block_size=10 + /// and alignment_size=10: one block [0,10) spanning both rows. + fn blockBounds(self: *const Self, local: usize) Bounds { + const block_size = self.mapping.layout_metadata.block_size; + // Runs shorter than a row align within that row. Larger runs + // span a whole number of rows, so alignment uses the run itself. + const alignment_size = self.mapping.layout_metadata.alignment_size; + const alignment_start = local - local % alignment_size; + // Round the position relative to alignment_start down to a run boundary. + const start = local - (local - alignment_start) % block_size; + // A row's final run may be shorter; stop at the alignment interval's end. + return .{ .start = start, .end = @min(start + block_size, alignment_start + alignment_size) }; + } + + fn blockAt(self: *const Self, region: HostPinnedAllocations.Region, offset: usize, len: usize, byte_offset: usize) Block { + return .{ + .offset = offset, + .len = len, + .byte_offset = byte_offset, + .replicas = self.mapping.allocations.replicas(region), + }; + } + + fn itemAt(ptr: [*]u8, byte_offset: usize) *T { + return @ptrCast(@alignCast(ptr + byte_offset)); + } + + const Location = struct { + region: HostPinnedAllocations.Region, + /// Coordinates of the element within its shard. + local: Coordinates, + /// Row-major index of the element within its shard. + offset: usize, + }; + + /// Locate the region holding an element, retaining its shard-local + /// coordinates for physical addressing. + fn locate(self: *const Self, global: Coordinates) Location { + const origins = self.mapping.allocations.entries.items(.origin); + for (self.mapping.allocations.regions.constSlice()) |region| { + const origin = origins[region.start]; + var local: Coordinates = .zero; + var offset: usize = 0; + for (self.mapping.shard_shape.dims(), 0..) |dim_, axis| { + const dim: usize = @intCast(dim_); + if (global.axes[axis] < origin.axes[axis] or global.axes[axis] - origin.axes[axis] >= dim) break; + local.axes[axis] = global.axes[axis] - origin.axes[axis]; + offset = offset * dim + local.axes[axis]; + } else { + return .{ .region = region, .local = local, .offset = offset }; + } + } + @panic("The union of all regions should cover the global shape."); + } + }; +} + +/// A position in an array, one coordinate per axis. Axes past the rank stay zero. +const Coordinates = struct { + axes: [Shape.MAX_RANK]usize, + + const zero: Coordinates = .{ .axes = @splat(0) }; + + /// The global coordinates of the first element of a device's shard. + fn ofShard(placement: *const Sharding.Placement, device: Sharding.Device) Coordinates { + var result: Coordinates = .zero; + for (placement.slices(device.coords).constSlice(), 0..) |slice, axis| result.axes[axis] = @intCast(slice.start); + return result; + } + + /// The coordinates of the element at `offset` of `shape`, in row-major order. + fn unflatten(shape_: Shape, offset: usize) Coordinates { + var result: Coordinates = .zero; + var remaining = offset; + var axis = shape_.rank(); + while (axis > 0) { + axis -= 1; + const dim: usize = @intCast(shape_.dim(axis)); + result.axes[axis] = remaining % dim; + remaining /= dim; + } + return result; + } + + /// Whether the coordinates match on the axes of `shape`. + fn eqlWithin(self: Coordinates, other: Coordinates, shape_: Shape) bool { + return std.mem.eql(usize, self.axes[0..shape_.rank()], other.axes[0..shape_.rank()]); + } + + /// Advance in row-major order, decoding only axes that carry or borrow. + /// The resulting coordinates must still be within the shape. + fn move(self: *Coordinates, comptime reverse: bool, shape_: Shape, count: usize) void { + const axes = &self.axes; + var carry = count; + var axis = shape_.rank(); + while (carry != 0 and axis > 0) { + axis -= 1; + const dim: usize = @intCast(shape_.dim(axis)); + if (reverse) { + if (carry <= axes[axis]) { + axes[axis] -= carry; + return; + } + carry -= axes[axis] + 1; + axes[axis] = dim - 1 - carry % dim; + } else { + const available = dim - axes[axis]; + if (carry < available) { + axes[axis] += carry; + return; + } + carry -= available; + axes[axis] = carry % dim; + } + carry = carry / dim + 1; + } + std.debug.assert(carry == 0); + } +}; + +/// Physical addressing and logical block boundaries prepared once per HostAccessible. +/// All metadata is stored by value; the source layout is needed only during init. +const LayoutMetadata = struct { + rank: u4, + element_size: usize, + block_size: usize, + alignment_size: usize, + addressing: Addressing, + + const Addressing = union(enum) { + row_major, + strided: ByteStrides, + tiled: Tiled, + + const ByteStrides = [Shape.MAX_RANK]usize; + + const Tiled = struct { + /// Unit and padding dimensions always hold index 0 and are left out. + digits: stdx.BoundedArray(Digit, Shape.MAX_RANK + pjrt.DefaultMemoryLayout.MAX_TILE_DIMS) = .empty, + + /// One physical dimension of a tiled layout, as a digit of one coordinate. + const Digit = struct { + byte_stride: usize, + /// Divides the coordinate, unless `div_shift` replaces it for a power of two. + divisor: usize, + /// A mask when `is_mask` is set; otherwise a general modulus. + modulus: usize, + axis: u3, + div_shift: ?std.math.Log2Int(usize), + is_mask: bool, + + fn init(axis: u3, dim: TiledDims.Dim, byte_stride: usize) Digit { + // Unbounded digits keep every bit of the quotient. + const modulus = if (dim.modulus == 0) std.math.maxInt(usize) else dim.modulus; + const is_mask = dim.modulus == 0 or std.math.isPowerOfTwo(modulus); + return .{ + .byte_stride = byte_stride, + .divisor = dim.divisor, + .modulus = if (is_mask and dim.modulus != 0) modulus - 1 else modulus, + .axis = axis, + .div_shift = if (std.math.isPowerOfTwo(dim.divisor)) @intCast(@ctz(dim.divisor)) else null, + .is_mask = is_mask, + }; + } + + fn value(self: Digit, coordinate: usize) usize { + const quotient = if (self.div_shift) |shift| coordinate >> shift else coordinate / self.divisor; + return if (self.is_mask) quotient & self.modulus else quotient % self.modulus; + } + }; + }; + }; + + fn init(shape: Shape, shard_shape: Shape, layout: pjrt.MemoryLayout) LayoutMetadata { + const last_dim_size: usize = if (shard_shape.rank() == 0) 1 else @intCast(shard_shape.dim(-1)); + const element_size: usize = shard_shape.dtype().sizeOf(); + + // Longest stretch where stepping to the next local index also steps to the next element in memory. + // Suppose a [2,5] shard tiled by (2,2): tiles are stored one after another, each row-major. + // With local indices the memory holds: + // `0 1 5 6 | 2 3 7 8 | 4 _ 9 _` + // 0 and 1 are adjacent but 1 and 2 aren't, so the run is 2. A row-major layout is a single run. + const addressing: Addressing, const physical_run: usize = addr: switch (layout) { + .strides => |strides| { + std.debug.assert(strides.byte_strides.len == shard_shape.rank()); + + const row_major = shard_shape.computeByteStrides(); + if (std.mem.eql(i64, strides.byte_strides, row_major.constSlice())) break :addr .{ .row_major, shard_shape.count() }; + + var byte_strides: Addressing.ByteStrides = @splat(0); + for (strides.byte_strides, 0..) |stride, axis| { + stdx.debug.assert(stride >= 0, "negative destination byte strides are unsupported, got {}", .{stride}); + byte_strides[axis] = @intCast(stride); + } + + const physical_run = + // Rows padded to an alignment keep their elements adjacent: a run is a row. + if (byte_strides[shard_shape.rank() - 1] == element_size) last_dim_size + // Permuted axes, e.g. a column-major [3,4] f32 with strides [4,12], put the + // innermost axis elsewhere in memory, so no two elements of a row are adjacent. + else 1; + break :addr .{ .{ .strided = byte_strides }, physical_run }; + }, + .tiled => |tiled| { + stdx.debug.assert(tiled.minor_to_major.len == shard_shape.rank(), "layout rank {} doesn't match shape rank {}", .{ tiled.minor_to_major.len, shard_shape.rank() }); + + if (tiled.tile_dims.len == 0 and isMinorToMajorRowMajor(tiled.minor_to_major, shard_shape.rank())) break :addr .{ .row_major, shard_shape.count() }; + + // The expanded dimensions are laid out row-major. + const expanded: TiledDims = .init(shard_shape, tiled); + var tiled_addressing: Addressing.Tiled = .{}; + var byte_stride = element_size; + // From the most minor physical dimension, the run grows while each + // dimension is the next digit of the innermost axis. + const last_axis = shard_shape.rank() -| 1; + var physical_run: usize = 1; + var run_grows = true; + var i = expanded.dims.len; + while (i > 0) { + i -= 1; + const dim = expanded.dims.get(i); + if (dim.size > 1) { + if (dim.axis) |axis| tiled_addressing.digits.appendAssumeCapacity(.init(axis, dim, byte_stride)); + if (run_grows) { + const next_digit = if (dim.axis) |axis| axis == last_axis and dim.divisor == physical_run else false; + if (!next_digit) { + run_grows = false; + } else if (dim.modulus == 0) { + // An unbounded digit holds the rest of the axis. + physical_run = last_dim_size; + run_grows = false; + } else { + physical_run *= dim.modulus; + // Padding after the digit's last value ends the run. + run_grows = dim.size == dim.modulus; + } + } + } + byte_stride *= dim.size; + } + physical_run = @min(physical_run, last_dim_size); + + break :addr .{ .{ .tiled = tiled_addressing }, physical_run }; + }, + }; + + // Longest stretch where stepping to the next local index also steps to the next global index. + // Suppose a global [2,6] split into two [2,3] shards, each holding half the columns. + // With global indices the shards hold: + // - shard 1: `0 1 2 | 6 7 8` + // - shard 2: `3 4 5 | 9 10 11` + // 2 and 6 are adjacent in shard 1 but not globally, so the run is 3. + const shard_run = run: { + var run: usize = 1; + var axis = shard_shape.rank(); + while (axis > 0) { + axis -= 1; + run *= @intCast(shard_shape.dim(axis)); + if (shape.dim(axis) != shard_shape.dim(axis)) break; + } + break :run run; + }; + const block_size = if (shard_shape.count() == 0) 0 else @min(physical_run, shard_run); + + return .{ + .rank = shard_shape.rank(), + .element_size = element_size, + .block_size = block_size, + // See `View.blockBounds`. + .alignment_size = @max(last_dim_size, block_size), + .addressing = addressing, + }; + } + + /// The byte offset in its shard of the element at shard-local row-major index + /// `local` and coordinates `coordinates`: row-major layouts use the index, + /// the others the coordinates. + fn byteOffset(self: *const LayoutMetadata, local: usize, coordinates: Coordinates) usize { + switch (self.addressing) { + .row_major => return local * self.element_size, + .strided => |*byte_strides| { + var offset: usize = 0; + for (coordinates.axes[0..self.rank], byte_strides[0..self.rank]) |coord, byte_stride| offset += coord * byte_stride; + return offset; + }, + .tiled => |*tiled| { + var offset: usize = 0; + for (tiled.digits.constSlice()) |digit| offset += digit.value(coordinates.axes[digit.axis]) * digit.byte_stride; + return offset; + }, + } + } +}; + +/// Whether a PJRT minor-to-major order lays the axes out row-major, like ZML shapes. +fn isMinorToMajorRowMajor(minor_to_major: []const i64, rank: usize) bool { + if (minor_to_major.len != rank) return false; + for (minor_to_major, 0..) |axis, i| { + if (axis != @as(i64, @intCast(rank - i - 1))) return false; + } + return true; +} + +/// The physical dimensions of a tiled layout, major to minor. XLA lays a tiled array +/// out row-major over these dimensions; see https://openxla.org/xla/tiled_layout. +const TiledDims = struct { + dims: stdx.BoundedArray(Dim, Shape.MAX_RANK + 2 * pjrt.DefaultMemoryLayout.MAX_TILE_DIMS) = .empty, + + /// Every dimension holds the digit `(coordinate[axis] / divisor) % modulus`. + const Dim = struct { + /// Null for padding, which only holds index 0. + axis: ?u3, + size: usize, + divisor: usize = 1, + /// Zero for a digit without an upper bound, which holds `coordinate / divisor`. + modulus: usize = 0, + + /// The index of the tile along this dimension. + fn tileCount(self: Dim, tile: usize) Dim { + const size = std.math.divCeil(usize, self.size, tile) catch unreachable; + const axis = self.axis orelse return .{ .axis = null, .size = size }; + if (self.modulus == 0) return .{ .axis = axis, .size = size, .divisor = self.divisor * tile }; + if (self.modulus % tile == 0) return .{ .axis = axis, .size = size, .divisor = self.divisor * tile, .modulus = self.modulus / tile }; + // One tile holds every value of the digit. + return .{ .axis = null, .size = size }; + } + + /// The index within the tile along this dimension. + fn tileIndex(self: Dim, tile: usize) Dim { + const axis = self.axis orelse return .{ .axis = null, .size = tile }; + // XLA assumes nested tiles add no padding. + stdx.debug.assert( + self.modulus == 0 or self.modulus % tile == 0 or tile % self.modulus == 0, + "nested tile {} must divide, or be a multiple of, the {} values it tiles", + .{ tile, self.modulus }, + ); + if (self.modulus != 0 and self.modulus < tile) return .{ .axis = axis, .size = tile, .divisor = self.divisor, .modulus = self.modulus }; + return .{ .axis = axis, .size = tile, .divisor = self.divisor, .modulus = tile }; + } + }; + + /// Follows XLA's LayoutUtil::LinearIndexForNestedTiling. The physical dimensions + /// start in minor_to_major order. Each tile then splits the most minor ones, + /// padded with leading unit dimensions if the tile has more, into tile counts + /// followed by indices within the tile. Tile sizes are listed major to minor, so + /// a later tile splits the previous tile's indices and possibly its counts. + /// + /// Two consequences are easy to get wrong: + /// - `tile_dims[j]` of a k-dimensional tile belongs to `minor_to_major[k-1-j]`, + /// not to `minor_to_major[j]` nor to logical axis j. TPU's default + /// `{2,1,0:T(8,128)}` for rank-3 32-bit arrays tiles rows by 8 and columns + /// by 128. + /// - A later tile splits the previous tile's dimensions, not the grid of + /// tiles: `(8,128)(2,1)`, the bf16 packing, interleaves pairs of rows + /// within each tile. TPU tiles bf16 and 8-bit arrays this way. + fn init(shape: Shape, layout: pjrt.MemoryLayout.Tiled) TiledDims { + const rank = shape.rank(); + stdx.debug.assert(layout.minor_to_major.len == rank, "layout rank {} doesn't match shape rank {}", .{ layout.minor_to_major.len, rank }); + var result: TiledDims = .{}; + var i = rank; + while (i > 0) { + i -= 1; + const axis: u3 = @intCast(layout.minor_to_major[i]); + result.dims.appendAssumeCapacity(.{ .axis = axis, .size = @intCast(shape.dim(axis)) }); + } + var cursor: usize = 0; + for (layout.tile_dims_sizes) |tile_rank| { + const tile = layout.tile_dims[cursor..][0..tile_rank]; + cursor += tile_rank; + while (result.dims.len < tile_rank) result.dims.insert(0, .{ .axis = null, .size = 1 }) catch unreachable; + const start = result.dims.len - tile_rank; + for (tile, start..) |size_, d| { + // XLA marks dimensions combined with the next one by a negative size. + stdx.debug.assert(size_ > 0, "unsupported tile dimension {}", .{size_}); + const size: usize = @intCast(size_); + result.dims.appendAssumeCapacity(result.dims.get(d).tileIndex(size)); + result.dims.set(d, result.dims.get(d).tileCount(size)); + } + } + stdx.debug.assert(cursor == layout.tile_dims.len, "layout tile metadata is inconsistent", .{}); + return result; + } +}; + +// +// ============= +// === TESTS === +// ============= +// + +/// The byte offset of an element, computed from the layout directly to check `LayoutMetadata`. +fn referenceByteOffset(shape_: Shape, layout: pjrt.MemoryLayout, index: []const usize) usize { + if (!builtin.is_test) @compileError("referenceByteOffset is only available in tests"); + + return switch (layout) { + .tiled => |tiled| tiledElementOffset(shape_, tiled, index) * shape_.dtype().sizeOf(), + .strides => |strides| b: { + var byte_offset: i64 = 0; + for (index, strides.byte_strides) |coord, stride| { + stdx.debug.assert(stride >= 0, "negative destination byte strides are unsupported, got {}", .{stride}); + byte_offset += @as(i64, @intCast(coord)) * stride; + } + stdx.debug.assert(byte_offset >= 0, "destination byte offset must be non-negative, got {}", .{byte_offset}); + break :b @as(usize, @intCast(byte_offset)); + }, + }; +} + +/// Computes the element offset for one logical index in XLA/PJRT tiled layout order, +/// as XLA's LayoutUtil::LinearIndexForNestedTiling does. A direct port of that +/// function, kept independent of `TiledDims`: a reference sharing the prepared +/// model of tiling once let tests miss offsets that disagreed with XLA. +fn tiledElementOffset(shape_: Shape, layout: pjrt.MemoryLayout.Tiled, index: []const usize) usize { + if (!builtin.is_test) @compileError("tiledElementOffset is only available in tests"); + + const rank = shape_.rank(); + stdx.debug.assert(layout.minor_to_major.len == rank, "layout rank {} doesn't match shape rank {}", .{ layout.minor_to_major.len, rank }); + + // Physical dimensions and indices, major to minor. + const Expanded = stdx.BoundedArray(usize, Shape.MAX_RANK + 2 * pjrt.DefaultMemoryLayout.MAX_TILE_DIMS); + var dims: Expanded = .empty; + var indices: Expanded = .empty; + var i = rank; + while (i > 0) { + i -= 1; + const axis: usize = @intCast(layout.minor_to_major[i]); + dims.appendAssumeCapacity(@intCast(shape_.dim(axis))); + indices.appendAssumeCapacity(index[axis]); + } + + var cursor: usize = 0; + for (layout.tile_dims_sizes) |tile_rank| { + const tile = layout.tile_dims[cursor..][0..tile_rank]; + cursor += tile_rank; + while (dims.len < tile_rank) { + dims.insert(0, 1) catch unreachable; + indices.insert(0, 0) catch unreachable; + } + // Tiled dimensions become tile counts, followed by the indices within the tile. + const start = dims.len - tile_rank; + for (tile, start..) |tile_dim_, d| { + const tile_dim: usize = @intCast(tile_dim_); + dims.appendAssumeCapacity(tile_dim); + indices.appendAssumeCapacity(indices.get(d) % tile_dim); + dims.set(d, std.math.divCeil(usize, dims.get(d), tile_dim) catch unreachable); + indices.set(d, indices.get(d) / tile_dim); + } + } + stdx.debug.assert(cursor == layout.tile_dims.len, "layout tile metadata is inconsistent", .{}); + + var offset: usize = 0; + for (dims.constSlice(), indices.constSlice()) |dim, coord| offset = offset * dim + coord; + return offset; +} + +test "LayoutMetadata byte offsets match physical layouts" { + const unit_tiles: [pjrt.DefaultMemoryLayout.MAX_TILE_DIMS]i64 = @splat(1); + const tile_ranks: [pjrt.DefaultMemoryLayout.MAX_NUM_TILES]usize = @splat(Shape.MAX_RANK); + const Case = struct { shape: Shape, layout: pjrt.MemoryLayout, block_size: usize }; + const cases = [_]Case{ + .{ .shape = .init(.{}, .i32), .layout = .{ .strides = .{ .byte_strides = &.{} } }, .block_size = 1 }, + .{ .shape = .init(.{ 3, 5 }, .i32), .layout = .{ .strides = .{ .byte_strides = &.{ 20, 4 } } }, .block_size = 15 }, + .{ .shape = .init(.{ 3, 5 }, .f16), .layout = .{ .strides = .{ .byte_strides = &.{ 16, 2 } } }, .block_size = 5 }, + .{ .shape = .init(.{ 3, 5 }, .i32), .layout = .{ .strides = .{ .byte_strides = &.{ 4, 16 } } }, .block_size = 1 }, + .{ + .shape = .init(.{ 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }, + .block_size = 15, + }, + .{ + .shape = .init(.{ 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }, + .block_size = 1, + }, + .{ + .shape = .init(.{ 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 2, 2 }, .tile_dims_sizes = &.{2} } }, + .block_size = 2, + }, + .{ + .shape = .init(.{ 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{ 2, 2 }, .tile_dims_sizes = &.{2} } }, + .block_size = 1, + }, + .{ + .shape = .init(.{ 3, 5 }, .f16), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 2, 2, 2, 1 }, .tile_dims_sizes = &.{ 2, 2 } } }, + .block_size = 1, + }, + .{ + .shape = .init(.{ 4, 8 }, .f16), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 2, 4, 2, 1 }, .tile_dims_sizes = &.{ 2, 2 } } }, + .block_size = 1, + }, + // The nested tile is larger than the values it tiles. + .{ + .shape = .init(.{ 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 2, 2, 4, 1 }, .tile_dims_sizes = &.{ 2, 2 } } }, + .block_size = 1, + }, + // The nested tile also splits the tile counts. + .{ + .shape = .init(.{ 4, 8 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 1, 4, 2, 1, 1 }, .tile_dims_sizes = &.{ 2, 3 } } }, + .block_size = 1, + }, + .{ + .shape = .init(.{ 2, 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 2, 1, 0 }, .tile_dims = &.{ 4, 2 }, .tile_dims_sizes = &.{2} } }, + .block_size = 2, + }, + .{ + .shape = .init(.{ 2, 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 2, 1, 0 }, .tile_dims = &.{ 3, 2 }, .tile_dims_sizes = &.{2} } }, + .block_size = 2, + }, + .{ + .shape = .init(.{ 2, 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 2, 1, 0 }, .tile_dims = &.{ 3, 2, 3, 1 }, .tile_dims_sizes = &.{ 2, 2 } } }, + .block_size = 1, + }, + // The nested tile moves pairs of the innermost axis to the most minor dimension. + .{ + .shape = .init(.{ 2, 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 0, 2, 1 }, .tile_dims = &.{ 2, 4, 2, 2, 1 }, .tile_dims_sizes = &.{ 3, 2 } } }, + .block_size = 2, + }, + // The tile has more dimensions than the shape. + .{ + .shape = .init(.{5}, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{0}, .tile_dims = &.{ 2, 4 }, .tile_dims_sizes = &.{2} } }, + .block_size = 4, + }, + .{ + .shape = .init(.{ 1, 1, 1, 1, 1, 1, 2, 3 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 7, 6, 5, 4, 3, 2, 1, 0 }, .tile_dims = &unit_tiles, .tile_dims_sizes = &tile_ranks } }, + .block_size = 3, + }, + }; + for (cases) |case| { + const metadata: LayoutMetadata = .init(case.shape, case.shape, case.layout); + try std.testing.expectEqual(case.block_size, metadata.block_size); + for (0..case.shape.count()) |local| { + const index = Coordinates.unflatten(case.shape, local); + const expected = referenceByteOffset(case.shape, case.layout, index.axes[0..case.shape.rank()]); + const actual = metadata.byteOffset(local, index); + try std.testing.expectEqual(expected, actual); + // Elements after the first of a block follow the previous one in memory. + if ((local % metadata.alignment_size) % metadata.block_size != 0) { + try std.testing.expectEqual(metadata.byteOffset(local - 1, Coordinates.unflatten(case.shape, local - 1)) + metadata.element_size, actual); + } + } + } +} + +test "tiled layout offsets follow XLA" { + const Tiled = pjrt.MemoryLayout.Tiled; + // Figure 1 of https://openxla.org/xla/tiled_layout: F32[3,5]{1,0:T(2,2)}. + const tiles: Tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 2, 2 }, .tile_dims_sizes = &.{2} }; + try std.testing.expectEqual(17, tiledElementOffset(.init(.{ 3, 5 }, .f32), tiles, &.{ 2, 3 })); + const shape_ = Shape.init(.{ 3, 4 }, .i32); + try std.testing.expectEqual(0, tiledElementOffset(shape_, tiles, &.{ 0, 0 })); + try std.testing.expectEqual(1, tiledElementOffset(shape_, tiles, &.{ 0, 1 })); + try std.testing.expectEqual(2, tiledElementOffset(shape_, tiles, &.{ 1, 0 })); + try std.testing.expectEqual(3, tiledElementOffset(shape_, tiles, &.{ 1, 1 })); + try std.testing.expectEqual(4, tiledElementOffset(shape_, tiles, &.{ 0, 2 })); + try std.testing.expectEqual(8, tiledElementOffset(shape_, tiles, &.{ 2, 0 })); + + // Figure 2: 4x8 tiled by (2,4)(2,1). The nested tile pairs rows within each + // tile, as the (8,128)(2,1) tiling of bf16 on TPU does. + const nested: Tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 2, 4, 2, 1 }, .tile_dims_sizes = &.{ 2, 2 } }; + const figure2 = [4][8]usize{ + .{ 0, 2, 4, 6, 8, 10, 12, 14 }, + .{ 1, 3, 5, 7, 9, 11, 13, 15 }, + .{ 16, 18, 20, 22, 24, 26, 28, 30 }, + .{ 17, 19, 21, 23, 25, 27, 29, 31 }, + }; + for (figure2, 0..) |offsets, row| { + for (offsets, 0..) |offset, col| try std.testing.expectEqual(offset, tiledElementOffset(.init(.{ 4, 8 }, .bf16), nested, &.{ row, col })); + } + + // A tile lists sizes for the most minor physical dimensions, major to minor. + // Like {2,1,0:T(8,128)} on TPU, this tiles rows by 2 and columns by 4: + // offset = a*32 + (b/2)*16 + (c/4)*8 + (b%2)*4 + c%4. + const rank3: Shape = .init(.{ 2, 3, 5 }, .f32); + const partial: Tiled = .{ .minor_to_major = &.{ 2, 1, 0 }, .tile_dims = &.{ 2, 4 }, .tile_dims_sizes = &.{2} }; + try std.testing.expectEqual(4, tiledElementOffset(rank3, partial, &.{ 0, 1, 0 })); + try std.testing.expectEqual(8, tiledElementOffset(rank3, partial, &.{ 0, 0, 4 })); + try std.testing.expectEqual(12, tiledElementOffset(rank3, partial, &.{ 0, 1, 4 })); + try std.testing.expectEqual(16, tiledElementOffset(rank3, partial, &.{ 0, 2, 0 })); + try std.testing.expectEqual(51, tiledElementOffset(rank3, partial, &.{ 1, 2, 3 })); + + // Tiles follow the physical order: offset = (c/2)*8 + (r/2)*4 + (c%2)*2 + r%2. + const transposed: Tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{ 2, 2 }, .tile_dims_sizes = &.{2} }; + try std.testing.expectEqual(1, tiledElementOffset(.init(.{ 3, 5 }, .f32), transposed, &.{ 1, 0 })); + try std.testing.expectEqual(2, tiledElementOffset(.init(.{ 3, 5 }, .f32), transposed, &.{ 0, 1 })); + try std.testing.expectEqual(4, tiledElementOffset(.init(.{ 3, 5 }, .f32), transposed, &.{ 2, 0 })); + try std.testing.expectEqual(8, tiledElementOffset(.init(.{ 3, 5 }, .f32), transposed, &.{ 0, 2 })); + try std.testing.expectEqual(14, tiledElementOffset(.init(.{ 3, 5 }, .f32), transposed, &.{ 2, 3 })); + + // A tile with more dimensions than the shape pads it with leading unit + // dimensions: offset = (c/4)*8 + c%4. + const wide: Tiled = .{ .minor_to_major = &.{0}, .tile_dims = &.{ 2, 4 }, .tile_dims_sizes = &.{2} }; + try std.testing.expectEqual(3, tiledElementOffset(.init(.{5}, .f32), wide, &.{3})); + try std.testing.expectEqual(8, tiledElementOffset(.init(.{5}, .f32), wide, &.{4})); + + // A later tile can also split the previous tile counts: + // offset = r*8 + (c%4)*2 + (c/4)%2. + const cross_tile: Tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 1, 4, 2, 1, 1 }, .tile_dims_sizes = &.{ 2, 3 } }; + try std.testing.expectEqual(1, tiledElementOffset(.init(.{ 4, 8 }, .f32), cross_tile, &.{ 0, 4 })); + try std.testing.expectEqual(2, tiledElementOffset(.init(.{ 4, 8 }, .f32), cross_tile, &.{ 0, 1 })); + try std.testing.expectEqual(11, tiledElementOffset(.init(.{ 4, 8 }, .f32), cross_tile, &.{ 1, 5 })); + try std.testing.expectEqual(31, tiledElementOffset(.init(.{ 4, 8 }, .f32), cross_tile, &.{ 3, 7 })); +} + +test "HostAccessible.HostPinnedAllocations keeps each region's replicas together as allocations are added" { + var allocations: HostAccessible.HostPinnedAllocations = .empty; + defer allocations.deinit(std.testing.allocator); + try allocations.entries.ensureTotalCapacity(std.testing.allocator, 5); + var storage: [5]u8 = undefined; + const base: [*]u8 = &storage; + // A vector of 12 split into shards of 4, holding regions A, B, A, C, B: + // replicas arrive interleaved with other regions. + const shard_shape: Shape = .init(.{4}, .u8); + for ([_]usize{ 0, 4, 0, 8, 4 }, 0..) |row, i| { + allocations.add(shard_shape, .{ .origin = .{ .axes = .{row} ++ .{0} ** (Shape.MAX_RANK - 1) }, .ptr = base + i, .shard_index = undefined, .handle = undefined }); + } + + try std.testing.expectEqualSlices([*]u8, &.{ base, base + 2, base + 1, base + 4, base + 3 }, allocations.entries.items(.ptr)); + for (allocations.regions.constSlice(), [_][2]u8{ .{ 0, 2 }, .{ 2, 2 }, .{ 4, 1 } }) |region, expected| { + try std.testing.expectEqual(expected, [2]u8{ region.start, region.len }); + } +} + +test "HostAccessible.format prints the elements in row-major order" { + const shape_ = Shape.init(.{ 2, 2 }, .i32); + const column_major: pjrt.MemoryLayout = .{ .tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }; + var storage = [_]i32{ 0, 2, 1, 3 }; + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, column_major, &.{0}, &.{@ptrCast(&storage)}); + defer pinned.deinit(std.testing.allocator); + var buffer: [128]u8 = undefined; + const printed = try std.fmt.bufPrint(&buffer, "{f}", .{pinned}); + try std.testing.expectStringEndsWith(printed, ") { 0 1 2 3 }"); +} + +test "HostAccessible.View borrows the buffer's pointers through slicing" { + const shape_ = Shape.init(.{4}, .i32); + const layout: pjrt.MemoryLayout = .{ .strides = .{ .byte_strides = &.{4} } }; + var storage = [_]i32{ 0, 1, 2, 3 }; + var other = [_]i32{ 10, 11, 12, 13 }; + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, layout, &.{0}, &.{@ptrCast(&storage)}); + defer pinned.deinit(std.testing.allocator); + const nested = pinned.view(i32).slice(1, null).slice(0, 2); + + // Slices read the pointers in place, which lets `await` refresh them + // without copying them into every view. + try std.testing.expectEqual(@as(*const HostAccessible.Mapping, pinned.mapping), nested.mapping); + pinned.mapping.allocations.entries.items(.ptr)[0] = @ptrCast(&other); + nested.fill(99); + try std.testing.expectEqualSlices(i32, &.{ 0, 1, 2, 3 }, &storage); + try std.testing.expectEqualSlices(i32, &.{ 10, 99, 99, 13 }, &other); +} + +test "HostAccessible.View blocks skip tile padding" { + const shape_ = Shape.init(.{ 3, 5 }, .i32); + const layout: pjrt.MemoryLayout = .{ .tiled = .{ + .minor_to_major = &.{ 1, 0 }, + .tile_dims = &.{ 2, 2 }, + .tile_dims_sizes = &.{2}, + } }; + var storage: [24]i32 = @splat(-1); + const ptrs = [_][*]u8{@ptrCast(&storage)}; + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, layout, &.{0}, &ptrs); + defer pinned.deinit(std.testing.allocator); + const values = pinned.view(i32); + try std.testing.expectEqual(2, pinned.mapping.layout_metadata.block_size); + try std.testing.expectEqual(HostAccessible.View(i32).Bounds{ .start = 0, .end = 2 }, values.blockBounds(1)); + try std.testing.expectEqual(HostAccessible.View(i32).Bounds{ .start = 4, .end = 5 }, values.blockBounds(4)); + var iterator = values.blocks(); + while (iterator.next()) |block| { + for (0..block.replicas.len) |i| { + const items = block.replica(i); + for (items, block.offset..) |*value, index| value.* = @intCast(index + 1); + } + } + try std.testing.expectEqualSlices(i32, &.{ + 1, 2, 6, 7, 3, 4, 8, 9, 5, -1, 10, -1, + 11, 12, -1, -1, 13, 14, -1, -1, 15, -1, -1, -1, + }, &storage); + values.slice(4, 3).fill(90); + try std.testing.expectEqualSlices(i32, &.{ + 1, 2, 90, 90, 3, 4, 8, 9, 90, -1, 10, -1, + 11, 12, -1, -1, 13, 14, -1, -1, 15, -1, -1, -1, + }, &storage); +} + +test "HostAccessible.View nested tiles" { + const shape_ = Shape.init(.{ 3, 5 }, .i32); + const layout: pjrt.MemoryLayout = .{ .tiled = .{ + .minor_to_major = &.{ 1, 0 }, + .tile_dims = &.{ 2, 2, 2, 1 }, + .tile_dims_sizes = &.{ 2, 2 }, + } }; + var storage: [24]i32 = @splat(-1); + const ptrs = [_][*]u8{@ptrCast(&storage)}; + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, layout, &.{0}, &ptrs); + defer pinned.deinit(std.testing.allocator); + const values = pinned.view(i32); + // Pairs of rows are interleaved within each tile, so every block is one element. + try std.testing.expectEqual(1, pinned.mapping.layout_metadata.block_size); + values.copyFrom(&.{ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }); + try std.testing.expectEqualSlices(i32, &.{ + 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, -1, -1, + 11, -1, 12, -1, 13, -1, 14, -1, 15, -1, -1, -1, + }, &storage); +} + +test "HostAccessible.View dense, transposed, strided and partial-rank tiled layouts" { + const shape_ = Shape.init(.{ 2, 3 }, .i32); + const Case = struct { layout: pjrt.MemoryLayout, block_size: usize, expected: []const i32 }; + const cases = [_]Case{ + .{ + .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }, + .block_size = 6, + .expected = &.{ 1, 2, 3, 4, 5, 6, -1, -1 }, + }, + .{ + .layout = .{ .tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }, + .block_size = 1, + .expected = &.{ 1, 4, 2, 5, 3, 6, -1, -1 }, + }, + .{ + .layout = .{ .strides = .{ .byte_strides = &.{ 12, 4 } } }, + .block_size = 6, + .expected = &.{ 1, 2, 3, 4, 5, 6, -1, -1 }, + }, + .{ + .layout = .{ .strides = .{ .byte_strides = &.{ 20, 4 } } }, + .block_size = 3, + .expected = &.{ 1, 2, 3, -1, -1, 4, 5, 6 }, + }, + .{ + .layout = .{ .strides = .{ .byte_strides = &.{ 4, 12 } } }, + .block_size = 1, + .expected = &.{ 1, 4, -1, 2, 5, -1, 3, 6 }, + }, + .{ + .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{2}, .tile_dims_sizes = &.{1} } }, + .block_size = 3, + .expected = &.{ 1, 2, 3, -1, 4, 5, 6, -1 }, + }, + }; + for (cases) |case| { + var storage: [8]i32 = @splat(-1); + const ptrs = [_][*]u8{@ptrCast(&storage)}; + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, case.layout, &.{0}, &ptrs); + defer pinned.deinit(std.testing.allocator); + const values = pinned.view(i32); + try std.testing.expectEqual(case.block_size, pinned.mapping.layout_metadata.block_size); + values.copyFrom(&.{ 1, 2, 3, 4, 5, 6 }); + try std.testing.expectEqualSlices(i32, case.expected, &storage); + } +} + +test "HostAccessible.View row-major layouts return one block across rows and replicas" { + const shape_: Shape = .init(.{ 3, 5 }, .i32); + const layouts = [_]pjrt.MemoryLayout{ + .{ .strides = .{ .byte_strides = &.{ 20, 4 } } }, + .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }, + }; + for (layouts) |layout| { + var storage: [2][15]i32 = @splat(@splat(0)); + const ptrs = [_][*]u8{ @ptrCast(&storage[0]), @ptrCast(&storage[1]) }; + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, layout, &.{ 0, 0 }, &ptrs); + defer pinned.deinit(std.testing.allocator); + const values = pinned.view(i32); + for ([_]HostAccessible.View(i32){ values, values.slice(1, null).slice(1, 10) }) |selected| { + var forward = selected.blocks(); + var reverse = selected.reverseBlocks(); + for ([_]HostAccessible.View(i32).Block{ forward.next().?, reverse.next().? }) |block| { + try std.testing.expectEqual(0, block.offset); + try std.testing.expectEqual(selected.len, block.len); + try std.testing.expectEqual(2, block.replicas.len); + for (0..block.replicas.len, &storage) |i, *replica| { + const items = block.replica(i); + try std.testing.expectEqual(replica[selected.start..].ptr, items.ptr); + try std.testing.expectEqual(selected.len, items.len); + } + } + try std.testing.expectEqual(null, forward.next()); + try std.testing.expectEqual(null, reverse.next()); + } + } +} + +test "HostAccessible.View dense shards stop at gaps in global row-major order" { + const Case = struct { + shape: Shape, + shard_shape: Shape, + block_size: usize, + origins: [2]usize, + expected: [2][]const i32, + }; + const cases = [_]Case{ + .{ + .shape = .init(.{ 2, 6 }, .i32), + .shard_shape = .init(.{ 2, 3 }, .i32), + .block_size = 3, + .origins = .{ 0, 3 }, + .expected = .{ &.{ 0, 1, 2, 6, 7, 8 }, &.{ 3, 4, 5, 9, 10, 11 } }, + }, + .{ + .shape = .init(.{ 2, 6, 2 }, .i32), + .shard_shape = .init(.{ 2, 3, 2 }, .i32), + .block_size = 6, + .origins = .{ 0, 6 }, + .expected = .{ + &.{ 0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 17 }, + &.{ 6, 7, 8, 9, 10, 11, 18, 19, 20, 21, 22, 23 }, + }, + }, + }; + for (cases) |case| { + const strides = case.shard_shape.computeByteStrides(); + var storage: [2][12]i32 = @splat(@splat(-1)); + var ptrs: [2][*]u8 = undefined; + for (&ptrs, &storage) |*ptr, *data| ptr.* = @ptrCast(data); + const pinned = try HostAccessible.initForTests(std.testing.allocator, case.shape, case.shard_shape, .{ .strides = .{ .byte_strides = strides.constSlice() } }, &case.origins, &ptrs); + defer pinned.deinit(std.testing.allocator); + const elements = pinned.view(i32); + try std.testing.expectEqual(case.block_size, pinned.mapping.layout_metadata.block_size); + try std.testing.expectEqual(case.block_size, elements.blockBounds(0).end); + var tail = elements.slice(case.block_size - 1, null).blocks(); + try std.testing.expectEqual(1, tail.next().?.len); + + elements.fillIota(0, 1); + for (storage, case.expected) |data, expected| { + try std.testing.expectEqualSlices(i32, expected, data[0..expected.len]); + } + var actual: [24]i32 = undefined; + elements.copyTo(actual[0..case.shape.count()]); + for (actual[0..case.shape.count()], 0..) |value, i| try std.testing.expectEqual(@as(i32, @intCast(i)), value); + + // A partial range crosses both a global gap and a physical shard boundary. + const start = case.block_size - 1; + const values = [_]i32{ 90, 91, 92, 93 }; + elements.slice(start, values.len).copyFrom(&values); + for (storage, case.expected) |data, expected| { + for (data[0..expected.len], expected) |value, global| { + const offset: usize = @intCast(global); + try std.testing.expectEqual(if (offset >= start and offset < start + values.len) values[offset - start] else global, value); + } + } + var copied: [4]i32 = undefined; + elements.slice(start, copied.len).copyTo(&copied); + try std.testing.expectEqualSlices(i32, &values, &copied); + } +} + +/// A 4x6 array split into 2x3 shards. Shards 4 to 7 are separate copies of shards 0 to 3. +const TestShards = struct { + storage: [8][8]i32 = @splat(@splat(-1)), + ptrs: [8][*]u8 = undefined, + origins: [8]usize = @splat(0), + pinned: ?HostAccessible = null, + + const shape_ = Shape.init(.{ 4, 6 }, .i32); + const shard_shape = Shape.init(.{ 2, 3 }, .i32); + const tiled: pjrt.MemoryLayout = .{ .tiled = .{ + .minor_to_major = &.{ 1, 0 }, + .tile_dims = &.{ 2, 2 }, + .tile_dims_sizes = &.{2}, + } }; + + fn deinit(self: *TestShards) void { + if (self.pinned) |pinned| pinned.deinit(std.testing.allocator); + } + + /// The view borrows `pinned`, which borrows the storage: the storage must + /// stay in place, and the next call replaces `pinned`. + fn view(self: *TestShards, layout: pjrt.MemoryLayout) !HostAccessible.View(i32) { + for (&self.ptrs, &self.origins, &self.storage, 0..) |*ptr, *origin, *data, i| { + ptr.* = @ptrCast(data); + const row = ((i % 4) / 2) * 2; + const column = (i % 2) * 3; + origin.* = row * 6 + column; + } + self.deinit(); + self.pinned = null; + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shard_shape, layout, &self.origins, &self.ptrs); + self.pinned = pinned; + return pinned.view(i32); + } + + /// With the tiled layout, every copy of each element holds `expected[global]` + /// and the tile padding is untouched. + fn expectValues(self: *const TestShards, expected: [24]i32) !void { + // Explicit offsets for the two rows of a 2x3 shard tiled by 2x2. + const physical_offsets = [_]usize{ 0, 1, 4, 2, 3, 6 }; + for (self.storage, self.origins) |data, origin| { + for (physical_offsets, 0..) |physical, local| { + const global = origin + (local / 3) * 6 + local % 3; + try std.testing.expectEqual(expected[global], data[physical]); + } + try std.testing.expectEqual(-1, data[5]); + try std.testing.expectEqual(-1, data[7]); + } + } +}; + +test "HostAccessible.View blocks visit partial global ranges in every shard" { + var fixture: TestShards = .{}; + defer fixture.deinit(); + const values = try fixture.view(TestShards.tiled); + var visits: [24]u8 = @splat(0); + // Nested slicing must keep block offsets relative to the selected range. + var iterator = values.slice(1, null).slice(1, 18).blocks(); + while (iterator.next()) |block| { + try std.testing.expect(block.offset + block.len <= 18); + try std.testing.expectEqual(2, block.replicas.len); + for (0..block.replicas.len) |i| { + const items = block.replica(i); + for (items, block.offset + 2..) |*value, global| { + visits[global] += 1; + value.* = @intCast(global); + } + } + } + var expected: [24]i32 = @splat(-1); + for (visits, &expected, 0..) |count, *value, global| { + const selected = global >= 2 and global < 20; + try std.testing.expectEqual(@as(u8, if (selected) 2 else 0), count); + if (selected) value.* = @intCast(global); + } + try fixture.expectValues(expected); + // Starting after the first row partition must skip those shards entirely. + values.set(17, 99); + try std.testing.expectEqual(99, fixture.storage[3][4]); + try std.testing.expectEqual(99, fixture.storage[7][4]); + + const Case = struct { step: i32, expected: [18]i32 }; + const cases = [_]Case{ + .{ .step = 3, .expected = .{ 7, 10, 13, 16, 19, 22, 25, 28, 31, 34, 37, 40, 43, 46, 49, 52, 55, 58 } }, + .{ .step = -2, .expected = .{ 7, 5, 3, 1, -1, -3, -5, -7, -9, -11, -13, -15, -17, -19, -21, -23, -25, -27 } }, + .{ .step = 0, .expected = @splat(7) }, + }; + for (cases) |case| { + values.slice(1, null).slice(1, 18).fillIota(7, case.step); + expected[2..20].* = case.expected; + try fixture.expectValues(expected); + } +} + +test "HostAccessible.View blocks merge shuffled shards and group replicas" { + var fixture: TestShards = .{}; + defer fixture.deinit(); + _ = try fixture.view(TestShards.tiled); + const order = [_]usize{ 3, 4, 1, 6, 2, 0, 7, 5 }; + var ptrs: [8][*]u8 = undefined; + var origins: [8]usize = undefined; + for (order, &ptrs, &origins) |index, *ptr, *origin| { + ptr.* = fixture.ptrs[index]; + origin.* = fixture.origins[index]; + } + const pinned = try HostAccessible.initForTests(std.testing.allocator, TestShards.shape_, TestShards.shard_shape, TestShards.tiled, &origins, &ptrs); + defer pinned.deinit(std.testing.allocator); + const values = pinned.view(i32); + const selected = values.slice(1, null).slice(1, 18); + const offsets = [_]usize{ 0, 1, 3, 4, 6, 7, 9, 10, 12, 13, 15, 16 }; + const lengths = [_]usize{ 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2 }; + const replicas = [_][2]usize{ .{ 4, 0 }, .{ 1, 5 }, .{ 6, 2 }, .{ 3, 7 } }; + const physical_offsets = [_]usize{ 0, 1, 4, 2, 3, 6 }; + var iterator = selected.blocks(); + for (offsets, lengths) |offset, len| { + const block = iterator.next().?; + try std.testing.expectEqual(offset, block.offset); + try std.testing.expectEqual(len, block.len); + try std.testing.expectEqual(2, block.replicas.len); + const block_start = offset + 2; + const region = block_start / 12 * 2 + block_start % 6 / 3; + const physical = physical_offsets[block_start / 6 % 2 * 3 + block_start % 3]; + for (0..block.replicas.len, replicas[region]) |i, storage_index| { + const items = block.replica(i); + try std.testing.expectEqual(fixture.storage[storage_index][physical..].ptr, items.ptr); + } + for (0..block.replicas.len) |i| { + const items = block.replica(i); + for (items, offset + 2..) |*value, global| value.* = @intCast(global); + } + } + try std.testing.expectEqual(null, iterator.next()); + var expected: [24]i32 = @splat(-1); + for (expected[2..20], 2..) |*value, global| value.* = @intCast(global); + try fixture.expectValues(expected); + try expectReverseBlocksMirrorForward(selected); +} + +test "HostAccessible.View blocks group the maximum number of replicas" { + var storage: [Platform.MAX_NUM_DEVICES][4]i32 = @splat(@splat(-1)); + var ptrs: [Platform.MAX_NUM_DEVICES][*]u8 = undefined; + const origins: [Platform.MAX_NUM_DEVICES]usize = @splat(0); + for (&ptrs, &storage) |*ptr, *items| ptr.* = @ptrCast(items); + const shape_: Shape = .init(.{4}, .i32); + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, .{ .strides = .{ .byte_strides = &.{4} } }, &origins, &ptrs); + defer pinned.deinit(std.testing.allocator); + const values = pinned.view(i32); + var iterator = values.blocks(); + const block = iterator.next().?; + try std.testing.expectEqual(0, block.offset); + try std.testing.expectEqual(4, block.len); + try std.testing.expectEqual(Platform.MAX_NUM_DEVICES, block.replicas.len); + try std.testing.expectEqual(null, iterator.next()); + // The descriptors remain valid after advancing the iterator. + for (0..block.replicas.len, &storage) |i, *replica| { + const items = block.replica(i); + try std.testing.expectEqual(replica[0..].ptr, items.ptr); + @memset(items, 42); + try std.testing.expectEqualSlices(i32, &.{ 42, 42, 42, 42 }, replica); + } + try expectReverseBlocksMirrorForward(values); +} + +test "HostAccessible.View visits the maximum number of distinct shuffled regions" { + var storage: [Platform.MAX_NUM_DEVICES]i32 = @splat(-1); + var ptrs: [Platform.MAX_NUM_DEVICES][*]u8 = undefined; + var origins: [Platform.MAX_NUM_DEVICES]usize = undefined; + for (&ptrs, &origins, 0..) |*ptr, *origin, shard| { + const index = storage.len - shard - 1; + ptr.* = @ptrCast(&storage[index]); + origin.* = index; + } + const shape_: Shape = .init(.{Platform.MAX_NUM_DEVICES}, .i32); + const shard_shape: Shape = .init(.{1}, .i32); + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shard_shape, .{ .strides = .{ .byte_strides = &.{4} } }, &origins, &ptrs); + defer pinned.deinit(std.testing.allocator); + const values = pinned.view(i32); + values.fillIota(0, 1); + var forward = values.blocks(); + var reverse = values.reverseBlocks(); + for (storage, 0..) |value, index| { + try std.testing.expectEqual(@as(i32, @intCast(index)), value); + const block = forward.next().?; + try std.testing.expectEqual(index, block.offset); + try std.testing.expectEqual(1, block.len); + try std.testing.expectEqual(1, block.replicas.len); + try std.testing.expectEqual(storage[index..].ptr, block.firstReplica().ptr); + const reverse_block = reverse.next().?; + const reverse_index = storage.len - index - 1; + try std.testing.expectEqual(reverse_index, reverse_block.offset); + try std.testing.expectEqual(storage[reverse_index..].ptr, reverse_block.firstReplica().ptr); + } + try std.testing.expectEqual(null, forward.next()); + try std.testing.expectEqual(null, reverse.next()); +} + +test "HostAccessible.View multidimensional slices preserve coordinates across carries" { + const shape_: Shape = .init(.{ 2, 3, 5 }, .i32); + const layouts = [_]pjrt.MemoryLayout{ + .{ .strides = .{ .byte_strides = &.{ 60, 20, 4 } } }, + .{ .tiled = .{ .minor_to_major = &.{ 2, 1, 0 }, .tile_dims = &.{ 3, 2 }, .tile_dims_sizes = &.{2} } }, + }; + for (layouts) |layout| { + var storage: [48]i32 = @splat(-1); + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, layout, &.{0}, &.{@ptrCast(&storage)}); + defer pinned.deinit(std.testing.allocator); + const values = pinned.view(i32); + values.fillIota(0, 1); + var expected: [48]i32 = @splat(-1); + for (0..shape_.count()) |local| { + const index = Coordinates.unflatten(shape_, local); + const physical = referenceByteOffset(shape_, layout, index.axes[0..shape_.rank()]) / @sizeOf(i32); + expected[physical] = @intCast(local); + try std.testing.expectEqual(@as(i32, @intCast(local)), values.get(local)); + } + try std.testing.expectEqualSlices(i32, &expected, &storage); + for (0..shape_.count() + 1) |start| { + for (0..shape_.count() - start + 1) |len| { + try expectReverseBlocksMirrorForward(values.slice(start, len)); + } + } + } +} + +test "HostAccessible.View reverse blocks mirror forward blocks" { + var fixture: TestShards = .{}; + defer fixture.deinit(); + const layouts = [_]pjrt.MemoryLayout{ + TestShards.tiled, + .{ .strides = .{ .byte_strides = &.{ 12, 4 } } }, + .{ .strides = .{ .byte_strides = &.{ 4, 8 } } }, + }; + for (layouts) |layout| { + const values = try fixture.view(layout); + for (0..values.len + 1) |start| { + for (0..values.len - start + 1) |len| { + try expectReverseBlocksMirrorForward(values.slice(start, len)); + } + } + } + // Dense blocks can span multiple rows, and scalar blocks have no row axis. + const dense_pinned = try HostAccessible.initForTests(std.testing.allocator, TestShards.shard_shape, TestShards.shard_shape, layouts[1], fixture.origins[0..1], fixture.ptrs[0..1]); + defer dense_pinned.deinit(std.testing.allocator); + const dense = dense_pinned.view(i32); + try expectReverseBlocksMirrorForward(dense.slice(1, 4)); + const scalar_shape: Shape = .init(.{}, .i32); + const scalar_pinned = try HostAccessible.initForTests(std.testing.allocator, scalar_shape, scalar_shape, .{ .strides = .{ .byte_strides = &.{} } }, fixture.origins[0..1], fixture.ptrs[0..1]); + defer scalar_pinned.deinit(std.testing.allocator); + const scalar = scalar_pinned.view(i32); + try expectReverseBlocksMirrorForward(scalar); +} + +fn expectReverseBlocksMirrorForward(values: HostAccessible.View(i32)) !void { + var expected: [48]HostAccessible.View(i32).Block = undefined; + var len: usize = 0; + var offset: usize = 0; + var forward = values.blocks(); + while (forward.next()) |block| : (len += 1) { + try std.testing.expectEqual(offset, block.offset); + expected[len] = block; + offset += block.len; + } + try std.testing.expectEqual(values.len, offset); + var reverse = values.reverseBlocks(); + while (len > 0) { + len -= 1; + const block = reverse.next().?; + try std.testing.expectEqual(expected[len].offset, block.offset); + try std.testing.expectEqual(expected[len].len, block.len); + try std.testing.expectEqual(expected[len].byte_offset, block.byte_offset); + try std.testing.expectEqualSlices([*]u8, expected[len].replicas, block.replicas); + } + try std.testing.expectEqual(null, reverse.next()); +} + +test "HostAccessible.View scalar, empty ranges and rank-one padding" { + var scalar: i32 = 0; + const scalar_ptrs = [_][*]u8{@ptrCast(&scalar)}; + const shape_ = Shape.scalar(.i32); + const scalar_layout: pjrt.MemoryLayout = .{ .tiled = .{ .minor_to_major = &.{}, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }; + const scalar_pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, scalar_layout, &.{0}, &scalar_ptrs); + defer scalar_pinned.deinit(std.testing.allocator); + const scalar_view = scalar_pinned.view(i32); + scalar_view.copyFrom(&.{42}); + try std.testing.expectEqual(42, scalar); + var past_end = scalar_view.slice(1, null).blocks(); + try std.testing.expectEqual(null, past_end.next()); + + var storage: [8]bool = @splat(false); + const vector_ptrs = [_][*]u8{@ptrCast(&storage)}; + const vector_shape = Shape.init(.{5}, .bool); + const vector_layout: pjrt.MemoryLayout = .{ .tiled = .{ .minor_to_major = &.{0}, .tile_dims = &.{4}, .tile_dims_sizes = &.{1} } }; + const vector_pinned = try HostAccessible.initForTests(std.testing.allocator, vector_shape, vector_shape, vector_layout, &.{0}, &vector_ptrs); + defer vector_pinned.deinit(std.testing.allocator); + const vector = vector_pinned.view(bool); + vector.fill(true); + try std.testing.expectEqualSlices(bool, &.{ true, true, true, true, true, false, false, false }, &storage); + + const empty_shape = Shape.init(.{0}, .bool); + const empty_pinned = try HostAccessible.initForTests(std.testing.allocator, empty_shape, empty_shape, vector_layout, &.{0}, &vector_ptrs); + defer empty_pinned.deinit(std.testing.allocator); + const empty_view = empty_pinned.view(bool); + var empty_blocks = empty_view.blocks(); + try std.testing.expectEqual(null, empty_blocks.next()); + + // The empty view has no storage or metadata to read. + const none: HostAccessible.View(u32) = .empty; + none.fill(0); + none.copyTo(&.{}); + var none_blocks = none.reverseBlocks(); + try std.testing.expectEqual(null, none_blocks.next()); +} + +test "HostAccessible devices holding a shard read one allocation" { + const platform = testing.env(); + const shape_ = Shape.init(.{ .b = @as(i64, @intCast(platform.devices.len)), .d = 5 }, .i32); + try expectDevicesReadShards(platform, shape_); + try expectDevicesReadShards(platform, shape_.withPartitioning(platform.meshes.get("model").?, .{ .b = .model })); +} + +test "HostAccessible devices holding a shard read one allocation in data-parallel groups" { + const allocator = std.testing.allocator; + const io = std.testing.io; + // Two data-parallel groups of two tensor-parallel devices. + const platform = Platform.auto(allocator, io, .{ + .physical_mesh = .{ .custom = buildMesh2x2 }, + .cpu = .{ .device_count = 4 }, + }) catch return error.SkipZigTest; + defer platform.deinit(allocator, io); + const mesh: Sharding.Mesh = try .init( + "data_parallel", + &platform.physical_mesh, + .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth }), + .parseBindings(.{ .batch = .link_x, .model = .link_y }), + ); + try expectDevicesReadShards(platform, Shape.init(.{ .b = 4, .d = 5 }, .i32).withPartitioning(&mesh, .{ .b = .batch })); +} + +/// Rows of `.d` elements written once through a view reach every device +/// holding them, and devices holding the same shard share its allocation +/// where the platform allows it. +fn expectDevicesReadShards(platform: *const Platform, shape_: Shape) !void { + if (!builtin.is_test) @compileError("expectDevicesReadShards is only available in tests"); + + const allocator = std.testing.allocator; + const io = std.testing.io; + const api = platform.pjrt_api; + const pinned = try HostAccessible.init(allocator, io, platform, shape_, .undonatable); + defer pinned.deinit(allocator); + const placement: Sharding.Placement = try .init(shape_); + const expected_allocations = switch (platform.target) { + .cpu, .cuda, .rocm => shape_.count() / placement.shape.count(), + else => platform.devices.len, + }; + try std.testing.expectEqual(expected_allocations, pinned.mapping.allocations.entries.len); + + pinned.view(i32).fillIota(0, 1); + const row_size: usize = @intCast(shape_.dim(.d)); + const values = try allocator.alloc(i32, placement.shape.count()); + defer allocator.free(values); + for (pinned.buffer._shards.constSlice(), platform.physical_mesh.devices_in_canonical_order) |shard, device| { + @memset(values, -1); + if (try shard.toHostBuffer(api, std.mem.sliceAsBytes(values))) |event| { + defer event.deinit(api); + try event.await(api, io); + } + const first_row = Coordinates.ofShard(&placement, device).axes[0]; + for (values, 0..) |value, i| try std.testing.expectEqual(@as(i32, @intCast(first_row * row_size + i)), value); + } +} + +/// Like `zml/io.zig`'s, a 2x2 torus over the first four devices. +fn buildMesh2x2( + allocator: std.mem.Allocator, + target: @import("../platform.zig").Target, + devices: []const @import("../platform.zig").Device, +) !Sharding.PhysicalMesh { + if (!builtin.is_test) @compileError("buildMesh2x2 is only available in tests"); + + if (devices.len < 4) return error.NotEnoughDevices; + const topology: Sharding.PhysicalMesh.Tree = .axis(.link_x, .{ .mesh = .torus }, &.{ + .axis(.link_y, .{ .mesh = .torus }, &.{ + .device(devices[0]), + .device(devices[1]), + }), + .axis(.link_y, .{ .mesh = .torus }, &.{ + .device(devices[2]), + .device(devices[3]), + }), + }); + return Sharding.PhysicalMesh.fromTree(allocator, target, topology); +} + +test "HostAccessible executions read shared inputs on every device" { + const zml = @import("../zml.zig"); + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = testing.env(); + const api = platform.pjrt_api; + const shape_ = Shape.init(.{ 3, 5 }, .i32); + const Test = struct { + fn increment(input: zml.Tensor) zml.Tensor { + return input.onMemory(.host_pinned).toMemory(.device).addConstant(1); + } + }; + var exe = try platform.compileFn(allocator, io, Test.increment, .{zml.Tensor.fromShape(shape_)}, .{}); + defer exe.deinit(); + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + + const pinned = try HostAccessible.init(allocator, io, platform, shape_, .undonatable); + defer pinned.deinit(allocator); + for (0..2) |iteration| { + pinned.view(i32).fillIota(@intCast(iteration * 100), 1); + var output: Buffer = undefined; + runner.run(io, .{pinned.buffer}, .{&output}, .{}); + defer output.deinit(); + try output.await(io); + for (output._shards.constSlice()) |shard| { + var values: [15]i32 = @splat(-1); + if (try shard.toHostBuffer(api, std.mem.asBytes(&values))) |event| { + defer event.deinit(api); + try event.await(api, io); + } + for (values, 0..) |value, i| try std.testing.expectEqual(@as(i32, @intCast(iteration * 100 + i + 1)), value); + } + } +} + +test "HostAccessible.View.constSlice borrows contiguous buffers and copies the others" { + const shape_ = Shape.init(.{ 2, 3 }, .i32); + var scratch: [6]i32 = @splat(-1); + + var dense: [2][6]i32 = .{ .{ 0, 1, 2, 3, 4, 5 }, @splat(-1) }; + const dense_ptrs = [_][*]u8{ @ptrCast(&dense[0]), @ptrCast(&dense[1]) }; + const dense_pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, .{ .strides = .{ .byte_strides = &.{ 12, 4 } } }, &.{ 0, 0 }, &dense_ptrs); + defer dense_pinned.deinit(std.testing.allocator); + const borrowed = dense_pinned.view(i32).slice(1, 4).constSlice(&scratch); + try std.testing.expectEqual(dense[0][1..].ptr, borrowed.ptr); + try std.testing.expectEqualSlices(i32, &.{ 1, 2, 3, 4 }, borrowed); + + var column_major = [_]i32{ 0, 3, 1, 4, 2, 5 }; + const transposed: pjrt.MemoryLayout = .{ .tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }; + const transposed_pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, transposed, &.{0}, &.{@ptrCast(&column_major)}); + defer transposed_pinned.deinit(std.testing.allocator); + const copied = transposed_pinned.view(i32).slice(1, 4).constSlice(&scratch); + try std.testing.expectEqual(@as([*]const i32, &scratch), copied.ptr); + try std.testing.expectEqualSlices(i32, &.{ 1, 2, 3, 4 }, copied); + + const none: HostAccessible.View(i32) = .empty; + try std.testing.expectEqual(0, none.constSlice(&.{}).len); +} + +test "HostAccessible.View.edit writes contiguous buffers in place and commits every replica" { + const shape_ = Shape.init(.{ 2, 3 }, .i32); + var scratch: [6]i32 = @splat(-1); + + // In place: the first replica takes the writes, and `commit` copies it to the others. + var dense: [2][6]i32 = @splat(@splat(0)); + const dense_ptrs = [_][*]u8{ @ptrCast(&dense[0]), @ptrCast(&dense[1]) }; + const dense_pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, .{ .strides = .{ .byte_strides = &.{ 12, 4 } } }, &.{ 0, 0 }, &dense_ptrs); + defer dense_pinned.deinit(std.testing.allocator); + const dense_edit = dense_pinned.view(i32).slice(1, 4).edit(&scratch); + try std.testing.expectEqual(dense[0][1..].ptr, dense_edit.items.ptr); + @memcpy(dense_edit.items, &[_]i32{ 1, 2, 3, 4 }); + dense_edit.commit(); + for (dense) |replica| try std.testing.expectEqualSlices(i32, &.{ 0, 1, 2, 3, 4, 0 }, &replica); + + // Through the scratch: the buffer only changes on `commit`. + var column_major: [6]i32 = @splat(0); + const transposed: pjrt.MemoryLayout = .{ .tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }; + const transposed_pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, transposed, &.{0}, &.{@ptrCast(&column_major)}); + defer transposed_pinned.deinit(std.testing.allocator); + const copied_edit = transposed_pinned.view(i32).slice(1, 4).edit(&scratch); + try std.testing.expectEqual(@as([*]i32, &scratch), copied_edit.items.ptr); + @memcpy(copied_edit.items, &[_]i32{ 1, 2, 3, 4 }); + try std.testing.expectEqualSlices(i32, &.{ 0, 0, 0, 0, 0, 0 }, &column_major); + copied_edit.commit(); + try std.testing.expectEqualSlices(i32, &.{ 0, 3, 1, 4, 2, 0 }, &column_major); + + const none: HostAccessible.View(i32) = .empty; + none.edit(&.{}).commit(); +} + +test "HostAccessible.View.indexOfScalar and indexOfAny search blocks in row-major order" { + var fixture: TestShards = .{}; + defer fixture.deinit(); + const values = try fixture.view(TestShards.tiled); + values.fillIota(0, 1); + values.set(20, 7); + // Tiles split the view into blocks, and shards into regions. + try std.testing.expectEqual(7, values.indexOfScalar(7)); + try std.testing.expectEqual(12, values.slice(8, null).indexOfScalar(7)); + try std.testing.expectEqual(null, values.slice(8, 12).indexOfScalar(7)); + try std.testing.expectEqual(null, values.indexOfScalar(-1)); + try std.testing.expectEqual(null, HostAccessible.View(i32).empty.indexOfScalar(0)); + + try std.testing.expectEqual(5, values.indexOfAny(&.{ 7, 5 })); + try std.testing.expectEqual(12, values.slice(8, null).indexOfAny(&.{ -1, 7 })); + try std.testing.expectEqual(null, values.slice(8, 12).indexOfAny(&.{ 7, -1 })); + try std.testing.expectEqual(null, values.indexOfAny(&.{})); +} + +test "HostAccessible.View indexes replicas of a dense array directly" { + const shape_ = Shape.init(.{ 2, 3 }, .i32); + var storage: [2][6]i32 = @splat(@splat(-1)); + const ptrs = [_][*]u8{ @ptrCast(&storage[0]), @ptrCast(&storage[1]) }; + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, .{ .strides = .{ .byte_strides = &.{ 12, 4 } } }, &.{ 0, 0 }, &ptrs); + defer pinned.deinit(std.testing.allocator); + try std.testing.expect(pinned.mapping.contiguous); + const values = pinned.view(i32); + for (0..6) |i| values.set(i, @intCast(10 + i)); + values.slice(4, null).set(1, 99); + for (storage) |replica| try std.testing.expectEqualSlices(i32, &.{ 10, 11, 12, 13, 14, 99 }, &replica); + try std.testing.expectEqual(99, values.get(5)); + try std.testing.expectEqual(13, values.slice(2, 3).get(1)); + + inline for (.{ false, true }) |reverse| { + const nested = values.slice(1, 4); + var blocks = if (reverse) nested.reverseBlocks() else nested.blocks(); + const block = blocks.next().?; + try std.testing.expectEqual(0, block.offset); + try std.testing.expectEqual(2, block.replicas.len); + for (0..block.replicas.len, &storage) |i, *replica| { + const items = block.replica(i); + try std.testing.expectEqualSlices(i32, &.{ 11, 12, 13, 14 }, items); + try std.testing.expectEqual(replica[1..].ptr, items.ptr); + } + try std.testing.expectEqual(null, blocks.next()); + } +} + +test "HostAccessible.View set and get address single elements in every replica" { + var fixture: TestShards = .{}; + defer fixture.deinit(); + const values = try fixture.view(TestShards.tiled); + var expected: [24]i32 = undefined; + var i: usize = expected.len; + while (i > 0) { + i -= 1; + expected[i] = @intCast(100 + i); + values.set(i, expected[i]); + } + try fixture.expectValues(expected); + for (expected, 0..) |value, index| try std.testing.expectEqual(value, values.get(index)); + + // Indices of a nested view are relative to its start. + const nested = values.slice(7, 10); + nested.set(3, -5); + expected[10] = -5; + try fixture.expectValues(expected); + try std.testing.expectEqual(-5, nested.get(3)); +} + +test "HostAccessible.View copies across layouts, shards and replicas" { + var fixture: TestShards = .{}; + defer fixture.deinit(); + const values = try fixture.view(TestShards.tiled); + values.fill(0); + values.slice(2, 20).fillIota(10, 1); + values.set(17, 99); + const expected = [_]i32{ 0, 0, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 99, 26, 27, 28, 29, 0, 0 }; + var actual: [24]i32 = undefined; + values.copyTo(&actual); + try std.testing.expectEqualSlices(i32, &expected, &actual); + for (expected, 0..) |value, i| try std.testing.expectEqual(value, values.get(i)); + try fixture.expectValues(expected); + + // Blocks expose every replica, respect physical boundaries, and stop at + // the end of a nested view even when the physical run continues. + const nested = values.slice(1, null).slice(2, 4); + var nested_blocks = nested.blocks(); + const first = nested_blocks.next().?; + try std.testing.expectEqual(0, first.offset); + try std.testing.expectEqual(2, first.replicas.len); + for (0..first.replicas.len, [_]usize{ 1, 5 }) |i, storage_index| { + const items = first.replica(i); + try std.testing.expectEqualSlices(i32, expected[3..5], items); + try std.testing.expectEqual(fixture.storage[storage_index][0..].ptr, items.ptr); + } + const middle = nested_blocks.next().?; + try std.testing.expectEqual(2, middle.offset); + for (0..middle.replicas.len) |i| try std.testing.expectEqualSlices(i32, expected[5..6], middle.replica(i)); + const last = nested_blocks.next().?; + try std.testing.expectEqual(3, last.offset); + for (0..last.replicas.len, [_]usize{ 0, 4 }) |i, storage_index| { + const items = last.replica(i); + try std.testing.expectEqualSlices(i32, expected[6..7], items); + try std.testing.expectEqual(fixture.storage[storage_index][2..].ptr, items.ptr); + } + try std.testing.expectEqual(null, nested_blocks.next()); + + // Copy between differently shaped/tiled ranges, including a nonzero source + // and destination start, and verify every destination replica physically. + const dst_shape = Shape.init(.{ 3, 7 }, .i32); + const dst_layout: pjrt.MemoryLayout = .{ .tiled = .{ + .minor_to_major = &.{ 0, 1 }, + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + } }; + var dst_storage: [2][21]i32 = @splat(@splat(-1)); + const dst_ptrs = [_][*]u8{ @ptrCast(&dst_storage[0]), @ptrCast(&dst_storage[1]) }; + const dst_pinned = try HostAccessible.initForTests(std.testing.allocator, dst_shape, dst_shape, dst_layout, &.{ 0, 0 }, &dst_ptrs); + defer dst_pinned.deinit(std.testing.allocator); + const dst = dst_pinned.view(i32); + dst.slice(1, 19).copyFromView(values.slice(1, 22).slice(1, 19)); + for (dst_storage) |data| { + for (0..21) |i| { + const physical = (i % 7) * 3 + i / 7; + try std.testing.expectEqual(if (i > 0 and i < 20) expected[i + 1] else -1, data[physical]); + } + } + var empty: [0]i32 = .{}; + values.slice(values.len, 0).copyTo(&empty); + values.slice(0, 0).copyFromView(dst.slice(0, 0)); + values.slice(values.len, null).fill(-1); + values.slice(values.len, null).fillIota(-1, 1); + values.slice(values.len, null).copyFrom(&empty); + values.copyTo(&actual); + try std.testing.expectEqualSlices(i32, &expected, &actual); +} + +test "HostAccessible await after executable donation permits the next host update" { + const zml = @import("../zml.zig"); + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = testing.env(); + const shape_ = Shape.init(.{ 3, 5 }, .i32); + const Test = struct { + fn increment(input: zml.Tensor) zml.Tensor { + const host_input = input.onMemory(.host_pinned); + return host_input.toMemory(.device).addConstant(1).toMemory(.host_pinned).reuseBuffer(host_input); + } + }; + var exe = try platform.compileFn(allocator, io, Test.increment, .{zml.Tensor.fromShape(shape_)}, .{}); + defer exe.deinit(); + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + + var pinned = try HostAccessible.init(allocator, io, platform, shape_, .donatable); + defer pinned.deinit(allocator); + try std.testing.expectEqual(pinned.buffer._shards.len, pinned.mapping.allocations.entries.len); + const first_ptr = pinned.mapping.allocations.entries.items(.ptr)[0]; + pinned.view(i32).fillIota(0, 1); + + // Reuse the same runner and output destination as the model execution paths. + // Only await waits for the asynchronous executable before host access. + for (0..3) |iteration| { + runner.run(io, .{pinned.buffer}, .{&pinned.buffer}, .{}); + try pinned.await(io); + + // Reacquire from the output handles, even when donation reuses memory. + const values = pinned.view(i32); + var actual: [15]i32 = undefined; + values.copyTo(&actual); + try std.testing.expectEqual(first_ptr, pinned.mapping.allocations.entries.items(.ptr)[0]); + for (actual, 0..) |value, i| { + const expected: i32 = if (iteration == 0 or i < 3 or i >= 12) + @intCast(i + iteration + 1) + else + @intCast(iteration * 100 + i - 3 + 1); + try std.testing.expectEqual(expected, value); + } + + // Also check the executable's logical output through PJRT, independently + // of the host-visible layout used above. + var device_values: [15]i32 = undefined; + try pinned.buffer.toSlice(io, .init(shape_, std.mem.asBytes(&device_values))); + try std.testing.expectEqualSlices(i32, &actual, &device_values); + + values.slice(3, 9).fillIota(@intCast((iteration + 1) * 100), 1); + } +} + +test "HostAccessible await follows an output that doesn't reuse the pinned allocation" { + const zml = @import("../zml.zig"); + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = testing.env(); + const shape_ = Shape.init(.{ 3, 5 }, .i32); + const Test = struct { + fn increment(input: zml.Tensor) zml.Tensor { + // Without reuseBuffer, execution writes a new host allocation. + return input.onMemory(.host_pinned).toMemory(.device).addConstant(1).toMemory(.host_pinned); + } + }; + var exe = try platform.compileFn(allocator, io, Test.increment, .{zml.Tensor.fromShape(shape_)}, .{}); + defer exe.deinit(); + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + + var pinned = try HostAccessible.init(allocator, io, platform, shape_, .donatable); + defer pinned.deinit(allocator); + pinned.view(i32).fillIota(0, 1); + + for (0..2) |iteration| { + // The input isn't donated, so release it once execution replaced it. + var input = pinned.buffer; + defer input.deinit(); + const previous = pinned.mapping.allocations.entries.items(.ptr)[0]; + runner.run(io, .{input}, .{&pinned.buffer}, .{}); + try pinned.await(io); + const values = pinned.view(i32); + try std.testing.expect(pinned.mapping.allocations.entries.items(.ptr)[0] != previous); + + var actual: [15]i32 = undefined; + values.copyTo(&actual); + for (actual, 0..) |value, i| try std.testing.expectEqual(@as(i32, @intCast(iteration * 100 + i + 1)), value); + var device_values: [15]i32 = undefined; + try pinned.buffer.toSlice(io, .init(shape_, std.mem.asBytes(&device_values))); + try std.testing.expectEqualSlices(i32, &actual, &device_values); + + // Host writes now land in the output, which the next execution reads. + values.fillIota(@intCast((iteration + 1) * 100), 1); + } +} diff --git a/zml/zml.zig b/zml/zml.zig index 7d457937..f648548f 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -33,8 +33,6 @@ pub const meta = @import("meta.zig"); pub const mlir = @import("mlirx.zig"); pub const moe = @import("moe/moe.zig"); pub const nn = @import("nn.zig"); -pub const quantization = @import("quantization.zig"); -pub const Quantization = quantization.Quantization; pub const ops = @import("ops.zig"); pub const pjrtx = @import("pjrtx.zig"); pub const platform = @import("platform.zig"); @@ -42,6 +40,8 @@ pub const Memory = platform.Memory; pub const Platform = platform.Platform; pub const Target = platform.Target; pub const tracer = @import("profiling/tracer.zig"); +pub const quantization = @import("quantization.zig"); +pub const Quantization = quantization.Quantization; pub const safetensors = @import("safetensors.zig"); pub const shape = @import("shape.zig"); pub const Shape = shape.Shape; From f82a474971d557962ee110dda5a1aab746f8e3d8 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 8 Oct 2026 16:40:59 +0200 Subject: [PATCH 769/881] zml/moe: add backend compliance tests (#886) This PR adds tests to the MoE backend. The comparison is done using a StableHLO backend that has been tested against an oracle with known value. It's adding quite some time to the test so I don't know if we want to keep that many tests. However, this is the only way we have to ensure compliance across all backends and configuration. Each backend is tested with the whole matrix it supports. A nice side-effect of this PR is that we have a working implementation of MoE that will work on all platforms. It's going to be slower but allows for a quick platform bringup. --- zml/BUILD.bazel | 1 + zml/moe/cutlass_flashinfer.zig | 1 + zml/moe/fused_experts.zig | 27 -- zml/moe/moe.zig | 10 +- zml/moe/stablehlo.zig | 61 +++- zml/moe/tests.zig | 613 +++++++++++++++++++++++++++++++++ 6 files changed, 670 insertions(+), 43 deletions(-) create mode 100644 zml/moe/tests.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 46ed4aea..46a72626 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -105,6 +105,7 @@ zig_library( "mem.zig", "meta.zig", "mlirx.zig", + "moe/tests.zig", "moe/cute_kernels/moe.zig", "moe/cute_kernels/mxfp4.zig", "moe/cute_kernels/persistent_mxfp4.zig", diff --git a/zml/moe/cutlass_flashinfer.zig b/zml/moe/cutlass_flashinfer.zig index f90a7882..734283cd 100644 --- a/zml/moe/cutlass_flashinfer.zig +++ b/zml/moe/cutlass_flashinfer.zig @@ -338,6 +338,7 @@ pub fn register(platform: *const zml.Platform) !void { } pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; if (platform.state.cuda.fi_cutlass_moe_runners == null) return false; _ = computeCapability(platform) catch return false; return true; diff --git a/zml/moe/fused_experts.zig b/zml/moe/fused_experts.zig index b59e892b..88f24b3b 100644 --- a/zml/moe/fused_experts.zig +++ b/zml/moe/fused_experts.zig @@ -258,33 +258,6 @@ test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { } } -// TODO(Corentin): Add better tests -//test "ReLU squared activation preserves width and applies threshold before squaring" { -// const allocator = std.testing.allocator; -// const io = std.testing.io; -// const platform = zml.testing.env(); -// const Local = struct { -// fn forward(x: Tensor, threshold: ?f32) Tensor { -// return applyExpertActivation(x, .relu, threshold, .concatenated); -// } -// }; -// const x: Tensor = .init(.{ .token = 1, .out = 5 }, .f32); -// const values = [_]f32{ -3, 0, 1, 2, 4 }; -// var input = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&values)); -// defer input.deinit(); -// for ([_]?f32{ null, 2 }) |threshold| { -// var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, threshold }, .{}); -// defer exe.deinit(); -// try zml.testing.expectEqualShapes(x.shape(), exe.output_shapes[0]); -// var output = try exe.eval(allocator, io, .{input}); -// defer output.deinit(); -// var actual = try output.toSliceAlloc(allocator, io); -// defer actual.free(allocator); -// const expected = [_]f32{ 0, 0, 1, 4, if (threshold != null) 4 else 16 }; -// try std.testing.expectEqualSlices(f32, &expected, actual.constItems(f32)); -// } -//} - /// Build the inputs tuple for FusedMoe and invoke it via `K.call(...)`. pub const GemmOptions = struct { input: Tensor, diff --git a/zml/moe/moe.zig b/zml/moe/moe.zig index 975aac94..32a370be 100644 --- a/zml/moe/moe.zig +++ b/zml/moe/moe.zig @@ -20,13 +20,16 @@ const log = std.log.scoped(.@"zml/moe"); test { std.testing.refAllDecls(@This()); + _ = @import("tests.zig"); } /// How a backend expects the expert weights to be stored /// The packer writes this layout and forwardMoe reads it pub const ExpertsLayout = struct { - /// Column order of the fused gate/up projection. + /// Whether the two projections occupy contiguous halves or alternate. gate_up: ProjectionLayout, + /// Projection order within the concatenated halves or interleaved pairs. + gate_up_order: enum { gate_up, up_gate } = .gate_up, /// Backend specific transformation of the weights and scales. packing: Packing, @@ -155,7 +158,10 @@ pub const Backend = enum { .stablehlo => .{ .gate_up = .concatenated, .packing = .plain }, .cute_mxfp4 => if (scheme == .mxfp4) .{ .gate_up = .interleaved, .packing = .swizzled_scales } else error.UnsupportedQuantization, .triton_mxfp4 => if (scheme == .mxfp4) .{ .gate_up = .interleaved, .packing = .plain } else error.UnsupportedQuantization, - .flashinfer_cutlass => if (scheme == .nvfp4) .{ .gate_up = .concatenated, .packing = .flashinfer_nvfp4 } else error.UnsupportedQuantization, + .flashinfer_cutlass => if (scheme) |s| switch (s) { + .nvfp4 => .{ .gate_up = .concatenated, .packing = .flashinfer_nvfp4 }, + else => error.UnsupportedQuantization, + } else .{ .gate_up = .concatenated, .gate_up_order = .up_gate, .packing = .plain }, .mosaic_tpu, .metal => .{ .gate_up = .concatenated, .packing = .plain }, .triton, .fly => if (scheme == .mxfp4) .{ .gate_up = .interleaved, .packing = .plain } else .{ .gate_up = .concatenated, .packing = .plain }, }; diff --git a/zml/moe/stablehlo.zig b/zml/moe/stablehlo.zig index 1e05db5e..29d17f9a 100644 --- a/zml/moe/stablehlo.zig +++ b/zml/moe/stablehlo.zig @@ -96,9 +96,12 @@ fn roundFp4(x: Tensor) Tensor { // Quantize and dequantize activation groups using tensor math, independently of // Triton's quantizer. The return value is the represented value in FP32. -fn roundedInput(x: Tensor, linear: zml.nn.Linear, quantize: bool, dtype: zml.DataType) Tensor { +fn roundedInput(x: Tensor, linear: zml.nn.Linear, opts: Options, dtype: zml.DataType) Tensor { + if (opts.input_quantization == .none) return x.convert(dtype).convert(.f32); const q = linear.quantization orelse return x.convert(dtype).convert(.f32); - if (q.scheme == .nvfp4) { + const fixed_mxfp8 = opts.input_quantization == .mxfp8 or opts.input_quantization == .mxfp8_bf16; + const scheme = if (fixed_mxfp8) .fp8_block32 else q.scheme; + if (scheme == .nvfp4) { const global = if (q.input_scale) |scale| scale.asMultiplier() else Tensor.scalar(1, .f32); stdx.debug.assert(global.count() == 1, "reference NVFP4 input scale must be global", .{}); const grouped = x.convert(.bf16).convert(.f32).div(global.asScalar()) @@ -110,28 +113,29 @@ fn roundedInput(x: Tensor, linear: zml.nn.Linear, quantize: bool, dtype: zml.Dat return roundFp4(grouped.mul(Tensor.scalar(1, .f32).div(divisor))).mul(scale) .reshape(x.shape().withDtype(.f32)).mul(global.asScalar()); } - if (!quantize or q.scheme == .mxfp4) return x.convert(dtype).convert(.f32); - stdx.debug.assert(q.scheme != .nvfp4, "reference FP8 input quantization does not accept NVFP4", .{}); - const group: i64 = switch (q.scheme) { + if (opts.input_quantization == .automatic and (!opts.quantize_input or scheme == .mxfp4)) return x.convert(dtype).convert(.f32); + const output_dtype = if (fixed_mxfp8) .f8e4m3fn else linear.weight.dtype(); + const group: i64 = switch (scheme) { .mxfp8, .fp8_block32 => 32, .fp8_block128 => 128, .fp8_per_channel, .fp8_per_tensor => x.dim(.in), else => unreachable, }; - const grouped = x.convert(.f32).splitAxis(.in, .{ .group = @divExact(x.dim(.in), group), .element = group }); - const epsilon: f32 = if (q.scheme == .fp8_block32) 1e-4 else 1e-10; - const max_value: f32 = if (linear.weight.dtype() == .f8e4m3fnuz) 224 else 448; + const source = if (opts.input_quantization == .mxfp8_bf16) x.convert(.bf16) else x; + const grouped = source.convert(.f32).splitAxis(.in, .{ .group = @divExact(x.dim(.in), group), .element = group }); + const epsilon: f32 = if (scheme == .fp8_block32) 1e-4 else 1e-10; + const max_value: f32 = if (output_dtype == .f8e4m3fnuz) 224 else 448; var scale = grouped.abs().max(.element).maximum(Tensor.scalar(epsilon, .f32)).scale(1 / max_value); - if (q.scheme == .mxfp8 or q.scheme == .fp8_block32) scale = scale.log().scale(1 / @log(@as(f32, 2))).ceil().scale(@log(@as(f32, 2))).exp(); + if (scheme == .mxfp8 or scheme == .fp8_block32) scale = scale.log().scale(1 / @log(@as(f32, 2))).ceil().scale(@log(@as(f32, 2))).exp(); const broad = scale.broad(grouped.shape()); return grouped.div(broad).clamp(Tensor.scalar(-max_value, .f32), Tensor.scalar(max_value, .f32)) - .convert(linear.weight.dtype()).convert(.f32).mul(broad).reshape(x.shape().withDtype(.f32)); + .convert(output_dtype).convert(.f32).mul(broad).reshape(x.shape().withDtype(.f32)); } -fn project(x: Tensor, linear: zml.nn.Linear, ids: Tensor, quantize: bool, dtype: zml.DataType) Tensor { +fn project(x: Tensor, linear: zml.nn.Linear, ids: Tensor, opts: Options, dtype: zml.DataType) Tensor { const weight = dequantize(linear).withTags(.{ .expert, .out, .in }); const selected = weight.gather(.{ .expert = ids }, .{}); - var result = roundedInput(x, linear, quantize, dtype).dot(selected, .in); + var result = roundedInput(x, linear, opts, dtype).dot(selected, .in); if (linear.bias) |bias| { const selected_bias = if (bias.rank() == 2) bias.withTags(.{ .expert, .out }).gather(.{ .expert = ids }, .{}) @@ -145,6 +149,10 @@ fn project(x: Tensor, linear: zml.nn.Linear, ids: Tensor, quantize: bool, dtype: pub const Options = struct { activation: zml.moe.Activation, quantize_input: bool = false, + /// Model fixed backend contracts independently of optional input quantization: + /// Metal uses weight-only quantization; specialized MXFP4 GEMMs always use MXFP8 inputs. + /// CuTe additionally rounds the intermediate activation to BF16 before quantizing. + input_quantization: enum { automatic, none, mxfp8, mxfp8_bf16 } = .automatic, routing_weight_placement: @import("fused_experts.zig").RoutingWeightPlacement = .after_down, }; @@ -163,10 +171,12 @@ pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: const hidden = input.reshape(.{ .token = b * s, .in = input.dim(.d) }) .insertAxes(.in, .{.topk}).broad(Shape.init(.{ .token = b * s, .topk = k, .in = input.dim(.d) }, input.dtype())) .merge(.{ .route = .{ .token, .topk } }); - const projected = project(hidden, gate_up, ids, opts.quantize_input, input.dtype()).convert(input.dtype()).convert(.f32); + // Keep the projection's rounding boundary when XLA fuses activation/routing + // arithmetic; otherwise the reference can disagree with explicit BF16 GEMMs. + const projected = project(hidden, gate_up, ids, opts, input.dtype()).convert(input.dtype()).optimizationBarrier().convert(.f32); var activated = activate(projected, opts.activation); if (opts.routing_weight_placement == .before_down) activated = activated.mul(weights.broad(activated.shape())); - var output = project(activated.rename(.{ .out = .in }), down, ids, opts.quantize_input, input.dtype()); + var output = project(activated.rename(.{ .out = .in }), down, ids, opts, input.dtype()); if (opts.routing_weight_placement == .after_down) output = output.mul(weights.broad(output.shape())); return output.convert(input.dtype()).convert(.f32) .reshape(.{ .b = b, .s = s, .topk = k, .d = output.dim(.out) }) @@ -180,3 +190,26 @@ pub fn fusedExperts(input: Tensor, ids: Tensor, weights: Tensor, gate_up: zml.nn .routing_weight_placement = opts.routing_weight_placement, }); } + +test "MoE compliance: FP4 midpoint rounding" { + const Local = struct { + fn call(x: Tensor) Tensor { + return roundFp4(x); + } + }; + const values = [_]f32{ -7, -5, -3.5, -2.5, -1.75, -1.25, -0.75, -0.25, 0, 0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5, 7 }; + const expected = [_]f32{ -6, -4, -4, -2, -2, -1, -1, 0, 0, 0, 1, 1, 2, 2, 4, 4, 6 }; + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + const x: Tensor = .init(.{values.len}, .f32); + var exe = try platform.compileFn(allocator, io, Local.call, .{x}, .{}); + defer exe.deinit(); + var input = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&values)); + defer input.deinit(); + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + const actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + try std.testing.expectEqualSlices(f32, &expected, actual.constItems(f32)); +} diff --git a/zml/moe/tests.zig b/zml/moe/tests.zig new file mode 100644 index 00000000..43b3e1ac --- /dev/null +++ b/zml/moe/tests.zig @@ -0,0 +1,613 @@ +//! Shared public-entry-point tests. Hardware absence is the only runtime skip: +//! supported cases are explicit, and errors in those cases must fail the test. +const std = @import("std"); +const zml = @import("../zml.zig"); +const Tensor = zml.Tensor; +const Shape = zml.Shape; +const reference = @import("stablehlo.zig"); +const Placement = @import("fused_experts.zig").RoutingWeightPlacement; + +const Case = struct { + backend: zml.moe.Backend, + activation: zml.moe.Activation = .{ .swiglu = .{} }, + scheme: ?zml.Quantization.Scheme = null, + activation_dtype: zml.DataType = .bf16, + quantize_input: bool = false, + placement: Placement = .after_down, + bias: bool = false, + tokens: i64 = 1, + topk: i64 = 1, + batch: i64 = 1, + width: i64 = 128, + intermediate: i64 = 128, + experts: i64 = 8, + + pub fn compile(c: Case, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Exe { + std.debug.assert(c.backend.isAvailable(platform)); + const x: Tensor = .init(.{ .b = c.batch, .s = c.tokens, .d = c.width }, c.activation_dtype); + return platform.compileFn(allocator, io, forward, .{ x, c }, .{}); + } + + pub fn check(c: Case, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, exe: *const zml.Exe) !void { + const x: Tensor = .init(.{ .b = c.batch, .s = c.tokens, .d = c.width }, c.activation_dtype); + + var host = try zml.Slice.alloc(allocator, x.shape()); + defer host.free(allocator); + switch (c.activation_dtype) { + inline .bf16, .f16, .f32 => |dtype| { + for (host.items(dtype.toZigType()), 0..) |*value, i| { + const f = @as(f32, @floatFromInt(@as(i32, @intCast((i * 13 + i / 128) % 23)) - 11)) / 32; + value.* = if (dtype == .bf16) .fromF32(f) else @floatCast(f); + } + }, + else => unreachable, + } + + var input = try zml.Buffer.fromSlice(io, platform, host); + defer input.deinit(); + + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + var output: zml.Bufferized(Outputs) = undefined; + runner.run(io, .{input}, .{&output}, .{ .wait = true }); + defer zml.Buffer.deinitAll(Outputs, &output); + + // BF16 intermediate rounding and different GEMM reduction orders. Every + // element must pass; near-zero values use the absolute bound. + try zml.testing.expectClose(io, output.expected, output.actual, .{ + // The NVFP4 matrix observed a maximum absolute error of 0.07519531. + // Allow a small margin; this is an empirical comparison budget. + .absolute_tolerance = if (c.scheme == .nvfp4) 0.078125 else 0.015625, + .relative_tolerance = 0.02, + .minimum_close_fraction = 1, + }); + } +}; + +/// Generate a reproducible Tensor with a simple pattern. +/// +/// Generate as many elements as the shape requires, with this formula: +/// value[i] = (((i * multiplier + 3) % 17) - 8) / divisor +/// +/// Then reshape to the requested shape and convert to f32. +fn generateDeterministicTensor(shape: Shape, multiplier: i64, divisor: f64) Tensor { + return Tensor.arange(.{ .end = @intCast(shape.count()) }, .i32) + .mul(Tensor.scalar(multiplier, .i32)).addConstant(3) + .remainder(Tensor.scalar(17, .i32)).addConstant(-8) + .convert(.f32).scale(1 / divisor).reshape(shape.withDtype(.f32)); +} + +/// Select exactly representable FP4 values with distinct even/odd sequences. +fn generateFp4Tensor(shape: Shape, multiplier: i64) Tensor { + const values = Tensor.constantTensor(.init(.{ .value = 16 }, .f32), std.mem.sliceAsBytes(&[_]f32{ + 0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6, + })); + const index = Tensor.arange(.{ .end = @intCast(shape.count()) }, .i32); + const pair = index.div(Tensor.scalar(2, .i32)); + // Prime periods distinguish experts, output rows, and gate/up halves. + const even = pair.mul(Tensor.scalar(multiplier, .i32)).addConstant(3) + .remainder(Tensor.scalar(17, .i32)).remainder(Tensor.scalar(16, .i32)); + const odd = pair.mul(Tensor.scalar(3, .i32)).addConstant(7) + .remainder(Tensor.scalar(19, .i32)).remainder(Tensor.scalar(16, .i32)); + const indices = index.remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)).select(even, odd); + return values.gather(.{ .value = indices }, .{}).convert(.f4e2m1).reshape(shape.withDtype(.f4e2m1)); +} + +/// Create a Linear fixture with a reproducible pattern, optionally quantized and biased. +fn linear(shape: Shape, scheme: ?zml.Quantization.Scheme, bias: bool, multiplier: i64) zml.nn.Linear { + const fixture_bias = if (bias) generateDeterministicTensor(shape.remove(2), 3, 32).convert(shape.dtype()) else null; + + const s = scheme orelse return .{ + .weight = generateDeterministicTensor(shape, multiplier, 16).convert(shape.dtype()), + .tag = shape.tag(2), + .bias = fixture_bias, + }; + + const weight = switch (s) { + .mxfp4, .nvfp4 => generateFp4Tensor(shape, multiplier), + else => generateDeterministicTensor(shape, multiplier, 16).convert(.f8e4m3fn), + }; + + const scale_shape: Shape = switch (s) { + .mxfp4, .mxfp8 => .init(.{ shape.dim(0), shape.dim(1), @divExact(shape.dim(2), 32) }, .f8e8m0), + .nvfp4 => .init(.{ shape.dim(0), shape.dim(1), @divExact(shape.dim(2), 16) }, .f8e4m3fn), + .fp8_per_tensor => .init(.{ shape.dim(0), 1, 1 }, .f32), + .fp8_per_channel => .init(.{ shape.dim(0), shape.dim(1), 1 }, .f32), + .fp8_block128 => .init(.{ shape.dim(0), @divExact(shape.dim(1), 128), @divExact(shape.dim(2), 128) }, .f32), + .fp8_block32 => .init(.{ shape.dim(0), @divExact(shape.dim(1), 32), @divExact(shape.dim(2), 32) }, .f32), + }; + + const scales = Tensor.arange(.{ .end = @intCast(scale_shape.count()) }, .i32) + .remainder(Tensor.scalar(2, .i32)).convert(.f32).addConstant(1).scale(1.0 / 16.0) + .reshape(scale_shape.withDtype(.f32)).convert(scale_shape.dtype()); + + const input_scale: ?zml.Quantization.GlobalScale = if (s == .nvfp4) .{ .value = Tensor.scalar(1, .f32), .operation = .multiply } else null; + const global_scale: ?zml.Quantization.GlobalScale = if (s == .nvfp4) .{ .value = Tensor.scalar(1, .f32).broad(.init(.{ .expert = shape.dim(0) }, .f32)), .operation = .multiply } else null; + + return .{ + .weight = weight, + .tag = shape.tag(2), + .bias = fixture_bias, + .quantization = .{ + .scheme = s, + .scales = scales, + .input_scale = input_scale, + .global_scale = global_scale, + }, + }; +} + +const ExpertFixture = struct { + gate_up: zml.nn.Linear, + down: zml.nn.Linear, +}; + +/// Canonical weights use logical dimensions, concatenated gate/up, and plain scales. +fn makeExpertFixture(c: Case) ExpertFixture { + const projection_width: i64 = switch (c.activation) { + .silu, .gelu, .relu => c.intermediate, + else => 2 * c.intermediate, + }; + return .{ + .gate_up = linear(.init(.{ .expert = c.experts, .dout = projection_width, .d = c.width }, c.activation_dtype), c.scheme, c.bias, 7), + .down = linear(.init(.{ .expert = c.experts, .d = c.width, .dout = c.intermediate }, c.activation_dtype), c.scheme, c.bias, 11), + }; +} + +/// Interleave the gate/up halves of the output axis: [E, 2, N/2, K] -> [E, N/2, 2, K]. +fn interleave(tensor: Tensor) Tensor { + const tags = tensor.shape().tags(); + return tensor.withTags(.{ .expert, .out, .in }) + .splitAxis(.out, .{ .projection = 2, .mid = @divExact(tensor.dim(1), 2) }) + .transpose(.{ .expert, .mid, .projection, .in }) + .merge(.{ .out = .{ .mid, .projection } }).withTags(tags); +} + +/// Blackwell block scales: [E, N/128, 4, 32, K/4, 4] -> +/// [E, N/128, K/4, 32, 4, 4], retaining the declared [E, N, K] shape. +fn swizzleScales(scales: Tensor) Tensor { + return scales.reshape(.{ scales.dim(0), @divExact(scales.dim(1), 128), 4, 32, @divExact(scales.dim(2), 4), 4 }) + .transpose(.{ 0, 1, 4, 3, 2, 5 }).reshape(scales.shape()); +} + +fn packFp4Bytes(weight: Tensor) Tensor { + std.debug.assert(weight.dtype() == .f4e2m1); + return weight.reshape(weight.shape().setDim(2, @divExact(weight.dim(2), 2)).append(.{ .nibble = 2 })).bitCast(.u8); +} + +fn prepareGateUp(tensor: Tensor, layout: zml.moe.ExpertsLayout) Tensor { + const ordered = switch (layout.gate_up_order) { + .gate_up => tensor, + .up_gate => blk: { + const mid = @divExact(tensor.dim(1), 2); + break :blk Tensor.concatenate(&.{ tensor.slice(1, .{ .start = mid }), tensor.slice(1, .{ .end = mid }) }, 1); + }, + }; + return switch (layout.gate_up) { + .concatenated => ordered, + .interleaved => interleave(ordered), + }; +} + +/// Adapt a copy of both projections to the backend's storage and layout contract. +fn prepareForBackend(canonical: ExpertFixture, backend: zml.moe.Backend) ExpertFixture { + var prepared = canonical; + const scheme = canonical.gate_up.quantizationScheme(); + const layout = backend.expertsLayout(scheme) catch |err| zml.stdx.debug.panic("fixture layout: {}", .{err}); + // Plain activations have one projection, so gate/up ordering does not apply. + const gated = canonical.gate_up.weight.dim(1) == 2 * canonical.down.weight.dim(2); + if (gated) { + prepared.gate_up.weight = prepareGateUp(canonical.gate_up.weight, layout); + if (canonical.gate_up.bias) |bias| prepared.gate_up.bias = prepareGateUp(bias.appendAxes(.{.dummy}), layout).squeeze(.dummy); + if (prepared.gate_up.quantization) |*q| q.scales = prepareGateUp(q.scales, layout); + } + + for ([_]*zml.nn.Linear{ &prepared.gate_up, &prepared.down }) |projection| { + if (layout.packing == .swizzled_scales or layout.packing == .flashinfer_nvfp4) { + const q = &projection.quantization.?; + q.scales = swizzleScales(q.scales); + q.swizzled_scales = true; + } + if (backend == .cute_mxfp4 or backend == .triton_mxfp4) { + projection.weight = packFp4Bytes(projection.weight); + projection.quantization.?.scales = projection.quantization.?.scales.bitCast(.u8); + } + if (backend == .mosaic_tpu) projection.weight = projection.weight.transpose(.{ 0, 2, 1 }); + if (backend == .metal) { + if (projection.quantization) |*q| { + if (q.scheme != .nvfp4) q.scales = q.scales.convert(.bf16); + } + } + } + if (backend == .metal and scheme == .nvfp4) { + const global = &prepared.gate_up.quantization.?.global_scale.?; + global.value = global.value.appendAxes(.{.proj}).repeat1d(.proj, 2); + } + return prepared; +} + +const Outputs = struct { actual: Tensor, expected: Tensor }; + +/// Forward a fixture through the MoE implementation under test, and also through the StableHLO +/// reference implementation. Return both outputs for comparison. +fn forward(input: Tensor, c: Case) Outputs { + const canonical = makeExpertFixture(c); + const prepared = prepareForBackend(canonical, c.backend); + + const route_shape = Shape.init(.{ .b = c.batch, .s = c.tokens, .topk = c.topk }, .i32); + const route = Tensor.arange(.{ .end = @intCast(route_shape.count()) }, .i32).reshape(route_shape); + + // Only six of eight experts receive routes, with nonuniform counts. + const ids = route.mul(Tensor.scalar(5, .i32)).addConstant(2).remainder(Tensor.scalar(@min(c.experts, 6), .i32)); + + const weights = route.remainder(Tensor.scalar(3, .i32)).convert(.f32).addConstant(1).scale(0.25); + const options: zml.moe.Options = .{ + .activation = c.activation, + .quantize_input = c.quantize_input, + .routing_weight_placement = c.placement, + }; + + const actual = zml.moe.forwardMoe(input, ids, weights, prepared.gate_up, prepared.down, c.backend, options); + const expected = reference.reference(input, ids, weights, canonical.gate_up, canonical.down, .{ + .activation = c.activation, + .quantize_input = c.quantize_input, + .routing_weight_placement = c.placement, + .input_quantization = switch (c.backend) { + .metal => .none, + .cute_mxfp4 => .mxfp8_bf16, + .triton_mxfp4 => .mxfp8, + else => .automatic, + }, + }); + + return .{ .actual = actual.reshape(input.shape()), .expected = expected }; +} + +const gated_activations = [_]zml.moe.Activation{ + .{ .swiglu = .{} }, + .{ .swiglu = .{ .limit = 0.25 } }, + .{ .swiglu = .{ .bias = 1 } }, + .{ .swiglu = .{ .limit = 0.25, .bias = 1 } }, + .{ .swiglu_step = .{} }, + .{ .swiglu_step = .{ .limit = 0.25 } }, + .geglu, + .geglu_tanh, +}; +const all_activations = gated_activations ++ [_]zml.moe.Activation{ .relu, .silu, .gelu }; +const clipped_swiglu = [_]zml.moe.Activation{ + .{ .swiglu = .{ .limit = 0.25 } }, + .{ .swiglu = .{ .limit = 7 } }, +}; + +const FixtureShape = struct { + batch: i64 = 1, + tokens: i64 = 1, + topk: i64 = 1, + width: i64 = 128, + intermediate: i64 = 128, + experts: i64 = 8, +}; + +// Every option combination runs on decode, prefill, and batched prefill. +// Shapes and floating-point parameter values are representative finite sets. +const fixture_shapes = [_]FixtureShape{ + .{}, + .{ .tokens = 17, .topk = 2 }, + .{ .batch = 2, .tokens = 17, .topk = 3 }, +}; +const specialized_shapes = [_]FixtureShape{ + .{ .width = 5120, .intermediate = 2304, .experts = 2 }, + .{ .width = 5120, .intermediate = 2304, .experts = 2, .tokens = 17, .topk = 2 }, + .{ .width = 5120, .intermediate = 2304, .experts = 2, .batch = 2, .tokens = 17, .topk = 2 }, +}; + +const Format = struct { + scheme: ?zml.Quantization.Scheme = null, + activation_dtype: zml.DataType = .bf16, +}; + +const Matrix = struct { + formats: []const Format = &.{.{}}, + activations: []const zml.moe.Activation, + quantize_inputs: []const bool = &.{false}, + placements: []const Placement = &.{.after_down}, + biases: []const bool = &.{false}, + shapes: []const FixtureShape = &fixture_shapes, + + pub fn count(self: Matrix) usize { + return self.formats.len * self.activations.len * self.quantize_inputs.len * + self.placements.len * self.biases.len * self.shapes.len; + } + + /// Shapes vary fastest, matching the nested loops in the backend tests. + pub fn caseAt(self: Matrix, backend: zml.moe.Backend, index: usize) Case { + std.debug.assert(index < self.count()); + var remaining = index; + const shape = self.shapes[remaining % self.shapes.len]; + remaining /= self.shapes.len; + const bias = self.biases[remaining % self.biases.len]; + remaining /= self.biases.len; + const placement = self.placements[remaining % self.placements.len]; + remaining /= self.placements.len; + const quantize_input = self.quantize_inputs[remaining % self.quantize_inputs.len]; + remaining /= self.quantize_inputs.len; + const activation = self.activations[remaining % self.activations.len]; + remaining /= self.activations.len; + const format = self.formats[remaining]; + return .{ + .backend = backend, + .activation = activation, + .scheme = format.scheme, + .activation_dtype = format.activation_dtype, + .quantize_input = quantize_input, + .placement = placement, + .bias = bias, + .batch = shape.batch, + .tokens = shape.tokens, + .topk = shape.topk, + .width = shape.width, + .intermediate = shape.intermediate, + .experts = shape.experts, + }; + } +}; + +const Options = struct { + producers: usize = 16, + consumers: usize = 1, + case_queue_capacity: usize = 16, + executable_queue_capacity: usize = 16, +}; + +const CaseJob = struct { + index: usize, + case: Case, +}; + +const ExecutableJob = struct { + job: CaseJob, + exe: zml.Exe, +}; + +const Result = struct { + case: Case, + outcome: union(enum) { + passed, + skipped, + compilation_failed: anyerror, + check_failed: anyerror, + }, +}; + +const Pipeline = struct { + platform: *const zml.Platform, + + fn generateCases(self: *const Pipeline, io: std.Io, backend: zml.moe.Backend, matrix: Matrix, case_queue: *std.Io.Queue(CaseJob), results: []?Result) std.Io.Cancelable!void { + defer case_queue.close(io); + const available = backend.isAvailable(self.platform); + + var index: usize = 0; + for (0..matrix.count()) |local_index| { + const c = matrix.caseAt(backend, local_index); + if (available) { + case_queue.putOne(io, .{ .index = index, .case = c }) catch |err| return switch (err) { + error.Closed => {}, + error.Canceled => error.Canceled, + }; + } else { + results[index] = .{ .case = c, .outcome = .skipped }; + } + index += 1; + } + } + + fn compileExecutablesWorker(self: *const Pipeline, io: std.Io, allocator: std.mem.Allocator, case_queue: *std.Io.Queue(CaseJob), executable_queue: *std.Io.Queue(ExecutableJob), results: []?Result) std.Io.Cancelable!void { + while (true) { + const job = case_queue.getOne(io) catch |err| return switch (err) { + error.Closed => {}, + error.Canceled => error.Canceled, + }; + var exe = job.case.compile(allocator, io, self.platform) catch |err| { + if (err == error.Canceled) return error.Canceled; + results[job.index] = .{ .case = job.case, .outcome = .{ .compilation_failed = err } }; + continue; + }; + // Ownership transfers only on a successful enqueue. + executable_queue.putOne(io, .{ .job = job, .exe = exe }) catch |err| { + exe.deinit(); + return switch (err) { + error.Closed => {}, + error.Canceled => error.Canceled, + }; + }; + } + } + + fn runTestCaseWorker(self: *const Pipeline, io: std.Io, allocator: std.mem.Allocator, executable_queue: *std.Io.Queue(ExecutableJob), results: []?Result) std.Io.Cancelable!void { + while (true) { + var compiled = executable_queue.getOne(io) catch |err| return switch (err) { + error.Closed => {}, + error.Canceled => error.Canceled, + }; + defer compiled.exe.deinit(); + const job = compiled.job; + job.case.check(allocator, io, self.platform, &compiled.exe) catch |err| { + if (err == error.Canceled) return error.Canceled; + results[job.index] = .{ .case = job.case, .outcome = .{ .check_failed = err } }; + continue; + }; + results[job.index] = .{ .case = job.case, .outcome = .passed }; + } + } + + fn run(self: *const Pipeline, io: std.Io, allocator: std.mem.Allocator, backend: zml.moe.Backend, matrix: Matrix, options: Options) !void { + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + + const cases_buffer = try arena.allocator().alloc(CaseJob, options.case_queue_capacity); + const executables_buffer = try arena.allocator().alloc(ExecutableJob, options.executable_queue_capacity); + const results = try arena.allocator().alloc(?Result, matrix.count()); + @memset(results, null); + + var case_queue: std.Io.Queue(CaseJob) = .init(cases_buffer); + var executable_queue: std.Io.Queue(ExecutableJob) = .init(executables_buffer); + + var enumerator: std.Io.Group = .init; + var producers: std.Io.Group = .init; + var consumers: std.Io.Group = .init; + defer { + case_queue.close(io); + executable_queue.close(io); + enumerator.cancel(io); + producers.cancel(io); + consumers.cancel(io); + // On partial startup or cancellation, queued executables still own resources. + while (executable_queue.getOneUncancelable(io)) |compiled| { + compiled.exe.deinit(); + } else |_| {} + } + + for (0..options.consumers) |_| try consumers.concurrent(io, runTestCaseWorker, .{ self, io, allocator, &executable_queue, results }); + for (0..options.producers) |_| try producers.concurrent(io, compileExecutablesWorker, .{ self, io, allocator, &case_queue, &executable_queue, results }); + try enumerator.concurrent(io, generateCases, .{ self, io, backend, matrix, &case_queue, results }); + + try enumerator.await(io); + try producers.await(io); + // Closed queues drain before getOne returns error.Closed. + executable_queue.close(io); + try consumers.await(io); + + try report(results); + } + + fn report(results: []const ?Result) !void { + var failed: usize = 0; + for (results, 0..) |result, index| { + const r = result orelse return error.MissingCaseResult; + switch (r.outcome) { + .passed, .skipped => {}, + inline .compilation_failed, .check_failed => |err, stage| { + failed += 1; + std.debug.print("MoE case {}/{} {s}: {any}\nError: {}\n", .{ + index + 1, results.len, @tagName(stage), r.case, err, + }); + }, + } + } + if (failed != 0) return error.MoeComplianceFailed; + } +}; + +test "Triton compatibility matrix" { + const platform = zml.testing.env(); + + const pipeline: Pipeline = .{ .platform = platform }; + + try pipeline.run(std.testing.io, std.testing.allocator, .triton, .{ + .formats = &.{ + .{}, + .{ .scheme = .mxfp4 }, + .{ .scheme = .mxfp8 }, + .{ .scheme = .fp8_per_tensor }, + .{ .scheme = .fp8_per_channel }, + .{ .scheme = .fp8_block128 }, + .{ .scheme = .fp8_block32 }, + }, + .activations = &all_activations, + // BF16 and MXFP4 ignore this option; exercise both accepted settings. + .quantize_inputs = &.{ false, true }, + .placements = &.{ .before_down, .after_down }, + .biases = &.{ false, true }, + }, .{}); +} + +test "FlashInfer compatibility matrix" { + const platform = zml.testing.env(); + + const pipeline: Pipeline = .{ .platform = platform }; + + // Only default SwiGLU, no linear bias, and routing after the down projection. + try pipeline.run(std.testing.io, std.testing.allocator, .flashinfer_cutlass, .{ + .formats = &.{ .{}, .{ .scheme = .nvfp4 } }, + .activations = &.{.{ .swiglu = .{} }}, + }, .{}); +} + +test "specialized Triton MXFP4 compatibility matrix" { + const platform = zml.testing.env(); + + const pipeline: Pipeline = .{ .platform = platform }; + + // The kernel always quantizes inputs to MXFP8, independently of the option. + try pipeline.run(std.testing.io, std.testing.allocator, .triton_mxfp4, .{ + .formats = &.{.{ .scheme = .mxfp4 }}, + .activations = &clipped_swiglu, + .quantize_inputs = &.{ false, true }, + .placements = &.{ .before_down, .after_down }, + }, .{}); +} + +test "specialized CuTe MXFP4 compatibility matrix" { + const platform = zml.testing.env(); + + const pipeline: Pipeline = .{ .platform = platform }; + + try pipeline.run(std.testing.io, std.testing.allocator, .cute_mxfp4, .{ + .formats = &.{.{ .scheme = .mxfp4 }}, + .activations = &clipped_swiglu, + .quantize_inputs = &.{ false, true }, + .placements = &.{.before_down}, + .shapes = &specialized_shapes, + }, .{}); +} + +test "Fly compatibility matrix" { + const platform = zml.testing.env(); + + const pipeline: Pipeline = .{ .platform = platform }; + + // These constraints keep execution on Fly rather than its Triton fallback. + try pipeline.run(std.testing.io, std.testing.allocator, .fly, .{ + .formats = &.{.{ .scheme = .mxfp4 }}, + .activations = &gated_activations, + .quantize_inputs = &.{ false, true }, + .placements = &.{.before_down}, + .shapes = &specialized_shapes, + }, .{}); +} + +test "Metal compatibility matrix" { + const platform = zml.testing.env(); + + const pipeline: Pipeline = .{ .platform = platform }; + + // Quantized Metal kernels require BF16 activations and do not support bias. + try pipeline.run(std.testing.io, std.testing.allocator, .metal, .{ + .formats = &.{ + .{}, + .{ .activation_dtype = .f16 }, + .{ .activation_dtype = .f32 }, + .{ .scheme = .nvfp4 }, + .{ .scheme = .mxfp8 }, + .{ .scheme = .fp8_per_tensor }, + .{ .scheme = .fp8_per_channel }, + .{ .scheme = .fp8_block128 }, + .{ .scheme = .fp8_block32 }, + }, + .activations = &all_activations, + }, .{}); +} + +test "Mosaic TPU compatibility matrix" { + const platform = zml.testing.env(); + + const pipeline: Pipeline = .{ .platform = platform }; + + // canonicalizeDown currently requires a gated projection; quantized weights + // are not supported by gmmDType. Plain activations need a backend fix first. + try pipeline.run(std.testing.io, std.testing.allocator, .mosaic_tpu, .{ + .formats = &.{ .{}, .{ .activation_dtype = .f16 }, .{ .activation_dtype = .f32 } }, + .activations = &gated_activations, + .biases = &.{ false, true }, + }, .{}); +} From 74562d518ddc8d7f4928741244c6dca2631bf2bc Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Fri, 9 Oct 2026 12:21:16 +0200 Subject: [PATCH 770/881] zml/platform: Add furiosa platform (#871) Co-authored-by: Raphael Nahoum --- MODULE.bazel | 3 ++ examples/llm/models/common.zig | 2 +- platforms/BUILD.bazel | 2 + platforms/furiosa/BUILD.bazel | 21 ++++++++++ platforms/furiosa/furiosa.bzl | 21 ++++++++++ platforms/furiosa/furiosa.zig | 42 ++++++++++++++++++++ platforms/furiosa/libzml_furiosa.BUILD.bazel | 23 +++++++++++ platforms/platforms.zig | 2 + zml/Sharding.zig | 6 +-- zml/attention.zig | 2 +- zml/attention/paged_attention.zig | 2 +- zml/exe.zig | 4 +- zml/io.zig | 2 +- zml/mem.zig | 2 +- zml/ops.zig | 8 ++-- zml/platform.zig | 30 +++++++++++--- zml/tensor.zig | 12 +++--- 17 files changed, 158 insertions(+), 26 deletions(-) create mode 100644 platforms/furiosa/BUILD.bazel create mode 100644 platforms/furiosa/furiosa.bzl create mode 100644 platforms/furiosa/furiosa.zig create mode 100644 platforms/furiosa/libzml_furiosa.BUILD.bazel diff --git a/MODULE.bazel b/MODULE.bazel index 5e5d9c17..900d84af 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -233,6 +233,9 @@ use_repo( oneapi = use_extension("//platforms/oneapi:oneapi.bzl", "oneapi_packages") use_repo(oneapi, "libzml_oneapi") +furiosa = use_extension("//platforms/furiosa:furiosa.bzl", "furiosa_packages") +use_repo(furiosa, "libzml_furiosa") + tpu = use_extension("//platforms/tpu:tpu.bzl", "tpu_packages") use_repo(tpu, "libzml_tpu") diff --git a/examples/llm/models/common.zig b/examples/llm/models/common.zig index 62fc7560..8a5c785d 100644 --- a/examples/llm/models/common.zig +++ b/examples/llm/models/common.zig @@ -63,7 +63,7 @@ pub const Meshes = struct { .experts = try platform.registerMeshWithStrategy("experts", .mesh(.{ .experts = .high_bandwidth }), strategy_experts), }; }, - .cuda, .rocm, .oneapi, .neuron, .metal, .cpu => return .{ + .cuda, .rocm, .oneapi, .neuron, .metal, .cpu, .furiosa => return .{ .model = try platform.registerMesh("model", .mesh(.{ .model = .high_bandwidth })), .experts = try platform.registerMesh("experts", .mesh(.{ .experts = .high_bandwidth })), }, diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index 104bdab8..5ee2f176 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -10,6 +10,7 @@ PLATFORMS = { "neuron": False, "oneapi": False, "metal": False, + "furiosa": False, } [ @@ -68,6 +69,7 @@ zig_options( "tpu", "neuron", "oneapi", + "furiosa", ] ] diff --git a/platforms/furiosa/BUILD.bazel b/platforms/furiosa/BUILD.bazel new file mode 100644 index 00000000..3118e6d0 --- /dev/null +++ b/platforms/furiosa/BUILD.bazel @@ -0,0 +1,21 @@ +load("@rules_shell//shell:sh_test.bzl", "sh_test") +load("@rules_zig//zig:defs.bzl", "zig_library") + +exports_files(["libzml_furiosa.BUILD.bazel"]) + +zig_library( + name = "furiosa", + import_name = "platforms/furiosa", + main = "furiosa.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//bazel", + "//pjrt", + "//platforms:options", + "//stdx", + ] + select({ + "//platforms:furiosa.enabled": ["@libzml_furiosa"], + "//conditions:default": [], + }), +) diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl new file mode 100644 index 00000000..8a2cb33a --- /dev/null +++ b/platforms/furiosa/furiosa.bzl @@ -0,0 +1,21 @@ +"""Consume the complete Furiosa sandbox produced by XLA.""" + +load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") + +def _furiosa_impl(mctx): + http_archive( + name = "libzml_furiosa", + build_file = "libzml_furiosa.BUILD.bazel", + url = "https://mirror.zml.ai/plugins/202610081500.175.1.4a98481373f6/zml-furiosa-linux-amd64.tar.zst", + sha256 = "fb5569b2c3ab8604924fb66b2e9f6fd3fe38de32f58aaec25ef1aa510dfe10ee", + ) + + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = ["libzml_furiosa"], + root_module_direct_dev_deps = [], + ) + +furiosa_packages = module_extension( + implementation = _furiosa_impl, +) diff --git a/platforms/furiosa/furiosa.zig b/platforms/furiosa/furiosa.zig new file mode 100644 index 00000000..352c4342 --- /dev/null +++ b/platforms/furiosa/furiosa.zig @@ -0,0 +1,42 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); +const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); +const stdx = @import("stdx"); + +const log = std.log.scoped(.@"zml/platforms/furiosa"); + +extern fn setenv(name: [*:0]const u8, value: [*:0]const u8, overwrite: c_int) c_int; + +fn setEnv(name: [*:0]const u8, value: [:0]const u8, overwrite: c_int) !void { + if (setenv(name, value, overwrite) != 0) return error.SetEnvFailed; +} + +pub fn isEnabled() bool { + return platforms_options.furiosa_enabled; +} + +pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { + if (comptime !isEnabled() or builtin.os.tag != .linux or builtin.cpu.arch != .x86_64) return error.Unavailable; + + const r = try bazel.runfiles(bazel_builtin.current_repository); + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox = try r.rlocation("libzml_furiosa/sandbox", &path_buf) orelse { + log.err("Missing Furiosa sandbox runfile", .{}); + return error.FileNotFound; + }; + + const root = try std.Io.Dir.cwd().realPathFileAlloc(io, sandbox, allocator); + defer allocator.free(root); + + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + try setEnv("XLA_FURIOSA_SDK_ROOT", try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{root}), 1); + try setEnv("XLA_FURIOSA_VISIBLE_DEVICES", "0", 0); + + const library = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ root, "lib", "libzml_furiosa.so" }); + log.info("Loading Furiosa plugin: {s}", .{library}); + return .loadFrom(library); +} diff --git a/platforms/furiosa/libzml_furiosa.BUILD.bazel b/platforms/furiosa/libzml_furiosa.BUILD.bazel new file mode 100644 index 00000000..3a2da19a --- /dev/null +++ b/platforms/furiosa/libzml_furiosa.BUILD.bazel @@ -0,0 +1,23 @@ +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +# XLA owns the full payload. ZML only places it in the platform's runfiles. +copy_to_directory( + name = "sandbox", + srcs = glob( + [ + "bin/**", + "include/**", + "lib/**", + ], + allow_empty = False, + ), + add_directory_to_runfiles = True, + visibility = ["//visibility:public"], +) + +cc_library( + name = "libzml_furiosa", + data = [":sandbox"], + visibility = ["//visibility:public"], +) diff --git a/platforms/platforms.zig b/platforms/platforms.zig index 3f0c56fd..6239ba77 100644 --- a/platforms/platforms.zig +++ b/platforms/platforms.zig @@ -2,6 +2,7 @@ const std = @import("std"); const cpu = @import("platforms/cpu"); const cuda = @import("platforms/cuda"); +const furiosa = @import("platforms/furiosa"); const metal = @import("platforms/metal"); const neuron = @import("platforms/neuron"); const oneapi = @import("platforms/oneapi"); @@ -20,6 +21,7 @@ pub const Platform = enum { neuron, oneapi, metal, + furiosa, pub fn selectFirstAcceleratorEnabled() error{Unavailable}!Platform { inline for (@typeInfo(Platform).@"enum".fields) |field| { diff --git a/zml/Sharding.zig b/zml/Sharding.zig index 761721fa..5ce487ee 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -168,7 +168,7 @@ pub const Partitioner = union(enum) { pub fn fromTarget(target: Target) Partitioner { return switch (target) { - .cpu, .cuda, .rocm, .tpu, .oneapi, .neuron, .metal => .shardy, + .cpu, .cuda, .rocm, .tpu, .oneapi, .neuron, .metal, .furiosa => .shardy, }; } }; @@ -894,7 +894,7 @@ pub const PhysicalMesh = struct { .tpu => &.{ .link_x, .link_y, .link_z }, .neuron => &.{ .link, .link_x, .link_y, .link_z }, .cuda, .rocm => &.{.link}, - .cpu, .metal => &.{.bus}, + .cpu, .metal, .furiosa => &.{.bus}, .oneapi => &.{ .link, .bus }, }; } @@ -1100,7 +1100,7 @@ pub const PhysicalMesh = struct { .cuda, .rocm => gpu(allocator, platform_devices), .tpu => tpu(allocator, platform_devices), .neuron => return neuron(allocator, platform_devices), - .oneapi, .metal => cpu(allocator, platform_devices), + .oneapi, .metal, .furiosa => cpu(allocator, platform_devices), }; errdefer freeNode(allocator, root); diff --git a/zml/attention.zig b/zml/attention.zig index f5399c5e..c5b51603 100644 --- a/zml/attention.zig +++ b/zml/attention.zig @@ -32,7 +32,7 @@ pub const Backend = enum { }, .neuron => .nki, .metal => .metal_fa, - .cpu, .rocm, .tpu, .oneapi => .vanilla, + .cpu, .rocm, .tpu, .oneapi, .furiosa => .vanilla, }; } diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig index 7f83a10b..5e752a81 100644 --- a/zml/attention/paged_attention.zig +++ b/zml/attention/paged_attention.zig @@ -34,7 +34,7 @@ pub const Backend = enum { .oneapi => .triton, .tpu => .mosaic_tpu, .metal => .metal, - .cpu => .stablehlo, + .cpu, .furiosa => .stablehlo, .neuron => stdx.debug.panic("Paged attention is not supported on {s} yet", .{@tagName(platform.target)}), }; } diff --git a/zml/exe.zig b/zml/exe.zig index 1db04837..2acc0b61 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -347,7 +347,7 @@ pub const Exe = struct { const partition_events = events[0..@intCast(self.num_partitions)]; const events_slice: ?[]?*pjrt.Event = switch (self.platform.target) { .neuron => partition_events, - .cpu, .cuda, .rocm, .tpu, .oneapi, .metal => if (opts.wait) partition_events else null, + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => if (opts.wait) partition_events else null, }; self.exe.execute(self.platform.pjrt_api, .{ @@ -375,7 +375,7 @@ pub const Exe = struct { } } }, - .cpu, .cuda, .rocm, .tpu, .oneapi, .metal => if (opts.wait) { + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => if (opts.wait) { for (events_slice.?) |e| { if (e) |ev| { ev.await(self.platform.pjrt_api, io) catch |err| { diff --git a/zml/io.zig b/zml/io.zig index 5fce0793..b94f1ff6 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -689,7 +689,7 @@ pub const MemoryWriter = union(enum) { .cuda, .rocm, .oneapi => .{ .direct = try .init(allocator, io, platform, pools, dma_allocators, dma_chunk_size, shape, buffer, memory), }, - .tpu, .neuron, .cpu, .metal => if (memory == .host_pinned) + .tpu, .neuron, .cpu, .metal, .furiosa => if (memory == .host_pinned) std.debug.panic("Host pinned memory is not supported on {}", .{platform.target}) else .{ .buffered = try .init(allocator, io, platform, shape, buffer) }, diff --git a/zml/mem.zig b/zml/mem.zig index f471a83a..4dbce05f 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -23,7 +23,7 @@ pub const DmaAllocator = union(enum) { return switch (device.platform.target) { .cuda => .{ .dmam = .init(parent, device.platform) }, .rocm, .oneapi, .tpu => .{ .uib = .init(device.memory(.host_pinned).?) }, - .cpu, .neuron, .metal => .{ .passthrough = parent }, + .cpu, .neuron, .metal, .furiosa => .{ .passthrough = parent }, }; } diff --git a/zml/ops.zig b/zml/ops.zig index 20853e95..fbfc67b8 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1150,7 +1150,7 @@ pub fn neuronNki(inputs: anytype, outputs: anytype, opts: NeuronNkiOps) [outputs const ctx = Compiler.current(); switch (ctx.platform.target) { .neuron => {}, - .cpu, .cuda, .rocm, .tpu, .oneapi, .metal => { + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => { stdx.debug.panic("neuronNki is only available on Neuron, got {s}", .{@tagName(ctx.platform.target)}); }, } @@ -2166,7 +2166,7 @@ pub const LoweringCompatibility = struct { pub fn preserveIntegerScalarBroadcast(self: Tensor, output_shape: Shape) ?Tensor { switch (Compiler.current().platform.target) { .neuron => {}, - .cpu, .cuda, .rocm, .tpu, .oneapi, .metal => return null, + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => return null, } if (self.rank() != 0 or output_shape.rank() == 0 or !self.dtype().isInteger()) return null; @@ -2185,7 +2185,7 @@ pub const LoweringCompatibility = struct { pub fn preserveGatherFillSemantics(indices: []Tensor) void { switch (Compiler.current().platform.target) { .neuron => {}, - .cpu, .cuda, .rocm, .tpu, .oneapi, .metal => return, + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => return, } const active_lanes = activeLanesForFillDropIndices(indices) orelse return; @@ -2197,7 +2197,7 @@ pub const LoweringCompatibility = struct { pub fn preserveScatterDropSemantics(indices: []Tensor, updates: anytype, opts: Tensor.ScatterOpts, update_values: *[updates.len]*const mlir.Value) void { const active_lanes: ?Tensor = switch (Compiler.current().platform.target) { .neuron => if (opts.update_fn == Tensor.ScatterOpts.increment) activeLanesForFillDropIndices(indices) else null, - .cpu, .cuda, .rocm, .tpu, .oneapi, .metal => null, + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => null, }; if (active_lanes) |active| replaceInactiveIndirectIndices(indices, active); diff --git a/zml/platform.zig b/zml/platform.zig index 7a9ee710..66f7df01 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -39,7 +39,7 @@ fn validateDeviceCount(target: Target, num_devices: usize) !void { return error.MissingDevices; } switch (target) { - .cpu, .cuda, .rocm, .tpu, .neuron, .metal, .oneapi => { + .cpu, .cuda, .rocm, .tpu, .neuron, .metal, .oneapi, .furiosa => { if (!std.math.isPowerOfTwo(num_devices)) { log.err("Platform {} requires a power-of-two device count, got {}", .{ target, num_devices }); return error.InvalidDeviceCount; @@ -88,7 +88,7 @@ pub const Memory = struct { pub fn isOfKind(self: Memory, kind_: Kind) bool { switch (self.platform.target) { - .cuda, .rocm, .oneapi, .tpu => { + .cuda, .rocm, .oneapi, .tpu, .furiosa => { const zml_kind: Memory.Kind = switch (self.kind().len) { "device".len => .device, "pinned_host".len => .host_pinned, @@ -224,7 +224,7 @@ pub const Device = struct { fn platformDeviceSortId(target: Target, device: Device) usize { return switch (target) { .neuron => @intCast(device.localHardwareId()), - .cuda, .rocm, .tpu, .cpu, .oneapi, .metal => device.id(), + .cuda, .rocm, .tpu, .cpu, .oneapi, .metal, .furiosa => device.id(), }; } @@ -255,6 +255,7 @@ pub const State = union(Target) { neuron: void, oneapi: void, metal: void, + furiosa: void, pub const CudaState = struct { fi_cutlass_moe_runners: ?*zml.moe.cutlass_flashinfer.Runners = null, @@ -276,6 +277,7 @@ pub const State = union(Target) { .neuron => .{ .neuron = {} }, .oneapi => .{ .oneapi = {} }, .metal => .{ .metal = {} }, + .furiosa => .{ .furiosa = {} }, }; } @@ -421,6 +423,7 @@ pub const Platform = struct { .cuda, .oneapi, .metal, + .furiosa, .cpu, }; return for (ordered_targets) |target| { @@ -690,10 +693,10 @@ pub const Platform = struct { // inline cause `default` is a huge ass struct allocated on the stack, // and toMemoryLayout returns slices into it. // There is probably a better way of doing this, - // but given it's compiled out (except for TPU), I'm not gonna care for now. + // The queried layout can include backend-specific tiling. return switch (platform.target) { - .tpu => { - if (comptime !Target.tpu.isEnabled()) unreachable; + .tpu, .furiosa => { + if (comptime !Target.tpu.isEnabled() and !Target.furiosa.isEnabled()) unreachable; const element_type = pjrtx.bufferTypeFromDtype(dtype); const default = platform.pjrt_client.defaultMemoryLayout(platform.pjrt_api, element_type, dims) catch @panic("Failed to get default memory layout"); return default.toMemoryLayout(); @@ -732,10 +735,19 @@ pub const CreateOptions = struct { cuda: XlaGpu = .{ .allocator = .{ .vmm = .{ .memory_fraction = 0.90 } } }, tpu: struct {} = .{}, neuron: struct {} = .{}, + furiosa: Furiosa = .{}, oneapi: XlaGpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.90 } } }, metal: XlaGpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.90 } } }, io_impl: Platform.IoImpl = .threaded, + pub const Furiosa = struct { + pe_count: u8 = 8, + + fn writeNamedValues(self: Furiosa, values: *std.ArrayList(pjrt.NamedValue)) void { + values.appendAssumeCapacity(.init(.int64, "pe_count", self.pe_count)); + } + }; + pub const Cpu = struct { device_count: u32, @@ -815,6 +827,7 @@ pub const CreateOptions = struct { values.shrinkRetainingCapacity(0); switch (target) { .cpu => self.cpu.writeNamedValues(&values), + .furiosa => self.furiosa.writeNamedValues(&values), .cuda => self.cuda.writeNamedValues(target, &values), .rocm => self.rocm.writeNamedValues(target, &values), .oneapi => self.oneapi.writeNamedValues(target, &values), @@ -1039,6 +1052,11 @@ test "platform defaultMemoryLayout is boring" { const default_layout = try platform.pjrt_client.defaultMemoryLayout(platform.pjrt_api, pjrtx.bufferTypeFromDtype(.f32), dims); const mem_layout = default_layout.toMemoryLayout(); + if (platform.target == .furiosa) { + try std.testing.expectEqualDeep(mem_layout, platform.defaultMemoryLayout(dims, .f32)); + continue; + } + // Note: I'm not just calling platform.defaultMemoryLayout because I'm investigating // wether TPU requires its special branch. try std.testing.expectEqualDeep(mem_layout, pjrt.MemoryLayout{ diff --git a/zml/tensor.zig b/zml/tensor.zig index 96b60c0c..c2b63f53 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -275,7 +275,7 @@ pub const Tensor = struct { const ctx = Compiler.current(); switch (ctx.platform.target) { .cpu, .neuron, .metal => return self, - .cuda, .rocm, .tpu, .oneapi => {}, + .cuda, .rocm, .tpu, .oneapi, .furiosa => {}, } const frontend_attributes: *const mlir.Attribute = .dict(ctx.mlir_ctx, &.{ @@ -341,7 +341,7 @@ pub const Tensor = struct { const ctx = Compiler.current(); switch (ctx.platform.target) { .cpu, .neuron, .metal => return flat_tensors, - .cuda, .rocm, .tpu, .oneapi => {}, + .cuda, .rocm, .tpu, .oneapi, .furiosa => {}, } var copy = flat_tensors; @@ -359,7 +359,7 @@ pub const Tensor = struct { const ctx = Compiler.current(); switch (ctx.platform.target) { .cpu, .neuron, .metal => return self, - .cuda, .rocm, .tpu, .oneapi => {}, + .cuda, .rocm, .tpu, .oneapi, .furiosa => {}, } if (ctx.currentScope().id_to_argument.get(self.id) == null) { @@ -378,7 +378,7 @@ pub const Tensor = struct { switch (ctx.platform.target) { // Only one memory kind on those platform .cpu, .neuron, .metal => return, - .cuda, .rocm, .tpu, .oneapi => {}, + .cuda, .rocm, .tpu, .oneapi, .furiosa => {}, } meta.visit(struct { @@ -3854,7 +3854,7 @@ pub const Tensor = struct { } break :blk .{ .values = values, .indices = indices }; }, - .cpu, .cuda, .rocm, .tpu, .oneapi, .metal => blk: { + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => blk: { var sorted = self.sort(a, .{ .descending = opts.descending }); sorted.values = sorted.values.slice(a, .{ .end = k }); sorted.indices = sorted.indices.slice(a, .{ .end = k }); @@ -4811,7 +4811,7 @@ pub const Tensor = struct { } }).body, .{ .input = input, .name = full_name }, {}); }, - .oneapi, .neuron => {}, + .oneapi, .neuron, .furiosa => {}, } } From 8173a5b7f665d8c525e9117dbb7d6d82ef4d0d63 Mon Sep 17 00:00:00 2001 From: Louis Lechevalier Date: Fri, 9 Oct 2026 17:19:04 +0200 Subject: [PATCH 771/881] platforms/cuda: Bump zml CUDA plugin (#918) --- platforms/cuda/cuda.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 451dd3c3..24dd691e 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -224,12 +224,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "128fd48dba2db29f3ffe1b8d57282cd171f0e2b95c9739260da068b703cd9a59", - "url": "https://mirror.zml.ai/plugins/202610081055.168.1.56e99704ada4/zml-cuda-linux-amd64.tar.zst", + "sha256": "82968f83dc8af1b8942e28a15e0d4bd0d5d3c1ecf512fb86f9f3d883288ae064", + "url": "https://mirror.zml.ai/plugins/202610091329.178.1.75e89af70664/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "5c86f09731ec75a7779be53b195ae7e8e0bbf7b4a20c7b71cccdcf47fe83378b", - "url": "https://mirror.zml.ai/plugins/202610081055.168.1.56e99704ada4/zml-cuda-linux-arm64.tar.zst", + "sha256": "3605e422a0659c2cef9a59d35ee2a962caa78ef591160cfd780110c49002db5a", + "url": "https://mirror.zml.ai/plugins/202610091329.178.1.75e89af70664/zml-cuda-linux-arm64.tar.zst", }, } From c0992073fd5a2209a2d22c49f8652b613d32bde1 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 9 Oct 2026 17:36:28 +0200 Subject: [PATCH 772/881] workspace: bump other plugins (#922) --- platforms/cpu/cpu.bzl | 12 ++++++------ platforms/furiosa/furiosa.bzl | 4 ++-- platforms/metal/metal.bzl | 4 ++-- platforms/rocm/rocm.bzl | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 7fc024dc..583fcad7 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -42,22 +42,22 @@ def _cpu_plugin_impl(mctx): http_archive( name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "5051c6719809f1dd86a83665b6bb65a4c0c8ec3be4613c8652eecf4856ab1d27", - url = "https://mirror.zml.ai/plugins/202610081055.168.1.56e99704ada4/zml-cpu-linux-amd64.tar.zst", + sha256 = "6397644e129e198431e56d8fee6871d5d9a71550dc6ca03a5abbf0a9ef3a3eaf", + url = "https://mirror.zml.ai/plugins/202610091329.178.1.75e89af70664/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "d2ea2bce16aba06cc3f75725b1cd8c2fc72169a8ec8cb72c21637161dc79a1c0", - url = "https://mirror.zml.ai/plugins/202610081055.168.1.56e99704ada4/zml-cpu-darwin-amd64.tar.zst", + sha256 = "8276a919970b9d5c2edf4cf6f962c689fa67c45d325f70849bfdd38403f8fb8b", + url = "https://mirror.zml.ai/plugins/202610091329.178.1.75e89af70664/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "7e614948453f9692ca76d7014ffae9f25d9fe80836088ca92a5f764b26a6fe43", - url = "https://mirror.zml.ai/plugins/202610081055.168.1.56e99704ada4/zml-cpu-darwin-arm64.tar.zst", + sha256 = "e08eb1986a8705c66f065d15c0cacf228219118aca7c44148e47502f011982b1", + url = "https://mirror.zml.ai/plugins/202610091329.178.1.75e89af70664/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index 8a2cb33a..d0b901ba 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202610081500.175.1.4a98481373f6/zml-furiosa-linux-amd64.tar.zst", - sha256 = "fb5569b2c3ab8604924fb66b2e9f6fd3fe38de32f58aaec25ef1aa510dfe10ee", + url = "https://mirror.zml.ai/plugins/202610091329.178.1.75e89af70664/zml-furiosa-linux-amd64.tar.zst", + sha256 = "dfc5bd915e3075712d1ec821379b2958a4c3767ba827b59e6b76d32571970f88", ) return mctx.extension_metadata( diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index b7a0bf3f..b0f9d22d 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "bde54cd2d67d28c7ed91fa953c3b13074d40e7d73c1157e4ad4a84aa90394988", - url = "https://mirror.zml.ai/plugins/202610081055.168.1.56e99704ada4/zml-metal-darwin-arm64.tar.zst", + sha256 = "a1262b90e5c746097447b24c45ee69f00d2923c53f5491d73ff06c3c52377fee", + url = "https://mirror.zml.ai/plugins/202610091329.178.1.75e89af70664/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 69bcf2d4..ec754177 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610081055.168.1.56e99704ada4/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "e8d24524364816e53b3deb55ca138a152b5c6c006a853bf410e930c681c2b087" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610091329.178.1.75e89af70664/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "980c0d86a31b8ee58e8efc7b2464752cc3982c040f12a3c3582ba1b9289e2bce" def _rocm_package_name(name): return name + _ROCM_VERSION From 3500e88cf84e0e98d37b0f6048e6af9d33bc6108 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Fri, 9 Oct 2026 17:49:32 +0200 Subject: [PATCH 773/881] zml/moe: fit triton kernel launch config with LDS limit in mind (#920) Temporary until we refactor zml.kernels --- zml/moe/fused_experts.zig | 2 +- zml/moe/tests.zig | 21 ++++++++++++++++++++- zml/moe/triton.zig | 18 +++++++++++++++++- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/zml/moe/fused_experts.zig b/zml/moe/fused_experts.zig index 88f24b3b..e7c23dfb 100644 --- a/zml/moe/fused_experts.zig +++ b/zml/moe/fused_experts.zig @@ -499,7 +499,7 @@ pub const Routing = struct { // Config / validation helpers // ============================================================================= -const LaunchConfig = struct { +pub const LaunchConfig = struct { block_size_m: usize, block_size_n: usize, block_size_k: usize, diff --git a/zml/moe/tests.zig b/zml/moe/tests.zig index 43b3e1ac..1b654fa3 100644 --- a/zml/moe/tests.zig +++ b/zml/moe/tests.zig @@ -1,11 +1,12 @@ //! Shared public-entry-point tests. Hardware absence is the only runtime skip: //! supported cases are explicit, and errors in those cases must fail the test. const std = @import("std"); + const zml = @import("../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; -const reference = @import("stablehlo.zig"); const Placement = @import("fused_experts.zig").RoutingWeightPlacement; +const reference = @import("stablehlo.zig"); const Case = struct { backend: zml.moe.Backend, @@ -521,6 +522,24 @@ test "Triton compatibility matrix" { }, .{}); } +test "Triton MoE prefill launch buckets fit shared memory" { + const platform = zml.testing.env(); + if (!zml.moe.Backend.triton.isAvailable(platform) or platform.target != .rocm) return error.SkipZigTest; + + for ([_]i64{ 8, 32, 96, 128, 256, 512, 1024, 2048, 3072, 4096 }) |tokens| { + const case: Case = .{ + .backend = .triton, + .tokens = tokens, + .width = 256, + .intermediate = 128, + .topk = 2, + }; + var exe = try case.compile(std.testing.allocator, std.testing.io, platform); + defer exe.deinit(); + try case.check(std.testing.allocator, std.testing.io, platform, &exe); + } +} + test "FlashInfer compatibility matrix" { const platform = zml.testing.env(); diff --git a/zml/moe/triton.zig b/zml/moe/triton.zig index 2394e1e5..cba8e113 100644 --- a/zml/moe/triton.zig +++ b/zml/moe/triton.zig @@ -16,6 +16,22 @@ const kernels = @import("triton_kernels/triton_kernels.zig"); const log = std.log.scoped(.@"zml/moe/triton"); +fn numStages(config: shared.LaunchConfig) i32 { + const platform = zml.Compiler.current().platform; + if (platform.target != .rocm) return config.num_stages; + + // ROCm buffers num_stages - 1 BF16 operand pairs in LDS. A single stage + // reuses scratch space between operand layout conversions. + const bytes_per_stage: i64 = @intCast(2 * config.block_size_k * (config.block_size_m + config.block_size_n)); + var stages = config.num_stages; + for (platform.devices) |device| { + const attribute = device.pjrt_desc.attribute(platform.pjrt_api, "shared_memory_per_block_optin"); + const limit = if (attribute) |value| value.int64 else 64 * 1024; + stages = @intCast(@min(stages, @divTrunc(limit, bytes_per_stage) + 1)); + } + return stages; +} + pub fn call(opts: shared.GemmOptions) Tensor { // Native FP4 tensors expose logical K; the Triton operand is byte-packed. const weight = if (opts.quant_scheme == .mxfp4 and opts.weight.dtype() == .f4e2m1) @@ -123,7 +139,7 @@ pub fn call(opts: shared.GemmOptions) Tensor { }, .grid = .{ @intCast(grid_x), 1, 1 }, .num_warps = opts.launch_config.num_warps, - .num_stages = opts.launch_config.num_stages, + .num_stages = numStages(opts.launch_config), }, ).c; } From ecea83bda7d0c1c8986caeb5bf5863f81a13f406 Mon Sep 17 00:00:00 2001 From: fior512 <161968556+fior512@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:08:52 +0200 Subject: [PATCH 774/881] examples/llm: Qwen 3.5 Dense and MoE memory leak (#919) Fix a memory leak in Qwen 3.5 Dense and MoE examples: `layer_index` was never freed. Before (74562d5) / After Dense 0.8B VRAM : 11079 MB vs 5461 MB Dense 0.8B tok/s : 169 vs 198 tok/s increases because it reduces `cuMemCreate` calls from 26 to 2 per tokens --- examples/llm/models/qwen3_5/inference.zig | 2 ++ examples/llm/models/qwen3_5_moe/inference.zig | 2 ++ 2 files changed, 4 insertions(+) diff --git a/examples/llm/models/qwen3_5/inference.zig b/examples/llm/models/qwen3_5/inference.zig index 6bfb71c5..6a52f064 100644 --- a/examples/llm/models/qwen3_5/inference.zig +++ b/examples/llm/models/qwen3_5/inference.zig @@ -179,6 +179,7 @@ pub fn run(runner: *KernelRunner, args: Args, layer_index_buffers: []const zml.B }, .{}); args.kv_cache_buffers.self_attn.k = layer_cache.k; args.kv_cache_buffers.self_attn.v = layer_cache.v; + layer_cache.layer_index.deinit(); }, .linear_attention => |*layer| { var layer_cache: zml.Bufferized(model.KvCache.GatedDeltaNetCache) = .{ @@ -196,6 +197,7 @@ pub fn run(runner: *KernelRunner, args: Args, layer_index_buffers: []const zml.B }, .{}); args.kv_cache_buffers.gated_delta_net.conv_state = layer_cache.conv_state; args.kv_cache_buffers.gated_delta_net.recurrent_state = layer_cache.recurrent_state; + layer_cache.layer_index.deinit(); }, } } diff --git a/examples/llm/models/qwen3_5_moe/inference.zig b/examples/llm/models/qwen3_5_moe/inference.zig index f29ecabd..ac0777d4 100644 --- a/examples/llm/models/qwen3_5_moe/inference.zig +++ b/examples/llm/models/qwen3_5_moe/inference.zig @@ -186,6 +186,7 @@ pub fn run(runner: *KernelRunner, args: RunArgs) void { }, .{}); args.kv_cache_buffers.self_attn.k = layer_cache.k; args.kv_cache_buffers.self_attn.v = layer_cache.v; + layer_cache.layer_index.deinit(); }, .linear_attention => |*layer| { const index_buffer = switch (layer_index_buffer) { @@ -207,6 +208,7 @@ pub fn run(runner: *KernelRunner, args: RunArgs) void { }, .{}); args.kv_cache_buffers.gated_delta_net.conv_state = layer_cache.conv_state; args.kv_cache_buffers.gated_delta_net.recurrent_state = layer_cache.recurrent_state; + layer_cache.layer_index.deinit(); }, } } From 3a201a67faf67717792d941832e7724f93caced8 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 12 Oct 2026 17:55:27 +0200 Subject: [PATCH 775/881] workspace: fix typo in nvim config (#927) `mod` was not defined resulting in bazel file formatting being broken. --- .nvim.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nvim.lua b/.nvim.lua index 65995b81..9982de80 100644 --- a/.nvim.lua +++ b/.nvim.lua @@ -58,7 +58,7 @@ else pattern = "bzl", callback = function() vim.keymap.set("n", "f", function() - mod.format() + conform.format() end, { remap = false, silent = true }) end, }) From bd0886c3e5ae0dd9a10ca79174cacd6297887c32 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 12 Oct 2026 20:31:17 +0200 Subject: [PATCH 776/881] zml: re-structure attention kernels (#926) This PR introduces a restructuration of the kernels in ZML. This is done to simplify and speedup bringup on new platforms. From now on each "kernel" will have its own directory under `zml/kernels`. "Kernel" is in quote because we need to think about it being more of an "operation in a graph" than an actual kernel. For example, `moe` will eventually be part of this `kernels` directory despite being an operation potentially launching multiple actual kernels on some hardware. Also it's probable that in the future, this `kernels` directory will be put outside of zml or, at least, properly separated so that we don't have circular imports in the codebase. Currently, we have `@import("../../../zml.zig")` in the kernel code and this is a recipe for disaster. On top of the reorganization, it also adds a level of indirection for attention. Where previously `attention` was a generic term, we now have `attention` and `flash_attention` because `flash_attention` is an implementation of `attention`. Here is a detailed summary of the changes introduced by this PR: * Kernels reorganization. `flash_attention` and `paged_attention` are now separated. Each folder has a subfolder for each backend we support. `flash_attention` is also now a `BackendType` of `zml.attention` with its own implementations. * Removed `cuda_fa2` and `cuda_fa3` backend because they were obsolete. They have been replaced by a `cute` backend that has been generated automatically following the operation documentation, showing that this approach of scoping kernels per DSL actually scales. * Removed `attnd` support for now --- examples/llm/README.md | 2 +- examples/llm/main.zig | 18 +- examples/llm/models/lfm2/inference.zig | 2 +- examples/llm/models/lfm2/model.zig | 2 +- examples/llm/models/lfm2/session.zig | 7 +- examples/llm/models/lfm2_tests.zig | 9 +- examples/llm/models/llama/inference.zig | 27 +- examples/llm/models/llama/model.zig | 15 +- examples/llm/models/llama/session.zig | 19 +- examples/llm/models/qwen3_5/model.zig | 4 +- examples/llm/models/qwen3_5_moe/model.zig | 4 +- platforms/neuron/libzml_neuron.BUILD.bazel | 2 +- zml/BUILD.bazel | 30 +- zml/attention.zig | 355 ---- zml/attention/attention.zig | 122 ++ zml/attention/attnd.zig | 500 ----- zml/attention/cute_kernels/flashmla_sm100.zig | 1355 ------------- zml/attention/cute_kernels/sparse_mla.zig | 266 --- zml/attention/flashattn.zig | 1792 ----------------- zml/attention/fly_kernels/sparse_mla.zig | 458 ----- zml/attention/metal_attention.zig | 113 -- zml/attention/paged_attention.zig | 1232 ------------ zml/attention/sparse_mla.zig | 195 -- .../triton_kernels/unified_sparse_mla.zig | 567 ------ zml/kernels/flash_attention/README.md | 287 +++ zml/kernels/flash_attention/cute/BUILD.bazel | 9 + zml/kernels/flash_attention/cute/README.md | 145 ++ .../flash_attention/cute/attention.zig | 300 +++ zml/kernels/flash_attention/cute/cute.zig | 108 + .../flash_attention/flash_attention.zig | 364 ++++ zml/kernels/flash_attention/metal/BUILD.bazel | 6 + zml/kernels/flash_attention/metal/metal.zig | 38 + .../flash_attention}/nki/BUILD.bazel | 4 +- .../flash_attention}/nki/attention.py | 0 .../flash_attention}/nki/attention_harness.py | 0 .../flash_attention/nki/nki.zig} | 2 +- zml/kernels/paged_attention/metal/BUILD.bazel | 6 + zml/kernels/paged_attention/metal/metal.zig | 81 + .../paged_attention/mosaic_tpu/BUILD.bazel | 9 + .../mosaic_tpu/mosaic_tpu.zig} | 9 +- .../mosaic_tpu}/ragged_attention.zig | 2 +- .../paged_attention/paged_attention.zig | 502 +++++ .../paged_attention/stablehlo/BUILD.bazel | 6 + .../paged_attention/stablehlo/stablehlo.zig | 253 +++ .../paged_attention/triton/BUILD.bazel | 10 + .../paged_attention/triton/triton.zig} | 283 +-- .../triton}/unified_attention.zig | 2 +- .../triton}/unified_attention_oneapi.zig | 2 +- zml/platform.zig | 7 - zml/zml.zig | 3 +- 50 files changed, 2310 insertions(+), 7224 deletions(-) delete mode 100644 zml/attention.zig create mode 100644 zml/attention/attention.zig delete mode 100644 zml/attention/attnd.zig delete mode 100644 zml/attention/cute_kernels/flashmla_sm100.zig delete mode 100644 zml/attention/cute_kernels/sparse_mla.zig delete mode 100644 zml/attention/flashattn.zig delete mode 100644 zml/attention/fly_kernels/sparse_mla.zig delete mode 100644 zml/attention/metal_attention.zig delete mode 100644 zml/attention/paged_attention.zig delete mode 100644 zml/attention/sparse_mla.zig delete mode 100644 zml/attention/triton_kernels/unified_sparse_mla.zig create mode 100644 zml/kernels/flash_attention/README.md create mode 100644 zml/kernels/flash_attention/cute/BUILD.bazel create mode 100644 zml/kernels/flash_attention/cute/README.md create mode 100644 zml/kernels/flash_attention/cute/attention.zig create mode 100644 zml/kernels/flash_attention/cute/cute.zig create mode 100644 zml/kernels/flash_attention/flash_attention.zig create mode 100644 zml/kernels/flash_attention/metal/BUILD.bazel create mode 100644 zml/kernels/flash_attention/metal/metal.zig rename zml/{attention => kernels/flash_attention}/nki/BUILD.bazel (92%) rename zml/{attention => kernels/flash_attention}/nki/attention.py (100%) rename zml/{attention => kernels/flash_attention}/nki/attention_harness.py (100%) rename zml/{attention/nki/attention.zig => kernels/flash_attention/nki/nki.zig} (99%) create mode 100644 zml/kernels/paged_attention/metal/BUILD.bazel create mode 100644 zml/kernels/paged_attention/metal/metal.zig create mode 100644 zml/kernels/paged_attention/mosaic_tpu/BUILD.bazel rename zml/{attention/tpu_attention.zig => kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig} (97%) rename zml/{attention/mosaic_tpu_kernels => kernels/paged_attention/mosaic_tpu}/ragged_attention.zig (89%) create mode 100644 zml/kernels/paged_attention/paged_attention.zig create mode 100644 zml/kernels/paged_attention/stablehlo/BUILD.bazel create mode 100644 zml/kernels/paged_attention/stablehlo/stablehlo.zig create mode 100644 zml/kernels/paged_attention/triton/BUILD.bazel rename zml/{attention/triton_attention.zig => kernels/paged_attention/triton/triton.zig} (68%) rename zml/{attention/triton_kernels => kernels/paged_attention/triton}/unified_attention.zig (99%) rename zml/{attention/triton_kernels => kernels/paged_attention/triton}/unified_attention_oneapi.zig (99%) diff --git a/examples/llm/README.md b/examples/llm/README.md index 1b581514..fcd061ba 100644 --- a/examples/llm/README.md +++ b/examples/llm/README.md @@ -38,4 +38,4 @@ bazel run //examples/llm --@zml//platforms:cuda=true -- --model=hf://meta-llama/ - `--model=`: Required. Model repository to load. This can be a local path or a huggingface/S3 URI such as `hf://...` or `s3://...`. - `--prompt=`: Optional. Runs a single prompt instead of opening the interactive chat loop. - `--seqlen=`: Optional. Maximum sequence length. Defaults to `2048`. -- `--backend=`: Optional. Attention backend. If omitted, the program auto-selects one for the current platform. +- `--backend=`: Optional. Attention backend. `cute` uses CuTe flash attention on CUDA GPUs with compute capability 8.0 or newer. If omitted, the program auto-selects one for the current platform, including `cute` on supported CUDA GPUs. diff --git a/examples/llm/main.zig b/examples/llm/main.zig index e10db570..b24c022a 100644 --- a/examples/llm/main.zig +++ b/examples/llm/main.zig @@ -18,8 +18,7 @@ const Args = struct { prompt: ?[]const u8 = null, seqlen: u32 = 2048, topk: u32 = 4, - backend: ?zml.attention.Backend = null, - attnd_ip: ?[]const u8 = null, + backend: ?enum { stablehlo, cute, nki, metal } = null, profile: bool = false, pub const help = @@ -32,8 +31,7 @@ const Args = struct { \\ --prompt= Prompt to use for generation (default: none) \\ --seqlen= Sequence length (default: 2048) \\ --topk= Top-k sampling cutoff (default: 4) - \\ --backend= Attention backend to use ([vanilla, attnd, nki, cuda_fa2, cuda_fa3], default: auto-selection) - \\ --attnd-ip= Register and prefer the `attnd` backend at the provided `IP:PORT` + \\ --backend= Attention backend to use ([stablehlo, cute, nki, metal], default: auto-selection) \\ --profile Capture a PJRT profile for non-interactive runs and write a Perfetto trace \\ ; @@ -88,14 +86,12 @@ pub fn main(init: std.process.Init) !void { log.info("\n{f}", .{platform.fmtVerbose()}); - const backend = args.backend orelse if (args.attnd_ip) |attnd_ip| b: { - try zml.attention.attnd.register(allocator, io, platform, .{ - .destination = try .parseLiteral(attnd_ip), - .mtu = 9000, - }); - break :b zml.attention.Backend.attnd; + const backend: zml.attention.Backend = if (args.backend) |selected| switch (selected) { + .stablehlo => .{ .stablehlo = {} }, + .cute => .{ .flash_attention = .cute }, + .nki => .{ .flash_attention = .nki }, + .metal => .{ .flash_attention = .metal }, } else zml.attention.Backend.auto(platform); - defer if (args.attnd_ip) |_| zml.attention.attnd.deinit(); log.info("Selected backend: {}", .{backend}); // diff --git a/examples/llm/models/lfm2/inference.zig b/examples/llm/models/lfm2/inference.zig index 92ab4462..4726b1da 100644 --- a/examples/llm/models/lfm2/inference.zig +++ b/examples/llm/models/lfm2/inference.zig @@ -42,7 +42,7 @@ pub const CompilationParameters = struct { .batch_dim = 1, .rng = .init(), .cache = cache, - .attention_metadata = .init(.fromBackend(backend, seqlen, config.num_attention_heads), meshes.model), + .attention_metadata = .init(.fromBackend(backend)), .attention_parameters = .init(.fromBackend(backend)), .seqlen = seqlen, .meshes = meshes, diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index 1d627025..009d167d 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -567,7 +567,7 @@ pub const Attention = struct { stdx.debug.assert(q.dim(.batch) == 1, "LFM attention currently expects batch size 1 for flash attention backend, got {}", .{q.dim(.batch)}); const attn = switch (attention_parameters) { - .vanilla => zml.attention.attention( + .stablehlo => zml.attention.attention( q, k, v, diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig index b59a0416..8951c528 100644 --- a/examples/llm/models/lfm2/session.zig +++ b/examples/llm/models/lfm2/session.zig @@ -133,8 +133,11 @@ pub const Session = struct { const params = self.compiled_model.params; var attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = switch (params.attention_metadata) { - .metal_fa => .{ .metal_fa = .{ .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32) } }, - else => try params.attention_metadata.initBuffer(self.io, self.platform), + .stablehlo => try params.attention_metadata.initBuffer(self.io, self.platform), + .flash_attention => |metadata| .{ .flash_attention = switch (metadata) { + .metal => .{ .metal = .{ .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32) } }, + else => try metadata.initBuffer(self.io, self.platform), + } }, }; defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig index 2da991ec..19b28df6 100644 --- a/examples/llm/models/lfm2_tests.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -13,7 +13,7 @@ pub const std_options: std.Options = .{ const Args = struct { model: []const u8, activations: []const u8, - backend: ?zml.attention.Backend = null, + backend: ?enum { stablehlo, cute, nki, metal } = null, pub const help = \\Use lfm2_tests --model= --activations= [options] @@ -52,7 +52,12 @@ pub fn main(init: std.process.Init) !void { var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress); defer repo_model.unloadBuffers(&model_buffers, allocator); - const backend = args.backend orelse zml.attention.Backend.auto(platform); + const backend: zml.attention.Backend = if (args.backend) |selected| switch (selected) { + .stablehlo => .{ .stablehlo = {} }, + .cute => .{ .flash_attention = .cute }, + .nki => .{ .flash_attention = .nki }, + .metal => .{ .flash_attention = .metal }, + } else zml.attention.Backend.auto(platform); const params = lfm2.CompilationParameters.init( repo_model.inner, repo_model.parsed_config.value, diff --git a/examples/llm/models/llama/inference.zig b/examples/llm/models/llama/inference.zig index 331ee0dc..4555e3d4 100644 --- a/examples/llm/models/llama/inference.zig +++ b/examples/llm/models/llama/inference.zig @@ -34,30 +34,9 @@ pub const CompilationParameters = struct { .hd = head_dim, }, mdl.model.embed_tokens.weight.dtype()), meshes.model), .rng = .init(), - .attention_metadata = switch (backend) { - .attnd => .{ .attnd = .init() }, - else => .init(.fromBackend(backend, @intCast(seqlen), @intCast(config.num_attention_heads)), meshes.model), - }, - .prefill_attention_parameters = switch (backend) { - .attnd => .{ .attnd = .init(.{ - .model_id = .@"llama-3.1-8B", - .head_dim = head_dim, - .num_attention_heads = config.num_attention_heads, - .num_kv_heads = @intCast(config.num_key_value_heads), - .is_prefill = true, - }) }, - else => .init(.fromBackend(backend)), - }, - .decode_attention_parameters = switch (backend) { - .attnd => .{ .attnd = .init(.{ - .model_id = .@"llama-3.1-8B", - .head_dim = head_dim, - .num_attention_heads = config.num_attention_heads, - .num_kv_heads = @intCast(config.num_key_value_heads), - .is_prefill = false, - }) }, - else => .init(.fromBackend(backend)), - }, + .attention_metadata = .init(.fromBackend(backend)), + .prefill_attention_parameters = .init(.fromBackend(backend)), + .decode_attention_parameters = .init(.fromBackend(backend)), .seqlen = seqlen, .meshes = meshes, }; diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index ae988b97..7d474593 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -579,25 +579,12 @@ const SelfAttn = struct { k = new_kv_cache.keysAt(kv_cache_index).convert(dtype); v = new_kv_cache.valuesAt(kv_cache_index).convert(dtype); - const layer_attention_metadata: zml.attention.Metadata = switch (attention_parameters) { - .attnd => .{ .attnd = .{ - .layer_id = kv_cache_index.convert(.u16), - .conversation_id = attention_metadata.attnd.conversation_id, - .num_tokens = attention_metadata.attnd.num_tokens, - } }, - .vanilla => attention_metadata, - .cuda_fa2 => attention_metadata, - .cuda_fa3 => attention_metadata, - .nki => attention_metadata, - .metal_fa => attention_metadata, - }; - const attn_output = zml.attention.attention( q, k, v, token_index, - layer_attention_metadata, + attention_metadata, attention_parameters, ); diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig index 6a04cf11..9b801118 100644 --- a/examples/llm/models/llama/session.zig +++ b/examples/llm/models/llama/session.zig @@ -142,13 +142,11 @@ pub const Session = struct { const params = self.compiled_model.params; var attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = switch (params.attention_metadata) { - .attnd => .{ .attnd = .{ - .conversation_id = try .scalar(self.io, self.platform, self.conversation_id, .u64), - .layer_id = try .scalar(self.io, self.platform, 0, .u16), - .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32), + .stablehlo => try params.attention_metadata.initBuffer(self.io, self.platform), + .flash_attention => |metadata| .{ .flash_attention = switch (metadata) { + .metal => .{ .metal = .{ .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32) } }, + else => try metadata.initBuffer(self.io, self.platform), } }, - .metal_fa => .{ .metal_fa = .{ .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32) } }, - .vanilla, .cuda_fa2, .cuda_fa3, .nki => try params.attention_metadata.initBuffer(self.io, self.platform), }; defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); @@ -177,14 +175,7 @@ pub const Session = struct { defer current_token_buffer.deinit(); const params = self.compiled_model.params; - var attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = switch (params.attention_metadata) { - .attnd => .{ .attnd = .{ - .conversation_id = try .scalar(self.io, self.platform, self.conversation_id, .u64), - .layer_id = try .scalar(self.io, self.platform, 0, .u16), - .num_tokens = try .scalar(self.io, self.platform, 1, .u32), - } }, - .vanilla, .cuda_fa2, .cuda_fa3, .nki, .metal_fa => try params.attention_metadata.initBuffer(self.io, self.platform), - }; + var attention_metadata_buffers = try params.attention_metadata.initBuffer(self.io, self.platform); defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); generation: while (true) { diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 14edd022..8d028cc5 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -656,8 +656,8 @@ pub const SelfAttn = struct { k, v, token_index, - zml.attention.Metadata.init(.fromBackend(.vanilla, x.dim(.s), self.num_heads), tp), - zml.attention.Parameters.init(.fromBackend(.vanilla)), + zml.attention.Metadata.init(.fromBackend(.stablehlo)), + zml.attention.Parameters.init(.fromBackend(.stablehlo)), ).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); const gated_output = attn_output.mul(gate.sigmoid()); diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 59d52815..fa60a26d 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -640,8 +640,8 @@ pub const SelfAttn = struct { k, v, token_index, - zml.attention.Metadata.init(.fromBackend(.vanilla, x.dim(.s), self.num_heads), tp), - zml.attention.Parameters.init(.fromBackend(.vanilla)), + zml.attention.Metadata.init(.fromBackend(.stablehlo)), + zml.attention.Parameters.init(.fromBackend(.stablehlo)), ).withPartitioning(tp, .{ .q = .replicated, .h = .model, .hd = .replicated }).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); const gated_output = attn_output.mul(gate.sigmoid()); diff --git a/platforms/neuron/libzml_neuron.BUILD.bazel b/platforms/neuron/libzml_neuron.BUILD.bazel index 824accb0..8d8a7119 100644 --- a/platforms/neuron/libzml_neuron.BUILD.bazel +++ b/platforms/neuron/libzml_neuron.BUILD.bazel @@ -107,7 +107,7 @@ copy_to_directory( visibility = [ "@zml//tools/neuron:__pkg__", "@zml//zml:__pkg__", - "@zml//zml/attention/nki:__pkg__", + "@zml//zml/kernels/flash_attention/nki:__pkg__", ], ) diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 46a72626..ec9ff4e0 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -78,21 +78,7 @@ zig_library( srcs = [ "Compiler.zig", "Sharding.zig", - "attention.zig", - "attention/attnd.zig", - "attention/cute_kernels/flashmla_sm100.zig", - "attention/cute_kernels/sparse_mla.zig", - "attention/flashattn.zig", - "attention/fly_kernels/sparse_mla.zig", - "attention/metal_attention.zig", - "attention/mosaic_tpu_kernels/ragged_attention.zig", - "attention/paged_attention.zig", - "attention/sparse_mla.zig", - "attention/tpu_attention.zig", - "attention/triton_attention.zig", - "attention/triton_kernels/unified_attention.zig", - "attention/triton_kernels/unified_attention_oneapi.zig", - "attention/triton_kernels/unified_sparse_mla.zig", + "attention/attention.zig", "buffer.zig", "buffer/HostAccessible.zig", "constants.zig", @@ -101,11 +87,12 @@ zig_library( "floats.zig", "io.zig", "kernel.zig", + "kernels/flash_attention/flash_attention.zig", + "kernels/paged_attention/paged_attention.zig", "logo.zig", "mem.zig", "meta.zig", "mlirx.zig", - "moe/tests.zig", "moe/cute_kernels/moe.zig", "moe/cute_kernels/mxfp4.zig", "moe/cute_kernels/persistent_mxfp4.zig", @@ -119,6 +106,7 @@ zig_library( "moe/mosaic_tpu.zig", "moe/mosaic_tpu_kernels/gmm_ep.zig", "moe/stablehlo.zig", + "moe/tests.zig", "moe/triton.zig", "moe/triton_kernels/mxfp4.zig", "moe/triton_kernels/triton_kernels.zig", @@ -136,11 +124,17 @@ zig_library( "tensor.zig", "testing.zig", "zml.zig", - "//zml/attention/nki:attention", + "//zml/kernels/flash_attention/cute", + "//zml/kernels/flash_attention/metal", + "//zml/kernels/flash_attention/nki", + "//zml/kernels/paged_attention/metal", + "//zml/kernels/paged_attention/mosaic_tpu", + "//zml/kernels/paged_attention/stablehlo", + "//zml/kernels/paged_attention/triton", ], data = select({ "//platforms:neuron.enabled": [ - "//zml/attention/nki:attention_kernel", + "//zml/kernels/flash_attention/nki:attention_kernel", ], "//conditions:default": [], }), diff --git a/zml/attention.zig b/zml/attention.zig deleted file mode 100644 index c5b51603..00000000 --- a/zml/attention.zig +++ /dev/null @@ -1,355 +0,0 @@ -const std = @import("std"); - -pub const attnd = @import("attention/attnd.zig"); -pub const cute = @import("attention/cute_kernels/sparse_mla.zig"); -pub const flashattn = @import("attention/flashattn.zig"); -pub const fly = @import("attention/fly_kernels/sparse_mla.zig"); -pub const metal = @import("attention/metal_attention.zig"); -pub const nki = @import("attention/nki/attention.zig"); -pub const paged_attention = @import("attention/paged_attention.zig"); -pub const tpu = @import("attention/tpu_attention.zig"); -pub const triton = @import("attention/triton_attention.zig"); -pub const triton_kernels = @import("attention/triton_kernels/unified_attention.zig"); -const zml = @import("zml.zig"); - -test { - std.testing.refAllDecls(@This()); -} - -pub const Backend = enum { - vanilla, - attnd, - nki, - cuda_fa2, - cuda_fa3, - metal_fa, - - pub fn auto(platform: *const zml.Platform) Backend { - return switch (platform.target) { - .cuda => b: { - const cc = zml.platform.cuda.computeCapability(platform) orelse break :b .vanilla; - break :b if (cc.eql(.{ .major = 9, .minor = 0 })) .cuda_fa3 else .cuda_fa2; - }, - .neuron => .nki, - .metal => .metal_fa, - .cpu, .rocm, .tpu, .oneapi, .furiosa => .vanilla, - }; - } - - pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { - return switch (backend) { - .vanilla => true, - .attnd => true, // attnd runs over network - .nki => platform.target == .neuron, - .metal_fa => platform.target == .metal, - .cuda_fa2 => platform.target == .cuda, - .cuda_fa3 => if (zml.platform.cuda.computeCapability(platform)) |cc| cc.eql(.{ .major = 9, .minor = 0 }) else false, - }; - } -}; - -pub const Parameters = union(Backend) { - vanilla: void, - attnd: attnd.Parameters, - nki: nki.Parameters, - cuda_fa2: flashattn.fa2.Parameters, - cuda_fa3: flashattn.fa3.Parameters, - metal_fa: void, - - pub const InitOptions = union(Backend) { - vanilla: void, - attnd: void, - nki: nki.Parameters, - cuda_fa2: flashattn.fa2.Parameters.InitOptions, - cuda_fa3: flashattn.fa3.Parameters.InitOptions, - metal_fa: void, - - pub fn fromBackend(backend: Backend) InitOptions { - return switch (backend) { - .vanilla => .{ .vanilla = {} }, - .attnd => @panic("Must be initialized manually"), - .nki => .{ .nki = .init() }, - .cuda_fa2 => .{ .cuda_fa2 = .{} }, - .cuda_fa3 => .{ .cuda_fa3 = .{} }, - .metal_fa => .{ .metal_fa = {} }, - }; - } - }; - - pub fn init(opts: InitOptions) Parameters { - return switch (opts) { - .vanilla => .{ .vanilla = {} }, - .attnd => @panic("Must be initialized manually"), - .nki => |v| .{ .nki = v }, - .cuda_fa2 => |v| .{ .cuda_fa2 = .init(v) }, - .cuda_fa3 => |v| .{ .cuda_fa3 = .init(v) }, - .metal_fa => .{ .metal_fa = {} }, - }; - } -}; - -pub const Metadata = union(Backend) { - vanilla: void, - attnd: attnd.Metadata, - nki: void, - cuda_fa2: flashattn.fa2.Metadata, - cuda_fa3: flashattn.fa3.Metadata, - metal_fa: metal.Metadata, - - pub const InitOptions = union(Backend) { - vanilla: void, - attnd: void, - nki: void, - cuda_fa2: flashattn.fa2.Metadata.InitOptions, - cuda_fa3: flashattn.fa3.Metadata.InitOptions, - metal_fa: void, - - pub fn fromBackend(backend: Backend, seqlen: i64, num_heads: i64) InitOptions { - return switch (backend) { - .vanilla => .{ .vanilla = {} }, - .attnd => .{ .attnd = {} }, - .nki => .{ .nki = {} }, - .cuda_fa2 => .{ .cuda_fa2 = .{ .seqlen = seqlen, .num_heads = num_heads } }, - .cuda_fa3 => .{ .cuda_fa3 = .{ .seqlen = seqlen, .num_heads = num_heads } }, - .metal_fa => .{ .metal_fa = {} }, - }; - } - }; - - pub fn init(opts: InitOptions, mesh: *const zml.Sharding.Mesh) Metadata { - return switch (opts) { - .vanilla => .{ .vanilla = {} }, - .attnd => @panic("Must be initialized manually"), - .nki => .{ .nki = {} }, - .cuda_fa2 => |o| .{ .cuda_fa2 = .init(o, mesh) }, - .cuda_fa3 => |o| .{ .cuda_fa3 = .init(o, mesh) }, - .metal_fa => .{ .metal_fa = .init() }, - }; - } - - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { - return switch (self) { - .vanilla => .{ .vanilla = {} }, - .nki => .{ .nki = {} }, - inline else => |v, tag| @unionInit(zml.Bufferized(Metadata), @tagName(tag), try v.initBuffer(io, platform)), - }; - } - - pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { - switch (self.*) { - .vanilla => {}, - .attnd => |*v| attnd.Metadata.deinitBuffer(v), - .nki => {}, - .cuda_fa2 => |*v| flashattn.fa2.Metadata.deinitBuffer(v), - .cuda_fa3 => |*v| flashattn.fa3.Metadata.deinitBuffer(v), - .metal_fa => |*v| metal.Metadata.deinitBuffer(v), - } - } -}; - -/// Causal attention as used in transformers. -/// -/// **Shapes**: -/// - q, result: .{ .q, .h, .hd } -/// - k, v: .{ .k, .h, .hd } -/// -/// Where: -/// - .h is the number of head -/// - .q is the number of queries -/// - .k is the number of keys -/// - .hd is the head dimension -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { - return switch (parameters) { - .vanilla => b: { - // Generate the attention mask. - const seq_len = k.dim(.k); - var attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, q.dtype(), null); - - // Note: in Pytorch it would be very inefficient to generate the full attn_mask, - // then slice into it, but XLA is able to optimize this correctly. - attn_mask = attn_mask.gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, attn_mask.dtype()), token_index.appendAxes(.{.coord}), .{}); - const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask }); - break :b attn_output; - }, - .attnd => attnd.causalAttention(q, k, v, token_index, metadata.attnd, parameters.attnd), - .nki => nki.attention(q, k, v, token_index, parameters.nki), - .cuda_fa2 => flashattn.fa2.attention(q, k, v, token_index, metadata.cuda_fa2, parameters.cuda_fa2), - .cuda_fa3 => flashattn.fa3.attention(q, k, v, token_index, metadata.cuda_fa3, parameters.cuda_fa3), - .metal_fa => metal.attention(q, k, v, token_index, metadata.metal_fa), - }; -} - -test "FlashAttention metadata initializes with explicit sharding outside compilation" { - try std.testing.expect(zml.Compiler.currentOrNull() == null); - const mesh: zml.Sharding.Mesh = .{ - .name = "metadata_mesh", - .physical = undefined, - .logical = .mesh(.{ .model = .high_bandwidth }), - .bindings = .init(&.{.init(&.{.link_x})}), - .folds = .empty, - .folds_consumed = .empty, - }; - inline for (.{ Backend.cuda_fa2, Backend.cuda_fa3 }) |backend| { - const metadata: Metadata = .init(.fromBackend(backend, 16, 8), &mesh); - const cuda_metadata = @field(metadata, @tagName(backend)); - inline for (std.meta.fields(@TypeOf(cuda_metadata))) |field| { - const shape = @field(cuda_metadata, field.name).shape(); - try std.testing.expectEqual(&mesh, shape._sharding.mesh); - if (shape.hasTag(.h)) |axis| { - try std.testing.expectEqual(zml.Sharding.PartitionSpec.sharded(0), shape.partition(axis)); - } else { - try std.testing.expectEqual(zml.Sharding.PartitionSpec.replicated, shape.partition(.meta)); - } - } - } -} - -test "attention: q=1,qh=64,kh=8" { - try testAttention( - .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{63}, - ); - - try testAttention( - .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{36}, - ); -} - -test "attention: b=4,q=1,qh=64,kh=8" { - // Full attention - try testAttention( - .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{ 63, 63, 63, 63 }, - ); - - // Partial attention - try testAttention( - .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{ 61, 57, 23, 63 }, - ); -} - -test "attention: q=1,qh=8,kh=8" { - try testAttention( - .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{62}, - ); - try testAttention( - .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{63}, - ); -} - -test "attention: q=8,qh=64,kh=8" { - try testAttention( - .init(.{ .q = 8, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{56}, - ); -} - -pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32) !void { - const io = std.testing.io; - const allocator = std.testing.allocator; - - const platform = zml.testing.env(); - - var arena_state: std.heap.ArenaAllocator = .init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - const token_index_shape: zml.Shape = if (q_shape.hasTag(.b)) |_| b: { - std.debug.assert(token_index_h.len == q_shape.dim(.b)); - break :b .init(.{ .b = q_shape.dim(.b) }, .u32); - } else .init(.{}, .u32); - - const max_k_idx: i64 = k_shape.dim(.k) - 1; - const max_q_idx: i64 = q_shape.dim(.q) - 1; - for (token_index_h) |index| { - // Check for out of bound reads - std.debug.assert(index + max_q_idx <= max_k_idx); - } - - const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor } = .{ - .q = .fromShape(q_shape), - .k = .fromShape(k_shape), - .v = .fromShape(k_shape), - .token_index = .fromShape(token_index_shape), - }; - - const rng_q = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.q.shape(), .{ .mean = 0, .stddev = 1 } }, .{}); - defer rng_q.deinit(); - const rng_k = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.k.shape(), .{ .mean = 0, .stddev = 1 } }, .{}); - defer rng_k.deinit(); - - var q = try rng_q.eval(allocator, io, {}); - defer q.deinit(); - var k = try rng_k.eval(allocator, io, {}); - defer k.deinit(); - var v = try rng_k.eval(allocator, io, {}); - defer v.deinit(); - var token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, @ptrCast(token_index_h)); - defer token_index.deinit(); - - const meshes = platform.meshes.values(); - const vanilla_exe = try platform.compileFn(allocator, io, attention, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, .vanilla, .vanilla }, .{ - .program_name = "attention_vanilla", - .meshes = meshes, - }); - defer vanilla_exe.deinit(); - - var vanilla_d = try vanilla_exe.eval(allocator, io, .{ q, k, v, token_index, .vanilla }); - defer vanilla_d.deinit(); - try vanilla_d.await(io); - const vanilla_h: zml.Slice = try vanilla_d.toSliceAlloc(allocator, io); - defer vanilla_h.free(allocator); - - for (std.enums.values(Backend)) |backend| { - switch (backend) { - .attnd, .vanilla => continue, - else => if (!backend.isAvailable(platform)) continue, - } - - const metadata: Metadata = .init(.fromBackend(backend, tensors.k.dim(.k), tensors.q.dim(.h)), platform.meshes.get("model").?); - const parameters: Parameters = .init(.fromBackend(backend)); - const exe = try platform.compileFn( - allocator, - io, - attention, - .{ tensors.q, tensors.k, tensors.v, tensors.token_index, metadata, parameters }, - .{ - .program_name = try std.fmt.allocPrint(arena, "attention_{t}", .{backend}), - .meshes = meshes, - }, - ); - defer exe.deinit(); - - var metadata_d = try metadata.initBuffer(io, platform); - defer Metadata.deinitBuffer(&metadata_d); - - var output_d = try exe.eval(allocator, io, .{ q, k, v, token_index, metadata_d }); - defer output_d.deinit(); - try output_d.await(io); - const output_h = try output_d.toSliceAlloc(allocator, io); - defer output_h.free(allocator); - - errdefer std.log.err( - \\ Attention test failed, {0t} output doesn't match reference. - \\ - reference: {1d} - \\ - {0t}: {2d} - , .{ backend, vanilla_h, output_h }); - try zml.testing.expectClose(io, vanilla_h, output_h, .{ - .absolute_tolerance = 5e-3, - .relative_tolerance = 1e-2, - .epsilon_relative = 1e-3, - .minimum_close_fraction = 0.99, - }); - } -} diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig new file mode 100644 index 00000000..49fe947b --- /dev/null +++ b/zml/attention/attention.zig @@ -0,0 +1,122 @@ +const std = @import("std"); + +const zml = @import("../zml.zig"); +const flash_attention = @import("../kernels/flash_attention/flash_attention.zig"); + +test { + std.testing.refAllDecls(@This()); +} + +pub const BackendType = enum { + stablehlo, + flash_attention, +}; + +pub const Backend = union(BackendType) { + stablehlo: void, + flash_attention: flash_attention.Backend, + + pub fn auto(platform: *const zml.Platform) Backend { + return if (flash_attention.isAvailable(platform)) + .{ .flash_attention = flash_attention.Backend.auto(platform) } + else + .{ .stablehlo = {} }; + } + + pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { + return switch (backend) { + .stablehlo => true, + .flash_attention => |b| b.isAvailable(platform), + }; + } +}; + +pub const Parameters = union(BackendType) { + stablehlo: void, + flash_attention: flash_attention.Parameters, + + pub const InitOptions = union(BackendType) { + stablehlo: void, + flash_attention: flash_attention.Parameters.InitOptions, + + pub fn fromBackend(backend: Backend) InitOptions { + return switch (backend) { + .stablehlo => .{ .stablehlo = {} }, + .flash_attention => |b| .{ .flash_attention = .fromBackend(b) }, + }; + } + }; + + pub fn init(opts: InitOptions) Parameters { + return switch (opts) { + .stablehlo => .{ .stablehlo = {} }, + .flash_attention => |v| .{ .flash_attention = .init(v) }, + }; + } +}; + +pub const Metadata = union(BackendType) { + stablehlo: void, + flash_attention: flash_attention.Metadata, + + pub const InitOptions = union(BackendType) { + stablehlo: void, + flash_attention: flash_attention.Metadata.InitOptions, + + pub fn fromBackend(backend: Backend) InitOptions { + return switch (backend) { + .stablehlo => .{ .stablehlo = {} }, + .flash_attention => |b| .{ .flash_attention = .fromBackend(b) }, + }; + } + }; + + pub fn init(opts: InitOptions) Metadata { + return switch (opts) { + .stablehlo => .{ .stablehlo = {} }, + .flash_attention => |v| .{ .flash_attention = .init(v) }, + }; + } + + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { + return switch (self) { + .stablehlo => .{ .stablehlo = {} }, + .flash_attention => |v| .{ .flash_attention = try v.initBuffer(io, platform) }, + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { + switch (self.*) { + .stablehlo => {}, + .flash_attention => |*v| flash_attention.Metadata.deinitBuffer(v), + } + } +}; + +/// Causal attention as used in transformers. +/// +/// **Shapes**: +/// - q, result: .{ .q, .h, .hd } +/// - k, v: .{ .k, .h, .hd } +/// +/// Where: +/// - .h is the number of head +/// - .q is the number of queries +/// - .k is the number of keys +/// - .hd is the head dimension +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { + return switch (parameters) { + .stablehlo => b: { + // Generate the attention mask. + const seq_len = k.dim(.k); + var attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, q.dtype(), null); + + // Note: in Pytorch it would be very inefficient to generate the full attn_mask, + // then slice into it, but XLA is able to optimize this correctly. + attn_mask = attn_mask.gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, attn_mask.dtype()), token_index.appendAxes(.{.coord}), .{}); + const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask }); + break :b attn_output; + }, + .flash_attention => flash_attention.attention(q, k, v, token_index, metadata.flash_attention, parameters.flash_attention), + }; +} diff --git a/zml/attention/attnd.zig b/zml/attention/attnd.zig deleted file mode 100644 index a986ab7b..00000000 --- a/zml/attention/attnd.zig +++ /dev/null @@ -1,500 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); - -const stdx = @import("stdx"); - -const zml = @import("../zml.zig"); - -const log = std.log.scoped(.@"zml/attnd"); -var context: ?*Context = null; - -pub const Config = struct { - destination: std.Io.net.IpAddress, - mtu: u16, -}; - -pub const Parameters = struct { - model_id: ModelId, - head_dim: u32, - head_bytes: u32, - num_kv_heads: u8, - num_q_per_head: u8, - num_q_per_packet: u8, - is_prefill: bool, - - pub const Init = struct { - model_id: ModelId, - head_dim: u32, - num_attention_heads: u32, - num_kv_heads: u8, - is_prefill: bool, - }; - - pub fn init(args: Init) Parameters { - const mtu: u32 = if (context) |ctx| @intCast(ctx.mtu) else @panic("attnd wasn't registered yet, use register() first."); - const num_q_per_head: u8 = @intCast(@divExact(args.num_attention_heads, args.num_kv_heads)); - const head_bytes = args.head_dim * @sizeOf(u16); // TODO: Here we assume bf16 or equivalent. - const kv_bytes = 2 * head_bytes; - - var max_q_per_packet: u8 = @intCast(std.math.divFloor(u32, mtu - @sizeOf(Header) - kv_bytes, head_bytes) catch unreachable); - max_q_per_packet = @min(max_q_per_packet, num_q_per_head); - - const num_packets_per_head: u8 = @intCast(std.math.divCeil(u32, num_q_per_head, max_q_per_packet) catch unreachable); - const num_q_per_packet = @divExact(num_q_per_head, num_packets_per_head); - if (num_q_per_packet < num_q_per_head) { - log.warn("Will split attnd requests in {d} packets of {d} queries", .{ num_packets_per_head, max_q_per_packet }); - } - - return .{ - .model_id = args.model_id, - .head_dim = args.head_dim, - .head_bytes = head_bytes, - .num_kv_heads = args.num_kv_heads, - .num_q_per_head = num_q_per_head, - .num_q_per_packet = num_q_per_packet, - .is_prefill = args.is_prefill, - }; - } -}; - -pub const Metadata = struct { - conversation_id: zml.Tensor, - layer_id: zml.Tensor, - num_tokens: zml.Tensor, - - pub fn init() Metadata { - return .{ - .conversation_id = .fromShape(.scalar(.u64)), - .layer_id = .fromShape(.scalar(.u16)), - .num_tokens = .fromShape(.scalar(.u32)), - }; - } - - pub fn initBuffer( - self: Metadata, - io: std.Io, - platform: *const zml.Platform, - ) !zml.Bufferized(Metadata) { - return .{ - .conversation_id = try zml.Buffer.scalar(io, platform, 749, .u64), - .layer_id = try zml.Buffer.uninitialized(io, platform, self.layer_id.shape(), .{}), - .num_tokens = try zml.Buffer.uninitialized(io, platform, self.num_tokens.shape(), .{}), - }; - } - - pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { - self.conversation_id.deinit(); - self.layer_id.deinit(); - self.num_tokens.deinit(); - } -}; - -pub fn register(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, config: Config) !void { - var ctx = try allocator.create(Context); - errdefer allocator.destroy(ctx); - - try ctx.initialize(io, allocator, platform, config); - - context = ctx; - - try targets.attnd.register(platform); -} - -pub fn deinit() void { - if (context) |ctx| { - ctx.deinit(); - context = null; - } -} - -const Context = struct { - allocator: std.mem.Allocator, - io: std.Io, - mtu: u16, - // One client per device. - clients: []Client, - - fn initialize(ctx: *Context, io: std.Io, allocator: std.mem.Allocator, platform: *const zml.Platform, config: Config) !void { - ctx.allocator = allocator; - ctx.io = io; - ctx.mtu = config.mtu; - ctx.clients = try allocator.alloc(Client, platform.devices.len); - errdefer allocator.free(ctx.clients); - - var c_len: usize = 0; - errdefer for (ctx.clients[0..c_len]) |*c| c.deinit(allocator, io); - for (ctx.clients) |*c| { - try c.initialize(allocator, io, config); - c_len += 1; - } - } - - fn deinit(self: *Context) void { - for (self.clients) |*c| { - c.deinit(self.allocator, self.io); - } - self.allocator.free(self.clients); - self.allocator.destroy(self); - } -}; - -pub fn causalAttention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_offset: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { - const tp = zml.Compiler.current().mesh(.model); - const num_partitions = tp.numPartitionsForLogicalAxis(.model); - - const actual_k, const actual_v = if (parameters.is_prefill) - .{ k, v } - else - .{ - k.slice(.k, .dynSingle(token_offset)), - v.slice(.k, .dynSingle(token_offset)), - }; - - // Concatenate q/k/v and metadata into a single byte buffer on the GPU. - // This reduces 7 separate D2H transfers to 1 during host-compute offload. - const packed_input = zml.Tensor.concatenate(&.{ - q.bytes(), - actual_k.bytes(), - actual_v.bytes(), - metadata.conversation_id.bytes(), - metadata.layer_id.bytes(), - token_offset.bytes(), - metadata.num_tokens.bytes(), - }, .bytes); - - const out = targets.attnd.call( - .{ .mqkv = packed_input }, - .{ - .attn = q.shape(), - }, - .{ - .model_id = parameters.model_id, - .head_dim = parameters.head_dim, - .head_size_in_bytes = parameters.head_bytes, - .num_kv_heads = parameters.num_kv_heads, - .num_q_per_head = parameters.num_q_per_head, - .num_q_per_packet = parameters.num_q_per_packet, - .num_partitions = @intCast(num_partitions), - .q_byte_size = @intCast(q.shape().byteSize()), - .k_byte_size = @intCast(actual_k.shape().byteSize()), - .v_byte_size = @intCast(actual_v.shape().byteSize()), - }, - ); - - return out.attn; -} - -pub const targets = struct { - const NAME = "attnd"; - - pub const attnd = zml.ops.CustomCall(Input, Output, Attributes, attndCall, .{ - .name = NAME, - .sharding_aware = true, - .has_side_effect = false, - .compute_on_host = true, - }); -}; - -const Input = struct { - mqkv: zml.Tensor, -}; - -const Output = struct { - attn: zml.Shape, -}; - -const Attributes = struct { - model_id: ModelId, - head_dim: u32, - head_size_in_bytes: u32, - num_kv_heads: u8, - num_q_per_head: u8, - num_q_per_packet: u8, - num_partitions: u32, - /// Byte sizes for unpacking the concatenated packed buffer. - q_byte_size: u32, - k_byte_size: u32, - v_byte_size: u32, -}; - -fn attndCall( - call_frame: *zml.pjrt.ffi.CallFrame, - input: zml.pjrtx.TensorToCustomCallBuffer(Input), - output: zml.pjrtx.ShapeToCustomCallBuffer(Output), - attrs: Attributes, -) !?*zml.pjrt.ffi.Error { - const ctx: *Context = context.?; - const device_ordinal: u64 = @intCast(try call_frame.ctx.getDeviceOrdinal(call_frame.api)); - - const client = &ctx.clients[device_ordinal]; - // Unpack the concatenated byte buffer: [q_bytes | k_bytes | v_bytes | conversation_id(8) | layer_id(2) | token_offset(4) | num_tokens(4)] - const out_full = output.attn.asHostSlice(); - const mqkv = input.mqkv.asHostConstSlice(); - const q_end = attrs.q_byte_size; - const k_end = q_end + attrs.k_byte_size; - const v_end = k_end + attrs.v_byte_size; - const q_full = mqkv[0..q_end]; - const k_full = mqkv[q_end..k_end]; - const v_full = mqkv[k_end..v_end]; - const conversation_id = std.mem.bytesAsValue(u64, mqkv[v_end..][0..@sizeOf(u64)]).*; - const layer_id = std.mem.bytesAsValue(u16, mqkv[v_end + @sizeOf(u64) ..][0..@sizeOf(u16)]).*; - const token_offset = std.mem.bytesAsValue(u32, mqkv[v_end + @sizeOf(u64) + @sizeOf(u16) ..][0..@sizeOf(u32)]).*; - const num_tokens = std.mem.bytesAsValue(u32, mqkv[v_end + @sizeOf(u64) + @sizeOf(u16) + @sizeOf(u32) ..][0..@sizeOf(u32)]).*; - - const q_size_in_bytes = attrs.num_q_per_head * attrs.head_size_in_bytes; - const head_size_in_bytes = attrs.head_size_in_bytes; - const kv_heads_per_partitions = @divExact(attrs.num_kv_heads, attrs.num_partitions); - const kv_head_offset = kv_heads_per_partitions * device_ordinal; - for (0..num_tokens) |t| { - const q = q_full[t * kv_heads_per_partitions * q_size_in_bytes ..][0 .. kv_heads_per_partitions * q_size_in_bytes]; - const k = k_full[t * kv_heads_per_partitions * head_size_in_bytes ..][0 .. kv_heads_per_partitions * head_size_in_bytes]; - const v = v_full[t * kv_heads_per_partitions * head_size_in_bytes ..][0 .. kv_heads_per_partitions * head_size_in_bytes]; - - var num_send_messages: usize = 0; - for (0..kv_heads_per_partitions) |kv_head_id| { - const k_offset = kv_head_id * head_size_in_bytes; - const v_offset = kv_head_id * head_size_in_bytes; - var q_offset = kv_head_id * head_size_in_bytes * attrs.num_q_per_head; - var q_sent: u8 = 0; - - while (q_sent < attrs.num_q_per_head) { - const nq = @min(attrs.num_q_per_head - q_sent, attrs.num_q_per_packet); - const header: Header = .{ - .type = .attn, - .kv_head_id = @intCast(kv_head_offset + kv_head_id), - .conversation_id = conversation_id, - .token_pos = token_offset + @as(u32, @intCast(t)), - .layer_id = layer_id, - .model_id = attrs.model_id, - .first_q_id = q_sent, - .num_queries = nq, - }; - try client.send( - header, - q[q_offset..][0 .. head_size_in_bytes * nq], - k[k_offset..][0..head_size_in_bytes], - v[v_offset..][0..head_size_in_bytes], - ); - - q_offset += head_size_in_bytes * nq; - q_sent += nq; - num_send_messages += 1; - } - } - - var out = out_full[t * kv_heads_per_partitions * q_size_in_bytes ..][0 .. kv_heads_per_partitions * q_size_in_bytes]; - var recv_q: usize = 0; - while (recv_q < kv_heads_per_partitions * attrs.num_q_per_head) { - const response = client.receive() catch |err| { - log.err("Failed to receive response: {any}", .{err}); - return err; - }; - const header = response.header; - const payload = response.payload; - - std.debug.assert(header.num_queries <= attrs.num_q_per_head); - std.debug.assert(header.kv_head_id < attrs.num_kv_heads); - std.debug.assert(header.first_q_id + header.num_queries <= attrs.num_q_per_head); - - const n = header.num_queries * attrs.head_size_in_bytes; - std.debug.assert(payload.len == n); - const kv_head_id = header.kv_head_id - kv_head_offset; - const q_offset = (@as(usize, kv_head_id) * attrs.num_q_per_head + header.first_q_id) * attrs.head_size_in_bytes; - @memcpy(out[q_offset .. q_offset + n], payload[0..n]); - recv_q += header.num_queries; - num_send_messages -= 1; - } - std.debug.assert(num_send_messages == 0); - } - - return null; -} - -const MAX_CONCURRENT_PACKETS_PER_CLIENT = 32; - -const Client = struct { - socket: std.Io.net.Socket, - destination: std.Io.net.IpAddress, - packet_buffer: []u8, - send_addr: std.Io.Threaded.PosixAddress, - send_namelen: std.posix.socklen_t, - - fn initialize(self: *Client, allocator: std.mem.Allocator, io: std.Io, config: Config) !void { - const src: std.Io.net.IpAddress = .{ .ip4 = .unspecified(0) }; - const socket = try src.bind(io, .{ .mode = .dgram, .protocol = .udp }); - errdefer socket.close(io); - - const socket_buffer_size = @as(u32, @intCast(config.mtu)) * MAX_CONCURRENT_PACKETS_PER_CLIENT; - try std.posix.setsockopt(socket.handle, std.posix.SOL.SOCKET, std.posix.SO.RCVBUF, @ptrCast(&socket_buffer_size)); - try std.posix.setsockopt(socket.handle, std.posix.SOL.SOCKET, std.posix.SO.SNDBUF, @ptrCast(&socket_buffer_size)); - - // Set the Don't Fragment bit to avoid fragmentation on IPv4 - switch (builtin.os.tag) { - .linux => { - // With PMTUDISC_DO the kernel will keep track of the ICMP messages to detect the real MTU. - // Best practice [1] seems to be to use IP_PMTUDISC_PROBE and do DPLPMTUD to avoid Blind Performance-Degrading Attack [2] - // But we assume to be within an internal network, otherwise attnd is useless anyway. - // [1] https://seemann.io/posts/2025-02-19---ip-fragmentation/ - // [2] https://www.rfc-editor.org/rfc/rfc5927#section-7 - // TODO: Do we actually try to use the max MTU? If we just assume it to be fine, we could just as well use IP_PMTUDISC_PROBE - const mtu_discover: u32 = std.os.linux.IP.PMTUDISC_DO; - try std.posix.setsockopt(socket.handle, std.posix.IPPROTO.IP, std.os.linux.IP.MTU_DISCOVER, @ptrCast(&mtu_discover)); - }, - .macos => { - // We could set IP_DONTFRAG on the socket, but Zig doesn't expose this option (yet). - }, - else => {}, - } - - const packet_buffer = try allocator.alloc(u8, @as(usize, @intCast(config.mtu))); - errdefer allocator.free(packet_buffer); - - self.socket = socket; - self.destination = config.destination; - self.packet_buffer = packet_buffer; - self.send_namelen = std.Io.Threaded.addressToPosix(&self.destination, &self.send_addr); - } - - pub fn send(self: *Client, header: Header, q: []const u8, k: []const u8, v: []const u8) !void { - const iovecs = [_]std.posix.iovec_const{ - .{ .base = std.mem.asBytes(&header).ptr, .len = @sizeOf(Header) }, - .{ .base = q.ptr, .len = q.len }, - .{ .base = k.ptr, .len = k.len }, - .{ .base = v.ptr, .len = v.len }, - }; - const msg: std.posix.msghdr_const = .{ - .name = &self.send_addr.any, - .namelen = self.send_namelen, - .iov = &iovecs, - .iovlen = iovecs.len, - .control = null, - .controllen = 0, - .flags = 0, - }; - - const rc = std.c.sendmsg(self.socket.handle, &msg, std.posix.MSG.DONTWAIT); - switch (std.posix.errno(rc)) { - .SUCCESS => { - std.debug.assert(rc == @sizeOf(Header) + q.len + k.len + v.len); - }, - else => |err| { - log.err("Failed to send response: {any}", .{err}); - @panic("Attnd failed"); - }, - } - } - - pub fn receive(self: *Client) !Response { - var iovecs = [_]std.posix.iovec{ - .{ .base = self.packet_buffer.ptr, .len = self.packet_buffer.len }, - }; - var msg: std.posix.msghdr = .{ - .name = null, - .namelen = 0, - .iov = &iovecs, - .iovlen = iovecs.len, - .control = null, - .controllen = 0, - .flags = 0, - }; - - const rc = std.c.recvmsg( - self.socket.handle, - &msg, - 0, - ); - switch (std.posix.errno(rc)) { - .SUCCESS => { - const n: usize = @intCast(rc); - return Response.parse(self.packet_buffer[0..n]); - }, - else => |err| { - log.err("Failed to receive response: {any}", .{err}); - @panic("Attnd failed"); - }, - } - } - - pub fn deinit(self: *Client, allocator: std.mem.Allocator, io: std.Io) void { - allocator.free(self.packet_buffer); - self.socket.close(io); - } -}; - -pub const Message = struct { - header: *align(1) const Header, - payload: []const u8, - - pub fn parse(bytes: []const u8) !Message { - const n = @sizeOf(Header); - if (bytes.len < n) return error.ShortMessage; - - const msg: Message = .{ - .header = std.mem.bytesAsValue(Header, bytes[0..n]), - .payload = bytes[n..], - }; - - if (!(msg.header.magic[0] == 'Z' and msg.header.magic[1] == 'M' and msg.header.magic[2] == 'L')) { - return error.InvalidHeader; - } - - return msg; - } -}; - -pub const Request = struct { - header: *align(1) const Header, - q: []const u8, - k: []const u8, - v: []const u8, -}; - -pub const Response = Message; - -pub const Header = extern struct { - pub const MAGIC: [3]u8 = .{ 'Z', 'M', 'L' }; // 0x5A4D4C - - pub const Type = enum(u8) { - // Compute attention - attn = 'a', - // Starts a conversation - start = 's', - // Ping: the server answers immediately skipping all compute - ping = 'p', - _, - }; - - type: Type, - kv_head_id: u8, - - conversation_id: u64 align(1), - token_pos: u32 align(1), - layer_id: u16 align(1), - - model_id: ModelId, - first_q_id: u8, - num_queries: u8, - magic: [3]u8 align(1) = MAGIC, - - comptime { - // destination MAC (6) + source MAC (6) + Ethertype (2) [+ 802.1Q / VLAN tag (4)] - // We the optional VLAN, we can only guarantee an alignment of 4 for the payload after the prefix + header. - const ETH_HEADER_SIZE = 6 + 6 + 2; - const IP_HEADER_SIZE = 20; - const UDP_HEADER_SIZE = 8; - const PREFIX_SIZE = ETH_HEADER_SIZE + IP_HEADER_SIZE + UDP_HEADER_SIZE; - stdx.debug.assertComptime( - (PREFIX_SIZE + @sizeOf(Header)) % 4 == 0, - "ETH_HEADER_SIZE ({}) + @sizeOf(AttnRequest) ({}) == {d} != k * 4", - .{ PREFIX_SIZE, @sizeOf(Header), PREFIX_SIZE + @sizeOf(Header) }, - ); - } -}; - -pub const ModelId = enum(u8) { - @"llama-3.1-8B" = 0, - @"llama-3.2-1B" = 1, - @"qwen3-14B" = 2, - @"qwen3-32B" = 3, - _, // Unsupported models -}; diff --git a/zml/attention/cute_kernels/flashmla_sm100.zig b/zml/attention/cute_kernels/flashmla_sm100.zig deleted file mode 100644 index ed03b394..00000000 --- a/zml/attention/cute_kernels/flashmla_sm100.zig +++ /dev/null @@ -1,1355 +0,0 @@ -//! Sparse MLA decode / prefill for DeepSeek-V4.1 on SM100: persistent CTAs, -//! each processing query tokens t = cta, cta + ctas, ... (one token per -//! cluster when a token's key blocks are split across a cluster). -//! -//! Swapped-operand formulation (the 16 real heads are the MMA N dimension): -//! S^T [64 keys, 16 heads] = K_blk [64, 512] . Q^T tcgen05 M=64 N=16, K-major A/B -//! O^T [512, 16] += V_blk^T [512, 64] . P^T tcgen05 M=128 N=16, MN-major A -//! K and V are the same dequantized BF16 tile (SW128 K-major, 64 KiB), read -//! through two descriptors. Nothing is materialized in global memory and there -//! is no split-KV reduction (a cluster merges its partials through DSMEM). -//! -//! Warp roles (384 threads): -//! 0-3 softmax (lazy rescale, one barrier vote per block), O rescale (TMEM), epilogue -//! 4 MMA issue (one elected lane) -//! 5-6 loaders: cp.async of the raw rows and scales of each key block (zero-fill -//! for invalid rows) -//! 7 front: per token, validated indices and block counts (one token ahead), Q -//! 8-11 FP8/FP4 -> BF16 dequantization into the tile, in 128-column parts -//! Key blocks are numbered across the CTA's tokens: block g uses buffer g & 1 -//! with parity (g >> 1) & 1. Per-token state (indices, Q, O in TMEM) is -//! double-buffered by token parity so consecutive tokens overlap. -const std = @import("std"); - -const zml = @import("../../zml.zig"); -const cute = zml.kernel.cute; -const B = cute.Builder; -const V = cute.Value; - -pub const Cfg = struct { - queries: i64, - window_capacity: i64, - compressed_capacity: i64, - window_slots: i64, - compressed_slots: i64, - scale: f64, - /// CTAs per query token (a thread-block cluster): each takes a contiguous - /// range of the token's key blocks and the partial results are merged - /// through distributed shared memory, inside the kernel. Power of two. - /// Clusters of more than one CTA require `ctas == queries`. - cluster: i64 = 1, - /// Persistent CTAs (clusters): token t is processed by CTA t % ctas. - ctas: i64 = 0, - - fn gridCtas(c: Cfg) i64 { - return if (c.ctas > 0) @min(c.ctas, c.queries) else c.queries; - } -}; - -pub const Program = cute.Program(Cfg, .{ - .name = "flashmla_sm100", - .inputs = &.{ "q", "wv", "ws", "cv", "cs", "wi", "ci", "lengths", "sink", "active" }, - .outputs = &.{"out"}, - .run = program, -}); - -const kernel_name = "flashmla_sm100_kernel"; -const H = 16; -const D = 512; -const BK = 64; -const threads = 384; -const log2e = 1.4426950408889634; -/// Lazy rescale: the running max only moves when a block exceeds it by this -/// many log2 units, so P stays below 2^32 (BF16 P, F32 l and O keep their -/// relative precision) and O is rarely rescaled. -const rescale_threshold = 32.0; - -// Shared memory plan (bytes). -const smem = struct { - const tile = [2]i32{ 0, 65536 }; - /// Q of token parity kb (SW128 K-major, the B operand of QK). - const q = [2]i32{ 131072, 147456 }; - /// Raw (quantized) rows: a ring of slots of 64 rows x 288 bytes (see `rawBase`). - /// An FP4 block takes one slot (256 data bytes, then the 32 scale bytes of each - /// row); an FP8 block takes two (row bytes 0-255, then 256-511; its 16 scale - /// bytes follow the second slot's data). - const raw_slot = 64 * 288; - const scale_off = 256; - const p = [2]i32{ 221184, 223232 }; - const red = [2]i32{ 225280, 225536 }; - const red_sum = [2]i32{ 225792, 226048 }; - const bars = 226304; - const holder = bars + 8 * 40; - /// Per token parity: physical row per candidate (or -1), window [128] then - /// compressed [512], followed by the token's block counts (window, compressed). - const idx = holder + 16; - const idx_bytes = 640 * 4 + 16; - const counts = 640 * 4; - /// Sink logits (f32 [16], prefetched at start). - const sink = idx + 2 * idx_bytes; - const total = sink + H * 4; - /// Output staging, bf16 [16 heads][512] (reuses Q after the CTA's last QK). - const ostage = q[0]; - /// Cluster merge (after every block of the cluster is done), in the raw ring: - /// per-head max and sum of this CTA, merge weights [16 CTAs][16 heads], and the - /// inbox of (m, l) per head from every rank [16 ranks][32] f32. - const stats = 163840; - const weights = stats + 2 * H * 4; - const stats_in = weights + 16 * H * 4; - /// Cluster merge: O^T [512][16] f32 of this CTA (reuses tile 0 after the last PV), - /// and the inbox of the dims this rank owns, [ranks][512 / C][16] f32 (tile 0, upper half). - const obuf = 0; - const inbox = 32768; -}; - -const Bar = enum(i32) { - raw_full = 0, // [4] - raw_empty = 4, // [4] - tile_full = 8, - s_full = 10, - p_full = 12, - pv_done = 14, - idx_ready = 16, - idx_free = 18, - o_free = 20, - q_free = 22, - q_ready = 24, -}; - -/// Arrivals on idx_free: softmax warps (4), MMA (1), loaders (2), dequant warps (4). -const idx_consumers = 11; - -fn bar(b: *B, base: V, comptime which: Bar, buf: anytype) V { - return base.add(smem.bars + 8 * @intFromEnum(which)).add(b.lift(buf).mul(8)); -} - -// TMEM columns: O^T of token parity ob in 64 * ob (4 chunks of 128 dims x 16 -// heads); S^T of buffer `buf` in 128 + 64 * buf: four partial sums over 128 -// latent dims each, 16 columns apart (independent accumulators so the QK MMAs pipeline). -const tmem_cols = 256; -const tmem_s = 128; -const qk_parts = 4; - -const Format = enum { - fp8, - fp4, - fn rowBytes(f: Format) i32 { - return if (f == .fp8) 512 else 256; - } - fn scaleBytes(f: Format) i32 { - return if (f == .fp8) 16 else 32; - } -}; - -const Globals = struct { - q: V, - wv: V, - ws: V, - cv: V, - cs: V, - wi: V, - ci: V, - lengths: V, - sink: V, - active: V, - out: V, -}; - -const Ctx = struct { - b: *B, - cfg: Cfg, - g: Globals, - s: V, // shared base - tid: V, - warp: V, - lane: V, - taddr: V, - rank: V, // CTA rank in the token's cluster - cta: V, // cluster index: first token - ntok: V, // tokens of this CTA - // Per token (see `forToken`). - k: V = undefined, // token ordinal within the CTA - t: V = undefined, // query token - kb: V = undefined, // token parity (index buffer, O buffer) - ib: V = undefined, // shared address of the token's index buffer - nwb: V = undefined, // window blocks - ncb: V = undefined, // compressed blocks - b0: V = undefined, // this CTA's key blocks [b0, b1) of the token - b1: V = undefined, - n: V = undefined, // b1 - b0; local block j is block b0 + j of the token - nwl: V = undefined, // local window (FP8) blocks: the first nwl local blocks - g0: V = undefined, // CTA-wide number of the token's first local block - slot0: V = undefined, // CTA-wide raw-slot use of the token's first local block -}; - -/// The per-token context of token ordinal k; `counts` waits for the token's -/// index buffer and loads the block counts. -fn forToken(c0: Ctx, k: V, g0: V, slot0: V, comptime counts: bool) Ctx { - const b = c0.b; - var c = c0; - c.k = k; - c.t = c0.cta.add(k.mul(@as(i32, @intCast(c0.cfg.gridCtas())))); - c.kb = k.bitAnd(1); - c.ib = c0.s.add(smem.idx).add(c.kb.mul(smem.idx_bytes)); - c.g0 = g0; - c.slot0 = slot0; - if (counts) { - b.mbarrierWait(bar(b, c.s, .idx_ready, c.kb), parity(k)); - const nc = b.loadWords(2, c.ib.add(smem.counts), .shared, .{}); - c.nwb = nc[0]; - c.ncb = nc[1]; - const nb = c.nwb.add(c.ncb); - const log_c: i32 = @intCast(std.math.log2_int(u64, @intCast(c0.cfg.cluster))); - c.b0 = c0.rank.mul(nb).shrLogical(log_c); - c.b1 = c0.rank.add(1).mul(nb).shrLogical(log_c); - c.n = c.b1.sub(c.b0); - c.nwl = c.b1.minimum(c.nwb).sub(c.b0).maximum(0); - } - return c; -} - -/// This warp is done with the token's index buffer. -fn releaseIndices(c: Ctx) void { - const b = c.b; - b.syncWarp(); - var l0 = b.openIf(c.lane.eq(0)); - b.mbarrierArrive(bar(b, c.s, .idx_free, c.kb), 1); - l0.yieldThen(.{}); -} - -/// First raw-slot use of local block j: FP8 (window) blocks come first and use two. -fn slotUse(c: Ctx, j: V) V { - return c.slot0.add(j).add(j.minimum(c.nwl)); -} - -/// With one token per CTA the second Q buffer is unused: the ring starts there and -/// has 4 slots (two FP8 blocks in flight); otherwise 3 slots after it. -fn oneTokenPerCta(cfg: Cfg) bool { - return cfg.gridCtas() == cfg.queries; -} - -fn rawBase(cfg: Cfg) i32 { - return if (oneTokenPerCta(cfg)) smem.q[1] else smem.q[1] + 16384; -} - -fn rawSlots(cfg: Cfg) i32 { - return if (oneTokenPerCta(cfg)) 4 else 3; -} - -fn slotAddr(c: Ctx, use: V) V { - return c.s.add(rawBase(c.cfg)).add(use.rem(rawSlots(c.cfg)).mul(smem.raw_slot)); -} - -fn scaleAddr(c: Ctx, use: V) V { - return slotAddr(c, use).add(smem.scale_off); -} - -fn slotParity(c: Ctx, use: V) V { - return use.div(rawSlots(c.cfg)).bitAnd(1); -} - -fn parity(j: V) V { - return j.shrLogical(1).bitAnd(1); -} - -fn tupleOf(comptime n: usize, arr: [n]V) std.meta.Tuple(&([_]type{V} ** n)) { - var t: std.meta.Tuple(&([_]type{V} ** n)) = undefined; - inline for (0..n) |i| t[i] = arr[i]; - return t; -} - -fn program(b: *B, cfg: Cfg) cute.FinishError!void { - b.beginFunction(kernel_name, .cuda_kernel); - try kernel(b, cfg); - b.endFunction(.{ threads, 1, 1 }); - - b.beginFunction(Program.name, .host); - const h = try b.declareArgs(.{ - .q = .{ .ptr = cute.DType.bf16 }, - .wv = .{ .ptr = cute.DType.i8 }, - .ws = .{ .ptr = cute.DType.i8 }, - .cv = .{ .ptr = cute.DType.i8 }, - .cs = .{ .ptr = cute.DType.i8 }, - .wi = .{ .ptr = cute.DType.i32 }, - .ci = .{ .ptr = cute.DType.i32 }, - .lengths = .{ .ptr = cute.DType.i32 }, - .sink = .{ .ptr = cute.DType.f32 }, - .active = .{ .ptr = cute.DType.i32 }, - .out = .{ .ptr = cute.DType.bf16 }, - }); - launch(b, cfg, .{ h.q, h.wv, h.ws, h.cv, h.cs, h.wi, h.ci, h.lengths, h.sink, h.active, h.out }); - b.endFunction(null); -} - -fn launch(b: *B, cfg: Cfg, args: anytype) void { - const one = b.cst(.i32, 1); - const config = b.makeLaunchConfig(.{ - .grid = .{ b.cst(.i32, @as(i32, @intCast(cfg.gridCtas() * cfg.cluster))), one, one }, - .block = .{ b.cst(.i32, threads), one, one }, - .dynamic_smem = b.kernelSmemSize(kernel_name), - .stream = b.cudaStream(), - .cluster = .{ b.cst(.i32, @as(i32, @intCast(cfg.cluster))), one, one }, - .use_pdl = true, - }); - const status = b.launchEx(kernel_name, config, args); - b.returnHostStatus(b.cudaResultStatus(status)); -} - -fn kernel(b: *B, cfg: Cfg) cute.FinishError!void { - const g = try globalsOf(b, try b.declareArgs(.{ - .q = .{ .ptr = cute.DType.bf16 }, - .wv = .{ .ptr = cute.DType.i8 }, - .ws = .{ .ptr = cute.DType.i8 }, - .cv = .{ .ptr = cute.DType.i8 }, - .cs = .{ .ptr = cute.DType.i8 }, - .wi = .{ .ptr = cute.DType.i32 }, - .ci = .{ .ptr = cute.DType.i32 }, - .lengths = .{ .ptr = cute.DType.i32 }, - .sink = .{ .ptr = cute.DType.f32 }, - .active = .{ .ptr = cute.DType.i32 }, - .out = .{ .ptr = cute.DType.bf16 }, - })); - try kernelBody(b, cfg, g); -} - -fn globalsOf(b: *B, a: anytype) !Globals { - b.setFunctionAttribute("nvvm.minctasm", .int(b.ctx, .i32, 1)); - b.setFunctionAttribute("cu_attrs", b.parseAttribute("{max_dynamic_shared_size_bytes = #cuda.dev_max_shared_memory_optin, non_portable_cluster_size_allowed = 1 : i32}")); - - return .{ - .q = b.ptrToInt(a.q, .i64), - .wv = b.ptrToInt(a.wv, .i64), - .ws = b.ptrToInt(a.ws, .i64), - .cv = b.ptrToInt(a.cv, .i64), - .cs = b.ptrToInt(a.cs, .i64), - .wi = b.ptrToInt(a.wi, .i64), - .ci = b.ptrToInt(a.ci, .i64), - .lengths = b.ptrToInt(a.lengths, .i64), - .sink = b.ptrToInt(a.sink, .i64), - .active = b.ptrToInt(a.active, .i64), - .out = b.ptrToInt(a.out, .i64), - }; -} - -fn kernelBody(b: *B, cfg: Cfg, g: Globals) cute.FinishError!void { - const tid = b.threadIdx().x; - const warp = b.makeWarpUniform(tid.shrLogical(5)); - const lane = tid.bitAnd(31); - std.debug.assert(std.math.isPowerOfTwo(cfg.cluster) and cfg.cluster <= 16); - std.debug.assert(cfg.cluster == 1 or cfg.gridCtas() == cfg.queries); - const log_c: i32 = @intCast(std.math.log2_int(u64, @intCast(cfg.cluster))); - const cta = b.blockIdx().x.shrLogical(log_c); - const stride: i32 = @intCast(cfg.gridCtas()); - const queries: i32 = @intCast(cfg.queries); - const rank = if (cfg.cluster > 1) b.clusterCtaRank() else b.cst(.i32, 0); - const s = sharedStorage(b, smem.total); - - // Inputs come from the previous kernels of the layer. - b.waitForDependency(); - const c0: Ctx = .{ - .b = b, - .cfg = cfg, - .g = g, - .s = s, - .tid = tid, - .warp = warp, - .lane = lane, - .taddr = undefined, - .rank = rank, - .cta = cta, - .ntok = cta.sub(queries).neg().add(stride - 1).div(stride), - }; - // The front warp issues the first token's loads right away, overlapping the setup - // (other warps load harmless in-bounds addresses and discard them). - const first = frontLoad(c0, b.select(warp.eq(7), cta, b.cst(.i32, 0))); - { - var w7 = b.openIf(warp.eq(7)); - issueQ(c0, cta, b.cst(.i32, 0)); - w7.yieldThen(.{}); - } - { - var t0 = b.openIf(tid.eq(0)); - for (0..@intCast(rawSlots(cfg))) |i| { - b.mbarrierInit(bar(b, s, .raw_full, @as(i32, @intCast(i))), 64); - b.mbarrierInit(bar(b, s, .raw_empty, @as(i32, @intCast(i))), 128); - } - for (0..2) |i| { - b.mbarrierInit(bar(b, s, .tile_full, @as(i32, @intCast(i))), 128); - b.mbarrierInit(bar(b, s, .s_full, @as(i32, @intCast(i))), 1); - b.mbarrierInit(bar(b, s, .p_full, @as(i32, @intCast(i))), 128); - b.mbarrierInit(bar(b, s, .pv_done, @as(i32, @intCast(i))), 1); - b.mbarrierInit(bar(b, s, .idx_ready, @as(i32, @intCast(i))), 32); - b.mbarrierInit(bar(b, s, .idx_free, @as(i32, @intCast(i))), idx_consumers); - b.mbarrierInit(bar(b, s, .o_free, @as(i32, @intCast(i))), 128); - b.mbarrierInit(bar(b, s, .q_ready, @as(i32, @intCast(i))), 32); - b.mbarrierInit(bar(b, s, .q_free, @as(i32, @intCast(i))), 1); - } - b.fenceMbarrierInit(); - t0.yieldThen(.{}); - } - { - var w0 = b.openIf(warp.eq(0)); - b.tmemAllocAt(s.add(smem.holder), tmem_cols); - b.relinquishTmemAllocPermit(.{}); - w0.yieldThen(.{}); - } - - b.tcgen05Fence(.before); - b.barrierSync(0, threads); - b.tcgen05Fence(.after); - var c = c0; - c.taddr = b.loadWords(1, s.add(smem.holder), .shared, .{})[0]; - { - var role = b.openIf(warp.lt(4)); - softmaxRole(c); - role.yieldThen(.{}); - } - { - var role = b.openIf(warp.eq(4)); - mmaRole(c); - role.yieldThen(.{}); - } - { - var role = b.openIf(warp.eq(5).bitOr(warp.eq(6))); - loaderRole(c); - role.yieldThen(.{}); - } - { - var role = b.openIf(warp.eq(7)); - frontRole(c, first); - role.yieldThen(.{}); - } - { - var role = b.openIf(warp.ge(8)); - dequantRole(c); - role.yieldThen(.{}); - } - - b.tcgen05Fence(.before); - b.barrierSync(0, threads); - { - var w0 = b.openIf(warp.eq(0)); - b.tcgen05Fence(.after); - b.tmemDeallocAt(c.taddr, tmem_cols); - w0.yieldThen(.{}); - } - if (cfg.cluster > 1) { - // One token per cluster. - var cm = c; - cm.t = cta; - clusterMerge(cm); - } - b.launchDependents(); -} - -// ---- front (warp 7): indices, lengths and Q of each token --------------------------------- - -const FrontRegs = struct { - active: V, - lens: [2]V, - w: [4]V, // this lane's 4 window indices - c: [16]V, // this lane's 16 compressed indices -}; - -/// Issue the global loads of token t's lengths and indices (lane l holds -/// candidates [4 l, 4 l + 4) of the window and [16 l, 16 l + 16) of the compressed stream). -fn frontLoad(c: Ctx, t: V) FrontRegs { - const b = c.b; - var r: FrontRegs = undefined; - r.active = b.loadWords(1, c.g.active, .global, .{ .invariant = true })[0]; - r.lens = b.loadWords(2, c.g.lengths.add(t.to(.i64).mul(8)), .global, .{ .invariant = true }); - const wcap: i32 = @intCast(c.cfg.window_capacity); - const ccap: i32 = @intCast(c.cfg.compressed_capacity); - r.w = b.loadWords(4, c.g.wi.add(t.mul(wcap).add(c.lane.mul(@divExact(wcap, 32))).to(.i64).mul(4)), .global, .{ .invariant = true }); - for (&r.c) |*x| x.* = b.cst(.i32, -1); - if (ccap > 0) { - const per_lane = @divExact(ccap, 32); - const src = c.g.ci.add(t.mul(ccap).add(c.lane.mul(per_lane)).to(.i64).mul(4)); - for (0..@intCast(@divExact(per_lane, 4))) |v| { - const y = b.loadWords(4, src.add(@as(i64, @intCast(v * 16))), .global, .{ .invariant = true }); - for (0..4) |i| r.c[v * 4 + i] = y[i]; - } - } - return r; -} - -/// Validate token t's candidates (invalid ones become -1), store them and the -/// block counts in index buffer `kb`, and publish them. -fn frontStore(c: Ctx, t: V, kb: V, r: FrontRegs) void { - const b = c.b; - const zero = b.cst(.i32, 0); - // Padded (inactive) query rows get no key blocks: their output is zero. - const is_active = t.lt(r.active); - const wlen = b.select(is_active, r.lens[0].minimum(@as(i32, @intCast(c.cfg.window_capacity))).maximum(0), zero); - const clen = if (c.cfg.compressed_capacity > 0) b.select(is_active, r.lens[1].minimum(@as(i32, @intCast(c.cfg.compressed_capacity))).maximum(0), zero) else zero; - const ib = c.s.add(smem.idx).add(kb.mul(smem.idx_bytes)); - inline for (.{ Format.fp8, Format.fp4 }) |f| { - const capacity: i32 = @intCast(if (f == .fp8) c.cfg.window_capacity else c.cfg.compressed_capacity); - if (capacity > 0) { - const slots: i32 = @intCast(if (f == .fp8) c.cfg.window_slots else c.cfg.compressed_slots); - const length = if (f == .fp8) wlen else clen; - const per_lane = @divExact(capacity, 32); // 4 (window) or 16 (compressed) - for (0..@intCast(per_lane)) |i| { - const pos = c.lane.mul(per_lane).add(@as(i32, @intCast(i))); - const ix = if (f == .fp8) r.w[i] else r.c[i]; - const ok = pos.lt(length).bitAnd(ix.ge(0)).bitAnd(ix.lt(slots)); - b.storeWords(1, ib.add(idxBase(f)).add(pos.mul(4)), .shared, .{b.select(ok, ix, b.cst(.i32, -1))}); - } - } - } - { - var l0 = b.openIf(c.lane.eq(0)); - b.storeWords(2, ib.add(smem.counts), .shared, .{ wlen.add(BK - 1).shrLogical(6), clen.add(BK - 1).shrLogical(6) }); - l0.yieldThen(.{}); - } - b.mbarrierArrive(bar(b, c.s, .idx_ready, kb), 1); -} - -/// Q [16, 512] BF16 of token t -> SW128 K-major shared tile (the B operand of QK). -fn issueQ(c: Ctx, t: V, kb: V) void { - const b = c.b; - const q_row = c.g.q.add(t.to(.i64).mul(H * D * 2)); - const q_smem = c.s.add(b.select(kb.eq(0), b.cst(.i32, smem.q[0]), b.cst(.i32, smem.q[1]))); - for (0..32) |it| { - const chunk = c.lane.add(@as(i32, @intCast(it * 32))); - const row = chunk.shrLogical(6); - const col = chunk.bitAnd(63).mul(8); - b.cpAsync16(q_smem.add(sw128Offset(b, row, col, H)), q_row.add(row.mul(D).add(col).mul(2).to(.i64)), null); - } -} - -fn frontRole(c0: Ctx, first: FrontRegs) void { - const b = c0.b; - frontStore(c0, c0.cta, b.cst(.i32, 0), first); - var loop = b.openFor(0, c0.ntok, 1, .{}); - const k = loop.iv; - const c = forToken(c0, k, undefined, undefined, false); - { - // Q of token k (token 0: issued at start) once every QK of token k - 2 has completed. - var later = b.openIf(k.gt(0)); - { - var reuse = b.openIf(k.ge(2)); - b.mbarrierWait(bar(b, c.s, .q_free, c.kb), parity(k).bitXor(1)); - reuse.yieldThen(.{}); - } - issueQ(c, c.t, c.kb); - later.yieldThen(.{}); - } - // The MMA reads Q through the async proxy. - b.cpAsyncWaitAll(); - b.fenceProxyAsyncShared(); - b.mbarrierArrive(bar(b, c.s, .q_ready, c.kb), 1); - { - var more = b.openIf(k.add(1).lt(c.ntok)); - const next_t = c.t.add(@as(i32, @intCast(c.cfg.gridCtas()))); - // Warm L2 with the next token's Q (16 KiB: 4 lines per lane). - const q_next = c.g.q.add(next_t.to(.i64).mul(H * D * 2)); - for (0..4) |i| b.prefetchL2EvictLast(q_next.add(c.lane.add(@as(i32, @intCast(i * 32))).mul(128).to(.i64))); - const r = frontLoad(c, next_t); - const nkb = k.add(1).bitAnd(1); - { - // The buffer's previous token (k - 1) must be released by every consumer. - var reuse = b.openIf(k.ge(1)); - b.mbarrierWait(bar(b, c.s, .idx_free, nkb), parity(k.add(1)).bitXor(1)); - reuse.yieldThen(.{}); - } - frontStore(c, next_t, nkb, r); - more.yieldThen(.{}); - } - loop.yield(.{}); -} - -// ---- loaders (warps 5-6) ------------------------------------------------------------ - -fn loaderRole(c0: Ctx) void { - const b = c0.b; - var loop = b.openFor(0, c0.ntok, 1, .{b.cst(.i32, 0)}); - const c = forToken(c0, loop.iv, undefined, loop.carried[0], true); - { - var blocks = b.openFor(c.b0.minimum(c.nwb), c.b1.minimum(c.nwb), 1, .{}); - loadBlock(c, .fp8, blocks.iv.sub(c.b0), blocks.iv); - blocks.yield(.{}); - } - if (c.cfg.compressed_capacity > 0) { - var blocks = b.openFor(c.b0.maximum(c.nwb), c.b1, 1, .{}); - loadBlock(c, .fp4, blocks.iv.sub(c.b0), blocks.iv.sub(c.nwb)); - blocks.yield(.{}); - } - releaseIndices(c); - loop.yield(.{slotUse(c, c.n)}); -} - -fn idxBase(comptime f: Format) i32 { - return if (f == .fp8) 0 else 128 * 4; -} - -fn loadBlock(c: Ctx, comptime f: Format, j: V, local: V) void { - const b = c.b; - const use_a = slotUse(c, j); - const use_b = use_a.add(1); // second slot of an FP8 block - const slot_a = use_a.rem(rawSlots(c.cfg)); - const slot_b = use_b.rem(rawSlots(c.cfg)); - for (0..@as(usize, if (f == .fp8) 2 else 1)) |k| { - const use = if (k == 0) use_a else use_b; - var reuse = b.openIf(use.ge(rawSlots(c.cfg))); - b.mbarrierWait(bar(b, c.s, .raw_empty, use.rem(rawSlots(c.cfg))), slotParity(c, use).bitXor(1)); - reuse.yieldThen(.{}); - } - const values = if (f == .fp8) c.g.wv else c.g.cv; - const scales = if (f == .fp8) c.g.ws else c.g.cs; - // FP8 rows are split across two slots: lanes 0-15 copy bytes 0-255 into slot a, - // lanes 16-31 bytes 256-511 into slot b (both with the 288-byte FP4 row stride). - const raw = if (f == .fp8) b.select(c.lane.lt(16), slotAddr(c, use_a), slotAddr(c, use_b)) else slotAddr(c, use_a); - const sc = scaleAddr(c, if (f == .fp8) use_b else use_a); - const idx = c.ib.add(idxBase(f)).add(local.mul(BK * 4)); - // Row data in 16-byte chunks. FP4 (16 chunks per row): lanes 0-15 copy rows - // 0-31 and lanes 16-31 rows 32-63, two rows per warp instruction. FP8 (32 - // chunks per row): one row per instruction. Invalid rows (-1) are zero-filled. - const chunks_per_row = @divExact(f.rowBytes(), 16); - const chunk16 = c.lane.bitAnd(chunks_per_row - 1).mul(16); - const chunk_off = chunk16.to(.i64); - const dst_chunk = c.lane.bitAnd(15).mul(16); // within a 288-byte slot row - // Two loader warps: warp 5 takes the first half of this lane's rows, warp 6 the second. - // Four rows per iteration: unrolled for FP4 (4 iterations), a runtime loop for FP8 (small code). - const half_rows: i32 = comptime if (f == .fp4) 16 else 32; - const second = c.warp.eq(6).to(.i32).bitAnd(1); - const first_row = (if (f == .fp4) c.lane.shrLogical(4).mul(32) else b.cst(.i32, 0)).add(second.mul(half_rows)); - const src_base = values.add(chunk_off); - const dst_base = raw.add(first_row.mul(288)).add(dst_chunk); - const idx_base = idx.add(first_row.mul(4)); - const copyRows = struct { - fn f_(bb: *B, r: V, dst_base_: V, src_base_: V, idx_base_: V) void { - const ixs = ldSharedBatch(bb, 1, 4, idx_base_.add(r.mul(4)), .{0}); - for (0..4) |k| { - cpAsync16Row(bb, dst_base_.add(r.add(@as(i32, @intCast(k))).mul(288)), src_base_, ixs[k], @intCast(f.rowBytes()), bb.cst(.i64, 0)); - } - } - }.f_; - if (f == .fp4) { - for (0..half_rows / 4) |i| copyRows(b, b.cst(.i32, @as(i32, @intCast(4 * i))), dst_base, src_base, idx_base); - } else { - var rows = b.openFor(0, b.opaqueI32(half_rows), 4, .{}); - copyRows(b, rows.iv, dst_base, src_base, idx_base); - rows.yield(.{}); - } - // Scales: 16 (FP8) or 32 (FP4) bytes per row, one 16-byte chunk per lane and step. - const scale_chunks = comptime @divExact(f.scaleBytes(), 16); - const scale_steps = comptime @divExact(BK * scale_chunks, 32); - const sidx_offsets = comptime blk: { - var o: [scale_steps]i32 = undefined; - for (0..scale_steps) |k| o[k] = @as(i32, @intCast(k)) * @divExact(32, scale_chunks) * 4; - break :blk o; - }; - const scale_row0 = c.lane.shrLogical(std.math.log2_int(u32, @intCast(scale_chunks))); - const part16 = c.lane.bitAnd(scale_chunks - 1).mul(16); - const srow = ldSharedBatch(b, scale_steps, 1, idx.add(scale_row0.mul(4)), sidx_offsets); - const second_warp = c.warp.eq(6); - for (0..@intCast(scale_steps)) |k| { - // Warp 5 copies the even scale steps, warp 6 the odd ones. - var mine = b.openIf(if (k % 2 == 1) second_warp else c.warp.eq(5)); - const row = scale_row0.add(@as(i32, @intCast(k)) * @divExact(32, scale_chunks)); - cpAsync16Row(b, sc.add(row.mul(288)).add(part16), scales, srow[k], @intCast(f.scaleBytes()), part16.to(.i64)); - mine.yieldThen(.{}); - } - b.cpAsyncMbarrierArrive(bar(b, c.s, .raw_full, slot_a), true); - if (f == .fp8) b.cpAsyncMbarrierArrive(bar(b, c.s, .raw_full, slot_b), true); -} - -// ---- dequant (warps 8-11) ------------------------------------------------------------ - -fn dequantRole(c0: Ctx) void { - const b = c0.b; - const tid = c0.tid.sub(256); - var loop = b.openFor(0, c0.ntok, 1, .{ b.cst(.i32, 0), b.cst(.i32, 0) }); - const c = forToken(c0, loop.iv, loop.carried[0], loop.carried[1], true); - releaseIndices(c); - { - var blocks = b.openFor(0, c.nwl, 1, .{}); - dequantBlock(c, .fp8, blocks.iv, tid); - blocks.yield(.{}); - } - if (c.cfg.compressed_capacity > 0) { - var blocks = b.openFor(c.nwl, c.n, 1, .{}); - dequantBlock(c, .fp4, blocks.iv, tid); - blocks.yield(.{}); - } - loop.yield(.{ c.g0.add(c.n), slotUse(c, c.n) }); -} - -/// Thread (g = tid / 8, i = tid % 8) converts 8 consecutive values per step: -/// rows g + 16 * r (r < 4), columns 64 * step + 8 * i. The 8 threads of a row -/// write one 128-byte swizzle row, so each STS.128 is conflict free. The tile is -/// produced in four parts of 128 columns (steps 2p, 2p + 1): an FP8 block's first -/// raw slot (columns 0-255) is released after part 1, so the next block's copies -/// can start while parts 2-3 are converted. -fn dequantBlock(c: Ctx, comptime f: Format, j: V, tid: V) void { - const b = c.b; - const gj = c.g0.add(j); - const buf = gj.bitAnd(1); - const par = parity(gj); - const use_a = slotUse(c, j); - const use_b = use_a.add(1); - b.mbarrierWait(bar(b, c.s, .raw_full, use_a.rem(rawSlots(c.cfg))), slotParity(c, use_a)); - if (f == .fp8) b.mbarrierWait(bar(b, c.s, .raw_full, use_b.rem(rawSlots(c.cfg))), slotParity(c, use_b)); - { - var reuse = b.openIf(gj.ge(2)); - b.mbarrierWait(bar(b, c.s, .pv_done, buf), par.bitXor(1)); - reuse.yieldThen(.{}); - } - const g8 = tid.shrLogical(3); - const r8 = g8.bitAnd(7); - const in8 = tid.bitAnd(7); - const swz = in8.bitXor(r8).shl(4); - const raw = slotAddr(c, use_a).add(g8.mul(288)); - const raw_b = slotAddr(c, use_b).add(g8.mul(288)); - // FP8 scales follow the second slot's data (the first slot is released early). - const sc = scaleAddr(c, if (f == .fp8) use_b else use_a).add(g8.mul(288)); - const tile = c.s.add(b.select(buf.eq(0), b.cst(.i32, smem.tile[0]), b.cst(.i32, smem.tile[1]))); - const dst = tile.add(g8.shrLogical(3).mul(1024)).add(r8.mul(128)).add(swz); - // Raw words of part p for the thread's 4 rows (row stride 16 * 288 bytes): - // FP8: 2 x 2 words per row, then one scale word per row; - // FP4: 2 words per row, then 2 scale words per row. - const nw = if (f == .fp8) 20 else 16; - const loadPart = struct { - fn f_(cc: Ctx, comptime ff: Format, raw_: V, raw_b_: V, sc_: V, in8_: V, p: V) [nw]V { - const bb = cc.b; - var out: [nw]V = undefined; - if (ff == .fp8) { - // Steps 2p, 2p + 1: bytes [128 (p & 1), + 128) of the first (p < 2) or second slot. - const src = bb.select(p.lt(2), raw_, raw_b_).add(in8_.mul(8)).add(p.bitAnd(1).mul(128)); - const d = ldSharedBatch(bb, 8, 2, src, .{ 0, 64, 4608, 4672, 9216, 9280, 13824, 13888 }); - for (0..16) |i| out[i] = d[i]; - const sw = ldSharedBatch(bb, 4, 1, sc_.add(p.mul(4)), .{ 0, 4608, 9216, 13824 }); - for (0..4) |i| out[16 + i] = sw[i]; - } else { - const d = ldSharedBatch(bb, 8, 1, raw_.add(in8_.mul(4)).add(p.mul(64)), .{ 0, 32, 4608, 4640, 9216, 9248, 13824, 13856 }); - for (0..8) |i| out[i] = d[i]; - const sw = ldSharedBatch(bb, 4, 2, sc_.add(p.mul(8)), .{ 0, 4608, 9216, 13824 }); - for (0..8) |i| out[8 + i] = sw[i]; - } - return out; - } - }.f_; - const first = loadPart(c, f, raw, raw_b, sc, in8, b.cst(.i32, 0)); - if (f == .fp4) { - // Unrolled: the four parts of an FP4 block are short. - var cur = first; - for (0..4) |pi| { - const p = b.cst(.i32, @as(i32, @intCast(pi))); - const next = if (pi < 3) loadPart(c, f, raw, raw_b, sc, in8, p.add(1)) else cur; - dequantPart(c, f, cur, dst, in8, p); - cur = next; - } - } else { - var parts = b.openFor(0, b.opaqueI32(4), 1, tupleOf(nw, first)); - const p = parts.iv; - var cur: [nw]V = undefined; - for (0..nw) |i| cur[i] = parts.carried[i]; - // The last iteration re-loads its own part (harmless) to keep the loop uniform. - const next = loadPart(c, f, raw, raw_b, sc, in8, p.add(1).minimum(3)); - dequantPart(c, f, cur, dst, in8, p); - // Parts 0-1 were the first slot's last reads (part 2's loads read the second slot). - var first_done = b.openIf(p.eq(1)); - b.mbarrierArrive(bar(b, c.s, .raw_empty, use_a.rem(rawSlots(c.cfg))), 1); - first_done.yieldThen(.{}); - parts.yield(tupleOf(nw, next)); - } - b.fenceProxyAsyncShared(); - b.mbarrierArrive(bar(b, c.s, .tile_full, buf), 1); - b.mbarrierArrive(bar(b, c.s, .raw_empty, (if (f == .fp8) use_b else use_a).rem(rawSlots(c.cfg))), 1); -} - -/// Converts part `p` (raw words `cur`, see `dequantBlock`) into the bf16 tile at `dst`. -fn dequantPart(c: Ctx, comptime f: Format, cur: [if (f == .fp8) 20 else 16]V, dst: V, in8: V, p: V) void { - const b = c.b; - const dst_p = dst.add(p.mul(2 * BK * 128)); - // Invalid rows arrive zero-filled (raw bytes and scales), so they dequantize to 0. - for (0..4) |r| { - for (0..2) |k| { - const out = if (f == .fp8) blk: { - // Scale index 2 * step + (i >= 4) = 4 p + 2 k + (i >> 2): byte 2 k + (i >> 2) of word p. - const e = cur[16 + r].shrLogical(in8.shrLogical(2).add(@as(i32, @intCast(2 * k))).mul(8)).bitAnd(0xFF); - const scale = e.shl(7).bitOr(e.shl(23)); - break :blk fp8x8ToBf16x2x4Scaled(b, cur[r * 4 + k * 2], cur[r * 4 + k * 2 + 1], scale); - } else blk: { - // Scale index 4 * step + i / 2: byte i / 2 of the step's word. - const e = cur[8 + r * 2 + k].shrLogical(in8.shrLogical(1).mul(8)).bitAnd(0xFF); - const scale = b.e4m3x2ToBf16x2(e.bitOr(e.shl(8))); - break :blk fp4x8ToBf16x2x4Scaled(b, cur[r * 2 + k], scale); - }; - b.storeWords(4, dst_p.add(@as(i32, @intCast(r * 2048 + k * BK * 128))), .shared, out); - } - } -} - -// ---- MMA (warp 4) ------------------------------------------------------------------------ - -fn mmaRole(c0: Ctx) void { - const b = c0.b; - var leader = b.openIf(b.electSync()); - // Base descriptors; an MMA adds a constant (byte offset >> 4) to the start address field. - const q_descs = [2]V{ smemDescSw128(b, c0.s.add(smem.q[0]), 16, 1024), smemDescSw128(b, c0.s.add(smem.q[1]), 16, 1024) }; - const k_desc = [2]V{ smemDescSw128(b, c0.s.add(smem.tile[0]), 16, 1024), smemDescSw128(b, c0.s.add(smem.tile[1]), 16, 1024) }; - const v_desc = [2]V{ smemDescSw128(b, c0.s.add(smem.tile[0]), 8192, 1024), smemDescSw128(b, c0.s.add(smem.tile[1]), 8192, 1024) }; - const p_desc = [2]V{ smemDescSw128(b, c0.s.add(smem.p[0]), 16, 1024), smemDescSw128(b, c0.s.add(smem.p[1]), 16, 1024) }; - const qk_idesc = b.cst(.i32, @as(i32, @bitCast(instrDescBf16(64, H, .k, .k)))); - const pv_idesc = b.cst(.i32, @as(i32, @bitCast(instrDescBf16(128, H, .mn, .k)))); - const i32t = cute.DType.i32.toMlir(b.ctx); - - var tokens = b.openFor(0, c0.ntok, 1, .{b.cst(.i32, 0)}); - const c = forToken(c0, tokens.iv, tokens.carried[0], undefined, true); - // The MMA thread only needs the block count; the elected lane releases for the warp. - b.mbarrierArrive(bar(b, c.s, .idx_free, c.kb), 1); - b.mbarrierWait(bar(b, c.s, .q_ready, c.kb), parity(c.k)); - const q_desc = b.select(c.kb.eq(0), q_descs[0], q_descs[1]); - { - // O^T buffer kb: its previous token's epilogue must have read it. - var reuse = b.openIf(c.k.ge(2)); - b.mbarrierWait(bar(b, c.s, .o_free, c.kb), parity(c.k).bitXor(1)); - reuse.yieldThen(.{}); - } - b.tcgen05Fence(.after); - { - // No QK: Q is free right away. - var none = b.openIf(c.n.eq(0)); - b.tcgen05Commit(bar(b, c.s, .q_free, c.kb)); - none.yieldThen(.{}); - } - const o_taddr = c.taddr.add(c.kb.mul(4 * H)); - // Iteration i issues QK(i + 1) and PV(i) in whichever order their inputs - // become ready; i = -1 only issues QK(0). - var loop = b.openFor(-1, c.n, 1, .{}); - const i = loop.iv; - const next = i.add(1); - var poll = b.openWhile(.{ next.lt(c.n).to(.i32).bitAnd(1), i.ge(0).to(.i32).bitAnd(1) }, .{ i32t, i32t }); - poll.yieldBefore(poll.before_carried[0].bitOr(poll.before_carried[1]).ne(0), .{ poll.before_carried[0], poll.before_carried[1] }); - const qk_pending = poll.after_carried[0]; - const pv_pending = poll.after_carried[1]; - const gn = c.g0.add(next); - const nbuf = gn.bitAnd(1); - const qk_go = qk_pending.ne(0).bitAnd(b.mbarrierTestParity(bar(b, c.s, .tile_full, nbuf), parity(gn))); - { - var go = b.openIf(qk_go); - b.tcgen05Fence(.after); - const kd = b.select(nbuf.eq(0), k_desc[0], k_desc[1]); - const d = c.taddr.add(tmem_s).add(nbuf.mul(qk_parts * H)); - // K step s of part p covers latent dims 128 p + 16 s: consecutive MMAs go to - // different accumulators. - for (0..8) |step| { - for (0..qk_parts) |part| { - const ks = part * 8 + step; - const kb: i64 = @intCast(ks / 4); - const kin: i64 = @intCast(ks % 4); - const adesc = kd.add(@divExact(kb * 8192 + kin * 32, 16)); - const bdesc = q_desc.add(@divExact(kb * 2048 + kin * 32, 16)); - b.tcgen05MmaF16(d.add(@as(i32, @intCast(part * H))), adesc, bdesc, qk_idesc, b.cst(.i32, @intFromBool(step > 0)).ne(0)); - } - } - b.tcgen05Commit(bar(b, c.s, .s_full, nbuf)); - { - // The token's last QK: Q may be replaced once it completes. - var last = b.openIf(next.eq(c.n.sub(1))); - b.tcgen05Commit(bar(b, c.s, .q_free, c.kb)); - last.yieldThen(.{}); - } - go.yieldThen(.{}); - } - const gi = c.g0.add(i); - const buf = gi.bitAnd(1); - const pv_go = pv_pending.ne(0).bitAnd(b.mbarrierTestParity(bar(b, c.s, .p_full, buf), parity(gi))); - { - var go = b.openIf(pv_go); - b.tcgen05Fence(.after); - const vd = b.select(buf.eq(0), v_desc[0], v_desc[1]); - const pd = b.select(buf.eq(0), p_desc[0], p_desc[1]); - const first = b.select(i.eq(0), b.cst(.i32, 0), b.cst(.i32, 1)); - for (0..4) |ks| { - for (0..4) |chunk| { - const adesc = vd.add(@as(i64, @intCast(@divExact(2 * chunk * 8192 + ks * 2048, 16)))); - const bdesc = pd.add(@as(i64, @intCast(@divExact(ks * 32, 16)))); - const acc = if (ks > 0) b.cst(.i32, 1) else first; - b.tcgen05MmaF16(o_taddr.add(@as(i32, @intCast(chunk * H))), adesc, bdesc, pv_idesc, acc.ne(0)); - } - } - b.tcgen05Commit(bar(b, c.s, .pv_done, buf)); - go.yieldThen(.{}); - } - poll.yieldAfter(.{ b.select(qk_go, b.cst(.i32, 0), qk_pending), b.select(pv_go, b.cst(.i32, 0), pv_pending) }); - loop.yield(.{}); - tokens.yield(.{c.g0.add(c.n)}); - leader.yieldThen(.{}); -} - -// ---- softmax / epilogue (warps 0-3) ------------------------------------------------------ - -fn softmaxRole(c0: Ctx) void { - const b = c0.b; - const scale_log2: f32 = @floatCast(c0.cfg.scale * log2e); - const lane_tmem = c0.taddr.add(c0.warp.mul(32).shl(16)); - const row = c0.warp.mul(16).add(c0.lane); // S^T row held by this lane (lanes 0-15) - const has_row = c0.lane.lt(16); - - { - // Prefetch the sink logits for the epilogue. - var h16 = b.openIf(c0.tid.lt(H)); - b.storeWords(1, c0.s.add(smem.sink).add(c0.tid.mul(4)), .shared, .{b.loadWords(1, c0.g.sink.add(c0.tid.mul(4).to(.i64)), .global, .{ .invariant = true })[0].bitCast(.f32).bitCast(.i32)}); - h16.yieldThen(.{}); - } - var tokens = b.openFor(0, c0.ntok, 1, .{b.cst(.i32, 0)}); - const c = forToken(c0, tokens.iv, tokens.carried[0], undefined, true); - const o_tmem = lane_tmem.add(c.kb.mul(4 * H)); - var init: [2 * H]V = undefined; - for (0..H) |h| { - // The running max only guards exp2 against overflow: it need not be exact, - // and starting at 0 lets typical first blocks skip the cross-warp max. Logits - // far below it just give small P (BF16 / F32 range), as with a stale max. - init[h] = b.cst(.f32, 0); - init[H + h] = b.cst(.f32, 0); - } - var loop = b.openFor(0, c.n, 1, tupleOf(2 * H, init)); - const j = loop.iv; // local block; token block is b0 + j - var m: [H]V = undefined; - var l: [H]V = undefined; - for (0..H) |h| { - m[h] = loop.carried[h]; - l[h] = loop.carried[H + h]; - } - const gj = c.g0.add(j); - const buf = gj.bitAnd(1); - const par = parity(gj); - b.mbarrierWait(bar(b, c.s, .s_full, buf), par); - b.tcgen05Fence(.after); - const sparts = b.tmemLoad32x32b(qk_parts * H, lane_tmem.add(tmem_s).add(buf.mul(qk_parts * H))); - b.fenceTmemLoad(); - var sv: [H]V = undefined; - for (0..H) |h| { - var acc = sparts[h].bitCast(.f32); - for (1..qk_parts) |p_| acc = acc.add(sparts[p_ * H + h].bitCast(.f32)); - sv[h] = acc.bitCast(.i32); - } - // Candidate index of this row: window blocks first, then compressed blocks. - const tb = c.b0.add(j); - const is_window = tb.lt(c.nwb); - const cand = b.select(is_window, b.cst(.i32, idxBase(.fp8)).add(tb.mul(BK * 4)), b.cst(.i32, idxBase(.fp4)).add(tb.sub(c.nwb).mul(BK * 4))); - const row_ix = b.loadWords(1, c.ib.add(cand).add(row.bitAnd(BK - 1).mul(4)), .shared, .{})[0]; - const valid = has_row.bitAnd(row_ix.ge(0)); - - var sc: [H]V = undefined; - var exceed = b.cst(.i32, 0); - for (0..H) |h| { - sc[h] = b.select(valid, sv[h].bitCast(.f32).mul(scale_log2), b.cst(.f32, -std.math.inf(f32))); - exceed = exceed.bitOr(sc[h].gt(m[h].add(rescale_threshold)).to(.i32)); - } - // Usually no score exceeds the running max by the threshold: one barrier vote - // then skips the block max entirely (m stays, alpha = 1). - const f32t = cute.DType.f32.toMlir(b.ctx); - const i32t = cute.DType.i32.toMlir(b.ctx); - var upd = b.openIfElse(b.barrierReduceOr(1, 128, exceed.ne(0)), .{f32t} ** (2 * H) ++ .{i32t}); - { - // Block max per head: warp redux, then across the 4 warps through shared memory. - const red = c.s.add(b.select(buf.eq(0), b.cst(.i32, smem.red[0]), b.cst(.i32, smem.red[1]))); - for (0..H) |h| { - const wm = b.warpMaxF32(sc[h]); - var l0 = b.openIf(c.lane.eq(0)); - b.storeWords(1, red.add(c.warp.mul(H * 4)).add(@as(i32, @intCast(h * 4))), .shared, .{wm.bitCast(.i32)}); - l0.yieldThen(.{}); - } - b.barrierSync(1, 128); - const bms = blockReduce(c, red, .max); - var out: [2 * H + 1]V = undefined; - var changed = b.cst(.i32, 0); - for (0..H) |h| { - const bm = bms[h]; - const move = bm.gt(m[h].add(rescale_threshold)); - out[h] = b.select(move, bm, m[h]); - out[H + h] = b.exp2Approx(m[h].sub(out[h])); - changed = changed.bitOr(move.to(.i32)); - } - out[2 * H] = changed; - upd.yieldThen(tupleOf(2 * H + 1, out)); - } - { - var out: [2 * H + 1]V = undefined; - for (0..H) |h| { - out[h] = m[h]; - out[H + h] = b.cst(.f32, 1); - } - out[2 * H] = b.cst(.i32, 0); - upd.yieldElse(tupleOf(2 * H + 1, out)); - } - var new_m: [H]V = undefined; - var alpha: [H]V = undefined; - for (0..H) |h| { - new_m[h] = upd.results[h]; - alpha[h] = upd.results[H + h]; - } - const changed = upd.results[2 * H]; - // P^T [16 heads, 64 keys] BF16, SW128 K-major (one 128-byte row per head). - const p = c.s.add(b.select(buf.eq(0), b.cst(.i32, smem.p[0]), b.cst(.i32, smem.p[1]))); - var next_l: [H]V = undefined; - for (0..H) |h| { - const pr = b.select(valid, b.exp2Approx(sc[h].sub(new_m[h])), b.cst(.f32, 0)); - next_l[h] = l[h].mul(alpha[h]).add(pr); - const off = row.shrLogical(3).bitXor(@as(i32, @intCast(h & 7))).shl(4).add(row.bitAnd(7).shl(1)); - stSharedBf16If(b, p.add(@as(i32, @intCast(h * 128))).add(off), pr, has_row); - } - // Rescale O^T (TMEM) once the previous PV has landed. - { - var rescale = b.openIf(j.gt(0).bitAnd(changed.ne(0))); - const prev = gj.sub(1); - b.mbarrierWait(bar(b, c.s, .pv_done, prev.bitAnd(1)), parity(prev)); - b.tcgen05Fence(.after); - for (0..4) |chunk| { - const addr = o_tmem.add(@as(i32, @intCast(chunk * H))); - const o = b.tmemLoad32x32b(H, addr); - b.fenceTmemLoad(); - var scaled: [H]V = undefined; - for (0..H) |h| scaled[h] = o[h].bitCast(.f32).mul(alpha[h]).bitCast(.i32); - b.tmemStore32x32b(H, addr, scaled); - } - b.fenceTmemStore(); - rescale.yieldThen(.{}); - } - b.fenceProxyAsyncShared(); - b.tcgen05Fence(.before); - b.mbarrierArrive(bar(b, c.s, .p_full, buf), 1); - var carry: [2 * H]V = undefined; - for (0..H) |h| { - carry[h] = new_m[h]; - carry[H + h] = next_l[h]; - } - loop.yield(tupleOf(2 * H, carry)); - releaseIndices(c); - epilogue(c, o_tmem, loop.results[0..H].*, loop.results[H .. 2 * H].*); - tokens.yield(.{c.g0.add(c.n)}); -} - -/// Per head, the reduction over the 4 softmax warps of `buf` [4 warps][16 heads] f32. -fn blockReduce(c: Ctx, buf: V, comptime op: ReduceOp) [H]V { - const b = c.b; - var r: [H]V = undefined; - for (0..4) |w| { - for (0..4) |q4| { - const v = b.loadWords(4, buf.add(@as(i32, @intCast((w * H + q4 * 4) * 4))), .shared, .{}); - for (0..4) |i| { - const x = v[i].bitCast(.f32); - const h = q4 * 4 + i; - r[h] = if (w == 0) x else if (op == .max) r[h].maxnum(x) else r[h].add(x); - } - } - } - return r; -} - -/// O^T / (l + exp(sink - m)) of the token (or this CTA's partial result, for a cluster merge). -fn epilogue(c: Ctx, o_tmem: V, m: [H]V, l: [H]V) void { - const b = c.b; - const lane_tmem = c.taddr.add(c.warp.mul(32).shl(16)); - // The sums overlap the token's last PV. - const red_sum = c.s.add(b.select(c.kb.eq(0), b.cst(.i32, smem.red_sum[0]), b.cst(.i32, smem.red_sum[1]))); - { - // Lanes 16-31 hold no key rows (l = 0); lane h (< 16) gets head h's sum. - for (0..H) |h| { - const lsum = warpSumF32(b, l[h]); - var l0 = b.openIf(c.lane.eq(0)); - b.storeWords(1, red_sum.add(c.warp.mul(H * 4)).add(@as(i32, @intCast(h * 4))), .shared, .{lsum.bitCast(.i32)}); - l0.yieldThen(.{}); - } - } - b.barrierSync(1, 128); - const total = blockReduce(c, red_sum, .sum); - const has_blocks = c.n.gt(0); - { - var any = b.openIf(has_blocks); - const last = c.g0.add(c.n).sub(1); - b.mbarrierWait(bar(b, c.s, .pv_done, last.bitAnd(1)), parity(last)); - any.yieldThen(.{}); - } - b.tcgen05Fence(.after); - if (c.cfg.cluster > 1) { - // Partial result for the cluster merge: (m, l) per head and unnormalized O^T. - { - var t0 = b.openIf(c.tid.eq(0)); - for (0..H) |h| { - b.storeWords(1, c.s.add(smem.stats + @as(i32, @intCast(h * 4))), .shared, .{m[h].bitCast(.i32)}); - b.storeWords(1, c.s.add(smem.stats + @as(i32, @intCast((H + h) * 4))), .shared, .{total[h].bitCast(.i32)}); - } - t0.yieldThen(.{}); - } - // Normalized by this CTA's own sum (a convex combination of cache rows: bounded, - // so it travels as FP16); the owner re-weights with exp2(m - M) * l / L. - var inv_l: [H]V = undefined; - for (0..H) |h| inv_l[h] = b.select(total[h].gt(0).bitAnd(has_blocks), b.rcpApprox(total[h]), b.cst(.f32, 0)); - for (0..4) |chunk| { - const o = b.tmemLoad32x32b(H, o_tmem.add(@as(i32, @intCast(chunk * H)))); - b.fenceTmemLoad(); - const dim = c.warp.mul(32).add(c.lane).add(@as(i32, @intCast(chunk * 128))); - const dst = c.s.add(smem.obuf).add(dim.mul(H * 4)); - for (0..4) |q4| { - var v: [4]V = undefined; - // A rank without key blocks never wrote its O^T (TMEM holds stale data, possibly - // NaN/Inf): select, do not multiply by 0. - for (0..4) |i| v[i] = b.select(has_blocks, o[q4 * 4 + i].bitCast(.f32).mul(inv_l[q4 * 4 + i]), b.cst(.f32, 0)).bitCast(.i32); - b.storeWords(4, dst.add(@as(i32, @intCast(q4 * 16))), .shared, v); - } - } - _ = lane_tmem; - } else { - var inv: [H]V = undefined; - for (0..H) |h| { - const sink = b.loadWords(1, c.s.add(smem.sink + @as(i32, @intCast(h * 4))), .shared, .{})[0].bitCast(.f32); - const denom = total[h].add(b.exp2Approx(sink.mul(@as(f32, log2e)).sub(m[h]))); - // denom underflows only if every logit and the sink are ~2^126 below the running max. - inv[h] = b.select(has_blocks.bitAnd(denom.gt(0)), b.rcpApprox(denom), b.cst(.f32, 0)); - } - // The CTA's last token is staged in shared memory (Q is no longer needed) and - // written with 16-byte stores; earlier tokens store straight to global memory - // (a warp writes 64 contiguous bytes per head). - const last_token = c.k.eq(c.ntok.sub(1)); - const out_row = c.g.out.add(c.t.to(.i64).mul(H * D * 2)); - { - var direct = b.openIf(last_token.eq(false)); - var os: [4][H]V = undefined; - for (0..4) |chunk| os[chunk] = b.tmemLoad32x32b(H, o_tmem.add(@as(i32, @intCast(chunk * H)))); - b.fenceTmemLoad(); - // Lane pairs (d, d + 1) swap half their heads: the even lane stores heads 0-7, - // the odd lane heads 8-15, each as bf16 pairs of dims (d, d + 1). - const odd = c.lane.bitAnd(1).ne(0); - const d_even = c.warp.mul(32).add(c.lane.bitAnd(30)); - const h0 = b.select(odd, b.cst(.i32, 8), b.cst(.i32, 0)); - const base = out_row.add(h0.mul(D).add(d_even).mul(2).to(.i64)); - for (0..4) |chunk| { - for (0..8) |i| { - const lo = b.select(has_blocks, os[chunk][i].bitCast(.f32).mul(inv[i]), b.cst(.f32, 0)); - const hi = b.select(has_blocks, os[chunk][8 + i].bitCast(.f32).mul(inv[8 + i]), b.cst(.f32, 0)); - const send = b.select(odd, lo, hi); - const keep = b.select(odd, hi, lo); - const recv = b.shuffleXor(send, 1).bitCast(.f32); - // Even lane: (own dim, partner's dim); odd lane: (partner's dim, own dim). - const pair = b.packHalf2(b.select(odd, recv, keep), b.select(odd, keep, recv), .bf16); - b.storeWords(1, base.add(@as(i64, @intCast((i * D + chunk * 128) * 2))), .global, .{pair}); - } - } - direct.yieldThen(.{}); - } - { - var staged = b.openIf(last_token); - // The staging buffer is Q buffer 0: a token without key blocks never waited for - // its Q copy (cp.async into that buffer), which must land before it is overwritten. - b.mbarrierWait(bar(b, c.s, .q_ready, c.kb), parity(c.k)); - var os: [4][H]V = undefined; - for (0..4) |chunk| os[chunk] = b.tmemLoad32x32b(H, o_tmem.add(@as(i32, @intCast(chunk * H)))); - b.fenceTmemLoad(); - for (0..4) |chunk| { - const o = os[chunk]; - const dim = c.warp.mul(32).add(c.lane).add(@as(i32, @intCast(chunk * 128))); - for (0..H) |h| { - const v = b.select(has_blocks, o[h].bitCast(.f32).mul(inv[h]), b.cst(.f32, 0)); - b.storeHalf(c.s.add(smem.ostage).add(dim.add(@as(i32, @intCast(h * D))).mul(2)), .shared, b.f32ToBf16Bits(v)); - } - } - staged.yieldThen(.{}); - } - // O^T buffer kb may take the token after next. - b.tcgen05Fence(.before); - b.mbarrierArrive(bar(b, c.s, .o_free, c.kb), 1); - { - var staged = b.openIf(last_token); - b.barrierSync(1, 128); - for (0..8) |it| { - const off = c.tid.add(@as(i32, @intCast(it * 128))).mul(16); - b.storeWords(4, out_row.add(off.to(.i64)), .global, b.loadWords(4, c.s.add(smem.ostage).add(off), .shared, .{})); - } - staged.yieldThen(.{}); - } - } -} - -// ---- cluster merge (all 384 threads, cluster > 1) ---------------------------------------------- - -/// Combine the cluster's partial results. Rank r owns output dims -/// [r * 512 / C, (r + 1) * 512 / C): after every rank has finished its blocks, -/// each pushes its (m, l) and the owners' slices of its O^T into their shared -/// memory (remote stores do not stall), then every owner combines locally. -/// No global partials and no second kernel. -fn clusterMerge(c: Ctx) void { - const b = c.b; - const C: i32 = @intCast(c.cfg.cluster); - const dims = @divExact(D, C); - // Every rank is done with its tiles (the inbox reuses tile 0). - b.clusterSync(); - { - // (m, l) of head h to every rank's inbox, slot `rank`. - var heads_ = b.openIf(c.tid.lt(2 * H)); - const v = b.loadWords(1, c.s.add(smem.stats).add(c.tid.mul(4)), .shared, .{})[0].bitCast(.f32); - const dst = c.s.add(smem.stats_in).add(c.rank.mul(2 * H * 4)).add(c.tid.mul(4)); - for (0..@intCast(C)) |p| b.storeWordsPtr(1, b.mapaShared(dst, b.cst(.i32, @as(i32, @intCast(p)))), .{v.bitCast(.i32)}); - heads_.yieldThen(.{}); - } - { - // O^T rows of peer p's dims to its inbox slot `rank`: 16-byte chunks. - const chunks = D * @divExact(H, 4); - var loop = b.openFor(c.tid, chunks, threads, .{}); - const chunk = loop.iv; - const d = chunk.shrLogical(2); - const q4 = chunk.bitAnd(3); - const v = b.loadWords(4, c.s.add(smem.obuf).add(d.mul(H * 4)).add(q4.mul(16)), .shared, .{}); - const owner = d.div(dims); - // Normalized partials travel as FP16. - const packed_ = [2]V{ b.packHalf2(v[0].bitCast(.f32), v[1].bitCast(.f32), .f16), b.packHalf2(v[2].bitCast(.f32), v[3].bitCast(.f32), .f16) }; - const dst = c.s.add(smem.inbox).add(c.rank.mul(dims * H * 2)).add(d.rem(dims).mul(H * 2)).add(q4.mul(8)); - b.storeWordsPtr(2, b.mapaShared(dst, owner), packed_); - loop.yield(.{}); - } - b.clusterSync(); - // Per-head merge weights exp2(m_p - M) / L, with L = sum_p exp2(m_p - M) l_p + exp2(sink - M). - { - var heads_ = b.openIf(c.tid.lt(H)); - const h = c.tid; - var ms: [16]V = undefined; - var ls: [16]V = undefined; - var mx = b.cst(.f32, @as(f32, -1e30)); - for (0..@intCast(C)) |p| { - const slot = c.s.add(smem.stats_in + @as(i32, @intCast(p * 2 * H * 4))).add(h.mul(4)); - ms[p] = b.loadWords(1, slot, .shared, .{})[0].bitCast(.f32); - ls[p] = b.loadWords(1, slot.add(H * 4), .shared, .{})[0].bitCast(.f32); - mx = mx.maxnum(ms[p]); - } - const sink = b.loadWords(1, c.s.add(smem.sink).add(h.mul(4)), .shared, .{})[0].bitCast(.f32); - var denom = b.exp2Approx(sink.mul(@as(f32, log2e)).sub(mx)); - var w: [16]V = undefined; - for (0..@intCast(C)) |p| { - w[p] = b.exp2Approx(ms[p].sub(mx)); - denom = denom.add(w[p].mul(ls[p])); - } - const inv = b.rcpApprox(denom); - // Partials are O_p / l_p: weight exp2(m_p - M) * l_p / L. - for (0..@intCast(C)) |p| b.storeWords(1, c.s.add(smem.weights + @as(i32, @intCast(p * H * 4))).add(h.mul(4)), .shared, .{w[p].mul(ls[p]).mul(inv).bitCast(.i32)}); - heads_.yieldThen(.{}); - } - b.barrierSync(0, threads); - // Output: this rank's dims x 16 heads, 4 heads (one 16-byte load per rank) per item. - const items = dims * @divExact(H, 4); - const out_row = c.g.out.add(c.t.to(.i64).mul(H * D * 2)); - { - var loop = b.openFor(c.tid, items, threads, .{}); - const item = loop.iv; - // Consecutive threads take consecutive dims so the output stores coalesce. - const dl = item.rem(dims); - const hq = item.div(dims); // head quad - var acc = [4]V{ b.cst(.f32, 0), b.cst(.f32, 0), b.cst(.f32, 0), b.cst(.f32, 0) }; - for (0..@intCast(C)) |p| { - const o2 = b.loadWords(2, c.s.add(smem.inbox + @as(i32, @intCast(p)) * dims * H * 2).add(dl.mul(H * 2)).add(hq.mul(8)), .shared, .{}); - const lo = b.unpackHalf2(o2[0], .f16); - const hi = b.unpackHalf2(o2[1], .f16); - const o = [4]V{ lo[0], lo[1], hi[0], hi[1] }; - const wv = b.loadWords(4, c.s.add(smem.weights + @as(i32, @intCast(p * H * 4))).add(hq.mul(16)), .shared, .{}); - for (0..4) |i| acc[i] = acc[i].add(o[i].mul(wv[i].bitCast(.f32))); - } - for (0..4) |i| { - const head = hq.mul(4).add(@as(i32, @intCast(i))); - b.storeHalf(c.s.add(smem.ostage).add(head.mul(dims).add(dl).mul(2)), .shared, b.f32ToBf16Bits(acc[i])); - } - loop.yield(.{}); - } - b.barrierSync(0, threads); - { - // This rank's slice of every head row: dims / 8 16-byte chunks per head. - const per_head = @divExact(dims, 8); - var loop = b.openFor(c.tid, H * per_head, threads, .{}); - const h = loop.iv.div(per_head); - const k = loop.iv.rem(per_head); - const v = b.loadWords(4, c.s.add(smem.ostage).add(h.mul(dims).add(k.mul(8)).mul(2)), .shared, .{}); - b.storeWords(4, out_row.add(h.mul(D).add(c.rank.mul(dims)).add(k.mul(8)).to(.i64).mul(2)), .global, v); - loop.yield(.{}); - } -} - -// ---- helpers ------------------------------------------------------------------------------ -// Addresses are raw integers: shared `i32` (shared window), global `i64`, TMEM `i32` -// (lane << 16 | column). - -/// `bytes` of compiler-visible dynamic shared memory (1024-aligned), as a shared address. -fn sharedStorage(b: *B, comptime bytes: i64) V { - const storage = b.allocSmemStorage(.i8, 1024, b.layoutType(b.layoutSpec(.{bytes}, .{1})), 0, &.{std.fmt.comptimePrint("storage:{d}:0", .{bytes})}); - const base = b.getIterTyped(storage, b.ptrTy(.i8, .smem, 1024) catch @panic("bad smem ptr")).value(); - return b.ptrToInt(base, .i32); -} - -/// 16-byte cp.async of `base + row * row_bytes + offset`; when `row < 0` it zero-fills -/// the destination without reading memory. -fn cpAsync16Row(b: *B, dst: V, base: V, row: V, comptime row_bytes: u32, offset: V) void { - const src = base.add(row.maximum(0).to(.i64).mul(@as(i64, row_bytes))).add(offset); - b.cpAsync16(dst, src, row.ge(0)); -} - -/// `n` shared loads of `words` 32-bit words each (n * words results), from `base + offsets[i]`. -fn ldSharedBatch(b: *B, comptime n: usize, comptime words: usize, base: V, comptime offsets: [n]i32) [n * words]V { - var out: [n * words]V = undefined; - for (0..n) |i| { - const w = b.loadWords(words, base.add(offsets[i]), .shared, .{}); - for (0..words) |k| out[i * words + k] = w[k]; - } - return out; -} - -/// Eight packed FP8 e4m3 (two words) -> four bf16x2, each multiplied by the bf16x2 `scale`. -fn fp8x8ToBf16x2x4Scaled(b: *B, lo: V, hi: V, scale: V) [4]V { - const pairs = [4]V{ lo, lo.shrLogical(16), hi, hi.shrLogical(16) }; - var out: [4]V = undefined; - for (pairs, &out) |pair, *o| o.* = b.mulBf16x2(b.e4m3x2ToBf16x2(pair), scale); - return out; -} - -/// Eight packed FP4 e2m1 (one word, even element in the low nibble) -> four bf16x2, -/// each multiplied by the bf16x2 `scale`. -fn fp4x8ToBf16x2x4Scaled(b: *B, word: V, scale: V) [4]V { - var out = b.e2m1x8ToBf16x2x4(word); - for (&out) |*o| o.* = b.mulBf16x2(o.*, scale); - return out; -} - -/// 16-bit shared store of the bf16 rounding of `v` (f32) when `pred` (i1). -fn stSharedBf16If(b: *B, addr: V, v: V, pred: V) void { - var only = b.openIf(pred); - b.storeHalf(addr, .shared, b.f32ToBf16Bits(v)); - only.yieldThen(.{}); -} - -const Major = enum(u1) { k = 0, mn = 1 }; - -/// kind::f16 instruction descriptor: BF16 A/B, F32 accumulator. -fn instrDescBf16(comptime m: u32, comptime n: u32, comptime a_major: Major, comptime b_major: Major) u32 { - std.debug.assert((m == 64 or m == 128) and n % 8 == 0 and n >= 8 and n <= 256); - return (1 << 4) | // c_format = F32 - (1 << 7) | // a_format = BF16 - (1 << 10) | // b_format = BF16 - (@as(u32, @intFromEnum(a_major)) << 15) | - (@as(u32, @intFromEnum(b_major)) << 16) | - ((n >> 3) << 17) | - ((m >> 4) << 24); -} - -/// SM100 shared-memory matrix descriptor with 128-byte swizzle. `lbo`/`sbo` in bytes. -fn smemDescSw128(b: *B, addr: V, comptime lbo: u32, comptime sbo: u32) V { - const hi: u64 = (@as(u64, sbo >> 4) << 32) | (@as(u64, 1) << 46) | (@as(u64, 2) << 61); - const lo: u64 = @as(u64, lbo >> 4) << 16; - const start = addr.to(.i64).shrLogical(4).bitAnd(0x3FFF); - return start.bitOr(b.cst(.i64, @as(i64, @bitCast(hi | lo)))); -} - -/// Byte offset of element (row, col) of a bf16 tile stored as SW128 K-major atoms. -fn sw128Offset(b: *B, row: V, col: V, comptime rows: i32) V { - _ = b; - const slab = col.shrLogical(6).mul(rows * 128); - const chunk = col.bitAnd(63).shrLogical(3).bitXor(row.bitAnd(7)); - return slab.add(row.mul(128)).add(chunk.shl(4)).add(col.bitAnd(7).shl(1)); -} - -fn warpSumF32(b: *B, v: V) V { - var x = v; - inline for (.{ 16, 8, 4, 2, 1 }) |o| x = x.add(b.shuffleXor(x, o)); - return x; -} - -const ReduceOp = enum { max, sum }; diff --git a/zml/attention/cute_kernels/sparse_mla.zig b/zml/attention/cute_kernels/sparse_mla.zig deleted file mode 100644 index 5328eeb3..00000000 --- a/zml/attention/cute_kernels/sparse_mla.zig +++ /dev/null @@ -1,266 +0,0 @@ -//! CuTe backend of sparse MLA (`paged_attention.Mla`, backend `.cute`) on Blackwell -//! (SM100/SM103): in-place sparse attention over DeepSeek-V4.1-style quantized caches. -//! -//! Q is BF16 [queries, heads, 512]. Keys and values are the same 512-wide latent -//! rows, selected per query from two quantized paged caches: a sliding window -//! cache (FP8 e4m3, E8M0 scale per 32 values) and a compressed cache (packed -//! FP4 e2m1, E4M3 scale per 16 values). The selected rows are read and -//! dequantized on chip (`flashmla_sm100.zig`): nothing is gathered in global -//! memory and there is no split-KV reduction. -//! -//! `pagedAttention` is the backend entry of `paged_attention.Mla.pagedSparseAttention`: -//! it resolves the selected positions to physical rows of the paged caches and runs the -//! kernel once per group of 16 heads. -const std = @import("std"); - -const zml = @import("../../zml.zig"); -const Mla = @import("../paged_attention.zig").Mla; -const Tensor = zml.Tensor; - -pub const Config = struct { - queries: i64, - window_capacity: i64, - compressed_capacity: i64, - window_slots: i64, - compressed_slots: i64, - heads: i64 = 16, - scale: f64 = 1.0 / std.math.sqrt(512.0), - /// CTAs per query token (power of two); null picks one from the SM count. - cluster: ?i64 = null, - - /// Enough CTAs to cover the SMs in a single wave, but no more than there - /// are key blocks (64 candidates each) and at most 8 (portable cluster size). - /// Clusters only co-schedule within a GPC, so large clusters do not fill - /// every SM: on GB300 (152 SMs, one 226 KiB CTA per SM) at most 14 clusters - /// of 8 (112 CTAs) and 36 clusters of 4 run at once (measured 2026-09-29). - /// Persistent CTAs (one per SM: one 226 KiB CTA fits) when tokens are not split; - /// larger batches and prefill loop over tokens inside the kernel. - pub fn persistentCtas(c: Config, sm_count: usize) i64 { - if (c.clusterSize(sm_count) > 1) return c.queries; - return @min(c.queries, @as(i64, @intCast(sm_count))); - } - - pub fn clusterSize(c: Config, sm_count: usize) i64 { - if (c.cluster) |n| return n; - // Window-only layers have at most two blocks: splitting them never pays for the merge. - if (c.compressed_capacity == 0) return 1; - const blocks = @divTrunc(c.window_capacity + 63, 64) + @divTrunc(c.compressed_capacity + 63, 64); - const sms: i64 = @intCast(sm_count); - var size: i64 = 8; - while (size > 1) : (size = @divExact(size, 2)) { - const resident = switch (size) { - 8 => @divTrunc(sms * 112, 152), - 4 => @divTrunc(sms * 144, 152), - else => sms, - }; - if (size <= blocks and c.queries * size <= resident) break; - } - return size; - } -}; - -/// Kernel inputs: flat cache rows and per-query physical row indices. -pub const Inputs = struct { - q: zml.Tensor, // bf16 [queries, heads, 512] - window_values: zml.Tensor, // u8 [window_slots, 512] e4m3 - window_scales: zml.Tensor, // u8 [window_slots, 16] e8m0 - compressed_values: zml.Tensor, // u8 [compressed_slots, 256] e2m1 x2 (even value in the low nibble) - compressed_scales: zml.Tensor, // u8 [compressed_slots, 32] e4m3 - window_indices: zml.Tensor, // i32 [queries, window_capacity], physical rows, <0 invalid - compressed_indices: zml.Tensor, // i32 [queries, compressed_capacity] - lengths: zml.Tensor, // i32 [queries, 2] scan bounds per stream - sink: zml.Tensor, // f32 [heads] per-head logit with zero value - active: zml.Tensor, // i32 [1] leading active query rows -}; - -const kernel = @import("flashmla_sm100.zig"); - -pub fn isAvailable(platform: *const zml.Platform) bool { - if (platform.target != .cuda) return false; - const cc = zml.platform.cuda.computeCapability(platform) orelse return false; - return cc.major == 10; -} - -/// Whether the kernel supports these inputs: Blackwell, BF16 Q with a 512-wide latent -/// and a multiple of 16 heads (at most 64; they run in groups of 16, one `kernelCall` -/// each), and the DeepSeek-V4.1 cache formats without a global scale: FP8 window rows -/// with E8M0 scales per 32 values, a window of 128 rows, and packed FP4 compressed rows -/// with E4M3 scales per 16 values, at most 512 (a multiple of 128) per query. -pub fn supports(args: Mla.SparseAttentionArgs) bool { - if (!isAvailable(zml.Compiler.current().platform)) return false; - const q = args.q.shape(); - const heads = q.dim(.h); - if (q.dtype() != .bf16 or q.dim(.hd) != 512 or @rem(heads, 16) != 0 or heads > 64) return false; - - const window = switch (args.kv.cache) { - .quantized => |input| input, - .latent => return false, - }; - if (window.global_scale != null) return false; - if (window.values.dtype() != .f8e4m3fn or lastDim(window.values) != 512) return false; - if (window.scales.dtype() != .f8e8m0 or lastDim(window.scales) != 16) return false; - if (args.kv.positions.dim(.topk) != 128) return false; - - const compressed = args.compressed orelse return true; - const input = switch (compressed.cache) { - .quantized => |input| input, - .latent => return false, - }; - if (input.global_scale != null) return false; - if (input.values.dtype() != .f4e2m1 or lastDim(input.values) != 512) return false; - if (input.scales.dtype() != .f8e4m3fn or lastDim(input.scales) != 32) return false; - const capacity = compressed.positions.dim(.topk); - return @rem(capacity, 128) == 0 and capacity <= 512; -} - -fn lastDim(t: Tensor) i64 { - return t.dim(t.rank() - 1); -} - -/// One kernel launch over 16 heads (see `Config` and `Inputs`). -pub fn kernelCall(c: Config, a: Inputs) zml.Tensor { - // The kernel is compiled for these shapes (see `supports`); anything else would - // read the caches with the wrong layout. - std.debug.assert(c.heads == 16); - std.debug.assert(c.window_capacity == 128); - std.debug.assert(@rem(c.compressed_capacity, 128) == 0 and c.compressed_capacity <= 512); - std.debug.assert(a.q.dtype() == .bf16 and a.q.dim(1) == 16 and a.q.dim(2) == 512); - std.debug.assert(a.window_values.dim(1) == 512 and a.window_scales.dim(1) == 16); - std.debug.assert(a.compressed_values.dim(1) == 256 and a.compressed_scales.dim(1) == 32); - const out = zml.Shape.init(.{ c.queries, c.heads, 512 }, .bf16); - return kernel.Program.call(.{ - .q = a.q, - .wv = a.window_values, - .ws = a.window_scales, - .cv = a.compressed_values, - .cs = a.compressed_scales, - .wi = a.window_indices, - .ci = a.compressed_indices, - .lengths = a.lengths, - .sink = a.sink, - .active = a.active, - }, .{ .out = out }, .{ .cfg = .{ - .queries = c.queries, - .window_capacity = c.window_capacity, - .compressed_capacity = c.compressed_capacity, - .window_slots = c.window_slots, - .compressed_slots = c.compressed_slots, - .scale = c.scale, - .cluster = c.clusterSize(zml.attention.triton.getCuCount()), - .ctas = c.persistentCtas(zml.attention.triton.getCuCount()), - } }).out; -} - -/// Backend `.cute` of `paged_attention.Mla.pagedSparseAttention`: the selected rows of a -/// quantized window cache and optionally of a quantized compressed cache, read in place. -/// Panics on inputs the kernel does not support (see `supports`). Each stream is scanned -/// up to its last valid entry. -pub fn pagedAttention(args: Mla.SparseAttentionArgs, opts: Mla.Options) Tensor { - if (!supports(args)) std.debug.panic("CuTe sparse MLA does not support q {f} over a {s} cache of {} rows per query and {s}: use another backend", .{ - args.q.shape(), - @tagName(args.kv.cache), - args.kv.positions.dim(.topk), - if (args.compressed) |c| @tagName(c.cache) else "no compressed cache", - }); - const sink = args.sink orelse std.debug.panic("CuTe sparse MLA requires an attention sink", .{}); - if (opts.value_rank != args.q.dim(.hd)) std.debug.panic("CuTe sparse MLA requires value_rank ({}) == head dim ({})", .{ opts.value_rank, args.q.dim(.hd) }); - // Physical rows of the selected positions, from each selection's page table. - const active_count = args.kv.activeCount(); - const rows = args.kv.physicalRows(args.tokens_pos, active_count); - const window = args.kv.cache.quantized; - const head_dim: f64 = @floatFromInt(args.q.dim(.hd)); - var context: ShardContext = .{ - .q = args.q.rename(.{ .q = .b }), - .window_values = window.values, - .window_scales = window.scales, - .window_indices = rows, - .compressed_values = null, - .compressed_scales = null, - // Window-only: a placeholder, not read (compressed capacity 0). - .compressed_indices = rows, - .compressed_capacity = 0, - .sink = sink, - .active_count = active_count, - .scale = if (opts.scale) |scale| scale else 1 / @sqrt(head_dim), - }; - if (args.compressed) |c| { - const compressed_rows = c.physicalRows(args.tokens_pos, active_count); - context.compressed_values = c.cache.quantized.values; - context.compressed_scales = c.cache.quantized.scales; - context.compressed_indices = compressed_rows; - context.compressed_capacity = compressed_rows.dim(.topk); - } - return zml.ops.manualComputation(ShardContext.run, context, args.q.shape().rename(.{ .q = .b })).rename(.{ .b = .q }); -} - -/// Per-device part: this shard's heads, run in groups of 16 (a TP4 shard has exactly 16). -const ShardContext = struct { - q: Tensor, - window_values: Tensor, - window_scales: Tensor, - window_indices: Tensor, - compressed_values: ?Tensor, // null for window-only layers - compressed_scales: ?Tensor, - compressed_indices: Tensor, - compressed_capacity: i64, - sink: Tensor, - active_count: Tensor, - scale: f64, - - fn run(self: ShardContext, _: zml.Shape) Tensor { - const q = self.q; - const heads = q.dim(.h); - const queries = q.dim(.b); - std.debug.assert(@rem(heads, 16) == 0); - const window_rows: i64 = @intCast(self.window_values.shape().count() / 512); - const compressed_rows: i64 = if (self.compressed_values) |v| @intCast(v.shape().count() / 512) else 1; - // Window-only layers: one zero row as a placeholder (compressed capacity 0, never read). - const compressed_values = if (self.compressed_values) |v| v.reshape(.{ compressed_rows, 256, 2 }).bitCast(.u8) else Tensor.zeroes(zml.Shape.init(.{ 1, 256 }, .u8)); - const compressed_scales = if (self.compressed_scales) |sc| sc.bitCast(.u8).reshape(.{ compressed_rows, 32 }) else Tensor.zeroes(zml.Shape.init(.{ 1, 32 }, .u8)); - const window_capacity = self.window_indices.dim(.topk); - const window_indices = self.window_indices.convert(.i32); - const compressed_indices = if (self.compressed_capacity > 0) - self.compressed_indices.convert(.i32) - else - Tensor.scalar(-1, .i32).broad(zml.Shape.init(.{ .q = queries, .topk = 0 }, .i32)); - const lengths = Tensor.concatenate(&.{ - scanLength(window_indices).insertAxes(.q, .{.stream}), - scanLength(compressed_indices).insertAxes(.q, .{.stream}), - }, .stream).transpose(.{ .q, .stream }).reshape(.{ queries, 2 }); - const cfg: Config = .{ - .queries = queries, - .window_capacity = window_capacity, - .compressed_capacity = self.compressed_capacity, - .window_slots = window_rows, - .compressed_slots = compressed_rows, - .scale = self.scale, - }; - const q_all = q.reshape(.{ queries, heads, 512 }); - const sink_all = self.sink.convert(.f32).reshape(.{heads}); - var outs: [4]Tensor = undefined; - const groups: usize = @intCast(@divExact(heads, 16)); - for (0..groups) |g| { - const start: i64 = @intCast(16 * g); - outs[g] = kernelCall(cfg, .{ - .q = if (groups == 1) q_all else q_all.slice(1, .{ .start = start, .end = start + 16 }), - .window_values = self.window_values.bitCast(.u8).reshape(.{ window_rows, 512 }), - .window_scales = self.window_scales.bitCast(.u8).reshape(.{ window_rows, 16 }), - .compressed_values = compressed_values, - .compressed_scales = compressed_scales, - .window_indices = window_indices.reshape(.{ queries, window_capacity }), - .compressed_indices = compressed_indices.reshape(.{ queries, self.compressed_capacity }), - .lengths = lengths, - .sink = if (groups == 1) sink_all else sink_all.slice(0, .{ .start = start, .end = start + 16 }), - .active = self.active_count.convert(.i32).reshape(.{1}), - }); - } - const out = if (groups == 1) outs[0] else Tensor.concatenate(outs[0..groups], 1); - return out.withTags(.{ .b, .h, .hd }).withTags(q.shape()); - } -}; - -/// Rows to scan per query: through the last valid (>= 0) entry ([.q] i32). -fn scanLength(indices: Tensor) Tensor { - return Tensor.iota(indices.shape(), .topk).addConstant(1) - .mask(indices.cmp(.GE, .scalar(0, .i32)), 0).max(.topk).squeeze(.topk); -} diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig deleted file mode 100644 index 1e583384..00000000 --- a/zml/attention/flashattn.zig +++ /dev/null @@ -1,1792 +0,0 @@ -const std = @import("std"); - -const flashattn = @import("platforms/cuda/flashattn"); -const platforms = @import("platforms"); -const stdx = @import("stdx"); - -const zml = @import("../zml.zig"); -const ffi = zml.pjrt.ffi; -const AttentionOptions = @import("paged_attention.zig").AttentionOptions; -const AttentionMask = @import("paged_attention.zig").AttentionMask; - -const log = std.log.scoped(.@"zml/attention/flashattn"); - -pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { - if (comptime platforms.isEnabled(.cuda)) { - try flashattn.load(allocator, io); - } -} - -pub fn register(platform: *const zml.Platform) !void { - if (comptime platforms.isEnabled(.cuda)) { - try fa3.register(platform); - try fa2.fa2_mha_varlen_fwd.register(platform); - try paged_fa2.Decode.register(platform); - try paged_fa2.Prefill.register(platform); - try paged_fa3.Decode.register(platform); - try paged_fa3.Prefill.register(platform); - } -} - -fn flashattnDataTypeFromZmlDataType(dtype: zml.DataType) flashattn.DataType { - return switch (dtype) { - .f32 => .f32, - .f16 => .f16, - .bf16 => .bf16, - .u32 => .i32, - .i32 => .i32, - .i8 => .i8, - else => std.debug.panic("Unsupported dtype for attention backend cu_fa2: {t}", .{dtype}), - }; -} - -fn toFlashattnTensor(buffer: zml.pjrtx.CustomCallBuffer) flashattn.Tensor { - return .init( - buffer.ptr, - buffer.shape.dims(), - buffer.shape.withDtype(.u8).computeByteStrides().constSlice(), - flashattnDataTypeFromZmlDataType(buffer.shape.dtype()), - ); -} - -const Buffer = struct { - shape: zml.Shape, - ptr: *anyopaque, - - fn toFlashattnTensor(buffer: Buffer) flashattn.Tensor { - return .init( - buffer.ptr, - buffer.shape.dims(), - buffer.shape.withDtype(.u8).computeByteStrides().constSlice(), - flashattnDataTypeFromZmlDataType(buffer.shape.dtype()), - ); - } - - pub fn format( - self: @This(), - writer: *std.Io.Writer, - ) std.Io.Writer.Error!void { - try writer.print("{f}@{*}", .{ self.shape, self.ptr }); - } -}; - -fn getPlatform(call_frame: *ffi.CallFrame) zml.Platform { - const pjrt_api_ptr = call_frame.attrs.getByName(.scalar, "pjrt_api") orelse unreachable; - std.debug.assert(pjrt_api_ptr.dtype == .u64); - const pjrt_api: ?*zml.pjrt.Api = @ptrFromInt(pjrt_api_ptr.get(usize)); - - const pjrt_client_ptr = call_frame.attrs.getByName(.scalar, "pjrt_client") orelse unreachable; - std.debug.assert(pjrt_client_ptr.dtype == .u64); - const pjrt_client: ?*zml.pjrt.Client = @ptrFromInt(pjrt_client_ptr.get(usize)); - - return .{ .target = .cuda, .pjrt_api = pjrt_api.?, .pjrt_client = pjrt_client.? }; -} - -fn dataTypeFromFfiDataType(ffi_dt: ffi.DataType) zml.DataType { - return switch (ffi_dt) { - .bool => .bool, - .i8 => .i8, - .i16 => .i16, - .i32 => .i32, - .i64 => .i64, - .u8 => .u8, - .u16 => .u16, - .u32 => .u32, - .u64 => .u64, - .f16 => .f16, - .f32 => .f32, - .f64 => .f64, - .bf16 => .bf16, - .c64 => .c64, - .c128 => .c128, - .f8e5m2 => .f8e5m2, - .f8e4m3fn => .f8e4m3fn, - .f8e4m3b11fnuz => .f8e4m3b11fnuz, - .f8e5m2fnuz => .f8e5m2fnuz, - .f8e4m3fnuz => .f8e4m3fnuz, - else => unreachable, - }; -} - -fn shapeFromFfiBuffer(buffer: *const ffi.Buffer) zml.Shape { - return .init(buffer.dims(), dataTypeFromFfiDataType(buffer.dtype)); -} - -fn bufferFromFfiBuffer(ffi_buffer: *const ffi.Buffer) Buffer { - return .{ - .shape = shapeFromFfiBuffer(ffi_buffer), - .ptr = ffi_buffer.data, - }; -} - -fn getScalarAttributeAs(comptime T: type, call_frame: *ffi.CallFrame, attribute_name: []const u8) ?T { - const attribute = call_frame.attrs.getByName(.scalar, attribute_name) orelse return null; - return attribute.get(T); -} - -pub fn Wrapper(comptime T: type, run_func: std.meta.DeclEnum(T)) type { - return struct { - pub fn register(platform: *const zml.Platform) !void { - try platform.registerFfi(.{ - .name = T.custom_call_name, - .platform_name = "cuda", - .handler = T.run, - .traits = .{ .command_buffer_compatible = true }, - }); - } - - pub fn run(call_frame: *ffi.CallFrame) callconv(.c) ?*ffi.Error { - return @field(T, @tagName(run_func))(call_frame) catch b: { - break :b ffi.Error.create(call_frame.api.?, .unknown, "Unknown"); - }; - } - }; -} - -pub const fa2 = struct { - const Input = struct { - q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, - cu_seqlens_q: zml.Tensor, - cu_seqlens_k: zml.Tensor, - seqused_k: zml.Tensor, - softmax_lse: zml.Tensor, - softmax_lse_accum: zml.Tensor, - out_accum: zml.Tensor, - }; - - const Output = struct { - o: zml.Shape, - }; - - const Attributes = struct { - softmax_scale: f32, - is_causal: bool, - window_size_left: i32, - window_size_right: i32, - max_seqlen_q: i32, - max_seqlen_k: i32, - num_heads: i32, - }; - - fn ffiCall( - call_frame: *zml.pjrt.ffi.CallFrame, - input: zml.pjrtx.TensorToCustomCallBuffer(Input), - output: zml.pjrtx.ShapeToCustomCallBuffer(Output), - attributes: Attributes, - ) !?*zml.pjrt.ffi.Error { - const params: flashattn.FA2MhaVarlenFwdParams = .{ - .max_seqlen_q = attributes.max_seqlen_q, - .max_seqlen_k = attributes.max_seqlen_k, - .is_causal = attributes.is_causal, - .softmax_scale = attributes.softmax_scale, - .window_size_left = attributes.window_size_left, - .window_size_right = attributes.window_size_right, - .num_splits = 0, - .num_heads = attributes.num_heads, - }; - - const stream = call_frame.api.stream(call_frame.ctx); - - flashattn.fa2_mha_varlen_fwd( - &toFlashattnTensor(input.q), - &toFlashattnTensor(input.k), - &toFlashattnTensor(input.v), - &toFlashattnTensor(output.o), - &toFlashattnTensor(input.cu_seqlens_q), - &toFlashattnTensor(input.cu_seqlens_k), - &toFlashattnTensor(input.seqused_k), - null, - &toFlashattnTensor(input.softmax_lse), - null, - &toFlashattnTensor(input.softmax_lse_accum), - &toFlashattnTensor(input.out_accum), - ¶ms, - stream, - ); - - return null; - } - - const fa2_mha_varlen_fwd = zml.ops.CustomCall(Input, Output, Attributes, ffiCall, .{ - .name = "fa2_mha_varlen_fwd", - .sharding_aware = true, - .has_side_effect = false, - .output_operand_aliases = .{ .o = .q }, - }); - - pub const Parameters = struct { - sliding_window: i32 = -1, - - pub const InitOptions = struct {}; - - pub fn init(_: InitOptions) fa2.Parameters { - return .{}; - } - }; - - pub const Metadata = struct { - softmax_lse: zml.Tensor, - softmax_lse_accum: zml.Tensor, - out_accum: zml.Tensor, - - pub const InitOptions = struct { - seqlen: i64, - num_heads: i64, - }; - - pub fn init(opts: InitOptions, mesh: *const zml.Sharding.Mesh) Metadata { - return .{ - .softmax_lse = zml.Tensor.init(.{ .s = opts.seqlen, .h = opts.num_heads, .dummy = 1 }, .f32) - .withPartitioning(mesh, .{ .h = .model }), - - .softmax_lse_accum = zml.Tensor.init(.{ .dummy = 1, .h = opts.num_heads, .hd = 128 }, .f32) - .withPartitioning(mesh, .{ .h = .model }), - - .out_accum = zml.Tensor.init(.{ .s = opts.seqlen, .h = opts.num_heads, .hd = 128 }, .f32) - .withPartitioning(mesh, .{ .h = .model }), - }; - } - - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { - return .{ - .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), .{}), - .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), .{}), - .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), - }; - } - - pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { - self.softmax_lse.deinit(); - self.softmax_lse_accum.deinit(); - self.out_accum.deinit(); - } - }; - - pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index_: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { - const ctx = zml.Compiler.current(); - - var bs: i64 = 1; - var q = q_; - var k = k_; - var v = v_; - var token_index = token_index_; - - // Follow cu_fa2 convention - // We are a bit more restrictive because we assume each sequence has the same number of queries and keys. - // https://deepwiki.com/Dao-AILab/flash-attention/3.1-flashattention-2-python-interface#flash_attn_varlen_func - if (q.shape().hasTag(.b)) |_| { - bs = q.dim(.b); - q = q.merge(.{ .tot = .{ .b, .q } }); - k = k.merge(.{ .tot = .{ .b, .k } }); - v = v.merge(.{ .tot = .{ .b, .k } }); - } else { - q = q.rename(.{ .q = .tot }); - k = k.rename(.{ .k = .tot }); - v = v.rename(.{ .k = .tot }); - token_index = token_index.broad(.init(.{ .b = 1 }, .i32)); - } - - const max_seqlen_q: i32 = @intCast(q_.dim(.q)); - const max_seqlen_k: i32 = @intCast(k_.dim(.k)); - - // We still have a rectangle layout, q and k haven't been compacted. - // So each sequence have the same number of queries, keys - const cu_seqlens_q: zml.Tensor = .arange(.{ .end = max_seqlen_q * (bs + 1), .step = max_seqlen_q }, .i32); - const cu_seqlens_k: zml.Tensor = .arange(.{ .end = max_seqlen_k * (bs + 1), .step = max_seqlen_k }, .i32); - // But here we correctly pass the number of used k per sequence - const seqused_k = token_index.addConstant(max_seqlen_q).convert(.i32); - - const original_tot = q.dim(.tot); - const num_heads: i32 = @intCast(q_.dim(.h)); - const num_k_heads = k_.dim(.h); - const head_dim = q.dim(.hd); - const ngroups = @divExact(num_heads, num_k_heads); - const seqlenq_ngroups_swapped = max_seqlen_q == 1 and num_heads > num_k_heads and @mod(head_dim, 8) == 0 and parameters.sliding_window < 0; - if (seqlenq_ngroups_swapped) { - q = q.splitAxis(.h, .{ .h = num_k_heads, .ngroups = ngroups }) - .transpose(.{ .tot, .ngroups, .h, .hd }) - .merge(.{ .tot = .{ .tot, .ngroups } }); - } - - const attn_sdy = ctx.resolveMesh(.{.model}); - const q_sharded = q.withPartitioning(attn_sdy, .{ .h = .model }); - const model_partitions: i32 = @intCast(attn_sdy.numPartitionsForLogicalAxis(.model)); - - const output = fa2_mha_varlen_fwd.call( - .{ - .q = q_sharded, - .k = k.withPartitioning(attn_sdy, .{ .h = .model }), - .v = v.withPartitioning(attn_sdy, .{ .h = .model }), - .cu_seqlens_q = cu_seqlens_q, - .cu_seqlens_k = cu_seqlens_k, - .seqused_k = seqused_k, - .softmax_lse = metadata.softmax_lse.withPartitioning(attn_sdy, .{ .h = .model }), - .softmax_lse_accum = metadata.softmax_lse_accum.withPartitioning(attn_sdy, .{ .h = .model }), - .out_accum = metadata.out_accum.withPartitioning(attn_sdy, .{ .h = .model }), - }, - .{ - .o = q_sharded.shape(), - }, - .{ - .softmax_scale = 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))), - .is_causal = true, - .window_size_left = parameters.sliding_window, - .window_size_right = -1, - .max_seqlen_q = max_seqlen_q, - .max_seqlen_k = max_seqlen_k, - .num_heads = @divExact(num_heads, model_partitions), - }, - ); - var o = output.o; - - if (seqlenq_ngroups_swapped) { - o = o.splitAxis(.tot, .{ .tot = original_tot, .ngroups = ngroups }) - .transpose(.{ .tot, .h, .ngroups, .hd }) - .merge(.{ .h = .{ .h, .ngroups } }); - } - - return if (q_.shape().hasTag(.b)) |_| - o.splitAxis(.tot, .{ .b = bs, .q = q_.dim(.q) }) - else - o.rename(.{ .tot = .q }); - } -}; - -pub const fa3 = struct { - const custom_call_name = "fa3_mha_fwd"; - const Wrapped = Wrapper(@This(), .runInner); - - const register = Wrapped.register; - const run = Wrapped.run; - - pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { - if (call_frame.registeringHook()) return null; - - const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); - const k = bufferFromFfiBuffer(call_frame.args.buffers()[1]); - const v = bufferFromFfiBuffer(call_frame.args.buffers()[2]); - const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); - const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); - const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[5]); - const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[6]); - const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); - const scheduler_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[8]); - const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); - - const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { - const head_dim = q.shape.dim(2); - break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); - }; - const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; - const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; - const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; - const max_seqlen_q: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_q").?; - const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; - - const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); - const stream = call_frame.api.stream(ctx); - - const params: flashattn.FA3MhaFwdParams = .{ - .max_seqlen_q = max_seqlen_q, - .max_seqlen_k = max_seqlen_k, - .softcap = 0.0, - .is_rotary_interleaved = false, - .num_splits = 0, - .sm_margin = 0, - .is_causal = is_causal, - .softmax_scale = softmax_scale, - .window_size_left = window_size_left, - .window_size_right = window_size_right, - .cp_world_size = 1, - .cp_rank = 0, - }; - - flashattn.fa3_mha_fwd( - &q.toFlashattnTensor(), - &k.toFlashattnTensor(), - &v.toFlashattnTensor(), - &o.toFlashattnTensor(), - &cu_seqlens_q.toFlashattnTensor(), - &cu_seqlens_k.toFlashattnTensor(), - null, - null, - null, - null, - null, - null, - &softmax_lse.toFlashattnTensor(), - &softmax_lse_accum.toFlashattnTensor(), - &out_accum.toFlashattnTensor(), - &scheduler_metadata.toFlashattnTensor(), - null, - null, - ¶ms, - stream, - ); - - return null; - } - - pub const Parameters = struct { - pub const InitOptions = struct {}; - - pub fn init(opts: InitOptions) fa3.Parameters { - _ = opts; - return .{}; - } - }; - - pub const Metadata = struct { - softmax_lse: zml.Tensor, - softmax_lse_accum: zml.Tensor, - out_accum: zml.Tensor, - scheduler_metadata: zml.Tensor, - - pub const InitOptions = struct { - seqlen: i64, - num_heads: i64, - }; - - pub fn init(opts: InitOptions, mesh: *const zml.Sharding.Mesh) Metadata { - return .{ - .softmax_lse = zml.Tensor.init(.{ .h = opts.num_heads * opts.seqlen * 4 }, .i8) - .withPartitioning(mesh, .{ .h = .model }), - .softmax_lse_accum = zml.Tensor.init(.{ .h = opts.num_heads * 128 * 4 }, .i8) - .withPartitioning(mesh, .{ .h = .model }), - .out_accum = zml.Tensor.init(.{ .h = opts.num_heads * opts.seqlen * 128 * 4 }, .i8) - .withPartitioning(mesh, .{ .h = .model }), - .scheduler_metadata = zml.Tensor.init(.{ .meta = 2 }, .i32) - .withPartitioning(mesh, .{ .meta = .replicated }), - }; - } - - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { - return .{ - .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), .{}), - .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), .{}), - .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), - .scheduler_metadata = try zml.Buffer.uninitialized(io, platform, self.scheduler_metadata.shape(), .{}), - }; - } - - pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { - self.softmax_lse.deinit(); - self.softmax_lse_accum.deinit(); - self.out_accum.deinit(); - self.scheduler_metadata.deinit(); - } - }; - - pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, _: Parameters) zml.Tensor { - stdx.debug.assert(q_.shape().hasTag(.b) == null or q_.dim(.b) == 1, "fa3.attention support for batch size != 1 is not supported yet.", .{}); - const seqused_k = token_index.addConstant(q_.dim(.q)).reshape(.{1}); - // TODO(Corendos): replace with cumsum - const cu_seqlens_k = b: { - const zero = zml.Tensor.constant(token_index.dtype().zero()).reshape(.{1}); - break :b zml.Tensor.concatenate(&.{ zero, seqused_k }, 0).convert(.i32); - }; - const max_seqlen_q: i32 = @intCast(q_.dim(.q)); - const max_seqlen_k: i32 = @intCast(k_.dim(.k)); - var q = q_.insertAxes(.q, .{.b}).merge(.{ .tot = .{ .b, .q } }); - const k = k_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); - const v = v_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); - // TODO(Corendos): replace with cumsum - const cu_seqlens_q = zml.Tensor.constantTensor(zml.Shape.init(.{2}, .i32), std.mem.sliceAsBytes(&[2]i32{ 0, max_seqlen_q })) - .withPartitioning(.model, .{ ._0 = .replicated }); - - var o = zml.ops.customCall( - custom_call_name, - .{ - q, - k, - v, - cu_seqlens_q, - cu_seqlens_k, - metadata.softmax_lse, - metadata.softmax_lse_accum, - metadata.out_accum, - metadata.scheduler_metadata, - }, - .{q.shape()}, - .{ - .is_causal = true, - .window_size_left = @as(i32, -1), - .window_size_right = @as(i32, -1), - .max_seqlen_q = max_seqlen_q, - .max_seqlen_k = max_seqlen_k, - }, - .{ - .output_operand_aliases = &.{.{ .output_index = 0, .operand_index = 0 }}, - .has_side_effect = false, - }, - ); - - return o.splitAxis(.tot, .{ .b = 1, .q = q_.dim(.q) }).squeeze(.b); - } -}; - -pub const paged_fa2 = struct { - // God knows why flash attention uses this number and not something else. - pub const MAX_NUM_SPLITS = 8; - - pub const Variant = enum { - decode, - mixed, - }; - - pub const Options = union(Variant) { - decode: DecodeOptions, - mixed: MixedOptions, - - pub fn isPrefill(self: Options) bool { - return self == .mixed; - } - - pub fn maxNumPages(self: Options) usize { - return switch (self) { - .decode => |decode_options| decode_options.max_num_pages, - .mixed => |mixed_options| mixed_options.max_num_pages, - }; - } - }; - - pub const Parameters = union(Variant) { - decode: DecodeParameters, - mixed: MixedParameters, - - pub fn init(options_: Options) Parameters { - return switch (options_) { - .decode => |decode_options| .{ .decode = DecodeParameters.init(decode_options) }, - .mixed => |mixed_options| .{ .mixed = MixedParameters.init(mixed_options) }, - }; - } - - pub fn allocationSize(self: Parameters) usize { - return switch (self) { - .decode => |decode| decode.allocationSize(), - .mixed => |mixed| mixed.allocationSize(), - }; - } - - pub fn options(self: Parameters) Options { - return switch (self) { - .decode => |v| .{ .decode = v.options }, - .mixed => |v| .{ .mixed = v.options }, - }; - } - }; - - pub const DecodeOptions = struct { - batch_size: usize, - max_num_pages: usize, - max_seqlen_k: usize, - max_token_count: usize, - num_heads: usize, - num_kv_heads: usize, - head_dim: usize, - }; - - pub const DecodeParameters = struct { - block_table: zml.Tensor, - cu_seqlens_q: zml.Tensor, - seqused_k: zml.Tensor, - - options: DecodeOptions, - - pub fn init(options: DecodeOptions) DecodeParameters { - return .{ - .block_table = zml.Tensor.init(.{ .b = options.batch_size, .p = options.max_num_pages }, .i32), - .cu_seqlens_q = zml.Tensor.init(.{ .b = options.batch_size + 1 }, .i32), - .seqused_k = zml.Tensor.init(.{ .b = options.batch_size }, .i32), - .options = options, - }; - } - - pub fn allocationSize(self: DecodeParameters) usize { - var allocation_size: usize = 0; - - allocation_size += self.block_table.byteSize(); - allocation_size += self.cu_seqlens_q.byteSize(); - allocation_size += self.seqused_k.byteSize(); - - const out_accum_shape: zml.Shape = .init(.{ - .splits = MAX_NUM_SPLITS, - .hkv = self.options.num_kv_heads, - .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), - .b = self.options.batch_size, - .hd = self.options.head_dim, - }, .f32); - allocation_size += out_accum_shape.byteSize(); - - const softmax_lse_shape: zml.Shape = .init(.{ - .hkv = self.options.num_kv_heads, - .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), - .b = self.options.batch_size, - }, .f32); - allocation_size += softmax_lse_shape.byteSize(); - - const softmax_lse_accum_shape: zml.Shape = .init(.{ - .splits = MAX_NUM_SPLITS, - .hkv = self.options.num_kv_heads, - .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), - .b = self.options.batch_size, - }, .f32); - allocation_size += softmax_lse_accum_shape.byteSize(); - - return allocation_size; - } - }; - - pub const MixedOptions = struct { - batch_size_prefill: usize, - batch_size_decode: usize, - max_num_pages: usize, - max_seqlen_k: usize, - max_seqlen_q: usize, - max_token_count: usize, - num_heads: usize, - num_kv_heads: usize, - head_dim: usize, - }; - - pub const MixedParameters = struct { - block_table_prefill: zml.Tensor, - cu_seqlens_q_prefill: zml.Tensor, - seqused_k_prefill: zml.Tensor, - - block_table_decode: zml.Tensor, - cu_seqlens_q_decode: zml.Tensor, - seqused_k_decode: zml.Tensor, - - metadata: MixedMetadata, - options: MixedOptions, - - pub fn init(options: MixedOptions) MixedParameters { - return .{ - .block_table_prefill = zml.Tensor.init( - .{ .b = options.batch_size_prefill, .p = options.max_num_pages }, - .i32, - ), - .cu_seqlens_q_prefill = zml.Tensor.init(.{ .b = options.batch_size_prefill + 1 }, .i32), - .seqused_k_prefill = zml.Tensor.init(.{ .b = options.batch_size_prefill }, .i32), - .block_table_decode = zml.Tensor.init(.{ .b = options.batch_size_decode, .p = options.max_num_pages }, .i32), - .cu_seqlens_q_decode = zml.Tensor.init(.{ .b = options.batch_size_decode + 1 }, .i32), - .seqused_k_decode = zml.Tensor.init(.{ .b = options.batch_size_decode }, .i32), - .metadata = MixedMetadata.init(options), - .options = options, - }; - } - - pub fn allocationSize(self: MixedParameters) usize { - var allocation_size: usize = 0; - allocation_size += self.block_table_decode.byteSize(); - allocation_size += self.cu_seqlens_q_prefill.byteSize(); - allocation_size += self.seqused_k_prefill.byteSize(); - - allocation_size += self.block_table_decode.byteSize(); - allocation_size += self.cu_seqlens_q_decode.byteSize(); - allocation_size += self.seqused_k_decode.byteSize(); - allocation_size += self.metadata.allocationSize(); - - const out_accum_prefill_shape: zml.Shape = .init(.{ - .splits = MAX_NUM_SPLITS, - .hkv = self.options.num_kv_heads, - .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), - .b = self.options.max_token_count, - .hd = self.options.head_dim, - }, .f32); - allocation_size += out_accum_prefill_shape.byteSize(); - - const softmax_lse_prefill_shape: zml.Shape = .init(.{ - .hkv = self.options.num_kv_heads, - .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), - .b = self.options.max_token_count, - }, .f32); - allocation_size += softmax_lse_prefill_shape.byteSize(); - - const softmax_lse_accum_prefill_shape: zml.Shape = .init(.{ - .splits = MAX_NUM_SPLITS, - .hkv = self.options.num_kv_heads, - .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), - .b = self.options.max_token_count, - }, .f32); - allocation_size += softmax_lse_accum_prefill_shape.byteSize(); - - const out_accum_decode_shape: zml.Shape = .init(.{ - .splits = MAX_NUM_SPLITS, - .hkv = self.options.num_kv_heads, - .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), - .b = self.options.batch_size_decode, - .hd = self.options.head_dim, - }, .f32); - allocation_size += out_accum_decode_shape.byteSize(); - - const softmax_lse_decode_shape: zml.Shape = .init(.{ - .hkv = self.options.num_kv_heads, - .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), - .b = self.options.max_token_count, - }, .f32); - allocation_size += softmax_lse_decode_shape.byteSize(); - - const softmax_lse_accum_decode_shape: zml.Shape = .init(.{ - .splits = MAX_NUM_SPLITS, - .hkv = self.options.num_kv_heads, - .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), - .b = self.options.batch_size_decode, - }, .f32); - allocation_size += softmax_lse_accum_decode_shape.byteSize(); - - return allocation_size; - } - }; - - pub const MixedMetadata = struct { - decode_offset: zml.Tensor, - - pub fn init(opts: MixedOptions) MixedMetadata { - _ = opts; - return .{ - .decode_offset = .init(.{}, .i32), - }; - } - - pub fn allocationSize(self: MixedMetadata) usize { - var allocation_size: usize = 0; - allocation_size += self.decode_offset.byteSize(); - return allocation_size; - } - }; - - pub const Prefill = struct { - pub const custom_call_name = "paged_fa2_prefill"; - const Wrapped = Wrapper(@This(), .runInner); - - const register = Wrapped.register; - const run = Wrapped.run; - - pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { - if (call_frame.registeringHook()) return null; - - const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); - const paged_k = bufferFromFfiBuffer(call_frame.args.buffers()[1]); - const paged_v = bufferFromFfiBuffer(call_frame.args.buffers()[2]); - const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); - const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); - const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[5]); - const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[6]); - const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[7]); - const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); - const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[9]); - const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); - - const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { - const head_dim = q.shape.dim(2); - break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); - }; - const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; - const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; - const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; - const max_seqlen_q: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_q").?; - const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; - const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; - - const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); - const stream = call_frame.api.stream(ctx); - - const params: flashattn.FA2MhaVarlenFwdParams = .{ - .max_seqlen_q = max_seqlen_q, - .max_seqlen_k = max_seqlen_k, - .is_causal = is_causal, - .softmax_scale = softmax_scale, - .window_size_left = window_size_left, - .window_size_right = window_size_right, - .num_splits = MAX_NUM_SPLITS, - .num_heads = num_heads, - }; - - flashattn.fa2_mha_varlen_fwd( - &q.toFlashattnTensor(), - &paged_k.toFlashattnTensor(), - &paged_v.toFlashattnTensor(), - &o.toFlashattnTensor(), - &cu_seqlens_q.toFlashattnTensor(), - &cu_seqlens_k.toFlashattnTensor(), - &seqused_k.toFlashattnTensor(), - &block_table.toFlashattnTensor(), - &softmax_lse.toFlashattnTensor(), - null, - &softmax_lse_accum.toFlashattnTensor(), - &out_accum.toFlashattnTensor(), - ¶ms, - stream, - ); - - return null; - } - }; - - pub const Decode = struct { - pub const custom_call_name = "paged_fa2_decode"; - const Wrapped = Wrapper(@This(), .runInner); - - const register = Wrapped.register; - const run = Wrapped.run; - - pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { - if (call_frame.registeringHook()) return null; - - const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); - const paged_k = bufferFromFfiBuffer(call_frame.args.buffers()[1]); - const paged_v = bufferFromFfiBuffer(call_frame.args.buffers()[2]); - const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); - const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); - const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[5]); - const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[6]); - const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[7]); - const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); - const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[9]); - const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); - - const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { - const head_dim = q.shape.dim(2); - break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); - }; - const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; - const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; - const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; - const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; - const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; - - const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); - const stream = call_frame.api.stream(ctx); - - const params: flashattn.FA2MhaVarlenFwdParams = .{ - .max_seqlen_q = 1, - .max_seqlen_k = max_seqlen_k, - .is_causal = is_causal, - .softmax_scale = softmax_scale, - .window_size_left = window_size_left, - .window_size_right = window_size_right, - .num_splits = MAX_NUM_SPLITS, - .num_heads = num_heads, - }; - - flashattn.fa2_mha_varlen_fwd( - &q.toFlashattnTensor(), - &paged_k.toFlashattnTensor(), - &paged_v.toFlashattnTensor(), - &o.toFlashattnTensor(), - &cu_seqlens_q.toFlashattnTensor(), - &cu_seqlens_k.toFlashattnTensor(), - &seqused_k.toFlashattnTensor(), - &block_table.toFlashattnTensor(), - &softmax_lse.toFlashattnTensor(), - null, - &softmax_lse_accum.toFlashattnTensor(), - &out_accum.toFlashattnTensor(), - ¶ms, - stream, - ); - - return null; - } - }; - - pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { - stdx.debug.assert(q.shape().hasTags(.{ .b, .hg, .hkv, .hd }), "Expected q to have tags .b, .h, .hd", .{}); - stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); - stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); - const ctx = zml.Compiler.current(); - - const window_size_left = windowSizeLeft(opts.mask); - - const num_head_groups = q.dim(.hg); - const num_kv_heads = q.dim(.hkv); - const head_dim = q.dim(.hd); - const num_heads = num_head_groups * num_kv_heads; - // FIXME: remove unreachable and propagate error correctly. - - const attn_sdy = ctx.resolveMesh(.{.model}); - const model_partitions: i32 = @intCast(attn_sdy.numPartitionsForLogicalAxis(.model)); - const num_heads_per_shard = @divExact(num_heads, model_partitions); - - const o = switch (parameters) { - .decode => |decode_parameters| b: { - const seqlenq_ngroups_swapped = num_heads > num_kv_heads and @mod(head_dim, 8) == 0 and opts.mask.slidingWindow() == null; - - const block_table = decode_parameters.block_table.withPartitioning(.model, .{ .b = .replicated }); - const cu_seqlens_q = decode_parameters.cu_seqlens_q.withPartitioning(.model, .{ .b = .replicated }); - const seqused_k = decode_parameters.seqused_k.withPartitioning(.model, .{ .b = .replicated }); - - const out_accum = zml.Tensor.uninitialized(.init(.{ - .splits = MAX_NUM_SPLITS, - .hkv = num_kv_heads, - .hg = num_head_groups, - .b = q.dim(.b), - .hd = head_dim, - }, .f32)).withPartitioning(.model, .{ .hkv = .model }); - - const softmax_lse = zml.Tensor.uninitialized(.init(.{ - .hkv = num_kv_heads, - .hg = num_head_groups, - .b = q.dim(.b), - }, .f32)).withPartitioning(.model, .{ .hkv = .model }); - const softmax_lse_accum = zml.Tensor.uninitialized(.init(.{ - .splits = MAX_NUM_SPLITS, - .hkv = num_kv_heads, - .hg = num_head_groups, - .b = q.dim(.b), - }, .f32)).withPartitioning(.model, .{ .hkv = .model }); - const dummy_cu_seqlens_k: zml.Tensor = .zeroes(cu_seqlens_q.shape()); - - const batch_dim = q.dim(.b); - var q2 = q; - if (seqlenq_ngroups_swapped) { - q2 = q2.transpose(.{ .b, .hg, .hkv, .hd }).merge(.{ .b = .{ .b, .hg } }).withPartitioning(.model, .{ .hkv = .model }); - } else { - q2 = q2.transpose(.{ .b, .hkv, .hg, .hd }).merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); - } - - const output_shape = q2.shape(); - var o = zml.ops.manualComputation( - (struct { - inputs: struct { zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor }, - metadata: struct { - is_causal: bool, - max_seqlen_k: usize, - num_heads: i64, - window_size_left: i32, - softmax_scale: ?f32, - }, - opts: zml.ops.CustomCallOptions, - - fn body(self: @This(), output: zml.Shape) zml.Tensor { - return zml.ops.customCall(Decode.custom_call_name, self.inputs, output, self.metadata, self.opts); - } - }).body, - .{ - .inputs = .{ - q2, - k_cache, - v_cache, - cu_seqlens_q, - dummy_cu_seqlens_k, - seqused_k, - block_table, - softmax_lse, - softmax_lse_accum, - out_accum, - }, - .metadata = .{ - .is_causal = opts.mask.isCausal(), - .max_seqlen_k = decode_parameters.options.max_seqlen_k, - .num_heads = num_heads_per_shard, - .window_size_left = window_size_left, - .softmax_scale = opts.scale, - }, - .opts = zml.ops.CustomCallOptions{ - .has_side_effect = false, - }, - }, - output_shape, - ); - - if (seqlenq_ngroups_swapped) { - o = o.splitAxis(.b, .{ .b = batch_dim, .hg = num_head_groups }).transpose(.{ .b, .hkv, .hg, .hd }); - } else { - o = o.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = num_head_groups }); - } - - break :b o; - }, - .mixed => |mixed_parameters| b: { - const seqlenq_ngroups_swapped = num_heads > num_kv_heads and @mod(head_dim, 8) == 0 and opts.mask.slidingWindow() == null; - - const block_table_prefill = mixed_parameters.block_table_prefill.withPartitioning(.model, .{ .b = .replicated }); - const cu_seqlens_q_prefill = mixed_parameters.cu_seqlens_q_prefill.withPartitioning(.model, .{ .b = .replicated }); - const seqused_k_prefill = mixed_parameters.seqused_k_prefill.withPartitioning(.model, .{ .b = .replicated }); - const block_table_decode = mixed_parameters.block_table_decode.withPartitioning(.model, .{ .b = .replicated }); - const cu_seqlens_q_decode = mixed_parameters.cu_seqlens_q_decode.withPartitioning(.model, .{ .b = .replicated }); - const seqused_k_decode = mixed_parameters.seqused_k_decode.withPartitioning(.model, .{ .b = .replicated }); - - const out_accum_prefill = zml.Tensor.uninitialized(.init(.{ - .splits = MAX_NUM_SPLITS, - .hkv = num_kv_heads, - .hg = num_head_groups, - .b = q.dim(.b), - .hd = head_dim, - }, .f32)).withPartitioning(.model, .{ .hkv = .model }); - const softmax_lse_prefill = zml.Tensor.uninitialized(.init(.{ - .hkv = num_kv_heads, - .hg = num_head_groups, - .b = q.dim(.b), - }, .f32)).withPartitioning(.model, .{ .hkv = .model }); - const softmax_lse_accum_prefill = zml.Tensor.uninitialized(.init(.{ - .splits = MAX_NUM_SPLITS, - .hkv = num_kv_heads, - .hg = num_head_groups, - .b = q.dim(.b), - }, .f32)).withPartitioning(.model, .{ .hkv = .model }); - const dummy_cu_seqlens_k_prefill: zml.Tensor = .zeroes(cu_seqlens_q_prefill.shape()); - - var q2 = q; - q2 = q2.transpose(.{ .b, .hkv, .hg, .hd }).merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); - - const output_shape = q2.shape(); - var o = zml.ops.manualComputation( - (struct { - inputs: struct { zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor }, - metadata: struct { - is_causal: bool, - max_seqlen_k: usize, - max_seqlen_q: usize, - num_heads: i64, - window_size_left: i32, - softmax_scale: ?f32, - }, - opts: zml.ops.CustomCallOptions, - - fn body(self: @This(), output: zml.Shape) zml.Tensor { - return zml.ops.customCall(Prefill.custom_call_name, self.inputs, output, self.metadata, self.opts); - } - }).body, - .{ - .inputs = .{ - q2, - k_cache, - v_cache, - cu_seqlens_q_prefill, - dummy_cu_seqlens_k_prefill, - seqused_k_prefill, - block_table_prefill, - softmax_lse_prefill, - softmax_lse_accum_prefill, - out_accum_prefill, - }, - .metadata = .{ - .is_causal = opts.mask.isCausal(), - .max_seqlen_k = mixed_parameters.options.max_seqlen_k, - .max_seqlen_q = mixed_parameters.options.max_seqlen_q, - .num_heads = num_heads_per_shard, - .window_size_left = window_size_left, - .softmax_scale = opts.scale, - }, - .opts = zml.ops.CustomCallOptions{ - .has_side_effect = false, - }, - }, - output_shape, - ); - - o = o.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = num_head_groups }); - - const batch_dim_decode = block_table_decode.dim(0); - const out_accum_decode = zml.Tensor.uninitialized(.init(.{ - .splits = MAX_NUM_SPLITS, - .hkv = num_kv_heads, - .hg = num_head_groups, - .b = batch_dim_decode, - .hd = head_dim, - }, .f32)).withPartitioning(.model, .{ .hkv = .model }); - const softmax_lse_decode = zml.Tensor.uninitialized(.init(.{ - .hkv = num_kv_heads, - .hg = num_head_groups, - .b = batch_dim_decode, - }, .f32)).withPartitioning(.model, .{ .hkv = .model }); - const softmax_lse_accum_decode = zml.Tensor.uninitialized(.init(.{ - .splits = MAX_NUM_SPLITS, - .hkv = num_kv_heads, - .hg = num_head_groups, - .b = batch_dim_decode, - }, .f32)).withPartitioning(.model, .{ .hkv = .model }); - const dummy_cu_seqlens_k_decode = zml.Tensor.zeroes(cu_seqlens_q_decode.shape()); - var q_decode = q.slice(0, .dyn(mixed_parameters.metadata.decode_offset, batch_dim_decode)); - - if (seqlenq_ngroups_swapped) { - q_decode = q_decode.transpose(.{ .b, .hg, .hkv, .hd }).merge(.{ .b = .{ .b, .hg } }).withPartitioning(.model, .{ .hkv = .model }); - } else { - q_decode = q_decode.transpose(.{ .b, .hkv, .hg, .hd }).merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); - } - - const output_shape_decode = q_decode.shape(); - var o_decode = zml.ops.manualComputation( - (struct { - inputs: struct { zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor }, - metadata: struct { - is_causal: bool, - max_seqlen_k: usize, - num_heads: i64, - window_size_left: i32, - softmax_scale: ?f32, - }, - opts: zml.ops.CustomCallOptions, - - fn body(self: @This(), output: zml.Shape) zml.Tensor { - return zml.ops.customCall(Decode.custom_call_name, self.inputs, output, self.metadata, self.opts); - } - }).body, - .{ - .inputs = .{ - q_decode, - k_cache, - v_cache, - cu_seqlens_q_decode, - dummy_cu_seqlens_k_decode, - seqused_k_decode, - block_table_decode, - softmax_lse_decode, - softmax_lse_accum_decode, - out_accum_decode, - }, - .metadata = .{ - .is_causal = opts.mask.isCausal(), - .max_seqlen_k = mixed_parameters.options.max_seqlen_k, - .num_heads = num_heads_per_shard, - .window_size_left = window_size_left, - .softmax_scale = opts.scale, - }, - .opts = zml.ops.CustomCallOptions{ - .has_side_effect = false, - }, - }, - output_shape_decode, - ); - - if (seqlenq_ngroups_swapped) { - o_decode = o_decode.splitAxis(.b, .{ .b = batch_dim_decode, .hg = num_head_groups }).transpose(.{ .b, .hkv, .hg, .hd }); - } else { - o_decode = o_decode.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = num_head_groups }); - } - - o = o.dynamicUpdateSlice1d(o_decode, 0, mixed_parameters.metadata.decode_offset); - break :b o; - }, - }; - - return o; - } -}; - -pub const paged_fa3 = struct { - // God knows why flash attention uses this number and not something else. - pub const MAX_NUM_SPLITS = 32; - - pub const Variant = enum { - decode, - mixed, - }; - - pub const Options = union(Variant) { - decode: DecodeOptions, - mixed: MixedOptions, - - pub fn isPrefill(self: Options) bool { - return self == .mixed; - } - - pub fn maxNumPages(self: Options) usize { - return switch (self) { - .decode => |decode_options| decode_options.max_num_pages, - .mixed => |mixed_options| mixed_options.max_num_pages, - }; - } - }; - - pub const Parameters = union(Variant) { - decode: DecodeParameters, - mixed: MixedParameters, - - pub fn init(options_: Options) Parameters { - return switch (options_) { - .decode => |decode_options| .{ .decode = DecodeParameters.init(decode_options) }, - .mixed => |mixed_options| .{ .mixed = MixedParameters.init(mixed_options) }, - }; - } - - pub fn allocationSize(self: Parameters) usize { - return switch (self) { - .decode => |decode| decode.allocationSize(), - .mixed => |mixed| mixed.allocationSize(), - }; - } - - pub fn options(self: Parameters) Options { - return switch (self) { - .decode => |v| .{ .decode = v.options }, - .mixed => |v| .{ .mixed = v.options }, - }; - } - }; - - pub const DecodeOptions = struct { - batch_size: usize, - max_num_pages: usize, - max_seqlen_k: usize, - max_token_count: usize, - num_heads: usize, - num_kv_heads: usize, - head_dim: usize, - }; - - pub const DecodeParameters = struct { - block_table: zml.Tensor, - cu_seqlens_q: zml.Tensor, - seqused_k: zml.Tensor, - - options: DecodeOptions, - - pub fn init(options: DecodeOptions) DecodeParameters { - return .{ - .block_table = .init(.{ .b = options.batch_size, .p = options.max_num_pages }, .i32), - .cu_seqlens_q = .init(.{ .b = options.batch_size + 1 }, .i32), - .seqused_k = .init(.{ .b = options.batch_size }, .i32), - .options = options, - }; - } - - pub fn allocationSize(self: DecodeParameters) usize { - var allocation_size: usize = 0; - - allocation_size += self.block_table.byteSize(); - allocation_size += self.cu_seqlens_q.byteSize(); - allocation_size += self.seqused_k.byteSize(); - - const out_accum_shape: zml.Shape = .init(.{ - .splits = MAX_NUM_SPLITS, - .hkv = self.options.num_kv_heads, - .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), - .b = self.options.batch_size, - .hd = self.options.head_dim, - }, .f32); - allocation_size += out_accum_shape.byteSize(); - - const softmax_lse_shape: zml.Shape = .init(.{ - .hkv = self.options.num_kv_heads, - .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), - .b = self.options.batch_size, - }, .f32); - allocation_size += softmax_lse_shape.byteSize(); - - const softmax_lse_accum_shape: zml.Shape = .init(.{ - .splits = MAX_NUM_SPLITS, - .hkv = self.options.num_kv_heads, - .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), - .b = self.options.batch_size, - }, .f32); - allocation_size += softmax_lse_accum_shape.byteSize(); - - return allocation_size; - } - }; - - pub const MixedOptions = struct { - batch_size_prefill: usize, - batch_size_decode: usize, - max_num_pages: usize, - max_seqlen_q: usize, - max_seqlen_k: usize, - max_token_count: usize, - num_heads: usize, - num_kv_heads: usize, - head_dim: usize, - }; - - pub const MixedParameters = struct { - block_table_prefill: zml.Tensor, - cu_seqlens_q_prefill: zml.Tensor, - seqused_k_prefill: zml.Tensor, - - block_table_decode: zml.Tensor, - cu_seqlens_q_decode: zml.Tensor, - seqused_k_decode: zml.Tensor, - - metadata: MixedMetadata, - options: MixedOptions, - - pub fn init(options: MixedOptions) MixedParameters { - return .{ - .block_table_prefill = .init(.{ .b = options.batch_size_prefill, .p = options.max_num_pages }, .i32), - .cu_seqlens_q_prefill = .init(.{ .b = options.batch_size_prefill + 1 }, .i32), - .seqused_k_prefill = .init(.{ .b = options.batch_size_prefill }, .i32), - .block_table_decode = .init(.{ .b = options.batch_size_decode, .p = options.max_num_pages }, .i32), - .cu_seqlens_q_decode = .init(.{ .b = options.batch_size_decode + 1 }, .i32), - .seqused_k_decode = .init(.{ .b = options.batch_size_decode }, .i32), - .metadata = MixedMetadata.init(options), - .options = options, - }; - } - - pub fn allocationSize(self: MixedParameters) usize { - var allocation_size: usize = 0; - allocation_size += self.block_table_decode.byteSize(); - allocation_size += self.cu_seqlens_q_prefill.byteSize(); - allocation_size += self.seqused_k_prefill.byteSize(); - - allocation_size += self.block_table_decode.byteSize(); - allocation_size += self.cu_seqlens_q_decode.byteSize(); - allocation_size += self.seqused_k_decode.byteSize(); - allocation_size += self.metadata.allocationSize(); - - const out_accum_prefill_shape: zml.Shape = .init(.{ - .splits = MAX_NUM_SPLITS, - .hkv = self.options.num_kv_heads, - .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), - .b = self.options.max_token_count, - .hd = self.options.head_dim, - }, .f32); - allocation_size += out_accum_prefill_shape.byteSize(); - - const softmax_lse_prefill_shape: zml.Shape = .init(.{ - .hkv = self.options.num_kv_heads, - .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), - .b = self.options.max_token_count, - }, .f32); - allocation_size += softmax_lse_prefill_shape.byteSize(); - - const softmax_lse_accum_prefill_shape: zml.Shape = .init(.{ - .splits = MAX_NUM_SPLITS, - .hkv = self.options.num_kv_heads, - .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), - .b = self.options.max_token_count, - }, .f32); - allocation_size += softmax_lse_accum_prefill_shape.byteSize(); - - const out_accum_decode_shape: zml.Shape = .init(.{ - .splits = MAX_NUM_SPLITS, - .hkv = self.options.num_kv_heads, - .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), - .b = self.options.batch_size_decode, - .hd = self.options.head_dim, - }, .f32); - allocation_size += out_accum_decode_shape.byteSize(); - - const softmax_lse_decode_shape: zml.Shape = .init(.{ - .hkv = self.options.num_kv_heads, - .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), - .b = self.options.max_token_count, - }, .f32); - allocation_size += softmax_lse_decode_shape.byteSize(); - - const softmax_lse_accum_decode_shape: zml.Shape = .init(.{ - .splits = MAX_NUM_SPLITS, - .hkv = self.options.num_kv_heads, - .hg = @divExact(self.options.num_heads, self.options.num_kv_heads), - .b = self.options.batch_size_decode, - }, .f32); - allocation_size += softmax_lse_accum_decode_shape.byteSize(); - - return allocation_size; - } - }; - - pub const MixedMetadata = struct { - decode_offset: zml.Tensor, - - pub fn init(opts: MixedOptions) MixedMetadata { - _ = opts; - return .{ - .decode_offset = .init(.{}, .i32), - }; - } - - pub fn allocationSize(self: MixedMetadata) usize { - var allocation_size: usize = 0; - allocation_size += self.decode_offset.byteSize(); - return allocation_size; - } - }; - - pub const Prefill = struct { - pub const custom_call_name = "paged_fa3_prefill"; - const Wrapped = Wrapper(@This(), .runInner); - - const register = Wrapped.register; - const run = Wrapped.run; - - pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { - if (call_frame.registeringHook()) return null; - - const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); - const paged_k = bufferFromFfiBuffer(call_frame.args.buffers()[1]); - const paged_v = bufferFromFfiBuffer(call_frame.args.buffers()[2]); - const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); - const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); - const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[5]); - const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[6]); - const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); - const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); - const scheduler_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[9]); - const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); - - const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { - const head_dim = q.shape.dim(2); - break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); - }; - const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; - const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; - const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; - const max_seqlen_q: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_q").?; - const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; - - const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); - const stream = call_frame.api.stream(ctx); - - const params: flashattn.FA3MhaFwdParams = .{ - .max_seqlen_q = max_seqlen_q, - .max_seqlen_k = max_seqlen_k, - .is_causal = is_causal, - .softmax_scale = softmax_scale, - .window_size_left = window_size_left, - .window_size_right = window_size_right, - .softcap = 0.0, - .is_rotary_interleaved = false, - .num_splits = MAX_NUM_SPLITS, - .sm_margin = 0, - .cp_world_size = 1, - .cp_rank = 0, - }; - - flashattn.fa3_mha_fwd( - &q.toFlashattnTensor(), - &paged_k.toFlashattnTensor(), - &paged_v.toFlashattnTensor(), - &o.toFlashattnTensor(), - &cu_seqlens_q.toFlashattnTensor(), - null, - null, - &seqused_k.toFlashattnTensor(), - &block_table.toFlashattnTensor(), - null, - null, - null, - &softmax_lse.toFlashattnTensor(), - &softmax_lse_accum.toFlashattnTensor(), - &out_accum.toFlashattnTensor(), - &scheduler_metadata.toFlashattnTensor(), - null, - null, - ¶ms, - stream, - ); - - return null; - } - }; - - pub const Decode = struct { - pub const custom_call_name = "paged_fa3_decode"; - const Wrapped = Wrapper(@This(), .runInner); - - const register = Wrapped.register; - const run = Wrapped.run; - - pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { - if (call_frame.registeringHook()) return null; - - const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); - const paged_k = bufferFromFfiBuffer(call_frame.args.buffers()[1]); - const paged_v = bufferFromFfiBuffer(call_frame.args.buffers()[2]); - const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); - const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); - const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[5]); - const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[6]); - const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); - const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); - const scheduler_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[9]); - const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); - - const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { - const head_dim = q.shape.dim(2); - break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); - }; - const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; - const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; - const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; - const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; - - const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); - const stream = call_frame.api.stream(ctx); - - const params: flashattn.FA3MhaFwdParams = .{ - .max_seqlen_q = 1, - .max_seqlen_k = max_seqlen_k, - .is_causal = is_causal, - .softmax_scale = softmax_scale, - .window_size_left = window_size_left, - .window_size_right = window_size_right, - .softcap = 0.0, - .is_rotary_interleaved = false, - .num_splits = MAX_NUM_SPLITS, - .sm_margin = 0, - .cp_world_size = 1, - .cp_rank = 0, - }; - - flashattn.fa3_mha_fwd( - &q.toFlashattnTensor(), - &paged_k.toFlashattnTensor(), - &paged_v.toFlashattnTensor(), - &o.toFlashattnTensor(), - &cu_seqlens_q.toFlashattnTensor(), - null, - null, - &seqused_k.toFlashattnTensor(), - &block_table.toFlashattnTensor(), - null, - null, - null, - &softmax_lse.toFlashattnTensor(), - &softmax_lse_accum.toFlashattnTensor(), - &out_accum.toFlashattnTensor(), - &scheduler_metadata.toFlashattnTensor(), - null, - null, - ¶ms, - stream, - ); - - return null; - } - }; - - pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { - stdx.debug.assert(q.shape().hasTags(.{ .b, .hg, .hkv, .hd }), "Expected q to have tags .b, .h, .hd", .{}); - stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); - stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); - const window_size_left = windowSizeLeft(opts.mask); - - const num_head_groups = q.dim(.hg); - const num_kv_heads = q.dim(.hkv); - const head_dim = q.dim(.hd); - const o = switch (parameters) { - .decode => |decode_parameters| b: { - const batch_size = decode_parameters.block_table.dim(0); - - const block_table = decode_parameters.block_table.withPartitioning(.model, .{ .b = .replicated }); - const cu_seqlens_q = decode_parameters.cu_seqlens_q.withPartitioning(.model, .{ .b = .replicated }); - const seqused_k = decode_parameters.seqused_k.withPartitioning(.model, .{ .b = .replicated }); - - const out_accum = zml.Tensor.uninitialized(.init(.{ - .splits = MAX_NUM_SPLITS, - .hkv = num_kv_heads, - .hg = num_head_groups, - .b = q.dim(.b), - .hd = head_dim, - }, .f32)).withPartitioning(.model, .{ .hkv = .model }); - const softmax_lse = zml.Tensor.uninitialized(.init(.{ - .hkv = num_kv_heads, - .hg = num_head_groups, - .q = q.dim(.b), - }, .f32)).withPartitioning(.model, .{ .hkv = .model }); - const softmax_lse_accum = zml.Tensor.uninitialized(.init(.{ - .splits = MAX_NUM_SPLITS, - .q = q.dim(.b), - .hkv = num_kv_heads, - .hg = num_head_groups, - }, .f32)).withPartitioning(.model, .{ .hkv = .model }); - const scheduler_metadata = zml.Tensor.zeroes(.init(.{ .b = batch_size + 1 }, .i32)); - - var q2 = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); - - const output_shape = q2.shape(); - var o = zml.ops.manualComputation( - (struct { - inputs: struct { zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor }, - metadata: struct { - is_causal: bool, - max_seqlen_k: usize, - window_size_left: i32, - }, - opts: zml.ops.CustomCallOptions, - - fn body(self: @This(), output: zml.Shape) zml.Tensor { - return zml.ops.customCall(Decode.custom_call_name, self.inputs, output, self.metadata, self.opts); - } - }).body, - .{ - .inputs = .{ - q2, - k_cache, - v_cache, - cu_seqlens_q, - seqused_k, - block_table, - softmax_lse, - softmax_lse_accum, - out_accum, - scheduler_metadata, - }, - .metadata = .{ - .is_causal = opts.mask.isCausal(), - .max_seqlen_k = decode_parameters.options.max_seqlen_k, - .window_size_left = window_size_left, - }, - .opts = zml.ops.CustomCallOptions{ - .has_side_effect = false, - }, - }, - output_shape, - ); - - o = o.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = num_head_groups }); - - break :b o; - }, - .mixed => |mixed_parameters| b: { - const batch_size_prefill = mixed_parameters.block_table_prefill.dim(0); - - const block_table_prefill = mixed_parameters.block_table_prefill.withPartitioning(.model, .{ .b = .replicated }); - const cu_seqlens_q_prefill = mixed_parameters.cu_seqlens_q_prefill.withPartitioning(.model, .{ .b = .replicated }); - const seqused_k_prefill = mixed_parameters.seqused_k_prefill.withPartitioning(.model, .{ .b = .replicated }); - const block_table_decode = mixed_parameters.block_table_decode.withPartitioning(.model, .{ .b = .replicated }); - const cu_seqlens_q_decode = mixed_parameters.cu_seqlens_q_decode.withPartitioning(.model, .{ .b = .replicated }); - const seqused_k_decode = mixed_parameters.seqused_k_decode.withPartitioning(.model, .{ .b = .replicated }); - - const out_accum_prefill = zml.Tensor.uninitialized(.init(.{ - .splits = MAX_NUM_SPLITS, - .hkv = num_kv_heads, - .hg = num_head_groups, - .b = q.dim(.b), - .hd = head_dim, - }, .f32)).withPartitioning(.model, .{ .hkv = .model }); - const softmax_lse_prefill = zml.Tensor.uninitialized(.init(.{ - .hkv = num_kv_heads, - .hg = num_head_groups, - .q = q.dim(.b), - }, .f32)).withPartitioning(.model, .{ .hkv = .model }); - const softmax_lse_accum_prefill = zml.Tensor.uninitialized(.init(.{ - .splits = MAX_NUM_SPLITS, - .q = q.dim(.b), - .hkv = num_kv_heads, - .hg = num_head_groups, - }, .f32)).withPartitioning(.model, .{ .hkv = .model }); - const scheduler_metadata_prefill = zml.Tensor.zeroes(.init(.{ .b = batch_size_prefill + 1 }, .i32)); - - var q2 = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); - - const output_shape = q2.shape(); - var o = zml.ops.manualComputation( - (struct { - inputs: struct { zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor }, - metadata: struct { - is_causal: bool, - max_seqlen_k: usize, - max_seqlen_q: usize, - window_size_left: i32, - }, - opts: zml.ops.CustomCallOptions, - - fn body(self: @This(), output: zml.Shape) zml.Tensor { - return zml.ops.customCall(Prefill.custom_call_name, self.inputs, output, self.metadata, self.opts); - } - }).body, - .{ - .inputs = .{ - q2, - k_cache, - v_cache, - cu_seqlens_q_prefill, - seqused_k_prefill, - block_table_prefill, - softmax_lse_prefill, - softmax_lse_accum_prefill, - out_accum_prefill, - scheduler_metadata_prefill, - }, - .metadata = .{ - .is_causal = opts.mask.isCausal(), - .max_seqlen_k = mixed_parameters.options.max_seqlen_k, - .max_seqlen_q = mixed_parameters.options.max_seqlen_q, - .window_size_left = window_size_left, - }, - .opts = zml.ops.CustomCallOptions{ .has_side_effect = false }, - }, - output_shape, - ); - - o = o.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = num_head_groups }); - - const batch_size_decode = mixed_parameters.block_table_prefill.dim(0); - const out_accum_decode = zml.Tensor.uninitialized(.init(.{ - .splits = MAX_NUM_SPLITS, - .hkv = num_kv_heads, - .hg = num_head_groups, - .b = batch_size_decode, - .hd = head_dim, - }, .f32)).withPartitioning(.model, .{ .hkv = .model }); - const softmax_lse_decode = zml.Tensor.uninitialized(.init(.{ - .hkv = num_kv_heads, - .hg = num_head_groups, - .q = q.dim(.b), - }, .f32)); - const softmax_lse_accum_decode = zml.Tensor.uninitialized(.init(.{ - .splits = MAX_NUM_SPLITS, - .q = q.dim(.b), - .hkv = num_kv_heads, - .hg = num_head_groups, - }, .f32)); - const scheduler_metadata_decode = zml.Tensor.zeroes(.init(.{ .b = batch_size_decode + 1 }, .i32)); - var q_decode = q.slice(0, .dyn(mixed_parameters.metadata.decode_offset, batch_size_decode)).withPartitioning(.model, .{ .hkv = .model }); - - q_decode = q_decode.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); - - const decode_output_shape = q_decode.shape(); - var o_decode = zml.ops.manualComputation( - (struct { - inputs: struct { zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor, zml.Tensor }, - metadata: struct { - is_causal: bool, - max_seqlen_k: usize, - window_size_left: i32, - }, - opts: zml.ops.CustomCallOptions, - - fn body(self: @This(), output: zml.Shape) zml.Tensor { - return zml.ops.customCall(Decode.custom_call_name, self.inputs, output, self.metadata, self.opts); - } - }).body, - .{ - .inputs = .{ - q_decode, - k_cache, - v_cache, - cu_seqlens_q_decode, - seqused_k_decode, - block_table_decode, - softmax_lse_decode, - softmax_lse_accum_decode, - out_accum_decode, - scheduler_metadata_decode, - }, - .metadata = .{ - .is_causal = opts.mask.isCausal(), - .max_seqlen_k = mixed_parameters.options.max_seqlen_k, - .window_size_left = window_size_left, - }, - .opts = zml.ops.CustomCallOptions{ .has_side_effect = false }, - }, - decode_output_shape, - ); - o_decode = o_decode.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = num_head_groups }); - - o = o.dynamicUpdateSlice1d(o_decode, 0, mixed_parameters.metadata.decode_offset); - break :b o; - }, - }; - - return o; - } -}; - -fn windowSizeLeft(mask: AttentionMask) i32 { - return if (mask.slidingWindow()) |window| @as(i32, @intCast(window)) - 1 else -1; -} - -test "FlashAttention sliding window uses an inclusive offset" { - try std.testing.expectEqual(@as(i32, 2047), windowSizeLeft(.{ .standard = .{ .sliding_window = 2048 } })); - try std.testing.expectEqual(@as(i32, 0), windowSizeLeft(.{ .standard = .{ .sliding_window = 1 } })); - try std.testing.expectEqual(@as(i32, -1), windowSizeLeft(.{ .standard = .{} })); - try std.testing.expectEqual(@as(i32, -1), windowSizeLeft(.none)); -} diff --git a/zml/attention/fly_kernels/sparse_mla.zig b/zml/attention/fly_kernels/sparse_mla.zig deleted file mode 100644 index f96293e1..00000000 --- a/zml/attention/fly_kernels/sparse_mla.zig +++ /dev/null @@ -1,458 +0,0 @@ -const std = @import("std"); -const zml = @import("../../zml.zig"); -const sparse_mla = @import("../sparse_mla.zig"); -const fly = zml.kernel.fly; -const Tensor = zml.Tensor; -const Value = fly.Value; - -pub const Config = struct { - queries: i64, - heads: i64, - pages: i64, - page_size: i64, - topk: i64, - splits: i64, - has_sink: bool, - all_decode: bool, -}; - -const main_inputs: []const [:0]const u8 = &.{ "query", "kv_cache", "sink", "indices", "active", "scale" }; -pub const Main2D = fly.Kernel(Config, .{ - .name = "sparse_mla_2d", - .inputs = main_inputs, - .outputs = &.{"output"}, - .run = runMain2D, -}); -pub const Main3D = fly.Kernel(Config, .{ - .name = "sparse_mla_3d", - .inputs = main_inputs, - .outputs = &.{ "partial_output", "partial_lse" }, - .run = runMain3D, -}); - -pub const ReduceConfig = struct { - queries: i64, - heads: i64, - input_splits: i64, - output_splits: i64, - has_sink: bool, - all_decode: bool, -}; -const reduce_inputs: []const [:0]const u8 = &.{ "input", "input_lse", "sink", "active" }; -pub const Reduce = fly.Kernel(ReduceConfig, .{ - .name = "sparse_mla_reduce", - .inputs = reduce_inputs, - .outputs = &.{"output"}, - .run = runReduce, -}); -pub const ReducePartials = fly.Kernel(ReduceConfig, .{ - .name = "sparse_mla_reduce_partials", - .inputs = reduce_inputs, - .outputs = &.{ "partial_output", "partial_lse" }, - .run = runReducePartials, -}); - -/// Shapes are local to a manual computation. Unsupported contracts fall back -/// before transposes or native launches enter the graph. -pub fn pagedAttention(q: Tensor, kv_cache: Tensor, sink: ?Tensor, indices: Tensor, active: Tensor, opts: sparse_mla.Options, cu_count: usize) ?Tensor { - if (q.rank() != 3 or q.dtype() != .bf16 or q.dim(.hd) != 512 or - q.dim(.q) <= 0 or q.dim(.h) <= 0 or opts.value_rank != 512 or - kv_cache.rank() != 4 or kv_cache.dtype() != .bf16 or kv_cache.dim(.hd) != 512 or - kv_cache.dim(.hkv) != 1 or kv_cache.dim(.page) <= 0 or kv_cache.dim(.k_chunk) <= 0 or - indices.rank() != 2 or indices.dtype() != .i32 or indices.dim(.q) != q.dim(.q) or indices.dim(.topk) <= 0 or - active.count() != 1 or active.dtype() != .i32) return null; - if (sink) |value| { - if (value.count() != @as(usize, @intCast(q.dim(.h)))) return null; - } - const config = sparse_mla.launchConfig(opts, @intCast(indices.dim(.topk)), cu_count); - if (config.block_m != 16 or config.tile_size == 0 or config.tile_size > 16 or - q.count() > std.math.maxInt(i32) / config.num_splits or indices.count() > std.math.maxInt(i32) or - kv_cache.count() / 4 > std.math.maxInt(i32)) return null; - return call( - q.transpose(.{ .q, .h, .hd }), - kv_cache.transpose(.{ .page, .k_chunk, .hkv, .hd }).squeeze(.hkv), - sink, - indices, - active, - opts, - config, - ).transpose(q.shape().tags()); -} - -/// Consumes local, row-major [query, head, 512] and [page, slot, 512] tensors. -/// Values span the full 512 columns; `config` supplies split/reduction settings. -pub fn call(q: Tensor, kv_cache: Tensor, sink: ?Tensor, indices: Tensor, active: Tensor, opts: sparse_mla.Options, config: sparse_mla.Config) Tensor { - std.debug.assert(q.rank() == 3 and q.dim(2) == 512 and q.dtype() == .bf16); - std.debug.assert(opts.value_rank == 512); - std.debug.assert(kv_cache.rank() == 3 and kv_cache.dim(2) == 512 and kv_cache.dtype() == .bf16); - std.debug.assert(indices.rank() == 2 and indices.dim(0) == q.dim(0) and indices.dtype() == .i32); - std.debug.assert(active.count() == 1 and active.dtype() == .i32); - std.debug.assert(config.block_m == 16 and config.tile_size > 0 and config.tile_size <= 16); - std.debug.assert(q.count() <= std.math.maxInt(i32) and indices.count() <= std.math.maxInt(i32)); - std.debug.assert(config.num_splits > 0 and q.count() <= std.math.maxInt(i32) / config.num_splits); - // Each i64 view element contains four adjacent BF16 values without copying. - // View counts are bounded; the kernel forms byte addresses in i64. - std.debug.assert(kv_cache.count() / 4 <= std.math.maxInt(i32)); - const cfg: Config = .{ - .queries = q.dim(0), - .heads = q.dim(1), - .pages = kv_cache.dim(0), - .page_size = kv_cache.dim(1), - .topk = indices.dim(1), - .splits = @intCast(config.num_splits), - .has_sink = sink != null, - .all_decode = opts.all_decode, - }; - const sink_ = if (sink) |value| value.convert(.f32).reshape(.{cfg.heads}) else Tensor.scalar(0, .f32).reshape(.{1}); - const inputs: Main3D.Inputs = .{ - .query = q.reshape(.{ cfg.queries, cfg.heads, 128, 4 }).bitCast(.i64), - .kv_cache = kv_cache.reshape(.{ cfg.pages, cfg.page_size, 128, 4 }).bitCast(.i64), - .sink = sink_, - .indices = indices.reshape(.{ cfg.queries, cfg.topk }), - .active = active.reshape(.{1}), - .scale = Tensor.scalar(opts.scale orelse @as(f32, 1.0 / @sqrt(512.0)), .f32).reshape(.{1}), - }; - const head_blocks = std.math.divCeil(i64, cfg.heads, 16) catch unreachable; - const grid: [3]i32 = .{ @intCast(cfg.queries * head_blocks), @intCast(cfg.splits), 1 }; - if (cfg.splits == 1) { - return Main2D.call(inputs, .{ .output = q.shape() }, .{ .cfg = cfg, .grid = grid, .threads = 256 }).output; - } - const partials = Main3D.call(inputs, .{ - .partial_output = zml.Shape.init(.{ cfg.queries, cfg.heads, cfg.splits, 512 }, .f32), - .partial_lse = zml.Shape.init(.{ cfg.queries, cfg.heads, cfg.splits }, .f32), - }, .{ .cfg = cfg, .grid = grid, .threads = 256 }); - var partial_output = partials.partial_output; - var partial_lse = partials.partial_lse; - var reduce_splits = cfg.splits; - if (cfg.splits > config.grouped_reduce_threshold) { - const output_splits = @divExact(cfg.splits, @as(i64, @intCast(config.splits_per_group))); - const grouped = ReducePartials.call(.{ - .input = partial_output, - .input_lse = partial_lse, - .sink = Tensor.scalar(0, .f32).reshape(.{1}), - .active = active.reshape(.{1}), - }, .{ - .partial_output = zml.Shape.init(.{ cfg.queries, cfg.heads, output_splits, 512 }, .f32), - .partial_lse = zml.Shape.init(.{ cfg.queries, cfg.heads, output_splits }, .f32), - }, .{ - .cfg = .{ .queries = cfg.queries, .heads = cfg.heads, .input_splits = cfg.splits, .output_splits = output_splits, .has_sink = false, .all_decode = cfg.all_decode }, - .grid = .{ @intCast(cfg.queries), @intCast(cfg.heads * output_splits), 1 }, - .threads = 64, - }); - partial_output = grouped.partial_output; - partial_lse = grouped.partial_lse; - reduce_splits = output_splits; - } - return Reduce.call(.{ - .input = partial_output, - .input_lse = partial_lse, - .sink = sink_, - .active = active.reshape(.{1}), - }, .{ .output = q.shape() }, .{ - .cfg = .{ .queries = cfg.queries, .heads = cfg.heads, .input_splits = reduce_splits, .output_splits = 1, .has_sink = cfg.has_sink, .all_decode = cfg.all_decode }, - .grid = .{ @intCast(cfg.queries), @intCast(cfg.heads), 1 }, - .threads = 64, - }).output; -} - -pub fn isAvailable(platform: *const zml.Platform) bool { - return sparse_mla.Backend.auto(platform, .bf16) == .fly; -} - -fn runMain2D(b: *fly.Builder, cfg: Config) fly.FinishError!void { - std.debug.assert(cfg.splits == 1); - try runMain(b, cfg, false, Main2D.args(b)); -} - -fn runMain3D(b: *fly.Builder, cfg: Config) fly.FinishError!void { - try runMain(b, cfg, true, Main3D.args(b)); -} - -fn loadVector(b: *fly.Builder, ptr: Value, offset: Value, comptime dtype: fly.DType, comptime count: i64) Value { - const atom = b.copyAtom(.{ .universal = @intCast(count * @as(i64, dtype.bitWidth())) }, dtype); - return b.copyAtomLoad(atom, ptr.addOffset(offset).view(fly.L(count, 1))); -} - -fn storeVector(b: *fly.Builder, ptr: Value, offset: Value, value: Value, comptime dtype: fly.DType, comptime count: i64) void { - const atom = b.copyAtom(.{ .universal = @intCast(count * @as(i64, dtype.bitWidth())) }, dtype); - b.copyAtomStore(atom, value, ptr.addOffset(offset).view(fly.L(count, 1))); -} - -// One CTA owns one query, sixteen heads and one KV split. The four waves -// partition the 512 contraction dimensions; their score sums meet in LDS. -fn runMain(b: *fly.Builder, cfg: Config, comptime three_d: bool, a: anytype) fly.FinishError!void { - const head_blocks = std.math.divCeil(i64, cfg.heads, 16) catch unreachable; - const tiles = std.math.divCeil(i64, cfg.topk, 16) catch unreachable; - const tiles_per_split = std.math.divCeil(i64, tiles, cfg.splits) catch unreachable; - std.debug.assert(cfg.queries > 0 and cfg.heads > 0 and cfg.topk > 0 and cfg.page_size > 0); - std.debug.assert(cfg.splits > 0 and cfg.splits <= tiles); - const zero = b.constant(.f32, 0.0); - const one = b.constant(.f32, 1.0); - const ninf = b.constant(.f32, -std.math.inf(f32)); - const zero4 = zero.splat(4); - const one4 = one.splat(4); - const ninf4 = ninf.splat(4); - const zero_bf16 = b.constant(.i16, 0).splat(4); - const zero_words = b.constant(.i64, 0).splat(2); - const block = b.blockId(.x).to(.i64); - const split = b.blockId(.y).to(.i64); - const qi = block.divUnsigned(head_blocks); - const head_base = block.remUnsigned(head_blocks).mul(16); - const active = b.constant(.i1, @intFromBool(cfg.all_decode)).bitOr(qi.cmp(.lt, a.active.at(0).to(.i64))); - const output = if (three_d) a.partial_output else a.output; - // Inactive queries leave their preallocated output rows untouched. - var enabled = b.openIf(active); - { - const tid = b.threadId(.x).to(.i64); - const lane = tid.remUnsigned(64); - const wave = tid.divUnsigned(64); - const mn = lane.remUnsigned(16); - const kg4 = lane.divUnsigned(16).mul(4); - const wave128 = wave.mul(128); - const query_head = head_base.add(mn); - const query_word = wave.mul(32).add(lane.divUnsigned(16)); - const valid_query_head = query_head.cmpUnsigned(.lt, cfg.heads); - var q: [8]Value = undefined; - for (0..8) |i| { - var valid = b.openIfElse(valid_query_head, .{zero_bf16.type_()}); - valid.yieldThen(.{a.query.at(.{ qi, query_head, query_word.add(i * 4) }).splat(1).bitcast(.i16)}); - valid.yieldElse(.{zero_bf16}); - q[i] = valid.results[0]; - } - const tile_start = split.mul(tiles).divUnsigned(cfg.splits); - const tile_end = split.add(1).mul(tiles).divUnsigned(cfg.splits); - const loader_key = lane.divUnsigned(4); - const loader_dimension = wave128.add(lane.remUnsigned(4).mul(32)); - const scale = a.scale.at(0).splat(4); - const cache_pointer = a.kv_cache.emitIter(); - const shared = b.sharedArray(.i16, 10496, 16); - const shared_ptr = shared.emitIter(); - const atom = b.mmaAtom((fly.rocdl.MmaOpCDNA3MFMAType.get(b.ctx, .{ - .m = 16, - .n = 16, - .k = 16, - .elemTyA = .float(b.ctx, .bf16), - .elemTyB = .float(b.ctx, .bf16), - .elemTyAcc = .float(b.ctx, .f32), - }) catch return error.InvalidMlir).type_()); - var loop = b.openFor(b.constant(.i64, 0), tiles_per_split, 1, .{ - zero4, zero4, zero4, zero4, zero4, zero4, zero4, zero4, ninf4, zero4, - }); - { - const tile_index = tile_start.add(loop.iv); - const load_pos = tile_index.mul(16).add(loader_key); - const valid_index = tile_index.cmpUnsigned(.lt, tile_end).bitAnd(load_pos.cmpUnsigned(.lt, cfg.topk)); - const invalid_index = b.constant(.i32, -1); - var index = b.openIfElse(valid_index, .{invalid_index.type_()}); - index.yieldThen(.{a.indices.at(.{ qi, load_pos })}); - index.yieldElse(.{invalid_index}); - const position = index.results[0]; - const load_valid = position.cmp(.ge, 0); - // Form the word offset in i64 before multiplying cache dimensions. - const page = position.to(.i64).divUnsigned(cfg.page_size); - const slot = position.to(.i64).remUnsigned(cfg.page_size); - const cache_base = page.mul(cfg.page_size).add(slot).mul(128).add(loader_dimension.divUnsigned(4)); - for (0..4) |i| { - var valid = b.openIfElse(load_valid, .{zero_words.type_()}); - valid.yieldThen(.{loadVector(b, cache_pointer, cache_base.add(i * 2), .i64, 2)}); - valid.yieldElse(.{zero_words}); - const bits = valid.results[0].bitcast(.i16); - const column = loader_dimension.add(i * 8).divUnsigned(8).bitXor(loader_key).mul(8); - storeVector(b, shared_ptr, loader_key.mul(512).add(column), bits, .i16, 8); - } - b.barrier(); - - var qk = zero4; - for (0..8) |i| { - const dimension = wave128.add(kg4).add(i * 16); - const column = dimension.divUnsigned(8).bitXor(mn).mul(8).add(dimension.remUnsigned(8)); - const k = loadVector(b, shared_ptr, mn.mul(512).add(column), .i16, 4); - qk = b.mmaAtomCall(atom, q[i], k, qk); - } - const score_lane = lane.mul(8); - storeVector(b, shared_ptr, wave.mul(512).add(score_lane).add(8192), qk.bitcast(.i16), .i16, 8); - b.barrier(); - var score_parts: [4]Value = undefined; - for (0..4) |w| score_parts[w] = loadVector(b, shared_ptr, score_lane.add(8192 + w * 512), .i16, 8).bitcast(.f32); - const score = score_parts[0].add(score_parts[1]).add(score_parts[2].add(score_parts[3])).mul(scale); - const key_valid = load_valid.to(.i32).shuffle(.idx, mn.mul(4).to(.i32), 64).cmp(.ne, 0); - var new_m = zero4; - var new_l = zero4; - var alpha = zero4; - var probability: [4]Value = undefined; - for (0..4) |i| { - const row = kg4.add(i); - const keep = head_base.add(row).cmpUnsigned(.lt, cfg.heads).bitAnd(key_valid); - const s = keep.select(score.extract(@intCast(i)), ninf); - var max = s; - inline for (.{ 1, 2, 4, 8 }) |shift| max = max.maxNum(max.shuffleXor(shift, 16)); - const old_m = loop.carried[8].extract(@intCast(i)); - const old_l = loop.carried[9].extract(@intCast(i)); - const next_max = old_m.maxNum(max); - const m = next_max.cmpf(.gt, ninf).select(next_max, zero); - const prob = b.exp(s.sub(m)); - var sum = prob; - inline for (.{ 1, 2, 4, 8 }) |shift| sum = sum.add(sum.shuffleXor(shift, 16)); - const rescale = b.exp(old_m.sub(m)); - new_m = new_m.insert(@intCast(i), m); - new_l = new_l.insert(@intCast(i), old_l.mul(rescale).add(sum)); - alpha = alpha.insert(@intCast(i), rescale); - probability[i] = prob; - } - var write_probability = b.openIf(wave.cmp(.eq, 0)); - for (0..4) |i| { - const offset = kg4.add(i).mul(16).add(mn).add(10240); - shared.set(offset, probability[i].to(.bf16).bitcast(.i16)); - } - write_probability.yieldThen(.{}); - b.barrier(); - - const p = loadVector(b, shared_ptr, mn.mul(16).add(kg4).add(10240), .i16, 4); - var next_acc: [8]Value = undefined; - for (0..8) |n| { - const dimension = wave128.add(mn).add(n * 16); - var v = zero_bf16; - for (0..4) |k| { - const row = kg4.add(k); - const column = dimension.divUnsigned(8).bitXor(row).mul(8).add(dimension.remUnsigned(8)); - v = v.insert(@intCast(k), shared.at(row.mul(512).add(column))); - } - next_acc[n] = b.mmaAtomCall(atom, p, v, loop.carried[n].mul(alpha)); - } - // All waves finish consuming K and P before the next tile reuses LDS. - b.barrier(); - loop.yield(.{ next_acc[0], next_acc[1], next_acc[2], next_acc[3], next_acc[4], next_acc[5], next_acc[6], next_acc[7], new_m, new_l }); - } - var denominator = loop.results[9]; - var sink_alpha = one4; - if (!three_d and cfg.has_sink) { - var sink = zero4; - for (0..4) |i| { - const head = head_base.add(kg4).add(i); - var valid = b.openIfElse(head.cmpUnsigned(.lt, cfg.heads), .{ninf.type_()}); - valid.yieldThen(.{a.sink.at(head)}); - valid.yieldElse(.{ninf}); - sink = sink.insert(@intCast(i), valid.results[0]); - } - const max = loop.results[8].maxNum(sink); - const delta = loop.results[8].sub(max); - const center = sink.sub(max); - var sink_probability = zero4; - sink_alpha = zero4; - for (0..4) |i| { - sink_alpha = sink_alpha.insert(@intCast(i), b.exp(delta.extract(@intCast(i)))); - sink_probability = sink_probability.insert(@intCast(i), b.exp(center.extract(@intCast(i)))); - } - denominator = denominator.mul(sink_alpha).add(sink_probability); - } - const has_values = denominator.cmpf(.gt, zero4); - const safe_denominator = has_values.select(denominator, one4); - const inverse = one4.div(safe_denominator); - for (0..8) |n| { - const acc = if (three_d) loop.results[n] else loop.results[n].mul(sink_alpha); - const normalized = has_values.select(acc.mul(inverse), zero4); - const col = wave128.add(mn).add(n * 16); - for (0..4) |i| { - const head = head_base.add(kg4).add(i); - var valid = b.openIf(head.cmpUnsigned(.lt, cfg.heads)); - const value = normalized.extract(@intCast(i)); - if (three_d) { - output.set(.{ qi, head, split, col }, value); - } else { - output.set(.{ qi, head, col }, value.to(.bf16)); - } - valid.yieldThen(.{}); - } - } - if (three_d) { - // Keep logarithms scalar: the published ROCm lowering rejects vector log. - var log_denominator = zero4; - for (0..4) |i| log_denominator = log_denominator.insert(@intCast(i), b.log(safe_denominator.extract(@intCast(i)))); - const lse = has_values.select(loop.results[8].add(log_denominator), ninf4); - var writer = b.openIf(wave.cmp(.eq, 0).bitAnd(mn.cmp(.eq, 0))); - for (0..4) |i| { - const head = head_base.add(kg4).add(i); - var valid = b.openIf(head.cmpUnsigned(.lt, cfg.heads)); - a.partial_lse.set(.{ qi, head, split }, lse.extract(@intCast(i))); - valid.yieldThen(.{}); - } - writer.yieldThen(.{}); - } - enabled.yieldThen(.{}); - } -} - -fn runReduce(b: *fly.Builder, cfg: ReduceConfig) fly.FinishError!void { - std.debug.assert(cfg.output_splits == 1); - runReduction(b, cfg, false, Reduce.args(b)); -} - -fn runReducePartials(b: *fly.Builder, cfg: ReduceConfig) fly.FinishError!void { - std.debug.assert(!cfg.has_sink); - runReduction(b, cfg, true, ReducePartials.args(b)); -} - -fn runReduction(b: *fly.Builder, cfg: ReduceConfig, comptime grouped: bool, a: anytype) void { - const group_splits = @divExact(cfg.input_splits, cfg.output_splits); - std.debug.assert(group_splits > 0 and group_splits <= 64); - const qi = b.blockId(.x).to(.i64); - const head_group = b.blockId(.y).to(.i64); - const head = head_group.divUnsigned(cfg.output_splits); - const group = head_group.remUnsigned(cfg.output_splits); - const group_start = group.mul(group_splits); - const active = b.constant(.i1, @intFromBool(cfg.all_decode)).bitOr(qi.cmp(.lt, a.active.at(0).to(.i64))); - const zero = b.constant(.f32, 0.0); - const one = b.constant(.f32, 1.0); - const ninf = b.constant(.f32, -std.math.inf(f32)); - const zero4 = zero.splat(4); - var enabled = b.openIf(active); - { - const lane = b.threadId(.x).to(.i64); - var valid_split = b.openIfElse(lane.cmpUnsigned(.lt, group_splits), .{ninf.type_()}); - valid_split.yieldThen(.{a.input_lse.at(.{ qi, head, group_start.add(lane) })}); - valid_split.yieldElse(.{ninf}); - const lse = valid_split.results[0]; - var max = lse; - inline for (.{ 32, 16, 8, 4, 2, 1 }) |shift| max = max.maxNum(max.shuffleXor(shift, 64)); - const sink = if (!grouped and cfg.has_sink) a.sink.at(head) else ninf; - const overall_max = max.maxNum(sink); - const has_values = overall_max.cmpf(.gt, ninf); - const safe_max = has_values.select(overall_max, zero); - const weight = b.exp(lse.sub(safe_max)); - var sum = weight; - inline for (.{ 32, 16, 8, 4, 2, 1 }) |shift| sum = sum.add(sum.shuffleXor(shift, 64)); - const sink_weight = if (!grouped and cfg.has_sink) b.exp(sink.sub(safe_max)) else zero; - const denominator = sum.add(sink_weight); - const safe_denominator = denominator.cmpf(.gt, zero).select(denominator, one); - const normalizer = one.div(safe_denominator).splat(4); - const group_flat = qi.mul(cfg.heads).add(head).mul(cfg.input_splits).add(group_start); - const dim0 = lane.mul(8); - var loop = b.openFor(b.constant(.i64, 0), group_splits, 1, .{ zero4, zero4 }); - { - const weight4 = weight.shuffle(.idx, loop.iv.to(.i32), 64).splat(4); - const base = group_flat.add(loop.iv).mul(512).add(dim0); - const part0 = loadVector(b, a.input.emitIter(), base, .f32, 4); - const part1 = loadVector(b, a.input.emitIter(), base.add(4), .f32, 4); - loop.yield(.{ loop.carried[0].add(part0.mul(weight4)), loop.carried[1].add(part1.mul(weight4)) }); - } - const output = if (grouped) a.partial_output else a.output; - const row = qi.mul(cfg.heads).add(head); - const base = if (grouped) row.mul(cfg.output_splits).add(group).mul(512) else row.mul(512); - for (0..2) |i| { - const value = has_values.select(loop.results[i].mul(normalizer), zero4); - const offset = base.add(dim0).add(i * 4); - if (grouped) { - storeVector(b, output.emitIter(), offset, value, .f32, 4); - } else { - storeVector(b, output.emitIter(), offset, value.to(.bf16), .bf16, 4); - } - } - if (grouped) { - var writer = b.openIf(lane.cmp(.eq, 0)); - const lse_value = has_values.select(safe_max.add(b.log(safe_denominator)), ninf); - a.partial_lse.set(.{ qi, head, group }, lse_value); - writer.yieldThen(.{}); - } - enabled.yieldThen(.{}); - } -} diff --git a/zml/attention/metal_attention.zig b/zml/attention/metal_attention.zig deleted file mode 100644 index fa1e1929..00000000 --- a/zml/attention/metal_attention.zig +++ /dev/null @@ -1,113 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -const zml = @import("../zml.zig"); -const AttentionOptions = @import("paged_attention.zig").AttentionOptions; - -const log = std.log.scoped(.@"zml/attention/metal"); - -pub const Metadata = struct { - num_tokens: zml.Tensor, - - pub fn init() Metadata { - return .{ .num_tokens = .fromShape(.scalar(.u32)) }; - } - - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { - _ = self; - return .{ .num_tokens = try zml.Buffer.scalar(io, platform, 0, .u32) }; - } - - pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { - self.num_tokens.deinit(); - } -}; - -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata) zml.Tensor { - const qc = q.transpose(.{ .h, .q, .hd }); - const kc = k.transpose(.{ .h, .k, .hd }); - const vc = v.transpose(.{ .h, .k, .hd }); - const tok_i32 = token_index.convert(.i32); - - const attn = if (q.dim(.q) > 1) - zml.ops.customCall("zml$flash_attn", .{ qc, kc, vc, tok_i32, metadata.num_tokens }, qc.shape(), .{}, .{ .has_side_effect = false }) - else - zml.ops.customCall("zml$flash_attn", .{ qc, kc, vc, tok_i32 }, qc.shape(), .{}, .{ .has_side_effect = false }); - - return attn.transpose(q.shape()); -} - -pub const paged = struct { - pub const Options = struct { - batch_size: usize, - max_num_pages: usize, - max_seqlen_q: usize, - is_prefill: bool, - - pub fn isPrefill(self: Options) bool { - return self.is_prefill; - } - - pub fn maxNumPages(self: Options) usize { - return self.max_num_pages; - } - }; - - pub const Parameters = struct { - block_table: zml.Tensor, - seq_lens: zml.Tensor, - query_start_len: zml.Tensor, - options_: Options, - - pub fn init(options_: Options) Parameters { - return .{ - .block_table = .init(.{ .b = options_.batch_size, .p = options_.max_num_pages }, .i32), - .seq_lens = .init(.{ .b = options_.batch_size }, .i32), - .query_start_len = .init(.{ .b = options_.batch_size + 1 }, .i32), - .options_ = options_, - }; - } - - pub fn allocationSize(self: Parameters) usize { - var allocation_size: usize = 0; - - allocation_size += self.block_table.byteSize(); - allocation_size += self.seq_lens.byteSize(); - allocation_size += self.query_start_len.byteSize(); - - return allocation_size; - } - - pub fn options(self: Parameters) Options { - return self.options_; - } - }; - - pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { - if (opts.mask == .indices) @panic("Sparse Attention is not implemented on Metal"); - - const num_kv_heads = q.dim(.hkv); - - const qh = q.merge(.{ .h = .{ .hkv, .hg } }).transpose(.{ .b, .h, .hd }); - const kc = k_cache.transpose(.{ .page, .k_chunk, .hkv, .hd }); - const vc = v_cache.transpose(.{ .page, .k_chunk, .hkv, .hd }); - - const sliding_window: i32 = if (opts.mask.slidingWindow()) |window| @intCast(window) else -1; - - const out = zml.ops.customCall( - "zml$paged_attn", - .{ qh, kc, vc, parameters.block_table, parameters.seq_lens, parameters.query_start_len }, - qh.shape(), - .{ - .is_causal = opts.mask.isCausal(), - .sliding_window = sliding_window, - .scale = opts.scale, - }, - .{ .has_side_effect = false }, - ); - - // [total_q_tokens, num_heads, head_dim] -> {.b, .hkv, .hg, .hd} (q's order) - return out.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = .auto }).transpose(q.shape()); - } -}; diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig deleted file mode 100644 index 5e752a81..00000000 --- a/zml/attention/paged_attention.zig +++ /dev/null @@ -1,1232 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -const zml = @import("../zml.zig"); -const cute = @import("cute_kernels/sparse_mla.zig"); -const flashattn = @import("flashattn.zig"); -const metal = @import("metal_attention.zig"); -const sparse_mla = @import("sparse_mla.zig"); -const tpu = @import("tpu_attention.zig"); -const triton = @import("triton_attention.zig"); - -const PagedAttention = @This(); - -test { - std.testing.refAllDecls(Backend); - std.testing.refAllDecls(Options); - std.testing.refAllDecls(Parameters); - std.testing.refAllDecls(KvCache); -} - -pub const Backend = enum { - cuda_fa2, - cuda_fa3, - triton, - mosaic_tpu, - metal, - stablehlo, - - pub fn auto(platform: *const zml.Platform) Backend { - return switch (platform.target) { - .cuda => .triton, - .rocm => .triton, - .oneapi => .triton, - .tpu => .mosaic_tpu, - .metal => .metal, - .cpu, .furiosa => .stablehlo, - .neuron => stdx.debug.panic("Paged attention is not supported on {s} yet", .{@tagName(platform.target)}), - }; - } - - pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { - return switch (backend) { - .stablehlo => true, - .triton => platform.target != .cpu, - .metal => platform.target == .metal, - .mosaic_tpu => platform.target == .tpu, - .cuda_fa2 => platform.target == .cuda, - .cuda_fa3 => if (zml.platform.cuda.computeCapability(platform)) |cc| cc.eql(.{ .major = 9, .minor = 0 }) else false, - }; - } -}; - -pub const Options = union(Backend) { - cuda_fa2: flashattn.paged_fa2.Options, - cuda_fa3: flashattn.paged_fa3.Options, - triton: triton.paged.Options, - mosaic_tpu: tpu.mosaic_tpu.Options, - metal: metal.paged.Options, - stablehlo: triton.paged.Options, - - const Args = struct { - backend: Backend, - is_prefill: bool, - batch_size: u32, - batch_size_prefill: ?u32 = null, - batch_size_decode: ?u32 = null, - seq_len: u32, - max_num_pages: u32, - max_token_count: u32, - num_heads: u32, - num_kv_heads: u32, - head_dim: u32, - max_seqlen_q: u32, - }; - - pub fn fromBackend(args: Args) Options { - return switch (args.backend) { - .cuda_fa2 => if (args.is_prefill) .{ - .cuda_fa2 = .{ - .mixed = .{ - .batch_size_decode = args.batch_size_decode orelse args.batch_size, - .batch_size_prefill = args.batch_size_prefill orelse args.batch_size, - .max_num_pages = args.max_num_pages, - .max_seqlen_k = args.seq_len, - .max_seqlen_q = args.max_seqlen_q, - .max_token_count = args.max_token_count, - .num_heads = args.num_heads, - .num_kv_heads = args.num_kv_heads, - .head_dim = args.head_dim, - }, - }, - } else .{ - .cuda_fa2 = .{ - .decode = .{ - .batch_size = args.batch_size, - .max_num_pages = args.max_num_pages, - .max_seqlen_k = args.seq_len, - .max_token_count = args.max_token_count, - .num_heads = args.num_heads, - .num_kv_heads = args.num_kv_heads, - .head_dim = args.head_dim, - }, - }, - }, - .cuda_fa3 => if (args.is_prefill) .{ - .cuda_fa3 = .{ - .mixed = .{ - .batch_size_decode = args.batch_size_decode orelse args.batch_size, - .batch_size_prefill = args.batch_size_prefill orelse args.batch_size, - .max_num_pages = args.max_num_pages, - .max_seqlen_k = args.seq_len, - .max_seqlen_q = args.max_seqlen_q, - .max_token_count = args.max_token_count, - .num_heads = args.num_heads, - .num_kv_heads = args.num_kv_heads, - .head_dim = args.head_dim, - }, - }, - } else .{ - .cuda_fa3 = .{ - .decode = .{ - .batch_size = args.batch_size, - .max_num_pages = args.max_num_pages, - .max_seqlen_k = args.seq_len, - .max_token_count = args.max_token_count, - .num_heads = args.num_heads, - .num_kv_heads = args.num_kv_heads, - .head_dim = args.head_dim, - }, - }, - }, - .mosaic_tpu => .{ - .mosaic_tpu = .{ - .is_prefill = args.is_prefill, - .batch_size = args.batch_size, - .max_num_pages = args.max_num_pages, - .max_seqlen_k = args.seq_len, - .max_token_count = args.max_token_count, - .num_heads = args.num_heads, - .num_kv_heads = args.num_kv_heads, - .head_dim = args.head_dim, - }, - }, - inline .triton, .metal, .stablehlo => |t| @unionInit(Options, @tagName(t), .{ - .batch_size = args.batch_size, - .max_num_pages = args.max_num_pages, - .max_seqlen_q = args.max_seqlen_q, - .is_prefill = args.is_prefill, - }), - }; - } - - pub fn isPrefill(self: Options) bool { - return switch (self) { - inline else => |v| v.isPrefill(), - }; - } - - pub fn maxNumPages(self: Options) usize { - return switch (self) { - inline else => |v| v.maxNumPages(), - }; - } -}; - -pub const Parameters = union(Backend) { - cuda_fa2: flashattn.paged_fa2.Parameters, - cuda_fa3: flashattn.paged_fa3.Parameters, - triton: triton.paged.Parameters, - mosaic_tpu: tpu.mosaic_tpu.Parameters, - metal: metal.paged.Parameters, - stablehlo: triton.paged.Parameters, - - pub fn init(options_: Options) Parameters { - return switch (options_) { - inline else => |opts, backend| @unionInit(Parameters, @tagName(backend), .init(opts)), - }; - } - - pub fn options(self: Parameters) Options { - return switch (self) { - inline else => |v, backend| @unionInit(Options, @tagName(backend), v.options()), - }; - } - - pub fn allocationSize(self: Parameters) usize { - return switch (self) { - inline else => |v| v.allocationSize(), - }; - } - - pub fn onMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { - zml.Tensor.onMemoryAll(&self, memory); - return self; - } - - pub fn toMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { - return zml.Tensor.toMemoryAll(self, memory); - } -}; - -pub const AttentionMask = union(enum) { - standard: struct { - /// Restrict attention to keys at or before the query position. - causal: bool = true, - /// Optional left window size, including the current token. - /// Non-causal attention also includes every later valid key. - sliding_window: ?u32 = null, - }, - /// Attend to keys at specific positions. - indices: zml.Tensor, - - /// Attend to every valid key in the sequence. - pub const none: AttentionMask = .{ .standard = .{ .causal = false } }; - - pub fn isCausal(mask: AttentionMask) bool { - return switch (mask) { - .standard => |options| options.causal, - .indices => false, - }; - } - - pub fn slidingWindow(mask: AttentionMask) ?u32 { - return switch (mask) { - .standard => |options| options.sliding_window, - .indices => null, - }; - } -}; - -pub const AttentionOptions = struct { - mask: AttentionMask = .{ .standard = .{} }, - scale: ?f32 = null, - sink: ?zml.Tensor = null, -}; - -pub const KvCache = union(enum) { - split: struct { - k: zml.Tensor, - v: zml.Tensor, - }, - // TODO: rename `dense` to `fused`. - dense: zml.Tensor, - latent: zml.Tensor, - - pub fn update( - self: KvCache, - new_k: zml.Tensor, - new_v: zml.Tensor, - slot_mapping: zml.Tensor, - chunk_size: u32, - backend: Backend, - ) KvCache { - const page_index, const offset = getPageAndOffsetFromSlotMapping(slot_mapping, chunk_size); - - const kv: KvCache = switch (self) { - .split => |split| switch (backend) { - .cuda_fa2, .cuda_fa3, .triton, .mosaic_tpu, .metal, .stablehlo => .{ - .split = .{ - .k = split.k.scatterSlices( - .{ .page = page_index, .k_chunk = offset }, - new_k, - .{ .update_fn = zml.Tensor.ScatterOpts.override, .indices_are_unique = false, .indices_are_sorted = false }, - ).reuseBuffer(self.split.k), - .v = split.v.scatterSlices( - .{ .page = page_index, .k_chunk = offset }, - new_v, - .{ .update_fn = zml.Tensor.ScatterOpts.override, .indices_are_unique = false, .indices_are_sorted = false }, - ).reuseBuffer(self.split.v), - }, - }, - }, - .dense => @panic("TODO"), - .latent => |latent_kv| .{ - .latent = latent_kv.scatterSlices( - .{ .page = page_index, .k_chunk = offset }, - new_k, - .{ .update_fn = zml.Tensor.ScatterOpts.override, .indices_are_unique = false, .indices_are_sorted = false }, - ).reuseBuffer(self.latent), - }, - }; - - return kv; - } - - pub fn getPageAndOffsetFromSlotMapping(slot_mapping: zml.Tensor, page_size: u32) [2]zml.Tensor { - const page_index = slot_mapping.divByConst(page_size); - const offset = slot_mapping.remainder(.scalar(page_size, slot_mapping.dtype())); - return .{ page_index, offset }; - } - - pub fn getPage( - self: KvCache, - page_index: zml.Tensor, - ) [2]zml.Tensor { - return switch (self) { - .split => |split| .{ - split.k.slice(split.k.axis(.page), .dynSingle(page_index)), - split.v.slice(split.k.axis(.page), .dynSingle(page_index)), - }, - .dense, .latent => @panic("TODO"), - }; - } -}; - -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache, opts: AttentionOptions) zml.Tensor { - _ = k; - _ = v; - return switch (parameters) { - .cuda_fa2 => |cuda_fa2_parameters| switch (kv_cache) { - .split => |split| flashattn.paged_fa2.pagedAttention(cuda_fa2_parameters, q, split.k, split.v, opts), - .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), - .latent => std.debug.panic("latent KV pages are only supported with Multi-Latent Attention", .{}), - }, - .cuda_fa3 => |cuda_fa3_parameters| switch (kv_cache) { - .split => |split| flashattn.paged_fa3.pagedAttention(cuda_fa3_parameters, q, split.k, split.v, opts), - .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), - .latent => std.debug.panic("latent KV pages are only supported with Multi-Latent Attention", .{}), - }, - .triton => |triton_parameters| switch (kv_cache) { - .split => |split| triton.paged.pagedAttention(triton_parameters, q, split.k, split.v, opts), - .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), - .latent => std.debug.panic("latent KV pages are only supported with Multi-Latent Attention", .{}), - }, - .mosaic_tpu => |mosaic_tpu_parameters| switch (kv_cache) { - .split => std.debug.panic("split KV pages is not supported with the mosaic_tpu backend", .{}), - .latent => std.debug.panic("latent KV pages are only supported with Multi-Latent Attention", .{}), - .dense => tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, q, kv_cache.dense, opts), - }, - .metal => |metal_parameters| switch (kv_cache) { - .split => |split| metal.paged.pagedAttention(metal_parameters, q, split.k, split.v, opts), - .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), - .latent => std.debug.panic("latent KV pages are only supported with Multi-Latent Attention", .{}), - }, - .stablehlo => |params| switch (kv_cache) { - .latent => std.debug.panic("latent KV pages are only supported with Multi-Latent Attention", .{}), - else => stablehlo_pagedAttention(params, q, kv_cache, opts), - }, - }; -} - -test "Backend.auto selects mosaic_tpu on TPU" { - const platform: zml.Platform = .{ - .arena = undefined, - .target = .tpu, - .pjrt_api = undefined, - .pjrt_client = undefined, - .state = zml.platform.State.init(.tpu), - .devices = &.{}, - .memories = &.{}, - .physical_mesh = undefined, - .replicated_mesh = undefined, - .meshes = .empty, - .io_impl = .threaded, - }; - - try std.testing.expectEqual(Backend.mosaic_tpu, Backend.auto(&platform)); -} - -test "Backend.auto selects triton on oneAPI" { - const platform: zml.Platform = .{ - .arena = undefined, - .target = .oneapi, - .pjrt_api = undefined, - .pjrt_client = undefined, - .state = zml.platform.State.init(.oneapi), - .devices = &.{}, - .memories = &.{}, - .physical_mesh = undefined, - .replicated_mesh = undefined, - .meshes = .empty, - .io_impl = .threaded, - }; - - try std.testing.expectEqual(Backend.triton, Backend.auto(&platform)); -} - -test pagedAttention { - const platform = zml.testing.env(); - const io = std.testing.io; - const allocator = std.testing.allocator; - var arena_state: std.heap.ArenaAllocator = .init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - const num_pages = 64; - const page_size = 16; - const max_num_pages = 16; - const prefill_token_count = 32; - const num_prefill = 1; - const num_decode = 6; - const active_batch_size = num_prefill + num_decode; - const batch_size = active_batch_size + 1; - const query_token_count = prefill_token_count + num_decode; - const dt: zml.DataType = .bf16; - const model_meshe = platform.meshes.get("model").?; - const partition = .{ .hkv = .model }; - const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache } = .{ - .q = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hg = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), - .k = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), - .v = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), - .kv_cache = .{ - .split = .{ - .k = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), - .v = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), - }, - }, - }; - - const meshes: []const *const zml.Mesh = &.{ platform.replicated_mesh, platform.meshes.get("model").? }; - const rng_q = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.q.shape(), .{} }, .{ .meshes = meshes }); - defer rng_q.deinit(); - const rng_k = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.k.shape(), .{} }, .{ .meshes = meshes }); - defer rng_k.deinit(); - const rng_kv_cache = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.kv_cache.split.k.shape(), .{} }, .{ .meshes = meshes }); - defer rng_kv_cache.deinit(); - - const triton_options_args: Options.Args = .{ - .backend = .triton, - .is_prefill = true, - .batch_size = batch_size, - .batch_size_prefill = num_prefill, - .batch_size_decode = num_decode, - .seq_len = max_num_pages * page_size, - .max_num_pages = max_num_pages, - .max_token_count = 16 * page_size, - .num_heads = @intCast(tensors.q.dim(.hg) * tensors.q.dim(.hkv)), - .num_kv_heads = @intCast(tensors.q.dim(.hkv)), - .head_dim = @intCast(tensors.q.dim(.hd)), - .max_seqlen_q = 16 * 2, - }; - const triton_parameters: Parameters = .init(.fromBackend(triton_options_args)); - var q = try rng_q.eval(allocator, io, {}); - defer q.deinit(); - var new_k = try rng_k.eval(allocator, io, {}); - defer new_k.deinit(); - var new_v = try rng_k.eval(allocator, io, {}); - defer new_v.deinit(); - - var kv_cache_d: zml.Bufferized(KvCache) = .{ .split = undefined }; - kv_cache_d.split.k = try rng_kv_cache.eval(allocator, io, .{}); - defer kv_cache_d.split.k.deinit(); - kv_cache_d.split.v = try rng_kv_cache.eval(allocator, io, .{}); - defer kv_cache_d.split.v.deinit(); - - const block_table: [batch_size][max_num_pages]i32 = .{ - // prefilling pages 9-10 - .{ 0, 1, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, - // generation - .{ 0, 1, 2, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, - .{ 0, 1, 2, 3, 4, 6, 7, 8, -1, -1, -1, -1, -1, -1, -1, -1 }, - .{ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63 }, - .{ 0, 1, 2, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, - .{ 0, 1, 2, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, - .{ 0, 1, 2, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, - .{ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, - }; - - const seq_lens: [batch_size]i32 = .{ - 4 * page_size, - 4 * page_size + 3, - 7 * page_size + 11, - 15 * page_size + 9, - 4 * page_size + 3, - 4 * page_size + 3, - 4 * page_size + 3, - 0, - }; - - const query_start_len: [batch_size + 1]i32 = .{ 0, 32, 33, 34, 35, 36, 37, 38, 38 }; - var triton_parameters_d: zml.Bufferized(Parameters) = .{ .triton = .{ - .block_table = try .fromBytes(io, platform, triton_parameters.triton.block_table.shape(), @ptrCast(&block_table)), - .seq_lens = try .fromBytes(io, platform, triton_parameters.triton.seq_lens.shape(), @ptrCast(&seq_lens)), - .query_start_len = try .fromBytes(io, platform, triton_parameters.triton.query_start_len.shape(), @ptrCast(&query_start_len)), - } }; - defer zml.Buffer.deinitAll(Parameters, &triton_parameters_d); - - const all_backends = std.enums.values(Backend); - // stablehlo ignores these options, cuda_fa3 has no materializer in this fixture, and - // mosaic_tpu requires a dense KV cache instead of the split cache used by this test. - const option_sensitive_backends = [_]Backend{ .cuda_fa2, .triton, .metal }; - const test_cases = [_]struct { - name: []const u8, - attention_options: AttentionOptions, - backends: []const Backend, - }{ - .{ - .name = "unbounded", - .attention_options = .{ .mask = .{ .standard = .{} } }, - .backends = all_backends, - }, - .{ - .name = "non_causal", - .attention_options = .{ .mask = .{ .standard = .{ .causal = false } } }, - .backends = &option_sensitive_backends, - }, - .{ - .name = "sliding_window", - .attention_options = .{ .mask = .{ .standard = .{ .sliding_window = page_size } } }, - .backends = &option_sensitive_backends, - }, - }; - - for (test_cases) |test_case| { - var results_per_backend: std.enums.EnumArray(Backend, ?zml.Slice) = .initFill(null); - defer { - for (std.enums.values(Backend)) |backend| { - if (results_per_backend.get(backend)) |slice| slice.free(allocator); - } - } - - for (test_case.backends) |backend| { - if (!backend.isAvailable(platform)) { - std.log.warn("paged_attention backend {t} not available", .{backend}); - continue; - } - std.log.warn("Testing paged_attention {s} with backend {t}", .{ test_case.name, backend }); - - var backend_options_args = triton_options_args; - backend_options_args.backend = backend; - const parameters: Parameters = .init(.fromBackend(backend_options_args)); - - const exe = try platform.compileFn( - allocator, - io, - pagedAttention, - .{ parameters, tensors.q, tensors.k, tensors.v, tensors.kv_cache, test_case.attention_options }, - .{ .program_name = try std.fmt.allocPrint(arena, "paged_attention_{s}_{t}", .{ test_case.name, backend }), .meshes = meshes }, - ); - defer exe.deinit(); - - var parameters_d: zml.Bufferized(Parameters) = switch (parameters) { - // No materializer implemented for cuda fa3 - .cuda_fa3 => continue, - .cuda_fa2 => |params| cuda_fa2: { - var block_table_prefill: [num_prefill][max_num_pages]i32 = undefined; - @memcpy(&block_table_prefill, block_table[0..num_prefill]); - var block_table_decode: [num_decode][max_num_pages]i32 = undefined; - @memcpy(&block_table_decode, block_table[num_prefill .. num_prefill + num_decode]); - - var cu_seqlens_q_prefill: [num_prefill + 1]i32 = undefined; - @memcpy(&cu_seqlens_q_prefill, query_start_len[0 .. num_prefill + 1]); - var cu_seqlens_q_decode: [num_decode + 1]i32 = undefined; - for (&cu_seqlens_q_decode, query_start_len[num_prefill .. num_prefill + num_decode + 1]) |*decode_len, query_start| { - decode_len.* = query_start - prefill_token_count; - } - - var seqused_k_prefill: [num_prefill]i32 = undefined; - @memcpy(&seqused_k_prefill, seq_lens[0..num_prefill]); - var seqused_k_decode: [num_decode]i32 = undefined; - @memcpy(&seqused_k_decode, seq_lens[num_prefill .. num_prefill + num_decode]); - - break :cuda_fa2 .{ .cuda_fa2 = .{ .mixed = .{ - .block_table_prefill = try .fromBytes(io, platform, params.mixed.block_table_prefill.shape(), @ptrCast(&block_table_prefill)), - .cu_seqlens_q_prefill = try .fromBytes(io, platform, params.mixed.cu_seqlens_q_prefill.shape(), @ptrCast(&cu_seqlens_q_prefill)), - .seqused_k_prefill = try .fromBytes(io, platform, params.mixed.seqused_k_prefill.shape(), @ptrCast(&seqused_k_prefill)), - - .block_table_decode = try .fromBytes(io, platform, params.mixed.block_table_decode.shape(), @ptrCast(&block_table_decode)), - .cu_seqlens_q_decode = try .fromBytes(io, platform, params.mixed.cu_seqlens_q_decode.shape(), @ptrCast(&cu_seqlens_q_decode)), - .seqused_k_decode = try .fromBytes(io, platform, params.mixed.seqused_k_decode.shape(), @ptrCast(&seqused_k_decode)), - - .metadata = .{ .decode_offset = try .scalar(io, platform, prefill_token_count, .i32) }, - } } }; - }, - .triton => triton_parameters_d, - inline .metal, .mosaic_tpu, .stablehlo => |_, t| @unionInit(zml.Bufferized(Parameters), @tagName(t), .{ - .block_table = triton_parameters_d.triton.block_table, - .seq_lens = triton_parameters_d.triton.seq_lens, - .query_start_len = triton_parameters_d.triton.query_start_len, - }), - }; - // cu_fa2 creates new buffers while other reuse triton buffers. - defer if (backend == .cuda_fa2) zml.Buffer.deinitAll(Parameters, ¶meters_d); - - var output_d = try exe.eval(allocator, io, .{ parameters_d, q, new_k, new_v, kv_cache_d, .{} }); - defer output_d.deinit(); - results_per_backend.set(backend, try output_d.toSliceAlloc(allocator, io)); - } - - const reference_backend: Backend = .triton; - const reference = results_per_backend.get(reference_backend) orelse return error.SkipZigTest; - - var num_failed: u32 = 0; - for (std.enums.values(Backend)) |backend| { - if (backend == reference_backend) continue; - const output_h = results_per_backend.get(backend) orelse continue; - - const tolerance: zml.testing.CompareOpts = .{ - .absolute_tolerance = 1e-2, - .relative_tolerance = 1e-2, - .epsilon_relative = 1e-6, - }; - - zml.testing.expectClose(io, reference, output_h, tolerance) catch |err| switch (err) { - error.TestUnexpectedResult => { - num_failed += 1; - std.log.err("test pagedAttention {s} failed on backend {t} against reference {t}", .{ - test_case.name, - backend, - reference_backend, - }); - }, - else => |e| return e, - }; - } - - if (num_failed > 0) return error.TestUnexpectedResult; - } -} - -fn stablehlo_pagedAttention( - parameters: triton.paged.Parameters, - q: zml.Tensor, - kv_cache: KvCache, - opts: AttentionOptions, -) zml.Tensor { - switch (opts.mask) { - .standard => |options| if (options.sliding_window != null) @panic("Sliding window attention mask is not supported on StableHLO"), - .indices => @panic("Sparse attention mask is not supported on StableHLO"), - } - - const page_size = kv_cache.split.k.dim(.k_chunk); - - const final_state = zml.ops.@"while"( - AttentionLoop, - .{ .q = q, .kv_cache = kv_cache, .parameters = parameters, .opts = opts }, - .{ - // i32 to match triton conventions - .slot_id = .scalar(0, .i32), - .seq_id = .scalar(0, .i32), - .q_page_idx = .scalar(0, .i32), - .k_page_idx = .scalar(0, .i32), - // Note: we do all partial softmax in f32 - .partial_softmax_prefill = .zeroes(q.shape().setDim(.b, page_size).withDtype(.f32)), - .partial_softmax_decode = .zeroes(q.shape().setDim(.b, 1).withDtype(.f32)), - .out = .zeroes(q.shape()), - }, - ); - - return final_state.out; -} - -const AttentionLoop = struct { - q: zml.Tensor, - kv_cache: KvCache, - parameters: triton.paged.Parameters, - opts: AttentionOptions, - - const While = @This(); - - pub const State = struct { - slot_id: zml.Tensor, - seq_id: zml.Tensor, - q_page_idx: zml.Tensor, - k_page_idx: zml.Tensor, - partial_softmax_prefill: PartialSoftmax, - partial_softmax_decode: PartialSoftmax, - out: zml.Tensor, - }; - - pub fn cond(self: While, state: State) zml.Tensor { - const has_more_sequences = state.seq_id.cmp(.LT, .scalar(self.parameters.seq_lens.count(), .i32)); - const query_start = self.parameters.query_start_len.slice(0, .dynSingle(state.seq_id)); - const query_end = self.parameters.query_start_len.slice(0, .dynSingle(state.seq_id.addConstant(1))); - const has_queries = query_start.cmp(.LT, query_end).asScalar(); - return has_more_sequences.logical(.AND, has_queries); - } - - pub fn body(self: While, state_: State) State { - const query_start = self.parameters.query_start_len.slice(0, .dynSingle(state_.seq_id)); - const query_end = self.parameters.query_start_len.slice(0, .dynSingle(state_.seq_id.addConstant(1))); - const seq_num_queries_: zml.Tensor = .asScalar(.sub(query_end, query_start)); - const page_size_: u32 = @intCast(self.kv_cache.split.k.dim(.k_chunk)); - - return zml.ops.@"if"( - struct { - const IfPrefill = @This(); - while_body: While, - state: State, - seq_num_queries: zml.Tensor, - page_size: u32, - - /// Prefill version: num_slots == page_size, we multiply a full page of queries with a page of keys. - pub fn onTrue(if_ctx: IfPrefill) State { - const while_body = if_ctx.while_body; - const state = if_ctx.state; - const page_size = if_ctx.page_size; - const num_slots: u32 = page_size; - - // seq_lens includes the current queries; subtracting their count gives the context length. - const seq_len = while_body.parameters.seq_lens.slice(0, .dynSingle(state.seq_id)); - const q_offset = seq_len.sub(if_ctx.seq_num_queries).add(state.q_page_idx.scale(page_size)); - const next_k_offset = state.k_page_idx.addConstant(1).scale(page_size); - - // The current K page is processed in this iteration. Continue if the next page contains - // any key visible to the current query chunk, whose exclusive end is q_offset + num_slots. - const query_has_more_keys: zml.Tensor = .cmp(next_k_offset, .LT, q_offset.addConstant(num_slots)); - const sequence_has_more_queries: zml.Tensor = .cmp(state.q_page_idx.addConstant(1).scale(num_slots), .LT, if_ctx.seq_num_queries); - - const partial_softmax = while_body.attentionOnePage(state, q_offset, num_slots, page_size); - return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax); - } - - /// Decode version: num_slots == 1, we multiply one query with a page of keys. - pub fn onFalse(if_ctx: IfPrefill) State { - const while_body = if_ctx.while_body; - const state = if_ctx.state; - const page_size = if_ctx.page_size; - const num_slots: u32 = 1; - - // A decode query is the final token in seq_len, so its zero-based offset is seq_len - 1. - const seq_len = while_body.parameters.seq_lens.slice(0, .dynSingle(state.seq_id)).asScalar(); - const q_offset = seq_len.sub(if_ctx.seq_num_queries); - const next_k_offset = state.k_page_idx.addConstant(1).scale(page_size); - - const query_has_more_keys: zml.Tensor = .cmp(next_k_offset, .LT, q_offset.addConstant(num_slots)); - const sequence_has_more_queries: zml.Tensor = .scalar(false, .bool); - - const partial_softmax = while_body.attentionOnePage(state, q_offset, num_slots, page_size); - return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax); - } - }, - .{ - .while_body = self, - .state = state_, - .seq_num_queries = seq_num_queries_, - .page_size = page_size_, - }, - .cmp(seq_num_queries_, .GT, .scalar(1, .i32)), - ); - } - - pub fn attentionOnePage(self: While, state: State, q_offset: zml.Tensor, q_chunk: u32, page_size: u32) PartialSoftmax { - const k_offset = state.k_page_idx.scale(page_size); - const active_q = self.q.slice(self.q.axis(.b), .dyn(state.slot_id, q_chunk)); - - const active_k_page_id = self.parameters.block_table.slices( - .{ .b, .p }, - &.{ .dynSingle(state.seq_id), .dynSingle(state.k_page_idx) }, - ); - const active_k, const active_v = self.kv_cache.getPage(active_k_page_id); - - const dtype = self.q.dtype(); - const attn_mask: zml.Tensor = attn_mask: { - const mask_shape: zml.Shape = .init(.{ .b = q_chunk, .k_chunk = page_size }, dtype); - - const q_idx = zml.Tensor.iota(mask_shape, .b).add(q_offset); - const k_idx = zml.Tensor.iota(mask_shape, .k_chunk).add(k_offset); - - const mask = zml.Tensor.cmp(k_idx, .LE, q_idx); - const minus_inf = zml.Tensor.constant(dtype.minValue()); - break :attn_mask .select(mask, .scalar(0, dtype), minus_inf); - }; - - const scale: f32 = self.opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(active_q.dim(.hd))))); - var attn_weights = active_q.dot(active_k, .hd).scale(scale); - attn_weights = attn_weights.add(attn_mask.broad(attn_weights.shape())); - - const og_softmax = if (q_chunk == 1) state.partial_softmax_decode else state.partial_softmax_prefill; - // Note: we compute q * k using q and k dtype, but the partial softmax is in f32. - const precise_dt = og_softmax.values.dtype(); - var partial = partialSoftmax(attn_weights.convert(precise_dt), .k_chunk); - partial.values = partial.values.dot(active_v.convert(precise_dt), .k_chunk).transpose(og_softmax.values.shape()); - partial.max_value = partial.max_value.transpose(og_softmax.max_value.shape()); - partial.exp_sum = partial.exp_sum.transpose(og_softmax.exp_sum.shape()); - - return og_softmax.merge(partial); - } - - pub fn updateState(state: State, num_slots: u32, query_has_more_keys: zml.Tensor, sequence_has_more_queries: zml.Tensor, softmax: PartialSoftmax) State { - return zml.ops.if2( - query_has_more_keys, - // If the query has more pages: update partial softmax with attention on current page, delay updating output. - State{ - .slot_id = state.slot_id, - .seq_id = state.seq_id, - .q_page_idx = state.q_page_idx, - .k_page_idx = state.k_page_idx.addConstant(1), - .partial_softmax_prefill = if (num_slots == 1) state.partial_softmax_prefill else softmax, - .partial_softmax_decode = if (num_slots == 1) softmax else state.partial_softmax_decode, - .out = state.out, - }, - // Query has no more pages, we commit softmax to output and increment slot - State{ - .k_page_idx = .scalar(0, .i32), - .partial_softmax_prefill = if (num_slots == 1) state.partial_softmax_prefill else .reset(softmax), - .partial_softmax_decode = if (num_slots == 1) .reset(softmax) else state.partial_softmax_decode, - .out = state.out.dynamicUpdateSlice(.{ .b = state.slot_id }, softmax.finalize().convert(state.out.dtype())), - .slot_id = state.slot_id.addConstant(num_slots), - // If the sequence has more query chunks: increment q_page_idx, otherwise start over at 0. - .q_page_idx = .select(sequence_has_more_queries, state.q_page_idx.addConstant(1), .scalar(0, .i32)), - // If the sequence has more query chunks: keep seq_id, otherwise increment seq_id. - .seq_id = .select(sequence_has_more_queries, state.seq_id, state.seq_id.addConstant(1)), - }, - ); - } -}; - -pub const PartialSoftmax = struct { - values: zml.Tensor, - exp_sum: zml.Tensor, - max_value: zml.Tensor, - - pub fn zeroes(q_shape: zml.Shape) PartialSoftmax { - return .{ - .values = .zeroes(q_shape), - .exp_sum = .zeroes(q_shape.drop(.hd)), - .max_value = .broad(.constant(q_shape.dtype().minValue()), q_shape.drop(.hd)), - }; - } - - pub fn reset(self: PartialSoftmax) PartialSoftmax { - return .zeroes(self.values.shape()); - } - - pub fn merge(self: PartialSoftmax, other: PartialSoftmax) PartialSoftmax { - // Rescale self and other using the new global_max. - const global_max = self.max_value.maximum(other.max_value.convert(self.max_value.dtype())); - const new_self = self.rescale(global_max); - const new_other = other.rescale(global_max); - - // Now that self and other are using the same scale, we can just add them: - return .{ - .max_value = global_max, - .values = new_self.values.add(new_other.values), - .exp_sum = new_self.exp_sum.add(new_other.exp_sum), - }; - } - - /// Update max_value and rescale attn and exp_sum accordingly. - pub fn rescale(self: PartialSoftmax, max_value: zml.Tensor) PartialSoftmax { - const max_diff_exp: zml.Tensor = .exp(self.max_value.sub(max_value)); - const sum_dtype = self.exp_sum.dtype(); - return .{ - .max_value = max_value, - .values = self.values.mul(max_diff_exp.broad(self.values.shape())), - .exp_sum = self.exp_sum.mul(max_diff_exp.convert(sum_dtype)), - }; - } - - /// Divides the intermediary results by the exp_sum to get the proper attention values. - pub fn finalize(self: PartialSoftmax) zml.Tensor { - return self.values.div(self.exp_sum.broad(self.values.shape()).convert(self.values.dtype())); - } -}; - -/// Compute softmax over a chunk. -/// Returns intermediary results to allow aggregating later. -pub fn partialSoftmax(self: zml.Tensor, axis: anytype) PartialSoftmax { - const a = self.axis(axis); - const max_val = self.max(a).squeeze(a).maximum(.scalar(-1e16, self.dtype())); - const out = self.sub(max_val.broad(self.shape())).exp(); - return .{ - .values = out, - .exp_sum = out.convert(.f32).sum(a).squeeze(a), - .max_value = max_val, - }; -} - -pub const Mla = struct { - pub const Backend = sparse_mla.Backend; - - pub const Options = struct { - backend: Mla.Backend = .triton, - rope_rank: i64, - value_rank: i64, - scale: ?f32 = null, - /// null selects automatically; 1 forces the 2D kernel; other values must be powers of two up to 128. - num_kv_splits: ?u8 = null, - }; - - fn stablehlo_pagedSparseAttention(q: zml.Tensor, kv: zml.Tensor, sink: ?zml.Tensor, topk: zml.Tensor, opts: Mla.Options) zml.Tensor { - stdx.debug.assert(kv.dim(.hkv) == 1, "StableHLO MLA expects one latent KV head, got {}", .{kv.dim(.hkv)}); - const kv_flat = kv.squeeze(.hkv).reshape(.{ - .kv = kv.dim(.page) * kv.dim(.k_chunk), - .hd = kv.dim(.hd), - }); - const valid_topk = topk.cmp(.GE, zml.Tensor.zeroes(topk.shape())); - const safe_topk = zml.Tensor.select(valid_topk, topk, zml.Tensor.zeroes(topk.shape())); - const mask = valid_topk.insertAxes(.topk, .{.h}); - const selected_kv = kv_flat.gather(.{ .kv = safe_topk.rename(.{ .q = .b }) }, .{}).rename(.{ .b = .q, .topk = .kv }).convert(.f32); - - const dims = zml.nn.collectDims(.{ .h, .q, .kv, .hd }, &.{ q, kv_flat }, .strict) catch { - stdx.debug.panic("Inputs have incompatible shapes (q: {f}, kv: {f}).", .{ q, kv_flat }); - }; - - const sqrt_head_dim = opts.scale orelse 1.0 / std.math.sqrt(@as(f32, @floatFromInt(dims.hd))); - const q_32 = q.convert(.f32); - var scores = q_32.dot(selected_kv, .hd).scale(sqrt_head_dim); - scores = zml.Tensor.select(mask.broad(scores.shape()), scores, zml.Tensor.constant(scores.dtype().minValue())); - - const sink_shape = q.shape().set(.hd, 1); - const attn_sink = sink orelse stdx.debug.panic("ragged MLA attention requires an attention sink", .{}); - const sink_ = attn_sink.insertAxes(0, .{.q}).insertAxes(.last, .{.hd}).broad(sink_shape); - const scores_sink = zml.Tensor.concatenate(&.{ scores, sink_.convert(scores.dtype()) }, .kv); - - const attn_weights = scores_sink.softmax(.kv); - const attn_weights_non_sink = attn_weights.slice(-1, .{ .end = topk.dim(.topk) }); - const selected_values = selected_kv.slice(.hd, .{ .end = opts.value_rank }); - return attn_weights_non_sink.dot(selected_values, .kv).convert(q.dtype()); - } - - /// Rows of a paged sparse MLA cache, as stored - pub const Cache = union(enum) { - latent: zml.Tensor, - quantized: zml.quantization.QuantizedInput, - - fn rowsPerPage(self: Cache) i64 { - return switch (self) { - .latent => |t| t.dim(.k_chunk), - .quantized => |input| input.values.dim(.k_chunk), - }; - } - }; - - /// The rows each query attends in a paged cache - pub const CacheSelection = struct { - cache: Cache, - /// Page table of this cache (block table, sequence lengths, query offsets) - parameters: Parameters, - /// The logical token positions to attend, -1 or a - /// position outside the sequence selects nothing - positions: zml.Tensor, - /// Ratio of tokens per row - compression_ratio: u32 = 1, - - /// Number of leading active query rows, from the page table - pub fn activeCount(self: CacheSelection) zml.Tensor { - return switch (self.parameters) { - .triton, .stablehlo => |p| p.query_start_len.slice(.b, .single(p.query_start_len.dim(.b) - 1)), - else => std.debug.panic("sparse MLA does not support {} parameters", .{std.meta.activeTag(self.parameters)}), - }; - } - - /// Flat physical row (page * rows per page + slot) of each selected position - /// -1 for none and for queries past `active_count` - pub fn physicalRows(self: CacheSelection, tokens_pos: zml.Tensor, active_count: zml.Tensor) zml.Tensor { - const page_tokens = self.cache.rowsPerPage() * self.compression_ratio; - const tokens = switch (self.parameters) { - .triton, .stablehlo => |parameters| triton.paged.topkToPhysical(parameters, self.positions, tokens_pos, page_tokens), - else => std.debug.panic("sparse MLA does not support {} parameters", .{std.meta.activeTag(self.parameters)}), - }; - const active = zml.Tensor.iota(tokens.shape(), .q).cmp(.LT, active_count); - const valid = tokens.cmp(.GE, .scalar(0, .i32)).logical(.AND, active); - return tokens.divByConst(self.compression_ratio).mask(valid, -1); - } - }; - - /// Inputs of `pagedSparseAttention`. - pub const SparseAttentionArgs = struct { - q: zml.Tensor, - kv: CacheSelection, - compressed: ?CacheSelection = null, - sink: ?zml.Tensor, - tokens_pos: zml.Tensor, - }; - - pub fn pagedSparseAttention(args: SparseAttentionArgs, opts: Mla.Options) zml.Tensor { - // Switch there because backends read input rows differently - return switch (opts.backend) { - .cute => cute.pagedAttention(args, opts), - .triton, .fly => latentAttention(args, opts), - }; - } - - fn latentAttention(args: SparseAttentionArgs, opts: Mla.Options) zml.Tensor { - const active_count = args.kv.activeCount(); - const rows = args.kv.physicalRows(args.tokens_pos, active_count); - const kv, const selected = if (args.compressed == null and args.kv.cache == .latent) - .{ args.kv.cache.latent, rows } - else blk: { - // These steps are made inside the cute kernel - var latent = gatherRows(args.kv.cache, rows, args.q.dtype()); - var valid = rows.cmp(.GE, .scalar(0, .i32)); - if (args.compressed) |c| { - const compressed_rows = c.physicalRows(args.tokens_pos, active_count); - latent = zml.Tensor.concatenate(&.{ latent, gatherRows(c.cache, compressed_rows, args.q.dtype()) }, .topk); - valid = zml.Tensor.concatenate(&.{ valid, compressed_rows.cmp(.GE, .scalar(0, .i32)) }, .topk); - } - // Query i reads its gathered rows i * topk + j. - const block_rows = zml.Tensor.iota(valid.shape().withDtype(.i32), .q).scale(valid.dim(.topk)) - .add(zml.Tensor.iota(valid.shape().withDtype(.i32), .topk)).mask(valid, -1); - break :blk .{ latent.rename(.{ .q = .page, .topk = .k_chunk }), block_rows }; - }; - const active = zml.Tensor.iota(.init(.{ .q = args.q.dim(.q) }, .i32), .q).cmp(.LT, active_count); - return latentSparseAttention(args.kv.parameters, args.q, kv, args.sink, selected, active_count, opts).mask(active, 0); - } - - /// Gathers the specified rows from the cache - fn gatherRows(cache: Cache, rows: zml.Tensor, dtype: zml.DataType) zml.Tensor { - const valid = rows.cmp(.GE, .scalar(0, .i32)); - const safe = rows.mask(valid, 0); - const values = switch (cache) { - .latent => |t| t.merge(.{ .slot = .{ .page, .k_chunk } }).gather(.{ .slot = safe }, .{}).convert(dtype), - .quantized => |input| blk: { - const v = input.values.merge(.{ .slot = .{ .page, .k_chunk } }).gather(.{ .slot = safe }, .{}); - const sc = input.scales.merge(.{ .slot = .{ .page, .k_chunk } }).gather(.{ .slot = safe }, .{}); - const grouped = v.convert(.f32).splitAxis(.hd, .{ .qb = sc.dim(.hd), .qi = .auto }); - var dq = grouped.mul(sc.convert(.f32).rename(.{ .hd = .qb }).broad(grouped.shape())).merge(.{ .hd = .{ .qb, .qi } }); - if (input.global_scale) |g| dq = dq.mul(g.convert(.f32).broad(dq.shape())); - break :blk dq.convert(dtype).insertAxes(.hd, .{.hkv}); - }, - }; - return values.mask(valid.insertAxes(.last, .{ .hkv, .hd }).broad(values.shape()), 0); - } - - fn latentSparseAttention(parameters: Parameters, q: zml.Tensor, latent_kv: zml.Tensor, sink: ?zml.Tensor, rows: zml.Tensor, active_count: zml.Tensor, opts: Mla.Options) zml.Tensor { - stdx.debug.assert(q.shape().hasTags(.{ .q, .h, .hd }), "expected q to have tags .q, .h, .hd after flattening, got {f}", .{q.shape()}); - stdx.debug.assert(q.dim(.hd) > opts.rope_rank, "expected q head dim ({}) to include a rope tail of {}", .{ q.dim(.hd), opts.rope_rank }); - stdx.debug.assert(opts.value_rank > 0, "expected MLA value rank to be positive, got {}", .{opts.value_rank}); - stdx.debug.assert(opts.value_rank == q.dim(.hd) or opts.value_rank + opts.rope_rank == q.dim(.hd), "expected MLA value rank ({}) to cover either the complete qk head ({}) or its non-RoPE prefix ({})", .{ opts.value_rank, q.dim(.hd), q.dim(.hd) - opts.rope_rank }); - stdx.debug.assert(latent_kv.shape().hasTags(.{ .page, .k_chunk, .hkv, .hd }), "expected paged latent KV cache to have tags .page, .k_chunk, .hkv, .hd, got {f}", .{latent_kv.shape()}); - stdx.debug.assert(latent_kv.dim(.hd) == q.dim(.hd), "expected q and kv cache head dims to match, got q={} kv={}", .{ q.dim(.hd), latent_kv.dim(.hd) }); - - return switch (parameters) { - .triton => |p| sparse_mla.attention(q, latent_kv, sink, rows, active_count, !p.options_.is_prefill, opts), - .stablehlo => stablehlo_pagedSparseAttention(q, latent_kv, sink, rows, opts), - else => @panic("NOPE"), - }; - } -}; - -test "Triton sparse MLA value ranks and padded queries" { - const platform = zml.testing.env(); - if (!Backend.triton.isAvailable(platform)) return error.SkipZigTest; - - const parameters = Parameters.init(.fromBackend(.{ - .backend = .triton, - .is_prefill = true, - .batch_size = 1, - .seq_len = 32, - .max_num_pages = 2, - .max_token_count = 2, - .num_heads = 16, - .num_kv_heads = 1, - .head_dim = 128, - .max_seqlen_q = 2, - })); - const q_shape = zml.Shape.init(.{ .q = 2, .h = 16, .hd = 128 }, .f32); - const kv_shape = zml.Shape.init(.{ .page = 2, .k_chunk = 16, .hkv = 1, .hd = 128 }, .f32); - const sink_shape = zml.Shape.init(.{ .h = 16 }, .f32); - const topk_shape = zml.Shape.init(.{ .q = 2, .topk = 32 }, .i32); - const tokens_pos_shape = zml.Shape.init(.{ .q = 2 }, .i32); - - var q_data: [2][16][128]f32 = undefined; - @memset(std.mem.sliceAsBytes(&q_data), 0); - for (&q_data[0]) |*head| head[64] = 1; - var kv_data: [2][16][1][128]f32 = undefined; - @memset(std.mem.sliceAsBytes(&kv_data), 0); - @memset(kv_data[1][0][0][0..64], 10); - @memset(kv_data[1][1][0][0..64], 20); - kv_data[1][0][0][64] = 0; - kv_data[1][1][0][64] = 2; - var sink_data: [16]f32 = undefined; - for (&sink_data) |*value| value.* = -std.math.inf(f32); - var topk_data: [2][32]i32 = undefined; - @memset(&topk_data[0], -1); - @memset(&topk_data[1], -1); - topk_data[0][0] = 0; - topk_data[0][1] = 1; - const tokens_pos_data: [2]i32 = .{ 31, 31 }; - const block_table: [1][2]i32 = .{.{ 1, 0 }}; - const seq_lens: [1]i32 = .{32}; - // Only the first of the two statically allocated query rows is active. - const query_start_len: [2]i32 = .{ 0, 1 }; - - const q = zml.Tensor.init(q_shape, .f32); - const kv = zml.Tensor.init(kv_shape, .f32); - const sink = zml.Tensor.init(sink_shape, .f32); - const topk = zml.Tensor.init(topk_shape, .i32); - const tokens_pos = zml.Tensor.init(tokens_pos_shape, .i32); - - var parameters_d: zml.Bufferized(Parameters) = .{ .triton = .{ - .block_table = try .fromBytes(std.testing.io, platform, parameters.triton.block_table.shape(), std.mem.sliceAsBytes(&block_table)), - .seq_lens = try .fromBytes(std.testing.io, platform, parameters.triton.seq_lens.shape(), std.mem.sliceAsBytes(&seq_lens)), - .query_start_len = try .fromBytes(std.testing.io, platform, parameters.triton.query_start_len.shape(), std.mem.sliceAsBytes(&query_start_len)), - } }; - defer zml.Buffer.deinitAll(Parameters, ¶meters_d); - var q_d = try zml.Buffer.fromBytes(std.testing.io, platform, q_shape, std.mem.sliceAsBytes(&q_data)); - defer q_d.deinit(); - var kv_d = try zml.Buffer.fromBytes(std.testing.io, platform, kv_shape, std.mem.sliceAsBytes(&kv_data)); - defer kv_d.deinit(); - var sink_d = try zml.Buffer.fromBytes(std.testing.io, platform, sink_shape, std.mem.sliceAsBytes(&sink_data)); - defer sink_d.deinit(); - var topk_d = try zml.Buffer.fromBytes(std.testing.io, platform, topk_shape, std.mem.sliceAsBytes(&topk_data)); - defer topk_d.deinit(); - var tokens_pos_d = try zml.Buffer.fromBytes(std.testing.io, platform, tokens_pos_shape, std.mem.sliceAsBytes(&tokens_pos_data)); - defer tokens_pos_d.deinit(); - - const TestCase = struct { - num_kv_splits: u8, - value_rank: i64, - }; - inline for ([_]TestCase{ - .{ .num_kv_splits = 2, .value_rank = 64 }, - .{ .num_kv_splits = 1, .value_rank = 128 }, - }) |case| { - var exe = try platform.compileFn( - std.testing.allocator, - std.testing.io, - Mla.pagedSparseAttention, - .{ Mla.SparseAttentionArgs{ .q = q, .kv = .{ .cache = .{ .latent = kv }, .parameters = parameters, .positions = topk }, .sink = sink, .tokens_pos = tokens_pos }, .{ - .rope_rank = 64, - .value_rank = case.value_rank, - .scale = 1, - .num_kv_splits = case.num_kv_splits, - } }, - .{}, - ); - defer exe.deinit(); - - var output_d = try exe.eval( - std.testing.allocator, - std.testing.io, - .{.{ .q = q_d, .kv = .{ .cache = .{ .latent = kv_d }, .parameters = parameters_d, .positions = topk_d }, .compressed = null, .sink = sink_d, .tokens_pos = tokens_pos_d }}, - ); - defer output_d.deinit(); - try std.testing.expect(output_d.shape().eql(q_shape.set(.hd, case.value_rank))); - - var output = try output_d.toSliceAlloc(std.testing.allocator, std.testing.io); - defer output.free(std.testing.allocator); - const active_output = output.subSlice(output.shape.axis(.q), 0, 1); - - const expected_shape = q_shape.set(.q, 1).set(.hd, case.value_rank); - const expected_data = try std.testing.allocator.alloc(f32, expected_shape.count()); - defer std.testing.allocator.free(expected_data); - const exp_two = @exp(@as(f32, 2)); - const second_weight = exp_two / (1 + exp_two); - for (expected_data, 0..) |*value, i| { - const dim = i % @as(usize, @intCast(case.value_rank)); - value.* = if (dim < 64) - 10 + 10 * second_weight - else if (dim == 64) - 2 * second_weight - else - 0; - } - const expected = zml.Slice.init(expected_shape, std.mem.sliceAsBytes(expected_data)); - try zml.testing.expectClose(std.testing.io, expected, active_output, .{ .absolute_tolerance = 0.001, .relative_tolerance = 0.001 }); - } -} - -test "execute stablehlo mla kernel" { - const platform = zml.testing.env(); - const parameters = Parameters.init(.fromBackend(.{ - .backend = .stablehlo, - .is_prefill = true, - .batch_size = 1, - .seq_len = 32, - .max_num_pages = 2, - .max_token_count = 1, - .num_heads = 16, - .num_kv_heads = 1, - .head_dim = 128, - .max_seqlen_q = 1, - })); - - const q_shape = zml.Shape.init(.{ .q = 1, .h = 16, .hd = 128 }, .f32); - const kv_shape = zml.Shape.init(.{ .page = 2, .k_chunk = 16, .hkv = 1, .hd = 128 }, .f32); - const sink_shape = zml.Shape.init(.{ .h = 16 }, .f32); - const topk_shape = zml.Shape.init(.{ .q = 1, .topk = 2 }, .i32); - const tokens_pos_shape = zml.Shape.init(.{ .q = 1 }, .i32); - - var q_data: [1][16][128]f32 = undefined; - @memset(std.mem.sliceAsBytes(&q_data), 0); - for (&q_data[0]) |*head| head[64] = 1; - var kv_data: [2][16][1][128]f32 = undefined; - @memset(std.mem.sliceAsBytes(&kv_data), 0); - @memset(kv_data[1][0][0][0..64], 10); - @memset(kv_data[1][1][0][0..64], 20); - kv_data[1][0][0][64] = 0; - kv_data[1][1][0][64] = 2; - var sink_data: [16]f32 = undefined; - for (&sink_data) |*value| value.* = -std.math.inf(f32); - const topk_data: [1][2]i32 = .{.{ 0, 1 }}; - const tokens_pos_data: [1]i32 = .{31}; - const block_table: [1][2]i32 = .{.{ 1, 0 }}; - const seq_lens: [1]i32 = .{32}; - const query_start_len: [2]i32 = .{ 0, 1 }; - - const q = zml.Tensor.init(q_shape, .f32); - const kv = zml.Tensor.init(kv_shape, .f32); - const sink = zml.Tensor.init(sink_shape, .f32); - const topk = zml.Tensor.init(topk_shape, .i32); - const tokens_pos = zml.Tensor.init(tokens_pos_shape, .i32); - - var parameters_d: zml.Bufferized(Parameters) = .{ .stablehlo = .{ - .block_table = try .fromBytes(std.testing.io, platform, parameters.stablehlo.block_table.shape(), std.mem.sliceAsBytes(&block_table)), - .seq_lens = try .fromBytes(std.testing.io, platform, parameters.stablehlo.seq_lens.shape(), std.mem.sliceAsBytes(&seq_lens)), - .query_start_len = try .fromBytes(std.testing.io, platform, parameters.stablehlo.query_start_len.shape(), std.mem.sliceAsBytes(&query_start_len)), - } }; - defer zml.Buffer.deinitAll(Parameters, ¶meters_d); - var q_d = try zml.Buffer.fromBytes(std.testing.io, platform, q_shape, std.mem.sliceAsBytes(&q_data)); - defer q_d.deinit(); - var kv_d = try zml.Buffer.fromBytes(std.testing.io, platform, kv_shape, std.mem.sliceAsBytes(&kv_data)); - defer kv_d.deinit(); - var sink_d = try zml.Buffer.fromBytes(std.testing.io, platform, sink_shape, std.mem.sliceAsBytes(&sink_data)); - defer sink_d.deinit(); - var topk_d = try zml.Buffer.fromBytes(std.testing.io, platform, topk_shape, std.mem.sliceAsBytes(&topk_data)); - defer topk_d.deinit(); - var tokens_pos_d = try zml.Buffer.fromBytes(std.testing.io, platform, tokens_pos_shape, std.mem.sliceAsBytes(&tokens_pos_data)); - defer tokens_pos_d.deinit(); - - const exe = try platform.compileFn( - std.testing.allocator, - std.testing.io, - Mla.pagedSparseAttention, - .{ Mla.SparseAttentionArgs{ .q = q, .kv = .{ .cache = .{ .latent = kv }, .parameters = parameters, .positions = topk }, .sink = sink, .tokens_pos = tokens_pos }, .{ - .rope_rank = 64, - .value_rank = 64, - .scale = 1, - } }, - .{}, - ); - defer exe.deinit(); - - var output_d = try exe.eval( - std.testing.allocator, - std.testing.io, - .{.{ .q = q_d, .kv = .{ .cache = .{ .latent = kv_d }, .parameters = parameters_d, .positions = topk_d }, .compressed = null, .sink = sink_d, .tokens_pos = tokens_pos_d }}, - ); - defer zml.Buffer.deinitAll(zml.Tensor, &output_d); - - const output_shape = q_shape.set(.hd, 64); - var expected_data: [1][16][64]f32 = undefined; - const exp_two = @exp(@as(f32, 2)); - const expected_value = (10 + 20 * exp_two) / (1 + exp_two); - for (&expected_data[0]) |*head| @memset(head, expected_value); - const expected = zml.Slice.init(output_shape, std.mem.sliceAsBytes(&expected_data)); - try zml.testing.expectClose(std.testing.io, expected, output_d, .{ .absolute_tolerance = 0.001, .relative_tolerance = 0.001 }); -} diff --git a/zml/attention/sparse_mla.zig b/zml/attention/sparse_mla.zig deleted file mode 100644 index 3edbbe67..00000000 --- a/zml/attention/sparse_mla.zig +++ /dev/null @@ -1,195 +0,0 @@ -const std = @import("std"); -const stdx = @import("stdx"); -const zml = @import("../zml.zig"); -const triton = @import("triton_attention.zig"); -const cute = @import("cute_kernels/sparse_mla.zig"); -const fly = @import("fly_kernels/sparse_mla.zig"); -const MlaOptions = @import("paged_attention.zig").Mla.Options; -const SparseAttentionArgs = @import("paged_attention.zig").Mla.SparseAttentionArgs; - -pub const Backend = enum { - triton, - fly, - cute, - - pub fn auto(args: SparseAttentionArgs) Backend { - const platform = zml.Compiler.current().platform; - return switch (platform.target) { - .cuda => if (cute.supports(args)) .cute else .triton, - .rocm => switch (zml.platform.rocm.computeCapability(platform) orelse return .triton) { - .gfx942 => switch (args.q.dtype()) { - .bf16 => .fly, - else => .triton, - }, - else => .triton, - }, - else => .triton, - }; - } - - fn call(self: Backend, q: zml.Tensor, kv_cache: zml.Tensor, sink: ?zml.Tensor, topk: zml.Tensor, active_query_count: zml.Tensor, opts: Options) zml.Tensor { - const backend: Backend = switch (self) { - .fly => blk: { - const platform = zml.Compiler.current().platform; - const gfx942 = platform.target == .rocm and zml.platform.rocm.computeCapability(platform) == .gfx942; - break :blk if (gfx942 and q.dtype() == .bf16) .fly else .triton; - }, - .triton => .triton, - .cute => .cute, - }; - switch (backend) { - .fly => if (fly.pagedAttention(q, kv_cache, sink, topk, active_query_count, opts, triton.getCuCount())) |output| return output, - .triton, .cute => {}, - } - return triton.paged.pagedSparseMlaKernel(q, kv_cache, sink, topk, active_query_count, opts); - } -}; - -pub const Options = struct { - qk_rank: usize, - value_rank: usize, - num_heads: usize, - block_size: usize, - rope_rank: usize, - scale: ?f32, - total_q_blocks: usize, - num_kv_splits: ?u8, - all_decode: bool, -}; - -pub const Config = struct { - block_m: usize, - tile_size: usize, - num_splits: usize, - direct_programs: usize, - main_num_warps: usize = 4, - main_num_stages: usize = 2, - grouped_reduce_threshold: usize = 32, - splits_per_group: usize = 16, - grouped_reduce_num_warps: usize = 1, - parallel_reduce_min_splits: usize = std.math.maxInt(usize), - parallel_reduce_num_warps: usize = 1, -}; - -pub fn launchConfig(paged_opts: Options, topk_count: usize, cu_count_: usize) Config { - stdx.debug.assert(paged_opts.total_q_blocks > 0, "sparse MLA requires at least one query", .{}); - stdx.debug.assert(paged_opts.num_heads > 0, "sparse MLA requires at least one query head", .{}); - stdx.debug.assert(topk_count > 0, "sparse MLA requires at least one top-k entry", .{}); - - const padded_heads = std.math.ceilPowerOfTwoAssert(usize, paged_opts.num_heads); - const padded_topk = std.math.ceilPowerOfTwoAssert(usize, topk_count); - var config: Config = .{ - .block_m = @min(padded_heads, 16), - .tile_size = @min(padded_topk, 16), - .num_splits = 1, - .direct_programs = undefined, - }; - const cc = zml.platform.cuda.computeCapability(zml.Compiler.current().platform); - const is_sm103 = if (cc) |value| value.eql(.{ .major = 10, .minor = 3 }) else false; - - // GB300 (sm_103): a single wide query benefits from more head blocks and - // wider sparse tiles. The split selection below still derives each layer's - // split count from query/head/top-k shapes (16 for DSV4 CSA/HCA and 4 for - // its 128-entry full-attention layers). - if (is_sm103 and - paged_opts.all_decode and - paged_opts.total_q_blocks == 1 and - paged_opts.value_rank >= 512) - { - config.block_m = @min(padded_heads, 8); - config.tile_size = @min(padded_topk, 32); - config.parallel_reduce_min_splits = 16; - config.parallel_reduce_num_warps = 2; - } - // Full 256-query prefill chunks already expose enough query parallelism. - // Retain BLOCK_M=16 and the generic split model, but process wider sparse - // tiles. The predicate uses the post-mesh flattened query shape. - if (is_sm103 and - !paged_opts.all_decode and - paged_opts.total_q_blocks >= 256 and - paged_opts.value_rank >= 512) - { - config.tile_size = @min(padded_topk, 32); - } - - const num_tiles = std.math.divCeil(usize, topk_count, config.tile_size) catch unreachable; - const head_blocks = std.math.divCeil(usize, paged_opts.num_heads, config.block_m) catch unreachable; - config.direct_programs = paged_opts.total_q_blocks * head_blocks; - const cu_count = @max(cu_count_, 1); - - if (paged_opts.num_kv_splits) |requested| { - const num_splits: usize = requested; - stdx.debug.assert(std.math.isPowerOfTwo(num_splits), "MLA num_kv_splits ({}) must be a power of two", .{num_splits}); - stdx.debug.assert(num_splits <= 128, "MLA num_kv_splits ({}) must not exceed 128", .{num_splits}); - stdx.debug.assert(num_splits <= num_tiles, "MLA num_kv_splits ({}) must not exceed sparse tile count ({})", .{ num_splits, num_tiles }); - config.num_splits = num_splits; - } else { - var best_cost: usize = std.math.maxInt(usize); - const candidates = [_]usize{ 1, 2, 4, 8, 16, 32, 64, 128 }; - for (candidates) |num_splits| { - if (num_splits > num_tiles) break; - const programs = config.direct_programs * num_splits; - const rounds = std.math.divCeil(usize, programs, cu_count) catch unreachable; - const tiles_per_program = std.math.divCeil(usize, num_tiles, num_splits) catch unreachable; - const cost = rounds * tiles_per_program; - // Candidates are ordered by split count, so ties retain 2D or the lower-overhead 3D launch. - if (cost < best_cost) { - best_cost = cost; - config.num_splits = num_splits; - } - } - } - - return config; -} - -/// Sparse MLA over a paged latent cache, `topk` holding logical token positions. -pub fn pagedAttention(parameters: triton.paged.Parameters, q: zml.Tensor, kv_cache: zml.Tensor, sink: ?zml.Tensor, topk: zml.Tensor, tokens_pos: zml.Tensor, opts: MlaOptions) zml.Tensor { - const rows = triton.paged.topkToPhysical(parameters, topk, tokens_pos, kv_cache.dim(.k_chunk)); - const active_count = parameters.query_start_len.slice(.b, .single(parameters.query_start_len.dim(.b) - 1)); - return attention(q, kv_cache, sink, rows, active_count, !parameters.options_.is_prefill, opts); -} - -/// Sparse MLA of `q` [.q, .h, .hd] over the latent cache `kv_cache` [.page, .k_chunk, .hkv, .hd] -/// with the Triton (or Fly) kernels: `rows` holds the physical row of each selected key -/// ([.q, .topk], -1 unused), `active_count` the number of leading active queries. -pub fn attention(q: zml.Tensor, kv_cache: zml.Tensor, sink: ?zml.Tensor, rows: zml.Tensor, active_count: zml.Tensor, all_decode: bool, opts: MlaOptions) zml.Tensor { - const output_shape = q.shape().set(.hd, opts.value_rank); - return zml.ops.manualComputation( - (struct { - q: zml.Tensor, - kv_cache: zml.Tensor, - sink: ?zml.Tensor, - rows: zml.Tensor, - active_count: zml.Tensor, - all_decode: bool, - opts: MlaOptions, - - fn body(self: @This(), _: zml.Shape) zml.Tensor { - stdx.debug.assert(self.rows.dim(.q) == self.q.dim(.q), "expected rows q dim ({}) to match q dim ({})", .{ self.rows.dim(.q), self.q.dim(.q) }); - const paged_opts: Options = .{ - .qk_rank = @intCast(self.q.dim(.hd)), - .value_rank = @intCast(self.opts.value_rank), - .num_heads = @intCast(self.q.dim(.h)), - .block_size = @intCast(self.kv_cache.dim(.k_chunk)), - .rope_rank = @intCast(self.opts.rope_rank), - .scale = self.opts.scale, - .total_q_blocks = @intCast(self.q.dim(.q)), - .num_kv_splits = self.opts.num_kv_splits, - .all_decode = self.all_decode, - }; - return self.opts.backend.call(self.q, self.kv_cache, self.sink, self.rows, self.active_count, paged_opts); - } - }).body, - .{ - .q = q, - .kv_cache = kv_cache, - .sink = sink, - .rows = rows, - .active_count = active_count, - .all_decode = all_decode, - .opts = opts, - }, - output_shape, - ); -} diff --git a/zml/attention/triton_kernels/unified_sparse_mla.zig b/zml/attention/triton_kernels/unified_sparse_mla.zig deleted file mode 100644 index 07ee1b78..00000000 --- a/zml/attention/triton_kernels/unified_sparse_mla.zig +++ /dev/null @@ -1,567 +0,0 @@ -const std = @import("std"); - -const zml = @import("../../zml.zig"); -const ops = zml.ops; -const tri = zml.kernel.triton; -const Builder = tri.Builder; -const DType = tri.DType; -const Tensor = zml.Tensor; -const Shape = zml.Shape; -const Value = tri.Value; - -pub const Config = struct { - q_dtype: DType = .bf16, - kv_dtype: DType = .bf16, - sink_dtype: DType = .f32, - o_dtype: DType = .bf16, - num_query_heads: i64 = 32, - block_size: i64 = 16, - stride_cache_dim: i64 = 1, - topk_count: i64 = 32, - block_m: i64 = 16, - rope_rank: i64 = 64, - value_rank: i64 = 512, - rope_offset: i64 = 512, - tile_size: i64 = 16, - num_splits: i64 = 1, - use_attn_sink: bool = false, - all_decode: bool = false, -}; - -pub const Kernel2D = tri.Kernel(Config, .{ - .name = "_kernel_unified_attention_sparse_mla_2d_ptr", - .inputs = &.{ - "query_ptr", - "kv_cache_ptr", - "attn_sink_ptr", - "topk_indices_ptr", - "active_query_count_ptr", - "scale_ptr", - "query_stride_0_ptr", - "query_stride_1_ptr", - "output_stride_0_ptr", - "output_stride_1_ptr", - "stride_cache_0_ptr", - "stride_cache_1_ptr", - }, - .outputs = &.{"output"}, - .run = run2D, -}); - -fn run2D(b: *tri.Builder, cfg: Config) tri.FinishError!void { - const a = try b.declareArgs(.{ - .query_ptr = .{ .ptr = cfg.q_dtype }, - .kv_cache_ptr = .{ .ptr = cfg.kv_dtype }, - .attn_sink_ptr = .{ .ptr = cfg.sink_dtype }, - .topk_indices_ptr = .{ .ptr = .i32 }, - .active_query_count_ptr = .{ .ptr = .i32 }, - .scale_ptr = .{ .ptr = .f32 }, - .query_stride_0_ptr = .{ .ptr = .i64 }, - .query_stride_1_ptr = .{ .ptr = .i64 }, - .output_stride_0_ptr = .{ .ptr = .i64 }, - .output_stride_1_ptr = .{ .ptr = .i64 }, - .stride_cache_0_ptr = .{ .ptr = .i64 }, - .stride_cache_1_ptr = .{ .ptr = .i64 }, - .output_ptr = .{ .ptr = cfg.o_dtype }, - }); - - const scale = b.load(a.scale_ptr); - const query_stride_0 = b.load(a.query_stride_0_ptr); - const query_stride_1 = b.load(a.query_stride_1_ptr); - const output_stride_0 = b.load(a.output_stride_0_ptr); - const output_stride_1 = b.load(a.output_stride_1_ptr); - const stride_cache_0 = b.load(a.stride_cache_0_ptr); - const stride_cache_1 = b.load(a.stride_cache_1_ptr); - - kernelUnifiedAttentionSparseMla( - b, - a.output_ptr, - null, - a.query_ptr, - a.kv_cache_ptr, - a.attn_sink_ptr, - a.topk_indices_ptr, - a.active_query_count_ptr, - scale, - query_stride_0, - query_stride_1, - output_stride_0, - output_stride_1, - stride_cache_0, - stride_cache_1, - cfg, - false, - ); -} - -pub const Kernel3D = tri.Kernel(Config, .{ - .name = "_kernel_unified_attention_sparse_mla_3d_ptr", - .inputs = &.{ - "query_ptr", - "kv_cache_ptr", - "attn_sink_ptr", - "topk_indices_ptr", - "active_query_count_ptr", - "scale_ptr", - "query_stride_0_ptr", - "query_stride_1_ptr", - "output_stride_0_ptr", - "output_stride_1_ptr", - "stride_cache_0_ptr", - "stride_cache_1_ptr", - }, - .outputs = &.{ "partial_output", "partial_lse" }, - .run = run3D, -}); - -fn run3D(b: *tri.Builder, cfg: Config) tri.FinishError!void { - const a = try b.declareArgs(.{ - .query_ptr = .{ .ptr = cfg.q_dtype }, - .kv_cache_ptr = .{ .ptr = cfg.kv_dtype }, - .attn_sink_ptr = .{ .ptr = cfg.sink_dtype }, - .topk_indices_ptr = .{ .ptr = .i32 }, - .active_query_count_ptr = .{ .ptr = .i32 }, - .scale_ptr = .{ .ptr = .f32 }, - .query_stride_0_ptr = .{ .ptr = .i64 }, - .query_stride_1_ptr = .{ .ptr = .i64 }, - .output_stride_0_ptr = .{ .ptr = .i64 }, - .output_stride_1_ptr = .{ .ptr = .i64 }, - .stride_cache_0_ptr = .{ .ptr = .i64 }, - .stride_cache_1_ptr = .{ .ptr = .i64 }, - .partial_output_ptr = .{ .ptr = .f32 }, - .partial_lse_ptr = .{ .ptr = .f32 }, - }); - - const scale = b.load(a.scale_ptr); - const query_stride_0 = b.load(a.query_stride_0_ptr); - const query_stride_1 = b.load(a.query_stride_1_ptr); - const output_stride_0 = b.load(a.output_stride_0_ptr); - const output_stride_1 = b.load(a.output_stride_1_ptr); - const stride_cache_0 = b.load(a.stride_cache_0_ptr); - const stride_cache_1 = b.load(a.stride_cache_1_ptr); - - kernelUnifiedAttentionSparseMla( - b, - a.partial_output_ptr, - a.partial_lse_ptr, - a.query_ptr, - a.kv_cache_ptr, - a.attn_sink_ptr, - a.topk_indices_ptr, - a.active_query_count_ptr, - scale, - query_stride_0, - query_stride_1, - output_stride_0, - output_stride_1, - stride_cache_0, - stride_cache_1, - cfg, - true, - ); -} - -fn kernelUnifiedAttentionSparseMla( - k: *Builder, - output_ptr: Value, - partial_lse_ptr: ?Value, - query_ptr: Value, - kv_cache_ptr: Value, - attn_sink_ptr: Value, - topk_indices_ptr: Value, - active_query_count_ptr: Value, - scale: Value, - query_stride_0: Value, - query_stride_1: Value, - output_stride_0: Value, - output_stride_1: Value, - stride_cache_0: Value, - stride_cache_1: Value, - config: Config, - comptime three_d: bool, -) void { - const BLOCK_M: i64 = config.block_m; - const BLOCK_SIZE: i64 = config.block_size; - const ROPE_RANK: i64 = config.rope_rank; - const VALUE_RANK: i64 = config.value_rank; - const ROPE_OFFSET: i64 = config.rope_offset; - const TILE_SIZE: i64 = config.tile_size; - const NUM_QUERY_HEADS: i64 = config.num_query_heads; - const NUM_HEAD_BLOCKS: i64 = @divTrunc(NUM_QUERY_HEADS + BLOCK_M - 1, BLOCK_M); - const NUM_TILES: i64 = @divTrunc(config.topk_count + TILE_SIZE - 1, TILE_SIZE); - const NUM_SPLITS: i64 = if (three_d) config.num_splits else 1; - const TILES_PER_SPLIT: i64 = @divTrunc(NUM_TILES + NUM_SPLITS - 1, NUM_SPLITS); - - const q_block_global_idx = k.programId(.x); - const split_idx = if (three_d) k.programId(.y) else k.liftAs(0, .i32); - const split_tile_start = split_idx.mul(@as(i32, @intCast(NUM_TILES))).div(@as(i32, @intCast(NUM_SPLITS))); - const split_tile_end = split_idx.add(1).mul(@as(i32, @intCast(NUM_TILES))).div(@as(i32, @intCast(NUM_SPLITS))); - const q_ind = q_block_global_idx.div(@as(i32, @intCast(NUM_HEAD_BLOCKS))); - const head_ind = q_block_global_idx.rem(@as(i32, @intCast(NUM_HEAD_BLOCKS))); - if (!config.all_decode) { - const active_query_count = k.load(active_query_count_ptr); - k.returnIf(q_ind.ge(active_query_count), .{}); - } - - const offs_h = k.arange(0, BLOCK_M, .i32).add(head_ind.mul(@as(i32, @intCast(BLOCK_M)))); - const offs_value = k.arange(0, VALUE_RANK, .i32); - const offs_rope: ?Value = if (ROPE_RANK > 0) - k.arange(ROPE_OFFSET, ROPE_OFFSET + ROPE_RANK, .i32) - else - null; - const offs_t = k.arange(0, TILE_SIZE, .i32); - - const query_offset_0 = k.splat(q_ind, &.{BLOCK_M}); - const query_offset_1 = offs_h; - const head_mask = offs_h.lt(@as(i32, @intCast(NUM_QUERY_HEADS))); - - const qo0_2d = query_offset_0.expandDims(1).mul(query_stride_0); - const qo1_2d = query_offset_1.expandDims(1).mul(query_stride_1); - const Q_rope: ?Value = if (ROPE_RANK > 0) blk: { - const q_rope_offset = qo0_2d.add(qo1_2d).add(offs_rope.?.expandDims(0)); - break :blk k.loadOpts(query_ptr.addPtr(q_rope_offset), .{ - .mask = head_mask.expandDims(1), - .other = k.zeros(&.{ BLOCK_M, ROPE_RANK }, config.q_dtype), - .cache_modifier = if (config.all_decode or BLOCK_M >= NUM_QUERY_HEADS) .cg else .none, - }); - } else null; - - const q_value_offset = qo0_2d.add(qo1_2d).add(offs_value.expandDims(0)); - const Q_value = k.loadOpts(query_ptr.addPtr(q_value_offset), .{ - .mask = head_mask.expandDims(1), - .other = k.zeros(&.{ BLOCK_M, VALUE_RANK }, config.q_dtype), - .cache_modifier = if (config.all_decode or BLOCK_M >= NUM_QUERY_HEADS) .cg else .none, - }); - - const m_init = k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32); - const l_init = k.full(&.{BLOCK_M}, 0.0, .f32); - const acc_init = k.zeros(&.{ BLOCK_M, VALUE_RANK }, .f32); - - var loop = k.openFor(0, TILES_PER_SPLIT, 1, .{ m_init, l_init, acc_init }); - { - const t = loop.iv; - const M = loop.carried[0]; - const L = loop.carried[1]; - const acc = loop.carried[2]; - - const tile_idx = split_tile_start.add(t); - const tile_start = tile_idx.mul(@as(i32, @intCast(TILE_SIZE))); - const tile_offsets = tile_start.add(offs_t); - var valid_t = tile_offsets.lt(@as(i32, @intCast(config.topk_count))) - .bitAnd(tile_idx.lt(split_tile_end)); - - const topk_row_ptr = topk_indices_ptr.addPtr(q_ind.mul(@as(i32, @intCast(config.topk_count)))); - const topk_pos = k.loadOpts(topk_row_ptr.addPtr(tile_start).addPtr(offs_t), .{ - .mask = valid_t, - .other = k.zeros(&.{TILE_SIZE}, .i32), - }); - valid_t = valid_t.bitAnd(topk_pos.ge(0)); - - const physical_block_idx = topk_pos.div(@as(i32, @intCast(BLOCK_SIZE))); - const slot = topk_pos.rem(@as(i32, @intCast(BLOCK_SIZE))); - - var S = k.zeros(&.{ BLOCK_M, TILE_SIZE }, .f32); - - const physical_block_idx_t = physical_block_idx.expandDims(0); - const cache_block_offsets = physical_block_idx_t.to(.i64).mul(stride_cache_0); - const cache_block_ptrs = kv_cache_ptr.addPtr(cache_block_offsets); - const slot_t = slot.expandDims(0); - const cache_slot_offsets = slot_t.to(.i64).mul(stride_cache_1); - if (ROPE_RANK > 0) { - const k_rope_dim_offsets = offs_rope.?.expandDims(1).mul(@as(i32, @intCast(config.stride_cache_dim))); - const k_rope_dim_ptrs = k.broadcastTo(cache_block_ptrs, &.{ ROPE_RANK, TILE_SIZE }) - .addPtr(k.broadcastTo(k_rope_dim_offsets, &.{ ROPE_RANK, TILE_SIZE })); - const K_rope = k.loadOpts(k_rope_dim_ptrs.addPtr(k.broadcastTo(cache_slot_offsets, &.{ ROPE_RANK, TILE_SIZE })), .{ - .mask = valid_t.expandDims(0), - .other = k.zeros(&.{ ROPE_RANK, TILE_SIZE }, config.kv_dtype), - .cache_modifier = if (config.all_decode) .cg else .none, - }); - S = S.add(scale.mul(k.dot(Q_rope.?, K_rope, k.zeros(&.{ BLOCK_M, TILE_SIZE }, .f32)))); - } - - const k_value_dim_offsets = offs_value.expandDims(1).mul(@as(i32, @intCast(config.stride_cache_dim))); - const k_value_dim_ptrs = k.broadcastTo(cache_block_ptrs, &.{ VALUE_RANK, TILE_SIZE }) - .addPtr(k.broadcastTo(k_value_dim_offsets, &.{ VALUE_RANK, TILE_SIZE })); - const K_value = k.loadOpts(k_value_dim_ptrs.addPtr(k.broadcastTo(cache_slot_offsets, &.{ VALUE_RANK, TILE_SIZE })), .{ - .mask = valid_t.expandDims(0), - .other = k.zeros(&.{ VALUE_RANK, TILE_SIZE }, config.kv_dtype), - .cache_modifier = if (config.all_decode) .cg else .none, - }); - - S = S.add(scale.mul(k.dot(Q_value, K_value, k.zeros(&.{ BLOCK_M, TILE_SIZE }, .f32)))); - - const keep_mask = head_mask.expandDims(1) - .bitAnd(valid_t.expandDims(0)); - S = k.where(keep_mask, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); - - var m_j = M.maximum(k.maxOpts(S, .{ .axis = 1 })); - m_j = k.where(m_j.gt(-std.math.inf(f32)), m_j, k.full(&.{BLOCK_M}, 0.0, .f32)); - const P = k.exp(S.sub(m_j.expandDims(1))); - const l_j = k.sumOpts(P, .{ .axis = 1 }); - const alpha = k.exp(M.sub(m_j)); - - const acc_scaled = acc.mul(alpha.expandDims(1)); - const new_L = L.mul(alpha).add(l_j); - - const V = k.trans(K_value, &.{ 1, 0 }); - const new_acc = k.dot(P.to(config.kv_dtype), V, acc_scaled); - - loop.yield(.{ m_j, new_L, new_acc }); - } - - const M = loop.results[0]; - var L = loop.results[1]; - var acc = loop.results[2]; - - if (!three_d and config.use_attn_sink) { - const sink_logits = k.loadOpts(attn_sink_ptr.addPtr(query_offset_1), .{ - .mask = head_mask, - .other = k.zeros(&.{BLOCK_M}, config.sink_dtype), - }).to(.f32); - const sink_score = k.where( - head_mask, - sink_logits, - k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32), - ); - const n_m = M.maximum(sink_score); - const alpha = k.exp(M.sub(n_m)); - const sink_p = k.exp(sink_score.sub(n_m)); - - acc = acc.mul(alpha.expandDims(1)); - L = L.mul(alpha).add(sink_p); - } - - const has_value = L.gt(0.0); - const safe_l = k.where(has_value, L, k.full(&.{BLOCK_M}, 1.0, .f32)); - const one_over_l = k.full(&.{ BLOCK_M, 1 }, 1.0, .f32).div(safe_l.expandDims(1)); - acc = acc.mul(k.broadcastTo(one_over_l, &.{ BLOCK_M, VALUE_RANK })); - acc = k.where( - has_value.expandDims(1), - acc, - k.zeros(&.{ BLOCK_M, VALUE_RANK }, .f32), - ); - - if (three_d) { - // Compiler-managed workspace layouts are [query, head, split, value] - // and [query, head, split], both in f32. - const partial_out_token_stride: i64 = NUM_QUERY_HEADS * NUM_SPLITS * VALUE_RANK; - const partial_out_head_stride: i64 = NUM_SPLITS * VALUE_RANK; - const partial_out_offset = q_ind.to(.i64).mul(partial_out_token_stride) - .add(offs_h.expandDims(1).to(.i64).mul(partial_out_head_stride)) - .add(split_idx.to(.i64).mul(VALUE_RANK)) - .add(offs_value.expandDims(0).to(.i64)); - k.storeOpts( - output_ptr.addPtr(partial_out_offset), - acc, - .{ .mask = head_mask.expandDims(1) }, - ); - - const partial_lse_token_stride: i64 = NUM_QUERY_HEADS * NUM_SPLITS; - const partial_lse_offset = q_ind.to(.i64).mul(partial_lse_token_stride) - .add(offs_h.to(.i64).mul(NUM_SPLITS)) - .add(split_idx.to(.i64)); - const partial_lse = k.where( - has_value, - M.add(k.log(safe_l)), - k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32), - ); - k.storeOpts( - partial_lse_ptr.?.addPtr(partial_lse_offset), - partial_lse, - .{ .mask = head_mask }, - ); - return; - } - - const output_offsets = query_offset_0.expandDims(1).mul(output_stride_0) - .add(query_offset_1.expandDims(1).mul(output_stride_1)) - .add(offs_value.expandDims(0)); - k.storeOpts( - output_ptr.addPtr(output_offsets), - acc.to(config.o_dtype), - .{ .mask = head_mask.expandDims(1) }, - ); -} - -const Reduce3DPartialsCfg = struct { - num_query_heads: i64, - value_rank: i64, - num_input_splits: i64, - num_output_splits: i64, - all_decode: bool, -}; - -/// Reduce a large split dimension in fixed-size groups before the final -/// attention-sink reduction. Keeping the local reduction width small avoids -/// register spilling when sparse decode uses enough splits to fill a MI300X. -pub const Reduce3DPartials = tri.Kernel(Reduce3DPartialsCfg, .{ - .name = "_kernel_reduce_sparse_mla_3d_partials_ptr", - .inputs = &.{ "input", "input_lse", "active_query_count_ptr" }, - .outputs = &.{ "partial_output", "partial_lse" }, - .run = runReduce3DPartials, -}); - -fn runReduce3DPartials(b: *tri.Builder, cfg: Reduce3DPartialsCfg) tri.FinishError!void { - const a = try b.declareArgs(.{ - .input_ptr = .{ .ptr = .f32 }, - .input_lse_ptr = .{ .ptr = .f32 }, - .active_query_count_ptr = .{ .ptr = .i32 }, - .partial_output_ptr = .{ .ptr = .f32 }, - .partial_lse_ptr = .{ .ptr = .f32 }, - }); - - const query_idx = b.programId(.x); - if (!cfg.all_decode) { - const active_query_count = b.load(a.active_query_count_ptr); - b.returnIf(query_idx.ge(active_query_count), .{}); - } - const head_group_idx = b.programId(.y); - const head_idx = head_group_idx.div(@as(i32, @intCast(cfg.num_output_splits))); - const group_idx = head_group_idx.rem(@as(i32, @intCast(cfg.num_output_splits))); - const splits_per_group = @divExact(cfg.num_input_splits, cfg.num_output_splits); - const offs_s = b.arange(0, splits_per_group, .i32); - const offs_d = b.arange(0, cfg.value_rank, .i32); - const group_start = group_idx.to(.i64).mul(splits_per_group); - - const input_lse_token_stride = cfg.num_query_heads * cfg.num_input_splits; - const input_lse_offset = query_idx.to(.i64).mul(input_lse_token_stride) - .add(head_idx.to(.i64).mul(cfg.num_input_splits)) - .add(group_start) - .add(offs_s.to(.i64)); - const input_lse = b.load(a.input_lse_ptr.addPtr(input_lse_offset)); - const overall_max = b.max(input_lse); - const has_value = overall_max.gt(-std.math.inf(f32)); - const safe_max = b.where(has_value, overall_max, b.liftAs(0.0, .f32)); - const split_weights = b.exp(input_lse.sub(safe_max)); - const denominator = b.sumOpts(split_weights, .{ .axis = 0 }); - - const input_output_token_stride = cfg.num_query_heads * cfg.num_input_splits * cfg.value_rank; - const input_output_head_stride = cfg.num_input_splits * cfg.value_rank; - const input_output_base = query_idx.to(.i64).mul(input_output_token_stride) - .add(head_idx.to(.i64).mul(input_output_head_stride)) - .add(group_start.mul(cfg.value_rank)); - const input_output_offset = b.broadcastTo( - input_output_base.add(offs_s.expandDims(1).to(.i64).mul(cfg.value_rank)), - &.{ splits_per_group, cfg.value_rank }, - ).add(b.broadcastTo(offs_d.expandDims(0).to(.i64), &.{ splits_per_group, cfg.value_rank })); - const input_output = b.load(a.input_ptr.addPtr(input_output_offset)); - const numerator = b.sumOpts(input_output.mul(split_weights.expandDims(1)), .{ .axis = 0 }); - const safe_denominator = b.where(denominator.gt(0.0), denominator, b.liftAs(1.0, .f32)); - const output = b.where( - has_value, - numerator.div(safe_denominator), - b.zeros(&.{cfg.value_rank}, .f32), - ); - - const output_token_stride = cfg.num_query_heads * cfg.num_output_splits * cfg.value_rank; - const output_head_stride = cfg.num_output_splits * cfg.value_rank; - const output_offset = query_idx.to(.i64).mul(output_token_stride) - .add(head_idx.to(.i64).mul(output_head_stride)) - .add(group_idx.to(.i64).mul(cfg.value_rank)) - .add(offs_d.to(.i64)); - b.store(a.partial_output_ptr.addPtr(output_offset), output); - - const output_lse_offset = query_idx.to(.i64).mul(cfg.num_query_heads * cfg.num_output_splits) - .add(head_idx.to(.i64).mul(cfg.num_output_splits)) - .add(group_idx.to(.i64)); - const output_lse = b.where( - has_value, - safe_max.add(b.log(safe_denominator)), - b.liftAs(-std.math.inf(f32), .f32), - ); - b.store(a.partial_lse_ptr.addPtr(output_lse_offset), output_lse); -} - -const Reduce3DCfg = struct { - sink_dtype: DType = .f32, - o_dtype: DType = .bf16, - num_query_heads: i64 = 32, - value_rank: i64 = 512, - num_splits: i64 = 1, - use_attn_sink: bool = false, - all_decode: bool = false, -}; - -pub const Reduce3D = tri.Kernel(Reduce3DCfg, .{ - .name = "_kernel_reduce_sparse_mla_3d_ptr", - .inputs = &.{ - "partial_output_ptr", - "partial_lse_ptr", - "attn_sink_ptr", - "active_query_count_ptr", - "output_stride_0_ptr", - "output_stride_1_ptr", - }, - .outputs = &.{"output"}, - .run = runReduce3D, -}); - -fn runReduce3D(b: *tri.Builder, cfg: Reduce3DCfg) tri.FinishError!void { - const a = try b.declareArgs(.{ - .partial_output_ptr = .{ .ptr = .f32 }, - .partial_lse_ptr = .{ .ptr = .f32 }, - .attn_sink_ptr = .{ .ptr = cfg.sink_dtype }, - .active_query_count_ptr = .{ .ptr = .i32 }, - .output_stride_0_ptr = .{ .ptr = .i64 }, - .output_stride_1_ptr = .{ .ptr = .i64 }, - .output_ptr = .{ .ptr = cfg.o_dtype }, - }); - - const output_stride_0 = b.load(a.output_stride_0_ptr); - const output_stride_1 = b.load(a.output_stride_1_ptr); - const query_idx = b.programId(.x); - const head_idx = b.programId(.y); - if (!cfg.all_decode) { - const active_query_count = b.load(a.active_query_count_ptr); - b.returnIf(query_idx.ge(active_query_count), .{}); - } - const offs_s = b.arange(0, cfg.num_splits, .i32); - const offs_d = b.arange(0, cfg.value_rank, .i32); - - const partial_lse_token_stride: i64 = cfg.num_query_heads * cfg.num_splits; - const partial_lse_offset = query_idx.to(.i64).mul(partial_lse_token_stride) - .add(head_idx.to(.i64).mul(cfg.num_splits)) - .add(offs_s.to(.i64)); - const partial_lse = b.load(a.partial_lse_ptr.addPtr(partial_lse_offset)); - - const sink_score = if (cfg.use_attn_sink) - b.load(a.attn_sink_ptr.addPtr(head_idx)).to(.f32) - else - b.liftAs(-std.math.inf(f32), .f32); - const overall_max = b.max(partial_lse).maximum(sink_score); - const has_value = overall_max.gt(-std.math.inf(f32)); - const safe_max = b.where(has_value, overall_max, b.liftAs(0.0, .f32)); - const split_weights = b.exp(partial_lse.sub(safe_max)); - const sink_weight = if (cfg.use_attn_sink) - b.exp(sink_score.sub(safe_max)) - else - b.liftAs(0.0, .f32); - const denominator = b.sumOpts(split_weights, .{ .axis = 0 }).add(sink_weight); - - const partial_out_token_stride: i64 = cfg.num_query_heads * cfg.num_splits * cfg.value_rank; - const partial_out_head_stride: i64 = cfg.num_splits * cfg.value_rank; - const partial_out_base = query_idx.to(.i64).mul(partial_out_token_stride) - .add(head_idx.to(.i64).mul(partial_out_head_stride)); - const partial_out_offset = b.broadcastTo( - partial_out_base.add(offs_s.expandDims(1).to(.i64).mul(cfg.value_rank)), - &.{ cfg.num_splits, cfg.value_rank }, - ).add(b.broadcastTo(offs_d.expandDims(0).to(.i64), &.{ cfg.num_splits, cfg.value_rank })); - const partial_output = b.load(a.partial_output_ptr.addPtr(partial_out_offset)); - const weighted_output = partial_output.mul(split_weights.expandDims(1)); - const numerator = b.sumOpts(weighted_output, .{ .axis = 0 }); - const safe_denominator = b.where( - denominator.gt(0.0), - denominator, - b.liftAs(1.0, .f32), - ); - const output = b.where( - has_value, - numerator.div(safe_denominator), - b.zeros(&.{cfg.value_rank}, .f32), - ); - - const output_offset = query_idx.to(.i64).mul(output_stride_0) - .add(head_idx.to(.i64).mul(output_stride_1)) - .add(offs_d.to(.i64)); - b.store(a.output_ptr.addPtr(output_offset), output.to(cfg.o_dtype)); -} diff --git a/zml/kernels/flash_attention/README.md b/zml/kernels/flash_attention/README.md new file mode 100644 index 00000000..b5fe7682 --- /dev/null +++ b/zml/kernels/flash_attention/README.md @@ -0,0 +1,287 @@ +# Flash attention semantics + +This directory exposes causal scaled dot-product attention through +[`zml.flash_attention.attention`](flash_attention.zig). This document describes +the attention operation, the FlashAttention algorithm, and ZML's current semantic +contract. Shapes below are logical tensor shapes. Sharding describes ownership +of those tensors across devices; platform-specific execution layouts are outside +the scope of this document. + +## The attention operation + +For one sequence and one head, let `Q` contain `Lq` query vectors, and let `K` and +`V` contain `Lk` key/value vectors. Each query is compared with each permitted key; +the resulting probabilities weight the corresponding values: + +```text +S = scale * Q @ transpose(K) + M +P = softmax(S, axis=keys) +O = P @ V +``` + +The usual scale is `1 / sqrt(D)`, where `D` is the query/key head dimension. +An additive mask `M` is zero for allowed pairs and negative infinity for forbidden +pairs. Softmax normalizes each query row independently over keys, so a masked key +has zero probability. Heads and batch elements are independent. ZML's +[`sdpa`](../../nn.zig) implements this reference computation. + +Attention in general permits a value dimension `Dv` different from the query/key +dimension `D`; the output then has dimension `Dv`. This module uses the same +`.hd = D` for queries, keys, values, and output. + +FlashAttention can evaluate either unrestricted attention or masked attention. +Causality is a choice of mask, rather than an inherent property of the algorithm. +Other attention APIs may offer local windows, additive score biases, a custom +scale, and dropout on the probabilities during training. These options change +the operation and must be matched when comparing results. The +[upstream FlashAttention API](https://github.com/Dao-AILab/flash-attention#how-to-use-flashattention) +documents such options; ZML's interface here exposes a narrower operation. + +## What makes attention "flash" + +FlashAttention computes the same mathematical attention result while avoiding +storage of the complete `Lq × Lk` score and probability matrices. It processes +query/key/value tiles and combines partial results using an online softmax. +This reduces intermediate memory and memory traffic. Dense attention still +requires `O(Lq * Lk * D)` arithmetic per head; for self-attention with `Lq = Lk`, +that remains quadratic in sequence length. "Exact" means the attention formula +is preserved, with ordinary floating-point rounding differences. See the +[FlashAttention paper](https://arxiv.org/abs/2205.14135). + +For each query row, online softmax maintains a maximum `m`, an exponential sum +`l`, and an unnormalized weighted-value vector `u`. Given scores `s_j` for a new +tile of permitted keys, update: + +```text +m_new = max(m, max_j(s_j)) +a = exp(m - m_new) +p_j = exp(s_j - m_new) +l_new = a * l + sum_j(p_j) +u_new = a * u + sum_j(p_j * V_j) +O = u / l # after all tiles +``` + +Initialize with `m = -infinity`, `l = 0`, and `u = 0`; skip tiles with no permitted +keys. Rescaling the previous accumulators preserves a single normalization across +all tiles. Independently normalizing each tile and adding its output would give +a different result. The [FlashAttention-2 paper, sections 2–3](https://arxiv.org/html/2307.08691v1) +describes this recurrence and how training can recompute probability tiles during +the backward pass instead of retaining the full matrix. This module exposes only +the attention output, with no backward entry point or returned probabilities. + +## Heads and head dimensions + +`D` counts features in **one head**. It is distinct from the number of heads and +the sequence length. In a conventional multi-head projection, +`D = hidden_size / Hq`; models with an explicit head dimension need not follow +that relation. The query projection has width `Hq * D`, while each key/value +projection has width `Hkv * D`. + +Typical dimensions include: + +| Head dimension | Context | +| --- | --- | +| `64` | A common transformer head size; also the size used by this module's shared tests. | +| `128` | Another common size; Qwen2.5-7B uses `3584 / 28 = 128`. | +| `256` | Used by models such as Gemma 2 2B and 9B. | +| `512` | Used by Gemma 4 global attention layers, for example in Gemma 4 31B. | + +The [FlashAttention-2 paper](https://arxiv.org/html/2307.08691v1) describes typical +head sizes around 64–128. The model examples follow the +[Qwen2.5-7B configuration](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct/raw/main/config.json) +and [Gemma configurations](https://github.com/google/gemma_pytorch/blob/main/gemma/config.py). +Gemma can also use different head dimensions within one model: the +[Gemma 4 31B configuration](https://huggingface.co/google/gemma-4-31B/blob/main/config.json) +specifies `head_dim = 256` for local attention and `global_head_dim = 512` for +global attention. Use the dimension configured for the specific layer. +These are model dimensions, not a list of supported kernel sizes or a mathematical +requirement that `D` be a power of two. + +The number of query heads `Hq` may differ from the number of key/value heads +`Hkv`. ZML uses contiguous groups of query heads: + +```text +G = Hq / Hkv +kv_head(h) = floor(h / G) +``` + +`Hq` must be divisible by `Hkv`, and keys and values must have the same head count. +This covers multi-head attention (MHA: `Hq = Hkv`), grouped-query attention +(GQA: `1 < Hkv < Hq`), and multi-query attention (MQA: `Hkv = 1`). For example, +with `Hq = 64` and `Hkv = 8`, query heads 0–7 use KV head 0, heads 8–15 use KV +head 1, and so on. Sharing keys and values does not merge query heads: each query +head still has its own scores, normalization, and output. + +## ZML inputs and output + +The entry point is: + +```zig +attention(q, k, v, token_index, metadata, parameters) +``` + +The reference contract uses these tagged shapes, with an optional `.b` batch axis: + +| Argument | Unbatched shape | Batched shape | Meaning | +| --- | --- | --- | --- | +| `q` | `{ .q = Lq, .h = Hq, .hd = D }` | `{ .b = B, .q = Lq, .h = Hq, .hd = D }` | Queries to evaluate. | +| `k`, `v` | `{ .k = Lk, .h = Hkv, .hd = D }` | `{ .b = B, .k = Lk, .h = Hkv, .hd = D }` | Keys and values in sequence order. | +| `token_index` | Scalar | `{ .b = B }` or scalar | Zero-based position of the first query in the key sequence; a scalar applies to the whole batch. | +| Result | Same as `q` | Same as `q` | Weighted values for every query head. | + +Use matching floating-point dtypes for `q`, `k`, and `v`; the output retains the +query dtype. The shared tests use `bf16` tensors and `u32` positions. `metadata` +and `parameters` provide execution configuration rather than additional masks, +score biases, or model projections. + +### Causality and query positions + +Let `t = token_index` for a sequence. Query row `i` represents position `t + i`, +and key row `j` represents position `j`. The inclusive causal rule is: + +```text +allowed(i, j) = (j <= t + i) +M[i, j] = 0 if allowed(i, j), otherwise -infinity +``` + +The reference path constructs a square causal mask over `Lk` positions, gathers +`Lq` consecutive rows starting at `t`, and passes that mask to `sdpa`. Valid inputs +require `Lq > 0`, `t >= 0`, and `t + Lq <= Lk`. The test helper checks the upper +bound; callers must satisfy it rather than rely on out-of-range gather behavior. +Each valid query can attend to at least its own key, so fully masked rows are +outside this contract. + +| Use case | Query positions | Keys visible to query row `i` | +| --- | --- | --- | +| Initial prefill | `t = 0`, usually `Lq = Lk` | `0` through `i`. | +| Chunked prefill | `Lq > 1`, starting at `t` | `0` through `t + i`. | +| Single-token decode | `Lq = 1`, current token at `t` | `0` through `t`, including the current token. | + +For `Lq = 2`, `Lk = 5`, and `t = 2`, the mask is: + +```text + key position + 0 1 2 3 4 +query at 2 1 1 1 0 0 +query at 3 1 1 1 1 0 +``` + +Here `1` means permitted. `Lk` may be the capacity of a preallocated KV buffer; +it does not determine the current query position. All permitted KV entries must +already be populated. Positions after the last query are masked, and the call +does not append to or update a KV cache. + +For unequal query and key lengths, causal alignment must be specified explicitly. +The [upstream FlashAttention API](https://github.com/Dao-AILab/flash-attention#21-change-behavior-of-causal-flag) +uses bottom-right alignment: `j <= i + Lk - Lq`. ZML's reference rule agrees with +that convention when `t = Lk - Lq`. An explicit `token_index` also describes +queries earlier in a larger buffer, as in the example above. + +### Sharding and local attention + +Sharding changes where attention is evaluated, not its mathematical result. +The global output still has the query's tagged shape and dtype. Each local call +must receive the matching queries, keys, values, and positions for the batch +elements and heads it owns. + +| Axis or input | Sharding semantics | +| --- | --- | +| Batch `.b` | Sequences are independent. Q, K, V, and the output must assign the same batch elements to each shard. | +| Heads `.h` | Each shard must have the KV heads needed by its query heads, preserving `kv_head(h) = floor(h / G)`. K and V must have matching head ownership. | +| Query sequence `.q` | Query rows are independent once the required KV entries are available. Their causal positions remain positions in the original sequence. | +| Key sequence `.k` | Softmax normalizes over all permitted keys for a query, including keys on other shards if the sequence is distributed. Separate local softmax outputs cannot simply be added or averaged. | +| Features `.hd` | Every score uses the complete head dimension and the global scale `1 / sqrt(D)`. Splitting features requires combining partial dot products before softmax. | +| `token_index` | A scalar applies to every shard. Per-sequence positions follow the batch partition and are replicated across head shards of the same sequence. Head or batch sharding does not change their values. | + +For head sharding without KV replication, a partition count `P` must divide +`Hkv`, and therefore also `Hq`. Each shard owns complete, contiguous GQA groups: +`Hq_local = Hq / P`, `Hkv_local = Hkv / P`, and the local ratio remains `G`. +For example, with `Hq = 32`, `Hkv = 8`, and two shards, the first shard owns query +heads 0–15 and KV heads 0–3; the second owns query heads 16–31 and KV heads 4–7. +Each query still uses exactly the same KV head as in the unsharded computation. +MQA has only one KV head, so distributing its query heads over multiple devices +requires replicating that KV head rather than dividing it into independent heads. + +The local-kernel execution contract keeps `.q`, `.k`, and `.hd` complete on each +shard, with parallelism across batches and/or complete head groups. Thus each +local call has the full sequence coordinates and feature vectors needed by the +causal mask, dot products, and normalization. The mathematical possibilities for +sequence and feature sharding above do not imply that these local kernels +implement distributed sequence or feature reductions. + +An **open** partition means placement is unresolved; it does not mean the tensor +must be replicated. Operations such as cache slicing and RoPE can reopen +partition annotations even when their inputs were sharded. Before entering a +local kernel, resolve compatible partitions on a common mesh and align Q, K, V, +and positions. The KV layout anchors this alignment: an explicit KV head +partition is retained, while an open head partition can use a compatible mesh +axis whose partition count divides `Hkv`. Preserve explicit batch ownership and +do not assign the same device partition independently to both batch and heads. +An explicit replicated KV head axis remains replicated; open heads can also +remain replicated when no compatible head partition is available. Automatic +partition selection is an execution choice, not an additional attention argument. + +The output follows the **resolved** query partition. Its partition annotations +need not match the incoming query's unresolved annotations, and aligning operands +may require communication. In particular, unnecessarily resolving open KV heads +to replicated can gather the full cache on every attention call even though the +head groups could be evaluated independently. This changes cost, not attention +semantics, and does not modify the caller's KV cache. + +Splitting KV work into internal tiles or partitions is distinct from sharding the +input tensors across devices. Either way, combining partial attention states +must preserve the single softmax normalization. For disjoint key subsets with +states `(m_r, l_r, u_r)` as defined above: + +```text +m = max_r(m_r) +l = sum_r(exp(m_r - m) * l_r) +u = sum_r(exp(m_r - m) * u_r) +O = u / l +``` + +An empty subset contributes zero (`m_r = -infinity`, `l_r = 0`, `u_r = 0`). +Every valid query has at least one permitted key overall, so its merged +normalization is nonempty. + +### Scope and current conformance + +The public call always requests causal attention with scale `1 / sqrt(D)` and no +dropout. It has no arguments for an arbitrary mask, padding lengths, packed +sequence boundaries, a local window, ALiBi, score soft-capping, a softmax sink, or +a custom scale. The lower-level `sdpa` and `causalAttnMask` helpers expose some +additional options, but this wrapper does not pass them through. Separate batch +elements can have different `token_index` values; concatenating independent +sequences along `.k` does not isolate them from each other. + +Q/K/V projections, positional transformations such as RoPE, and any Q/K +normalization must happen before this call. Head concatenation and the output +projection happen afterward. A model's use of a familiar head dimension alone +does not establish compatibility with this operation. + +The reference contract above is broader than current kernel coverage. In +particular, nonzero-offset multi-query prefill and the optional batch dimension +are not uniformly supported by the current kernels; an existing prefill path +requires equal query/key lengths and assumes positions begin at zero. Treat +these as current conformance gaps, not alternative definitions of causality. + +The tests in [flash_attention.zig](flash_attention.zig) compare results +against `sdpa` with the gathered causal mask. The original shared cases cover +`D = 64`, MHA and GQA, +single-token decode at full and partial context lengths, batched decode with +different positions, and an eight-query chunk beginning at position 56 in a +64-position KV buffer. Additional conformance cases exercise head dimensions +through `512`, MQA, `bf16`/`f16`/`f32`, initial and offset prefill, partial tiles, +transposed tagged layouts, scalar batch positions, head/batch sharding, open +query/KV partitions, and split decode with empty partitions and cache tails. These +cases are enabled for kernels that support them; their presence does not imply +uniform coverage across every kernel. FP32 results are also checked against an +independent host calculation using FP64 arithmetic. + +Numerical agreement is tolerance-based. The reference evaluates softmax in +`f32`, then converts probabilities to the query dtype before multiplying by +values. Different accumulation orders and rounding points can produce small +differences even when the mathematical operation is identical. The shared tests +use absolute tolerance `5e-3`, relative tolerance `1e-2`, relative epsilon `1e-3`, +and a minimum close fraction of `0.99`; these are test criteria rather than a +universal error bound. diff --git a/zml/kernels/flash_attention/cute/BUILD.bazel b/zml/kernels/flash_attention/cute/BUILD.bazel new file mode 100644 index 00000000..b83a642e --- /dev/null +++ b/zml/kernels/flash_attention/cute/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "cute", + srcs = [ + "attention.zig", + "cute.zig", + ], +) diff --git a/zml/kernels/flash_attention/cute/README.md b/zml/kernels/flash_attention/cute/README.md new file mode 100644 index 00000000..8c55d2c8 --- /dev/null +++ b/zml/kernels/flash_attention/cute/README.md @@ -0,0 +1,145 @@ +# CuTe flash attention scheduling + +The [parent README](../README.md) defines attention and sharding semantics. +This document describes decode scheduling in this implementation and the +FlashInfer heuristics that inform it. Scheduling changes parallelism and +intermediate storage, not the attention formula or public arguments. + +## What FlashInfer does + +These references are pinned to FlashInfer commit +`2a2cceab07c7bfca0525065fbfdc5fb2cff0c33a`; FlashInfer has multiple execution paths, +so these rules describe the cited CUDA decode paths rather than every backend. + +[Single-sequence decode](https://github.com/flashinfer-ai/flashinfer/blob/2a2cceab07c7bfca0525065fbfdc5fb2cff0c33a/include/flashinfer/attention/decode.cuh#L705-L744) +uses one attention launch when the live sequence has at most 256 tokens or no +scratch buffer is supplied. Otherwise it queries the compiled kernel's resident +blocks per SM and the device's SM count, then computes: + +```text +grid_budget = resident_blocks_per_SM * SM_count +split_budget = floor(grid_budget / KV_heads) +chunk_tokens = max(ceil(live_tokens / split_budget), 256) +splits = ceil(live_tokens / chunk_tokens) +``` + +It launches partial attention over these chunks followed by `MergeStates`. +The merge preserves a single softmax normalization across the chunks. + +[Batched paged decode](https://github.com/flashinfer-ai/flashinfer/blob/2a2cceab07c7bfca0525065fbfdc5fb2cff0c33a/include/flashinfer/attention/scheduler.cuh#L75-L98) +uses the per-request lengths to search for a chunk size whose total block count +fits the occupancy budget. Its +[work estimator](https://github.com/flashinfer-ai/flashinfer/blob/2a2cceab07c7bfca0525065fbfdc5fb2cff0c33a/include/flashinfer/attention/scheduler.cuh#L174-L205) +disables splitting if `batch_size * KV_heads` already fills that budget. The +minimum chunk is `max(floor(128 / page_size), 1)` pages. Without CUDA graphs it +also avoids splitting when partitioning adds no requests. Graph execution can +retain a padded split schedule to keep launch and allocation sizes stable. + +## Adaptation to this kernel + +[`decodeSplits`](attention.zig) runs after sharding has established each device's +local dimensions. A block owns at most four query heads belonging to one KV +head, so our unsplit decode block count is: + +```text +G = local_query_heads / local_KV_heads +base_blocks = local_batch * local_KV_heads * ceil(G / 4) +``` + +Counting query-head groups matters: a large GQA group already launches multiple +blocks per KV head in this kernel. Using global batch or head counts would +underestimate the need for splitting after partitioning across devices. + +XLA compiles and owns the CUDA function after this planner runs. We cannot call +CUDA's occupancy calculator on that function here. Instead, the planner uses +PJRT's `core_count` and `shared_memory_per_block_optin` attributes and a +static limit of twelve 128-thread blocks per SM: + +```text +padded_D = next_power_of_two(max(D, 32)) +shared_bytes_per_block = 2 * 16 * padded_D * sizeof(dtype) # K and V tiles +blocks_per_SM = max(1, min(12, floor(shared_budget / shared_bytes_per_block))) +grid_budget = 4 * SM_count * blocks_per_SM +``` + +The per-block shared-memory limit serves as a conservative shared-memory budget; +it is not a measurement of SM capacity or compiled-kernel occupancy. The +twelve-block limit is a scheduling estimate, not a measured occupancy or a +hardware constant. It provides enough resident work to hide latency in the +128-thread kernel; register usage and other hardware limits can reduce actual +occupancy. Four scheduling waves provide additional work for the serial +KV-tile loop; this is a measured adaptation, whereas the cited FlashInfer CUDA +planner targets one resident grid. Larger heads and FP32 reduce the budget through their shared-memory requirements. For +multiple visible devices, the smallest reported SM count and shared-memory +budget are used. Missing or invalid attributes disable splitting. + +The selection rules are: + +1. Prefill (`Lq > 1`) remains unsplit. +2. A cache capacity of at most 64 tokens remains unsplit. +3. If `base_blocks >= grid_budget`, keep one partition per block's head group. +4. Otherwise set `split_budget = ceil(grid_budget / base_blocks)`, + `minimum_chunk = 16` for capacities through 256, otherwise `32`, + `chunk = max(minimum_chunk, ceil(cache_capacity / split_budget))`, and + `splits = ceil(cache_capacity / chunk)`. +5. If the result is one split, launch attention directly into the output. Other + cases allocate FP32 partial states and launch attention followed by a merge. + +The chunk floors are one or two of this kernel's 16-key tiles. They adapt +FlashInfer's minimum-work-per-partition heuristic to our serial per-tile score +loop. Applying its literal 128/256-token floors and a smaller block budget +regressed several measured cases. The workload-based budget and split/merge +structure follow FlashInfer, but these constants are specific to this kernel. +The split budget rounds up a partial wave: rounding 32 partitions down to 31, +for example, can add a whole tile to every block. The grid budget is therefore +a target, and the launched grid may exceed it by less than one split's worth +of `base_blocks`. + +## Capacity and live lengths + +Our graph receives `token_index` on device, potentially with a different value +for each batch element. Planning uses static cache capacity to bound the grid +and scratch allocation without reading positions back to the host. At runtime, +each sequence distributes its **live** prefix of `token_index + 1` keys evenly +across the planned splits. Therefore the actual chunk can be smaller than the +planning floor. Empty partitions contribute zero to the merge. + +This is not FlashInfer's host-side ragged-length search. A short live sequence +in a large cache can still use the split path, and the merge launch remains in +the graph even when only one partition has keys. The schedule does not change +from token to token, so it can be reused during generation and graph capture. + +Partial states contain the weighted-value accumulator, maximum, and exponential +sum in FP32. Their storage is +`4 * local_batch * splits * local_query_heads * (D + 2)` bytes. Reducing splits +as the batch grows reduces both scratch traffic and merge work. Scores are +scaled by `log2(e) / sqrt(D)` so attention and merging use base-2 exponentials. + +## Validation and timing + +The existing attention harness covers GQA/MQA, different positions within a +batch, partial tiles, empty partitions, sharding, three dtypes, and head +dimensions through 512. Planner tests cover the short-cache boundary, decreasing +split counts as batch size grows, resource limits, and extra query-head groups. +Runtime checks include batches through 2048 at the 64/65-token scheduling +boundary; they establish correctness, not throughput at that batch size. + +An opt-in benchmark in [cute.zig](cute.zig) compares the previous 32-split policy +with the planner at batches 1, 4, 16, and 64, and full contexts of 128, 2048, and +8192 tokens. It uses BF16, 16 query heads, four KV heads, and `D = 128` per device. +Sixteen dependent attention calls amortize host overhead; reported microseconds +per attention call are the median of 16 samples after three warmups. This is a +repeated-cache kernel benchmark, not end-to-end model throughput. + +```sh +bazel test //zml:test --//platforms:cuda=true --//platforms:cpu=false \ + --@rules_zig//zig/settings:zigopt=--test-filter \ + --@rules_zig//zig/settings:zigopt=zml.test.zml \ + --@rules_zig//zig/settings:zigopt=--test-filter \ + '--@rules_zig//zig/settings:zigopt=CuTe decode' \ + --test_env=CUDA_VISIBLE_DEVICES=0 \ + --test_env=ZML_CUTE_ATTENTION_BENCHMARK=1 --test_output=all +``` + +`CUTE_DECODE_BENCH` rows contain batch, context length, policy, split count, and +microseconds per call. Timing is not a correctness assertion. diff --git a/zml/kernels/flash_attention/cute/attention.zig b/zml/kernels/flash_attention/cute/attention.zig new file mode 100644 index 00000000..016dee40 --- /dev/null +++ b/zml/kernels/flash_attention/cute/attention.zig @@ -0,0 +1,300 @@ +const std = @import("std"); + +const zml = @import("../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const Value = cute.Value; + +pub const Config = struct { + batch: i64, + query_length: i64, + key_length: i64, + query_heads: i64, + kv_heads: i64, + head_dim: i64, + dtype: cute.DType, + splits: i64 = 1, +}; + +/// Planning uses local (post-sharding) dimensions. XLA owns the compiled CUDA +/// function, so this is an estimated resource budget, not an occupancy query. +pub const DecodeResources = struct { + multiprocessors: i64, + shared_bytes: i64, +}; + +pub fn decodeSplits(q: zml.Shape, k: zml.Shape, resources: DecodeResources) i64 { + const query_length = q.dim(.q); + const key_length = k.dim(.k); + const num_heads = q.dim(.h); + const num_kv_heads = k.dim(.h); + const head_dim = q.dim(.hd); + const batch = if (q.hasTag(.b) != null) q.dim(.b) else 1; + const dtype = q.dtype(); + + if (query_length != 1 or key_length <= 64) return 1; + if (resources.multiprocessors <= 0 or resources.shared_bytes <= 0) return 1; + const group = @divExact(num_heads, num_kv_heads); + const blocks_per_sequence = num_kv_heads * @divTrunc(group + 3, 4); + const base_blocks = batch * blocks_per_sequence; + var padded_dim: i64 = 32; + while (padded_dim < head_dim) padded_dim *= 2; + const bytes_per_element: i64 = if (dtype == .f32) 4 else 2; + const shared_per_block = 2 * tile_keys * padded_dim * bytes_per_element; + const blocks_per_sm = @max(1, @min(12, @divTrunc(resources.shared_bytes, shared_per_block))); + // Multiple scheduling waves keep the serial tile loop supplied with work. + // The measured tradeoff and distinction from CUDA occupancy are in README. + const grid_budget = 4 * resources.multiprocessors * blocks_per_sm; + if (base_blocks >= grid_budget) return 1; + // Round up a partial wave: flooring can leave SMs idle and add a whole + // extra KV tile to every block (e.g. 31 partitions versus 32). + const split_budget = @divTrunc(grid_budget + base_blocks - 1, base_blocks); + // One tile for short caches, two otherwise. FlashInfer's larger chunk + // floor is tuned for a different per-block KV processing rate. + // With static dense shapes, capacity bounds the scratch space and grid; + // each invocation balances its live prefix over these partitions on device. + const min_chunk: i64 = if (key_length <= 256) tile_keys else 2 * tile_keys; + const chunk = @max(min_chunk, @divTrunc(key_length + split_budget - 1, split_budget)); + return @divTrunc(key_length + chunk - 1, chunk); +} + +test "CuTe decode planner respects workload and resource limits" { + const resources: DecodeResources = .{ .multiprocessors = 170, .shared_bytes = 101376 }; + var q = zml.Shape.init(.{ .b = 1, .q = 1, .h = 16, .hd = 128 }, .bf16); + var k = zml.Shape.init(.{ .b = 1, .k = 2048, .h = 4, .hd = 128 }, .bf16); + try std.testing.expectEqual(64, decodeSplits(q, k, resources)); + k = k.setDim(.k, 64); + try std.testing.expectEqual(1, decodeSplits(q, k, resources)); + k = k.setDim(.k, 65); + try std.testing.expectEqual(5, decodeSplits(q, k, resources)); + k = k.setDim(.k, 2048); + var previous: i64 = 64; + for ([_]i64{ 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048 }) |batch| { + q = q.setDim(.b, batch); + k = k.setDim(.b, batch); + const splits = decodeSplits(q, k, resources); + try std.testing.expect(splits <= previous); + try std.testing.expect(splits >= 1); + try std.testing.expect(splits == 1 or (splits - 1) * batch * 4 < 8160); + previous = splits; + } + try std.testing.expectEqual(1, previous); + q = q.setDim(.b, 16); + k = k.setDim(.b, 16); + q = q.setDim(.hd, 512); + k = k.setDim(.hd, 512); + q = q.withDtype(.f32); + k = k.withDtype(.f32); + try std.testing.expectEqual(11, decodeSplits(q, k, resources)); + q = q.setDim(.q, 7); + try std.testing.expectEqual(1, decodeSplits(q, k, resources)); + q = q.setDim(.q, 1); + try std.testing.expectEqual(1, decodeSplits(q, k, .{ .multiprocessors = 0, .shared_bytes = 0 })); + // More query-head groups already create more independent CTAs. + q = q.setDim(.b, 1); + k = k.setDim(.b, 1); + q = q.setDim(.hd, 128); + k = k.setDim(.hd, 128); + q = q.withDtype(.bf16); + k = k.withDtype(.bf16); + q = q.setDim(.h, 1024); + try std.testing.expectEqual(32, decodeSplits(q, k, resources)); +} + +pub const block: [3]i32 = .{ 128, 1, 1 }; +const tile_keys = 16; + +pub const Kernel = cute.Kernel(Config, .{ + .name = "flash_attention_cute", + .inputs = &.{ "q", "k", "v", "positions" }, + .outputs = &.{"out"}, + .run = run, +}); + +fn run(b: *B, cfg: Config) cute.FinishError!void { + std.debug.assert(cfg.head_dim > 0 and cfg.head_dim <= 512); + inline for (.{ 1, 2, 4, 8, 16 }) |fragments| { + if (cfg.head_dim <= fragments * 32) return runSized(b, cfg, fragments); + } + unreachable; +} + +fn tupleOf(array: anytype) std.meta.Tuple(&([_]type{Value} ** array.len)) { + var result: std.meta.Tuple(&([_]type{Value} ** array.len)) = undefined; + inline for (0..array.len) |i| result[i] = array[i]; + return result; +} + +/// Four warps own four (query position, query head) rows sharing one KV head. +/// Only a KV tile lives in shared memory; maxima, sums, and outputs stay in +/// registers. All lanes participate in shuffles, including padded rows/features. +fn runSized(b: *B, cfg: Config, comptime fragments: usize) cute.FinishError!void { + const padded_dim = fragments * 32; + const split = cfg.splits > 1; + const out_shape = if (split) &[_]i64{ cfg.batch, cfg.splits, cfg.query_length, cfg.query_heads, cfg.head_dim + 2 } else &[_]i64{ cfg.batch, cfg.query_length, cfg.query_heads, cfg.head_dim }; + const a = try b.declareArgs(.{ + .q = .{ .tensor = .{ .dtype = cfg.dtype, .shape = &.{ cfg.batch, cfg.query_length, cfg.query_heads, cfg.head_dim } } }, + .k = .{ .tensor = .{ .dtype = cfg.dtype, .shape = &.{ cfg.batch, cfg.key_length, cfg.kv_heads, cfg.head_dim } } }, + .v = .{ .tensor = .{ .dtype = cfg.dtype, .shape = &.{ cfg.batch, cfg.key_length, cfg.kv_heads, cfg.head_dim } } }, + .positions = .{ .tensor = .{ .dtype = .i32, .shape = &.{cfg.batch} } }, + .out = .{ .tensor = .{ .dtype = if (split) .f32 else cfg.dtype, .shape = out_shape } }, + }); + const tid = b.threadIdx().x; + const lane = tid.rem(32); + const group = @divExact(cfg.query_heads, cfg.kv_heads); + const batch = b.blockIdx().y.div(cfg.kv_heads); + const kv_head = b.blockIdx().y.rem(cfg.kv_heads); + const first_row = b.blockIdx().x.mul(4); + const row = first_row.add(tid.div(32)); + // Padded warps compute a valid dummy row and suppress their final stores. + const query = row.div(group).minimum(cfg.query_length - 1); + const head = kv_head.mul(group).add(row.rem(group)); + const start = a.positions.get(.{batch}); + const position = start.add(query); + const causal_end = start.add(first_row.add(3).div(group).minimum(cfg.query_length - 1)).add(1); + // Partition the live cache, rather than its capacity, so decode uses all + // blocks even when only a prefix of the cache is populated. + const chunk = causal_end.add(cfg.splits - 1).div(cfg.splits); + const begin = if (split) b.blockIdx().z.mul(chunk) else b.cst(.i32, 0); + const end = if (split) begin.add(chunk).minimum(causal_end) else causal_end; + const zero = b.cst(.f32, 0); + const neg_inf = b.cst(.f32, -std.math.inf(f32)); + + var queries: [fragments]Value = undefined; + inline for (0..fragments) |f| { + const d = lane.add(f * 32); + var valid = b.openIfElse(d.lt(cfg.head_dim), .{cute.DType.f32.toMlir(b.ctx)}); + valid.yieldThen(.{a.q.get(.{ batch, query, head, d }).to(.f32)}); + valid.yieldElse(.{zero}); + queries[f] = valid.results[0]; + } + + const layout = b.makeLayout(&.{ tile_keys, padded_dim }, .{ .stride = &.{ padded_dim, 1 } }); + const keys = b.makeTensor(b.allocSmem(cfg.dtype, tile_keys * padded_dim, 16), layout); + const values = b.makeTensor(b.allocSmem(cfg.dtype, tile_keys * padded_dim, 16), layout); + var initial: [fragments + 2]Value = @splat(zero); + initial[0] = neg_inf; + var tiles = b.openFor(begin, end, tile_keys, tupleOf(initial)); + + // Zero-fill partial tiles and feature padding before the CTA-wide barrier. + var copy = b.openFor(tid, tile_keys * padded_dim, 128, .{}); + const tile_key = copy.iv.div(padded_dim); + const d = copy.iv.rem(padded_dim); + const key = tiles.iv.add(tile_key); + var valid_load = b.openIfElse(key.lt(end).bitAnd(d.lt(cfg.head_dim)), .{ cfg.dtype.toMlir(b.ctx), cfg.dtype.toMlir(b.ctx) }); + valid_load.yieldThen(.{ a.k.get(.{ batch, key, kv_head, d }), a.v.get(.{ batch, key, kv_head, d }) }); + valid_load.yieldElse(.{ zero.to(cfg.dtype), zero.to(cfg.dtype) }); + keys.set(.{ tile_key, d }, valid_load.results[0]); + values.set(.{ tile_key, d }, valid_load.results[1]); + copy.yield(.{}); + b.syncThreads(); + + // Each lane keeps one score; dot products reduce across the feature lanes. + var scores = b.openFor(b.cst(.i32, 0), tile_keys, 1, .{neg_inf}); + var dot = zero; + inline for (0..fragments) |f| { + dot = dot.add(queries[f].mul(keys.get(.{ scores.iv, lane.add(f * 32) }).to(.f32))); + } + inline for (.{ 16, 8, 4, 2, 1 }) |offset| dot = dot.add(b.shuffleXor(dot, offset)); + const scale = std.math.log2e / std.math.sqrt(@as(f32, @floatFromInt(cfg.head_dim))); + const score = b.select(tiles.iv.add(scores.iv).le(position).bitAnd(tiles.iv.add(scores.iv).lt(end)), dot.mul(scale), neg_inf); + scores.yield(.{b.select(lane.eq(scores.iv), score, scores.carried[0])}); + var maximum = scores.results[0]; + inline for (.{ 16, 8, 4, 2, 1 }) |offset| maximum = maximum.maximum(b.shuffleXor(maximum, offset)); + maximum = maximum.maximum(tiles.carried[0]); + const rescale = b.exp2Approx(tiles.carried[0].sub(maximum)); + const probability = b.exp2Approx(scores.results[0].sub(maximum)); + var sum = probability; + inline for (.{ 16, 8, 4, 2, 1 }) |offset| sum = sum.add(b.shuffleXor(sum, offset)); + sum = sum.add(tiles.carried[1].mul(rescale)); + + var accumulators: [fragments]Value = undefined; + inline for (0..fragments) |f| accumulators[f] = tiles.carried[f + 2].mul(rescale); + var weighted = b.openFor(b.cst(.i32, 0), tile_keys, 1, tupleOf(accumulators)); + const weight = b.shuffleIdx(probability, weighted.iv); + inline for (0..fragments) |f| { + accumulators[f] = weighted.carried[f].add(weight.mul(values.get(.{ weighted.iv, lane.add(f * 32) }).to(.f32))); + } + weighted.yield(tupleOf(accumulators)); + // No warp may overwrite the tile before every warp has consumed it. + b.syncThreads(); + var next: [fragments + 2]Value = undefined; + next[0] = maximum; + next[1] = sum; + inline for (0..fragments) |f| next[f + 2] = weighted.results[f]; + tiles.yield(tupleOf(next)); + + inline for (0..fragments) |f| { + const col = lane.add(f * 32); + var valid_store = b.openIf(row.lt(cfg.query_length * group).bitAnd(col.lt(cfg.head_dim))); + if (split) { + a.out.set(.{ batch, b.blockIdx().z, query, head, col }, tiles.results[f + 2]); + } else { + a.out.set(.{ batch, query, head, col }, tiles.results[f + 2].div(tiles.results[1])); + } + valid_store.yieldThen(.{}); + } + if (split) { + var stats = b.openIf(row.lt(cfg.query_length * group).bitAnd(lane.eq(0))); + a.out.set(.{ batch, b.blockIdx().z, query, head, cfg.head_dim }, tiles.results[0]); + a.out.set(.{ batch, b.blockIdx().z, query, head, cfg.head_dim + 1 }, tiles.results[1]); + stats.yieldThen(.{}); + } +} + +/// Merge unnormalized partial attention states in FP32. Empty partitions have +/// m=-inf, l=0, u=0 and contribute zero, including at the first decode token. +pub const Merge = cute.Kernel(Config, .{ + .name = "flash_attention_cute_merge", + .inputs = &.{"partials"}, + .outputs = &.{"out"}, + .run = merge, +}); + +fn merge(b: *B, cfg: Config) cute.FinishError!void { + const a = try b.declareArgs(.{ + .partials = .{ .tensor = .{ .dtype = .f32, .shape = &.{ cfg.batch, cfg.splits, cfg.query_heads, cfg.head_dim + 2 } } }, + .out = .{ .tensor = .{ .dtype = cfg.dtype, .shape = &.{ cfg.batch, cfg.query_heads, cfg.head_dim } } }, + }); + const batch = b.blockIdx().y; + const head = b.blockIdx().x; + const lane = b.threadIdx().x.rem(32); + const zero = b.cst(.f32, 0); + const neg_inf = b.cst(.f32, -std.math.inf(f32)); + // Every warp reduces the partition statistics, then independently handles + // a contiguous range of output features. + var maxima = b.openFor(lane, cfg.splits, 32, .{neg_inf}); + maxima.yield(.{maxima.carried[0].maximum(a.partials.get(.{ batch, maxima.iv, head, cfg.head_dim }))}); + var maximum = maxima.results[0]; + inline for (.{ 16, 8, 4, 2, 1 }) |offset| maximum = maximum.maximum(b.shuffleXor(maximum, offset)); + const weights = b.makeTensor(b.allocSmem(.f32, cfg.splits, 16), b.makeLayout(&.{cfg.splits}, .{})); + var stats = b.openFor(b.threadIdx().x, cfg.splits, 128, .{}); + weights.set(.{stats.iv}, b.exp2Approx(a.partials.get(.{ batch, stats.iv, head, cfg.head_dim }).sub(maximum))); + stats.yield(.{}); + b.syncThreads(); + var features = b.openFor(b.threadIdx().x, cfg.head_dim, 128, .{}); + var sum = b.openFor(b.cst(.i32, 0), cfg.splits, 1, .{ zero, zero }); + const weight = weights.get(.{sum.iv}); + sum.yield(.{ + sum.carried[0].add(weight.mul(a.partials.get(.{ batch, sum.iv, head, features.iv }))), + sum.carried[1].add(weight.mul(a.partials.get(.{ batch, sum.iv, head, cfg.head_dim + 1 }))), + }); + a.out.set(.{ batch, head, features.iv }, sum.results[0].div(sum.results[1])); + features.yield(.{}); +} + +test "CuTe attention emits tiled online softmax through head dimension 512" { + for ([_]i64{ 64, 80, 128, 256, 512 }) |head_dim| { + const ir = try Kernel.emit(std.testing.allocator, .{ + .batch = 2, + .query_length = 7, + .key_length = 37, + .query_heads = 6, + .kv_heads = 2, + .head_dim = head_dim, + .dtype = .bf16, + }, block); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "scf.for") != null); + } +} diff --git a/zml/kernels/flash_attention/cute/cute.zig b/zml/kernels/flash_attention/cute/cute.zig new file mode 100644 index 00000000..2958888d --- /dev/null +++ b/zml/kernels/flash_attention/cute/cute.zig @@ -0,0 +1,108 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const kernel = @import("attention.zig"); + +test { + _ = kernel; +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + const cc = zml.platform.cuda.computeCapability(platform) orelse return false; + return cc.major >= 8; +} + +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor) zml.Tensor { + stdx.debug.assert(q.shape().hasTags(.{ .q, .h, .hd }) and k.shape().hasTags(.{ .k, .h, .hd }) and v.shape().hasTags(.{ .k, .h, .hd }), "CuTe attention requires q/h/hd and k/h/hd tags", .{}); + const batched = q.shape().hasTag(.b) != null; + const rank: usize = if (batched) 4 else 3; + stdx.debug.assert(q.rank() == rank and k.rank() == rank and v.rank() == rank, "CuTe attention expects rank-3 tensors or rank-4 tensors with a batch axis", .{}); + stdx.debug.assert(q.dtype() == k.dtype() and q.dtype() == v.dtype(), "CuTe attention requires matching Q/K/V dtypes", .{}); + stdx.debug.assert(q.dtype() == .bf16 or q.dtype() == .f16 or q.dtype() == .f32, "CuTe attention supports bf16, f16, and f32", .{}); + stdx.debug.assert(q.dim(.hd) > 0 and q.dim(.hd) <= 512 and q.dim(.hd) == k.dim(.hd) and k.dim(.hd) == v.dim(.hd), "CuTe attention requires matching head dimensions in [1, 512]", .{}); + stdx.debug.assert(k.dim(.h) > 0 and q.dim(.h) > 0 and @rem(q.dim(.h), k.dim(.h)) == 0 and k.dim(.h) == v.dim(.h), "CuTe attention requires Hq divisible by Hkv and matching K/V heads", .{}); + stdx.debug.assert(q.dim(.q) > 0 and q.dim(.q) <= k.dim(.k) and k.dim(.k) == v.dim(.k), "CuTe attention requires 0 < Lq <= Lk and matching K/V lengths", .{}); + stdx.debug.assert(token_index.dtype().isInteger(), "CuTe attention requires integer token positions", .{}); + if (batched) { + stdx.debug.assert(k.shape().hasTag(.b) != null and v.shape().hasTag(.b) != null and q.dim(.b) == k.dim(.b) and q.dim(.b) == v.dim(.b), "CuTe attention requires matching batch dimensions", .{}); + stdx.debug.assert(token_index.rank() == 0 or (token_index.rank() == 1 and token_index.shape().hasTag(.b) != null and token_index.dim(.b) == q.dim(.b)), "CuTe attention requires a scalar position or one position per batch element", .{}); + } else { + stdx.debug.assert(token_index.rank() == 0, "Unbatched CuTe attention requires a scalar token position", .{}); + } + + const mesh = zml.Compiler.current().resolveMesh(.{.model}); + const q_sharded = q.withPartitioning(mesh, .{ .h = .model }); + const k_sharded = k.withPartitioning(mesh, .{ .h = .model }); + const v_sharded = v.withPartitioning(mesh, .{ .h = .model }); + const positions = if (batched) + token_index.convert(.i32).broad(zml.Shape.init(.{ .b = q.dim(.b) }, .i32)) + else + token_index.convert(.i32); + return zml.ops.manualComputation(Context.body, Context{ .q = q_sharded, .k = k_sharded, .v = v_sharded, .positions = positions }, q_sharded.shape()); +} + +const Context = struct { + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + positions: zml.Tensor, + + fn body(self: Context, output_shape: zml.Shape) zml.Tensor { + const batched = self.q.shape().hasTag(.b) != null; + const q = if (batched) self.q.transpose(.{ .b, .q, .h, .hd }) else self.q.transpose(.{ .q, .h, .hd }); + const k = if (batched) self.k.transpose(.{ .b, .k, .h, .hd }) else self.k.transpose(.{ .k, .h, .hd }); + const v = if (batched) self.v.transpose(.{ .b, .k, .h, .hd }) else self.v.transpose(.{ .k, .h, .hd }); + + const splits = kernel.decodeSplits(q.shape(), k.shape(), decodeResources(zml.Compiler.current().platform)); + + const cfg: kernel.Config = .{ + .batch = if (batched) q.dim(.b) else 1, + .query_length = q.dim(.q), + .key_length = k.dim(.k), + .query_heads = q.dim(.h), + .kv_heads = k.dim(.h), + .head_dim = q.dim(.hd), + .dtype = zml.kernel.cute.from(q.dtype()), + .splits = splits, + }; + + const out = if (cfg.splits > 1) b: { + const partials = kernel.Kernel.call(.{ .q = q, .k = k, .v = v, .positions = self.positions }, .{ + .out = zml.Shape.init(.{ cfg.batch, cfg.splits, cfg.query_length, cfg.query_heads, cfg.head_dim + 2 }, .f32), + }, .{ + .cfg = cfg, + .grid = .{ @intCast(@divTrunc(@divExact(cfg.query_heads, cfg.kv_heads) + 3, 4)), @intCast(cfg.kv_heads * cfg.batch), @intCast(cfg.splits) }, + .block = kernel.block, + }).out; + break :b kernel.Merge.call(.{ .partials = partials }, .{ .out = q.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(cfg.query_heads), @intCast(cfg.batch), 1 }, + .block = kernel.block, + }).out; + } else kernel.Kernel.call(.{ .q = q, .k = k, .v = v, .positions = self.positions }, .{ .out = q.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(@divTrunc(cfg.query_length * @divExact(cfg.query_heads, cfg.kv_heads) + 3, 4)), @intCast(cfg.kv_heads * cfg.batch), 1 }, + .block = kernel.block, + }).out; + + return out.transpose(output_shape); + } +}; + +/// Returns the minimum number of multiprocessors and shared memory per block across all devices in the platform. +/// If there are no devices, returns 0 for both values. +/// If any device does not have the required attributes, returns 0 for both values. +fn decodeResources(platform: *const zml.Platform) kernel.DecodeResources { + var result: kernel.DecodeResources = .{ .multiprocessors = std.math.maxInt(i64), .shared_bytes = std.math.maxInt(i64) }; + if (platform.devices.len == 0) return .{ .multiprocessors = 0, .shared_bytes = 0 }; + for (platform.devices) |device| { + const sms = device.pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return .{ .multiprocessors = 0, .shared_bytes = 0 }; + const shared = device.pjrt_desc.attribute(platform.pjrt_api, "shared_memory_per_block_optin") orelse return .{ .multiprocessors = 0, .shared_bytes = 0 }; + result.multiprocessors = @min(result.multiprocessors, sms.int64); + result.shared_bytes = @min(result.shared_bytes, shared.int64); + } + return result; +} diff --git a/zml/kernels/flash_attention/flash_attention.zig b/zml/kernels/flash_attention/flash_attention.zig new file mode 100644 index 00000000..c1a27a36 --- /dev/null +++ b/zml/kernels/flash_attention/flash_attention.zig @@ -0,0 +1,364 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +pub const cute = @import("cute/cute.zig"); +pub const metal = @import("metal/metal.zig"); +pub const nki = @import("nki/nki.zig"); +const zml = @import("../../zml.zig"); + +test { + std.testing.refAllDecls(@This()); +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + return switch (platform.target) { + .neuron => true, + .metal => true, + .cuda => if (cute.isAvailable(platform)) true else false, + .cpu, .rocm, .tpu, .oneapi, .furiosa => false, + }; +} + +pub const Backend = enum { + cute, + nki, + metal, + + pub fn auto(platform: *const zml.Platform) Backend { + return switch (platform.target) { + .neuron => .nki, + .metal => .metal, + .cuda => .cute, + .cpu, .rocm, .tpu, .oneapi, .furiosa => stdx.debug.panic("Flash attention is not available on the {} platform", .{platform.target}), + }; + } + + pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { + return switch (backend) { + .cute => cute.isAvailable(platform), + .nki => platform.target == .neuron, + .metal => platform.target == .metal, + }; + } +}; + +pub const Parameters = union(Backend) { + cute: void, + nki: nki.Parameters, + metal: void, + + pub const InitOptions = union(Backend) { + cute: void, + nki: nki.Parameters, + metal: void, + + pub fn fromBackend(backend: Backend) InitOptions { + return switch (backend) { + .cute => .{ .cute = {} }, + .nki => .{ .nki = .init() }, + .metal => .{ .metal = {} }, + }; + } + }; + + pub fn init(opts: InitOptions) Parameters { + return switch (opts) { + .cute => .{ .cute = {} }, + .nki => |v| .{ .nki = v }, + .metal => .{ .metal = {} }, + }; + } +}; + +pub const Metadata = union(Backend) { + cute: void, + nki: void, + metal: metal.Metadata, + + pub const InitOptions = union(Backend) { + cute: void, + nki: void, + metal: void, + + pub fn fromBackend(backend: Backend) InitOptions { + return switch (backend) { + .cute => .{ .cute = {} }, + .nki => .{ .nki = {} }, + .metal => .{ .metal = {} }, + }; + } + }; + + pub fn init(opts: InitOptions) Metadata { + return switch (opts) { + .cute => .{ .cute = {} }, + .nki => .{ .nki = {} }, + .metal => .{ .metal = .init() }, + }; + } + + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { + return switch (self) { + .cute => .{ .cute = {} }, + .nki => .{ .nki = {} }, + inline else => |v, tag| @unionInit(zml.Bufferized(Metadata), @tagName(tag), try v.initBuffer(io, platform)), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { + switch (self.*) { + .cute => {}, + .nki => {}, + .metal => |*v| metal.Metadata.deinitBuffer(v), + } + } +}; + +/// Causal attention as used in transformers. +/// +/// **Shapes**: +/// - q, result: .{ .q, .h, .hd } +/// - k, v: .{ .k, .h, .hd } +/// +/// Where: +/// - .h is the number of head +/// - .q is the number of queries +/// - .k is the number of keys +/// - .hd is the head dimension +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { + return switch (parameters) { + .cute => cute.attention(q, k, v, token_index), + .nki => nki.attention(q, k, v, token_index, parameters.nki), + .metal => metal.attention(q, k, v, token_index, metadata.metal), + }; +} + +test "attention: q=1,qh=64,kh=8" { + try testAttention( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{63}, + ); + + try testAttention( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{36}, + ); +} + +test "attention: b=4,q=1,qh=64,kh=8" { + // Full attention + try testAttention( + .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{ 63, 63, 63, 63 }, + ); + + // Partial attention + try testAttention( + .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{ 61, 57, 23, 63 }, + ); +} + +test "attention: q=1,qh=8,kh=8" { + try testAttention( + .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{62}, + ); + try testAttention( + .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{63}, + ); +} + +test "attention: q=8,qh=64,kh=8" { + try testAttention( + .init(.{ .q = 8, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{56}, + ); +} + +pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32) !void { + return testAttentionBackends(q_shape, k_shape, token_index_h, std.enums.values(Backend)); +} + +fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, backends: []const Backend) !void { + const io = std.testing.io; + const allocator = std.testing.allocator; + + const platform = zml.testing.env(); + + var arena_state: std.heap.ArenaAllocator = .init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + const token_index_shape: zml.Shape = if (token_index_h.len > 1) b: { + std.debug.assert(token_index_h.len == q_shape.dim(.b)); + break :b .init(.{ .b = q_shape.dim(.b) }, .u32); + } else .init(.{}, .u32); + + const max_k_idx: i64 = k_shape.dim(.k) - 1; + const max_q_idx: i64 = q_shape.dim(.q) - 1; + for (token_index_h) |index| { + // Check for out of bound reads + std.debug.assert(index + max_q_idx <= max_k_idx); + } + + const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor } = .{ + .q = .fromShape(q_shape), + .k = .fromShape(k_shape), + .v = .fromShape(k_shape), + .token_index = .fromShape(token_index_shape), + }; + + var q = try attentionTestInput(q_shape, 17); + defer q.deinit(); + var k = try attentionTestInput(k_shape, 29); + defer k.deinit(); + var v = try attentionTestInput(k_shape, 43); + defer v.deinit(); + var token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, @ptrCast(token_index_h)); + defer token_index.deinit(); + + const meshes = platform.meshes.values(); + const reference_metadata: zml.attention.Metadata = .{ .stablehlo = {} }; + const reference_parameters: zml.attention.Parameters = .{ .stablehlo = {} }; + const reference_exe = try platform.compileFn(allocator, io, zml.attention.attention, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, reference_metadata, reference_parameters }, .{ + .program_name = "attention_stablehlo", + .meshes = meshes, + }); + defer reference_exe.deinit(); + + var reference_d = try reference_exe.eval(allocator, io, .{ q, k, v, token_index, reference_metadata }); + defer reference_d.deinit(); + try reference_d.await(io); + const reference_h: zml.Slice = try reference_d.toSliceAlloc(allocator, io); + defer reference_h.free(allocator); + + for (backends) |backend| { + if (!backend.isAvailable(platform)) continue; + + const metadata: Metadata = .init(.fromBackend(backend)); + const parameters: Parameters = .init(.fromBackend(backend)); + const exe = try platform.compileFn( + allocator, + io, + attention, + .{ tensors.q, tensors.k, tensors.v, tensors.token_index, metadata, parameters }, + .{ + .program_name = try std.fmt.allocPrint(arena, "attention_{t}", .{backend}), + .meshes = meshes, + }, + ); + defer exe.deinit(); + + var metadata_d = try metadata.initBuffer(io, platform); + defer Metadata.deinitBuffer(&metadata_d); + + var output_d = try exe.eval(allocator, io, .{ q, k, v, token_index, metadata_d }); + defer output_d.deinit(); + try output_d.await(io); + const output_h = try output_d.toSliceAlloc(allocator, io); + defer output_h.free(allocator); + + errdefer std.log.err( + \\ Attention test failed, {0t} output doesn't match reference. + \\ - reference: {1d} + \\ - {0t}: {2d} + , .{ backend, reference_h, output_h }); + try zml.testing.expectClose(io, reference_h, output_h, .{ + .absolute_tolerance = 5e-3, + .relative_tolerance = 1e-2, + .epsilon_relative = 1e-3, + .minimum_close_fraction = 0.99, + }); + } +} + +fn attentionTestInput(shape: zml.Shape, seed: u64) !zml.Buffer { + const host = try zml.Slice.alloc(std.testing.allocator, shape); + defer host.free(std.testing.allocator); + var prng: std.Random.DefaultPrng = .init(seed); + const random = prng.random(); + switch (shape.dtype()) { + inline .bf16, .f16, .f32 => |dtype| { + for (host.items(dtype.toZigType())) |*value| { + const x = random.float(f32) * 4 - 2; + value.* = if (dtype == .bf16) .fromF32(x) else @floatCast(x); + } + }, + else => unreachable, + } + return zml.Buffer.fromSlice(std.testing.io, zml.testing.env(), host); +} + +/// Independent f64 oracle: the SDPA reference uses fast dot precision, so its +/// f32 result alone cannot establish the accuracy of the kernel's f32 arithmetic. +fn checkAttentionF32(q: zml.Slice, k: zml.Slice, v: zml.Slice, positions: []const u32, output: zml.Slice) !void { + const Layout = struct { + b: usize, + s: usize, + h: usize, + d: usize, + + fn stride(shape: zml.Shape, tag: anytype) usize { + const axis = shape.hasTag(tag) orelse return 0; + var result: usize = 1; + for (shape.dims()[axis + 1 ..]) |dim| result *= @intCast(dim); + return result; + } + + fn init(shape: zml.Shape) @This() { + return .{ .b = stride(shape, .b), .s = if (shape.hasTag(.q) != null) stride(shape, .q) else stride(shape, .k), .h = stride(shape, .h), .d = stride(shape, .hd) }; + } + + fn offset(self: @This(), b: usize, s: usize, h: usize, d: usize) usize { + return b * self.b + s * self.s + h * self.h + d * self.d; + } + }; + const ql = Layout.init(q.shape); + const kl = Layout.init(k.shape); + const vl = Layout.init(v.shape); + const batch: usize = if (q.shape.hasTag(.b) != null) @intCast(q.shape.dim(.b)) else 1; + const queries: usize = @intCast(q.shape.dim(.q)); + const heads: usize = @intCast(q.shape.dim(.h)); + const dim: usize = @intCast(q.shape.dim(.hd)); + const group: usize = @intCast(@divExact(q.shape.dim(.h), k.shape.dim(.h))); + const scores = try std.testing.allocator.alloc(f64, @intCast(k.shape.dim(.k))); + defer std.testing.allocator.free(scores); + const scale = 1.0 / @sqrt(@as(f64, @floatFromInt(dim))); + for (0..batch) |b| { + const start: usize = positions[if (positions.len == 1) 0 else b]; + for (0..queries) |i| { + for (0..heads) |h| { + var maximum: f64 = -std.math.inf(f64); + const active = scores[0 .. start + i + 1]; + for (active, 0..) |*score, j| { + var dot: f64 = 0; + for (0..dim) |d| dot += @as(f64, q.items(f32)[ql.offset(b, i, h, d)]) * @as(f64, k.items(f32)[kl.offset(b, j, h / group, d)]); + score.* = dot * scale; + maximum = @max(maximum, score.*); + } + var sum: f64 = 0; + for (active) |*score| { + score.* = @exp(score.* - maximum); + sum += score.*; + } + for (0..dim) |d| { + var expected: f64 = 0; + for (active, 0..) |weight, j| expected += weight * @as(f64, v.items(f32)[vl.offset(b, j, h / group, d)]); + expected /= sum; + const actual = output.items(f32)[ql.offset(b, i, h, d)]; + try std.testing.expectApproxEqAbs(expected, @as(f64, actual), 2e-5 + 2e-5 * @abs(expected)); + } + } + } + } +} diff --git a/zml/kernels/flash_attention/metal/BUILD.bazel b/zml/kernels/flash_attention/metal/BUILD.bazel new file mode 100644 index 00000000..855bdcca --- /dev/null +++ b/zml/kernels/flash_attention/metal/BUILD.bazel @@ -0,0 +1,6 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "metal", + srcs = ["metal.zig"], +) diff --git a/zml/kernels/flash_attention/metal/metal.zig b/zml/kernels/flash_attention/metal/metal.zig new file mode 100644 index 00000000..9327c2a1 --- /dev/null +++ b/zml/kernels/flash_attention/metal/metal.zig @@ -0,0 +1,38 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); + +const log = std.log.scoped(.@"zml/attention/metal"); + +pub const Metadata = struct { + num_tokens: zml.Tensor, + + pub fn init() Metadata { + return .{ .num_tokens = .fromShape(.scalar(.u32)) }; + } + + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { + _ = self; + return .{ .num_tokens = try zml.Buffer.scalar(io, platform, 0, .u32) }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { + self.num_tokens.deinit(); + } +}; + +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata) zml.Tensor { + const qc = q.transpose(.{ .h, .q, .hd }); + const kc = k.transpose(.{ .h, .k, .hd }); + const vc = v.transpose(.{ .h, .k, .hd }); + const tok_i32 = token_index.convert(.i32); + + const attn = if (q.dim(.q) > 1) + zml.ops.customCall("zml$flash_attn", .{ qc, kc, vc, tok_i32, metadata.num_tokens }, qc.shape(), .{}, .{ .has_side_effect = false }) + else + zml.ops.customCall("zml$flash_attn", .{ qc, kc, vc, tok_i32 }, qc.shape(), .{}, .{ .has_side_effect = false }); + + return attn.transpose(q.shape()); +} diff --git a/zml/attention/nki/BUILD.bazel b/zml/kernels/flash_attention/nki/BUILD.bazel similarity index 92% rename from zml/attention/nki/BUILD.bazel rename to zml/kernels/flash_attention/nki/BUILD.bazel index a8b165de..f8e4f470 100644 --- a/zml/attention/nki/BUILD.bazel +++ b/zml/kernels/flash_attention/nki/BUILD.bazel @@ -8,8 +8,8 @@ filegroup( ) filegroup( - name = "attention", - srcs = ["attention.zig"], + name = "nki", + srcs = ["nki.zig"], ) py_binary( diff --git a/zml/attention/nki/attention.py b/zml/kernels/flash_attention/nki/attention.py similarity index 100% rename from zml/attention/nki/attention.py rename to zml/kernels/flash_attention/nki/attention.py diff --git a/zml/attention/nki/attention_harness.py b/zml/kernels/flash_attention/nki/attention_harness.py similarity index 100% rename from zml/attention/nki/attention_harness.py rename to zml/kernels/flash_attention/nki/attention_harness.py diff --git a/zml/attention/nki/attention.zig b/zml/kernels/flash_attention/nki/nki.zig similarity index 99% rename from zml/attention/nki/attention.zig rename to zml/kernels/flash_attention/nki/nki.zig index cdbad308..72f0a66d 100644 --- a/zml/attention/nki/attention.zig +++ b/zml/kernels/flash_attention/nki/nki.zig @@ -1,6 +1,6 @@ const platforms = @import("platforms"); -const zml = @import("../../zml.zig"); +const zml = @import("../../../zml.zig"); const stdx = zml.stdx; const KernelSpec = struct { diff --git a/zml/kernels/paged_attention/metal/BUILD.bazel b/zml/kernels/paged_attention/metal/BUILD.bazel new file mode 100644 index 00000000..855bdcca --- /dev/null +++ b/zml/kernels/paged_attention/metal/BUILD.bazel @@ -0,0 +1,6 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "metal", + srcs = ["metal.zig"], +) diff --git a/zml/kernels/paged_attention/metal/metal.zig b/zml/kernels/paged_attention/metal/metal.zig new file mode 100644 index 00000000..b337638a --- /dev/null +++ b/zml/kernels/paged_attention/metal/metal.zig @@ -0,0 +1,81 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); + +const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; + +const log = std.log.scoped(.@"zml/attention/metal"); + +pub const Options = struct { + batch_size: usize, + max_num_pages: usize, + max_seqlen_q: usize, + is_prefill: bool, + + pub fn isPrefill(self: Options) bool { + return self.is_prefill; + } + + pub fn maxNumPages(self: Options) usize { + return self.max_num_pages; + } +}; + +pub const Parameters = struct { + block_table: zml.Tensor, + seq_lens: zml.Tensor, + query_start_len: zml.Tensor, + options_: Options, + + pub fn init(options_: Options) Parameters { + return .{ + .block_table = .init(.{ .b = options_.batch_size, .p = options_.max_num_pages }, .i32), + .seq_lens = .init(.{ .b = options_.batch_size }, .i32), + .query_start_len = .init(.{ .b = options_.batch_size + 1 }, .i32), + .options_ = options_, + }; + } + + pub fn allocationSize(self: Parameters) usize { + var allocation_size: usize = 0; + + allocation_size += self.block_table.byteSize(); + allocation_size += self.seq_lens.byteSize(); + allocation_size += self.query_start_len.byteSize(); + + return allocation_size; + } + + pub fn options(self: Parameters) Options { + return self.options_; + } +}; + +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { + if (opts.mask == .indices) @panic("Sparse Attention is not implemented on Metal"); + + const num_kv_heads = q.dim(.hkv); + + const qh = q.merge(.{ .h = .{ .hkv, .hg } }).transpose(.{ .b, .h, .hd }); + const kc = k_cache.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const vc = v_cache.transpose(.{ .page, .k_chunk, .hkv, .hd }); + + const sliding_window: i32 = if (opts.mask.slidingWindow()) |window| @intCast(window) else -1; + + const out = zml.ops.customCall( + "zml$paged_attn", + .{ qh, kc, vc, parameters.block_table, parameters.seq_lens, parameters.query_start_len }, + qh.shape(), + .{ + .is_causal = opts.mask.isCausal(), + .sliding_window = sliding_window, + .scale = opts.scale, + }, + .{ .has_side_effect = false }, + ); + + // [total_q_tokens, num_heads, head_dim] -> {.b, .hkv, .hg, .hd} (q's order) + return out.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = .auto }).transpose(q.shape()); +} diff --git a/zml/kernels/paged_attention/mosaic_tpu/BUILD.bazel b/zml/kernels/paged_attention/mosaic_tpu/BUILD.bazel new file mode 100644 index 00000000..6a0e4e87 --- /dev/null +++ b/zml/kernels/paged_attention/mosaic_tpu/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "mosaic_tpu", + srcs = [ + "mosaic_tpu.zig", + "ragged_attention.zig", + ], +) diff --git a/zml/attention/tpu_attention.zig b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig similarity index 97% rename from zml/attention/tpu_attention.zig rename to zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig index f6e26f5a..f1cc85f4 100644 --- a/zml/attention/tpu_attention.zig +++ b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig @@ -4,10 +4,9 @@ const mlir = @import("mlir"); const ragged_paged = @import("platforms/tpu/ragged_paged"); const stdx = @import("stdx"); -const Compiler = @import("../Compiler.zig"); -const zml = @import("../zml.zig"); -const AttentionOptions = @import("paged_attention.zig").AttentionOptions; -const ragged_attention = @import("mosaic_tpu_kernels/ragged_attention.zig"); +const zml = @import("../../../zml.zig"); +const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; +const ragged_attention = @import("ragged_attention.zig"); const log = std.log.scoped(.tpu_attention); @@ -89,7 +88,7 @@ pub const mosaic_tpu = struct { num_seqs: zml.Tensor, cfg: ragged_paged.Cfg, ) zml.Tensor { - const mlir_ctx = Compiler.current().mlir_ctx; + const mlir_ctx = zml.Compiler.current().mlir_ctx; const out = ragged_attention.Kernel.call( .{ .kv_lens = seq_lens, diff --git a/zml/attention/mosaic_tpu_kernels/ragged_attention.zig b/zml/kernels/paged_attention/mosaic_tpu/ragged_attention.zig similarity index 89% rename from zml/attention/mosaic_tpu_kernels/ragged_attention.zig rename to zml/kernels/paged_attention/mosaic_tpu/ragged_attention.zig index a402a7e5..30536306 100644 --- a/zml/attention/mosaic_tpu_kernels/ragged_attention.zig +++ b/zml/kernels/paged_attention/mosaic_tpu/ragged_attention.zig @@ -1,6 +1,6 @@ const ragged_paged = @import("platforms/tpu/ragged_paged"); -const zml = @import("../../zml.zig"); +const zml = @import("../../../zml.zig"); test { _ = Kernel; diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig new file mode 100644 index 00000000..c3a747f1 --- /dev/null +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -0,0 +1,502 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../zml.zig"); +const metal = @import("metal/metal.zig"); +const tpu = @import("mosaic_tpu/mosaic_tpu.zig"); +const triton = @import("triton/triton.zig"); +const stablehlo = @import("stablehlo/stablehlo.zig"); + +const PagedAttention = @This(); + +test { + std.testing.refAllDecls(Backend); + std.testing.refAllDecls(Options); + std.testing.refAllDecls(Parameters); + std.testing.refAllDecls(KvCache); +} + +pub const Backend = enum { + triton, + mosaic_tpu, + metal, + stablehlo, + + pub fn auto(platform: *const zml.Platform) Backend { + return switch (platform.target) { + .cuda => .triton, + .rocm => .triton, + .oneapi => .triton, + .tpu => .mosaic_tpu, + .metal => .metal, + .cpu, .furiosa => .stablehlo, + .neuron => stdx.debug.panic("Paged attention is not supported on {s} yet", .{@tagName(platform.target)}), + }; + } + + pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { + return switch (backend) { + .stablehlo => true, + .triton => platform.target != .cpu, + .metal => platform.target == .metal, + .mosaic_tpu => platform.target == .tpu, + }; + } +}; + +pub const Options = union(Backend) { + triton: triton.paged.Options, + mosaic_tpu: tpu.mosaic_tpu.Options, + metal: metal.Options, + stablehlo: triton.paged.Options, + + const Args = struct { + backend: Backend, + is_prefill: bool, + batch_size: u32, + batch_size_prefill: ?u32 = null, + batch_size_decode: ?u32 = null, + seq_len: u32, + max_num_pages: u32, + max_token_count: u32, + num_heads: u32, + num_kv_heads: u32, + head_dim: u32, + max_seqlen_q: u32, + }; + + pub fn fromBackend(args: Args) Options { + return switch (args.backend) { + .mosaic_tpu => .{ + .mosaic_tpu = .{ + .is_prefill = args.is_prefill, + .batch_size = args.batch_size, + .max_num_pages = args.max_num_pages, + .max_seqlen_k = args.seq_len, + .max_token_count = args.max_token_count, + .num_heads = args.num_heads, + .num_kv_heads = args.num_kv_heads, + .head_dim = args.head_dim, + }, + }, + inline .triton, .metal, .stablehlo => |t| @unionInit(Options, @tagName(t), .{ + .batch_size = args.batch_size, + .max_num_pages = args.max_num_pages, + .max_seqlen_q = args.max_seqlen_q, + .is_prefill = args.is_prefill, + }), + }; + } + + pub fn isPrefill(self: Options) bool { + return switch (self) { + inline else => |v| v.isPrefill(), + }; + } + + pub fn maxNumPages(self: Options) usize { + return switch (self) { + inline else => |v| v.maxNumPages(), + }; + } +}; + +pub const Parameters = union(Backend) { + triton: triton.paged.Parameters, + mosaic_tpu: tpu.mosaic_tpu.Parameters, + metal: metal.Parameters, + stablehlo: triton.paged.Parameters, + + pub fn init(options_: Options) Parameters { + return switch (options_) { + inline else => |opts, backend| @unionInit(Parameters, @tagName(backend), .init(opts)), + }; + } + + pub fn options(self: Parameters) Options { + return switch (self) { + inline else => |v, backend| @unionInit(Options, @tagName(backend), v.options()), + }; + } + + pub fn allocationSize(self: Parameters) usize { + return switch (self) { + inline else => |v| v.allocationSize(), + }; + } + + pub fn onMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { + zml.Tensor.onMemoryAll(&self, memory); + return self; + } + + pub fn toMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { + return zml.Tensor.toMemoryAll(self, memory); + } +}; + +pub const AttentionMask = union(enum) { + standard: struct { + /// Restrict attention to keys at or before the query position. + causal: bool = true, + /// Optional left window size, including the current token. + /// Non-causal attention also includes every later valid key. + sliding_window: ?u32 = null, + }, + /// Attend to keys at specific positions. + indices: zml.Tensor, + + /// Attend to every valid key in the sequence. + pub const none: AttentionMask = .{ .standard = .{ .causal = false } }; + + pub fn isCausal(mask: AttentionMask) bool { + return switch (mask) { + .standard => |options| options.causal, + .indices => false, + }; + } + + pub fn slidingWindow(mask: AttentionMask) ?u32 { + return switch (mask) { + .standard => |options| options.sliding_window, + .indices => null, + }; + } +}; + +pub const AttentionOptions = struct { + mask: AttentionMask = .{ .standard = .{} }, + scale: ?f32 = null, + sink: ?zml.Tensor = null, +}; + +pub const KvCache = union(enum) { + split: struct { + k: zml.Tensor, + v: zml.Tensor, + }, + // TODO: rename `dense` to `fused`. + dense: zml.Tensor, + latent: zml.Tensor, + + pub fn update( + self: KvCache, + new_k: zml.Tensor, + new_v: zml.Tensor, + slot_mapping: zml.Tensor, + chunk_size: u32, + backend: Backend, + ) KvCache { + const page_index, const offset = getPageAndOffsetFromSlotMapping(slot_mapping, chunk_size); + + const kv: KvCache = switch (self) { + .split => |split| switch (backend) { + .triton, .mosaic_tpu, .metal, .stablehlo => .{ + .split = .{ + .k = split.k.scatterSlices( + .{ .page = page_index, .k_chunk = offset }, + new_k, + .{ .update_fn = zml.Tensor.ScatterOpts.override, .indices_are_unique = false, .indices_are_sorted = false }, + ).reuseBuffer(self.split.k), + .v = split.v.scatterSlices( + .{ .page = page_index, .k_chunk = offset }, + new_v, + .{ .update_fn = zml.Tensor.ScatterOpts.override, .indices_are_unique = false, .indices_are_sorted = false }, + ).reuseBuffer(self.split.v), + }, + }, + }, + .dense => @panic("TODO"), + .latent => |latent_kv| .{ + .latent = latent_kv.scatterSlices( + .{ .page = page_index, .k_chunk = offset }, + new_k, + .{ .update_fn = zml.Tensor.ScatterOpts.override, .indices_are_unique = false, .indices_are_sorted = false }, + ).reuseBuffer(self.latent), + }, + }; + + return kv; + } + + pub fn getPageAndOffsetFromSlotMapping(slot_mapping: zml.Tensor, page_size: u32) [2]zml.Tensor { + const page_index = slot_mapping.divByConst(page_size); + const offset = slot_mapping.remainder(.scalar(page_size, slot_mapping.dtype())); + return .{ page_index, offset }; + } + + pub fn getPage( + self: KvCache, + page_index: zml.Tensor, + ) [2]zml.Tensor { + return switch (self) { + .split => |split| .{ + split.k.slice(split.k.axis(.page), .dynSingle(page_index)), + split.v.slice(split.k.axis(.page), .dynSingle(page_index)), + }, + .dense, .latent => @panic("TODO"), + }; + } +}; + +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache, opts: AttentionOptions) zml.Tensor { + _ = k; + _ = v; + return switch (parameters) { + .triton => |triton_parameters| switch (kv_cache) { + .split => |split| triton.paged.pagedAttention(triton_parameters, q, split.k, split.v, opts), + .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + .latent => std.debug.panic("latent KV pages are only supported with Multi-Latent Attention", .{}), + }, + .mosaic_tpu => |mosaic_tpu_parameters| switch (kv_cache) { + .split => std.debug.panic("split KV pages is not supported with the mosaic_tpu backend", .{}), + .latent => std.debug.panic("latent KV pages are only supported with Multi-Latent Attention", .{}), + .dense => tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, q, kv_cache.dense, opts), + }, + .metal => |metal_parameters| switch (kv_cache) { + .split => |split| metal.pagedAttention(metal_parameters, q, split.k, split.v, opts), + .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + .latent => std.debug.panic("latent KV pages are only supported with Multi-Latent Attention", .{}), + }, + .stablehlo => |params| switch (kv_cache) { + .latent => std.debug.panic("latent KV pages are only supported with Multi-Latent Attention", .{}), + else => stablehlo.pagedAttention(params, q, kv_cache, opts), + }, + }; +} + +test "Backend.auto selects mosaic_tpu on TPU" { + const platform: zml.Platform = .{ + .arena = undefined, + .target = .tpu, + .pjrt_api = undefined, + .pjrt_client = undefined, + .state = zml.platform.State.init(.tpu), + .devices = &.{}, + .memories = &.{}, + .physical_mesh = undefined, + .replicated_mesh = undefined, + .meshes = .empty, + .io_impl = .threaded, + }; + + try std.testing.expectEqual(Backend.mosaic_tpu, Backend.auto(&platform)); +} + +test "Backend.auto selects triton on oneAPI" { + const platform: zml.Platform = .{ + .arena = undefined, + .target = .oneapi, + .pjrt_api = undefined, + .pjrt_client = undefined, + .state = zml.platform.State.init(.oneapi), + .devices = &.{}, + .memories = &.{}, + .physical_mesh = undefined, + .replicated_mesh = undefined, + .meshes = .empty, + .io_impl = .threaded, + }; + + try std.testing.expectEqual(Backend.triton, Backend.auto(&platform)); +} + +test pagedAttention { + const platform = zml.testing.env(); + const io = std.testing.io; + const allocator = std.testing.allocator; + var arena_state: std.heap.ArenaAllocator = .init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + const num_pages = 64; + const page_size = 16; + const max_num_pages = 16; + const prefill_token_count = 32; + const num_prefill = 1; + const num_decode = 6; + const active_batch_size = num_prefill + num_decode; + const batch_size = active_batch_size + 1; + const query_token_count = prefill_token_count + num_decode; + const dt: zml.DataType = .bf16; + const model_meshe = platform.meshes.get("model").?; + const partition = .{ .hkv = .model }; + const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache } = .{ + .q = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hg = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), + .k = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), + .v = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), + .kv_cache = .{ + .split = .{ + .k = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), + .v = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), + }, + }, + }; + + const meshes: []const *const zml.Mesh = &.{ platform.replicated_mesh, platform.meshes.get("model").? }; + const rng_q = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.q.shape(), .{} }, .{ .meshes = meshes }); + defer rng_q.deinit(); + const rng_k = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.k.shape(), .{} }, .{ .meshes = meshes }); + defer rng_k.deinit(); + const rng_kv_cache = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.kv_cache.split.k.shape(), .{} }, .{ .meshes = meshes }); + defer rng_kv_cache.deinit(); + + const triton_options_args: Options.Args = .{ + .backend = .triton, + .is_prefill = true, + .batch_size = batch_size, + .batch_size_prefill = num_prefill, + .batch_size_decode = num_decode, + .seq_len = max_num_pages * page_size, + .max_num_pages = max_num_pages, + .max_token_count = 16 * page_size, + .num_heads = @intCast(tensors.q.dim(.hg) * tensors.q.dim(.hkv)), + .num_kv_heads = @intCast(tensors.q.dim(.hkv)), + .head_dim = @intCast(tensors.q.dim(.hd)), + .max_seqlen_q = 16 * 2, + }; + const triton_parameters: Parameters = .init(.fromBackend(triton_options_args)); + var q = try rng_q.eval(allocator, io, {}); + defer q.deinit(); + var new_k = try rng_k.eval(allocator, io, {}); + defer new_k.deinit(); + var new_v = try rng_k.eval(allocator, io, {}); + defer new_v.deinit(); + + var kv_cache_d: zml.Bufferized(KvCache) = .{ .split = undefined }; + kv_cache_d.split.k = try rng_kv_cache.eval(allocator, io, .{}); + defer kv_cache_d.split.k.deinit(); + kv_cache_d.split.v = try rng_kv_cache.eval(allocator, io, .{}); + defer kv_cache_d.split.v.deinit(); + + const block_table: [batch_size][max_num_pages]i32 = .{ + // prefilling pages 9-10 + .{ 0, 1, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, + // generation + .{ 0, 1, 2, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, + .{ 0, 1, 2, 3, 4, 6, 7, 8, -1, -1, -1, -1, -1, -1, -1, -1 }, + .{ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63 }, + .{ 0, 1, 2, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, + .{ 0, 1, 2, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, + .{ 0, 1, 2, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, + .{ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, + }; + + const seq_lens: [batch_size]i32 = .{ + 4 * page_size, + 4 * page_size + 3, + 7 * page_size + 11, + 15 * page_size + 9, + 4 * page_size + 3, + 4 * page_size + 3, + 4 * page_size + 3, + 0, + }; + + const query_start_len: [batch_size + 1]i32 = .{ 0, 32, 33, 34, 35, 36, 37, 38, 38 }; + var triton_parameters_d: zml.Bufferized(Parameters) = .{ .triton = .{ + .block_table = try .fromBytes(io, platform, triton_parameters.triton.block_table.shape(), @ptrCast(&block_table)), + .seq_lens = try .fromBytes(io, platform, triton_parameters.triton.seq_lens.shape(), @ptrCast(&seq_lens)), + .query_start_len = try .fromBytes(io, platform, triton_parameters.triton.query_start_len.shape(), @ptrCast(&query_start_len)), + } }; + defer zml.Buffer.deinitAll(Parameters, &triton_parameters_d); + + const all_backends = std.enums.values(Backend); + // stablehlo ignores these options, mosaic_tpu requires a dense + // KV cache instead of the split cache used by this test. + const option_sensitive_backends = [_]Backend{ .triton, .metal }; + const test_cases = [_]struct { + name: []const u8, + attention_options: AttentionOptions, + backends: []const Backend, + }{ + .{ + .name = "unbounded", + .attention_options = .{ .mask = .{ .standard = .{} } }, + .backends = all_backends, + }, + .{ + .name = "non_causal", + .attention_options = .{ .mask = .{ .standard = .{ .causal = false } } }, + .backends = &option_sensitive_backends, + }, + .{ + .name = "sliding_window", + .attention_options = .{ .mask = .{ .standard = .{ .sliding_window = page_size } } }, + .backends = &option_sensitive_backends, + }, + }; + + for (test_cases) |test_case| { + var results_per_backend: std.enums.EnumArray(Backend, ?zml.Slice) = .initFill(null); + defer { + for (std.enums.values(Backend)) |backend| { + if (results_per_backend.get(backend)) |slice| slice.free(allocator); + } + } + + for (test_case.backends) |backend| { + if (!backend.isAvailable(platform)) { + std.log.warn("paged_attention backend {t} not available", .{backend}); + continue; + } + std.log.warn("Testing paged_attention {s} with backend {t}", .{ test_case.name, backend }); + + var backend_options_args = triton_options_args; + backend_options_args.backend = backend; + const parameters: Parameters = .init(.fromBackend(backend_options_args)); + + const exe = try platform.compileFn( + allocator, + io, + pagedAttention, + .{ parameters, tensors.q, tensors.k, tensors.v, tensors.kv_cache, test_case.attention_options }, + .{ .program_name = try std.fmt.allocPrint(arena, "paged_attention_{s}_{t}", .{ test_case.name, backend }), .meshes = meshes }, + ); + defer exe.deinit(); + + const parameters_d: zml.Bufferized(Parameters) = switch (parameters) { + .triton => triton_parameters_d, + inline .metal, .mosaic_tpu, .stablehlo => |_, t| @unionInit(zml.Bufferized(Parameters), @tagName(t), .{ + .block_table = triton_parameters_d.triton.block_table, + .seq_lens = triton_parameters_d.triton.seq_lens, + .query_start_len = triton_parameters_d.triton.query_start_len, + }), + }; + + var output_d = try exe.eval(allocator, io, .{ parameters_d, q, new_k, new_v, kv_cache_d, .{} }); + defer output_d.deinit(); + results_per_backend.set(backend, try output_d.toSliceAlloc(allocator, io)); + } + + const reference_backend: Backend = .triton; + const reference = results_per_backend.get(reference_backend) orelse return error.SkipZigTest; + + var num_failed: u32 = 0; + for (std.enums.values(Backend)) |backend| { + if (backend == reference_backend) continue; + const output_h = results_per_backend.get(backend) orelse continue; + + const tolerance: zml.testing.CompareOpts = .{ + .absolute_tolerance = 1e-2, + .relative_tolerance = 1e-2, + .epsilon_relative = 1e-6, + }; + + zml.testing.expectClose(io, reference, output_h, tolerance) catch |err| switch (err) { + error.TestUnexpectedResult => { + num_failed += 1; + std.log.err("test pagedAttention {s} failed on backend {t} against reference {t}", .{ + test_case.name, + backend, + reference_backend, + }); + }, + else => |e| return e, + }; + } + + if (num_failed > 0) return error.TestUnexpectedResult; + } +} diff --git a/zml/kernels/paged_attention/stablehlo/BUILD.bazel b/zml/kernels/paged_attention/stablehlo/BUILD.bazel new file mode 100644 index 00000000..f60464f4 --- /dev/null +++ b/zml/kernels/paged_attention/stablehlo/BUILD.bazel @@ -0,0 +1,6 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "stablehlo", + srcs = ["stablehlo.zig"], +) diff --git a/zml/kernels/paged_attention/stablehlo/stablehlo.zig b/zml/kernels/paged_attention/stablehlo/stablehlo.zig new file mode 100644 index 00000000..ab0ffee2 --- /dev/null +++ b/zml/kernels/paged_attention/stablehlo/stablehlo.zig @@ -0,0 +1,253 @@ +const zml = @import("../../../zml.zig"); +const KvCache = @import("../paged_attention.zig").KvCache; +const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; +const triton = @import("../triton/triton.zig"); + +pub fn pagedAttention( + parameters: triton.paged.Parameters, + q: zml.Tensor, + kv_cache: KvCache, + opts: AttentionOptions, +) zml.Tensor { + switch (opts.mask) { + .standard => |options| if (options.sliding_window != null) @panic("Sliding window attention mask is not supported on StableHLO"), + .indices => @panic("Sparse attention mask is not supported on StableHLO"), + } + + const page_size = kv_cache.split.k.dim(.k_chunk); + + const final_state = zml.ops.@"while"( + AttentionLoop, + .{ .q = q, .kv_cache = kv_cache, .parameters = parameters, .opts = opts }, + .{ + // i32 to match triton conventions + .slot_id = .scalar(0, .i32), + .seq_id = .scalar(0, .i32), + .q_page_idx = .scalar(0, .i32), + .k_page_idx = .scalar(0, .i32), + // Note: we do all partial softmax in f32 + .partial_softmax_prefill = .zeroes(q.shape().setDim(.b, page_size).withDtype(.f32)), + .partial_softmax_decode = .zeroes(q.shape().setDim(.b, 1).withDtype(.f32)), + .out = .zeroes(q.shape()), + }, + ); + + return final_state.out; +} + +const AttentionLoop = struct { + q: zml.Tensor, + kv_cache: KvCache, + parameters: triton.paged.Parameters, + opts: AttentionOptions, + + const While = @This(); + + pub const State = struct { + slot_id: zml.Tensor, + seq_id: zml.Tensor, + q_page_idx: zml.Tensor, + k_page_idx: zml.Tensor, + partial_softmax_prefill: PartialSoftmax, + partial_softmax_decode: PartialSoftmax, + out: zml.Tensor, + }; + + pub fn cond(self: While, state: State) zml.Tensor { + const has_more_sequences = state.seq_id.cmp(.LT, .scalar(self.parameters.seq_lens.count(), .i32)); + const query_start = self.parameters.query_start_len.slice(0, .dynSingle(state.seq_id)); + const query_end = self.parameters.query_start_len.slice(0, .dynSingle(state.seq_id.addConstant(1))); + const has_queries = query_start.cmp(.LT, query_end).asScalar(); + return has_more_sequences.logical(.AND, has_queries); + } + + pub fn body(self: While, state_: State) State { + const query_start = self.parameters.query_start_len.slice(0, .dynSingle(state_.seq_id)); + const query_end = self.parameters.query_start_len.slice(0, .dynSingle(state_.seq_id.addConstant(1))); + const seq_num_queries_: zml.Tensor = .asScalar(.sub(query_end, query_start)); + const page_size_: u32 = @intCast(self.kv_cache.split.k.dim(.k_chunk)); + + return zml.ops.@"if"( + struct { + const IfPrefill = @This(); + while_body: While, + state: State, + seq_num_queries: zml.Tensor, + page_size: u32, + + /// Prefill version: num_slots == page_size, we multiply a full page of queries with a page of keys. + pub fn onTrue(if_ctx: IfPrefill) State { + const while_body = if_ctx.while_body; + const state = if_ctx.state; + const page_size = if_ctx.page_size; + const num_slots: u32 = page_size; + + // seq_lens includes the current queries; subtracting their count gives the context length. + const seq_len = while_body.parameters.seq_lens.slice(0, .dynSingle(state.seq_id)); + const q_offset = seq_len.sub(if_ctx.seq_num_queries).add(state.q_page_idx.scale(page_size)); + const next_k_offset = state.k_page_idx.addConstant(1).scale(page_size); + + // The current K page is processed in this iteration. Continue if the next page contains + // any key visible to the current query chunk, whose exclusive end is q_offset + num_slots. + const query_has_more_keys: zml.Tensor = .cmp(next_k_offset, .LT, q_offset.addConstant(num_slots)); + const sequence_has_more_queries: zml.Tensor = .cmp(state.q_page_idx.addConstant(1).scale(num_slots), .LT, if_ctx.seq_num_queries); + + const partial_softmax = while_body.attentionOnePage(state, q_offset, num_slots, page_size); + return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax); + } + + /// Decode version: num_slots == 1, we multiply one query with a page of keys. + pub fn onFalse(if_ctx: IfPrefill) State { + const while_body = if_ctx.while_body; + const state = if_ctx.state; + const page_size = if_ctx.page_size; + const num_slots: u32 = 1; + + // A decode query is the final token in seq_len, so its zero-based offset is seq_len - 1. + const seq_len = while_body.parameters.seq_lens.slice(0, .dynSingle(state.seq_id)).asScalar(); + const q_offset = seq_len.sub(if_ctx.seq_num_queries); + const next_k_offset = state.k_page_idx.addConstant(1).scale(page_size); + + const query_has_more_keys: zml.Tensor = .cmp(next_k_offset, .LT, q_offset.addConstant(num_slots)); + const sequence_has_more_queries: zml.Tensor = .scalar(false, .bool); + + const partial_softmax = while_body.attentionOnePage(state, q_offset, num_slots, page_size); + return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax); + } + }, + .{ + .while_body = self, + .state = state_, + .seq_num_queries = seq_num_queries_, + .page_size = page_size_, + }, + .cmp(seq_num_queries_, .GT, .scalar(1, .i32)), + ); + } + + pub fn attentionOnePage(self: While, state: State, q_offset: zml.Tensor, q_chunk: u32, page_size: u32) PartialSoftmax { + const k_offset = state.k_page_idx.scale(page_size); + const active_q = self.q.slice(self.q.axis(.b), .dyn(state.slot_id, q_chunk)); + + const active_k_page_id = self.parameters.block_table.slices( + .{ .b, .p }, + &.{ .dynSingle(state.seq_id), .dynSingle(state.k_page_idx) }, + ); + const active_k, const active_v = self.kv_cache.getPage(active_k_page_id); + + const dtype = self.q.dtype(); + const attn_mask: zml.Tensor = attn_mask: { + const mask_shape: zml.Shape = .init(.{ .b = q_chunk, .k_chunk = page_size }, dtype); + + const q_idx = zml.Tensor.iota(mask_shape, .b).add(q_offset); + const k_idx = zml.Tensor.iota(mask_shape, .k_chunk).add(k_offset); + + const mask = zml.Tensor.cmp(k_idx, .LE, q_idx); + const minus_inf = zml.Tensor.constant(dtype.minValue()); + break :attn_mask .select(mask, .scalar(0, dtype), minus_inf); + }; + + const scale: f32 = self.opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(active_q.dim(.hd))))); + var attn_weights = active_q.dot(active_k, .hd).scale(scale); + attn_weights = attn_weights.add(attn_mask.broad(attn_weights.shape())); + + const og_softmax = if (q_chunk == 1) state.partial_softmax_decode else state.partial_softmax_prefill; + // Note: we compute q * k using q and k dtype, but the partial softmax is in f32. + const precise_dt = og_softmax.values.dtype(); + var partial = partialSoftmax(attn_weights.convert(precise_dt), .k_chunk); + partial.values = partial.values.dot(active_v.convert(precise_dt), .k_chunk).transpose(og_softmax.values.shape()); + partial.max_value = partial.max_value.transpose(og_softmax.max_value.shape()); + partial.exp_sum = partial.exp_sum.transpose(og_softmax.exp_sum.shape()); + + return og_softmax.merge(partial); + } + + pub fn updateState(state: State, num_slots: u32, query_has_more_keys: zml.Tensor, sequence_has_more_queries: zml.Tensor, softmax: PartialSoftmax) State { + return zml.ops.if2( + query_has_more_keys, + // If the query has more pages: update partial softmax with attention on current page, delay updating output. + State{ + .slot_id = state.slot_id, + .seq_id = state.seq_id, + .q_page_idx = state.q_page_idx, + .k_page_idx = state.k_page_idx.addConstant(1), + .partial_softmax_prefill = if (num_slots == 1) state.partial_softmax_prefill else softmax, + .partial_softmax_decode = if (num_slots == 1) softmax else state.partial_softmax_decode, + .out = state.out, + }, + // Query has no more pages, we commit softmax to output and increment slot + State{ + .k_page_idx = .scalar(0, .i32), + .partial_softmax_prefill = if (num_slots == 1) state.partial_softmax_prefill else .reset(softmax), + .partial_softmax_decode = if (num_slots == 1) .reset(softmax) else state.partial_softmax_decode, + .out = state.out.dynamicUpdateSlice(.{ .b = state.slot_id }, softmax.finalize().convert(state.out.dtype())), + .slot_id = state.slot_id.addConstant(num_slots), + // If the sequence has more query chunks: increment q_page_idx, otherwise start over at 0. + .q_page_idx = .select(sequence_has_more_queries, state.q_page_idx.addConstant(1), .scalar(0, .i32)), + // If the sequence has more query chunks: keep seq_id, otherwise increment seq_id. + .seq_id = .select(sequence_has_more_queries, state.seq_id, state.seq_id.addConstant(1)), + }, + ); + } +}; + +pub const PartialSoftmax = struct { + values: zml.Tensor, + exp_sum: zml.Tensor, + max_value: zml.Tensor, + + pub fn zeroes(q_shape: zml.Shape) PartialSoftmax { + return .{ + .values = .zeroes(q_shape), + .exp_sum = .zeroes(q_shape.drop(.hd)), + .max_value = .broad(.constant(q_shape.dtype().minValue()), q_shape.drop(.hd)), + }; + } + + pub fn reset(self: PartialSoftmax) PartialSoftmax { + return .zeroes(self.values.shape()); + } + + pub fn merge(self: PartialSoftmax, other: PartialSoftmax) PartialSoftmax { + // Rescale self and other using the new global_max. + const global_max = self.max_value.maximum(other.max_value.convert(self.max_value.dtype())); + const new_self = self.rescale(global_max); + const new_other = other.rescale(global_max); + + // Now that self and other are using the same scale, we can just add them: + return .{ + .max_value = global_max, + .values = new_self.values.add(new_other.values), + .exp_sum = new_self.exp_sum.add(new_other.exp_sum), + }; + } + + /// Update max_value and rescale attn and exp_sum accordingly. + pub fn rescale(self: PartialSoftmax, max_value: zml.Tensor) PartialSoftmax { + const max_diff_exp: zml.Tensor = .exp(self.max_value.sub(max_value)); + const sum_dtype = self.exp_sum.dtype(); + return .{ + .max_value = max_value, + .values = self.values.mul(max_diff_exp.broad(self.values.shape())), + .exp_sum = self.exp_sum.mul(max_diff_exp.convert(sum_dtype)), + }; + } + + /// Divides the intermediary results by the exp_sum to get the proper attention values. + pub fn finalize(self: PartialSoftmax) zml.Tensor { + return self.values.div(self.exp_sum.broad(self.values.shape()).convert(self.values.dtype())); + } +}; + +/// Compute softmax over a chunk. +/// Returns intermediary results to allow aggregating later. +pub fn partialSoftmax(self: zml.Tensor, axis: anytype) PartialSoftmax { + const a = self.axis(axis); + const max_val = self.max(a).squeeze(a).maximum(.scalar(-1e16, self.dtype())); + const out = self.sub(max_val.broad(self.shape())).exp(); + return .{ + .values = out, + .exp_sum = out.convert(.f32).sum(a).squeeze(a), + .max_value = max_val, + }; +} diff --git a/zml/kernels/paged_attention/triton/BUILD.bazel b/zml/kernels/paged_attention/triton/BUILD.bazel new file mode 100644 index 00000000..fbce732b --- /dev/null +++ b/zml/kernels/paged_attention/triton/BUILD.bazel @@ -0,0 +1,10 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "triton", + srcs = [ + "triton.zig", + "unified_attention.zig", + "unified_attention_oneapi.zig", + ], +) diff --git a/zml/attention/triton_attention.zig b/zml/kernels/paged_attention/triton/triton.zig similarity index 68% rename from zml/attention/triton_attention.zig rename to zml/kernels/paged_attention/triton/triton.zig index 9c2cf38c..f4212706 100644 --- a/zml/attention/triton_attention.zig +++ b/zml/kernels/paged_attention/triton/triton.zig @@ -2,14 +2,11 @@ const std = @import("std"); const stdx = @import("stdx"); -const zml = @import("../zml.zig"); +const zml = @import("../../../zml.zig"); const triton = zml.kernel.triton; -const AttentionOptions = @import("paged_attention.zig").AttentionOptions; -const MlaOptions = @import("paged_attention.zig").Mla.Options; -const sparse_mla = @import("sparse_mla.zig"); -const kernels = @import("triton_kernels/unified_attention.zig"); -const kernels_oneapi = @import("triton_kernels/unified_attention_oneapi.zig"); -const mla_kernels = @import("triton_kernels/unified_sparse_mla.zig"); +const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; +const kernels = @import("unified_attention.zig"); +const kernels_oneapi = @import("unified_attention_oneapi.zig"); const log = std.log.scoped(.@"zml/attention/triton"); @@ -717,276 +714,4 @@ pub const paged = struct { return output.output; } - - pub fn pagedSparseMlaKernel(q: zml.Tensor, kv_cache: zml.Tensor, sink: ?zml.Tensor, topk: zml.Tensor, active_query_count: zml.Tensor, paged_opts: PagedSparseMlaOptions) zml.Tensor { - const value_rank: i64 = @intCast(paged_opts.value_rank); - const rope_rank: i64 = if (paged_opts.value_rank == paged_opts.qk_rank) 0 else @intCast(paged_opts.rope_rank); - stdx.debug.assert(std.math.isPowerOfTwo(paged_opts.value_rank), "sparse MLA value rank ({}) must be a power of two", .{paged_opts.value_rank}); - stdx.debug.assert(rope_rank == 0 or std.math.isPowerOfTwo(@as(usize, @intCast(rope_rank))), "sparse MLA separate RoPE rank ({}) must be a power of two", .{rope_rank}); - stdx.debug.assert(value_rank + rope_rank == paged_opts.qk_rank, "sparse MLA expects either a full-width value or adjacent value/RoPE regions, got qk={} value={} rope={}", .{ paged_opts.qk_rank, value_rank, rope_rank }); - - const out_shape = q.shape().set(.hd, value_rank); - - const q_strides = q.shape().computeElementStrides().constSlice(); - const out_strides = out_shape.computeElementStrides().constSlice(); - const kv_strides = kv_cache.shape().computeElementStrides().constSlice(); - - const sink_: zml.Tensor = sink orelse .scalar(0, .i8); - const use_sink = sink != null; - const sm_scale: f32 = paged_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(paged_opts.qk_rank)))); - - const config = sparse_mla.launchConfig(paged_opts, @intCast(topk.dim(.topk)), getCuCount()); - - const kernel_config: mla_kernels.Config = .{ - .q_dtype = triton.from(q.dtype()), - .kv_dtype = triton.from(kv_cache.dtype()), - .sink_dtype = triton.from(sink_.dtype()), - .o_dtype = triton.from(q.dtype()), - .num_query_heads = @intCast(paged_opts.num_heads), - .block_size = @intCast(paged_opts.block_size), - .block_m = @intCast(config.block_m), - .topk_count = topk.dim(.topk), - .rope_rank = rope_rank, - .value_rank = value_rank, - .rope_offset = value_rank, - .tile_size = @intCast(config.tile_size), - .num_splits = @intCast(config.num_splits), - .use_attn_sink = use_sink, - .all_decode = paged_opts.all_decode, - }; - log.debug("pagedSparseMla config: {any}, kernel: {any}", .{ config, kernel_config }); - - const kernel_inputs: mla_kernels.Kernel2D.Inputs = .{ - .query_ptr = q, - .kv_cache_ptr = kv_cache, - .attn_sink_ptr = sink_, - .topk_indices_ptr = topk, - .active_query_count_ptr = active_query_count, - .scale_ptr = zml.Tensor.constant(zml.DataType.f32.constant(sm_scale)), - .query_stride_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(q_strides[0])), - .query_stride_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(q_strides[1])), - .output_stride_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(out_strides[0])), - .output_stride_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(out_strides[1])), - .stride_cache_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(kv_strides[kv_cache.shape().axis(.page)])), - .stride_cache_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(kv_strides[kv_cache.shape().axis(.k_chunk)])), - }; - - if (config.num_splits == 1) { - const out = mla_kernels.Kernel2D.call( - kernel_inputs, - .{ .output = out_shape }, - .{ - .cfg = kernel_config, - .grid = .{ @intCast(config.direct_programs), 1, 1 }, - .num_warps = @intCast(config.main_num_warps), - .num_stages = @intCast(config.main_num_stages), - }, - ); - return out.output.reshape(out_shape); - } - - const num_splits: i64 = @intCast(config.num_splits); - const kernel_3d_inputs: mla_kernels.Kernel3D.Inputs = .{ - .query_ptr = kernel_inputs.query_ptr, - .kv_cache_ptr = kernel_inputs.kv_cache_ptr, - .attn_sink_ptr = kernel_inputs.attn_sink_ptr, - .topk_indices_ptr = kernel_inputs.topk_indices_ptr, - .active_query_count_ptr = kernel_inputs.active_query_count_ptr, - .scale_ptr = kernel_inputs.scale_ptr, - .query_stride_0_ptr = kernel_inputs.query_stride_0_ptr, - .query_stride_1_ptr = kernel_inputs.query_stride_1_ptr, - .output_stride_0_ptr = kernel_inputs.output_stride_0_ptr, - .output_stride_1_ptr = kernel_inputs.output_stride_1_ptr, - .stride_cache_0_ptr = kernel_inputs.stride_cache_0_ptr, - .stride_cache_1_ptr = kernel_inputs.stride_cache_1_ptr, - }; - const partials = mla_kernels.Kernel3D.call( - kernel_3d_inputs, - .{ - .partial_output = zml.Shape.init(.{ q.dim(.q), @as(i64, @intCast(paged_opts.num_heads)), num_splits, value_rank }, .f32), - .partial_lse = zml.Shape.init(.{ q.dim(.q), @as(i64, @intCast(paged_opts.num_heads)), num_splits }, .f32), - }, - .{ - .cfg = kernel_config, - .grid = .{ @intCast(config.direct_programs), @intCast(config.num_splits), 1 }, - .num_warps = @intCast(config.main_num_warps), - .num_stages = @intCast(config.main_num_stages), - }, - ); - var reduce_partial_output = partials.partial_output; - var reduce_partial_lse = partials.partial_lse; - var reduce_num_splits = num_splits; - if (num_splits > config.grouped_reduce_threshold) { - const grouped_splits: i64 = @divExact(num_splits, @as(i64, @intCast(config.splits_per_group))); - const grouped = mla_kernels.Reduce3DPartials.call( - .{ - .input = partials.partial_output, - .input_lse = partials.partial_lse, - .active_query_count_ptr = active_query_count, - }, - .{ - .partial_output = zml.Shape.init(.{ q.dim(.q), @as(i64, @intCast(paged_opts.num_heads)), grouped_splits, value_rank }, .f32), - .partial_lse = zml.Shape.init(.{ q.dim(.q), @as(i64, @intCast(paged_opts.num_heads)), grouped_splits }, .f32), - }, - .{ - .cfg = .{ - .num_query_heads = @intCast(paged_opts.num_heads), - .value_rank = value_rank, - .num_input_splits = num_splits, - .num_output_splits = grouped_splits, - .all_decode = paged_opts.all_decode, - }, - .grid = .{ @intCast(q.dim(.q)), @intCast(paged_opts.num_heads * @as(usize, @intCast(grouped_splits))), 1 }, - .num_warps = @intCast(config.grouped_reduce_num_warps), - .num_stages = 1, - }, - ); - reduce_partial_output = grouped.partial_output; - reduce_partial_lse = grouped.partial_lse; - reduce_num_splits = grouped_splits; - } - - const out = mla_kernels.Reduce3D.call( - .{ - .partial_output_ptr = reduce_partial_output, - .partial_lse_ptr = reduce_partial_lse, - .attn_sink_ptr = sink_, - .active_query_count_ptr = active_query_count, - .output_stride_0_ptr = zml.Tensor.constant(zml.DataType.i64.constant(out_strides[0])), - .output_stride_1_ptr = zml.Tensor.constant(zml.DataType.i64.constant(out_strides[1])), - }, - .{ .output = out_shape }, - .{ - .cfg = .{ - .sink_dtype = triton.from(sink_.dtype()), - .o_dtype = triton.from(q.dtype()), - .num_query_heads = @intCast(paged_opts.num_heads), - .value_rank = value_rank, - .num_splits = reduce_num_splits, - .use_attn_sink = use_sink, - .all_decode = paged_opts.all_decode, - }, - .grid = .{ @intCast(q.dim(.q)), @intCast(paged_opts.num_heads), 1 }, - .num_warps = @intCast(if (reduce_num_splits >= config.parallel_reduce_min_splits) - config.parallel_reduce_num_warps - else - 1), - .num_stages = 1, - }, - ); - - return out.output.reshape(out_shape); - } - - pub fn tokenToSequence(query_start_len: zml.Tensor, query_count: i64) zml.Tensor { - const sequence_count = query_start_len.dim(.b) - 1; - const starts = query_start_len.slice(.b, .{ .end = sequence_count }).rename(.{ .b = .seq }); - const ends = query_start_len.slice(.b, .{ .start = 1 }).rename(.{ .b = .seq }); - const query_x_sequence_shape = zml.Shape.init(.{ .q = query_count, .seq = sequence_count }, .i32); - const query = zml.Tensor.iota(query_x_sequence_shape, .q).convert(.i32); - const in_range = query.cmp(.GE, starts.broad(query_x_sequence_shape)).convert(.i32) - .mul(query.cmp(.LT, ends.broad(query_x_sequence_shape)).convert(.i32)) - .cmp(.NE, zml.Tensor.zeroes(query_x_sequence_shape)); - const sequence = zml.Tensor.iota(query_x_sequence_shape, .seq).convert(.i32); - return zml.Tensor.select(in_range, sequence, zml.Tensor.zeroes(query_x_sequence_shape)).sum(.seq).squeeze(.seq); - } - - pub fn topkToPhysical(parameters: anytype, topk: zml.Tensor, tokens_pos: zml.Tensor, block_size: i64) zml.Tensor { - const topk_i32 = topk.convert(.i32); - const topk_shape = topk_i32.shape(); - - stdx.debug.assert(topk_shape.hasTags(.{ .q, .topk }), "paged MLA topk must have .q and .topk axes, got {f}", .{topk_shape}); - stdx.debug.assert(tokens_pos.shape().hasTags(.{.q}), "paged MLA token positions must have a .q axis, got {f}", .{tokens_pos.shape()}); - - const query_to_sequence = tokenToSequence(parameters.query_start_len, topk_i32.dim(.q)); - const sequence_ends = parameters.query_start_len.slice(.b, .{ .start = 1 }).rename(.{ .b = .seq }); - const last_query = sequence_ends.gather(.{ .seq = query_to_sequence }, .{}).sub(.scalar(1, .i32)); - const last_token_pos = tokens_pos - .gather(.{ .q = last_query.rename(.{ .q = .lookup }) }, .{}) - .rename(.{ .lookup = .q }) - .convert(.i32); - const seq_lens = parameters.seq_lens.rename(.{ .b = .seq }).gather(.{ .seq = query_to_sequence }, .{}); - const first_visible_token = last_token_pos.addConstant(1).sub(seq_lens); - const relative_topk = topk_i32.sub(first_visible_token.broad(topk_shape)); - - const block_size_scalar = zml.Tensor.scalar(@as(i32, @intCast(block_size)), .i32).broad(topk_shape); - const zero = zml.Tensor.zeroes(topk_shape); - const valid_nonnegative = relative_topk.cmp(.GE, zero); - const valid_in_sequence = relative_topk.cmp(.LT, seq_lens.broad(topk_shape)); - const valid_topk = valid_nonnegative.logical(.AND, valid_in_sequence); - - const safe_topk = zml.Tensor.select(valid_topk, relative_topk, zero); - const logical_block = safe_topk.div(block_size_scalar); - const slot = safe_topk.remainder(block_size_scalar); - - const sequence = query_to_sequence.broad(topk_shape); - const block_table = parameters.block_table.rename(.{ .b = .seq }); - const physical_block = block_table.gather(.{ .seq = sequence, .p = logical_block }, .{}); - - const physical_topk = physical_block.mul(block_size_scalar).add(slot); - - return zml.Tensor.select(valid_topk, physical_topk, zml.Tensor.scalar(-1, .i32).broad(topk_shape)); - } - - pub const PagedSparseMlaOptions = sparse_mla.Options; - - pub fn pagedSparseMla(parameters: Parameters, q: zml.Tensor, kv_cache: zml.Tensor, sink: ?zml.Tensor, topk: zml.Tensor, tokens_pos: zml.Tensor, opts: MlaOptions) zml.Tensor { - return sparse_mla.pagedAttention(parameters, q, kv_cache, sink, topk, tokens_pos, opts); - } }; - -test "sparse MLA emits 2D and 3D Triton kernels" { - const platform = zml.testing.env(); - var compilation = zml.Compiler.init(std.testing.allocator, std.testing.io, platform, .{}); - defer compilation.deinit(); - compilation.activate(); - defer compilation.deactivate(); - - const block = @import("mlir").Block.init(&.{}, &.{}); - const scope = compilation.pushBlock(block); - defer scope.pop(); - - const q = zml.Tensor.zeroes(zml.Shape.init(.{ .q = 1, .h = 6, .hd = 576 }, .bf16)); - const kv = zml.Tensor.zeroes(zml.Shape.init(.{ .page = 32, .k_chunk = 1, .hkv = 1, .hd = 576 }, .bf16)); - const sink = zml.Tensor.zeroes(zml.Shape.init(.{ .h = 6 }, .bf16)); - const topk = zml.Tensor.zeroes(zml.Shape.init(.{ .q = 1, .topk = 32 }, .i32)); - const active_query_count = zml.Tensor.scalar(1, .i32); - const two_d_opts: paged.PagedSparseMlaOptions = .{ - .qk_rank = 576, - .value_rank = 512, - .num_heads = 6, - .block_size = 1, - .rope_rank = 64, - .scale = null, - .total_q_blocks = 1, - .num_kv_splits = 1, - .all_decode = true, - }; - const output_shape = q.shape().set(.hd, 512); - - const two_d = paged.pagedSparseMlaKernel(q, kv, sink, topk, active_query_count, two_d_opts); - try std.testing.expect(two_d.shape().eql(output_shape)); - try std.testing.expect(two_d.value().owner().verify()); - - var three_d_opts = two_d_opts; - three_d_opts.num_kv_splits = 2; - const three_d = paged.pagedSparseMlaKernel(q, kv, sink, topk, active_query_count, three_d_opts); - try std.testing.expect(three_d.shape().eql(output_shape)); - try std.testing.expect(three_d.value().owner().verify()); - - const dsv4_q = zml.Tensor.zeroes(zml.Shape.init(.{ .q = 1, .h = 6, .hd = 512 }, .bf16)); - const dsv4_kv = zml.Tensor.zeroes(zml.Shape.init(.{ .page = 32, .k_chunk = 1, .hkv = 1, .hd = 512 }, .bf16)); - var dsv4_opts = two_d_opts; - dsv4_opts.qk_rank = 512; - dsv4_opts.value_rank = 512; - const dsv4_output_shape = dsv4_q.shape(); - - const dsv4_two_d = paged.pagedSparseMlaKernel(dsv4_q, dsv4_kv, sink, topk, active_query_count, dsv4_opts); - try std.testing.expect(dsv4_two_d.shape().eql(dsv4_output_shape)); - try std.testing.expect(dsv4_two_d.value().owner().verify()); - - dsv4_opts.num_kv_splits = 2; - const dsv4_three_d = paged.pagedSparseMlaKernel(dsv4_q, dsv4_kv, sink, topk, active_query_count, dsv4_opts); - try std.testing.expect(dsv4_three_d.shape().eql(dsv4_output_shape)); - try std.testing.expect(dsv4_three_d.value().owner().verify()); -} diff --git a/zml/attention/triton_kernels/unified_attention.zig b/zml/kernels/paged_attention/triton/unified_attention.zig similarity index 99% rename from zml/attention/triton_kernels/unified_attention.zig rename to zml/kernels/paged_attention/triton/unified_attention.zig index dd219a88..b6ca3f12 100644 --- a/zml/attention/triton_kernels/unified_attention.zig +++ b/zml/kernels/paged_attention/triton/unified_attention.zig @@ -4,7 +4,7 @@ const mlir = @import("mlir"); const dialects = @import("mlir/dialects"); const ttir = dialects.ttir; -const zml = @import("../../zml.zig"); +const zml = @import("../../../zml.zig"); const tri = zml.kernel.triton; const Builder = tri.Builder; const Value = tri.Value; diff --git a/zml/attention/triton_kernels/unified_attention_oneapi.zig b/zml/kernels/paged_attention/triton/unified_attention_oneapi.zig similarity index 99% rename from zml/attention/triton_kernels/unified_attention_oneapi.zig rename to zml/kernels/paged_attention/triton/unified_attention_oneapi.zig index c1feea1c..3514f00f 100644 --- a/zml/attention/triton_kernels/unified_attention_oneapi.zig +++ b/zml/kernels/paged_attention/triton/unified_attention_oneapi.zig @@ -4,7 +4,7 @@ const mlir = @import("mlir"); const dialects = @import("mlir/dialects"); const ttir = dialects.ttir; -const zml = @import("../../zml.zig"); +const zml = @import("../../../zml.zig"); const tri = zml.kernel.triton; const Builder = tri.Builder; const Value = tri.Value; diff --git a/zml/platform.zig b/zml/platform.zig index 66f7df01..354ec3c1 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -6,7 +6,6 @@ const pjrt = @import("pjrt"); const stdx = @import("stdx"); pub const Target = @import("platforms").Platform; -const attention = @import("attention.zig"); const constants = @import("constants.zig"); const Exe = @import("exe.zig").Exe; const pjrtx = @import("pjrtx.zig"); @@ -382,12 +381,6 @@ pub const Platform = struct { switch (target) { .cuda => { - zml.attention.flashattn.load(arena, io) catch { - log.warn("Failed to load flashattn", .{}); - }; - zml.attention.flashattn.register(platform) catch { - log.warn("Failed to register flashattn custom call", .{}); - }; if (zml.moe.cutlass_flashinfer.load(arena, io, platform)) { zml.moe.cutlass_flashinfer.register(platform) catch |err| { log.warn( diff --git a/zml/zml.zig b/zml/zml.zig index f648548f..d5ee6f08 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -13,7 +13,8 @@ const runfiles = @import("runfiles"); pub const stdx = @import("stdx"); pub const tokenizer = @import("zml/tokenizer"); -pub const attention = @import("attention.zig"); +pub const attention = @import("attention/attention.zig"); +pub const paged_attention = @import("kernels/paged_attention/paged_attention.zig"); pub const Buffer = @import("buffer.zig").Buffer; pub const Compiler = @import("Compiler.zig"); pub const constants = @import("constants.zig"); From a0c358db12ae5aacfdffb4e171a9d130c7b64ee5 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 13 Oct 2026 13:34:13 +0200 Subject: [PATCH 777/881] zml/kernels/paged_attention: removed latent kvcache (#929) It's not used anymore --- .../paged_attention/paged_attention.zig | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index c3a747f1..da919554 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -178,7 +178,6 @@ pub const KvCache = union(enum) { }, // TODO: rename `dense` to `fused`. dense: zml.Tensor, - latent: zml.Tensor, pub fn update( self: KvCache, @@ -208,13 +207,6 @@ pub const KvCache = union(enum) { }, }, .dense => @panic("TODO"), - .latent => |latent_kv| .{ - .latent = latent_kv.scatterSlices( - .{ .page = page_index, .k_chunk = offset }, - new_k, - .{ .update_fn = zml.Tensor.ScatterOpts.override, .indices_are_unique = false, .indices_are_sorted = false }, - ).reuseBuffer(self.latent), - }, }; return kv; @@ -235,7 +227,7 @@ pub const KvCache = union(enum) { split.k.slice(split.k.axis(.page), .dynSingle(page_index)), split.v.slice(split.k.axis(.page), .dynSingle(page_index)), }, - .dense, .latent => @panic("TODO"), + .dense => @panic("TODO"), }; } }; @@ -247,22 +239,16 @@ pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: z .triton => |triton_parameters| switch (kv_cache) { .split => |split| triton.paged.pagedAttention(triton_parameters, q, split.k, split.v, opts), .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), - .latent => std.debug.panic("latent KV pages are only supported with Multi-Latent Attention", .{}), }, .mosaic_tpu => |mosaic_tpu_parameters| switch (kv_cache) { .split => std.debug.panic("split KV pages is not supported with the mosaic_tpu backend", .{}), - .latent => std.debug.panic("latent KV pages are only supported with Multi-Latent Attention", .{}), .dense => tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, q, kv_cache.dense, opts), }, .metal => |metal_parameters| switch (kv_cache) { .split => |split| metal.pagedAttention(metal_parameters, q, split.k, split.v, opts), .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), - .latent => std.debug.panic("latent KV pages are only supported with Multi-Latent Attention", .{}), - }, - .stablehlo => |params| switch (kv_cache) { - .latent => std.debug.panic("latent KV pages are only supported with Multi-Latent Attention", .{}), - else => stablehlo.pagedAttention(params, q, kv_cache, opts), }, + .stablehlo => |params| stablehlo.pagedAttention(params, q, kv_cache, opts), }; } From 8e95da3f2df23cff4ea5cbeae33906b8f97158c3 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 13 Oct 2026 13:34:14 +0200 Subject: [PATCH 778/881] zml/kernels/paged_attention: add kernel documentation (#930) --- zml/kernels/paged_attention/README.md | 184 ++++++++++++++++++ zml/kernels/paged_attention/metal/README.md | 24 +++ .../paged_attention/mosaic_tpu/README.md | 28 +++ .../paged_attention/stablehlo/README.md | 30 +++ zml/kernels/paged_attention/triton/README.md | 28 +++ 5 files changed, 294 insertions(+) create mode 100644 zml/kernels/paged_attention/README.md create mode 100644 zml/kernels/paged_attention/metal/README.md create mode 100644 zml/kernels/paged_attention/mosaic_tpu/README.md create mode 100644 zml/kernels/paged_attention/stablehlo/README.md create mode 100644 zml/kernels/paged_attention/triton/README.md diff --git a/zml/kernels/paged_attention/README.md b/zml/kernels/paged_attention/README.md new file mode 100644 index 00000000..20ba9d7d --- /dev/null +++ b/zml/kernels/paged_attention/README.md @@ -0,0 +1,184 @@ +# Paged attention semantics + +[paged_attention.zig](paged_attention.zig) evaluates attention over a KV cache +whose logical sequences are stored in fixed-size physical pages. Paging changes +how keys and values are addressed; softmax still normalizes over all permitted +keys in one sequence, independently for each query and query head. + +The public entry point is: + +```zig +pagedAttention(parameters, q, k, v, kv_cache, opts) +``` + +The result has the query's shape and dtype. The `k` and `v` arguments are currently +ignored: attention reads exclusively from `kv_cache`. Populate the cache, +including the current queries' keys and values, **before** calling attention. +The attention call does not update the cache or allocate pages. Projections, +RoPE, and any Q/K normalization happen before this operation; the output +projection happens afterward. + +## Tensor layouts and heads + +Let `T` be the packed query token count, `B` the sequence capacity, `N` the +physical page count, `P` the tokens per page, `M` the page-table capacity per +sequence, `Hkv` the KV head count, `G` the query heads per KV head, and `D` the +features per head. + +| Input | Tagged shape | Meaning | +| --- | --- | --- | +| `q` | `{ .b = T, .hkv = Hkv, .hg = G, .hd = D }` | Queries packed in sequence order. Here `.b` counts tokens, not sequences. | +| Split cache `k`, `v` | `{ .page = N, .k_chunk = P, .hkv = Hkv, .hd = D }` | Physical key/value storage. | +| `parameters.block_table` | `{ .b = B, .p = M }`, `i32` | Logical page to physical page mapping for each sequence. | +| `parameters.seq_lens` | `{ .b = B }`, `i32` | Valid KV token counts, including the current queries. | +| `parameters.query_start_len` | `{ .b = B + 1 }`, `i32` | Cumulative offsets into packed queries. | + +The total query head count is `Hq = Hkv * G`. Flattening the two head axes gives +`h = hkv * G + hg`, so query head `h` reads KV head `floor(h / G)`. Each query +head has its own scores and softmax. This covers MHA (`G = 1`), GQA, and MQA +(`Hkv = 1`). `D` is the feature count of one head, not the number of heads or +the sequence length. Use matching floating-point query and cache dtypes. + +The TPU backend instead requires `KvCache.dense`, a fused tensor with layout +`{ .page = N, .k_chunk = P, .hkv = 2 * Hkv, .hd = Dpad }`. The combined head +axis interleaves `K0, V0, K1, V1, ...`. `Dpad` must equal `D` rounded up to a +multiple of 128. Pad unused cache features with zeros. The wrapper pads queries, +uses the logical `D` for the default scale, and trims the output back to `D`. +Its accepted query/cache dtypes are `bf16`, `f16`, and `f32`. + +## Packed sequences and positions + +For sequence `s`, define: + +```text +start = query_start_len[s] +end = query_start_len[s + 1] +Lq = end - start +Lk = seq_lens[s] +context_len = Lk - Lq +position(i) = context_len + i # 0 <= i < Lq +``` + +Queries `q[start:end]` represent the final `Lq` tokens of that sequence. The +kernel infers their positions from lengths; there is no separate position input. +An initial prefill has `Lq = Lk`, a chunked prefill has `Lq > 1` after a cached +prefix, and a decode normally has `Lq = 1` at position `Lk - 1`. A batch may mix +prefill and decode sequences. Set `is_prefill = true` for such mixed batches; +`false` selects the all-decode execution configuration on Triton. It does not +change the attention equation. `max_seqlen_q` is a configuration bound for the +largest query chunk, not a runtime sequence length. + +Offsets must begin at zero and be nondecreasing. Active sequences require +`0 < Lq <= Lk <= M * P`. Place inactive sequences after the active sequences, +with repeated offsets and normally `seq_lens = 0`. This ordering matters: +StableHLO stops at the first sequence with no queries, and TPU passes a count of +active sequences to its kernel. The final offset is the active packed token +count; any extra query capacity is padding whose output should not be consumed. + +## Page addressing and cache updates + +A logical key position `j` in sequence `s` is addressed as: + +```text +logical_page = floor(j / P) +physical_page = block_table[s, logical_page] +offset = j % P +K(s, j) = k_cache[physical_page, offset] +V(s, j) = v_cache[physical_page, offset] +``` + +Physical pages need not be contiguous or ordered. Different sequences may map +to the same physical pages to share a prefix. Every page covering a valid token +must have a valid physical ID in `[0, N)` and populated KV data. Entries beyond +`ceil(Lk / P)` are unused; tests use `-1` there. A negative ID is not a portable +way to mask a valid logical page. `seq_lens`, rather than allocated page count, +determines which tokens in the last page are valid. + +`KvCache.update(new_k, new_v, slot_mapping, chunk_size, backend)` writes split +cache entries using physical flattened slots: + +```text +chunk_size = P +physical_page = slot_mapping[t] / P +offset = slot_mapping[t] % P +``` + +`new_k` and `new_v` have packed token/head/feature axes matching the split +cache. `slot_mapping` is a physical address, whereas `block_table` translates +logical sequence positions. The caller maintains both mappings and lengths; +`update` does not derive or update sequence metadata. Writes override existing +entries and request buffer reuse. Use valid, distinct slots for writes requiring +deterministic results; conflicting scatter writes have no ordering guarantee. +The generic update method currently panics for `.dense`. + +## Attention equation and masks + +For local query row `i`, KV head `a`, and query group head `g`: + +```text +p = Lk - Lq + i +score_j = scale * dot(q[start + i, a, g], K(s, j, a)) +weight = softmax(score over permitted j) +out = sum_j(weight_j * V(s, j, a)) +``` + +`opts.scale` defaults to `1 / sqrt(D)`. There is no dropout or returned +probability matrix. Internal paging, tiling, or splitting of the key range must +preserve one softmax normalization; summing separately normalized page outputs +would compute a different result. Accumulation order and dtype conversions can +cause numerical differences between backends. + +The default mask is `.standard = .{ .causal = true }`: + +| Mask | Permitted logical key positions | +| --- | --- | +| Default causal | `0 <= j < Lk` and `j <= p`. Includes the query's own key. | +| `AttentionMask.none` | Every `0 <= j < Lk`. | +| Causal with `sliding_window = W` | `0 <= j < Lk` and `p - W < j <= p`. | +| Non-causal with `sliding_window = W` | `0 <= j < Lk` and `p - W < j`. Includes all later valid keys. | + +Use a positive `W`; it counts the current token as part of the left window. +Use `null` for no window. A non-causal window is a left bound, not a symmetric +window around the query. + +Triton also accepts `.indices`, an `i32` tensor shaped +`{ .b = T, .topk = K }` with `K > 0`. Each packed query row supplies logical +key positions in its own sequence, shared across its heads. Positions outside +`[0, Lk)` are ignored. This replaces the standard mask, so future valid keys +are permitted. Duplicate positions are processed repeatedly and therefore +change their weight; provide unique positions to represent a set of keys. + +On Triton, `opts.sink` supplies one softmax logit per flattened query head. +It adds `exp(sink[h])` to the denominator and contributes zero to the weighted +value numerator. It is not a cache token and does not consume a page or sequence +position. Other wrappers currently ignore `sink`. For portable use, ensure each +active query has at least one permitted key; fully masked rows are not given a +shared cross-backend output contract. + +## Example + +With `P = 4`, consider a two-query chunk after a four-token prefix and a decode +after a two-token prefix, followed by one inactive sequence: + +```text +query_start_len = [0, 2, 3, 3] +seq_lens = [6, 3, 0] +block_table = [[5, 2], [8, -1], [-1, -1]] +``` + +Packed queries 0 and 1 belong to sequence 0 at positions 4 and 5. Their current +KV entries occupy physical page 2, offsets 0 and 1 (slots 8 and 9). Packed query +2 belongs to sequence 1 at position 2, physical page 8, offset 2 (slot 34). +The default causal mask permits keys 0–4, 0–5, and 0–2 respectively, always +within the query's own sequence. With a window of 3, the permitted positions +become 2–4, 3–5, and 0–2. + +## Backend implementations + +See each backend's README for execution details, supported options, and current +limitations: + +- [Triton](triton/README.md) — CUDA, ROCm, and oneAPI. +- [Mosaic TPU](mosaic_tpu/README.md) — TPU. +- [Metal](metal/README.md) — Metal. +- [StableHLO](stablehlo/README.md) — CPU and Furiosa fallback. diff --git a/zml/kernels/paged_attention/metal/README.md b/zml/kernels/paged_attention/metal/README.md new file mode 100644 index 00000000..8e3e5d6e --- /dev/null +++ b/zml/kernels/paged_attention/metal/README.md @@ -0,0 +1,24 @@ +# Metal paged attention + +This backend is automatically selected for Metal. See the +[parent README](../README.md) for the attention equation, packed sequence +metadata, and page addressing contract. + +[The wrapper](metal.zig) requires split KV caches. It merges query head groups, +transposes queries and caches into the custom call's layouts, invokes +`zml$paged_attn`, and restores the query's tagged layout for the output. + +Causal and non-causal standard masks, left sliding windows, and a custom scale +are passed to the custom call. Explicit key indices are rejected, and +`opts.sink` is currently ignored. Fused caches are rejected by +ordinary paged attention. + +When sharding heads, keep complete GQA groups together with matching query and +cache head ownership. Each local computation needs the full required key range, +features, and matching sequence metadata. + +The shared tests in [paged_attention.zig](../paged_attention.zig) compare this +backend against StableHLO for causal attention and include option-specific +comparisons. They exercise `bf16`, mixed prefill/decode, shared prefix pages, +partial final KV pages, and trailing inactive sequences. These cases do not +establish support for every shape or dtype. diff --git a/zml/kernels/paged_attention/mosaic_tpu/README.md b/zml/kernels/paged_attention/mosaic_tpu/README.md new file mode 100644 index 00000000..6fff585c --- /dev/null +++ b/zml/kernels/paged_attention/mosaic_tpu/README.md @@ -0,0 +1,28 @@ +# Mosaic TPU paged attention + +This backend is automatically selected for TPU. See the +[parent README](../README.md) for the attention equation and packed sequence +metadata contract. + +[The wrapper](mosaic_tpu.zig) requires fused `KvCache.dense` storage with shape +`{ .page = N, .k_chunk = P, .hkv = 2 * Hkv, .hd = Dpad }`. The combined head +axis interleaves `K0, V0, K1, V1, ...`. `Dpad` must equal the logical query head +dimension rounded up to a multiple of 128, with unused cache features zeroed. +Queries are padded before execution and outputs trimmed afterward. The default +scale uses the logical head dimension. Accepted query/cache dtypes are `bf16`, +`f16`, and `f32`. + +Causal and non-causal standard masks and left sliding windows are supported. +Explicit key indices are rejected. `opts.sink` is currently ignored. Split +caches are rejected by ordinary paged attention, and the generic +`KvCache.update` method does not implement fused-cache writes. + +The wrapper partitions heads on the model mesh and replicates sequence metadata. +It counts sequences with nonempty query intervals and passes that count to the +ragged kernel, so active sequences must precede trailing inactive entries. +Each local head computation must have the full required key range and features. + +The shared split-cache comparison test does not exercise this backend. Wrapper +unit tests cover head-dimension alignment and configuration, including scale +and standard mask options; they do not establish runtime support for every +shape or dtype. diff --git a/zml/kernels/paged_attention/stablehlo/README.md b/zml/kernels/paged_attention/stablehlo/README.md new file mode 100644 index 00000000..34144345 --- /dev/null +++ b/zml/kernels/paged_attention/stablehlo/README.md @@ -0,0 +1,30 @@ +# StableHLO paged attention + +This fallback is automatically selected for CPU and Furiosa. See the +[parent README](../README.md) for the general attention equation, packed +sequence metadata, and page addressing contract. + +[The implementation](stablehlo.zig) iterates over sequences, query chunks, and +KV pages using StableHLO control flow. It merges partial softmax states with +`f32` maxima, exponential sums, and weighted-value accumulators, then converts +the final output to the query dtype. Query-key dot products use the input dtype. + +## Current limitations + +- Only split KV caches work. Although public dispatch forwards `.dense`, this + implementation accesses `.split`. Latent caches are rejected by dispatch. +- Attention is always causal: `causal = false` is currently ignored. Sliding + windows and explicit key indices are rejected. `opts.sink` is ignored; custom + scale is supported. +- Multi-query chunks are processed a full page at a time, and the packed output + cursor advances by the page size. Use page-multiple query chunks; arbitrary + partial query chunks are not covered by this implementation. Single-query + decode uses a separate one-row path. +- The sequence loop stops at the first empty query interval. Active sequences + must precede trailing inactive entries. + +The shared tests in [paged_attention.zig](../paged_attention.zig) use this path +as the causal comparison reference with `bf16`, page size 16, mixed +prefill/decode, shared prefix pages, partial final KV pages, and trailing +inactive sequences. Those cases do not establish support for every shape or +dtype, or conformance for the unsupported options above. diff --git a/zml/kernels/paged_attention/triton/README.md b/zml/kernels/paged_attention/triton/README.md new file mode 100644 index 00000000..aca6b52e --- /dev/null +++ b/zml/kernels/paged_attention/triton/README.md @@ -0,0 +1,28 @@ +# Triton paged attention + +This backend is automatically selected for CUDA, ROCm, and oneAPI. See the +[parent README](../README.md) for the attention equation, packed sequence +metadata, and page addressing contract. + +[Triton's wrapper](triton.zig) uses split KV caches and supports causal and +non-causal attention, a left sliding window, explicit key indices, and softmax +sinks. Query heads are grouped by KV head. Keep complete GQA groups together +when sharding `.hkv`, with matching query and cache ownership and complete +`.hg` and `.hd` axes. Each local computation needs its full key range and the +matching sequence metadata. + +The wrapper chooses between a 2D kernel and a 3D split-key kernel followed by +segment reduction. Prefill uses the 2D path; all-decode batches select a path +based on batch size and KV head count. oneAPI forces split-key execution for +all-decode batches and has a specialized path for standard masks. Segment +reduction combines softmax maxima, exponential sums, and weighted values to +preserve one normalization over the key range. + +Set `is_prefill = true` for mixed prefill/decode batches. `max_seqlen_q` guides +kernel configuration. These options do not change logical query positions. + +The shared tests in [paged_attention.zig](../paged_attention.zig) exercise +`bf16`, mixed prefill/decode, shared prefix pages, partial final KV pages, +trailing inactive sequences, and option-specific comparisons. Kernel selection +and numerical behavior depend on the target; these cases do not establish +support for every shape or dtype. From 00a0af4a55201a9dc888f29d93d2d4b18079691c Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 13 Oct 2026 17:11:38 +0200 Subject: [PATCH 779/881] zig 0.17 (second attempt) (#923) Bump for zig 0.17 Deps: - uucode bumped to https://github.com/jacobsandlund/uucode/commit/1fb73433bba5d93366c57f23ff2e9d7939746500 - zigimg bumped to https://github.com/zigimg/zigimg/commit/734fa919fb15b7917d4eb124f53600348f4ffa30 - libvaxis bumped to https://github.com/rockorager/libvaxis/commit/14b7684272234a3d66316c0d1bbdc876d4e8997c - zio bumped to: https://github.com/lalinsky/zio/commit/417757935ef6d0f0a2396d4023a44ca82ac1901c - rules_zig upgraded to [version 0.17](https://github.com/hermeticbuild/rules_zig/releases/tag/v0.17.0) - zls is using a commit from master, but they don't have yet a tagged release so Codex did a bit of packaging. I confirmed it correctly autocompletes on my machine --- .bazelrc | 3 +- .github/workflows/ci.yaml | 2 +- MODULE.bazel | 19 +- MODULE.bazel.lock | 210 +- bazel/bazel.zig | 2 +- bazel/zig_index.json | 223 + bin/zml-smi/api/client.zig | 16 +- bin/zml-smi/csv/csv.zig | 15 +- bin/zml-smi/main.zig | 4 +- bin/zml-smi/platforms/neuron/nrt.zig | 12 +- bin/zml-smi/platforms/oneapi/monitor.zig | 2 +- bin/zml-smi/platforms/oneapi/oneapi.zig | 18 +- bin/zml-smi/prometheus/exposition.zig | 6 +- bin/zml-smi/tui/image_cache.zig | 2 +- bin/zml-smi/tui/print.zig | 13 +- bin/zml-smi/tui/top.zig | 5 +- bin/zml-smi/utils/dynlib.zig | 4 +- bin/zml-smi/utils/sandbox.zig | 2 +- kernels/common/args.zig | 6 +- kernels/common/control_flow.zig | 18 +- kernels/cuda_tile/builder.zig | 146 +- kernels/cuda_tile/control_flow.zig | 8 +- kernels/cuda_tile/dtype.zig | 4 +- kernels/cute/builder.zig | 56 +- kernels/cute/dtype.zig | 4 +- kernels/fly/builder.zig | 47 +- kernels/fly/dtype.zig | 6 +- kernels/fly/layout.zig | 8 +- kernels/mosaic_tpu/builder.zig | 64 +- kernels/mosaic_tpu/dtype.zig | 4 +- kernels/triton/builder.zig | 96 +- kernels/triton/dtype.zig | 4 +- mlir/dialects/arith.zig | 8 +- mlir/dialects/cuda_tile/cuda_tile.zig | 18 +- mlir/dialects/cute_ir/cuda.zig | 12 +- mlir/dialects/cute_ir/cute_ir.zig | 47 +- mlir/dialects/cute_ir/cute_nvgpu.zig | 442 +- mlir/dialects/cute_ir/nvvm.zig | 381 +- mlir/dialects/fly/attributes.zig | 6 +- mlir/dialects/fly/fly.zig | 2 +- mlir/dialects/gpu.zig | 20 +- mlir/dialects/llvm_dialect.zig | 2 +- mlir/dialects/memref.zig | 2 +- mlir/dialects/mosaic_tpu/mosaic_tpu.zig | 42 +- mlir/dialects/stablehlo/stablehlo.zig | 4 +- mlir/dialects/ttir/ttir.zig | 24 +- mlir/dialects/vector.zig | 20 +- mlir/mlir.zig | 10 +- pjrt/ffi.zig | 4 +- pjrt/pjrt.zig | 50 +- platforms/cuda/compat_probe.zig | 2 +- platforms/cuda/cuda.zig | 4 +- platforms/cuda/flashattn/flashattn.zig | 2 +- platforms/neuron/libzml_neuron.zig | 9 +- platforms/neuron/neuron.zig | 8 +- platforms/neuron/python_launcher.zig | 6 +- platforms/platforms.zig | 4 +- stdx/flags.zig | 197 +- stdx/fmt.zig | 6 +- stdx/json.zig | 50 +- stdx/meta.zig | 30 +- stdx/segmented_list.zig | 2 +- stdx/stdx.zig | 61 +- third_party/arocc/arocc.bazel | 12 +- third_party/arocc/repo.bzl | 5 +- third_party/libvaxis/bump-zig.patch | 2019 -------- third_party/libvaxis/fixes.patch | 263 +- third_party/libvaxis/repo.bzl | 4 +- third_party/non_module_deps.bzl | 2 + third_party/translate-c/repo.bzl | 5 +- third_party/translate-c/translate-c.bazel | 9 + third_party/uucode/repo.bzl | 3 +- third_party/uucode/uucode.bazel | 2 + third_party/zigimg/bump-zig.patch | 4491 ----------------- third_party/zigimg/repo.bzl | 27 +- third_party/zio/repo.bzl | 2 +- third_party/zls/BUILD.bazel | 36 + third_party/zls/build_options.zig | 5 + third_party/zls/diffz.bazel | 8 + third_party/zls/exe_options.zig | 3 + third_party/zls/known_folders.bazel | 8 + third_party/zls/lsp.bazel | 51 + third_party/zls/repo.bzl | 28 + third_party/zls/tracy_options.zig | 3 + third_party/zls/zls.bazel | 53 + vfs/BUILD.bazel | 9 +- vfs/base.zig | 52 +- vfs/file.zig | 115 +- vfs/gcs.zig | 6 +- vfs/hf.zig | 4 +- vfs/http.zig | 4 +- vfs/s3.zig | 6 +- vfs/vfs.zig | 90 +- zml/Compiler.zig | 4 +- zml/Sharding.zig | 74 +- zml/buffer/HostAccessible.zig | 2 +- zml/dtype.zig | 7 +- zml/exe.zig | 49 +- zml/floats.zig | 14 +- zml/kernel.zig | 8 +- .../flash_attention/cute/attention.zig | 4 +- zml/mem.zig | 6 +- zml/meta.zig | 204 +- zml/moe/cute_kernels/persistent_mxfp4.zig | 4 +- zml/moe/cutlass_flashinfer.zig | 4 +- zml/moe/triton_kernels/mxfp4.zig | 4 +- zml/nn.zig | 11 +- zml/ops.zig | 114 +- zml/pjrtx.zig | 8 +- zml/platform.zig | 4 +- zml/profiling/profiler.zig | 2 +- zml/profiling/tracer.zig | 62 +- zml/quantization.zig | 2 +- zml/shape.zig | 72 +- zml/slice.zig | 10 +- zml/tensor.zig | 14 +- zml/tokenizer/iree/tokenizer.zig | 8 +- 117 files changed, 2232 insertions(+), 8288 deletions(-) delete mode 100644 third_party/libvaxis/bump-zig.patch delete mode 100644 third_party/zigimg/bump-zig.patch create mode 100644 third_party/zls/BUILD.bazel create mode 100644 third_party/zls/build_options.zig create mode 100644 third_party/zls/diffz.bazel create mode 100644 third_party/zls/exe_options.zig create mode 100644 third_party/zls/known_folders.bazel create mode 100644 third_party/zls/lsp.bazel create mode 100644 third_party/zls/repo.bzl create mode 100644 third_party/zls/tracy_options.zig create mode 100644 third_party/zls/zls.bazel diff --git a/.bazelrc b/.bazelrc index 81972716..1b47d230 100644 --- a/.bazelrc +++ b/.bazelrc @@ -61,8 +61,6 @@ common --incompatible_disallow_empty_glob=false # Use the C linker to link Zig code common --@rules_zig//zig/settings:use_cc_common_link=true -# Zig self hosted backend is too fragile for now -common --@rules_zig//zig/settings:zigopt=-fllvm common --@rules_zig//zig/settings:use_standalone_translate_c=True @@ -109,6 +107,7 @@ common:release --compilation_mode=opt common:release --copt=-Ofast common:release --strip=always common:release --@rules_zig//zig/settings:mode=release_safe +common:release --@rules_zig//zig/settings:zigopt=-fllvm # --config=native builds everything optimized for the local CPU common:native --copt=-march=native diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c996de04..13d153b3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -118,7 +118,7 @@ jobs: - name: Setup Zig uses: mlugg/setup-zig@v2.2.1 with: - version: 0.16.0 + version: 0.17.0 mirror: https://mirror.zml.ai/zig - name: Zig Format diff --git a/MODULE.bazel b/MODULE.bazel index 900d84af..c0cddf52 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -49,7 +49,8 @@ bazel_dep(name = "rules_shell", version = "0.6.1") # repo_name = None prevents rules_swift from eagerly auto-registering toolchains. bazel_dep(name = "rules_swift", version = "3.6.1", repo_name = None) -bazel_dep(name = "rules_zig", version = "0.15.1") +bazel_dep(name = "rules_zig", version = "0.17.0") + bazel_dep(name = "tar.bzl", version = "0.10.4") bazel_dep(name = "llvm", version = "0.8.24") bazel_dep(name = "libxml2", version = "2.15.3") @@ -99,7 +100,7 @@ zig.index(file = "//bazel:zig_index.json") zig.toolchain( # Ensure Zig actions are run locally only extra_exec_compatible_with = ["//platforms:local_execution"], - zig_version = "0.16.0", + zig_version = "0.17.0", ) zig.mirrors(urls = [ "https://pkg.machengine.org/zig", @@ -134,15 +135,7 @@ register_toolchains("//:translate_c_toolchain") # build graph. register_execution_platforms("//platforms:local_execution_platform") -zls = use_extension("@rules_zig//zig/zls:extensions.bzl", "zls") -zls.index(file = "@rules_zig//zig/zls/private:versions.json") -zls.toolchain( - zig_version = "0.16.0", - zls_version = "0.16.0", -) -use_repo(zls, "zls_toolchains") - -register_toolchains("@zls_toolchains//:all") +register_toolchains("//third_party/zls:toolchain") ### @@ -328,6 +321,10 @@ use_repo( "xla", "zigimg", "zio", + "zls", + "zls_diffz", + "zls_known_folders", + "zls_lsp", ) xla = use_extension("//third_party/xla:xla.bzl", "xla") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index b019f47e..b4bf3b82 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -431,8 +431,8 @@ "https://bcr.bazel.build/modules/rules_swift/3.1.2/MODULE.bazel": "72c8f5cf9d26427cee6c76c8e3853eb46ce6b0412a081b2b6db6e8ad56267400", "https://bcr.bazel.build/modules/rules_swift/3.6.1/MODULE.bazel": "5a68bb05b6b8044f8a7bd7184d56150189e7eca05522ce525517c234fed50ed8", "https://bcr.bazel.build/modules/rules_swift/3.6.1/source.json": "9af2e521ffdb41961b6b9ac05293bd6519ee2fb52849a2be658a20b0221cb8db", - "https://bcr.bazel.build/modules/rules_zig/0.15.1/MODULE.bazel": "f0b813d9191c8d201e0b3ecbe5c5220ddedb717a15009fc711a78b3e1db8710d", - "https://bcr.bazel.build/modules/rules_zig/0.15.1/source.json": "0bd91d7459f504196dfdc90cd0fffbf1f0df2667c9cd5eb708feb3fd27578e6a", + "https://bcr.bazel.build/modules/rules_zig/0.17.0/MODULE.bazel": "2c1c08adc986cfdb5fee97e21982cac846f8a20a6a8054ae9cab8b02721c2011", + "https://bcr.bazel.build/modules/rules_zig/0.17.0/source.json": "488f71f57c596ed9b7635ebeb28066b23c6fc1802c30749d06bdad261d7b04a2", "https://bcr.bazel.build/modules/stardoc/0.5.0/MODULE.bazel": "f9f1f46ba8d9c3362648eea571c6f9100680efc44913618811b58cc9c02cd678", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", @@ -1516,19 +1516,19 @@ }, "@@rules_zig+//zig:extensions.bzl%zig": { "general": { - "bzlTransitiveDigest": "VGLOg2kVw+STO9R5OrQnLFbv7Vl5TPdtIebIs324IpU=", - "usagesDigest": "4xBoe2eIScx2QI0jVieHWXdflezUUDIApwMqS+SWGbw=", + "bzlTransitiveDigest": "VY2CKW7wm6B3wCpZq7W6z3hCnHiiJlXtB4JpuB3OPD8=", + "usagesDigest": "FDLiNUluzC0WBatQ/5WFpxzyQP4ubA/DUgTwOzYLGw4=", "recordedInputs": [ "REPO_MAPPING:rules_zig+,bazel_skylib bazel_skylib+", "REPO_MAPPING:rules_zig+,bazel_tools bazel_tools", - "FILE:@@//bazel/zig_index.json 2e3b4b83f1add5d3ca12fd05797c04c38af21d86c7fc7c1a9b356560141a4797", - "FILE:@@rules_zig+//zig/private/versions.json 3e4e370dee9c3f3fdc8bfc7a600e76743278fde40815eead5d0bcda3aec3e2bd" + "FILE:@@//bazel/zig_index.json 387d1ab130896456fb092fb08bfaf0c57ccba854121955c752052600d1643d37", + "FILE:@@rules_zig+//zig/private/versions.json a33370258a702099b35362d4220fe8f02c869c2b845abcbf0aa78200abab097a" ], "generatedRepoSpecs": { - "zig_0.16.0_aarch64-linux": { + "zig_0.17.0_aarch64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.16.0/zig-aarch64-linux-0.16.0.tar.xz", + "url": "https://ziglang.org/download/0.17.0/zig-aarch64-linux-0.17.0.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1558,15 +1558,15 @@ "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds" ], - "sha256": "ea4b09bfb22ec6f6c6ceac57ab63efb6b46e17ab08d21f69f3a48b38e1534f17", - "zig_version": "0.16.0", + "sha256": "9e8d11661d4ae3bd57702a3832781e23ad151dde5798e16a5ccd503f65234ff8", + "zig_version": "0.17.0", "platform": "aarch64-linux" } }, - "zig_0.16.0_aarch64-macos": { + "zig_0.17.0_aarch64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.16.0/zig-aarch64-macos-0.16.0.tar.xz", + "url": "https://ziglang.org/download/0.17.0/zig-aarch64-macos-0.17.0.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1596,15 +1596,15 @@ "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds" ], - "sha256": "b23d70deaa879b5c2d486ed3316f7eaa53e84acf6fc9cc747de152450d401489", - "zig_version": "0.16.0", + "sha256": "b607e9b9234790a008116ae5bdb71c6243b84b9fb42a53a9e70fde41c06c536a", + "zig_version": "0.17.0", "platform": "aarch64-macos" } }, - "zig_0.16.0_aarch64-windows": { + "zig_0.17.0_aarch64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.16.0/zig-aarch64-windows-0.16.0.zip", + "url": "https://ziglang.org/download/0.17.0/zig-aarch64-windows-0.17.0.zip", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1634,15 +1634,15 @@ "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds" ], - "sha256": "aee38316ee4111717900f45dd3130145c39289e105541d737eb8c5ed653c78ef", - "zig_version": "0.16.0", + "sha256": "0a59d91fa1cb40cf068e9b0954434ce973500c7a2ea749f1e01af62cdab52d26", + "zig_version": "0.17.0", "platform": "aarch64-windows" } }, - "zig_0.16.0_x86_64-linux": { + "zig_0.17.0_x86_64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.16.0/zig-x86_64-linux-0.16.0.tar.xz", + "url": "https://ziglang.org/download/0.17.0/zig-x86_64-linux-0.17.0.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1672,15 +1672,15 @@ "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds" ], - "sha256": "70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00", - "zig_version": "0.16.0", + "sha256": "1cbe9df9f27e6b78d14ccbca43b6703a404ef79ef1c463de901d7f088d4e2026", + "zig_version": "0.17.0", "platform": "x86_64-linux" } }, - "zig_0.16.0_x86_64-macos": { + "zig_0.17.0_x86_64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.16.0/zig-x86_64-macos-0.16.0.tar.xz", + "url": "https://ziglang.org/download/0.17.0/zig-x86_64-macos-0.17.0.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1710,15 +1710,15 @@ "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds" ], - "sha256": "0387557ed1877bc6a2e1802c8391953baddba76081876301c522f52977b52ba7", - "zig_version": "0.16.0", + "sha256": "4f9a1c5269aa17ebda5e6d3c2b89d6cbf36f7d2b22a0306e9ab98f25f95529c6", + "zig_version": "0.17.0", "platform": "x86_64-macos" } }, - "zig_0.16.0_x86_64-windows": { + "zig_0.17.0_x86_64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.16.0/zig-x86_64-windows-0.16.0.zip", + "url": "https://ziglang.org/download/0.17.0/zig-x86_64-windows-0.17.0.zip", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1748,8 +1748,8 @@ "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds" ], - "sha256": "68659eb5f1e4eb1437a722f1dd889c5a322c9954607f5edcf337bc3684a75a7e", - "zig_version": "0.16.0", + "sha256": "b5663f69581dcf391293fbf16c06cb80d81d806545ce618b4d0bab7f0eb8c428", + "zig_version": "0.17.0", "platform": "x86_64-windows" } }, @@ -1757,28 +1757,28 @@ "repoRuleId": "@@rules_zig+//zig/private/repo:toolchains_repo.bzl%toolchains_repo", "attributes": { "names": [ - "zig_0.16.0_aarch64-linux", - "zig_0.16.0_aarch64-macos", - "zig_0.16.0_aarch64-windows", - "zig_0.16.0_x86_64-linux", - "zig_0.16.0_x86_64-macos", - "zig_0.16.0_x86_64-windows" + "zig_0.17.0_aarch64-linux", + "zig_0.17.0_aarch64-macos", + "zig_0.17.0_aarch64-windows", + "zig_0.17.0_x86_64-linux", + "zig_0.17.0_x86_64-macos", + "zig_0.17.0_x86_64-windows" ], "labels": [ - "@zig_0.16.0_aarch64-linux//:zig_toolchain", - "@zig_0.16.0_aarch64-macos//:zig_toolchain", - "@zig_0.16.0_aarch64-windows//:zig_toolchain", - "@zig_0.16.0_x86_64-linux//:zig_toolchain", - "@zig_0.16.0_x86_64-macos//:zig_toolchain", - "@zig_0.16.0_x86_64-windows//:zig_toolchain" + "@zig_0.17.0_aarch64-linux//:zig_toolchain", + "@zig_0.17.0_aarch64-macos//:zig_toolchain", + "@zig_0.17.0_aarch64-windows//:zig_toolchain", + "@zig_0.17.0_x86_64-linux//:zig_toolchain", + "@zig_0.17.0_x86_64-macos//:zig_toolchain", + "@zig_0.17.0_x86_64-windows//:zig_toolchain" ], "zig_versions": [ - "0.16.0", - "0.16.0", - "0.16.0", - "0.16.0", - "0.16.0", - "0.16.0" + "0.17.0", + "0.17.0", + "0.17.0", + "0.17.0", + "0.17.0", + "0.17.0" ], "exec_lengths": [ 3, @@ -1834,123 +1834,21 @@ "@@rules_zig+//zig/zls:extensions.bzl%zls": { "general": { "bzlTransitiveDigest": "ZuceGxE17jROQ+kQkaPqIcK7gMKGb56I97Aq1/Dubdc=", - "usagesDigest": "b/ERjMTWazfa8VFWuTc0PTtQQzLDuloWSHlzXj6Zb5w=", + "usagesDigest": "kOrP52crJGijpHNzo4S7msXvbGpfmzvkE4tuhpRWTsI=", "recordedInputs": [ - "REPO_MAPPING:,rules_zig rules_zig+", "REPO_MAPPING:rules_zig+,bazel_skylib bazel_skylib+", "REPO_MAPPING:rules_zig+,bazel_tools bazel_tools", "FILE:@@rules_zig+//zig/zls/private/versions.json 127803289dfd79b5255db5f74be1d9a2f790f03292d6147009af702d888aba69" ], "generatedRepoSpecs": { - "zls_0.16.0_aarch64-linux": { - "repoRuleId": "@@rules_zig+//zig/zls/private/repo:zls_repository.bzl%zls_repository", - "attributes": { - "url": "https://builds.zigtools.org/zls-aarch64-linux-0.16.0.tar.xz", - "mirrors": [], - "sha256": "430cd293d201eb70ae2519dbc96c854bf8791b8df7fc9392e8d2dc9680a2bed7", - "zls_version": "0.16.0", - "platform": "aarch64-linux" - } - }, - "zls_0.16.0_aarch64-macos": { - "repoRuleId": "@@rules_zig+//zig/zls/private/repo:zls_repository.bzl%zls_repository", - "attributes": { - "url": "https://builds.zigtools.org/zls-aarch64-macos-0.16.0.tar.xz", - "mirrors": [], - "sha256": "b93ec549f8558a7e85984a840e9276d274f1059b54ade4254296ef4982958359", - "zls_version": "0.16.0", - "platform": "aarch64-macos" - } - }, - "zls_0.16.0_aarch64-windows": { - "repoRuleId": "@@rules_zig+//zig/zls/private/repo:zls_repository.bzl%zls_repository", - "attributes": { - "url": "https://builds.zigtools.org/zls-aarch64-windows-0.16.0.zip", - "mirrors": [], - "sha256": "ef4c5ccb93c80c9f023105c5f558ae8774ac6668d560ba6f92a2f87d95df2311", - "zls_version": "0.16.0", - "platform": "aarch64-windows" - } - }, - "zls_0.16.0_x86_64-linux": { - "repoRuleId": "@@rules_zig+//zig/zls/private/repo:zls_repository.bzl%zls_repository", - "attributes": { - "url": "https://builds.zigtools.org/zls-x86_64-linux-0.16.0.tar.xz", - "mirrors": [], - "sha256": "ded6d562a0b86ee878b1ddf70ffab2797ce3cdca3b02d6077548f9d56dff96b6", - "zls_version": "0.16.0", - "platform": "x86_64-linux" - } - }, - "zls_0.16.0_x86_64-macos": { - "repoRuleId": "@@rules_zig+//zig/zls/private/repo:zls_repository.bzl%zls_repository", - "attributes": { - "url": "https://builds.zigtools.org/zls-x86_64-macos-0.16.0.tar.xz", - "mirrors": [], - "sha256": "49f716ea96c1aadaecaa5d9c0a50874cbcf443dc42b825f1e7ee35499ad3eb96", - "zls_version": "0.16.0", - "platform": "x86_64-macos" - } - }, - "zls_0.16.0_x86_64-windows": { - "repoRuleId": "@@rules_zig+//zig/zls/private/repo:zls_repository.bzl%zls_repository", - "attributes": { - "url": "https://builds.zigtools.org/zls-x86_64-windows-0.16.0.zip", - "mirrors": [], - "sha256": "35cbb7163224e8cf92d21099c1b1391f2aba927f25d389f021b13a21d40b96dd", - "zls_version": "0.16.0", - "platform": "x86_64-windows" - } - }, "zls_toolchains": { "repoRuleId": "@@rules_zig+//zig/zls/private/repo:toolchains_repo.bzl%toolchains_repo", "attributes": { - "names": [ - "0.16.0_for_0.16.0_aarch64-linux", - "0.16.0_for_0.16.0_aarch64-macos", - "0.16.0_for_0.16.0_aarch64-windows", - "0.16.0_for_0.16.0_x86_64-linux", - "0.16.0_for_0.16.0_x86_64-macos", - "0.16.0_for_0.16.0_x86_64-windows" - ], - "labels": [ - "@zls_0.16.0_aarch64-linux//:zls_toolchain", - "@zls_0.16.0_aarch64-macos//:zls_toolchain", - "@zls_0.16.0_aarch64-windows//:zls_toolchain", - "@zls_0.16.0_x86_64-linux//:zls_toolchain", - "@zls_0.16.0_x86_64-macos//:zls_toolchain", - "@zls_0.16.0_x86_64-windows//:zls_toolchain" - ], - "zig_versions": [ - "0.16.0", - "0.16.0", - "0.16.0", - "0.16.0", - "0.16.0", - "0.16.0" - ], - "exec_lengths": [ - 2, - 2, - 2, - 2, - 2, - 2 - ], - "exec_constraints": [ - "@platforms//os:linux", - "@platforms//cpu:aarch64", - "@platforms//os:macos", - "@platforms//cpu:aarch64", - "@platforms//os:windows", - "@platforms//cpu:aarch64", - "@platforms//os:linux", - "@platforms//cpu:x86_64", - "@platforms//os:macos", - "@platforms//cpu:x86_64", - "@platforms//os:windows", - "@platforms//cpu:x86_64" - ] + "names": [], + "labels": [], + "zig_versions": [], + "exec_lengths": [], + "exec_constraints": [] } } } diff --git a/bazel/bazel.zig b/bazel/bazel.zig index 0cf23580..2ae3324a 100644 --- a/bazel/bazel.zig +++ b/bazel/bazel.zig @@ -12,7 +12,7 @@ const init_array_section = switch (builtin.object_format) { else => "", }; -export const _bazel_runfiles_init linksection(init_array_section) = &struct { +export const _bazel_runfiles_init align(@alignOf(usize)) linksection(init_array_section) = &struct { fn call(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { _ = argc; _ = envp; diff --git a/bazel/zig_index.json b/bazel/zig_index.json index 8b39b699..b8cf9b9e 100644 --- a/bazel/zig_index.json +++ b/bazel/zig_index.json @@ -1,4 +1,227 @@ { + "0.17.0": { + "version": "0.17.0", + "date": "2026-10-01", + "docs": "https://ziglang.org/documentation/0.17.0/", + "stdDocs": "https://ziglang.org/documentation/0.17.0/std/", + "notes": "https://ziglang.org/download/0.17.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.17.0/zig-0.17.0.tar.xz", + "shasum": "b6c7f1728f043700d6529bac980800792f824256a9d2f1839b3d62beed0b8abd", + "size": "23049960" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.17.0/zig-bootstrap-0.17.0.tar.xz", + "shasum": "1e9e9b8e3c753b35dfb1d9ea48f097579fadbd3e5c1e724ab3fc5bce05477b71", + "size": "57030388" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-macos-0.17.0.tar.xz", + "shasum": "4f9a1c5269aa17ebda5e6d3c2b89d6cbf36f7d2b22a0306e9ab98f25f95529c6", + "size": "59317572" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-macos-0.17.0.tar.xz", + "shasum": "b607e9b9234790a008116ae5bdb71c6243b84b9fb42a53a9e70fde41c06c536a", + "size": "53985220" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-linux-0.17.0.tar.xz", + "shasum": "1cbe9df9f27e6b78d14ccbca43b6703a404ef79ef1c463de901d7f088d4e2026", + "size": "57332648" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-linux-0.17.0.tar.xz", + "shasum": "9e8d11661d4ae3bd57702a3832781e23ad151dde5798e16a5ccd503f65234ff8", + "size": "52877280" + }, + "arm-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-arm-linux-0.17.0.tar.xz", + "shasum": "53f0045cdef7ba06da70a12b2ef654e4c5283b0139bc441aa7d84f364b46de43", + "size": "53841896" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-riscv64-linux-0.17.0.tar.xz", + "shasum": "18ff032ac6cecf746a84259f6463264f13160ff88bc3b80722ce06f9479b82ed", + "size": "57116976" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-powerpc64le-linux-0.17.0.tar.xz", + "shasum": "731e2887e66fff41fdc34f7fd9d8afa814d86d7426f5827805f8854b2977684b", + "size": "56990076" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86-linux-0.17.0.tar.xz", + "shasum": "55e39e175cd5b3098afc29ec47b2590134f5b874175369fb9ae7b8e836d7531a", + "size": "59823964" + }, + "loongarch64-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-loongarch64-linux-0.17.0.tar.xz", + "shasum": "5a6000b866201c3b3820b563fcfd29641185393e7d35039e50235db418d8367c", + "size": "54257324" + }, + "s390x-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-s390x-linux-0.17.0.tar.xz", + "shasum": "13905f11989b65566c544d191d4bc5bd5d6f858e303208cf1d2f090e5d91b512", + "size": "57136832" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-windows-0.17.0.zip", + "shasum": "b5663f69581dcf391293fbf16c06cb80d81d806545ce618b4d0bab7f0eb8c428", + "size": "100268137" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-windows-0.17.0.zip", + "shasum": "0a59d91fa1cb40cf068e9b0954434ce973500c7a2ea749f1e01af62cdab52d26", + "size": "95992853" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86-windows-0.17.0.zip", + "shasum": "c7b83507549c7bcd42cd58c4910b47037f4150ed7778e11b3684f130620969d1", + "size": "101891323" + }, + "aarch64-freebsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-freebsd-0.17.0.tar.xz", + "shasum": "3cf53d07b3587a26727b196a6eb7918704b907d951520daa68e4e1820fc6f48e", + "size": "52860244" + }, + "arm-freebsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-arm-freebsd-0.17.0.tar.xz", + "shasum": "e0c3ecf21c186a48d3b8e8d0b4459f9dc67ff8024445401506a7e970782da5f8", + "size": "54273908" + }, + "powerpc64le-freebsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-powerpc64le-freebsd-0.17.0.tar.xz", + "shasum": "94e127c2a4beb5151a8c58e8af4b82dc3649ed95f343ffbbfa381e2afba50234", + "size": "57027652" + }, + "riscv64-freebsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-riscv64-freebsd-0.17.0.tar.xz", + "shasum": "4ab73eb01a0cb910dfa23d8efc88d38dcdd5ebb2174537cf531738ba520354f1", + "size": "57199160" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-freebsd-0.17.0.tar.xz", + "shasum": "a314f1cc92d098bd39caf9e5607a0de2a8fd0d00064776cb3fd5b3bee38a67c9", + "size": "57468884" + }, + "aarch64-netbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-netbsd-0.17.0.tar.xz", + "shasum": "421fefe3910dedc8b8a6ae9a8905ea9200ecb0760499bcf0f56af7a5ba4930b5", + "size": "52769844" + }, + "arm-netbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-arm-netbsd-0.17.0.tar.xz", + "shasum": "2537d66f6dbd57438355d100180d09f19640c98895d4052e4e2994a02cac320a", + "size": "55345456" + }, + "riscv64-netbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-riscv64-netbsd-0.17.0.tar.xz", + "shasum": "687c92430ab3e7a5329e941e95f0ec4368d8d8ac0a4048f61c2dd714e675ca12", + "size": "57157568" + }, + "x86-netbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86-netbsd-0.17.0.tar.xz", + "shasum": "2025c508390cc78d9c469675d2a0602f2a975c92ec197af51faf09a565b27ee3", + "size": "60380332" + }, + "x86_64-netbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-netbsd-0.17.0.tar.xz", + "shasum": "10f71cc89efbafc6e22447f23c44047d53c5d2061c82b1c32efd1ef55ea2d9b6", + "size": "57363300" + }, + "aarch64-openbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-openbsd-0.17.0.tar.xz", + "shasum": "1cf6c33a0cdd9a58a0b40c7ee86d8de9c190ed7bd8d47ad61687908955ecac10", + "size": "53265892" + }, + "arm-openbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-arm-openbsd-0.17.0.tar.xz", + "shasum": "068d5200b027541c2b0a2e16efc3fb367f1fae78f9c3954e57512868c9a0ecc2", + "size": "54024240" + }, + "riscv64-openbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-riscv64-openbsd-0.17.0.tar.xz", + "shasum": "4858839a9e4d96fb50d6e4894d10270f5377a087b2929d238b1a12b8293e274a", + "size": "57490512" + }, + "x86_64-openbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-openbsd-0.17.0.tar.xz", + "shasum": "5e9606fc322ef4c346c199e162d26e4c293358a36ea3ca72e877d42ef71e24be", + "size": "58612444" + } + }, + "0.17.0-dev.1936+5a625d5f3": { + "version": "0.17.0-dev.1936+5a625d5f3", + "docs": "https://ziglang.org/documentation/master/", + "stdDocs": "https://ziglang.org/documentation/master/std/", + "x86_64-macos": { + "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.17.0-dev.1936+5a625d5f3.tar.xz", + "shasum": "8ab9f0d015abb75c0c9c039f7191711a7a2fc72c984458f553926725794791dd", + "size": "59162280" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.17.0-dev.1936+5a625d5f3.tar.xz", + "shasum": "e08d4485c96d2d31a8ce3ae80a58badec202920b675149ad6b0c3b974ec7c005", + "size": "53867184" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.17.0-dev.1936+5a625d5f3.tar.xz", + "shasum": "38620c06ca57b175f7dab8f65c8001f9a67b0ff1987c344d4e64ccb72b57174f", + "size": "57180268" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.17.0-dev.1936+5a625d5f3.tar.xz", + "shasum": "6c559ff4bca3e84fbbf7b6ad8521ce70bb3e460371b3d8803112353fb8ce70cc", + "size": "52741876" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.17.0-dev.1936+5a625d5f3.zip", + "shasum": "e0d62a00e067de41a0a34af2af73694e00692e045ca244748965a7b3729633a1", + "size": "100793891" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.17.0-dev.1936+5a625d5f3.zip", + "shasum": "e75f340afee0772fe4daed61d92b999534c8460601175b66fe890703cf752850", + "size": "96526396" + } + }, + "0.17.0-dev.1509+bb296ab9b": { + "version": "0.17.0-dev.1509+bb296ab9b", + "date": "2026-07-29", + "docs": "https://ziglang.org/documentation/master/", + "stdDocs": "https://ziglang.org/documentation/master/std/", + "x86_64-macos": { + "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.17.0-dev.1509+bb296ab9b.tar.xz", + "shasum": "087aad38fead6c0ba7a52b5db3c405e63708c4765eda643d4144b9e4933d45e5", + "size": "59064872" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.17.0-dev.1509+bb296ab9b.tar.xz", + "shasum": "24bd83c1d435b8ab6192f58afcc22f4e5252077a938ffbc35eaa4ca97c5be709", + "size": "53773824" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.17.0-dev.1509+bb296ab9b.tar.xz", + "shasum": "48cc865b8b410ec84eaa97e50c2bd7a657871802ce3aaaf04dd1da2294d4b28a", + "size": "57056988" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.17.0-dev.1509+bb296ab9b.tar.xz", + "shasum": "7f17e09b675cfe39805c5b551b251151f7750b9df8a168538007890c1c00dc8e", + "size": "52649416" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.17.0-dev.1509+bb296ab9b.zip", + "shasum": "a4f537761ca1f5274c9bb0e7da598c6e82205a4ea78f329690a26eade555d7e8", + "size": "100743514" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.17.0-dev.1509+bb296ab9b.zip", + "shasum": "1fa7a2b5f5442628287fead0f7451b8cb93ed2ca5dd6c79f8026fc8ca83580d2", + "size": "96467368" + } + }, "master": { "version": "0.16.0-dev.3153+d6f43caad", "date": "2026-04-12", diff --git a/bin/zml-smi/api/client.zig b/bin/zml-smi/api/client.zig index 17e49917..ddec6d2d 100644 --- a/bin/zml-smi/api/client.zig +++ b/bin/zml-smi/api/client.zig @@ -6,10 +6,10 @@ const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.Ar const Collector = @import("zml-smi/collector").Collector; fn setRemote(device: *DeviceInfo) void { - inline for (@typeInfo(DeviceInfo).@"union".fields) |field| { - if (device.* == @field(smi_info.device_info.Target, field.name)) { - @field(device, field.name).values[0].remote = true; - @field(device, field.name).values[1].remote = true; + inline for (@typeInfo(DeviceInfo).@"union".field_names) |field_name| { + if (device.* == @field(smi_info.device_info.Target, field_name)) { + @field(device, field_name).values[0].remote = true; + @field(device, field_name).values[1].remote = true; } } } @@ -86,10 +86,10 @@ fn pollOnce(poll_arena: *std.heap.ArenaAllocator, gpa: std.mem.Allocator, io: st var device = std.json.parseFromValueLeaky(DeviceInfo, poll_arena.allocator(), item, .{ .ignore_unknown_fields = true }) catch continue; setRemote(&device); - inline for (@typeInfo(DeviceInfo).@"union".fields) |field| { - if (devices[i].* == @field(smi_info.device_info.Target, field.name)) { - @field(devices[i], field.name).back().* = @field(device, field.name).front().*; - @field(devices[i], field.name).swap(); + inline for (@typeInfo(DeviceInfo).@"union".field_names) |field_name| { + if (devices[i].* == @field(smi_info.device_info.Target, field_name)) { + @field(devices[i], field_name).back().* = @field(device, field_name).front().*; + @field(devices[i], field_name).swap(); } } } diff --git a/bin/zml-smi/csv/csv.zig b/bin/zml-smi/csv/csv.zig index a05c5a4f..dbd80dd9 100644 --- a/bin/zml-smi/csv/csv.zig +++ b/bin/zml-smi/csv/csv.zig @@ -3,8 +3,9 @@ const smi_info = @import("zml-smi/info"); const DeviceInfo = smi_info.device_info.DeviceInfo; pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo) !void { - inline for (@typeInfo(DeviceInfo).@"union".fields) |tp| { - const tag = @field(smi_info.Target, tp.name); + const info = @typeInfo(DeviceInfo).@"union"; + inline for (info.field_names, info.field_types) |type_name, DeviceType| { + const tag = @field(smi_info.Target, type_name); var header_printed = false; for (devices, 0..) |dev, i| { @@ -12,8 +13,8 @@ pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo) !void { tag => |*db| { if (!header_printed) { try writer.writeAll("index,type"); - inline for (@typeInfo(tp.type.Value).@"struct".fields) |f| { - try writer.writeAll("," ++ f.name); + inline for (@typeInfo(DeviceType.Value).@"struct".field_names) |field_name| { + try writer.writeAll("," ++ field_name); } try writer.writeAll("\n"); @@ -22,10 +23,10 @@ pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo) !void { const val = db.front().*; - try writer.print("{d},{s}", .{ i, tp.name }); - inline for (@typeInfo(tp.type.Value).@"struct".fields) |f| { + try writer.print("{d},{s}", .{ i, type_name }); + inline for (@typeInfo(DeviceType.Value).@"struct".field_names) |field_name| { try writer.writeAll(","); - try writeValue(writer, @field(val, f.name)); + try writeValue(writer, @field(val, field_name)); } try writer.writeAll("\n"); }, diff --git a/bin/zml-smi/main.zig b/bin/zml-smi/main.zig index 47c861ff..f066dd34 100644 --- a/bin/zml-smi/main.zig +++ b/bin/zml-smi/main.zig @@ -163,9 +163,9 @@ pub fn main(init: std.process.Init) !void { try json.write(&json_writer.interface, collector.device_infos.items, procs.items); try json_writer.flush(); } else if (args.top) { - try tui.run(gpa, io, &state); + try tui.run(gpa, io, init.environ_map, &state); } else { - try static_print.run(arena, io, &state); + try static_print.run(arena, io, init.environ_map, &state); } } diff --git a/bin/zml-smi/platforms/neuron/nrt.zig b/bin/zml-smi/platforms/neuron/nrt.zig index 8fe21a27..cd77b3c0 100644 --- a/bin/zml-smi/platforms/neuron/nrt.zig +++ b/bin/zml-smi/platforms/neuron/nrt.zig @@ -151,11 +151,11 @@ pub fn hbmSize(dev: *c.ndl_device_t) usize { } pub fn deviceType(dev: *c.ndl_device_t) DeviceType { - return @enumFromInt(dev.device_type); + return @fromBackingInt(@intCast(dev.device_type)); } fn resolveFromElf(handle: *anyopaque, io: std.Io, comptime known_sym: [:0]const u8) !PrivateFns { - const fields = std.meta.fields(PrivateFns); + const field_names = comptime std.meta.fieldNames(PrivateFns); const sym_addr = std.c.dlsym(handle, known_sym) orelse return error.SymbolNotFound; @@ -181,7 +181,7 @@ fn resolveFromElf(handle: *anyopaque, io: std.Io, comptime known_sym: [:0]const defer std.posix.munmap(mmap); var result: PrivateFns = undefined; - var remaining: usize = fields.len; + var remaining: usize = field_names.len; const ehdr: *const elf.ElfN.Ehdr = @ptrCast(@alignCast(mmap)); const shdrs: []const elf.ElfN.Shdr = @as([*]const elf.ElfN.Shdr, @ptrCast(@alignCast(mmap[ehdr.shoff..])))[0..ehdr.shnum]; @@ -201,9 +201,9 @@ fn resolveFromElf(handle: *anyopaque, io: std.Io, comptime known_sym: [:0]const const sym_name = std.mem.span(@as([*c]const u8, @ptrCast(strtab[@as(usize, s.name)..]))); - inline for (fields) |field| { - if (std.mem.eql(u8, sym_name, field.name)) { - @field(result, field.name) = @ptrFromInt(elf_base + @as(usize, s.value)); + inline for (field_names) |field_name| { + if (std.mem.eql(u8, sym_name, field_name)) { + @field(result, field_name) = @ptrFromInt(elf_base + @as(usize, s.value)); remaining -= 1; } } diff --git a/bin/zml-smi/platforms/oneapi/monitor.zig b/bin/zml-smi/platforms/oneapi/monitor.zig index 503ae0df..dd5f97a9 100644 --- a/bin/zml-smi/platforms/oneapi/monitor.zig +++ b/bin/zml-smi/platforms/oneapi/monitor.zig @@ -149,7 +149,7 @@ inline fn openDevice(allocator: std.mem.Allocator, io: std.Io, render_idx: usize } pub fn device(self: *const Monitor, handle: Handle) !*const Device { - const idx = @intFromEnum(handle); + const idx = @backingInt(handle); if (idx >= self.devices.len) return error.NotFound; return &self.devices[idx]; } diff --git a/bin/zml-smi/platforms/oneapi/oneapi.zig b/bin/zml-smi/platforms/oneapi/oneapi.zig index 0e89a6a2..2e1e5deb 100644 --- a/bin/zml-smi/platforms/oneapi/oneapi.zig +++ b/bin/zml-smi/platforms/oneapi/oneapi.zig @@ -18,7 +18,7 @@ pub fn init(allocator: std.mem.Allocator, io: std.Io) !OneApi { pub fn handleByIndex(self: *const OneApi, device_id: usize) !Handle { if (device_id >= self.monitor.devices.len) return error.NotFound; - return @enumFromInt(@as(u32, @intCast(device_id))); + return @fromBackingInt(@intCast(@as(u32, @intCast(device_id)))); } pub fn name(self: *const OneApi, allocator: std.mem.Allocator, handle: Handle) ![]const u8 { @@ -111,7 +111,7 @@ pub fn driverVersion(self: *const OneApi, allocator: std.mem.Allocator) ![]const pub fn powerUsage(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { const dev = block: { - const idx = @intFromEnum(handle); + const idx = @backingInt(handle); if (idx >= self.monitor.devices.len) return error.NotFound; break :block &self.monitor.devices[idx]; }; @@ -144,7 +144,7 @@ pub fn fanSpeed(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u6 pub fn gpuUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { const dev = block: { - const idx = @intFromEnum(handle); + const idx = @backingInt(handle); if (idx >= self.monitor.devices.len) return error.NotFound; break :block &self.monitor.devices[idx]; }; @@ -155,7 +155,7 @@ pub fn gpuUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 }; defer usage.deinit(allocator); - const dev_idx: u16 = @intCast(@intFromEnum(handle)); + const dev_idx: u16 = @intCast(@backingInt(handle)); const current = if (usage.get(dev_idx)) |sample| sample.engine else null; const util = Monitor.processUtil(dev.activity_prev, current) orelse 0; dev.activity_prev = current; @@ -164,7 +164,7 @@ pub fn gpuUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 pub fn encoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { const dev = block: { - const idx = @intFromEnum(handle); + const idx = @backingInt(handle); if (idx >= self.monitor.devices.len) return error.NotFound; break :block &self.monitor.devices[idx]; }; @@ -175,7 +175,7 @@ pub fn encoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) }; defer usage.deinit(allocator); - const dev_idx: u16 = @intCast(@intFromEnum(handle)); + const dev_idx: u16 = @intCast(@backingInt(handle)); const current = if (usage.get(dev_idx)) |sample| sample.encoder else null; const util = Monitor.processUtil(dev.encoder_prev, current) orelse { dev.encoder_prev = current; @@ -187,7 +187,7 @@ pub fn encoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) pub fn decoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { const dev = block: { - const idx = @intFromEnum(handle); + const idx = @backingInt(handle); if (idx >= self.monitor.devices.len) return error.NotFound; break :block &self.monitor.devices[idx]; }; @@ -198,7 +198,7 @@ pub fn decoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) }; defer usage.deinit(allocator); - const dev_idx: u16 = @intCast(@intFromEnum(handle)); + const dev_idx: u16 = @intCast(@backingInt(handle)); const current = if (usage.get(dev_idx)) |sample| sample.decoder else null; const util = Monitor.processUtil(dev.decoder_prev, current) orelse { dev.decoder_prev = current; @@ -219,7 +219,7 @@ pub fn maxClockGraphics(self: *OneApi, allocator: std.mem.Allocator, handle: Han } pub fn memUsed(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev_idx: u16 = @intCast(@intFromEnum(handle)); + const dev_idx: u16 = @intCast(@backingInt(handle)); var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch return 0; defer usage.deinit(allocator); diff --git a/bin/zml-smi/prometheus/exposition.zig b/bin/zml-smi/prometheus/exposition.zig index c955226f..54d48cb6 100644 --- a/bin/zml-smi/prometheus/exposition.zig +++ b/bin/zml-smi/prometheus/exposition.zig @@ -48,14 +48,14 @@ pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo, host: *const try writer.writeAll("# HELP zml_device_info Device metadata\n"); try writer.writeAll("# TYPE zml_device_info gauge\n"); for (devices, 0..) |dev, i| { - inline for (@typeInfo(DeviceInfo).@"union".fields) |tp| { - const tag = @field(smi_info.device_info.Target, tp.name); + inline for (@typeInfo(DeviceInfo).@"union".field_names) |type_name| { + const tag = @field(smi_info.device_info.Target, type_name); switch (dev.*) { tag => |*db| { const val = db.front().*; - try writer.print("zml_device_info{{device=\"{d}\",type=\"{s}\"", .{ i, tp.name }); + try writer.print("zml_device_info{{device=\"{d}\",type=\"{s}\"", .{ i, type_name }); inline for (.{ .{ "name", "name" }, diff --git a/bin/zml-smi/tui/image_cache.zig b/bin/zml-smi/tui/image_cache.zig index cc841691..1aea4778 100644 --- a/bin/zml-smi/tui/image_cache.zig +++ b/bin/zml-smi/tui/image_cache.zig @@ -19,7 +19,7 @@ pub fn load( key: []const u8, data: []const u8, ) void { - const image = vx.loadImage(allocator, undefined, writer, .{ .mem = data }) catch return; + const image = vx.loadImage(allocator, writer, .{ .mem = data }) catch return; self.map.put(allocator, key, image) catch return; } diff --git a/bin/zml-smi/tui/print.zig b/bin/zml-smi/tui/print.zig index 7338d20e..238534cd 100644 --- a/bin/zml-smi/tui/print.zig +++ b/bin/zml-smi/tui/print.zig @@ -14,11 +14,12 @@ fn silentResize(vx: *vaxis.Vaxis, allocator: std.mem.Allocator, winsize: vaxis.W vx.screen_last = try vaxis.AllocatingScreen.init(allocator, winsize.cols, winsize.rows); } -pub fn run(allocator: std.mem.Allocator, io: std.Io, state: *data.SystemState) !void { - var tty = try vaxis.Tty.init(io); +pub fn run(allocator: std.mem.Allocator, io: std.Io, env_map: *std.process.Environ.Map, state: *data.SystemState) !void { + var tty_buffer: [4096]u8 = undefined; + var tty = try vaxis.Tty.init(io, &tty_buffer); defer tty.deinit(); - var vx = try vaxis.init(allocator, .{}); + var vx = try vaxis.init(io, allocator, env_map, .{}); defer { // Reset terminal modes without clearing screen content. vx.resetModes(tty.writer()) catch {}; @@ -27,12 +28,12 @@ pub fn run(allocator: std.mem.Allocator, io: std.Io, state: *data.SystemState) ! vx.deinit(allocator, &discarding.writer); } - const ws = try vaxis.Tty.getWinsize(tty.fd); + const ws = try tty.getWinsize(); try silentResize(&vx, allocator, ws); { const EventLoop = vaxis.Loop(vxfw.Event); - var loop: EventLoop = .{ .tty = &tty, .vaxis = &vx, .io = io, .queue = .{ .io = io } }; + var loop: EventLoop = .init(io, &tty, &vx); try loop.start(); defer loop.stop(); @@ -41,7 +42,7 @@ pub fn run(allocator: std.mem.Allocator, io: std.Io, state: *data.SystemState) ! const writer = tty.writer(); try writer.writeAll("\x1b7"); // DECSC: save cursor try writer.flush(); - try vx.queryTerminal(writer, io, 1 * std.time.ns_per_s); + try vx.queryTerminal(writer, .fromSeconds(1)); try writer.writeAll("\x1b8"); // DECRC: restore cursor try writer.flush(); } diff --git a/bin/zml-smi/tui/top.zig b/bin/zml-smi/tui/top.zig index 9a5b8624..74b3796b 100644 --- a/bin/zml-smi/tui/top.zig +++ b/bin/zml-smi/tui/top.zig @@ -151,8 +151,9 @@ const Model = struct { } }; -pub fn run(allocator: std.mem.Allocator, io: std.Io, state: *data.SystemState) !void { - var app = try vxfw.App.init(allocator, io); +pub fn run(allocator: std.mem.Allocator, io: std.Io, env_map: *std.process.Environ.Map, state: *data.SystemState) !void { + var tty_buffer: [4096]u8 = undefined; + var app = try vxfw.App.init(io, allocator, env_map, &tty_buffer); defer app.deinit(); var model: Model = .{ diff --git a/bin/zml-smi/utils/dynlib.zig b/bin/zml-smi/utils/dynlib.zig index 730b495d..4a478107 100644 --- a/bin/zml-smi/utils/dynlib.zig +++ b/bin/zml-smi/utils/dynlib.zig @@ -2,8 +2,8 @@ const std = @import("std"); pub fn lookupStruct(dynlib: *std.DynLib, comptime VTable: type) !VTable { var result: VTable = undefined; - inline for (std.meta.fields(VTable)) |field| { - @field(result, field.name) = dynlib.lookup(field.type, field.name) orelse return error.SymbolResolutionFailed; + inline for (comptime std.meta.fieldNames(VTable), comptime std.meta.fieldTypes(VTable)) |field_name, Field| { + @field(result, field_name) = dynlib.lookup(Field, field_name) orelse return error.SymbolResolutionFailed; } return result; diff --git a/bin/zml-smi/utils/sandbox.zig b/bin/zml-smi/utils/sandbox.zig index 1b53c9ea..d4e268bd 100644 --- a/bin/zml-smi/utils/sandbox.zig +++ b/bin/zml-smi/utils/sandbox.zig @@ -11,7 +11,7 @@ const init_array_section = switch (builtin.object_format) { else => "", }; -export const _ linksection(init_array_section) = &struct { +export const _ align(@alignOf(usize)) linksection(init_array_section) = &struct { fn call(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { _ = argc; _ = argv; diff --git a/kernels/common/args.zig b/kernels/common/args.zig index 6d89de34..84d57d49 100644 --- a/kernels/common/args.zig +++ b/kernels/common/args.zig @@ -1,6 +1,4 @@ pub fn NamedArgs(comptime Spec: type, comptime ValueT: type) type { - const in = @typeInfo(Spec).@"struct".fields; - comptime var names: [in.len][]const u8 = undefined; - for (in, 0..) |f, i| names[i] = f.name; - return @Struct(.auto, null, &names, &@splat(ValueT), &@splat(.{})); + const field_names = @typeInfo(Spec).@"struct".field_names; + return @Struct(.auto, null, field_names, &@splat(ValueT), &@splat(.{})); } diff --git a/kernels/common/control_flow.zig b/kernels/common/control_flow.zig index 252184f3..afd79e8f 100644 --- a/kernels/common/control_flow.zig +++ b/kernels/common/control_flow.zig @@ -13,7 +13,7 @@ pub fn tupleArity(comptime T: type, comptime what: []const u8) comptime_int { const info = @typeInfo(T); if (info != .@"struct" or !info.@"struct".is_tuple) @compileError(what ++ " must be a tuple literal like `.{ v1, v2 }`"); - return info.@"struct".fields.len; + return info.@"struct".field_names.len; } pub fn emitScfYield( @@ -27,13 +27,13 @@ pub fn emitScfYield( const info = @typeInfo(@TypeOf(values)); if (info != .@"struct" or !info.@"struct".is_tuple) @compileError(what ++ " expects a tuple literal"); - if (info.@"struct".fields.len != N) + if (info.@"struct".field_names.len != N) @compileError(what ++ ": yield arity must match the scope's declared arity"); var buf: [N]*const mlir.Value = undefined; - inline for (info.@"struct".fields, 0..) |f, i| { - if (f.type != ValueT) + inline for (info.@"struct".field_names, info.@"struct".field_types, 0..) |field_name, Field, i| { + if (Field != ValueT) @compileError(what ++ ": every tuple element must be a Value"); - buf[i] = @field(values, f.name).inner; + buf[i] = @field(values, field_name).inner; } _ = scf.yield(kernel.ctx, &buf, kernel.loc()).appendTo(block); } @@ -151,14 +151,14 @@ pub fn WhileScope(comptime BuilderT: type, comptime ValueT: type, comptime N: us const info = @typeInfo(@TypeOf(forwarded)); if (info != .@"struct" or !info.@"struct".is_tuple) @compileError("WhileScope.yieldBefore: forwarded must be a tuple literal"); - if (info.@"struct".fields.len != M) + if (info.@"struct".field_names.len != M) @compileError("WhileScope.yieldBefore: forwarded arity must match after_types arity"); const k = self.kernel; var buf: [M]*const mlir.Value = undefined; - inline for (info.@"struct".fields, 0..) |f, i| { - if (f.type != ValueT) + inline for (info.@"struct".field_names, info.@"struct".field_types, 0..) |field_name, Field, i| { + if (Field != ValueT) @compileError("WhileScope.yieldBefore: every forwarded element must be a Value"); - buf[i] = @field(forwarded, f.name).inner; + buf[i] = @field(forwarded, field_name).inner; } _ = scf.condition(k.ctx, cond.inner, &buf, k.loc()).appendTo(self.before_block); k.popBlock(); diff --git a/kernels/cuda_tile/builder.zig b/kernels/cuda_tile/builder.zig index d73742a1..6067ed4b 100644 --- a/kernels/cuda_tile/builder.zig +++ b/kernels/cuda_tile/builder.zig @@ -84,8 +84,8 @@ pub const Value = struct { pub fn isFloatElem(self: Value) bool { const et = self.elemType(); - inline for (std.meta.fields(mlir.FloatTypes)) |f| { - if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, f.name))) != null) return true; + inline for (comptime std.meta.fieldNames(mlir.FloatTypes)) |f_name| { + if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, f_name))) != null) return true; } return false; } @@ -509,13 +509,13 @@ pub const Builder = struct { pub fn declareArgsOpts(self: *Builder, spec: anytype, opts: Opts) !dsl.NamedArgs(@TypeOf(spec), Value) { const Spec = @TypeOf(spec); - const fields = @typeInfo(Spec).@"struct".fields; + const field_names = @typeInfo(Spec).@"struct".field_names; - var arg_specs: [fields.len]ArgSpec = undefined; - inline for (fields, 0..) |f, i| { - const raw = @field(spec, f.name); + var arg_specs: [field_names.len]ArgSpec = undefined; + inline for (0.., field_names) |i, field_name| { + const raw = @field(spec, field_name); const kind: ArgSpec.Kind = if (@TypeOf(raw) == ArgSpec.Kind) raw else blk: { - const variant = @typeInfo(@TypeOf(raw)).@"struct".fields[0].name; + const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; const tag = @field(std.meta.Tag(ArgSpec.Kind), variant); const inner = @field(raw, variant); break :blk switch (tag) { @@ -526,14 +526,14 @@ pub const Builder = struct { } }, }; }; - arg_specs[i] = .{ .name = f.name, .kind = kind }; + arg_specs[i] = .{ .name = field_name, .kind = kind }; } try self.declareArgsLowOpts(&arg_specs, opts); var named: dsl.NamedArgs(Spec, Value) = undefined; - inline for (fields, 0..) |f, i| { - @field(named, f.name) = self.arg(i); + inline for (0.., field_names) |i, field_name| { + @field(named, field_name) = self.arg(i); } return named; } @@ -643,12 +643,12 @@ pub const Builder = struct { const info = @typeInfo(T); if (info != .@"struct" or !info.@"struct".is_tuple) @compileError("Builder.yield expects a tuple literal like `.{ v1, v2 }`"); - const n = info.@"struct".fields.len; + const n = info.@"struct".field_names.len; const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.yield OOM"); - inline for (info.@"struct".fields, 0..) |f, i| { - if (f.type != Value) - @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(f.type)); - out[i] = @field(values, f.name); + inline for (info.@"struct".field_names, 0..) |f, i| { + if (@FieldType(T, f) != Value) + @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(@FieldType(T, f))); + out[i] = @field(values, f); } return out; } @@ -1633,7 +1633,7 @@ pub const Builder = struct { inits: anytype, ) ForScope(tupleArity(@TypeOf(inits), "openFor: inits")) { const N = comptime tupleArity(@TypeOf(inits), "openFor: inits"); - const fields = @typeInfo(@TypeOf(inits)).@"struct".fields; + const fields = @typeInfo(@TypeOf(inits)).@"struct".field_names; // Literals take the type of the first bound that is a Value; all // three must agree (`AllTypesMatch` on the op). @@ -1651,8 +1651,8 @@ pub const Builder = struct { block_locs[0] = self.loc(); var inits_inner: [N]*const mlir.Value = undefined; inline for (fields, 0..) |f, i| { - const raw = @field(inits, f.name); - const v: Value = if (f.type == Value) raw else self.lift(raw); + const raw = @field(inits, f); + const v: Value = if (@FieldType(@TypeOf(inits), f) == Value) raw else self.lift(raw); block_types[i + 1] = v.type_(); block_locs[i + 1] = self.loc(); inits_inner[i] = v.inner; @@ -1692,13 +1692,13 @@ pub const Builder = struct { result_types: anytype, ) IfScope(tupleArity(@TypeOf(result_types), "openIfElse: result_types")) { const N = comptime tupleArity(@TypeOf(result_types), "openIfElse: result_types"); - const fields = @typeInfo(@TypeOf(result_types)).@"struct".fields; + const fields = @typeInfo(@TypeOf(result_types)).@"struct".field_names; var types: [N]*const mlir.Type = undefined; inline for (fields, 0..) |f, i| { - if (f.type != *const mlir.Type) + if (@FieldType(@TypeOf(result_types), f) != *const mlir.Type) @compileError("openIfElse: every result_type must be *const mlir.Type (use b.scalarTy/tileTy)"); - types[i] = @field(result_types, f.name); + types[i] = @field(result_types, f); } const then_block = mlir.Block.init(&.{}, &.{}); @@ -1718,14 +1718,14 @@ pub const Builder = struct { /// inside an `if`, and close the body with `scope.yield`. pub fn openLoop(self: *Builder, inits: anytype) LoopScope(tupleArity(@TypeOf(inits), "openLoop: inits")) { const N = comptime tupleArity(@TypeOf(inits), "openLoop: inits"); - const fields = @typeInfo(@TypeOf(inits)).@"struct".fields; + const fields = @typeInfo(@TypeOf(inits)).@"struct".field_names; var block_types: [N]*const mlir.Type = undefined; var block_locs: [N]*const mlir.Location = undefined; var inits_inner: [N]*const mlir.Value = undefined; inline for (fields, 0..) |f, i| { - const raw = @field(inits, f.name); - const v: Value = if (f.type == Value) raw else self.lift(raw); + const raw = @field(inits, f); + const v: Value = if (@FieldType(@TypeOf(inits), f) == Value) raw else self.lift(raw); block_types[i] = v.type_(); block_locs[i] = self.loc(); inits_inner[i] = v.inner; @@ -1769,7 +1769,7 @@ pub const Builder = struct { try al.writer.print("{f}", .{self.module.operation()}); - return try self.allocator.dupeZ(u8, al.written()); + return try self.allocator.dupeSentinel(u8, al.written(), 0); } }; @@ -1799,12 +1799,12 @@ test "empty entry round-trips" { const ir = try b.finish(&.{}); defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.module @empty") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "entry @empty(") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "assume") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "div_by<16>") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.entry") == null); - try std.testing.expect(std.mem.indexOf(u8, ir, "return") != null); + try std.testing.expect(std.mem.find(u8, ir, "cuda_tile.module @empty") != null); + try std.testing.expect(std.mem.find(u8, ir, "entry @empty(") != null); + try std.testing.expect(std.mem.find(u8, ir, "assume") != null); + try std.testing.expect(std.mem.find(u8, ir, "div_by<16>") != null); + try std.testing.expect(std.mem.find(u8, ir, "cuda_tile.entry") == null); + try std.testing.expect(std.mem.find(u8, ir, "return") != null); try expectRoundTrip(ctx, ir); } @@ -1827,14 +1827,14 @@ test "add_one over pointer tiles matches the XLA milestone kernel" { const ir = try b.finish(&.{}); defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "iota : tile<128xi32>") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "reshape") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "broadcast") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "offset") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "load_ptr_tko weak") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "addf") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "store_ptr_tko weak") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "token = ") != null); + try std.testing.expect(std.mem.find(u8, ir, "iota : tile<128xi32>") != null); + try std.testing.expect(std.mem.find(u8, ir, "reshape") != null); + try std.testing.expect(std.mem.find(u8, ir, "broadcast") != null); + try std.testing.expect(std.mem.find(u8, ir, "offset") != null); + try std.testing.expect(std.mem.find(u8, ir, "load_ptr_tko weak") != null); + try std.testing.expect(std.mem.find(u8, ir, "addf") != null); + try std.testing.expect(std.mem.find(u8, ir, "store_ptr_tko weak") != null); + try std.testing.expect(std.mem.find(u8, ir, "token = ") != null); try expectRoundTrip(ctx, ir); } @@ -1866,13 +1866,13 @@ test "vector_add over partition views" { const ir = try b.finish(&.{}); defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "optimization_hints=") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "make_tensor_view") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "partition_view") != null or std.mem.indexOf(u8, ir, "addf") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "store_view_tko weak") != null); + try std.testing.expect(std.mem.find(u8, ir, "optimization_hints=") != null); + try std.testing.expect(std.mem.find(u8, ir, "make_tensor_view") != null); + try std.testing.expect(std.mem.find(u8, ir, "partition_view") != null or std.mem.find(u8, ir, "addf") != null); + try std.testing.expect(std.mem.find(u8, ir, "store_view_tko weak") != null); try expectRoundTrip(ctx, ir); } @@ -1907,10 +1907,10 @@ test "gemm with a for loop and mmaf" { const ir = try b.finish(&.{}); defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "for ") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "iter_values") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "mmaf") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "continue ") != null); + try std.testing.expect(std.mem.find(u8, ir, "for ") != null); + try std.testing.expect(std.mem.find(u8, ir, "iter_values") != null); + try std.testing.expect(std.mem.find(u8, ir, "mmaf") != null); + try std.testing.expect(std.mem.find(u8, ir, "continue ") != null); try expectRoundTrip(ctx, ir); } @@ -1997,14 +1997,14 @@ test "if, if-else, loop with break, reduce, scan, cast, select" { const ir = try b.finish(&.{}); defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "reduce ") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "scan ") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "ftoi") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "itof") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "select") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "loop ") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "break ") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "yield") != null); + try std.testing.expect(std.mem.find(u8, ir, "reduce ") != null); + try std.testing.expect(std.mem.find(u8, ir, "scan ") != null); + try std.testing.expect(std.mem.find(u8, ir, "ftoi") != null); + try std.testing.expect(std.mem.find(u8, ir, "itof") != null); + try std.testing.expect(std.mem.find(u8, ir, "select") != null); + try std.testing.expect(std.mem.find(u8, ir, "loop ") != null); + try std.testing.expect(std.mem.find(u8, ir, "break ") != null); + try std.testing.expect(std.mem.find(u8, ir, "yield") != null); try expectRoundTrip(ctx, ir); } @@ -2055,20 +2055,20 @@ test "atomics, print, globals, alloca, dynamic views, strided view, tokens" { const ir = try b.finish(&.{}); defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "global @lut") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "get_global @lut") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "tensor_view") != null); + try std.testing.expect(std.mem.find(u8, ir, "rounding") != null); try expectRoundTrip(ctx, ir); } diff --git a/kernels/cuda_tile/control_flow.zig b/kernels/cuda_tile/control_flow.zig index 2f971f1d..f51e8571 100644 --- a/kernels/cuda_tile/control_flow.zig +++ b/kernels/cuda_tile/control_flow.zig @@ -13,13 +13,13 @@ fn collect(comptime ValueT: type, comptime N: usize, values: anytype, comptime w const info = @typeInfo(@TypeOf(values)); if (info != .@"struct" or !info.@"struct".is_tuple) @compileError(what ++ " expects a tuple literal"); - if (info.@"struct".fields.len != N) + if (info.@"struct".field_names.len != N) @compileError(what ++ ": arity must match the scope's declared arity"); var buf: [N]*const mlir.Value = undefined; - inline for (info.@"struct".fields, 0..) |f, i| { - if (f.type != ValueT) + inline for (info.@"struct".field_names, 0..) |f, i| { + if (@FieldType(@TypeOf(values), f) != ValueT) @compileError(what ++ ": every tuple element must be a Value"); - buf[i] = @field(values, f.name).inner; + buf[i] = @field(values, f).inner; } return buf; } diff --git a/kernels/cuda_tile/dtype.zig b/kernels/cuda_tile/dtype.zig index 4552e662..a9193839 100644 --- a/kernels/cuda_tile/dtype.zig +++ b/kernels/cuda_tile/dtype.zig @@ -73,8 +73,8 @@ pub fn intBitwidth(dt: DType) u32 { } pub fn mlirElemToDType(ctx: *mlir.Context, elem: *const mlir.Type) DType { - inline for (std.meta.fields(DType)) |f| { - const dt = @field(DType, f.name); + inline for (comptime std.meta.fieldNames(DType)) |f_name| { + const dt = @field(DType, f_name); if (elem.eql(dt.toMlir(ctx))) return dt; } @panic("element type not a recognized CUDA Tile IR DType"); diff --git a/kernels/cute/builder.zig b/kernels/cute/builder.zig index d012b222..286b92aa 100644 --- a/kernels/cute/builder.zig +++ b/kernels/cute/builder.zig @@ -98,8 +98,8 @@ pub const Value = struct { pub fn isFloat(self: Value) bool { const t = self.type_(); - inline for (std.meta.fields(mlir.FloatTypes)) |f| { - if (t.isA(mlir.FloatType(@field(mlir.FloatTypes, f.name))) != null) return true; + inline for (@typeInfo(mlir.FloatTypes).@"enum".field_names) |f| { + if (t.isA(mlir.FloatType(@field(mlir.FloatTypes, f))) != null) return true; } return false; } @@ -670,14 +670,14 @@ pub const Builder = struct { /// yields a `Tensor`, `.{ .p = .{ .ptr = .f32 } }` a pointer `Value`. pub fn declareArgs(self: *Builder, spec: anytype) FinishError!ArgsOf(@TypeOf(spec)) { const Spec = @TypeOf(spec); - const fields = @typeInfo(Spec).@"struct".fields; + const fields = @typeInfo(Spec).@"struct".field_names; const arg_specs = try self.arena.allocator().alloc(ArgSpec, fields.len); inline for (fields, 0..) |f, i| { - const raw = @field(spec, f.name); - const variant = @typeInfo(@TypeOf(raw)).@"struct".fields[0].name; + const raw = @field(spec, f); + const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; const inner = @field(raw, variant); - arg_specs[i] = .{ .name = f.name, .kind = switch (@field(std.meta.Tag(ArgSpec.Kind), variant)) { + arg_specs[i] = .{ .name = f, .kind = switch (@field(std.meta.Tag(ArgSpec.Kind), variant)) { .ptr => .{ .ptr = inner }, .tensor => .{ .tensor = .{ .dtype = inner.dtype, @@ -699,7 +699,7 @@ pub const Builder = struct { var named: ArgsOf(Spec) = undefined; inline for (fields, 0..) |f, i| { - @field(named, f.name) = switch (@FieldType(ArgsOf(Spec), f.name)) { + @field(named, f) = switch (@FieldType(ArgsOf(Spec), f)) { Tensor => self.tensors[i].?, else => self.arg(i), }; @@ -708,12 +708,12 @@ pub const Builder = struct { } fn ArgsOf(comptime Spec: type) type { - const in = @typeInfo(Spec).@"struct".fields; + const in = @typeInfo(Spec).@"struct".field_names; comptime var names: [in.len][]const u8 = undefined; comptime var types: [in.len]type = undefined; inline for (in, 0..) |f, i| { - names[i] = f.name; - types[i] = if (@hasField(f.type, "tensor")) Tensor else Value; + names[i] = f; + types[i] = if (@hasField(@FieldType(Spec, f), "tensor")) Tensor else Value; } return @Struct(.auto, null, &names, &types, &@splat(.{})); } @@ -998,7 +998,7 @@ pub const Builder = struct { // `cute.make_composed_layout`, or `llvm.load`/`llvm.store` printing // their alignment as a `` property). try printed.writer.print("{f}", .{self.module.operation().fmt(.{ .print_generic_op_form = true })}); - return try self.allocator.dupeZ(u8, printed.written()); + return try self.allocator.dupeSentinel(u8, printed.written(), 0); } // ==================== types ==================== @@ -1097,9 +1097,9 @@ pub const Builder = struct { .comptime_int, .int => writer.print("{d}", .{value}) catch @panic("OOM"), .@"struct" => |info| { writer.writeByte('(') catch @panic("OOM"); - inline for (info.fields, 0..) |field, i| { + inline for (info.field_names, 0..) |field, i| { if (i != 0) writer.writeByte(',') catch @panic("OOM"); - self.writeAlgebra(writer, @field(value, field.name)); + self.writeAlgebra(writer, @field(value, field)); } writer.writeByte(')') catch @panic("OOM"); }, @@ -1120,10 +1120,10 @@ pub const Builder = struct { pub fn basis(self: *Builder, numerator: i64, denominator: i64, modes: anytype) ScaledBasis { _ = self; if (denominator <= 0) @panic("CuTe scaled-basis denominator must be positive"); - const fields = @typeInfo(@TypeOf(modes)).@"struct".fields; + const fields = @typeInfo(@TypeOf(modes)).@"struct".field_names; if (fields.len == 0 or fields.len > MAX_RANK) @panic("CuTe scaled basis needs 1..MAX_RANK modes"); var result: ScaledBasis = .{ .numerator = numerator, .denominator = denominator, .mode_count = fields.len }; - inline for (fields, 0..) |field, i| result.modes[i] = @intCast(@field(modes, field.name)); + inline for (fields, 0..) |field, i| result.modes[i] = @intCast(@field(modes, field)); return result; } @@ -1312,13 +1312,13 @@ pub const Builder = struct { switch (@typeInfo(T)) { .comptime_int, .int => writer.print("{d}", .{coord}) catch @panic("OOM"), .@"struct" => |info| { - if (info.fields.len == 0) @compileError("makeCoord: empty tuple"); - if (info.fields.len > 1) writer.writeByte('(') catch @panic("OOM"); - inline for (info.fields, 0..) |field, i| { + if (info.field_names.len == 0) @compileError("makeCoord: empty tuple"); + if (info.field_names.len > 1) writer.writeByte('(') catch @panic("OOM"); + inline for (info.field_names, 0..) |field, i| { if (i != 0) writer.writeByte(',') catch @panic("OOM"); - self.writeCoord(writer, dyn, @field(coord, field.name)); + self.writeCoord(writer, dyn, @field(coord, field)); } - if (info.fields.len > 1) writer.writeByte(')') catch @panic("OOM"); + if (info.field_names.len > 1) writer.writeByte(')') catch @panic("OOM"); }, else => @compileError("makeCoord: unsupported coordinate leaf " ++ @typeName(T)), } @@ -2973,8 +2973,8 @@ pub const Builder = struct { pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(tupleArity(@TypeOf(result_types), "openIfElse: result_types")) { const N = comptime tupleArity(@TypeOf(result_types), "openIfElse: result_types"); var types: [N]*const mlir.Type = undefined; - inline for (@typeInfo(@TypeOf(result_types)).@"struct".fields, 0..) |f, i| { - types[i] = @field(result_types, f.name); + inline for (@typeInfo(@TypeOf(result_types)).@"struct".field_names, 0..) |f, i| { + types[i] = @field(result_types, f); } const then_block = mlir.Block.init(&.{}, &.{}); const else_block = mlir.Block.init(&.{}, &.{}); @@ -2994,8 +2994,8 @@ pub const Builder = struct { block_types[0] = lb.type_(); block_locs[0] = self.loc(); var inits_inner: [N]*const mlir.Value = undefined; - inline for (@typeInfo(@TypeOf(inits)).@"struct".fields, 0..) |f, i| { - const v = self.lift(@field(inits, f.name)); + inline for (@typeInfo(@TypeOf(inits)).@"struct".field_names, 0..) |f, i| { + const v = self.lift(@field(inits, f)); block_types[i + 1] = v.type_(); block_locs[i + 1] = self.loc(); inits_inner[i] = v.inner; @@ -3034,8 +3034,8 @@ pub const Builder = struct { var init_types: [N]*const mlir.Type = undefined; var init_locs: [N]*const mlir.Location = undefined; var inits_inner: [N]*const mlir.Value = undefined; - inline for (@typeInfo(@TypeOf(inits)).@"struct".fields, 0..) |f, i| { - const v = self.lift(@field(inits, f.name)); + inline for (@typeInfo(@TypeOf(inits)).@"struct".field_names, 0..) |f, i| { + const v = self.lift(@field(inits, f)); init_types[i] = v.type_(); init_locs[i] = self.loc(); inits_inner[i] = v.inner; @@ -3043,8 +3043,8 @@ pub const Builder = struct { var result_types: [M]*const mlir.Type = undefined; var result_locs: [M]*const mlir.Location = undefined; - inline for (@typeInfo(@TypeOf(after_types)).@"struct".fields, 0..) |f, i| { - result_types[i] = @field(after_types, f.name); + inline for (@typeInfo(@TypeOf(after_types)).@"struct".field_names, 0..) |f, i| { + result_types[i] = @field(after_types, f); result_locs[i] = self.loc(); } diff --git a/kernels/cute/dtype.zig b/kernels/cute/dtype.zig index 6dc9cc14..ae98cb08 100644 --- a/kernels/cute/dtype.zig +++ b/kernels/cute/dtype.zig @@ -55,8 +55,8 @@ pub fn dtypeBitwidth(dt: DType) usize { } pub fn mlirToDType(ctx: *mlir.Context, t: *const mlir.Type) DType { - inline for (std.meta.fields(DType)) |f| { - const dt = @field(DType, f.name); + inline for (@typeInfo(DType).@"enum".field_names) |f| { + const dt = @field(DType, f); if (t.eql(dt.toMlir(ctx))) return dt; } std.debug.panic("type {f} is not a CuTe DType", .{t}); diff --git a/kernels/fly/builder.zig b/kernels/fly/builder.zig index c7c087dc..ed3b18f9 100644 --- a/kernels/fly/builder.zig +++ b/kernels/fly/builder.zig @@ -727,22 +727,22 @@ pub const Builder = struct { /// `.{ .tensor = .{ .dtype = ..., .dims = &.{ ... } } }`. pub fn declareArgs(self: *Builder, spec: anytype) !dsl.NamedArgs(@TypeOf(spec), Value) { const Spec = @TypeOf(spec); - const fields = @typeInfo(Spec).@"struct".fields; + const fields = @typeInfo(Spec).@"struct".field_names; var specs: [fields.len]ArgSpec = undefined; inline for (fields, 0..) |f, i| { - const raw = @field(spec, f.name); - const variant = @typeInfo(@TypeOf(raw)).@"struct".fields[0].name; + const raw = @field(spec, f); + const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; const inner = @field(raw, variant); specs[i] = if (comptime std.mem.eql(u8, variant, "ptr")) - .{ .name = f.name, .dtype = inner } + .{ .name = f, .dtype = inner } else if (comptime std.mem.eql(u8, variant, "tensor")) - .{ .name = f.name, .dtype = inner.dtype, .dims = inner.dims, .space = if (@hasField(@TypeOf(inner), "space")) inner.space else .global } + .{ .name = f, .dtype = inner.dtype, .dims = inner.dims, .space = if (@hasField(@TypeOf(inner), "space")) inner.space else .global } else - @compileError("fly.declareArgs: argument `" ++ f.name ++ "` must be `.{ .ptr = dtype }` or `.{ .tensor = .{ .dtype, .dims } }`"); + @compileError("fly.declareArgs: argument `" ++ f ++ "` must be `.{ .ptr = dtype }` or `.{ .tensor = .{ .dtype, .dims } }`"); } try self.declareArgsLow(&specs); var named: dsl.NamedArgs(Spec, Value) = undefined; - inline for (fields, 0..) |f, i| @field(named, f.name) = self.arg(i); + inline for (fields, 0..) |f, i| @field(named, f) = self.arg(i); return named; } @@ -960,8 +960,8 @@ pub const Builder = struct { .comptime_int, .int => return .static(@intCast(v)), .@"struct" => |s| { if (!s.is_tuple) @compileError("fly.intTuple: expected an integer, null, Value, IntTuple or tuple, got " ++ @typeName(T)); - const elems = self.alloc(IntTuple, s.fields.len); - inline for (s.fields, 0..) |f, i| elems[i] = self.intTupleSpec(operands, @field(v, f.name)); + const elems = self.alloc(IntTuple, s.field_names.len); + inline for (s.field_names, 0..) |f, i| elems[i] = self.intTupleSpec(operands, @field(v, f)); return .{ .tup = elems }; }, else => @compileError("fly.intTuple: expected an integer, null, Value, IntTuple or tuple, got " ++ @typeName(T)), @@ -1335,7 +1335,7 @@ pub const Builder = struct { inits: anytype, ) ForScope(tupleArity(@TypeOf(inits), "openFor: inits")) { const N = comptime tupleArity(@TypeOf(inits), "openFor: inits"); - const fields = @typeInfo(@TypeOf(inits)).@"struct".fields; + const fields = @typeInfo(@TypeOf(inits)).@"struct".field_names; const lb_v: Value = if (@TypeOf(lower) == Value) lower else self.constIndex(lower); const ub_v: Value = if (@TypeOf(upper) == Value) upper else self.constLike(upper, lb_v); @@ -1347,8 +1347,8 @@ pub const Builder = struct { block_locs[0] = self.loc(); var inits_inner: [N]*const mlir.Value = undefined; inline for (fields, 0..) |f, i| { - if (f.type != Value) @compileError("openFor: every init must be a Value"); - const v: Value = @field(inits, f.name); + if (@TypeOf(@field(inits, f)) != Value) @compileError("openFor: every init must be a Value"); + const v: Value = @field(inits, f); block_types[i + 1] = v.type_(); block_locs[i + 1] = self.loc(); inits_inner[i] = v.inner; @@ -1391,11 +1391,11 @@ pub const Builder = struct { pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(tupleArity(@TypeOf(result_types), "openIfElse: result_types")) { const N = comptime tupleArity(@TypeOf(result_types), "openIfElse: result_types"); - const fields = @typeInfo(@TypeOf(result_types)).@"struct".fields; + const fields = @typeInfo(@TypeOf(result_types)).@"struct".field_names; var types: [N]*const mlir.Type = undefined; inline for (fields, 0..) |f, i| { - if (f.type != *const mlir.Type) @compileError("openIfElse: every result_type must be *const mlir.Type"); - types[i] = @field(result_types, f.name); + if (@TypeOf(@field(result_types, f)) != *const mlir.Type) @compileError("openIfElse: every result_type must be *const mlir.Type"); + types[i] = @field(result_types, f); } const then_block = mlir.Block.init(&.{}, &.{}); const else_block = mlir.Block.init(&.{}, &.{}); @@ -1413,16 +1413,16 @@ pub const Builder = struct { ) { const N = comptime tupleArity(@TypeOf(inits), "openWhile: inits"); const M = comptime tupleArity(@TypeOf(after_types), "openWhile: after_types"); - const init_fields = @typeInfo(@TypeOf(inits)).@"struct".fields; - const type_fields = @typeInfo(@TypeOf(after_types)).@"struct".fields; + const init_fields = @typeInfo(@TypeOf(inits)).@"struct".field_names; + const type_fields = @typeInfo(@TypeOf(after_types)).@"struct".field_names; var before_types: [N]*const mlir.Type = undefined; var before_locs: [N]*const mlir.Location = undefined; var inits_inner: [N]*const mlir.Value = undefined; inline for (init_fields, 0..) |f, i| { - if (f.type != Value) + if (@TypeOf(@field(inits, f)) != Value) @compileError("openWhile: every init must be a Value"); - const v: Value = @field(inits, f.name); + const v: Value = @field(inits, f); before_types[i] = v.type_(); before_locs[i] = self.loc(); inits_inner[i] = v.inner; @@ -1432,9 +1432,9 @@ pub const Builder = struct { var after_tys: [M]*const mlir.Type = undefined; var after_locs: [M]*const mlir.Location = undefined; inline for (type_fields, 0..) |f, i| { - if (f.type != *const mlir.Type) + if (@TypeOf(@field(after_types, f)) != *const mlir.Type) @compileError("openWhile: every after_type must be *const mlir.Type"); - after_tys[i] = @field(after_types, f.name); + after_tys[i] = @field(after_types, f); after_locs[i] = self.loc(); } const after_block = mlir.Block.init(&after_tys, &after_locs); @@ -1468,7 +1468,7 @@ pub const Builder = struct { var al: std.Io.Writer.Allocating = .init(self.allocator); defer al.deinit(); try al.writer.print("{f}", .{self.module.operation().fmt(.{ .debug_info = false })}); - return try self.allocator.dupeZ(u8, al.written()); + return try self.allocator.dupeSentinel(u8, al.written(), 0); } }; @@ -1752,8 +1752,7 @@ test "tiledMma builds, verifies and re-parses" { test "MmaFlavor.atomThreads matches the dialect" { const ctx = try testContext(); defer ctx.deinit(); - inline for (std.meta.fields(MmaFlavor)) |field| { - const flavor: MmaFlavor = @enumFromInt(field.value); + inline for (comptime std.enums.values(MmaFlavor)) |flavor| { const atom = try fly.types.MmaAtomType.get(ctx, .{ .mmaOp = flavor.atomType(ctx) }); // `mma_atom.thr_layout` is the wavefront an atom occupies; the launch // geometry is derived from it, so a wrong table silently corrupts C. diff --git a/kernels/fly/dtype.zig b/kernels/fly/dtype.zig index 6962a898..ecc9f070 100644 --- a/kernels/fly/dtype.zig +++ b/kernels/fly/dtype.zig @@ -65,8 +65,7 @@ pub const DType = enum { /// Inverse of `toMlir`. pub fn fromMlir(ctx: *mlir.Context, ty: *const mlir.Type) ?DType { - inline for (std.meta.fields(DType)) |f| { - const dt: DType = @enumFromInt(f.value); + inline for (comptime std.enums.values(DType)) |dt| { if (ty.eql(dt.toMlir(ctx))) return dt; } return null; @@ -77,8 +76,7 @@ test "dtype types round-trip" { defer registry.deinit(); const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); defer ctx.deinit(); - inline for (std.meta.fields(DType)) |f| { - const dt: DType = @enumFromInt(f.value); + inline for (comptime std.enums.values(DType)) |dt| { try std.testing.expectEqual(dt, fromMlir(ctx, dt.toMlir(ctx)).?); } try std.testing.expectEqual(DType.i8, DType.i1.storageElem()); diff --git a/kernels/fly/layout.zig b/kernels/fly/layout.zig index 85495ed4..e70e0e00 100644 --- a/kernels/fly/layout.zig +++ b/kernels/fly/layout.zig @@ -189,8 +189,8 @@ fn itImpl(comptime v: anytype) IntTuple { .comptime_int, .int => return IntTuple.static(v), .@"struct" => |s| { if (!s.is_tuple) @compileError("fly.it: expected an integer, null, IntTuple or tuple, got " ++ @typeName(T)); - var elems: [s.fields.len]IntTuple = undefined; - for (s.fields, 0..) |f, i| elems[i] = itImpl(@field(v, f.name)); + var elems: [s.field_names.len]IntTuple = undefined; + for (s.field_names, 0..) |f, i| elems[i] = itImpl(@field(v, f)); const frozen = elems; return .{ .tup = &frozen }; }, @@ -413,8 +413,8 @@ fn tileImpl(comptime v: anytype) Tile { .comptime_int, .int => return .{ .leaf = .{ .s = v } }, .@"struct" => |s| { if (!s.is_tuple) @compileError("fly.tile: expected an integer, null, Layout, Tile or tuple, got " ++ @typeName(T)); - var elems: [s.fields.len]Tile = undefined; - for (s.fields, 0..) |f, i| elems[i] = tileImpl(@field(v, f.name)); + var elems: [s.field_names.len]Tile = undefined; + for (s.field_names, 0..) |f, i| elems[i] = tileImpl(@field(v, f)); const frozen = elems; return .{ .modes = &frozen }; }, diff --git a/kernels/mosaic_tpu/builder.zig b/kernels/mosaic_tpu/builder.zig index 52307bca..d88fd6e5 100644 --- a/kernels/mosaic_tpu/builder.zig +++ b/kernels/mosaic_tpu/builder.zig @@ -106,8 +106,8 @@ pub const Value = struct { /// True if the element type is a floating-point type (f16/bf16/f32/f64/fp8/…). pub fn isFloatElem(self: Value) bool { const et = self.elemType(); - inline for (std.meta.fields(mlir.FloatTypes)) |f| { - if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, f.name))) != null) return true; + inline for (comptime std.meta.fieldNames(mlir.FloatTypes)) |field_name| { + if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, field_name))) != null) return true; } return false; } @@ -603,13 +603,13 @@ pub const Builder = struct { opts: Opts, ) !dsl.NamedArgs(@TypeOf(spec), Value) { const Spec = @TypeOf(spec); - const fields = @typeInfo(Spec).@"struct".fields; + const field_names = @typeInfo(Spec).@"struct".field_names; - var arg_specs: [fields.len]ArgSpec = undefined; - inline for (fields, 0..) |f, i| { - const raw = @field(spec, f.name); + var arg_specs: [field_names.len]ArgSpec = undefined; + inline for (field_names, 0..) |field_name, i| { + const raw = @field(spec, field_name); const kind: ArgSpec.Kind = if (@TypeOf(raw) == ArgSpec.Kind) raw else blk: { - const variant = @typeInfo(@TypeOf(raw)).@"struct".fields[0].name; + const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; const tag = @field(std.meta.Tag(ArgSpec.Kind), variant); const inner = @field(raw, variant); break :blk switch (tag) { @@ -632,14 +632,14 @@ pub const Builder = struct { .sem => .{ .sem = inner }, }; }; - arg_specs[i] = .{ .name = f.name, .kind = kind }; + arg_specs[i] = .{ .name = field_name, .kind = kind }; } try self.declareArgsLowOpts(&arg_specs, result_types, opts); var named: dsl.NamedArgs(Spec, Value) = undefined; - inline for (fields, 0..) |f, i| { - @field(named, f.name) = self.arg(i); + inline for (field_names, 0..) |field_name, i| { + @field(named, field_name) = self.arg(i); } return named; } @@ -899,12 +899,12 @@ pub const Builder = struct { const info = @typeInfo(T); if (info != .@"struct" or !info.@"struct".is_tuple) @compileError("Builder.yield expects a tuple literal like `.{ v1, v2 }`"); - const n = info.@"struct".fields.len; + const n = info.@"struct".field_names.len; const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.yield OOM"); - inline for (info.@"struct".fields, 0..) |f, i| { - if (f.type != Value) - @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(f.type)); - out[i] = @field(values, f.name); + inline for (info.@"struct".field_names, info.@"struct".field_types, 0..) |field_name, Field, i| { + if (Field != Value) + @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(Field)); + out[i] = @field(values, field_name); } return out; } @@ -2057,7 +2057,7 @@ pub const Builder = struct { inits: anytype, ) dsl.ForScope(Builder, Value, dsl.tupleArity(@TypeOf(inits), "openFor: inits")) { const N = comptime dsl.tupleArity(@TypeOf(inits), "openFor: inits"); - const fields = @typeInfo(@TypeOf(inits)).@"struct".fields; + const info = @typeInfo(@TypeOf(inits)).@"struct"; // scf.for bounds default to `index`. const lb_v: Value = if (@TypeOf(lower) == Value) lower else self.cIndex(@intCast(lower)); @@ -2069,9 +2069,9 @@ pub const Builder = struct { block_types[0] = lb_v.type_(); block_locs[0] = self.loc(); var inits_inner: [N]*const mlir.Value = undefined; - inline for (fields, 0..) |f, i| { - const raw = @field(inits, f.name); - const v: Value = if (f.type == Value) raw else self.lift(raw); + inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { + const raw = @field(inits, field_name); + const v: Value = if (Field == Value) raw else self.lift(raw); block_types[i + 1] = v.type_(); block_locs[i + 1] = self.loc(); inits_inner[i] = v.inner; @@ -2111,13 +2111,13 @@ pub const Builder = struct { result_types: anytype, ) dsl.IfScope(Builder, Value, dsl.tupleArity(@TypeOf(result_types), "openIfElse: result_types")) { const N = comptime dsl.tupleArity(@TypeOf(result_types), "openIfElse: result_types"); - const fields = @typeInfo(@TypeOf(result_types)).@"struct".fields; + const info = @typeInfo(@TypeOf(result_types)).@"struct"; var types: [N]*const mlir.Type = undefined; - inline for (fields, 0..) |f, i| { - if (f.type != *const mlir.Type) + inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { + if (Field != *const mlir.Type) @compileError("openIfElse: every result_type must be *const mlir.Type (use k.scalarTy/vectorTy)"); - types[i] = @field(result_types, f.name); + types[i] = @field(result_types, field_name); } const then_block = mlir.Block.init(&.{}, &.{}); @@ -2145,16 +2145,16 @@ pub const Builder = struct { ) { const N = comptime dsl.tupleArity(@TypeOf(inits), "openWhile: inits"); const M = comptime dsl.tupleArity(@TypeOf(after_types), "openWhile: after_types"); - const init_fields = @typeInfo(@TypeOf(inits)).@"struct".fields; - const type_fields = @typeInfo(@TypeOf(after_types)).@"struct".fields; + const init_info = @typeInfo(@TypeOf(inits)).@"struct"; + const type_info = @typeInfo(@TypeOf(after_types)).@"struct"; var before_types: [N]*const mlir.Type = undefined; var before_locs: [N]*const mlir.Location = undefined; var inits_inner: [N]*const mlir.Value = undefined; - inline for (init_fields, 0..) |f, i| { - if (f.type != Value) + inline for (init_info.field_names, init_info.field_types, 0..) |field_name, Field, i| { + if (Field != Value) @compileError("openWhile: every init must be a Value"); - const v: Value = @field(inits, f.name); + const v: Value = @field(inits, field_name); before_types[i] = v.type_(); before_locs[i] = self.loc(); inits_inner[i] = v.inner; @@ -2163,10 +2163,10 @@ pub const Builder = struct { var after_tys: [M]*const mlir.Type = undefined; var after_locs: [M]*const mlir.Location = undefined; - inline for (type_fields, 0..) |f, i| { - if (f.type != *const mlir.Type) + inline for (type_info.field_names, type_info.field_types, 0..) |field_name, Field, i| { + if (Field != *const mlir.Type) @compileError("openWhile: every after_type must be *const mlir.Type"); - after_tys[i] = @field(after_types, f.name); + after_tys[i] = @field(after_types, field_name); after_locs[i] = self.loc(); } const after_block = mlir.Block.init(&after_tys, &after_locs); @@ -2232,7 +2232,7 @@ pub const Builder = struct { .debug_info = false, })}); - return try self.allocator.dupeZ(u8, al.written()); + return try self.allocator.dupeSentinel(u8, al.written(), 0); } /// Emit `transform_N` stub funcs — one per non-skipped i/o ref-arg. diff --git a/kernels/mosaic_tpu/dtype.zig b/kernels/mosaic_tpu/dtype.zig index d22370f6..549c69da 100644 --- a/kernels/mosaic_tpu/dtype.zig +++ b/kernels/mosaic_tpu/dtype.zig @@ -60,8 +60,8 @@ pub fn intBitwidth(dt: DType) u32 { } pub fn mlirElemToDType(ctx: *mlir.Context, elem: *const mlir.Type) DType { - inline for (std.meta.fields(DType)) |f| { - const dt = @field(DType, f.name); + inline for (comptime std.meta.fieldNames(DType)) |field_name| { + const dt = @field(DType, field_name); if (elem.eql(dt.toMlir(ctx))) return dt; } @panic("element type not a recognized Mosaic-TPU DType"); diff --git a/kernels/triton/builder.zig b/kernels/triton/builder.zig index 702fba0d..d585e8c1 100644 --- a/kernels/triton/builder.zig +++ b/kernels/triton/builder.zig @@ -66,8 +66,8 @@ pub const Value = struct { pub fn isFloatElem(self: Value) bool { const et = self.elemType(); - inline for (std.meta.fields(mlir.FloatTypes)) |f| { - if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, f.name))) != null) return true; + inline for (comptime std.meta.fieldNames(mlir.FloatTypes)) |field_name| { + if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, field_name))) != null) return true; } return false; } @@ -391,25 +391,25 @@ pub const ReturnIfScope = struct { pub fn yieldReturn(self: *ReturnIfScope, values: anytype) void { const k = self.kernel; - const fields = @typeInfo(@TypeOf(values)).@"struct".fields; + const info = @typeInfo(@TypeOf(values)).@"struct"; if (self.inline_return) { - var ret_operands: [fields.len]*const mlir.Value = undefined; - inline for (fields, 0..) |f, i| { - if (f.type != Value) + var ret_operands: [info.field_names.len]*const mlir.Value = undefined; + inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { + if (Field != Value) @compileError("ReturnIfScope.yieldReturn: every value must be a Value"); - ret_operands[i] = @field(values, f.name).inner; + ret_operands[i] = @field(values, field_name).inner; } _ = ttir.return_(k.ctx, &ret_operands, k.loc()).appendTo(self.ret_block); } else { if (k.exit_block == null) { k.exit_block = mlir.Block.init(&.{}, &.{}); - var ret_operands: [fields.len]*const mlir.Value = undefined; - inline for (fields, 0..) |f, i| { - if (f.type != Value) + var ret_operands: [info.field_names.len]*const mlir.Value = undefined; + inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { + if (Field != Value) @compileError("ReturnIfScope.yieldReturn: every value must be a Value"); - ret_operands[i] = @field(values, f.name).inner; + ret_operands[i] = @field(values, field_name).inner; } _ = ttir.return_(k.ctx, &ret_operands, k.loc()).appendTo(k.exit_block.?); } @@ -533,13 +533,13 @@ pub const Builder = struct { opts: Opts, ) !dsl.NamedArgs(@TypeOf(spec), Value) { const Spec = @TypeOf(spec); - const fields = @typeInfo(Spec).@"struct".fields; + const field_names = @typeInfo(Spec).@"struct".field_names; - var arg_specs: [fields.len]ArgSpec = undefined; - inline for (fields, 0..) |f, i| { - const raw = @field(spec, f.name); + var arg_specs: [field_names.len]ArgSpec = undefined; + inline for (field_names, 0..) |field_name, i| { + const raw = @field(spec, field_name); const kind: ArgSpec.Kind = if (@TypeOf(raw) == ArgSpec.Kind) raw else blk: { - const variant = @typeInfo(@TypeOf(raw)).@"struct".fields[0].name; + const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; const tag = @field(std.meta.Tag(ArgSpec.Kind), variant); const inner = @field(raw, variant); break :blk switch (tag) { @@ -557,14 +557,14 @@ pub const Builder = struct { } }, }; }; - arg_specs[i] = .{ .name = f.name, .kind = kind }; + arg_specs[i] = .{ .name = field_name, .kind = kind }; } try self.declareArgsLowOpts(&arg_specs, result_types, opts); var named: dsl.NamedArgs(Spec, Value) = undefined; - inline for (fields, 0..) |f, i| { - @field(named, f.name) = self.arg(i); + inline for (field_names, 0..) |field_name, i| { + @field(named, field_name) = self.arg(i); } return named; } @@ -679,12 +679,12 @@ pub const Builder = struct { const info = @typeInfo(T); if (info != .@"struct" or !info.@"struct".is_tuple) @compileError("Builder.yield expects a tuple literal like `.{ v1, v2 }`"); - const n = info.@"struct".fields.len; + const n = info.@"struct".field_names.len; const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.yield OOM"); - inline for (info.@"struct".fields, 0..) |f, i| { - if (f.type != Value) - @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(f.type)); - out[i] = @field(values, f.name); + inline for (info.@"struct".field_names, info.@"struct".field_types, 0..) |field_name, Field, i| { + if (Field != Value) + @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(Field)); + out[i] = @field(values, field_name); } return out; } @@ -898,8 +898,8 @@ pub const Builder = struct { } fn mlirElemToDType(self: *const Builder, elem: *const mlir.Type) DType { - inline for (std.meta.fields(DType)) |f| { - const dt = @field(DType, f.name); + inline for (comptime std.meta.fieldNames(DType)) |field_name| { + const dt = @field(DType, field_name); if (elem.eql(dt.toMlir(self.ctx))) return dt; } @panic("element type not a recognized DSL DType"); @@ -2012,7 +2012,7 @@ pub const Builder = struct { inits: anytype, ) ForScope(tupleArity(@TypeOf(inits), "openFor: inits")) { const N = comptime tupleArity(@TypeOf(inits), "openFor: inits"); - const fields = @typeInfo(@TypeOf(inits)).@"struct".fields; + const info = @typeInfo(@TypeOf(inits)).@"struct"; const lb_v: Value = if (@TypeOf(lower) == Value) lower else self.lift(lower); const ub_v: Value = if (@TypeOf(upper) == Value) upper else self.constMatching(upper, lb_v.type_()); @@ -2023,9 +2023,9 @@ pub const Builder = struct { block_types[0] = lb_v.type_(); block_locs[0] = self.loc(); var inits_inner: [N]*const mlir.Value = undefined; - inline for (fields, 0..) |f, i| { - const raw = @field(inits, f.name); - const v: Value = if (f.type == Value) raw else self.lift(raw); + inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { + const raw = @field(inits, field_name); + const v: Value = if (Field == Value) raw else self.lift(raw); block_types[i + 1] = v.type_(); block_locs[i + 1] = self.loc(); inits_inner[i] = v.inner; @@ -2058,12 +2058,12 @@ pub const Builder = struct { const region = current.parentRegion(); const cont_block = mlir.Block.init(&.{}, &.{}); - const fields = @typeInfo(@TypeOf(values)).@"struct".fields; - var ret_operands: [fields.len]*const mlir.Value = undefined; - inline for (fields, 0..) |f, i| { - if (f.type != Value) + const info = @typeInfo(@TypeOf(values)).@"struct"; + var ret_operands: [info.field_names.len]*const mlir.Value = undefined; + inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { + if (Field != Value) @compileError("returnIfOpts: every value must be a Value"); - ret_operands[i] = @field(values, f.name).inner; + ret_operands[i] = @field(values, field_name).inner; } const ret_target: *mlir.Block = if (opts.inline_return) blk: { @@ -2144,13 +2144,13 @@ pub const Builder = struct { result_types: anytype, ) IfScope(tupleArity(@TypeOf(result_types), "openIfElse: result_types")) { const N = comptime tupleArity(@TypeOf(result_types), "openIfElse: result_types"); - const fields = @typeInfo(@TypeOf(result_types)).@"struct".fields; + const info = @typeInfo(@TypeOf(result_types)).@"struct"; var types: [N]*const mlir.Type = undefined; - inline for (fields, 0..) |f, i| { - if (f.type != *const mlir.Type) + inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { + if (Field != *const mlir.Type) @compileError("openIfElse: every result_type must be *const mlir.Type (use k.scalarTy/tensorTy)"); - types[i] = @field(result_types, f.name); + types[i] = @field(result_types, field_name); } const then_block = mlir.Block.init(&.{}, &.{}); @@ -2176,16 +2176,16 @@ pub const Builder = struct { ) { const N = comptime tupleArity(@TypeOf(inits), "openWhile: inits"); const M = comptime tupleArity(@TypeOf(after_types), "openWhile: after_types"); - const init_fields = @typeInfo(@TypeOf(inits)).@"struct".fields; - const type_fields = @typeInfo(@TypeOf(after_types)).@"struct".fields; + const init_info = @typeInfo(@TypeOf(inits)).@"struct"; + const type_info = @typeInfo(@TypeOf(after_types)).@"struct"; var before_types: [N]*const mlir.Type = undefined; var before_locs: [N]*const mlir.Location = undefined; var inits_inner: [N]*const mlir.Value = undefined; - inline for (init_fields, 0..) |f, i| { - if (f.type != Value) + inline for (init_info.field_names, init_info.field_types, 0..) |field_name, Field, i| { + if (Field != Value) @compileError("openWhile: every init must be a Value"); - const v: Value = @field(inits, f.name); + const v: Value = @field(inits, field_name); before_types[i] = v.type_(); before_locs[i] = self.loc(); inits_inner[i] = v.inner; @@ -2194,10 +2194,10 @@ pub const Builder = struct { var after_tys: [M]*const mlir.Type = undefined; var after_locs: [M]*const mlir.Location = undefined; - inline for (type_fields, 0..) |f, i| { - if (f.type != *const mlir.Type) + inline for (type_info.field_names, type_info.field_types, 0..) |field_name, Field, i| { + if (Field != *const mlir.Type) @compileError("openWhile: every after_type must be *const mlir.Type"); - after_tys[i] = @field(after_types, f.name); + after_tys[i] = @field(after_types, field_name); after_locs[i] = self.loc(); } const after_block = mlir.Block.init(&after_tys, &after_locs); @@ -2249,6 +2249,6 @@ pub const Builder = struct { .print_name_loc_as_prefix = true, })}); - return try self.allocator.dupeZ(u8, al.written()); + return try self.allocator.dupeSentinel(u8, al.written(), 0); } }; diff --git a/kernels/triton/dtype.zig b/kernels/triton/dtype.zig index c8c1d623..45018e67 100644 --- a/kernels/triton/dtype.zig +++ b/kernels/triton/dtype.zig @@ -62,8 +62,8 @@ pub fn intBitwidth(dt: DType) u32 { } pub fn mlirElemToDType(ctx: *mlir.Context, elem: *const mlir.Type) DType { - inline for (std.meta.fields(DType)) |f| { - const dt = @field(DType, f.name); + inline for (comptime std.meta.fieldNames(DType)) |field_name| { + const dt = @field(DType, field_name); if (elem.eql(dt.toMlir(ctx))) return dt; } @panic("element type not a recognized Triton DType"); diff --git a/mlir/dialects/arith.zig b/mlir/dialects/arith.zig index aee1c869..70c30574 100644 --- a/mlir/dialects/arith.zig +++ b/mlir/dialects/arith.zig @@ -360,7 +360,7 @@ pub fn convertf( ) *mlir.Operation { var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; if (opts.rounding) |rm| { - attrs.appendAssumeCapacity(.named(ctx, "roundingmode", .int(ctx, .i32, @intFromEnum(rm)))); + attrs.appendAssumeCapacity(.named(ctx, "roundingmode", .int(ctx, .i32, @backingInt(rm)))); } return mlir.Operation.make(ctx, "arith.convertf", .{ .operands = .{ .flat = &.{src} }, @@ -381,7 +381,7 @@ pub fn scaling_truncf( ) *mlir.Operation { var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; if (opts.rounding) |rm| { - attrs.appendAssumeCapacity(.named(ctx, "roundingmode", .int(ctx, .i32, @intFromEnum(rm)))); + attrs.appendAssumeCapacity(.named(ctx, "roundingmode", .int(ctx, .i32, @backingInt(rm)))); } return mlir.Operation.make(ctx, "arith.scaling_truncf", .{ .operands = .{ .flat = &.{ src, scale } }, @@ -406,7 +406,7 @@ pub fn cmpi( .operands = .{ .flat = &.{ lhs, rhs } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "predicate", .int(ctx, .i64, @intFromEnum(predicate))), + .named(ctx, "predicate", .int(ctx, .i64, @backingInt(predicate))), }, .location = location, }); @@ -423,7 +423,7 @@ pub fn cmpf( .operands = .{ .flat = &.{ lhs, rhs } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "predicate", .int(ctx, .i64, @intFromEnum(predicate))), + .named(ctx, "predicate", .int(ctx, .i64, @backingInt(predicate))), }, .location = location, }); diff --git a/mlir/dialects/cuda_tile/cuda_tile.zig b/mlir/dialects/cuda_tile/cuda_tile.zig index e32ddc26..d4975fd4 100644 --- a/mlir/dialects/cuda_tile/cuda_tile.zig +++ b/mlir/dialects/cuda_tile/cuda_tile.zig @@ -506,16 +506,16 @@ pub const OptimizationHintsAttr = opaque { /// The hint dictionary of one architecture: the non-null fields of `hint` /// but `arch`, integers as `i64` like the textual form. fn hintDictionary(ctx: *mlir.Context, hint: anytype) *const mlir.Attribute { - const fields = @typeInfo(@TypeOf(hint)).@"struct".fields; - var entries: stdx.BoundedArray(mlir.NamedAttribute, fields.len) = .empty; - inline for (fields) |f| { - if (comptime std.mem.eql(u8, f.name, "arch")) continue; - if (@field(hint, f.name)) |v| { + const field_names = @typeInfo(@TypeOf(hint)).@"struct".field_names; + var entries: stdx.BoundedArray(mlir.NamedAttribute, field_names.len) = .empty; + inline for (field_names) |field_name| { + if (comptime std.mem.eql(u8, field_name, "arch")) continue; + if (@field(hint, field_name)) |v| { const value: *const mlir.Attribute = switch (@TypeOf(v)) { bool => .boolean(ctx, v), else => .int(ctx, .i64, v), }; - entries.appendAssumeCapacity(.named(ctx, f.name, value)); + entries.appendAssumeCapacity(.named(ctx, field_name, value)); } } return .dict(ctx, entries.constSlice()); @@ -524,7 +524,7 @@ fn hintDictionary(ctx: *mlir.Context, hint: anytype) *const mlir.Attribute { /// `#cuda_tile.optimization_hints` from a slice of `EntryHint`, /// `LoadStoreHint` or `PtrLoadStoreHint`. pub fn optimizationHints(ctx: *mlir.Context, hints: anytype) *const mlir.Attribute { - var entries: stdx.BoundedArray(mlir.NamedAttribute, @typeInfo(Arch).@"enum".fields.len) = .empty; + var entries: stdx.BoundedArray(mlir.NamedAttribute, std.meta.fieldNames(Arch).len) = .empty; for (hints) |h| entries.appendAssumeCapacity(.named(ctx, @tagName(h.arch), hintDictionary(ctx, h))); const value: *const mlir.DictionaryAttribute = @ptrCast(mlir.Attribute.dict(ctx, entries.constSlice())); const attr = OptimizationHintsAttr.get(ctx, .{ .value = value }) catch @@ -1430,7 +1430,7 @@ pub fn load_view_tko(ctx: *mlir.Context, view: *const mlir.Value, index: []const segs.add(tok.slice()); var attrs = memAttrs(ctx, opts); // Preserve bounds checks unless the caller has proved a coordinate in bounds. - const inbounds = [_]i32{0} ** mlir.ShapedType.MAX_RANK; + const inbounds: [mlir.ShapedType.MAX_RANK]i32 = @splat(0); attrs.appendAssumeCapacity(.named(ctx, "inbounds", .denseArray(ctx, .bool, inbounds[0..index.len]))); attrs.appendAssumeCapacity(segs.attr(ctx)); return mlir.Operation.make(ctx, opName("load_view_tko"), .{ @@ -1450,7 +1450,7 @@ pub fn store_view_tko(ctx: *mlir.Context, tile: *const mlir.Value, view: *const segs.add(index); segs.add(tok.slice()); var attrs = memAttrs(ctx, opts); - const inbounds = [_]i32{0} ** mlir.ShapedType.MAX_RANK; + const inbounds: [mlir.ShapedType.MAX_RANK]i32 = @splat(0); attrs.appendAssumeCapacity(.named(ctx, "inbounds", .denseArray(ctx, .bool, inbounds[0..index.len]))); attrs.appendAssumeCapacity(segs.attr(ctx)); return mlir.Operation.make(ctx, opName("store_view_tko"), .{ diff --git a/mlir/dialects/cute_ir/cuda.zig b/mlir/dialects/cute_ir/cuda.zig index 6c546113..f3cfcc01 100644 --- a/mlir/dialects/cute_ir/cuda.zig +++ b/mlir/dialects/cute_ir/cuda.zig @@ -353,20 +353,20 @@ pub const ComputeTargetAttr = opaque { archs: []const GpuArchitecture, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCudaComputeTargetAttrGet(ctx.ptr(), @intFromEnum(args.representation), @intFromEnum(args.portability), @intCast(args.archs.len), @ptrCast(args.archs.ptr)); + const result = c.mlirCudaComputeTargetAttrGet(ctx.ptr(), @backingInt(args.representation), @backingInt(args.portability), @intCast(args.archs.len), @ptrCast(args.archs.ptr)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getRepresentation(self: *const Self) ExecutableRepresentation { - return @enumFromInt(c.mlirCudaComputeTargetAttrGetRepresentation(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCudaComputeTargetAttrGetRepresentation(self.ptr()))); } pub fn getPortability(self: *const Self) ArchPortability { - return @enumFromInt(c.mlirCudaComputeTargetAttrGetPortability(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCudaComputeTargetAttrGetPortability(self.ptr()))); } pub fn getNumArchs(self: *const Self) usize { return @intCast(c.mlirCudaComputeTargetAttrGetNumArchs(self.ptr())); } pub fn getArch(self: *const Self, pos: usize) GpuArchitecture { - return @enumFromInt(c.mlirCudaComputeTargetAttrGetArchs(self.ptr(), @intCast(pos))); + return @fromBackingInt(@intCast(c.mlirCudaComputeTargetAttrGetArchs(self.ptr(), @intCast(pos)))); } }; @@ -439,11 +439,11 @@ pub const ExecutableAttr = opaque { format: ExecutableFormat, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCudaExecutableAttrGet(ctx.ptr(), @intFromEnum(args.format)); + const result = c.mlirCudaExecutableAttrGet(ctx.ptr(), @backingInt(args.format)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getFormat(self: *const Self) ExecutableFormat { - return @enumFromInt(c.mlirCudaExecutableAttrGetFormat(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCudaExecutableAttrGetFormat(self.ptr()))); } }; diff --git a/mlir/dialects/cute_ir/cute_ir.zig b/mlir/dialects/cute_ir/cute_ir.zig index 8420bcbf..b36565ac 100644 --- a/mlir/dialects/cute_ir/cute_ir.zig +++ b/mlir/dialects/cute_ir/cute_ir.zig @@ -641,14 +641,14 @@ pub const PtrType = opaque { bitlayout: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCutePtrTypeGet(ctx.ptr(), if (args.valueType) |v| v.ptr() else c.MlirType{ .ptr = null }, @intFromEnum(args.addressSpace), args.alignment, if (args.swizzle) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, if (args.bitlayout) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCutePtrTypeGet(ctx.ptr(), if (args.valueType) |v| v.ptr() else c.MlirType{ .ptr = null }, @backingInt(args.addressSpace), args.alignment, if (args.swizzle) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, if (args.bitlayout) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValueType(self: *const Self) ?*const mlir.Type { return @ptrCast(c.mlirCutePtrTypeGetValueType(self.ptr()).ptr); } pub fn getAddressSpace(self: *const Self) AddressSpace { - return @enumFromInt(c.mlirCutePtrTypeGetAddressSpace(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCutePtrTypeGetAddressSpace(self.ptr()))); } pub fn getAlignment(self: *const Self) u64 { return c.mlirCutePtrTypeGetAlignment(self.ptr()); @@ -1225,11 +1225,11 @@ pub const ReductionOpAttr = opaque { value: ReductionOp, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteReductionOpAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteReductionOpAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) ReductionOp { - return @enumFromInt(c.mlirCuteReductionOpAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteReductionOpAttrGetValue(self.ptr()))); } }; @@ -2861,16 +2861,16 @@ test { std.testing.refAllDecls(cuda); inline for (.{ nvvm, cuda }) |namespace| { inline for (comptime std.meta.declarations(namespace)) |decl| { - const T = @field(namespace, decl.name); + const T = @field(namespace, decl); if (@TypeOf(T) == type) std.testing.refAllDecls(T); } } inline for (comptime std.meta.declarations(@This())) |decl| { - const T = @field(@This(), decl.name); + const T = @field(@This(), decl); if (@TypeOf(T) == type) std.testing.refAllDecls(T); } inline for (comptime std.meta.declarations(nvgpu)) |decl| { - const T = @field(nvgpu, decl.name); + const T = @field(nvgpu, decl); if (@TypeOf(T) == type) std.testing.refAllDecls(T); } } @@ -2994,13 +2994,14 @@ test "generated types and attributes rebuild from their getters" { // Every case of every enum attribute. inline for (.{ cute, nvgpu }) |namespace| { inline for (comptime std.meta.declarations(namespace)) |decl| { - const T = @field(namespace, decl.name); + const T = @field(namespace, decl); if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque" and @hasDecl(T, "InitArgs")) { - const fields = @typeInfo(T.InitArgs).@"struct".fields; - if (fields.len == 1 and comptime std.mem.eql(u8, fields[0].name, "value")) { - inline for (@typeInfo(fields[0].type).@"enum".fields) |case| { - const attr = try T.get(ctx, .{ .value = @enumFromInt(case.value) }); - try std.testing.expectEqual(case.value, @intFromEnum(attr.getValue())); + const fields = @typeInfo(T.InitArgs).@"struct".field_names; + if (fields.len == 1 and comptime std.mem.eql(u8, fields[0], "value")) { + inline for (@typeInfo(@FieldType(T.InitArgs, fields[0])).@"enum".field_names) |case| { + const case_value = @field(@FieldType(T.InitArgs, fields[0]), case); + const attr = try T.get(ctx, .{ .value = case_value }); + try std.testing.expectEqual(@backingInt(case_value), @backingInt(attr.getValue())); try std.testing.expect((try rebuild(T, ctx, attr)).eql(attr)); } } @@ -3070,17 +3071,17 @@ test "nvvm target rejects what the compiler rejects" { fn rebuildEnums(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T { var args: T.InitArgs = undefined; var buffers: [4][16]u32 = undefined; - inline for (@typeInfo(T.InitArgs).@"struct".fields, 0..) |field, i| { - const name = [_]u8{comptime std.ascii.toUpper(field.name[0])} ++ field.name[1..]; - const info = @typeInfo(field.type); + inline for (@typeInfo(T.InitArgs).@"struct".field_names, 0..) |field, i| { + const name = [_]u8{comptime std.ascii.toUpper(field[0])} ++ field[1..]; + const info = @typeInfo(@FieldType(T.InitArgs, field)); if (info == .pointer and info.pointer.size == .slice and @typeInfo(info.pointer.child) == .@"enum") { const E = info.pointer.child; const count = @field(T, "getNum" ++ name)(value); const elements: []E = @ptrCast(buffers[i][0..count]); for (elements, 0..) |*element, pos| element.* = @field(T, "get" ++ name[0 .. name.len - 1])(value, pos); - @field(args, field.name) = elements; + @field(args, field) = elements; } else { - @field(args, field.name) = @field(T, "get" ++ name)(value); + @field(args, field) = @field(T, "get" ++ name)(value); } } return T.get(ctx, args); @@ -3089,14 +3090,14 @@ fn rebuildEnums(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T fn rebuild(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T { var args: T.InitArgs = undefined; var elements: [8]*const mlir.Type = undefined; - inline for (@typeInfo(T.InitArgs).@"struct".fields) |field| { - const name = [_]u8{comptime std.ascii.toUpper(field.name[0])} ++ field.name[1..]; - if (field.type == []const *const mlir.Type) { + inline for (@typeInfo(T.InitArgs).@"struct".field_names) |field| { + const name = [_]u8{comptime std.ascii.toUpper(field[0])} ++ field[1..]; + if (@FieldType(T.InitArgs, field) == []const *const mlir.Type) { const count = @field(T, "getNum" ++ name)(value); for (elements[0..count], 0..) |*element, i| element.* = @field(T, "get" ++ name[0 .. name.len - 1])(value, i); - @field(args, field.name) = elements[0..count]; + @field(args, field) = elements[0..count]; } else { - @field(args, field.name) = @field(T, "get" ++ name)(value); + @field(args, field) = @field(T, "get" ++ name)(value); } } return T.get(ctx, args); diff --git a/mlir/dialects/cute_ir/cute_nvgpu.zig b/mlir/dialects/cute_ir/cute_nvgpu.zig index 7d7ac63d..652ff9e1 100644 --- a/mlir/dialects/cute_ir/cute_nvgpu.zig +++ b/mlir/dialects/cute_ir/cute_nvgpu.zig @@ -564,7 +564,7 @@ pub const CopyAtomG2RType = opaque { invariant: bool = false, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomG2RTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.memOrder), @intFromEnum(args.memScope), @intFromEnum(args.l2PrefetchSize), @intFromEnum(args.l1CacheEvictPriority), @intFromEnum(args.loadCacheMode), @intFromEnum(args.sharedSpace), args.invariant); + const result = c.mlirCuteNVGPUCopyAtomG2RTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.l2PrefetchSize), @backingInt(args.l1CacheEvictPriority), @backingInt(args.loadCacheMode), @backingInt(args.sharedSpace), args.invariant); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -574,22 +574,22 @@ pub const CopyAtomG2RType = opaque { return c.mlirCuteNVGPUCopyAtomG2RTypeGetCopyBits(self.ptr()); } pub fn getMemOrder(self: *const Self) MemOrderKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomG2RTypeGetMemOrder(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetMemOrder(self.ptr()))); } pub fn getMemScope(self: *const Self) MemScopeKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomG2RTypeGetMemScope(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetMemScope(self.ptr()))); } pub fn getL2PrefetchSize(self: *const Self) L2PrefetchSize { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomG2RTypeGetL2PrefetchSize(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetL2PrefetchSize(self.ptr()))); } pub fn getL1CacheEvictPriority(self: *const Self) CacheEvictionPriority { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomG2RTypeGetL1CacheEvictPriority(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetL1CacheEvictPriority(self.ptr()))); } pub fn getLoadCacheMode(self: *const Self) LoadCacheMode { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomG2RTypeGetLoadCacheMode(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetLoadCacheMode(self.ptr()))); } pub fn getSharedSpace(self: *const Self) SharedSpace { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomG2RTypeGetSharedSpace(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetSharedSpace(self.ptr()))); } pub fn getInvariant(self: *const Self) bool { return c.mlirCuteNVGPUCopyAtomG2RTypeGetInvariant(self.ptr()); @@ -705,7 +705,7 @@ pub const CopyAtomLdsmType = opaque { transpose: bool = false, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomLdsmTypeGet(ctx.ptr(), args.valType.ptr(), args.mode.ptr(), @intFromEnum(args.szPattern), args.numMatrices, args.transpose); + const result = c.mlirCuteNVGPUCopyAtomLdsmTypeGet(ctx.ptr(), args.valType.ptr(), args.mode.ptr(), @backingInt(args.szPattern), args.numMatrices, args.transpose); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -715,7 +715,7 @@ pub const CopyAtomLdsmType = opaque { return @ptrCast(c.mlirCuteNVGPUCopyAtomLdsmTypeGetMode(self.ptr()).ptr.?); } pub fn getSzPattern(self: *const Self) LdsmSzPattern { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomLdsmTypeGetSzPattern(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomLdsmTypeGetSzPattern(self.ptr()))); } pub fn getNumMatrices(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomLdsmTypeGetNumMatrices(self.ptr()); @@ -748,11 +748,11 @@ pub const CopyAtomNonExec2DGather4TmaLoadType = opaque { tmaFormat: ?TmaDataFormat = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGet(ctx.ptr(), @intFromEnum(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); + const result = c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getKind(self: *const Self) GatherScatterTmaLoad { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetKind(self.ptr()))); } pub fn getValType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetValType(self.ptr()).ptr.?); @@ -765,7 +765,7 @@ pub const CopyAtomNonExec2DGather4TmaLoadType = opaque { } pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @enumFromInt(value); + return if (value < 0) null else @fromBackingInt(@intCast(value)); } }; @@ -791,7 +791,7 @@ pub const CopyAtomNonExec2DScatter4TmaStoreType = opaque { tmaFormat: ?TmaDataFormat = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); + const result = c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -805,7 +805,7 @@ pub const CopyAtomNonExec2DScatter4TmaStoreType = opaque { } pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @enumFromInt(value); + return if (value < 0) null else @fromBackingInt(@intCast(value)); } }; @@ -832,11 +832,11 @@ pub const CopyAtomNonExecIm2ColTmaLoadType = opaque { tmaFormat: ?TmaDataFormat = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGet(ctx.ptr(), @intFromEnum(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); + const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getKind(self: *const Self) Im2ColTmaLoad { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetKind(self.ptr()))); } pub fn getValType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetValType(self.ptr()).ptr.?); @@ -849,7 +849,7 @@ pub const CopyAtomNonExecIm2ColTmaLoadType = opaque { } pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @enumFromInt(value); + return if (value < 0) null else @fromBackingInt(@intCast(value)); } }; @@ -875,7 +875,7 @@ pub const CopyAtomNonExecIm2ColTmaStoreType = opaque { tmaFormat: ?TmaDataFormat = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); + const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -889,7 +889,7 @@ pub const CopyAtomNonExecIm2ColTmaStoreType = opaque { } pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @enumFromInt(value); + return if (value < 0) null else @fromBackingInt(@intCast(value)); } }; @@ -916,11 +916,11 @@ pub const CopyAtomNonExecTiledTmaLoadType = opaque { tmaFormat: ?TmaDataFormat = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGet(ctx.ptr(), @intFromEnum(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); + const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getKind(self: *const Self) TiledTmaLoad { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetKind(self.ptr()))); } pub fn getValType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetValType(self.ptr()).ptr.?); @@ -933,7 +933,7 @@ pub const CopyAtomNonExecTiledTmaLoadType = opaque { } pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @enumFromInt(value); + return if (value < 0) null else @fromBackingInt(@intCast(value)); } }; @@ -960,11 +960,11 @@ pub const CopyAtomNonExecTiledTmaReduceType = opaque { tmaFormat: ?TmaDataFormat = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGet(ctx.ptr(), @intFromEnum(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); + const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getKind(self: *const Self) ReductionKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetKind(self.ptr()))); } pub fn getValType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetValType(self.ptr()).ptr.?); @@ -977,7 +977,7 @@ pub const CopyAtomNonExecTiledTmaReduceType = opaque { } pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @enumFromInt(value); + return if (value < 0) null else @fromBackingInt(@intCast(value)); } }; @@ -1003,7 +1003,7 @@ pub const CopyAtomNonExecTiledTmaStoreType = opaque { tmaFormat: ?TmaDataFormat = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @intFromEnum(v) else -1); + const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1017,7 +1017,7 @@ pub const CopyAtomNonExecTiledTmaStoreType = opaque { } pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @enumFromInt(value); + return if (value < 0) null else @fromBackingInt(@intCast(value)); } }; @@ -1046,7 +1046,7 @@ pub const CopyAtomR2GType = opaque { sharedSpace: SharedSpace = .CTA, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomR2GTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.memOrder), @intFromEnum(args.memScope), @intFromEnum(args.l1CacheEvictPriority), @intFromEnum(args.storeCacheMode), @intFromEnum(args.sharedSpace)); + const result = c.mlirCuteNVGPUCopyAtomR2GTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.l1CacheEvictPriority), @backingInt(args.storeCacheMode), @backingInt(args.sharedSpace)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1056,19 +1056,19 @@ pub const CopyAtomR2GType = opaque { return c.mlirCuteNVGPUCopyAtomR2GTypeGetCopyBits(self.ptr()); } pub fn getMemOrder(self: *const Self) MemOrderKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2GTypeGetMemOrder(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetMemOrder(self.ptr()))); } pub fn getMemScope(self: *const Self) MemScopeKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2GTypeGetMemScope(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetMemScope(self.ptr()))); } pub fn getL1CacheEvictPriority(self: *const Self) CacheEvictionPriority { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2GTypeGetL1CacheEvictPriority(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetL1CacheEvictPriority(self.ptr()))); } pub fn getStoreCacheMode(self: *const Self) StoreCacheMode { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2GTypeGetStoreCacheMode(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetStoreCacheMode(self.ptr()))); } pub fn getSharedSpace(self: *const Self) SharedSpace { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2GTypeGetSharedSpace(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetSharedSpace(self.ptr()))); } }; @@ -1095,7 +1095,7 @@ pub const CopyAtomR2SType = opaque { sharedSpace: SharedSpace = .CTA, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomR2STypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.memOrder), @intFromEnum(args.memScope), @intFromEnum(args.sharedSpace)); + const result = c.mlirCuteNVGPUCopyAtomR2STypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.sharedSpace)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1105,13 +1105,13 @@ pub const CopyAtomR2SType = opaque { return c.mlirCuteNVGPUCopyAtomR2STypeGetCopyBits(self.ptr()); } pub fn getMemOrder(self: *const Self) MemOrderKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2STypeGetMemOrder(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2STypeGetMemOrder(self.ptr()))); } pub fn getMemScope(self: *const Self) MemScopeKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2STypeGetMemScope(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2STypeGetMemScope(self.ptr()))); } pub fn getSharedSpace(self: *const Self) SharedSpace { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomR2STypeGetSharedSpace(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2STypeGetSharedSpace(self.ptr()))); } }; @@ -1138,7 +1138,7 @@ pub const CopyAtomS2RType = opaque { sharedSpace: SharedSpace = .CTA, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomS2RTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.memOrder), @intFromEnum(args.memScope), @intFromEnum(args.sharedSpace)); + const result = c.mlirCuteNVGPUCopyAtomS2RTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.sharedSpace)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1148,13 +1148,13 @@ pub const CopyAtomS2RType = opaque { return c.mlirCuteNVGPUCopyAtomS2RTypeGetCopyBits(self.ptr()); } pub fn getMemOrder(self: *const Self) MemOrderKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomS2RTypeGetMemOrder(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetMemOrder(self.ptr()))); } pub fn getMemScope(self: *const Self) MemScopeKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomS2RTypeGetMemScope(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetMemScope(self.ptr()))); } pub fn getSharedSpace(self: *const Self) SharedSpace { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomS2RTypeGetSharedSpace(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetSharedSpace(self.ptr()))); } }; @@ -1179,14 +1179,14 @@ pub const CopyAtomSIMTAsyncCopyType = opaque { copyBits: c_int, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGet(ctx.ptr(), args.valType.ptr(), @intFromEnum(args.cache), args.copyBits); + const result = c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGet(ctx.ptr(), args.valType.ptr(), @backingInt(args.cache), args.copyBits); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetValType(self.ptr()).ptr.?); } pub fn getCache(self: *const Self) LoadCacheMode { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCache(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCache(self.ptr()))); } pub fn getCopyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCopyBits(self.ptr()); @@ -1217,7 +1217,7 @@ pub const CopyAtomSIMTMultimemLdReduceType = opaque { memScope: MemScopeKind = .SYS, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.reduction), if (args.ldReduceAccPrecision) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, @intFromEnum(args.memOrder), @intFromEnum(args.memScope)); + const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.reduction), if (args.ldReduceAccPrecision) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, @backingInt(args.memOrder), @backingInt(args.memScope)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1227,16 +1227,16 @@ pub const CopyAtomSIMTMultimemLdReduceType = opaque { return c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetCopyBits(self.ptr()); } pub fn getReduction(self: *const Self) ReductionKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetReduction(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetReduction(self.ptr()))); } pub fn getLdReduceAccPrecision(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetLdReduceAccPrecision(self.ptr()).ptr); } pub fn getMemOrder(self: *const Self) MemOrderKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemOrder(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemOrder(self.ptr()))); } pub fn getMemScope(self: *const Self) MemScopeKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemScope(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemScope(self.ptr()))); } }; @@ -1263,7 +1263,7 @@ pub const CopyAtomSIMTMultimemRedType = opaque { memScope: MemScopeKind = .SYS, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.reduction), @intFromEnum(args.memOrder), @intFromEnum(args.memScope)); + const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.reduction), @backingInt(args.memOrder), @backingInt(args.memScope)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1273,13 +1273,13 @@ pub const CopyAtomSIMTMultimemRedType = opaque { return c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetCopyBits(self.ptr()); } pub fn getReduction(self: *const Self) ReductionKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetReduction(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetReduction(self.ptr()))); } pub fn getMemOrder(self: *const Self) MemOrderKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemOrder(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemOrder(self.ptr()))); } pub fn getMemScope(self: *const Self) MemScopeKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemScope(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemScope(self.ptr()))); } }; @@ -1305,7 +1305,7 @@ pub const CopyAtomSIMTMultimemStType = opaque { memScope: MemScopeKind = .SYS, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.memOrder), @intFromEnum(args.memScope)); + const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1315,10 +1315,10 @@ pub const CopyAtomSIMTMultimemStType = opaque { return c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetCopyBits(self.ptr()); } pub fn getMemOrder(self: *const Self) MemOrderKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemOrder(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemOrder(self.ptr()))); } pub fn getMemScope(self: *const Self) MemScopeKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemScope(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemScope(self.ptr()))); } }; @@ -1344,7 +1344,7 @@ pub const CopyAtomSIMTSyncCopyType = opaque { dstSpace: cute.AddressSpace = .generic, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.srcSpace), @intFromEnum(args.dstSpace)); + const result = c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.srcSpace), @backingInt(args.dstSpace)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1354,10 +1354,10 @@ pub const CopyAtomSIMTSyncCopyType = opaque { return c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetCopyBits(self.ptr()); } pub fn getSrcSpace(self: *const Self) cute.AddressSpace { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetSrcSpace(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetSrcSpace(self.ptr()))); } pub fn getDstSpace(self: *const Self) cute.AddressSpace { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetDstSpace(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetDstSpace(self.ptr()))); } }; @@ -1384,7 +1384,7 @@ pub const CopyAtomSM100CopyS2TType = opaque { broadcast: CopyS2TBroadcast = .none, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numCta, @intFromEnum(args.broadcast)); + const result = c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numCta, @backingInt(args.broadcast)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1400,7 +1400,7 @@ pub const CopyAtomSM100CopyS2TType = opaque { return c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumCta(self.ptr()); } pub fn getBroadcast(self: *const Self) CopyS2TBroadcast { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetBroadcast(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetBroadcast(self.ptr()))); } }; @@ -1428,7 +1428,7 @@ pub const CopyAtomSM100S2TCopyV2Type = opaque { broadcast: CopyS2TBroadcast = .none, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numCta, @intFromEnum(args.smemMajor), @intFromEnum(args.broadcast)); + const result = c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numCta, @backingInt(args.smemMajor), @backingInt(args.broadcast)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1444,10 +1444,10 @@ pub const CopyAtomSM100S2TCopyV2Type = opaque { return c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumCta(self.ptr()); } pub fn getSmemMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetSmemMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetSmemMajor(self.ptr()))); } pub fn getBroadcast(self: *const Self) CopyS2TBroadcast { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetBroadcast(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetBroadcast(self.ptr()))); } }; @@ -1562,7 +1562,7 @@ pub const CopyAtomSM107TmemLoadSPCompressType = opaque { nan: bool = false, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, @intFromEnum(args.redOp), args.red, args.nan); + const result = c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, @backingInt(args.redOp), args.red, args.nan); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1578,7 +1578,7 @@ pub const CopyAtomSM107TmemLoadSPCompressType = opaque { return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumRep(self.ptr()); } pub fn getRedOp(self: *const Self) TmemLoadRedOp { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRedOp(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRedOp(self.ptr()))); } pub fn getRed(self: *const Self) bool { return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRed(self.ptr()); @@ -1613,7 +1613,7 @@ pub const CopyAtomSM10xTmemLoadRedType = opaque { halfSplitOff: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, @intFromEnum(args.redOp), args.nan, if (args.halfSplitOff) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, @backingInt(args.redOp), args.nan, if (args.halfSplitOff) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1629,7 +1629,7 @@ pub const CopyAtomSM10xTmemLoadRedType = opaque { return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumRep(self.ptr()); } pub fn getRedOp(self: *const Self) TmemLoadRedOp { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetRedOp(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetRedOp(self.ptr()))); } pub fn getNan(self: *const Self) bool { return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNan(self.ptr()); @@ -1706,7 +1706,7 @@ pub const CopyAtomTmaLoadType = opaque { noFullyOobTile: bool = false, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomTmaLoadTypeGet(ctx.ptr(), args.valType.ptr(), args.sparsity, args.copyBits, @intFromEnum(args.mode), args.numCta, args.gStride.ptr(), args.mcast, if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }, args.override, args.noFullyOobTile); + const result = c.mlirCuteNVGPUCopyAtomTmaLoadTypeGet(ctx.ptr(), args.valType.ptr(), args.sparsity, args.copyBits, @backingInt(args.mode), args.numCta, args.gStride.ptr(), args.mcast, if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }, args.override, args.noFullyOobTile); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1719,7 +1719,7 @@ pub const CopyAtomTmaLoadType = opaque { return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetCopyBits(self.ptr()); } pub fn getMode(self: *const Self) TmaLoadMode { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetMode(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetMode(self.ptr()))); } pub fn getNumCta(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetNumCta(self.ptr()); @@ -1765,7 +1765,7 @@ pub const CopyAtomTmaReduceType = opaque { tmaGbasis: ?*const mlir.Type = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomTmaReduceTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @intFromEnum(args.mode), @intFromEnum(args.kind), args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }); + const result = c.mlirCuteNVGPUCopyAtomTmaReduceTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.mode), @backingInt(args.kind), args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1775,10 +1775,10 @@ pub const CopyAtomTmaReduceType = opaque { return c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetCopyBits(self.ptr()); } pub fn getMode(self: *const Self) TmaStoreMode { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetMode(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetMode(self.ptr()))); } pub fn getKind(self: *const Self) ReductionKind { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetKind(self.ptr()))); } pub fn getGStride(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetGStride(self.ptr()).ptr.?); @@ -1814,7 +1814,7 @@ pub const CopyAtomTmaStoreType = opaque { noFullyOobTile: bool = false, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.sparsity, args.copyBits, @intFromEnum(args.mode), args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }, args.override, args.noFullyOobTile); + const result = c.mlirCuteNVGPUCopyAtomTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.sparsity, args.copyBits, @backingInt(args.mode), args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }, args.override, args.noFullyOobTile); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValType(self: *const Self) *const mlir.Type { @@ -1827,7 +1827,7 @@ pub const CopyAtomTmaStoreType = opaque { return c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetCopyBits(self.ptr()); } pub fn getMode(self: *const Self) TmaStoreMode { - return @enumFromInt(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetMode(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetMode(self.ptr()))); } pub fn getGStride(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetGStride(self.ptr()).ptr.?); @@ -1874,7 +1874,7 @@ pub const MmaAtomSM100UMMABlockScaledSparseType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @intFromEnum(args.sparseMetadataFormat), @intFromEnum(args.aFragKind), args.vecSize, @intFromEnum(args.archPromote), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.archPromote), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -1884,10 +1884,10 @@ pub const MmaAtomSM100UMMABlockScaledSparseType = opaque { return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetNumCta(self.ptr()); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAMajor(self.ptr()))); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBMajor(self.ptr()))); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAType(self.ptr()).ptr.?); @@ -1902,16 +1902,16 @@ pub const MmaAtomSM100UMMABlockScaledSparseType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSfType(self.ptr()).ptr.?); } pub fn getSparseMetadataFormat(self: *const Self) SparseMetadataFormat { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSparseMetadataFormat(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSparseMetadataFormat(self.ptr()))); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAFragKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAFragKind(self.ptr()))); } pub fn getVecSize(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetVecSize(self.ptr()); } pub fn getArchPromote(self: *const Self) Arch { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetArchPromote(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetArchPromote(self.ptr()))); } pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetIntOverflow(self.ptr()).ptr); @@ -1948,7 +1948,7 @@ pub const MmaAtomSM100UMMABlockScaledType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @intFromEnum(args.aFragKind), args.vecSize, @intFromEnum(args.archPromote), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.archPromote), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -1958,10 +1958,10 @@ pub const MmaAtomSM100UMMABlockScaledType = opaque { return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetNumCta(self.ptr()); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAMajor(self.ptr()))); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBMajor(self.ptr()))); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAType(self.ptr()).ptr.?); @@ -1976,13 +1976,13 @@ pub const MmaAtomSM100UMMABlockScaledType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetSfType(self.ptr()).ptr.?); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAFragKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAFragKind(self.ptr()))); } pub fn getVecSize(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetVecSize(self.ptr()); } pub fn getArchPromote(self: *const Self) Arch { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetArchPromote(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetArchPromote(self.ptr()))); } pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetIntOverflow(self.ptr()).ptr); @@ -2019,7 +2019,7 @@ pub const MmaAtomSM100UMMASparseType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.eType.ptr(), @intFromEnum(args.sparseMetadataFormat), @intFromEnum(args.aFragKind), args.cScaleExp, if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.eType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.cScaleExp, if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -2029,10 +2029,10 @@ pub const MmaAtomSM100UMMASparseType = opaque { return c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetNumCta(self.ptr()); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAMajor(self.ptr()))); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBMajor(self.ptr()))); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAType(self.ptr()).ptr.?); @@ -2047,10 +2047,10 @@ pub const MmaAtomSM100UMMASparseType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetEType(self.ptr()).ptr.?); } pub fn getSparseMetadataFormat(self: *const Self) SparseMetadataFormat { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetSparseMetadataFormat(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetSparseMetadataFormat(self.ptr()))); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAFragKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAFragKind(self.ptr()))); } pub fn getCScaleExp(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCScaleExp(self.ptr()); @@ -2088,7 +2088,7 @@ pub const MmaAtomSM100UMMAType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM100UMMATypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @intFromEnum(args.aFragKind), args.cScaleExp, if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM100UMMATypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.aFragKind), args.cScaleExp, if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -2098,10 +2098,10 @@ pub const MmaAtomSM100UMMAType = opaque { return c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetNumCta(self.ptr()); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAMajor(self.ptr()))); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetBMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetBMajor(self.ptr()))); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAType(self.ptr()).ptr.?); @@ -2113,7 +2113,7 @@ pub const MmaAtomSM100UMMAType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetCType(self.ptr()).ptr.?); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAFragKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAFragKind(self.ptr()))); } pub fn getCScaleExp(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetCScaleExp(self.ptr()); @@ -2155,7 +2155,7 @@ pub const MmaAtomSM107UMMABlockScaledSparseType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @intFromEnum(args.sparseMetadataFormat), @intFromEnum(args.aFragKind), args.vecSize, @intFromEnum(args.aCollectorOp), @intFromEnum(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -2165,10 +2165,10 @@ pub const MmaAtomSM107UMMABlockScaledSparseType = opaque { return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetNumCta(self.ptr()); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAMajor(self.ptr()))); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBMajor(self.ptr()))); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAType(self.ptr()).ptr.?); @@ -2183,19 +2183,19 @@ pub const MmaAtomSM107UMMABlockScaledSparseType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSfType(self.ptr()).ptr.?); } pub fn getSparseMetadataFormat(self: *const Self) SparseMetadataFormat { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSparseMetadataFormat(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSparseMetadataFormat(self.ptr()))); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAFragKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAFragKind(self.ptr()))); } pub fn getVecSize(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetVecSize(self.ptr()); } pub fn getACollectorOp(self: *const Self) MmaCollectorOp { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetACollectorOp(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetACollectorOp(self.ptr()))); } pub fn getBCollectorOp(self: *const Self) MmaCollectorOp { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBCollectorOp(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBCollectorOp(self.ptr()))); } pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetIntOverflow(self.ptr()).ptr); @@ -2233,7 +2233,7 @@ pub const MmaAtomSM107UMMABlockScaledType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @intFromEnum(args.aFragKind), args.vecSize, @intFromEnum(args.aCollectorOp), @intFromEnum(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -2243,10 +2243,10 @@ pub const MmaAtomSM107UMMABlockScaledType = opaque { return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetNumCta(self.ptr()); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAMajor(self.ptr()))); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBMajor(self.ptr()))); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAType(self.ptr()).ptr.?); @@ -2261,16 +2261,16 @@ pub const MmaAtomSM107UMMABlockScaledType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetSfType(self.ptr()).ptr.?); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAFragKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAFragKind(self.ptr()))); } pub fn getVecSize(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetVecSize(self.ptr()); } pub fn getACollectorOp(self: *const Self) MmaCollectorOp { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetACollectorOp(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetACollectorOp(self.ptr()))); } pub fn getBCollectorOp(self: *const Self) MmaCollectorOp { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBCollectorOp(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBCollectorOp(self.ptr()))); } pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetIntOverflow(self.ptr()).ptr); @@ -2309,7 +2309,7 @@ pub const MmaAtomSM107UMMASparseType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.eType.ptr(), @intFromEnum(args.sparseMetadataFormat), @intFromEnum(args.aFragKind), args.cScaleExp, @intFromEnum(args.aCollectorOp), @intFromEnum(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.eType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.cScaleExp, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -2319,10 +2319,10 @@ pub const MmaAtomSM107UMMASparseType = opaque { return c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetNumCta(self.ptr()); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAMajor(self.ptr()))); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBMajor(self.ptr()))); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAType(self.ptr()).ptr.?); @@ -2337,19 +2337,19 @@ pub const MmaAtomSM107UMMASparseType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetEType(self.ptr()).ptr.?); } pub fn getSparseMetadataFormat(self: *const Self) SparseMetadataFormat { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetSparseMetadataFormat(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetSparseMetadataFormat(self.ptr()))); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAFragKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAFragKind(self.ptr()))); } pub fn getCScaleExp(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCScaleExp(self.ptr()); } pub fn getACollectorOp(self: *const Self) MmaCollectorOp { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetACollectorOp(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetACollectorOp(self.ptr()))); } pub fn getBCollectorOp(self: *const Self) MmaCollectorOp { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBCollectorOp(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBCollectorOp(self.ptr()))); } pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetIntOverflow(self.ptr()).ptr); @@ -2386,7 +2386,7 @@ pub const MmaAtomSM107UMMAType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM107UMMATypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @intFromEnum(args.aFragKind), args.cScaleExp, @intFromEnum(args.aCollectorOp), @intFromEnum(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM107UMMATypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.aFragKind), args.cScaleExp, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -2396,10 +2396,10 @@ pub const MmaAtomSM107UMMAType = opaque { return c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetNumCta(self.ptr()); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAMajor(self.ptr()))); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBMajor(self.ptr()))); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAType(self.ptr()).ptr.?); @@ -2411,16 +2411,16 @@ pub const MmaAtomSM107UMMAType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetCType(self.ptr()).ptr.?); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAFragKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAFragKind(self.ptr()))); } pub fn getCScaleExp(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetCScaleExp(self.ptr()); } pub fn getACollectorOp(self: *const Self) MmaCollectorOp { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetACollectorOp(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetACollectorOp(self.ptr()))); } pub fn getBCollectorOp(self: *const Self) MmaCollectorOp { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBCollectorOp(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBCollectorOp(self.ptr()))); } pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetIntOverflow(self.ptr()).ptr); @@ -2502,7 +2502,7 @@ pub const MmaAtomSM80SparseType = opaque { intOverflow: ?*const mlir.Attribute = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM80SparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @intFromEnum(args.sparseMetadataFormat), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + const result = c.mlirCuteNVGPUMmaAtomSM80SparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.sparseMetadataFormat), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { @@ -2518,7 +2518,7 @@ pub const MmaAtomSM80SparseType = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetCType(self.ptr()).ptr.?); } pub fn getSparseMetadataFormat(self: *const Self) SparseMetadataFormat { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetSparseMetadataFormat(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetSparseMetadataFormat(self.ptr()))); } pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetIntOverflow(self.ptr()).ptr); @@ -2639,17 +2639,17 @@ pub const MmaAtomSM90Type = opaque { bNeg: bool = false, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM90TypeGet(ctx.ptr(), args.shapeMnk.ptr(), @intFromEnum(args.aMajor), @intFromEnum(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @intFromEnum(args.aFragKind), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, args.aNeg, args.bNeg); + const result = c.mlirCuteNVGPUMmaAtomSM90TypeGet(ctx.ptr(), args.shapeMnk.ptr(), @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.aFragKind), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, args.aNeg, args.bNeg); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetShapeMnk(self.ptr()).ptr.?); } pub fn getAMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM90TypeGetAMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetAMajor(self.ptr()))); } pub fn getBMajor(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM90TypeGetBMajor(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetBMajor(self.ptr()))); } pub fn getAType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetAType(self.ptr()).ptr.?); @@ -2661,7 +2661,7 @@ pub const MmaAtomSM90Type = opaque { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetCType(self.ptr()).ptr.?); } pub fn getAFragKind(self: *const Self) MmaFragKind { - return @enumFromInt(c.mlirCuteNVGPUMmaAtomSM90TypeGetAFragKind(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetAFragKind(self.ptr()))); } pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetIntOverflow(self.ptr()).ptr); @@ -2932,11 +2932,11 @@ pub const AtomCopyFieldBulkCopyG2SAttr = opaque { value: AtomCopyFieldBulkCopyG2S, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldBulkCopyG2S { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGetValue(self.ptr()))); } }; @@ -2959,11 +2959,11 @@ pub const AtomCopyFieldBulkCopyS2GAttr = opaque { value: AtomCopyFieldBulkCopyS2G, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldBulkCopyS2G { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGetValue(self.ptr()))); } }; @@ -2986,11 +2986,11 @@ pub const AtomCopyFieldBulkCopyS2SAttr = opaque { value: AtomCopyFieldBulkCopyS2S, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldBulkCopyS2S { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGetValue(self.ptr()))); } }; @@ -3013,11 +3013,11 @@ pub const AtomCopyFieldDsmemStoreAttr = opaque { value: AtomCopyFieldDsmemStore, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldDsmemStore { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGetValue(self.ptr()))); } }; @@ -3040,11 +3040,11 @@ pub const AtomCopyFieldLoadGlobalAttr = opaque { value: AtomCopyFieldLoadGlobal, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldLoadGlobal { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGetValue(self.ptr()))); } }; @@ -3067,11 +3067,11 @@ pub const AtomCopyFieldNonExec2DGather4TmaLoadAttr = opaque { value: AtomCopyFieldNonExec2DGather4TmaLoad, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldNonExec2DGather4TmaLoad { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGetValue(self.ptr()))); } }; @@ -3094,11 +3094,11 @@ pub const AtomCopyFieldNonExec2DScatter4TmaStoreAttr = opaque { value: AtomCopyFieldNonExec2DScatter4TmaStore, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldNonExec2DScatter4TmaStore { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGetValue(self.ptr()))); } }; @@ -3121,11 +3121,11 @@ pub const AtomCopyFieldNonExecIm2ColTmaLoadAttr = opaque { value: AtomCopyFieldNonExecIm2ColTmaLoad, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldNonExecIm2ColTmaLoad { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGetValue(self.ptr()))); } }; @@ -3148,11 +3148,11 @@ pub const AtomCopyFieldNonExecIm2ColTmaStoreAttr = opaque { value: AtomCopyFieldNonExecIm2ColTmaStore, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldNonExecIm2ColTmaStore { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGetValue(self.ptr()))); } }; @@ -3175,11 +3175,11 @@ pub const AtomCopyFieldNonExecTiledTmaLoadAttr = opaque { value: AtomCopyFieldNonExecTiledTmaLoad, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldNonExecTiledTmaLoad { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGetValue(self.ptr()))); } }; @@ -3202,11 +3202,11 @@ pub const AtomCopyFieldNonExecTiledTmaReduceAttr = opaque { value: AtomCopyFieldNonExecTiledTmaReduce, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldNonExecTiledTmaReduce { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGetValue(self.ptr()))); } }; @@ -3229,11 +3229,11 @@ pub const AtomCopyFieldNonExecTiledTmaStoreAttr = opaque { value: AtomCopyFieldNonExecTiledTmaStore, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldNonExecTiledTmaStore { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGetValue(self.ptr()))); } }; @@ -3256,11 +3256,11 @@ pub const AtomCopyFieldStoreGlobalAttr = opaque { value: AtomCopyFieldStoreGlobal, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldStoreGlobal { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGetValue(self.ptr()))); } }; @@ -3283,11 +3283,11 @@ pub const AtomCopyFieldTmaLoadAttr = opaque { value: AtomCopyFieldTmaLoad, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldTmaLoad { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGetValue(self.ptr()))); } }; @@ -3310,11 +3310,11 @@ pub const AtomCopyFieldTmaReduceAttr = opaque { value: AtomCopyFieldTmaReduce, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldTmaReduce { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGetValue(self.ptr()))); } }; @@ -3337,11 +3337,11 @@ pub const AtomCopyFieldTmaStoreAttr = opaque { value: AtomCopyFieldTmaStore, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomCopyFieldTmaStore { - return @enumFromInt(c.mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGetValue(self.ptr()))); } }; @@ -3364,11 +3364,11 @@ pub const AtomMmaFieldSM100Attr = opaque { value: AtomMmaFieldSM100, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM100AttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomMmaFieldSM100AttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomMmaFieldSM100 { - return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM100AttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100AttrGetValue(self.ptr()))); } }; @@ -3391,11 +3391,11 @@ pub const AtomMmaFieldSM100BlockScaledAttr = opaque { value: AtomMmaFieldSM100BlockScaled, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomMmaFieldSM100BlockScaled { - return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGetValue(self.ptr()))); } }; @@ -3418,11 +3418,11 @@ pub const AtomMmaFieldSM100BlockScaledSparseAttr = opaque { value: AtomMmaFieldSM100BlockScaledSparse, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomMmaFieldSM100BlockScaledSparse { - return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGetValue(self.ptr()))); } }; @@ -3445,11 +3445,11 @@ pub const AtomMmaFieldSM100SparseAttr = opaque { value: AtomMmaFieldSM100Sparse, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomMmaFieldSM100Sparse { - return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGetValue(self.ptr()))); } }; @@ -3472,11 +3472,11 @@ pub const AtomMmaFieldSM120BlockScaledAttr = opaque { value: AtomMmaFieldSM120BlockScaled, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomMmaFieldSM120BlockScaled { - return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGetValue(self.ptr()))); } }; @@ -3499,11 +3499,11 @@ pub const AtomMmaFieldSM80SparseAttr = opaque { value: AtomMmaFieldSM80Sparse, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomMmaFieldSM80Sparse { - return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGetValue(self.ptr()))); } }; @@ -3526,11 +3526,11 @@ pub const AtomMmaFieldSM90Attr = opaque { value: AtomMmaFieldSM90, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM90AttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUAtomMmaFieldSM90AttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomMmaFieldSM90 { - return @enumFromInt(c.mlirCuteNVGPUAtomMmaFieldSM90AttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM90AttrGetValue(self.ptr()))); } }; @@ -3553,11 +3553,11 @@ pub const BinaryOpAttr = opaque { value: BinaryOp, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUBinaryOpAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUBinaryOpAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) BinaryOp { - return @enumFromInt(c.mlirCuteNVGPUBinaryOpAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUBinaryOpAttrGetValue(self.ptr()))); } }; @@ -3580,11 +3580,11 @@ pub const CopyS2TBroadcastAttr = opaque { value: CopyS2TBroadcast, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyS2TBroadcastAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUCopyS2TBroadcastAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) CopyS2TBroadcast { - return @enumFromInt(c.mlirCuteNVGPUCopyS2TBroadcastAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyS2TBroadcastAttrGetValue(self.ptr()))); } }; @@ -3607,11 +3607,11 @@ pub const GatherScatterTmaLoadAttr = opaque { value: GatherScatterTmaLoad, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUGatherScatterTmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUGatherScatterTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) GatherScatterTmaLoad { - return @enumFromInt(c.mlirCuteNVGPUGatherScatterTmaLoadAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUGatherScatterTmaLoadAttrGetValue(self.ptr()))); } }; @@ -3634,11 +3634,11 @@ pub const Im2ColTmaLoadAttr = opaque { value: Im2ColTmaLoad, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUIm2ColTmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUIm2ColTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) Im2ColTmaLoad { - return @enumFromInt(c.mlirCuteNVGPUIm2ColTmaLoadAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUIm2ColTmaLoadAttrGetValue(self.ptr()))); } }; @@ -3661,11 +3661,11 @@ pub const LdReduceAccPrecisionKindAttr = opaque { value: LdReduceAccPrecisionKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPULdReduceAccPrecisionKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPULdReduceAccPrecisionKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) LdReduceAccPrecisionKind { - return @enumFromInt(c.mlirCuteNVGPULdReduceAccPrecisionKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPULdReduceAccPrecisionKindAttrGetValue(self.ptr()))); } }; @@ -3688,11 +3688,11 @@ pub const LdsmSzPatternAttr = opaque { value: LdsmSzPattern, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPULdsmSzPatternAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPULdsmSzPatternAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) LdsmSzPattern { - return @enumFromInt(c.mlirCuteNVGPULdsmSzPatternAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPULdsmSzPatternAttrGetValue(self.ptr()))); } }; @@ -3715,11 +3715,11 @@ pub const LoadCacheModeAttr = opaque { value: LoadCacheMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPULoadCacheModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPULoadCacheModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) LoadCacheMode { - return @enumFromInt(c.mlirCuteNVGPULoadCacheModeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPULoadCacheModeAttrGetValue(self.ptr()))); } }; @@ -3742,11 +3742,11 @@ pub const MMAIntOverflowAttr = opaque { value: MMAIntOverflow, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMMAIntOverflowAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUMMAIntOverflowAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MMAIntOverflow { - return @enumFromInt(c.mlirCuteNVGPUMMAIntOverflowAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMMAIntOverflowAttrGetValue(self.ptr()))); } }; @@ -3769,11 +3769,11 @@ pub const MajorModeAttr = opaque { value: cute.MajorMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMajorModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUMajorModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUMajorModeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMajorModeAttrGetValue(self.ptr()))); } }; @@ -3796,11 +3796,11 @@ pub const MmaCollectorOpAttr = opaque { value: MmaCollectorOp, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaCollectorOpAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUMmaCollectorOpAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MmaCollectorOp { - return @enumFromInt(c.mlirCuteNVGPUMmaCollectorOpAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaCollectorOpAttrGetValue(self.ptr()))); } }; @@ -3823,11 +3823,11 @@ pub const MmaFragKindAttr = opaque { value: MmaFragKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaFragKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUMmaFragKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MmaFragKind { - return @enumFromInt(c.mlirCuteNVGPUMmaFragKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaFragKindAttrGetValue(self.ptr()))); } }; @@ -3873,11 +3873,11 @@ pub const ReductionKindAttr = opaque { value: ReductionKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUReductionKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUReductionKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) ReductionKind { - return @enumFromInt(c.mlirCuteNVGPUReductionKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUReductionKindAttrGetValue(self.ptr()))); } }; @@ -3902,14 +3902,14 @@ pub const RmemFrgAttr = opaque { deriveElemType: bool = false, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPURmemFrgAttrGet(ctx.ptr(), args.valueType.ptr(), @intFromEnum(args.operand), args.deriveElemType); + const result = c.mlirCuteNVGPURmemFrgAttrGet(ctx.ptr(), args.valueType.ptr(), @backingInt(args.operand), args.deriveElemType); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValueType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPURmemFrgAttrGetValueType(self.ptr()).ptr.?); } pub fn getOperand(self: *const Self) cute.MmaOperand { - return @enumFromInt(c.mlirCuteNVGPURmemFrgAttrGetOperand(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPURmemFrgAttrGetOperand(self.ptr()))); } pub fn getDeriveElemType(self: *const Self) bool { return c.mlirCuteNVGPURmemFrgAttrGetDeriveElemType(self.ptr()); @@ -3935,11 +3935,11 @@ pub const SM100CircularSmemFrgAttr = opaque { majorMode: cute.MajorMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSM100CircularSmemFrgAttrGet(ctx.ptr(), @intFromEnum(args.majorMode)); + const result = c.mlirCuteNVGPUSM100CircularSmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getMajorMode(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUSM100CircularSmemFrgAttrGetMajorMode(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100CircularSmemFrgAttrGetMajorMode(self.ptr()))); } }; @@ -3962,11 +3962,11 @@ pub const SM100SmemFrgAttr = opaque { majorMode: cute.MajorMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSM100SmemFrgAttrGet(ctx.ptr(), @intFromEnum(args.majorMode)); + const result = c.mlirCuteNVGPUSM100SmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getMajorMode(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUSM100SmemFrgAttrGetMajorMode(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100SmemFrgAttrGetMajorMode(self.ptr()))); } }; @@ -4023,7 +4023,7 @@ pub const SM100TmemFrgAttr = opaque { tmemAllocMode: TmemAllocMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSM100TmemFrgAttrGet(ctx.ptr(), args.dataType.ptr(), args.storageType.ptr(), args.ctaGroup, @intFromEnum(args.tmemAllocMode)); + const result = c.mlirCuteNVGPUSM100TmemFrgAttrGet(ctx.ptr(), args.dataType.ptr(), args.storageType.ptr(), args.ctaGroup, @backingInt(args.tmemAllocMode)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getDataType(self: *const Self) *const mlir.Type { @@ -4036,7 +4036,7 @@ pub const SM100TmemFrgAttr = opaque { return c.mlirCuteNVGPUSM100TmemFrgAttrGetCtaGroup(self.ptr()); } pub fn getTmemAllocMode(self: *const Self) TmemAllocMode { - return @enumFromInt(c.mlirCuteNVGPUSM100TmemFrgAttrGetTmemAllocMode(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100TmemFrgAttrGetTmemAllocMode(self.ptr()))); } }; @@ -4063,7 +4063,7 @@ pub const SM100TmemSfFrgAttr = opaque { tmemAllocMode: TmemAllocMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSM100TmemSfFrgAttrGet(ctx.ptr(), args.sfType.ptr(), args.sfVecSize, args.ctaGroup, args.isSfa, @intFromEnum(args.tmemAllocMode)); + const result = c.mlirCuteNVGPUSM100TmemSfFrgAttrGet(ctx.ptr(), args.sfType.ptr(), args.sfVecSize, args.ctaGroup, args.isSfa, @backingInt(args.tmemAllocMode)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getSfType(self: *const Self) *const mlir.Type { @@ -4079,7 +4079,7 @@ pub const SM100TmemSfFrgAttr = opaque { return c.mlirCuteNVGPUSM100TmemSfFrgAttrGetIsSfa(self.ptr()); } pub fn getTmemAllocMode(self: *const Self) TmemAllocMode { - return @enumFromInt(c.mlirCuteNVGPUSM100TmemSfFrgAttrGetTmemAllocMode(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100TmemSfFrgAttrGetTmemAllocMode(self.ptr()))); } }; @@ -4102,11 +4102,11 @@ pub const SM107SmemFrgAttr = opaque { majorMode: cute.MajorMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSM107SmemFrgAttrGet(ctx.ptr(), @intFromEnum(args.majorMode)); + const result = c.mlirCuteNVGPUSM107SmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getMajorMode(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUSM107SmemFrgAttrGetMajorMode(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM107SmemFrgAttrGetMajorMode(self.ptr()))); } }; @@ -4129,11 +4129,11 @@ pub const SM90SmemFrgAttr = opaque { majorMode: cute.MajorMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSM90SmemFrgAttrGet(ctx.ptr(), @intFromEnum(args.majorMode)); + const result = c.mlirCuteNVGPUSM90SmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getMajorMode(self: *const Self) cute.MajorMode { - return @enumFromInt(c.mlirCuteNVGPUSM90SmemFrgAttrGetMajorMode(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM90SmemFrgAttrGetMajorMode(self.ptr()))); } }; @@ -4156,11 +4156,11 @@ pub const TiledTmaLoadAttr = opaque { value: TiledTmaLoad, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTiledTmaLoadAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUTiledTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TiledTmaLoad { - return @enumFromInt(c.mlirCuteNVGPUTiledTmaLoadAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUTiledTmaLoadAttrGetValue(self.ptr()))); } }; @@ -4183,11 +4183,11 @@ pub const TmaDataFormatAttr = opaque { value: TmaDataFormat, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTmaDataFormatAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUTmaDataFormatAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TmaDataFormat { - return @enumFromInt(c.mlirCuteNVGPUTmaDataFormatAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmaDataFormatAttrGetValue(self.ptr()))); } }; @@ -4210,11 +4210,11 @@ pub const TmaLoadModeAttr = opaque { value: TmaLoadMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTmaLoadModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUTmaLoadModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TmaLoadMode { - return @enumFromInt(c.mlirCuteNVGPUTmaLoadModeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmaLoadModeAttrGetValue(self.ptr()))); } }; @@ -4237,11 +4237,11 @@ pub const TmaStoreModeAttr = opaque { value: TmaStoreMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTmaStoreModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUTmaStoreModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TmaStoreMode { - return @enumFromInt(c.mlirCuteNVGPUTmaStoreModeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmaStoreModeAttrGetValue(self.ptr()))); } }; @@ -4264,11 +4264,11 @@ pub const TmemAllocModeAttr = opaque { value: TmemAllocMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTmemAllocModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUTmemAllocModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TmemAllocMode { - return @enumFromInt(c.mlirCuteNVGPUTmemAllocModeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmemAllocModeAttrGetValue(self.ptr()))); } }; @@ -4291,11 +4291,11 @@ pub const TmemLoadRedOpAttr = opaque { value: TmemLoadRedOp, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTmemLoadRedOpAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVGPUTmemLoadRedOpAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TmemLoadRedOp { - return @enumFromInt(c.mlirCuteNVGPUTmemLoadRedOpAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmemLoadRedOpAttrGetValue(self.ptr()))); } }; diff --git a/mlir/dialects/cute_ir/nvvm.zig b/mlir/dialects/cute_ir/nvvm.zig index 75cfcae3..bfcd16b5 100644 --- a/mlir/dialects/cute_ir/nvvm.zig +++ b/mlir/dialects/cute_ir/nvvm.zig @@ -835,11 +835,11 @@ pub const AtomicOpKindAttr = opaque { value: AtomicOpKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMAtomicOpKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMAtomicOpKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) AtomicOpKind { - return @enumFromInt(c.mlirCuteNVVMAtomicOpKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMAtomicOpKindAttrGetValue(self.ptr()))); } }; @@ -862,11 +862,11 @@ pub const BarrierReductionAttr = opaque { value: BarrierReduction, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMBarrierReductionAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMBarrierReductionAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) BarrierReduction { - return @enumFromInt(c.mlirCuteNVVMBarrierReductionAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMBarrierReductionAttrGetValue(self.ptr()))); } }; @@ -889,11 +889,11 @@ pub const BarrierReduxKindAttr = opaque { value: BarrierReduxKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMBarrierReduxKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMBarrierReduxKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) BarrierReduxKind { - return @enumFromInt(c.mlirCuteNVVMBarrierReduxKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMBarrierReduxKindAttrGetValue(self.ptr()))); } }; @@ -916,11 +916,11 @@ pub const BlockScaleFormatAttr = opaque { value: BlockScaleFormat, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMBlockScaleFormatAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMBlockScaleFormatAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) BlockScaleFormat { - return @enumFromInt(c.mlirCuteNVVMBlockScaleFormatAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMBlockScaleFormatAttrGetValue(self.ptr()))); } }; @@ -943,11 +943,11 @@ pub const CTAGroupKindAttr = opaque { value: CTAGroupKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMCTAGroupKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMCTAGroupKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) CTAGroupKind { - return @enumFromInt(c.mlirCuteNVVMCTAGroupKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMCTAGroupKindAttrGetValue(self.ptr()))); } }; @@ -970,11 +970,11 @@ pub const CVTPackFloatKindAttr = opaque { value: CVTPackFloatKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMCVTPackFloatKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMCVTPackFloatKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) CVTPackFloatKind { - return @enumFromInt(c.mlirCuteNVVMCVTPackFloatKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMCVTPackFloatKindAttrGetValue(self.ptr()))); } }; @@ -997,11 +997,11 @@ pub const CacheEvictionPriorityAttr = opaque { value: CacheEvictionPriority, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMCacheEvictionPriorityAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMCacheEvictionPriorityAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) CacheEvictionPriority { - return @enumFromInt(c.mlirCuteNVVMCacheEvictionPriorityAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMCacheEvictionPriorityAttrGetValue(self.ptr()))); } }; @@ -1024,11 +1024,11 @@ pub const ClusterLaunchControlQueryTypeAttr = opaque { value: ClusterLaunchControlQueryType, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMClusterLaunchControlQueryTypeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMClusterLaunchControlQueryTypeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) ClusterLaunchControlQueryType { - return @enumFromInt(c.mlirCuteNVVMClusterLaunchControlQueryTypeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMClusterLaunchControlQueryTypeAttrGetValue(self.ptr()))); } }; @@ -1051,11 +1051,11 @@ pub const CompareOpKindAttr = opaque { value: CompareOpKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMCompareOpKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMCompareOpKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) CompareOpKind { - return @enumFromInt(c.mlirCuteNVVMCompareOpKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMCompareOpKindAttrGetValue(self.ptr()))); } }; @@ -1078,11 +1078,11 @@ pub const ConvertFP4TypeAttr = opaque { value: ConvertFP4Type, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMConvertFP4TypeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMConvertFP4TypeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) ConvertFP4Type { - return @enumFromInt(c.mlirCuteNVVMConvertFP4TypeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMConvertFP4TypeAttrGetValue(self.ptr()))); } }; @@ -1105,11 +1105,11 @@ pub const ConvertFP8TypeAttr = opaque { value: ConvertFP8Type, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMConvertFP8TypeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMConvertFP8TypeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) ConvertFP8Type { - return @enumFromInt(c.mlirCuteNVVMConvertFP8TypeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMConvertFP8TypeAttrGetValue(self.ptr()))); } }; @@ -1132,11 +1132,11 @@ pub const ConvertScaleKindAttr = opaque { value: ConvertScaleKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMConvertScaleKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMConvertScaleKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) ConvertScaleKind { - return @enumFromInt(c.mlirCuteNVVMConvertScaleKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMConvertScaleKindAttrGetValue(self.ptr()))); } }; @@ -1159,11 +1159,11 @@ pub const DotAccumulateTypeAttr = opaque { value: DotAccumulateType, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMDotAccumulateTypeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMDotAccumulateTypeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) DotAccumulateType { - return @enumFromInt(c.mlirCuteNVVMDotAccumulateTypeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMDotAccumulateTypeAttrGetValue(self.ptr()))); } }; @@ -1186,11 +1186,11 @@ pub const EvictKindAttr = opaque { value: EvictKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMEvictKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMEvictKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) EvictKind { - return @enumFromInt(c.mlirCuteNVVMEvictKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMEvictKindAttrGetValue(self.ptr()))); } }; @@ -1213,11 +1213,11 @@ pub const FPRoundingModeAttr = opaque { value: FPRoundingMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMFPRoundingModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMFPRoundingModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) FPRoundingMode { - return @enumFromInt(c.mlirCuteNVVMFPRoundingModeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMFPRoundingModeAttrGetValue(self.ptr()))); } }; @@ -1240,11 +1240,11 @@ pub const GridDepActionKindAttr = opaque { value: GridDepActionKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMGridDepActionKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMGridDepActionKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) GridDepActionKind { - return @enumFromInt(c.mlirCuteNVVMGridDepActionKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMGridDepActionKindAttrGetValue(self.ptr()))); } }; @@ -1267,11 +1267,11 @@ pub const IntegerRoundingModeAttr = opaque { value: IntegerRoundingMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMIntegerRoundingModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMIntegerRoundingModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) IntegerRoundingMode { - return @enumFromInt(c.mlirCuteNVVMIntegerRoundingModeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMIntegerRoundingModeAttrGetValue(self.ptr()))); } }; @@ -1294,11 +1294,11 @@ pub const L2PrefetchSizeAttr = opaque { value: L2PrefetchSize, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVML2PrefetchSizeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVML2PrefetchSizeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) L2PrefetchSize { - return @enumFromInt(c.mlirCuteNVVML2PrefetchSizeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVML2PrefetchSizeAttrGetValue(self.ptr()))); } }; @@ -1321,11 +1321,11 @@ pub const LdStMatrixEltTypeAttr = opaque { value: LdStMatrixEltType, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMLdStMatrixEltTypeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMLdStMatrixEltTypeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) LdStMatrixEltType { - return @enumFromInt(c.mlirCuteNVVMLdStMatrixEltTypeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMLdStMatrixEltTypeAttrGetValue(self.ptr()))); } }; @@ -1348,11 +1348,11 @@ pub const LoadCacheModifierExtKindAttr = opaque { value: LoadCacheModifierExtKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMLoadCacheModifierExtKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMLoadCacheModifierExtKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) LoadCacheModifierExtKind { - return @enumFromInt(c.mlirCuteNVVMLoadCacheModifierExtKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMLoadCacheModifierExtKindAttrGetValue(self.ptr()))); } }; @@ -1375,11 +1375,11 @@ pub const LoadCacheModifierKindAttr = opaque { value: LoadCacheModifierKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMLoadCacheModifierKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMLoadCacheModifierKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) LoadCacheModifierKind { - return @enumFromInt(c.mlirCuteNVVMLoadCacheModifierKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMLoadCacheModifierKindAttrGetValue(self.ptr()))); } }; @@ -1402,11 +1402,11 @@ pub const LoadShapeAttr = opaque { value: LoadShape, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMLoadShapeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMLoadShapeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) LoadShape { - return @enumFromInt(c.mlirCuteNVVMLoadShapeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMLoadShapeAttrGetValue(self.ptr()))); } }; @@ -1429,11 +1429,11 @@ pub const LoadSrcFormatAttr = opaque { value: LoadSrcFormat, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMLoadSrcFormatAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMLoadSrcFormatAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) LoadSrcFormat { - return @enumFromInt(c.mlirCuteNVVMLoadSrcFormatAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMLoadSrcFormatAttrGetValue(self.ptr()))); } }; @@ -1456,11 +1456,11 @@ pub const MBarrierLayoutAttr = opaque { value: MBarrierLayout, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMBarrierLayoutAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMMBarrierLayoutAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MBarrierLayout { - return @enumFromInt(c.mlirCuteNVVMMBarrierLayoutAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierLayoutAttrGetValue(self.ptr()))); } }; @@ -1483,11 +1483,11 @@ pub const MBarrierPhaseAttr = opaque { value: MBarrierPhase, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMBarrierPhaseAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMMBarrierPhaseAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MBarrierPhase { - return @enumFromInt(c.mlirCuteNVVMMBarrierPhaseAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierPhaseAttrGetValue(self.ptr()))); } }; @@ -1510,11 +1510,11 @@ pub const MBarrierScopeKindAttr = opaque { value: MBarrierScopeKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMBarrierScopeKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMMBarrierScopeKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MBarrierScopeKind { - return @enumFromInt(c.mlirCuteNVVMMBarrierScopeKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierScopeKindAttrGetValue(self.ptr()))); } }; @@ -1537,11 +1537,11 @@ pub const MBarrierSpaceKindAttr = opaque { value: MBarrierSpaceKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMBarrierSpaceKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMMBarrierSpaceKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MBarrierSpaceKind { - return @enumFromInt(c.mlirCuteNVVMMBarrierSpaceKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierSpaceKindAttrGetValue(self.ptr()))); } }; @@ -1564,11 +1564,11 @@ pub const MBarrierTxnKindAttr = opaque { value: MBarrierTxnKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMBarrierTxnKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMMBarrierTxnKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MBarrierTxnKind { - return @enumFromInt(c.mlirCuteNVVMMBarrierTxnKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierTxnKindAttrGetValue(self.ptr()))); } }; @@ -1591,11 +1591,11 @@ pub const MBarrierWaitKindAttr = opaque { value: MBarrierWaitKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMBarrierWaitKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMMBarrierWaitKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MBarrierWaitKind { - return @enumFromInt(c.mlirCuteNVVMMBarrierWaitKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierWaitKindAttrGetValue(self.ptr()))); } }; @@ -1618,11 +1618,11 @@ pub const MMAB1OpAttr = opaque { value: MMAB1Op, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMMAB1OpAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMMMAB1OpAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MMAB1Op { - return @enumFromInt(c.mlirCuteNVVMMMAB1OpAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMMMAB1OpAttrGetValue(self.ptr()))); } }; @@ -1645,11 +1645,11 @@ pub const MMABlockScaleKindAttr = opaque { value: MMABlockScaleKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMMABlockScaleKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMMMABlockScaleKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MMABlockScaleKind { - return @enumFromInt(c.mlirCuteNVVMMMABlockScaleKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMMMABlockScaleKindAttrGetValue(self.ptr()))); } }; @@ -1672,11 +1672,11 @@ pub const MMACtaCountAttr = opaque { value: MMACtaCount, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMMACtaCountAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMMMACtaCountAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MMACtaCount { - return @enumFromInt(c.mlirCuteNVVMMMACtaCountAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMMMACtaCountAttrGetValue(self.ptr()))); } }; @@ -1699,11 +1699,11 @@ pub const MMAFragAttr = opaque { value: MMAFrag, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMMAFragAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMMMAFragAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MMAFrag { - return @enumFromInt(c.mlirCuteNVVMMMAFragAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMMMAFragAttrGetValue(self.ptr()))); } }; @@ -1726,11 +1726,11 @@ pub const MMAIntOverflowAttr = opaque { value: MMAIntOverflow, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMMAIntOverflowAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMMMAIntOverflowAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MMAIntOverflow { - return @enumFromInt(c.mlirCuteNVVMMMAIntOverflowAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMMMAIntOverflowAttrGetValue(self.ptr()))); } }; @@ -1753,11 +1753,11 @@ pub const MMAKindAttr = opaque { value: MMAKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMMAKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMMMAKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MMAKind { - return @enumFromInt(c.mlirCuteNVVMMMAKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMMMAKindAttrGetValue(self.ptr()))); } }; @@ -1780,11 +1780,11 @@ pub const MMALayoutAttr = opaque { value: MMALayout, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMMALayoutAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMMMALayoutAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MMALayout { - return @enumFromInt(c.mlirCuteNVVMMMALayoutAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMMMALayoutAttrGetValue(self.ptr()))); } }; @@ -1807,11 +1807,11 @@ pub const MMATypesAttr = opaque { value: MMATypes, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMMATypesAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMMMATypesAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MMATypes { - return @enumFromInt(c.mlirCuteNVVMMMATypesAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMMMATypesAttrGetValue(self.ptr()))); } }; @@ -1834,11 +1834,11 @@ pub const MatchSyncKindAttr = opaque { value: MatchSyncKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMatchSyncKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMMatchSyncKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MatchSyncKind { - return @enumFromInt(c.mlirCuteNVVMMatchSyncKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMMatchSyncKindAttrGetValue(self.ptr()))); } }; @@ -1861,11 +1861,11 @@ pub const MemOrderKindAttr = opaque { value: MemOrderKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMemOrderKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMMemOrderKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MemOrderKind { - return @enumFromInt(c.mlirCuteNVVMMemOrderKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMMemOrderKindAttrGetValue(self.ptr()))); } }; @@ -1888,11 +1888,11 @@ pub const MemScopeKindAttr = opaque { value: MemScopeKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMemScopeKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMMemScopeKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MemScopeKind { - return @enumFromInt(c.mlirCuteNVVMMemScopeKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMMemScopeKindAttrGetValue(self.ptr()))); } }; @@ -1915,11 +1915,11 @@ pub const MulModeAttr = opaque { value: MulMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMulModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMMulModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MulMode { - return @enumFromInt(c.mlirCuteNVVMMulModeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMMulModeAttrGetValue(self.ptr()))); } }; @@ -1942,11 +1942,11 @@ pub const NVVMMemorySpaceAttr = opaque { value: NVVMMemorySpace, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMNVVMMemorySpaceAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMNVVMMemorySpaceAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) NVVMMemorySpace { - return @enumFromInt(c.mlirCuteNVVMNVVMMemorySpaceAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMNVVMMemorySpaceAttrGetValue(self.ptr()))); } }; @@ -1969,11 +1969,11 @@ pub const PermuteModeAttr = opaque { value: PermuteMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMPermuteModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMPermuteModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) PermuteMode { - return @enumFromInt(c.mlirCuteNVVMPermuteModeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMPermuteModeAttrGetValue(self.ptr()))); } }; @@ -1996,11 +1996,11 @@ pub const PrefetchCacheLevelAttr = opaque { value: PrefetchCacheLevel, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMPrefetchCacheLevelAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMPrefetchCacheLevelAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) PrefetchCacheLevel { - return @enumFromInt(c.mlirCuteNVVMPrefetchCacheLevelAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMPrefetchCacheLevelAttrGetValue(self.ptr()))); } }; @@ -2023,11 +2023,11 @@ pub const ProxyKindAttr = opaque { value: ProxyKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMProxyKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMProxyKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) ProxyKind { - return @enumFromInt(c.mlirCuteNVVMProxyKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMProxyKindAttrGetValue(self.ptr()))); } }; @@ -2050,11 +2050,11 @@ pub const ReductionKindAttr = opaque { value: ReductionKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMReductionKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMReductionKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) ReductionKind { - return @enumFromInt(c.mlirCuteNVVMReductionKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMReductionKindAttrGetValue(self.ptr()))); } }; @@ -2077,11 +2077,11 @@ pub const ReductionOpAttr = opaque { value: ReductionOp, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMReductionOpAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMReductionOpAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) ReductionOp { - return @enumFromInt(c.mlirCuteNVVMReductionOpAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMReductionOpAttrGetValue(self.ptr()))); } }; @@ -2104,11 +2104,11 @@ pub const ReductionTypeAttr = opaque { value: ReductionType, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMReductionTypeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMReductionTypeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) ReductionType { - return @enumFromInt(c.mlirCuteNVVMReductionTypeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMReductionTypeAttrGetValue(self.ptr()))); } }; @@ -2131,11 +2131,11 @@ pub const SPCompressElemSizeAttr = opaque { value: SPCompressElemSize, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSPCompressElemSizeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMSPCompressElemSizeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) SPCompressElemSize { - return @enumFromInt(c.mlirCuteNVVMSPCompressElemSizeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPCompressElemSizeAttrGetValue(self.ptr()))); } }; @@ -2158,11 +2158,11 @@ pub const SPCompressFactorTypeAttr = opaque { value: SPCompressFactorType, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSPCompressFactorTypeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMSPCompressFactorTypeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) SPCompressFactorType { - return @enumFromInt(c.mlirCuteNVVMSPCompressFactorTypeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPCompressFactorTypeAttrGetValue(self.ptr()))); } }; @@ -2185,11 +2185,11 @@ pub const SPCompressIndexSizeAttr = opaque { value: SPCompressIndexSize, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSPCompressIndexSizeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMSPCompressIndexSizeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) SPCompressIndexSize { - return @enumFromInt(c.mlirCuteNVVMSPCompressIndexSizeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPCompressIndexSizeAttrGetValue(self.ptr()))); } }; @@ -2212,11 +2212,11 @@ pub const SPCompressOpKindAttr = opaque { value: SPCompressOpKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSPCompressOpKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMSPCompressOpKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) SPCompressOpKind { - return @enumFromInt(c.mlirCuteNVVMSPCompressOpKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPCompressOpKindAttrGetValue(self.ptr()))); } }; @@ -2239,11 +2239,11 @@ pub const SPCompressRepFactorAttr = opaque { value: SPCompressRepFactor, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSPCompressRepFactorAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMSPCompressRepFactorAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) SPCompressRepFactor { - return @enumFromInt(c.mlirCuteNVVMSPCompressRepFactorAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPCompressRepFactorAttrGetValue(self.ptr()))); } }; @@ -2266,11 +2266,11 @@ pub const SPDecompressElemSizeAttr = opaque { value: SPDecompressElemSize, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSPDecompressElemSizeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMSPDecompressElemSizeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) SPDecompressElemSize { - return @enumFromInt(c.mlirCuteNVVMSPDecompressElemSizeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPDecompressElemSizeAttrGetValue(self.ptr()))); } }; @@ -2293,11 +2293,11 @@ pub const SPDecompressFactorTypeAttr = opaque { value: SPDecompressFactorType, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSPDecompressFactorTypeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMSPDecompressFactorTypeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) SPDecompressFactorType { - return @enumFromInt(c.mlirCuteNVVMSPDecompressFactorTypeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPDecompressFactorTypeAttrGetValue(self.ptr()))); } }; @@ -2320,11 +2320,11 @@ pub const SPDecompressIndexSizeAttr = opaque { value: SPDecompressIndexSize, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSPDecompressIndexSizeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMSPDecompressIndexSizeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) SPDecompressIndexSize { - return @enumFromInt(c.mlirCuteNVVMSPDecompressIndexSizeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPDecompressIndexSizeAttrGetValue(self.ptr()))); } }; @@ -2347,11 +2347,11 @@ pub const SPDecompressRepFactorAttr = opaque { value: SPDecompressRepFactor, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSPDecompressRepFactorAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMSPDecompressRepFactorAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) SPDecompressRepFactor { - return @enumFromInt(c.mlirCuteNVVMSPDecompressRepFactorAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPDecompressRepFactorAttrGetValue(self.ptr()))); } }; @@ -2374,11 +2374,11 @@ pub const SaturationModeAttr = opaque { value: SaturationMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSaturationModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMSaturationModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) SaturationMode { - return @enumFromInt(c.mlirCuteNVVMSaturationModeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMSaturationModeAttrGetValue(self.ptr()))); } }; @@ -2401,11 +2401,11 @@ pub const SaturationModeKindAttr = opaque { value: SaturationModeKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSaturationModeKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMSaturationModeKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) SaturationModeKind { - return @enumFromInt(c.mlirCuteNVVMSaturationModeKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMSaturationModeKindAttrGetValue(self.ptr()))); } }; @@ -2428,11 +2428,11 @@ pub const ScaleVecSizeAttr = opaque { value: ScaleVecSize, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMScaleVecSizeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMScaleVecSizeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) ScaleVecSize { - return @enumFromInt(c.mlirCuteNVVMScaleVecSizeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMScaleVecSizeAttrGetValue(self.ptr()))); } }; @@ -2455,11 +2455,11 @@ pub const SetMaxRegisterActionAttr = opaque { value: SetMaxRegisterAction, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSetMaxRegisterActionAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMSetMaxRegisterActionAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) SetMaxRegisterAction { - return @enumFromInt(c.mlirCuteNVVMSetMaxRegisterActionAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMSetMaxRegisterActionAttrGetValue(self.ptr()))); } }; @@ -2482,11 +2482,11 @@ pub const SharedSpaceAttr = opaque { value: SharedSpace, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSharedSpaceAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMSharedSpaceAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) SharedSpace { - return @enumFromInt(c.mlirCuteNVVMSharedSpaceAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMSharedSpaceAttrGetValue(self.ptr()))); } }; @@ -2509,11 +2509,11 @@ pub const ShflKindAttr = opaque { value: ShflKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMShflKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMShflKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) ShflKind { - return @enumFromInt(c.mlirCuteNVVMShflKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMShflKindAttrGetValue(self.ptr()))); } }; @@ -2536,11 +2536,11 @@ pub const SparsityFormatAttr = opaque { value: SparsityFormat, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSparsityFormatAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMSparsityFormatAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) SparsityFormat { - return @enumFromInt(c.mlirCuteNVVMSparsityFormatAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMSparsityFormatAttrGetValue(self.ptr()))); } }; @@ -2563,11 +2563,11 @@ pub const StateSpaceAttr = opaque { value: StateSpace, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMStateSpaceAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMStateSpaceAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) StateSpace { - return @enumFromInt(c.mlirCuteNVVMStateSpaceAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMStateSpaceAttrGetValue(self.ptr()))); } }; @@ -2590,11 +2590,11 @@ pub const StoreCacheModifierKindAttr = opaque { value: StoreCacheModifierKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMStoreCacheModifierKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMStoreCacheModifierKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) StoreCacheModifierKind { - return @enumFromInt(c.mlirCuteNVVMStoreCacheModifierKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMStoreCacheModifierKindAttrGetValue(self.ptr()))); } }; @@ -2617,11 +2617,11 @@ pub const StoreShapeAttr = opaque { value: StoreShape, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMStoreShapeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMStoreShapeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) StoreShape { - return @enumFromInt(c.mlirCuteNVVMStoreShapeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMStoreShapeAttrGetValue(self.ptr()))); } }; @@ -2644,11 +2644,11 @@ pub const TCBarParamAttr = opaque { value: TCBarParam, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTCBarParamAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMTCBarParamAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TCBarParam { - return @enumFromInt(c.mlirCuteNVVMTCBarParamAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMTCBarParamAttrGetValue(self.ptr()))); } }; @@ -2671,11 +2671,11 @@ pub const TMALoadModeAttr = opaque { value: TMALoadMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTMALoadModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMTMALoadModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TMALoadMode { - return @enumFromInt(c.mlirCuteNVVMTMALoadModeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMTMALoadModeAttrGetValue(self.ptr()))); } }; @@ -2698,11 +2698,11 @@ pub const TMAReduxKindAttr = opaque { value: TMAReduxKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTMAReduxKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMTMAReduxKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TMAReduxKind { - return @enumFromInt(c.mlirCuteNVVMTMAReduxKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMTMAReduxKindAttrGetValue(self.ptr()))); } }; @@ -2725,11 +2725,11 @@ pub const TMAStoreModeAttr = opaque { value: TMAStoreMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTMAStoreModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMTMAStoreModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TMAStoreMode { - return @enumFromInt(c.mlirCuteNVVMTMAStoreModeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMTMAStoreModeAttrGetValue(self.ptr()))); } }; @@ -2752,11 +2752,11 @@ pub const Tcgen05CpMulticastAttr = opaque { value: Tcgen05CpMulticast, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTcgen05CpMulticastAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMTcgen05CpMulticastAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) Tcgen05CpMulticast { - return @enumFromInt(c.mlirCuteNVVMTcgen05CpMulticastAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05CpMulticastAttrGetValue(self.ptr()))); } }; @@ -2779,11 +2779,11 @@ pub const Tcgen05CpShapeAttr = opaque { value: Tcgen05CpShape, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTcgen05CpShapeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMTcgen05CpShapeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) Tcgen05CpShape { - return @enumFromInt(c.mlirCuteNVVMTcgen05CpShapeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05CpShapeAttrGetValue(self.ptr()))); } }; @@ -2806,11 +2806,11 @@ pub const Tcgen05CpSrcFormatAttr = opaque { value: Tcgen05CpSrcFormat, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTcgen05CpSrcFormatAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMTcgen05CpSrcFormatAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) Tcgen05CpSrcFormat { - return @enumFromInt(c.mlirCuteNVVMTcgen05CpSrcFormatAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05CpSrcFormatAttrGetValue(self.ptr()))); } }; @@ -2833,11 +2833,11 @@ pub const Tcgen05FenceKindAttr = opaque { value: Tcgen05FenceKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTcgen05FenceKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMTcgen05FenceKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) Tcgen05FenceKind { - return @enumFromInt(c.mlirCuteNVVMTcgen05FenceKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05FenceKindAttrGetValue(self.ptr()))); } }; @@ -2860,11 +2860,11 @@ pub const Tcgen05LdStShapeAttr = opaque { value: Tcgen05LdStShape, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTcgen05LdStShapeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMTcgen05LdStShapeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) Tcgen05LdStShape { - return @enumFromInt(c.mlirCuteNVVMTcgen05LdStShapeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05LdStShapeAttrGetValue(self.ptr()))); } }; @@ -2887,11 +2887,11 @@ pub const Tcgen05MMABlockScaleAttr = opaque { value: Tcgen05MMABlockScale, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTcgen05MMABlockScaleAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMTcgen05MMABlockScaleAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) Tcgen05MMABlockScale { - return @enumFromInt(c.mlirCuteNVVMTcgen05MMABlockScaleAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05MMABlockScaleAttrGetValue(self.ptr()))); } }; @@ -2914,11 +2914,11 @@ pub const Tcgen05MMACollectorBBufferAttr = opaque { value: Tcgen05MMACollectorBBuffer, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTcgen05MMACollectorBBufferAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMTcgen05MMACollectorBBufferAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) Tcgen05MMACollectorBBuffer { - return @enumFromInt(c.mlirCuteNVVMTcgen05MMACollectorBBufferAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05MMACollectorBBufferAttrGetValue(self.ptr()))); } }; @@ -2941,11 +2941,11 @@ pub const Tcgen05MMACollectorOpAttr = opaque { value: Tcgen05MMACollectorOp, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTcgen05MMACollectorOpAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMTcgen05MMACollectorOpAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) Tcgen05MMACollectorOp { - return @enumFromInt(c.mlirCuteNVVMTcgen05MMACollectorOpAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05MMACollectorOpAttrGetValue(self.ptr()))); } }; @@ -2968,11 +2968,11 @@ pub const Tcgen05MMAKindAttr = opaque { value: Tcgen05MMAKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTcgen05MMAKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMTcgen05MMAKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) Tcgen05MMAKind { - return @enumFromInt(c.mlirCuteNVVMTcgen05MMAKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05MMAKindAttrGetValue(self.ptr()))); } }; @@ -2995,11 +2995,11 @@ pub const Tcgen05WaitKindAttr = opaque { value: Tcgen05WaitKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTcgen05WaitKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMTcgen05WaitKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) Tcgen05WaitKind { - return @enumFromInt(c.mlirCuteNVVMTcgen05WaitKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05WaitKindAttrGetValue(self.ptr()))); } }; @@ -3022,11 +3022,11 @@ pub const TensormapElemtypeAttr = opaque { value: TensormapElemtype, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTensormapElemtypeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMTensormapElemtypeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TensormapElemtype { - return @enumFromInt(c.mlirCuteNVVMTensormapElemtypeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapElemtypeAttrGetValue(self.ptr()))); } }; @@ -3049,11 +3049,11 @@ pub const TensormapFieldAttr = opaque { value: TensormapField, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTensormapFieldAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMTensormapFieldAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TensormapField { - return @enumFromInt(c.mlirCuteNVVMTensormapFieldAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapFieldAttrGetValue(self.ptr()))); } }; @@ -3076,11 +3076,11 @@ pub const TensormapFillModeAttr = opaque { value: TensormapFillMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTensormapFillModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMTensormapFillModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TensormapFillMode { - return @enumFromInt(c.mlirCuteNVVMTensormapFillModeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapFillModeAttrGetValue(self.ptr()))); } }; @@ -3103,11 +3103,11 @@ pub const TensormapInterleaveLayoutAttr = opaque { value: TensormapInterleaveLayout, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTensormapInterleaveLayoutAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMTensormapInterleaveLayoutAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TensormapInterleaveLayout { - return @enumFromInt(c.mlirCuteNVVMTensormapInterleaveLayoutAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapInterleaveLayoutAttrGetValue(self.ptr()))); } }; @@ -3130,11 +3130,11 @@ pub const TensormapSwizzleAtomicityAttr = opaque { value: TensormapSwizzleAtomicity, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTensormapSwizzleAtomicityAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMTensormapSwizzleAtomicityAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TensormapSwizzleAtomicity { - return @enumFromInt(c.mlirCuteNVVMTensormapSwizzleAtomicityAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapSwizzleAtomicityAttrGetValue(self.ptr()))); } }; @@ -3157,11 +3157,11 @@ pub const TensormapSwizzleModeAttr = opaque { value: TensormapSwizzleMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTensormapSwizzleModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMTensormapSwizzleModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TensormapSwizzleMode { - return @enumFromInt(c.mlirCuteNVVMTensormapSwizzleModeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapSwizzleModeAttrGetValue(self.ptr()))); } }; @@ -3184,11 +3184,11 @@ pub const TmemLayoutAttr = opaque { value: TmemLayout, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTmemLayoutAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMTmemLayoutAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) TmemLayout { - return @enumFromInt(c.mlirCuteNVVMTmemLayoutAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMTmemLayoutAttrGetValue(self.ptr()))); } }; @@ -3211,11 +3211,11 @@ pub const ValidatePatternAttr = opaque { value: ValidatePattern, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMValidatePatternAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMValidatePatternAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) ValidatePattern { - return @enumFromInt(c.mlirCuteNVVMValidatePatternAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMValidatePatternAttrGetValue(self.ptr()))); } }; @@ -3238,11 +3238,11 @@ pub const VoteSyncKindAttr = opaque { value: VoteSyncKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMVoteSyncKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMVoteSyncKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) VoteSyncKind { - return @enumFromInt(c.mlirCuteNVVMVoteSyncKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMVoteSyncKindAttrGetValue(self.ptr()))); } }; @@ -3265,11 +3265,11 @@ pub const WGMMAScaleInAttr = opaque { value: WGMMAScaleIn, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMWGMMAScaleInAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMWGMMAScaleInAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) WGMMAScaleIn { - return @enumFromInt(c.mlirCuteNVVMWGMMAScaleInAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMWGMMAScaleInAttrGetValue(self.ptr()))); } }; @@ -3292,11 +3292,11 @@ pub const WGMMAScaleOutAttr = opaque { value: WGMMAScaleOut, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMWGMMAScaleOutAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMWGMMAScaleOutAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) WGMMAScaleOut { - return @enumFromInt(c.mlirCuteNVVMWGMMAScaleOutAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMWGMMAScaleOutAttrGetValue(self.ptr()))); } }; @@ -3319,11 +3319,11 @@ pub const WGMMATypesAttr = opaque { value: WGMMATypes, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMWGMMATypesAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirCuteNVVMWGMMATypesAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) WGMMATypes { - return @enumFromInt(c.mlirCuteNVVMWGMMATypesAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirCuteNVVMWGMMATypesAttrGetValue(self.ptr()))); } }; @@ -6844,13 +6844,14 @@ test "every enum attribute prints as the compiler does" { .{ WGMMATypesAttr, "#nvvm.wgmma_type<{s}>" }, }) |example| { const T = example[0]; - inline for (@typeInfo(@FieldType(T.InitArgs, "value")).@"enum".fields) |case| { - const attr = try T.get(ctx, .{ .value = @enumFromInt(case.value) }); - try std.testing.expectEqual(case.value, @intFromEnum(attr.getValue())); + const E = @FieldType(T.InitArgs, "value"); + inline for (@typeInfo(E).@"enum".field_names) |name| { + const attr = try T.get(ctx, .{ .value = @field(E, name) }); + try std.testing.expectEqual(@field(E, name), attr.getValue()); var buf: [256]u8 = undefined; var w: std.Io.Writer = .fixed(&buf); try w.print("{f}", .{attr}); - try std.testing.expectEqualStrings(std.fmt.comptimePrint(example[1], .{case.name}), w.buffered()); + try std.testing.expectEqualStrings(std.fmt.comptimePrint(example[1], .{name}), w.buffered()); } } } diff --git a/mlir/dialects/fly/attributes.zig b/mlir/dialects/fly/attributes.zig index eceb8b0f..a30336fc 100644 --- a/mlir/dialects/fly/attributes.zig +++ b/mlir/dialects/fly/attributes.zig @@ -206,7 +206,7 @@ pub const AddressSpaceAttr = opaque { }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirFlyAddressSpaceAttrGet(ctx.ptr(), @intFromEnum(args.addressSpace)); + const result = c.mlirFlyAddressSpaceAttrGet(ctx.ptr(), @backingInt(args.addressSpace)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } }; @@ -214,7 +214,7 @@ pub const AddressSpaceAttr = opaque { pub const MmaOperand = enum(i32) { a = 0, b = 1, c = 2, d = 3 }; pub fn mmaOperandAttr(ctx: *mlir.Context, operand: MmaOperand) mlir.Error!*const mlir.Attribute { - const result = c.mlirFlyMmaOperandAttrGet(ctx.ptr(), @intFromEnum(operand)); + const result = c.mlirFlyMmaOperandAttrGet(ctx.ptr(), @backingInt(operand)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } @@ -234,6 +234,6 @@ pub const GemmTraversalOrder = enum(i32) { }; pub fn gemmTraversalOrderAttr(ctx: *mlir.Context, order: GemmTraversalOrder) mlir.Error!*const mlir.Attribute { - const result = c.mlirFlyGemmTraversalOrderAttrGet(ctx.ptr(), @intFromEnum(order)); + const result = c.mlirFlyGemmTraversalOrderAttrGet(ctx.ptr(), @backingInt(order)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } diff --git a/mlir/dialects/fly/fly.zig b/mlir/dialects/fly/fly.zig index 97571450..8bfafbce 100644 --- a/mlir/dialects/fly/fly.zig +++ b/mlir/dialects/fly/fly.zig @@ -132,7 +132,7 @@ test { inline for (.{ types, attributes, ops, rocdl, rocdl.ops }) |module| { std.testing.refAllDecls(module); inline for (comptime std.meta.declarations(module)) |decl| { - const value = @field(module, decl.name); + const value = @field(module, decl); if (@TypeOf(value) == type and @typeInfo(value) == .@"opaque") std.testing.refAllDecls(value); } } diff --git a/mlir/dialects/gpu.zig b/mlir/dialects/gpu.zig index d43687fc..a6e16ebf 100644 --- a/mlir/dialects/gpu.zig +++ b/mlir/dialects/gpu.zig @@ -38,11 +38,11 @@ pub const DimensionAttr = opaque { value: Dim, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirGPUDimensionAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirGPUDimensionAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) Dim { - return @enumFromInt(c.mlirGPUDimensionAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirGPUDimensionAttrGetValue(self.ptr()))); } }; @@ -65,11 +65,11 @@ pub const ShuffleModeAttr = opaque { value: ShuffleMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirGPUShuffleModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirGPUShuffleModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) ShuffleMode { - return @enumFromInt(c.mlirGPUShuffleModeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirGPUShuffleModeAttrGetValue(self.ptr()))); } }; @@ -174,17 +174,17 @@ test "gpu enum attributes are built through the C API" { const ctx = try mlir.Context.init(.{}); defer ctx.deinit(); - inline for (std.meta.fields(Dim)) |field| { - const value: Dim = @enumFromInt(field.value); + inline for (@typeInfo(Dim).@"enum".field_names) |name| { + const value: Dim = @field(Dim, name); const attr = try DimensionAttr.get(ctx, .{ .value = value }); try std.testing.expectEqual(value, attr.getValue()); try std.testing.expect(attr.attribute().isA(DimensionAttr) != null); - try expectPrints("#gpu.dim<" ++ field.name ++ ">", attr); + try expectPrints("#gpu.dim<" ++ name ++ ">", attr); } - inline for (std.meta.fields(ShuffleMode)) |field| { - const value: ShuffleMode = @enumFromInt(field.value); + inline for (@typeInfo(ShuffleMode).@"enum".field_names) |name| { + const value: ShuffleMode = @field(ShuffleMode, name); const attr = try ShuffleModeAttr.get(ctx, .{ .value = value }); try std.testing.expectEqual(value, attr.getValue()); - try expectPrints("#gpu.shuffle_mode<" ++ field.name ++ ">", attr); + try expectPrints("#gpu.shuffle_mode<" ++ name ++ ">", attr); } } diff --git a/mlir/dialects/llvm_dialect.zig b/mlir/dialects/llvm_dialect.zig index 2b7e904d..541c1373 100644 --- a/mlir/dialects/llvm_dialect.zig +++ b/mlir/dialects/llvm_dialect.zig @@ -23,7 +23,7 @@ pub const NVPTXAddressSpace = enum(u32) { /// `!llvm.ptr`, an opaque pointer. Loads the LLVM dialect into `ctx`. pub fn pointerType(ctx: *mlir.Context, space: anytype) *const mlir.Type { _ = c.mlirContextGetOrLoadDialect(ctx.ptr(), mlir.stringRef("llvm")); - return @ptrCast(c.mlirLLVMPointerTypeGet(ctx.ptr(), @intCast(@intFromEnum(space))).ptr); + return @ptrCast(c.mlirLLVMPointerTypeGet(ctx.ptr(), @intCast(@backingInt(space))).ptr); } // ============================================================================= diff --git a/mlir/dialects/memref.zig b/mlir/dialects/memref.zig index 77acbc5d..338d1732 100644 --- a/mlir/dialects/memref.zig +++ b/mlir/dialects/memref.zig @@ -216,7 +216,7 @@ pub fn atomic_rmw( .operands = .{ .flat = buf.constSlice() }, .results = .{ .flat = &.{value.type_()} }, .attributes = &.{ - .named(ctx, "kind", .int(ctx, .i64, @intFromEnum(kind))), + .named(ctx, "kind", .int(ctx, .i64, @backingInt(kind))), }, .location = location, }); diff --git a/mlir/dialects/mosaic_tpu/mosaic_tpu.zig b/mlir/dialects/mosaic_tpu/mosaic_tpu.zig index 3cd164c0..b70e3c3e 100644 --- a/mlir/dialects/mosaic_tpu/mosaic_tpu.zig +++ b/mlir/dialects/mosaic_tpu/mosaic_tpu.zig @@ -158,11 +158,11 @@ pub const ReductionKindAttr = opaque { value: ReductionKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirTpuReductionKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirTpuReductionKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) ReductionKind { - return @enumFromInt(c.mlirTpuReductionKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirTpuReductionKindAttrGetValue(self.ptr()))); } }; @@ -197,11 +197,11 @@ pub const ContractPrecisionAttr = opaque { value: ContractPrecision, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirTpuContractPrecisionAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirTpuContractPrecisionAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) ContractPrecision { - return @enumFromInt(c.mlirTpuContractPrecisionAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirTpuContractPrecisionAttrGetValue(self.ptr()))); } }; @@ -235,11 +235,11 @@ pub const RoundingModeAttr = opaque { value: RoundingMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirTpuRoundingModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirTpuRoundingModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) RoundingMode { - return @enumFromInt(c.mlirTpuRoundingModeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirTpuRoundingModeAttrGetValue(self.ptr()))); } }; @@ -274,11 +274,11 @@ pub const CoreTypeAttr = opaque { value: CoreType, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirTpuCoreTypeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirTpuCoreTypeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) CoreType { - return @enumFromInt(c.mlirTpuCoreTypeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirTpuCoreTypeAttrGetValue(self.ptr()))); } }; @@ -314,11 +314,11 @@ pub const DimensionSemanticsAttr = opaque { value: DimensionSemantics, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirTpuDimensionSemanticsAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirTpuDimensionSemanticsAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) DimensionSemantics { - return @enumFromInt(c.mlirTpuDimensionSemanticsAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirTpuDimensionSemanticsAttrGetValue(self.ptr()))); } }; @@ -352,11 +352,11 @@ pub const PipelineModeAttr = opaque { value: PipelineMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirTpuPipelineModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirTpuPipelineModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) PipelineMode { - return @enumFromInt(c.mlirTpuPipelineModeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirTpuPipelineModeAttrGetValue(self.ptr()))); } }; @@ -390,11 +390,11 @@ pub const RevisitModeAttr = opaque { value: RevisitMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirTpuRevisitModeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirTpuRevisitModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) RevisitMode { - return @enumFromInt(c.mlirTpuRevisitModeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirTpuRevisitModeAttrGetValue(self.ptr()))); } }; @@ -435,16 +435,16 @@ pub const MemorySpaceAttr = opaque { coreType: ?CoreType = null, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const core_type: i64 = if (args.coreType) |ct| @intFromEnum(ct) else -1; - const result = c.mlirTpuMemorySpaceAttrGet(ctx.ptr(), @intFromEnum(args.value), core_type); + const core_type: i64 = if (args.coreType) |ct| @backingInt(ct) else -1; + const result = c.mlirTpuMemorySpaceAttrGet(ctx.ptr(), @backingInt(args.value), core_type); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) MemorySpace { - return @enumFromInt(c.mlirTpuMemorySpaceAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirTpuMemorySpaceAttrGetValue(self.ptr()))); } pub fn getCoreType(self: *const Self) ?CoreType { const core_type = c.mlirTpuMemorySpaceAttrGetCoreType(self.ptr()); - return if (core_type < 0) null else @enumFromInt(core_type); + return if (core_type < 0) null else @fromBackingInt(@intCast(core_type)); } }; @@ -1682,13 +1682,13 @@ fn expectPrints(expected: []const u8, value: anytype) !void { fn expectEnumAttr(comptime Attr: type, ctx: *mlir.Context) !void { const Enum = @FieldType(Attr.InitArgs, "value"); - inline for (std.meta.fields(Enum)) |field| { - const value: Enum = @enumFromInt(field.value); + inline for (@typeInfo(Enum).@"enum".field_names) |name| { + const value: Enum = @field(Enum, name); const attr = try Attr.get(ctx, .{ .value = value }); try std.testing.expectEqual(value, attr.getValue()); try std.testing.expect(attr.attribute().isA(Attr) != null); try std.testing.expect(value.attribute(ctx).eql(attr.attribute())); - try expectPrints("#tpu." ++ Attr.mnemonic ++ "<" ++ field.name ++ ">", attr); + try expectPrints("#tpu." ++ Attr.mnemonic ++ "<" ++ name ++ ">", attr); } } diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index efae3018..f9ea1e7e 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -805,7 +805,7 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result const backend_config_api_version: ?CustomCallOpts.ApiVersion = if (opts.backend_config) |bc| bc else null; if (opts.api_version orelse backend_config_api_version) |v| { attrs.appendAssumeCapacity( - .named(ctx, "api_version", .int(ctx, .i32, @intFromEnum(v))), + .named(ctx, "api_version", .int(ctx, .i32, @backingInt(v))), ); } if (opts.backend_config) |backend_config| { @@ -1463,7 +1463,7 @@ pub fn channelHandle( return @ptrCast(c.stablehloChannelHandleGet( ctx.ptr(), handle, - @intFromEnum(channel_type), + @backingInt(channel_type), ).ptr); } diff --git a/mlir/dialects/ttir/ttir.zig b/mlir/dialects/ttir/ttir.zig index 81d926dc..ee302f92 100644 --- a/mlir/dialects/ttir/ttir.zig +++ b/mlir/dialects/ttir/ttir.zig @@ -142,51 +142,51 @@ pub const DescriptorReduceKind = enum(i32) { }; pub fn programDim(ctx: *mlir.Context, v: ProgramIDDim) *const mlir.Attribute { - return @ptrCast(c.mlirTritonProgramDimGet(ctx.ptr(), @intFromEnum(v)).ptr); + return @ptrCast(c.mlirTritonProgramDimGet(ctx.ptr(), @backingInt(v)).ptr); } pub fn cacheModifier(ctx: *mlir.Context, v: CacheModifier) *const mlir.Attribute { - return @ptrCast(c.mlirTritonCacheModifierGet(ctx.ptr(), @intFromEnum(v)).ptr); + return @ptrCast(c.mlirTritonCacheModifierGet(ctx.ptr(), @backingInt(v)).ptr); } pub fn evictionPolicy(ctx: *mlir.Context, v: EvictionPolicy) *const mlir.Attribute { - return @ptrCast(c.mlirTritonEvictionPolicyGet(ctx.ptr(), @intFromEnum(v)).ptr); + return @ptrCast(c.mlirTritonEvictionPolicyGet(ctx.ptr(), @backingInt(v)).ptr); } pub fn paddingOption(ctx: *mlir.Context, v: PaddingOption) *const mlir.Attribute { - return @ptrCast(c.mlirTritonPaddingOptionGet(ctx.ptr(), @intFromEnum(v)).ptr); + return @ptrCast(c.mlirTritonPaddingOptionGet(ctx.ptr(), @backingInt(v)).ptr); } pub fn roundingMode(ctx: *mlir.Context, v: RoundingMode) *const mlir.Attribute { - return @ptrCast(c.mlirTritonRoundingModeGet(ctx.ptr(), @intFromEnum(v)).ptr); + return @ptrCast(c.mlirTritonRoundingModeGet(ctx.ptr(), @backingInt(v)).ptr); } pub fn inputPrecision(ctx: *mlir.Context, v: InputPrecision) *const mlir.Attribute { - return @ptrCast(c.mlirTritonInputPrecisionGet(ctx.ptr(), @intFromEnum(v)).ptr); + return @ptrCast(c.mlirTritonInputPrecisionGet(ctx.ptr(), @backingInt(v)).ptr); } pub fn rmwOp(ctx: *mlir.Context, v: RMWOp) *const mlir.Attribute { - return @ptrCast(c.mlirTritonRMWOpGet(ctx.ptr(), @intFromEnum(v)).ptr); + return @ptrCast(c.mlirTritonRMWOpGet(ctx.ptr(), @backingInt(v)).ptr); } pub fn memSemantic(ctx: *mlir.Context, v: MemSemantic) *const mlir.Attribute { - return @ptrCast(c.mlirTritonMemSemanticGet(ctx.ptr(), @intFromEnum(v)).ptr); + return @ptrCast(c.mlirTritonMemSemanticGet(ctx.ptr(), @backingInt(v)).ptr); } pub fn memSyncScope(ctx: *mlir.Context, v: MemSyncScope) *const mlir.Attribute { - return @ptrCast(c.mlirTritonMemSyncScopeGet(ctx.ptr(), @intFromEnum(v)).ptr); + return @ptrCast(c.mlirTritonMemSyncScopeGet(ctx.ptr(), @backingInt(v)).ptr); } pub fn propagateNan(ctx: *mlir.Context, v: PropagateNan) *const mlir.Attribute { - return @ptrCast(c.mlirTritonPropagateNanGet(ctx.ptr(), @intFromEnum(v)).ptr); + return @ptrCast(c.mlirTritonPropagateNanGet(ctx.ptr(), @backingInt(v)).ptr); } pub fn scaleDotElemType(ctx: *mlir.Context, v: ScaleDotElemType) *const mlir.Attribute { - return @ptrCast(c.mlirTritonScaleDotElemTypeGet(ctx.ptr(), @intFromEnum(v)).ptr); + return @ptrCast(c.mlirTritonScaleDotElemTypeGet(ctx.ptr(), @backingInt(v)).ptr); } pub fn descriptorReduceKind(ctx: *mlir.Context, v: DescriptorReduceKind) *const mlir.Attribute { - return @ptrCast(c.mlirTritonDescriptorReduceKindGet(ctx.ptr(), @intFromEnum(v)).ptr); + return @ptrCast(c.mlirTritonDescriptorReduceKindGet(ctx.ptr(), @backingInt(v)).ptr); } // ============================================================================= diff --git a/mlir/dialects/vector.zig b/mlir/dialects/vector.zig index 04e2384f..d2cca4f2 100644 --- a/mlir/dialects/vector.zig +++ b/mlir/dialects/vector.zig @@ -60,11 +60,11 @@ pub const CombiningKindAttr = opaque { value: CombiningKind, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirVectorCombiningKindAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirVectorCombiningKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) CombiningKind { - return @enumFromInt(c.mlirVectorCombiningKindAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirVectorCombiningKindAttrGetValue(self.ptr()))); } }; @@ -87,11 +87,11 @@ pub const IteratorTypeAttr = opaque { value: IteratorType, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirVectorIteratorTypeAttrGet(ctx.ptr(), @intFromEnum(args.value)); + const result = c.mlirVectorIteratorTypeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getValue(self: *const Self) IteratorType { - return @enumFromInt(c.mlirVectorIteratorTypeAttrGetValue(self.ptr())); + return @fromBackingInt(@intCast(c.mlirVectorIteratorTypeAttrGetValue(self.ptr()))); } }; @@ -834,17 +834,17 @@ test "vector enum attributes are built through the C API" { const ctx = try mlir.Context.init(.{}); defer ctx.deinit(); - inline for (std.meta.fields(CombiningKind)) |field| { - const value: CombiningKind = @enumFromInt(field.value); + inline for (@typeInfo(CombiningKind).@"enum".field_names) |name| { + const value: CombiningKind = @field(CombiningKind, name); const attr = try CombiningKindAttr.get(ctx, .{ .value = value }); try std.testing.expectEqual(value, attr.getValue()); try std.testing.expect(value.attribute(ctx).isA(CombiningKindAttr) != null); - try expectPrints("#vector.kind<" ++ field.name ++ ">", attr); + try expectPrints("#vector.kind<" ++ name ++ ">", attr); } - inline for (std.meta.fields(IteratorType)) |field| { - const value: IteratorType = @enumFromInt(field.value); + inline for (@typeInfo(IteratorType).@"enum".field_names) |name| { + const value: IteratorType = @field(IteratorType, name); const attr = try IteratorTypeAttr.get(ctx, .{ .value = value }); try std.testing.expectEqual(value, attr.getValue()); - try expectPrints("#vector.iterator_type<" ++ field.name ++ ">", attr); + try expectPrints("#vector.iterator_type<" ++ name ++ ">", attr); } } diff --git a/mlir/mlir.zig b/mlir/mlir.zig index d759dbfa..83bec2f5 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -88,16 +88,16 @@ pub fn registerFuncExtensions(registry: *DialectRegistry) void { } fn isPtrConst(comptime T: type) bool { - return @typeInfo(T).pointer.is_const; + return @typeInfo(T).pointer.attrs.@"const"; } fn CastedPtr(comptime T1: type, comptime T2: type) type { - return if (@typeInfo(T1).pointer.is_const) *const T2 else *T2; + return if (@typeInfo(T1).pointer.attrs.@"const") *const T2 else *T2; } pub fn Methods(comptime T: type, comptime NativeT: type) type { return struct { - pub const IsPtrConst = @typeInfo(@typeInfo(@FieldType(NativeT, "ptr")).optional.child).pointer.is_const; + pub const IsPtrConst = @typeInfo(@typeInfo(@FieldType(NativeT, "ptr")).optional.child).pointer.attrs.@"const"; pub const Ptr = if (IsPtrConst) *const T else *T; pub fn ptr(self: anytype) NativeT { @@ -1467,11 +1467,11 @@ pub const Operation = opaque { (struct { pub fn callback(op: c.MlirOperation, ctx_: ?*anyopaque) callconv(.c) c.MlirWalkResult { const inner_ctx_: *ContextType = @ptrCast(@alignCast(ctx_)); - return @intFromEnum(walkfn(inner_ctx_.ctx, @ptrCast(op.ptr))); + return @backingInt(walkfn(inner_ctx_.ctx, @ptrCast(op.ptr))); } }).callback, &inner_ctx, - @intFromEnum(order), + @backingInt(order), ); } diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index 7c7ed2f6..cebec841 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -11,7 +11,7 @@ const Stream = @import("pjrt.zig").Stream; const log = std.log.scoped(.pjrt); comptime { - if (@typeInfo(TypeId).@"struct".fields.len != 1) @compileError("TypeId has changed"); + if (@typeInfo(TypeId).@"struct".field_names.len != 1) @compileError("TypeId has changed"); } /// The signature of a generic custom call. @@ -450,7 +450,7 @@ pub const Error = opaque { pub fn create(api: *const Api, error_code: ErrorCode, message: []const u8) *Error { var ret: pjrt.meta.Struct(c.XLA_FFI_Error_Create_Args) = .{ .message = message.ptr, - .errc = @intFromEnum(error_code), + .errc = @backingInt(error_code), }; return fromInner(api.inner().XLA_FFI_Error_Create.?(@ptrCast(&ret)).?); } diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 44a72aca..73cadf44 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -31,18 +31,18 @@ pub const meta = struct { } pub fn Struct(comptime T: type) type { - const fields = std.meta.fields(T); - var names: [fields.len][]const u8 = undefined; - var types: [fields.len]type = undefined; - var attributes: [fields.len]std.builtin.Type.StructField.Attributes = undefined; - for (fields, &names, &types, &attributes) |field, *name, *type_, *attr| { - name.* = field.name; - type_.* = field.type; + const info = @typeInfo(T).@"struct"; + var names: [info.field_names.len][]const u8 = undefined; + var types: [info.field_names.len]type = undefined; + var attributes: [info.field_names.len]std.builtin.Type.Struct.FieldAttributes = undefined; + for (info.field_names, info.field_types, &names, &types, &attributes) |field_name, Field, *name, *type_, *attr| { + name.* = field_name; + type_.* = Field; attr.* = .{ - .default_value_ptr = @ptrCast(if (std.mem.eql(u8, field.name, "struct_size")) + .default_value_ptr = @ptrCast(if (std.mem.eql(u8, field_name, "struct_size")) &structSize(T) else - &std.mem.zeroes(field.type)), + &std.mem.zeroes(Field)), }; } return @Struct( @@ -153,7 +153,7 @@ pub const Api = struct { const fti = @typeInfo(@FieldType(c.PJRT_Api, @tagName(func))); const fn_ptr = @typeInfo(fti.optional.child); const fn_type_info = @typeInfo(fn_ptr.pointer.child); - const arg_array_type_info = @typeInfo(fn_type_info.@"fn".params[0].type.?); + const arg_array_type_info = @typeInfo(fn_type_info.@"fn".param_types[0].?); return arg_array_type_info.pointer.child; } @@ -255,7 +255,7 @@ pub const Api = struct { pub fn stablehloCurrentVersion(self: *const Api) ?[]const u8 { const state = struct { var buf: [32]u8 = undefined; - var str: ?[:0]const u8 = null; + var str: ?[]const u8 = null; }; if (state.str) |str| { return str; @@ -263,7 +263,7 @@ pub const Api = struct { if (self.pluginAttribute("stablehlo_current_version")) |nv| { switch (nv.value()) { .int64list => |v| { - state.str = std.fmt.bufPrintZ(&state.buf, "{d}.{d}.{d}", .{ v[0], v[1], v[2] }) catch unreachable; + state.str = std.mem.print(&state.buf, "{d}.{d}.{d}", .{ v[0], v[1], v[2] }) catch unreachable; }, else => unreachable, } @@ -365,7 +365,7 @@ pub const Error = opaque { const ret = api.call(.PJRT_Error_GetCode, .{ .@"error" = self.inner(), }) catch unreachable; - return @enumFromInt(ret.code); + return @fromBackingInt(@intCast(ret.code)); } pub fn getMessage(self: *Error, api: *const Api) []const u8 { @@ -390,7 +390,7 @@ pub const ShapeSpec = extern struct { .inner = .{ .dims = @ptrCast(@constCast(dims_)), .num_dims = dims_.len, - .element_type = @intFromEnum(bt), + .element_type = @backingInt(bt), }, }; } @@ -400,7 +400,7 @@ pub const ShapeSpec = extern struct { } pub fn bufferType(self: ShapeSpec) BufferType { - return @enumFromInt(self.inner.element_type); + return @fromBackingInt(@intCast(self.inner.element_type)); } }; @@ -494,13 +494,13 @@ pub const Client = opaque { const ret = try api.call(.PJRT_Client_BufferFromHostBuffer, .{ .client = self.inner(), .data = @constCast(args.data), - .type = @intFromEnum(args.buffer_type), + .type = @backingInt(args.buffer_type), .dims = @ptrCast(@constCast(args.dims)), .num_dims = args.dims.len, .byte_strides = if (args.byte_strides) |bs| @ptrCast(@constCast(bs)) else null, .num_byte_strides = if (args.byte_strides) |bs| bs.len else 0, .device_layout = @ptrCast(@constCast(&args.layout.toCStruct())), - .host_buffer_semantics = @intFromEnum(args.host_buffer_semantics), + .host_buffer_semantics = @backingInt(args.host_buffer_semantics), .device = if (args.dst == .device) @ptrCast(@constCast(args.dst.device)) else null, .memory = if (args.dst == .memory) args.dst.memory.inner() else null, }); @@ -543,7 +543,7 @@ pub const Client = opaque { .device_buffer_ptr = @constCast(args.device_buffer_ptr), .dims = args.dims.ptr, .num_dims = args.dims.len, - .element_type = @intFromEnum(args.element_type), + .element_type = @backingInt(args.element_type), .layout = @ptrCast(@constCast(&layout)), .device = @ptrCast(@constCast(args.device)), .on_delete_callback = args.on_delete_callback, @@ -609,7 +609,7 @@ pub const Client = opaque { .struct_size = c.PJRT_Layouts_PJRT_Client_GetDefaultLayout_Args_STRUCT_SIZE, .extension_start = null, .client = self.inner(), - .type = @intFromEnum(element_type), + .type = @backingInt(element_type), .dims = dims.ptr, .num_dims = dims.len, .layout = null, @@ -670,7 +670,7 @@ pub const Client = opaque { .client = self.inner(), .shape_dims = args.dims.ptr, .shape_num_dims = args.dims.len, - .shape_element_type = @intFromEnum(args.element_type), + .shape_element_type = @backingInt(args.element_type), .shape_layout = @ptrCast(&layout), .device = if (args.dst == .device) @ptrCast(@constCast(args.dst.device)) else null, .memory = if (args.dst == .memory) args.dst.memory.inner() else null, @@ -808,7 +808,7 @@ pub const Device = opaque { const ret = try api.call(.PJRT_Device_MemoryStats, .{ .device = self.inner(), }); - return .fromCStruct(@bitCast(ret)); + return .fromCStruct(ret); } }; @@ -1287,7 +1287,7 @@ pub const Buffer = opaque { const ret = api.call(.PJRT_Buffer_ElementType, .{ .buffer = self.inner(), }) catch unreachable; - return @enumFromInt(ret.type); + return @fromBackingInt(@intCast(ret.type)); } pub fn dimensions(self: *const Buffer, api: *const Api) []const i64 { @@ -1617,7 +1617,7 @@ pub const NamedValue = extern struct { }; pub fn kind(self: NamedValue) Kind { - return @enumFromInt(self.inner.type); + return @fromBackingInt(@intCast(self.inner.type)); } pub fn name(self: NamedValue) []const u8 { @@ -1634,14 +1634,14 @@ pub const NamedValue = extern struct { }; } - pub fn init(comptime kind_: Kind, name_: []const u8, value_: std.meta.fieldInfo(Value, kind_).type) NamedValue { + pub fn init(comptime kind_: Kind, name_: []const u8, value_: @FieldType(Value, @tagName(kind_))) NamedValue { return .{ .inner = .{ .struct_size = c.PJRT_NamedValue_STRUCT_SIZE, .extension_start = null, .name = @ptrCast(name_), .name_size = name_.len, - .type = @intFromEnum(kind_), + .type = @backingInt(kind_), .unnamed_0 = switch (kind_) { .string => .{ .string_value = @ptrCast(@constCast(value_)) }, .int64 => .{ .int64_value = value_ }, diff --git a/platforms/cuda/compat_probe.zig b/platforms/cuda/compat_probe.zig index 9e962577..3f890c6a 100644 --- a/platforms/cuda/compat_probe.zig +++ b/platforms/cuda/compat_probe.zig @@ -22,7 +22,7 @@ pub fn main(init: std.process.Init) !u8 { const cuInit = libcuda.lookup(*const fn (c_uint) callconv(.c) c_int, "cuInit").?; const cuGetErrorString = libcuda.lookup(*const fn (c_int, *[*c]const u8) callconv(.c) c_int, "cuGetErrorString").?; - return @intFromEnum(switch (cuInit(0)) { + return @backingInt(switch (cuInit(0)) { CUDA_SUCCESS => ExitCode.Success, CUDA_ERROR_SYSTEM_DRIVER_MISMATCH => ExitCode.SystemDriverMismatch, CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE => ExitCode.CompatNotSupportedOnDevice, diff --git a/platforms/cuda/cuda.zig b/platforms/cuda/cuda.zig index 562fdc56..692ea226 100644 --- a/platforms/cuda/cuda.zig +++ b/platforms/cuda/cuda.zig @@ -47,7 +47,7 @@ pub fn needsCudaCompat(io: std.Io, sandbox_path: []const u8) !bool { log.err("Failed to run CUDA compatibility probe: {any}", .{err}); return err; }; - const result: compat_probe.ExitCode = @enumFromInt(res.exited); + const result: compat_probe.ExitCode = @fromBackingInt(@intCast(res.exited)); return switch (result) { .Success => true, @@ -69,7 +69,7 @@ fn hasNvidiaDevice(io: std.Io) bool { fn hasCudaPathInLDPath() bool { const ldLibraryPath = std.c.getenv("LD_LIBRARY_PATH") orelse return false; - return std.ascii.indexOfIgnoreCase(std.mem.span(ldLibraryPath), nvidiaLibsPath) != null; + return std.ascii.findIgnoreCase(std.mem.span(ldLibraryPath), nvidiaLibsPath) != null; } fn setupXlaGpuCudaDirFlag(allocator: std.mem.Allocator, sandbox: []const u8) !void { diff --git a/platforms/cuda/flashattn/flashattn.zig b/platforms/cuda/flashattn/flashattn.zig index b4014c25..7d9c72b0 100644 --- a/platforms/cuda/flashattn/flashattn.zig +++ b/platforms/cuda/flashattn/flashattn.zig @@ -20,7 +20,7 @@ pub const Tensor = struct { .rank = @intCast(rank), .dims = undefined, .strides = undefined, - .dtype = @intCast(@intFromEnum(dtype)), + .dtype = @intCast(@backingInt(dtype)), }, }; @memcpy(ret.inner.dims[0..rank], dims); diff --git a/platforms/neuron/libzml_neuron.zig b/platforms/neuron/libzml_neuron.zig index 968f1584..404d8cc0 100644 --- a/platforms/neuron/libzml_neuron.zig +++ b/platforms/neuron/libzml_neuron.zig @@ -46,7 +46,7 @@ fn setupNeuronEnv(io: std.Io, sandbox_path: []const u8) !void { var root_comm_buf: [256]u8 = undefined; _ = setenv( "NEURON_RT_ROOT_COMM_ID", - try std.fmt.bufPrintZ(&root_comm_buf, "127.0.0.1:{d}", .{try findFreeTcpPort(io)}), + try std.mem.printSentinel(&root_comm_buf, "127.0.0.1:{d}", .{try findFreeTcpPort(io)}, 0), 1, ); _ = setenv( @@ -69,9 +69,9 @@ fn setupNeuronEnv(io: std.Io, sandbox_path: []const u8) !void { const old_path = if (std.c.getenv("PATH")) |path| std.mem.span(path) else ""; var new_path_buf: [std.posix.PATH_MAX:0]u8 = undefined; const new_path = if (old_path.len == 0) - try std.fmt.bufPrintZ(&new_path_buf, "{s}", .{bin_path}) + try std.mem.printSentinel(&new_path_buf, "{s}", .{bin_path}, 0) else - try std.fmt.bufPrintZ(&new_path_buf, "{s}:{s}", .{ bin_path, old_path }); + try std.mem.printSentinel(&new_path_buf, "{s}:{s}", .{ bin_path, old_path }, 0); if (setenv("PATH", new_path.ptr, 1) != 0) return error.SetEnvFailed; if (std.c.getenv("NEURON_CC_FLAGS") != null) return; @@ -81,7 +81,7 @@ fn setupNeuronEnv(io: std.Io, sandbox_path: []const u8) !void { var flags_buf: [512:0]u8 = undefined; const instance = try neuron.instance(); const target = @tagName(instance.compilerTarget()); - const flags = try std.fmt.bufPrintZ( + const flags = try std.mem.printSentinel( &flags_buf, "--target={s} " ++ "--optlevel=1 " ++ @@ -98,6 +98,7 @@ fn setupNeuronEnv(io: std.Io, sandbox_path: []const u8) !void { log_level, log_level, }, + 0, ); _ = setenv("NEURON_CC_FLAGS", flags.ptr, 0); diff --git a/platforms/neuron/neuron.zig b/platforms/neuron/neuron.zig index 039cc566..e0365fe9 100644 --- a/platforms/neuron/neuron.zig +++ b/platforms/neuron/neuron.zig @@ -82,16 +82,16 @@ pub const Instance = struct { pub fn instance() !Instance { var info = std.mem.zeroes(c.struct_nrt_instance_info); - info.family = @intFromEnum(Family.unknown); - info.size = @intFromEnum(Size.unknown); + info.family = @backingInt(Family.unknown); + info.size = @backingInt(Size.unknown); if (c.nrt_get_instance_info(&info, @sizeOf(c.struct_nrt_instance_info)) != c.NRT_SUCCESS) { return error.Unavailable; } return .{ - .family = @enumFromInt(info.family), - .size = @enumFromInt(info.size), + .family = @fromBackingInt(@intCast(info.family)), + .size = @fromBackingInt(@intCast(info.size)), }; } diff --git a/platforms/neuron/python_launcher.zig b/platforms/neuron/python_launcher.zig index 26ca9766..075034ab 100644 --- a/platforms/neuron/python_launcher.zig +++ b/platforms/neuron/python_launcher.zig @@ -113,7 +113,7 @@ fn configureInterpreter(init: std.process.Init, config: *c.PyConfig) ![]const u8 const args = try init.minimal.args.toSlice(arena.allocator()); const c_args = try arena.allocator().alloc([*:0]const u8, args.len); for (args, 0..) |arg, i| { - c_args[i] = (try arena.allocator().dupeZ(u8, arg)).ptr; + c_args[i] = (try arena.allocator().dupeSentinel(u8, arg, 0)).ptr; } pyStatusCheck(c.PyConfig_SetBytesArgv(config, @intCast(c_args.len), @ptrCast(c_args.ptr))); @@ -125,7 +125,7 @@ fn configureInterpreter(init: std.process.Init, config: *c.PyConfig) ![]const u8 { var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const home = try std.fmt.bufPrintZ(&buf, "{f}{d}.{d}", .{ + const home = try std.mem.printSentinel(&buf, "{f}{d}.{d}", .{ std.Io.Dir.path.fmtJoin(&.{ self_exe_dir, "..", @@ -134,7 +134,7 @@ fn configureInterpreter(init: std.process.Init, config: *c.PyConfig) ![]const u8 }), c.PY_MAJOR_VERSION, c.PY_MINOR_VERSION, - }); + }, 0); pyStatusCheck(c.PyConfig_SetBytesString(config, &config.home, home)); pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(home))); } diff --git a/platforms/platforms.zig b/platforms/platforms.zig index 6239ba77..789c9a10 100644 --- a/platforms/platforms.zig +++ b/platforms/platforms.zig @@ -24,8 +24,8 @@ pub const Platform = enum { furiosa, pub fn selectFirstAcceleratorEnabled() error{Unavailable}!Platform { - inline for (@typeInfo(Platform).@"enum".fields) |field| { - const target: Platform = @enumFromInt(field.value); + inline for (@typeInfo(Platform).@"enum".field_names) |field| { + const target: Platform = @field(Platform, field); if (comptime (target != .cpu and target.isEnabled())) return target; } diff --git a/stdx/flags.zig b/stdx/flags.zig index 6ee06eaf..52a9e6a4 100644 --- a/stdx/flags.zig +++ b/stdx/flags.zig @@ -94,7 +94,8 @@ pub fn parseProcessArgs(init_: std.process.Init.Minimal, comptime CliArgs: type) fn parse_commands(args: *std.process.Args.Iterator, comptime Commands: type) Commands { comptime assert(@typeInfo(Commands) == .@"union"); - comptime assert(std.meta.fields(Commands).len >= 2); + const field_names = std.meta.fieldNames(Commands); + comptime assert(field_names.len >= 2); const first_arg = args.next() orelse fatal( "subcommand required, expected {s}", @@ -109,10 +110,10 @@ fn parse_commands(args: *std.process.Args.Iterator, comptime Commands: type) Com } } - inline for (comptime std.meta.fields(Commands)) |field| { - comptime assert(std.mem.indexOf(u8, field.name, "_") == null); - if (std.mem.eql(u8, first_arg, field.name)) { - return @unionInit(Commands, field.name, parse_flags(args, field.type)); + inline for (field_names, std.meta.fieldTypes(Commands)) |field_name, FieldType| { + comptime assert(std.mem.indexOf(u8, field_name, "_") == null); + if (std.mem.eql(u8, first_arg, field_name)) { + return @unionInit(Commands, field_name, parse_flags(args, FieldType)); } } fatal("unknown subcommand: '{s}'", .{first_arg}); @@ -128,52 +129,61 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { return {}; } - assert(@typeInfo(Flags) == .@"struct"); + const flags_info = @typeInfo(Flags).@"struct"; - comptime var fields: [std.meta.fields(Flags).len]std.builtin.Type.StructField = undefined; + const total_num_fields = flags_info.field_names.len; + comptime var field_names: [total_num_fields][]const u8 = undefined; + comptime var field_types: [total_num_fields]type = undefined; + comptime var field_defaults: [total_num_fields]?*const anyopaque = undefined; comptime var field_count = 0; - comptime var positional_fields: []const std.builtin.Type.StructField = &.{}; + const positional_infos: ?std.lang.Type.Struct = if (@hasField(Flags, "positional")) + @typeInfo(@FieldType(Flags, "positional")).@"struct" + else + null; - comptime for (std.meta.fields(Flags)) |field| { - if (std.mem.eql(u8, field.name, "positional")) { - assert(@typeInfo(field.type) == .@"struct"); - positional_fields = std.meta.fields(field.type); + comptime for (flags_info.field_names, flags_info.field_types, flags_info.field_attrs) |field_name, FieldType, field_attr| { + if (std.mem.eql(u8, field_name, "positional")) { var optional_tail = false; - for (positional_fields) |positional_field| { - if (default_value(positional_field) == null) { + for (positional_infos.?.field_types, positional_infos.?.field_attrs) |PosType, pos_attr| { + const default: ?PosType = pos_attr.defaultValue(PosType); + if (default == null) { if (optional_tail) @panic("optional positional arguments must be last"); } else { optional_tail = true; } - switch (@typeInfo(positional_field.type)) { + switch (@typeInfo(PosType)) { .optional => |optional| { // optional flags should have a default - assert(default_value(positional_field) != null); - assert(default_value(positional_field).? == null); + assert(default != null); + assert(default.? == null); assert_valid_value_type(optional.child); }, else => { - assert_valid_value_type(positional_field.type); + assert_valid_value_type(PosType); }, } } } else { - fields[field_count] = field; + field_names[field_count] = field_name; + field_types[field_count] = FieldType; + field_defaults[field_count] = field_attr.default_value_ptr; field_count += 1; - switch (@typeInfo(field.type)) { + const default: ?FieldType = field_attr.defaultValue(FieldType); + + switch (@typeInfo(FieldType)) { .bool => { // boolean flags should have a default - debug.assertComptime(default_value(field) != null and default_value(field).? == false, "boolean flag --{s} should default to false", .{field.name}); + debug.assertComptime(default != null and default.? == false, "boolean flag --{s} should default to false", .{field_name}); }, .optional => |optional| { // optional flags should have a default - debug.assertComptime(default_value(field) != null and default_value(field).? == null, "optional flag --{s} should have a null default value", .{field.name}); + debug.assertComptime(default != null and default.? == null, "optional flag --{s} should have a null default value", .{field_name}); assert_valid_value_type(optional.child); }, else => { - assert_valid_value_type(field.type); + assert_valid_value_type(FieldType); }, } } @@ -182,29 +192,33 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { var result: Flags = undefined; // Would use std.enums.EnumFieldStruct(Flags, u32, 0) here but Flags is a struct not an Enum. var counts = comptime blk: { - const f = std.meta.fields(Flags); - var count_field_names: [fields.len][]const u8 = undefined; - var count_field_types: [fields.len]type = undefined; - var count_field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; - for (f, &count_field_names, &count_field_types, &count_field_attrs) |field, *field_name, *field_type, *field_attr| { - field_name.* = field.name; - field_type.* = u32; - field_attr.* = .{ - .@"align" = @alignOf(u32), - .default_value_ptr = @ptrCast(&@as(u32, 0)), - }; - } - break :blk @Struct(.auto, null, &count_field_names, &count_field_types, &count_field_attrs){}; + const count_field_types: [total_num_fields]type = @splat(u32); + const count_field_attrs: [total_num_fields]std.builtin.Type.Struct.FieldAttributes = @splat(.{ + .@"align" = @alignOf(u32), + .default_value_ptr = @ptrCast(&@as(u32, 0)), + }); + break :blk @Struct(.auto, null, flags_info.field_names, &count_field_types, &count_field_attrs){}; }; // When parsing arguments, we must consider longer arguments first, such that `--foo-bar=92` is // not confused for a misspelled `--foo=92`. Using `std.sort` for comptime-only values does not // work, so open-code insertion sort, and comptime assert order during the actual parsing. comptime { - for (fields[0..field_count], 0..) |*field_right, i| { - for (fields[0..i]) |*field_left| { - if (field_left.name.len < field_right.name.len) { - std.mem.swap(std.builtin.Type.StructField, field_left, field_right); + for ( + 0.., + field_names[0..field_count], + field_types[0..field_count], + field_defaults[0..field_count], + ) |i, *name_right, *type_right, *default_right| { + for ( + field_names[0..i], + field_types[0..i], + field_defaults[0..i], + ) |*name_left, *type_left, *default_left| { + if (name_left.len < name_right.len) { + std.mem.swap([]const u8, name_left, name_right); + std.mem.swap(type, type_left, type_right); + std.mem.swap(?*const anyopaque, default_left, default_right); } } } @@ -221,29 +235,29 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { } comptime var field_len_prev = std.math.maxInt(usize); - inline for (fields[0..field_count]) |field| { - const flag = comptime flag_name(field); + inline for (field_names[0..field_count], field_types[0..field_count]) |field_name, FieldType| { + const flag = comptime flag_name(field_name); - comptime assert(field_len_prev >= field.name.len); - field_len_prev = field.name.len; + comptime assert(field_len_prev >= field_name.len); + field_len_prev = field_name.len; if (std.mem.startsWith(u8, arg, flag)) { if (parsed_positional) { fatal("unexpected trailing option: '{s}'", .{arg}); } - @field(counts, field.name) += 1; - const flag_value = parse_flag(field.type, flag, arg); - @field(result, field.name) = flag_value; + @field(counts, field_name) += 1; + const flag_value = parse_flag(FieldType, flag, arg); + @field(result, field_name) = flag_value; continue :next_arg; } } - if (@hasField(Flags, "positional")) { + if (positional_infos) |pos_infos| { counts.positional += 1; switch (counts.positional - 1) { - inline 0...positional_fields.len - 1 => |positional_index| { - const positional_field = positional_fields[positional_index]; - const flag = comptime flag_name_positional(positional_field); + inline 0...pos_infos.field_names.len - 1 => |positional_index| { + const positional_field_name = pos_infos.field_names[positional_index]; + const flag = comptime flag_name_positional(positional_field_name); if (arg.len == 0) fatal("{s}: empty argument", .{flag}); // Prevent ambiguity between a flag and positional argument value. We could add @@ -252,8 +266,8 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { if (arg[0] == '-') fatal("unexpected argument: '{s}'", .{arg}); parsed_positional = true; - @field(result.positional, positional_field.name) = - parse_value(positional_field.type, flag, arg); + @field(result.positional, positional_field_name) = + parse_value(pos_infos.field_types[positional_index], flag, arg); continue :next_arg; }, else => {}, // Fall-through to the unexpected argument error. @@ -263,11 +277,15 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { fatal("unexpected argument: '{s}'", .{arg}); } - inline for (fields[0..field_count]) |field| { - const flag = flag_name(field); - switch (@field(counts, field.name)) { - 0 => if (default_value(field)) |default| { - @field(result, field.name) = default; + inline for ( + field_names[0..field_count], + field_types[0..field_count], + field_defaults[0..field_count], + ) |field_name, FieldType, default_ptr| { + const flag = flag_name(field_name); + switch (@field(counts, field_name)) { + 0 => if (defaultValue(FieldType, default_ptr)) |default| { + @field(result, field_name) = default; } else { fatal("{s}: argument is required", .{flag}); }, @@ -276,15 +294,19 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { } } - if (@hasField(Flags, "positional")) { - assert(counts.positional <= positional_fields.len); - inline for (positional_fields, 0..) |positional_field, positional_index| { + if (positional_infos) |pos_infos| { + assert(counts.positional <= pos_infos.field_names.len); + inline for ( + 0.., + pos_infos.field_names, + pos_infos.field_types, + pos_infos.field_attrs, + ) |positional_index, name, PosFieldType, attr| { if (positional_index >= counts.positional) { - const flag = comptime flag_name_positional(positional_field); - if (default_value(positional_field)) |default| { - @field(result.positional, positional_field.name) = default; + if (attr.defaultValue(PosFieldType)) |default| { + @field(result.positional, name) = default; } else { - fatal("{s}: argument is required", .{flag}); + fatal("<{s}>: argument is required", .{name}); } } } @@ -299,8 +321,8 @@ fn assert_valid_value_type(comptime T: type) void { if (@typeInfo(T) == .@"enum") { const info = @typeInfo(T).@"enum"; - assert(info.is_exhaustive); - assert(info.fields.len >= 2); + assert(info.mode == .exhaustive); + assert(info.field_names.len >= 2); return; } @@ -449,7 +471,7 @@ pub const ByteSize = struct { else ByteUnit.bytes; - _ = std.math.mul(u64, amount, @intFromEnum(unit)) catch { + _ = std.math.mul(u64, amount, @backingInt(unit)) catch { return ByteSizeParseError.BytesOverflow; }; @@ -460,7 +482,7 @@ pub const ByteSize = struct { return std.math.mul( u64, size.value, - @intFromEnum(size.unit), + @backingInt(size.unit), ) catch unreachable; } @@ -542,7 +564,7 @@ fn parse_value_float(comptime T: type, flag: []const u8, value: [:0]const u8) T fn parse_value_enum(comptime E: type, flag: []const u8, value: [:0]const u8) E { assert((flag[0] == '-' and flag[1] == '-') or flag[0] == '<'); - comptime assert(@typeInfo(E).@"enum".is_exhaustive); + comptime assert(@typeInfo(E).@"enum".mode == .exhaustive); return std.meta.stringToEnum(E, value) orelse fatal( "{s}: expected one of {s}, but found '{s}'", @@ -552,53 +574,50 @@ fn parse_value_enum(comptime E: type, flag: []const u8, value: [:0]const u8) E { fn fields_to_comma_list(comptime E: type) []const u8 { comptime { - const field_count = std.meta.fields(E).len; + const field_names = std.meta.fieldNames(E); + const field_count = field_names.len; assert(field_count >= 2); var result: []const u8 = ""; - for (std.meta.fields(E), 0..) |field, field_index| { + for (field_names, 0..) |field_name, field_index| { const separator = switch (field_index) { 0 => "", else => ", ", field_count - 1 => if (field_count == 2) " or " else ", or ", }; - result = result ++ separator ++ "'" ++ field.name ++ "'"; + result = result ++ separator ++ "'" ++ field_name ++ "'"; } return result; } } -pub fn flag_name(comptime field: std.builtin.Type.StructField) []const u8 { +pub fn flag_name(comptime field_name: []const u8) []const u8 { // TODO(Zig): Cleanup when this is fixed after Zig 0.11. // Without comptime blk, the compiler thinks the result is a runtime slice returning a UAF. return comptime blk: { - assert(!std.mem.eql(u8, field.name, "positional")); + assert(!std.mem.eql(u8, field_name, "positional")); var result: []const u8 = "--"; var index = 0; - while (std.mem.indexOf(u8, field.name[index..], "_")) |i| { - result = result ++ field.name[index..][0..i] ++ "-"; + while (std.mem.indexOf(u8, field_name[index..], "_")) |i| { + result = result ++ field_name[index..][0..i] ++ "-"; index = index + i + 1; } - result = result ++ field.name[index..]; + result = result ++ field_name[index..]; break :blk result; }; } test flag_name { - const field = @typeInfo(struct { statsd: bool }).@"struct".fields[0]; - try std.testing.expectEqualStrings(flag_name(field), "--statsd"); + try std.testing.expectEqualStrings(flag_name("statsd"), "--statsd"); } -fn flag_name_positional(comptime field: std.builtin.Type.StructField) []const u8 { - comptime assert(std.mem.indexOf(u8, field.name, "_") == null); - return "<" ++ field.name ++ ">"; +fn flag_name_positional(comptime field_name: []const u8) []const u8 { + comptime assert(std.mem.indexOf(u8, field_name, "_") == null); + return "<" ++ field_name ++ ">"; } -/// This is essentially `field.default_value`, but with a useful type instead of `?*anyopaque`. -pub fn default_value(comptime field: std.builtin.Type.StructField) ?field.type { - return if (field.default_value_ptr) |default_opaque| - @as(*const field.type, @ptrCast(@alignCast(default_opaque))).* - else - null; +pub fn defaultValue(comptime FieldType: type, default_value_ptr: ?*const anyopaque) ?FieldType { + const dp: *const FieldType = @ptrCast(@alignCast(default_value_ptr orelse return null)); + return dp.*; } diff --git a/stdx/fmt.zig b/stdx/fmt.zig index 33351d93..a62d4f7d 100644 --- a/stdx/fmt.zig +++ b/stdx/fmt.zig @@ -222,15 +222,15 @@ fn printValue( switch (@typeInfo(T)) { .@"struct" => |info| { try w.writeAll(".{ "); - inline for (info.fields, 0..) |f, i| { + inline for (info.field_names, 0..) |field_name, i| { if (i > 0) try w.writeAll(", "); if (!info.is_tuple) { try w.writeByte('.'); - try w.writeAll(f.name); + try w.writeAll(field_name); try w.writeAll(" = "); } - try printValue(w, options, @field(value, f.name), max_depth - 1); + try printValue(w, options, @field(value, field_name), max_depth - 1); } try w.writeAll(" }"); }, diff --git a/stdx/json.zig b/stdx/json.zig index d1143efc..c1f71a4a 100644 --- a/stdx/json.zig +++ b/stdx/json.zig @@ -42,22 +42,22 @@ pub fn UnionHelpers(comptime T: type) type { } pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!T { - inline for (std.meta.fields(T)) |field| { - switch (field.type) { - bool => if (source == .bool) return @unionInit(T, field.name, source.bool), + inline for (comptime std.meta.fieldNames(T), comptime std.meta.fieldTypes(T)) |field_name, Field| { + switch (Field) { + bool => if (source == .bool) return @unionInit(T, field_name, source.bool), []const u8 => switch (source) { .string, .number_string => |v| { const dupe = try allocator.dupe(u8, v); - return @unionInit(T, field.name, dupe); + return @unionInit(T, field_name, dupe); }, else => {}, }, - else => switch (@typeInfo(field.type)) { - .int => if (source == .integer) return @unionInit(T, field.name, @intCast(source.integer)), - .float => if (source == .float) return @unionInit(T, field.name, @floatCast(source.float)), - .@"struct" => if (source == .object) return @unionInit(T, field.name, try std.json.innerParseFromValue(field.type, allocator, source, options)), + else => switch (@typeInfo(Field)) { + .int => if (source == .integer) return @unionInit(T, field_name, @intCast(source.integer)), + .float => if (source == .float) return @unionInit(T, field_name, @floatCast(source.float)), + .@"struct" => if (source == .object) return @unionInit(T, field_name, try std.json.innerParseFromValue(Field, allocator, source, options)), inline else => switch (source) { - .number_string, .array => return @unionInit(T, field.name, try std.json.innerParseFromValue(field.type, allocator, source, options)), + .number_string, .array => return @unionInit(T, field_name, try std.json.innerParseFromValue(Field, allocator, source, options)), else => {}, }, }, @@ -130,9 +130,9 @@ pub fn TaggedUnionHelpers(comptime T: type, comptime tag_name: [:0]const u8) typ if (@typeInfo(T) != .@"union") { @compileError("TaggedUnion expects a union type, found " ++ @typeName(T)); } - for (std.meta.fields(T)) |field| { - if (@typeInfo(field.type) != .@"struct") { - @compileError("TaggedUnion member '" ++ field.name ++ "' must be a struct, found " ++ @typeName(field.type)); + for (std.meta.fieldNames(T), std.meta.fieldTypes(T)) |field_name, Field| { + if (@typeInfo(Field) != .@"struct") { + @compileError("TaggedUnion member '" ++ field_name ++ "' must be a struct, found " ++ @typeName(Field)); } } } @@ -152,13 +152,13 @@ pub fn TaggedUnionHelpers(comptime T: type, comptime tag_name: [:0]const u8) typ var o = source.object; const tag = (o.fetchSwapRemove(tag_name) orelse return error.MissingField).value; if (tag != .string) return error.LengthMismatch; - inline for (std.meta.fields(T)) |field| { - if (std.mem.eql(u8, field.name, tag.string)) { - const inner: field.type = std.json.parseFromValueLeaky(field.type, allocator, .{ .object = o }, options) catch |err| { - std.log.warn("failed to interpret {s} as a {s}: {}", .{ tag.string, @typeName(field.type), err }); + inline for (comptime std.meta.fieldNames(T), comptime std.meta.fieldTypes(T)) |field_name, Field| { + if (std.mem.eql(u8, field_name, tag.string)) { + const inner: Field = std.json.parseFromValueLeaky(Field, allocator, .{ .object = o }, options) catch |err| { + std.log.warn("failed to interpret {s} as a {s}: {}", .{ tag.string, @typeName(Field), err }); return err; }; - return @unionInit(T, field.name, inner); + return @unionInit(T, field_name, inner); } } return error.InvalidEnumTag; @@ -172,9 +172,9 @@ pub fn TaggedUnionHelpers(comptime T: type, comptime tag_name: [:0]const u8) typ try jw.objectField(tag_name); try jw.write(@tagName(tag)); switch (@typeInfo(@TypeOf(v))) { - .@"struct" => inline for (std.meta.fields(@TypeOf(v))) |field| { - try jw.objectField(field.name); - try jw.write(@field(v, field.name)); + .@"struct" => inline for (comptime std.meta.fieldNames(@TypeOf(v))) |name| { + try jw.objectField(name); + try jw.write(@field(v, name)); }, else => unreachable, // Would have failed with comptime check. } @@ -230,11 +230,11 @@ pub fn NeverNull(comptime T: type, comptime default_value: T) type { } pub fn fillDefaultStructValues(comptime T: type, r: *T) !void { - inline for (@typeInfo(T).Struct.fields) |field| { - if (field.default_value) |default_ptr| { - if (@field(r, field.name) == null) { - const default = @as(*align(1) const field.type, @ptrCast(default_ptr)).*; - @field(r, field.name) = default; + const info = @typeInfo(T).@"struct"; + inline for (info.field_names, info.field_types, info.field_attrs) |field_name, Field, attrs| { + if (attrs.defaultValue(Field)) |default| { + if (@field(r, field_name) == null) { + @field(r, field_name) = default; } } } diff --git a/stdx/meta.zig b/stdx/meta.zig index cb9aaeba..9b1c05af 100644 --- a/stdx/meta.zig +++ b/stdx/meta.zig @@ -20,8 +20,8 @@ pub fn isTuple(comptime T: type) bool { pub fn isStructOf(comptime T: type, comptime Elem: type) bool { return switch (@typeInfo(T)) { .@"struct" => |info| blk: { - inline for (info.fields) |field| { - if (field.type != Elem) { + inline for (info.field_types) |FieldType| { + if (FieldType != Elem) { break :blk false; } } @@ -34,8 +34,8 @@ pub fn isStructOf(comptime T: type, comptime Elem: type) bool { pub fn isStructOfAny(comptime T: type, comptime f: fn (comptime type) bool) bool { return switch (@typeInfo(T)) { .@"struct" => |info| blk: { - inline for (info.fields) |field| { - if (f(field.type) == false) { + inline for (info.field_types) |FieldType| { + if (f(FieldType) == false) { break :blk false; } } @@ -121,9 +121,9 @@ pub fn asSlice(comptime T: type) type { } pub fn TupleRange(comptime T: type, comptime start: ?usize, comptime end: ?usize) type { - const fields = std.meta.fields(T); + const all_field_types = std.meta.fieldTypes(T); const start_ = start orelse 0; - const end_ = end orelse fields.len; + const end_ = end orelse all_field_types.len; if (start_ == end_) { return @Tuple(&.{}); @@ -131,17 +131,17 @@ pub fn TupleRange(comptime T: type, comptime start: ?usize, comptime end: ?usize var field_types: [end_ - start_]type = undefined; inline for (start_..end_, 0..) |i, j| { - field_types[j] = fields[i].type; + field_types[j] = all_field_types[i]; } return @Tuple(&field_types); } pub fn FnParam(comptime func: anytype, comptime n: comptime_int) type { - const params = @typeInfo(@TypeOf(func)).@"fn".params; + const params = @typeInfo(@TypeOf(func)).@"fn".param_types; if (n >= params.len) { @compileError("param doesn't exist in func"); } - return params[n].type orelse @compileError("anytype is not supported"); + return params[n] orelse @compileError("anytype is not supported"); } pub fn FnReturn(comptime func: anytype) type { @@ -167,8 +167,8 @@ pub fn FnReturnErrorSet(comptime func: anytype) ?type { pub fn Head(comptime Tuple: type) type { return switch (@typeInfo(Tuple)) { .@"struct" => |struct_info| { - if (struct_info.fields.len == 0) @compileError("Can't tail empty tuple"); - return struct_info.fields[0].type; + if (struct_info.field_types.len == 0) @compileError("Can't tail empty tuple"); + return struct_info.field_types[0]; }, else => @compileError("Head works on tuple type"), }; @@ -177,10 +177,10 @@ pub fn Head(comptime Tuple: type) type { pub fn Tail(comptime Tuple: type) type { return switch (@typeInfo(Tuple)) { .@"struct" => |struct_info| { - if (struct_info.fields.len == 0) @compileError("Can't tail empty tuple"); - var types: [struct_info.fields.len - 1]type = undefined; - for (struct_info.fields[1..], &types) |field, *type_| { - type_.* = field.type; + if (struct_info.field_types.len == 0) @compileError("Can't tail empty tuple"); + var types: [struct_info.field_types.len - 1]type = undefined; + for (struct_info.field_types[1..], &types) |FieldType, *type_| { + type_.* = FieldType; } return @Tuple(&types); }, diff --git a/stdx/segmented_list.zig b/stdx/segmented_list.zig index ee115ce2..567534be 100644 --- a/stdx/segmented_list.zig +++ b/stdx/segmented_list.zig @@ -107,7 +107,7 @@ pub fn SegmentedList(comptime T: type, comptime prealloc_item_count: usize) type pub const prealloc_count = prealloc_item_count; fn AtType(comptime SelfType: type) type { - if (@typeInfo(SelfType).pointer.is_const) { + if (@typeInfo(SelfType).pointer.attrs.@"const") { return *const T; } else { return *T; diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 8d1fb590..1b19e638 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -28,7 +28,7 @@ pub inline fn stackSlice(comptime max_len: usize, T: type, len: usize) []T { return storage[0..len]; } -pub const noalloc: std.mem.Allocator = if (builtin.mode == .ReleaseFast) undefined else std.testing.failing_allocator; +pub const noalloc: std.mem.Allocator = if (builtin.mode == .fast) undefined else std.testing.failing_allocator; pub fn arenaWithCapacity(parent: std.mem.Allocator, initial_capacity: usize) std.mem.Allocator.Error!std.heap.ArenaAllocator { var a: std.heap.ArenaAllocator = .init(parent); @@ -43,7 +43,7 @@ pub fn pinToCore(core_id: usize) void { if (builtin.os.tag == .linux) { const CPUSet = std.bit_set.ArrayBitSet(usize, std.os.linux.CPU_SETSIZE * @sizeOf(usize)); - var set: CPUSet = .initEmpty(); + var set: CPUSet = .empty; set.set(core_id); std.os.linux.sched_setaffinity(0, @ptrCast(&set.masks)) catch {}; } @@ -86,3 +86,60 @@ pub fn Once(comptime f: anytype) type { } }; } + +/// Packed storage of sub-byte values +/// +/// Note: before Zig 0.17, @Vector(u2, 4) was always one byte and could be used as a storage type, +/// But that's not true anymore. +pub fn Packed(T: type) type { + return switch (@bitSizeOf(T)) { + 2 => packed struct(u8) { + x: T, + y: T, + z: T, + w: T, + + const P = @This(); + pub fn pack(x: T, y: T, z: T, w: T) P { + return .{ .x = x, .y = y, .z = z, .w = w }; + } + + pub fn unpack(p: P) [4]T { + return .{ p.x, p.y, p.z, p.w }; + } + + pub fn formatNumber(p: P, w: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + try formatPackedNumber(T, &p.unpack(), w, n); + } + }, + 4 => packed struct(u8) { + x: T, + y: T, + + const P = @This(); + pub fn pack(x: T, y: T) P { + return .{ .x = x, .y = y }; + } + + pub fn unpack(p: P) [2]T { + return .{ p.x, p.y }; + } + + pub fn formatNumber(p: P, w: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + try formatPackedNumber(T, &p.unpack(), w, n); + } + }, + else => debug.compileError("stdx.Packed expects a bitsize of 2 or 4, got type {} with bitsize {d}", .{ T, @bitSizeOf(T) }), + }; +} + +fn formatPackedNumber(T: type, values: []const T, w: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + for (0.., values) |i, val| { + if (i > 0) try w.writeByte(','); + switch (@typeInfo(T)) { + .@"struct" => try val.formatNumber(w, n), + .int => try fmt.formatInt(val, n, w), + else => try w.print("{}", .{val}), + } + } +} diff --git a/third_party/arocc/arocc.bazel b/third_party/arocc/arocc.bazel index 853399f5..a5de1d98 100644 --- a/third_party/arocc/arocc.bazel +++ b/third_party/arocc/arocc.bazel @@ -80,19 +80,10 @@ zig_options( ], ) -zig_library( - name = "zig", - srcs = glob(["deps/zig/**/*.zig"]), - main = "deps/zig/lib.zig", -) - zig_library( name = "backend", main = "src/backend.zig", - deps = [ - ":build_options", - ":zig", - ], + deps = [":build_options"], ) zig_library( @@ -127,7 +118,6 @@ zig_binary( ":assembly_backend", ":build_options", ], - zigopts = ["-fm"], ) zig_binary( diff --git a/third_party/arocc/repo.bzl b/third_party/arocc/repo.bzl index 98a10e38..15ff6d7f 100644 --- a/third_party/arocc/repo.bzl +++ b/third_party/arocc/repo.bzl @@ -1,9 +1,10 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") def repo(): + # https://github.com/zml/arocc/releases/tag/0.17.0 git_repository( name = "arocc", - remote = "https://github.com/Vexu/arocc.git", - commit = "5f5a050569a95ecc40a426f0c3666ae7ef987ede", + remote = "https://github.com/zml/arocc", + commit = "d0c8c4d9c55daa7ef6e40cf0f630a5b5e900989b", build_file = Label("//third_party/arocc:arocc.bazel"), ) diff --git a/third_party/libvaxis/bump-zig.patch b/third_party/libvaxis/bump-zig.patch deleted file mode 100644 index 45e221b7..00000000 --- a/third_party/libvaxis/bump-zig.patch +++ /dev/null @@ -1,2019 +0,0 @@ -diff --git a/build.zig.zon b/build.zig.zon -index 6192c1a..4ca04f8 100644 ---- a/build.zig.zon -+++ b/build.zig.zon -@@ -2,15 +2,15 @@ - .name = .vaxis, - .fingerprint = 0x14fbbb94fc556305, - .version = "0.5.1", -- .minimum_zig_version = "0.15.1", -+ .minimum_zig_version = "0.16.0-dev.2261+d6b3dd25a", - .dependencies = .{ - .zigimg = .{ -- .url = "git+https://github.com/zigimg/zigimg#eab2522c023b9259db8b13f2f90d609b7437e5f6", -- .hash = "zigimg-0.1.0-8_eo2vUZFgAAtN1c6dAO5DdqL0d4cEWHtn6iR5ucZJti", -+ .url = "https://github.com/unexge/zigimg/archive/refs/heads/master.zip", -+ .hash = "zigimg-0.1.0-8_eo2tGrFwD-VzWa_E8HuIuNenM2US7PRXBj5EJdGfrw", - }, - .uucode = .{ -- .url = "git+https://github.com/jacobsandlund/uucode#5f05f8f83a75caea201f12cc8ea32a2d82ea9732", -- .hash = "uucode-0.1.0-ZZjBPj96QADXyt5sqwBJUnhaDYs_qBeeKijZvlRa0eqM", -+ .url = "https://github.com/jacobsandlund/uucode/archive/refs/heads/zig-0.16.zip", -+ .hash = "uucode-0.2.0-ZZjBPkJiVABQGPdyw9BOtU2sflHv3vovBzmYtPRj2hvm", - }, - }, - .paths = .{ -diff --git a/examples/cli.zig b/examples/cli.zig -index 3b44676..2b9a288 100644 ---- a/examples/cli.zig -+++ b/examples/cli.zig -@@ -4,30 +4,23 @@ const Cell = vaxis.Cell; - const TextInput = vaxis.widgets.TextInput; - - const log = std.log.scoped(.main); --pub fn main() !void { -- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -- defer { -- const deinit_status = gpa.deinit(); -- if (deinit_status == .leak) { -- log.err("memory leak", .{}); -- } -- } -- const alloc = gpa.allocator(); -+pub fn main(init: std.process.Init) !void { -+ const alloc = init.gpa; - - var buffer: [1024]u8 = undefined; -- var tty = try vaxis.Tty.init(&buffer); -+ var tty = try vaxis.Tty.init(init.io, &buffer); - defer tty.deinit(); - - var vx = try vaxis.init(alloc, .{}); - defer vx.deinit(alloc, tty.writer()); - -- var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx }; -+ var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx, .io = init.io, .queue = .{ .io = init.io } }; - try loop.init(); - - try loop.start(); - defer loop.stop(); - -- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_s); -+ try vx.queryTerminal(tty.writer(), init.io, 1 * std.time.ns_per_s); - - var text_input = TextInput.init(alloc); - defer text_input.deinit(); -diff --git a/examples/counter.zig b/examples/counter.zig -index d19af42..e68420b 100644 ---- a/examples/counter.zig -+++ b/examples/counter.zig -@@ -111,13 +111,10 @@ const Model = struct { - } - }; - --pub fn main() !void { -- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -- defer _ = gpa.deinit(); -+pub fn main(init: std.process.Init) !void { -+ const allocator = init.gpa; - -- const allocator = gpa.allocator(); -- -- var app = try vxfw.App.init(allocator); -+ var app = try vxfw.App.init(allocator, init.io); - defer app.deinit(); - - // We heap allocate our model because we will require a stable pointer to it in our Button -diff --git a/examples/fuzzy.zig b/examples/fuzzy.zig -index 6ffbe6a..7fef980 100644 ---- a/examples/fuzzy.zig -+++ b/examples/fuzzy.zig -@@ -11,7 +11,6 @@ const Model = struct { - - /// Used for filtered RichText Spans and result - arena: std.heap.ArenaAllocator, -- filtered: std.ArrayList(vxfw.RichText), - result: []const u8, - - pub fn init(gpa: std.mem.Allocator) !*Model { -@@ -201,31 +200,24 @@ fn toLower(arena: std.mem.Allocator, src: []const u8) std.mem.Allocator.Error![] - return lower; - } - --pub fn main() !void { -- var debug_allocator = std.heap.GeneralPurposeAllocator(.{}){}; -- defer _ = debug_allocator.deinit(); -- -- const gpa = debug_allocator.allocator(); -+pub fn main(init: std.process.Init) !void { -+ const gpa = init.gpa; - -- var app = try vxfw.App.init(gpa); -+ var app = try vxfw.App.init(gpa, init.io); - errdefer app.deinit(); - - const model = try Model.init(gpa); - defer model.deinit(gpa); - - // Run the command -- var fd = std.process.Child.init(&.{"fd"}, gpa); -- fd.stdout_behavior = .Pipe; -- fd.stderr_behavior = .Pipe; -- var stdout = std.ArrayList(u8).empty; -- var stderr = std.ArrayList(u8).empty; -- defer stdout.deinit(gpa); -- defer stderr.deinit(gpa); -- try fd.spawn(); -- try fd.collectOutput(gpa, &stdout, &stderr, 10_000_000); -- _ = try fd.wait(); -- -- var iter = std.mem.splitScalar(u8, stdout.items, '\n'); -+ const result = try std.process.run(gpa, init.io, .{ -+ .argv = &.{"fd"}, -+ .stdout_limit = .limited(10_000_000), -+ }); -+ defer gpa.free(result.stdout); -+ defer gpa.free(result.stderr); -+ -+ var iter = std.mem.splitScalar(u8, result.stdout, '\n'); - while (iter.next()) |line| { - if (line.len == 0) continue; - try model.list.append(gpa, .{ .text = line }); -@@ -235,8 +227,8 @@ pub fn main() !void { - app.deinit(); - - if (model.result.len > 0) { -- _ = try std.posix.write(std.posix.STDOUT_FILENO, model.result); -- _ = try std.posix.write(std.posix.STDOUT_FILENO, "\n"); -+ try std.Io.File.stdout().writeStreamingAll(init.io, model.result); -+ try std.Io.File.stdout().writeStreamingAll(init.io, "\n"); - } else { - std.process.exit(130); - } -diff --git a/examples/image.zig b/examples/image.zig -index c14bbdb..af62862 100644 ---- a/examples/image.zig -+++ b/examples/image.zig -@@ -8,41 +8,34 @@ const Event = union(enum) { - winsize: vaxis.Winsize, - }; - --pub fn main() !void { -- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -- defer { -- const deinit_status = gpa.deinit(); -- if (deinit_status == .leak) { -- log.err("memory leak", .{}); -- } -- } -- const alloc = gpa.allocator(); -+pub fn main(init: std.process.Init) !void { -+ const alloc = init.gpa; - - var buffer: [1024]u8 = undefined; -- var tty = try vaxis.Tty.init(&buffer); -+ var tty = try vaxis.Tty.init(init.io, &buffer); - defer tty.deinit(); - - var vx = try vaxis.init(alloc, .{}); - defer vx.deinit(alloc, tty.writer()); - -- var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx }; -+ var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx, .io = init.io, .queue = .{ .io = init.io } }; - try loop.init(); - - try loop.start(); - defer loop.stop(); - - try vx.enterAltScreen(tty.writer()); -- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_s); -+ try vx.queryTerminal(tty.writer(), init.io, 1 * std.time.ns_per_s); - - var read_buffer: [1024 * 1024]u8 = undefined; // 1MB buffer -- var img1 = try vaxis.zigimg.Image.fromFilePath(alloc, "examples/zig.png", &read_buffer); -+ var img1 = try vaxis.zigimg.Image.fromFilePath(alloc, init.io, "examples/zig.png", &read_buffer); - defer img1.deinit(alloc); - - const imgs = [_]vaxis.Image{ - try vx.transmitImage(alloc, tty.writer(), &img1, .rgba), - // var img1 = try vaxis.zigimg.Image.fromFilePath(alloc, "examples/zig.png"); -- // try vx.loadImage(alloc, tty.writer(), .{ .path = "examples/zig.png" }), -- try vx.loadImage(alloc, tty.writer(), .{ .path = "examples/vaxis.png" }), -+ // try vx.loadImage(alloc, init.io, tty.writer(), .{ .path = "examples/zig.png" }), -+ try vx.loadImage(alloc, init.io, tty.writer(), .{ .path = "examples/vaxis.png" }), - }; - defer vx.freeImage(tty.writer(), imgs[0].id); - defer vx.freeImage(tty.writer(), imgs[1].id); -diff --git a/examples/list_view.zig b/examples/list_view.zig -index 24cbf8d..5b961e9 100644 ---- a/examples/list_view.zig -+++ b/examples/list_view.zig -@@ -52,13 +52,10 @@ const Model = struct { - } - }; - --pub fn main() !void { -- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -- defer _ = gpa.deinit(); -+pub fn main(init: std.process.Init) !void { -+ const allocator = init.gpa; - -- const allocator = gpa.allocator(); -- -- var app = try vxfw.App.init(allocator); -+ var app = try vxfw.App.init(allocator, init.io); - defer app.deinit(); - - const model = try allocator.create(Model); -diff --git a/examples/main.zig b/examples/main.zig -index 2a4b96b..3caa7d9 100644 ---- a/examples/main.zig -+++ b/examples/main.zig -@@ -3,25 +3,17 @@ const vaxis = @import("vaxis"); - const Cell = vaxis.Cell; - - const log = std.log.scoped(.main); --pub fn main() !void { -- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -- defer { -- const deinit_status = gpa.deinit(); -- //fail test; can't try in defer as defer is executed after we return -- if (deinit_status == .leak) { -- log.err("memory leak", .{}); -- } -- } -- const alloc = gpa.allocator(); -+pub fn main(init: std.process.Init) !void { -+ const alloc = init.gpa; - - var buffer: [1024]u8 = undefined; -- var tty = try vaxis.Tty.init(&buffer); -+ var tty = try vaxis.Tty.init(init.io, &buffer); - defer tty.deinit(); - - var vx = try vaxis.init(alloc, .{}); - defer vx.deinit(alloc, tty.writer()); - -- var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx }; -+ var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx, .io = init.io, .queue = .{ .io = init.io } }; - try loop.init(); - - try loop.start(); -@@ -29,7 +21,7 @@ pub fn main() !void { - - // Optionally enter the alternate screen - try vx.enterAltScreen(tty.writer()); -- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_s); -+ try vx.queryTerminal(tty.writer(), init.io, 1 * std.time.ns_per_s); - - // We'll adjust the color index every keypress - var color_idx: u8 = 0; -diff --git a/examples/scroll.zig b/examples/scroll.zig -index 0f92573..adf1f44 100644 ---- a/examples/scroll.zig -+++ b/examples/scroll.zig -@@ -151,13 +151,10 @@ const Model = struct { - } - }; - --pub fn main() !void { -- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -- defer _ = gpa.deinit(); -+pub fn main(init: std.process.Init) !void { -+ const allocator = init.gpa; - -- const allocator = gpa.allocator(); -- -- var app = try vxfw.App.init(allocator); -+ var app = try vxfw.App.init(allocator, init.io); - errdefer app.deinit(); - - var arena = std.heap.ArenaAllocator.init(allocator); -diff --git a/examples/split_view.zig b/examples/split_view.zig -index 78d4e6f..5c48432 100644 ---- a/examples/split_view.zig -+++ b/examples/split_view.zig -@@ -49,13 +49,10 @@ const Model = struct { - } - }; - --pub fn main() !void { -- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -- defer _ = gpa.deinit(); -+pub fn main(init: std.process.Init) !void { -+ const allocator = init.gpa; - -- const allocator = gpa.allocator(); -- -- var app = try vxfw.App.init(allocator); -+ var app = try vxfw.App.init(allocator, init.io); - defer app.deinit(); - - const model = try allocator.create(Model); -diff --git a/examples/table.zig b/examples/table.zig -index 430c3d5..c489828 100644 ---- a/examples/table.zig -+++ b/examples/table.zig -@@ -14,16 +14,14 @@ const ActiveSection = enum { - btm, - }; - --pub fn main() !void { -- var gpa = heap.GeneralPurposeAllocator(.{}){}; -- defer if (gpa.detectLeaks()) log.err("Memory leak detected!", .{}); -- const alloc = gpa.allocator(); -+pub fn main(init: std.process.Init) !void { -+ const alloc = init.gpa; - - // Users set up below the main function - const users_buf = try alloc.dupe(User, users[0..]); - - var buffer: [1024]u8 = undefined; -- var tty = try vaxis.Tty.init(&buffer); -+ var tty = try vaxis.Tty.init(init.io, &buffer); - defer tty.deinit(); - const tty_writer = tty.writer(); - var vx = try vaxis.init(alloc, .{ -@@ -35,12 +33,12 @@ pub fn main() !void { - key_press: vaxis.Key, - winsize: vaxis.Winsize, - table_upd, -- }) = .{ .tty = &tty, .vaxis = &vx }; -+ }) = .{ .tty = &tty, .vaxis = &vx, .io = init.io, .queue = .{ .io = init.io } }; - try loop.init(); - try loop.start(); - defer loop.stop(); - try vx.enterAltScreen(tty.writer()); -- try vx.queryTerminal(tty.writer(), 250 * std.time.ns_per_ms); -+ try vx.queryTerminal(tty.writer(), init.io, 250 * std.time.ns_per_ms); - - const logo = - \\░█░█░█▀█░█░█░▀█▀░█▀▀░░░▀█▀░█▀█░█▀▄░█░░░█▀▀░ -diff --git a/examples/text_input.zig b/examples/text_input.zig -index cea0466..5c3c6c6 100644 ---- a/examples/text_input.zig -+++ b/examples/text_input.zig -@@ -18,20 +18,12 @@ const Event = union(enum) { - foo: u8, - }; - --pub fn main() !void { -- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -- defer { -- const deinit_status = gpa.deinit(); -- //fail test; can't try in defer as defer is executed after we return -- if (deinit_status == .leak) { -- log.err("memory leak", .{}); -- } -- } -- const alloc = gpa.allocator(); -+pub fn main(init: std.process.Init) !void { -+ const alloc = init.gpa; - - // Initalize a tty - var buffer: [1024]u8 = undefined; -- var tty = try vaxis.Tty.init(&buffer); -+ var tty = try vaxis.Tty.init(init.io, &buffer); - defer tty.deinit(); - - // Use a buffered writer for better performance. There are a lot of writes -@@ -47,6 +39,8 @@ pub fn main() !void { - var loop: vaxis.Loop(Event) = .{ - .vaxis = &vx, - .tty = &tty, -+ .io = init.io, -+ .queue = .{ .io = init.io }, - }; - try loop.init(); - -@@ -71,7 +65,7 @@ pub fn main() !void { - try writer.flush(); - // Sends queries to terminal to detect certain features. This should - // _always_ be called, but is left to the application to decide when -- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_s); -+ try vx.queryTerminal(tty.writer(), init.io, 1 * std.time.ns_per_s); - - // The main event loop. Vaxis provides a thread safe, blocking, buffered - // queue which can serve as the primary event queue for an application -@@ -94,8 +88,10 @@ pub fn main() !void { - } else if (key.matches('n', .{ .ctrl = true })) { - try vx.notify(tty.writer(), "vaxis", "hello from vaxis"); - loop.stop(); -- var child = std.process.Child.init(&.{"nvim"}, alloc); -- _ = try child.spawnAndWait(); -+ var child = try std.process.spawn(init.io, .{ -+ .argv = &.{"nvim"}, -+ }); -+ _ = try child.wait(init.io); - try loop.start(); - try vx.enterAltScreen(tty.writer()); - vx.queueRefresh(); -diff --git a/examples/text_view.zig b/examples/text_view.zig -index 73e8e54..d49cb5b 100644 ---- a/examples/text_view.zig -+++ b/examples/text_view.zig -@@ -9,31 +9,24 @@ const Event = union(enum) { - winsize: vaxis.Winsize, - }; - --pub fn main() !void { -- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -- -- defer { -- const deinit_status = gpa.deinit(); -- if (deinit_status == .leak) { -- log.err("memory leak", .{}); -- } -- } -- -- const alloc = gpa.allocator(); -+pub fn main(init: std.process.Init) !void { -+ const alloc = init.gpa; - var buffer: [1024]u8 = undefined; -- var tty = try vaxis.Tty.init(&buffer); -+ var tty = try vaxis.Tty.init(init.io, &buffer); - defer tty.deinit(); - var vx = try vaxis.init(alloc, .{}); - defer vx.deinit(alloc, tty.writer()); - var loop: vaxis.Loop(Event) = .{ - .vaxis = &vx, - .tty = &tty, -+ .io = init.io, -+ .queue = .{ .io = init.io }, - }; - try loop.init(); - try loop.start(); - defer loop.stop(); - try vx.enterAltScreen(tty.writer()); -- try vx.queryTerminal(tty.writer(), 20 * std.time.ns_per_s); -+ try vx.queryTerminal(tty.writer(), init.io, 20 * std.time.ns_per_s); - var text_view = TextView{}; - var text_view_buffer = TextView.Buffer{}; - defer text_view_buffer.deinit(alloc); -@@ -61,6 +54,6 @@ pub fn main() !void { - win.clear(); - text_view.draw(win, text_view_buffer); - try vx.render(tty.writer()); -- try tty.writer.flush(); -+ try tty.writer().flush(); - } - } -diff --git a/examples/vaxis.zig b/examples/vaxis.zig -index 5ca566c..8a12a66 100644 ---- a/examples/vaxis.zig -+++ b/examples/vaxis.zig -@@ -9,32 +9,24 @@ const Event = union(enum) { - - pub const panic = vaxis.panic_handler; - --pub fn main() !void { -- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -- defer { -- const deinit_status = gpa.deinit(); -- //fail test; can't try in defer as defer is executed after we return -- if (deinit_status == .leak) { -- std.log.err("memory leak", .{}); -- } -- } -- const alloc = gpa.allocator(); -+pub fn main(init: std.process.Init) !void { -+ const alloc = init.gpa; - - var buffer: [1024]u8 = undefined; -- var tty = try vaxis.Tty.init(&buffer); -+ var tty = try vaxis.Tty.init(init.io, &buffer); - defer tty.deinit(); - - var vx = try vaxis.init(alloc, .{}); - defer vx.deinit(alloc, tty.writer()); - -- var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx }; -+ var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx, .io = init.io, .queue = .{ .io = init.io } }; - try loop.init(); - - try loop.start(); - defer loop.stop(); - - try vx.enterAltScreen(tty.writer()); -- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_s); -+ try vx.queryTerminal(tty.writer(), init.io, 1 * std.time.ns_per_s); - - try vx.queryColor(tty.writer(), .fg); - try vx.queryColor(tty.writer(), .bg); -@@ -84,7 +76,7 @@ pub fn main() !void { - // try vx.render(bw.writer().any()); - // try bw.flush(); - try vx.render(tty.writer()); -- std.Thread.sleep(16 * std.time.ns_per_ms); -+ try std.Io.sleep(init.io, std.Io.Duration.fromMilliseconds(16), .real); - switch (dir) { - .up => { - pct += 1; -diff --git a/examples/view.zig b/examples/view.zig -index 1b01dfd..3f5108a 100644 ---- a/examples/view.zig -+++ b/examples/view.zig -@@ -13,15 +13,8 @@ const Event = union(enum) { - winsize: vaxis.Winsize, - }; - --pub fn main() !void { -- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -- defer { -- const deinit_status = gpa.deinit(); -- if (deinit_status == .leak) { -- log.err("memory leak", .{}); -- } -- } -- const alloc = gpa.allocator(); -+pub fn main(init: std.process.Init) !void { -+ const alloc = init.gpa; - - var world_map: []const u8 = lg_world_map; - var map_width = lg_map_width; -@@ -45,7 +38,7 @@ pub fn main() !void { - }); - - var buffer: [1024]u8 = undefined; -- var tty = try vaxis.Tty.init(&buffer); -+ var tty = try vaxis.Tty.init(init.io, &buffer); - defer tty.deinit(); - - const writer = tty.writer(); -@@ -58,13 +51,15 @@ pub fn main() !void { - var loop: vaxis.Loop(Event) = .{ - .vaxis = &vx, - .tty = &tty, -+ .io = init.io, -+ .queue = .{ .io = init.io }, - }; - try loop.init(); - try loop.start(); - defer loop.stop(); - try vx.enterAltScreen(writer); - try writer.flush(); -- try vx.queryTerminal(tty.writer(), 20 * std.time.ns_per_s); -+ try vx.queryTerminal(tty.writer(), init.io, 20 * std.time.ns_per_s); - - // Initialize Views - // - Large Map -diff --git a/examples/vt.zig b/examples/vt.zig -index fc13149..6d7f647 100644 ---- a/examples/vt.zig -+++ b/examples/vt.zig -@@ -9,33 +9,24 @@ const Event = union(enum) { - - pub const panic = vaxis.panic_handler; - --pub fn main() !void { -- var gpa = std.heap.GeneralPurposeAllocator(.{}){}; -- defer { -- const deinit_status = gpa.deinit(); -- //fail test; can't try in defer as defer is executed after we return -- if (deinit_status == .leak) { -- std.log.err("memory leak", .{}); -- } -- } -- const alloc = gpa.allocator(); -+pub fn main(init: std.process.Init) !void { -+ const alloc = init.gpa; - - var buffer: [1024]u8 = undefined; -- var tty = try vaxis.Tty.init(&buffer); -+ var tty = try vaxis.Tty.init(init.io, &buffer); - const writer = tty.writer(); - var vx = try vaxis.init(alloc, .{}); - defer vx.deinit(alloc, writer); - -- var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx }; -+ var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx, .io = init.io, .queue = .{ .io = init.io } }; - try loop.init(); - - try loop.start(); - defer loop.stop(); - - try vx.enterAltScreen(writer); -- try vx.queryTerminal(writer, 1 * std.time.ns_per_s); -- var env = try std.process.getEnvMap(alloc); -- defer env.deinit(); -+ try vx.queryTerminal(writer, init.io, 1 * std.time.ns_per_s); -+ const env = init.environ_map; - - const vt_opts: vaxis.widgets.Terminal.Options = .{ - .winsize = .{ -@@ -52,8 +43,9 @@ pub fn main() !void { - var write_buf: [4096]u8 = undefined; - var vt = try vaxis.widgets.Terminal.init( - alloc, -+ init.io, - &argv, -- &env, -+ env, - vt_opts, - &write_buf, - ); -@@ -62,7 +54,7 @@ pub fn main() !void { - - var redraw: bool = false; - while (true) { -- std.Thread.sleep(8 * std.time.ns_per_ms); -+ std.Io.sleep(init.io, .fromMilliseconds(8), .awake) catch {}; - // try vt events first - while (vt.tryEvent()) |event| { - redraw = true; -diff --git a/src/Loop.zig b/src/Loop.zig -index a11e2f7..cd7e9d9 100644 ---- a/src/Loop.zig -+++ b/src/Loop.zig -@@ -18,8 +18,9 @@ pub fn Loop(comptime T: type) type { - - tty: *Tty, - vaxis: *Vaxis, -+ io: std.Io, - -- queue: Queue(T, 512) = .{}, -+ queue: Queue(T, 512), - thread: ?std.Thread = null, - should_quit: bool = false, - -@@ -214,7 +215,7 @@ pub fn handleEventGeneric(self: anytype, vx: *Vaxis, cache: *GraphemeCache, Even - } - }, - .cap_da1 => { -- std.Thread.Futex.wake(&vx.query_futex, 10); -+ std.Io.futexWake(self.io, u32, &vx.query_futex.raw, 10); - vx.queries_done.store(true, .unordered); - }, - .mouse => |mouse| { -@@ -359,7 +360,7 @@ pub fn handleEventGeneric(self: anytype, vx: *Vaxis, cache: *GraphemeCache, Even - vx.caps.multi_cursor = true; - }, - .cap_da1 => { -- std.Thread.Futex.wake(&vx.query_futex, 10); -+ std.Io.futexWake(self.io, u32, &vx.query_futex.raw, 10); - vx.queries_done.store(true, .unordered); - }, - .winsize => |winsize| { -@@ -386,13 +387,13 @@ test Loop { - foo: u8, - }; - -- var tty = try vaxis.Tty.init(&.{}); -+ var tty = try vaxis.Tty.init(std.testing.io); - defer tty.deinit(); - - var vx = try vaxis.init(std.testing.allocator, .{}); - defer vx.deinit(std.testing.allocator, tty.writer()); - -- var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx }; -+ var loop: vaxis.Loop(Event) = .{ .tty = &tty, .vaxis = &vx, .io = std.testing.io, .queue = .{ .io = std.testing.io } }; - try loop.init(); - - try loop.start(); -@@ -400,5 +401,5 @@ test Loop { - - // Optionally enter the alternate screen - try vx.enterAltScreen(tty.writer()); -- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_ms); -+ try vx.queryTerminal(tty.writer(), std.testing.io, 1 * std.time.ns_per_ms); - } -diff --git a/src/Parser.zig b/src/Parser.zig -index 6358269..1b125dd 100644 ---- a/src/Parser.zig -+++ b/src/Parser.zig -@@ -118,7 +118,7 @@ inline fn parseGround(input: []const u8) !Result { - var grapheme_len: usize = 0; - var cp_count: usize = 0; - -- while (grapheme_iter.next()) |result| { -+ while (grapheme_iter.nextCodePoint()) |result| { - cp_count += 1; - if (result.is_break) { - // Found the first grapheme boundary -diff --git a/src/Vaxis.zig b/src/Vaxis.zig -index a6a3830..2e24e9b 100644 ---- a/src/Vaxis.zig -+++ b/src/Vaxis.zig -@@ -3,7 +3,7 @@ const builtin = @import("builtin"); - const atomic = std.atomic; - const base64Encoder = std.base64.standard.Encoder; - const zigimg = @import("zigimg"); --const IoWriter = std.io.Writer; -+const IoWriter = std.Io.Writer; - - const Cell = @import("Cell.zig"); - const Image = @import("Image.zig"); -@@ -250,10 +250,10 @@ pub fn exitAltScreen(self: *Vaxis, tty: *IoWriter) !void { - /// - /// This call will block until Vaxis.query_futex is woken up, or the timeout. - /// Event loops can wake up this futex when cap_da1 is received --pub fn queryTerminal(self: *Vaxis, tty: *IoWriter, timeout_ns: u64) !void { -+pub fn queryTerminal(self: *Vaxis, tty: *IoWriter, io: std.Io, timeout_ns: u64) !void { - try self.queryTerminalSend(tty); - // 1 second timeout -- std.Thread.Futex.timedWait(&self.query_futex, 0, timeout_ns) catch {}; -+ std.Io.futexWaitTimeout(io, u32, &self.query_futex.raw, 0, .{ .duration = .{ .raw = .{ .nanoseconds = @intCast(timeout_ns) }, .clock = .awake } }) catch {}; - self.queries_done.store(true, .unordered); - try self.enableDetectedFeatures(tty); - } -@@ -265,7 +265,7 @@ pub fn queryTerminalSend(vx: *Vaxis, tty: *IoWriter) !void { - vx.queries_done.store(false, .unordered); - - // TODO: re-enable this -- // const colorterm = std.posix.getenv("COLORTERM") orelse ""; -+ // const colorterm = std.c.getenv("COLORTERM") orelse ""; - // if (std.mem.eql(u8, colorterm, "truecolor") or - // std.mem.eql(u8, colorterm, "24bit")) - // { -@@ -320,22 +320,22 @@ pub fn enableDetectedFeatures(self: *Vaxis, tty: *IoWriter) !void { - }, - else => { - // Apply any environment variables -- if (std.posix.getenv("TERMUX_VERSION")) |_| -+ if (std.c.getenv("TERMUX_VERSION")) |_| - self.sgr = .legacy; -- if (std.posix.getenv("VHS_RECORD")) |_| { -+ if (std.c.getenv("VHS_RECORD")) |_| { - self.caps.unicode = .wcwidth; - self.caps.kitty_keyboard = false; - self.sgr = .legacy; - } -- if (std.posix.getenv("TERM_PROGRAM")) |prg| { -- if (std.mem.eql(u8, prg, "vscode")) -+ if (std.c.getenv("TERM_PROGRAM")) |prg| { -+ if (std.mem.eql(u8, std.mem.span(prg), "vscode")) - self.sgr = .legacy; - } -- if (std.posix.getenv("VAXIS_FORCE_LEGACY_SGR")) |_| -+ if (std.c.getenv("VAXIS_FORCE_LEGACY_SGR")) |_| - self.sgr = .legacy; -- if (std.posix.getenv("VAXIS_FORCE_WCWIDTH")) |_| -+ if (std.c.getenv("VAXIS_FORCE_WCWIDTH")) |_| - self.caps.unicode = .wcwidth; -- if (std.posix.getenv("VAXIS_FORCE_UNICODE")) |_| -+ if (std.c.getenv("VAXIS_FORCE_UNICODE")) |_| - self.caps.unicode = .unicode; - - // enable detected features -@@ -1067,6 +1067,7 @@ pub fn transmitImage( - pub fn loadImage( - self: *Vaxis, - alloc: std.mem.Allocator, -+ io: std.Io, - tty: *IoWriter, - src: Image.Source, - ) !Image { -@@ -1074,7 +1075,7 @@ pub fn loadImage( - - var read_buffer: [1024 * 1024]u8 = undefined; // 1MB buffer - var img = switch (src) { -- .path => |path| try zigimg.Image.fromFilePath(alloc, path, &read_buffer), -+ .path => |path| try zigimg.Image.fromFilePath(alloc, io, path, &read_buffer), - .mem => |bytes| try zigimg.Image.fromMemory(alloc, bytes), - }; - defer img.deinit(alloc); -@@ -1481,7 +1482,7 @@ pub fn setTerminalWorkingDirectory(_: *Vaxis, tty: *IoWriter, path: []const u8) - return error.InvalidAbsolutePath; - const hostname = switch (builtin.os.tag) { - .windows => null, -- else => std.posix.getenv("HOSTNAME"), -+ else => std.c.getenv("HOSTNAME"), - } orelse "localhost"; - - const uri: std.Uri = .{ -@@ -1495,11 +1496,11 @@ pub fn setTerminalWorkingDirectory(_: *Vaxis, tty: *IoWriter, path: []const u8) - - test "render: no output when no changes" { - var vx = try Vaxis.init(std.testing.allocator, .{}); -- var deinit_writer = std.io.Writer.Allocating.init(std.testing.allocator); -+ var deinit_writer: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer deinit_writer.deinit(); - defer vx.deinit(std.testing.allocator, &deinit_writer.writer); - -- var render_writer = std.io.Writer.Allocating.init(std.testing.allocator); -+ var render_writer: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer render_writer.deinit(); - try vx.render(&render_writer.writer); - const output = try render_writer.toOwnedSlice(); -diff --git a/src/gwidth.zig b/src/gwidth.zig -index 194139f..ecffaf7 100644 ---- a/src/gwidth.zig -+++ b/src/gwidth.zig -@@ -55,10 +55,10 @@ pub fn gwidth(str: []const u8, method: Method) u16 { - var grapheme_start: usize = 0; - var prev_break: bool = true; - -- while (grapheme_iter.next()) |result| { -+ while (grapheme_iter.nextCodePoint()) |result| { - if (prev_break and !result.is_break) { - // Start of a new grapheme -- const cp_len: usize = std.unicode.utf8CodepointSequenceLength(result.cp) catch 1; -+ const cp_len: usize = std.unicode.utf8CodepointSequenceLength(result.code_point) catch 1; - grapheme_start = grapheme_iter.i - cp_len; - } - -diff --git a/src/queue.zig b/src/queue.zig -index 6d847ea..908cf70 100644 ---- a/src/queue.zig -+++ b/src/queue.zig -@@ -1,7 +1,7 @@ - const std = @import("std"); - const assert = std.debug.assert; - const atomic = std.atomic; --const Condition = std.Thread.Condition; -+const Condition = std.Io.Condition; - - /// Thread safe. Fixed size. Blocking push and pop. - pub fn Queue( -@@ -11,23 +11,25 @@ pub fn Queue( - return struct { - buf: [size]T = undefined, - -+ io: std.Io, -+ - read_index: usize = 0, - write_index: usize = 0, - -- mutex: std.Thread.Mutex = .{}, -+ mutex: std.Io.Mutex = .init, - // blocks when the buffer is full -- not_full: Condition = .{}, -+ not_full: Condition = .init, - // ...or empty -- not_empty: Condition = .{}, -+ not_empty: Condition = .init, - - const Self = @This(); - - /// Pop an item from the queue. Blocks until an item is available. - pub fn pop(self: *Self) T { -- self.mutex.lock(); -- defer self.mutex.unlock(); -+ self.mutex.lockUncancelable(self.io); -+ defer self.mutex.unlock(self.io); - while (self.isEmptyLH()) { -- self.not_empty.wait(&self.mutex); -+ self.not_empty.waitUncancelable(self.io, &self.mutex); - } - std.debug.assert(!self.isEmptyLH()); - return self.popAndSignalLH(); -@@ -36,10 +38,10 @@ pub fn Queue( - /// Push an item into the queue. Blocks until an item has been - /// put in the queue. - pub fn push(self: *Self, item: T) void { -- self.mutex.lock(); -- defer self.mutex.unlock(); -+ self.mutex.lockUncancelable(self.io); -+ defer self.mutex.unlock(self.io); - while (self.isFullLH()) { -- self.not_full.wait(&self.mutex); -+ self.not_full.waitUncancelable(self.io, &self.mutex); - } - std.debug.assert(!self.isFullLH()); - self.pushAndSignalLH(item); -@@ -49,8 +51,8 @@ pub fn Queue( - /// was successfully placed in the queue, false if the queue - /// was full. - pub fn tryPush(self: *Self, item: T) bool { -- self.mutex.lock(); -- defer self.mutex.unlock(); -+ self.mutex.lockUncancelable(self.io); -+ defer self.mutex.unlock(self.io); - if (self.isFullLH()) return false; - self.pushAndSignalLH(item); - return true; -@@ -59,28 +61,28 @@ pub fn Queue( - /// Pop an item from the queue. Returns null when no item is - /// available. - pub fn tryPop(self: *Self) ?T { -- self.mutex.lock(); -- defer self.mutex.unlock(); -+ self.mutex.lockUncancelable(self.io); -+ defer self.mutex.unlock(self.io); - if (self.isEmptyLH()) return null; - return self.popAndSignalLH(); - } - - /// Poll the queue. This call blocks until events are in the queue - pub fn poll(self: *Self) void { -- self.mutex.lock(); -- defer self.mutex.unlock(); -+ self.mutex.lockUncancelable(self.io); -+ defer self.mutex.unlock(self.io); - while (self.isEmptyLH()) { -- self.not_empty.wait(&self.mutex); -+ self.not_empty.waitUncancelable(self.io, &self.mutex); - } - std.debug.assert(!self.isEmptyLH()); - } - - pub fn lock(self: *Self) void { -- self.mutex.lock(); -+ self.mutex.lockUncancelable(self.io); - } - - pub fn unlock(self: *Self) void { -- self.mutex.unlock(); -+ self.mutex.unlock(self.io); - } - - /// Used to efficiently drain the queue while the lock is externally held -@@ -92,7 +94,7 @@ pub fn Queue( - const was_full = self.isFullLH(); - const item = self.popLH(); - if (was_full) { -- self.not_full.signal(); -+ self.not_full.signal(self.io); - } - return item; - } -@@ -108,15 +110,15 @@ pub fn Queue( - - /// Returns `true` if the queue is empty and `false` otherwise. - pub fn isEmpty(self: *Self) bool { -- self.mutex.lock(); -- defer self.mutex.unlock(); -+ self.mutex.lockUncancelable(self.io); -+ defer self.mutex.unlock(self.io); - return self.isEmptyLH(); - } - - /// Returns `true` if the queue is full and `false` otherwise. - pub fn isFull(self: *Self) bool { -- self.mutex.lock(); -- defer self.mutex.unlock(); -+ self.mutex.lockUncancelable(self.io); -+ defer self.mutex.unlock(self.io); - return self.isFullLH(); - } - -@@ -143,7 +145,7 @@ pub fn Queue( - self.buf[self.mask(self.write_index)] = item; - self.write_index = self.mask2(self.write_index + 1); - if (was_empty) { -- self.not_empty.signal(); -+ self.not_empty.signal(self.io); - } - } - -@@ -151,7 +153,7 @@ pub fn Queue( - const was_full = self.isFullLH(); - const result = self.popLH(); - if (was_full) { -- self.not_full.signal(); -+ self.not_full.signal(self.io); - } - return result; - } -@@ -167,7 +169,7 @@ pub fn Queue( - const testing = std.testing; - const cfg = Thread.SpawnConfig{ .allocator = testing.allocator }; - test "Queue: simple push / pop" { -- var queue: Queue(u8, 16) = .{}; -+ var queue: Queue(u8, 16) = .{ .io = std.testing.io }; - queue.push(1); - queue.push(2); - const pop = queue.pop(); -@@ -182,7 +184,7 @@ fn testPushPop(q: *Queue(u8, 2)) !void { - } - - test "Fill, wait to push, pop once in another thread" { -- var queue: Queue(u8, 2) = .{}; -+ var queue: Queue(u8, 2) = .{ .io = std.testing.io }; - queue.push(1); - queue.push(2); - const t = try Thread.spawn(cfg, testPushPop, .{&queue}); -@@ -202,7 +204,7 @@ fn testPush(q: *Queue(u8, 2)) void { - } - - test "Try to pop, fill from another thread" { -- var queue: Queue(u8, 2) = .{}; -+ var queue: Queue(u8, 2) = .{ .io = std.testing.io }; - const thread = try Thread.spawn(cfg, testPush, .{&queue}); - for (0..5) |idx| { - try testing.expectEqual(@as(u8, @intCast(idx)), queue.pop()); -@@ -217,8 +219,8 @@ fn sleepyPop(q: *Queue(u8, 2), state: *atomic.Value(u8)) !void { - - // Then we spuriously wake it up, because that's a thing that can - // happen. -- q.not_full.signal(); -- q.not_empty.signal(); -+ q.not_full.signal(q.io); -+ q.not_empty.signal(q.io); - - // Then give the other thread a good chance of waking up. It's not - // clear that yield guarantees the other thread will be scheduled, -@@ -226,7 +228,7 @@ fn sleepyPop(q: *Queue(u8, 2), state: *atomic.Value(u8)) !void { - // still full and the push in the other thread is still blocked - // waiting for space. - try Thread.yield(); -- std.Thread.sleep(10 * std.time.ns_per_ms); -+ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; - // Finally, let that other thread go. - try std.testing.expectEqual(1, q.pop()); - -@@ -235,15 +237,15 @@ fn sleepyPop(q: *Queue(u8, 2), state: *atomic.Value(u8)) !void { - try Thread.yield(); - // But we want to ensure that there's a second push waiting, so - // here's another sleep. -- std.Thread.sleep(10 * std.time.ns_per_ms); -+ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; - - // Another spurious wake... -- q.not_full.signal(); -- q.not_empty.signal(); -+ q.not_full.signal(q.io); -+ q.not_empty.signal(q.io); - // And another chance for the other thread to see that it's - // spurious and go back to sleep. - try Thread.yield(); -- std.Thread.sleep(10 * std.time.ns_per_ms); -+ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; - - // Pop that thing and we're done. - try std.testing.expectEqual(2, q.pop()); -@@ -256,19 +258,20 @@ test "Fill, block, fill, block" { - // that too (after some time) then drain the queue. This test - // fails if the while loop in `push` is turned into an `if`. - -- var queue: Queue(u8, 2) = .{}; -+ var queue: Queue(u8, 2) = .{ .io = std.testing.io }; - var state = atomic.Value(u8).init(0); - const thread = try Thread.spawn(cfg, sleepyPop, .{ &queue, &state }); - queue.push(1); - queue.push(2); - state.store(1, .release); -- const now = std.time.milliTimestamp(); -+ const start = std.Io.Clock.Timestamp.now(std.testing.io, .awake); - queue.push(3); // This one should block. -- const then = std.time.milliTimestamp(); -+ const end = std.Io.Clock.Timestamp.now(std.testing.io, .awake); - - // Just to make sure the sleeps are yielding to this thread, make - // sure it took at least 5ms to do the push. -- try std.testing.expect(then - now > 5); -+ const elapsed = start.durationTo(end); -+ try std.testing.expect(elapsed.raw.toMilliseconds() > 5); - - state.store(2, .release); - // This should block again, waiting for the other thread. -@@ -283,14 +286,14 @@ test "Fill, block, fill, block" { - fn sleepyPush(q: *Queue(u8, 1), state: *atomic.Value(u8)) !void { - // Try to ensure the other thread has already started trying to pop. - try Thread.yield(); -- std.Thread.sleep(10 * std.time.ns_per_ms); -+ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; - - // Spurious wake -- q.not_full.signal(); -- q.not_empty.signal(); -+ q.not_full.signal(q.io); -+ q.not_empty.signal(q.io); - - try Thread.yield(); -- std.Thread.sleep(10 * std.time.ns_per_ms); -+ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; - - // Stick something in the queue so it can be popped. - q.push(1); -@@ -299,11 +302,11 @@ fn sleepyPush(q: *Queue(u8, 1), state: *atomic.Value(u8)) !void { - try Thread.yield(); - // Give the other thread time to block again. - try Thread.yield(); -- std.Thread.sleep(10 * std.time.ns_per_ms); -+ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; - - // Spurious wake -- q.not_full.signal(); -- q.not_empty.signal(); -+ q.not_full.signal(q.io); -+ q.not_empty.signal(q.io); - - q.push(2); - } -@@ -313,7 +316,7 @@ test "Drain, block, drain, block" { - // test should fail if the `while` loop in `pop` is turned into an - // `if`. - -- var queue: Queue(u8, 1) = .{}; -+ var queue: Queue(u8, 1) = .{ .io = std.testing.io }; - var state = atomic.Value(u8).init(0); - const thread = try Thread.spawn(cfg, sleepyPush, .{ &queue, &state }); - try std.testing.expectEqual(1, queue.pop()); -@@ -328,11 +331,11 @@ fn readerThread(q: *Queue(u8, 1)) !void { - - test "2 readers" { - // 2 threads read, one thread writes -- var queue: Queue(u8, 1) = .{}; -+ var queue: Queue(u8, 1) = .{ .io = std.testing.io }; - const t1 = try Thread.spawn(cfg, readerThread, .{&queue}); - const t2 = try Thread.spawn(cfg, readerThread, .{&queue}); - try Thread.yield(); -- std.Thread.sleep(10 * std.time.ns_per_ms); -+ std.Io.sleep(std.testing.io, .fromMilliseconds(10), .awake) catch {}; - queue.push(1); - queue.push(1); - t1.join(); -@@ -344,7 +347,7 @@ fn writerThread(q: *Queue(u8, 1)) !void { - } - - test "2 writers" { -- var queue: Queue(u8, 1) = .{}; -+ var queue: Queue(u8, 1) = .{ .io = std.testing.io }; - const t1 = try Thread.spawn(cfg, writerThread, .{&queue}); - const t2 = try Thread.spawn(cfg, writerThread, .{&queue}); - -diff --git a/src/tty.zig b/src/tty.zig -index 4a3c142..c1da795 100644 ---- a/src/tty.zig -+++ b/src/tty.zig -@@ -33,8 +33,11 @@ pub const PosixTty = struct { - /// The file descriptor of the tty - fd: posix.fd_t, - -+ io: std.Io, -+ - /// File.Writer for efficient buffered writing -- tty_writer: std.fs.File.Writer, -+ tty_buffer: [1024]u8 = undefined, -+ tty_writer: ?std.Io.File.Writer = null, - - pub const SignalHandler = struct { - context: *anyopaque, -@@ -43,7 +46,8 @@ pub const PosixTty = struct { - - /// global signal handlers - var handlers: [8]SignalHandler = undefined; -- var handler_mutex: std.Thread.Mutex = .{}; -+ var handler_mutex: std.Io.Mutex = .init; -+ var handler_io: std.Io = undefined; - var handler_idx: usize = 0; - - var handler_installed: bool = false; -@@ -51,9 +55,9 @@ pub const PosixTty = struct { - /// initializes a Tty instance by opening /dev/tty and "making it raw". A - /// signal handler is installed for SIGWINCH. No callbacks are installed, be - /// sure to register a callback when initializing the event loop -- pub fn init(buffer: []u8) !PosixTty { -+ pub fn init(io: std.Io) !PosixTty { - // Open our tty -- const fd = try posix.open("/dev/tty", .{ .ACCMODE = .RDWR }, 0); -+ const fd = try posix.openat(posix.AT.FDCWD, "/dev/tty", .{ .ACCMODE = .RDWR }, 0); - - // Set the termios of the tty - const termios = try makeRaw(fd); -@@ -69,13 +73,8 @@ pub const PosixTty = struct { - posix.sigaction(posix.SIG.WINCH, &act, null); - handler_installed = true; - -- const file = std.fs.File{ .handle = fd }; -- -- const self: PosixTty = .{ -- .fd = fd, -- .termios = termios, -- .tty_writer = .initStreaming(file, buffer), -- }; -+ handler_io = io; -+ const self: PosixTty = .{ .fd = fd, .io = io, .termios = termios }; - - global_tty = self; - -@@ -88,7 +87,7 @@ pub const PosixTty = struct { - std.log.err("couldn't restore terminal: {}", .{err}); - }; - if (builtin.os.tag != .macos) // closing /dev/tty may block indefinitely on macos -- posix.close(self.fd); -+ (std.Io.File{ .handle = self.fd, .flags = .{ .nonblocking = false } }).close(handler_io); - } - - /// Resets the signal handler to it's default -@@ -107,7 +106,12 @@ pub const PosixTty = struct { - } - - pub fn writer(self: *PosixTty) *std.Io.Writer { -- return &self.tty_writer.interface; -+ if (self.tty_writer == null) { -+ const file: std.Io.File = .{ .handle = self.fd, .flags = .{ .nonblocking = false } }; -+ self.tty_writer = .initStreaming(file, self.io, &self.tty_buffer); -+ } -+ -+ return &self.tty_writer.?.interface; - } - - pub fn read(self: *const PosixTty, buf: []u8) !usize { -@@ -117,16 +121,16 @@ pub const PosixTty = struct { - /// Install a signal handler for winsize. A maximum of 8 handlers may be - /// installed - pub fn notifyWinsize(handler: SignalHandler) !void { -- handler_mutex.lock(); -- defer handler_mutex.unlock(); -+ handler_mutex.lockUncancelable(handler_io); -+ defer handler_mutex.unlock(handler_io); - if (handler_idx == handlers.len) return error.OutOfMemory; - handlers[handler_idx] = handler; - handler_idx += 1; - } - -- fn handleWinch(_: c_int) callconv(.c) void { -- handler_mutex.lock(); -- defer handler_mutex.unlock(); -+ fn handleWinch(_: posix.SIG) callconv(.c) void { -+ if (!handler_mutex.tryLock()) return; -+ defer handler_mutex.unlock(handler_io); - var i: usize = 0; - while (i < handler_idx) : (i += 1) { - const handler = handlers[i]; -@@ -189,6 +193,7 @@ pub const PosixTty = struct { - pub const WindowsTty = struct { - stdin: windows.HANDLE, - stdout: windows.HANDLE, -+ io: std.Io, - - initial_codepage: c_uint, - initial_input_mode: CONSOLE_MODE_INPUT, -@@ -198,7 +203,8 @@ pub const WindowsTty = struct { - buf: [4]u8 = undefined, - - /// File.Writer for efficient buffered writing -- tty_writer: std.fs.File.Writer, -+ tty_buffer: [1024]u8 = undefined, -+ tty_writer: ?std.Io.File.Writer = null, - - /// The last mouse button that was pressed. We store the previous state of button presses on each - /// mouse event so we can detect which button was released -@@ -221,7 +227,7 @@ pub const WindowsTty = struct { - .ENABLE_LVB_GRID_WORLDWIDE = 1, // enables reverse video and underline - }; - -- pub fn init(buffer: []u8) !Tty { -+ pub fn init(io: std.Io) !Tty { - const stdin: std.fs.File = .stdin(); - const stdout: std.fs.File = .stdout(); - -@@ -237,12 +243,12 @@ pub const WindowsTty = struct { - return windows.unexpectedError(windows.kernel32.GetLastError()); - - const self: Tty = .{ -+ .io = io, - .stdin = stdin.handle, - .stdout = stdout.handle, - .initial_codepage = initial_output_codepage, - .initial_input_mode = initial_input_mode, - .initial_output_mode = initial_output_mode, -- .tty_writer = .initStreaming(stdout, buffer), - }; - - // save a copy of this tty as the global_tty for panic handling -@@ -298,7 +304,12 @@ pub const WindowsTty = struct { - } - - pub fn writer(self: *Tty) *std.Io.Writer { -- return &self.tty_writer.interface; -+ if (self.tty_writer == null) { -+ const stdout: std.fs.File = .stdout(); -+ self.tty_writer = .initStreaming(stdout, &self.tty_buffer); -+ } -+ -+ return &self.tty_writer.?.interface; - } - - pub fn read(self: *const Tty, buf: []u8) !usize { -@@ -694,20 +705,20 @@ pub const WindowsTty = struct { - pub const TestTty = struct { - /// Used for API compat - fd: posix.fd_t, -+ io: std.Io, - pipe_read: posix.fd_t, - pipe_write: posix.fd_t, - tty_writer: *std.Io.Writer.Allocating, - - /// Initializes a TestTty. -- pub fn init(buffer: []u8) !TestTty { -- _ = buffer; -- -+ pub fn init(io: std.Io) !TestTty { - if (builtin.os.tag == .windows) return error.SkipZigTest; - const list = try std.testing.allocator.create(std.Io.Writer.Allocating); - list.* = .init(std.testing.allocator); -- const r, const w = try posix.pipe(); -+ const r, const w = try std.Io.Threaded.pipe2(.{}); - return .{ - .fd = r, -+ .io = io, - .pipe_read = r, - .pipe_write = w, - .tty_writer = list, -@@ -715,8 +726,9 @@ pub const TestTty = struct { - } - - pub fn deinit(self: TestTty) void { -- std.posix.close(self.pipe_read); -- std.posix.close(self.pipe_write); -+ const io = std.testing.io; -+ (std.Io.File{ .handle = self.pipe_read, .flags = .{ .nonblocking = false } }).close(io); -+ (std.Io.File{ .handle = self.pipe_write, .flags = .{ .nonblocking = false } }).close(io); - self.tty_writer.deinit(); - std.testing.allocator.destroy(self.tty_writer); - } -diff --git a/src/unicode.zig b/src/unicode.zig -index a5253ac..4ff2e4c 100644 ---- a/src/unicode.zig -+++ b/src/unicode.zig -@@ -26,10 +26,10 @@ pub const GraphemeIterator = struct { - } - - pub fn next(self: *GraphemeIterator) ?Grapheme { -- while (self.inner.next()) |res| { -+ while (self.inner.nextCodePoint()) |res| { - // When leaving a break and entering a non-break, set the start of a cluster - if (self.prev_break and !res.is_break) { -- const cp_len: usize = std.unicode.utf8CodepointSequenceLength(res.cp) catch 1; -+ const cp_len: usize = std.unicode.utf8CodepointSequenceLength(res.code_point) catch 1; - self.start = self.inner.i - cp_len; - } - -diff --git a/src/vxfw/App.zig b/src/vxfw/App.zig -index 6d394a9..fc3962f 100644 ---- a/src/vxfw/App.zig -+++ b/src/vxfw/App.zig -@@ -16,7 +16,7 @@ tty: vaxis.Tty, - vx: vaxis.Vaxis, - timers: std.ArrayList(vxfw.Tick), - wants_focus: ?vxfw.Widget, --buffer: [1024]u8, -+io: std.Io, - - /// Runtime options - pub const Options = struct { -@@ -27,10 +27,10 @@ pub const Options = struct { - /// Create an application. We require stable pointers to do the set up, so this will create an App - /// object on the heap. Call destroy when the app is complete to reset terminal state and release - /// resources --pub fn init(allocator: Allocator) !App { -- var app: App = .{ -+pub fn init(allocator: Allocator, io: std.Io) !App { -+ return .{ - .allocator = allocator, -- .tty = undefined, -+ .tty = try vaxis.Tty.init(io), - .vx = try vaxis.init(allocator, .{ - .system_clipboard_allocator = allocator, - .kitty_keyboard_flags = .{ -@@ -39,10 +39,8 @@ pub fn init(allocator: Allocator) !App { - }), - .timers = std.ArrayList(vxfw.Tick){}, - .wants_focus = null, -- .buffer = undefined, -+ .io = io, - }; -- app.tty = try vaxis.Tty.init(&app.buffer); -- return app; - } - - pub fn deinit(self: *App) void { -@@ -55,7 +53,8 @@ pub fn run(self: *App, widget: vxfw.Widget, opts: Options) anyerror!void { - const tty = &self.tty; - const vx = &self.vx; - -- var loop: EventLoop = .{ .tty = tty, .vaxis = vx }; -+ const io = self.io; -+ var loop: EventLoop = .{ .tty = tty, .vaxis = vx, .io = io, .queue = .{ .io = io } }; - try loop.start(); - defer loop.stop(); - -@@ -65,7 +64,7 @@ pub fn run(self: *App, widget: vxfw.Widget, opts: Options) anyerror!void { - loop.postEvent(.focus_in); - - try vx.enterAltScreen(tty.writer()); -- try vx.queryTerminal(tty.writer(), 1 * std.time.ns_per_s); -+ try vx.queryTerminal(tty.writer(), io, 1 * std.time.ns_per_s); - try vx.setBracketedPaste(tty.writer(), true); - try vx.subscribeToColorSchemeUpdates(tty.writer()); - -@@ -97,13 +96,14 @@ pub fn run(self: *App, widget: vxfw.Widget, opts: Options) anyerror!void { - defer focus_handler.deinit(self.allocator); - - // Timestamp of our next frame -- var next_frame_ms: u64 = @intCast(std.time.milliTimestamp()); -+ var next_frame_ms: u64 = nowMs(io); - - // Create our event context - var ctx: vxfw.EventContext = .{ - .alloc = self.allocator, - .phase = .capturing, - .cmds = vxfw.CommandList{}, -+ .io = io, - .consume_event = false, - .redraw = false, - .quit = false, -@@ -111,13 +111,13 @@ pub fn run(self: *App, widget: vxfw.Widget, opts: Options) anyerror!void { - defer ctx.cmds.deinit(self.allocator); - - while (true) { -- const now_ms: u64 = @intCast(std.time.milliTimestamp()); -+ const now_ms: u64 = nowMs(io); - if (now_ms >= next_frame_ms) { - // Deadline exceeded. Schedule the next frame - next_frame_ms = now_ms + tick_ms; - } else { - // Sleep until the deadline -- std.Thread.sleep((next_frame_ms - now_ms) * std.time.ns_per_ms); -+ try std.Io.sleep(io, .fromMilliseconds(@intCast(next_frame_ms - now_ms)), .real); - next_frame_ms += tick_ms; - } - -@@ -296,7 +296,7 @@ fn handleCommand(self: *App, cmds: *vxfw.CommandList) Allocator.Error!void { - } - - fn checkTimers(self: *App, ctx: *vxfw.EventContext) anyerror!void { -- const now_ms = std.time.milliTimestamp(); -+ const now_ms = nowMs(ctx.io); - - // timers are always sorted descending - while (self.timers.pop()) |tick| { -@@ -349,12 +349,16 @@ const MouseHandler = struct { - .surface = last_frame, - .z_index = 0, - }; -- const mouse_point: vxfw.Point = .{ -- .row = @intCast(mouse.row), -- .col = @intCast(mouse.col), -- }; -- if (sub.containsPoint(mouse_point)) { -- try last_frame.hitTest(app.allocator, &hits, mouse_point); -+ -+ if (mouse.row >= 0 and mouse.col >= 0) { -+ const mouse_point: vxfw.Point = .{ -+ .row = @intCast(mouse.row), -+ .col = @intCast(mouse.col), -+ }; -+ -+ if (sub.containsPoint(mouse_point)) { -+ try last_frame.hitTest(app.allocator, &hits, mouse_point); -+ } - } - - // We store the hit list from the last mouse event to determine mouse_enter and mouse_leave -@@ -408,12 +412,16 @@ const MouseHandler = struct { - .surface = last_frame, - .z_index = 0, - }; -- const mouse_point: vxfw.Point = .{ -- .row = @intCast(mouse.row), -- .col = @intCast(mouse.col), -- }; -- if (sub.containsPoint(mouse_point)) { -- try last_frame.hitTest(app.allocator, &hits, mouse_point); -+ -+ if (mouse.row >= 0 and mouse.col >= 0) { -+ const mouse_point: vxfw.Point = .{ -+ .row = @intCast(mouse.row), -+ .col = @intCast(mouse.col), -+ }; -+ -+ if (sub.containsPoint(mouse_point)) { -+ try last_frame.hitTest(app.allocator, &hits, mouse_point); -+ } - } - - // Handle mouse_enter and mouse_leave events -@@ -602,3 +610,8 @@ const FocusHandler = struct { - } - } - }; -+ -+fn nowMs(io: std.Io) u64 { -+ const now = std.Io.Clock.now(.real, io); -+ return @intCast(now.toMilliseconds()); -+} -diff --git a/src/vxfw/Button.zig b/src/vxfw/Button.zig -index d4b2a9c..075627e 100644 ---- a/src/vxfw/Button.zig -+++ b/src/vxfw/Button.zig -@@ -142,6 +142,7 @@ test Button { - var ctx: vxfw.EventContext = .{ - .alloc = std.testing.allocator, - .cmds = .empty, -+ .io = std.testing.io, - }; - defer ctx.cmds.deinit(ctx.alloc); - -diff --git a/src/vxfw/ListView.zig b/src/vxfw/ListView.zig -index 8ff9551..de93d5d 100644 ---- a/src/vxfw/ListView.zig -+++ b/src/vxfw/ListView.zig -@@ -564,6 +564,7 @@ test ListView { - var ctx: vxfw.EventContext = .{ - .alloc = std.testing.allocator, - .cmds = .empty, -+ .io = std.testing.io, - }; - defer ctx.cmds.deinit(ctx.alloc); - -@@ -731,6 +732,7 @@ test "ListView: uneven scroll" { - var ctx: vxfw.EventContext = .{ - .alloc = std.testing.allocator, - .cmds = .empty, -+ .io = std.testing.io, - }; - defer ctx.cmds.deinit(ctx.alloc); - -diff --git a/src/vxfw/ScrollView.zig b/src/vxfw/ScrollView.zig -index 36319a4..38859ea 100644 ---- a/src/vxfw/ScrollView.zig -+++ b/src/vxfw/ScrollView.zig -@@ -646,6 +646,7 @@ test ScrollView { - var ctx: vxfw.EventContext = .{ - .alloc = std.testing.allocator, - .cmds = .empty, -+ .io = std.testing.io, - }; - defer ctx.cmds.deinit(ctx.alloc); - -@@ -1044,6 +1045,7 @@ test "ScrollView: uneven scroll" { - var ctx: vxfw.EventContext = .{ - .alloc = std.testing.allocator, - .cmds = .empty, -+ .io = std.testing.io, - }; - defer ctx.cmds.deinit(ctx.alloc); - -diff --git a/src/vxfw/Spinner.zig b/src/vxfw/Spinner.zig -index 6619522..eb3d8ba 100644 ---- a/src/vxfw/Spinner.zig -+++ b/src/vxfw/Spinner.zig -@@ -20,11 +20,11 @@ frame: u4 = 0, - was_spinning: std.atomic.Value(bool) = .{ .raw = false }, - - /// Start, or add one, to the spinner counter. Thread safe. --pub fn start(self: *Spinner) ?vxfw.Command { -+pub fn start(self: *Spinner, io: std.Io) ?vxfw.Command { - self.was_spinning.store(true, .unordered); - const count = self.count.fetchAdd(1, .monotonic); - if (count == 0) { -- return vxfw.Tick.in(time_lapse, self.widget()); -+ return vxfw.Tick.in(io, time_lapse, self.widget()); - } - return null; - } -@@ -111,13 +111,13 @@ test Spinner { - // Start the spinner. This (maybe) returns a Tick command to schedule the next frame. If the - // spinner is already running, no command is returned. Calling start is thread safe. The - // returned command can be added to an EventContext to schedule the frame -- const maybe_cmd = spinner.start(); -+ const maybe_cmd = spinner.start(std.testing.io); - try std.testing.expect(maybe_cmd != null); - try std.testing.expect(maybe_cmd.? == .tick); - try std.testing.expectEqual(1, spinner.count.load(.unordered)); - - // If we call start again, we won't get another command but our counter will go up -- const maybe_cmd2 = spinner.start(); -+ const maybe_cmd2 = spinner.start(std.testing.io); - try std.testing.expect(maybe_cmd2 == null); - try std.testing.expectEqual(2, spinner.count.load(.unordered)); - -@@ -126,6 +126,7 @@ test Spinner { - var ctx: vxfw.EventContext = .{ - .alloc = arena.allocator(), - .cmds = .empty, -+ .io = std.testing.io, - }; - - // The event loop handles the tick event and calls us back with a .tick event. If we should keep -diff --git a/src/vxfw/SplitView.zig b/src/vxfw/SplitView.zig -index 9a355a0..591e5e9 100644 ---- a/src/vxfw/SplitView.zig -+++ b/src/vxfw/SplitView.zig -@@ -229,6 +229,7 @@ test SplitView { - var ctx: vxfw.EventContext = .{ - .alloc = arena.allocator(), - .cmds = .empty, -+ .io = std.testing.io, - }; - try split_widget.handleEvent(&ctx, .{ .mouse = mouse }); - // We should get a command to change the mouse shape -diff --git a/src/vxfw/Text.zig b/src/vxfw/Text.zig -index 1f76cba..a7b71bb 100644 ---- a/src/vxfw/Text.zig -+++ b/src/vxfw/Text.zig -@@ -223,7 +223,7 @@ pub const SoftwrapIterator = struct { - // Advance the hard iterator - if (self.index == self.line.len) { - self.line = self.hard_iter.next() orelse return null; -- self.line = std.mem.trimRight(u8, self.line, " \t"); -+ self.line = std.mem.trimEnd(u8, self.line, " \t"); - self.index = 0; - } - -@@ -237,7 +237,7 @@ pub const SoftwrapIterator = struct { - if (self.ctx.max.width) |max| { - if (cur_width + next_width > max) { - // Trim the word to see if it can fit on a line by itself -- const trimmed = std.mem.trimLeft(u8, word, " \t"); -+ const trimmed = std.mem.trimStart(u8, word, " \t"); - const trimmed_bytes = word.len - trimmed.len; - // The number of bytes we trimmed is equal to the reduction in length - const trimmed_width = next_width - trimmed_bytes; -diff --git a/src/vxfw/TextField.zig b/src/vxfw/TextField.zig -index 6ad108b..e5a4d5b 100644 ---- a/src/vxfw/TextField.zig -+++ b/src/vxfw/TextField.zig -@@ -346,7 +346,7 @@ pub fn deleteAfterCursor(self: *TextField) void { - /// Moves the cursor backward by words. If the character before the cursor is a space, the cursor is - /// positioned just after the next previous space - pub fn moveBackwardWordwise(self: *TextField) void { -- const trimmed = std.mem.trimRight(u8, self.buf.firstHalf(), " "); -+ const trimmed = std.mem.trimEnd(u8, self.buf.firstHalf(), " "); - const idx = if (std.mem.lastIndexOfScalar(u8, trimmed, ' ')) |last| - last + 1 - else -@@ -562,6 +562,7 @@ test TextField { - var ctx: vxfw.EventContext = .{ - .alloc = arena.allocator(), - .cmds = .empty, -+ .io = std.testing.io, - }; - - // Enough boiler plate...Create the text field -diff --git a/src/vxfw/vxfw.zig b/src/vxfw/vxfw.zig -index 1510e71..f768d56 100644 ---- a/src/vxfw/vxfw.zig -+++ b/src/vxfw/vxfw.zig -@@ -61,8 +61,8 @@ pub const Tick = struct { - return lhs.deadline_ms > rhs.deadline_ms; - } - -- pub fn in(ms: u32, widget: Widget) Command { -- const now = std.time.milliTimestamp(); -+ pub fn in(io: std.Io, ms: u32, widget: Widget) Command { -+ const now = std.Io.Clock.now(.real, io).toMilliseconds(); - return .{ .tick = .{ - .deadline_ms = now + ms, - .widget = widget, -@@ -101,6 +101,7 @@ pub const EventContext = struct { - phase: Phase = .at_target, - alloc: Allocator, - cmds: CommandList, -+ io: std.Io, - - /// The event was handled, do not pass it on - consume_event: bool = false, -@@ -120,7 +121,7 @@ pub const EventContext = struct { - } - - pub fn tick(self: *EventContext, ms: u32, widget: Widget) Allocator.Error!void { -- try self.addCmd(Tick.in(ms, widget)); -+ try self.addCmd(Tick.in(self.io, ms, widget)); - } - - pub fn consumeAndRedraw(self: *EventContext) void { -@@ -541,10 +542,10 @@ test "All widgets have a doctest and refAllDecls test" { - // it easy to fail CI early, or spot bad tests vs non-existant tests - const excludes = &[_][]const u8{ "vxfw.zig", "App.zig" }; - -- var cwd = try std.fs.cwd().openDir("./src/vxfw", .{ .iterate = true }); -+ var cwd = try std.Io.Dir.cwd().openDir(std.testing.io, "./src/vxfw", .{ .iterate = true }); - var iter = cwd.iterate(); -- defer cwd.close(); -- outer: while (try iter.next()) |file| { -+ defer cwd.close(std.testing.io); -+ outer: while (try iter.next(std.testing.io)) |file| { - if (file.kind != .file) continue; - for (excludes) |ex| if (std.mem.eql(u8, ex, file.name)) continue :outer; - -@@ -552,7 +553,7 @@ test "All widgets have a doctest and refAllDecls test" { - file.name[0..idx] - else - continue; -- const data = try cwd.readFileAllocOptions(std.testing.allocator, file.name, 10_000_000, null, .of(u8), 0x00); -+ const data = try cwd.readFileAllocOptions(std.testing.io, file.name, std.testing.allocator, std.Io.Limit.limited(10_000_000), .of(u8), 0x00); - defer std.testing.allocator.free(data); - var ast = try std.zig.Ast.parse(std.testing.allocator, data, .zig); - defer ast.deinit(std.testing.allocator); -diff --git a/src/widgets/TextInput.zig b/src/widgets/TextInput.zig -index a6236a4..6b8871b 100644 ---- a/src/widgets/TextInput.zig -+++ b/src/widgets/TextInput.zig -@@ -266,7 +266,7 @@ pub fn deleteAfterCursor(self: *TextInput) void { - /// Moves the cursor backward by words. If the character before the cursor is a space, the cursor is - /// positioned just after the next previous space - pub fn moveBackwardWordwise(self: *TextInput) void { -- const trimmed = std.mem.trimRight(u8, self.buf.firstHalf(), " "); -+ const trimmed = std.mem.trimEnd(u8, self.buf.firstHalf(), " "); - const idx = if (std.mem.lastIndexOfScalar(u8, trimmed, ' ')) |last| - last + 1 - else -diff --git a/src/widgets/TextView.zig b/src/widgets/TextView.zig -index 45f9e44..ee326cf 100644 ---- a/src/widgets/TextView.zig -+++ b/src/widgets/TextView.zig -@@ -79,52 +79,22 @@ pub const Buffer = struct { - var cols: usize = self.last_cols; - var iter = uucode.grapheme.Iterator(uucode.utf8.Iterator).init(.init(content.bytes)); - -- var grapheme_start: usize = 0; -- var prev_break: bool = true; -- -- while (iter.next()) |result| { -- if (prev_break and !result.is_break) { -- // Start of a new grapheme -- const cp_len: usize = std.unicode.utf8CodepointSequenceLength(result.cp) catch 1; -- grapheme_start = iter.i - cp_len; -- } -- -- if (result.is_break) { -- // End of a grapheme -- const grapheme_end = iter.i; -- const grapheme_len = grapheme_end - grapheme_start; -- -- try self.grapheme.append(allocator, .{ -- .len = @intCast(grapheme_len), -- .offset = @intCast(self.content.items.len + grapheme_start), -- }); -- -- const cluster = content.bytes[grapheme_start..grapheme_end]; -- if (std.mem.eql(u8, cluster, "\n")) { -- self.cols = @max(self.cols, cols); -- cols = 0; -- } else { -- // Calculate width using gwidth -- const w = vaxis.gwidth.gwidth(cluster, .unicode); -- cols +|= w; -- } -- -- grapheme_start = grapheme_end; -- } -- prev_break = result.is_break; -- } -- -- // Flush the last grapheme if we ended mid-cluster -- if (!prev_break and grapheme_start < content.bytes.len) { -- const grapheme_len = content.bytes.len - grapheme_start; -+ while (iter.nextGrapheme()) |grapheme| { -+ const grapheme_start = grapheme.start; -+ const grapheme_end = grapheme.end; -+ const grapheme_len = grapheme_end - grapheme_start; - - try self.grapheme.append(allocator, .{ - .len = @intCast(grapheme_len), - .offset = @intCast(self.content.items.len + grapheme_start), - }); - -- const cluster = content.bytes[grapheme_start..]; -- if (!std.mem.eql(u8, cluster, "\n")) { -+ const cluster = content.bytes[grapheme_start..grapheme_end]; -+ if (std.mem.eql(u8, cluster, "\n")) { -+ self.cols = @max(self.cols, cols); -+ cols = 0; -+ } else { -+ // Calculate width using gwidth - const w = vaxis.gwidth.gwidth(cluster, .unicode); - cols +|= w; - } -diff --git a/src/widgets/terminal/Command.zig b/src/widgets/terminal/Command.zig -index 0cd24f7..ed41553 100644 ---- a/src/widgets/terminal/Command.zig -+++ b/src/widgets/terminal/Command.zig -@@ -14,7 +14,7 @@ working_directory: ?[]const u8, - // Set after spawn() - pid: ?std.posix.pid_t = null, - --env_map: *const std.process.EnvMap, -+env_map: *const std.process.Environ.Map, - - pty: Pty, - -@@ -43,8 +43,8 @@ pub fn spawn(self: *Command, allocator: std.mem.Allocator) !void { - try posix.dup2(self.pty.tty.handle, std.posix.STDOUT_FILENO); - try posix.dup2(self.pty.tty.handle, std.posix.STDERR_FILENO); - -- self.pty.tty.close(); -- if (self.pty.pty.handle > 2) self.pty.pty.close(); -+ _ = std.os.linux.close(self.pty.tty.handle); -+ if (self.pty.pty.handle > 2) _ = std.os.linux.close(self.pty.pty.handle); - - if (self.working_directory) |wd| { - try std.posix.chdir(wd); -@@ -78,8 +78,8 @@ pub fn spawn(self: *Command, allocator: std.mem.Allocator) !void { - fn handleSigChild(_: c_int) callconv(.c) void { - const result = std.posix.waitpid(-1, 0); - -- Terminal.global_vt_mutex.lock(); -- defer Terminal.global_vt_mutex.unlock(); -+ if (!Terminal.global_vt_mutex.tryLock()) return; -+ defer Terminal.global_vt_mutex.unlock(Terminal.global_vt_io); - if (Terminal.global_vts) |vts| { - var vt = vts.get(result.pid) orelse return; - vt.event_queue.push(.exited); -@@ -97,7 +97,7 @@ pub fn kill(self: *Command) void { - /// hash map plus options. - fn createEnvironFromMap( - arena: std.mem.Allocator, -- map: *const std.process.EnvMap, -+ map: *const std.process.Environ.Map, - ) ![:null]?[*:0]u8 { - const envp_count: usize = map.count(); - -diff --git a/src/widgets/terminal/Pty.zig b/src/widgets/terminal/Pty.zig -index d9f4138..4c67328 100644 ---- a/src/widgets/terminal/Pty.zig -+++ b/src/widgets/terminal/Pty.zig -@@ -7,8 +7,8 @@ const Winsize = @import("../../main.zig").Winsize; - - const posix = std.posix; - --pty: std.fs.File, --tty: std.fs.File, -+pty: std.Io.File, -+tty: std.Io.File, - - /// opens a new tty/pty pair - pub fn init() !Pty { -@@ -19,9 +19,9 @@ pub fn init() !Pty { - } - - /// closes the tty and pty --pub fn deinit(self: Pty) void { -- self.pty.close(); -- self.tty.close(); -+pub fn deinit(self: Pty, io: std.Io) void { -+ self.pty.close(io); -+ self.tty.close(io); - } - - /// sets the size of the pty -@@ -37,7 +37,7 @@ pub fn setSize(self: Pty, ws: Winsize) !void { - } - - fn openPtyLinux() !Pty { -- const p = try posix.open("/dev/ptmx", .{ .ACCMODE = .RDWR, .NOCTTY = true }, 0); -+ const p = try posix.openat(posix.AT.FDCWD, "/dev/ptmx", .{ .ACCMODE = .RDWR, .NOCTTY = true }, 0); - errdefer posix.close(p); - - // unlockpt -@@ -50,7 +50,7 @@ fn openPtyLinux() !Pty { - const sname = try std.fmt.bufPrint(&buf, "/dev/pts/{d}", .{n}); - std.log.debug("pts: {s}", .{sname}); - -- const t = try posix.open(sname, .{ .ACCMODE = .RDWR, .NOCTTY = true }, 0); -+ const t = try posix.openat(posix.AT.FDCWD, sname, .{ .ACCMODE = .RDWR, .NOCTTY = true }, 0); - - return .{ - .pty = .{ .handle = p }, -diff --git a/src/widgets/terminal/Terminal.zig b/src/widgets/terminal/Terminal.zig -index 36d61d7..786ca85 100644 ---- a/src/widgets/terminal/Terminal.zig -+++ b/src/widgets/terminal/Terminal.zig -@@ -43,21 +43,23 @@ pub const InputEvent = union(enum) { - key_press: vaxis.Key, - }; - --pub var global_vt_mutex: std.Thread.Mutex = .{}; -+pub var global_vt_mutex: std.Io.Mutex = .init; -+var global_vt_io: std.Io = undefined; - pub var global_vts: ?std.AutoHashMap(i32, *Terminal) = null; - pub var global_sigchild_installed: bool = false; - - allocator: std.mem.Allocator, -+io: std.Io, - scrollback_size: u16, - - pty: Pty, --pty_writer: std.fs.File.Writer, -+pty_writer: std.Io.File.Writer, - cmd: Command, - thread: ?std.Thread = null, - - /// the screen we draw from - front_screen: Screen, --front_mutex: std.Thread.Mutex = .{}, -+front_mutex: std.Io.Mutex = .init, - - /// the back screens - back_screen: *Screen = undefined, -@@ -65,7 +67,7 @@ back_screen_pri: Screen, - back_screen_alt: Screen, - // only applies to primary screen - scroll_offset: usize = 0, --back_mutex: std.Thread.Mutex = .{}, -+back_mutex: std.Io.Mutex = .init, - // dirty is protected by back_mutex. Only access this field when you hold that mutex - dirty: bool = false, - -@@ -79,14 +81,15 @@ working_directory: std.ArrayList(u8) = .empty, - - last_printed: []const u8 = "", - --event_queue: Queue = .{}, -+event_queue: Queue, - - /// initialize a Terminal. This sets the size of the underlying pty and allocates the sizes of the - /// screen - pub fn init( - allocator: std.mem.Allocator, -+ io: std.Io, - argv: []const []const u8, -- env: *const std.process.EnvMap, -+ env: *const std.process.Environ.Map, - opts: Options, - write_buf: []u8, - ) !Terminal { -@@ -107,8 +110,10 @@ pub fn init( - while (col < opts.winsize.cols) : (col += 8) { - try tabs.append(allocator, col); - } -+ global_vt_io = io; - return .{ - .allocator = allocator, -+ .io = io, - .pty = pty, - .pty_writer = pty.pty.writerStreaming(write_buf), - .cmd = cmd, -@@ -117,6 +122,7 @@ pub fn init( - .back_screen_pri = try Screen.init(allocator, opts.winsize.cols, opts.winsize.rows + opts.scrollback_size), - .back_screen_alt = try Screen.init(allocator, opts.winsize.cols, opts.winsize.rows), - .tab_stops = tabs, -+ .event_queue = .{ .io = io }, - }; - } - -@@ -125,8 +131,8 @@ pub fn deinit(self: *Terminal) void { - self.should_quit = true; - - pid: { -- global_vt_mutex.lock(); -- defer global_vt_mutex.unlock(); -+ global_vt_mutex.lockUncancelable(global_vt_io); -+ defer global_vt_mutex.unlock(global_vt_io); - var vts = global_vts orelse break :pid; - if (self.cmd.pid) |pid| - _ = vts.remove(pid); -@@ -143,7 +149,7 @@ pub fn deinit(self: *Terminal) void { - thread.join(); - self.thread = null; - } -- self.pty.deinit(); -+ self.pty.deinit(self.io); - self.front_screen.deinit(self.allocator); - self.back_screen_pri.deinit(self.allocator); - self.back_screen_alt.deinit(self.allocator); -@@ -162,16 +168,15 @@ pub fn spawn(self: *Terminal) !void { - if (self.cmd.working_directory) |pwd| { - try self.working_directory.appendSlice(self.allocator, pwd); - } else { -- const pwd = std.fs.cwd(); - var buffer: [std.fs.max_path_bytes]u8 = undefined; -- const out_path = try std.os.getFdPath(pwd.fd, &buffer); -+ const out_path = try std.posix.readlinkZ("/proc/self/cwd", &buffer); - try self.working_directory.appendSlice(self.allocator, out_path); - } - - { - // add to our global list -- global_vt_mutex.lock(); -- defer global_vt_mutex.unlock(); -+ global_vt_mutex.lockUncancelable(global_vt_io); -+ defer global_vt_mutex.unlock(global_vt_io); - if (global_vts == null) - global_vts = std.AutoHashMap(i32, *Terminal).init(self.allocator); - if (self.cmd.pid) |pid| -@@ -190,8 +195,8 @@ pub fn resize(self: *Terminal, ws: Winsize) !void { - ws.rows == self.front_screen.height) - return; - -- self.back_mutex.lock(); -- defer self.back_mutex.unlock(); -+ self.back_mutex.lockUncancelable(self.io); -+ defer self.back_mutex.unlock(self.io); - - self.front_screen.deinit(self.allocator); - self.front_screen = try Screen.init(self.allocator, ws.cols, ws.rows); -@@ -206,7 +211,7 @@ pub fn resize(self: *Terminal, ws: Winsize) !void { - - pub fn draw(self: *Terminal, allocator: std.mem.Allocator, win: vaxis.Window) !void { - if (self.back_mutex.tryLock()) { -- defer self.back_mutex.unlock(); -+ defer self.back_mutex.unlock(self.io); - // We keep this as a separate condition so we don't deadlock by obtaining the lock but not - // having sync - if (!self.mode.sync) { -@@ -249,7 +254,7 @@ pub fn get_pty_writer(self: *Terminal) *std.Io.Writer { - return &self.pty_writer.interface; - } - --fn reader(self: *const Terminal, buf: []u8) std.fs.File.Reader { -+fn reader(self: *const Terminal, buf: []u8) std.Io.File.Reader { - return self.pty.pty.readerStreaming(buf); - } - -@@ -265,8 +270,8 @@ fn run(self: *Terminal) !void { - - while (!self.should_quit) { - const event = try parser.parseReader(&reader_.interface); -- self.back_mutex.lock(); -- defer self.back_mutex.unlock(); -+ self.back_mutex.lockUncancelable(self.io); -+ defer self.back_mutex.unlock(self.io); - - if (!self.dirty and self.event_queue.tryPush(.redraw)) - self.dirty = true; diff --git a/third_party/libvaxis/fixes.patch b/third_party/libvaxis/fixes.patch index aca9c136..effbb2aa 100644 --- a/third_party/libvaxis/fixes.patch +++ b/third_party/libvaxis/fixes.patch @@ -1,86 +1,108 @@ -diff --git a/examples/table.zig b/examples/table.zig -index c489828..16aefdc 100644 ---- a/examples/table.zig -+++ b/examples/table.zig -@@ -20,8 +20,8 @@ pub fn main(init: std.process.Init) !void { - // Users set up below the main function - const users_buf = try alloc.dupe(User, users[0..]); - -- var buffer: [1024]u8 = undefined; -- var tty = try vaxis.Tty.init(init.io, &buffer); -+ // var buffer: [1024]u8 = undefined; -+ var tty = try vaxis.Tty.init(init.io); - defer tty.deinit(); - const tty_writer = tty.writer(); - var vx = try vaxis.init(alloc, .{ +diff --git a/src/Loop.zig b/src/Loop.zig +--- a/src/Loop.zig ++++ b/src/Loop.zig +@@ -352,7 +352,7 @@ + } + }, + .cap_da1 => { +- std.Io.futexWake(vx.io, std.atomic.Value(u32), &vx.query_futex, 10); ++ std.Io.futexWake(vx.io, u32, &vx.query_futex.raw, 10); + vx.queries_done.store(true, .unordered); + }, + .mouse => |mouse| { +@@ -484,7 +484,7 @@ + vx.caps.program_status = true; + }, + .cap_da1 => { +- std.Io.futexWake(vx.io, std.atomic.Value(u32), &vx.query_futex, 10); ++ std.Io.futexWake(vx.io, u32, &vx.query_futex.raw, 10); + vx.queries_done.store(true, .unordered); + }, + .winsize => |winsize| { diff --git a/src/Vaxis.zig b/src/Vaxis.zig -index 2e24e9b..2b2ee49 100644 --- a/src/Vaxis.zig +++ b/src/Vaxis.zig -@@ -127,14 +127,13 @@ pub fn deinit(self: *Vaxis, alloc: ?std.mem.Allocator, tty: *IoWriter) void { +@@ -139,14 +139,13 @@ } } -/// resets enabled features, sends cursor to home and clears below cursor --pub fn resetState(self: *Vaxis, tty: *IoWriter) !void { +-pub fn resetState(self: *Vaxis, tty: *std.Io.Writer) !void { - // always show the cursor on state reset +/// Resets enabled terminal features without clearing screen content. +/// Use this for non-interactive / single-shot rendering where the +/// output should remain visible after exit. -+pub fn resetModes(self: *Vaxis, tty: *IoWriter) !void { ++pub fn resetModes(self: *Vaxis, tty: *std.Io.Writer) !void { tty.writeAll(ctlseqs.show_cursor) catch {}; tty.writeAll(ctlseqs.sgr_reset) catch {}; if (self.screen.cursor_shape != .default) { - // In many terminals, `.default` will set to the configured cursor shape. Others, it will - // change to a blinking block. - tty.print(ctlseqs.cursor_shape, .{@intFromEnum(Cell.CursorShape.default)}) catch {}; + tty.print(ctlseqs.cursor_shape, .{@backingInt(Cell.CursorShape.default)}) catch {}; } if (self.state.kitty_keyboard) { -@@ -147,18 +146,6 @@ pub fn resetState(self: *Vaxis, tty: *IoWriter) !void { +@@ -159,6 +158,32 @@ if (self.state.bracketed_paste) { try self.setBracketedPaste(tty, false); } -- if (self.state.alt_screen) { -- try tty.writeAll(ctlseqs.home); -- try tty.writeAll(ctlseqs.erase_below_cursor); -- try self.exitAltScreen(tty); -- } else { -- try tty.writeByte('\r'); -- var i: u16 = 0; -- while (i < self.state.cursor.row) : (i += 1) { -- try tty.writeAll(ctlseqs.ri); -- } -- try tty.writeAll(ctlseqs.erase_below_cursor); -- } - if (self.state.color_scheme_updates) { - try tty.writeAll(ctlseqs.color_scheme_reset); - self.state.color_scheme_updates = false; -@@ -179,7 +166,24 @@ pub fn resetState(self: *Vaxis, tty: *IoWriter) !void { - try tty.writeAll(ctlseqs.osc12_reset); - self.state.changed_cursor_color = false; - } ++ if (self.state.color_scheme_updates) { ++ try tty.writeAll(ctlseqs.color_scheme_reset); ++ self.state.color_scheme_updates = false; ++ } ++ if (self.state.in_band_resize) { ++ try tty.writeAll(ctlseqs.in_band_resize_reset); ++ self.state.in_band_resize = false; ++ } ++ if (self.state.changed_default_fg) { ++ try tty.writeAll(ctlseqs.osc10_reset); ++ self.state.changed_default_fg = false; ++ } ++ if (self.state.changed_default_bg) { ++ try tty.writeAll(ctlseqs.osc11_reset); ++ self.state.changed_default_bg = false; ++ } ++ if (self.state.changed_cursor_color) { ++ try tty.writeAll(ctlseqs.osc12_reset); ++ self.state.changed_cursor_color = false; ++ } + try tty.flush(); +} - ++ +/// resets enabled features, sends cursor to home and clears below cursor -+pub fn resetState(self: *Vaxis, tty: *IoWriter) !void { ++pub fn resetState(self: *Vaxis, tty: *std.Io.Writer) !void { + self.resetModes(tty) catch {}; -+ if (self.state.alt_screen) { -+ try tty.writeAll(ctlseqs.home); -+ try tty.writeAll(ctlseqs.erase_below_cursor); -+ try self.exitAltScreen(tty); -+ } else { -+ try tty.writeByte('\r'); -+ var i: u16 = 0; -+ while (i < self.state.cursor.row) : (i += 1) { -+ try tty.writeAll(ctlseqs.ri); -+ } -+ try tty.writeAll(ctlseqs.erase_below_cursor); -+ } + if (self.state.alt_screen) { + try tty.writeAll(ctlseqs.home); + try tty.writeAll(ctlseqs.erase_below_cursor); +@@ -171,27 +196,6 @@ + } + try tty.writeAll(ctlseqs.erase_below_cursor); + } +- if (self.state.color_scheme_updates) { +- try tty.writeAll(ctlseqs.color_scheme_reset); +- self.state.color_scheme_updates = false; +- } +- if (self.state.in_band_resize) { +- try tty.writeAll(ctlseqs.in_band_resize_reset); +- self.state.in_band_resize = false; +- } +- if (self.state.changed_default_fg) { +- try tty.writeAll(ctlseqs.osc10_reset); +- self.state.changed_default_fg = false; +- } +- if (self.state.changed_default_bg) { +- try tty.writeAll(ctlseqs.osc11_reset); +- self.state.changed_default_bg = false; +- } +- if (self.state.changed_cursor_color) { +- try tty.writeAll(ctlseqs.osc12_reset); +- self.state.changed_cursor_color = false; +- } +- try tty.flush(); } -@@ -943,19 +947,19 @@ pub fn transmitLocalImagePath( +@@ -1101,19 +1105,19 @@ switch (format) { .rgb => { try tty.print( @@ -103,7 +125,7 @@ index 2e24e9b..2b2ee49 100644 .{ id, medium_char, encoded }, ); }, -@@ -991,7 +995,7 @@ pub fn transmitPreEncodedImage( +@@ -1149,7 +1153,7 @@ if (bytes.len < 4096) { try tty.print( @@ -112,7 +134,7 @@ index 2e24e9b..2b2ee49 100644 .{ fmt, width, -@@ -1004,14 +1008,14 @@ pub fn transmitPreEncodedImage( +@@ -1162,14 +1166,14 @@ var n: usize = 4096; try tty.print( @@ -129,11 +151,22 @@ index 2e24e9b..2b2ee49 100644 .{ m, bytes[n..end], +diff --git a/src/c_api.zig b/src/c_api.zig +--- a/src/c_api.zig ++++ b/src/c_api.zig +@@ -965,7 +965,7 @@ + .cap_multi_cursor => vx.caps.multi_cursor = true, + .cap_program_status => vx.caps.program_status = true, + .cap_da1 => { +- std.Io.futexWake(vx.io, std.atomic.Value(u32), &vx.query_futex, 10); ++ std.Io.futexWake(vx.io, u32, &vx.query_futex.raw, 10); + vx.queries_done.store(true, .unordered); + }, + .winsize => { diff --git a/src/ctlseqs.zig b/src/ctlseqs.zig -index ab415c6..4baa496 100644 --- a/src/ctlseqs.zig +++ b/src/ctlseqs.zig -@@ -138,8 +138,8 @@ pub const osc52_clipboard_copy = "\x1b]52;c;{s}\x1b\\"; +@@ -140,8 +140,8 @@ pub const osc52_clipboard_request = "\x1b]52;c;?\x1b\\"; // Kitty graphics @@ -144,118 +177,10 @@ index ab415c6..4baa496 100644 pub const kitty_graphics_closing = ",C=1\x1b\\"; // Color control sequences -diff --git a/src/tty.zig b/src/tty.zig -index c1da795..7f6a303 100644 ---- a/src/tty.zig -+++ b/src/tty.zig -@@ -52,6 +52,10 @@ pub const PosixTty = struct { - - var handler_installed: bool = false; - -+ /// Atomic flag set by the SIGWINCH signal handler. Checked and cleared -+ /// by pollWinch() from a non-signal context to avoid deadlocks. -+ var winch_pending: std.atomic.Value(bool) = .{ .raw = false }; -+ - /// initializes a Tty instance by opening /dev/tty and "making it raw". A - /// signal handler is installed for SIGWINCH. No callbacks are installed, be - /// sure to register a callback when initializing the event loop -@@ -129,7 +133,20 @@ pub const PosixTty = struct { - } - - fn handleWinch(_: posix.SIG) callconv(.c) void { -- if (!handler_mutex.tryLock()) return; -+ // Only set an atomic flag. Calling mutex.lock() or queue.push() -+ // from a signal handler is not async-signal-safe and causes -+ // deadlocks when the signal is delivered to a thread that already -+ // holds one of those locks. Registered handlers are dispatched -+ // from pollWinch() instead. -+ winch_pending.store(true, .release); -+ } -+ -+ /// Check if a SIGWINCH was received and dispatch registered handlers. -+ /// Must be called from a non-signal context (e.g., the main event loop) -+ /// to avoid the async-signal-safety issues with the old approach. -+ pub fn pollWinch() void { -+ if (!winch_pending.swap(false, .acquire)) return; -+ handler_mutex.lockUncancelable(handler_io); - defer handler_mutex.unlock(handler_io); - var i: usize = 0; - while (i < handler_idx) : (i += 1) { -diff --git a/src/vxfw/App.zig b/src/vxfw/App.zig -index fc3962f..2661cf4 100644 ---- a/src/vxfw/App.zig -+++ b/src/vxfw/App.zig -@@ -1,4 +1,5 @@ - const std = @import("std"); -+const builtin = @import("builtin"); - const vaxis = @import("../main.zig"); - const vxfw = @import("vxfw.zig"); - -@@ -36,8 +37,8 @@ fn init(allocator: Allocator, io: std.Io) !App { - .kitty_keyboard_flags = .{ - .report_events = true, - }, - }), -- .timers = std.ArrayList(vxfw.Tick){}, -+ .timers = .empty, - .wants_focus = null, - .io = io, - }; -@@ -102,7 +103,7 @@ - var ctx: vxfw.EventContext = .{ - .alloc = self.allocator, - .phase = .capturing, -- .cmds = vxfw.CommandList{}, -+ .cmds = .empty, - .io = io, - .consume_event = false, - .redraw = false, -@@ -123,6 +124,13 @@ pub fn run(self: *App, widget: vxfw.Widget, opts: Options) anyerror!void { - - try self.checkTimers(&ctx); - -+ // Poll for pending SIGWINCH outside of queue lock. -+ // The signal handler only sets an atomic flag; we dispatch -+ // the registered callbacks here where mutex operations are safe. -+ if (builtin.os.tag != .windows and !builtin.is_test) { -+ if (!vx.state.in_band_resize) vaxis.Tty.pollWinch(); -+ } -+ - { - loop.queue.lock(); - defer loop.queue.unlock(); -@@ -342,7 +350,7 @@ - // For mouse events we store the last frame and use that for hit testing - const last_frame = surface; - -- var hits = std.ArrayList(vxfw.HitResult){}; -+ var hits: std.ArrayList(vxfw.HitResult) = .empty; - defer hits.deinit(app.allocator); - const sub: vxfw.SubSurface = .{ - .origin = .{ .row = 0, .col = 0 }, -@@ -405,7 +413,7 @@ - const last_frame = self.last_frame; - self.mouse = mouse; - -- var hits = std.ArrayList(vxfw.HitResult){}; -+ var hits: std.ArrayList(vxfw.HitResult) = .empty; - defer hits.deinit(app.allocator); - const sub: vxfw.SubSurface = .{ - .origin = .{ .row = 0, .col = 0 }, -@@ -524,7 +532,7 @@ - return .{ - .root = root, - .focused_widget = root, -- .path_to_focused = std.ArrayList(Widget){}, -+ .path_to_focused = .empty, - }; - } - diff --git a/src/vxfw/Padding.zig b/src/vxfw/Padding.zig -index e60162f..6e5f40b 100644 --- a/src/vxfw/Padding.zig +++ b/src/vxfw/Padding.zig -@@ -6,7 +6,7 @@ const Allocator = std.mem.Allocator; +@@ -6,7 +6,7 @@ const vxfw = @import("vxfw.zig"); const Padding = @This(); @@ -264,7 +189,7 @@ index e60162f..6e5f40b 100644 left: u16 = 0, right: u16 = 0, top: u16 = 0, -@@ -54,10 +54,6 @@ fn typeErasedDrawFn(ptr: *anyopaque, ctx: vxfw.DrawContext) Allocator.Error!vxfw +@@ -54,10 +54,6 @@ pub fn draw(self: *const Padding, ctx: vxfw.DrawContext) Allocator.Error!vxfw.Surface { const pad = self.padding; diff --git a/third_party/libvaxis/repo.bzl b/third_party/libvaxis/repo.bzl index 8305d1c0..e10982a4 100644 --- a/third_party/libvaxis/repo.bzl +++ b/third_party/libvaxis/repo.bzl @@ -4,8 +4,8 @@ def repo(): git_repository( name = "libvaxis", remote = "https://github.com/rockorager/libvaxis.git", - commit = "a3ae1d53feeeeaeb6218de3d38837559811acae4", + commit = "14b7684272234a3d66316c0d1bbdc876d4e8997c", build_file = "//third_party/libvaxis:libvaxis.bazel", - patches = ["//third_party/libvaxis:bump-zig.patch", "//third_party/libvaxis:fixes.patch"], + patches = ["//third_party/libvaxis:fixes.patch"], patch_args = ["-p1"], ) diff --git a/third_party/non_module_deps.bzl b/third_party/non_module_deps.bzl index 44632c4d..bd792d4c 100644 --- a/third_party/non_module_deps.bzl +++ b/third_party/non_module_deps.bzl @@ -17,6 +17,7 @@ load("//third_party/uucode:repo.bzl", uucode = "repo") load("//third_party/xla:repo.bzl", xla = "repo") load("//third_party/zigimg:repo.bzl", zigimg = "repo") load("//third_party/zio:repo.bzl", zio = "repo") +load("//third_party/zls:repo.bzl", zls = "repo") def _non_module_deps_impl(mctx): cloud_accelerator_diagnostics() @@ -38,6 +39,7 @@ def _non_module_deps_impl(mctx): flydsl() cute_ir() zio() + zls() return mctx.extension_metadata( reproducible = True, diff --git a/third_party/translate-c/repo.bzl b/third_party/translate-c/repo.bzl index 8315b2af..dfd8835d 100644 --- a/third_party/translate-c/repo.bzl +++ b/third_party/translate-c/repo.bzl @@ -3,8 +3,9 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") def repo(): git_repository( name = "translate-c", - # Use ZML Github mirror of translate-c because Codeber is too unstable + # Use the ZML GitHub mirror because Codeberg is unreliable. remote = "https://github.com/zml/translate-c", - commit = "46b5609b5ac4c0a896217d1d984f3ae50e4810b5", + # 2.0.0 + commit = "0da7a16c3235b935b82421646076e0657cda21f6", build_file = Label("//third_party/translate-c:translate-c.bazel"), ) diff --git a/third_party/translate-c/translate-c.bazel b/third_party/translate-c/translate-c.bazel index b2ff9c98..7de2a5eb 100644 --- a/third_party/translate-c/translate-c.bazel +++ b/third_party/translate-c/translate-c.bazel @@ -1,4 +1,12 @@ load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_configure_binary", "zig_library") +load("@zml//bazel:zig_options.bzl", "zig_option", "zig_options") + +zig_option(name = "debug_allocations", bool = False) + +zig_options( + name = "build_options", + settings = [":debug_allocations"], +) zig_library( name = "c_builtins", @@ -19,6 +27,7 @@ zig_binary( main = "src/main.zig", linkopts = ["-lc"], deps = [ + ":build_options", ":c_builtins", ":helpers", "@arocc//:aro", diff --git a/third_party/uucode/repo.bzl b/third_party/uucode/repo.bzl index 20a2f080..5f7b1cbf 100644 --- a/third_party/uucode/repo.bzl +++ b/third_party/uucode/repo.bzl @@ -21,7 +21,8 @@ _uucode_repo = repository_rule( def repo(): _uucode_repo( name = "uucode", - commit = "f1c687f09174f9f11eda14d3e3ce497d68ab09a6", + # main branch + commit = "1fb73433bba5d93366c57f23ff2e9d7939746500", build_config = "//third_party/uucode:build_config.zig", build_file = "//third_party/uucode:uucode.bazel", config_storage_patch = "//third_party/uucode:config-storage-module.patch", diff --git a/third_party/uucode/uucode.bazel b/third_party/uucode/uucode.bazel index f2b5634d..4a3710f2 100644 --- a/third_party/uucode/uucode.bazel +++ b/third_party/uucode/uucode.bazel @@ -8,6 +8,7 @@ zig_library( main = "src/config.zig", srcs = [ "src/components.zig", + "src/ucd_inputs.zig", "src/fields.zig", "src/multi_slice.zig", "src/quirks.zig", @@ -51,6 +52,7 @@ zig_library( main = "src/config.zig", srcs = [ "src/components.zig", + "src/ucd_inputs.zig", "src/fields.zig", "src/multi_slice.zig", "src/quirks.zig", diff --git a/third_party/zigimg/bump-zig.patch b/third_party/zigimg/bump-zig.patch deleted file mode 100644 index b668dadc..00000000 --- a/third_party/zigimg/bump-zig.patch +++ /dev/null @@ -1,4491 +0,0 @@ -diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml -index 400aee8..72ad6f9 100644 ---- a/.github/workflows/main.yml -+++ b/.github/workflows/main.yml -@@ -25,7 +25,7 @@ jobs: - - name: Setup Zig - uses: mlugg/setup-zig@v2 - with: -- version: 0.15.2 -+ version: 0.16.0-dev.2261+d6b3dd25a - - name: AST Verify - run: zig fmt --check --ast-check src/ - working-directory: zigimg -diff --git a/build.zig.zon b/build.zig.zon -index 3ff4816..536d37b 100644 ---- a/build.zig.zon -+++ b/build.zig.zon -@@ -1,7 +1,7 @@ - .{ - .name = .zigimg, - .fingerprint = 0x6d2b443cdaa8f7f3, -- .minimum_zig_version = "0.15.2", -+ .minimum_zig_version = "0.16.0-dev.2261+d6b3dd25a", - .version = "0.1.0", - .paths = .{ - "build.zig", -diff --git a/src/Image.zig b/src/Image.zig -index d67cd61..6472129 100644 ---- a/src/Image.zig -+++ b/src/Image.zig -@@ -71,7 +71,7 @@ pub const WriteError = Error || - std.mem.Allocator.Error || - io.ReadStream.Error || - io.WriteStream.Error || -- std.fs.File.OpenError || -+ std.Io.File.OpenError || - error{ EndOfStream, InvalidData, UnfinishedBits }; - - pub const ConvertError = Error || -@@ -101,7 +101,7 @@ - }; - - pub const Animation = struct { -- frames: FrameList = .{}, -+ frames: FrameList = .empty, - loop_count: i32 = AnimationLoopInfinite, - - pub const FrameList = std.ArrayList(AnimationFrame); -@@ -150,16 +150,16 @@ pub fn deinit(self: *Image, allocator: std.mem.Allocator) void { - } - - /// Detect which image format is used by the file path --pub fn detectFormatFromFilePath(file_path: []const u8, read_buffer: []u8) !Format { -- var file = try std.fs.cwd().openFile(file_path, .{}); -- defer file.close(); -+pub fn detectFormatFromFilePath(std_io: std.Io, file_path: []const u8, read_buffer: []u8) !Format { -+ var file = try std.Io.Dir.openFile(std.Io.Dir.cwd(), std_io, file_path, .{}); -+ defer file.close(std_io); - -- return detectFormatFromFile(file, read_buffer); -+ return detectFormatFromFile(std_io, file, read_buffer); - } - - /// Detect which image format is used by the file --pub fn detectFormatFromFile(file: std.fs.File, read_buffer: []u8) !Format { -- var read_stream = io.ReadStream.initFile(file, read_buffer); -+pub fn detectFormatFromFile(std_io: std.Io, file: std.Io.File, read_buffer: []u8) !Format { -+ var read_stream = io.ReadStream.initFile(std_io, file, read_buffer); - return internalDetectFormat(&read_stream); - } - -@@ -170,16 +170,16 @@ pub fn detectFormatFromMemory(buffer: []const u8) !Format { - } - - /// Load an image from a file path --pub fn fromFilePath(allocator: std.mem.Allocator, file_path: []const u8, read_buffer: []u8) !Image { -- var file = try std.fs.cwd().openFile(file_path, .{}); -- defer file.close(); -+pub fn fromFilePath(allocator: std.mem.Allocator, std_io: std.Io, file_path: []const u8, read_buffer: []u8) !Image { -+ var file = try std.Io.Dir.openFile(std.Io.Dir.cwd(), std_io, file_path, .{}); -+ defer file.close(std_io); - -- return fromFile(allocator, file, read_buffer); -+ return fromFile(allocator, std_io, file, read_buffer); - } - --/// Load an image from a standard library std.fs.File --pub fn fromFile(allocator: std.mem.Allocator, file: std.fs.File, read_buffer: []u8) !Image { -- var read_stream = io.ReadStream.initFile(file, read_buffer); -+/// Load an image from a standard library std.Io.File -+pub fn fromFile(allocator: std.mem.Allocator, std_io: std.Io, file: std.Io.File, read_buffer: []u8) !Image { -+ var read_stream = io.ReadStream.initFile(std_io, file, read_buffer); - return internalRead(allocator, &read_stream); - } - -@@ -260,16 +260,16 @@ pub fn toManaged(self: Image, allocator: std.mem.Allocator) Managed { - } - - /// Write the image to an image format to the specified path --pub fn writeToFilePath(self: Image, allocator: std.mem.Allocator, file_path: []const u8, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { -- var file = try std.fs.cwd().createFile(file_path, .{}); -- defer file.close(); -+pub fn writeToFilePath(self: Image, allocator: std.mem.Allocator, std_io: std.Io, file_path: []const u8, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { -+ var file = try std.Io.Dir.createFile(std.Io.Dir.cwd(), std_io, file_path, .{}); -+ defer file.close(std_io); - -- try self.writeToFile(allocator, file, write_buffer, encoder_options); -+ try self.writeToFile(allocator, std_io, file, write_buffer, encoder_options); - } - --/// Write the image to an image format to the specified std.fs.File --pub fn writeToFile(self: Image, allocator: std.mem.Allocator, file: std.fs.File, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { -- var write_stream = io.WriteStream.initFile(file, write_buffer); -+/// Write the image to an image format to the specified std.Io.File -+pub fn writeToFile(self: Image, allocator: std.mem.Allocator, std_io: std.Io, file: std.Io.File, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { -+ var write_stream = io.WriteStream.initFile(std_io, file, write_buffer); - - try self.internalWrite(allocator, &write_stream, encoder_options); - } -diff --git a/src/Image/Managed.zig b/src/Image/Managed.zig -index 608b1d6..1517e7c 100644 ---- a/src/Image/Managed.zig -+++ b/src/Image/Managed.zig -@@ -20,14 +20,16 @@ width: usize = 0, - height: usize = 0, - pixels: color.PixelStorage = .{ .invalid = void{} }, - animation: Animation = .{}, -+io: std.Io = undefined, - allocator: std.mem.Allocator = undefined, // Allocator needs to be last in order to be able to ptrCast to Image - - const Managed = @This(); - - /// Init an empty image with no pixel data --pub fn init(allocator: std.mem.Allocator) Managed { -+pub fn init(allocator: std.mem.Allocator, io: std.Io) Managed { - return Managed{ - .allocator = allocator, -+ .io = io, - }; - } - -@@ -37,13 +39,13 @@ pub fn deinit(self: *Managed) void { - } - - /// Detect which image format is used by the file path --pub fn detectFormatFromFilePath(file_path: []const u8, read_buffer: []u8) !Format { -- return Image.detectFormatFromFilePath(file_path, read_buffer); -+pub fn detectFormatFromFilePath(io: std.Io, file_path: []const u8, read_buffer: []u8) !Format { -+ return Image.detectFormatFromFilePath(io, file_path, read_buffer); - } - - /// Detect which image format is used by the file --pub fn detectFormatFromFile(file: std.fs.File, read_buffer: []u8) !Format { -- return Image.detectFormatFromFile(file, read_buffer); -+pub fn detectFormatFromFile(io: std.Io, file: std.Io.File, read_buffer: []u8) !Format { -+ return Image.detectFormatFromFile(io, file, read_buffer); - } - - /// Detect which image format is used by the memory buffer -@@ -52,13 +54,13 @@ pub fn detectFormatFromMemory(buffer: []const u8) !Format { - } - - /// Load an image from a file path --pub fn fromFilePath(allocator: std.mem.Allocator, file_path: []const u8, read_buffer: []u8) !Managed { -- return (try Image.fromFilePath(allocator, file_path, read_buffer)).toManaged(allocator); -+pub fn fromFilePath(allocator: std.mem.Allocator, io: std.Io, file_path: []const u8, read_buffer: []u8) !Managed { -+ return (try Image.fromFilePath(allocator, io, file_path, read_buffer)).toManaged(allocator); - } - --/// Load an image from a standard library std.fs.File --pub fn fromFile(allocator: std.mem.Allocator, file: std.fs.File, read_buffer: []u8) !Managed { -- return (try Image.fromFile(allocator, file, read_buffer)).toManaged(allocator); -+/// Load an image from a standard library std.Io.File -+pub fn fromFile(allocator: std.mem.Allocator, io: std.Io, file: std.Io.File, read_buffer: []u8) !Managed { -+ return (try Image.fromFile(allocator, io, file, read_buffer)).toManaged(allocator); - } - - /// Load an image from a memory buffer -@@ -116,12 +118,12 @@ pub fn isAnimation(self: Managed) bool { - - /// Write the image to an image format to the specified path - pub fn writeToFilePath(self: Managed, file_path: []const u8, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { -- return Image.writeToFilePath(self.toUnmanaged(), self.allocator, file_path, write_buffer, encoder_options); -+ return Image.writeToFilePath(self.toUnmanaged(), self.allocator, self.io, file_path, write_buffer, encoder_options); - } - --/// Write the image to an image format to the specified std.fs.File --pub fn writeToFile(self: Managed, file: std.fs.File, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { -- return Image.writeToFile(self.toUnmanaged(), self.allocator, file, write_buffer, encoder_options); -+/// Write the image to an image format to the specified std.Io.File -+pub fn writeToFile(self: Managed, file: std.Io.File, write_buffer: []u8, encoder_options: EncoderOptions) WriteError!void { -+ return Image.writeToFile(self.toUnmanaged(), self.allocator, file, self.io, write_buffer, encoder_options); - } - - /// Write the image to an image format in a memory buffer. The memory buffer is not grown -diff --git a/src/color.zig b/src/color.zig -index 4591bce..8a1a6ac 100644 ---- a/src/color.zig -+++ b/src/color.zig -@@ -55,10 +55,7 @@ pub fn ScaleValue(comptime T: type) fn (anytype) T { - // In order to fit `value * out_max` we need an - // int of size `in_bits + out_bits` in the worst - // case scenario. -- const FitInt = @Type(.{ .int = .{ -- .bits = in_bits + out_bits, -- .signedness = .unsigned, -- } }); -+ const FitInt = @Int(.unsigned, in_bits + out_bits); - - return @truncate((@as(FitInt, value) * out_max + in_max / 2) / in_max); - } else return value; -@@ -395,7 +392,7 @@ fn ToMethods( - return packed struct(u0) { - const To = @This(); - -- fn getSelf(to: *const To) *align(1) const T { -+ fn getSelf(to: anytype) *align(1) const T { - // @fieldParentPtr is broken for packed structs. - // See: https://github.com/ziglang/zig/issues/20458 - if (@typeInfo(T).@"struct".layout == .@"packed") { -@@ -408,47 +405,47 @@ fn ToMethods( - } - - /// Assumes the target color type has `FromMethods` for it. -- pub fn color(to: *const To, ColorT: type) ColorT { -- return ColorT.from.color(to.getSelf().*); -+ pub fn color(to: anytype, ColorT: type) ColorT { -+ return ColorT.from.color(getSelf(to).*); - } - -- pub fn u32Rgba(to: *const To) u32 { -- const self = to.getSelf(); -+ pub fn u32Rgba(to: anytype) u32 { -+ const self = getSelf(to); - return @as(u32, toU8(self.r)) << 24 | - @as(u32, toU8(self.g)) << 16 | - @as(u32, toU8(self.b)) << 8 | - if (has_alpha) toU8(self.a) else 0xff; - } - -- pub fn u32Rgb(to: *const To) u32 { -- const self = to.getSelf(); -+ pub fn u32Rgb(to: anytype) u32 { -+ const self = getSelf(to); - return @as(u32, toU8(self.r)) << 16 | - @as(u32, toU8(self.g)) << 8 | - toU8(self.b); - } - -- pub fn u64Rgba(to: *const To) u64 { -- const self = to.getSelf(); -+ pub fn u64Rgba(to: anytype) u64 { -+ const self = getSelf(to); - return @as(u64, toU16(self.r)) << 48 | - @as(u64, toU16(self.g)) << 32 | - @as(u64, toU16(self.b)) << 16 | - if (has_alpha) toU16(self.a) else 0xffff; - } - -- pub fn u64Rgb(to: *const To) u64 { -- const self = to.getSelf(); -+ pub fn u64Rgb(to: anytype) u64 { -+ const self = getSelf(to); - return @as(u64, toU16(self.r)) << 32 | - @as(u64, toU16(self.g)) << 16 | - toU16(self.b); - } - - /// Only valid for color types where all channels are the same type. -- pub fn array(to: *const To) [4]RedT { -+ pub fn array(to: anytype) [4]RedT { - if (comptime multiple_channel_types) { - @compileError("Color.to.array may only be used when all channels in the color are the same type."); - } - -- const self = to.getSelf(); -+ const self = getSelf(to); - - return .{ - self.r, -@@ -461,8 +458,8 @@ fn ToMethods( - }; - } - -- pub fn float4(to: *const To) math.float4 { -- const self = to.getSelf(); -+ pub fn float4(to: anytype) math.float4 { -+ const self = getSelf(to); - - return .{ - toF32(self.r), -@@ -477,8 +474,8 @@ fn ToMethods( - - /// For int channels, premultiplication - /// is done with a round-trip to f32. -- pub fn premultipliedAlpha(to: *const To) T { -- const self = to.getSelf(); -+ pub fn premultipliedAlpha(to: anytype) T { -+ const self = getSelf(to); - var res = self.*; - if (!has_alpha) return res; - -diff --git a/src/formats/pam.zig b/src/formats/pam.zig -index f366d95..7ef9833 100644 ---- a/src/formats/pam.zig -+++ b/src/formats/pam.zig -@@ -70,7 +70,7 @@ - var maybe_maxval: ?u16 = null; - var maybe_tuple_type: ?TupleType = null; -- var comments = std.ArrayListUnmanaged([]const u8){}; -+ var comments: std.ArrayListUnmanaged([]const u8) = .empty; - defer { - for (comments.items) |comment| allocator.free(comment); - comments.deinit(allocator); -@@ -77,7 +77,7 @@ const Header = struct { - } - - { -- var line_buffer_stream = try std.io.Writer.Allocating.initCapacity(allocator, 32); -+ var line_buffer_stream = try std.Io.Writer.Allocating.initCapacity(allocator, 32); - defer line_buffer_stream.deinit(); - - while (true) { -diff --git a/src/formats/pcx.zig b/src/formats/pcx.zig -index b694762..52c2bb0 100644 ---- a/src/formats/pcx.zig -+++ b/src/formats/pcx.zig -@@ -218,7 +218,7 @@ test "PCX RLE Fast encoder" { - const uncompressed_data = [_]u8{ 1, 1, 1, 1, 1, 1, 1, 1, 1, 64, 64, 2, 2, 2, 2, 2, 215, 215, 215, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 200, 200, 200, 200, 210, 210 }; - const compressed_data = [_]u8{ 0xC9, 0x01, 0xC2, 0x40, 0xC5, 0x02, 0xC3, 0xD7, 0xCA, 0x03, 0xC4, 0xC8, 0xC2, 0xD2 }; - -- var writer_alloc = std.io.Writer.Allocating.init(std.testing.allocator); -+ var writer_alloc = std.Io.Writer.Allocating.init(std.testing.allocator); - defer writer_alloc.deinit(); - - try RLEFastEncoder.encode(uncompressed_data[0..], &writer_alloc.writer); -@@ -233,7 +233,7 @@ test "PCX RLE Fast encoder should encore more than 63 bytes similar" { - - const compressed_data = [_]u8{ 0xFF, 0x45, 0x45, 0x45, 0xC4, 0x1 }; - -- var writer_alloc = std.io.Writer.Allocating.init(std.testing.allocator); -+ var writer_alloc = std.Io.Writer.Allocating.init(std.testing.allocator); - defer writer_alloc.deinit(); - - try RLEFastEncoder.encode(uncompressed_data[0..], &writer_alloc.writer); -diff --git a/src/formats/tiff/types.zig b/src/formats/tiff/types.zig -index 8bcce86..067480a 100644 ---- a/src/formats/tiff/types.zig -+++ b/src/formats/tiff/types.zig -@@ -335,7 +335,7 @@ pub const IFD = struct { - // Skip unknown tags instead of panicking on invalid enum values. - // Many TIFF files contain vendor/proprietary tags (Make, Model, ICC profile, etc.) - // that we don't need to handle for image decoding. -- const maybe_tag_id = std.meta.intToEnum(TagId, tag_id_raw) catch continue; -+ const maybe_tag_id = std.enums.fromInt(TagId, tag_id_raw) orelse continue; - - try self.tags_map.put(maybe_tag_id, TagField{ - .data_type = std.mem.toNative(u16, tag.data_type, endianess), -diff --git a/src/formats/xbm.zig b/src/formats/xbm.zig -index d72e4ec..19ac9c0 100644 ---- a/src/formats/xbm.zig -+++ b/src/formats/xbm.zig -@@ -100,7 +100,7 @@ pub const XBM = struct { - - var line_full = try reader.takeDelimiterInclusive('\n'); - while (line_full.len > 0) { -- const line = std.mem.trimRight(u8, line_full, " \r\n"); -+ const line = std.mem.trimEnd(u8, line_full, " \r\n"); - if (isDefineLine(line)) { - if (parseDefineValue(line)) |value| { - if (width == null) { -diff --git a/src/io.zig b/src/io.zig -index 2c1363d..8d463a1 100644 ---- a/src/io.zig -+++ b/src/io.zig -@@ -5,11 +5,11 @@ pub const DEFAULT_BUFFER_SIZE = 4096; - - pub const ReadStream = union(enum) { - memory: std.Io.Reader, -- file: std.fs.File.Reader, -+ file: std.Io.File.Reader, - - pub const Error = SeekError || EndPosError || std.Io.Reader.Error || std.Io.Reader.StreamError; -- pub const SeekError = std.fs.File.Reader.SeekError; -- pub const EndPosError = std.fs.File.Reader.SizeError; -+ pub const SeekError = std.Io.File.Reader.SeekError; -+ pub const EndPosError = std.Io.File.Reader.SizeError; - - pub fn initMemory(buffer: []const u8) ReadStream { - return .{ -@@ -17,9 +17,9 @@ pub const ReadStream = union(enum) { - }; - } - -- pub fn initFile(file: std.fs.File, buffer: []u8) ReadStream { -+ pub fn initFile(io: std.Io, file: std.Io.File, buffer: []u8) ReadStream { - return .{ -- .file = file.reader(buffer), -+ .file = file.reader(io, buffer), - }; - } - -@@ -58,7 +58,7 @@ pub const ReadStream = union(enum) { - .memory => |*memory| { - const new_pos: i64 = @as(i64, @intCast(memory.seek)) + offset; - if (new_pos < 0 or new_pos >= memory.end) { -- return std.fs.File.SeekError.Unseekable; -+ return std.Io.File.SeekError.Unseekable; - } - - memory.seek = @intCast(new_pos); -@@ -96,10 +96,10 @@ pub const ReadStream = union(enum) { - - pub const WriteStream = union(enum) { - memory: std.Io.Writer, -- file: std.fs.File.Writer, -+ file: std.Io.File.Writer, - - pub const Error = SeekError || std.Io.Writer.Error; -- pub const SeekError = std.fs.File.SeekError; -+ pub const SeekError = std.Io.File.SeekError || std.Io.Writer.Error; - - pub fn initMemory(buffer: []u8) WriteStream { - return .{ -@@ -107,9 +107,9 @@ pub const WriteStream = union(enum) { - }; - } - -- pub fn initFile(file: std.fs.File, buffer: []u8) WriteStream { -+ pub fn initFile(io: std.Io, file: std.Io.File, buffer: []u8) WriteStream { - return .{ -- .file = file.writer(buffer), -+ .file = file.writer(io, buffer), - }; - } - -diff --git a/src/math.zig b/src/math.zig -index c4e5910..bad9687 100644 ---- a/src/math.zig -+++ b/src/math.zig -@@ -24,11 +24,11 @@ pub fn Matrix(comptime T: type) type { - var temp = self; - - // To find the determinant, use a Gaussian elimination to transform the temp matrix into the row echelon form -- for (0..(ComponentSize - 1)) |row| { -- for ((row + 1)..ComponentSize) |next_row| { -+ inline for (0..(ComponentSize - 1)) |row| { -+ inline for ((row + 1)..ComponentSize) |next_row| { - const factor = temp.matrix[next_row][row] / temp.matrix[row][row]; - -- for (0..ComponentSize) |column| { -+ inline for (0..ComponentSize) |column| { - temp.matrix[next_row][column] = temp.matrix[next_row][column] - factor * temp.matrix[row][column]; - } - } -@@ -37,7 +37,7 @@ pub fn Matrix(comptime T: type) type { - // Once the temp matrix is in row echelon form, multiply the diagonal to find the determinant - var result: f32 = temp.matrix[0][0]; - -- for (1..ComponentSize) |diagonal| { -+ inline for (1..ComponentSize) |diagonal| { - result *= temp.matrix[diagonal][diagonal]; - } - -@@ -48,9 +48,9 @@ pub fn Matrix(comptime T: type) type { - pub fn fromArray(array: [ComponentSize * ComponentSize]f32) Self { - var result: Self = undefined; - -- for (0..ComponentSize) |row| { -+ inline for (0..ComponentSize) |row| { - const stride = row * ComponentSize; -- for (0..ComponentSize) |column| { -+ inline for (0..ComponentSize) |column| { - result.matrix[row][column] = array[stride + column]; - } - } -@@ -72,20 +72,20 @@ pub fn Matrix(comptime T: type) type { - var temp = self; - var result: Self = identity(); - -- for (0..ComponentSize) |row| { -+ inline for (0..ComponentSize) |row| { - // Transform the pivot to 1 by multiplying the row by its inverse - const inverse_pivot = 1.0 / temp.matrix[row][row]; - -- for (0..ComponentSize) |column| { -+ inline for (0..ComponentSize) |column| { - temp.matrix[row][column] *= inverse_pivot; - result.matrix[row][column] *= inverse_pivot; - } - - // Then do Gaussian elimination from current row to bottom -- for ((row + 1)..ComponentSize) |next_row| { -+ inline for ((row + 1)..ComponentSize) |next_row| { - const factor = temp.matrix[next_row][row] / temp.matrix[row][row]; - -- for (0..ComponentSize) |column| { -+ inline for (0..ComponentSize) |column| { - temp.matrix[next_row][column] = temp.matrix[next_row][column] - factor * temp.matrix[row][column]; - result.matrix[next_row][column] = result.matrix[next_row][column] - factor * result.matrix[row][column]; - } -@@ -94,14 +94,12 @@ pub fn Matrix(comptime T: type) type { - - // Then do a Gaussian elimination bottom-up to transform the temp matrix into an identity matrix. - // The result will be the inverse of the matrix -- var row: usize = ComponentSize -% 1; -- while (row >= 1 and row < ComponentSize) : (row -%= 1) { -- var previous_row = row -% 1; -- -- while (previous_row >= 0 and previous_row < ComponentSize) : (previous_row -%= 1) { -+ inline for (1..ComponentSize) |i| { -+ const row = ComponentSize - i; -+ inline for (0..row) |previous_row| { - const factor = temp.matrix[previous_row][row] / temp.matrix[row][row]; - -- for (0..ComponentSize) |column| { -+ inline for (0..ComponentSize) |column| { - temp.matrix[previous_row][column] = temp.matrix[previous_row][column] - factor * temp.matrix[row][column]; - result.matrix[previous_row][column] = result.matrix[previous_row][column] - factor * result.matrix[row][column]; - } -@@ -126,8 +124,8 @@ pub fn Matrix(comptime T: type) type { - - const transposed_right = right.transpose(); - -- for (0..ComponentSize) |row| { -- for (0..ComponentSize) |column| { -+ inline for (0..ComponentSize) |row| { -+ inline for (0..ComponentSize) |column| { - result.matrix[row][column] = @reduce(.Add, self.matrix[row] * transposed_right.matrix[column]); - } - } -@@ -138,8 +136,8 @@ pub fn Matrix(comptime T: type) type { - pub fn transpose(self: Self) Self { - var result = std.mem.zeroes(Self); - -- for (0..ComponentSize) |row| { -- for (0..ComponentSize) |column| { -+ inline for (0..ComponentSize) |row| { -+ inline for (0..ComponentSize) |column| { - result.matrix[row][column] = self.matrix[column][row]; - } - } -diff --git a/src/simd.zig b/src/simd.zig -index 20c5dcb..cb115c9 100644 ---- a/src/simd.zig -+++ b/src/simd.zig -@@ -78,8 +78,10 @@ pub fn vpermd(v: @Vector(8, i32), mask: @Vector(8, i32)) @Vector(8, i32) { - [v] "x" (v), - ); - } else { -- var res: @Vector(8, i32) = undefined; -- inline for (0..8) |i| res[i] = v[@as(u32, @bitCast(mask[i]))]; -+ const v_arr: [8]i32 = v; -+ const mask_arr: [8]i32 = mask; -+ var res: [8]i32 = undefined; -+ for (0..8) |i| res[i] = v_arr[@intCast(mask_arr[i] & 7)]; - return res; - } - } -diff --git a/src/utils.zig b/src/utils.zig -index f55c12c..a12733e 100644 ---- a/src/utils.zig -+++ b/src/utils.zig -@@ -4,7 +4,7 @@ const std = @import("std"); - const native_endian = builtin.target.cpu.arch.endian(); - - pub const StructReadError = error{InvalidData} || std.Io.Reader.Error; --pub const StructWriteError = std.io.StreamSource.WriteError; -+pub const StructWriteError = std.Io.Writer.Error; - - pub fn FixedStorage(comptime T: type, comptime storage_size: usize) type { - return struct { -diff --git a/tests/formats/bmp_test.zig b/tests/formats/bmp_test.zig -index cc80b74..635dc04 100644 ---- a/tests/formats/bmp_test.zig -+++ b/tests/formats/bmp_test.zig -@@ -87,12 +87,12 @@ fn verifyBitmapRGBAV5(the_bitmap: bmp.BMP, pixels: zigimg.color.PixelStorage) !v - - test "Read simple version 4 24-bit RGB bitmap" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_bitmap = bmp.BMP{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_bitmap.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -145,10 +145,10 @@ test "Read simple version 4 24-bit RGB bitmap" { - - test "Read a valid version 5 RGBA bitmap from file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/windows_rgba_v5.bmp"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var the_bitmap = bmp.BMP{}; - -@@ -174,10 +174,10 @@ test "Read a valid version 5 RGBA bitmap from memory" { - - test "Should error when reading an invalid file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/notbmp.png"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var the_bitmap = bmp.BMP{}; - -@@ -215,17 +215,17 @@ test "Write a v4 bitmap when writing an image with bgr24 pixel format" { - pixels.bgr24[7] = zigimg.color.Bgr24.from.rgb(255, 255, 0); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .bmp = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .bmp = .{} }); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var read_bmp = bmp.BMP{}; - -@@ -277,17 +277,17 @@ test "Write a v5 bitmap when writing an image with bgra32 pixel format" { - pixels.bgra32[8] = zigimg.color.Bgra32.from.rgba(0, 0, 0, 0); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .bmp = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .bmp = .{} }); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var read_bmp = bmp.BMP{}; - -diff --git a/tests/formats/farbfeld_test.zig b/tests/formats/farbfeld_test.zig -index a48a7fb..00a73a0 100644 ---- a/tests/formats/farbfeld_test.zig -+++ b/tests/formats/farbfeld_test.zig -@@ -5,10 +5,10 @@ const farbfeld = zigimg.formats.farbfeld; - - test "Farbfeld: Read yellow file" { - const yellow_file = try helpers.testOpenFile(helpers.fixtures_path ++ "farbfeld/yellow-1x1-semitransparent.png.ff"); -- defer yellow_file.close(); -+ defer yellow_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var yellow_read_stream = zigimg.io.ReadStream.initFile(yellow_file, read_buffer[0..]); -+ var yellow_read_stream = zigimg.io.ReadStream.initFile(std.testing.io, yellow_file, read_buffer[0..]); - - var yellow_image = farbfeld.Farbfeld{}; - const yellow_pixels = try yellow_image.read(helpers.zigimg_test_allocator, &yellow_read_stream); -@@ -21,10 +21,10 @@ test "Farbfeld: Read yellow file" { - - test "Farbfeld: read dragon file" { - const dragon_file = try helpers.testOpenFile(helpers.fixtures_path ++ "farbfeld/dragon.ff"); -- defer dragon_file.close(); -+ defer dragon_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var dragon_read_stream = zigimg.io.ReadStream.initFile(dragon_file, read_buffer[0..]); -+ var dragon_read_stream = zigimg.io.ReadStream.initFile(std.testing.io, dragon_file, read_buffer[0..]); - - var dragon_image = farbfeld.Farbfeld{}; - const dragon_pixels = try dragon_image.read(helpers.zigimg_test_allocator, &dragon_read_stream); -@@ -37,10 +37,10 @@ test "Farbfeld: read dragon file" { - - test "Farbfeld: invalid file format" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "farbfeld/dragon.png"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var farbfeld_image = farbfeld.Farbfeld{}; - const image_error = farbfeld_image.read(helpers.zigimg_test_allocator, &read_stream); -@@ -50,10 +50,10 @@ test "Farbfeld: invalid file format" { - - test "Farbfeld: read writeImage output" { - const source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "farbfeld/yellow-1x1-semitransparent.png.ff"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var source_read_stream = zigimg.io.ReadStream.initFile(source_file, read_buffer[0..]); -+ var source_read_stream = zigimg.io.ReadStream.initFile(std.testing.io, source_file, read_buffer[0..]); - - var source_image: farbfeld.Farbfeld = .{}; - const source_pixels = try source_image.read(helpers.zigimg_test_allocator, &source_read_stream); -diff --git a/tests/formats/gif_test.zig b/tests/formats/gif_test.zig -index 496e14f..bdd84f6 100644 ---- a/tests/formats/gif_test.zig -+++ b/tests/formats/gif_test.zig -@@ -325,10 +325,10 @@ test "GIF writer roundtrip read existing GIF, write, read again" { - - // Read an existing GIF file using low-level reader - const gif_input_file = try helpers.testOpenFile(helpers.fixtures_path ++ "gif/rotating_earth.gif"); -- defer gif_input_file.close(); -+ defer gif_input_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(gif_input_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, gif_input_file, read_buffer[0..]); - - var original_gif = gif.GIF.init(allocator); - defer original_gif.deinit(); -@@ -489,10 +489,10 @@ test "GIF writer rejects non-indexed data without auto convert" { - - test "Should error on non GIF images" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var gif_file = gif.GIF.init(helpers.zigimg_test_allocator); - defer gif_file.deinit(); -@@ -563,7 +563,7 @@ const SINGLE_GIF_FILE_TEST = false; - fn loadRotatingEarthImage(allocator: std.mem.Allocator) !zigimg.Image { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_path = helpers.fixtures_path ++ "gif/rotating_earth.gif"; -- return zigimg.Image.fromFilePath(allocator, image_path, read_buffer[0..]); -+ return zigimg.Image.fromFilePath(allocator, std.testing.io, image_path, read_buffer[0..]); - } - - test "GIF test suite" { -@@ -575,10 +575,10 @@ test "GIF test suite" { - defer test_list.deinit(helpers.zigimg_test_allocator); - - const test_list_file = try helpers.testOpenFile(helpers.fixtures_path ++ "gif/TESTS"); -- defer test_list_file.close(); -+ defer test_list_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(test_list_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, test_list_file, read_buffer[0..]); - - var reader = read_stream.reader(); - -@@ -598,11 +598,11 @@ test "GIF test suite" { - for (test_list.items) |entry| { - doGifTest(entry) catch |err| { - has_failed_file = true; -- std.debug.print("Error: {}\n", .{err}); -+ std.log.info("Error: {}\n", .{err}); - continue; - }; - -- std.debug.print("OK\n", .{}); -+ std.log.info("OK\n", .{}); - } - - if (has_failed_file) { -@@ -612,10 +612,10 @@ test "GIF test suite" { - - test "Rotating Earth GIF" { - const gif_input_file = try helpers.testOpenFile(helpers.fixtures_path ++ "gif/rotating_earth.gif"); -- defer gif_input_file.close(); -+ defer gif_input_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(gif_input_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, gif_input_file, read_buffer[0..]); - - var gif_file = gif.GIF.init(helpers.zigimg_test_allocator); - defer gif_file.deinit(); -@@ -710,8 +710,8 @@ const IniFile = struct { - const equals_sign_position_opt = std.mem.indexOf(u8, read_line[0..], "="); - - if (equals_sign_position_opt) |equals_sign_position| { -- const key_name = std.mem.trimRight(u8, read_line[0..(equals_sign_position - 1)], " "); -- const string_value = std.mem.trimLeft(u8, read_line[(equals_sign_position + 1)..], " "); -+ const key_name = std.mem.trimEnd(u8, read_line[0..(equals_sign_position - 1)], " "); -+ const string_value = std.mem.trimStart(u8, read_line[(equals_sign_position + 1)..], " "); - - if (self.sections.getPtr(current_section)) |section_entry| { - const value = blk: { -@@ -760,7 +760,7 @@ const IniFile = struct { - }; - - fn doGifTest(entry_name: []const u8) !void { -- std.debug.print("GIF test {s}... ", .{entry_name}); -+ std.log.info("GIF test {s}... ", .{entry_name}); - - var area_alloc = std.heap.ArenaAllocator.init(helpers.zigimg_test_allocator); - const area_allocator = area_alloc.allocator(); -@@ -770,13 +770,13 @@ fn doGifTest(entry_name: []const u8) !void { - const config_filepath = try std.fs.path.resolve(area_allocator, &[_][]const u8{ helpers.fixtures_path, "gif", config_filename }); - - const config_file = try helpers.testOpenFile(config_filepath); -- defer config_file.close(); -+ defer config_file.close(std.testing.io); - - var config_ini = IniFile.init(helpers.zigimg_test_allocator); - defer config_ini.deinit(); - - var ini_read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var ini_read_stream = zigimg.io.ReadStream.initFile(config_file, ini_read_buffer[0..]); -+ var ini_read_stream = zigimg.io.ReadStream.initFile(std.testing.io, config_file, ini_read_buffer[0..]); - - try config_ini.parse(ini_read_stream.reader()); - -@@ -804,10 +804,10 @@ fn doGifTest(entry_name: []const u8) !void { - - const gif_input_filepath = try std.fs.path.resolve(area_allocator, &[_][]const u8{ helpers.fixtures_path, "gif", input_filename.string }); - const gif_input_file = try helpers.testOpenFile(gif_input_filepath); -- defer gif_input_file.close(); -+ defer gif_input_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(gif_input_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, gif_input_file, read_buffer[0..]); - - var gif_file = gif.GIF.init(helpers.zigimg_test_allocator); - defer gif_file.deinit(); -@@ -856,10 +856,10 @@ fn doGifTest(entry_name: []const u8) !void { - const pixels_filename = frame_section.getValue("pixels") orelse return error.InvalidGifConfigFile; - const pixels_filepath = try std.fs.path.resolve(area_allocator, &[_][]const u8{ helpers.fixtures_path, "gif", pixels_filename.string }); - const pixels_file = try helpers.testOpenFile(pixels_filepath); -- defer pixels_file.close(); -+ defer pixels_file.close(std.testing.io); - - var pixels_read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var pixels_read_stream = zigimg.io.ReadStream.initFile(pixels_file, pixels_read_buffer[0..]); -+ var pixels_read_stream = zigimg.io.ReadStream.initFile(std.testing.io, pixels_file, pixels_read_buffer[0..]); - - var pixels_reader = pixels_read_stream.reader(); - -@@ -894,7 +894,7 @@ fn doGifTest(entry_name: []const u8) !void { - try helpers.expectEq(actual_color.to.color(zigimg.color.Rgba32), gif_background_color); - } else { - helpers.expectEq(actual_color.to.color(zigimg.color.Rgba32), expected_color) catch |err| { -- std.debug.print("Pixel #{} does not match: expected={}, actual={}\n", .{ index, expected_color, actual_color.to.color(zigimg.color.Rgba32) }); -+ std.log.info("Pixel #{} does not match: expected={}, actual={}\n", .{ index, expected_color, actual_color.to.color(zigimg.color.Rgba32) }); - return err; - }; - } -@@ -929,27 +929,27 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { - - // Build path to GIF file - const gif_filename = std.fmt.allocPrint(area_allocator, "{s}.gif", .{entry_name}) catch { -- std.debug.print("FAIL (allocPrint failed)\n", .{}); -+ std.log.info("FAIL (allocPrint failed)\n", .{}); - return .failed; - }; - const gif_filepath = std.fs.path.resolve(area_allocator, &[_][]const u8{ helpers.fixtures_path, "gif", gif_filename }) catch { -- std.debug.print("FAIL (path resolve failed)\n", .{}); -+ std.log.info("FAIL (path resolve failed)\n", .{}); - return .failed; - }; - - // Read original GIF -- const gif_input_file = try std.fs.cwd().openFile(gif_filepath, .{}); -- defer gif_input_file.close(); -+ const gif_input_file = try std.Io.Dir.openFile(std.Io.Dir.cwd(), std.testing.io, gif_filepath, .{}); -+ defer gif_input_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(gif_input_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, gif_input_file, read_buffer[0..]); - - var original_gif = gif.GIF.init(helpers.zigimg_test_allocator); - defer original_gif.deinit(); - - var original_frames = original_gif.read(&read_stream) catch { - // Reader correctly rejected bad data -- std.debug.print("OK (reader correctly rejected bad data)\n", .{}); -+ std.log.info("OK (reader correctly rejected bad data)\n", .{}); - return .passed; - }; - defer { -@@ -961,7 +961,7 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { - - if (original_frames.items.len == 0) { - // No frames means nothing to write -- std.debug.print("OK (no frames to write)\n", .{}); -+ std.log.info("OK (no frames to write)\n", .{}); - return .passed; - } - -@@ -974,13 +974,13 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { - - // Non-indexed formats can't be written without auto_convert - if (!image.pixelFormat().isIndexed()) { -- std.debug.print("OK (non-indexed format correctly not written: {s})\n", .{@tagName(image.pixels)}); -+ std.log.info("OK (non-indexed format correctly not written: {s})\n", .{@tagName(image.pixels)}); - return .passed; - } - - // Zero dimensions can't be written - if (image.width == 0 or image.height == 0) { -- std.debug.print("OK (zero dimensions correctly not written)\n", .{}); -+ std.log.info("OK (zero dimensions correctly not written)\n", .{}); - return .passed; - } - -@@ -991,16 +991,16 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { - gif.GIF.writeImage(helpers.zigimg_test_allocator, &write_stream, image, .{ .gif = .{} }) catch |err| { - // For very large images, running out of buffer space is expected - if (err == error.WriteFailed and (image.width > 1000 or image.height > 1000)) { -- std.debug.print("OK (large image exceeded buffer)\n", .{}); -+ std.log.info("OK (large image exceeded buffer)\n", .{}); - return .passed; - } -- std.debug.print("FAIL (write error: {})\n", .{err}); -+ std.log.info("FAIL (write error: {})\n", .{err}); - return .failed; - }; - - const written_len = write_stream.writer().end; - if (written_len == 0) { -- std.debug.print("FAIL (wrote 0 bytes)\n", .{}); -+ std.log.info("FAIL (wrote 0 bytes)\n", .{}); - return .failed; - } - -@@ -1011,7 +1011,7 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { - defer decoded_gif.deinit(); - - var decoded_frames = decoded_gif.read(&read_back_stream) catch |err| { -- std.debug.print("FAIL (read-back error: {})\n", .{err}); -+ std.log.info("FAIL (read-back error: {})\n", .{err}); - return .failed; - }; - defer { -@@ -1025,7 +1025,7 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { - if (original_gif.header.width != decoded_gif.header.width or - original_gif.header.height != decoded_gif.header.height) - { -- std.debug.print("FAIL (dimension mismatch: {}x{} vs {}x{})\n", .{ -+ std.log.info("FAIL (dimension mismatch: {}x{} vs {}x{})\n", .{ - original_gif.header.width, - original_gif.header.height, - decoded_gif.header.width, -@@ -1036,7 +1036,7 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { - - // Verify we got at least one frame - if (decoded_frames.items.len == 0) { -- std.debug.print("FAIL (no frames in output)\n", .{}); -+ std.log.info("FAIL (no frames in output)\n", .{}); - return .failed; - } - -@@ -1048,14 +1048,14 @@ fn doGifRoundtripTest(entry_name: []const u8) !RoundtripResult { - while (original_iter.next()) |original_color| { - const decoded_color_opt = decoded_iter.next(); - if (decoded_color_opt == null) { -- std.debug.print("FAIL (missing pixel at index {})\n", .{pixel_index}); -+ std.log.info("FAIL (missing pixel at index {})\n", .{pixel_index}); - return .failed; - } - const original = original_color.to.color(zigimg.color.Rgba32); - const decoded = decoded_color_opt.?.to.color(zigimg.color.Rgba32); - - if (original.r != decoded.r or original.g != decoded.g or original.b != decoded.b) { -- std.debug.print("FAIL (pixel {} mismatch: ({},{},{}) vs ({},{},{}))\n", .{ -+ std.log.info("FAIL (pixel {} mismatch: ({},{},{}) vs ({},{},{}))\n", .{ - pixel_index, original.r, original.g, original.b, decoded.r, decoded.g, decoded.b, - }); - return .failed; -@@ -1071,10 +1071,10 @@ test "GIF writer roundtrip test suite" { - defer test_list.deinit(helpers.zigimg_test_allocator); - - const test_list_file = try helpers.testOpenFile(helpers.fixtures_path ++ "gif/TESTS"); -- defer test_list_file.close(); -+ defer test_list_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(test_list_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, test_list_file, read_buffer[0..]); - - var reader = read_stream.reader(); - -diff --git a/tests/formats/iff_test.zig b/tests/formats/iff_test.zig -index ab2dd6b..6a3594d 100644 ---- a/tests/formats/iff_test.zig -+++ b/tests/formats/iff_test.zig -@@ -5,12 +5,12 @@ const zigimg = @import("zigimg"); - - test "IFF-PBM indexed8 (chunky Deluxe Paint DOS file)" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-pbm.iff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_iff = iff.IFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -37,12 +37,12 @@ test "IFF-PBM indexed8 (chunky Deluxe Paint DOS file)" { - - test "IFF-ILBM indexed8 8 bitplanes" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-ilbm-8bit-compressed.iff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_iff = iff.IFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -63,12 +63,12 @@ test "IFF-ILBM indexed8 8 bitplanes" { - - test "IFF-ILBM indexed8 8 bitplanes uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-ilbm-8bit-uncompressed.iff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_iff = iff.IFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -89,12 +89,12 @@ test "IFF-ILBM indexed8 8 bitplanes uncompressed" { - - test "IFF-ILBM indexed8 6 bitplanes EHB" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-ehb.iff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_iff = iff.IFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -121,12 +121,12 @@ test "IFF-ILBM indexed8 6 bitplanes EHB" { - - test "IFF-ILBM indexed8 4 bitplanes HAM" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-ham.iff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_iff = iff.IFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -150,12 +150,12 @@ test "IFF-ILBM indexed8 4 bitplanes HAM" { - - test "IFF-ILBM indexed8 6 bitplanes HAM8" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-ham8.iff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_iff = iff.IFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -179,12 +179,12 @@ test "IFF-ILBM indexed8 6 bitplanes HAM8" { - - test "IFF-ILBM 24bit" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-24bit.iff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_iff = iff.IFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -208,12 +208,12 @@ test "IFF-ILBM 24bit" { - - test "IFF-ILBM indexed8 4 bitplanes Atari ST" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-ilbm-4bit-compressed-atari.iff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_iff = iff.IFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -240,12 +240,12 @@ test "IFF-ILBM indexed8 4 bitplanes Atari ST" { - - test "IFF-ACBM indexed8 3 bitplanes uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ilbm/sample-8bit.acbm"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_iff = iff.IFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_iff.read(&read_stream, helpers.zigimg_test_allocator); - defer pixels.deinit(helpers.zigimg_test_allocator); -diff --git a/tests/formats/jpeg_test.zig b/tests/formats/jpeg_test.zig -index 8288b62..0f1254f 100644 ---- a/tests/formats/jpeg_test.zig -+++ b/tests/formats/jpeg_test.zig -@@ -7,10 +7,10 @@ const color = zigimg.color; - - test "Should error on non JPEG images" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); - defer jpeg_file.deinit(); -@@ -28,10 +28,10 @@ test "Should error on non JPEG images" { - - test "Read JFIF header properly and decode simple Huffman stream" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "jpeg/huff_simple0.jpg"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); - defer jpeg_file.deinit(); -@@ -59,10 +59,10 @@ test "Read JFIF header properly and decode simple Huffman stream" { - - test "Read the tuba properly" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "jpeg/tuba.jpg"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); - defer jpeg_file.deinit(); -@@ -95,10 +95,10 @@ test "Read the tuba properly" { - - test "Read grayscale images" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "jpeg/grayscale_sample0.jpg"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); - defer jpeg_file.deinit(); -@@ -130,19 +130,19 @@ test "Read grayscale images" { - } - - test "Read subsampling images" { -- var testdir = std.fs.cwd().openDir(helpers.fixtures_path ++ "jpeg/", .{ .access_sub_paths = false, .no_follow = true, .iterate = true }) catch null; -+ var testdir = std.Io.Dir.openDir(std.Io.Dir.cwd(), std.testing.io, helpers.fixtures_path ++ "jpeg/", .{ .access_sub_paths = false, .iterate = true }) catch null; - if (testdir) |*idir| { -- defer idir.close(); -+ defer idir.close(std.testing.io); - - var it = idir.iterate(); -- while (try it.next()) |entry| { -+ while (try it.next(std.testing.io)) |entry| { - if (entry.kind != .file or !std.mem.endsWith(u8, entry.name, ".jpg") or !std.mem.startsWith(u8, entry.name, "subsampling_")) continue; - -- var test_file = try idir.openFile(entry.name, .{ .mode = .read_only }); -- defer test_file.close(); -+ var test_file = try idir.openFile(std.testing.io, entry.name, .{ .mode = .read_only }); -+ defer test_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(test_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, test_file, read_buffer[0..]); - - var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); - defer jpeg_file.deinit(); -@@ -182,10 +182,10 @@ test "Read subsampling images" { - - test "Read progressive jpeg with restart intervals" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "jpeg/tuba_restart_prog.jpg"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); - defer jpeg_file.deinit(); -@@ -403,10 +403,10 @@ test "JPEG writer round-trip with all test fixtures" { - for (test_fixtures) |fixture_path| { - // Skip if file doesn't exist - const file = helpers.testOpenFile(fixture_path) catch continue; -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [4096]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - var jpeg_file = jpeg.JPEG.init(helpers.zigimg_test_allocator); - defer jpeg_file.deinit(); - -diff --git a/tests/formats/netpbm_test.zig b/tests/formats/netpbm_test.zig -index ee06575..e0ffe05 100644 ---- a/tests/formats/netpbm_test.zig -+++ b/tests/formats/netpbm_test.zig -@@ -5,10 +5,10 @@ const zigimg = @import("zigimg"); - - test "Load ASCII PBM image" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/pbm_ascii.pbm"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var pbm_file = netpbm.PBM{}; - -@@ -27,10 +27,10 @@ test "Load ASCII PBM image" { - - test "Load binary PBM image" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/pbm_binary.pbm"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var pbm_file = netpbm.PBM{}; - -@@ -49,10 +49,10 @@ test "Load binary PBM image" { - - test "Load ASCII PGM 8-bit grayscale image" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/pgm_ascii_grayscale8.pgm"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var pgm_file = netpbm.PGM{}; - -@@ -71,10 +71,10 @@ test "Load ASCII PGM 8-bit grayscale image" { - - test "Load Binary PGM 8-bit grayscale image" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/pgm_binary_grayscale8.pgm"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var pgm_file = netpbm.PGM{}; - -@@ -93,10 +93,10 @@ test "Load Binary PGM 8-bit grayscale image" { - - test "Load ASCII PGM 16-bit grayscale image" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/pgm_ascii_grayscale16.pgm"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var pgm_file = netpbm.PGM{}; - -@@ -115,10 +115,10 @@ test "Load ASCII PGM 16-bit grayscale image" { - - test "Load Binary PGM 16-bit grayscale image" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/pgm_binary_grayscale16.pgm"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var pgm_file = netpbm.PGM{}; - -@@ -137,10 +137,10 @@ test "Load Binary PGM 16-bit grayscale image" { - - test "Load ASCII PPM image" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/ppm_ascii_rgb24.ppm"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var ppm_file = netpbm.PPM{}; - -@@ -176,10 +176,10 @@ test "Load ASCII PPM image" { - - test "Load binary PPM image" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "netpbm/ppm_binary_rgb24.ppm"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var ppm_file = netpbm.PPM{}; - -@@ -234,16 +234,16 @@ test "Write bitmap(grayscale1) ASCII PBM file" { - } - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .pbm = .{ .binary = false }, - }); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch unreachable; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; - } - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, image_file_name, read_buffer[0..]); -+ var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, read_buffer[0..]); - defer read_image.deinit(helpers.zigimg_test_allocator); - - try helpers.expectEq(read_image.width, width); -@@ -281,16 +281,16 @@ test "Write bitmap(Grayscale1) binary PBM file" { - } - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .pbm = .{ .binary = true }, - }); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch unreachable; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; - } - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, image_file_name, read_buffer[0..]); -+ var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, read_buffer[0..]); - defer read_image.deinit(helpers.zigimg_test_allocator); - - try helpers.expectEq(read_image.width, width); -@@ -324,16 +324,16 @@ test "Write grayscale8 ASCII PGM file" { - } - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .pgm = .{ .binary = false }, - }); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch unreachable; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; - } - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, image_file_name, read_buffer[0..]); -+ var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, read_buffer[0..]); - defer read_image.deinit(helpers.zigimg_test_allocator); - - try helpers.expectEq(read_image.width, width); -@@ -367,16 +367,16 @@ test "Write grayscale8 binary PGM file" { - } - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .pgm = .{ .binary = true }, - }); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch unreachable; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; - } - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, image_file_name, read_buffer[0..]); -+ var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, read_buffer[0..]); - defer read_image.deinit(helpers.zigimg_test_allocator); - - try helpers.expectEq(read_image.width, width); -@@ -420,16 +420,16 @@ test "Writing Rgb24 ASCII PPM format" { - pixels.rgb24[7] = zigimg.color.Rgb24.from.rgb(255, 255, 0); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .ppm = .{ .binary = false }, - }); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch unreachable; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; - } - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, image_file_name, read_buffer[0..]); -+ var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, read_buffer[0..]); - defer read_image.deinit(helpers.zigimg_test_allocator); - - try helpers.expectEq(read_image.width, width); -@@ -474,16 +474,16 @@ test "Writing Rgb24 binary PPM format" { - pixels.rgb24[7] = zigimg.color.Rgb24.from.rgb(255, 255, 0); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .ppm = .{ .binary = true }, - }); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch unreachable; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; - } - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, image_file_name, read_buffer[0..]); -+ var read_image = try zigimg.Image.fromFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, read_buffer[0..]); - defer read_image.deinit(helpers.zigimg_test_allocator); - - try helpers.expectEq(read_image.width, width); -@@ -505,7 +505,7 @@ test "Trying to write a bitmap or grayscale Netpbm with an true color pixel form - defer source_image.deinit(helpers.zigimg_test_allocator); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch unreachable; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; - } - - const pixels = source_image.pixels; -@@ -527,12 +527,12 @@ test "Trying to write a bitmap or grayscale Netpbm with an true color pixel form - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - - { -- const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .pbm = .{} }); -+ const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .pbm = .{} }); - try std.testing.expectError(zigimg.Image.WriteError.Unsupported, write_error); - } - - { -- const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .pgm = .{} }); -+ const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .pgm = .{} }); - try std.testing.expectError(zigimg.Image.WriteError.Unsupported, write_error); - } - } -@@ -551,7 +551,7 @@ test "Trying to write a bitmap or true color Netpbm with a 8-bit grayscale pixel - defer source_image.deinit(helpers.zigimg_test_allocator); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch unreachable; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; - } - - const source = source_image.pixels; -@@ -562,12 +562,12 @@ test "Trying to write a bitmap or true color Netpbm with a 8-bit grayscale pixel - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - - { -- const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .pbm = .{} }); -+ const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .pbm = .{} }); - try std.testing.expectError(zigimg.Image.WriteError.Unsupported, write_error); - } - - { -- const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .ppm = .{} }); -+ const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .ppm = .{} }); - try std.testing.expectError(zigimg.Image.WriteError.Unsupported, write_error); - } - } -@@ -589,7 +589,7 @@ test "Trying to write a grayscale or true color Netbpm with a 1-bit grayscale pi - defer source_image.deinit(helpers.zigimg_test_allocator); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch unreachable; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch unreachable; - } - - const source = source_image.pixels; -@@ -601,12 +601,12 @@ test "Trying to write a grayscale or true color Netbpm with a 1-bit grayscale pi - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - - { -- const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .pgm = .{} }); -+ const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .pgm = .{} }); - try std.testing.expectError(zigimg.Image.WriteError.Unsupported, write_error); - } - - { -- const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .ppm = .{} }); -+ const write_error = source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .ppm = .{} }); - try std.testing.expectError(zigimg.Image.WriteError.Unsupported, write_error); - } - } -diff --git a/tests/formats/pam_test.zig b/tests/formats/pam_test.zig -index 2d58d45..5742f77 100644 ---- a/tests/formats/pam_test.zig -+++ b/tests/formats/pam_test.zig -@@ -5,10 +5,10 @@ const zigimg = @import("zigimg"); - - test "rejects non-PAM images" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - -@@ -17,10 +17,10 @@ test "rejects non-PAM images" { - - test "rejects PAM images with unsupported depth" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/unsupported_depth.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - -@@ -29,10 +29,10 @@ test "rejects PAM images with unsupported depth" { - - test "rejects PAM images with invalid maxval" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/invalid_maxval.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - -@@ -41,10 +41,10 @@ test "rejects PAM images with invalid maxval" { - - test "rejects PAM images with component values greater than maxval" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/value_greater_than_maxval.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - try helpers.expectError(invalid, zigimg.Image.ReadError.InvalidData); -@@ -52,10 +52,10 @@ test "rejects PAM images with component values greater than maxval" { - - test "rejects PAM images with unknown tuple type" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/unknown_tupletype.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - -@@ -64,10 +64,10 @@ test "rejects PAM images with unknown tuple type" { - - test "rejects PAM images with invalid first token" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/invalid_first_token.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - -@@ -76,10 +76,10 @@ test "rejects PAM images with invalid first token" { - - test "rejects PAM images with tuple type not matching other parameters" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/non_matching_tuple_type.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const invalid = pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - -@@ -88,10 +88,10 @@ test "rejects PAM images with tuple type not matching other parameters" { - - test "accepts comments" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_blackandwhite_comments.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - defer image.deinit(helpers.zigimg_test_allocator); -@@ -110,10 +110,10 @@ test "accepts comments" { - - test "reads blackandwhite pam" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_blackandwhite.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - defer image.deinit(helpers.zigimg_test_allocator); -@@ -132,10 +132,10 @@ test "reads blackandwhite pam" { - - test "reads blackandwhite_alpha pam" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_blackandwhite_alpha.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - defer image.deinit(helpers.zigimg_test_allocator); -@@ -154,10 +154,10 @@ test "reads blackandwhite_alpha pam" { - - test "reads grayscale pam with maxval 255" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_grayscale_maxval_255.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - defer image.deinit(helpers.zigimg_test_allocator); -@@ -176,10 +176,10 @@ test "reads grayscale pam with maxval 255" { - - test "reads grayscale alpha pam with maxval 255" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_grayscale_alpha_maxval_255.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - defer image.deinit(helpers.zigimg_test_allocator); -@@ -198,10 +198,10 @@ test "reads grayscale alpha pam with maxval 255" { - - test "read of rgb pam with maxval 255" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/horse.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - defer image.deinit(helpers.zigimg_test_allocator); -@@ -211,10 +211,10 @@ test "read of rgb pam with maxval 255" { - - test "basic read-write-read produces same result" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_grayscale_alpha_maxval_255.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - defer image.deinit(helpers.zigimg_test_allocator); -@@ -239,7 +239,7 @@ test "basic read-write-read produces same result" { - - var decoded_image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_back_stream); - defer decoded_image.deinit(helpers.zigimg_test_allocator); -- -+ - try std.testing.expectEqual(@as(usize, 4), decoded_image.height); - try std.testing.expectEqual(@as(usize, 4), decoded_image.width); - try helpers.expectEqSlice(zigimg.color.Grayscale8Alpha, decoded_image.pixels.grayscale8Alpha, &[16]zigimg.color.Grayscale8Alpha{ -@@ -254,10 +254,10 @@ test "basic read-write-read produces same result" { - - test "reads rgba pam with maxval 255" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_rgba_maxval_255.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - defer image.deinit(helpers.zigimg_test_allocator); -@@ -273,10 +273,10 @@ test "reads rgba pam with maxval 255" { - - test "reads rgba pam with maxval 65535" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pam/simple_rgba_maxval_65535.pam"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var image = try pam.PAM.readImage(helpers.zigimg_test_allocator, &read_stream); - defer image.deinit(helpers.zigimg_test_allocator); -diff --git a/tests/formats/pcx_test.zig b/tests/formats/pcx_test.zig -index db5d556..ee51851 100644 ---- a/tests/formats/pcx_test.zig -+++ b/tests/formats/pcx_test.zig -@@ -7,10 +7,10 @@ const toU8 = zigimg.color.ScaleValue(u8); - - test "PCX indexed1 (linear)" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp1.pcx"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var pcx_file = pcx.PCX{}; - -@@ -45,10 +45,10 @@ test "PCX indexed1 (linear)" { - - test "PCX indexed4 (linear)" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp4.pcx"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var pcx_file = pcx.PCX{}; - -@@ -84,10 +84,10 @@ test "PCX indexed4 (linear)" { - - test "PCX indexed8 (linear)" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp8.pcx"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var pcx_file = pcx.PCX{}; - -@@ -125,10 +125,10 @@ test "PCX indexed8 (linear)" { - - test "PCX indexed24 (planar)" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp24.pcx"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var pcx_file = pcx.PCX{}; - -@@ -169,25 +169,25 @@ test "Write PCX indexed1 (odd width)" { - const image_file_name = "zigimg_pcx_indexed1_odd.pcx"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp1.pcx"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .pcx = .{}, - }); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var pcx_file = pcx.PCX{}; - -@@ -242,18 +242,18 @@ test "Write PCX indexed 1 (even width)" { - const image_file_name = "zigimg_pcx_indexed1_even.pcx"; - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try image_pattern.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try image_pattern.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .pcx = .{}, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var pcx_file = pcx.PCX{}; - -@@ -285,25 +285,25 @@ test "Write PCX indexed4 (odd width)" { - const image_file_name = "zigimg_pcx_indexed4_odd.pcx"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp4.pcx"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .pcx = .{}, - }); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var pcx_file = pcx.PCX{}; - -@@ -359,18 +359,18 @@ test "Write PCX indexed 4 (even width)" { - const image_file_name = "zigimg_pcx_indexed4_even.pcx"; - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try rainbow_test.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try rainbow_test.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .pcx = .{}, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var pcx_file = pcx.PCX{}; - -@@ -400,25 +400,25 @@ test "Write PCX indexed8 (odd width)" { - const image_file_name = "zigimg_pcx_indexed8_odd.pcx"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp8.pcx"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .pcx = .{}, - }); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var pcx_file = pcx.PCX{}; - -@@ -472,18 +472,18 @@ test "Write PCX indexed 8 (even width)" { - const image_file_name = "zigimg_pcx_indexed8_even.pcx"; - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try rainbow_test.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try rainbow_test.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .pcx = .{}, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var pcx_file = pcx.PCX{}; - -@@ -513,25 +513,25 @@ test "Write PCX rgb24 (odd width)" { - const image_file_name = "zigimg_pcx_bpp24_odd.pcx"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "pcx/test-bpp24.pcx"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .pcx = .{}, - }); - - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var pcx_file = pcx.PCX{}; - -@@ -594,18 +594,18 @@ test "Write PCX rgb24 (even width)" { - const image_file_name = "zigimg_pcx_rgb24_even.pcx"; - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try rainbow_test.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ -+ try rainbow_test.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ - .pcx = .{}, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var pcx_file = pcx.PCX{}; - -diff --git a/tests/formats/png_test.zig b/tests/formats/png_test.zig -index ec39855..def25c7 100644 ---- a/tests/formats/png_test.zig -+++ b/tests/formats/png_test.zig -@@ -8,10 +8,10 @@ const valid_header_data = png.magic_header ++ "\x00\x00\x00\x0d" ++ png.Chunks.I - - test "png: Should error on non PNG images" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const invalidFile = png.PNG.readImage(helpers.zigimg_test_allocator, &read_stream); - -@@ -111,10 +111,10 @@ test "png: Indexed PNG with transparency (Aseprite output)" { - // mlarouche: While the full test suite already test this image, I like having a smaller test that I can verify - // some specific info myself - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "png/aseprite_indexed_transparent.png"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var png_image = try png.PNG.readImage(helpers.zigimg_test_allocator, &read_stream); - defer png_image.deinit(helpers.zigimg_test_allocator); -@@ -180,18 +180,18 @@ test "png: Don't write tRNS chunk in indexed format when there is no alpha" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_indexed_no_trns.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .png = .{}, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var check_trns_processor: CheckTrnsPresentProcessor = .{}; - -@@ -220,18 +220,18 @@ test "png: Write tRNS chunk in indexed format only when alpha is present" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_indexed_trns.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .png = .{}, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var check_trns_processor: CheckTrnsPresentProcessor = .{}; - -@@ -268,16 +268,16 @@ test "png: Write indexed1 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_indexed1.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -322,16 +322,16 @@ test "png: Write indexed2 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_indexed2.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -376,16 +376,16 @@ test "png: Write indexed4 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_indexed4.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -430,16 +430,16 @@ test "png: Write indexed8 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_indexed8.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -475,16 +475,16 @@ test "png: Write grayscale1 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_grayscale1.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -513,16 +513,16 @@ test "png: Write grayscale2 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_grayscale2.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -551,16 +551,16 @@ test "png: Write grayscale4 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_grayscale4.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -589,16 +589,16 @@ test "png: Write grayscale8 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_grayscale8.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -628,16 +628,16 @@ test "png: Write grayscale8Alpha format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_grayscale8Alpha.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -667,16 +667,16 @@ test "png: Write grayscale16 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_grayscale16.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -706,16 +706,16 @@ test "png: Write grayscale16Alpha format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_grayscale16Alpha.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -747,16 +747,16 @@ test "png: Write rgb24 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_rgb24.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -790,16 +790,16 @@ test "png: Write rgba32 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_rgba32.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -833,16 +833,16 @@ test "png: Write rgb48 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_rgb48.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -876,16 +876,16 @@ test "png: Write rgba64 format" { - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - const image_file_name = "zigimg_png_rgba64.png"; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], .{ .png = .{} }); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], .{ .png = .{} }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var options = png.DefaultOptions.init(.{}); - -@@ -910,32 +910,32 @@ test "png: Official Test Suite" { - - // Useful to quickly test everything on full dir of images - pub fn testWithDir(directory: []const u8, test_md5_signature: bool) !void { -- var test_dir_opt = std.fs.cwd().openDir(directory, .{ .access_sub_paths = false, .no_follow = true, .iterate = true }) catch null; -+ var test_dir_opt = std.Io.Dir.openDir(std.Io.Dir.cwd(), std.testing.io, directory, .{ .access_sub_paths = false, .follow_symlinks = false, .iterate = true }) catch null; - if (test_dir_opt) |*test_dir| { -- defer test_dir.close(); -+ defer test_dir.close(std.testing.io); - var it = test_dir.iterate(); - if (test_md5_signature) { -- std.debug.print("\n", .{}); -+ std.log.info("\n", .{}); - } -- while (try it.next()) |entry| { -+ while (try it.next(std.testing.io)) |entry| { - if (entry.kind != .file or !std.mem.eql(u8, std.fs.path.extension(entry.name), ".png")) { - continue; - } - - if (test_md5_signature) { -- std.debug.print("Testing file {s} ... ", .{entry.name}); -+ std.log.info("Testing file {s} ... ", .{entry.name}); - } - -- var test_file = try test_dir.openFile(entry.name, .{ .mode = .read_only }); -- defer test_file.close(); -+ var test_file = try test_dir.openFile(std.testing.io, entry.name, .{ .mode = .read_only }); -+ defer test_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(test_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, test_file, read_buffer[0..]); - - if (entry.name[0] == 'x' and entry.name[2] != 't' and entry.name[2] != 's') { - try std.testing.expectError(zigimg.Image.ReadError.InvalidData, png.loadHeader(&read_stream)); - if (test_md5_signature) { -- std.debug.print("OK\n", .{}); -+ std.log.info("OK\n", .{}); - } - continue; - } -@@ -946,7 +946,7 @@ pub fn testWithDir(directory: []const u8, test_md5_signature: bool) !void { - const error_result = png.loadWithHeader(&read_stream, &header, std.testing.allocator, default_options.get()); - try std.testing.expectError(zigimg.Image.ReadError.InvalidData, error_result); - if (test_md5_signature) { -- std.debug.print("OK\n", .{}); -+ std.log.info("OK\n", .{}); - } - continue; - } -@@ -968,11 +968,11 @@ pub fn testWithDir(directory: []const u8, test_md5_signature: bool) !void { - @memcpy(test_data_name[len - 3 .. len], "tsd"); - - // Read test data and check with it -- if (test_dir.openFile(test_data_name[0..len], .{ .mode = .read_only })) |test_data_file| { -- defer test_data_file.close(); -+ if (test_dir.openFile(std.testing.io, test_data_name[0..len], .{ .mode = .read_only })) |test_data_file| { -+ defer test_data_file.close(std.testing.io); - - var test_read_buffer: [512]u8 = undefined; -- var test_file_reader = test_data_file.reader(test_read_buffer[0..]); -+ var test_file_reader = test_data_file.reader(std.testing.io, test_read_buffer[0..]); - var test_reader = &test_file_reader.interface; - - var expected_md5: [16]u8 = undefined; -@@ -983,31 +983,31 @@ pub fn testWithDir(directory: []const u8, test_md5_signature: bool) !void { - const str_md5 = try test_reader.takeDelimiterExclusive('\n'); - _ = try std.fmt.hexToBytes(expected_md5[0..], str_md5); - try std.testing.expectEqual(expected_pixel_format, std.meta.activeTag(result)); -- try std.testing.expectEqualSlices(u8, expected_md5[0..], md5_val[0..]); // catch std.debug.print("MD5 Expected: {s} Got {s}\n", .{std.fmt.fmtSliceHexUpper(expected_md5[0..]), std.fmt.fmtSliceHexUpper(md5_val[0..])}); -+ try std.testing.expectEqualSlices(u8, expected_md5[0..], md5_val[0..]); // catch std.log.info("MD5 Expected: {s} Got {s}\n", .{std.fmt.fmtSliceHexUpper(expected_md5[0..]), std.fmt.fmtSliceHexUpper(md5_val[0..])}); - } else |_| { - // If there is no test data assume test is correct and write it out - try writeTestData(test_dir, test_data_name[0..len], &result, md5_val[0..]); - } - - if (test_md5_signature) { -- std.debug.print("OK\n", .{}); -+ std.log.info("OK\n", .{}); - } - - // Write Raw bytes - // std.mem.copyForwards(u8, tst_data_name[len - 3 .. len + 1], "data"); - // var rawoutput = try idir.createFile(tst_data_name[0 .. len + 1], .{}); -- // defer rawoutput.close(); -+ // defer rawoutput.close(std.testing.io); - // try rawoutput.writeAll(result_bytes); - } - } - } - --fn writeTestData(dir: *std.fs.Dir, test_data_name: []const u8, result: *zigimg.color.PixelStorage, md5_value: []const u8) !void { -- var test_output = try dir.createFile(test_data_name, .{}); -- defer test_output.close(); -+fn writeTestData(dir: *std.Io.Dir, test_data_name: []const u8, result: *zigimg.color.PixelStorage, md5_value: []const u8) !void { -+ var test_output = try dir.createFile(std.testing.io, test_data_name, .{}); -+ defer test_output.close(std.testing.io); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var file_writer = test_output.writer(write_buffer[0..]); -+ var file_writer = test_output.writer(std.testing.io, write_buffer[0..]); - var writer = &file_writer.interface; - - try writer.print("{s}\n{X}", .{ @tagName(result.*), md5_value }); -@@ -1018,14 +1018,14 @@ fn writeTestData(dir: *std.fs.Dir, test_data_name: []const u8, result: *zigimg.c - test "png: InfoProcessor on Png Test suite" { - const directory = helpers.fixtures_path ++ "png/"; - -- var test_dir_opt = std.fs.cwd().openDir(directory, .{ .access_sub_paths = false, .no_follow = true, .iterate = true }) catch null; -+ var test_dir_opt = std.Io.Dir.openDir(std.Io.Dir.cwd(), std.testing.io, directory, .{ .access_sub_paths = false, .iterate = true }) catch null; - if (test_dir_opt) |*test_dir| { -- defer test_dir.close(); -+ defer test_dir.close(std.testing.io); - var it = test_dir.iterate(); - - var info_buffer: [16384]u8 = undefined; - -- while (try it.next()) |entry| { -+ while (try it.next(std.testing.io)) |entry| { - if (entry.kind != .file or !std.mem.eql(u8, std.fs.path.extension(entry.name), ".png")) { - continue; - } -@@ -1033,11 +1033,11 @@ test "png: InfoProcessor on Png Test suite" { - var info_stream = zigimg.io.WriteStream.initMemory(info_buffer[0..]); - var options = png.InfoProcessor.PngInfoOptions.init(png.InfoProcessor.init(info_stream.writer())); - -- var test_file = try test_dir.openFile(entry.name, .{ .mode = .read_only }); -- defer test_file.close(); -+ var test_file = try test_dir.openFile(std.testing.io, entry.name, .{ .mode = .read_only }); -+ defer test_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(test_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, test_file, read_buffer[0..]); - if (entry.name[0] == 'x') { - continue; - } -@@ -1051,13 +1051,13 @@ test "png: InfoProcessor on Png Test suite" { - @memcpy(test_data_name[len - 4 .. len], "info"); - - // Read test data and check with it -- if (test_dir.openFile(test_data_name[0..len], .{ .mode = .read_only })) |test_data_file| { -- defer test_data_file.close(); -+ if (test_dir.openFile(std.testing.io, test_data_name[0..len], .{ .mode = .read_only })) |test_data_file| { -+ defer test_data_file.close(std.testing.io); - - var expected_data_buffer: [16384]u8 = undefined; - - var test_read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var file_reader = test_data_file.reader(test_read_buffer[0..]); -+ var file_reader = test_data_file.reader(std.testing.io, test_read_buffer[0..]); - var reader = &file_reader.interface; - - const loaded = try reader.readSliceShort(expected_data_buffer[0..]); -@@ -1065,11 +1065,11 @@ test "png: InfoProcessor on Png Test suite" { - try std.testing.expectEqualSlices(u8, expected_data_buffer[0..loaded], info_buffer[0..loaded]); - } else |_| { - // If there is no test data assume test is correct and write it out -- var test_output_file = try test_dir.createFile(test_data_name[0..len], .{}); -- defer test_output_file.close(); -+ var test_output_file = try test_dir.createFile(std.testing.io, test_data_name[0..len], .{}); -+ defer test_output_file.close(std.testing.io); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var file_writer = test_output_file.writer(write_buffer[0..]); -+ var file_writer = test_output_file.writer(std.testing.io, write_buffer[0..]); - var writer = &file_writer.interface; - - try writer.writeAll(info_buffer[0..@intCast(info_stream.getPos())]); -diff --git a/tests/formats/qoi_test.zig b/tests/formats/qoi_test.zig -index 96256e3..5a9f985 100644 ---- a/tests/formats/qoi_test.zig -+++ b/tests/formats/qoi_test.zig -@@ -8,10 +8,10 @@ const zero_qoi_file = helpers.fixtures_path ++ "qoi/zero.qoi"; - - test "Should error on non QOI images" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var qoi_file = qoi.QOI{}; - -@@ -22,10 +22,10 @@ test "Should error on non QOI images" { - - test "Read zero.qoi file" { - const file = try helpers.testOpenFile(zero_qoi_file); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var qoi_file = qoi.QOI{}; - -diff --git a/tests/formats/ras_test.zig b/tests/formats/ras_test.zig -index d7240a1..5452098 100644 ---- a/tests/formats/ras_test.zig -+++ b/tests/formats/ras_test.zig -@@ -5,10 +5,10 @@ const helpers = @import("../helpers.zig"); - - test "Should error on non RAS images" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var ras_file = ras.RAS{}; - -@@ -18,12 +18,12 @@ test "Should error on non RAS images" { - - test "Sun-Raster 24-bit RGB24 uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-rgb24.ras"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var ras_file = ras.RAS{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -46,12 +46,12 @@ test "Sun-Raster 24-bit RGB24 uncompressed" { - - test "Sun-Raster 24-bit BGR24 uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-bgr24.ras"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var ras_file = ras.RAS{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -74,12 +74,12 @@ test "Sun-Raster 24-bit BGR24 uncompressed" { - - test "Sun-Raster 32-bit xRGB uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-xrgb32.ras"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var ras_file = ras.RAS{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -98,12 +98,12 @@ test "Sun-Raster 32-bit xRGB uncompressed" { - - test "Sun-Raster 8-bit with palette uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-8bit.ras"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var ras_file = ras.RAS{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -124,12 +124,12 @@ test "Sun-Raster 8-bit with palette uncompressed" { - - test "Sun-Raster 8-bit grayscale uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-8bit-grayscale.ras"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var ras_file = ras.RAS{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -144,12 +144,12 @@ test "Sun-Raster 8-bit grayscale uncompressed" { - - test "Sun-Raster 1-bit black & white uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-blackwhite.ras"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var ras_file = ras.RAS{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -164,12 +164,12 @@ test "Sun-Raster 1-bit black & white uncompressed" { - - test "Sun-Raster bgr24 rle compressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "ras/sample-24bit-bgr-rle.ras"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var ras_file = ras.RAS{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try ras_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -diff --git a/tests/formats/sgi_test.zig b/tests/formats/sgi_test.zig -index d098ef7..ed23769 100644 ---- a/tests/formats/sgi_test.zig -+++ b/tests/formats/sgi_test.zig -@@ -5,10 +5,10 @@ const zigimg = @import("zigimg"); - - test "Should error on non SGI images" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var sgi_file = sgi.SGI{}; - -@@ -18,12 +18,12 @@ test "Should error on non SGI images" { - - test "SGI 24-bit uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-rgb24.sgi"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var sgi_file = sgi.SGI{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -46,12 +46,12 @@ test "SGI 24-bit uncompressed" { - - test "SGI grayscale uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-blackwhite.sgi"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var sgi_file = sgi.SGI{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -66,12 +66,12 @@ test "SGI grayscale uncompressed" { - - test "SGI 32-bit RGBA uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-rgba.sgi"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var sgi_file = sgi.SGI{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -94,12 +94,12 @@ test "SGI 32-bit RGBA uncompressed" { - - test "SGI RGB48be uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-rgb48be.sgi"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var sgi_file = sgi.SGI{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -122,12 +122,12 @@ test "SGI RGB48be uncompressed" { - - test "SGI grayscale rle compressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-gray-rle.sgi"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var sgi_file = sgi.SGI{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -142,12 +142,12 @@ test "SGI grayscale rle compressed" { - - test "SGI 24-bit rle compressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-24bit-rle.sgi"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var sgi_file = sgi.SGI{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -170,12 +170,12 @@ test "SGI 24-bit rle compressed" { - - test "SGI RGB48be rle uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-rgb48be-rle.sgi"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var sgi_file = sgi.SGI{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -198,12 +198,12 @@ test "SGI RGB48be rle uncompressed" { - - test "SGI 32-bit RGBA rle compressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "sgi/sample-rgba-rle.sgi"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var sgi_file = sgi.SGI{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try sgi_file.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -diff --git a/tests/formats/tga_test.zig b/tests/formats/tga_test.zig -index 83f2d76..c3182d4 100644 ---- a/tests/formats/tga_test.zig -+++ b/tests/formats/tga_test.zig -@@ -5,10 +5,10 @@ const zigimg = @import("zigimg"); - - test "Should error on non TGA images" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -19,10 +19,10 @@ test "Should error on non TGA images" { - - test "Read ubw8 TGA file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ubw8.tga"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -56,10 +56,10 @@ test "Read ubw8 TGA file" { - - test "Read ucm8 TGA file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ucm8.tga"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -101,10 +101,10 @@ test "Read ucm8 TGA file" { - - test "Read utc16 TGA file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc16.tga"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -140,10 +140,10 @@ test "Read utc16 TGA file" { - - test "Read utc24 TGA file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc24.tga"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -179,10 +179,10 @@ test "Read utc24 TGA file" { - - test "Read utc32 TGA file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc32.tga"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -218,10 +218,10 @@ test "Read utc32 TGA file" { - - test "Read cbw8 TGA file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/cbw8.tga"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -255,10 +255,10 @@ test "Read cbw8 TGA file" { - - test "Read ccm8 TGA file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ccm8.tga"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -300,10 +300,10 @@ test "Read ccm8 TGA file" { - - test "Read ctc24 TGA file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ctc24.tga"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -339,10 +339,10 @@ test "Read ctc24 TGA file" { - - test "Read matte-01 TGA file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/matte-01.tga"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -384,10 +384,10 @@ test "Read matte-01 TGA file" { - - test "Read font TGA file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/font.tga"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -411,10 +411,10 @@ test "Read font TGA file" { - - test "Read stopsignsmall TGA v1 file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/stopsignsmall.tga"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -436,10 +436,10 @@ test "Read stopsignsmall TGA v1 file" { - - test "Read stopsignsmallcompressed TGA v1 file" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/stopsignsmallcompressed.tga"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -463,11 +463,11 @@ test "Write TGA uncompressed grayscale8" { - const image_file_name = "zigimg_tga_uncompressed_grayscale8.tga"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ubw8.tga"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - const encoder_options = zigimg.Image.EncoderOptions{ -@@ -500,15 +500,15 @@ test "Write TGA uncompressed grayscale8" { - }; - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], encoder_options); -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], encoder_options); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -558,15 +558,15 @@ test "Write TGA compressed grayscale8" { - const image_file_name = "zigimg_tga_compressed_grayscale8.tga"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/cbw8.tga"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = true, - .color_map_depth = 16, -@@ -575,13 +575,13 @@ test "Write TGA compressed grayscale8" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -617,15 +617,15 @@ test "Write uncompressed indexed8 (color map 16-bit)" { - const image_file_name = "zigimg_tga_uncompressed_indexed8.tga"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ucm8.tga"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = false, - .color_map_depth = 16, -@@ -634,13 +634,13 @@ test "Write uncompressed indexed8 (color map 16-bit)" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -684,15 +684,15 @@ test "Write uncompressed indexed8 (color map 24-bit)" { - const image_file_name = "zigimg_tga_uncompressed_indexed8.tga"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ucm8.tga"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = false, - .color_map_depth = 24, -@@ -701,13 +701,13 @@ test "Write uncompressed indexed8 (color map 24-bit)" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -751,15 +751,15 @@ test "Write compressed indexed8 (color map 16-bit)" { - const image_file_name = "zigimg_tga_compressed_indexed8.tga"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ccm8.tga"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = true, - .color_map_depth = 16, -@@ -768,13 +768,13 @@ test "Write compressed indexed8 (color map 16-bit)" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -818,15 +818,15 @@ test "Write compressed indexed8 (color map 24-bit)" { - const image_file_name = "zigimg_tga_compressed_indexed8.tga"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ccm8.tga"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = true, - .color_map_depth = 24, -@@ -835,13 +835,13 @@ test "Write compressed indexed8 (color map 24-bit)" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -885,15 +885,15 @@ test "Write uncompressed 16-bit true color TGA" { - const image_file_name = "zigimg_tga_uncompressed_true_color_16.tga"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc16.tga"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = false, - .color_map_depth = 16, -@@ -902,13 +902,13 @@ test "Write uncompressed 16-bit true color TGA" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -946,15 +946,15 @@ test "Write compressed 16-bit true color TGA" { - const image_file_name = "zigimg_tga_compressed_true_color_16.tga"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc16.tga"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = true, - .color_map_depth = 16, -@@ -963,13 +963,13 @@ test "Write compressed 16-bit true color TGA" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -1007,15 +1007,15 @@ test "Write uncompressed 24-bit true color TGA" { - const image_file_name = "zigimg_tga_uncompressed_true_color_24.tga"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc24.tga"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = false, - .color_map_depth = 24, -@@ -1024,13 +1024,13 @@ test "Write uncompressed 24-bit true color TGA" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -1068,15 +1068,15 @@ test "Write compressed 24-bit true color TGA" { - const image_file_name = "zigimg_tga_compressed_true_color_24.tga"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ctc24.tga"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = true, - .color_map_depth = 24, -@@ -1085,13 +1085,13 @@ test "Write compressed 24-bit true color TGA" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -1129,15 +1129,15 @@ test "Write uncompressed 32-bit true color TGA" { - const image_file_name = "zigimg_tga_uncompressed_true_color_32.tga"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/utc32.tga"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = false, - .color_map_depth = 24, -@@ -1146,13 +1146,13 @@ test "Write uncompressed 32-bit true color TGA" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -1190,15 +1190,15 @@ test "Write compressed 32-bit true color TGA" { - const image_file_name = "zigimg_tga_compressed_true_color_32.tga"; - - var source_file = try helpers.testOpenFile(helpers.fixtures_path ++ "tga/ctc32.tga"); -- defer source_file.close(); -+ defer source_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; - -- var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, source_file, read_buffer[0..]); -+ var source_image = try zigimg.Image.fromFile(helpers.zigimg_test_allocator, std.testing.io, source_file, read_buffer[0..]); - defer source_image.deinit(helpers.zigimg_test_allocator); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = true, - .color_map_depth = 24, -@@ -1207,13 +1207,13 @@ test "Write compressed 32-bit true color TGA" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -1270,7 +1270,7 @@ test "Write uncompressed Rgb24 to TGA" { - @memcpy(source_image.pixels.rgb24[0..], uncompressed_source[0..]); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = false, - .color_map_depth = 24, -@@ -1279,14 +1279,14 @@ test "Write uncompressed Rgb24 to TGA" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -1329,7 +1329,7 @@ test "Write compressed Rgb24 to TGA" { - @memcpy(source_image.pixels.rgb24[0..], uncompressed_source[0..]); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = true, - .color_map_depth = 24, -@@ -1338,14 +1338,14 @@ test "Write compressed Rgb24 to TGA" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -1403,7 +1403,7 @@ test "Write uncompressed Rgba32 to TGA" { - @memcpy(source_image.pixels.rgba32[0..], uncompressed_source[0..]); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = false, - .color_map_depth = 24, -@@ -1412,14 +1412,14 @@ test "Write uncompressed Rgba32 to TGA" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -@@ -1478,7 +1478,7 @@ test "Write compressed Rgba32 to TGA" { - @memcpy(source_image.pixels.rgba32[0..], uncompressed_source[0..]); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- try source_image.writeToFilePath(helpers.zigimg_test_allocator, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ -+ try source_image.writeToFilePath(helpers.zigimg_test_allocator, std.testing.io, image_file_name, write_buffer[0..], zigimg.Image.EncoderOptions{ - .tga = .{ - .rle_compressed = true, - .color_map_depth = 24, -@@ -1487,14 +1487,14 @@ test "Write compressed Rgba32 to TGA" { - }, - }); - defer { -- std.fs.cwd().deleteFile(image_file_name) catch {}; -+ std.Io.Dir.deleteFile(std.Io.Dir.cwd(), std.testing.io, image_file_name) catch {}; - } - - const read_file = try helpers.testOpenFile(image_file_name); -- defer read_file.close(); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var tga_file = tga.TGA{}; - -diff --git a/tests/formats/tiff_test.zig b/tests/formats/tiff_test.zig -index cce4cfa..2a8a042 100644 ---- a/tests/formats/tiff_test.zig -+++ b/tests/formats/tiff_test.zig -@@ -5,10 +5,10 @@ const zigimg = @import("zigimg"); - - test "Should error on non TIFF images" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "bmp/simple_v4.bmp"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - var sgi_file = tiff.TIFF{}; - -@@ -18,12 +18,12 @@ test "Should error on non TIFF images" { - - test "TIFF/LE monochrome black uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-monob-raw.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -39,12 +39,12 @@ test "TIFF/LE monochrome black uncompressed" { - - test "TIFF/LE grayscale8 uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-grayscale8-raw.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -61,12 +61,12 @@ test "TIFF/LE grayscale8 uncompressed" { - - test "TIFF/LE 8-bit with colormap uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-pal8-raw.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -87,12 +87,12 @@ test "TIFF/LE 8-bit with colormap uncompressed" { - - test "TIFF/LE 24-bit uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgb24-raw.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -115,12 +115,12 @@ test "TIFF/LE 24-bit uncompressed" { - - test "TIFF/BE rgb24 gray single strip uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/big-endian/sample-rgb24-single-strip.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -143,12 +143,12 @@ test "TIFF/BE rgb24 gray single strip uncompressed" { - - test "TIFF/BE rgb24 color single strip uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/big-endian/sample-pal8-raw.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -171,12 +171,12 @@ test "TIFF/BE rgb24 color single strip uncompressed" { - - test "TIFF/BE 24-bit uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/big-endian/sample-rgb24-raw.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -199,12 +199,12 @@ test "TIFF/BE 24-bit uncompressed" { - - test "TIFF/LE RGBA uncompressed" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgba-raw.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -223,12 +223,12 @@ test "TIFF/LE RGBA uncompressed" { - - test "TIFF/LE monochrome black packbits" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-monob-packbits.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -244,12 +244,12 @@ test "TIFF/LE monochrome black packbits" { - - test "TIFF/LE grayscale8 packbits" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-grayscale8-packbits.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -266,12 +266,12 @@ test "TIFF/LE grayscale8 packbits" { - - test "TIFF/LE 8-bit with colormap packbits" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-pal8-packbits.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -292,12 +292,12 @@ test "TIFF/LE 8-bit with colormap packbits" { - - test "TIFF/LE 24-bit packbits" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgb24-packbits.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -320,12 +320,12 @@ test "TIFF/LE 24-bit packbits" { - - test "TIFF/LE RGBA packbits" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgba-packbits.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -344,12 +344,12 @@ test "TIFF/LE RGBA packbits" { - - test "TIFF/LE monochrome black CCITT" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/ccitt_rle.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -364,12 +364,12 @@ test "TIFF/LE monochrome black CCITT" { - - test "TIFF/LE monochrome black LZW" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-monob-lzw.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -385,12 +385,12 @@ test "TIFF/LE monochrome black LZW" { - - test "TIFF/LE grayscale8 LZW" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-grayscale8-lzw.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -407,12 +407,12 @@ test "TIFF/LE grayscale8 LZW" { - - test "TIFF/LE 8-bit with colormap LZW" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-pal8-lzw.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -433,12 +433,12 @@ test "TIFF/LE 8-bit with colormap LZW" { - - test "TIFF/LE 24-bit LZW" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgb24-lzw.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -461,12 +461,12 @@ test "TIFF/LE 24-bit LZW" { - - test "TIFF/LE RGBA LZW" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgba-lzw.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -485,12 +485,12 @@ test "TIFF/LE RGBA LZW" { - - test "TIFF/LE monochrome black Deflate" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-monob-deflate.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_bitmap = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_bitmap.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -506,12 +506,12 @@ test "TIFF/LE monochrome black Deflate" { - - test "TIFF/LE grayscale8 Deflate" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-grayscale8-deflate.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -528,12 +528,12 @@ test "TIFF/LE grayscale8 Deflate" { - - test "TIFF/LE 8-bit with colormap Deflate" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-pal8-deflate.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -554,12 +554,12 @@ test "TIFF/LE 8-bit with colormap Deflate" { - - test "TIFF/LE 24-bit Deflate" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgb24-deflate.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -@@ -582,12 +582,12 @@ test "TIFF/LE 24-bit Deflate" { - - test "TIFF/LE RGBA Deflate" { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "tiff/sample-rgba-deflate.tiff"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_tiff = tiff.TIFF{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const pixels = try the_tiff.read(helpers.zigimg_test_allocator, &read_stream); - defer pixels.deinit(helpers.zigimg_test_allocator); -diff --git a/tests/formats/xbm_test.zig b/tests/formats/xbm_test.zig -index 52eb29c..053e41e 100644 ---- a/tests/formats/xbm_test.zig -+++ b/tests/formats/xbm_test.zig -@@ -30,12 +30,12 @@ const simple_4x4_xbm = - test "XBM: invalid file format" { - { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "xbm/bad_missing_dim.xbm"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_xbm = xbm.XBM{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const actual_error = the_xbm.read(helpers.zigimg_test_allocator, &read_stream); - try helpers.expectError(actual_error, zigimg.Image.ReadError.InvalidData); -@@ -43,12 +43,12 @@ test "XBM: invalid file format" { - - { - const file = try helpers.testOpenFile(helpers.fixtures_path ++ "xbm/bad_missing_pixels.xbm"); -- defer file.close(); -+ defer file.close(std.testing.io); - - var the_xbm = xbm.XBM{}; - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, file, read_buffer[0..]); - - const actual_error = the_xbm.read(helpers.zigimg_test_allocator, &read_stream); - try helpers.expectError(actual_error, zigimg.Image.ReadError.InvalidData); -diff --git a/tests/helpers.zig b/tests/helpers.zig -index 9a4ef29..b9f06b1 100644 ---- a/tests/helpers.zig -+++ b/tests/helpers.zig -@@ -30,17 +30,17 @@ pub inline fn expectApproxEqRel(actual: anytype, expected: anytype, tolerance: a - return try std.testing.expectApproxEqRel(expected, actual, tolerance); - } - --pub fn testOpenFile(file_path: []const u8) !std.fs.File { -- return std.fs.cwd().openFile(file_path, .{}) catch |err| -+pub fn testOpenFile(file_path: []const u8) !std.Io.File { -+ return std.Io.Dir.openFile(std.Io.Dir.cwd(), std.testing.io, file_path, .{}) catch |err| - if (err == error.FileNotFound) return error.SkipZigTest else return err; - } - - pub fn testImageFromFile(image_path: []const u8, buffer: []u8) !zigimg.Image { -- return zigimg.Image.fromFilePath(zigimg_test_allocator, image_path, buffer) catch |err| -+ return zigimg.Image.fromFilePath(zigimg_test_allocator, std.testing.io, image_path, buffer) catch |err| - if (err == error.FileNotFound) return error.SkipZigTest else return err; - } - - pub fn testReadFile(file_path: []const u8, buffer: []u8) ![]u8 { -- return std.fs.cwd().readFile(file_path, buffer) catch |err| -+ return std.Io.Dir.readFile(std.Io.Dir.cwd(), std.testing.io, file_path, buffer) catch |err| - if (err == error.FileNotFound) return error.SkipZigTest else return err; - } -diff --git a/tests/image_test.zig b/tests/image_test.zig -index ccd2d11..529be41 100644 ---- a/tests/image_test.zig -+++ b/tests/image_test.zig -@@ -187,7 +187,7 @@ test "Should detect BMP properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .bmp); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -213,7 +213,7 @@ test "Should detect GIF properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .gif); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -231,7 +231,7 @@ test "Should detect PCX properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .pcx); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -247,7 +247,7 @@ test "Should detect PBM properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .pbm); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -265,7 +265,7 @@ test "Should detect PGM properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .pgm); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -281,7 +281,7 @@ test "Should detect PPM properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .ppm); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -297,7 +297,7 @@ test "Should detect PNG properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .png); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -321,7 +321,7 @@ test "Should detect TGA properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .tga); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -334,7 +334,7 @@ test "Should detect QOI properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .qoi); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -350,7 +350,7 @@ test "Should detect JPEG properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .jpeg); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -366,7 +366,7 @@ test "Should detect Farbfeld properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .farbfeld); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -381,7 +381,7 @@ test "Should detect IFF/PBM properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .iff); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -396,7 +396,7 @@ test "Should detect RAS properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .ras); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -411,7 +411,7 @@ test "Should detect SGI properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .sgi); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -426,7 +426,7 @@ test "Should detect TIFF properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .tiff); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -@@ -449,7 +449,7 @@ test "Should detect XBM properly" { - - for (image_tests) |image_path| { - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- const format = try Image.detectFormatFromFilePath(image_path, read_buffer[0..]); -+ const format = try Image.detectFormatFromFilePath(std.testing.io, image_path, read_buffer[0..]); - try std.testing.expect(format == .xbm); - - var test_image = try helpers.testImageFromFile(image_path, read_buffer[0..]); -diff --git a/tests/io_test.zig b/tests/io_test.zig -index 2dc6f7a..1ec5d60 100644 ---- a/tests/io_test.zig -+++ b/tests/io_test.zig -@@ -55,12 +55,12 @@ test "io.ReadStream: should read and seek properly within a unbuffered file" { - var temp_folder = std.testing.tmpDir(.{}); - defer temp_folder.cleanup(); - -- try temp_folder.dir.writeFile(.{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); -+ try temp_folder.dir.writeFile(std.testing.io, .{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); - -- const read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); -- defer read_file.close(); -+ const read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, &.{}); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, &.{}); - - var reader = read_stream.reader(); - -@@ -99,12 +99,12 @@ test "io.ReadStream: should error on invalid seek within unbuffered file" { - var temp_folder = std.testing.tmpDir(.{}); - defer temp_folder.cleanup(); - -- try temp_folder.dir.writeFile(.{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); -+ try temp_folder.dir.writeFile(std.testing.io, .{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); - -- const read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); -- defer read_file.close(); -+ const read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); -+ defer read_file.close(std.testing.io); - -- var read_stream = zigimg.io.ReadStream.initFile(read_file, &.{}); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, &.{}); - - const seek_to_error = read_stream.seekTo(123456); - try helpers.expectError(seek_to_error, zigimg.io.ReadStream.SeekError.Unseekable); -@@ -122,13 +122,13 @@ test "io.ReadStream: should read and seek properly within a file with a small bu - var temp_folder = std.testing.tmpDir(.{}); - defer temp_folder.cleanup(); - -- try temp_folder.dir.writeFile(.{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); -+ try temp_folder.dir.writeFile(std.testing.io, .{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); - -- const read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); -- defer read_file.close(); -+ const read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); -+ defer read_file.close(std.testing.io); - - var small_buffer: [30]u8 = @splat(0); -- var read_stream = zigimg.io.ReadStream.initFile(read_file, small_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, small_buffer[0..]); - - var reader = read_stream.reader(); - -@@ -164,13 +164,13 @@ test "io.ReadStream: should read and seek properly within a file with a large bu - var temp_folder = std.testing.tmpDir(.{}); - defer temp_folder.cleanup(); - -- try temp_folder.dir.writeFile(.{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); -+ try temp_folder.dir.writeFile(std.testing.io, .{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); - -- const read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); -- defer read_file.close(); -+ const read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - var reader = read_stream.reader(); - -@@ -206,13 +206,13 @@ test "io.ReadStream: should error on invalid seek within a file with a large buf - var temp_folder = std.testing.tmpDir(.{}); - defer temp_folder.cleanup(); - -- try temp_folder.dir.writeFile(.{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); -+ try temp_folder.dir.writeFile(std.testing.io, .{ .sub_path = TEST_FILENAME, .data = TEST_FILE_CONTENTS }); - -- const read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); -- defer read_file.close(); -+ const read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); -+ defer read_file.close(std.testing.io); - - var read_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var read_stream = zigimg.io.ReadStream.initFile(read_file, read_buffer[0..]); -+ var read_stream = zigimg.io.ReadStream.initFile(std.testing.io, read_file, read_buffer[0..]); - - const seek_to_error = read_stream.seekTo(123456); - try helpers.expectError(seek_to_error, zigimg.io.ReadStream.SeekError.Unseekable); -@@ -261,10 +261,10 @@ test "io.WriteStream: should write property within a file with a small buffer" { - defer temp_folder.cleanup(); - - { -- const write_file = try temp_folder.dir.createFile(TEST_FILENAME, .{}); -- defer write_file.close(); -+ const write_file = try temp_folder.dir.createFile(std.testing.io, TEST_FILENAME, .{}); -+ defer write_file.close(std.testing.io); - -- var write_stream = zigimg.io.WriteStream.initFile(write_file, buffer[0..]); -+ var write_stream = zigimg.io.WriteStream.initFile(std.testing.io, write_file, buffer[0..]); - - var writer = write_stream.writer(); - try writer.writeAll(TEST_FILE_CONTENTS[0..3]); -@@ -283,11 +283,11 @@ test "io.WriteStream: should write property within a file with a small buffer" { - try writer.flush(); - } - -- var read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); -- defer read_file.close(); -+ var read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); -+ defer read_file.close(std.testing.io); - - var read_buffer: [1024]u8 = @splat(0); -- var file_reader = read_file.reader(read_buffer[0..]); -+ var file_reader = read_file.reader(std.testing.io, read_buffer[0..]); - - var reader = &file_reader.interface; - -@@ -305,11 +305,11 @@ test "io.WriteStream: should write property within a file with a large buffer" { - defer temp_folder.cleanup(); - - { -- const write_file = try temp_folder.dir.createFile(TEST_FILENAME, .{}); -- defer write_file.close(); -+ const write_file = try temp_folder.dir.createFile(std.testing.io, TEST_FILENAME, .{}); -+ defer write_file.close(std.testing.io); - - var write_buffer: [zigimg.io.DEFAULT_BUFFER_SIZE]u8 = undefined; -- var write_stream = zigimg.io.WriteStream.initFile(write_file, write_buffer[0..]); -+ var write_stream = zigimg.io.WriteStream.initFile(std.testing.io, write_file, write_buffer[0..]); - - var writer = write_stream.writer(); - try writer.writeAll(TEST_FILE_CONTENTS[0..3]); -@@ -328,11 +328,11 @@ test "io.WriteStream: should write property within a file with a large buffer" { - try writer.flush(); - } - -- var read_file = try temp_folder.dir.openFile(TEST_FILENAME, .{}); -- defer read_file.close(); -+ var read_file = try temp_folder.dir.openFile(std.testing.io, TEST_FILENAME, .{}); -+ defer read_file.close(std.testing.io); - - var read_buffer: [1024]u8 = @splat(0); -- var file_reader = read_file.reader(read_buffer[0..]); -+ var file_reader = read_file.reader(std.testing.io, read_buffer[0..]); - - var reader = &file_reader.interface; - -diff --git a/zigimg.zig b/zigimg.zig -index 9f6b1a8..a53b23c 100644 ---- a/zigimg.zig -+++ b/zigimg.zig -@@ -38,5 +38,5 @@ test { - @import("tests/octree_quantizer_test.zig"), - @import("tests/pixel_format_converter_test.zig"), - @import("tests/pixel_format_test.zig"), -- }) |source_file| std.testing.refAllDeclsRecursive(source_file); -+ }) |source_file| std.testing.refAllDecls(source_file); - } -diff --git a/src/formats/gif.zig b/src/formats/gif.zig ---- a/src/formats/gif.zig -+++ b/src/formats/gif.zig -@@ -8,7 +8,7 @@ - const std = @import("std"); - const utils = @import("../utils.zig"); - --pub const HeaderFlags = packed struct { -+pub const HeaderFlags = packed struct(u8) { - global_color_table_size: u3 = 0, - sorted: bool = false, - color_resolution: u3 = 0, -@@ -129,9 +129,9 @@ - pub const GIF = struct { - header: Header = .{}, - global_color_table: utils.FixedStorage(color.Rgb24, 256) = .{}, -- frames: std.ArrayList(FrameData) = .{}, -- comments: std.ArrayList(CommentExtension) = .{}, -- application_infos: std.ArrayList(ApplicationExtension) = .{}, -+ frames: std.ArrayList(FrameData) = .empty, -+ comments: std.ArrayList(CommentExtension) = .empty, -+ application_infos: std.ArrayList(ApplicationExtension) = .empty, - arena_allocator: std.heap.ArenaAllocator = undefined, - - pub const SubImage = struct { -@@ -147,6 +147,6 @@ - pub const FrameData = struct { - graphics_control: ?GraphicControlExtension = null, -- sub_images: std.ArrayList(*SubImage) = .{}, -+ sub_images: std.ArrayList(*SubImage) = .empty, - - pub fn deinit(self: *FrameData, allocator: std.mem.Allocator) void { - for (self.sub_images.items) |sub_image| { -@@ -683,7 +683,7 @@ - const final_pixel_format = self.findBestPixelFormat(); - - const frame_list_allocator = self.arena_allocator.child_allocator; - -- var frame_list = Image.Animation.FrameList{}; -+ var frame_list: Image.Animation.FrameList = .empty; - - if (self.frames.items.len == 0) { -diff --git a/src/compressions/lzw.zig b/src/compressions/lzw.zig ---- a/src/compressions/lzw.zig -+++ b/src/compressions/lzw.zig -@@ -11,9 +11,10 @@ - next_code: u13 = 0, - previous_code: ?u13 = null, -- dictionary: std.AutoArrayHashMap(u13, []const u8), -+ dictionary: std.AutoArrayHashMapUnmanaged(u13, []const u8) = .empty, -+ dict_allocator: std.mem.Allocator, - - remaining_data: ?u13 = null, - remaining_bits: u4 = 0, -@@ -27,7 +28,7 @@ - var result = Self{ - .area_allocator = std.heap.ArenaAllocator.init(allocator), - .code_size = initial_code_size, -- .dictionary = std.AutoArrayHashMap(u13, []const u8).init(allocator), -+ .dict_allocator = allocator, - .initial_code_size = initial_code_size, - .clear_code = @as(u13, 1) << @intCast(initial_code_size), - .end_information_code = (@as(u13, 1) << @intCast(initial_code_size)) + 1, -@@ -43,7 +44,7 @@ - - pub fn deinit(self: *Self) void { - self.area_allocator.deinit(); -- self.dictionary.deinit(); -+ self.dictionary.deinit(self.dict_allocator); - } - - pub fn decode(self: *Self, reader: *std.Io.Reader, writer: *std.Io.Writer) !void { -@@ -85,7 +86,7 @@ - var new_value = try allocator.alloc(u8, previous_value.len + 1); - std.mem.copyForwards(u8, new_value, previous_value); - new_value[previous_value.len] = value[0]; -- try self.dictionary.put(self.next_code, new_value); -+ try self.dictionary.put(self.dict_allocator, self.next_code, new_value); - - self.next_code += 1; - -@@ -109,7 +110,7 @@ - var new_value = try allocator.alloc(u8, previous_value.len + 1); - std.mem.copyForwards(u8, new_value, previous_value); - new_value[previous_value.len] = previous_value[0]; -- try self.dictionary.put(self.next_code, new_value); -+ try self.dictionary.put(self.dict_allocator, self.next_code, new_value); - - _ = try writer.write(new_value); - -@@ -154,7 +155,7 @@ - var data = try allocator.alloc(u8, 1); - data[0] = @as(u8, @truncate(index)); - -- try self.dictionary.put(index, data); -+ try self.dictionary.put(self.dict_allocator, index, data); - } - } - }; -diff --git a/src/formats/jpeg/huffman.zig b/src/formats/jpeg/huffman.zig ---- a/src/formats/jpeg/huffman.zig -+++ b/src/formats/jpeg/huffman.zig -@@ -9,7 +9,7 @@ - - const HuffmanCode = struct { length_minus_one: u4, code: u16 }; --const HuffmanCodeMap = std.AutoArrayHashMap(HuffmanCode, u8); -+const HuffmanCodeMap = std.AutoArrayHashMapUnmanaged(HuffmanCode, u8); - - const JPEG_DEBUG = false; - const JPEG_VERY_DEBUG = false; -@@ -40,8 +40,8 @@ - for (code_counts) |count| total_huffman_codes += count; - -- var huffman_code_map = HuffmanCodeMap.init(allocator); -- errdefer huffman_code_map.deinit(); -+ var huffman_code_map: HuffmanCodeMap = .empty; -+ errdefer huffman_code_map.deinit(allocator); - - var fast_table: [1 << fast_bits]u8 = @splat(255); - var fast_size: [1 << fast_bits]u5 = @splat(0); -@@ -65,7 +65,7 @@ - - const byte = try reader.takeByte(); -- try huffman_code_map.put(.{ .length_minus_one = @as(u4, @intCast(i)), .code = code }, byte); -+ try huffman_code_map.put(allocator, .{ .length_minus_one = @as(u4, @intCast(i)), .code = code }, byte); - - // construct accelaration structure see stb_image - if (i + 1 <= fast_bits) { -@@ -96,5 +96,5 @@ - - pub fn deinit(self: *Table) void { -- self.code_map.deinit(); -+ self.code_map.deinit(self.allocator); - } - }; - diff --git a/third_party/zigimg/repo.bzl b/third_party/zigimg/repo.bzl index aa375cfc..a4bb178f 100644 --- a/third_party/zigimg/repo.bzl +++ b/third_party/zigimg/repo.bzl @@ -1,25 +1,10 @@ -def _zigimg_repo_impl(ctx): - ctx.download_and_extract( - url = "https://github.com/zigimg/zigimg/archive/{}.tar.gz".format(ctx.attr.commit), - stripPrefix = "zigimg-{}".format(ctx.attr.commit), - ) - - ctx.patch(ctx.attr.patch, strip = 1) - ctx.symlink(ctx.attr.build_file, "BUILD.bazel") - -_zigimg_repo = repository_rule( - implementation = _zigimg_repo_impl, - attrs = { - "commit": attr.string(mandatory = True), - "patch": attr.label(mandatory = True, allow_single_file = True), - "build_file": attr.label(mandatory = True, allow_single_file = True), - }, -) +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") def repo(): - _zigimg_repo( + git_repository( name = "zigimg", - commit = "7b98e82621fe302a9edc147df1191f4d1b7ff7a5", - patch = "//third_party/zigimg:bump-zig.patch", - build_file = "//third_party/zigimg:zigimg.bazel", + remote = "https://github.com/zigimg/zigimg.git", + # Match libvaxis build.zig.zon. + commit = "734fa919fb15b7917d4eb124f53600348f4ffa30", + build_file = Label("//third_party/zigimg:zigimg.bazel"), ) diff --git a/third_party/zio/repo.bzl b/third_party/zio/repo.bzl index 16b9e277..96133ef0 100644 --- a/third_party/zio/repo.bzl +++ b/third_party/zio/repo.bzl @@ -4,6 +4,6 @@ def repo(): git_repository( name = "zio", remote = "https://github.com/lalinsky/zio", - commit = "281d969fdefe04f2e8b28d1f45fac0011f881c73", # v0.19.0 + commit = "1b4e9787f8a4a67d91372a603b23258364f89bf0", build_file = Label("//third_party/zio:zio.bazel"), ) diff --git a/third_party/zls/BUILD.bazel b/third_party/zls/BUILD.bazel new file mode 100644 index 00000000..964fe87b --- /dev/null +++ b/third_party/zls/BUILD.bazel @@ -0,0 +1,36 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") +load("@rules_zig//zig/zls:toolchain.bzl", "zls_toolchain") + +exports_files(glob(["*.bazel"])) + +zig_library( + name = "build_options", + main = "build_options.zig", + visibility = ["@zls//:__pkg__"], +) + +zig_library( + name = "exe_options", + main = "exe_options.zig", + visibility = ["@zls//:__pkg__"], +) + +zig_library( + name = "tracy_options", + import_name = "options", + main = "tracy_options.zig", + visibility = ["@zls//:__pkg__"], +) + +zls_toolchain( + name = "zls", + bin = "@zls//:zls", + zls_version = "0.17.0-dev+eab2be0fd744", +) + +toolchain( + name = "toolchain", + target_settings = ["@zig_toolchains//:0.17.0"], + toolchain = ":zls", + toolchain_type = "@rules_zig//zig/zls:toolchain_type", +) diff --git a/third_party/zls/build_options.zig b/third_party/zls/build_options.zig new file mode 100644 index 00000000..b50cb2d8 --- /dev/null +++ b/third_party/zls/build_options.zig @@ -0,0 +1,5 @@ +const std = @import("std"); + +pub const version_string = "0.17.0-dev+eab2be0fd744"; +pub const version = std.SemanticVersion.parse(version_string) catch unreachable; +pub const minimum_runtime_zig_version_string = "0.17.0"; diff --git a/third_party/zls/diffz.bazel b/third_party/zls/diffz.bazel new file mode 100644 index 00000000..fb4b85fa --- /dev/null +++ b/third_party/zls/diffz.bazel @@ -0,0 +1,8 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "diffz", + srcs = glob(["**/*.zig"]), + main = "DiffMatchPatch.zig", + visibility = ["@zls//:__pkg__"], +) diff --git a/third_party/zls/exe_options.zig b/third_party/zls/exe_options.zig new file mode 100644 index 00000000..eaad61b5 --- /dev/null +++ b/third_party/zls/exe_options.zig @@ -0,0 +1,3 @@ +pub const enable_failing_allocator = false; +pub const enable_failing_allocator_likelihood: u32 = 256; +pub const debug_gpa = false; diff --git a/third_party/zls/known_folders.bazel b/third_party/zls/known_folders.bazel new file mode 100644 index 00000000..234db4b0 --- /dev/null +++ b/third_party/zls/known_folders.bazel @@ -0,0 +1,8 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "known-folders", + srcs = glob(["**/*.zig"]), + main = "known-folders.zig", + visibility = ["@zls//:__pkg__"], +) diff --git a/third_party/zls/lsp.bazel b/third_party/zls/lsp.bazel new file mode 100644 index 00000000..b7c6b6dd --- /dev/null +++ b/third_party/zls/lsp.bazel @@ -0,0 +1,51 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") + +# @embedFile resolves a module's root file without parsing it as Zig. +genrule( + name = "meta_model_source", + srcs = ["metaModel.json"], + outs = ["meta_model.zig"], + cmd = "cp $(<) $(@)", +) + +zig_library( + name = "meta-model", + main = ":meta_model.zig", +) + +zig_binary( + name = "codegen", + srcs = glob(["src/codegen/*.zig"]), + extra_srcs = ["src/codegen/config.zon"], + main = "src/codegen/codegen.zig", + deps = [":meta-model"], +) + +genrule( + name = "generate_types", + outs = ["lsp_types.zig"], + cmd = "$(location :codegen) $(@)", + tools = [":codegen"], +) + +zig_library( + name = "parser", + main = "src/parser.zig", +) + +zig_library( + name = "types", + main = ":lsp_types.zig", + deps = [":parser"], +) + +zig_library( + name = "lsp", + srcs = glob(["src/*.zig"]), + main = "src/lsp.zig", + visibility = ["@zls//:__pkg__"], + deps = [ + ":parser", + ":types", + ], +) diff --git a/third_party/zls/repo.bzl b/third_party/zls/repo.bzl new file mode 100644 index 00000000..27f05c5c --- /dev/null +++ b/third_party/zls/repo.bzl @@ -0,0 +1,28 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + # ZLS 0.17 development revision; dependencies match its build.zig.zon. + git_repository( + name = "zls", + remote = "https://github.com/zigtools/zls.git", + commit = "eab2be0fd74443a27662da809b771c4ad0d2afcf", + build_file = Label("//third_party/zls:zls.bazel"), + ) + git_repository( + name = "zls_lsp", + remote = "https://github.com/zigtools/lsp-kit.git", + commit = "d148676ace1eaecf3c703ce7d350d0e94e502ed7", + build_file = Label("//third_party/zls:lsp.bazel"), + ) + git_repository( + name = "zls_diffz", + remote = "https://github.com/ziglibs/diffz.git", + commit = "aac8aa99c436ab8277b0711922aad062c0167b12", + build_file = Label("//third_party/zls:diffz.bazel"), + ) + git_repository( + name = "zls_known_folders", + remote = "https://github.com/ziglibs/known-folders.git", + commit = "6da0d0c41b78b9ed2d34fa364fcb81b5ebece6c4", + build_file = Label("//third_party/zls:known_folders.bazel"), + ) diff --git a/third_party/zls/tracy_options.zig b/third_party/zls/tracy_options.zig new file mode 100644 index 00000000..01dfcb1d --- /dev/null +++ b/third_party/zls/tracy_options.zig @@ -0,0 +1,3 @@ +pub const enable = false; +pub const enable_allocation = false; +pub const enable_callstack = false; diff --git a/third_party/zls/zls.bazel b/third_party/zls/zls.bazel new file mode 100644 index 00000000..56ccbff8 --- /dev/null +++ b/third_party/zls/zls.bazel @@ -0,0 +1,53 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") + +zig_binary( + name = "config_gen", + extra_srcs = ["src/tools/config.json"], + main = "src/tools/config_gen.zig", +) + +genrule( + name = "generate_version_data", + srcs = ["src/tools/langref.html.in"], + outs = ["version_data.zig"], + cmd = "$(location :config_gen) --langref-version master --langref-path $(<) --generate-version-data $(@)", + tools = [":config_gen"], +) + +zig_library( + name = "version_data", + main = ":version_data.zig", +) + +zig_library( + name = "tracy", + main = "src/tracy.zig", + deps = ["@zml//third_party/zls:tracy_options"], +) + +zig_library( + name = "zls_lib", + srcs = glob(["src/**/*.zig"]), + import_name = "zls", + main = "src/zls.zig", + deps = [ + ":tracy", + ":version_data", + "@zls_diffz//:diffz", + "@zls_lsp//:lsp", + "@zml//third_party/zls:build_options", + ], +) + +zig_binary( + name = "zls", + srcs = glob(["src/**/*.zig"]), + main = "src/main.zig", + visibility = ["//visibility:public"], + deps = [ + ":tracy", + ":zls_lib", + "@zls_known_folders//:known-folders", + "@zml//third_party/zls:exe_options", + ], +) diff --git a/vfs/BUILD.bazel b/vfs/BUILD.bazel index 0ebc39e1..c8de1e0c 100644 --- a/vfs/BUILD.bazel +++ b/vfs/BUILD.bazel @@ -1,8 +1,7 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "vfs", - main = "vfs.zig", srcs = [ "base.zig", "file.zig", @@ -12,9 +11,15 @@ zig_library( "parallel_read.zig", "s3.zig", ], + main = "vfs.zig", tags = ["manual"], visibility = ["//visibility:public"], deps = [ "//stdx", ], ) + +zig_test( + name = "test", + deps = [":vfs"], +) diff --git a/vfs/base.zig b/vfs/base.zig index 33d79509..1e66ca89 100644 --- a/vfs/base.zig +++ b/vfs/base.zig @@ -93,12 +93,12 @@ pub const VFSBase = struct { .processSetCurrentDir = processSetCurrentDir, .processSetCurrentPath = processSetCurrentPath, .processReplace = processReplace, - .processReplacePath = processReplacePath, .processSpawn = processSpawn, - .processSpawnPath = processSpawnPath, .childWait = childWait, .childKill = childKill, .progressParentFile = progressParentFile, + .inheritParentDir = inheritParentDir, + .inheritParentFile = inheritParentFile, .now = now, .clockResolution = clockResolution, .sleep = sleep, @@ -111,9 +111,6 @@ pub const VFSBase = struct { .netListenUnix = netListenUnix, .netConnectUnix = netConnectUnix, .netSocketCreatePair = netSocketCreatePair, - .netSend = netSend, - .netRead = netRead, - .netWrite = netWrite, .netWriteFile = netWriteFile, .netClose = netClose, .netShutdown = netShutdown, @@ -261,7 +258,7 @@ pub const VFSBase = struct { return self.inner.vtable.dirAccess(self.inner.userdata, dir, sub_path, options); } - pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.CreateFlags) std.Io.File.OpenError!std.Io.File { + pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.CreateFileOptions) std.Io.File.OpenError!std.Io.File { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirCreateFile(self.inner.userdata, dir, sub_path, flags); } @@ -271,7 +268,7 @@ pub const VFSBase = struct { return self.inner.vtable.dirCreateFileAtomic(self.inner.userdata, dir, sub_path, options); } - pub fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + pub fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirOpenFile(self.inner.userdata, dir, sub_path, flags); } @@ -496,7 +493,7 @@ pub const VFSBase = struct { return self.inner.vtable.fileMemoryMapWrite(self.inner.userdata, memory_map); } - pub fn processExecutableOpen(userdata: ?*anyopaque, flags: std.Io.File.OpenFlags) std.process.OpenExecutableError!std.Io.File { + pub fn processExecutableOpen(userdata: ?*anyopaque, flags: std.Io.Dir.OpenFileOptions) std.process.OpenExecutableError!std.Io.File { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.processExecutableOpen(self.inner.userdata, flags); } @@ -541,21 +538,11 @@ pub const VFSBase = struct { return self.inner.vtable.processReplace(self.inner.userdata, options); } - pub fn processReplacePath(userdata: ?*anyopaque, dir: std.Io.Dir, options: std.process.ReplaceOptions) std.process.ReplaceError { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.processReplacePath(self.inner.userdata, dir, options); - } - pub fn processSpawn(userdata: ?*anyopaque, options: std.process.SpawnOptions) std.process.SpawnError!std.process.Child { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.processSpawn(self.inner.userdata, options); } - pub fn processSpawnPath(userdata: ?*anyopaque, dir: std.Io.Dir, options: std.process.SpawnOptions) std.process.SpawnError!std.process.Child { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.processSpawnPath(self.inner.userdata, dir, options); - } - pub fn childWait(userdata: ?*anyopaque, child: *std.process.Child) std.process.Child.WaitError!std.process.Child.Term { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.childWait(self.inner.userdata, child); @@ -571,6 +558,16 @@ pub const VFSBase = struct { return self.inner.vtable.progressParentFile(self.inner.userdata); } + pub fn inheritParentDir(userdata: ?*anyopaque, handle: std.Io.Dir.Handle) std.Io.InheritParentHandleError!std.Io.Dir { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.inheritParentDir(self.inner.userdata, handle); + } + + pub fn inheritParentFile(userdata: ?*anyopaque, handle: std.Io.File.Handle, flags: std.Io.File.Flags) std.Io.InheritParentHandleError!std.Io.File { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.inheritParentFile(self.inner.userdata, handle, flags); + } + pub fn now(userdata: ?*anyopaque, clock: std.Io.Clock) std.Io.Timestamp { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.now(self.inner.userdata, clock); @@ -631,29 +628,14 @@ pub const VFSBase = struct { return self.inner.vtable.netSocketCreatePair(self.inner.userdata, options); } - pub fn netSend(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, msgs: []std.Io.net.OutgoingMessage, flags: std.Io.net.SendFlags) struct { ?std.Io.net.Socket.SendError, usize } { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.netSend(self.inner.userdata, handle, msgs, flags); - } - - pub fn netRead(userdata: ?*anyopaque, src: std.Io.net.Socket.Handle, data: [][]u8) std.Io.net.Stream.Reader.Error!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.netRead(self.inner.userdata, src, data); - } - - pub fn netWrite(userdata: ?*anyopaque, dest: std.Io.net.Socket.Handle, header: []const u8, data: []const []const u8, splat: usize) std.Io.net.Stream.Writer.Error!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.netWrite(self.inner.userdata, dest, header, data, splat); - } - pub fn netWriteFile(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.net.Stream.Writer.WriteFileError!usize { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netWriteFile(self.inner.userdata, handle, header, reader, limit); } - pub fn netClose(userdata: ?*anyopaque, handles: []const std.Io.net.Socket.Handle) void { + pub fn netClose(userdata: ?*anyopaque, sockets: []const std.Io.net.Socket) void { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - self.inner.vtable.netClose(self.inner.userdata, handles); + self.inner.vtable.netClose(self.inner.userdata, sockets); } pub fn netShutdown(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, how: std.Io.net.ShutdownHow) std.Io.net.ShutdownError!void { diff --git a/vfs/file.zig b/vfs/file.zig index 627f79d9..1b077c33 100644 --- a/vfs/file.zig +++ b/vfs/file.zig @@ -21,9 +21,30 @@ fn canUseDirectIO() bool { return false; } +fn getStatusFlags(file: std.Io.File) DirectIoError!c_int { + while (true) { + const result: c_int = std.posix.system.fcntl(file.handle, std.posix.F.GETFL, @as(c_int, 0)); + switch (std.posix.errno(result)) { + .SUCCESS => return result, + .INTR => continue, + else => return DirectIoError.UnexpectedFcntlResult, + } + } +} + +fn setStatusFlags(file: std.Io.File, flags: c_int) DirectIoError!void { + while (true) { + switch (std.posix.errno(std.posix.system.fcntl(file.handle, std.posix.F.SETFL, flags))) { + .SUCCESS => return, + .INTR => continue, + else => return DirectIoError.UnexpectedFcntlResult, + } + } +} + fn useDirectIO(file: std.Io.File) DirectIoError!bool { if (comptime canUseDirectIO()) { - const flags = std.posix.system.fcntl(file.handle, std.posix.F.GETFL); + const flags = try getStatusFlags(file); const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); return (flags & direct_flag) != 0; } else { @@ -33,12 +54,11 @@ fn useDirectIO(file: std.Io.File) DirectIoError!bool { fn switchToBufferedIO(file: std.fs.File) DirectIoError!void { if (comptime canUseDirectIO()) { - const flags = std.posix.system.fcntl(file.handle, std.posix.F.GETFL); + const flags = try getStatusFlags(file); const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); if ((flags & direct_flag) == 0) return; - const result = try std.posix.fcntl(file.handle, std.posix.F.SETFL, flags & ~@as(c_uint, @bitCast(@as(u32, @intCast(direct_flag))))); - if (result != 0) return DirectIoError.UnexpectedFcntlResult; + try setStatusFlags(file, flags & ~direct_flag); } else { return DirectIoError.UnsupportedPlatform; } @@ -46,11 +66,9 @@ fn switchToBufferedIO(file: std.fs.File) DirectIoError!void { fn switchToDirectIO(file: std.Io.File) DirectIoError!void { if (builtin.os.tag == .linux and canUseDirectIO()) { - const flags = std.posix.system.fcntl(file.handle, std.posix.F.GETFL); + const flags = try getStatusFlags(file); const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); - - const result = std.posix.system.fcntl(file.handle, std.posix.F.SETFL, flags | direct_flag); - if (result != 0) return DirectIoError.UnexpectedFcntlResult; + try setStatusFlags(file, flags | direct_flag); } else { return DirectIoError.UnsupportedPlatform; } @@ -111,6 +129,7 @@ pub const File = struct { .dirStat = dirStat, .dirStatFile = dirStatFile, .dirAccess = dirAccess, + .dirCreateFile = dirCreateFile, .dirOpenFile = dirOpenFile, .dirClose = dirClose, .dirRead = dirRead, @@ -119,6 +138,9 @@ pub const File = struct { .fileStat = fileStat, .fileLength = fileLength, .fileClose = fileClose, + .fileWritePositional = fileWritePositional, + .fileWriteFileStreaming = fileWriteFileStreaming, + .fileWriteFilePositional = fileWriteFilePositional, .fileReadPositional = fileReadPositional, .fileSeekBy = fileSeekBy, .fileSeekTo = fileSeekTo, @@ -151,7 +173,7 @@ pub const File = struct { return self.handles.at(@intCast(file.handle)); } - fn canOpenWithDirectIO(self: *File, file: std.Io.File, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!bool { + fn canOpenWithDirectIO(self: *File, file: std.Io.File, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!bool { const file_stat = self.base.inner.vtable.fileStat(self.base.inner.userdata, file) catch |err| switch (err) { else => return std.Io.File.OpenError.Unexpected, }; @@ -201,7 +223,18 @@ pub const File = struct { }, }); }, - .file_write_streaming, .device_io_control, .net_receive => { + .file_write_streaming => |o| { + const handle = self.getFileHandle(o.file); + return self.base.inner.vtable.operate(self.base.inner.userdata, .{ + .file_write_streaming = .{ + .file = handle.innerFile(), + .header = o.header, + .data = o.data, + .splat = o.splat, + }, + }); + }, + .device_io_control, .net_receive, .net_read, .net_send, .net_write => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -227,7 +260,21 @@ pub const File = struct { return self.base.inner.vtable.dirAccess(self.base.inner.userdata, dir, sub_path, options); } - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.CreateFileOptions) std.Io.File.OpenError!std.Io.File { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const inner_file = try self.base.inner.vtable.dirCreateFile(self.base.inner.userdata, dir, sub_path, flags); + errdefer inner_file.close(self.base.inner); + + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; + handle.* = .{ + .inner_handle = inner_file.handle, + .inner_flags = inner_file.flags, + .direct_io_handle = null, + }; + return .{ .handle = @intCast(idx), .flags = inner_file.flags }; + } + + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); const inner_file = try self.base.inner.vtable.dirOpenFile(self.base.inner.userdata, dir, sub_path, flags); errdefer inner_file.close(self.base.inner); @@ -311,6 +358,52 @@ pub const File = struct { } } + fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) std.Io.File.WritePositionalError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return self.base.inner.vtable.fileWritePositional(self.base.inner.userdata, handle.innerFile(), header, data, splat, offset); + } + + fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const own_io = self.io(); + if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; + + const destination = self.getFileHandle(file); + const source = self.getFileHandle(reader.file); + + const original_source_io = reader.io; + const original_source_file = reader.file; + reader.io = self.base.inner; + reader.file = source.innerFile(); + defer { + reader.io = original_source_io; + reader.file = original_source_file; + } + + return self.base.inner.vtable.fileWriteFileStreaming(self.base.inner.userdata, destination.innerFile(), header, reader, limit); + } + + fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const own_io = self.io(); + if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; + + const destination = self.getFileHandle(file); + const source = self.getFileHandle(reader.file); + + const original_source_io = reader.io; + const original_source_file = reader.file; + reader.io = self.base.inner; + reader.file = source.innerFile(); + defer { + reader.io = original_source_io; + reader.file = original_source_file; + } + + return self.base.inner.vtable.fileWriteFilePositional(self.base.inner.userdata, destination.innerFile(), header, reader, limit, offset); + } + fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); const inner_file = self.innerFile(file, data, offset); diff --git a/vfs/gcs.zig b/vfs/gcs.zig index 649e9958..acc7f634 100644 --- a/vfs/gcs.zig +++ b/vfs/gcs.zig @@ -652,7 +652,7 @@ pub const GCS = struct { handle.pos += @intCast(total); return .{ .file_read_streaming = total }; }, - .file_write_streaming, .device_io_control, .net_receive => { + .file_write_streaming, .device_io_control, .net_receive, .net_read, .net_send, .net_write => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -709,7 +709,7 @@ pub const GCS = struct { fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { error.FileNotFound => return std.Io.File.OpenError.FileNotFound, @@ -875,7 +875,7 @@ pub const GCS = struct { return try std.fmt.bufPrint(buf, "{d:0>4}{d:0>2}{d:0>2}T{d:0>2}{d:0>2}{d:0>2}Z", .{ year_day.year, - @intFromEnum(month_day.month), + @backingInt(month_day.month), month_day.day_index + 1, day_seconds.getHoursIntoDay(), day_seconds.getMinutesIntoHour(), diff --git a/vfs/hf.zig b/vfs/hf.zig index 2a1e9249..8770fe18 100644 --- a/vfs/hf.zig +++ b/vfs/hf.zig @@ -652,7 +652,7 @@ pub const HF = struct { handle.pos += @intCast(total); return .{ .file_read_streaming = total }; }, - .file_write_streaming, .device_io_control, .net_receive => { + .file_write_streaming, .device_io_control, .net_receive, .net_read, .net_send, .net_write => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -732,7 +732,7 @@ pub const HF = struct { if (node == null) return std.Io.Dir.AccessError.FileNotFound; } - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); var path_buffer: [8 * 1024]u8 = undefined; diff --git a/vfs/http.zig b/vfs/http.zig index ed8c5ee4..9054983e 100644 --- a/vfs/http.zig +++ b/vfs/http.zig @@ -156,7 +156,7 @@ pub const HTTP = struct { handle.pos += @intCast(total); return .{ .file_read_streaming = total }; }, - .file_write_streaming, .device_io_control, .net_receive => { + .file_write_streaming, .device_io_control, .net_receive, .net_read, .net_send, .net_write => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -210,7 +210,7 @@ pub const HTTP = struct { fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); const size = self.fetchSize(dir, sub_path) catch return std.Io.File.OpenError.Unexpected; diff --git a/vfs/s3.zig b/vfs/s3.zig index 51adcc7c..eb849e21 100644 --- a/vfs/s3.zig +++ b/vfs/s3.zig @@ -544,7 +544,7 @@ pub const S3 = struct { handle.pos += @intCast(total); return .{ .file_read_streaming = total }; }, - .file_write_streaming, .device_io_control, .net_receive => { + .file_write_streaming, .device_io_control, .net_receive, .net_read, .net_send, .net_write => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -601,7 +601,7 @@ pub const S3 = struct { fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { error.FileNotFound => return std.Io.File.OpenError.FileNotFound, @@ -767,7 +767,7 @@ pub const S3 = struct { return try std.fmt.bufPrint(buf, "{d:0>4}{d:0>2}{d:0>2}T{d:0>2}{d:0>2}{d:0>2}Z", .{ year_day.year, - @intFromEnum(month_day.month), + @backingInt(month_day.month), month_day.day_index + 1, day_seconds.getHoursIntoDay(), day_seconds.getMinutesIntoHour(), diff --git a/vfs/vfs.zig b/vfs/vfs.zig index b5e5c580..fe626ff9 100644 --- a/vfs/vfs.zig +++ b/vfs/vfs.zig @@ -187,7 +187,16 @@ fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable .data = o.data, } }); }, - .device_io_control, .file_write_streaming, .net_receive => { + .file_write_streaming => |o| { + const handle, const backend = self.getFileHandle(o.file); + return backend.vtable.operate(backend.userdata, .{ .file_write_streaming = .{ + .file = .{ .handle = @intCast(handle.handle), .flags = handle.flags }, + .header = o.header, + .data = o.data, + .splat = o.splat, + } }); + }, + .device_io_control, .net_receive, .net_read, .net_send, .net_write => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -235,7 +244,7 @@ pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, o return backend.vtable.dirAccess(backend.userdata, dir_, stripScheme(sub_path), options); } -pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.CreateFlags) std.Io.File.OpenError!std.Io.File { +pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.CreateFileOptions) std.Io.File.OpenError!std.Io.File { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { log.err("Failed to lookup backend for dir create file '{s}' : {any}", .{ sub_path, err }); @@ -243,25 +252,29 @@ pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u }; const file = try backend.vtable.dirCreateFile(backend.userdata, dir_, stripScheme(sub_path), flags); + errdefer file.close(backend); const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; handle.* = .{ .handle = @intCast(file.handle), .backend_idx = backend_idx, + .flags = file.flags, }; - return .{ .handle = @intCast(idx), .flags = .{ .nonblocking = false } }; + return .{ .handle = @intCast(idx), .flags = file.flags }; } -fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { +fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { log.err("Failed to lookup backend for opening file '{s}' : {any}", .{ sub_path, err }); return std.Io.File.OpenError.Unexpected; }; const file = try backend.vtable.dirOpenFile(backend.userdata, dir_, stripScheme(sub_path), flags); + errdefer file.close(backend); const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; handle.* = .{ .handle = @intCast(file.handle), .backend_idx = backend_idx, + .flags = file.flags, }; return .{ .handle = @intCast(idx), .flags = handle.flags }; } @@ -353,26 +366,42 @@ fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const own_io = self.io(); + if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; const dst_handle, const dst_backend = self.getFileHandle(file); - const src_handle, _ = self.getFileHandle(reader.file); + const src_handle, const src_backend = self.getFileHandle(reader.file); + if (src_backend.userdata != dst_backend.userdata or src_backend.vtable != dst_backend.vtable) return error.Unimplemented; + const original_src_io = reader.io; const original_src_handle = reader.file; + reader.io = src_backend; reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; - defer reader.file = original_src_handle; + defer { + reader.io = original_src_io; + reader.file = original_src_handle; + } return dst_backend.vtable.fileWriteFileStreaming(dst_backend.userdata, .{ .handle = @intCast(dst_handle.handle), .flags = dst_handle.flags }, header, reader, limit); } fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const own_io = self.io(); + if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; const dst_handle, const dst_backend = self.getFileHandle(file); - const src_handle, _ = self.getFileHandle(reader.file); + const src_handle, const src_backend = self.getFileHandle(reader.file); + if (src_backend.userdata != dst_backend.userdata or src_backend.vtable != dst_backend.vtable) return error.Unimplemented; + const original_src_io = reader.io; const original_src_handle = reader.file; + reader.io = src_backend; reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; - defer reader.file = original_src_handle; + defer { + reader.io = original_src_io; + reader.file = original_src_handle; + } return dst_backend.vtable.fileWriteFilePositional(dst_backend.userdata, .{ .handle = @intCast(dst_handle.handle), .flags = dst_handle.flags }, header, reader, limit, offset); } @@ -407,3 +436,48 @@ fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std. return try backend.vtable.fileRealPath(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, out_buffer); } } + +test "copy a local file through the VFS" { + const allocator = std.testing.allocator; + const process_io = std.testing.io; + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + const source_data: [8192]u8 = @splat('x'); + try tmp.dir.writeFile(process_io, .{ + .sub_path = "source", + .data = &source_data, + }); + + var path_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path_len = try tmp.dir.realPath(process_io, &path_buffer); + const source_uri = try std.fmt.allocPrint(allocator, "file://{s}/source", .{path_buffer[0..path_len]}); + defer allocator.free(source_uri); + const destination_uri = try std.fmt.allocPrint(allocator, "file://{s}/destination", .{path_buffer[0..path_len]}); + defer allocator.free(destination_uri); + + var file_backend: File = .init(allocator, process_io, .{}); + defer file_backend.deinit(); + var vfs: VFS = try .init(allocator, process_io); + defer vfs.deinit(); + try vfs.register("file", file_backend.io()); + + const vfs_io = vfs.io(); + { + var source = try std.Io.Dir.openFile(.cwd(), vfs_io, source_uri, .{}); + defer source.close(vfs_io); + var destination = try std.Io.Dir.createFile(.cwd(), vfs_io, destination_uri, .{}); + defer destination.close(vfs_io); + + var reader: std.Io.File.Reader = .initStreaming(source, vfs_io, &.{}); + var copy_buffer: [4096]u8 align(4096) = undefined; + var writer = destination.writer(vfs_io, ©_buffer); + try std.testing.expectEqual(source_data.len, try reader.interface.streamRemaining(&writer.interface)); + try writer.interface.flush(); + } + + const copied = try tmp.dir.readFileAlloc(process_io, "destination", allocator, .limited(source_data.len + 1)); + defer allocator.free(copied); + try std.testing.expectEqualSlices(u8, &source_data, copied); +} diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 200f2265..15685740 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -164,7 +164,7 @@ pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, var mlir_known_types: std.enums.EnumArray(DataType, *const mlir.Type) = .initUndefined(); { for (0.., &mlir_known_types.values) |i, *mlir_type| { - mlir_type.* = mlirx.Type.fromDType(mlir_ctx, @enumFromInt(i)); + mlir_type.* = mlirx.Type.fromDType(mlir_ctx, @fromBackingInt(@intCast(i))); } } @@ -316,7 +316,7 @@ pub fn mlirType(self: *const Compiler, dt: DataType) *const mlir.Type { pub fn dtype(self: *const Compiler, mlir_type: *const mlir.Type) DataType { @setRuntimeSafety(false); for (0.., &self.mlir_known_types.values) |i, known_type| { - if (known_type == mlir_type) return @enumFromInt(i); + if (known_type == mlir_type) return @fromBackingInt(@intCast(i)); } std.debug.panic("Can't convert unknown mlir type to dtype: {f}", .{mlir_type}); } diff --git a/zml/Sharding.zig b/zml/Sharding.zig index 5ce487ee..792dff69 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -135,21 +135,21 @@ pub const PartitionSpec = enum(u4) { open = 15, pub fn sharded(mesh_axis: u3) PartitionSpec { - return @enumFromInt(mesh_axis); + return @fromBackingInt(@intCast(mesh_axis)); } /// Extract the mesh axis along which we are sharded. Null if not sharded. pub fn meshAxis(self: PartitionSpec) ?u3 { - const ax = @intFromEnum(self); - return if (ax < @intFromEnum(PartitionSpec.replicated)) @intCast(ax) else null; + const ax = @backingInt(self); + return if (ax < @backingInt(PartitionSpec.replicated)) @intCast(ax) else null; } pub fn isSharded(self: PartitionSpec) bool { - return @intFromEnum(self) < @intFromEnum(PartitionSpec.replicated); + return @backingInt(self) < @backingInt(PartitionSpec.replicated); } pub fn isClosed(self: PartitionSpec) bool { - return @intFromEnum(self) <= @intFromEnum(PartitionSpec.replicated); + return @backingInt(self) <= @backingInt(PartitionSpec.replicated); } test isClosed { @@ -285,13 +285,13 @@ pub const Partitioning = packed struct { } pub fn splat(spec: PartitionSpec) Partitioning { - const vec: Vec = @splat(@intFromEnum(spec)); + const vec: Vec = @splat(@backingInt(spec)); return @bitCast(vec); } pub fn repeat(spec: PartitionSpec, rank_: usize) Partitioning { std.debug.assert(rank_ <= MAX_RANK); - const splatted: Vec = @splat(@intFromEnum(spec)); + const splatted: Vec = @splat(@backingInt(spec)); const mask = std.simd.iota(u4, 8) < @as(Vec, @splat(@truncate(rank_))); return @bitCast(@select(u4, mask, splatted, @as(Vec, @bitCast(out_of_bound)))); } @@ -300,7 +300,7 @@ pub const Partitioning = packed struct { std.debug.assert(ax < MAX_RANK); const pack: u32 = @bitCast(p); const shift: u5 = @intCast(4 * ax); - return @enumFromInt(@as(u4, @truncate(pack >> shift))); + return @fromBackingInt(@truncate(pack >> shift)); } pub fn set(p: Partitioning, ax: usize, spec: PartitionSpec) Partitioning { @@ -308,7 +308,7 @@ pub const Partitioning = packed struct { const pack: u32 = @bitCast(p); const shift: u5 = @intCast(4 * ax); const mask = @as(u32, 0xf) << shift; - return @bitCast((pack & ~mask) | (@as(u32, @intFromEnum(spec)) << shift)); + return @bitCast((pack & ~mask) | (@as(u32, @backingInt(spec)) << shift)); } pub fn rank(p: Partitioning) u8 { @@ -325,7 +325,7 @@ pub const Partitioning = packed struct { const pack: u32 = @bitCast(p); const shift: u5 = @intCast(4 * ax); const lower_mask = (@as(u32, 1) << shift) - 1; - return @bitCast((pack & lower_mask) | ((pack & ~lower_mask) << 4) | (@as(u32, @intFromEnum(spec)) << shift)); + return @bitCast((pack & lower_mask) | ((pack & ~lower_mask) << 4) | (@as(u32, @backingInt(spec)) << shift)); } /// Removes a spec, shifting subsequent slots left and filling the last with unknown. @@ -334,7 +334,7 @@ pub const Partitioning = packed struct { const pack: u32 = @bitCast(p); const shift: u5 = @intCast(4 * ax); const lower_mask = (@as(u32, 1) << shift) - 1; - return @bitCast((pack & lower_mask) | ((pack >> 4) & ~lower_mask) | (@as(u32, @intFromEnum(PartitionSpec.out_of_bound)) << 28)); + return @bitCast((pack & lower_mask) | ((pack >> 4) & ~lower_mask) | (@as(u32, @backingInt(PartitionSpec.out_of_bound)) << 28)); } pub fn toArray(p: Partitioning) [MAX_RANK]PartitionSpec { @@ -344,7 +344,7 @@ pub const Partitioning = packed struct { } pub fn hasUniqueAxes(p: Partitioning) bool { - var used_mesh_axes: std.StaticBitSet(MAX_RANK) = .initEmpty(); + var used_mesh_axes: std.StaticBitSet(MAX_RANK) = .empty; for (0..MAX_RANK) |shape_ax| { const spec = p.get(shape_ax); if (spec.meshAxis()) |mesh_axis| { @@ -383,7 +383,7 @@ pub const Partitioning = packed struct { test "insertion and removal at every slot" { var parts: Partitioning = .out_of_bound; - for (0..MAX_RANK) |ax| parts = parts.set(ax, @enumFromInt(ax)); + for (0..MAX_RANK) |ax| parts = parts.set(ax, @fromBackingInt(@intCast(ax))); const original = parts.toArray(); for (0..MAX_RANK) |ax| { var inserted = original; @@ -414,13 +414,13 @@ pub const Partitioning = packed struct { const T = @TypeOf(partitioning); stdx.debug.assertComptime(stdx.meta.isStruct(T), "parsePartitioning expected a struct of enum literals eg {{ .b = .data, .d = .model }}, got: {any}", .{T}); - inline for (std.meta.fields(T)) |field| { - const shape_tag = Shape.toTag(field); + inline for (comptime std.meta.fieldNames(T)) |field_name| { + const shape_tag = Shape.toTag(field_name); const shape_ax = Shape.axisFromTagMaybe(tags, shape_tag) orelse { - std.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), field.name }); + std.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), field_name }); }; - const value = @field(partitioning, field.name); + const value = @field(partitioning, field_name); const spec: PartitionSpec = if (@TypeOf(value) == PartitionSpec) value else switch (value) { .replicated => .replicated, .out_of_bound => std.debug.panic("value out_of_bound not allowed", .{}), @@ -603,11 +603,11 @@ pub const PhysicalMesh = struct { try writer.writeAll("], depth_by_tag={"); - const fields = std.meta.fields(PhysicalAxisTag); - inline for (fields, 0..) |field, i| { + const field_names = comptime std.meta.fieldNames(PhysicalAxisTag); + inline for (field_names, 0..) |field_name, i| { if (i > 0) try writer.writeAll(", "); - const tag = @field(PhysicalAxisTag, field.name); - try writer.writeAll(field.name); + const tag = @field(PhysicalAxisTag, field_name); + try writer.writeAll(field_name); try writer.writeAll("="); if (self.depth_by_tag.get(tag)) |d| { try writer.print("{d}", .{d}); @@ -1349,9 +1349,9 @@ pub const LogicalMesh = struct { var axes: Axes = .empty; var intents: Intents = .empty; - inline for (std.meta.fields(T)) |field| { - const value = @field(axes_, field.name); - axes.appendAssumeCapacity(Shape.toTag(field)); + inline for (comptime std.meta.fieldNames(T)) |field_name| { + const value = @field(axes_, field_name); + axes.appendAssumeCapacity(Shape.toTag(field_name)); intents.appendAssumeCapacity(intentFromValue(value)); } @@ -1922,15 +1922,15 @@ pub const Strategy = struct { pub fn parseBindings(bindings: anytype) Strategy { const err_msg = "Strategy.parseBindings excepts fields to be PhysicalAxisTag or tuple of PhysicalAxisTag, got {}"; var res: Strategy = .{ .bindings = .empty, .folding = .empty }; - const fields = @typeInfo(@TypeOf(bindings)).@"struct".fields; - if (fields.len == 0) @compileError("Strategy.parseBindings requires at least one binding"); - inline for (fields) |field_info| { - switch (@typeInfo(field_info.type)) { - .enum_literal, .@"enum" => res.addBinding(field_info, @field(bindings, field_info.name)), + const info = @typeInfo(@TypeOf(bindings)).@"struct"; + if (info.field_names.len == 0) @compileError("Strategy.parseBindings requires at least one binding"); + inline for (info.field_names, info.field_types) |field_name, Field| { + switch (@typeInfo(Field)) { + .enum_literal, .@"enum" => res.addBinding(field_name, @field(bindings, field_name)), .@"struct" => |struct_info| { stdx.debug.assertComptime(struct_info.is_tuple, err_msg, .{@TypeOf(bindings)}); - inline for (@field(bindings, field_info.name)) |axis_tag| { - res.addBinding(field_info, axis_tag); + inline for (@field(bindings, field_name)) |axis_tag| { + res.addBinding(field_name, axis_tag); } }, else => stdx.debug.compileError(err_msg, .{@TypeOf(bindings)}), @@ -2042,13 +2042,13 @@ pub const Placement = struct { // Note this shape is mainly use to indicate the dims of the sharded buffer. // We may want to only store that. shape: Shape, - global_shape: if (builtin.mode == .Debug) Shape else void, + global_shape: if (builtin.mode == .debug) Shape else void, axis_plans: stdx.BoundedArray(AxisSplit, Shape.MAX_RANK), pub fn init(shape: Shape) error{IncompatibleSharding}!Placement { var pl: Placement = .{ .shape = shape, // modified below - .global_shape = if (builtin.mode == .Debug) shape else {}, + .global_shape = if (builtin.mode == .debug) shape else {}, .axis_plans = .empty, // set below }; // mesh == null => replicated => no axis plan @@ -2088,7 +2088,7 @@ pub const Placement = struct { } pub fn shardPtr(pl: *const Placement, device: Device.Coords, slice: Slice) [*]const u8 { - if (builtin.mode == .Debug) { + if (builtin.mode == .debug) { // there is a bug in caller code that used a placement for a different shape std.debug.assert(pl.global_shape.eql(slice.shape)); } @@ -2276,10 +2276,10 @@ const ShardingTest = struct { /// Creates a 1D-3D PhysicalMesh from simple dimensions. pub fn physical(self: ShardingTest, dims: anytype, geometry: AxisGeometry) !PhysicalMesh { const info = @typeInfo(@TypeOf(dims)).@"struct"; - const N = info.fields.len; + const N = info.field_names.len; var sizes: [N]usize = undefined; - inline for (info.fields, sizes[0..]) |field, *s| { - s.* = @intCast(@field(dims, field.name)); + inline for (info.field_names, sizes[0..]) |field_name, *s| { + s.* = @intCast(@field(dims, field_name)); } const tags: [3]PhysicalAxisTag = .{ .link_x, .link_y, .link_z }; diff --git a/zml/buffer/HostAccessible.zig b/zml/buffer/HostAccessible.zig index 212c460d..76dab0ba 100644 --- a/zml/buffer/HostAccessible.zig +++ b/zml/buffer/HostAccessible.zig @@ -1264,7 +1264,7 @@ test "HostAccessible.HostPinnedAllocations keeps each region's replicas together // replicas arrive interleaved with other regions. const shard_shape: Shape = .init(.{4}, .u8); for ([_]usize{ 0, 4, 0, 8, 4 }, 0..) |row, i| { - allocations.add(shard_shape, .{ .origin = .{ .axes = .{row} ++ .{0} ** (Shape.MAX_RANK - 1) }, .ptr = base + i, .shard_index = undefined, .handle = undefined }); + allocations.add(shard_shape, .{ .origin = .{ .axes = .{row} ++ @as([Shape.MAX_RANK - 1]usize, @splat(0)) }, .ptr = base + i, .shard_index = undefined, .handle = undefined }); } try std.testing.expectEqualSlices([*]u8, &.{ base, base + 2, base + 1, base + 4, base + 3 }, allocations.entries.items(.ptr)); diff --git a/zml/dtype.zig b/zml/dtype.zig index c56fdd5b..f8d24089 100644 --- a/zml/dtype.zig +++ b/zml/dtype.zig @@ -1,5 +1,7 @@ const std = @import("std"); +const stdx = @import("stdx"); + const floats = @import("floats.zig"); const C64 = std.math.Complex(f32); @@ -139,8 +141,9 @@ pub const DataType = enum(u8) { pub fn toPackedZigType(comptime dtype: DataType) type { return switch (dtype) { .bool => bool, - .i2, .i4, .u2, .u4 => |dt| @Vector(8 / dt.bitSizeOf(), @FieldType(Value, @tagName(dtype))), + .i2, .i4, .u2, .u4 => |dt| stdx.Packed(@FieldType(Value, @tagName(dt))), .f4e2m1 => floats.Float4E2M1.Packed, + inline .c64, .c128 => |dt| @FieldType(Value, @tagName(dt)), else => { const T = @FieldType(Value, @tagName(dtype)); if (@bitSizeOf(T) < 8) @compileLog("Forgot type !", T); @@ -165,6 +168,8 @@ pub const DataType = enum(u8) { pub fn bitSizeOf(self: DataType) u16 { return switch (self) { .bool => 8, // In PJRT/stablehlo bool always occupy a full byte + .c64 => 64, + .c128 => 128, inline else => |tag| @bitSizeOf(tag.toZigType()), }; } diff --git a/zml/exe.zig b/zml/exe.zig index 2acc0b61..1dd1a2f2 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -396,7 +396,7 @@ pub fn FnExe(comptime function_: anytype) type { .@"fn" => |info| info, else => @compileError("FnExe expects a function, got " ++ @typeName(@TypeOf(function_))), }; - if (function_info.is_var_args or function_info.params.len != 1) { + if (function_info.attrs.varargs or function_info.param_types.len != 1) { @compileError("FnExe function must accept exactly one input struct"); } @@ -446,7 +446,7 @@ pub fn FnExe(comptime function_: anytype) type { const RunnerSelf = @This(); pub const BakedInput = structFieldRange(Input, 0, count); - pub const NonBakedInput = structFieldRange(Input, count, @typeInfo(Input).@"struct".fields.len); + pub const NonBakedInput = structFieldRange(Input, count, @typeInfo(Input).@"struct".field_names.len); pub fn init(exe: *const Self, allocator: std.mem.Allocator, baked: BakedInput) !RunnerSelf { var arguments = try exe.raw.args(allocator); @@ -488,42 +488,30 @@ fn countBackedFields(comptime Input: type, comptime baked_fields: anytype) usize @compileError("FnExe baked fields must be a tuple of enum literals"); } - const input_fields = @typeInfo(Input).@"struct".fields; - if (baked_info.fields.len > input_fields.len) { + const input_info = @typeInfo(Input).@"struct"; + if (baked_info.field_names.len > input_info.field_names.len) { @compileError("FnExe baked fields must be a prefix of its bufferized input fields"); } - inline for (baked_info.fields, 0..) |tuple_field, i| { - const baked_field = @field(baked_fields, tuple_field.name); + inline for (baked_info.field_names, 0..) |tuple_field_name, i| { + const baked_field = @field(baked_fields, tuple_field_name); if (@TypeOf(baked_field) != @EnumLiteral()) { @compileError("FnExe baked fields must be enum literals"); } - if (!std.mem.eql(u8, @tagName(baked_field), input_fields[i].name)) { + if (!std.mem.eql(u8, @tagName(baked_field), input_info.field_names[i])) { @compileError("FnExe baked fields must be an ordered prefix of its bufferized input fields"); } } - return baked_info.fields.len; + return baked_info.field_names.len; } fn structFieldRange(comptime Struct: type, comptime start: usize, comptime end: usize) type { - const fields = @typeInfo(Struct).@"struct".fields[start..end]; - var field_names: [fields.len][]const u8 = undefined; - var field_types: [fields.len]type = undefined; - var field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; - for (&field_names, &field_types, &field_attrs, fields) |*name, *T, *attrs, field| { - name.* = field.name; - T.* = field.type; - attrs.* = .{ - .@"comptime" = field.is_comptime, - .@"align" = field.alignment, - .default_value_ptr = field.default_value_ptr, - }; - } - return @Struct(.auto, null, &field_names, &field_types, &field_attrs); + const info = @typeInfo(Struct).@"struct"; + return @Struct(.auto, null, info.field_names[start..end], info.field_types[start..end], info.field_attrs[start..end]); } fn typedFunctionInput(comptime function_: anytype) type { - const Input = @typeInfo(@TypeOf(function_)).@"fn".params[0].type orelse + const Input = @typeInfo(@TypeOf(function_)).@"fn".param_types[0] orelse @compileError("FnExe function must have a concrete input type"); validateFnExeStruct("input", Input); return Input; @@ -547,16 +535,13 @@ fn validateFnExeStruct(comptime role: []const u8, comptime T: type) void { fn typedOutputDestinations(comptime BufferizedOutput: type) type { const output_info = @typeInfo(BufferizedOutput).@"struct"; - const fields = output_info.fields; - var field_names: [fields.len][]const u8 = undefined; - var field_types: [fields.len]type = undefined; - var field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; - for (&field_names, &field_types, &field_attrs, fields) |*name, *T, *attrs, field| { - name.* = field.name; - T.* = *field.type; - attrs.* = .{ .@"align" = @alignOf(*field.type) }; + var field_types: [output_info.field_names.len]type = undefined; + var field_attrs: [output_info.field_names.len]std.builtin.Type.Struct.FieldAttributes = undefined; + for (&field_types, &field_attrs, output_info.field_types) |*T, *attrs, Field| { + T.* = *Field; + attrs.* = .{ .@"align" = @alignOf(*Field) }; } - return @Struct(.auto, null, &field_names, &field_types, &field_attrs); + return @Struct(.auto, null, output_info.field_names, &field_types, &field_attrs); } test "FnExe derives baked and runtime calls from a function" { diff --git a/zml/floats.zig b/zml/floats.zig index 7bf08824..689c22a8 100644 --- a/zml/floats.zig +++ b/zml/floats.zig @@ -8,8 +8,8 @@ fn FloatConversionHelpers(Float: type) type { @compileError(err_msg); } comptime { - for (info.@"struct".fields, &.{ "mantissa", "exponent", "sign" }) |field, expected_name| { - if (!std.mem.eql(u8, field.name, expected_name)) + for (info.@"struct".field_names, &.{ "mantissa", "exponent", "sign" }) |field_name, expected_name| { + if (!std.mem.eql(u8, field_name, expected_name)) @compileError(err_msg); } } @@ -20,7 +20,7 @@ fn FloatConversionHelpers(Float: type) type { const mantissa_bits: u8 = @typeInfo(@FieldType(Float, "mantissa")).int.bits; const exponent_bits: u8 = @typeInfo(@FieldType(Float, "exponent")).int.bits; const f32_mantissa_bits: u8 = @typeInfo(@FieldType(Float32, "mantissa")).int.bits; - const exp_bias: i16 = std.math.maxInt(std.meta.Int(.unsigned, exponent_bits - 1)); + const exp_bias: i16 = std.math.maxInt(@Int(.unsigned, exponent_bits - 1)); const exp_off: u8 = FloatConversionHelpers(Float32).exp_bias - exp_bias; /// Lossy conversion from f32, similar to @floatCast @@ -106,8 +106,8 @@ fn FloatHelpers(Float: type) type { @compileError(err_msg); } comptime { - for (info.@"struct".fields, &.{ "mantissa", "exponent", "sign" }) |field, expected_name| { - if (!std.mem.eql(u8, field.name, expected_name)) + for (info.@"struct".field_names, &.{ "mantissa", "exponent", "sign" }) |field_name, expected_name| { + if (!std.mem.eql(u8, field_name, expected_name)) @compileError(err_msg); } } @@ -517,7 +517,7 @@ pub const Float4E2M1 = packed struct(u4) { pub fn formatNumber(xy: Packed, w: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { try xy.x.formatNumber(w, n); - w.writeByte(','); + try w.writeByte(','); try xy.y.formatNumber(w, n); } }; @@ -544,7 +544,7 @@ pub fn isInf(x: anytype) bool { } if (!@hasDecl(Float, "inf")) return false; - const FBits = std.meta.Int(.unsigned, @bitSizeOf(Float)); + const FBits = @Int(.unsigned, @bitSizeOf(Float)); const remove_sign = ~@as(FBits, 0) >> 1; return @as(FBits, @bitCast(x)) & remove_sign == @as(FBits, @bitCast(Float.inf)); } diff --git a/zml/kernel.zig b/zml/kernel.zig index 33c685cc..204f224d 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -50,10 +50,10 @@ fn resolveOutputOperandAliases( var out: stdx.BoundedArray(dialects.stablehlo.CustomCallOpts.OutputOperandAlias, dialects.stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".fields, 0..) |field, output_index| { - if (@field(a, field.name)) |operand| out.appendAssumeCapacity(.{ + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@intFromEnum(operand))) + operand_offset, + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, }); } } @@ -832,7 +832,7 @@ pub const fly = struct { /// `zeroed_args` index of an output. pub fn zeroedOutput(comptime f: std.meta.FieldEnum(Outputs)) i32 { - return @intCast(spec.inputs.len + @intFromEnum(f)); + return @intCast(spec.inputs.len + @backingInt(f)); } /// `shapes` is inputs, then outputs. diff --git a/zml/kernels/flash_attention/cute/attention.zig b/zml/kernels/flash_attention/cute/attention.zig index 016dee40..396537fe 100644 --- a/zml/kernels/flash_attention/cute/attention.zig +++ b/zml/kernels/flash_attention/cute/attention.zig @@ -119,8 +119,8 @@ fn run(b: *B, cfg: Config) cute.FinishError!void { unreachable; } -fn tupleOf(array: anytype) std.meta.Tuple(&([_]type{Value} ** array.len)) { - var result: std.meta.Tuple(&([_]type{Value} ** array.len)) = undefined; +fn tupleOf(array: anytype) @Tuple(&@as([array.len]type, @splat(Value))) { + var result: @Tuple(&@as([array.len]type, @splat(Value))) = undefined; inline for (0..array.len) |i| result[i] = array[i]; return result; } diff --git a/zml/mem.zig b/zml/mem.zig index 4dbce05f..58067e07 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -341,8 +341,8 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu const type_info = @typeInfo(ModelBufferized); switch (type_info) { .@"struct" => |struct_type_info| { - inline for (struct_type_info.fields) |field| { - try bufferizeInner(allocator, @field(model, field.name), &@field(bufferized_, field.name)); + inline for (struct_type_info.field_names) |field_name| { + try bufferizeInner(allocator, @field(model, field_name), &@field(bufferized_, field_name)); } }, .@"union" => { @@ -364,7 +364,7 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu .pointer => |p| { switch (p.size) { .slice => { - bufferized_.* = try allocator.alignedAlloc(p.child, .fromByteUnits(p.alignment orelse @alignOf(p.child)), model.len); + bufferized_.* = try allocator.alignedAlloc(p.child, .fromByteUnits(p.attrs.@"align" orelse @alignOf(p.child)), model.len); for (model, bufferized_.*) |src, *dst| { try bufferizeInner(allocator, src, dst); } diff --git a/zml/meta.zig b/zml/meta.zig index 08738e11..c9a43f25 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -20,27 +20,30 @@ pub fn MapType(From: type, To: type) type { return switch (@typeInfo(T)) { .@"struct" => |struct_infos| { - const fields = struct_infos.fields; + const field_names = struct_infos.field_names; var same: bool = true; - var struct_field_names: [fields.len][]const u8 = undefined; - var struct_field_types: [fields.len]type = undefined; - var struct_field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; - for (struct_field_names[0..], struct_field_types[0..], struct_field_attrs[0..], fields) |*field_name, *field_type, *field_attr, field| { - if (!field.is_comptime) { - const R = map(field.type); - if (R == field.type) { - field_name.* = field.name; - field_type.* = field.type; - field_attr.* = .{ - .@"comptime" = field.is_comptime, - .@"align" = field.alignment, - .default_value_ptr = field.default_value_ptr, - }; + var struct_field_names: [field_names.len][]const u8 = undefined; + var struct_field_types: [field_names.len]type = undefined; + var struct_field_attrs: [field_names.len]std.builtin.Type.Struct.FieldAttributes = undefined; + for ( + struct_field_names[0..], + struct_field_types[0..], + struct_field_attrs[0..], + struct_infos.field_names, + struct_infos.field_types, + struct_infos.field_attrs, + ) |*field_name, *field_type, *field_attr, source_name, SourceField, source_attrs| { + if (!source_attrs.@"comptime") { + const R = map(SourceField); + if (R == SourceField) { + field_name.* = source_name; + field_type.* = SourceField; + field_attr.* = source_attrs; } else { - field_name.* = field.name; + field_name.* = source_name; field_type.* = R; field_attr.* = .{ - .@"comptime" = field.is_comptime, + .@"comptime" = source_attrs.@"comptime", .@"align" = @alignOf(R), }; same = false; @@ -52,34 +55,35 @@ pub fn MapType(From: type, To: type) type { } } } else { - field_name.* = field.name; - field_type.* = field.type; - field_attr.* = .{ - .@"comptime" = field.is_comptime, - .@"align" = field.alignment, - .default_value_ptr = field.default_value_ptr, - }; + field_name.* = source_name; + field_type.* = SourceField; + field_attr.* = source_attrs; } } if (same) return T; return if (struct_infos.is_tuple) @Tuple(&struct_field_types) else @Struct(.auto, null, &struct_field_names, &struct_field_types, &struct_field_attrs); }, .@"union" => |union_info| { - const fields = union_info.fields; + const field_names = union_info.field_names; var same: bool = true; - var union_field_names: [fields.len][]const u8 = undefined; - var union_field_types: [fields.len]type = undefined; - var union_field_attrs: [fields.len]std.builtin.Type.UnionField.Attributes = undefined; - for (union_field_names[0..], union_field_types[0..], union_field_attrs[0..], fields) |*field_name, *field_type, *field_attr, field| { - const R = map(field.type); - if (R == field.type) { - field_name.* = field.name; - field_type.* = field.type; - field_attr.* = .{ - .@"align" = field.alignment, - }; + var union_field_names: [field_names.len][]const u8 = undefined; + var union_field_types: [field_names.len]type = undefined; + var union_field_attrs: [field_names.len]std.builtin.Type.Union.FieldAttributes = undefined; + for ( + union_field_names[0..], + union_field_types[0..], + union_field_attrs[0..], + union_info.field_names, + union_info.field_types, + union_info.field_attrs, + ) |*field_name, *field_type, *field_attr, source_name, SourceField, source_attrs| { + const R = map(SourceField); + if (R == SourceField) { + field_name.* = source_name; + field_type.* = SourceField; + field_attr.* = source_attrs; } else { - field_name.* = field.name; + field_name.* = source_name; field_type.* = R; field_attr.* = .{ .@"align" = @alignOf(R), @@ -92,11 +96,11 @@ pub fn MapType(From: type, To: type) type { }, .array => |arr_info| [arr_info.len]map(arr_info.child), .pointer => |ptr_info| switch (ptr_info.size) { - .slice => if (ptr_info.is_const) + .slice => if (ptr_info.attrs.@"const") []const map(ptr_info.child) else []map(ptr_info.child), - .one => if (ptr_info.is_const) + .one => if (ptr_info.attrs.@"const") *const map(ptr_info.child) else *map(ptr_info.child), @@ -179,26 +183,26 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam if (@sizeOf(ToStruct) == 0) return; switch (type_info_to) { - .@"struct" => |info| inline for (info.fields) |field| { - if (field.is_comptime or @sizeOf(field.type) == 0) continue; - const field_type_info = @typeInfo(field.type); + .@"struct" => |info| inline for (info.field_names, info.field_types, info.field_attrs) |field_name, Field, field_attrs| { + if (field_attrs.@"comptime" or @sizeOf(Field) == 0) continue; + const field_type_info = @typeInfo(Field); // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. switch (field_type_info) { // .pointer => try convertType(From, To, allocator, @field(from, field.name), @field(to, field.name), Ctx, ctx, cb), - .array, .optional, .@"union", .@"struct", .pointer => if (@hasField(FromStruct, field.name)) { + .array, .optional, .@"union", .@"struct", .pointer => if (@hasField(FromStruct, field_name)) { try mapAlloc( cb, allocator, ctx, - @field(from, field.name), - &@field(to, field.name), + @field(from, field_name), + &@field(to, field_name), ); - } else if (field.default_value_ptr) |_| { - @field(to, field.name) = null; + } else if (field_attrs.default_value_ptr) |_| { + @field(to, field_name) = null; } else { - stdx.debug.compileError("Mapping {} -> {} inside {} failed. Missing field {s} in {}", .{ From, To, FromStruct, field.name, ToStruct }); + stdx.debug.compileError("Mapping {} -> {} inside {} failed. Missing field {s} in {}", .{ From, To, FromStruct, field_name, ToStruct }); }, - else => @field(to, field.name) = @field(from, field.name), + else => @field(to, field_name) = @field(from, field_name), } }, .@"union" => { @@ -332,25 +336,21 @@ pub fn MapRestrict(From: type, To: type) type { .@"struct" => |struct_infos| { // We know that at least one of the struct field contains a From. // We map each field individually. Fields without From and comptime fields are removed. - const fields = struct_infos.fields; + const field_names = struct_infos.field_names; var num_fields: usize = 0; - var struct_field_names: [fields.len][]const u8 = undefined; - var struct_field_types: [fields.len]type = undefined; - var struct_field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; - for (fields) |field| { - if (!field.is_comptime and Contains(field.type, From)) { - const R = map(field.type); - if (R == field.type) { - struct_field_names[num_fields] = field.name; - struct_field_types[num_fields] = field.type; - struct_field_attrs[num_fields] = .{ - .@"comptime" = field.is_comptime, - .@"align" = field.alignment, - .default_value_ptr = field.default_value_ptr, - }; + var struct_field_names: [field_names.len][]const u8 = undefined; + var struct_field_types: [field_names.len]type = undefined; + var struct_field_attrs: [field_names.len]std.builtin.Type.Struct.FieldAttributes = undefined; + for (struct_infos.field_names, struct_infos.field_types, struct_infos.field_attrs) |field_name, Field, field_attrs| { + if (!field_attrs.@"comptime" and Contains(Field, From)) { + const R = map(Field); + if (R == Field) { + struct_field_names[num_fields] = field_name; + struct_field_types[num_fields] = Field; + struct_field_attrs[num_fields] = field_attrs; } else { - const name = if (struct_infos.is_tuple) struct_infos.fields[num_fields].name else field.name; + const name = if (struct_infos.is_tuple) struct_infos.field_names[num_fields] else field_name; struct_field_names[num_fields] = name; struct_field_types[num_fields] = R; struct_field_attrs[num_fields] = .{ @@ -377,13 +377,13 @@ pub fn MapRestrict(From: type, To: type) type { .@"union" => |union_info| { // We know that at least one of the union field contains a From. // We map each field individually. Fields without From, are replaced by "void". - const fields = union_info.fields; - var union_field_names: [fields.len][]const u8 = undefined; - var union_field_types: [fields.len]type = undefined; - var union_field_attrs: [fields.len]std.builtin.Type.UnionField.Attributes = undefined; - for (0.., fields) |i, field| { - const FT = map(field.type); - union_field_names[i] = field.name; + const field_names = union_info.field_names; + var union_field_names: [field_names.len][]const u8 = undefined; + var union_field_types: [field_names.len]type = undefined; + var union_field_attrs: [field_names.len]std.builtin.Type.Union.FieldAttributes = undefined; + for (0.., union_info.field_names, union_info.field_types) |i, field_name, Field| { + const FT = map(Field); + union_field_names[i] = field_name; union_field_types[i] = FT; union_field_attrs[i] = .{ .@"align" = @alignOf(FT), @@ -393,19 +393,19 @@ pub fn MapRestrict(From: type, To: type) type { }, .array => |arr_info| [arr_info.len]map(arr_info.child), .pointer => |ptr_info| switch (ptr_info.size) { - .slice => if (ptr_info.is_const) + .slice => if (ptr_info.attrs.@"const") []const map(ptr_info.child) else []map(ptr_info.child), - .one => if (ptr_info.is_const) + .one => if (ptr_info.attrs.@"const") *const map(ptr_info.child) else *map(ptr_info.child), - .many => if (ptr_info.is_const) + .many => if (ptr_info.attrs.@"const") [*]const map(ptr_info.child) else [*]map(ptr_info.child), - .c => if (ptr_info.is_const) + .c => if (ptr_info.attrs.@"const") [*c]map(ptr_info.child) else [*c]map(ptr_info.child), @@ -499,7 +499,7 @@ pub fn visit(comptime callback: anytype, ctx: FnParam(callback, 0), v: anytype) .callback => callback(ctx, v), .recurse => { const TargetType, const mutating_cb = switch (@typeInfo(FnParam(callback, 1))) { - .pointer => |info| .{ info.child, !info.is_const }, + .pointer => |info| .{ info.child, !info.attrs.@"const" }, else => stdx.debug.compileError("zml.meta.visit({any}) is expecting a callback with a pointer as second argument but found {any}", .{ @TypeOf(callback), FnParam(callback, 1) }), }; @@ -511,12 +511,12 @@ pub fn visit(comptime callback: anytype, ctx: FnParam(callback, 0), v: anytype) switch (ptr_info_v.size) { .one => switch (@typeInfo(ChildTypeV)) { - .@"struct" => |s| inline for (s.fields) |field| { - if (field.is_comptime or comptime !Contains(field.type, TargetType)) continue; - if (@typeInfo(field.type) == .pointer) { - if (can_error) try visit(callback, ctx, @field(v, field.name)) else visit(callback, ctx, @field(v, field.name)); + .@"struct" => |s| inline for (s.field_names, s.field_types, s.field_attrs) |field_name, Field, field_attrs| { + if (field_attrs.@"comptime" or comptime !Contains(Field, TargetType)) continue; + if (@typeInfo(Field) == .pointer) { + if (can_error) try visit(callback, ctx, @field(v, field_name)) else visit(callback, ctx, @field(v, field_name)); } else { - if (can_error) try visit(callback, ctx, &@field(v, field.name)) else visit(callback, ctx, &@field(v, field.name)); + if (can_error) try visit(callback, ctx, &@field(v, field_name)) else visit(callback, ctx, &@field(v, field_name)); } }, .array => for (v[0..]) |*elem| { @@ -654,11 +654,11 @@ pub fn visitFlatStruct(comptime callback: anytype, ctx: FnParam(callback, 0), v: return switch (ptr_info_v.size) { .one => switch (@typeInfo(ChildTypeV)) { - .@"struct" => |s| inline for (s.fields) |field| { - if (field.is_comptime or comptime !Contains(field.type, TargetType)) continue; - stdx.debug.assertComptime(@typeInfo(field.type) != .pointer, err_msg, err_args); + .@"struct" => |s| inline for (s.field_names, s.field_types, s.field_attrs) |field_name, Field, field_attrs| { + if (field_attrs.@"comptime" or comptime !Contains(Field, TargetType)) continue; + stdx.debug.assertComptime(@typeInfo(Field) != .pointer, err_msg, err_args); - if (can_error) try visitFlatStruct(callback, ctx, &@field(v, field.name)) else visitFlatStruct(callback, ctx, &@field(v, field.name)); + if (can_error) try visitFlatStruct(callback, ctx, &@field(v, field_name)) else visitFlatStruct(callback, ctx, &@field(v, field_name)); }, .array => for (v[0..]) |*elem| { if (can_error) try visitFlatStruct(callback, ctx, elem) else visitFlatStruct(callback, ctx, elem); @@ -786,17 +786,17 @@ pub fn zip(comptime func: anytype, allocator: std.mem.Allocator, values: anytype .pointer => stdx.debug.compileError("zip only accept by value arguments. Received: {}", .{V}), .@"struct" => |struct_info| { var out: V = values[0]; - inline for (struct_info.fields) |f| { - if (f.is_comptime) continue; - if (@typeInfo(f.type) == .pointer) { + inline for (struct_info.field_names, struct_info.field_types, struct_info.field_attrs) |field_name, Field, field_attrs| { + if (field_attrs.@"comptime") continue; + if (@typeInfo(Field) == .pointer) { stdx.debug.compileError("zip doesn't follow pointers and don't accept struct containing them. Received: {}", .{V}); } - var fields = try allocator.alloc(f.type, values.len); + var fields = try allocator.alloc(Field, values.len); defer allocator.free(fields); for (values, 0..) |val, i| { - fields[i] = @field(val, f.name); + fields[i] = @field(val, field_name); } - @field(out, f.name) = try zip(func, allocator, fields, args); + @field(out, field_name) = try zip(func, allocator, fields, args); } return out; }, @@ -844,7 +844,7 @@ pub fn collectAlloc( allocator: std.mem.Allocator, obj: anytype, ) std.mem.Allocator.Error![]stdx.meta.FnReturn(func) { - stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".params.len <= 2, "zml.meta.collect expects a func with two arguments, got: {}", .{@TypeOf(func)}); + stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".param_types.len <= 2, "zml.meta.collect expects a func with two arguments, got: {}", .{@TypeOf(func)}); const CollectAllocCtx = struct { func_ctx: _CollectCtx(func), @@ -871,7 +871,7 @@ pub fn collectAlloc( /// finds all X in the given object, and write the result of func(X) into a given slice. /// Asserts that the number of X found is equal to the slice len. pub fn collectBuf(func: anytype, func_ctx: _CollectCtx(func), obj: anytype, out: []stdx.meta.FnReturn(func)) void { - stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".params.len <= 2, "zml.meta.collectBuf expects a func with one or two arguments, got: {}", .{@TypeOf(func)}); + stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".param_types.len <= 2, "zml.meta.collectBuf expects a func with one or two arguments, got: {}", .{@TypeOf(func)}); const LocalContext = struct { func_ctx: _CollectCtx(func), out: @TypeOf(out), @@ -916,14 +916,14 @@ pub fn collectPtrs( } fn _CollectCtx(func: anytype) type { - const params = @typeInfo(@TypeOf(func)).@"fn".params; + const params = @typeInfo(@TypeOf(func)).@"fn".param_types; if (params.len == 1) return void; - return params[0].type orelse @compileError("anytype not supported in collect"); + return params[0] orelse @compileError("anytype not supported in collect"); } fn _CollectArg(func: anytype) type { - const params = @typeInfo(@TypeOf(func)).@"fn".params; - return params[params.len - 1].type orelse @compileError("anytype not supported in collect"); + const params = @typeInfo(@TypeOf(func)).@"fn".param_types; + return params[params.len - 1] orelse @compileError("anytype not supported in collect"); } pub fn Contains(Haystack: type, T: type) bool { @@ -939,8 +939,8 @@ pub fn Contains(Haystack: type, T: type) bool { return switch (@typeInfo(Haystack)) { .@"struct" => |info| { if (@hasDecl(Haystack, "zml_no_meta_visit")) return false; - inline for (info.fields) |field| { - if (!field.is_comptime and Contains(field.type, T)) { + inline for (info.field_types, info.field_attrs) |Field, field_attrs| { + if (!field_attrs.@"comptime" and Contains(Field, T)) { return true; } } @@ -948,8 +948,8 @@ pub fn Contains(Haystack: type, T: type) bool { }, .@"union" => |info| { if (@hasDecl(Haystack, "zml_no_meta_visit")) return false; - inline for (info.fields) |field| { - if (Contains(field.type, T)) + inline for (info.field_types) |Field| { + if (Contains(Field, T)) return true; } return false; diff --git a/zml/moe/cute_kernels/persistent_mxfp4.zig b/zml/moe/cute_kernels/persistent_mxfp4.zig index 1f4b25cb..7208e6ce 100644 --- a/zml/moe/cute_kernels/persistent_mxfp4.zig +++ b/zml/moe/cute_kernels/persistent_mxfp4.zig @@ -1032,11 +1032,11 @@ fn runTmaProducer( } fn Tuple(comptime T: type, comptime len: usize) type { - return std.meta.Tuple(&([_]type{T} ** len)); + return @Tuple(&@as([len]type, @splat(T))); } fn arity(comptime T: type) usize { - return @typeInfo(T).@"struct".fields.len; + return @typeInfo(T).@"struct".field_names.len; } /// The MLIR types of a tuple of values, as `openWhile` expects them. diff --git a/zml/moe/cutlass_flashinfer.zig b/zml/moe/cutlass_flashinfer.zig index 734283cd..75b52eeb 100644 --- a/zml/moe/cutlass_flashinfer.zig +++ b/zml/moe/cutlass_flashinfer.zig @@ -95,7 +95,7 @@ pub const Variant = enum { const max_num_devices = zml.platform.Platform.MAX_NUM_DEVICES; pub const Runners = struct { - runners: [std.meta.fields(Variant).len][max_num_devices]?DeviceRunner = + runners: [std.meta.fieldNames(Variant).len][max_num_devices]?DeviceRunner = @splat(@splat(null)), pub fn deinit(self: *Runners) void { @@ -112,7 +112,7 @@ pub const Runners = struct { if (device < 0 or device >= max_num_devices) return error.UnsupportedDevice; const index: usize = @intCast(device); - const variant_index: usize = @intFromEnum(variant); + const variant_index: usize = @backingInt(variant); if (self.runners[variant_index][index]) |runner| return runner; const api = try fi_cutlass_moe.apiForDevice(device); diff --git a/zml/moe/triton_kernels/mxfp4.zig b/zml/moe/triton_kernels/mxfp4.zig index 4da51a5a..74170966 100644 --- a/zml/moe/triton_kernels/mxfp4.zig +++ b/zml/moe/triton_kernels/mxfp4.zig @@ -64,8 +64,8 @@ pub fn validateInputs(c: Config, a: Inputs) !void { .w2 = zml.Shape.init(.{ c.experts, c.hidden, @divTrunc(c.intermediate, 2) }, .u8), .s2 = zml.Shape.init(.{ c.experts, c.hidden, a.s2.dim(2) }, .f8e8m0), }; - inline for (std.meta.fields(Inputs)) |field| { - if (!@field(a, field.name).shape().eql(@field(shapes, field.name))) return error.InvalidInputShape; + inline for (comptime std.meta.fieldNames(Inputs)) |field| { + if (!@field(a, field).shape().eql(@field(shapes, field))) return error.InvalidInputShape; } } diff --git a/zml/nn.zig b/zml/nn.zig index 1403788f..7519a435 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -2259,8 +2259,13 @@ pub fn collectDims( return context.res; } -fn shapeToDims(shape: anytype) [@divExact(@sizeOf(@TypeOf(shape)), @sizeOf(i64))]i64 { - return @bitCast(shape); +fn shapeToDims(shape: anytype) [std.meta.fieldNames(@TypeOf(shape)).len]i64 { + const field_names = comptime std.meta.fieldNames(@TypeOf(shape)); + var result: [field_names.len]i64 = undefined; + inline for (field_names, 0..) |field_name, i| { + result[i] = @field(shape, field_name); + } + return result; } test collectDims { @@ -2315,7 +2320,7 @@ fn ShapeStruct(comptime dims: anytype) type { @setEvalBranchQuota(rank + 5); var struct_field_names: [rank][]const u8 = undefined; var struct_field_types: [rank]type = undefined; - var struct_field_attrs: [rank]std.builtin.Type.StructField.Attributes = undefined; + var struct_field_attrs: [rank]std.builtin.Type.Struct.FieldAttributes = undefined; const default: i64 = NOT_SET; for (&struct_field_names, &struct_field_types, &struct_field_attrs, dims) |*field_name, *field_type, *field_attr, axis| { field_name.* = @tagName(axis); diff --git a/zml/ops.zig b/zml/ops.zig index fbfc67b8..11a31ea0 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -34,7 +34,7 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@T if (!struct_info.is_tuple) { @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"); } - break :b struct_info.fields.len; + break :b struct_info.field_names.len; }, .array => |array_info| b: { if (array_info.child != Tensor) { @@ -72,7 +72,7 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@T const reducer_block = b: { const ArgsTypes: [input_tensors.len]type = @splat(ReduceArgs); - var args: std.meta.Tuple(&ArgsTypes) = undefined; + var args: @Tuple(&ArgsTypes) = undefined; var block_types: [2 * input_tensors.len]*const mlir.Type = undefined; inline for (0..input_tensors.len) |i| { @@ -158,8 +158,8 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@T } var out: @TypeOf(inputs) = undefined; - inline for (struct_info.fields, 0..) |field, i| { - @field(out, field.name) = Tensor._result(input_tensors[i].shape(), op.result(i)); + inline for (struct_info.field_names, 0..) |field_name, i| { + @field(out, field_name) = Tensor._result(input_tensors[i].shape(), op.result(i)); } break :b out; }, @@ -216,7 +216,7 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func // Note: inputs and inits are infered to be tuple of ReduceArgs const reduce_block, var result = b: { const ArgsTypes: [inits.len]type = @splat(ReduceArgs); - const ArgsType = std.meta.Tuple(&ArgsTypes); + const ArgsType = @Tuple(&ArgsTypes); var args: ArgsType = undefined; var block_types: [2 * inits.len]*const mlir.Type = undefined; @@ -315,7 +315,7 @@ pub const ReduceWindowOpts = struct { pub fn ReduceWindowFn(N: comptime_int) type { return @Fn( &@as([N]type, @splat(ReduceArgs)), - &@as([N]std.builtin.Type.Fn.Param.Attributes, @splat(.{})), + &@as([N]std.builtin.Type.Fn.ParamAttributes, @splat(.{})), [N]Tensor, .{}, ); @@ -405,7 +405,7 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytyp const sort_block = b: { const ArgsTypes: [inputs.len]type = @splat(SortArgs); - const ArgsType = std.meta.Tuple(&ArgsTypes); + const ArgsType = @Tuple(&ArgsTypes); var args: ArgsType = undefined; var block_types: [2 * inputs.len]*const mlir.Type = undefined; @@ -547,7 +547,7 @@ pub fn @"while"( const result: While.State = captured_context.body(initial_state); - if (builtin.mode == .Debug) { + if (builtin.mode == .debug) { const result_tensors = meta.collectPtrs(Tensor, allocator, &result) catch @panic("OOM"); stdx.debug.assert(flat_operands.len == result_tensors.len, "zml.ops.while({s}) expects body to return {d} Tensor values, got {d} values in ", .{ @typeName(While), flat_operands.len, result_tensors.len }); for (0.., flat_operands, result_tensors) |i, in, out| { @@ -1623,7 +1623,7 @@ test "cuda_tile grid" { var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Mod.forward, .{a}, platform, .{}); defer exe.deinit(); - const input = [_]f32{0} ** n; + const input: [n]f32 = @splat(0); var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); defer a_buffer.deinit(); @@ -1729,7 +1729,7 @@ pub fn scatter( const update_block, var result = b: { const ArgsTypes: [inputs.len]type = @splat(ScatterArgs); - const ArgsType = std.meta.Tuple(&ArgsTypes); + const ArgsType = @Tuple(&ArgsTypes); var args: ArgsType = undefined; var block_types: [2 * inputs.len]*const mlir.Type = undefined; @@ -2063,7 +2063,7 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, var self_kind: stdx.BoundedArray(GatherAxisKind, constants.MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; for (self._shape.tags(), 0..self.rank()) |t, self_ax| { - const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, 1, @intCast(self_ax)); + const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, @intCast(self_ax), 1); if (indices_shape.hasTag(t)) |id_ax| { // tag is both in self and indices -> it's a batching dim // Note: tags are required for batching. @@ -2087,7 +2087,7 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, if (ax == idx_axes[0]) { // The first val_ax is special cause this is the place where we insert indices axes. for (0.., indices_shape.tags(), indices_shape.dims()) |id_axis_order, id_axis, id_inserted_dim| { - const is_batching_axis = std.mem.containsAtLeastScalar(i64, idx_batch_axes.constSlice(), 1, @intCast(id_axis_order)); + const is_batching_axis = std.mem.containsAtLeastScalar(i64, idx_batch_axes.constSlice(), @intCast(id_axis_order), 1); // Batching axis is already in self. if (is_batching_axis) continue; @@ -2278,10 +2278,10 @@ fn CustomCallResultTypeFromOutputSpec(comptime OutputSpecT: type) type { .@"struct" => |struct_info| b: { if (OutputSpecT == Shape) break :b Tensor; if (!struct_info.is_tuple) @compileError("Expected tuple output shape spec"); - break :b if (struct_info.fields.len == 1) + break :b if (struct_info.field_names.len == 1) Tensor else - [struct_info.fields.len]Tensor; + [struct_info.field_names.len]Tensor; }, .array => |array_info| b: { break :b if (array_info.len == 1) @@ -2322,20 +2322,20 @@ pub fn customCallOutputOperandAliases( ) ?[]const dialects.stablehlo.CustomCallOpts.OutputOperandAlias { const aliases_ = aliases orelse return null; - const output_fields = @typeInfo(O).@"struct".fields; + const output_field_names = @typeInfo(O).@"struct".field_names; comptime var num_aliases = 0; - inline for (output_fields) |field| { - if (@field(aliases_, field.name) != null) num_aliases += 1; + inline for (output_field_names) |field_name| { + if (@field(aliases_, field_name) != null) num_aliases += 1; } const output_operand_aliases = comptime blk: { var aliases_buffer: [num_aliases]dialects.stablehlo.CustomCallOpts.OutputOperandAlias = undefined; var i = 0; - for (output_fields, 0..) |field, output_index| { - if (@field(aliases_, field.name)) |operand| { + for (output_field_names, 0..) |field_name, output_index| { + if (@field(aliases_, field_name)) |operand| { aliases_buffer[i] = .{ .output_index = @intCast(output_index), - .operand_index = @intCast(@intFromEnum(operand)), + .operand_index = @intCast(@backingInt(operand)), }; i += 1; } @@ -2447,7 +2447,7 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, break :b &[1]Tensor{inputs}; } if (!struct_info.is_tuple) @compileError("Expected tuple"); - var inputs_: [struct_info.fields.len]Tensor = undefined; + var inputs_: [struct_info.field_names.len]Tensor = undefined; meta.collectBuf((struct { pub fn func(t: Tensor) Tensor { return t; @@ -2471,7 +2471,7 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, break :b &[1]Shape{outputs}; } if (!struct_info.is_tuple) @compileError("Expected tuple"); - var output_shapes: [struct_info.fields.len]Shape = undefined; + var output_shapes: [struct_info.field_names.len]Shape = undefined; meta.collectBuf((struct { pub fn func(t: Shape) Shape { return t; @@ -2504,14 +2504,14 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, .@"struct" => |struct_info| b: { if (@TypeOf(outputs) == Shape) break :b outputs_flat[0]; if (!struct_info.is_tuple) @compileError("Expected tuple"); - if (struct_info.fields.len == 1) break :b outputs_flat[0]; - var outputs_: [struct_info.fields.len]Tensor = undefined; + if (struct_info.field_names.len == 1) break :b outputs_flat[0]; + var outputs_: [struct_info.field_names.len]Tensor = undefined; @memcpy(&outputs_, outputs_flat); break :b outputs_; }, .array => |array_info| b: { if (array_info.len == 1) break :b outputs_flat[0]; - var outputs_: [array_info.fields.len]Tensor = undefined; + var outputs_: [array_info.len]Tensor = undefined; @memcpy(&outputs_, outputs_flat); break :b outputs_; }, @@ -2536,7 +2536,7 @@ pub fn manualComputation( break :b &[1]Shape{outputs}; } if (!struct_info.is_tuple) @compileError("Expected tuple output shapes"); - var output_shapes_flat: [struct_info.fields.len]Shape = undefined; + var output_shapes_flat: [struct_info.field_names.len]Shape = undefined; meta.collectBuf((struct { pub fn func(t: Shape) Shape { return t; @@ -2947,7 +2947,7 @@ fn manualComputationOutputShapesArg(comptime OutputShapesT: type, local_output_s if (pointer_info.size != .slice or pointer_info.child != Shape) { @compileError("manualComputation body output shapes must be void, Shape, []Shape, or []const Shape; got " ++ @typeName(OutputShapesT)); } - break :b if (pointer_info.is_const) + break :b if (pointer_info.attrs.@"const") local_output_shapes else @constCast(local_output_shapes); @@ -3001,7 +3001,7 @@ fn metadataFieldToMlirAttribute(mlir_ctx: *mlir.Context, comptime T: type, value return switch (type_info) { .comptime_int => .int(mlir_ctx, .u64, @as(u64, value)), .@"enum" => |enum_field| switch (@typeInfo(enum_field.tag_type)) { - .int => |int_tag| metadataIntegerFieldToMlirAttribute(mlir_ctx, int_tag, @intFromEnum(value)), + .int => |int_tag| metadataIntegerFieldToMlirAttribute(mlir_ctx, int_tag, @backingInt(value)), else => @compileError("Unsupported tag type for enum metadata: " ++ @typeName(enum_field.tag_type)), }, .int => |int_field| metadataIntegerFieldToMlirAttribute(mlir_ctx, int_field, value), @@ -3108,15 +3108,11 @@ pub fn ShapeToTensor(T: type) type { } pub fn CustomCallOutputOperandAliases(I: type, O: type) type { - const output_fields = @typeInfo(O).@"struct".fields; - var field_names: [output_fields.len][]const u8 = undefined; - inline for (output_fields, 0..) |field, i| { - field_names[i] = field.name; - } + const field_names = @typeInfo(O).@"struct".field_names; return @Struct( .auto, null, - &field_names, + field_names, &@splat(?std.meta.FieldEnum(I)), &@splat(.{ .default_value_ptr = &@as(?std.meta.FieldEnum(I), null) }), ); @@ -3133,9 +3129,9 @@ pub fn shardingAwareTypedCustomCall( const Output = @TypeOf(output); const Attributes = @TypeOf(attributes); - var output_shapes: [@typeInfo(Output).@"struct".fields.len]Shape = undefined; - inline for (@typeInfo(Output).@"struct".fields, 0..) |field, i| { - output_shapes[i] = @field(output, field.name); + var output_shapes: [@typeInfo(Output).@"struct".field_names.len]Shape = undefined; + inline for (@typeInfo(Output).@"struct".field_names, 0..) |field_name, i| { + output_shapes[i] = @field(output, field_name); } const Handler = struct { @@ -3154,8 +3150,8 @@ pub fn shardingAwareTypedCustomCall( // Convert the slice back to a struct var out: ShapeToTensor(Output) = undefined; - inline for (@typeInfo(Output).@"struct".fields, 0..) |field, i| { - @field(out, field.name) = output_tensors[i]; + inline for (@typeInfo(Output).@"struct".field_names, 0..) |field_name, i| { + @field(out, field_name) = output_tensors[i]; } return out; } @@ -3178,9 +3174,9 @@ pub fn typedCustomCall( const input_tensors: []const Tensor = switch (@typeInfo(Input)) { .@"struct" => |struct_info| b: { - var input_tensors: [struct_info.fields.len]Tensor = undefined; - inline for (struct_info.fields, 0..) |field, i| { - input_tensors[i] = @field(input, field.name); + var input_tensors: [struct_info.field_names.len]Tensor = undefined; + inline for (struct_info.field_names, 0..) |field_name, i| { + input_tensors[i] = @field(input, field_name); } break :b &input_tensors; }, @@ -3194,9 +3190,9 @@ pub fn typedCustomCall( const output_shapes: []const Shape = switch (@typeInfo(Output)) { .@"struct" => |struct_info| b: { - var output_shapes: [struct_info.fields.len]Shape = undefined; - inline for (struct_info.fields, 0..) |field, i| { - output_shapes[i] = @field(output, field.name); + var output_shapes: [struct_info.field_names.len]Shape = undefined; + inline for (struct_info.field_names, 0..) |field_name, i| { + output_shapes[i] = @field(output, field_name); } break :b &output_shapes; }, @@ -3210,13 +3206,13 @@ pub fn typedCustomCall( var metadata_attribute_list = switch (@typeInfo(Attributes)) { .void => std.ArrayList(mlir.NamedAttribute).initCapacity(allocator, 2) catch unreachable, - .@"struct" => |struct_info| std.ArrayList(mlir.NamedAttribute).initCapacity(allocator, struct_info.fields.len + 2) catch unreachable, + .@"struct" => |struct_info| std.ArrayList(mlir.NamedAttribute).initCapacity(allocator, struct_info.field_names.len + 2) catch unreachable, else => @compileError("Unsupported type: " ++ @typeName(Attributes)), }; if (@typeInfo(Attributes) == .@"struct") { - inline for (@typeInfo(Attributes).@"struct".fields) |field| { - if (metadataFieldToMlirAttribute(ctx.mlir_ctx, field.type, @field(attributes, field.name))) |attribute| { - metadata_attribute_list.appendAssumeCapacity(mlir.NamedAttribute.named(ctx.mlir_ctx, field.name, attribute)); + inline for (@typeInfo(Attributes).@"struct".field_names, @typeInfo(Attributes).@"struct".field_types) |field_name, Field| { + if (metadataFieldToMlirAttribute(ctx.mlir_ctx, Field, @field(attributes, field_name))) |attribute| { + metadata_attribute_list.appendAssumeCapacity(mlir.NamedAttribute.named(ctx.mlir_ctx, field_name, attribute)); } } } @@ -3272,8 +3268,8 @@ pub fn typedCustomCall( switch (@typeInfo(Output)) { .@"struct" => |struct_info| { var out: ShapeToTensor(Output) = undefined; - inline for (output_shapes, struct_info.fields, 0..) |output_shape, field, i| { - @field(out, field.name) = Tensor._result(output_shape, op.result(i)); + inline for (output_shapes, struct_info.field_names, 0..) |output_shape, field_name, i| { + @field(out, field_name) = Tensor._result(output_shape, op.result(i)); } return out; }, @@ -3296,26 +3292,26 @@ fn customCallArgsFromPjrtCallFrame(I: type, O: type, A: type, call_frame: *pjrt. A, } { var input: TensorToCustomCallBuffer(I) = undefined; - inline for (@typeInfo(I).@"struct".fields, 0..) |field, i| { + inline for (@typeInfo(I).@"struct".field_names, 0..) |field_name, i| { const buf = call_frame.args.buffers()[i]; - @field(input, field.name) = .fromPjrt(buf); + @field(input, field_name) = .fromPjrt(buf); } var output: ShapeToCustomCallBuffer(O) = undefined; - inline for (@typeInfo(O).@"struct".fields, 0..) |field, i| { + inline for (@typeInfo(O).@"struct".field_names, 0..) |field_name, i| { const shape_buf = call_frame.results.buffers()[i]; - @field(output, field.name) = .fromPjrt(shape_buf); + @field(output, field_name) = .fromPjrt(shape_buf); } const attributes: A = switch (@typeInfo(A)) { .void => {}, .@"struct" => blk: { var attrs: A = undefined; - inline for (@typeInfo(A).@"struct".fields) |field| { + inline for (@typeInfo(A).@"struct".field_names, @typeInfo(A).@"struct".field_types) |field_name, Field| { // TODO: Use getByIndex instead? - const attribute = call_frame.attrs.getByName(.scalar, field.name) orelse - @panic("Attribute not found: " ++ field.name); - @field(attrs, field.name) = attribute.get(field.type); + const attribute = call_frame.attrs.getByName(.scalar, field_name) orelse + @panic("Attribute not found: " ++ field_name); + @field(attrs, field_name) = attribute.get(Field); } break :blk attrs; }, diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index f492b120..8543b591 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -117,13 +117,13 @@ pub fn dtypeFromBufferType(pjrt_type: pjrt.BufferType) DataType { } test bufferTypeFromDtype { - inline for (@typeInfo(DataType).@"enum".fields) |field| { - const dt: DataType = @enumFromInt(field.value); + inline for (@typeInfo(DataType).@"enum".field_values) |field_value| { + const dt: DataType = @fromBackingInt(@intCast(field_value)); try std.testing.expectEqual(dt, dtypeFromBufferType(bufferTypeFromDtype(dt))); } - inline for (@typeInfo(pjrt.BufferType).@"enum".fields) |field| { - const dt: pjrt.BufferType = @enumFromInt(field.value); + inline for (@typeInfo(pjrt.BufferType).@"enum".field_values) |field_value| { + const dt: pjrt.BufferType = @fromBackingInt(@intCast(field_value)); if (dt == .invalid) continue; try std.testing.expectEqual(dt, bufferTypeFromDtype(dtypeFromBufferType(dt))); } diff --git a/zml/platform.zig b/zml/platform.zig index 354ec3c1..9e513451 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -216,7 +216,7 @@ pub const Device = struct { } pub fn memory(self: *const Device, memory_kind: Memory.Kind) ?*const Memory { - return self.memory_by_kind.values[@intFromEnum(memory_kind)]; + return self.memory_by_kind.values[@backingInt(memory_kind)]; } }; @@ -238,7 +238,7 @@ fn sortDevicesById(target: Target, devices: []Device) void { std.mem.sort(Device, devices, Context{ .target = target }, Context.lessThan); - if (builtin.mode == .Debug) { + if (builtin.mode == .debug) { for (devices, 0..) |device, expected_id| { std.debug.assert(platformDeviceSortId(target, device) == expected_id); } diff --git a/zml/profiling/profiler.zig b/zml/profiling/profiler.zig index 3f3c8409..d46a0c4c 100644 --- a/zml/profiling/profiler.zig +++ b/zml/profiling/profiler.zig @@ -92,7 +92,7 @@ pub const ProfilerOptions = struct { c.tensorflow_ProfileOptions_set_repository_path(proto, upb.stringView(self.repository_path)); if (self.device_type) |value| { - c.tensorflow_ProfileOptions_set_device_type(proto, @intFromEnum(value)); + c.tensorflow_ProfileOptions_set_device_type(proto, @backingInt(value)); } if (self.include_dataset_ops) |value| { c.tensorflow_ProfileOptions_set_include_dataset_ops(proto, value); diff --git a/zml/profiling/tracer.zig b/zml/profiling/tracer.zig index e3157d10..5aed01df 100644 --- a/zml/profiling/tracer.zig +++ b/zml/profiling/tracer.zig @@ -3,6 +3,7 @@ const builtin = @import("builtin"); const c = @import("c"); const platforms = @import("platforms"); +const stdx = @import("stdx"); const zffi = @import("ffi"); pub const Span = struct { @@ -44,14 +45,14 @@ pub fn span(comptime name: []const u8, metadata: anytype) Span { return .start(name); }, .@"struct" => |info| { - if (info.fields.len == 0) { + if (info.field_names.len == 0) { return .start(name); } var buffer: [computeEncodedNameLen(name, @TypeOf(metadata))]u8 = undefined; var encoded: std.ArrayList(u8) = .initBuffer(&buffer); - appendEncodedName(&encoded, .failing, name, info, metadata) catch unreachable; + appendEncodedName(&encoded, .failing, name, metadata) catch unreachable; return .start(encoded.items); }, else => @compileError("Unsupported metadata."), @@ -62,34 +63,34 @@ pub fn span(comptime name: []const u8, metadata: anytype) Span { /// bounded by `span` at comptime. The caller owns the returned slice. pub fn formatSpanName(allocator: std.mem.Allocator, name: []const u8, metadata: anytype) ![]const u8 { switch (@typeInfo(@TypeOf(metadata))) { - .@"struct" => |info| { + .@"struct" => { var encoded: std.ArrayList(u8) = .empty; errdefer encoded.deinit(allocator); - try appendEncodedName(&encoded, allocator, name, info, metadata); + try appendEncodedName(&encoded, allocator, name, metadata); return try encoded.toOwnedSlice(allocator); }, else => @compileError("trace metadata must be a struct literal like .{ .step_num = 42 }"), } } -fn appendEncodedName(buffer: *std.ArrayList(u8), allocator: std.mem.Allocator, name: []const u8, info: std.builtin.Type.Struct, metadata: anytype) !void { +fn appendEncodedName(buffer: *std.ArrayList(u8), allocator: std.mem.Allocator, name: []const u8, metadata: anytype) !void { try buffer.appendSlice(allocator, name); var field_count: usize = 0; try buffer.append(allocator, '#'); - inline for (info.fields) |field| { - if (shouldEncodeField(field, @field(metadata, field.name))) { + inline for (comptime std.meta.fieldNames(@TypeOf(metadata))) |field_name| { + if (shouldEncodeField(field_name, @field(metadata, field_name))) { if (field_count != 0) { try buffer.append(allocator, ','); } field_count += 1; - const key = comptime metadataKey(field.name); + const key = comptime metadataKey(field_name); validateMetadataToken(key); try buffer.appendSlice(allocator, key); try buffer.append(allocator, '='); - try appendMetadataValue(buffer, allocator, @field(metadata, field.name), field.name); + try appendMetadataValue(buffer, allocator, @field(metadata, field_name), field_name); } } @@ -114,33 +115,38 @@ fn metadataKey(comptime field_name: []const u8) []const u8 { fn computeEncodedNameLen(comptime name: []const u8, comptime Metadata: type) usize { switch (@typeInfo(Metadata)) { .@"struct" => |info| { - if (info.fields.len == 0) return name.len; + if (info.field_names.len == 0) return name.len; // name#[attrs]# var max_len = name.len + 2; - inline for (info.fields) |field| { - const key = comptime metadataKey(field.name); + inline for ( + info.field_names, + info.field_types, + info.field_attrs, + ) |field_name, FieldType, field_attr| { + const key = comptime metadataKey(field_name); validateMetadataToken(key); + const default: ?FieldType = field_attr.defaultValue(FieldType); // [key]=[value], max_len += key.len + 2; - max_len += switch (@typeInfo(field.type)) { + max_len += switch (@typeInfo(FieldType)) { .bool => 1, .int => @max( - std.fmt.comptimePrint("{}", .{std.math.minInt(field.type)}).len, - std.fmt.comptimePrint("{}", .{std.math.maxInt(field.type)}).len, + std.fmt.comptimePrint("{}", .{std.math.minInt(FieldType)}).len, + std.fmt.comptimePrint("{}", .{std.math.maxInt(FieldType)}).len, ), .@"enum" => |ty| b: { var max_enum_len: usize = 0; - for (ty.fields) |enum_field| { - max_enum_len = @max(max_enum_len, enum_field.name.len); + for (ty.field_names) |enum_field_name| { + max_enum_len = @max(max_enum_len, enum_field_name.len); } break :b max_enum_len; }, - .comptime_int => std.fmt.comptimePrint("{}", .{defaultMetadataValue(field).?}).len, - .comptime_float => std.fmt.comptimePrint("{d}", .{defaultMetadataValue(field).?}).len, - .enum_literal => @tagName(defaultMetadataValue(field).?).len, - else => @compileError("trace metadata field '" ++ field.name ++ "' is not statically bounded; use formatspanName"), + .comptime_int => std.fmt.comptimePrint("{}", .{default.?}).len, + .comptime_float => std.fmt.comptimePrint("{d}", .{default.?}).len, + .enum_literal => @tagName(default.?).len, + else => @compileError("trace metadata field '" ++ field_name ++ "' is not statically bounded; use formatspanName"), }; } return max_len; @@ -149,17 +155,11 @@ fn computeEncodedNameLen(comptime name: []const u8, comptime Metadata: type) usi } } -fn defaultMetadataValue(comptime field: std.builtin.Type.StructField) ?field.type { - return if (field.default_value_ptr) |default_opaque| - @as(*const field.type, @ptrCast(@alignCast(default_opaque))).* - else - null; -} - -fn shouldEncodeField(field: std.builtin.Type.StructField, value: anytype) bool { - return switch (@typeInfo(field.type)) { +fn shouldEncodeField(comptime field_name: []const u8, value: anytype) bool { + const FieldType = @TypeOf(value); + return switch (@typeInfo(FieldType)) { .optional => value != null, - .bool => if (comptime std.mem.eql(u8, field.name, "root")) value else true, + .bool => if (comptime std.mem.eql(u8, field_name, "root")) value else true, else => true, }; } diff --git a/zml/quantization.zig b/zml/quantization.zig index b125513a..89e1b210 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -474,7 +474,7 @@ test "block FP8 E8M0 scales preserve power-of-two boundaries and nonminor axes" try zml.testing.expectEqualShapes(x.shape().setDim(.k, 8).withDtype(.f32), exe.output_shapes[1]); const fp8_max: f32 = if (dtype == .f8e4m3fn) 448.0 else 224.0; var host: [2 * 256 * 3]f32 = undefined; - var maxima = [_]f32{0} ** (2 * 8 * 3); + var maxima: [2 * 8 * 3]f32 = @splat(0); for (&host, 0..) |*value, i| { const block = (i / 3) % 256 / 32; const scale_index = (i / (256 * 3)) * 24 + block * 3 + i % 3; diff --git a/zml/shape.zig b/zml/shape.zig index 11a3efac..3e3fb56b 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -58,19 +58,19 @@ pub const Shape = struct { if (comptime stdx.meta.isStruct(T)) { var dims_: DimsArray = .empty; var tags_: TagsArray = .empty; - inline for (std.meta.fields(T)) |field| { - const fv = @field(v, field.name); - if (comptime stdx.meta.isInteger(field.type)) { + inline for (comptime std.meta.fieldNames(T), comptime std.meta.fieldTypes(T)) |field_name, Field| { + const fv = @field(v, field_name); + if (comptime stdx.meta.isInteger(Field)) { dims_.appendAssumeCapacity(@intCast(fv)); } else if (@TypeOf(fv) == @EnumLiteral() and comptime isAutoDim(fv)) { dims_.appendAssumeCapacity(-1); } else { - stdx.debug.compileError("Field {s} should be an integer or an auto dimension, got {}", .{ field.name, field.type }); + stdx.debug.compileError("Field {s} should be an integer or an auto dimension, got {}", .{ field_name, Field }); } if (comptime stdx.meta.isTuple(T)) { tags_.appendAssumeCapacity(TagUnknown); } else { - tags_.appendAssumeCapacity(toTag(field)); + tags_.appendAssumeCapacity(toTag(field_name)); } } @@ -107,9 +107,9 @@ pub const Shape = struct { } if (comptime stdx.meta.isTupleOfAny(T, isAxisConvertible)) { - inline for (std.meta.fields(T)) |field| { - axes_.appendAssumeCapacity(self.axis(@field(v, field.name))); - tags_.appendAssumeCapacity(self.tag(@field(v, field.name))); + inline for (comptime std.meta.fieldNames(T)) |field_name| { + axes_.appendAssumeCapacity(self.axis(@field(v, field_name))); + tags_.appendAssumeCapacity(self.tag(@field(v, field_name))); } return .{ axes_, tags_ }; } @@ -191,7 +191,7 @@ pub const Shape = struct { fn isTagConvertible(comptime T: type) bool { return switch (T) { - @EnumLiteral(), std.builtin.Type.StructField, Tag => true, + @EnumLiteral(), [:0]const u8, Tag => true, else => false, }; } @@ -200,14 +200,14 @@ pub const Shape = struct { const T = @TypeOf(v); return switch (T) { @EnumLiteral() => @tagName(v).ptr, - std.builtin.Type.StructField => v.name.ptr, + [:0]const u8 => v.ptr, Tag => v, - else => stdx.debug.compileError("Shape tag should be an @EnumLiteral(), a Shape.Tag or a StructField, got {}", .{T}), + else => stdx.debug.compileError("Shape tag should be an @EnumLiteral(), a Shape.Tag or a field name, got {}", .{T}), }; } fn ensureAttributesAreSync(self: Shape) void { - if (builtin.mode == .Debug) { + if (builtin.mode == .debug) { stdx.debug.assert(self._dims.len == self._tags.len, "Tags, dims and partitioning have diverged! dims={d} tags={d}", .{ self._dims.len, self._tags.len }); } } @@ -312,8 +312,8 @@ pub const Shape = struct { } if (comptime stdx.meta.isStruct(T)) { - inline for (std.meta.fields(T)) |field| { - res.appendAssumeCapacity(self.axis(@field(axes_, field.name))); + inline for (comptime std.meta.fieldNames(T)) |field_name| { + res.appendAssumeCapacity(self.axis(@field(axes_, field_name))); } return res; } @@ -861,11 +861,11 @@ pub const Shape = struct { var res = self; for (0..self.rank()) |ax| { const mesh_axis = self.partition(ax).meshAxis() orelse continue; - inline for (std.meta.fields(T)) |field| { - const old_axis = mesh.resolveLogicalAxis(toTag(field)) orelse - stdx.debug.panic("Mesh '{s}' has no logical axis '{s}'", .{ mesh.name, field.name }); + inline for (comptime std.meta.fieldNames(T)) |field_name| { + const old_axis = mesh.resolveLogicalAxis(toTag(field_name)) orelse + stdx.debug.panic("Mesh '{s}' has no logical axis '{s}'", .{ mesh.name, field_name }); if (mesh_axis == old_axis) { - const new_tag = toTag(@field(mapping, field.name)); + const new_tag = toTag(@field(mapping, field_name)); const new_axis = mesh.resolveLogicalAxis(new_tag) orelse stdx.debug.panic("Mesh '{s}' has no logical axis '{s}'", .{ mesh.name, new_tag }); res._sharding = res._sharding.set(ax, .sharded(@intCast(new_axis))); @@ -1015,10 +1015,10 @@ pub const Shape = struct { const T = @TypeOf(renames); stdx.debug.assertComptime(stdx.meta.isStructOfAny(T, isAxisConvertible), "Must pass a struct of enum literals. Passed: {any}", .{T}); var res = self; - inline for (std.meta.fields(T)) |field| { - const new_field = @field(renames, field.name); + inline for (comptime std.meta.fieldNames(T)) |field_name| { + const new_field = @field(renames, field_name); stdx.debug.assert(self.hasTag(new_field) == null, "{f}.rename({any}) failed because of duplicated axis {any}", .{ self, renames, new_field }); - res._tags.set(self.axis(field), toTag(new_field)); + res._tags.set(self.axis(field_name), toTag(new_field)); } return res; } @@ -1232,8 +1232,8 @@ pub const Shape = struct { stdx.debug.assertComptime(stdx.meta.isStruct(T), "Must pass struct of enum literals like .{ .a = .{ .a1, .a2 } }. Passed: {any}", .{T}); var res = self; - inline for (std.meta.fields(T)) |field| { - res = res.splitAxis(field, @field(axes_, field.name)); + inline for (comptime std.meta.fieldNames(T)) |field_name| { + res = res.splitAxis(field_name, @field(axes_, field_name)); } return res; } @@ -1344,9 +1344,9 @@ pub const Shape = struct { stdx.debug.assertComptime(stdx.meta.isStruct(T), "Must pass struct of enum literals like .{ .a = .{ .a1, .a2 } }. Passed: {any}", .{T}); var res = self; - inline for (std.meta.fields(T)) |field| { - stdx.debug.assertComptime(stdx.meta.isTupleOfAny(field.type, isAxisConvertible) or stdx.meta.isSliceOfAny(field.type, isAxisConvertible), "Must pass struct of axes. Passed: {any}", .{field.type}); - res = res.mergeAxis(field, @field(axes_, field.name)); + inline for (comptime std.meta.fieldNames(T), comptime std.meta.fieldTypes(T)) |field_name, Field| { + stdx.debug.assertComptime(stdx.meta.isTupleOfAny(Field, isAxisConvertible) or stdx.meta.isSliceOfAny(Field, isAxisConvertible), "Must pass struct of axes. Passed: {any}", .{Field}); + res = res.mergeAxis(field_name, @field(axes_, field_name)); } return res; } @@ -1398,14 +1398,14 @@ pub const Shape = struct { } if (comptime stdx.meta.isStruct(V)) { - const fields = std.meta.fields(V); - stdx.debug.assertComptime(fields.len <= constants.MAX_RANK, "Too many fields in struct {} ({d}). Max supported is {d}.", .{ V, fields.len, constants.MAX_RANK }); - inline for (fields) |field| { - const fv = @field(v, field.name); + const field_names = comptime std.meta.fieldNames(V); + stdx.debug.assertComptime(field_names.len <= constants.MAX_RANK, "Too many fields in struct {} ({d}). Max supported is {d}.", .{ V, field_names.len, constants.MAX_RANK }); + inline for (field_names) |field_name| { + const fv = @field(v, field_name); vals_.appendAssumeCapacity(fv); if (!comptime stdx.meta.isTuple(V)) { - tags_.appendAssumeCapacity(toTag(field)); + tags_.appendAssumeCapacity(toTag(field_name)); } } return .{ vals_, tags_ }; @@ -1435,11 +1435,11 @@ pub const Shape = struct { if (comptime stdx.meta.isStruct(V)) { for (0..self.rank()) |_| res.appendAssumeCapacity(default); - const fields = std.meta.fields(V); - stdx.debug.assertComptime(fields.len <= constants.MAX_RANK, "expects up to {} options struct literal, got {}", .{ V, constants.MAX_RANK, fields.len }); - inline for (fields) |field| { - const a = self.axis(field); - res.buffer[a] = @field(options, field.name); + const field_names = comptime std.meta.fieldNames(V); + stdx.debug.assertComptime(field_names.len <= constants.MAX_RANK, "expects up to {} options struct literal, got {}", .{ V, constants.MAX_RANK, field_names.len }); + inline for (field_names) |field_name| { + const a = self.axis(field_name); + res.buffer[a] = @field(options, field_name); } return res; } diff --git a/zml/slice.zig b/zml/slice.zig index f096608b..2deb96d2 100644 --- a/zml/slice.zig +++ b/zml/slice.zig @@ -31,7 +31,7 @@ pub const Slice = struct { const byte_strides = shape.computeByteStrides(); return .{ .bytes = bytes, - .mutable = !type_info.pointer.is_const, + .mutable = !type_info.pointer.attrs.@"const", .shape = shape, .offset_bytes = 0, .byte_strides = byte_strides, @@ -147,12 +147,12 @@ pub const Slice = struct { } pub fn items(slice: Slice, comptime T: type) []T { - stdx.debug.assertComptime(T == bool or @bitSizeOf(T) >= 8, "zml.Slice stores packed sub-bytes type so you need to pass a packed type here like @Vector({}, {}). Got: {}", .{ @divFloor(8, @bitSizeOf(T)), T, T }); + stdx.debug.assertComptime(T == bool or @bitSizeOf(T) >= 8, "zml.Slice stores packed sub-bytes type so you need to pass a packed struct here like stdx.Packed({0}). Got: {0}", .{T}); return @ptrCast(@alignCast(slice.data())); } pub fn constItems(slice: Slice, comptime T: type) []const T { - stdx.debug.assertComptime(T == bool or @bitSizeOf(T) >= 8, "zml.Slice stores packed sub-bytes type so you need to pass a packed type here like @Vector({}, {}). Got: {}", .{ @divFloor(8, @bitSizeOf(T)), T, T }); + stdx.debug.assertComptime(T == bool or @bitSizeOf(T) >= 8, "zml.Slice stores packed sub-bytes type so you need to pass a packed struct here like zml.Float4E2M1.Packed. Got: {}", .{T}); return @ptrCast(@alignCast(slice.constData())); } @@ -289,6 +289,10 @@ pub const Slice = struct { const raw_values: []const T = @ptrCast(@alignCast(slice.bytes[slice.offset_bytes..])); const values = raw_values[0..needed_len]; + if (comptime dt.bitSizeOf() < 8) { + return try stdx.fmt.formatSliceAny(values, options, stride, writer); + } + switch (comptime dt.class()) { .float => try stdx.fmt.formatFloatSlice(values, options, stride, writer), .integer => try stdx.fmt.formatIntSlice(values, options, stride, writer), diff --git a/zml/tensor.zig b/zml/tensor.zig index c2b63f53..275f42d3 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -38,7 +38,7 @@ pub const Tensor = struct { } fn nextTensorId() Id { - return @enumFromInt(Tensor.current_id.fetchAdd(1, .seq_cst)); + return @fromBackingInt(@intCast(Tensor.current_id.fetchAdd(1, .seq_cst))); } pub fn fromShape(shape_: Shape) Tensor { @@ -83,7 +83,7 @@ pub const Tensor = struct { var res: Tensor = .{ ._shape = sh, ._value = val, .id = nextTensorId() }; res._shape._sharding.partition = .open(sh.rank()); - if (builtin.mode == .Debug) { + if (builtin.mode == .debug) { // Check that the MLIR value actually have the same shape. const other = fromMlirValue(val); const eql_dtype = sh.dtype() == other.dtype(); @@ -468,7 +468,7 @@ pub const Tensor = struct { const platform = zml.testing.env(); const io = std.testing.io; - const inputs: [6]@Vector(2, i4) = .{ .{ -3.0, -2 }, .{ -1, 1 }, .{ 2, 3 }, .{ 1, 2 }, .{ 3, 4 }, .{ 5, -5 } }; + const inputs: [6]stdx.Packed(i4) = .{ .pack(-3.0, -2), .pack(-1, 1), .pack(2, 3), .pack(1, 2), .pack(3, 4), .pack(5, -5) }; const x_t = Tensor.init(.{ 6, 2 }, .i4); const y_t = Tensor.init(.{ 6, 2 }, .i4); const z_t = Tensor.init(.{ 6, 2 }, .i4); @@ -1432,8 +1432,8 @@ pub const Tensor = struct { test "convert u2 -> u8" { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const x_u2: [2]@Vector(4, u2) = .{ .{ 0, 1, 2, 3 }, .{ 3, 2, 1, 0 } }; - std.debug.assert(@sizeOf(@TypeOf(x_u2)) == 2); + const x_u2: [2]stdx.Packed(u2) = .{ .pack(0, 1, 2, 3), .pack(3, 2, 1, 0) }; + comptime std.debug.assert(@sizeOf(@TypeOf(x_u2)) == 2); const x_u8: [8]u8 = .{ 0, 1, 2, 3, 3, 2, 1, 0 }; const x_u2_t: Tensor = .init(.{8}, .u2); @@ -3809,8 +3809,8 @@ pub const Tensor = struct { const has_name: ?[:0]const u8, const a = switch (@typeInfo(@TypeOf(named_axis_))) { .int, .comptime_int => .{ null, self.axis(@as(i64, @intCast(named_axis_))) }, .@"struct" => |info| blk: { - stdx.debug.assertComptime(info.fields.len == 1, err_msg, .{}); - break :blk .{ info.fields[0].name, self.axis(@field(named_axis_, info.fields[0].name)) }; + stdx.debug.assertComptime(info.field_names.len == 1, err_msg, .{}); + break :blk .{ info.field_names[0], self.axis(@field(named_axis_, info.field_names[0])) }; }, else => stdx.debug.compileError(err_msg, .{}), }; diff --git a/zml/tokenizer/iree/tokenizer.zig b/zml/tokenizer/iree/tokenizer.zig index bc986d4b..0ae25c42 100644 --- a/zml/tokenizer/iree/tokenizer.zig +++ b/zml/tokenizer/iree/tokenizer.zig @@ -577,8 +577,8 @@ pub const Normalizer = struct { ), }, .Strip => |direction| b: { - const strip_left = @intFromEnum(direction) & 0b10 != 0; - const strip_right = @intFromEnum(direction) & 0b01 != 0; + const strip_left = @backingInt(direction) & 0b10 != 0; + const strip_right = @backingInt(direction) & 0b01 != 0; break :b c.iree_tokenizer_normalizer_strip_allocate(strip_left, strip_right, iree_allocator, &norm); }, }; @@ -785,10 +785,10 @@ test "normalizer sequence preserves composition across tiles" { const allocator = std.testing.allocator; var normalizer = try Normalizer.init(allocator, &.{ .NFC, .Lowercase }); defer normalizer.deinit(); - const input = "a" ** 63 ++ "e\u{301}b"; + const input = @as([63]u8, @splat('a')) ++ "e\u{301}b"; const output = try normalizer.normalize(allocator, input); defer allocator.free(output); - try std.testing.expectEqualSlices(u8, "a" ** 63 ++ "éb", output); + try std.testing.expectEqualSlices(u8, @as([63]u8, @splat('a')) ++ "éb", output); } test "normalizer sequence allocation failure retains child ownership" { From a5c40b7614b8ae83d0792e653e41d8ad4931d38c Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Tue, 13 Oct 2026 19:54:27 +0200 Subject: [PATCH 780/881] zml/platform/furiosa: rename environment variables from to FURIOSA_* (#921) --- platforms/furiosa/furiosa.bzl | 4 ++-- platforms/furiosa/furiosa.zig | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index d0b901ba..1e0ba026 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202610091329.178.1.75e89af70664/zml-furiosa-linux-amd64.tar.zst", - sha256 = "dfc5bd915e3075712d1ec821379b2958a4c3767ba827b59e6b76d32571970f88", + url = "https://mirror.zml.ai/plugins/202610111616.184.1.0536437b83ad/zml-furiosa-linux-amd64.tar.zst", + sha256 = "730469a37671f94ebf095a245d96a9c8585954d92a98698ae675bf73c765e75d", ) return mctx.extension_metadata( diff --git a/platforms/furiosa/furiosa.zig b/platforms/furiosa/furiosa.zig index 352c4342..b8057c71 100644 --- a/platforms/furiosa/furiosa.zig +++ b/platforms/furiosa/furiosa.zig @@ -33,8 +33,8 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { defer allocator.free(root); var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - try setEnv("XLA_FURIOSA_SDK_ROOT", try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{root}), 1); - try setEnv("XLA_FURIOSA_VISIBLE_DEVICES", "0", 0); + try setEnv("FURIOSA_SDK_ROOT", try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{root}), 1); + try setEnv("FURIOSA_VISIBLE_DEVICES", "0", 0); const library = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ root, "lib", "libzml_furiosa.so" }); log.info("Loading Furiosa plugin: {s}", .{library}); From f83f5ece0bc9a6a6ce78a1d6e1e1fd5a4b36e837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20Gauti=C3=A9?= <51361266+tgautie@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:12:54 +0200 Subject: [PATCH 781/881] zml/kernels/attention: Resolve TPU and NKI attention meshes by logical axis (#933) We align TPU and NKI attention meshes expectation on the CuTe GPU attention, which simply resolves the mesh from the logical axis. This avoids expecting a hard-coded `model` mesh, which breaks in some TPU workflows where the mesh is called `tp_mesh`. --- zml/kernels/flash_attention/nki/nki.zig | 2 +- .../paged_attention/mosaic_tpu/mosaic_tpu.zig | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/zml/kernels/flash_attention/nki/nki.zig b/zml/kernels/flash_attention/nki/nki.zig index 72f0a66d..6f1488ca 100644 --- a/zml/kernels/flash_attention/nki/nki.zig +++ b/zml/kernels/flash_attention/nki/nki.zig @@ -43,7 +43,7 @@ pub const Parameters = struct { /// Attention for Neuron. Both prefill and decode lower to NKI custom calls. pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, parameters: Parameters) zml.Tensor { - const tp_mesh = zml.Compiler.current().mesh(.model); + const tp_mesh = zml.Compiler.current().resolveMesh(.{.model}); const q_sharded = q.withPartitioning(tp_mesh, .{ .q = .replicated, .h = .model, .hd = .replicated }); const k_sharded = k.withPartitioning(tp_mesh, .{ .k = .replicated, .h = .model, .hd = .replicated }); const v_sharded = v.withPartitioning(tp_mesh, .{ .k = .replicated, .h = .model, .hd = .replicated }); diff --git a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig index f1cc85f4..fbe276d5 100644 --- a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig @@ -154,12 +154,13 @@ pub const mosaic_tpu = struct { .{ q.dim(.hd), target_head_dim }, ); return q.pad(0, .{ .hd = zml.Tensor.Pad{ .high = target_head_dim - q.dim(.hd) } }) - .withPartitioning(.model, .{ .h = .model }); + .withPartitioning(q.shape()._sharding.mesh.?, .{ .h = q.shape().partition(.h) }); } inline fn prepareInputs(parameters: Parameters, q: zml.Tensor, kv_pages: zml.Tensor) PreparedInputs { - const kv_pages_partitioned = kv_pages.withPartitioning(.model, .{ .hkv = .model }); - const q_merged = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(.model, .{ .h = .model }); + const mesh = zml.Compiler.current().resolveMesh(.{.model}); + const kv_pages_partitioned = kv_pages.withPartitioning(mesh, .{ .hkv = .model }); + const q_merged = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(mesh, .{ .h = .model }); const logical_head_dim: usize = @intCast(q_merged.dim(.hd)); const cache_head_dim: usize = @intCast(kv_pages_partitioned.dim(.hd)); stdx.debug.assert( @@ -168,15 +169,15 @@ pub const mosaic_tpu = struct { .{ cache_head_dim, logical_head_dim }, ); const q_ragged = alignQueryHeadDimForKernel(q_merged, kv_pages_partitioned.dim(.hd)); - const query_start_len = parameters.query_start_len.withPartitioning(.model, .{ .b = .replicated }); + const query_start_len = parameters.query_start_len.withPartitioning(mesh, .{ .b = .replicated }); return .{ .q = q_ragged, .kv_pages = kv_pages_partitioned, - .seq_lens = parameters.seq_lens.withPartitioning(.model, .{ ._0 = .replicated }), - .block_table = parameters.block_table.withPartitioning(.model, .{ ._0 = .replicated, ._1 = .replicated }), + .seq_lens = parameters.seq_lens.withPartitioning(mesh, .{ ._0 = .replicated }), + .block_table = parameters.block_table.withPartitioning(mesh, .{ ._0 = .replicated, ._1 = .replicated }), .query_start_len = query_start_len, - .num_seqs = activeSequenceCount(query_start_len).withPartitioning(.model, .{ ._0 = .replicated }), + .num_seqs = activeSequenceCount(query_start_len).withPartitioning(mesh, .{ ._0 = .replicated }), }; } From 3b0f556a742fb62479b7ef1708ace10efb79e2a4 Mon Sep 17 00:00:00 2001 From: fior512 <161968556+fior512@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:31:42 +0200 Subject: [PATCH 782/881] zml/nn: reducing sync in GDN decode (#932) Fast-path inside the forward of `GatedDeltaNet` in order to reduce syncs in decode path. Master vs this commit: - tok/s : 167 to 204 - cuStreamSynchronize call/tok : 37.24 to 1.24 - GPU idle/tok : 3504 us to 2240 us --- zml/nn.zig | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/zml/nn.zig b/zml/nn.zig index 7519a435..d0d038a2 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -1787,7 +1787,17 @@ pub const GatedDeltaNet = struct { .outputs = .zeroes(values.shape()), }; validateInitialState(gdn, state); - const final_state = ops.@"while"(GatedDeltaNet, gdn, state); + + const final_state = if (queries.dim(.s) == 1) blk: { + // avoids a `stablehlo.while`, which costs a host sync per call on GPU. + const next = gdn.body(state); + const run = gdn.cond(state); + break :blk GatedDeltaNet.State{ + .s = run.broad(state.s.shape()).select(next.s, state.s), + .outputs = run.broad(state.outputs.shape()).select(next.outputs, state.outputs), + .step = run.select(next.step, state.step), + }; + } else ops.@"while"(GatedDeltaNet, gdn, state); return .{ .outputs = final_state.outputs, From 82e800dc5cbc53b02ab9d9cba809bf2ef4d6216b Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 14 Oct 2026 15:18:04 +0200 Subject: [PATCH 783/881] workspace: bump ZML plugins (#937) This introduce the new tiling path support. --- platforms/cpu/cpu.bzl | 12 ++++++------ platforms/cuda/cuda.bzl | 8 ++++---- platforms/furiosa/furiosa.bzl | 4 ++-- platforms/metal/metal.bzl | 4 ++-- platforms/rocm/rocm.bzl | 4 ++-- zml/Compiler.zig | 5 ----- 6 files changed, 16 insertions(+), 21 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 583fcad7..5b26d9e7 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -42,22 +42,22 @@ def _cpu_plugin_impl(mctx): http_archive( name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "6397644e129e198431e56d8fee6871d5d9a71550dc6ca03a5abbf0a9ef3a3eaf", - url = "https://mirror.zml.ai/plugins/202610091329.178.1.75e89af70664/zml-cpu-linux-amd64.tar.zst", + sha256 = "0e3477f5c4b5f2446b47b905184987176886656e9f8bd0b7b0ff6020022f26b3", + url = "https://mirror.zml.ai/plugins/202610141143.207.1.b026d193d9cb/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "8276a919970b9d5c2edf4cf6f962c689fa67c45d325f70849bfdd38403f8fb8b", - url = "https://mirror.zml.ai/plugins/202610091329.178.1.75e89af70664/zml-cpu-darwin-amd64.tar.zst", + sha256 = "ff9437164e2f7b7714d8ef6bf6c88d593cab076ff9bcf55eaa83f3a66753a49b", + url = "https://mirror.zml.ai/plugins/202610141143.207.1.b026d193d9cb/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "e08eb1986a8705c66f065d15c0cacf228219118aca7c44148e47502f011982b1", - url = "https://mirror.zml.ai/plugins/202610091329.178.1.75e89af70664/zml-cpu-darwin-arm64.tar.zst", + sha256 = "3c1569e4cd35803360750fb87b93c9f381e49fb36943ef6d4de8b445548b3797", + url = "https://mirror.zml.ai/plugins/202610141143.207.1.b026d193d9cb/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 24dd691e..e74f792b 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -224,12 +224,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "82968f83dc8af1b8942e28a15e0d4bd0d5d3c1ecf512fb86f9f3d883288ae064", - "url": "https://mirror.zml.ai/plugins/202610091329.178.1.75e89af70664/zml-cuda-linux-amd64.tar.zst", + "sha256": "226e8b04611efe6c8ac3ceaf3010499b5957d498e585c970afb69c2ec47ff9fe", + "url": "https://mirror.zml.ai/plugins/202610141143.207.1.b026d193d9cb/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "3605e422a0659c2cef9a59d35ee2a962caa78ef591160cfd780110c49002db5a", - "url": "https://mirror.zml.ai/plugins/202610091329.178.1.75e89af70664/zml-cuda-linux-arm64.tar.zst", + "sha256": "5b55fcd3173485b0e96034bbbf078e354c393d938aadb2ab8fa9bf658b384759", + "url": "https://mirror.zml.ai/plugins/202610141143.207.1.b026d193d9cb/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index 1e0ba026..970d2249 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202610111616.184.1.0536437b83ad/zml-furiosa-linux-amd64.tar.zst", - sha256 = "730469a37671f94ebf095a245d96a9c8585954d92a98698ae675bf73c765e75d", + url = "https://mirror.zml.ai/plugins/202610141143.207.1.b026d193d9cb/zml-furiosa-linux-amd64.tar.zst", + sha256 = "1225cb3eecc40745669d6e2d8280830d5966a6abe33f7aa1e43b6e5197b6e08a", ) return mctx.extension_metadata( diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index b0f9d22d..38839d2f 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "a1262b90e5c746097447b24c45ee69f00d2923c53f5491d73ff06c3c52377fee", - url = "https://mirror.zml.ai/plugins/202610091329.178.1.75e89af70664/zml-metal-darwin-arm64.tar.zst", + sha256 = "b7c1a5c4b30fb70c7901f41a003da6cb5bb4bb2f37f965b92c14656ae7cc6dca", + url = "https://mirror.zml.ai/plugins/202610141143.207.1.b026d193d9cb/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index ec754177..fcf0037b 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610091329.178.1.75e89af70664/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "980c0d86a31b8ee58e8efc7b2464752cc3982c040f12a3c3582ba1b9289e2bce" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610141143.207.1.b026d193d9cb/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "ec895a112ec2f36e4f73a845498235748da47032f32c218605a39c25d39c97ba" def _rocm_package_name(name): return name + _ROCM_VERSION diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 15685740..6c3e6421 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -945,8 +945,6 @@ fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.Load try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_nccl_user_buffers", true, upb_arena); // Enable for both Blackwell+ and Ampere+ // try setXlaOverrideFlag(overrides_map, "xla_gpu_cudnn_gemm_fusion_level", 2, upb_arena); - // Need to adapt XLA to support that (for other platforms too) - try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_enable_tiling_propagation", false, upb_arena); }, .rocm => { try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_scheduling_mode", "CONCURRENT", upb_arena); @@ -961,11 +959,9 @@ fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.Load // Enable user buffers to assign color 1 and receive the reclaim exemption: // https://github.com/openxla/xla/pull/46029 try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_nccl_user_buffers", true, upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_enable_tiling_propagation", false, upb_arena); }, .metal => { try setXlaOverrideFlag(overrides_map, "xla_gpu_metal_fast_math", false, upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_enable_tiling_propagation", false, upb_arena); }, .oneapi => { // More efficient for the allgather/broadcast implementation of the collective permute. @@ -976,7 +972,6 @@ fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.Load // Grouped GEMM still needs a oneDNN lowering. try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_ragged_dot_grouped_gemm", false, upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_enable_tiling_propagation", false, upb_arena); }, else => {}, } From fe25f16b460e27af32041f8b0bb12796e354d6b4 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Wed, 14 Oct 2026 18:09:21 +0200 Subject: [PATCH 784/881] zml/buffer: replace emptyShell by PjrtBufferMetadata.init (#935) --- zml/buffer.zig | 48 ++++++++++++++++++----------------- zml/buffer/HostAccessible.zig | 11 +++++--- 2 files changed, 33 insertions(+), 26 deletions(-) diff --git a/zml/buffer.zig b/zml/buffer.zig index 10f0f8d1..03dd5b3d 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -141,7 +141,12 @@ pub const Buffer = struct { data_: []const u8, opts: FromOptions, ) !Buffer { - var buffer, const metadata = emptyShell(platform, shape_); + var buffer: Buffer = .{ + ._platform = platform, + ._shape = shape_, + ._shards = .empty, + }; + const metadata: PjrtBufferMetadata = .init(platform, shape_); errdefer for (buffer._shards.slice()) |shard| { shard.deinit(platform.pjrt_api); @@ -220,7 +225,12 @@ pub const Buffer = struct { opts: UnitializedOptions, ) !Buffer { std.log.debug("uninitialized {f}", .{shape_}); - var buffer, const metadata = emptyShell(platform, shape_); + var buffer: Buffer = .{ + ._platform = platform, + ._shape = shape_, + ._shards = .empty, + }; + const metadata: PjrtBufferMetadata = .init(platform, shape_); errdefer for (buffer._shards.slice()) |shard| { shard.deinit(platform.pjrt_api); }; @@ -398,38 +408,30 @@ test "device round-trip" { } } -const PjrtBufferMetadata = struct { +pub const PjrtBufferMetadata = struct { shape: Shape, ty: pjrt.BufferType, placement: Sharding.Placement, layout: pjrt.MemoryLayout, -}; -/// Inline because the layout may point to stack memory, as with -/// `Platform.defaultMemoryLayout`. Outside `Buffer` so that -/// `Buffer.HostAccessible` can use it without it being part of the API: -/// zml.zig only exports `Buffer` from this file. -pub inline fn emptyShell(platform: *const Platform, shape_: Shape) struct { Buffer, PjrtBufferMetadata } { - const buf: Buffer = .{ - ._platform = platform, - ._shape = shape_, - ._shards = .empty, - }; + /// Inline because the layout may point to stack memory, as with + /// `Platform.defaultMemoryLayout`. Outside `Buffer` so that + /// `Buffer.HostAccessible` can use it without it being part of the API: + /// zml.zig only exports `Buffer` from this file. + pub inline fn init(platform: *const Platform, shape_: Shape) PjrtBufferMetadata { - // Shards use the PJRT shape, which packs sub-byte elements. - const packed_shape = shape_.packedShape(); - const placement = placementOrPanic(platform, packed_shape); + // Shards use the PJRT shape, which packs sub-byte elements. + const packed_shape = shape_.packedShape(); + const placement = placementOrPanic(platform, packed_shape); - return .{ - buf, - .{ + return .{ .shape = packed_shape, .ty = pjrtx.bufferTypeFromDtype(packed_shape.dtype()), .placement = placement, .layout = platform.defaultMemoryLayout(placement.shape.dims(), packed_shape.dtype()), - }, - }; -} + }; + } +}; fn placementOrPanic(platform: *const Platform, shape: Shape) Sharding.Placement { var sh = shape; diff --git a/zml/buffer/HostAccessible.zig b/zml/buffer/HostAccessible.zig index 76dab0ba..2d326e75 100644 --- a/zml/buffer/HostAccessible.zig +++ b/zml/buffer/HostAccessible.zig @@ -17,7 +17,7 @@ const pjrt = @import("pjrt"); const stdx = @import("stdx"); const Buffer = @import("../buffer.zig").Buffer; -const emptyShell = @import("../buffer.zig").emptyShell; +const PjrtBufferMetadata = @import("../buffer.zig").PjrtBufferMetadata; const Platform = @import("../platform.zig").Platform; const Shape = @import("../shape.zig").Shape; const Sharding = @import("../Sharding.zig"); @@ -242,10 +242,15 @@ pub fn init( .cpu, .cuda, .rocm => donation == .undonatable, else => false, }; - const api = platform.pjrt_api; - var buffer, const metadata = emptyShell(platform, shape_); + var buffer: Buffer = .{ + ._platform = platform, + ._shape = shape_, + ._shards = .empty, + }; + const metadata: PjrtBufferMetadata = .init(platform, shape_); const devices = platform.physical_mesh.devices_in_canonical_order; + const api = platform.pjrt_api; var allocations: HostPinnedAllocations = .empty; errdefer allocations.deinit(allocator); From 016376f2f4721fc9b8969a3cac2ae84992113421 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Wed, 14 Oct 2026 18:29:02 +0200 Subject: [PATCH 785/881] workspace: remove flashattn (#939) --- MODULE.bazel | 2 - platforms/cuda/flashattn/BUILD.bazel | 20 --- platforms/cuda/flashattn/flashattn.zig | 125 ------------------ third_party/flashattn/BUILD.bazel | 0 .../flashattn/flashattention.BUILD.bazel | 13 -- third_party/flashattn/repo.bzl | 30 ----- third_party/non_module_deps.bzl | 2 - zml/BUILD.bazel | 1 - 8 files changed, 193 deletions(-) delete mode 100644 platforms/cuda/flashattn/BUILD.bazel delete mode 100644 platforms/cuda/flashattn/flashattn.zig delete mode 100644 third_party/flashattn/BUILD.bazel delete mode 100644 third_party/flashattn/flashattention.BUILD.bazel delete mode 100644 third_party/flashattn/repo.bzl diff --git a/MODULE.bazel b/MODULE.bazel index c0cddf52..1acbbf6d 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -305,8 +305,6 @@ use_repo( "com_google_sentencepiece", "cuda_tile", "cute_ir", - "flashattn_linux_amd64", - "flashattn_linux_arm64", "flashinfer_cutlass_moe_linux_amd64", "flashinfer_cutlass_moe_linux_arm64", "flydsl", diff --git a/platforms/cuda/flashattn/BUILD.bazel b/platforms/cuda/flashattn/BUILD.bazel deleted file mode 100644 index 34bf0b14..00000000 --- a/platforms/cuda/flashattn/BUILD.bazel +++ /dev/null @@ -1,20 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "flashattn", - tags = ["manual"], - import_name = "platforms/cuda/flashattn", - main = "flashattn.zig", - visibility = ["//visibility:public"], - deps = [ - "//bazel", - "@rules_zig//zig/runfiles", - ] + select({ - "@llvm//platforms/config:linux_x86_64": ["@flashattn_linux_amd64//:c"], - "@llvm//platforms/config:linux_aarch64": ["@flashattn_linux_arm64//:c"], - }), - data = select({ - "@llvm//platforms/config:linux_x86_64": ["@flashattn_linux_amd64//:libflashattn"], - "@llvm//platforms/config:linux_aarch64": ["@flashattn_linux_arm64//:libflashattn"], - }), -) diff --git a/platforms/cuda/flashattn/flashattn.zig b/platforms/cuda/flashattn/flashattn.zig deleted file mode 100644 index 7d9c72b0..00000000 --- a/platforms/cuda/flashattn/flashattn.zig +++ /dev/null @@ -1,125 +0,0 @@ -const std = @import("std"); - -const bazel = @import("bazel"); -const bazel_builtin = @import("bazel_builtin"); -const builtin = @import("builtin"); -const c = @import("c"); -pub const FA2MhaFwdParams = c.FA2MhaFwdParams; -pub const FA2MhaVarlenFwdParams = c.FA2MhaVarlenFwdParams; -pub const FA3MhaFwdParams = c.FA3MhaFwdParams; -const runfiles = @import("runfiles"); - -pub const Tensor = struct { - inner: c.FlashattnTensor, - - pub fn init(ptr: *anyopaque, dims: []const i64, strides: []const i64, dtype: DataType) Tensor { - const rank = dims.len; - var ret: Tensor = .{ - .inner = .{ - .ptr = ptr, - .rank = @intCast(rank), - .dims = undefined, - .strides = undefined, - .dtype = @intCast(@backingInt(dtype)), - }, - }; - @memcpy(ret.inner.dims[0..rank], dims); - @memcpy(ret.inner.strides[0..rank], strides); - return ret; - } -}; - -pub const DataType = enum(c_int) { - f32 = c.CAPI_FLOAT, - f16 = c.CAPI_FLOAT16, - bf16 = c.CAPI_BFLOAT16, - i32 = c.CAPI_INT32, - i8 = c.CAPI_INT8, -}; - -const Fa2MhaFwdFunc = *const fn ( - q: *const Tensor, - k: *const Tensor, - v: *const Tensor, - out: *const Tensor, - softmax_lse: ?*const Tensor, - alibi_slopes_: ?*const Tensor, - softmax_lse_accum: ?*const Tensor, - out_accum: ?*const Tensor, - params: ?*const FA2MhaFwdParams, - stream: ?*anyopaque, -) callconv(.c) void; - -const Fa2MhaVarlenFwdFunc = *const fn ( - q: *const Tensor, - k: *const Tensor, - v: *const Tensor, - out: *const Tensor, - cu_seqlens_q: ?*const Tensor, - cu_seqlens_k: ?*const Tensor, - seqused_k: ?*const Tensor, - block_table: ?*const Tensor, - softmax_lse: ?*const Tensor, - alibi_slopes_: ?*const Tensor, - softmax_lse_accum: ?*const Tensor, - out_accum: ?*const Tensor, - params: ?*const FA2MhaVarlenFwdParams, - stream: ?*anyopaque, -) callconv(.c) void; - -const Fa3MhaFwdFunc = *const fn ( - q: ?*const Tensor, - k: ?*const Tensor, - v: ?*const Tensor, - out: ?*const Tensor, - cu_seqlens_q: ?*const Tensor, - cu_seqlens_k: ?*const Tensor, - seqused_q: ?*const Tensor, - seqused_k: ?*const Tensor, - page_table: ?*const Tensor, - q_descale: ?*const Tensor, - k_descale: ?*const Tensor, - v_descale: ?*const Tensor, - softmax_lse: ?*const Tensor, - softmax_lse_accum: ?*const Tensor, - out_accum: ?*const Tensor, - scheduler_metadata: ?*const Tensor, - s_aux: ?*const Tensor, - cp_tot_seqused_k: ?*const Tensor, - params: ?*const FA3MhaFwdParams, - stream: ?*anyopaque, -) callconv(.c) void; - -pub var fa2_mha_fwd: Fa2MhaFwdFunc = undefined; -pub var fa2_mha_varlen_fwd: Fa2MhaVarlenFwdFunc = undefined; -pub var fa3_mha_fwd: Fa3MhaFwdFunc = undefined; - -fn findFlashattnLibrary( - r: anytype, - buffer: *[std.Io.Dir.max_path_bytes]u8, -) !?[]const u8 { - const candidate = switch (builtin.cpu.arch) { - .aarch64 => "flashattn_linux_arm64/lib/libflashattn.so", - .x86_64 => "flashattn_linux_amd64/lib/libflashattn.so", - else => return null, - }; - return try r.rlocation(candidate, buffer); -} - -pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { - _ = io; - _ = allocator; - - const r = try bazel.runfiles(bazel_builtin.current_repository); - - var buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; - const library = (try findFlashattnLibrary(r, &buffer)) orelse return error.NotFound; - var lib = std.DynLib.open(library) catch |err| { - std.log.err("Failed to open libflashattn.so: {any}", .{err}); - return err; - }; - - fa2_mha_varlen_fwd = lib.lookup(Fa2MhaVarlenFwdFunc, "fa2_mha_varlen_fwd") orelse return error.NotFound; - fa2_mha_fwd = lib.lookup(Fa2MhaFwdFunc, "fa2_mha_fwd") orelse return error.NotFound; - fa3_mha_fwd = lib.lookup(Fa3MhaFwdFunc, "fa3_mha_fwd") orelse return error.NotFound; -} diff --git a/third_party/flashattn/BUILD.bazel b/third_party/flashattn/BUILD.bazel deleted file mode 100644 index e69de29b..00000000 diff --git a/third_party/flashattn/flashattention.BUILD.bazel b/third_party/flashattn/flashattention.BUILD.bazel deleted file mode 100644 index 030e3d3f..00000000 --- a/third_party/flashattn/flashattention.BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_cc//cc:cc_library.bzl", "cc_library") - -filegroup( - name = "libflashattn", - visibility = ["//visibility:public"], - srcs = ["lib/libflashattn.so"], -) - -cc_library( - name = "c", - hdrs = ["include/capi.h"], - visibility = ["//visibility:public"], -) diff --git a/third_party/flashattn/repo.bzl b/third_party/flashattn/repo.bzl deleted file mode 100644 index bcad73fe..00000000 --- a/third_party/flashattn/repo.bzl +++ /dev/null @@ -1,30 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") - -_FLASHATTN_RELEASE = "v0.0.6-rc9" - -_FLASHATTN_ASSETS = { - "amd64": { - "sha256": "cb341c14ee0fc5f02a262497cbc80b552c23bce594d087a2aa278f14cfc82ef2", - "url": "https://github.com/zml/flash-attention/releases/download/{release}/flash-attention_linux-amd64.tar.gz", - }, - "arm64": { - "sha256": "55feeccb4b3728dec08da2aa051109d0b9fbd8c0d0786c878976c1b713e3b407", - "url": "https://github.com/zml/flash-attention/releases/download/{release}/flash-attention_linux-arm64.tar.gz", - }, -} - -def flasshattn_archive(name, arch): - asset = _FLASHATTN_ASSETS[arch] - http_archive( - name = name, - url = asset["url"].format(release = _FLASHATTN_RELEASE), - sha256 = asset["sha256"], - build_file = "//:third_party/flashattn/flashattention.BUILD.bazel", - ) - -def repo(): - flasshattn_archive("flashattn_linux_amd64", "amd64") - flasshattn_archive("flashattn_linux_arm64", "arm64") - - #new_local_repository(name = "flashattn", build_file="//:third_party/flashattn/BUILD.bazel", path="/home/corendos/flashattn/") diff --git a/third_party/non_module_deps.bzl b/third_party/non_module_deps.bzl index bd792d4c..876b0db2 100644 --- a/third_party/non_module_deps.bzl +++ b/third_party/non_module_deps.bzl @@ -3,7 +3,6 @@ load("//third_party/cloud_accelerator_diagnostics:repo.bzl", cloud_accelerator_d load("//third_party/com_google_sentencepiece:repo.bzl", com_google_sentencepiece = "repo") load("//third_party/cuda_tile:repo.bzl", cuda_tile = "repo") load("//third_party/cute_ir:repo.bzl", cute_ir = "repo") -load("//third_party/flashattn:repo.bzl", flashattn = "repo") load("//third_party/flashinfer_cutlass_moe:repo.bzl", flashinfer_cutlass_moe = "repo") load("//third_party/flydsl:repo.bzl", flydsl = "repo") load("//third_party/iree:repo.bzl", iree = "repo") @@ -27,7 +26,6 @@ def _non_module_deps_impl(mctx): xla() arocc() translate_c() - flashattn() flashinfer_cutlass_moe() linenoise() stb() diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index ec9ff4e0..dff0bb0f 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -181,7 +181,6 @@ zig_library( "//conditions:default": [], }) + select({ "//platforms:cuda.enabled": [ - "//platforms/cuda/flashattn", "//platforms/cuda/flashinfer_cutlass_moe", ], "//conditions:default": [], From cddc9cf60743e3c4e7e0d5400d5ab6df909630df Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Wed, 14 Oct 2026 19:32:13 +0200 Subject: [PATCH 786/881] zml/io: load Furiosa weights through pinned host memory (#928) Staging blocks are uninitialized `.host_pinned` buffers, and tensors are sent with the direct writer. The plugin backs `.host_pinned` with SDK pinned (dma-buf) memory, which the SDK recognizes by address and transfers without pinning pages on every request. Pinned memory comes from `/dev/dma_heap/system`, so the loading user needs access to it. --- platforms/cpu/cpu.bzl | 6 +++--- platforms/cuda/cuda.bzl | 4 ++-- platforms/furiosa/furiosa.bzl | 4 ++-- platforms/furiosa/furiosa.zig | 16 ++++++++++++++++ platforms/metal/metal.bzl | 2 +- platforms/rocm/rocm.bzl | 2 +- zml/io.zig | 4 ++-- zml/mem.zig | 4 ++-- 8 files changed, 29 insertions(+), 13 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 5b26d9e7..6994888d 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -43,21 +43,21 @@ def _cpu_plugin_impl(mctx): name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, sha256 = "0e3477f5c4b5f2446b47b905184987176886656e9f8bd0b7b0ff6020022f26b3", - url = "https://mirror.zml.ai/plugins/202610141143.207.1.b026d193d9cb/zml-cpu-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610141523.216.1.9e9d3e15652f/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "ff9437164e2f7b7714d8ef6bf6c88d593cab076ff9bcf55eaa83f3a66753a49b", - url = "https://mirror.zml.ai/plugins/202610141143.207.1.b026d193d9cb/zml-cpu-darwin-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610141523.216.1.9e9d3e15652f/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "3c1569e4cd35803360750fb87b93c9f381e49fb36943ef6d4de8b445548b3797", - url = "https://mirror.zml.ai/plugins/202610141143.207.1.b026d193d9cb/zml-cpu-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610141523.216.1.9e9d3e15652f/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index e74f792b..b00a8d34 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -225,11 +225,11 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { "sha256": "226e8b04611efe6c8ac3ceaf3010499b5957d498e585c970afb69c2ec47ff9fe", - "url": "https://mirror.zml.ai/plugins/202610141143.207.1.b026d193d9cb/zml-cuda-linux-amd64.tar.zst", + "url": "https://mirror.zml.ai/plugins/202610141523.216.1.9e9d3e15652f/zml-cuda-linux-amd64.tar.zst", }, "arm64": { "sha256": "5b55fcd3173485b0e96034bbbf078e354c393d938aadb2ab8fa9bf658b384759", - "url": "https://mirror.zml.ai/plugins/202610141143.207.1.b026d193d9cb/zml-cuda-linux-arm64.tar.zst", + "url": "https://mirror.zml.ai/plugins/202610141523.216.1.9e9d3e15652f/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index 970d2249..b7f6e543 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202610141143.207.1.b026d193d9cb/zml-furiosa-linux-amd64.tar.zst", - sha256 = "1225cb3eecc40745669d6e2d8280830d5966a6abe33f7aa1e43b6e5197b6e08a", + url = "https://mirror.zml.ai/plugins/202610141523.216.1.9e9d3e15652f/zml-furiosa-linux-amd64.tar.zst", + sha256 = "23d8c7c3aa210dae91ca56bb66bcaa0630d40ea2048f3971bc4ca436beb4b0b3", ) return mctx.extension_metadata( diff --git a/platforms/furiosa/furiosa.zig b/platforms/furiosa/furiosa.zig index b8057c71..ebbd9e3c 100644 --- a/platforms/furiosa/furiosa.zig +++ b/platforms/furiosa/furiosa.zig @@ -19,8 +19,24 @@ pub fn isEnabled() bool { return platforms_options.furiosa_enabled; } +fn hasFuriosaDevice(io: std.Io) bool { + std.Io.Dir.accessAbsolute(io, "/dev/rngd", .{ .read = true }) catch return false; + return true; +} + +fn hasDmaHeapAccess(io: std.Io) bool { + std.Io.Dir.accessAbsolute(io, "/dev/dma_heap/system", .{ .read = true }) catch return false; + return true; +} + pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { if (comptime !isEnabled() or builtin.os.tag != .linux or builtin.cpu.arch != .x86_64) return error.Unavailable; + if (!hasFuriosaDevice(io)) { + return error.Unavailable; + } + if (!hasDmaHeapAccess(io)) { + std.debug.panic("Furiosa needs read access to /dev/dma_heap/system for pinned host memory", .{}); + } const r = try bazel.runfiles(bazel_builtin.current_repository); var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 38839d2f..85bc0d86 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -5,7 +5,7 @@ def _metal_impl(mctx): name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", sha256 = "b7c1a5c4b30fb70c7901f41a003da6cb5bb4bb2f37f965b92c14656ae7cc6dca", - url = "https://mirror.zml.ai/plugins/202610141143.207.1.b026d193d9cb/zml-metal-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610141523.216.1.9e9d3e15652f/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index fcf0037b..5e654733 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,7 +11,7 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610141143.207.1.b026d193d9cb/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610141523.216.1.9e9d3e15652f/zml-rocm-linux-amd64.tar.zst" _PLUGIN_ROCM_SHA256 = "ec895a112ec2f36e4f73a845498235748da47032f32c218605a39c25d39c97ba" def _rocm_package_name(name): diff --git a/zml/io.zig b/zml/io.zig index b94f1ff6..eb9596b5 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -686,10 +686,10 @@ pub const MemoryWriter = union(enum) { memory: Memory.Kind, ) !MemoryWriter { return switch (platform.target) { - .cuda, .rocm, .oneapi => .{ + .cuda, .rocm, .oneapi, .furiosa => .{ .direct = try .init(allocator, io, platform, pools, dma_allocators, dma_chunk_size, shape, buffer, memory), }, - .tpu, .neuron, .cpu, .metal, .furiosa => if (memory == .host_pinned) + .tpu, .neuron, .cpu, .metal => if (memory == .host_pinned) std.debug.panic("Host pinned memory is not supported on {}", .{platform.target}) else .{ .buffered = try .init(allocator, io, platform, shape, buffer) }, diff --git a/zml/mem.zig b/zml/mem.zig index 58067e07..a64ae968 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -22,8 +22,8 @@ pub const DmaAllocator = union(enum) { pub fn init(parent: std.mem.Allocator, device: *const Device) DmaAllocator { return switch (device.platform.target) { .cuda => .{ .dmam = .init(parent, device.platform) }, - .rocm, .oneapi, .tpu => .{ .uib = .init(device.memory(.host_pinned).?) }, - .cpu, .neuron, .metal, .furiosa => .{ .passthrough = parent }, + .rocm, .oneapi, .tpu, .furiosa => .{ .uib = .init(device.memory(.host_pinned).?) }, + .cpu, .neuron, .metal => .{ .passthrough = parent }, }; } From e385d9c272dcc99a5a735b06be4108e0c872416b Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Thu, 15 Oct 2026 14:45:20 +0200 Subject: [PATCH 787/881] workspace: bump plugins to fix Metal GDN overflow (#940) Metal GDN used to overflow on state indexes since the rework. --- platforms/cpu/cpu.bzl | 6 +++--- platforms/cuda/cuda.bzl | 4 ++-- platforms/furiosa/furiosa.bzl | 2 +- platforms/metal/metal.bzl | 4 ++-- platforms/rocm/rocm.bzl | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 6994888d..a83719ed 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -43,21 +43,21 @@ def _cpu_plugin_impl(mctx): name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, sha256 = "0e3477f5c4b5f2446b47b905184987176886656e9f8bd0b7b0ff6020022f26b3", - url = "https://mirror.zml.ai/plugins/202610141523.216.1.9e9d3e15652f/zml-cpu-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610150915.220.1.f2470651679c/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "ff9437164e2f7b7714d8ef6bf6c88d593cab076ff9bcf55eaa83f3a66753a49b", - url = "https://mirror.zml.ai/plugins/202610141523.216.1.9e9d3e15652f/zml-cpu-darwin-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610150915.220.1.f2470651679c/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "3c1569e4cd35803360750fb87b93c9f381e49fb36943ef6d4de8b445548b3797", - url = "https://mirror.zml.ai/plugins/202610141523.216.1.9e9d3e15652f/zml-cpu-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610150915.220.1.f2470651679c/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index b00a8d34..8604d025 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -225,11 +225,11 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { "sha256": "226e8b04611efe6c8ac3ceaf3010499b5957d498e585c970afb69c2ec47ff9fe", - "url": "https://mirror.zml.ai/plugins/202610141523.216.1.9e9d3e15652f/zml-cuda-linux-amd64.tar.zst", + "url": "https://mirror.zml.ai/plugins/202610150915.220.1.f2470651679c/zml-cuda-linux-amd64.tar.zst", }, "arm64": { "sha256": "5b55fcd3173485b0e96034bbbf078e354c393d938aadb2ab8fa9bf658b384759", - "url": "https://mirror.zml.ai/plugins/202610141523.216.1.9e9d3e15652f/zml-cuda-linux-arm64.tar.zst", + "url": "https://mirror.zml.ai/plugins/202610150915.220.1.f2470651679c/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index b7f6e543..30da0600 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,7 +6,7 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202610141523.216.1.9e9d3e15652f/zml-furiosa-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610150915.220.1.f2470651679c/zml-furiosa-linux-amd64.tar.zst", sha256 = "23d8c7c3aa210dae91ca56bb66bcaa0630d40ea2048f3971bc4ca436beb4b0b3", ) diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 85bc0d86..fabf2dfe 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "b7c1a5c4b30fb70c7901f41a003da6cb5bb4bb2f37f965b92c14656ae7cc6dca", - url = "https://mirror.zml.ai/plugins/202610141523.216.1.9e9d3e15652f/zml-metal-darwin-arm64.tar.zst", + sha256 = "2413f6952055d9d336a7236a8b9f2a5ceba2f77048000662bfde3b202244fbac", + url = "https://mirror.zml.ai/plugins/202610150915.220.1.f2470651679c/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 5e654733..0368b288 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,7 +11,7 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610141523.216.1.9e9d3e15652f/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610150915.220.1.f2470651679c/zml-rocm-linux-amd64.tar.zst" _PLUGIN_ROCM_SHA256 = "ec895a112ec2f36e4f73a845498235748da47032f32c218605a39c25d39c97ba" def _rocm_package_name(name): From bc9d6c2c8d3ed0a113bbc0397f89e86ae110a1db Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Fri, 16 Oct 2026 10:31:34 +0200 Subject: [PATCH 788/881] workspace: bump ZML plugins and CUDA/ROCm runtimes (#942) Update XLA plugins to release 202610151602.227.1.e215eb02db73 for CPU, CUDA, ROCm, Metal, and Furiosa. Updated CUDA to 13.4.3 and ROCm with TheRock 10.1. Now we use CUDA compatibility libraries and NCCL 2.32.3 through NVIDIA's APT repositories. Starting with CUDA 13.4's redistribution manifest omits the generic cuda_compat package (https://docs.nvidia.com/deploy/cuda-compatibility/why-cuda-compatibility.html). So we resolve it through NVIDIA's APT repositories instead. These repositories expose flat package indexes at separate x86_64 and SBSA URLs, so we extend rules_distroless to support flat sources. For NCCL we leverage the same APT source workflow, replacing separate cryptic wheel URLs. --- MODULE.bazel | 1 + platforms/cpu/cpu.bzl | 12 +- platforms/cuda/cuda.bzl | 120 +- platforms/cuda/libzml_cuda.BUILD.bazel | 2 +- platforms/cuda/packages.lock.json | 108 + platforms/cuda/packages.yaml | 8 + platforms/furiosa/furiosa.bzl | 4 +- platforms/metal/metal.bzl | 4 +- platforms/rocm/packages.lock.json | 3547 ++++++++--------- platforms/rocm/packages.yaml | 2 +- platforms/rocm/rocm.bzl | 4 +- .../rules_distroless/flat-repositories.patch | 59 + third_party/xla/repo.bzl | 2 +- 13 files changed, 1818 insertions(+), 2055 deletions(-) create mode 100644 third_party/rules_distroless/flat-repositories.patch diff --git a/MODULE.bazel b/MODULE.bazel index 1acbbf6d..e9c4c9d3 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -37,6 +37,7 @@ single_version_override( patch_strip = 1, patches = [ "//:third_party/rules_distroless/accept-nospace-version-constraints.patch", + "//:third_party/rules_distroless/flat-repositories.patch", ], version = "0.8.0", ) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index a83719ed..c30f125b 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -42,22 +42,22 @@ def _cpu_plugin_impl(mctx): http_archive( name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "0e3477f5c4b5f2446b47b905184987176886656e9f8bd0b7b0ff6020022f26b3", - url = "https://mirror.zml.ai/plugins/202610150915.220.1.f2470651679c/zml-cpu-linux-amd64.tar.zst", + sha256 = "9d82f2612decf7066e1c66e3da8fa8883f5f3714fe3aaa6b1ee6041b435552e6", + url = "https://mirror.zml.ai/plugins/202610151602.227.1.e215eb02db73/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "ff9437164e2f7b7714d8ef6bf6c88d593cab076ff9bcf55eaa83f3a66753a49b", - url = "https://mirror.zml.ai/plugins/202610150915.220.1.f2470651679c/zml-cpu-darwin-amd64.tar.zst", + sha256 = "f365f0092a6b493888d3ce87d3bc859c02116df455037f609414e7bfeac0873e", + url = "https://mirror.zml.ai/plugins/202610151602.227.1.e215eb02db73/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "3c1569e4cd35803360750fb87b93c9f381e49fb36943ef6d4de8b445548b3797", - url = "https://mirror.zml.ai/plugins/202610150915.220.1.f2470651679c/zml-cpu-darwin-arm64.tar.zst", + sha256 = "3ebd6bec2c05599f250415b309ffcd5f34e5c1c37b7b806eeb35c88254e8eb16", + url = "https://mirror.zml.ai/plugins/202610151602.227.1.e215eb02db73/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 8604d025..3b18d71c 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -6,17 +6,18 @@ load("//platforms:packages.bzl", "packages") ARCHS = ["linux-x86_64", "linux-sbsa"] CUDA_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cuda/redist/" -CUDA_VERSION = "13.3.0" -CUDA_VARIANT = "cuda13.3" -CUDA_REDIST_JSON_SHA256 = "507eddaab1360336bc0fe17b77552e0b7dfe1e74da888671c3a2f5fad7775db1" +CUDA_VERSION = "13.4.3" +CUDA_MAJOR_MINOR = ".".join(CUDA_VERSION.split(".")[:2]) +CUDA_VARIANT = "cuda{}".format(CUDA_MAJOR_MINOR) +CUDA_REDIST_JSON_SHA256 = "8e99511eee1bd78664bdb6cf43dc17a7ba6afeab78c8162bd5f23571ed6423c6" CUDNN_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cudnn/redist/" -CUDNN_VERSION = "9.24.0" -CUDNN_REDIST_JSON_SHA256 = "89a0626f1925d9f4ceb8d6bc0b2de160b30e92afaae5471e6cfbb50234ae5488" +CUDNN_VERSION = "9.27.0" +CUDNN_REDIST_JSON_SHA256 = "d3f870db59da3fd12581aeca5fda381afa50ce1d4d9cccc8dcd79d7a628c22a2" NVSHMEM_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/nvshmem/redist/" -NVSHMEM_VERSION = "3.6.5" -NVSHMEM_REDIST_JSON_SHA256 = "afbf1ad5c4174c25a66cadd5b925d6814e679e8f836ef1fee1442afb6cb7fdd3" +NVSHMEM_VERSION = "3.8.0" +NVSHMEM_REDIST_JSON_SHA256 = "5920f88ccae8261083570f740463eddfdf8ad2708d88ebede380ce55d84eaf6c" _BUILD_FILE_DEFAULT_VISIBILITY = """\ package(default_visibility = ["//visibility:public"]) @@ -34,6 +35,41 @@ CUDA_COMPAT_FILES = [ "libnvidia-ptxjitcompiler.so.1", ] +def _cuda_compat_packages(driver_version): + return [ + packages.patchelf( + name = "{}.patchelf".format(file), + set_rpath = "$ORIGIN", + src = "compat/{}".format(file), + soname = file, + ) + for file in CUDA_COMPAT_FILES + ] + [ + packages.patchelf( + name = "nvcuextend", + set_rpath = "$ORIGIN", + src = "compat/libnvcuextend.so.1", + soname = "libnvcuextend.so.1", + ), + """filegroup( + name = "cuda_compat", + srcs = [ + "compat/libnvidia-gpucomp.so.{driver_version}", + "compat/libnvidia-tileiras.so.{driver_version}", + ] + {} + select({{ + "@llvm//platforms/config:linux_x86_64": [ + "compat/libnvidia-pkcs11-openssl3.so.{driver_version}", + ], + "@llvm//platforms/config:linux_aarch64": [ + ":nvcuextend", + ], + }}), + )""".format(repr([ + ":{}.patchelf".format(file) + for file in CUDA_COMPAT_FILES + ]), driver_version = driver_version), + ] + CUDA_PACKAGES = { "cuda_nvml_dev": [ packages.cc_library( @@ -73,31 +109,6 @@ CUDA_PACKAGES = { srcs = ["lib/libnvtx3interop.so"], ), ], - "cuda_compat": [ - packages.patchelf( - name = "{}.patchelf".format(file), - set_rpath = "$ORIGIN", - src = "compat/{}".format(file), - soname = file, - ) - for file in CUDA_COMPAT_FILES - ] + [ - """filegroup( - name = "cuda_compat", - srcs = [ - "compat/libnvidia-gpucomp.so.610.43.02", - "compat/libnvidia-tileiras.so.610.43.02", - ] + {} + select({{ - "@llvm//platforms/config:linux_x86_64": [ - "compat/libnvidia-pkcs11-openssl3.so.610.43.02", - ], - "@llvm//platforms/config:linux_aarch64": [], - }}), - )""".format(repr([ - ":{}.patchelf".format(file) - for file in CUDA_COMPAT_FILES - ])), - ], "libcufft": [ packages.filegroup( name = "libcufft", @@ -154,7 +165,7 @@ CUDA_PACKAGES = { name = "cuda_nvrtc", srcs = [ "lib/libnvrtc.so.13", - "lib/libnvrtc-builtins.so.13.3", + "lib/libnvrtc-builtins.so.13.4", ], ), ], @@ -183,6 +194,7 @@ CUDNN_PACKAGES = { "lib/libcudnn_engines_runtime_compiled.so.9", "lib/libcudnn_engines_tensor_ir.so.9", "lib/libcudnn_heuristic.so.9", + "lib/libcudnn_ext.so.9", ], ), ], @@ -195,7 +207,7 @@ NVSHMEM_PACKAGES = { srcs = [ "lib/libnvshmem_host.so.3", "lib/nvshmem_bootstrap_uid.so.3", - "lib/nvshmem_transport_ibrc.so.5", + "lib/nvshmem_transport_ibrc.so.7", ], ), ], @@ -224,23 +236,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "226e8b04611efe6c8ac3ceaf3010499b5957d498e585c970afb69c2ec47ff9fe", - "url": "https://mirror.zml.ai/plugins/202610150915.220.1.f2470651679c/zml-cuda-linux-amd64.tar.zst", + "sha256": "85101fe4efaa4419afaa3c9b9a9154ada87a1c16665bff53215de145b39751fc", + "url": "https://mirror.zml.ai/plugins/202610151602.227.1.e215eb02db73/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "5b55fcd3173485b0e96034bbbf078e354c393d938aadb2ab8fa9bf658b384759", - "url": "https://mirror.zml.ai/plugins/202610150915.220.1.f2470651679c/zml-cuda-linux-arm64.tar.zst", - }, -} - -_NCCL_ASSETS = { - "amd64": { - "url": "https://pypi.nvidia.com/nvidia-nccl-cu13/nvidia_nccl_cu13-2.30.4-py3-none-manylinux_2_18_x86_64.whl", - "sha256": "534dbf3058cadb625f08ab0d17f1dffad3b961a2bfa360d66633fcf21be53f57", - }, - "arm64": { - "url": "https://pypi.nvidia.com/nvidia-nccl-cu13/nvidia_nccl_cu13-2.30.4-py3-none-manylinux_2_18_aarch64.whl", - "sha256": "e99308a3a89fba78918d50886e81072a6c8b0b4199feb02c3903e63713a6525a", + "sha256": "1c985db48fd8f8a3691092bba2c0c7585377210db7e1b0c06a9b04047158d590", + "url": "https://mirror.zml.ai/plugins/202610151602.227.1.e215eb02db73/zml-cuda-linux-arm64.tar.zst", }, } @@ -293,6 +294,16 @@ def _cuda_impl(mctx): build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + "\n".join(build_file_content), ) + compat_package = "cuda-compat-{}".format(CUDA_MAJOR_MINOR.replace(".", "-")) + for arch, pkg in loaded_packages[compat_package].items(): + http_deb_archive( + name = "cuda_compat_linux_{}".format({"amd64": "x86_64", "arm64": "sbsa"}[arch]), + urls = pkg["urls"], + sha256 = pkg["sha256"], + strip_prefix = "./usr/local/cuda-{}".format(CUDA_MAJOR_MINOR), + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + "\n".join(_cuda_compat_packages(pkg["version"].split("-")[0])), + ) + for pkg, build_file_content in CUDA_PACKAGES.items(): pkg_data = CUDA_REDIST[pkg] for arch in ARCHS: @@ -341,17 +352,16 @@ def _cuda_impl(mctx): strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), ) - for arch, arch_config in _NCCL_ASSETS.items(): - http_archive( + for arch, pkg in loaded_packages["libnccl2"].items(): + http_deb_archive( name = _repo_name("nccl", arch), - urls = [arch_config["url"]], - type = "zip", - sha256 = arch_config["sha256"], + urls = pkg["urls"], + sha256 = pkg["sha256"], build_file_content = "\n".join([ _BUILD_FILE_DEFAULT_VISIBILITY, packages.filegroup( name = "nccl", - srcs = ["nvidia/nccl/lib/libnccl.so.2"], + srcs = ["usr/lib/{}-linux-gnu/libnccl.so.2".format({"amd64": "x86_64", "arm64": "aarch64"}[arch])], ), ]), ) diff --git a/platforms/cuda/libzml_cuda.BUILD.bazel b/platforms/cuda/libzml_cuda.BUILD.bazel index e79ab900..8c327a80 100644 --- a/platforms/cuda/libzml_cuda.BUILD.bazel +++ b/platforms/cuda/libzml_cuda.BUILD.bazel @@ -68,7 +68,7 @@ copy_to_directory( include_external_repositories = ["**"], replace_prefixes = { "compat": "lib/compat", - "nvidia/nccl/lib": "lib", + "nvcuextend": "lib/compat", "nvvm/lib64": "lib", "libzml_cuda_so": "lib", "lib/x86_64-linux-gnu": "lib", diff --git a/platforms/cuda/packages.lock.json b/platforms/cuda/packages.lock.json index f582fa45..3563bc61 100755 --- a/platforms/cuda/packages.lock.json +++ b/platforms/cuda/packages.lock.json @@ -87,6 +87,60 @@ ], "version": "1:1.2.13.dfsg-1" }, + { + "arch": "amd64", + "dependencies": [], + "key": "cuda-compat-13-4_615.71.09-2ubuntu1_amd64", + "name": "cuda-compat-13-4", + "sha256": "1b814aa705b4dfe955420c967bd545f39ef7b67f32bd0f285a005ea296b02831", + "urls": [ + "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/./cuda-compat-13-4_615.71.09-2ubuntu1_amd64.deb" + ], + "version": "615.71.09-2ubuntu1" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "libstdc-p--p-6_12.2.0-14-p-deb12u1_amd64", + "name": "libstdc++6", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "libc6_2.36-9-p-deb12u10_amd64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + } + ], + "key": "libnccl2_2.32.3-1-p-cuda13.4_amd64", + "name": "libnccl2", + "sha256": "55a3e32a3f560f20e2f423cd38f1f284951e92093d1e8fbaabb4c42ac76f3195", + "urls": [ + "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/./libnccl2_2.32.3-1+cuda13.4_amd64.deb" + ], + "version": "2.32.3-1+cuda13.4" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libstdc-p--p-6_12.2.0-14-p-deb12u1_amd64", + "name": "libstdc++6", + "sha256": "5cd3171216d4ab0fc911cfe9c35509bf2dd8f47761c43b7f6a4296701551a24d", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/libstdc++6_12.2.0-14+deb12u1_amd64.deb" + ], + "version": "12.2.0-14+deb12u1" + }, { "arch": "arm64", "dependencies": [ @@ -173,6 +227,60 @@ "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_arm64.deb" ], "version": "1:1.2.13.dfsg-1" + }, + { + "arch": "arm64", + "dependencies": [], + "key": "cuda-compat-13-4_615.71.09-2ubuntu1_arm64", + "name": "cuda-compat-13-4", + "sha256": "83cdca90db73a36784a077693e3b48565d852158c63f604194c1f0d42c2e5458", + "urls": [ + "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa/./cuda-compat-13-4_615.71.09-2ubuntu1_arm64.deb" + ], + "version": "615.71.09-2ubuntu1" + }, + { + "arch": "arm64", + "dependencies": [ + { + "key": "libstdc-p--p-6_12.2.0-14-p-deb12u1_arm64", + "name": "libstdc++6", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_arm64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "libc6_2.36-9-p-deb12u10_arm64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_arm64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + } + ], + "key": "libnccl2_2.32.3-1-p-cuda13.4_arm64", + "name": "libnccl2", + "sha256": "809a6eea2f8a570050229bc218e30d4280829cbe01e824e052c96ee9203a7fe5", + "urls": [ + "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa/./libnccl2_2.32.3-1+cuda13.4_arm64.deb" + ], + "version": "2.32.3-1+cuda13.4" + }, + { + "arch": "arm64", + "dependencies": [], + "key": "libstdc-p--p-6_12.2.0-14-p-deb12u1_arm64", + "name": "libstdc++6", + "sha256": "26e138c677a985775331373828a6c286c551ff397cb735d00e2383cb273d1cb2", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/libstdc++6_12.2.0-14+deb12u1_arm64.deb" + ], + "version": "12.2.0-14+deb12u1" } ], "version": 1 diff --git a/platforms/cuda/packages.yaml b/platforms/cuda/packages.yaml index 9f6ac8b9..010ca572 100644 --- a/platforms/cuda/packages.yaml +++ b/platforms/cuda/packages.yaml @@ -8,6 +8,12 @@ sources: url: https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z - channel: llvm-toolchain-bookworm-22 main url: https://apt.llvm.org/bookworm + - channel: / + url: https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 + archs: [amd64] + - channel: / + url: https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa + archs: [arm64] archs: - "amd64" @@ -16,3 +22,5 @@ archs: packages: - "llvm-libunwind1" # LLVM 22 runtime from llvm-toolchain-bookworm-22. - "zlib1g" + - "cuda-compat-13-4" + - "libnccl2" diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index 30da0600..60e63905 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202610150915.220.1.f2470651679c/zml-furiosa-linux-amd64.tar.zst", - sha256 = "23d8c7c3aa210dae91ca56bb66bcaa0630d40ea2048f3971bc4ca436beb4b0b3", + url = "https://mirror.zml.ai/plugins/202610151602.227.1.e215eb02db73/zml-furiosa-linux-amd64.tar.zst", + sha256 = "e474a6ba2f3b5e810dba24dc7339bf45d5e7ea05025c924d7952d968ef6f1335", ) return mctx.extension_metadata( diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index fabf2dfe..d6995cbf 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "2413f6952055d9d336a7236a8b9f2a5ceba2f77048000662bfde3b202244fbac", - url = "https://mirror.zml.ai/plugins/202610150915.220.1.f2470651679c/zml-metal-darwin-arm64.tar.zst", + sha256 = "5ff70796f49a67880fe15cb639ab7b358547719a9afd69d184e6cb3cc7119250", + url = "https://mirror.zml.ai/plugins/202610151602.227.1.e215eb02db73/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/packages.lock.json b/platforms/rocm/packages.lock.json index e22d3b1e..2b2a4836 100644 --- a/platforms/rocm/packages.lock.json +++ b/platforms/rocm/packages.lock.json @@ -123,9 +123,9 @@ "arch": "amd64", "dependencies": [ { - "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", "name": "amdrocm-sysdeps10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -348,24 +348,24 @@ "version": "3.10.6-1~22.04.1" } ], - "key": "amdrocm-amdsmi10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-amdsmi10.1_10.1.0-3_amd64", "name": "amdrocm-amdsmi10.1", - "sha256": "9d722a1406c1701e48d93f6c2616eb8b0cf71787de2d074a70124eaae9ea221d", + "sha256": "994e715d38a16d9c408eebd508c66123ff2f64cf4d4f4ca15780c4500e85103a", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-amdsmi10.1_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-amdsmi10.1_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", "name": "amdrocm-sysdeps10.1", - "sha256": "ccfd6b77ecb56e982bbf18c855c4b402cd642b8fba411112f6499987f4d60ba9", + "sha256": "a2493d98144a586df49489c4b4b7a7449c9eb123597515929699b1b5fccfd916", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sysdeps10.1_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sysdeps10.1_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", @@ -811,19 +811,19 @@ "arch": "amd64", "dependencies": [ { - "key": "amdrocm-runtime10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", "name": "amdrocm-runtime10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-llvm10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", "name": "amdrocm-llvm10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", "name": "amdrocm-sysdeps10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -1136,37 +1136,37 @@ "version": "3.10.6-1~22.04.1" } ], - "key": "amdrocm-base10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-base10.1_10.1.0-3_amd64", "name": "amdrocm-base10.1", - "sha256": "2449f0433a9d3b93af4ad4bc42cef5429b3318745e809f83e21b1c20cc838f63", + "sha256": "a3d48032edee53c51016b28884a1fe81e2d2f84e087c2546dd20753402ad53c9", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-base10.1_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-base10.1_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-runtime10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", "name": "amdrocm-runtime10.1", - "sha256": "62accad621bfeb32147c1dff30dfdd9b22ca6bbc6e69c07e9a39d0eca6863e18", + "sha256": "2cf4fa3b82f9b37be9fc18ca6886e1791585c1b8041758ccba9e3c5dadb7c6b1", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-runtime10.1_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-runtime10.1_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [ { - "key": "amdrocm-runtime10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", "name": "amdrocm-runtime10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", "name": "amdrocm-sysdeps10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -1389,9 +1389,9 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "amdrocm-base10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-base10.1_10.1.0-3_amd64", "name": "amdrocm-base10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -1484,13 +1484,13 @@ "version": "3.10.6-1~22.04.1" } ], - "key": "amdrocm-llvm10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", "name": "amdrocm-llvm10.1", - "sha256": "5f185639c5f2cba6aefe1b1a945b0a05fadb13763ee094e933114e5f153471ae", + "sha256": "6a4727102d830f6c97b911351192b15623189278afad9b4558514a6fe51ca58a", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-llvm10.1_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-llvm10.1_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", @@ -1683,39 +1683,39 @@ "arch": "amd64", "dependencies": [ { - "key": "amdrocm-blas10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx950_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx950", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx950_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx950", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx950_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx950", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse-host10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse-host10.1_10.1.0-3_amd64", "name": "amdrocm-sparse-host10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-runtime10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", "name": "amdrocm-runtime10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-llvm10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", "name": "amdrocm-llvm10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", "name": "amdrocm-sysdeps10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -1938,9 +1938,9 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "amdrocm-base10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-base10.1_10.1.0-3_amd64", "name": "amdrocm-base10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -2033,490 +2033,445 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "amdrocm-profiler-base10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-profiler-base10.1_10.1.0-3_amd64", "name": "amdrocm-profiler-base10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver-host10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver-host10.1_10.1.0-3_amd64", "name": "amdrocm-solver-host10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-math-common10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-math-common10.1_10.1.0-3_amd64", "name": "amdrocm-math-common10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas-host10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas-host10.1_10.1.0-3_amd64", "name": "amdrocm-blas-host10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx942_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx942", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx942_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx942", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx942_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx942", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx90c_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-blas10.1-gfx90c", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-solver10.1-gfx90c_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-solver10.1-gfx90c", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-sparse10.1-gfx90c_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-sparse10.1-gfx90c", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-blas10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx90a_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx90a", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx90a_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx90a", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx90a_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx90a", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx908_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx908", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx908_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx908", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx908_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx908", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx906_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-blas10.1-gfx906", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-solver10.1-gfx906_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-solver10.1-gfx906", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-sparse10.1-gfx906_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-sparse10.1-gfx906", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-blas10.1-gfx900_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-blas10.1-gfx900", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-solver10.1-gfx900_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-solver10.1-gfx900", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-sparse10.1-gfx900_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-sparse10.1-gfx900", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-blas10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1250_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1250", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1250_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1250", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1250_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1250", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1201_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1201", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1201_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1201", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1201_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1201", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1200_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1200", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1200_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1200", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1200_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1200", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1153_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1153", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1153_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1153", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1153_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1153", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1152_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1152", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1152_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1152", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1152_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1152", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1151_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1151", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1151_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1151", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1151_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1151", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1150_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1150", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1150_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1150", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1150_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1150", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1103_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1103", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1103_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1103", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1103_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1103", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1102_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1102", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1102_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1102", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1102_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1102", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1101_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1101", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1101_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1101", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1101_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1101", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1100_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1100", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1100_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1100", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1100_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1100", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1036_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1036", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1036_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1036", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1036_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1036", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1035_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1035", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1035_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1035", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1035_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1035", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1034_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1034", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1034_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1034", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1034_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1034", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1033_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1033", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1033_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1033", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1033_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1033", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1032_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1032", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1032_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1032", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1032_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1032", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1031_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1031", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1031_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1031", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1031_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1031", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1030_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1030", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1030_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1030", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1030_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1030", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1012_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1012", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1012_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1012", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1012_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1012", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1011_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1011", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1011_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1011", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1011_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1011", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1010_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1010", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1010_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1010", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1010_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1010", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" } ], - "key": "amdrocm-blas10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1_10.1.0-3_amd64", "name": "amdrocm-blas10.1", - "sha256": "70bf09bab42dc86acc6489580311c267751d848122c13f4af7902b84677e86cb", + "sha256": "2a8a63f7c3e5b5e11974433c9cb6abdd297c9939c07cd061f83e0a8d3867e785", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx950_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx950", - "sha256": "32712f653543a86f71b0631d587ad1818c33b3b549fe5a37ad0cdc2fa5d0ba28", + "sha256": "ba944cc7b634d098344315357b126a6ab7fff39ab34970f798d20a3e8b1177d2", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx950_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx950_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx950_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx950", - "sha256": "4f6f3f49115cebeb52c7fd9694f912794fc94a22f632584251107723dc974efd", + "sha256": "089bf10ade33b38698d8eb02af7cd8ec9768ad1946afd2e76b18175d7dbbaa26", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx950_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx950_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx950_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx950", - "sha256": "855eb5f6bf63e15d3e03d23ec09eb86deac4df68bb4ec3eb43fc7e8fba6fa633", + "sha256": "b0714db7f27322097d7d1940e9c95a431d9ec11ff7ea93081ca549f8b72a379c", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx950_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx950_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse-host10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse-host10.1_10.1.0-3_amd64", "name": "amdrocm-sparse-host10.1", - "sha256": "3b5cf4021a4c7e3d01bf626cda47c27044aa9f88ce714868dd411cf98a168d6b", + "sha256": "44faf2440577e65c51b9b6474c1b70a96503a44c7939e6b8ec75512ab928f08e", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse-host10.1_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse-host10.1_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [ { - "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", "name": "amdrocm-sysdeps10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -2539,14 +2494,14 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "amdrocm-runtime10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", "name": "amdrocm-runtime10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-llvm10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", "name": "amdrocm-llvm10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", @@ -2749,9 +2704,9 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "amdrocm-base10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-base10.1_10.1.0-3_amd64", "name": "amdrocm-base10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -2844,970 +2799,871 @@ "version": "3.10.6-1~22.04.1" } ], - "key": "amdrocm-profiler-base10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-profiler-base10.1_10.1.0-3_amd64", "name": "amdrocm-profiler-base10.1", - "sha256": "a11a01214b24e38623907268b9f1d8b7fd1100bed59b4e147d6226ca3a7a3874", + "sha256": "5f027faf16f293f66fe42cd794ec6b984377183417f49c49c8c2daf3a5675d23", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-profiler-base10.1_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-profiler-base10.1_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver-host10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver-host10.1_10.1.0-3_amd64", "name": "amdrocm-solver-host10.1", - "sha256": "9fcc4c65c59801311f60ea8d111bcfbfaea75fff56c1988d68987aea9c0da4c3", + "sha256": "1ec2e8677941eadcb4b87f48c5cfae18511b1fc2ac08794d61740843eabba022", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver-host10.1_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver-host10.1_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-math-common10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-math-common10.1_10.1.0-3_amd64", "name": "amdrocm-math-common10.1", - "sha256": "3a7656ad8f819d060a508c4a95f662cb344b865d4b95c5e99ac2e630b716d824", + "sha256": "1e4b3e1fc77bc8431e3f5bf81b9193376c9025dd22b6c8a28f87d448cd3d7eaf", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-math-common10.1_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-math-common10.1_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas-host10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas-host10.1_10.1.0-3_amd64", "name": "amdrocm-blas-host10.1", - "sha256": "f0c3f8b9a213e3a51871b9304bb021e5bd1ee9030ccf3a002d3a7e95f9672ee0", + "sha256": "9c35dc27bc27b797f80d42d01cf3d8e042c5f6cc1db66f1a54180d5f21ccf3a3", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas-host10.1_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas-host10.1_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx942_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx942", - "sha256": "a031f800c28e9681f26d92a044b14f62232c250b89b20dda46a4cf8c6a656400", + "sha256": "f988f42ba63f83ca7e65086a5439ac693a0c76c5a963e871b1d8440c7e892c34", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx942_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx942_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx942_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx942", - "sha256": "07d39e2cc097f041eeb26a9161ca51bacd40f1462b2436a6d226dd885c7b00a1", + "sha256": "c7964290972e15f3db7d73a91c460332ecbb19fa1c187cbdf5836c73804c2748", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx942_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx942_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx942_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx942", - "sha256": "629a30e05403cd6e7a016f519e3bff9cf56eee79ab3a8360a0fa89948f71053d", + "sha256": "0dadc1402ca074a627918c52680d77dfe99b22abdad643ad36fdfb964ef3fa16", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx942_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx942_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas10.1-gfx90c_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-blas10.1-gfx90c", - "sha256": "5ba71bbc88659437e83cc00b2d47e5e7bfcdd2d805c7cbfeda7a50ff9b06fd7c", - "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx90c_10.1.0~20260903-33697653391_amd64.deb" - ], - "version": "10.1.0~20260903-33697653391" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx90c_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-solver10.1-gfx90c", - "sha256": "5ff7061523f7dff14e4bb8f77ef2a3b8ee03f5053907cc836784b6cabfb48fa4", - "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx90c_10.1.0~20260903-33697653391_amd64.deb" - ], - "version": "10.1.0~20260903-33697653391" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx90c_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-sparse10.1-gfx90c", - "sha256": "bb714e03c8b27c820a33d0d60ec970ac038316e8dbf042cb87da96acfd6b9820", - "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx90c_10.1.0~20260903-33697653391_amd64.deb" - ], - "version": "10.1.0~20260903-33697653391" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx90a_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx90a", - "sha256": "242c5128627accd2626e2fb4cd9a0ab893fa08e167062decb6166baee98894c5", + "sha256": "a0799f0aad9e9378f245288ce5ae653ff1b4f716962fe0f8d6eb160820fb9870", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx90a_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx90a_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx90a_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx90a", - "sha256": "5fb7638b8bcee5b18e697e8346470dfd44a91f8d98abcf7c67f478b75fd802d3", + "sha256": "f329191044bc2aee16a5eb33ab989f393dfbef42c4705542e4d5a0112e406b8d", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx90a_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx90a_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx90a_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx90a", - "sha256": "ada1cdc38b161e37925e61d3b3cdd5f5b3a15cb3222a151d459769e8a75b4a77", + "sha256": "200cae46a18b621385e50b191f9acbacceea04b8c7757148442661c65ea58c40", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx90a_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx90a_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx908_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx908", - "sha256": "db26b9bd2daaa5ffb6a21bc555275ed2ec92c1bd5c1167db00d43a225924b9ad", + "sha256": "da0bf3ce9c571e9c2a60cc1964e287ff0c2496ebb156003962212650c8a3a42c", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx908_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx908_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx908_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx908", - "sha256": "63cb1dc258a0fee7fb07604ca67f91c62e61c819700a0e4fdc29c93d89992f62", + "sha256": "0c7e23e418f4892d6d341d9c89f774477d52e105595204801272f2834be1f40b", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx908_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx908_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx908_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx908", - "sha256": "c95acca10f7081a3073a25be93be28b21d5192354dec725481421aa49b7ffee8", + "sha256": "54d6ddf65e98722e9e5c0b39909d7ccd689ec273c6d7020f6095ba7274404152", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx908_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx908_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas10.1-gfx906_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-blas10.1-gfx906", - "sha256": "a26b3fbe79056c73d2bade44e29ec6f49c66bfde869f5b8118ccc118cd4e33c3", - "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx906_10.1.0~20260903-33697653391_amd64.deb" - ], - "version": "10.1.0~20260903-33697653391" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx906_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-solver10.1-gfx906", - "sha256": "412dcf592e94a8cbbe836243fbc110361829339d33156b7d4e11b2f8b2e97103", - "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx906_10.1.0~20260903-33697653391_amd64.deb" - ], - "version": "10.1.0~20260903-33697653391" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx906_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-sparse10.1-gfx906", - "sha256": "927ee1317af52aae8db2fd0ce97eacda1de14d2c1e290a09797fb32b2c5737ca", - "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx906_10.1.0~20260903-33697653391_amd64.deb" - ], - "version": "10.1.0~20260903-33697653391" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx900_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-blas10.1-gfx900", - "sha256": "6f53bdceed424a60184c1ea1569dd2be57499c102f33f70dbafc086165eae955", - "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx900_10.1.0~20260903-33697653391_amd64.deb" - ], - "version": "10.1.0~20260903-33697653391" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx900_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-solver10.1-gfx900", - "sha256": "ec941ec97045c84f01c2bf8f547527f1ed6ee80e9b18af6bec9df70b7f6f8889", - "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx900_10.1.0~20260903-33697653391_amd64.deb" - ], - "version": "10.1.0~20260903-33697653391" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx900_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-sparse10.1-gfx900", - "sha256": "0a14ecd08b7067782644c2c20f6c24c18e56513db79a7938900aad676fe3e5dc", - "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx900_10.1.0~20260903-33697653391_amd64.deb" - ], - "version": "10.1.0~20260903-33697653391" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1250_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1250", - "sha256": "5e1fa4edf09fdb53468da5d42c94b8253ce28d74cc162890cdc8b8bffe3a0008", + "sha256": "44db7d9e96ecddd26bd8b069a0a5a89610276c8db859ca7cad9f619f0bf0dd60", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1250_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1250_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1250_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1250", - "sha256": "e0c7baa13a70c0e72ad452a7f74ef3f2c3fcd9d0bbd1db1c0f4d42893bb24932", + "sha256": "7e76b6e1b314d87b55c268f5b6f0b9e300f571eb34de551524bc6302b9ab276b", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1250_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1250_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1250_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1250", - "sha256": "c9a70085ee25b49b0e46a6dd0fa06ec5278c7846b082c19a7cdd139e99fe8686", + "sha256": "d555ef3296a63874af1fa99fc7f8f48e57d932abce3c57ec16b2d4cb24ee027e", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1250_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1250_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1201_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1201", - "sha256": "10d913f2ec18d6e16e77c0f6435abdebb8393bc66747ee70ebd7448c6e19b02c", + "sha256": "1906724e7d338bcbf8013e6f02a96618120d39d5d7c8f3b5a40da185be236bc2", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1201_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1201_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1201_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1201", - "sha256": "ea16125989307dbce3aff9b3dded900751616f720a61d445bc633b6f9a9d6315", + "sha256": "43214b4df226aac08647646f71c13dec6cd3be561f9278f0a9fa36b3a5827337", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1201_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1201_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1201_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1201", - "sha256": "017d35a09886dc97c288d5d8abaa59d24c1c879bf820ddd104e27fe54b90a4ba", + "sha256": "4612811ba76d77f1faa19b1f4da9db829a473253058aa2b170a2ee555de26a15", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1201_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1201_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1200_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1200", - "sha256": "56347ab3dcf22044e6420b998afa866857d5c76cf4223f659100cd4cd725ad6b", + "sha256": "b4df7e69d7c5c0e6e0ebe262411574153cca723c1ef4e9d445dbb539e1724a16", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1200_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1200_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1200_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1200", - "sha256": "df27d26d46a78796db5b97243a6073205b94155b81fa41a0d46543c078530539", + "sha256": "757a0c725ebcf7701ced94331d1cb4d615aef9e34849c96a84c3aa9dc0e4a754", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1200_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1200_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1200_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1200", - "sha256": "4ff73a8d4186d7b877d8f4647327905817bcb72c4ce55e2c260f4a0841f40614", + "sha256": "740dd6b691abe24651412c7d5eac34f10a3878eec939759b5dc39f720d530084", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1200_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1200_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1153_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1153", - "sha256": "40b06296573a2c23b74ab50247b2466fa3ed51fc7e311686c7451517219e527b", + "sha256": "dfaec5607aad9e14282c27c1d1f7440cedbde1d147d2e42c11aada69cfb5c30d", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1153_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1153_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1153_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1153", - "sha256": "bb5f4863fda70d26179e17dd0922493ae2aa45096ecb6d1696897dba5c729b83", + "sha256": "b9cfe52bea768867d858a4b0db6de109b95455c1470528cfb51ba5ef4a62f58d", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1153_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1153_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1153_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1153", - "sha256": "cdd2a35b80a85fd95c0b7bd627a6e1cb3dac5d41f277ce89d00613c4cf51428c", + "sha256": "fac6655f10a9e857c844e0f0c9941213ad543c375335272e97ca9a05c3adf64a", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1153_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1153_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1152_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1152", - "sha256": "2cb29cd88839762ed0b7847f1a9a79a6cae0e0ac49b796c0bb4432b427e7b8cc", + "sha256": "3cb059af8f1ce850cf6fabe3182a1904ada1f2460f638476ad2fd57c43423988", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1152_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1152_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1152_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1152", - "sha256": "07b9b06b28fd11ae4926efa29597eb444141f365f070dac1154bce12808687d3", + "sha256": "93cccffec257c00cbc45e0970f980fcefe06b2a55893dda128bd97b1b6be29cb", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1152_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1152_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1152_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1152", - "sha256": "ea3bdb9d7a636fa55b003b21faefb3d22629ff63bcb31cd40d13432ae6bdbd04", + "sha256": "2b348c1379a9ffbfafea1d606d47574d6b009fb7e729eeaa586a4bcac19830b5", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1152_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1152_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1151_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1151", - "sha256": "857154648bbdac75f539ebe79706a72689988036fae694df042ebe7ea02957a6", + "sha256": "9de48ee15f185b471fc6b0e836abea882153e7d3cced6e3157a5f091f21ea4cd", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1151_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1151_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1151_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1151", - "sha256": "f957615d37f11493426e278bf19727e45149a5b2551d9d49ff70b8a9a0458811", + "sha256": "d22416ee83fc82dddf4d094eebe28a1a699b09153ed4d996b7763d8686365ab4", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1151_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1151_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1151_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1151", - "sha256": "9b741dfc30bc8a5f0130cfdcbb337c41e2d651d5e077c12b2177158a6f1f01a3", + "sha256": "8ab4582f863ceeb2bd0f9bc607786f57d03d34515bed3fb38e4bcaa89b05e8f2", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1151_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1151_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1150_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1150", - "sha256": "ec9eb850c647a69b82cf80ec8f8ea063bf06c6ddb0d2bf9d680803ddd67af1c5", + "sha256": "7f2fa46ad609d5cf9b8ef181cf0f76e288880d8916530e33b74df33efe508d37", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1150_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1150_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1150_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1150", - "sha256": "4bb76ca1d2954da0808635c7172def023b6f80672dbfa299a4269c2b17a41371", + "sha256": "ffd9c28d8313d64a83401614d6164e4308ade4f7eca52209206c7974f605e541", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1150_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1150_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1150_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1150", - "sha256": "abc915dda44d75f699c5f1122049334c73be02da751f3752fff2e7aa5fd68a93", + "sha256": "81626b7367523f6bb0ba918aa284c9ddcbe59093fad6110462d76cf51dfe06a5", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1150_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1150_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1103_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1103", - "sha256": "621a18815c4294ef802b0622defa030e5a1bbc23f7195f61abd1c7747f4450c9", + "sha256": "2bda07dcfbf333c35dd9b063e8eb4436bb653afea0aa0ef23fac58ebc54ae021", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1103_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1103_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1103_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1103", - "sha256": "c6ff77ba5593df096f3efd524a7e8c8bd2e588274b7e403c9b179e87a76c4709", + "sha256": "b20f76bd6424de48b4a3d7881bc7ee4dc7c0e060970b237a8db53297b46c2acc", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1103_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1103_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1103_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1103", - "sha256": "0001e6ac717e66b3dc7efe41246ae4c643f9063a7889bfe39deda66fd6b2bc53", + "sha256": "1089660ba5ab633ac0c6f7acc0da10e39926713e71c581f3e27bb7add04cfa59", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1103_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1103_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1102_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1102", - "sha256": "dbcd542dfaaf7ea141216c54bb980771a158cb09d5f2af83729b062db3233b69", + "sha256": "626a668f339768a2842a9679a417009ad5c885e9a727e0e34ff7386ab9262f71", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1102_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1102_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1102_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1102", - "sha256": "22f9745ca43d9ba3f78585a272b701cd62a813824a7ac711e25ed3814f0f2d96", + "sha256": "33d78823a628a6b0bf77f4a36cf977c1d1c816f1aa7e1b55bd2af86370bd3f6c", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1102_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1102_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1102_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1102", - "sha256": "71c25b880004010ce89b1d6db2899658360ccb76550320d666d96afcc37639b4", + "sha256": "fcabe97aadc69b9a6f4d006191d0e4fb77df22b734b688763b2c437071c9e11a", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1102_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1102_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1101_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1101", - "sha256": "107a517470ed0667ac6771ce2c6534d0d4dc22bf5669fc59d5a8c434b28ba3b4", + "sha256": "736385257ca8d3680f49500667fd7bc3fe483e249e80b550da6386e17ac92cb8", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1101_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1101_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1101_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1101", - "sha256": "115a5371068a7e6f2f418cc2fa556686c3ed7e41a40933798d165e51f34cc30f", + "sha256": "49610fda5782cbf81a0ade0ecea1e4cda4a23ea5f619bbeb19954fd3159bc511", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1101_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1101_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1101_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1101", - "sha256": "5beb485c1e423bff82e4f8eacb2454aef31c7db5afee58c09eb1267068c0245f", + "sha256": "28470e954ec30f8a6cbfff23b85fcdbc0c486b839f904d61a2c922e3157349d0", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1101_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1101_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1100_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1100", - "sha256": "e0b6684ebeafaac46b0c2b93333b71b4b665b22dc992b2c4ed999eda43971b3e", + "sha256": "7998acee523f9ac9e2ce6c36514c8343fb9a651629275f0354404e47e080f076", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1100_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1100_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1100_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1100", - "sha256": "bc92ee0916d6355257559e81669e8e406f89b4402f292b8b78284c8295049117", + "sha256": "71d7fdbe3619d77d01db2f7895f73330365ca3e1fb792ccc80f8accfb067db7b", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1100_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1100_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1100_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1100", - "sha256": "91ccf0dd8400e0a795ea09b62412ec531cfcb426f08251348ffb0aedfa05107d", + "sha256": "15f0a8421d49acdc19365063b7ce4d3b85102bbfbb517e83bc888e4107061694", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1100_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1100_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1036_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1036", - "sha256": "2c34a119a2de5884511933718aa38e410af24c9f7a6bacac06c1dd18a4f0abc1", + "sha256": "dcf5d14b3781669526b279a40866068d60e4d119aa5b628f38fd69e065fba450", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1036_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1036_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1036_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1036", - "sha256": "d924ee3f4b70cfce1dd44e266ade058a5bd862508824860b202fa03909bb1b4c", + "sha256": "7e65239be07b34e784e22f1fd98aa1b02a89a713aad05c3640ee88b08cf38a7d", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1036_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1036_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1036_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1036", - "sha256": "f3bd7e470e081cc2b7226e98a901c1855c17084e348ce8a9a46a8985a928f680", + "sha256": "028a6598a0f90384b08bdfb0ee42ca2031206e8bf5b65730eeef2a77c201cf87", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1036_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1036_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1035_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1035", - "sha256": "731831b0f2f3e134548525c2f5a6195fb5511fa7b0b4cfc5b8121a9c47dd5680", + "sha256": "5e0397e891522100a5432d433039e8877479b5111f7226401ed185700a56d2ef", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1035_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1035_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1035_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1035", - "sha256": "789256f456ed6b3888f37f354b7277f75f9463f700363b50c2aad2842662dadc", + "sha256": "2763f459121dcca48edc4997dc861763cb3c237d102a0cf771209f5d4e0865db", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1035_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1035_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1035_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1035", - "sha256": "606e905784e54fecc1449a38e5256bf938b80a4bf305f1ce9207dfefdbcbcf52", + "sha256": "2aed6082214e2ef53cc472d169128238a64ec7ec9300291759a5b6416e173d20", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1035_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1035_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1034_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1034", - "sha256": "f1e9629f774876e4d58544e70c68ff4e8a865398ad6ec9a79e5f5f29fd9c08b0", + "sha256": "370b7644ad5031bde2dcbb0fdc32441a741869d44d953ff315abe7ba263dbb72", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1034_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1034_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1034_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1034", - "sha256": "e3c0b83128f0d9cfb14d6490ee11c7cd7541a4e047884b0f133e6733d4475c4c", + "sha256": "aa24693000c9b848f66135dde49722d19673ef1d3385736b91b5ebe298509b28", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1034_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1034_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1034_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1034", - "sha256": "136009871feac8958f5d06717fd208ed37d15b003e981c58cc4a34874cd253bf", + "sha256": "0ab0d5238ba6a8d2c0b46e57ace593316c10d93759aa1328937e170c348d1a1f", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1034_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1034_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1033_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1033", - "sha256": "ef577d055b5196c03a94dc3ecb7e0dc2620d8c33f622ad3d24a74cf33f685b76", + "sha256": "cf3d658fb0a76fbead063bad0c9c892d7624f9b3c40e41de67cbe49169dfae58", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1033_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1033_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1033_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1033", - "sha256": "bc2faaef8650a97d0003a6122ed701f4b7ca0ace79815d29fe0588bcfbb23e76", + "sha256": "208976c7e784551983664f24330c8ba9734b4bb505e25ae6dac233c98a711052", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1033_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1033_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1033_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1033", - "sha256": "8058ebcfa77d219dccbdb9758ddd4203a39775afbbbe02ad634e67b143587be3", + "sha256": "ced0d2f76cf3113b6bf7246ce1ddd90c638293a61f0c78df1b8099f3cc89a5d7", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1033_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1033_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1032_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1032", - "sha256": "8740c616bf9a536d80fb0873e7dc6014040b2c5cdf980e6c45ed192fcada168c", + "sha256": "03aa4d150527f6fb4ab0c84ed650e8ce63a132935ec50641e595693d378529da", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1032_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1032_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1032_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1032", - "sha256": "d091bea4179f5581dccf63b537bbf184cd886853773043ed6b6804dd18800bf2", + "sha256": "9a79590868ea114964ce9ca634b73213600f7800c0cd2d5b26f3e964bf496d28", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1032_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1032_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1032_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1032", - "sha256": "d9ed5b581f3c63192926fc03893f3f7fb5b75b887fb1d68cd502d3658e8c3a96", + "sha256": "4182454edef4240bbc911ec01f00deae83e71a8ff878fbcceff70db3d52eaefe", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1032_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1032_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1031_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1031", - "sha256": "998cb3eb22e39c6a4a0934216750e8a5c575e7b34c4f246cebbaea3cc80ade86", + "sha256": "2553208d7552421a7e0100334a0135320dfd07d70746a04d6a75ac49c7c5514d", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1031_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1031_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1031_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1031", - "sha256": "2f6d6e1a9b1192a63c9a468699324467de491ae5d9c97efdd3020842b6432b8f", + "sha256": "dc2453d80d4e7690f918ff3dbe9eff6663b1f837aaae3975144282126d7b1527", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1031_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1031_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1031_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1031", - "sha256": "230c6600dd9587138cdcabf3274e04452a1bb5875c126b620cdf79b2f703a5f4", + "sha256": "3f0bb24ff77b733eee320eba671a0b169b1527fc8bf2dfae6c0b4cc5c363ec3f", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1031_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1031_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1030_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1030", - "sha256": "7ef186f51905b8d38c52b815a665f5f76a99d036af646c1db48c3ee150d2d238", + "sha256": "4b56e03309b8339d2584a7af18609b0f5f8d041dbf08653c1a32f4a36fa9e73b", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1030_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1030_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1030_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1030", - "sha256": "62f654b869ba1208db04eb67f4b94904ef330eac8534eb7c9a24c05f2a01a882", + "sha256": "8c3802bb51b012a36a9012d86c020694f26ebea07492f9bdb3189248b87ec2d1", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1030_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1030_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1030_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1030", - "sha256": "8863c44c88ec1e6aa1d09b0e221ee57ea60fb66b8280535beb14d0c6d12bd7a8", + "sha256": "e8c255c31835bcf6a5193f9ff8ed91b6a94369097ed8c364168dd71d2c163e6e", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1030_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1030_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1012_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1012", - "sha256": "c5e4e2928e67a7d501af132396470afde4c4c279358f2b7a213875a2475d0e87", + "sha256": "95c5aa73aaa87b4fb944d5cd05088b1ce500d71f012d3ee1eae76d7db105b906", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1012_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1012_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1012_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1012", - "sha256": "66d350644d469e3b6f15f481823d91cf5d9dc3de16cff231b885a3a6723ed208", + "sha256": "11afcd2eebbb54385d0334e126a2ff63b90ccf820b04ed0f81ae4042273bfd66", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1012_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1012_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1012_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1012", - "sha256": "3e30d0262fa41c16e22e86a9ee76908f6b9fb51b04694d553757a3d191c6ac1e", + "sha256": "ade96cb7aa4f0a16f23920eafb683686efabee5e5aaaac023d5b11434037d181", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1012_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1012_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1011_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1011", - "sha256": "d066c2bcb5ff469ed53fed0e24512cfafb281026b099a31d385ba3891e05f943", + "sha256": "7142eefd24ec5363036bb03267e49caf7af820a8f275df4ae63afcb53471cee6", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1011_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1011_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1011_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1011", - "sha256": "c99f5d01203a5a19753790bc8b1dfd6cf6285504855dfbf66de922047d47b9b8", + "sha256": "b689b157fd60e30ba111dbea21077086020e0e1eda146995e83b5b44cd117b43", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1011_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1011_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1011_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1011", - "sha256": "07dcb85b5fdbb062e2b1427b24568b744d0db90ea2c36437b90c2311c4d49db7", + "sha256": "4a4e94348c97148c7332ceb32073cbf330623f507a6022ae398ccfeb6ddc2878", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1011_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1011_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-blas10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1010_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1010", - "sha256": "645de7ff587ad83f7374f55d9ece589d5d651aaa8d6b9ee0b7ba7525e4e50fd7", + "sha256": "052bdaa575e9a47e59ec21d492b5288ee289999d0121c7d0b82484f4a4f3c19f", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas10.1-gfx1010_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1010_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-solver10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1010_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1010", - "sha256": "c4c344044a99c336ce2a6c72e057ca676576a0587c2124de0511479cbf4c5392", + "sha256": "969e5b48158bb841dbdc3db2388e7b13d58016649104b138e2ce307a087f4afa", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1-gfx1010_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1010_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1010_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1010", - "sha256": "1fd269cdda8880606b595c502756db2f0e7d53be239deacfca23e3a8d52956b2", + "sha256": "905dca5b90e3e301512e422eb0d5ae5631e341cb4b8dab4f5f2324ec2947a33f", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1-gfx1010_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1010_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [ { - "key": "amdrocm-hipblas-common-dev10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-hipblas-common-dev10.1_10.1.0-3_amd64", "name": "amdrocm-hipblas-common-dev10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-runtime-dev10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-runtime-dev10.1_10.1.0-3_amd64", "name": "amdrocm-runtime-dev10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-llvm-dev10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-llvm-dev10.1_10.1.0-3_amd64", "name": "amdrocm-llvm-dev10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-llvm10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", "name": "amdrocm-llvm10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-runtime10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", "name": "amdrocm-runtime10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-base10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-base10.1_10.1.0-3_amd64", "name": "amdrocm-base10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -4030,9 +3886,9 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", "name": "amdrocm-sysdeps10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", @@ -4125,47 +3981,47 @@ "version": "2.35-0ubuntu3.10" } ], - "key": "amdrocm-blas-dev10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas-dev10.1_10.1.0-3_amd64", "name": "amdrocm-blas-dev10.1", - "sha256": "1bef633435f6e92942021d66f6ec69f62e1f0f04d23fcd1268df8602365e2f0c", + "sha256": "c7b6b25d7f1143742a7b7a2bc6819cab56a166995b60ebdc5b282d5ddc67770f", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-blas-dev10.1_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas-dev10.1_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-hipblas-common-dev10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-hipblas-common-dev10.1_10.1.0-3_amd64", "name": "amdrocm-hipblas-common-dev10.1", - "sha256": "52aa990a98bbcb45ec5b6cc9e392f8cea38a895e0827eb0d6e96eff8a47381f3", + "sha256": "e8456d1a340533ae73e5a7cbe9aa66feea1f8cedef72971f00930167b80037a3", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-hipblas-common-dev10.1_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-hipblas-common-dev10.1_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [ { - "key": "amdrocm-llvm-dev10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-llvm-dev10.1_10.1.0-3_amd64", "name": "amdrocm-llvm-dev10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-llvm10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", "name": "amdrocm-llvm10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-runtime10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", "name": "amdrocm-runtime10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-base10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-base10.1_10.1.0-3_amd64", "name": "amdrocm-base10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -4388,9 +4244,9 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", "name": "amdrocm-sysdeps10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", @@ -4483,67 +4339,67 @@ "version": "2.35-0ubuntu3.10" } ], - "key": "amdrocm-runtime-dev10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-runtime-dev10.1_10.1.0-3_amd64", "name": "amdrocm-runtime-dev10.1", - "sha256": "30cc57738702d459a1e13452bb2c1a0a7195c7b31f2e23b9f5c9b7adf216b750", + "sha256": "452b3a3b31a4fa4da8d4c08976a5698aa05732b52ab88daa85cb4b4d2c84754c", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-runtime-dev10.1_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-runtime-dev10.1_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-llvm-dev10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-llvm-dev10.1_10.1.0-3_amd64", "name": "amdrocm-llvm-dev10.1", - "sha256": "94255d09d463bd77e01773bb43a7c8a12d64382c283b94a8d8b78e0ed27ef453", + "sha256": "351db702fd37be1fda7e5221aedf19484e99d27f2a3e7d2df5f0f531bcebcfd5", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-llvm-dev10.1_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-llvm-dev10.1_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [ { - "key": "amdrocm-dnn10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx950_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx950", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx950_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx950", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx950_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx950", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx950_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx950", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse-host10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse-host10.1_10.1.0-3_amd64", "name": "amdrocm-sparse-host10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-runtime10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", "name": "amdrocm-runtime10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-llvm10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", "name": "amdrocm-llvm10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", "name": "amdrocm-sysdeps10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -4766,9 +4622,9 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "amdrocm-base10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-base10.1_10.1.0-3_amd64", "name": "amdrocm-base10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -4861,915 +4717,822 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "amdrocm-profiler-base10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-profiler-base10.1_10.1.0-3_amd64", "name": "amdrocm-profiler-base10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver-host10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver-host10.1_10.1.0-3_amd64", "name": "amdrocm-solver-host10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-math-common10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-math-common10.1_10.1.0-3_amd64", "name": "amdrocm-math-common10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas-host10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas-host10.1_10.1.0-3_amd64", "name": "amdrocm-blas-host10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-dnn-host10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn-host10.1_10.1.0-3_amd64", "name": "amdrocm-dnn-host10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-dnn10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx942_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx942", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx942_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx942", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx942_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx942", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx942_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx942", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-dnn10.1-gfx90c_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-dnn10.1-gfx90c", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-blas10.1-gfx90c_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-blas10.1-gfx90c", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-solver10.1-gfx90c_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-solver10.1-gfx90c", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-sparse10.1-gfx90c_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-sparse10.1-gfx90c", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-dnn10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx90a_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx90a", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx90a_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx90a", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx90a_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx90a", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx90a_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx90a", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-dnn10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx908_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx908", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx908_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx908", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx908_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx908", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx908_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx908", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-dnn10.1-gfx906_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-dnn10.1-gfx906", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-blas10.1-gfx906_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-blas10.1-gfx906", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-solver10.1-gfx906_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-solver10.1-gfx906", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-sparse10.1-gfx906_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-sparse10.1-gfx906", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-dnn10.1-gfx900_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-dnn10.1-gfx900", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-blas10.1-gfx900_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-blas10.1-gfx900", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-solver10.1-gfx900_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-solver10.1-gfx900", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-sparse10.1-gfx900_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-sparse10.1-gfx900", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-dnn10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1250_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1250", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1250_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1250", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1250_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1250", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1250_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1250", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-dnn10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1201_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1201", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1201_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1201", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1201_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1201", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1201_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1201", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-dnn10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1200_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1200", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1200_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1200", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1200_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1200", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1200_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1200", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-dnn10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1153_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1153", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1153_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1153", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1153_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1153", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1153_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1153", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-dnn10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1152_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1152", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1152_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1152", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1152_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1152", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1152_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1152", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-dnn10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1151_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1151", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1151_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1151", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1151_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1151", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1151_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1151", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-dnn10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1150_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1150", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1150_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1150", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1150_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1150", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1150_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1150", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-dnn10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1103_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1103", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1103_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1103", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1103_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1103", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1103_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1103", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-dnn10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1102_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1102", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1102_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1102", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1102_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1102", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1102_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1102", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-dnn10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1101_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1101", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1101_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1101", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1101_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1101", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1101_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1101", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-dnn10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1100_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1100", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1100_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1100", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1100_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1100", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1100_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1100", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-dnn10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1036_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1036", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1036_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1036", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1036_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1036", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1036_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1036", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-dnn10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1035_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1035", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1035_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1035", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1035_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1035", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1035_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1035", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-dnn10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1034_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1034", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1034_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1034", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1034_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1034", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1034_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1034", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-dnn10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1033_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1033", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1033_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1033", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1033_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1033", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1033_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1033", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-dnn10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1032_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1032", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1032_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1032", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1032_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1032", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1032_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1032", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-dnn10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1031_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1031", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1031_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1031", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1031_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1031", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1031_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1031", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-dnn10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1030_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1030", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1030_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1030", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1030_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1030", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1030_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1030", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-dnn10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1012_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1012", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1012_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1012", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1012_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1012", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1012_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1012", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-dnn10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1011_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1011", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1011_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1011", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1011_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1011", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1011_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1011", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-dnn10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1010_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1010", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1010_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1010", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1010_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1010", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1010_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1010", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" } ], - "key": "amdrocm-dnn10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1_10.1.0-3_amd64", "name": "amdrocm-dnn10.1", - "sha256": "fabb0943631b83b6cec3e430b42ff6f5731de8087e62d68adac1683ee0d9391f", + "sha256": "2a65817ed4bd283ff908a6789c1beaa95e1232043b9c0261edb4f7b031e1880e", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx950_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx950", - "sha256": "4cb346e12960579f71d348a7d876e4b563b661ba7d97cc47066ee32593d74f8a", + "sha256": "b945f4d2f35c53226a43ef798c036e0ea23eb4844c12a9f917fb450a41e756d4", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx950_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx950_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn-host10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn-host10.1_10.1.0-3_amd64", "name": "amdrocm-dnn-host10.1", - "sha256": "996c4605b993f63e59508cd41f41854f39c9d79105be6a2189a36f9bd3bc00e4", + "sha256": "199c66da1e43484d21dfe1581bcf84deec3ee82d106605058b932d8c3dc86359", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn-host10.1_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn-host10.1_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx942_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx942", - "sha256": "f60883c8e76e1e896f9c8a3f0ef5315bc3fb475add98a99c3ccfb0b182d1c9e7", + "sha256": "78e43e7ecf4a52b895fc9500e778abc0d4882a7ad5f60caff8830bb2afd64119", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx942_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx942_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx90c_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-dnn10.1-gfx90c", - "sha256": "914f8c011c3157b09a80b75a6c93da4e208066cc27858f4369b5005cdc9ef379", - "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx90c_10.1.0~20260903-33697653391_amd64.deb" - ], - "version": "10.1.0~20260903-33697653391" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx90a_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx90a", - "sha256": "c0e10e44b49fa26dc954599cfe2892d0ccc01f6a7fbbe761b7f5b2d159f45f46", + "sha256": "6fa5ee1edecb61cc9434659cfece977c636da00b63d9886646848b876a60cba6", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx90a_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx90a_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx908_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx908", - "sha256": "9416803c73afc49146416549723c1c773477c35eadc5ba61bdfa889a5529cd5f", + "sha256": "349a3f6c0c9cf7981d5cc2476be98e0893a5b54033a541660e084a3011efba17", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx908_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx908_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx906_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-dnn10.1-gfx906", - "sha256": "518d95df9a8db9e400ab0373d8c15d8aa95b9016da450e44f9aaefc856adda77", - "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx906_10.1.0~20260903-33697653391_amd64.deb" - ], - "version": "10.1.0~20260903-33697653391" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx900_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-dnn10.1-gfx900", - "sha256": "f16c2aa5b65adce059ab714af64c950b4e6d4ebe4601e396d1a6f62aca07473b", - "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx900_10.1.0~20260903-33697653391_amd64.deb" - ], - "version": "10.1.0~20260903-33697653391" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1250_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1250", - "sha256": "ba1a9af51d934a5ce69ed28ae4392e39fd14c0f99607d9913cedc1ad4253d5d5", + "sha256": "bffbfd01dd43330cdc2a6957b4100c64983d9e94e0fecda43dd799ce6b545534", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1250_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1250_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1201_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1201", - "sha256": "4af7048c36883c5f8081446386508fa1ec7d1b786924498f0bb374a0b7236570", + "sha256": "d53c6f605df4faa6cfbc4ea9a3f3e75549dbec3400ab899ab502bb8d68681d3f", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1201_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1201_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1200_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1200", - "sha256": "e54eb6d0352b23bb13ce31bf51418e6b1d48c0fdcb0b5ef1a4789cf88f0a5180", + "sha256": "e8e27e5fe95a8092af0c7b287e1c8b2ebb4a4099677cb274a56d09d04f6effd6", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1200_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1200_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1153_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1153", - "sha256": "7346c8fa685deb046589ec848cda162f6b4b61e6f46e10af29ca61edb75e0198", + "sha256": "39609097b14134dfce5937940c0d6118f089f8963f41b88aaf82c6dbe36d8d88", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1153_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1153_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1152_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1152", - "sha256": "a92b0bb304eb8241f03c8c5fa7137c8ff219b685011a8ed522a950e8cb7ab551", + "sha256": "f9383ce30973bca37538eab93173f7a6f277189dc1f2c32285902284524b9f6c", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1152_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1152_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1151_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1151", - "sha256": "036da64732312dd730cc10442001d5606ab16fff39380e2023efc12986bfcfd3", + "sha256": "daf6239c37012e979b4b31612cc27c1f6b5df16c4d0a3eb9ffdb7149f6c0c88e", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1151_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1151_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1150_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1150", - "sha256": "e19418cc9c8520eba50f9a47a5efc6fdf8a899e17da9e180b9d03aafcbc371b9", + "sha256": "57370823faa258ee6c28650dbaa1afe4a983f0d431c0a4f59b45c52bfd5aa776", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1150_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1150_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1103_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1103", - "sha256": "e45cba470dd199188673f8d76e2f667d9417c49b0460b439a6854b6f3f6f2c7a", + "sha256": "31103c9e996fb78bb9fbee801b67401fb757bf0f909d35d22ee3c555f6f2419c", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1103_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1103_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1102_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1102", - "sha256": "d8a35cc470cc3c125d35af9c6bfccec90fe692ea7561753ba5060b48bb5cf9d3", + "sha256": "ba17cd8597c96637216596edc2146297bf3606e32e9bac7dce06147a8d171a06", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1102_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1102_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1101_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1101", - "sha256": "b5614e8e4533d2a9af33e98ba9d2de4c83f74898f6558a217aefb50a69c269c4", + "sha256": "25844db830121a1bb8cb3e64e1d04eb8fbda654a14c7e08b96eafce54891c789", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1101_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1101_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1100_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1100", - "sha256": "374a434a45a6b0f173c96886eadc168dc72b65fde7e23c49ee6766102ce82335", + "sha256": "7b742fc6696895809f20edaf373a22cd50fd19f17086c56d6cb144d025b32712", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1100_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1100_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1036_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1036", - "sha256": "44ef31ce86a418b83c63244b298092b553223f03f5b074a0359ace3f2887f6e8", + "sha256": "46658d6cf72bfc08aae94e13c7d7c34209714e700dd033b2c109203addc158b1", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1036_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1036_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1035_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1035", - "sha256": "5dbc1a29d58783dc73c722665c062cf215f2919f3c75c7c3bf25580f40586a95", + "sha256": "9d2165f2a2771936204ab97105a427c3b489ddc6984cebf8d3786822fbd566e9", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1035_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1035_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1034_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1034", - "sha256": "21f7823c4fd75c4f442283870856f64e94c7a92d86d60d7ca5812256f723d480", + "sha256": "d3c75ffa1b9f79165bfcad631d009149742780114e132b3cd3e4912eb0f23632", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1034_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1034_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1033_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1033", - "sha256": "8e0b558c18bab1a9757061e3286b3dd0ed54e940c05a41956764fa6fa24f31c4", + "sha256": "ebb48929bd2faf317aaa457096d4cefd6a9a2f8b9cbe2492bffb2e52135bdea3", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1033_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1033_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1032_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1032", - "sha256": "c7b3c1eed6dc0f5ecbb01e606aeda85390e70a36f681dc0b9d07d5e28a1103e8", + "sha256": "ee57ec63c768f031899bb630b42ca780bd753ce5c2543d04aea25bd86a04bf33", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1032_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1032_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1031_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1031", - "sha256": "708590387b6d623f62dd37f46bf2869ac70003b8906f8e9678e1968b5e97f021", + "sha256": "6ce6789fcbaaac92404402adeec89e1170ccd04cccc7340240338f506f3e0e27", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1031_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1031_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1030_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1030", - "sha256": "2be1d98c784bd9b0f9d3404e97f396e65c52aa9bdf0deb2ce4e0766831624c92", + "sha256": "e33aeb4b4acd112e37a278e006fe214a19927451e63d8bfd8db7cf4fb04a03a6", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1030_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1030_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1012_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1012", - "sha256": "fdf531b7ba1b5b5481c992917421da3b821693a1a1d58f1174b45c078a53d7fa", + "sha256": "21723b94ee22e0cc5caf839dab16c431e0b6b7f91d9a06cb22e58cc486ae7a5f", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1012_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1012_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1011_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1011", - "sha256": "f37873e0de4116d7ad6755e3237d09e9083d452f035006b017a8e0ad6f995f82", + "sha256": "4b1722d77fc4e0509802a1c56466cc72a8e208ade66f83b92c32cb54ec9c357e", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1011_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1011_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-dnn10.1-gfx1010_10.1.0-3_amd64", "name": "amdrocm-dnn10.1-gfx1010", - "sha256": "420f999e607c8788d05d14653e5b29c9162b968edfdaf27aa870055b0dae4590", + "sha256": "c9e57b5b1c75dcbf41dc69310adb3cdac7ef807e3119dd51a0357f2a9429f97a", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-dnn10.1-gfx1010_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1010_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [ { - "key": "amdrocm-fft10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx950_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx950", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-fft-host10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft-host10.1_10.1.0-3_amd64", "name": "amdrocm-fft-host10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", "name": "amdrocm-sysdeps10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -5792,14 +5555,14 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "amdrocm-runtime10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", "name": "amdrocm-runtime10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-llvm10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", "name": "amdrocm-llvm10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", @@ -6002,9 +5765,9 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "amdrocm-base10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-base10.1_10.1.0-3_amd64", "name": "amdrocm-base10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -6097,490 +5860,442 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "amdrocm-fft10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx942_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx942", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-fft10.1-gfx90c_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-fft10.1-gfx90c", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-fft10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx90a_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx90a", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-fft10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx908_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx908", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-fft10.1-gfx906_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-fft10.1-gfx906", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-fft10.1-gfx900_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-fft10.1-gfx900", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-fft10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1250_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1250", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-fft10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1201_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1201", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-fft10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1200_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1200", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-fft10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1153_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1153", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-fft10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1152_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1152", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-fft10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1151_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1151", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-fft10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1150_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1150", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-fft10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1103_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1103", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-fft10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1102_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1102", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-fft10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1101_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1101", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-fft10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1100_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1100", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-fft10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1036_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1036", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-fft10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1035_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1035", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-fft10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1034_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1034", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-fft10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1033_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1033", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-fft10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1032_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1032", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-fft10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1031_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1031", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-fft10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1030_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1030", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-fft10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1012_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1012", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-fft10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1011_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1011", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-fft10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1010_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1010", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" } ], - "key": "amdrocm-fft10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1_10.1.0-3_amd64", "name": "amdrocm-fft10.1", - "sha256": "6538002afe1d2a4f8d31e56f3cc83b2cd99092cadcf3324be831d2e7bb16c7bb", + "sha256": "0246b0530ab01388018cafba38ee64736933c3b5eeea0e21dcd9dafb44a5950b", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx950_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx950", - "sha256": "3ead73072ea39ebfecdb9d0736f50e83124fe2bf52ced522e02c762bc69ab367", + "sha256": "3be1ec8ef93a0d7c4d3c463fa92a0ab0f7422f09e5c278d486e42424897704c8", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx950_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx950_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft-host10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft-host10.1_10.1.0-3_amd64", "name": "amdrocm-fft-host10.1", - "sha256": "56ebdf77beb71f04cc4cf916fc9281ac4f24a09a0d9b4b6801c6052411497b02", + "sha256": "5eb075b4dd89ea9d500921959be55a469c60675f53c89e846e186da1d353c27e", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft-host10.1_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft-host10.1_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx942_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx942", - "sha256": "ac2378092eb6af2caeb6c483baadb4737121697675e906a6d55e1d4db30e1e37", + "sha256": "415771e4caefb3af0e91264a6ac232ab0a1401aa68d0e9c9b836cc177c609ac2", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx942_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx942_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft10.1-gfx90c_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-fft10.1-gfx90c", - "sha256": "a0f062551dec371c5d42ca76ca006860b2e6e80d3a0981beae56069642079a25", - "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx90c_10.1.0~20260903-33697653391_amd64.deb" - ], - "version": "10.1.0~20260903-33697653391" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx90a_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx90a", - "sha256": "d4a7a2a4cf2bba3332a6d1331246cc0ba2d4b9633ff69b62517efac22796e4a1", + "sha256": "c626c38512d954de19bcc298cfde38e419d4fc21d57665c10d92523c18419265", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx90a_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx90a_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx908_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx908", - "sha256": "43209ba15011876fc19e3910775c4b978f7f5c2628d45167c1b1f3defe1c8884", + "sha256": "3310f7fe026228c270df2b9b8100c2a64ef7aaae783a54e55ca5099fe57a718b", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx908_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx908_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft10.1-gfx906_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-fft10.1-gfx906", - "sha256": "c290e7eb143b593897dc3ee4b5fbdf5a9ccc5b211499471be13796ad1562ede0", - "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx906_10.1.0~20260903-33697653391_amd64.deb" - ], - "version": "10.1.0~20260903-33697653391" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx900_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-fft10.1-gfx900", - "sha256": "65dc89e130d932c97838a27123a7fec83433c3e3d1aac617f634a1b4ecc0754b", - "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx900_10.1.0~20260903-33697653391_amd64.deb" - ], - "version": "10.1.0~20260903-33697653391" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1250_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1250", - "sha256": "0d4392b50e74b7ca9c2c6b6263167a5aada416d9415bc3d29ed63c25dfdcfca1", + "sha256": "7b8cb472a25096913a4329f5dfc2e124e98621764aff2e04b79db5d7f6ce3ea8", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1250_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1250_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1201_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1201", - "sha256": "fbb4030d3040af10987cf0652439b1f4f59e6f6b62335392ff9462e17217bd90", + "sha256": "eb10f46c03ffdbc1be901e2fefc16fcd3dcd36079acc17bf2a991dda010ca22e", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1201_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1201_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1200_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1200", - "sha256": "c34b93ca4721dd052e7ba00a2450892bca08014686849a2c383bf61e22224e72", + "sha256": "04f88bce83b753730025bc352c61b16027c2a6d565f85763ce8990faf155231f", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1200_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1200_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1153_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1153", - "sha256": "b0cd1b3a33e8d6170631f82c5b38e703664750d6ff31ecdf2293c990b557bacb", + "sha256": "cd7270fd487b7abea0498a832122c11186ac234981822274af84520b8980c0c3", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1153_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1153_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1152_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1152", - "sha256": "750516dba73b09a083f7a1ac2cd14097200f2e59c2d0230b3fedddefd3c1f7b8", + "sha256": "67250ae45a6190b8e0f0bea35a8019a8fd5a5da070432206a32c4ae3388f2e0e", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1152_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1152_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1151_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1151", - "sha256": "d647b4847f0780c6e4474fd201571373693c6abbece5577e209c99391ffb90cb", + "sha256": "f70434a361d0207188a0feff30d677164c1fdcdb0af970acbc55561b3bfc42f5", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1151_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1151_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1150_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1150", - "sha256": "1abe94fe014ac7af44c6547de43b3a14a0db82a5b18fde7456c1a4c144abfffc", + "sha256": "26b23859562047c84e8eb4ff7387bcab69b475eeb8655e577b0c95e436e6c2f4", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1150_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1150_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1103_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1103", - "sha256": "a5b3ebadfb033886fd7c637cb8122f397171e511a1d28616da8cf4511a22cbfa", + "sha256": "0cd8464f08f8b6b08ed10773352a34d1d1efea4de3c039a36e6f3bb4fdfb0f9d", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1103_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1103_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1102_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1102", - "sha256": "d9b467c06abc268e5ca2d26b8a2c0da64b48f733e54cabdc18b06c18de23de16", + "sha256": "ec4b9439df683ee782241ec4551daa4e0ec4e4ae4a8c2b92f1bd41285cad596e", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1102_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1102_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1101_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1101", - "sha256": "9c5e08a8690073a9dd9113f99265321a913e3624fea6d8ed7a0ab65bdf060a60", + "sha256": "2ed4b2fbabb310fdb7fdf8ed01c9ca1f0f37fe97b5e3027451737be4815492a8", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1101_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1101_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1100_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1100", - "sha256": "f85432c5c9282e643212c3692ebc9a0805f5d15be6f13f2432294a699b8a488e", + "sha256": "df323baa9cfb3a2c2dd81bd81510021db4780eb074717b22abb9dbd80d992dc0", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1100_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1100_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1036_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1036", - "sha256": "b4fd30b8f178a66727f26baa6459a8e73153853f4a4a79bcf99f45ca2aa1ac91", + "sha256": "6eb45f4a815e10d2119370e0ed68fe44bc34c2999c3b0a799d15d23d96f17ed8", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1036_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1036_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1035_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1035", - "sha256": "40305b1b8c983ac3c12a553058deeddc8b47b67c247570214eea10baa1c3c27d", + "sha256": "1da409970e160027c8e5fbf9d446394fa9108b92163ba69796b7d7f598f65d18", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1035_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1035_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1034_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1034", - "sha256": "8b8ac4bafef4808c60d2f96a36755c33f167d22a5fec79f9588f5fb8501d80c8", + "sha256": "58351fe6a54cc374a3d94a61416c7c4079303e2b38e974d54381bd4f5b53f83e", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1034_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1034_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1033_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1033", - "sha256": "06e3a930cee8af98bb654201afec55d2d7541cad785d2bba9bf50e14bdd88f4b", + "sha256": "98bb8653582f9840ba723b0344204be8f6c55c758e756acec6ee372e9233119d", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1033_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1033_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1032_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1032", - "sha256": "5e991ac1f2724adb5c13a901404317c67b863a53b507abf8ac2e8be4de57f257", + "sha256": "e4b0fc5dab2eaac470a54811d19bef3b9c1f903e94d5dfa56a4f3f05b8d5977d", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1032_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1032_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1031_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1031", - "sha256": "dc5a63b123e63d8c816557e75c3f2b571e6001027941635c5c4af497d9ce3ada", + "sha256": "998eae13396245c496da21504fff6832654b50323cc40cb694060a79f943a51c", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1031_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1031_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1030_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1030", - "sha256": "0b4174ac0339450b0c63d9477afeab84fb98310453401e5a73a408e2dbc50eba", + "sha256": "e321ced70752e27d4c8b70eaf3b8590b9ce6019d64896cb5cfca75d4ad26bf89", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1030_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1030_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1012_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1012", - "sha256": "1997587ecc2aa1eb380b62b51061b50c2f7ffc00036214162b16191deffa56d5", + "sha256": "b8910ca5eea38a9b9e68d3752637e214ded07e84b4ac83da39f15157cc56bf64", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1012_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1012_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1011_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1011", - "sha256": "da905e999e4165475884afcc0eae9f37363ab4cca3ae9423d5e665fcf7b502d2", + "sha256": "f6cc9a8657250ba79d739ca65bc8cb13491629695b9c486060652ea9fb508656", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1011_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1011_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-fft10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-fft10.1-gfx1010_10.1.0-3_amd64", "name": "amdrocm-fft10.1-gfx1010", - "sha256": "4be01dc7c5f64496bf197efefa26859a5ae1f70d6f657e2a7dd5025ed05c4fc5", + "sha256": "378078f09766726732a4f192397b6f1d9fee346b68efd519b58238f9c41f895a", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-fft10.1-gfx1010_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1010_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [ { - "key": "amdrocm-rccl10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx950_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx950", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-rccl-host10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl-host10.1_10.1.0-3_amd64", "name": "amdrocm-rccl-host10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-profiler-base10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-profiler-base10.1_10.1.0-3_amd64", "name": "amdrocm-profiler-base10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", "name": "amdrocm-sysdeps10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -6603,14 +6318,14 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "amdrocm-runtime10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", "name": "amdrocm-runtime10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-llvm10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", "name": "amdrocm-llvm10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", @@ -6813,9 +6528,9 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "amdrocm-base10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-base10.1_10.1.0-3_amd64", "name": "amdrocm-base10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -6908,436 +6623,388 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "amdrocm-amdsmi10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-amdsmi10.1_10.1.0-3_amd64", "name": "amdrocm-amdsmi10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-rccl10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx942_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx942", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-rccl10.1-gfx90c_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-rccl10.1-gfx90c", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-rccl10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx90a_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx90a", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-rccl10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx908_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx908", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-rccl10.1-gfx906_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-rccl10.1-gfx906", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-rccl10.1-gfx900_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-rccl10.1-gfx900", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-rccl10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1250_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1250", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-rccl10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1201_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1201", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-rccl10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1200_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1200", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-rccl10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1151_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1151", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-rccl10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1102_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1102", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-rccl10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1101_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1101", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-rccl10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1100_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1100", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-rccl10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1036_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1036", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-rccl10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1035_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1035", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-rccl10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1034_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1034", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-rccl10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1033_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1033", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-rccl10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1032_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1032", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-rccl10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1031_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1031", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-rccl10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1030_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1030", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-rccl10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1012_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1012", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-rccl10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1011_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1011", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-rccl10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1010_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1010", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" } ], - "key": "amdrocm-rccl10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1_10.1.0-3_amd64", "name": "amdrocm-rccl10.1", - "sha256": "790722d01adc9c1bf8c800534068fcb372caf43fe6fbff51ce8fc8e1ab7cc8e5", + "sha256": "18b5e340191a729524ede9ec9d1810f3673d035b4940743307189e1b4a96d293", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-rccl10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx950_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx950", - "sha256": "24bc92cdaa7974eaab279e38e1af94478adf57be1e2ab13399e684a2b65a5be7", + "sha256": "d9a77415ca9392af357d188df8e805f86c8b0f7574705133ee919b88fec66f4d", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx950_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx950_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-rccl-host10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl-host10.1_10.1.0-3_amd64", "name": "amdrocm-rccl-host10.1", - "sha256": "6444b12d36c247cfc955829de234ef784ee6298beed04a5f5a6bd560fe791be8", + "sha256": "b38172c360fc8fdf89d038edfa3cb5c3cb874804f089a4c9ba91c384be7c9707", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl-host10.1_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl-host10.1_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-rccl10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx942_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx942", - "sha256": "7691821aebf05ff3beeb7db947b60999de23a00b12f585fb3b66462cb71e903d", + "sha256": "c0c4be063b49364858d76af509cbf88e4642146489a1fcde2088f66a2e1f31a9", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx942_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx942_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-rccl10.1-gfx90c_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-rccl10.1-gfx90c", - "sha256": "52564c6944164aa422f2b2cb898ce833e8ae48617c8ac6ee56adc2146e2c10c4", - "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx90c_10.1.0~20260903-33697653391_amd64.deb" - ], - "version": "10.1.0~20260903-33697653391" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx90a_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx90a", - "sha256": "774ef85c35b4824f5689f70de9092b9cb18a9a467fec78779404d6ddba7b0426", + "sha256": "ffcc6bce6b2271b3c6cd1c5d7b568fe2773a0022a2ba4d24c177f7df5d6c5225", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx90a_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx90a_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-rccl10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx908_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx908", - "sha256": "416468a8580570e18bf4b82ab276babdd6f5e8e870667d661c4e4f5fb73fa64b", + "sha256": "2eebb6a869970c182dd64105da18f956801ed7a048693f47f99a05a087b01f59", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx908_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx908_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-rccl10.1-gfx906_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-rccl10.1-gfx906", - "sha256": "e3a50d7f0e3104bdaea8dd2e20a1c9ad6cb77ad8147eea9510cad838bf523223", - "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx906_10.1.0~20260903-33697653391_amd64.deb" - ], - "version": "10.1.0~20260903-33697653391" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl10.1-gfx900_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-rccl10.1-gfx900", - "sha256": "94d3cefa9c8993b3437fd20cb2f6628deac396d5ae23c9b7c641908ee77c8e50", - "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx900_10.1.0~20260903-33697653391_amd64.deb" - ], - "version": "10.1.0~20260903-33697653391" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1250_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1250", - "sha256": "570a58e63a70e13dad9d14334a48857cb9a759ac60651f36fa2df231eb3e9e6e", + "sha256": "941da91fc0f827732d0c4e0010403723643a27dd3ca2494270aa97d0a8a38f35", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1250_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1250_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1201_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1201", - "sha256": "5abe6dfe6097048cfa5a22fd6705889cf61363667008fd8cc0bee1f585d69ca7", + "sha256": "4c589250c3c587d4865551b15f32adb46cfbb5aaa07e147eb82b3d6634cff213", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1201_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1201_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1200_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1200", - "sha256": "03278667e2e95a8cbc979e4de10104aecc1b521624dbb043ba2567931469bcec", + "sha256": "33086fcdf5aa8e353c90ac38effddca41cb3d8ae88272333df2c580b08826abe", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1200_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1200_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1151_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1151", - "sha256": "554fa2650e4619f324562ccbc568f1f44c16cb22816d51680c48c132fd7c8693", + "sha256": "4d7b24fc17351d69c63028867b5bdf4b9015e563175c868a814c8ca34b7bbcce", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1151_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1151_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1102_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1102", - "sha256": "021051e9cf9cb49a82647041ec449c35de2e540302b8a0da0200a4933713197c", + "sha256": "935b928d8f2901861021d3f9cb2c43953d309975c4e5803b4ac496d6bd4ee007", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1102_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1102_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1101_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1101", - "sha256": "66563039aeb86fdda465d6be01d0429f205bbc028deaf75048c023ceb167d462", + "sha256": "4d418a9fb372ec15bd9e42faa02f9bf945adc5de2853cb77db54cb658286c67c", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1101_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1101_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1100_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1100", - "sha256": "f3a239c13110330a21a84327e15c631c35e7d2522c736899d2a4c7b453084ea2", + "sha256": "ceddf293510c927d2a5a8fee30cc76fce6cd0d3cc6977422396514bedbcc597e", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1100_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1100_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1036_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1036", - "sha256": "dce7d2a9c8136e63a33b432e58fa64efb6bf9bc00c39485b2ad472332ce54b22", + "sha256": "8fb0c4dfa528a82c872df49b0feeb7205e9664ce06d9760d0fa85a1892d97f79", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1036_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1036_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1035_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1035", - "sha256": "be1dc0fa1a1d01aa9fe831a18342690383c1497e746162e42b4cf5d6b899357e", + "sha256": "174385d65d68e5c5aa7c51f5cfcf90fae9005c37944f4515883f370760830b74", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1035_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1035_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1034_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1034", - "sha256": "b515cda1f55817a4350fad0957dff5693a6b55577cc4923afd2a9a41d09ab56a", + "sha256": "c3e4727c061c15aa67b89b61539d58be182dca850df13defe1790b7b5b7ffd4d", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1034_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1034_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1033_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1033", - "sha256": "70ea2ed56caf665952eb520d059503bca46f45b00c3f34501ace894af09ce9d2", + "sha256": "521d5e2bc7c7d8f4d941044472fd9f800d5f172d4c9a3c015b035e2454c2dba1", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1033_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1033_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1032_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1032", - "sha256": "1d99c301bfe389f335fee87b99436bab7da3f11dd56c4b9732330e6f03079d93", + "sha256": "198345dc05c10db202080744337fee857c367786a315004e662dc24340cff787", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1032_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1032_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1031_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1031", - "sha256": "e1740e46bd4ada5565cac1ec827971838ee4e23b30eba1ba242e3be47fa1a23d", + "sha256": "91761516a93977fcc70cb771e5ad2e7f958f8147905ca86adf592e5f7d79dab5", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1031_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1031_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1030_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1030", - "sha256": "9608e4772bcd714e8fdca999501539acd36e504114e3cda32ee5d10fcabe82a9", + "sha256": "076013938bd6ef83e31d5c4f444d7fb136d0733a4edfdfccf05d8108776c3388", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1030_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1030_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1012_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1012", - "sha256": "6a4e521f0bbc6d07d49eb8856fae118f78cfd14a13210f2c7926fcd3191f0721", + "sha256": "473c3be9bf38e2f30dac5f6019a521c3aad2b96e10e28389f542c799141b8f09", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1012_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1012_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1011_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1011", - "sha256": "8a05e6996a2c97d37014fb743fe751f0e5f7caf87424f846f64e9e785d315032", + "sha256": "842068f0334f6c3d4977394c9ee5098f8a8254012206b5f66f5b2aaa0e510ce1", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1011_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1011_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-rccl10.1-gfx1010_10.1.0-3_amd64", "name": "amdrocm-rccl10.1-gfx1010", - "sha256": "7098b435b3bba65a309532d154d71d8e477d54a3b28fc8b0b7f976a866f61b67", + "sha256": "09c782685d0d6a93e767a580dc5e35183cf9f742859953271d8abaa977a74c6e", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-rccl10.1-gfx1010_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1010_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [ { - "key": "amdrocm-solver10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx950_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx950", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx950_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx950", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas-host10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas-host10.1_10.1.0-3_amd64", "name": "amdrocm-blas-host10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-profiler-base10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-profiler-base10.1_10.1.0-3_amd64", "name": "amdrocm-profiler-base10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", "name": "amdrocm-sysdeps10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -7360,14 +7027,14 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "amdrocm-runtime10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", "name": "amdrocm-runtime10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-llvm10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", "name": "amdrocm-llvm10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", @@ -7570,9 +7237,9 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "amdrocm-base10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-base10.1_10.1.0-3_amd64", "name": "amdrocm-base10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -7665,466 +7332,421 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "amdrocm-sparse10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx950_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx950", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse-host10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse-host10.1_10.1.0-3_amd64", "name": "amdrocm-sparse-host10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver-host10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver-host10.1_10.1.0-3_amd64", "name": "amdrocm-solver-host10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-math-common10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-math-common10.1_10.1.0-3_amd64", "name": "amdrocm-math-common10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx942_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx942", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx942_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx942", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx942_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx942", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx90c_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-solver10.1-gfx90c", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-blas10.1-gfx90c_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-blas10.1-gfx90c", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-sparse10.1-gfx90c_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-sparse10.1-gfx90c", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-solver10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx90a_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx90a", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx90a_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx90a", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx90a_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx90a", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx908_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx908", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx908_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx908", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx908_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx908", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx906_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-solver10.1-gfx906", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-blas10.1-gfx906_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-blas10.1-gfx906", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-sparse10.1-gfx906_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-sparse10.1-gfx906", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-solver10.1-gfx900_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-solver10.1-gfx900", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-blas10.1-gfx900_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-blas10.1-gfx900", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-sparse10.1-gfx900_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-sparse10.1-gfx900", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-solver10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1250_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1250", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1250_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1250", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1250_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1250", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1201_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1201", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1201_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1201", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1201_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1201", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1200_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1200", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1200_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1200", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1200_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1200", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1153_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1153", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1153_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1153", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1153_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1153", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1152_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1152", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1152_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1152", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1152_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1152", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1151_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1151", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1151_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1151", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1151_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1151", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1150_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1150", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1150_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1150", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1150_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1150", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1103_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1103", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1103_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1103", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1103_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1103", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1102_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1102", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1102_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1102", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1102_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1102", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1101_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1101", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1101_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1101", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1101_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1101", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1100_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1100", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1100_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1100", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1100_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1100", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1036_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1036", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1036_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1036", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1036_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1036", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1035_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1035", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1035_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1035", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1035_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1035", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1034_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1034", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1034_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1034", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1034_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1034", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1033_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1033", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1033_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1033", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1033_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1033", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1032_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1032", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1032_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1032", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1032_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1032", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1031_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1031", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1031_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1031", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1031_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1031", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1030_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1030", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1030_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1030", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1030_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1030", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1012_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1012", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1012_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1012", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1012_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1012", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1011_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1011", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1011_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1011", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1011_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1011", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1010_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1010", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1010_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1010", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1010_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1010", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" } ], - "key": "amdrocm-solver10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1_10.1.0-3_amd64", "name": "amdrocm-solver10.1", - "sha256": "7fdefce3a45950fd85acd08a7dad35a227be225b4abc544fbb9292cce850df79", + "sha256": "66f914aa374d13a700da9006af9dcd4aeae20c02a50b4807c45d3ed990da0711", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-solver10.1_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [ { - "key": "amdrocm-sparse10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx950_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx950", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx950_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx950", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx950_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx950_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx950", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver-host10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver-host10.1_10.1.0-3_amd64", "name": "amdrocm-solver-host10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-math-common10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-math-common10.1_10.1.0-3_amd64", "name": "amdrocm-math-common10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -8147,29 +7769,29 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "amdrocm-blas-host10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas-host10.1_10.1.0-3_amd64", "name": "amdrocm-blas-host10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-profiler-base10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-profiler-base10.1_10.1.0-3_amd64", "name": "amdrocm-profiler-base10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sysdeps10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", "name": "amdrocm-sysdeps10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-runtime10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", "name": "amdrocm-runtime10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-llvm10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", "name": "amdrocm-llvm10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", @@ -8372,9 +7994,9 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "amdrocm-base10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-base10.1_10.1.0-3_amd64", "name": "amdrocm-base10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -8467,423 +8089,378 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "amdrocm-sparse-host10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse-host10.1_10.1.0-3_amd64", "name": "amdrocm-sparse-host10.1", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx942_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx942", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx942_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx942", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx942_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx942_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx942", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx90c_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-sparse10.1-gfx90c", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-blas10.1-gfx90c_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-blas10.1-gfx90c", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-solver10.1-gfx90c_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-solver10.1-gfx90c", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-sparse10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx90a_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx90a", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx90a_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx90a", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx90a_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx90a_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx90a", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx908_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx908", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx908_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx908", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx908_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx908_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx908", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx906_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-sparse10.1-gfx906", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-blas10.1-gfx906_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-blas10.1-gfx906", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-solver10.1-gfx906_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-solver10.1-gfx906", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-sparse10.1-gfx900_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-sparse10.1-gfx900", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-blas10.1-gfx900_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-blas10.1-gfx900", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-solver10.1-gfx900_10.1.0_20260903-33697653391_amd64", - "name": "amdrocm-solver10.1-gfx900", - "version": "10.1.0~20260903-33697653391" - }, - { - "key": "amdrocm-sparse10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1250_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1250", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1250_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1250", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1250_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1250_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1250", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1201_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1201", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1201_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1201", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1201_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1201_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1201", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1200_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1200", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1200_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1200", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1200_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1200_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1200", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1153_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1153", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1153_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1153", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1153_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1153_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1153", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1152_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1152", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1152_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1152", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1152_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1152_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1152", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1151_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1151", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1151_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1151", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1151_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1151_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1151", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1150_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1150", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1150_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1150", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1150_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1150_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1150", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1103_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1103", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1103_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1103", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1103_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1103_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1103", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1102_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1102", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1102_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1102", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1102_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1102_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1102", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1101_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1101", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1101_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1101", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1101_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1101_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1101", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1100_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1100", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1100_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1100", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1100_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1100_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1100", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1036_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1036", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1036_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1036", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1036_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1036_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1036", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1035_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1035", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1035_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1035", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1035_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1035_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1035", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1034_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1034", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1034_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1034", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1034_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1034_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1034", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1033_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1033", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1033_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1033", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1033_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1033_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1033", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1032_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1032", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1032_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1032", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1032_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1032_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1032", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1031_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1031", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1031_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1031", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1031_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1031_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1031", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1030_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1030", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1030_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1030", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1030_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1030_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1030", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1012_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1012", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1012_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1012", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1012_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1012_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1012", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1011_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1011", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1011_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1011", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1011_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1011_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1011", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-sparse10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1-gfx1010_10.1.0-3_amd64", "name": "amdrocm-sparse10.1-gfx1010", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-blas10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-blas10.1-gfx1010_10.1.0-3_amd64", "name": "amdrocm-blas10.1-gfx1010", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" }, { - "key": "amdrocm-solver10.1-gfx1010_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-solver10.1-gfx1010_10.1.0-3_amd64", "name": "amdrocm-solver10.1-gfx1010", - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" } ], - "key": "amdrocm-sparse10.1_10.1.0_20260903-33697653391_amd64", + "key": "amdrocm-sparse10.1_10.1.0-3_amd64", "name": "amdrocm-sparse10.1", - "sha256": "ce618ab8b73fe2d57e6d62f1edec18cd6a850e2bfd33a31fcb6d7bcf0a375a3d", + "sha256": "71a68bcbb89fdac799535916b602a7f456269a24e2e40d0151abd003c7ad1b58", "urls": [ - "https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391/pool/main/amdrocm-sparse10.1_10.1.0~20260903-33697653391_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1_10.1.0-3_amd64.deb" ], - "version": "10.1.0~20260903-33697653391" + "version": "10.1.0-3" } ], "version": 1 diff --git a/platforms/rocm/packages.yaml b/platforms/rocm/packages.yaml index 40b74c93..c9116561 100644 --- a/platforms/rocm/packages.yaml +++ b/platforms/rocm/packages.yaml @@ -7,7 +7,7 @@ sources: - channel: llvm-toolchain-jammy-22 main url: https://apt.llvm.org/jammy - channel: stable main - url: https://nightly.repo.amd.com/rocm/core/packages/ubuntu2204/20260903-33697653391 + url: https://stable.repo.amd.com/rocm/core/packages/ubuntu2204 - channel: jammy main url: https://snapshot.ubuntu.com/ubuntu/20250711T030400Z - channel: jammy-security main diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 0368b288..819a3297 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610150915.220.1.f2470651679c/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "ec895a112ec2f36e4f73a845498235748da47032f32c218605a39c25d39c97ba" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610151602.227.1.e215eb02db73/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "7d418c7404c99e078de6fcba2e4fccf324f25f1219cd0176caac9e8d14ab222c" def _rocm_package_name(name): return name + _ROCM_VERSION diff --git a/third_party/rules_distroless/flat-repositories.patch b/third_party/rules_distroless/flat-repositories.patch new file mode 100644 index 00000000..e4ba73bc --- /dev/null +++ b/third_party/rules_distroless/flat-repositories.patch @@ -0,0 +1,59 @@ +diff --git a/apt/private/deb_resolve.bzl b/apt/private/deb_resolve.bzl +--- a/apt/private/deb_resolve.bzl ++++ b/apt/private/deb_resolve.bzl +@@ -46,7 +46,10 @@ + sources = [] + + for src in manifest["sources"]: +- distr, components = src["channel"].split(" ", 1) ++ if src["channel"].endswith("/"): ++ distr, components = src["channel"], "" ++ else: ++ distr, components = src["channel"].split(" ", 1) + for comp in components.split(" "): + # TODO: only support urls before 1.0 + if "urls" in src: +@@ -60,6 +63,7 @@ + urls, + distr, + comp, ++ src.get("archs", manifest["archs"]), + )) + + repository = deb_repository.new(rctx, sources = sources, archs = manifest["archs"]) +diff --git a/apt/private/apt_deb_repository.bzl b/apt/private/apt_deb_repository.bzl +--- a/apt/private/apt_deb_repository.bzl ++++ b/apt/private/apt_deb_repository.bzl +@@ -13,7 +13,7 @@ + return use_netrc(netrc, urls, {}) + + def _fetch_package_index(rctx, urls, dist, comp, arch, integrity): +- target_triple = "{dist}/{comp}/{arch}".format(dist = dist, comp = comp, arch = arch) ++ target_triple = "{dist}/{comp}/{arch}".format(dist = dist.strip("/") or ".", comp = comp, arch = arch) + + # See https://linux.die.net/man/1/xz , https://linux.die.net/man/1/gzip , and https://linux.die.net/man/1/bzip2 + # --keep -> keep the original file (Bazel might be still committing the output to the cache) +@@ -35,7 +35,10 @@ + download = None + for (ext, cmd) in supported_extensions: + output = "{}/Packages{}".format(target_triple, ext) +- dist_url = "{}/dists/{}/{}/binary-{}/Packages{}".format(url, dist, comp, arch, ext) ++ if dist.endswith("/"): ++ dist_url = "{}/{}Packages{}".format(url, dist.lstrip("/"), ext) ++ else: ++ dist_url = "{}/dists/{}/{}/binary-{}/Packages{}".format(url, dist, comp, arch, ext) + auth = {} + if url in base_auth: + auth = {dist_url: base_auth[url]} +@@ -167,7 +170,10 @@ + ) + + for arch in archs: +- for (urls, dist, comp) in sources: ++ for (urls, dist, comp, source_archs) in sources: ++ if arch not in source_archs: ++ continue ++ + # We assume that `url` does not contain a trailing forward slash when passing to + # functions below. If one is present, remove it. Some HTTP servers do not handle + # redirects properly when a path contains "//" diff --git a/third_party/xla/repo.bzl b/third_party/xla/repo.bzl index 8db1747f..0e1d9620 100644 --- a/third_party/xla/repo.bzl +++ b/third_party/xla/repo.bzl @@ -4,7 +4,7 @@ def repo(): git_repository( name = "xla", remote = "https://github.com/openxla/xla.git", - commit = "f3dee9ae87745e151837b2ff1b0485928f8e5a5d", + commit = "65aed3e0ed7b0f965825676dd27e312965c846c8", patches = [ "//third_party/xla:cuda-root-path-local-defines.patch", ], From 7bb457ebebbaa4e4a3ab52c35c05d15e94cdb80f Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 16 Oct 2026 12:30:05 +0200 Subject: [PATCH 789/881] zml: refactor manualComputation (#925) here I'm promoting a pattern I would like to extend to more of ZML: function that takes `anytype` argument should do as little work as possible and called a fully typed function. I kept `manualComputation` signature to not break existing code, I moved the code to `zml/ops/manual_computation.zig` to better encapsulate the required helper functions. (and `zml/ops.zig` is pretty big atm) Similarly `Sharding.Partitioning.parse` now calls `const logical = comptime Logical.parse(partitioning);` then `logical.applyUseAll(tags, mesh, default_value);` which is fully typed. I'm also documenting `manualComputation` better --- examples/llm/models/qwen3_5/model.zig | 44 +- examples/llm/models/qwen3_5_moe/model.zig | 26 +- mlir/dialects/shardy.zig | 14 +- stdx/bounded_array.zig | 9 + zml/BUILD.bazel | 1 + zml/Compiler.zig | 15 +- zml/Sharding.zig | 107 +++- zml/buffer.zig | 4 +- zml/buffer/HostAccessible.zig | 4 +- zml/kernels/flash_attention/nki/nki.zig | 11 +- .../paged_attention/mosaic_tpu/mosaic_tpu.zig | 5 +- zml/meta.zig | 39 ++ zml/ops.zig | 455 +----------------- zml/ops/manual_computation.zig | 301 ++++++++++++ zml/tensor.zig | 2 +- zml/testing.zig | 7 + 16 files changed, 521 insertions(+), 523 deletions(-) create mode 100644 zml/ops/manual_computation.zig diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 8d028cc5..d43bf7ee 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -592,8 +592,8 @@ pub const SelfAttn = struct { return .{ k, v }; } - fn partitionProjectedKv(kv: zml.Tensor, kv_head_meshe: zml.Sharding.DimMeshe) zml.Tensor { - return switch (kv_head_meshe) { + fn partitionProjectedKv(kv: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { + return switch (kv_head_sharding) { .sharded => |heads| blk: { const sharded_kv = if (heads.factor != 1) kv.stutter1d(kv.axis(.h), heads.factor) else kv; break :blk sharded_kv.withPartitioning(.model, .{ .s = .replicated, .h = .model, .hd = .replicated }); @@ -602,9 +602,9 @@ pub const SelfAttn = struct { }; } - fn partitionCachedKv(tensor: zml.Tensor, kv_head_meshe: zml.Sharding.DimMeshe) zml.Tensor { + fn partitionCachedKv(tensor: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { var cache_tensor = tensor.rename(.{ .s = .k }); - return switch (kv_head_meshe) { + return switch (kv_head_sharding) { .sharded => |heads| blk: { if (heads.factor != 1) { cache_tensor = cache_tensor.stutter1d(cache_tensor.axis(.h), heads.factor); @@ -626,10 +626,10 @@ pub const SelfAttn = struct { var q, var gate = self.projectQAndGate(x_qkv); var k, var v = self.projectKV(x_qkv); - const kv_head_meshe = tp.shardableDim(k.dim(.h), .model, q.dim(.h)); + const kv_head_sharding = tp.shardableDim(k.dim(.h), .model, q.dim(.h)); - k = partitionProjectedKv(k, kv_head_meshe); - v = partitionProjectedKv(v, kv_head_meshe); + k = partitionProjectedKv(k, kv_head_sharding); + v = partitionProjectedKv(v, kv_head_sharding); q = self.q_norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); k = self.k_norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); @@ -641,15 +641,15 @@ pub const SelfAttn = struct { const cos, const sin = self.rotary_embed.getCosAndSin(position_ids, dtype); q = self.rotary_embed.applyRope(q, cos, sin); k = self.rotary_embed.applyRope(k, cos, sin); - k = partitionProjectedKv(k, kv_head_meshe); - v = partitionProjectedKv(v, kv_head_meshe); + k = partitionProjectedKv(k, kv_head_sharding); + v = partitionProjectedKv(v, kv_head_sharding); const new_kv_cache = kv_cache.update(k, v, token_index.convert(.u32)); k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); q = q.rename(.{ .s = .q }); - k = partitionCachedKv(k, kv_head_meshe); - v = partitionCachedKv(v, kv_head_meshe); + k = partitionCachedKv(k, kv_head_sharding); + v = partitionCachedKv(v, kv_head_sharding); const attn_output = zml.attention.attention( q, @@ -1021,7 +1021,7 @@ pub const KvCache = struct { pub const Buffers = zml.Bufferized(SelfAttnCache); - pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, model_meshe: *const zml.Mesh) SelfAttnCache { + pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, tp: *const zml.Mesh) SelfAttnCache { const num_self_attn_layers = countLayers(config.text_config.layer_types, .full_attention); const kv_shape = zml.Shape.init(.{ .b = batch_dim, @@ -1030,10 +1030,10 @@ pub const KvCache = struct { .h = config.text_config.num_key_value_heads, .hd = config.text_config.head_dim, }, dtype); - const kv_head_meshe = model_meshe.shardableDim(kv_shape.dim(.h), .model, config.text_config.num_attention_heads); - const sharded_kv_shape = switch (kv_head_meshe) { - .sharded => |heads| kv_shape.setDim(.h, heads.dim).withPartitioning(model_meshe, .{ .h = .model }), - .replicated => kv_shape.withPartitioning(model_meshe, .{ .h = .replicated }), + const kv_head_sharding = tp.shardableDim(kv_shape.dim(.h), .model, config.text_config.num_attention_heads); + const sharded_kv_shape = switch (kv_head_sharding) { + .sharded => |heads| kv_shape.setDim(.h, heads.dim).withPartitioning(tp, .{ .h = .model }), + .replicated => kv_shape.withPartitioning(tp, .{ .h = .replicated }), }; return .{ .k = .fromShape(sharded_kv_shape), @@ -1122,7 +1122,7 @@ pub const KvCache = struct { pub const Buffers = zml.Bufferized(GatedDeltaNetCache); - pub fn init(config: Config, batch_dim: i64, conv_dtype: zml.DataType, recurrent_dtype: zml.DataType, model_meshe: *const zml.Mesh) GatedDeltaNetCache { + pub fn init(config: Config, batch_dim: i64, conv_dtype: zml.DataType, recurrent_dtype: zml.DataType, tp: *const zml.Mesh) GatedDeltaNetCache { const num_linear_attn_layers = countLayers(config.text_config.layer_types, .linear_attention); const conv_dim = 2 * config.text_config.linear_num_key_heads * config.text_config.linear_key_head_dim + @@ -1140,8 +1140,8 @@ pub const KvCache = struct { .khd = config.text_config.linear_key_head_dim, .vhd = config.text_config.linear_value_head_dim, }, recurrent_dtype); - const sharded_conv_state_shape = conv_state_shape.withPartitioning(model_meshe, .{ .mix = .model }); - const sharded_recurrent_state_shape = recurrent_state_shape.withPartitioning(model_meshe, .{ .vh = .model }); + const sharded_conv_state_shape = conv_state_shape.withPartitioning(tp, .{ .mix = .model }); + const sharded_recurrent_state_shape = recurrent_state_shape.withPartitioning(tp, .{ .vh = .model }); return .{ .conv_state = zml.Tensor.fromShape(sharded_conv_state_shape), .recurrent_state = zml.Tensor.fromShape(sharded_recurrent_state_shape), @@ -1226,12 +1226,12 @@ pub const KvCache = struct { max_seq_len: i64, cache_dtype: zml.DataType, recurrent_dtype: zml.DataType, - model_meshe: *const zml.Mesh, + tp: *const zml.Mesh, ) KvCache { return .{ .layer_types = config.text_config.layer_types, - .self_attn = .init(config, batch_dim, max_seq_len, cache_dtype, model_meshe), - .gated_delta_net = .init(config, batch_dim, cache_dtype, recurrent_dtype, model_meshe), + .self_attn = .init(config, batch_dim, max_seq_len, cache_dtype, tp), + .gated_delta_net = .init(config, batch_dim, cache_dtype, recurrent_dtype, tp), }; } diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index fa60a26d..d8e00890 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -47,8 +47,8 @@ pub const RopeParameters = struct { rope_theta: f32, }; -fn partitionProjectedKv(kv: zml.Tensor, kv_head_meshe: zml.Sharding.DimMeshe) zml.Tensor { - return switch (kv_head_meshe) { +fn partitionProjectedKv(kv: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { + return switch (kv_head_sharding) { .sharded => |heads| blk: { const sharded_kv = if (heads.factor != 1) kv.stutter1d(kv.axis(.h), heads.factor) else kv; break :blk sharded_kv.withPartitioning(.model, .{ .s = .replicated, .h = .model, .hd = .replicated }); @@ -57,9 +57,9 @@ fn partitionProjectedKv(kv: zml.Tensor, kv_head_meshe: zml.Sharding.DimMeshe) zm }; } -fn partitionCachedKv(tensor: zml.Tensor, kv_head_meshe: zml.Sharding.DimMeshe) zml.Tensor { +fn partitionCachedKv(tensor: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { var kv = tensor.rename(.{ .s = .k }); - return switch (kv_head_meshe) { + return switch (kv_head_sharding) { .sharded => |heads| blk: { if (heads.factor != 1) { kv = kv.stutter1d(kv.axis(.h), heads.factor); @@ -607,12 +607,12 @@ pub const SelfAttn = struct { const x_qkv = x.withPartitioning(tp, .{ .d = .replicated }); var q, var gate = self.projectQAndGate(x_qkv); var k, var v = self.projectKV(x_qkv); - const kv_head_meshe = tp.shardableDim(k.dim(.h), .model, q.dim(.h)); + const kv_head_sharding = tp.shardableDim(k.dim(.h), .model, q.dim(.h)); q = q.withPartitioning(tp, .{ .s = .replicated, .h = .model, .hd = .replicated }); gate = gate.withPartitioning(tp, .{ .s = .replicated, .d_out_proj = .model }); - k = partitionProjectedKv(k, kv_head_meshe); - v = partitionProjectedKv(v, kv_head_meshe); + k = partitionProjectedKv(k, kv_head_sharding); + v = partitionProjectedKv(v, kv_head_sharding); q = self.q_norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); k = self.k_norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); @@ -625,15 +625,15 @@ pub const SelfAttn = struct { q = self.rotary_embed.applyRope(q, cos, sin); k = self.rotary_embed.applyRope(k, cos, sin); q = q.withPartitioning(tp, .{ .s = .replicated, .h = .model, .hd = .replicated }); - k = partitionProjectedKv(k, kv_head_meshe); - v = partitionProjectedKv(v, kv_head_meshe); + k = partitionProjectedKv(k, kv_head_sharding); + v = partitionProjectedKv(v, kv_head_sharding); const new_kv_cache = kv_cache.update(k, v, token_index.convert(.u32)); k = new_kv_cache.keys().convert(dtype); v = new_kv_cache.values().convert(dtype); q = q.rename(.{ .s = .q }).withPartitioning(tp, .{ .q = .replicated, .h = .model, .hd = .replicated }); - k = partitionCachedKv(k, kv_head_meshe); - v = partitionCachedKv(v, kv_head_meshe); + k = partitionCachedKv(k, kv_head_sharding); + v = partitionCachedKv(v, kv_head_sharding); const attn_output = zml.attention.attention( q, @@ -1147,8 +1147,8 @@ pub const KvCache = struct { .h = config.text_config.num_key_value_heads, .hd = config.text_config.head_dim, }, dtype); - const kv_head_meshe = model_meshe.shardableDim(kv_shape.dim(.h), .model, config.text_config.num_attention_heads); - const sharded_kv_shape = switch (kv_head_meshe) { + const kv_head_sharding = model_meshe.shardableDim(kv_shape.dim(.h), .model, config.text_config.num_attention_heads); + const sharded_kv_shape = switch (kv_head_sharding) { .sharded => |heads| kv_shape.setDim(.h, heads.dim).withPartitioning(model_meshe, .{ .h = .model }), .replicated => kv_shape.withPartitioning(model_meshe, .{ .h = .replicated }), }; diff --git a/mlir/dialects/shardy.zig b/mlir/dialects/shardy.zig index 1e2307fa..b391d261 100644 --- a/mlir/dialects/shardy.zig +++ b/mlir/dialects/shardy.zig @@ -15,6 +15,16 @@ pub fn registerDialect(registry: *mlir.DialectRegistry) void { dialectHandle().insertDialect(registry); } +pub const attr = struct { + pub fn mesh(ctx: *mlir.Context, mesh_: MeshAttribute.Mesh) *const mlir.Attribute { + return MeshAttribute.init(ctx, mesh_).asAttr(); + } + + pub fn shardingsPerValue(ctx: *mlir.Context, shardings: []const *const TensorShardingAttribute) *const mlir.Attribute { + return TensorShardingPerValueAttribute.init(ctx, shardings).asAttr(); + } +}; + pub const MeshAxisAttribute = opaque { const M = mlir.Methods(MeshAxisAttribute, c.MlirAttribute); @@ -124,8 +134,8 @@ pub const AxisRefAttribute = opaque { } pub fn subAxisInfo(self: *const AxisRefAttribute) ?*const SubAxisInfoAttribute { - const attr = c.sdyAxisRefAttrGetSubAxisInfo(self.ptr()); - return if (attr.ptr == null) null else @ptrCast(attr.ptr); + const attr_ = c.sdyAxisRefAttrGetSubAxisInfo(self.ptr()); + return if (attr_.ptr == null) null else @ptrCast(attr_.ptr); } }; diff --git a/stdx/bounded_array.zig b/stdx/bounded_array.zig index 190f30fc..e541f417 100644 --- a/stdx/bounded_array.zig +++ b/stdx/bounded_array.zig @@ -47,6 +47,8 @@ pub fn BoundedArrayAligned( buffer: [buffer_capacity]T align(alignment.toByteUnits()), len: usize, + pub const max_len: usize = buffer_capacity; + pub const empty: Self = .{ .buffer = undefined, .len = 0 }; pub fn init(m: []const T) Self { @@ -55,6 +57,13 @@ pub fn BoundedArrayAligned( return buf; } + pub fn repeat(value: T, len: usize) Self { + std.debug.assert(len <= buffer_capacity); + var buf: Self = .{ .buffer = undefined, .len = len }; + @memset(buf.buffer[0..len], value); + return buf; + } + /// View the internal array as a slice whose size was previously set. pub fn slice(self: anytype) switch (@TypeOf(&self.buffer)) { *align(alignment.toByteUnits()) [buffer_capacity]T => []align(alignment.toByteUnits()) T, diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index dff0bb0f..f7c17184 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -113,6 +113,7 @@ zig_library( "moe/triton_mxfp4.zig", "nn.zig", "ops.zig", + "ops/manual_computation.zig", "pjrtx.zig", "platform.zig", "profiling/profiler.zig", diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 6c3e6421..2ac5680c 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -136,6 +136,12 @@ pub const Scope = struct { pub fn registerTensorAsBlockArgument(scope: *Scope, id: Tensor.Id, arg_id: usize) void { scope.id_to_argument.put(scope.arena.allocator(), id, scope.block.argument(arg_id)) catch @panic("OOM"); } + + pub fn value(scope: *Scope, x: Tensor) *const mlir.Value { + return scope.id_to_argument.get(x.id) orelse + x._value orelse + @panic("Something went really wrong, tensor is not an argument nor has an mlir.Value"); + } }; pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: Options) Compiler { @@ -557,12 +563,12 @@ pub const TensorInfo = struct { var attrs: AttributeList = .empty; const mlir_ctx = compiler.mlir_ctx; - const meshe_attr = try compiler.tensorShardingAttr(arena, mlir_ctx, info.shape); + const mesh_attr = try compiler.tensorShardingAttr(arena, info.shape); const name = switch (compiler.partitioner) { .gspmd => "mhlo.sharding", .shardy => "sdy.sharding", }; - attrs.appendAssumeCapacity(.named(mlir_ctx, name, meshe_attr)); + attrs.appendAssumeCapacity(.named(mlir_ctx, name, mesh_attr)); const memory = scope.id_to_memory.get(info.id) orelse .device; if (memory != .device) { @@ -814,13 +820,12 @@ fn validateMeshes(meshes: []const *const Sharding.Mesh) !void { pub fn tensorShardingAttr( compiler: *const Compiler, allocator: std.mem.Allocator, - mlir_ctx: *mlir.Context, shape: Shape, ) error{OutOfMemory}!*const mlir.Attribute { const mesh_ = shape._sharding.mesh orelse compiler.platform.replicated_mesh; return switch (compiler.partitioner) { - .shardy => (try mesh_.sdyShardingAttrForShape(allocator, mlir_ctx, shape)).asAttr(), - .gspmd => mesh_.gspmdShardingAttrForShape(allocator, mlir_ctx, shape) catch |err| switch (err) { + .shardy => (try mesh_.sdyShardingAttrForShape(allocator, compiler.mlir_ctx, shape)).asAttr(), + .gspmd => mesh_.gspmdShardingAttrForShape(allocator, compiler.mlir_ctx, shape) catch |err| switch (err) { error.WriteFailed => error.OutOfMemory, // We're writing to memory error.OutOfMemory => error.OutOfMemory, error.IncompatibleSharding => @panic("Shape dimensions must be divisible by their mesh partitions"), diff --git a/zml/Sharding.zig b/zml/Sharding.zig index 792dff69..3bba887c 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -180,7 +180,7 @@ fn shapeHasAxisPartition(shape: Shape) bool { return false; } -pub const DimMeshe = union(enum) { +pub const DimSharding = union(enum) { sharded: struct { dim: i64, factor: u32, @@ -410,33 +410,86 @@ pub const Partitioning = packed struct { /// The default value is important. /// In Tensor code unspecified axis will be inferred as `.open` while in shape code, it's `.replicated`. pub fn parse(tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec, partitioning: anytype) Partitioning { - var partition_: Partitioning = .repeat(default_value, tags.len); + const logical = if (@TypeOf(partitioning) == Logical) + partitioning + else + // capture all the comptime logic in the Logical struct. + comptime Logical.parse(partitioning); - const T = @TypeOf(partitioning); - stdx.debug.assertComptime(stdx.meta.isStruct(T), "parsePartitioning expected a struct of enum literals eg {{ .b = .data, .d = .model }}, got: {any}", .{T}); - inline for (comptime std.meta.fieldNames(T)) |field_name| { - const shape_tag = Shape.toTag(field_name); - const shape_ax = Shape.axisFromTagMaybe(tags, shape_tag) orelse { - std.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), field_name }); - }; + return logical.applyUseAll(tags, mesh, default_value); + } - const value = @field(partitioning, field_name); - const spec: PartitionSpec = if (@TypeOf(value) == PartitionSpec) value else switch (value) { - .replicated => .replicated, - .out_of_bound => std.debug.panic("value out_of_bound not allowed", .{}), - .open => .open, - else => spec: { - const mesh_tag = Shape.toTag(value); + pub const Logical = struct { + src: Shape.TagsArray, + tgt: Shape.TagsArray, + + const replicated: Shape.Tag = Shape.toTag(.replicated); + const open: Shape.Tag = Shape.toTag(.open); + + pub fn parse(v: anytype) Logical { + const V = @TypeOf(v); + + var src_: Shape.TagsArray = .empty; + var tgt_: Shape.TagsArray = .empty; + + if (comptime stdx.meta.isStruct(V)) { + const field_names = std.meta.fieldNames(V); + stdx.debug.assertComptime(field_names.len <= @TypeOf(src_).max_len, "Too many tags in struct {} ({d})", .{ V, field_names.len }); + inline for (field_names) |field_name| { + const fv = @field(v, field_name); + tgt_.appendAssumeCapacity(@tagName(fv)); + + if (!comptime stdx.meta.isTuple(V)) { + src_.appendAssumeCapacity(field_name); + } + } + return .{ .src = src_, .tgt = tgt_ }; + } + + stdx.debug.compileError("Sharding.Partitioning.parse expects a struct of enum literals, eg {{ .b = .data, .d = .model }}, got: {any}", .{V}); + } + + test "Logical.parse" { + const logical = comptime Logical.parse(.{ .a = .x, .b = .y }); + + try std.testing.expectEqualSlices(Shape.Tag, &.{ "a".ptr, "b".ptr }, logical.src.constSlice()); + try std.testing.expectEqualSlices(Shape.Tag, &.{ "x".ptr, "y".ptr }, logical.tgt.constSlice()); + } + + pub fn apply(logical: *const Logical, tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec) struct { Partitioning, ?u8 } { + var partition_: Partitioning = .repeat(default_value, tags.len); + var missing_src_axis: ?u8 = null; + + for (0.., logical.src.slice(), logical.tgt.slice()) |i, shape_tag, mesh_tag| { + const shape_ax = Shape.axisFromTagMaybe(tags, shape_tag) orelse { + missing_src_axis = @intCast(i); + continue; + }; + + const spec: PartitionSpec = if (mesh_tag == Logical.replicated) + .replicated + else if (mesh_tag == Logical.open) + .open + else spec: { const mesh_ax = mesh.resolveLogicalAxis(mesh_tag) orelse { stdx.debug.panic("{f} has no logical axis '{s}'", .{ mesh, mesh_tag }); }; break :spec .sharded(@intCast(mesh_ax)); - }, - }; - partition_ = partition_.set(shape_ax, spec); + }; + partition_ = partition_.set(shape_ax, spec); + } + return .{ partition_, missing_src_axis }; } - return partition_; - } + + pub fn applyUseAll(logical: *const Logical, tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec) Partitioning { + const p, const missing_shape_axis = logical.apply(tags, mesh, default_value); + + if (missing_shape_axis) |ax| { + std.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), logical.src.get(ax) }); + } + return p; + } + }; }; /// Device as part of a PhysicalMesh. @@ -1449,7 +1502,7 @@ pub const Mesh = struct { folds: Folds, folds_consumed: std.EnumSet(PhysicalAxisTag), - pub fn shardableDim(mesh: *const Mesh, dim: i64, logical_axis: anytype, must_divide: i64) DimMeshe { + pub fn shardableDim(mesh: *const Mesh, dim: i64, logical_axis: anytype, must_divide: i64) DimSharding { const partitions = mesh.numPartitionsForLogicalAxis(logical_axis); if (@mod(dim, partitions) == 0) { @@ -1471,8 +1524,14 @@ pub const Mesh = struct { } pub fn resolveLogicalAxis(self: *const Mesh, tag: Shape.Tag) ?u8 { - for (0.., self.logical.axes.slice()) |i, logical_axis| { - if (std.mem.eql(u8, std.mem.span(logical_axis), std.mem.span(tag))) return @truncate(i); + if (@inComptime()) { + for (0.., self.logical.axes.slice()) |i, logical_axis| { + if (std.mem.eql(u8, std.mem.span(logical_axis), std.mem.span(tag))) return @truncate(i); + } + } else { + for (0.., self.logical.axes.slice()) |i, logical_axis| { + if (logical_axis == tag) return @truncate(i); + } } return null; } diff --git a/zml/buffer.zig b/zml/buffer.zig index 03dd5b3d..b8ab1c8d 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -367,6 +367,7 @@ test "device round-trip" { const io = std.testing.io; const allocator = std.testing.allocator; const platform = zml.testing.env(); + const tp = zml.testing.tpMesh(); const x: [8][8]u32 = .{ .{ 0, 1, 2, 3, 4, 5, 6, 7 }, @@ -379,10 +380,9 @@ test "device round-trip" { .{ 56, 57, 58, 59, 60, 61, 62, 63 }, }; - const mp: *const zml.Mesh = platform.meshes.get("model").?; const x_h: zml.Slice = .init(.withPartitioning( .init(.{ .b = 8, .d = 8 }, .u32), - mp, + tp, .{ .b = .model }, ), std.mem.asBytes(&x)); // no free: x_h is stack allocated diff --git a/zml/buffer/HostAccessible.zig b/zml/buffer/HostAccessible.zig index 2d326e75..644704c2 100644 --- a/zml/buffer/HostAccessible.zig +++ b/zml/buffer/HostAccessible.zig @@ -1818,9 +1818,11 @@ test "HostAccessible.View scalar, empty ranges and rank-one padding" { test "HostAccessible devices holding a shard read one allocation" { const platform = testing.env(); + const tp = testing.tpMesh(); + const shape_ = Shape.init(.{ .b = @as(i64, @intCast(platform.devices.len)), .d = 5 }, .i32); try expectDevicesReadShards(platform, shape_); - try expectDevicesReadShards(platform, shape_.withPartitioning(platform.meshes.get("model").?, .{ .b = .model })); + try expectDevicesReadShards(platform, shape_.withPartitioning(tp, .{ .b = .model })); } test "HostAccessible devices holding a shard read one allocation in data-parallel groups" { diff --git a/zml/kernels/flash_attention/nki/nki.zig b/zml/kernels/flash_attention/nki/nki.zig index 6f1488ca..282da132 100644 --- a/zml/kernels/flash_attention/nki/nki.zig +++ b/zml/kernels/flash_attention/nki/nki.zig @@ -43,14 +43,15 @@ pub const Parameters = struct { /// Attention for Neuron. Both prefill and decode lower to NKI custom calls. pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, parameters: Parameters) zml.Tensor { - const tp_mesh = zml.Compiler.current().resolveMesh(.{.model}); - const q_sharded = q.withPartitioning(tp_mesh, .{ .q = .replicated, .h = .model, .hd = .replicated }); - const k_sharded = k.withPartitioning(tp_mesh, .{ .k = .replicated, .h = .model, .hd = .replicated }); - const v_sharded = v.withPartitioning(tp_mesh, .{ .k = .replicated, .h = .model, .hd = .replicated }); + // TODO: https://github.com/zml/zml/issues/914 + const tp = zml.Compiler.current().resolveMesh(.{.model}); + const q_sharded = q.withPartitioning(tp, .{ .h = .model }); + const k_sharded = k.withPartitioning(tp, .{ .h = .model }); + const v_sharded = v.withPartitioning(tp, .{ .h = .model }); if (q.dim(.q) == 1) { const token_index_2d = token_index.broad(zml.Shape.init(.{ .row = 1, .col = 1 }, token_index.dtype())) - .withPartitioning(tp_mesh, .{ .row = .replicated, .col = .replicated }); + .withPartitioning(tp, .{ .row = .replicated, .col = .replicated }); return zml.ops.manualComputation( (struct { diff --git a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig index fbe276d5..ef9e852c 100644 --- a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig @@ -153,8 +153,9 @@ pub const mosaic_tpu = struct { "mosaic_tpu ragged paged attention q head_dim {} exceeds kernel/cache head_dim {}", .{ q.dim(.hd), target_head_dim }, ); - return q.pad(0, .{ .hd = zml.Tensor.Pad{ .high = target_head_dim - q.dim(.hd) } }) - .withPartitioning(q.shape()._sharding.mesh.?, .{ .h = q.shape().partition(.h) }); + var q_padded = q.pad(0, .{ .hd = zml.Tensor.Pad{ .high = target_head_dim - q.dim(.hd) } }); + q_padded._shape._sharding = q._shape._sharding; + return q_padded; } inline fn prepareInputs(parameters: Parameters, q: zml.Tensor, kv_pages: zml.Tensor) PreparedInputs { diff --git a/zml/meta.zig b/zml/meta.zig index c9a43f25..1eefea11 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -915,6 +915,45 @@ pub fn collectPtrs( return context.out.toOwnedSlice(allocator); } +/// Finds all X in the given object, and write their mutable pointers into an arraylist. +pub fn collectMutPtrs( + T: type, + allocator: std.mem.Allocator, + obj: anytype, +) std.mem.Allocator.Error![]*T { + const ok = switch (@typeInfo(@TypeOf(obj))) { + .pointer => |ptr_info| !ptr_info.attrs.@"const", + else => false, + }; + stdx.debug.assertComptime(ok, "zml.meta.collectMutPtrs requires a mutable pointer input, got: {}", .{@TypeOf(obj)}); + return @ptrCast(try collectPtrs(T, allocator, obj)); +} + +/// Finds all X in the given object, and copy them by value into an arraylist. +pub fn collectValues( + T: type, + allocator: std.mem.Allocator, + obj: anytype, +) std.mem.Allocator.Error![]T { + const CollectValuesCtx = struct { + allocator: std.mem.Allocator, + out: std.ArrayList(T) = .empty, + oom: bool = false, + + fn cb(ctx: *@This(), val: *const T) void { + if (ctx.oom) return; + ctx.out.append(ctx.allocator, val.*) catch { + ctx.oom = true; + }; + } + }; + var context = CollectValuesCtx{ .allocator = allocator }; + visit(CollectValuesCtx.cb, &context, obj); + if (context.oom) return error.OutOfMemory; + + return context.out.toOwnedSlice(allocator); +} + fn _CollectCtx(func: anytype) type { const params = @typeInfo(@TypeOf(func)).@"fn".param_types; if (params.len == 1) return void; diff --git a/zml/ops.zig b/zml/ops.zig index 11a31ea0..555537f5 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -7,6 +7,13 @@ const pjrt = @import("pjrt"); const platforms = @import("platforms"); const stdx = @import("stdx"); +const manual_computation = @import("ops/manual_computation.zig"); +pub const manualComputation = manual_computation.manualComputation; + +test { + _ = manual_computation; +} + const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("mem.zig").Bufferized; const Compiler = @import("Compiler.zig"); @@ -2523,451 +2530,6 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, }; } -/// Runs a per-shard body using one mesh for all input and output partition specs. -pub fn manualComputation( - comptime body_fn: anytype, - inputs: stdx.meta.FnParam(body_fn, 0), - outputs: anytype, -) manualComputationReturnType(body_fn) { - const output_shapes: []const Shape = switch (@typeInfo(@TypeOf(outputs))) { - .void => &.{}, - .@"struct" => |struct_info| b: { - if (@TypeOf(outputs) == Shape) { - break :b &[1]Shape{outputs}; - } - if (!struct_info.is_tuple) @compileError("Expected tuple output shapes"); - var output_shapes_flat: [struct_info.field_names.len]Shape = undefined; - meta.collectBuf((struct { - pub fn func(t: Shape) Shape { - return t; - } - }).func, {}, &outputs, &output_shapes_flat); - break :b &output_shapes_flat; - }, - .array => &outputs, - .pointer => |pointer_info| b: { - if (pointer_info.size != .slice) @compileError("Expected output shape slice"); - break :b outputs; - }, - else => @compileError("Unsupported manualComputation output type: " ++ @typeName(@TypeOf(outputs))), - }; - - const sharded_outputs: []const Tensor = manualComputationInternal(inputs, output_shapes, body_fn) catch |err| switch (err) { - error.OutOfMemory => @panic("OOM"), - }; - const ReturnT = manualComputationReturnType(body_fn); - return manualComputationSliceToReturn(ReturnT, sharded_outputs); -} - -fn manualComputationLocalizeInputs(allocator: std.mem.Allocator, inputs: anytype, local_tensors: []const Tensor) !@TypeOf(inputs) { - const Context = struct { - local_tensors: []const Tensor, - input_idx: usize = 0, - - fn localize(self: *@This(), _: Tensor) Tensor { - stdx.debug.assert(self.input_idx < self.local_tensors.len, "manualComputation ran out of localized inputs", .{}); - defer self.input_idx += 1; - return self.local_tensors[self.input_idx]; - } - }; - - var context: Context = .{ .local_tensors = local_tensors }; - var local_inputs: @TypeOf(inputs) = undefined; - try meta.mapAlloc(Context.localize, allocator, &context, inputs, &local_inputs); - stdx.debug.assert(context.input_idx == local_tensors.len, "manualComputation localized {} inputs, expected {}", .{ context.input_idx, local_tensors.len }); - return local_inputs; -} - -fn manualComputationMesh(inputs: []const Shape, outputs: []const Shape, fallback: *const Sharding.Mesh) error{IncompatibleMeshes}!*const Sharding.Mesh { - var mesh: ?*const Sharding.Mesh = null; - for ([_][]const Shape{ inputs, outputs }) |shapes| { - for (shapes) |shape| { - if (shape._sharding.mesh) |shape_mesh| { - if (mesh) |m| { - if (m != shape_mesh) return error.IncompatibleMeshes; - } else mesh = shape_mesh; - } - } - } - return mesh orelse fallback; -} - -test "manualComputation infers one mesh from inputs and outputs" { - const fallback: Sharding.Mesh = .{ - .name = "fallback", - .physical = undefined, - .logical = .mesh(.{ .batch = .balanced }), - .bindings = .empty, - .folds = .empty, - .folds_consumed = .empty, - }; - var first = fallback; - first.name = "first"; - var second = fallback; - second.name = "second"; - const replicated: Shape = .init(.{8}, .f32); - var sharded = replicated; - sharded._sharding = .{ .mesh = &first, .partition = .init(&.{.mesh_axis_0}) }; - var other = sharded; - other._sharding.mesh = &second; - - try std.testing.expectEqual(&fallback, try manualComputationMesh(&.{replicated}, &.{replicated}, &fallback)); - try std.testing.expectEqual(&first, try manualComputationMesh(&.{ replicated, sharded, sharded }, &.{sharded}, &fallback)); - try std.testing.expectEqual(&first, try manualComputationMesh(&.{replicated}, &.{sharded}, &fallback)); - try std.testing.expectEqual(&first, try manualComputationMesh(&.{}, &.{sharded}, &fallback)); - try std.testing.expectError(error.IncompatibleMeshes, manualComputationMesh(&.{ sharded, other }, &.{}, &fallback)); - try std.testing.expectError(error.IncompatibleMeshes, manualComputationMesh(&.{sharded}, &.{other}, &fallback)); -} - -fn manualComputationInternal( - inputs: anytype, - outputs: []const Shape, - comptime body_fn: anytype, -) error{OutOfMemory}![]Tensor { - const BodyReturnT = manualComputationReturnType(body_fn); - const BodyOutputShapesT = stdx.meta.FnParam(body_fn, 1); - - const ctx = Compiler.current(); - const scope = ctx.currentScope(); - - var arena_state: std.heap.ArenaAllocator = .init(std.heap.page_allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - const input_shapes = try meta.collectAlloc(Tensor.shape, {}, arena, &inputs); - const input_values = try meta.collectAlloc(Tensor.value, {}, arena, &inputs); - - const local_input_shapes = try arena.alloc(Shape, input_shapes.len); - const local_output_shapes = try arena.alloc(Shape, outputs.len); - const manual_mesh = manualComputationMesh(input_shapes, outputs, ctx.platform.replicated_mesh) catch - @panic("manualComputation expects all inputs and outputs to use the same mesh"); - log.debug("manualComputation({s}) will use mesh {s}", .{ @typeName(@TypeOf(body_fn)), manual_mesh.name }); - - for (input_shapes, 0..) |shape, i| { - local_input_shapes[i] = Sharding.shardedShape(shape) catch std.debug.panic("can't shard {f} for {f}", .{ shape, manual_mesh }); - log.debug("\t- {f} -> {f}", .{ shape, local_input_shapes[i] }); - } - for (outputs, 0..) |shape, i| { - local_output_shapes[i] = Sharding.shardedShape(shape) catch std.debug.panic("can't shard {f} for {f}", .{ shape, manual_mesh }); - } - - return switch (ctx.partitioner) { - .shardy => { - const in_meshes_attr = try Sharding.sdyPerValueShardingAttr(arena, ctx.mlir_ctx, input_shapes, manual_mesh); - const out_meshes_attr = try Sharding.sdyPerValueShardingAttr(arena, ctx.mlir_ctx, outputs, manual_mesh); - const manual_axes_attr = try Sharding.sdyManualAxesAttr(arena, ctx.mlir_ctx, input_shapes, outputs, manual_mesh); - - const block_types = try arena.alloc(*const mlir.Type, input_shapes.len); - for (local_input_shapes, 0..) |input_shape, i| { - block_types[i] = mlirx.Type.rankedTensor(ctx.mlir_ctx, input_shape); - } - const block_locs = try arena.alloc(*const mlir.Location, input_shapes.len); - @memset(block_locs, ctx.unknown_location); - - const manual_block = mlir.Block.init(block_types, block_locs); - errdefer manual_block.deinit(); - - const manual_scope = ctx.pushBlock(manual_block); - defer manual_scope.pop(); - - const local_input_tensors = try arena.alloc(Tensor, input_shapes.len); - for (0..input_shapes.len) |i| { - local_input_tensors[i] = Tensor._result(local_input_shapes[i], manual_block.argument(i)); - } - - ctx.manual_computation_depth += 1; - defer ctx.manual_computation_depth -= 1; - - const local_inputs = try manualComputationLocalizeInputs(arena, inputs, local_input_tensors); - const body_output_shapes = manualComputationOutputShapesArg(BodyOutputShapesT, local_output_shapes); - const body_result = @call(.auto, body_fn, .{ local_inputs, body_output_shapes }); - const local_outputs = manualComputationBodyToSlice(BodyReturnT, arena, body_result); - stdx.debug.assert(local_outputs.len == outputs.len, "manualComputation body returned {} values, expected {}", .{ local_outputs.len, outputs.len }); - - const local_output_values = try arena.alloc(*const mlir.Value, outputs.len); - for (0..outputs.len) |i| { - stdx.debug.assert(local_outputs[i].shape().eql(local_output_shapes[i]), "manualComputation body returned shape {f}, expected {f}", .{ local_outputs[i].shape(), local_output_shapes[i] }); - local_output_values[i] = local_outputs[i].value(); - } - - _ = mlir.Operation.make(ctx.mlir_ctx, "sdy.return", .{ - .operands = .{ .flat = local_output_values }, - .verify = false, - .location = ctx.location, - }).appendTo(manual_block); - - const global_result_types = try arena.alloc(*const mlir.Type, outputs.len); - for (outputs, 0..) |output_shape, i| { - global_result_types[i] = mlirx.Type.rankedTensor(ctx.mlir_ctx, output_shape); - } - - const op = mlir.Operation.make(ctx.mlir_ctx, "sdy.manual_computation", .{ - .operands = .{ .flat = input_values }, - .results = .{ .flat = global_result_types }, - .blocks = &.{manual_block}, - .attributes = &.{ - .named(ctx.mlir_ctx, "in_shardings", in_meshes_attr), - .named(ctx.mlir_ctx, "out_shardings", out_meshes_attr), - .named(ctx.mlir_ctx, "manual_axes", manual_axes_attr), - }, - .verify = true, - .location = ctx.location, - }).appendTo(scope.block); - - // Use the compiler allocator to return memory to the parent - const sharded_outputs = ctx.alloc(Tensor, outputs.len); - for (outputs, 0..) |output, i| { - sharded_outputs[i] = Tensor._resultPropagateSharding(output, op.result(i)); - } - return sharded_outputs; - }, - .gspmd => { - const local_input_values = try arena.alloc(*const mlir.Value, input_shapes.len); - for (0..input_shapes.len) |i| { - const local_type = mlirx.Type.rankedTensor(ctx.mlir_ctx, local_input_shapes[i]); - const full_to_shard = dialects.stablehlo.custom_call( - ctx.mlir_ctx, - &.{input_values[i]}, - &.{local_type}, - .{ - .call_target_name = "SPMDFullToShardShape", - .has_side_effect = false, - .backend_config = .{ .original = "" }, - .additional_attributes = &.{ - .named(ctx.mlir_ctx, "mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")), - }, - }, - ctx.location, - ).appendTo(scope.block); - local_input_values[i] = full_to_shard.result(0); - } - - const local_input_tensors = try arena.alloc(Tensor, input_shapes.len); - for (0..input_shapes.len) |i| { - local_input_tensors[i] = Tensor._result(local_input_shapes[i], local_input_values[i]); - } - - ctx.manual_computation_depth += 1; - defer ctx.manual_computation_depth -= 1; - const local_inputs = try manualComputationLocalizeInputs(arena, inputs, local_input_tensors); - const body_output_shapes = manualComputationOutputShapesArg(BodyOutputShapesT, local_output_shapes); - const body_result = @call(.auto, body_fn, .{ local_inputs, body_output_shapes }); - const local_outputs = manualComputationBodyToSlice(BodyReturnT, arena, body_result); - stdx.debug.assert(local_outputs.len == outputs.len, "manualComputation body returned {} values, expected {}", .{ local_outputs.len, outputs.len }); - for (0..outputs.len) |i| { - stdx.debug.assert(local_outputs[i].shape().eql(local_output_shapes[i]), "manualComputation body returned shape {f}, expected {f}", .{ local_outputs[i].shape(), local_output_shapes[i] }); - } - - // Skip optimization barrier for custom call that don't return values - if (outputs.len == 0) return &.{}; - - const global_values = try arena.alloc(*const mlir.Value, outputs.len); - const global_types = try arena.alloc(*const mlir.Type, outputs.len); - for (outputs, 0..) |output_shape, i| { - const gspmd_attr = try ctx.tensorShardingAttr(arena, ctx.mlir_ctx, output_shape); - - global_types[i] = mlirx.Type.rankedTensor(ctx.mlir_ctx, output_shape); - const shard_to_full = dialects.stablehlo.custom_call( - ctx.mlir_ctx, - &.{local_outputs[i].value()}, - &.{global_types[i]}, - .{ - .call_target_name = "SPMDShardToFullShape", - .has_side_effect = false, - .backend_config = .{ .original = "" }, - .additional_attributes = &.{ - .named(ctx.mlir_ctx, "mhlo.sharding", gspmd_attr), - }, - }, - ctx.location, - ).appendTo(scope.block); - global_values[i] = shard_to_full.result(0); - } - - const barrier = dialects.stablehlo.optimizationBarrier( - ctx.mlir_ctx, - global_values, - global_types, - ctx.location, - ).appendTo(scope.block); - - const sharded_outputs = ctx.alloc(Tensor, outputs.len); - for (outputs, 0..) |output_shape, i| { - sharded_outputs[i] = Tensor._resultPropagateSharding(output_shape, barrier.result(i)); - } - return sharded_outputs; - }, - }; -} - -test "manualComputation handler API" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); - defer comp.deinit(); - comp.activate(); - defer comp.deactivate(); - - const block = mlir.Block.init(&.{}, &.{}); - const scope = comp.pushBlock(block); - defer scope.pop(); - - const shape = Shape.init(.{8}, .f32); - const lhs = Tensor.constant(DataType.f32.constant(1)).broad(shape); - const rhs = Tensor.constant(DataType.f32.constant(2)).broad(shape); - - const Configured = struct { - lhs: Tensor, - bias: ?Tensor, - missing_bias: ?Tensor, - enabled: bool, - rhs: Tensor, - - pub fn compute(self: @This(), output_shape: Shape) Tensor { - stdx.debug.assert(self.enabled, "manualComputation did not preserve handler configuration", .{}); - stdx.debug.assert(self.missing_bias == null, "manualComputation did not preserve a null optional tensor", .{}); - return self.lhs.add(self.rhs).add(self.bias.?).reshape(output_shape); - } - }; - const configured = manualComputation( - Configured.compute, - .{ - .lhs = lhs, - .bias = rhs, - .missing_bias = null, - .enabled = true, - .rhs = rhs, - }, - shape, - ); - try zml.testing.expectEqualShapes(shape, configured.shape()); - - const passthrough = manualComputation( - (struct { - input: Tensor, - - pub fn call(self: @This(), output_shape: Shape) Tensor { - return self.input.reshape(output_shape); - } - }).call, - .{ .input = configured }, - shape, - ); - try zml.testing.expectEqualShapes(shape, passthrough.shape()); - - const NestedInputs = struct { - pair: [2]Tensor, - optional: ?struct { tensor: Tensor }, - }; - const slice_inputs = [_]Tensor{ lhs, rhs }; - const original_values = [_]usize{ - @intFromPtr(lhs.value()), - @intFromPtr(rhs.value()), - @intFromPtr(lhs.value()), - @intFromPtr(lhs.value()), - @intFromPtr(rhs.value()), - }; - - const NestedStruct = struct { - nested_inputs: NestedInputs, - slice_inputs: []const Tensor, - original_values: [5]usize, - - pub fn compute(self: @This(), output_shape: Shape) Tensor { - const localized_inputs = [_]Tensor{ - self.nested_inputs.pair[0], - self.nested_inputs.pair[1], - self.nested_inputs.optional.?.tensor, - self.slice_inputs[0], - self.slice_inputs[1], - }; - for (localized_inputs, self.original_values) |input, original_value| { - stdx.debug.assert(@intFromPtr(input.value()) != original_value, "manualComputation did not localize a nested input", .{}); - } - - var result = localized_inputs[0]; - for (localized_inputs[1..]) |input| result = result.add(input); - return result.reshape(output_shape); - } - }; - const nested = manualComputation( - NestedStruct.compute, - .{ - .nested_inputs = .{ - .pair = .{ lhs, rhs }, - .optional = .{ .tensor = lhs }, - }, - .slice_inputs = &slice_inputs, - .original_values = original_values, - }, - shape, - ); - try zml.testing.expectEqualShapes(shape, nested.shape()); -} - -fn manualComputationReturnType(comptime body_fn: anytype) type { - const ReturnT = stdx.meta.FnReturn(body_fn); - if (ReturnT == void or ReturnT == Tensor) return ReturnT; - - return switch (@typeInfo(ReturnT)) { - .pointer => |pointer_info| b: { - if (pointer_info.size != .slice or pointer_info.child != Tensor) { - @compileError("manualComputation body must return void, Tensor, []Tensor, or []const Tensor; got " ++ @typeName(ReturnT)); - } - break :b ReturnT; - }, - else => @compileError("manualComputation body must return void, Tensor, []Tensor, or []const Tensor; got " ++ @typeName(ReturnT)), - }; -} - -fn manualComputationBodyToSlice(comptime ReturnT: type, allocator: std.mem.Allocator, result: ReturnT) []const Tensor { - if (ReturnT == void) return &.{}; - if (ReturnT == Tensor) { - const out = allocator.alloc(Tensor, 1) catch unreachable; - out[0] = result; - return out; - } - - return result; -} - -fn manualComputationOutputShapesArg(comptime OutputShapesT: type, local_output_shapes: []const Shape) OutputShapesT { - if (OutputShapesT == void) { - stdx.debug.assert(local_output_shapes.len == 0, "manualComputation body expects no output shapes, got {}", .{local_output_shapes.len}); - return; - } - if (OutputShapesT == Shape) { - stdx.debug.assert(local_output_shapes.len == 1, "manualComputation body expects one output shape, got {}", .{local_output_shapes.len}); - return local_output_shapes[0]; - } - - return switch (@typeInfo(OutputShapesT)) { - .pointer => |pointer_info| b: { - if (pointer_info.size != .slice or pointer_info.child != Shape) { - @compileError("manualComputation body output shapes must be void, Shape, []Shape, or []const Shape; got " ++ @typeName(OutputShapesT)); - } - break :b if (pointer_info.attrs.@"const") - local_output_shapes - else - @constCast(local_output_shapes); - }, - else => @compileError("manualComputation body output shapes must be void, Shape, []Shape, or []const Shape; got " ++ @typeName(OutputShapesT)), - }; -} - -fn manualComputationSliceToReturn(comptime ReturnT: type, outputs: []const Tensor) ReturnT { - if (ReturnT == void) { - stdx.debug.assert(outputs.len == 0, "manualComputation expected no outputs, got {}", .{outputs.len}); - return; - } - if (ReturnT == Tensor) { - stdx.debug.assert(outputs.len == 1, "manualComputation expected one output, got {}", .{outputs.len}); - return outputs[0]; - } - return outputs; -} - fn metadataIntegerFieldToMlirAttribute(mlir_ctx: *mlir.Context, int_field: std.builtin.Type.Int, value: anytype) *const mlir.Attribute { return switch (int_field.signedness) { .signed => switch (int_field.bits) { @@ -3324,6 +2886,7 @@ fn customCallArgsFromPjrtCallFrame(I: type, O: type, A: type, call_frame: *pjrt. test customCall { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const tp = zml.testing.tpMesh(); var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); @@ -3334,7 +2897,7 @@ test customCall { const scope = comp.pushBlock(block); defer scope.pop(); - const shape = zml.Shape.init(.{128}, .bf16).withPartitioning(platform.meshes.get("model").?, .{ ._0 = .model }); + const shape = zml.Shape.init(.{128}, .bf16).withPartitioning(tp, .{ ._0 = .model }); const input = Tensor.constant(zml.DataType.bf16.constant(0)).broad(shape); const output = customCall("my_custom_call", .{input}, .{zml.Shape.init(.{128}, .bf16)}, .{}, .{ .has_side_effect = false, diff --git a/zml/ops/manual_computation.zig b/zml/ops/manual_computation.zig new file mode 100644 index 00000000..982c7b0b --- /dev/null +++ b/zml/ops/manual_computation.zig @@ -0,0 +1,301 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const dialects = @import("mlir/dialects"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +const zml = struct { + // Nested to disambiguate from stdx.meta + const meta = @import("../meta.zig"); + const testing = @import("../testing.zig"); +}; + +const Compiler = @import("../Compiler.zig"); +const DataType = @import("../dtype.zig").DataType; +const mlirx = @import("../mlirx.zig"); +const Shape = @import("../shape.zig").Shape; +const Sharding = @import("../Sharding.zig"); +const Tensor = @import("../tensor.zig").Tensor; +const log = std.log.scoped(.@"zml/Compiler"); + +/// ZML operator describes computation across several devices, +/// but `manualComputation` allows to partition the devices in small group, +/// split the `Tensor` on those partitions, +/// and describes sub-computation that work inside a partition. +/// +/// See: +/// https://github.com/openxla/shardy/blob/main/docs/sdy_dialect.md#sdymanual_computation-sdymanualcomputationop +/// +/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R` where: +/// - `I` is user defined and contains inputs tensors and any metadata required by the computation +/// - `O` contains the partionned output shapes, types must be `void`, `Shape` or `[]const Shape` for 0, 1 or many outputs +/// - `R` must be `void`, `Tensor, or `[]const Tensor` for 0, 1 or many outputs +/// * `inputs` must be using the same mesh (using .withPartitioning), or be fully replicated +/// * `outputs` contains the global output shapes, +/// and `manualComputation` will pass to `body_fn` the shapes of each local output shard. +/// +/// This depart from the rest of ZML where output shapes are generally inferred, +/// but in this case since `body_fn` explains how to compute parts of the result, +/// it's not clear how to put them back together, +/// hence why the user is asked to explicitly provided the expected output shape. +pub fn manualComputation( + comptime body_fn: anytype, + inputs: stdx.meta.FnParam(body_fn, 0), + output_shapes: stdx.meta.FnParam(body_fn, 1), +) stdx.meta.FnReturn(body_fn) { + const Inputs = stdx.meta.FnParam(body_fn, 0); + const OutputShapes = stdx.meta.FnParam(body_fn, 1); + const Outputs = stdx.meta.FnReturn(body_fn); + + const compiler = Compiler.current(); + var arena_state: std.heap.ArenaAllocator = .init(compiler.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + var owned_inputs: Inputs = inputs; + const flat_input_tensors: []*Tensor = zml.meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); + defer arena.free(flat_input_tensors); + + const flat_output_shapes: []const Shape = zml.meta.collectValues(Shape, arena, &output_shapes) catch @panic("OOM"); + + const WrappedInputs = struct { + inputs: *const Inputs, + output_shapes: OutputShapes, + + fn wrap(allocator: std.mem.Allocator, opaque_wrapper: *const anyopaque, opaque_outputs: *anyopaque) error{OutOfMemory}![]*Tensor { + const wrapped: *const @This() = @ptrCast(@alignCast(opaque_wrapper)); + const outputs_ptr: *Outputs = @ptrCast(@alignCast(opaque_outputs)); + + outputs_ptr.* = body_fn(wrapped.inputs.*, wrapped.output_shapes); + + return try zml.meta.collectMutPtrs(Tensor, allocator, outputs_ptr); + } + }; + + var outputs_struct: Outputs = undefined; + var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = output_shapes }; + _ = manualComputationInternal( + compiler, + arena, + @typeName(@TypeOf(body_fn)), + &wrapped_inputs, + flat_input_tensors, + @ptrCast(&outputs_struct), + flat_output_shapes, + WrappedInputs.wrap, + ) catch |err| switch (err) { + error.OutOfMemory => @panic("OOM"), + }; + return outputs_struct; +} + +const LocalComputationFn = *const fn (std.mem.Allocator, *const anyopaque, *anyopaque) error{OutOfMemory}![]*Tensor; + +/// Reified version of manualComputation. +/// This one is compiled exactly once, user code just need to compile the `collectPtrs` +noinline fn manualComputationInternal( + compiler: *Compiler, + arena: std.mem.Allocator, + functor_name: []const u8, + input_struct: *anyopaque, + input_tensors: []*Tensor, + output_struct: *anyopaque, + output_shapes: []const Shape, + local_fn: LocalComputationFn, +) error{OutOfMemory}![]*Tensor { + const global_scope = compiler.currentScope(); + const mlir_ctx = compiler.mlir_ctx; + + const mesh = switch (chooseCommonMesh(input_tensors, compiler.platform.replicated_mesh)) { + .mesh => |m| m, + .incompatible => |incomp| std.debug.panic("manualComputation expects all inputs and outputs to use the same mesh, got input {}: {} and input {}: {}", .{ incomp[0], input_tensors[incomp[0]], incomp[1], input_tensors[incomp[1]] }), + }; + log.debug("manualComputation({s}) will use mesh {s}", .{ functor_name, mesh.name }); + + const input_types = try arena.alloc(*const mlir.Type, input_tensors.len); + for (input_types, input_tensors) |*input_ty, tensor_ptr| { + const global = tensor_ptr._shape; + const local = Sharding.shardedShape(global) catch std.debug.panic("can't shard {f} for {f}", .{ global, mesh }); + log.debug("\t- {f} -> {f}", .{ global, local }); + input_ty.* = mlirx.Type.rankedTensor(mlir_ctx, local); + } + const input_locs = try arena.alloc(*const mlir.Location, input_tensors.len); + @memset(input_locs, compiler.unknown_location); + + const input_values = try arena.alloc(*const mlir.Value, input_tensors.len); + for (input_values, input_tensors) |*input_val, tensor_ptr| { + input_val.* = global_scope.value(tensor_ptr.*); + } + + const input_shapes = try arena.alloc(Shape, input_tensors.len); + for (input_shapes, input_tensors) |*shape, tensor_ptr| { + shape.* = tensor_ptr._shape; + } + + const manual_block = mlir.Block.init(input_types, input_locs); + errdefer manual_block.deinit(); + + const local_scope = compiler.pushBlock(manual_block); + defer local_scope.pop(); + for (0.., input_tensors) |i, tensor_ptr| { + const global = tensor_ptr._shape; + const local = Sharding.shardedShape(global) catch unreachable; // we already computed this above + tensor_ptr.* = switch (compiler.partitioner) { + .shardy => Tensor._result(local, manual_block.argument(i)), + .gspmd => gspmd.globalToLocal(mlir_ctx, global_scope, tensor_ptr.*, local, compiler.location), + }; + } + + compiler.manual_computation_depth += 1; + defer compiler.manual_computation_depth -= 1; + + const local_outputs = try local_fn(arena, input_struct, output_struct); + stdx.debug.assert(local_outputs.len == output_shapes.len, "manualComputation body returned {} values, expected {}", .{ local_outputs.len, output_shapes.len }); + for (0.., local_outputs, output_shapes) |i, local_tensor, global_shape| { + const expected_local = Sharding.shardedShape(global_shape) catch std.debug.panic("can't shard output {f} for {f}", .{ global_shape, mesh }); + stdx.debug.assert(expected_local.eql(local_tensor._shape), "manualComputation body output {d} expected {f}, got {f}", .{ i, expected_local, local_tensor._shape }); + } + + const global_output_types = try arena.alloc(*const mlir.Type, output_shapes.len); + for (output_shapes, 0..) |output_shape, i| { + global_output_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output_shape); + } + + const local_output_values = try arena.alloc(*const mlir.Value, local_outputs.len); + for (local_output_values, local_outputs) |*output_val, tensor_ptr| { + output_val.* = local_scope.value(tensor_ptr.*); + } + + const op: *const mlir.Operation = switch (compiler.partitioner) { + .shardy => op: { + // Shardy creates a sdy.manual_computation block. + _ = mlir.Operation.make(mlir_ctx, "sdy.return", .{ + .operands = .{ .flat = local_output_values }, + .verify = false, + .location = compiler.location, + }).appendTo(manual_block); + + const out_mesh_attrs = try arena.alloc(*const dialects.shardy.TensorShardingAttribute, output_shapes.len); + for (out_mesh_attrs, output_shapes) |*attr, shape| { + attr.* = try mesh.sdyShardingAttrForShape(arena, mlir_ctx, shape); + } + + break :op mlir.Operation.make(mlir_ctx, "sdy.manual_computation", .{ + .operands = .{ .flat = input_values }, + .results = .{ .flat = global_output_types }, + .blocks = &.{manual_block}, + .attributes = &.{ + .named(mlir_ctx, "in_shardings", try Sharding.sdyPerValueShardingAttr(arena, mlir_ctx, input_shapes, mesh)), + .named(mlir_ctx, "out_shardings", dialects.shardy.attr.shardingsPerValue(mlir_ctx, out_mesh_attrs)), + .named(mlir_ctx, "manual_axes", try Sharding.sdyManualAxesAttr(arena, mlir_ctx, input_shapes, output_shapes, mesh)), + }, + .verify = true, + .location = compiler.location, + }).appendTo(global_scope.block); + }, + .gspmd => op: { + // Gspmd calls SPMDShardToFullShape and wrap into an optimizationBarrier + // Shortcut to avoid empty optimization barrier + if (output_shapes.len == 0) return &.{}; + + const global_values = try arena.alloc(*const mlir.Value, output_shapes.len); + const global_types = try arena.alloc(*const mlir.Type, output_shapes.len); + for (output_shapes, 0..) |global_shape, i| { + global_values[i], global_types[i] = try gspmd.localToGlobal(compiler, arena, global_scope, local_output_values[i], global_shape); + } + + break :op dialects.stablehlo.optimizationBarrier( + mlir_ctx, + global_values, + global_types, + compiler.location, + ).appendTo(global_scope.block); + }, + }; + + // reuse local_outputs buffer to store the final result. + for (0.., local_outputs, output_shapes) |o, output_tensor_ptr, output_shape| { + output_tensor_ptr.* = Tensor._resultPropagateSharding(output_shape, op.result(o)); + } + return local_outputs; +} + +const ResolvedMesh = union(enum) { + mesh: *const Sharding.Mesh, + incompatible: [2]u32, +}; + +fn chooseCommonMesh(inputs: []const *const Tensor, fallback: *const Sharding.Mesh) ResolvedMesh { + var mesh: ?*const Sharding.Mesh = null; + var mesh_source_id: u32 = @intCast(inputs.len); + for (0.., inputs) |i, input| { + if (input.shape()._sharding.mesh) |shape_mesh| { + if (mesh) |m| { + if (m != shape_mesh) + return .{ .incompatible = .{ mesh_source_id, @intCast(i) } }; + } else { + mesh = shape_mesh; + mesh_source_id = @intCast(i); + } + } + } + return .{ .mesh = mesh orelse fallback }; +} + +test chooseCommonMesh { + const fallback: Sharding.Mesh = .initTest(.fallback, .{.all}); + const first: Sharding.Mesh = .initTest(.first, .{ .data, .model }); + const second: Sharding.Mesh = .initTest(.second, .{ .experts, .model }); + const replicated = Tensor.init(.{ .a = 8 }, .f32); + const sharded = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&first, .{ .a = .data }); + const other = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&second, .{ .a = .experts }); + + try std.testing.expectEqual(ResolvedMesh{ .mesh = &fallback }, chooseCommonMesh(&.{}, &fallback)); + try std.testing.expectEqual(ResolvedMesh{ .mesh = &fallback }, chooseCommonMesh(&.{&replicated}, &fallback)); + try std.testing.expectEqual(ResolvedMesh{ .mesh = &first }, chooseCommonMesh(&.{&sharded}, &fallback)); + try std.testing.expectEqual(ResolvedMesh{ .mesh = &first }, chooseCommonMesh(&.{ &replicated, &sharded, &sharded }, &fallback)); + try std.testing.expectEqual(ResolvedMesh{ .incompatible = .{ 0, 1 } }, chooseCommonMesh(&.{ &sharded, &other }, &fallback)); +} + +const gspmd = struct { + fn globalToLocal(mlir_ctx: *mlir.Context, scope: *Compiler.Scope, local_tensor: Tensor, local_shape: Shape, location: *const mlir.Location) Tensor { + const local_type = mlirx.Type.rankedTensor(mlir_ctx, local_shape); + const full_to_shard = dialects.stablehlo.custom_call( + mlir_ctx, + &.{scope.value(local_tensor)}, + &.{local_type}, + .{ + .call_target_name = "SPMDFullToShardShape", + .has_side_effect = false, + .backend_config = .{ .original = "" }, + .additional_attributes = &.{ + .named(mlir_ctx, "mhlo.sharding", .string(mlir_ctx, "{manual}")), + }, + }, + location, + ).appendTo(scope.block); + return Tensor._result(local_shape, full_to_shard.result(0)); + } + + fn localToGlobal(compiler: *Compiler, arena: std.mem.Allocator, scope: *Compiler.Scope, local_value: *const mlir.Value, global_shape: Shape) !struct { *const mlir.Value, *const mlir.Type } { + const gspmd_attr = try compiler.tensorShardingAttr(arena, global_shape); + const global_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, global_shape); + + const shard_to_full = dialects.stablehlo.custom_call( + compiler.mlir_ctx, + &.{local_value}, + &.{global_type}, + .{ + .call_target_name = "SPMDShardToFullShape", + .has_side_effect = false, + .backend_config = .{ .original = "" }, + .additional_attributes = &.{ + .named(compiler.mlir_ctx, "mhlo.sharding", gspmd_attr), + }, + }, + compiler.location, + ).appendTo(scope.block); + return .{ shard_to_full.result(0), global_type }; + } +}; diff --git a/zml/tensor.zig b/zml/tensor.zig index 275f42d3..796709ad 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -190,7 +190,7 @@ pub const Tensor = struct { return res; }; - const attr = ctx.tensorShardingAttr(ctx.allocator, ctx.mlir_ctx, partitioned_shape) catch @panic("OOM"); + const attr = ctx.tensorShardingAttr(ctx.allocator, partitioned_shape) catch @panic("OOM"); const op_result = switch (ctx.partitioner) { .shardy => blk: { diff --git a/zml/testing.zig b/zml/testing.zig index 366351d7..5e3d6333 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -4,6 +4,7 @@ const builtin = @import("builtin"); const stdx = @import("stdx"); const Platform = @import("platform.zig").Platform; +const Sharding = @import("Sharding.zig"); const Slice = @import("slice.zig").Slice; const zml = @import("zml.zig"); @@ -34,6 +35,12 @@ pub fn env() *const Platform { return _platform.?; } +pub fn tpMesh() *const Sharding.Mesh { + if (!builtin.is_test) @compileError("Cannot use zml.testing.tpMesh() outside of a test block"); + + return _platform.?.meshes.get("model").?; +} + /// In neural network we generally care about the relative precision, /// but on a given dimension, if the output is close to 0, then the precision /// don't matter as much. From c36b3f8f1e8256965b7c7a5fa2ef7d85fabf8ecf Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Fri, 16 Oct 2026 13:43:03 +0200 Subject: [PATCH 790/881] platforms: Bump plugins for reduced Thunk Prepare/Initialize overhead (#946) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For DeepSeek 4.1-Flash on 4 GB300 ``` | | current | new | |------------------------------------|--------------:|--------:| | Start of XlaModule → cuGraphLaunch | 1,008 µs | 132 µs | | ...of which on CPU | 896 µs | 70 µs | | Prepare + Initialize | ~790 µs | ~2 µs | ``` --- platforms/cpu/cpu.bzl | 6 +++--- platforms/cuda/cuda.bzl | 8 ++++---- platforms/furiosa/furiosa.bzl | 2 +- platforms/metal/metal.bzl | 4 ++-- platforms/rocm/rocm.bzl | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index c30f125b..1e20d1fa 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -43,21 +43,21 @@ def _cpu_plugin_impl(mctx): name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, sha256 = "9d82f2612decf7066e1c66e3da8fa8883f5f3714fe3aaa6b1ee6041b435552e6", - url = "https://mirror.zml.ai/plugins/202610151602.227.1.e215eb02db73/zml-cpu-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610160837.234.1.e1adc5dca29d/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "f365f0092a6b493888d3ce87d3bc859c02116df455037f609414e7bfeac0873e", - url = "https://mirror.zml.ai/plugins/202610151602.227.1.e215eb02db73/zml-cpu-darwin-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610160837.234.1.e1adc5dca29d/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "3ebd6bec2c05599f250415b309ffcd5f34e5c1c37b7b806eeb35c88254e8eb16", - url = "https://mirror.zml.ai/plugins/202610151602.227.1.e215eb02db73/zml-cpu-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610160837.234.1.e1adc5dca29d/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 3b18d71c..6ab141ac 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -236,12 +236,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "85101fe4efaa4419afaa3c9b9a9154ada87a1c16665bff53215de145b39751fc", - "url": "https://mirror.zml.ai/plugins/202610151602.227.1.e215eb02db73/zml-cuda-linux-amd64.tar.zst", + "sha256": "b1b33616999bfe479dfff050e6934f7b2d935c08621a269014aff585925614c2", + "url": "https://mirror.zml.ai/plugins/202610160837.234.1.e1adc5dca29d/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "1c985db48fd8f8a3691092bba2c0c7585377210db7e1b0c06a9b04047158d590", - "url": "https://mirror.zml.ai/plugins/202610151602.227.1.e215eb02db73/zml-cuda-linux-arm64.tar.zst", + "sha256": "f6819007f6d1ab5f3a1c7b79d7e2ddc9b33eba8f61d45589c322931d6379e672", + "url": "https://mirror.zml.ai/plugins/202610160837.234.1.e1adc5dca29d/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index 60e63905..3dae64a3 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,7 +6,7 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202610151602.227.1.e215eb02db73/zml-furiosa-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610160837.234.1.e1adc5dca29d/zml-furiosa-linux-amd64.tar.zst", sha256 = "e474a6ba2f3b5e810dba24dc7339bf45d5e7ea05025c924d7952d968ef6f1335", ) diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index d6995cbf..ecfb19e2 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "5ff70796f49a67880fe15cb639ab7b358547719a9afd69d184e6cb3cc7119250", - url = "https://mirror.zml.ai/plugins/202610151602.227.1.e215eb02db73/zml-metal-darwin-arm64.tar.zst", + sha256 = "70fbd60acacf40268d9b05d692f75e6e5360169c083819d27dd95ddd2214a0be", + url = "https://mirror.zml.ai/plugins/202610160837.234.1.e1adc5dca29d/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 819a3297..4ebd9a1c 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610151602.227.1.e215eb02db73/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "7d418c7404c99e078de6fcba2e4fccf324f25f1219cd0176caac9e8d14ab222c" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610160837.234.1.e1adc5dca29d/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "22a1cf08945e194164af39175c58a119c57536f5ef9f15c72804808c78f1157a" def _rocm_package_name(name): return name + _ROCM_VERSION From 8329631da1de33e38bba5d29fcae736276372be4 Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Fri, 16 Oct 2026 15:22:13 +0200 Subject: [PATCH 791/881] platforms/cuda: fix redist json checksum (#949) --- platforms/cuda/cuda.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 6ab141ac..7a32188c 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -9,7 +9,7 @@ CUDA_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cuda/redist/ CUDA_VERSION = "13.4.3" CUDA_MAJOR_MINOR = ".".join(CUDA_VERSION.split(".")[:2]) CUDA_VARIANT = "cuda{}".format(CUDA_MAJOR_MINOR) -CUDA_REDIST_JSON_SHA256 = "8e99511eee1bd78664bdb6cf43dc17a7ba6afeab78c8162bd5f23571ed6423c6" +CUDA_REDIST_JSON_SHA256 = "1420d5c676cad98a5ff423d529f116fee92f6b99559a3afdad41e782d6a11dba" CUDNN_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cudnn/redist/" CUDNN_VERSION = "9.27.0" From a7a4fcd3f8199bfd03d4aec859f598769027e80c Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 16 Oct 2026 15:27:30 +0200 Subject: [PATCH 792/881] zml/sharding: use logical meshes (#943) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation This is the big sharding rewrite we need to achieve complex sharding. Currently we always resolve a mesh logical axis to a mesh physical axis (or several of them). But on GPU we always have 1D topology, so we can't have 2D logical meshes. So this PR drops the logical -> physical axis bindings, and instead store a logical -> devices binding. This is more flexible, and has the nice side effect of simplifying `Buffer.from` On stablehlo side this changes how we generate the MLIR because we now use logical meshes, and `manualComputation` uses logical axes. (In shardy, GSPM is not affected AFAICT because it uses device ID directly) ## Buffer.from This also accelerates `Buffer.from` that I rewrote to be FAST. Version | Fixed cost | Per device -- | -- | -- origin/master | 109.6 ns | 35.39 ns Rewrite | 35.8 ns | 4.38 ns Improvement | 3.1× | 8.1× ## Creating mesh I'm adding a new syntax to create LogicalMesh, to allow the more complex meshes we want to do, and in particular, I now allow to force a specific number of dimensions. You can see it in action in `mnist.zig`. ``` const dp_mp = try platform.registerMesh("DP-MP", try .init( .{ .data, .model }, &.{ .low_bandwidth, .high_bandwidth }, &.{ 2, -1 }, )); info: topology used: Mesh(name=DP-MP, partitions=4, replicas=1) ┌ Mappings (logical coordinates from physical coordinates): ├── data (low_bandwidth) -> (bus / 2) └── model (high_bandwidth) -> (bus % 2) ┌ PhysicalMesh(platform=cpu shardable_axes={ .bus } num_devices=4) └── [bus] 0..4 (tree) ├── Device(id=0, coords={ 0 }) ├── Device(id=1, coords={ 1 }) ├── Device(id=2, coords={ 2 }) └── Device(id=3, coords={ 3 }) ┌ Logical coordinates: └── data=0 └── model=0,1 -> devices=0,1 └── data=1 └── model=0,1 -> devices=2,3 ``` The previous syntax is still available, and ZML will align logical axes to physical axes like we do before this PR. This part as unit tests, and I checked it work as I expect on CUDA but I would need to double check the logics still works on TPU/Neuron, because there is a few platform specific branches in that code. ## Test plan As part of the benchmark Codex generated a test suite with hard-coded offsets, allowing to check that Buffer.from didn't change behavior through the commits, checking we correctly compute each individual shard. TODO check that LLMD perf isn't degraded. --- examples/mnist/BUILD.bazel | 19 + examples/mnist/mnist.zig | 17 +- examples/mnist/mnist_data_parallel.zig | 196 +++ pjrt/pjrt.zig | 33 +- stdx/bounded_array.zig | 17 +- zml/Compiler.zig | 40 +- zml/Sharding.zig | 1606 ++++++++++++++++-------- zml/buffer.zig | 75 +- zml/buffer/HostAccessible.zig | 51 +- zml/exe.zig | 18 +- zml/io.zig | 97 +- zml/ops.zig | 2 +- zml/ops/manual_computation.zig | 80 +- zml/platform.zig | 5 +- zml/shape.zig | 34 +- zml/slice.zig | 5 +- zml/tensor.zig | 6 +- 17 files changed, 1517 insertions(+), 784 deletions(-) create mode 100644 examples/mnist/mnist_data_parallel.zig diff --git a/examples/mnist/BUILD.bazel b/examples/mnist/BUILD.bazel index 0f9664c1..f0ddf811 100644 --- a/examples/mnist/BUILD.bazel +++ b/examples/mnist/BUILD.bazel @@ -23,6 +23,25 @@ zig_binary( ], ) +zig_binary( + name = "mnist_dp", + args = [ + "$(location @mnist//:mnist.safetensors)", + "$(location @mnist//:t10k-images.idx3-ubyte)", + ], + data = [ + "@mnist//:mnist.safetensors", + "@mnist//:t10k-images.idx3-ubyte", + ], + main = "mnist_data_parallel.zig", + srcs = ["mnist.zig"], + visibility = ["//visibility:public"], + deps = [ + "//bazel", + "//zml", + ], +) + mtree_spec( name = "mtree", srcs = [":mnist"], diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index d1291c1b..f5842af8 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -2,12 +2,13 @@ const std = @import("std"); const log = std.log; const zml = @import("zml"); -const stdx = zml.stdx; pub const std_options: std.Options = .{ .log_level = .info, }; +pub const Img = [28 * 28]u8; + /// Model definition const Mnist = struct { fc1: Layer, @@ -137,13 +138,13 @@ pub fn main(init: std.process.Init) !void { // inference - can be looped const idx = rng.random().uintLessThan(u64, 10000); - var sample: [28 * 28]u8 align(16) = undefined; - _ = try dataset.readPositionalAll(io, &sample, 16 + (idx * 28 * 28)); + var sample: Img align(16) = undefined; + _ = try dataset.readPositionalAll(io, &sample, 16 + (idx * @sizeOf(Img))); var input_buffer: zml.Buffer = try .fromSlice(io, platform, zml.Slice.init(input.shape(), &sample)); defer input_buffer.deinit(); - printDigit(sample); + printDigit(&sample); args.set(.{ mnist_buffers, input_buffer }); exe.call(io, args, &results, .{}); @@ -159,14 +160,14 @@ pub fn main(init: std.process.Init) !void { , .{digits[try result.getValue(u8, io)]}); } -fn printDigit(digit: [28 * 28]u8) void { +pub fn printDigit(digit_flat: *const Img) void { + const digit: *const [28][28]u8 = @ptrCast(digit_flat); var buffer: [28][30][2]u8 = undefined; for (0..28) |y| { buffer[y][0] = .{ '|', ' ' }; buffer[y][29] = .{ '|', '\n' }; for (1..29) |x| { - const idx = (y * 28) + (x - 1); - const val = digit[idx]; + const val = digit[y][x - 1]; buffer[y][x] = blk: { if (val > 240) break :blk .{ '*', '*' }; if (val > 225) break :blk .{ 'o', 'o' }; @@ -185,7 +186,7 @@ fn printDigit(digit: [28 * 28]u8) void { , .{std.mem.asBytes(&buffer)}); } -const digits = [_][]const u8{ +pub const digits = [_][]const u8{ \\ | ### | \\ | # # | \\ | # # | diff --git a/examples/mnist/mnist_data_parallel.zig b/examples/mnist/mnist_data_parallel.zig new file mode 100644 index 00000000..a539ae5a --- /dev/null +++ b/examples/mnist/mnist_data_parallel.zig @@ -0,0 +1,196 @@ +//! Data-parallel version of mnist.zig, +//! Separated to kept `mnist.zig` minimal + +const std = @import("std"); +const zml = @import("zml"); + +const base = @import("mnist.zig"); +const log = std.log.scoped(.mnist); + +pub const std_options: std.Options = .{ + .log_level = .info, +}; + +/// Model definition +const Mnist = struct { + fc1: Layer, + fc2: Layer, + + pub fn init(store: zml.io.TensorStore.View) Mnist { + return .{ + // Layer 1 is sharded following it's output axis + .fc1 = .{ + .weight = store.createTensor("fc1.weight", .{ .d_out, .d }, .dp_mp, .{ .d_out = .model }), + .bias = store.createTensor("fc1.bias", .{.d_out}, .dp_mp, .{ .d_out = .model }), + }, + // Layer 2 is sharded following it's input axis (and bias is fully replicated) + .fc2 = .{ + .weight = store.createTensor("fc2.weight", .{ .d_out, .d }, .dp_mp, .{ .d = .model }), + .bias = store.createReplicatedTensor("fc2.bias", .{.d_out}), + }, + }; + } + + pub fn load( + self: *const Mnist, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *const zml.io.TensorStore, + ) !zml.Bufferized(Mnist) { + var buffers = try zml.mem.bufferize(allocator, Mnist, self); + errdefer zml.Buffer.deinitAll(Mnist, &buffers); + + var loader: zml.io.Loader = try .init(allocator, platform, .default); + errdefer loader.deinit(); + + try loader.load(io, Mnist, self, &buffers, store, .{}); + try loader.await(io); + + return buffers; + } + + /// just two linear layers + relu activation + pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { + var x = input.onMemory(.host_pinned).toMemory(.device).merge(.{ .d = .{ .x, .y } }).convert(.f32); + const layers: []const Layer = &.{ self.fc1, self.fc2 }; + const compiler = zml.Compiler.current(); + for (0.., layers) |i, layer| { + compiler.pushLocationFmt(@src(), "layer[{d}]", .{i}); + defer compiler.popLocation(); + x = layer.forward(x); + } + return x.argMax(.d).indices.convert(.u8).withPartitioning(input.shape()._sharding.mesh.?, .{ .b = .data }).toMemory(.host_pinned); + } + + const Layer = struct { + weight: zml.Tensor, + bias: zml.Tensor, + + pub fn forward(self: Layer, input: zml.Tensor) zml.Tensor { + log.warn("layer(input={f}, w={f}, b={f}) -> {f}", .{ input, self.weight, self.bias, input.dot(self.weight, .d) }); + const x = input.dot(self.weight, .d); + return x.add(self.bias.broad(x.shape())).relu().rename(.{ .d_out = .d }); + } + }; +}; + +pub fn main(init: std.process.Init) !void { + const arena = init.arena; + const allocator = init.gpa; + const io = init.io; + + // Parse program args + const process_args = try init.minimal.args.toSlice(arena.allocator()); + const model_path = process_args[1]; + const t10kfilename = process_args[2]; + + // Read model shapes. + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); + defer registry.deinit(); + + // Auto-select platform + const platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator, io); + + // Decide how to partition the devices of our hardware. + // Here we use a mixed Data parallel / Model parallel + const dp_mp = try platform.registerMesh("dp_mp", try .init( + .{ .data, .model }, + &.{ .low_bandwidth, .high_bandwidth }, + &.{ 2, -1 }, + )); + log.info("topology used: {f}", .{dp_mp}); + + var profiler = try platform.profiler(allocator, io, .{ .session_id = "mnist" }); + defer profiler.deinit(); + + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{dp_mp}); + defer store.deinit(); + const mnist_model: Mnist = .init(store.view()); + + // // Compile model + const bs: u32 = 4; + const input: zml.Tensor = .withPartitioning(.init(.{ .b = bs, .x = 28, .y = 28 }, .u8), dp_mp, .{ .b = .data }); + var exe = exe: { + log.info("Compiling model....", .{}); + const start: std.Io.Timestamp = .now(io, .awake); + defer log.info("✅ Compiled model [{f}]", .{start.untilNow(io, .awake)}); + break :exe try platform.compile( + allocator, + io, + mnist_model, + .forward, + .{input}, + .{ + .program_name = "mnist", + .meshes = &.{dp_mp}, + .xla_dump_to = "/tmp/zml/mnist", + }, + ); + }; + defer exe.deinit(); + + // Load buffers + var mnist_buffers = blk: { + log.info("Transfering weights....", .{}); + const start: std.Io.Timestamp = .now(io, .awake); + defer log.info("✅ Transferred weights [{f}]", .{ + start.untilNow(io, .awake), + }); + break :blk try mnist_model.load(init.arena.allocator(), io, platform, &store); + }; + defer zml.Buffer.deinitAll(Mnist, &mnist_buffers); + + var args = try exe.args(allocator); + defer args.deinit(allocator); + + var results = try exe.results(allocator); + defer results.deinit(allocator); + + // Load a random digit image from the dataset. + const dataset = try std.Io.Dir.openFile(.cwd(), io, t10kfilename, .{ .mode = .read_only }); + defer dataset.close(io); + + var rng: std.Random.DefaultPrng = blk: { + const now: std.Io.Timestamp = .now(io, .awake); + break :blk .init(@intCast(now.toMilliseconds())); + }; + + // inference - can be looped + var batch: []align(16) base.Img = try allocator.alignedAlloc(base.Img, .@"16", bs); + defer allocator.free(batch); + + for (0..bs) |i| { + const rand_id = rng.random().uintLessThan(u64, 10000); + _ = try dataset.readPositionalAll(io, @ptrCast(&batch[i]), 16 + (rand_id * @sizeOf(base.Img))); + } + + // This performs a host to host pinned memcpy. + // TODO: show how to allocate a Host pinned buffer and write into it. + var input_buffer: zml.Buffer = try .fromBytesOpts(io, platform, input.shape(), std.mem.sliceAsBytes(batch), .{ .memory = .host_pinned }); + defer input_buffer.deinit(); + + for (batch[0..]) |*sample| base.printDigit(sample); + + args.set(.{ mnist_buffers, input_buffer }); + + try profiler.start(); + exe.call(io, args, &results, .{}); + if (try profiler.stop()) |report| { + log.info("Wrote profiler files to {s} and {s}", .{ report.protobuf_path, report.perfetto_path }); + } + var recognized_digits_d: zml.Buffer = results.get(zml.Buffer); + defer recognized_digits_d.deinit(); + + const recognized_digits = try recognized_digits_d.getValue([bs]u8, io); + for (0..bs) |i| { + log.info( + \\✅ Shard {d} RECOGNIZED DIGIT: + \\ +-------------+ + \\{s} + \\ +-------------+ + \\ + , .{ i, base.digits[recognized_digits[i]] }); + } +} diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 73cadf44..3715361f 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -5,7 +5,8 @@ const c = @import("c"); pub const ffi = @import("ffi.zig"); -const log = std.log.scoped(.pjrt); +const log_scope = .pjrt; +const log = std.log.scoped(log_scope); test { std.testing.refAllDecls(@This()); @@ -74,10 +75,12 @@ pub const ApiError = error{ Unauthenticated, }; -inline fn interpretPjrtError(api: *const Api, pjrt_error: *Error, context: []const u8) ApiError { +inline fn interpretPjrtError(api: *const Api, pjrt_error: *Error, context: []const u8, comptime log_level: std.log.Level) ApiError { defer pjrt_error.deinit(api); const err_code = pjrt_error.getCode(api).toApiError(); - log.warn("[{s}] {t}: {s}", .{ context, err_code, pjrt_error.getMessage(api) }); + if (comptime std.log.logEnabled(log_level, log_scope)) { + std.options.logFn(log_level, log_scope, "[{s}] {t}: {s}", .{ context, err_code, pjrt_error.getMessage(api) }); + } return err_code; } @@ -165,24 +168,33 @@ pub const Api = struct { }; } - inline fn innerCall(self: *const Api, comptime method: Funcs, arg: *PJRTFnArg(method)) ApiError!void { + inline fn innerCall(self: *const Api, comptime method: Funcs, arg: *PJRTFnArg(method)) ?*Error { if (@offsetOf(c.PJRT_Api, @tagName(method)) > self.inner.struct_size) { std.debug.panic("PJRT Api method {s} not available in this plugin", .{@tagName(method)}); } const fn_ptr = @field(&self.inner, @tagName(method)).?; const result = fn_ptr(arg); if (@TypeOf(result) == void) { - return; + return null; } if (result) |pjrt_c_error| { const pjrt_error: *Error = @ptrCast(pjrt_c_error); - return interpretPjrtError(self, pjrt_error, @tagName(method)); + return pjrt_error; } + return null; } inline fn call(self: *const Api, comptime method: Funcs, arg: PJRTFnArgWithDefault(method)) ApiError!PJRTFnArgWithDefault(method) { var ret = arg; - try innerCall(self, method, @ptrCast(&ret)); + const pjrt_error = innerCall(self, method, @ptrCast(&ret)); + if (pjrt_error) |err| return interpretPjrtError(self, err, @tagName(method), .warn); + return ret; + } + + inline fn callSilent(self: *const Api, comptime method: Funcs, arg: PJRTFnArgWithDefault(method)) ApiError!PJRTFnArgWithDefault(method) { + var ret = arg; + const pjrt_error = innerCall(self, method, @ptrCast(&ret)); + if (pjrt_error) |err| return interpretPjrtError(self, err, @tagName(method), .debug); return ret; } @@ -805,7 +817,8 @@ pub const Device = opaque { }; pub fn memoryStats(self: *const Device, api: *const Api) ApiError!MemoryStats { - const ret = try api.call(.PJRT_Device_MemoryStats, .{ + // callSilent: If this API is not available no point in printing it at every call + const ret = try api.callSilent(.PJRT_Device_MemoryStats, .{ .device = self.inner(), }); return .fromCStruct(ret); @@ -1409,7 +1422,7 @@ pub const Event = opaque { pub fn await(self: *Event, api: *const Api, io: std.Io) ApiError!void { if (self.isReady(api)) { const err = self.getEventError(api) orelse return; - return interpretPjrtError(api, err, "PJRT_Event_Await"); + return interpretPjrtError(api, err, "PJRT_Event_Await", .warn); } const Ctx = struct { @@ -1426,7 +1439,7 @@ pub const Event = opaque { }.call, &ctx); ctx.event.waitUncancelable(io); - if (ctx.err) |err| return interpretPjrtError(api, err, "PJRT_Event_OnReady"); + if (ctx.err) |err| return interpretPjrtError(api, err, "PJRT_Event_OnReady", .warn); } pub fn awaitRaw(self: *const Event, api: *const Api) ApiError!void { diff --git a/stdx/bounded_array.zig b/stdx/bounded_array.zig index e541f417..1f77a14d 100644 --- a/stdx/bounded_array.zig +++ b/stdx/bounded_array.zig @@ -98,12 +98,14 @@ pub fn BoundedArrayAligned( /// Return the element at index `i` of the slice. pub fn get(self: Self, i: usize) T { - return self.constSlice()[i]; + std.debug.assert(i < self.len); + return self.buffer[i]; } /// Set the value of the element at index `i` of the slice. pub fn set(self: *Self, i: usize, item: T) void { - self.slice()[i] = item; + std.debug.assert(i < self.len); + self.buffer[i] = item; } /// Return the maximum length of a slice. @@ -286,17 +288,6 @@ pub fn BoundedArrayAligned( @memset(self.slice()[old_len..self.len], value); } - //pub const Writer = if (T != u8) - // @compileError("The Writer interface is only defined for BoundedArray(u8, ...) " ++ - // "but the given type is BoundedArray(" ++ @typeName(T) ++ ", ...)") - //else - // std.io.GenericWriter(*Self, error{Overflow}, appendWrite); - - ///// Initializes a writer which will write into the array. - //pub fn writer(self: *Self) Writer { - // return .{ .context = self }; - //} - /// Same as `appendSlice` except it returns the number of bytes written, which is always the same /// as `m.len`. The purpose of this function existing is to match `std.io.GenericWriter` API. fn appendWrite(self: *Self, m: []const u8) error{Overflow}!usize { diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 2ac5680c..c6c30394 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -471,8 +471,9 @@ pub fn compileInternal( _ = result.func.appendTo(compiler.module.body()); - const num_partitions = compiler.meshes[0].numPartitions(); - const num_replicas = compiler.meshes[0].numReplicas(); + // Mesh replication is encoded in shardings within one SPMD executable. + const num_partitions = compiler.meshes[0].numDevices(); + const num_replicas = 1; const num_devices = num_partitions * num_replicas; compiler.module.operation().setAttributeByName( @@ -502,7 +503,7 @@ pub fn compileInternal( allocator, platform, loaded_executable, - @intCast(num_devices), + num_devices, num_partitions, // This will get copied into exe result.input_info.items(.shape), @@ -515,13 +516,16 @@ pub fn compileInternal( return exe; } +/// Declare all logical meshes used by this program fn addPartitionerOperations(ctx: *Compiler) !void { const allocator = ctx.arena.allocator(); const mlir_ctx = ctx.mlir_ctx; const module = ctx.module; switch (ctx.partitioner) { - .gspmd => {}, + .gspmd => { + // GSPMD doesn't use meshes, but explicit partitioning operations. + }, .shardy => { for (ctx.meshes) |shd| { const attr_str = try shd.sdyMeshAttr(allocator); @@ -805,12 +809,25 @@ fn repack(compiler: *Compiler, scope: *Scope, og_shape: Shape, value: *const mli fn validateMeshes(meshes: []const *const Sharding.Mesh) !void { stdx.debug.assert(meshes.len >= 1, "Waiting at leat 1 mesh strategy to be implemented", .{}); - const first = meshes[0]; - const partitions = first.numPartitions(); - const replicas = first.numReplicas(); + for (meshes) |m| { + const num_replicas = m.numReplicas(); + if (num_replicas > 1 and !std.mem.eql(u8, "replicated", m.name)) { + // This seems like a performance footgun, rather that something anyone would want to use. + // It's generally better to have meshes that cover all devices, + // even if later it means that some mesh axis is never used. + // Otherwise a "sharded" tensor can in fact be replicated because the underlying mesh has replicas + // TODO(#914): decide if we should forbid this altogether + // currently DirectMemoryWriter tests have replicated meshes depending on the platform they target + log.warn("Mesh({s}) has numReplicas() == {d}. This imply that all tensor put on this mesh will be implicitly replicated {d} times.\n{f}", .{ m.name, num_replicas, num_replicas, m }); + } + log.debug("Using {f}", .{m}); + } - for (meshes[1..]) |s| { - if (s.numPartitions() != partitions or s.numReplicas() != replicas) { + const first = meshes[0]; + const devices = first.numDevices(); + + for (meshes[1..]) |m| { + if (m.numDevices() != devices) { // todo: deviceAssignments should also be checked for consistency here, but for simplicity we just check the cardinality numbers return error.InconsistentMesheCardinality; } @@ -863,8 +880,9 @@ fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.Load const platform = compiler.platform; const main_mesh = compiler.meshes[0]; - const num_partitions = main_mesh.numPartitions(); - const num_replicas = main_mesh.numReplicas(); + // Mesh replicas are part of the SPMD device set, not XLA program replicas. + const num_partitions: i32 = @intCast(main_mesh.numDevices()); + const num_replicas = 1; const device_assignment = try main_mesh.deviceAssignment(compiler.allocator); defer compiler.allocator.free(device_assignment); diff --git a/zml/Sharding.zig b/zml/Sharding.zig index 3bba887c..bfb70240 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -509,13 +509,6 @@ pub const Device = struct { fn hasValidCoords(device: Device) bool { return @as(u32, @bitCast(device.coords)) != @as(u32, @bitCast(undefined_coords)); } - - pub fn format(device: Device, writer: *std.Io.Writer) !void { - try writer.print( - "Device(id={d} coords={any})", - .{ device.id, device.coords }, - ); - } }; /// Physical axis tags represent hardware interconnect tiers. @@ -646,32 +639,6 @@ pub const PhysicalMesh = struct { return self.depth_by_tag.get(tag); } - pub fn format(self: AxisTraversal, writer: *std.Io.Writer) !void { - try writer.writeAll("AxisTraversal(order=["); - - for (self.order.slice(), 0..) |tag, i| { - if (i > 0) try writer.writeAll(", "); - try writer.writeAll(@tagName(tag)); - } - - try writer.writeAll("], depth_by_tag={"); - - const field_names = comptime std.meta.fieldNames(PhysicalAxisTag); - inline for (field_names, 0..) |field_name, i| { - if (i > 0) try writer.writeAll(", "); - const tag = @field(PhysicalAxisTag, field_name); - try writer.writeAll(field_name); - try writer.writeAll("="); - if (self.depth_by_tag.get(tag)) |d| { - try writer.print("{d}", .{d}); - } else { - try writer.writeAll("null"); - } - } - - try writer.writeAll("})"); - } - /// Canonical DFS axis order: /// /// Tree: @@ -738,6 +705,89 @@ pub const PhysicalMesh = struct { return mesh; } + pub fn torus1d( + allocator: std.mem.Allocator, + target: Target, + devices: []const PlatformDevice, + ) !Sharding.PhysicalMesh { + const nodes = try allocator.alloc(PhysicalNode, devices.len); + for (nodes, devices) |*node, device| { + node.* = .device(device); + } + + return .fromOwnedTree(allocator, target, .axis(.link_x, .{ .mesh = .torus }, nodes)); + } + + pub fn torus2d( + allocator: std.mem.Allocator, + target: Target, + devices: []const PlatformDevice, + ) !Sharding.PhysicalMesh { + // Find the best split: 4 -> 2x2, 6 -> 2x3, 8 -> 2x4 + var link_x: usize = 1; + var link_y: usize = devices.len; + if (devices.len < 2) return error.NotEnoughDevices; + + for (2..devices.len) |i| { + if (@mod(devices.len, i) != 0) continue; + const j = @divExact(devices.len, i); + if (i <= j) { + link_x = i; + link_y = j; + } else break; + } + + // This only happen if number of devices is 1 or a prime number + if (link_x == 1) return error.NotEnoughDevices; + + const x_axes = try allocator.alloc(PhysicalNode, link_x); + var device_queue = devices; + for (x_axes) |*x_ax| { + const y_devices = try allocator.alloc(PhysicalNode, link_y); + for (y_devices) |*y_dev| { + y_dev.* = .device(device_queue[0]); + device_queue = device_queue[1..]; + } + + x_ax.* = .axis(.link_y, .{ .mesh = .torus }, y_devices); + } + + return .fromOwnedTree(allocator, target, .axis(.link_x, .{ .mesh = .torus }, x_axes)); + } + + pub fn torus3d( + allocator: std.mem.Allocator, + target: Target, + devices: []const PlatformDevice, + ) !Sharding.PhysicalMesh { + if (devices.len < 8) return error.NotEnoughDevices; + if (devices.len > 8) @panic("TODO: adapt logic from torus2D"); + const topology: Sharding.PhysicalMesh.Tree = .axis(.link_x, .{ .mesh = .torus }, &.{ + .axis(.link_y, .{ .mesh = .torus }, &.{ + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[0]), + .device(devices[1]), + }), + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[2]), + .device(devices[3]), + }), + }), + .axis(.link_y, .{ .mesh = .torus }, &.{ + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[4]), + .device(devices[5]), + }), + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[6]), + .device(devices[7]), + }), + }), + }); + + return .fromTree(allocator, target, topology); + } + fn populateDevicesInCanonicalOrder(self: *PhysicalMesh, allocator: std.mem.Allocator) !void { var devices_: std.ArrayList(Device) = .empty; errdefer devices_.deinit(allocator); @@ -961,29 +1011,25 @@ pub const PhysicalMesh = struct { } pub fn format(self: PhysicalMesh, writer: *std.Io.Writer) !void { - try writer.print("\nPhysicalMesh(platform={s} shardable_axes={any} num_devices={d})\n", .{ @tagName(self.target), self.shardableAxes(), self.countDevices() }); - try writer.print("├── {f}\n", .{self.axis_traversal}); - try writer.print("│ \n", .{}); - - try formatNode(self.root, writer, "", true); + try writer.print("PhysicalMesh(platform={s} shardable_axes={any} num_devices={d})\n", .{ @tagName(self.target), self.shardableAxes(), self.countDevices() }); + try formatNode(self.root, writer, "", 0, true); } - fn formatNode(node: PhysicalNode, writer: *std.Io.Writer, prefix: []const u8, is_last: bool) !void { + fn formatNode(node: PhysicalNode, writer: *std.Io.Writer, prefix: []const u8, depth: u8, is_last: bool) !void { try writer.print("{s}{s}", .{ prefix, if (is_last) "└── " else "├── " }); - switch (node) { - .leaf => |d| { - try writer.print("{f}\n", .{d}); + .leaf => |device| { + try writer.print("Device(id={d}, coords={any})\n", .{ device.id, device.coords[0..depth] }); }, .branch => |b| { - try writer.print("[{s}] x{d} ({s})\n", .{ @tagName(b.tag), b.children.len, @tagName(b.geometry) }); + try writer.print("[{s}] 0..{d} ({s})\n", .{ @tagName(b.tag), b.children.len, @tagName(b.geometry) }); var buf: [256]u8 = undefined; const new_prefix = std.fmt.bufPrint(&buf, "{s}{s}", .{ prefix, if (is_last) " " else "│ " }) catch unreachable; for (b.children, 0..) |child, i| { const child_is_last = (i == b.children.len - 1); - try formatNode(child, writer, new_prefix, child_is_last); + try formatNode(child, writer, new_prefix, depth +| 1, child_is_last); } }, } @@ -1373,28 +1419,38 @@ pub const LogicalAxisIntent = enum { low_bandwidth, }; -// Defines the logical mesh used for mesh the model and data. -// E.g., batch, model, context, experts, etc. -// Each axis represents a dimension along which the workload can be partitioned. The value is the intent of the axis. -/// LogicalMesh defines *semantic* axes only (batch/model/context/...). -/// It stores intent, not size. -/// -/// Why: -/// - Logical axes represent model meaning, not hardware. -/// - Sizes come from tensors later, during assignment. -/// -/// How it works: -/// - init() takes a struct of axis intents. -/// - each field name becomes a logical axis tag. -/// -/// Limitations: -/// - No sizes here; no divisibility checks here. +/// Semantic axes, their communication intents, and optional device-count constraints. pub const LogicalMesh = struct { pub const Axes = stdx.BoundedArray(Shape.Tag, MAX_MESH_RANK); pub const Intents = stdx.BoundedArray(LogicalAxisIntent, MAX_MESH_RANK); axes: Axes, intents: Intents, + dimensions: [MAX_MESH_RANK]i32, + + /// Create a logical mesh. + /// * names of the logical axes: .{ .a, .b }, + /// * intents: intent for each axis + /// * dimensions: dimension for each axis, -1 infers the size from underlying topology + pub fn init(names: anytype, intents: []const LogicalAxisIntent, dimensions: []const i32) !LogicalMesh { + if (@TypeOf(names) != []const Shape.Tag) { + const parsed_names = comptime Shape.parseTags(names); + return .init(@as([]const Shape.Tag, parsed_names.slice()), intents, dimensions); + } + + if (names.len == 0 or names.len > MAX_MESH_RANK or names.len != intents.len or names.len != dimensions.len) @panic("invalid logical mesh"); + + var result: LogicalMesh = .{ .axes = .init(names), .intents = .init(intents), .dimensions = @splat(-1) }; + for (names, dimensions, 0..) |name, dimension, i| { + if (dimension != -1 and dimension <= 0) return error.InvalidLogicalDimension; + if (std.mem.findScalar(Shape.Tag, names[0..i], name)) |_| { + @panic("invalid logical mesh"); + } + + result.dimensions[i] = dimension; + } + return result; + } pub fn mesh(axes_: anytype) LogicalMesh { const T = @TypeOf(axes_); @@ -1412,10 +1468,7 @@ pub const LogicalMesh = struct { stdx.debug.panic("LogicalMesh must have at least one axis defined", .{}); } - return .{ - .axes = axes, - .intents = intents, - }; + return .{ .axes = axes, .intents = intents, .dimensions = @splat(-1) }; } pub fn intent(self: LogicalMesh, tag: Shape.Tag) ?LogicalAxisIntent { @@ -1440,8 +1493,9 @@ pub const LogicalMesh = struct { pub fn format(self: LogicalMesh, writer: *std.Io.Writer) !void { try writer.writeAll("LogicalMesh("); - for (self.axes.slice(), self.intents.slice()) |axis, int| { + for (self.axes.slice(), self.intents.slice(), 0..) |axis, int, i| { try writer.print(" {s}={s}", .{ axis, @tagName(int) }); + if (self.dimensions[i] != -1) try writer.print("[{d}]", .{self.dimensions[i]}); } try writer.writeAll(")"); } @@ -1449,45 +1503,6 @@ pub const LogicalMesh = struct { pub const PhysicalList = stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK); -pub const Fold = struct { - target: PhysicalAxisTag, - sources: PhysicalList, -}; -pub const Folds = stdx.BoundedArray(Fold, MAX_MESH_RANK); - -pub const Axis = struct { - tag: PhysicalAxisTag, - size: i64, - geometry: ?AxisGeometry, - folded: stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK), - - pub fn contains(self: *const Axis, tag: PhysicalAxisTag) bool { - if (self.tag == tag) return true; - for (self.folded.slice()) |t| if (t == tag) return true; - return false; - } -}; - -pub const PhysicalView = struct { - axes: stdx.BoundedArray(Axis, MAX_MESH_RANK), - total_devices: i64, - - pub fn axisCoordFromLinearShard(self: *const PhysicalView, axis_index: usize, linear_idx: usize) usize { - const stride = self.axisStrideForLinear(axis_index); - const axis_size: usize = @intCast(self.axes.slice()[axis_index].size); - return (linear_idx / stride) % axis_size; - } - - pub fn axisStrideForLinear(self: *const PhysicalView, axis_index: usize) usize { - var stride: usize = 1; - var j = axis_index + 1; - while (j < self.axes.len) : (j += 1) { - stride *= @intCast(self.axes.slice()[j].size); - } - return stride; - } -}; - pub const Mesh = struct { pub const zml_no_meta_visit: void = {}; @@ -1495,13 +1510,11 @@ pub const Mesh = struct { physical: *const PhysicalMesh, logical: LogicalMesh, + grid: DeviceGrid, + /// Compact binding table: logical axis -> physical axes bindings: stdx.BoundedArray(PhysicalList, MAX_MESH_RANK), - /// Explicit folding rules: kept axis -> ordered source axes. - folds: Folds, - folds_consumed: std.EnumSet(PhysicalAxisTag), - pub fn shardableDim(mesh: *const Mesh, dim: i64, logical_axis: anytype, must_divide: i64) DimSharding { const partitions = mesh.numPartitionsForLogicalAxis(logical_axis); @@ -1542,6 +1555,8 @@ pub const Mesh = struct { logical: LogicalMesh, strategy: Strategy, ) !Mesh { + const num_devices = physical.countDevices(); + std.debug.assert(num_devices == physical.devices_in_canonical_order.len); const axis_order = physical.axisOrder().slice(); if (axis_order.len == 0) return error.InvalidPhysicalMesh; if (strategy.bindings.len == 0) return error.InvalidStrategy; @@ -1555,7 +1570,7 @@ pub const Mesh = struct { for (strategy.bindings.slice()) |strat_binding| { for (logical.axes.slice(), bindings.slice()) |logical_axis, *binding_| { if (std.mem.eql(u8, std.mem.span(logical_axis), std.mem.span(strat_binding.logical))) { - binding_.* = strat_binding.physical; + for (strat_binding.physical.constSlice()) |part| binding_.appendAssumeCapacity(part.axis); break; } } else { @@ -1563,24 +1578,17 @@ pub const Mesh = struct { } } - var folds: Folds = .empty; - var folds_consumed: std.EnumSet(PhysicalAxisTag) = .empty; - for (strategy.folding.slice()) |entry| { - for (entry.sources.slice()) |src| { - if (!physical.hasAxis(src)) return error.InvalidPhysicalAxis; - folds_consumed.insert(src); - } - folds.appendAssumeCapacity(.{ .target = entry.target, .sources = entry.sources }); - } - - return .{ + var mesh: Mesh = .{ .name = name_, .logical = logical, + .grid = undefined, .physical = physical, .bindings = bindings, - .folds = folds, - .folds_consumed = folds_consumed, }; + mesh.grid = try DeviceGrid.init(&mesh, &strategy); + std.debug.assert(num_devices == physical.devices_in_canonical_order.len); + std.debug.assert(@mod(mesh.numDevices(), mesh.numPartitions()) == 0); + return mesh; } pub fn initTest(name_: @EnumLiteral(), tagz: anytype) Mesh { @@ -1591,162 +1599,90 @@ pub const Mesh = struct { .logical = .{ .axes = .init(axes_tags.slice()), .intents = .{ .buffer = @splat(.high_bandwidth), .len = axes_tags.len }, + .dimensions = @splat(-1), }, .physical = undefined, + .grid = undefined, .bindings = .empty, - .folds = .empty, - .folds_consumed = .empty, }; } - fn foldSources(self: *const Mesh, tag: PhysicalAxisTag) ?[]const PhysicalAxisTag { - for (self.folds.slice()) |*f| { - if (f.target == tag) return f.sources.slice(); - } - return null; - } - pub fn devicesInCanonicalOrder(mesh: *const Mesh) []const Device { return mesh.physical.devices_in_canonical_order; } - pub fn physicalView(self: *const Mesh) PhysicalView { - const physical = self.physical; - var view: PhysicalView = .{ - .axes = .empty, - .total_devices = 1, - }; - - const axis_order = physical.axisOrder().slice(); - - for (axis_order) |tag| { - if (!physical.hasAxis(tag)) continue; - if (self.folds_consumed.contains(tag) and self.foldSources(tag) == null) continue; - - var folded: stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK) = .empty; - var size: i64 = 1; - - if (self.foldSources(tag)) |sources| { - for (sources) |src| { - folded.appendAssumeCapacity(src); - size *= physical.axis(src); - } - } else { - folded.appendAssumeCapacity(tag); - size = physical.axis(tag); - } - - view.total_devices *= size; - view.axes.appendAssumeCapacity(.{ - .tag = tag, - .size = size, - .geometry = physical.geometry(tag), - .folded = folded, - }); - } - - return view; - } - pub fn logicalIndexFromCoords(self: *const Mesh, coords: []const u8) usize { return self.physical.linearIndexFromCoords(coords); } - pub fn numPartitions(self: *const Mesh) i32 { - return @intCast(self.physicalView().total_devices); + pub fn numPartitions(self: *const Mesh) u32 { + var partitions: u32 = 1; + for (self.grid.axis_sizes[0..self.grid.rank]) |size| partitions *= size; + return @intCast(partitions); } - pub fn numPartitionsForLogicalAxis(self: *const Mesh, logical_axis: anytype) i64 { + pub fn numPartitionsForLogicalAxis(self: *const Mesh, logical_axis: anytype) u32 { const logical_tag = Shape.toTag(logical_axis); + const logical_ax = self.resolveLogicalAxis(logical_tag) orelse std.debug.panic("mesh {f} has no axis {s}", .{ self, logical_tag }); - const bound_axes = self.binding(logical_ax); - - var physical_axes: std.EnumSet(PhysicalAxisTag) = .empty; - for (bound_axes) |bound_axis| { - if (self.foldSources(bound_axis)) |sources| { - for (sources) |source| physical_axes.insert(source); - } else { - physical_axes.insert(bound_axis); - } - } - - var partitions: i64 = 1; - for (std.enums.values(PhysicalAxisTag)) |physical_axis| { - if (physical_axes.contains(physical_axis)) { - partitions *= self.physical.axis(physical_axis); - } - } - return partitions; + return self.grid.axis_sizes[logical_ax]; } - pub fn numReplicas(_: *const Mesh) i32 { - return 1; + pub fn numDevices(self: *const Mesh) u32 { + return @intCast(self.physical.devices_in_canonical_order.len); } - pub fn numDevices(self: *const Mesh) i32 { - return self.numPartitions() * self.numReplicas(); + pub fn numReplicas(self: *const Mesh) u32 { + return @divExact(self.numDevices(), self.numPartitions()); } pub fn sdyMeshAttr(self: *const Mesh, allocator: std.mem.Allocator) ![]const u8 { var out: std.Io.Writer.Allocating = .init(allocator); errdefer out.deinit(); - const view = self.physicalView(); try out.writer.writeAll("#sdy.mesh<["); - for (view.axes.slice(), 0..) |p, i| { + var tile_count: usize = 1; + for (self.logical.axes.slice(), 0..) |axis, i| { if (i > 0) try out.writer.writeAll(", "); - try out.writer.print("\"{s}\"={d}", .{ @tagName(p.tag), p.size }); + const size = self.grid.axis_sizes[i]; + tile_count *= size; + try out.writer.print("\"{s}\"={d}", .{ std.mem.span(axis), size }); + } + const replicas = @divExact(self.grid.device_count, tile_count); + if (replicas > 1) { + try out.writer.print(", \"{s}\"={d}", .{ self.sdyReplicaAxis(), replicas }); + } + try out.writer.writeAll("], device_ids=["); + var replica_indices: [Platform.MAX_NUM_DEVICES]usize = @splat(0); + var ids: [Platform.MAX_NUM_DEVICES]usize = undefined; + for (0..self.grid.device_count) |partition_index| { + const coords: [MAX_MESH_RANK]u8 = self.grid.logical_coords[partition_index]; + var tile_index: usize = 0; + for (0..self.logical.axes.len) |axis| { + tile_index = tile_index * self.grid.axis_sizes[axis] + coords[axis]; + } + // Device IDs here are executable partition indices, not PJRT device IDs. + ids[tile_index * replicas + replica_indices[tile_index]] = partition_index; + replica_indices[tile_index] += 1; + } + for (ids[0..self.grid.device_count], 0..) |id, i| { + if (i > 0) try out.writer.writeAll(", "); + try out.writer.print("{d}", .{id}); } try out.writer.writeAll("]>"); return try out.toOwnedSlice(); } - const DimMapping = struct { - /// For each tensor dimension, the indices of physical axes (in PhysicalView) mesh it. - axes_per_dim: stdx.BoundedArray(stdx.BoundedArray(usize, Shape.MAX_RANK), Shape.MAX_RANK), - /// Indices of physical axes used for replication. - replicated_axes: stdx.BoundedArray(usize, Shape.MAX_RANK), - /// Reference to the view for sizes. - view: PhysicalView, - }; - - /// Common logic to map tensor dimensions to physical mesh indices. - fn getDimMapping(self: *const Mesh, shape: Shape) DimMapping { - const view = self.physicalView(); - var axes_per_dim: stdx.BoundedArray(stdx.BoundedArray(usize, Shape.MAX_RANK), Shape.MAX_RANK) = .empty; - var used_mask: [Shape.MAX_RANK]bool = @splat(false); - var globally_used: std.EnumSet(PhysicalAxisTag) = .empty; - - for (0..shape.rank()) |ax| { - var dim_axes: stdx.BoundedArray(usize, Shape.MAX_RANK) = .empty; - const spec = shape.partition(ax); - - if (spec.meshAxis()) |mesh_ax| { - const binding_ = self.bindings.get(mesh_ax); - for (binding_.slice()) |p_tag| { - for (view.axes.slice(), 0..) |v_ax, i| { - // Only use the axis if it's bound and hasn't been consumed by a previous dimension - if (v_ax.contains(p_tag) and !globally_used.contains(v_ax.tag)) { - dim_axes.appendAssumeCapacity(i); - globally_used.insert(v_ax.tag); - used_mask[i] = true; - } - } - } - } - axes_per_dim.appendAssumeCapacity(dim_axes); + fn sdyReplicaAxis(self: *const Mesh) []const u8 { + // There are at most MAX_MESH_RANK logical names, so one candidate is free. + inline for (0..MAX_MESH_RANK + 1) |i| { + const name = std.fmt.comptimePrint("__replica_{d}", .{i}); + const taken = for (self.logical.axes.slice()) |axis| { + if (std.mem.eql(u8, std.mem.span(axis), name)) break true; + } else false; + if (!taken) return name; } - - var replicated_axes: stdx.BoundedArray(usize, Shape.MAX_RANK) = .empty; - for (0..view.axes.len) |i| { - if (!used_mask[i]) replicated_axes.appendAssumeCapacity(i); - } - - return .{ - .axes_per_dim = axes_per_dim, - .replicated_axes = replicated_axes, - .view = view, - }; + unreachable; } pub fn sdyShardingAttrForShape( @@ -1758,37 +1694,32 @@ pub const Mesh = struct { var arena = try stdx.arenaWithCapacity(parent_allocator, 1024); defer arena.deinit(); const allocator = arena.allocator(); - const mapping = data.getDimMapping(shape); const dimensions = try allocator.alloc(*const dialects.shardy.DimensionShardingAttribute, shape.rank()); - - // Note: here we map from the logical axis to physical. Is this a good idea ? - // why not stay in logical realm ? + var used: [MAX_MESH_RANK]bool = @splat(false); for (0.., dimensions) |ax, *d| { d.* = switch (shape.partition(ax)) { .out_of_bound => @panic("undefined"), .replicated => .replicated(ctx), .open => .open(ctx, &.{}), else => d: { - const dim_phys_indices = mapping.axes_per_dim.get(ax); - if (dim_phys_indices.len == 0) { - break :d .replicated(ctx); - } else { - const axes = try allocator.alloc(*const dialects.shardy.AxisRefAttribute, dim_phys_indices.len); - for (dim_phys_indices.slice(), 0..) |p_idx, i| { - axes[i] = .named(ctx, @tagName(mapping.view.axes.get(p_idx).tag)); - } - break :d .closed(ctx, axes); - } + const axis = shape.partition(ax).meshAxis().?; + used[axis] = true; + break :d .closed(ctx, &.{.named(ctx, std.mem.span(data.logical.axes.get(axis)))}); }, }; } - const replicated_axes = try allocator.alloc(*const dialects.shardy.AxisRefAttribute, mapping.replicated_axes.len); - for (replicated_axes, mapping.replicated_axes.slice()) |*r, p_idx| { - r.* = .named(ctx, @tagName(mapping.view.axes.get(p_idx).tag)); + var replicated_axes: stdx.BoundedArray(*const dialects.shardy.AxisRefAttribute, MAX_MESH_RANK + 1) = .empty; + var tile_count: usize = 1; + for (data.logical.axes.slice(), 0..) |axis, i| { + tile_count *= data.grid.axis_sizes[i]; + if (!used[i]) replicated_axes.appendAssumeCapacity(.named(ctx, std.mem.span(axis))); + } + if (tile_count < data.grid.device_count) { + replicated_axes.appendAssumeCapacity(.named(ctx, data.sdyReplicaAxis())); } - return .init(ctx, .{ .mesh = data.name, .dimensions = dimensions, .replicated_axes = replicated_axes }); + return .init(ctx, .{ .mesh = data.name, .dimensions = dimensions, .replicated_axes = replicated_axes.slice() }); } pub fn gspmdShardingAttrForShape(self: *const Mesh, parent_allocator: std.mem.Allocator, ctx: *mlir.Context, shape: Shape) !*const mlir.Attribute { @@ -1796,36 +1727,20 @@ pub const Mesh = struct { defer arena.deinit(); const allocator = arena.allocator(); - var has_sharding = false; - for (0..shape.rank()) |ax| { - if (shape.partition(ax).isSharded()) { - has_sharding = true; - break; - } - } - if (!has_sharding) return .string(ctx, try allocator.dupe(u8, "{replicated}")); - - const mapping = self.getDimMapping(shape); var tile_shape: stdx.BoundedArray(i64, Shape.MAX_RANK + 1) = .empty; - - // Calculate tile sizes per tensor dimension - for (mapping.axes_per_dim.slice()) |dim_axes| { - var combined: i64 = 1; - for (dim_axes.slice()) |idx| { - combined *= mapping.view.axes.get(idx).size; - } - tile_shape.appendAssumeCapacity(combined); + var tile_count: usize = 1; + for (0..shape.rank()) |ax| { + const size = if (shape.partition(ax).meshAxis()) |axis| self.grid.axis_sizes[axis] else 1; + tile_shape.appendAssumeCapacity(size); + tile_count *= size; } + if (tile_count == 1) return .string(ctx, "{replicated}"); - // Add replication dimension if physical axes are left over. - const has_replication = mapping.replicated_axes.len > 0; - if (has_replication) { - var repl_size: i64 = 1; - for (mapping.replicated_axes.slice()) |idx| { - repl_size *= mapping.view.axes.get(idx).size; - } - tile_shape.appendAssumeCapacity(repl_size); - } + // All devices at the same logical tile coordinate are replicas, including + // unused logical axes and physical axes that have no logical binding. + const replicas = @divExact(self.grid.device_count, tile_count); + const has_replication = replicas > 1; + if (has_replication) tile_shape.appendAssumeCapacity(@intCast(replicas)); var out: std.Io.Writer.Allocating = .init(allocator); errdefer out.deinit(); @@ -1858,20 +1773,31 @@ pub const Mesh = struct { } fn gspmdTileAssignment(self: *const Mesh, allocator: std.mem.Allocator, shape: Shape) ![]u32 { - const placement = try Placement.init(shape); + _ = try Sharding.Placement.init(&shape); const devices = self.devicesInCanonicalOrder(); + var tile_axes: stdx.BoundedArray(u8, MAX_MESH_RANK) = .empty; + var used_axes: u8 = 0; var tile_count: usize = 1; - for (placement.axis_plans.constSlice()) |plan| tile_count *= @intCast(plan.num_devices); + for (0..shape.rank()) |ax| { + const logical_axis = shape.partition(ax).meshAxis() orelse continue; + const bit = @as(u8, 1) << @intCast(logical_axis); + if (used_axes & bit != 0) continue; + used_axes |= bit; + tile_axes.appendAssumeCapacity(@intCast(logical_axis)); + tile_count *= self.grid.axis_sizes[logical_axis]; + } const replicas_per_tile = @divExact(devices.len, tile_count); const replica_indices = try allocator.alloc(usize, tile_count); defer allocator.free(replica_indices); @memset(replica_indices, 0); const ids = try allocator.alloc(u32, devices.len); - for (devices, 0..) |device, partition_index| { + for (0..devices.len) |partition_index| { + const coords: [MAX_MESH_RANK]u8 = self.grid.logical_coords[partition_index]; var tile_index: usize = 0; - for (placement.axis_plans.constSlice()) |plan| { - tile_index = tile_index * @as(usize, @intCast(plan.num_devices)) + @as(usize, @intCast(plan.linearIndex(device.coords))); + for (tile_axes.constSlice()) |logical_axis| { + tile_index = tile_index * self.grid.axis_sizes[logical_axis] + + coords[logical_axis]; } // HLO shardings refer to logical partitions; executable device assignment // separately maps those partitions to physical PJRT device IDs. @@ -1882,8 +1808,7 @@ pub const Mesh = struct { } pub fn deviceAssignment(self: *const Mesh, allocator: std.mem.Allocator) ![]u32 { - const view = self.physicalView(); - const count: u32 = @intCast(view.total_devices); + const count = self.physical.devices_in_canonical_order.len; var ids = try allocator.alloc(u32, count); @memset(ids, std.math.maxInt(u32)); @@ -1901,59 +1826,236 @@ pub const Mesh = struct { return ids; } - /// Extract coordinate data from the physical mesh for a specific axis. - fn addPhysicalToSplit(mesh: *const Mesh, plan: *AxisSplit, tag: PhysicalAxisTag) void { - const physical = mesh.physical; - const info = physical.axisInfo(tag) orelse return; - const depth = physical.axis_traversal.depth(tag) orelse return; - plan.add(@intCast(info.size), depth); + pub fn format(self: Mesh, writer: *std.Io.Writer) !void { + try writer.print("Mesh(name={s}, partitions={d}, replicas={d})\n", .{ self.name, self.numPartitions(), self.numReplicas() }); + + try writer.writeAll("┌ Mappings (logical coordinates from physical coordinates):\n"); + const num_axes = self.logical.axes.len; + for (self.logical.axes.slice(), self.logical.intents.slice(), 0..) |tag, intent, logical_axis| { + try writer.print("{s}── {s} ({s}) -> ", .{ if (logical_axis == num_axes - 1) "└" else "├", tag, @tagName(intent) }); + try self.formatAxisMapping(writer, logical_axis); + try writer.writeAll("\n"); + } + try writer.writeAll("\n"); + try writer.print("┌ {f}\n", .{self.physical}); + + try writer.writeAll("┌ Logical coordinates:\n"); + var coords: [MAX_MESH_RANK]usize = @splat(0); + try self.formatLogicalCoordinates(writer, &coords, 0); } - pub fn format(self: Mesh, writer: *std.Io.Writer) !void { - try writer.print("Mesh(name={s})\n", .{self.name}); - - try writer.writeAll("Bindings:\n"); - for (self.logical.axes.slice(), self.logical.intents.slice(), self.bindings.slice()) |l_tag, l_intent, bind| { - try writer.print(" - {s} ({s}) -> ", .{ l_tag, @tagName(l_intent) }); - - if (bind.len == 0) { - try writer.writeAll("unbound\n"); - } else { - for (bind.slice(), 0..) |physical, i| { - if (i > 0) try writer.writeAll(", "); - try writer.writeAll(@tagName(physical)); + /// Reverse engineer the mapping logical -> physical from the device grid. + fn formatAxisMapping(self: Mesh, writer: *std.Io.Writer, logical_axis: usize) !void { + const Part = struct { + tag: PhysicalAxisTag, + physical_size: usize, + physical_stride: usize, + size: usize, + logical_stride: usize, + }; + // A u8 coordinate has at most eight nontrivial mixed-radix digits. + var parts: stdx.BoundedArray(Part, @bitSizeOf(u8)) = .empty; + const logical_size = self.grid.axis_sizes[logical_axis]; + var logical_stride: usize = 1; + while (logical_stride < logical_size) { + const remaining = @divExact(logical_size, logical_stride); + var size = remaining; + const part: Part = found: { + // Prefer the largest matching portion, so whole axes stay readable. + while (size > 1) : (size -= 1) { + if (remaining % size != 0) continue; + for (self.physical.axisOrder().constSlice()) |tag| { + const physical_size: usize = @intCast(self.physical.axis(tag)); + const depth = self.physical.axis_traversal.depth(tag).?; + var physical_stride: usize = 1; + while (physical_stride * size <= physical_size) : (physical_stride += 1) { + if (physical_size % (physical_stride * size) != 0) continue; + for (self.devicesInCanonicalOrder(), 0..) |device, i| { + const coords: [MAX_MESH_RANK]u8 = self.grid.logical_coords[i]; + if ((coords[logical_axis] / logical_stride) % size != + (device.coords[depth] / physical_stride) % size) break; + } else { + break :found .{ + .tag = tag, + .physical_size = physical_size, + .physical_stride = physical_stride, + .size = size, + .logical_stride = logical_stride, + }; + } + } + } } - try writer.writeAll("\n"); + try writer.writeAll("device-dependent (see logical coordinates)"); + return; + }; + parts.appendAssumeCapacity(part); + logical_stride *= part.size; + } + if (parts.len == 0) { + try writer.writeAll("0 (replicated)"); + return; + } + var i: usize = parts.len; + while (i > 0) { + i -= 1; + const part = parts.constSlice()[i]; + if (i + 1 < parts.len) try writer.writeAll(" + "); + const partial = part.size != part.physical_size; + if (partial) try writer.writeAll("("); + try writer.writeAll(@tagName(part.tag)); + if (part.physical_stride > 1) try writer.print(" / {d}", .{part.physical_stride}); + if (part.physical_stride * part.size < part.physical_size) try writer.print(" % {d}", .{part.size}); + if (partial) try writer.writeAll(")"); + if (part.logical_stride > 1) try writer.print(" * {d}", .{part.logical_stride}); + } + } + + fn formatLogicalCoordinates(self: Mesh, writer: *std.Io.Writer, coords: *[MAX_MESH_RANK]usize, depth: usize) std.Io.Writer.Error!void { + const axes = self.logical.axes.constSlice(); + if (axes.len == 0) return; + const last = axes.len - 1; + if (depth < last) { + const size: usize = @intCast(self.numPartitionsForLogicalAxis(axes[depth])); + for (0..size) |coord| { + coords[depth] = coord; + try writer.splatByteAll(' ', 4 * depth); + try writer.print("└── {s}={d}\n", .{ axes[depth], coord }); + try self.formatLogicalCoordinates(writer, coords, depth + 1); } + return; } - const view = self.physicalView(); - - try writer.print("Physical capacity: {d}\n", .{view.total_devices}); - - try writer.writeAll("Physical axes: "); - if (view.axes.len == 0) { - try writer.writeAll("(none)\n"); - } else { - for (view.axes.slice(), 0..) |axis, i| { - if (i > 0) try writer.writeAll(" × "); - try writer.print("{s}[{d}]", .{ @tagName(axis.tag), axis.size }); - } - try writer.writeAll("\n"); + try writer.splatByteAll(' ', 4 * depth); + try writer.print("└── {s}=", .{axes[last]}); + const size: usize = @intCast(self.numPartitionsForLogicalAxis(axes[last])); + for (0..size) |coord| { + if (coord > 0) try writer.writeAll(","); + try writer.print("{d}", .{coord}); } - - try writer.writeAll("Geometries: "); - if (view.axes.len == 0) { - try writer.writeAll("(none)\n"); - } else { - for (view.axes.slice(), 0..) |axis, i| { - if (i > 0) try writer.writeAll(", "); - try writer.print("{s}={s}", .{ - @tagName(axis.tag), - @tagName(axis.geometry orelse .point_to_point), - }); + try writer.writeAll(" -> devices="); + for (0..size) |coord| { + coords[last] = coord; + if (coord > 0) try writer.writeAll(","); + var count: usize = 0; + for (0..self.grid.device_count) |device_index| { + if (self.matchesLogicalCoordinates(device_index, coords[0..axes.len])) count += 1; } - try writer.writeAll("\n"); + // Keep replicas grouped so each entry corresponds to one last-axis coordinate. + if (count != 1) try writer.writeAll("["); + var written: usize = 0; + for (self.devicesInCanonicalOrder(), 0..) |device, device_index| { + if (!self.matchesLogicalCoordinates(device_index, coords[0..axes.len])) continue; + if (written > 0) try writer.writeAll(","); + try writer.print("{d}", .{device.id}); + written += 1; + } + if (count != 1) try writer.writeAll("]"); + } + try writer.writeAll("\n"); + } + + fn matchesLogicalCoordinates(self: Mesh, device_index: usize, coords: []const usize) bool { + const actual: [MAX_MESH_RANK]u8 = self.grid.logical_coords[device_index]; + for (coords, 0..) |expected, logical_axis| { + if (actual[logical_axis] != expected) return false; + } + return true; + } +}; + +/// Resolved logical coordinates in canonical physical-device order. +/// Unused logical axes have coordinate zero. +pub const DeviceGrid = struct { + pub const Coordinates = [MAX_MESH_RANK]u8; + + comptime { + std.debug.assert(Platform.MAX_NUM_DEVICES <= 256); + } + + rank: u8, + device_count: usize, + axis_sizes: [MAX_MESH_RANK]u32, + logical_coords: [Platform.MAX_NUM_DEVICES]Coordinates align(16), + + fn init(mesh_: *const Mesh, strategy: *const Strategy) !DeviceGrid { + for (strategy.folding.constSlice()) |entry| { + for (entry.sources.constSlice()) |source| { + if (!mesh_.physical.hasAxis(source)) return error.InvalidPhysicalAxis; + } + } + const devices = mesh_.devicesInCanonicalOrder(); + if (devices.len > Platform.MAX_NUM_DEVICES) return error.InvalidPhysicalMesh; + var grid: DeviceGrid = .{ + .rank = @intCast(mesh_.logical.axes.len), + .device_count = devices.len, + .axis_sizes = @splat(1), + .logical_coords = @splat(@splat(0)), + }; + for (mesh_.logical.axes.constSlice(), 0..) |logical_tag, logical_axis| { + var used: std.EnumSet(PhysicalAxisTag) = .empty; + for (strategy.bindings.constSlice()) |binding_| { + if (!std.mem.eql(u8, std.mem.span(binding_.logical), std.mem.span(logical_tag))) continue; + for (binding_.physical.constSlice()) |part| { + if (part.size) |size| { + const info = mesh_.physical.axisInfo(part.axis) orelse return error.InvalidPhysicalAxis; + const physical_size: u32 = @intCast(info.size); + if (size == 0 or part.stride == 0 or size > physical_size or part.stride > physical_size / size) return error.InvalidPartialBinding; + if (physical_size % (part.stride * size) != 0) return error.InvalidPartialBinding; + try grid.appendAxis(mesh_, logical_axis, part.axis, part.stride, size); + } else { + for (strategy.foldSources(part.axis) orelse &.{part.axis}) |source| { + if (used.contains(source)) continue; + used.insert(source); + const info = mesh_.physical.axisInfo(source) orelse continue; + try grid.appendAxis(mesh_, logical_axis, source, 1, @intCast(info.size)); + } + } + } + } + const dimension = mesh_.logical.dimensions[logical_axis]; + if (dimension != -1) { + if (dimension <= 0) return error.InvalidLogicalDimension; + const size: u32 = @intCast(dimension); + if (grid.axis_sizes[logical_axis] % size != 0) return error.IncompatibleLogicalDimension; + grid.axis_sizes[logical_axis] = size; + // Keep the least-significant portion of the ordered binding; the rest replicates. + for (grid.logical_coords[0..devices.len]) |*row| { + var coords: [MAX_MESH_RANK]u8 = row.*; + coords[logical_axis] = @intCast(coords[logical_axis] % size); + row.* = coords; + } + } + } + // Shared physical axes are valid only when their portions produce a complete + // Cartesian logical grid with the same number of replicas at every coordinate. + var partitions: usize = 1; + for (grid.axis_sizes[0..grid.rank]) |size| { + if (size > devices.len / partitions) return error.OverlappingLogicalAxes; + partitions *= size; + } + if (devices.len % partitions != 0) return error.OverlappingLogicalAxes; + var counts: [Platform.MAX_NUM_DEVICES]usize = @splat(0); + for (grid.logical_coords[0..devices.len]) |row| { + const coords: [MAX_MESH_RANK]u8 = row; + var index: usize = 0; + for (coords[0..grid.rank], grid.axis_sizes[0..grid.rank]) |coord, size| index = index * size + coord; + counts[index] += 1; + } + for (counts[0..partitions]) |count| { + if (count != devices.len / partitions) return error.OverlappingLogicalAxes; + } + return grid; + } + + fn appendAxis(self: *DeviceGrid, mesh_: *const Mesh, logical_axis: usize, tag: PhysicalAxisTag, stride: u32, size: u32) !void { + if (size > Platform.MAX_NUM_DEVICES / self.axis_sizes[logical_axis]) return error.InvalidPartialBinding; + const depth = mesh_.physical.axis_traversal.depth(tag).?; + self.axis_sizes[logical_axis] *= size; + for (mesh_.devicesInCanonicalOrder(), 0..) |device, i| { + var coords: [MAX_MESH_RANK]u8 = self.logical_coords[i]; + coords[logical_axis] = @intCast(@as(u32, coords[logical_axis]) * size + (device.coords[depth] / stride) % size); + self.logical_coords[i] = coords; } } }; @@ -1963,9 +2065,15 @@ pub const Strategy = struct { target: PhysicalAxisTag, sources: PhysicalList, }; + pub const PhysicalBinding = struct { + axis: PhysicalAxisTag, + stride: u32 = 1, + /// null binds the whole axis (including any explicit fold). + size: ?u32 = null, + }; pub const Binding = struct { logical: Shape.Tag, - physical: PhysicalList, + physical: stdx.BoundedArray(PhysicalBinding, MAX_MESH_RANK), }; pub const Bindings = stdx.BoundedArray(Binding, MAX_MESH_RANK); @@ -1999,19 +2107,31 @@ pub const Strategy = struct { } pub fn addBinding(self: *Strategy, logical: anytype, physical: PhysicalAxisTag) void { - const raw_tag = Shape.toTag(logical); - const logical_tag: []const u8 = std.mem.span(raw_tag); + self.appendBinding(Shape.toTag(logical), .{ .axis = physical }); + } - for (self.bindings.slice()) |*b| { - if (std.mem.eql(u8, std.mem.span(b.logical), logical_tag)) { - b.physical.appendAssumeCapacity(physical); + /// Append the mixed-radix portion `(coordinate / stride) % size` of a physical axis. + /// `stride * size` must divide the physical axis size. Partial bindings do not expand folds. + /// Portions are concatenated in insertion order, most significant first. + pub fn addPartialBinding(self: *Strategy, logical: anytype, physical: PhysicalAxisTag, stride: u32, size: u32) void { + self.appendBinding(Shape.toTag(logical), .{ .axis = physical, .stride = stride, .size = size }); + } + + fn appendBinding(self: *Strategy, logical: Shape.Tag, physical: PhysicalBinding) void { + for (self.bindings.slice()) |*binding_| { + if (std.mem.eql(u8, std.mem.span(binding_.logical), std.mem.span(logical))) { + binding_.physical.appendAssumeCapacity(physical); return; } } + self.bindings.appendAssumeCapacity(.{ .logical = logical, .physical = .init(&.{physical}) }); + } - var list: PhysicalList = .empty; - list.appendAssumeCapacity(physical); - self.bindings.appendAssumeCapacity(.{ .logical = raw_tag, .physical = list }); + fn foldSources(self: *const Strategy, tag: PhysicalAxisTag) ?[]const PhysicalAxisTag { + for (self.folding.constSlice()) |*fold| { + if (fold.target == tag) return fold.sources.constSlice(); + } + return null; } /// Explicitly fold axes: `target` is the kept axis, `sources` define order. @@ -2054,24 +2174,56 @@ pub const Strategy = struct { for (physical.axisOrder().slice()) |tag| available.appendAssumeCapacity(tag); } - const avail = available.slice(); - var counters = std.EnumArray(LogicalAxisIntent, usize).initFill(0); + const avail = available.constSlice(); + if (avail.len == 0) return strategy; + var remaining: [MAX_MESH_RANK]u32 = @splat(1); + for (avail, 0..) |tag, i| remaining[i] = @intCast(physical.axis(tag)); - inline for (std.meta.tags(LogicalAxisIntent)) |target_intent| { - for (logical.axes.slice(), logical.intents.slice()) |l_tag, l_intent| { - if (l_intent != target_intent) continue; - - const i = counters.get(target_intent); - const p_tag = switch (target_intent) { - .high_bandwidth => avail[i % avail.len], - .balanced => avail[(avail.len / 2 + i) % avail.len], - .low_bandwidth => avail[avail.len - 1 - (i % avail.len)], - }; - - strategy.addBinding(l_tag, p_tag); - counters.set(target_intent, i + 1); + // Reserve fixed dimensions before automatic axes can consume their capacity. + // Carve fixed portions from the most-significant end, leaving a contiguous + // low-order portion for automatic axes (e.g. model parallelism on one link). + for ([_]bool{ true, false }) |fixed| { + var counters: std.EnumArray(LogicalAxisIntent, u32) = .initFill(0); + inline for (std.meta.tags(LogicalAxisIntent)) |target_intent| { + for (logical.axes.constSlice(), logical.intents.constSlice(), 0..) |tag, intent, logical_axis| { + if (intent != target_intent) continue; + const i = counters.get(target_intent); + counters.set(target_intent, @intCast(i + 1)); + const dimension = logical.dimensions[logical_axis]; + if ((dimension != -1) != fixed) continue; + const preferred = switch (target_intent) { + .high_bandwidth => i % avail.len, + .balanced => (avail.len / 2 + i) % avail.len, + .low_bandwidth => avail.len - 1 - (i % avail.len), + }; + if (fixed) { + if (dimension <= 1) { + strategy.addPartialBinding(tag, avail[preferred], 1, 1); + continue; + } + var needed: u32 = @intCast(dimension); + for (0..avail.len) |offset| { + const axis = if (target_intent == .low_bandwidth) + (preferred + avail.len - offset) % avail.len + else + (preferred + offset) % avail.len; + const size = std.math.gcd(needed, remaining[axis]); + if (size == 1) continue; + remaining[axis] = @divExact(remaining[axis], size); + strategy.addPartialBinding(tag, avail[axis], remaining[axis], size); + needed = @divExact(needed, size); + if (needed == 1) break; + } + // Mesh.init reports an incompatible dimension if capacity was insufficient. + if (needed == @as(u32, @intCast(dimension))) strategy.addPartialBinding(tag, avail[preferred], 1, 1); + } else if (remaining[preferred] > 1) { + strategy.addPartialBinding(tag, avail[preferred], 1, remaining[preferred]); + remaining[preferred] = 1; + } + } } } + if (strategy.bindings.len == 0) strategy.addPartialBinding(logical.axes.constSlice()[0], avail[0], 1, 1); return strategy; } @@ -2080,8 +2232,8 @@ pub const Strategy = struct { /// For a given shape, compute the shape of the slice each shard will receive. pub fn shardedShape(shape: Shape) error{IncompatibleSharding}!Shape { if (shape._sharding.mesh != null) { - const pl: Placement = try .init(shape); - var local_shape = pl.shape; + const pl: Placement = try Sharding.Placement.init(&shape); + var local_shape = pl.localShape(shape); local_shape._sharding = .replicated; return local_shape; } else { @@ -2090,7 +2242,7 @@ pub fn shardedShape(shape: Shape) error{IncompatibleSharding}!Shape { } } -/// Precompute mesh information common to all devices for a given shape. +/// Owns per-device addressing metadata for a shape. pub const Placement = struct { pub const Slices = stdx.BoundedArray(Slice1d, Shape.MAX_RANK); pub const Slice1d = struct { @@ -2098,208 +2250,127 @@ pub const Placement = struct { size: i64, }; - // Note this shape is mainly use to indicate the dims of the sharded buffer. - // We may want to only store that. - shape: Shape, - global_shape: if (builtin.mode == .debug) Shape else void, - axis_plans: stdx.BoundedArray(AxisSplit, Shape.MAX_RANK), + local_dims: stdx.BoundedArray(i64, Shape.MAX_RANK), + mesh_ptr: ?*const Mesh, + byte_steps: [MAX_MESH_RANK]u64, - pub fn init(shape: Shape) error{IncompatibleSharding}!Placement { - var pl: Placement = .{ - .shape = shape, // modified below - .global_shape = if (builtin.mode == .debug) shape else {}, - .axis_plans = .empty, // set below + pub fn init(shape: *const Shape) error{IncompatibleSharding}!Placement { + const byte_strides = shape.computeByteStrides(); + return initWithByteStrides(shape, byte_strides.constSlice()); + } + + /// Reuse the caller's contiguous byte strides without recomputing them. + pub fn initWithByteStrides(shape: *const Shape, byte_strides: []const i64) error{IncompatibleSharding}!Placement { + std.debug.assert(byte_strides.len == shape.rank()); + if (builtin.mode == .debug) std.debug.assert(shape._sharding.partition.hasUniqueAxes()); + var result: Placement = .{ + .local_dims = .{ .buffer = undefined, .len = shape.rank() }, + .mesh_ptr = shape._sharding.mesh, + .byte_steps = @splat(0), }; - // mesh == null => replicated => no axis plan - if (shape._sharding.mesh == null) return pl; - const mesh_ = shape._sharding.mesh.?; - var used_axes: std.EnumSet(PhysicalAxisTag) = .empty; - - for (0..shape.rank()) |ax| { - const axis_index: u8 = @intCast(ax); - pl.axis_plans.appendAssumeCapacity(try axisSplit(mesh_, shape, &used_axes, axis_index)); + for (shape.dims(), result.local_dims.slice(), 0..) |dim, *result_local_dim, ax| { + result_local_dim.* = dim; + const mesh_ = result.mesh_ptr orelse continue; + const logical_axis = shape.partition(ax).meshAxis() orelse continue; + const count = mesh_.grid.axis_sizes[logical_axis]; + if (@rem(dim, count) != 0) return error.IncompatibleSharding; + const local_dim = @divExact(dim, count); + result_local_dim.* = local_dim; + result.byte_steps[logical_axis] = @intCast(local_dim * byte_strides[ax]); } - - for (pl.shape._dims.slice(), shape.dims(), pl.axis_plans.slice()) |*shard_dim, dim, plan| { - shard_dim.* = @divExact(dim, plan.num_devices); - } - return pl; + return result; } pub fn mesh(pl: *const Placement) ?*const Mesh { - return pl.shape._sharding.mesh; + return pl.mesh_ptr; } - pub fn slices(pl: *const Placement, device: Device.Coords) Slices { - var res: Slices = .{ .buffer = undefined, .len = pl.shape.rank() }; - if (pl.mesh() == null) { - for (res.slice(), pl.shape.dims()) |*slice, dim| slice.* = .{ .start = 0, .size = dim }; + /// Reconstruct metadata only for consumers that need a full shard shape. + pub fn localShape(pl: *const Placement, source: Shape) Shape { + std.debug.assert(source.rank() == pl.local_dims.len); + var result = source; + @memcpy(result._dims.slice(), pl.local_dims.constSlice()); + result._sharding.mesh = pl.mesh_ptr; + return result; + } + + pub fn slices(pl: *const Placement, device_index: usize, source: Shape) Slices { + std.debug.assert(source.rank() == pl.local_dims.len); + var res: Slices = .{ .buffer = undefined, .len = pl.local_dims.len }; + if (pl.mesh_ptr == null) { + for (res.slice(), pl.local_dims.constSlice()) |*slice, dim| slice.* = .{ .start = 0, .size = dim }; return res; } - for (res.slice(), pl.shape.dims(), pl.axis_plans.slice()) |*r, size, plan| { - const start = plan.linearIndex(device) * size; - r.* = .{ - .start = start, - .size = size, - }; + const grid = &pl.mesh_ptr.?.grid; + std.debug.assert(device_index < grid.device_count); + const coords: [MAX_MESH_RANK]u8 = grid.logical_coords[device_index]; + var used_axes: u8 = 0; + for (res.slice(), pl.local_dims.constSlice(), 0..) |*r, size, ax| { + var coordinate: i64 = 0; + if (source.partition(ax).meshAxis()) |logical_axis| { + const bit = @as(u8, 1) << @intCast(logical_axis); + if (used_axes & bit == 0) coordinate = coords[logical_axis]; + used_axes |= bit; + } + r.* = .{ .start = coordinate * size, .size = size }; } return res; } - pub fn shardPtr(pl: *const Placement, device: Device.Coords, slice: Slice) [*]const u8 { + /// Validate the contiguous source slice once before iterating over devices. + /// The returned pointer includes the slice's byte offset. + pub fn basePtr(pl: *const Placement, slice: Slice) [*]const u8 { if (builtin.mode == .debug) { - // there is a bug in caller code that used a placement for a different shape - std.debug.assert(pl.global_shape.eql(slice.shape)); + std.debug.assert(pl.local_dims.len == slice.shape.rank()); } + return slice.constData().ptr; + } - var ptr: [*]const u8 = slice.constData().ptr; - // mesh = null => implies replicated => each device read from the start. - if (pl.shape._sharding.mesh == null) return ptr; - for (pl.shape.dims(), pl.axis_plans.slice(), slice.byte_strides.slice()) |size, plan, stride| { - const start = plan.linearIndex(device) * size; - ptr = ptr[@intCast(start * stride)..]; - } - return ptr; + /// device_index is a canonical ordinal, never a PJRT device ID. + /// base_ptr must reference validated contiguous data for the global shape. + pub fn shardPtr(pl: *const Placement, device_index: usize, base_ptr: [*]const u8) [*]const u8 { + return base_ptr + pl.byteOffset(device_index); + } + + fn byteOffset(pl: *const Placement, device_index: usize) usize { + const mesh_ = pl.mesh() orelse return 0; + std.debug.assert(device_index < mesh_.grid.device_count); + const coords: [MAX_MESH_RANK]u8 = mesh_.grid.logical_coords[device_index]; + const steps = pl.byte_steps; + // Placement coefficients and grid coordinates describe offsets within the + // validated source slice. Keep arithmetic checks in debug only. + return offset: { + @setRuntimeSafety(builtin.mode == .debug); + var offset: u64 = 0; + inline for (0..MAX_MESH_RANK) |axis| { + offset += @as(u64, coords[axis]) * steps[axis]; + } + break :offset @intCast(offset); + }; } /// Given a Slice, returns the subslice corresponding to a specific device. /// Depending on the layout, the sub-slice may or may not be contiguous. - pub fn shardSlice(pl: *const Placement, device: Device.Coords, slice: Slice) Slice { - const shard_ptr = pl.shardPtr(device, slice); + pub fn shardSlice(pl: *const Placement, device_index: usize, slice: Slice) Slice { + const shard_ptr = pl.shardPtr(device_index, pl.basePtr(slice)); const offset_bytes = @intFromPtr(shard_ptr) - @intFromPtr(slice.bytes.ptr); return .{ .bytes = slice.bytes, .mutable = slice.mutable, - .shape = pl.shape, + .shape = pl.localShape(slice.shape), .offset_bytes = offset_bytes, .byte_strides = slice.byte_strides, }; } - pub fn format(pl: Placement, writer: *std.Io.Writer) std.Io.Writer.Error!void { - const ordered_devices = if (pl.mesh()) |m| m.devicesInCanonicalOrder() else { - return try writer.print("Placement(shape={f}, replicated)\n", .{pl.shape}); - }; - - try pl.formatPlacementSummary(pl.shape, ordered_devices.len, writer); - try writer.writeAll("Per-shard placement:\n"); - for (ordered_devices, 0..) |device, shard_index| { - try writer.print("└─ Shard[{d}] device={d} coords={any}, slices=[", .{ shard_index, device.id, device.coords }); - for (0.., pl.slices(device.coords).slice()) |ax, s| { - if (ax > 0) try writer.writeAll(", "); - const axis_label = pl.shape.debugTag(ax); - try writer.print("{s}:[{d}:{d}]", .{ axis_label, s.start, s.start + s.size }); - } - try writer.writeAll("]\n"); - } - - for (0.., pl.axis_plans.slice()) |ax, plan| { - try writer.print("- axis{d} -> plan {f}\n", .{ ax, plan }); - } - } - - fn formatPlacementSummary(pl: Placement, shape: Shape, shard_count: usize, writer: *std.Io.Writer) std.Io.Writer.Error!void { - try writer.print("Placement(shape={f} shards={d})\n", .{ shape, shard_count }); - - try writer.writeAll("Axis partitioning summary:\n"); - for (0.., pl.slices(@splat(0)).slice()) |ax, slice| { - const dim = shape.dim(ax); - const spec = shape.partition(ax); - const axis_label = shape.debugTag(ax); - - const shard_size: i64 = slice.size; - const shards_count: i64 = if (shard_size > 0 and @rem(dim, shard_size) == 0) - @divExact(dim, shard_size) - else - 0; - - try writer.print( - "└─ {s}: dim={d}, spec={s}, shard_size={d}, shards={d}\n", - .{ axis_label, dim, @tagName(spec), shard_size, shards_count }, - ); - } + pub fn format(pl: *const Placement, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print( + "Placement(local_dims={any}, mesh={*}, byte_steps={any})\n", + .{ pl.local_dims.constSlice(), pl.mesh_ptr, pl.byte_steps }, + ); } }; -const AxisSplit = struct { - /// For each physical coordinate depth, the contribution of that coordinate - /// to the linear shard index. Unused depths have stride 0. - coord_strides: Device.Coords, - counts: Device.Coords, - num_devices: u32, - - pub const empty: AxisSplit = .{ - .coord_strides = @splat(0), - .counts = @splat(0), - .num_devices = 1, - }; - - pub fn add(split: *AxisSplit, size: u8, depth: u8) void { - for (&split.coord_strides) |*stride| stride.* *= size; - split.coord_strides[depth] = 1; - split.counts[depth] = size; - split.num_devices *= size; - } - - pub fn linearIndex(split: AxisSplit, device_coords: Device.Coords) u32 { - @setRuntimeSafety(false); - const coords_u8: @Vector(MAX_MESH_RANK, u8) = device_coords; - const strides: @Vector(MAX_MESH_RANK, u8) = split.coord_strides; - return @reduce(.Add, coords_u8 * strides); - } - - pub fn format(split: AxisSplit, writer: *std.Io.Writer) std.Io.Writer.Error!void { - try writer.print("Plan(counts={any},strides={any})", .{ split.counts, split.coord_strides }); - } -}; - -fn axisSplit( - mesh: *const Mesh, - shape: Shape, - used_axes: *std.EnumSet(PhysicalAxisTag), - axis_index: u8, -) !AxisSplit { - const dim = shape.dim(axis_index); - const spec = shape.partition(axis_index); - - return if (spec.meshAxis()) |mesh_axis| { - const binding = mesh.binding(mesh_axis); - - // Calculate the split based on the physical coordinates of the current device. - return try calculateSplit(mesh, dim, binding, used_axes); - } else .empty; -} - -fn calculateSplit( - mesh: *const Mesh, - dim: i64, - binding: []const PhysicalAxisTag, - used_axes: *std.EnumSet(PhysicalAxisTag), -) error{IncompatibleSharding}!AxisSplit { - var plan: AxisSplit = .empty; - - for (binding) |p_tag| { - // Expand the physical tag: check if it's a target for folded source axes. - if (mesh.foldSources(p_tag)) |sources| { - for (sources) |src| { - if (used_axes.contains(src)) continue; - mesh.addPhysicalToSplit(&plan, src); - used_axes.insert(src); - } - } else { - // Standard case: directly bound, not part of an explicit fold. - if (used_axes.contains(p_tag)) continue; - mesh.addPhysicalToSplit(&plan, p_tag); - used_axes.insert(p_tag); - } - } - - if (plan.num_devices > 0 and @rem(dim, plan.num_devices) != 0) { - return error.IncompatibleSharding; - } - return plan; -} - const ShardingTest = struct { pub const ExpectedShard = struct { // TODO: remove device_id from here, we always have ids from 0 to N @@ -2311,12 +2382,14 @@ const ShardingTest = struct { mesh: *const Mesh, shape: Shape, + expected_num_partitions: u32, + expected_num_replicas: u32, expect_error: ?anyerror = null, expected_sdy: ?[]const u8 = null, expected_shards: []const ExpectedShard = &.{}, pub fn format(scenario: Scenario, writer: *std.Io.Writer) std.Io.Writer.Error!void { - try writer.print("Scenario:\n", .{}); + try writer.print("Scenario: partitions={d}, replicas={d}\n", .{ scenario.expected_num_partitions, scenario.expected_num_replicas }); if (scenario.expect_error) |err| try writer.print("- error {}\n", .{err}); if (scenario.expected_sdy) |sdy| try writer.print("- shardy: {s}\n", .{sdy}); if (scenario.expected_shards.len > 0) try writer.print("- shards:\n", .{}); @@ -2365,6 +2438,10 @@ const ShardingTest = struct { const mesh = s.mesh; errdefer std.log.warn("Failed mesh {f}", .{s}); + try std.testing.expectEqual(s.expected_num_partitions, mesh.numPartitions()); + try std.testing.expectEqual(s.expected_num_replicas, mesh.numReplicas()); + try std.testing.expectEqual(mesh.grid.device_count, @as(usize, @intCast(mesh.numDevices()))); + // Verify MLIR String if (s.expected_sdy) |expected_attr| { errdefer std.log.warn("Expected shardy annotation failed", .{}); @@ -2389,21 +2466,21 @@ const ShardingTest = struct { if (s.expect_error) |err| { errdefer std.log.warn("Expected error failed", .{}); try std.testing.expect(ordered_devices.len > 0); - try std.testing.expectError(err, Sharding.Placement.init(s.shape)); + try std.testing.expectError(err, Sharding.Placement.init(&s.shape)); return; } // Verify Shard Slices (Math) if (s.expected_shards.len > 0) { - const pl: Placement = try .init(s.shape); + const pl: Placement = try Sharding.Placement.init(&s.shape); errdefer std.log.warn("Expected shards failed. Got {f}", .{pl}); try std.testing.expectEqual(s.expected_shards.len, ordered_devices.len); - for (s.expected_shards, ordered_devices) |expected, device| { + for (s.expected_shards, ordered_devices, 0..) |expected, device, device_index| { { try std.testing.expectEqual(expected.device_id, device.id); // Check pl.slices(device) - const actual_slices = pl.slices(device.coords); + const actual_slices = pl.slices(device_index, s.shape); // Consider rewriting test cases to use Slice1D. This requires passing the axis const actual_start_len = try self.allocator.alloc([2]i64, actual_slices.len); @@ -2422,7 +2499,7 @@ const ShardingTest = struct { expected_size.* = slice[1]; } - try std.testing.expectEqualSlices(i64, expected_sizes, pl.shape.dims()); + try std.testing.expectEqualSlices(i64, expected_sizes, pl.local_dims.constSlice()); } } } @@ -2441,10 +2518,12 @@ test "mesh: unknown partitioning implies replication" { const mesh: Sharding.Mesh = try .init("folded_mesh", &physical, logical, strategy); try runner.run(.{ + .expected_num_partitions = 2, + .expected_num_replicas = 2, .mesh = &mesh, // No partitioning on shape .shape = Shape.init(.{ .batch = 8 }, .f32), - .expected_sdy = "#sdy.sharding<@folded_mesh, [{}], replicated={\"link_x\", \"link_y\"}>", + .expected_sdy = "#sdy.sharding<@folded_mesh, [{}], replicated={\"batch\", \"__replica_0\"}>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{.{ 0, 8 }} }, .{ .device_id = 1, .slices = &.{.{ 0, 8 }} }, @@ -2454,6 +2533,49 @@ test "mesh: unknown partitioning implies replication" { }); } +test "suggest splits a one-dimensional topology for fixed data and automatic model axes" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + const runner: ShardingTest = .init(allocator); + const logical = try LogicalMesh.init(.{ .data, .model }, &.{ .low_bandwidth, .high_bandwidth }, &.{ 2, -1 }); + for ([_]usize{ 2, 4, 8 }) |count| { + const physical = try runner.physical(.{count}, .{ .mesh = .torus }); + const mesh_ = try Mesh.init("DP-MP", &physical, logical, .suggest(logical, &physical)); + try std.testing.expectEqual(2, mesh_.numPartitionsForLogicalAxis(.data)); + try std.testing.expectEqual(@as(u32, @intCast(count / 2)), mesh_.numPartitionsForLogicalAxis(.model)); + try std.testing.expectEqual(1, mesh_.numReplicas()); + for (0..count) |i| { + const coords: [MAX_MESH_RANK]u8 = mesh_.grid.logical_coords[i]; + try std.testing.expectEqual(@as(u8, @intCast(i / (count / 2))), coords[0]); + try std.testing.expectEqual(@as(u8, @intCast(i % (count / 2))), coords[1]); + } + const shape = Shape.init(.{ .b = 8, .m = 8 }, .f32).withPartitioning(&mesh_, .{ .b = .data, .m = .model }); + try runner.run(.{ .mesh = &mesh_, .shape = shape, .expected_num_partitions = @intCast(count), .expected_num_replicas = 1 }); + } +} + +test "suggest reserves fixed dimensions across physical axes before automatic allocation" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 4, 3 }, .{ .mesh = .torus }); + const logical = try LogicalMesh.init(.{ .model, .data }, &.{ .high_bandwidth, .low_bandwidth }, &.{ -1, 6 }); + const mesh_ = try Mesh.init("split", &physical, logical, .suggest(logical, &physical)); + try std.testing.expectEqual(2, mesh_.numPartitionsForLogicalAxis(.model)); + try std.testing.expectEqual(6, mesh_.numPartitionsForLogicalAxis(.data)); + try std.testing.expectEqual(1, mesh_.numReplicas()); + + const fixed = try LogicalMesh.init(.{ .data, .model }, &.{ .low_bandwidth, .high_bandwidth }, &.{ 3, 4 }); + const fixed_mesh = try Mesh.init("fixed", &physical, fixed, .suggest(fixed, &physical)); + try std.testing.expectEqual(12, fixed_mesh.numPartitions()); + const impossible = try LogicalMesh.init(.{ .data, .model }, &.{ .low_bandwidth, .high_bandwidth }, &.{ 5, -1 }); + try std.testing.expectError(error.IncompatibleLogicalDimension, Mesh.init("impossible", &physical, impossible, .suggest(impossible, &physical))); + const replica_logical = try LogicalMesh.init(.{ .data, .model }, &.{ .low_bandwidth, .high_bandwidth }, &.{ 1, 1 }); + const replica_mesh = try Mesh.init("replicated", &physical, replica_logical, .suggest(replica_logical, &physical)); + try std.testing.expectEqual(12, replica_mesh.numReplicas()); +} + test "mesh: suggest strategy realization" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); @@ -2470,10 +2592,12 @@ test "mesh: suggest strategy realization" { const mesh: Mesh = try .init("suggested_mesh", &physical, logical, strategy); try runner.run(.{ + .expected_num_partitions = 4, + .expected_num_replicas = 2, .mesh = &mesh, .shape = Shape.init(.{ .batch = 4, .model = 4 }, .f32) .withPartitioning(&mesh, .{ .batch = .batch, .model = .model }), - .expected_sdy = "#sdy.sharding<@suggested_mesh, [{\"link_z\"}, {\"link_x\"}], replicated={\"link_y\"}>", + .expected_sdy = "#sdy.sharding<@suggested_mesh, [{\"batch\"}, {\"model\"}], replicated={\"__replica_0\"}>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{ .{ 0, 2 }, .{ 0, 2 } } }, // x0, y0, z0 .{ .device_id = 1, .slices = &.{ .{ 2, 2 }, .{ 0, 2 } } }, // x0, y0, z1 (batch split) @@ -2487,7 +2611,7 @@ test "mesh: suggest strategy realization" { }); } -test "mesh: suggest folds logical axes with same intent" { +test "mesh: suggest leaves conflicting logical axes replicated" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const runner: ShardingTest = .init(arena.allocator()); @@ -2503,10 +2627,12 @@ test "mesh: suggest folds logical axes with same intent" { const mesh: Mesh = try .init("fold_mesh", &physical, logical, strategy); try runner.run(.{ + .expected_num_partitions = 4, + .expected_num_replicas = 1, .mesh = &mesh, .shape = Shape.init(.{ .model = 4, .experts = 4 }, .f32) .withPartitioning(&mesh, .{ .model = .model, .experts = .experts }), - .expected_sdy = "#sdy.sharding<@fold_mesh, [{\"link_x\"}, {}]>", + .expected_sdy = "#sdy.sharding<@fold_mesh, [{\"model\"}, {\"experts\"}]>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{ .{ 0, 1 }, .{ 0, 4 } } }, .{ .device_id = 1, .slices = &.{ .{ 1, 1 }, .{ 0, 4 } } }, @@ -2514,6 +2640,20 @@ test "mesh: suggest folds logical axes with same intent" { .{ .device_id = 3, .slices = &.{ .{ 3, 1 }, .{ 0, 4 } } }, }, }); + // Swapping tensor dimensions must not change which logical axis owns the devices. + try runner.run(.{ + .expected_num_partitions = 4, + .expected_num_replicas = 1, + .mesh = &mesh, + .shape = Shape.init(.{ .experts = 4, .model = 4 }, .f32) + .withPartitioning(&mesh, .{ .experts = .experts, .model = .model }), + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{ .{ 0, 4 }, .{ 0, 1 } } }, + .{ .device_id = 1, .slices = &.{ .{ 0, 4 }, .{ 1, 1 } } }, + .{ .device_id = 2, .slices = &.{ .{ 0, 4 }, .{ 2, 1 } } }, + .{ .device_id = 3, .slices = &.{ .{ 0, 4 }, .{ 3, 1 } } }, + }, + }); } test "mesh: multiple physical axes on one logical dimension (folding)" { @@ -2528,9 +2668,11 @@ test "mesh: multiple physical axes on one logical dimension (folding)" { const strategy: Strategy = .parseBindings(.{ .model = .{ .link_x, .link_y } }); const mesh: Mesh = try .init("folded_mesh", &physical, logical, strategy); try runner.run(.{ + .expected_num_partitions = 4, + .expected_num_replicas = 1, .mesh = &mesh, .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(&mesh, .{ .model = .model }), - .expected_sdy = "#sdy.sharding<@folded_mesh, [{\"link_x\", \"link_y\"}]>", + .expected_sdy = "#sdy.sharding<@folded_mesh, [{\"model\"}]>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, .{ .device_id = 1, .slices = &.{.{ 4, 4 }} }, @@ -2546,9 +2688,11 @@ test "mesh: multiple physical axes on one logical dimension (folding)" { const strategy: Strategy = .parseBindings(.{ .model = .{ .link_y, .link_x } }); const mesh: Mesh = try .init("folded_mesh", &physical, logical, strategy); try runner.run(.{ + .expected_num_partitions = 4, + .expected_num_replicas = 1, .mesh = &mesh, .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(&mesh, .{ .model = .model }), - .expected_sdy = "#sdy.sharding<@folded_mesh, [{\"link_y\", \"link_x\"}]>", + .expected_sdy = "#sdy.sharding<@folded_mesh, [{\"model\"}]>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, .{ .device_id = 1, .slices = &.{.{ 8, 4 }} }, @@ -2572,9 +2716,11 @@ test "mesh: explicit strategy folding" { const mesh: Mesh = try .init("strategy_fold", &physical, logical, strategy); try runner.run(.{ + .expected_num_partitions = 4, + .expected_num_replicas = 2, .mesh = &mesh, .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(&mesh, .{ .model = .model }), - .expected_sdy = "#sdy.sharding<@strategy_fold, [{\"link_x\"}], replicated={\"link_y\"}>", + .expected_sdy = "#sdy.sharding<@strategy_fold, [{\"model\"}], replicated={\"__replica_0\"}>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, .{ .device_id = 1, .slices = &.{.{ 4, 4 }} }, @@ -2600,9 +2746,11 @@ test "mesh: open and replicated dimension mix" { const mesh: Mesh = try .init("mix_mesh", &physical, logical, strategy); try runner.run(.{ + .expected_num_partitions = 4, + .expected_num_replicas = 1, .mesh = &mesh, .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(&mesh, .{ .batch = .open, .model = .replicated }), - .expected_sdy = "#sdy.sharding<@mix_mesh, [{?}, {}], replicated={\"link_x\", \"link_y\"}>", + .expected_sdy = "#sdy.sharding<@mix_mesh, [{?}, {}], replicated={\"batch\", \"model\"}>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, .{ .device_id = 1, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, @@ -2628,11 +2776,13 @@ test "mesh: full 3D cluster mesh" { const mesh: Mesh = try .init("3d_mesh", &physical, logical, strategy); try runner.run(.{ + .expected_num_partitions = 8, + .expected_num_replicas = 1, .mesh = &mesh, // Note: .context and .model are swapped compared to .link_x and .link_z .shape = Shape.init(.{ .batch = 4, .context = 4, .model = 4 }, .f32) .withPartitioning(&mesh, .{ .batch = .batch, .model = .model, .context = .context }), - .expected_sdy = "#sdy.sharding<@\"3d_mesh\", [{\"link_x\"}, {\"link_z\"}, {\"link_y\"}]>", + .expected_sdy = "#sdy.sharding<@\"3d_mesh\", [{\"batch\"}, {\"context\"}, {\"model\"}]>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{ .{ 0, 2 }, .{ 0, 2 }, .{ 0, 2 } } }, .{ .device_id = 1, .slices = &.{ .{ 0, 2 }, .{ 2, 2 }, .{ 0, 2 } } }, @@ -2666,6 +2816,36 @@ test "mesh: num partitions for logical axis" { try std.testing.expectEqual(1, mesh.numPartitionsForLogicalAxis(.batch)); } +test "mesh: size-one and unbound logical axes leave all devices replicated" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 1, 6 }, .{ .mesh = .torus }); + const mesh_ = try Mesh.init("replicas", &physical, .mesh(.{ .data = .balanced, .model = .balanced }), .parseBindings(.{ .data = .link_x })); + try runner.run(.{ + .mesh = &mesh_, + .shape = Shape.init(.{ .batch = 12 }, .f32).withPartitioning(&mesh_, .{ .batch = .data }), + .expected_num_partitions = 1, + .expected_num_replicas = 6, + }); +} + +test "mesh: logical partitions and replicas can have different sizes" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 2, 3, 2 }, .{ .mesh = .torus }); + var strategy: Strategy = .parseBindings(.{ .model = .link_x }); + strategy.addFold(.link_x, &.{ .link_z, .link_x }); + const mesh_ = try Mesh.init("folded", &physical, .mesh(.{ .model = .balanced }), strategy); + try runner.run(.{ + .mesh = &mesh_, + .shape = Shape.init(.{ .model = 8 }, .f32).withPartitioning(&mesh_, .{ .model = .model }), + .expected_num_partitions = 4, + .expected_num_replicas = 3, + }); +} + test "localized shapes do not retain global sharding" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); @@ -2692,3 +2872,329 @@ test "GSPMD tiles use tensor axis order and logical partition indices" { const ids = try mesh_.gspmdTileAssignment(arena.allocator(), shape); try std.testing.expectEqualSlices(u32, &.{ 0, 2, 4, 6, 1, 3, 5, 7 }, ids); } + +test "DeviceGrid stores aligned logical coordinates in canonical order" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); + for (physical.devices_in_canonical_order, 0..) |*device, i| device.id = @intCast(3 - i); + const mesh_ = try Mesh.init("grid", &physical, .mesh(.{ .model = .balanced, .data = .balanced }), .parseBindings(.{ .model = .link_y, .data = .link_x })); + const expected = [_][4]u8{ .{ 0, 0, 0, 0 }, .{ 1, 0, 0, 0 }, .{ 0, 1, 0, 0 }, .{ 1, 1, 0, 0 } }; + try std.testing.expectEqual(@as(usize, 0), @intFromPtr(&mesh_.grid.logical_coords) % 16); + for (mesh_.grid.logical_coords[0..mesh_.grid.device_count], expected, 0..) |coords, want, i| { + try std.testing.expectEqual(@intFromPtr(&mesh_.grid.logical_coords) + 4 * i, @intFromPtr(&mesh_.grid.logical_coords[i])); + const actual: [4]u8 = coords; + try std.testing.expectEqualSlices(u8, &want, &actual); + } + try std.testing.expectEqual(@as(usize, 4), @sizeOf(DeviceGrid.Coordinates)); + const shape = Shape.init(.{ .m = 16, .n = 24 }, .f32).withPartitioning(&mesh_, .{ .m = .model, .n = .data }); + const placement = try Sharding.Placement.init(&shape); + const offsets = [_]usize{ 0, 768, 48, 816 }; + var data: [16 * 24 * 4 + 16]u8 = undefined; + var slice = Slice.init(shape, data[16..]); + slice.bytes = &data; + slice.offset_bytes = 16; + const base_ptr = placement.basePtr(slice); + try std.testing.expectEqual(@intFromPtr(&data) + 16, @intFromPtr(base_ptr)); + for (offsets, 0..) |offset, i| { + try std.testing.expectEqual(offset, placement.byteOffset(i)); + try std.testing.expectEqual(@intFromPtr(&data) + 16 + offset, @intFromPtr(placement.shardPtr(i, base_ptr))); + } +} + +test "partial bindings share physical axes and combine portions in order" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + const runner: ShardingTest = .init(allocator); + const physical = try runner.physical(.{ 4, 6 }, .{ .mesh = .torus }); + const logical = try LogicalMesh.init(.{ .model, .data }, &.{ .balanced, .low_bandwidth }, &.{ -1, -1 }); + var strategy: Strategy = .init; + strategy.addPartialBinding(.model, .link_y, 2, 3); + strategy.addPartialBinding(.model, .link_x, 1, 2); + strategy.addPartialBinding(.data, .link_x, 2, 2); + strategy.addPartialBinding(.data, .link_y, 1, 2); + const mesh_ = try Mesh.init("partial", &physical, logical, strategy); + try std.testing.expectEqual(6, mesh_.numPartitionsForLogicalAxis(.model)); + try std.testing.expectEqual(4, mesh_.numPartitionsForLogicalAxis(.data)); + try std.testing.expectEqual(1, mesh_.numReplicas()); + for (physical.devices_in_canonical_order, 0..) |device, i| { + const coords: [MAX_MESH_RANK]u8 = mesh_.grid.logical_coords[i]; + try std.testing.expectEqual((device.coords[1] / 2) * 2 + device.coords[0] % 2, coords[0]); + try std.testing.expectEqual((device.coords[0] / 2) * 2 + device.coords[1] % 2, coords[1]); + } + const formatted = try std.fmt.allocPrint(allocator, "{f}", .{mesh_}); + try std.testing.expect(std.mem.indexOf(u8, formatted, "model (balanced) -> (link_y / 2) * 2 + (link_x % 2)") != null); + const shape = Shape.init(.{ .m = 12, .d = 8 }, .f32).withPartitioning(&mesh_, .{ .m = .model, .d = .data }); + try runner.run(.{ .mesh = &mesh_, .shape = shape, .expected_num_partitions = 24, .expected_num_replicas = 1 }); +} + +test "fixed logical dimensions constrain whole partial and folded bindings" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 4, 6 }, .{ .mesh = .torus }); + const logical = try LogicalMesh.init(.{ .model, .data, .unused }, &.{ .balanced, .low_bandwidth, .balanced }, &.{ 2, -1, 1 }); + var strategy: Strategy = .parseBindings(.{ .model = .link_x, .data = .link_y }); + const mesh_ = try Mesh.init("constrained", &physical, logical, strategy); + try std.testing.expectEqual(2, mesh_.numPartitionsForLogicalAxis(.model)); + try std.testing.expectEqual(6, mesh_.numPartitionsForLogicalAxis(.data)); + try std.testing.expectEqual(1, mesh_.numPartitionsForLogicalAxis(.unused)); + try std.testing.expectEqual(2, mesh_.numReplicas()); + for (physical.devices_in_canonical_order, 0..) |device, i| { + try std.testing.expectEqual(device.coords[0] % 2, mesh_.grid.logical_coords[i][0]); + } + + const fixed = try LogicalMesh.init(.{.model}, &.{.balanced}, &.{2}); + strategy = .init; + strategy.addPartialBinding(.model, .link_x, 1, 4); + const partial = try Mesh.init("partial", &physical, fixed, strategy); + try std.testing.expectEqual(2, partial.numPartitions()); + try std.testing.expectEqual(12, partial.numReplicas()); + + strategy = .parseBindings(.{ .model = .link_x }); + strategy.addFold(.link_x, &.{ .link_y, .link_x }); + const folded = try Mesh.init("folded", &physical, try LogicalMesh.init(.{.model}, &.{.balanced}, &.{8}), strategy); + try std.testing.expectEqual(8, folded.numPartitions()); + try std.testing.expectEqual(3, folded.numReplicas()); + for (physical.devices_in_canonical_order, 0..) |device, i| { + try std.testing.expectEqual((device.coords[1] * 4 + device.coords[0]) % 8, folded.grid.logical_coords[i][0]); + } + for ([_]i32{ 5, 32 }) |dimension| { + try std.testing.expectError(error.IncompatibleLogicalDimension, Mesh.init("invalid", &physical, try LogicalMesh.init(.{.model}, &.{.balanced}, &.{dimension}), strategy)); + } +} + +test "partial bindings reject invalid portions and correlated logical axes" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{8}, .{ .mesh = .torus }); + for ([_][2]u32{ .{ 0, 2 }, .{ 1, 0 }, .{ 3, 2 }, .{ 1, 3 }, .{ 4, 4 }, .{ 1, std.math.maxInt(u32) } }) |portion| { + var strategy: Strategy = .init; + strategy.addPartialBinding(.model, .link_x, portion[0], portion[1]); + try std.testing.expectError(error.InvalidPartialBinding, Mesh.init("invalid", &physical, .mesh(.{ .model = .balanced }), strategy)); + } + var strategy: Strategy = .init; + strategy.addPartialBinding(.model, .link_y, 1, 2); + try std.testing.expectError(error.InvalidPhysicalAxis, Mesh.init("invalid", &physical, .mesh(.{ .model = .balanced }), strategy)); + strategy = .init; + strategy.addPartialBinding(.model, .link_x, 1, 2); + strategy.addPartialBinding(.data, .link_x, 1, 2); + // Four claimed partitions fit on eight devices, but only two coordinates occur. + try std.testing.expectError(error.OverlappingLogicalAxes, Mesh.init("overlap", &physical, .mesh(.{ .model = .balanced, .data = .balanced }), strategy)); +} + +test "DeviceGrid resolves folds once in strategy order" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + const runner: ShardingTest = .init(allocator); + const physical = try runner.physical(.{ 2, 3, 2 }, .{ .mesh = .torus }); + for (physical.devices_in_canonical_order, 0..) |*device, i| device.id = @intCast(11 - i); + var strategy: Strategy = .parseBindings(.{ .model = .{ .link_x, .link_z } }); + strategy.addFold(.link_x, &.{ .link_z, .link_x }); + const mesh_ = try Mesh.init("folded", &physical, .mesh(.{ .model = .balanced }), strategy); + strategy.folding = .empty; + + try std.testing.expectEqual(4, mesh_.numPartitions()); + try std.testing.expectEqual(3, mesh_.numReplicas()); + const expected = [_]u8{ 0, 2, 0, 2, 0, 2, 1, 3, 1, 3, 1, 3 }; + for (expected, 0..) |coord, i| { + try std.testing.expectEqual(coord, mesh_.grid.logical_coords[i][0]); + } + const formatted = try std.fmt.allocPrint(allocator, "{f}", .{mesh_}); + try std.testing.expect(std.mem.indexOf(u8, formatted, "model (balanced) -> link_z * 2 + link_x\n") != null); + try std.testing.expect(std.mem.indexOf(u8, formatted, "model=0,1,2,3 -> devices=[11,9,7],[5,3,1],[10,8,6],[4,2,0]") != null); +} + +test "DeviceGrid validates fold sources and overlapping folded axes" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); + var strategy: Strategy = .parseBindings(.{ .model = .link_x }); + strategy.addFold(.link_y, &.{ .link_y, .link_z }); + try std.testing.expectError(error.InvalidPhysicalAxis, Mesh.init("invalid", &physical, .mesh(.{ .model = .balanced }), strategy)); + + strategy = .parseBindings(.{ .model = .link_x, .data = .link_y }); + strategy.addFold(.link_x, &.{ .link_x, .link_y }); + try std.testing.expectError(error.OverlappingLogicalAxes, Mesh.init("overlap", &physical, .mesh(.{ .model = .balanced, .data = .balanced }), strategy)); +} + +test "Placement owns dimensions and byte steps across copies" { + const Helper = struct { + fn create() !Placement { + const shape = Shape.init(.{ .m = 3, .n = 5 }, .f32); + return Placement.init(&shape); + } + }; + var original = try Helper.create(); + original.byte_steps[0] = 60; + const copied = original; + original.local_dims.buffer[0] = 99; + original.byte_steps[0] = 100; + try std.testing.expectEqualSlices(i64, &.{ 3, 5 }, copied.local_dims.constSlice()); + try std.testing.expectEqualSlices(u64, &.{ 60, 0, 0, 0 }, &copied.byte_steps); + const source = Shape.init(.{ .m = 3, .n = 5 }, .f32); + try std.testing.expect(copied.localShape(source).eqlWithTags(source)); + var data: [3 * 5 * 4]u8 = undefined; + try std.testing.expectEqual(@intFromPtr(&data), @intFromPtr(copied.basePtr(.init(source, &data)))); +} + +test "Placement byte steps and dot product exceed 32 bits" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{Platform.MAX_NUM_DEVICES}, .{ .mesh = .torus }); + const mesh_ = try Mesh.init("wide", &physical, .mesh(.{ .model = .balanced }), .parseBindings(.{ .model = .link_x })); + const shape = Shape.init(.{ .m = 2 * Platform.MAX_NUM_DEVICES, .n = 1 << 30 }, .f32).withPartitioning(&mesh_, .{ .m = .model }); + const placement = try Sharding.Placement.init(&shape); + try std.testing.expectEqual(@as(usize, 1) << 33, placement.byteOffset(1)); + const last_device = Platform.MAX_NUM_DEVICES - 1; + const coords: [MAX_MESH_RANK]u8 = mesh_.grid.logical_coords[last_device]; + try std.testing.expectEqual(@as(u8, @intCast(last_device)), coords[0]); + try std.testing.expectEqual(@as(usize, last_device) << 33, placement.byteOffset(last_device)); +} + +test "DeviceGrid rejects overlapping logical axes" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + try std.testing.expectError(error.OverlappingLogicalAxes, Mesh.init("overlap", &physical, .mesh(.{ .model = .balanced, .data = .balanced }), .parseBindings(.{ .model = .{ .link_x, .link_y }, .data = .{ .link_y, .link_z } }))); +} + +test "Shardy meshes preserve logical order and unused physical replicas" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + const runner: ShardingTest = .init(allocator); + const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + const mesh_ = try Mesh.init("logical", &physical, .mesh(.{ .batch = .balanced, .model = .balanced }), .parseBindings(.{ .batch = .link_z, .model = .link_x })); + const registry: *mlir.DialectRegistry = try .init(); + defer registry.deinit(); + registry.registerDialect("sdy"); + const ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + const mesh_attr = try mesh_.sdyMeshAttr(allocator); + _ = try mlir.Attribute.parse(ctx, mesh_attr); + try std.testing.expectEqualStrings( + "#sdy.mesh<[\"batch\"=2, \"model\"=2, \"__replica_0\"=2], device_ids=[0, 2, 4, 6, 1, 3, 5, 7]>", + mesh_attr, + ); +} + +test "Shardy replica axis avoids logical name collisions" { + const mesh_ = Mesh.initTest(.collision, .{ .__replica_0, .__replica_1 }); + try std.testing.expectEqualStrings("__replica_2", mesh_.sdyReplicaAxis()); +} + +test "GSPMD attributes use logical tile sizes and replica groups" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + const ctx: *mlir.Context = try .init(.{ .threading = false }); + defer ctx.deinit(); + const runner: ShardingTest = .init(allocator); + const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + for (physical.devices_in_canonical_order, 0..) |*device, i| device.id = @intCast(7 - i); + const mesh_ = try Mesh.init("tiles", &physical, .mesh(.{ .batch = .balanced, .model = .balanced, .unused = .balanced }), .parseBindings(.{ .batch = .link_z, .model = .link_x })); + + const Case = struct { shape: Shape, expected: []const u8 }; + const cases = [_]Case{ + .{ + .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .batch, .model = .model }), + .expected = "{devices=[2,2,2]0,2,4,6,1,3,5,7 last_tile_dim_replicate}", + }, + .{ + .shape = Shape.init(.{ .model = 8, .batch = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .batch, .model = .model }), + .expected = "{devices=[2,2,2]0,2,1,3,4,6,5,7 last_tile_dim_replicate}", + }, + .{ + .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .batch }), + .expected = "{devices=[2,1,4]0,2,4,6,1,3,5,7 last_tile_dim_replicate}", + }, + .{ + .shape = Shape.init(.{ .batch = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .unused }), + .expected = "{replicated}", + }, + .{ + .shape = Shape.init(.{ .batch = 8 }, .f32), + .expected = "{replicated}", + }, + .{ + .shape = Shape.init(.{}, .f32), + .expected = "{replicated}", + }, + }; + for (cases) |case| { + const actual = try mesh_.gspmdShardingAttrForShape(allocator, ctx, case.shape); + try std.testing.expect(actual.eql(.string(ctx, case.expected))); + } +} + +test "GSPMD attributes preserve folded logical axis order" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + const ctx: *mlir.Context = try .init(.{ .threading = false }); + defer ctx.deinit(); + const runner: ShardingTest = .init(allocator); + const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + var strategy: Strategy = .parseBindings(.{ .model = .link_x, .batch = .link_y }); + strategy.addFold(.link_x, &.{ .link_z, .link_x }); + const mesh_ = try Mesh.init("folded", &physical, .mesh(.{ .model = .balanced, .batch = .balanced }), strategy); + const sharded = Shape.init(.{ .model = 8, .batch = 8 }, .f32).withPartitioning(&mesh_, .{ .model = .model, .batch = .batch }); + const actual = try mesh_.gspmdShardingAttrForShape(allocator, ctx, sharded); + try std.testing.expect(actual.eql(.string(ctx, "{devices=[4,2]0,2,4,6,1,3,5,7}"))); + + const partial = Shape.init(.{ .model = 8 }, .f32).withPartitioning(&mesh_, .{ .model = .model }); + const partial_attr = try mesh_.gspmdShardingAttrForShape(allocator, ctx, partial); + try std.testing.expect(partial_attr.eql(.string(ctx, "{devices=[4,2]0,2,4,6,1,3,5,7 last_tile_dim_replicate}"))); +} + +test "GSPMD executes a reduction across a folded logical axis" { + if (comptime !Target.cpu.isEnabled()) return error.SkipZigTest; + + const zml = @import("zml.zig"); + const io = std.testing.io; + const allocator = std.testing.allocator; + + const platform = try Platform.init(allocator, io, .cpu, .{ + .cpu = .{ .device_count = 8 }, + .physical_mesh = .{ .custom = PhysicalMesh.torus3d }, + }); + defer platform.deinit(allocator, io); + var strategy: Strategy = .parseBindings(.{ .model = .link_x, .data = .link_y }); + strategy.addFold(.link_x, &.{ .link_z, .link_x }); + const mesh_ = try platform.registerMeshWithStrategy("logical", .mesh(.{ .data = .balanced, .model = .balanced }), strategy); + + const Functions = struct { + fn forward(input: zml.Tensor) zml.Tensor { + return input.sum(.m).squeeze(.m).withPartitioning(input.shape()._sharding.mesh.?, .{ .n = .data }); + } + }; + const input = zml.Tensor.init(.{ .m = 8, .n = 4 }, .i32).withPartitioning(mesh_, .{ .m = .model, .n = .data }); + const exe = try platform.compileFn(allocator, io, Functions.forward, .{input}, .{ + .meshes = &.{mesh_}, + .partitioner = .gspmd, + }); + defer exe.deinit(); + var values: [8][4]i32 = undefined; + var expected: [4]i32 = @splat(0); + for (&values, 0..) |*row, m| { + for (row, 0..) |*value, n| { + value.* = @intCast(10 * m + n); + expected[n] += value.*; + } + } + + var input_buffer = try zml.Buffer.fromBytes(io, platform, input.shape(), std.mem.asBytes(&values)); + defer input_buffer.deinit(); + var output = try exe.eval(allocator, io, .{input_buffer}); + defer output.deinit(); + try std.testing.expectEqual(expected, try output.getValue([4]i32, io)); +} diff --git a/zml/buffer.zig b/zml/buffer.zig index b8ab1c8d..8714d80f 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -127,7 +127,7 @@ pub const Buffer = struct { } pub fn format(self: Buffer, writer: *std.Io.Writer) !void { - const pl = Sharding.Placement.init(self._shape) catch |err| { + const pl = Sharding.Placement.init(&self._shape) catch |err| { return try writer.print("Buffer({f}) with invalid sharding {} ({})", .{ self._shape, self._shape._sharding, err }); }; try writer.print("{f}", .{pl}); @@ -141,45 +141,54 @@ pub const Buffer = struct { data_: []const u8, opts: FromOptions, ) !Buffer { - var buffer: Buffer = .{ - ._platform = platform, - ._shape = shape_, - ._shards = .empty, + var pjrt_shards: Shards = .empty; + var packed_shape = shape_.packedShape(); + if (packed_shape._sharding.mesh == null) packed_shape._sharding.mesh = platform.replicated_mesh; + const byte_strides = packed_shape.computeByteStrides(); + const placement = Sharding.Placement.initWithByteStrides(&packed_shape, byte_strides.constSlice()) catch |err| { + @branchHint(.cold); + std.debug.panic("Buffer shape {f} is incompatible with its mesh: {t}", .{ packed_shape, err }); }; - const metadata: PjrtBufferMetadata = .init(platform, shape_); + const ty = pjrtx.bufferTypeFromDtype(packed_shape.dtype()); + const layout = platform.defaultMemoryLayout(placement.local_dims.constSlice(), packed_shape.dtype()); - errdefer for (buffer._shards.slice()) |shard| { + errdefer for (pjrt_shards.slice()) |shard| { shard.deinit(platform.pjrt_api); }; stdx.debug.assert(platform.devices[0].memory(opts.memory) != null, "Device doesn't have {} memory", .{opts.memory}); - const slice = Slice.init(metadata.shape, data_); + const byte_size: usize = if (packed_shape.rank() == 0) + packed_shape.dtype().sizeOf() + else + @intCast(packed_shape.dim(0) * byte_strides.get(0)); + stdx.debug.assert(data_.len == byte_size, "Expected data length {}, got {}", .{ byte_size, data_.len }); + const base_ptr = data_.ptr; - for (platform.physical_mesh.devices_in_canonical_order) |device| { + for (platform.physical_mesh.devices_in_canonical_order, 0..) |device, device_index| { const memory = platform.devices[device.id].memory(opts.memory).?; const args: pjrt.Client.BufferFromHostBufferArgs = .{ // Change for each device - .data = metadata.placement.shardPtr(device.coords, slice), + .data = placement.shardPtr(device_index, base_ptr), .dst = .{ .memory = memory.pjrt_memory }, // Constant across devices - .layout = metadata.layout, - .dims = metadata.placement.shape.dims(), - .buffer_type = metadata.ty, - .byte_strides = slice.byte_strides.constSlice(), + .layout = layout, + .dims = placement.local_dims.constSlice(), + .buffer_type = ty, + .byte_strides = byte_strides.constSlice(), .host_buffer_semantics = .ImmutableUntilTransferCompletes, }; const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); if (event) |ev| ev.deinit(platform.pjrt_api); - buffer._shards.appendAssumeCapacity(pjrt_buffer); + pjrt_shards.appendAssumeCapacity(pjrt_buffer); } if (opts.wait) { - try buffer.await(io); + try awaitShards(pjrt_shards.constSlice(), platform, io); } - return buffer; + return .{ ._platform = platform, ._shape = shape_, ._shards = pjrt_shards }; } /// Copies the given Zig bytes to the accelerator memory and @@ -209,10 +218,14 @@ pub const Buffer = struct { } pub fn await(self: Buffer, io: std.Io) !void { - for (self._shards.constSlice()) |buffer| { - const ev = buffer.readyEvent(self._platform.pjrt_api); - defer ev.deinit(self._platform.pjrt_api); - try ev.await(self._platform.pjrt_api, io); + try awaitShards(self._shards.constSlice(), self._platform, io); + } + + fn awaitShards(pjrt_shards: []const *pjrt.Buffer, platform: *const Platform, io: std.Io) !void { + for (pjrt_shards) |shard| { + const event = shard.readyEvent(platform.pjrt_api); + defer event.deinit(platform.pjrt_api); + try event.await(platform.pjrt_api, io); } } @@ -244,7 +257,7 @@ pub const Buffer = struct { .dst = .{ .memory = memory.pjrt_memory }, // Constant across devices .layout = metadata.layout, - .dims = metadata.placement.shape.dims(), + .dims = metadata.placement.local_dims.constSlice(), .element_type = metadata.ty, }; @@ -282,12 +295,12 @@ pub const Buffer = struct { stdx.debug.assert(self._shape.eql(slice.shape), "Buffer shape {f} doesn't match destination slice {f}", .{ self._shape, slice.shape }); const placement = placementOrPanic(self._platform, self._shape); - for (self._platform.physical_mesh.devices_in_canonical_order, 0..) |device, shard_index| { + for (0..self._platform.physical_mesh.devices_in_canonical_order.len) |shard_index| { // TODO: handle replicated information, we shouldn't iterate over all the devices unless needed - const sub_slice = placement.shardSlice(device.coords, slice); + const sub_slice = placement.shardSlice(shard_index, slice); if (!sub_slice.isContiguous()) return error.NonContiguousShardRead; - const size_bytes = placement.shape.byteSize(); + const size_bytes = placement.localShape(self._shape).byteSize(); const destination = sub_slice.data()[0..size_bytes]; const maybe_event = try self._shards.get(shard_index).toHostBuffer(self._platform.pjrt_api, destination); @@ -306,11 +319,11 @@ pub const Buffer = struct { const placement = placementOrPanic(self._platform, self._shape); - var shard_slice = try Slice.alloc(allocator, placement.shape); + var shard_slice = try Slice.alloc(allocator, placement.localShape(self._shape)); defer shard_slice.free(allocator); - for (self._platform.physical_mesh.devices_in_canonical_order, 0..) |device, shard_index| { - const sub_slice = placement.shardSlice(device.coords, slice); + for (0..self._platform.physical_mesh.devices_in_canonical_order.len) |shard_index| { + const sub_slice = placement.shardSlice(shard_index, slice); const maybe_event = try self._shards.get(shard_index).toHostBuffer(self._platform.pjrt_api, shard_slice.data()); if (maybe_event) |event| { defer event.deinit(self._platform.pjrt_api); @@ -330,7 +343,7 @@ pub const Buffer = struct { /// `shard_byte_size` can be up to `self.shape().byteSize()` when the buffer is fully replicated. pub fn byteSize(self: Buffer) usize { const placement = placementOrPanic(self._platform, self._shape); - return placement.shape.byteSize() * self._shards.len; + return placement.localShape(self._shape).byteSize() * self._shards.len; } pub fn opaqueDevicePtr(self: Buffer, device_id: usize) *anyopaque { @@ -428,7 +441,7 @@ pub const PjrtBufferMetadata = struct { .shape = packed_shape, .ty = pjrtx.bufferTypeFromDtype(packed_shape.dtype()), .placement = placement, - .layout = platform.defaultMemoryLayout(placement.shape.dims(), packed_shape.dtype()), + .layout = platform.defaultMemoryLayout(placement.local_dims.constSlice(), packed_shape.dtype()), }; } }; @@ -436,7 +449,7 @@ pub const PjrtBufferMetadata = struct { fn placementOrPanic(platform: *const Platform, shape: Shape) Sharding.Placement { var sh = shape; if (sh._sharding.mesh == null) sh._sharding.mesh = platform.replicated_mesh; - return Sharding.Placement.init(sh) catch |err| { + return Sharding.Placement.init(&sh) catch |err| { @branchHint(.cold); std.debug.panic("Buffer shape {f} is incompatible with its mesh: {t}", .{ shape, err }); }; diff --git a/zml/buffer/HostAccessible.zig b/zml/buffer/HostAccessible.zig index 644704c2..b3e82a9c 100644 --- a/zml/buffer/HostAccessible.zig +++ b/zml/buffer/HostAccessible.zig @@ -260,8 +260,8 @@ pub fn init( for (devices, 0..) |device, shard_index| { const memory = platform.devices[device.id].memory(.host_pinned).?.pjrt_memory; - const origin: Coordinates = .ofShard(&metadata.placement, device); - const shared_region = if (can_share_allocations) allocations.findRegion(origin, metadata.placement.shape) else null; + const origin: Coordinates = .ofShard(&metadata.placement, shard_index, metadata.shape); + const shared_region = if (can_share_allocations) allocations.findRegion(origin, metadata.placement.localShape(metadata.shape)) else null; if (shared_region) |region_index| { // Devices sharing allocations add no replicas: the region has a single entry. @@ -273,7 +273,7 @@ pub fn init( } buffer._shards.appendAssumeCapacity(try platform.pjrt_client.createViewOfDeviceBuffer(api, .{ .device_buffer_ptr = try existing.handle.opaqueDeviceMemoryDataPointer(api), - .dims = metadata.placement.shape.dims(), + .dims = metadata.placement.local_dims.constSlice(), .element_type = metadata.ty, .layout = try existing.handle.memoryLayout(api), // The backing host allocation is accessible across devices @@ -284,14 +284,14 @@ pub fn init( })); } else { const handle = try platform.pjrt_client.createUninitializedBuffer(api, .{ - .dims = metadata.placement.shape.dims(), + .dims = metadata.placement.local_dims.constSlice(), .element_type = metadata.ty, .layout = metadata.layout, .dst = .{ .memory = memory }, }); buffer._shards.appendAssumeCapacity(handle); // The pointer is read once the creation completes, after `await` below. - allocations.add(metadata.placement.shape, .{ + allocations.add(metadata.placement.localShape(metadata.shape), .{ .origin = origin, .ptr = undefined, .shard_index = shard_index, @@ -305,7 +305,7 @@ pub fn init( return .{ .buffer = buffer, - .mapping = try .create(allocator, metadata.shape, metadata.placement.shape, metadata.layout, allocations), + .mapping = try .create(allocator, metadata.shape, metadata.placement.localShape(metadata.shape), metadata.layout, allocations), }; } @@ -708,9 +708,10 @@ const Coordinates = struct { const zero: Coordinates = .{ .axes = @splat(0) }; /// The global coordinates of the first element of a device's shard. - fn ofShard(placement: *const Sharding.Placement, device: Sharding.Device) Coordinates { + fn ofShard(placement: *const Sharding.Placement, device_index: usize, shape: Shape) Coordinates { + // TODO: derive shard coordinates directly from DeviceGrid instead of reconstructing slices. var result: Coordinates = .zero; - for (placement.slices(device.coords).constSlice(), 0..) |slice, axis| result.axes[axis] = @intCast(slice.start); + for (placement.slices(device_index, shape).constSlice(), 0..) |slice, axis| result.axes[axis] = @intCast(slice.start); return result; } @@ -1830,7 +1831,7 @@ test "HostAccessible devices holding a shard read one allocation in data-paralle const io = std.testing.io; // Two data-parallel groups of two tensor-parallel devices. const platform = Platform.auto(allocator, io, .{ - .physical_mesh = .{ .custom = buildMesh2x2 }, + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, .cpu = .{ .device_count = 4 }, }) catch return error.SkipZigTest; defer platform.deinit(allocator, io); @@ -1854,50 +1855,28 @@ fn expectDevicesReadShards(platform: *const Platform, shape_: Shape) !void { const api = platform.pjrt_api; const pinned = try HostAccessible.init(allocator, io, platform, shape_, .undonatable); defer pinned.deinit(allocator); - const placement: Sharding.Placement = try .init(shape_); + const placement: Sharding.Placement = try Sharding.Placement.init(&shape_); const expected_allocations = switch (platform.target) { - .cpu, .cuda, .rocm => shape_.count() / placement.shape.count(), + .cpu, .cuda, .rocm => shape_.count() / placement.localShape(shape_).count(), else => platform.devices.len, }; try std.testing.expectEqual(expected_allocations, pinned.mapping.allocations.entries.len); pinned.view(i32).fillIota(0, 1); const row_size: usize = @intCast(shape_.dim(.d)); - const values = try allocator.alloc(i32, placement.shape.count()); + const values = try allocator.alloc(i32, placement.localShape(shape_).count()); defer allocator.free(values); - for (pinned.buffer._shards.constSlice(), platform.physical_mesh.devices_in_canonical_order) |shard, device| { + for (pinned.buffer._shards.constSlice(), 0..) |shard, device_index| { @memset(values, -1); if (try shard.toHostBuffer(api, std.mem.sliceAsBytes(values))) |event| { defer event.deinit(api); try event.await(api, io); } - const first_row = Coordinates.ofShard(&placement, device).axes[0]; + const first_row = Coordinates.ofShard(&placement, device_index, shape_).axes[0]; for (values, 0..) |value, i| try std.testing.expectEqual(@as(i32, @intCast(first_row * row_size + i)), value); } } -/// Like `zml/io.zig`'s, a 2x2 torus over the first four devices. -fn buildMesh2x2( - allocator: std.mem.Allocator, - target: @import("../platform.zig").Target, - devices: []const @import("../platform.zig").Device, -) !Sharding.PhysicalMesh { - if (!builtin.is_test) @compileError("buildMesh2x2 is only available in tests"); - - if (devices.len < 4) return error.NotEnoughDevices; - const topology: Sharding.PhysicalMesh.Tree = .axis(.link_x, .{ .mesh = .torus }, &.{ - .axis(.link_y, .{ .mesh = .torus }, &.{ - .device(devices[0]), - .device(devices[1]), - }), - .axis(.link_y, .{ .mesh = .torus }, &.{ - .device(devices[2]), - .device(devices[3]), - }), - }); - return Sharding.PhysicalMesh.fromTree(allocator, target, topology); -} - test "HostAccessible executions read shared inputs on every device" { const zml = @import("../zml.zig"); const allocator = std.testing.allocator; diff --git a/zml/exe.zig b/zml/exe.zig index 1dd1a2f2..85baa7e7 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -25,10 +25,10 @@ pub const Exe = struct { /// Inputs whose buffers are donated to an output (`reuseBuffer`). /// Runners destroy their PJRT handles after the raw call returns. - donated_input_indices: []const usize, + donated_input_indices: []const u32, - num_devices: usize, - num_partitions: i32, + num_devices: u32, + num_partitions: u32, arena: std.heap.ArenaAllocator, @@ -36,8 +36,8 @@ pub const Exe = struct { allocator: std.mem.Allocator, platform: *const Platform, exe: *pjrt.LoadedExecutable, - num_devices: usize, - num_partitions: i32, + num_devices: u32, + num_partitions: u32, input_shapes: []const Shape, output_shapes: []const Shape, input_aliasing: []const ?u32, @@ -51,12 +51,12 @@ pub const Exe = struct { var donated_count: usize = 0; for (input_aliasing) |aliasing| donated_count += @intFromBool(aliasing != null); - const donated_input_indices = try arena.allocator().alloc(usize, donated_count); + const donated_input_indices = try arena.allocator().alloc(u32, donated_count); var donated_index: usize = 0; for (input_aliasing, 0..) |aliasing, input_index| { if (aliasing == null) continue; - donated_input_indices[donated_index] = input_index; + donated_input_indices[donated_index] = @intCast(input_index); donated_index += 1; } @@ -607,7 +607,7 @@ test "Exe raw calls leave donated input handle cleanup to the caller" { const weight: Tensor = .init(.{2}, .i32); const exe = try platform.compileFn(allocator, io, Functions.forward, .{ value, weight }, .{}); defer exe.deinit(); - try std.testing.expectEqualSlices(usize, &.{0}, exe.donated_input_indices); + try std.testing.expectEqualSlices(u32, &.{0}, exe.donated_input_indices); var weight_buffer = try Buffer.fromBytes(io, platform, weight.shape(), std.mem.asBytes(&[2]i32{ 10, 20 })); defer weight_buffer.deinit(); @@ -682,7 +682,7 @@ test "FnExe runner replaces unchanged donated buffers" { const value: Tensor = .init(.{2}, .i32); const exe = try Model.compile(allocator, io, platform, .{}, .{.{ .value = value }}); defer exe.deinit(); - try std.testing.expectEqualSlices(usize, &.{0}, exe.raw.donated_input_indices); + try std.testing.expectEqualSlices(u32, &.{0}, exe.raw.donated_input_indices); var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), std.mem.asBytes(&[2]i32{ 1, 2 })); defer value_buffer.deinit(); diff --git a/zml/io.zig b/zml/io.zig index eb9596b5..43bfdf23 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -980,12 +980,12 @@ const DispatchSpans = struct { mirror_writers: []usize, fn init(allocator: std.mem.Allocator, shape: Shape) !DispatchSpans { - const placement = try Sharding.Placement.init(shape); + const placement = try Sharding.Placement.init(&shape); const ordered_devices = placement.mesh().?.devicesInCanonicalOrder(); var placement_span_count: usize = 0; - for (ordered_devices) |device| { - placement_span_count += placementSpanCount(shape, placement.slices(device.coords).constSlice()); + for (0..ordered_devices.len) |device_index| { + placement_span_count += placementSpanCount(shape, placement.slices(device_index, shape).constSlice()); } var placement_spans: std.ArrayList(PlacementSpan) = try .initCapacity(allocator, placement_span_count); @@ -993,8 +993,8 @@ const DispatchSpans = struct { const byte_strides = shape.computeByteStrides(); - for (ordered_devices, 0..) |device, writer_index| { - appendShardPlacementSpans(&placement_spans, shape, placement.slices(device.coords).constSlice(), byte_strides.constSlice(), writer_index); + for (0..ordered_devices.len) |writer_index| { + appendShardPlacementSpans(&placement_spans, shape, placement.slices(writer_index, shape).constSlice(), byte_strides.constSlice(), writer_index); } std.debug.assert(placement_spans.items.len == placement_span_count); @@ -1199,7 +1199,7 @@ pub const DirectMemoryWriter = struct { ) !DirectMemoryWriter { var placed_shape = shape; if (placed_shape._sharding.mesh == null) placed_shape._sharding.mesh = platform.replicated_mesh; - const placement = try Sharding.Placement.init(placed_shape); + const placement = try Sharding.Placement.init(&placed_shape); const mesh = placement.mesh().?; const ordered_devices = mesh.devicesInCanonicalOrder(); var shard_writers = try allocator.alloc(DirectShardWriter, ordered_devices.len); @@ -1216,7 +1216,7 @@ pub const DirectMemoryWriter = struct { const shard_dma_allocator = dma_allocators[device.id].allocator(); const pjrt_mem = platform.devices[device.id].memory(memory).?; - shard_writers[i] = try .init(shard_dma_allocator, io, pjrt_mem, pool, placement.shape); + shard_writers[i] = try .init(shard_dma_allocator, io, pjrt_mem, pool, placement.localShape(placed_shape)); initialized += 1; pjrt_buffers.appendAssumeCapacity(shard_writers[i].pjrt_buffer); @@ -1446,72 +1446,6 @@ pub const DirectMemoryWriter = struct { } }; -fn buildMesh2( - allocator: std.mem.Allocator, - target: @import("platform.zig").Target, - devices: []const @import("platform.zig").Device, -) !Sharding.PhysicalMesh { - if (devices.len < 2) return error.NotEnoughDevices; - const topology: Sharding.PhysicalMesh.Tree = .axis(.link_x, .{ .mesh = .torus }, &.{ - .device(devices[0]), - .device(devices[1]), - }); - - return Sharding.PhysicalMesh.fromTree(allocator, target, topology); -} - -fn buildMesh2x2( - allocator: std.mem.Allocator, - target: @import("platform.zig").Target, - devices: []const @import("platform.zig").Device, -) !Sharding.PhysicalMesh { - if (devices.len < 4) return error.NotEnoughDevices; - const topology: Sharding.PhysicalMesh.Tree = .axis(.link_x, .{ .mesh = .torus }, &.{ - .axis(.link_y, .{ .mesh = .torus }, &.{ - .device(devices[0]), - .device(devices[1]), - }), - .axis(.link_y, .{ .mesh = .torus }, &.{ - .device(devices[2]), - .device(devices[3]), - }), - }); - - return Sharding.PhysicalMesh.fromTree(allocator, target, topology); -} - -fn buildMesh2x2x2( - allocator: std.mem.Allocator, - target: @import("platform.zig").Target, - devices: []const @import("platform.zig").Device, -) !Sharding.PhysicalMesh { - if (devices.len < 8) return error.NotEnoughDevices; - const topology: Sharding.PhysicalMesh.Tree = .axis(.link_x, .{ .mesh = .torus }, &.{ - .axis(.link_y, .{ .mesh = .torus }, &.{ - .axis(.link_z, .{ .mesh = .torus }, &.{ - .device(devices[0]), - .device(devices[1]), - }), - .axis(.link_z, .{ .mesh = .torus }, &.{ - .device(devices[2]), - .device(devices[3]), - }), - }), - .axis(.link_y, .{ .mesh = .torus }, &.{ - .axis(.link_z, .{ .mesh = .torus }, &.{ - .device(devices[4]), - .device(devices[5]), - }), - .axis(.link_z, .{ .mesh = .torus }, &.{ - .device(devices[6]), - .device(devices[7]), - }), - }), - }); - - return Sharding.PhysicalMesh.fromTree(allocator, target, topology); -} - const DirectMemoryWriterDeviceTest = struct { const WriteMode = enum { stream_remaining, @@ -1539,6 +1473,7 @@ const DirectMemoryWriterDeviceTest = struct { defer platform.deinit(self.allocator, self.io); const mesh: Sharding.Mesh = try .init(scenario.name, &platform.physical_mesh, scenario.logical_mesh, scenario.strategy); + std.log.warn("mesh: {f}", .{mesh}); try self.runDirectMemoryWriter( platform, scenario.shape.withPartitioning(&mesh, partitioning), @@ -1627,7 +1562,7 @@ const DirectMemoryWriterDeviceTest = struct { } }; -test "DirectMemoryWriter: replicated with auto topology" { +test "DirectMemoryWriter: replicated tensor" { const case: DirectMemoryWriterDeviceTest = .{ .allocator = std.testing.allocator, .io = std.testing.io, @@ -1636,7 +1571,7 @@ test "DirectMemoryWriter: replicated with auto topology" { try case.run(.{ .name = "replicated_auto", .create_options = .{ - .physical_mesh = .auto, + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, .cpu = .{ .device_count = 4 }, }, .shape = Shape.init(.{ .rows = 8, .cols = 128 }, .f32), @@ -1654,7 +1589,7 @@ test "DirectMemoryWriter: 1D model split with 2x2 physical mesh" { try case.run(.{ .name = "model_auto", .create_options = .{ - .physical_mesh = .{ .custom = buildMesh2x2 }, + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, .cpu = .{ .device_count = 4 }, }, .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32), @@ -1672,7 +1607,7 @@ test "DirectMemoryWriter: 2D batch/model split with 2x2 physical mesh" { try case.run(.{ .name = "batch_model_2d_torus", .create_options = .{ - .physical_mesh = .{ .custom = buildMesh2x2 }, + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, .cpu = .{ .device_count = 4 }, }, .shape = Shape.init(.{ .batch = 8, .model = 1024 }, .f32), @@ -1693,7 +1628,7 @@ test "DirectMemoryWriter: folded model mesh with 2x2 physical mesh" { try case.run(.{ .name = "model_folded_2d_torus", .create_options = .{ - .physical_mesh = .{ .custom = buildMesh2x2 }, + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, .cpu = .{ .device_count = 4 }, }, .shape = Shape.init(.{ .model = 4096 }, .f32), @@ -1715,7 +1650,7 @@ test "DirectMemoryWriter: writableSliceGreedy with mirrored shards" { try case.run(.{ .name = "model_auto_writable_slice", .create_options = .{ - .physical_mesh = .{ .custom = buildMesh2x2 }, + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, .cpu = .{ .device_count = 4 }, }, .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32), @@ -1736,7 +1671,7 @@ test "DirectMemoryWriter: 3D topology folded model + replicated batch" { try case.run(.{ .name = "topology_3d_folded_model", .create_options = .{ - .physical_mesh = .{ .custom = buildMesh2x2x2 }, + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus3d }, .cpu = .{ .device_count = 8 }, }, .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), @@ -1761,7 +1696,7 @@ test "MemoryWriter can produce a host pinned buffer" { try case.run(.{ .name = "host_pinned", .create_options = .{ - .physical_mesh = .{ .custom = buildMesh2 }, + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus1d }, }, .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), .logical_mesh = .mesh(.{ .model = .high_bandwidth }), diff --git a/zml/ops.zig b/zml/ops.zig index 555537f5..54eedda9 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -74,7 +74,7 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@T else => @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"), }; - const num_devices = ctx.meshes[0].numPartitions(); + const num_devices = ctx.meshes[0].numDevices(); if (num_devices <= 1) return inputs; const reducer_block = b: { diff --git a/zml/ops/manual_computation.zig b/zml/ops/manual_computation.zig index 982c7b0b..b71a78de 100644 --- a/zml/ops/manual_computation.zig +++ b/zml/ops/manual_computation.zig @@ -111,7 +111,11 @@ noinline fn manualComputationInternal( .mesh => |m| m, .incompatible => |incomp| std.debug.panic("manualComputation expects all inputs and outputs to use the same mesh, got input {}: {} and input {}: {}", .{ incomp[0], input_tensors[incomp[0]], incomp[1], input_tensors[incomp[1]] }), }; - log.debug("manualComputation({s}) will use mesh {s}", .{ functor_name, mesh.name }); + const manual_axes = collectManualAxes(mesh, input_tensors); + log.debug("manualComputation({s}) will use mesh {s}, manual axes: {f}", .{ functor_name, mesh.name, stdx.fmt.stringsZ(manual_axes.constSlice()) }); + + const input_shapes = try arena.alloc(Shape, input_tensors.len); + for (input_shapes, input_tensors) |*shape, tensor| shape.* = tensor._shape; const input_types = try arena.alloc(*const mlir.Type, input_tensors.len); for (input_types, input_tensors) |*input_ty, tensor_ptr| { @@ -128,11 +132,6 @@ noinline fn manualComputationInternal( input_val.* = global_scope.value(tensor_ptr.*); } - const input_shapes = try arena.alloc(Shape, input_tensors.len); - for (input_shapes, input_tensors) |*shape, tensor_ptr| { - shape.* = tensor_ptr._shape; - } - const manual_block = mlir.Block.init(input_types, input_locs); errdefer manual_block.deinit(); @@ -221,12 +220,12 @@ noinline fn manualComputationInternal( return local_outputs; } -const ResolvedMesh = union(enum) { +const ChosenMesh = union(enum) { mesh: *const Sharding.Mesh, incompatible: [2]u32, }; -fn chooseCommonMesh(inputs: []const *const Tensor, fallback: *const Sharding.Mesh) ResolvedMesh { +fn chooseCommonMesh(inputs: []const *const Tensor, fallback: *const Sharding.Mesh) ChosenMesh { var mesh: ?*const Sharding.Mesh = null; var mesh_source_id: u32 = @intCast(inputs.len); for (0.., inputs) |i, input| { @@ -251,11 +250,58 @@ test chooseCommonMesh { const sharded = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&first, .{ .a = .data }); const other = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&second, .{ .a = .experts }); - try std.testing.expectEqual(ResolvedMesh{ .mesh = &fallback }, chooseCommonMesh(&.{}, &fallback)); - try std.testing.expectEqual(ResolvedMesh{ .mesh = &fallback }, chooseCommonMesh(&.{&replicated}, &fallback)); - try std.testing.expectEqual(ResolvedMesh{ .mesh = &first }, chooseCommonMesh(&.{&sharded}, &fallback)); - try std.testing.expectEqual(ResolvedMesh{ .mesh = &first }, chooseCommonMesh(&.{ &replicated, &sharded, &sharded }, &fallback)); - try std.testing.expectEqual(ResolvedMesh{ .incompatible = .{ 0, 1 } }, chooseCommonMesh(&.{ &sharded, &other }, &fallback)); + try std.testing.expectEqual(ChosenMesh{ .mesh = &fallback }, chooseCommonMesh(&.{}, &fallback)); + try std.testing.expectEqual(ChosenMesh{ .mesh = &fallback }, chooseCommonMesh(&.{&replicated}, &fallback)); + try std.testing.expectEqual(ChosenMesh{ .mesh = &first }, chooseCommonMesh(&.{&sharded}, &fallback)); + try std.testing.expectEqual(ChosenMesh{ .mesh = &first }, chooseCommonMesh(&.{ &replicated, &sharded, &sharded }, &fallback)); + try std.testing.expectEqual(ChosenMesh{ .incompatible = .{ 0, 1 } }, chooseCommonMesh(&.{ &sharded, &other }, &fallback)); +} + +fn collectManualAxes(mesh: *const Sharding.Mesh, inputs: []const *const Tensor) stdx.BoundedArray(Shape.Tag, Sharding.MAX_MESH_RANK) { + var used: [Sharding.MAX_MESH_RANK]bool = @splat(false); + for (inputs) |input| { + const sharding = input.shape()._sharding; + if (sharding.mesh == null) continue; + std.debug.assert(sharding.mesh.? == mesh); + for (0..input.rank()) |dim| { + if (sharding.partition.get(dim).meshAxis()) |axis| used[axis] = true; + } + } + + var axes: stdx.BoundedArray(Shape.Tag, Sharding.MAX_MESH_RANK) = .empty; + for (mesh.logical.axes.constSlice(), 0..) |axis, i| { + if (used[i]) axes.appendAssumeCapacity(axis); + } + return axes; +} + +test collectManualAxes { + const mesh: Sharding.Mesh = .initTest(.manual, .{ .data, .model, .unused }); + var data = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .data }); + var model = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .model }); + const replicated = Tensor.init(.{ .a = 8 }, .f32); + const open = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .open }); + + const axes = collectManualAxes(&mesh, &.{ &model, &replicated, &data, &model, &open }); + data = replicated; + model = replicated; + try std.testing.expectEqual(@as(usize, 2), axes.len); + try std.testing.expectEqualStrings("data", std.mem.span(axes.get(0))); + try std.testing.expectEqualStrings("model", std.mem.span(axes.get(1))); + try std.testing.expectEqual(@as(usize, 0), collectManualAxes(&mesh, &.{ &replicated, &open }).len); + try std.testing.expectEqual(@as(usize, 0), collectManualAxes(&mesh, &.{}).len); + + const registry: *mlir.DialectRegistry = try .init(); + defer registry.deinit(); + registry.registerDialect("sdy"); + const ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + const expected = dialects.shardy.ManualAxesAttribute.init(ctx, &.{ + .init(ctx, "data"), + .init(ctx, "model"), + }).asAttr(); + try std.testing.expect(shardy.manualAxesAttr(ctx, axes.constSlice()).eql(expected)); } const gspmd = struct { @@ -299,3 +345,11 @@ const gspmd = struct { return .{ shard_to_full.result(0), global_type }; } }; + +const shardy = struct { + pub fn manualAxesAttr(mlir_ctx: *mlir.Context, axes: []const Shape.Tag) *const mlir.Attribute { + var attrs: stdx.BoundedArray(*const mlir.StringAttribute, Sharding.MAX_MESH_RANK) = .empty; + for (axes) |axis| attrs.appendAssumeCapacity(.init(mlir_ctx, std.mem.span(axis))); + return dialects.shardy.ManualAxesAttribute.init(mlir_ctx, attrs.constSlice()).asAttr(); + } +}; diff --git a/zml/platform.zig b/zml/platform.zig index 9e513451..3b52c08e 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -643,9 +643,12 @@ pub const Platform = struct { logical, .suggest(logical, &platform.physical_mesh), ) catch |err| switch (err) { - error.InvalidPhysicalMesh, error.InvalidStrategy, error.InvalidPhysicalAxis => { + error.InvalidPhysicalMesh, error.InvalidStrategy, error.InvalidPhysicalAxis, error.OverlappingLogicalAxes => { std.debug.panic("ZML failed to create a valid mesh for logical mesh: {f}\nand physical_mesh: {f}\nPlease report this bug.", .{ logical, platform.physical_mesh }); }, + error.InvalidPartialBinding, error.InvalidLogicalDimension, error.IncompatibleLogicalDimension => { + std.debug.panic("ZML cannot satisfy the dimensions of logical mesh {f} on physical mesh {f}", .{ logical, platform.physical_mesh }); + }, error.OutOfMemory => |e| return e, }; } diff --git a/zml/shape.zig b/zml/shape.zig index 3e3fb56b..b0863367 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -381,7 +381,7 @@ pub const Shape = struct { pub fn count(self: Shape) usize { var res: i64 = 1; for (self.dims()) |d| { - stdx.debug.assert(d >= 0, "Can't count elements in shape with negative dimension: {f}", .{self}); + stdx.debug.assert(d >= 0, "Can't count elements in shape with negative dimension: {f}", .{&self}); res *= d; } return @intCast(res); @@ -824,8 +824,7 @@ pub const Shape = struct { .init(&.{.link_y}), .init(&.{.link_z}), }), - .folds = .empty, - .folds_consumed = .empty, + .grid = undefined, }; pub fn withPartitioning(self: Shape, mesh: *const Sharding.Mesh, partitioning: anytype) Shape { @@ -1133,21 +1132,28 @@ pub const Shape = struct { } } + test "byte strides handle zero dimensions and maximum rank" { + const empty = Shape.init(.{ 2, 0, 5 }, .f32).computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 0, 20, 4 }, empty.constSlice()); + const full_rank = Shape.init(.{ 2, 3, 5, 7, 11, 13, 17, 19 }, .u8).computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 4849845, 1616615, 323323, 46189, 4199, 323, 19, 1 }, full_rank.constSlice()); + const large = Shape.init(.{std.math.maxInt(i64)}, .f64).computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{8}, large.constSlice()); + } + fn computeStrides(self: Shape, element_size: i64) stdx.BoundedArray(i64, constants.MAX_RANK) { + @setRuntimeSafety(false); const rk = self.rank(); var strides: stdx.BoundedArray(i64, constants.MAX_RANK) = .{ .buffer = undefined, .len = rk }; - if (rk == 0) return strides; + var stride = element_size; + var ax: usize = rk; + while (ax > 0) { + ax -= 1; + strides.buffer[ax] = stride; + // Axis 0 is not part of any stride; avoid computing the total size. + if (ax > 0) stride *= self._dims.buffer[ax]; + } - const V = @Vector(constants.MAX_RANK, i64); - const rank_mask = std.simd.iota(u8, constants.MAX_RANK) < @as(@Vector(constants.MAX_RANK, u8), @splat(rk)); - // For each axis compute the product of all following dimensions - // and the element size in bytes. - var d: V = @bitCast(self._dims.buffer); - d = @select(i64, rank_mask, d, @as(V, @splat(1))); - d = std.simd.shiftElementsLeft(d, 1, element_size); - d = std.simd.prefixScan(.Mul, -1, d); - - strides.buffer = @bitCast(d); return strides; } diff --git a/zml/slice.zig b/zml/slice.zig index 2deb96d2..9451165b 100644 --- a/zml/slice.zig +++ b/zml/slice.zig @@ -6,6 +6,7 @@ const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const floats = @import("floats.zig"); const Shape = @import("shape.zig").Shape; +const builtin = @import("builtin"); pub fn isBytes(comptime T: type) bool { const type_info = @typeInfo(T); @@ -135,13 +136,13 @@ pub const Slice = struct { pub fn data(slice: Slice) []u8 { std.debug.assert(slice.mutable); - std.debug.assert(slice.isContiguous()); + if (builtin.mode == .debug) std.debug.assert(slice.isContiguous()); const end = slice.offset_bytes + slice.shape.byteSize(); return @constCast(slice.bytes[slice.offset_bytes..end]); } pub fn constData(slice: Slice) []const u8 { - std.debug.assert(slice.isContiguous()); + if (builtin.mode == .debug) std.debug.assert(slice.isContiguous()); const end = slice.offset_bytes + slice.shape.byteSize(); return slice.bytes[slice.offset_bytes..end]; } diff --git a/zml/tensor.zig b/zml/tensor.zig index 796709ad..c4ac5360 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -242,8 +242,7 @@ pub const Tensor = struct { .physical = undefined, .logical = .mesh(.{ .data = .low_bandwidth, .model = .high_bandwidth }), .bindings = .init(&.{.init(&.{.link_x})}), - .folds = .empty, - .folds_consumed = .empty, + .grid = undefined, }; const mp_dp: Sharding.Mesh = .{ @@ -251,8 +250,7 @@ pub const Tensor = struct { .physical = undefined, .logical = .mesh(.{ .model = .low_bandwidth, .data = .high_bandwidth }), .bindings = .init(&.{.init(&.{.link_x})}), - .folds = .empty, - .folds_consumed = .empty, + .grid = undefined, }; const x = Tensor.init(.{ .h = 8 }, .f32).withPartitioning(&dp_mp, .{ .h = .model }); From 2d516eeb789eeb26781a8a929a3a91fa01882957 Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Fri, 16 Oct 2026 18:46:10 +0200 Subject: [PATCH 793/881] zml/platforms: rollback to CUDA 13.3.0 (#955) CUDA 13.4.2 ships PTAX 13.4.92, which miscompiles a valid atomic CAS in some LLM models, causing crashses. The generated code uses a different byte offset for the load and atomic update, so the two operations access different memory locations. --- platforms/cuda/cuda.bzl | 120 ++++++++++++------------- platforms/cuda/libzml_cuda.BUILD.bazel | 2 +- 2 files changed, 56 insertions(+), 66 deletions(-) diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 7a32188c..8604d025 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -6,18 +6,17 @@ load("//platforms:packages.bzl", "packages") ARCHS = ["linux-x86_64", "linux-sbsa"] CUDA_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cuda/redist/" -CUDA_VERSION = "13.4.3" -CUDA_MAJOR_MINOR = ".".join(CUDA_VERSION.split(".")[:2]) -CUDA_VARIANT = "cuda{}".format(CUDA_MAJOR_MINOR) -CUDA_REDIST_JSON_SHA256 = "1420d5c676cad98a5ff423d529f116fee92f6b99559a3afdad41e782d6a11dba" +CUDA_VERSION = "13.3.0" +CUDA_VARIANT = "cuda13.3" +CUDA_REDIST_JSON_SHA256 = "507eddaab1360336bc0fe17b77552e0b7dfe1e74da888671c3a2f5fad7775db1" CUDNN_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cudnn/redist/" -CUDNN_VERSION = "9.27.0" -CUDNN_REDIST_JSON_SHA256 = "d3f870db59da3fd12581aeca5fda381afa50ce1d4d9cccc8dcd79d7a628c22a2" +CUDNN_VERSION = "9.24.0" +CUDNN_REDIST_JSON_SHA256 = "89a0626f1925d9f4ceb8d6bc0b2de160b30e92afaae5471e6cfbb50234ae5488" NVSHMEM_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/nvshmem/redist/" -NVSHMEM_VERSION = "3.8.0" -NVSHMEM_REDIST_JSON_SHA256 = "5920f88ccae8261083570f740463eddfdf8ad2708d88ebede380ce55d84eaf6c" +NVSHMEM_VERSION = "3.6.5" +NVSHMEM_REDIST_JSON_SHA256 = "afbf1ad5c4174c25a66cadd5b925d6814e679e8f836ef1fee1442afb6cb7fdd3" _BUILD_FILE_DEFAULT_VISIBILITY = """\ package(default_visibility = ["//visibility:public"]) @@ -35,41 +34,6 @@ CUDA_COMPAT_FILES = [ "libnvidia-ptxjitcompiler.so.1", ] -def _cuda_compat_packages(driver_version): - return [ - packages.patchelf( - name = "{}.patchelf".format(file), - set_rpath = "$ORIGIN", - src = "compat/{}".format(file), - soname = file, - ) - for file in CUDA_COMPAT_FILES - ] + [ - packages.patchelf( - name = "nvcuextend", - set_rpath = "$ORIGIN", - src = "compat/libnvcuextend.so.1", - soname = "libnvcuextend.so.1", - ), - """filegroup( - name = "cuda_compat", - srcs = [ - "compat/libnvidia-gpucomp.so.{driver_version}", - "compat/libnvidia-tileiras.so.{driver_version}", - ] + {} + select({{ - "@llvm//platforms/config:linux_x86_64": [ - "compat/libnvidia-pkcs11-openssl3.so.{driver_version}", - ], - "@llvm//platforms/config:linux_aarch64": [ - ":nvcuextend", - ], - }}), - )""".format(repr([ - ":{}.patchelf".format(file) - for file in CUDA_COMPAT_FILES - ]), driver_version = driver_version), - ] - CUDA_PACKAGES = { "cuda_nvml_dev": [ packages.cc_library( @@ -109,6 +73,31 @@ CUDA_PACKAGES = { srcs = ["lib/libnvtx3interop.so"], ), ], + "cuda_compat": [ + packages.patchelf( + name = "{}.patchelf".format(file), + set_rpath = "$ORIGIN", + src = "compat/{}".format(file), + soname = file, + ) + for file in CUDA_COMPAT_FILES + ] + [ + """filegroup( + name = "cuda_compat", + srcs = [ + "compat/libnvidia-gpucomp.so.610.43.02", + "compat/libnvidia-tileiras.so.610.43.02", + ] + {} + select({{ + "@llvm//platforms/config:linux_x86_64": [ + "compat/libnvidia-pkcs11-openssl3.so.610.43.02", + ], + "@llvm//platforms/config:linux_aarch64": [], + }}), + )""".format(repr([ + ":{}.patchelf".format(file) + for file in CUDA_COMPAT_FILES + ])), + ], "libcufft": [ packages.filegroup( name = "libcufft", @@ -165,7 +154,7 @@ CUDA_PACKAGES = { name = "cuda_nvrtc", srcs = [ "lib/libnvrtc.so.13", - "lib/libnvrtc-builtins.so.13.4", + "lib/libnvrtc-builtins.so.13.3", ], ), ], @@ -194,7 +183,6 @@ CUDNN_PACKAGES = { "lib/libcudnn_engines_runtime_compiled.so.9", "lib/libcudnn_engines_tensor_ir.so.9", "lib/libcudnn_heuristic.so.9", - "lib/libcudnn_ext.so.9", ], ), ], @@ -207,7 +195,7 @@ NVSHMEM_PACKAGES = { srcs = [ "lib/libnvshmem_host.so.3", "lib/nvshmem_bootstrap_uid.so.3", - "lib/nvshmem_transport_ibrc.so.7", + "lib/nvshmem_transport_ibrc.so.5", ], ), ], @@ -236,12 +224,23 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "b1b33616999bfe479dfff050e6934f7b2d935c08621a269014aff585925614c2", - "url": "https://mirror.zml.ai/plugins/202610160837.234.1.e1adc5dca29d/zml-cuda-linux-amd64.tar.zst", + "sha256": "226e8b04611efe6c8ac3ceaf3010499b5957d498e585c970afb69c2ec47ff9fe", + "url": "https://mirror.zml.ai/plugins/202610150915.220.1.f2470651679c/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "f6819007f6d1ab5f3a1c7b79d7e2ddc9b33eba8f61d45589c322931d6379e672", - "url": "https://mirror.zml.ai/plugins/202610160837.234.1.e1adc5dca29d/zml-cuda-linux-arm64.tar.zst", + "sha256": "5b55fcd3173485b0e96034bbbf078e354c393d938aadb2ab8fa9bf658b384759", + "url": "https://mirror.zml.ai/plugins/202610150915.220.1.f2470651679c/zml-cuda-linux-arm64.tar.zst", + }, +} + +_NCCL_ASSETS = { + "amd64": { + "url": "https://pypi.nvidia.com/nvidia-nccl-cu13/nvidia_nccl_cu13-2.30.4-py3-none-manylinux_2_18_x86_64.whl", + "sha256": "534dbf3058cadb625f08ab0d17f1dffad3b961a2bfa360d66633fcf21be53f57", + }, + "arm64": { + "url": "https://pypi.nvidia.com/nvidia-nccl-cu13/nvidia_nccl_cu13-2.30.4-py3-none-manylinux_2_18_aarch64.whl", + "sha256": "e99308a3a89fba78918d50886e81072a6c8b0b4199feb02c3903e63713a6525a", }, } @@ -294,16 +293,6 @@ def _cuda_impl(mctx): build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + "\n".join(build_file_content), ) - compat_package = "cuda-compat-{}".format(CUDA_MAJOR_MINOR.replace(".", "-")) - for arch, pkg in loaded_packages[compat_package].items(): - http_deb_archive( - name = "cuda_compat_linux_{}".format({"amd64": "x86_64", "arm64": "sbsa"}[arch]), - urls = pkg["urls"], - sha256 = pkg["sha256"], - strip_prefix = "./usr/local/cuda-{}".format(CUDA_MAJOR_MINOR), - build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + "\n".join(_cuda_compat_packages(pkg["version"].split("-")[0])), - ) - for pkg, build_file_content in CUDA_PACKAGES.items(): pkg_data = CUDA_REDIST[pkg] for arch in ARCHS: @@ -352,16 +341,17 @@ def _cuda_impl(mctx): strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), ) - for arch, pkg in loaded_packages["libnccl2"].items(): - http_deb_archive( + for arch, arch_config in _NCCL_ASSETS.items(): + http_archive( name = _repo_name("nccl", arch), - urls = pkg["urls"], - sha256 = pkg["sha256"], + urls = [arch_config["url"]], + type = "zip", + sha256 = arch_config["sha256"], build_file_content = "\n".join([ _BUILD_FILE_DEFAULT_VISIBILITY, packages.filegroup( name = "nccl", - srcs = ["usr/lib/{}-linux-gnu/libnccl.so.2".format({"amd64": "x86_64", "arm64": "aarch64"}[arch])], + srcs = ["nvidia/nccl/lib/libnccl.so.2"], ), ]), ) diff --git a/platforms/cuda/libzml_cuda.BUILD.bazel b/platforms/cuda/libzml_cuda.BUILD.bazel index 8c327a80..e79ab900 100644 --- a/platforms/cuda/libzml_cuda.BUILD.bazel +++ b/platforms/cuda/libzml_cuda.BUILD.bazel @@ -68,7 +68,7 @@ copy_to_directory( include_external_repositories = ["**"], replace_prefixes = { "compat": "lib/compat", - "nvcuextend": "lib/compat", + "nvidia/nccl/lib": "lib", "nvvm/lib64": "lib", "libzml_cuda_so": "lib", "lib/x86_64-linux-gnu": "lib", From 2a57062b298835427b5bffc0040904ff030f6c47 Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Mon, 19 Oct 2026 10:22:38 +0200 Subject: [PATCH 794/881] zml/nn: softplus (#954) --- examples/llm/models/qwen3_5/model.zig | 6 +-- examples/llm/models/qwen3_5_moe/model.zig | 8 +--- zml/nn.zig | 48 +++++++++++++++++++++++ 3 files changed, 50 insertions(+), 12 deletions(-) diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index d43bf7ee..ac786f05 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -920,7 +920,7 @@ pub const GatedDeltaNet = struct { const beta = b.sigmoid(); const aLog_type = self.aLog.dtype(); - const g = self.aLog.broad(a.shape()).exp().mul(softplus(a.convert(aLog_type).add(self.dt_bias.convert(aLog_type).broad(a.shape())))).negate(); + const g = self.aLog.broad(a.shape()).exp().mul(zml.nn.softplus(a.convert(aLog_type).add(self.dt_bias.convert(aLog_type).broad(a.shape())), .{})).negate(); const query_for_rule = b: { if (self.qk_head_repetition == 1) break :b query; @@ -1298,7 +1298,3 @@ pub const KvCache = struct { }; } }; - -fn softplus(x: zml.Tensor) zml.Tensor { - return x.exp().addConstant(1).log(); -} diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index d8e00890..8908a2b6 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -1049,7 +1049,7 @@ pub const GatedDeltaNet = struct { const beta = b.sigmoid(); const aLog_type = self.aLog.dtype(); - const g = self.aLog.broad(a.shape()).exp().mul(softplus(a.convert(aLog_type).add(self.dt_bias.convert(aLog_type).broad(a.shape())))).negate(); + const g = self.aLog.broad(a.shape()).exp().mul(zml.nn.softplus(a.convert(aLog_type).add(self.dt_bias.convert(aLog_type).broad(a.shape())), .{})).negate(); const query_for_rule = if (self.qk_head_repetition == 1) query else query.stutter1d(query.axis(.kh), self.qk_head_repetition); const key_for_rule = if (self.qk_head_repetition == 1) key else key.stutter1d(key.axis(.kh), self.qk_head_repetition); @@ -1407,9 +1407,3 @@ pub const KvCache = struct { }; } }; - -//========================Utils======================== - -fn softplus(x: zml.Tensor) zml.Tensor { - return x.exp().addConstant(1).log(); -} diff --git a/zml/nn.zig b/zml/nn.zig index d0d038a2..56fd8b51 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -2343,3 +2343,51 @@ fn ShapeStruct(comptime dims: anytype) type { return @Struct(.@"extern", null, &struct_field_names, &struct_field_types, &struct_field_attrs); } + +/// Smooth approximation of ReLU function: log(1 + exp(x)) +/// For numerical stability, falls back to the linear function when `x > opts.threshold`. +pub fn softplus( + x: Tensor, + opts: struct { threshold: f32 = 20.0 }, +) zml.Tensor { + // Compute log1p(z) as z * log(u) / (u - 1), where u = 1 + z. + // Using log(1 + exp(x)) directly loses precision for x < ~-16.6, causing strongly negative logits to produce zero scores. + const z = x.exp(); + const u = z.addConstant(1); + const log1p = u.cmp(.EQ, .scalar(1, .f32)).select(z, z.mul(u.log()).div(u.subConstant(1))); + return x.cmp(.GT, .scalar(opts.threshold, .f32)).select(x, log1p); +} + +test "softplus" { + const inputs = [_]f32{ -20.0, -1.0, 0.0, 1.0, 2.0, 2.1, 3.0 }; + const expected = [_]f32{ 2.0611537e-9, 0.3132617, 0.6931472, 1.3132616, 2.126928, 2.1, 3.0 }; + + const platform = zml.testing.env(); + const input: Tensor = .init(.{ .x = inputs.len }, .f32); + const Local = struct { + fn call(x: Tensor) Tensor { + return softplus(x, .{ .threshold = 2.0 }); + } + }; + + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Local.call, .{input}, .{}); + defer exe.deinit(); + + var input_buffer: zml.Buffer = try .fromBytes( + std.testing.io, + platform, + input.shape(), + std.mem.sliceAsBytes(&inputs), + ); + defer input_buffer.deinit(); + + var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); + defer result.deinit(); + + try zml.testing.expectClose( + std.testing.io, + Slice.init(input.shape(), std.mem.sliceAsBytes(&expected)), + result, + .{}, + ); +} From c651a9de45a8aaf2135139fb90d489ec50ce8ed9 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Mon, 19 Oct 2026 12:45:42 +0200 Subject: [PATCH 795/881] rbe: use GCR image registry instead of Docker Hub (#962) GCR mirror is more reliable than fetching through the Docker Hub --- BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BUILD.bazel b/BUILD.bazel index cefaac75..734a3792 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -57,7 +57,7 @@ platform( "@bazel_tools//tools/cpp:clang", ], exec_properties = { - "container-image": "docker://ubuntu:22.04", + "container-image": "docker://mirror.gcr.io/library/ubuntu:22.04", "OSFamily": "Linux", }, visibility = ["//visibility:public"], From 9c7448da3c654dbe60feba8472c8cec190c2c953 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 19 Oct 2026 15:15:11 +0200 Subject: [PATCH 796/881] zml: re-structure moe kernels (#941) Akin to #926, this PR introduces a restructuration of the moe kernels in ZML. No need to re-explaing the organizatiopn because it follows what has been done in the mentioned PR. Here is the detailed changes of this PR: * Kernels reorganization. `zml/kernels/moe` is the place to put all the moe implementation. There is one sub-folder per backend. * Changed the expected shape and tags when calling the moe forward function. It's now expecting a 2D tensor tagged `.{.t , .d}`. Tests still passes without key modifications. The only small tweak that was done is related to fp32 -> fp8 triton quantization on sm89. Due to how triton lowers the convert, it was producing slightly incorrect result. A small preparation was added to ensure the conversion is identical on this platform. **Future TODO**: * Ideally, this conversion on sm89 should be gated so that we don't pay the cost on other platforms. We don't have a way to detect the platform in the triton `Builder` yet but it could probably be added * Settle on a complete API that encodes all the scenario we could encounter. One example of that is what is required for Deepseek V4.1 Flash, the moe gemms should be done in `mxfp8 x mxfp4` and we should have a way to request that in a better way than `quantize_input: bool`. --- examples/llm/models/qwen3_5_moe/model.zig | 8 +- zml/BUILD.bazel | 27 +- zml/kernels/moe/cute/BUILD.bazel | 11 + .../moe/cute/cute.zig} | 31 +- .../cute_kernels => kernels/moe/cute}/moe.zig | 89 +- .../moe/cute}/mxfp4.zig | 2 +- .../moe/cute}/persistent_mxfp4.zig | 2 +- .../moe/cutlass_flashinfer/BUILD.bazel | 8 + .../cutlass_flashinfer.zig | 58 +- zml/kernels/moe/fly/BUILD.bazel | 9 + zml/kernels/moe/fly/fly.zig | 356 +++++ zml/kernels/moe/fly/moe.zig | 661 ++++++++++ zml/kernels/moe/metal/BUILD.bazel | 8 + zml/{moe => kernels/moe/metal}/metal.zig | 8 +- zml/{ => kernels}/moe/moe.zig | 62 +- zml/kernels/moe/mosaic_tpu/BUILD.bazel | 9 + .../moe/mosaic_tpu}/gmm_ep.zig | 2 +- .../moe/mosaic_tpu}/mosaic_tpu.zig | 17 +- zml/kernels/moe/stablehlo/BUILD.bazel | 8 + .../moe/stablehlo}/stablehlo.zig | 21 +- zml/{ => kernels}/moe/tests.zig | 107 +- zml/kernels/moe/triton/BUILD.bazel | 9 + .../moe/triton/moe.zig} | 345 ++++- zml/kernels/moe/triton/triton.zig | 1161 +++++++++++++++++ zml/moe/fly_kernels/moe.zig | 289 ---- zml/moe/fly_kernels/mxfp4.zig | 462 ------- zml/moe/fused_experts.zig | 550 -------- zml/moe/triton.zig | 326 ----- zml/moe/triton_kernels/mxfp4.zig | 465 ------- zml/moe/triton_mxfp4.zig | 116 -- zml/zml.zig | 2 +- 31 files changed, 2832 insertions(+), 2397 deletions(-) create mode 100644 zml/kernels/moe/cute/BUILD.bazel rename zml/{moe/cute_mxfp4.zig => kernels/moe/cute/cute.zig} (74%) rename zml/{moe/cute_kernels => kernels/moe/cute}/moe.zig (66%) rename zml/{moe/cute_kernels => kernels/moe/cute}/mxfp4.zig (99%) rename zml/{moe/cute_kernels => kernels/moe/cute}/persistent_mxfp4.zig (99%) create mode 100644 zml/kernels/moe/cutlass_flashinfer/BUILD.bazel rename zml/{moe => kernels/moe/cutlass_flashinfer}/cutlass_flashinfer.zig (95%) create mode 100644 zml/kernels/moe/fly/BUILD.bazel create mode 100644 zml/kernels/moe/fly/fly.zig create mode 100644 zml/kernels/moe/fly/moe.zig create mode 100644 zml/kernels/moe/metal/BUILD.bazel rename zml/{moe => kernels/moe/metal}/metal.zig (97%) rename zml/{ => kernels}/moe/moe.zig (79%) create mode 100644 zml/kernels/moe/mosaic_tpu/BUILD.bazel rename zml/{moe/mosaic_tpu_kernels => kernels/moe/mosaic_tpu}/gmm_ep.zig (99%) rename zml/{moe => kernels/moe/mosaic_tpu}/mosaic_tpu.zig (96%) create mode 100644 zml/kernels/moe/stablehlo/BUILD.bazel rename zml/{moe => kernels/moe/stablehlo}/stablehlo.zig (91%) rename zml/{ => kernels}/moe/tests.zig (89%) create mode 100644 zml/kernels/moe/triton/BUILD.bazel rename zml/{moe/triton_kernels/triton_kernels.zig => kernels/moe/triton/moe.zig} (70%) create mode 100644 zml/kernels/moe/triton/triton.zig delete mode 100644 zml/moe/fly_kernels/moe.zig delete mode 100644 zml/moe/fly_kernels/mxfp4.zig delete mode 100644 zml/moe/fused_experts.zig delete mode 100644 zml/moe/triton.zig delete mode 100644 zml/moe/triton_kernels/mxfp4.zig delete mode 100644 zml/moe/triton_mxfp4.zig diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 8908a2b6..75719aed 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -769,9 +769,9 @@ pub const Moe = struct { const routing_scores, const topk_ids = self.router.forward(x); const moe_output = zml.moe.forwardMoe( - x, - topk_ids, - routing_scores, + x.merge(.{ .t = .{ .b, .s } }), + topk_ids.merge(.{ .t = .{ .b, .s } }), + routing_scores.merge(.{ .t = .{ .b, .s } }), self.gate_up_proj, self.down_proj, moe_backend, @@ -780,7 +780,7 @@ pub const Moe = struct { .quantize_input = false, .routing_weight_placement = .after_down, }, - ); + ).reshape(x.shape()); const shared_gate = self.shared_expert_gate.forward(x, x.dtype()).sigmoid().broad(x.shape()); const shared = self.shared_expert.forward(x).rename(.{ .dout = .d }).mul(shared_gate).withPartitioning(.model, .{ diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index f7c17184..a396efb0 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -88,29 +88,13 @@ zig_library( "io.zig", "kernel.zig", "kernels/flash_attention/flash_attention.zig", + "kernels/moe/moe.zig", + "kernels/moe/tests.zig", "kernels/paged_attention/paged_attention.zig", "logo.zig", "mem.zig", "meta.zig", "mlirx.zig", - "moe/cute_kernels/moe.zig", - "moe/cute_kernels/mxfp4.zig", - "moe/cute_kernels/persistent_mxfp4.zig", - "moe/cute_mxfp4.zig", - "moe/cutlass_flashinfer.zig", - "moe/fly_kernels/moe.zig", - "moe/fly_kernels/mxfp4.zig", - "moe/fused_experts.zig", - "moe/metal.zig", - "moe/moe.zig", - "moe/mosaic_tpu.zig", - "moe/mosaic_tpu_kernels/gmm_ep.zig", - "moe/stablehlo.zig", - "moe/tests.zig", - "moe/triton.zig", - "moe/triton_kernels/mxfp4.zig", - "moe/triton_kernels/triton_kernels.zig", - "moe/triton_mxfp4.zig", "nn.zig", "ops.zig", "ops/manual_computation.zig", @@ -128,6 +112,13 @@ zig_library( "//zml/kernels/flash_attention/cute", "//zml/kernels/flash_attention/metal", "//zml/kernels/flash_attention/nki", + "//zml/kernels/moe/cute", + "//zml/kernels/moe/cutlass_flashinfer", + "//zml/kernels/moe/fly", + "//zml/kernels/moe/metal", + "//zml/kernels/moe/mosaic_tpu", + "//zml/kernels/moe/stablehlo", + "//zml/kernels/moe/triton", "//zml/kernels/paged_attention/metal", "//zml/kernels/paged_attention/mosaic_tpu", "//zml/kernels/paged_attention/stablehlo", diff --git a/zml/kernels/moe/cute/BUILD.bazel b/zml/kernels/moe/cute/BUILD.bazel new file mode 100644 index 00000000..1c3e9683 --- /dev/null +++ b/zml/kernels/moe/cute/BUILD.bazel @@ -0,0 +1,11 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "cute", + srcs = [ + "cute.zig", + "moe.zig", + "mxfp4.zig", + "persistent_mxfp4.zig", + ], +) diff --git a/zml/moe/cute_mxfp4.zig b/zml/kernels/moe/cute/cute.zig similarity index 74% rename from zml/moe/cute_mxfp4.zig rename to zml/kernels/moe/cute/cute.zig index 650378ef..ced6dd89 100644 --- a/zml/moe/cute_mxfp4.zig +++ b/zml/kernels/moe/cute/cute.zig @@ -2,9 +2,8 @@ const stdx = @import("stdx"); -const zml = @import("../zml.zig"); -const triton_mxfp4 = @import("triton_mxfp4.zig"); -pub const kernels = @import("cute_kernels/moe.zig"); +const zml = @import("../../../zml.zig"); +pub const kernels = @import("moe.zig"); // `moe.zig`'s `refAllDecls` reaches this file but not its imports, so the // kernel emit tests are only collected if `kernels` is referenced here. @@ -12,8 +11,6 @@ test { _ = kernels; } -pub const Parameters = triton_mxfp4.Parameters; - pub fn isAvailable(platform: *const zml.Platform) bool { if (platform.target != .cuda) return false; const cc = zml.platform.cuda.computeCapability(platform) orelse return false; @@ -21,10 +18,10 @@ pub fn isAvailable(platform: *const zml.Platform) bool { } pub fn validateOptions(opts: zml.moe.Options) void { - stdx.debug.assert(opts.activation == .swiglu, "cute_mxfp4 backend only accepts swiglu activation, got {}", .{opts.activation}); - stdx.debug.assert(opts.activation.swiglu.limit != null, "cute_mxfp4 backend requires swiglu limit to be set", .{}); - stdx.debug.assert(opts.activation.swiglu.bias == null, "cute_mxfp4 backend requires swiglu bias to be null", .{}); - stdx.debug.assert(opts.activation.swiglu.scale == null, "cute_mxfp4 backend requires swiglu scale to be null", .{}); + stdx.debug.assert(opts.activation == .swiglu, "cute backend only accepts swiglu activation, got {}", .{opts.activation}); + stdx.debug.assert(opts.activation.swiglu.limit != null, "cute backend requires swiglu limit to be set", .{}); + stdx.debug.assert(opts.activation.swiglu.bias == null, "cute backend requires swiglu bias to be null", .{}); + stdx.debug.assert(opts.activation.swiglu.scale == null, "cute backend requires swiglu scale to be null", .{}); } pub fn fusedExperts( @@ -36,16 +33,16 @@ pub fn fusedExperts( options: zml.moe.Options, ) zml.Tensor { validateOptions(options); - stdx.debug.assert(input.dtype() == .bf16, "cute_mxfp4 backend only supports bf16 inputs, got {}", .{input.dtype()}); - stdx.debug.assert(gate_up.bias == null and down.bias == null, "cute_mxfp4 backend expects gate_up bias and down bias to be null", .{}); + stdx.debug.assert(input.dtype() == .bf16, "cute backend only supports bf16 inputs, got {}", .{input.dtype()}); + stdx.debug.assert(gate_up.bias == null and down.bias == null, "cute backend expects gate_up bias and down bias to be null", .{}); - const gq = gate_up.quantization orelse @panic("cute_mxfp4 backend requires gate_up quantization to be set"); - const dq = down.quantization orelse @panic("cute_mxfp4 backend requires down quantization to be set"); - stdx.debug.assert(gq.scheme == .mxfp4 and dq.scheme == .mxfp4, "cute_mxfp4 expects gate_up and down quantization scheme to be mxfp4, got {} and {}", .{ gq.scheme, dq.scheme }); + const gq = gate_up.quantization orelse @panic("cute backend requires gate_up quantization to be set"); + const dq = down.quantization orelse @panic("cute backend requires down quantization to be set"); + stdx.debug.assert(gq.scheme == .mxfp4 and dq.scheme == .mxfp4, "cute expects gate_up and down quantization scheme to be mxfp4, got {} and {}", .{ gq.scheme, dq.scheme }); // Weight storage for mxfp4 in HF is expressed as u8 or i8 - stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "cute_mxfp4 expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); - stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "cute_mxfp4 expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); + stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "cute expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); + stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "cute expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); const expert_parallelism = gate_up.weight.shape().partition(.expert).isSharded(); const hidden = down.weight.dim(1); @@ -57,7 +54,7 @@ pub fn fusedExperts( // The SwiGLU clamp is a kernel parameter, but the routing weight is // multiplied in the up epilogue, before the down projection: applying it // after would have to move into the down epilogue or the reduction. - stdx.debug.assert(options.routing_weight_placement == .before_down, "cute_mxfp4 backend only supports routing_weight_placement = .before_down, got {}", .{options.routing_weight_placement}); + stdx.debug.assert(options.routing_weight_placement == .before_down, "cute backend only supports routing_weight_placement = .before_down, got {}", .{options.routing_weight_placement}); const context: Context = .{ .input = input, .ids = ids, diff --git a/zml/moe/cute_kernels/moe.zig b/zml/kernels/moe/cute/moe.zig similarity index 66% rename from zml/moe/cute_kernels/moe.zig rename to zml/kernels/moe/cute/moe.zig index e5794dbb..638685e3 100644 --- a/zml/moe/cute_kernels/moe.zig +++ b/zml/kernels/moe/cute/moe.zig @@ -6,10 +6,10 @@ const std = @import("std"); const stdx = @import("stdx"); -const zml = @import("../../zml.zig"); -const triton_mxfp4 = @import("../triton_mxfp4.zig"); +const zml = @import("../../../zml.zig"); pub const boundary = @import("mxfp4.zig"); pub const persistent = @import("persistent_mxfp4.zig"); +const triton_kernels = @import("../triton/moe.zig"); pub const Inputs = struct { /// `[tokens, hidden]` BF16. @@ -29,8 +29,8 @@ pub const Inputs = struct { /// The kernels are tuned for the DeepSeek V4.1 expert geometry, shared by the /// main MoE and the D-Spark drafter. Expert count and top-k are parameters. pub fn validateShapes(hidden: i64, intermediate: i64) void { - stdx.debug.assert(hidden == 5120, "cute_mxfp4 backend expects hidden dimension to be 5120, got {}", .{hidden}); - stdx.debug.assert(intermediate == 2304, "cute_mxfp4 backend expects intermediate dimension to be 2304, got {}", .{intermediate}); + stdx.debug.assert(hidden == 5120, "cute backend expects hidden dimension to be 5120, got {}", .{hidden}); + stdx.debug.assert(intermediate == 2304, "cute backend expects intermediate dimension to be 2304, got {}", .{intermediate}); } /// Route each token/top-k pair to its own one-row group instead of grouping @@ -69,8 +69,10 @@ fn persistentCtas() i64 { return if (attribute.int64 > 0) attribute.int64 else 148; } +// TODO(Corentin): settle on groups/capacity. + /// Upper bound on the number of `group`-row expert groups for all routes. -fn capacity(tokens: i64, topk: i64, experts: i64, group: i64) i64 { +fn computeCapacity(tokens: i64, topk: i64, experts: i64, group: i64) i64 { return @divTrunc(tokens * topk + group - 1, group) + experts - 1; } @@ -88,23 +90,72 @@ const Routing = struct { schedule: zml.Tensor, }; -fn route(tokens: i64, hidden: i64, intermediate: i64, experts: i64, topk: i64, n: i64, direct: bool, ids: zml.Tensor) Routing { +const atomic_route_threshold = 6 * 2048; + +fn route(tokens: i64, experts: i64, topk: i64, n: i64, direct: bool, ids: zml.Tensor) Routing { const routes = tokens * topk; const flat_ids = ids.convert(.i32).reshape(.{routes}); if (direct) return .{ .groups = routes, .route_map = null, .route_inverse = flat_ids, .schedule = flat_ids }; - const groups = capacity(tokens, topk, experts, n); - const cfg: triton_mxfp4.kernels.Config = .{ - .tokens = tokens, - .hidden = hidden, - .intermediate = intermediate, - .experts = experts, - .global_experts = experts, - .topk = topk, - // Unused: only the routing and scheduling kernels of that config run. - .swiglu_limit = 0, + const capacity = computeCapacity(tokens, topk, experts, n); + + std.debug.assert(capacity > 0); + // `route` rescans every route once per expert: cheapest for decode, but + // quadratic. Past a few thousand tokens, count with atomics instead. + const r: struct { counts: zml.Tensor, pos: zml.Tensor } = if (routes >= atomic_route_threshold) blk: { + const counted = triton_kernels.RouteAtomic.Kernel.call( + .{ .ids = ids, .zeros = zml.Tensor.zeroes(.init(.{experts}, .i32)) }, + .{ .counts = .init(.{experts}, .i32), .pos = .init(.{routes}, .i32) }, + .{ + .cfg = .{ + .tokens = tokens, + .topk = topk, + .expert_offset = 0, + .experts = experts, + }, + .grid = .{ @intCast(@divTrunc(routes + triton_kernels.RouteAtomic.route_block - 1, triton_kernels.RouteAtomic.route_block)), 1, 1 }, + .num_warps = 4, + .num_stages = 1, + .output_operand_aliases = .{ .counts = .zeros }, + }, + ); + break :blk .{ .counts = counted.counts, .pos = counted.pos }; + } else blk: { + const scanned = triton_kernels.Route.Kernel.call( + .{ .ids = ids }, + .{ .counts = .init(.{experts}, .i32), .pos = .init(.{routes}, .i32) }, + .{ + .cfg = .{ .tokens = tokens, .topk = topk, .expert_offset = 0 }, + .grid = .{ @intCast(experts), 1, 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ); + break :blk .{ .counts = scanned.counts, .pos = scanned.pos }; }; - const schedule = triton_mxfp4.kernels.scheduleForCute(cfg, flat_ids.reshape(.{ tokens, topk }), groups, n); - return .{ .groups = groups, .route_map = schedule.route_map, .route_inverse = schedule.route_inverse, .schedule = schedule.schedule }; + + const s = triton_kernels.Schedule.Kernel.call( + .{ .ids = ids, .counts = r.counts, .pos = r.pos }, + .{ + .map = .init(.{capacity * n}, .i32), + .perm = .init(.{tokens * topk}, .i32), + .sched = .init(.{2 * capacity + 1}, .i32), + }, + .{ + .cfg = .{ + .tokens = tokens, + .topk = topk, + .expert_offset = 0, + .experts = experts, + .capacity = capacity, + .group = n, + }, + .grid = .{ @intCast(@divTrunc(tokens * topk + 127, 128)), 1, 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ); + + return .{ .groups = capacity, .route_map = s.perm, .route_inverse = s.map, .schedule = s.sched }; } /// `[tokens, hidden]` BF16 output of the MoE layer. @@ -117,7 +168,7 @@ pub fn forward(tokens: i64, hidden: i64, intermediate: i64, experts: i64, topk: pub fn forwardWithTile(tokens: i64, hidden: i64, intermediate: i64, experts: i64, topk: i64, swiglu_limit: f32, a: Inputs, tile_n: ?i64) zml.Tensor { const n = tile_n orelse tileN(tokens, experts, topk); const direct = isDirect(tokens); - const routing = route(tokens, hidden, intermediate, experts, topk, n, direct, a.ids); + const routing = route(tokens, experts, topk, n, direct, a.ids); const routes = tokens * topk; const up_cfg: persistent.Config = .{ diff --git a/zml/moe/cute_kernels/mxfp4.zig b/zml/kernels/moe/cute/mxfp4.zig similarity index 99% rename from zml/moe/cute_kernels/mxfp4.zig rename to zml/kernels/moe/cute/mxfp4.zig index 98b16ccf..caffea33 100644 --- a/zml/moe/cute_kernels/mxfp4.zig +++ b/zml/kernels/moe/cute/mxfp4.zig @@ -3,7 +3,7 @@ //! reduction. const std = @import("std"); -const zml = @import("../../zml.zig"); +const zml = @import("../../../zml.zig"); const cute = zml.kernel.cute; const B = cute.Builder; diff --git a/zml/moe/cute_kernels/persistent_mxfp4.zig b/zml/kernels/moe/cute/persistent_mxfp4.zig similarity index 99% rename from zml/moe/cute_kernels/persistent_mxfp4.zig rename to zml/kernels/moe/cute/persistent_mxfp4.zig index 7208e6ce..920e1256 100644 --- a/zml/moe/cute_kernels/persistent_mxfp4.zig +++ b/zml/kernels/moe/cute/persistent_mxfp4.zig @@ -13,7 +13,7 @@ const std = @import("std"); const mlir = @import("mlir"); -const zml = @import("../../zml.zig"); +const zml = @import("../../../zml.zig"); const cute = zml.kernel.cute; const B = cute.Builder; const V = cute.Value; diff --git a/zml/kernels/moe/cutlass_flashinfer/BUILD.bazel b/zml/kernels/moe/cutlass_flashinfer/BUILD.bazel new file mode 100644 index 00000000..54b8604c --- /dev/null +++ b/zml/kernels/moe/cutlass_flashinfer/BUILD.bazel @@ -0,0 +1,8 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "cutlass_flashinfer", + srcs = [ + "cutlass_flashinfer.zig", + ], +) diff --git a/zml/moe/cutlass_flashinfer.zig b/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig similarity index 95% rename from zml/moe/cutlass_flashinfer.zig rename to zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig index 75b52eeb..ffaec26b 100644 --- a/zml/moe/cutlass_flashinfer.zig +++ b/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig @@ -3,7 +3,7 @@ const stdx = @import("stdx"); const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); const platforms = @import("platforms"); -const zml = @import("../zml.zig"); +const zml = @import("../../../zml.zig"); const log = std.log.scoped(.@"zml/moe/cutlass_flashinfer"); @@ -34,7 +34,7 @@ pub const Options = struct { pub fn validateOptions(opts: zml.moe.Options) void { stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); - stdx.debug.assert(opts.activation == .swiglu, "cute_mxfp4 backend only accepts swiglu activation, got {}", .{opts.activation}); + stdx.debug.assert(opts.activation == .swiglu, "flashinfer_cutlass backend only accepts swiglu activation, got {}", .{opts.activation}); stdx.debug.assert(opts.activation.swiglu.limit == null, "Activation thresholds require the Triton MoE backend", .{}); stdx.debug.assert(opts.activation.swiglu.bias == null, "flashinfer_cutlass backend requires swiglu bias to be null", .{}); stdx.debug.assert(opts.activation.swiglu.scale == null, "flashinfer_cutlass backend requires swiglu scale to be null", .{}); @@ -431,27 +431,26 @@ fn validateInputs( { return error.UnsupportedType; } - if (hidden_states.rank() != 3 or + if (hidden_states.rank() != 2 or fc1_weights.rank() != 3 or fc2_weights.rank() != 3 or - topk_weights.rank() != 3 or - topk_ids.rank() != 3) + topk_weights.rank() != 2 or + topk_ids.rank() != 2) { return error.InvalidShape; } - const batch = hidden_states.dim(0); - const sequence = hidden_states.dim(1); - const hidden_size = hidden_states.dim(2); + const tokens = hidden_states.dim(.t); + const hidden_size = hidden_states.dim(.d); const num_experts = fc1_weights.dim(0); const fc1_rows = fc1_weights.dim(1); const intermediate_size = switch (options.activation) { .silu, .gelu, .relu => fc1_rows, .swiglu, .geglu, .geglu_tanh, .swiglu_step => @divExact(fc1_rows, 2), }; - const top_k = topk_ids.dim(2); + const top_k = topk_ids.dim(.topk); - if (batch <= 0 or sequence <= 0 or hidden_size <= 0 or + if (tokens <= 0 or hidden_size <= 0 or num_experts <= 0 or intermediate_size <= 0 or @mod(hidden_size, 16) != 0 or @mod(intermediate_size, 16) != 0 or top_k <= 0 or top_k > num_experts) @@ -462,11 +461,9 @@ fn validateInputs( fc2_weights.dim(0) != num_experts or fc2_weights.dim(1) != hidden_size or fc2_weights.dim(2) != intermediate_size or - topk_weights.dim(0) != batch or - topk_weights.dim(1) != sequence or - topk_weights.dim(2) != top_k or - topk_ids.dim(0) != batch or - topk_ids.dim(1) != sequence or + topk_weights.dim(.t) != tokens or + topk_weights.dim(.topk) != top_k or + topk_ids.dim(.t) != tokens or !fc1_weight_block.shape().eql(fc1BlockScaleShape( num_experts, hidden_size, @@ -487,7 +484,7 @@ fn validateInputs( } return .{ - .num_tokens = batch * sequence, + .num_tokens = tokens, .hidden_size = hidden_size, .intermediate_size = intermediate_size, .num_experts = @intCast(num_experts), @@ -586,38 +583,35 @@ pub fn fusedExpertsBf16( fc2_weights.dtype() != .bf16 or topk_weights.dtype() != .f32 or topk_ids.dtype() != .i32 or - hidden_states.rank() != 3 or + hidden_states.rank() != 2 or fc1_weights.rank() != 3 or fc2_weights.rank() != 3 or - topk_weights.rank() != 3 or - topk_ids.rank() != 3) + topk_weights.rank() != 2 or + topk_ids.rank() != 2) { // TODO(Corentin): Better error message @panic("InvalidInput"); } - const batch = hidden_states.dim(0); - const sequence = hidden_states.dim(1); - const hiddenSize = hidden_states.dim(2); + const tokens = hidden_states.dim(.t); + const hiddenSize = hidden_states.dim(.d); const numExperts = fc1_weights.dim(0); const fc1Rows = fc1_weights.dim(1); const intermediateSize = switch (options.activation) { .silu, .gelu, .relu => fc1Rows, .swiglu, .geglu, .geglu_tanh, .swiglu_step => @divExact(fc1Rows, 2), }; - const topK = topk_ids.dim(2); - if (batch <= 0 or sequence <= 0 or hiddenSize <= 0 or + const topK = topk_ids.dim(.topk); + if (tokens <= 0 or hiddenSize <= 0 or numExperts <= 0 or intermediateSize <= 0 or topK <= 0 or topK > numExperts or fc1_weights.dim(2) != hiddenSize or fc2_weights.dim(0) != numExperts or fc2_weights.dim(1) != hiddenSize or fc2_weights.dim(2) != intermediateSize or - topk_weights.dim(0) != batch or - topk_weights.dim(1) != sequence or - topk_weights.dim(2) != topK or - topk_ids.dim(0) != batch or - topk_ids.dim(1) != sequence) + topk_weights.dim(.t) != tokens or + topk_weights.dim(.topk) != topK or + topk_ids.dim(.t) != tokens) { // TODO(Corentin): Better error message @panic("InvalidShape"); @@ -626,7 +620,7 @@ pub fn fusedExpertsBf16( const runners = currentRunners() catch |e| stdx.debug.panic("Failed to get runners: {}", .{e}); const attributes: Attributes = .{ .runners = @intFromPtr(runners), - .num_tokens = batch * sequence, + .num_tokens = tokens, .hidden_size = hiddenSize, .intermediate_size = intermediateSize, .num_experts = @intCast(numExperts), @@ -758,7 +752,7 @@ pub fn fusedExperts( ); const local_reshaped = local_output .reshape(self.input.shape().dims()) - .withTags(.{ .b, .s, .d }); + .withTags(.{ .t, .d }); return zml.ops.allReduce(local_reshaped, zml.Tensor.add); } }).body, @@ -851,7 +845,7 @@ pub fn fusedExperts( ); const local_reshaped = local_output .reshape(self.input.shape().dims()) - .withTags(.{ .b, .s, .d }); + .withTags(.{ .t, .d }); return zml.ops.allReduce(local_reshaped, zml.Tensor.add); } }).body, diff --git a/zml/kernels/moe/fly/BUILD.bazel b/zml/kernels/moe/fly/BUILD.bazel new file mode 100644 index 00000000..cb3b2ff4 --- /dev/null +++ b/zml/kernels/moe/fly/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "fly", + srcs = [ + "fly.zig", + "moe.zig", + ], +) diff --git a/zml/kernels/moe/fly/fly.zig b/zml/kernels/moe/fly/fly.zig new file mode 100644 index 00000000..f3f6ece6 --- /dev/null +++ b/zml/kernels/moe/fly/fly.zig @@ -0,0 +1,356 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const Tensor = zml.Tensor; +const Shape = zml.Shape; +const DataType = zml.DataType; +const fly = @import("moe.zig"); +const ProjectionLayout = zml.moe.ProjectionLayout; +const RoutingWeightPlacement = zml.moe.RoutingWeightPlacement; + +const log = std.log.scoped(.@"zml/moe/fused_experts"); + +test { + std.testing.refAllDecls(@This()); +} + +pub fn fusedExperts( + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + opts: zml.moe.Options, +) zml.Tensor { + const gate_up_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; + const layout: zml.moe.ExpertsLayout = if (gate_up_scheme == .mxfp4) + .{ .gate_up = .interleaved, .packing = .plain } + else + .{ .gate_up = .concatenated, .packing = .plain }; + + const args: FusedExpertsArgs = .{ + .hidden_states = input, + .gate_up = gate_up, + .down = down, + .topk_weights = topk_weights, + .topk_ids = topk_ids, + .activation = opts.activation, + .quantize_input = opts.quantize_input, + .gate_up_layout = layout.gate_up, + .routing_weight_placement = opts.routing_weight_placement, + }; + const expert_partition = gate_up.weight.shape().partition(.expert); + + if (!expert_partition.isSharded()) { + return fusedExpertsImpl(args); + } + + return zml.ops.manualComputation( + (struct { + args: FusedExpertsArgs, + global_num_experts: i64, + + fn call(self: @This(), _: zml.Shape) zml.Tensor { + const local_args = self.args; + const local_num_experts = local_args.gate_up.weight.dim(.expert); + const partition_id = zml.ops.partitionId().convert(.i32); + const expert_start = partition_id.scale(local_num_experts).convert(.i32); + const global_expert_ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); + + // Map global expert ids to local ids, or -1 for experts outside this partition. + const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) + .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); + var mapped_args = local_args; + mapped_args.expert_map = local_expert_mask.select( + global_expert_ids.sub(expert_start), + zml.Tensor.scalar(-1, .i32), + ); + + const local_output = fusedExpertsImpl(mapped_args); + const local_reshaped = local_output.reshape(local_args.hidden_states.shape()).withTags(.{ .t, .d }); + return zml.ops.allReduce(local_reshaped, zml.Tensor.add); + } + }).call, + .{ .args = args, .global_num_experts = gate_up.weight.dim(.expert) }, + input.shape(), + ); +} + +pub const FusedExpertsArgs = struct { + hidden_states: Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + topk_weights: Tensor, + topk_ids: Tensor, + activation: zml.moe.Activation, + expert_map: ?Tensor = null, + /// Use FP8 activations for FP8 weights; false keeps BF16 activations. + quantize_input: bool, + gate_up_layout: ProjectionLayout, + routing_weight_placement: RoutingWeightPlacement, +}; + +pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { + const hidden_states = opts.hidden_states; + const topk_weights = opts.topk_weights; + const topk_ids = opts.topk_ids; + const num_tokens = hidden_states.dim(.t); + var launch_config = launchConfigForTokens(num_tokens); + + const gate_up_scheme = opts.gate_up.quantizationScheme(); + const down_scheme = opts.down.quantizationScheme(); + if (gate_up_scheme) |scheme| switch (scheme) { + .nvfp4 => stdx.debug.panic("fly backend doesn't support gate_up nvfp4 quantization scheme", .{}), + .fp8_block128 => launch_config.block_size_k = 128, + .fp8_block32 => launch_config.block_size_k = 32, + .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, + }; + + var down_launch_config = launchConfigForTokens(num_tokens); + if (down_scheme) |scheme| switch (scheme) { + .nvfp4 => stdx.debug.panic("fly backend doesn't support down nvfp4 quantization scheme", .{}), + .fp8_block128 => down_launch_config.block_size_k = 128, + .fp8_block32 => down_launch_config.block_size_k = 32, + .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, + }; + + const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); + const gate_up = opts.gate_up.weight.withTags(.{ .expert, .out, .in }); + const down = opts.down.weight.withTags(.{ .expert, .out, .mid }); + const routing_weights = topk_weights.reshape(.{ .token = num_tokens, .in = topk_weights.dim(.topk) }).withTags(.{ .token, .topk }); + const ids = topk_ids.reshape(.{ .token = num_tokens, .in = topk_ids.dim(.topk) }).withTags(.{ .token, .topk }); + + stdx.debug.assert(hidden.dtype() == .bf16, "fly backend expected BF16 hidden states, got {}", .{hidden.dtype()}); + if (gate_up_scheme == .mxfp4) { + stdx.debug.assert(gate_up.dtype() == .u8 or gate_up.dtype() == .i8 or gate_up.dtype() == .f4e2m1, "fly backend expected u8, i8 or f4e2m1 gate_up data type when quantization scheme is mxfp4, got {}", .{gate_up.dtype()}); + } else { + stdx.debug.assert(gate_up.dtype() == .bf16 or gate_up.dtype() == .f8e4m3fn or gate_up.dtype() == .f8e4m3fnuz, "fly backend expected gate_up data type to be bf16, fp8e4m3fn or f8e4m3fnuz, got {}", .{gate_up.dtype()}); + } + if (down_scheme == .mxfp4) { + stdx.debug.assert(down.dtype() == .u8 or down.dtype() == .i8 or down.dtype() == .f4e2m1, "fly backend expected u8, i8 or f4e2m1 down data type when quantization scheme is mxfp4, got {}", .{down.dtype()}); + } else { + stdx.debug.assert(down.dtype() == .bf16 or down.dtype() == .f8e4m3fn or down.dtype() == .f8e4m3fnuz, "fly backend expected down data type to be bf16, fp8e4m3fn or f8e4m3fnuz, got {}", .{down.dtype()}); + } + + stdx.debug.assert(routing_weights.dtype() == .f32 or routing_weights.dtype() == .bf16, "fly backend expected f32 or bf16 routing weights, got {}", .{routing_weights.dtype()}); + stdx.debug.assert(ids.dtype() == .i32, "fly backend expected i32 expert ids, got {}", .{ids.dtype()}); + const gate_up_k = gate_up.dim(.in) * @as(i64, if (gate_up_scheme == .mxfp4 and gate_up.dtype() != .f4e2m1) 2 else 1); + const down_k = down.dim(.mid) * @as(i64, if (down_scheme == .mxfp4 and down.dtype() != .f4e2m1) 2 else 1); + stdx.debug.assert(hidden.dim(.in) == gate_up_k, "fly backend expected hidden width to match gate/up input width, got {} and {}", .{ hidden.dim(.in), gate_up_k }); + const activation_reduction: i64 = switch (opts.activation) { + .gelu, .relu, .silu => 1, + .swiglu, .swiglu_step, .geglu, .geglu_tanh => 2, + }; + stdx.debug.assert(@rem(gate_up.dim(.out), activation_reduction) == 0, "fly backend expected gate/up output width to be divisible by {}, got {}", .{ activation_reduction, gate_up.dim(.out) }); + stdx.debug.assert(down_k == @divFloor(gate_up.dim(.out), activation_reduction), "fly backend expected down input width to match activated width, got {} and {}", .{ down_k, @divFloor(gate_up.dim(.out), activation_reduction) }); + stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "fly backend expected routing ids and weights to match hidden token count, got {}, {} and {}", .{ ids.dim(.token), routing_weights.dim(.token), hidden.dim(.token) }); + stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "fly backend expected routing ids and weights to have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), routing_weights.dim(.topk) }); + stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "fly backend expected gate/up and down expert counts to match, got {} and {}", .{ gate_up.dim(.expert), down.dim(.expert) }); + if (opts.expert_map) |expert_map| + stdx.debug.assert(expert_map.dtype() == .i32 and expert_map.rank() == 1, "fly backend expected a rank-1 i32 expert map, got rank {} and dtype {}", .{ expert_map.rank(), expert_map.dtype() }); + + const num_experts = if (opts.expert_map) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); + const routing = prepareRouting(ids, num_experts, @intCast(launch_config.block_size_m)); + + const expert_ids = if (opts.expert_map) |expert_map| + expert_map.gather(.{ .expert = routing.expert_ids }, .{}).withTags(.{.g}) + else + routing.expert_ids; + + const hidden_quant, const input_scale = prepareInput(hidden, gate_up_scheme, gate_up.dtype(), opts.quantize_input); + + const gate_up_out = gemm(.{ + .input = hidden_quant, + .weight = gate_up, + .bias = opts.gate_up.bias, + .input_scale = input_scale, + .weight_scale = opts.gate_up.quantizationScales(), + .routing = routing, + .expert_ids = expert_ids, + .launch_config = launch_config, + .quant_scheme = gate_up_scheme, + .top_k = @intCast(ids.dim(.topk)), + .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), + }); + + var activated = zml.moe.applyActivation(gate_up_out, opts.activation, opts.gate_up_layout); + if (opts.routing_weight_placement == .before_down) { + const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); + activated = activated.mul(weights.broad(activated.shape())); + } + const activated_quant, const down_input_scale = prepareInput(activated, down_scheme, down.dtype(), opts.quantize_input); + + const down_out = gemm(.{ + .input = activated_quant, + .weight = down, + .bias = opts.down.bias, + .input_scale = down_input_scale, + .weight_scale = opts.down.quantizationScales(), + .routing_weights = if (opts.routing_weight_placement == .after_down) routing_weights else null, + .routing = routing, + .expert_ids = expert_ids, + .launch_config = down_launch_config, + .quant_scheme = down_scheme, + .top_k = 1, + .output_shape = Shape.init(.{ .token = num_tokens, .topk = ids.dim(.topk), .out = down.dim(.out) }, .bf16), + }); + + const output = down_out.convert(.f32).sum(.topk).squeeze(.topk).convert(.bf16); + + return output.reshape(.{ .t = num_tokens, .d = down.dim(.out) }); +} + +// TODO(Corentin): unused for now +pub fn supports(opts: FusedExpertsArgs) bool { + if (opts.hidden_states.dtype() != .bf16 or opts.gate_up.quantizationScheme() != .mxfp4 or + opts.down.quantizationScheme() != .mxfp4 or opts.gate_up.bias != null or opts.down.bias != null or + opts.routing_weight_placement != .before_down) return false; + const gate_up = opts.gate_up.weight; + const down = opts.down.weight; + if (gate_up.rank() != 3 or down.rank() != 3 or opts.hidden_states.dim(.d) != 5120) return false; + const gate_up_scale = opts.gate_up.quantizationScales() orelse return false; + const down_scale = opts.down.quantizationScales() orelse return false; + const experts = if (opts.expert_map) |map| map.dim(.expert) else gate_up.dim(0); + if (experts < 1 or experts > 512) return false; + inline for (.{ .{ gate_up, gate_up_scale, 4608, 5120 }, .{ down, down_scale, 5120, 2304 } }) |projection| { + const weight, const scale, const n, const k = projection; + const packing_factor: i64 = switch (weight.dtype()) { + .u8, .i8 => 2, + .f4e2m1 => 1, + else => return false, + }; + if (weight.dim(0) < 1 or weight.dim(0) > 512 or weight.dim(1) != n or weight.dim(2) * packing_factor != k or + scale.rank() != 3 or scale.dim(0) != weight.dim(0) or scale.dim(1) != n or scale.dim(2) != k / 32) return false; + switch (scale.dtype()) { + .u8, .i8, .f8e8m0 => {}, + else => return false, + } + } + const routes = opts.hidden_states.dim(.t) * opts.topk_ids.dim(.topk); + return routes > 0 and routes <= std.math.maxInt(i32) / 5120; +} + +pub fn prepareInput(x: Tensor, scheme: ?zml.Quantization.Scheme, _: zml.DataType, _: bool) struct { Tensor, ?Tensor } { + std.debug.assert(scheme == .mxfp4); + return .{ x.convert(.bf16), null }; +} + +pub const GemmOptions = struct { + input: Tensor, + weight: Tensor, + bias: ?Tensor = null, + input_scale: ?Tensor = null, + weight_scale: ?Tensor = null, + routing_weights: ?Tensor = null, + routing: Routing, + expert_ids: Tensor, + launch_config: LaunchConfig, + quant_scheme: ?zml.Quantization.Scheme = null, + top_k: usize, + output_shape: Shape, +}; + +pub fn gemm(opts: GemmOptions) zml.Tensor { + zml.stdx.debug.assert(opts.quant_scheme == .mxfp4 and opts.bias == null and opts.routing_weights == null and opts.input_scale == null, "Fly MoE requires MXFP4 weights, BF16 activations and routing weights before the down projection", .{}); + const weight = if (opts.weight.dtype() == .f4e2m1) + opts.weight.reshape(opts.weight.shape().setDim(2, @divExact(opts.weight.dim(2), 2)).append(.{ .nibble = 2 })).bitCast(.u8) + else + opts.weight; + return fly.Mfma.call(opts.input, weight, opts.weight_scale.?.bitCast(.u8), opts.routing.sorted_token_ids, opts.expert_ids, opts.routing.num_tokens_post_padded, opts.output_shape, opts.top_k, opts.routing.naive_block_assignment, opts.launch_config.block_size_m); +} + +pub const Routing = struct { + sorted_token_ids: Tensor, + expert_ids: Tensor, + num_tokens_post_padded: Tensor, + max_num_tokens_padded: i64, + num_assignments: i64, + naive_block_assignment: bool, +}; + +pub fn prepareRouting(topk_ids: Tensor, num_experts: i64, block_size_m: i64) Routing { + std.debug.assert(num_experts > 0 and num_experts <= 512); + std.debug.assert(block_size_m > 0 and block_size_m <= 128 and @mod(block_size_m, 16) == 0); + const ids = topk_ids.withTags(.{ .token, .topk }).convert(.i32); + const num_assignments = ids.dim(.token) * ids.dim(.topk); + // GEMV handles up to six routes; MFMA shares weights across sorted rows. + const naive_block_assignment = num_assignments <= 6; + const max_num_tokens_padded = if (naive_block_assignment or num_assignments < num_experts) + num_assignments * block_size_m + else + (std.math.divCeil(i64, num_assignments + num_experts * (block_size_m - 1), block_size_m) catch unreachable) * block_size_m; + const sorted_token_ids, const expert_ids, const num_tokens_post_padded = if (naive_block_assignment) blk: { + break :blk .{ + Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)), + ids.reshape(.{ .g = num_assignments }), + Tensor.constant(.{ .i32 = @as(i32, @intCast(max_num_tokens_padded)) }).reshape(.{1}), + }; + } else blk: { + const flat_experts = ids.reshape(.{ .g = num_assignments }); + const aligned = fly.Align.call( + flat_experts, + Tensor.zeroes(Shape.init(.{ .g = max_num_tokens_padded }, .i32)), + Tensor.zeroes(Shape.init(.{ .g = @divExact(max_num_tokens_padded, block_size_m) }, .i32)), + Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)), + Tensor.zeroes(Shape.init(.{ .g = num_experts + 1 }, .i32)), + block_size_m, + ); + const sorted = fly.Sort.call(flat_experts, aligned.sorted_token_ids, aligned.cumsum); + break :blk .{ sorted.sorted_token_ids, aligned.expert_ids, aligned.num_tokens_post_pad }; + }; + return .{ + .sorted_token_ids = sorted_token_ids, + .expert_ids = expert_ids, + .num_tokens_post_padded = num_tokens_post_padded, + .max_num_tokens_padded = max_num_tokens_padded, + .num_assignments = num_assignments, + .naive_block_assignment = naive_block_assignment, + }; +} + +// ============================================================================= +// Config / validation helpers +// ============================================================================= + +const LaunchConfig = struct { + block_size_m: usize, + block_size_n: usize, + block_size_k: usize, + group_size_m: usize, + num_warps: i32, + num_stages: i32, +}; + +const launch_configs = [_]struct { tokens: i64, config: LaunchConfig }{ + .{ .tokens = 1, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 4 } }, + .{ .tokens = 2, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 4 } }, + .{ .tokens = 4, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, + .{ .tokens = 8, .config = .{ .block_size_m = 16, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 16, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 64, .group_size_m = 64, .num_warps = 4, .num_stages = 5 } }, + .{ .tokens = 24, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, + .{ .tokens = 32, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 128, .group_size_m = 1, .num_warps = 4, .num_stages = 2 } }, + .{ .tokens = 48, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 128, .group_size_m = 64, .num_warps = 4, .num_stages = 2 } }, + .{ .tokens = 64, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 128, .group_size_m = 1, .num_warps = 4, .num_stages = 2 } }, + .{ .tokens = 96, .config = .{ .block_size_m = 16, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 128, .config = .{ .block_size_m = 16, .block_size_n = 256, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, + .{ .tokens = 256, .config = .{ .block_size_m = 16, .block_size_n = 256, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, + .{ .tokens = 512, .config = .{ .block_size_m = 32, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 1024, .config = .{ .block_size_m = 64, .block_size_n = 128, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, + .{ .tokens = 1536, .config = .{ .block_size_m = 64, .block_size_n = 128, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, + .{ .tokens = 2048, .config = .{ .block_size_m = 128, .block_size_n = 128, .block_size_k = 64, .group_size_m = 16, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 3072, .config = .{ .block_size_m = 128, .block_size_n = 256, .block_size_k = 64, .group_size_m = 1, .num_warps = 8, .num_stages = 4 } }, + .{ .tokens = 4096, .config = .{ .block_size_m = 128, .block_size_n = 256, .block_size_k = 64, .group_size_m = 16, .num_warps = 8, .num_stages = 4 } }, +}; + +fn launchConfigForTokens(num_tokens: i64) LaunchConfig { + var best = launch_configs[0]; + for (launch_configs[1..]) |candidate| { + // The table is ascending, so ties retain the smaller token bucket. + if (@abs(candidate.tokens - num_tokens) < @abs(best.tokens - num_tokens)) { + best = candidate; + } + } + return best.config; +} diff --git a/zml/kernels/moe/fly/moe.zig b/zml/kernels/moe/fly/moe.zig new file mode 100644 index 00000000..ea3cdd62 --- /dev/null +++ b/zml/kernels/moe/fly/moe.zig @@ -0,0 +1,661 @@ +const std = @import("std"); +const zml = @import("../../../zml.zig"); +const Tensor = zml.Tensor; +const Shape = zml.Shape; +const fly = zml.kernel.fly; +const Builder = fly.Builder; +const Value = fly.Value; + +pub const Align = struct { + const Result = struct { + sorted_token_ids: Tensor, + expert_ids: Tensor, + num_tokens_post_pad: Tensor, + cumsum: Tensor, + }; + + fn supports(ids: Tensor, sorted: Tensor, experts: Tensor, total: Tensor, cumsum: Tensor, block_size: i64) bool { + for ([_]Tensor{ ids, sorted, experts, total, cumsum }) |tensor| { + if (tensor.rank() != 1 or tensor.dtype() != .i32) return false; + } + const num_experts = cumsum.dim(0) - 1; + if (num_experts < 1 or num_experts > 512 or total.dim(0) != 1 or ids.dim(0) > std.math.maxInt(i32) or + block_size <= 0 or block_size > 128 or @mod(block_size, 16) != 0) return false; + const capacity_bound = if (ids.dim(0) < num_experts) + ids.dim(0) * block_size + else + @divTrunc(ids.dim(0) + num_experts * (block_size - 1) + block_size - 1, block_size) * block_size; + return capacity_bound <= std.math.maxInt(i32) and sorted.dim(0) >= capacity_bound and + sorted.dim(0) <= std.math.maxInt(i32) and @mod(sorted.dim(0), block_size) == 0 and + experts.dim(0) == @divTrunc(sorted.dim(0), block_size); + } + + /// All four outputs alias their corresponding input buffers. Invalid expert IDs + /// are ignored, and all unused expert blocks are marked -1. Prefix ends in LDS + /// let each thread find its output block's expert without scanning every expert. + pub fn call(ids: Tensor, sorted: Tensor, experts: Tensor, total: Tensor, cumsum: Tensor, block_size: i64) Result { + std.debug.assert(Align.supports(ids, sorted, experts, total, cumsum, block_size)); + const cfg: Config = .{ + .routes = ids.dim(0), + .experts = cumsum.dim(0) - 1, + .capacity = sorted.dim(0), + .blocks = experts.dim(0), + .block_size = block_size, + .padding = block_size - 1, + }; + const result = Kernel.call(.{ .ids = ids, .sorted = sorted, .experts = experts, .total = total, .cumsum_in = cumsum }, .{ .sorted_token_ids = sorted.shape(), .expert_ids = experts.shape(), .num_tokens_post_pad = total.shape(), .cumsum = cumsum.shape() }, .{ + .cfg = cfg, + .threads = 512, + .grid = .{ 2, 1, 1 }, + .output_operand_aliases = .{ .sorted_token_ids = .sorted, .expert_ids = .experts, .num_tokens_post_pad = .total, .cumsum = .cumsum_in }, + }); + return .{ .sorted_token_ids = result.sorted_token_ids, .expert_ids = result.expert_ids, .num_tokens_post_pad = result.num_tokens_post_pad, .cumsum = result.cumsum }; + } + + const Config = struct { + routes: i64, + experts: i64, + capacity: i64, + blocks: i64, + block_size: i64, + padding: i64, + }; + + const Kernel = zml.kernel.fly.Kernel(Config, .{ + .name = "fly_moe_align", + .inputs = &.{ "ids", "sorted", "experts", "total", "cumsum_in" }, + .outputs = &.{ "sorted_token_ids", "expert_ids", "num_tokens_post_pad", "cumsum" }, + .run = run, + }); + + fn run(b: *Builder, cfg: Config) fly.FinishError!void { + const args = Kernel.args(b); + const tid = b.threadId(.x); + const tid64 = tid.to(.i64); + const zero = b.constant(.i32, 0); + const one = b.constant(.i32, 1); + const experts = b.constant(.i32, cfg.experts); + // Distinct arrays match the histogram, inclusive ends and wave totals. + const counts = b.sharedArray(.i32, 512, 16); + const ends = b.sharedArray(.i32, 512, 16); + const waves = b.sharedArray(.i32, 8, 16); + var fill = b.openIfElse(b.blockId(.x).cmp(.ne, 0), .{}); + var positions = b.openFor(tid64, cfg.capacity, 512, .{}); + args.sorted_token_ids.set(positions.iv, b.constant(.i32, cfg.routes)); + positions.yield(.{}); + fill.yieldThen(.{}); + + counts.set(tid, zero); + b.barrier(); + var routes = b.openFor(tid64, cfg.routes, 512, .{}); + const id = args.ids.at(routes.iv); + var valid_id = b.openIf(id.cmp(.ge, 0).bitAnd(id.cmp(.lt, cfg.experts))); + _ = b.ptrAtomicAdd(counts.emitIter().addOffset(id), one, .workgroup); + valid_id.yieldThen(.{}); + routes.yield(.{}); + b.barrier(); + + const padded = counts.at(tid).add(cfg.padding).divUnsigned(cfg.block_size).mul(cfg.block_size); + const lane = tid.remUnsigned(64); + const wave = tid.divUnsigned(64); + var prefix = padded; + inline for (.{ 1, 2, 4, 8, 16, 32 }) |shift| { + const peer = prefix.shuffle(.up, shift, 64); + prefix = prefix.add(lane.cmpUnsigned(.ge, shift).select(peer, zero)); + } + var last_lane = b.openIf(lane.cmp(.eq, 63)); + waves.set(wave, prefix); + last_lane.yieldThen(.{}); + b.barrier(); + + var sums = b.openFor(zero, 8, 1, .{ zero, zero }); + const value = waves.at(sums.iv); + const preceding = sums.carried[0].add(sums.iv.cmpUnsigned(.lt, wave).select(value, zero)); + const total = sums.carried[1].add(value); + sums.yield(.{ preceding, total }); + const end = sums.results[0].add(prefix); + const start = end.sub(padded); + ends.set(tid, end); + var has_expert = b.openIf(tid.cmpUnsigned(.lt, cfg.experts)); + args.cumsum.set(tid64, start); + has_expert.yieldThen(.{}); + var first = b.openIf(tid.cmp(.eq, 0)); + args.cumsum.set(b.constant(.i64, cfg.experts), sums.results[1]); + args.num_tokens_post_pad.set(b.constant(.i64, 0), sums.results[1]); + first.yieldThen(.{}); + b.barrier(); + + var blocks = b.openFor(tid64, cfg.blocks, 512, .{}); + const offset = blocks.iv.to(.i32).mul(cfg.block_size); + var block_expert = b.openIfElse(offset.cmpUnsigned(.lt, sums.results[1]), .{zero.type_()}); + // Upper bound of inclusive expert ends handles empty experts without a scan. + var bounds = b.openWhile(.{ zero, experts }, .{ zero.type_(), zero.type_() }); + bounds.yieldBefore(bounds.before_carried[0].cmpUnsigned(.lt, bounds.before_carried[1]), .{ bounds.before_carried[0], bounds.before_carried[1] }); + const lo = bounds.after_carried[0]; + const hi = bounds.after_carried[1]; + const mid = lo.add(hi).shrU(1); + const advance = ends.at(mid).cmpUnsigned(.le, offset); + bounds.yieldAfter(.{ advance.select(mid.add(1), lo), advance.select(hi, mid) }); + block_expert.yieldThen(.{bounds.results[0]}); + block_expert.yieldElse(.{b.constant(.i32, -1)}); + args.expert_ids.set(blocks.iv, block_expert.results[0]); + blocks.yield(.{}); + fill.yieldElse(.{}); + } +}; + +pub const Sort = struct { + const Result = struct { + sorted_token_ids: Tensor, + cumsum: Tensor, + }; + + fn supports(ids: Tensor, sorted_token_ids: Tensor, cumsum: Tensor) bool { + return ids.rank() == 1 and sorted_token_ids.rank() == 1 and cumsum.rank() == 1 and + ids.dtype() == .i32 and sorted_token_ids.dtype() == .i32 and cumsum.dtype() == .i32 and + cumsum.dim(0) >= 2 and cumsum.dim(0) <= std.math.maxInt(i32) and + ids.dim(0) <= std.math.maxInt(i32) and sorted_token_ids.dim(0) >= ids.dim(0); + } + + /// cumsum contains one starting offset per expert followed by the total padded + /// capacity. Each valid route atomically advances its expert's cursor; ordering + /// within an expert is unspecified. Invalid expert IDs leave both buffers alone. + pub fn call(ids: Tensor, sorted_token_ids: Tensor, cumsum: Tensor) Result { + std.debug.assert(Sort.supports(ids, sorted_token_ids, cumsum)); + if (ids.dim(0) == 0) return .{ .sorted_token_ids = sorted_token_ids, .cumsum = cumsum }; + const routes = ids.dim(0); + const grid = @min(@divTrunc(routes + 255, 256), 65535); + const cfg: Config = .{ .routes = routes, .experts = cumsum.dim(0) - 1, .step = grid * 256 }; + const result = Kernel.call(.{ .ids = ids, .sorted = sorted_token_ids, .cumsum_in = cumsum }, .{ .sorted_token_ids = sorted_token_ids.shape(), .cumsum = cumsum.shape() }, .{ + .cfg = cfg, + .threads = 256, + .grid = .{ @intCast(grid), 1, 1 }, + .output_operand_aliases = .{ .sorted_token_ids = .sorted, .cumsum = .cumsum_in }, + }); + return .{ .sorted_token_ids = result.sorted_token_ids, .cumsum = result.cumsum }; + } + + const Config = struct { + routes: i64, + experts: i64, + step: i64, + }; + + const Kernel = zml.kernel.fly.Kernel(Config, .{ + .name = "fly_count_and_sort_expert_tokens", + .inputs = &.{ "ids", "sorted", "cumsum_in" }, + .outputs = &.{ "sorted_token_ids", "cumsum" }, + .run = run, + }); + + fn run(b: *Builder, cfg: Config) fly.FinishError!void { + const args = Kernel.args(b); + const start = b.blockId(.x).to(.i64).mul(256).add(b.threadId(.x).to(.i64)); + var routes = b.openFor(start, cfg.routes, cfg.step, .{}); + const id = args.ids.at(routes.iv); + var valid = b.openIf(id.cmp(.ge, 0).bitAnd(id.cmp(.lt, cfg.experts))); + const cursor = args.cumsum.emitIter().addOffset(id.to(.i64)); + const rank = b.ptrAtomicAdd(cursor, b.constant(.i32, 1), .agent); + args.sorted_token_ids.set(rank.to(.i64), routes.iv.to(.i32)); + valid.yieldThen(.{}); + routes.yield(.{}); + } +}; + +pub const Mfma = struct { + /// The caller owns route localization, activation and reduction. + /// `expert_ids` contains local bank indices, including -1 for remote experts. + /// Sorted routes must contain a real prefix followed by padding per expert. + /// `block_m` is their alignment; native tiles contain 16 or 32 rows. + pub fn call( + input: Tensor, + weight: Tensor, + scales: Tensor, + sorted_ids: Tensor, + expert_ids: Tensor, + padded_count: Tensor, + output: zml.Shape, + topk: usize, + naive: bool, + block_m: usize, + ) Tensor { + std.debug.assert(input.rank() == 2 and input.dtype() == .bf16 and weight.rank() == 3 and + (weight.dtype() == .u8 or weight.dtype() == .i8) and scales.rank() == 3 and scales.dtype() == .u8 and + sorted_ids.rank() == 1 and sorted_ids.dtype() == .i32 and expert_ids.rank() == 1 and expert_ids.dtype() == .i32 and + padded_count.count() == 1 and padded_count.dtype() == .i32 and output.rank() >= 2 and output.dtype() == .bf16 and + topk > 0 and block_m > 0 and block_m % 16 == 0); + const n = weight.dim(1); + const k = input.dim(1); + std.debug.assert((n == 4608 and k == 5120) or (n == 5120 and k == 2304)); + std.debug.assert(input.dim(0) > 0 and weight.dim(0) > 0 and weight.dim(2) == @divExact(k, 2) and + scales.dim(0) == weight.dim(0) and scales.dim(1) == n and scales.dim(2) == @divExact(k, 32) and + output.dim(-1) == n); + // Keep packed-word counts and route/output indices within signed i32. + std.debug.assert(weight.count() / 4 <= std.math.maxInt(i32) and output.count() <= std.math.maxInt(i32) and + sorted_ids.count() <= std.math.maxInt(i32)); + const routes: i64 = @intCast(output.count() / @as(usize, @intCast(n))); + std.debug.assert(topk <= routes and @mod(@as(usize, @intCast(routes)), topk) == 0 and + input.dim(0) == @divTrunc(routes, @as(i64, @intCast(topk)))); + const capacity: i64 = @intCast(sorted_ids.count()); + const blocks: i64 = @intCast(expert_ids.count()); + if (naive) { + std.debug.assert(blocks == routes); + } else { + std.debug.assert(capacity > 0 and block_m <= capacity and @mod(capacity, @as(i64, @intCast(block_m))) == 0 and + blocks == @divExact(capacity, @as(i64, @intCast(block_m)))); + } + if (naive and routes <= 6) return Gemv.call(input, weight, scales, expert_ids, output, topk); + return callMfma(input, weight, scales, sorted_ids, expert_ids, padded_count, output, topk, naive, block_m); + } + + fn callMfma(input: Tensor, weight: Tensor, scales: Tensor, sorted_ids: Tensor, expert_ids: Tensor, padded_count: Tensor, output: zml.Shape, topk: usize, naive: bool, block_m: usize) Tensor { + const n = weight.dim(1); + const k = input.dim(1); + const routes = input.dim(0) * @as(i64, @intCast(topk)); + const native_m: i64 = if (!naive and block_m % 32 == 0) 32 else 16; + const cfg: Config = .{ + .tokens = input.dim(0), + .experts = weight.dim(0), + .n = n, + .k = k, + .ka = @divExact(k, 4), + .kb = @divExact(k, 8), + .ks = @divExact(k, 32), + .topk = @as(i64, @intCast(topk)), + .routes = routes, + .capacity = @as(i64, @intCast(sorted_ids.count())), + .blocks = @as(i64, @intCast(expert_ids.count())), + .subtiles = @divExact(@as(i64, @intCast(block_m)), native_m), + .native_m = native_m, + .wide = @as(i64, @intFromBool(native_m == 32)), + .stage_elements = native_m * 128, + .shared_elements = native_m * 256, + .naive = @as(i64, @intFromBool(naive)), + }; + const a = input.reshape(.{ input.dim(0), @divExact(k, 4), 4 }).bitCast(.i64); + const b = weight.reshape(.{ weight.dim(0), n, @divExact(k, 8), 4 }).bitCast(.i32); + return Kernel.call(.{ .a = a, .b = b, .scales = scales, .sorted = sorted_ids, .experts = expert_ids, .padded = padded_count.reshape(.{1}) }, .{ + .output = zml.Shape.init(.{ routes, n }, .bf16), + }, .{ + .cfg = cfg, + .threads = 256, + .grid = .{ @intCast(@divExact(n, 64)), @intCast(if (naive) routes else @divExact(cfg.capacity, native_m)), 1 }, + }).output.reshape(output); + } + + const Config = struct { + tokens: i64, + experts: i64, + n: i64, + k: i64, + ka: i64, + kb: i64, + ks: i64, + topk: i64, + routes: i64, + capacity: i64, + blocks: i64, + subtiles: i64, + native_m: i64, + wide: i64, + stage_elements: i64, + shared_elements: i64, + naive: i64, + }; + + const Kernel = zml.kernel.fly.Kernel(Config, .{ + .name = "mxfp4_raw_mfma", + .inputs = &.{ "a", "b", "scales", "sorted", "experts", "padded" }, + .outputs = &.{"output"}, + .run = run, + }); + + fn run(b: *Builder, cfg: Config) fly.FinishError!void { + const args = Kernel.args(b); + const zero = b.constant(.i32, 0); + const fzero = b.constant(.f32, 0.0); + const zeros = fzero.splat(4); + const block_n = b.blockId(.x).to(.i64); + const block_m = b.blockId(.y).to(.i64); + const route_start = block_m.mul(cfg.native_m); + const has_rows = if (cfg.naive != 0) b.constant(.i1, 1) else blk: { + const padded = args.padded.at(b.constant(.i64, 0)).to(.i64); + var has_block = b.openIfElse(route_start.cmpUnsigned(.lt, padded), .{b.constant(.i1, 0).type_()}); + has_block.yieldThen(.{args.sorted.at(route_start).cmp(.ne, cfg.routes)}); + has_block.yieldElse(.{b.constant(.i1, 0)}); + break :blk has_block.results[0]; + }; + var block = b.openIf(has_rows); + const expert_block = if (cfg.naive != 0) block_m else block_m.divUnsigned(cfg.subtiles); + const eid = args.experts.at(expert_block); + const valid_expert = eid.cmp(.ge, 0).bitAnd(eid.cmp(.lt, cfg.experts)); + const thread = b.threadId(.x).to(.i64); + const lane = thread.remUnsigned(64); + const wave = thread.divUnsigned(64); + const mn_lane = lane.remUnsigned(16); + const k_lane = lane.divUnsigned(16); + const out_n = block_n.mul(64).add(wave.mul(16)).add(mn_lane); + // Each expert has a contiguous real prefix before route-count padding. + const second_active = if (cfg.wide != 0) blk: { + const first = args.sorted.at(route_start.add(16)); + break :blk first.cmp(.ge, 0).bitAnd(first.cmp(.lt, cfg.routes)); + } else b.constant(.i1, 0); + + var sums = b.openIfElse(valid_expert, .{ zeros.type_(), zeros.type_() }); + const expert = eid.to(.i64); + const load_row = thread.divUnsigned(16); + const load_chunk = thread.remUnsigned(16); + const a_route = routeForRow(b, args.sorted, block_m, route_start, load_row, cfg.naive != 0); + const a_valid = a_route.cmp(.ge, 0).bitAnd(a_route.cmp(.lt, cfg.routes)); + const a_base = a_route.to(.i64).divUnsigned(cfg.topk).mul(cfg.ka); + var second_route = b.openIfElse(second_active, .{zero.type_()}); + second_route.yieldThen(.{args.sorted.at(route_start.add(load_row).add(16))}); + second_route.yieldElse(.{b.constant(.i32, -1)}); + const a_route_g1 = second_route.results[0]; + const a_valid_g1 = a_route_g1.cmp(.ge, 0).bitAnd(a_route_g1.cmp(.lt, cfg.routes)); + const a_base_g1 = a_route_g1.to(.i64).divUnsigned(cfg.topk).mul(cfg.ka); + + // One allocation, two disjoint ping-pong regions; never two identical leaves. + const shared = b.sharedArray(.i16, cfg.shared_elements, 16); + const load_a = b.copyAtom(.{ .universal = 128 }, .i64); + const load_b = b.copyAtom(.{ .universal = 128 }, .i32); + const copy_shared = b.copyAtom(.{ .universal = 128 }, .i16); + const loader: TileLoader = .{ + .a = args.a, + .weight = args.b, + .scales = args.scales, + .shared = shared, + .load_a = load_a, + .load_b = load_b, + .copy_shared = copy_shared, + .a_base = a_base, + .a_base_g1 = a_base_g1, + .a_valid = a_valid, + .a_valid_g1 = a_valid_g1, + .second_active = second_active, + .a_col = load_chunk.mul(2), + .a_store = load_row.mul(128).add(load_chunk.bitXor(load_row).mul(8)), + .b_base = expert.mul(cfg.n * cfg.kb).add(out_n.mul(cfg.kb)), + .expert = expert, + .out_n = out_n, + .k_lane = k_lane, + }; + const initial = loader.load(b, b.constant(.i64, 0), b.constant(.i64, 0)); + b.barrier(); + const atom = b.mmaAtom((fly.rocdl.MmaOpCDNA3MFMAType.get(b.ctx, .{ + .m = 16, + .n = 16, + .k = 16, + .elemTyA = fly.DType.bf16.toMlir(b.ctx), + .elemTyB = fly.DType.bf16.toMlir(b.ctx), + .elemTyAcc = fly.DType.f32.toMlir(b.ctx), + }) catch return error.InvalidMlir).type_()); + var k = b.openFor(b.constant(.i64, 0), cfg.k, 128, .{ zeros, zeros, initial.words, initial.scale }); + const read_base = k.iv.divUnsigned(128).bitAnd(1).mul(cfg.stage_elements); + const write_base = k.iv.divUnsigned(128).bitAnd(1).bitXor(1).mul(cfg.stage_elements); + var a_fragments: [4]Value = undefined; + var a_fragments_g1: [4]Value = undefined; + inline for (0..4) |u| { + const offset = read_base.add(mn_lane.mul(128)).add(k_lane.mul(4).add(u).bitXor(mn_lane).mul(8)); + a_fragments[u] = b.copyAtomLoad(copy_shared, vectorView(shared, offset, 8)); + } + inline for (0..4) |u| { + const offset = read_base.add(mn_lane.mul(128)).add(k_lane.mul(4).add(u).bitXor(mn_lane).mul(8)).add(2048); + const azero = b.constant(.i16, 0).splat(8); + var second = b.openIfElse(second_active, .{azero.type_()}); + second.yieldThen(.{b.copyAtomLoad(copy_shared, vectorView(shared, offset, 8))}); + second.yieldElse(.{azero}); + a_fragments_g1[u] = second.results[0]; + } + // Prefetch the other LDS region only after all current A fragments are read. + const next_k = k.iv.add(128); + var next = b.openIfElse(next_k.cmpUnsigned(.lt, cfg.k), .{ initial.words.type_(), initial.scale.type_() }); + const next_tile = loader.load(b, next_k, write_base); + next.yieldThen(.{ next_tile.words, next_tile.scale }); + next.yieldElse(.{ b.constant(.i32, 0).splat(4), fzero }); + var acc = k.carried[0]; + var acc_g1 = k.carried[1]; + inline for (0..4) |u| { + const decoded = decodeWeight(b, k.carried[2].extract(u), k.carried[3]); + const a0 = a_fragments[u].shuffleVector(a_fragments[u], &.{ 0, 1, 2, 3 }).bitcast(.bf16); + const a1 = a_fragments[u].shuffleVector(a_fragments[u], &.{ 4, 5, 6, 7 }).bitcast(.bf16); + const w0 = decoded.shuffleVector(decoded, &.{ 0, 1, 2, 3 }); + const w1 = decoded.shuffleVector(decoded, &.{ 4, 5, 6, 7 }); + acc = b.mmaAtomCall(atom, a0, w0, acc); + acc = b.mmaAtomCall(atom, a1, w1, acc); + var second = b.openIfElse(second_active, .{zeros.type_()}); + const a0_g1 = a_fragments_g1[u].shuffleVector(a_fragments_g1[u], &.{ 0, 1, 2, 3 }).bitcast(.bf16); + const a1_g1 = a_fragments_g1[u].shuffleVector(a_fragments_g1[u], &.{ 4, 5, 6, 7 }).bitcast(.bf16); + const first_acc = b.mmaAtomCall(atom, a0_g1, w0, acc_g1); + second.yieldThen(.{b.mmaAtomCall(atom, a1_g1, w1, first_acc)}); + second.yieldElse(.{acc_g1}); + acc_g1 = second.results[0]; + } + // All current reads finish before a later iteration reuses this LDS region. + b.barrier(); + k.yield(.{ acc, acc_g1, next.results[0], next.results[1] }); + sums.yieldThen(.{ k.results[0], k.results[1] }); + sums.yieldElse(.{ zeros, zeros }); + + inline for (0..4) |i| { + storeRow(b, args.output, args.sorted, block_m, route_start, k_lane.mul(4).add(i), out_n, sums.results[0].extract(i), cfg); + } + var store_second = b.openIf(second_active); + inline for (0..4) |i| { + storeRow(b, args.output, args.sorted, block_m, route_start, k_lane.mul(4).add(i + 16), out_n, sums.results[1].extract(i), cfg); + } + store_second.yieldThen(.{}); + block.yieldThen(.{}); + } + + fn routeForRow(b: *Builder, sorted: Value, block: Value, start: Value, row: Value, naive: bool) Value { + return if (naive) row.cmp(.eq, 0).select(block.to(.i32), b.constant(.i32, -1)) else sorted.at(start.add(row)); + } + + fn storeRow(b: *Builder, output: Value, sorted: Value, block: Value, start: Value, row: Value, col: Value, value: Value, cfg: Config) void { + const route = routeForRow(b, sorted, block, start, row, cfg.naive != 0); + var valid = b.openIf(route.cmp(.ge, 0).bitAnd(route.cmp(.lt, cfg.routes))); + output.set(.{ route.to(.i64), col }, value.to(.bf16)); + valid.yieldThen(.{}); + } + + fn vectorView(tensor: Value, offset: Value, comptime n: i64) Value { + return tensor.emitIter().addOffset(offset).view(fly.L(n, 1)); + } + + const TileLoader = struct { + a: Value, + weight: Value, + scales: Value, + shared: Value, + load_a: Value, + load_b: Value, + copy_shared: Value, + a_base: Value, + a_base_g1: Value, + a_valid: Value, + a_valid_g1: Value, + second_active: Value, + a_col: Value, + a_store: Value, + b_base: Value, + expert: Value, + out_n: Value, + k_lane: Value, + + fn load(self: TileLoader, b: *Builder, k: Value, write_base: Value) struct { words: Value, scale: Value } { + self.storeActivation(b, self.a_base, self.a_valid, k, self.a_store.add(write_base)); + var second = b.openIf(self.second_active); + self.storeActivation(b, self.a_base_g1, self.a_valid_g1, k, self.a_store.add(write_base).add(2048)); + second.yieldThen(.{}); + const offset = self.b_base.add(k.divUnsigned(8)).add(self.k_lane.mul(4)); + const words = b.copyAtomLoad(self.load_b, vectorView(self.weight, offset, 4)); + const byte = self.scales.at(.{ self.expert, self.out_n, k.divUnsigned(32).add(self.k_lane) }); + const scale = byte.toUnsigned(.i32); + const finite = scale.cmp(.eq, 0).select(b.constant(.i32, 0x00400000), scale.shl(23)); + return .{ .words = words, .scale = scale.cmp(.eq, 255).select(b.constant(.i32, 0x7fc00000), finite).bitcast(.f32) }; + } + + fn storeActivation(self: TileLoader, b: *Builder, base: Value, valid: Value, k: Value, destination: Value) void { + const zeros = b.constant(.i64, 0).splat(2); + var guarded = b.openIfElse(valid, .{zeros.type_()}); + const offset = base.add(k.divUnsigned(4)).add(self.a_col); + guarded.yieldThen(.{b.copyAtomLoad(self.load_a, vectorView(self.a, offset, 2))}); + guarded.yieldElse(.{zeros}); + b.copyAtomStore(self.copy_shared, guarded.results[0].bitcast(.i16), vectorView(self.shared, destination, 8)); + } + }; + + fn decodeWeight(b: *Builder, word: Value, scale: Value) Value { + const lo = b.constant(.i32, 0x44403800); + const hi = b.constant(.i32, 0x54504c48); + const even = b.perm(hi, lo, word.bitAnd(0x07070707)); + const odd = b.perm(hi, lo, word.shrU(4).bitAnd(0x07070707)); + const ev0 = b.cvtPkF32Fp8(even, false); + const od0 = b.cvtPkF32Fp8(odd, false); + const ev1 = b.cvtPkF32Fp8(even, true); + const od1 = b.cvtPkF32Fp8(odd, true); + const first = ev0.shuffleVector(od0, &.{ 0, 2, 1, 3 }); + const last = ev1.shuffleVector(od1, &.{ 0, 2, 1, 3 }); + const magnitude = first.shuffleVector(last, &.{ 0, 1, 2, 3, 4, 5, 6, 7 }); + var shifts = b.constant(.i32, 0).splat(8); + inline for (0..8) |i| shifts = shifts.insert(i, b.constant(.i32, i * 4)); + const signs = word.splat(8).shrU(shifts).bitAnd(8).shl(28); + // FNUZ has no -0: restore FP4's sign in F32 before scaling and BF16 rounding. + return magnitude.bitcast(.i32).bitOr(signs).bitcast(.f32).mul(scale).to(.bf16); + } +}; + +const Gemv = struct { + // Packed-word GEMV derived from the supplied moe-mi300x/fly_word.py algorithm. + // Several rows share a CTA, and the shorter down projection uses subwaves to + // expose independent outputs without paying for a full wave reduction per row. + const Config = struct { rows: usize, lanes: usize, unroll: usize }; + + /// One local expert ID per original route; negative/out-of-bank IDs write zero. + fn call(x: Tensor, weight: Tensor, scales: Tensor, ids: Tensor, output: zml.Shape, topk: usize) Tensor { + const cfg: Gemv.Config = if (weight.dim(1) == 4608) + .{ .rows = 4, .lanes = 64, .unroll = 1 } + else + .{ .rows = 4, .lanes = 32, .unroll = 5 }; + const rows = cfg.rows; + const lanes = cfg.lanes; + const unroll = cfg.unroll; + const n = weight.dim(1); + const k = x.dim(1); + const routes: i64 = @intCast(ids.count()); + std.debug.assert(x.dtype() == .bf16 and (weight.dtype() == .u8 or weight.dtype() == .i8)); + std.debug.assert(@mod(n, 8) == 0 and @mod(k, 32) == 0); + std.debug.assert(topk > 0 and routes == x.dim(0) * @as(i64, @intCast(topk))); + std.debug.assert(weight.dim(2) * 2 == k); + std.debug.assert(scales.dim(0) == weight.dim(0) and scales.dim(1) == n and scales.dim(2) == @divExact(k, 32)); + const tile_rows = rows * (64 / lanes); + std.debug.assert(@mod(n, @as(i64, @intCast(tile_rows * 8))) == 0); + const fields: EmitConfig = .{ .unroll = unroll, .rows = tile_rows, .lanes = lanes, .stride = unroll * lanes, .ntiles = @divExact(n, @as(i64, @intCast(tile_rows))), .tokens = x.dim(0), .experts = weight.dim(0), .n = n, .k_pairs = @divExact(k, 2), .k_words = @divExact(k, 8), .k_scales = @divExact(k, 32), .routes = routes, .grid = @divExact(n, @as(i64, @intCast(tile_rows))) * routes, .topk = @as(i64, @intCast(topk)) }; + const a = x.reshape(.{ .token = x.dim(0), .in = @divExact(k, 2), .pair = 2 }).bitCast(.i32); + const b = weight.reshape(.{ .expert = weight.dim(0), .out = n, .in = @divExact(weight.dim(2), 4), .word = 4 }).bitCast(.i32); + const matrix_output = zml.Shape.init(.{ .token = routes, .out = n }, output.dtype()); + return Gemv.Kernel.call(.{ .a = a, .b = b, .scales = scales.bitCast(.u8), .ids = ids.reshape(.{routes}) }, .{ + .output = matrix_output, + }, .{ + .cfg = fields, + .threads = @intCast(rows * 64), + .grid = .{ @intCast(fields.grid), 1, 1 }, + }).output.reshape(output); + } + + const EmitConfig = struct { + unroll: usize, + rows: usize, + lanes: usize, + stride: usize, + ntiles: i64, + tokens: i64, + experts: i64, + n: i64, + k_pairs: i64, + k_words: i64, + k_scales: i64, + routes: i64, + grid: i64, + topk: i64, + }; + + const Kernel = zml.kernel.fly.Kernel(EmitConfig, .{ + .name = "mxfp4_gemv", + .inputs = &.{ "a", "b", "scales", "ids" }, + .outputs = &.{"output"}, + .run = Gemv.run, + }); + + fn run(b: *Builder, cfg: EmitConfig) fly.FinishError!void { + const args = Gemv.Kernel.args(b); + const block = b.blockId(.x).to(.i64); + const thread = b.threadId(.x).to(.i64); + const lane = thread.remUnsigned(cfg.lanes); + const rowgroup = thread.divUnsigned(cfg.lanes); + // Preserve the eight-XCD workgroup permutation and original subwave layout. + const pid = block.remUnsigned(8).mul(@divExact(cfg.grid, 8)).add(block.divUnsigned(8)); + const row = pid.remUnsigned(cfg.ntiles).mul(cfg.rows).add(rowgroup); + const route = pid.divUnsigned(cfg.ntiles); + const expert32 = args.ids.at(route); + const expert = expert32.to(.i64); + const token = route.divUnsigned(cfg.topk); + const local = expert32.cmp(.ge, 0).bitAnd(expert32.cmp(.lt, cfg.experts)); + const inrow = row.cmpUnsigned(.lt, cfg.n); + const fzero = b.constant(.f32, 0.0); + var sum = b.openIfElse(local.bitAnd(inrow), .{fzero.type_()}); + var k = b.openFor(lane, cfg.k_words, cfg.stride, .{fzero}); + var carried = k.carried[0]; + for (0..cfg.unroll) |u| { + const pos = k.iv.add(u * cfg.lanes); + var inbounds = b.openIfElse(pos.cmpUnsigned(.lt, cfg.k_words), .{fzero.type_()}); + const word = args.b.at(.{ expert, row, pos }); + const scale8 = args.scales.at(.{ expert, row, pos.divUnsigned(4) }); + const scale = decodeScale(b, scale8); + const low = word.bitAnd(0x07070707); + const high = word.shrU(4).bitAnd(0x07070707); + const even_bits = b.perm(b.constant(.i32, 0xd4d0ccc8), b.constant(.i32, 0xc4c0b800), low); + const odd_bits = b.perm(b.constant(.i32, 0xd4d0ccc8), b.constant(.i32, 0xc4c0b800), high); + const even = even_bits.bitAnd(word.shl(4).bitOr(0x7f7f7f7f)); + const odd = odd_bits.bitAnd(word.bitOr(0x7f7f7f7f)); + // Retain the scalar decoder and its FP32 scale -> BF16 -> FP32 boundary. + const weights = [_]Value{ + b.cvtPkF32Fp8(even, false).mul(scale).to(.bf16).to(.f32), + b.cvtPkF32Fp8(odd, false).mul(scale).to(.bf16).to(.f32), + b.cvtPkF32Fp8(even, true).mul(scale).to(.bf16).to(.f32), + b.cvtPkF32Fp8(odd, true).mul(scale).to(.bf16).to(.f32), + }; + var partial = fzero; + inline for (0..4) |pair| { + const x = args.a.at(.{ token, pos.mul(4).add(pair) }); + const xlo = x.to(.i16).bitcast(.bf16).to(.f32); + const xhi = x.shrU(16).to(.i16).bitcast(.bf16).to(.f32); + const wlo = weights[(pair / 2) * 2].extract(pair % 2); + const whi = weights[(pair / 2) * 2 + 1].extract(pair % 2); + // Odd then even FMA, followed by one add to the K-loop accumulator. + partial = b.fma(xlo, wlo, b.fma(xhi, whi, partial)); + } + inbounds.yieldThen(.{partial.add(carried)}); + inbounds.yieldElse(.{carried}); + carried = inbounds.results[0]; + } + k.yield(.{carried}); + var reduced = k.results[0]; + var shift: i32 = 1; + while (shift < @as(i32, @intCast(cfg.lanes))) : (shift *= 2) { + reduced = reduced.add(reduced.shuffleXor(shift, 64)); + } + sum.yieldThen(.{reduced}); + sum.yieldElse(.{fzero}); + var store = b.openIf(lane.cmp(.eq, 0).bitAnd(inrow)); + args.output.set(.{ route, row }, sum.results[0].to(.bf16)); + store.yieldThen(.{}); + } + + fn decodeScale(b: *Builder, byte: Value) Value { + const scale = byte.toUnsigned(.i32); + const finite = scale.cmp(.eq, 0).select(b.constant(.i32, 0x00400000), scale.shl(23)); + return scale.cmp(.eq, 255).select(b.constant(.i32, 0x7fc00000), finite).bitcast(.f32); + } +}; diff --git a/zml/kernels/moe/metal/BUILD.bazel b/zml/kernels/moe/metal/BUILD.bazel new file mode 100644 index 00000000..b4240b00 --- /dev/null +++ b/zml/kernels/moe/metal/BUILD.bazel @@ -0,0 +1,8 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "metal", + srcs = [ + "metal.zig", + ], +) diff --git a/zml/moe/metal.zig b/zml/kernels/moe/metal/metal.zig similarity index 97% rename from zml/moe/metal.zig rename to zml/kernels/moe/metal/metal.zig index e437b364..e129c76c 100644 --- a/zml/moe/metal.zig +++ b/zml/kernels/moe/metal/metal.zig @@ -1,7 +1,7 @@ const std = @import("std"); const stdx = @import("stdx"); -const zml = @import("../zml.zig"); +const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const log = std.log.scoped(.@"zml/moe/metal"); @@ -149,10 +149,8 @@ pub fn fusedExpertsImpl( stdx.debug.assert(is_topk_weights_valid_dtype, "metal backend expected topk_weights data type to be bf16, f16 or f32, got {}", .{topk_weights.dtype()}); stdx.debug.assert(topk_ids.dtype() == .i32, "metal backend expected topk_ids data type to be i32, got {}", .{topk_ids.dtype()}); - const b = hidden_states.dim(.b); - const s = hidden_states.dim(.s); const d = hidden_states.dim(.d); - const num_tokens = b * s; + const num_tokens = hidden_states.dim(.t); const topk = topk_ids.dim(.topk); const num_routes = num_tokens * topk; @@ -193,5 +191,5 @@ pub fn fusedExpertsImpl( const combined = weighted.splitAxis(.r, .{ .token = num_tokens, .topk = topk }) .sum(.topk).squeeze(.topk); - return combined.reshape(.{ .b = b, .s = s, .d = down.dim(.d) }); + return combined.reshape(.{ .t = num_tokens, .d = down.dim(.d) }); } diff --git a/zml/moe/moe.zig b/zml/kernels/moe/moe.zig similarity index 79% rename from zml/moe/moe.zig rename to zml/kernels/moe/moe.zig index 32a370be..fcfd269c 100644 --- a/zml/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -2,19 +2,15 @@ const std = @import("std"); const platforms = @import("platforms"); -const zml = @import("../zml.zig"); +const zml = @import("../../zml.zig"); const stdx = zml.stdx; -pub const stablehlo = @import("stablehlo.zig"); -pub const triton_mxfp4 = @import("triton_mxfp4.zig"); -pub const cute_mxfp4 = @import("cute_mxfp4.zig"); -pub const cutlass_flashinfer = @import("cutlass_flashinfer.zig"); -pub const metal = @import("metal.zig"); -pub const mosaic_tpu = @import("mosaic_tpu.zig"); -pub const triton = @import("triton.zig"); -pub const fly = @import("fly_kernels/moe.zig"); -const fused_experts = @import("fused_experts.zig"); -pub const triton_kernels = @import("triton_kernels/triton_kernels.zig"); -pub const ProjectionLayout = fused_experts.ProjectionLayout; +pub const stablehlo = @import("stablehlo/stablehlo.zig"); +pub const cute = @import("cute/cute.zig"); +pub const cutlass_flashinfer = @import("cutlass_flashinfer/cutlass_flashinfer.zig"); +pub const metal = @import("metal/metal.zig"); +pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); +pub const triton = @import("triton/triton.zig"); +pub const fly = @import("fly/fly.zig"); const log = std.log.scoped(.@"zml/moe"); @@ -23,6 +19,9 @@ test { _ = @import("tests.zig"); } +pub const ProjectionLayout = enum { concatenated, interleaved }; +pub const RoutingWeightPlacement = enum { before_down, after_down }; + /// How a backend expects the expert weights to be stored /// The packer writes this layout and forwardMoe reads it pub const ExpertsLayout = struct { @@ -78,8 +77,7 @@ pub const Activation = union(ActivationKind) { pub const Backend = enum { stablehlo, - cute_mxfp4, - triton_mxfp4, + cute, flashinfer_cutlass, triton, fly, @@ -93,10 +91,8 @@ pub const Backend = enum { .cuda => b: { const s = scheme orelse break :b .triton; break :b switch (s) { - .mxfp4 => if (cute_mxfp4.isAvailable(platform)) - .cute_mxfp4 - else if (triton_mxfp4.isAvailable(platform)) - .triton_mxfp4 + .mxfp4 => if (cute.isAvailable(platform)) + .cute else .triton, .nvfp4 => if (cutlass_flashinfer.isNvfp4Supported(platform)) @@ -136,8 +132,7 @@ pub const Backend = enum { pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { return switch (backend) { .stablehlo => true, - .triton_mxfp4 => triton_mxfp4.isAvailable(platform), - .cute_mxfp4 => cute_mxfp4.isAvailable(platform), + .cute => cute.isAvailable(platform), .flashinfer_cutlass => cutlass_flashinfer.isAvailable(platform), .fly => switch (platform.target) { .rocm => zml.platform.rocm.computeCapability(platform) == .gfx942, @@ -156,8 +151,7 @@ pub const Backend = enum { pub fn expertsLayout(backend: Backend, scheme: ?zml.Quantization.Scheme) !ExpertsLayout { return switch (backend) { .stablehlo => .{ .gate_up = .concatenated, .packing = .plain }, - .cute_mxfp4 => if (scheme == .mxfp4) .{ .gate_up = .interleaved, .packing = .swizzled_scales } else error.UnsupportedQuantization, - .triton_mxfp4 => if (scheme == .mxfp4) .{ .gate_up = .interleaved, .packing = .plain } else error.UnsupportedQuantization, + .cute => if (scheme == .mxfp4) .{ .gate_up = .interleaved, .packing = .swizzled_scales } else error.UnsupportedQuantization, .flashinfer_cutlass => if (scheme) |s| switch (s) { .nvfp4 => .{ .gate_up = .concatenated, .packing = .flashinfer_nvfp4 }, else => error.UnsupportedQuantization, @@ -170,7 +164,7 @@ pub const Backend = enum { pub fn register(backend: Backend, platform: *zml.Platform) !void { return switch (backend) { .stablehlo => {}, - .cute_mxfp4, .triton_mxfp4 => {}, + .cute => {}, .flashinfer_cutlass => cutlass_flashinfer.register(platform), .triton, .fly => {}, .mosaic_tpu => {}, @@ -184,10 +178,21 @@ pub const Options = struct { /// Quantize activations for Triton FP8 GEMMs; false keeps BF16 activations. quantize_input: bool, /// Where routing weights are applied; FlashInfer, Mosaic and Metal require after_down. - routing_weight_placement: fused_experts.RoutingWeightPlacement, + routing_weight_placement: RoutingWeightPlacement, }; -/// Routing IDs and weights have shape { b, s, topk }; input has shape { b, s, d }. +/// Check the shared token-wise contract for activations and routing tensors. +pub fn validateRoutingInputs(input: zml.Tensor, ids: zml.Tensor, weights: zml.Tensor) void { + stdx.debug.assert(input.rank() == 2 and ids.rank() == 2 and weights.rank() == 2, "MoE expects input {{.t, .d}} and routing {{.t, .topk}}, got {f}, {f}, {f}", .{ input.shape(), ids.shape(), weights.shape() }); + stdx.debug.assert(input.dim(.t) == ids.dim(.t) and input.dim(.t) == weights.dim(.t), "MoE input and routing token counts must match", .{}); + stdx.debug.assert(ids.dim(.topk) == weights.dim(.topk), "MoE routing ids and weights must have matching top-k dimensions", .{}); + stdx.debug.assert(input.shape().hasTag(.t) == 0 and input.shape().hasTag(.d) == 1 and + ids.shape().hasTag(.t) == 0 and ids.shape().hasTag(.topk) == 1 and + weights.shape().hasTag(.t) == 0 and weights.shape().hasTag(.topk) == 1, "MoE expects ordered tags {{.t, .d}} for input and {{.t, .topk}} for routing", .{}); +} + +/// Input has shape {.t, .d}; routing IDs and weights have shape {.t, .topk}. +/// Returns {.t, .d}. Flatten batched input with merge(.{ .t = .{ .b, .s } }). pub fn forwardMoe( input: zml.Tensor, topk_ids: zml.Tensor, @@ -197,12 +202,13 @@ pub fn forwardMoe( backend: Backend, opts: Options, ) zml.Tensor { + validateRoutingInputs(input, topk_ids, topk_weights); return switch (backend) { .stablehlo => stablehlo.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), - .cute_mxfp4 => cute_mxfp4.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), - .triton_mxfp4 => triton_mxfp4.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + .cute => cute.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), .flashinfer_cutlass => cutlass_flashinfer.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), - inline .triton, .fly => |b| fused_experts.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts, b), + .triton => triton.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + .fly => fly.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), .mosaic_tpu => mosaic_tpu.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), .metal => metal.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), }; diff --git a/zml/kernels/moe/mosaic_tpu/BUILD.bazel b/zml/kernels/moe/mosaic_tpu/BUILD.bazel new file mode 100644 index 00000000..ed4dfcd9 --- /dev/null +++ b/zml/kernels/moe/mosaic_tpu/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "mosaic_tpu", + srcs = [ + "gmm_ep.zig", + "mosaic_tpu.zig", + ], +) diff --git a/zml/moe/mosaic_tpu_kernels/gmm_ep.zig b/zml/kernels/moe/mosaic_tpu/gmm_ep.zig similarity index 99% rename from zml/moe/mosaic_tpu_kernels/gmm_ep.zig rename to zml/kernels/moe/mosaic_tpu/gmm_ep.zig index 2fca09f5..59638114 100644 --- a/zml/moe/mosaic_tpu_kernels/gmm_ep.zig +++ b/zml/kernels/moe/mosaic_tpu/gmm_ep.zig @@ -1,6 +1,6 @@ const std = @import("std"); -const zml = @import("../../zml.zig"); +const zml = @import("../../../zml.zig"); const mtt = @import("kernels/mosaic_tpu/builder"); const pipeline = mtt.pipeline; const mtt_kernel = zml.kernel.mosaic_tpu; diff --git a/zml/moe/mosaic_tpu.zig b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig similarity index 96% rename from zml/moe/mosaic_tpu.zig rename to zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig index f43d1b84..8d7807bf 100644 --- a/zml/moe/mosaic_tpu.zig +++ b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig @@ -3,9 +3,9 @@ const std = @import("std"); const mtt = @import("kernels/mosaic_tpu/builder"); const stdx = @import("stdx"); -const zml = @import("../zml.zig"); +const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; -pub const gmm_ep = @import("mosaic_tpu_kernels/gmm_ep.zig"); +pub const gmm_ep = @import("gmm_ep.zig"); const log = std.log.scoped(.@"zml/moe/mosaic_tpu"); @@ -282,7 +282,7 @@ pub fn fusedExperts( .w2_bias = self.bias_down, }, ); - return local_output.reshape(self.input.shape().dims()).withTags(.{ .b, .s, .d }); + return local_output.reshape(self.input.shape()).withTags(.{ .t, .d }); } }).body, .{ @@ -328,14 +328,13 @@ pub fn fusedExpertsImpl( topk_ids: Tensor, opts: Options, ) Tensor { - const b = hidden_states.dim(.b); - const s = hidden_states.dim(.s); - const hidden = hidden_states.reshape(.{ .token = b * s, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); + const tokens = hidden_states.dim(.t); + const hidden = hidden_states.reshape(.{ .token = tokens, .in = hidden_states.dim(.d) }); // TODO(Corentin): Better error message const gate_up = canonicalizeGateUp(w1, hidden.dim(.in)) catch |e| stdx.debug.panic("Invalid gate_up weights for fusedExpertsImpl: {}", .{e}); const down = canonicalizeDown(w2, hidden.dim(.in), gate_up.dim(.out)) catch |e| stdx.debug.panic("Invalid down weights for fusedExpertsImpl: {}", .{e}); - const weights = topk_weights.reshape(.{ .token = b * s, .topk = topk_weights.dim(.topk) }).withTags(.{ .token, .topk }); - const ids = topk_ids.reshape(.{ .token = b * s, .topk = topk_ids.dim(.topk) }).withTags(.{ .token, .topk }); + const weights = topk_weights.reshape(.{ .token = tokens, .topk = topk_weights.dim(.topk) }); + const ids = topk_ids.reshape(.{ .token = tokens, .topk = topk_ids.dim(.topk) }); const num_experts = if (opts.global_num_experts != -1) opts.global_num_experts else gate_up.dim(.expert); if (opts.expert_map) |expert_map| { @@ -403,5 +402,5 @@ pub fn fusedExpertsImpl( const unsorted = weighted.gather(.{ .token = restore_indices }, .{}).rename(.{ .restore = .route }); const restored = unsorted.reshape(.{ .topk = ids.dim(.topk), .token = hidden.dim(.token), .out = down.dim(.out) }); const combined = restored.sum(.topk).squeeze(.topk); - return combined.reshape(.{ .b = b, .s = s, .d = down.dim(.out) }); + return combined.reshape(.{ .t = tokens, .d = down.dim(.out) }); } diff --git a/zml/kernels/moe/stablehlo/BUILD.bazel b/zml/kernels/moe/stablehlo/BUILD.bazel new file mode 100644 index 00000000..c31b27f0 --- /dev/null +++ b/zml/kernels/moe/stablehlo/BUILD.bazel @@ -0,0 +1,8 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "stablehlo", + srcs = [ + "stablehlo.zig", + ], +) diff --git a/zml/moe/stablehlo.zig b/zml/kernels/moe/stablehlo/stablehlo.zig similarity index 91% rename from zml/moe/stablehlo.zig rename to zml/kernels/moe/stablehlo/stablehlo.zig index 29d17f9a..f32769e8 100644 --- a/zml/moe/stablehlo.zig +++ b/zml/kernels/moe/stablehlo/stablehlo.zig @@ -3,7 +3,7 @@ const std = @import("std"); const stdx = @import("stdx"); -const zml = @import("../zml.zig"); +const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; @@ -129,7 +129,7 @@ fn roundedInput(x: Tensor, linear: zml.nn.Linear, opts: Options, dtype: zml.Data if (scheme == .mxfp8 or scheme == .fp8_block32) scale = scale.log().scale(1 / @log(@as(f32, 2))).ceil().scale(@log(@as(f32, 2))).exp(); const broad = scale.broad(grouped.shape()); return grouped.div(broad).clamp(Tensor.scalar(-max_value, .f32), Tensor.scalar(max_value, .f32)) - .convert(output_dtype).convert(.f32).mul(broad).reshape(x.shape().withDtype(.f32)); + .convert(output_dtype).optimizationBarrier().convert(.f32).mul(broad).reshape(x.shape().withDtype(.f32)); } fn project(x: Tensor, linear: zml.nn.Linear, ids: Tensor, opts: Options, dtype: zml.DataType) Tensor { @@ -153,23 +153,20 @@ pub const Options = struct { /// Metal uses weight-only quantization; specialized MXFP4 GEMMs always use MXFP8 inputs. /// CuTe additionally rounds the intermediate activation to BF16 before quantizing. input_quantization: enum { automatic, none, mxfp8, mxfp8_bf16 } = .automatic, - routing_weight_placement: @import("fused_experts.zig").RoutingWeightPlacement = .after_down, + routing_weight_placement: zml.moe.RoutingWeightPlacement = .after_down, }; pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: Options) Tensor { - stdx.debug.assert(input.rank() == 3 and topk_ids.rank() == 3 and topk_weights.rank() == 3, "reference expects rank-three input and routing tensors", .{}); stdx.debug.assert(topk_ids.dtype() == .i32, "reference expects i32 expert ids", .{}); stdx.debug.assert(gate_up.weight.rank() == 3 and down.weight.rank() == 3, "reference expects rank-three expert weights", .{}); stdx.debug.assert(gate_up.weight.dim(.expert) == down.weight.dim(.expert), "reference expert counts must match", .{}); - const b = input.dim(.b); - const s = input.dim(.s); + const tokens = input.dim(.t); const k = topk_ids.dim(.topk); stdx.debug.assert(k > 0 and k <= gate_up.weight.dim(.expert), "reference top-k must fit the expert count", .{}); - stdx.debug.assert(topk_ids.dim(.b) == b and topk_ids.dim(.s) == s and topk_weights.dim(.b) == b and topk_weights.dim(.s) == s and topk_weights.dim(.topk) == k, "reference routing dimensions must match input and ids", .{}); - const ids = topk_ids.reshape(.{ .route = b * s * k }); - const weights = topk_weights.reshape(.{ .route = b * s * k }).convert(.f32); - const hidden = input.reshape(.{ .token = b * s, .in = input.dim(.d) }) - .insertAxes(.in, .{.topk}).broad(Shape.init(.{ .token = b * s, .topk = k, .in = input.dim(.d) }, input.dtype())) + const ids = topk_ids.reshape(.{ .route = tokens * k }); + const weights = topk_weights.reshape(.{ .route = tokens * k }).convert(.f32); + const hidden = input.reshape(.{ .token = tokens, .in = input.dim(.d) }) + .insertAxes(.in, .{.topk}).broad(Shape.init(.{ .token = tokens, .topk = k, .in = input.dim(.d) }, input.dtype())) .merge(.{ .route = .{ .token, .topk } }); // Keep the projection's rounding boundary when XLA fuses activation/routing // arithmetic; otherwise the reference can disagree with explicit BF16 GEMMs. @@ -179,7 +176,7 @@ pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: var output = project(activated.rename(.{ .out = .in }), down, ids, opts, input.dtype()); if (opts.routing_weight_placement == .after_down) output = output.mul(weights.broad(output.shape())); return output.convert(input.dtype()).convert(.f32) - .reshape(.{ .b = b, .s = s, .topk = k, .d = output.dim(.out) }) + .reshape(.{ .t = tokens, .topk = k, .d = output.dim(.out) }) .sum(.topk).squeeze(.topk).convert(input.dtype()); } diff --git a/zml/moe/tests.zig b/zml/kernels/moe/tests.zig similarity index 89% rename from zml/moe/tests.zig rename to zml/kernels/moe/tests.zig index 1b654fa3..ec92bbeb 100644 --- a/zml/moe/tests.zig +++ b/zml/kernels/moe/tests.zig @@ -2,11 +2,11 @@ //! supported cases are explicit, and errors in those cases must fail the test. const std = @import("std"); -const zml = @import("../zml.zig"); +const zml = @import("../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; -const Placement = @import("fused_experts.zig").RoutingWeightPlacement; -const reference = @import("stablehlo.zig"); +const reference = @import("stablehlo/stablehlo.zig"); +const Placement = @import("moe.zig").RoutingWeightPlacement; const Case = struct { backend: zml.moe.Backend, @@ -18,19 +18,18 @@ const Case = struct { bias: bool = false, tokens: i64 = 1, topk: i64 = 1, - batch: i64 = 1, width: i64 = 128, intermediate: i64 = 128, experts: i64 = 8, pub fn compile(c: Case, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Exe { std.debug.assert(c.backend.isAvailable(platform)); - const x: Tensor = .init(.{ .b = c.batch, .s = c.tokens, .d = c.width }, c.activation_dtype); + const x: Tensor = .init(.{ .t = c.tokens, .d = c.width }, c.activation_dtype); return platform.compileFn(allocator, io, forward, .{ x, c }, .{}); } pub fn check(c: Case, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, exe: *const zml.Exe) !void { - const x: Tensor = .init(.{ .b = c.batch, .s = c.tokens, .d = c.width }, c.activation_dtype); + const x: Tensor = .init(.{ .t = c.tokens, .d = c.width }, c.activation_dtype); var host = try zml.Slice.alloc(allocator, x.shape()); defer host.free(allocator); @@ -209,17 +208,24 @@ fn prepareForBackend(canonical: ExpertFixture, backend: zml.moe.Backend) ExpertF q.scales = swizzleScales(q.scales); q.swizzled_scales = true; } - if (backend == .cute_mxfp4 or backend == .triton_mxfp4) { - projection.weight = packFp4Bytes(projection.weight); - projection.quantization.?.scales = projection.quantization.?.scales.bitCast(.u8); - } - if (backend == .mosaic_tpu) projection.weight = projection.weight.transpose(.{ 0, 2, 1 }); - if (backend == .metal) { - if (projection.quantization) |*q| { + switch (backend) { + .cute => { + projection.weight = packFp4Bytes(projection.weight); + projection.quantization.?.scales = projection.quantization.?.scales.bitCast(.u8); + }, + .triton => if (scheme == .mxfp4) { + projection.weight = packFp4Bytes(projection.weight); + }, + .mosaic_tpu => { + projection.weight = projection.weight.transpose(.{ 0, 2, 1 }); + }, + .metal => if (projection.quantization) |*q| { if (q.scheme != .nvfp4) q.scales = q.scales.convert(.bf16); - } + }, + else => {}, } } + if (backend == .metal and scheme == .nvfp4) { const global = &prepared.gate_up.quantization.?.global_scale.?; global.value = global.value.appendAxes(.{.proj}).repeat1d(.proj, 2); @@ -235,7 +241,7 @@ fn forward(input: Tensor, c: Case) Outputs { const canonical = makeExpertFixture(c); const prepared = prepareForBackend(canonical, c.backend); - const route_shape = Shape.init(.{ .b = c.batch, .s = c.tokens, .topk = c.topk }, .i32); + const route_shape = Shape.init(.{ .t = c.tokens, .topk = c.topk }, .i32); const route = Tensor.arange(.{ .end = @intCast(route_shape.count()) }, .i32).reshape(route_shape); // Only six of eight experts receive routes, with nonuniform counts. @@ -255,13 +261,13 @@ fn forward(input: Tensor, c: Case) Outputs { .routing_weight_placement = c.placement, .input_quantization = switch (c.backend) { .metal => .none, - .cute_mxfp4 => .mxfp8_bf16, - .triton_mxfp4 => .mxfp8, + .cute => .mxfp8_bf16, + .triton => if (c.scheme == .mxfp4 and zml.Compiler.current().platform.target == .cuda) .mxfp8 else .automatic, else => .automatic, }, }); - return .{ .actual = actual.reshape(input.shape()), .expected = expected }; + return .{ .actual = actual, .expected = expected }; } const gated_activations = [_]zml.moe.Activation{ @@ -281,7 +287,6 @@ const clipped_swiglu = [_]zml.moe.Activation{ }; const FixtureShape = struct { - batch: i64 = 1, tokens: i64 = 1, topk: i64 = 1, width: i64 = 128, @@ -289,17 +294,17 @@ const FixtureShape = struct { experts: i64 = 8, }; -// Every option combination runs on decode, prefill, and batched prefill. +// Every option combination runs with 1, 17, and 34 flattened tokens. // Shapes and floating-point parameter values are representative finite sets. const fixture_shapes = [_]FixtureShape{ .{}, .{ .tokens = 17, .topk = 2 }, - .{ .batch = 2, .tokens = 17, .topk = 3 }, + .{ .tokens = 34, .topk = 3 }, }; const specialized_shapes = [_]FixtureShape{ .{ .width = 5120, .intermediate = 2304, .experts = 2 }, .{ .width = 5120, .intermediate = 2304, .experts = 2, .tokens = 17, .topk = 2 }, - .{ .width = 5120, .intermediate = 2304, .experts = 2, .batch = 2, .tokens = 17, .topk = 2 }, + .{ .width = 5120, .intermediate = 2304, .experts = 2, .tokens = 34, .topk = 2 }, }; const Format = struct { @@ -343,7 +348,6 @@ const Matrix = struct { .quantize_input = quantize_input, .placement = placement, .bias = bias, - .batch = shape.batch, .tokens = shape.tokens, .topk = shape.topk, .width = shape.width, @@ -507,7 +511,6 @@ test "Triton compatibility matrix" { try pipeline.run(std.testing.io, std.testing.allocator, .triton, .{ .formats = &.{ .{}, - .{ .scheme = .mxfp4 }, .{ .scheme = .mxfp8 }, .{ .scheme = .fp8_per_tensor }, .{ .scheme = .fp8_per_channel }, @@ -515,11 +518,49 @@ test "Triton compatibility matrix" { .{ .scheme = .fp8_block32 }, }, .activations = &all_activations, - // BF16 and MXFP4 ignore this option; exercise both accepted settings. + // BF16 ignores this option; exercise both accepted settings. .quantize_inputs = &.{ false, true }, .placements = &.{ .before_down, .after_down }, .biases = &.{ false, true }, }, .{}); + + if (platform.target != .cuda) { + // Other platforms retain the general MXFP4 path and its full contract. + try pipeline.run(std.testing.io, std.testing.allocator, .triton, .{ + .formats = &.{.{ .scheme = .mxfp4 }}, + .activations = &all_activations, + .quantize_inputs = &.{ false, true }, + .placements = &.{ .before_down, .after_down }, + .biases = &.{ false, true }, + }, .{}); + return; + } + + // Preserve the 12 supported cases from the former general MXFP4 matrix. + // Its other 252 cases require activations or projection bias unsupported + // by the simplified path. The former specialized matrix remains below. + try pipeline.run(std.testing.io, std.testing.allocator, .triton, .{ + .formats = &.{.{ .scheme = .mxfp4 }}, + .activations = &.{gated_activations[1]}, + .quantize_inputs = &.{ false, true }, + .placements = &.{ .before_down, .after_down }, + }, .{}); +} + +// Keep triton mxfp4 separate because it only supports a subset for now +test "Triton MXFP4 compatibility matrix" { + const platform = zml.testing.env(); + if (platform.target != .cuda) return error.SkipZigTest; + const pipeline: Pipeline = .{ .platform = platform }; + + // The merged backend uses the former specialized MXFP4 path: clipped + // SwiGLU without projection bias, with unconditional MXFP8 activations. + try pipeline.run(std.testing.io, std.testing.allocator, .triton, .{ + .formats = &.{.{ .scheme = .mxfp4 }}, + .activations = &clipped_swiglu, + .quantize_inputs = &.{ false, true }, + .placements = &.{ .before_down, .after_down }, + }, .{}); } test "Triton MoE prefill launch buckets fit shared memory" { @@ -552,26 +593,12 @@ test "FlashInfer compatibility matrix" { }, .{}); } -test "specialized Triton MXFP4 compatibility matrix" { - const platform = zml.testing.env(); - - const pipeline: Pipeline = .{ .platform = platform }; - - // The kernel always quantizes inputs to MXFP8, independently of the option. - try pipeline.run(std.testing.io, std.testing.allocator, .triton_mxfp4, .{ - .formats = &.{.{ .scheme = .mxfp4 }}, - .activations = &clipped_swiglu, - .quantize_inputs = &.{ false, true }, - .placements = &.{ .before_down, .after_down }, - }, .{}); -} - test "specialized CuTe MXFP4 compatibility matrix" { const platform = zml.testing.env(); const pipeline: Pipeline = .{ .platform = platform }; - try pipeline.run(std.testing.io, std.testing.allocator, .cute_mxfp4, .{ + try pipeline.run(std.testing.io, std.testing.allocator, .cute, .{ .formats = &.{.{ .scheme = .mxfp4 }}, .activations = &clipped_swiglu, .quantize_inputs = &.{ false, true }, diff --git a/zml/kernels/moe/triton/BUILD.bazel b/zml/kernels/moe/triton/BUILD.bazel new file mode 100644 index 00000000..8dda7f88 --- /dev/null +++ b/zml/kernels/moe/triton/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "triton", + srcs = [ + "moe.zig", + "triton.zig", + ], +) diff --git a/zml/moe/triton_kernels/triton_kernels.zig b/zml/kernels/moe/triton/moe.zig similarity index 70% rename from zml/moe/triton_kernels/triton_kernels.zig rename to zml/kernels/moe/triton/moe.zig index 0561b074..b3a22c18 100644 --- a/zml/moe/triton_kernels/triton_kernels.zig +++ b/zml/kernels/moe/triton/moe.zig @@ -7,7 +7,7 @@ const std = @import("std"); -const zml = @import("../../zml.zig"); +const zml = @import("../../../zml.zig"); const tri = zml.kernel.triton; const Builder = tri.Builder; const Value = tri.Value; @@ -842,3 +842,346 @@ pub const CountAndSortExpertTokens = struct { } } }; + +/// Returns the next power of two. +fn pow2(x: i64) i64 { + return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(x)) catch unreachable); +} + +pub fn quantRows(tokens: i64) i64 { + return if (tokens >= 16) 16 else 1; +} + +fn ar(b: *Builder, n: i64) Value { + return b.arange(0, n, .i32); +} + +fn ld(b: *Builder, p: Value, mask: Value, dt: tri.DType, other: i64) Value { + return b.loadOpts(p, .{ .mask = mask, .other = if (p.isTensor()) b.full(p.shape().constSlice(), other, dt) else b.liftAs(other, dt) }); +} + +fn ci(b: *Builder, n: i64) Value { + return b.liftAs(n, .i32); +} + +fn quant32(b: *Builder, x: Value) struct { q: Value, s: Value } { + const rows = x.shape().constSlice()[0]; + const cols = x.shape().constSlice()[1]; + const groups = @divExact(cols, 32); + const blocks = b.reshape(x, &.{ rows * groups, 32 }); + const sf = b.exp2(b.ceil(b.log2(b.maxOpts(b.absf(blocks), .{ .axis = 1 }).maximum(1e-4).mul(1.0 / 448.0)))); + const scaled = blocks.div(sf.expandDims(1)).maximum(-448.0).minimum(448.0); + // TODO(Corentin): We should gate that behind a platform check so that we don't do useless stuff on non-SM89 platforms. + // Find a way to add that to the builder, maybe a `b.platform()` or something like that. + const q = b: { + // SM89 lowers FP32 -> FP8 through FP16 with truncation. Round to odd at + // FP16 precision first so that this intermediate cannot turn a value just + // above an FP8 midpoint into an exact tie. Direct FP8 conversion is unchanged. + const scaled_bits = b.bitcast(scaled, .i32); + const sticky = b.select(scaled_bits.bitAnd(0x1fff).ne(0), b.full(scaled.shape().constSlice(), 0x2000, .i32), b.zeros(scaled.shape().constSlice(), .i32)); + const prepared = b.bitcast(scaled_bits.bitAnd(-0x2000).bitOr(sticky), .f32); + break :b prepared.to(.f8e4m3fn); + }; + const bits = b.shrui(b.bitcast(sf, .i32), b.full(sf.shape().constSlice(), 23, .i32)).bitAnd(255).to(.i8); + return .{ .q = b.reshape(q, &.{ rows, cols }), .s = b.reshape(bits, &.{ rows, groups }) }; +} + +pub fn combineBlock(tokens: i64, hidden: i64) i64 { + // Keep at least 512 CTAs before increasing per-CTA reduction work. + const elements = tokens * hidden; + if (tokens > 1 and elements >= 512 * 2048) return 2048; + if (tokens > 1 and elements >= 512 * 1024) return 1024; + return 256; +} + +/// Computes an upper bound on the number of expert tiles needed. +/// When there is only one token, reserve a single tile per selected expert. +pub fn tiles(tokens: i64, topk: i64, experts: i64, bn: i64) i64 { + if (tokens == 1) return topk; + return @min(tokens * topk, @divTrunc(tokens * topk + experts * (bn - 1) + bn - 1, bn)); +} + +// ============================================================================= +// mxfp4_triton_quant +// ============================================================================= + +pub const Quant = struct { + pub const Cfg = struct { + tokens: i64, + hidden: i64, + }; + + pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_quant", .inputs = &.{"x"}, .outputs = &.{ "q", "s" }, .run = run }); + + fn run(b: *Builder, c: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ .x = .{ .ptr = .bf16 }, .q = .{ .ptr = .f8e4m3fn }, .s = .{ .ptr = .i8 } }); + const rows = b.programId(.x).mul(quantRows(c.tokens)).add(ar(b, quantRows(c.tokens))); + const block = b.programId(.y); + const cols = block.mul(128).add(ar(b, 128)); + const offsets = rows.expandDims(1).mul(c.hidden).add(cols.expandDims(0)); + const mask = rows.lt(c.tokens).expandDims(1); + const x = ld(b, a.x.addPtr(offsets), mask, .bf16, 0).to(.f32); + const q = quant32(b, x); + b.storeOpts(a.q.addPtr(offsets), q.q, .{ .mask = mask }); + const scale_offsets = rows.expandDims(1).mul(@divTrunc(c.hidden, 32)).add(block.mul(4).add(ar(b, 4)).expandDims(0)); + b.storeOpts(a.s.addPtr(scale_offsets), q.s, .{ .mask = mask }); + } +}; + +// ============================================================================= +// mxfp4_triton_route +// ============================================================================= + +pub const Route = struct { + pub const Cfg = struct { + tokens: i64, + topk: i64, + expert_offset: i64, + }; + + pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_route", .inputs = &.{"ids"}, .outputs = &.{ "counts", "pos" }, .run = run }); + + fn run(b: *Builder, c: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ .ids = .{ .ptr = .i32 }, .counts = .{ .ptr = .i32 }, .pos = .{ .ptr = .i32 } }); + const e = b.programId(.x); + const p = ar(b, pow2(c.tokens * c.topk)); + const ids = ld(b, a.ids.addPtr(p), p.lt(c.tokens * c.topk), .i32, -1); + const valid = p.lt(c.tokens * c.topk).bitAnd(ids.eq(e.add(c.expert_offset))); + b.storeOpts(a.pos.addPtr(p), valid.to(.i32).cumsum().sub(1), .{ .mask = valid }); + b.store(a.counts.addPtr(e), valid.to(.i32).sum()); + } +}; + +// ============================================================================= +// mxfp4_triton_route_atomic +// ============================================================================= + +pub const RouteAtomic = struct { + pub const route_block = 1024; + + pub const Cfg = struct { + tokens: i64, + topk: i64, + expert_offset: i64, + experts: i64, + }; + + pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_route_atomic", .inputs = &.{ "ids", "zeros" }, .outputs = &.{ "counts", "pos" }, .run = run }); + + /// Routing for large batches, used by `scheduleForCute`. + /// Each route takes the next free slot of its expert with an atomic + /// increment of `counts` (zero-initialized through `zeros`). Slots within an + /// expert are therefore unordered, which the grouped GEMMs do not care + /// about: every routed row is computed independently. + fn run(b: *Builder, c: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ .ids = .{ .ptr = .i32 }, .zeros = .{ .ptr = .i32 }, .counts = .{ .ptr = .i32 }, .pos = .{ .ptr = .i32 } }); + const p = b.programId(.x).mul(route_block).add(ar(b, route_block)); + const in_range = p.lt(c.tokens * c.topk); + const e = ld(b, a.ids.addPtr(p), in_range, .i32, -1).sub(ci(b, c.expert_offset)); + const valid = in_range.bitAnd(e.ge(0)).bitAnd(e.lt(c.experts)); + const slot = b.atomicRmwOpts(.add, a.counts.addPtr(e.maximum(0)), b.full(&.{route_block}, 1, .i32), .{ .mask = valid, .sem = .relaxed }); + b.storeOpts(a.pos.addPtr(p), slot, .{ .mask = valid }); + } +}; + +// ============================================================================= +// mxfp4_triton_schedule +// ============================================================================= + +pub const Schedule = struct { + pub const Cfg = struct { + tokens: i64, + topk: i64, + expert_offset: i64, + experts: i64, + /// Expert groups the schedule may fill. + capacity: i64, + /// Routed rows per expert group, i.e. the consuming GEMM's tile N. + group: i64, + }; + + pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_schedule", .inputs = &.{ "ids", "counts", "pos" }, .outputs = &.{ "map", "perm", "sched" }, .run = run }); + + fn run(b: *Builder, cfg: Cfg) tri.FinishError!void { + const nt = cfg.capacity; + // Route positions are i32 tensors; keep the group size in that type. + const group: i32 = @intCast(cfg.group); + const a = try b.declareArgs(.{ .ids = .{ .ptr = .i32 }, .counts = .{ .ptr = .i32 }, .pos = .{ .ptr = .i32 }, .map = .{ .ptr = .i32 }, .perm = .{ .ptr = .i32 }, .sched = .{ .ptr = .i32 } }); + const p = b.programId(.x).mul(128).add(ar(b, 128)); + const e = ld(b, a.ids.addPtr(p), p.lt(cfg.tokens * cfg.topk), .i32, -1).sub(ci(b, cfg.expert_offset)); + const es = ar(b, pow2(cfg.experts)); + const counts = ld(b, a.counts.addPtr(es), es.lt(cfg.experts), .i32, 0); + const ts = counts.cdiv(group); + const starts = ts.cumsum().sub(ts); + var first = b.openIf(b.programId(.x).eq(0)); + b.store(a.sched.addPtr(2 * nt), ts.sum()); + first.yieldThen(.{}); + const live = p.lt(cfg.tokens * cfg.topk).bitAnd(e.ge(0)).bitAnd(e.lt(cfg.experts)); + const safe = e.maximum(0).minimum(pow2(cfg.experts) - 1); + const pos = ld(b, a.pos.addPtr(p), live, .i32, 0); + const start = starts.gather(safe, 0); + const tile = start.add(pos.div(group)); + const row = pos.rem(group); + b.storeOpts(a.perm.addPtr(p), b.select(live, tile.mul(group).add(row), b.full(&.{128}, -1, .i32)), .{ .mask = p.lt(cfg.tokens * cfg.topk) }); + b.storeOpts(a.map.addPtr(tile.mul(group).add(row)), p, .{ .mask = live }); + b.storeOpts(a.sched.addPtr(tile), e, .{ .mask = live.bitAnd(row.eq(0)) }); + b.storeOpts(a.sched.addPtr(nt).addPtr(tile), counts.gather(safe, 0).sub(pos).minimum(group), .{ .mask = live.bitAnd(row.eq(0)) }); + } +}; + +// ============================================================================= +// mxfp4_triton_combine +// ============================================================================= + +pub const Combine = struct { + pub const Cfg = struct { + tokens: i64, + hidden: i64, + topk: i64, + expert_offset: i64, + experts: i64, + routing_weight_placement: zml.moe.RoutingWeightPlacement = .before_down, + bn: i64, + }; + pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_combine", .inputs = &.{ "d", "ids", "perm", "rw" }, .outputs = &.{"y"}, .run = run }); + + fn run(b: *Builder, c: Cfg) tri.FinishError!void { + return combineWithBn(b, c); + } + + fn combineWithBn(b: *Builder, c: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ .d = .{ .ptr = .bf16 }, .ids = .{ .ptr = .i32 }, .perm = .{ .ptr = .i32 }, .rw = .{ .ptr = .f32 }, .y = .{ .ptr = .bf16 } }); + const t = b.programId(.x); + const block = combineBlock(c.tokens, c.hidden); + const h = b.programId(.y).mul(block).add(ar(b, block)); + var acc = b.zeros(&.{block}, .f32); + for (0..@intCast(c.topk)) |s| { + const pair = t.mul(c.topk).add(@as(i64, @intCast(s))); + const row = if (c.tokens == 1) pair.mul(c.bn) else b.load(a.perm.addPtr(pair)); + const valid = if (c.tokens == 1) v: { + const e = b.load(a.ids.addPtr(pair)).sub(ci(b, c.expert_offset)); + break :v e.ge(0).bitAnd(e.lt(c.experts)); + } else row.ge(0); + var value = ld(b, a.d.addPtr(row.mul(c.hidden).add(h)), valid.bitAnd(h.lt(c.hidden)), .bf16, 0).to(.f32); + if (c.routing_weight_placement == .after_down) { + value = value.mul(ld(b, a.rw.addPtr(pair), valid, .f32, 0)); + } + acc = acc.add(value); + } + b.storeOpts(a.y.addPtr(t.mul(c.hidden).add(h)), acc.to(.bf16), .{ .mask = h.lt(c.hidden) }); + } +}; + +// ============================================================================= +// mxfp4 gemm +// ============================================================================= + +pub const GateUp = struct { + pub const Cfg = GemmConfig; + + pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_gate_up", .inputs = &.{ "q", "s", "w", "ws", "ids", "rw", "map", "sched" }, .outputs = &.{ "mid", "ms" }, .run = run }); + + fn run(b: *Builder, c: Cfg) tri.FinishError!void { + return gemm(b, c, .gate_up); + } +}; + +pub const Down = struct { + pub const Cfg = GemmConfig; + + pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_down", .inputs = &.{ "mid", "ms", "w", "ws", "ids", "sched" }, .outputs = &.{"d"}, .run = run }); + + fn run(b: *Builder, c: Cfg) tri.FinishError!void { + return gemm(b, c, .down); + } +}; + +const GemmConfig = struct { + tokens: i64, + hidden: i64, + intermediate: i64, + experts: i64, + topk: i64, + expert_offset: i64 = 0, + // Swiglu clamping. Set to 0 to disable. + swiglu_limit: f32, + scale_stride: i64, + routing_weight_placement: zml.moe.RoutingWeightPlacement = .before_down, + bn: i64, + bk: i64, +}; + +const Projection = enum { gate_up, down }; + +fn gemm(b: *Builder, c: GemmConfig, comptime projection: Projection) tri.FinishError!void { + const is_up = projection == .gate_up; + const a = if (is_up) try b.declareArgs(.{ .q = .{ .ptr = .f8e4m3fn }, .s = .{ .ptr = .i8 }, .w = .{ .ptr = .i8 }, .ws = .{ .ptr = .i8 }, .ids = .{ .ptr = .i32 }, .rw = .{ .ptr = .f32 }, .map = .{ .ptr = .i32 }, .sched = .{ .ptr = .i32 }, .mid = .{ .ptr = .f8e4m3fn }, .ms = .{ .ptr = .i8 } }) else try b.declareArgs(.{ .q = .{ .ptr = .f8e4m3fn }, .s = .{ .ptr = .i8 }, .w = .{ .ptr = .i8 }, .ws = .{ .ptr = .i8 }, .ids = .{ .ptr = .i32 }, .sched = .{ .ptr = .i32 }, .d = .{ .ptr = .bf16 } }); + const block = b.programId(.x); + const tile = b.programId(.y); + const n = ar(b, c.bn); + const nt = tiles(c.tokens, c.topk, c.experts, c.bn); + const expert = if (c.tokens == 1) b.load(a.ids.addPtr(tile)).sub(ci(b, c.expert_offset)) else ld(b, a.sched.addPtr(tile), tile.lt(b.load(a.sched.addPtr(2 * nt))), .i32, 0); + const valid = if (c.tokens == 1) expert.ge(0).bitAnd(expert.lt(c.experts)) else tile.lt(b.load(a.sched.addPtr(2 * nt))); + var live = b.openIf(valid); + const count = if (c.tokens == 1) ci(b, 1) else b.load(a.sched.addPtr(nt).addPtr(tile)); + const pair = if (c.tokens == 1) b.full(&.{c.bn}, 0, .i32).add(tile) else if (is_up) ld(b, a.map.addPtr(tile.mul(c.bn).add(n)), n.lt(count), .i32, 0) else n; + const width: i64 = if (is_up and c.tokens > 8) 256 else 128; + const ksize = if (is_up) c.hidden else c.intermediate; + const rows = if (is_up) 2 * c.intermediate else c.hidden; + const m = block.mul(width).add(ar(b, width)); + const k = ar(b, c.bk); + const ks = ar(b, @divExact(c.bk, 32)); + const row = if (is_up) pair.div(c.topk) else tile.mul(c.bn).add(n); + const wd = b.makeTensorDescriptor(a.w.addPtr(expert.to(.i64).mul(@divTrunc(rows * ksize, 2))), &.{ ci(b, rows), ci(b, @divTrunc(ksize, 2)) }, &.{ b.liftAs(@divTrunc(ksize, 2), .i64), b.liftAs(1, .i64) }, .pad_zero, &.{ width, @divExact(c.bk, 2) }, .i8); + var loop = b.openFor(0, @divTrunc(ksize + c.bk - 1, c.bk), 1, .{b.zeros(&.{ width, c.bn }, .f32)}); + const sk = loop.iv.mul(@divExact(c.bk, 32)).add(ks); + const kk = loop.iv.mul(c.bk).add(k); + const w = b.descriptorLoad(wd, &.{ block.mul(ci(b, width)), loop.iv.mul(ci(b, @divExact(c.bk, 2))) }, &.{ width, @divExact(c.bk, 2) }, .i8); + const scale_offset = m.expandDims(1).mul(c.scale_stride).add(sk.expandDims(0)); + const ws = ld(b, a.ws.addPtr(expert.to(.i64).mul(rows * c.scale_stride)).addPtr(scale_offset), sk.expandDims(0).lt(@divTrunc(ksize, 32)), .i8, 127); + var mask = kk.expandDims(0).lt(ksize); + var smask = sk.expandDims(0).lt(@divTrunc(ksize, 32)); + // The small-batch epilogue leaves padded rows unwritten. + if (is_up or c.tokens <= 8) { + mask = mask.bitAnd(n.lt(count).expandDims(1)); + smask = smask.bitAnd(n.lt(count).expandDims(1)); + } + const x = ld(b, a.q.addPtr(row.expandDims(1).mul(ksize).add(kk.expandDims(0))), mask, .f8e4m3fn, 0); + const xs = ld(b, a.s.addPtr(row.expandDims(1).mul(@divTrunc(ksize, 32)).add(sk.expandDims(0))), smask, .i8, 127); + loop.yield(.{b.dotScaled(w, b.permute(x, &.{ 1, 0 }), loop.carried[0], ws, xs, .e2m1, .e4m3)}); + const epilogue_n: i64 = if (c.tokens == 1) 1 else c.bn; + const out_n = ar(b, epilogue_n); + const acc = if (c.tokens == 1) live_row: { + // The remaining MMA columns are padding during single-token decode. + // Extract column zero before activation/reduction and output stores. + var v = loop.results[0]; + var cols: i64 = c.bn; + while (cols > 1) : (cols = @divTrunc(cols, 2)) { + v = b.split(b.reshape(v, &.{ width, @divTrunc(cols, 2), 2 }))[0]; + } + break :live_row b.reshape(v, &.{ 1, width }).to(.bf16); + } else b.permute(loop.results[0], &.{ 1, 0 }).to(.bf16); + if (is_up) { + const columns = @divExact(width, 2); + const halves = b.split(b.reshape(acc.to(.f32), &.{ epilogue_n, columns, 2 })); + var g = halves[0]; + var u = halves[1]; + if (c.swiglu_limit > 0) { + g = g.minimum(c.swiglu_limit); + u = u.minimum(c.swiglu_limit).maximum(-c.swiglu_limit); + } + const ex = b.externElementwise(&.{g.mul(-1.0)}, &.{ epilogue_n, columns }, .f32, "libdevice", "", "__nv_expf"); + var mid = u.mul(b.divRn(g, ex.add(1.0))); + if (c.routing_weight_placement == .before_down) { + const rw = if (c.tokens == 1) b.load(a.rw.addPtr(tile)) else ld(b, a.rw.addPtr(pair), n.lt(count), .f32, 0).expandDims(1); + mid = mid.mul(rw); + } + // Compute scales and quantize directly from the FP32 SwiGLU result. + const q = quant32(b, mid); + const col = block.mul(columns).add(ar(b, columns)); + b.store(a.mid.addPtr(tile.mul(c.bn).add(out_n).expandDims(1).mul(c.intermediate).add(col.expandDims(0))), q.q); + const scale_cols = block.mul(@divExact(columns, 32)).add(ar(b, @divExact(columns, 32))); + b.store(a.ms.addPtr(tile.mul(c.bn).add(out_n).expandDims(1).mul(@divTrunc(c.intermediate, 32)).add(scale_cols.expandDims(0))), q.s); + } else b.store(a.d.addPtr(tile.mul(c.bn).add(out_n).expandDims(1).mul(c.hidden).add(m.expandDims(0))), acc); + live.yieldThen(.{}); +} diff --git a/zml/kernels/moe/triton/triton.zig b/zml/kernels/moe/triton/triton.zig new file mode 100644 index 00000000..908cd163 --- /dev/null +++ b/zml/kernels/moe/triton/triton.zig @@ -0,0 +1,1161 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const Tensor = zml.Tensor; +const Shape = zml.Shape; +const DataType = zml.DataType; +const kernels = @import("moe.zig"); +const toDType = zml.kernel.triton.from; +const ProjectionLayout = zml.moe.ProjectionLayout; +const RoutingWeightPlacement = zml.moe.RoutingWeightPlacement; + +const log = std.log.scoped(.@"zml/moe/triton"); + +fn numStages(config: LaunchConfig) i32 { + const platform = zml.Compiler.current().platform; + if (platform.target != .rocm) return config.num_stages; + + // ROCm buffers num_stages - 1 BF16 operand pairs in LDS. A single stage + // reuses scratch space between operand layout conversions. + const bytes_per_stage: i64 = @intCast(2 * config.block_size_k * (config.block_size_m + config.block_size_n)); + var stages = config.num_stages; + for (platform.devices) |device| { + const attribute = device.pjrt_desc.attribute(platform.pjrt_api, "shared_memory_per_block_optin"); + const limit = if (attribute) |value| value.int64 else 64 * 1024; + stages = @intCast(@min(stages, @divTrunc(limit, bytes_per_stage) + 1)); + } + return stages; +} + +test { + std.testing.refAllDecls(@This()); +} + +pub fn fusedExperts( + input: zml.Tensor, + ids: zml.Tensor, + weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + options: zml.moe.Options, +) zml.Tensor { + // HACK(Corentin): only rely on the scheme of gate_up for now. + + if (gate_up.quantization) |q| switch (q.scheme) { + .mxfp4 => if (zml.Compiler.current().platform.target == .cuda) { + return fusedExpertsMxfp4(input, ids, weights, gate_up, down, options); + }, + else => {}, + }; + + return fusedExpertsBase(input, ids, weights, gate_up, down, options); +} + +pub fn fusedExpertsBase( + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + opts: zml.moe.Options, +) zml.Tensor { + const gate_up_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; + const layout: zml.moe.ExpertsLayout = if (gate_up_scheme == .mxfp4) + .{ .gate_up = .interleaved, .packing = .plain } + else + .{ .gate_up = .concatenated, .packing = .plain }; + + const args: FusedExpertsArgs = .{ + .hidden_states = input, + .gate_up = gate_up, + .down = down, + .topk_weights = topk_weights, + .topk_ids = topk_ids, + .activation = opts.activation, + .quantize_input = opts.quantize_input, + .gate_up_layout = layout.gate_up, + .routing_weight_placement = opts.routing_weight_placement, + }; + const expert_partition = gate_up.weight.shape().partition(.expert); + + if (!expert_partition.isSharded()) { + return fusedExpertsImpl(args); + } + + return zml.ops.manualComputation( + (struct { + args: FusedExpertsArgs, + global_num_experts: i64, + + fn call(self: @This(), _: zml.Shape) zml.Tensor { + const local_args = self.args; + const local_num_experts = local_args.gate_up.weight.dim(.expert); + const partition_id = zml.ops.partitionId().convert(.i32); + const expert_start = partition_id.scale(local_num_experts).convert(.i32); + const global_expert_ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); + + // Map global expert ids to local ids, or -1 for experts outside this partition. + const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) + .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); + var mapped_args = local_args; + mapped_args.expert_map = local_expert_mask.select( + global_expert_ids.sub(expert_start), + zml.Tensor.scalar(-1, .i32), + ); + + const local_output = fusedExpertsImpl(mapped_args); + const local_reshaped = local_output.reshape(local_args.hidden_states.shape()).withTags(.{ .t, .d }); + return zml.ops.allReduce(local_reshaped, zml.Tensor.add); + } + }).call, + .{ .args = args, .global_num_experts = gate_up.weight.dim(.expert) }, + input.shape(), + ); +} + +pub const FusedExpertsArgs = struct { + hidden_states: Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + topk_weights: Tensor, + topk_ids: Tensor, + activation: zml.moe.Activation, + expert_map: ?Tensor = null, + /// Use FP8 activations for FP8 weights; false keeps BF16 activations. + quantize_input: bool, + gate_up_layout: ProjectionLayout, + routing_weight_placement: RoutingWeightPlacement, +}; + +pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { + const hidden_states = opts.hidden_states; + const topk_weights = opts.topk_weights; + const topk_ids = opts.topk_ids; + const num_tokens = hidden_states.dim(.t); + var launch_config = launchConfigForTokens(num_tokens); + + const gate_up_scheme = opts.gate_up.quantizationScheme(); + const down_scheme = opts.down.quantizationScheme(); + if (gate_up_scheme) |scheme| switch (scheme) { + .nvfp4 => stdx.debug.panic("triton backend doesn't support gate_up nvfp4 quantization scheme", .{}), + .fp8_block128 => launch_config.block_size_k = 128, + .fp8_block32 => launch_config.block_size_k = 32, + .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, + }; + + var down_launch_config = launchConfigForTokens(num_tokens); + if (down_scheme) |scheme| switch (scheme) { + .nvfp4 => stdx.debug.panic("triton backend doesn't support down nvfp4 quantization scheme", .{}), + .fp8_block128 => down_launch_config.block_size_k = 128, + .fp8_block32 => down_launch_config.block_size_k = 32, + .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, + }; + + const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); + const gate_up = opts.gate_up.weight.withTags(.{ .expert, .out, .in }); + const down = opts.down.weight.withTags(.{ .expert, .out, .mid }); + const routing_weights = topk_weights.reshape(.{ .token = num_tokens, .in = topk_weights.dim(.topk) }).withTags(.{ .token, .topk }); + const ids = topk_ids.reshape(.{ .token = num_tokens, .in = topk_ids.dim(.topk) }).withTags(.{ .token, .topk }); + + stdx.debug.assert(hidden.dtype() == .bf16, "triton backend expected BF16 hidden states, got {}", .{hidden.dtype()}); + if (gate_up_scheme == .mxfp4) { + stdx.debug.assert(gate_up.dtype() == .u8 or gate_up.dtype() == .i8 or gate_up.dtype() == .f4e2m1, "triton backend expected u8, i8 or f4e2m1 gate_up data type when quantization scheme is mxfp4, got {}", .{gate_up.dtype()}); + } else { + stdx.debug.assert(gate_up.dtype() == .bf16 or gate_up.dtype() == .f8e4m3fn or gate_up.dtype() == .f8e4m3fnuz, "triton backend expected gate_up data type to be bf16, fp8e4m3fn or f8e4m3fnuz, got {}", .{gate_up.dtype()}); + } + if (down_scheme == .mxfp4) { + stdx.debug.assert(down.dtype() == .u8 or down.dtype() == .i8 or down.dtype() == .f4e2m1, "triton backend expected u8, i8 or f4e2m1 down data type when quantization scheme is mxfp4, got {}", .{down.dtype()}); + } else { + stdx.debug.assert(down.dtype() == .bf16 or down.dtype() == .f8e4m3fn or down.dtype() == .f8e4m3fnuz, "triton backend expected down data type to be bf16, fp8e4m3fn or f8e4m3fnuz, got {}", .{down.dtype()}); + } + + stdx.debug.assert(routing_weights.dtype() == .f32 or routing_weights.dtype() == .bf16, "triton backend expected f32 or bf16 routing weights, got {}", .{routing_weights.dtype()}); + stdx.debug.assert(ids.dtype() == .i32, "triton backend expected i32 expert ids, got {}", .{ids.dtype()}); + const gate_up_k = gate_up.dim(.in) * @as(i64, if (gate_up_scheme == .mxfp4 and gate_up.dtype() != .f4e2m1) 2 else 1); + const down_k = down.dim(.mid) * @as(i64, if (down_scheme == .mxfp4 and down.dtype() != .f4e2m1) 2 else 1); + stdx.debug.assert(hidden.dim(.in) == gate_up_k, "triton backend expected hidden width to match gate/up input width, got {} and {}", .{ hidden.dim(.in), gate_up_k }); + const activation_reduction: i64 = switch (opts.activation) { + .gelu, .relu, .silu => 1, + .swiglu, .swiglu_step, .geglu, .geglu_tanh => 2, + }; + stdx.debug.assert(@rem(gate_up.dim(.out), activation_reduction) == 0, "triton backend expected gate/up output width to be divisible by {}, got {}", .{ activation_reduction, gate_up.dim(.out) }); + stdx.debug.assert(down_k == @divFloor(gate_up.dim(.out), activation_reduction), "triton backend expected down input width to match activated width, got {} and {}", .{ down_k, @divFloor(gate_up.dim(.out), activation_reduction) }); + stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "triton backend expected routing ids and weights to match hidden token count, got {}, {} and {}", .{ ids.dim(.token), routing_weights.dim(.token), hidden.dim(.token) }); + stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "triton backend expected routing ids and weights to have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), routing_weights.dim(.topk) }); + stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "triton backend expected gate/up and down expert counts to match, got {} and {}", .{ gate_up.dim(.expert), down.dim(.expert) }); + if (opts.expert_map) |expert_map| + stdx.debug.assert(expert_map.dtype() == .i32 and expert_map.rank() == 1, "triton backend expected a rank-1 i32 expert map, got rank {} and dtype {}", .{ expert_map.rank(), expert_map.dtype() }); + + const num_experts = if (opts.expert_map) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); + const routing = prepareRouting(ids, num_experts, @intCast(launch_config.block_size_m)); + + const expert_ids = if (opts.expert_map) |expert_map| + expert_map.gather(.{ .expert = routing.expert_ids }, .{}).withTags(.{.g}) + else + routing.expert_ids; + + const hidden_quant, const input_scale = prepareInput(hidden, gate_up_scheme, gate_up.dtype(), opts.quantize_input); + + const gate_up_out = gemm(.{ + .input = hidden_quant, + .weight = gate_up, + .bias = opts.gate_up.bias, + .input_scale = input_scale, + .weight_scale = opts.gate_up.quantizationScales(), + .routing = routing, + .expert_ids = expert_ids, + .launch_config = launch_config, + .quant_scheme = gate_up_scheme, + .top_k = @intCast(ids.dim(.topk)), + .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), + }); + + var activated = zml.moe.applyActivation(gate_up_out, opts.activation, opts.gate_up_layout); + if (opts.routing_weight_placement == .before_down) { + const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); + activated = activated.mul(weights.broad(activated.shape())); + } + const activated_quant, const down_input_scale = prepareInput(activated, down_scheme, down.dtype(), opts.quantize_input); + + const down_out = gemm(.{ + .input = activated_quant, + .weight = down, + .bias = opts.down.bias, + .input_scale = down_input_scale, + .weight_scale = opts.down.quantizationScales(), + .routing_weights = if (opts.routing_weight_placement == .after_down) routing_weights else null, + .routing = routing, + .expert_ids = expert_ids, + .launch_config = down_launch_config, + .quant_scheme = down_scheme, + .top_k = 1, + .output_shape = Shape.init(.{ .token = num_tokens, .topk = ids.dim(.topk), .out = down.dim(.out) }, .bf16), + }); + + const output = down_out.convert(.f32).sum(.topk).squeeze(.topk).convert(.bf16); + + return output.reshape(.{ .t = num_tokens, .d = down.dim(.out) }); +} + +pub const GemmOptions = struct { + input: Tensor, + weight: Tensor, + bias: ?Tensor = null, + input_scale: ?Tensor = null, + weight_scale: ?Tensor = null, + routing_weights: ?Tensor = null, + routing: Routing, + expert_ids: Tensor, + launch_config: LaunchConfig, + quant_scheme: ?zml.Quantization.Scheme = null, + top_k: usize, + output_shape: Shape, +}; + +pub fn gemm(opts: GemmOptions) Tensor { + // Native FP4 tensors expose logical K; the Triton operand is byte-packed. + const weight = if (opts.quant_scheme == .mxfp4 and opts.weight.dtype() == .f4e2m1) + opts.weight.reshape(opts.weight.shape().setDim(2, @divExact(opts.weight.dim(2), 2)).append(.{ .nibble = 2 })).bitCast(.u8) + else + opts.weight; + const weight_k = weight.dim(2) * @as(i64, if (opts.quant_scheme == .mxfp4) 2 else 1); + stdx.debug.assert(opts.input.dim(1) == weight_k, "input width {} must match logical weight K {}", .{ opts.input.dim(1), weight_k }); + if (opts.quant_scheme == .mxfp4) { + stdx.debug.assert(@mod(weight_k, 32) == 0 and opts.weight_scale != null, "MXFP4 requires K divisible by 32 and weight scales", .{}); + stdx.debug.assert(opts.input.dtype() == .bf16 and opts.input_scale == null, "MXFP4 requires BF16 activations without input scales", .{}); + } + + if (opts.quant_scheme == .fp8_block32) { + const scales = opts.weight_scale orelse stdx.debug.panic("block32 FP8 requires weight scales", .{}); + stdx.debug.assert(weight.dtype() == .f8e4m3fn and @mod(weight.dim(1), 32) == 0 and @mod(weight_k, 32) == 0, "block32 FP8 requires E4M3FN weights with N and K divisible by 32", .{}); + stdx.debug.assert(scales.rank() == 3 and scales.dim(0) == weight.dim(0) and scales.dim(1) == @divExact(weight.dim(1), 32) and scales.dim(2) == @divExact(weight_k, 32), "block32 FP8 requires scales shaped [expert, N/32, K/32], got {f}", .{scales.shape()}); + stdx.debug.assert(scales.dtype() == .f8e8m0 or scales.dtype() == .bf16 or scales.dtype() == .f32, "unsupported block32 FP8 scale dtype {}", .{scales.dtype()}); + } + + stdx.debug.assert(opts.quant_scheme != null or (opts.input_scale == null and opts.weight_scale == null), "scales require a quantization scheme", .{}); + for ([_]i64{ weight.dim(1), weight.dim(2), opts.input.dim(1), weight.dim(1) * weight.dim(2), opts.output_shape.dim(-1) }) |dim_or_stride| { + stdx.debug.assert(dim_or_stride > 0 and @mod(dim_or_stride, 16) == 0, "FusedMoe dimensions and matrix strides must be positive multiples of 16, got {}", .{dim_or_stride}); + } + const block_size_m: i64 = @intCast(opts.launch_config.block_size_m); + const block_size_n: i64 = @intCast(opts.launch_config.block_size_n); + const m_tokens = opts.input.dim(0); + const em_effective = if (m_tokens < block_size_m) + @min(opts.routing.max_num_tokens_padded, opts.routing.num_assignments * block_size_m) + else + opts.routing.max_num_tokens_padded; + stdx.debug.assert(@mod(em_effective, block_size_m) == 0, "routing capacity {} must be a multiple of block size {}", .{ em_effective, block_size_m }); + const grid_x = + @divExact(em_effective, block_size_m) * + (std.math.divCeil(i64, weight.dim(1), block_size_n) catch unreachable); + + const weight_scale: ?Tensor = if (opts.weight_scale) |scale| blk: { + const scheme = opts.quant_scheme orelse break :blk scale; + break :blk switch (scheme) { + // dot_scaled takes E8M0 encodings as bytes, including native E8M0 tensors. + .mxfp4, .mxfp8 => scale.bitCast(.u8), + .fp8_per_channel => scale.reshape(.{ weight.dim(0), weight.dim(1), 1 }), + .fp8_per_tensor => if (scale.count() == 1) scale else scale.reshape(.{ weight.dim(0), 1, 1 }), + // Tiled FP8 uses ordinary dot products with FP32 scale arithmetic. + .fp8_block32 => scale.convert(.f32), + .fp8_block128, .nvfp4 => scale, + }; + } else null; + + const stride_asm: i64 = if (opts.input_scale) |scale| (if (scale.rank() == 2) scale.dim(1) else 0) else 0; + const stride_ask: i64 = if (opts.input_scale) |scale| (if (scale.rank() == 2) 1 else 0) else 0; + const has_weight_scale_expert_stride = if (weight_scale) |scale| + !(opts.quant_scheme == .fp8_per_tensor and scale.count() == 1) + else + false; + const stride_bse: i64 = if (weight_scale) |scale| (if (scale.rank() == 3 and has_weight_scale_expert_stride) scale.dim(1) * scale.dim(2) else 0) else 0; + const stride_bsk: i64 = if (weight_scale) |scale| (if (scale.rank() == 3) 1 else 0) else 0; + const stride_bsn: i64 = if (weight_scale) |scale| (if (scale.rank() == 3) scale.dim(2) else 0) else 0; + + return kernels.FusedMoe.Kernel.call( + .{ + .a_ptr = opts.input, + .b_ptr = weight, + .b_bias_ptr = opts.bias orelse Tensor.scalar(0, .f32), + .a_scale_ptr = opts.input_scale orelse Tensor.scalar(1.0, .f32), + .b_scale_ptr = weight_scale orelse Tensor.scalar(1.0, .f32), + .routing_weights_ptr = opts.routing_weights orelse Tensor.scalar(1.0, .f32), + .sorted_token_ids_ptr = opts.routing.sorted_token_ids, + .expert_ids_ptr = opts.expert_ids, + .num_tokens_post_padded_ptr = opts.routing.num_tokens_post_padded, + .N_ptr = Tensor.constant(.{ .i64 = weight.dim(1) }).reshape(.{1}), + .K_ptr = Tensor.constant(.{ .i64 = weight_k }).reshape(.{1}), + .EM_ptr = Tensor.constant(.{ .i64 = em_effective }).reshape(.{1}), + .num_valid_tokens_ptr = Tensor.constant(.{ .i64 = opts.routing.num_assignments }).reshape(.{1}), + .stride_am_ptr = Tensor.constant(.{ .i64 = opts.input.dim(1) }).reshape(.{1}), + .stride_be_ptr = Tensor.constant(.{ .i64 = weight.dim(1) * weight.dim(2) }).reshape(.{1}), + .stride_bn_ptr = Tensor.constant(.{ .i64 = weight.dim(2) }).reshape(.{1}), + .stride_cm_ptr = Tensor.constant(.{ .i64 = weight.dim(.out) }).reshape(.{1}), + .stride_asm_ptr = Tensor.constant(.{ .i64 = stride_asm }).reshape(.{1}), + .stride_ask_ptr = Tensor.constant(.{ .i64 = stride_ask }).reshape(.{1}), + .stride_bse_ptr = Tensor.constant(.{ .i64 = stride_bse }).reshape(.{1}), + .stride_bsk_ptr = Tensor.constant(.{ .i64 = stride_bsk }).reshape(.{1}), + .stride_bsn_ptr = Tensor.constant(.{ .i64 = stride_bsn }).reshape(.{1}), + .stride_bbe_ptr = Tensor.constant(.{ .i64 = if (opts.bias) |bias| (if (bias.rank() == 2 and bias.dim(0) > 1) bias.dim(1) else 0) else 0 }).reshape(.{1}), + .stride_bbn_ptr = Tensor.constant(.{ .i64 = if (opts.bias) |bias| (if (bias.rank() > 0 and bias.dim(-1) > 1) 1 else 0) else 0 }).reshape(.{1}), + }, + .{ .c = opts.output_shape }, + .{ + .cfg = .{ + .a_dtype = toDType(opts.input.dtype()), + .b_dtype = toDType(weight.dtype()), + .c_dtype = toDType(opts.output_shape.dtype()), + .a_scale_dtype = if (opts.input_scale) |scale| toDType(scale.dtype()) else null, + .b_scale_dtype = if (weight_scale) |scale| toDType(scale.dtype()) else null, + .b_bias_dtype = if (opts.bias) |bias| toDType(bias.dtype()) else null, + .routing_weights_dtype = if (opts.routing_weights) |weights| toDType(weights.dtype()) else null, + .block_size_m = opts.launch_config.block_size_m, + .block_size_n = opts.launch_config.block_size_n, + .block_size_k = opts.launch_config.block_size_k, + .group_size_m = opts.launch_config.group_size_m, + .top_k = opts.top_k, + .naive_block_assignment = opts.routing.naive_block_assignment, + .compute_type = .bf16, + .quant_scheme = opts.quant_scheme, + }, + .grid = .{ @intCast(grid_x), 1, 1 }, + .num_warps = opts.launch_config.num_warps, + .num_stages = numStages(opts.launch_config), + }, + ).c; +} + +pub const Routing = struct { + sorted_token_ids: Tensor, + expert_ids: Tensor, + num_tokens_post_padded: Tensor, + max_num_tokens_padded: i64, + num_assignments: i64, + naive_block_assignment: bool, +}; + +pub fn prepareRouting(topk_ids: Tensor, num_experts: i64, block_size_m: i64) Routing { + stdx.debug.assert(block_size_m > 0 and @mod(block_size_m, 16) == 0, "routing block size must be a positive multiple of 16, got {}", .{block_size_m}); + const ids = topk_ids.withTags(.{ .token, .topk }).convert(.i32); + const num_assignments = ids.dim(.token) * ids.dim(.topk); + const sparsity_factor: i64 = 4; + const naive_block_assignment = num_assignments * sparsity_factor <= num_experts; + const max_num_tokens_padded = if (naive_block_assignment) + num_assignments * block_size_m + else if (num_assignments < num_experts) + num_assignments * block_size_m + else + (std.math.divCeil(i64, num_assignments + num_experts * (block_size_m - 1), block_size_m) catch unreachable) * block_size_m; + + const sorted_token_ids, const expert_ids, const num_tokens_post_padded = if (naive_block_assignment) blk: { + log.debug("Using naive block assignment for MoE kernels. Num assignments: {d}, Num experts: {d}", .{ num_assignments, num_experts }); + break :blk .{ + Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)), + ids.reshape(.{ .g = num_assignments }), + Tensor.constant(.{ .i32 = @as(i32, @intCast(max_num_tokens_padded)) }).reshape(.{1}), + }; + } else alignBlockSize(ids, num_experts, block_size_m); + + return .{ + .sorted_token_ids = sorted_token_ids, + .expert_ids = expert_ids, + .num_tokens_post_padded = num_tokens_post_padded, + .max_num_tokens_padded = max_num_tokens_padded, + .num_assignments = num_assignments, + .naive_block_assignment = naive_block_assignment, + }; +} + +fn alignBlockSize(topk_ids: Tensor, num_experts: i64, block_size_m: i64) struct { Tensor, Tensor, Tensor } { + const topk_ids_ = topk_ids.withTags(.{ .token, .topk }).convert(.i32); + const num_tokens = topk_ids_.dim(.token); + const topk = topk_ids_.dim(.topk); + const num_assignments = num_tokens * topk; + const max_num_tokens_padded = if (num_assignments < num_experts) + num_assignments * block_size_m + else + (std.math.divCeil(i64, num_assignments + num_experts * (block_size_m - 1), block_size_m) catch unreachable) * block_size_m; + const max_num_m_blocks = @divExact(max_num_tokens_padded, block_size_m); + const warp_size: i64 = 32; + const padded_num_experts: i64 = @intCast(std.math.ceilPowerOfTwoAssert(u64, @intCast(@max(num_experts, warp_size)))); + const sort_block_size: i64 = 256; + const sort_grid_x: i64 = @min(std.math.divCeil(i64, num_assignments, sort_block_size) catch unreachable, 65535); + + const flat_experts = topk_ids_.reshape(.{ .g = num_assignments }); + var cumsums = Tensor.zeroes(Shape.init(.{ .g = num_experts + 1 }, .i32)); + var expert_ids = Tensor.zeroes(Shape.init(.{ .g = max_num_m_blocks }, .i32)); + var sorted_token_ids = Tensor.zeroes(Shape.init(.{ .g = max_num_tokens_padded }, .i32)); + var num_tokens_post_padded = Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)); + + { + const align_outs = kernels.MoeAlignBlockSize.Kernel.call( + .{ + .topk_ids_ptr = flat_experts, + .sorted_token_ids_ptr = sorted_token_ids, + .expert_ids_ptr = expert_ids, + .num_tokens_post_pad_ptr = num_tokens_post_padded, + .cumsum_ptr = cumsums, + }, + .{ + .sorted_token_ids = sorted_token_ids.shape(), + .expert_ids = expert_ids.shape(), + .num_tokens_post_pad = num_tokens_post_padded.shape(), + .cumsum = cumsums.shape(), + }, + .{ + .cfg = .{ + .numel = @intCast(num_assignments), + .num_experts = @intCast(num_experts), + .padded_num_experts = @intCast(padded_num_experts), + .max_num_tokens_padded = @intCast(max_num_tokens_padded), + .max_num_m_blocks = @intCast(max_num_m_blocks), + .block_size_m = @intCast(block_size_m), + .hist_block = 256, + }, + .grid = .{ 2, 1, 1 }, + .num_stages = 1, + .num_warps = 8, + .output_operand_aliases = .{ + .sorted_token_ids = .sorted_token_ids_ptr, + .expert_ids = .expert_ids_ptr, + .num_tokens_post_pad = .num_tokens_post_pad_ptr, + .cumsum = .cumsum_ptr, + }, + }, + ); + sorted_token_ids = align_outs.sorted_token_ids; + expert_ids = align_outs.expert_ids; + num_tokens_post_padded = align_outs.num_tokens_post_pad; + cumsums = align_outs.cumsum; + } + + { + const sort_outs = kernels.CountAndSortExpertTokens.Kernel.call( + .{ + .topk_ids_ptr = flat_experts, + .sorted_token_ids_ptr = sorted_token_ids, + .cumsum_ptr = cumsums, + }, + .{ + .sorted_token_ids = sorted_token_ids.shape(), + .cumsum = cumsums.shape(), + }, + .{ + .cfg = .{ + .numel = @intCast(num_assignments), + .num_experts = @intCast(num_experts), + .sort_block_size = @intCast(sort_block_size), + }, + .grid = .{ @intCast(sort_grid_x), 1, 1 }, + .num_stages = 1, + .num_warps = 4, + .output_operand_aliases = .{ + .sorted_token_ids = .sorted_token_ids_ptr, + .cumsum = .cumsum_ptr, + }, + }, + ); + sorted_token_ids = sort_outs.sorted_token_ids; + cumsums = sort_outs.cumsum; + } + + return .{ sorted_token_ids, expert_ids, num_tokens_post_padded }; +} + +pub fn prepareInput(x: Tensor, scheme: ?zml.Quantization.Scheme, output_dtype: zml.DataType, quantize: bool) struct { Tensor, ?Tensor } { + if (quantize) { + if (scheme) |scheme_| { + if (scheme_ != .mxfp4) { + const input, const scale = quantizeFp8Input(x, scheme_, output_dtype); + return .{ input, scale }; + } + } + } + return .{ x.convert(.bf16), null }; +} + +fn quantizeFp8Input(x: Tensor, scheme: zml.Quantization.Scheme, output_dtype: DataType) struct { Tensor, Tensor } { + const group_size: i64, const scale_dtype: DataType, const absmax_epsilon: f32 = switch (scheme) { + .mxfp8 => .{ 32, .u8, 1e-10 }, + .fp8_per_channel, .fp8_per_tensor => .{ x.dim(1), .f32, 1e-10 }, + .fp8_block128 => .{ 128, .f32, 1e-10 }, + .fp8_block32 => .{ 32, .f32, 1e-4 }, + .mxfp4, .nvfp4 => unreachable, + }; + stdx.debug.assert(x.rank() == 2, "expected a rank-2 activation matrix, got {f}", .{x.shape()}); + stdx.debug.assert(@mod(x.dim(1), group_size) == 0, "activation width must be divisible by group size {d}, got {d}", .{ group_size, x.dim(1) }); + + const groups_per_row = @divExact(x.dim(1), group_size); + const quantized = Tensor.zeroes(Shape.init(.{ .token = x.dim(0), .feature = x.dim(1) }, output_dtype)); + const scales = Tensor.zeroes(Shape.init(.{ .token = x.dim(0), .group = groups_per_row }, scale_dtype)); + + const outs = kernels.PerTokenGroupQuantFp8.Kernel.call( + .{ + .y_ptr = x, + .group_size_ptr = Tensor.constant(.{ .i64 = group_size }).reshape(.{1}), + .y_num_columns_ptr = Tensor.constant(.{ .i64 = x.dim(1) }).reshape(.{1}), + .y_row_stride_ptr = Tensor.constant(.{ .i64 = x.dim(1) }).reshape(.{1}), + .eps_ptr = Tensor.scalar(absmax_epsilon, .f32), + }, + .{ .y_q = quantized.shape(), .y_s = scales.shape() }, + .{ + .cfg = .{ + .input_dtype = toDType(x.dtype()), + .output_dtype = toDType(output_dtype), + .scale_dtype = toDType(scale_dtype), + .block = std.math.ceilPowerOfTwoAssert(usize, @intCast(group_size)), + .quant_scheme = scheme, + }, + .grid = .{ @intCast(x.dim(0) * groups_per_row), 1, 1 }, + .num_stages = 1, + .num_warps = 1, + }, + ); + + return .{ outs.y_q, outs.y_s }; +} + +test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + const Local = struct { + fn forward(x: Tensor, layout: ProjectionLayout, activation: zml.moe.Activation) Tensor { + return zml.moe.applyActivation(x, activation, layout); + } + }; + const x: Tensor = .init(.{ .token = 1, .out = 6 }, .bf16); + const gates = [_]f32{ 0.75, -1.25, 3.5 }; + const ups = [_]f32{ 0.875, -2.25, 4.5 }; + for ([_]ProjectionLayout{ .concatenated, .interleaved }) |layout| { + var values: [6]zml.floats.BFloat16 = undefined; + for (gates, ups, 0..) |gate, up, i| { + values[if (layout == .concatenated) i else 2 * i] = .fromF32(gate); + values[if (layout == .concatenated) i + 3 else 2 * i + 1] = .fromF32(up); + } + var input = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&values)); + defer input.deinit(); + for ([_]?f32{ null, 2 }) |threshold| { + const activation: zml.moe.Activation = .{ .swiglu = .{ .limit = threshold } }; + var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, activation }, .{}); + defer exe.deinit(); + try zml.testing.expectEqualShapes(Shape.init(.{ .token = 1, .out = 3 }, .f32), exe.output_shapes[0]); + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + var actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + for (gates, ups, actual.constItems(f32)) |gate, up, value| { + const g = if (threshold) |limit| @min(gate, limit) else gate; + const u = if (threshold) |limit| std.math.clamp(up, -limit, limit) else up; + try std.testing.expectApproxEqAbs(g / (1 + @exp(-g)) * u, value, 1e-6); + } + } + } +} + +test "FP8 routed GEMM with bias matches dequantized weights" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + if (platform.target != .cuda) return error.SkipZigTest; + + const Local = struct { + const Outputs = struct { actual: Tensor, expected: Tensor }; + + fn expandScales(scales: Tensor, shape: Shape) Tensor { + var expanded = scales.convert(.f32); + for (0..shape.rank()) |axis| { + const factor = @divExact(shape.dim(axis), expanded.dim(axis)); + const broad_shape = expanded.shape().insert(axis + 1, .{factor}); + const axes = Shape.range(broad_shape.rank(), .i64).remove(axis + 1); + expanded = expanded.broadcast(broad_shape, axes.dims()) + .reshape(expanded.shape().setDim(axis, shape.dim(axis))); + } + return expanded.withTags(shape.tags()); + } + + fn forward(x: Tensor, w: Tensor, scales: Tensor, ids: Tensor, weights: Tensor, bias: Tensor, scheme: zml.Quantization.Scheme, mx_native: bool, quantize_input: bool) Outputs { + const weight = w.convert(.f8e4m3fn); + var input = x; + var input_scale: ?Tensor = null; + var reference_input = x.convert(.f32); + if (quantize_input) { + input, input_scale = prepareInput(x, scheme, .f8e4m3fn, true); + const scale = if (scheme == .mxfp8) input_scale.?.bitCast(.f8e8m0) else input_scale.?; + reference_input = input.convert(.f32).withTags(x.shape().tags()) + .mul(expandScales(scale.convert(.f32), x.shape())); + } + const weight_scale = if (scheme == .mxfp8) blk: { + const native = scales.convert(.f8e8m0); + break :blk if (mx_native) native else native.bitCast(.u8); + } else scales; + const config = launchConfigForTokens(8); + const routing = prepareRouting(ids, w.dim(.expert), @intCast(config.block_size_m)); + const actual = gemm(.{ + .input = input, + .weight = weight, + .bias = bias, + .input_scale = input_scale, + .weight_scale = weight_scale, + .routing_weights = weights, + .routing = routing, + .expert_ids = routing.expert_ids, + .launch_config = config, + .quant_scheme = scheme, + .top_k = 1, + .output_shape = Shape.init(.{ .token = x.dim(.token), .topk = 1, .out = w.dim(.out) }, .bf16), + }).squeeze(.topk); + const expanded_weight = weight.convert(.f32).mul(expandScales(scales, weight.shape())); + const selected_weight = expanded_weight.gather(.{ .expert = ids.squeeze(.topk) }, .{}); + const dot = reference_input.dot(selected_weight, .in); + const selected_bias = if (bias.rank() == 2) + bias.gather(.{ .expert = ids.squeeze(.topk) }, .{}) + else + bias; + const unweighted = dot.add(selected_bias.convert(.f32).broad(dot.shape())); + const expected = unweighted.mul(weights.squeeze(.topk).convert(.f32).broad(unweighted.shape())).convert(.bf16); + return .{ .actual = actual, .expected = expected }; + } + }; + + for ([_]zml.Quantization.Scheme{ .mxfp8, .fp8_per_tensor, .fp8_per_channel, .fp8_block128 }) |scheme| { + for ([_]bool{ false, true }) |quantize_input| { + for ([_]i64{ 1, 17 }) |tokens| { + const n: i64 = if (scheme == .fp8_block128) 256 else 48; + const scale_dims: [3]i64 = switch (scheme) { + .mxfp8 => .{ 8, n, 8 }, + .fp8_per_tensor => if (tokens == 1) .{ 1, 1, 1 } else .{ 8, 1, 1 }, + .fp8_per_channel => .{ 8, n, 1 }, + .fp8_block128 => .{ 8, 2, 2 }, + else => unreachable, + }; + const x: Tensor = .init(.{ .token = tokens, .in = 256 }, .bf16); + const w: Tensor = .init(.{ .expert = 8, .out = n, .in = 256 }, .bf16); + const scales: Tensor = .init(.{ scale_dims[0], scale_dims[1], scale_dims[2] }, .f32); + const ids: Tensor = .init(.{ .token = tokens, .topk = 1 }, .i32); + const weights: Tensor = .init(.{ .token = tokens, .topk = 1 }, .bf16); + const bias: Tensor = if (tokens == 1) + .init(.{ .out = n }, .bf16) + else + .init(.{ .expert = 8, .out = n }, .f32); + var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, w, scales, ids, weights, bias, scheme, tokens == 1, quantize_input }, .{}); + defer exe.deinit(); + var buffers: [6]zml.Buffer = undefined; + var initialized: usize = 0; + defer for (buffers[0..initialized]) |*buffer| buffer.deinit(); + for ([_]Shape{ x.shape(), w.shape(), scales.shape(), ids.shape(), weights.shape(), bias.shape() }, 0..) |shape, i| { + const slice = try zml.Slice.alloc(allocator, shape); + defer slice.free(allocator); + for (0..shape.count()) |j| { + switch (i) { + 2 => slice.items(f32)[j] = @as(f32, @floatFromInt(@as(u32, 1) << @as(u5, @intCast(j % 4)))) / 16, + 3 => slice.items(i32)[j] = @intCast((j * 3 + 5) % 8), + 5 => { + const value = @as(f32, @floatFromInt(@as(i32, @intCast(j % 17)) - 8)) / 2; + if (shape.dtype() == .f32) { + slice.items(f32)[j] = value; + } else { + slice.items(zml.floats.BFloat16)[j] = .fromF32(value); + } + }, + else => slice.items(zml.floats.BFloat16)[j] = .fromF32(if (i == 4) + @as(f32, @floatFromInt(1 + j % 3)) / 4 + else + @as(f32, @floatFromInt(@as(i32, @intCast((j * 7 + j / 256) % 13)) - 6)) / 4), + } + } + buffers[i] = try zml.Buffer.fromSlice(io, platform, slice); + initialized += 1; + } + var output: zml.Bufferized(Local.Outputs) = undefined; + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + runner.run(io, .{ buffers[0], buffers[1], buffers[2], buffers[3], buffers[4], buffers[5] }, .{&output}, .{ .wait = true }); + defer zml.Buffer.deinitAll(Local.Outputs, &output); + zml.testing.expectClose(io, output.expected, output.actual, .{ .absolute_tolerance = 0.03125, .relative_tolerance = 0.01 }) catch |err| { + log.warn("FP8 routed GEMM failed for scheme={s}, quantize_input={}, tokens={}", .{ @tagName(scheme), quantize_input, tokens }); + return err; + }; + } + } + } +} + +test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + if (platform.target != .cuda) return error.SkipZigTest; + const Local = struct { + fn forward(x: Tensor, layout: ProjectionLayout, placement: RoutingWeightPlacement, storage_dtype: DataType) Tensor { + const fp4 = storage_dtype != .bf16; + const columns = Tensor.arange(.{ .end = 256 }, .i32).withTags(.{.dout}); + const is_gate = switch (layout) { + .concatenated => columns.cmp(.LT, Tensor.scalar(128, .i32)), + .interleaved => columns.remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), + }; + const dtype: DataType = if (fp4) .u8 else .bf16; + // E2M1 codes 1, 2, 3 encode 0.5, 1, 1.5; each byte holds two values. + const gate_value = if (fp4) Tensor.scalar(0x11, .u8) else Tensor.scalar(0.5, .bf16); + const up_value = if (fp4) Tensor.scalar(0x33, .u8) else Tensor.scalar(1.5, .bf16); + const gate_up_values = is_gate.select(gate_value, up_value); + var gate_up: zml.nn.Linear = .{ + .weight = gate_up_values.broad(Shape.init(.{ .expert = 8, .dout = 256, .d = @as(i64, if (fp4) 64 else 128) }, dtype)), + .tag = Shape.toTag(.d), + .quantization = if (fp4) .{ + .scheme = .mxfp4, + .scales = Tensor.scalar(127, .u8).broad(Shape.init(.{ .expert = 8, .dout = 256, .d = 4 }, .u8)), + } else null, + }; + var down: zml.nn.Linear = .{ + .weight = (if (fp4) Tensor.scalar(0x22, .u8) else Tensor.scalar(1, .bf16)).broad(Shape.init(.{ .expert = 8, .d = 128, .dout = @as(i64, if (fp4) 64 else 128) }, dtype)), + .bias = Tensor.scalar(2, .bf16).broad(Shape.init(.{ .expert = 8, .d = 128 }, .bf16)), + .tag = Shape.toTag(.dout), + .quantization = if (fp4) .{ + .scheme = .mxfp4, + .scales = Tensor.scalar(127, .u8).broad(Shape.init(.{ .expert = 8, .d = 128, .dout = 4 }, .u8)), + } else null, + }; + if (storage_dtype == .f4e2m1) { + gate_up.weight = gate_up.weight.bitCast(.f4e2m1).reshape(.{ .expert = 8, .dout = 256, .d = 128 }); + down.weight = down.weight.bitCast(.f4e2m1).reshape(.{ .expert = 8, .d = 128, .dout = 128 }); + gate_up.quantization.?.scales = gate_up.quantization.?.scales.bitCast(.f8e8m0); + down.quantization.?.scales = down.quantization.?.scales.bitCast(.f8e8m0); + } + const route = Tensor.arange(.{ .end = 2 }, .i32).reshape(.{ .t = 1, .topk = 2 }) + .broad(Shape.init(.{ .t = x.dim(.t), .topk = 2 }, .i32)); + return fusedExpertsImpl(.{ + .hidden_states = x, + .gate_up = gate_up, + .down = down, + .topk_ids = route.addConstant(2), + .topk_weights = route.convert(.f32).addConstant(1).scale(0.25), + .gate_up_layout = layout, + .routing_weight_placement = placement, + .quantize_input = false, + .activation = .{ .swiglu = .{} }, + }); + } + }; + for ([_]DataType{ .bf16, .u8, .f4e2m1 }) |storage_dtype| { + const tokens: i64 = if (storage_dtype == .u8) 17 else 1; + const x: Tensor = .init(.{ .t = tokens, .d = 128 }, .bf16); + const host = try zml.Slice.alloc(allocator, x.shape()); + defer host.free(allocator); + @memset(host.items(zml.floats.BFloat16), .fromF32(1.0 / 128.0)); + var input = try zml.Buffer.fromSlice(io, platform, host); + defer input.deinit(); + for ([_]ProjectionLayout{ .concatenated, .interleaved }) |layout| { + for ([_]RoutingWeightPlacement{ .before_down, .after_down }) |placement| { + var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, placement, storage_dtype }, .{}); + defer exe.deinit(); + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + var actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + const activation: f32 = 0.5 / (1 + @exp(@as(f32, -0.5))) * 1.5; + // The two routes have weights 0.25 and 0.5. Bias distinguishes placement. + const bias: f32 = if (placement == .before_down) 4 else 1.5; + const expected = 128 * activation * 0.75 + bias; + for (actual.constItems(zml.floats.BFloat16)) |value| { + try std.testing.expectApproxEqAbs(expected, value.toF32(), 0.5); + } + } + } + } +} + +// ============================================================================= +// Config / validation helpers +// ============================================================================= + +const LaunchConfig = struct { + block_size_m: usize, + block_size_n: usize, + block_size_k: usize, + group_size_m: usize, + num_warps: i32, + num_stages: i32, +}; + +const launch_configs = [_]struct { tokens: i64, config: LaunchConfig }{ + .{ .tokens = 1, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 4 } }, + .{ .tokens = 2, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 4 } }, + .{ .tokens = 4, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, + .{ .tokens = 8, .config = .{ .block_size_m = 16, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 16, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 64, .group_size_m = 64, .num_warps = 4, .num_stages = 5 } }, + .{ .tokens = 24, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, + .{ .tokens = 32, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 128, .group_size_m = 1, .num_warps = 4, .num_stages = 2 } }, + .{ .tokens = 48, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 128, .group_size_m = 64, .num_warps = 4, .num_stages = 2 } }, + .{ .tokens = 64, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 128, .group_size_m = 1, .num_warps = 4, .num_stages = 2 } }, + .{ .tokens = 96, .config = .{ .block_size_m = 16, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 128, .config = .{ .block_size_m = 16, .block_size_n = 256, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, + .{ .tokens = 256, .config = .{ .block_size_m = 16, .block_size_n = 256, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, + .{ .tokens = 512, .config = .{ .block_size_m = 32, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 1024, .config = .{ .block_size_m = 64, .block_size_n = 128, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, + .{ .tokens = 1536, .config = .{ .block_size_m = 64, .block_size_n = 128, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, + .{ .tokens = 2048, .config = .{ .block_size_m = 128, .block_size_n = 128, .block_size_k = 64, .group_size_m = 16, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 3072, .config = .{ .block_size_m = 128, .block_size_n = 256, .block_size_k = 64, .group_size_m = 1, .num_warps = 8, .num_stages = 4 } }, + .{ .tokens = 4096, .config = .{ .block_size_m = 128, .block_size_n = 256, .block_size_k = 64, .group_size_m = 16, .num_warps = 8, .num_stages = 4 } }, +}; + +fn launchConfigForTokens(num_tokens: i64) LaunchConfig { + var best = launch_configs[0]; + for (launch_configs[1..]) |candidate| { + // The table is ascending, so ties retain the smaller token bucket. + if (@abs(candidate.tokens - num_tokens) < @abs(best.tokens - num_tokens)) { + best = candidate; + } + } + return best.config; +} + +test "MoE launch config selects the nearest token bucket" { + try std.testing.expectEqualDeep(launchConfigForTokens(4), launchConfigForTokens(6)); + try std.testing.expectEqualDeep(launchConfigForTokens(8), launchConfigForTokens(7)); + try std.testing.expectEqualDeep(launchConfigForTokens(1), launchConfigForTokens(0)); + try std.testing.expectEqualDeep(launchConfigForTokens(4096), launchConfigForTokens(8192)); + try std.testing.expectEqual(@as(usize, 64), launchConfigForTokens(16).group_size_m); + try std.testing.expectEqual(@as(i32, 5), launchConfigForTokens(16).num_stages); +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + const cc = zml.platform.cuda.computeCapability(platform) orelse return false; + return cc.major == 10; +} + +pub fn validateOptions(opts: zml.moe.Options) void { + stdx.debug.assert(opts.activation == .swiglu, "triton backend only accepts swiglu activation, got {}", .{opts.activation}); + stdx.debug.assert(opts.activation.swiglu.limit != null, "triton backend requires swiglu limit to be set", .{}); + stdx.debug.assert(opts.activation.swiglu.bias == null, "triton backend requires swiglu bias to be null", .{}); + stdx.debug.assert(opts.activation.swiglu.scale == null, "triton backend requires swiglu scale to be null", .{}); +} + +/// Row-major MXFP4 E2M1 weights and linear E8M0 block32 scales. +/// BF16 activations are quantized on the GPU to FP8 with per-32 E8M0 scales. +pub fn fusedExpertsMxfp4( + input: zml.Tensor, + ids: zml.Tensor, + weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + options: zml.moe.Options, +) zml.Tensor { + validateOptions(options); + stdx.debug.assert(input.dtype() == .bf16, "triton backend only supports bf16 inputs, got {}", .{input.dtype()}); + stdx.debug.assert(gate_up.bias == null and down.bias == null, "triton backend expects gate_up bias and down bias to be null", .{}); + + const gq = gate_up.quantization orelse @panic("triton backend requires gate_up quantization to be set"); + const dq = down.quantization orelse @panic("triton backend requires down quantization to be set"); + stdx.debug.assert(gq.scheme == .mxfp4 and dq.scheme == .mxfp4, "triton expects gate_up and down quantization scheme to be mxfp4, got {} and {}", .{ gq.scheme, dq.scheme }); + + // Weight storage for mxfp4 in HF is expressed as u8 or i8 + stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "triton expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); + stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "triton expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); + + const expert_parallelism = gate_up.weight.shape().partition(.expert).isSharded(); + + const context: Context = .{ + .input = input, + .ids = ids, + .weights = weights, + .w1 = gate_up.weight.bitCast(.u8), + .s1 = gq.scales, + .w2 = down.weight.bitCast(.u8), + .s2 = dq.scales, + .global_experts = gate_up.weight.dim(.expert), + .topk = ids.dim(.topk), + .limit = options.activation.swiglu.limit.?, + .routing_weight_placement = options.routing_weight_placement, + .expert_parallel = expert_parallelism, + }; + + return if (expert_parallelism) + zml.ops.manualComputation(Context.body, context, input.shape()) + else + context.body(input.shape()); +} + +pub const Config = struct { + tokens: i64, + hidden: i64, + intermediate: i64, + experts: i64, + global_experts: i64, + topk: i64, + expert_offset: i64 = 0, + // Swiglu clamping. Set to 0 to disable. + swiglu_limit: f32, + /// Scale SwiGLU before FP8 quantization, or scale the down result before reduction. + routing_weight_placement: RoutingWeightPlacement = .before_down, + + pub fn validate(c: Config) !void { + if (c.tokens <= 0 or + c.experts <= 0 or + c.topk <= 0 or c.topk > c.global_experts or + c.expert_offset < 0 or c.expert_offset + c.experts > c.global_experts) + { + return error.InvalidRoutingShape; + } + + if (c.hidden < 128 or @mod(c.hidden, 128) != 0 or + c.intermediate < 128 or @mod(c.intermediate, 128) != 0) + { + return error.InvalidHiddenShape; + } + + if (!std.math.isFinite(c.swiglu_limit) or c.swiglu_limit < 0) { + return error.InvalidScale; + } + } +}; + +pub fn validateInputs( + c: Config, + input: zml.Tensor, + ids: zml.Tensor, + weights: zml.Tensor, + w1: zml.Tensor, + s1: zml.Tensor, + w2: zml.Tensor, + s2: zml.Tensor, +) !void { + if (s1.rank() != 3 or s2.rank() != 3) return error.InvalidInputShape; + if (s1.dim(2) < @divTrunc(c.hidden, 32) or + s2.dim(2) < @divTrunc(c.intermediate, 32)) return error.InvalidInputShape; + const shapes = .{ + .x = zml.Shape.init(.{ c.tokens, c.hidden }, .bf16), + .ids = zml.Shape.init(.{ c.tokens, c.topk }, .i32), + .scales = zml.Shape.init(.{ c.tokens, c.topk }, .f32), + .w1 = zml.Shape.init(.{ c.experts, 2 * c.intermediate, @divTrunc(c.hidden, 2) }, .u8), + .s1 = zml.Shape.init(.{ c.experts, 2 * c.intermediate, s1.dim(2) }, .f8e8m0), + .w2 = zml.Shape.init(.{ c.experts, c.hidden, @divTrunc(c.intermediate, 2) }, .u8), + .s2 = zml.Shape.init(.{ c.experts, c.hidden, s2.dim(2) }, .f8e8m0), + }; + + if (!input.shape().eql(shapes.x)) return error.InvalidInputShape; + if (!ids.shape().eql(shapes.ids)) return error.InvalidInputShape; + if (!weights.shape().eql(shapes.scales)) return error.InvalidInputShape; + if (!w1.shape().eql(shapes.w1)) return error.InvalidInputShape; + if (!s1.shape().eql(shapes.s1)) return error.InvalidInputShape; + if (!w2.shape().eql(shapes.w2)) return error.InvalidInputShape; + if (!s2.shape().eql(shapes.s2)) return error.InvalidInputShape; +} + +const Context = struct { + input: zml.Tensor, + ids: zml.Tensor, + weights: zml.Tensor, + w1: zml.Tensor, + s1: zml.Tensor, + w2: zml.Tensor, + s2: zml.Tensor, + global_experts: i64, + topk: i64, + limit: f32, + routing_weight_placement: RoutingWeightPlacement, + expert_parallel: bool, + + fn body(self: Context, _: zml.Shape) zml.Tensor { + const experts = self.w1.dim(.expert); + const hidden = self.w2.dim(1); + const intermediate = self.w2.dim(2) * 2; + var ids = self.ids.convert(.i32).reshape(.{ .token = .auto, .topk = self.topk }); + const tokens = ids.dim(.token); + if (self.expert_parallel) { + const partition_id = zml.ops.partitionId().convert(.i32); + ids = ids.sub(partition_id.scale(experts)); + } + const x = self.input.reshape(.{ tokens, hidden }); + const scales = self.weights.convert(.f32).reshape(.{ tokens, self.topk }); + + const cfg: Config = .{ + .tokens = tokens, + .hidden = hidden, + .intermediate = intermediate, + .experts = experts, + .global_experts = self.global_experts, + .topk = self.topk, + .swiglu_limit = self.limit, + .routing_weight_placement = self.routing_weight_placement, + }; + + cfg.validate() catch @panic("Invalid MXFP4 MoE configuration"); + validateInputs(cfg, x, ids, scales, self.w1, self.s1, self.w2, self.s2) catch @panic("Invalid MXFP4 tensor layout"); + + const bn = 16; + const bk = 512; + + const tile_count = kernels.tiles(cfg.tokens, cfg.topk, cfg.experts, bn); + + // Quantize activations + const q = kernels.Quant.Kernel.call( + .{ .x = x }, + .{ + .q = .init(.{ cfg.tokens, cfg.hidden }, .f8e4m3fn), + .s = .init(.{ cfg.tokens, @divTrunc(cfg.hidden, 32) }, .u8), + }, + .{ + .cfg = .{ .tokens = cfg.tokens, .hidden = cfg.hidden }, + .grid = .{ @intCast(@divTrunc(cfg.tokens + kernels.quantRows(cfg.tokens) - 1, kernels.quantRows(cfg.tokens))), @intCast(@divTrunc(cfg.hidden, 128)), 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ); + + const dummy = zml.Tensor.scalar(0, .i32); + var map = dummy; + var perm = dummy; + var sched = dummy; + + if (cfg.tokens != 1) { + const r = kernels.Route.Kernel.call( + .{ .ids = ids }, + .{ .counts = .init(.{cfg.experts}, .i32), .pos = .init(.{cfg.tokens * cfg.topk}, .i32) }, + .{ .cfg = .{ + .tokens = cfg.tokens, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + }, .grid = .{ @intCast(cfg.experts), 1, 1 }, .num_warps = 4, .num_stages = 1 }, + ); + + const s = kernels.Schedule.Kernel.call( + .{ .ids = ids, .counts = r.counts, .pos = r.pos }, + .{ .map = .init(.{tile_count * bn}, .i32), .perm = .init(.{cfg.tokens * cfg.topk}, .i32), .sched = .init(.{2 * tile_count + 1}, .i32) }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .experts = cfg.experts, + .capacity = tile_count, + .group = bn, + }, + .grid = .{ @intCast(@divTrunc(cfg.tokens * cfg.topk + 127, 128)), 1, 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ); + + map = s.map; + perm = s.perm; + sched = s.sched; + } + + // Each narrow tile produces 64 activations: two complete scale groups. + const up_columns: i64 = if (cfg.tokens <= 8) 64 else 128; + const m = kernels.GateUp.Kernel.call( + .{ .q = q.q, .s = q.s, .w = self.w1, .ws = self.s1.bitCast(.u8), .ids = ids, .rw = scales, .map = map, .sched = sched }, + .{ .mid = .init(.{ tile_count * bn, cfg.intermediate }, .f8e4m3fn), .ms = .init(.{ tile_count * bn, @divTrunc(cfg.intermediate, 32) }, .u8) }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .hidden = cfg.hidden, + .intermediate = cfg.intermediate, + .experts = cfg.experts, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .swiglu_limit = cfg.swiglu_limit, + .scale_stride = self.s1.dim(2), + .routing_weight_placement = cfg.routing_weight_placement, + .bn = bn, + .bk = bk, + }, + .grid = .{ @intCast(@divTrunc(cfg.intermediate, up_columns)), @intCast(tile_count), 1 }, + .num_warps = if (cfg.tokens <= 8) 4 else 8, + .num_stages = 2, + .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.intermediate, up_columns) * tile_count), + }, + ); + + const d = kernels.Down.Kernel.call( + .{ .mid = m.mid, .ms = m.ms, .w = self.w2, .ws = self.s2.bitCast(.u8), .ids = ids, .sched = sched }, + .{ .d = .init(.{ tile_count * bn, cfg.hidden }, .bf16) }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .hidden = cfg.hidden, + .intermediate = cfg.intermediate, + .experts = cfg.experts, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .swiglu_limit = cfg.swiglu_limit, + .scale_stride = self.s2.dim(2), + .routing_weight_placement = cfg.routing_weight_placement, + .bn = bn, + .bk = bk, + }, + .grid = .{ @intCast(@divTrunc(cfg.hidden, 128)), @intCast(tile_count), 1 }, + .num_warps = if (cfg.tokens <= 8) 4 else 8, + .num_stages = 2, + .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.hidden, 128) * tile_count), + }, + ); + + const result = kernels.Combine.Kernel.call( + .{ .d = d.d, .ids = ids, .perm = perm, .rw = scales }, + .{ .y = x.shape() }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .hidden = cfg.hidden, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .experts = cfg.experts, + .routing_weight_placement = cfg.routing_weight_placement, + .bn = bn, + }, + .grid = .{ @intCast(cfg.tokens), @intCast(@divTrunc(cfg.hidden + kernels.combineBlock(cfg.tokens, cfg.hidden) - 1, kernels.combineBlock(cfg.tokens, cfg.hidden))), 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ).y.reshape(self.input.shape().dims()).withTags(self.input.shape()); + + return if (self.expert_parallel) + zml.ops.allReduce(result, zml.Tensor.add) + else + result; + } +}; diff --git a/zml/moe/fly_kernels/moe.zig b/zml/moe/fly_kernels/moe.zig deleted file mode 100644 index 201ef3ac..00000000 --- a/zml/moe/fly_kernels/moe.zig +++ /dev/null @@ -1,289 +0,0 @@ -const std = @import("std"); -const zml = @import("../../zml.zig"); -const shared = @import("../fused_experts.zig"); -const mxfp4 = @import("mxfp4.zig"); -const Tensor = zml.Tensor; -const Shape = zml.Shape; -const fly = zml.kernel.fly; -const Builder = fly.Builder; - -pub const Parameters = shared.Parameters; - -pub fn supports(opts: shared.FusedExpertsArgs) bool { - if (opts.hidden_states.dtype() != .bf16 or opts.gate_up.quantizationScheme() != .mxfp4 or - opts.down.quantizationScheme() != .mxfp4 or opts.gate_up.bias != null or opts.down.bias != null or - opts.routing_weight_placement != .before_down) return false; - const gate_up = opts.gate_up.weight; - const down = opts.down.weight; - if (gate_up.rank() != 3 or down.rank() != 3 or opts.hidden_states.dim(.d) != 5120) return false; - const gate_up_scale = opts.gate_up.quantizationScales() orelse return false; - const down_scale = opts.down.quantizationScales() orelse return false; - const experts = if (opts.expert_map) |map| map.dim(.expert) else gate_up.dim(0); - if (experts < 1 or experts > 512) return false; - inline for (.{ .{ gate_up, gate_up_scale, 4608, 5120 }, .{ down, down_scale, 5120, 2304 } }) |projection| { - const weight, const scale, const n, const k = projection; - const packing_factor: i64 = switch (weight.dtype()) { - .u8, .i8 => 2, - .f4e2m1 => 1, - else => return false, - }; - if (weight.dim(0) < 1 or weight.dim(0) > 512 or weight.dim(1) != n or weight.dim(2) * packing_factor != k or - scale.rank() != 3 or scale.dim(0) != weight.dim(0) or scale.dim(1) != n or scale.dim(2) != k / 32) return false; - switch (scale.dtype()) { - .u8, .i8, .f8e8m0 => {}, - else => return false, - } - } - const routes = opts.hidden_states.dim(.b) * opts.hidden_states.dim(.s) * opts.topk_ids.dim(.topk); - return routes > 0 and routes <= std.math.maxInt(i32) / 5120; -} - -pub fn prepareInput(x: Tensor, scheme: ?zml.Quantization.Scheme, _: zml.DataType, _: bool) struct { Tensor, ?Tensor } { - std.debug.assert(scheme == .mxfp4); - return .{ x.convert(.bf16), null }; -} - -pub fn call(opts: shared.GemmOptions) zml.Tensor { - zml.stdx.debug.assert(opts.quant_scheme == .mxfp4 and opts.bias == null and opts.routing_weights == null and opts.input_scale == null, "Fly MoE requires MXFP4 weights, BF16 activations and routing weights before the down projection", .{}); - const weight = if (opts.weight.dtype() == .f4e2m1) - opts.weight.reshape(opts.weight.shape().setDim(2, @divExact(opts.weight.dim(2), 2)).append(.{ .nibble = 2 })).bitCast(.u8) - else - opts.weight; - return mxfp4.call(opts.input, weight, opts.weight_scale.?.bitCast(.u8), opts.routing.sorted_token_ids, opts.expert_ids, opts.routing.num_tokens_post_padded, opts.output_shape, opts.top_k, opts.routing.naive_block_assignment, opts.launch_config.block_size_m); -} - -pub fn prepareRouting(topk_ids: Tensor, num_experts: i64, block_size_m: i64) shared.Routing { - std.debug.assert(num_experts > 0 and num_experts <= 512); - std.debug.assert(block_size_m > 0 and block_size_m <= 128 and @mod(block_size_m, 16) == 0); - const ids = topk_ids.withTags(.{ .token, .topk }).convert(.i32); - const num_assignments = ids.dim(.token) * ids.dim(.topk); - // GEMV handles up to six routes; MFMA shares weights across sorted rows. - const naive_block_assignment = num_assignments <= 6; - const max_num_tokens_padded = if (naive_block_assignment or num_assignments < num_experts) - num_assignments * block_size_m - else - (std.math.divCeil(i64, num_assignments + num_experts * (block_size_m - 1), block_size_m) catch unreachable) * block_size_m; - const sorted_token_ids, const expert_ids, const num_tokens_post_padded = if (naive_block_assignment) blk: { - break :blk .{ - Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)), - ids.reshape(.{ .g = num_assignments }), - Tensor.constant(.{ .i32 = @as(i32, @intCast(max_num_tokens_padded)) }).reshape(.{1}), - }; - } else blk: { - const flat_experts = ids.reshape(.{ .g = num_assignments }); - const aligned = Align.call( - flat_experts, - Tensor.zeroes(Shape.init(.{ .g = max_num_tokens_padded }, .i32)), - Tensor.zeroes(Shape.init(.{ .g = @divExact(max_num_tokens_padded, block_size_m) }, .i32)), - Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)), - Tensor.zeroes(Shape.init(.{ .g = num_experts + 1 }, .i32)), - block_size_m, - ); - const sorted = Sort.call(flat_experts, aligned.sorted_token_ids, aligned.cumsum); - break :blk .{ sorted.sorted_token_ids, aligned.expert_ids, aligned.num_tokens_post_pad }; - }; - return .{ - .sorted_token_ids = sorted_token_ids, - .expert_ids = expert_ids, - .num_tokens_post_padded = num_tokens_post_padded, - .max_num_tokens_padded = max_num_tokens_padded, - .num_assignments = num_assignments, - .naive_block_assignment = naive_block_assignment, - }; -} - -const Align = struct { - const Result = struct { - sorted_token_ids: Tensor, - expert_ids: Tensor, - num_tokens_post_pad: Tensor, - cumsum: Tensor, - }; - - fn supports(ids: Tensor, sorted: Tensor, experts: Tensor, total: Tensor, cumsum: Tensor, block_size: i64) bool { - for ([_]Tensor{ ids, sorted, experts, total, cumsum }) |tensor| { - if (tensor.rank() != 1 or tensor.dtype() != .i32) return false; - } - const num_experts = cumsum.dim(0) - 1; - if (num_experts < 1 or num_experts > 512 or total.dim(0) != 1 or ids.dim(0) > std.math.maxInt(i32) or - block_size <= 0 or block_size > 128 or @mod(block_size, 16) != 0) return false; - const capacity_bound = if (ids.dim(0) < num_experts) - ids.dim(0) * block_size - else - @divTrunc(ids.dim(0) + num_experts * (block_size - 1) + block_size - 1, block_size) * block_size; - return capacity_bound <= std.math.maxInt(i32) and sorted.dim(0) >= capacity_bound and - sorted.dim(0) <= std.math.maxInt(i32) and @mod(sorted.dim(0), block_size) == 0 and - experts.dim(0) == @divTrunc(sorted.dim(0), block_size); - } - - /// All four outputs alias their corresponding input buffers. Invalid expert IDs - /// are ignored, and all unused expert blocks are marked -1. Prefix ends in LDS - /// let each thread find its output block's expert without scanning every expert. - fn call(ids: Tensor, sorted: Tensor, experts: Tensor, total: Tensor, cumsum: Tensor, block_size: i64) Result { - std.debug.assert(Align.supports(ids, sorted, experts, total, cumsum, block_size)); - const cfg: Config = .{ - .routes = ids.dim(0), - .experts = cumsum.dim(0) - 1, - .capacity = sorted.dim(0), - .blocks = experts.dim(0), - .block_size = block_size, - .padding = block_size - 1, - }; - const result = Kernel.call(.{ .ids = ids, .sorted = sorted, .experts = experts, .total = total, .cumsum_in = cumsum }, .{ .sorted_token_ids = sorted.shape(), .expert_ids = experts.shape(), .num_tokens_post_pad = total.shape(), .cumsum = cumsum.shape() }, .{ - .cfg = cfg, - .threads = 512, - .grid = .{ 2, 1, 1 }, - .output_operand_aliases = .{ .sorted_token_ids = .sorted, .expert_ids = .experts, .num_tokens_post_pad = .total, .cumsum = .cumsum_in }, - }); - return .{ .sorted_token_ids = result.sorted_token_ids, .expert_ids = result.expert_ids, .num_tokens_post_pad = result.num_tokens_post_pad, .cumsum = result.cumsum }; - } - - const Config = struct { - routes: i64, - experts: i64, - capacity: i64, - blocks: i64, - block_size: i64, - padding: i64, - }; - - const Kernel = zml.kernel.fly.Kernel(Config, .{ - .name = "fly_moe_align", - .inputs = &.{ "ids", "sorted", "experts", "total", "cumsum_in" }, - .outputs = &.{ "sorted_token_ids", "expert_ids", "num_tokens_post_pad", "cumsum" }, - .run = run, - }); - - fn run(b: *Builder, cfg: Config) fly.FinishError!void { - const args = Kernel.args(b); - const tid = b.threadId(.x); - const tid64 = tid.to(.i64); - const zero = b.constant(.i32, 0); - const one = b.constant(.i32, 1); - const experts = b.constant(.i32, cfg.experts); - // Distinct arrays match the histogram, inclusive ends and wave totals. - const counts = b.sharedArray(.i32, 512, 16); - const ends = b.sharedArray(.i32, 512, 16); - const waves = b.sharedArray(.i32, 8, 16); - var fill = b.openIfElse(b.blockId(.x).cmp(.ne, 0), .{}); - var positions = b.openFor(tid64, cfg.capacity, 512, .{}); - args.sorted_token_ids.set(positions.iv, b.constant(.i32, cfg.routes)); - positions.yield(.{}); - fill.yieldThen(.{}); - - counts.set(tid, zero); - b.barrier(); - var routes = b.openFor(tid64, cfg.routes, 512, .{}); - const id = args.ids.at(routes.iv); - var valid_id = b.openIf(id.cmp(.ge, 0).bitAnd(id.cmp(.lt, cfg.experts))); - _ = b.ptrAtomicAdd(counts.emitIter().addOffset(id), one, .workgroup); - valid_id.yieldThen(.{}); - routes.yield(.{}); - b.barrier(); - - const padded = counts.at(tid).add(cfg.padding).divUnsigned(cfg.block_size).mul(cfg.block_size); - const lane = tid.remUnsigned(64); - const wave = tid.divUnsigned(64); - var prefix = padded; - inline for (.{ 1, 2, 4, 8, 16, 32 }) |shift| { - const peer = prefix.shuffle(.up, shift, 64); - prefix = prefix.add(lane.cmpUnsigned(.ge, shift).select(peer, zero)); - } - var last_lane = b.openIf(lane.cmp(.eq, 63)); - waves.set(wave, prefix); - last_lane.yieldThen(.{}); - b.barrier(); - - var sums = b.openFor(zero, 8, 1, .{ zero, zero }); - const value = waves.at(sums.iv); - const preceding = sums.carried[0].add(sums.iv.cmpUnsigned(.lt, wave).select(value, zero)); - const total = sums.carried[1].add(value); - sums.yield(.{ preceding, total }); - const end = sums.results[0].add(prefix); - const start = end.sub(padded); - ends.set(tid, end); - var has_expert = b.openIf(tid.cmpUnsigned(.lt, cfg.experts)); - args.cumsum.set(tid64, start); - has_expert.yieldThen(.{}); - var first = b.openIf(tid.cmp(.eq, 0)); - args.cumsum.set(b.constant(.i64, cfg.experts), sums.results[1]); - args.num_tokens_post_pad.set(b.constant(.i64, 0), sums.results[1]); - first.yieldThen(.{}); - b.barrier(); - - var blocks = b.openFor(tid64, cfg.blocks, 512, .{}); - const offset = blocks.iv.to(.i32).mul(cfg.block_size); - var block_expert = b.openIfElse(offset.cmpUnsigned(.lt, sums.results[1]), .{zero.type_()}); - // Upper bound of inclusive expert ends handles empty experts without a scan. - var bounds = b.openWhile(.{ zero, experts }, .{ zero.type_(), zero.type_() }); - bounds.yieldBefore(bounds.before_carried[0].cmpUnsigned(.lt, bounds.before_carried[1]), .{ bounds.before_carried[0], bounds.before_carried[1] }); - const lo = bounds.after_carried[0]; - const hi = bounds.after_carried[1]; - const mid = lo.add(hi).shrU(1); - const advance = ends.at(mid).cmpUnsigned(.le, offset); - bounds.yieldAfter(.{ advance.select(mid.add(1), lo), advance.select(hi, mid) }); - block_expert.yieldThen(.{bounds.results[0]}); - block_expert.yieldElse(.{b.constant(.i32, -1)}); - args.expert_ids.set(blocks.iv, block_expert.results[0]); - blocks.yield(.{}); - fill.yieldElse(.{}); - } -}; - -const Sort = struct { - const Result = struct { - sorted_token_ids: Tensor, - cumsum: Tensor, - }; - - fn supports(ids: Tensor, sorted_token_ids: Tensor, cumsum: Tensor) bool { - return ids.rank() == 1 and sorted_token_ids.rank() == 1 and cumsum.rank() == 1 and - ids.dtype() == .i32 and sorted_token_ids.dtype() == .i32 and cumsum.dtype() == .i32 and - cumsum.dim(0) >= 2 and cumsum.dim(0) <= std.math.maxInt(i32) and - ids.dim(0) <= std.math.maxInt(i32) and sorted_token_ids.dim(0) >= ids.dim(0); - } - - /// cumsum contains one starting offset per expert followed by the total padded - /// capacity. Each valid route atomically advances its expert's cursor; ordering - /// within an expert is unspecified. Invalid expert IDs leave both buffers alone. - fn call(ids: Tensor, sorted_token_ids: Tensor, cumsum: Tensor) Result { - std.debug.assert(Sort.supports(ids, sorted_token_ids, cumsum)); - if (ids.dim(0) == 0) return .{ .sorted_token_ids = sorted_token_ids, .cumsum = cumsum }; - const routes = ids.dim(0); - const grid = @min(@divTrunc(routes + 255, 256), 65535); - const cfg: Config = .{ .routes = routes, .experts = cumsum.dim(0) - 1, .step = grid * 256 }; - const result = Kernel.call(.{ .ids = ids, .sorted = sorted_token_ids, .cumsum_in = cumsum }, .{ .sorted_token_ids = sorted_token_ids.shape(), .cumsum = cumsum.shape() }, .{ - .cfg = cfg, - .threads = 256, - .grid = .{ @intCast(grid), 1, 1 }, - .output_operand_aliases = .{ .sorted_token_ids = .sorted, .cumsum = .cumsum_in }, - }); - return .{ .sorted_token_ids = result.sorted_token_ids, .cumsum = result.cumsum }; - } - - const Config = struct { - routes: i64, - experts: i64, - step: i64, - }; - - const Kernel = zml.kernel.fly.Kernel(Config, .{ - .name = "fly_count_and_sort_expert_tokens", - .inputs = &.{ "ids", "sorted", "cumsum_in" }, - .outputs = &.{ "sorted_token_ids", "cumsum" }, - .run = run, - }); - - fn run(b: *Builder, cfg: Config) fly.FinishError!void { - const args = Kernel.args(b); - const start = b.blockId(.x).to(.i64).mul(256).add(b.threadId(.x).to(.i64)); - var routes = b.openFor(start, cfg.routes, cfg.step, .{}); - const id = args.ids.at(routes.iv); - var valid = b.openIf(id.cmp(.ge, 0).bitAnd(id.cmp(.lt, cfg.experts))); - const cursor = args.cumsum.emitIter().addOffset(id.to(.i64)); - const rank = b.ptrAtomicAdd(cursor, b.constant(.i32, 1), .agent); - args.sorted_token_ids.set(rank.to(.i64), routes.iv.to(.i32)); - valid.yieldThen(.{}); - routes.yield(.{}); - } -}; diff --git a/zml/moe/fly_kernels/mxfp4.zig b/zml/moe/fly_kernels/mxfp4.zig deleted file mode 100644 index 13c10bf6..00000000 --- a/zml/moe/fly_kernels/mxfp4.zig +++ /dev/null @@ -1,462 +0,0 @@ -const std = @import("std"); -const zml = @import("../../zml.zig"); -const Tensor = zml.Tensor; -const fly = zml.kernel.fly; -const Value = fly.Value; -const Builder = fly.Builder; - -/// The caller owns route localization, activation and reduction. -/// `expert_ids` contains local bank indices, including -1 for remote experts. -/// Sorted routes must contain a real prefix followed by padding per expert. -/// `block_m` is their alignment; native tiles contain 16 or 32 rows. -pub fn call( - input: Tensor, - weight: Tensor, - scales: Tensor, - sorted_ids: Tensor, - expert_ids: Tensor, - padded_count: Tensor, - output: zml.Shape, - topk: usize, - naive: bool, - block_m: usize, -) Tensor { - std.debug.assert(input.rank() == 2 and input.dtype() == .bf16 and weight.rank() == 3 and - (weight.dtype() == .u8 or weight.dtype() == .i8) and scales.rank() == 3 and scales.dtype() == .u8 and - sorted_ids.rank() == 1 and sorted_ids.dtype() == .i32 and expert_ids.rank() == 1 and expert_ids.dtype() == .i32 and - padded_count.count() == 1 and padded_count.dtype() == .i32 and output.rank() >= 2 and output.dtype() == .bf16 and - topk > 0 and block_m > 0 and block_m % 16 == 0); - const n = weight.dim(1); - const k = input.dim(1); - std.debug.assert((n == 4608 and k == 5120) or (n == 5120 and k == 2304)); - std.debug.assert(input.dim(0) > 0 and weight.dim(0) > 0 and weight.dim(2) == @divExact(k, 2) and - scales.dim(0) == weight.dim(0) and scales.dim(1) == n and scales.dim(2) == @divExact(k, 32) and - output.dim(-1) == n); - // Keep packed-word counts and route/output indices within signed i32. - std.debug.assert(weight.count() / 4 <= std.math.maxInt(i32) and output.count() <= std.math.maxInt(i32) and - sorted_ids.count() <= std.math.maxInt(i32)); - const routes: i64 = @intCast(output.count() / @as(usize, @intCast(n))); - std.debug.assert(topk <= routes and @mod(@as(usize, @intCast(routes)), topk) == 0 and - input.dim(0) == @divTrunc(routes, @as(i64, @intCast(topk)))); - const capacity: i64 = @intCast(sorted_ids.count()); - const blocks: i64 = @intCast(expert_ids.count()); - if (naive) { - std.debug.assert(blocks == routes); - } else { - std.debug.assert(capacity > 0 and block_m <= capacity and @mod(capacity, @as(i64, @intCast(block_m))) == 0 and - blocks == @divExact(capacity, @as(i64, @intCast(block_m)))); - } - if (naive and routes <= 6) return Gemv.call(input, weight, scales, expert_ids, output, topk); - return callMfma(input, weight, scales, sorted_ids, expert_ids, padded_count, output, topk, naive, block_m); -} - -fn callMfma(input: Tensor, weight: Tensor, scales: Tensor, sorted_ids: Tensor, expert_ids: Tensor, padded_count: Tensor, output: zml.Shape, topk: usize, naive: bool, block_m: usize) Tensor { - const n = weight.dim(1); - const k = input.dim(1); - const routes = input.dim(0) * @as(i64, @intCast(topk)); - const native_m: i64 = if (!naive and block_m % 32 == 0) 32 else 16; - const cfg: Config = .{ - .tokens = input.dim(0), - .experts = weight.dim(0), - .n = n, - .k = k, - .ka = @divExact(k, 4), - .kb = @divExact(k, 8), - .ks = @divExact(k, 32), - .topk = @as(i64, @intCast(topk)), - .routes = routes, - .capacity = @as(i64, @intCast(sorted_ids.count())), - .blocks = @as(i64, @intCast(expert_ids.count())), - .subtiles = @divExact(@as(i64, @intCast(block_m)), native_m), - .native_m = native_m, - .wide = @as(i64, @intFromBool(native_m == 32)), - .stage_elements = native_m * 128, - .shared_elements = native_m * 256, - .naive = @as(i64, @intFromBool(naive)), - }; - const a = input.reshape(.{ input.dim(0), @divExact(k, 4), 4 }).bitCast(.i64); - const b = weight.reshape(.{ weight.dim(0), n, @divExact(k, 8), 4 }).bitCast(.i32); - return Kernel.call(.{ .a = a, .b = b, .scales = scales, .sorted = sorted_ids, .experts = expert_ids, .padded = padded_count.reshape(.{1}) }, .{ - .output = zml.Shape.init(.{ routes, n }, .bf16), - }, .{ - .cfg = cfg, - .threads = 256, - .grid = .{ @intCast(@divExact(n, 64)), @intCast(if (naive) routes else @divExact(cfg.capacity, native_m)), 1 }, - }).output.reshape(output); -} - -const Config = struct { - tokens: i64, - experts: i64, - n: i64, - k: i64, - ka: i64, - kb: i64, - ks: i64, - topk: i64, - routes: i64, - capacity: i64, - blocks: i64, - subtiles: i64, - native_m: i64, - wide: i64, - stage_elements: i64, - shared_elements: i64, - naive: i64, -}; - -const Kernel = zml.kernel.fly.Kernel(Config, .{ - .name = "mxfp4_raw_mfma", - .inputs = &.{ "a", "b", "scales", "sorted", "experts", "padded" }, - .outputs = &.{"output"}, - .run = run, -}); - -fn run(b: *Builder, cfg: Config) fly.FinishError!void { - const args = Kernel.args(b); - const zero = b.constant(.i32, 0); - const fzero = b.constant(.f32, 0.0); - const zeros = fzero.splat(4); - const block_n = b.blockId(.x).to(.i64); - const block_m = b.blockId(.y).to(.i64); - const route_start = block_m.mul(cfg.native_m); - const has_rows = if (cfg.naive != 0) b.constant(.i1, 1) else blk: { - const padded = args.padded.at(b.constant(.i64, 0)).to(.i64); - var has_block = b.openIfElse(route_start.cmpUnsigned(.lt, padded), .{b.constant(.i1, 0).type_()}); - has_block.yieldThen(.{args.sorted.at(route_start).cmp(.ne, cfg.routes)}); - has_block.yieldElse(.{b.constant(.i1, 0)}); - break :blk has_block.results[0]; - }; - var block = b.openIf(has_rows); - const expert_block = if (cfg.naive != 0) block_m else block_m.divUnsigned(cfg.subtiles); - const eid = args.experts.at(expert_block); - const valid_expert = eid.cmp(.ge, 0).bitAnd(eid.cmp(.lt, cfg.experts)); - const thread = b.threadId(.x).to(.i64); - const lane = thread.remUnsigned(64); - const wave = thread.divUnsigned(64); - const mn_lane = lane.remUnsigned(16); - const k_lane = lane.divUnsigned(16); - const out_n = block_n.mul(64).add(wave.mul(16)).add(mn_lane); - // Each expert has a contiguous real prefix before route-count padding. - const second_active = if (cfg.wide != 0) blk: { - const first = args.sorted.at(route_start.add(16)); - break :blk first.cmp(.ge, 0).bitAnd(first.cmp(.lt, cfg.routes)); - } else b.constant(.i1, 0); - - var sums = b.openIfElse(valid_expert, .{ zeros.type_(), zeros.type_() }); - const expert = eid.to(.i64); - const load_row = thread.divUnsigned(16); - const load_chunk = thread.remUnsigned(16); - const a_route = routeForRow(b, args.sorted, block_m, route_start, load_row, cfg.naive != 0); - const a_valid = a_route.cmp(.ge, 0).bitAnd(a_route.cmp(.lt, cfg.routes)); - const a_base = a_route.to(.i64).divUnsigned(cfg.topk).mul(cfg.ka); - var second_route = b.openIfElse(second_active, .{zero.type_()}); - second_route.yieldThen(.{args.sorted.at(route_start.add(load_row).add(16))}); - second_route.yieldElse(.{b.constant(.i32, -1)}); - const a_route_g1 = second_route.results[0]; - const a_valid_g1 = a_route_g1.cmp(.ge, 0).bitAnd(a_route_g1.cmp(.lt, cfg.routes)); - const a_base_g1 = a_route_g1.to(.i64).divUnsigned(cfg.topk).mul(cfg.ka); - - // One allocation, two disjoint ping-pong regions; never two identical leaves. - const shared = b.sharedArray(.i16, cfg.shared_elements, 16); - const load_a = b.copyAtom(.{ .universal = 128 }, .i64); - const load_b = b.copyAtom(.{ .universal = 128 }, .i32); - const copy_shared = b.copyAtom(.{ .universal = 128 }, .i16); - const loader: TileLoader = .{ - .a = args.a, - .weight = args.b, - .scales = args.scales, - .shared = shared, - .load_a = load_a, - .load_b = load_b, - .copy_shared = copy_shared, - .a_base = a_base, - .a_base_g1 = a_base_g1, - .a_valid = a_valid, - .a_valid_g1 = a_valid_g1, - .second_active = second_active, - .a_col = load_chunk.mul(2), - .a_store = load_row.mul(128).add(load_chunk.bitXor(load_row).mul(8)), - .b_base = expert.mul(cfg.n * cfg.kb).add(out_n.mul(cfg.kb)), - .expert = expert, - .out_n = out_n, - .k_lane = k_lane, - }; - const initial = loader.load(b, b.constant(.i64, 0), b.constant(.i64, 0)); - b.barrier(); - const atom = b.mmaAtom((fly.rocdl.MmaOpCDNA3MFMAType.get(b.ctx, .{ - .m = 16, - .n = 16, - .k = 16, - .elemTyA = fly.DType.bf16.toMlir(b.ctx), - .elemTyB = fly.DType.bf16.toMlir(b.ctx), - .elemTyAcc = fly.DType.f32.toMlir(b.ctx), - }) catch return error.InvalidMlir).type_()); - var k = b.openFor(b.constant(.i64, 0), cfg.k, 128, .{ zeros, zeros, initial.words, initial.scale }); - const read_base = k.iv.divUnsigned(128).bitAnd(1).mul(cfg.stage_elements); - const write_base = k.iv.divUnsigned(128).bitAnd(1).bitXor(1).mul(cfg.stage_elements); - var a_fragments: [4]Value = undefined; - var a_fragments_g1: [4]Value = undefined; - inline for (0..4) |u| { - const offset = read_base.add(mn_lane.mul(128)).add(k_lane.mul(4).add(u).bitXor(mn_lane).mul(8)); - a_fragments[u] = b.copyAtomLoad(copy_shared, vectorView(shared, offset, 8)); - } - inline for (0..4) |u| { - const offset = read_base.add(mn_lane.mul(128)).add(k_lane.mul(4).add(u).bitXor(mn_lane).mul(8)).add(2048); - const azero = b.constant(.i16, 0).splat(8); - var second = b.openIfElse(second_active, .{azero.type_()}); - second.yieldThen(.{b.copyAtomLoad(copy_shared, vectorView(shared, offset, 8))}); - second.yieldElse(.{azero}); - a_fragments_g1[u] = second.results[0]; - } - // Prefetch the other LDS region only after all current A fragments are read. - const next_k = k.iv.add(128); - var next = b.openIfElse(next_k.cmpUnsigned(.lt, cfg.k), .{ initial.words.type_(), initial.scale.type_() }); - const next_tile = loader.load(b, next_k, write_base); - next.yieldThen(.{ next_tile.words, next_tile.scale }); - next.yieldElse(.{ b.constant(.i32, 0).splat(4), fzero }); - var acc = k.carried[0]; - var acc_g1 = k.carried[1]; - inline for (0..4) |u| { - const decoded = decodeWeight(b, k.carried[2].extract(u), k.carried[3]); - const a0 = a_fragments[u].shuffleVector(a_fragments[u], &.{ 0, 1, 2, 3 }).bitcast(.bf16); - const a1 = a_fragments[u].shuffleVector(a_fragments[u], &.{ 4, 5, 6, 7 }).bitcast(.bf16); - const w0 = decoded.shuffleVector(decoded, &.{ 0, 1, 2, 3 }); - const w1 = decoded.shuffleVector(decoded, &.{ 4, 5, 6, 7 }); - acc = b.mmaAtomCall(atom, a0, w0, acc); - acc = b.mmaAtomCall(atom, a1, w1, acc); - var second = b.openIfElse(second_active, .{zeros.type_()}); - const a0_g1 = a_fragments_g1[u].shuffleVector(a_fragments_g1[u], &.{ 0, 1, 2, 3 }).bitcast(.bf16); - const a1_g1 = a_fragments_g1[u].shuffleVector(a_fragments_g1[u], &.{ 4, 5, 6, 7 }).bitcast(.bf16); - const first_acc = b.mmaAtomCall(atom, a0_g1, w0, acc_g1); - second.yieldThen(.{b.mmaAtomCall(atom, a1_g1, w1, first_acc)}); - second.yieldElse(.{acc_g1}); - acc_g1 = second.results[0]; - } - // All current reads finish before a later iteration reuses this LDS region. - b.barrier(); - k.yield(.{ acc, acc_g1, next.results[0], next.results[1] }); - sums.yieldThen(.{ k.results[0], k.results[1] }); - sums.yieldElse(.{ zeros, zeros }); - - inline for (0..4) |i| { - storeRow(b, args.output, args.sorted, block_m, route_start, k_lane.mul(4).add(i), out_n, sums.results[0].extract(i), cfg); - } - var store_second = b.openIf(second_active); - inline for (0..4) |i| { - storeRow(b, args.output, args.sorted, block_m, route_start, k_lane.mul(4).add(i + 16), out_n, sums.results[1].extract(i), cfg); - } - store_second.yieldThen(.{}); - block.yieldThen(.{}); -} - -fn routeForRow(b: *Builder, sorted: Value, block: Value, start: Value, row: Value, naive: bool) Value { - return if (naive) row.cmp(.eq, 0).select(block.to(.i32), b.constant(.i32, -1)) else sorted.at(start.add(row)); -} - -fn storeRow(b: *Builder, output: Value, sorted: Value, block: Value, start: Value, row: Value, col: Value, value: Value, cfg: Config) void { - const route = routeForRow(b, sorted, block, start, row, cfg.naive != 0); - var valid = b.openIf(route.cmp(.ge, 0).bitAnd(route.cmp(.lt, cfg.routes))); - output.set(.{ route.to(.i64), col }, value.to(.bf16)); - valid.yieldThen(.{}); -} - -fn vectorView(tensor: Value, offset: Value, comptime n: i64) Value { - return tensor.emitIter().addOffset(offset).view(fly.L(n, 1)); -} - -const TileLoader = struct { - a: Value, - weight: Value, - scales: Value, - shared: Value, - load_a: Value, - load_b: Value, - copy_shared: Value, - a_base: Value, - a_base_g1: Value, - a_valid: Value, - a_valid_g1: Value, - second_active: Value, - a_col: Value, - a_store: Value, - b_base: Value, - expert: Value, - out_n: Value, - k_lane: Value, - - fn load(self: TileLoader, b: *Builder, k: Value, write_base: Value) struct { words: Value, scale: Value } { - self.storeActivation(b, self.a_base, self.a_valid, k, self.a_store.add(write_base)); - var second = b.openIf(self.second_active); - self.storeActivation(b, self.a_base_g1, self.a_valid_g1, k, self.a_store.add(write_base).add(2048)); - second.yieldThen(.{}); - const offset = self.b_base.add(k.divUnsigned(8)).add(self.k_lane.mul(4)); - const words = b.copyAtomLoad(self.load_b, vectorView(self.weight, offset, 4)); - const byte = self.scales.at(.{ self.expert, self.out_n, k.divUnsigned(32).add(self.k_lane) }); - const scale = byte.toUnsigned(.i32); - const finite = scale.cmp(.eq, 0).select(b.constant(.i32, 0x00400000), scale.shl(23)); - return .{ .words = words, .scale = scale.cmp(.eq, 255).select(b.constant(.i32, 0x7fc00000), finite).bitcast(.f32) }; - } - - fn storeActivation(self: TileLoader, b: *Builder, base: Value, valid: Value, k: Value, destination: Value) void { - const zeros = b.constant(.i64, 0).splat(2); - var guarded = b.openIfElse(valid, .{zeros.type_()}); - const offset = base.add(k.divUnsigned(4)).add(self.a_col); - guarded.yieldThen(.{b.copyAtomLoad(self.load_a, vectorView(self.a, offset, 2))}); - guarded.yieldElse(.{zeros}); - b.copyAtomStore(self.copy_shared, guarded.results[0].bitcast(.i16), vectorView(self.shared, destination, 8)); - } -}; - -fn decodeWeight(b: *Builder, word: Value, scale: Value) Value { - const lo = b.constant(.i32, 0x44403800); - const hi = b.constant(.i32, 0x54504c48); - const even = b.perm(hi, lo, word.bitAnd(0x07070707)); - const odd = b.perm(hi, lo, word.shrU(4).bitAnd(0x07070707)); - const ev0 = b.cvtPkF32Fp8(even, false); - const od0 = b.cvtPkF32Fp8(odd, false); - const ev1 = b.cvtPkF32Fp8(even, true); - const od1 = b.cvtPkF32Fp8(odd, true); - const first = ev0.shuffleVector(od0, &.{ 0, 2, 1, 3 }); - const last = ev1.shuffleVector(od1, &.{ 0, 2, 1, 3 }); - const magnitude = first.shuffleVector(last, &.{ 0, 1, 2, 3, 4, 5, 6, 7 }); - var shifts = b.constant(.i32, 0).splat(8); - inline for (0..8) |i| shifts = shifts.insert(i, b.constant(.i32, i * 4)); - const signs = word.splat(8).shrU(shifts).bitAnd(8).shl(28); - // FNUZ has no -0: restore FP4's sign in F32 before scaling and BF16 rounding. - return magnitude.bitcast(.i32).bitOr(signs).bitcast(.f32).mul(scale).to(.bf16); -} - -const Gemv = struct { - // Packed-word GEMV derived from the supplied moe-mi300x/fly_word.py algorithm. - // Several rows share a CTA, and the shorter down projection uses subwaves to - // expose independent outputs without paying for a full wave reduction per row. - const Config = struct { rows: usize, lanes: usize, unroll: usize }; - - /// One local expert ID per original route; negative/out-of-bank IDs write zero. - fn call(x: Tensor, weight: Tensor, scales: Tensor, ids: Tensor, output: zml.Shape, topk: usize) Tensor { - const cfg: Gemv.Config = if (weight.dim(1) == 4608) - .{ .rows = 4, .lanes = 64, .unroll = 1 } - else - .{ .rows = 4, .lanes = 32, .unroll = 5 }; - const rows = cfg.rows; - const lanes = cfg.lanes; - const unroll = cfg.unroll; - const n = weight.dim(1); - const k = x.dim(1); - const routes: i64 = @intCast(ids.count()); - std.debug.assert(x.dtype() == .bf16 and (weight.dtype() == .u8 or weight.dtype() == .i8)); - std.debug.assert(@mod(n, 8) == 0 and @mod(k, 32) == 0); - std.debug.assert(topk > 0 and routes == x.dim(0) * @as(i64, @intCast(topk))); - std.debug.assert(weight.dim(2) * 2 == k); - std.debug.assert(scales.dim(0) == weight.dim(0) and scales.dim(1) == n and scales.dim(2) == @divExact(k, 32)); - const tile_rows = rows * (64 / lanes); - std.debug.assert(@mod(n, @as(i64, @intCast(tile_rows * 8))) == 0); - const fields: EmitConfig = .{ .unroll = unroll, .rows = tile_rows, .lanes = lanes, .stride = unroll * lanes, .ntiles = @divExact(n, @as(i64, @intCast(tile_rows))), .tokens = x.dim(0), .experts = weight.dim(0), .n = n, .k_pairs = @divExact(k, 2), .k_words = @divExact(k, 8), .k_scales = @divExact(k, 32), .routes = routes, .grid = @divExact(n, @as(i64, @intCast(tile_rows))) * routes, .topk = @as(i64, @intCast(topk)) }; - const a = x.reshape(.{ .token = x.dim(0), .in = @divExact(k, 2), .pair = 2 }).bitCast(.i32); - const b = weight.reshape(.{ .expert = weight.dim(0), .out = n, .in = @divExact(weight.dim(2), 4), .word = 4 }).bitCast(.i32); - const matrix_output = zml.Shape.init(.{ .token = routes, .out = n }, output.dtype()); - return Gemv.Kernel.call(.{ .a = a, .b = b, .scales = scales.bitCast(.u8), .ids = ids.reshape(.{routes}) }, .{ - .output = matrix_output, - }, .{ - .cfg = fields, - .threads = @intCast(rows * 64), - .grid = .{ @intCast(fields.grid), 1, 1 }, - }).output.reshape(output); - } - - const EmitConfig = struct { - unroll: usize, - rows: usize, - lanes: usize, - stride: usize, - ntiles: i64, - tokens: i64, - experts: i64, - n: i64, - k_pairs: i64, - k_words: i64, - k_scales: i64, - routes: i64, - grid: i64, - topk: i64, - }; - - const Kernel = zml.kernel.fly.Kernel(EmitConfig, .{ - .name = "mxfp4_gemv", - .inputs = &.{ "a", "b", "scales", "ids" }, - .outputs = &.{"output"}, - .run = Gemv.run, - }); - - fn run(b: *Builder, cfg: EmitConfig) fly.FinishError!void { - const args = Gemv.Kernel.args(b); - const block = b.blockId(.x).to(.i64); - const thread = b.threadId(.x).to(.i64); - const lane = thread.remUnsigned(cfg.lanes); - const rowgroup = thread.divUnsigned(cfg.lanes); - // Preserve the eight-XCD workgroup permutation and original subwave layout. - const pid = block.remUnsigned(8).mul(@divExact(cfg.grid, 8)).add(block.divUnsigned(8)); - const row = pid.remUnsigned(cfg.ntiles).mul(cfg.rows).add(rowgroup); - const route = pid.divUnsigned(cfg.ntiles); - const expert32 = args.ids.at(route); - const expert = expert32.to(.i64); - const token = route.divUnsigned(cfg.topk); - const local = expert32.cmp(.ge, 0).bitAnd(expert32.cmp(.lt, cfg.experts)); - const inrow = row.cmpUnsigned(.lt, cfg.n); - const fzero = b.constant(.f32, 0.0); - var sum = b.openIfElse(local.bitAnd(inrow), .{fzero.type_()}); - var k = b.openFor(lane, cfg.k_words, cfg.stride, .{fzero}); - var carried = k.carried[0]; - for (0..cfg.unroll) |u| { - const pos = k.iv.add(u * cfg.lanes); - var inbounds = b.openIfElse(pos.cmpUnsigned(.lt, cfg.k_words), .{fzero.type_()}); - const word = args.b.at(.{ expert, row, pos }); - const scale8 = args.scales.at(.{ expert, row, pos.divUnsigned(4) }); - const scale = decodeScale(b, scale8); - const low = word.bitAnd(0x07070707); - const high = word.shrU(4).bitAnd(0x07070707); - const even_bits = b.perm(b.constant(.i32, 0xd4d0ccc8), b.constant(.i32, 0xc4c0b800), low); - const odd_bits = b.perm(b.constant(.i32, 0xd4d0ccc8), b.constant(.i32, 0xc4c0b800), high); - const even = even_bits.bitAnd(word.shl(4).bitOr(0x7f7f7f7f)); - const odd = odd_bits.bitAnd(word.bitOr(0x7f7f7f7f)); - // Retain the scalar decoder and its FP32 scale -> BF16 -> FP32 boundary. - const weights = [_]Value{ - b.cvtPkF32Fp8(even, false).mul(scale).to(.bf16).to(.f32), - b.cvtPkF32Fp8(odd, false).mul(scale).to(.bf16).to(.f32), - b.cvtPkF32Fp8(even, true).mul(scale).to(.bf16).to(.f32), - b.cvtPkF32Fp8(odd, true).mul(scale).to(.bf16).to(.f32), - }; - var partial = fzero; - inline for (0..4) |pair| { - const x = args.a.at(.{ token, pos.mul(4).add(pair) }); - const xlo = x.to(.i16).bitcast(.bf16).to(.f32); - const xhi = x.shrU(16).to(.i16).bitcast(.bf16).to(.f32); - const wlo = weights[(pair / 2) * 2].extract(pair % 2); - const whi = weights[(pair / 2) * 2 + 1].extract(pair % 2); - // Odd then even FMA, followed by one add to the K-loop accumulator. - partial = b.fma(xlo, wlo, b.fma(xhi, whi, partial)); - } - inbounds.yieldThen(.{partial.add(carried)}); - inbounds.yieldElse(.{carried}); - carried = inbounds.results[0]; - } - k.yield(.{carried}); - var reduced = k.results[0]; - var shift: i32 = 1; - while (shift < @as(i32, @intCast(cfg.lanes))) : (shift *= 2) { - reduced = reduced.add(reduced.shuffleXor(shift, 64)); - } - sum.yieldThen(.{reduced}); - sum.yieldElse(.{fzero}); - var store = b.openIf(lane.cmp(.eq, 0).bitAnd(inrow)); - args.output.set(.{ route, row }, sum.results[0].to(.bf16)); - store.yieldThen(.{}); - } - - fn decodeScale(b: *Builder, byte: Value) Value { - const scale = byte.toUnsigned(.i32); - const finite = scale.cmp(.eq, 0).select(b.constant(.i32, 0x00400000), scale.shl(23)); - return scale.cmp(.eq, 255).select(b.constant(.i32, 0x7fc00000), finite).bitcast(.f32); - } -}; diff --git a/zml/moe/fused_experts.zig b/zml/moe/fused_experts.zig deleted file mode 100644 index e7c23dfb..00000000 --- a/zml/moe/fused_experts.zig +++ /dev/null @@ -1,550 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -const zml = @import("../zml.zig"); -const Tensor = zml.Tensor; -const Shape = zml.Shape; -const DataType = zml.DataType; -const fly = @import("fly_kernels/moe.zig"); -const triton = @import("triton.zig"); -const callFusedMoe = triton.call; - -const log = std.log.scoped(.@"zml/moe/fused_experts"); - -test { - std.testing.refAllDecls(@This()); -} - -pub fn fusedExperts( - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - opts: zml.moe.Options, - comptime backend: zml.moe.Backend, -) zml.Tensor { - stdx.debug.assertComptime(backend == .triton or backend == .fly, "fusedExperts only supports the triton and fly backend", .{}); - - const gate_up_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; - const layout: zml.moe.ExpertsLayout = if (gate_up_scheme == .mxfp4) - .{ .gate_up = .interleaved, .packing = .plain } - else - .{ .gate_up = .concatenated, .packing = .plain }; - - const args: FusedExpertsArgs = .{ - .hidden_states = input, - .gate_up = gate_up, - .down = down, - .topk_weights = topk_weights, - .topk_ids = topk_ids, - .activation = opts.activation, - .quantize_input = opts.quantize_input, - .gate_up_layout = layout.gate_up, - .routing_weight_placement = opts.routing_weight_placement, - }; - const expert_partition = gate_up.weight.shape().partition(.expert); - - if (!expert_partition.isSharded()) { - return fusedExpertsImpl(args, backend); - } - - return zml.ops.manualComputation( - (struct { - args: FusedExpertsArgs, - global_num_experts: i64, - - fn call(self: @This(), _: zml.Shape) zml.Tensor { - const local_args = self.args; - const local_num_experts = local_args.gate_up.weight.dim(.expert); - const partition_id = zml.ops.partitionId().convert(.i32); - const expert_start = partition_id.scale(local_num_experts).convert(.i32); - const global_expert_ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); - - // Map global expert ids to local ids, or -1 for experts outside this partition. - const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) - .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); - var mapped_args = local_args; - mapped_args.expert_map = local_expert_mask.select( - global_expert_ids.sub(expert_start), - zml.Tensor.scalar(-1, .i32), - ); - - const local_output = fusedExpertsImpl(mapped_args, backend); - const local_reshaped = local_output.reshape(local_args.hidden_states.shape().dims()).withTags(.{ .b, .s, .d }); - return zml.ops.allReduce(local_reshaped, zml.Tensor.add); - } - }).call, - .{ .args = args, .global_num_experts = gate_up.weight.dim(.expert) }, - input.shape(), - ); -} - -pub const ProjectionLayout = enum { concatenated, interleaved }; -pub const RoutingWeightPlacement = enum { before_down, after_down }; - -pub const FusedExpertsArgs = struct { - hidden_states: Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - topk_weights: Tensor, - topk_ids: Tensor, - activation: zml.moe.Activation, - expert_map: ?Tensor = null, - /// Use FP8 activations for FP8 weights; false keeps BF16 activations. - quantize_input: bool, - gate_up_layout: ProjectionLayout, - routing_weight_placement: RoutingWeightPlacement, -}; - -pub fn fusedExpertsImpl(opts: FusedExpertsArgs, comptime backend: zml.moe.Backend) Tensor { - const Impl = switch (backend) { - .fly => fly, - .triton => triton, - else => @compileError("unsupported fused-experts backend"), - }; - const backend_name = @tagName(backend); - if (backend == .fly and (!backend.isAvailable(zml.Compiler.current().platform) or !fly.supports(opts))) { - return fusedExpertsImpl(opts, .triton); - } - - const hidden_states = opts.hidden_states; - const topk_weights = opts.topk_weights; - const topk_ids = opts.topk_ids; - const b = hidden_states.dim(.b); - const s = hidden_states.dim(.s); - const num_tokens = b * s; - var launch_config = launchConfigForTokens(num_tokens); - - const gate_up_scheme = opts.gate_up.quantizationScheme(); - const down_scheme = opts.down.quantizationScheme(); - if (gate_up_scheme) |scheme| switch (scheme) { - .nvfp4 => stdx.debug.panic("{s} backend doesn't support gate_up nvfp4 quantization scheme", .{backend_name}), - .fp8_block128 => launch_config.block_size_k = 128, - .fp8_block32 => launch_config.block_size_k = 32, - .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, - }; - - var down_launch_config = launchConfigForTokens(num_tokens); - if (down_scheme) |scheme| switch (scheme) { - .nvfp4 => stdx.debug.panic("{s} backend doesn't support down nvfp4 quantization scheme", .{backend_name}), - .fp8_block128 => down_launch_config.block_size_k = 128, - .fp8_block32 => down_launch_config.block_size_k = 32, - .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, - }; - - const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); - const gate_up = opts.gate_up.weight.withTags(.{ .expert, .out, .in }); - const down = opts.down.weight.withTags(.{ .expert, .out, .mid }); - const routing_weights = topk_weights.reshape(.{ .token = num_tokens, .in = topk_weights.dim(.topk) }).withTags(.{ .token, .topk }); - const ids = topk_ids.reshape(.{ .token = num_tokens, .in = topk_ids.dim(.topk) }).withTags(.{ .token, .topk }); - - stdx.debug.assert(hidden.dtype() == .bf16, "{s} backend expected BF16 hidden states, got {}", .{ backend_name, hidden.dtype() }); - if (gate_up_scheme == .mxfp4) { - stdx.debug.assert(gate_up.dtype() == .u8 or gate_up.dtype() == .i8 or gate_up.dtype() == .f4e2m1, "{s} backend expected u8, i8 or f4e2m1 gate_up data type when quantization scheme is mxfp4, got {}", .{ backend_name, gate_up.dtype() }); - } else { - stdx.debug.assert(gate_up.dtype() == .bf16 or gate_up.dtype() == .f8e4m3fn or gate_up.dtype() == .f8e4m3fnuz, "{s} backend expected gate_up data type to be bf16, fp8e4m3fn or f8e4m3fnuz, got {}", .{ backend_name, gate_up.dtype() }); - } - if (down_scheme == .mxfp4) { - stdx.debug.assert(down.dtype() == .u8 or down.dtype() == .i8 or down.dtype() == .f4e2m1, "{s} backend expected u8, i8 or f4e2m1 down data type when quantization scheme is mxfp4, got {}", .{ backend_name, down.dtype() }); - } else { - stdx.debug.assert(down.dtype() == .bf16 or down.dtype() == .f8e4m3fn or down.dtype() == .f8e4m3fnuz, "{s} backend expected down data type to be bf16, fp8e4m3fn or f8e4m3fnuz, got {}", .{ backend_name, down.dtype() }); - } - - stdx.debug.assert(routing_weights.dtype() == .f32 or routing_weights.dtype() == .bf16, "{s} backend expected f32 or bf16 routing weights, got {}", .{ backend_name, routing_weights.dtype() }); - stdx.debug.assert(ids.dtype() == .i32, "{s} backend expected i32 expert ids, got {}", .{ backend_name, ids.dtype() }); - const gate_up_k = gate_up.dim(.in) * @as(i64, if (gate_up_scheme == .mxfp4 and gate_up.dtype() != .f4e2m1) 2 else 1); - const down_k = down.dim(.mid) * @as(i64, if (down_scheme == .mxfp4 and down.dtype() != .f4e2m1) 2 else 1); - stdx.debug.assert(hidden.dim(.in) == gate_up_k, "{s} backend expected hidden width to match gate/up input width, got {} and {}", .{ backend_name, hidden.dim(.in), gate_up_k }); - const activation_reduction: i64 = switch (opts.activation) { - .gelu, .relu, .silu => 1, - .swiglu, .swiglu_step, .geglu, .geglu_tanh => 2, - }; - stdx.debug.assert(@rem(gate_up.dim(.out), activation_reduction) == 0, "{s} backend expected gate/up output width to be divisible by {}, got {}", .{ backend_name, activation_reduction, gate_up.dim(.out) }); - stdx.debug.assert(down_k == @divFloor(gate_up.dim(.out), activation_reduction), "{s} backend expected down input width to match activated width, got {} and {}", .{ backend_name, down_k, @divFloor(gate_up.dim(.out), activation_reduction) }); - stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "{s} backend expected routing ids and weights to match hidden token count, got {}, {} and {}", .{ backend_name, ids.dim(.token), routing_weights.dim(.token), hidden.dim(.token) }); - stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "{s} backend expected routing ids and weights to have matching top-k dimensions, got {} and {}", .{ backend_name, ids.dim(.topk), routing_weights.dim(.topk) }); - stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "{s} backend expected gate/up and down expert counts to match, got {} and {}", .{ backend_name, gate_up.dim(.expert), down.dim(.expert) }); - if (opts.expert_map) |expert_map| - stdx.debug.assert(expert_map.dtype() == .i32 and expert_map.rank() == 1, "{s} backend expected a rank-1 i32 expert map, got rank {} and dtype {}", .{ backend_name, expert_map.rank(), expert_map.dtype() }); - - const num_experts = if (opts.expert_map) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); - const routing = Impl.prepareRouting(ids, num_experts, @intCast(launch_config.block_size_m)); - - const expert_ids = if (opts.expert_map) |expert_map| - expert_map.gather(.{ .expert = routing.expert_ids }, .{}).withTags(.{.g}) - else - routing.expert_ids; - - const hidden_quant, const input_scale = Impl.prepareInput(hidden, gate_up_scheme, gate_up.dtype(), opts.quantize_input); - - const gate_up_out = Impl.call(.{ - .input = hidden_quant, - .weight = gate_up, - .bias = opts.gate_up.bias, - .input_scale = input_scale, - .weight_scale = opts.gate_up.quantizationScales(), - .routing = routing, - .expert_ids = expert_ids, - .launch_config = launch_config, - .quant_scheme = gate_up_scheme, - .top_k = @intCast(ids.dim(.topk)), - .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), - }); - - var activated = zml.moe.applyActivation(gate_up_out, opts.activation, opts.gate_up_layout); - if (opts.routing_weight_placement == .before_down) { - const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); - activated = activated.mul(weights.broad(activated.shape())); - } - const activated_quant, const down_input_scale = Impl.prepareInput(activated, down_scheme, down.dtype(), opts.quantize_input); - - const down_out = Impl.call(.{ - .input = activated_quant, - .weight = down, - .bias = opts.down.bias, - .input_scale = down_input_scale, - .weight_scale = opts.down.quantizationScales(), - .routing_weights = if (opts.routing_weight_placement == .after_down) routing_weights else null, - .routing = routing, - .expert_ids = expert_ids, - .launch_config = down_launch_config, - .quant_scheme = down_scheme, - .top_k = 1, - .output_shape = Shape.init(.{ .token = b * s, .topk = ids.dim(.topk), .out = down.dim(.out) }, .bf16), - }); - - const output = down_out.convert(.f32).sum(.topk).squeeze(.topk).convert(.bf16); - - return output.reshape(.{ .b = b, .token = s, .out = down.dim(.out) }); -} - -test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = zml.testing.env(); - const Local = struct { - fn forward(x: Tensor, layout: ProjectionLayout, activation: zml.moe.Activation) Tensor { - return zml.moe.applyActivation(x, activation, layout); - } - }; - const x: Tensor = .init(.{ .token = 1, .out = 6 }, .bf16); - const gates = [_]f32{ 0.75, -1.25, 3.5 }; - const ups = [_]f32{ 0.875, -2.25, 4.5 }; - for ([_]ProjectionLayout{ .concatenated, .interleaved }) |layout| { - var values: [6]zml.floats.BFloat16 = undefined; - for (gates, ups, 0..) |gate, up, i| { - values[if (layout == .concatenated) i else 2 * i] = .fromF32(gate); - values[if (layout == .concatenated) i + 3 else 2 * i + 1] = .fromF32(up); - } - var input = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&values)); - defer input.deinit(); - for ([_]?f32{ null, 2 }) |threshold| { - const activation: zml.moe.Activation = .{ .swiglu = .{ .limit = threshold } }; - var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, activation }, .{}); - defer exe.deinit(); - try zml.testing.expectEqualShapes(Shape.init(.{ .token = 1, .out = 3 }, .f32), exe.output_shapes[0]); - var output = try exe.eval(allocator, io, .{input}); - defer output.deinit(); - var actual = try output.toSliceAlloc(allocator, io); - defer actual.free(allocator); - for (gates, ups, actual.constItems(f32)) |gate, up, value| { - const g = if (threshold) |limit| @min(gate, limit) else gate; - const u = if (threshold) |limit| std.math.clamp(up, -limit, limit) else up; - try std.testing.expectApproxEqAbs(g / (1 + @exp(-g)) * u, value, 1e-6); - } - } - } -} - -/// Build the inputs tuple for FusedMoe and invoke it via `K.call(...)`. -pub const GemmOptions = struct { - input: Tensor, - weight: Tensor, - bias: ?Tensor = null, - input_scale: ?Tensor = null, - weight_scale: ?Tensor = null, - routing_weights: ?Tensor = null, - routing: Routing, - expert_ids: Tensor, - launch_config: LaunchConfig, - quant_scheme: ?zml.Quantization.Scheme = null, - top_k: usize, - output_shape: Shape, -}; - -test "FP8 routed GEMM with bias matches dequantized weights" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = zml.testing.env(); - if (platform.target != .cuda) return error.SkipZigTest; - - const Local = struct { - const Outputs = struct { actual: Tensor, expected: Tensor }; - - fn expandScales(scales: Tensor, shape: Shape) Tensor { - var expanded = scales.convert(.f32); - for (0..shape.rank()) |axis| { - const factor = @divExact(shape.dim(axis), expanded.dim(axis)); - const broad_shape = expanded.shape().insert(axis + 1, .{factor}); - const axes = Shape.range(broad_shape.rank(), .i64).remove(axis + 1); - expanded = expanded.broadcast(broad_shape, axes.dims()) - .reshape(expanded.shape().setDim(axis, shape.dim(axis))); - } - return expanded.withTags(shape.tags()); - } - - fn forward(x: Tensor, w: Tensor, scales: Tensor, ids: Tensor, weights: Tensor, bias: Tensor, scheme: zml.Quantization.Scheme, mx_native: bool, quantize_input: bool) Outputs { - const weight = w.convert(.f8e4m3fn); - var input = x; - var input_scale: ?Tensor = null; - var reference_input = x.convert(.f32); - if (quantize_input) { - input, input_scale = triton.prepareInput(x, scheme, .f8e4m3fn, true); - const scale = if (scheme == .mxfp8) input_scale.?.bitCast(.f8e8m0) else input_scale.?; - reference_input = input.convert(.f32).withTags(x.shape().tags()) - .mul(expandScales(scale.convert(.f32), x.shape())); - } - const weight_scale = if (scheme == .mxfp8) blk: { - const native = scales.convert(.f8e8m0); - break :blk if (mx_native) native else native.bitCast(.u8); - } else scales; - const config = launchConfigForTokens(8); - const routing = triton.prepareRouting(ids, w.dim(.expert), @intCast(config.block_size_m)); - const actual = callFusedMoe(.{ - .input = input, - .weight = weight, - .bias = bias, - .input_scale = input_scale, - .weight_scale = weight_scale, - .routing_weights = weights, - .routing = routing, - .expert_ids = routing.expert_ids, - .launch_config = config, - .quant_scheme = scheme, - .top_k = 1, - .output_shape = Shape.init(.{ .token = x.dim(.token), .topk = 1, .out = w.dim(.out) }, .bf16), - }).squeeze(.topk); - const expanded_weight = weight.convert(.f32).mul(expandScales(scales, weight.shape())); - const selected_weight = expanded_weight.gather(.{ .expert = ids.squeeze(.topk) }, .{}); - const dot = reference_input.dot(selected_weight, .in); - const selected_bias = if (bias.rank() == 2) - bias.gather(.{ .expert = ids.squeeze(.topk) }, .{}) - else - bias; - const unweighted = dot.add(selected_bias.convert(.f32).broad(dot.shape())); - const expected = unweighted.mul(weights.squeeze(.topk).convert(.f32).broad(unweighted.shape())).convert(.bf16); - return .{ .actual = actual, .expected = expected }; - } - }; - - for ([_]zml.Quantization.Scheme{ .mxfp8, .fp8_per_tensor, .fp8_per_channel, .fp8_block128 }) |scheme| { - for ([_]bool{ false, true }) |quantize_input| { - for ([_]i64{ 1, 17 }) |tokens| { - const n: i64 = if (scheme == .fp8_block128) 256 else 48; - const scale_dims: [3]i64 = switch (scheme) { - .mxfp8 => .{ 8, n, 8 }, - .fp8_per_tensor => if (tokens == 1) .{ 1, 1, 1 } else .{ 8, 1, 1 }, - .fp8_per_channel => .{ 8, n, 1 }, - .fp8_block128 => .{ 8, 2, 2 }, - else => unreachable, - }; - const x: Tensor = .init(.{ .token = tokens, .in = 256 }, .bf16); - const w: Tensor = .init(.{ .expert = 8, .out = n, .in = 256 }, .bf16); - const scales: Tensor = .init(.{ scale_dims[0], scale_dims[1], scale_dims[2] }, .f32); - const ids: Tensor = .init(.{ .token = tokens, .topk = 1 }, .i32); - const weights: Tensor = .init(.{ .token = tokens, .topk = 1 }, .bf16); - const bias: Tensor = if (tokens == 1) - .init(.{ .out = n }, .bf16) - else - .init(.{ .expert = 8, .out = n }, .f32); - var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, w, scales, ids, weights, bias, scheme, tokens == 1, quantize_input }, .{}); - defer exe.deinit(); - var buffers: [6]zml.Buffer = undefined; - var initialized: usize = 0; - defer for (buffers[0..initialized]) |*buffer| buffer.deinit(); - for ([_]Shape{ x.shape(), w.shape(), scales.shape(), ids.shape(), weights.shape(), bias.shape() }, 0..) |shape, i| { - const slice = try zml.Slice.alloc(allocator, shape); - defer slice.free(allocator); - for (0..shape.count()) |j| { - switch (i) { - 2 => slice.items(f32)[j] = @as(f32, @floatFromInt(@as(u32, 1) << @as(u5, @intCast(j % 4)))) / 16, - 3 => slice.items(i32)[j] = @intCast((j * 3 + 5) % 8), - 5 => { - const value = @as(f32, @floatFromInt(@as(i32, @intCast(j % 17)) - 8)) / 2; - if (shape.dtype() == .f32) { - slice.items(f32)[j] = value; - } else { - slice.items(zml.floats.BFloat16)[j] = .fromF32(value); - } - }, - else => slice.items(zml.floats.BFloat16)[j] = .fromF32(if (i == 4) - @as(f32, @floatFromInt(1 + j % 3)) / 4 - else - @as(f32, @floatFromInt(@as(i32, @intCast((j * 7 + j / 256) % 13)) - 6)) / 4), - } - } - buffers[i] = try zml.Buffer.fromSlice(io, platform, slice); - initialized += 1; - } - var output: zml.Bufferized(Local.Outputs) = undefined; - var runner = try exe.runner(allocator); - defer runner.deinit(allocator); - runner.run(io, .{ buffers[0], buffers[1], buffers[2], buffers[3], buffers[4], buffers[5] }, .{&output}, .{ .wait = true }); - defer zml.Buffer.deinitAll(Local.Outputs, &output); - zml.testing.expectClose(io, output.expected, output.actual, .{ .absolute_tolerance = 0.03125, .relative_tolerance = 0.01 }) catch |err| { - log.warn("FP8 routed GEMM failed for scheme={s}, quantize_input={}, tokens={}", .{ @tagName(scheme), quantize_input, tokens }); - return err; - }; - } - } - } -} - -test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = zml.testing.env(); - if (platform.target != .cuda) return error.SkipZigTest; - const Local = struct { - fn forward(x: Tensor, layout: ProjectionLayout, placement: RoutingWeightPlacement, storage_dtype: DataType) Tensor { - const fp4 = storage_dtype != .bf16; - const columns = Tensor.arange(.{ .end = 256 }, .i32).withTags(.{.dout}); - const is_gate = switch (layout) { - .concatenated => columns.cmp(.LT, Tensor.scalar(128, .i32)), - .interleaved => columns.remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), - }; - const dtype: DataType = if (fp4) .u8 else .bf16; - // E2M1 codes 1, 2, 3 encode 0.5, 1, 1.5; each byte holds two values. - const gate_value = if (fp4) Tensor.scalar(0x11, .u8) else Tensor.scalar(0.5, .bf16); - const up_value = if (fp4) Tensor.scalar(0x33, .u8) else Tensor.scalar(1.5, .bf16); - const gate_up_values = is_gate.select(gate_value, up_value); - var gate_up: zml.nn.Linear = .{ - .weight = gate_up_values.broad(Shape.init(.{ .expert = 8, .dout = 256, .d = @as(i64, if (fp4) 64 else 128) }, dtype)), - .tag = Shape.toTag(.d), - .quantization = if (fp4) .{ - .scheme = .mxfp4, - .scales = Tensor.scalar(127, .u8).broad(Shape.init(.{ .expert = 8, .dout = 256, .d = 4 }, .u8)), - } else null, - }; - var down: zml.nn.Linear = .{ - .weight = (if (fp4) Tensor.scalar(0x22, .u8) else Tensor.scalar(1, .bf16)).broad(Shape.init(.{ .expert = 8, .d = 128, .dout = @as(i64, if (fp4) 64 else 128) }, dtype)), - .bias = Tensor.scalar(2, .bf16).broad(Shape.init(.{ .expert = 8, .d = 128 }, .bf16)), - .tag = Shape.toTag(.dout), - .quantization = if (fp4) .{ - .scheme = .mxfp4, - .scales = Tensor.scalar(127, .u8).broad(Shape.init(.{ .expert = 8, .d = 128, .dout = 4 }, .u8)), - } else null, - }; - if (storage_dtype == .f4e2m1) { - gate_up.weight = gate_up.weight.bitCast(.f4e2m1).reshape(.{ .expert = 8, .dout = 256, .d = 128 }); - down.weight = down.weight.bitCast(.f4e2m1).reshape(.{ .expert = 8, .d = 128, .dout = 128 }); - gate_up.quantization.?.scales = gate_up.quantization.?.scales.bitCast(.f8e8m0); - down.quantization.?.scales = down.quantization.?.scales.bitCast(.f8e8m0); - } - const route = Tensor.arange(.{ .end = 2 }, .i32).reshape(.{ .b = 1, .s = 1, .topk = 2 }) - .broad(Shape.init(.{ .b = 1, .s = x.dim(.s), .topk = 2 }, .i32)); - return fusedExpertsImpl(.{ - .hidden_states = x, - .gate_up = gate_up, - .down = down, - .topk_ids = route.addConstant(2), - .topk_weights = route.convert(.f32).addConstant(1).scale(0.25), - .gate_up_layout = layout, - .routing_weight_placement = placement, - .quantize_input = false, - .activation = .{ .swiglu = .{} }, - }, .triton); - } - }; - for ([_]DataType{ .bf16, .u8, .f4e2m1 }) |storage_dtype| { - const tokens: i64 = if (storage_dtype == .u8) 17 else 1; - const x: Tensor = .init(.{ .b = 1, .s = tokens, .d = 128 }, .bf16); - const host = try zml.Slice.alloc(allocator, x.shape()); - defer host.free(allocator); - @memset(host.items(zml.floats.BFloat16), .fromF32(1.0 / 128.0)); - var input = try zml.Buffer.fromSlice(io, platform, host); - defer input.deinit(); - for ([_]ProjectionLayout{ .concatenated, .interleaved }) |layout| { - for ([_]RoutingWeightPlacement{ .before_down, .after_down }) |placement| { - var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, placement, storage_dtype }, .{}); - defer exe.deinit(); - var output = try exe.eval(allocator, io, .{input}); - defer output.deinit(); - var actual = try output.toSliceAlloc(allocator, io); - defer actual.free(allocator); - const activation: f32 = 0.5 / (1 + @exp(@as(f32, -0.5))) * 1.5; - // The two routes have weights 0.25 and 0.5. Bias distinguishes placement. - const bias: f32 = if (placement == .before_down) 4 else 1.5; - const expected = 128 * activation * 0.75 + bias; - for (actual.constItems(zml.floats.BFloat16)) |value| { - try std.testing.expectApproxEqAbs(expected, value.toF32(), 0.5); - } - } - } - } -} - -pub const Routing = struct { - sorted_token_ids: Tensor, - expert_ids: Tensor, - num_tokens_post_padded: Tensor, - max_num_tokens_padded: i64, - num_assignments: i64, - naive_block_assignment: bool, -}; - -// ============================================================================= -// Config / validation helpers -// ============================================================================= - -pub const LaunchConfig = struct { - block_size_m: usize, - block_size_n: usize, - block_size_k: usize, - group_size_m: usize, - num_warps: i32, - num_stages: i32, -}; - -const launch_configs = [_]struct { tokens: i64, config: LaunchConfig }{ - .{ .tokens = 1, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 4 } }, - .{ .tokens = 2, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 4 } }, - .{ .tokens = 4, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, - .{ .tokens = 8, .config = .{ .block_size_m = 16, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, - .{ .tokens = 16, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 64, .group_size_m = 64, .num_warps = 4, .num_stages = 5 } }, - .{ .tokens = 24, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, - .{ .tokens = 32, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 128, .group_size_m = 1, .num_warps = 4, .num_stages = 2 } }, - .{ .tokens = 48, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 128, .group_size_m = 64, .num_warps = 4, .num_stages = 2 } }, - .{ .tokens = 64, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 128, .group_size_m = 1, .num_warps = 4, .num_stages = 2 } }, - .{ .tokens = 96, .config = .{ .block_size_m = 16, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, - .{ .tokens = 128, .config = .{ .block_size_m = 16, .block_size_n = 256, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, - .{ .tokens = 256, .config = .{ .block_size_m = 16, .block_size_n = 256, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, - .{ .tokens = 512, .config = .{ .block_size_m = 32, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, - .{ .tokens = 1024, .config = .{ .block_size_m = 64, .block_size_n = 128, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, - .{ .tokens = 1536, .config = .{ .block_size_m = 64, .block_size_n = 128, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, - .{ .tokens = 2048, .config = .{ .block_size_m = 128, .block_size_n = 128, .block_size_k = 64, .group_size_m = 16, .num_warps = 8, .num_stages = 3 } }, - .{ .tokens = 3072, .config = .{ .block_size_m = 128, .block_size_n = 256, .block_size_k = 64, .group_size_m = 1, .num_warps = 8, .num_stages = 4 } }, - .{ .tokens = 4096, .config = .{ .block_size_m = 128, .block_size_n = 256, .block_size_k = 64, .group_size_m = 16, .num_warps = 8, .num_stages = 4 } }, -}; - -fn launchConfigForTokens(num_tokens: i64) LaunchConfig { - var best = launch_configs[0]; - for (launch_configs[1..]) |candidate| { - // The table is ascending, so ties retain the smaller token bucket. - if (@abs(candidate.tokens - num_tokens) < @abs(best.tokens - num_tokens)) { - best = candidate; - } - } - return best.config; -} - -test "MoE launch config selects the nearest token bucket" { - try std.testing.expectEqualDeep(launchConfigForTokens(4), launchConfigForTokens(6)); - try std.testing.expectEqualDeep(launchConfigForTokens(8), launchConfigForTokens(7)); - try std.testing.expectEqualDeep(launchConfigForTokens(1), launchConfigForTokens(0)); - try std.testing.expectEqualDeep(launchConfigForTokens(4096), launchConfigForTokens(8192)); - try std.testing.expectEqual(@as(usize, 64), launchConfigForTokens(16).group_size_m); - try std.testing.expectEqual(@as(i32, 5), launchConfigForTokens(16).num_stages); -} diff --git a/zml/moe/triton.zig b/zml/moe/triton.zig deleted file mode 100644 index cba8e113..00000000 --- a/zml/moe/triton.zig +++ /dev/null @@ -1,326 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -const zml = @import("../zml.zig"); -const Tensor = zml.Tensor; -const Shape = zml.Shape; -const DataType = zml.DataType; -const toDType = zml.kernel.triton.from; -const shared = @import("fused_experts.zig"); -pub const Parameters = shared.Parameters; -pub const ProjectionLayout = shared.ProjectionLayout; -pub const RoutingWeightPlacement = shared.RoutingWeightPlacement; -pub const FusedExpertsArgs = shared.FusedExpertsArgs; -const kernels = @import("triton_kernels/triton_kernels.zig"); - -const log = std.log.scoped(.@"zml/moe/triton"); - -fn numStages(config: shared.LaunchConfig) i32 { - const platform = zml.Compiler.current().platform; - if (platform.target != .rocm) return config.num_stages; - - // ROCm buffers num_stages - 1 BF16 operand pairs in LDS. A single stage - // reuses scratch space between operand layout conversions. - const bytes_per_stage: i64 = @intCast(2 * config.block_size_k * (config.block_size_m + config.block_size_n)); - var stages = config.num_stages; - for (platform.devices) |device| { - const attribute = device.pjrt_desc.attribute(platform.pjrt_api, "shared_memory_per_block_optin"); - const limit = if (attribute) |value| value.int64 else 64 * 1024; - stages = @intCast(@min(stages, @divTrunc(limit, bytes_per_stage) + 1)); - } - return stages; -} - -pub fn call(opts: shared.GemmOptions) Tensor { - // Native FP4 tensors expose logical K; the Triton operand is byte-packed. - const weight = if (opts.quant_scheme == .mxfp4 and opts.weight.dtype() == .f4e2m1) - opts.weight.reshape(opts.weight.shape().setDim(2, @divExact(opts.weight.dim(2), 2)).append(.{ .nibble = 2 })).bitCast(.u8) - else - opts.weight; - const weight_k = weight.dim(2) * @as(i64, if (opts.quant_scheme == .mxfp4) 2 else 1); - stdx.debug.assert(opts.input.dim(1) == weight_k, "input width {} must match logical weight K {}", .{ opts.input.dim(1), weight_k }); - if (opts.quant_scheme == .mxfp4) { - stdx.debug.assert(@mod(weight_k, 32) == 0 and opts.weight_scale != null, "MXFP4 requires K divisible by 32 and weight scales", .{}); - stdx.debug.assert(opts.input.dtype() == .bf16 and opts.input_scale == null, "MXFP4 requires BF16 activations without input scales", .{}); - } - - if (opts.quant_scheme == .fp8_block32) { - const scales = opts.weight_scale orelse stdx.debug.panic("block32 FP8 requires weight scales", .{}); - stdx.debug.assert(weight.dtype() == .f8e4m3fn and @mod(weight.dim(1), 32) == 0 and @mod(weight_k, 32) == 0, "block32 FP8 requires E4M3FN weights with N and K divisible by 32", .{}); - stdx.debug.assert(scales.rank() == 3 and scales.dim(0) == weight.dim(0) and scales.dim(1) == @divExact(weight.dim(1), 32) and scales.dim(2) == @divExact(weight_k, 32), "block32 FP8 requires scales shaped [expert, N/32, K/32], got {f}", .{scales.shape()}); - stdx.debug.assert(scales.dtype() == .f8e8m0 or scales.dtype() == .bf16 or scales.dtype() == .f32, "unsupported block32 FP8 scale dtype {}", .{scales.dtype()}); - } - - stdx.debug.assert(opts.quant_scheme != null or (opts.input_scale == null and opts.weight_scale == null), "scales require a quantization scheme", .{}); - for ([_]i64{ weight.dim(1), weight.dim(2), opts.input.dim(1), weight.dim(1) * weight.dim(2), opts.output_shape.dim(-1) }) |dim_or_stride| { - stdx.debug.assert(dim_or_stride > 0 and @mod(dim_or_stride, 16) == 0, "FusedMoe dimensions and matrix strides must be positive multiples of 16, got {}", .{dim_or_stride}); - } - const block_size_m: i64 = @intCast(opts.launch_config.block_size_m); - const block_size_n: i64 = @intCast(opts.launch_config.block_size_n); - const m_tokens = opts.input.dim(0); - const em_effective = if (m_tokens < block_size_m) - @min(opts.routing.max_num_tokens_padded, opts.routing.num_assignments * block_size_m) - else - opts.routing.max_num_tokens_padded; - stdx.debug.assert(@mod(em_effective, block_size_m) == 0, "routing capacity {} must be a multiple of block size {}", .{ em_effective, block_size_m }); - const grid_x = - @divExact(em_effective, block_size_m) * - (std.math.divCeil(i64, weight.dim(1), block_size_n) catch unreachable); - - const weight_scale: ?Tensor = if (opts.weight_scale) |scale| blk: { - const scheme = opts.quant_scheme orelse break :blk scale; - break :blk switch (scheme) { - // dot_scaled takes E8M0 encodings as bytes, including native E8M0 tensors. - .mxfp4, .mxfp8 => scale.bitCast(.u8), - .fp8_per_channel => scale.reshape(.{ weight.dim(0), weight.dim(1), 1 }), - .fp8_per_tensor => if (scale.count() == 1) scale else scale.reshape(.{ weight.dim(0), 1, 1 }), - // Tiled FP8 uses ordinary dot products with FP32 scale arithmetic. - .fp8_block32 => scale.convert(.f32), - .fp8_block128, .nvfp4 => scale, - }; - } else null; - - const stride_asm: i64 = if (opts.input_scale) |scale| (if (scale.rank() == 2) scale.dim(1) else 0) else 0; - const stride_ask: i64 = if (opts.input_scale) |scale| (if (scale.rank() == 2) 1 else 0) else 0; - const has_weight_scale_expert_stride = if (weight_scale) |scale| - !(opts.quant_scheme == .fp8_per_tensor and scale.count() == 1) - else - false; - const stride_bse: i64 = if (weight_scale) |scale| (if (scale.rank() == 3 and has_weight_scale_expert_stride) scale.dim(1) * scale.dim(2) else 0) else 0; - const stride_bsk: i64 = if (weight_scale) |scale| (if (scale.rank() == 3) 1 else 0) else 0; - const stride_bsn: i64 = if (weight_scale) |scale| (if (scale.rank() == 3) scale.dim(2) else 0) else 0; - - return kernels.FusedMoe.Kernel.call( - .{ - .a_ptr = opts.input, - .b_ptr = weight, - .b_bias_ptr = opts.bias orelse Tensor.scalar(0, .f32), - .a_scale_ptr = opts.input_scale orelse Tensor.scalar(1.0, .f32), - .b_scale_ptr = weight_scale orelse Tensor.scalar(1.0, .f32), - .routing_weights_ptr = opts.routing_weights orelse Tensor.scalar(1.0, .f32), - .sorted_token_ids_ptr = opts.routing.sorted_token_ids, - .expert_ids_ptr = opts.expert_ids, - .num_tokens_post_padded_ptr = opts.routing.num_tokens_post_padded, - .N_ptr = Tensor.constant(.{ .i64 = weight.dim(1) }).reshape(.{1}), - .K_ptr = Tensor.constant(.{ .i64 = weight_k }).reshape(.{1}), - .EM_ptr = Tensor.constant(.{ .i64 = em_effective }).reshape(.{1}), - .num_valid_tokens_ptr = Tensor.constant(.{ .i64 = opts.routing.num_assignments }).reshape(.{1}), - .stride_am_ptr = Tensor.constant(.{ .i64 = opts.input.dim(1) }).reshape(.{1}), - .stride_be_ptr = Tensor.constant(.{ .i64 = weight.dim(1) * weight.dim(2) }).reshape(.{1}), - .stride_bn_ptr = Tensor.constant(.{ .i64 = weight.dim(2) }).reshape(.{1}), - .stride_cm_ptr = Tensor.constant(.{ .i64 = weight.dim(.out) }).reshape(.{1}), - .stride_asm_ptr = Tensor.constant(.{ .i64 = stride_asm }).reshape(.{1}), - .stride_ask_ptr = Tensor.constant(.{ .i64 = stride_ask }).reshape(.{1}), - .stride_bse_ptr = Tensor.constant(.{ .i64 = stride_bse }).reshape(.{1}), - .stride_bsk_ptr = Tensor.constant(.{ .i64 = stride_bsk }).reshape(.{1}), - .stride_bsn_ptr = Tensor.constant(.{ .i64 = stride_bsn }).reshape(.{1}), - .stride_bbe_ptr = Tensor.constant(.{ .i64 = if (opts.bias) |bias| (if (bias.rank() == 2 and bias.dim(0) > 1) bias.dim(1) else 0) else 0 }).reshape(.{1}), - .stride_bbn_ptr = Tensor.constant(.{ .i64 = if (opts.bias) |bias| (if (bias.rank() > 0 and bias.dim(-1) > 1) 1 else 0) else 0 }).reshape(.{1}), - }, - .{ .c = opts.output_shape }, - .{ - .cfg = .{ - .a_dtype = toDType(opts.input.dtype()), - .b_dtype = toDType(weight.dtype()), - .c_dtype = toDType(opts.output_shape.dtype()), - .a_scale_dtype = if (opts.input_scale) |scale| toDType(scale.dtype()) else null, - .b_scale_dtype = if (weight_scale) |scale| toDType(scale.dtype()) else null, - .b_bias_dtype = if (opts.bias) |bias| toDType(bias.dtype()) else null, - .routing_weights_dtype = if (opts.routing_weights) |weights| toDType(weights.dtype()) else null, - .block_size_m = opts.launch_config.block_size_m, - .block_size_n = opts.launch_config.block_size_n, - .block_size_k = opts.launch_config.block_size_k, - .group_size_m = opts.launch_config.group_size_m, - .top_k = opts.top_k, - .naive_block_assignment = opts.routing.naive_block_assignment, - .compute_type = .bf16, - .quant_scheme = opts.quant_scheme, - }, - .grid = .{ @intCast(grid_x), 1, 1 }, - .num_warps = opts.launch_config.num_warps, - .num_stages = numStages(opts.launch_config), - }, - ).c; -} - -pub fn prepareRouting(topk_ids: Tensor, num_experts: i64, block_size_m: i64) shared.Routing { - stdx.debug.assert(block_size_m > 0 and @mod(block_size_m, 16) == 0, "routing block size must be a positive multiple of 16, got {}", .{block_size_m}); - const ids = topk_ids.withTags(.{ .token, .topk }).convert(.i32); - const num_assignments = ids.dim(.token) * ids.dim(.topk); - const sparsity_factor: i64 = 4; - const naive_block_assignment = num_assignments * sparsity_factor <= num_experts; - const max_num_tokens_padded = if (naive_block_assignment) - num_assignments * block_size_m - else if (num_assignments < num_experts) - num_assignments * block_size_m - else - (std.math.divCeil(i64, num_assignments + num_experts * (block_size_m - 1), block_size_m) catch unreachable) * block_size_m; - - const sorted_token_ids, const expert_ids, const num_tokens_post_padded = if (naive_block_assignment) blk: { - log.debug("Using naive block assignment for MoE kernels. Num assignments: {d}, Num experts: {d}", .{ num_assignments, num_experts }); - break :blk .{ - Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)), - ids.reshape(.{ .g = num_assignments }), - Tensor.constant(.{ .i32 = @as(i32, @intCast(max_num_tokens_padded)) }).reshape(.{1}), - }; - } else alignBlockSize(ids, num_experts, block_size_m); - - return .{ - .sorted_token_ids = sorted_token_ids, - .expert_ids = expert_ids, - .num_tokens_post_padded = num_tokens_post_padded, - .max_num_tokens_padded = max_num_tokens_padded, - .num_assignments = num_assignments, - .naive_block_assignment = naive_block_assignment, - }; -} - -fn alignBlockSize(topk_ids: Tensor, num_experts: i64, block_size_m: i64) struct { Tensor, Tensor, Tensor } { - const topk_ids_ = topk_ids.withTags(.{ .token, .topk }).convert(.i32); - const num_tokens = topk_ids_.dim(.token); - const topk = topk_ids_.dim(.topk); - const num_assignments = num_tokens * topk; - const max_num_tokens_padded = if (num_assignments < num_experts) - num_assignments * block_size_m - else - (std.math.divCeil(i64, num_assignments + num_experts * (block_size_m - 1), block_size_m) catch unreachable) * block_size_m; - const max_num_m_blocks = @divExact(max_num_tokens_padded, block_size_m); - const warp_size: i64 = 32; - const padded_num_experts: i64 = @intCast(std.math.ceilPowerOfTwoAssert(u64, @intCast(@max(num_experts, warp_size)))); - const sort_block_size: i64 = 256; - const sort_grid_x: i64 = @min(std.math.divCeil(i64, num_assignments, sort_block_size) catch unreachable, 65535); - - const flat_experts = topk_ids_.reshape(.{ .g = num_assignments }); - var cumsums = Tensor.zeroes(Shape.init(.{ .g = num_experts + 1 }, .i32)); - var expert_ids = Tensor.zeroes(Shape.init(.{ .g = max_num_m_blocks }, .i32)); - var sorted_token_ids = Tensor.zeroes(Shape.init(.{ .g = max_num_tokens_padded }, .i32)); - var num_tokens_post_padded = Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)); - - { - const align_outs = kernels.MoeAlignBlockSize.Kernel.call( - .{ - .topk_ids_ptr = flat_experts, - .sorted_token_ids_ptr = sorted_token_ids, - .expert_ids_ptr = expert_ids, - .num_tokens_post_pad_ptr = num_tokens_post_padded, - .cumsum_ptr = cumsums, - }, - .{ - .sorted_token_ids = sorted_token_ids.shape(), - .expert_ids = expert_ids.shape(), - .num_tokens_post_pad = num_tokens_post_padded.shape(), - .cumsum = cumsums.shape(), - }, - .{ - .cfg = .{ - .numel = @intCast(num_assignments), - .num_experts = @intCast(num_experts), - .padded_num_experts = @intCast(padded_num_experts), - .max_num_tokens_padded = @intCast(max_num_tokens_padded), - .max_num_m_blocks = @intCast(max_num_m_blocks), - .block_size_m = @intCast(block_size_m), - .hist_block = 256, - }, - .grid = .{ 2, 1, 1 }, - .num_stages = 1, - .num_warps = 8, - .output_operand_aliases = .{ - .sorted_token_ids = .sorted_token_ids_ptr, - .expert_ids = .expert_ids_ptr, - .num_tokens_post_pad = .num_tokens_post_pad_ptr, - .cumsum = .cumsum_ptr, - }, - }, - ); - sorted_token_ids = align_outs.sorted_token_ids; - expert_ids = align_outs.expert_ids; - num_tokens_post_padded = align_outs.num_tokens_post_pad; - cumsums = align_outs.cumsum; - } - - { - const sort_outs = kernels.CountAndSortExpertTokens.Kernel.call( - .{ - .topk_ids_ptr = flat_experts, - .sorted_token_ids_ptr = sorted_token_ids, - .cumsum_ptr = cumsums, - }, - .{ - .sorted_token_ids = sorted_token_ids.shape(), - .cumsum = cumsums.shape(), - }, - .{ - .cfg = .{ - .numel = @intCast(num_assignments), - .num_experts = @intCast(num_experts), - .sort_block_size = @intCast(sort_block_size), - }, - .grid = .{ @intCast(sort_grid_x), 1, 1 }, - .num_stages = 1, - .num_warps = 4, - .output_operand_aliases = .{ - .sorted_token_ids = .sorted_token_ids_ptr, - .cumsum = .cumsum_ptr, - }, - }, - ); - sorted_token_ids = sort_outs.sorted_token_ids; - cumsums = sort_outs.cumsum; - } - - return .{ sorted_token_ids, expert_ids, num_tokens_post_padded }; -} - -pub fn prepareInput(x: Tensor, scheme: ?zml.Quantization.Scheme, output_dtype: zml.DataType, quantize: bool) struct { Tensor, ?Tensor } { - if (quantize) { - if (scheme) |scheme_| { - if (scheme_ != .mxfp4) { - const input, const scale = quantizeFp8Input(x, scheme_, output_dtype); - return .{ input, scale }; - } - } - } - return .{ x.convert(.bf16), null }; -} - -fn quantizeFp8Input(x: Tensor, scheme: zml.Quantization.Scheme, output_dtype: DataType) struct { Tensor, Tensor } { - const group_size: i64, const scale_dtype: DataType, const absmax_epsilon: f32 = switch (scheme) { - .mxfp8 => .{ 32, .u8, 1e-10 }, - .fp8_per_channel, .fp8_per_tensor => .{ x.dim(1), .f32, 1e-10 }, - .fp8_block128 => .{ 128, .f32, 1e-10 }, - .fp8_block32 => .{ 32, .f32, 1e-4 }, - .mxfp4, .nvfp4 => unreachable, - }; - stdx.debug.assert(x.rank() == 2, "expected a rank-2 activation matrix, got {f}", .{x.shape()}); - stdx.debug.assert(@mod(x.dim(1), group_size) == 0, "activation width must be divisible by group size {d}, got {d}", .{ group_size, x.dim(1) }); - - const groups_per_row = @divExact(x.dim(1), group_size); - const quantized = Tensor.zeroes(Shape.init(.{ .token = x.dim(0), .feature = x.dim(1) }, output_dtype)); - const scales = Tensor.zeroes(Shape.init(.{ .token = x.dim(0), .group = groups_per_row }, scale_dtype)); - - const outs = kernels.PerTokenGroupQuantFp8.Kernel.call( - .{ - .y_ptr = x, - .group_size_ptr = Tensor.constant(.{ .i64 = group_size }).reshape(.{1}), - .y_num_columns_ptr = Tensor.constant(.{ .i64 = x.dim(1) }).reshape(.{1}), - .y_row_stride_ptr = Tensor.constant(.{ .i64 = x.dim(1) }).reshape(.{1}), - .eps_ptr = Tensor.scalar(absmax_epsilon, .f32), - }, - .{ .y_q = quantized.shape(), .y_s = scales.shape() }, - .{ - .cfg = .{ - .input_dtype = toDType(x.dtype()), - .output_dtype = toDType(output_dtype), - .scale_dtype = toDType(scale_dtype), - .block = std.math.ceilPowerOfTwoAssert(usize, @intCast(group_size)), - .quant_scheme = scheme, - }, - .grid = .{ @intCast(x.dim(0) * groups_per_row), 1, 1 }, - .num_stages = 1, - .num_warps = 1, - }, - ); - - return .{ outs.y_q, outs.y_s }; -} diff --git a/zml/moe/triton_kernels/mxfp4.zig b/zml/moe/triton_kernels/mxfp4.zig deleted file mode 100644 index 74170966..00000000 --- a/zml/moe/triton_kernels/mxfp4.zig +++ /dev/null @@ -1,465 +0,0 @@ -//! FP4 x FP8 MoE emitted through the ZML Triton DSL. -//! BF16 activations use per-32 FP8 scaling; weights retain per-32 E8M0 scales. -//! Gate/up weight and scale rows are interleaved gate-first. -const std = @import("std"); -const zml = @import("../../zml.zig"); -const tri = zml.kernel.triton; -const B = tri.Builder; -const V = tri.Value; - -pub const Config = struct { - tokens: i64, - hidden: i64, - intermediate: i64, - experts: i64, - global_experts: i64, - topk: i64, - expert_offset: i64 = 0, - // Swiglu clamping. Set to 0 to disable. - swiglu_limit: f32, - /// Scale SwiGLU before FP8 quantization, or scale the down result before reduction. - routing_weight_placement: zml.moe.triton.RoutingWeightPlacement = .before_down, - - pub fn validate(c: Config) !void { - if (c.tokens <= 0 or - c.experts <= 0 or - c.topk <= 0 or c.topk > c.global_experts or - c.expert_offset < 0 or c.expert_offset + c.experts > c.global_experts) - { - return error.InvalidRoutingShape; - } - - if (c.hidden < 128 or @mod(c.hidden, 128) != 0 or - c.intermediate < 128 or @mod(c.intermediate, 128) != 0) - { - return error.InvalidHiddenShape; - } - - if (!std.math.isFinite(c.swiglu_limit) or c.swiglu_limit < 0) { - return error.InvalidScale; - } - } -}; - -pub const Inputs = struct { - x: zml.Tensor, - ids: zml.Tensor, - scales: zml.Tensor, - w1: zml.Tensor, - s1: zml.Tensor, - w2: zml.Tensor, - s2: zml.Tensor, -}; - -pub fn validateInputs(c: Config, a: Inputs) !void { - if (a.s1.rank() != 3 or a.s2.rank() != 3) return error.InvalidInputShape; - if (a.s1.dim(2) < @divTrunc(c.hidden, 32) or - a.s2.dim(2) < @divTrunc(c.intermediate, 32)) return error.InvalidInputShape; - const shapes = .{ - .x = zml.Shape.init(.{ c.tokens, c.hidden }, .bf16), - .ids = zml.Shape.init(.{ c.tokens, c.topk }, .i32), - .scales = zml.Shape.init(.{ c.tokens, c.topk }, .f32), - .w1 = zml.Shape.init(.{ c.experts, 2 * c.intermediate, @divTrunc(c.hidden, 2) }, .u8), - .s1 = zml.Shape.init(.{ c.experts, 2 * c.intermediate, a.s1.dim(2) }, .f8e8m0), - .w2 = zml.Shape.init(.{ c.experts, c.hidden, @divTrunc(c.intermediate, 2) }, .u8), - .s2 = zml.Shape.init(.{ c.experts, c.hidden, a.s2.dim(2) }, .f8e8m0), - }; - inline for (comptime std.meta.fieldNames(Inputs)) |field| { - if (!@field(a, field).shape().eql(@field(shapes, field))) return error.InvalidInputShape; - } -} - -const bn = 16; -const bk = 512; - -/// Computes an upper bound on the number of expert tiles needed. -/// When there is only one token, reserve a single tile per selected expert. -fn tiles(c: Config) i64 { - if (c.tokens == 1) return c.topk; - return @min(c.tokens * c.topk, @divTrunc(c.tokens * c.topk + c.experts * (bn - 1) + bn - 1, bn)); -} - -/// Returns the next power of two. -fn pow2(x: i64) i64 { - return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(x)) catch unreachable); -} - -fn quantRows(c: Config) i64 { - return if (c.tokens >= 16) 16 else 1; -} - -fn combineBlock(c: Config) i64 { - // Keep at least 512 CTAs before increasing per-CTA reduction work. - const elements = c.tokens * c.hidden; - if (c.tokens > 1 and elements >= 512 * 2048) return 2048; - if (c.tokens > 1 and elements >= 512 * 1024) return 1024; - return 256; -} - -fn ar(b: *B, n: i64) V { - return b.arange(0, n, .i32); -} - -fn ci(b: *B, n: i64) V { - return b.liftAs(n, .i32); -} - -fn ld(b: *B, p: V, mask: V, dt: tri.DType, other: i64) V { - return b.loadOpts(p, .{ .mask = mask, .other = if (p.isTensor()) b.full(p.shape().constSlice(), other, dt) else b.liftAs(other, dt) }); -} - -// Scale rows can be tightly packed or padded. Specialize the GEMM on their -// physical width without imposing an alignment requirement on the tensor. -const GemmConfig = struct { - moe: Config, - scale_stride: i64, -}; - -const Quant = tri.Kernel(Config, .{ .name = "mxfp4_triton_quant", .inputs = &.{"x"}, .outputs = &.{ "q", "s" }, .run = quantInput }); -const Route = tri.Kernel(Config, .{ .name = "mxfp4_triton_route", .inputs = &.{"ids"}, .outputs = &.{ "counts", "pos" }, .run = route }); -const RouteAtomic = tri.Kernel(Config, .{ .name = "mxfp4_triton_route_atomic", .inputs = &.{ "ids", "zeros" }, .outputs = &.{ "counts", "pos" }, .run = routeAtomic }); -// The same device code serves both backends; only the group geometry differs. -const ScheduleConfig = struct { - moe: Config, - /// Expert groups the schedule may fill. - capacity: i64, - /// Routed rows per expert group, i.e. the consuming GEMM's tile N. - group: i64, -}; -// Two symbols for one kernel: both backends can run in the same process, and -// distinct names keep them apart in profiles. -const Schedule = tri.Kernel(ScheduleConfig, .{ .name = "mxfp4_triton_schedule", .inputs = &.{ "ids", "counts", "pos" }, .outputs = &.{ "map", "perm", "sched" }, .run = scheduleGroups }); -const GateUp = tri.Kernel(GemmConfig, .{ .name = "mxfp4_triton_up", .inputs = &.{ "q", "s", "w", "ws", "ids", "rw", "map", "sched" }, .outputs = &.{ "mid", "ms" }, .run = gateUp }); -const Down = tri.Kernel(GemmConfig, .{ .name = "mxfp4_triton_down", .inputs = &.{ "mid", "ms", "w", "ws", "ids", "sched" }, .outputs = &.{"d"}, .run = down }); -const Combine = tri.Kernel(Config, .{ .name = "mxfp4_triton_combine", .inputs = &.{ "d", "ids", "perm", "rw" }, .outputs = &.{"y"}, .run = combine }); - -/// MXFP4 routing for the CuTe backend (`zml/moe/cute_mxfp4.zig`), whose -/// persistent GEMMs consume it. It reuses this backend's `route` and -/// `schedule` kernels with `group`-row expert groups (the CuTe GEMM tile N) -/// and a compact `capacity`, instead of the fixed `bn` tiles used here. -/// `route_map` maps routes to routed rows, `route_inverse` rows back to -/// routes, and `schedule` is `[group experts | group sizes | active group -/// count]`. -pub fn scheduleForCute(c: Config, ids: zml.Tensor, capacity: i64, group: i64) struct { - route_inverse: zml.Tensor, - route_map: zml.Tensor, - schedule: zml.Tensor, -} { - c.validate() catch @panic("Invalid MXFP4 MoE configuration"); - std.debug.assert(capacity > 0); - const routes = c.tokens * c.topk; - // `route` rescans every route once per expert: cheapest for decode, but - // quadratic. Past a few thousand tokens, count with atomics instead. - const r: struct { counts: zml.Tensor, pos: zml.Tensor } = if (routes >= atomic_route_threshold) blk: { - const counted = RouteAtomic.call( - .{ .ids = ids, .zeros = zml.Tensor.zeroes(.init(.{c.experts}, .i32)) }, - .{ .counts = .init(.{c.experts}, .i32), .pos = .init(.{routes}, .i32) }, - .{ - .cfg = c, - .grid = .{ @intCast(@divTrunc(routes + route_block - 1, route_block)), 1, 1 }, - .num_warps = 4, - .num_stages = 1, - .output_operand_aliases = .{ .counts = .zeros }, - }, - ); - break :blk .{ .counts = counted.counts, .pos = counted.pos }; - } else blk: { - const scanned = Route.call( - .{ .ids = ids }, - .{ .counts = .init(.{c.experts}, .i32), .pos = .init(.{routes}, .i32) }, - .{ .cfg = c, .grid = .{ @intCast(c.experts), 1, 1 }, .num_warps = 4, .num_stages = 1 }, - ); - break :blk .{ .counts = scanned.counts, .pos = scanned.pos }; - }; - const s = Schedule.call( - .{ .ids = ids, .counts = r.counts, .pos = r.pos }, - .{ - .map = .init(.{capacity * group}, .i32), - .perm = .init(.{c.tokens * c.topk}, .i32), - .sched = .init(.{2 * capacity + 1}, .i32), - }, - .{ - .cfg = .{ .moe = c, .capacity = capacity, .group = group }, - .grid = .{ @intCast(@divTrunc(c.tokens * c.topk + 127, 128)), 1, 1 }, - .num_warps = 4, - .num_stages = 1, - }, - ); - return .{ .route_inverse = s.map, .route_map = s.perm, .schedule = s.sched }; -} - -pub fn forward(c: Config, a: Inputs) zml.Tensor { - c.validate() catch @panic("Invalid MXFP4 MoE configuration"); - validateInputs(c, a) catch @panic("Invalid MXFP4 tensor layout"); - - const tile_count = tiles(c); - - // Quantize activations - const q = Quant.call( - .{ .x = a.x }, - .{ - .q = .init(.{ c.tokens, c.hidden }, .f8e4m3fn), - .s = .init(.{ c.tokens, @divTrunc(c.hidden, 32) }, .u8), - }, - .{ - .cfg = c, - .grid = .{ @intCast(@divTrunc(c.tokens + quantRows(c) - 1, quantRows(c))), @intCast(@divTrunc(c.hidden, 128)), 1 }, - .num_warps = 4, - .num_stages = 1, - }, - ); - - const dummy = zml.Tensor.scalar(0, .i32); - var map = dummy; - var perm = dummy; - var sched = dummy; - - if (c.tokens != 1) { - const r = Route.call( - .{ .ids = a.ids }, - .{ .counts = .init(.{c.experts}, .i32), .pos = .init(.{c.tokens * c.topk}, .i32) }, - .{ .cfg = c, .grid = .{ @intCast(c.experts), 1, 1 }, .num_warps = 4, .num_stages = 1 }, - ); - - const s = Schedule.call( - .{ .ids = a.ids, .counts = r.counts, .pos = r.pos }, - .{ .map = .init(.{tile_count * bn}, .i32), .perm = .init(.{c.tokens * c.topk}, .i32), .sched = .init(.{2 * tile_count + 1}, .i32) }, - .{ - .cfg = .{ .moe = c, .capacity = tile_count, .group = bn }, - .grid = .{ @intCast(@divTrunc(c.tokens * c.topk + 127, 128)), 1, 1 }, - .num_warps = 4, - .num_stages = 1, - }, - ); - - map = s.map; - perm = s.perm; - sched = s.sched; - } - - // Each narrow tile produces 64 activations: two complete scale groups. - const up_columns: i64 = if (c.tokens <= 8) 64 else 128; - const m = GateUp.call( - .{ .q = q.q, .s = q.s, .w = a.w1, .ws = a.s1.bitCast(.u8), .ids = a.ids, .rw = a.scales, .map = map, .sched = sched }, - .{ .mid = .init(.{ tile_count * bn, c.intermediate }, .f8e4m3fn), .ms = .init(.{ tile_count * bn, @divTrunc(c.intermediate, 32) }, .u8) }, - .{ - .cfg = .{ .moe = c, .scale_stride = a.s1.dim(2) }, - .grid = .{ @intCast(@divTrunc(c.intermediate, up_columns)), @intCast(tile_count), 1 }, - .num_warps = if (c.tokens <= 8) 4 else 8, - .num_stages = 2, - .global_scratch_memory_size = @intCast(128 * @divTrunc(c.intermediate, up_columns) * tile_count), - }, - ); - - const d = Down.call( - .{ .mid = m.mid, .ms = m.ms, .w = a.w2, .ws = a.s2.bitCast(.u8), .ids = a.ids, .sched = sched }, - .{ .d = .init(.{ tile_count * bn, c.hidden }, .bf16) }, - .{ - .cfg = .{ .moe = c, .scale_stride = a.s2.dim(2) }, - .grid = .{ @intCast(@divTrunc(c.hidden, 128)), @intCast(tile_count), 1 }, - .num_warps = if (c.tokens <= 8) 4 else 8, - .num_stages = 2, - .global_scratch_memory_size = @intCast(128 * @divTrunc(c.hidden, 128) * tile_count), - }, - ); - - return Combine.call( - .{ .d = d.d, .ids = a.ids, .perm = perm, .rw = a.scales }, - .{ .y = a.x.shape() }, - .{ - .cfg = c, - .grid = .{ @intCast(c.tokens), @intCast(@divTrunc(c.hidden + combineBlock(c) - 1, combineBlock(c))), 1 }, - .num_warps = 4, - .num_stages = 1, - }, - ).y; -} - -fn quant32(b: *B, x: V) struct { q: V, s: V } { - const rows = x.shape().constSlice()[0]; - const cols = x.shape().constSlice()[1]; - const groups = @divExact(cols, 32); - const blocks = b.reshape(x, &.{ rows * groups, 32 }); - const sf = b.exp2(b.ceil(b.log2(b.maxOpts(b.absf(blocks), .{ .axis = 1 }).maximum(1e-4).mul(1.0 / 448.0)))); - const q = blocks.div(sf.expandDims(1)).maximum(-448.0).minimum(448.0).to(.f8e4m3fn); - const bits = b.shrui(b.bitcast(sf, .i32), b.full(sf.shape().constSlice(), 23, .i32)).bitAnd(255).to(.i8); - return .{ .q = b.reshape(q, &.{ rows, cols }), .s = b.reshape(bits, &.{ rows, groups }) }; -} - -fn quantInput(b: *B, c: Config) tri.FinishError!void { - const a = try b.declareArgs(.{ .x = .{ .ptr = .bf16 }, .q = .{ .ptr = .f8e4m3fn }, .s = .{ .ptr = .i8 } }); - const rows = b.programId(.x).mul(quantRows(c)).add(ar(b, quantRows(c))); - const block = b.programId(.y); - const cols = block.mul(128).add(ar(b, 128)); - const offsets = rows.expandDims(1).mul(c.hidden).add(cols.expandDims(0)); - const mask = rows.lt(c.tokens).expandDims(1); - const x = ld(b, a.x.addPtr(offsets), mask, .bf16, 0).to(.f32); - const q = quant32(b, x); - b.storeOpts(a.q.addPtr(offsets), q.q, .{ .mask = mask }); - const scale_offsets = rows.expandDims(1).mul(@divTrunc(c.hidden, 32)).add(block.mul(4).add(ar(b, 4)).expandDims(0)); - b.storeOpts(a.s.addPtr(scale_offsets), q.s, .{ .mask = mask }); -} - -fn route(b: *B, c: Config) tri.FinishError!void { - const a = try b.declareArgs(.{ .ids = .{ .ptr = .i32 }, .counts = .{ .ptr = .i32 }, .pos = .{ .ptr = .i32 } }); - const e = b.programId(.x); - const p = ar(b, pow2(c.tokens * c.topk)); - const ids = ld(b, a.ids.addPtr(p), p.lt(c.tokens * c.topk), .i32, -1); - const valid = p.lt(c.tokens * c.topk).bitAnd(ids.eq(e.add(c.expert_offset))); - b.storeOpts(a.pos.addPtr(p), valid.to(.i32).cumsum().sub(1), .{ .mask = valid }); - b.store(a.counts.addPtr(e), valid.to(.i32).sum()); -} - -const route_block = 1024; -const atomic_route_threshold = 6 * 2048; - -/// Routing for large batches, used by `scheduleForCute`. -/// Each route takes the next free slot of its expert with an atomic -/// increment of `counts` (zero-initialized through `zeros`). Slots within an -/// expert are therefore unordered, which the grouped GEMMs do not care -/// about: every routed row is computed independently. -fn routeAtomic(b: *B, c: Config) tri.FinishError!void { - const a = try b.declareArgs(.{ .ids = .{ .ptr = .i32 }, .zeros = .{ .ptr = .i32 }, .counts = .{ .ptr = .i32 }, .pos = .{ .ptr = .i32 } }); - const p = b.programId(.x).mul(route_block).add(ar(b, route_block)); - const in_range = p.lt(c.tokens * c.topk); - const e = ld(b, a.ids.addPtr(p), in_range, .i32, -1).sub(ci(b, c.expert_offset)); - const valid = in_range.bitAnd(e.ge(0)).bitAnd(e.lt(c.experts)); - const slot = b.atomicRmwOpts(.add, a.counts.addPtr(e.maximum(0)), b.full(&.{route_block}, 1, .i32), .{ .mask = valid, .sem = .relaxed }); - b.storeOpts(a.pos.addPtr(p), slot, .{ .mask = valid }); -} - -fn scheduleGroups(b: *B, cfg: ScheduleConfig) tri.FinishError!void { - const c = cfg.moe; - const nt = cfg.capacity; - // Route positions are i32 tensors; keep the group size in that type. - const group: i32 = @intCast(cfg.group); - const a = try b.declareArgs(.{ .ids = .{ .ptr = .i32 }, .counts = .{ .ptr = .i32 }, .pos = .{ .ptr = .i32 }, .map = .{ .ptr = .i32 }, .perm = .{ .ptr = .i32 }, .sched = .{ .ptr = .i32 } }); - const p = b.programId(.x).mul(128).add(ar(b, 128)); - const e = ld(b, a.ids.addPtr(p), p.lt(c.tokens * c.topk), .i32, -1).sub(ci(b, c.expert_offset)); - const es = ar(b, pow2(c.experts)); - const counts = ld(b, a.counts.addPtr(es), es.lt(c.experts), .i32, 0); - const ts = counts.cdiv(group); - const starts = ts.cumsum().sub(ts); - var first = b.openIf(b.programId(.x).eq(0)); - b.store(a.sched.addPtr(2 * nt), ts.sum()); - first.yieldThen(.{}); - const live = p.lt(c.tokens * c.topk).bitAnd(e.ge(0)).bitAnd(e.lt(c.experts)); - const safe = e.maximum(0).minimum(pow2(c.experts) - 1); - const pos = ld(b, a.pos.addPtr(p), live, .i32, 0); - const start = starts.gather(safe, 0); - const tile = start.add(pos.div(group)); - const row = pos.rem(group); - b.storeOpts(a.perm.addPtr(p), b.select(live, tile.mul(group).add(row), b.full(&.{128}, -1, .i32)), .{ .mask = p.lt(c.tokens * c.topk) }); - b.storeOpts(a.map.addPtr(tile.mul(group).add(row)), p, .{ .mask = live }); - b.storeOpts(a.sched.addPtr(tile), e, .{ .mask = live.bitAnd(row.eq(0)) }); - b.storeOpts(a.sched.addPtr(nt).addPtr(tile), counts.gather(safe, 0).sub(pos).minimum(group), .{ .mask = live.bitAnd(row.eq(0)) }); -} - -fn gateUp(b: *B, c: GemmConfig) tri.FinishError!void { - return gemm(b, c.moe, .gate_up, c.scale_stride); -} - -fn down(b: *B, c: GemmConfig) tri.FinishError!void { - return gemm(b, c.moe, .down, c.scale_stride); -} - -const Projection = enum { gate_up, down }; - -fn gemm(b: *B, c: Config, comptime projection: Projection, scale_stride: i64) tri.FinishError!void { - const is_up = projection == .gate_up; - const a = if (is_up) try b.declareArgs(.{ .q = .{ .ptr = .f8e4m3fn }, .s = .{ .ptr = .i8 }, .w = .{ .ptr = .i8 }, .ws = .{ .ptr = .i8 }, .ids = .{ .ptr = .i32 }, .rw = .{ .ptr = .f32 }, .map = .{ .ptr = .i32 }, .sched = .{ .ptr = .i32 }, .mid = .{ .ptr = .f8e4m3fn }, .ms = .{ .ptr = .i8 } }) else try b.declareArgs(.{ .q = .{ .ptr = .f8e4m3fn }, .s = .{ .ptr = .i8 }, .w = .{ .ptr = .i8 }, .ws = .{ .ptr = .i8 }, .ids = .{ .ptr = .i32 }, .sched = .{ .ptr = .i32 }, .d = .{ .ptr = .bf16 } }); - const block = b.programId(.x); - const tile = b.programId(.y); - const n = ar(b, bn); - const nt = tiles(c); - const expert = if (c.tokens == 1) b.load(a.ids.addPtr(tile)).sub(ci(b, c.expert_offset)) else ld(b, a.sched.addPtr(tile), tile.lt(b.load(a.sched.addPtr(2 * nt))), .i32, 0); - const valid = if (c.tokens == 1) expert.ge(0).bitAnd(expert.lt(c.experts)) else tile.lt(b.load(a.sched.addPtr(2 * nt))); - var live = b.openIf(valid); - const count = if (c.tokens == 1) ci(b, 1) else b.load(a.sched.addPtr(nt).addPtr(tile)); - const pair = if (c.tokens == 1) b.full(&.{bn}, 0, .i32).add(tile) else if (is_up) ld(b, a.map.addPtr(tile.mul(bn).add(n)), n.lt(count), .i32, 0) else n; - const width: i64 = if (is_up and c.tokens > 8) 256 else 128; - const ksize = if (is_up) c.hidden else c.intermediate; - const rows = if (is_up) 2 * c.intermediate else c.hidden; - const m = block.mul(width).add(ar(b, width)); - const k = ar(b, bk); - const ks = ar(b, bk / 32); - const row = if (is_up) pair.div(c.topk) else tile.mul(bn).add(n); - const wd = b.makeTensorDescriptor(a.w.addPtr(expert.to(.i64).mul(@divTrunc(rows * ksize, 2))), &.{ ci(b, rows), ci(b, @divTrunc(ksize, 2)) }, &.{ b.liftAs(@divTrunc(ksize, 2), .i64), b.liftAs(1, .i64) }, .pad_zero, &.{ width, bk / 2 }, .i8); - var loop = b.openFor(0, @divTrunc(ksize + bk - 1, bk), 1, .{b.zeros(&.{ width, bn }, .f32)}); - const sk = loop.iv.mul(bk / 32).add(ks); - const kk = loop.iv.mul(bk).add(k); - const w = b.descriptorLoad(wd, &.{ block.mul(ci(b, width)), loop.iv.mul(bk / 2) }, &.{ width, bk / 2 }, .i8); - const scale_offset = m.expandDims(1).mul(scale_stride).add(sk.expandDims(0)); - const ws = ld(b, a.ws.addPtr(expert.to(.i64).mul(rows * scale_stride)).addPtr(scale_offset), sk.expandDims(0).lt(@divTrunc(ksize, 32)), .i8, 127); - var mask = kk.expandDims(0).lt(ksize); - var smask = sk.expandDims(0).lt(@divTrunc(ksize, 32)); - // The small-batch epilogue leaves padded rows unwritten. - if (is_up or c.tokens <= 8) { - mask = mask.bitAnd(n.lt(count).expandDims(1)); - smask = smask.bitAnd(n.lt(count).expandDims(1)); - } - const x = ld(b, a.q.addPtr(row.expandDims(1).mul(ksize).add(kk.expandDims(0))), mask, .f8e4m3fn, 0); - const xs = ld(b, a.s.addPtr(row.expandDims(1).mul(@divTrunc(ksize, 32)).add(sk.expandDims(0))), smask, .i8, 127); - loop.yield(.{b.dotScaled(w, b.permute(x, &.{ 1, 0 }), loop.carried[0], ws, xs, .e2m1, .e4m3)}); - const epilogue_n: i64 = if (c.tokens == 1) 1 else bn; - const out_n = ar(b, epilogue_n); - const acc = if (c.tokens == 1) live_row: { - // The remaining MMA columns are padding during single-token decode. - // Extract column zero before activation/reduction and output stores. - var v = loop.results[0]; - var cols: i64 = bn; - while (cols > 1) : (cols = @divTrunc(cols, 2)) { - v = b.split(b.reshape(v, &.{ width, @divTrunc(cols, 2), 2 }))[0]; - } - break :live_row b.reshape(v, &.{ 1, width }).to(.bf16); - } else b.permute(loop.results[0], &.{ 1, 0 }).to(.bf16); - if (is_up) { - const columns = @divExact(width, 2); - const halves = b.split(b.reshape(acc.to(.f32), &.{ epilogue_n, columns, 2 })); - var g = halves[0]; - var u = halves[1]; - if (c.swiglu_limit > 0) { - g = g.minimum(c.swiglu_limit); - u = u.minimum(c.swiglu_limit).maximum(-c.swiglu_limit); - } - const ex = b.externElementwise(&.{g.mul(-1.0)}, &.{ epilogue_n, columns }, .f32, "libdevice", "", "__nv_expf"); - var mid = u.mul(b.divRn(g, ex.add(1.0))); - if (c.routing_weight_placement == .before_down) { - const rw = if (c.tokens == 1) b.load(a.rw.addPtr(tile)) else ld(b, a.rw.addPtr(pair), n.lt(count), .f32, 0).expandDims(1); - mid = mid.mul(rw); - } - // Compute scales and quantize directly from the FP32 SwiGLU result. - const q = quant32(b, mid); - const col = block.mul(columns).add(ar(b, columns)); - b.store(a.mid.addPtr(tile.mul(bn).add(out_n).expandDims(1).mul(c.intermediate).add(col.expandDims(0))), q.q); - const scale_cols = block.mul(@divExact(columns, 32)).add(ar(b, @divExact(columns, 32))); - b.store(a.ms.addPtr(tile.mul(bn).add(out_n).expandDims(1).mul(@divTrunc(c.intermediate, 32)).add(scale_cols.expandDims(0))), q.s); - } else b.store(a.d.addPtr(tile.mul(bn).add(out_n).expandDims(1).mul(c.hidden).add(m.expandDims(0))), acc); - live.yieldThen(.{}); -} - -fn combine(b: *B, c: Config) tri.FinishError!void { - return combineWithBn(b, c, bn); -} - -fn combineWithBn(b: *B, c: Config, tile_n: i64) tri.FinishError!void { - const a = try b.declareArgs(.{ .d = .{ .ptr = .bf16 }, .ids = .{ .ptr = .i32 }, .perm = .{ .ptr = .i32 }, .rw = .{ .ptr = .f32 }, .y = .{ .ptr = .bf16 } }); - const t = b.programId(.x); - const block = combineBlock(c); - const h = b.programId(.y).mul(block).add(ar(b, block)); - var acc = b.zeros(&.{block}, .f32); - for (0..@intCast(c.topk)) |s| { - const pair = t.mul(c.topk).add(@as(i64, @intCast(s))); - const row = if (c.tokens == 1) pair.mul(tile_n) else b.load(a.perm.addPtr(pair)); - const valid = if (c.tokens == 1) v: { - const e = b.load(a.ids.addPtr(pair)).sub(ci(b, c.expert_offset)); - break :v e.ge(0).bitAnd(e.lt(c.experts)); - } else row.ge(0); - var value = ld(b, a.d.addPtr(row.mul(c.hidden).add(h)), valid.bitAnd(h.lt(c.hidden)), .bf16, 0).to(.f32); - if (c.routing_weight_placement == .after_down) { - value = value.mul(ld(b, a.rw.addPtr(pair), valid, .f32, 0)); - } - acc = acc.add(value); - } - b.storeOpts(a.y.addPtr(t.mul(c.hidden).add(h)), acc.to(.bf16), .{ .mask = h.lt(c.hidden) }); -} diff --git a/zml/moe/triton_mxfp4.zig b/zml/moe/triton_mxfp4.zig deleted file mode 100644 index d20ffe8f..00000000 --- a/zml/moe/triton_mxfp4.zig +++ /dev/null @@ -1,116 +0,0 @@ -const std = @import("std"); -const stdx = @import("stdx"); - -const zml = @import("../zml.zig"); -pub const kernels = @import("triton_kernels/mxfp4.zig"); - -pub fn isAvailable(platform: *const zml.Platform) bool { - if (platform.target != .cuda) return false; - const cc = zml.platform.cuda.computeCapability(platform) orelse return false; - return cc.major == 10; -} - -pub fn validateOptions(opts: zml.moe.Options) void { - stdx.debug.assert(opts.activation == .swiglu, "cute_mxfp4 backend only accepts swiglu activation, got {}", .{opts.activation}); - stdx.debug.assert(opts.activation.swiglu.limit != null, "cute_mxfp4 backend requires swiglu limit to be set", .{}); - stdx.debug.assert(opts.activation.swiglu.bias == null, "cute_mxfp4 backend requires swiglu bias to be null", .{}); - stdx.debug.assert(opts.activation.swiglu.scale == null, "cute_mxfp4 backend requires swiglu scale to be null", .{}); -} - -/// Row-major MXFP4 E2M1 weights and linear E8M0 block32 scales. -/// BF16 activations are quantized on the GPU to FP8 with per-32 E8M0 scales. -pub fn fusedExperts( - input: zml.Tensor, - ids: zml.Tensor, - weights: zml.Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - options: zml.moe.Options, -) zml.Tensor { - validateOptions(options); - stdx.debug.assert(input.dtype() == .bf16, "triton_mxfp4 backend only supports bf16 inputs, got {}", .{input.dtype()}); - stdx.debug.assert(gate_up.bias == null and down.bias == null, "triton_mxfp4 backend expects gate_up bias and down bias to be null", .{}); - - const gq = gate_up.quantization orelse @panic("triton_mxfp4 backend requires gate_up quantization to be set"); - const dq = down.quantization orelse @panic("triton_mxfp4 backend requires down quantization to be set"); - stdx.debug.assert(gq.scheme == .mxfp4 and dq.scheme == .mxfp4, "triton_mxfp4 expects gate_up and down quantization scheme to be mxfp4, got {} and {}", .{ gq.scheme, dq.scheme }); - - // Weight storage for mxfp4 in HF is expressed as u8 or i8 - stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "triton_mxfp4 expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); - stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "triton_mxfp4 expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); - - const expert_parallelism = gate_up.weight.shape().partition(.expert).isSharded(); - - const context: Context = .{ - .input = input, - .ids = ids, - .weights = weights, - .w1 = gate_up.weight.bitCast(.u8), - .s1 = gq.scales, - .w2 = down.weight.bitCast(.u8), - .s2 = dq.scales, - .global_experts = gate_up.weight.dim(.expert), - .topk = ids.dim(.topk), - .limit = options.activation.swiglu.limit.?, - .routing_weight_placement = options.routing_weight_placement, - .expert_parallel = expert_parallelism, - }; - - return if (expert_parallelism) - zml.ops.manualComputation(Context.body, context, input.shape()) - else - context.body(input.shape()); -} - -const Context = struct { - input: zml.Tensor, - ids: zml.Tensor, - weights: zml.Tensor, - w1: zml.Tensor, - s1: zml.Tensor, - w2: zml.Tensor, - s2: zml.Tensor, - global_experts: i64, - topk: i64, - limit: f32, - routing_weight_placement: zml.moe.triton.RoutingWeightPlacement, - expert_parallel: bool, - - fn body(self: Context, _: zml.Shape) zml.Tensor { - const experts = self.w1.dim(.expert); - const hidden = self.w2.dim(1); - const intermediate = self.w2.dim(2) * 2; - var ids = self.ids.convert(.i32).reshape(.{ .token = .auto, .topk = self.topk }); - const tokens = ids.dim(.token); - if (self.expert_parallel) { - const partition_id = zml.ops.partitionId().convert(.i32); - ids = ids.sub(partition_id.scale(experts)); - } - - const cfg: kernels.Config = .{ - .tokens = tokens, - .hidden = hidden, - .intermediate = intermediate, - .experts = experts, - .global_experts = self.global_experts, - .topk = self.topk, - .swiglu_limit = self.limit, - .routing_weight_placement = self.routing_weight_placement, - }; - - const result = kernels.forward(cfg, .{ - .x = self.input.reshape(.{ tokens, hidden }), - .ids = ids, - .scales = self.weights.convert(.f32).reshape(.{ tokens, self.topk }), - .w1 = self.w1, - .s1 = self.s1, - .w2 = self.w2, - .s2 = self.s2, - }).reshape(self.input.shape().dims()).withTags(self.input.shape()); - - return if (self.expert_parallel) - zml.ops.allReduce(result, zml.Tensor.add) - else - result; - } -}; diff --git a/zml/zml.zig b/zml/zml.zig index d5ee6f08..23d12e75 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -32,7 +32,7 @@ pub const mem = @import("mem.zig"); pub const Bufferized = mem.Bufferized; pub const meta = @import("meta.zig"); pub const mlir = @import("mlirx.zig"); -pub const moe = @import("moe/moe.zig"); +pub const moe = @import("kernels/moe/moe.zig"); pub const nn = @import("nn.zig"); pub const ops = @import("ops.zig"); pub const pjrtx = @import("pjrtx.zig"); From 94ead7094e5f317f97506d31cc7053681e32b9f0 Mon Sep 17 00:00:00 2001 From: Louis Lechevalier Date: Mon, 19 Oct 2026 15:18:23 +0200 Subject: [PATCH 797/881] zml/platforms: bump ZML plugins and enable all reduce collective in cmd buffer (#959) This enable Lamport all reduce collective if compute capabilities matched on CUDA and ROCm --- platforms/cpu/cpu.bzl | 12 ++++++------ platforms/cuda/cuda.bzl | 8 ++++---- platforms/furiosa/furiosa.bzl | 4 ++-- platforms/metal/metal.bzl | 4 ++-- platforms/rocm/rocm.bzl | 4 ++-- zml/Compiler.zig | 4 ++-- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 1e20d1fa..f6d1393f 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -42,22 +42,22 @@ def _cpu_plugin_impl(mctx): http_archive( name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "9d82f2612decf7066e1c66e3da8fa8883f5f3714fe3aaa6b1ee6041b435552e6", - url = "https://mirror.zml.ai/plugins/202610160837.234.1.e1adc5dca29d/zml-cpu-linux-amd64.tar.zst", + sha256 = "ed12c7d89f2a9f914368ede117c6086ea36c557cb3f0ddfde20a92dbaa4192a1", + url = "https://mirror.zml.ai/plugins/202610191219.248.1.c2619532effe/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "f365f0092a6b493888d3ce87d3bc859c02116df455037f609414e7bfeac0873e", - url = "https://mirror.zml.ai/plugins/202610160837.234.1.e1adc5dca29d/zml-cpu-darwin-amd64.tar.zst", + sha256 = "c45347542e35a289e49bd4682436a041491a992f6ed26747f6885827c0f527f8", + url = "https://mirror.zml.ai/plugins/202610191219.248.1.c2619532effe/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "3ebd6bec2c05599f250415b309ffcd5f34e5c1c37b7b806eeb35c88254e8eb16", - url = "https://mirror.zml.ai/plugins/202610160837.234.1.e1adc5dca29d/zml-cpu-darwin-arm64.tar.zst", + sha256 = "b06e61dee9d796cccebb8f80abcd125d4ed39350820905400e8668e61590723b", + url = "https://mirror.zml.ai/plugins/202610191219.248.1.c2619532effe/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 8604d025..a936c467 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -224,12 +224,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "226e8b04611efe6c8ac3ceaf3010499b5957d498e585c970afb69c2ec47ff9fe", - "url": "https://mirror.zml.ai/plugins/202610150915.220.1.f2470651679c/zml-cuda-linux-amd64.tar.zst", + "sha256": "b4a87415714f44c9adc2ce38bfad5ef42518195349a6a548a549cb2de2f0af2e", + "url": "https://mirror.zml.ai/plugins/202610191219.248.1.c2619532effe/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "5b55fcd3173485b0e96034bbbf078e354c393d938aadb2ab8fa9bf658b384759", - "url": "https://mirror.zml.ai/plugins/202610150915.220.1.f2470651679c/zml-cuda-linux-arm64.tar.zst", + "sha256": "2fccf6a376d9e735a6103acbfca1aaf6acf08791ed2b64dedece322cab8c5153", + "url": "https://mirror.zml.ai/plugins/202610191219.248.1.c2619532effe/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index 3dae64a3..2e665538 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202610160837.234.1.e1adc5dca29d/zml-furiosa-linux-amd64.tar.zst", - sha256 = "e474a6ba2f3b5e810dba24dc7339bf45d5e7ea05025c924d7952d968ef6f1335", + url = "https://mirror.zml.ai/plugins/202610191219.248.1.c2619532effe/zml-furiosa-linux-amd64.tar.zst", + sha256 = "3913a592fbc35a50ac60700011529addb9c18a51b614dfc5d1aa4f346b500712", ) return mctx.extension_metadata( diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index ecfb19e2..7a937584 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "70fbd60acacf40268d9b05d692f75e6e5360169c083819d27dd95ddd2214a0be", - url = "https://mirror.zml.ai/plugins/202610160837.234.1.e1adc5dca29d/zml-metal-darwin-arm64.tar.zst", + sha256 = "e484f78bb14df98c4271f0d91760c32d167716b936ac465258fb7874282b3b2a", + url = "https://mirror.zml.ai/plugins/202610191219.248.1.c2619532effe/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 4ebd9a1c..7d4206be 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610160837.234.1.e1adc5dca29d/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "22a1cf08945e194164af39175c58a119c57536f5ef9f15c72804808c78f1157a" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610191219.248.1.c2619532effe/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "777213f6cba1c8f0cdcd88d2ff12f9445e41cb498b1d382a3fb1bfad0fcf99d0" def _rocm_package_name(name): return name + _ROCM_VERSION diff --git a/zml/Compiler.zig b/zml/Compiler.zig index c6c30394..d04fccec 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -972,11 +972,11 @@ fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.Load .rocm => { try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_scheduling_mode", "CONCURRENT", upb_arena); try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_update_mode", "SKIP_TEMP", upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_collective_kernels", "", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_collective_kernels", "COLLECTIVE_KERNEL_ALL_REDUCE", upb_arena); // Only capture all-reduce and all-gather collectives in HIP graphs; other collective types execute outside command buffers since they are unsupported in HIP graphs. try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_collectives_command_buffer_filter", "ALLREDUCE,ALLGATHER", upb_arena); // Add collectives to the default list - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES_KERNEL,COLLECTIVES,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); // With SKIP_TEMP, captured NCCL collectives can retain stale physical mappings // if VMM reclaims their backing under memory pressure, causing silent corruption. // Enable user buffers to assign color 1 and receive the reclaim exemption: From 231154ef14d6c14507659896ba09a321c46befe0 Mon Sep 17 00:00:00 2001 From: Arthur <124417490+Arthur031221@users.noreply.github.com> Date: Tue, 20 Oct 2026 00:45:42 +0800 Subject: [PATCH 798/881] zml/dtype: fix Value.as panicking for f4e2m1/f8e3m4/f8e4m3/f8e8m0 (#878) `DataType.Value.as` converts a tagged-union `Value` to a native Zig type. The float branch of the inline switch lists which float dtypes it knows how to convert via `.toF32()`, but that list was never updated when f4e2m1, f8e3m4, f8e4m3 (the plain variant, distinct from f8e4m3b11fnuz/fn/fnuz), and f8e8m0 were added to `DataType`. All four already implement `.toF32()` in zml/floats.zig, so calling `.as(f32)` on a `Value` of any of these dtypes panics with "Unsupported conversion ... -> f32" instead of returning the converted float. The fix is a one-line addition of the four dtypes to the existing switch arm, the same pattern as the other float dtypes already listed there. --- zml/dtype.zig | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/zml/dtype.zig b/zml/dtype.zig index f8d24089..f5cd9da8 100644 --- a/zml/dtype.zig +++ b/zml/dtype.zig @@ -363,7 +363,7 @@ pub const DataType = enum(u8) { .bool => return self.bool, .float => switch (self) { inline .f16, .f32, .f64 => |v| return @floatCast(v), - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16 => |v| return @floatCast(v.toF32()), + inline .f4e2m1, .f8e3m4, .f8e4m3, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .f8e8m0, .bf16 => |v| return @floatCast(v.toF32()), else => {}, }, .int => switch (self) { @@ -374,6 +374,13 @@ pub const DataType = enum(u8) { } std.debug.panic("Unsupported conversion {} -> {s}", .{ self.dtype(), @typeName(T) }); } + + test as { + try std.testing.expectEqual(@as(f32, 1.0), Value.init(.f4e2m1, 1).as(f32)); + try std.testing.expectEqual(@as(f32, 1.0), Value.init(.f8e3m4, 1).as(f32)); + try std.testing.expectEqual(@as(f32, 1.0), Value.init(.f8e4m3, 1).as(f32)); + try std.testing.expectEqual(@as(f32, 1.0), Value.init(.f8e8m0, 1).as(f32)); + } }; }; From a5dfa567d46b6046c39f8f9b32b91622c283ad5d Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 20 Oct 2026 09:30:12 +0200 Subject: [PATCH 799/881] bazel: update to 9.3.0 (#964) --- .bazelversion | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bazelversion b/.bazelversion index 44931da2..b13d146a 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -9.1.1 +9.3.0 From 1179f605f778aaa87b02adcea31377937b05dc6d Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 20 Oct 2026 09:31:03 +0200 Subject: [PATCH 800/881] ci: remote build (#924) Local runner growing disk usage will be handled in another PR. --- .bazelrc | 2 ++ .github/workflows/ci.yaml | 35 ++++++++++++++--------------------- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/.bazelrc b/.bazelrc index 1b47d230..09c50737 100644 --- a/.bazelrc +++ b/.bazelrc @@ -120,8 +120,10 @@ common:remote --remote_cache=grpcs://remote.buildbuddy.io common:remote --experimental_remote_downloader=grpcs://remote.buildbuddy.io common:remote --remote_executor=grpcs://remote.buildbuddy.io common:remote --remote_download_toplevel +common:remote --remote_download_outputs=minimal common:remote --nobuild_runfile_links common:remote --remote_timeout=3600 +common:remote --experimental_remote_cache_chunking common:remote --noexperimental_throttle_remote_action_building common:remote --remote_cache_compression # Re-runs actions instead of failing on transient lost remote/disk cache inputs. diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 13d153b3..bace1a65 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -20,7 +20,6 @@ jobs: include: - target: cpu bazel_opts: "" - self_hosted: false run_tests: true runner_labels: - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-cpu @@ -29,9 +28,9 @@ jobs: - image=ubuntu24-amd64 - volume=80gb - extras=s3-cache + - target: neuron bazel_opts: "--@zml//platforms:neuron=true --@zml//platforms:cpu=false" - self_hosted: false run_tests: false runner_labels: - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-neuron @@ -40,9 +39,9 @@ jobs: - image=ubuntu24-amd64 - volume=80gb - extras=s3-cache + - target: tpu bazel_opts: "--@zml//platforms:tpu=true --@zml//platforms:cpu=false" - self_hosted: false run_tests: false runner_labels: - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-tpu @@ -51,16 +50,19 @@ jobs: - image=ubuntu24-amd64 - volume=80gb - extras=s3-cache + - target: cuda bazel_opts: "--@zml//platforms:cuda=true --@zml//platforms:cpu=false" - self_hosted: true run_tests: true runner_labels: ["self-hosted", "platform=cuda"] + - target: rocm bazel_opts: "--@zml//platforms:rocm=true --@zml//platforms:cpu=false" - self_hosted: true run_tests: true runner_labels: ["self-hosted", "platform=rocm"] + + # Todo : add missing platforms like Metal / oneAPI / Furiosa ... + runs-on: ${{ matrix.runner_labels }} steps: - uses: runs-on/action@v2.1.0 @@ -68,33 +70,24 @@ jobs: uses: actions/checkout@v6 - name: Ensure self hosted cache permissions - if: ${{ matrix.self_hosted }} run: mkdir -p /tmp/zig-cache && mkdir -p /tmp/zml && sudo chmod -R 777 /tmp/zig-cache && sudo chmod -R 777 /tmp/zml - name: Setup Bazel uses: bazel-contrib/setup-bazel@0.19.0 with: - bazelisk-version: 1.28.1 - bazelisk-cache: ${{ !matrix.self_hosted }} - repository-cache: ${{ !matrix.self_hosted }} - external-cache: ${{ !matrix.self_hosted }} + bazelisk-version: 1.29.0 + bazelisk-cache: true + cache-save: false bazelrc: | + startup --experimental_remote_repo_contents_cache common --verbose_failures common --keep_going common --test_output=streamed common --color=yes common --show_timestamps - build --experimental_disk_cache_gc_max_size=100G - build --experimental_disk_cache_gc_max_age=14d - build --experimental_disk_cache_gc_idle_delay=1m - common --experimental_action_cache_gc_max_age=14d - common --experimental_action_cache_gc_idle_delay=1m - common --repo_contents_cache_gc_max_age=14d - common --repo_contents_cache_gc_idle_delay=1m - ${{ !matrix.self_hosted && secrets.BUILDBUDDY_CI_API_KEY != '' && 'common --config=remote' || '' }} - ${{ !matrix.self_hosted && secrets.BUILDBUDDY_CI_API_KEY != '' && format('common:remote --remote_header=x-buildbuddy-api-key={0}', secrets.BUILDBUDDY_CI_API_KEY) || '' }} - ${{ !matrix.self_hosted && secrets.BUILDBUDDY_CI_API_KEY != '' && 'common:remote --remote_download_outputs=minimal' || '' }} - ${{ matrix.self_hosted && 'common --disk_cache=/home/actions/.cache/bazel-disk' || '' }} + common --config=remote + common --strategy=TestRunner=local + ${{ format('common:remote --remote_header=x-buildbuddy-api-key={0}', secrets.BUILDBUDDY_CI_API_KEY) || '' }} - run: bazel mod graph - run: bazel query --output=graph ${{ matrix.bazel_opts }} //... From dd82a8a10b83b59b6c1756c015d09155a87b08a1 Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Tue, 20 Oct 2026 11:36:07 +0200 Subject: [PATCH 801/881] third_party/zls: bump to 0.17.0 (#966) --- MODULE.bazel.lock | 603 +----------------------------------- third_party/zls/BUILD.bazel | 2 +- 2 files changed, 10 insertions(+), 595 deletions(-) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index b4bf3b82..1e746605 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1,5 +1,5 @@ { - "lockFileVersion": 26, + "lockFileVersion": 28, "registryFileHashes": { "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", "https://bcr.bazel.build/modules/abseil-cpp/20260526.0/MODULE.bazel": "b4a76a8d70ba08aa0253fd5e374ca5a5fa6b15aba985a7bbe4467c138954830f", @@ -17,6 +17,7 @@ "https://bcr.bazel.build/modules/apple_support/1.24.1/MODULE.bazel": "f46e8ddad60aef170ee92b2f3d00ef66c147ceafea68b6877cb45bd91737f5f8", "https://bcr.bazel.build/modules/apple_support/1.24.2/MODULE.bazel": "0e62471818affb9f0b26f128831d5c40b074d32e6dda5a0d3852847215a41ca4", "https://bcr.bazel.build/modules/apple_support/2.3.0/MODULE.bazel": "d48f824ae8eeea5f837eb3038cef3615075d996d3e82eb9187192c2820605a81", + "https://bcr.bazel.build/modules/apple_support/2.5.2/MODULE.bazel": "bb1b7081e390a8d3885f642cb0073dfcf8e7a36525e079fa8f5835af7079c0fe", "https://bcr.bazel.build/modules/apple_support/2.5.4/MODULE.bazel": "964e9268b37edb64e9a0fa525e5b5e535ba56bf63cadc68375f92e69fcb734f6", "https://bcr.bazel.build/modules/apple_support/2.8.0/MODULE.bazel": "c45f5176057092afba0be6c48a9ea02101666802f8be536947e099e94757f8ed", "https://bcr.bazel.build/modules/apple_support/2.8.0/source.json": "b7f6d87669c206adaa7d82cc44733349c63e341b80a0e049496ebd91d78c3472", @@ -484,7 +485,7 @@ "moduleExtensions": { "@@aspect_rules_py+//py:extensions.bzl%py_tools": { "general": { - "bzlTransitiveDigest": "LaqXRQ7hUz2pKp1jB+4lExnjHXPJFKoz6+nOJqccLrQ=", + "bzlTransitiveDigest": "6o8lyMtImPc3Si+zcGss6WH8IY80mTWhC0MC3r0ra8Y=", "usagesDigest": "9BiP3ASHlGl4doGhzXJwK17taNzv/q6x/IZ4Hb6LxNY=", "recordedInputs": [ "REPO_MAPPING:aspect_rules_py+,aspect_tools_telemetry_report aspect_tools_telemetry++telemetry+aspect_tools_telemetry_report", @@ -535,49 +536,6 @@ } } }, - "@@aspect_rules_py+//uv/private/tomltool:extension.bzl%tomltool": { - "general": { - "bzlTransitiveDigest": "TwqHTUjLecbltZP/kGFRhI/12V79YPpuOflrAhnMaR8=", - "usagesDigest": "tQ76lH+Nn5OlAFLvBbnfvnCzNIiwGgY6tOumYZXH44w=", - "recordedInputs": [ - "REPO_MAPPING:aspect_rules_py+,bazel_tools bazel_tools" - ], - "generatedRepoSpecs": { - "toml2json_aarch64_osx_libsystem": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_darwin_arm64", - "sha256": "9adc44d976e8f5baf5a6d613ceb3db6e5f56b2e22e75ac56521fdce62f227d88", - "executable": true - } - }, - "toml2json_x86_64_osx_libsystem": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_darwin_amd64", - "sha256": "425bac4015394fd9840fabcd52ac60fc796a9c10655b36c57e36ffb7dc9f3dd4", - "executable": true - } - }, - "toml2json_aarch64_linux_gnu": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_linux_arm64", - "sha256": "be8376c8e3232a242eae0d187f741a475498a949b942361a9af6e95072ef5670", - "executable": true - } - }, - "toml2json_x86_64_linux_gnu": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "url": "https://github.com/dzbarsky/toml2json/releases/download/v0.0.4/toml2json_linux_amd64", - "sha256": "f3dd54fabf2d27d0c027b0421860e5d9d909080be1613f0ffd87057633b65e9a", - "executable": true - } - } - } - } - }, "@@aspect_tools_telemetry+//:extension.bzl%telemetry": { "general": { "bzlTransitiveDigest": "MePriaXmQNSqUfE+YQvEtzBc2bU1mjITINIffFiZYgo=", @@ -599,126 +557,6 @@ } } }, - "@@envoy_api+//bazel:repositories.bzl%non_module_deps": { - "general": { - "bzlTransitiveDigest": "zeEpd78tNW+9P6M7a2DhAjl5PQl47oXTBJOoYY0WuNI=", - "usagesDigest": "OP2p9QPxt4WFY/lah8Iecqt6SOUwZHs89hYRyaV+rSg=", - "recordedInputs": [ - "REPO_MAPPING:envoy_api+,bazel_tools bazel_tools", - "REPO_MAPPING:envoy_api+,envoy_api envoy_api+" - ], - "generatedRepoSpecs": { - "prometheus_metrics_model": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/prometheus/client_model/archive/v0.6.1.tar.gz" - ], - "sha256": "b9b690bc35d80061f255faa7df7621eae39fe157179ccd78ff6409c3b004f05e", - "strip_prefix": "client_model-0.6.1", - "build_file_content": "\nload(\"@envoy_api//bazel:api_build_system.bzl\", \"api_cc_py_proto_library\")\nload(\"@io_bazel_rules_go//proto:def.bzl\", \"go_proto_library\")\n\napi_cc_py_proto_library(\n name = \"client_model\",\n srcs = [\n \"io/prometheus/client/metrics.proto\",\n ],\n visibility = [\"//visibility:public\"],\n)\n\ngo_proto_library(\n name = \"client_model_go_proto\",\n importpath = \"github.com/prometheus/client_model/go\",\n proto = \":client_model\",\n visibility = [\"//visibility:public\"],\n)\n" - } - }, - "com_github_bufbuild_buf": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/bufbuild/buf/releases/download/v1.49.0/buf-Linux-x86_64.tar.gz" - ], - "sha256": "ee8da9748249f7946d79191e36469ce7bc3b8ba80019bff1fa4289a44cbc23bf", - "strip_prefix": "buf", - "build_file_content": "\npackage(\n default_visibility = [\"//visibility:public\"],\n)\n\nfilegroup(\n name = \"buf\",\n srcs = [\n \"@com_github_bufbuild_buf//:bin/buf\",\n ],\n tags = [\"manual\"], # buf is downloaded as a linux binary; tagged manual to prevent build for non-linux users\n)\n" - } - }, - "envoy_toolshed": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "urls": [ - "https://github.com/envoyproxy/toolshed/archive/bazel-v0.2.2.tar.gz" - ], - "sha256": "443fe177aba0cef8c17b7a48905c925c67b09005b10dd70ff12cd9f729a72d51", - "strip_prefix": "toolshed-bazel-v0.2.2/bazel" - } - } - } - } - }, - "@@googleapis+//:extensions.bzl%switched_rules": { - "general": { - "bzlTransitiveDigest": "dGjdAR2OztzWMSyz8eMU1fObvEYPBQh2DdtbMY2PmNc=", - "usagesDigest": "eoktHDHIPNOPltVMlzzSHLOYyaf7GneTwkOWrpNh8PQ=", - "recordedInputs": [], - "generatedRepoSpecs": {} - } - }, - "@@grpc+//bazel:extensions.bzl%exec_properties": { - "general": { - "bzlTransitiveDigest": "tdXJDtOnvTse7uoYV1eSKHW4jguoFr52HSAV6fIOAeI=", - "usagesDigest": "hImqXzHtSPMkGXUwrV9REcdKmXLsLUdEto6iGSUMItA=", - "recordedInputs": [ - "REPO_MAPPING:grpc+,bazel_toolchains grpc++http_archive+bazel_toolchains" - ], - "generatedRepoSpecs": { - "grpc_custom_exec_properties": { - "repoRuleId": "@@grpc++http_archive+bazel_toolchains//rules/exec_properties:exec_properties.bzl%_exec_property_sets_repository", - "attributes": { - "constants_bzl_content": "LARGE_MACHINE = {\"label:os\": \"ubuntu\", \"label:machine_size\": \"large\"}\n" - } - } - } - } - }, - "@@protobuf+//python/dist:system_python.bzl%system_python_extension": { - "general": { - "bzlTransitiveDigest": "XIzUjPrPvfZw+ee/BZaUhkKywlpB0qAKyTi/+DyC/XI=", - "usagesDigest": "tCi55FyqtOJ2jXh9vcjrHCl4ov3kpWiwKl103nA9BOI=", - "recordedInputs": [], - "generatedRepoSpecs": { - "system_python": { - "repoRuleId": "@@protobuf+//python/dist:system_python.bzl%system_python", - "attributes": { - "minimum_python_version": "3.9" - } - } - } - } - }, - "@@rules_android+//bzlmod_extensions:apksig.bzl%apksig_extension": { - "general": { - "bzlTransitiveDigest": "eYxUKP6jh5Ixolg9KdkZ/OBudos9hU2RBV+5XdqppZ4=", - "usagesDigest": "zr/niBQ/s2fHozWAsg4vI70wAxcuFjG+QtM15qGkq9o=", - "recordedInputs": [ - "REPO_MAPPING:rules_android+,bazel_tools bazel_tools" - ], - "generatedRepoSpecs": { - "apksig": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://android.googlesource.com/platform/tools/apksig/+archive/24e3075e68ebe17c0b529bb24bfda819db5e2f3b.tar.gz", - "build_file": "@@rules_android+//bzlmod_extensions:apksig.BUILD" - } - } - } - } - }, - "@@rules_android+//bzlmod_extensions:com_android_dex.bzl%com_android_dex_extension": { - "general": { - "bzlTransitiveDigest": "gy5TiFZm16I25Ieuw8gSwq8vLHlpWECIMp0gWY9YynI=", - "usagesDigest": "c1Y/KGGjUYCyd8zNIVTUh1bynVXRFz6xGKaSCBpQANM=", - "recordedInputs": [ - "REPO_MAPPING:rules_android+,bazel_tools bazel_tools" - ], - "generatedRepoSpecs": { - "com_android_dex": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://android.googlesource.com/platform/dalvik/+archive/5a81c499a569731e2395f7c8d13c0e0d4e17a2b6.tar.gz", - "build_file": "@@rules_android+//bzlmod_extensions:com_android_dex.BUILD" - } - } - } - } - }, "@@rules_android+//rules/android_sdk_repository:rule.bzl%android_sdk_repository_extension": { "general": { "bzlTransitiveDigest": "aLDGLKolYFwLsBssOQfwOs+bo3Aof0yKkW0HQRPX3ms=", @@ -734,7 +572,7 @@ }, "@@rules_oci+//oci:extensions.bzl%oci": { "general": { - "bzlTransitiveDigest": "u2chY5noKxJ9cum/JgI8hogPYhYniVKnJLd6f2ChPI4=", + "bzlTransitiveDigest": "cbfE3+b9HnO8wGMMbdSzaHKjcON7SL8gb4fKTNiVHug=", "usagesDigest": "JLbamplho20rviuebatMGqnQnaoj32ewCzLuRVusGoU=", "recordedInputs": [ "REPO_MAPPING:aspect_bazel_lib+,bazel_tools bazel_tools", @@ -929,72 +767,9 @@ } } }, - "@@rules_perl+//perl:extensions.bzl%perl_repositories": { - "general": { - "bzlTransitiveDigest": "kNKYG2Q4r+exUypJYjau+ehY90be7nvRR1ffN6loz7w=", - "usagesDigest": "qSSNDdCNVxNhY36wMndEAFacdhR0ooLTmumfad0km9s=", - "recordedInputs": [ - "REPO_MAPPING:rules_perl+,bazel_tools bazel_tools", - "REPO_MAPPING:rules_perl+,rules_perl rules_perl+" - ], - "generatedRepoSpecs": { - "perl_darwin_arm64": { - "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", - "attributes": { - "strip_prefix": "perl-darwin-arm64", - "sha256": "285769f3c50c339fb59a3987b216ae3c5c573b95babe6875a1ef56fb178433da", - "urls": [ - "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-darwin-arm64.tar.xz" - ] - } - }, - "perl_darwin_amd64": { - "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", - "attributes": { - "strip_prefix": "perl-darwin-amd64", - "sha256": "63bc5ee36f5394d71c50cca6cafdd333ee58f9eaa40bca63c85f9bd06f2c1fd6", - "urls": [ - "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-darwin-amd64.tar.xz" - ] - } - }, - "perl_linux_amd64": { - "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", - "attributes": { - "strip_prefix": "perl-linux-amd64", - "sha256": "3bdffa9d7a3f97c0207314637b260ba5115b1d0829f97e3e2e301191a4d4d076", - "urls": [ - "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-linux-amd64.tar.xz" - ] - } - }, - "perl_linux_arm64": { - "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", - "attributes": { - "strip_prefix": "perl-linux-arm64", - "sha256": "6fa4ece99e790ecbc2861f6ecb7b52694c01c2eeb215b4370f16a3b12d952117", - "urls": [ - "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-linux-arm64.tar.xz" - ] - } - }, - "perl_windows_x86_64": { - "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", - "attributes": { - "strip_prefix": "", - "sha256": "aeb973da474f14210d3e1a1f942dcf779e2ae7e71e4c535e6c53ebabe632cc98", - "urls": [ - "https://mirror.bazel.build/strawberryperl.com/download/5.32.1.1/strawberry-perl-5.32.1.1-64bit.zip", - "https://strawberryperl.com/download/5.32.1.1/strawberry-perl-5.32.1.1-64bit.zip" - ] - } - } - } - } - }, "@@rules_python+//python/uv:uv.bzl%uv": { "general": { - "bzlTransitiveDigest": "IzGWfoGARM6X2a4AM7BN5gheLtDsFeP9Fra2qTod+mA=", + "bzlTransitiveDigest": "ZtcPw94gqLP8+ocBKipnnyLIivxEC/OUYen7Kx7hnEA=", "usagesDigest": "Ucfum/vsVnlkhtglTERDLTj+bBeG7dcpbV9h0LP9HOQ=", "recordedInputs": [ "REPO_MAPPING:rules_python+,bazel_tools bazel_tools", @@ -1153,370 +928,9 @@ } } }, - "@@rules_rust+//crate_universe:extension.bzl%crate": { - "general": { - "bzlTransitiveDigest": "sv4+a0gzluPfXdGS4VRD5W4GeawxvNnNht0NHBgIAPs=", - "usagesDigest": "EuFUqVKVHF263jHTWOHXs4tFACdRNVOhwpoytdk19bs=", - "recordedInputs": [ - "ENV:CARGO_BAZEL_DEBUG \\0", - "ENV:CARGO_BAZEL_GENERATOR_SHA256 \\0", - "ENV:CARGO_BAZEL_GENERATOR_URL \\0", - "ENV:CARGO_BAZEL_ISOLATED \\0", - "ENV:CARGO_BAZEL_REPIN \\0", - "ENV:CARGO_BAZEL_REPIN_ONLY \\0", - "ENV:CARGO_BAZEL_TIMEOUT \\0", - "ENV:REPIN \\0", - "REPO_MAPPING:bazel_features+,bazel_features_globals bazel_features++version_extension+bazel_features_globals", - "REPO_MAPPING:bazel_features+,bazel_features_version bazel_features++version_extension+bazel_features_version", - "REPO_MAPPING:rules_cc+,bazel_skylib bazel_skylib+", - "REPO_MAPPING:rules_cc+,bazel_tools bazel_tools", - "REPO_MAPPING:rules_cc+,cc_compatibility_proxy rules_cc++compatibility_proxy+cc_compatibility_proxy", - "REPO_MAPPING:rules_cc+,platforms platforms", - "REPO_MAPPING:rules_cc+,rules_cc rules_cc+", - "REPO_MAPPING:rules_cc++compatibility_proxy+cc_compatibility_proxy,rules_cc rules_cc+", - "REPO_MAPPING:rules_rust+,bazel_features bazel_features+", - "REPO_MAPPING:rules_rust+,bazel_skylib bazel_skylib+", - "REPO_MAPPING:rules_rust+,bazel_tools bazel_tools", - "REPO_MAPPING:rules_rust+,rules_cc rules_cc+", - "REPO_MAPPING:rules_rust+,rules_rust rules_rust+" - ], - "generatedRepoSpecs": { - "crates": { - "repoRuleId": "@@rules_rust+//crate_universe:extensions.bzl%_generate_repo", - "attributes": { - "contents": { - "BUILD.bazel": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\npackage(default_visibility = [\"//visibility:public\"])\n\nexports_files(\n [\n \"cargo-bazel.json\",\n \"crates.bzl\",\n \"defs.bzl\",\n ] + glob(\n allow_empty = True,\n include = [\"*.bazel\"],\n ),\n)\n\nfilegroup(\n name = \"srcs\",\n srcs = glob(\n allow_empty = True,\n include = [\n \"*.bazel\",\n \"*.bzl\",\n ],\n ),\n)\n\n# Workspace Member Dependencies\nalias(\n name = \"googletest-0.14.3\",\n actual = \"@crates__googletest-0.14.3//:googletest\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"googletest\",\n actual = \"@crates__googletest-0.14.3//:googletest\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"linkme-0.3.36\",\n actual = \"@crates__linkme-0.3.36//:linkme\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"linkme\",\n actual = \"@crates__linkme-0.3.36//:linkme\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"paste-1.0.15\",\n actual = \"@crates__paste-1.0.15//:paste\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"paste\",\n actual = \"@crates__paste-1.0.15//:paste\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"quote-1.0.46\",\n actual = \"@crates__quote-1.0.46//:quote\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"quote\",\n actual = \"@crates__quote-1.0.46//:quote\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"syn-2.0.118\",\n actual = \"@crates__syn-2.0.118//:syn\",\n tags = [\"manual\"],\n)\n\nalias(\n name = \"syn\",\n actual = \"@crates__syn-2.0.118//:syn\",\n tags = [\"manual\"],\n)\n", - "alias_rules.bzl": "\"\"\"Alias that transitions its target to `compilation_mode=opt`. Use `transition_alias=\"opt\"` to enable.\"\"\"\n\nload(\"@rules_cc//cc:defs.bzl\", \"CcInfo\")\nload(\"@rules_rust//rust:rust_common.bzl\", \"COMMON_PROVIDERS\")\n\ndef _transition_alias_impl(ctx):\n # `ctx.attr.actual` is a list of 1 item due to the transition\n providers = [ctx.attr.actual[0][provider] for provider in COMMON_PROVIDERS]\n if CcInfo in ctx.attr.actual[0]:\n providers.append(ctx.attr.actual[0][CcInfo])\n return providers\n\ndef _change_compilation_mode(compilation_mode):\n def _change_compilation_mode_impl(_settings, _attr):\n return {\n \"//command_line_option:compilation_mode\": compilation_mode,\n }\n\n return transition(\n implementation = _change_compilation_mode_impl,\n inputs = [],\n outputs = [\n \"//command_line_option:compilation_mode\",\n ],\n )\n\ndef _transition_alias_rule(compilation_mode):\n return rule(\n implementation = _transition_alias_impl,\n provides = COMMON_PROVIDERS,\n attrs = {\n \"actual\": attr.label(\n mandatory = True,\n doc = \"`rust_library()` target to transition to `compilation_mode=opt`.\",\n providers = COMMON_PROVIDERS,\n cfg = _change_compilation_mode(compilation_mode),\n ),\n \"_allowlist_function_transition\": attr.label(\n default = \"@bazel_tools//tools/allowlists/function_transition_allowlist\",\n ),\n },\n doc = \"Transitions a Rust library crate to the `compilation_mode=opt`.\",\n )\n\ntransition_alias_dbg = _transition_alias_rule(\"dbg\")\ntransition_alias_fastbuild = _transition_alias_rule(\"fastbuild\")\ntransition_alias_opt = _transition_alias_rule(\"opt\")\n", - "defs.bzl": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\"\"\"\n# `crates_repository` API\n\n- [aliases](#aliases)\n- [crate_deps](#crate_deps)\n- [all_crate_deps](#all_crate_deps)\n- [crate_repositories](#crate_repositories)\n\n\"\"\"\n\nload(\"@bazel_tools//tools/build_defs/repo:git.bzl\", \"new_git_repository\")\nload(\"@bazel_tools//tools/build_defs/repo:http.bzl\", \"http_archive\")\nload(\"@bazel_tools//tools/build_defs/repo:utils.bzl\", \"maybe\")\nload(\"@bazel_skylib//lib:selects.bzl\", \"selects\")\nload(\"@rules_rust//crate_universe/private:local_crate_mirror.bzl\", \"local_crate_mirror\")\n\n###############################################################################\n# MACROS API\n###############################################################################\n\n# An identifier that represent common dependencies (unconditional).\n_COMMON_CONDITION = \"\"\n\ndef _flatten_dependency_maps(all_dependency_maps):\n \"\"\"Flatten a list of dependency maps into one dictionary.\n\n Dependency maps have the following structure:\n\n ```python\n DEPENDENCIES_MAP = {\n # The first key in the map is a Bazel package\n # name of the workspace this file is defined in.\n \"workspace_member_package\": {\n\n # Not all dependencies are supported for all platforms.\n # the condition key is the condition required to be true\n # on the host platform.\n \"condition\": {\n\n # An alias to a crate target. # The label of the crate target the\n # Aliases are only crate names. # package name refers to.\n \"package_name\": \"@full//:label\",\n }\n }\n }\n ```\n\n Args:\n all_dependency_maps (list): A list of dicts as described above\n\n Returns:\n dict: A dictionary as described above\n \"\"\"\n dependencies = {}\n\n for workspace_deps_map in all_dependency_maps:\n for pkg_name, conditional_deps_map in workspace_deps_map.items():\n if pkg_name not in dependencies:\n non_frozen_map = dict()\n for key, values in conditional_deps_map.items():\n non_frozen_map.update({key: dict(values.items())})\n dependencies.setdefault(pkg_name, non_frozen_map)\n continue\n\n for condition, deps_map in conditional_deps_map.items():\n # If the condition has not been recorded, do so and continue\n if condition not in dependencies[pkg_name]:\n dependencies[pkg_name].setdefault(condition, dict(deps_map.items()))\n continue\n\n # Alert on any miss-matched dependencies\n inconsistent_entries = []\n for crate_name, crate_label in deps_map.items():\n existing = dependencies[pkg_name][condition].get(crate_name)\n if existing and existing != crate_label:\n inconsistent_entries.append((crate_name, existing, crate_label))\n dependencies[pkg_name][condition].update({crate_name: crate_label})\n\n return dependencies\n\ndef crate_deps(deps, package_name = None):\n \"\"\"Finds the fully qualified label of the requested crates for the package where this macro is called.\n\n Args:\n deps (list): The desired list of crate targets.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()`.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if not deps:\n return []\n\n if package_name == None:\n package_name = native.package_name()\n\n # Join both sets of dependencies\n dependencies = _flatten_dependency_maps([\n _NORMAL_DEPENDENCIES,\n _NORMAL_DEV_DEPENDENCIES,\n _PROC_MACRO_DEPENDENCIES,\n _PROC_MACRO_DEV_DEPENDENCIES,\n _BUILD_DEPENDENCIES,\n _BUILD_PROC_MACRO_DEPENDENCIES,\n ]).pop(package_name, {})\n\n # Combine all conditional packages so we can easily index over a flat list\n # TODO: Perhaps this should actually return select statements and maintain\n # the conditionals of the dependencies\n flat_deps = {}\n for deps_set in dependencies.values():\n for crate_name, crate_label in deps_set.items():\n flat_deps.update({crate_name: crate_label})\n\n missing_crates = []\n crate_targets = []\n for crate_target in deps:\n if crate_target not in flat_deps:\n missing_crates.append(crate_target)\n else:\n crate_targets.append(flat_deps[crate_target])\n\n if missing_crates:\n fail(\"Could not find crates `{}` among dependencies of `{}`. Available dependencies were `{}`\".format(\n missing_crates,\n package_name,\n dependencies,\n ))\n\n return crate_targets\n\ndef all_crate_deps(\n normal = False, \n normal_dev = False, \n proc_macro = False, \n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Finds the fully qualified label of all requested direct crate dependencies \\\n for the package where this macro is called.\n\n If no parameters are set, all normal dependencies are returned. Setting any one flag will\n otherwise impact the contents of the returned list.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list.\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n list: A list of labels to generated rust targets (str)\n \"\"\"\n\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_dependency_maps = []\n if normal:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n if normal_dev:\n all_dependency_maps.append(_NORMAL_DEV_DEPENDENCIES)\n if proc_macro:\n all_dependency_maps.append(_PROC_MACRO_DEPENDENCIES)\n if proc_macro_dev:\n all_dependency_maps.append(_PROC_MACRO_DEV_DEPENDENCIES)\n if build:\n all_dependency_maps.append(_BUILD_DEPENDENCIES)\n if build_proc_macro:\n all_dependency_maps.append(_BUILD_PROC_MACRO_DEPENDENCIES)\n\n # Default to always using normal dependencies\n if not all_dependency_maps:\n all_dependency_maps.append(_NORMAL_DEPENDENCIES)\n\n dependencies = _flatten_dependency_maps(all_dependency_maps).pop(package_name, None)\n\n if not dependencies:\n if dependencies == None:\n fail(\"Tried to get all_crate_deps for package \" + package_name + \" but that package had no Cargo.toml file\")\n else:\n return []\n\n crate_deps = list(dependencies.pop(_COMMON_CONDITION, {}).values())\n for condition, deps in dependencies.items():\n crate_deps += selects.with_or({\n tuple(_CONDITIONS[condition]): deps.values(),\n \"//conditions:default\": [],\n })\n\n return crate_deps\n\ndef aliases(\n normal = False,\n normal_dev = False,\n proc_macro = False,\n proc_macro_dev = False,\n build = False,\n build_proc_macro = False,\n package_name = None):\n \"\"\"Produces a map of Crate alias names to their original label\n\n If no dependency kinds are specified, `normal` and `proc_macro` are used by default.\n Setting any one flag will otherwise determine the contents of the returned dict.\n\n Args:\n normal (bool, optional): If True, normal dependencies are included in the\n output list.\n normal_dev (bool, optional): If True, normal dev dependencies will be\n included in the output list..\n proc_macro (bool, optional): If True, proc_macro dependencies are included\n in the output list.\n proc_macro_dev (bool, optional): If True, dev proc_macro dependencies are\n included in the output list.\n build (bool, optional): If True, build dependencies are included\n in the output list.\n build_proc_macro (bool, optional): If True, build proc_macro dependencies are\n included in the output list.\n package_name (str, optional): The package name of the set of dependencies to look up.\n Defaults to `native.package_name()` when unset.\n\n Returns:\n dict: The aliases of all associated packages\n \"\"\"\n if package_name == None:\n package_name = native.package_name()\n\n # Determine the relevant maps to use\n all_aliases_maps = []\n if normal:\n all_aliases_maps.append(_NORMAL_ALIASES)\n if normal_dev:\n all_aliases_maps.append(_NORMAL_DEV_ALIASES)\n if proc_macro:\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n if proc_macro_dev:\n all_aliases_maps.append(_PROC_MACRO_DEV_ALIASES)\n if build:\n all_aliases_maps.append(_BUILD_ALIASES)\n if build_proc_macro:\n all_aliases_maps.append(_BUILD_PROC_MACRO_ALIASES)\n\n # Default to always using normal aliases\n if not all_aliases_maps:\n all_aliases_maps.append(_NORMAL_ALIASES)\n all_aliases_maps.append(_PROC_MACRO_ALIASES)\n\n aliases = _flatten_dependency_maps(all_aliases_maps).pop(package_name, None)\n\n if not aliases:\n return dict()\n\n common_items = aliases.pop(_COMMON_CONDITION, {}).items()\n\n # If there are only common items in the dictionary, immediately return them\n if not len(aliases.keys()) == 1:\n return dict(common_items)\n\n # Build a single select statement where each conditional has accounted for the\n # common set of aliases.\n crate_aliases = {\"//conditions:default\": dict(common_items)}\n for condition, deps in aliases.items():\n condition_triples = _CONDITIONS[condition]\n for triple in condition_triples:\n if triple in crate_aliases:\n crate_aliases[triple].update(deps)\n else:\n crate_aliases.update({triple: dict(deps.items() + common_items)})\n\n return select(crate_aliases)\n\n###############################################################################\n# WORKSPACE MEMBER DEPS AND ALIASES\n###############################################################################\n\n_NORMAL_DEPENDENCIES = {\n \"\": {\n _COMMON_CONDITION: {\n \"googletest\": Label(\"@crates//:googletest-0.14.3\"),\n \"linkme\": Label(\"@crates//:linkme-0.3.36\"),\n \"quote\": Label(\"@crates//:quote-1.0.46\"),\n \"syn\": Label(\"@crates//:syn-2.0.118\"),\n },\n },\n}\n\n\n_NORMAL_ALIASES = {\n \"\": {\n _COMMON_CONDITION: {\n },\n },\n}\n\n\n_NORMAL_DEV_DEPENDENCIES = {\n \"\": {\n },\n}\n\n\n_NORMAL_DEV_ALIASES = {\n \"\": {\n },\n}\n\n\n_PROC_MACRO_DEPENDENCIES = {\n \"\": {\n _COMMON_CONDITION: {\n \"paste\": Label(\"@crates//:paste-1.0.15\"),\n },\n },\n}\n\n\n_PROC_MACRO_ALIASES = {\n \"\": {\n },\n}\n\n\n_PROC_MACRO_DEV_DEPENDENCIES = {\n \"\": {\n },\n}\n\n\n_PROC_MACRO_DEV_ALIASES = {\n \"\": {\n },\n}\n\n\n_BUILD_DEPENDENCIES = {\n \"\": {\n },\n}\n\n\n_BUILD_ALIASES = {\n \"\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_DEPENDENCIES = {\n \"\": {\n },\n}\n\n\n_BUILD_PROC_MACRO_ALIASES = {\n \"\": {\n },\n}\n\n\n_CONDITIONS = {\n \"aarch64-apple-darwin\": [\"@rules_rust//rust/platform:aarch64-apple-darwin\"],\n \"aarch64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\"],\n \"wasm32-unknown-unknown\": [\"@rules_rust//rust/platform:wasm32-unknown-unknown\"],\n \"wasm32-wasip1\": [\"@rules_rust//rust/platform:wasm32-wasip1\"],\n \"x86_64-pc-windows-msvc\": [\"@rules_rust//rust/platform:x86_64-pc-windows-msvc\"],\n \"x86_64-unknown-linux-gnu\": [\"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\"],\n \"x86_64-unknown-nixos-gnu\": [\"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\"],\n}\n\n###############################################################################\n\ndef crate_repositories():\n \"\"\"A macro for defining repositories for all generated crates.\n\n Returns:\n A list of repos visible to the module through the module extension.\n \"\"\"\n maybe(\n http_archive,\n name = \"crates__aho-corasick-1.1.4\",\n sha256 = \"ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/aho-corasick/1.1.4/download\"],\n strip_prefix = \"aho-corasick-1.1.4\",\n build_file = Label(\"@crates//crates:BUILD.aho-corasick-1.1.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__autocfg-1.5.1\",\n sha256 = \"f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/autocfg/1.5.1/download\"],\n strip_prefix = \"autocfg-1.5.1\",\n build_file = Label(\"@crates//crates:BUILD.autocfg-1.5.1.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__googletest-0.14.3\",\n sha256 = \"f6b5e2f2b556b7b90297a5a35c8267dd43a537923d2b329beefdba2b4ec19d94\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/googletest/0.14.3/download\"],\n strip_prefix = \"googletest-0.14.3\",\n build_file = Label(\"@crates//crates:BUILD.googletest-0.14.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__googletest_macro-0.14.3\",\n sha256 = \"2ae6abc96141edd26bf5aeec0f119c129c44de3ced09e5073711a02cb74725d0\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/googletest_macro/0.14.3/download\"],\n strip_prefix = \"googletest_macro-0.14.3\",\n build_file = Label(\"@crates//crates:BUILD.googletest_macro-0.14.3.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__linkme-0.3.36\",\n sha256 = \"e83272d46373fb8decca684579ac3e7c8f3d71d4cc3aa693df8759e260ae41cf\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/linkme/0.3.36/download\"],\n strip_prefix = \"linkme-0.3.36\",\n build_file = Label(\"@crates//crates:BUILD.linkme-0.3.36.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__linkme-impl-0.3.36\",\n sha256 = \"32d59e20403c7d08fe62b4376edfe5c7fb2ef1e6b1465379686d0f21c8df444b\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/linkme-impl/0.3.36/download\"],\n strip_prefix = \"linkme-impl-0.3.36\",\n build_file = Label(\"@crates//crates:BUILD.linkme-impl-0.3.36.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__memchr-2.8.2\",\n sha256 = \"88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/memchr/2.8.2/download\"],\n strip_prefix = \"memchr-2.8.2\",\n build_file = Label(\"@crates//crates:BUILD.memchr-2.8.2.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__num-traits-0.2.19\",\n sha256 = \"071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/num-traits/0.2.19/download\"],\n strip_prefix = \"num-traits-0.2.19\",\n build_file = Label(\"@crates//crates:BUILD.num-traits-0.2.19.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__paste-1.0.15\",\n sha256 = \"57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/paste/1.0.15/download\"],\n strip_prefix = \"paste-1.0.15\",\n build_file = Label(\"@crates//crates:BUILD.paste-1.0.15.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__proc-macro2-1.0.106\",\n sha256 = \"8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/proc-macro2/1.0.106/download\"],\n strip_prefix = \"proc-macro2-1.0.106\",\n build_file = Label(\"@crates//crates:BUILD.proc-macro2-1.0.106.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__quote-1.0.46\",\n sha256 = \"dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/quote/1.0.46/download\"],\n strip_prefix = \"quote-1.0.46\",\n build_file = Label(\"@crates//crates:BUILD.quote-1.0.46.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-1.12.4\",\n sha256 = \"f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex/1.12.4/download\"],\n strip_prefix = \"regex-1.12.4\",\n build_file = Label(\"@crates//crates:BUILD.regex-1.12.4.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-automata-0.4.14\",\n sha256 = \"6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-automata/0.4.14/download\"],\n strip_prefix = \"regex-automata-0.4.14\",\n build_file = Label(\"@crates//crates:BUILD.regex-automata-0.4.14.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__regex-syntax-0.8.11\",\n sha256 = \"d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/regex-syntax/0.8.11/download\"],\n strip_prefix = \"regex-syntax-0.8.11\",\n build_file = Label(\"@crates//crates:BUILD.regex-syntax-0.8.11.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__rustversion-1.0.22\",\n sha256 = \"b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/rustversion/1.0.22/download\"],\n strip_prefix = \"rustversion-1.0.22\",\n build_file = Label(\"@crates//crates:BUILD.rustversion-1.0.22.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__syn-2.0.118\",\n sha256 = \"1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/syn/2.0.118/download\"],\n strip_prefix = \"syn-2.0.118\",\n build_file = Label(\"@crates//crates:BUILD.syn-2.0.118.bazel\"),\n )\n\n maybe(\n http_archive,\n name = \"crates__unicode-ident-1.0.24\",\n sha256 = \"e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75\",\n type = \"tar.gz\",\n urls = [\"https://static.crates.io/crates/unicode-ident/1.0.24/download\"],\n strip_prefix = \"unicode-ident-1.0.24\",\n build_file = Label(\"@crates//crates:BUILD.unicode-ident-1.0.24.bazel\"),\n )\n\n return [\n struct(repo=\"crates__googletest-0.14.3\", is_dev_dep = False),\n struct(repo=\"crates__linkme-0.3.36\", is_dev_dep = False),\n struct(repo=\"crates__paste-1.0.15\", is_dev_dep = False),\n struct(repo=\"crates__quote-1.0.46\", is_dev_dep = False),\n struct(repo=\"crates__syn-2.0.118\", is_dev_dep = False),\n ]\n" - } - } - }, - "crates__aho-corasick-1.1.4": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "remote_patch_strip": 1, - "sha256": "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/aho-corasick/1.1.4/download" - ], - "strip_prefix": "aho-corasick-1.1.4", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"aho_corasick\",\n deps = [\n \"@crates__memchr-2.8.2//:memchr\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"perf-literal\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=aho-corasick\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.1.4\",\n)\n" - } - }, - "crates__autocfg-1.5.1": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "remote_patch_strip": 1, - "sha256": "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/autocfg/1.5.1/download" - ], - "strip_prefix": "autocfg-1.5.1", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"autocfg\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2015\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=autocfg\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.5.1\",\n)\n" - } - }, - "crates__googletest-0.14.3": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "remote_patch_strip": 1, - "sha256": "f6b5e2f2b556b7b90297a5a35c8267dd43a537923d2b329beefdba2b4ec19d94", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/googletest/0.14.3/download" - ], - "strip_prefix": "googletest-0.14.3", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"googletest\",\n deps = [\n \"@crates__num-traits-0.2.19//:num_traits\",\n \"@crates__regex-1.12.4//:regex\",\n ],\n proc_macro_deps = [\n \"@crates__googletest_macro-0.14.3//:googletest_macro\",\n \"@crates__rustversion-1.0.22//:rustversion\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=googletest\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.14.3\",\n)\n" - } - }, - "crates__googletest_macro-0.14.3": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "remote_patch_strip": 1, - "sha256": "2ae6abc96141edd26bf5aeec0f119c129c44de3ced09e5073711a02cb74725d0", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/googletest_macro/0.14.3/download" - ], - "strip_prefix": "googletest_macro-0.14.3", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"googletest_macro\",\n deps = [\n \"@crates__proc-macro2-1.0.106//:proc_macro2\",\n \"@crates__quote-1.0.46//:quote\",\n \"@crates__syn-2.0.118//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=googletest_macro\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.14.3\",\n)\n" - } - }, - "crates__linkme-0.3.36": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "remote_patch_strip": 1, - "sha256": "e83272d46373fb8decca684579ac3e7c8f3d71d4cc3aa693df8759e260ae41cf", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/linkme/0.3.36/download" - ], - "strip_prefix": "linkme-0.3.36", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"linkme\",\n deps = [\n \"@crates__linkme-0.3.36//:build_script_build\",\n ],\n proc_macro_deps = [\n \"@crates__linkme-impl-0.3.36//:linkme_impl\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=linkme\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.36\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"linkme\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=linkme\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.3.36\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__linkme-impl-0.3.36": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "remote_patch_strip": 1, - "sha256": "32d59e20403c7d08fe62b4376edfe5c7fb2ef1e6b1465379686d0f21c8df444b", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/linkme-impl/0.3.36/download" - ], - "strip_prefix": "linkme-impl-0.3.36", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"linkme_impl\",\n deps = [\n \"@crates__linkme-impl-0.3.36//:build_script_build\",\n \"@crates__proc-macro2-1.0.106//:proc_macro2\",\n \"@crates__quote-1.0.46//:quote\",\n \"@crates__syn-2.0.118//:syn\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=linkme-impl\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.3.36\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"linkme-impl\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=linkme-impl\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.3.36\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__memchr-2.8.2": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "remote_patch_strip": 1, - "sha256": "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/memchr/2.8.2/download" - ], - "strip_prefix": "memchr-2.8.2", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"memchr\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=memchr\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.8.2\",\n)\n" - } - }, - "crates__num-traits-0.2.19": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "remote_patch_strip": 1, - "sha256": "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/num-traits/0.2.19/download" - ], - "strip_prefix": "num-traits-0.2.19", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"num_traits\",\n deps = [\n \"@crates__num-traits-0.2.19//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=num-traits\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.19\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n deps = [\n \"@crates__autocfg-1.5.1//:autocfg\",\n ],\n edition = \"2021\",\n pkg_name = \"num-traits\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=num-traits\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"0.2.19\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__paste-1.0.15": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "remote_patch_strip": 1, - "sha256": "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/paste/1.0.15/download" - ], - "strip_prefix": "paste-1.0.15", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"paste\",\n deps = [\n \"@crates__paste-1.0.15//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=paste\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.15\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2018\",\n pkg_name = \"paste\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=paste\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.15\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__proc-macro2-1.0.106": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "remote_patch_strip": 1, - "sha256": "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/proc-macro2/1.0.106/download" - ], - "strip_prefix": "proc-macro2-1.0.106", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"proc_macro2\",\n deps = [\n \"@crates__proc-macro2-1.0.106//:build_script_build\",\n \"@crates__unicode-ident-1.0.24//:unicode_ident\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"proc-macro\",\n ] + select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [\n \"default\", # aarch64-apple-darwin\n ],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [\n \"default\", # aarch64-unknown-linux-gnu\n ],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [\n \"default\", # x86_64-pc-windows-msvc\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [\n \"default\", # x86_64-unknown-linux-gnu\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [\n \"default\", # x86_64-unknown-nixos-gnu\n ],\n \"//conditions:default\": [],\n }),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=proc-macro2\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.106\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"proc-macro\",\n ] + select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [\n \"default\", # aarch64-apple-darwin\n ],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [\n \"default\", # aarch64-unknown-linux-gnu\n ],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [\n \"default\", # x86_64-pc-windows-msvc\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [\n \"default\", # x86_64-unknown-linux-gnu\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [\n \"default\", # x86_64-unknown-nixos-gnu\n ],\n \"//conditions:default\": [],\n }),\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"proc-macro2\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=proc-macro2\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.106\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__quote-1.0.46": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "remote_patch_strip": 1, - "sha256": "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/quote/1.0.46/download" - ], - "strip_prefix": "quote-1.0.46", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"quote\",\n deps = [\n \"@crates__proc-macro2-1.0.106//:proc_macro2\",\n \"@crates__quote-1.0.46//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"proc-macro\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=quote\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.46\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"proc-macro\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2021\",\n pkg_name = \"quote\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=quote\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.46\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__regex-1.12.4": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "remote_patch_strip": 1, - "sha256": "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/regex/1.12.4/download" - ], - "strip_prefix": "regex-1.12.4", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"regex\",\n deps = [\n \"@crates__aho-corasick-1.1.4//:aho_corasick\",\n \"@crates__memchr-2.8.2//:memchr\",\n \"@crates__regex-automata-0.4.14//:regex_automata\",\n \"@crates__regex-syntax-0.8.11//:regex_syntax\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"perf\",\n \"perf-backtrack\",\n \"perf-cache\",\n \"perf-dfa\",\n \"perf-inline\",\n \"perf-literal\",\n \"perf-onepass\",\n \"std\",\n \"unicode\",\n \"unicode-age\",\n \"unicode-bool\",\n \"unicode-case\",\n \"unicode-gencat\",\n \"unicode-perl\",\n \"unicode-script\",\n \"unicode-segment\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=regex\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.12.4\",\n)\n" - } - }, - "crates__regex-automata-0.4.14": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "remote_patch_strip": 1, - "sha256": "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/regex-automata/0.4.14/download" - ], - "strip_prefix": "regex-automata-0.4.14", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"regex_automata\",\n deps = [\n \"@crates__aho-corasick-1.1.4//:aho_corasick\",\n \"@crates__memchr-2.8.2//:memchr\",\n \"@crates__regex-syntax-0.8.11//:regex_syntax\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"alloc\",\n \"dfa-onepass\",\n \"hybrid\",\n \"meta\",\n \"nfa-backtrack\",\n \"nfa-pikevm\",\n \"nfa-thompson\",\n \"perf-inline\",\n \"perf-literal\",\n \"perf-literal-multisubstring\",\n \"perf-literal-substring\",\n \"std\",\n \"syntax\",\n \"unicode\",\n \"unicode-age\",\n \"unicode-bool\",\n \"unicode-case\",\n \"unicode-gencat\",\n \"unicode-perl\",\n \"unicode-script\",\n \"unicode-segment\",\n \"unicode-word-boundary\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=regex-automata\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.4.14\",\n)\n" - } - }, - "crates__regex-syntax-0.8.11": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "remote_patch_strip": 1, - "sha256": "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/regex-syntax/0.8.11/download" - ], - "strip_prefix": "regex-syntax-0.8.11", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"regex_syntax\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"std\",\n \"unicode\",\n \"unicode-age\",\n \"unicode-bool\",\n \"unicode-case\",\n \"unicode-gencat\",\n \"unicode-perl\",\n \"unicode-script\",\n \"unicode-segment\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=regex-syntax\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.8.11\",\n)\n" - } - }, - "crates__rustversion-1.0.22": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "remote_patch_strip": 1, - "sha256": "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/rustversion/1.0.22/download" - ], - "strip_prefix": "rustversion-1.0.22", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_proc_macro\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_proc_macro(\n name = \"rustversion\",\n deps = [\n \"@crates__rustversion-1.0.22//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rustversion\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.22\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_name = \"build_script_build\",\n crate_root = \"build/build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n edition = \"2018\",\n pkg_name = \"rustversion\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=rustversion\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n version = \"1.0.22\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" - } - }, - "crates__syn-2.0.118": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "remote_patch_strip": 1, - "sha256": "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/syn/2.0.118/download" - ], - "strip_prefix": "syn-2.0.118", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"syn\",\n deps = [\n \"@crates__proc-macro2-1.0.106//:proc_macro2\",\n \"@crates__quote-1.0.46//:quote\",\n \"@crates__unicode-ident-1.0.24//:unicode_ident\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"clone-impls\",\n \"default\",\n \"derive\",\n \"parsing\",\n \"printing\",\n \"proc-macro\",\n ] + select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [\n \"extra-traits\", # aarch64-apple-darwin\n \"full\", # aarch64-apple-darwin\n ],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [\n \"extra-traits\", # aarch64-unknown-linux-gnu\n \"full\", # aarch64-unknown-linux-gnu\n ],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [\n \"extra-traits\", # x86_64-pc-windows-msvc\n \"full\", # x86_64-pc-windows-msvc\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [\n \"extra-traits\", # x86_64-unknown-linux-gnu\n \"full\", # x86_64-unknown-linux-gnu\n ],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [\n \"extra-traits\", # x86_64-unknown-nixos-gnu\n \"full\", # x86_64-unknown-nixos-gnu\n ],\n \"//conditions:default\": [],\n }),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=syn\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"2.0.118\",\n)\n" - } - }, - "crates__unicode-ident-1.0.24": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "remote_patch_strip": 1, - "sha256": "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75", - "type": "tar.gz", - "urls": [ - "https://static.crates.io/crates/unicode-ident/1.0.24/download" - ], - "strip_prefix": "unicode-ident-1.0.24", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'protobuf'\n###############################################################################\n\nload(\"@rules_rust//cargo:defs.bzl\", \"cargo_toml_env_vars\")\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"unicode_ident\",\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_root = \"src/lib.rs\",\n edition = \"2021\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=unicode-ident\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@rules_rust//rust/platform:aarch64-apple-darwin\": [],\n \"@rules_rust//rust/platform:aarch64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:wasm32-unknown-unknown\": [],\n \"@rules_rust//rust/platform:wasm32-wasip1\": [],\n \"@rules_rust//rust/platform:x86_64-pc-windows-msvc\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-linux-gnu\": [],\n \"@rules_rust//rust/platform:x86_64-unknown-nixos-gnu\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"1.0.24\",\n)\n" - } - } - } - } - }, - "@@rules_rust+//crate_universe/private:internal_extensions.bzl%cu_nr": { - "general": { - "bzlTransitiveDigest": "R1Y9HkqS/IoqUbr25FqrM9MNwlYxdxjkDgGabT4WKKU=", - "usagesDigest": "tG3p3Nb5XxC7vWY/bcKdb//g0HoAxpxxH3F5/jBVlk4=", - "recordedInputs": [ - "REPO_MAPPING:bazel_features+,bazel_features_globals bazel_features++version_extension+bazel_features_globals", - "REPO_MAPPING:bazel_features+,bazel_features_version bazel_features++version_extension+bazel_features_version", - "REPO_MAPPING:rules_cc+,bazel_skylib bazel_skylib+", - "REPO_MAPPING:rules_cc+,bazel_tools bazel_tools", - "REPO_MAPPING:rules_cc+,cc_compatibility_proxy rules_cc++compatibility_proxy+cc_compatibility_proxy", - "REPO_MAPPING:rules_cc+,platforms platforms", - "REPO_MAPPING:rules_cc+,rules_cc rules_cc+", - "REPO_MAPPING:rules_cc++compatibility_proxy+cc_compatibility_proxy,rules_cc rules_cc+", - "REPO_MAPPING:rules_rust+,bazel_features bazel_features+", - "REPO_MAPPING:rules_rust+,bazel_skylib bazel_skylib+", - "REPO_MAPPING:rules_rust+,bazel_tools bazel_tools", - "REPO_MAPPING:rules_rust+,cui rules_rust++cu+cui", - "REPO_MAPPING:rules_rust+,rrc rules_rust++i2+rrc", - "REPO_MAPPING:rules_rust+,rules_cc rules_cc+", - "REPO_MAPPING:rules_rust+,rules_rust rules_rust+" - ], - "generatedRepoSpecs": { - "cargo_bazel_bootstrap": { - "repoRuleId": "@@rules_rust+//cargo/private:cargo_bootstrap.bzl%cargo_bootstrap_repository", - "attributes": { - "srcs": [ - "@@rules_rust+//crate_universe:src/api.rs", - "@@rules_rust+//crate_universe:src/api/lockfile.rs", - "@@rules_rust+//crate_universe:src/cli.rs", - "@@rules_rust+//crate_universe:src/cli/generate.rs", - "@@rules_rust+//crate_universe:src/cli/query.rs", - "@@rules_rust+//crate_universe:src/cli/render.rs", - "@@rules_rust+//crate_universe:src/cli/splice.rs", - "@@rules_rust+//crate_universe:src/cli/vendor.rs", - "@@rules_rust+//crate_universe:src/config.rs", - "@@rules_rust+//crate_universe:src/context.rs", - "@@rules_rust+//crate_universe:src/context/crate_context.rs", - "@@rules_rust+//crate_universe:src/context/platforms.rs", - "@@rules_rust+//crate_universe:src/lib.rs", - "@@rules_rust+//crate_universe:src/lockfile.rs", - "@@rules_rust+//crate_universe:src/main.rs", - "@@rules_rust+//crate_universe:src/metadata.rs", - "@@rules_rust+//crate_universe:src/metadata/cargo_bin.rs", - "@@rules_rust+//crate_universe:src/metadata/cargo_tree_resolver.rs", - "@@rules_rust+//crate_universe:src/metadata/cargo_tree_rustc_wrapper.bat", - "@@rules_rust+//crate_universe:src/metadata/cargo_tree_rustc_wrapper.sh", - "@@rules_rust+//crate_universe:src/metadata/dependency.rs", - "@@rules_rust+//crate_universe:src/metadata/metadata_annotation.rs", - "@@rules_rust+//crate_universe:src/rendering.rs", - "@@rules_rust+//crate_universe:src/rendering/template_engine.rs", - "@@rules_rust+//crate_universe:src/rendering/templates/module_bzl.j2", - "@@rules_rust+//crate_universe:src/rendering/templates/partials/header.j2", - "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/aliases_map.j2", - "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/deps_map.j2", - "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/repo_git.j2", - "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/repo_http.j2", - "@@rules_rust+//crate_universe:src/rendering/templates/vendor_module.j2", - "@@rules_rust+//crate_universe:src/rendering/verbatim/alias_rules.bzl", - "@@rules_rust+//crate_universe:src/select.rs", - "@@rules_rust+//crate_universe:src/splicing.rs", - "@@rules_rust+//crate_universe:src/splicing/cargo_config.rs", - "@@rules_rust+//crate_universe:src/splicing/crate_index_lookup.rs", - "@@rules_rust+//crate_universe:src/splicing/splicer.rs", - "@@rules_rust+//crate_universe:src/test.rs", - "@@rules_rust+//crate_universe:src/utils.rs", - "@@rules_rust+//crate_universe:src/utils/starlark.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/glob.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/label.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/select.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/select_dict.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/select_list.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/select_scalar.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/select_set.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/serialize.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/target_compatible_with.rs", - "@@rules_rust+//crate_universe:src/utils/symlink.rs", - "@@rules_rust+//crate_universe:src/utils/target_triple.rs" - ], - "binary": "cargo-bazel", - "cargo_lockfile": "@@rules_rust+//crate_universe:Cargo.lock", - "cargo_toml": "@@rules_rust+//crate_universe:Cargo.toml", - "version": "1.93.1", - "timeout": 900, - "rust_toolchain_cargo_template": "@rust_host_tools//:bin/{tool}", - "rust_toolchain_rustc_template": "@rust_host_tools//:bin/{tool}", - "compressed_windows_toolchain_names": false - } - } - }, - "moduleExtensionMetadata": { - "explicitRootModuleDirectDeps": [ - "cargo_bazel_bootstrap" - ], - "explicitRootModuleDirectDevDeps": [], - "useAllRepos": "NO", - "reproducible": false - } - } - }, "@@rules_zig+//zig:extensions.bzl%zig": { "general": { - "bzlTransitiveDigest": "VY2CKW7wm6B3wCpZq7W6z3hCnHiiJlXtB4JpuB3OPD8=", + "bzlTransitiveDigest": "hPjDj28qKZsSFAE45ZtQPcgDkeRXAcwxiuAQyeEF7ng=", "usagesDigest": "FDLiNUluzC0WBatQ/5WFpxzyQP4ubA/DUgTwOzYLGw4=", "recordedInputs": [ "REPO_MAPPING:rules_zig+,bazel_skylib bazel_skylib+", @@ -1833,7 +1247,7 @@ }, "@@rules_zig+//zig/zls:extensions.bzl%zls": { "general": { - "bzlTransitiveDigest": "ZuceGxE17jROQ+kQkaPqIcK7gMKGb56I97Aq1/Dubdc=", + "bzlTransitiveDigest": "8UokkNRK4sShFNMJ3aDabiCS4Q/ldgChV/NYNszfmCg=", "usagesDigest": "kOrP52crJGijpHNzo4S7msXvbGpfmzvkE4tuhpRWTsI=", "recordedInputs": [ "REPO_MAPPING:rules_zig+,bazel_skylib bazel_skylib+", @@ -2731,5 +2145,6 @@ ] } } - } + }, + "factsVersions": {} } diff --git a/third_party/zls/BUILD.bazel b/third_party/zls/BUILD.bazel index 964fe87b..d53e20be 100644 --- a/third_party/zls/BUILD.bazel +++ b/third_party/zls/BUILD.bazel @@ -25,7 +25,7 @@ zig_library( zls_toolchain( name = "zls", bin = "@zls//:zls", - zls_version = "0.17.0-dev+eab2be0fd744", + zls_version = "0.17.0", ) toolchain( From e9685a36626a61098a286adecdadab4efb9bc3cb Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Tue, 20 Oct 2026 11:49:41 +0200 Subject: [PATCH 802/881] zml/bazel: simplify patchelf execution and fix RPATH removal (#916) The `@patchelf` version 0.18.0 is more capable and can now construct something like ```bash patchelf \ --add-needed somelib.so \ --force-rpath \ --set-rpath '/home/kevin/sdk/lib' \ --rename-dynamic-symbols patched_tcc.symbols \ --output patched_tcc/furiosa-tcc \ somebinary ``` So now we do not need to spawn a shell and do all the cp, chmod, and the successive patchelf command Before ```bash remove_rpath = ["some/directory"] ``` `--remove-rpath` is a flag that `Removes the entire embedded search path` and not an individual directory will contruct something like `patchelf --remove-rpath some/directory patched.so` but `--remove-rpath` do not take arguments Switch to `remove_rpath = True` to ensure something like `patchelf --remove-rpath --output patched.so original.so` the `--remove-rpath` is now used just after the patchelf to ensure correct usage flag placement. --- bazel/patchelf.bzl | 108 +++++++++++++++++++++++---------------------- 1 file changed, 55 insertions(+), 53 deletions(-) diff --git a/bazel/patchelf.bzl b/bazel/patchelf.bzl index 50691b82..bafaa6ec 100644 --- a/bazel/patchelf.bzl +++ b/bazel/patchelf.bzl @@ -1,76 +1,78 @@ def _patchelf_impl(ctx): - output_name = ctx.file.src.basename - if ctx.attr.soname: - output_name = ctx.attr.soname - output = ctx.actions.declare_file("{}/{}".format(ctx.attr.name, output_name)) + output_name = ctx.attr.soname or ctx.file.src.basename + output = ctx.actions.declare_file("{}/{}".format(ctx.label.name, output_name)) - commands = [ - "set -e", - 'cp -f "$2" "$3"', - 'chmod +w "$3"', + inputs = [ctx.file.src] + args = ctx.actions.args() + + # --- soname ----------------------------------------------------------- + if ctx.attr.soname: + args.add("--set-soname", ctx.attr.soname) + + # --- DT_NEEDED -------------------------------------------------------- + args.add_all(ctx.attr.remove_needed, before_each = "--remove-needed") + args.add_all(ctx.attr.add_needed, before_each = "--add-needed") + for old, new in ctx.attr.replace_needed.items(): + args.add("--replace-needed") + args.add(old) + args.add(new) + + # --- RPATH / RUNPATH -------------------------------------------------- + # patchelf only accepts one rpath operation per invocation. + rpath_ops = [ + bool(ctx.attr.set_rpath), + bool(ctx.attr.add_rpath), + ctx.attr.remove_rpath, ] - - if ctx.attr.soname: - commands.append(""" "$1" --set-soname '{}' "$3" """.format(ctx.attr.soname)) - if ctx.attr.remove_needed: - for v in ctx.attr.remove_needed: - commands.append(""" "$1" --remove-needed '{}' "$3" """.format(v)) - if ctx.attr.add_needed: - for v in ctx.attr.add_needed: - commands.append(""" "$1" --add-needed '{}' "$3" """.format(v)) - if ctx.attr.replace_needed: - for k, v in ctx.attr.replace_needed.items(): - commands.append(""" "$1" --replace-needed '{}' '{}' "$3" """.format(k, v)) + if len([op for op in rpath_ops if op]) > 1: + fail("patchelf: set_rpath, add_rpath and remove_rpath are mutually exclusive") if ctx.attr.set_rpath: - commands.append(""" "$1" --set-rpath '{}' --force-rpath "$3" """.format(ctx.attr.set_rpath)) - if ctx.attr.add_rpath: - for path in ctx.attr.add_rpath: - commands.append(""" "$1" --add-rpath '{}' --force-rpath "$3" """.format(path)) - if ctx.attr.remove_rpath: - for path in ctx.attr.remove_rpath: - commands.append(""" "$1" --remove-rpath '{}' "$3" """.format(path)) + args.add("--force-rpath") + args.add("--set-rpath", ctx.attr.set_rpath) + elif ctx.attr.add_rpath: + args.add("--force-rpath") + args.add("--add-rpath", ":".join(ctx.attr.add_rpath)) + elif ctx.attr.remove_rpath: + args.add("--remove-rpath") - renamed_syms = ctx.actions.declare_file("{}.rename.txt".format(ctx.label.name)) + # --- dynamic symbol renames ------------------------------------------ if ctx.attr.rename_dynamic_symbols: - content = "\n".join([ - "{} {}".format(k, v) - for k, v in ctx.attr.rename_dynamic_symbols.items() - ]) - ctx.actions.write(renamed_syms, content) - commands.append(""" "$1" --rename-dynamic-symbols '{}' "$3" """.format(renamed_syms.path)) - else: - ctx.actions.write(renamed_syms, "") + symbols = ctx.actions.declare_file(ctx.label.name + ".symbols") + ctx.actions.write(symbols, "".join([ + "{} {}\n".format(old, new) + for old, new in ctx.attr.rename_dynamic_symbols.items() + ])) + inputs.append(symbols) + args.add("--rename-dynamic-symbols", symbols) - ctx.actions.run_shell( - inputs = [ctx.file.src, renamed_syms], + args.add("--output", output) + args.add(ctx.file.src) + + ctx.actions.run( + executable = ctx.executable._patchelf, + inputs = inputs, outputs = [output], - arguments = [ctx.executable._patchelf.path, ctx.file.src.path, output.path], - command = "\n".join(commands), - tools = [ctx.executable._patchelf], + arguments = [args], + mnemonic = "Patchelf", + progress_message = "Patching ELF %{output}", ) - - return [ - DefaultInfo( - files = depset([output]), - ), - ] + return [DefaultInfo(files = depset([output]))] patchelf = rule( implementation = _patchelf_impl, attrs = { "src": attr.label(allow_single_file = True, mandatory = True), - "soname": attr.string(), + "soname": attr.string(doc = "Set DT_SONAME; also used as the output file name."), "add_needed": attr.string_list(), "remove_needed": attr.string_list(), - "replace_needed": attr.string_dict(), - "rename_dynamic_symbols": attr.string_dict(), + "replace_needed": attr.string_dict(doc = "old -> new DT_NEEDED replacements."), + "rename_dynamic_symbols": attr.string_dict(doc = "old -> new symbol renames."), "set_rpath": attr.string(), - "add_rpath": attr.string_list(), - "remove_rpath": attr.string_list(), + "add_rpath": attr.string_list(doc = "Entries appended to the existing RPATH (joined with ':')."), + "remove_rpath": attr.bool(default = False, doc = "Strip the DT_RPATH/DT_RUNPATH entry entirely."), "_patchelf": attr.label( default = "@patchelf", - allow_single_file = True, executable = True, cfg = "exec", ), From b238e5e6e74d87e73549f4004f3858688510d5db Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 20 Oct 2026 12:51:06 +0200 Subject: [PATCH 803/881] ci: update runs-on images and runners manifest (#968) Needed configuration for an upcoming CI change. Also remove not used anymore bzlcache over S3 since we use BuildBuddy. --- .github/runs-on.yml | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/.github/runs-on.yml b/.github/runs-on.yml index 384121f7..aefeccc9 100644 --- a/.github/runs-on.yml +++ b/.github/runs-on.yml @@ -3,9 +3,30 @@ images: platform: "linux" arch: "amd64" ami: "ami-04a92520784b93e73" - preinstall: | - #!/bin/bash - wget https://github.com/jbdalido/bazel-remote/releases/download/test/bazel-remote -O /usr/local/bin/bzlcache - chmod +x /usr/local/bin/bzlcache - echo "Run bazel cache" - systemd-run --property=Restart=always /usr/local/bin/bzlcache --max_size=1000 --grpc_address 127.0.0.1:15501 --dir=/tmp/cache --s3.region eu-west-3 --s3.bucket runs-on-s3bucketcache-jycgicewbssh --s3.auth_method iam_role --s3.prefix cache --s3.endpoint=s3.eu-west-3.amazonaws.com --storage_mode uncompressed + + ubuntu24-arm64: + platform: "linux" + arch: "arm64" + ami: "ami-035c36ba717c3d79f" + +runners: + large-runner-amd64: + cpu: [16] + family: ["c8", "c7"] + image: ubuntu24-amd64 + volume: 80gb + extras: ["s3-cache"] + + large-runner-arm64: + cpu: [16] + family: ["c8g", "c7g"] + image: ubuntu24-arm64 + volume: 80gb + extras: ["s3-cache"] + + small-runner-amd64: + cpu: [4] + family: ["c8", "c7"] + image: ubuntu24-amd64 + volume: 80gb + extras: ["s3-cache"] From 6cd00a0d00c72a516638f88d55dcf3bf4538429d Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 20 Oct 2026 14:06:49 +0200 Subject: [PATCH 804/881] ci: add supported platforms with bumped and audited deps (#967) --- .github/workflows/ci.yaml | 150 +++++++++++++++++++++++++---------- platforms/neuron/BUILD.bazel | 8 ++ 2 files changed, 117 insertions(+), 41 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bace1a65..31bc46e8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -13,67 +13,92 @@ concurrency: jobs: ci: - name: ci-${{ matrix.target }} + name: ci-${{ matrix.target }}-${{ matrix.arch }} strategy: fail-fast: false matrix: include: - target: cpu - bazel_opts: "" + arch: amd64 + bazel_common_opts: "" run_tests: true runner_labels: - - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-cpu - - runner=16cpu-linux-x64 - - family=c8+c7 - - image=ubuntu24-amd64 - - volume=80gb - - extras=s3-cache + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-cpu-amd64 + - runner=large-runner-amd64 - target: neuron - bazel_opts: "--@zml//platforms:neuron=true --@zml//platforms:cpu=false" + arch: amd64 + bazel_common_opts: "--@zml//platforms:neuron=true --@zml//platforms:cpu=false" run_tests: false runner_labels: - - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-neuron - - runner=16cpu-linux-x64 - - family=c8+c7 - - image=ubuntu24-amd64 - - volume=80gb - - extras=s3-cache + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-neuron-amd64 + - runner=large-runner-amd64 - target: tpu - bazel_opts: "--@zml//platforms:tpu=true --@zml//platforms:cpu=false" + arch: amd64 + bazel_common_opts: "--@zml//platforms:tpu=true --@zml//platforms:cpu=false" run_tests: false runner_labels: - - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-tpu - - runner=16cpu-linux-x64 - - family=c8+c7 - - image=ubuntu24-amd64 - - volume=80gb - - extras=s3-cache + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-tpu-amd64 + - runner=large-runner-amd64 - target: cuda - bazel_opts: "--@zml//platforms:cuda=true --@zml//platforms:cpu=false" + arch: amd64 + bazel_common_opts: "--@zml//platforms:cuda=true --@zml//platforms:cpu=false" run_tests: true - runner_labels: ["self-hosted", "platform=cuda"] + runner_labels: ["self-hosted", "linux", "x64", "platform=cuda"] + + - target: cuda + arch: arm64 + bazel_common_opts: "--@zml//platforms:cuda=true --@zml//platforms:cpu=false" + run_tests: false + runner_labels: + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-cuda-arm64 + - runner=large-runner-arm64 - target: rocm - bazel_opts: "--@zml//platforms:rocm=true --@zml//platforms:cpu=false" + arch: amd64 + bazel_common_opts: "--@zml//platforms:rocm=true --@zml//platforms:cpu=false" run_tests: true - runner_labels: ["self-hosted", "platform=rocm"] + runner_labels: ["self-hosted", "linux", "x64", "platform=rocm"] - # Todo : add missing platforms like Metal / oneAPI / Furiosa ... + - target: oneapi + arch: amd64 + bazel_common_opts: "--@zml//platforms:oneapi=true --@zml//platforms:cpu=false" + run_tests: false + runner_labels: + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-oneapi-amd64 + - runner=large-runner-amd64 + + - target: metal + arch: arm64 + bazel_common_opts: "--@zml//platforms:metal=true --@zml//platforms:cpu=false" + bazel_build_opts: "--platforms=@zml//platforms:macos_arm64" + run_tests: false + runner_labels: + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-metal-arm64 + - runner=large-runner-amd64 + + - target: furiosa + arch: amd64 + bazel_common_opts: "--@zml//platforms:furiosa=true --@zml//platforms:cpu=false" + run_tests: false + runner_labels: + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-furiosa-amd64 + - runner=large-runner-amd64 runs-on: ${{ matrix.runner_labels }} steps: - - uses: runs-on/action@v2.1.0 + - uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0 - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Ensure self hosted cache permissions + if: ${{ contains(matrix.runner_labels, 'self-hosted') }} run: mkdir -p /tmp/zig-cache && mkdir -p /tmp/zml && sudo chmod -R 777 /tmp/zig-cache && sudo chmod -R 777 /tmp/zml - name: Setup Bazel - uses: bazel-contrib/setup-bazel@0.19.0 + uses: bazel-contrib/setup-bazel@104e3eea9447cdaa4b00d0fb46191a3985d26ab3 # 0.19.1 with: bazelisk-version: 1.29.0 bazelisk-cache: true @@ -89,27 +114,70 @@ jobs: common --strategy=TestRunner=local ${{ format('common:remote --remote_header=x-buildbuddy-api-key={0}', secrets.BUILDBUDDY_CI_API_KEY) || '' }} + - name: Capture self hosted storage usage before + id: storage_before + if: ${{ contains(matrix.runner_labels, 'self-hosted') }} + continue-on-error: true + shell: bash + run: &measure-self-hosted-storage | + output_base=$(bazel info output_base) || output_base="" + bazel_size="unavailable" + if [[ -n "$output_base" ]]; then + bazel_size=$(du -sh --si -- "$output_base" | cut -f1) || bazel_size="unavailable" + fi + disks=$(df -H --local --output=source,size,used,avail,pcent,target | + awk '$1 ~ /^\/dev\// && $1 !~ /^\/dev\/loop/ && !seen[$1]++ { + printf "%s%s: %s / %s used (%s), %s free", separator, $6, $3, $2, $5, $4 + separator = "; " + } END {if (separator == "") printf "none"}') || disks="unavailable" + { + printf 'output_base=%s\n' "$output_base" + printf 'bazel_size=%s\n' "$bazel_size" + printf 'disks=%s\n' "$disks" + } >> "$GITHUB_OUTPUT" + - run: bazel mod graph - - run: bazel query --output=graph ${{ matrix.bazel_opts }} //... - - run: bazel cquery ${{ matrix.bazel_opts }} //... - - run: bazel build ${{ matrix.bazel_opts }} //... + - run: bazel query ${{ matrix.bazel_common_opts }} --output=graph //... + - run: bazel cquery ${{ matrix.bazel_common_opts }} ${{ matrix.bazel_build_opts }} //... + - run: bazel build ${{ matrix.bazel_common_opts }} ${{ matrix.bazel_build_opts }} //... - if: ${{ matrix.run_tests }} - run: bazel test ${{ matrix.bazel_opts }} //... + run: bazel test ${{ matrix.bazel_common_opts }} ${{ matrix.bazel_build_opts }} //... + + - name: Capture self hosted storage usage after + id: storage_after + if: ${{ always() && contains(matrix.runner_labels, 'self-hosted') }} + continue-on-error: true + shell: bash + run: *measure-self-hosted-storage + + - name: Report self hosted storage usage + if: ${{ always() && contains(matrix.runner_labels, 'self-hosted') }} + env: + STORAGE_JOB: ${{ github.job }}-${{ matrix.target }}-${{ matrix.arch }} + OUTPUT_BASE: ${{ steps.storage_after.outputs.output_base || steps.storage_before.outputs.output_base || 'unavailable' }} + SIZE_BEFORE: ${{ steps.storage_before.outputs.bazel_size || 'unavailable' }} + SIZE_AFTER: ${{ steps.storage_after.outputs.bazel_size || 'unavailable' }} + DISKS_BEFORE: ${{ steps.storage_before.outputs.disks || 'unavailable' }} + DISKS_AFTER: ${{ steps.storage_after.outputs.disks || 'unavailable' }} + run: | + message=$(printf 'Runner: %s\nBazel output base: %s\nBazel directory size: %s -> %s\nDisks before: %s\nDisks after: %s' \ + "$RUNNER_NAME" "$OUTPUT_BASE" "$SIZE_BEFORE" "$SIZE_AFTER" "$DISKS_BEFORE" "$DISKS_AFTER") + message=${message//'%'/'%25'} + message=${message//$'\r'/'%0D'} + message=${message//$'\n'/'%0A'} + printf '::notice title=Disk usage - %s::%s\n' "$STORAGE_JOB" "$message" lint: runs-on: - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-lint - - runner=4cpu-linux-x64 - - image=ubuntu24-amd64 - - volume=80gb - - extras=s3-cache + - runner=small-runner-amd64 steps: - - uses: runs-on/action@v2.1.0 + - uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0 - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Zig - uses: mlugg/setup-zig@v2.2.1 + uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 with: version: 0.17.0 mirror: https://mirror.zml.ai/zig diff --git a/platforms/neuron/BUILD.bazel b/platforms/neuron/BUILD.bazel index d427fd47..ad537191 100644 --- a/platforms/neuron/BUILD.bazel +++ b/platforms/neuron/BUILD.bazel @@ -85,12 +85,20 @@ cc_import( name = "neuron_ncfw", shared_library = "@aws-neuronx-runtime-lib//:libncfw.patchelf", tags = ["manual"], + target_compatible_with = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], ) cc_import( name = "neuron_nrt", shared_library = "@aws-neuronx-runtime-lib//:libnrt.patchelf", tags = ["manual"], + target_compatible_with = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], deps = [":neuron_ncfw"], ) From 886a6b8a972ed8862eb0c6bfc99c62e8ff01cfff Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 20 Oct 2026 16:41:55 +0200 Subject: [PATCH 805/881] zml/kernels/paged_attention: add cute backend (#963) This PR is the ground work that adds a `cute` backend for paged attention. It's not use by default yet because it's a very naive implementation that is far from performant. This PR also includes two targets that are useful when working on the kernels. The first one is `//zml/kernels/paged_attention/cute:correctness_test`. As its name suggest, it's a correctness test that ensures the computation done is correct and returns valid results. It's verified against a CPU reference implementation (that is also checked against a hardcoded truth). It relies on `Case` representing various real-life scenarios. The second one is `//zml/kernels/paged_attention/cute:benchmark`. It's an executable that can be run and will output execution stats for various `Case`. This is the target to use when working on the performance of paged attention cute kernels. --- kernels/cute/README.md | 9 ++ zml/BUILD.bazel | 1 + zml/kernels/paged_attention/cute/BUILD.bazel | 45 ++++++ .../paged_attention/cute/attention.zig | 123 ++++++++++++++++ .../paged_attention/cute/benchmark.zig | 120 ++++++++++++++++ zml/kernels/paged_attention/cute/cases.zig | 82 +++++++++++ .../paged_attention/cute/correctness_test.zig | 65 +++++++++ zml/kernels/paged_attention/cute/cute.zig | 78 ++++++++++ zml/kernels/paged_attention/cute/fixtures.zig | 133 ++++++++++++++++++ .../paged_attention/cute/reference.zig | 126 +++++++++++++++++ .../paged_attention/paged_attention.zig | 17 ++- 11 files changed, 795 insertions(+), 4 deletions(-) create mode 100644 zml/kernels/paged_attention/cute/BUILD.bazel create mode 100644 zml/kernels/paged_attention/cute/attention.zig create mode 100644 zml/kernels/paged_attention/cute/benchmark.zig create mode 100644 zml/kernels/paged_attention/cute/cases.zig create mode 100644 zml/kernels/paged_attention/cute/correctness_test.zig create mode 100644 zml/kernels/paged_attention/cute/cute.zig create mode 100644 zml/kernels/paged_attention/cute/fixtures.zig create mode 100644 zml/kernels/paged_attention/cute/reference.zig diff --git a/kernels/cute/README.md b/kernels/cute/README.md index 52aae51c..c96509a9 100644 --- a/kernels/cute/README.md +++ b/kernels/cute/README.md @@ -62,6 +62,15 @@ const c = NaiveElementwiseAdd.call(.{ .gA = a, .gB = b }, .{ .gC = a.shape() }, ## Python to Zig +Put each control-flow region in a Zig `{ ... }` block immediately after +`openFor`, `openIf`, or `openWhile`, with its corresponding `yield` call inside +the block. For `openIfElse`, use separate blocks ending in `yieldThen` and +`yieldElse`; likewise, give `openWhile` separate `yieldBefore` and `yieldAfter` +blocks. Keep the scope variable outside these blocks so its `.results` remain +accessible afterward. The braces make the generated control flow visible and +keep region-local values from escaping their scope; the builder's `open` and +`yield` calls still create and close the actual IR regions. + | `cutlass.cute` | `kernels/cute` | |--------------------------------------------|-------------------------------------------------| | `gA: cute.Tensor` kernel argument | `.gA = .{ .tensor = .{ .dtype, .shape } }` | diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index a396efb0..435afb5c 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -123,6 +123,7 @@ zig_library( "//zml/kernels/paged_attention/mosaic_tpu", "//zml/kernels/paged_attention/stablehlo", "//zml/kernels/paged_attention/triton", + "//zml/kernels/paged_attention/cute", ], data = select({ "//platforms:neuron.enabled": [ diff --git a/zml/kernels/paged_attention/cute/BUILD.bazel b/zml/kernels/paged_attention/cute/BUILD.bazel new file mode 100644 index 00000000..9f44e41f --- /dev/null +++ b/zml/kernels/paged_attention/cute/BUILD.bazel @@ -0,0 +1,45 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_test") + +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "cute", + srcs = [ + "attention.zig", + "cute.zig", + ], +) + +zig_test( + name = "correctness_test", + srcs = [ + "cases.zig", + "fixtures.zig", + "reference.zig", + ], + main = "correctness_test.zig", + tags = [ + "exclusive", + "requires-gpu", + ], + target_compatible_with = select({ + "//platforms:cuda.enabled": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + deps = ["//zml"], +) + +zig_binary( + name = "benchmark", + srcs = [ + "cases.zig", + "fixtures.zig", + ], + main = "benchmark.zig", + tags = ["manual"], + target_compatible_with = select({ + "//platforms:cuda.enabled": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + deps = ["//zml"], +) diff --git a/zml/kernels/paged_attention/cute/attention.zig b/zml/kernels/paged_attention/cute/attention.zig new file mode 100644 index 00000000..245ef7e8 --- /dev/null +++ b/zml/kernels/paged_attention/cute/attention.zig @@ -0,0 +1,123 @@ +const std = @import("std"); + +const zml = @import("../../../zml.zig"); +const cute = zml.kernel.cute; +const V = cute.Value; + +pub const Config = struct { + head_dim: i64, + tokens: i64, + batch: i64, + kv_heads: i64, + group: i64, + pages: i64, + pages_per_seq: i64, + page_size: i64, + dtype: cute.DType, + causal: bool, + window: i64, + scale: f32, + sink: bool, + indexed: bool, + topk: i64, +}; + +pub const Kernel = cute.Kernel(Config, .{ + .name = "paged_attention_cute_generic", + .inputs = &.{ "q", "k", "v", "table", "lengths", "starts", "sink", "indices" }, + .outputs = &.{"out"}, + .run = run, +}); + +fn tuple(comptime T: type, comptime n: usize, values: [n]T) @Tuple(&@as([n]type, @splat(T))) { + var result: @Tuple(&@as([n]type, @splat(T))) = undefined; + inline for (0..n) |i| result[i] = values[i]; + return result; +} + +fn run(b: *cute.Builder, c: Config) cute.FinishError!void { + switch (c.head_dim) { + 32 => try runDim(b, c, 1), + 64 => try runDim(b, c, 2), + 128 => try runDim(b, c, 4), + 256 => try runDim(b, c, 8), + 512 => try runDim(b, c, 16), + else => unreachable, + } +} + +fn runDim(b: *cute.Builder, c: Config, comptime chunks: usize) cute.FinishError!void { + const a = try b.declareArgs(.{ + .q = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.tokens, c.kv_heads * c.group, c.head_dim } } }, + .k = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.pages, c.page_size, c.kv_heads, c.head_dim } } }, + .v = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.pages, c.page_size, c.kv_heads, c.head_dim } } }, + .table = .{ .tensor = .{ .dtype = .i32, .shape = &.{ c.batch, c.pages_per_seq } } }, + .lengths = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch} } }, + .starts = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch + 1} } }, + .sink = .{ .tensor = .{ .dtype = .f32, .shape = &.{c.kv_heads * c.group} } }, + .indices = .{ .tensor = .{ .dtype = .i32, .shape = &.{ c.tokens, c.topk } } }, + .out = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.tokens, c.kv_heads * c.group, c.head_dim } } }, + }); + const token = b.blockIdx().x; + const head = b.blockIdx().y; + const lane = b.threadIdx().x; + const zero = b.cst(.f32, 0); + // One full warp owns a query/head; all control flow is warp-uniform. + // Also define output for unused capacity at the end of the packed queries. + for (0..chunks) |i| a.out.set(.{ token, head, lane.add(i * 32) }, zero.to(c.dtype)); + var active = b.openIf(token.lt(a.starts.get(.{c.batch}))); + { + var find_seq = b.openFor(0, c.batch, 1, .{b.cst(.i32, 0)}); + { + const owns = token.ge(a.starts.get(.{find_seq.iv})).bitAnd(token.lt(a.starts.get(.{find_seq.iv.add(1)}))); + find_seq.yield(.{b.select(owns, find_seq.iv, find_seq.carried[0])}); + } + const seq = find_seq.results[0]; + const start = a.starts.get(.{seq}); + const qlen = a.starts.get(.{seq.add(1)}).sub(start); + const len = a.lengths.get(.{seq}); + const position = len.sub(qlen).add(token.sub(start)); + const begin = if (c.window > 0) position.sub(c.window - 1).maximum(0) else b.cst(.i32, 0); + const end = if (c.causal) position.add(1).minimum(len) else len; + var query: [chunks]V = undefined; + for (&query, 0..) |*q, i| q.* = a.q.get(.{ token, head, lane.add(i * 32) }).to(.f32); + + // State is running max, normalizer, then each lane's output features. + // A sink is an initial softmax entry with zero value. + var initial: [chunks + 2]V = @splat(zero); + initial[0] = if (c.sink) a.sink.get(.{head}).mul(std.math.log2e) else b.cst(.f32, -std.math.inf(f32)); + initial[1] = if (c.sink) b.cst(.f32, 1) else zero; + var keys = b.openFor(if (c.indexed) b.cst(.i32, 0) else begin, if (c.indexed) b.cst(.i32, c.topk) else end, 1, tuple(V, chunks + 2, initial)); + { + const key = if (c.indexed) a.indices.get(.{ token, keys.iv }) else keys.iv; + const valid = key.ge(0).bitAnd(key.lt(len)); + // Invalid indexed entries must not load the page table, K, or V. Duplicates + // deliberately contribute once per occurrence, as independent softmax entries. + var entry = b.openIfElse(valid, tuple(@TypeOf(zero.type_()), chunks + 2, @splat(zero.type_()))); + { + const page = a.table.get(.{ seq, key.div(c.page_size) }); + const offset = key.rem(c.page_size); + const kv_head = head.div(c.group); + var score = zero; + for (query, 0..) |q, i| score = score.add(q.mul(a.k.get(.{ page, offset, kv_head, lane.add(i * 32) }).to(.f32))); + inline for (.{ 16, 8, 4, 2, 1 }) |step| score = score.add(b.shuffleXor(score, step)); + score = score.mul(c.scale * std.math.log2e); + const maximum = keys.carried[0].maximum(score); + const alpha = b.exp2Approx(keys.carried[0].sub(maximum)); + const weight = b.exp2Approx(score.sub(maximum)); + var next: [chunks + 2]V = undefined; + next[0] = maximum; + next[1] = keys.carried[1].mul(alpha).add(weight); + for (0..chunks) |i| next[i + 2] = keys.carried[i + 2].mul(alpha).add(weight.mul(a.v.get(.{ page, offset, kv_head, lane.add(i * 32) }).to(.f32))); + entry.yieldThen(tuple(V, chunks + 2, next)); + } + { + entry.yieldElse(tuple(V, chunks + 2, keys.carried)); + } + keys.yield(tuple(V, chunks + 2, entry.results)); + } + const norm = b.select(keys.results[1].gt(0), b.rcpApprox(keys.results[1]), zero); + for (0..chunks) |i| a.out.set(.{ token, head, lane.add(i * 32) }, keys.results[i + 2].mul(norm).to(c.dtype)); + active.yieldThen(.{}); + } +} diff --git a/zml/kernels/paged_attention/cute/benchmark.zig b/zml/kernels/paged_attention/cute/benchmark.zig new file mode 100644 index 00000000..c4bf1165 --- /dev/null +++ b/zml/kernels/paged_attention/cute/benchmark.zig @@ -0,0 +1,120 @@ +const std = @import("std"); +const zml = @import("zml"); +const cases = @import("cases.zig"); +const fixtures = @import("fixtures.zig"); +const Inputs = fixtures.Inputs; + +pub fn main(init: std.process.Init) !void { + const platform = try zml.Platform.auto(init.gpa, init.io, .{}); + defer platform.deinit(init.gpa, init.io); + + if (platform.target != .cuda) return error.CudaRequired; + + const mesh = try platform.registerMesh("tp", .mesh(.{ .model = .high_bandwidth })); + + const cc = zml.platform.cuda.computeCapability(platform) orelse return error.UnknownComputeCapability; + std.debug.print("PAGED_ATTENTION dtype=bf16 page_size=16 seed=42 cc={d}.{d}\n{f}\n", .{ + cc.major, + cc.minor, + platform.fmtVerbose(), + }); + + std.debug.print("timing=synchronized_invocation_us devices={d} hkv=per_device (KV heads sharded across all visible GPUs; includes runtime, output allocation, attention, merge and wait; excludes compile, upload and output destruction)\n", .{platform.devices.len}); + var failures: usize = 0; + for (cases.benchmarks) |c| { + std.debug.print("CASE name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} mask={s} backend=cute\n", .{ + c.name, + c.head_dim, + c.batch, + c.kv_heads, + c.group, + c.prefill, + if (c.prefill > 0) c.chunk else 1, + c.kv_len, + @tagName(c.mask), + }); + + runBenchmark(init.gpa, init.io, platform, mesh, c) catch |err| { + failures += 1; + std.debug.print("FAIL error={s}\n", .{@errorName(err)}); + }; + } + + if (failures != 0) return error.BenchmarkFailed; +} + +fn runBenchmark(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mesh: *const zml.Mesh, c: cases.Case) !void { + // Keep number of head per GPU constant. + var global_case = c; + global_case.kv_heads *= platform.devices.len; + var inputs = Inputs.init(global_case); + inputs.q = inputs.q.withPartitioning(mesh, .{ .hkv = .model }); + inputs.k = inputs.k.withPartitioning(mesh, .{ .hkv = .model }); + inputs.v = inputs.v.withPartitioning(mesh, .{ .hkv = .model }); + inputs.sinks = inputs.sinks.withPartitioning(mesh, .{ ._0 = .model }); + + const compile_start: std.Io.Timestamp = .now(io, .awake); + const exe = try platform.compileFn(a, io, fixtures.forward, .{ inputs, global_case, zml.paged_attention.Backend.cute }, .{ + .program_name = "paged_attention_harness", + .meshes = &.{mesh}, + }); + defer exe.deinit(); + const compile_us = elapsedUs(compile_start, io); + + var runner = try exe.runner(a); + defer runner.deinit(a); + + var arena = std.heap.ArenaAllocator.init(a); + defer arena.deinit(); + + const fixture = try fixtures.Fixture.init(arena.allocator(), global_case); + + var buffers = try fixtures.upload(io, platform, inputs, fixture); + defer zml.Buffer.deinitAll(Inputs, &buffers); + + var output: zml.Buffer = undefined; + + // Warmup + for (0..5) |_| { + runner.run(io, .{buffers}, .{&output}, .{ .wait = true }); + output.deinit(); + } + + var samples: [101]f64 = undefined; + var sample_count: usize = 0; + var accumulated_us: f64 = 0; + while (sample_count < samples.len and (sample_count < 21 or accumulated_us < 100_000)) { + const call_start: std.Io.Timestamp = .now(io, .awake); + runner.run(io, .{buffers}, .{&output}, .{ .wait = true }); + samples[sample_count] = elapsedUs(call_start, io); + accumulated_us += samples[sample_count]; + sample_count += 1; + output.deinit(); + } + + std.mem.sort(f64, samples[0..sample_count], {}, std.sort.asc(f64)); + const median = if (sample_count % 2 == 0) + (samples[sample_count / 2 - 1] + samples[sample_count / 2]) / 2 + else + samples[sample_count / 2]; + + std.debug.print("RESULT name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} backend=cute compile_ms={d:.3} n={d} median_us={d:.3} p10_us={d:.3} p90_us={d:.3}\n", .{ + c.name, + c.head_dim, + c.batch, + c.kv_heads, + c.group, + c.prefill, + if (c.prefill > 0) c.chunk else 1, + c.kv_len, + compile_us / 1000, + sample_count, + median, + samples[sample_count / 10], + samples[(sample_count - 1) * 9 / 10], + }); +} + +fn elapsedUs(start: std.Io.Timestamp, io: std.Io) f64 { + return @as(f64, @floatFromInt(start.untilNow(io, .awake).toNanoseconds())) / 1000; +} diff --git a/zml/kernels/paged_attention/cute/cases.zig b/zml/kernels/paged_attention/cute/cases.zig new file mode 100644 index 00000000..0bf417f4 --- /dev/null +++ b/zml/kernels/paged_attention/cute/cases.zig @@ -0,0 +1,82 @@ +const std = @import("std"); + +pub const page_size = 16; +pub const topk = 8; +pub const Mask = enum { causal, non_causal, window, indices }; + +pub const Case = struct { + name: []const u8 = "standard", + head_dim: usize = 128, + batch: usize = 2, + kv_heads: usize = 2, + group: usize = 4, + prefill: usize = 0, + chunk: usize = 17, + kv_len: usize = 65, + skew: bool = false, + mask: Mask = .causal, + empty_indices: bool = false, + window: u32 = 17, + sink: bool = false, + scale: ?f32 = null, + + pub fn tokens(c: Case) usize { + return c.prefill * c.chunk + c.batch - c.prefill; + } + pub fn pages(c: Case) usize { + return std.math.divCeil(usize, c.kv_len, page_size) catch unreachable; + } + pub fn heads(c: Case) usize { + return c.kv_heads * c.group; + } + pub fn queryLen(c: Case, seq: usize) usize { + return if (seq < c.prefill) c.chunk else 1; + } + pub fn kvLen(c: Case, seq: usize) usize { + const qlen = c.queryLen(seq); + const reduction = if (c.skew) (seq * 17) % c.kv_len else 0; + return @max(qlen, c.kv_len - @min(reduction, c.kv_len - 1)); + } +}; + +// Eight geometry cases plus orthogonal semantics/boundary cases, not a Cartesian product. +pub const correctness = blk: { + var result: [17]Case = undefined; + var i: usize = 0; + for ([_]usize{ 64, 128, 256, 512 }) |dim| { + for ([_]bool{ false, true }) |mixed| { + result[i] = .{ .head_dim = dim, .prefill = if (mixed) 1 else 0 }; + i += 1; + } + } + result[8] = .{ .name = "mha_page_tail", .group = 1, .kv_len = 15 }; + result[9] = .{ .name = "mqa_page_exact", .kv_heads = 1, .group = 8, .kv_len = 16 }; + result[10] = .{ .name = "non_causal", .head_dim = 256, .prefill = 1, .mask = .non_causal }; + result[11] = .{ .name = "window_sink", .head_dim = 256, .prefill = 1, .mask = .window, .sink = true, .scale = 0.125 }; + result[12] = .{ .name = "indices", .head_dim = 256, .prefill = 1, .mask = .indices, .sink = true }; + result[13] = .{ .name = "long_decode", .head_dim = 512, .kv_heads = 1, .group = 2, .batch = 1, .kv_len = 2049 }; + result[14] = .{ .name = "skew", .head_dim = 256, .batch = 4, .prefill = 2, .skew = true, .kv_len = 129 }; + result[15] = .{ .name = "mixed_short_chunks", .batch = 4, .prefill = 2, .chunk = 16, .skew = true, .kv_len = 64 }; + result[16] = .{ .name = "empty_indices", .prefill = 1, .mask = .indices, .empty_indices = true }; + break :blk result; +}; + +pub const benchmarks = blk: { + var result: [24]Case = undefined; + var i: usize = 0; + for ([_]usize{ 64, 128, 256, 512 }) |dim| { + for ([_]usize{ 2, 8, 32 }) |batch| { + for ([_]bool{ false, true }) |mixed| { + result[i] = .{ + .head_dim = dim, + .batch = batch, + .prefill = if (mixed) @max(1, batch / 4) else 0, + .chunk = 64, + .kv_len = 2048, + }; + i += 1; + } + } + } + break :blk result; +}; diff --git a/zml/kernels/paged_attention/cute/correctness_test.zig b/zml/kernels/paged_attention/cute/correctness_test.zig new file mode 100644 index 00000000..e61ff44b --- /dev/null +++ b/zml/kernels/paged_attention/cute/correctness_test.zig @@ -0,0 +1,65 @@ +const reference = @import("reference.zig"); +const std = @import("std"); +const zml = @import("zml"); +const cases = @import("cases.zig"); +const fixtures = @import("fixtures.zig"); +const Inputs = fixtures.Inputs; + +test "CPU reference self checks" { + _ = @import("reference.zig"); +} + +test "paged attention correctness" { + const platform = zml.testing.env(); + if (platform.target != .cuda) return error.CudaRequired; + + const a = std.testing.allocator; + const io = std.testing.io; + + const cc = zml.platform.cuda.computeCapability(platform) orelse return error.UnknownComputeCapability; + std.debug.print("PAGED_ATTENTION dtype=bf16 page_size=16 seed=42 cc={d}.{d}\n{f}\n", .{ + cc.major, cc.minor, platform.fmtVerbose(), + }); + + var failures: usize = 0; + for (cases.correctness) |c| { + std.debug.print("CASE name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} mask={s} backend=cute\n", .{ + c.name, c.head_dim, c.batch, c.kv_heads, c.group, c.prefill, if (c.prefill > 0) c.chunk else 1, c.kv_len, @tagName(c.mask), + }); + + runCorrectness(a, io, platform, c) catch |err| { + failures += 1; + std.debug.print("FAIL error={s}\n", .{@errorName(err)}); + }; + } + + try std.testing.expectEqual(@as(usize, 0), failures); +} + +fn runCorrectness(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, c: cases.Case) !void { + const exe = try platform.compileFn(a, io, fixtures.forward, .{ Inputs.init(c), c, zml.paged_attention.Backend.cute }, .{ .program_name = "paged_attention_harness" }); + defer exe.deinit(); + + var runner = try exe.runner(a); + defer runner.deinit(a); + + var arena = std.heap.ArenaAllocator.init(a); + defer arena.deinit(); + + const fixture = try fixtures.Fixture.init(arena.allocator(), c); + + var buffers = try fixtures.upload(io, platform, Inputs.init(c), fixture); + defer zml.Buffer.deinitAll(Inputs, &buffers); + + var output: zml.Buffer = undefined; + runner.run(io, .{buffers}, .{&output}, .{ .wait = true }); + defer output.deinit(); + + var host = try output.toSliceAlloc(a, io); + defer host.free(a); + + const max_error = try reference.check(a, c, fixture, host.items(zml.floats.BFloat16)); + std.debug.print("RESULT name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} backend=cute check=all_outputs max_abs={d:.6}\n", .{ + c.name, c.head_dim, c.batch, c.kv_heads, c.group, c.prefill, if (c.prefill > 0) c.chunk else 1, c.kv_len, max_error, + }); +} diff --git a/zml/kernels/paged_attention/cute/cute.zig b/zml/kernels/paged_attention/cute/cute.zig new file mode 100644 index 00000000..60302a72 --- /dev/null +++ b/zml/kernels/paged_attention/cute/cute.zig @@ -0,0 +1,78 @@ +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const paged = @import("../paged_attention.zig"); +const triton = @import("../triton/triton.zig"); +const kernel = @import("attention.zig"); + +pub const Options = triton.paged.Options; +pub const Parameters = triton.paged.Parameters; + +test { + _ = kernel; +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + const cc = zml.platform.cuda.computeCapability(platform) orelse return false; + return cc.major > 8 or (cc.major == 8 and cc.minor >= 9); +} + +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { + stdx.debug.assert(isAvailable(zml.Compiler.current().platform), "CuTe paged attention requires SM89 or newer", .{}); + stdx.debug.assert(q.dtype() == k.dtype() and q.dtype() == v.dtype() and (q.dtype() == .bf16 or q.dtype() == .f16), "CuTe paged attention requires matching BF16 or FP16 Q/K/V", .{}); + stdx.debug.assert(q.dim(.hd) == 32 or q.dim(.hd) == 64 or q.dim(.hd) == 128 or q.dim(.hd) == 256 or q.dim(.hd) == 512, "CuTe paged attention supports head dimensions 32, 64, 128, 256 and 512", .{}); + return zml.ops.manualComputation(Context.body, Context{ .q = q, .k = k, .v = v, .parameters = parameters, .opts = opts }, q.shape()); +} + +const Context = struct { + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + parameters: Parameters, + opts: paged.AttentionOptions, + + fn body(self: Context, output_shape: zml.Shape) zml.Tensor { + const q = self.q.transpose(.{ .b, .hkv, .hg, .hd }); + const k = self.k.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const v = self.v.transpose(.{ .page, .k_chunk, .hkv, .hd }); + stdx.debug.assert(q.dim(.hkv) == k.dim(.hkv) and k.shape().eql(v.shape()) and k.dim(.hd) == q.dim(.hd) and q.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); + const p = self.parameters; + const indices = switch (self.opts.mask) { + .indices => |indices| indices.transpose(.{ .b, .topk }), + else => zml.Tensor.zeroes(.init(.{ q.dim(.b), 1 }, .i32)), + }; + const cfg: kernel.Config = .{ + .head_dim = q.dim(.hd), + .tokens = q.dim(.b), + .batch = p.block_table.dim(.b), + .kv_heads = k.dim(.hkv), + .group = q.dim(.hg), + .pages = k.dim(.page), + .pages_per_seq = p.block_table.dim(.p), + .page_size = k.dim(.k_chunk), + .dtype = zml.kernel.cute.from(q.dtype()), + .causal = self.opts.mask.isCausal(), + .window = self.opts.mask.slidingWindow() orelse 0, + .scale = self.opts.scale orelse (1 / @sqrt(@as(f32, @floatFromInt(q.dim(.hd))))), + .sink = self.opts.sink != null, + .indexed = self.opts.mask == .indices, + .topk = indices.dim(-1), + }; + const sink = if (self.opts.sink) |s| s.convert(.f32).reshape(.{cfg.kv_heads * cfg.group}) else zml.Tensor.zeroes(.init(.{cfg.kv_heads * cfg.group}, .f32)); + return kernel.Kernel.call(.{ + .q = q, + .k = k, + .v = v, + .table = p.block_table, + .lengths = p.seq_lens, + .starts = p.query_start_len, + .sink = sink, + .indices = indices, + }, .{ .out = q.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(cfg.tokens), @intCast(cfg.kv_heads * cfg.group), 1 }, + .block = .{ 32, 1, 1 }, + }).out.transpose(output_shape); + } +}; diff --git a/zml/kernels/paged_attention/cute/fixtures.zig b/zml/kernels/paged_attention/cute/fixtures.zig new file mode 100644 index 00000000..b7cc2224 --- /dev/null +++ b/zml/kernels/paged_attention/cute/fixtures.zig @@ -0,0 +1,133 @@ +const std = @import("std"); +const zml = @import("zml"); +const cases = @import("cases.zig"); + +pub const Fixture = struct { + q: []zml.floats.BFloat16, + k: []zml.floats.BFloat16, + v: []zml.floats.BFloat16, + table: []i32, + lengths: []i32, + starts: []i32, + sinks: []f32, + indices: []i32, + + // The caller owns an arena; all arrays live until the case is complete. + pub fn init(a: std.mem.Allocator, c: cases.Case) !Fixture { + const f: Fixture = .{ + .q = try a.alloc(zml.floats.BFloat16, c.tokens() * c.heads() * c.head_dim), + .k = try a.alloc(zml.floats.BFloat16, c.batch * c.pages() * cases.page_size * c.kv_heads * c.head_dim), + .v = try a.alloc(zml.floats.BFloat16, c.batch * c.pages() * cases.page_size * c.kv_heads * c.head_dim), + .table = try a.alloc(i32, c.batch * c.pages()), + .lengths = try a.alloc(i32, c.batch), + .starts = try a.alloc(i32, c.batch + 1), + .sinks = try a.alloc(f32, c.heads()), + .indices = try a.alloc(i32, c.tokens() * cases.topk), + }; + var rng = std.Random.DefaultPrng.init(42); + const random = rng.random(); + for (f.q) |*v| v.* = .fromF32(random.float(f32) * 2 - 1); + for (f.k) |*v| v.* = .fromF32(random.float(f32) * 2 - 1); + for (f.v) |*v| v.* = .fromF32(random.float(f32) * 2 - 1); + for (f.sinks, 0..) |*v, h| v.* = @as(f32, @floatFromInt(h % 5)) - 2; + @memset(f.indices, -1); + f.starts[0] = 0; + for (0..c.batch) |s| { + const len = c.kvLen(s); + f.lengths[s] = @intCast(len); + f.starts[s + 1] = f.starts[s] + @as(i32, @intCast(c.queryLen(s))); + for (0..c.pages()) |p| { + f.table[s * c.pages() + p] = if (p * cases.page_size < len) @intCast(s * c.pages() + p) else -1; + } + // Unused physical slots must not contribute to PV, even via zero * NaN. + for (len..c.pages() * cases.page_size) |j| { + const offset = (s * c.pages() * cases.page_size + j) * c.kv_heads * c.head_dim; + @memset(f.k[offset..][0 .. c.kv_heads * c.head_dim], .fromF32(std.math.nan(f32))); + @memset(f.v[offset..][0 .. c.kv_heads * c.head_dim], .fromF32(std.math.nan(f32))); + } + for (@intCast(f.starts[s])..@intCast(f.starts[s + 1])) |t| { + const l: i32 = @intCast(len); + @memcpy(f.indices[t * cases.topk ..][0..cases.topk], &[_]i32{ 0, l - 1, @divTrunc(l, 2), -1, l, 0, 1, l - 2 }); + } + } + if (c.empty_indices) { + for (f.indices, 0..) |*index, i| index.* = if (i % 2 == 0) -1 else @intCast(c.kv_len); + @memset(f.k, .fromF32(std.math.nan(f32))); + @memset(f.v, .fromF32(std.math.nan(f32))); + } + return f; + } + + pub fn kvOffset(f: Fixture, c: cases.Case, seq: usize, key: usize, head: usize) usize { + const page: usize = @intCast(f.table[seq * c.pages() + key / cases.page_size]); + return ((page * cases.page_size + key % cases.page_size) * c.kv_heads + head) * c.head_dim; + } +}; + +pub const Inputs = struct { + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + table: zml.Tensor, + lengths: zml.Tensor, + starts: zml.Tensor, + sinks: zml.Tensor, + indices: zml.Tensor, + + pub fn init(c: cases.Case) Inputs { + return .{ + .q = .init(.{ .b = c.tokens(), .hkv = c.kv_heads, .hg = c.group, .hd = c.head_dim }, .bf16), + .k = .init(.{ .page = c.batch * c.pages(), .k_chunk = cases.page_size, .hkv = c.kv_heads, .hd = c.head_dim }, .bf16), + .v = .init(.{ .page = c.batch * c.pages(), .k_chunk = cases.page_size, .hkv = c.kv_heads, .hd = c.head_dim }, .bf16), + .table = .init(.{ .b = c.batch, .p = c.pages() }, .i32), + .lengths = .init(.{ .b = c.batch }, .i32), + .starts = .init(.{ .b = c.batch + 1 }, .i32), + .sinks = .init(.{c.heads()}, .f32), + .indices = .init(.{ .b = c.tokens(), .topk = cases.topk }, .i32), + }; + } + + fn options(i: Inputs, c: cases.Case) zml.paged_attention.AttentionOptions { + return .{ + .mask = switch (c.mask) { + .causal => .{ .standard = .{} }, + .non_causal => .none, + .window => .{ .standard = .{ .sliding_window = c.window } }, + .indices => .{ .indices = i.indices }, + }, + .sink = if (c.sink) i.sinks else null, + .scale = c.scale, + }; + } +}; + +pub fn forward(i: Inputs, c: cases.Case, backend: zml.paged_attention.Backend) zml.Tensor { + const parameters: zml.paged_attention.Parameters = switch (backend) { + inline .cute, .triton => |b| @unionInit(zml.paged_attention.Parameters, @tagName(b), .{ + .block_table = i.table, + .seq_lens = i.lengths, + .query_start_len = i.starts, + .options_ = .{ + .batch_size = c.batch, + .max_num_pages = c.pages(), + .max_seqlen_q = if (c.prefill > 0) c.chunk else 1, + .is_prefill = c.prefill > 0, + }, + }), + else => unreachable, + }; + return zml.paged_attention.pagedAttention(parameters, i.q, i.k, i.v, .{ .split = .{ .k = i.k, .v = i.v } }, i.options(c)); +} + +pub fn upload(io: std.Io, platform: *const zml.Platform, inputs: Inputs, f: Fixture) !zml.Bufferized(Inputs) { + var result: zml.Bufferized(Inputs) = undefined; + var initialized: usize = 0; + errdefer inline for (@typeInfo(Inputs).@"struct".field_names, 0..) |name, index| { + if (index < initialized) @field(result, name).deinit(); + }; + inline for (@typeInfo(Inputs).@"struct".field_names) |name| { + @field(result, name) = try .fromBytes(io, platform, @field(inputs, name).shape(), std.mem.sliceAsBytes(@field(f, name))); + initialized += 1; + } + return result; +} diff --git a/zml/kernels/paged_attention/cute/reference.zig b/zml/kernels/paged_attention/cute/reference.zig new file mode 100644 index 00000000..49b9dc43 --- /dev/null +++ b/zml/kernels/paged_attention/cute/reference.zig @@ -0,0 +1,126 @@ +const std = @import("std"); +const zml = @import("zml"); +const cases = @import("cases.zig"); +const fixtures = @import("fixtures.zig"); + +/// One query/head, independent of GPU tensor operations. Duplicate indices are +/// separate softmax entries; invalid indices are excluded. +pub fn row(c: cases.Case, f: fixtures.Fixture, seq: usize, token: usize, head: usize, scores: []f32, out: []f32) void { + const len: usize = @intCast(f.lengths[seq]); + const start: usize = @intCast(f.starts[seq]); + const qlen: usize = @intCast(f.starts[seq + 1] - f.starts[seq]); + const position = len - qlen + token - start; + const count = if (c.mask == .indices) cases.topk else len; + const scale = c.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(c.head_dim))); + const q = f.q[(token * c.heads() + head) * c.head_dim ..][0..c.head_dim]; + var maximum: f32 = if (c.sink) f.sinks[head] else -std.math.inf(f32); + for (0..count) |j| { + const key = keyAt(c, f, token, j); + const valid = key >= 0 and key < len and switch (c.mask) { + .indices, .non_causal => true, + .causal => key <= position, + .window => key <= position and @as(i64, @intCast(key)) > @as(i64, @intCast(position)) - c.window, + }; + scores[j] = -std.math.inf(f32); + if (!valid) continue; + const offset = f.kvOffset(c, seq, @intCast(key), head / c.group); + var dot: f32 = 0; + for (q, f.k[offset..][0..c.head_dim]) |qv, kv| dot += qv.toF32() * kv.toF32(); + scores[j] = dot * scale; + maximum = @max(maximum, scores[j]); + } + @memset(out, 0); + if (maximum == -std.math.inf(f32)) return; + var denominator: f32 = if (c.sink) @exp(f.sinks[head] - maximum) else 0; + for (0..count) |j| { + if (scores[j] == -std.math.inf(f32)) continue; + const weight = @exp(scores[j] - maximum); + denominator += weight; + const offset = f.kvOffset(c, seq, @intCast(keyAt(c, f, token, j)), head / c.group); + for (out, f.v[offset..][0..c.head_dim]) |*o, v| o.* += weight * v.toF32(); + } + for (out) |*v| v.* /= denominator; +} + +fn keyAt(c: cases.Case, f: fixtures.Fixture, token: usize, j: usize) i64 { + return if (c.mask == .indices) f.indices[token * cases.topk + j] else @intCast(j); +} + +pub fn check(a: std.mem.Allocator, c: cases.Case, f: fixtures.Fixture, actual: []const zml.floats.BFloat16) !f32 { + const active_tokens: usize = @intCast(f.starts[c.batch]); + for (actual[0 .. active_tokens * c.heads() * c.head_dim]) |v| { + if (!std.math.isFinite(v.toF32())) return error.NonFiniteOutput; + } + const scores = try a.alloc(f32, @max(c.kv_len, cases.topk)); + defer a.free(scores); + const expected = try a.alloc(f32, c.head_dim); + defer a.free(expected); + var max_error: f32 = 0; + var failures: usize = 0; + for (0..c.batch) |s| { + for (@intCast(f.starts[s])..@intCast(f.starts[s + 1])) |t| { + for (0..c.heads()) |h| { + row(c, f, s, t, h, scores, expected); + const base = (t * c.heads() + h) * c.head_dim; + for (expected, actual[base..][0..c.head_dim], 0..) |want, got_bf16, d| { + const got = got_bf16.toF32(); + const err = @abs(want - got); + max_error = @max(max_error, err); + // Fixed BF16 error budget; never adapted to observed failures. + if (!std.math.isFinite(want) or err > 0.003 + 0.02 * @abs(want)) { + if (failures == 0) std.debug.print("Mismatch seq={d} token={d} head={d} dim={d}: expected={d} actual={d}\n", .{ s, t, h, d, want, got }); + failures += 1; + } + } + } + } + } + if (failures > 0) return error.ReferenceMismatch; + return max_error; +} + +test "reference single key, uniform causal scores, and sink" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var c: cases.Case = .{ .head_dim = 64, .batch = 1, .kv_heads = 1, .group = 1, .prefill = 1, .chunk = 2, .kv_len = 2 }; + const f = try fixtures.Fixture.init(arena.allocator(), c); + @memset(f.q, .fromF32(0)); + @memset(f.v[0..64], .fromF32(2)); + @memset(f.v[64..128], .fromF32(4)); + var scores: [2]f32 = undefined; + var out: [64]f32 = undefined; + row(c, f, 0, 0, 0, &scores, &out); + for (out) |v| try std.testing.expectEqual(@as(f32, 2), v); + row(c, f, 0, 1, 0, &scores, &out); + for (out) |v| try std.testing.expectEqual(@as(f32, 3), v); + c.sink = true; + f.sinks[0] = 0; + row(c, f, 0, 1, 0, &scores, &out); + for (out) |v| try std.testing.expectEqual(@as(f32, 2), v); +} + +test "reference windows, noncausal, duplicate and invalid indices" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var c: cases.Case = .{ .head_dim = 64, .batch = 1, .kv_heads = 1, .group = 1, .prefill = 1, .chunk = 2, .kv_len = 2 }; + const f = try fixtures.Fixture.init(arena.allocator(), c); + @memset(f.q, .fromF32(0)); + @memset(f.v[0..64], .fromF32(2)); + @memset(f.v[64..128], .fromF32(4)); + var scores: [cases.topk]f32 = undefined; + var out: [64]f32 = undefined; + c.mask = .non_causal; + row(c, f, 0, 0, 0, &scores, &out); + for (out) |v| try std.testing.expectEqual(@as(f32, 3), v); + c.mask = .window; + c.window = 1; + row(c, f, 0, 1, 0, &scores, &out); + for (out) |v| try std.testing.expectEqual(@as(f32, 4), v); + c.mask = .indices; + @memset(f.indices, -1); + @memcpy(f.indices[0..4], &[_]i32{ 0, 0, 1, 2 }); + row(c, f, 0, 0, 0, &scores, &out); + for (out) |v| try std.testing.expectApproxEqAbs(@as(f32, 8.0 / 3.0), v, 1e-6); + row(c, f, 0, 1, 0, &scores, &out); + for (out) |v| try std.testing.expectEqual(@as(f32, 0), v); +} diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index da919554..5cc2ba22 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -3,6 +3,7 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../../zml.zig"); +const cute = @import("cute/cute.zig"); const metal = @import("metal/metal.zig"); const tpu = @import("mosaic_tpu/mosaic_tpu.zig"); const triton = @import("triton/triton.zig"); @@ -18,6 +19,7 @@ test { } pub const Backend = enum { + cute, triton, mosaic_tpu, metal, @@ -37,6 +39,7 @@ pub const Backend = enum { pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { return switch (backend) { + .cute => cute.isAvailable(platform), .stablehlo => true, .triton => platform.target != .cpu, .metal => platform.target == .metal, @@ -46,6 +49,7 @@ pub const Backend = enum { }; pub const Options = union(Backend) { + cute: cute.Options, triton: triton.paged.Options, mosaic_tpu: tpu.mosaic_tpu.Options, metal: metal.Options, @@ -80,7 +84,7 @@ pub const Options = union(Backend) { .head_dim = args.head_dim, }, }, - inline .triton, .metal, .stablehlo => |t| @unionInit(Options, @tagName(t), .{ + inline .cute, .triton, .metal, .stablehlo => |t| @unionInit(Options, @tagName(t), .{ .batch_size = args.batch_size, .max_num_pages = args.max_num_pages, .max_seqlen_q = args.max_seqlen_q, @@ -103,6 +107,7 @@ pub const Options = union(Backend) { }; pub const Parameters = union(Backend) { + cute: cute.Parameters, triton: triton.paged.Parameters, mosaic_tpu: tpu.mosaic_tpu.Parameters, metal: metal.Parameters, @@ -191,7 +196,7 @@ pub const KvCache = union(enum) { const kv: KvCache = switch (self) { .split => |split| switch (backend) { - .triton, .mosaic_tpu, .metal, .stablehlo => .{ + .cute, .triton, .mosaic_tpu, .metal, .stablehlo => .{ .split = .{ .k = split.k.scatterSlices( .{ .page = page_index, .k_chunk = offset }, @@ -236,6 +241,10 @@ pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: z _ = k; _ = v; return switch (parameters) { + .cute => |p| switch (kv_cache) { + .split => |split| cute.pagedAttention(p, q, split.k, split.v, opts), + .dense => @panic("CuTe paged attention requires split KV pages"), + }, .triton => |triton_parameters| switch (kv_cache) { .split => |split| triton.paged.pagedAttention(triton_parameters, q, split.k, split.v, opts), .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), @@ -391,7 +400,7 @@ test pagedAttention { const all_backends = std.enums.values(Backend); // stablehlo ignores these options, mosaic_tpu requires a dense // KV cache instead of the split cache used by this test. - const option_sensitive_backends = [_]Backend{ .triton, .metal }; + const option_sensitive_backends = [_]Backend{ .triton, .cute, .metal }; const test_cases = [_]struct { name: []const u8, attention_options: AttentionOptions, @@ -444,7 +453,7 @@ test pagedAttention { const parameters_d: zml.Bufferized(Parameters) = switch (parameters) { .triton => triton_parameters_d, - inline .metal, .mosaic_tpu, .stablehlo => |_, t| @unionInit(zml.Bufferized(Parameters), @tagName(t), .{ + inline .cute, .metal, .mosaic_tpu, .stablehlo => |_, t| @unionInit(zml.Bufferized(Parameters), @tagName(t), .{ .block_table = triton_parameters_d.triton.block_table, .seq_lens = triton_parameters_d.triton.seq_lens, .query_start_len = triton_parameters_d.triton.query_start_len, From 0db5229cd43c18c491814fcea95bb602befdaeb0 Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Wed, 21 Oct 2026 10:58:19 +0200 Subject: [PATCH 806/881] zml/kernels: parallelize MoE expert block assignment (#961) Assign blocks for multiple experts in parallel instead of processing experts one at a time. Each lane writes the expert ids for its own block range. Reduces assignment cost from 7.6 us to 1.8 us per call. --- zml/kernels/moe/triton/moe.zig | 38 +++++++++++----------------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/zml/kernels/moe/triton/moe.zig b/zml/kernels/moe/triton/moe.zig index b3a22c18..0d2bab5b 100644 --- a/zml/kernels/moe/triton/moe.zig +++ b/zml/kernels/moe/triton/moe.zig @@ -639,6 +639,9 @@ test "FusedMoe emits each FP8 and MXFP4 scaling path" { /// Direct port of Python `moe_align_block_size_kernel`. pid==0 runs the /// histogram + cumsum + block-to-expert assignment pass; pid==1 fills /// sorted_token_ids with NUMEL. +/// +/// Block assignment are set up to `num_tokens_post_padded` and the remaining blocks of +/// `expert_ids` are not set to a sentinel value. pub const MoeAlignBlockSize = struct { pub const Cfg = struct { numel: usize, @@ -673,7 +676,6 @@ pub const MoeAlignBlockSize = struct { const num_experts: i32 = @intCast(cfg.num_experts); const padded_num_experts: i64 = @intCast(cfg.padded_num_experts); const max_num_tokens_padded: i32 = @intCast(cfg.max_num_tokens_padded); - const max_num_m_blocks: i64 = @intCast(cfg.max_num_m_blocks); const hist_block: i64 = @intCast(cfg.hist_block); const pid = b.programId(.x); @@ -731,32 +733,16 @@ pub const MoeAlignBlockSize = struct { b.store(a.cumsum_ptr.addPtr(num_experts), total_tokens_post_pad); b.store(a.num_tokens_post_pad_ptr, total_tokens_post_pad); - const block_offs = b.arange(0, hist_block, .i32); - var assign_loop = b.openFor(0, max_num_m_blocks, hist_block, .{}); + // Each expert owns a disjoint padded range, including empty ranges. + // Scatter from register-resident prefixes; no cross-lane global reads. + const block_counts = padded_counts.div(block_size_m); + const max_blocks = b.maxOpts(block_counts, .{ .axis = 0 }); + var scatter = b.openFor(0, max_blocks, 1, .{}); { - const block_ids = assign_loop.iv.add(block_offs); - const block_mask = block_ids.lt(@as(i32, @intCast(max_num_m_blocks))); - const block_offsets = block_ids.mul(block_size_m); - const block_expert_init = b.full(&.{hist_block}, -1, .i32); - - var exp_loop = b.openFor(0, num_experts, 1, .{block_expert_init}); - { - const e_iv = exp_loop.iv; - const start_v = b.load(a.cumsum_ptr.addPtr(e_iv)); - const end_v = b.load(a.cumsum_ptr.addPtr(e_iv.add(1))); - const in_range = block_mask - .bitAnd(block_offsets.ge(start_v)) - .bitAnd(block_offsets.lt(end_v)); - exp_loop.yield(.{ - b.select(in_range, e_iv.splatTo(&.{hist_block}), exp_loop.carried[0]), - }); - } - b.storeOpts( - a.expert_ids_ptr.addPtr(block_ids), - exp_loop.results[0], - .{ .mask = block_mask }, - ); - assign_loop.yield(.{}); + const block_ids = starts.div(block_size_m).add(scatter.iv); + const mask = expert_mask.bitAnd(block_counts.gt(scatter.iv)); + b.storeOpts(a.expert_ids_ptr.addPtr(block_ids), expert_offs, .{ .mask = mask }); + scatter.yield(.{}); } } }; From 21f5b1a55fdafeb95a7ad19e13afbfc4bd6705cd Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Wed, 21 Oct 2026 11:08:35 +0200 Subject: [PATCH 807/881] zml/kernels: vectorize KV load for triton paged attention (#971) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make KV-cache strides compile-time constants so Triton can infer alignment and simplify address calculations. On Qwen3.5-35B-A3B, both 2D and 3D attention kernels now use 128-bit vector loads for K/V, fetching eight bf16 values per instruction per active thread instead of one. Thanks @fior512 for your contribution 🙏 --------- Co-authored-by: fior512 <161968556+fior512@users.noreply.github.com> --- zml/kernels/paged_attention/triton/triton.zig | 24 ++--- .../triton/unified_attention.zig | 89 ++++++------------- 2 files changed, 35 insertions(+), 78 deletions(-) diff --git a/zml/kernels/paged_attention/triton/triton.zig b/zml/kernels/paged_attention/triton/triton.zig index f4212706..b0b69b8d 100644 --- a/zml/kernels/paged_attention/triton/triton.zig +++ b/zml/kernels/paged_attention/triton/triton.zig @@ -368,10 +368,14 @@ pub const paged = struct { pub fn pagedAttention2d(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, paged_attention_opts: PagedAttentionOptions) zml.Tensor { const config = select2dConfig(paged_attention_opts); + const k_strides = k_cache.shape().computeElementStrides().constSlice(); + const v_strides = v_cache.shape().computeElementStrides().constSlice(); const kernel_config: kernels.KernelUnifiedAttention2dPtr.Config = .{ .q_dtype = triton.from(q.dtype()), .kv_dtype = triton.from(k_cache.dtype()), + .k_strides = .{ k_strides[0], k_strides[1], k_strides[2] }, + .v_strides = .{ v_strides[0], v_strides[1], v_strides[2] }, .o_dtype = triton.from(q.dtype()), .num_query_heads = @intCast(paged_attention_opts.num_heads), .num_queries_per_kv = @intCast(paged_attention_opts.numQueriesPerKv()), @@ -398,8 +402,6 @@ pub const paged = struct { const q_shape = q.shape().mergeAxes(.{ .h = .{ .hkv, .hg } }); const q_strides = q_shape.computeElementStrides().constSlice(); - const k_strides = k_cache.shape().computeElementStrides().constSlice(); - const v_strides = v_cache.shape().computeElementStrides().constSlice(); const scale: f32 = paged_attention_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))); const num_seqs = parameters.block_table.dim(0); @@ -431,12 +433,6 @@ pub const paged = struct { .output_stride_0_ptr = .scalar(q_strides[0], .i64), .output_stride_1_ptr = .scalar(q_strides[1], .i64), .qq_bias_stride_0_ptr = dummy, - .stride_k_cache_0_ptr = .scalar(k_strides[0], .i64), - .stride_k_cache_1_ptr = .scalar(k_strides[1], .i64), - .stride_k_cache_2_ptr = .scalar(k_strides[2], .i64), - .stride_v_cache_0_ptr = .scalar(v_strides[0], .i64), - .stride_v_cache_1_ptr = .scalar(v_strides[1], .i64), - .stride_v_cache_2_ptr = .scalar(v_strides[2], .i64), .query_start_len_ptr = parameters.query_start_len, .num_seqs_ptr = .scalar(num_seqs, .i32), }, @@ -453,11 +449,15 @@ pub const paged = struct { pub fn pagedAttention3d(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, paged_attention_opts: PagedAttentionOptions) zml.Tensor { const config = select3dConfig(paged_attention_opts); + const k_strides = k_cache.shape().computeElementStrides().constSlice(); + const v_strides = v_cache.shape().computeElementStrides().constSlice(); const head_size_padded: i64 = @intCast(std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim)); const attn_kernel_config: kernels.KernelUnifiedAttention3dPtr.Config = .{ .q_dtype = triton.from(q.dtype()), .kv_dtype = triton.from(k_cache.dtype()), + .k_strides = .{ k_strides[0], k_strides[1], k_strides[2] }, + .v_strides = .{ v_strides[0], v_strides[1], v_strides[2] }, .num_query_heads = @intCast(paged_attention_opts.num_heads), .num_queries_per_kv = @intCast(paged_attention_opts.numQueriesPerKv()), .block_size = @intCast(paged_attention_opts.block_size), @@ -496,8 +496,6 @@ pub const paged = struct { const q_shape = q.shape().mergeAxes(.{ .h = .{ .hkv, .hg } }); const q_strides = q_shape.computeElementStrides().constSlice(); - const k_strides = k_cache.shape().computeElementStrides().constSlice(); - const v_strides = v_cache.shape().computeElementStrides().constSlice(); const scale: f32 = paged_attention_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))); const num_seqs = parameters.block_table.dim(0); @@ -532,12 +530,6 @@ pub const paged = struct { .query_stride_0_ptr = .scalar(q_strides[0], .i64), .query_stride_1_ptr = .scalar(q_strides[1], .i64), .qq_bias_stride_0_ptr = dummy, - .stride_k_cache_0_ptr = .scalar(k_strides[0], .i64), - .stride_k_cache_1_ptr = .scalar(k_strides[1], .i64), - .stride_k_cache_2_ptr = .scalar(k_strides[2], .i64), - .stride_v_cache_0_ptr = .scalar(v_strides[0], .i64), - .stride_v_cache_1_ptr = .scalar(v_strides[1], .i64), - .stride_v_cache_2_ptr = .scalar(v_strides[2], .i64), .query_start_len_ptr = parameters.query_start_len, .num_seqs_ptr = .scalar(num_seqs, .i32), }, diff --git a/zml/kernels/paged_attention/triton/unified_attention.zig b/zml/kernels/paged_attention/triton/unified_attention.zig index b6ca3f12..bf3fadbd 100644 --- a/zml/kernels/paged_attention/triton/unified_attention.zig +++ b/zml/kernels/paged_attention/triton/unified_attention.zig @@ -22,6 +22,8 @@ pub const KernelUnifiedAttention2dPtr = struct { pub const Config = struct { q_dtype: DType, kv_dtype: DType, + k_strides: [3]i64, + v_strides: [3]i64, o_dtype: DType, scale_dtype: DType = .f32, @@ -55,13 +57,12 @@ pub const KernelUnifiedAttention2dPtr = struct { pub const Kernel = tri.Kernel(Config, .{ .name = "kernel_unified_attention_2d_ptr", .inputs = &.{ - "query_ptr", "key_cache_ptr", "value_cache_ptr", "sink_ptr", - "block_tables_ptr", "seq_lens_ptr", "alibi_slopes_ptr", "qq_bias_ptr", - "scale_ptr", "k_scale_ptr", "v_scale_ptr", "out_scale_ptr", - "softcap_ptr", "block_table_stride_ptr", "query_stride_0_ptr", "query_stride_1_ptr", - "output_stride_0_ptr", "output_stride_1_ptr", "qq_bias_stride_0_ptr", "stride_k_cache_0_ptr", - "stride_k_cache_1_ptr", "stride_k_cache_2_ptr", "stride_v_cache_0_ptr", "stride_v_cache_1_ptr", - "stride_v_cache_2_ptr", "query_start_len_ptr", "num_seqs_ptr", "indices_ptr", + "query_ptr", "key_cache_ptr", "value_cache_ptr", "sink_ptr", + "block_tables_ptr", "seq_lens_ptr", "alibi_slopes_ptr", "qq_bias_ptr", + "scale_ptr", "k_scale_ptr", "v_scale_ptr", "out_scale_ptr", + "softcap_ptr", "block_table_stride_ptr", "query_stride_0_ptr", "query_stride_1_ptr", + "output_stride_0_ptr", "output_stride_1_ptr", "qq_bias_stride_0_ptr", "query_start_len_ptr", + "num_seqs_ptr", "indices_ptr", }, .outputs = &.{"output"}, .run = run, @@ -87,12 +88,6 @@ pub const KernelUnifiedAttention2dPtr = struct { .output_stride_0_ptr = .{ .ptr = .i64 }, .output_stride_1_ptr = .{ .ptr = .i64 }, .qq_bias_stride_0_ptr = .{ .ptr = .i64 }, - .stride_k_cache_0_ptr = .{ .ptr = .i64 }, - .stride_k_cache_1_ptr = .{ .ptr = .i64 }, - .stride_k_cache_2_ptr = .{ .ptr = .i64 }, - .stride_v_cache_0_ptr = .{ .ptr = .i64 }, - .stride_v_cache_1_ptr = .{ .ptr = .i64 }, - .stride_v_cache_2_ptr = .{ .ptr = .i64 }, .query_start_len_ptr = .{ .ptr = .i32 }, .num_seqs_ptr = .{ .ptr = .i32 }, .indices_ptr = .{ .ptr = .i32 }, @@ -110,12 +105,6 @@ pub const KernelUnifiedAttention2dPtr = struct { const output_stride_0 = b.load(a.output_stride_0_ptr); const output_stride_1 = b.load(a.output_stride_1_ptr); const qq_bias_stride_0 = b.load(a.qq_bias_stride_0_ptr); - const stride_k_cache_0 = b.load(a.stride_k_cache_0_ptr); - const stride_k_cache_1 = b.load(a.stride_k_cache_1_ptr); - const stride_k_cache_2 = b.load(a.stride_k_cache_2_ptr); - const stride_v_cache_0 = b.load(a.stride_v_cache_0_ptr); - const stride_v_cache_1 = b.load(a.stride_v_cache_1_ptr); - const stride_v_cache_2 = b.load(a.stride_v_cache_2_ptr); const num_seqs = b.load(a.num_seqs_ptr); kernelUnifiedAttention2d( @@ -141,12 +130,6 @@ pub const KernelUnifiedAttention2dPtr = struct { output_stride_0, output_stride_1, qq_bias_stride_0, - stride_k_cache_0, - stride_k_cache_1, - stride_k_cache_2, - stride_v_cache_0, - stride_v_cache_1, - stride_v_cache_2, a.query_start_len_ptr, num_seqs, cfg, @@ -159,6 +142,8 @@ pub const KernelUnifiedAttention3dPtr = struct { pub const Config = struct { q_dtype: DType, kv_dtype: DType, + k_strides: [3]i64, + v_strides: [3]i64, scale_dtype: DType = .f32, num_query_heads: i64, @@ -189,13 +174,11 @@ pub const KernelUnifiedAttention3dPtr = struct { pub const Kernel = tri.Kernel(Config, .{ .name = "kernel_unified_attention_3d_ptr", .inputs = &.{ - "query_ptr", "key_cache_ptr", "value_cache_ptr", "sink_ptr", - "block_tables_ptr", "seq_lens_ptr", "alibi_slopes_ptr", "qq_bias_ptr", - "scale_ptr", "k_scale_ptr", "v_scale_ptr", "softcap_ptr", - "block_table_stride_ptr", "query_stride_0_ptr", "query_stride_1_ptr", "qq_bias_stride_0_ptr", - "stride_k_cache_0_ptr", "stride_k_cache_1_ptr", "stride_k_cache_2_ptr", "stride_v_cache_0_ptr", - "stride_v_cache_1_ptr", "stride_v_cache_2_ptr", "query_start_len_ptr", "num_seqs_ptr", - "indices_ptr", + "query_ptr", "key_cache_ptr", "value_cache_ptr", "sink_ptr", + "block_tables_ptr", "seq_lens_ptr", "alibi_slopes_ptr", "qq_bias_ptr", + "scale_ptr", "k_scale_ptr", "v_scale_ptr", "softcap_ptr", + "block_table_stride_ptr", "query_stride_0_ptr", "query_stride_1_ptr", "qq_bias_stride_0_ptr", + "query_start_len_ptr", "num_seqs_ptr", "indices_ptr", }, .outputs = &.{ "segm_output", "segm_max", "segm_expsum" }, .run = run, @@ -218,12 +201,6 @@ pub const KernelUnifiedAttention3dPtr = struct { .query_stride_0_ptr = .{ .ptr = .i64 }, .query_stride_1_ptr = .{ .ptr = .i64 }, .qq_bias_stride_0_ptr = .{ .ptr = .i64 }, - .stride_k_cache_0_ptr = .{ .ptr = .i64 }, - .stride_k_cache_1_ptr = .{ .ptr = .i64 }, - .stride_k_cache_2_ptr = .{ .ptr = .i64 }, - .stride_v_cache_0_ptr = .{ .ptr = .i64 }, - .stride_v_cache_1_ptr = .{ .ptr = .i64 }, - .stride_v_cache_2_ptr = .{ .ptr = .i64 }, .query_start_len_ptr = .{ .ptr = .i32 }, .num_seqs_ptr = .{ .ptr = .i32 }, .indices_ptr = .{ .ptr = .i32 }, @@ -240,12 +217,6 @@ pub const KernelUnifiedAttention3dPtr = struct { const query_stride_0 = b.load(a.query_stride_0_ptr); const query_stride_1 = b.load(a.query_stride_1_ptr); const qq_bias_stride_0 = b.load(a.qq_bias_stride_0_ptr); - const stride_k_cache_0 = b.load(a.stride_k_cache_0_ptr); - const stride_k_cache_1 = b.load(a.stride_k_cache_1_ptr); - const stride_k_cache_2 = b.load(a.stride_k_cache_2_ptr); - const stride_v_cache_0 = b.load(a.stride_v_cache_0_ptr); - const stride_v_cache_1 = b.load(a.stride_v_cache_1_ptr); - const stride_v_cache_2 = b.load(a.stride_v_cache_2_ptr); const num_seqs = b.load(a.num_seqs_ptr); kernelUnifiedAttention3d( @@ -270,12 +241,6 @@ pub const KernelUnifiedAttention3dPtr = struct { query_stride_0, query_stride_1, qq_bias_stride_0, - stride_k_cache_0, - stride_k_cache_1, - stride_k_cache_2, - stride_v_cache_0, - stride_v_cache_1, - stride_v_cache_2, a.query_start_len_ptr, num_seqs, cfg, @@ -449,12 +414,6 @@ fn kernelUnifiedAttention2d( output_stride_0: Value, output_stride_1: Value, qq_bias_stride_0: Value, - stride_k_cache_0: Value, - stride_k_cache_1: Value, - stride_k_cache_2: Value, - stride_v_cache_0: Value, - stride_v_cache_1: Value, - stride_v_cache_2: Value, query_start_len_ptr: Value, num_seqs: Value, config: KernelUnifiedAttention2dPtr.Config, @@ -468,6 +427,12 @@ fn kernelUnifiedAttention2d( const NUM_QUERIES_PER_KV: i64 = config.num_queries_per_kv; const NUM_QUERY_HEADS: i64 = config.num_query_heads; const SLIDING_WINDOW: i64 = config.sliding_window; + const stride_k_cache_0 = k.liftAs(config.k_strides[0], .i64); + const stride_k_cache_1 = k.liftAs(config.k_strides[1], .i64); + const stride_k_cache_2 = k.liftAs(config.k_strides[2], .i64); + const stride_v_cache_0 = k.liftAs(config.v_strides[0], .i64); + const stride_v_cache_1 = k.liftAs(config.v_strides[1], .i64); + const stride_v_cache_2 = k.liftAs(config.v_strides[2], .i64); const kv_head_idx = k.programId(.x); const q_block_global_idx = k.programId(.y); @@ -777,12 +742,6 @@ fn kernelUnifiedAttention3d( query_stride_0: Value, query_stride_1: Value, qq_bias_stride_0: Value, - stride_k_cache_0: Value, - stride_k_cache_1: Value, - stride_k_cache_2: Value, - stride_v_cache_0: Value, - stride_v_cache_1: Value, - stride_v_cache_2: Value, query_start_len_ptr: Value, num_seqs: Value, config: KernelUnifiedAttention3dPtr.Config, @@ -797,6 +756,12 @@ fn kernelUnifiedAttention3d( const NUM_QUERY_HEADS: i64 = config.num_query_heads; const NUM_SEGMENTS_PER_SEQ: i64 = config.num_segments_per_seq; const SLIDING_WINDOW: i64 = config.sliding_window; + const stride_k_cache_0 = k.liftAs(config.k_strides[0], .i64); + const stride_k_cache_1 = k.liftAs(config.k_strides[1], .i64); + const stride_k_cache_2 = k.liftAs(config.k_strides[2], .i64); + const stride_v_cache_0 = k.liftAs(config.v_strides[0], .i64); + const stride_v_cache_1 = k.liftAs(config.v_strides[1], .i64); + const stride_v_cache_2 = k.liftAs(config.v_strides[2], .i64); const q_block_global_idx = k.programId(.x); const kv_head_idx = k.programId(.y); From 24f8c2b9863a2c40552b8b0ba3bad127f6794b5c Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Wed, 21 Oct 2026 11:47:42 +0200 Subject: [PATCH 808/881] examples/llm: sample only the last prompt token (#965) The LM head normalized and projected every prompt position at prefill, and the session read back the whole token buffer just to keep the last token. Every model now slices the hidden state at `active_length - 1` before the norm, so prefill computes logits for a single position and returns a single token. Decode writes the next token over the one it read, so it allocates no buffer per step. lfm2's `actual_seq_len` is renamed `active_length`, the name the other models use. --- examples/llm/models/lfm2/inference.zig | 13 ++++++---- examples/llm/models/lfm2/model.zig | 25 +++++++++++-------- examples/llm/models/lfm2/session.zig | 23 +++++++++-------- examples/llm/models/llama/inference.zig | 11 +++++--- examples/llm/models/llama/model.zig | 11 ++++---- examples/llm/models/llama/session.zig | 13 ++++++++-- examples/llm/models/qwen3_5/inference.zig | 7 +++++- examples/llm/models/qwen3_5/model.zig | 14 ++++++++--- examples/llm/models/qwen3_5/session.zig | 8 +++--- examples/llm/models/qwen3_5_moe/inference.zig | 7 +++++- examples/llm/models/qwen3_5_moe/model.zig | 14 ++++++++--- examples/llm/models/qwen3_5_moe/session.zig | 7 ++++-- 12 files changed, 103 insertions(+), 50 deletions(-) diff --git a/examples/llm/models/lfm2/inference.zig b/examples/llm/models/lfm2/inference.zig index 4726b1da..0e37a999 100644 --- a/examples/llm/models/lfm2/inference.zig +++ b/examples/llm/models/lfm2/inference.zig @@ -55,8 +55,9 @@ pub const CompilationOptions = CompilationParameters; pub const Args = struct { io: std.Io, tokens_buf: *zml.Buffer, + next_token_buf: *zml.Buffer, tokens_pos_buf: *zml.Buffer, - actual_seq_len_buf: *zml.Buffer, + active_length_buf: *zml.Buffer, rng_buf: *zml.Bufferized(zml.Tensor.Rng), cache_buffers: *zml.Bufferized(model.Cache), attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata), @@ -169,7 +170,7 @@ pub fn run( .inputs = .{ .hidden = hidden_buffer, .tokens_position_offset = args.tokens_pos_buf.*, - .actual_seq_len = args.actual_seq_len_buf.*, + .active_length = args.active_length_buf.*, .cache = args.cache_buffers.*, .conv_cache_index = conv_cache_index_buffer.*, .kv_cache_index = kv_cache_index_buffer.*, @@ -188,10 +189,11 @@ pub fn run( .inputs = .{ .hidden = hidden_buffer, .tokens = args.tokens_buf.*, + .active_length = args.active_length_buf.*, .rng = args.rng_buf.*, }, .outputs = .{ - .tokens = args.tokens_buf, + .tokens = args.next_token_buf, .rng = args.rng_buf, }, }, .{}); @@ -279,7 +281,7 @@ fn compileLayer( .layer = layer, .hidden = zml.Tensor.init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()), .tokens_position_offset = zml.Tensor.init(.{ .batch = opts.batch_dim }, .u32), - .actual_seq_len = zml.Tensor.init(.{}, .u32), + .active_length = .init(.{}, .u32), .cache = opts.cache, .conv_cache_index = zml.Tensor.init(.{}, .u32), .kv_cache_index = zml.Tensor.init(.{}, .u32), @@ -312,7 +314,8 @@ fn compileSample( .lm_head = mdl.lm_head, .embed_tokens = mdl.embed_tokens, .hidden = zml.Tensor.init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()), - .tokens = zml.Tensor.init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32), + .tokens = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32), + .active_length = .init(.{}, .u32), .rng = opts.rng, }}); } diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index 009d167d..dd21c7f9 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -209,7 +209,7 @@ pub const Model = struct { self: Model, tokens: zml.Tensor, tokens_position_offset: zml.Tensor, - actual_seq_len: zml.Tensor, + active_length: zml.Tensor, rng: zml.Tensor.Rng, cache_: Cache, attention_metadata: zml.attention.Metadata, @@ -233,7 +233,7 @@ pub const Model = struct { .layer = layer, .hidden = hidden, .tokens_position_offset = tokens_position_offset, - .actual_seq_len = actual_seq_len, + .active_length = active_length, .cache = cache, .conv_cache_index = conv_cache_index, .kv_cache_index = kv_cache_index, @@ -251,6 +251,7 @@ pub const Model = struct { .embed_tokens = self.embed_tokens, .hidden = hidden, .tokens = tokens, + .active_length = active_length, .rng = rng, }); return .{ result.tokens, cache.reuseBuffer(cache_), result.rng }; @@ -298,6 +299,7 @@ pub const LmHead = struct { embed_tokens: TokenEmbedding, hidden: zml.Tensor, tokens: zml.Tensor, + active_length: zml.Tensor, rng: zml.Tensor.Rng, }; @@ -315,9 +317,12 @@ pub const LmHead = struct { pub fn forward(input: Input) Output { const self = input.lm_head; - const logits = input.embed_tokens.unembed(self.embedding_norm.forward(input.hidden)); + const last_index = input.active_length.convert(.i32).addConstant(-1); + const hidden = input.hidden.slice(.seq, .dyn(last_index, 1)); + const logits = input.embed_tokens.unembed(self.embedding_norm.forward(hidden)); const new_tokens, const new_rng = zml.nn.sampleTokens(logits, self.sampling_strategy, input.rng); - return .{ .tokens = new_tokens.convert(input.tokens.dtype()).reuseBuffer(input.tokens), .rng = new_rng }; + const tokens = new_tokens.convert(.u32); + return .{ .tokens = if (input.tokens.dim(.seq) == 1) tokens.reuseBuffer(input.tokens) else tokens, .rng = new_rng }; } pub fn unloadBuffers(self: *zml.Bufferized(LmHead)) void { @@ -338,7 +343,7 @@ pub const DecoderLayer = struct { layer: DecoderLayer, hidden: zml.Tensor, tokens_position_offset: zml.Tensor, - actual_seq_len: zml.Tensor, + active_length: zml.Tensor, cache: Cache, conv_cache_index: zml.Tensor, kv_cache_index: zml.Tensor, @@ -388,7 +393,7 @@ pub const DecoderLayer = struct { const residual, const updated_conv_cache = operator.forward( self.operator_norm.forward(input.hidden), input.tokens_position_offset, - input.actual_seq_len, + input.active_length, cache.conv, conv_cache_index, input.conv_parameters, @@ -451,7 +456,7 @@ pub const ShortConv = struct { }; } - pub fn forward(self: ShortConv, input: zml.Tensor, tokens_position_offset: zml.Tensor, actual_seq_len: zml.Tensor, cache_: ConvCache, cache_index: zml.Tensor, parameters: ConvParameters) struct { zml.Tensor, ConvCache } { + pub fn forward(self: ShortConv, input: zml.Tensor, tokens_position_offset: zml.Tensor, active_length: zml.Tensor, cache_: ConvCache, cache_index: zml.Tensor, parameters: ConvParameters) struct { zml.Tensor, ConvCache } { var cache = cache_; const BCx = self.in_proj.forward(input.withPartitioning(.model, .{ .d = .replicated })); @@ -459,10 +464,10 @@ pub const ShortConv = struct { const Bx = B.mul(x); const conv_out = if (parameters.is_prefill) b: { - const actual_seq_len_i32 = actual_seq_len.convert(.i32); - const start = actual_seq_len_i32.sub(zml.Tensor.scalar(@as(i32, @intCast(self.config.conv_L_cache)), .i32)).maximum(zml.Tensor.scalar(@as(i32, 0), .i32)); + const active_length_i32 = active_length.convert(.i32); + const start = active_length_i32.sub(.scalar(self.config.conv_L_cache, .i32)).maximum(.scalar(0, .i32)); const cache_seq_indices = lkp: { - const n = actual_seq_len_i32.sub(start); + const n = active_length_i32.sub(start); const left_pad = zml.Tensor.scalar(@as(i32, @intCast(self.config.conv_L_cache)), .i32).sub(n); const sh = tokens_position_offset.shape().insert(.last, .{ .seq = self.config.conv_L_cache }); break :lkp zml.Tensor.iota(sh, .seq).convert(.u32).add(left_pad.convert(.u32).broad(sh)).broad(sh); diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig index 8951c528..ecc5abf8 100644 --- a/examples/llm/models/lfm2/session.zig +++ b/examples/llm/models/lfm2/session.zig @@ -127,9 +127,9 @@ pub const Session = struct { var tokens_pos_buf: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice); defer tokens_pos_buf.deinit(); - const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(all_tokens.len)})); - var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice); - defer actual_seq_len_buf.deinit(); + const active_length_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(all_tokens.len)})); + var active_length_buf: zml.Buffer = try .fromSlice(self.io, self.platform, active_length_slice); + defer active_length_buf.deinit(); const params = self.compiled_model.params; var attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = switch (params.attention_metadata) { @@ -146,18 +146,20 @@ pub const Session = struct { var kv_cache_index_buffer: zml.Buffer = try .scalar(self.io, self.platform, 0, .u32); defer kv_cache_index_buffer.deinit(); + var next_token_buf: zml.Buffer = undefined; inference.run(&self.prefill, .{ .io = self.io, .tokens_buf = &tokens_buf, + .next_token_buf = &next_token_buf, .tokens_pos_buf = &tokens_pos_buf, - .actual_seq_len_buf = &actual_seq_len_buf, + .active_length_buf = &active_length_buf, .rng_buf = &self.rng_buf, .cache_buffers = &self.cache_buffers, .attention_metadata_buffers = attention_metadata_buffers, }, &conv_cache_index_buffer, &kv_cache_index_buffer); - try tokens_buf.toSlice(self.io, tokens_slice); - self.generated_token_slice.items(u32)[0] = tokens_slice.items(u32)[all_tokens.len - 1]; + defer next_token_buf.deinit(); + try next_token_buf.toSlice(self.io, self.generated_token_slice); } pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { @@ -167,9 +169,9 @@ pub const Session = struct { var current_token_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, self.generated_token_slice); defer current_token_buffer.deinit(); - const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{0})); - var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice); - defer actual_seq_len_buf.deinit(); + const active_length_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{1})); + var active_length_buf: zml.Buffer = try .fromSlice(self.io, self.platform, active_length_slice); + defer active_length_buf.deinit(); const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); defer self.allocator.free(out_tokens_buffer); @@ -208,8 +210,9 @@ pub const Session = struct { inference.run(&self.decode, .{ .io = self.io, .tokens_buf = ¤t_token_buffer, + .next_token_buf = ¤t_token_buffer, .tokens_pos_buf = &token_pos_buffer, - .actual_seq_len_buf = &actual_seq_len_buf, + .active_length_buf = &active_length_buf, .rng_buf = &self.rng_buf, .cache_buffers = &self.cache_buffers, .attention_metadata_buffers = attention_metadata_buffers, diff --git a/examples/llm/models/llama/inference.zig b/examples/llm/models/llama/inference.zig index 4555e3d4..51ea3f9c 100644 --- a/examples/llm/models/llama/inference.zig +++ b/examples/llm/models/llama/inference.zig @@ -48,7 +48,9 @@ pub const CompilationOptions = CompilationParameters; pub const Args = struct { io: std.Io, tokens_buf: *zml.Buffer, + next_token_buf: *zml.Buffer, token_index_buf: *zml.Buffer, + active_length_buf: *zml.Buffer, kv_cache_buffers: *zml.Bufferized(model.KvCache), rng_buffers: *zml.Bufferized(zml.Tensor.Rng), attention_metadata_buffers: *const zml.Bufferized(zml.attention.Metadata), @@ -94,6 +96,7 @@ const Forward = struct { weights: model.Model, tokens: zml.Tensor, token_index: zml.Tensor, + active_length: zml.Tensor, kv_cache: model.KvCache, rng: zml.Tensor.Rng, attention_metadata: zml.attention.Metadata, @@ -129,11 +132,11 @@ const Forward = struct { const result = model.LmHead.forward(.{ .lm_head = .init(input.weights), .hidden = hidden, - .tokens = input.tokens, + .active_length = input.active_length, .rng = input.rng, }); return .{ - .tokens = result.tokens, + .tokens = if (input.tokens.dim(.s) == 1) result.tokens.reuseBuffer(input.tokens) else result.tokens, .kv_cache = kv_cache.reuseBuffer(input.kv_cache), .rng = .{ ._state = result.rng._state.reuseBuffer(input.rng._state) }, }; @@ -159,12 +162,13 @@ pub fn run(runner: *KernelRunner, args: Args) void { .inputs = .{ .tokens = args.tokens_buf.*, .token_index = args.token_index_buf.*, + .active_length = args.active_length_buf.*, .kv_cache = args.kv_cache_buffers.*, .rng = args.rng_buffers.*, .attention_metadata = args.attention_metadata_buffers.*, }, .outputs = .{ - .tokens = args.tokens_buf, + .tokens = args.next_token_buf, .kv_cache = args.kv_cache_buffers, .rng = args.rng_buffers, }, @@ -196,6 +200,7 @@ fn compileKernel( .weights = llama_model, .tokens = zml.Tensor.init(.{ .s = seqlen }, .u32), .token_index = parameters.token_index, + .active_length = .init(.{}, .u32), .kv_cache = parameters.kv_cache, .rng = parameters.rng, .attention_metadata = parameters.attention_metadata, diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index 7d474593..05610dc1 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -211,6 +211,7 @@ pub const Model = struct { self: Model, tokens_: zml.Tensor, token_index: zml.Tensor, + active_length: zml.Tensor, kv_cache: KvCache, rng: zml.Tensor.Rng, attention_metadata: zml.attention.Metadata, @@ -229,7 +230,7 @@ pub const Model = struct { const sample = LmHead.forward(.{ .lm_head = self.lmHead(), .hidden = out, - .tokens = tokens, + .active_length = active_length, .rng = rng, }); @@ -350,7 +351,7 @@ pub const LmHead = struct { pub const Input = struct { lm_head: LmHead, hidden: zml.Tensor, - tokens: zml.Tensor, + active_length: zml.Tensor, rng: zml.Tensor.Rng, }; @@ -361,8 +362,8 @@ pub const LmHead = struct { pub fn forward(input: Input) Output { const self = input.lm_head; - const tokens = input.tokens.withPartialTags(.{.s}); - const hidden = self.norm.forward(input.hidden.withPartialTags(.{ .s, .d })); + const last_index = input.active_length.convert(.i32).addConstant(-1); + const hidden = self.norm.forward(input.hidden.withPartialTags(.{ .s, .d }).slice(.s, .dyn(last_index, 1))); var logits = blk: { if (self.lm_head) |lm_head| { @@ -376,7 +377,7 @@ pub const LmHead = struct { logits = logits.rename(.{ .d = .voc }); const next_tokens, const new_rng = zml.nn.sampleTokens(logits, self.gen_opts, input.rng); - return .{ .tokens = next_tokens.convert(tokens.dtype()).reuseBuffer(tokens), .rng = new_rng }; + return .{ .tokens = next_tokens.convert(.u32), .rng = new_rng }; } }; diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig index 9b801118..8a6532e6 100644 --- a/examples/llm/models/llama/session.zig +++ b/examples/llm/models/llama/session.zig @@ -139,6 +139,8 @@ pub const Session = struct { var prefill_tokens_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, prefill_tokens_slice); defer prefill_tokens_buffer.deinit(); + var active_length_buffer: zml.Buffer = try .scalar(self.io, self.platform, all_tokens.len, .u32); + defer active_length_buffer.deinit(); const params = self.compiled_model.params; var attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = switch (params.attention_metadata) { @@ -150,17 +152,20 @@ pub const Session = struct { }; defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); + var next_token_buffer: zml.Buffer = undefined; inference.run(&self.prefill, .{ .io = self.io, .tokens_buf = &prefill_tokens_buffer, + .next_token_buf = &next_token_buffer, .token_index_buf = &self.token_index_buffers[0], + .active_length_buf = &active_length_buffer, .kv_cache_buffers = &self.kv_cache_buffers, .rng_buffers = &self.rng_buffers, .attention_metadata_buffers = &attention_metadata_buffers, }); - try prefill_tokens_buffer.toSlice(self.io, prefill_tokens_slice); - self.last_generated_token = prefill_tokens_slice.items(u32)[all_tokens.len - 1]; + defer next_token_buffer.deinit(); + self.last_generated_token = try next_token_buffer.getValue(u32, self.io); } pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { @@ -173,6 +178,8 @@ pub const Session = struct { var last_token_id: u32 = self.last_generated_token; var current_token_buffer: zml.Buffer = try .fromBytes(self.io, self.platform, .init(.{ .s = 1 }, .u32), @ptrCast(&last_token_id)); defer current_token_buffer.deinit(); + var active_length_buffer: zml.Buffer = try .scalar(self.io, self.platform, 1, .u32); + defer active_length_buffer.deinit(); const params = self.compiled_model.params; var attention_metadata_buffers = try params.attention_metadata.initBuffer(self.io, self.platform); @@ -190,7 +197,9 @@ pub const Session = struct { inference.run(&self.decode, .{ .io = self.io, .tokens_buf = ¤t_token_buffer, + .next_token_buf = ¤t_token_buffer, .token_index_buf = &self.token_index_buffers[all_tokens.items.len], + .active_length_buf = &active_length_buffer, .kv_cache_buffers = &self.kv_cache_buffers, .rng_buffers = &self.rng_buffers, .attention_metadata_buffers = &attention_metadata_buffers, diff --git a/examples/llm/models/qwen3_5/inference.zig b/examples/llm/models/qwen3_5/inference.zig index 6a52f064..50d3da30 100644 --- a/examples/llm/models/qwen3_5/inference.zig +++ b/examples/llm/models/qwen3_5/inference.zig @@ -36,6 +36,7 @@ pub const CompilationOptions = CompilationParameters; pub const Args = struct { io: std.Io, tokens_buf: *zml.Buffer, + next_token_buf: *zml.Buffer, token_index_buf: *zml.Buffer, active_length_buf: *zml.Buffer, kv_cache_buffers: *zml.Bufferized(model.KvCache), @@ -205,11 +206,13 @@ pub fn run(runner: *KernelRunner, args: Args, layer_index_buffers: []const zml.B runner.sample.run(args.io, .{ .inputs = .{ .hidden = hidden_buffer, + .tokens = args.tokens_buf.*, + .active_length = args.active_length_buf.*, .rng = args.rng_buffers.*, .token_index = args.token_index_buf.*, }, .outputs = .{ - .tokens = args.tokens_buf, + .tokens = args.next_token_buf, .rng = args.rng_buffers, .token_index = args.token_index_buf, }, @@ -297,6 +300,8 @@ fn compileSample(allocator: std.mem.Allocator, io: std.Io, platform: *const zml. return zml.FnExe(model.Sampler.sampleTokens).compile(allocator, io, platform, .{ .meshes = ¶meters.meshes.all(), .program_name = phase.programName("qwen3_5", "sampler") }, .{.{ .sampler = mdl.sampler(), .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), + .tokens = .init(.{ .b = 1, .s = seqlen }, .u32), + .active_length = .init(.{}, .u32), .rng = parameters.rng, .token_index = parameters.token_index, }}); diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index ac786f05..77d3884e 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -222,10 +222,12 @@ pub const Model = struct { const result = Sampler.sampleTokens(.{ .sampler = self.sampler(), .hidden = text_model_output, + .tokens = tokens, + .active_length = active_length, .rng = rng, .token_index = token_index, }); - return .{ result.tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, result.rng }; + return .{ result.tokens, updated_kv_cache, result.rng }; } pub fn sampler(self: Model) Sampler { @@ -245,6 +247,8 @@ pub const Sampler = struct { pub const Input = struct { sampler: Sampler, hidden: zml.Tensor, + tokens: zml.Tensor, + active_length: zml.Tensor, rng: zml.Tensor.Rng, token_index: zml.Tensor, }; @@ -257,13 +261,15 @@ pub const Sampler = struct { pub fn sampleTokens(input: Input) Output { const self = input.sampler; - const x = self.norm.forward(input.hidden); + const last_index = input.active_length.convert(.i32).addConstant(-1); + const x = self.norm.forward(input.hidden.slice(.s, .dyn(last_index, 1))); const logits = self.lm_head.forward(x.withPartialTags(.{.d}), x.dtype()).rename(.{ .dout = .voc }); const next_tokens, const new_rng = zml.nn.sampleTokens(logits, self.gen_options.sampling_strategy, input.rng); + const tokens = next_tokens.convert(.u32); return .{ - .tokens = next_tokens.convert(.u32), + .tokens = if (input.tokens.dim(.s) == 1) tokens.reuseBuffer(input.tokens) else tokens, .rng = new_rng, - .token_index = input.token_index.addConstant(1), + .token_index = input.token_index.addConstant(1).reuseBuffer(input.token_index), }; } }; diff --git a/examples/llm/models/qwen3_5/session.zig b/examples/llm/models/qwen3_5/session.zig index 84963773..5121fd30 100644 --- a/examples/llm/models/qwen3_5/session.zig +++ b/examples/llm/models/qwen3_5/session.zig @@ -119,18 +119,19 @@ pub const Session = struct { var active_length_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.len)), .u32); defer active_length_buffer.deinit(); + var next_token_buffer: zml.Buffer = undefined; inference.run(&self.prefill, .{ .io = self.io, .tokens_buf = &prefill_tokens_buffer, + .next_token_buf = &next_token_buffer, .token_index_buf = &prefill_token_index_buffer, .active_length_buf = &active_length_buffer, .kv_cache_buffers = &self.kv_cache_buffers, .rng_buffers = &self.rng_buffers, }, self.layer_index_buffers); - try prefill_tokens_buffer.toSlice(self.io, prefill_tokens_slice); - const generated_token = prefill_tokens_slice.items(u32)[all_tokens.len - 1]; - self.generated_token_slice.items(u32)[0] = generated_token; + defer next_token_buffer.deinit(); + try next_token_buffer.toSlice(self.io, self.generated_token_slice); } pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { @@ -168,6 +169,7 @@ pub const Session = struct { inference.run(&self.decode, .{ .io = self.io, .tokens_buf = ¤t_token_buffer, + .next_token_buf = ¤t_token_buffer, .token_index_buf = &token_index_buffer, .active_length_buf = &active_length_buffer, .kv_cache_buffers = &self.kv_cache_buffers, diff --git a/examples/llm/models/qwen3_5_moe/inference.zig b/examples/llm/models/qwen3_5_moe/inference.zig index ac0777d4..896744b3 100644 --- a/examples/llm/models/qwen3_5_moe/inference.zig +++ b/examples/llm/models/qwen3_5_moe/inference.zig @@ -38,6 +38,7 @@ pub const LayerIndexBuffer = union(enum) { pub const RunArgs = struct { io: std.Io, tokens_buffer: *zml.Buffer, + next_token_buffer: *zml.Buffer, token_index_buffer: *zml.Buffer, active_length_buffer: *zml.Buffer, kv_cache_buffers: *zml.Bufferized(model.KvCache), @@ -216,11 +217,13 @@ pub fn run(runner: *KernelRunner, args: RunArgs) void { runner.sample.run(args.io, .{ .inputs = .{ .hidden = hidden_buffer, + .tokens = args.tokens_buffer.*, + .active_length = args.active_length_buffer.*, .rng = args.rng_buffers.*, .token_index = args.token_index_buffer.*, }, .outputs = .{ - .tokens = args.tokens_buffer, + .tokens = args.next_token_buffer, .rng = args.rng_buffers, .token_index = args.token_index_buffer, }, @@ -307,6 +310,8 @@ fn compileSample(allocator: std.mem.Allocator, io: std.Io, platform: *const zml. return compileExe(allocator, io, platform, model.Sampler.sampleTokens, .{.{ .sampler = mdl.text_model.sampler(), .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), + .tokens = .init(.{ .b = 1, .s = seqlen }, .u32), + .active_length = .init(.{}, .u32), .rng = parameters.rng, .token_index = zml.Tensor.init(.{}, .u32), }}, parameters, progress, phase, "sampling"); diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 75719aed..11ce8151 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -235,7 +235,7 @@ pub const Model = struct { ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { const tokens = tokens_.withPartialTags(.{.s}); const new_tokens, const updated_kv_cache, const new_rng = self.text_model.forward(tokens, token_index, active_length, kv_cache, self.config, rng, moe_backend); - return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; + return .{ new_tokens, updated_kv_cache, new_rng }; } }; @@ -250,6 +250,8 @@ pub const Sampler = struct { pub const Input = struct { sampler: Sampler, hidden: zml.Tensor, + tokens: zml.Tensor, + active_length: zml.Tensor, rng: zml.Tensor.Rng, token_index: zml.Tensor, }; @@ -262,13 +264,15 @@ pub const Sampler = struct { pub fn sampleTokens(input: Input) Output { const self = input.sampler; - const x = self.norm.forward(input.hidden); + const last_index = input.active_length.convert(.i32).addConstant(-1); + const x = self.norm.forward(input.hidden.slice(.s, .dyn(last_index, 1))); const logits = self.lm_head.forward(x.withPartialTags(.{.d}), x.dtype()).rename(.{ .dout = .voc }); const next_tokens, const new_rng = zml.nn.sampleTokens(logits, self.gen_options.sampling_strategy, input.rng); + const tokens = next_tokens.convert(.u32); return .{ - .tokens = next_tokens.convert(.u32), + .tokens = if (input.tokens.dim(.s) == 1) tokens.reuseBuffer(input.tokens) else tokens, .rng = new_rng, - .token_index = input.token_index.addConstant(1), + .token_index = input.token_index.addConstant(1).reuseBuffer(input.token_index), }; } }; @@ -368,6 +372,8 @@ pub const TextModel = struct { const result = Sampler.sampleTokens(.{ .sampler = self.sampler(), .hidden = hidden_states, + .tokens = tokens, + .active_length = active_length, .rng = rng, .token_index = token_index, }); diff --git a/examples/llm/models/qwen3_5_moe/session.zig b/examples/llm/models/qwen3_5_moe/session.zig index 35998797..6a819f26 100644 --- a/examples/llm/models/qwen3_5_moe/session.zig +++ b/examples/llm/models/qwen3_5_moe/session.zig @@ -128,9 +128,11 @@ pub const Session = struct { var active_length_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.len)), .u32); defer active_length_buffer.deinit(); + var next_token_buffer: zml.Buffer = undefined; inference.run(&self.prefill, .{ .io = self.io, .tokens_buffer = &tokens_buffer, + .next_token_buffer = &next_token_buffer, .token_index_buffer = &token_index_buffer, .active_length_buffer = &active_length_buffer, .kv_cache_buffers = &self.kv_cache_buffers, @@ -138,8 +140,8 @@ pub const Session = struct { .layer_index_buffers = self.layer_index_buffers, }); - try tokens_buffer.toSlice(self.io, tokens_slice); - self.generated_token_slice.items(u32)[0] = tokens_slice.items(u32)[all_tokens.len - 1]; + defer next_token_buffer.deinit(); + try next_token_buffer.toSlice(self.io, self.generated_token_slice); } pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { @@ -171,6 +173,7 @@ pub const Session = struct { inference.run(&self.decode, .{ .io = self.io, .tokens_buffer = ¤t_token_buffer, + .next_token_buffer = ¤t_token_buffer, .token_index_buffer = &token_index_buffer, .active_length_buffer = &active_length_buffer, .kv_cache_buffers = &self.kv_cache_buffers, From b168417cce5017ae424126d1c2b3301850a1fc72 Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Wed, 21 Oct 2026 13:20:31 +0200 Subject: [PATCH 809/881] zml/kernels: rework sparse GQA (#945) Move sparse GQA into an standalone kernel rather than maintaining it as a separate branch of the unified attention kernel. This separation improves maintainability and keeps the unified attention kernel more manageable, as previous performance optimizations had significantly increased its complexity. To reviewer: there's remaining work to merge the oneapi kernel into unified attention and clean the kernels/kernels_sparse import. --- .../paged_attention/triton/BUILD.bazel | 1 + zml/kernels/paged_attention/triton/README.md | 34 +- .../triton/sparse_unified_attention.zig | 318 ++++++++++++++++++ zml/kernels/paged_attention/triton/triton.zig | 304 ++++++++++++++--- .../triton/unified_attention.zig | 116 ++----- 5 files changed, 639 insertions(+), 134 deletions(-) create mode 100644 zml/kernels/paged_attention/triton/sparse_unified_attention.zig diff --git a/zml/kernels/paged_attention/triton/BUILD.bazel b/zml/kernels/paged_attention/triton/BUILD.bazel index fbce732b..eff7d096 100644 --- a/zml/kernels/paged_attention/triton/BUILD.bazel +++ b/zml/kernels/paged_attention/triton/BUILD.bazel @@ -3,6 +3,7 @@ package(default_visibility = ["//zml:__pkg__"]) filegroup( name = "triton", srcs = [ + "sparse_unified_attention.zig", "triton.zig", "unified_attention.zig", "unified_attention_oneapi.zig", diff --git a/zml/kernels/paged_attention/triton/README.md b/zml/kernels/paged_attention/triton/README.md index aca6b52e..68526150 100644 --- a/zml/kernels/paged_attention/triton/README.md +++ b/zml/kernels/paged_attention/triton/README.md @@ -11,9 +11,10 @@ when sharding `.hkv`, with matching query and cache ownership and complete `.hg` and `.hd` axes. Each local computation needs its full key range and the matching sequence metadata. -The wrapper chooses between a 2D kernel and a 3D split-key kernel followed by -segment reduction. Prefill uses the 2D path; all-decode batches select a path -based on batch size and KV head count. oneAPI forces split-key execution for +For standard masks, the wrapper chooses between a 2D kernel and a 3D split-key +kernel followed by segment reduction. Prefill uses the 2D path; all-decode +batches select a path based on batch size and KV head count. Explicit indices +use the dedicated sparse path described below. oneAPI forces split-key execution for all-decode batches and has a specialized path for standard masks. Segment reduction combines softmax maxima, exponential sums, and weighted values to preserve one normalization over the key range. @@ -26,3 +27,30 @@ The shared tests in [paged_attention.zig](../paged_attention.zig) exercise trailing inactive sequences, and option-specific comparisons. Kernel selection and numerical behavior depend on the target; these cases do not establish support for every shape or dtype. + +## Sparse attention + +`.indices` mask dispatches to a dedicated sparse kernel in `unified_attention.zig`. +The `i32` tensor has shape `{ .b = T, .topk = K }`, with one list per packed +query shared across heads. Each entry is a logical position within that query's +sequence, including its cached prefix. The kernel translates each selected +position through the existing page table; it does not compute top-k selection. +Negative and out-of-sequence entries are ignored, including holes before valid +tail entries. Duplicate entries count repeatedly. Indices replace the standard +mask, so they do not implicitly enforce causality. Empty selections return zero; +optional attention sinks contribute only to the softmax denominator. + +Prefill processes one query and one KV-head group per program and writes the +normalized result directly. Decode uses 16- or 32-position gather tiles and an +occupancy-based split count bounded by selection length. Single-token decode +addresses sequence slots directly; other query layouts use packed-query lookup. +Inactive sequence slots and unused query capacity are skipped. Only segments +covering selection slots launch. Small empty segments skip Q and KV loading, +and larger segments skip wholly empty tiles. Known strides and scale are +specialized, and sparse gathers use the default cache policy. + +Split-K decode writes FP32 partials and contiguous-head statistics. Its reducer +handles at most 64 features per program, ignores unlaunched and zero-mass +segments, and never reads their unwritten partial outputs. The direct and split +paths use the same online softmax and support non-power-of-two GQA groups and +padded head dimensions. Dense attention retains its existing dispatch and kernels. diff --git a/zml/kernels/paged_attention/triton/sparse_unified_attention.zig b/zml/kernels/paged_attention/triton/sparse_unified_attention.zig new file mode 100644 index 00000000..04f5bb71 --- /dev/null +++ b/zml/kernels/paged_attention/triton/sparse_unified_attention.zig @@ -0,0 +1,318 @@ +const std = @import("std"); +const zml = @import("../../../zml.zig"); +const tri = zml.kernel.triton; +const Builder = tri.Builder; +const Value = tri.Value; +const DType = tri.DType; + +/// `log_2(e)` — pre-multiply with this to turn `exp(x)` into `exp2(x * RCP_LN2)`. +const RCP_LN2: f32 = 1.4426950408889634; + +/// Partition selection slots, not logical KV positions. Scratch capacity may be +/// larger than the number of segments that have any selection slots to visit. +pub const SparseSegments = struct { + tiles_per_segment: i64, + span: i64, + launched: i64, + + pub fn init(count: i64, capacity: i64, tile: i64) SparseSegments { + std.debug.assert(count > 0 and capacity > 0 and tile > 0); + const tiles = std.math.divCeil(i64, count, capacity * tile) catch unreachable; + return .{ + .tiles_per_segment = tiles, + .span = tiles * tile, + .launched = std.math.divCeil(i64, count, tiles * tile) catch unreachable, + }; + } +}; + +pub const SparseAttentionConfig = struct { + dtype: DType, + num_seqs: i32, + num_query_heads: i64, + num_queries_per_kv: i64, + head_size: i64, + head_size_padded: i64, + block_m: i64, + block_size: i64, + tile_size: i64, + indices_count: i64, + num_segments: i64 = 1, + decode_sequence_grid: bool = false, + use_sinks: bool = false, + scale: f32, + block_table_stride: i64, + query_strides: [3]i64, + key_strides: [4]i64, + value_strides: [4]i64, + indices_strides: [2]i64, +}; + +fn findSeqIdx( + k: *Builder, + query_start_len_ptr: Value, + target_idx: Value, + num_seqs: Value, + block_q: i64, + use_q_block_mode: bool, +) Value { + const left_init = k.liftAs(0, .i32); + + var w = k.openWhile(.{ left_init, num_seqs }, .{ k.scalarTy(.i32), k.scalarTy(.i32) }); + { + const left = w.before_carried[0]; + const right = w.before_carried[1]; + w.yieldBefore(left.lt(right), .{ left, right }); + } + { + const left = w.after_carried[0]; + const right = w.after_carried[1]; + const mid = left.add(right).div(2); + const val = k.load(query_start_len_ptr.addPtr(mid)); + const mid_val = if (use_q_block_mode) + val.div(@as(i32, @intCast(block_q))).add(mid) + else + val; + + var i = k.openIfElse(mid_val.le(target_idx), .{ k.scalarTy(.i32), k.scalarTy(.i32) }); + { + i.yieldThen(.{ mid.add(1), right }); + } + { + i.yieldElse(.{ left, mid }); + } + w.yieldAfter(.{ i.results[0], i.results[1] }); + } + + return w.results[0].sub(1); +} + +/// One program processes all selected keys for one query/KV-head group and write the normalized output. +pub const KernelSparseAttention2dPtr = KernelSparseAttentionPtr(false); + +/// Also known as split-K in the litterature, this 3d variant uses multiple programs process different +/// portions of that same selection list. They write partial results, then a reduction kernels combines +/// them into one normalized output. +/// +/// Mostly useful during decode when there are few queries and a long top-k list. +pub const KernelSparseAttention3dPtr = KernelSparseAttentionPtr(true); + +/// One query and one GQA group per program. The direct specialization writes +/// normalized output; split decode writes FP32 partials for SparseReducePtr. +fn KernelSparseAttentionPtr(comptime split_k: bool) type { + return struct { + pub const Config = SparseAttentionConfig; + pub const Kernel = tri.Kernel(Config, .{ + .name = if (split_k) "kernel_sparse_attention_3d_ptr" else "kernel_sparse_attention_2d_ptr", + .inputs = &.{ + "query_ptr", "key_ptr", "value_ptr", "indices_ptr", + "block_tables_ptr", "seq_lens_ptr", "query_start_len_ptr", "sink_ptr", + }, + .outputs = if (split_k) &.{ "output", "segm_max", "segm_expsum" } else &.{"output"}, + .run = run, + }); + + fn run(k: *Builder, cfg: Config) tri.FinishError!void { + const a = if (split_k) try k.declareArgs(.{ + .query_ptr = .{ .ptr = cfg.dtype }, + .key_ptr = .{ .ptr = cfg.dtype }, + .value_ptr = .{ .ptr = cfg.dtype }, + .indices_ptr = .{ .ptr = .i32 }, + .block_tables_ptr = .{ .ptr = .i32 }, + .seq_lens_ptr = .{ .ptr = .i32 }, + .query_start_len_ptr = .{ .ptr = .i32 }, + .sink_ptr = .{ .ptr = .f32 }, + .output_ptr = .{ .ptr = .f32 }, + .segm_max_ptr = .{ .ptr = .f32 }, + .segm_expsum_ptr = .{ .ptr = .f32 }, + }) else try k.declareArgs(.{ + .query_ptr = .{ .ptr = cfg.dtype }, + .key_ptr = .{ .ptr = cfg.dtype }, + .value_ptr = .{ .ptr = cfg.dtype }, + .indices_ptr = .{ .ptr = .i32 }, + .block_tables_ptr = .{ .ptr = .i32 }, + .seq_lens_ptr = .{ .ptr = .i32 }, + .query_start_len_ptr = .{ .ptr = .i32 }, + .sink_ptr = .{ .ptr = .f32 }, + .output_ptr = .{ .ptr = cfg.dtype }, + }); + + std.debug.assert(cfg.block_m >= cfg.num_queries_per_kv); + std.debug.assert(split_k or cfg.num_segments == 1); + + const partition = SparseSegments.init(cfg.indices_count, cfg.num_segments, cfg.tile_size); + const program = k.programId(.x); + const num_seqs = k.liftAs(cfg.num_seqs, .i32); + if (!cfg.decode_sequence_grid) { + const active_queries = k.load(a.query_start_len_ptr.addPtr(num_seqs)); + k.returnIf(program.ge(active_queries), .{}); + } + + const seq = if (cfg.decode_sequence_grid) program else findSeqIdx(k, a.query_start_len_ptr, program, num_seqs, 1, false); + const seq_len = k.load(a.seq_lens_ptr.addPtr(seq)); + const query_start = k.load(a.query_start_len_ptr.addPtr(seq)); + const query_end = k.load(a.query_start_len_ptr.addPtr(seq).addPtr(1)); + if (cfg.decode_sequence_grid) k.returnIf(query_start.eq(query_end), .{}); + const query = if (cfg.decode_sequence_grid) query_start else program; + const kv_head = if (cfg.num_query_heads == cfg.num_queries_per_kv) k.liftAs(0, .i32) else k.programId(.y); + + // Launch the partial tail first, without launching unused capacity. + const slot = if (split_k) k.programId(.z) else k.liftAs(0, .i32); + const segment = if (split_k) k.where(slot.eq(0), k.liftAs(@as(i32, @intCast(partition.launched - 1)), .i32), slot.sub(1)) else slot; + const segment_start = segment.mul(@as(i32, @intCast(partition.span))); + const heads = k.arange(0, cfg.block_m, .i32); + const head = kv_head.mul(@as(i32, @intCast(cfg.num_queries_per_kv))).add(heads); + const head_mask = heads.lt(@as(i32, @intCast(cfg.num_queries_per_kv))); + const dims = k.arange(0, cfg.head_size_padded, .i32); + const dim_mask = dims.lt(@as(i32, @intCast(cfg.head_size))); + const indices_row = a.indices_ptr.addPtr(query.to(.i64).mul(cfg.indices_strides[0])); + + const sink = if (cfg.use_sinks) k.loadOpts(a.sink_ptr.addPtr(head), .{ + .mask = head_mask.bitAnd(segment.eq(0)), + .other = k.full(&.{cfg.block_m}, -std.math.inf(f32), .f32), + }).mul(RCP_LN2) else k.full(&.{cfg.block_m}, -std.math.inf(f32), .f32); + const has_sink = if (cfg.use_sinks) segment.eq(0).bitAnd(head_mask) else k.full(&.{cfg.block_m}, 0, .i1); + const initial_max = k.where(has_sink, sink, k.full(&.{cfg.block_m}, -std.math.inf(f32), .f32)); + const initial_sum = has_sink.to(.f32); + const initial_acc = k.zeros(&.{ cfg.block_m, cfg.head_size_padded }, .f32); + + if (partition.span <= 256) { + const scan_size: i64 = @intCast(std.math.ceilPowerOfTwoAssert(u64, @intCast(partition.span))); + const selected = segment_start.add(k.arange(0, scan_size, .i32)); + const scan_mask = selected.lt(cfg.indices_count).bitAnd(selected.lt(segment_start.add(@as(i32, @intCast(partition.span))))); + const positions = k.loadOpts(indices_row.addPtr(selected.to(.i64).mul(cfg.indices_strides[1])), .{ + .mask = scan_mask, + .other = k.full(&.{scan_size}, -1, .i32), + }); + const empty = k.sum(scan_mask.bitAnd(positions.ge(0)).bitAnd(positions.lt(seq_len)).to(.i32)).eq(0); + var branch = k.openIfElse(empty, .{}); + storeResult(k, cfg, a, query, head, head_mask, segment, dims, dim_mask, initial_max, initial_sum, initial_acc); + branch.yieldThen(.{}); + branch.yieldElse(.{}); + k.returnIf(empty, .{}); + } + + const q_offset = query.to(.i64).mul(cfg.query_strides[0]) + .add(head.to(.i64).mul(cfg.query_strides[1]).expandDims(1)) + .add(dims.to(.i64).mul(cfg.query_strides[2]).expandDims(0)); + const Q = k.loadOpts(a.query_ptr.addPtr(q_offset), .{ + .mask = k.mask2d(head_mask, dim_mask, cfg.block_m, cfg.head_size_padded), + .other = initial_acc.to(cfg.dtype), + }); + const table = a.block_tables_ptr.addPtr(seq.to(.i64).mul(cfg.block_table_stride)); + const offsets = k.arange(0, cfg.tile_size, .i32); + var loop = k.openFor(k.liftAs(0, .i32), k.liftAs(@as(i32, @intCast(partition.tiles_per_segment)), .i32), 1, .{ initial_max, initial_sum, initial_acc }); + { + const selected = segment_start.add(loop.iv.mul(@as(i32, @intCast(cfg.tile_size)))).add(offsets); + const position = k.loadOpts(indices_row.addPtr(selected.to(.i64).mul(cfg.indices_strides[1])), .{ + .mask = selected.lt(cfg.indices_count), + .other = k.full(&.{cfg.tile_size}, -1, .i32), + }); + const valid = selected.lt(cfg.indices_count).bitAnd(position.ge(0)).bitAnd(position.lt(seq_len)); + // A bounded single-tile segment was already checked above. + const nonempty = if (partition.tiles_per_segment == 1 and partition.span <= 256) k.liftAs(1, .i1) else k.sum(valid.to(.i32)).gt(0); + var branch = k.openIfElse(nonempty, .{ k.tensorTy(&.{cfg.block_m}, .f32), k.tensorTy(&.{cfg.block_m}, .f32), k.tensorTy(&.{ cfg.block_m, cfg.head_size_padded }, .f32) }); + const safe_position = k.where(valid, position, k.zeros(&.{cfg.tile_size}, .i32)); + const page = k.loadOpts(table.addPtr(safe_position.div(@as(i32, @intCast(cfg.block_size)))), .{ + .mask = valid, + .other = k.zeros(&.{cfg.tile_size}, .i32), + }).to(.i64); + const in_page = safe_position.rem(@as(i32, @intCast(cfg.block_size))).to(.i64); + const key_offset = page.mul(cfg.key_strides[0]).add(in_page.mul(cfg.key_strides[1])).add(kv_head.to(.i64).mul(cfg.key_strides[2])).expandDims(0) + .add(dims.to(.i64).mul(cfg.key_strides[3]).expandDims(1)); + const value_offset = page.mul(cfg.value_strides[0]).add(in_page.mul(cfg.value_strides[1])).add(kv_head.to(.i64).mul(cfg.value_strides[2])).expandDims(1) + .add(dims.to(.i64).mul(cfg.value_strides[3]).expandDims(0)); + const K = k.loadOpts(a.key_ptr.addPtr(key_offset), .{ + .mask = k.mask2d(dim_mask, valid, cfg.head_size_padded, cfg.tile_size), + .other = k.zeros(&.{ cfg.head_size_padded, cfg.tile_size }, cfg.dtype), + }); + const V = k.loadOpts(a.value_ptr.addPtr(value_offset), .{ + .mask = k.mask2d(valid, dim_mask, cfg.tile_size, cfg.head_size_padded), + .other = k.zeros(&.{ cfg.tile_size, cfg.head_size_padded }, cfg.dtype), + }); + const scores = k.dot(Q, K, k.zeros(&.{ cfg.block_m, cfg.tile_size }, .f32)).mul(cfg.scale * RCP_LN2); + const S = k.where(k.mask2d(head_mask, valid, cfg.block_m, cfg.tile_size), scores, k.full(&.{ cfg.block_m, cfg.tile_size }, -std.math.inf(f32), .f32)); + const max_score = loop.carried[0].maximum(k.maxOpts(S, .{ .axis = 1 })); + const next_max = k.where(max_score.gt(-std.math.inf(f32)), max_score, k.zeros(&.{cfg.block_m}, .f32)); + const alpha = k.exp2(loop.carried[0].sub(next_max)); + const P = k.exp2(S.sub(next_max.expandDims(1))); + const sum = loop.carried[1].mul(alpha).add(k.sumOpts(P, .{ .axis = 1 })); + const acc = k.dot(P.to(cfg.dtype), V, loop.carried[2].mul(alpha.expandDims(1))); + branch.yieldThen(.{ next_max, sum, acc }); + branch.yieldElse(.{ loop.carried[0], loop.carried[1], loop.carried[2] }); + loop.yield(.{ branch.results[0], branch.results[1], branch.results[2] }); + } + + storeResult(k, cfg, a, query, head, head_mask, segment, dims, dim_mask, loop.results[0], loop.results[1], loop.results[2]); + } + + fn storeResult(k: *Builder, cfg: Config, a: anytype, query: Value, head: Value, head_mask: Value, segment: Value, dims: Value, dim_mask: Value, max_score: Value, sum: Value, acc: Value) void { + if (split_k) { + // Adjacent heads are written together by the producer. + const stats_offset = query.to(.i64).mul(cfg.num_segments).add(segment.to(.i64)).mul(cfg.num_query_heads).add(head.to(.i64)); + k.storeOpts(a.segm_max_ptr.addPtr(stats_offset), max_score, .{ .mask = head_mask }); + k.storeOpts(a.segm_expsum_ptr.addPtr(stats_offset), sum, .{ .mask = head_mask }); + const offset = query.to(.i64).mul(cfg.num_query_heads).add(head.to(.i64)).mul(cfg.num_segments).add(segment.to(.i64)).mul(cfg.head_size_padded).expandDims(1).add(dims.to(.i64).expandDims(0)); + // Zero-mass segments need no partial output; the reducer masks them. + k.storeOpts(a.output_ptr.addPtr(offset), acc, .{ .mask = k.mask2d(head_mask.bitAnd(sum.gt(0.0)), dim_mask, cfg.block_m, cfg.head_size_padded) }); + } else { + const denominator = k.where(sum.gt(0.0), sum, k.full(&.{cfg.block_m}, 1.0, .f32)); + const offset = query.to(.i64).mul(cfg.query_strides[0]).add(head.to(.i64).mul(cfg.query_strides[1]).expandDims(1)).add(dims.to(.i64).mul(cfg.query_strides[2]).expandDims(0)); + k.storeOpts(a.output_ptr.addPtr(offset), acc.div(denominator.expandDims(1)).to(cfg.dtype), .{ .mask = k.mask2d(head_mask, dim_mask, cfg.block_m, cfg.head_size_padded) }); + } + } + }; +} + +pub const SparseReducePtr = struct { + pub const Config = struct { + dtype: DType, + num_seqs: i32, + num_query_heads: i64, + head_size: i64, + head_size_padded: i64, + head_tile: i64, + num_segments: i64, + launched_segments: i64, + output_strides: [3]i64, + }; + pub const Kernel = tri.Kernel(Config, .{ + .name = "reduce_sparse_attention_ptr", + .inputs = &.{ "segm_output_ptr", "segm_max_ptr", "segm_expsum_ptr", "query_start_len_ptr" }, + .outputs = &.{"output"}, + .run = run, + }); + fn run(k: *Builder, cfg: Config) tri.FinishError!void { + const a = try k.declareArgs(.{ + .segm_output_ptr = .{ .ptr = .f32 }, + .segm_max_ptr = .{ .ptr = .f32 }, + .segm_expsum_ptr = .{ .ptr = .f32 }, + .query_start_len_ptr = .{ .ptr = .i32 }, + .output_ptr = .{ .ptr = cfg.dtype }, + }); + const query = k.programId(.x); + k.returnIf(query.ge(k.load(a.query_start_len_ptr.addPtr(cfg.num_seqs))), .{}); + const head = k.programId(.y); + const segments = k.arange(0, cfg.num_segments, .i32); + const segment_mask = segments.lt(cfg.launched_segments); + const dims = k.programId(.z).mul(@as(i32, @intCast(cfg.head_tile))).add(k.arange(0, cfg.head_tile, .i32)); + const dim_mask = dims.lt(cfg.head_size); + const stats = query.to(.i64).mul(cfg.num_segments).add(segments.to(.i64)).mul(cfg.num_query_heads).add(head.to(.i64)); + const maxima = k.loadOpts(a.segm_max_ptr.addPtr(stats), .{ .mask = segment_mask, .other = k.full(&.{cfg.num_segments}, -std.math.inf(f32), .f32) }); + const sums = k.loadOpts(a.segm_expsum_ptr.addPtr(stats), .{ .mask = segment_mask, .other = k.zeros(&.{cfg.num_segments}, .f32) }); + const nonempty = segment_mask.bitAnd(sums.gt(0.0)); + const maximum = k.max(k.where(nonempty, maxima, k.full(&.{cfg.num_segments}, -std.math.inf(f32), .f32))); + const safe_maximum = k.where(maximum.gt(-std.math.inf(f32)), maximum, k.liftAs(0.0, .f32)); + const rescale = k.exp2(maxima.sub(safe_maximum)); + const denominator = k.sum(sums.mul(rescale)); + const offset = query.to(.i64).mul(cfg.num_query_heads).add(head.to(.i64)).mul(cfg.num_segments).add(segments.to(.i64)).mul(cfg.head_size_padded).expandDims(1).add(dims.to(.i64).expandDims(0)); + const partial = k.loadOpts(a.segm_output_ptr.addPtr(offset), .{ + .mask = k.mask2d(nonempty, dim_mask, cfg.num_segments, cfg.head_tile), + .other = k.zeros(&.{ cfg.num_segments, cfg.head_tile }, .f32), + }); + const numerator = k.sumOpts(partial.mul(rescale.expandDims(1)), .{ .axis = 0 }); + const output = numerator.div(k.where(denominator.gt(0.0), denominator, k.liftAs(1.0, .f32))); + const output_offset = query.to(.i64).mul(cfg.output_strides[0]).add(head.to(.i64).mul(cfg.output_strides[1])).add(dims.to(.i64).mul(cfg.output_strides[2])); + k.storeOpts(a.output_ptr.addPtr(output_offset), output.to(cfg.dtype), .{ .mask = dim_mask }); + } +}; diff --git a/zml/kernels/paged_attention/triton/triton.zig b/zml/kernels/paged_attention/triton/triton.zig index b0b69b8d..1ccbeba8 100644 --- a/zml/kernels/paged_attention/triton/triton.zig +++ b/zml/kernels/paged_attention/triton/triton.zig @@ -6,6 +6,7 @@ const zml = @import("../../../zml.zig"); const triton = zml.kernel.triton; const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; const kernels = @import("unified_attention.zig"); +const kernels_sparse = @import("sparse_unified_attention.zig"); const kernels_oneapi = @import("unified_attention_oneapi.zig"); const log = std.log.scoped(.@"zml/attention/triton"); @@ -59,26 +60,7 @@ fn isCudaComputeCapability(expected: zml.platform.cuda.ComputeCapability) bool { /// Triton codegen bug is not diagnosed; this only keeps us out of it. const max_query_tile_elements: usize = 16 * 1024; -fn select2dSparseConfig(options: paged.PagedAttentionOptions) Config2D { - const tile_size: usize = switch (options.indices_count) { - 0...16 => 16, - 17...32 => 32, - else => if (options.head_dim > 128) 32 else 64, - }; - - return .{ - .block_m = @max(16, std.math.ceilPowerOfTwoAssert(usize, options.numQueriesPerKv())), - .block_q = 1, - .tile_size = tile_size, - .num_warps = 4, - .num_stages = 1, - .total_q_blocks = options.num_tokens + options.batch_size, - }; -} - fn select2dConfig(options: paged.PagedAttentionOptions) Config2D { - if (options.indices_count > 0) return select2dSparseConfig(options); - const max_num_stages_2d: usize = if (options.head_dim <= 128) 4 else 2; // Until we test on other platforms, gate the fix to GB300 @@ -145,7 +127,7 @@ fn select3dConfig(options: paged.PagedAttentionOptions) Config3D { var reduce_num_warps: usize = 2; // Intel decode needs more warps to spread the work and avoid register spill. const attn_warps: usize = if (options.all_decode and isOneapiTarget()) 8 else 2; - const tile_size = if (options.indices_count > 0) 32 else options.block_size; + const tile_size = options.block_size; //const MAX_SEGMENTS: usize = @min(128, std.math.divCeil(usize, max_seqlen_k, tile_size)); var num_segments = std.math.divCeil(usize, options.target_num_prgms, options.num_2d_prgms) catch unreachable; @@ -192,6 +174,26 @@ pub fn getCuCount() usize { return @intCast(attribute.int64); } +const SparseConfig = struct { + tile_size: usize, + num_segments: usize, +}; + +fn selectSparseConfig(count: usize, batch_size: usize, num_kv_heads: usize, cu_count: usize, all_decode: bool) SparseConfig { + std.debug.assert(count > 0 and batch_size > 0 and num_kv_heads > 0); + if (!all_decode) return .{ .tile_size = 32, .num_segments = 1 }; + + const target_num_prgms = cu_count * 4; + const occupancy_splits = std.math.ceilPowerOfTwoAssert(usize, std.math.divCeil(usize, target_num_prgms * 4, batch_size * num_kv_heads) catch unreachable); + const tile: usize = if (occupancy_splits > 128) 16 else 32; + const tile_splits = std.math.ceilPowerOfTwoAssert(usize, std.math.divCeil(usize, count, tile) catch unreachable); + + return .{ + .tile_size = tile, + .num_segments = @min(tile_splits, @min(if (tile == 16) @as(usize, 256) else 128, @max(8, occupancy_splits))), + }; +} + pub const paged = struct { pub const Options = struct { batch_size: usize, @@ -257,8 +259,6 @@ pub const paged = struct { num_2d_prgms: usize, max_seqlen_q: usize, scale: ?f32, - /// Number of indices for sparse attention - indices_count: usize, pub fn numQueriesPerKv(self: PagedAttentionOptions) usize { return self.num_heads / self.num_kv_heads; @@ -270,6 +270,10 @@ pub const paged = struct { }; pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { + if (opts.mask == .indices) { + return pagedSparseAttention(parameters, q, k_cache, v_cache, opts); + } + const output = zml.ops.manualComputation( (struct { q: zml.Tensor, @@ -296,12 +300,10 @@ pub const paged = struct { // Intel decode: pack exactly one GQA group per tile (block_q == 1) so the // single decode query token doesn't carry masked-out fp32 acc lanes. // oneAPI decode keeps one GQA group per tile, padded to a power of two so tt.make_range emits legal Triton IR. - const block_m: usize = if (self.opts.mask == .indices) - @max(16, std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv)) - else if (!self.options.is_prefill and isOneapiTarget()) + const block_m: usize = if (!self.options.is_prefill and isOneapiTarget()) std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv) else if (num_queries_per_kv <= 16) 16 else std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv); - const block_q: usize = if (self.opts.mask == .indices) 1 else block_m / num_queries_per_kv; + const block_q: usize = block_m / num_queries_per_kv; const num_tokens: usize = @intCast(self.q.dim(.b)); const num_seqs: usize = @intCast(parameters_.block_table.dim(.b)); const total_q_blocks: usize = num_tokens / block_q + num_seqs; @@ -329,10 +331,6 @@ pub const paged = struct { .num_2d_prgms = num_2d_prgms, .max_seqlen_q = self.options.max_seqlen_q, .scale = self.opts.scale, - .indices_count = switch (self.opts.mask) { - .indices => |indices| @intCast(indices.dim(.topk)), - else => 0, - }, }; const use_2d_kernel = use2dKernel( @@ -342,7 +340,7 @@ pub const paged = struct { ); const output = if (use_2d_kernel) pagedAttention2d(parameters_, self.q, self.k_cache, self.v_cache, self.opts, paged_attention_opts) - else if (isOneapiTarget() and self.opts.mask != .indices) + else if (isOneapiTarget()) pagedAttention3dOneapi(parameters_, self.q, self.k_cache, self.v_cache, self.opts, paged_attention_opts) else pagedAttention3d(parameters_, self.q, self.k_cache, self.v_cache, self.opts, paged_attention_opts); @@ -366,6 +364,239 @@ pub const paged = struct { return output; } + pub fn pagedSparseAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { + stdx.debug.assert(opts.mask == .indices, "Expected indices mask but got {}", .{opts.mask}); + const indices = opts.mask.indices; + stdx.debug.assert(indices.rank() == 2 and indices.dtype() == .i32, "Sparse indices must be an i32 [query, topk] tensor, got {f}", .{indices.shape()}); + stdx.debug.assert(indices.dim(.b) == q.dim(.b) and indices.dim(.topk) > 0, "Sparse indices {f} must match queries {f} with a nonempty selection capacity", .{ indices.shape(), q.shape() }); + stdx.debug.assert(indices.shape().hasTags(.{ .b, .topk }), "Sparse indices expect to have .b, .topk tags", .{}); + stdx.debug.assert(q.dtype() == k_cache.dtype() and q.dtype() == v_cache.dtype(), "Sparse attention requires matching query and KV dtypes", .{}); + + return zml.ops.manualComputation( + (struct { + q: zml.Tensor, + k_cache: zml.Tensor, + v_cache: zml.Tensor, + block_table: zml.Tensor, + seq_lens: zml.Tensor, + query_start_len: zml.Tensor, + opts: AttentionOptions, + options: Options, + + fn body(self: @This(), _: zml.Shape) zml.Tensor { + const parameters_: Parameters = .{ + .block_table = self.block_table, + .seq_lens = self.seq_lens, + .query_start_len = self.query_start_len, + .options_ = self.options, + }; + + const config = selectSparseConfig( + @intCast(self.opts.mask.indices.dim(.topk)), + @intCast(self.block_table.dim(.b)), + @intCast(self.q.dim(.hkv)), + getCuCount(), + !self.options.is_prefill, + ); + + return if (config.num_segments == 1) + pagedSparseAttention2d(parameters_, self.q, self.k_cache, self.v_cache, self.opts, config) + else + pagedSparseAttention3d(parameters_, self.q, self.k_cache, self.v_cache, self.opts, config); + } + }).body, + .{ + .q = q, + .k_cache = k_cache, + .v_cache = v_cache, + .block_table = parameters.block_table, + .seq_lens = parameters.seq_lens, + .query_start_len = parameters.query_start_len, + .opts = opts, + .options = parameters.options_, + }, + q.shape(), + ); + } + + pub fn pagedSparseAttention2d(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, tuning: SparseConfig) zml.Tensor { + std.debug.assert(tuning.num_segments == 1); + + const indices = opts.mask.indices; + + const group = q.dim(.hg); + const num_kv_heads = q.dim(.hkv); + std.debug.assert(num_kv_heads == k_cache.dim(.hkv) and num_kv_heads == v_cache.dim(.hkv)); + std.debug.assert(q.dim(.hd) == k_cache.dim(.hd) and q.dim(.hd) == v_cache.dim(.hd)); + + const all_decode = !parameters.options_.is_prefill; + + const q_shape = q.shape().mergeAxes(.{ .h = .{ .hkv, .hg } }); + + const qs = q_shape.computeElementStrides().constSlice(); + const ks = k_cache.shape().computeElementStrides().constSlice(); + const vs = v_cache.shape().computeElementStrides().constSlice(); + const is = indices.shape().computeElementStrides().constSlice(); + const ts = parameters.block_table.shape().computeElementStrides().constSlice(); + + const head_size_padded: i64 = @intCast(std.math.ceilPowerOfTwoAssert(u64, @intCast(@max(16, q.dim(.hd))))); + + const config: kernels_sparse.SparseAttentionConfig = .{ + .dtype = triton.from(q.dtype()), + .num_seqs = @intCast(parameters.block_table.dim(.b)), + .num_query_heads = num_kv_heads * group, + .num_queries_per_kv = group, + .head_size = q.dim(.hd), + .head_size_padded = head_size_padded, + .block_m = @intCast(@max(16, std.math.ceilPowerOfTwoAssert(u64, @intCast(group)))), + .block_size = k_cache.dim(.k_chunk), + .tile_size = @intCast(tuning.tile_size), + .indices_count = indices.dim(.topk), + .num_segments = @intCast(tuning.num_segments), + .decode_sequence_grid = all_decode and parameters.options_.max_seqlen_q == 1, + .use_sinks = opts.sink != null, + .scale = opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))), + .block_table_stride = ts[parameters.block_table.axis(.b)], + .query_strides = .{ qs[q_shape.axis(.b)], qs[q_shape.axis(.h)], qs[q_shape.axis(.hd)] }, + .key_strides = .{ ks[k_cache.axis(.page)], ks[k_cache.axis(.k_chunk)], ks[k_cache.axis(.hkv)], ks[k_cache.axis(.hd)] }, + .value_strides = .{ vs[v_cache.axis(.page)], vs[v_cache.axis(.k_chunk)], vs[v_cache.axis(.hkv)], vs[v_cache.axis(.hd)] }, + .indices_strides = .{ is[indices.axis(.b)], is[indices.axis(.topk)] }, + }; + + const inputs: kernels_sparse.KernelSparseAttention2dPtr.Kernel.Inputs = .{ + .query_ptr = q, + .key_ptr = k_cache, + .value_ptr = v_cache, + .indices_ptr = indices, + .block_tables_ptr = parameters.block_table, + .seq_lens_ptr = parameters.seq_lens, + .query_start_len_ptr = parameters.query_start_len, + .sink_ptr = if (opts.sink) |sink| sink.convert(.f32) else .scalar(0, .f32), + }; + + const grid: [3]i32 = .{ + if (config.decode_sequence_grid) config.num_seqs else @intCast(q.dim(.b)), + @intCast(num_kv_heads), + 1, + }; + + const warps: i32 = if (all_decode and isOneapiTarget()) 8 else 4; + + const output = kernels_sparse.KernelSparseAttention2dPtr.Kernel.call(inputs, .{ + .output = q.shape(), + }, .{ + .cfg = config, + .grid = grid, + .num_stages = 1, + .num_warps = warps, + }); + + return output.output; + } + + pub fn pagedSparseAttention3d(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, tuning: SparseConfig) zml.Tensor { + std.debug.assert(tuning.num_segments > 1); + + const indices = opts.mask.indices; + const group = q.dim(.hg); + const num_kv_heads = q.dim(.hkv); + std.debug.assert(num_kv_heads == k_cache.dim(.hkv) and num_kv_heads == v_cache.dim(.hkv)); + std.debug.assert(q.dim(.hd) == k_cache.dim(.hd) and q.dim(.hd) == v_cache.dim(.hd)); + + const all_decode = !parameters.options_.is_prefill; + + const q_shape = q.shape().mergeAxes(.{ .h = .{ .hkv, .hg } }); + + const qs = q_shape.computeElementStrides().constSlice(); + const ks = k_cache.shape().computeElementStrides().constSlice(); + const vs = v_cache.shape().computeElementStrides().constSlice(); + const is = indices.shape().computeElementStrides().constSlice(); + const ts = parameters.block_table.shape().computeElementStrides().constSlice(); + + const head_size_padded: i64 = @intCast(std.math.ceilPowerOfTwoAssert(u64, @intCast(@max(16, q.dim(.hd))))); + + const config: kernels_sparse.SparseAttentionConfig = .{ + .dtype = triton.from(q.dtype()), + .num_seqs = @intCast(parameters.block_table.dim(.b)), + .num_query_heads = num_kv_heads * group, + .num_queries_per_kv = group, + .head_size = q.dim(.hd), + .head_size_padded = head_size_padded, + .block_m = @intCast(@max(16, std.math.ceilPowerOfTwoAssert(u64, @intCast(group)))), + .block_size = k_cache.dim(.k_chunk), + .tile_size = @intCast(tuning.tile_size), + .indices_count = indices.dim(.topk), + .num_segments = @intCast(tuning.num_segments), + .decode_sequence_grid = all_decode and parameters.options_.max_seqlen_q == 1, + .use_sinks = opts.sink != null, + .scale = opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))), + .block_table_stride = ts[parameters.block_table.axis(.b)], + .query_strides = .{ qs[q_shape.axis(.b)], qs[q_shape.axis(.h)], qs[q_shape.axis(.hd)] }, + .key_strides = .{ ks[k_cache.axis(.page)], ks[k_cache.axis(.k_chunk)], ks[k_cache.axis(.hkv)], ks[k_cache.axis(.hd)] }, + .value_strides = .{ vs[v_cache.axis(.page)], vs[v_cache.axis(.k_chunk)], vs[v_cache.axis(.hkv)], vs[v_cache.axis(.hd)] }, + .indices_strides = .{ is[indices.axis(.b)], is[indices.axis(.topk)] }, + }; + + const inputs: kernels_sparse.KernelSparseAttention3dPtr.Kernel.Inputs = .{ + .query_ptr = q, + .key_ptr = k_cache, + .value_ptr = v_cache, + .indices_ptr = indices, + .block_tables_ptr = parameters.block_table, + .seq_lens_ptr = parameters.seq_lens, + .query_start_len_ptr = parameters.query_start_len, + .sink_ptr = if (opts.sink) |sink| sink.convert(.f32) else .scalar(0, .f32), + }; + + const partition = kernels_sparse.SparseSegments.init(config.indices_count, config.num_segments, config.tile_size); + const grid: [3]i32 = .{ + if (config.decode_sequence_grid) config.num_seqs else @intCast(q.dim(.b)), + @intCast(num_kv_heads), + @intCast(partition.launched), + }; + + const warps: i32 = if (all_decode) 8 else 4; + + const stats_shape = zml.Shape.init(.{ q.dim(.b), config.num_segments, config.num_query_heads }, .f32); + + const partials = kernels_sparse.KernelSparseAttention3dPtr.Kernel.call(inputs, .{ + .output = .init(.{ q.dim(.b), config.num_query_heads, config.num_segments, head_size_padded }, .f32), + .segm_max = stats_shape, + .segm_expsum = stats_shape, + }, .{ + .cfg = config, + .grid = grid, + .num_stages = 1, + .num_warps = warps, + }); + + const head_tile = @min(64, head_size_padded); + + const output = kernels_sparse.SparseReducePtr.Kernel.call(.{ + .segm_output_ptr = partials.output, + .segm_max_ptr = partials.segm_max, + .segm_expsum_ptr = partials.segm_expsum, + .query_start_len_ptr = parameters.query_start_len, + }, .{ .output = q.shape() }, .{ + .cfg = .{ + .dtype = config.dtype, + .num_seqs = config.num_seqs, + .num_query_heads = config.num_query_heads, + .head_size = config.head_size, + .head_size_padded = head_size_padded, + .head_tile = head_tile, + .num_segments = config.num_segments, + .launched_segments = partition.launched, + .output_strides = config.query_strides, + }, + .grid = .{ @intCast(q.dim(.b)), @intCast(config.num_query_heads), @intCast(@divExact(head_size_padded, head_tile)) }, + .num_stages = 1, + .num_warps = 4, + }).output; + + return output; + } + pub fn pagedAttention2d(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, paged_attention_opts: PagedAttentionOptions) zml.Tensor { const config = select2dConfig(paged_attention_opts); const k_strides = k_cache.shape().computeElementStrides().constSlice(); @@ -393,7 +624,6 @@ pub const paged = struct { .use_fp8 = false, .all_decode = paged_attention_opts.all_decode, .is_causal = opts.mask.isCausal(), - .indices_count = @intCast(paged_attention_opts.indices_count), }; log.debug("pagedAttention2d config: {any}", .{kernel_config}); @@ -411,10 +641,6 @@ pub const paged = struct { const output = kernels.KernelUnifiedAttention2dPtr.Kernel.call( .{ .query_ptr = q, - .indices_ptr = switch (opts.mask) { - .indices => |indices| indices.transpose(.{ .b, .topk }), - else => dummy, - }, .key_cache_ptr = k_cache, .value_cache_ptr = v_cache, .sink_ptr = sink, @@ -474,7 +700,6 @@ pub const paged = struct { .num_segments_per_seq = @intCast(config.attention.num_segments_per_seq), .all_decode = paged_attention_opts.all_decode, .is_causal = opts.mask.isCausal(), - .indices_count = @intCast(paged_attention_opts.indices_count), }; log.debug("pagedAttention3d attention config: {any}", .{attn_kernel_config}); @@ -486,7 +711,6 @@ pub const paged = struct { .head_size_padded = head_size_padded, .block_q = @intCast(config.reduce.block_q), .num_segments_per_seq = @intCast(config.reduce.num_segments_per_seq), - .indices_count = @intCast(paged_attention_opts.indices_count), .use_fp8 = false, }; log.debug("pagedAttention3d reduce config: {any}", .{reduce_kernel_config}); @@ -511,10 +735,6 @@ pub const paged = struct { const attn_output = kernels.KernelUnifiedAttention3dPtr.Kernel.call( .{ .query_ptr = q, - .indices_ptr = switch (opts.mask) { - .indices => |indices| indices.transpose(.{ .b, .topk }), - else => dummy, - }, .key_cache_ptr = k_cache, .value_cache_ptr = v_cache, .sink_ptr = sink, diff --git a/zml/kernels/paged_attention/triton/unified_attention.zig b/zml/kernels/paged_attention/triton/unified_attention.zig index bf3fadbd..ce28905f 100644 --- a/zml/kernels/paged_attention/triton/unified_attention.zig +++ b/zml/kernels/paged_attention/triton/unified_attention.zig @@ -51,7 +51,6 @@ pub const KernelUnifiedAttention2dPtr = struct { fp8_max: f32 = FP8_E4M3_MAX, all_decode: bool = false, is_causal: bool = true, - indices_count: i64 = 0, }; pub const Kernel = tri.Kernel(Config, .{ @@ -62,7 +61,7 @@ pub const KernelUnifiedAttention2dPtr = struct { "scale_ptr", "k_scale_ptr", "v_scale_ptr", "out_scale_ptr", "softcap_ptr", "block_table_stride_ptr", "query_stride_0_ptr", "query_stride_1_ptr", "output_stride_0_ptr", "output_stride_1_ptr", "qq_bias_stride_0_ptr", "query_start_len_ptr", - "num_seqs_ptr", "indices_ptr", + "num_seqs_ptr", }, .outputs = &.{"output"}, .run = run, @@ -90,7 +89,6 @@ pub const KernelUnifiedAttention2dPtr = struct { .qq_bias_stride_0_ptr = .{ .ptr = .i64 }, .query_start_len_ptr = .{ .ptr = .i32 }, .num_seqs_ptr = .{ .ptr = .i32 }, - .indices_ptr = .{ .ptr = .i32 }, .output_ptr = .{ .ptr = cfg.o_dtype }, }); @@ -111,7 +109,6 @@ pub const KernelUnifiedAttention2dPtr = struct { b, a.output_ptr, a.query_ptr, - a.indices_ptr, a.key_cache_ptr, a.value_cache_ptr, a.sink_ptr, @@ -168,7 +165,6 @@ pub const KernelUnifiedAttention3dPtr = struct { num_segments_per_seq: i64, all_decode: bool = false, is_causal: bool = true, - indices_count: i64 = 0, }; pub const Kernel = tri.Kernel(Config, .{ @@ -178,7 +174,7 @@ pub const KernelUnifiedAttention3dPtr = struct { "block_tables_ptr", "seq_lens_ptr", "alibi_slopes_ptr", "qq_bias_ptr", "scale_ptr", "k_scale_ptr", "v_scale_ptr", "softcap_ptr", "block_table_stride_ptr", "query_stride_0_ptr", "query_stride_1_ptr", "qq_bias_stride_0_ptr", - "query_start_len_ptr", "num_seqs_ptr", "indices_ptr", + "query_start_len_ptr", "num_seqs_ptr", }, .outputs = &.{ "segm_output", "segm_max", "segm_expsum" }, .run = run, @@ -203,7 +199,6 @@ pub const KernelUnifiedAttention3dPtr = struct { .qq_bias_stride_0_ptr = .{ .ptr = .i64 }, .query_start_len_ptr = .{ .ptr = .i32 }, .num_seqs_ptr = .{ .ptr = .i32 }, - .indices_ptr = .{ .ptr = .i32 }, .segm_output_ptr = .{ .ptr = .f32 }, .segm_max_ptr = .{ .ptr = .f32 }, .segm_expsum_ptr = .{ .ptr = .f32 }, @@ -225,7 +220,6 @@ pub const KernelUnifiedAttention3dPtr = struct { a.segm_max_ptr, a.segm_expsum_ptr, a.query_ptr, - a.indices_ptr, a.key_cache_ptr, a.value_cache_ptr, a.sink_ptr, @@ -262,7 +256,6 @@ pub const ReduceSegmentsPtr = struct { block_q: i64, num_segments_per_seq: i64, - indices_count: i64 = 0, use_fp8: bool, fp8_min: f32 = FP8_E4M3_MIN, fp8_max: f32 = FP8_E4M3_MAX, @@ -395,7 +388,6 @@ fn kernelUnifiedAttention2d( k: *Builder, output_ptr: Value, query_ptr: Value, - indices_ptr: Value, key_cache_ptr: Value, value_cache_ptr: Value, sink_ptr: Value, @@ -461,11 +453,7 @@ fn kernelUnifiedAttention2d( // query_pos = q_block_local_idx * BLOCK_Q + offs_m // NUM_QUERIES_PER_KV const q_local_bq = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))); - // Sparse tiles contain one query token and its GQA heads, with padded head lanes. - const query_pos = if (config.indices_count > 0) - q_local_bq.splatTo(&.{BLOCK_M}) - else - q_local_bq.add(offs_m.div(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); + const query_pos = q_local_bq.add(offs_m.div(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); const query_offset_0 = cur_batch_in_all_start_index.add(query_pos); const query_offset_1 = kv_head_idx.mul(@as(i32, @intCast(NUM_QUERIES_PER_KV))) @@ -480,10 +468,7 @@ fn kernelUnifiedAttention2d( else k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); const query_mask_0 = query_pos.lt(cur_batch_query_len); - const query_mask_1 = if (config.indices_count > 0) - offs_m.lt(@as(i32, @intCast(NUM_QUERIES_PER_KV))) - else - query_offset_1.lt(@as(i32, @intCast(NUM_QUERY_HEADS))); + const query_mask_1 = query_offset_1.lt(@as(i32, @intCast(NUM_QUERY_HEADS))); // Q : (BLOCK_M, HEAD_SIZE_PADDED) const q_mask_ab = k.mask2d(query_mask_0, dim_mask, BLOCK_M, HEAD_SIZE_PADDED); @@ -534,7 +519,7 @@ fn kernelUnifiedAttention2d( // represented in this block. In non-causal mode, scan the whole sequence // so draft rows can attend to all rows written for the same sequence. const pad_term: i32 = @intCast(@divTrunc(BLOCK_M - 1, NUM_QUERIES_PER_KV) + 1); - const max_seq_prefix_len = if (config.indices_count > 0) k.liftAs(config.indices_count, .i32) else if (config.is_causal) b: { + const max_seq_prefix_len = if (config.is_causal) b: { const max_prefix_raw = context_len.add(q_local_bq).add(pad_term); break :b max_prefix_raw.minimum(seq_len); } else seq_len; @@ -572,22 +557,13 @@ fn kernelUnifiedAttention2d( const L = loop.carried[1]; const acc = loop.carried[2]; - const selection_offset = j.mul(@as(i32, @intCast(TILE_SIZE))).add(offs_t); - const selection_mask = selection_offset.lt(max_seq_prefix_len); - const seq_offset = if (config.indices_count > 0) k.loadOpts( - indices_ptr.addPtr(cur_batch_in_all_start_index.add(q_local_bq).to(.i64).mul(config.indices_count)).addPtr(selection_offset), - .{ .mask = selection_mask, .other = k.full(&.{TILE_SIZE}, -1, .i32) }, - ) else selection_offset; - var tile_mask = if (config.indices_count > 0) - selection_mask.bitAnd(seq_offset.ge(0)).bitAnd(seq_offset.lt(seq_len)) - else - selection_mask; - const safe_offset = k.where(tile_mask, seq_offset, k.zeros(&.{TILE_SIZE}, .i32)); - const physical_block_idx = k.loadOpts( - block_tables_ptr_shifted.addPtr(safe_offset.div(@as(i32, @intCast(BLOCK_SIZE)))), - .{ .mask = tile_mask, .other = k.full(&.{TILE_SIZE}, -1, .i32) }, + const seq_offset = j.mul(@as(i32, @intCast(TILE_SIZE))).add(offs_t); + + const tile_mask: Value = seq_offset.lt(max_seq_prefix_len); + + const physical_block_idx = k.load( + block_tables_ptr_shifted.addPtr(seq_offset.div(@as(i32, @intCast(BLOCK_SIZE)))), ).to(.i64); - tile_mask = tile_mask.bitAnd(physical_block_idx.ge(0)); const v_offset = k.expandDims(physical_block_idx, 1).mul(stride_v_cache_0) .add(kv_head_idx.to(.i64).mul(stride_v_cache_2)) @@ -679,12 +655,7 @@ fn kernelUnifiedAttention2d( const alpha_2d = alpha.expandDims(1); const acc_scaled = acc.mul(alpha_2d); const new_L = L.mul(alpha).add(l_j); - // An empty sparse tile must not establish a maximum of zero: later - // selected keys can have very negative scores and still carry all mass. - const new_M = if (config.indices_count > 0) - k.where(new_L.gt(0.0), m_j, k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32)) - else - m_j; + const new_M = m_j; const P_cast = P.to(config.q_dtype); const new_acc = k.dot(P_cast, V, acc_scaled); @@ -695,7 +666,7 @@ fn kernelUnifiedAttention2d( var acc_final = loop.results[2]; const L_2d = k.expandDims(L_final, 1); - const one_over_L = k.where(L_2d.gt(0.0), k.full(&.{ BLOCK_M, 1 }, 1.0, .f32).div(L_2d), k.zeros(&.{ BLOCK_M, 1 }, .f32)); + const one_over_L = k.full(&.{ BLOCK_M, 1 }, 1.0, .f32).div(L_2d); acc_final = acc_final.mul(k.broadcastTo(one_over_L, &.{ BLOCK_M, HEAD_SIZE_PADDED })); if (config.use_fp8) { @@ -726,7 +697,6 @@ fn kernelUnifiedAttention3d( segm_max_ptr: Value, segm_expsum_ptr: Value, query_ptr: Value, - indices_ptr: Value, key_cache_ptr: Value, value_cache_ptr: Value, sink_ptr: Value, @@ -784,11 +754,10 @@ fn kernelUnifiedAttention3d( k.returnIf(q_out_of_range, .{}); const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); - const attention_len = if (config.indices_count > 0) k.liftAs(config.indices_count, .i32) else seq_len; - const tiles_per_segment = cdivFn(attention_len, @as(i32, @intCast(NUM_SEGMENTS_PER_SEQ * TILE_SIZE))); + const tiles_per_segment = cdivFn(seq_len, @as(i32, @intCast(NUM_SEGMENTS_PER_SEQ * TILE_SIZE))); const segm_lo = segm_idx.mul(tiles_per_segment).mul(@as(i32, @intCast(TILE_SIZE))); - const segm_out_of_range = segm_lo.ge(attention_len); + const segm_out_of_range = segm_lo.ge(seq_len); k.returnIf(segm_out_of_range, .{}); const offs_m = k.arange(0, BLOCK_M, .i32); @@ -796,11 +765,7 @@ fn kernelUnifiedAttention3d( const offs_t = k.arange(0, TILE_SIZE, .i32); const q_local_bq = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))); - // Sparse tiles contain one query token and its GQA heads, with padded head lanes. - const query_pos = if (config.indices_count > 0) - q_local_bq.splatTo(&.{BLOCK_M}) - else - q_local_bq.add(offs_m.div(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); + const query_pos = q_local_bq.add(offs_m.div(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); const query_offset_0 = cur_batch_in_all_start_index.add(query_pos); const query_offset_1 = kv_head_idx.mul(@as(i32, @intCast(NUM_QUERIES_PER_KV))) @@ -818,10 +783,7 @@ fn kernelUnifiedAttention3d( else k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); const query_mask_0 = query_pos.lt(cur_batch_query_len); - const query_mask_1 = if (config.indices_count > 0) - offs_m.lt(@as(i32, @intCast(NUM_QUERIES_PER_KV))) - else - query_offset_1.lt(@as(i32, @intCast(NUM_QUERY_HEADS))); + const query_mask_1 = query_offset_1.lt(@as(i32, @intCast(NUM_QUERY_HEADS))); const q_mask_pre = k.expandDims(query_mask_0, 1).bitAnd(k.expandDims(query_mask_1, 1)); const q_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) @@ -874,7 +836,7 @@ fn kernelUnifiedAttention3d( qq_bias_ptr.splatTo(&.{BLOCK_M}); const pad_term: i32 = @intCast(@divTrunc(BLOCK_M - 1, NUM_QUERIES_PER_KV) + 1); - const max_seq_prefix_len = if (config.indices_count > 0) k.liftAs(config.indices_count, .i32) else if (config.is_causal) b: { + const max_seq_prefix_len = if (config.is_causal) b: { const max_prefix_raw = context_len.add(q_local_bq).add(pad_term); break :b max_prefix_raw.minimum(seq_len); } else seq_len; @@ -893,23 +855,12 @@ fn kernelUnifiedAttention3d( const L = loop.carried[1]; const acc = loop.carried[2]; - const selection_offset = j.mul(@as(i32, @intCast(TILE_SIZE))).add(offs_t); - const selection_mask = selection_offset.lt(max_seq_prefix_len); - const seq_offset = if (config.indices_count > 0) k.loadOpts( - indices_ptr.addPtr(cur_batch_in_all_start_index.add(q_local_bq).to(.i64).mul(config.indices_count)).addPtr(selection_offset), - .{ .mask = selection_mask, .other = k.full(&.{TILE_SIZE}, -1, .i32) }, - ) else selection_offset; - var tile_mask = if (config.indices_count > 0) - selection_mask.bitAnd(seq_offset.ge(0)).bitAnd(seq_offset.lt(seq_len)) - else - selection_mask; - const safe_offset = k.where(tile_mask, seq_offset, k.zeros(&.{TILE_SIZE}, .i32)); - const physical_block_idx = k.loadOpts( - block_tables_ptr.addPtr(block_table_offset).addPtr(safe_offset.div(@as(i32, @intCast(BLOCK_SIZE)))), - .{ .mask = tile_mask, .other = k.full(&.{TILE_SIZE}, -1, .i32) }, - ).to(.i64); - tile_mask = tile_mask.bitAnd(physical_block_idx.ge(0)); + const seq_offset = j.mul(@as(i32, @intCast(TILE_SIZE))).add(offs_t); + const tile_mask: Value = seq_offset.lt(max_seq_prefix_len); + const physical_block_idx = k.load( + block_tables_ptr.addPtr(block_table_offset).addPtr(seq_offset.div(@as(i32, @intCast(BLOCK_SIZE)))), + ).to(.i64); const v_offset = k.expandDims(physical_block_idx, 1).mul(stride_v_cache_0) .add(kv_head_idx.to(.i64).mul(stride_v_cache_2)) .add(k.expandDims(offs_d, 0).mul(config.stride_v_cache_3)) @@ -1003,11 +954,7 @@ fn kernelUnifiedAttention3d( const P_cast = P.to(config.q_dtype); const new_acc = k.dot(P_cast, V, acc_scaled); - const new_M = if (config.indices_count > 0) - k.where(new_L.gt(0.0), m_j, k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32)) - else - m_j; - loop.yield(.{ new_M, new_L, new_acc }); + loop.yield(.{ m_j, new_L, new_acc }); } const M_final = loop.results[0]; const L_final = loop.results[1]; @@ -1073,17 +1020,12 @@ fn reduceSegments( const query_token_idx = k.programId(.x); const query_head_idx = k.programId(.y); - if (config.indices_count > 0) { - const active_queries = k.load(query_start_len_ptr.addPtr(num_seqs)); - k.returnIf(query_token_idx.ge(active_queries), .{}); - } const seq_idx = findSeqIdx(k, query_start_len_ptr, query_token_idx, num_seqs, BLOCK_Q, false); const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); - const attention_len = if (config.indices_count > 0) k.liftAs(config.indices_count, .i32) else seq_len; - const tiles_per_segment = cdivFn(attention_len, @as(i32, @intCast(NUM_SEGMENTS_PER_SEQ * TILE_SIZE))); - const act_num_segments = cdivFn(attention_len, tiles_per_segment.mul(@as(i32, @intCast(TILE_SIZE)))); + const tiles_per_segment = cdivFn(seq_len, @as(i32, @intCast(NUM_SEGMENTS_PER_SEQ * TILE_SIZE))); + const act_num_segments = cdivFn(seq_len, tiles_per_segment.mul(@as(i32, @intCast(TILE_SIZE)))); const seg_range = k.arange(0, NUM_SEGMENTS_PER_SEQ, .i32); const segm_mask = seg_range.lt(act_num_segments); @@ -1105,11 +1047,7 @@ fn reduceSegments( .mask = segm_mask, .other = k.full(&.{NUM_SEGMENTS_PER_SEQ}, -std.math.inf(f32), .f32), }); - const max_value = k.max(segm_max); - const overall_max = if (config.indices_count > 0) - k.where(max_value.gt(-std.math.inf(f32)), max_value, k.liftAs(0.0, .f32)) - else - max_value; + const overall_max = k.max(segm_max); var segm_expsum = k.loadOpts(segm_expsum_ptr.addPtr(segm_offset), .{ .mask = segm_mask, From a1c9049f4410b83cbece625011899ab10b0cd353 Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Thu, 22 Oct 2026 10:38:37 +0200 Subject: [PATCH 810/881] zml/kernels: relax CuTe MoE conditions (#976) Changes: - introduce `clamp` function to CuTe builder - make swiglu limit field optional - add 16 and 32 reduction tile --- kernels/cute/builder.zig | 50 +++++++++++++++++++++++ zml/kernels/moe/cute/cute.zig | 7 ++-- zml/kernels/moe/cute/moe.zig | 21 +++++++--- zml/kernels/moe/cute/persistent_mxfp4.zig | 19 +++++---- 4 files changed, 80 insertions(+), 17 deletions(-) diff --git a/kernels/cute/builder.zig b/kernels/cute/builder.zig index 286b92aa..61c9aed7 100644 --- a/kernels/cute/builder.zig +++ b/kernels/cute/builder.zig @@ -173,6 +173,14 @@ pub const Value = struct { return k.emit(if (l.isFloat()) arith.maximumf(k.ctx, l.inner, r.inner, k.loc()) else arith.maxsi(k.ctx, l.inner, r.inner, k.loc())); } + pub fn clamp(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + if (l.isFloat()) return k.emit(math.clampf(k.ctx, l.inner, r.neg().inner, r.inner, k.loc())); + const upper = k.emit(arith.minsi(k.ctx, l.inner, r.inner, k.loc())); + return k.emit(arith.maxsi(k.ctx, upper.inner, r.neg().inner, k.loc())); + } + /// Float max that ignores NaN operands (`arith.maxnumf`, a single `max.f32`), unlike /// the NaN-propagating `maximum`. pub fn maxnum(self: Value, rhs: anytype) Value { @@ -3138,6 +3146,48 @@ test "naive elementwise add matches the Python notebook kernel" { try std.testing.expect(parsed.operation().verify()); } +test "integer clamp emits both bounds" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "integer_clamp"); + defer b.deinit(); + const a = try b.declareArgs(.{ .out = .{ .tensor = .{ .dtype = .i32, .shape = &.{8} } } }); + const tid = b.threadIdx().x; + a.out.set(.{tid}, tid.clamp(4)); + + const ir = try b.finish(.{ 8, 1, 1 }); + defer std.testing.allocator.free(ir); + try expectContains(ir, "arith.minsi"); + try expectContains(ir, "arith.maxsi"); + try expectContains(ir, "arith.subi"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "float clamp emits math.clampf" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "float_clamp"); + defer b.deinit(); + const a = try b.declareArgs(.{ + .input = .{ .tensor = .{ .dtype = .f32, .shape = &.{8} } }, + .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{8} } }, + }); + const tid = b.threadIdx().x; + a.out.set(.{tid}, a.input.get(.{tid}).clamp(4.0)); + + const ir = try b.finish(.{ 8, 1, 1 }); + defer std.testing.allocator.free(ir); + try expectContains(ir, "arith.negf"); + try expectContains(ir, "math.clampf"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + test "guard, shared memory, sync, layouts, for loop" { const ctx = try testContext(); defer ctx.deinit(); diff --git a/zml/kernels/moe/cute/cute.zig b/zml/kernels/moe/cute/cute.zig index ced6dd89..ed153b5d 100644 --- a/zml/kernels/moe/cute/cute.zig +++ b/zml/kernels/moe/cute/cute.zig @@ -19,7 +19,6 @@ pub fn isAvailable(platform: *const zml.Platform) bool { pub fn validateOptions(opts: zml.moe.Options) void { stdx.debug.assert(opts.activation == .swiglu, "cute backend only accepts swiglu activation, got {}", .{opts.activation}); - stdx.debug.assert(opts.activation.swiglu.limit != null, "cute backend requires swiglu limit to be set", .{}); stdx.debug.assert(opts.activation.swiglu.bias == null, "cute backend requires swiglu bias to be null", .{}); stdx.debug.assert(opts.activation.swiglu.scale == null, "cute backend requires swiglu scale to be null", .{}); } @@ -48,7 +47,7 @@ pub fn fusedExperts( const hidden = down.weight.dim(1); const intermediate = down.weight.dim(2) * 2; - // The CuTe GEMMs are specialized on one set of dimensions + // Check the boundary quantizer and CuTe GEMM tile constraints. kernels.validateShapes(hidden, intermediate); // The SwiGLU clamp is a kernel parameter, but the routing weight is @@ -64,7 +63,7 @@ pub fn fusedExperts( .w2 = down.weight, .s2 = dq.scales, .topk = ids.dim(.topk), - .swiglu_limit = options.activation.swiglu.limit.?, + .swiglu_limit = options.activation.swiglu.limit, .expert_parallel = expert_parallelism, }; return if (expert_parallelism) @@ -82,7 +81,7 @@ const Context = struct { w2: zml.Tensor, s2: zml.Tensor, topk: i64, - swiglu_limit: f32, + swiglu_limit: ?f32, expert_parallel: bool, fn body(self: Context, _: zml.Shape) zml.Tensor { diff --git a/zml/kernels/moe/cute/moe.zig b/zml/kernels/moe/cute/moe.zig index 638685e3..809b374e 100644 --- a/zml/kernels/moe/cute/moe.zig +++ b/zml/kernels/moe/cute/moe.zig @@ -26,11 +26,20 @@ pub const Inputs = struct { ids: zml.Tensor, }; -/// The kernels are tuned for the DeepSeek V4.1 expert geometry, shared by the -/// main MoE and the D-Spark drafter. Expert count and top-k are parameters. +/// The boundary quantizer handles four 128-column tiles per CTA. The GEMMs +/// support 128-column tiles and a fixed set of reduction tile counts. pub fn validateShapes(hidden: i64, intermediate: i64) void { - stdx.debug.assert(hidden == 5120, "cute backend expects hidden dimension to be 5120, got {}", .{hidden}); - stdx.debug.assert(intermediate == 2304, "cute backend expects intermediate dimension to be 2304, got {}", .{intermediate}); + stdx.debug.assert(hidden > 0 and @rem(hidden, 4 * 128) == 0, "cute backend requires hidden dimension to be a positive multiple of 512, got {}", .{hidden}); + stdx.debug.assert(intermediate > 0 and @rem(intermediate, 128) == 0, "cute backend requires intermediate dimension to be a positive multiple of 128, got {}", .{intermediate}); + stdx.debug.assert(supportedReductionTiles(@divTrunc(hidden, 128)), "cute backend does not support hidden reduction tile count {}, got hidden dimension {}", .{ @divTrunc(hidden, 128), hidden }); + stdx.debug.assert(supportedReductionTiles(@divTrunc(intermediate, 128)), "cute backend does not support intermediate reduction tile count {}, got intermediate dimension {}", .{ @divTrunc(intermediate, 128), intermediate }); +} + +fn supportedReductionTiles(tiles: i64) bool { + return switch (tiles) { + 16, 18, 32, 40 => true, + else => false, + }; } /// Route each token/top-k pair to its own one-row group instead of grouping @@ -159,13 +168,13 @@ fn route(tokens: i64, experts: i64, topk: i64, n: i64, direct: bool, ids: zml.Te } /// `[tokens, hidden]` BF16 output of the MoE layer. -pub fn forward(tokens: i64, hidden: i64, intermediate: i64, experts: i64, topk: i64, swiglu_limit: f32, a: Inputs) zml.Tensor { +pub fn forward(tokens: i64, hidden: i64, intermediate: i64, experts: i64, topk: i64, swiglu_limit: ?f32, a: Inputs) zml.Tensor { return forwardWithTile(tokens, hidden, intermediate, experts, topk, swiglu_limit, a, null); } /// `forward` with an explicit GEMM tile N, for tuning. `null` selects it from /// the batch size. -pub fn forwardWithTile(tokens: i64, hidden: i64, intermediate: i64, experts: i64, topk: i64, swiglu_limit: f32, a: Inputs, tile_n: ?i64) zml.Tensor { +pub fn forwardWithTile(tokens: i64, hidden: i64, intermediate: i64, experts: i64, topk: i64, swiglu_limit: ?f32, a: Inputs, tile_n: ?i64) zml.Tensor { const n = tile_n orelse tileN(tokens, experts, topk); const direct = isDirect(tokens); const routing = route(tokens, experts, topk, n, direct, a.ids); diff --git a/zml/kernels/moe/cute/persistent_mxfp4.zig b/zml/kernels/moe/cute/persistent_mxfp4.zig index 920e1256..dc5732c4 100644 --- a/zml/kernels/moe/cute/persistent_mxfp4.zig +++ b/zml/kernels/moe/cute/persistent_mxfp4.zig @@ -245,7 +245,7 @@ pub const Config = struct { routes: i64, /// Clamp applied to both halves of the fused SwiGLU. Emitted as an IR /// constant, so a threaded value costs exactly what a literal did. - swiglu_limit: f32 = 10.0, + swiglu_limit: ?f32 = 10.0, /// Ungrouped routing: group `g` is route `g` and holds one live row. /// `schedule` is then just the `[routes]` expert ids; its size and /// active-count sections and `route_inverse` are not read. @@ -1408,7 +1408,7 @@ const EpilogueOperands = struct { /// `Config.groupRows`. group_rows: i64 = 0, /// `Config.swiglu_limit`. - swiglu_limit: f32 = 10.0, + swiglu_limit: ?f32 = 10.0, }; /// Warps 0-3 drain each TMEM accumulator stage into registers, release it to @@ -1617,9 +1617,14 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo for (columns[first..last], activations[first..last], first..) |value, *activation, column| { const c: i32 = @intCast(column); // The reference model rounds the projection to BF16. - const limit = operands.swiglu_limit; - const gate = value.to(.bf16).to(.f32).minimum(limit); - const up_value = b.shuffleXor(value, 1).to(.bf16).to(.f32).maximum(-limit).minimum(limit); + var gate = value.to(.bf16).to(.f32); + if (operands.swiglu_limit) |limit| { + gate = gate.minimum(limit); + } + var up_value = b.shuffleXor(value, 1).to(.bf16).to(.f32); + if (operands.swiglu_limit) |limit| { + up_value = up_value.clamp(limit); + } // up * SiLU(gate), with the hardware approximations Python's // fastmath selects: exp(-g) = 2^(-g * log2 e). const exp_neg = b.unaryF32("ex2.approx.ftz.f32", gate.mul(-std.math.log2e)); @@ -1852,7 +1857,7 @@ fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E // AB and accumulator mbarriers carry all cross-warp dependencies. var tma_warp = b.openIf(warp.eq(5)); switch (k_tiles) { - inline 18, 40 => |kt| runTmaProducer(b, tile, kt, storage, active, d.tiles_m, expert_ids, direct, partitioned_a.targets[0], partitioned_b.targets[0], exec_a, exec_b, exec_sfa, exec_sfb), + inline 16, 18, 32, 40 => |kt| runTmaProducer(b, tile, kt, storage, active, d.tiles_m, expert_ids, direct, partitioned_a.targets[0], partitioned_b.targets[0], exec_a, exec_b, exec_sfa, exec_sfb), else => std.debug.panic("unsupported MXFP4 reduction tile count {d}", .{k_tiles}), } tma_warp.yieldThen(.{}); @@ -1861,7 +1866,7 @@ fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E // Python's MMA role reaches its own TmemAllocator.wait_for_alloc site. b.namedBarrier(2, 160); switch (k_tiles) { - inline 18, 40 => |kt| runMmaConsumer(b, tile, kt, storage, tiled_mma, tmem_holding, active, d.tiles_m), + inline 16, 18, 32, 40 => |kt| runMmaConsumer(b, tile, kt, storage, tiled_mma, tmem_holding, active, d.tiles_m), else => unreachable, } mma_warp.yieldThen(.{}); From ddb520cf397aaefce7a98d96e3ad35a1a4ca4651 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 22 Oct 2026 10:48:08 +0200 Subject: [PATCH 811/881] zml/kernels/paged_attention/cute: add SM100 specialization for gemma 4 (#969) This PR adds a `sm10x` specific implementation for the head dim used in Gemma 4 --- zml/kernels/paged_attention/cute/BUILD.bazel | 2 + zml/kernels/paged_attention/cute/cute.zig | 9 +- zml/kernels/paged_attention/cute/sm100.zig | 114 +++ .../paged_attention/cute/sm100_attention.zig | 716 ++++++++++++++++++ 4 files changed, 838 insertions(+), 3 deletions(-) create mode 100644 zml/kernels/paged_attention/cute/sm100.zig create mode 100644 zml/kernels/paged_attention/cute/sm100_attention.zig diff --git a/zml/kernels/paged_attention/cute/BUILD.bazel b/zml/kernels/paged_attention/cute/BUILD.bazel index 9f44e41f..d57a13f8 100644 --- a/zml/kernels/paged_attention/cute/BUILD.bazel +++ b/zml/kernels/paged_attention/cute/BUILD.bazel @@ -7,6 +7,8 @@ filegroup( srcs = [ "attention.zig", "cute.zig", + "sm100.zig", + "sm100_attention.zig", ], ) diff --git a/zml/kernels/paged_attention/cute/cute.zig b/zml/kernels/paged_attention/cute/cute.zig index 60302a72..454f413c 100644 --- a/zml/kernels/paged_attention/cute/cute.zig +++ b/zml/kernels/paged_attention/cute/cute.zig @@ -2,14 +2,15 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const paged = @import("../paged_attention.zig"); -const triton = @import("../triton/triton.zig"); +const sm100 = @import("sm100.zig"); const kernel = @import("attention.zig"); -pub const Options = triton.paged.Options; -pub const Parameters = triton.paged.Parameters; +pub const Options = sm100.Options; +pub const Parameters = sm100.Parameters; test { _ = kernel; + _ = sm100; } pub fn isAvailable(platform: *const zml.Platform) bool { @@ -22,6 +23,8 @@ pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: z stdx.debug.assert(isAvailable(zml.Compiler.current().platform), "CuTe paged attention requires SM89 or newer", .{}); stdx.debug.assert(q.dtype() == k.dtype() and q.dtype() == v.dtype() and (q.dtype() == .bf16 or q.dtype() == .f16), "CuTe paged attention requires matching BF16 or FP16 Q/K/V", .{}); stdx.debug.assert(q.dim(.hd) == 32 or q.dim(.hd) == 64 or q.dim(.hd) == 128 or q.dim(.hd) == 256 or q.dim(.hd) == 512, "CuTe paged attention supports head dimensions 32, 64, 128, 256 and 512", .{}); + if (sm100.isAvailable(zml.Compiler.current().platform) and (q.dim(.hd) == 256 or q.dim(.hd) == 512) and opts.mask != .indices) + return sm100.pagedAttention(parameters, q, k, v, opts); return zml.ops.manualComputation(Context.body, Context{ .q = q, .k = k, .v = v, .parameters = parameters, .opts = opts }, q.shape()); } diff --git a/zml/kernels/paged_attention/cute/sm100.zig b/zml/kernels/paged_attention/cute/sm100.zig new file mode 100644 index 00000000..007f153d --- /dev/null +++ b/zml/kernels/paged_attention/cute/sm100.zig @@ -0,0 +1,114 @@ +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const paged = @import("../paged_attention.zig"); +const triton = @import("../triton/triton.zig"); +const kernel = @import("sm100_attention.zig"); + +pub const Options = triton.paged.Options; +pub const Parameters = triton.paged.Parameters; + +test { + _ = kernel; +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + const cc = zml.platform.cuda.computeCapability(platform) orelse return false; + return cc.major == 10; +} + +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { + stdx.debug.assert(q.dtype() == k.dtype() and q.dtype() == v.dtype(), "CuTe paged attention requires matching Q/K/V types", .{}); + return zml.ops.manualComputation(Context.body, Context{ .q = q, .k = k, .v = v, .parameters = parameters, .opts = opts }, q.shape()); +} + +const Context = struct { + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + parameters: Parameters, + opts: paged.AttentionOptions, + + fn body(self: Context, output_shape: zml.Shape) zml.Tensor { + const q = self.q.transpose(.{ .b, .hkv, .hg, .hd }); + const k = self.k.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const v = self.v.transpose(.{ .page, .k_chunk, .hkv, .hd }); + stdx.debug.assert(q.dim(.hkv) == k.dim(.hkv) and k.shape().eql(v.shape()) and k.dim(.hd) == q.dim(.hd) and q.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); + const p = self.parameters; + const all_decode = !p.options_.is_prefill; + const sms: i64 = @intCast(triton.getCuCount()); + const rows = kernel.decodeRows(q.dim(.hg)); + const head_tiles = @divTrunc(q.dim(.hg) + rows - 1, rows); + const max_splits = if (all_decode) @min(128, @divTrunc(sms * 2 + k.dim(.hkv) * head_tiles - 1, k.dim(.hkv) * head_tiles)) else 1; + const window = self.opts.mask.slidingWindow() orelse 0; + const large_decode_tile = all_decode and q.dim(.hd) == 256 and + q.dim(.hg) <= 4 and window >= 128 and window <= 1024 and + p.block_table.dim(.b) * k.dim(.hkv) * head_tiles >= @divTrunc(sms, 2); + const kv_tile: i64 = if (large_decode_tile) 128 else 64; + const decode_work = p.block_table.dim(.b) * k.dim(.hkv) * head_tiles; + const decode_waves: i64 = if (large_decode_tile and decode_work < sms * 2) 4 else 2; + // A full decode batch supplies enough independent sequence/head CTAs + // to stream the window without split scratch or a separate merge. + // Smaller grids retain the existing partitioned path. + const max_window_tiles = @divTrunc(@as(i64, window) + 2 * kv_tile - 2, kv_tile); + const tiles_per_split: i64 = if (!large_decode_tile and all_decode and q.dim(.hd) == 256 and window > 0) @max(1, @divTrunc(max_window_tiles + 127, 128)) else 0; + const splits = if (tiles_per_split > 0) + @divTrunc(max_window_tiles + tiles_per_split - 1, tiles_per_split) + else if (all_decode) + @min(max_splits, @max(1, @divTrunc(sms * decode_waves + p.block_table.dim(.b) * k.dim(.hkv) * head_tiles - 1, p.block_table.dim(.b) * k.dim(.hkv) * head_tiles))) + else + max_splits; + // Keep independent split CTAs for low-occupancy shapes. At full + // occupancy, a fixed number of workers reuses Q and TMEM and avoids + // launching the entire window bound for short runtime contexts. + const split_stride: i64 = if (tiles_per_split == 1 and splits > 2 and + p.block_table.dim(.b) * k.dim(.hkv) * head_tiles >= sms * 2) 2 else 0; + const cfg: kernel.Config = .{ + .head_dim = q.dim(.hd), + .kv_tile = kv_tile, + // Prefetch an extra feature stage for non-split D512 decode; + // split workloads retain the smaller shared-memory footprint. + .load_buffers = if (kv_tile == 128 or (all_decode and q.dim(.hd) == 512 and splits == 1)) 3 else 2, + .tokens = q.dim(.b), + .batch = p.block_table.dim(.b), + .kv_heads = k.dim(.hkv), + .group = q.dim(.hg), + .pages_per_seq = p.block_table.dim(.p), + .page_size = k.dim(.k_chunk), + .max_query = if (all_decode) 1 else @min(q.dim(.b), @as(i64, @intCast(p.options_.max_seqlen_q))), + .dtype = zml.kernel.cute.from(q.dtype()), + .all_decode = all_decode, + .causal = self.opts.mask.isCausal(), + .window = window, + .scale = self.opts.scale orelse (1 / @sqrt(@as(f32, @floatFromInt(q.dim(.hd))))), + .sink = self.opts.sink != null, + .sms = sms, + .splits = @max(1, splits), + .tiles_per_split = tiles_per_split, + .split_stride = split_stride, + }; + const sink = if (self.opts.sink) |s| s.convert(.f32).reshape(.{cfg.kv_heads * cfg.group}) else zml.Tensor.zeroes(.init(.{cfg.kv_heads * cfg.group}, .f32)); + const out_shape = if (cfg.splits > 1) zml.Shape.init(.{ cfg.splits, cfg.tokens, cfg.kv_heads * cfg.group, cfg.head_dim + 2 }, .f32) else q.shape(); + const out = kernel.Kernel.call(.{ + .q = q, + .k = k, + .v = v, + .table = p.block_table, + .lengths = p.seq_lens, + .starts = p.query_start_len, + .sink = sink, + }, .{ .out = out_shape }, .{ + .cfg = cfg, + .grid = .{ @intCast(cfg.batch * cfg.queryTiles() * (if (cfg.headsInX()) cfg.kv_heads else 1)), @intCast(if (cfg.headsInX()) 1 else cfg.kv_heads * cfg.outputTiles()), @intCast(if (cfg.split_stride > 0) cfg.split_stride else cfg.splits) }, + .block = kernel.block, + }).out; + const result = if (cfg.splits > 1) kernel.Merge.call(.{ + .partials = out, + .lengths = p.seq_lens, + .starts = p.query_start_len, + .sink = sink, + }, .{ .out = q.shape() }, .{ .cfg = cfg, .grid = .{ @intCast(cfg.kv_heads * cfg.group), @intCast(cfg.batch), 1 }, .block = .{ @intCast(cfg.head_dim), 1, 1 } }).out else out; + return result.transpose(output_shape); + } +}; diff --git a/zml/kernels/paged_attention/cute/sm100_attention.zig b/zml/kernels/paged_attention/cute/sm100_attention.zig new file mode 100644 index 00000000..12a6fe19 --- /dev/null +++ b/zml/kernels/paged_attention/cute/sm100_attention.zig @@ -0,0 +1,716 @@ +const std = @import("std"); + +const zml = @import("../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const V = cute.Value; + +pub const Config = struct { + head_dim: i64 = 512, + // Logical key tile; independent of physical cache page size. + kv_tile: i64 = 64, + load_buffers: i64 = 2, + tokens: i64, + batch: i64, + kv_heads: i64, + group: i64, + pages_per_seq: i64, + page_size: i64, + max_query: i64, + dtype: cute.DType, + all_decode: bool, + causal: bool, + window: i64, + scale: f32, + sink: bool, + sms: i64, + splits: i64, + // A positive bound partitions sliding decode by work per CTA, even when + // the batch already provides enough CTAs to fill the device. + tiles_per_split: i64 = 0, + // Fixed workers per sequence/head tile; zero launches each split separately. + split_stride: i64 = 0, + + pub fn headsInX(c: Config) bool { + return c.split_stride > 0 or (c.all_decode and c.kv_tile > 64); + } + pub fn rows(c: Config) i64 { + return if (c.all_decode) decodeRows(c.group) else 64; + } + pub fn queryTiles(c: Config) i64 { + return @divTrunc(c.max_query * c.group + c.rows() - 1, c.rows()); + } + pub fn outputTiles(c: Config) i64 { + return if (c.all_decode) 1 else @divExact(c.head_dim, 256); + } +}; + +// Use legal MMA widths, with masked tails for arbitrary GQA ratios. +pub fn decodeRows(group: i64) i64 { + return if (group <= 8) 8 else 16; +} + +pub const block: [3]i32 = .{ 256, 1, 1 }; +pub const Kernel = cute.Kernel(Config, .{ + .name = "paged_attention_cute", + .inputs = &.{ "q", "k", "v", "table", "lengths", "starts", "sink" }, + .outputs = &.{"out"}, + .run = run, +}); + +// KV buffers hold kv_tile keys by 128 features. The load ring streams D/128 +// K stages followed by D/128 V stages while Q stays resident. Prefill retains +// 64-key tiles and 256 output features per CTA. +fn kvBytes(c: Config) i64 { + return c.kv_tile * 128 * 2; +} +fn qBase(c: Config) i32 { + return @intCast(c.load_buffers * kvBytes(c)); +} +fn pBase(c: Config) i32 { + return qBase(c) + @as(i32, @intCast(c.rows())) * @as(i32, @intCast(c.head_dim)) * 2; +} +fn barBase(c: Config) i32 { + return pBase(c) + @as(i32, @intCast(c.rows() * c.kv_tile)) * 2; +} +fn holder(c: Config) i32 { + return barBase(c) + @as(i32, @intCast(c.load_buffers * 16)) + 16; +} +fn redBase(c: Config) i32 { + return holder(c) + 16; +} +fn sharedBytes(c: Config) i32 { + return redBase(c) + 4 * 16 * 4; +} +fn scoreCol(c: Config) i32 { + return if (c.all_decode) @intCast(@divExact(c.head_dim, 128) * c.rows()) else 256; +} +fn tmemCols(c: Config) u32 { + return if (c.all_decode) @intCast(std.math.ceilPowerOfTwoAssert(u32, @intCast((@divExact(c.head_dim, 128) + @divTrunc(c.kv_tile + 127, 128)) * c.rows()))) else 512; +} + +const Ctx = struct { + b: *B, + c: Config, + q: V, + k: V, + v: V, + table: V, + sink: V, + out: V, + s: V, + tmem: V, + tid: V, + warp: V, + lane: V, + seq: V, + head: V, + first_row: V, + start: V, + qlen: V, + klen: V, + first_tile: V, + tiles: V, + split: V, + output_half: V, +}; + +fn loadI32(b: *B, ptr: V, index: V) V { + return b.loadWords(1, ptr.add(index.to(.i64).mul(4)), .global, .{})[0]; +} + +// All-decode metadata gives the live request count without host readback. The +// fixed grid/scratch bound is independent of it and safe for graph replay. +fn liveSplits(b: *B, c: Config, active: V, tiles: V) V { + if (c.splits == 1) return b.cst(.i32, 1); + if (c.tiles_per_split > 0) + return tiles.add(c.tiles_per_split - 1).div(c.tiles_per_split).maximum(1).minimum(c.splits); + const work = active.maximum(1).mul(c.kv_heads * @divTrunc(c.group + c.rows() - 1, c.rows())); + const waves: i64 = if (c.kv_tile > 64 and c.window > 0) 4 else 2; + const budget = work.add(c.sms * waves - 1).div(work).maximum(1).minimum(c.splits); + return budget.minimum(tiles.add(1).div(2).maximum(1)); +} + +fn run(b: *B, c: Config) cute.FinishError!void { + std.debug.assert(c.kv_tile == 64 or c.kv_tile == 128 or c.kv_tile == 256); + std.debug.assert(c.all_decode or c.kv_tile == 64); + std.debug.assert(c.load_buffers == 2 or c.load_buffers == 3); + // Persistent independent partitions restart stage numbering each time. + std.debug.assert(c.split_stride == 0 or c.load_buffers == 2); + const a = try b.declareArgs(.{ + .q = .{ .ptr = c.dtype }, + .k = .{ .ptr = c.dtype }, + .v = .{ .ptr = c.dtype }, + .table = .{ .ptr = cute.DType.i32 }, + .lengths = .{ .ptr = cute.DType.i32 }, + .starts = .{ .ptr = cute.DType.i32 }, + .sink = .{ .ptr = cute.DType.f32 }, + .out = .{ .ptr = if (c.splits > 1) cute.DType.f32 else c.dtype }, + }); + b.setFunctionAttribute("cu_attrs", b.parseAttribute("{max_dynamic_shared_size_bytes = #cuda.dev_max_shared_memory_optin}")); + const starts = b.ptrToInt(a.starts, .i64); + const seq = b.blockIdx().x.div(c.queryTiles() * (if (c.headsInX()) c.kv_heads else 1)); + const first_row = b.blockIdx().x.rem(c.queryTiles()).mul(c.rows()); + const start = loadI32(b, starts, seq); + const qlen = loadI32(b, starts, seq.add(1)).sub(start); + const klen = loadI32(b, b.ptrToInt(a.lengths, .i64), seq); + const first_pos = klen.sub(qlen).add(first_row.div(c.group)); + const last_pos = klen.sub(qlen).add(first_row.add(c.rows() - 1).div(c.group).minimum(qlen.sub(1))); + const begin = if (c.window > 0) first_pos.sub(c.window - 1).maximum(0) else b.cst(.i32, 0); + const end = if (c.causal) last_pos.add(1) else klen; + const first_tile = begin.div(c.kv_tile); + const total_tiles = end.add(c.kv_tile - 1).div(c.kv_tile).sub(first_tile).maximum(0); + const splits = liveSplits(b, c, loadI32(b, starts, b.cst(.i32, c.batch)), total_tiles); + const split = b.blockIdx().z; + var active = b.openIf(first_row.lt(qlen.mul(c.group)).bitAnd(split.lt(splits))); + { + const lo = if (c.tiles_per_split > 0) split.mul(c.tiles_per_split) else total_tiles.mul(split).div(splits); + const hi = if (c.tiles_per_split > 0) lo.add(c.tiles_per_split).minimum(total_tiles) else total_tiles.mul(split.add(1)).div(splits); + const tid = b.threadIdx().x; + const s = b.ptrToInt(b.allocSmem(.i8, @intCast(sharedBytes(c)), 1024), .i32); + var ctx: Ctx = .{ + .b = b, + .c = c, + .q = b.ptrToInt(a.q, .i64), + .k = b.ptrToInt(a.k, .i64), + .v = b.ptrToInt(a.v, .i64), + .table = b.ptrToInt(a.table, .i64), + .sink = b.ptrToInt(a.sink, .i64), + .out = b.ptrToInt(a.out, .i64), + .s = s, + .tmem = undefined, + .tid = tid, + .warp = tid.div(32), + .lane = tid.rem(32), + .seq = seq, + .head = if (c.headsInX()) b.blockIdx().x.div(c.queryTiles()).rem(c.kv_heads) else b.blockIdx().y.div(c.outputTiles()), + .output_half = b.blockIdx().y.rem(c.outputTiles()), + .first_row = first_row, + .start = start, + .qlen = qlen, + .klen = klen, + .first_tile = first_tile.add(lo), + .tiles = if (c.tiles_per_split == 1) b.cst(.i32, 1) else hi.sub(lo), + .split = split, + }; + var init = b.openIf(tid.eq(0)); + { + for (0..@as(usize, @intCast(c.load_buffers))) |i| { + b.mbarrierInit(s.add(barBase(c) + @as(i32, @intCast(i * 8))), 128); + b.mbarrierInit(s.add(barBase(c) + @as(i32, @intCast(c.load_buffers * 8)) + @as(i32, @intCast(i * 8))), 1); + } + b.mbarrierInit(s.add(barBase(c) + @as(i32, @intCast(c.load_buffers * 16))), 1); + b.fenceMbarrierInit(); + init.yieldThen(.{}); + } + var alloc = b.openIf(tid.lt(32)); + { + b.tmemAllocAt(s.add(holder(c)), tmemCols(c)); + b.relinquishTmemAllocPermit(.{}); + alloc.yieldThen(.{}); + } + // Queries are packed_row by token, KV head, and group head. Mask the final + // query tile without ever forming an out-of-range source load. + var copy = b.openFor(tid, c.rows() * @divExact(c.head_dim, 8), 256, .{}); + { + const row = copy.iv.div(@divExact(c.head_dim, 8)); + const col = copy.iv.rem(@divExact(c.head_dim, 8)).mul(8); + const packed_row = first_row.add(row); + const valid = packed_row.lt(qlen.mul(c.group)); + const token = start.add(b.select(valid, packed_row.div(c.group), b.cst(.i32, 0))); + const qi = token.to(.i64).mul(c.kv_heads).add(ctx.head.to(.i64)).mul(c.group).add(packed_row.rem(c.group).to(.i64)).mul(c.head_dim).add(col.to(.i64)); + b.cpAsync16(s.add(qBase(c)).add(swizzle(row, col, c.rows())), ctx.q.add(qi.mul(2)), valid); + copy.yield(.{}); + } + b.cpAsyncWaitAll(); + b.syncThreads(); + ctx.tmem = b.loadWords(1, s.add(holder(c)), .shared, .{})[0]; + b.tcgen05Fence(.after); + if (c.split_stride > 0) { + // A fixed grid reuses Q, shared memory and TMEM across independent + // split partitions. Runtime lengths bound this device loop only. + // D256 has four load stages: both two-stage rings and the QK/PV + // completion barrier return to parity zero after every partition. + var partitions = b.openFor(split, splits, c.split_stride, .{}); + { + ctx.first_tile = first_tile.add(partitions.iv); + ctx.split = partitions.iv; + var producer = b.openIf(tid.ge(128)); + { + loadStages(ctx); + producer.yieldThen(.{}); + } + var consumer = b.openIf(tid.lt(128)); + { + decode(ctx); + consumer.yieldThen(.{}); + } + b.tcgen05Fence(.before); + b.syncThreads(); + b.tcgen05Fence(.after); + partitions.yield(.{}); + } + } else { + var producer = b.openIf(tid.ge(128)); + { + loadStages(ctx); + producer.yieldThen(.{}); + } + var consumer = b.openIf(tid.lt(128)); + { + if (c.all_decode) decode(ctx) else prefill(ctx); + consumer.yieldThen(.{}); + } + b.tcgen05Fence(.before); + b.syncThreads(); + } + var free = b.openIf(tid.lt(32)); + { + b.tmemDeallocAt(ctx.tmem, tmemCols(c)); + free.yieldThen(.{}); + } + active.yieldThen(.{}); + } +} + +fn ready(c: Ctx, id: V) V { + return c.s.add(barBase(c.c)).add(id.rem(c.c.load_buffers).mul(8)); +} +fn released(c: Ctx, id: V) V { + return ready(c, id).add(c.c.load_buffers * 8); +} +fn phase(c: Ctx, id: V) V { + return id.div(c.c.load_buffers).rem(2); +} + +fn loadStages(c: Ctx) void { + if (c.c.all_decode) return loadDecodeStages(c); + const b = c.b; + // Prefill computes half the output features per CTA; decode computes all. + const k_stages = @divExact(c.c.head_dim, 128); + const stages = k_stages + if (c.c.all_decode) k_stages else 2; + var work = b.openFor(0, c.tiles.mul(stages), 1, .{}); + { + const id = work.iv; + const part = id.rem(stages); + var reuse = b.openIf(id.ge(c.c.load_buffers)); + { + b.mbarrierWait(released(c, id), phase(c, id).bitXor(1)); + reuse.yieldThen(.{}); + } + const is_k = part.lt(k_stages); + const d0 = b.select(is_k, part.mul(128), part.sub(k_stages).mul(128).add(c.output_half.mul(256))); + const source = b.select(is_k, c.k, c.v); + const key0 = c.first_tile.add(id.div(stages)).mul(64); + var vec = b.openFor(c.tid.sub(128), 64 * 16, 128, .{}); + { + const row = vec.iv.div(16); + const col = vec.iv.rem(16).mul(8); + const key = key0.add(row); + const valid = key.lt(c.klen); + // Do not read unused page table entries (which may contain -1). + const safe_key = b.select(valid, key, b.cst(.i32, 0)); + const page = loadI32(b, c.table, c.seq.mul(c.c.pages_per_seq).add(safe_key.div(c.c.page_size))); + const offset = page.to(.i64).mul(c.c.page_size).add(safe_key.rem(c.c.page_size).to(.i64)).mul(c.c.kv_heads).add(c.head.to(.i64)).mul(c.c.head_dim).add(d0.add(col).to(.i64)); + b.cpAsync16(c.s.add(id.rem(c.c.load_buffers).mul(kvBytes(c.c))).add(swizzle(row, col, 64)), source.add(offset.mul(2)), valid); + vec.yield(.{}); + } + b.cpAsyncMbarrierArrive(ready(c, id), true); + work.yield(.{}); + } +} + +// Reuse the paged addresses across all K/V feature stages of a decode tile. +fn loadDecodeStages(c: Ctx) void { + const b = c.b; + const k_stages: usize = @intCast(@divExact(c.c.head_dim, 128)); + const stages = 2 * k_stages; + const tid = c.tid.sub(128); + const row0 = tid.div(16); + const col = tid.rem(16).mul(8); + var tiles = b.openFor(0, c.tiles, 1, .{}); + { + const key0 = c.first_tile.add(tiles.iv).mul(c.c.kv_tile).add(row0); + var offsets: [32]V = undefined; + var masks: [32]V = undefined; + for (0..@as(usize, @intCast(@divExact(c.c.kv_tile, 8)))) |i| { + const key = key0.add(@as(i32, @intCast(i * 8))); + masks[i] = key.lt(c.klen); + const safe_key = b.select(masks[i], key, b.cst(.i32, 0)); + const page = loadI32(b, c.table, c.seq.mul(c.c.pages_per_seq).add(safe_key.div(c.c.page_size))); + offsets[i] = page.to(.i64).mul(c.c.page_size).add(safe_key.rem(c.c.page_size).to(.i64)).mul(c.c.kv_heads).add(c.head.to(.i64)).mul(c.c.head_dim).add(col.to(.i64)).mul(2); + } + for (0..stages) |part| { + const id = tiles.iv.mul(stages).add(@as(i32, @intCast(part))); + var reuse = b.openIf(id.ge(c.c.load_buffers)); + { + b.mbarrierWait(released(c, id), phase(c, id).bitXor(1)); + reuse.yieldThen(.{}); + } + const source = if (part < k_stages) c.k else c.v; + const d0: i32 = @intCast((part % k_stages) * 128 * 2); + for (0..@as(usize, @intCast(@divExact(c.c.kv_tile, 8)))) |i| { + const row = row0.add(@as(i32, @intCast(i * 8))); + b.cpAsync16(c.s.add(id.rem(c.c.load_buffers).mul(kvBytes(c.c))).add(swizzle(row, col, c.c.kv_tile)), source.add(offsets[i]).add(d0), masks[i]); + } + b.cpAsyncMbarrierArrive(ready(c, id), true); + } + tiles.yield(.{}); + } +} + +fn mma(c: Ctx, tile: V, comptime qk: bool) void { + const b = c.b; + const k_stages = @divExact(c.c.head_dim, 128); + const stages = k_stages + if (c.c.all_decode) k_stages else 2; + const count: usize = if (qk or c.c.all_decode) @intCast(k_stages) else 2; + var leader = b.openIf(c.tid.eq(0)); + { + for (0..count) |part| { + const id = tile.mul(stages).add(@as(i32, @intCast(part + if (qk) @as(usize, 0) else @as(usize, @intCast(k_stages))))); + b.mbarrierWait(ready(c, id), phase(c, id)); + b.tcgen05Fence(.after); + const kv = c.s.add(id.rem(c.c.load_buffers).mul(kvBytes(c.c))); + const kd = descriptor(b, kv, if (qk) 16 else @intCast(c.c.kv_tile * 128)); + const qd = descriptor(b, c.s.add(if (qk) qBase(c.c) else pBase(c.c)), 16); + const score_slabs: usize = if (qk and c.c.all_decode) @intCast(@divTrunc(c.c.kv_tile + 127, 128)) else 1; + const steps: usize = if (qk) 8 else @intCast(@divExact(c.c.kv_tile, 16)); + for (0..score_slabs) |slab| { + for (0..steps) |step| { + const step_off: i64 = @as(i64, @intCast(step / 4)) * c.c.kv_tile * 128 + @as(i64, @intCast(step % 4)) * 32; + const kvd = kd.add(if (qk) @divExact(step_off + @as(i64, @intCast(slab)) * 128 * 128, 16) else @as(i64, @intCast(step * 2048 / 16))); + const other_off: i64 = if (qk) @as(i64, @intCast(part * 2 + step / 4)) * c.c.rows() * 128 + @as(i64, @intCast(step % 4)) * 32 else @as(i64, @intCast(step / 4)) * c.c.rows() * 128 + @as(i64, @intCast(step % 4)) * 32; + const other = qd.add(@divExact(other_off, 16)); + const swapped = c.c.all_decode; + const ad = if (swapped) kvd else other; + const bd = if (swapped) other else kvd; + const m: u32 = if (swapped) (if (qk) @intCast(@min(128, c.c.kv_tile)) else 128) else 64; + const n: u32 = if (swapped) @intCast(c.c.rows()) else if (qk) 64 else 128; + const instruction = instrDesc(c.c.dtype, m, n, swapped and !qk, !swapped and !qk); + const dst = c.tmem.add(if (qk) scoreCol(c.c) + @as(i32, @intCast(slab)) * @as(i32, @intCast(c.c.rows())) else @as(i32, @intCast(part * (if (swapped) @as(usize, @intCast(c.c.rows())) else 128)))); + const acc = if (qk) b.cst(.i32, @intFromBool(part > 0 or step > 0)).ne(0) else tile.gt(0).bitOr(b.cst(.i32, @intFromBool(step > 0)).ne(0)); + b.tcgen05MmaF16(dst, ad, bd, b.cst(.i32, @as(i32, @bitCast(instruction))), acc); + } + } + b.tcgen05Commit(released(c, id)); + } + b.tcgen05Commit(c.s.add(barBase(c.c) + @as(i32, @intCast(c.c.load_buffers * 16)))); + leader.yieldThen(.{}); + } + // Two completions per tile: QK parity 0 and PV parity 1. + b.mbarrierWait(c.s.add(barBase(c.c) + @as(i32, @intCast(c.c.load_buffers * 16))), b.cst(.i32, if (qk) 0 else 1)); + b.tcgen05Fence(.after); + // All consumers must observe PV completion before the issuing warp can + // complete the next QK and wrap this shared barrier's parity again. + // Single-tile splits rendezvous at the outer full-CTA barrier instead. + if (!qk and c.c.tiles_per_split != 1) b.barrierSync(1, 128); +} + +fn validScore(c: Ctx, row: V, key: V) V { + const pos = c.klen.sub(c.qlen).add(row.div(c.c.group)); + var valid = row.lt(c.qlen.mul(c.c.group)).bitAnd(key.lt(c.klen)); + if (c.c.causal) valid = valid.bitAnd(key.le(pos)); + if (c.c.window > 0) valid = valid.bitAnd(key.gt(pos.sub(c.c.window))); + return valid; +} + +fn storeP(c: Ctx, row: V, col: V, value: V) void { + c.b.storeHalf(c.s.add(pBase(c.c)).add(swizzle(row, col, c.c.rows())), .shared, value.to(c.c.dtype).bitCast(.i16).to(.i32)); +} + +fn consumerSync(c: Ctx) void { + c.b.fenceProxyAsyncShared(); + c.b.tcgen05Fence(.before); + c.b.barrierSync(1, 128); + c.b.tcgen05Fence(.after); +} + +fn prefill(c: Ctx) void { + const b = c.b; + const row = c.first_row.add(c.warp.mul(16)).add(c.lane); + const owns = c.lane.lt(16); + const lane_tmem = c.tmem.add(c.warp.mul(32).shl(16)); + const zero = b.cst(.f32, 0); + const neg_inf = b.cst(.f32, -std.math.inf(f32)); + var loop = b.openFor(0, c.tiles, 1, .{ neg_inf, zero }); + { + mma(c, loop.iv, true); + const raw = b.tmemLoad32x32b(64, lane_tmem.add(scoreCol(c.c))); + b.fenceTmemLoad(); + var scores: [64]V = undefined; + var maximum = loop.carried[0]; + for (0..64) |j| { + const key = c.first_tile.add(loop.iv).mul(64).add(@as(i32, @intCast(j))); + scores[j] = b.select(validScore(c, row, key).bitAnd(owns), raw[j].bitCast(.f32).mul(c.c.scale * std.math.log2e), neg_inf); + maximum = maximum.maximum(scores[j]); + } + const safe_max = b.select(maximum.eq(neg_inf), zero, maximum); + const alpha = b.exp2Approx(loop.carried[0].sub(safe_max)); + var sum = loop.carried[1].mul(alpha); + var probabilities: [64]V = undefined; + for (0..64) |j| { + probabilities[j] = b.exp2Approx(scores[j].sub(safe_max)); + sum = sum.add(probabilities[j]); + } + var write = b.openIf(owns); + { + for (0..64) |j| storeP(c, c.warp.mul(16).add(c.lane), b.cst(.i32, @as(i32, @intCast(j))), probabilities[j]); + write.yieldThen(.{}); + } + var rescale = b.openIf(loop.iv.gt(0)); + { + for (0..8) |chunk| { + const addr = lane_tmem.add(@as(i32, @intCast(chunk * 32))); + var out = b.tmemLoad32x32b(32, addr); + b.fenceTmemLoad(); + for (&out) |*v| v.* = v.bitCast(.f32).mul(alpha).bitCast(.i32); + b.tmemStore32x32b(32, addr, out); + } + b.fenceTmemStore(); + rescale.yieldThen(.{}); + } + consumerSync(c); + mma(c, loop.iv, false); + loop.yield(.{ maximum, sum }); + } + const norm = normalization(c, row, loop.results[0], loop.results[1]); + for (0..8) |chunk| { + const out = b.tmemLoad32x32b(32, lane_tmem.add(@as(i32, @intCast(chunk * 32)))); + b.fenceTmemLoad(); + var valid = b.openIf(owns.bitAnd(row.lt(c.qlen.mul(c.c.group)))); + { + for (0..32) |j| { + const dim = c.output_half.mul(256).add(@as(i32, @intCast(chunk * 32 + j))); + storeOutput(c, row, dim, out[j].bitCast(.f32).mul(norm)); + } + valid.yieldThen(.{}); + } + } +} + +fn tuple(comptime n: usize, a: [n]V) @Tuple(&@as([n]type, @splat(V))) { + var t: @Tuple(&@as([n]type, @splat(V))) = undefined; + inline for (0..n) |i| t[i] = a[i]; + return t; +} + +fn reduceHeads(c: Ctx, comptime heads: usize, values: [heads]V, comptime max: bool) [heads]V { + const b = c.b; + for (values, 0..) |v, h| { + var r = v; + inline for (.{ 16, 8, 4, 2, 1 }) |step| r = if (max) r.maximum(b.shuffleXor(r, step)) else r.add(b.shuffleXor(r, step)); + var leader = b.openIf(c.lane.eq(0)); + { + b.storeWords(1, c.s.add(redBase(c.c)).add(c.warp.mul(64)).add(@as(i32, @intCast(h * 4))), .shared, .{r.bitCast(.i32)}); + leader.yieldThen(.{}); + } + } + b.barrierSync(1, 128); + var result: [heads]V = undefined; + for (0..heads) |h| { + var value = b.loadWords(1, c.s.add(redBase(c.c) + @as(i32, @intCast(h * 4))), .shared, .{})[0].bitCast(.f32); + for (1..4) |w| { + const next = b.loadWords(1, c.s.add(redBase(c.c) + @as(i32, @intCast(w * 64 + h * 4))), .shared, .{})[0].bitCast(.f32); + value = if (max) value.maximum(next) else value.add(next); + } + result[h] = value; + } + b.barrierSync(1, 128); + return result; +} + +fn decode(c: Ctx) void { + if (c.c.kv_tile == 256) decodeTile(c, 2) else decodeTile(c, 1); +} + +fn decodeTile(c: Ctx, comptime slabs: usize) void { + // MMA still uses its legal 8/16-row tile. Only process the statically + // occupied head columns in the scalar softmax and output paths. + if (c.c.group <= 2) decodeSized(c, 2, slabs) else if (c.c.group <= 4) decodeSized(c, 4, slabs) else if (c.c.rows() == 8) decodeSized(c, 8, slabs) else decodeSized(c, 16, slabs); +} + +fn decodeSized(c: Ctx, comptime heads: usize, comptime slabs: usize) void { + const b = c.b; + const lane_tmem = c.tmem.add(c.warp.mul(32).shl(16)); + const key_row = c.warp.mul(@as(i32, if (c.c.kv_tile == 64) 16 else 32)).add(c.lane); + const owns = if (c.c.kv_tile == 64) c.lane.lt(16) else b.cst(.i32, 1).ne(0); + const zero = b.cst(.f32, 0); + const neg_inf = b.cst(.f32, -std.math.inf(f32)); + var initial: [2 * heads]V = undefined; + for (0..heads) |h| { + initial[h] = neg_inf; + initial[h + heads] = zero; + } + var loop = b.openFor(0, c.tiles, 1, tuple(2 * heads, initial)); + { + mma(c, loop.iv, true); + var scores: [slabs][heads]V = undefined; + var local_max: [heads]V = @splat(neg_inf); + for (0..slabs) |slab| { + const raw = b.tmemLoad32x32b(heads, lane_tmem.add(scoreCol(c.c) + @as(i32, @intCast(slab)) * @as(i32, @intCast(c.c.rows())))); + b.fenceTmemLoad(); + const key = c.first_tile.add(loop.iv).mul(c.c.kv_tile).add(key_row).add(@as(i32, @intCast(slab * 128))); + for (0..heads) |h| { + scores[slab][h] = b.select(owns.bitAnd(validScore(c, c.first_row.add(@as(i32, @intCast(h))), key)), raw[h].bitCast(.f32).mul(c.c.scale * std.math.log2e), neg_inf); + local_max[h] = local_max[h].maximum(scores[slab][h]); + } + } + const tile_max = reduceHeads(c, heads, local_max, true); + var next: [2 * heads]V = undefined; + var alpha: [heads]V = undefined; + var probabilities: [slabs][heads]V = undefined; + var local_sum: [heads]V = @splat(zero); + for (0..heads) |h| { + next[h] = loop.carried[h].maximum(tile_max[h]); + const safe_max = b.select(next[h].eq(neg_inf), zero, next[h]); + alpha[h] = b.exp2Approx(loop.carried[h].sub(safe_max)); + for (0..slabs) |slab| { + probabilities[slab][h] = b.exp2Approx(scores[slab][h].sub(safe_max)); + local_sum[h] = local_sum[h].add(probabilities[slab][h]); + } + } + const sums = reduceHeads(c, heads, local_sum, false); + for (0..heads) |h| next[h + heads] = loop.carried[h + heads].mul(alpha[h]).add(sums[h]); + var write = b.openIf(owns); + { + for (0..slabs) |slab| { + const key_col = key_row.add(@as(i32, @intCast(slab * 128))); + for (0..heads) |h| storeP(c, b.cst(.i32, @as(i32, @intCast(h))), key_col, probabilities[slab][h]); + for (heads..@as(usize, @intCast(c.c.rows()))) |h| storeP(c, b.cst(.i32, @as(i32, @intCast(h))), key_col, zero); + } + write.yieldThen(.{}); + } + if (c.c.tiles_per_split != 1) { + var rescale = b.openIf(loop.iv.gt(0)); + { + for (0..@as(usize, @intCast(@divExact(c.c.head_dim, 128)))) |chunk| { + const addr = lane_tmem.add(@as(i32, @intCast(chunk * @as(usize, @intCast(c.c.rows()))))); + var out = b.tmemLoad32x32b(heads, addr); + b.fenceTmemLoad(); + for (0..heads) |h| out[h] = out[h].bitCast(.f32).mul(alpha[h]).bitCast(.i32); + b.tmemStore32x32b(heads, addr, out); + } + b.fenceTmemStore(); + rescale.yieldThen(.{}); + } + } + consumerSync(c); + mma(c, loop.iv, false); + loop.yield(tuple(2 * heads, next)); + } + var norms: [heads]V = undefined; + for (0..heads) |h| norms[h] = if (c.c.splits > 1) b.cst(.f32, 1) else normalization(c, c.first_row.add(@as(i32, @intCast(h))), loop.results[h], loop.results[h + heads]); + for (0..@as(usize, @intCast(@divExact(c.c.head_dim, 128)))) |chunk| { + const out = b.tmemLoad32x32b(heads, lane_tmem.add(@as(i32, @intCast(chunk * @as(usize, @intCast(c.c.rows())))))); + b.fenceTmemLoad(); + const dim = c.warp.mul(32).add(c.lane).add(@as(i32, @intCast(chunk * 128))); + for (0..heads) |h| { + const row = c.first_row.add(@as(i32, @intCast(h))); + var valid = b.openIf(row.lt(c.qlen.mul(c.c.group))); + { + storeOutput(c, row, dim, out[h].bitCast(.f32).mul(norms[h])); + if (c.c.splits > 1 and chunk == 0) { + var stats = b.openIf(c.tid.eq(0)); + { + storeOutput(c, row, b.cst(.i32, c.c.head_dim), loop.results[h]); + storeOutput(c, row, b.cst(.i32, c.c.head_dim + 1), loop.results[h + heads]); + stats.yieldThen(.{}); + } + } + valid.yieldThen(.{}); + } + } + } +} + +fn normalization(c: Ctx, row: V, m: V, l: V) V { + const b = c.b; + if (!c.c.sink) return b.select(l.gt(0), b.rcpApprox(l), b.cst(.f32, 0)); + const h = c.head.mul(c.c.group).add(row.rem(c.c.group)); + const sink = loadI32(b, c.sink, h).bitCast(.f32).mul(std.math.log2e); + const maximum = m.maximum(sink); + const weight = b.exp2Approx(m.sub(maximum)); + return weight.div(l.mul(weight).add(b.exp2Approx(sink.sub(maximum)))); +} + +fn storeOutput(c: Ctx, row: V, dim: V, value: V) void { + const token = c.start.add(row.div(c.c.group)); + var offset = token.to(.i64).mul(c.c.kv_heads).add(c.head.to(.i64)).mul(c.c.group).add(row.rem(c.c.group).to(.i64)); + if (c.c.splits > 1) { + offset = c.split.to(.i64).mul(c.c.tokens * c.c.kv_heads * c.c.group).add(offset).mul(c.c.head_dim + 2).add(dim.to(.i64)); + c.b.storeWords(1, c.out.add(offset.mul(4)), .global, .{value.bitCast(.i32)}); + } else { + offset = offset.mul(c.c.head_dim).add(dim.to(.i64)); + c.b.storeHalf(c.out.add(offset.mul(2)), .global, value.to(c.c.dtype).bitCast(.i16).to(.i32)); + } +} + +fn descriptor(b: *B, address: V, lbo: u64) V { + const flags: u64 = (64 << 32) | (1 << 46) | (@as(u64, 2) << 61) | ((lbo / 16) << 16); + return address.to(.i64).shrLogical(4).bitAnd(0x3fff).bitOr(b.cst(.i64, @as(i64, @bitCast(flags)))); +} +fn swizzle(row: V, col: V, rows: i64) V { + return col.div(64).mul(rows * 128).add(row.mul(128)).add(col.rem(64).div(8).bitXor(row.rem(8)).mul(16)).add(col.rem(8).mul(2)); +} +fn instrDesc(dtype: cute.DType, m: u32, n: u32, a_mn: bool, b_mn: bool) u32 { + const format: u32 = if (dtype == .bf16) 1 else 0; + return (1 << 4) | (format << 7) | (format << 10) | (@as(u32, @intFromBool(a_mn)) << 15) | (@as(u32, @intFromBool(b_mn)) << 16) | ((n >> 3) << 17) | ((m >> 4) << 24); +} + +pub const Merge = cute.Kernel(Config, .{ + .name = "paged_attention_cute_merge", + .inputs = &.{ "partials", "lengths", "starts", "sink" }, + .outputs = &.{"out"}, + .run = merge, +}); + +fn merge(b: *B, c: Config) cute.FinishError!void { + const a = try b.declareArgs(.{ + .partials = .{ .tensor = .{ .dtype = .f32, .shape = &.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 } } }, + .lengths = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch} } }, + .starts = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch + 1} } }, + .sink = .{ .tensor = .{ .dtype = .f32, .shape = &.{c.kv_heads * c.group} } }, + .out = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.tokens, c.kv_heads * c.group, c.head_dim } } }, + }); + const seq = b.blockIdx().y; + const token = a.starts.get(.{seq}); + var valid = b.openIf(token.lt(a.starts.get(.{seq.add(1)}))); + { + const head = b.blockIdx().x; + const len = a.lengths.get(.{seq}); + const begin = if (c.window > 0) len.sub(c.window).maximum(0) else b.cst(.i32, 0); + const tiles = len.add(c.kv_tile - 1).div(c.kv_tile).sub(begin.div(c.kv_tile)); + const splits = liveSplits(b, c, a.starts.get(.{c.batch}), tiles); + const neg_inf = b.cst(.f32, -std.math.inf(f32)); + var maximum = b.openFor(0, splits, 1, .{if (c.sink) a.sink.get(.{head}).mul(std.math.log2e) else neg_inf}); + { + maximum.yield(.{maximum.carried[0].maximum(a.partials.get(.{ maximum.iv, token, head, c.head_dim }))}); + } + const m = maximum.results[0]; + var dims = b.openFor(b.threadIdx().x, c.head_dim, b.blockDim().x, .{}); + { + const zero = b.cst(.f32, 0); + var sum = b.openFor(0, splits, 1, .{ zero, if (c.sink) b.exp2Approx(a.sink.get(.{head}).mul(std.math.log2e).sub(m)) else zero }); + { + const weight = b.exp2Approx(a.partials.get(.{ sum.iv, token, head, c.head_dim }).sub(m)); + sum.yield(.{ sum.carried[0].add(weight.mul(a.partials.get(.{ sum.iv, token, head, dims.iv }))), sum.carried[1].add(weight.mul(a.partials.get(.{ sum.iv, token, head, c.head_dim + 1 }))) }); + } + a.out.set(.{ token, head, dims.iv }, sum.results[0].div(sum.results[1]).to(c.dtype)); + dims.yield(.{}); + } + valid.yieldThen(.{}); + } +} + +test "CuTe paged attention emits prefill and decode tensor cores" { + for ([_]i64{ 256, 512 }) |head_dim| for ([_]bool{ false, true }) |all_decode| { + const cfg: Config = .{ .head_dim = head_dim, .tokens = 80, .batch = 4, .kv_heads = 2, .group = 6, .pages_per_seq = 128, .page_size = 16, .max_query = if (all_decode) 1 else 65, .dtype = .bf16, .all_decode = all_decode, .causal = true, .window = 0, .scale = 0.044194174, .sink = true, .sms = 148, .splits = if (all_decode) 64 else 1 }; + const ir = try Kernel.emit(std.testing.allocator, cfg, block); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "tcgen05.mma") != null); + if (all_decode) { + const merge_ir = try Merge.emit(std.testing.allocator, cfg, .{ 128, 1, 1 }); + defer std.testing.allocator.free(merge_ir); + } + }; +} From be49a71bfff25d2a4db8797bcb6975612644655f Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 22 Oct 2026 10:48:08 +0200 Subject: [PATCH 812/881] zml/kernels/paged_attention/cute: add sm120 specific kernels (#972) This PR adds a `sm120` specific implementation of paged attention in `cute` --- zml/kernels/paged_attention/cute/BUILD.bazel | 4 +- zml/kernels/paged_attention/cute/cases.zig | 23 +- .../paged_attention/cute/correctness_test.zig | 38 +- zml/kernels/paged_attention/cute/cute.zig | 6 +- zml/kernels/paged_attention/cute/fixtures.zig | 9 +- .../paged_attention/cute/reference.zig | 1 + .../paged_attention/cute/sm100/BUILD.bazel | 9 + .../attention.zig} | 2 +- .../cute/{ => sm100}/sm100.zig | 8 +- .../paged_attention/cute/sm120/BUILD.bazel | 9 + .../paged_attention/cute/sm120/attention.zig | 457 ++++++++++++++++++ .../paged_attention/cute/sm120/sm120.zig | 83 ++++ 12 files changed, 630 insertions(+), 19 deletions(-) create mode 100644 zml/kernels/paged_attention/cute/sm100/BUILD.bazel rename zml/kernels/paged_attention/cute/{sm100_attention.zig => sm100/attention.zig} (99%) rename zml/kernels/paged_attention/cute/{ => sm100}/sm100.zig (97%) create mode 100644 zml/kernels/paged_attention/cute/sm120/BUILD.bazel create mode 100644 zml/kernels/paged_attention/cute/sm120/attention.zig create mode 100644 zml/kernels/paged_attention/cute/sm120/sm120.zig diff --git a/zml/kernels/paged_attention/cute/BUILD.bazel b/zml/kernels/paged_attention/cute/BUILD.bazel index d57a13f8..b96272b2 100644 --- a/zml/kernels/paged_attention/cute/BUILD.bazel +++ b/zml/kernels/paged_attention/cute/BUILD.bazel @@ -7,8 +7,8 @@ filegroup( srcs = [ "attention.zig", "cute.zig", - "sm100.zig", - "sm100_attention.zig", + "//zml/kernels/paged_attention/cute/sm100", + "//zml/kernels/paged_attention/cute/sm120", ], ) diff --git a/zml/kernels/paged_attention/cute/cases.zig b/zml/kernels/paged_attention/cute/cases.zig index 0bf417f4..63aae6ec 100644 --- a/zml/kernels/paged_attention/cute/cases.zig +++ b/zml/kernels/paged_attention/cute/cases.zig @@ -2,7 +2,7 @@ const std = @import("std"); pub const page_size = 16; pub const topk = 8; -pub const Mask = enum { causal, non_causal, window, indices }; +pub const Mask = enum { causal, non_causal, window, non_causal_window, indices }; pub const Case = struct { name: []const u8 = "standard", @@ -13,6 +13,9 @@ pub const Case = struct { prefill: usize = 0, chunk: usize = 17, kv_len: usize = 65, + kv_capacity: ?usize = null, + inactive: usize = 0, + shuffle_pages: bool = false, skew: bool = false, mask: Mask = .causal, empty_indices: bool = false, @@ -21,19 +24,20 @@ pub const Case = struct { scale: ?f32 = null, pub fn tokens(c: Case) usize { - return c.prefill * c.chunk + c.batch - c.prefill; + return c.prefill * c.chunk + c.batch - c.prefill - c.inactive; } pub fn pages(c: Case) usize { - return std.math.divCeil(usize, c.kv_len, page_size) catch unreachable; + return std.math.divCeil(usize, c.kv_capacity orelse c.kv_len, page_size) catch unreachable; } pub fn heads(c: Case) usize { return c.kv_heads * c.group; } pub fn queryLen(c: Case, seq: usize) usize { - return if (seq < c.prefill) c.chunk else 1; + return if (seq >= c.batch - c.inactive) 0 else if (seq < c.prefill) c.chunk else 1; } pub fn kvLen(c: Case, seq: usize) usize { const qlen = c.queryLen(seq); + if (qlen == 0) return 0; const reduction = if (c.skew) (seq * 17) % c.kv_len else 0; return @max(qlen, c.kv_len - @min(reduction, c.kv_len - 1)); } @@ -41,7 +45,7 @@ pub const Case = struct { // Eight geometry cases plus orthogonal semantics/boundary cases, not a Cartesian product. pub const correctness = blk: { - var result: [17]Case = undefined; + var result: [26]Case = undefined; var i: usize = 0; for ([_]usize{ 64, 128, 256, 512 }) |dim| { for ([_]bool{ false, true }) |mixed| { @@ -58,6 +62,15 @@ pub const correctness = blk: { result[14] = .{ .name = "skew", .head_dim = 256, .batch = 4, .prefill = 2, .skew = true, .kv_len = 129 }; result[15] = .{ .name = "mixed_short_chunks", .batch = 4, .prefill = 2, .chunk = 16, .skew = true, .kv_len = 64 }; result[16] = .{ .name = "empty_indices", .prefill = 1, .mask = .indices, .empty_indices = true }; + result[17] = .{ .name = "d32", .head_dim = 32, .group = 1, .kv_len = 33 }; + result[18] = .{ .name = "odd_group", .group = 5, .prefill = 1, .chunk = 9, .kv_len = 79 }; + result[19] = .{ .name = "group_tile_tail", .head_dim = 256, .batch = 1, .group = 19, .kv_len = 257 }; + result[20] = .{ .name = "shuffled_mixed", .head_dim = 512, .batch = 4, .prefill = 2, .group = 3, .chunk = 11, .kv_len = 129, .shuffle_pages = true }; + result[21] = .{ .name = "split_sink", .head_dim = 512, .batch = 1, .kv_len = 4097, .sink = true, .shuffle_pages = true }; + result[22] = .{ .name = "split_window_sink", .head_dim = 256, .batch = 1, .kv_len = 4097, .mask = .window, .window = 257, .sink = true }; + result[23] = .{ .name = "inactive_sequence", .batch = 4, .prefill = 1, .inactive = 1, .kv_len = 129 }; + result[24] = .{ .name = "non_causal_window", .head_dim = 64, .prefill = 1, .kv_len = 65, .mask = .non_causal_window, .window = 7 }; + result[25] = .{ .name = "empty_splits", .batch = 1, .kv_len = 1, .kv_capacity = 4096, .sink = true }; break :blk result; }; diff --git a/zml/kernels/paged_attention/cute/correctness_test.zig b/zml/kernels/paged_attention/cute/correctness_test.zig index e61ff44b..efa34c0c 100644 --- a/zml/kernels/paged_attention/cute/correctness_test.zig +++ b/zml/kernels/paged_attention/cute/correctness_test.zig @@ -27,7 +27,7 @@ test "paged attention correctness" { c.name, c.head_dim, c.batch, c.kv_heads, c.group, c.prefill, if (c.prefill > 0) c.chunk else 1, c.kv_len, @tagName(c.mask), }); - runCorrectness(a, io, platform, c) catch |err| { + runCorrectness(a, io, platform, c, false) catch |err| { failures += 1; std.debug.print("FAIL error={s}\n", .{@errorName(err)}); }; @@ -36,8 +36,27 @@ test "paged attention correctness" { try std.testing.expectEqual(@as(usize, 0), failures); } -fn runCorrectness(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, c: cases.Case) !void { - const exe = try platform.compileFn(a, io, fixtures.forward, .{ Inputs.init(c), c, zml.paged_attention.Backend.cute }, .{ .program_name = "paged_attention_harness" }); +test "FP16 paged attention" { + const platform = zml.testing.env(); + if (platform.target != .cuda) return error.CudaRequired; + for ([_]cases.Case{ + .{ .name = "fp16_decode", .head_dim = 64, .kv_len = 257 }, + .{ .name = "fp16_mixed", .prefill = 1, .mask = .non_causal }, + .{ .name = "fp16_window_sink", .head_dim = 256, .prefill = 1, .mask = .window, .sink = true }, + .{ .name = "fp16_split_sink", .head_dim = 512, .batch = 1, .kv_len = 2049, .sink = true, .shuffle_pages = true }, + }) |c| try runCorrectness(std.testing.allocator, std.testing.io, platform, c, true); +} + +fn forwardF16(i: Inputs, c: cases.Case, backend: zml.paged_attention.Backend) zml.Tensor { + var inputs = i; + inputs.q = i.q.convert(.f16); + inputs.k = i.k.convert(.f16); + inputs.v = i.v.convert(.f16); + return fixtures.forward(inputs, c, backend).convert(.bf16); +} + +fn runCorrectness(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, c: cases.Case, comptime compute_f16: bool) !void { + const exe = try platform.compileFn(a, io, if (compute_f16) forwardF16 else fixtures.forward, .{ Inputs.init(c), c, zml.paged_attention.Backend.cute }, .{ .program_name = "paged_attention_harness" }); defer exe.deinit(); var runner = try exe.runner(a); @@ -47,6 +66,15 @@ fn runCorrectness(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platfor defer arena.deinit(); const fixture = try fixtures.Fixture.init(arena.allocator(), c); + if (compute_f16) { + // Exactly representable in both formats, so the existing CPU reference + // receives precisely the values used by the FP16 tensor-core path. + for ([_][]zml.floats.BFloat16{ fixture.q, fixture.k, fixture.v }) |values| { + for (values) |*v| if (std.math.isFinite(v.toF32())) { + v.* = .fromF32(@round(v.toF32() * 64) / 64); + }; + } + } var buffers = try fixtures.upload(io, platform, Inputs.init(c), fixture); defer zml.Buffer.deinitAll(Inputs, &buffers); @@ -59,7 +87,7 @@ fn runCorrectness(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platfor defer host.free(a); const max_error = try reference.check(a, c, fixture, host.items(zml.floats.BFloat16)); - std.debug.print("RESULT name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} backend=cute check=all_outputs max_abs={d:.6}\n", .{ - c.name, c.head_dim, c.batch, c.kv_heads, c.group, c.prefill, if (c.prefill > 0) c.chunk else 1, c.kv_len, max_error, + std.debug.print("RESULT name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} backend=cute compute_dtype={s} check=all_outputs max_abs={d:.6}\n", .{ + c.name, c.head_dim, c.batch, c.kv_heads, c.group, c.prefill, if (c.prefill > 0) c.chunk else 1, c.kv_len, if (compute_f16) "f16" else "bf16", max_error, }); } diff --git a/zml/kernels/paged_attention/cute/cute.zig b/zml/kernels/paged_attention/cute/cute.zig index 454f413c..999f6c24 100644 --- a/zml/kernels/paged_attention/cute/cute.zig +++ b/zml/kernels/paged_attention/cute/cute.zig @@ -2,7 +2,8 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const paged = @import("../paged_attention.zig"); -const sm100 = @import("sm100.zig"); +const sm100 = @import("sm100/sm100.zig"); +const sm120 = @import("sm120/sm120.zig"); const kernel = @import("attention.zig"); pub const Options = sm100.Options; @@ -11,6 +12,7 @@ pub const Parameters = sm100.Parameters; test { _ = kernel; _ = sm100; + _ = sm120; } pub fn isAvailable(platform: *const zml.Platform) bool { @@ -25,6 +27,8 @@ pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: z stdx.debug.assert(q.dim(.hd) == 32 or q.dim(.hd) == 64 or q.dim(.hd) == 128 or q.dim(.hd) == 256 or q.dim(.hd) == 512, "CuTe paged attention supports head dimensions 32, 64, 128, 256 and 512", .{}); if (sm100.isAvailable(zml.Compiler.current().platform) and (q.dim(.hd) == 256 or q.dim(.hd) == 512) and opts.mask != .indices) return sm100.pagedAttention(parameters, q, k, v, opts); + if (sm120.isAvailable(zml.Compiler.current().platform) and opts.mask != .indices) + return sm120.pagedAttention(parameters, q, k, v, opts); return zml.ops.manualComputation(Context.body, Context{ .q = q, .k = k, .v = v, .parameters = parameters, .opts = opts }, q.shape()); } diff --git a/zml/kernels/paged_attention/cute/fixtures.zig b/zml/kernels/paged_attention/cute/fixtures.zig index b7cc2224..210d7184 100644 --- a/zml/kernels/paged_attention/cute/fixtures.zig +++ b/zml/kernels/paged_attention/cute/fixtures.zig @@ -39,9 +39,15 @@ pub const Fixture = struct { for (0..c.pages()) |p| { f.table[s * c.pages() + p] = if (p * cases.page_size < len) @intCast(s * c.pages() + p) else -1; } + const live_pages = std.math.divCeil(usize, len, cases.page_size) catch unreachable; + if (c.shuffle_pages) random.shuffle(i32, f.table[s * c.pages() ..][0..live_pages]); // Unused physical slots must not contribute to PV, even via zero * NaN. for (len..c.pages() * cases.page_size) |j| { - const offset = (s * c.pages() * cases.page_size + j) * c.kv_heads * c.head_dim; + const page = if (j / cases.page_size < live_pages) + @as(usize, @intCast(f.table[s * c.pages() + j / cases.page_size])) + else + s * c.pages() + j / cases.page_size; + const offset = (page * cases.page_size + j % cases.page_size) * c.kv_heads * c.head_dim; @memset(f.k[offset..][0 .. c.kv_heads * c.head_dim], .fromF32(std.math.nan(f32))); @memset(f.v[offset..][0 .. c.kv_heads * c.head_dim], .fromF32(std.math.nan(f32))); } @@ -93,6 +99,7 @@ pub const Inputs = struct { .causal => .{ .standard = .{} }, .non_causal => .none, .window => .{ .standard = .{ .sliding_window = c.window } }, + .non_causal_window => .{ .standard = .{ .causal = false, .sliding_window = c.window } }, .indices => .{ .indices = i.indices }, }, .sink = if (c.sink) i.sinks else null, diff --git a/zml/kernels/paged_attention/cute/reference.zig b/zml/kernels/paged_attention/cute/reference.zig index 49b9dc43..1a030646 100644 --- a/zml/kernels/paged_attention/cute/reference.zig +++ b/zml/kernels/paged_attention/cute/reference.zig @@ -20,6 +20,7 @@ pub fn row(c: cases.Case, f: fixtures.Fixture, seq: usize, token: usize, head: u .indices, .non_causal => true, .causal => key <= position, .window => key <= position and @as(i64, @intCast(key)) > @as(i64, @intCast(position)) - c.window, + .non_causal_window => @as(i64, @intCast(key)) > @as(i64, @intCast(position)) - c.window, }; scores[j] = -std.math.inf(f32); if (!valid) continue; diff --git a/zml/kernels/paged_attention/cute/sm100/BUILD.bazel b/zml/kernels/paged_attention/cute/sm100/BUILD.bazel new file mode 100644 index 00000000..c8c2a20d --- /dev/null +++ b/zml/kernels/paged_attention/cute/sm100/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernels/paged_attention/cute:__pkg__"]) + +filegroup( + name = "sm100", + srcs = [ + "attention.zig", + "sm100.zig", + ], +) diff --git a/zml/kernels/paged_attention/cute/sm100_attention.zig b/zml/kernels/paged_attention/cute/sm100/attention.zig similarity index 99% rename from zml/kernels/paged_attention/cute/sm100_attention.zig rename to zml/kernels/paged_attention/cute/sm100/attention.zig index 12a6fe19..0787108a 100644 --- a/zml/kernels/paged_attention/cute/sm100_attention.zig +++ b/zml/kernels/paged_attention/cute/sm100/attention.zig @@ -1,6 +1,6 @@ const std = @import("std"); -const zml = @import("../../../zml.zig"); +const zml = @import("../../../../zml.zig"); const cute = zml.kernel.cute; const B = cute.Builder; const V = cute.Value; diff --git a/zml/kernels/paged_attention/cute/sm100.zig b/zml/kernels/paged_attention/cute/sm100/sm100.zig similarity index 97% rename from zml/kernels/paged_attention/cute/sm100.zig rename to zml/kernels/paged_attention/cute/sm100/sm100.zig index 007f153d..9c0f5ade 100644 --- a/zml/kernels/paged_attention/cute/sm100.zig +++ b/zml/kernels/paged_attention/cute/sm100/sm100.zig @@ -1,9 +1,9 @@ const stdx = @import("stdx"); -const zml = @import("../../../zml.zig"); -const paged = @import("../paged_attention.zig"); -const triton = @import("../triton/triton.zig"); -const kernel = @import("sm100_attention.zig"); +const zml = @import("../../../../zml.zig"); +const paged = @import("../../paged_attention.zig"); +const triton = @import("../../triton/triton.zig"); +const kernel = @import("attention.zig"); pub const Options = triton.paged.Options; pub const Parameters = triton.paged.Parameters; diff --git a/zml/kernels/paged_attention/cute/sm120/BUILD.bazel b/zml/kernels/paged_attention/cute/sm120/BUILD.bazel new file mode 100644 index 00000000..f93bd5e0 --- /dev/null +++ b/zml/kernels/paged_attention/cute/sm120/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernels/paged_attention/cute:__pkg__"]) + +filegroup( + name = "sm120", + srcs = [ + "attention.zig", + "sm120.zig", + ], +) diff --git a/zml/kernels/paged_attention/cute/sm120/attention.zig b/zml/kernels/paged_attention/cute/sm120/attention.zig new file mode 100644 index 00000000..92a7ba30 --- /dev/null +++ b/zml/kernels/paged_attention/cute/sm120/attention.zig @@ -0,0 +1,457 @@ +const std = @import("std"); +const mlir = @import("mlir"); +const llvm = @import("mlir/dialects").llvm; +const vector = @import("mlir/dialects").vector; + +const zml = @import("../../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const V = cute.Value; + +pub const Config = struct { + head_dim: i64, + tokens: i64, + batch: i64, + kv_heads: i64, + group: i64, + pages_per_seq: i64, + page_size: i64, + max_query: i64, + dtype: cute.DType, + warps_per_query: i64, + kv_tile: i64, + causal: bool, + window: i64, + scale: f32, + sink: bool, + splits: i64 = 1, + + pub fn rows(c: Config) i64 { + return @divExact(64, c.warps_per_query); + } + pub fn queryTiles(c: Config) i64 { + return @divTrunc(c.max_query * c.group + c.rows() - 1, c.rows()); + } + fn kvBase(c: Config) i64 { + return c.rows() * c.head_dim * 2; + } + fn pBase(c: Config) i64 { + return c.kvBase() + c.kv_tile * c.head_dim * 2; + } + fn redBase(c: Config) i64 { + return c.pBase() + c.rows() * c.kv_tile * 2; + } + fn sharedBytes(c: Config) i64 { + return c.redBase() + 2 * 64 * 4; + } +}; + +pub const Kernel = cute.Kernel(Config, .{ + .name = "paged_attention_cute_sm120", + .inputs = &.{ "q", "k", "v", "table", "lengths", "starts", "sink" }, + .outputs = &.{"out"}, + .run = run, +}); + +fn tuple(comptime n: usize, values: [n]V) @Tuple(&@as([n]type, @splat(V))) { + var result: @Tuple(&@as([n]type, @splat(V))) = undefined; + inline for (0..n) |i| result[i] = values[i]; + return result; +} + +fn flatten(comptime n: usize, values: [n][4]V) [n * 4]V { + var result: [n * 4]V = undefined; + for (0..n) |i| @memcpy(result[i * 4 ..][0..4], &values[i]); + return result; +} + +fn packAcc(b: *B, values: [4]V) V { + return b.emit(vector.from_elements(b.ctx, &.{ values[0].inner, values[1].inner, values[2].inner, values[3].inner }, b.parseType("vector<4xf32>"), b.loc())); +} + +fn loadI32(b: *B, ptr: V, index: V) V { + return b.loadWords(1, ptr.add(index.to(.i64).mul(4)), .global, .{})[0]; +} + +// XOR whole 16-byte vectors so cp.async and ldmatrix agree on bank layout. +fn swizzle(row: V, col: V, width: i64) V { + return row.mul(width).add(col.bitXor(row.rem(@min(8, @divExact(width, 8))).mul(8))).mul(2); +} + +fn unpackStruct(b: *B, comptime n: usize, value: V, dtype: cute.DType) [n]V { + var result: [n]V = undefined; + for (&result, 0..) |*r, i| r.* = b.emit(mlir.Operation.make(b.ctx, "llvm.extractvalue", .{ + .operands = .{ .flat = &.{value.inner} }, + .results = .{ .flat = &.{dtype.toMlir(b.ctx)} }, + .attributes = &.{.named(b.ctx, "position", .denseArray(b.ctx, .i64, &.{@intCast(i)}))}, + .location = b.loc(), + })); + return result; +} + +fn matrixLoad(b: *B, comptime n: usize, comptime transpose: bool, address: V) [n]V { + const assembly = std.fmt.comptimePrint("ldmatrix.sync.aligned.m8n8.x{d}{s}.shared.b16 {{", .{ n, if (transpose) ".trans" else "" }) ++ + (if (n == 4) "$0, $1, $2, $3}, [$4];" else "$0, $1}, [$2];"); + const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{address.inner}, b.parseType(if (n == 4) "!llvm.struct<(i32, i32, i32, i32)>" else "!llvm.struct<(i32, i32)>"), .{ .constraints = if (n == 4) "=r,=r,=r,=r,r" else "=r,=r,r", .has_side_effects = true }, b.loc())); + return unpackStruct(b, n, result, .i32); +} + +fn mma(b: *B, comptime dtype: cute.DType, a: [4]V, k: [2]V, acc: [4]V) [4]V { + const assembly = "mma.sync.aligned.m16n8k16.row.col.f32." ++ @tagName(dtype) ++ "." ++ @tagName(dtype) ++ ".f32 " ++ + "{$0, $1, $2, $3}, {$4, $5, $6, $7}, {$8, $9}, {$10, $11, $12, $13};"; + const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{ a[0].inner, a[1].inner, a[2].inner, a[3].inner, k[0].inner, k[1].inner, acc[0].inner, acc[1].inner, acc[2].inner, acc[3].inner }, b.parseType("!llvm.struct<(f32, f32, f32, f32)>"), .{ .constraints = "=f,=f,=f,=f,r,r,r,r,r,r,f,f,f,f" }, b.loc())); + return unpackStruct(b, 4, result, .f32); +} + +fn run(b: *B, c: Config) cute.FinishError!void { + switch (c.head_dim) { + inline 32, 64, 128, 256, 512 => |d| switch (c.warps_per_query) { + inline 2, 4 => |w| switch (c.kv_tile) { + inline 32, 64 => |n| switch (c.dtype) { + inline .bf16, .f16 => |dt| try runTile(b, c, d, w, n, dt), + else => unreachable, + }, + else => unreachable, + }, + else => unreachable, + }, + else => unreachable, + } +} + +const TileContext = struct { + b: *B, + c: Config, + q: V, + k: V, + v: V, + table: V, + sink: V, + out: V, + shared: V, + tid: V, + lane: V, + warp: V, + seq: V, + head: V, + first_row: V, + start: V, + qlen: V, + klen: V, + split: V, + + // Every load resolves its logical page on device. Tail copies are zeroed + // before either MMA, including V tails whose backing allocation holds NaN. + fn loadKV(t: TileContext, source: V, key_base: V) void { + const b = t.b; + const c = t.c; + if (c.page_size == 16 and c.head_dim >= 64) { + // A page ID is shared by all feature vectors in its 16 rows. + // Resolve it once, then issue independent vector copies instead + // of serializing a page-table load before every cp.async. + var pages = b.openFor(0, c.kv_tile, 16, .{}); + { + const first_key = key_base.add(pages.iv); + const page_valid = first_key.lt(t.klen); + const logical_page = b.select(page_valid, first_key.div(16), b.cst(.i32, 0)); + const page = loadI32(b, t.table, t.seq.mul(c.pages_per_seq).add(logical_page)); + const base = page.to(.i64).mul(16 * c.kv_heads * c.head_dim).add(t.head.to(.i64).mul(c.head_dim)); + for (0..@as(usize, @intCast(@divExact(c.head_dim, 64)))) |i| { + const vector_index = t.tid.add(i * 128); + const row = vector_index.div(@divExact(c.head_dim, 8)); + const col = vector_index.rem(@divExact(c.head_dim, 8)).mul(8); + const offset = base.add(row.to(.i64).mul(c.kv_heads * c.head_dim)).add(col.to(.i64)); + b.cpAsync16(t.shared.add(c.kvBase()).add(swizzle(pages.iv.add(row), col, c.head_dim)), source.add(offset.mul(2)), first_key.add(row).lt(t.klen)); + } + pages.yield(.{}); + } + return; + } + var copy = b.openFor(t.tid, c.kv_tile * @divExact(c.head_dim, 8), 128, .{}); + { + const row = copy.iv.div(@divExact(c.head_dim, 8)); + const col = copy.iv.rem(@divExact(c.head_dim, 8)).mul(8); + const key = key_base.add(row); + const valid = key.lt(t.klen); + const safe_key = b.select(valid, key, b.cst(.i32, 0)); + const page = loadI32(b, t.table, t.seq.mul(c.pages_per_seq).add(safe_key.div(c.page_size))); + const offset = page.to(.i64).mul(c.page_size).add(safe_key.rem(c.page_size).to(.i64)).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.head_dim).add(col.to(.i64)); + b.cpAsync16(t.shared.add(c.kvBase()).add(swizzle(row, col, c.head_dim)), source.add(offset.mul(2)), valid); + copy.yield(.{}); + } + } + + fn publishReduction(t: TileContext, row: V, value: V, comptime maximum: bool) void { + const b = t.b; + var v = value; + inline for (.{ 2, 1 }) |step| v = if (maximum) v.maximum(b.shuffleXor(v, step)) else v.add(b.shuffleXor(v, step)); + const address = t.shared.add(t.c.redBase() + (if (maximum) @as(i64, 0) else 256)).add(t.warp.mul(16).add(row).mul(4)); + var leader = b.openIf(t.lane.rem(4).eq(0)); + { + b.storeWords(1, address, .shared, .{v.bitCast(.i32)}); + leader.yieldThen(.{}); + } + // Callers synchronize once both row halves have been published. + } + + fn readReduction(t: TileContext, comptime w: usize, row: V, comptime maximum: bool) V { + const b = t.b; + const base = t.shared.add(t.c.redBase() + (if (maximum) @as(i64, 0) else 256)).add(t.warp.div(w).mul(w * 16).add(row).mul(4)); + var result = b.loadWords(1, base, .shared, .{})[0].bitCast(.f32); + for (1..w) |i| { + const v = b.loadWords(1, base.add(i * 16 * 4), .shared, .{})[0].bitCast(.f32); + result = if (maximum) result.maximum(v) else result.add(v); + } + return result; + } + + fn store(t: TileContext, row: V, dim: V, value: V) void { + const c = t.c; + const token = t.start.add(row.div(c.group)); + var offset = token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(row.rem(c.group).to(.i64)); + if (c.splits > 1) { + offset = t.split.to(.i64).mul(c.tokens * c.kv_heads * c.group).add(offset).mul(c.head_dim + 2).add(dim.to(.i64)); + t.b.storeWords(1, t.out.add(offset.mul(4)), .global, .{value.bitCast(.i32)}); + } else { + offset = offset.mul(c.head_dim).add(dim.to(.i64)); + t.b.storeHalf(t.out.add(offset.mul(2)), .global, value.to(c.dtype).bitCast(.i16).to(.i32)); + } + } +}; + +fn runTile(b: *B, c: Config, comptime d: usize, comptime w: usize, comptime n: usize, comptime dt: cute.DType) cute.FinishError!void { + std.debug.assert(c.sharedBytes() <= 99 * 1024); + const a = try b.declareArgs(.{ + .q = .{ .ptr = dt }, + .k = .{ .ptr = dt }, + .v = .{ .ptr = dt }, + .table = .{ .ptr = cute.DType.i32 }, + .lengths = .{ .ptr = cute.DType.i32 }, + .starts = .{ .ptr = cute.DType.i32 }, + .sink = .{ .ptr = cute.DType.f32 }, + .out = .{ .ptr = if (c.splits > 1) cute.DType.f32 else dt }, + }); + b.setFunctionAttribute("cu_attrs", b.parseAttribute("{max_dynamic_shared_size_bytes = #cuda.dev_max_shared_memory_optin}")); + const seq = b.blockIdx().x.div(c.queryTiles()); + const first_row = b.blockIdx().x.rem(c.queryTiles()).mul(c.rows()); + const starts = b.ptrToInt(a.starts, .i64); + const start = loadI32(b, starts, seq); + const qlen = loadI32(b, starts, seq.add(1)).sub(start); + var active = b.openIf(first_row.lt(qlen.mul(c.group))); + { + const tid = b.threadIdx().x; + const t: TileContext = .{ + .b = b, + .c = c, + .q = b.ptrToInt(a.q, .i64), + .k = b.ptrToInt(a.k, .i64), + .v = b.ptrToInt(a.v, .i64), + .table = b.ptrToInt(a.table, .i64), + .sink = b.ptrToInt(a.sink, .i64), + .out = b.ptrToInt(a.out, .i64), + .shared = b.ptrToInt(b.allocSmem(.i8, c.sharedBytes(), 128), .i32), + .tid = tid, + .lane = tid.rem(32), + .warp = tid.div(32), + .seq = seq, + .head = b.blockIdx().y, + .first_row = first_row, + .start = start, + .qlen = qlen, + .klen = loadI32(b, b.ptrToInt(a.lengths, .i64), seq), + .split = b.blockIdx().z, + }; + var copy_q = b.openFor(tid, c.rows() * (d / 8), 128, .{}); + { + const row = copy_q.iv.div(d / 8); + const col = copy_q.iv.rem(d / 8).mul(8); + const packed_row = first_row.add(row); + const valid = packed_row.lt(qlen.mul(c.group)); + const token = start.add(b.select(valid, packed_row.div(c.group), b.cst(.i32, 0))); + const offset = token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(packed_row.rem(c.group).to(.i64)).mul(d).add(col.to(.i64)); + b.cpAsync16(t.shared.add(swizzle(row, col, d)), t.q.add(offset.mul(2)), valid); + copy_q.yield(.{}); + } + const first_pos = t.klen.sub(qlen).add(first_row.div(c.group)); + const last_pos = t.klen.sub(qlen).add(first_row.add(c.rows() - 1).div(c.group).minimum(qlen.sub(1))); + const begin = if (c.window > 0) first_pos.sub(c.window - 1).maximum(0) else b.cst(.i32, 0); + const end = if (c.causal) last_pos.add(1).minimum(t.klen) else t.klen; + const first_tile = begin.div(n); + const tiles = end.add(n - 1).div(n).sub(first_tile).maximum(0); + const lo = first_tile.add(tiles.mul(t.split).div(c.splits)); + const hi = first_tile.add(tiles.mul(t.split.add(1)).div(c.splits)); + const row_base = t.warp.div(w).mul(16); + const key_part = t.warp.rem(w).mul(n / w); + const dim_part = t.warp.rem(w).mul(d / w); + const row0 = t.lane.div(4); + const col0 = t.lane.rem(4).mul(2); + const zero = b.cst(.f32, 0); + const neg_inf = b.cst(.f32, -std.math.inf(f32)); + const nf = n / w / 8; + const of = d / w / 8; + const count = 4 + of; + var initial: [count]V = @splat(zero); + initial[0] = neg_inf; + initial[1] = neg_inf; + for (initial[4..]) |*v| v.* = packAcc(b, .{ zero, zero, zero, zero }); + b.cpAsyncWaitAll(); + b.syncThreads(); + var loop = b.openFor(lo, hi, 1, tuple(count, initial)); + { + const key_base = loop.iv.mul(n); + t.loadKV(t.k, key_base); + b.cpAsyncWaitAll(); + b.syncThreads(); + var scores: [nf][4]V = @splat(@splat(zero)); + // Keep only MMA operands live in registers; Q remains in shared + // memory so D512 does not consume the entire per-thread register file. + var qk = b.openFor(0, d, 16, tuple(nf * 4, @splat(zero))); + { + const qa = matrixLoad(b, 4, false, t.shared.add(swizzle(row_base.add(t.lane.rem(16)), qk.iv.add(t.lane.div(16).mul(8)), d))); + for (0..nf) |i| { + const kb = matrixLoad(b, 2, false, t.shared.add(c.kvBase()).add(swizzle(key_part.add(i * 8).add(t.lane.rem(8)), qk.iv.add(t.lane.div(8).rem(2).mul(8)), d))); + scores[i] = mma(b, dt, qa, kb, qk.carried[i * 4 ..][0..4].*); + } + qk.yield(tuple(nf * 4, flatten(nf, scores))); + } + for (0..nf) |i| scores[i] = qk.results[i * 4 ..][0..4].*; + var local_max: [2]V = @splat(neg_inf); + for (0..nf) |i| { + for (0..4) |j| { + const row = first_row.add(row_base).add(row0).add((j / 2) * 8); + const position = t.klen.sub(qlen).add(row.div(c.group)); + const key = key_base.add(key_part).add(i * 8).add(col0).add(j % 2); + var valid = row.lt(qlen.mul(c.group)).bitAnd(key.lt(t.klen)); + if (c.causal) valid = valid.bitAnd(key.le(position)); + if (c.window > 0) valid = valid.bitAnd(key.gt(position.sub(c.window))); + scores[i][j] = b.select(valid, scores[i][j].mul(c.scale * std.math.log2e), neg_inf); + local_max[j / 2] = local_max[j / 2].maximum(scores[i][j]); + } + } + for (0..2) |h| t.publishReduction(row0.add(h * 8), local_max[h], true); + b.syncThreads(); + // All K reads have completed. Reuse its allocation for V, and + // overlap these global loads with softmax and probability stores. + t.loadKV(t.v, key_base); + var m: [2]V = undefined; + var alpha: [2]V = undefined; + var local_sum: [2]V = @splat(zero); + for (0..2) |h| { + m[h] = loop.carried[h].maximum(t.readReduction(w, row0.add(h * 8), true)); + // Empty/fully masked row tiles have m=-inf and denominator 0. + alpha[h] = b.exp2Approx(loop.carried[h].sub(m[h].maximum(-std.math.floatMax(f32)))); + } + for (0..nf) |i| { + for (0..4) |j| { + scores[i][j] = b.exp2Approx(scores[i][j].sub(m[j / 2].maximum(-std.math.floatMax(f32)))); + local_sum[j / 2] = local_sum[j / 2].add(scores[i][j]); + } + for (0..2) |h| { + const row = row_base.add(row0).add(h * 8); + const col = key_part.add(i * 8).add(col0); + b.storeWords(1, t.shared.add(c.pBase()).add(swizzle(row, col, n)), .shared, .{b.packHalf2(scores[i][h * 2], scores[i][h * 2 + 1], dt)}); + } + } + for (0..2) |h| t.publishReduction(row0.add(h * 8), local_sum[h], false); + b.cpAsyncWaitAll(); + b.syncThreads(); + var next: [count]V = undefined; + for (0..2) |h| { + next[h] = m[h]; + next[h + 2] = loop.carried[h + 2].mul(alpha[h]).add(t.readReduction(w, row0.add(h * 8), false)); + } + var acc: [of][4]V = undefined; + for (0..of) |i| for (0..4) |j| { + acc[i][j] = b.vectorExtract(loop.carried[4 + i], @intCast(j), .f32).mul(alpha[j / 2]); + }; + var packed_acc: [of]V = undefined; + for (0..of) |i| packed_acc[i] = packAcc(b, acc[i]); + var pv = b.openFor(0, n, 16, tuple(of, packed_acc)); + { + const pa = matrixLoad(b, 4, false, t.shared.add(c.pBase()).add(swizzle(row_base.add(t.lane.rem(16)), pv.iv.add(t.lane.div(16).mul(8)), n))); + for (0..of) |i| { + const vb = matrixLoad(b, 2, true, t.shared.add(c.kvBase()).add(swizzle(pv.iv.add(t.lane.rem(16)), dim_part.add(i * 8), d))); + var prev: [4]V = undefined; + for (0..4) |j| prev[j] = b.vectorExtract(pv.carried[i], @intCast(j), .f32); + acc[i] = mma(b, dt, pa, vb, prev); + packed_acc[i] = packAcc(b, acc[i]); + } + pv.yield(tuple(of, packed_acc)); + } + @memcpy(next[4..], &pv.results); + b.syncThreads(); + loop.yield(tuple(count, next)); + } + for (0..2) |h| { + const row = first_row.add(row_base).add(row0).add(h * 8); + var valid = b.openIf(row.lt(qlen.mul(c.group))); + { + const m = loop.results[h]; + const l = loop.results[h + 2]; + var norm = b.select(l.gt(0), b.rcpApprox(l), zero); + if (c.splits == 1 and c.sink) { + const sink = loadI32(b, t.sink, t.head.mul(c.group).add(row.rem(c.group))).bitCast(.f32).mul(std.math.log2e); + const maximum = m.maximum(sink); + const weight = b.exp2Approx(m.sub(maximum)); + norm = weight.div(l.mul(weight).add(b.exp2Approx(sink.sub(maximum)))); + } + for (0..of) |i| for (0..2) |j| { + const value = b.vectorExtract(loop.results[4 + i], @intCast(h * 2 + j), .f32); + t.store(row, dim_part.add(i * 8).add(col0).add(j), if (c.splits > 1) value else value.mul(norm)); + }; + if (c.splits > 1) { + var leader = b.openIf(t.warp.rem(w).eq(0).bitAnd(t.lane.rem(4).eq(0))); + { + t.store(row, b.cst(.i32, d), m); + t.store(row, b.cst(.i32, d + 1), l); + leader.yieldThen(.{}); + } + } + valid.yieldThen(.{}); + } + } + active.yieldThen(.{}); + } +} + +pub const Merge = cute.Kernel(Config, .{ + .name = "paged_attention_cute_sm120_merge", + .inputs = &.{ "partials", "starts", "sink" }, + .outputs = &.{"out"}, + .run = merge, +}); + +fn merge(b: *B, c: Config) cute.FinishError!void { + const a = try b.declareArgs(.{ + .partials = .{ .tensor = .{ .dtype = .f32, .shape = &.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 } } }, + .starts = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch + 1} } }, + .sink = .{ .tensor = .{ .dtype = .f32, .shape = &.{c.kv_heads * c.group} } }, + .out = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.tokens, c.kv_heads * c.group, c.head_dim } } }, + }); + const token = b.blockIdx().x; + const head = b.blockIdx().y; + var valid = b.openIf(token.lt(a.starts.get(.{c.batch}))); + { + const zero = b.cst(.f32, 0); + const sink = if (c.sink) a.sink.get(.{head}).mul(std.math.log2e) else b.cst(.f32, -std.math.inf(f32)); + var maximum = b.openFor(0, c.splits, 1, .{sink}); + { + maximum.yield(.{maximum.carried[0].maximum(a.partials.get(.{ maximum.iv, token, head, c.head_dim }))}); + } + const m = maximum.results[0].maximum(-std.math.floatMax(f32)); + var dims = b.openFor(b.threadIdx().x, c.head_dim, 128, .{}); + { + var sum = b.openFor(0, c.splits, 1, .{ zero, if (c.sink) b.exp2Approx(sink.sub(m)) else zero }); + { + const weight = b.exp2Approx(a.partials.get(.{ sum.iv, token, head, c.head_dim }).sub(m)); + sum.yield(.{ + sum.carried[0].add(weight.mul(a.partials.get(.{ sum.iv, token, head, dims.iv }))), + sum.carried[1].add(weight.mul(a.partials.get(.{ sum.iv, token, head, c.head_dim + 1 }))), + }); + } + a.out.set(.{ token, head, dims.iv }, sum.results[0].mul(b.select(sum.results[1].gt(0), b.rcpApprox(sum.results[1]), zero)).to(c.dtype)); + dims.yield(.{}); + } + valid.yieldThen(.{}); + } +} diff --git a/zml/kernels/paged_attention/cute/sm120/sm120.zig b/zml/kernels/paged_attention/cute/sm120/sm120.zig new file mode 100644 index 00000000..45699739 --- /dev/null +++ b/zml/kernels/paged_attention/cute/sm120/sm120.zig @@ -0,0 +1,83 @@ +const stdx = @import("stdx"); + +const zml = @import("../../../../zml.zig"); +const paged = @import("../../paged_attention.zig"); +const triton = @import("../../triton/triton.zig"); +const kernel = @import("attention.zig"); + +pub const Options = triton.paged.Options; +pub const Parameters = triton.paged.Parameters; + +test { + _ = kernel; +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + const cc = zml.platform.cuda.computeCapability(platform) orelse return false; + return cc.major == 12; +} + +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { + return zml.ops.manualComputation(Context.body, Context{ .q = q, .k = k, .v = v, .parameters = parameters, .opts = opts }, q.shape()); +} + +const Context = struct { + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + parameters: Parameters, + opts: paged.AttentionOptions, + + fn body(self: Context, output_shape: zml.Shape) zml.Tensor { + const q = self.q.transpose(.{ .b, .hkv, .hg, .hd }); + const k = self.k.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const v = self.v.transpose(.{ .page, .k_chunk, .hkv, .hd }); + stdx.debug.assert(q.dim(.hkv) == k.dim(.hkv) and k.shape().eql(v.shape()) and k.dim(.hd) == q.dim(.hd) and q.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); + const p = self.parameters; + const decode = !p.options_.is_prefill; + var c: kernel.Config = .{ + .head_dim = q.dim(.hd), + .tokens = q.dim(.b), + .batch = p.block_table.dim(.b), + .kv_heads = k.dim(.hkv), + .group = q.dim(.hg), + .pages_per_seq = p.block_table.dim(.p), + .page_size = k.dim(.k_chunk), + .max_query = if (decode) 1 else @min(q.dim(.b), @as(i64, @intCast(p.options_.max_seqlen_q))), + .dtype = zml.kernel.cute.from(q.dtype()), + .warps_per_query = if (decode) 4 else 2, + .kv_tile = if (q.dim(.hd) == 512) 32 else 64, + .causal = self.opts.mask.isCausal(), + .window = self.opts.mask.slidingWindow() orelse 0, + .scale = self.opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q.dim(.hd)))), + .sink = self.opts.sink != null, + }; + // Mixed batches also contain single-token rows. Partition their KV + // range so one long decode cannot hold up all the prefill CTAs. + const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); + const work = query_tiles * c.kv_heads; + const bound = if (c.window > 0) @min(c.window, c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; + c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(triton.getCuCount())) * 2 + work - 1, work), @divTrunc(bound + 127, 128)))); + const sink = if (self.opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.zeroes(.init(.{c.kv_heads * c.group}, .f32)); + const out = kernel.Kernel.call(.{ + .q = q, + .k = k, + .v = v, + .table = p.block_table, + .lengths = p.seq_lens, + .starts = p.query_start_len, + .sink = sink, + }, .{ .out = if (c.splits > 1) zml.Shape.init(.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 }, .f32) else q.shape() }, .{ + .cfg = c, + .grid = .{ @intCast(c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, + .block = .{ 128, 1, 1 }, + }).out; + const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .starts = p.query_start_len, .sink = sink }, .{ .out = q.shape() }, .{ + .cfg = c, + .grid = .{ @intCast(c.tokens), @intCast(c.kv_heads * c.group), 1 }, + .block = .{ 128, 1, 1 }, + }).out else out; + return result.transpose(output_shape); + } +}; From 93c0fd5ddb628c07266055b2d77209f171db117b Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 22 Oct 2026 10:48:09 +0200 Subject: [PATCH 813/881] zml/kernels/paged_attention/cute: add sm10x specific kernels (#974) This PR adds a `sm10x` specific implementation of paged attention in `cute`. It replaces the gemma 4 specific introduced in #969 --- zml/kernels/paged_attention/cute/BUILD.bazel | 2 +- .../paged_attention/cute/benchmark.zig | 2 +- zml/kernels/paged_attention/cute/cases.zig | 10 +- .../paged_attention/cute/correctness_test.zig | 6 +- zml/kernels/paged_attention/cute/cute.zig | 55 +- zml/kernels/paged_attention/cute/fixtures.zig | 9 +- .../paged_attention/cute/sm100/attention.zig | 716 ------------------ .../paged_attention/cute/sm100/sm100.zig | 114 --- .../cute/{sm100 => sm10x}/BUILD.bazel | 5 +- .../paged_attention/cute/sm10x/attention.zig | 673 ++++++++++++++++ .../paged_attention/cute/sm10x/sm103.zig | 44 ++ .../paged_attention/cute/sm10x/sm10x.zig | 165 ++++ .../paged_attention/cute/sm120/sm120.zig | 8 +- 13 files changed, 956 insertions(+), 853 deletions(-) delete mode 100644 zml/kernels/paged_attention/cute/sm100/attention.zig delete mode 100644 zml/kernels/paged_attention/cute/sm100/sm100.zig rename zml/kernels/paged_attention/cute/{sm100 => sm10x}/BUILD.bazel (68%) create mode 100644 zml/kernels/paged_attention/cute/sm10x/attention.zig create mode 100644 zml/kernels/paged_attention/cute/sm10x/sm103.zig create mode 100644 zml/kernels/paged_attention/cute/sm10x/sm10x.zig diff --git a/zml/kernels/paged_attention/cute/BUILD.bazel b/zml/kernels/paged_attention/cute/BUILD.bazel index b96272b2..6329ae12 100644 --- a/zml/kernels/paged_attention/cute/BUILD.bazel +++ b/zml/kernels/paged_attention/cute/BUILD.bazel @@ -7,7 +7,7 @@ filegroup( srcs = [ "attention.zig", "cute.zig", - "//zml/kernels/paged_attention/cute/sm100", + "//zml/kernels/paged_attention/cute/sm10x", "//zml/kernels/paged_attention/cute/sm120", ], ) diff --git a/zml/kernels/paged_attention/cute/benchmark.zig b/zml/kernels/paged_attention/cute/benchmark.zig index c4bf1165..9fd6a355 100644 --- a/zml/kernels/paged_attention/cute/benchmark.zig +++ b/zml/kernels/paged_attention/cute/benchmark.zig @@ -54,7 +54,7 @@ fn runBenchmark(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, inputs.sinks = inputs.sinks.withPartitioning(mesh, .{ ._0 = .model }); const compile_start: std.Io.Timestamp = .now(io, .awake); - const exe = try platform.compileFn(a, io, fixtures.forward, .{ inputs, global_case, zml.paged_attention.Backend.cute }, .{ + const exe = try platform.compileFn(a, io, fixtures.forward, .{ inputs, global_case }, .{ .program_name = "paged_attention_harness", .meshes = &.{mesh}, }); diff --git a/zml/kernels/paged_attention/cute/cases.zig b/zml/kernels/paged_attention/cute/cases.zig index 63aae6ec..e8dedeaa 100644 --- a/zml/kernels/paged_attention/cute/cases.zig +++ b/zml/kernels/paged_attention/cute/cases.zig @@ -45,7 +45,7 @@ pub const Case = struct { // Eight geometry cases plus orthogonal semantics/boundary cases, not a Cartesian product. pub const correctness = blk: { - var result: [26]Case = undefined; + var result: [34]Case = undefined; var i: usize = 0; for ([_]usize{ 64, 128, 256, 512 }) |dim| { for ([_]bool{ false, true }) |mixed| { @@ -71,6 +71,14 @@ pub const correctness = blk: { result[23] = .{ .name = "inactive_sequence", .batch = 4, .prefill = 1, .inactive = 1, .kv_len = 129 }; result[24] = .{ .name = "non_causal_window", .head_dim = 64, .prefill = 1, .kv_len = 65, .mask = .non_causal_window, .window = 7 }; result[25] = .{ .name = "empty_splits", .batch = 1, .kv_len = 1, .kv_capacity = 4096, .sink = true }; + result[26] = .{ .name = "d32_mixed", .head_dim = 32, .group = 3, .prefill = 1, .chunk = 9, .kv_len = 257, .shuffle_pages = true }; + result[27] = .{ .name = "d64_group_tail", .head_dim = 64, .group = 19, .kv_len = 257 }; + result[28] = .{ .name = "split_mixed_sink", .head_dim = 128, .batch = 1, .prefill = 1, .chunk = 3, .kv_len = 4097, .sink = true, .shuffle_pages = true }; + result[29] = .{ .name = "d512_prefill", .head_dim = 512, .batch = 2, .prefill = 2, .chunk = 65, .kv_len = 257, .sink = true, .shuffle_pages = true }; + result[30] = .{ .name = "negative_scale", .head_dim = 128, .prefill = 1, .kv_len = 257, .mask = .non_causal, .scale = -0.125 }; + result[31] = .{ .name = "zero_scale", .head_dim = 256, .prefill = 1, .kv_len = 257, .sink = true, .scale = 0 }; + result[32] = .{ .name = "d512_window_prefill", .head_dim = 512, .batch = 1, .prefill = 1, .chunk = 33, .kv_len = 513, .mask = .window, .window = 129, .sink = true }; + result[33] = .{ .name = "prefill_inactive", .head_dim = 64, .batch = 4, .prefill = 2, .inactive = 2, .kv_len = 257, .group = 5 }; break :blk result; }; diff --git a/zml/kernels/paged_attention/cute/correctness_test.zig b/zml/kernels/paged_attention/cute/correctness_test.zig index efa34c0c..436b9d3a 100644 --- a/zml/kernels/paged_attention/cute/correctness_test.zig +++ b/zml/kernels/paged_attention/cute/correctness_test.zig @@ -47,16 +47,16 @@ test "FP16 paged attention" { }) |c| try runCorrectness(std.testing.allocator, std.testing.io, platform, c, true); } -fn forwardF16(i: Inputs, c: cases.Case, backend: zml.paged_attention.Backend) zml.Tensor { +fn forwardF16(i: Inputs, c: cases.Case) zml.Tensor { var inputs = i; inputs.q = i.q.convert(.f16); inputs.k = i.k.convert(.f16); inputs.v = i.v.convert(.f16); - return fixtures.forward(inputs, c, backend).convert(.bf16); + return fixtures.forward(inputs, c).convert(.bf16); } fn runCorrectness(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, c: cases.Case, comptime compute_f16: bool) !void { - const exe = try platform.compileFn(a, io, if (compute_f16) forwardF16 else fixtures.forward, .{ Inputs.init(c), c, zml.paged_attention.Backend.cute }, .{ .program_name = "paged_attention_harness" }); + const exe = try platform.compileFn(a, io, if (compute_f16) forwardF16 else fixtures.forward, .{ Inputs.init(c), c }, .{ .program_name = "paged_attention_harness" }); defer exe.deinit(); var runner = try exe.runner(a); diff --git a/zml/kernels/paged_attention/cute/cute.zig b/zml/kernels/paged_attention/cute/cute.zig index 999f6c24..e572cce1 100644 --- a/zml/kernels/paged_attention/cute/cute.zig +++ b/zml/kernels/paged_attention/cute/cute.zig @@ -2,16 +2,59 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const paged = @import("../paged_attention.zig"); -const sm100 = @import("sm100/sm100.zig"); +const sm10x = @import("sm10x/sm10x.zig"); const sm120 = @import("sm120/sm120.zig"); const kernel = @import("attention.zig"); -pub const Options = sm100.Options; -pub const Parameters = sm100.Parameters; +pub const Options = struct { + batch_size: usize, + max_num_pages: usize, + max_seqlen_q: usize, + is_prefill: bool, + + pub fn isPrefill(self: Options) bool { + return self.is_prefill; + } + + pub fn maxNumPages(self: Options) usize { + return self.max_num_pages; + } +}; + +pub const Parameters = struct { + block_table: zml.Tensor, + seq_lens: zml.Tensor, + query_start_len: zml.Tensor, + options_: Options, + + pub fn init(options_: Options) Parameters { + return .{ + .block_table = .init(.{ .b = options_.batch_size, .p = options_.max_num_pages }, .i32), + .seq_lens = .init(.{ .b = options_.batch_size }, .i32), + .query_start_len = .init(.{ .b = options_.batch_size + 1 }, .i32), + .options_ = options_, + }; + } + + pub fn allocationSize(self: Parameters) usize { + return self.block_table.byteSize() + self.seq_lens.byteSize() + self.query_start_len.byteSize(); + } + + pub fn options(self: Parameters) Options { + return self.options_; + } +}; + +// TODO(Corentin): This should be moved in the platform probably +pub fn multiprocessorCount(platform: *const zml.Platform) usize { + const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return 1; + if (attribute.int64 <= 0) return 1; + return @intCast(attribute.int64); +} test { _ = kernel; - _ = sm100; + _ = sm10x; _ = sm120; } @@ -25,8 +68,8 @@ pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: z stdx.debug.assert(isAvailable(zml.Compiler.current().platform), "CuTe paged attention requires SM89 or newer", .{}); stdx.debug.assert(q.dtype() == k.dtype() and q.dtype() == v.dtype() and (q.dtype() == .bf16 or q.dtype() == .f16), "CuTe paged attention requires matching BF16 or FP16 Q/K/V", .{}); stdx.debug.assert(q.dim(.hd) == 32 or q.dim(.hd) == 64 or q.dim(.hd) == 128 or q.dim(.hd) == 256 or q.dim(.hd) == 512, "CuTe paged attention supports head dimensions 32, 64, 128, 256 and 512", .{}); - if (sm100.isAvailable(zml.Compiler.current().platform) and (q.dim(.hd) == 256 or q.dim(.hd) == 512) and opts.mask != .indices) - return sm100.pagedAttention(parameters, q, k, v, opts); + if (sm10x.isAvailable(zml.Compiler.current().platform) and opts.mask != .indices) + return sm10x.pagedAttention(parameters, q, k, v, opts); if (sm120.isAvailable(zml.Compiler.current().platform) and opts.mask != .indices) return sm120.pagedAttention(parameters, q, k, v, opts); return zml.ops.manualComputation(Context.body, Context{ .q = q, .k = k, .v = v, .parameters = parameters, .opts = opts }, q.shape()); diff --git a/zml/kernels/paged_attention/cute/fixtures.zig b/zml/kernels/paged_attention/cute/fixtures.zig index 210d7184..8c289b04 100644 --- a/zml/kernels/paged_attention/cute/fixtures.zig +++ b/zml/kernels/paged_attention/cute/fixtures.zig @@ -108,9 +108,9 @@ pub const Inputs = struct { } }; -pub fn forward(i: Inputs, c: cases.Case, backend: zml.paged_attention.Backend) zml.Tensor { - const parameters: zml.paged_attention.Parameters = switch (backend) { - inline .cute, .triton => |b| @unionInit(zml.paged_attention.Parameters, @tagName(b), .{ +pub fn forward(i: Inputs, c: cases.Case) zml.Tensor { + const parameters: zml.paged_attention.Parameters = .{ + .cute = .{ .block_table = i.table, .seq_lens = i.lengths, .query_start_len = i.starts, @@ -120,8 +120,7 @@ pub fn forward(i: Inputs, c: cases.Case, backend: zml.paged_attention.Backend) z .max_seqlen_q = if (c.prefill > 0) c.chunk else 1, .is_prefill = c.prefill > 0, }, - }), - else => unreachable, + }, }; return zml.paged_attention.pagedAttention(parameters, i.q, i.k, i.v, .{ .split = .{ .k = i.k, .v = i.v } }, i.options(c)); } diff --git a/zml/kernels/paged_attention/cute/sm100/attention.zig b/zml/kernels/paged_attention/cute/sm100/attention.zig deleted file mode 100644 index 0787108a..00000000 --- a/zml/kernels/paged_attention/cute/sm100/attention.zig +++ /dev/null @@ -1,716 +0,0 @@ -const std = @import("std"); - -const zml = @import("../../../../zml.zig"); -const cute = zml.kernel.cute; -const B = cute.Builder; -const V = cute.Value; - -pub const Config = struct { - head_dim: i64 = 512, - // Logical key tile; independent of physical cache page size. - kv_tile: i64 = 64, - load_buffers: i64 = 2, - tokens: i64, - batch: i64, - kv_heads: i64, - group: i64, - pages_per_seq: i64, - page_size: i64, - max_query: i64, - dtype: cute.DType, - all_decode: bool, - causal: bool, - window: i64, - scale: f32, - sink: bool, - sms: i64, - splits: i64, - // A positive bound partitions sliding decode by work per CTA, even when - // the batch already provides enough CTAs to fill the device. - tiles_per_split: i64 = 0, - // Fixed workers per sequence/head tile; zero launches each split separately. - split_stride: i64 = 0, - - pub fn headsInX(c: Config) bool { - return c.split_stride > 0 or (c.all_decode and c.kv_tile > 64); - } - pub fn rows(c: Config) i64 { - return if (c.all_decode) decodeRows(c.group) else 64; - } - pub fn queryTiles(c: Config) i64 { - return @divTrunc(c.max_query * c.group + c.rows() - 1, c.rows()); - } - pub fn outputTiles(c: Config) i64 { - return if (c.all_decode) 1 else @divExact(c.head_dim, 256); - } -}; - -// Use legal MMA widths, with masked tails for arbitrary GQA ratios. -pub fn decodeRows(group: i64) i64 { - return if (group <= 8) 8 else 16; -} - -pub const block: [3]i32 = .{ 256, 1, 1 }; -pub const Kernel = cute.Kernel(Config, .{ - .name = "paged_attention_cute", - .inputs = &.{ "q", "k", "v", "table", "lengths", "starts", "sink" }, - .outputs = &.{"out"}, - .run = run, -}); - -// KV buffers hold kv_tile keys by 128 features. The load ring streams D/128 -// K stages followed by D/128 V stages while Q stays resident. Prefill retains -// 64-key tiles and 256 output features per CTA. -fn kvBytes(c: Config) i64 { - return c.kv_tile * 128 * 2; -} -fn qBase(c: Config) i32 { - return @intCast(c.load_buffers * kvBytes(c)); -} -fn pBase(c: Config) i32 { - return qBase(c) + @as(i32, @intCast(c.rows())) * @as(i32, @intCast(c.head_dim)) * 2; -} -fn barBase(c: Config) i32 { - return pBase(c) + @as(i32, @intCast(c.rows() * c.kv_tile)) * 2; -} -fn holder(c: Config) i32 { - return barBase(c) + @as(i32, @intCast(c.load_buffers * 16)) + 16; -} -fn redBase(c: Config) i32 { - return holder(c) + 16; -} -fn sharedBytes(c: Config) i32 { - return redBase(c) + 4 * 16 * 4; -} -fn scoreCol(c: Config) i32 { - return if (c.all_decode) @intCast(@divExact(c.head_dim, 128) * c.rows()) else 256; -} -fn tmemCols(c: Config) u32 { - return if (c.all_decode) @intCast(std.math.ceilPowerOfTwoAssert(u32, @intCast((@divExact(c.head_dim, 128) + @divTrunc(c.kv_tile + 127, 128)) * c.rows()))) else 512; -} - -const Ctx = struct { - b: *B, - c: Config, - q: V, - k: V, - v: V, - table: V, - sink: V, - out: V, - s: V, - tmem: V, - tid: V, - warp: V, - lane: V, - seq: V, - head: V, - first_row: V, - start: V, - qlen: V, - klen: V, - first_tile: V, - tiles: V, - split: V, - output_half: V, -}; - -fn loadI32(b: *B, ptr: V, index: V) V { - return b.loadWords(1, ptr.add(index.to(.i64).mul(4)), .global, .{})[0]; -} - -// All-decode metadata gives the live request count without host readback. The -// fixed grid/scratch bound is independent of it and safe for graph replay. -fn liveSplits(b: *B, c: Config, active: V, tiles: V) V { - if (c.splits == 1) return b.cst(.i32, 1); - if (c.tiles_per_split > 0) - return tiles.add(c.tiles_per_split - 1).div(c.tiles_per_split).maximum(1).minimum(c.splits); - const work = active.maximum(1).mul(c.kv_heads * @divTrunc(c.group + c.rows() - 1, c.rows())); - const waves: i64 = if (c.kv_tile > 64 and c.window > 0) 4 else 2; - const budget = work.add(c.sms * waves - 1).div(work).maximum(1).minimum(c.splits); - return budget.minimum(tiles.add(1).div(2).maximum(1)); -} - -fn run(b: *B, c: Config) cute.FinishError!void { - std.debug.assert(c.kv_tile == 64 or c.kv_tile == 128 or c.kv_tile == 256); - std.debug.assert(c.all_decode or c.kv_tile == 64); - std.debug.assert(c.load_buffers == 2 or c.load_buffers == 3); - // Persistent independent partitions restart stage numbering each time. - std.debug.assert(c.split_stride == 0 or c.load_buffers == 2); - const a = try b.declareArgs(.{ - .q = .{ .ptr = c.dtype }, - .k = .{ .ptr = c.dtype }, - .v = .{ .ptr = c.dtype }, - .table = .{ .ptr = cute.DType.i32 }, - .lengths = .{ .ptr = cute.DType.i32 }, - .starts = .{ .ptr = cute.DType.i32 }, - .sink = .{ .ptr = cute.DType.f32 }, - .out = .{ .ptr = if (c.splits > 1) cute.DType.f32 else c.dtype }, - }); - b.setFunctionAttribute("cu_attrs", b.parseAttribute("{max_dynamic_shared_size_bytes = #cuda.dev_max_shared_memory_optin}")); - const starts = b.ptrToInt(a.starts, .i64); - const seq = b.blockIdx().x.div(c.queryTiles() * (if (c.headsInX()) c.kv_heads else 1)); - const first_row = b.blockIdx().x.rem(c.queryTiles()).mul(c.rows()); - const start = loadI32(b, starts, seq); - const qlen = loadI32(b, starts, seq.add(1)).sub(start); - const klen = loadI32(b, b.ptrToInt(a.lengths, .i64), seq); - const first_pos = klen.sub(qlen).add(first_row.div(c.group)); - const last_pos = klen.sub(qlen).add(first_row.add(c.rows() - 1).div(c.group).minimum(qlen.sub(1))); - const begin = if (c.window > 0) first_pos.sub(c.window - 1).maximum(0) else b.cst(.i32, 0); - const end = if (c.causal) last_pos.add(1) else klen; - const first_tile = begin.div(c.kv_tile); - const total_tiles = end.add(c.kv_tile - 1).div(c.kv_tile).sub(first_tile).maximum(0); - const splits = liveSplits(b, c, loadI32(b, starts, b.cst(.i32, c.batch)), total_tiles); - const split = b.blockIdx().z; - var active = b.openIf(first_row.lt(qlen.mul(c.group)).bitAnd(split.lt(splits))); - { - const lo = if (c.tiles_per_split > 0) split.mul(c.tiles_per_split) else total_tiles.mul(split).div(splits); - const hi = if (c.tiles_per_split > 0) lo.add(c.tiles_per_split).minimum(total_tiles) else total_tiles.mul(split.add(1)).div(splits); - const tid = b.threadIdx().x; - const s = b.ptrToInt(b.allocSmem(.i8, @intCast(sharedBytes(c)), 1024), .i32); - var ctx: Ctx = .{ - .b = b, - .c = c, - .q = b.ptrToInt(a.q, .i64), - .k = b.ptrToInt(a.k, .i64), - .v = b.ptrToInt(a.v, .i64), - .table = b.ptrToInt(a.table, .i64), - .sink = b.ptrToInt(a.sink, .i64), - .out = b.ptrToInt(a.out, .i64), - .s = s, - .tmem = undefined, - .tid = tid, - .warp = tid.div(32), - .lane = tid.rem(32), - .seq = seq, - .head = if (c.headsInX()) b.blockIdx().x.div(c.queryTiles()).rem(c.kv_heads) else b.blockIdx().y.div(c.outputTiles()), - .output_half = b.blockIdx().y.rem(c.outputTiles()), - .first_row = first_row, - .start = start, - .qlen = qlen, - .klen = klen, - .first_tile = first_tile.add(lo), - .tiles = if (c.tiles_per_split == 1) b.cst(.i32, 1) else hi.sub(lo), - .split = split, - }; - var init = b.openIf(tid.eq(0)); - { - for (0..@as(usize, @intCast(c.load_buffers))) |i| { - b.mbarrierInit(s.add(barBase(c) + @as(i32, @intCast(i * 8))), 128); - b.mbarrierInit(s.add(barBase(c) + @as(i32, @intCast(c.load_buffers * 8)) + @as(i32, @intCast(i * 8))), 1); - } - b.mbarrierInit(s.add(barBase(c) + @as(i32, @intCast(c.load_buffers * 16))), 1); - b.fenceMbarrierInit(); - init.yieldThen(.{}); - } - var alloc = b.openIf(tid.lt(32)); - { - b.tmemAllocAt(s.add(holder(c)), tmemCols(c)); - b.relinquishTmemAllocPermit(.{}); - alloc.yieldThen(.{}); - } - // Queries are packed_row by token, KV head, and group head. Mask the final - // query tile without ever forming an out-of-range source load. - var copy = b.openFor(tid, c.rows() * @divExact(c.head_dim, 8), 256, .{}); - { - const row = copy.iv.div(@divExact(c.head_dim, 8)); - const col = copy.iv.rem(@divExact(c.head_dim, 8)).mul(8); - const packed_row = first_row.add(row); - const valid = packed_row.lt(qlen.mul(c.group)); - const token = start.add(b.select(valid, packed_row.div(c.group), b.cst(.i32, 0))); - const qi = token.to(.i64).mul(c.kv_heads).add(ctx.head.to(.i64)).mul(c.group).add(packed_row.rem(c.group).to(.i64)).mul(c.head_dim).add(col.to(.i64)); - b.cpAsync16(s.add(qBase(c)).add(swizzle(row, col, c.rows())), ctx.q.add(qi.mul(2)), valid); - copy.yield(.{}); - } - b.cpAsyncWaitAll(); - b.syncThreads(); - ctx.tmem = b.loadWords(1, s.add(holder(c)), .shared, .{})[0]; - b.tcgen05Fence(.after); - if (c.split_stride > 0) { - // A fixed grid reuses Q, shared memory and TMEM across independent - // split partitions. Runtime lengths bound this device loop only. - // D256 has four load stages: both two-stage rings and the QK/PV - // completion barrier return to parity zero after every partition. - var partitions = b.openFor(split, splits, c.split_stride, .{}); - { - ctx.first_tile = first_tile.add(partitions.iv); - ctx.split = partitions.iv; - var producer = b.openIf(tid.ge(128)); - { - loadStages(ctx); - producer.yieldThen(.{}); - } - var consumer = b.openIf(tid.lt(128)); - { - decode(ctx); - consumer.yieldThen(.{}); - } - b.tcgen05Fence(.before); - b.syncThreads(); - b.tcgen05Fence(.after); - partitions.yield(.{}); - } - } else { - var producer = b.openIf(tid.ge(128)); - { - loadStages(ctx); - producer.yieldThen(.{}); - } - var consumer = b.openIf(tid.lt(128)); - { - if (c.all_decode) decode(ctx) else prefill(ctx); - consumer.yieldThen(.{}); - } - b.tcgen05Fence(.before); - b.syncThreads(); - } - var free = b.openIf(tid.lt(32)); - { - b.tmemDeallocAt(ctx.tmem, tmemCols(c)); - free.yieldThen(.{}); - } - active.yieldThen(.{}); - } -} - -fn ready(c: Ctx, id: V) V { - return c.s.add(barBase(c.c)).add(id.rem(c.c.load_buffers).mul(8)); -} -fn released(c: Ctx, id: V) V { - return ready(c, id).add(c.c.load_buffers * 8); -} -fn phase(c: Ctx, id: V) V { - return id.div(c.c.load_buffers).rem(2); -} - -fn loadStages(c: Ctx) void { - if (c.c.all_decode) return loadDecodeStages(c); - const b = c.b; - // Prefill computes half the output features per CTA; decode computes all. - const k_stages = @divExact(c.c.head_dim, 128); - const stages = k_stages + if (c.c.all_decode) k_stages else 2; - var work = b.openFor(0, c.tiles.mul(stages), 1, .{}); - { - const id = work.iv; - const part = id.rem(stages); - var reuse = b.openIf(id.ge(c.c.load_buffers)); - { - b.mbarrierWait(released(c, id), phase(c, id).bitXor(1)); - reuse.yieldThen(.{}); - } - const is_k = part.lt(k_stages); - const d0 = b.select(is_k, part.mul(128), part.sub(k_stages).mul(128).add(c.output_half.mul(256))); - const source = b.select(is_k, c.k, c.v); - const key0 = c.first_tile.add(id.div(stages)).mul(64); - var vec = b.openFor(c.tid.sub(128), 64 * 16, 128, .{}); - { - const row = vec.iv.div(16); - const col = vec.iv.rem(16).mul(8); - const key = key0.add(row); - const valid = key.lt(c.klen); - // Do not read unused page table entries (which may contain -1). - const safe_key = b.select(valid, key, b.cst(.i32, 0)); - const page = loadI32(b, c.table, c.seq.mul(c.c.pages_per_seq).add(safe_key.div(c.c.page_size))); - const offset = page.to(.i64).mul(c.c.page_size).add(safe_key.rem(c.c.page_size).to(.i64)).mul(c.c.kv_heads).add(c.head.to(.i64)).mul(c.c.head_dim).add(d0.add(col).to(.i64)); - b.cpAsync16(c.s.add(id.rem(c.c.load_buffers).mul(kvBytes(c.c))).add(swizzle(row, col, 64)), source.add(offset.mul(2)), valid); - vec.yield(.{}); - } - b.cpAsyncMbarrierArrive(ready(c, id), true); - work.yield(.{}); - } -} - -// Reuse the paged addresses across all K/V feature stages of a decode tile. -fn loadDecodeStages(c: Ctx) void { - const b = c.b; - const k_stages: usize = @intCast(@divExact(c.c.head_dim, 128)); - const stages = 2 * k_stages; - const tid = c.tid.sub(128); - const row0 = tid.div(16); - const col = tid.rem(16).mul(8); - var tiles = b.openFor(0, c.tiles, 1, .{}); - { - const key0 = c.first_tile.add(tiles.iv).mul(c.c.kv_tile).add(row0); - var offsets: [32]V = undefined; - var masks: [32]V = undefined; - for (0..@as(usize, @intCast(@divExact(c.c.kv_tile, 8)))) |i| { - const key = key0.add(@as(i32, @intCast(i * 8))); - masks[i] = key.lt(c.klen); - const safe_key = b.select(masks[i], key, b.cst(.i32, 0)); - const page = loadI32(b, c.table, c.seq.mul(c.c.pages_per_seq).add(safe_key.div(c.c.page_size))); - offsets[i] = page.to(.i64).mul(c.c.page_size).add(safe_key.rem(c.c.page_size).to(.i64)).mul(c.c.kv_heads).add(c.head.to(.i64)).mul(c.c.head_dim).add(col.to(.i64)).mul(2); - } - for (0..stages) |part| { - const id = tiles.iv.mul(stages).add(@as(i32, @intCast(part))); - var reuse = b.openIf(id.ge(c.c.load_buffers)); - { - b.mbarrierWait(released(c, id), phase(c, id).bitXor(1)); - reuse.yieldThen(.{}); - } - const source = if (part < k_stages) c.k else c.v; - const d0: i32 = @intCast((part % k_stages) * 128 * 2); - for (0..@as(usize, @intCast(@divExact(c.c.kv_tile, 8)))) |i| { - const row = row0.add(@as(i32, @intCast(i * 8))); - b.cpAsync16(c.s.add(id.rem(c.c.load_buffers).mul(kvBytes(c.c))).add(swizzle(row, col, c.c.kv_tile)), source.add(offsets[i]).add(d0), masks[i]); - } - b.cpAsyncMbarrierArrive(ready(c, id), true); - } - tiles.yield(.{}); - } -} - -fn mma(c: Ctx, tile: V, comptime qk: bool) void { - const b = c.b; - const k_stages = @divExact(c.c.head_dim, 128); - const stages = k_stages + if (c.c.all_decode) k_stages else 2; - const count: usize = if (qk or c.c.all_decode) @intCast(k_stages) else 2; - var leader = b.openIf(c.tid.eq(0)); - { - for (0..count) |part| { - const id = tile.mul(stages).add(@as(i32, @intCast(part + if (qk) @as(usize, 0) else @as(usize, @intCast(k_stages))))); - b.mbarrierWait(ready(c, id), phase(c, id)); - b.tcgen05Fence(.after); - const kv = c.s.add(id.rem(c.c.load_buffers).mul(kvBytes(c.c))); - const kd = descriptor(b, kv, if (qk) 16 else @intCast(c.c.kv_tile * 128)); - const qd = descriptor(b, c.s.add(if (qk) qBase(c.c) else pBase(c.c)), 16); - const score_slabs: usize = if (qk and c.c.all_decode) @intCast(@divTrunc(c.c.kv_tile + 127, 128)) else 1; - const steps: usize = if (qk) 8 else @intCast(@divExact(c.c.kv_tile, 16)); - for (0..score_slabs) |slab| { - for (0..steps) |step| { - const step_off: i64 = @as(i64, @intCast(step / 4)) * c.c.kv_tile * 128 + @as(i64, @intCast(step % 4)) * 32; - const kvd = kd.add(if (qk) @divExact(step_off + @as(i64, @intCast(slab)) * 128 * 128, 16) else @as(i64, @intCast(step * 2048 / 16))); - const other_off: i64 = if (qk) @as(i64, @intCast(part * 2 + step / 4)) * c.c.rows() * 128 + @as(i64, @intCast(step % 4)) * 32 else @as(i64, @intCast(step / 4)) * c.c.rows() * 128 + @as(i64, @intCast(step % 4)) * 32; - const other = qd.add(@divExact(other_off, 16)); - const swapped = c.c.all_decode; - const ad = if (swapped) kvd else other; - const bd = if (swapped) other else kvd; - const m: u32 = if (swapped) (if (qk) @intCast(@min(128, c.c.kv_tile)) else 128) else 64; - const n: u32 = if (swapped) @intCast(c.c.rows()) else if (qk) 64 else 128; - const instruction = instrDesc(c.c.dtype, m, n, swapped and !qk, !swapped and !qk); - const dst = c.tmem.add(if (qk) scoreCol(c.c) + @as(i32, @intCast(slab)) * @as(i32, @intCast(c.c.rows())) else @as(i32, @intCast(part * (if (swapped) @as(usize, @intCast(c.c.rows())) else 128)))); - const acc = if (qk) b.cst(.i32, @intFromBool(part > 0 or step > 0)).ne(0) else tile.gt(0).bitOr(b.cst(.i32, @intFromBool(step > 0)).ne(0)); - b.tcgen05MmaF16(dst, ad, bd, b.cst(.i32, @as(i32, @bitCast(instruction))), acc); - } - } - b.tcgen05Commit(released(c, id)); - } - b.tcgen05Commit(c.s.add(barBase(c.c) + @as(i32, @intCast(c.c.load_buffers * 16)))); - leader.yieldThen(.{}); - } - // Two completions per tile: QK parity 0 and PV parity 1. - b.mbarrierWait(c.s.add(barBase(c.c) + @as(i32, @intCast(c.c.load_buffers * 16))), b.cst(.i32, if (qk) 0 else 1)); - b.tcgen05Fence(.after); - // All consumers must observe PV completion before the issuing warp can - // complete the next QK and wrap this shared barrier's parity again. - // Single-tile splits rendezvous at the outer full-CTA barrier instead. - if (!qk and c.c.tiles_per_split != 1) b.barrierSync(1, 128); -} - -fn validScore(c: Ctx, row: V, key: V) V { - const pos = c.klen.sub(c.qlen).add(row.div(c.c.group)); - var valid = row.lt(c.qlen.mul(c.c.group)).bitAnd(key.lt(c.klen)); - if (c.c.causal) valid = valid.bitAnd(key.le(pos)); - if (c.c.window > 0) valid = valid.bitAnd(key.gt(pos.sub(c.c.window))); - return valid; -} - -fn storeP(c: Ctx, row: V, col: V, value: V) void { - c.b.storeHalf(c.s.add(pBase(c.c)).add(swizzle(row, col, c.c.rows())), .shared, value.to(c.c.dtype).bitCast(.i16).to(.i32)); -} - -fn consumerSync(c: Ctx) void { - c.b.fenceProxyAsyncShared(); - c.b.tcgen05Fence(.before); - c.b.barrierSync(1, 128); - c.b.tcgen05Fence(.after); -} - -fn prefill(c: Ctx) void { - const b = c.b; - const row = c.first_row.add(c.warp.mul(16)).add(c.lane); - const owns = c.lane.lt(16); - const lane_tmem = c.tmem.add(c.warp.mul(32).shl(16)); - const zero = b.cst(.f32, 0); - const neg_inf = b.cst(.f32, -std.math.inf(f32)); - var loop = b.openFor(0, c.tiles, 1, .{ neg_inf, zero }); - { - mma(c, loop.iv, true); - const raw = b.tmemLoad32x32b(64, lane_tmem.add(scoreCol(c.c))); - b.fenceTmemLoad(); - var scores: [64]V = undefined; - var maximum = loop.carried[0]; - for (0..64) |j| { - const key = c.first_tile.add(loop.iv).mul(64).add(@as(i32, @intCast(j))); - scores[j] = b.select(validScore(c, row, key).bitAnd(owns), raw[j].bitCast(.f32).mul(c.c.scale * std.math.log2e), neg_inf); - maximum = maximum.maximum(scores[j]); - } - const safe_max = b.select(maximum.eq(neg_inf), zero, maximum); - const alpha = b.exp2Approx(loop.carried[0].sub(safe_max)); - var sum = loop.carried[1].mul(alpha); - var probabilities: [64]V = undefined; - for (0..64) |j| { - probabilities[j] = b.exp2Approx(scores[j].sub(safe_max)); - sum = sum.add(probabilities[j]); - } - var write = b.openIf(owns); - { - for (0..64) |j| storeP(c, c.warp.mul(16).add(c.lane), b.cst(.i32, @as(i32, @intCast(j))), probabilities[j]); - write.yieldThen(.{}); - } - var rescale = b.openIf(loop.iv.gt(0)); - { - for (0..8) |chunk| { - const addr = lane_tmem.add(@as(i32, @intCast(chunk * 32))); - var out = b.tmemLoad32x32b(32, addr); - b.fenceTmemLoad(); - for (&out) |*v| v.* = v.bitCast(.f32).mul(alpha).bitCast(.i32); - b.tmemStore32x32b(32, addr, out); - } - b.fenceTmemStore(); - rescale.yieldThen(.{}); - } - consumerSync(c); - mma(c, loop.iv, false); - loop.yield(.{ maximum, sum }); - } - const norm = normalization(c, row, loop.results[0], loop.results[1]); - for (0..8) |chunk| { - const out = b.tmemLoad32x32b(32, lane_tmem.add(@as(i32, @intCast(chunk * 32)))); - b.fenceTmemLoad(); - var valid = b.openIf(owns.bitAnd(row.lt(c.qlen.mul(c.c.group)))); - { - for (0..32) |j| { - const dim = c.output_half.mul(256).add(@as(i32, @intCast(chunk * 32 + j))); - storeOutput(c, row, dim, out[j].bitCast(.f32).mul(norm)); - } - valid.yieldThen(.{}); - } - } -} - -fn tuple(comptime n: usize, a: [n]V) @Tuple(&@as([n]type, @splat(V))) { - var t: @Tuple(&@as([n]type, @splat(V))) = undefined; - inline for (0..n) |i| t[i] = a[i]; - return t; -} - -fn reduceHeads(c: Ctx, comptime heads: usize, values: [heads]V, comptime max: bool) [heads]V { - const b = c.b; - for (values, 0..) |v, h| { - var r = v; - inline for (.{ 16, 8, 4, 2, 1 }) |step| r = if (max) r.maximum(b.shuffleXor(r, step)) else r.add(b.shuffleXor(r, step)); - var leader = b.openIf(c.lane.eq(0)); - { - b.storeWords(1, c.s.add(redBase(c.c)).add(c.warp.mul(64)).add(@as(i32, @intCast(h * 4))), .shared, .{r.bitCast(.i32)}); - leader.yieldThen(.{}); - } - } - b.barrierSync(1, 128); - var result: [heads]V = undefined; - for (0..heads) |h| { - var value = b.loadWords(1, c.s.add(redBase(c.c) + @as(i32, @intCast(h * 4))), .shared, .{})[0].bitCast(.f32); - for (1..4) |w| { - const next = b.loadWords(1, c.s.add(redBase(c.c) + @as(i32, @intCast(w * 64 + h * 4))), .shared, .{})[0].bitCast(.f32); - value = if (max) value.maximum(next) else value.add(next); - } - result[h] = value; - } - b.barrierSync(1, 128); - return result; -} - -fn decode(c: Ctx) void { - if (c.c.kv_tile == 256) decodeTile(c, 2) else decodeTile(c, 1); -} - -fn decodeTile(c: Ctx, comptime slabs: usize) void { - // MMA still uses its legal 8/16-row tile. Only process the statically - // occupied head columns in the scalar softmax and output paths. - if (c.c.group <= 2) decodeSized(c, 2, slabs) else if (c.c.group <= 4) decodeSized(c, 4, slabs) else if (c.c.rows() == 8) decodeSized(c, 8, slabs) else decodeSized(c, 16, slabs); -} - -fn decodeSized(c: Ctx, comptime heads: usize, comptime slabs: usize) void { - const b = c.b; - const lane_tmem = c.tmem.add(c.warp.mul(32).shl(16)); - const key_row = c.warp.mul(@as(i32, if (c.c.kv_tile == 64) 16 else 32)).add(c.lane); - const owns = if (c.c.kv_tile == 64) c.lane.lt(16) else b.cst(.i32, 1).ne(0); - const zero = b.cst(.f32, 0); - const neg_inf = b.cst(.f32, -std.math.inf(f32)); - var initial: [2 * heads]V = undefined; - for (0..heads) |h| { - initial[h] = neg_inf; - initial[h + heads] = zero; - } - var loop = b.openFor(0, c.tiles, 1, tuple(2 * heads, initial)); - { - mma(c, loop.iv, true); - var scores: [slabs][heads]V = undefined; - var local_max: [heads]V = @splat(neg_inf); - for (0..slabs) |slab| { - const raw = b.tmemLoad32x32b(heads, lane_tmem.add(scoreCol(c.c) + @as(i32, @intCast(slab)) * @as(i32, @intCast(c.c.rows())))); - b.fenceTmemLoad(); - const key = c.first_tile.add(loop.iv).mul(c.c.kv_tile).add(key_row).add(@as(i32, @intCast(slab * 128))); - for (0..heads) |h| { - scores[slab][h] = b.select(owns.bitAnd(validScore(c, c.first_row.add(@as(i32, @intCast(h))), key)), raw[h].bitCast(.f32).mul(c.c.scale * std.math.log2e), neg_inf); - local_max[h] = local_max[h].maximum(scores[slab][h]); - } - } - const tile_max = reduceHeads(c, heads, local_max, true); - var next: [2 * heads]V = undefined; - var alpha: [heads]V = undefined; - var probabilities: [slabs][heads]V = undefined; - var local_sum: [heads]V = @splat(zero); - for (0..heads) |h| { - next[h] = loop.carried[h].maximum(tile_max[h]); - const safe_max = b.select(next[h].eq(neg_inf), zero, next[h]); - alpha[h] = b.exp2Approx(loop.carried[h].sub(safe_max)); - for (0..slabs) |slab| { - probabilities[slab][h] = b.exp2Approx(scores[slab][h].sub(safe_max)); - local_sum[h] = local_sum[h].add(probabilities[slab][h]); - } - } - const sums = reduceHeads(c, heads, local_sum, false); - for (0..heads) |h| next[h + heads] = loop.carried[h + heads].mul(alpha[h]).add(sums[h]); - var write = b.openIf(owns); - { - for (0..slabs) |slab| { - const key_col = key_row.add(@as(i32, @intCast(slab * 128))); - for (0..heads) |h| storeP(c, b.cst(.i32, @as(i32, @intCast(h))), key_col, probabilities[slab][h]); - for (heads..@as(usize, @intCast(c.c.rows()))) |h| storeP(c, b.cst(.i32, @as(i32, @intCast(h))), key_col, zero); - } - write.yieldThen(.{}); - } - if (c.c.tiles_per_split != 1) { - var rescale = b.openIf(loop.iv.gt(0)); - { - for (0..@as(usize, @intCast(@divExact(c.c.head_dim, 128)))) |chunk| { - const addr = lane_tmem.add(@as(i32, @intCast(chunk * @as(usize, @intCast(c.c.rows()))))); - var out = b.tmemLoad32x32b(heads, addr); - b.fenceTmemLoad(); - for (0..heads) |h| out[h] = out[h].bitCast(.f32).mul(alpha[h]).bitCast(.i32); - b.tmemStore32x32b(heads, addr, out); - } - b.fenceTmemStore(); - rescale.yieldThen(.{}); - } - } - consumerSync(c); - mma(c, loop.iv, false); - loop.yield(tuple(2 * heads, next)); - } - var norms: [heads]V = undefined; - for (0..heads) |h| norms[h] = if (c.c.splits > 1) b.cst(.f32, 1) else normalization(c, c.first_row.add(@as(i32, @intCast(h))), loop.results[h], loop.results[h + heads]); - for (0..@as(usize, @intCast(@divExact(c.c.head_dim, 128)))) |chunk| { - const out = b.tmemLoad32x32b(heads, lane_tmem.add(@as(i32, @intCast(chunk * @as(usize, @intCast(c.c.rows())))))); - b.fenceTmemLoad(); - const dim = c.warp.mul(32).add(c.lane).add(@as(i32, @intCast(chunk * 128))); - for (0..heads) |h| { - const row = c.first_row.add(@as(i32, @intCast(h))); - var valid = b.openIf(row.lt(c.qlen.mul(c.c.group))); - { - storeOutput(c, row, dim, out[h].bitCast(.f32).mul(norms[h])); - if (c.c.splits > 1 and chunk == 0) { - var stats = b.openIf(c.tid.eq(0)); - { - storeOutput(c, row, b.cst(.i32, c.c.head_dim), loop.results[h]); - storeOutput(c, row, b.cst(.i32, c.c.head_dim + 1), loop.results[h + heads]); - stats.yieldThen(.{}); - } - } - valid.yieldThen(.{}); - } - } - } -} - -fn normalization(c: Ctx, row: V, m: V, l: V) V { - const b = c.b; - if (!c.c.sink) return b.select(l.gt(0), b.rcpApprox(l), b.cst(.f32, 0)); - const h = c.head.mul(c.c.group).add(row.rem(c.c.group)); - const sink = loadI32(b, c.sink, h).bitCast(.f32).mul(std.math.log2e); - const maximum = m.maximum(sink); - const weight = b.exp2Approx(m.sub(maximum)); - return weight.div(l.mul(weight).add(b.exp2Approx(sink.sub(maximum)))); -} - -fn storeOutput(c: Ctx, row: V, dim: V, value: V) void { - const token = c.start.add(row.div(c.c.group)); - var offset = token.to(.i64).mul(c.c.kv_heads).add(c.head.to(.i64)).mul(c.c.group).add(row.rem(c.c.group).to(.i64)); - if (c.c.splits > 1) { - offset = c.split.to(.i64).mul(c.c.tokens * c.c.kv_heads * c.c.group).add(offset).mul(c.c.head_dim + 2).add(dim.to(.i64)); - c.b.storeWords(1, c.out.add(offset.mul(4)), .global, .{value.bitCast(.i32)}); - } else { - offset = offset.mul(c.c.head_dim).add(dim.to(.i64)); - c.b.storeHalf(c.out.add(offset.mul(2)), .global, value.to(c.c.dtype).bitCast(.i16).to(.i32)); - } -} - -fn descriptor(b: *B, address: V, lbo: u64) V { - const flags: u64 = (64 << 32) | (1 << 46) | (@as(u64, 2) << 61) | ((lbo / 16) << 16); - return address.to(.i64).shrLogical(4).bitAnd(0x3fff).bitOr(b.cst(.i64, @as(i64, @bitCast(flags)))); -} -fn swizzle(row: V, col: V, rows: i64) V { - return col.div(64).mul(rows * 128).add(row.mul(128)).add(col.rem(64).div(8).bitXor(row.rem(8)).mul(16)).add(col.rem(8).mul(2)); -} -fn instrDesc(dtype: cute.DType, m: u32, n: u32, a_mn: bool, b_mn: bool) u32 { - const format: u32 = if (dtype == .bf16) 1 else 0; - return (1 << 4) | (format << 7) | (format << 10) | (@as(u32, @intFromBool(a_mn)) << 15) | (@as(u32, @intFromBool(b_mn)) << 16) | ((n >> 3) << 17) | ((m >> 4) << 24); -} - -pub const Merge = cute.Kernel(Config, .{ - .name = "paged_attention_cute_merge", - .inputs = &.{ "partials", "lengths", "starts", "sink" }, - .outputs = &.{"out"}, - .run = merge, -}); - -fn merge(b: *B, c: Config) cute.FinishError!void { - const a = try b.declareArgs(.{ - .partials = .{ .tensor = .{ .dtype = .f32, .shape = &.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 } } }, - .lengths = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch} } }, - .starts = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch + 1} } }, - .sink = .{ .tensor = .{ .dtype = .f32, .shape = &.{c.kv_heads * c.group} } }, - .out = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.tokens, c.kv_heads * c.group, c.head_dim } } }, - }); - const seq = b.blockIdx().y; - const token = a.starts.get(.{seq}); - var valid = b.openIf(token.lt(a.starts.get(.{seq.add(1)}))); - { - const head = b.blockIdx().x; - const len = a.lengths.get(.{seq}); - const begin = if (c.window > 0) len.sub(c.window).maximum(0) else b.cst(.i32, 0); - const tiles = len.add(c.kv_tile - 1).div(c.kv_tile).sub(begin.div(c.kv_tile)); - const splits = liveSplits(b, c, a.starts.get(.{c.batch}), tiles); - const neg_inf = b.cst(.f32, -std.math.inf(f32)); - var maximum = b.openFor(0, splits, 1, .{if (c.sink) a.sink.get(.{head}).mul(std.math.log2e) else neg_inf}); - { - maximum.yield(.{maximum.carried[0].maximum(a.partials.get(.{ maximum.iv, token, head, c.head_dim }))}); - } - const m = maximum.results[0]; - var dims = b.openFor(b.threadIdx().x, c.head_dim, b.blockDim().x, .{}); - { - const zero = b.cst(.f32, 0); - var sum = b.openFor(0, splits, 1, .{ zero, if (c.sink) b.exp2Approx(a.sink.get(.{head}).mul(std.math.log2e).sub(m)) else zero }); - { - const weight = b.exp2Approx(a.partials.get(.{ sum.iv, token, head, c.head_dim }).sub(m)); - sum.yield(.{ sum.carried[0].add(weight.mul(a.partials.get(.{ sum.iv, token, head, dims.iv }))), sum.carried[1].add(weight.mul(a.partials.get(.{ sum.iv, token, head, c.head_dim + 1 }))) }); - } - a.out.set(.{ token, head, dims.iv }, sum.results[0].div(sum.results[1]).to(c.dtype)); - dims.yield(.{}); - } - valid.yieldThen(.{}); - } -} - -test "CuTe paged attention emits prefill and decode tensor cores" { - for ([_]i64{ 256, 512 }) |head_dim| for ([_]bool{ false, true }) |all_decode| { - const cfg: Config = .{ .head_dim = head_dim, .tokens = 80, .batch = 4, .kv_heads = 2, .group = 6, .pages_per_seq = 128, .page_size = 16, .max_query = if (all_decode) 1 else 65, .dtype = .bf16, .all_decode = all_decode, .causal = true, .window = 0, .scale = 0.044194174, .sink = true, .sms = 148, .splits = if (all_decode) 64 else 1 }; - const ir = try Kernel.emit(std.testing.allocator, cfg, block); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "tcgen05.mma") != null); - if (all_decode) { - const merge_ir = try Merge.emit(std.testing.allocator, cfg, .{ 128, 1, 1 }); - defer std.testing.allocator.free(merge_ir); - } - }; -} diff --git a/zml/kernels/paged_attention/cute/sm100/sm100.zig b/zml/kernels/paged_attention/cute/sm100/sm100.zig deleted file mode 100644 index 9c0f5ade..00000000 --- a/zml/kernels/paged_attention/cute/sm100/sm100.zig +++ /dev/null @@ -1,114 +0,0 @@ -const stdx = @import("stdx"); - -const zml = @import("../../../../zml.zig"); -const paged = @import("../../paged_attention.zig"); -const triton = @import("../../triton/triton.zig"); -const kernel = @import("attention.zig"); - -pub const Options = triton.paged.Options; -pub const Parameters = triton.paged.Parameters; - -test { - _ = kernel; -} - -pub fn isAvailable(platform: *const zml.Platform) bool { - if (platform.target != .cuda) return false; - const cc = zml.platform.cuda.computeCapability(platform) orelse return false; - return cc.major == 10; -} - -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { - stdx.debug.assert(q.dtype() == k.dtype() and q.dtype() == v.dtype(), "CuTe paged attention requires matching Q/K/V types", .{}); - return zml.ops.manualComputation(Context.body, Context{ .q = q, .k = k, .v = v, .parameters = parameters, .opts = opts }, q.shape()); -} - -const Context = struct { - q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, - parameters: Parameters, - opts: paged.AttentionOptions, - - fn body(self: Context, output_shape: zml.Shape) zml.Tensor { - const q = self.q.transpose(.{ .b, .hkv, .hg, .hd }); - const k = self.k.transpose(.{ .page, .k_chunk, .hkv, .hd }); - const v = self.v.transpose(.{ .page, .k_chunk, .hkv, .hd }); - stdx.debug.assert(q.dim(.hkv) == k.dim(.hkv) and k.shape().eql(v.shape()) and k.dim(.hd) == q.dim(.hd) and q.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); - const p = self.parameters; - const all_decode = !p.options_.is_prefill; - const sms: i64 = @intCast(triton.getCuCount()); - const rows = kernel.decodeRows(q.dim(.hg)); - const head_tiles = @divTrunc(q.dim(.hg) + rows - 1, rows); - const max_splits = if (all_decode) @min(128, @divTrunc(sms * 2 + k.dim(.hkv) * head_tiles - 1, k.dim(.hkv) * head_tiles)) else 1; - const window = self.opts.mask.slidingWindow() orelse 0; - const large_decode_tile = all_decode and q.dim(.hd) == 256 and - q.dim(.hg) <= 4 and window >= 128 and window <= 1024 and - p.block_table.dim(.b) * k.dim(.hkv) * head_tiles >= @divTrunc(sms, 2); - const kv_tile: i64 = if (large_decode_tile) 128 else 64; - const decode_work = p.block_table.dim(.b) * k.dim(.hkv) * head_tiles; - const decode_waves: i64 = if (large_decode_tile and decode_work < sms * 2) 4 else 2; - // A full decode batch supplies enough independent sequence/head CTAs - // to stream the window without split scratch or a separate merge. - // Smaller grids retain the existing partitioned path. - const max_window_tiles = @divTrunc(@as(i64, window) + 2 * kv_tile - 2, kv_tile); - const tiles_per_split: i64 = if (!large_decode_tile and all_decode and q.dim(.hd) == 256 and window > 0) @max(1, @divTrunc(max_window_tiles + 127, 128)) else 0; - const splits = if (tiles_per_split > 0) - @divTrunc(max_window_tiles + tiles_per_split - 1, tiles_per_split) - else if (all_decode) - @min(max_splits, @max(1, @divTrunc(sms * decode_waves + p.block_table.dim(.b) * k.dim(.hkv) * head_tiles - 1, p.block_table.dim(.b) * k.dim(.hkv) * head_tiles))) - else - max_splits; - // Keep independent split CTAs for low-occupancy shapes. At full - // occupancy, a fixed number of workers reuses Q and TMEM and avoids - // launching the entire window bound for short runtime contexts. - const split_stride: i64 = if (tiles_per_split == 1 and splits > 2 and - p.block_table.dim(.b) * k.dim(.hkv) * head_tiles >= sms * 2) 2 else 0; - const cfg: kernel.Config = .{ - .head_dim = q.dim(.hd), - .kv_tile = kv_tile, - // Prefetch an extra feature stage for non-split D512 decode; - // split workloads retain the smaller shared-memory footprint. - .load_buffers = if (kv_tile == 128 or (all_decode and q.dim(.hd) == 512 and splits == 1)) 3 else 2, - .tokens = q.dim(.b), - .batch = p.block_table.dim(.b), - .kv_heads = k.dim(.hkv), - .group = q.dim(.hg), - .pages_per_seq = p.block_table.dim(.p), - .page_size = k.dim(.k_chunk), - .max_query = if (all_decode) 1 else @min(q.dim(.b), @as(i64, @intCast(p.options_.max_seqlen_q))), - .dtype = zml.kernel.cute.from(q.dtype()), - .all_decode = all_decode, - .causal = self.opts.mask.isCausal(), - .window = window, - .scale = self.opts.scale orelse (1 / @sqrt(@as(f32, @floatFromInt(q.dim(.hd))))), - .sink = self.opts.sink != null, - .sms = sms, - .splits = @max(1, splits), - .tiles_per_split = tiles_per_split, - .split_stride = split_stride, - }; - const sink = if (self.opts.sink) |s| s.convert(.f32).reshape(.{cfg.kv_heads * cfg.group}) else zml.Tensor.zeroes(.init(.{cfg.kv_heads * cfg.group}, .f32)); - const out_shape = if (cfg.splits > 1) zml.Shape.init(.{ cfg.splits, cfg.tokens, cfg.kv_heads * cfg.group, cfg.head_dim + 2 }, .f32) else q.shape(); - const out = kernel.Kernel.call(.{ - .q = q, - .k = k, - .v = v, - .table = p.block_table, - .lengths = p.seq_lens, - .starts = p.query_start_len, - .sink = sink, - }, .{ .out = out_shape }, .{ - .cfg = cfg, - .grid = .{ @intCast(cfg.batch * cfg.queryTiles() * (if (cfg.headsInX()) cfg.kv_heads else 1)), @intCast(if (cfg.headsInX()) 1 else cfg.kv_heads * cfg.outputTiles()), @intCast(if (cfg.split_stride > 0) cfg.split_stride else cfg.splits) }, - .block = kernel.block, - }).out; - const result = if (cfg.splits > 1) kernel.Merge.call(.{ - .partials = out, - .lengths = p.seq_lens, - .starts = p.query_start_len, - .sink = sink, - }, .{ .out = q.shape() }, .{ .cfg = cfg, .grid = .{ @intCast(cfg.kv_heads * cfg.group), @intCast(cfg.batch), 1 }, .block = .{ @intCast(cfg.head_dim), 1, 1 } }).out else out; - return result.transpose(output_shape); - } -}; diff --git a/zml/kernels/paged_attention/cute/sm100/BUILD.bazel b/zml/kernels/paged_attention/cute/sm10x/BUILD.bazel similarity index 68% rename from zml/kernels/paged_attention/cute/sm100/BUILD.bazel rename to zml/kernels/paged_attention/cute/sm10x/BUILD.bazel index c8c2a20d..05cd7edc 100644 --- a/zml/kernels/paged_attention/cute/sm100/BUILD.bazel +++ b/zml/kernels/paged_attention/cute/sm10x/BUILD.bazel @@ -1,9 +1,10 @@ package(default_visibility = ["//zml/kernels/paged_attention/cute:__pkg__"]) filegroup( - name = "sm100", + name = "sm10x", srcs = [ "attention.zig", - "sm100.zig", + "sm103.zig", + "sm10x.zig", ], ) diff --git a/zml/kernels/paged_attention/cute/sm10x/attention.zig b/zml/kernels/paged_attention/cute/sm10x/attention.zig new file mode 100644 index 00000000..17f5552e --- /dev/null +++ b/zml/kernels/paged_attention/cute/sm10x/attention.zig @@ -0,0 +1,673 @@ +const std = @import("std"); + +const zml = @import("../../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const V = cute.Value; +const sm103 = @import("sm103.zig"); + +pub const Architecture = enum { sm100, sm103 }; + +pub const Config = struct { + architecture: Architecture, + head_dim: i64, + tokens: i64, + batch: i64, + kv_heads: i64, + group: i64, + pages_per_seq: i64, + page_size: i64, + max_query: i64, + dtype: cute.DType, + decode: bool, + causal: bool, + window: i64, + scale: f32, + sink: bool, + splits: i64 = 1, + + pub fn rows(c: Config) i64 { + return if (c.decode) (if (c.group <= 8) 8 else 16) else 64; + } + pub fn keys(c: Config) i64 { + return if (c.decode) 128 else 64; + } + pub fn queryTiles(c: Config) i64 { + return @divTrunc(c.max_query * c.group + c.rows() - 1, c.rows()); + } + fn paddedDim(c: Config) i64 { + return @max(128, c.head_dim); + } + fn features(c: Config) i64 { + return @divExact(c.paddedDim(), 128); + } + fn buffers(c: Config) i64 { + return if (c.decode) 2 else 3; + } + fn kvBytes(c: Config) i64 { + return c.keys() * 128 * 2; + } + fn qBase(c: Config) i64 { + return c.buffers() * c.kvBytes(); + } + fn pBase(c: Config) i64 { + return c.qBase() + c.rows() * c.paddedDim() * 2; + } + fn barriers(c: Config) i64 { + return c.pBase() + c.rows() * c.keys() * 2; + } + fn completion(c: Config) i64 { + return c.barriers() + c.buffers() * 16; + } + fn holder(c: Config) i64 { + return c.completion() + 16; + } + fn reductions(c: Config) i64 { + return c.holder() + 16; + } + fn sharedBytes(c: Config) i64 { + return c.reductions() + 4 * 16 * 4; + } + fn scoreColumn(c: Config) i64 { + return if (c.decode) c.features() * c.rows() else @min(c.paddedDim(), 448); + } + fn columns(c: Config) u32 { + return std.math.ceilPowerOfTwoAssert(u32, @intCast(@max(32, c.scoreColumn() + (if (c.decode) c.rows() else c.keys())))); + } +}; + +pub const block: [3]i32 = .{ 256, 1, 1 }; +pub const Attention = cute.Kernel(Config, .{ + .name = "paged_attention_cute", + .inputs = &.{ "q", "k", "v", "table", "lengths", "starts", "sink" }, + .outputs = &.{"out"}, + .run = attention, +}); + +fn tuple(comptime T: type, comptime n: usize, values: [n]T) @Tuple(&@as([n]type, @splat(T))) { + var result: @Tuple(&@as([n]type, @splat(T))) = undefined; + inline for (0..n) |i| result[i] = values[i]; + return result; +} + +fn loadInt(b: *B, pointer: V, index: V) V { + return b.loadWords(1, pointer.add(index.to(.i64).mul(4)), .global, .{})[0]; +} + +// Each 64-feature plane uses a 128-byte XOR swizzle. The descriptor and +// vector copies use this same layout for Q, K, V and probabilities. +fn swizzle(row: V, col: V, rows: i64) V { + return col.div(64).mul(rows * 128).add(row.mul(128)).add(col.rem(64).div(8).bitXor(row.rem(8)).mul(16)).add(col.rem(8).mul(2)); +} + +fn descriptor(b: *B, address: V, leading_bytes: i64) V { + const bits: u64 = (64 << 32) | (1 << 46) | (@as(u64, 2) << 61) | (@as(u64, @intCast(@divExact(leading_bytes, 16))) << 16); + return address.to(.i64).shrLogical(4).bitAnd(0x3fff).bitOr(b.cst(.i64, @as(i64, @bitCast(bits)))); +} + +fn instruction(c: Config, qk: bool) u32 { + const format: u32 = if (c.dtype == .bf16) 1 else 0; + const m: u32 = if (c.decode) 128 else 64; + const n: u32 = @intCast(if (c.decode) c.rows() else if (qk) c.keys() else 128); + return (1 << 4) | (format << 7) | (format << 10) | + (@as(u32, @intFromBool(c.decode and !qk)) << 15) | + (@as(u32, @intFromBool(!c.decode and !qk)) << 16) | ((n >> 3) << 17) | ((m >> 4) << 24); +} + +const Range = struct { first: V, count: V }; + +// Used by both attention and merge: every output row must merge exactly the +// partitions produced for its query tile, including causal/window boundaries. +fn keyRange(b: *B, c: Config, first_row: V, qlen: V, klen: V) Range { + const first_pos = klen.sub(qlen).add(first_row.div(c.group)); + const last_pos = klen.sub(qlen).add(first_row.add(c.rows() - 1).div(c.group).minimum(qlen.sub(1))); + const begin = if (c.window > 0) first_pos.sub(c.window - 1).maximum(0) else b.cst(.i32, 0); + const end = if (c.causal) last_pos.add(1).minimum(klen) else klen; + const first = begin.div(c.keys()); + return .{ .first = first, .count = end.add(c.keys() - 1).div(c.keys()).sub(first).maximum(0) }; +} + +fn liveSplits(c: Config, tiles: V) V { + return tiles.add(1).div(2).maximum(1).minimum(c.splits); +} + +const Tile = struct { + b: *B, + c: Config, + q: V, + k: V, + v: V, + table: V, + sink: V, + out: V, + shared: V, + tmem: V, + tid: V, + lane: V, + warp: V, + seq: V, + head: V, + first_row: V, + start: V, + qlen: V, + klen: V, + first_key_tile: V, + tiles: V, + split: V, + + fn ready(t: Tile, id: V) V { + return t.shared.add(t.c.barriers()).add(id.rem(t.c.buffers()).mul(8)); + } + fn released(t: Tile, id: V) V { + return t.ready(id).add(t.c.buffers() * 8); + } + fn phase(t: Tile, id: V) V { + return id.div(t.c.buffers()).rem(2); + } + fn valid(t: Tile, row: V, key: V) V { + const pos = t.klen.sub(t.qlen).add(row.div(t.c.group)); + var result = row.lt(t.qlen.mul(t.c.group)).bitAnd(key.lt(t.klen)); + if (t.c.causal) result = result.bitAnd(key.le(pos)); + if (t.c.window > 0) result = result.bitAnd(key.gt(pos.sub(t.c.window))); + return result; + } + fn putProbability(t: Tile, row: V, key: V, value: V) void { + t.b.storeHalf(t.shared.add(t.c.pBase()).add(swizzle(row, key, t.c.rows())), .shared, value.to(t.c.dtype).bitCast(.i16).to(.i32)); + } + fn put(t: Tile, row: V, dim: V, value: V) void { + const token = t.start.add(row.div(t.c.group)); + var index = token.to(.i64).mul(t.c.kv_heads).add(t.head.to(.i64)).mul(t.c.group).add(row.rem(t.c.group).to(.i64)); + if (t.c.splits > 1) { + index = t.split.to(.i64).mul(t.c.tokens * t.c.kv_heads * t.c.group).add(index).mul(t.c.head_dim + 2).add(dim.to(.i64)); + t.b.storeWords(1, t.out.add(index.mul(4)), .global, .{value.bitCast(.i32)}); + } else { + index = index.mul(t.c.head_dim).add(dim.to(.i64)); + t.b.storeHalf(t.out.add(index.mul(2)), .global, value.to(t.c.dtype).bitCast(.i16).to(.i32)); + } + } + fn norm(t: Tile, row: V, maximum: V, denominator: V) V { + const b = t.b; + if (t.c.splits > 1) return b.cst(.f32, 1); + if (!t.c.sink) return b.select(denominator.gt(0), b.rcpApprox(denominator), b.cst(.f32, 0)); + const sink = loadInt(b, t.sink, t.head.mul(t.c.group).add(row.rem(t.c.group))).bitCast(.f32).mul(std.math.log2e); + const reference = maximum.maximum(sink).maximum(-std.math.floatMax(f32)); + const weight = b.exp2Approx(maximum.sub(reference)); + return weight.div(denominator.mul(weight).add(b.exp2Approx(sink.sub(reference)))); + } +}; + +fn attention(b: *B, c: Config) cute.FinishError!void { + const a = try b.declareArgs(.{ + .q = .{ .ptr = c.dtype }, + .k = .{ .ptr = c.dtype }, + .v = .{ .ptr = c.dtype }, + .table = .{ .ptr = cute.DType.i32 }, + .lengths = .{ .ptr = cute.DType.i32 }, + .starts = .{ .ptr = cute.DType.i32 }, + .sink = .{ .ptr = cute.DType.f32 }, + .out = .{ .ptr = if (c.splits > 1) cute.DType.f32 else c.dtype }, + }); + std.debug.assert(c.sharedBytes() <= 227 * 1024); + b.setFunctionAttribute("cu_attrs", b.parseAttribute("{max_dynamic_shared_size_bytes = #cuda.dev_max_shared_memory_optin}")); + const seq = b.blockIdx().x.div(c.queryTiles()); + const first_row = b.blockIdx().x.rem(c.queryTiles()).mul(c.rows()); + const starts = b.ptrToInt(a.starts, .i64); + const start = loadInt(b, starts, seq); + const qlen = loadInt(b, starts, seq.add(1)).sub(start); + const klen = loadInt(b, b.ptrToInt(a.lengths, .i64), seq); + const range = keyRange(b, c, first_row, qlen, klen); + const splits = liveSplits(c, range.count); + const split = b.blockIdx().z; + var active = b.openIf(first_row.lt(qlen.mul(c.group)).bitAnd(split.lt(splits))); + { + const tid = b.threadIdx().x; + const lo = range.count.mul(split).div(splits); + const hi = range.count.mul(split.add(1)).div(splits); + const shared = b.ptrToInt(b.allocSmem(.i8, @intCast(c.sharedBytes()), 1024), .i32); + var t: Tile = .{ + .b = b, + .c = c, + .q = b.ptrToInt(a.q, .i64), + .k = b.ptrToInt(a.k, .i64), + .v = b.ptrToInt(a.v, .i64), + .table = b.ptrToInt(a.table, .i64), + .sink = b.ptrToInt(a.sink, .i64), + .out = b.ptrToInt(a.out, .i64), + .shared = shared, + .tmem = undefined, + .tid = tid, + .lane = tid.rem(32), + .warp = tid.div(32), + .seq = seq, + .head = b.blockIdx().y, + .first_row = first_row, + .start = start, + .qlen = qlen, + .klen = klen, + .first_key_tile = range.first.add(lo), + .tiles = hi.sub(lo), + .split = split, + }; + var leader = b.openIf(tid.eq(0)); + { + for (0..@as(usize, @intCast(c.buffers()))) |i| { + b.mbarrierInit(shared.add(c.barriers() + @as(i64, @intCast(i * 8))), 128); + b.mbarrierInit(shared.add(c.barriers() + c.buffers() * 8 + @as(i64, @intCast(i * 8))), 1); + } + b.mbarrierInit(shared.add(c.completion()), 1); + b.fenceMbarrierInit(); + leader.yieldThen(.{}); + } + var alloc = b.openIf(tid.lt(32)); + { + b.tmemAllocAt(shared.add(c.holder()), c.columns()); + b.relinquishTmemAllocPermit(.{}); + alloc.yieldThen(.{}); + } + var queries = b.openFor(tid, c.rows() * @divExact(c.paddedDim(), 8), 256, .{}); + { + const row = queries.iv.div(@divExact(c.paddedDim(), 8)); + const col = queries.iv.rem(@divExact(c.paddedDim(), 8)).mul(8); + const packed_row = first_row.add(row); + const valid = packed_row.lt(qlen.mul(c.group)).bitAnd(col.lt(c.head_dim)); + const token = start.add(b.select(valid, packed_row.div(c.group), b.cst(.i32, 0))); + const offset = token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(packed_row.rem(c.group).to(.i64)).mul(c.head_dim).add(col.to(.i64)); + b.cpAsync16(shared.add(c.qBase()).add(swizzle(row, col, c.rows())), t.q.add(offset.mul(2)), valid); + queries.yield(.{}); + } + b.cpAsyncWaitAll(); + b.syncThreads(); + t.tmem = b.loadWords(1, shared.add(c.holder()), .shared, .{})[0]; + b.tcgen05Fence(.after); + var producer = b.openIf(tid.ge(128)); + { + produce(t); + producer.yieldThen(.{}); + } + var consumer = b.openIf(tid.lt(128)); + { + if (c.decode) { + if (c.group <= 2) decode(t, 2) else if (c.group <= 4) decode(t, 4) else if (c.group <= 8) decode(t, 8) else decode(t, 16); + } else prefill(t); + consumer.yieldThen(.{}); + } + b.tcgen05Fence(.before); + b.syncThreads(); + var free = b.openIf(tid.lt(32)); + { + b.tmemDeallocAt(t.tmem, c.columns()); + free.yieldThen(.{}); + } + active.yieldThen(.{}); + } +} + +fn produce(t: Tile) void { + const b = t.b; + const c = t.c; + const row0 = t.tid.sub(128).div(16); + const col = t.tid.rem(16).mul(8); + var tiles = b.openFor(0, t.tiles, 1, .{}); + { + // Paged offsets are shared by K/V and every 128-feature stage. + var offsets: [16]V = undefined; + var valid: [16]V = undefined; + for (0..@as(usize, @intCast(@divExact(c.keys(), 8)))) |i| { + const key = t.first_key_tile.add(tiles.iv).mul(c.keys()).add(row0).add(@as(i32, @intCast(i * 8))); + valid[i] = key.lt(t.klen).bitAnd(col.lt(c.head_dim)); + const safe = b.select(key.lt(t.klen), key, b.cst(.i32, 0)); + const page = loadInt(b, t.table, t.seq.mul(c.pages_per_seq).add(safe.div(c.page_size))); + offsets[i] = page.to(.i64).mul(c.page_size).add(safe.rem(c.page_size).to(.i64)).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.head_dim).add(col.to(.i64)).mul(2); + } + for (0..@as(usize, @intCast(c.features() * 2))) |stage| { + const id = tiles.iv.mul(c.features() * 2).add(@as(i32, @intCast(stage))); + var reuse = b.openIf(id.ge(c.buffers())); + { + b.mbarrierWait(t.released(id), t.phase(id).bitXor(1)); + reuse.yieldThen(.{}); + } + const source = if (stage < c.features()) t.k else t.v; + const feature_offset: i64 = @mod(@as(i64, @intCast(stage)), c.features()) * 256; + for (0..@as(usize, @intCast(@divExact(c.keys(), 8)))) |i| { + const row = row0.add(@as(i32, @intCast(i * 8))); + b.cpAsync16(t.shared.add(id.rem(c.buffers()).mul(c.kvBytes())).add(swizzle(row, col, c.keys())), source.add(offsets[i]).add(feature_offset), valid[i]); + } + b.cpAsyncMbarrierArrive(t.ready(id), true); + } + tiles.yield(.{}); + } +} + +fn multiply(t: Tile, tile: V, comptime qk: bool) void { + const b = t.b; + const c = t.c; + var leader = b.openIf(t.tid.eq(0)); + { + for (0..@as(usize, @intCast(c.features()))) |part| { + const id = tile.mul(c.features() * 2).add(@as(i64, @intCast(part)) + if (qk) @as(i64, 0) else c.features()); + b.mbarrierWait(t.ready(id), t.phase(id)); + b.tcgen05Fence(.after); + const kv = descriptor(b, t.shared.add(id.rem(c.buffers()).mul(c.kvBytes())), if (qk) 16 else c.keys() * 128); + const qp = descriptor(b, t.shared.add(if (qk) c.qBase() else c.pBase()), 16); + const steps: usize = @intCast(if (qk) 8 else @divExact(c.keys(), 16)); + for (0..steps) |step| { + const kv_offset: i64 = if (qk) @as(i64, @intCast(step / 4)) * c.keys() * 128 + @as(i64, @intCast(step % 4)) * 32 else @as(i64, @intCast(step)) * 2048; + const qp_offset: i64 = (if (qk) @as(i64, @intCast(part * 2 + step / 4)) else @as(i64, @intCast(step / 4))) * c.rows() * 128 + @as(i64, @intCast(step % 4)) * 32; + const left = if (c.decode) kv.add(@divExact(kv_offset, 16)) else qp.add(@divExact(qp_offset, 16)); + const right = if (c.decode) qp.add(@divExact(qp_offset, 16)) else kv.add(@divExact(kv_offset, 16)); + const column: i64 = if (qk) c.scoreColumn() else @as(i64, @intCast(part)) * (if (c.decode) c.rows() else 128); + const accumulate = if (qk) b.cst(.i32, @intFromBool(part > 0 or step > 0)).ne(0) else tile.gt(0).bitOr(b.cst(.i32, @intFromBool(step > 0)).ne(0)); + b.tcgen05MmaF16(t.tmem.add(column), left, right, b.cst(.i32, @as(i32, @bitCast(instruction(c, qk)))), accumulate); + } + b.tcgen05Commit(t.released(id)); + } + b.tcgen05Commit(t.shared.add(c.completion())); + leader.yieldThen(.{}); + } + b.mbarrierWait(t.shared.add(c.completion()), b.cst(.i32, if (qk) 0 else 1)); + b.tcgen05Fence(.after); + // A fast warp must not wrap completion parity before every consumer has + // observed the preceding PV completion. + if (!qk) b.barrierSync(1, 128); +} + +fn publishProbabilities(t: Tile) void { + t.b.fenceProxyAsyncShared(); + t.b.tcgen05Fence(.before); + t.b.barrierSync(1, 128); + t.b.tcgen05Fence(.after); +} + +fn reduceRow(comptime n: usize, values: [n]V, comptime maximum: bool) V { + var tree = values; + var width: usize = n; + while (width > 1) { + width /= 2; + for (0..width) |i| tree[i] = if (maximum) tree[i * 2].maximum(tree[i * 2 + 1]) else tree[i * 2].add(tree[i * 2 + 1]); + } + return tree[0]; +} + +fn reduceHeads(t: Tile, comptime heads: usize, values: [heads]V, comptime maximum: bool) [heads]V { + const b = t.b; + for (values, 0..) |value, h| { + var reduced = value; + inline for (.{ 16, 8, 4, 2, 1 }) |step| reduced = if (maximum) reduced.maximum(b.shuffleXor(reduced, step)) else reduced.add(b.shuffleXor(reduced, step)); + var leader = b.openIf(t.lane.eq(0)); + { + b.storeWords(1, t.shared.add(t.c.reductions()).add(t.warp.mul(64)).add(@as(i32, @intCast(h * 4))), .shared, .{reduced.bitCast(.i32)}); + leader.yieldThen(.{}); + } + } + b.barrierSync(1, 128); + var result: [heads]V = undefined; + for (0..heads) |h| { + var value = b.loadWords(1, t.shared.add(t.c.reductions() + @as(i64, @intCast(h * 4))), .shared, .{})[0].bitCast(.f32); + for (1..4) |w| { + const next = b.loadWords(1, t.shared.add(t.c.reductions() + @as(i64, @intCast(w * 64 + h * 4))), .shared, .{})[0].bitCast(.f32); + value = if (maximum) value.maximum(next) else value.add(next); + } + result[h] = value; + } + b.barrierSync(1, 128); + return result; +} + +fn decode(t: Tile, comptime heads: usize) void { + const b = t.b; + const c = t.c; + const address = t.tmem.add(t.warp.mul(32).shl(16)); + const key_row = t.warp.mul(32).add(t.lane); + const zero = b.cst(.f32, 0); + const neg_inf = b.cst(.f32, -std.math.inf(f32)); + var initial: [2 * heads]V = undefined; + for (0..heads) |h| { + initial[h] = neg_inf; + initial[heads + h] = zero; + } + var tiles = b.openFor(0, t.tiles, 1, tuple(V, 2 * heads, initial)); + { + multiply(t, tiles.iv, true); + const raw = b.tmemLoad32x32b(heads, address.add(c.scoreColumn())); + b.fenceTmemLoad(); + const key = t.first_key_tile.add(tiles.iv).mul(c.keys()).add(key_row); + var scores: [heads]V = undefined; + for (0..heads) |h| scores[h] = b.select(t.valid(t.first_row.add(@as(i32, @intCast(h))), key), raw[h].bitCast(.f32).mul(c.scale * std.math.log2e), neg_inf); + const maxima = reduceHeads(t, heads, scores, true); + var next: [2 * heads]V = undefined; + var alpha: [heads]V = undefined; + var probabilities: [heads]V = undefined; + for (0..heads) |h| { + next[h] = tiles.carried[h].maximum(maxima[h]); + const reference = next[h].maximum(-std.math.floatMax(f32)); + alpha[h] = b.exp2Approx(tiles.carried[h].sub(reference)); + probabilities[h] = b.exp2Approx(scores[h].sub(reference)); + } + const sums = reduceHeads(t, heads, probabilities, false); + for (0..heads) |h| { + next[heads + h] = tiles.carried[heads + h].mul(alpha[h]).add(sums[h]); + t.putProbability(b.cst(.i32, @as(i32, @intCast(h))), key_row, probabilities[h]); + } + for (heads..@as(usize, @intCast(c.rows()))) |h| t.putProbability(b.cst(.i32, @as(i32, @intCast(h))), key_row, zero); + var rescale = b.openIf(tiles.iv.gt(0)); + { + for (0..@as(usize, @intCast(c.features()))) |part| { + const addr = address.add(@as(i64, @intCast(part)) * c.rows()); + var out = b.tmemLoad32x32b(heads, addr); + b.fenceTmemLoad(); + for (0..heads) |h| out[h] = out[h].bitCast(.f32).mul(alpha[h]).bitCast(.i32); + b.tmemStore32x32b(heads, addr, out); + } + b.fenceTmemStore(); + rescale.yieldThen(.{}); + } + publishProbabilities(t); + multiply(t, tiles.iv, false); + tiles.yield(tuple(V, 2 * heads, next)); + } + for (0..@as(usize, @intCast(c.features()))) |part| { + const out = b.tmemLoad32x32b(heads, address.add(@as(i64, @intCast(part)) * c.rows())); + b.fenceTmemLoad(); + const dim = key_row.add(@as(i32, @intCast(part * 128))); + for (0..heads) |h| { + const row = t.first_row.add(@as(i32, @intCast(h))); + var valid = b.openIf(row.lt(t.qlen.mul(c.group))); + { + var feature = b.openIf(dim.lt(c.head_dim)); + { + const value = b.select(tiles.results[heads + h].gt(0), out[h].bitCast(.f32).mul(t.norm(row, tiles.results[h], tiles.results[heads + h])), zero); + t.put(row, dim, value); + feature.yieldThen(.{}); + } + if (c.splits > 1 and part == 0) { + var stats = b.openIf(t.tid.eq(0)); + { + t.put(row, b.cst(.i32, c.head_dim), tiles.results[h]); + t.put(row, b.cst(.i32, c.head_dim + 1), tiles.results[heads + h]); + stats.yieldThen(.{}); + } + } + valid.yieldThen(.{}); + } + } + } +} + +fn prefill(t: Tile) void { + const b = t.b; + const c = t.c; + const address = t.tmem.add(t.warp.mul(32).shl(16)); + const local_row = t.warp.mul(16).add(t.lane); + const row = t.first_row.add(local_row); + const owns = t.lane.lt(16); + const zero = b.cst(.f32, 0); + const neg_inf = b.cst(.f32, -std.math.inf(f32)); + var tiles = b.openFor(0, t.tiles, 1, .{ neg_inf, zero }); + { + // D512 fills TMEM with O. Preserve its last 64 columns in registers + // while QK reuses them for scores, avoiding duplicate QK/softmax CTAs. + var saved: [64]V = undefined; + if (c.head_dim == 512) { + var preserve = b.openIfElse(tiles.iv.gt(0), tuple(@TypeOf(zero.type_()), 64, @splat(zero.type_()))); + { + const tail = b.tmemLoad32x32b(64, address.add(c.scoreColumn())); + b.fenceTmemLoad(); + var words: [64]V = undefined; + for (tail, 0..) |value, i| words[i] = value.bitCast(.f32); + preserve.yieldThen(tuple(V, 64, words)); + } + { + preserve.yieldElse(tuple(V, 64, @as([64]V, @splat(zero)))); + } + @memcpy(&saved, &preserve.results); + b.barrierSync(1, 128); + } + multiply(t, tiles.iv, true); + var scores: [64]V = undefined; + var hardware_max = neg_inf; + if (c.architecture == .sm103) { + const loaded = sm103.loadScores(b, address.add(c.scoreColumn())); + scores = loaded.values; + hardware_max = loaded.maximum; + } else { + const loaded = b.tmemLoad32x32b(64, address.add(c.scoreColumn())); + b.fenceTmemLoad(); + for (loaded, 0..) |value, i| scores[i] = value.bitCast(.f32); + } + const key0 = t.first_key_tile.add(tiles.iv).mul(c.keys()); + // Hardware maxima describe raw scores. Only fully visible tiles may + // use them; page tails and mask boundaries reduce post-mask values. + const full = owns.bitAnd(t.valid(row, key0)).bitAnd(t.valid(row, key0.add(63))); + for (0..64) |i| { + scores[i] = b.select(owns.bitAnd(t.valid(row, key0.add(@as(i32, @intCast(i))))), scores[i].mul(c.scale * std.math.log2e), neg_inf); + } + const tile_max = if (c.architecture == .sm103 and c.scale > 0) blk: { + var choose = b.openIfElse(full, .{zero.type_()}); + { + choose.yieldThen(.{hardware_max.mul(c.scale * std.math.log2e)}); + } + { + choose.yieldElse(.{reduceRow(64, scores, true)}); + } + break :blk choose.results[0]; + } else reduceRow(64, scores, true); + const maximum = tiles.carried[0].maximum(tile_max); + const reference = maximum.maximum(-std.math.floatMax(f32)); + const alpha = b.exp2Approx(tiles.carried[0].sub(reference)); + var probabilities: [64]V = undefined; + for (0..64) |i| { + probabilities[i] = b.exp2Approx(scores[i].sub(reference)); + } + const denominator = tiles.carried[1].mul(alpha).add(reduceRow(64, probabilities, false)); + var write = b.openIf(owns); + { + for (0..64) |i| t.putProbability(local_row, b.cst(.i32, @as(i32, @intCast(i))), probabilities[i]); + write.yieldThen(.{}); + } + var rescale = b.openIf(tiles.iv.gt(0)); + { + for (0..@as(usize, @intCast(@divExact(if (c.head_dim == 512) c.scoreColumn() else c.paddedDim(), 32)))) |chunk| { + const addr = address.add(@as(i32, @intCast(chunk * 32))); + var out = b.tmemLoad32x32b(32, addr); + b.fenceTmemLoad(); + for (&out) |*value| value.* = value.bitCast(.f32).mul(alpha).bitCast(.i32); + b.tmemStore32x32b(32, addr, out); + } + if (c.head_dim == 512) { + for (&saved) |*value| value.* = value.mul(alpha).bitCast(.i32); + b.tmemStore32x32b(64, address.add(c.scoreColumn()), saved); + } + b.fenceTmemStore(); + rescale.yieldThen(.{}); + } + publishProbabilities(t); + multiply(t, tiles.iv, false); + tiles.yield(.{ maximum, denominator }); + } + const norm = t.norm(row, tiles.results[0], tiles.results[1]); + for (0..@as(usize, @intCast(@divExact(c.paddedDim(), 32)))) |chunk| { + const out = b.tmemLoad32x32b(32, address.add(@as(i32, @intCast(chunk * 32)))); + b.fenceTmemLoad(); + var valid = b.openIf(owns.bitAnd(row.lt(t.qlen.mul(c.group)))); + { + for (0..32) |i| { + const dim = chunk * 32 + i; + if (dim < c.head_dim) t.put(row, b.cst(.i32, @as(i32, @intCast(dim))), b.select(tiles.results[1].gt(0), out[i].bitCast(.f32).mul(norm), zero)); + } + if (c.splits > 1 and chunk == 0) { + t.put(row, b.cst(.i32, c.head_dim), tiles.results[0]); + t.put(row, b.cst(.i32, c.head_dim + 1), tiles.results[1]); + } + valid.yieldThen(.{}); + } + } +} + +pub const Merge = cute.Kernel(Config, .{ + .name = "paged_attention_cute_merge", + .inputs = &.{ "partials", "lengths", "starts", "sink" }, + .outputs = &.{"out"}, + .run = merge, +}); + +fn merge(b: *B, c: Config) cute.FinishError!void { + const a = try b.declareArgs(.{ + .partials = .{ .tensor = .{ .dtype = .f32, .shape = &.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 } } }, + .lengths = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch} } }, + .starts = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch + 1} } }, + .sink = .{ .tensor = .{ .dtype = .f32, .shape = &.{if (c.sink) c.kv_heads * c.group else 1} } }, + .out = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.tokens, c.kv_heads * c.group, c.head_dim } } }, + }); + const token = b.blockIdx().x; + const head = b.blockIdx().y; + var active = b.openIf(token.lt(a.starts.get(.{c.batch}))); + { + var find = b.openFor(0, c.batch, 1, .{b.cst(.i32, 0)}); + { + const owns = token.ge(a.starts.get(.{find.iv})).bitAnd(token.lt(a.starts.get(.{find.iv.add(1)}))); + find.yield(.{b.select(owns, find.iv, find.carried[0])}); + } + const seq = find.results[0]; + const start = a.starts.get(.{seq}); + const qlen = a.starts.get(.{seq.add(1)}).sub(start); + const row = token.sub(start).mul(c.group).add(head.rem(c.group)); + const range = keyRange(b, c, row.div(c.rows()).mul(c.rows()), qlen, a.lengths.get(.{seq})); + const splits = liveSplits(c, range.count); + const zero = b.cst(.f32, 0); + const sink = if (c.sink) a.sink.get(.{head}).mul(std.math.log2e) else b.cst(.f32, -std.math.inf(f32)); + var maximum = b.openFor(0, splits, 1, .{sink}); + { + maximum.yield(.{maximum.carried[0].maximum(a.partials.get(.{ maximum.iv, token, head, c.head_dim }))}); + } + const reference = maximum.results[0].maximum(-std.math.floatMax(f32)); + var dims = b.openFor(b.threadIdx().x, c.head_dim, 128, .{}); + { + var sum = b.openFor(0, splits, 1, .{ zero, if (c.sink) b.exp2Approx(sink.sub(reference)) else zero }); + { + const weight = b.exp2Approx(a.partials.get(.{ sum.iv, token, head, c.head_dim }).sub(reference)); + sum.yield(.{ + sum.carried[0].add(weight.mul(a.partials.get(.{ sum.iv, token, head, dims.iv }))), + sum.carried[1].add(weight.mul(a.partials.get(.{ sum.iv, token, head, c.head_dim + 1 }))), + }); + } + const norm = b.select(sum.results[1].gt(0), b.rcpApprox(sum.results[1]), zero); + a.out.set(.{ token, head, dims.iv }, sum.results[0].mul(norm).to(c.dtype)); + dims.yield(.{}); + } + active.yieldThen(.{}); + } +} + +test "SM10x layouts and instruction feature gates" { + for ([_]i64{ 32, 64, 128, 256, 512 }) |dim| for ([_]bool{ true, false }) |decode_only| for ([_]Architecture{ .sm100, .sm103 }) |architecture| { + const c: Config = .{ .architecture = architecture, .head_dim = dim, .tokens = 80, .batch = 4, .kv_heads = 2, .group = 5, .pages_per_seq = 256, .page_size = 16, .max_query = if (decode_only) 1 else 17, .dtype = .bf16, .decode = decode_only, .causal = true, .window = 0, .scale = 0.125, .sink = true, .splits = 4 }; + try std.testing.expect(c.columns() >= 32 and c.columns() <= 512); + try std.testing.expect(c.sharedBytes() <= 227 * 1024); + const ir = try Attention.emit(std.testing.allocator, c, block); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "tcgen05.mma") != null); + try std.testing.expectEqual(architecture == .sm103 and !decode_only, std.mem.indexOf(u8, ir, "tcgen05.ld.red") != null); + const merge_ir = try Merge.emit(std.testing.allocator, c, .{ 128, 1, 1 }); + defer std.testing.allocator.free(merge_ir); + }; +} diff --git a/zml/kernels/paged_attention/cute/sm10x/sm103.zig b/zml/kernels/paged_attention/cute/sm10x/sm103.zig new file mode 100644 index 00000000..bc1da928 --- /dev/null +++ b/zml/kernels/paged_attention/cute/sm10x/sm103.zig @@ -0,0 +1,44 @@ +const std = @import("std"); +const mlir = @import("mlir"); +const llvm = @import("mlir/dialects").llvm; + +const cute = @import("../../../../zml.zig").kernel.cute; + +pub const Scores = struct { + values: [64]cute.Value, + maximum: cute.Value, +}; + +// SM103 adds a per-TMEM-lane reduction across the loaded columns. This is +// useful for query-major scores; swapped decode reduces across lanes instead. +pub fn loadScores(b: *cute.Builder, address: cute.Value) Scores { + const asm_parts = comptime blk: { + @setEvalBranchQuota(20000); + var registers: []const u8 = ""; + var constraints: []const u8 = ""; + var fields: []const u8 = ""; + for (0..65) |i| { + if (i < 64) registers = registers ++ (if (i == 0) "" else ", ") ++ std.fmt.comptimePrint("${d}", .{i}); + constraints = constraints ++ (if (i == 0) "" else ",") ++ "=f"; + fields = fields ++ (if (i == 0) "" else ", ") ++ "f32"; + } + break :blk .{ + "tcgen05.ld.red.sync.aligned.32x32b.x64.max.f32 {" ++ registers ++ "}, $64, [$65];", + constraints ++ ",r", + "!llvm.struct<(" ++ fields ++ ")>", + }; + }; + const result = b.emit(llvm.inline_asm(b.ctx, asm_parts[0], &.{address.inner}, b.parseType(asm_parts[2]), .{ .constraints = asm_parts[1], .has_side_effects = true }, b.loc())); + var scores: Scores = undefined; + for (0..65) |i| { + const value = b.emit(mlir.Operation.make(b.ctx, "llvm.extractvalue", .{ + .operands = .{ .flat = &.{result.inner} }, + .results = .{ .flat = &.{cute.DType.f32.toMlir(b.ctx)} }, + .attributes = &.{.named(b.ctx, "position", .denseArray(b.ctx, .i64, &.{@intCast(i)}))}, + .location = b.loc(), + })); + if (i < 64) scores.values[i] = value else scores.maximum = value; + } + b.fenceTmemLoad(); + return scores; +} diff --git a/zml/kernels/paged_attention/cute/sm10x/sm10x.zig b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig new file mode 100644 index 00000000..664720ee --- /dev/null +++ b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig @@ -0,0 +1,165 @@ +const std = @import("std"); +const stdx = @import("stdx"); + +const zml = @import("../../../../zml.zig"); +const paged = @import("../../paged_attention.zig"); +const cute = @import("../cute.zig"); +const kernel = @import("attention.zig"); + +pub const Options = cute.Options; +pub const Parameters = cute.Parameters; + +test { + _ = kernel; +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + const cc = zml.platform.cuda.computeCapability(platform) orelse return false; + return cc.major == 10; +} + +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { + return zml.ops.manualComputation(Context.body, Context{ .q = q, .k = k, .v = v, .parameters = parameters, .opts = opts }, q.shape()); +} + +const Context = struct { + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + parameters: Parameters, + opts: paged.AttentionOptions, + + fn body(self: Context, output_shape: zml.Shape) zml.Tensor { + const q = self.q.transpose(.{ .b, .hkv, .hg, .hd }); + const k = self.k.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const v = self.v.transpose(.{ .page, .k_chunk, .hkv, .hd }); + stdx.debug.assert(q.dim(.hkv) == k.dim(.hkv) and k.shape().eql(v.shape()) and k.dim(.hd) == q.dim(.hd) and q.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); + const p = self.parameters; + const decode = !p.options_.is_prefill; + const cc = zml.platform.cuda.computeCapability(zml.Compiler.current().platform).?; + var c: kernel.Config = .{ + .architecture = if (cc.minor >= 3) .sm103 else .sm100, + .head_dim = q.dim(.hd), + .tokens = q.dim(.b), + .batch = p.block_table.dim(.b), + .kv_heads = k.dim(.hkv), + .group = q.dim(.hg), + .pages_per_seq = p.block_table.dim(.p), + .page_size = k.dim(.k_chunk), + .max_query = if (decode) 1 else @min(q.dim(.b), @as(i64, @intCast(p.options_.max_seqlen_q))), + .dtype = zml.kernel.cute.from(q.dtype()), + .decode = decode, + .causal = self.opts.mask.isCausal(), + .window = self.opts.mask.slidingWindow() orelse 0, + .scale = self.opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q.dim(.hd)))), + .sink = self.opts.sink != null, + }; + const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); + const work = query_tiles * c.kv_heads; + const bound = if (c.window > 0 and c.causal) @min(c.window + @divTrunc(c.rows() + c.group - 1, c.group), c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; + c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(cute.multiprocessorCount(zml.Compiler.current().platform))) * 2 + work - 1, work), @divTrunc(bound + 2 * c.keys() - 1, 2 * c.keys())))); + // An unused scalar stays a constant buffer; broadcasting a dummy for + // every head would add a helper kernel to each attention invocation. + const sink = if (self.opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.scalar(0, .f32).reshape(.{1}); + const out = kernel.Attention.call(.{ .q = q, .k = k, .v = v, .table = p.block_table, .lengths = p.seq_lens, .starts = p.query_start_len, .sink = sink }, .{ + .out = if (c.splits > 1) zml.Shape.init(.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 }, .f32) else q.shape(), + }, .{ .cfg = c, .grid = .{ @intCast(c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, .block = kernel.block }).out; + const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .lengths = p.seq_lens, .starts = p.query_start_len, .sink = sink }, .{ .out = q.shape() }, .{ + .cfg = c, + .grid = .{ @intCast(c.tokens), @intCast(c.kv_heads * c.group), 1 }, + .block = .{ 128, 1, 1 }, + }).out else out; + return result.transpose(output_shape); + } +}; + +const ArchitectureCheck = struct { + fn forward(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, c: kernel.Config) zml.Tensor { + const table_values: [9]i32 = .{ 4, 2, 8, 1, 6, 0, 7, 3, 5 }; + const table = zml.Tensor.constantTensor(.init(.{ 1, 9 }, .i32), std.mem.asBytes(&table_values)); + const lengths = zml.Tensor.constantTensor(.init(.{1}, .i32), std.mem.asBytes(&[_]i32{129})); + const starts = zml.Tensor.constantTensor(.init(.{2}, .i32), std.mem.asBytes(&[_]i32{ 0, 3 })); + const sink = zml.Tensor.zeroes(.init(.{3}, .f32)); + const partials = kernel.Attention.call(.{ .q = q, .k = k, .v = v, .table = table, .lengths = lengths, .starts = starts, .sink = sink }, .{ + .out = .init(.{ 2, 3, 3, c.head_dim + 2 }, .f32), + }, .{ .cfg = c, .grid = .{ 1, 1, 2 }, .block = kernel.block }).out; + return kernel.Merge.call(.{ .partials = partials, .lengths = lengths, .starts = starts, .sink = sink }, .{ .out = q.shape() }, .{ + .cfg = c, + .grid = .{ 3, 3, 1 }, + .block = .{ 128, 1, 1 }, + }).out; + } +}; + +test "SM10x GB200 and GB300 instruction paths match CPU reference" { + const platform = zml.testing.env(); + if (!isAvailable(platform)) return error.SkipZigTest; + const cc = zml.platform.cuda.computeCapability(platform).?; + const a = std.testing.allocator; + const io = std.testing.io; + const BF16 = zml.floats.BFloat16; + const table: [9]usize = .{ 4, 2, 8, 1, 6, 0, 7, 3, 5 }; + for ([_]i64{ 128, 512 }) |head_dim| { + const d: usize = @intCast(head_dim); + const q_host = try a.alloc(BF16, 9 * d); + defer a.free(q_host); + const k_host = try a.alloc(BF16, 144 * d); + defer a.free(k_host); + const v_host = try a.alloc(BF16, 144 * d); + defer a.free(v_host); + for (q_host, 0..) |*value, i| value.* = .fromF32(@as(f32, @floatFromInt(i % 13)) / 16 - 0.375); + @memset(k_host, .fromF32(std.math.nan(f32))); + @memset(v_host, .fromF32(std.math.nan(f32))); + for (0..129) |key| for (0..d) |feature| { + const offset = (table[key / 16] * 16 + key % 16) * d + feature; + k_host[offset] = .fromF32(@as(f32, @floatFromInt((key * 7 + feature) % 17)) / 16 - 0.5); + v_host[offset] = .fromF32(@as(f32, @floatFromInt((key + feature * 3) % 23)) / 16 - 0.625); + }; + const q: zml.Tensor = .init(.{ .b = 3, .hkv = 1, .hg = 3, .hd = d }, .bf16); + const k: zml.Tensor = .init(.{ .page = 9, .k_chunk = 16, .hkv = 1, .hd = d }, .bf16); + const v: zml.Tensor = .fromShape(k.shape()); + var qb = try zml.Buffer.fromBytes(io, platform, q.shape(), std.mem.sliceAsBytes(q_host)); + defer qb.deinit(); + var kb = try zml.Buffer.fromBytes(io, platform, k.shape(), std.mem.sliceAsBytes(k_host)); + defer kb.deinit(); + var vb = try zml.Buffer.fromBytes(io, platform, k.shape(), std.mem.sliceAsBytes(v_host)); + defer vb.deinit(); + for ([_]kernel.Architecture{ .sm100, .sm103 }) |architecture| { + if (architecture == .sm103 and cc.minor < 3) continue; + const c: kernel.Config = .{ .architecture = architecture, .head_dim = head_dim, .tokens = 3, .batch = 1, .kv_heads = 1, .group = 3, .pages_per_seq = 9, .page_size = 16, .max_query = 3, .dtype = .bf16, .decode = false, .causal = true, .window = 0, .scale = 0.125, .sink = true, .splits = 2 }; + const exe = try platform.compileFn(a, io, ArchitectureCheck.forward, .{ q, k, v, c }, .{}); + defer exe.deinit(); + var output = try exe.eval(a, io, .{ qb, kb, vb }); + defer output.deinit(); + var host = try output.toSliceAlloc(a, io); + defer host.free(a); + const actual = host.items(BF16); + for (0..9) |row| { + const keys = 127 + row / 3; + var scores: [129]f32 = undefined; + var maximum: f32 = 0; // Sink score, with zero value. + for (0..keys) |key| { + const offset = (table[key / 16] * 16 + key % 16) * d; + var score: f32 = 0; + for (0..d) |feature| score += q_host[row * d + feature].toF32() * k_host[offset + feature].toF32(); + scores[key] = score * c.scale; + maximum = @max(maximum, scores[key]); + } + var denominator = @exp(-maximum); + for (0..keys) |key| { + scores[key] = @exp(scores[key] - maximum); + denominator += scores[key]; + } + for (0..d) |feature| { + var expected: f32 = 0; + for (0..keys) |key| expected += scores[key] * v_host[(table[key / 16] * 16 + key % 16) * d + feature].toF32(); + expected /= denominator; + const got = actual[row * d + feature].toF32(); + try std.testing.expect(std.math.isFinite(got)); + try std.testing.expect(@abs(got - expected) <= 0.003 + 0.02 * @abs(expected)); + } + } + } + } +} diff --git a/zml/kernels/paged_attention/cute/sm120/sm120.zig b/zml/kernels/paged_attention/cute/sm120/sm120.zig index 45699739..6dfeb7e2 100644 --- a/zml/kernels/paged_attention/cute/sm120/sm120.zig +++ b/zml/kernels/paged_attention/cute/sm120/sm120.zig @@ -2,11 +2,11 @@ const stdx = @import("stdx"); const zml = @import("../../../../zml.zig"); const paged = @import("../../paged_attention.zig"); -const triton = @import("../../triton/triton.zig"); +const cute = @import("../cute.zig"); const kernel = @import("attention.zig"); -pub const Options = triton.paged.Options; -pub const Parameters = triton.paged.Parameters; +pub const Options = cute.Options; +pub const Parameters = cute.Parameters; test { _ = kernel; @@ -58,7 +58,7 @@ const Context = struct { const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); const work = query_tiles * c.kv_heads; const bound = if (c.window > 0) @min(c.window, c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; - c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(triton.getCuCount())) * 2 + work - 1, work), @divTrunc(bound + 127, 128)))); + c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(cute.multiprocessorCount(zml.Compiler.current().platform))) * 2 + work - 1, work), @divTrunc(bound + 127, 128)))); const sink = if (self.opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.zeroes(.init(.{c.kv_heads * c.group}, .f32)); const out = kernel.Kernel.call(.{ .q = q, From 9658a71ec0db98fd33406c4ec85df4122b049f3a Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Thu, 22 Oct 2026 13:00:00 +0200 Subject: [PATCH 814/881] zml/kernels: relax MoE Fly restrictions (#978) Remove hardcoded hidden and intermediate hidden dimensions check. --- zml/kernels/moe/fly/fly.zig | 17 +++++++++++------ zml/kernels/moe/fly/moe.zig | 9 +++++++-- zml/kernels/moe/tests.zig | 18 ++++++++++++++++++ 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/zml/kernels/moe/fly/fly.zig b/zml/kernels/moe/fly/fly.zig index f3f6ece6..e5c5b6de 100644 --- a/zml/kernels/moe/fly/fly.zig +++ b/zml/kernels/moe/fly/fly.zig @@ -209,27 +209,32 @@ pub fn supports(opts: FusedExpertsArgs) bool { opts.routing_weight_placement != .before_down) return false; const gate_up = opts.gate_up.weight; const down = opts.down.weight; - if (gate_up.rank() != 3 or down.rank() != 3 or opts.hidden_states.dim(.d) != 5120) return false; + if (gate_up.rank() != 3 or down.rank() != 3) return false; const gate_up_scale = opts.gate_up.quantizationScales() orelse return false; const down_scale = opts.down.quantizationScales() orelse return false; const experts = if (opts.expert_map) |map| map.dim(.expert) else gate_up.dim(0); if (experts < 1 or experts > 512) return false; - inline for (.{ .{ gate_up, gate_up_scale, 4608, 5120 }, .{ down, down_scale, 5120, 2304 } }) |projection| { - const weight, const scale, const n, const k = projection; + var max_width: i64 = opts.hidden_states.dim(.d); + inline for (.{ .{ gate_up, gate_up_scale }, .{ down, down_scale } }) |projection| { + const weight, const scale = projection; const packing_factor: i64 = switch (weight.dtype()) { .u8, .i8 => 2, .f4e2m1 => 1, else => return false, }; - if (weight.dim(0) < 1 or weight.dim(0) > 512 or weight.dim(1) != n or weight.dim(2) * packing_factor != k or - scale.rank() != 3 or scale.dim(0) != weight.dim(0) or scale.dim(1) != n or scale.dim(2) != k / 32) return false; + const n = weight.dim(1); + const k = weight.dim(2) * packing_factor; + if (!fly.Mfma.supportsShape(n, k)) return false; + max_width = @max(max_width, n); + if (weight.dim(0) < 1 or weight.dim(0) > 512 or + scale.rank() != 3 or scale.dim(0) != weight.dim(0) or scale.dim(1) != n or scale.dim(2) != @divExact(k, 32)) return false; switch (scale.dtype()) { .u8, .i8, .f8e8m0 => {}, else => return false, } } const routes = opts.hidden_states.dim(.t) * opts.topk_ids.dim(.topk); - return routes > 0 and routes <= std.math.maxInt(i32) / 5120; + return routes > 0 and routes <= @divTrunc(std.math.maxInt(i32), max_width); } pub fn prepareInput(x: Tensor, scheme: ?zml.Quantization.Scheme, _: zml.DataType, _: bool) struct { Tensor, ?Tensor } { diff --git a/zml/kernels/moe/fly/moe.zig b/zml/kernels/moe/fly/moe.zig index ea3cdd62..01d5e412 100644 --- a/zml/kernels/moe/fly/moe.zig +++ b/zml/kernels/moe/fly/moe.zig @@ -203,6 +203,11 @@ pub const Sort = struct { }; pub const Mfma = struct { + /// MFMA loads full 64-column by 128-reduction tiles without tail masks. + pub fn supportsShape(n: i64, k: i64) bool { + return n > 0 and k > 0 and @mod(n, 64) == 0 and @mod(k, 128) == 0; + } + /// The caller owns route localization, activation and reduction. /// `expert_ids` contains local bank indices, including -1 for remote experts. /// Sorted routes must contain a real prefix followed by padding per expert. @@ -226,7 +231,7 @@ pub const Mfma = struct { topk > 0 and block_m > 0 and block_m % 16 == 0); const n = weight.dim(1); const k = input.dim(1); - std.debug.assert((n == 4608 and k == 5120) or (n == 5120 and k == 2304)); + std.debug.assert(supportsShape(n, k)); std.debug.assert(input.dim(0) > 0 and weight.dim(0) > 0 and weight.dim(2) == @divExact(k, 2) and scales.dim(0) == weight.dim(0) and scales.dim(1) == n and scales.dim(2) == @divExact(k, 32) and output.dim(-1) == n); @@ -534,7 +539,7 @@ const Gemv = struct { /// One local expert ID per original route; negative/out-of-bank IDs write zero. fn call(x: Tensor, weight: Tensor, scales: Tensor, ids: Tensor, output: zml.Shape, topk: usize) Tensor { - const cfg: Gemv.Config = if (weight.dim(1) == 4608) + const cfg: Gemv.Config = if (x.dim(1) >= 4096) .{ .rows = 4, .lanes = 64, .unroll = 1 } else .{ .rows = 4, .lanes = 32, .unroll = 5 }; diff --git a/zml/kernels/moe/tests.zig b/zml/kernels/moe/tests.zig index ec92bbeb..cfeb05ba 100644 --- a/zml/kernels/moe/tests.zig +++ b/zml/kernels/moe/tests.zig @@ -622,6 +622,24 @@ test "Fly compatibility matrix" { }, .{}); } +test "Fly general projection shapes" { + const pipeline: Pipeline = .{ .platform = zml.testing.env() }; + + // Cover GEMV, sorted MFMA with 16- and 32-row tiles, and wider projections. + try pipeline.run(std.testing.io, std.testing.allocator, .fly, .{ + .formats = &.{.{ .scheme = .mxfp4 }}, + .activations = &.{.{ .swiglu = .{} }}, + .placements = &.{.before_down}, + .shapes = &.{ + .{ .width = 384, .intermediate = 128, .experts = 2 }, + .{ .width = 384, .intermediate = 128, .experts = 2, .tokens = 7 }, + .{ .width = 384, .intermediate = 128, .experts = 2, .tokens = 1024 }, + .{ .width = 7168, .intermediate = 2048, .experts = 2 }, + .{ .width = 7168, .intermediate = 2048, .experts = 2, .tokens = 7 }, + }, + }, .{}); +} + test "Metal compatibility matrix" { const platform = zml.testing.env(); From 5d85f0f52181d73388c781ac639fc0fbb034d75b Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Thu, 22 Oct 2026 14:25:28 +0200 Subject: [PATCH 815/881] platforms: bump plugins and stop preallocating by default on Metal (#979) Metal does not need to pre-allocate memory --- platforms/cpu/cpu.bzl | 12 ++++++------ platforms/cuda/cuda.bzl | 8 ++++---- platforms/furiosa/furiosa.bzl | 4 ++-- platforms/metal/metal.bzl | 4 ++-- platforms/rocm/rocm.bzl | 4 ++-- zml/platform.zig | 2 +- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index f6d1393f..0f4a1668 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -42,22 +42,22 @@ def _cpu_plugin_impl(mctx): http_archive( name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "ed12c7d89f2a9f914368ede117c6086ea36c557cb3f0ddfde20a92dbaa4192a1", - url = "https://mirror.zml.ai/plugins/202610191219.248.1.c2619532effe/zml-cpu-linux-amd64.tar.zst", + sha256 = "309b737f996024b98b1ee18a44e37df1b328e7c1502a98f5829c10120eb35d63", + url = "https://mirror.zml.ai/plugins/202610220840.276.1.f955d5e5c947/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "c45347542e35a289e49bd4682436a041491a992f6ed26747f6885827c0f527f8", - url = "https://mirror.zml.ai/plugins/202610191219.248.1.c2619532effe/zml-cpu-darwin-amd64.tar.zst", + sha256 = "bd4a67d94401022b1d5ee0677e427d6ffe5140fa57cddfafb8189a5743565662", + url = "https://mirror.zml.ai/plugins/202610220840.276.1.f955d5e5c947/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "b06e61dee9d796cccebb8f80abcd125d4ed39350820905400e8668e61590723b", - url = "https://mirror.zml.ai/plugins/202610191219.248.1.c2619532effe/zml-cpu-darwin-arm64.tar.zst", + sha256 = "6be93f4527ca7634904ca4eaa4c2a197f2d8f017f89c26cd59ffc8753ed5d248", + url = "https://mirror.zml.ai/plugins/202610220840.276.1.f955d5e5c947/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index a936c467..c34bb2b2 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -224,12 +224,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "b4a87415714f44c9adc2ce38bfad5ef42518195349a6a548a549cb2de2f0af2e", - "url": "https://mirror.zml.ai/plugins/202610191219.248.1.c2619532effe/zml-cuda-linux-amd64.tar.zst", + "sha256": "3f62507f5c9140624cf0c36ba15a5a768e9bd80a59e25de5dddcfef83a38b3c0", + "url": "https://mirror.zml.ai/plugins/202610220840.276.1.f955d5e5c947/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "2fccf6a376d9e735a6103acbfca1aaf6acf08791ed2b64dedece322cab8c5153", - "url": "https://mirror.zml.ai/plugins/202610191219.248.1.c2619532effe/zml-cuda-linux-arm64.tar.zst", + "sha256": "4ef15281d857e41ebc3b967444df377ed8cc91636f3f8e80f35331528b09fa95", + "url": "https://mirror.zml.ai/plugins/202610220840.276.1.f955d5e5c947/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index 2e665538..63d6870f 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202610191219.248.1.c2619532effe/zml-furiosa-linux-amd64.tar.zst", - sha256 = "3913a592fbc35a50ac60700011529addb9c18a51b614dfc5d1aa4f346b500712", + url = "https://mirror.zml.ai/plugins/202610220840.276.1.f955d5e5c947/zml-furiosa-linux-amd64.tar.zst", + sha256 = "0624dd1317047e3230c105baf118539ea5fc0ae4d0a6464db493acf493d09d8b", ) return mctx.extension_metadata( diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 7a937584..0a1802f5 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "e484f78bb14df98c4271f0d91760c32d167716b936ac465258fb7874282b3b2a", - url = "https://mirror.zml.ai/plugins/202610191219.248.1.c2619532effe/zml-metal-darwin-arm64.tar.zst", + sha256 = "114b63b4324c77a9bd932919e6a7e47e35f437de62a3d2343133b9c3d1623b29", + url = "https://mirror.zml.ai/plugins/202610220840.276.1.f955d5e5c947/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 7d4206be..8c19d254 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610191219.248.1.c2619532effe/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "777213f6cba1c8f0cdcd88d2ff12f9445e41cb498b1d382a3fb1bfad0fcf99d0" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610220840.276.1.f955d5e5c947/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "e2c2b3db78782c897976bdc43c46f05c93c0f61f65569579c55868adcafe15df" def _rocm_package_name(name): return name + _ROCM_VERSION diff --git a/zml/platform.zig b/zml/platform.zig index 3b52c08e..e10a47f2 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -733,7 +733,7 @@ pub const CreateOptions = struct { neuron: struct {} = .{}, furiosa: Furiosa = .{}, oneapi: XlaGpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.90 } } }, - metal: XlaGpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.90 } } }, + metal: XlaGpu = .{ .allocator = .{ .bfc = .{ .preallocate = false, .memory_fraction = 0.90 } } }, io_impl: Platform.IoImpl = .threaded, pub const Furiosa = struct { From 29b67dd10e1d943bf6fef47a5ecec1e27d51618f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20Gauti=C3=A9?= <51361266+tgautie@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:45:17 +0200 Subject: [PATCH 816/881] zml/manual_computation: translate output shapes to local before calling the callback (#980) We fix a bug introduced in https://github.com/zml/zml/pull/925: the output shapes passed to the local callback were left in their global state and not translated any more to the local state. --- zml/ops/manual_computation.zig | 98 +++++++++++++++++++++++++--------- 1 file changed, 74 insertions(+), 24 deletions(-) diff --git a/zml/ops/manual_computation.zig b/zml/ops/manual_computation.zig index b71a78de..49ebe199 100644 --- a/zml/ops/manual_computation.zig +++ b/zml/ops/manual_computation.zig @@ -57,24 +57,26 @@ pub fn manualComputation( const flat_input_tensors: []*Tensor = zml.meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); defer arena.free(flat_input_tensors); - const flat_output_shapes: []const Shape = zml.meta.collectValues(Shape, arena, &output_shapes) catch @panic("OOM"); + var owned_output_shapes: OutputShapes = output_shapes; + const flat_output_shapes: []*Shape = zml.meta.collectMutPtrs(Shape, arena, &owned_output_shapes) catch @panic("OOM"); + defer arena.free(flat_output_shapes); const WrappedInputs = struct { inputs: *const Inputs, - output_shapes: OutputShapes, + output_shapes: *const OutputShapes, fn wrap(allocator: std.mem.Allocator, opaque_wrapper: *const anyopaque, opaque_outputs: *anyopaque) error{OutOfMemory}![]*Tensor { const wrapped: *const @This() = @ptrCast(@alignCast(opaque_wrapper)); const outputs_ptr: *Outputs = @ptrCast(@alignCast(opaque_outputs)); - outputs_ptr.* = body_fn(wrapped.inputs.*, wrapped.output_shapes); + outputs_ptr.* = body_fn(wrapped.inputs.*, wrapped.output_shapes.*); return try zml.meta.collectMutPtrs(Tensor, allocator, outputs_ptr); } }; var outputs_struct: Outputs = undefined; - var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = output_shapes }; + var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &owned_output_shapes }; _ = manualComputationInternal( compiler, arena, @@ -101,7 +103,7 @@ noinline fn manualComputationInternal( input_struct: *anyopaque, input_tensors: []*Tensor, output_struct: *anyopaque, - output_shapes: []const Shape, + output_shapes: []*Shape, local_fn: LocalComputationFn, ) error{OutOfMemory}![]*Tensor { const global_scope = compiler.currentScope(); @@ -114,8 +116,10 @@ noinline fn manualComputationInternal( const manual_axes = collectManualAxes(mesh, input_tensors); log.debug("manualComputation({s}) will use mesh {s}, manual axes: {f}", .{ functor_name, mesh.name, stdx.fmt.stringsZ(manual_axes.constSlice()) }); - const input_shapes = try arena.alloc(Shape, input_tensors.len); - for (input_shapes, input_tensors) |*shape, tensor| shape.* = tensor._shape; + const global_input_shapes = try arena.alloc(Shape, input_tensors.len); + for (global_input_shapes, input_tensors) |*shape, tensor| shape.* = tensor._shape; + const global_output_shapes = try arena.alloc(Shape, output_shapes.len); + for (global_output_shapes, output_shapes) |*shape, shape_ptr| shape.* = shape_ptr.*; const input_types = try arena.alloc(*const mlir.Type, input_tensors.len); for (input_types, input_tensors) |*input_ty, tensor_ptr| { @@ -137,8 +141,7 @@ noinline fn manualComputationInternal( const local_scope = compiler.pushBlock(manual_block); defer local_scope.pop(); - for (0.., input_tensors) |i, tensor_ptr| { - const global = tensor_ptr._shape; + for (0.., input_tensors, global_input_shapes) |i, tensor_ptr, global| { const local = Sharding.shardedShape(global) catch unreachable; // we already computed this above tensor_ptr.* = switch (compiler.partitioner) { .shardy => Tensor._result(local, manual_block.argument(i)), @@ -146,18 +149,21 @@ noinline fn manualComputationInternal( }; } + for (output_shapes, global_output_shapes) |shape_ptr, global| { + shape_ptr.* = Sharding.shardedShape(global) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); + } + compiler.manual_computation_depth += 1; defer compiler.manual_computation_depth -= 1; const local_outputs = try local_fn(arena, input_struct, output_struct); - stdx.debug.assert(local_outputs.len == output_shapes.len, "manualComputation body returned {} values, expected {}", .{ local_outputs.len, output_shapes.len }); - for (0.., local_outputs, output_shapes) |i, local_tensor, global_shape| { - const expected_local = Sharding.shardedShape(global_shape) catch std.debug.panic("can't shard output {f} for {f}", .{ global_shape, mesh }); - stdx.debug.assert(expected_local.eql(local_tensor._shape), "manualComputation body output {d} expected {f}, got {f}", .{ i, expected_local, local_tensor._shape }); + stdx.debug.assert(local_outputs.len == global_output_shapes.len, "manualComputation body returned {} values, expected {}", .{ local_outputs.len, global_output_shapes.len }); + for (0.., local_outputs, output_shapes) |i, local_tensor, local_shape| { + stdx.debug.assert(local_shape.eql(local_tensor._shape), "manualComputation body output {d} expected {f}, got {f}", .{ i, local_shape.*, local_tensor._shape }); } - const global_output_types = try arena.alloc(*const mlir.Type, output_shapes.len); - for (output_shapes, 0..) |output_shape, i| { + const global_output_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); + for (global_output_shapes, 0..) |output_shape, i| { global_output_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output_shape); } @@ -175,8 +181,8 @@ noinline fn manualComputationInternal( .location = compiler.location, }).appendTo(manual_block); - const out_mesh_attrs = try arena.alloc(*const dialects.shardy.TensorShardingAttribute, output_shapes.len); - for (out_mesh_attrs, output_shapes) |*attr, shape| { + const out_mesh_attrs = try arena.alloc(*const dialects.shardy.TensorShardingAttribute, global_output_shapes.len); + for (out_mesh_attrs, global_output_shapes) |*attr, shape| { attr.* = try mesh.sdyShardingAttrForShape(arena, mlir_ctx, shape); } @@ -185,9 +191,9 @@ noinline fn manualComputationInternal( .results = .{ .flat = global_output_types }, .blocks = &.{manual_block}, .attributes = &.{ - .named(mlir_ctx, "in_shardings", try Sharding.sdyPerValueShardingAttr(arena, mlir_ctx, input_shapes, mesh)), + .named(mlir_ctx, "in_shardings", try Sharding.sdyPerValueShardingAttr(arena, mlir_ctx, global_input_shapes, mesh)), .named(mlir_ctx, "out_shardings", dialects.shardy.attr.shardingsPerValue(mlir_ctx, out_mesh_attrs)), - .named(mlir_ctx, "manual_axes", try Sharding.sdyManualAxesAttr(arena, mlir_ctx, input_shapes, output_shapes, mesh)), + .named(mlir_ctx, "manual_axes", try Sharding.sdyManualAxesAttr(arena, mlir_ctx, global_input_shapes, global_output_shapes, mesh)), }, .verify = true, .location = compiler.location, @@ -196,11 +202,11 @@ noinline fn manualComputationInternal( .gspmd => op: { // Gspmd calls SPMDShardToFullShape and wrap into an optimizationBarrier // Shortcut to avoid empty optimization barrier - if (output_shapes.len == 0) return &.{}; + if (global_output_shapes.len == 0) return &.{}; - const global_values = try arena.alloc(*const mlir.Value, output_shapes.len); - const global_types = try arena.alloc(*const mlir.Type, output_shapes.len); - for (output_shapes, 0..) |global_shape, i| { + const global_values = try arena.alloc(*const mlir.Value, global_output_shapes.len); + const global_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); + for (global_output_shapes, 0..) |global_shape, i| { global_values[i], global_types[i] = try gspmd.localToGlobal(compiler, arena, global_scope, local_output_values[i], global_shape); } @@ -214,7 +220,7 @@ noinline fn manualComputationInternal( }; // reuse local_outputs buffer to store the final result. - for (0.., local_outputs, output_shapes) |o, output_tensor_ptr, output_shape| { + for (0.., local_outputs, global_output_shapes) |o, output_tensor_ptr, output_shape| { output_tensor_ptr.* = Tensor._resultPropagateSharding(output_shape, op.result(o)); } return local_outputs; @@ -353,3 +359,47 @@ const shardy = struct { return dialects.shardy.ManualAxesAttribute.init(mlir_ctx, attrs.constSlice()).asAttr(); } }; + +test "manualComputation passes local output shapes to the callback" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + const mesh = zml.testing.tpMesh(); + const partitions = mesh.numPartitionsForLogicalAxis(.model); + if (partitions == 1) return error.SkipZigTest; + + var compiler: Compiler = .init(allocator, io, platform, .{ .meshes = &.{mesh} }); + defer compiler.deinit(); + compiler.activate(); + defer compiler.deactivate(); + + const block = mlir.Block.init(&.{}, &.{}); + defer block.deinit(); + const scope = compiler.pushBlock(block); + defer scope.pop(); + + const global_shape = Shape.init(.{ .b = 4, .h = 3 * partitions, .hd = 128 }, .bf16) + .withPartitioning(mesh, .{ .h = .model }); + const local_shape = Shape.init(.{ .b = 4, .h = 3, .hd = 128 }, .bf16); + const input = Tensor.zeroes(global_shape).withPartitioning(mesh, .{ .h = .model }); + var received_shape: ?Shape = null; + const Context = struct { + input: Tensor, + received_shape: *?Shape, + + fn body(self: @This(), output_shape: Shape) Tensor { + self.received_shape.* = output_shape; + return self.input; + } + }; + const output = manualComputation( + Context.body, + .{ .input = input, .received_shape = &received_shape }, + global_shape, + ); + + try std.testing.expect(received_shape != null); + try std.testing.expect(received_shape.?.eql(local_shape)); + try std.testing.expect(output.shape().eql(global_shape)); + try std.testing.expectEqualDeep(global_shape._sharding, output.shape()._sharding); +} From fcdef99e1ffdbf7d3f8ef1825b4da8b6cdda080c Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Thu, 22 Oct 2026 18:00:35 +0200 Subject: [PATCH 817/881] zml/kernels: remove clamp to CuTe DSL (#983) Seems like this is not properly lowered to CuTe DSL. Revert and investigate later why. --- kernels/cute/builder.zig | 50 ----------------------- zml/kernels/moe/cute/persistent_mxfp4.zig | 2 +- 2 files changed, 1 insertion(+), 51 deletions(-) diff --git a/kernels/cute/builder.zig b/kernels/cute/builder.zig index 61c9aed7..286b92aa 100644 --- a/kernels/cute/builder.zig +++ b/kernels/cute/builder.zig @@ -173,14 +173,6 @@ pub const Value = struct { return k.emit(if (l.isFloat()) arith.maximumf(k.ctx, l.inner, r.inner, k.loc()) else arith.maxsi(k.ctx, l.inner, r.inner, k.loc())); } - pub fn clamp(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - if (l.isFloat()) return k.emit(math.clampf(k.ctx, l.inner, r.neg().inner, r.inner, k.loc())); - const upper = k.emit(arith.minsi(k.ctx, l.inner, r.inner, k.loc())); - return k.emit(arith.maxsi(k.ctx, upper.inner, r.neg().inner, k.loc())); - } - /// Float max that ignores NaN operands (`arith.maxnumf`, a single `max.f32`), unlike /// the NaN-propagating `maximum`. pub fn maxnum(self: Value, rhs: anytype) Value { @@ -3146,48 +3138,6 @@ test "naive elementwise add matches the Python notebook kernel" { try std.testing.expect(parsed.operation().verify()); } -test "integer clamp emits both bounds" { - const ctx = try testContext(); - defer ctx.deinit(); - - var b = try Builder.open(std.testing.allocator, ctx, "integer_clamp"); - defer b.deinit(); - const a = try b.declareArgs(.{ .out = .{ .tensor = .{ .dtype = .i32, .shape = &.{8} } } }); - const tid = b.threadIdx().x; - a.out.set(.{tid}, tid.clamp(4)); - - const ir = try b.finish(.{ 8, 1, 1 }); - defer std.testing.allocator.free(ir); - try expectContains(ir, "arith.minsi"); - try expectContains(ir, "arith.maxsi"); - try expectContains(ir, "arith.subi"); - const parsed = try mlir.Module.parse(ctx, ir); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "float clamp emits math.clampf" { - const ctx = try testContext(); - defer ctx.deinit(); - - var b = try Builder.open(std.testing.allocator, ctx, "float_clamp"); - defer b.deinit(); - const a = try b.declareArgs(.{ - .input = .{ .tensor = .{ .dtype = .f32, .shape = &.{8} } }, - .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{8} } }, - }); - const tid = b.threadIdx().x; - a.out.set(.{tid}, a.input.get(.{tid}).clamp(4.0)); - - const ir = try b.finish(.{ 8, 1, 1 }); - defer std.testing.allocator.free(ir); - try expectContains(ir, "arith.negf"); - try expectContains(ir, "math.clampf"); - const parsed = try mlir.Module.parse(ctx, ir); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - test "guard, shared memory, sync, layouts, for loop" { const ctx = try testContext(); defer ctx.deinit(); diff --git a/zml/kernels/moe/cute/persistent_mxfp4.zig b/zml/kernels/moe/cute/persistent_mxfp4.zig index dc5732c4..0e18af45 100644 --- a/zml/kernels/moe/cute/persistent_mxfp4.zig +++ b/zml/kernels/moe/cute/persistent_mxfp4.zig @@ -1623,7 +1623,7 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo } var up_value = b.shuffleXor(value, 1).to(.bf16).to(.f32); if (operands.swiglu_limit) |limit| { - up_value = up_value.clamp(limit); + up_value = up_value.maximum(-limit).minimum(limit); } // up * SiLU(gate), with the hardware approximations Python's // fastmath selects: exp(-g) = 2^(-g * log2 e). From e81b0ecdc94ecd601179797a6f590e134c6580d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20Gauti=C3=A9?= <51361266+tgautie@users.noreply.github.com> Date: Thu, 22 Oct 2026 18:07:07 +0200 Subject: [PATCH 818/881] llmd/tpu/ragged_paged_attn: fix for sequences with empty kv (#938) Drafter scheduling in speculative decoding (https://github.com/zml/monorepo/pull/685) can create empty slots with empty kv content in full-attn layers of the drafter. This fixes this issue by aligning the TPU logic with other platforms (eg Triton) by avoiding to request DMA transfers of KV buffers for invalid slots. --- platforms/tpu/ragged_paged.zig | 55 ++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/platforms/tpu/ragged_paged.zig b/platforms/tpu/ragged_paged.zig index 2b1009a8..ea44e812 100644 --- a/platforms/tpu/ragged_paged.zig +++ b/platforms/tpu/ragged_paged.zig @@ -547,10 +547,18 @@ pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { { var i = k.openIf(k.cmpi(.eq, k.addi(a.heads_blk_idx, a.q_blk_idx), k.lift(@as(i32, 0)))); - createKvAsyncCopyDescriptors(k, cfg, a, a.heads_blk_idx, init_seq_idx, k.lift(@as(i32, 0)), init_buf_idx); + const first_seq_idx = nextNonemptyKvSequence(k, cfg, a, init_seq_idx, num_seqs); + { + var nonempty = k.openIf(k.cmpi(.slt, first_seq_idx, num_seqs)); + createKvAsyncCopyDescriptors(k, cfg, a, a.heads_blk_idx, first_seq_idx, k.lift(@as(i32, 0)), init_buf_idx); + nonempty.yieldThen(.{}); + } i.yieldThen(.{}); } + // Empty rows skip flashAttention, so their accumulator must already be zero. + k.vectorStoreAt(a.acc_ref, k.zeros(&.{ cfg.numQPerBlk(), cfg.numQHeadsPerBlk(), cfg.head_dim }, .f32), &.{ k.cIndex(0), k.cIndex(0), k.cIndex(0) }); + var q_loop = k.openWhile(.{ k.lift(@as(i32, 0)), init_seq_idx, init_buf_idx }, .{ k.scalarTy(.i32), k.scalarTy(.i32), k.scalarTy(.i32) }); { const done = q_loop.before_carried[0]; @@ -589,10 +597,20 @@ pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { const is_last_kv = k.cmpi(.sge, k.muli(next_kv_blk_raw, k.lift(@as(i32, @intCast(cfg.numKvPerBlk())))), kv_len); const next_kv_blk = k.select(is_last_kv, k.lift(@as(i32, 0)), next_kv_blk_raw); const cur_seq_end = k.cmpi(.sle, q_end, q_len_end); - const next_seq_a = k.select(cur_seq_end, k.addi(cur_seq_idx, k.lift(@as(i32, 1))), cur_seq_idx); - const next_seq_b = k.select(is_last_kv, next_seq_a, cur_seq_idx); - const is_last_seq = k.cmpi(.eq, next_seq_b, num_seqs); - const next_seq_idx = k.select(is_last_seq, k.lift(@as(i32, 0)), next_seq_b); + const next_nonempty_seq = blk: { + // Only sequence transitions need to scan for empty KV rows. + var next_seq = k.openIfElse(k.andi(is_last_kv, cur_seq_end), .{k.scalarTy(.i32)}); + { + const start_seq_idx = k.addi(cur_seq_idx, k.lift(@as(i32, 1))); + next_seq.yieldThen(.{nextNonemptyKvSequence(k, cfg, a, start_seq_idx, num_seqs)}); + } + { + next_seq.yieldElse(.{cur_seq_idx}); + } + break :blk next_seq.results[0]; + }; + const is_last_seq = k.cmpi(.eq, next_nonempty_seq, num_seqs); + const next_seq_idx = k.select(is_last_seq, k.lift(@as(i32, 0)), next_nonempty_seq); const next_heads_blk_idx = k.select(is_last_seq, k.addi(a.heads_blk_idx, k.lift(@as(i32, 1))), a.heads_blk_idx); const next_buf = k.extui(k.cmpi(.eq, cur_buf, k.lift(@as(i32, 0))), .i32); @@ -659,6 +677,33 @@ pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { k.vectorStoreAt(a.o, k.refLoad(a.acc_ref).to(cfg.q_dtype), &.{ k.cIndex(0), k.cIndex(0), k.cIndex(0) }); } +// Fixed drafter blocks can have query tokens without visible KV tokens. +// Keep their query positions, but never schedule a DMA for an empty KV row. +fn nextNonemptyKvSequence( + k: *mtt.Builder, + cfg: Cfg, + a: anytype, + start_seq_idx: mtt.Value, + num_seqs: mtt.Value, +) mtt.Value { + var loop = k.openWhile(.{start_seq_idx}, .{k.scalarTy(.i32)}); + { + const seq_idx = loop.before_carried[0]; + // The terminal index can equal the metadata array length. + const safe_seq_idx = k.minsi(seq_idx, k.lift(@as(i32, @intCast(cfg.max_num_seqs - 1)))); + const kv_len = k.scalarLoad(a.kv_lens, &.{k.toIndex(safe_seq_idx)}); + const skip = k.andi( + k.cmpi(.slt, seq_idx, num_seqs), + k.cmpi(.sle, kv_len, k.lift(@as(i32, 0))), + ); + loop.yieldBefore(skip, .{seq_idx}); + } + { + loop.yieldAfter(.{k.addi(loop.after_carried[0], k.lift(@as(i32, 1)))}); + } + return loop.results[0]; +} + /// Open a builder, run the kernel body, and return the canonicalized MLIR IR. /// Used by offline tooling (`examples/mosaic_ragged_paged`); production model /// code goes through `zml.kernel.mosaic_tpu.Kernel(...).emit(...)` instead. From 2e7871a80241ea5b1eeab800e3d84301555f5e86 Mon Sep 17 00:00:00 2001 From: Louis Lechevalier Date: Fri, 23 Oct 2026 15:30:55 +0200 Subject: [PATCH 819/881] zml/platforms: bump ZML plugins (#984) --- platforms/cpu/cpu.bzl | 6 +++--- platforms/cuda/cuda.bzl | 8 ++++---- platforms/furiosa/furiosa.bzl | 4 ++-- platforms/metal/metal.bzl | 4 ++-- platforms/rocm/rocm.bzl | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 0f4a1668..782c1c9f 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -43,21 +43,21 @@ def _cpu_plugin_impl(mctx): name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, sha256 = "309b737f996024b98b1ee18a44e37df1b328e7c1502a98f5829c10120eb35d63", - url = "https://mirror.zml.ai/plugins/202610220840.276.1.f955d5e5c947/zml-cpu-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610231235.289.1.bfc209c9e2ec/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "bd4a67d94401022b1d5ee0677e427d6ffe5140fa57cddfafb8189a5743565662", - url = "https://mirror.zml.ai/plugins/202610220840.276.1.f955d5e5c947/zml-cpu-darwin-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610231235.289.1.bfc209c9e2ec/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "6be93f4527ca7634904ca4eaa4c2a197f2d8f017f89c26cd59ffc8753ed5d248", - url = "https://mirror.zml.ai/plugins/202610220840.276.1.f955d5e5c947/zml-cpu-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610231235.289.1.bfc209c9e2ec/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index c34bb2b2..5bd2f667 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -224,12 +224,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "3f62507f5c9140624cf0c36ba15a5a768e9bd80a59e25de5dddcfef83a38b3c0", - "url": "https://mirror.zml.ai/plugins/202610220840.276.1.f955d5e5c947/zml-cuda-linux-amd64.tar.zst", + "sha256": "2528d0612a9503ada8b5a46dbe70e52ae0c21bce4ae11f92c9e3bf8f83045cff", + "url": "https://mirror.zml.ai/plugins/202610231235.289.1.bfc209c9e2ec/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "4ef15281d857e41ebc3b967444df377ed8cc91636f3f8e80f35331528b09fa95", - "url": "https://mirror.zml.ai/plugins/202610220840.276.1.f955d5e5c947/zml-cuda-linux-arm64.tar.zst", + "sha256": "9926b0a66a2e0c95d0c202f3cc82fb4321a6da2cfe903204f353df8e0209c288", + "url": "https://mirror.zml.ai/plugins/202610231235.289.1.bfc209c9e2ec/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index 63d6870f..c0d40593 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202610220840.276.1.f955d5e5c947/zml-furiosa-linux-amd64.tar.zst", - sha256 = "0624dd1317047e3230c105baf118539ea5fc0ae4d0a6464db493acf493d09d8b", + url = "https://mirror.zml.ai/plugins/202610231235.289.1.bfc209c9e2ec/zml-furiosa-linux-amd64.tar.zst", + sha256 = "f8d6bb03fb4cd5dad9b37856b2f8f12b28ec1a8d46f59ca03c401bea668215e9", ) return mctx.extension_metadata( diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 0a1802f5..9b9edef0 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "114b63b4324c77a9bd932919e6a7e47e35f437de62a3d2343133b9c3d1623b29", - url = "https://mirror.zml.ai/plugins/202610220840.276.1.f955d5e5c947/zml-metal-darwin-arm64.tar.zst", + sha256 = "39441a3c19a6d8400133461cca252e4397162f0a03d096457fade953facf151e", + url = "https://mirror.zml.ai/plugins/202610231235.289.1.bfc209c9e2ec/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 8c19d254..2182445c 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610220840.276.1.f955d5e5c947/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "e2c2b3db78782c897976bdc43c46f05c93c0f61f65569579c55868adcafe15df" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610231235.289.1.bfc209c9e2ec/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "f9c75c7b9b08a7022b8039d02ebf7bb635ed1a65335969f3a0b241d09b2170b8" def _rocm_package_name(name): return name + _ROCM_VERSION From 79bcc6af031e1b24b4a83368508bd7ca79e2a986 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Fri, 23 Oct 2026 17:08:20 +0200 Subject: [PATCH 820/881] workspace: bump plugins (#985) Update the metal plugin to improve performance under load --- platforms/cpu/cpu.bzl | 6 +++--- platforms/cuda/cuda.bzl | 4 ++-- platforms/furiosa/furiosa.bzl | 2 +- platforms/metal/metal.bzl | 4 ++-- platforms/rocm/rocm.bzl | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 782c1c9f..7e2e227c 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -43,21 +43,21 @@ def _cpu_plugin_impl(mctx): name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, sha256 = "309b737f996024b98b1ee18a44e37df1b328e7c1502a98f5829c10120eb35d63", - url = "https://mirror.zml.ai/plugins/202610231235.289.1.bfc209c9e2ec/zml-cpu-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610231457.291.1.799ad5c39a34/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "bd4a67d94401022b1d5ee0677e427d6ffe5140fa57cddfafb8189a5743565662", - url = "https://mirror.zml.ai/plugins/202610231235.289.1.bfc209c9e2ec/zml-cpu-darwin-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610231457.291.1.799ad5c39a34/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "6be93f4527ca7634904ca4eaa4c2a197f2d8f017f89c26cd59ffc8753ed5d248", - url = "https://mirror.zml.ai/plugins/202610231235.289.1.bfc209c9e2ec/zml-cpu-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610231457.291.1.799ad5c39a34/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 5bd2f667..38dc796c 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -225,11 +225,11 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { "sha256": "2528d0612a9503ada8b5a46dbe70e52ae0c21bce4ae11f92c9e3bf8f83045cff", - "url": "https://mirror.zml.ai/plugins/202610231235.289.1.bfc209c9e2ec/zml-cuda-linux-amd64.tar.zst", + "url": "https://mirror.zml.ai/plugins/202610231457.291.1.799ad5c39a34/zml-cuda-linux-amd64.tar.zst", }, "arm64": { "sha256": "9926b0a66a2e0c95d0c202f3cc82fb4321a6da2cfe903204f353df8e0209c288", - "url": "https://mirror.zml.ai/plugins/202610231235.289.1.bfc209c9e2ec/zml-cuda-linux-arm64.tar.zst", + "url": "https://mirror.zml.ai/plugins/202610231457.291.1.799ad5c39a34/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index c0d40593..40e711e7 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,7 +6,7 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202610231235.289.1.bfc209c9e2ec/zml-furiosa-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610231457.291.1.799ad5c39a34/zml-furiosa-linux-amd64.tar.zst", sha256 = "f8d6bb03fb4cd5dad9b37856b2f8f12b28ec1a8d46f59ca03c401bea668215e9", ) diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 9b9edef0..303b3843 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "39441a3c19a6d8400133461cca252e4397162f0a03d096457fade953facf151e", - url = "https://mirror.zml.ai/plugins/202610231235.289.1.bfc209c9e2ec/zml-metal-darwin-arm64.tar.zst", + sha256 = "1493fb9a9a2c1e217aa67914a982825cdc298fcb246d406e4e2ea217c66cbeec", + url = "https://mirror.zml.ai/plugins/202610231457.291.1.799ad5c39a34/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 2182445c..193ad367 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,7 +11,7 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610231235.289.1.bfc209c9e2ec/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610231457.291.1.799ad5c39a34/zml-rocm-linux-amd64.tar.zst" _PLUGIN_ROCM_SHA256 = "f9c75c7b9b08a7022b8039d02ebf7bb635ed1a65335969f3a0b241d09b2170b8" def _rocm_package_name(name): From 3f60e626e4d1e0caac2cd3815fba63514537e232 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Mon, 26 Oct 2026 11:41:05 +0100 Subject: [PATCH 821/881] mlir/dialects: split into one package per dialect and unify them (#982) Splits the monolithic mlir/dialects library into one Bazel package per dialect, then makes every package follow the same layout, C API and Zig API. Each package owns its Zig bindings, its C API when upstream doesn't provide one, and its tests. Dialect registration moves out of the core mlir package, so a target only pulls in the dialects it actually uses. The conventions are documented in `mlir/dialects/README.md`, with a porting checklist and conformance checks. Along the way: - cute_ir is flattened, and its Cuda, CuteNVGPU and NVVM sub-dialects become their own packages - fly's ROCDL extension moves to fly_rocdl - new packages for ROCDL and UB - kernels/ is renamed to dsl/, since it holds the DSLs - every package registers through its own registerDialect, and each DSL through registerDialects, instead of dialect name strings - the C and C++ follow the LLVM and MLIR C API conventions, formatted with a new mlir/.clang-format - BUILD files share one template - accessors are named as nouns like in the rest of ZML - each package tests itself in its own test.zig, including that every bound operation is registered and that every wrapper method compiles Fixes: - memref atomic_rmw emitted the wrong kind, because the AtomicRMWKind values didn't match the pinned LLVM - the pinned Triton and Mosaic no longer have tt.cat and tpu.repeat, so the DSL's cat and repeat now lower the way tl.cat and pltpu.repeat do - the ROCDL bindings are completed: buffer address spaces, vector types, waves_per_eu range Breaking changes: - register dialects through their package instead of DialectHandle.fromString, and mlir.registerFuncExtensions becomes func.registerExtensions - accessors drop the get prefix, so getValue() becomes value() - ttir.cat and mosaic_tpu.repeat are removed, and stablehlo pad and mosaic_tpu sort take their options before the location - zml.kernel.fly.rocdl becomes fly_rocdl - Mesh.sdyMeshAttr takes a context and returns an attribute instead of text - func.func no longer prints public, its default visibility --------- Co-authored-by: Steeve Morin --- docs/tutorials/write_first_model.md | 2 +- {kernels => dsl}/common/BUILD.bazel | 5 +- {kernels => dsl}/common/args.zig | 0 {kernels => dsl}/common/control_flow.zig | 22 +- {kernels => dsl}/common/dsl.zig | 0 {kernels => dsl}/cuda_tile/BUILD.bazel | 6 +- {kernels => dsl}/cuda_tile/README.md | 18 +- {kernels => dsl}/cuda_tile/builder.zig | 318 +- {kernels => dsl}/cuda_tile/control_flow.zig | 20 +- {kernels => dsl}/cuda_tile/dtype.zig | 0 dsl/cute/BUILD.bazel | 34 + {kernels => dsl}/cute/README.md | 42 +- {kernels => dsl}/cute/builder.zig | 1208 +-- {kernels => dsl}/cute/dtype.zig | 0 dsl/fly/BUILD.bazel | 35 + {kernels => dsl}/fly/PORTING.md | 12 +- {kernels => dsl}/fly/README.md | 19 +- {kernels => dsl}/fly/builder.zig | 135 +- {kernels => dsl}/fly/dtype.zig | 0 {kernels => dsl}/fly/layout.zig | 2 +- {kernels => dsl}/mosaic_tpu/BUILD.bazel | 15 +- {kernels => dsl}/mosaic_tpu/README.md | 6 +- {kernels => dsl}/mosaic_tpu/builder.zig | 203 +- {kernels => dsl}/mosaic_tpu/dtype.zig | 0 {kernels => dsl}/mosaic_tpu/pipeline.zig | 34 +- {kernels => dsl}/triton/BUILD.bazel | 13 +- {kernels => dsl}/triton/PORTING.md | 12 +- {kernels => dsl}/triton/README.md | 6 +- {kernels => dsl}/triton/builder.zig | 89 +- {kernels => dsl}/triton/dtype.zig | 0 kernels/cute/BUILD.bazel | 24 - kernels/fly/BUILD.bazel | 25 - mlir/.clang-format | 1 + mlir/BUILD.bazel | 16 +- mlir/dialects/BUILD.bazel | 70 - mlir/dialects/README.md | 262 + mlir/dialects/affine/BUILD.bazel | 35 + mlir/dialects/{ => affine}/affine.zig | 47 +- mlir/dialects/affine/capi.cc | 9 + mlir/dialects/affine/capi.h | 18 + mlir/dialects/affine/test.zig | 40 + mlir/dialects/arith/BUILD.bazel | 36 + mlir/dialects/{ => arith}/arith.zig | 47 +- mlir/dialects/arith/capi.cc | 25 + mlir/dialects/{arith_capi.h => arith/capi.h} | 13 +- mlir/dialects/arith/test.zig | 106 + mlir/dialects/arith_capi.cc | 20 - mlir/dialects/cf/BUILD.bazel | 23 + mlir/dialects/{ => cf}/cf.zig | 14 +- mlir/dialects/cf/test.zig | 31 + .../Dialect/Cuda/IR => cuda}/BUILD.bazel | 69 +- .../Dialect/Cuda/IR => cuda}/CudaAttrs.td | 12 +- .../Dialect/Cuda/IR => cuda}/CudaDialect.cpp | 22 +- .../Dialect/Cuda/IR => cuda}/CudaDialect.h | 18 +- .../Dialect/Cuda/IR => cuda}/CudaDialect.td | 8 +- .../Dialect/Cuda/IR => cuda}/CudaEnums.td | 8 +- .../Cuda/IR => cuda}/CudaInference.cpp | 4 +- .../Dialect/Cuda/IR => cuda}/CudaOps.td | 12 +- .../Dialect/Cuda/IR => cuda}/CudaTypes.td | 10 +- .../lib/CAPI/CudaTypes.cpp => cuda/capi.cc} | 253 +- mlir/dialects/cuda/capi.h | 253 + mlir/dialects/{ => cuda}/cuda.zig | 22 +- .../{cute_ir/cuda.zig => cuda/ops.zig} | 568 +- mlir/dialects/cuda/test.zig | 324 + mlir/dialects/cuda_tile/BUILD.bazel | 20 +- .../cuda_tile/{cuda_tile_capi.cc => capi.cc} | 58 +- mlir/dialects/cuda_tile/capi.h | 84 + mlir/dialects/cuda_tile/cuda_tile.zig | 510 +- mlir/dialects/cuda_tile/cuda_tile_capi.h | 79 - mlir/dialects/cuda_tile/test.zig | 391 + mlir/dialects/cute_ir/BUILD.bazel | 215 +- mlir/dialects/cute_ir/CuteAttrsAll.td | 14 + .../Dialect/Cute/IR => }/CuteAttrsCompiler.td | 6 +- .../cute_ir/Dialect/Cute/IR => }/CuteEnums.td | 6 +- .../{lib/Dialect/IR => }/CuteInference.cpp | 2168 +++-- .../Dialect/Cute/IR => }/CuteInference.h | 12 +- .../cute_ir/{lib/Dialect/IR => }/CuteOps.cpp | 4 +- .../cute_ir/Dialect/Cute/IR => }/CuteOps.td | 8 +- .../{lib/Dialect/IR => }/CuteTypes.cpp | 91 +- mlir/dialects/cute_ir/CuteTypesAll.td | 12 + .../Dialect/Cute/IR => }/CuteTypesCompiler.td | 10 +- mlir/dialects/cute_ir/README.md | 83 +- mlir/dialects/cute_ir/REGENERATING.md | 63 +- mlir/dialects/cute_ir/capi.cc | 1450 +++ mlir/dialects/cute_ir/capi.h | 491 + mlir/dialects/cute_ir/cute_ir.zig | 1261 +-- .../include/cute_ir-c/Dialect/BUILD.bazel | 47 - .../cute_ir/include/cute_ir-c/Dialect/Cuda.h | 22 - .../cute_ir-c/Dialect/CudaAttributes.h | 52 - .../include/cute_ir-c/Dialect/CudaTypes.h | 149 - .../cute_ir/include/cute_ir-c/Dialect/Cute.h | 24 - .../cute_ir-c/Dialect/CuteAttributes.h | 65 - .../Dialect/CuteCompilerAttributes.h | 225 - .../include/cute_ir-c/Dialect/CuteNVGPU.h | 22 - .../cute_ir-c/Dialect/CuteNVGPUAttributes.h | 480 - .../include/cute_ir-c/Dialect/CuteTypes.h | 185 - .../cute_ir/include/cute_ir-c/Dialect/NVVM.h | 21 - .../cute_ir-c/Dialect/NVVMAttributes.h | 509 - .../cute_ir/Dialect/Cute/IR/BUILD.bazel | 138 - .../cute_ir/Dialect/Cute/IR/CuteAttrsAll.td | 7 - .../cute_ir/Dialect/Cute/IR/CuteTypesAll.td | 5 - mlir/dialects/cute_ir/lib/CAPI/BUILD.bazel | 63 - mlir/dialects/cute_ir/lib/CAPI/Cuda.cpp | 10 - .../cute_ir/lib/CAPI/CudaAttributes.cpp | 136 - mlir/dialects/cute_ir/lib/CAPI/Cute.cpp | 17 - .../cute_ir/lib/CAPI/CuteAttributes.cpp | 130 - .../lib/CAPI/CuteCompilerAttributes.cpp | 672 -- mlir/dialects/cute_ir/lib/CAPI/CuteNVGPU.cpp | 19 - .../cute_ir/lib/CAPI/CuteNVGPUAttributes.cpp | 1183 --- mlir/dialects/cute_ir/lib/CAPI/CuteTypes.cpp | 508 - mlir/dialects/cute_ir/lib/CAPI/NVVM.cpp | 10 - .../cute_ir/lib/Dialect/Cuda/IR/BUILD.bazel | 18 - .../lib/Dialect/CuteNVGPU/IR/BUILD.bazel | 15 - .../cute_ir/lib/Dialect/IR/BUILD.bazel | 30 - .../cute_ir/lib/Dialect/NVVM/IR/BUILD.bazel | 18 - .../lib/Dialect/NVVM/IR/NVVMInference.cpp | 227 - mlir/dialects/cute_ir/test.zig | 332 + .../CuteNVGPU/IR => cute_nvgpu}/BUILD.bazel | 76 +- .../IR => cute_nvgpu}/CuteNVGPUAttrs.td | 12 +- .../IR => cute_nvgpu}/CuteNVGPUDialect.cpp | 1222 ++- .../IR => cute_nvgpu}/CuteNVGPUDialect.h | 25 +- .../IR => cute_nvgpu}/CuteNVGPUDialect.td | 6 +- .../IR => cute_nvgpu}/CuteNVGPUEnums.td | 8 +- .../IR => cute_nvgpu}/CuteNVGPUOps.td | 10 +- .../IR => cute_nvgpu}/CuteNVGPUTypes.td | 8 +- .../CuteNVGPUTypes.cpp => cute_nvgpu/capi.cc} | 1207 ++- .../CuteNVGPUTypes.h => cute_nvgpu/capi.h} | 499 +- .../{cute_ir => cute_nvgpu}/cute_nvgpu.zig | 1830 ++-- mlir/dialects/cute_nvgpu/test.zig | 210 + mlir/dialects/dialects.zig | 26 - mlir/dialects/fly/BUILD.bazel | 38 +- mlir/dialects/fly/REGENERATING.md | 53 +- mlir/dialects/fly/attributes.zig | 42 +- mlir/dialects/fly/capi.cc | 529 + mlir/dialects/fly/capi.h | 168 + mlir/dialects/fly/fly.zig | 261 +- mlir/dialects/fly/fly_capi.cc | 519 - mlir/dialects/fly/fly_capi.h | 143 - mlir/dialects/fly/ops.zig | 304 +- mlir/dialects/fly/rocdl_ops.zig | 69 - mlir/dialects/fly/test.zig | 378 + mlir/dialects/fly/types.zig | 118 +- mlir/dialects/fly_rocdl/BUILD.bazel | 43 + mlir/dialects/fly_rocdl/capi.cc | 107 + mlir/dialects/fly_rocdl/capi.h | 45 + .../rocdl.zig => fly_rocdl/fly_rocdl.zig} | 32 +- mlir/dialects/fly_rocdl/ops.zig | 83 + mlir/dialects/fly_rocdl/test.zig | 30 + mlir/dialects/func/BUILD.bazel | 35 + mlir/dialects/func/capi.cc | 10 + mlir/dialects/func/capi.h | 20 + mlir/dialects/{ => func}/func.zig | 12 +- mlir/dialects/func/test.zig | 41 + mlir/dialects/gpu/BUILD.bazel | 35 + mlir/dialects/gpu/capi.cc | 52 + mlir/dialects/{gpu_capi.h => gpu/capi.h} | 16 +- mlir/dialects/{ => gpu}/gpu.zig | 48 +- mlir/dialects/gpu/test.zig | 62 + mlir/dialects/gpu_capi.cc | 47 - mlir/dialects/llvm/BUILD.bazel | 24 + mlir/dialects/llvm/llvm.zig | 17 + .../{llvm_dialect.zig => llvm/ops.zig} | 71 +- mlir/dialects/llvm/test.zig | 44 + mlir/dialects/math/BUILD.bazel | 22 + mlir/dialects/{ => math}/math.zig | 10 +- mlir/dialects/math/test.zig | 73 + mlir/dialects/memref/BUILD.bazel | 23 + mlir/dialects/{ => memref}/memref.zig | 48 +- mlir/dialects/memref/test.zig | 77 + mlir/dialects/mosaic_tpu/BUILD.bazel | 19 +- .../{mosaic_tpu_capi.cc => capi.cc} | 174 +- mlir/dialects/mosaic_tpu/capi.h | 145 + mlir/dialects/mosaic_tpu/mosaic_tpu.zig | 374 +- mlir/dialects/mosaic_tpu/mosaic_tpu_capi.h | 105 - mlir/dialects/mosaic_tpu/test.zig | 269 + .../Dialect/NVVM/IR => nvvm}/BUILD.bazel | 67 +- .../Dialect/NVVM/IR => nvvm}/NVVMAttrs.td | 10 +- .../Dialect/NVVM/IR => nvvm}/NVVMDialect.cpp | 31 +- .../Dialect/NVVM/IR => nvvm}/NVVMDialect.h | 16 +- .../Dialect/NVVM/IR => nvvm}/NVVMDialect.td | 8 +- .../Dialect/NVVM/IR => nvvm}/NVVMEnums.td | 8 +- mlir/dialects/nvvm/NVVMInference.cpp | 253 + .../Dialect/NVVM/IR => nvvm}/NVVMOps.td | 10 +- .../CAPI/NVVMAttributes.cpp => nvvm/capi.cc} | 829 +- mlir/dialects/nvvm/capi.h | 785 ++ mlir/dialects/{ => nvvm}/nvvm.zig | 67 +- .../{cute_ir/nvvm.zig => nvvm/ops.zig} | 3967 +++++--- mlir/dialects/nvvm/test.zig | 472 + mlir/dialects/rocdl/BUILD.bazel | 46 + mlir/dialects/rocdl/README.md | 54 + mlir/dialects/rocdl/REGENERATING.md | 62 + mlir/dialects/rocdl/annotations.zig | 56 + mlir/dialects/rocdl/attributes.zig | 417 + mlir/dialects/rocdl/capi.cc | 440 + mlir/dialects/rocdl/capi.h | 140 + mlir/dialects/rocdl/enums.zig | 208 + mlir/dialects/rocdl/ops.zig | 8600 +++++++++++++++++ mlir/dialects/rocdl/rocdl.zig | 56 + mlir/dialects/rocdl/target.zig | 100 + mlir/dialects/rocdl/test.zig | 592 ++ mlir/dialects/rocdl/types.zig | 147 + mlir/dialects/scf/BUILD.bazel | 23 + mlir/dialects/{ => scf}/scf.zig | 20 +- mlir/dialects/scf/test.zig | 39 + mlir/dialects/shardy.zig | 459 - mlir/dialects/shardy/BUILD.bazel | 23 + mlir/dialects/shardy/shardy.zig | 312 + mlir/dialects/shardy/test.zig | 182 + mlir/dialects/stablehlo/BUILD.bazel | 11 +- mlir/dialects/stablehlo/stablehlo.zig | 352 +- mlir/dialects/stablehlo/test.zig | 102 + mlir/dialects/ttir/BUILD.bazel | 26 +- mlir/dialects/ttir/capi.cc | 167 + mlir/dialects/ttir/capi.h | 59 + mlir/dialects/ttir/test.zig | 163 + mlir/dialects/ttir/ttir.zig | 310 +- mlir/dialects/ttir/ttir_capi.cc | 115 - mlir/dialects/ttir/ttir_capi.h | 47 - mlir/dialects/ub/BUILD.bazel | 34 + mlir/dialects/ub/capi.cc | 8 + mlir/dialects/ub/capi.h | 18 + mlir/dialects/ub/test.zig | 23 + mlir/dialects/ub/ub.zig | 5 + mlir/dialects/vector/BUILD.bazel | 36 + mlir/dialects/vector/capi.cc | 51 + mlir/dialects/vector/capi.h | 36 + mlir/dialects/vector/test.zig | 86 + mlir/dialects/{ => vector}/vector.zig | 46 +- mlir/dialects/vector_capi.cc | 44 - mlir/dialects/vector_capi.h | 32 - mlir/mlir.h | 38 +- mlir/mlir.zig | 31 +- mlir/register.cc | 13 - platforms/tpu/BUILD.bazel | 2 +- platforms/tpu/ragged_paged.zig | 2 +- zml/BUILD.bazel | 31 +- zml/Compiler.zig | 56 +- zml/Sharding.zig | 56 +- zml/kernel.zig | 47 +- zml/kernels/moe/cute/persistent_mxfp4.zig | 206 +- zml/kernels/moe/fly/moe.zig | 2 +- zml/kernels/moe/mosaic_tpu/gmm_ep.zig | 2 +- zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig | 2 +- .../paged_attention/cute/sm10x/attention.zig | 2 +- .../paged_attention/cute/sm10x/sm103.zig | 7 +- .../paged_attention/cute/sm120/attention.zig | 12 +- .../triton/unified_attention.zig | 3 +- .../triton/unified_attention_oneapi.zig | 3 +- zml/ops.zig | 125 +- zml/ops/manual_computation.zig | 25 +- zml/pjrtx.zig | 8 +- zml/tensor.zig | 176 +- 252 files changed, 32484 insertions(+), 15685 deletions(-) rename {kernels => dsl}/common/BUILD.bazel (79%) rename {kernels => dsl}/common/args.zig (100%) rename {kernels => dsl}/common/control_flow.zig (92%) rename {kernels => dsl}/common/dsl.zig (100%) rename {kernels => dsl}/cuda_tile/BUILD.bazel (85%) rename {kernels => dsl}/cuda_tile/README.md (88%) rename {kernels => dsl}/cuda_tile/builder.zig (83%) rename {kernels => dsl}/cuda_tile/control_flow.zig (85%) rename {kernels => dsl}/cuda_tile/dtype.zig (100%) create mode 100644 dsl/cute/BUILD.bazel rename {kernels => dsl}/cute/README.md (85%) rename {kernels => dsl}/cute/builder.zig (72%) rename {kernels => dsl}/cute/dtype.zig (100%) create mode 100644 dsl/fly/BUILD.bazel rename {kernels => dsl}/fly/PORTING.md (93%) rename {kernels => dsl}/fly/README.md (93%) rename {kernels => dsl}/fly/builder.zig (94%) rename {kernels => dsl}/fly/dtype.zig (100%) rename {kernels => dsl}/fly/layout.zig (99%) rename {kernels => dsl}/mosaic_tpu/BUILD.bazel (56%) rename {kernels => dsl}/mosaic_tpu/README.md (99%) rename {kernels => dsl}/mosaic_tpu/builder.zig (93%) rename {kernels => dsl}/mosaic_tpu/dtype.zig (100%) rename {kernels => dsl}/mosaic_tpu/pipeline.zig (97%) rename {kernels => dsl}/triton/BUILD.bazel (60%) rename {kernels => dsl}/triton/PORTING.md (98%) rename {kernels => dsl}/triton/README.md (99%) rename {kernels => dsl}/triton/builder.zig (97%) rename {kernels => dsl}/triton/dtype.zig (100%) delete mode 100644 kernels/cute/BUILD.bazel delete mode 100644 kernels/fly/BUILD.bazel create mode 100644 mlir/.clang-format delete mode 100644 mlir/dialects/BUILD.bazel create mode 100644 mlir/dialects/README.md create mode 100644 mlir/dialects/affine/BUILD.bazel rename mlir/dialects/{ => affine}/affine.zig (93%) create mode 100644 mlir/dialects/affine/capi.cc create mode 100644 mlir/dialects/affine/capi.h create mode 100644 mlir/dialects/affine/test.zig create mode 100644 mlir/dialects/arith/BUILD.bazel rename mlir/dialects/{ => arith}/arith.zig (91%) create mode 100644 mlir/dialects/arith/capi.cc rename mlir/dialects/{arith_capi.h => arith/capi.h} (66%) create mode 100644 mlir/dialects/arith/test.zig delete mode 100644 mlir/dialects/arith_capi.cc create mode 100644 mlir/dialects/cf/BUILD.bazel rename mlir/dialects/{ => cf}/cf.zig (96%) create mode 100644 mlir/dialects/cf/test.zig rename mlir/dialects/{cute_ir/include/cute_ir/Dialect/Cuda/IR => cuda}/BUILD.bazel (73%) rename mlir/dialects/{cute_ir/include/cute_ir/Dialect/Cuda/IR => cuda}/CudaAttrs.td (89%) rename mlir/dialects/{cute_ir/lib/Dialect/Cuda/IR => cuda}/CudaDialect.cpp (51%) rename mlir/dialects/{cute_ir/include/cute_ir/Dialect/Cuda/IR => cuda}/CudaDialect.h (58%) rename mlir/dialects/{cute_ir/include/cute_ir/Dialect/Cuda/IR => cuda}/CudaDialect.td (81%) rename mlir/dialects/{cute_ir/include/cute_ir/Dialect/Cuda/IR => cuda}/CudaEnums.td (99%) rename mlir/dialects/{cute_ir/lib/Dialect/Cuda/IR => cuda}/CudaInference.cpp (90%) rename mlir/dialects/{cute_ir/include/cute_ir/Dialect/Cuda/IR => cuda}/CudaOps.td (99%) rename mlir/dialects/{cute_ir/include/cute_ir/Dialect/Cuda/IR => cuda}/CudaTypes.td (97%) rename mlir/dialects/{cute_ir/lib/CAPI/CudaTypes.cpp => cuda/capi.cc} (56%) create mode 100644 mlir/dialects/cuda/capi.h rename mlir/dialects/{ => cuda}/cuda.zig (87%) rename mlir/dialects/{cute_ir/cuda.zig => cuda/ops.zig} (88%) create mode 100644 mlir/dialects/cuda/test.zig rename mlir/dialects/cuda_tile/{cuda_tile_capi.cc => capi.cc} (85%) create mode 100644 mlir/dialects/cuda_tile/capi.h delete mode 100644 mlir/dialects/cuda_tile/cuda_tile_capi.h create mode 100644 mlir/dialects/cuda_tile/test.zig create mode 100644 mlir/dialects/cute_ir/CuteAttrsAll.td rename mlir/dialects/cute_ir/{include/cute_ir/Dialect/Cute/IR => }/CuteAttrsCompiler.td (98%) rename mlir/dialects/cute_ir/{include/cute_ir/Dialect/Cute/IR => }/CuteEnums.td (94%) rename mlir/dialects/cute_ir/{lib/Dialect/IR => }/CuteInference.cpp (60%) rename mlir/dialects/cute_ir/{include/cute_ir/Dialect/Cute/IR => }/CuteInference.h (71%) rename mlir/dialects/cute_ir/{lib/Dialect/IR => }/CuteOps.cpp (69%) rename mlir/dialects/cute_ir/{include/cute_ir/Dialect/Cute/IR => }/CuteOps.td (99%) rename mlir/dialects/cute_ir/{lib/Dialect/IR => }/CuteTypes.cpp (96%) create mode 100644 mlir/dialects/cute_ir/CuteTypesAll.td rename mlir/dialects/cute_ir/{include/cute_ir/Dialect/Cute/IR => }/CuteTypesCompiler.td (97%) create mode 100644 mlir/dialects/cute_ir/capi.cc create mode 100644 mlir/dialects/cute_ir/capi.h delete mode 100644 mlir/dialects/cute_ir/include/cute_ir-c/Dialect/BUILD.bazel delete mode 100644 mlir/dialects/cute_ir/include/cute_ir-c/Dialect/Cuda.h delete mode 100644 mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CudaAttributes.h delete mode 100644 mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CudaTypes.h delete mode 100644 mlir/dialects/cute_ir/include/cute_ir-c/Dialect/Cute.h delete mode 100644 mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteAttributes.h delete mode 100644 mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteCompilerAttributes.h delete mode 100644 mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPU.h delete mode 100644 mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPUAttributes.h delete mode 100644 mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteTypes.h delete mode 100644 mlir/dialects/cute_ir/include/cute_ir-c/Dialect/NVVM.h delete mode 100644 mlir/dialects/cute_ir/include/cute_ir-c/Dialect/NVVMAttributes.h delete mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/BUILD.bazel delete mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteAttrsAll.td delete mode 100644 mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesAll.td delete mode 100644 mlir/dialects/cute_ir/lib/CAPI/BUILD.bazel delete mode 100644 mlir/dialects/cute_ir/lib/CAPI/Cuda.cpp delete mode 100644 mlir/dialects/cute_ir/lib/CAPI/CudaAttributes.cpp delete mode 100644 mlir/dialects/cute_ir/lib/CAPI/Cute.cpp delete mode 100644 mlir/dialects/cute_ir/lib/CAPI/CuteAttributes.cpp delete mode 100644 mlir/dialects/cute_ir/lib/CAPI/CuteCompilerAttributes.cpp delete mode 100644 mlir/dialects/cute_ir/lib/CAPI/CuteNVGPU.cpp delete mode 100644 mlir/dialects/cute_ir/lib/CAPI/CuteNVGPUAttributes.cpp delete mode 100644 mlir/dialects/cute_ir/lib/CAPI/CuteTypes.cpp delete mode 100644 mlir/dialects/cute_ir/lib/CAPI/NVVM.cpp delete mode 100644 mlir/dialects/cute_ir/lib/Dialect/Cuda/IR/BUILD.bazel delete mode 100644 mlir/dialects/cute_ir/lib/Dialect/CuteNVGPU/IR/BUILD.bazel delete mode 100644 mlir/dialects/cute_ir/lib/Dialect/IR/BUILD.bazel delete mode 100644 mlir/dialects/cute_ir/lib/Dialect/NVVM/IR/BUILD.bazel delete mode 100644 mlir/dialects/cute_ir/lib/Dialect/NVVM/IR/NVVMInference.cpp create mode 100644 mlir/dialects/cute_ir/test.zig rename mlir/dialects/{cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR => cute_nvgpu}/BUILD.bazel (67%) rename mlir/dialects/{cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR => cute_nvgpu}/CuteNVGPUAttrs.td (93%) rename mlir/dialects/{cute_ir/lib/Dialect/CuteNVGPU/IR => cute_nvgpu}/CuteNVGPUDialect.cpp (63%) rename mlir/dialects/{cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR => cute_nvgpu}/CuteNVGPUDialect.h (59%) rename mlir/dialects/{cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR => cute_nvgpu}/CuteNVGPUDialect.td (86%) rename mlir/dialects/{cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR => cute_nvgpu}/CuteNVGPUEnums.td (99%) rename mlir/dialects/{cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR => cute_nvgpu}/CuteNVGPUOps.td (99%) rename mlir/dialects/{cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR => cute_nvgpu}/CuteNVGPUTypes.td (99%) rename mlir/dialects/{cute_ir/lib/CAPI/CuteNVGPUTypes.cpp => cute_nvgpu/capi.cc} (74%) rename mlir/dialects/{cute_ir/include/cute_ir-c/Dialect/CuteNVGPUTypes.h => cute_nvgpu/capi.h} (68%) rename mlir/dialects/{cute_ir => cute_nvgpu}/cute_nvgpu.zig (76%) create mode 100644 mlir/dialects/cute_nvgpu/test.zig delete mode 100644 mlir/dialects/dialects.zig create mode 100644 mlir/dialects/fly/capi.cc create mode 100644 mlir/dialects/fly/capi.h delete mode 100644 mlir/dialects/fly/fly_capi.cc delete mode 100644 mlir/dialects/fly/fly_capi.h delete mode 100644 mlir/dialects/fly/rocdl_ops.zig create mode 100644 mlir/dialects/fly/test.zig create mode 100644 mlir/dialects/fly_rocdl/BUILD.bazel create mode 100644 mlir/dialects/fly_rocdl/capi.cc create mode 100644 mlir/dialects/fly_rocdl/capi.h rename mlir/dialects/{fly/rocdl.zig => fly_rocdl/fly_rocdl.zig} (84%) create mode 100644 mlir/dialects/fly_rocdl/ops.zig create mode 100644 mlir/dialects/fly_rocdl/test.zig create mode 100644 mlir/dialects/func/BUILD.bazel create mode 100644 mlir/dialects/func/capi.cc create mode 100644 mlir/dialects/func/capi.h rename mlir/dialects/{ => func}/func.zig (92%) create mode 100644 mlir/dialects/func/test.zig create mode 100644 mlir/dialects/gpu/BUILD.bazel create mode 100644 mlir/dialects/gpu/capi.cc rename mlir/dialects/{gpu_capi.h => gpu/capi.h} (72%) rename mlir/dialects/{ => gpu}/gpu.zig (78%) create mode 100644 mlir/dialects/gpu/test.zig delete mode 100644 mlir/dialects/gpu_capi.cc create mode 100644 mlir/dialects/llvm/BUILD.bazel create mode 100644 mlir/dialects/llvm/llvm.zig rename mlir/dialects/{llvm_dialect.zig => llvm/ops.zig} (73%) create mode 100644 mlir/dialects/llvm/test.zig create mode 100644 mlir/dialects/math/BUILD.bazel rename mlir/dialects/{ => math}/math.zig (98%) create mode 100644 mlir/dialects/math/test.zig create mode 100644 mlir/dialects/memref/BUILD.bazel rename mlir/dialects/{ => memref}/memref.zig (97%) create mode 100644 mlir/dialects/memref/test.zig rename mlir/dialects/mosaic_tpu/{mosaic_tpu_capi.cc => capi.cc} (56%) create mode 100644 mlir/dialects/mosaic_tpu/capi.h delete mode 100644 mlir/dialects/mosaic_tpu/mosaic_tpu_capi.h create mode 100644 mlir/dialects/mosaic_tpu/test.zig rename mlir/dialects/{cute_ir/include/cute_ir/Dialect/NVVM/IR => nvvm}/BUILD.bazel (70%) rename mlir/dialects/{cute_ir/include/cute_ir/Dialect/NVVM/IR => nvvm}/NVVMAttrs.td (89%) rename mlir/dialects/{cute_ir/lib/Dialect/NVVM/IR => nvvm}/NVVMDialect.cpp (56%) rename mlir/dialects/{cute_ir/include/cute_ir/Dialect/NVVM/IR => nvvm}/NVVMDialect.h (60%) rename mlir/dialects/{cute_ir/include/cute_ir/Dialect/NVVM/IR => nvvm}/NVVMDialect.td (80%) rename mlir/dialects/{cute_ir/include/cute_ir/Dialect/NVVM/IR => nvvm}/NVVMEnums.td (99%) create mode 100644 mlir/dialects/nvvm/NVVMInference.cpp rename mlir/dialects/{cute_ir/include/cute_ir/Dialect/NVVM/IR => nvvm}/NVVMOps.td (99%) rename mlir/dialects/{cute_ir/lib/CAPI/NVVMAttributes.cpp => nvvm/capi.cc} (67%) create mode 100644 mlir/dialects/nvvm/capi.h rename mlir/dialects/{ => nvvm}/nvvm.zig (84%) rename mlir/dialects/{cute_ir/nvvm.zig => nvvm/ops.zig} (65%) create mode 100644 mlir/dialects/nvvm/test.zig create mode 100644 mlir/dialects/rocdl/BUILD.bazel create mode 100644 mlir/dialects/rocdl/README.md create mode 100644 mlir/dialects/rocdl/REGENERATING.md create mode 100644 mlir/dialects/rocdl/annotations.zig create mode 100644 mlir/dialects/rocdl/attributes.zig create mode 100644 mlir/dialects/rocdl/capi.cc create mode 100644 mlir/dialects/rocdl/capi.h create mode 100644 mlir/dialects/rocdl/enums.zig create mode 100644 mlir/dialects/rocdl/ops.zig create mode 100644 mlir/dialects/rocdl/rocdl.zig create mode 100644 mlir/dialects/rocdl/target.zig create mode 100644 mlir/dialects/rocdl/test.zig create mode 100644 mlir/dialects/rocdl/types.zig create mode 100644 mlir/dialects/scf/BUILD.bazel rename mlir/dialects/{ => scf}/scf.zig (97%) create mode 100644 mlir/dialects/scf/test.zig delete mode 100644 mlir/dialects/shardy.zig create mode 100644 mlir/dialects/shardy/BUILD.bazel create mode 100644 mlir/dialects/shardy/shardy.zig create mode 100644 mlir/dialects/shardy/test.zig create mode 100644 mlir/dialects/stablehlo/test.zig create mode 100644 mlir/dialects/ttir/capi.cc create mode 100644 mlir/dialects/ttir/capi.h create mode 100644 mlir/dialects/ttir/test.zig delete mode 100644 mlir/dialects/ttir/ttir_capi.cc delete mode 100644 mlir/dialects/ttir/ttir_capi.h create mode 100644 mlir/dialects/ub/BUILD.bazel create mode 100644 mlir/dialects/ub/capi.cc create mode 100644 mlir/dialects/ub/capi.h create mode 100644 mlir/dialects/ub/test.zig create mode 100644 mlir/dialects/ub/ub.zig create mode 100644 mlir/dialects/vector/BUILD.bazel create mode 100644 mlir/dialects/vector/capi.cc create mode 100644 mlir/dialects/vector/capi.h create mode 100644 mlir/dialects/vector/test.zig rename mlir/dialects/{ => vector}/vector.zig (94%) delete mode 100644 mlir/dialects/vector_capi.cc delete mode 100644 mlir/dialects/vector_capi.h delete mode 100644 mlir/register.cc diff --git a/docs/tutorials/write_first_model.md b/docs/tutorials/write_first_model.md index da9113a9..312c9afc 100644 --- a/docs/tutorials/write_first_model.md +++ b/docs/tutorials/write_first_model.md @@ -318,7 +318,7 @@ debug(zml/module): ******** ZML generated MLIR ******** module @zml attributes {mhlo.num_partitions = 4 : i32, mhlo.num_replicas = 1 : i32} { sdy.mesh @replicated = <["bus"=4]> - func.func public @main(%arg0: tensor<3xf16> {sdy.sharding = #sdy.sharding<@replicated, [{}], replicated={"bus"}>}, %arg1: tensor<3xf16> {sdy.sharding = #sdy.sharding<@replicated, [{}], replicated={"bus"}>}, %arg2: tensor<3xf16> {sdy.sharding = #sdy.sharding<@replicated, [{}], replicated={"bus"}>}) -> (tensor<3xf16> {sdy.sharding = #sdy.sharding<@replicated, [{}], replicated={"bus"}>}) { + func.func @main(%arg0: tensor<3xf16> {sdy.sharding = #sdy.sharding<@replicated, [{}], replicated={"bus"}>}, %arg1: tensor<3xf16> {sdy.sharding = #sdy.sharding<@replicated, [{}], replicated={"bus"}>}, %arg2: tensor<3xf16> {sdy.sharding = #sdy.sharding<@replicated, [{}], replicated={"bus"}>}) -> (tensor<3xf16> {sdy.sharding = #sdy.sharding<@replicated, [{}], replicated={"bus"}>}) { %0 = stablehlo.multiply %arg1, %arg2 : tensor<3xf16> %1 = stablehlo.add %0, %arg0 : tensor<3xf16> return %1 : tensor<3xf16> diff --git a/kernels/common/BUILD.bazel b/dsl/common/BUILD.bazel similarity index 79% rename from kernels/common/BUILD.bazel rename to dsl/common/BUILD.bazel index 11151f04..9631c34e 100644 --- a/kernels/common/BUILD.bazel +++ b/dsl/common/BUILD.bazel @@ -6,13 +6,12 @@ zig_library( "args.zig", "control_flow.zig", ], - import_name = "kernels/common", + import_name = "dsl/common", main = "dsl.zig", visibility = ["//visibility:public"], deps = [ "//mlir", - "//mlir/dialects", - "//stdx", + "//mlir/dialects/scf", ], ) diff --git a/kernels/common/args.zig b/dsl/common/args.zig similarity index 100% rename from kernels/common/args.zig rename to dsl/common/args.zig diff --git a/kernels/common/control_flow.zig b/dsl/common/control_flow.zig similarity index 92% rename from kernels/common/control_flow.zig rename to dsl/common/control_flow.zig index afd79e8f..b57c6ece 100644 --- a/kernels/common/control_flow.zig +++ b/dsl/common/control_flow.zig @@ -5,9 +5,7 @@ const std = @import("std"); const mlir = @import("mlir"); -const dialects = @import("mlir/dialects"); - -const scf = dialects.scf; +const scf = @import("mlir/dialects/scf"); pub fn tupleArity(comptime T: type, comptime what: []const u8) comptime_int { const info = @typeInfo(T); @@ -85,14 +83,7 @@ pub fn IfOnlyScope(comptime BuilderT: type, comptime ValueT: type) type { const k = self.kernel; emitScfYield(ValueT, k, self.then_block, 0, values, "IfOnlyScope.yieldThen"); k.popBlock(); - const if_op = scf.if_( - k.ctx, - self.cond_inner, - &.{}, - self.then_block, - null, - k.loc(), - ); + const if_op = scf.if_(k.ctx, self.cond_inner, &.{}, self.then_block, .{}, k.loc()); _ = if_op.appendTo(k.currentBlock()); } }; @@ -120,14 +111,7 @@ pub fn IfScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize const k = self.kernel; emitScfYield(ValueT, k, self.else_block, N, values, "IfScope.yieldElse"); k.popBlock(); - const if_op = scf.if_( - k.ctx, - self.cond_inner, - &self.result_types, - self.then_block, - self.else_block, - k.loc(), - ); + const if_op = scf.if_(k.ctx, self.cond_inner, &self.result_types, self.then_block, .{ .else_block = self.else_block }, k.loc()); _ = if_op.appendTo(k.currentBlock()); for (0..N) |i| self.results[i] = .{ .inner = if_op.result(i), .kernel = k }; } diff --git a/kernels/common/dsl.zig b/dsl/common/dsl.zig similarity index 100% rename from kernels/common/dsl.zig rename to dsl/common/dsl.zig diff --git a/kernels/cuda_tile/BUILD.bazel b/dsl/cuda_tile/BUILD.bazel similarity index 85% rename from kernels/cuda_tile/BUILD.bazel rename to dsl/cuda_tile/BUILD.bazel index 03208310..20ed77ad 100644 --- a/kernels/cuda_tile/BUILD.bazel +++ b/dsl/cuda_tile/BUILD.bazel @@ -2,16 +2,16 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "cuda_tile_builder", - tags = ["manual"], srcs = [ "control_flow.zig", "dtype.zig", ], - import_name = "kernels/cuda_tile/builder", + import_name = "dsl/cuda_tile/builder", main = "builder.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ - "//kernels/common", + "//dsl/common", "//mlir", "//mlir/dialects/cuda_tile", "//stdx", diff --git a/kernels/cuda_tile/README.md b/dsl/cuda_tile/README.md similarity index 88% rename from kernels/cuda_tile/README.md rename to dsl/cuda_tile/README.md index aac90f2f..ef25428d 100644 --- a/kernels/cuda_tile/README.md +++ b/dsl/cuda_tile/README.md @@ -1,8 +1,8 @@ # CUDA Tile IR kernels in Zig -`kernels/cuda_tile` is a Zig DSL that emits NVIDIA's `cuda_tile` MLIR dialect -(CUDA Tile IR). It is the third instance of the `kernels/triton` / -`kernels/mosaic_tpu` pattern: a `Builder` that owns an MLIR module and appends +`dsl/cuda_tile` is a Zig DSL that emits NVIDIA's `cuda_tile` MLIR dialect +(CUDA Tile IR). It is the third instance of the `dsl/triton` / +`dsl/mosaic_tpu` pattern: a `Builder` that owns an MLIR module and appends ops to a block stack, a `Value` with fluent arithmetic, and a `zml.kernel.cuda_tile.Kernel` wrapper that turns a `run` function into a `stablehlo.custom_call` XLA compiles with `tileiras`. @@ -110,23 +110,23 @@ grep -n '^pub fn ' mlir/dialects/cuda_tile/cuda_tile.zig grep -n '^pub const ' mlir/dialects/cuda_tile/cuda_tile.zig # the DSL: Value's fluent methods (add, mul, lt, to, sum, reshape, ...) -awk '/^pub const Value = struct/,/^};/' kernels/cuda_tile/builder.zig | grep 'pub fn ' +awk '/^pub const Value = struct/,/^};/' dsl/cuda_tile/builder.zig | grep 'pub fn ' # the DSL: everything on Builder (views, load/store, math, casts, reductions, # atomics, control flow) -- the `*Opts` variant of a call takes the options -awk '/^pub const Builder = struct/,0' kernels/cuda_tile/builder.zig | grep ' pub fn ' +awk '/^pub const Builder = struct/,0' dsl/cuda_tile/builder.zig | grep ' pub fn ' # what each option struct holds, and its defaults -grep -n '^pub const [A-Za-z]*Opts = struct' -A 8 kernels/cuda_tile/builder.zig +grep -n '^pub const [A-Za-z]*Opts = struct' -A 8 dsl/cuda_tile/builder.zig # for / if / loop scopes and their terminators -grep -n 'pub fn ' kernels/cuda_tile/control_flow.zig +grep -n 'pub fn ' dsl/cuda_tile/control_flow.zig # element types -grep -n 'pub const DType = enum' -A 30 kernels/cuda_tile/dtype.zig +grep -n 'pub const DType = enum' -A 30 dsl/cuda_tile/dtype.zig ``` -Every op also has a test: `grep -n '^test "' kernels/cuda_tile/builder.zig` +Every op also has a test: `grep -n '^test "' dsl/cuda_tile/builder.zig` lists them, and each one prints IR that round-trips through the verifier, so they double as usage examples. A worked, runnable example (saxpy, softmax, gemm, in-place relu) lives in `examples/cuda_tile`. diff --git a/kernels/cuda_tile/builder.zig b/dsl/cuda_tile/builder.zig similarity index 83% rename from kernels/cuda_tile/builder.zig rename to dsl/cuda_tile/builder.zig index 6067ed4b..f534a9c8 100644 --- a/kernels/cuda_tile/builder.zig +++ b/dsl/cuda_tile/builder.zig @@ -1,7 +1,7 @@ const std = @import("std"); -const ct = @import("mlir/dialects/cuda_tile"); -const dsl = @import("kernels/common"); +const cuda_tile = @import("mlir/dialects/cuda_tile"); +const dsl = @import("dsl/common"); const tupleArity = dsl.tupleArity; const mlir = @import("mlir"); const stdx = @import("stdx"); @@ -13,19 +13,19 @@ const isFloatDtype = dtypes.isFloatDtype; const dtypeBitwidth = dtypes.dtypeBitwidth; const intBitwidth = dtypes.intBitwidth; -pub const RoundingMode = ct.RoundingMode; -pub const Signedness = ct.Signedness; -pub const IntegerOverflow = ct.IntegerOverflow; -pub const ComparisonPredicate = ct.ComparisonPredicate; -pub const ComparisonOrdering = ct.ComparisonOrdering; -pub const AtomicRMWMode = ct.AtomicRMWMode; -pub const MemoryScope = ct.MemoryScope; -pub const MemoryOrdering = ct.MemoryOrdering; -pub const PaddingValue = ct.PaddingValue; -pub const Arch = ct.Arch; -pub const EntryHint = ct.EntryHint; -pub const LoadStoreHint = ct.LoadStoreHint; -pub const PtrLoadStoreHint = ct.PtrLoadStoreHint; +pub const RoundingMode = cuda_tile.RoundingMode; +pub const Signedness = cuda_tile.Signedness; +pub const IntegerOverflow = cuda_tile.IntegerOverflow; +pub const ComparisonPredicate = cuda_tile.ComparisonPredicate; +pub const ComparisonOrdering = cuda_tile.ComparisonOrdering; +pub const AtomicRMWMode = cuda_tile.AtomicRMWMode; +pub const MemoryScope = cuda_tile.MemoryScope; +pub const MemoryOrdering = cuda_tile.MemoryOrdering; +pub const PaddingValue = cuda_tile.PaddingValue; +pub const Arch = cuda_tile.Arch; +pub const EntryHint = cuda_tile.EntryHint; +pub const LoadStoreHint = cuda_tile.LoadStoreHint; +pub const PtrLoadStoreHint = cuda_tile.PtrLoadStoreHint; test { std.testing.refAllDecls(@This()); @@ -33,7 +33,9 @@ test { std.testing.refAllDecls(Value); } -pub const dialects_needed = [_][]const u8{"cuda_tile"}; +pub fn registerDialects(registry: *mlir.DialectRegistry) void { + inline for (.{cuda_tile}) |dialect| dialect.registerDialect(registry); +} pub const FinishError = error{InvalidMlir} || std.mem.Allocator.Error || std.Io.Writer.Error; @@ -51,12 +53,12 @@ pub const Value = struct { return self.kernel orelse @panic("Value has no owning kernel; use Builder.* helpers instead"); } - pub fn tile(self: Value) *const ct.TileType { - return self.type_().isA(ct.TileType) orelse std.debug.panic("Value {f} is not a tile", .{self.type_()}); + pub fn tile(self: Value) *const cuda_tile.TileType { + return self.type_().isA(cuda_tile.TileType) orelse std.debug.panic("Value {f} is not a tile", .{self.type_()}); } pub fn isTile(self: Value) bool { - return self.type_().isA(ct.TileType) != null; + return self.type_().isA(cuda_tile.TileType) != null; } pub fn elemType(self: Value) *const mlir.Type { @@ -95,7 +97,7 @@ pub const Value = struct { } pub fn isPtrElem(self: Value) bool { - return self.elemType().isA(ct.PointerType) != null; + return self.elemType().isA(cuda_tile.PointerType) != null; } pub fn add(self: Value, rhs: anytype) Value { @@ -377,7 +379,7 @@ pub const AllocaOpts = struct { pub const GlobalOpts = struct { alignment: ?i64 = null, constant: bool = false, - visibility: ?ct.SymbolVisibility = null, + visibility: ?cuda_tile.SymbolVisibility = null, }; pub const ScanOpts = struct { @@ -556,13 +558,13 @@ pub const Builder = struct { const entry = mlir.Block.init(arg_types, arg_locs); const ct_body = mlir.Block.init(&.{}, &.{}); - const ct_module = ct.module(ctx, self.name, ct_body, unknown_loc); + const ct_module = cuda_tile.module(ctx, self.name, ct_body, unknown_loc); _ = ct_module.appendTo(self.module.body()); - const func_op = ct.entry(ctx, .{ + const func_op = cuda_tile.entry(ctx, .{ .name = self.name, .block = entry, - .optimization_hints = if (opts.hints.len > 0) ct.optimizationHints(ctx, opts.hints) else null, + .optimization_hints = if (opts.hints.len > 0) cuda_tile.optimizationHintsAttr(ctx, opts.hints) else null, .location = unknown_loc, }); _ = func_op.appendTo(ct_body); @@ -662,7 +664,7 @@ pub const Builder = struct { if (!dtypes.tileLegal(shape)) { std.debug.panic("Builder.tileTy: {any} is not a legal tile shape (power-of-two dims, at most 2^24 elements)", .{shape}); } - return ct.tileType(self.ctx, shape, dtype.toMlir(self.ctx)); + return cuda_tile.tileType(self.ctx, shape, dtype.toMlir(self.ctx)); } pub fn scalarTy(self: *const Builder, dtype: DType) *const mlir.Type { @@ -679,15 +681,15 @@ pub const Builder = struct { /// declared as `.i8` and `unpack`ed in the kernel. pub fn ptrTileTy(self: *const Builder, shape: []const i64, dtype: DType) *const mlir.Type { if (dtype == .i4) @panic("Builder: i4 is a tile element type only, never a pointee; declare the buffer as .i8 and unpack in-kernel"); - return ct.tileType(self.ctx, shape, ct.pointerType(self.ctx, dtype.toMlir(self.ctx))); + return cuda_tile.tileType(self.ctx, shape, cuda_tile.pointerType(self.ctx, dtype.toMlir(self.ctx))); } pub fn tokenTy(self: *const Builder) *const mlir.Type { - return ct.tokenType(self.ctx); + return cuda_tile.tokenType(self.ctx); } fn tileLike(self: *const Builder, v: Value, elem: *const mlir.Type) *const mlir.Type { - return ct.tileType(self.ctx, v.shape().constSlice(), elem); + return cuda_tile.tileType(self.ctx, v.shape().constSlice(), elem); } fn mlirElemToDType(self: *const Builder, elem: *const mlir.Type) DType { @@ -695,7 +697,7 @@ pub const Builder = struct { } fn pointee(v: Value) *const mlir.Type { - const p = v.elemType().isA(ct.PointerType) orelse std.debug.panic("expected a pointer tile, got {f}", .{v.type_()}); + const p = v.elemType().isA(cuda_tile.PointerType) orelse std.debug.panic("expected a pointer tile, got {f}", .{v.type_()}); return p.pointee(); } @@ -708,7 +710,7 @@ pub const Builder = struct { .comptime_float, .float => @floatCast(value), else => @compileError("Builder: unsupported scalar " ++ @typeName(T)), }; - return ct.floatElem(self.ctx, elem, v); + return cuda_tile.floatElemAttr(self.ctx, elem, v); } const v: i64 = switch (@typeInfo(T)) { .comptime_int => blk: { @@ -723,11 +725,11 @@ pub const Builder = struct { .comptime_float, .float => @intFromFloat(value), else => @compileError("Builder: unsupported scalar " ++ @typeName(T)), }; - return ct.intElem(elem, v); + return cuda_tile.intElemAttr(elem, v); } fn splat(self: *Builder, tile_ty: *const mlir.Type, dtype: DType, value: anytype) Value { - return self.emit(ct.constant(self.ctx, ct.splatAttr(tile_ty, self.elemAttr(dtype, value)), tile_ty, self.loc())); + return self.emit(cuda_tile.constant(self.ctx, cuda_tile.splatAttr(tile_ty, self.elemAttr(dtype, value)), tile_ty, self.loc())); } pub fn cst(self: *Builder, dtype: DType, value: anytype) Value { @@ -752,7 +754,7 @@ pub const Builder = struct { pub fn dense(self: *Builder, shape: []const i64, dtype: DType, values: anytype) Value { checkDenseBytes("Builder.dense", shape, dtype, values); const ty = self.tileTy(shape, dtype); - return self.emit(ct.constant(self.ctx, ct.denseAttr(ty, values), ty, self.loc())); + return self.emit(cuda_tile.constant(self.ctx, cuda_tile.denseAttr(ty, values), ty, self.loc())); } fn checkDenseBytes(comptime who: []const u8, shape: []const i64, dtype: DType, values: anytype) void { @@ -767,7 +769,7 @@ pub const Builder = struct { } pub fn iota(self: *Builder, n: i64, dtype: DType) Value { - return self.emit(ct.iota(self.ctx, self.tileTy(&.{n}, dtype), self.loc())); + return self.emit(cuda_tile.iota(self.ctx, self.tileTy(&.{n}, dtype), self.loc())); } /// The element attribute `reduce`/`scan` need as an identity. @@ -814,7 +816,7 @@ pub const Builder = struct { } fn constMatching(self: *Builder, value: anytype, ref: *const mlir.Type) Value { - const elem = if (ref.isA(ct.TileType)) |t| t.elementType() else ref; + const elem = if (ref.isA(cuda_tile.TileType)) |t| t.elementType() else ref; return self.liftAs(value, self.mlirElemToDType(elem)); } @@ -877,34 +879,34 @@ pub const Builder = struct { } pub fn tileBlockId(self: *Builder) BlockId { - const r = self.emitMulti(ct.get_tile_block_id(self.ctx, self.loc()), 3); + const r = self.emitMulti(cuda_tile.get_tile_block_id(self.ctx, self.loc()), 3); return .{ .x = r[0], .y = r[1], .z = r[2] }; } pub fn numTileBlocks(self: *Builder) BlockId { - const r = self.emitMulti(ct.get_num_tile_blocks(self.ctx, self.loc()), 3); + const r = self.emitMulti(cuda_tile.get_num_tile_blocks(self.ctx, self.loc()), 3); return .{ .x = r[0], .y = r[1], .z = r[2] }; } pub fn assumeDivBy(self: *Builder, v: Value, n: u64) Value { - return self.emit(ct.assume(self.ctx, v.inner, ct.divBy(self.ctx, n, null, null), self.loc())); + return self.emit(cuda_tile.assume(self.ctx, v.inner, cuda_tile.divByAttr(self.ctx, n, .{}), self.loc())); } pub fn assumeDivByEvery(self: *Builder, v: Value, n: u64, every: i64, along: i64) Value { - return self.emit(ct.assume(self.ctx, v.inner, ct.divBy(self.ctx, n, every, along), self.loc())); + return self.emit(cuda_tile.assume(self.ctx, v.inner, cuda_tile.divByAttr(self.ctx, n, .{ .every = every, .along = along }), self.loc())); } pub fn assumeSameElements(self: *Builder, v: Value, values: []const i64) Value { - return self.emit(ct.assume(self.ctx, v.inner, ct.sameElements(self.ctx, values), self.loc())); + return self.emit(cuda_tile.assume(self.ctx, v.inner, cuda_tile.sameElementsAttr(self.ctx, values), self.loc())); } pub fn assumeBounded(self: *Builder, v: Value, lb: ?i64, ub: ?i64) Value { - return self.emit(ct.assume(self.ctx, v.inner, ct.bounded(self.ctx, lb, ub), self.loc())); + return self.emit(cuda_tile.assume(self.ctx, v.inner, cuda_tile.boundedAttr(self.ctx, .{ .lb = lb, .ub = ub }), self.loc())); } pub fn tensorView(self: *Builder, base: Value, shape: []const i64, strides: []const i64) Value { - const ty = ct.tensorViewType(self.ctx, pointee(base), shape, strides); - return self.emit(ct.make_tensor_view(self.ctx, base.inner, &.{}, &.{}, ty, self.loc())); + const ty = cuda_tile.tensorViewType(self.ctx, pointee(base), shape, strides); + return self.emit(cuda_tile.make_tensor_view(self.ctx, base.inner, &.{}, &.{}, ty, self.loc())); } pub const Dim = union(enum) { @@ -921,19 +923,19 @@ pub const Builder = struct { for (shape) |d| switch (d) { .static => |s| static_shape.appendAssumeCapacity(s), .dynamic => |v| { - static_shape.appendAssumeCapacity(ct.TensorViewType.dynamic()); + static_shape.appendAssumeCapacity(cuda_tile.TensorViewType.dynamic()); dyn_shape.append(scratch, v.inner) catch @panic("OOM"); }, }; for (strides) |d| switch (d) { .static => |s| static_strides.appendAssumeCapacity(s), .dynamic => |v| { - static_strides.appendAssumeCapacity(ct.TensorViewType.dynamic()); + static_strides.appendAssumeCapacity(cuda_tile.TensorViewType.dynamic()); dyn_strides.append(scratch, v.inner) catch @panic("OOM"); }, }; - const ty = ct.tensorViewType(self.ctx, pointee(base), static_shape.constSlice(), static_strides.constSlice()); - return self.emit(ct.make_tensor_view(self.ctx, base.inner, dyn_shape.items, dyn_strides.items, ty, self.loc())); + const ty = cuda_tile.tensorViewType(self.ctx, pointee(base), static_shape.constSlice(), static_strides.constSlice()); + return self.emit(cuda_tile.make_tensor_view(self.ctx, base.inner, dyn_shape.items, dyn_strides.items, ty, self.loc())); } fn i32Slice(self: *Builder, values: []const i64) []const i32 { @@ -947,24 +949,30 @@ pub const Builder = struct { /// masked. That is why this DSL has no mask tile. pub fn partitionView(self: *Builder, tv: Value, tile: []const i64, opts: PartitionOpts) Value { if (!dtypes.tileLegal(tile)) std.debug.panic("Builder.partitionView: {any} is not a legal tile shape", .{tile}); - const ty: *const mlir.Type = @ptrCast(ct.PartitionViewType.get(self.ctx, self.i32Slice(tile), tv.type_(), opts.dim_map, opts.padding)); - return self.emit(ct.make_partition_view(self.ctx, tv.inner, ty, self.loc())); + const ty: *const mlir.Type = @ptrCast(cuda_tile.PartitionViewType.get(self.ctx, .{ .tile_shape = self.i32Slice(tile), .tensor_view = tv.type_(), .dim_map = opts.dim_map, .padding = opts.padding })); + return self.emit(cuda_tile.make_partition_view(self.ctx, tv.inner, ty, self.loc())); } /// 13.3: like `partitionView`, stepping by `traversal_strides` elements /// per index instead of a whole tile. pub fn stridedView(self: *Builder, tv: Value, tile: []const i64, traversal_strides: []const i64, opts: PartitionOpts) Value { if (!dtypes.tileLegal(tile)) std.debug.panic("Builder.stridedView: {any} is not a legal tile shape", .{tile}); - const ty: *const mlir.Type = @ptrCast(ct.StridedViewType.get(self.ctx, self.i32Slice(tile), self.i32Slice(traversal_strides), tv.type_(), opts.dim_map, opts.padding)); - return self.emit(ct.make_strided_view(self.ctx, tv.inner, ty, self.loc())); + const ty: *const mlir.Type = @ptrCast(cuda_tile.StridedViewType.get(self.ctx, .{ + .tile_shape = self.i32Slice(tile), + .traversal_strides = self.i32Slice(traversal_strides), + .tensor_view = tv.type_(), + .dim_map = opts.dim_map, + .padding = opts.padding, + })); + return self.emit(cuda_tile.make_strided_view(self.ctx, tv.inner, ty, self.loc())); } /// 13.3: a view whose `sparse_dim` is indexed by a rank-1 tile of /// `tile[sparse_dim]` indices (every other index stays rank 0). pub fn gatherScatterView(self: *Builder, tv: Value, tile: []const i64, sparse_dim: u32, padding: ?PaddingValue) Value { if (!dtypes.tileLegal(tile)) std.debug.panic("Builder.gatherScatterView: {any} is not a legal tile shape", .{tile}); - const ty = ct.gatherScatterViewType(self.ctx, tile, tv.type_(), sparse_dim, padding); - return self.emit(ct.make_gather_scatter_view(self.ctx, tv.inner, ty, self.loc())); + const ty = cuda_tile.gatherScatterViewType(self.ctx, tile, tv.type_(), sparse_dim, .{ .padding = padding }); + return self.emit(cuda_tile.make_gather_scatter_view(self.ctx, tv.inner, ty, self.loc())); } fn i32Scalars(self: *Builder, n: usize) []const *const mlir.Type { @@ -974,22 +982,22 @@ pub const Builder = struct { } pub fn indexSpaceShape(self: *Builder, view: Value) []Value { - const n = ct.indexRankOfView(view.type_()); - return self.emitMulti(ct.get_index_space_shape(self.ctx, view.inner, self.i32Scalars(n), self.loc()), n); + const n = cuda_tile.indexRankOfView(view.type_()); + return self.emitMulti(cuda_tile.get_index_space_shape(self.ctx, view.inner, self.i32Scalars(n), self.loc()), n); } pub fn tensorShape(self: *Builder, tv: Value) []Value { - const t = tv.type_().isA(ct.TensorViewType) orelse std.debug.panic("expected a tensor_view, got {f}", .{tv.type_()}); + const t = tv.type_().isA(cuda_tile.TensorViewType) orelse std.debug.panic("expected a tensor_view, got {f}", .{tv.type_()}); const n = t.rank(); - return self.emitMulti(ct.get_tensor_shape(self.ctx, tv.inner, self.i32Scalars(n), self.loc()), n); + return self.emitMulti(cuda_tile.get_tensor_shape(self.ctx, tv.inner, self.i32Scalars(n), self.loc()), n); } - fn memOpts(self: *Builder, ordering: MemoryOrdering, scope: ?MemoryScope, token: ?Value, hints: anytype) ct.MemOpts { + fn memOpts(self: *Builder, ordering: MemoryOrdering, scope: ?MemoryScope, token: ?Value, hints: anytype) cuda_tile.MemOpts { return .{ .ordering = ordering, .scope = scope, .token = innerOpt(token), - .optimization_hints = if (hints.len > 0) ct.optimizationHints(self.ctx, hints) else null, + .optimization_hints = if (hints.len > 0) cuda_tile.optimizationHintsAttr(self.ctx, hints) else null, }; } @@ -998,9 +1006,9 @@ pub const Builder = struct { } pub fn loadOpts(self: *Builder, view: Value, index: []const Value, opts: LoadOpts) Loaded { - const expected = ct.indexRankOfView(view.type_()); + const expected = cuda_tile.indexRankOfView(view.type_()); if (index.len != expected) std.debug.panic("Builder.load: view takes {d} indices, got {d}", .{ expected, index.len }); - const r = self.emitMulti(ct.load_view_tko(self.ctx, view.inner, self.innerSlice(index), self.memOpts(opts.ordering, opts.scope, opts.token, opts.hints), self.loc()), 2); + const r = self.emitMulti(cuda_tile.load_view_tko(self.ctx, view.inner, self.innerSlice(index), self.memOpts(opts.ordering, opts.scope, opts.token, opts.hints), self.loc()), 2); return .{ .tile = r[0], .token = r[1] }; } @@ -1009,9 +1017,9 @@ pub const Builder = struct { } pub fn storeOpts(self: *Builder, tile: Value, view: Value, index: []const Value, opts: StoreOpts) Value { - const expected = ct.indexRankOfView(view.type_()); + const expected = cuda_tile.indexRankOfView(view.type_()); if (index.len != expected) std.debug.panic("Builder.store: view takes {d} indices, got {d}", .{ expected, index.len }); - return self.emit(ct.store_view_tko(self.ctx, tile.inner, view.inner, self.innerSlice(index), self.memOpts(opts.ordering, opts.scope, opts.token, opts.hints), self.loc())); + return self.emit(cuda_tile.store_view_tko(self.ctx, tile.inner, view.inner, self.innerSlice(index), self.memOpts(opts.ordering, opts.scope, opts.token, opts.hints), self.loc())); } /// Gather through a tile of pointers, dropping the token. This arm cannot @@ -1023,7 +1031,7 @@ pub const Builder = struct { pub fn loadPtrOpts(self: *Builder, ptrs: Value, opts: LoadPtrOpts) Loaded { const result_ty = self.tileLike(ptrs, pointee(ptrs)); const mask: ?Value = if (opts.mask) |m| self.expandTo(m, ptrs.shape().constSlice()) else null; - const r = self.emitMulti(ct.load_ptr_tko(self.ctx, ptrs.inner, result_ty, .{ + const r = self.emitMulti(cuda_tile.load_ptr_tko(self.ctx, ptrs.inner, result_ty, .{ .mask = innerOpt(mask), .padding = innerOpt(opts.padding), .mem = self.memOpts(opts.ordering, opts.scope, opts.token, opts.hints), @@ -1037,18 +1045,18 @@ pub const Builder = struct { pub fn storePtrOpts(self: *Builder, ptrs: Value, value: Value, opts: StorePtrOpts) Value { const mask: ?Value = if (opts.mask) |m| self.expandTo(m, ptrs.shape().constSlice()) else null; - return self.emit(ct.store_ptr_tko(self.ctx, ptrs.inner, value.inner, .{ + return self.emit(cuda_tile.store_ptr_tko(self.ctx, ptrs.inner, value.inner, .{ .mask = innerOpt(mask), .mem = self.memOpts(opts.ordering, opts.scope, opts.token, opts.hints), }, self.loc())); } pub fn makeToken(self: *Builder) Value { - return self.emit(ct.make_token(self.ctx, self.loc())); + return self.emit(cuda_tile.make_token(self.ctx, self.loc())); } pub fn joinTokens(self: *Builder, tokens: []const Value) Value { - return self.emit(ct.join_tokens(self.ctx, self.innerSlice(tokens), self.loc())); + return self.emit(cuda_tile.join_tokens(self.ctx, self.innerSlice(tokens), self.loc())); } pub fn print(self: *Builder, str: []const u8, args: []const Value) Value { @@ -1056,27 +1064,27 @@ pub const Builder = struct { } pub fn printOpts(self: *Builder, str: []const u8, args: []const Value, token: ?Value) Value { - return self.emit(ct.print_tko(self.ctx, str, self.innerSlice(args), innerOpt(token), self.loc())); + return self.emit(cuda_tile.print_tko(self.ctx, str, self.innerSlice(args), .{ .token = innerOpt(token) }, self.loc())); } - fn atomicOpts(opts: AtomicOpts) ct.AtomicOpts { + fn atomicOpts(opts: AtomicOpts) cuda_tile.AtomicOpts { return .{ .ordering = opts.ordering, .scope = opts.scope, .mask = innerOpt(opts.mask), .token = innerOpt(opts.token) }; } pub fn atomicCas(self: *Builder, ptrs: Value, cmp: Value, val: Value, opts: AtomicOpts) Loaded { - const r = self.emitMulti(ct.atomic_cas_tko(self.ctx, ptrs.inner, cmp.inner, val.inner, atomicOpts(opts), self.loc()), 2); + const r = self.emitMulti(cuda_tile.atomic_cas_tko(self.ctx, ptrs.inner, cmp.inner, val.inner, atomicOpts(opts), self.loc()), 2); return .{ .tile = r[0], .token = r[1] }; } pub fn atomicRmw(self: *Builder, mode: AtomicRMWMode, ptrs: Value, operand: Value, opts: AtomicOpts) Loaded { - const r = self.emitMulti(ct.atomic_rmw_tko(self.ctx, ptrs.inner, mode, operand.inner, atomicOpts(opts), self.loc()), 2); + const r = self.emitMulti(cuda_tile.atomic_rmw_tko(self.ctx, ptrs.inner, mode, operand.inner, atomicOpts(opts), self.loc()), 2); return .{ .tile = r[0], .token = r[1] }; } /// 13.3: an atomic reduction into an unpadded tiled view; returns the /// token. `xchg` is not a reduction and the dialect rejects it. pub fn atomicRedView(self: *Builder, mode: AtomicRMWMode, view: Value, index: []const Value, value: Value, opts: RedViewOpts) Value { - return self.emit(ct.atomic_red_view_tko(self.ctx, view.inner, self.innerSlice(index), mode, value.inner, .{ + return self.emit(cuda_tile.atomic_red_view_tko(self.ctx, view.inner, self.innerSlice(index), mode, value.inner, .{ .scope = opts.scope, .token = innerOpt(opts.token), }, self.loc())); @@ -1084,13 +1092,13 @@ pub const Builder = struct { /// 13.3: `num_elem` elements of `dtype`, as a rank-0 pointer tile. pub fn alloca(self: *Builder, dtype: DType, num_elem: i64, opts: AllocaOpts) Value { - return self.emit(ct.alloca(self.ctx, num_elem, opts.alignment, opts.global, self.ptrTy(dtype), self.loc())); + return self.emit(cuda_tile.alloca(self.ctx, num_elem, opts.alignment, opts.global, self.ptrTy(dtype), self.loc())); } pub fn global(self: *Builder, name: []const u8, shape: []const i64, dtype: DType, value: anytype, opts: GlobalOpts) void { const ty = self.tileTy(shape, dtype); - const attr = ct.splatAttr(ty, self.elemAttr(dtype, value)); - self.appendGlobal(ct.global(self.ctx, name, attr, .{ .alignment = opts.alignment, .constant = opts.constant, .visibility = opts.visibility }, self.loc())); + const attr = cuda_tile.splatAttr(ty, self.elemAttr(dtype, value)); + self.appendGlobal(cuda_tile.global(self.ctx, name, attr, .{ .alignment = opts.alignment, .constant = opts.constant, .visibility = opts.visibility }, self.loc())); } /// A module-level dense buffer from a slice of Zig scalars (see `dense` @@ -1098,7 +1106,7 @@ pub const Builder = struct { pub fn globalDense(self: *Builder, name: []const u8, shape: []const i64, dtype: DType, values: anytype, opts: GlobalOpts) void { checkDenseBytes("Builder.globalDense", shape, dtype, values); const ty = self.tileTy(shape, dtype); - self.appendGlobal(ct.global(self.ctx, name, ct.denseAttr(ty, values), .{ .alignment = opts.alignment, .constant = opts.constant, .visibility = opts.visibility }, self.loc())); + self.appendGlobal(cuda_tile.global(self.ctx, name, cuda_tile.denseAttr(ty, values), .{ .alignment = opts.alignment, .constant = opts.constant, .visibility = opts.visibility }, self.loc())); } fn appendGlobal(self: *Builder, op: *mlir.Operation) void { @@ -1111,207 +1119,207 @@ pub const Builder = struct { } pub fn getGlobal(self: *Builder, name: []const u8, dtype: DType) Value { - return self.emit(ct.get_global(self.ctx, name, self.ptrTy(dtype), self.loc())); + return self.emit(cuda_tile.get_global(self.ctx, name, self.ptrTy(dtype), self.loc())); } pub fn addf(self: *Builder, lhs: Value, rhs: Value) Value { return self.addfOpts(lhs, rhs, .{}); } pub fn addfOpts(self: *Builder, lhs: Value, rhs: Value, opts: FloatOpts) Value { - return self.emit(ct.addf(self.ctx, lhs.inner, rhs.inner, opts.rounding, opts.flush_to_zero, self.loc())); + return self.emit(cuda_tile.addf(self.ctx, lhs.inner, rhs.inner, opts.rounding, opts.flush_to_zero, self.loc())); } pub fn subf(self: *Builder, lhs: Value, rhs: Value) Value { return self.subfOpts(lhs, rhs, .{}); } pub fn subfOpts(self: *Builder, lhs: Value, rhs: Value, opts: FloatOpts) Value { - return self.emit(ct.subf(self.ctx, lhs.inner, rhs.inner, opts.rounding, opts.flush_to_zero, self.loc())); + return self.emit(cuda_tile.subf(self.ctx, lhs.inner, rhs.inner, opts.rounding, opts.flush_to_zero, self.loc())); } pub fn mulf(self: *Builder, lhs: Value, rhs: Value) Value { return self.mulfOpts(lhs, rhs, .{}); } pub fn mulfOpts(self: *Builder, lhs: Value, rhs: Value, opts: FloatOpts) Value { - return self.emit(ct.mulf(self.ctx, lhs.inner, rhs.inner, opts.rounding, opts.flush_to_zero, self.loc())); + return self.emit(cuda_tile.mulf(self.ctx, lhs.inner, rhs.inner, opts.rounding, opts.flush_to_zero, self.loc())); } pub fn divf(self: *Builder, lhs: Value, rhs: Value) Value { return self.divfOpts(lhs, rhs, .{}); } pub fn divfOpts(self: *Builder, lhs: Value, rhs: Value, opts: FloatOpts) Value { - return self.emit(ct.divf(self.ctx, lhs.inner, rhs.inner, opts.rounding, opts.flush_to_zero, self.loc())); + return self.emit(cuda_tile.divf(self.ctx, lhs.inner, rhs.inner, opts.rounding, opts.flush_to_zero, self.loc())); } pub fn remf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(ct.remf(self.ctx, lhs.inner, rhs.inner, self.loc())); + return self.emit(cuda_tile.remf(self.ctx, lhs.inner, rhs.inner, self.loc())); } pub fn negf(self: *Builder, x: Value) Value { - return self.emit(ct.negf(self.ctx, x.inner, self.loc())); + return self.emit(cuda_tile.negf(self.ctx, x.inner, self.loc())); } pub fn absf(self: *Builder, x: Value) Value { - return self.emit(ct.absf(self.ctx, x.inner, self.loc())); + return self.emit(cuda_tile.absf(self.ctx, x.inner, self.loc())); } pub fn fma(self: *Builder, a: Value, b: Value, acc: Value) Value { return self.fmaOpts(a, b, acc, .{}); } pub fn fmaOpts(self: *Builder, a: Value, b: Value, acc: Value, opts: FloatOpts) Value { - return self.emit(ct.fma(self.ctx, a.inner, b.inner, acc.inner, opts.rounding, opts.flush_to_zero, self.loc())); + return self.emit(cuda_tile.fma(self.ctx, a.inner, b.inner, acc.inner, opts.rounding, opts.flush_to_zero, self.loc())); } pub fn maxf(self: *Builder, lhs: Value, rhs: Value) Value { return self.maxfOpts(lhs, rhs, .{}); } pub fn maxfOpts(self: *Builder, lhs: Value, rhs: Value, opts: MinMaxOpts) Value { - return self.emit(ct.maxf(self.ctx, lhs.inner, rhs.inner, opts.propagate_nan, opts.flush_to_zero, self.loc())); + return self.emit(cuda_tile.maxf(self.ctx, lhs.inner, rhs.inner, opts.propagate_nan, opts.flush_to_zero, self.loc())); } pub fn minf(self: *Builder, lhs: Value, rhs: Value) Value { return self.minfOpts(lhs, rhs, .{}); } pub fn minfOpts(self: *Builder, lhs: Value, rhs: Value, opts: MinMaxOpts) Value { - return self.emit(ct.minf(self.ctx, lhs.inner, rhs.inner, opts.propagate_nan, opts.flush_to_zero, self.loc())); + return self.emit(cuda_tile.minf(self.ctx, lhs.inner, rhs.inner, opts.propagate_nan, opts.flush_to_zero, self.loc())); } pub fn pow(self: *Builder, x: Value, exponent: Value) Value { - return self.emit(ct.pow(self.ctx, x.inner, exponent.inner, self.loc())); + return self.emit(cuda_tile.pow(self.ctx, x.inner, exponent.inner, self.loc())); } pub fn atan2(self: *Builder, x: Value, y: Value) Value { - return self.emit(ct.atan2(self.ctx, x.inner, y.inner, self.loc())); + return self.emit(cuda_tile.atan2(self.ctx, x.inner, y.inner, self.loc())); } pub fn exp(self: *Builder, x: Value) Value { - return self.emit(ct.exp(self.ctx, x.inner, null, self.loc())); + return self.emit(cuda_tile.exp(self.ctx, x.inner, .{}, self.loc())); } /// 13.3: `exp` with an explicit rounding mode. pub fn expRound(self: *Builder, x: Value, rounding: RoundingMode) Value { - return self.emit(ct.exp(self.ctx, x.inner, rounding, self.loc())); + return self.emit(cuda_tile.exp(self.ctx, x.inner, .{ .rounding = rounding }, self.loc())); } pub fn exp2(self: *Builder, x: Value) Value { return self.exp2Opts(x, false); } pub fn exp2Opts(self: *Builder, x: Value, flush_to_zero: bool) Value { - return self.emit(ct.exp2(self.ctx, x.inner, flush_to_zero, self.loc())); + return self.emit(cuda_tile.exp2(self.ctx, x.inner, flush_to_zero, self.loc())); } pub fn log(self: *Builder, x: Value) Value { - return self.emit(ct.log(self.ctx, x.inner, self.loc())); + return self.emit(cuda_tile.log(self.ctx, x.inner, self.loc())); } pub fn log2(self: *Builder, x: Value) Value { - return self.emit(ct.log2(self.ctx, x.inner, self.loc())); + return self.emit(cuda_tile.log2(self.ctx, x.inner, self.loc())); } pub fn sqrt(self: *Builder, x: Value) Value { return self.sqrtOpts(x, .{}); } pub fn sqrtOpts(self: *Builder, x: Value, opts: FloatOpts) Value { - return self.emit(ct.sqrt(self.ctx, x.inner, opts.rounding, opts.flush_to_zero, self.loc())); + return self.emit(cuda_tile.sqrt(self.ctx, x.inner, opts.rounding, opts.flush_to_zero, self.loc())); } pub fn rsqrt(self: *Builder, x: Value) Value { return self.rsqrtOpts(x, false); } pub fn rsqrtOpts(self: *Builder, x: Value, flush_to_zero: bool) Value { - return self.emit(ct.rsqrt(self.ctx, x.inner, flush_to_zero, self.loc())); + return self.emit(cuda_tile.rsqrt(self.ctx, x.inner, flush_to_zero, self.loc())); } pub fn sin(self: *Builder, x: Value) Value { - return self.emit(ct.sin(self.ctx, x.inner, self.loc())); + return self.emit(cuda_tile.sin(self.ctx, x.inner, self.loc())); } pub fn cos(self: *Builder, x: Value) Value { - return self.emit(ct.cos(self.ctx, x.inner, self.loc())); + return self.emit(cuda_tile.cos(self.ctx, x.inner, self.loc())); } pub fn tan(self: *Builder, x: Value) Value { - return self.emit(ct.tan(self.ctx, x.inner, self.loc())); + return self.emit(cuda_tile.tan(self.ctx, x.inner, self.loc())); } pub fn sinh(self: *Builder, x: Value) Value { - return self.emit(ct.sinh(self.ctx, x.inner, self.loc())); + return self.emit(cuda_tile.sinh(self.ctx, x.inner, self.loc())); } pub fn cosh(self: *Builder, x: Value) Value { - return self.emit(ct.cosh(self.ctx, x.inner, self.loc())); + return self.emit(cuda_tile.cosh(self.ctx, x.inner, self.loc())); } pub fn tanh(self: *Builder, x: Value) Value { - return self.emit(ct.tanh(self.ctx, x.inner, null, self.loc())); + return self.emit(cuda_tile.tanh(self.ctx, x.inner, .{}, self.loc())); } /// 13.2: `tanh` with an explicit rounding mode. pub fn tanhRound(self: *Builder, x: Value, rounding: RoundingMode) Value { - return self.emit(ct.tanh(self.ctx, x.inner, rounding, self.loc())); + return self.emit(cuda_tile.tanh(self.ctx, x.inner, .{ .rounding = rounding }, self.loc())); } pub fn ceil(self: *Builder, x: Value) Value { - return self.emit(ct.ceil(self.ctx, x.inner, self.loc())); + return self.emit(cuda_tile.ceil(self.ctx, x.inner, self.loc())); } pub fn floor(self: *Builder, x: Value) Value { - return self.emit(ct.floor(self.ctx, x.inner, self.loc())); + return self.emit(cuda_tile.floor(self.ctx, x.inner, self.loc())); } pub fn cmpf(self: *Builder, predicate: ComparisonPredicate, lhs: Value, rhs: Value) Value { return self.cmpfOpts(predicate, lhs, rhs, .ordered); } pub fn cmpfOpts(self: *Builder, predicate: ComparisonPredicate, lhs: Value, rhs: Value, ordering: ComparisonOrdering) Value { - return self.emit(ct.cmpf(self.ctx, predicate, ordering, lhs.inner, rhs.inner, self.loc())); + return self.emit(cuda_tile.cmpf(self.ctx, predicate, ordering, lhs.inner, rhs.inner, self.loc())); } pub fn mmaf(self: *Builder, a: Value, b: Value, acc: Value) Value { return self.mmafOpts(a, b, acc, .{}); } pub fn mmafOpts(self: *Builder, a: Value, b: Value, acc: Value, opts: MmaOpts) Value { - return self.emit(ct.mmaf(self.ctx, a.inner, b.inner, acc.inner, opts.fast_acc, self.loc())); + return self.emit(cuda_tile.mmaf(self.ctx, a.inner, b.inner, acc.inner, opts.fast_acc, self.loc())); } /// 13.3: block-scaled MMA. Scales are plain tiles in logical layout — /// `a_scale` is `[M, K/V]`, `b_scale` is `[K/V, N]` — never pre-swizzled. pub fn mmafScaled(self: *Builder, a: Value, b: Value, acc: Value, a_scale: Value, b_scale: Value) Value { - return self.emit(ct.mmaf_scaled(self.ctx, a.inner, b.inner, acc.inner, a_scale.inner, b_scale.inner, self.loc())); + return self.emit(cuda_tile.mmaf_scaled(self.ctx, a.inner, b.inner, acc.inner, a_scale.inner, b_scale.inner, self.loc())); } pub fn mmai(self: *Builder, a: Value, b: Value, acc: Value, a_signedness: Signedness, b_signedness: Signedness) Value { - return self.emit(ct.mmai(self.ctx, a.inner, b.inner, acc.inner, a_signedness, b_signedness, self.loc())); + return self.emit(cuda_tile.mmai(self.ctx, a.inner, b.inner, acc.inner, a_signedness, b_signedness, self.loc())); } pub fn addi(self: *Builder, lhs: Value, rhs: Value) Value { return self.addiOpts(lhs, rhs, .{}); } pub fn addiOpts(self: *Builder, lhs: Value, rhs: Value, opts: IntOpts) Value { - return self.emit(ct.addi(self.ctx, lhs.inner, rhs.inner, opts.overflow, self.loc())); + return self.emit(cuda_tile.addi(self.ctx, lhs.inner, rhs.inner, opts.overflow, self.loc())); } pub fn subi(self: *Builder, lhs: Value, rhs: Value) Value { return self.subiOpts(lhs, rhs, .{}); } pub fn subiOpts(self: *Builder, lhs: Value, rhs: Value, opts: IntOpts) Value { - return self.emit(ct.subi(self.ctx, lhs.inner, rhs.inner, opts.overflow, self.loc())); + return self.emit(cuda_tile.subi(self.ctx, lhs.inner, rhs.inner, opts.overflow, self.loc())); } pub fn muli(self: *Builder, lhs: Value, rhs: Value) Value { return self.muliOpts(lhs, rhs, .{}); } pub fn muliOpts(self: *Builder, lhs: Value, rhs: Value, opts: IntOpts) Value { - return self.emit(ct.muli(self.ctx, lhs.inner, rhs.inner, opts.overflow, self.loc())); + return self.emit(cuda_tile.muli(self.ctx, lhs.inner, rhs.inner, opts.overflow, self.loc())); } pub fn mulhii(self: *Builder, x: Value, y: Value) Value { - return self.emit(ct.mulhii(self.ctx, x.inner, y.inner, self.loc())); + return self.emit(cuda_tile.mulhii(self.ctx, x.inner, y.inner, self.loc())); } pub fn divi(self: *Builder, lhs: Value, rhs: Value, signedness: Signedness) Value { return self.diviOpts(lhs, rhs, .{ .signedness = signedness }); } pub fn diviOpts(self: *Builder, lhs: Value, rhs: Value, opts: DivIOpts) Value { - return self.emit(ct.divi(self.ctx, lhs.inner, rhs.inner, opts.signedness, opts.rounding, self.loc())); + return self.emit(cuda_tile.divi(self.ctx, lhs.inner, rhs.inner, opts.signedness, .{ .rounding = opts.rounding }, self.loc())); } pub fn remi(self: *Builder, lhs: Value, rhs: Value, signedness: Signedness) Value { - return self.emit(ct.remi(self.ctx, lhs.inner, rhs.inner, signedness, self.loc())); + return self.emit(cuda_tile.remi(self.ctx, lhs.inner, rhs.inner, signedness, self.loc())); } pub fn maxi(self: *Builder, lhs: Value, rhs: Value, signedness: Signedness) Value { - return self.emit(ct.maxi(self.ctx, lhs.inner, rhs.inner, signedness, self.loc())); + return self.emit(cuda_tile.maxi(self.ctx, lhs.inner, rhs.inner, signedness, self.loc())); } pub fn mini(self: *Builder, lhs: Value, rhs: Value, signedness: Signedness) Value { - return self.emit(ct.mini(self.ctx, lhs.inner, rhs.inner, signedness, self.loc())); + return self.emit(cuda_tile.mini(self.ctx, lhs.inner, rhs.inner, signedness, self.loc())); } pub fn negi(self: *Builder, x: Value) Value { - return self.emit(ct.negi(self.ctx, x.inner, .none, self.loc())); + return self.emit(cuda_tile.negi(self.ctx, x.inner, .none, self.loc())); } pub fn absi(self: *Builder, x: Value) Value { - return self.emit(ct.absi(self.ctx, x.inner, self.loc())); + return self.emit(cuda_tile.absi(self.ctx, x.inner, self.loc())); } pub fn shli(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(ct.shli(self.ctx, lhs.inner, rhs.inner, .none, self.loc())); + return self.emit(cuda_tile.shli(self.ctx, lhs.inner, rhs.inner, .none, self.loc())); } pub fn shri(self: *Builder, lhs: Value, rhs: Value, signedness: Signedness) Value { - return self.emit(ct.shri(self.ctx, lhs.inner, rhs.inner, signedness, self.loc())); + return self.emit(cuda_tile.shri(self.ctx, lhs.inner, rhs.inner, signedness, self.loc())); } pub fn andi(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(ct.andi(self.ctx, lhs.inner, rhs.inner, self.loc())); + return self.emit(cuda_tile.andi(self.ctx, lhs.inner, rhs.inner, self.loc())); } pub fn ori(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(ct.ori(self.ctx, lhs.inner, rhs.inner, self.loc())); + return self.emit(cuda_tile.ori(self.ctx, lhs.inner, rhs.inner, self.loc())); } pub fn xori(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(ct.xori(self.ctx, lhs.inner, rhs.inner, self.loc())); + return self.emit(cuda_tile.xori(self.ctx, lhs.inner, rhs.inner, self.loc())); } pub fn cmpi(self: *Builder, predicate: ComparisonPredicate, lhs: Value, rhs: Value, signedness: Signedness) Value { - return self.emit(ct.cmpi(self.ctx, predicate, signedness, lhs.inner, rhs.inner, self.loc())); + return self.emit(cuda_tile.cmpi(self.ctx, predicate, signedness, lhs.inner, rhs.inner, self.loc())); } pub fn maximum(self: *Builder, a: anytype, b: anytype) Value { @@ -1325,7 +1333,7 @@ pub const Builder = struct { } pub fn select(self: *Builder, cond: Value, t: Value, f: Value) Value { - return self.emit(ct.select(self.ctx, cond.inner, t.inner, f.inner, self.loc())); + return self.emit(cuda_tile.select(self.ctx, cond.inner, t.inner, f.inner, self.loc())); } pub fn where(self: *Builder, cond: Value, x: anytype, y: anytype) Value { @@ -1338,20 +1346,20 @@ pub const Builder = struct { } pub fn bitcast(self: *Builder, src: Value, dtype: DType) Value { - return self.emit(ct.bitcast(self.ctx, src.inner, self.tileLike(src, dtype.toMlir(self.ctx)), self.loc())); + return self.emit(cuda_tile.bitcast(self.ctx, src.inner, self.tileLike(src, dtype.toMlir(self.ctx)), self.loc())); } pub fn exti(self: *Builder, src: Value, dtype: DType, signedness: Signedness) Value { - return self.emit(ct.exti(self.ctx, src.inner, signedness, self.tileLike(src, dtype.toMlir(self.ctx)), self.loc())); + return self.emit(cuda_tile.exti(self.ctx, src.inner, signedness, self.tileLike(src, dtype.toMlir(self.ctx)), self.loc())); } pub fn trunci(self: *Builder, src: Value, dtype: DType) Value { - return self.emit(ct.trunci(self.ctx, src.inner, self.tileLike(src, dtype.toMlir(self.ctx)), .none, self.loc())); + return self.emit(cuda_tile.trunci(self.ctx, src.inner, self.tileLike(src, dtype.toMlir(self.ctx)), .none, self.loc())); } pub fn ftof(self: *Builder, src: Value, dtype: DType) Value { return self.ftofOpts(src, dtype, .{}); } pub fn ftofOpts(self: *Builder, src: Value, dtype: DType, opts: CastOpts) Value { const rounding: RoundingMode = opts.rounding orelse (if (dtype == .f8e8m0fnu) .zero else .nearest_even); - return self.emit(ct.ftof(self.ctx, src.inner, self.tileLike(src, dtype.toMlir(self.ctx)), rounding, self.loc())); + return self.emit(cuda_tile.ftof(self.ctx, src.inner, self.tileLike(src, dtype.toMlir(self.ctx)), .{ .rounding = rounding }, self.loc())); } /// Float -> integer, truncating toward zero (the only rounding `ftoi` /// admits). @@ -1359,7 +1367,7 @@ pub const Builder = struct { return self.ftoiOpts(src, dtype, .{}); } pub fn ftoiOpts(self: *Builder, src: Value, dtype: DType, opts: FToIOpts) Value { - return self.emit(ct.ftoi(self.ctx, src.inner, opts.signedness, .nearest_int_to_zero, self.tileLike(src, dtype.toMlir(self.ctx)), self.loc())); + return self.emit(cuda_tile.ftoi(self.ctx, src.inner, opts.signedness, .nearest_int_to_zero, self.tileLike(src, dtype.toMlir(self.ctx)), self.loc())); } /// Integer -> float. `f8e8m0fnu` is not a legal target; go through `f32`. pub fn itof(self: *Builder, src: Value, dtype: DType) Value { @@ -1367,24 +1375,24 @@ pub const Builder = struct { } pub fn itofOpts(self: *Builder, src: Value, dtype: DType, opts: CastOpts) Value { if (dtype == .f8e8m0fnu) @panic("Builder.itof: the dialect refuses integer -> f8e8m0fnu; convert to .f32 first (Builder.cast does)"); - return self.emit(ct.itof(self.ctx, src.inner, opts.signedness, opts.rounding orelse .nearest_even, self.tileLike(src, dtype.toMlir(self.ctx)), self.loc())); + return self.emit(cuda_tile.itof(self.ctx, src.inner, opts.signedness, opts.rounding orelse .nearest_even, self.tileLike(src, dtype.toMlir(self.ctx)), self.loc())); } pub fn intToPtr(self: *Builder, src: Value, pointee_dtype: DType) Value { - return self.emit(ct.int_to_ptr(self.ctx, src.inner, self.ptrTileTy(src.shape().constSlice(), pointee_dtype), self.loc())); + return self.emit(cuda_tile.int_to_ptr(self.ctx, src.inner, self.ptrTileTy(src.shape().constSlice(), pointee_dtype), self.loc())); } pub fn ptrToInt(self: *Builder, src: Value) Value { - return self.emit(ct.ptr_to_int(self.ctx, src.inner, self.tileLike(src, DType.i64.toMlir(self.ctx)), self.loc())); + return self.emit(cuda_tile.ptr_to_int(self.ctx, src.inner, self.tileLike(src, DType.i64.toMlir(self.ctx)), self.loc())); } pub fn ptrToPtr(self: *Builder, src: Value, pointee_dtype: DType) Value { - return self.emit(ct.ptr_to_ptr(self.ctx, src.inner, self.ptrTileTy(src.shape().constSlice(), pointee_dtype), self.loc())); + return self.emit(cuda_tile.ptr_to_ptr(self.ctx, src.inner, self.ptrTileTy(src.shape().constSlice(), pointee_dtype), self.loc())); } /// 13.3: a sub-byte tile packed into `tile`. pub fn pack(self: *Builder, src: Value, shape: []const i64) Value { - return self.emit(ct.pack(self.ctx, src.inner, self.tileTy(shape, .i8), self.loc())); + return self.emit(cuda_tile.pack(self.ctx, src.inner, self.tileTy(shape, .i8), self.loc())); } /// 13.3: an i8 tile unpacked into `tile` of a sub-byte `dtype`. pub fn unpack(self: *Builder, src: Value, shape: []const i64, dtype: DType) Value { - return self.emit(ct.unpack(self.ctx, src.inner, self.tileTy(shape, dtype), self.loc())); + return self.emit(cuda_tile.unpack(self.ctx, src.inner, self.tileTy(shape, dtype), self.loc())); } /// Numeric cast with auto-dispatch: ftof / ftoi / itof / exti / trunci / @@ -1411,16 +1419,16 @@ pub const Builder = struct { } pub fn reshape(self: *Builder, src: Value, shape: []const i64) Value { - return self.emit(ct.reshape(self.ctx, src.inner, self.tileTyOf(shape, src.elemType()), self.loc())); + return self.emit(cuda_tile.reshape(self.ctx, src.inner, self.tileTyOf(shape, src.elemType()), self.loc())); } fn tileTyOf(self: *const Builder, shape: []const i64, elem: *const mlir.Type) *const mlir.Type { if (!dtypes.tileLegal(shape)) std.debug.panic("Builder: {any} is not a legal tile shape", .{shape}); - return ct.tileType(self.ctx, shape, elem); + return cuda_tile.tileType(self.ctx, shape, elem); } pub fn broadcastTo(self: *Builder, src: Value, shape: []const i64) Value { - return self.emit(ct.broadcast(self.ctx, src.inner, self.tileTyOf(shape, src.elemType()), self.loc())); + return self.emit(cuda_tile.broadcast(self.ctx, src.inner, self.tileTyOf(shape, src.elemType()), self.loc())); } pub fn expandDims(self: *Builder, src: Value, axis: usize) Value { @@ -1435,11 +1443,11 @@ pub const Builder = struct { pub fn cat(self: *Builder, lhs: Value, rhs: Value, dim: usize) Value { var out = lhs.shape(); out.set(dim, out.get(dim) + rhs.dim(dim)); - return self.emit(ct.cat(self.ctx, lhs.inner, rhs.inner, @intCast(dim), self.tileTyOf(out.constSlice(), lhs.elemType()), self.loc())); + return self.emit(cuda_tile.cat(self.ctx, lhs.inner, rhs.inner, @intCast(dim), self.tileTyOf(out.constSlice(), lhs.elemType()), self.loc())); } pub fn extract(self: *Builder, src: Value, indices: []const Value, shape: []const i64) Value { - return self.emit(ct.extract(self.ctx, src.inner, self.innerSlice(indices), self.tileTyOf(shape, src.elemType()), self.loc())); + return self.emit(cuda_tile.extract(self.ctx, src.inner, self.innerSlice(indices), self.tileTyOf(shape, src.elemType()), self.loc())); } pub fn permute(self: *Builder, src: Value, order: []const i32) Value { @@ -1447,7 +1455,7 @@ pub const Builder = struct { std.debug.assert(order.len == src_shape.len); var out: Value.Shape = .empty; for (order) |i| out.appendAssumeCapacity(src_shape.get(@intCast(i))); - return self.emit(ct.permute(self.ctx, src.inner, order, self.tileTyOf(out.constSlice(), src.elemType()), self.loc())); + return self.emit(cuda_tile.permute(self.ctx, src.inner, order, self.tileTyOf(out.constSlice(), src.elemType()), self.loc())); } pub fn transpose(self: *Builder, src: Value) Value { @@ -1456,7 +1464,7 @@ pub const Builder = struct { } pub fn offset(self: *Builder, ptrs: Value, off: Value) Value { - return self.emit(ct.offset(self.ctx, ptrs.inner, off.inner, self.loc())); + return self.emit(cuda_tile.offset(self.ctx, ptrs.inner, off.inner, self.loc())); } fn reducedShape(src: Value, axis: usize) Value.Shape { @@ -1472,13 +1480,13 @@ pub const Builder = struct { } fn combineBody(self: *Builder, ctx: anytype, elem: *const mlir.Type, combine: anytype) *mlir.Block { - const scalar = ct.tileType(self.ctx, &.{}, elem); + const scalar = cuda_tile.tileType(self.ctx, &.{}, elem); const body = mlir.Block.init(&.{ scalar, scalar }, &.{ self.loc(), self.loc() }); self.pushBlock(body); const element: Value = .{ .inner = body.argument(0), .kernel = self }; const acc: Value = .{ .inner = body.argument(1), .kernel = self }; const combined: Value = combine(self, element, acc, ctx); - _ = ct.yield(self.ctx, &.{combined.inner}, self.loc()).appendTo(body); + _ = cuda_tile.yield(self.ctx, &.{combined.inner}, self.loc()).appendTo(body); self.popBlock(); return body; } @@ -1486,13 +1494,13 @@ pub const Builder = struct { pub fn reduce(self: *Builder, ctx: anytype, args: ReduceArgs(@TypeOf(ctx))) Value { const result_ty = self.tileTyOf(reducedShape(args.src, args.axis).constSlice(), args.src.elemType()); const body = self.combineBody(ctx, args.src.elemType(), args.combine); - return self.emit(ct.reduce(self.ctx, &.{args.src.inner}, @intCast(args.axis), &.{args.identity}, body, &.{result_ty}, self.loc())); + return self.emit(cuda_tile.reduce(self.ctx, &.{args.src.inner}, @intCast(args.axis), &.{args.identity}, body, &.{result_ty}, self.loc())); } pub fn scan(self: *Builder, ctx: anytype, args: ScanArgs(@TypeOf(ctx))) Value { checkAxis(args.src, args.axis); const body = self.combineBody(ctx, args.src.elemType(), args.combine); - return self.emit(ct.scan(self.ctx, &.{args.src.inner}, @intCast(args.axis), args.reverse, &.{args.identity}, body, &.{args.src.type_()}, self.loc())); + return self.emit(cuda_tile.scan(self.ctx, &.{args.src.inner}, @intCast(args.axis), args.reverse, &.{args.identity}, body, &.{args.src.type_()}, self.loc())); } /// The `2N`-argument body of a variadic `reduce`/`scan`: arguments come in @@ -1503,7 +1511,7 @@ pub const Builder = struct { const types = scratch.alloc(*const mlir.Type, 2 * n) catch @panic("OOM"); const locs = scratch.alloc(*const mlir.Location, 2 * n) catch @panic("OOM"); for (srcs, 0..) |s, i| { - const scalar = ct.tileType(self.ctx, &.{}, s.elemType()); + const scalar = cuda_tile.tileType(self.ctx, &.{}, s.elemType()); types[2 * i] = scalar; types[2 * i + 1] = scalar; locs[2 * i] = self.loc(); @@ -1519,7 +1527,7 @@ pub const Builder = struct { } const out: []const Value = combine(self, elements, accs, ctx); if (out.len != n) std.debug.panic("Builder.reduceMulti: combine returned {d} values for {d} sources", .{ out.len, n }); - _ = ct.yield(self.ctx, self.innerSlice(out), self.loc()).appendTo(body); + _ = cuda_tile.yield(self.ctx, self.innerSlice(out), self.loc()).appendTo(body); self.popBlock(); return body; } @@ -1529,7 +1537,7 @@ pub const Builder = struct { const result_types = self.arena.allocator().alloc(*const mlir.Type, args.srcs.len) catch @panic("OOM"); for (args.srcs, 0..) |s, i| result_types[i] = self.tileTyOf(reducedShape(s, args.axis).constSlice(), s.elemType()); const body = self.combineBodyMulti(ctx, args.srcs, args.combine); - return self.emitMulti(ct.reduce(self.ctx, self.innerSlice(args.srcs), @intCast(args.axis), args.identities, body, result_types, self.loc()), args.srcs.len); + return self.emitMulti(cuda_tile.reduce(self.ctx, self.innerSlice(args.srcs), @intCast(args.axis), args.identities, body, result_types, self.loc()), args.srcs.len); } pub fn scanMulti(self: *Builder, ctx: anytype, args: ScanMultiArgs(@TypeOf(ctx))) []Value { @@ -1540,7 +1548,7 @@ pub const Builder = struct { result_types[i] = s.type_(); } const body = self.combineBodyMulti(ctx, args.srcs, args.combine); - return self.emitMulti(ct.scan(self.ctx, self.innerSlice(args.srcs), @intCast(args.axis), args.reverse, args.identities, body, result_types, self.loc()), args.srcs.len); + return self.emitMulti(cuda_tile.scan(self.ctx, self.innerSlice(args.srcs), @intCast(args.axis), args.reverse, args.identities, body, result_types, self.loc()), args.srcs.len); } const ReduceKind = enum { sum, max, min, prod }; @@ -1747,7 +1755,7 @@ pub const Builder = struct { } pub fn deviceAssert(self: *Builder, condition: Value, message: []const u8) void { - _ = ct.assert_(self.ctx, condition.inner, message, self.loc()).appendTo(self.currentBlock()); + _ = cuda_tile.assert_(self.ctx, condition.inner, message, self.loc()).appendTo(self.currentBlock()); } /// Append `return`, verify the module, and serialize it to a NUL-terminated @@ -1757,7 +1765,7 @@ pub const Builder = struct { std.debug.assert(results.len == 0); const current = self.currentBlock(); if (current.terminator() == null) { - _ = ct.return_(self.ctx, &.{}, self.loc()).appendTo(current); + _ = cuda_tile.return_(self.ctx, &.{}, self.loc()).appendTo(current); } if (!self.module.operation().verify()) { @@ -1776,7 +1784,7 @@ pub const Builder = struct { fn testContext() !*mlir.Context { const registry = try mlir.DialectRegistry.init(); defer registry.deinit(); - inline for (dialects_needed) |d| mlir.DialectHandle.fromString(d).insertDialect(registry); + registerDialects(registry); const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); ctx.loadAllAvailableDialects(); return ctx; diff --git a/kernels/cuda_tile/control_flow.zig b/dsl/cuda_tile/control_flow.zig similarity index 85% rename from kernels/cuda_tile/control_flow.zig rename to dsl/cuda_tile/control_flow.zig index f51e8571..8d69a764 100644 --- a/kernels/cuda_tile/control_flow.zig +++ b/dsl/cuda_tile/control_flow.zig @@ -1,11 +1,11 @@ -//! `cuda_tile` scope wrappers, like `kernels/common/control_flow.zig` but with +//! `cuda_tile` scope wrappers, like `dsl/common/control_flow.zig` but with //! the dialect's terminators: `continue` ends a `for`/`loop` body, `yield` an //! `if`, `break` leaves a `loop`. An already-terminated branch is left alone. const std = @import("std"); +const cuda_tile = @import("mlir/dialects/cuda_tile"); +const dsl = @import("dsl/common"); const mlir = @import("mlir"); -const ct = @import("mlir/dialects/cuda_tile"); -const dsl = @import("kernels/common"); pub const tupleArity = dsl.tupleArity; @@ -28,9 +28,9 @@ fn terminate(comptime ValueT: type, comptime N: usize, kernel: anytype, block: * if (block.terminator() != null) return; const buf = collect(ValueT, N, values, what); const op = switch (mnemonic) { - .yield => ct.yield(kernel.ctx, &buf, kernel.loc()), - .continue_ => ct.continue_(kernel.ctx, &buf, kernel.loc()), - .break_ => ct.break_(kernel.ctx, &buf, kernel.loc()), + .yield => cuda_tile.yield(kernel.ctx, &buf, kernel.loc()), + .continue_ => cuda_tile.continue_(kernel.ctx, &buf, kernel.loc()), + .break_ => cuda_tile.break_(kernel.ctx, &buf, kernel.loc()), }; _ = op.appendTo(block); } @@ -62,7 +62,7 @@ pub fn ForScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usiz const k = self.kernel; terminate(ValueT, N, k, self.body, .continue_, values, "ForScope.yield"); k.popBlock(); - const for_op = ct.for_(k.ctx, self.lb_inner, self.ub_inner, self.step_inner, &self.inits_inner, self.body, self.unsigned_cmp, k.loc()); + const for_op = cuda_tile.for_(k.ctx, self.lb_inner, self.ub_inner, self.step_inner, &self.inits_inner, self.body, self.unsigned_cmp, k.loc()); _ = for_op.appendTo(k.currentBlock()); for (0..N) |i| self.results[i] = .{ .inner = for_op.result(i), .kernel = k }; } @@ -81,7 +81,7 @@ pub fn IfOnlyScope(comptime BuilderT: type, comptime ValueT: type) type { const k = self.kernel; terminate(ValueT, 0, k, self.then_block, .yield, values, "IfOnlyScope.yieldThen"); k.popBlock(); - _ = ct.if_(k.ctx, self.cond_inner, &.{}, self.then_block, null, k.loc()).appendTo(k.currentBlock()); + _ = cuda_tile.if_(k.ctx, self.cond_inner, &.{}, self.then_block, .{}, k.loc()).appendTo(k.currentBlock()); } }; } @@ -108,7 +108,7 @@ pub fn IfScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize const k = self.kernel; terminate(ValueT, N, k, self.else_block, .yield, values, "IfScope.yieldElse"); k.popBlock(); - const if_op = ct.if_(k.ctx, self.cond_inner, &self.result_types, self.then_block, self.else_block, k.loc()); + const if_op = cuda_tile.if_(k.ctx, self.cond_inner, &self.result_types, self.then_block, .{ .else_block = self.else_block }, k.loc()); _ = if_op.appendTo(k.currentBlock()); for (0..N) |i| self.results[i] = .{ .inner = if_op.result(i), .kernel = k }; } @@ -143,7 +143,7 @@ pub fn LoopScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usi const k = self.kernel; terminate(ValueT, N, k, self.body, .continue_, values, "LoopScope.yield"); k.popBlock(); - const loop_op = ct.loop(k.ctx, &self.inits_inner, &self.result_types, self.body, k.loc()); + const loop_op = cuda_tile.loop(k.ctx, &self.inits_inner, &self.result_types, self.body, k.loc()); _ = loop_op.appendTo(k.currentBlock()); for (0..N) |i| self.results[i] = .{ .inner = loop_op.result(i), .kernel = k }; } diff --git a/kernels/cuda_tile/dtype.zig b/dsl/cuda_tile/dtype.zig similarity index 100% rename from kernels/cuda_tile/dtype.zig rename to dsl/cuda_tile/dtype.zig diff --git a/dsl/cute/BUILD.bazel b/dsl/cute/BUILD.bazel new file mode 100644 index 00000000..5693ed5c --- /dev/null +++ b/dsl/cute/BUILD.bazel @@ -0,0 +1,34 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "cute_builder", + srcs = [ + "dtype.zig", + ], + import_name = "dsl/cute/builder", + main = "builder.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//dsl/common", + "//mlir", + "//mlir/dialects/arith", + "//mlir/dialects/cf", + "//mlir/dialects/cuda", + "//mlir/dialects/cute_ir", + "//mlir/dialects/cute_nvgpu", + "//mlir/dialects/func", + "//mlir/dialects/gpu", + "//mlir/dialects/llvm", + "//mlir/dialects/math", + "//mlir/dialects/nvvm", + "//mlir/dialects/scf", + "//mlir/dialects/vector", + "//stdx", + ], +) + +zig_test( + name = "test", + deps = [":cute_builder"], +) diff --git a/kernels/cute/README.md b/dsl/cute/README.md similarity index 85% rename from kernels/cute/README.md rename to dsl/cute/README.md index c96509a9..d8de2608 100644 --- a/kernels/cute/README.md +++ b/dsl/cute/README.md @@ -1,17 +1,18 @@ # CuTe DSL kernels in Zig -`kernels/cute` is a Zig front end for NVIDIA's CuTe DSL, shaped after the +`dsl/cute` is a Zig front end for NVIDIA's CuTe DSL, shaped after the Python `cutlass.cute` API: a kernel takes tensors, indexes them with coordinates, and reads `thread_idx()` and friends from `cute.arch`. It is the -fourth instance of the `kernels/triton` / `kernels/mosaic_tpu` / -`kernels/cuda_tile` pattern: a `Builder` that owns an MLIR module and appends +fourth instance of the `dsl/triton` / `dsl/mosaic_tpu` / +`dsl/cuda_tile` pattern: a `Builder` that owns an MLIR module and appends ops to a block stack, a `Value` with fluent scalar arithmetic, and a `zml.kernel.cute.Kernel` wrapper that turns a `run` function into a `stablehlo.custom_call` XLA compiles in process with `libcute_compiler.so`. -The dialect binding lives in `mlir/dialects/cute_ir` (`cute`, `cute_nvgpu`, -and NVIDIA's `nvvm` and `cuda`, types and attributes through the dialects' -C API). +The dialect bindings live in `mlir/dialects/cute_ir` (`cute`), +`mlir/dialects/cute_nvgpu`, and `mlir/dialects/nvvm` and `mlir/dialects/cuda` +for NVIDIA's `nvvm` and `cuda` (operation builders under `.ops`), with types +and attributes through each dialect's C API. ## A kernel @@ -71,7 +72,7 @@ accessible afterward. The braces make the generated control flow visible and keep region-local values from escaping their scope; the builder's `open` and `yield` calls still create and close the actual IR regions. -| `cutlass.cute` | `kernels/cute` | +| `cutlass.cute` | `dsl/cute` | |--------------------------------------------|-------------------------------------------------| | `gA: cute.Tensor` kernel argument | `.gA = .{ .tensor = .{ .dtype, .shape } }` | | `gA[mi, ni]` / `gC[mi, ni] = v` | `gA.get(.{ mi, ni })` / `gC.set(.{ mi, ni }, v)`| @@ -85,7 +86,7 @@ keep region-local values from escaping their scope; the builder's `open` and | `cute.arch.griddepcontrol_launch_dependents()` | `b.launchDependents()` | | `x.to(Float32)` | `x.to(.f32)` | | `if cond:` / `for i in range(...)` | `b.openIf(cond)` / `b.openFor(lb, ub, step, .{})` | -| `cpasync.make_tiled_tma_atom(...)` | `b.makeTiledTmaLoadAtom(...)` / `...Typed(...)` | +| `cpasync.make_tiled_tma_atom(...)` | `b.makeTiledTmaLoadAtom(...)` | | `cpasync.tma_partition(...)` | `b.tmaPartition(...)` | | `cute.copy(atom, src, dst)` | `b.copy(atom, src, dst, pred)` | | `cute.make_tiled_mma(atom)` | `b.makeTiledMma(atom, result_type)` | @@ -105,8 +106,9 @@ keep region-local values from escaping their scope; the builder's `open` and `cst(dtype, v)` picks another type and `coerce` lifts literals to the other operand's type. * **`nvvm` and `cuda` are the compiler's.** The context registers NVIDIA's - `nvvm` and `cuda` dialects (`cute.nvvm`, `cute.cuda`, carried in - `mlir/dialects/cute_ir`) and upstream's `llvm`, so every operation is + `nvvm` and `cuda` dialects (`nvvm.ops`, `cuda.ops`, from the + `mlir/dialects/nvvm` and `mlir/dialects/cuda` packages) and upstream's + `llvm`, so every operation is verified as it is built; nothing is left unregistered. * **The module is just the kernel.** `finish` prints a module holding only `func.func @name(...)`; the CuTe compiler makes every public function a @@ -119,10 +121,10 @@ keep region-local values from escaping their scope; the builder's `open` and their layouts in the compiler, so callers spell out the resulting CuTe type with `View` instead of relying on Python's dynamic type objects. * **Not covered yet:** dynamic tensor shapes and automatic CuTe layout algebra. -* **Typed variants.** Operations whose result type CuTe derives exist in two - forms: one takes the result type as MLIR text (`localTile`, `slice`, - `tmaPartition`, ...), the `...Typed` one takes an `*mlir.Type` assembled - with `layoutSpec`, `memrefType` and `coordTensorType`. +* **Typed result types.** Operations whose result type CuTe derives + (`localTile`, `slice`, `tmaPartition`, ...) take an `*mlir.Type` assembled + with `layoutSpec`, `layoutType`, `memrefType`, `coordTensorType`, + `smemDescViewType` or `tiledCopyType`; nothing is parsed from text. * **Architecture sections.** `Builder` groups TMA (`cute.nvgpu.cpasync`, SM90+) and tcgen05 / tensor memory (`cute.nvgpu.tcgen05`, SM100) in their own sections; mbarriers, fences and bulk-copy groups sit with `cute.arch`. @@ -138,7 +140,7 @@ scheduler parameters) use `zml.kernel.cute.Program` instead of `Kernel`. Its buffers and calls `b.launchEx`. The custom call carries no grid or block: XLA runs the host function with the real buffers and launches (or records in a command buffer) whatever it launched. `CallOpts.scalars` are passed to the -host function after the buffers. `zml/moe/cute_kernels/persistent_mxfp4.zig` +host function after the buffers. `zml/kernels/moe/cute/persistent_mxfp4.zig` is the reference: a warp-specialized SM100 block-scaled GEMM whose host function builds four TMA tensor maps and a PDL launch. @@ -163,15 +165,15 @@ To see what XLA received, run with `--xla_dump_to=

`: it writes ```sh # the DSL: Value's fluent methods (add, mul, lt, to, exp, ...) -awk '/^pub const Value = struct/,/^};/' kernels/cute/builder.zig | grep 'pub fn ' +awk '/^pub const Value = struct/,/^};/' dsl/cute/builder.zig | grep 'pub fn ' # the DSL: everything on Tensor and Builder -awk '/^pub const Tensor = struct/,/^};/' kernels/cute/builder.zig | grep 'pub fn ' -awk '/^pub const Builder = struct/,0' kernels/cute/builder.zig | grep ' pub fn ' +awk '/^pub const Tensor = struct/,/^};/' dsl/cute/builder.zig | grep 'pub fn ' +awk '/^pub const Builder = struct/,0' dsl/cute/builder.zig | grep ' pub fn ' # every bound op of the dialect -grep -n '^pub fn ' mlir/dialects/cute_ir/cute_ir.zig mlir/dialects/cute_ir/cute_nvgpu.zig +grep -n '^pub fn ' mlir/dialects/cute_ir/cute_ir.zig mlir/dialects/cute_nvgpu/cute_nvgpu.zig ``` -`grep -n '^test "' kernels/cute/builder.zig` lists the tests; each prints a +`grep -n '^test "' dsl/cute/builder.zig` lists the tests; each prints a kernel that round-trips through the verifier. diff --git a/kernels/cute/builder.zig b/dsl/cute/builder.zig similarity index 72% rename from kernels/cute/builder.zig rename to dsl/cute/builder.zig index 286b92aa..59afe202 100644 --- a/kernels/cute/builder.zig +++ b/dsl/cute/builder.zig @@ -1,21 +1,22 @@ const std = @import("std"); -const cute = @import("mlir/dialects/cute_ir"); -const dialects = @import("mlir/dialects"); -const dsl = @import("kernels/common"); +const arith = @import("mlir/dialects/arith"); +const cf = @import("mlir/dialects/cf"); +const cuda = @import("mlir/dialects/cuda"); +const cute_ir = @import("mlir/dialects/cute_ir"); +const cute_nvgpu = @import("mlir/dialects/cute_nvgpu"); +const dsl = @import("dsl/common"); +const func = @import("mlir/dialects/func"); +const gpu = @import("mlir/dialects/gpu"); +const llvm = @import("mlir/dialects/llvm"); +const math = @import("mlir/dialects/math"); +const nvvm = @import("mlir/dialects/nvvm"); +const scf = @import("mlir/dialects/scf"); +const vector = @import("mlir/dialects/vector"); const tupleArity = dsl.tupleArity; const mlir = @import("mlir"); const stdx = @import("stdx"); -const arith = dialects.arith; -const cuda = dialects.cuda; -const func = dialects.func; -const gpu = dialects.gpu; -const llvm = dialects.llvm; -const math = dialects.math; -const nvvm = dialects.nvvm; -const vector = dialects.vector; - const dtypes = @import("dtype.zig"); pub const DType = dtypes.DType; const isFloatDtype = dtypes.isFloatDtype; @@ -30,23 +31,30 @@ test { std.testing.refAllDecls(Atom); } -/// `nvvm` and `cuda` are NVIDIA's (`cute.nvvm`, `cute.cuda`, handles `cute_nvvm` +/// `nvvm` and `cuda` are NVIDIA's (`nvvm.ops`, `cuda.ops`, handles `cute_nvvm` /// and `cuda`); `llvm` is upstream's, whose types and operations NVIDIA's /// compiler reads as its own. -pub const dialects_needed = [_][]const u8{ "func", "gpu", "cute", "cute_nvgpu", cute.nvvm.dialect_handle, cute.cuda.dialect_handle, "llvm", "arith", "scf", "math", "cf", "vector" }; +pub fn registerDialects(registry: *mlir.DialectRegistry) void { + inline for (.{ func, gpu, cute_ir, cute_nvgpu, nvvm, cuda, llvm, arith, scf, math, cf, vector }) |dialect| dialect.registerDialect(registry); +} pub const FinishError = error{InvalidMlir} || std.mem.Allocator.Error || std.Io.Writer.Error; pub const MAX_RANK = 8; pub const Dims = stdx.BoundedArray(i64, MAX_RANK); -pub const MemorySpace = cute.MemorySpace; +pub const MemorySpace = cute_ir.MemorySpace; /// A CuTe algebra expression assembled from Zig values. Kernel authors use -/// `Builder.expr`, `Builder.basis`, and `Builder.layoutSpec`; only this DSL -/// layer serializes the expression required by CuTe's C API. -pub const AlgebraExpr = struct { - text: []const u8, +/// `Builder.expr`, `Builder.basis`, and `Builder.layoutSpec`; the DSL builds +/// the algebra attribute of the kind each use needs. +pub const AlgebraExpr = union(enum) { + static: i64, + /// A dynamic leaf known to be a multiple of this, 1 when nothing is known. + dynamic: i64, + all, + basis: ScaledBasis, + tuple: []const AlgebraExpr, }; pub const AlgebraToken = enum { @@ -73,10 +81,6 @@ pub const ConstrainedDynamic = struct { pub const LayoutSpec = struct { shape: AlgebraExpr, stride: AlgebraExpr, - - pub fn payload(self: LayoutSpec, b: *Builder) []const u8 { - return std.fmt.allocPrint(b.arena.allocator(), "{s}:{s}", .{ self.shape.text, self.stride.text }) catch @panic("OOM"); - } }; /// A scalar SSA value (`Int32`, `Float32`, ... in the Python DSL). @@ -109,7 +113,7 @@ pub const Value = struct { } pub fn isPtr(self: Value) bool { - return self.type_().isA(cute.PtrType) != null; + return self.type_().isA(cute_ir.PtrType) != null; } pub fn add(self: Value, rhs: anytype) Value { @@ -362,7 +366,7 @@ pub const Tensor = struct { pub fn get(self: Tensor, coord: anytype) Value { const k = self.kernel; const c = k.makeCoord(coord); - return k.emit(cute.memref_load(k.ctx, self.inner, c, self.dtype.toMlir(k.ctx), k.loc())); + return k.emit(cute_ir.memref_load(k.ctx, self.inner, c, self.dtype.toMlir(k.ctx), k.loc())); } /// `t[coord] = value`; the value is converted to the element type. @@ -370,7 +374,7 @@ pub const Tensor = struct { const k = self.kernel; const c = k.makeCoord(coord); const v = k.liftAs(value_, self.dtype).to(self.dtype); - _ = cute.memref_store(k.ctx, self.inner, c, v.inner, k.loc()).appendTo(k.currentBlock()); + _ = cute_ir.memref_store(k.ctx, self.inner, c, v.inner, k.loc()).appendTo(k.currentBlock()); } /// Erase Zig's tracked layout while retaining the MLIR memref. CuTe @@ -401,11 +405,11 @@ pub const View = struct { pub fn get(self: View, coord: anytype, result_dtype: DType) Value { const c = self.kernel.makeCoord(coord); - return self.kernel.emit(cute.memref_load(self.kernel.ctx, self.inner, c, result_dtype.toMlir(self.kernel.ctx), self.kernel.loc())); + return self.kernel.emit(cute_ir.memref_load(self.kernel.ctx, self.inner, c, result_dtype.toMlir(self.kernel.ctx), self.kernel.loc())); } pub fn set(self: View, coord: anytype, value_: Value) void { - self.kernel.emitVoid(cute.memref_store(self.kernel.ctx, self.inner, self.kernel.makeCoord(coord), value_.inner, self.kernel.loc())); + self.kernel.emitVoid(cute_ir.memref_store(self.kernel.ctx, self.inner, self.kernel.makeCoord(coord), value_.inner, self.kernel.loc())); } }; @@ -429,7 +433,7 @@ pub const Atom = struct { self.kernel.asValue(value_) else self.kernel.lift(value_); - const result = self.kernel.emit(cute.nvgpu.atom_set_value( + const result = self.kernel.emit(cute_nvgpu.atom_set_value( self.kernel.ctx, self.inner, new_value.inner, @@ -441,21 +445,6 @@ pub const Atom = struct { } }; -/// Attribute payload accepted by target-specific CuTe operations. -pub const Attr = union(enum) { - str: []const u8, - text: []const u8, - raw: *const mlir.Attribute, - - pub fn get(self: Attr, ctx: *mlir.Context) *const mlir.Attribute { - return switch (self) { - .str => |s| .string(ctx, s), - .text => |s| mlir.Attribute.parse(ctx, s) catch std.debug.panic("invalid MLIR attribute: {s}", .{s}), - .raw => |a| a, - }; - } -}; - pub const TmaDescriptor = struct { atom: Atom, tensor: View, @@ -482,7 +471,6 @@ pub const TmaConfig = struct { coordinate_layout: LayoutSpec, coordinate_rank: u8, format: TmaFormat = .default, - num_multicast: u32 = 1, }; pub const BlockScaledMma = struct { @@ -499,50 +487,22 @@ pub const BlockScaledMmaConfig = struct { vec_size: u8 = 32, }; -fn dtypeName(dtype: DType) []const u8 { - return switch (dtype) { - .i1 => "i1", - .i8 => "i8", - .i16 => "i16", - .i32 => "i32", - .i64 => "i64", - .f16 => "f16", - .bf16 => "bf16", - .f32 => "f32", - .f64 => "f64", - .f4e2m1fn => "f4E2M1FN", - .f8e4m3fn => "f8E4M3FN", - .f8e5m2 => "f8E5M2", - .f8e8m0fnu => "f8E8M0FNU", - }; -} - -fn formatName(format: TmaFormat, dtype: DType) []const u8 { +fn tmaDataFormat(format: TmaFormat, dtype: DType) cute_nvgpu.TmaDataFormat { return switch (format) { - .u4_unpack_u8 => "U4_UNPACK_U8", - .u16 => "U16", + .u4_unpack_u8 => .U4_UNPACK_U8, + .u16 => .U16, .default => switch (dtype) { - .f8e4m3fn => "U8", - .f32 => "F32_RN", + .f8e4m3fn => .U8, + .f32 => .F32_RN, else => @panic("TMA format must be specified for this dtype"), }, }; } -fn tmaFormatAttribute(b: *Builder, format: TmaFormat) ?*const mlir.Attribute { - return switch (format) { - .default => null, - .u4_unpack_u8 => b.parseAttribute("#cute_nvgpu.tma_data_format"), - .u16 => b.parseAttribute("#cute_nvgpu.tma_data_format"), - }; -} - -fn blockScaledPayload(b: *Builder, cfg: BlockScaledMmaConfig) []const u8 { - return std.fmt.allocPrint( - b.arena.allocator(), - "<{d}x{d}x{d}, num_cta = {d}, ab_major = (k, k), elem_type = (f4E2M1FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, frag_kind = ss, vec_size = {d}>", - .{ cfg.m, cfg.n, cfg.k, cfg.num_cta, cfg.vec_size }, - ) catch @panic("OOM"); +fn tmaFormatAttribute(b: *Builder, config: TmaConfig) ?*const mlir.Attribute { + if (config.format == .default) return null; + const attr = cute_nvgpu.TmaDataFormatAttr.get(b.ctx, .{ .value = tmaDataFormat(config.format, config.dtype) }) catch @panic("invalid TMA format"); + return attr.attribute(); } pub const Index3 = struct { x: Value, y: Value, z: Value }; @@ -568,8 +528,6 @@ pub const ArgSpec = struct { tensor: TensorSpec, /// A type constructed through a dialect/type API. mlir_type: MlirTypeSpec, - /// A by-value kernel ABI argument such as a CUDA tensor-map atom. - raw: RawSpec, }; pub const MlirTypeSpec = struct { @@ -577,11 +535,6 @@ pub const ArgSpec = struct { grid_constant: bool = false, }; - pub const RawSpec = struct { - type_text: []const u8, - grid_constant: bool = false, - }; - pub const TensorSpec = struct { dtype: DType, shape: []const i64, @@ -689,10 +642,6 @@ pub const Builder = struct { .type_ = inner.type_, .grid_constant = if (@hasField(@TypeOf(inner), "grid_constant")) inner.grid_constant else false, } }, - .raw => .{ .raw = .{ - .type_text = inner.type_text, - .grid_constant = if (@hasField(@TypeOf(inner), "grid_constant")) inner.grid_constant else false, - } }, } }; } try self.declareArgsLow(arg_specs); @@ -730,7 +679,6 @@ pub const Builder = struct { .ptr => |dt| self.ptrTy(dt, .gmem, 16), .tensor => |t| self.ptrTy(t.dtype, .gmem, t.alignment), .mlir_type => |typed| typed.type_, - .raw => |raw| self.parseType(raw.type_text), } catch return error.InvalidMlir; l.* = self.loc(); } @@ -744,9 +692,6 @@ pub const Builder = struct { .mlir_type => |typed| if (typed.grid_constant) .dict(ctx, &.{ .named(ctx, "cute_nvgpu.grid_constant", .unit(ctx)), }) else empty_dict, - .raw => |raw| if (raw.grid_constant) .dict(ctx, &.{ - .named(ctx, "cute_nvgpu.grid_constant", .unit(ctx)), - }) else empty_dict, else => empty_dict, }; if (attr.* != empty_dict) any_arg_attr = true; @@ -759,7 +704,6 @@ pub const Builder = struct { .args_attributes = if (any_arg_attr) arg_attrs else null, .results = &.{}, .location = self.loc(), - .visibility = null, .verify = false, .extra_attributes = &.{ .named(ctx, "cute.kernel", .unit(ctx)), @@ -783,7 +727,6 @@ pub const Builder = struct { .args_attributes = if (any_arg_attr) arg_attrs else null, .results = &.{.int(ctx, .i32)}, .location = self.loc(), - .visibility = null, .verify = false, .extra_attributes = &.{.named(ctx, "llvm.emit_c_interface", .unit(ctx))}, }), @@ -805,7 +748,6 @@ pub const Builder = struct { .stride = spec.stride orelse self.rowMajor(spec.shape), })), .mlir_type => null, - .raw => null, }; } } @@ -887,6 +829,12 @@ pub const Builder = struct { return .unknown(self.ctx); } + /// `#cuda.dev_max_shared_memory_optin`, the device's opt-in maximum of + /// shared memory per block, for a kernel's `cu_attrs`. + pub fn devMaxSharedMemoryOptin(self: *Builder) *const mlir.Attribute { + return (cuda.ops.DevMaxSharedMemoryOptinAttr.get(self.ctx, .{}) catch @panic("invalid CUDA attribute")).attribute(); + } + pub fn setFunctionAttribute(self: *Builder, name: []const u8, attribute: *const mlir.Attribute) void { const function = self.func_op orelse @panic("setFunctionAttribute called before declareArgs"); function.setAttributeByName(name, attribute); @@ -894,12 +842,16 @@ pub const Builder = struct { // ==================== cuda host launch ==================== + /// `@kernels::@`, a kernel of the nested GPU module. + fn kernelSymbol(self: *Builder, kernel_name: []const u8) *const mlir.Attribute { + return .symbolRef(self.ctx, "kernels", &.{.init(self.ctx, kernel_name)}); + } + pub fn kernelSmemSize(self: *Builder, kernel_name: []const u8) Value { - const symbol = std.fmt.allocPrint(self.arena.allocator(), "@kernels::@{s}", .{kernel_name}) catch @panic("OOM"); - return self.emit(cute.kernel_smem_size( + return self.emit(cute_ir.kernel_smem_size( self.ctx, .int(self.ctx, .i64), - self.parseAttribute(symbol), + self.kernelSymbol(kernel_name), self.loc(), )); } @@ -944,11 +896,10 @@ pub const Builder = struct { /// actual launch on its stream. pub fn launchEx(self: *Builder, kernel_name: []const u8, config: LaunchConfig, arguments: anytype) Value { const values = self.innerSlice(arguments); - const symbol = std.fmt.allocPrint(self.arena.allocator(), "@kernels::@{s}", .{kernel_name}) catch @panic("OOM"); return self.emit(cuda.launch_ex(self.ctx, .{ .config = config.inner, .inputs = values, - .callee = self.parseAttribute(symbol), + .callee = self.kernelSymbol(kernel_name), .assume_kernel_attr = cuda.assumeKernelAttr(self.ctx, true), .location = self.loc(), })); @@ -1004,29 +955,35 @@ pub const Builder = struct { // ==================== types ==================== pub fn ptrTy(self: *Builder, dt: DType, space: MemorySpace, alignment: u64) !*const mlir.Type { - return cute.pointerType(self.ctx, dt.toMlir(self.ctx), space, alignment); + return cute_ir.pointerType(self.ctx, space, alignment, .{ .element_type = dt.toMlir(self.ctx) }); } pub fn swizzledPtrTy(self: *Builder, dt: DType, space: MemorySpace, alignment: u64, swizzle: Swizzle) !*const mlir.Type { - const text = std.fmt.allocPrint(self.arena.allocator(), "S<{d},{d},{d}>", .{ swizzle.bits, swizzle.base, swizzle.shift }) catch @panic("OOM"); - const attr = try cute.SwizzleAttr.get(self.ctx, text); - return (try cute.PtrType.get(self.ctx, .{ + return (try cute_ir.PtrType.get(self.ctx, .{ .valueType = dt.toMlir(self.ctx), .addressSpace = space, .alignment = alignment, - .swizzle = attr.attribute(), + .swizzle = (try self.swizzleAttr(swizzle)).attribute(), })).type_(); } + fn swizzleAttr(self: *Builder, swizzle: Swizzle) !*const cute_ir.SwizzleAttr { + return cute_ir.SwizzleAttr.get(self.ctx, .{ + .bits = std.math.cast(u32, swizzle.bits) orelse return error.InvalidMlir, + .base = std.math.cast(u32, swizzle.base) orelse return error.InvalidMlir, + .shift = std.math.cast(i32, swizzle.shift) orelse return error.InvalidMlir, + }); + } + pub fn intToPtr(self: *Builder, address: anytype, dt: DType, space: MemorySpace, alignment: u64) Value { const ptr_type = self.ptrTy(dt, space, alignment) catch @panic("invalid CuTe pointer type"); - return self.emit(cute.inttoptr(self.ctx, self.lift(address).inner, ptr_type, self.loc())); + return self.emit(cute_ir.inttoptr(self.ctx, self.lift(address).inner, ptr_type, self.loc())); } /// `cute.ptrtoint`: the integer address of a pointer (`i32` for shared /// memory, `i64` for global memory), for hand-written PTX operands. pub fn ptrToInt(self: *Builder, pointer: anytype, dt: DType) Value { - return self.emit(cute.ptrtoint(self.ctx, self.asValue(pointer).inner, dt.toMlir(self.ctx), self.loc())); + return self.emit(cute_ir.ptrtoint(self.ctx, self.asValue(pointer).inner, dt.toMlir(self.ctx), self.loc())); } /// Reinterpret a pointer without changing its address. This is the typed @@ -1034,89 +991,82 @@ pub const Builder = struct { /// to expose their byte storage to a scalar correctness implementation. pub fn recastPointer(self: *Builder, pointer: anytype, dt: DType, space: MemorySpace, alignment: u64) Value { const result_type = self.ptrTy(dt, space, alignment) catch @panic("invalid recast pointer type"); - return self.emit(cute.recast_iter(self.ctx, self.asValue(pointer).inner, result_type, self.loc())); + return self.emit(cute_ir.recast_iter(self.ctx, self.asValue(pointer).inner, result_type, self.loc())); } pub fn recastSwizzledPointer(self: *Builder, pointer: anytype, dt: DType, space: MemorySpace, alignment: u64, swizzle: Swizzle) Value { const result_type = self.swizzledPtrTy(dt, space, alignment, swizzle) catch @panic("invalid swizzled pointer type"); - return self.emit(cute.recast_iter(self.ctx, self.asValue(pointer).inner, result_type, self.loc())); + return self.emit(cute_ir.recast_iter(self.ctx, self.asValue(pointer).inner, result_type, self.loc())); } - /// Parse a CuTe/CuTe-NVGPU type that has no compact C-API constructor. - /// This is primarily used for architecture-specific atom and descriptor - /// types whose complete layout is part of the type itself. - pub fn parseType(self: *Builder, text: []const u8) *const mlir.Type { - return mlir.Type.parse(self.ctx, text) catch std.debug.panic("invalid MLIR type: {s}", .{text}); + /// The scalar of a single extent, otherwise the tuple of `dims`. + fn dimsExpr(self: *Builder, dims: []const i64) AlgebraExpr { + if (dims.len == 1) return .{ .static = dims[0] }; + const elements = self.arena.allocator().alloc(AlgebraExpr, dims.len) catch @panic("OOM"); + for (dims, elements) |d, *e| e.* = .{ .static = d }; + return .{ .tuple = elements }; } - /// Parse a target-specific MLIR attribute. - pub fn parseAttribute(self: *Builder, text: []const u8) *const mlir.Attribute { - return mlir.Attribute.parse(self.ctx, text) catch std.debug.panic("invalid MLIR attribute: {s}", .{text}); - } - - fn algebra(self: *Builder, dims: []const i64) []const u8 { - var out: std.Io.Writer.Allocating = .init(self.arena.allocator()); - if (dims.len == 1) { - out.writer.print("{d}", .{dims[0]}) catch @panic("OOM"); - } else { - out.writer.writeByte('(') catch @panic("OOM"); - for (dims, 0..) |d, i| { - if (i > 0) out.writer.writeByte(',') catch @panic("OOM"); - out.writer.print("{d}", .{d}) catch @panic("OOM"); - } - out.writer.writeByte(')') catch @panic("OOM"); - } - return out.written(); - } - - fn writeAlgebra(self: *Builder, writer: *std.Io.Writer, value: anytype) void { + /// Build a CuTe algebra expression from integers, nested Zig tuples, + /// `AlgebraToken`s, `ConstrainedDynamic`s and `ScaledBasis` values. + pub fn expr(self: *Builder, value: anytype) AlgebraExpr { const T = @TypeOf(value); - if (T == AlgebraExpr) { - writer.writeAll(value.text) catch @panic("OOM"); - return; - } - if (T == AlgebraToken) { - writer.writeByte(switch (value) { - .all => '_', - .dynamic => '?', - }) catch @panic("OOM"); - return; - } - if (T == ScaledBasis) { - writer.print("{d}", .{value.numerator}) catch @panic("OOM"); - if (value.denominator != 1) writer.print("/{d}", .{value.denominator}) catch @panic("OOM"); - for (value.modes[0..value.mode_count]) |mode| writer.print("@{d}", .{mode}) catch @panic("OOM"); - return; - } + if (T == AlgebraExpr) return value; + if (T == AlgebraToken) return switch (value) { + .all => .all, + .dynamic => .{ .dynamic = 1 }, + }; + if (T == ScaledBasis) return .{ .basis = value }; if (T == ConstrainedDynamic) { if (value.divisible_by == 0) @panic("CuTe constrained dynamic divisor must be positive"); - writer.print("?{{div={d}}}", .{value.divisible_by}) catch @panic("OOM"); - return; + return .{ .dynamic = @intCast(value.divisible_by) }; } switch (@typeInfo(T)) { - .comptime_int, .int => writer.print("{d}", .{value}) catch @panic("OOM"), + .comptime_int, .int => return .{ .static = @intCast(value) }, .@"struct" => |info| { - writer.writeByte('(') catch @panic("OOM"); - inline for (info.field_names, 0..) |field, i| { - if (i != 0) writer.writeByte(',') catch @panic("OOM"); - self.writeAlgebra(writer, @field(value, field)); - } - writer.writeByte(')') catch @panic("OOM"); + const elements = self.arena.allocator().alloc(AlgebraExpr, info.field_names.len) catch @panic("OOM"); + inline for (info.field_names, 0..) |field, i| elements[i] = self.expr(@field(value, field)); + return .{ .tuple = elements }; }, else => @compileError("unsupported CuTe algebra value " ++ @typeName(T)), } } - /// Build a CuTe algebra expression from integers, nested Zig tuples, - /// `AlgebraToken`s and `ScaledBasis` values. - pub fn expr(self: *Builder, value: anytype) AlgebraExpr { - var out: std.Io.Writer.Allocating = .init(self.arena.allocator()); - self.writeAlgebra(&out.writer, value); - return .{ .text = out.written() }; + /// The algebra attribute `A` (`cute_ir.IntTupleAttr`, `ShapeAttr`, ...) + /// of `e`. + fn algebraAttr(self: *Builder, comptime A: type, e: AlgebraExpr) *const A { + const result = switch (e) { + .static => |v| if (@hasDecl(A, "getStatic")) A.getStatic(self.ctx, v) else std.debug.panic("{s} has no integer leaf", .{@typeName(A)}), + .dynamic => |divisibility| if (@hasDecl(A, "getDynamic")) A.getDynamic(self.ctx, 32, divisibility) else std.debug.panic("{s} has no dynamic leaf", .{@typeName(A)}), + .all => if (@hasDecl(A, "getUnderscore")) A.getUnderscore(self.ctx) else std.debug.panic("{s} has no `_` leaf", .{@typeName(A)}), + .basis => |b| if (@hasDecl(A, "getBasis")) blk: { + const modes = self.arena.allocator().alloc(i32, b.mode_count) catch @panic("OOM"); + for (modes, b.modes[0..b.mode_count]) |*mode, m| mode.* = m; + break :blk A.getBasis(self.ctx, b.numerator, b.denominator, modes); + } else std.debug.panic("{s} has no scaled-basis leaf", .{@typeName(A)}), + .tuple => |elements| blk: { + const attrs = self.arena.allocator().alloc(*const A, elements.len) catch @panic("OOM"); + for (elements, attrs) |element, *attr| attr.* = self.algebraAttr(A, element); + break :blk A.getTuple(self.ctx, attrs); + }, + }; + return result catch std.debug.panic("invalid CuTe algebra for {s}", .{@typeName(A)}); } - /// `coefficient @ mode...`, including rational coefficients such as - /// `1/2@0`. The mode path maps a coordinate into a nested CuTe mode. + fn layoutAttr(self: *Builder, spec: LayoutSpec) *const cute_ir.LayoutAttr { + return cute_ir.LayoutAttr.get(self.ctx, .{ + .shape = self.algebraAttr(cute_ir.ShapeAttr, spec.shape), + .stride = self.algebraAttr(cute_ir.StrideAttr, spec.stride), + }) catch @panic("invalid CuTe layout"); + } + + fn intTupleType(self: *Builder, e: AlgebraExpr) *const mlir.Type { + return (cute_ir.IntTupleType.get(self.ctx, .{ .attr = self.algebraAttr(cute_ir.IntTupleAttr, e).attribute() }) catch @panic("invalid CuTe integer tuple")).type_(); + } + + /// `numerator/denominator` along `modes`, the mode path outermost first like + /// Python's `ScaledBasis(value, modes)`. CuTe prints it innermost first: + /// `basis(1, 2, .{ 0, 1 })` is `1/2@1@0`. pub fn basis(self: *Builder, numerator: i64, denominator: i64, modes: anytype) ScaledBasis { _ = self; if (denominator <= 0) @panic("CuTe scaled-basis denominator must be positive"); @@ -1132,62 +1082,76 @@ pub const Builder = struct { } pub fn layoutType(self: *Builder, spec: LayoutSpec) *const mlir.Type { - return cute.algebraType(self.ctx, .layout, spec.payload(self)) catch @panic("invalid CuTe layout"); + return (cute_ir.LayoutType.get(self.ctx, .{ .attr = self.layoutAttr(spec).attribute() }) catch @panic("invalid CuTe layout")).type_(); } pub fn composedLayoutType(self: *Builder, swizzle: Swizzle, offset: anytype, outer: LayoutSpec) *const mlir.Type { - const payload = std.fmt.allocPrint( - self.arena.allocator(), - "S<{d},{d},{d}> o {s} o {s}", - .{ swizzle.bits, swizzle.base, swizzle.shift, self.expr(offset).text, outer.payload(self) }, - ) catch @panic("OOM"); - return cute.algebraType(self.ctx, .composed_layout, payload) catch @panic("invalid CuTe composed layout"); + const attr = cute_ir.ComposedLayoutAttr.get(self.ctx, .{ + .inner = .{ .swizzle = self.swizzleAttr(swizzle) catch @panic("invalid CuTe swizzle") }, + .offset = self.algebraAttr(cute_ir.IntTupleAttr, self.expr(offset)), + .outer = self.layoutAttr(outer), + }) catch @panic("invalid CuTe composed layout"); + return (cute_ir.ComposedLayoutType.get(self.ctx, .{ .attr = attr.attribute() }) catch @panic("invalid CuTe composed layout")).type_(); } pub fn memrefType(self: *Builder, dtype: DType, space: MemorySpace, alignment: u64, layout_type: *const mlir.Type) *const mlir.Type { const pointer = self.ptrTy(dtype, space, alignment) catch @panic("invalid CuTe pointer type"); - return (cute.MemRefType.get(self.ctx, .{ .ptr = pointer, .layout = layout_type }) catch @panic("invalid CuTe memref type")).type_(); + return (cute_ir.MemRefType.get(self.ctx, .{ .ptr = pointer, .layout = layout_type }) catch @panic("invalid CuTe memref type")).type_(); + } + + pub fn smemDescType(self: *Builder) *const mlir.Type { + return (cute_nvgpu.SmemDescType.get(self.ctx, .{}) catch @panic("invalid shared-memory descriptor")).type_(); + } + + /// `!cute_nvgpu.smem_desc_view` of a shared-memory descriptor over `layout`. + pub fn smemDescViewType(self: *Builder, layout: LayoutSpec) *const mlir.Type { + return (cute_nvgpu.SmemDescViewType.get(self.ctx, .{ + .desc = self.smemDescType(), + .layout = self.layoutAttr(layout).attribute(), + }) catch @panic("invalid shared-memory descriptor view")).type_(); + } + + /// `!cute.tiled_copy` of `copy_atom` over `layout_copy_tv`, tiled by the + /// modes of `tiler_mn`. + pub fn tiledCopyType(self: *Builder, copy_atom: *const mlir.Type, layout_copy_tv: LayoutSpec, tiler_mn: []const LayoutSpec) *const mlir.Type { + return (cute_ir.TiledCopyType.get(self.ctx, .{ + .copyAtom = copy_atom, + .layoutCopyTv = self.layoutAttr(layout_copy_tv).attribute(), + .tilerMn = self.tileAttr(tiler_mn).attribute(), + }) catch @panic("invalid tiled copy")).type_(); } pub fn memrefTypeFromPointer(self: *Builder, pointer_type: *const mlir.Type, layout_type: *const mlir.Type) *const mlir.Type { - return (cute.MemRefType.get(self.ctx, .{ .ptr = pointer_type, .layout = layout_type }) catch @panic("invalid CuTe memref type")).type_(); + return (cute_ir.MemRefType.get(self.ctx, .{ .ptr = pointer_type, .layout = layout_type }) catch @panic("invalid CuTe memref type")).type_(); } - pub fn staticLayout(self: *Builder, spec: LayoutSpec) View { - const result = self.emit(cute.static(self.ctx, self.layoutType(spec), self.loc())); + /// `cute.static` of an integer tuple such as `128` or `.{ 0, 4 }`. + pub fn staticIntTuple(self: *Builder, value: anytype) View { + const result = self.emit(cute_ir.static(self.ctx, self.intTupleType(self.expr(value)), self.loc())); return .{ .inner = result.inner, .kernel = self }; } - /// Materialize a rational scaled-basis layout used by block-scale TMA. - /// The local standalone LayoutAttr constructor only accepts integer - /// strides, while CoordTensorType owns the more general coordinate - /// algebra. Recovering its layout through the typed C API avoids exposing - /// that parser distinction to kernel code. - pub fn staticCoordinateLayout(self: *Builder, rank: usize, spec: LayoutSpec) View { - const tensor_type = self.coordTensorType(rank, spec); - const coords = tensor_type.isA(cute.CoordTensorType) orelse @panic("invalid coordinate tensor type"); - const result = self.emit(cute.static(self.ctx, coords.getLayout(), self.loc())); + pub fn staticLayout(self: *Builder, spec: LayoutSpec) View { + const result = self.emit(cute_ir.static(self.ctx, self.layoutType(spec), self.loc())); return .{ .inner = result.inner, .kernel = self }; } pub const Swizzle = struct { bits: i64, base: i64, shift: i64 }; pub fn staticSwizzle(self: *Builder, spec: Swizzle) View { - const payload = std.fmt.allocPrint(self.arena.allocator(), "S<{d},{d},{d}>", .{ spec.bits, spec.base, spec.shift }) catch @panic("OOM"); - const ty = cute.algebraType(self.ctx, .swizzle, payload) catch @panic("invalid CuTe swizzle"); - const result = self.emit(cute.static(self.ctx, ty, self.loc())); + const attr = self.swizzleAttr(spec) catch @panic("invalid CuTe swizzle"); + const ty = (cute_ir.SwizzleType.get(self.ctx, .{ .attr = attr.attribute() }) catch @panic("invalid CuTe swizzle")).type_(); + const result = self.emit(cute_ir.static(self.ctx, ty, self.loc())); return .{ .inner = result.inner, .kernel = self }; } /// Construct `swizzle o offset o outer_layout` through the CuTe dialect. pub fn makeComposedLayout(self: *Builder, swizzle: Swizzle, offset: anytype, outer: LayoutSpec) View { const inner = self.staticSwizzle(swizzle); - const offset_expr = self.expr(offset); - const offset_ty = cute.algebraType(self.ctx, .int_tuple, offset_expr.text) catch @panic("invalid CuTe composed-layout offset"); - const offset_value = self.emit(cute.make_int_tuple(self.ctx, &.{}, offset_ty, self.loc())); + const offset_value = self.emit(cute_ir.make_int_tuple(self.ctx, &.{}, self.intTupleType(self.expr(offset)), self.loc())); const outer_value = self.staticLayout(outer); const result_ty = self.composedLayoutType(swizzle, offset, outer); - const result = self.emit(cute.make_composed_layout(self.ctx, inner.inner, offset_value.inner, outer_value.inner, result_ty, self.loc())); + const result = self.emit(cute_ir.make_composed_layout(self.ctx, inner.inner, offset_value.inner, outer_value.inner, result_ty, self.loc())); return .{ .inner = result.inner, .kernel = self }; } @@ -1196,49 +1160,23 @@ pub const Builder = struct { pub fn makeTensorView(self: *Builder, pointer: anytype, layout: anytype) View { const p = self.asValue(pointer); const l = self.asValue(layout); - const result_ty = (cute.MemRefType.get(self.ctx, .{ .ptr = p.type_(), .layout = l.type_() }) catch @panic("invalid CuTe tensor view")).type_(); - const result = self.emit(cute.make_view(self.ctx, p.inner, l.inner, result_ty, self.loc())); + const result_ty = (cute_ir.MemRefType.get(self.ctx, .{ .ptr = p.type_(), .layout = l.type_() }) catch @panic("invalid CuTe tensor view")).type_(); + const result = self.emit(cute_ir.make_view(self.ctx, p.inner, result_ty, .{ .layout = l.inner }, self.loc())); return .{ .inner = result.inner, .kernel = self }; } pub fn coordTensorType(self: *Builder, rank: usize, layout: LayoutSpec) *const mlir.Type { if (rank == 0 or rank > MAX_RANK) @panic("coordTensorType rank must be in 1..MAX_RANK"); - var zeros: std.Io.Writer.Allocating = .init(self.arena.allocator()); - if (rank > 1) zeros.writer.writeByte('(') catch @panic("OOM"); - for (0..rank) |i| { - if (i != 0) zeros.writer.writeByte(',') catch @panic("OOM"); - zeros.writer.writeByte('0') catch @panic("OOM"); - } - if (rank > 1) zeros.writer.writeByte(')') catch @panic("OOM"); - const tuple = cute.algebraType(self.ctx, .int_tuple, zeros.written()) catch @panic("invalid coordinate tuple"); - const payload = layout.payload(self); - // The standalone LayoutAttr C API intentionally restricts scaled - // basis coefficients to integers. CoordTensorType additionally - // accepts the rational basis used by block-scaled scale-factor maps. - // Keep that parser workaround inside the DSL type constructor. - if (std.mem.indexOfScalar(u8, payload, '/') != null) { - const type_text = std.fmt.allocPrint(self.arena.allocator(), "!cute.coord_tensor<\"{s}\", \"{s}\">", .{ zeros.written(), payload }) catch @panic("OOM"); - return self.parseType(type_text); - } - return (cute.CoordTensorType.get(self.ctx, .{ .arithTuple = tuple, .layout = self.layoutType(layout) }) catch @panic("invalid coordinate tensor")).type_(); + const zeros: [MAX_RANK]i64 = @splat(0); + return self.coordTensorTypeWithTuple(self.dimsExpr(zeros[0..rank]), layout); } - /// Coordinate-tensor type whose coordinate tuple contains dynamic and - /// static leaves. `tuple_payload` is CuTe algebra such as `(0,?,?)`; - /// layouts remain structured `LayoutSpec` values in Zig. - pub fn coordTensorTypePayload(self: *Builder, tuple_payload: []const u8, layout: LayoutSpec) *const mlir.Type { - const attr = cute.IntTupleAttr.get(self.ctx, tuple_payload) catch std.debug.panic("invalid coordinate tuple {s}", .{tuple_payload}); - const tuple = (cute.IntTupleType.get(self.ctx, .{ .attr = attr.attribute() }) catch @panic("invalid coordinate tuple type")).type_(); - return (cute.CoordTensorType.get(self.ctx, .{ .arithTuple = tuple, .layout = self.layoutType(layout) }) catch @panic("invalid coordinate tensor")).type_(); - } - - /// Coordinate-tensor type with an explicit arithmetic-tuple expression. + /// Coordinate-tensor type with an explicit arithmetic-tuple expression, + /// such as `.{ 0, ConstrainedDynamic{ .divisible_by = 128 }, AlgebraToken.dynamic }`. /// This represents the divisibility information produced by CuTe tiling /// without requiring kernel code to spell textual MLIR types. pub fn coordTensorTypeWithTuple(self: *Builder, tuple: anytype, layout: LayoutSpec) *const mlir.Type { - const tuple_text = self.expr(tuple).text; - const tuple_type = cute.algebraType(self.ctx, .int_tuple, tuple_text) catch std.debug.panic("invalid coordinate tuple {s}", .{tuple_text}); - return (cute.CoordTensorType.get(self.ctx, .{ .arithTuple = tuple_type, .layout = self.layoutType(layout) }) catch @panic("invalid coordinate tensor")).type_(); + return (cute_ir.CoordTensorType.get(self.ctx, .{ .arithTuple = self.intTupleType(self.expr(tuple)), .layout = self.layoutType(layout) }) catch @panic("invalid coordinate tensor")).type_(); } fn rowMajor(self: *Builder, shape: []const i64) []const i64 { @@ -1264,61 +1202,45 @@ pub const Builder = struct { // ==================== cute.core ==================== - /// Materialize a value whose complete static CuTe type is already known. - /// - /// Nested layouts and integer tuples produced by CuTe's layout algebra - /// cannot be represented by the flat `Layout` helper. Keeping this - /// escape hatch typed still lets MLIR verify every consumer. - pub fn staticValue(self: *Builder, type_text: []const u8) View { - const result = self.emit(cute.static(self.ctx, self.parseType(type_text), self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - /// `cute.make_layout(shape, stride=...)`, static only. pub fn makeLayout(self: *Builder, shape: []const i64, opts: LayoutOpts) Layout { if (shape.len == 0 or shape.len > MAX_RANK) std.debug.panic("makeLayout: rank {d} not in 1..{d}", .{ shape.len, MAX_RANK }); const stride = opts.stride orelse self.leftMost(shape); if (stride.len != shape.len) std.debug.panic("makeLayout: stride rank {d} != shape rank {d}", .{ stride.len, shape.len }); - const ctx = self.ctx; - const text = std.fmt.allocPrint(self.arena.allocator(), "{s}:{s}", .{ self.algebra(shape), self.algebra(stride) }) catch @panic("OOM"); - const layout_ty = cute.algebraType(ctx, .layout, text) catch @panic("bad layout"); - const l = self.emit(cute.static(ctx, layout_ty, self.loc())); + const layout_ty = self.layoutType(.{ .shape = self.dimsExpr(shape), .stride = self.dimsExpr(stride) }); + const l = self.emit(cute_ir.static(self.ctx, layout_ty, self.loc())); return .{ .inner = l.inner, .shape = Dims.fromSlice(shape) catch unreachable, .stride = Dims.fromSlice(stride) catch unreachable }; } /// `cute.make_tensor(ptr, layout)`. pub fn makeTensor(self: *Builder, ptr: Value, layout: Layout) Tensor { - const pt = ptr.type_().isA(cute.PtrType) orelse std.debug.panic("makeTensor: {f} is not a !cute.ptr", .{ptr.type_()}); - const elem = pt.getValueType() orelse @panic("makeTensor: untyped pointer"); - const memref_ty = (cute.MemRefType.get(self.ctx, .{ .ptr = ptr.type_(), .layout = layout.inner.type_() }) catch @panic("bad memref")).type_(); - const v = self.emit(cute.make_view(self.ctx, ptr.inner, layout.inner, memref_ty, self.loc())); - return .{ .inner = v.inner, .kernel = self, .layout = layout, .dtype = dtypes.mlirToDType(self.ctx, elem), .space = pt.getAddressSpace() }; + const pt = ptr.type_().isA(cute_ir.PtrType) orelse std.debug.panic("makeTensor: {f} is not a !cute.ptr", .{ptr.type_()}); + const elem = pt.valueType() orelse @panic("makeTensor: untyped pointer"); + const memref_ty = (cute_ir.MemRefType.get(self.ctx, .{ .ptr = ptr.type_(), .layout = layout.inner.type_() }) catch @panic("bad memref")).type_(); + const v = self.emit(cute_ir.make_view(self.ctx, ptr.inner, memref_ty, .{ .layout = layout.inner }, self.loc())); + return .{ .inner = v.inner, .kernel = self, .layout = layout, .dtype = dtypes.mlirToDType(self.ctx, elem), .space = pt.addressSpace() }; } - fn writeCoord(self: *Builder, writer: *std.Io.Writer, dyn: *stdx.BoundedArray(*const mlir.Value, MAX_RANK), coord: anytype) void { + /// The algebra of a coordinate whose `Value` leaves are dynamic and + /// appended to `dyn`. A one-element tuple is its element. + fn coordExpr(self: *Builder, dyn: *stdx.BoundedArray(*const mlir.Value, MAX_RANK), coord: anytype) AlgebraExpr { const T = @TypeOf(coord); if (T == Value) { - writer.writeByte('?') catch @panic("OOM"); dyn.append(self.lift(coord).to(.i32).inner) catch @panic("makeCoord: too many dynamic leaves"); - return; - } - if (T == AlgebraToken) { - writer.writeByte(switch (coord) { - .all => '_', - .dynamic => @panic("makeCoord: AlgebraToken.dynamic needs an SSA value"), - }) catch @panic("OOM"); - return; + return .{ .dynamic = 1 }; } + if (T == AlgebraToken) return switch (coord) { + .all => .all, + .dynamic => @panic("makeCoord: AlgebraToken.dynamic needs an SSA value"), + }; switch (@typeInfo(T)) { - .comptime_int, .int => writer.print("{d}", .{coord}) catch @panic("OOM"), + .comptime_int, .int => return .{ .static = @intCast(coord) }, .@"struct" => |info| { if (info.field_names.len == 0) @compileError("makeCoord: empty tuple"); - if (info.field_names.len > 1) writer.writeByte('(') catch @panic("OOM"); - inline for (info.field_names, 0..) |field, i| { - if (i != 0) writer.writeByte(',') catch @panic("OOM"); - self.writeCoord(writer, dyn, @field(coord, field)); - } - if (info.field_names.len > 1) writer.writeByte(')') catch @panic("OOM"); + if (info.field_names.len == 1) return self.coordExpr(dyn, @field(coord, info.field_names[0])); + const elements = self.arena.allocator().alloc(AlgebraExpr, info.field_names.len) catch @panic("OOM"); + inline for (info.field_names, 0..) |field, i| elements[i] = self.coordExpr(dyn, @field(coord, field)); + return .{ .tuple = elements }; }, else => @compileError("makeCoord: unsupported coordinate leaf " ++ @typeName(T)), } @@ -1328,22 +1250,19 @@ pub const Builder = struct { /// placeholders. Integer leaves stay static in the dependent CuTe type. pub fn makeCoord(self: *Builder, coord: anytype) *const mlir.Value { if (@TypeOf(coord) == Value) return self.makeCoord(.{coord}); - var text: std.Io.Writer.Allocating = .init(self.arena.allocator()); var dyn: stdx.BoundedArray(*const mlir.Value, MAX_RANK) = .empty; - self.writeCoord(&text.writer, &dyn, coord); - const ty = cute.algebraType(self.ctx, .coord, text.written()) catch @panic("bad coord"); - return self.emit(cute.make_coord(self.ctx, dyn.constSlice(), ty, self.loc())).inner; + const attr = self.algebraAttr(cute_ir.CoordAttr, self.coordExpr(&dyn, coord)); + const ty = (cute_ir.CoordType.get(self.ctx, .{ .attr = attr.attribute() }) catch @panic("bad coord")).type_(); + return self.emit(cute_ir.make_coord(self.ctx, dyn.constSlice(), ty, self.loc())).inner; } /// A dynamic/static CuTe integer tuple, used for pointer arithmetic in /// tensor and shared memory. It accepts the same nested Zig values as a /// coordinate, but preserves integer-tuple semantics. pub fn makeIntTuple(self: *Builder, values: anytype) View { - var text: std.Io.Writer.Allocating = .init(self.arena.allocator()); var dyn: stdx.BoundedArray(*const mlir.Value, MAX_RANK) = .empty; - self.writeCoord(&text.writer, &dyn, values); - const ty = cute.algebraType(self.ctx, .int_tuple, text.written()) catch @panic("bad integer tuple"); - const value = self.emit(cute.make_int_tuple(self.ctx, dyn.constSlice(), ty, self.loc())); + const ty = self.intTupleType(self.coordExpr(&dyn, values)); + const value = self.emit(cute_ir.make_int_tuple(self.ctx, dyn.constSlice(), ty, self.loc())); return .{ .inner = value.inner, .kernel = self }; } @@ -1352,8 +1271,7 @@ pub const Builder = struct { /// `cute.add_offset`, need that to keep the operand's alignment. pub fn makeIntTupleDivBy(self: *Builder, value: anytype, comptime divisor: u32) View { const constrained = self.assumeDivBy(value, divisor); - const payload = std.fmt.allocPrint(self.arena.allocator(), "!cute.int_tuple<\"?{{div={d}}}\">", .{divisor}) catch @panic("OOM"); - const tuple = self.emit(cute.make_int_tuple(self.ctx, &.{constrained.inner}, self.parseType(payload), self.loc())); + const tuple = self.emit(cute_ir.make_int_tuple(self.ctx, &.{constrained.inner}, self.intTupleType(.{ .dynamic = divisor }), self.loc())); return .{ .inner = tuple.inner, .kernel = self }; } @@ -1361,23 +1279,8 @@ pub const Builder = struct { /// derives while slicing a tiled coordinate tensor. pub fn assumeDivBy(self: *Builder, value: anytype, comptime divisor: u32) Value { if (divisor == 0) @compileError("assumeDivBy divisor must be non-zero"); - const type_text = std.fmt.allocPrint(self.arena.allocator(), "!cute.i32", .{divisor}) catch @panic("OOM"); - return self.emit(cute.assume(self.ctx, self.lift(value).to(.i32).inner, self.parseType(type_text), self.loc())); - } - - /// Build a coordinate tensor with an explicit dependent arithmetic-tuple - /// type. This is the source-level equivalent of the tuple produced by - /// `slice` after TMA partitioning (for example `?{div=128}`). - pub fn makeCoordTensorWithTuple(self: *Builder, values: anytype, tuple_payload: []const u8, layout: anytype) View { - const tuple_text = std.fmt.allocPrint(self.arena.allocator(), "!cute.int_tuple<\"{s}\">", .{tuple_payload}) catch @panic("OOM"); - const tuple_type = self.parseType(tuple_text); - const tuple = self.emit(cute.make_int_tuple(self.ctx, self.innerSlice(values), tuple_type, self.loc())); - const iterator_type = (cute.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = tuple_type }) catch @panic("invalid arithmetic tuple iterator")).type_(); - const iterator = self.emit(cute.make_arith_tuple_iter(self.ctx, tuple.inner, iterator_type, self.loc())); - const layout_value = self.asValue(layout); - const result_type = (cute.CoordTensorType.get(self.ctx, .{ .arithTuple = tuple_type, .layout = layout_value.type_() }) catch @panic("invalid coordinate tensor")).type_(); - const result = self.emit(cute.make_view(self.ctx, iterator.inner, layout_value.inner, result_type, self.loc())); - return .{ .inner = result.inner, .kernel = self }; + const constrained = cute_ir.constrainedIntType(self.ctx, .i32, divisor) catch @panic("invalid CuTe constrained integer"); + return self.emit(cute_ir.assume(self.ctx, self.lift(value).to(.i32).inner, constrained, self.loc())); } /// Build a coordinate tensor from a runtime origin and a static layout. @@ -1387,10 +1290,10 @@ pub const Builder = struct { pub fn makeCoordTensor(self: *Builder, origin: anytype, layout: anytype) View { const tuple = self.makeIntTuple(origin); const layout_value = self.asValue(layout); - const iterator_type = (cute.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = tuple.type_() }) catch @panic("invalid arithmetic tuple iterator")).type_(); - const iterator = self.emit(cute.make_arith_tuple_iter(self.ctx, tuple.inner, iterator_type, self.loc())); - const result_type = (cute.CoordTensorType.get(self.ctx, .{ .arithTuple = tuple.type_(), .layout = layout_value.type_() }) catch @panic("invalid coordinate tensor")).type_(); - const result = self.emit(cute.make_view(self.ctx, iterator.inner, layout_value.inner, result_type, self.loc())); + const iterator_type = (cute_ir.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = tuple.type_() }) catch @panic("invalid arithmetic tuple iterator")).type_(); + const iterator = self.emit(cute_ir.make_arith_tuple_iter(self.ctx, iterator_type, .{ .value_ = tuple.inner }, self.loc())); + const result_type = (cute_ir.CoordTensorType.get(self.ctx, .{ .arithTuple = tuple.type_(), .layout = layout_value.type_() }) catch @panic("invalid coordinate tensor")).type_(); + const result = self.emit(cute_ir.make_view(self.ctx, iterator.inner, result_type, .{ .layout = layout_value.inner }, self.loc())); return .{ .inner = result.inner, .kernel = self }; } @@ -1408,19 +1311,19 @@ pub const Builder = struct { self: *Builder, comptime leaf_count: usize, input: anytype, - tuple_payload: []const u8, + tuple: anytype, comptime leaf_divisibility: [leaf_count]u32, layout: anytype, ) View { if (leaf_count == 0 or leaf_count > MAX_RANK) @panic("invalid coordinate tuple leaf count"); const source = self.asValue(input); - const source_type = source.type_().isA(cute.CoordTensorType) orelse + const source_type = source.type_().isA(cute_ir.CoordTensorType) orelse std.debug.panic("rebuildCoordTensorIterator: expected coordinate tensor, got {f}", .{source.type_()}); - const source_tuple_type = source_type.getArithTuple(); - const source_iterator_type = (cute.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = source_tuple_type }) catch + const source_tuple_type = source_type.arithTuple(); + const source_iterator_type = (cute_ir.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = source_tuple_type }) catch @panic("invalid source arithmetic tuple iterator")).type_(); - const source_iterator = self.emit(cute.get_iter(self.ctx, source.inner, source_iterator_type, self.loc())); - const source_tuple = self.emit(cute.deref_arith_tuple_iter( + const source_iterator = self.emit(cute_ir.get_iter(self.ctx, source.inner, source_iterator_type, self.loc())); + const source_tuple = self.emit(cute_ir.deref_arith_tuple_iter( self.ctx, source_iterator.inner, source_tuple_type, @@ -1431,68 +1334,43 @@ pub const Builder = struct { // 0 marks a leaf the producer knows to be statically zero, 1 a leaf // with no known divisibility. inline for (leaf_divisibility, 0..) |divisor, i| { - const leaf_payload = if (divisor == 0) - "0" - else if (divisor == 1) - "?" - else - std.fmt.allocPrint(self.arena.allocator(), "?{{div={d}}}", .{divisor}) catch @panic("OOM"); - leaf_types[i] = cute.algebraType(self.ctx, .int_tuple, leaf_payload) catch - std.debug.panic("invalid dynamic integer tuple leaf {s}", .{leaf_payload}); + leaf_types[i] = self.intTupleType(if (divisor == 0) .{ .static = 0 } else .{ .dynamic = divisor }); } - const leaves = self.emitResults(leaf_count, cute.get_leaves( + const leaves = self.emitResults(leaf_count, cute_ir.get_leaves( self.ctx, source_tuple.inner, leaf_types, self.loc(), )); - const rebuilt_tuple_type = cute.algebraType(self.ctx, .int_tuple, tuple_payload) catch - std.debug.panic("invalid rebuilt coordinate tuple {s}", .{tuple_payload}); - const rebuilt_tuple = self.emit(cute.make_int_tuple( + const rebuilt_tuple_type = self.intTupleType(self.expr(tuple)); + const rebuilt_tuple = self.emit(cute_ir.make_int_tuple( self.ctx, self.innerSlice(leaves[0..]), rebuilt_tuple_type, self.loc(), )); - const rebuilt_iterator_type = (cute.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = rebuilt_tuple_type }) catch + const rebuilt_iterator_type = (cute_ir.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = rebuilt_tuple_type }) catch @panic("invalid rebuilt arithmetic tuple iterator")).type_(); - const rebuilt_iterator = self.emit(cute.make_arith_tuple_iter( - self.ctx, - rebuilt_tuple.inner, - rebuilt_iterator_type, - self.loc(), - )); + const rebuilt_iterator = self.emit(cute_ir.make_arith_tuple_iter(self.ctx, rebuilt_iterator_type, .{ .value_ = rebuilt_tuple.inner }, self.loc())); const layout_value = self.asValue(layout); - const result_type = (cute.CoordTensorType.get(self.ctx, .{ + const result_type = (cute_ir.CoordTensorType.get(self.ctx, .{ .arithTuple = rebuilt_tuple_type, .layout = layout_value.type_(), }) catch @panic("invalid rebuilt coordinate tensor")).type_(); - const result = self.emit(cute.make_view( - self.ctx, - rebuilt_iterator.inner, - layout_value.inner, - result_type, - self.loc(), - )); + const result = self.emit(cute_ir.make_view(self.ctx, rebuilt_iterator.inner, result_type, .{ .layout = layout_value.inner }, self.loc())); return .{ .inner = result.inner, .kernel = self }; } - /// A coordinate made only of CuTe placeholders, such as `(_,_,_)`. - /// Placeholders select every element of a mode and therefore cannot be - /// represented by Zig integer values passed to `makeCoord`. - pub fn staticCoord(self: *Builder, payload: []const u8) View { - const type_text = std.fmt.allocPrint(self.arena.allocator(), "!cute.coord<\"{s}\">", .{payload}) catch @panic("OOM"); - const result = self.emit(cute.make_coord(self.ctx, &.{}, self.parseType(type_text), self.loc())); - return .{ .inner = result.inner, .kernel = self }; + /// `!cute.shape` of a CuTe algebra value such as `.{ .{ 128, 16 }, 1, 1, 2 }` + /// or `.{ AlgebraToken.dynamic, AlgebraToken.dynamic }`. + pub fn shapeType(self: *Builder, shape: anytype) *const mlir.Type { + return (cute_ir.ShapeType.get(self.ctx, .{ .attr = self.algebraAttr(cute_ir.ShapeAttr, self.expr(shape)).attribute() }) catch @panic("invalid CuTe shape")).type_(); } - /// Static `cute.make_shape`; `payload` is CuTe algebra such as - /// `((128,16),1,1,2)`. Dynamic leaves can be added when a kernel needs - /// them, but all current SM100 instruction tiles are static. - pub fn makeShape(self: *Builder, payload: []const u8) View { - const ty_text = std.fmt.allocPrint(self.arena.allocator(), "!cute.shape<\"{s}\">", .{payload}) catch @panic("OOM"); - const value = self.emit(cute.make_shape(self.ctx, &.{}, self.parseType(ty_text), self.loc())); + /// Static `cute.make_shape` of a CuTe algebra value. + pub fn makeShape(self: *Builder, shape: anytype) View { + const value = self.emit(cute_ir.make_shape(self.ctx, &.{}, self.shapeType(shape), self.loc())); return .{ .inner = value.inner, .kernel = self }; } @@ -1501,12 +1379,12 @@ pub const Builder = struct { /// leaves dynamic is significant to CuTe's TMA layout algebra when a /// runtime extent happens to be one. pub fn makeShapeValues(self: *Builder, values: anytype, result_type: *const mlir.Type) View { - const value = self.emit(cute.make_shape(self.ctx, self.innerSlice(values), result_type, self.loc())); + const value = self.emit(cute_ir.make_shape(self.ctx, self.innerSlice(values), result_type, self.loc())); return .{ .inner = value.inner, .kernel = self }; } - pub fn tileToShapeTyped(self: *Builder, input: anytype, shape: anytype, order: View, result_type: *const mlir.Type) View { - const result = self.emit(cute.tile_to_shape( + pub fn tileToShape(self: *Builder, input: anytype, shape: anytype, order: View, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.tile_to_shape( self.ctx, self.asValue(input).inner, self.asValue(shape).inner, @@ -1519,42 +1397,39 @@ pub const Builder = struct { /// Static `cute.make_tile` from mode extents. pub fn makeTile(self: *Builder, shape: []const i64) View { - var payload: std.Io.Writer.Allocating = .init(self.arena.allocator()); - payload.writer.writeByte('[') catch @panic("OOM"); - for (shape, 0..) |extent, i| { - if (i != 0) payload.writer.writeByte(';') catch @panic("OOM"); - payload.writer.print("{d}:1", .{extent}) catch @panic("OOM"); - } - payload.writer.writeByte(']') catch @panic("OOM"); - const ty_text = std.fmt.allocPrint(self.arena.allocator(), "!cute.tile<\"{s}\">", .{payload.written()}) catch @panic("OOM"); - const value = self.emit(cute.make_tile(self.ctx, &.{}, self.parseType(ty_text), self.loc())); + const modes = self.arena.allocator().alloc(LayoutSpec, shape.len) catch @panic("OOM"); + for (shape, modes) |extent, *mode| mode.* = .{ .shape = .{ .static = extent }, .stride = .{ .static = 1 } }; + const value = self.emit(cute_ir.make_tile(self.ctx, &.{}, self.tileType(modes), self.loc())); return .{ .inner = value.inner, .kernel = self }; } + /// `!cute.tile` whose modes are `layouts`, such as `[512:1;1:0;4:1]`. + pub fn tileType(self: *Builder, layouts: []const LayoutSpec) *const mlir.Type { + return (cute_ir.TileType.get(self.ctx, .{ .attr = self.tileAttr(layouts).attribute() }) catch @panic("invalid CuTe tile")).type_(); + } + + fn tileAttr(self: *Builder, layouts: []const LayoutSpec) *const cute_ir.TileAttr { + const modes = self.arena.allocator().alloc(*const cute_ir.TileAttr, layouts.len) catch @panic("OOM"); + for (layouts, modes) |layout, *mode| mode.* = cute_ir.TileAttr.getLayout(self.ctx, self.layoutAttr(layout)) catch @panic("invalid CuTe tile mode"); + return cute_ir.TileAttr.getTuple(self.ctx, modes) catch @panic("invalid CuTe tile"); + } + /// Construct a target atom whose complete architecture payload is /// encoded in `atom_type` (for example `!cute_nvgpu.sm100.mma_bs<...>`). - pub fn makeAtom(self: *Builder, atom_type: []const u8, values: anytype) Atom { - const result = self.emit(cute.make_atom(self.ctx, self.innerSlice(values), self.parseType(atom_type), self.loc())); + pub fn makeAtom(self: *Builder, atom_type: *const mlir.Type, values: anytype) Atom { + const result = self.emit(cute_ir.make_atom(self.ctx, self.innerSlice(values), atom_type, self.loc())); return .{ .inner = result.inner, .kernel = self }; } - pub fn makeTiledMma(self: *Builder, atom: Atom, result_type: []const u8) Atom { - const result = self.emit(cute.make_tiled_mma(self.ctx, atom.inner, self.parseType(result_type), self.loc())); + pub fn makeTiledMma(self: *Builder, atom: Atom, result_type: *const mlir.Type) Atom { + const result = self.emit(cute_ir.make_tiled_mma(self.ctx, atom.inner, result_type, self.loc())); return .{ .inner = result.inner, .kernel = self }; } - /// Allocate a statically typed shared-memory tensor. This is used for - /// swizzled layouts which cannot be represented by `Layout` alone. - pub fn allocSmemView(self: *Builder, memref_type: []const u8) View { - const result = self.emit(cute.memref_alloc_smem(self.ctx, self.parseType(memref_type), self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - /// Allocate shared memory with a type assembled by the Zig DSL. This is - /// the non-string counterpart of `allocSmemView` used by source kernels. - pub fn allocSmemTyped(self: *Builder, dtype: DType, alignment: u64, layout_type: *const mlir.Type) View { + /// Allocate shared memory with a type assembled by the Zig DSL. + pub fn allocSmemView(self: *Builder, dtype: DType, alignment: u64, layout_type: *const mlir.Type) View { const memref_type = self.memrefType(dtype, .smem, alignment, layout_type); - const result = self.emit(cute.memref_alloc_smem(self.ctx, memref_type, self.loc())); + const result = self.emit(cute_ir.memref_alloc_smem(self.ctx, memref_type, self.loc())); return .{ .inner = result.inner, .kernel = self }; } @@ -1586,96 +1461,53 @@ pub const Builder = struct { /// Allocate a register-memory tensor whose layout is represented by a /// normal CuTe layout value. - pub fn allocRmemTyped(self: *Builder, dtype: DType, alignment: u64, layout: anytype) View { + pub fn allocRmem(self: *Builder, dtype: DType, alignment: u64, layout: anytype) View { const layout_value = self.asValue(layout); const memref_type = self.memrefType(dtype, .rmem, alignment, layout_value.type_()); - const result = self.emit(cute.memref_alloca(self.ctx, layout_value.inner, memref_type, self.loc())); + const result = self.emit(cute_ir.memref_alloca(self.ctx, memref_type, .{ .layout = layout_value.inner }, self.loc())); return .{ .inner = result.inner, .kernel = self }; } - pub fn makeView(self: *Builder, pointer: anytype, layout: anytype, result_type: []const u8) View { - const result = self.emit(cute.make_view( - self.ctx, - self.asValue(pointer).inner, - self.asValue(layout).inner, - self.parseType(result_type), - self.loc(), - )); + pub fn makeView(self: *Builder, pointer: anytype, layout: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.make_view(self.ctx, self.asValue(pointer).inner, result_type, .{ .layout = self.asValue(layout).inner }, self.loc())); return .{ .inner = result.inner, .kernel = self }; } - pub fn makeViewTyped(self: *Builder, pointer: anytype, layout: anytype, result_type: *const mlir.Type) View { - const result = self.emit(cute.make_view( - self.ctx, - self.asValue(pointer).inner, - self.asValue(layout).inner, - result_type, - self.loc(), - )); + pub fn makeViewNoLayout(self: *Builder, pointer: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.make_view(self.ctx, self.asValue(pointer).inner, result_type, .{}, self.loc())); return .{ .inner = result.inner, .kernel = self }; } - pub fn makeViewNoLayout(self: *Builder, pointer: anytype, result_type: []const u8) View { - const result = self.emit(cute.make_view(self.ctx, self.asValue(pointer).inner, null, self.parseType(result_type), self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn localTile(self: *Builder, input: anytype, tile: anytype, coord: anytype, result_type: []const u8, projection: ?Attr) View { + /// `projection`, like Python's `proj`, selects the modes of `tile` that + /// apply, such as `b.expr(.{ 1, AlgebraToken.all, 1 })`. + pub fn localTile(self: *Builder, input: anytype, tile: anytype, coord: anytype, result_type: *const mlir.Type, projection: ?AlgebraExpr) View { const coord_value = if (@TypeOf(coord) == View) coord.inner else self.makeCoord(coord); - const result = self.emit(cute.local_tile( - self.ctx, - self.asValue(input).inner, - self.asValue(tile).inner, - coord_value, - self.parseType(result_type), - if (projection) |p| p.get(self.ctx) else null, - self.loc(), - )); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn localTileTyped(self: *Builder, input: anytype, tile: anytype, coord: anytype, result_type: *const mlir.Type, projection: ?Attr) View { - const coord_value = if (@TypeOf(coord) == View) coord.inner else self.makeCoord(coord); - const result = self.emit(cute.local_tile( + const result = self.emit(cute_ir.local_tile( self.ctx, self.asValue(input).inner, self.asValue(tile).inner, coord_value, result_type, - if (projection) |p| p.get(self.ctx) else null, + .{ .proj = if (projection) |p| self.algebraAttr(cute_ir.CoordAttr, p).attribute() else null }, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub fn localPartition(self: *Builder, input: anytype, tiler: anytype, index: anytype, result_type: []const u8) View { - const result = self.emit(cute.local_partition( + pub fn localPartition(self: *Builder, input: anytype, tiler: anytype, index: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.local_partition( self.ctx, self.asValue(input).inner, self.asValue(tiler).inner, self.makeCoord(index), - self.parseType(result_type), + result_type, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - /// Apply CuTe layout algebra while keeping the compiler-derived result - /// type explicit. These operations are the building blocks used between - /// TMA partitioning and the block-scaled MMA fragments. - pub fn slice(self: *Builder, input: anytype, coord: anytype, result_type: []const u8) View { - const result = self.emit(cute.slice( - self.ctx, - self.asValue(input).inner, - self.makeCoord(coord), - self.parseType(result_type), - self.loc(), - )); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn sliceTyped(self: *Builder, input: anytype, coord: anytype, result_type: *const mlir.Type) View { - const result = self.emit(cute.slice( + pub fn slice(self: *Builder, input: anytype, coord: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.slice( self.ctx, self.asValue(input).inner, self.makeCoord(coord), @@ -1694,29 +1526,29 @@ pub const Builder = struct { const value = self.asValue(input); const layout_value = self.asValue(layout); const input_type = value.type_(); - const result_type: *const mlir.Type = if (input_type.isA(cute.MemRefType)) |memref| - (cute.MemRefType.get(self.ctx, .{ .ptr = memref.getPtr(), .layout = layout_value.type_() }) catch @panic("invalid regrouped memref")).type_() - else if (input_type.isA(cute.CoordTensorType)) |coords| - (cute.CoordTensorType.get(self.ctx, .{ .arithTuple = coords.getArithTuple(), .layout = layout_value.type_() }) catch @panic("invalid regrouped coordinate tensor")).type_() + const result_type: *const mlir.Type = if (input_type.isA(cute_ir.MemRefType)) |memref| + (cute_ir.MemRefType.get(self.ctx, .{ .ptr = memref.ptr_(), .layout = layout_value.type_() }) catch @panic("invalid regrouped memref")).type_() + else if (input_type.isA(cute_ir.CoordTensorType)) |coords| + (cute_ir.CoordTensorType.get(self.ctx, .{ .arithTuple = coords.arithTuple(), .layout = layout_value.type_() }) catch @panic("invalid regrouped coordinate tensor")).type_() else std.debug.panic("withLayout: expected memref or coordinate tensor, got {f}", .{input_type}); - const iterator_type: *const mlir.Type = if (input_type.isA(cute.MemRefType)) |memref| - memref.getPtr() - else if (input_type.isA(cute.CoordTensorType)) |coords| - (cute.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = coords.getArithTuple() }) catch @panic("invalid arithmetic tuple iterator")).type_() + const iterator_type: *const mlir.Type = if (input_type.isA(cute_ir.MemRefType)) |memref| + memref.ptr_() + else if (input_type.isA(cute_ir.CoordTensorType)) |coords| + (cute_ir.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = coords.arithTuple() }) catch @panic("invalid arithmetic tuple iterator")).type_() else unreachable; - const iterator = self.emit(cute.get_iter(self.ctx, value.inner, iterator_type, self.loc())); - const result = self.emit(cute.make_view(self.ctx, iterator.inner, layout_value.inner, result_type, self.loc())); + const iterator = self.emit(cute_ir.get_iter(self.ctx, value.inner, iterator_type, self.loc())); + const result = self.emit(cute_ir.make_view(self.ctx, iterator.inner, result_type, .{ .layout = layout_value.inner }, self.loc())); return .{ .inner = result.inner, .kernel = self }; } - pub fn groupModes(self: *Builder, input: anytype, begin: u32, end: u32, result_type: []const u8) View { - const result = self.emit(cute.group_modes( + pub fn groupModes(self: *Builder, input: anytype, begin: u32, end: u32, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.group_modes( self.ctx, self.asValue(input).inner, - self.parseType(result_type), + result_type, .int(self.ctx, .i32, begin), .int(self.ctx, .i32, end), self.loc(), @@ -1724,85 +1556,68 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn tiledDivide(self: *Builder, input: anytype, tiler: anytype, result_type: []const u8) View { - const result = self.emit(cute.tiled_divide( + pub fn tiledDivide(self: *Builder, input: anytype, tiler: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.tiled_divide( self.ctx, self.asValue(input).inner, self.asValue(tiler).inner, - self.parseType(result_type), + result_type, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub fn flatDivide(self: *Builder, input: anytype, tiler: anytype, result_type: []const u8) View { - const result = self.emit(cute.flat_divide( + pub fn flatDivide(self: *Builder, input: anytype, tiler: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.flat_divide( self.ctx, self.asValue(input).inner, self.asValue(tiler).inner, - self.parseType(result_type), + result_type, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub fn coalesce(self: *Builder, input: anytype, target_profile: ?View, result_type: []const u8) View { - const result = self.emit(cute.coalesce( + pub fn coalesce(self: *Builder, input: anytype, target_profile: ?View, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.coalesce( self.ctx, self.asValue(input).inner, - if (target_profile) |target| target.inner else null, - self.parseType(result_type), + result_type, + .{ .target_profile = if (target_profile) |target| target.inner else null }, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub fn filterZeros(self: *Builder, input: anytype, target_profile: ?View, result_type: []const u8) View { - const result = self.emit(cute.filter_zeros( + pub fn filterZeros(self: *Builder, input: anytype, target_profile: ?View, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.filter_zeros( self.ctx, self.asValue(input).inner, - if (target_profile) |target| target.inner else null, - self.parseType(result_type), + result_type, + .{ .target_profile = if (target_profile) |target| target.inner else null }, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub fn getLayout(self: *Builder, input: anytype, result_type: []const u8) View { - const result = self.emit(cute.get_layout(self.ctx, self.asValue(input).inner, self.parseType(result_type), self.loc())); + pub fn getLayout(self: *Builder, input: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.get_layout(self.ctx, self.asValue(input).inner, result_type, self.loc())); return .{ .inner = result.inner, .kernel = self }; } - pub fn getIter(self: *Builder, input: anytype, result_type: []const u8) View { - const result = self.emit(cute.get_iter(self.ctx, self.asValue(input).inner, self.parseType(result_type), self.loc())); + /// `cute.get_iter`: the iterator of `input`, of type `result_type`. + pub fn getIter(self: *Builder, input: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.get_iter(self.ctx, self.asValue(input).inner, result_type, self.loc())); return .{ .inner = result.inner, .kernel = self }; } - /// Typed form of `cute.get_iter`, used when the pointer type is already - /// available through the CuTe C API and no textual type is necessary. - pub fn getIterTyped(self: *Builder, input: anytype, result_type: *const mlir.Type) View { - const result = self.emit(cute.get_iter(self.ctx, self.asValue(input).inner, result_type, self.loc())); + pub fn getShape(self: *Builder, input: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.get_shape(self.ctx, self.asValue(input).inner, result_type, self.loc())); return .{ .inner = result.inner, .kernel = self }; } - pub fn getShape(self: *Builder, input: anytype, result_type: []const u8) View { - const result = self.emit(cute.get_shape(self.ctx, self.asValue(input).inner, self.parseType(result_type), self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn addOffset(self: *Builder, input: anytype, offset: anytype, result_type: []const u8) View { - const result = self.emit(cute.add_offset( - self.ctx, - self.asValue(input).inner, - self.asValue(offset).inner, - self.parseType(result_type), - self.loc(), - )); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn addOffsetTyped(self: *Builder, input: anytype, offset: anytype, result_type: *const mlir.Type) View { - const result = self.emit(cute.add_offset( + pub fn addOffset(self: *Builder, input: anytype, offset: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.add_offset( self.ctx, self.asValue(input).inner, self.asValue(offset).inner, @@ -1812,63 +1627,50 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn recastIter(self: *Builder, input: anytype, result_type: []const u8) View { - const result = self.emit(cute.recast_iter(self.ctx, self.asValue(input).inner, self.parseType(result_type), self.loc())); + pub fn recastIter(self: *Builder, input: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.recast_iter(self.ctx, self.asValue(input).inner, result_type, self.loc())); return .{ .inner = result.inner, .kernel = self }; } - pub fn appendToRank(self: *Builder, input: anytype, element: anytype, rank: u32, result_type: []const u8) View { - const result = self.emit(cute.append_to_rank( + pub fn appendToRank(self: *Builder, input: anytype, element: anytype, rank: u32, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.append_to_rank( self.ctx, self.asValue(input).inner, self.asValue(element).inner, - self.parseType(result_type), + result_type, .int(self.ctx, .i32, rank), self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub fn makeTiledCopy(self: *Builder, atom: Atom, result_type: []const u8) Atom { - const result = self.emit(cute.make_tiled_copy(self.ctx, atom.inner, self.parseType(result_type), self.loc())); + pub fn makeTiledCopy(self: *Builder, atom: Atom, result_type: *const mlir.Type) Atom { + const result = self.emit(cute_ir.make_tiled_copy(self.ctx, atom.inner, result_type, self.loc())); return .{ .inner = result.inner, .kernel = self }; } - pub fn tiledCopyPartition(self: *Builder, tiled: Atom, input: anytype, coord: anytype, destination: bool, result_type: []const u8) View { + pub fn tiledCopyPartition(self: *Builder, tiled: Atom, input: anytype, coord: anytype, destination: bool, result_type: *const mlir.Type) View { const op = if (destination) - cute.tiled_copy_partition_D(self.ctx, tiled.inner, self.asValue(input).inner, self.makeCoord(coord), self.parseType(result_type), self.loc()) + cute_ir.tiled_copy_partition_D(self.ctx, tiled.inner, self.asValue(input).inner, self.makeCoord(coord), result_type, self.loc()) else - cute.tiled_copy_partition_S(self.ctx, tiled.inner, self.asValue(input).inner, self.makeCoord(coord), self.parseType(result_type), self.loc()); + cute_ir.tiled_copy_partition_S(self.ctx, tiled.inner, self.asValue(input).inner, self.makeCoord(coord), result_type, self.loc()); const result = self.emit(op); return .{ .inner = result.inner, .kernel = self }; } - pub fn tiledCopyRetile(self: *Builder, tiled: Atom, input: anytype, result_type: []const u8) View { - const result = self.emit(cute.tiled_copy_retile( + pub fn tiledCopyRetile(self: *Builder, tiled: Atom, input: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.tiled_copy_retile( self.ctx, tiled.inner, self.asValue(input).inner, - self.parseType(result_type), + result_type, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub fn tiledMmaPartition(self: *Builder, tiled: Atom, input: anytype, coord: anytype, operand_id: u2, result_type: []const u8) View { - const result = self.emit(cute.tiled_mma_partition( - self.ctx, - tiled.inner, - self.asValue(input).inner, - self.makeCoord(coord), - self.parseType(result_type), - .int(self.ctx, .i32, operand_id), - self.loc(), - )); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn tiledMmaPartitionTyped(self: *Builder, tiled: Atom, input: anytype, coord: anytype, operand_id: u2, result_type: *const mlir.Type) View { - const result = self.emit(cute.tiled_mma_partition( + pub fn tiledMmaPartition(self: *Builder, tiled: Atom, input: anytype, coord: anytype, operand_id: u2, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.tiled_mma_partition( self.ctx, tiled.inner, self.asValue(input).inner, @@ -1880,12 +1682,12 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn mmaMakeFragment(self: *Builder, tiled: Atom, input: anytype, operand_id: u2, result_type: []const u8) View { - const result = self.emit(cute.mma_make_fragment( + pub fn mmaMakeFragment(self: *Builder, tiled: Atom, input: anytype, operand_id: u2, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.mma_make_fragment( self.ctx, tiled.inner, self.asValue(input).inner, - self.parseType(result_type), + result_type, .int(self.ctx, .i32, operand_id), self.loc(), )); @@ -1893,18 +1695,11 @@ pub const Builder = struct { } pub fn copy(self: *Builder, atom: Atom, src: anytype, dst: anytype, pred: ?Value) void { - self.emitVoid(cute.copy( - self.ctx, - atom.inner, - self.innerSlice(src), - self.innerSlice(dst), - if (pred) |p| p.inner else null, - self.loc(), - )); + self.emitVoid(cute_ir.copy(self.ctx, atom.inner, self.innerSlice(src), self.innerSlice(dst), .{ .pred = if (pred) |p| p.inner else null }, self.loc())); } pub fn gemm(self: *Builder, tiled: Atom, d: anytype, a: anytype, rhs: anytype, c: anytype) void { - self.emitVoid(cute.gemm( + self.emitVoid(cute_ir.gemm( self.ctx, tiled.inner, self.asValue(d).inner, @@ -1926,54 +1721,23 @@ pub const Builder = struct { )); } - // ==================== cute.nvgpu.cpasync: TMA (SM90+) ==================== + // ==================== cute_nvgpu.cpasync: TMA (SM90+) ==================== // The host function builds tiled TMA atoms (Python `make_tiled_tma_atom`); // the kernel receives them as grid-constant arguments and makes them // executable before issuing copies. - pub const TmaLoadOptions = struct { - atom_type: []const u8, - tensor_type: []const u8, - kind: Attr = .{ .str = "tma_load" }, - num_multicast: u32 = 1, - tma_format: ?Attr = null, - }; - - /// Build the host-visible TMA descriptor and its coordinate tensor. The - /// result types are supplied explicitly because they depend on CuTe's - /// layout algebra and may contain divisibility constraints. - pub fn makeTiledTmaLoadAtom(self: *Builder, gmem: anytype, smem_layout: anytype, cta_map: anytype, opts: TmaLoadOptions) TmaDescriptor { - const results = self.emitResults(2, cute.nvgpu.atom_make_non_exec_tiled_tma_load( - self.ctx, - self.asValue(gmem).inner, - self.asValue(smem_layout).inner, - self.asValue(cta_map).inner, - self.parseType(opts.atom_type), - self.parseType(opts.tensor_type), - opts.kind.get(self.ctx), - .int(self.ctx, .i32, opts.num_multicast), - if (opts.tma_format) |format| format.get(self.ctx) else null, - self.loc(), - )); - return .{ - .atom = .{ .inner = results[0].inner, .kernel = self }, - .tensor = .{ .inner = results[1].inner, .kernel = self }, - }; - } - - /// Typed counterpart of `makeTiledTmaLoadAtom`: the atom and coordinate - /// tensor types are assembled from `config`. - pub fn makeTiledTmaLoadAtomTyped(self: *Builder, gmem: anytype, smem_layout: anytype, cta_map: anytype, config: TmaConfig) TmaDescriptor { - const results = self.emitResults(2, cute.nvgpu.atom_make_non_exec_tiled_tma_load( + /// Build the host-visible TMA descriptor and its coordinate tensor, whose + /// types are assembled from `config`. + pub fn makeTiledTmaLoadAtom(self: *Builder, gmem: anytype, smem_layout: anytype, cta_map: anytype, config: TmaConfig) TmaDescriptor { + const results = self.emitResults(2, cute_nvgpu.atom_make_non_exec_tiled_tma_load( self.ctx, self.asValue(gmem).inner, self.asValue(smem_layout).inner, self.asValue(cta_map).inner, self.tmaLoadAtomType(config), self.coordTensorType(config.coordinate_rank, config.coordinate_layout), - self.parseAttribute("#cute_nvgpu.tiled_tma_load"), - .int(self.ctx, .i32, config.num_multicast), - tmaFormatAttribute(self, config.format), + (cute_nvgpu.TiledTmaLoadAttr.get(self.ctx, .{ .value = .sm_90 }) catch @panic("invalid tiled TMA load")).attribute(), + .{ .num_multicast = .int(self.ctx, .i32, 1), .tma_format = tmaFormatAttribute(self, config) }, self.loc(), )); return .{ @@ -1982,40 +1746,16 @@ pub const Builder = struct { }; } - pub const TmaStoreOptions = struct { - atom_type: []const u8, - tensor_type: []const u8, - tma_format: ?Attr = null, - }; - /// Host-visible tiled TMA store atom and its coordinate tensor. - pub fn makeTiledTmaStoreAtom(self: *Builder, gmem: anytype, smem_layout: anytype, cta_map: anytype, opts: TmaStoreOptions) TmaDescriptor { - const results = self.emitResults(2, cute.nvgpu.atom_make_non_exec_tiled_tma_store( - self.ctx, - self.asValue(gmem).inner, - self.asValue(smem_layout).inner, - self.asValue(cta_map).inner, - self.parseType(opts.atom_type), - self.parseType(opts.tensor_type), - if (opts.tma_format) |format| format.get(self.ctx) else null, - self.loc(), - )); - return .{ - .atom = .{ .inner = results[0].inner, .kernel = self }, - .tensor = .{ .inner = results[1].inner, .kernel = self }, - }; - } - - /// Typed counterpart of `makeTiledTmaStoreAtom`. - pub fn makeTiledTmaStoreAtomTyped(self: *Builder, gmem: anytype, smem_layout: anytype, cta_map: anytype, config: TmaConfig) TmaDescriptor { - const results = self.emitResults(2, cute.nvgpu.atom_make_non_exec_tiled_tma_store( + pub fn makeTiledTmaStoreAtom(self: *Builder, gmem: anytype, smem_layout: anytype, cta_map: anytype, config: TmaConfig) TmaDescriptor { + const results = self.emitResults(2, cute_nvgpu.atom_make_non_exec_tiled_tma_store( self.ctx, self.asValue(gmem).inner, self.asValue(smem_layout).inner, self.asValue(cta_map).inner, self.tmaStoreAtomType(config), self.coordTensorType(config.coordinate_rank, config.coordinate_layout), - tmaFormatAttribute(self, config.format), + .{ .tma_format = tmaFormatAttribute(self, config) }, self.loc(), )); return .{ @@ -2026,102 +1766,80 @@ pub const Builder = struct { /// Host-side tiled TMA load atom type described by `config`. pub fn tmaLoadAtomType(self: *Builder, config: TmaConfig) *const mlir.Type { - return self.parseType(std.fmt.allocPrint( - self.arena.allocator(), - "!cute_nvgpu.atom.non_exec_tiled_tma_load, tma_format = {s}>", - .{ dtypeName(config.dtype), config.copy_bits, config.global_basis.payload(self), formatName(config.format, config.dtype) }, - ) catch @panic("OOM")); + return (cute_nvgpu.CopyAtomNonExecTiledTmaLoadType.get(self.ctx, .{ + .kind = .sm_90, + .valType = config.dtype.toMlir(self.ctx), + .copyBits = @intCast(config.copy_bits), + .tmaGbasis = self.layoutType(config.global_basis), + .tmaFormat = tmaDataFormat(config.format, config.dtype), + }) catch @panic("invalid TMA load atom")).type_(); } /// Executable device-side form of a tiled TMA load atom. `make_exec_tma` /// preserves the tensor-map payload and adds the launch mode used by /// `copy`. pub fn tmaLoadExecAtomType(self: *Builder, config: TmaConfig) *const mlir.Type { - return self.parseType(std.fmt.allocPrint( - self.arena.allocator(), - "!cute_nvgpu.atom.tma_load<{s}, copy_bits = {d}, mode = tiled, num_cta = {d}, g_stride = <\"()\"> tma_gbasis = <\"{s}\">>", - .{ dtypeName(config.dtype), config.copy_bits, config.num_multicast, config.global_basis.payload(self) }, - ) catch @panic("OOM")); + return (cute_nvgpu.CopyAtomTmaLoadType.get(self.ctx, .{ + .valType = config.dtype.toMlir(self.ctx), + .copyBits = @intCast(config.copy_bits), + .mode = .tiled, + .numCta = 1, + .gStride = self.emptyStrideType(), + .tmaGbasis = self.layoutType(config.global_basis), + }) catch @panic("invalid TMA load atom")).type_(); } /// Host-side tiled TMA store atom type described by `config`. pub fn tmaStoreAtomType(self: *Builder, config: TmaConfig) *const mlir.Type { - return self.parseType(std.fmt.allocPrint( - self.arena.allocator(), - "!cute_nvgpu.atom.non_exec_tiled_tma_store<{s}, copy_bits = {d}, tma_gbasis = <\"{s}\">, tma_format = {s}>", - .{ dtypeName(config.dtype), config.copy_bits, config.global_basis.payload(self), formatName(config.format, config.dtype) }, - ) catch @panic("OOM")); + return (cute_nvgpu.CopyAtomNonExecTiledTmaStoreType.get(self.ctx, .{ + .valType = config.dtype.toMlir(self.ctx), + .copyBits = @intCast(config.copy_bits), + .tmaGbasis = self.layoutType(config.global_basis), + .tmaFormat = tmaDataFormat(config.format, config.dtype), + }) catch @panic("invalid TMA store atom")).type_(); } /// Executable device-side form of a tiled TMA store atom. pub fn tmaStoreExecAtomType(self: *Builder, config: TmaConfig) *const mlir.Type { - return self.parseType(std.fmt.allocPrint( - self.arena.allocator(), - "!cute_nvgpu.atom.tma_store<{s}, copy_bits = {d}, mode = tiled, g_stride = <\"()\"> tma_gbasis = <\"{s}\">>", - .{ dtypeName(config.dtype), config.copy_bits, config.global_basis.payload(self) }, - ) catch @panic("OOM")); + return (cute_nvgpu.CopyAtomTmaStoreType.get(self.ctx, .{ + .valType = config.dtype.toMlir(self.ctx), + .copyBits = @intCast(config.copy_bits), + .mode = .tiled, + .gStride = self.emptyStrideType(), + .tmaGbasis = self.layoutType(config.global_basis), + }) catch @panic("invalid TMA store atom")).type_(); + } + + /// The `()` global stride of tiled TMA atoms. + fn emptyStrideType(self: *Builder) *const mlir.Type { + return (cute_ir.StrideType.get(self.ctx, .{ .attr = self.algebraAttr(cute_ir.StrideAttr, .{ .tuple = &.{} }).attribute() }) catch @panic("invalid CuTe stride")).type_(); } pub fn prefetchTmaDesc(self: *Builder, descriptor: Atom) void { - self.emitVoid(cute.nvgpu.prefetch_tma_desc(self.ctx, descriptor.inner, self.loc())); + self.emitVoid(cute_nvgpu.prefetch_tma_desc(self.ctx, descriptor.inner, self.loc())); } - pub fn makeExecTma(self: *Builder, descriptor: Atom, result_type: []const u8) Atom { - const result = self.emit(cute.nvgpu.atom_make_exec_tma(self.ctx, descriptor.inner, self.parseType(result_type), null, null, self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn makeExecTmaTyped(self: *Builder, descriptor: Atom, result_type: *const mlir.Type) Atom { - const result = self.emit(cute.nvgpu.atom_make_exec_tma(self.ctx, descriptor.inner, result_type, null, null, self.loc())); + pub fn makeExecTma(self: *Builder, descriptor: Atom, result_type: *const mlir.Type) Atom { + const result = self.emit(cute_nvgpu.atom_make_exec_tma(self.ctx, descriptor.inner, result_type, .{}, self.loc())); return .{ .inner = result.inner, .kernel = self }; } /// Attach the mbarrier that an executable TMA load atom completes. pub fn setTmaBarrier(self: *Builder, atom: Atom, barrier: Value) Atom { - const result = self.emit(cute.nvgpu.atom_set_value( + const result = self.emit(cute_nvgpu.atom_set_value( self.ctx, atom.inner, barrier.inner, atom.type_(), - self.parseAttribute("#cute_nvgpu.atom_copy_field_tmaload"), + (cute_nvgpu.AtomCopyFieldTmaLoadAttr.get(self.ctx, .{ .value = .tma_bar }) catch @panic("invalid TMA load field")).attribute(), self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } + /// Partition `smem` and `targets` for the TMA atom `descriptor`; the + /// dependent result types are built by the caller. pub fn tmaPartition( - self: *Builder, - descriptor: Atom, - cta_coord: anytype, - cta_layout: anytype, - smem: anytype, - targets: anytype, - smem_type: []const u8, - target_types: []const []const u8, - ) TmaPartition { - if (targets.len != target_types.len) @panic("tmaPartition: target count does not match result type count"); - const result_types = self.arena.allocator().alloc(*const mlir.Type, target_types.len) catch @panic("OOM"); - const views = self.arena.allocator().alloc(View, target_types.len) catch @panic("OOM"); - for (target_types, result_types) |type_text, *ty| ty.* = self.parseType(type_text); - const op = cute.nvgpu.atom_tma_partition( - self.ctx, - descriptor.inner, - self.makeCoord(cta_coord), - self.asValue(cta_layout).inner, - self.asValue(smem).inner, - self.innerSlice(targets), - self.parseType(smem_type), - result_types, - self.loc(), - ); - _ = op.appendTo(self.currentBlock()); - for (views, 0..) |*view, i| view.* = .{ .inner = op.result(i + 1), .kernel = self }; - return .{ .smem = .{ .inner = op.result(0), .kernel = self }, .targets = views }; - } - - /// Typed form of `tmaPartition`. The dependent result types are built by - /// the Zig CuTe DSL instead of being round-tripped through textual MLIR. - pub fn tmaPartitionTyped( self: *Builder, descriptor: Atom, cta_coord: anytype, @@ -2131,9 +1849,9 @@ pub const Builder = struct { smem_type: *const mlir.Type, target_types: []const *const mlir.Type, ) TmaPartition { - if (targets.len != target_types.len) @panic("tmaPartitionTyped: target count does not match result type count"); + if (targets.len != target_types.len) @panic("tmaPartition: target count does not match result type count"); const views = self.arena.allocator().alloc(View, target_types.len) catch @panic("OOM"); - const op = cute.nvgpu.atom_tma_partition( + const op = cute_nvgpu.atom_tma_partition( self.ctx, descriptor.inner, self.makeCoord(cta_coord), @@ -2149,15 +1867,30 @@ pub const Builder = struct { return .{ .smem = .{ .inner = op.result(0), .kernel = self }, .targets = views }; } - // ==================== cute.nvgpu.tcgen05 (SM100) ==================== + // ==================== cute_nvgpu.tcgen05 (SM100) ==================== /// Tiled MMA type of the SM100 block-scaled MXFP4 x MXFP8 atom. pub fn blockScaledMmaType(self: *Builder, cfg: BlockScaledMmaConfig) *const mlir.Type { - return self.parseType(std.fmt.allocPrint( - self.arena.allocator(), - "!cute.tiled_mma, permutation_MNK = <\"[_;_;_]\">>", - .{blockScaledPayload(self, cfg)}, - ) catch @panic("OOM")); + return (cute_ir.TiledMmaType.get(self.ctx, .{ + .mmaAtom = self.blockScaledAtomType(cfg), + .atomLayoutMNK = self.layoutAttr(self.layoutSpec(.{ 1, 1, 1 }, .{ 0, 0, 0 })).attribute(), + .permutationMNK = self.algebraAttr(cute_ir.TileAttr, self.expr(.{ AlgebraToken.all, AlgebraToken.all, AlgebraToken.all })).attribute(), + }) catch @panic("invalid block-scaled tiled MMA")).type_(); + } + + fn blockScaledAtomType(self: *Builder, cfg: BlockScaledMmaConfig) *const mlir.Type { + return (cute_nvgpu.MmaAtomSM100UMMABlockScaledType.get(self.ctx, .{ + .shapeMnk = self.algebraAttr(cute_ir.ShapeAttr, self.expr(.{ cfg.m, cfg.n, cfg.k })).attribute(), + .numCta = cfg.num_cta, + .aMajor = .k, + .bMajor = .k, + .aType = DType.f4e2m1fn.toMlir(self.ctx), + .bType = DType.f8e4m3fn.toMlir(self.ctx), + .cType = DType.f32.toMlir(self.ctx), + .sfType = DType.f8e8m0fnu.toMlir(self.ctx), + .aFragKind = .smem_desc, + .vecSize = cfg.vec_size, + }) catch @panic("invalid block-scaled MMA atom")).type_(); } /// SS, K-major MXFP4 x MXFP8 block-scaled atom and its tiled MMA, as the @@ -2168,20 +1901,19 @@ pub const Builder = struct { std.debug.panic("unsupported SM100 block-scaled MMA shape {d}x{d}x{d}", .{ cfg.m, cfg.n, cfg.k }); } const disabled = self.cst(.i1, false); - const atom_type = std.fmt.allocPrint(self.arena.allocator(), "!cute_nvgpu.sm100.mma_bs{s}", .{blockScaledPayload(self, cfg)}) catch @panic("OOM"); - const atom = self.makeAtom(atom_type, .{ disabled, disabled, disabled, scale_ptr, scale_ptr }); - const tiled = self.emit(cute.make_tiled_mma(self.ctx, atom.inner, self.blockScaledMmaType(cfg), self.loc())); + const atom = self.makeAtom(self.blockScaledAtomType(cfg), .{ disabled, disabled, disabled, scale_ptr, scale_ptr }); + const tiled = self.emit(cute_ir.make_tiled_mma(self.ctx, atom.inner, self.blockScaledMmaType(cfg), self.loc())); return .{ .atom = atom, .tiled = .{ .inner = tiled.inner, .kernel = self } }; } /// Enable (or disable) accumulation into C for the next block-scaled MMA. pub fn setMmaAccumulate(self: *Builder, mma: Atom, enabled: Value) Atom { - const result = self.emit(cute.nvgpu.atom_set_value( + const result = self.emit(cute_nvgpu.atom_set_value( self.ctx, mma.inner, enabled.inner, mma.type_(), - self.parseAttribute("#cute_nvgpu.atom_mma_field_sm100_block_scaled"), + (cute_nvgpu.AtomMmaFieldSM100BlockScaledAttr.get(self.ctx, .{ .value = .accum_c }) catch @panic("invalid block-scaled MMA field")).attribute(), self.loc(), )); return .{ .inner = result.inner, .kernel = self }; @@ -2201,32 +1933,30 @@ pub const Builder = struct { pub const TmemOptions = struct { two_cta: bool = false, exclusive: bool = false }; pub fn allocTmem(self: *Builder, columns: u16, holding_ptr: anytype, opts: TmemOptions) void { - self.emitVoid(cute.nvgpu.arch_sm100_alloc_tmem( + self.emitVoid(cute_nvgpu.arch_sm100_alloc_tmem( self.ctx, self.cst(.i32, columns).inner, self.asValue(holding_ptr).inner, - if (opts.two_cta) .unit(self.ctx) else null, - if (opts.exclusive) .unit(self.ctx) else null, + .{ .is_two_cta = if (opts.two_cta) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, self.loc(), )); } pub fn retrieveTmemPtr(self: *Builder, holding_ptr: anytype, dtype: DType, alignment: u64) Value { const ptr_type = self.ptrTy(dtype, .tmem, alignment) catch @panic("invalid tensor-memory pointer type"); - return self.emit(cute.nvgpu.arch_sm100_retrieve_tmem_ptr(self.ctx, self.asValue(holding_ptr).inner, ptr_type, self.loc())); + return self.emit(cute_nvgpu.arch_sm100_retrieve_tmem_ptr(self.ctx, self.asValue(holding_ptr).inner, ptr_type, self.loc())); } pub fn relinquishTmemAllocPermit(self: *Builder, opts: TmemOptions) void { - self.emitVoid(cute.nvgpu.arch_sm100_relinquish_tmem_alloc_permit(self.ctx, if (opts.two_cta) .unit(self.ctx) else null, self.loc())); + self.emitVoid(cute_nvgpu.arch_sm100_relinquish_tmem_alloc_permit(self.ctx, .{ .is_two_cta = if (opts.two_cta) .unit(self.ctx) else null }, self.loc())); } pub fn deallocTmem(self: *Builder, pointer: anytype, columns: u16, opts: TmemOptions) void { - self.emitVoid(cute.nvgpu.arch_sm100_dealloc_tmem( + self.emitVoid(cute_nvgpu.arch_sm100_dealloc_tmem( self.ctx, self.asValue(pointer).inner, self.cst(.i32, columns).inner, - if (opts.two_cta) .unit(self.ctx) else null, - if (opts.exclusive) .unit(self.ctx) else null, + .{ .is_two_cta = if (opts.two_cta) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, self.loc(), )); } @@ -2238,15 +1968,15 @@ pub const Builder = struct { pack_16: bool = false, }; - pub fn tmemLoad(self: *Builder, src: anytype, result_type: []const u8, opts: TmemLoadOptions) View { - const result = self.emit(cute.nvgpu.arch_copy_SM100_tmem_load( + pub fn tmemLoad(self: *Builder, src: anytype, result_type: *const mlir.Type, opts: TmemLoadOptions) View { + const result = self.emit(cute_nvgpu.arch_copy_SM100_tmem_load( self.ctx, self.asValue(src).inner, - self.parseType(result_type), + result_type, .int(self.ctx, .i32, opts.num_dp), .int(self.ctx, .i32, opts.num_b), .int(self.ctx, .i32, opts.num_rep), - if (opts.pack_16) .boolean(self.ctx, true) else null, + .{ .pack_16 = if (opts.pack_16) .boolean(self.ctx, true) else null }, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; @@ -2258,46 +1988,64 @@ pub const Builder = struct { // The NVVM-facing operation transports raw 32-bit register words. // Callers reinterpret each lane as the accumulator element type. const result_type = mlir.Type.vector(&.{lanes}, DType.i32.toMlir(self.ctx)); - return self.emit(cute.nvgpu.arch_copy_SM100_tmem_load( + return self.emit(cute_nvgpu.arch_copy_SM100_tmem_load( self.ctx, self.asValue(src).inner, result_type, .int(self.ctx, .i32, opts.num_dp), .int(self.ctx, .i32, opts.num_b), .int(self.ctx, .i32, opts.num_rep), - if (opts.pack_16) .boolean(self.ctx, true) else null, + .{ .pack_16 = if (opts.pack_16) .boolean(self.ctx, true) else null }, self.loc(), )); } - pub fn makeS2tCopy(self: *Builder, atom: Atom, tmem: anytype, result_type: []const u8) Atom { - const result = self.emit(cute.nvgpu.atom_make_s2t_copy( + pub const S2tCopyConfig = struct { + num_dp: i32, + num_bit: i32, + num_cta: i32 = 1, + broadcast: cute_nvgpu.CopyS2TBroadcast = .none, + }; + + /// `!cute_nvgpu.atom.s2t_copy`, a `tcgen05.cp` atom moving `dtype`. + pub fn s2tCopyAtomType(self: *Builder, dtype: DType, cfg: S2tCopyConfig) *const mlir.Type { + return (cute_nvgpu.CopyAtomSM100CopyS2TType.get(self.ctx, .{ + .valType = dtype.toMlir(self.ctx), + .numDp = cfg.num_dp, + .numBit = cfg.num_bit, + .numCta = cfg.num_cta, + .broadcast = cfg.broadcast, + }) catch @panic("invalid S2T copy atom")).type_(); + } + + pub fn makeS2tCopy(self: *Builder, atom: Atom, tmem: anytype, result_type: *const mlir.Type) Atom { + const result = self.emit(cute_nvgpu.atom_make_s2t_copy( self.ctx, atom.inner, self.asValue(tmem).inner, - self.parseType(result_type), + result_type, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub fn makeTmemCopy(self: *Builder, atom: Atom, tmem: anytype, result_type: []const u8) Atom { - const result = self.emit(cute.nvgpu.atom_make_tmem_copy( + pub fn makeTmemCopy(self: *Builder, atom: Atom, tmem: anytype, result_type: *const mlir.Type) Atom { + const result = self.emit(cute_nvgpu.atom_make_tmem_copy( self.ctx, atom.inner, self.asValue(tmem).inner, - self.parseType(result_type), + result_type, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub fn s2tSmemDescriptor(self: *Builder, atom: Atom, input: anytype, result_type: []const u8) View { - const result = self.emit(cute.nvgpu.atom_get_copy_s2t_smem_desc_view( + pub fn s2tSmemDescriptor(self: *Builder, atom: Atom, input: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_nvgpu.atom_get_copy_s2t_smem_desc_view( self.ctx, atom.inner, self.asValue(input).inner, - self.parseType(result_type), + result_type, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; @@ -2317,17 +2065,16 @@ pub const Builder = struct { mode: *const mlir.Attribute, num_cta: u8, ) void { - self.emitVoid(cute.nvgpu.arch_copy_SM100_tma_load( + self.emitVoid(cute_nvgpu.arch_copy_SM100_tma_load( self.ctx, descriptor_addr.inner, data_addr.inner, barrier_addr.inner, self.innerSlice(coords), - if (multicast_mask) |v| v.inner else null, self.innerSlice(offsets), - if (cache_policy) |v| v.inner else null, mode, .int(self.ctx, .i32, num_cta), + .{ .multicast_mask = if (multicast_mask) |v| v.inner else null, .cache_policy = if (cache_policy) |v| v.inner else null }, self.loc(), )); } @@ -2341,13 +2088,13 @@ pub const Builder = struct { cache_policy: ?Value, mode: *const mlir.Attribute, ) void { - self.emitVoid(cute.nvgpu.arch_copy_SM100_tma_store( + self.emitVoid(cute_nvgpu.arch_copy_SM100_tma_store( self.ctx, descriptor_addr.inner, data_addr.inner, self.innerSlice(coords), - if (cache_policy) |v| v.inner else null, mode, + .{ .cache_policy = if (cache_policy) |v| v.inner else null }, self.loc(), )); } @@ -2388,7 +2135,7 @@ pub const Builder = struct { // `arch.make_warp_uniform` on i1 parses, but the native SM100 // pipeline does not lower it all the way to NVVM. const uniform_input = if (scalar.dtype() == .i1) scalar.to(.i32) else scalar; - const uniform = self.emit(cute.nvgpu.arch_make_warp_uniform( + const uniform = self.emit(cute_nvgpu.arch_make_warp_uniform( self.ctx, uniform_input.inner, uniform_input.type_(), @@ -2417,7 +2164,7 @@ pub const Builder = struct { "bar.sync {d}, {d};", .{ id, participants }, ) catch @panic("namedBarrier OOM"); - _ = llvm.inline_asm(self.ctx, assembly, &.{}, null, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + _ = llvm.inline_asm(self.ctx, assembly, &.{}, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); } /// Build the CuTe first-class divisor used by CUTLASS's persistent tile @@ -2425,10 +2172,10 @@ pub const Builder = struct { /// sequence as Python CuTe DSL rather than an integer div/rem pair. pub fn fastDivmodCreate(self: *Builder, divisor_: anytype) Value { const divisor = self.lift(divisor_).to(.i32); - return self.emit(cute.fast_divmod_create_divisor( + return self.emit(cute_ir.fast_divmod_create_divisor( self.ctx, divisor.inner, - self.parseType("!cute.fast_divmod_divisor<32>"), + (cute_ir.FastDivmodDivisorType.get(self.ctx, .{ .width = 32, .isPow2 = false }) catch @panic("invalid fast divmod divisor")).type_(), self.loc(), )); } @@ -2436,7 +2183,7 @@ pub const Builder = struct { /// Return `{ quotient, remainder }` for a CUTLASS FastDivmod divisor. pub fn fastDivmod(self: *Builder, dividend_: anytype, divisor: Value) [2]Value { const dividend = self.lift(dividend_).to(.i32); - return self.emitResults(2, cute.fast_divmod_compute( + return self.emitResults(2, cute_ir.fast_divmod_compute( self.ctx, dividend.inner, divisor.inner, @@ -2461,7 +2208,13 @@ pub const Builder = struct { const dtype = value.dtype(); if (dtypeBitwidth(dtype) != 32) @panic("warpMaxNonNegative requires a 32-bit scalar"); const bits = if (dtype == .i32) value else value.bitCast(.i32); - const reduced = self.emit(llvm.inline_asm(self.ctx, "redux.sync.max.u32 $0, $1, 0xffffffff;", &.{bits.inner}, .int(self.ctx, .i32), .{ .constraints = "=r,r" }, self.loc())); + const reduced = self.emit(llvm.inline_asm( + self.ctx, + "redux.sync.max.u32 $0, $1, 0xffffffff;", + &.{bits.inner}, + .{ .constraints = "=r,r", .result_type = .int(self.ctx, .i32) }, + self.loc(), + )); return if (dtype == .i32) reduced else reduced.bitCast(dtype); } @@ -2469,7 +2222,13 @@ pub const Builder = struct { const dtype = value.dtype(); if (dtypeBitwidth(dtype) != 32) @panic("shuffleXor currently requires a 32-bit scalar"); const bits = if (dtype == .i32) value else value.bitCast(.i32); - const shuffled = self.emit(llvm.inline_asm(self.ctx, "shfl.sync.bfly.b32 $0, $1, $2, 31, 0xffffffff;", &.{ bits.inner, self.cst(.i32, offset).inner }, .int(self.ctx, .i32), .{ .constraints = "=r,r,r" }, self.loc())); + const shuffled = self.emit(llvm.inline_asm( + self.ctx, + "shfl.sync.bfly.b32 $0, $1, $2, 31, 0xffffffff;", + &.{ bits.inner, self.cst(.i32, offset).inner }, + .{ .constraints = "=r,r,r", .result_type = .int(self.ctx, .i32) }, + self.loc(), + )); return if (dtype == .i32) shuffled else shuffled.bitCast(dtype); } @@ -2479,7 +2238,13 @@ pub const Builder = struct { const dtype = value.dtype(); if (dtypeBitwidth(dtype) != 32) @panic("shuffleIdx currently requires a 32-bit scalar"); const bits = if (dtype == .i32) value else value.bitCast(.i32); - const shuffled = self.emit(llvm.inline_asm(self.ctx, "shfl.sync.idx.b32 $0, $1, $2, 31, 0xffffffff;", &.{ bits.inner, self.liftAs(lane, .i32).inner }, .int(self.ctx, .i32), .{ .constraints = "=r,r,r" }, self.loc())); + const shuffled = self.emit(llvm.inline_asm( + self.ctx, + "shfl.sync.idx.b32 $0, $1, $2, 31, 0xffffffff;", + &.{ bits.inner, self.liftAs(lane, .i32).inner }, + .{ .constraints = "=r,r,r", .result_type = .int(self.ctx, .i32) }, + self.loc(), + )); return if (dtype == .i32) shuffled else shuffled.bitCast(dtype); } @@ -2490,28 +2255,33 @@ pub const Builder = struct { self.ctx, "{ .reg .b16 lo, hi; cvt.rn.satfinite.e4m3x2.f32 lo, $2, $1; cvt.rn.satfinite.e4m3x2.f32 hi, $4, $3; mov.b32 $0, {lo, hi}; }", &.{ a.to(.f32).inner, b.to(.f32).inner, c.to(.f32).inner, d.to(.f32).inner }, - .int(self.ctx, .i32), - .{ .constraints = "=r,f,f,f,f" }, + .{ .constraints = "=r,f,f,f,f", .result_type = .int(self.ctx, .i32) }, self.loc(), )); } /// One-operand FP32 PTX instruction, e.g. `ex2.approx.ftz.f32`. pub fn unaryF32(self: *Builder, comptime instruction: []const u8, value: Value) Value { - return self.emit(llvm.inline_asm(self.ctx, instruction ++ " $0, $1;", &.{value.to(.f32).inner}, DType.f32.toMlir(self.ctx), .{ .constraints = "=f,f" }, self.loc())); + return self.emit(llvm.inline_asm( + self.ctx, + instruction ++ " $0, $1;", + &.{value.to(.f32).inner}, + .{ .constraints = "=f,f", .result_type = DType.f32.toMlir(self.ctx) }, + self.loc(), + )); } /// Signal programmatic dependent launches after this grid has consumed /// its inputs. The caller must also launch the producer/consumer chain /// with PDL enabled. pub fn launchDependents(self: *Builder) void { - _ = llvm.inline_asm(self.ctx, "griddepcontrol.launch_dependents;", &.{}, null, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + _ = llvm.inline_asm(self.ctx, "griddepcontrol.launch_dependents;", &.{}, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); } /// Wait until the preceding grid has made its programmatic dependency /// available. This is the consumer-side half of a PDL launch chain. pub fn waitForDependency(self: *Builder) void { - _ = llvm.inline_asm(self.ctx, "griddepcontrol.wait;", &.{}, null, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + _ = llvm.inline_asm(self.ctx, "griddepcontrol.wait;", &.{}, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); } /// `cute.arch.alloc_smem(dtype, size_in_elems, alignment)`: a static @@ -2519,7 +2289,7 @@ pub const Builder = struct { pub fn allocSmem(self: *Builder, dt: DType, size_in_elems: i64, alignment: ?u64) Value { const al = alignment orelse @max(1, dtypeBitwidth(dt) / 8); const ty = self.ptrTy(dt, .smem, al) catch @panic("bad smem pointer"); - return self.emit(cute.nvgpu.arch_alloc_smem(self.ctx, ty, .int(self.ctx, .i32, size_in_elems), self.loc())); + return self.emit(cute_nvgpu.arch_alloc_smem(self.ctx, ty, .int(self.ctx, .i32, size_in_elems), self.loc())); } // mbarriers, async-proxy fences and bulk-copy groups. @@ -2687,7 +2457,7 @@ pub const Builder = struct { const alloc = self.arena.allocator(); const inner = alloc.alloc(*const mlir.Value, operands.len) catch @panic("OOM"); for (operands, inner) |o, *x| x.* = o.inner; - const op = llvm.call_intrinsic(self.ctx, name, inner, result, self.loc()); + const op = llvm.call_intrinsic(self.ctx, name, inner, .{ .result_type = result }, self.loc()); if (result == null) { self.emitVoid(op); return null; @@ -3085,8 +2855,8 @@ pub const Builder = struct { fn testContext() !*mlir.Context { const registry = try mlir.DialectRegistry.init(); defer registry.deinit(); - inline for (dialects_needed) |d| mlir.DialectHandle.fromString(d).insertDialect(registry); - mlir.registerFuncExtensions(registry); + registerDialects(registry); + func.registerExtensions(registry); const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); ctx.loadAllAvailableDialects(); return ctx; @@ -3295,18 +3065,22 @@ test "complete program emits a Zig-built CUDA host launch" { test "SM100 block-scaled MMA atoms use the CuTe DSL instruction shapes" { const registry = try mlir.DialectRegistry.init(); defer registry.deinit(); - inline for (dialects_needed) |dialect| mlir.DialectHandle.fromString(dialect).insertDialect(registry); - mlir.registerFuncExtensions(registry); + registerDialects(registry); + func.registerExtensions(registry); const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); defer ctx.deinit(); ctx.loadAllAvailableDialects(); var b = try Builder.open(std.testing.allocator, ctx, "sm100_blockscaled_atoms"); defer b.deinit(); - _ = try b.declareArgs(.{ .descriptor = .{ .raw = .{ - .type_text = "!cute_nvgpu.atom.non_exec_tiled_tma_load, tma_format = U8>", - .grid_constant = true, - } } }); + const descriptor = try cute_nvgpu.CopyAtomNonExecTiledTmaLoadType.get(ctx, .{ + .kind = .sm_90, + .valType = DType.f8e4m3fn.toMlir(ctx), + .copyBits = 16384, + .tmaGbasis = b.layoutType(b.layoutSpec(.{ 128, 16, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) })), + .tmaFormat = .U8, + }); + _ = try b.declareArgs(.{ .descriptor = .{ .mlir_type = .{ .type_ = descriptor.type_(), .grid_constant = true } } }); const sf_ptr = b.intToPtr(0, .f8e8m0fnu, .tmem, 1); _ = b.makeBlockScaledMma(.{ .n = 16 }, sf_ptr); diff --git a/kernels/cute/dtype.zig b/dsl/cute/dtype.zig similarity index 100% rename from kernels/cute/dtype.zig rename to dsl/cute/dtype.zig diff --git a/dsl/fly/BUILD.bazel b/dsl/fly/BUILD.bazel new file mode 100644 index 00000000..492134dc --- /dev/null +++ b/dsl/fly/BUILD.bazel @@ -0,0 +1,35 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "fly_builder", + srcs = [ + "dtype.zig", + "layout.zig", + ], + import_name = "dsl/fly/builder", + main = "builder.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//dsl/common", + "//mlir", + "//mlir/dialects/arith", + "//mlir/dialects/fly", + "//mlir/dialects/fly_rocdl", + "//mlir/dialects/func", + "//mlir/dialects/gpu", + "//mlir/dialects/llvm", + "//mlir/dialects/math", + "//mlir/dialects/memref", + "//mlir/dialects/rocdl", + "//mlir/dialects/scf", + "//mlir/dialects/ub", + "//mlir/dialects/vector", + "//stdx", + ], +) + +zig_test( + name = "test", + deps = [":fly_builder"], +) diff --git a/kernels/fly/PORTING.md b/dsl/fly/PORTING.md similarity index 93% rename from kernels/fly/PORTING.md rename to dsl/fly/PORTING.md index 21673dbe..491db667 100644 --- a/kernels/fly/PORTING.md +++ b/dsl/fly/PORTING.md @@ -31,12 +31,12 @@ runtime must agree on the dialect version. FlyDSL HEAD may differ from both. | `fx.make_view(ptr, layout)` | `ptr.view(layout)` | | `fx.make_copy_atom(fx.UniversalCopy128b(), fx.Float32)` | `b.copyAtom(.{ .universal = 128 }, .f32)` | | `fx.rocdl.BufferCopy32b()` / `BufferCopyLDS128b()` | `.{ .buffer_copy = 32 }` / `.{ .buffer_copy_lds = 128 }` | -| `fx.make_mma_atom(fx.rocdl.MFMA(16,16,4,fx.Float32))` | `b.mmaAtom((try fly.rocdl.MmaOpCDNA3MFMAType.get(b.ctx, .{ .m = 16, .n = 16, .k = 4, ... })).type_())` | +| `fx.make_mma_atom(fx.rocdl.MFMA(16,16,4,fx.Float32))` | `b.mmaAtom((try fly.fly_rocdl.MmaOpCDNA3MFMAType.get(b.ctx, .{ .m = 16, .n = 16, .k = 4, ... })).type_())` | | `fx.make_tiled_mma(atom, layout)` | `b.tiledMma(atom, layout, null)` | | `fx.make_tiled_copy_tv(atom, thr, val)` | `b.tiledCopyTV(atom, thr, val)` | | `fx.make_tiled_copy_A(atom, tiled_mma)` | `b.tiledCopyA(atom, tiled_mma)` | | `tiled_copy.tile_mn` | `tc.tileMN()` | -| `tiled_mma.tile_size_mnk` etc. | `fly.expect(tm.value.type_(), .tiled_mma).getTileSizeMNK()` | +| `tiled_mma.tile_size_mnk` etc. | `@import("mlir/dialects/fly").expect(tm.value.type_(), .tiled_mma).tileSizeMNK()` | | `tiled_copy.get_slice(tid).partition_S(t)` | `tc.getSlice(tid).partitionS(t)` | | `thr_copy.retile(t)` | `thr.retile(t)` | | `tiled_mma.make_fragment_A(t)` | `tm.makeFragmentA(t)` | @@ -70,7 +70,7 @@ runtime must agree on the dialect version. FlyDSL HEAD may differ from both. `b.copyAtomLoad` returns vectors, `b.mmaAtomCall` returns the accumulator, and `b.copyAtomStore` writes vectors. Carry accumulator values through `openFor`; do not put mutable register tensors across control-flow regions. -3. `bazel test //kernels/fly:test` (or a `K.emit` test): the module must +3. `bazel test //dsl/fly:test` (or a `K.emit` test): the module must verify and re-parse. A panic from `fly.: cannot create operation` lists the operand types; compare them with the Python trace. 4. Run on the ROCm box against a host reference @@ -97,8 +97,8 @@ runtime must agree on the dialect version. FlyDSL HEAD may differ from both. ## Model kernels -The [MoE kernels](../../zml/moe/fly_kernels/) include native routing and -MXFP4 GEMV/MFMA implementations. The -[sparse MLA kernel](../../zml/attention/fly_kernels/sparse_mla.zig) uses SSA +The [MoE kernels](../../zml/kernels/moe/fly/) include native routing and +MXFP4 GEMV/MFMA implementations. llmd's sparse MLA kernel +(`llmd/kernels/mla/fly/sparse_mla.zig`, downstream) uses SSA matrix accumulators through attention loops. These are examples of the builder APIs with model tensor layouts and launch wrappers. diff --git a/kernels/fly/README.md b/dsl/fly/README.md similarity index 93% rename from kernels/fly/README.md rename to dsl/fly/README.md index 0f0a86f7..266d643e 100644 --- a/kernels/fly/README.md +++ b/dsl/fly/README.md @@ -7,7 +7,7 @@ call; ZML never links a GPU compiler. Two layers define the surface: -- **Low layer** — `kernels/fly/builder.zig`: `Builder`, `Value`, `TiledCopy`, +- **Low layer** — `dsl/fly/builder.zig`: `Builder`, `Value`, `TiledCopy`, `TiledMma`, the comptime layout literals in `layout.zig`, and `DType`. `Builder.open` takes an `*mlir.Context` from `zml.kernel.fly.newContext()`. - **High layer** — `zml.kernel.fly.Kernel(Config, spec)`: names the kernel's @@ -85,12 +85,11 @@ and `zml/kernel.zig`. Model kernels live beside their consumers: -- [MoE routing](../../zml/moe/fly_kernels/moe.zig) and - [MXFP4 projections](../../zml/moe/fly_kernels/mxfp4.zig). -- [Sparse MLA](../../zml/attention/fly_kernels/sparse_mla.zig). +- [MoE routing and MXFP4 projections](../../zml/kernels/moe/fly/moe.zig). +- Sparse MLA, downstream in llmd (`llmd/kernels/mla/fly/sparse_mla.zig`). -Fly is a separate backend. The [MoE selector](../../zml/moe/moe.zig) and -[sparse-MLA selector](../../zml/attention/sparse_mla.zig) select by platform, +Fly is a separate backend. The [MoE selector](../../zml/kernels/moe/moe.zig) and +llmd's sparse-MLA selector select by platform, architecture and dtype. The current native kernels target gfx942; their callers check shape support before routing or launching kernels and use Triton for unsupported contracts. @@ -168,8 +167,8 @@ inferred type back on the host; `v.emitShape()` / `v.emitSize()` emit ops. types and layout; this call performs no tiled partitioning. - `b.bufferTensor(t)` is `fx.rocdl.make_buffer_tensor`: the same tensor over a CDNA raw-buffer descriptor, for bounds-checked `buffer_copy` atoms. -- Derived layouts come off the type: `fly.expect(tm.value.type_(), .tiled_mma)` - then `.getTileSizeMNK()`, `.getTiledThrValLayoutA()`, ... +- Derived layouts come off the type: `@import("mlir/dialects/fly").expect(tm.value.type_(), .tiled_mma)` + then `.tileSizeMNK()`, `.tiledThrValLayoutA()`, ... ## Memory and scalars @@ -213,7 +212,7 @@ Use `workgroup` scope for LDS pointers and `agent` scope for global pointers. The ordering is monotonic; callers provide barriers or kernel dependencies before consuming completed counters. -Control flow uses `kernels/common`: `b.openFor(lo, hi, step, .{ inits })` +Control flow uses `dsl/common`: `b.openFor(lo, hi, step, .{ inits })` with `.yield(.{ ... })`, `b.openIf(cond)` with `.yieldThen(.{})`, `b.openIfElse(cond, .{ types })` with `.yieldThen` / `.yieldElse`. `openWhile` has separate condition and body regions: `before_carried` @@ -241,6 +240,6 @@ runtime. Straight-line kernels can still use `rmemTensor`, - A failing `inferReturnTypes` panics with the op name and every operand type (`fly.make_tiled_copy: cannot create operation ... operand 1: !fly.layout<...>`). -- `Kernel.emit` returns the module text; `bazel test //kernels/fly:test` +- `Kernel.emit` returns the module text; `bazel test //dsl/fly:test` builds the example kernels on any host, no GPU needed. - On the device side, `--xla_dump_to=` dumps the module the plugin saw. diff --git a/kernels/fly/builder.zig b/dsl/fly/builder.zig similarity index 94% rename from kernels/fly/builder.zig rename to dsl/fly/builder.zig index ed3b18f9..89ff45cc 100644 --- a/kernels/fly/builder.zig +++ b/dsl/fly/builder.zig @@ -1,11 +1,17 @@ const std = @import("std"); -const dialects = @import("mlir/dialects"); -const arith = dialects.arith; -const gpu = dialects.gpu; -const scf = dialects.scf; +const arith = @import("mlir/dialects/arith"); +const dsl = @import("dsl/common"); const fly = @import("mlir/dialects/fly"); -const dsl = @import("kernels/common"); +const func = @import("mlir/dialects/func"); +const gpu = @import("mlir/dialects/gpu"); +const llvm = @import("mlir/dialects/llvm"); +const math = @import("mlir/dialects/math"); +const memref = @import("mlir/dialects/memref"); +const rocdl = @import("mlir/dialects/rocdl"); +const scf = @import("mlir/dialects/scf"); +const ub = @import("mlir/dialects/ub"); +const vector = @import("mlir/dialects/vector"); const tupleArity = dsl.tupleArity; const mlir = @import("mlir"); const stdx = @import("stdx"); @@ -14,7 +20,7 @@ pub const layout = @import("layout.zig"); pub const dtypes = @import("dtype.zig"); pub const DType = dtypes.DType; -pub const rocdl = fly.rocdl; +pub const fly_rocdl = @import("mlir/dialects/fly_rocdl"); pub const IntTuple = layout.IntTuple; pub const Layout = layout.Layout; pub const Tile = layout.Tile; @@ -25,7 +31,9 @@ pub const colMajor = layout.colMajor; pub const ordered = layout.ordered; pub const tile = layout.tile; -pub const dialects_needed = fly.dialects_needed; +pub fn registerDialects(registry: *mlir.DialectRegistry) void { + inline for (.{ func, arith, scf, math, vector, memref, llvm, gpu, rocdl, ub, fly, fly_rocdl }) |dialect| dialect.registerDialect(registry); +} pub const FinishError = error{ InvalidMlir, OutOfMemory } || std.Io.Writer.Error; @@ -159,11 +167,11 @@ pub const Value = struct { } pub fn coalesce(self: Value) Value { const k = self.kern(); - return k.emit(fly.ops.coalesce(k.ctx, self.inner, null, k.loc())); + return k.emit(fly.ops.coalesce(k.ctx, self.inner, .{}, k.loc())); } pub fn complement(self: Value) Value { const k = self.kern(); - return k.emit(fly.ops.complement(k.ctx, self.inner, null, k.loc())); + return k.emit(fly.ops.complement(k.ctx, self.inner, .{}, k.loc())); } /// `t[coord]` with a `null` somewhere; `coord` is a tuple literal @@ -224,7 +232,7 @@ pub const Value = struct { pub fn makeFragmentLike(self: Value, dtype: ?DType) Value { const k = self.kern(); const dtype_attr: ?*const mlir.Attribute = if (dtype) |d| .typeAttr(d.toMlir(k.ctx)) else null; - return k.emit(fly.ops.make_fragment_like(k.ctx, self.inner, dtype_attr, k.loc())); + return k.emit(fly.ops.make_fragment_like(k.ctx, self.inner, .{ .dtype = dtype_attr }, k.loc())); } pub fn elemLess(self: Value, rhs: anytype) Value { @@ -253,9 +261,9 @@ pub const Value = struct { return self.unary(fly.ops.memref_load_vec); } - pub fn store(self: Value, vector: Value) void { + pub fn store(self: Value, vector_: Value) void { const k = self.kern(); - k.emitNone(fly.ops.memref_store_vec(k.ctx, vector.inner, self.inner, k.loc())); + k.emitNone(fly.ops.memref_store_vec(k.ctx, vector_.inner, self.inner, k.loc())); } /// Store a splat vector of the tensor's static size. @@ -265,7 +273,7 @@ pub const Value = struct { const dt = self.elemDType(); const cst = k.constant(dt, value); const vec_ty = mlir.Type.vector(&.{n}, dt.toMlir(k.ctx)); - const vec = k.emit(dialects.vector.broadcast(k.ctx, cst.inner, vec_ty, k.loc())); + const vec = k.emit(vector.broadcast(k.ctx, cst.inner, vec_ty, k.loc())); self.store(vec); } @@ -302,9 +310,9 @@ pub const Value = struct { const r: Value = if (@TypeOf(rhs) == Value) rhs else k.constLike(rhs, self); if (self.type_().eql(r.type_())) return .{ self, r }; if (self.type_().isA(mlir.VectorType) != null and r.type_().isA(mlir.VectorType) == null) - return .{ self, k.emit(dialects.vector.broadcast(k.ctx, r.inner, self.type_(), k.loc())) }; + return .{ self, k.emit(vector.broadcast(k.ctx, r.inner, self.type_(), k.loc())) }; if (self.type_().isA(mlir.VectorType) == null and r.type_().isA(mlir.VectorType) != null) - return .{ k.emit(dialects.vector.broadcast(k.ctx, self.inner, r.type_(), k.loc())), r }; + return .{ k.emit(vector.broadcast(k.ctx, self.inner, r.type_(), k.loc())), r }; std.debug.panic("fly: cannot combine {f} with {f}", .{ self.type_(), r.type_() }); } @@ -421,7 +429,7 @@ pub const Value = struct { pub fn splat(self: Value, n: i64) Value { const k = self.kern(); std.debug.assert(self.type_().isA(mlir.VectorType) == null and n > 0); - return k.emit(dialects.vector.broadcast(k.ctx, self.inner, .vector(&.{n}, self.type_()), k.loc())); + return k.emit(vector.broadcast(k.ctx, self.inner, .vector(&.{n}, self.type_()), k.loc())); } /// Preserve total bits, adjusting vector length to the destination element. @@ -430,7 +438,7 @@ pub const Value = struct { const src = self.scalarDType(); if (self.type_().isA(mlir.VectorType)) |vec| { const n = @divExact(vec.dimension(0) * @as(i64, src.bitWidth()), dtype.bitWidth()); - return k.emit(dialects.vector.bitcast(k.ctx, self.inner, .vector(&.{n}, dtype.toMlir(k.ctx)), k.loc())); + return k.emit(vector.bitcast(k.ctx, self.inner, .vector(&.{n}, dtype.toMlir(k.ctx)), k.loc())); } std.debug.assert(src.bitWidth() == dtype.bitWidth()); return k.emit(arith.bitcast(k.ctx, self.inner, dtype.toMlir(k.ctx), k.loc())); @@ -438,12 +446,12 @@ pub const Value = struct { pub fn insert(self: Value, index: i64, value: Value) Value { const k = self.kern(); - return k.emit(dialects.vector.insert(k.ctx, value.inner, self.inner, &.{index}, &.{}, k.loc())); + return k.emit(vector.insert(k.ctx, value.inner, self.inner, &.{index}, &.{}, k.loc())); } pub fn shuffleVector(self: Value, rhs: Value, indices: []const i64) Value { const k = self.kern(); - return k.emit(dialects.vector.shuffle(k.ctx, self.inner, rhs.inner, indices, .vector(&.{@intCast(indices.len)}, self.scalarElemType()), k.loc())); + return k.emit(vector.shuffle(k.ctx, self.inner, rhs.inner, indices, .vector(&.{@intCast(indices.len)}, self.scalarElemType()), k.loc())); } fn withElemType(self: Value, elem: *const mlir.Type) *const mlir.Type { @@ -546,7 +554,7 @@ pub const Value = struct { pub fn reduce(self: Value, reduce_kind: ReduceKind) Value { const k = self.kern(); const result = self.scalarElemType(); - const ck: dialects.vector.CombiningKind = switch (reduce_kind) { + const ck: vector.CombiningKind = switch (reduce_kind) { .add => .add, .mul => .mul, .minf => .minnumf, @@ -557,14 +565,14 @@ pub const Value = struct { .or_ => .@"or", .xor => .xor, }; - const op = dialects.vector.reduction(k.ctx, ck, self.inner, result, .{}, k.loc()); + const op = vector.reduction(k.ctx, ck, self.inner, result, .{}, k.loc()); return k.emitFast(op); } /// Static position. pub fn extract(self: Value, i: i64) Value { const k = self.kern(); - return k.emit(dialects.vector.extract(k.ctx, self.inner, &.{i}, &.{}, self.scalarElemType(), k.loc())); + return k.emit(vector.extract(k.ctx, self.inner, &.{i}, &.{}, self.scalarElemType(), k.loc())); } }; @@ -622,7 +630,7 @@ pub const TiledMma = struct { const k = self.value.kern(); const operand_id = Builder.must(fly.attributes.mmaOperandAttr(k.ctx, operand)); const stages: ?*const mlir.Attribute = if (opts.stages) |st| .int(k.ctx, .i32, st) else null; - return k.emit(fly.ops.mma_make_fragment(k.ctx, self.value.inner, input.inner, operand_id, stages, k.loc())); + return k.emit(fly.ops.mma_make_fragment(k.ctx, self.value.inner, input.inner, operand_id, .{ .stages = stages }, k.loc())); } /// Per-thread register fragment for the A operand. @@ -825,15 +833,15 @@ pub const Builder = struct { /// Arena-allocated. pub fn readIntTuple(self: *Builder, tuple_ty: *const fly.types.IntTupleType) IntTuple { if (tuple_ty.isLeaf()) { - return switch (tuple_ty.getLeaf()) { + return switch (tuple_ty.leaf()) { .static => |v| IntTuple.static(v), .dynamic, .basis => IntTuple.dyn, .none => IntTuple.star, }; } - const n = tuple_ty.getNumElements(); + const n = tuple_ty.numElements(); const elems = self.alloc(IntTuple, n); - for (0..n) |i| elems[i] = self.readIntTuple(tuple_ty.getElement(i)); + for (0..n) |i| elems[i] = self.readIntTuple(tuple_ty.element(i)); return .{ .tup = elems }; } @@ -844,8 +852,8 @@ pub const Builder = struct { /// The shape and stride of a static `!fly.layout`, as host literals. pub fn readLayout(self: *Builder, layout_ty: *const mlir.Type) Layout { const lay = fly.expect(layout_ty, .layout); - const shape = lay.getShape(); - const stride = lay.getStride(); + const shape = lay.shape(); + const stride = lay.stride(); if (!shape.isStatic() or !stride.isStatic()) { std.debug.panic("fly: `{f}` is not static", .{layout_ty}); } @@ -1036,43 +1044,42 @@ pub const Builder = struct { } pub fn rsqrt(self: *Builder, x: Value) Value { - return self.emitFast(dialects.math.rsqrt(self.ctx, x.inner, self.loc())); + return self.emitFast(math.rsqrt(self.ctx, x.inner, self.loc())); } pub fn sqrt(self: *Builder, x: Value) Value { - return self.emitFast(dialects.math.sqrt(self.ctx, x.inner, self.loc())); + return self.emitFast(math.sqrt(self.ctx, x.inner, self.loc())); } pub fn exp(self: *Builder, x: Value) Value { - return self.emitFast(dialects.math.exp(self.ctx, x.inner, self.loc())); + return self.emitFast(math.exp(self.ctx, x.inner, self.loc())); } pub fn log(self: *Builder, x: Value) Value { - return self.emitFast(dialects.math.log(self.ctx, x.inner, self.loc())); + return self.emitFast(math.log(self.ctx, x.inner, self.loc())); } pub fn fma(self: *Builder, a: Value, b: Value, c: Value) Value { - return self.emitFast(dialects.math.fma(self.ctx, a.inner, b.inner, c.inner, self.loc())); + return self.emitFast(math.fma(self.ctx, a.inner, b.inner, c.inner, self.loc())); } /// A matrix atom with SSA operands; unlike `gemm`, this needs no register /// allocas and can carry its accumulator through control flow. pub fn mmaAtomCall(self: *Builder, atom: Value, a: Value, b: Value, c: Value) Value { - return self.emit(fly.ops.mma_atom_call_ssa(self.ctx, atom.inner, null, &.{a.inner}, &.{b.inner}, c.inner, &.{c.type_()}, self.loc())); + return self.emit(fly.ops.mma_atom_call_ssa(self.ctx, atom.inner, &.{a.inner}, &.{b.inner}, c.inner, &.{c.type_()}, .{}, self.loc())); } /// AMD byte permutation. ROCDL has no corresponding operation at our pin. pub fn perm(self: *Builder, hi: Value, lo: Value, selector: Value) Value { - return self.emit(dialects.llvm.inline_asm(self.ctx, "v_perm_b32 $0, $1, $2, $3", &.{ hi.inner, lo.inner, selector.inner }, .int(self.ctx, .i32), .{ - .constraints = "=v,v,v,v", - }, self.loc())); + return self.emit(llvm.inline_asm( + self.ctx, + "v_perm_b32 $0, $1, $2, $3", + &.{ hi.inner, lo.inner, selector.inner }, + .{ .constraints = "=v,v,v,v", .result_type = .int(self.ctx, .i32) }, + self.loc(), + )); } /// CDNA3 packed FP8 conversion, selecting two bytes from the low/high word. pub fn cvtPkF32Fp8(self: *Builder, bits: Value, high: bool) Value { - return self.emit(fly.make(self.ctx, "rocdl.cvt.pk.f32.fp8", .{ - .operands = .{ .flat = &.{bits.inner} }, - .results = .{ .flat = &.{.vector(&.{2}, .float(self.ctx, .f32))} }, - .attributes = &.{.named(self.ctx, "wordSel", .boolean(self.ctx, high))}, - .location = self.loc(), - })); + return self.emit(rocdl.cvtPkF32Fp8(self.ctx, bits.inner, high, self.loc())); } pub const AtomicScope = enum { workgroup, agent }; @@ -1113,14 +1120,14 @@ pub const Builder = struct { .named(self.ctx, "allocBytes", .int(self.ctx, .i64, bytes)), .named(self.ctx, "allocAlign", .int(self.ctx, .i64, alignment)), }); - const raw = self.emit(fly.ops.make_ptr(self.ctx, &.{}, raw_ty, dict, self.loc())); + const raw = self.emit(fly.ops.make_ptr(self.ctx, &.{}, raw_ty, .{ .dict_attrs = dict }, self.loc())); const typed = self.recast(raw, dtype); return typed.view(self.static(Layout{ .shape = .static(n), .stride = .static(1) })); } /// A single atom-sized copy. pub fn copyAtomCall(self: *Builder, atom: Value, src: Value, dst: Value) void { - self.emitNone(fly.ops.copy_atom_call(self.ctx, atom.inner, src.inner, dst.inner, null, self.loc())); + self.emitNone(fly.ops.copy_atom_call(self.ctx, atom.inner, src.inner, dst.inner, .{}, self.loc())); } /// One atom-sized load straight into a `vector`. @@ -1132,12 +1139,12 @@ pub const Builder = struct { const n = src.sizeStatic(); const dt = src.elemDType(); const vec_ty = mlir.Type.vector(&.{n}, dt.toMlir(self.ctx)); - return self.emit(fly.ops.copy_atom_call_ssa(self.ctx, atom.inner, src.inner, null, null, &.{vec_ty}, self.loc())); + return self.emit(fly.ops.copy_atom_call_ssa(self.ctx, atom.inner, src.inner, &.{vec_ty}, .{}, self.loc())); } /// One atom-sized store of a `vector`; see `copyAtomLoad`. pub fn copyAtomStore(self: *Builder, atom: Value, vec: Value, dst: Value) void { - self.emitNone(fly.ops.copy_atom_call_ssa(self.ctx, atom.inner, vec.inner, dst.inner, null, &.{}, self.loc())); + self.emitNone(fly.ops.copy_atom_call_ssa(self.ctx, atom.inner, vec.inner, &.{}, .{ .dst = dst.inner }, self.loc())); } // ---------------------------------------------------------------- tensors @@ -1195,8 +1202,8 @@ pub const Builder = struct { fn type_(self: CopyOp, ctx: *mlir.Context) *const mlir.Type { return switch (self) { .universal => |b| must(fly.types.CopyOpUniversalCopyType.get(ctx, .{ .bitSize = b })).type_(), - .buffer_copy => |b| must(fly.rocdl.CopyOpCDNA3BufferCopyType.get(ctx, .{ .bitSize = b })).type_(), - .buffer_copy_lds => |b| must(fly.rocdl.CopyOpCDNA3BufferCopyLDSType.get(ctx, .{ .bitSize = b })).type_(), + .buffer_copy => |b| must(fly_rocdl.CopyOpCDNA3BufferCopyType.get(ctx, .{ .bitSize = b })).type_(), + .buffer_copy_lds => |b| must(fly_rocdl.CopyOpCDNA3BufferCopyLDSType.get(ctx, .{ .bitSize = b })).type_(), }; } }; @@ -1207,7 +1214,7 @@ pub const Builder = struct { return self.emit(fly.ops.make_copy_atom(self.ctx, &.{}, t, .int(self.ctx, .i32, bits), self.loc())); } - /// `op` is an MMA op type, such as a `fly.rocdl.MmaOpCDNA3MFMAType`. + /// `op` is an MMA op type, such as a `fly_rocdl.MmaOpCDNA3MFMAType`. pub fn mmaAtom(self: *Builder, op: *const mlir.Type) Value { const t = must(fly.types.MmaAtomType.get(self.ctx, .{ .mmaOp = op })).type_(); return self.emit(fly.ops.make_mma_atom(self.ctx, t, self.loc())); @@ -1247,12 +1254,12 @@ pub const Builder = struct { pub fn tiledCopyFor(self: *Builder, operand: fly.MmaOperand, copy_atom: Value, tm: TiledMma) TiledCopy { const tm_ty = fly.expect(tm.value.type_(), .tiled_mma); const layout_tv = self.staticTy((switch (operand) { - .a => tm_ty.getTiledThrValLayoutA(), - .b => tm_ty.getTiledThrValLayoutB(), - .c => tm_ty.getTiledThrValLayoutC(), + .a => tm_ty.tiledThrValLayoutA(), + .b => tm_ty.tiledThrValLayoutB(), + .c => tm_ty.tiledThrValLayoutC(), .d => @panic("fly.tiledCopyFor: no D operand copy"), }).type_()); - const tile_size = self.staticTy(tm_ty.getTileSizeMNK().type_()); + const tile_size = self.staticTy(tm_ty.tileSizeMNK().type_()); const modes: [2]i32 = switch (operand) { .a => .{ 0, 2 }, .b => .{ 1, 2 }, @@ -1289,14 +1296,14 @@ pub const Builder = struct { // (7 << 12) | (4 << 15): the CDNA descriptor flags FlyDSL and XLA use. const flags = self.constant(.i32, 0x27000); const elem = t.elemDType(); - const buf_ty = self.ptrType(elem, (must(fly.rocdl.BufferDescAddressAttr.get(self.ctx))).attribute()); - const buf_ptr = self.emit(fly.ops.make_ptr(self.ctx, &.{ ptr.inner, c0.inner, nrec.inner, flags.inner }, buf_ty, null, self.loc())); + const buf_ty = self.ptrType(elem, (must(fly_rocdl.BufferDescAddressAttr.get(self.ctx))).attribute()); + const buf_ptr = self.emit(fly.ops.make_ptr(self.ctx, &.{ ptr.inner, c0.inner, nrec.inner, flags.inner }, buf_ty, .{}, self.loc())); return buf_ptr.view(lay); } pub fn tiledMma(self: *Builder, atom: Value, atom_layout: anytype, permutation: ?Value) TiledMma { const al = self.lift(atom_layout); - const v = self.emit(fly.ops.make_tiled_mma(self.ctx, atom.inner, al.inner, if (permutation) |p| p.inner else null, self.loc())); + const v = self.emit(fly.ops.make_tiled_mma(self.ctx, atom.inner, al.inner, .{ .permutation = if (permutation) |p| p.inner else null }, self.loc())); return .{ .value = v }; } @@ -1305,11 +1312,11 @@ pub const Builder = struct { }; pub fn copy(self: *Builder, atom: Value, src: Value, dst: Value, opts: CopyOpts) void { - self.emitNone(fly.ops.copy(self.ctx, atom.inner, src.inner, dst.inner, if (opts.pred) |p| p.inner else null, self.loc())); + self.emitNone(fly.ops.copy(self.ctx, atom.inner, src.inner, dst.inner, .{ .pred = if (opts.pred) |p| p.inner else null }, self.loc())); } pub fn gemm(self: *Builder, atom: Value, d: Value, a: Value, b: Value, c: Value) void { - self.emitNone(fly.ops.gemm(self.ctx, atom.inner, d.inner, &.{a.inner}, &.{b.inner}, c.inner, null, null, self.loc())); + self.emitNone(fly.ops.gemm(self.ctx, atom.inner, d.inner, &.{a.inner}, &.{b.inner}, c.inner, .{}, self.loc())); } // ---------------------------------------------------------------- control flow @@ -1479,7 +1486,7 @@ pub const Builder = struct { fn testContext() !*mlir.Context { const registry = try mlir.DialectRegistry.init(); defer registry.deinit(); - fly.insertDialects(registry); + registerDialects(registry); const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); ctx.loadAllAvailableDialects(); return ctx; @@ -1588,7 +1595,7 @@ pub const MmaFlavor = enum { const operand = self.operandDType().toMlir(ctx); const acc: *const mlir.Type = .float(ctx, .f32); return switch (self) { - .cdna3_mfma => (fly.rocdl.MmaOpCDNA3MFMAType.get(ctx, .{ + .cdna3_mfma => (fly_rocdl.MmaOpCDNA3MFMAType.get(ctx, .{ .m = 16, .n = 16, .k = 4, @@ -1597,7 +1604,7 @@ pub const MmaFlavor = enum { .elemTyAcc = acc, }) catch unreachable).type_(), inline .gfx11_wmma, .gfx120x_wmma => |f| blk: { - const T = if (f == .gfx11_wmma) fly.rocdl.MmaOpGFX11WMMAType else fly.rocdl.MmaOpGFX120XWMMAType; + const T = if (f == .gfx11_wmma) fly_rocdl.MmaOpGFX11WMMAType else fly_rocdl.MmaOpGFX120XWMMAType; break :blk (T.get(ctx, .{ .m = 16, .n = 16, @@ -1756,7 +1763,7 @@ test "MmaFlavor.atomThreads matches the dialect" { const atom = try fly.types.MmaAtomType.get(ctx, .{ .mmaOp = flavor.atomType(ctx) }); // `mma_atom.thr_layout` is the wavefront an atom occupies; the launch // geometry is derived from it, so a wrong table silently corrupts C. - const threads = switch (atom.getThrLayout().getShape().getLeaf()) { + const threads = switch (atom.thrLayout().shape().leaf()) { .static => |v| v, else => return error.TestUnexpectedResult, }; @@ -1924,7 +1931,7 @@ test "SSA matrix atom builds, verifies and re-parses" { var b = try Builder.open(std.testing.allocator, ctx, "mma_atom_ssa"); defer b.deinit(); _ = try b.declareArgs(.{}); - const atom_type = try fly.rocdl.MmaOpCDNA3MFMAType.get(ctx, .{ + const atom_type = try fly_rocdl.MmaOpCDNA3MFMAType.get(ctx, .{ .m = 16, .n = 16, .k = 16, diff --git a/kernels/fly/dtype.zig b/dsl/fly/dtype.zig similarity index 100% rename from kernels/fly/dtype.zig rename to dsl/fly/dtype.zig diff --git a/kernels/fly/layout.zig b/dsl/fly/layout.zig similarity index 99% rename from kernels/fly/layout.zig rename to dsl/fly/layout.zig index e70e0e00..06dd64c6 100644 --- a/kernels/fly/layout.zig +++ b/dsl/fly/layout.zig @@ -500,7 +500,7 @@ test "static queries" { fn testContext() !*mlir.Context { const registry = try mlir.DialectRegistry.init(); defer registry.deinit(); - fly.insertDialects(registry); + @import("builder.zig").registerDialects(registry); const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); ctx.loadAllAvailableDialects(); return ctx; diff --git a/kernels/mosaic_tpu/BUILD.bazel b/dsl/mosaic_tpu/BUILD.bazel similarity index 56% rename from kernels/mosaic_tpu/BUILD.bazel rename to dsl/mosaic_tpu/BUILD.bazel index 651d9826..81a4474d 100644 --- a/kernels/mosaic_tpu/BUILD.bazel +++ b/dsl/mosaic_tpu/BUILD.bazel @@ -2,19 +2,26 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "mosaic_tpu_builder", - tags = ["manual"], srcs = [ "dtype.zig", "pipeline.zig", ], - import_name = "kernels/mosaic_tpu/builder", + import_name = "dsl/mosaic_tpu/builder", main = "builder.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ - "//kernels/common", + "//dsl/common", "//mlir", - "//mlir/dialects", + "//mlir/dialects/affine", + "//mlir/dialects/arith", + "//mlir/dialects/cf", + "//mlir/dialects/func", + "//mlir/dialects/math", + "//mlir/dialects/memref", "//mlir/dialects/mosaic_tpu", + "//mlir/dialects/scf", + "//mlir/dialects/vector", "//stdx", ], ) diff --git a/kernels/mosaic_tpu/README.md b/dsl/mosaic_tpu/README.md similarity index 99% rename from kernels/mosaic_tpu/README.md rename to dsl/mosaic_tpu/README.md index a6b01a00..09e5a429 100644 --- a/kernels/mosaic_tpu/README.md +++ b/dsl/mosaic_tpu/README.md @@ -14,7 +14,7 @@ its C API come from the pinned XLA repository. Two layers define the surface: -- **Low layer** — `kernels/mosaic_tpu/builder.zig`: the IR builder primitives. +- **Low layer** — `dsl/mosaic_tpu/builder.zig`: the IR builder primitives. `Builder.open(allocator, ctx, name)` → `b.declareArgsOpts(spec, results, opts)` → body via `b.*` helpers → `b.finishOpts(results, .{ .canonicalize = true })` for the final IR string. Pass `Builder.open` / `Builder.init` a @@ -130,7 +130,7 @@ windows) can be runtime values from a config. > Kernel re-exports — `MyKernel.Kernel.Config`, `.Inputs`, `.Outputs`, > `.Results`, `.CallOpts`). > -> Single-kernel files (e.g. `zml/attention/mosaic_tpu_kernels/ragged_attention.zig`) +> Single-kernel files (e.g. `zml/kernels/paged_attention/mosaic_tpu/ragged_attention.zig`) > can skip the wrapper and put `pub const Cfg` / `pub const Kernel` at > module scope directly — the wrapper is just convenient when several > kernels share a file. @@ -476,7 +476,7 @@ Use these for SMEM args (e.g. `cu_seqlens`) where Pallas emits scalar | `b.broadcastTo(src, shape)` | `vector.broadcast` — leading-dim broadcast or rank-prepend. | | `b.shapeCast(src, new_shape)` | `vector.shape_cast` — reshape preserving total lanes. | | `b.reshape(src, new_shape)` | `tpu.reshape` (TPU-flavored reshape; differs from `shapeCast`). | -| `b.repeat(src, dimension, times)` | `tpu.repeat`. | +| `b.repeat(src, dimension, times)` | `tpu.concatenate` of `times` copies, as `pltpu.repeat`. | | `b.concatenate(sources, dimension)` | `tpu.concatenate`. | | `b.transpose(src, permutation)` | `tpu.transpose`. | | `b.broadcastInSublanes(src, lane)` | `tpu.broadcast_in_sublanes`. | diff --git a/kernels/mosaic_tpu/builder.zig b/dsl/mosaic_tpu/builder.zig similarity index 93% rename from kernels/mosaic_tpu/builder.zig rename to dsl/mosaic_tpu/builder.zig index d88fd6e5..5f54315c 100644 --- a/kernels/mosaic_tpu/builder.zig +++ b/dsl/mosaic_tpu/builder.zig @@ -7,31 +7,31 @@ const std = @import("std"); -const dialects = @import("mlir/dialects"); -const arith = dialects.arith; -const cf = dialects.cf; -const func = dialects.func; -const math = dialects.math; -const memref = dialects.memref; -const scf = dialects.scf; -const vector = dialects.vector; +const affine = @import("mlir/dialects/affine"); +const arith = @import("mlir/dialects/arith"); +const cf = @import("mlir/dialects/cf"); +const func = @import("mlir/dialects/func"); +const math = @import("mlir/dialects/math"); +const memref = @import("mlir/dialects/memref"); +const scf = @import("mlir/dialects/scf"); +const vector = @import("mlir/dialects/vector"); /// `vector`-dialect enums re-exported for convenience. /// Used by `vectorReduction` / `multiReduction`. pub const CombiningKind = vector.CombiningKind; -const dsl = @import("kernels/common"); +const dsl = @import("dsl/common"); const tupleArity = dsl.tupleArity; const mlir = @import("mlir"); const stdx = @import("stdx"); -const tpu = @import("mlir/dialects/mosaic_tpu"); +const mosaic_tpu = @import("mlir/dialects/mosaic_tpu"); /// `tpu`-dialect enums re-exported for convenience. -pub const MemorySpace = tpu.MemorySpace; -pub const ReductionKind = tpu.ReductionKind; -pub const ContractPrecision = tpu.ContractPrecision; -pub const RoundingMode = tpu.RoundingMode; -pub const DimensionSemantics = tpu.DimensionSemantics; -pub const CoreType = tpu.CoreType; -pub const PipelineMode = tpu.PipelineMode; -pub const RevisitMode = tpu.RevisitMode; +pub const MemorySpace = mosaic_tpu.MemorySpace; +pub const ReductionKind = mosaic_tpu.ReductionKind; +pub const ContractPrecision = mosaic_tpu.ContractPrecision; +pub const RoundingMode = mosaic_tpu.RoundingMode; +pub const DimensionSemantics = mosaic_tpu.DimensionSemantics; +pub const CoreType = mosaic_tpu.CoreType; +pub const PipelineMode = mosaic_tpu.PipelineMode; +pub const RevisitMode = mosaic_tpu.RevisitMode; const dtypes = @import("dtype.zig"); pub const DType = dtypes.DType; @@ -383,7 +383,7 @@ pub const MatmulOpts = struct { transpose_lhs_hint: bool = false, precision: ?ContractPrecision = null, /// Optional `#tpu.dot_dimension_numbers<...>` attribute. When omitted the - /// canonicalizer derives one. Build with `tpu.dotDimensionNumbers`. + /// canonicalizer derives one. Build with `mosaic_tpu.dotDimensionNumbersAttr`. dimension_numbers: ?*const mlir.Attribute = null, }; @@ -445,7 +445,9 @@ pub const MinMaxOpts = struct { // Builder — the main DSL builder. // ============================================================================= -pub const dialects_needed = [_][]const u8{ "func", "tpu", "arith", "scf", "math", "cf", "memref", "vector", "affine" }; +pub fn registerDialects(registry: *mlir.DialectRegistry) void { + inline for (.{ func, mosaic_tpu, arith, scf, math, cf, memref, vector, affine }) |dialect| dialect.registerDialect(registry); +} /// The main DSL builder. Create with `init`, populate the body with helpers, /// then call `finish` to get the IR string. @@ -668,13 +670,13 @@ pub const Builder = struct { r.memory_space.attribute(ctx), ), .sem => |s| switch (s) { - .regular => tpu.semaphoreType(ctx), - .dma => tpu.dmaSemaphoreType(ctx), + .regular => mosaic_tpu.semaphoreType(ctx), + .dma => mosaic_tpu.dmaSemaphoreType(ctx), }, .sem_array => |sa| .memRef( switch (sa.kind) { - .regular => tpu.semaphoreType(ctx), - .dma => tpu.dmaSemaphoreType(ctx), + .regular => mosaic_tpu.semaphoreType(ctx), + .dma => mosaic_tpu.dmaSemaphoreType(ctx), }, sa.shape, null, @@ -817,7 +819,7 @@ pub const Builder = struct { inner.appendAssumeCapacity(.named(ctx, "transform_indices", .flatSymbolRef(ctx, sym_name))); inner.appendAssumeCapacity(.named(ctx, "window_bounds", .denseArray(ctx, .i64, block_shape))); if (w.element_window) |ew| { - inner.appendAssumeCapacity(.named(ctx, "window_kind", tpu.elementWindowAttribute(ctx, ew.pad_low, ew.pad_high))); + inner.appendAssumeCapacity(.named(ctx, "window_kind", mosaic_tpu.elementWindowAttr(ctx, ew.pad_low, ew.pad_high))); } wp_buf[idx] = .dict(ctx, inner.constSlice()); idx += 1; @@ -838,8 +840,6 @@ pub const Builder = struct { // Body is empty at declare time — verify only at `finish`. .verify = false, .extra_attributes = extra_attrs.constSlice(), - // sym_visibility omitted — defaults to public, elided when unset. - .visibility = null, }); _ = func_op.appendTo(self.module.body()); @@ -1306,25 +1306,25 @@ pub const Builder = struct { /// `tpu.bitcast` — same-bitwidth element-type swap. Shape preserved. pub fn bitcastTo(self: *Builder, src: Value, dt: DType) Value { - return self.emit(tpu.bitcast(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + return self.emit(mosaic_tpu.bitcast(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); } pub fn extf(self: *Builder, src: Value, dt: DType) Value { - return self.emit(tpu.extf(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + return self.emit(mosaic_tpu.extf(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); } pub fn truncf(self: *Builder, src: Value, dt: DType) Value { - return self.emit(tpu.truncf(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + return self.emit(mosaic_tpu.truncf(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); } pub fn sitofp(self: *Builder, src: Value, dt: DType) Value { - return self.emit(tpu.sitofp(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + return self.emit(mosaic_tpu.sitofp(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); } pub fn uitofp(self: *Builder, src: Value, dt: DType) Value { - return self.emit(tpu.uitofp(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + return self.emit(mosaic_tpu.uitofp(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); } pub fn fptosi(self: *Builder, src: Value, dt: DType) Value { - return self.emit(tpu.fptosi(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + return self.emit(mosaic_tpu.fptosi(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); } pub fn fptoui(self: *Builder, src: Value, dt: DType) Value { - return self.emit(tpu.fptoui(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + return self.emit(mosaic_tpu.fptoui(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); } /// Numeric cast with auto-dispatch. Routes through `arith.*` ops for the @@ -1462,15 +1462,7 @@ pub const Builder = struct { const result_ty = self.refVectorType(ref); const idx_inner = self.indicesOrZero(ref, indices); const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; - return self.emit(tpu.vector_load( - ctx, - ref.inner, - idx_inner, - mask_inner, - .{ .strides = opts.strides }, - result_ty, - self.loc(), - )); + return self.emit(mosaic_tpu.vector_load(ctx, ref.inner, idx_inner, result_ty, .{ .strides = opts.strides, .mask = mask_inner }, self.loc())); } /// `tpu.vector_load` with an explicit result vector shape. @@ -1479,15 +1471,7 @@ pub const Builder = struct { const result_ty = mlir.Type.vector(shape, ref.elemType()); const idx_inner = self.indicesOrZero(ref, indices); const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; - return self.emit(tpu.vector_load( - ctx, - ref.inner, - idx_inner, - mask_inner, - .{ .strides = opts.strides }, - result_ty, - self.loc(), - )); + return self.emit(mosaic_tpu.vector_load(ctx, ref.inner, idx_inner, result_ty, .{ .strides = opts.strides, .mask = mask_inner }, self.loc())); } /// `tpu.vector_store(value, ref)` — full-shape vector store. @@ -1502,15 +1486,7 @@ pub const Builder = struct { pub fn refStoreOpts(self: *Builder, ref: Value, value: Value, indices: []const Value, opts: VectorStoreOpts) void { const idx_inner = self.indicesOrZero(ref, indices); const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; - _ = tpu.vector_store( - self.ctx, - value.inner, - ref.inner, - idx_inner, - mask_inner, - .{ .strides = opts.strides, .add = opts.add }, - self.loc(), - ).appendTo(self.currentBlock()); + _ = mosaic_tpu.vector_store(self.ctx, value.inner, ref.inner, idx_inner, .{ .strides = opts.strides, .add = opts.add, .mask = mask_inner }, self.loc()).appendTo(self.currentBlock()); } /// `memref.load` — scalar load. Indices must be `index`-typed. @@ -1563,7 +1539,7 @@ pub const Builder = struct { s.appendAssumeCapacity(@divTrunc(dim + stride - 1, stride)); } const result_ty = mlir.Type.vector(s.constSlice(), mr.elementType()); - return self.emit(tpu.strided_load(self.ctx, ref.inner, self.innerSlice(indices), strides, result_ty, self.loc())); + return self.emit(mosaic_tpu.strided_load(self.ctx, ref.inner, self.innerSlice(indices), strides, result_ty, self.loc())); } /// `tpu.strided_load` with an explicit result shape — when you know the @@ -1572,12 +1548,12 @@ pub const Builder = struct { pub fn stridedLoadShape(self: *Builder, ref: Value, indices: []const Value, strides: []const i32, result_shape: []const i64) Value { const mr = ref.type_().isA(mlir.MemRefType) orelse @panic("stridedLoadShape: not a memref"); const result_ty = mlir.Type.vector(result_shape, mr.elementType()); - return self.emit(tpu.strided_load(self.ctx, ref.inner, self.innerSlice(indices), strides, result_ty, self.loc())); + return self.emit(mosaic_tpu.strided_load(self.ctx, ref.inner, self.innerSlice(indices), strides, result_ty, self.loc())); } /// `tpu.strided_store` — multi-dim strided store. pub fn stridedStore(self: *Builder, ref: Value, value: Value, indices: []const Value, strides: []const i32) void { - _ = tpu.strided_store(self.ctx, value.inner, ref.inner, self.innerSlice(indices), strides, self.loc()).appendTo(self.currentBlock()); + _ = mosaic_tpu.strided_store(self.ctx, value.inner, ref.inner, self.innerSlice(indices), strides, self.loc()).appendTo(self.currentBlock()); } // ==================== TPU compute: matmul / iota / reciprocal ==================== @@ -1589,11 +1565,12 @@ pub const Builder = struct { } pub fn matmulOpts(self: *Builder, lhs: Value, rhs: Value, acc: Value, opts: MatmulOpts) Value { - return self.emit(tpu.matmul( + return self.emit(mosaic_tpu.matmul( self.ctx, lhs.inner, rhs.inner, acc.inner, + acc.type_(), .{ .transpose_lhs = opts.transpose_lhs, .transpose_rhs = opts.transpose_rhs, @@ -1601,7 +1578,6 @@ pub const Builder = struct { .precision = opts.precision, .dimension_numbers = opts.dimension_numbers, }, - acc.type_(), self.loc(), )); } @@ -1610,7 +1586,7 @@ pub const Builder = struct { /// `dimensions` to use the default. pub fn iota(self: *Builder, shape: []const i64, dtype: DType, dimensions: ?[]const i32) Value { const ty = mlir.Type.vector(shape, dtype.toMlir(self.ctx)); - return self.emit(tpu.iota(self.ctx, dimensions, ty, self.loc())); + return self.emit(mosaic_tpu.iota(self.ctx, ty, .{ .dimensions = dimensions }, self.loc())); } /// 1-D `arange(0, n)`. TPU layout pass requires rank ≥ 2 with the lane dim @@ -1618,7 +1594,7 @@ pub const Builder = struct { pub fn arange(self: *Builder, n: i64, dtype: DType) Value { const elem = dtype.toMlir(self.ctx); const ty_2d = .vector(&.{ 1, n }, elem); - const iota_2d = self.emit(tpu.iota(self.ctx, &.{1}, ty_2d, self.loc())); + const iota_2d = self.emit(mosaic_tpu.iota(self.ctx, ty_2d, .{ .dimensions = &.{1} }, self.loc())); return self.shapeCast(iota_2d, &.{n}); } @@ -1628,7 +1604,7 @@ pub const Builder = struct { } pub fn reciprocalOpts(self: *Builder, x: Value, opts: ReciprocalOpts) Value { - return self.emit(tpu.reciprocal(self.ctx, x.inner, .{ + return self.emit(mosaic_tpu.reciprocal(self.ctx, x.inner, .{ .approx = opts.approx, .full_range = opts.full_range, }, self.loc())); @@ -1636,7 +1612,7 @@ pub const Builder = struct { // ==================== Vector dialect helpers ==================== // - // The `tpu.*` shape ops (`reshape`, `repeat`, `transpose`, + // The `tpu.*` shape ops (`reshape`, `transpose`, // `broadcastInSublanes`) are Mosaic-specific layout-aware variants. /// `vector.broadcast` to `shape`. For Zig scalar literals prefer `splat` (dense-splat fast path). @@ -1726,7 +1702,7 @@ pub const Builder = struct { pub fn reshape(self: *Builder, src: Value, new_shape: []const i64) Value { const ty = mlir.Type.vector(new_shape, src.elemType()); - return self.emit(tpu.reshape(self.ctx, src.inner, ty, self.loc())); + return self.emit(mosaic_tpu.reshape(self.ctx, src.inner, ty, self.loc())); } pub fn repeat(self: *Builder, src: Value, dimension: i32, times: i32) Value { @@ -1736,8 +1712,11 @@ pub const Builder = struct { const d = in_shape.get(i); out.appendAssumeCapacity(if (i == @as(usize, @intCast(dimension))) d * times else d); } + if (times == 1) return src; const ty = mlir.Type.vector(out.constSlice(), src.elemType()); - return self.emit(tpu.repeat(self.ctx, src.inner, dimension, times, ty, self.loc())); + const sources = self.arena.allocator().alloc(*const mlir.Value, @intCast(times)) catch @panic("OOM"); + @memset(sources, src.inner); + return self.emit(mosaic_tpu.concatenate(self.ctx, sources, dimension, ty, self.loc())); } pub fn concatenate(self: *Builder, sources: []const Value, dimension: i32) Value { @@ -1749,7 +1728,7 @@ pub const Builder = struct { var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; for (0..head.len) |i| out.appendAssumeCapacity(if (i == ax) sum else head.get(i)); const ty = mlir.Type.vector(out.constSlice(), sources[0].elemType()); - return self.emit(tpu.concatenate(self.ctx, self.innerSlice(sources), dimension, ty, self.loc())); + return self.emit(mosaic_tpu.concatenate(self.ctx, self.innerSlice(sources), dimension, ty, self.loc())); } pub fn transpose(self: *Builder, src: Value, permutation: []const i64) Value { @@ -1758,17 +1737,17 @@ pub const Builder = struct { var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; for (permutation) |p| out.appendAssumeCapacity(in_shape.get(@intCast(p))); const ty = mlir.Type.vector(out.constSlice(), src.elemType()); - return self.emit(tpu.transpose(self.ctx, src.inner, permutation, ty, self.loc())); + return self.emit(mosaic_tpu.transpose(self.ctx, src.inner, permutation, ty, self.loc())); } pub fn broadcastInSublanes(self: *Builder, src: Value, lane: i32) Value { // Result shape = src shape (the op replicates within sublanes). - return self.emit(tpu.broadcast_in_sublanes(self.ctx, src.inner, lane, src.type_(), self.loc())); + return self.emit(mosaic_tpu.broadcast_in_sublanes(self.ctx, src.inner, lane, src.type_(), self.loc())); } pub fn rotate(self: *Builder, value: Value, opts: RotateOpts) Value { const amount = self.lift(@as(i32, opts.amount)); - return self.emit(tpu.dynamic_rotate(self.ctx, value.inner, amount.inner, .{ + return self.emit(mosaic_tpu.dynamic_rotate(self.ctx, value.inner, amount.inner, .{ .dimension = opts.dimension, .stride = opts.stride, .stride_dimension = opts.stride_dimension, @@ -1783,7 +1762,7 @@ pub const Builder = struct { var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; for (0..in_shape.len) |i| out.appendAssumeCapacity(if (i == @as(usize, @intCast(dim))) 1 else in_shape.get(i)); const ty = mlir.Type.vector(out.constSlice(), input.elemType()); - return self.emit(tpu.all_reduce(self.ctx, input.inner, dim, kind, ty, self.loc())); + return self.emit(mosaic_tpu.all_reduce(self.ctx, input.inner, dim, kind, ty, self.loc())); } /// `tpu.reduce_index(input, kind, axis)` — drop axis, replace with @@ -1795,14 +1774,14 @@ pub const Builder = struct { if (@as(i32, @intCast(i)) != axis) out.appendAssumeCapacity(d); } const ty = mlir.Type.vector(out.constSlice(), .int(self.ctx, .i32)); - return self.emit(tpu.reduce_index(self.ctx, input.inner, axis, kind, ty, self.loc())); + return self.emit(mosaic_tpu.reduce_index(self.ctx, input.inner, axis, kind, ty, self.loc())); } /// Scan the last dimension; a mask must be rank one with that dimension /// as its length. The result has the same shape as the input. pub fn scan(self: *Builder, input: Value, kind: ReductionKind, mask: ?Value) Value { const m_inner: ?*const mlir.Value = if (mask) |m| m.inner else null; - return self.emit(tpu.scan(self.ctx, input.inner, kind, m_inner, input.type_(), self.loc())); + return self.emit(mosaic_tpu.scan(self.ctx, input.inner, kind, input.type_(), .{ .mask = m_inner }, self.loc())); } /// `tpu.sort(keys, values, mask?)` — returns (output_mask, sorted_keys, sorted_values). @@ -1812,15 +1791,14 @@ pub const Builder = struct { // Default: vector. break :blk .vector(keys.shape().constSlice(), .integer(self.ctx, .i1)); }; - const op = tpu.sort( + const op = mosaic_tpu.sort( self.ctx, keys.inner, values.inner, - m_inner, - .{ .descending = opts.descending }, out_mask_ty, keys.type_(), values.type_(), + .{ .descending = opts.descending, .mask = m_inner }, self.loc(), ); _ = op.appendTo(self.currentBlock()); @@ -1842,7 +1820,7 @@ pub const Builder = struct { ) Value { const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefSlice: not a memref"); const result_ty = mlir.Type.memRef(mr.elementType(), result_shape, null, mr.memorySpace()); - return self.emit(tpu.memref_slice(self.ctx, mem_ref.inner, .{ + return self.emit(mosaic_tpu.memref_slice(self.ctx, mem_ref.inner, .{ .base_idx = self.innerSlice(base_idx), .dynamic_sizes = self.innerSlice(dynamic_sizes), }, result_ty, self.loc())); @@ -1851,13 +1829,13 @@ pub const Builder = struct { pub fn memRefSqueeze(self: *Builder, mem_ref: Value, result_shape: []const i64) Value { const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefSqueeze: not a memref"); const result_ty = mlir.Type.memRef(mr.elementType(), result_shape, null, mr.memorySpace()); - return self.emit(tpu.memref_squeeze(self.ctx, mem_ref.inner, result_ty, self.loc())); + return self.emit(mosaic_tpu.memref_squeeze(self.ctx, mem_ref.inner, result_ty, self.loc())); } pub fn memRefReshape(self: *Builder, mem_ref: Value, result_shape: []const i64) Value { const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefReshape: not a memref"); const result_ty = mlir.Type.memRef(mr.elementType(), result_shape, null, mr.memorySpace()); - return self.emit(tpu.memref_reshape(self.ctx, mem_ref.inner, result_ty, self.loc())); + return self.emit(mosaic_tpu.memref_reshape(self.ctx, mem_ref.inner, result_ty, self.loc())); } pub fn memRefAlloca(self: *Builder, shape: []const i64, dtype: DType, memory_space: MemorySpace) Value { @@ -1869,7 +1847,7 @@ pub const Builder = struct { var s: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; for (0..mr.rank()) |i| s.appendAssumeCapacity(mr.dimension(i)); const result_ty = .memRef(dt.toMlir(self.ctx), s.constSlice(), null, mr.memorySpace()); - return self.emit(tpu.memref_bitcast(self.ctx, mem_ref.inner, result_ty, self.loc())); + return self.emit(mosaic_tpu.memref_bitcast(self.ctx, mem_ref.inner, result_ty, self.loc())); } /// `tpu.memref_bitcast` with an explicit result shape — needed for the @@ -1878,17 +1856,17 @@ pub const Builder = struct { pub fn memRefBitcastShape(self: *Builder, mem_ref: Value, result_shape: []const i64, dt: DType) Value { const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefBitcastShape: not a memref"); const result_ty = mlir.Type.memRef(dt.toMlir(self.ctx), result_shape, null, mr.memorySpace()); - return self.emit(tpu.memref_bitcast(self.ctx, mem_ref.inner, result_ty, self.loc())); + return self.emit(mosaic_tpu.memref_bitcast(self.ctx, mem_ref.inner, result_ty, self.loc())); } pub fn reinterpretCast(self: *Builder, mem_ref: Value, result_shape: []const i64, dt: DType) Value { const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("reinterpretCast: not a memref"); const result_ty = mlir.Type.memRef(dt.toMlir(self.ctx), result_shape, null, mr.memorySpace()); - return self.emit(tpu.reinterpret_cast(self.ctx, mem_ref.inner, result_ty, self.loc())); + return self.emit(mosaic_tpu.reinterpret_cast(self.ctx, mem_ref.inner, result_ty, self.loc())); } pub fn assumeLayout(self: *Builder, src: Value) Value { - return self.emit(tpu.assume_layout(self.ctx, src.inner, src.type_(), self.loc())); + return self.emit(mosaic_tpu.assume_layout(self.ctx, src.inner, src.type_(), self.loc())); } pub fn eraseMemRefLayout(self: *Builder, mem_ref: Value) Value { @@ -1897,7 +1875,7 @@ pub const Builder = struct { for (0..mr.rank()) |i| s.appendAssumeCapacity(mr.dimension(i)); // Result drops the layout but keeps shape, element type, memory space. const result_ty = .memRef(mr.elementType(), s.constSlice(), null, mr.memorySpace()); - return self.emit(tpu.erase_memref_layout(self.ctx, mem_ref.inner, result_ty, self.loc())); + return self.emit(mosaic_tpu.erase_memref_layout(self.ctx, mem_ref.inner, result_ty, self.loc())); } // ==================== Mask building ==================== @@ -1905,7 +1883,7 @@ pub const Builder = struct { /// `tpu.create_mask` — 2-sided mask `[low, high)` per dim. pub fn createMask(self: *Builder, low_bounds: []const Value, high_bounds: []const Value, shape: []const i64) Value { const ty = mlir.Type.vector(shape, .integer(self.ctx, .i1)); - return self.emit(tpu.create_mask(self.ctx, self.innerSlice(low_bounds), self.innerSlice(high_bounds), ty, self.loc())); + return self.emit(mosaic_tpu.create_mask(self.ctx, self.innerSlice(low_bounds), self.innerSlice(high_bounds), ty, self.loc())); } /// Convenience: 1-sided range mask `(0 <= idx < limit)` — emits two @@ -1921,25 +1899,25 @@ pub const Builder = struct { pub fn semAlloc(self: *Builder, kind: ArgSpec.SemKind) Value { const elem = switch (kind) { - .regular => tpu.semaphoreType(self.ctx), - .dma => tpu.dmaSemaphoreType(self.ctx), + .regular => mosaic_tpu.semaphoreType(self.ctx), + .dma => mosaic_tpu.dmaSemaphoreType(self.ctx), }; const ty = mlir.Type.memRef(elem, &.{}, null, MemorySpace.semaphore_mem.attribute(self.ctx)); - return self.emit(tpu.sem_alloc(self.ctx, ty, self.loc())); + return self.emit(mosaic_tpu.sem_alloc(self.ctx, ty, self.loc())); } pub fn semBarrier(self: *Builder) Value { - const ty = mlir.Type.memRef(tpu.semaphoreType(self.ctx), &.{}, null, MemorySpace.semaphore_mem.attribute(self.ctx)); - return self.emit(tpu.sem_barrier(self.ctx, ty, self.loc())); + const ty = mlir.Type.memRef(mosaic_tpu.semaphoreType(self.ctx), &.{}, null, MemorySpace.semaphore_mem.attribute(self.ctx)); + return self.emit(mosaic_tpu.sem_barrier(self.ctx, ty, self.loc())); } pub fn semRead(self: *Builder, semaphore: Value) Value { - return self.emit(tpu.sem_read(self.ctx, semaphore.inner, self.loc())); + return self.emit(mosaic_tpu.sem_read(self.ctx, semaphore.inner, self.loc())); } pub fn semWait(self: *Builder, semaphore: Value, amount: anytype) void { const a = if (@TypeOf(amount) == Value) amount else self.lift(amount); - _ = tpu.sem_wait(self.ctx, semaphore.inner, a.inner, self.loc()).appendTo(self.currentBlock()); + _ = mosaic_tpu.sem_wait(self.ctx, semaphore.inner, a.inner, self.loc()).appendTo(self.currentBlock()); } pub fn semSignal(self: *Builder, semaphore: Value, amount: anytype) void { @@ -1951,7 +1929,7 @@ pub const Builder = struct { const dev: ?*const mlir.Value = if (opts.device_id) |d| d.inner else null; const core: ?*const mlir.Value = if (opts.core_id) |c_| c_.inner else null; const subcore: ?*const mlir.Value = if (opts.subcore_id) |sc| sc.inner else null; - _ = tpu.sem_signal(self.ctx, semaphore.inner, a.inner, .{ + _ = mosaic_tpu.sem_signal(self.ctx, semaphore.inner, a.inner, .{ .device_id = dev, .core_id = core, .subcore_id = subcore, @@ -1959,7 +1937,7 @@ pub const Builder = struct { } pub fn barrier(self: *Builder, barrier_id: Value) void { - _ = tpu.barrier(self.ctx, barrier_id.inner, self.loc()).appendTo(self.currentBlock()); + _ = mosaic_tpu.barrier(self.ctx, barrier_id.inner, self.loc()).appendTo(self.currentBlock()); } pub fn enqueueDma(self: *Builder, source: Value, target: Value, target_semaphore: Value, opts: EnqueueDmaOpts) void { @@ -1967,7 +1945,7 @@ pub const Builder = struct { const dev: ?*const mlir.Value = if (opts.device_id) |d| d.inner else null; const core: ?*const mlir.Value = if (opts.core_id) |c_| c_.inner else null; const subcore: ?*const mlir.Value = if (opts.subcore_id) |sc| sc.inner else null; - _ = tpu.enqueue_dma(self.ctx, source.inner, target.inner, target_semaphore.inner, .{ + _ = mosaic_tpu.enqueue_dma(self.ctx, source.inner, target.inner, target_semaphore.inner, .{ .source_semaphore = src_sem, .device_id = dev, .core_id = core, @@ -1980,7 +1958,7 @@ pub const Builder = struct { pub fn waitDma2(self: *Builder, semaphore: Value, src: Value, dst: Value, opts: WaitDma2Opts) void { const dev: ?*const mlir.Value = if (opts.device_id) |d| d.inner else null; const core: ?*const mlir.Value = if (opts.core_id) |c_| c_.inner else null; - _ = tpu.wait_dma2(self.ctx, semaphore.inner, src.inner, dst.inner, .{ + _ = mosaic_tpu.wait_dma2(self.ctx, semaphore.inner, src.inner, dst.inner, .{ .device_id = dev, .core_id = core, .strict_ordering = opts.strict_ordering, @@ -1988,17 +1966,17 @@ pub const Builder = struct { } pub fn deviceId(self: *Builder) Value { - return self.emit(tpu.device_id(self.ctx, self.loc())); + return self.emit(mosaic_tpu.device_id(self.ctx, self.loc())); } /// `tpu.trace_start` — pair with `traceStop`. Used by Pallas to mark /// einsum boundaries (e.g. `"nd,md->nm"`). pub fn traceStart(self: *Builder, level: i32, message: []const u8) void { - _ = tpu.trace_start(self.ctx, level, message, self.loc()).appendTo(self.currentBlock()); + _ = mosaic_tpu.trace_start(self.ctx, level, message, self.loc()).appendTo(self.currentBlock()); } pub fn traceStop(self: *Builder) void { - _ = tpu.trace_stop(self.ctx, self.loc()).appendTo(self.currentBlock()); + _ = mosaic_tpu.trace_stop(self.ctx, self.loc()).appendTo(self.currentBlock()); } /// Build a `#tpu.dot_dimension_numbers<...>` attribute for `matmulOpts`. @@ -2012,7 +1990,7 @@ pub const Builder = struct { lhs_batch: []const i64, rhs_batch: []const i64, ) *const mlir.Attribute { - return tpu.dotDimensionNumbers( + return mosaic_tpu.dotDimensionNumbersAttr( self.ctx, lhs_contracting, rhs_contracting, @@ -2026,11 +2004,11 @@ pub const Builder = struct { pub fn delay(self: *Builder, cycles: anytype) void { const c_ = if (@TypeOf(cycles) == Value) cycles else self.lift(cycles); - _ = tpu.delay(self.ctx, c_.inner, self.loc()).appendTo(self.currentBlock()); + _ = mosaic_tpu.delay(self.ctx, c_.inner, self.loc()).appendTo(self.currentBlock()); } pub fn assumeMultiple(self: *Builder, src: Value, multiple: i32) Value { - return self.emit(tpu.assume_multiple(self.ctx, src.inner, multiple, self.loc())); + return self.emit(mosaic_tpu.assume_multiple(self.ctx, src.inner, multiple, self.loc())); } pub fn openRegion(self: *Builder) *mlir.Block { @@ -2040,9 +2018,9 @@ pub const Builder = struct { } pub fn closeRegion(self: *Builder, block: *mlir.Block) void { - _ = tpu.yield(self.ctx, &.{}, self.loc()).appendTo(block); + _ = mosaic_tpu.yield(self.ctx, &.{}, self.loc()).appendTo(block); self.popBlock(); - _ = tpu.region(self.ctx, block, &.{}, self.loc()).appendTo(self.currentBlock()); + _ = mosaic_tpu.region(self.ctx, block, &.{}, self.loc()).appendTo(self.currentBlock()); } // ==================== SCF regions ==================== @@ -2303,7 +2281,6 @@ pub const Builder = struct { .results = result_types, .block = block, .location = unknown_loc, - .visibility = null, .verify = false, .extra_attributes = extra_attrs.constSlice(), }); @@ -2343,8 +2320,8 @@ test { test "static rotate uses the current dynamic rotate operation" { const registry = try mlir.DialectRegistry.init(); defer registry.deinit(); - inline for (dialects_needed) |dialect| registry.registerDialect(dialect); - mlir.registerFuncExtensions(registry); + registerDialects(registry); + func.registerExtensions(registry); const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); defer ctx.deinit(); ctx.loadAllAvailableDialects(); diff --git a/kernels/mosaic_tpu/dtype.zig b/dsl/mosaic_tpu/dtype.zig similarity index 100% rename from kernels/mosaic_tpu/dtype.zig rename to dsl/mosaic_tpu/dtype.zig diff --git a/kernels/mosaic_tpu/pipeline.zig b/dsl/mosaic_tpu/pipeline.zig similarity index 97% rename from kernels/mosaic_tpu/pipeline.zig rename to dsl/mosaic_tpu/pipeline.zig index 9d0a051d..5726c764 100644 --- a/kernels/mosaic_tpu/pipeline.zig +++ b/dsl/mosaic_tpu/pipeline.zig @@ -5,14 +5,12 @@ //! inputs peeled as sync copy; `trace_scopes`. Unsupported features panic. const std = @import("std"); +const func = @import("mlir/dialects/func"); const mlir = @import("mlir"); -const dialects = @import("mlir/dialects"); +const scf = @import("mlir/dialects/scf"); const builder = @import("builder.zig"); -const tpu = @import("mlir/dialects/mosaic_tpu"); -const arith = dialects.arith; -const scf = dialects.scf; -const memref = dialects.memref; +const mosaic_tpu = @import("mlir/dialects/mosaic_tpu"); const Builder = builder.Builder; const Value = builder.Value; @@ -257,9 +255,9 @@ pub fn emitPipeline( } } - _ = tpu.yield(b.ctx, &.{}, b.loc()).appendTo(region_block); + _ = mosaic_tpu.yield(b.ctx, &.{}, b.loc()).appendTo(region_block); b.popBlock(); - _ = tpu.region(b.ctx, region_block, &.{}, b.loc()).appendTo(b.currentBlock()); + _ = mosaic_tpu.region(b.ctx, region_block, &.{}, b.loc()).appendTo(b.currentBlock()); } pub const BufferType = enum { input, output, input_output }; @@ -512,7 +510,7 @@ pub const BufferedRef = struct { const tiling: i64 = if (d == bs.len - 1) 128 else if (bs.len >= 2 and d == bs.len - 2) 8 else 1; const clamped = @min(n, dim_size); const sz_const = @divFloor(clamped + tiling - 1, tiling) * tiling; - size[d] = b.emit(tpu.assume_multiple(b.ctx, b.lift(@as(i32, @intCast(sz_const))).inner, @as(i32, @intCast(tiling)), b.loc())); + size[d] = b.emit(mosaic_tpu.assume_multiple(b.ctx, b.lift(@as(i32, @intCast(sz_const))).inner, @as(i32, @intCast(tiling)), b.loc())); } }, .bounded_slice => { @@ -524,14 +522,14 @@ pub const BufferedRef = struct { const dim_size = self.spec.full_shape[d]; base[d] = sl.start; if (self.spec.bounded_slice_skip_clamp) { - size[d] = b.emit(tpu.assume_multiple(b.ctx, sl.size.inner, tiling, b.loc())); + size[d] = b.emit(mosaic_tpu.assume_multiple(b.ctx, sl.size.inner, tiling, b.loc())); continue; } // Prologue shortcut: Python-int 0 start ⇒ `_round_up_to_nearest_multiple` folds. if (static_zero_start and sl.start.asConstantInt() == @as(?i64, 0)) { const is_oob = b.cmpi(.sgt, sl.size, b.lift(@as(i32, @intCast(dim_size)))); const sz = b.select(is_oob, b.lift(@as(i32, @intCast(dim_size))), sl.size); - size[d] = b.emit(tpu.assume_multiple(b.ctx, sz.inner, tiling, b.loc())); + size[d] = b.emit(mosaic_tpu.assume_multiple(b.ctx, sz.inner, tiling, b.loc())); continue; } const is_oob = b.cmpi(.sgt, b.addi(sl.start, sl.size), b.lift(@as(i32, @intCast(dim_size)))); @@ -551,7 +549,7 @@ pub const BufferedRef = struct { break :blk b.addi(round_down, m); }; const sz = b.select(is_oob, rounded, sl.size); - size[d] = b.emit(tpu.assume_multiple(b.ctx, sz.inner, tiling, b.loc())); + size[d] = b.emit(mosaic_tpu.assume_multiple(b.ctx, sz.inner, tiling, b.loc())); }, } } @@ -647,9 +645,9 @@ pub const BufferedRef = struct { const sem = semAllocOp(b, dmaSemRefTy(b, null)); b.enqueueDma(self.src, self.window_ref, sem, .{}); b.waitDma2(sem, self.src, self.window_ref, .{}); - _ = tpu.yield(b.ctx, &.{}, b.loc()).appendTo(blk); + _ = mosaic_tpu.yield(b.ctx, &.{}, b.loc()).appendTo(blk); b.popBlock(); - _ = tpu.region(b.ctx, blk, &.{}, b.loc()).appendTo(b.currentBlock()); + _ = mosaic_tpu.region(b.ctx, blk, &.{}, b.loc()).appendTo(b.currentBlock()); } fn syncCopyOut(self: *BufferedRef, b: *Builder) void { @@ -658,9 +656,9 @@ pub const BufferedRef = struct { const sem = semAllocOp(b, dmaSemRefTy(b, null)); b.enqueueDma(self.window_ref, self.src, sem, .{}); b.waitDma2(sem, self.window_ref, self.src, .{}); - _ = tpu.yield(b.ctx, &.{}, b.loc()).appendTo(blk); + _ = mosaic_tpu.yield(b.ctx, &.{}, b.loc()).appendTo(blk); b.popBlock(); - _ = tpu.region(b.ctx, blk, &.{}, b.loc()).appendTo(b.currentBlock()); + _ = mosaic_tpu.region(b.ctx, blk, &.{}, b.loc()).appendTo(b.currentBlock()); } // Op order mirrors pipeline.py: copy_in/copy_out/wait_out take slot rem @@ -1038,7 +1036,7 @@ pub fn semAllocOp(b: *Builder, ty: *const mlir.Type) Value { /// `pl.SemaphoreType.DMA((n,))`, or rank-0 for `n == null`. pub fn dmaSemRefTy(b: *Builder, n: ?usize) *const mlir.Type { const shape: []const i64 = if (n) |m| &.{@intCast(m)} else &.{}; - return mlir.Type.memRef(tpu.dmaSemaphoreType(b.ctx), shape, null, MemorySpace.semaphore_mem.attribute(b.ctx)); + return mlir.Type.memRef(mosaic_tpu.dmaSemaphoreType(b.ctx), shape, null, MemorySpace.semaphore_mem.attribute(b.ctx)); } fn gridSize(b: *Builder, grid: []const Value) Value { @@ -1109,8 +1107,8 @@ test { test "trivial input-output ref is copied in and out" { const registry = try mlir.DialectRegistry.init(); defer registry.deinit(); - inline for (builder.dialects_needed) |dialect| registry.registerDialect(dialect); - mlir.registerFuncExtensions(registry); + builder.registerDialects(registry); + func.registerExtensions(registry); const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); defer ctx.deinit(); diff --git a/kernels/triton/BUILD.bazel b/dsl/triton/BUILD.bazel similarity index 60% rename from kernels/triton/BUILD.bazel rename to dsl/triton/BUILD.bazel index cc80054c..780454a8 100644 --- a/kernels/triton/BUILD.bazel +++ b/dsl/triton/BUILD.bazel @@ -2,17 +2,22 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "triton_builder", - tags = ["manual"], srcs = [ "dtype.zig", ], - import_name = "kernels/triton/builder", + import_name = "dsl/triton/builder", main = "builder.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ - "//kernels/common", + "//dsl/common", "//mlir", - "//mlir/dialects", + "//mlir/dialects/arith", + "//mlir/dialects/cf", + "//mlir/dialects/func", + "//mlir/dialects/llvm", + "//mlir/dialects/math", + "//mlir/dialects/scf", "//mlir/dialects/ttir", "//stdx", ], diff --git a/kernels/triton/PORTING.md b/dsl/triton/PORTING.md similarity index 98% rename from kernels/triton/PORTING.md rename to dsl/triton/PORTING.md index 659baadf..7512cfb8 100644 --- a/kernels/triton/PORTING.md +++ b/dsl/triton/PORTING.md @@ -85,7 +85,7 @@ function. ### Step 1.2 — Sketch the Zig declaration -Inside `zml/moe/triton_kernels.zig` or `zml/attention/triton_kernels.zig` +Inside `zml/kernels/moe/triton/` or `zml/kernels/paged_attention/triton/` (or whichever production file fits), add: ```zig @@ -677,9 +677,9 @@ Once the LLIR matches *and* the visual-transparency pass is done: | Problem | File | |---------------------------------------------------|----------------------------------------------| -| "How do I express `tl.X(...)` in Zig?" | `kernels/triton/README.md` (Python ↔ Zig table) | -| "What does `b.declareArgs(...)` accept?" | `kernels/triton/README.md` (Argument specs) | -| "How do I call this kernel from `forward`?" | `kernels/triton/README.md` (`K.call(...)`) | +| "How do I express `tl.X(...)` in Zig?" | `dsl/triton/README.md` (Python ↔ Zig table) | +| "What does `b.declareArgs(...)` accept?" | `dsl/triton/README.md` (Argument specs) | +| "How do I call this kernel from `forward`?" | `dsl/triton/README.md` (`K.call(...)`) | | "My kernel diverges from Python at LLIR" | This file (Phase 3) | | "How do I add a kernel to the comparison harness?"| Phase 1 above + `examples/triton_emitter/` | | "I want to inspect raw TTIR ops" | `mlir/dialects/ttir/ttir.zig` (escape hatch) | @@ -699,7 +699,7 @@ If a divergence resists all the standard fixes: 3. **Verify the Python override args match the Zig config.** Mismatched constexpr values (e.g. `BLOCK_SIZE=64` in Zig vs `1024` auto-default in Python) generate completely different IR. -4. **Compare against `zml/moe/triton_kernels.zig` MoE kernels** — they +4. **Compare against the `zml/kernels/moe/triton/` MoE kernels** — they went through this loop and are reasonable references for the patterns above. @@ -708,7 +708,7 @@ If a divergence resists all the standard fixes: ## Before "fixing" the DSL Every time a divergence makes you suspect the DSL itself is buggy and you -reach for `kernels/triton/builder.zig` — **verify against the Triton sources +reach for `dsl/triton/builder.zig` — **verify against the Triton sources first** at `/Users/raph/Documents/Git-Repos/triton`. The DSL exists to mirror what Python's frontend emits; the source of truth is the Python trace, not what the rule "looks like" or what feels symmetric. diff --git a/kernels/triton/README.md b/dsl/triton/README.md similarity index 99% rename from kernels/triton/README.md rename to dsl/triton/README.md index 8860107d..58cc1939 100644 --- a/kernels/triton/README.md +++ b/dsl/triton/README.md @@ -1,13 +1,13 @@ # ZML Triton DSL — Author's Guide A Zig builder for Triton IR (TTIR) that tries to read like `triton.language` -Python while staying in pure Zig. Used by `zml/moe/triton_kernels.zig`, -`zml/attention/triton_kernels.zig`, and by anything else that wants to call +Python while staying in pure Zig. Used by `zml/kernels/moe/triton/`, +`zml/kernels/paged_attention/triton/`, and by anything else that wants to call Triton kernels from a ZML model. Two layers define the surface: -- **Low layer** — `kernels/triton/builder.zig`: the IR builder primitives. +- **Low layer** — `dsl/triton/builder.zig`: the IR builder primitives. `Builder`, `Value`, `DType`, `ArgSpec`. Use these directly when you need full control (custom op insertion, dynamic-arity arg lists, escape hatches). `Builder.open` / `Builder.init` take an `*mlir.Context` — pass them a diff --git a/kernels/triton/builder.zig b/dsl/triton/builder.zig similarity index 97% rename from kernels/triton/builder.zig rename to dsl/triton/builder.zig index d585e8c1..ae802c9c 100644 --- a/kernels/triton/builder.zig +++ b/dsl/triton/builder.zig @@ -1,12 +1,13 @@ const std = @import("std"); -const dialects = @import("mlir/dialects"); -const arith = dialects.arith; -const cf = dialects.cf; -const math = dialects.math; -const scf = dialects.scf; -const ttir = dialects.ttir; -const dsl = @import("kernels/common"); +const arith = @import("mlir/dialects/arith"); +const cf = @import("mlir/dialects/cf"); +const dsl = @import("dsl/common"); +const func = @import("mlir/dialects/func"); +const llvm = @import("mlir/dialects/llvm"); +const math = @import("mlir/dialects/math"); +const scf = @import("mlir/dialects/scf"); +const ttir = @import("mlir/dialects/ttir"); const tupleArity = dsl.tupleArity; const mlir = @import("mlir"); const stdx = @import("stdx"); @@ -456,7 +457,9 @@ pub fn ScanArgs(comptime CtxT: type) type { }; } -pub const dialects_needed = [_][]const u8{ "func", "tt", "arith", "scf", "math", "cf", "llvm" }; +pub fn registerDialects(registry: *mlir.DialectRegistry) void { + inline for (.{ func, ttir, arith, scf, math, cf, llvm }) |dialect| dialect.registerDialect(registry); +} pub const Builder = struct { allocator: std.mem.Allocator, @@ -1097,7 +1100,16 @@ pub const Builder = struct { } const mask_inner: ?*const mlir.Value = if (mask_v) |m| m.inner else null; const other_inner: ?*const mlir.Value = if (other_v) |o| o.inner else null; - return self.emit(ttir.load(self.ctx, ptr.inner, self.loadResultType(ptr), mask_inner, other_inner, opts.cache_modifier, opts.eviction_policy, opts.@"volatile", self.loc())); + return self.emit(ttir.load( + self.ctx, + ptr.inner, + self.loadResultType(ptr), + opts.cache_modifier, + opts.eviction_policy, + opts.@"volatile", + .{ .mask = mask_inner, .other = other_inner }, + self.loc(), + )); } pub fn store(self: *Builder, ptr: Value, value: Value) void { @@ -1112,7 +1124,7 @@ pub const Builder = struct { } } const mask_inner: ?*const mlir.Value = if (mask_v) |m| m.inner else null; - _ = ttir.store(self.ctx, ptr.inner, value.inner, mask_inner, opts.cache_modifier, opts.eviction_policy, self.loc()).appendTo(self.currentBlock()); + _ = ttir.store(self.ctx, ptr.inner, value.inner, opts.cache_modifier, opts.eviction_policy, .{ .mask = mask_inner }, self.loc()).appendTo(self.currentBlock()); } // ==================== arith: binary ==================== @@ -1367,7 +1379,7 @@ pub const Builder = struct { } /// tt.fp_to_fp with rounding option. pub fn fpToFpOpts(self: *Builder, src: Value, out_dtype: DType, opts: FpToFpOpts) Value { - return self.emit(ttir.fp_to_fp(self.ctx, src.inner, self.swapElem(src, out_dtype), opts.rounding, self.loc())); + return self.emit(ttir.fp_to_fp(self.ctx, src.inner, self.swapElem(src, out_dtype), .{ .rounding = opts.rounding }, self.loc())); } /// tt.clampf with propagate_nan=.none. Use `clampfOpts` to override. pub fn clampf(self: *Builder, x: Value, lo: Value, hi: Value) Value { @@ -1426,23 +1438,27 @@ pub const Builder = struct { return self.trans(src, order); } - /// tt.cat — concatenate along dim 0. Use `catOpts` for can_reorder / dim. + /// `tl.cat` along dim 0. Use `catOpts` for can_reorder / dim. pub fn cat(self: *Builder, lhs: Value, rhs: Value) Value { return self.catOpts(lhs, rhs, .{}); } - /// tt.cat with full options. dim != 0 is not yet supported. + /// `tl.cat` with full options: `join`, `permute` and `reshape`, as Triton lowers it. pub fn catOpts(self: *Builder, lhs: Value, rhs: Value, opts: CatOpts) Value { - std.debug.assert(opts.dim == 0); // TODO: dim != 0 requires join+permute+reshape - _ = opts.can_reorder; // forwarded to ttir.cat when supported + _ = opts.can_reorder; const ls = lhs.shape(); const rs = rhs.shape(); std.debug.assert(ls.len == rs.len); - var out_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; - out_shape.appendAssumeCapacity(ls.get(0) + rs.get(0)); - for (1..ls.len) |i| out_shape.appendAssumeCapacity(ls.get(i)); - const result_type = mlir.Type.rankedTensor(out_shape.constSlice(), lhs.elemType()); - return self.emit(ttir.cat(self.ctx, lhs.inner, rhs.inner, result_type, self.loc())); + for (0..ls.len) |i| std.debug.assert(ls.get(i) == rs.get(i)); + const rank: i32 = @intCast(ls.len); + const dim: usize = @intCast(if (opts.dim < 0) opts.dim + rank else opts.dim); + var order: stdx.BoundedArray(i32, mlir.ShapedType.MAX_RANK) = .empty; + for (0..ls.len) |i| order.appendAssumeCapacity(@intCast(i)); + order.insert(dim, rank) catch unreachable; + var new_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..ls.len) |i| new_shape.appendAssumeCapacity(ls.get(i)); + new_shape.slice()[dim] = ls.get(dim) + rs.get(dim); + return self.reshape(self.permute(self.join(lhs, rhs), order.constSlice()), new_shape.constSlice()); } /// `tt.join` — stack two equal-shape tensors along a new minor dim of size 2. @@ -1491,7 +1507,7 @@ pub const Builder = struct { pub fn histogramOpts(self: *Builder, src: Value, num_bins: i64, opts: HistogramOpts) Value { const result_ty = mlir.Type.rankedTensor(&.{num_bins}, DType.i32.toMlir(self.ctx)); const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; - return self.emit(ttir.histogram(self.ctx, src.inner, mask_inner, result_ty, self.loc())); + return self.emit(ttir.histogram(self.ctx, src.inner, result_ty, .{ .mask = mask_inner }, self.loc())); } /// tt.assert — device-side assert (i1 or tensor<...xi1>). Use `deviceAssertOpts` for mask. @@ -1514,7 +1530,7 @@ pub const Builder = struct { /// tt.atomic_rmw with full options. pub fn atomicRmwOpts(self: *Builder, rmw: ttir.RMWOp, ptr: Value, val: Value, opts: AtomicRMWOpts) Value { const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; - return self.emit(ttir.atomic_rmw(self.ctx, rmw, ptr.inner, val.inner, mask_inner, opts.sem, opts.scope, self.loc())); + return self.emit(ttir.atomic_rmw(self.ctx, rmw, ptr.inner, val.inner, opts.sem, opts.scope, .{ .mask = mask_inner }, self.loc())); } /// tt.atomic_cas — returns old value. Use `atomicCasOpts` for sem / scope. @@ -1529,7 +1545,7 @@ pub const Builder = struct { /// `tt.call` — call another tt.func in this module by symbol name. pub fn call(self: *Builder, callee: []const u8, operands: []const Value, result_types: []const *const mlir.Type) []Value { - const op = ttir.call(self.ctx, callee, self.innerSlice(operands), result_types, null, null, self.loc()); + const op = ttir.call(self.ctx, callee, self.innerSlice(operands), result_types, .{}, self.loc()); return self.emitMulti(op, result_types.len); } @@ -1566,14 +1582,13 @@ pub const Builder = struct { a.inner, b.inner, c_acc.inner, - a_s, - b_s, a_elem_type, b_elem_type, c_acc.type_(), opts.fast_math, opts.lhs_k_pack, opts.rhs_k_pack, + .{ .a_scale = a_s, .b_scale = b_s }, self.loc(), )); } @@ -1652,7 +1667,7 @@ pub const Builder = struct { block_shape: []const i64, dtype: DType, ) Value { - const result_type = ttir.tensorDescType(block_shape, dtype.toMlir(self.ctx), null); + const result_type = ttir.tensorDescType(block_shape, dtype.toMlir(self.ctx), .{}); return self.emit(ttir.make_tensor_descriptor( self.ctx, base.inner, @@ -2081,16 +2096,7 @@ pub const Builder = struct { region.appendOwnedBlock(cont_block); - _ = cf.cond_br( - self.ctx, - cond.inner, - ret_target, - &.{}, - cont_block, - &.{}, - null, - self.loc(), - ).appendTo(current); + _ = cf.cond_br(self.ctx, cond.inner, ret_target, &.{}, cont_block, &.{}, .{}, self.loc()).appendTo(current); if (self.block_stack.items.len == 0) { self.pushBlock(cont_block); @@ -2108,16 +2114,7 @@ pub const Builder = struct { region.appendOwnedBlock(ret_block); region.appendOwnedBlock(cont_block); - _ = cf.cond_br( - self.ctx, - cond.inner, - ret_block, - &.{}, - cont_block, - &.{}, - null, - self.loc(), - ).appendTo(current); + _ = cf.cond_br(self.ctx, cond.inner, ret_block, &.{}, cont_block, &.{}, .{}, self.loc()).appendTo(current); self.pushBlock(ret_block); diff --git a/kernels/triton/dtype.zig b/dsl/triton/dtype.zig similarity index 100% rename from kernels/triton/dtype.zig rename to dsl/triton/dtype.zig diff --git a/kernels/cute/BUILD.bazel b/kernels/cute/BUILD.bazel deleted file mode 100644 index 97b6ddd9..00000000 --- a/kernels/cute/BUILD.bazel +++ /dev/null @@ -1,24 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "cute_builder", - tags = ["manual"], - srcs = [ - "dtype.zig", - ], - import_name = "kernels/cute/builder", - main = "builder.zig", - visibility = ["//visibility:public"], - deps = [ - "//kernels/common", - "//mlir", - "//mlir/dialects", - "//mlir/dialects/cute_ir", - "//stdx", - ], -) - -zig_test( - name = "test", - deps = [":cute_builder"], -) diff --git a/kernels/fly/BUILD.bazel b/kernels/fly/BUILD.bazel deleted file mode 100644 index 5ee1a1f8..00000000 --- a/kernels/fly/BUILD.bazel +++ /dev/null @@ -1,25 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "fly_builder", - tags = ["manual"], - srcs = [ - "dtype.zig", - "layout.zig", - ], - import_name = "kernels/fly/builder", - main = "builder.zig", - visibility = ["//visibility:public"], - deps = [ - "//kernels/common", - "//mlir", - "//mlir/dialects", - "//mlir/dialects/fly", - "//stdx", - ], -) - -zig_test( - name = "test", - deps = [":fly_builder"], -) diff --git a/mlir/.clang-format b/mlir/.clang-format new file mode 100644 index 00000000..9b3aa8b7 --- /dev/null +++ b/mlir/.clang-format @@ -0,0 +1 @@ +BasedOnStyle: LLVM diff --git a/mlir/BUILD.bazel b/mlir/BUILD.bazel index e292a030..8333c579 100644 --- a/mlir/BUILD.bazel +++ b/mlir/BUILD.bazel @@ -3,30 +3,18 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") cc_library( name = "c", - tags = ["manual"], - srcs = ["register.cc"], hdrs = ["mlir.h"], - visibility = ["//mlir:__subpackages__"], + tags = ["manual"], deps = [ - "@llvm-project//mlir:AffineDialect", - "@llvm-project//mlir:CAPIArith", - "@llvm-project//mlir:CAPICF", "@llvm-project//mlir:CAPIIR", - "@llvm-project//mlir:CAPILLVM", - "@llvm-project//mlir:CAPIMath", - "@llvm-project//mlir:CAPIMemRef", - "@llvm-project//mlir:CAPIROCDL", - "@llvm-project//mlir:CAPISCF", "@llvm-project//mlir:CAPITransforms", - "@llvm-project//mlir:CAPIVector", - "@llvm-project//mlir:FuncExtensions", ], ) zig_library( name = "mlir", - tags = ["manual"], main = "mlir.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ ":c", diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/BUILD.bazel deleted file mode 100644 index 220394e8..00000000 --- a/mlir/dialects/BUILD.bazel +++ /dev/null @@ -1,70 +0,0 @@ -load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -# What upstream's C API leaves out of the dialects bound here. -cc_library( - name = "capi", - srcs = [ - "arith_capi.cc", - "gpu_capi.cc", - "vector_capi.cc", - ], - hdrs = [ - "arith_capi.h", - "gpu_capi.h", - "vector_capi.h", - ], - strip_include_prefix = ".", - tags = ["manual"], - visibility = ["//mlir/dialects:__subpackages__"], - deps = [ - "@llvm-project//mlir:ArithDialect", - "@llvm-project//mlir:CAPIIR", - "@llvm-project//mlir:GPUDialect", - "@llvm-project//mlir:VectorDialect", - ], - alwayslink = True, -) - -zig_library( - name = "dialects", - srcs = [ - "affine.zig", - "arith.zig", - "cf.zig", - "cuda.zig", - "func.zig", - "gpu.zig", - "llvm_dialect.zig", - "math.zig", - "memref.zig", - "nvvm.zig", - "scf.zig", - "shardy.zig", - "vector.zig", - ], - import_name = "mlir/dialects", - main = "dialects.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - ":capi", - "//mlir", - "//mlir:c", - "//mlir/dialects/cuda_tile", - "//mlir/dialects/cute_ir", - "//mlir/dialects/fly", - "//mlir/dialects/mosaic_tpu", - "//mlir/dialects/stablehlo", - "//mlir/dialects/ttir", - "//stdx", - "@shardy//shardy/integrations/c:sdy_capi", - ], -) - -zig_test( - name = "test", - deps = [ - ":dialects", - ], -) diff --git a/mlir/dialects/README.md b/mlir/dialects/README.md new file mode 100644 index 00000000..d95f7a1c --- /dev/null +++ b/mlir/dialects/README.md @@ -0,0 +1,262 @@ +# MLIR dialects + +One Bazel package per MLIR dialect that ZML emits. The rules come in three +tiers: + +- **Guarantees** hold for every package. Tests and the conformance checks + below enforce them, and callers rely on them. +- **Conventions** are the defaults. Follow them in new code; existing code + that differs is fine unless it confuses callers. +- **Exceptions** list the known, justified departures from the conventions. + +References: `cuda/`, `nvvm/` and `cute_nvgpu/` (generated from ODS, with +their own TableGen and C API), `gpu/`, `arith/` and `func/` (upstream +dialects, with a small C API for what upstream lacks). + +## Guarantees + +- **Registration.** The package exposes `registerDialect(registry)`, which + calls `registry.registerDialect("")` with a string literal, where + `` is the second argument of + `MLIR_DECLARE_CAPI_DIALECT_REGISTRATION`. Tests and consumers register + through `.registerDialect`, never through raw strings or + `DialectHandle.fromString`. A DSL exposes `registerDialects(registry)`, + which calls the `registerDialect` of each package it emits. +- **Operation list.** A package that builds operations has, in its + `test.zig`, a sorted `const operation_names: []const []const u8` naming + every operation its builders create. `test "every bound operation is + registered"` checks each name against the registry, and checks that + unregistered dialects are not allowed. A builder for an operation the pinned dialect does not register + does not exist; a DSL method builds it from the operations the Python DSL + lowers it to instead. +- **Bindings, not text.** A dialect package builds attributes and types + through C API constructors (`get`/`getChecked`), never by printing and + parsing text. A type or attribute the C API lacks gets a binding (a C + constructor and a Zig wrapper) before anything uses it. `mlir.Type.parse` + and `mlir.Attribute.parse` are for tests. Consumers use the bindings too. +- **Everything compiles.** `test.zig` runs `std.testing.refAllDecls` on the + package and on each of its wrapper types, so no declaration hides behind + lazy analysis. +- **Argument order.** A builder takes `ctx`, the required operands, the + result types (omitted when inferred), the required attributes, then the + optional arguments (an `opts` struct), then `location`, which is always + last. +- **Verification.** A builder verifies the operation it creates. One that + cannot (its region is filled later, or it only verifies inside a parent) + passes `.verify = false` and says so in its doc comment. +- **C API safety.** Every declaration is `MLIR_CAPI_EXPORTED`. Constructors + load the dialect, return a null handle for a null context, and verifying + constructors (`getChecked`) return a null handle for invalid input. No + `alwayslink` and no static initializers: the Zig side calls the + registration function by name, so the linker keeps what it uses. +- **Hygiene.** LLVM file banners and include-path guards (below), clean under + `mlir/.clang-format` and the Bazel toolchain's `zig fmt` (0.17), tests only + in `test.zig`, and the BUILD essentials: `capi` and the `zig_library` + tagged `manual`, an explicit `srcs` list in `td_files`, no `glob(`. + +## Conventions + +### Package layout + +- `mlir/dialects//` is imported as `@import("mlir/dialects/")`, + main file `.zig`. Generated builders go in `ops.zig`, exposed as + `.ops`, with hand-written helpers in `.zig`. +- The C API is `capi.h` / `capi.cc`. Dialects written here use + `{Dialect,Ops,Types,Attributes,Enums}.{td,h,cpp}`; vendored + sources keep their upstream file names. +- A generated file starts with a header naming its source and saying it is + generated. The package that generates it has a `REGENERATING.md` of agent + instructions (sources, generated vs maintained files, steps, checks) that + points here for conventions. Generator scripts are not committed. + +### BUILD.bazel + +Targets in this order: `capi` (`cc_library`, deps `:dialect`, `:headers`, +`@llvm-project//mlir:CAPIIR`), the `zig_library` (public, deps `:capi`, +`//mlir`), `test` (`zig_test`, `main = "test.zig"`, plus `:capi` when +`test.zig` imports `c`), then `td_files` (`includes = ["/"]`, direct td +deps such as `@llvm-project//mlir:OpBaseTdFiles`), the `gentbl_cc_library` +targets the dialect needs (`dialect_inc_gen`, `enums_inc_gen`, +`attrs_inc_gen`, `types_inc_gen`, `ops_inc_gen`), `headers` and `dialect`. +Every dep a source includes or imports is listed directly. Format with +`./tools/buildifier.sh`. + +### C and C++ + +- The first line is an 80-column LLVM banner: + `//===- capi.h - C API for NVIDIA's cuda dialect -------------------*- C -*-===//`. + The tag is `*- C -*-` for C headers, `*- C++ -*-` for C++ headers and + `*- tablegen -*-` for `.td`; `.cc`/`.cpp`/`.inc` banners end in `---===//`. + An optional `//` description follows, then a blank line. +- The guard is the include path in upper case with `/` and `.` as `_`, no + `ZML_` prefix (`MLIR_DIALECTS_CUDA_CUDADIALECT_H`, `..._TD`), closed with + `#endif // `. +- Includes follow LLVM order: own header, package headers, `mlir/` and + `llvm/`, then system headers. `capi.h` includes `mlir-c/IR.h` (which + provides `stdbool` and `stdint`) and the upstream `mlir-c/Dialect/.h` it + extends, inside `extern "C"` under `#ifdef __cplusplus`. +- Hand-written C APIs use fixed-width integers (`uint32_t`, `int64_t`), + parameters `MlirContext context`, `MlirAttribute attr`, `MlirType type`, + and C++ locals `MLIRContext *ctx`. `capi.cc` uses `static` helpers, not + anonymous namespaces, and one static helper loads the dialect + (`loadGPU` in `gpu/capi.cc`). + +### Names and where they come from + +`` is the dialect's C prefix, the first argument of +`MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(, )`: `Cuda`, `CuteNVGPU`, +`Triton`, `ROCDL`. Everything else is named as ODS names it: + +| ODS | C API | Zig | +|---|---|---| +| `AttrDef` (class `NameAttr`) | `mlirAttributeIsAName`, `mlirNameAttrGet` | `NameAttr`, `NameAttr.get(ctx, .{ ... })` | +| `TypeDef` (class `NameType`) | `mlirTypeIsAName`, `mlirNameTypeGet` | `NameType` | +| parameter `$elementType` | `mlirNameAttrGetElementType` | field `.elementType`, accessor `elementType()` | +| list parameter `$archs` | `...GetNumArchs`, `...GetArchs(attr, pos)` | `numArchs()`, `arch(i)` | +| `EnumAttr` / `I32EnumAttr` | `...AttrGet(context, value)`, `...AttrGetValue` | Zig enum, `XAttr.get(ctx, .{ .value = ... })`, `value()` | +| op mnemonic `fast_sin.f32` | generic operation API | builder `fast_sin_f32` | + +- Predicates have no `Attr`/`Type` suffix (`mlirTypeIsACudaEvent`). +- C getters keep MLIR's `Get`. Zig accessors are nouns, as in the rest + of ZML (`value()`, `elementType()`, `numAxes()`, `axis(i)`), never + `get`-prefixed. Predicates keep `is`/`has`. +- Enum fields use the IR spelling (`@"and"`, `@"false"`). +- A builder is named after its mnemonic with dots turned into underscores, + case kept. Its parameters are the ODS operand and attribute names, and its + result types `_type` (`_types` when variadic). +- A name that is a Zig keyword, or that collides with a member, a declaration + of the file or a builder local, takes a trailing underscore (`for_`, + `return_`, `ptr_()`, `format_()`). It is quoted only when that name is taken + too (`TupleType.@"type"(i)`, next to the `type_()` upcast). +- Alternative constructors are `mlirNameAttr
Get` in C, as MLIR's + `mlirDenseElementsAttrSplatGet`, and `get(ctx, ...)` in Zig. + +### Zig API + +```zig +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("cuda"); +} + +/// `#cuda.executable`: Format of a compiled executable +pub const ExecutableAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaExecutable; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const InitArgs = struct { format: ExecutableFormat }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { ... } + pub fn attribute(self: *const Self) *const mlir.Attribute { ... } + pub fn format_(self: *const Self) ExecutableFormat { ... } +}; +``` + +- Wrappers use `mlir.Methods` with `c.MlirAttribute` or `c.MlirType`, upcast + with `attribute()` or `type_()`, and build with `get(ctx, InitArgs)`, a + struct even for one field. `get` returns `mlir.Error!*const X` when the C + getter can return null, a plain pointer otherwise. +- Free helpers are `Attr(ctx, ...)` and `Type(ctx, ...)`. +- Optional builder arguments go in `opts`: generated builders declare it + inline (`opts: struct { ... }`), hand-written ones may name it `Opts`. + Its fields are the optional operands, result types and optional, defaulted + or unit attributes, each defaulting to `null` (or the ODS default), so + callers pass `.{}`. +- Sorted imports (`std` first), no unused declarations. + +### Comments + +- Porting adds no explanatory comments; doc comments on public API stay + short. +- A generated wrapper's doc comment is the IR spelling, a colon and the ODS + summary (``/// `#cuda.executable`: Format of a compiled executable``); a + generated builder's is the IR name, a period, then the summary. + +### Tests + +Beyond the guaranteed `refAllDecls` and registration tests: a +`testContext()` that registers the package and its dependencies and calls +`mlir.Context.init(.{ .registry = registry, .threading = false })` and +`loadAllAvailableDialects()`; attributes and types round-tripped (build with +`get`, print with `{f}`, reparse, compare with `eql`); operations built and +verified. + +## Exceptions + +- Generated `.td` files start with a "generated" header instead of a banner. +- Generated C APIs keep ODS's integer types (`int`) and repeat the null check + and dialect load in every constructor. +- `cute_ir/` and `cute_nvgpu/` keep their generated builders in `.zig`; + `llvm/` keeps its builders in a private `ops.zig`. +- `cuda/`, `cute_nvgpu/` and `nvvm/` are regenerated through + `cute_ir/REGENERATING.md`, `fly_rocdl/` through `fly/REGENERATING.md`. +- `cute_ir:capi` and `cute_nvgpu:capi` each depend on the other's `:dialect`: + their C++ sources use each other's types. +- `cf`, `llvm`, `math`, `memref`, `scf`, `shardy` and `stablehlo` have no + `capi` of their own and bind upstream's C API target directly. +- Hand-written builders with many parameters take an `Args` struct + (`func.func`, `gpu.func`, `ttir.func`, `cuda.kernel`, `cuda_tile.entry`, + `memref.subview`, `vector.transfer_read`, ...); function-like ones carry + `location` inside it. +- Some hand-written `Opts` hold required or non-null defaulted fields where + named arguments read better (`stablehlo.PadOpts`, + `mosaic_tpu.LoadOpts.sublane_mask`, `affine.ForOpts.step`). +- stablehlo, shardy, cuda_tile and ttir getters return plain pointers (their + C calls cannot fail); `ttir.pointerType` and `ttir.tensorDescType` take no + `ctx` (the C API derives it from the element type). +- Enum cases whose IR spelling is not an identifier use the C++ symbol + (rocdl's `volatile_op`), as do the generated cute_nvgpu and ttir enums. +- fly's attributes and cute_ir's algebra attributes have named constructors + (`getStatic`, `getDynamic`, `getTuple`, ...); fly's generated builders use + snake_case parameters. + +## Porting checklist + +1. Choose ``, the C prefix and the handle; create the package with the + BUILD targets above. +2. Vendor or write the `.td`/C++ sources and list every `.td` in `td_files`. +3. Write `capi.h`/`capi.cc` for registration and whatever upstream's C API + lacks: per attribute and type, a predicate, a `Get` and the parameter + getters. +4. Bind in `.zig`, or generate `ops.zig`: `registerDialect`, enums, + wrappers, helpers, builders. +5. Write `test.zig`, with the `operation_names` list. +6. If anything is generated, add a `REGENERATING.md`. +7. Make consumers depend on `//mlir/dialects/` and call + `.registerDialect(registry)`. +8. Run the verification below. + +## Verification + +Run from the repository root, with `ZIG` set to the Bazel toolchain's `zig` +(`AGENTS.md` shows how to find its standard library; `zig` is next to it). + +```bash +#!/usr/bin/env bash +d=mlir/dialects/ +bazel test "//$d:test" && bazel test //mlir/... && bazel build //... +./tools/buildifier.sh -mode=check -r "$PWD/$d" +"$ZIG" fmt --check "$d" +src=(); while IFS= read -r f; do src+=("$f"); done < <(find "$d" -maxdepth 1 \( -name '*.h' -o -name '*.cc' -o -name '*.cpp' -o -name '*.td' -o -name '*.inc' \)) +cc=(); for f in "${src[@]}"; do [[ $f == *.td || $f == *.inc ]] || cc+=("$f"); done +[ ${#cc[@]} -eq 0 ] || clang-format --dry-run --Werror --style=file:mlir/.clang-format "${cc[@]}" + +# Conformance: prints nothing on a conforming package. +for f in "${src[@]}"; do l=$(head -1 "$f"); [ ${#l} -eq 80 ] || grep -q generated <<<"$l" || echo "banner: $f"; done +grep -rln --include='*.h' --include='*.td' 'ZML_' "$d" +grep -nE 'alwayslink|glob\(' "$d/BUILD.bazel" +[ -f "$d/capi.cc" ] && grep -nE 'extern "C"|^namespace \{' "$d/capi.cc" +[ -f "$d/capi.h" ] && grep -nw unsigned "$d/capi.h" +grep -rnE --include='*.zig' 'pub fn get[A-Z][A-Za-z]*\(self' "$d" +grep -rnE --include='*.zig' --exclude=test.zig '(Type|Attribute)\.parse\(' "$d" +grep -rln --include='*.zig' --exclude=test.zig '^test ' "$d" +grep -rqF --include='*.zig' 'pub fn registerDialect' "$d" || echo "missing: registerDialect" +if grep -rqF --include='*.zig' '.make(' "$d"; then + grep -qF 'const operation_names' "$d/test.zig" || echo "missing: operation_names" + grep -qF 'test "every bound operation is registered"' "$d/test.zig" || echo "missing: registration test" +fi +git grep -n 'DialectHandle.fromString' -- dsl zml examples +exit 0 +``` diff --git a/mlir/dialects/affine/BUILD.bazel b/mlir/dialects/affine/BUILD.bazel new file mode 100644 index 00000000..06f31849 --- /dev/null +++ b/mlir/dialects/affine/BUILD.bazel @@ -0,0 +1,35 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:AffineDialect", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "affine", + import_name = "mlir/dialects/affine", + main = "affine.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":affine", + "//mlir", + ], +) diff --git a/mlir/dialects/affine.zig b/mlir/dialects/affine/affine.zig similarity index 93% rename from mlir/dialects/affine.zig rename to mlir/dialects/affine/affine.zig index be0c560c..c808619c 100644 --- a/mlir/dialects/affine.zig +++ b/mlir/dialects/affine/affine.zig @@ -1,9 +1,10 @@ -const std = @import("std"); - -const c = @import("c"); const mlir = @import("mlir"); const stdx = @import("stdx"); +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("affine"); +} + // ============================================================================= // affine.apply / affine.min / affine.max // ============================================================================= @@ -64,7 +65,7 @@ pub fn max( // affine.for / affine.yield // ============================================================================= -pub const ForArgs = struct { +pub const ForOpts = struct { /// Constant lower bound. Pass `null` when supplying a `lower_bound_map`. lower_bound: ?i64 = null, /// Constant upper bound. Pass `null` when supplying an `upper_bound_map`. @@ -87,33 +88,33 @@ pub const ForArgs = struct { /// block takes `index, *init_arg_types` and must terminate with `affine.yield`. pub fn for_( ctx: *mlir.Context, - args: ForArgs, body: *mlir.Block, + opts: ForOpts, location: *const mlir.Location, ) *mlir.Operation { // Resolve final maps. - const lb_map = args.lower_bound_map orelse blk: { - const cst = mlir.AffineExpr.constant(ctx, args.lower_bound orelse 0); + const lb_map = opts.lower_bound_map orelse blk: { + const cst = mlir.AffineExpr.constant(ctx, opts.lower_bound orelse 0); break :blk mlir.AffineMap.get(ctx, 0, 0, &.{cst}); }; - const ub_map = args.upper_bound_map orelse blk: { - const cst = mlir.AffineExpr.constant(ctx, args.upper_bound orelse 0); + const ub_map = opts.upper_bound_map orelse blk: { + const cst = mlir.AffineExpr.constant(ctx, opts.upper_bound orelse 0); break :blk mlir.AffineMap.get(ctx, 0, 0, &.{cst}); }; var operands: stdx.BoundedArray(*const mlir.Value, 64) = .empty; - operands.appendSliceAssumeCapacity(args.lower_bound_operands); - operands.appendSliceAssumeCapacity(args.upper_bound_operands); - operands.appendSliceAssumeCapacity(args.inits); + operands.appendSliceAssumeCapacity(opts.lower_bound_operands); + operands.appendSliceAssumeCapacity(opts.upper_bound_operands); + operands.appendSliceAssumeCapacity(opts.inits); const seg_sizes = [3]i32{ - @intCast(args.lower_bound_operands.len), - @intCast(args.upper_bound_operands.len), - @intCast(args.inits.len), + @intCast(opts.lower_bound_operands.len), + @intCast(opts.upper_bound_operands.len), + @intCast(opts.inits.len), }; var result_types: stdx.BoundedArray(*const mlir.Type, 64) = .empty; - for (args.inits) |v| result_types.appendAssumeCapacity(v.type_()); + for (opts.inits) |v| result_types.appendAssumeCapacity(v.type_()); return mlir.Operation.make(ctx, "affine.for", .{ .operands = .{ .flat = operands.constSlice() }, @@ -123,7 +124,7 @@ pub fn for_( .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), .named(ctx, "lowerBoundMap", .affineMap(lb_map)), .named(ctx, "upperBoundMap", .affineMap(ub_map)), - .named(ctx, "step", .int(ctx, .i64, args.step)), + .named(ctx, "step", .int(ctx, .i64, opts.step)), }, // The for body may reference outer SSA values that only become // reachable once it's appended to its parent — defer to module-level @@ -155,7 +156,9 @@ pub fn if_( operands: []const *const mlir.Value, result_types: []const *const mlir.Type, then_block: *mlir.Block, - else_block: ?*mlir.Block, + opts: struct { + else_block: ?*mlir.Block = null, + }, location: *const mlir.Location, ) *mlir.Operation { var state: mlir.OperationState = .init("affine.if", location); @@ -168,7 +171,7 @@ pub fn if_( const then_region = mlir.Region.init(); then_region.appendOwnedBlock(then_block); const else_region = mlir.Region.init(); - if (else_block) |b| else_region.appendOwnedBlock(b); + if (opts.else_block) |b| else_region.appendOwnedBlock(b); state.addOwnedRegions(&.{ then_region, else_region }); return mlir.Operation.init(&state) catch @panic("Failed to create affine.if"); @@ -271,7 +274,7 @@ pub fn vector_store( pub const ParallelArgs = struct { /// Per-loop-dim reductions, one entry per result. Use the same kinds as - /// `mlir.dialects.memref.AtomicRMWKind` (the dialect uses + /// `AtomicRMWKind` in `mlir/dialects/memref` (the dialect uses /// `arith::AtomicRMWKind`). Pass `&.{}` for parallel loops with no /// loop-carried reductions. reductions: []const i64 = &.{}, @@ -408,7 +411,3 @@ pub fn linearize_index( .location = location, }); } - -test { - std.testing.refAllDecls(@This()); -} diff --git a/mlir/dialects/affine/capi.cc b/mlir/dialects/affine/capi.cc new file mode 100644 index 00000000..2110365f --- /dev/null +++ b/mlir/dialects/affine/capi.cc @@ -0,0 +1,9 @@ +//===- capi.cc - C API for the affine dialect -----------------------------===// + +#include "capi.h" + +#include "mlir/CAPI/Registration.h" +#include "mlir/Dialect/Affine/IR/AffineOps.h" + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Affine, affine, + mlir::affine::AffineDialect) diff --git a/mlir/dialects/affine/capi.h b/mlir/dialects/affine/capi.h new file mode 100644 index 00000000..5ae66467 --- /dev/null +++ b/mlir/dialects/affine/capi.h @@ -0,0 +1,18 @@ +//===- capi.h - C API for the affine dialect ----------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_AFFINE_CAPI_H +#define MLIR_DIALECTS_AFFINE_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Affine, affine); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_AFFINE_CAPI_H diff --git a/mlir/dialects/affine/test.zig b/mlir/dialects/affine/test.zig new file mode 100644 index 00000000..4248d4c3 --- /dev/null +++ b/mlir/dialects/affine/test.zig @@ -0,0 +1,40 @@ +const std = @import("std"); + +const affine = @import("mlir/dialects/affine"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "affine.apply", + "affine.delinearize_index", + "affine.for", + "affine.if", + "affine.linearize_index", + "affine.load", + "affine.max", + "affine.min", + "affine.parallel", + "affine.store", + "affine.vector_load", + "affine.vector_store", + "affine.yield", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(affine); + inline for (comptime std.meta.declarations(affine)) |decl| { + const T = @field(affine, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + affine.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/arith/BUILD.bazel b/mlir/dialects/arith/BUILD.bazel new file mode 100644 index 00000000..90c84ff0 --- /dev/null +++ b/mlir/dialects/arith/BUILD.bazel @@ -0,0 +1,36 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "arith", + import_name = "mlir/dialects/arith", + main = "arith.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//stdx", + "@llvm-project//mlir:CAPIArith", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":arith", + "//mlir", + ], +) diff --git a/mlir/dialects/arith.zig b/mlir/dialects/arith/arith.zig similarity index 91% rename from mlir/dialects/arith.zig rename to mlir/dialects/arith/arith.zig index 70c30574..2e52324e 100644 --- a/mlir/dialects/arith.zig +++ b/mlir/dialects/arith/arith.zig @@ -1,9 +1,11 @@ -const std = @import("std"); - const c = @import("c"); const mlir = @import("mlir"); const stdx = @import("stdx"); +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("arith"); +} + // ============================================================================= // Predicate enums (match ArithBase.td exactly) // ============================================================================= @@ -24,7 +26,7 @@ pub const CmpIPredicate = enum(i64) { /// Arith cmpf predicates. Values match mlir::arith::CmpFPredicate. pub const CmpFPredicate = enum(i64) { - always_false = 0, + false = 0, oeq = 1, ogt = 2, oge = 3, @@ -39,7 +41,7 @@ pub const CmpFPredicate = enum(i64) { ule = 12, une = 13, uno = 14, - always_true = 15, + true = 15, }; // ============================================================================= @@ -68,11 +70,9 @@ pub const FastMathAttr = opaque { pub const isAFn = c.mlirAttributeIsAArithFastMath; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "fastmath"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -83,7 +83,7 @@ pub const FastMathAttr = opaque { const result = c.mlirArithFastMathAttrGet(ctx.ptr(), @bitCast(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) FastMathFlags { + pub fn value(self: *const Self) FastMathFlags { return @bitCast(c.mlirArithFastMathAttrGetValue(self.ptr())); } }; @@ -447,34 +447,3 @@ pub fn select( .location = location, }); } - -test { - std.testing.refAllDecls(@This()); -} - -fn expectPrints(expected: []const u8, value: anytype) !void { - var buf: [128]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{value}); - try std.testing.expectEqualStrings(expected, w.buffered()); -} - -test "arith fastmath attribute is built through the C API" { - const ctx = try mlir.Context.init(.{}); - defer ctx.deinit(); - - const cases = [_]struct { FastMathFlags, []const u8 }{ - .{ .none, "#arith.fastmath" }, - .{ .fast, "#arith.fastmath" }, - .{ .{ .reassoc = true }, "#arith.fastmath" }, - .{ .{ .nnan = true, .ninf = true, .afn = true }, "#arith.fastmath" }, - .{ .{ .nsz = true, .arcp = true, .contract = true }, "#arith.fastmath" }, - }; - for (cases) |case| { - const attr = try FastMathAttr.get(ctx, .{ .value = case[0] }); - try std.testing.expectEqual(case[0], attr.getValue()); - try std.testing.expect(attr.attribute().isA(FastMathAttr) != null); - try expectPrints(case[1], attr); - } - try std.testing.expectError(error.InvalidMlir, FastMathAttr.get(ctx, .{ .value = .{ ._padding = 1 } })); -} diff --git a/mlir/dialects/arith/capi.cc b/mlir/dialects/arith/capi.cc new file mode 100644 index 00000000..228c1d09 --- /dev/null +++ b/mlir/dialects/arith/capi.cc @@ -0,0 +1,25 @@ +//===- capi.cc - C API for the arith dialect ------------------------------===// + +#include "capi.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/Dialect/Arith/IR/Arith.h" + +bool mlirAttributeIsAArithFastMath(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirArithFastMathAttrGet(MlirContext context, uint32_t value) { + mlir::MLIRContext *ctx = unwrap(context); + std::optional flags = + mlir::arith::symbolizeFastMathFlags(value); + if (!ctx || !flags) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::arith::FastMathFlagsAttr::get(ctx, *flags)); +} + +uint32_t mlirArithFastMathAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} diff --git a/mlir/dialects/arith_capi.h b/mlir/dialects/arith/capi.h similarity index 66% rename from mlir/dialects/arith_capi.h rename to mlir/dialects/arith/capi.h index 6012e6ed..7c87aa22 100644 --- a/mlir/dialects/arith_capi.h +++ b/mlir/dialects/arith/capi.h @@ -1,10 +1,10 @@ +//===- capi.h - C API for the arith dialect -----------------------*- C -*-===// +// // C entry points for the `arith` dialect's flag attributes, which upstream's // C API leaves out. -#ifndef ZML_MLIR_DIALECTS_ARITH_CAPI_H -#define ZML_MLIR_DIALECTS_ARITH_CAPI_H -#include -#include +#ifndef MLIR_DIALECTS_ARITH_CAPI_H +#define MLIR_DIALECTS_ARITH_CAPI_H #include "mlir-c/IR.h" @@ -15,11 +15,12 @@ extern "C" { // #arith.fastmath<...>: `value` is the `FastMathFlags` bit set; `Get` returns // a null attribute when it has bits outside the enum. MLIR_CAPI_EXPORTED bool mlirAttributeIsAArithFastMath(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirArithFastMathAttrGet(MlirContext ctx, uint32_t value); +MLIR_CAPI_EXPORTED MlirAttribute mlirArithFastMathAttrGet(MlirContext context, + uint32_t value); MLIR_CAPI_EXPORTED uint32_t mlirArithFastMathAttrGetValue(MlirAttribute attr); #ifdef __cplusplus } #endif -#endif // ZML_MLIR_DIALECTS_ARITH_CAPI_H +#endif // MLIR_DIALECTS_ARITH_CAPI_H diff --git a/mlir/dialects/arith/test.zig b/mlir/dialects/arith/test.zig new file mode 100644 index 00000000..ec1cdf29 --- /dev/null +++ b/mlir/dialects/arith/test.zig @@ -0,0 +1,106 @@ +const std = @import("std"); + +const arith = @import("mlir/dialects/arith"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "arith.addf", + "arith.addi", + "arith.addui_extended", + "arith.andi", + "arith.bitcast", + "arith.ceildivsi", + "arith.ceildivui", + "arith.cmpf", + "arith.cmpi", + "arith.constant", + "arith.convertf", + "arith.divf", + "arith.divsi", + "arith.divui", + "arith.extf", + "arith.extsi", + "arith.extui", + "arith.floordivsi", + "arith.flush_denormals", + "arith.fptosi", + "arith.fptoui", + "arith.index_cast", + "arith.index_castui", + "arith.maximumf", + "arith.maxnumf", + "arith.maxsi", + "arith.maxui", + "arith.minimumf", + "arith.minnumf", + "arith.minsi", + "arith.minui", + "arith.mulf", + "arith.muli", + "arith.mulsi_extended", + "arith.mului_extended", + "arith.negf", + "arith.ori", + "arith.remf", + "arith.remsi", + "arith.remui", + "arith.scaling_truncf", + "arith.select", + "arith.shli", + "arith.shrsi", + "arith.shrui", + "arith.sitofp", + "arith.subf", + "arith.subi", + "arith.truncf", + "arith.trunci", + "arith.uitofp", + "arith.xori", +}; + +fn expectPrints(expected: []const u8, value: anytype) !void { + var buf: [128]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(expected, w.buffered()); +} + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(arith); + inline for (comptime std.meta.declarations(arith)) |decl| { + const T = @field(arith, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "arith fastmath attribute is built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + const cases = [_]struct { arith.FastMathFlags, []const u8 }{ + .{ .none, "#arith.fastmath" }, + .{ .fast, "#arith.fastmath" }, + .{ .{ .reassoc = true }, "#arith.fastmath" }, + .{ .{ .nnan = true, .ninf = true, .afn = true }, "#arith.fastmath" }, + .{ .{ .nsz = true, .arcp = true, .contract = true }, "#arith.fastmath" }, + }; + for (cases) |case| { + const attr = try arith.FastMathAttr.get(ctx, .{ .value = case[0] }); + try std.testing.expectEqual(case[0], attr.value()); + try std.testing.expect(attr.attribute().isA(arith.FastMathAttr) != null); + try expectPrints(case[1], attr); + } + try std.testing.expectError(error.InvalidMlir, arith.FastMathAttr.get(ctx, .{ .value = .{ ._padding = 1 } })); +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + arith.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/arith_capi.cc b/mlir/dialects/arith_capi.cc deleted file mode 100644 index 7fb07c15..00000000 --- a/mlir/dialects/arith_capi.cc +++ /dev/null @@ -1,20 +0,0 @@ -#include "arith_capi.h" - -#include "mlir/CAPI/IR.h" -#include "mlir/Dialect/Arith/IR/Arith.h" - -bool mlirAttributeIsAArithFastMath(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirArithFastMathAttrGet(MlirContext ctx, uint32_t value) { - mlir::MLIRContext *context = unwrap(ctx); - std::optional flags = mlir::arith::symbolizeFastMathFlags(value); - if (!context || !flags) return {nullptr}; - context->getOrLoadDialect(); - return wrap(mlir::arith::FastMathFlagsAttr::get(context, *flags)); -} - -uint32_t mlirArithFastMathAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); -} diff --git a/mlir/dialects/cf/BUILD.bazel b/mlir/dialects/cf/BUILD.bazel new file mode 100644 index 00000000..3d308f63 --- /dev/null +++ b/mlir/dialects/cf/BUILD.bazel @@ -0,0 +1,23 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "cf", + import_name = "mlir/dialects/cf", + main = "cf.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "//stdx", + "@llvm-project//mlir:CAPICF", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":cf", + "//mlir", + ], +) diff --git a/mlir/dialects/cf.zig b/mlir/dialects/cf/cf.zig similarity index 96% rename from mlir/dialects/cf.zig rename to mlir/dialects/cf/cf.zig index 37df9737..9a63297c 100644 --- a/mlir/dialects/cf.zig +++ b/mlir/dialects/cf/cf.zig @@ -3,6 +3,10 @@ const std = @import("std"); const mlir = @import("mlir"); const stdx = @import("stdx"); +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("cf"); +} + // ============================================================================= // cf.assert // ============================================================================= @@ -64,7 +68,9 @@ pub fn cond_br( true_operands: []const *const mlir.Value, false_dest: *mlir.Block, false_operands: []const *const mlir.Value, - branch_weights: ?[]const i32, + opts: struct { + branch_weights: ?[]const i32 = null, + }, location: *const mlir.Location, ) *mlir.Operation { var operands_buf: stdx.BoundedArray(*const mlir.Value, 128) = .empty; @@ -81,7 +87,7 @@ pub fn cond_br( var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; attrs.appendAssumeCapacity(.named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes))); - if (branch_weights) |w| { + if (opts.branch_weights) |w| { std.debug.assert(w.len == 2); attrs.appendAssumeCapacity(.named(ctx, "branch_weights", .denseArray(ctx, .i32, w))); } @@ -160,7 +166,3 @@ pub fn switch_( return mlir.Operation.init(&state) catch @panic("Failed to create cf.switch"); } - -test { - std.testing.refAllDecls(@This()); -} diff --git a/mlir/dialects/cf/test.zig b/mlir/dialects/cf/test.zig new file mode 100644 index 00000000..5a91f8f4 --- /dev/null +++ b/mlir/dialects/cf/test.zig @@ -0,0 +1,31 @@ +const std = @import("std"); + +const cf = @import("mlir/dialects/cf"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "cf.assert", + "cf.br", + "cf.cond_br", + "cf.switch", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(cf); + inline for (comptime std.meta.declarations(cf)) |decl| { + const T = @field(cf, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + cf.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/BUILD.bazel b/mlir/dialects/cuda/BUILD.bazel similarity index 73% rename from mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/BUILD.bazel rename to mlir/dialects/cuda/BUILD.bazel index 615d9b29..bcfabc4a 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/BUILD.bazel +++ b/mlir/dialects/cuda/BUILD.bazel @@ -1,12 +1,55 @@ load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") package(default_visibility = ["//visibility:public"]) +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + ":dialect", + ":headers", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "cuda", + srcs = [ + "ops.zig", + ], + import_name = "mlir/dialects/cuda", + main = "cuda.zig", + tags = ["manual"], + deps = [ + ":capi", + "//mlir", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":cuda", + "//mlir", + ], +) + td_library( name = "td_files", - srcs = glob(["*.td"]), - includes = ["/mlir/dialects/cute_ir/include"], + srcs = [ + "CudaAttrs.td", + "CudaDialect.td", + "CudaEnums.td", + "CudaOps.td", + "CudaTypes.td", + ], + includes = ["/"], deps = [ "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", "@llvm-project//mlir:OpBaseTdFiles", @@ -16,7 +59,6 @@ td_library( gentbl_cc_library( name = "dialect_inc_gen", - includes = ["../../../.."], tbl_outs = [ ( [ @@ -40,7 +82,6 @@ gentbl_cc_library( gentbl_cc_library( name = "enums_inc_gen", - includes = ["../../../.."], tbl_outs = [ ( ["-gen-enum-decls"], @@ -58,7 +99,6 @@ gentbl_cc_library( gentbl_cc_library( name = "attrs_inc_gen", - includes = ["../../../.."], tbl_outs = [ ( [ @@ -82,7 +122,6 @@ gentbl_cc_library( gentbl_cc_library( name = "types_inc_gen", - includes = ["../../../.."], tbl_outs = [ ( [ @@ -104,10 +143,8 @@ gentbl_cc_library( deps = [":td_files"], ) -# Every cuda operation of the compiler's registry. gentbl_cc_library( name = "ops_inc_gen", - includes = ["../../../.."], tbl_outs = [ ( ["-gen-op-decls"], @@ -126,7 +163,6 @@ gentbl_cc_library( cc_library( name = "headers", hdrs = ["CudaDialect.h"], - includes = ["../../../.."], deps = [ ":attrs_inc_gen", ":dialect_inc_gen", @@ -140,3 +176,18 @@ cc_library( "@llvm-project//mlir:SideEffectInterfaces", ], ) + +cc_library( + name = "dialect", + srcs = [ + "CudaDialect.cpp", + "CudaInference.cpp", + ], + deps = [ + ":headers", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:LLVMDialect", + ], +) diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaAttrs.td b/mlir/dialects/cuda/CudaAttrs.td similarity index 89% rename from mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaAttrs.td rename to mlir/dialects/cuda/CudaAttrs.td index abfbb5ee..4d5e6f08 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaAttrs.td +++ b/mlir/dialects/cuda/CudaAttrs.td @@ -1,4 +1,4 @@ -//===- CudaAttrs.td - CUDA host attributes of the CuTe DSL --*- tablegen -*-===// +//===- CudaAttrs.td - CUDA host attributes of the CuTe DSL -*- tablegen -*-===// // // Generated for nvidia-cutlass-dsl 4.8.0 from the DSL compiler // (`_cutlass_ir.cu13` parser messages, its mnemonics, and probing its @@ -12,11 +12,11 @@ // //===----------------------------------------------------------------------===// -#ifndef CUTE_IR_DIALECT_CUDA_IR_CUDA_ATTRS_TD -#define CUTE_IR_DIALECT_CUDA_IR_CUDA_ATTRS_TD +#ifndef MLIR_DIALECTS_CUDA_CUDAATTRS_TD +#define MLIR_DIALECTS_CUDA_CUDAATTRS_TD -include "cute_ir/Dialect/Cuda/IR/CudaDialect.td" -include "cute_ir/Dialect/Cuda/IR/CudaEnums.td" +include "mlir/dialects/cuda/CudaDialect.td" +include "mlir/dialects/cuda/CudaEnums.td" defvar Cuda_Ns = "::mlir::cutlass_compiler::cuda"; @@ -65,4 +65,4 @@ def Cuda_FuncAttributesAttr : Cuda_Attr<"FuncAttributes", "func_attributes"> { let assemblyFormat = "$values"; } -#endif // CUTE_IR_DIALECT_CUDA_IR_CUDA_ATTRS_TD +#endif // MLIR_DIALECTS_CUDA_CUDAATTRS_TD diff --git a/mlir/dialects/cute_ir/lib/Dialect/Cuda/IR/CudaDialect.cpp b/mlir/dialects/cuda/CudaDialect.cpp similarity index 51% rename from mlir/dialects/cute_ir/lib/Dialect/Cuda/IR/CudaDialect.cpp rename to mlir/dialects/cuda/CudaDialect.cpp index 53353381..0ce2910d 100644 --- a/mlir/dialects/cute_ir/lib/Dialect/Cuda/IR/CudaDialect.cpp +++ b/mlir/dialects/cuda/CudaDialect.cpp @@ -1,38 +1,38 @@ -//===- CudaDialect.cpp - NVIDIA's cuda dialect of the CuTe DSL -----------===// +//===- CudaDialect.cpp - NVIDIA's cuda dialect of the CuTe DSL ------------===// -#include "cute_ir/Dialect/Cuda/IR/CudaDialect.h" +#include "mlir/dialects/cuda/CudaDialect.h" -#include "llvm/ADT/TypeSwitch.h" #include "mlir/IR/Builders.h" #include "mlir/IR/DialectImplementation.h" +#include "llvm/ADT/TypeSwitch.h" using namespace mlir; using namespace mlir::cutlass_compiler::cuda; -#include "cute_ir/Dialect/Cuda/IR/CudaDialect.cpp.inc" +#include "mlir/dialects/cuda/CudaDialect.cpp.inc" -#include "cute_ir/Dialect/Cuda/IR/CudaEnums.cpp.inc" +#include "mlir/dialects/cuda/CudaEnums.cpp.inc" #define GET_ATTRDEF_CLASSES -#include "cute_ir/Dialect/Cuda/IR/CudaAttrs.cpp.inc" +#include "mlir/dialects/cuda/CudaAttrs.cpp.inc" #define GET_TYPEDEF_CLASSES -#include "cute_ir/Dialect/Cuda/IR/CudaTypes.cpp.inc" +#include "mlir/dialects/cuda/CudaTypes.cpp.inc" #define GET_OP_CLASSES -#include "cute_ir/Dialect/Cuda/IR/CudaOps.cpp.inc" +#include "mlir/dialects/cuda/CudaOps.cpp.inc" void CudaDialect::initialize() { addAttributes< #define GET_ATTRDEF_LIST -#include "cute_ir/Dialect/Cuda/IR/CudaAttrs.cpp.inc" +#include "mlir/dialects/cuda/CudaAttrs.cpp.inc" >(); addTypes< #define GET_TYPEDEF_LIST -#include "cute_ir/Dialect/Cuda/IR/CudaTypes.cpp.inc" +#include "mlir/dialects/cuda/CudaTypes.cpp.inc" >(); addOperations< #define GET_OP_LIST -#include "cute_ir/Dialect/Cuda/IR/CudaOps.cpp.inc" +#include "mlir/dialects/cuda/CudaOps.cpp.inc" >(); } diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaDialect.h b/mlir/dialects/cuda/CudaDialect.h similarity index 58% rename from mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaDialect.h rename to mlir/dialects/cuda/CudaDialect.h index fac57ef2..a96cdd17 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaDialect.h +++ b/mlir/dialects/cuda/CudaDialect.h @@ -1,7 +1,7 @@ -//===- CudaDialect.h - NVIDIA's cuda dialect of the CuTe DSL ------*- C++ -*-===// +//===- CudaDialect.h - NVIDIA's cuda dialect of the CuTe DSL ----*- C++ -*-===// -#ifndef CUTE_IR_DIALECT_CUDA_IR_CUDA_DIALECT_H -#define CUTE_IR_DIALECT_CUDA_IR_CUDA_DIALECT_H +#ifndef MLIR_DIALECTS_CUDA_CUDADIALECT_H +#define MLIR_DIALECTS_CUDA_CUDADIALECT_H #include "mlir/Bytecode/BytecodeOpInterface.h" #include "mlir/Dialect/LLVMIR/LLVMAttrs.h" @@ -14,17 +14,17 @@ #include "mlir/Interfaces/InferTypeOpInterface.h" #include "mlir/Interfaces/SideEffectInterfaces.h" -#include "cute_ir/Dialect/Cuda/IR/CudaDialect.h.inc" +#include "mlir/dialects/cuda/CudaDialect.h.inc" -#include "cute_ir/Dialect/Cuda/IR/CudaEnums.h.inc" +#include "mlir/dialects/cuda/CudaEnums.h.inc" #define GET_ATTRDEF_CLASSES -#include "cute_ir/Dialect/Cuda/IR/CudaAttrs.h.inc" +#include "mlir/dialects/cuda/CudaAttrs.h.inc" #define GET_TYPEDEF_CLASSES -#include "cute_ir/Dialect/Cuda/IR/CudaTypes.h.inc" +#include "mlir/dialects/cuda/CudaTypes.h.inc" #define GET_OP_CLASSES -#include "cute_ir/Dialect/Cuda/IR/CudaOps.h.inc" +#include "mlir/dialects/cuda/CudaOps.h.inc" -#endif // CUTE_IR_DIALECT_CUDA_IR_CUDA_DIALECT_H +#endif // MLIR_DIALECTS_CUDA_CUDADIALECT_H diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaDialect.td b/mlir/dialects/cuda/CudaDialect.td similarity index 81% rename from mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaDialect.td rename to mlir/dialects/cuda/CudaDialect.td index d8c9d551..939e2d85 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaDialect.td +++ b/mlir/dialects/cuda/CudaDialect.td @@ -1,7 +1,7 @@ -//===- CudaDialect.td - NVIDIA's CUDA host dialect of the CuTe DSL -*- tablegen -*-===// +//===- CudaDialect.td - CUDA host dialect of the CuTe DSL --*- tablegen -*-===// -#ifndef CUTE_IR_DIALECT_CUDA_IR_CUDA_DIALECT_TD -#define CUTE_IR_DIALECT_CUDA_IR_CUDA_DIALECT_TD +#ifndef MLIR_DIALECTS_CUDA_CUDADIALECT_TD +#define MLIR_DIALECTS_CUDA_CUDADIALECT_TD include "mlir/IR/AttrTypeBase.td" include "mlir/IR/DialectBase.td" @@ -35,4 +35,4 @@ class Cuda_Attr let mnemonic = attrMnemonic; } -#endif // CUTE_IR_DIALECT_CUDA_IR_CUDA_DIALECT_TD +#endif // MLIR_DIALECTS_CUDA_CUDADIALECT_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaEnums.td b/mlir/dialects/cuda/CudaEnums.td similarity index 99% rename from mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaEnums.td rename to mlir/dialects/cuda/CudaEnums.td index cc415457..f7980167 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaEnums.td +++ b/mlir/dialects/cuda/CudaEnums.td @@ -5,10 +5,10 @@ // as a dialect attribute also has its EnumAttr, printed `#` // or, where the compiler prints it so, `#nvvm`. -#ifndef CUTE_IR_GENERATED_CUDA_ENUMS_TD -#define CUTE_IR_GENERATED_CUDA_ENUMS_TD +#ifndef MLIR_DIALECTS_CUDA_CUDAENUMS_TD +#define MLIR_DIALECTS_CUDA_CUDAENUMS_TD -include "cute_ir/Dialect/Cuda/IR/CudaDialect.td" +include "mlir/dialects/cuda/CudaDialect.td" include "mlir/IR/EnumAttr.td" def Cuda_ArchPortabilityEnum : I32EnumAttr<"ArchPortability", "Enumerated type describing portability", [ @@ -333,4 +333,4 @@ def Cuda_StreamCaptureModeEnumEnum : I32EnumAttr<"StreamCaptureModeEnum", "Enume let genSpecializedAttr = 0; } -#endif // CUTE_IR_GENERATED_CUDA_ENUMS_TD +#endif // MLIR_DIALECTS_CUDA_CUDAENUMS_TD diff --git a/mlir/dialects/cute_ir/lib/Dialect/Cuda/IR/CudaInference.cpp b/mlir/dialects/cuda/CudaInference.cpp similarity index 90% rename from mlir/dialects/cute_ir/lib/Dialect/Cuda/IR/CudaInference.cpp rename to mlir/dialects/cuda/CudaInference.cpp index b13ce484..1289679e 100644 --- a/mlir/dialects/cute_ir/lib/Dialect/Cuda/IR/CudaInference.cpp +++ b/mlir/dialects/cuda/CudaInference.cpp @@ -1,10 +1,10 @@ -//===- CudaInference.cpp - result types the compiler infers ------------===// +//===- CudaInference.cpp - result types the compiler infers ---------------===// // // Generated from the nvidia-cutlass-dsl 4.8.0 compiler (its inference // probed by varying operand types); do not edit. An operation without a // recovered rule fails inference, so the result types written stand. -#include "cute_ir/Dialect/Cuda/IR/CudaDialect.h" +#include "mlir/dialects/cuda/CudaDialect.h" using namespace mlir; using namespace ::mlir::cutlass_compiler::cuda; diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaOps.td b/mlir/dialects/cuda/CudaOps.td similarity index 99% rename from mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaOps.td rename to mlir/dialects/cuda/CudaOps.td index ee468f71..991814c1 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaOps.td +++ b/mlir/dialects/cuda/CudaOps.td @@ -18,12 +18,12 @@ // // Constraints not recovered: -#ifndef CUTE_IR_GENERATED_CUDA_OPS_TD -#define CUTE_IR_GENERATED_CUDA_OPS_TD +#ifndef MLIR_DIALECTS_CUDA_CUDAOPS_TD +#define MLIR_DIALECTS_CUDA_CUDAOPS_TD -include "cute_ir/Dialect/Cuda/IR/CudaAttrs.td" -include "cute_ir/Dialect/Cuda/IR/CudaEnums.td" -include "cute_ir/Dialect/Cuda/IR/CudaTypes.td" +include "mlir/dialects/cuda/CudaAttrs.td" +include "mlir/dialects/cuda/CudaEnums.td" +include "mlir/dialects/cuda/CudaTypes.td" include "mlir/Interfaces/InferTypeOpInterface.td" include "mlir/Interfaces/SideEffectInterfaces.td" @@ -3069,4 +3069,4 @@ def Cuda_TensorMapEncodeTiledOp : Cuda_Op<"tensor_map_encode_tiled"> { ); } -#endif // CUTE_IR_GENERATED_CUDA_OPS_TD +#endif // MLIR_DIALECTS_CUDA_CUDAOPS_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaTypes.td b/mlir/dialects/cuda/CudaTypes.td similarity index 97% rename from mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaTypes.td rename to mlir/dialects/cuda/CudaTypes.td index 6dc97d51..d149651e 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR/CudaTypes.td +++ b/mlir/dialects/cuda/CudaTypes.td @@ -1,4 +1,4 @@ -//===- CudaTypes.td - CUDA host types of the CuTe DSL -----*- tablegen -*-===// +//===- CudaTypes.td - CUDA host types of the CuTe DSL ------*- tablegen -*-===// // // Generated for nvidia-cutlass-dsl 4.8.0 from the DSL compiler (its typed // Python type classes, its mnemonics, `_cutlass_ir.cu13` parser messages and @@ -11,10 +11,10 @@ // //===----------------------------------------------------------------------===// -#ifndef CUTE_IR_DIALECT_CUDA_IR_CUDA_TYPES_TD -#define CUTE_IR_DIALECT_CUDA_IR_CUDA_TYPES_TD +#ifndef MLIR_DIALECTS_CUDA_CUDATYPES_TD +#define MLIR_DIALECTS_CUDA_CUDATYPES_TD -include "cute_ir/Dialect/Cuda/IR/CudaDialect.td" +include "mlir/dialects/cuda/CudaDialect.td" def Cuda_ArchSetType : Cuda_Type<"ArchSet", "arch_set"> { let summary = "`!cuda.arch_set`"; @@ -209,4 +209,4 @@ def Cuda_TensorMapType : Cuda_Type<"TensorMap", "tensor_map"> { let assemblyFormat = ""; } -#endif // CUTE_IR_DIALECT_CUDA_IR_CUDA_TYPES_TD +#endif // MLIR_DIALECTS_CUDA_CUDATYPES_TD diff --git a/mlir/dialects/cute_ir/lib/CAPI/CudaTypes.cpp b/mlir/dialects/cuda/capi.cc similarity index 56% rename from mlir/dialects/cute_ir/lib/CAPI/CudaTypes.cpp rename to mlir/dialects/cuda/capi.cc index 751f7ed5..4cdff5ee 100644 --- a/mlir/dialects/cute_ir/lib/CAPI/CudaTypes.cpp +++ b/mlir/dialects/cuda/capi.cc @@ -1,17 +1,26 @@ -// C API for the cuda types, generated from the dialect's .td files. -#include "cute_ir-c/Dialect/CudaTypes.h" +//===- capi.cc - C API for NVIDIA's cuda dialect --------------------------===// + +#include "capi.h" + +#include "mlir/dialects/cuda/CudaDialect.h" -#include "cute_ir/Dialect/Cuda/IR/CudaDialect.h" #include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" #include "mlir/CAPI/Support.h" +// Registration + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Cuda, cuda, + mlir::cutlass_compiler::cuda::CudaDialect) + +// Attributes + namespace cuda = mlir::cutlass_compiler::cuda; -namespace { // Unwraps `handle` into `out`: false when it is of another context or not a // T, or when it is null and not optional. template -bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { +static bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { auto value = unwrap(handle); if (!value) return optional; @@ -20,7 +29,138 @@ bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { out = llvm::dyn_cast(value); return static_cast(out); } -} // namespace + +bool mlirAttributeIsACudaAssumeKernel(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCudaAssumeKernelAttrGet(MlirContext context, bool value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::AssumeKernelAttr::get(ctx, value)); +} +bool mlirCudaAssumeKernelAttrGetValue(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getValue(); +} + +bool mlirAttributeIsACudaComputeTarget(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCudaComputeTargetAttrGet(MlirContext context, + uint32_t representation, + uint32_t portability, + intptr_t numArchs, + const uint32_t *archs) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto representationValue = + ::mlir::cutlass_compiler::cuda::symbolizeExecutableRepresentation( + representation); + if (!representationValue) + return {nullptr}; + auto portabilityValue = + ::mlir::cutlass_compiler::cuda::symbolizeArchPortability(portability); + if (!portabilityValue) + return {nullptr}; + llvm::SmallVector<::mlir::cutlass_compiler::cuda::GpuArchitecture> archsValue; + for (intptr_t i = 0; i < numArchs; ++i) { + auto value = + ::mlir::cutlass_compiler::cuda::symbolizeGpuArchitecture(archs[i]); + if (!value) + return {nullptr}; + archsValue.push_back(*value); + } + return wrap(cuda::ComputeTargetAttr::get(ctx, *representationValue, + *portabilityValue, archsValue)); +} +uint32_t mlirCudaComputeTargetAttrGetRepresentation(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getRepresentation()); +} +uint32_t mlirCudaComputeTargetAttrGetPortability(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getPortability()); +} +intptr_t mlirCudaComputeTargetAttrGetNumArchs(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getArchs().size(); +} +uint32_t mlirCudaComputeTargetAttrGetArchs(MlirAttribute attr, intptr_t pos) { + return static_cast( + llvm::cast(unwrap(attr)).getArchs()[pos]); +} + +bool mlirAttributeIsACudaDevMaxSharedMemoryOptin(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCudaDevMaxSharedMemoryOptinAttrGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::DevMaxSharedMemoryOptinAttr::get(ctx)); +} + +bool mlirAttributeIsACudaDeviceAttributes(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCudaDeviceAttributesAttrGet(MlirContext context, + MlirAttribute values) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::DictionaryAttr valuesValue; + if (!unwrapAs(ctx, values, false, valuesValue)) + return {nullptr}; + return wrap(cuda::DeviceAttributesAttr::get(ctx, valuesValue)); +} +MlirAttribute mlirCudaDeviceAttributesAttrGetValues(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getValues())); +} + +bool mlirAttributeIsACudaExecutable(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCudaExecutableAttrGet(MlirContext context, uint32_t format) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto formatValue = + ::mlir::cutlass_compiler::cuda::symbolizeExecutableFormat(format); + if (!formatValue) + return {nullptr}; + return wrap(cuda::ExecutableAttr::get(ctx, *formatValue)); +} +uint32_t mlirCudaExecutableAttrGetFormat(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getFormat()); +} + +bool mlirAttributeIsACudaFuncAttributes(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCudaFuncAttributesAttrGet(MlirContext context, + MlirAttribute values) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::DictionaryAttr valuesValue; + if (!unwrapAs(ctx, values, false, valuesValue)) + return {nullptr}; + return wrap(cuda::FuncAttributesAttr::get(ctx, valuesValue)); +} +MlirAttribute mlirCudaFuncAttributesAttrGetValues(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getValues())); +} + +// Types bool mlirTypeIsACudaArchSet(MlirType type) { return llvm::isa_and_nonnull(unwrap(type)); @@ -113,7 +253,8 @@ MlirType mlirCudaKernelTypeGet(MlirContext context) { bool mlirTypeIsACudaLaunchConfig(MlirType type) { return llvm::isa_and_nonnull(unwrap(type)); } -MlirType mlirCudaLaunchConfigTypeGet(MlirContext context, uint32_t maxNumAttrs) { +MlirType mlirCudaLaunchConfigTypeGet(MlirContext context, + uint32_t maxNumAttrs) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; @@ -201,21 +342,30 @@ MlirType mlirCudaRuntimeKernelNodeParamsTypeGet(MlirContext context) { return wrap(cuda::RuntimeKernelNodeParamsType::get(ctx)); } -bool mlirTypeIsACudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreference(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); +bool mlirTypeIsACudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreference( + MlirType type) { + return llvm::isa_and_nonnull< + cuda::RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType>( + unwrap(type)); } -MlirType mlirCudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceTypeGet(MlirContext context) { +MlirType +mlirCudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceTypeGet( + MlirContext context) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - return wrap(cuda::RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType::get(ctx)); + return wrap( + cuda::RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType:: + get(ctx)); } bool mlirTypeIsACudaRuntimeLaunchAttributeValueCooperative(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); + return llvm::isa_and_nonnull< + cuda::RuntimeLaunchAttributeValueCooperativeType>(unwrap(type)); } -MlirType mlirCudaRuntimeLaunchAttributeValueCooperativeTypeGet(MlirContext context) { +MlirType +mlirCudaRuntimeLaunchAttributeValueCooperativeTypeGet(MlirContext context) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; @@ -224,9 +374,11 @@ MlirType mlirCudaRuntimeLaunchAttributeValueCooperativeTypeGet(MlirContext conte } bool mlirTypeIsACudaRuntimeLaunchAttributeValueMemSyncDomain(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); + return llvm::isa_and_nonnull< + cuda::RuntimeLaunchAttributeValueMemSyncDomainType>(unwrap(type)); } -MlirType mlirCudaRuntimeLaunchAttributeValueMemSyncDomainTypeGet(MlirContext context) { +MlirType +mlirCudaRuntimeLaunchAttributeValueMemSyncDomainTypeGet(MlirContext context) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; @@ -234,32 +386,45 @@ MlirType mlirCudaRuntimeLaunchAttributeValueMemSyncDomainTypeGet(MlirContext con return wrap(cuda::RuntimeLaunchAttributeValueMemSyncDomainType::get(ctx)); } -bool mlirTypeIsACudaRuntimeLaunchAttributeValueNvlinkUtilCentricScheduling(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); +bool mlirTypeIsACudaRuntimeLaunchAttributeValueNvlinkUtilCentricScheduling( + MlirType type) { + return llvm::isa_and_nonnull< + cuda::RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType>( + unwrap(type)); } -MlirType mlirCudaRuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingTypeGet(MlirContext context) { +MlirType mlirCudaRuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingTypeGet( + MlirContext context) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - return wrap(cuda::RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType::get(ctx)); + return wrap( + cuda::RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType::get( + ctx)); } -bool mlirTypeIsACudaRuntimeLaunchAttributeValuePortableClusterSizeMode(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); +bool mlirTypeIsACudaRuntimeLaunchAttributeValuePortableClusterSizeMode( + MlirType type) { + return llvm::isa_and_nonnull< + cuda::RuntimeLaunchAttributeValuePortableClusterSizeModeType>( + unwrap(type)); } -MlirType mlirCudaRuntimeLaunchAttributeValuePortableClusterSizeModeTypeGet(MlirContext context) { +MlirType mlirCudaRuntimeLaunchAttributeValuePortableClusterSizeModeTypeGet( + MlirContext context) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - return wrap(cuda::RuntimeLaunchAttributeValuePortableClusterSizeModeType::get(ctx)); + return wrap( + cuda::RuntimeLaunchAttributeValuePortableClusterSizeModeType::get(ctx)); } bool mlirTypeIsACudaRuntimeLaunchAttributeValuePriority(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); + return llvm::isa_and_nonnull( + unwrap(type)); } -MlirType mlirCudaRuntimeLaunchAttributeValuePriorityTypeGet(MlirContext context) { +MlirType +mlirCudaRuntimeLaunchAttributeValuePriorityTypeGet(MlirContext context) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; @@ -267,21 +432,33 @@ MlirType mlirCudaRuntimeLaunchAttributeValuePriorityTypeGet(MlirContext context) return wrap(cuda::RuntimeLaunchAttributeValuePriorityType::get(ctx)); } -bool mlirTypeIsACudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowed(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); +bool mlirTypeIsACudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowed( + MlirType type) { + return llvm::isa_and_nonnull< + cuda:: + RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType>( + unwrap(type)); } -MlirType mlirCudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedTypeGet(MlirContext context) { +MlirType +mlirCudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedTypeGet( + MlirContext context) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - return wrap(cuda::RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType::get(ctx)); + return wrap( + cuda:: + RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType:: + get(ctx)); } -bool mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemCarveout(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); +bool mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemCarveout( + MlirType type) { + return llvm::isa_and_nonnull< + cuda::RuntimeLaunchAttributeValueSharedMemCarveoutType>(unwrap(type)); } -MlirType mlirCudaRuntimeLaunchAttributeValueSharedMemCarveoutTypeGet(MlirContext context) { +MlirType mlirCudaRuntimeLaunchAttributeValueSharedMemCarveoutTypeGet( + MlirContext context) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; @@ -290,9 +467,11 @@ MlirType mlirCudaRuntimeLaunchAttributeValueSharedMemCarveoutTypeGet(MlirContext } bool mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemoryMode(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); + return llvm::isa_and_nonnull< + cuda::RuntimeLaunchAttributeValueSharedMemoryModeType>(unwrap(type)); } -MlirType mlirCudaRuntimeLaunchAttributeValueSharedMemoryModeTypeGet(MlirContext context) { +MlirType mlirCudaRuntimeLaunchAttributeValueSharedMemoryModeTypeGet( + MlirContext context) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; @@ -301,9 +480,11 @@ MlirType mlirCudaRuntimeLaunchAttributeValueSharedMemoryModeTypeGet(MlirContext } bool mlirTypeIsACudaRuntimeLaunchAttributeValueSyncPolicy(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); + return llvm::isa_and_nonnull( + unwrap(type)); } -MlirType mlirCudaRuntimeLaunchAttributeValueSyncPolicyTypeGet(MlirContext context) { +MlirType +mlirCudaRuntimeLaunchAttributeValueSyncPolicyTypeGet(MlirContext context) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; diff --git a/mlir/dialects/cuda/capi.h b/mlir/dialects/cuda/capi.h new file mode 100644 index 00000000..126a87d8 --- /dev/null +++ b/mlir/dialects/cuda/capi.h @@ -0,0 +1,253 @@ +//===- capi.h - C API for NVIDIA's cuda dialect -------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_CUDA_CAPI_H +#define MLIR_DIALECTS_CUDA_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Registration + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Cuda, cuda); + +// Attributes + +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles. Enums are their integer +// values (the dialect's *Enums.td). Getters require a value accepted by the +// matching IsA. + +// `#cuda.assume_kernel_attr`: Whether `cuda.launch` may assume its callee is a +// kernel +MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaAssumeKernel(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaAssumeKernelAttrGet(MlirContext context, bool value); +MLIR_CAPI_EXPORTED bool mlirCudaAssumeKernelAttrGetValue(MlirAttribute attr); + +// `#cuda.compute_target`: Representation, portability and architectures of a +// compiled executable +MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaComputeTarget(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCudaComputeTargetAttrGet( + MlirContext context, uint32_t representation, uint32_t portability, + intptr_t numArchs, const uint32_t *archs); +MLIR_CAPI_EXPORTED uint32_t +mlirCudaComputeTargetAttrGetRepresentation(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t +mlirCudaComputeTargetAttrGetPortability(MlirAttribute attr); +MLIR_CAPI_EXPORTED intptr_t +mlirCudaComputeTargetAttrGetNumArchs(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t +mlirCudaComputeTargetAttrGetArchs(MlirAttribute attr, intptr_t pos); + +// `#cuda.dev_max_shared_memory_optin`: The device's opt-in maximum of shared +// memory per block +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACudaDevMaxSharedMemoryOptin(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaDevMaxSharedMemoryOptinAttrGet(MlirContext context); + +// `#cuda.device_attributes`: CUDA device attributes +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACudaDeviceAttributes(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaDeviceAttributesAttrGet(MlirContext context, MlirAttribute values); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaDeviceAttributesAttrGetValues(MlirAttribute attr); + +// `#cuda.executable`: Format of a compiled executable +MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaExecutable(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCudaExecutableAttrGet(MlirContext context, + uint32_t format); +MLIR_CAPI_EXPORTED uint32_t mlirCudaExecutableAttrGetFormat(MlirAttribute attr); + +// `#cuda.func_attributes`: CUDA function attributes of a kernel +MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaFuncAttributes(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaFuncAttributesAttrGet(MlirContext context, MlirAttribute values); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaFuncAttributesAttrGetValues(MlirAttribute attr); + +// Types + +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles. Enums are their integer +// values (the dialect's *Enums.td). Getters require a value accepted by the +// matching IsA. + +// `!cuda.arch_set`: `!cuda.arch_set` +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaArchSet(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaArchSetTypeGet(MlirContext context); + +// `!cuda.event`: `Event` represents CUDA event +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaEvent(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaEventTypeGet(MlirContext context); + +// `!cuda.graph_device_node`: `GraphDeviceNode` represents CUDA device node +// handle for device-side node update +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaGraphDeviceNode(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaGraphDeviceNodeTypeGet(MlirContext context); + +// `!cuda.graph_exec`: CUDA Graph Execution +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaGraphExec(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaGraphExecTypeGet(MlirContext context); + +// `!cuda.graph_node`: CUDA Graph Node handle +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaGraphNode(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaGraphNodeTypeGet(MlirContext context); + +// `!cuda.graph`: CUDA Graph +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaGraph(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaGraphTypeGet(MlirContext context); + +// `!cuda.int`: `cuda.int` is a integer type with a bitwidth matching the +// underlying ABI integer bitwidth +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaInt(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaIntTypeGet(MlirContext context); + +// `!cuda.kernel`: CUDA Kernel handle +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaKernel(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaKernelTypeGet(MlirContext context); + +// `!cuda.launch_cfg`: Representation of CUDA's extensible launch configuration +// mirroring cuda runtime API +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaLaunchConfig(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaLaunchConfigTypeGet(MlirContext context, + uint32_t maxNumAttrs); +MLIR_CAPI_EXPORTED uint32_t +mlirCudaLaunchConfigTypeGetMaxNumAttrs(MlirType type); + +// `!cuda.library`: CUDA Library handle +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaLibrary(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaLibraryTypeGet(MlirContext context); + +// `!cuda.module`: CUDA Module handle +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaModule(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaModuleTypeGet(MlirContext context); + +// `!cuda.result`: CUDA Result type +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaResult(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaResultTypeGet(MlirContext context); + +// `!cuda.runtime.device_prop`: CUDA cudaDeviceProp struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeDeviceProp(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeDevicePropTypeGet(MlirContext context); + +// `!cuda.runtime.dim_3`: CUDA dim3 struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeDim3(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeDim3TypeGet(MlirContext context); + +// `!cuda.runtime.extent`: CUDA cudaExtent struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeExtent(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeExtentTypeGet(MlirContext context); + +// `!cuda.runtime.kernel_node_params`: CUDA cudaKernelNodeParams struct type +// (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeKernelNodeParams(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeKernelNodeParamsTypeGet(MlirContext context); + +// `!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreference( + MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceTypeGet( + MlirContext context); + +// `!cuda.runtime.launch_attribute_value.cooperative` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValueCooperative(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValueCooperativeTypeGet(MlirContext context); + +// `!cuda.runtime.launch_attribute_value.mem_sync_domain` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValueMemSyncDomain(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValueMemSyncDomainTypeGet(MlirContext context); + +// `!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValueNvlinkUtilCentricScheduling( + MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingTypeGet( + MlirContext context); + +// `!cuda.runtime.launch_attribute_value.portable_cluster_size_mode` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValuePortableClusterSizeMode( + MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValuePortableClusterSizeModeTypeGet( + MlirContext context); + +// `!cuda.runtime.launch_attribute_value.priority` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValuePriority(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValuePriorityTypeGet(MlirContext context); + +// `!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowed( + MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedTypeGet( + MlirContext context); + +// `!cuda.runtime.launch_attribute_value.shared_mem_carveout` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemCarveout(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValueSharedMemCarveoutTypeGet( + MlirContext context); + +// `!cuda.runtime.launch_attribute_value.shared_memory_mode` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemoryMode(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValueSharedMemoryModeTypeGet(MlirContext context); + +// `!cuda.runtime.launch_attribute_value.sync_policy` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValueSyncPolicy(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValueSyncPolicyTypeGet(MlirContext context); + +// `!cuda.runtime.memcpy_3d_params`: CUDA cudaMemcpy3DParms struct type +// (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeMemcpy3DParams(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeMemcpy3DParamsTypeGet(MlirContext context); + +// `!cuda.runtime.pitched_ptr`: CUDA cudaPitchedPtr struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimePitchedPtr(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimePitchedPtrTypeGet(MlirContext context); + +// `!cuda.runtime.pos`: CUDA cudaPos struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimePos(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimePosTypeGet(MlirContext context); + +// `!cuda.runtime.uuid`: CUDA cudaUUID_t struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeUuid(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeUuidTypeGet(MlirContext context); + +// `!cuda.stream`: CUDA Stream +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaStream(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaStreamTypeGet(MlirContext context); + +// `!cuda.tensor_map`: CUDA Tensor Map type +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaTensorMap(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaTensorMapTypeGet(MlirContext context); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_CUDA_CAPI_H diff --git a/mlir/dialects/cuda.zig b/mlir/dialects/cuda/cuda.zig similarity index 87% rename from mlir/dialects/cuda.zig rename to mlir/dialects/cuda/cuda.zig index d610e623..5fb18635 100644 --- a/mlir/dialects/cuda.zig +++ b/mlir/dialects/cuda/cuda.zig @@ -1,17 +1,15 @@ //! Builders for the CUDA host-program operations CuTe kernels emit, over the -//! bindings of NVIDIA's `cuda` dialect (`mlir/dialects/cute_ir`, `cute.cuda`), +//! bindings of NVIDIA's `cuda` dialect (`mlir/dialects/cuda`, `ops`), //! which the CuTe-DSL compiler registers. Each builder fixes the attributes the //! kernels do not choose. -const std = @import("std"); - -const cute = @import("mlir/dialects/cute_ir"); const mlir = @import("mlir"); -const stdx = @import("stdx"); -pub const ops = cute.cuda; +pub const ops = @import("ops.zig"); -pub const dialect_namespace = ops.dialect_namespace; +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("cuda"); +} pub fn streamType(ctx: *mlir.Context) *const mlir.Type { return (ops.StreamType.get(ctx, .{}) catch unreachable).type_(); @@ -42,13 +40,13 @@ pub const KernelArgs = struct { /// `cuda.kernel`; `body` is filled afterwards. pub fn kernel(ctx: *mlir.Context, args: KernelArgs) *mlir.Operation { - const op = ops.kernel(ctx, .string(ctx, args.name), .typeAttr(args.function_type), args.arg_attrs, null, null, .{args.body}, args.location); + const op = ops.kernel(ctx, .string(ctx, args.name), .typeAttr(args.function_type), .{args.body}, .{ .arg_attrs = args.arg_attrs }, args.location); for (args.extra_attributes) |attr| op.setAttributeByName(attr.name().str(), attr.attribute()); return op; } pub fn return_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return ops.@"return"(ctx, values, location); + return ops.return_(ctx, values, location); } pub fn cast(ctx: *mlir.Context, dst: *const mlir.Type, src: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { @@ -102,9 +100,5 @@ pub const LaunchExArgs = struct { }; pub fn launch_ex(ctx: *mlir.Context, args: LaunchExArgs) *mlir.Operation { - return ops.launch_ex(ctx, args.config, args.inputs, resultType(ctx), args.callee, null, null, args.assume_kernel_attr, args.location); -} - -test { - std.testing.refAllDecls(@This()); + return ops.launch_ex(ctx, args.config, args.inputs, resultType(ctx), args.callee, .{ .assume_kernel_attr = args.assume_kernel_attr }, args.location); } diff --git a/mlir/dialects/cute_ir/cuda.zig b/mlir/dialects/cuda/ops.zig similarity index 88% rename from mlir/dialects/cute_ir/cuda.zig rename to mlir/dialects/cuda/ops.zig index f3cfcc01..baeb0ded 100644 --- a/mlir/dialects/cute_ir/cuda.zig +++ b/mlir/dialects/cuda/ops.zig @@ -1,13 +1,10 @@ //! Zig bindings of NVIDIA's CUDA host dialect of the CuTe DSL (nvidia-cutlass-dsl 4.8.0), generated //! from the dialect's .td files; do not edit. -const std = @import("std"); - const c = @import("c"); const mlir = @import("mlir"); const stdx = @import("stdx"); -pub const dialect_namespace = "cuda"; /// The C dialect handle, `mlirGetDialectHandle__cuda__`. pub const dialect_handle = "cuda"; @@ -312,11 +309,9 @@ pub const AssumeKernelAttr = opaque { pub const isAFn = c.mlirAttributeIsACudaAssumeKernel; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "assume_kernel_attr"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -327,7 +322,7 @@ pub const AssumeKernelAttr = opaque { const result = c.mlirCudaAssumeKernelAttrGet(ctx.ptr(), args.value); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) bool { + pub fn value(self: *const Self) bool { return c.mlirCudaAssumeKernelAttrGetValue(self.ptr()); } }; @@ -339,11 +334,9 @@ pub const ComputeTargetAttr = opaque { pub const isAFn = c.mlirAttributeIsACudaComputeTarget; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "compute_target"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -356,16 +349,16 @@ pub const ComputeTargetAttr = opaque { const result = c.mlirCudaComputeTargetAttrGet(ctx.ptr(), @backingInt(args.representation), @backingInt(args.portability), @intCast(args.archs.len), @ptrCast(args.archs.ptr)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getRepresentation(self: *const Self) ExecutableRepresentation { + pub fn representation(self: *const Self) ExecutableRepresentation { return @fromBackingInt(@intCast(c.mlirCudaComputeTargetAttrGetRepresentation(self.ptr()))); } - pub fn getPortability(self: *const Self) ArchPortability { + pub fn portability(self: *const Self) ArchPortability { return @fromBackingInt(@intCast(c.mlirCudaComputeTargetAttrGetPortability(self.ptr()))); } - pub fn getNumArchs(self: *const Self) usize { + pub fn numArchs(self: *const Self) usize { return @intCast(c.mlirCudaComputeTargetAttrGetNumArchs(self.ptr())); } - pub fn getArch(self: *const Self, pos: usize) GpuArchitecture { + pub fn arch(self: *const Self, pos: usize) GpuArchitecture { return @fromBackingInt(@intCast(c.mlirCudaComputeTargetAttrGetArchs(self.ptr(), @intCast(pos)))); } }; @@ -377,11 +370,9 @@ pub const DevMaxSharedMemoryOptinAttr = opaque { pub const isAFn = c.mlirAttributeIsACudaDevMaxSharedMemoryOptin; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "dev_max_shared_memory_optin"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -400,11 +391,9 @@ pub const DeviceAttributesAttr = opaque { pub const isAFn = c.mlirAttributeIsACudaDeviceAttributes; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "device_attributes"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -415,7 +404,7 @@ pub const DeviceAttributesAttr = opaque { const result = c.mlirCudaDeviceAttributesAttrGet(ctx.ptr(), args.values.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValues(self: *const Self) *const mlir.Attribute { + pub fn values(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCudaDeviceAttributesAttrGetValues(self.ptr()).ptr.?); } }; @@ -427,11 +416,9 @@ pub const ExecutableAttr = opaque { pub const isAFn = c.mlirAttributeIsACudaExecutable; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "executable"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -442,7 +429,7 @@ pub const ExecutableAttr = opaque { const result = c.mlirCudaExecutableAttrGet(ctx.ptr(), @backingInt(args.format)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getFormat(self: *const Self) ExecutableFormat { + pub fn format_(self: *const Self) ExecutableFormat { return @fromBackingInt(@intCast(c.mlirCudaExecutableAttrGetFormat(self.ptr()))); } }; @@ -454,11 +441,9 @@ pub const FuncAttributesAttr = opaque { pub const isAFn = c.mlirAttributeIsACudaFuncAttributes; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "func_attributes"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -469,7 +454,7 @@ pub const FuncAttributesAttr = opaque { const result = c.mlirCudaFuncAttributesAttrGet(ctx.ptr(), args.values.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValues(self: *const Self) *const mlir.Attribute { + pub fn values(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCudaFuncAttributesAttrGetValues(self.ptr()).ptr.?); } }; @@ -481,11 +466,9 @@ pub const ArchSetType = opaque { pub const isAFn = c.mlirTypeIsACudaArchSet; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "arch_set"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -504,11 +487,9 @@ pub const EventType = opaque { pub const isAFn = c.mlirTypeIsACudaEvent; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "event"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -527,11 +508,9 @@ pub const GraphDeviceNodeType = opaque { pub const isAFn = c.mlirTypeIsACudaGraphDeviceNode; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "graph_device_node"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -550,11 +529,9 @@ pub const GraphExecType = opaque { pub const isAFn = c.mlirTypeIsACudaGraphExec; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "graph_exec"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -573,11 +550,9 @@ pub const GraphNodeType = opaque { pub const isAFn = c.mlirTypeIsACudaGraphNode; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "graph_node"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -596,11 +571,9 @@ pub const GraphType = opaque { pub const isAFn = c.mlirTypeIsACudaGraph; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "graph"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -619,11 +592,9 @@ pub const IntType = opaque { pub const isAFn = c.mlirTypeIsACudaInt; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "int"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -642,11 +613,9 @@ pub const KernelType = opaque { pub const isAFn = c.mlirTypeIsACudaKernel; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "kernel"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -665,11 +634,9 @@ pub const LaunchConfigType = opaque { pub const isAFn = c.mlirTypeIsACudaLaunchConfig; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "launch_cfg"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -680,7 +647,7 @@ pub const LaunchConfigType = opaque { const result = c.mlirCudaLaunchConfigTypeGet(ctx.ptr(), args.maxNumAttrs); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getMaxNumAttrs(self: *const Self) u32 { + pub fn maxNumAttrs(self: *const Self) u32 { return c.mlirCudaLaunchConfigTypeGetMaxNumAttrs(self.ptr()); } }; @@ -692,11 +659,9 @@ pub const LibraryType = opaque { pub const isAFn = c.mlirTypeIsACudaLibrary; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "library"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -715,11 +680,9 @@ pub const ModuleType = opaque { pub const isAFn = c.mlirTypeIsACudaModule; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "module"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -738,11 +701,9 @@ pub const ResultType = opaque { pub const isAFn = c.mlirTypeIsACudaResult; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "result"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -761,11 +722,9 @@ pub const RuntimeDevicePropType = opaque { pub const isAFn = c.mlirTypeIsACudaRuntimeDeviceProp; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "runtime.device_prop"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -784,11 +743,9 @@ pub const RuntimeDim3Type = opaque { pub const isAFn = c.mlirTypeIsACudaRuntimeDim3; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "runtime.dim_3"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -807,11 +764,9 @@ pub const RuntimeExtentType = opaque { pub const isAFn = c.mlirTypeIsACudaRuntimeExtent; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "runtime.extent"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -830,11 +785,9 @@ pub const RuntimeKernelNodeParamsType = opaque { pub const isAFn = c.mlirTypeIsACudaRuntimeKernelNodeParams; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "runtime.kernel_node_params"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -853,11 +806,9 @@ pub const RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType = opa pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreference; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "runtime.launch_attribute_value.cluster_scheduling_policy_preference"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -876,11 +827,9 @@ pub const RuntimeLaunchAttributeValueCooperativeType = opaque { pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueCooperative; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "runtime.launch_attribute_value.cooperative"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -899,11 +848,9 @@ pub const RuntimeLaunchAttributeValueMemSyncDomainType = opaque { pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueMemSyncDomain; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "runtime.launch_attribute_value.mem_sync_domain"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -922,11 +869,9 @@ pub const RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType = opaque { pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueNvlinkUtilCentricScheduling; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "runtime.launch_attribute_value.nvlink_util_centric_scheduling"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -945,11 +890,9 @@ pub const RuntimeLaunchAttributeValuePortableClusterSizeModeType = opaque { pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValuePortableClusterSizeMode; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "runtime.launch_attribute_value.portable_cluster_size_mode"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -968,11 +911,9 @@ pub const RuntimeLaunchAttributeValuePriorityType = opaque { pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValuePriority; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "runtime.launch_attribute_value.priority"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -991,11 +932,9 @@ pub const RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowed; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "runtime.launch_attribute_value.programmatic_stream_serialization_allowed"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1014,11 +953,9 @@ pub const RuntimeLaunchAttributeValueSharedMemCarveoutType = opaque { pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemCarveout; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "runtime.launch_attribute_value.shared_mem_carveout"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1037,11 +974,9 @@ pub const RuntimeLaunchAttributeValueSharedMemoryModeType = opaque { pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemoryMode; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "runtime.launch_attribute_value.shared_memory_mode"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1060,11 +995,9 @@ pub const RuntimeLaunchAttributeValueSyncPolicyType = opaque { pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueSyncPolicy; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "runtime.launch_attribute_value.sync_policy"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1083,11 +1016,9 @@ pub const RuntimeMemcpy3DParamsType = opaque { pub const isAFn = c.mlirTypeIsACudaRuntimeMemcpy3DParams; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "runtime.memcpy_3d_params"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1106,11 +1037,9 @@ pub const RuntimePitchedPtrType = opaque { pub const isAFn = c.mlirTypeIsACudaRuntimePitchedPtr; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "runtime.pitched_ptr"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1129,11 +1058,9 @@ pub const RuntimePosType = opaque { pub const isAFn = c.mlirTypeIsACudaRuntimePos; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "runtime.pos"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1152,11 +1079,9 @@ pub const RuntimeUuidType = opaque { pub const isAFn = c.mlirTypeIsACudaRuntimeUuid; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "runtime.uuid"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1175,11 +1100,9 @@ pub const StreamType = opaque { pub const isAFn = c.mlirTypeIsACudaStream; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "stream"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1198,11 +1121,9 @@ pub const TensorMapType = opaque { pub const isAFn = c.mlirTypeIsACudaTensorMap; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "tensor_map"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1258,9 +1179,16 @@ pub fn compose(ctx: *mlir.Context, fields: []const *const mlir.Value, result_typ } /// `cuda.decompose`. Result types are explicit; attributes use mlir.Attribute. -pub fn decompose(ctx: *mlir.Context, source: *const mlir.Value, fields_type: ?*const mlir.Type, location: *const mlir.Location) *mlir.Operation { +pub fn decompose( + ctx: *mlir.Context, + source: *const mlir.Value, + opts: struct { + fields_type: ?*const mlir.Type = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; - if (fields_type) |value| results.appendAssumeCapacity(value); + if (opts.fields_type) |value| results.appendAssumeCapacity(value); return mlir.Operation.make(ctx, "cuda.decompose", .{ .operands = .{ .flat = &.{source} }, .results = .{ .flat = results.constSlice() }, @@ -1474,21 +1402,39 @@ pub fn event_elapsed_time(ctx: *mlir.Context, start_event: *const mlir.Value, en } /// `cuda.event.record`. Result types are explicit; attributes use mlir.Attribute. -pub fn event_record(ctx: *mlir.Context, event: *const mlir.Value, stream: ?*const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +pub fn event_record( + ctx: *mlir.Context, + event: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + stream: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cuda.event.record", .{ - .operands = .{ .flat = if (stream) |value| &.{ event, value } else &.{event} }, + .operands = .{ .flat = if (opts.stream) |value| &.{ event, value } else &.{event} }, .results = .{ .flat = &.{result_type} }, .location = location, }); } /// `cuda.func`. Result types are explicit; attributes use mlir.Attribute. Not verified on creation: the blocks are usually filled afterwards. -pub fn func(ctx: *mlir.Context, sym_name: *const mlir.Attribute, function_type: *const mlir.Attribute, arg_attrs: ?*const mlir.Attribute, res_attrs: ?*const mlir.Attribute, blocks: [1]*mlir.Block, location: *const mlir.Location) *mlir.Operation { +pub fn func( + ctx: *mlir.Context, + sym_name: *const mlir.Attribute, + function_type: *const mlir.Attribute, + blocks: [1]*mlir.Block, + opts: struct { + arg_attrs: ?*const mlir.Attribute = null, + res_attrs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; attributes.appendAssumeCapacity(.named(ctx, "sym_name", sym_name)); attributes.appendAssumeCapacity(.named(ctx, "function_type", function_type)); - if (arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); - if (res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); + if (opts.arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); + if (opts.res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); return mlir.Operation.make(ctx, "cuda.func", .{ .attributes = attributes.constSlice(), .blocks = &blocks, @@ -1543,13 +1489,24 @@ pub fn graph_exec_destroy(ctx: *mlir.Context, graph_exec: *const mlir.Value, res } /// `cuda.kernel`. Result types are explicit; attributes use mlir.Attribute. Not verified on creation: the blocks are usually filled afterwards. -pub fn kernel(ctx: *mlir.Context, sym_name: *const mlir.Attribute, function_type: *const mlir.Attribute, arg_attrs: ?*const mlir.Attribute, res_attrs: ?*const mlir.Attribute, cu_func_attrs: ?*const mlir.Attribute, blocks: [1]*mlir.Block, location: *const mlir.Location) *mlir.Operation { +pub fn kernel( + ctx: *mlir.Context, + sym_name: *const mlir.Attribute, + function_type: *const mlir.Attribute, + blocks: [1]*mlir.Block, + opts: struct { + arg_attrs: ?*const mlir.Attribute = null, + res_attrs: ?*const mlir.Attribute = null, + cu_func_attrs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; attributes.appendAssumeCapacity(.named(ctx, "sym_name", sym_name)); attributes.appendAssumeCapacity(.named(ctx, "function_type", function_type)); - if (arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); - if (res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); - if (cu_func_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "cu_func_attrs", value)); + if (opts.arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); + if (opts.res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); + if (opts.cu_func_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "cu_func_attrs", value)); return mlir.Operation.make(ctx, "cuda.kernel", .{ .attributes = attributes.constSlice(), .blocks = &blocks, @@ -1559,12 +1516,27 @@ pub fn kernel(ctx: *mlir.Context, sym_name: *const mlir.Attribute, function_type } /// `cuda.launch`. Result types are explicit; attributes use mlir.Attribute. -pub fn launch(ctx: *mlir.Context, grid: []const *const mlir.Value, block: []const *const mlir.Value, dynamicSmem: *const mlir.Value, cudaStream: *const mlir.Value, inputs: []const *const mlir.Value, cuda_result_type: *const mlir.Type, callee: *const mlir.Attribute, arg_attrs: ?*const mlir.Attribute, res_attrs: ?*const mlir.Attribute, assume_kernel_attr: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn launch( + ctx: *mlir.Context, + grid: []const *const mlir.Value, + block: []const *const mlir.Value, + dynamicSmem: *const mlir.Value, + cudaStream: *const mlir.Value, + inputs: []const *const mlir.Value, + cuda_result_type: *const mlir.Type, + callee: *const mlir.Attribute, + opts: struct { + arg_attrs: ?*const mlir.Attribute = null, + res_attrs: ?*const mlir.Attribute = null, + assume_kernel_attr: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; attributes.appendAssumeCapacity(.named(ctx, "callee", callee)); - if (arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); - if (res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); - if (assume_kernel_attr) |value| attributes.appendAssumeCapacity(.named(ctx, "assume_kernel_attr", value)); + if (opts.arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); + if (opts.res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); + if (opts.assume_kernel_attr) |value| attributes.appendAssumeCapacity(.named(ctx, "assume_kernel_attr", value)); return mlir.Operation.make(ctx, "cuda.launch", .{ .operands = .{ .variadic = &.{ grid, @@ -1753,15 +1725,27 @@ pub fn launch_cfg_sync_policy(ctx: *mlir.Context, launch_cfg: *const mlir.Value, } /// `cuda.launch_ex`. Result types are explicit; attributes use mlir.Attribute. -pub fn launch_ex(ctx: *mlir.Context, launch_cfg: *const mlir.Value, inputs: []const *const mlir.Value, cuda_result_type: *const mlir.Type, callee: *const mlir.Attribute, arg_attrs: ?*const mlir.Attribute, res_attrs: ?*const mlir.Attribute, assume_kernel_attr: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn launch_ex( + ctx: *mlir.Context, + launch_cfg: *const mlir.Value, + inputs: []const *const mlir.Value, + cuda_result_type: *const mlir.Type, + callee: *const mlir.Attribute, + opts: struct { + arg_attrs: ?*const mlir.Attribute = null, + res_attrs: ?*const mlir.Attribute = null, + assume_kernel_attr: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var operands: stdx.BoundedArray(*const mlir.Value, 256) = .empty; operands.appendAssumeCapacity(launch_cfg); operands.appendSliceAssumeCapacity(inputs); var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; attributes.appendAssumeCapacity(.named(ctx, "callee", callee)); - if (arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); - if (res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); - if (assume_kernel_attr) |value| attributes.appendAssumeCapacity(.named(ctx, "assume_kernel_attr", value)); + if (opts.arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); + if (opts.res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); + if (opts.assume_kernel_attr) |value| attributes.appendAssumeCapacity(.named(ctx, "assume_kernel_attr", value)); return mlir.Operation.make(ctx, "cuda.launch_ex", .{ .operands = .{ .flat = operands.constSlice() }, .results = .{ .flat = &.{cuda_result_type} }, @@ -1771,10 +1755,24 @@ pub fn launch_ex(ctx: *mlir.Context, launch_cfg: *const mlir.Value, inputs: []co } /// `cuda.launch_kernel`. Result types are explicit; attributes use mlir.Attribute. -pub fn launch_kernel(ctx: *mlir.Context, kernel_: *const mlir.Value, grid: []const *const mlir.Value, block: []const *const mlir.Value, dynamicSmem: *const mlir.Value, cudaStream: *const mlir.Value, kernel_args: *const mlir.Value, cuda_result_type: *const mlir.Type, arg_attrs: ?*const mlir.Attribute, res_attrs: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn launch_kernel( + ctx: *mlir.Context, + kernel_: *const mlir.Value, + grid: []const *const mlir.Value, + block: []const *const mlir.Value, + dynamicSmem: *const mlir.Value, + cudaStream: *const mlir.Value, + kernel_args: *const mlir.Value, + cuda_result_type: *const mlir.Type, + opts: struct { + arg_attrs: ?*const mlir.Attribute = null, + res_attrs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); - if (res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); + if (opts.arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); + if (opts.res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); return mlir.Operation.make(ctx, "cuda.launch_kernel", .{ .operands = .{ .variadic = &.{ &.{kernel_}, @@ -1866,9 +1864,20 @@ pub fn memory_copy(ctx: *mlir.Context, src: *const mlir.Value, dst: *const mlir. } /// `cuda.memory.copy_async`. Result types are explicit; attributes use mlir.Attribute. -pub fn memory_copy_async(ctx: *mlir.Context, src: *const mlir.Value, dst: *const mlir.Value, bytes: *const mlir.Value, stream: ?*const mlir.Value, cuda_result_type: *const mlir.Type, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn memory_copy_async( + ctx: *mlir.Context, + src: *const mlir.Value, + dst: *const mlir.Value, + bytes: *const mlir.Value, + cuda_result_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + stream: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cuda.memory.copy_async", .{ - .operands = .{ .flat = if (stream) |value| &.{ src, dst, bytes, value } else &.{ src, dst, bytes } }, + .operands = .{ .flat = if (opts.stream) |value| &.{ src, dst, bytes, value } else &.{ src, dst, bytes } }, .results = .{ .flat = &.{cuda_result_type} }, .attributes = &.{ .named(ctx, "kind", kind), @@ -1914,9 +1923,19 @@ pub fn memory_set(ctx: *mlir.Context, ptr: *const mlir.Value, value_: *const mli } /// `cuda.memory.set_async`. Result types are explicit; attributes use mlir.Attribute. -pub fn memory_set_async(ctx: *mlir.Context, ptr: *const mlir.Value, value_: *const mlir.Value, bytes: *const mlir.Value, stream: ?*const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +pub fn memory_set_async( + ctx: *mlir.Context, + ptr: *const mlir.Value, + value_: *const mlir.Value, + bytes: *const mlir.Value, + cuda_result_type: *const mlir.Type, + opts: struct { + stream: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cuda.memory.set_async", .{ - .operands = .{ .flat = if (stream) |value| &.{ ptr, value_, bytes, value } else &.{ ptr, value_, bytes } }, + .operands = .{ .flat = if (opts.stream) |value| &.{ ptr, value_, bytes, value } else &.{ ptr, value_, bytes } }, .results = .{ .flat = &.{cuda_result_type} }, .location = location, }); @@ -1983,7 +2002,7 @@ pub fn result_succeeded(ctx: *mlir.Context, cuda_result: *const mlir.Value, succ } /// `cuda.return`. Result types are explicit; attributes use mlir.Attribute. Not verified on creation: a terminator is only one in its block. -pub fn @"return"(ctx: *mlir.Context, inputs: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { +pub fn return_(ctx: *mlir.Context, inputs: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "cuda.return", .{ .operands = .{ .flat = inputs }, .verify = false, @@ -3301,9 +3320,17 @@ pub fn stream_end_capture(ctx: *mlir.Context, stream: *const mlir.Value, result_ } /// `cuda.stream.synchronize`. Result types are explicit; attributes use mlir.Attribute. -pub fn stream_synchronize(ctx: *mlir.Context, stream: *const mlir.Value, event: ?*const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +pub fn stream_synchronize( + ctx: *mlir.Context, + stream: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + event: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cuda.stream.synchronize", .{ - .operands = .{ .flat = if (event) |value| &.{ stream, value } else &.{stream} }, + .operands = .{ .flat = if (opts.event) |value| &.{ stream, value } else &.{stream} }, .results = .{ .flat = &.{result_type} }, .location = location, }); @@ -3317,234 +3344,3 @@ pub fn tensor_map_encode_tiled(ctx: *mlir.Context, tensorDataType: *const mlir.V .location = location, }); } - -/// Every operation bound here. -pub const operation_names: []const []const u8 = &.{ - "cuda.addressof", - "cuda.binary", - "cuda.cast", - "cuda.compose", - "cuda.decompose", - "cuda.device.sync", - "cuda.driver.event.create", - "cuda.driver.event.destroy", - "cuda.driver.event.elapsed_time", - "cuda.driver.event.query", - "cuda.driver.event.record", - "cuda.driver.event.record_with_flags", - "cuda.driver.event.synchronize", - "cuda.driver.library.get_global", - "cuda.driver.library.get_kernel", - "cuda.driver.library.get_kernel_count", - "cuda.driver.library.get_managed", - "cuda.driver.library.get_module", - "cuda.driver.library.get_unified_function", - "cuda.driver.library.load_data", - "cuda.driver.library.load_from_file", - "cuda.driver.library.unload", - "cuda.driver.tensor_map.encode_im2col", - "cuda.driver.tensor_map.encode_im2col_wide", - "cuda.driver.tensor_map.replace_address", - "cuda.event.create", - "cuda.event.destroy", - "cuda.event.elapsed_time", - "cuda.event.record", - "cuda.func", - "cuda.func.get_name", - "cuda.graph.destroy", - "cuda.graph.instantiate", - "cuda.graph.launch", - "cuda.graph_exec.destroy", - "cuda.kernel", - "cuda.launch", - "cuda.launch_cfg.access_policy_window", - "cuda.launch_cfg.cluster_dim", - "cuda.launch_cfg.cluster_scheduling_policy", - "cuda.launch_cfg.cooperative", - "cuda.launch_cfg.create", - "cuda.launch_cfg.create_from_stream", - "cuda.launch_cfg.device_updatable_kernel_node", - "cuda.launch_cfg.grid_dimensions", - "cuda.launch_cfg.launch_completion_event", - "cuda.launch_cfg.mem_sync_domain", - "cuda.launch_cfg.mem_sync_domain_map", - "cuda.launch_cfg.nvlink_util_centric_scheduling", - "cuda.launch_cfg.preferred_cluster_dim", - "cuda.launch_cfg.print", - "cuda.launch_cfg.priority", - "cuda.launch_cfg.programmatic_event", - "cuda.launch_cfg.programmatic_stream_serialization_allowed", - "cuda.launch_cfg.shared_mem_carveout", - "cuda.launch_cfg.sync_policy", - "cuda.launch_ex", - "cuda.launch_kernel", - "cuda.library.get_kernel", - "cuda.library.load_data", - "cuda.library.unload", - "cuda.memory.alloc", - "cuda.memory.alloc_async", - "cuda.memory.alloc_host", - "cuda.memory.alloc_managed", - "cuda.memory.copy", - "cuda.memory.copy_async", - "cuda.memory.free", - "cuda.memory.free_async", - "cuda.memory.free_host", - "cuda.memory.set", - "cuda.memory.set_async", - "cuda.occupancy_max_active_blocks_per_multiprocessor", - "cuda.profiler.start", - "cuda.profiler.stop", - "cuda.result.assert_success", - "cuda.result.failed", - "cuda.result.print_on_error", - "cuda.result.succeeded", - "cuda.return", - "cuda.return_if_error", - "cuda.runtime.device.properties", - "cuda.runtime.device_prop.get_access_policy_max_window_size", - "cuda.runtime.device_prop.get_async_engine_count", - "cuda.runtime.device_prop.get_can_map_host_memory", - "cuda.runtime.device_prop.get_can_use_host_pointer_for_registered_mem", - "cuda.runtime.device_prop.get_cluster_launch", - "cuda.runtime.device_prop.get_compute_preemption_supported", - "cuda.runtime.device_prop.get_concurrent_kernels", - "cuda.runtime.device_prop.get_concurrent_managed_access", - "cuda.runtime.device_prop.get_cooperative_launch", - "cuda.runtime.device_prop.get_deferred_mapping_cuda_array_supported", - "cuda.runtime.device_prop.get_device_numa_config", - "cuda.runtime.device_prop.get_device_numa_id", - "cuda.runtime.device_prop.get_direct_managed_mem_access_from_host", - "cuda.runtime.device_prop.get_ecc_enabled", - "cuda.runtime.device_prop.get_global_l1cache_supported", - "cuda.runtime.device_prop.get_gpu_direct_rdma_flush_writes_options", - "cuda.runtime.device_prop.get_gpu_direct_rdma_supported", - "cuda.runtime.device_prop.get_gpu_direct_rdma_writes_ordering", - "cuda.runtime.device_prop.get_gpu_pci_device_id", - "cuda.runtime.device_prop.get_gpu_pci_subsystem_id", - "cuda.runtime.device_prop.get_host_native_atomic_supported", - "cuda.runtime.device_prop.get_host_numa_id", - "cuda.runtime.device_prop.get_host_numa_multinode_ipc_supported", - "cuda.runtime.device_prop.get_host_register_read_only_supported", - "cuda.runtime.device_prop.get_host_register_supported", - "cuda.runtime.device_prop.get_integrated", - "cuda.runtime.device_prop.get_ipc_event_supported", - "cuda.runtime.device_prop.get_is_multi_gpu_board", - "cuda.runtime.device_prop.get_l_2cache_size", - "cuda.runtime.device_prop.get_local_l1cache_supported", - "cuda.runtime.device_prop.get_luid", - "cuda.runtime.device_prop.get_luid_device_node_mask", - "cuda.runtime.device_prop.get_major", - "cuda.runtime.device_prop.get_managed_memory", - "cuda.runtime.device_prop.get_max_blocks_per_multi_processor", - "cuda.runtime.device_prop.get_max_grid_size", - "cuda.runtime.device_prop.get_max_surface_1d", - "cuda.runtime.device_prop.get_max_surface_1d_layered", - "cuda.runtime.device_prop.get_max_surface_2d", - "cuda.runtime.device_prop.get_max_surface_2d_layered", - "cuda.runtime.device_prop.get_max_surface_3d", - "cuda.runtime.device_prop.get_max_surface_cubemap", - "cuda.runtime.device_prop.get_max_surface_cubemap_layered", - "cuda.runtime.device_prop.get_max_texture_1d", - "cuda.runtime.device_prop.get_max_texture_1d_layered", - "cuda.runtime.device_prop.get_max_texture_1d_mipmap", - "cuda.runtime.device_prop.get_max_texture_2d", - "cuda.runtime.device_prop.get_max_texture_2d_gather", - "cuda.runtime.device_prop.get_max_texture_2d_layered", - "cuda.runtime.device_prop.get_max_texture_2d_linear", - "cuda.runtime.device_prop.get_max_texture_2d_mipmap", - "cuda.runtime.device_prop.get_max_texture_3d", - "cuda.runtime.device_prop.get_max_texture_3d_alt", - "cuda.runtime.device_prop.get_max_texture_cubemap", - "cuda.runtime.device_prop.get_max_texture_cubemap_layered", - "cuda.runtime.device_prop.get_max_threads_dim", - "cuda.runtime.device_prop.get_max_threads_per_block", - "cuda.runtime.device_prop.get_max_threads_per_multi_processor", - "cuda.runtime.device_prop.get_mem_pitch", - "cuda.runtime.device_prop.get_memory_bus_width", - "cuda.runtime.device_prop.get_memory_pool_supported_handle_types", - "cuda.runtime.device_prop.get_memory_pools_supported", - "cuda.runtime.device_prop.get_minor", - "cuda.runtime.device_prop.get_mps_enabled", - "cuda.runtime.device_prop.get_multi_gpu_board_group_id", - "cuda.runtime.device_prop.get_multi_processor_count", - "cuda.runtime.device_prop.get_name", - "cuda.runtime.device_prop.get_pageable_memory_access", - "cuda.runtime.device_prop.get_pageable_memory_access_uses_host_page_tables", - "cuda.runtime.device_prop.get_pci_bus_id", - "cuda.runtime.device_prop.get_pci_device_id", - "cuda.runtime.device_prop.get_pci_domain_id", - "cuda.runtime.device_prop.get_persisting_l2cache_max_size", - "cuda.runtime.device_prop.get_regs_per_block", - "cuda.runtime.device_prop.get_regs_per_multiprocessor", - "cuda.runtime.device_prop.get_reserved_shared_mem_per_block", - "cuda.runtime.device_prop.get_shared_mem_per_block", - "cuda.runtime.device_prop.get_shared_mem_per_block_optin", - "cuda.runtime.device_prop.get_shared_mem_per_multiprocessor", - "cuda.runtime.device_prop.get_sparse_cuda_array_supported", - "cuda.runtime.device_prop.get_stream_priorities_supported", - "cuda.runtime.device_prop.get_surface_alignment", - "cuda.runtime.device_prop.get_tcc_driver", - "cuda.runtime.device_prop.get_texture_alignment", - "cuda.runtime.device_prop.get_texture_pitch_alignment", - "cuda.runtime.device_prop.get_timeline_semaphore_interop_supported", - "cuda.runtime.device_prop.get_total_const_mem", - "cuda.runtime.device_prop.get_total_global_mem", - "cuda.runtime.device_prop.get_unified_addressing", - "cuda.runtime.device_prop.get_unified_function_pointers", - "cuda.runtime.device_prop.get_uuid", - "cuda.runtime.device_prop.get_warp_size", - "cuda.runtime.dim_3.get_x", - "cuda.runtime.dim_3.get_y", - "cuda.runtime.dim_3.get_z", - "cuda.runtime.event.create_with_flags", - "cuda.runtime.event.query", - "cuda.runtime.event.record_with_flags", - "cuda.runtime.event.synchronize", - "cuda.runtime.extent.get_depth", - "cuda.runtime.extent.get_height", - "cuda.runtime.extent.get_width", - "cuda.runtime.graph.add_kernel_node", - "cuda.runtime.graph.create", - "cuda.runtime.graph.kernel_node_get_attribute", - "cuda.runtime.graph.kernel_node_set_attribute", - "cuda.runtime.kernel_node_params.get_block_dim", - "cuda.runtime.kernel_node_params.get_extra", - "cuda.runtime.kernel_node_params.get_func", - "cuda.runtime.kernel_node_params.get_grid_dim", - "cuda.runtime.kernel_node_params.get_kernel_params", - "cuda.runtime.kernel_node_params.get_shared_mem_bytes", - "cuda.runtime.library.get_global", - "cuda.runtime.library.get_kernel", - "cuda.runtime.library.get_kernel_count", - "cuda.runtime.library.get_managed", - "cuda.runtime.library.get_unified_function", - "cuda.runtime.library.load_data", - "cuda.runtime.library.load_from_file", - "cuda.runtime.library.unload", - "cuda.runtime.memcpy_3d_params.get_dst_array", - "cuda.runtime.memcpy_3d_params.get_dst_pos", - "cuda.runtime.memcpy_3d_params.get_dst_ptr", - "cuda.runtime.memcpy_3d_params.get_extent", - "cuda.runtime.memcpy_3d_params.get_kind", - "cuda.runtime.memcpy_3d_params.get_src_array", - "cuda.runtime.memcpy_3d_params.get_src_pos", - "cuda.runtime.memcpy_3d_params.get_src_ptr", - "cuda.runtime.memory.cpy3d", - "cuda.runtime.memory.cpy3d_async", - "cuda.runtime.pitched_ptr.get_pitch", - "cuda.runtime.pitched_ptr.get_ptr", - "cuda.runtime.pitched_ptr.get_xsize", - "cuda.runtime.pitched_ptr.get_ysize", - "cuda.runtime.pos.get_x", - "cuda.runtime.pos.get_y", - "cuda.runtime.pos.get_z", - "cuda.runtime.uuid.get_bytes", - "cuda.stream.add_callback", - "cuda.stream.begin_capture", - "cuda.stream.create", - "cuda.stream.destroy", - "cuda.stream.end_capture", - "cuda.stream.synchronize", - "cuda.tensor_map_encode_tiled", -}; diff --git a/mlir/dialects/cuda/test.zig b/mlir/dialects/cuda/test.zig new file mode 100644 index 00000000..b17f50cd --- /dev/null +++ b/mlir/dialects/cuda/test.zig @@ -0,0 +1,324 @@ +const std = @import("std"); + +const cuda = @import("mlir/dialects/cuda"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "cuda.addressof", + "cuda.binary", + "cuda.cast", + "cuda.compose", + "cuda.decompose", + "cuda.device.sync", + "cuda.driver.event.create", + "cuda.driver.event.destroy", + "cuda.driver.event.elapsed_time", + "cuda.driver.event.query", + "cuda.driver.event.record", + "cuda.driver.event.record_with_flags", + "cuda.driver.event.synchronize", + "cuda.driver.library.get_global", + "cuda.driver.library.get_kernel", + "cuda.driver.library.get_kernel_count", + "cuda.driver.library.get_managed", + "cuda.driver.library.get_module", + "cuda.driver.library.get_unified_function", + "cuda.driver.library.load_data", + "cuda.driver.library.load_from_file", + "cuda.driver.library.unload", + "cuda.driver.tensor_map.encode_im2col", + "cuda.driver.tensor_map.encode_im2col_wide", + "cuda.driver.tensor_map.replace_address", + "cuda.event.create", + "cuda.event.destroy", + "cuda.event.elapsed_time", + "cuda.event.record", + "cuda.func", + "cuda.func.get_name", + "cuda.graph.destroy", + "cuda.graph.instantiate", + "cuda.graph.launch", + "cuda.graph_exec.destroy", + "cuda.kernel", + "cuda.launch", + "cuda.launch_cfg.access_policy_window", + "cuda.launch_cfg.cluster_dim", + "cuda.launch_cfg.cluster_scheduling_policy", + "cuda.launch_cfg.cooperative", + "cuda.launch_cfg.create", + "cuda.launch_cfg.create_from_stream", + "cuda.launch_cfg.device_updatable_kernel_node", + "cuda.launch_cfg.grid_dimensions", + "cuda.launch_cfg.launch_completion_event", + "cuda.launch_cfg.mem_sync_domain", + "cuda.launch_cfg.mem_sync_domain_map", + "cuda.launch_cfg.nvlink_util_centric_scheduling", + "cuda.launch_cfg.preferred_cluster_dim", + "cuda.launch_cfg.print", + "cuda.launch_cfg.priority", + "cuda.launch_cfg.programmatic_event", + "cuda.launch_cfg.programmatic_stream_serialization_allowed", + "cuda.launch_cfg.shared_mem_carveout", + "cuda.launch_cfg.sync_policy", + "cuda.launch_ex", + "cuda.launch_kernel", + "cuda.library.get_kernel", + "cuda.library.load_data", + "cuda.library.unload", + "cuda.memory.alloc", + "cuda.memory.alloc_async", + "cuda.memory.alloc_host", + "cuda.memory.alloc_managed", + "cuda.memory.copy", + "cuda.memory.copy_async", + "cuda.memory.free", + "cuda.memory.free_async", + "cuda.memory.free_host", + "cuda.memory.set", + "cuda.memory.set_async", + "cuda.occupancy_max_active_blocks_per_multiprocessor", + "cuda.profiler.start", + "cuda.profiler.stop", + "cuda.result.assert_success", + "cuda.result.failed", + "cuda.result.print_on_error", + "cuda.result.succeeded", + "cuda.return", + "cuda.return_if_error", + "cuda.runtime.device.properties", + "cuda.runtime.device_prop.get_access_policy_max_window_size", + "cuda.runtime.device_prop.get_async_engine_count", + "cuda.runtime.device_prop.get_can_map_host_memory", + "cuda.runtime.device_prop.get_can_use_host_pointer_for_registered_mem", + "cuda.runtime.device_prop.get_cluster_launch", + "cuda.runtime.device_prop.get_compute_preemption_supported", + "cuda.runtime.device_prop.get_concurrent_kernels", + "cuda.runtime.device_prop.get_concurrent_managed_access", + "cuda.runtime.device_prop.get_cooperative_launch", + "cuda.runtime.device_prop.get_deferred_mapping_cuda_array_supported", + "cuda.runtime.device_prop.get_device_numa_config", + "cuda.runtime.device_prop.get_device_numa_id", + "cuda.runtime.device_prop.get_direct_managed_mem_access_from_host", + "cuda.runtime.device_prop.get_ecc_enabled", + "cuda.runtime.device_prop.get_global_l1cache_supported", + "cuda.runtime.device_prop.get_gpu_direct_rdma_flush_writes_options", + "cuda.runtime.device_prop.get_gpu_direct_rdma_supported", + "cuda.runtime.device_prop.get_gpu_direct_rdma_writes_ordering", + "cuda.runtime.device_prop.get_gpu_pci_device_id", + "cuda.runtime.device_prop.get_gpu_pci_subsystem_id", + "cuda.runtime.device_prop.get_host_native_atomic_supported", + "cuda.runtime.device_prop.get_host_numa_id", + "cuda.runtime.device_prop.get_host_numa_multinode_ipc_supported", + "cuda.runtime.device_prop.get_host_register_read_only_supported", + "cuda.runtime.device_prop.get_host_register_supported", + "cuda.runtime.device_prop.get_integrated", + "cuda.runtime.device_prop.get_ipc_event_supported", + "cuda.runtime.device_prop.get_is_multi_gpu_board", + "cuda.runtime.device_prop.get_l_2cache_size", + "cuda.runtime.device_prop.get_local_l1cache_supported", + "cuda.runtime.device_prop.get_luid", + "cuda.runtime.device_prop.get_luid_device_node_mask", + "cuda.runtime.device_prop.get_major", + "cuda.runtime.device_prop.get_managed_memory", + "cuda.runtime.device_prop.get_max_blocks_per_multi_processor", + "cuda.runtime.device_prop.get_max_grid_size", + "cuda.runtime.device_prop.get_max_surface_1d", + "cuda.runtime.device_prop.get_max_surface_1d_layered", + "cuda.runtime.device_prop.get_max_surface_2d", + "cuda.runtime.device_prop.get_max_surface_2d_layered", + "cuda.runtime.device_prop.get_max_surface_3d", + "cuda.runtime.device_prop.get_max_surface_cubemap", + "cuda.runtime.device_prop.get_max_surface_cubemap_layered", + "cuda.runtime.device_prop.get_max_texture_1d", + "cuda.runtime.device_prop.get_max_texture_1d_layered", + "cuda.runtime.device_prop.get_max_texture_1d_mipmap", + "cuda.runtime.device_prop.get_max_texture_2d", + "cuda.runtime.device_prop.get_max_texture_2d_gather", + "cuda.runtime.device_prop.get_max_texture_2d_layered", + "cuda.runtime.device_prop.get_max_texture_2d_linear", + "cuda.runtime.device_prop.get_max_texture_2d_mipmap", + "cuda.runtime.device_prop.get_max_texture_3d", + "cuda.runtime.device_prop.get_max_texture_3d_alt", + "cuda.runtime.device_prop.get_max_texture_cubemap", + "cuda.runtime.device_prop.get_max_texture_cubemap_layered", + "cuda.runtime.device_prop.get_max_threads_dim", + "cuda.runtime.device_prop.get_max_threads_per_block", + "cuda.runtime.device_prop.get_max_threads_per_multi_processor", + "cuda.runtime.device_prop.get_mem_pitch", + "cuda.runtime.device_prop.get_memory_bus_width", + "cuda.runtime.device_prop.get_memory_pool_supported_handle_types", + "cuda.runtime.device_prop.get_memory_pools_supported", + "cuda.runtime.device_prop.get_minor", + "cuda.runtime.device_prop.get_mps_enabled", + "cuda.runtime.device_prop.get_multi_gpu_board_group_id", + "cuda.runtime.device_prop.get_multi_processor_count", + "cuda.runtime.device_prop.get_name", + "cuda.runtime.device_prop.get_pageable_memory_access", + "cuda.runtime.device_prop.get_pageable_memory_access_uses_host_page_tables", + "cuda.runtime.device_prop.get_pci_bus_id", + "cuda.runtime.device_prop.get_pci_device_id", + "cuda.runtime.device_prop.get_pci_domain_id", + "cuda.runtime.device_prop.get_persisting_l2cache_max_size", + "cuda.runtime.device_prop.get_regs_per_block", + "cuda.runtime.device_prop.get_regs_per_multiprocessor", + "cuda.runtime.device_prop.get_reserved_shared_mem_per_block", + "cuda.runtime.device_prop.get_shared_mem_per_block", + "cuda.runtime.device_prop.get_shared_mem_per_block_optin", + "cuda.runtime.device_prop.get_shared_mem_per_multiprocessor", + "cuda.runtime.device_prop.get_sparse_cuda_array_supported", + "cuda.runtime.device_prop.get_stream_priorities_supported", + "cuda.runtime.device_prop.get_surface_alignment", + "cuda.runtime.device_prop.get_tcc_driver", + "cuda.runtime.device_prop.get_texture_alignment", + "cuda.runtime.device_prop.get_texture_pitch_alignment", + "cuda.runtime.device_prop.get_timeline_semaphore_interop_supported", + "cuda.runtime.device_prop.get_total_const_mem", + "cuda.runtime.device_prop.get_total_global_mem", + "cuda.runtime.device_prop.get_unified_addressing", + "cuda.runtime.device_prop.get_unified_function_pointers", + "cuda.runtime.device_prop.get_uuid", + "cuda.runtime.device_prop.get_warp_size", + "cuda.runtime.dim_3.get_x", + "cuda.runtime.dim_3.get_y", + "cuda.runtime.dim_3.get_z", + "cuda.runtime.event.create_with_flags", + "cuda.runtime.event.query", + "cuda.runtime.event.record_with_flags", + "cuda.runtime.event.synchronize", + "cuda.runtime.extent.get_depth", + "cuda.runtime.extent.get_height", + "cuda.runtime.extent.get_width", + "cuda.runtime.graph.add_kernel_node", + "cuda.runtime.graph.create", + "cuda.runtime.graph.kernel_node_get_attribute", + "cuda.runtime.graph.kernel_node_set_attribute", + "cuda.runtime.kernel_node_params.get_block_dim", + "cuda.runtime.kernel_node_params.get_extra", + "cuda.runtime.kernel_node_params.get_func", + "cuda.runtime.kernel_node_params.get_grid_dim", + "cuda.runtime.kernel_node_params.get_kernel_params", + "cuda.runtime.kernel_node_params.get_shared_mem_bytes", + "cuda.runtime.library.get_global", + "cuda.runtime.library.get_kernel", + "cuda.runtime.library.get_kernel_count", + "cuda.runtime.library.get_managed", + "cuda.runtime.library.get_unified_function", + "cuda.runtime.library.load_data", + "cuda.runtime.library.load_from_file", + "cuda.runtime.library.unload", + "cuda.runtime.memcpy_3d_params.get_dst_array", + "cuda.runtime.memcpy_3d_params.get_dst_pos", + "cuda.runtime.memcpy_3d_params.get_dst_ptr", + "cuda.runtime.memcpy_3d_params.get_extent", + "cuda.runtime.memcpy_3d_params.get_kind", + "cuda.runtime.memcpy_3d_params.get_src_array", + "cuda.runtime.memcpy_3d_params.get_src_pos", + "cuda.runtime.memcpy_3d_params.get_src_ptr", + "cuda.runtime.memory.cpy3d", + "cuda.runtime.memory.cpy3d_async", + "cuda.runtime.pitched_ptr.get_pitch", + "cuda.runtime.pitched_ptr.get_ptr", + "cuda.runtime.pitched_ptr.get_xsize", + "cuda.runtime.pitched_ptr.get_ysize", + "cuda.runtime.pos.get_x", + "cuda.runtime.pos.get_y", + "cuda.runtime.pos.get_z", + "cuda.runtime.uuid.get_bytes", + "cuda.stream.add_callback", + "cuda.stream.begin_capture", + "cuda.stream.create", + "cuda.stream.destroy", + "cuda.stream.end_capture", + "cuda.stream.synchronize", + "cuda.tensor_map_encode_tiled", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(cuda); + std.testing.refAllDecls(cuda.ops); + inline for (comptime std.meta.declarations(cuda.ops)) |decl| { + const T = @field(cuda.ops, decl); + if (@TypeOf(T) == type) std.testing.refAllDecls(T); + } +} + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + cuda.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +test "every bound operation is registered" { + const ctx = try testContext(); + defer ctx.deinit(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "cuda types and attributes print as the compiler does" { + @setEvalBranchQuota(100_000); + const ctx = try testContext(); + defer ctx.deinit(); + // The compiler's printed forms, which these must print byte for byte. + inline for (.{ + .{ cuda.ops.AssumeKernelAttr, "#cuda.assume_kernel_attr" }, + .{ cuda.ops.ComputeTargetAttr, "#cuda.compute_target" }, + .{ cuda.ops.DevMaxSharedMemoryOptinAttr, "#cuda.dev_max_shared_memory_optin" }, + .{ cuda.ops.ExecutableAttr, "#cuda.executable" }, + .{ cuda.ops.LaunchConfigType, "!cuda.launch_cfg" }, + .{ cuda.ops.StreamType, "!cuda.stream" }, + .{ cuda.ops.ResultType, "!cuda.result" }, + .{ cuda.ops.IntType, "!cuda.int" }, + .{ cuda.ops.RuntimeDim3Type, "!cuda.runtime.dim_3" }, + .{ cuda.ops.RuntimeLaunchAttributeValueSyncPolicyType, "!cuda.runtime.launch_attribute_value.sync_policy" }, + .{ cuda.ops.RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType, "!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference" }, + .{ cuda.ops.RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType, "!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling" }, + .{ cuda.ops.RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType, "!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed" }, + }) |example| { + const T = example[0]; + const is_type = @hasDecl(T, "type_"); + const parsed = if (is_type) try mlir.Type.parse(ctx, example[1]) else try mlir.Attribute.parse(ctx, example[1]); + const value = parsed.isA(T) orelse return error.TestUnexpectedResult; + try std.testing.expect((try rebuildEnums(T, ctx, value)).eql(value)); + var buf: [512]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(example[1], w.buffered()); + } + // The compiler prints these without the space before the dictionary; it + // parses both. + inline for (.{ + .{ cuda.ops.DeviceAttributesAttr, "#cuda" }, + .{ cuda.ops.FuncAttributesAttr, "#cuda" }, + }) |example| { + const parsed = try mlir.Attribute.parse(ctx, example[1]); + const value = parsed.isA(example[0]) orelse return error.TestUnexpectedResult; + try std.testing.expect((try rebuildEnums(example[0], ctx, value)).eql(value)); + } +} + +/// `rebuild` for parameters that are slices of enums. +fn rebuildEnums(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T { + var args: T.InitArgs = undefined; + var buffers: [4][16]u32 = undefined; + inline for (@typeInfo(T.InitArgs).@"struct".field_names, 0..) |field, i| { + const name = [_]u8{comptime std.ascii.toUpper(field[0])} ++ field[1..]; + const noun = [_]u8{comptime std.ascii.toLower(field[0])} ++ field[1..]; + const accessor = if (@hasDecl(T, noun ++ "_")) noun ++ "_" else noun; + const info = @typeInfo(@FieldType(T.InitArgs, field)); + if (info == .pointer and info.pointer.size == .slice and @typeInfo(info.pointer.child) == .@"enum") { + const E = info.pointer.child; + const count = @field(T, "num" ++ name)(value); + const elements: []E = @ptrCast(buffers[i][0..count]); + for (elements, 0..) |*element, pos| element.* = @field(T, noun[0 .. noun.len - 1])(value, pos); + @field(args, field) = elements; + } else { + @field(args, field) = @field(T, accessor)(value); + } + } + return T.get(ctx, args); +} diff --git a/mlir/dialects/cuda_tile/BUILD.bazel b/mlir/dialects/cuda_tile/BUILD.bazel index 48e50fdc..92d622cf 100644 --- a/mlir/dialects/cuda_tile/BUILD.bazel +++ b/mlir/dialects/cuda_tile/BUILD.bazel @@ -3,11 +3,10 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") # The TileView type interface, which NVIDIA's C API does not expose. cc_library( - name = "cuda_tile_capi", - srcs = ["cuda_tile_capi.cc"], - hdrs = ["cuda_tile_capi.h"], - strip_include_prefix = ".", - visibility = ["//visibility:private"], + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], deps = [ "@cuda_tile//:CudaTileDialect", "@llvm-project//mlir:CAPIIR", @@ -18,11 +17,11 @@ zig_library( name = "cuda_tile", import_name = "mlir/dialects/cuda_tile", main = "cuda_tile.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ - ":cuda_tile_capi", + ":capi", "//mlir", - "//mlir:c", "//stdx", "@cuda_tile//:cuda_tile_capi", ], @@ -30,5 +29,10 @@ zig_library( zig_test( name = "test", - deps = [":cuda_tile"], + main = "test.zig", + deps = [ + ":capi", + ":cuda_tile", + "//mlir", + ], ) diff --git a/mlir/dialects/cuda_tile/cuda_tile_capi.cc b/mlir/dialects/cuda_tile/capi.cc similarity index 85% rename from mlir/dialects/cuda_tile/cuda_tile_capi.cc rename to mlir/dialects/cuda_tile/capi.cc index de37f42f..879e3757 100644 --- a/mlir/dialects/cuda_tile/cuda_tile_capi.cc +++ b/mlir/dialects/cuda_tile/capi.cc @@ -1,30 +1,30 @@ -#include "cuda_tile_capi.h" +//===- capi.cc - C API for the cuda_tile dialect --------------------------===// -#include +#include "capi.h" #include "cuda_tile/Dialect/CudaTile/IR/Attributes.h" #include "cuda_tile/Dialect/CudaTile/IR/Dialect.h" #include "cuda_tile/Dialect/CudaTile/IR/Types.h" + #include "mlir/CAPI/IR.h" #include "mlir/IR/BuiltinAttributes.h" #include "mlir/IR/Diagnostics.h" +#include + using namespace mlir::cuda_tile; -namespace { - -mlir::MLIRContext *loadCudaTile(MlirContext context) { +static mlir::MLIRContext *loadCudaTile(MlirContext context) { mlir::MLIRContext *ctx = unwrap(context); - if (ctx) ctx->getOrLoadDialect(); + if (ctx) + ctx->getOrLoadDialect(); return ctx; } -std::optional optional(bool has, int64_t value) { +static std::optional optional(bool has, int64_t value) { return has ? std::optional(value) : std::nullopt; } -} // namespace - //===----------------------------------------------------------------------===// // TileView //===----------------------------------------------------------------------===// @@ -50,20 +50,20 @@ bool mlirTypeIsACudaTileGatherScatterView(MlirType type) { return llvm::isa(unwrap(type)); } -MlirType mlirCudaTileGatherScatterViewTypeGet(MlirContext context, - intptr_t rank, - const int32_t *tileShape, - MlirType tensorView, - uint32_t sparseDim, - MlirAttribute paddingValue) { +MlirType mlirCudaTileGatherScatterViewTypeGet( + MlirContext context, intptr_t rank, const int32_t *tileShape, + MlirType tensorView, uint32_t sparseDim, MlirAttribute paddingValue) { mlir::MLIRContext *ctx = loadCudaTile(context); - if (!ctx) return {nullptr}; + if (!ctx) + return {nullptr}; auto view = llvm::dyn_cast_or_null(unwrap(tensorView)); - if (!view) return {nullptr}; + if (!view) + return {nullptr}; PaddingValueAttr padding; if (!mlirAttributeIsNull(paddingValue)) { padding = llvm::dyn_cast(unwrap(paddingValue)); - if (!padding) return {nullptr}; + if (!padding) + return {nullptr}; } auto shape = mlir::DenseI32ArrayAttr::get( ctx, llvm::ArrayRef(tileShape, rank)); @@ -85,7 +85,8 @@ uint32_t mlirCudaTileGatherScatterViewTypeGetSparseDim(MlirType type) { } MlirAttribute mlirCudaTileGatherScatterViewTypeGetPaddingValue(MlirType type) { - return wrap(llvm::cast(unwrap(type)).getPaddingValue()); + return wrap( + llvm::cast(unwrap(type)).getPaddingValue()); } //===----------------------------------------------------------------------===// @@ -95,9 +96,11 @@ MlirAttribute mlirCudaTileGatherScatterViewTypeGetPaddingValue(MlirType type) { MlirAttribute mlirCudaTileOptimizationHintsAttrGet(MlirContext context, MlirAttribute value) { mlir::MLIRContext *ctx = loadCudaTile(context); - if (!ctx) return {nullptr}; + if (!ctx) + return {nullptr}; auto dict = llvm::dyn_cast_or_null(unwrap(value)); - if (!dict) return {nullptr}; + if (!dict) + return {nullptr}; return wrap(OptimizationHintsAttr::getChecked( [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, dict)); } @@ -119,7 +122,8 @@ MlirAttribute mlirCudaTileDivByAttrGet(MlirContext context, uint64_t divisor, bool hasAlong, int64_t along) { mlir::MLIRContext *ctx = loadCudaTile(context); // The printer writes `every` and `along` together. - if (!ctx || hasEvery != hasAlong) return {nullptr}; + if (!ctx || hasEvery != hasAlong) + return {nullptr}; return wrap(DivByAttr::get(ctx, divisor, optional(hasEvery, every), optional(hasAlong, along))); } @@ -156,10 +160,11 @@ MlirAttribute mlirCudaTileSameElementsAttrGet(MlirContext context, intptr_t rank, const int64_t *values) { mlir::MLIRContext *ctx = loadCudaTile(context); - if (!ctx) return {nullptr}; + if (!ctx) + return {nullptr}; return wrap(SameElementsAttr::get( - ctx, mlir::DenseI64ArrayAttr::get(ctx, - llvm::ArrayRef(values, rank)))); + ctx, mlir::DenseI64ArrayAttr::get( + ctx, llvm::ArrayRef(values, rank)))); } MlirAttribute mlirCudaTileSameElementsAttrGetValues(MlirAttribute attr) { @@ -177,7 +182,8 @@ bool mlirAttributeIsACudaTileBounded(MlirAttribute attr) { MlirAttribute mlirCudaTileBoundedAttrGet(MlirContext context, bool hasLb, int64_t lb, bool hasUb, int64_t ub) { mlir::MLIRContext *ctx = loadCudaTile(context); - if (!ctx) return {nullptr}; + if (!ctx) + return {nullptr}; return wrap(BoundedAttr::get(ctx, optional(hasLb, lb), optional(hasUb, ub))); } diff --git a/mlir/dialects/cuda_tile/capi.h b/mlir/dialects/cuda_tile/capi.h new file mode 100644 index 00000000..6f299c3f --- /dev/null +++ b/mlir/dialects/cuda_tile/capi.h @@ -0,0 +1,84 @@ +//===- capi.h - C API for the cuda_tile dialect -------------------*- C -*-===// +// +// C entry points for what NVIDIA's cuda_tile C API leaves out: the +// `TileView` type interface, shared by the partition, strided and +// gather/scatter views, the gather/scatter view type, the general +// `optimization_hints` constructor and the `assume` predicates. + +#ifndef MLIR_DIALECTS_CUDA_TILE_CAPI_H +#define MLIR_DIALECTS_CUDA_TILE_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Constructors return null for invalid parameters; optional attribute +// parameters take null handles, and std::optional integers a `has` flag. +// Getters require a value accepted by the matching IsA. + +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaTileTileView(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaTileTileViewTypeGetViewTileType(MlirType type); +MLIR_CAPI_EXPORTED intptr_t +mlirCudaTileTileViewTypeGetViewIndexRank(MlirType type); + +// `!cuda_tile.gather_scatter_view`. `paddingValue` is a +// `#cuda_tile.padding_value` attribute, or null for none. +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaTileGatherScatterView(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaTileGatherScatterViewTypeGet( + MlirContext context, intptr_t rank, const int32_t *tileShape, + MlirType tensorView, uint32_t sparseDim, MlirAttribute paddingValue); +// A DenseI32ArrayAttr. +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaTileGatherScatterViewTypeGetTileShape(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaTileGatherScatterViewTypeGetTensorView(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCudaTileGatherScatterViewTypeGetSparseDim(MlirType type); +// Null when the view has no padding value. +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaTileGatherScatterViewTypeGetPaddingValue(MlirType type); + +// `#cuda_tile.optimization_hints` over its dictionary of per-architecture +// dictionaries (IsA: NVIDIA's mlirCudaTileAttributeIsAOptimizationHintsAttr). +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaTileOptimizationHintsAttrGet(MlirContext context, MlirAttribute value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaTileOptimizationHintsAttrGetValue(MlirAttribute attr); + +// `#cuda_tile.div_by`. `every` and `along` are both present or both absent. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaTileDivBy(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaTileDivByAttrGet(MlirContext context, uint64_t divisor, bool hasEvery, + int64_t every, bool hasAlong, int64_t along); +MLIR_CAPI_EXPORTED uint64_t mlirCudaTileDivByAttrGetDivisor(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirCudaTileDivByAttrHasEvery(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirCudaTileDivByAttrGetEvery(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirCudaTileDivByAttrHasAlong(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirCudaTileDivByAttrGetAlong(MlirAttribute attr); + +// `#cuda_tile.same_elements`. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACudaTileSameElements(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCudaTileSameElementsAttrGet( + MlirContext context, intptr_t rank, const int64_t *values); +// A DenseI64ArrayAttr. +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaTileSameElementsAttrGetValues(MlirAttribute attr); + +// `#cuda_tile.bounded`, each bound optional. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaTileBounded(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCudaTileBoundedAttrGet( + MlirContext context, bool hasLb, int64_t lb, bool hasUb, int64_t ub); +MLIR_CAPI_EXPORTED bool mlirCudaTileBoundedAttrHasLb(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirCudaTileBoundedAttrGetLb(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirCudaTileBoundedAttrHasUb(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirCudaTileBoundedAttrGetUb(MlirAttribute attr); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_CUDA_TILE_CAPI_H diff --git a/mlir/dialects/cuda_tile/cuda_tile.zig b/mlir/dialects/cuda_tile/cuda_tile.zig index d4975fd4..ddf0a0d5 100644 --- a/mlir/dialects/cuda_tile/cuda_tile.zig +++ b/mlir/dialects/cuda_tile/cuda_tile.zig @@ -1,6 +1,6 @@ //! Zig bindings for NVIDIA's `cuda_tile` MLIR dialect (CUDA Tile IR): one //! function per op in `Ops.td` order. Types and attributes go through the -//! dialect's C API, completed by `cuda_tile_capi.h` where it has no getter. +//! dialect's C API, completed by `capi.h` where it has no getter. const std = @import("std"); @@ -8,7 +8,11 @@ const c = @import("c"); const mlir = @import("mlir"); const stdx = @import("stdx"); -pub const dialect_namespace = "cuda_tile"; +const dialect_namespace = "cuda_tile"; + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("cuda_tile"); +} fn stringRef(s: []const u8) c.MlirStringRef { return .{ .data = s.ptr, .length = s.len }; @@ -41,6 +45,10 @@ pub const PointerType = opaque { pub fn pointee(self: *const PointerType) *const mlir.Type { return typeFromC(c.mlirCudaTilePointerTypeGetPointeeType(self.ptr())); } + + pub fn type_(self: *const PointerType) *const mlir.Type { + return @ptrCast(self); + } }; pub fn pointerType(ctx: *mlir.Context, pointee_type: *const mlir.Type) *const mlir.Type { @@ -58,8 +66,13 @@ pub const TileType = opaque { pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; - pub fn get(ctx: *mlir.Context, shape: []const i64, elem: *const mlir.Type) *const TileType { - return @ptrCast(typeFromC(c.mlirCudaTileTileTypeGet(ctx.ptr(), @intCast(shape.len), shape.ptr, elem.ptr()))); + pub const InitArgs = struct { + shape: []const i64, + elem: *const mlir.Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const TileType { + return @ptrCast(typeFromC(c.mlirCudaTileTileTypeGet(ctx.ptr(), @intCast(args.shape.len), args.shape.ptr, args.elem.ptr()))); } pub fn elementType(self: *const TileType) *const mlir.Type { @@ -73,10 +86,14 @@ pub const TileType = opaque { pub fn dimension(self: *const TileType, i: usize) i64 { return c.mlirCudaTileTileTypeGetDimSize(self.ptr(), @intCast(i)); } + + pub fn type_(self: *const TileType) *const mlir.Type { + return @ptrCast(self); + } }; pub fn tileType(ctx: *mlir.Context, shape: []const i64, elem: *const mlir.Type) *const mlir.Type { - return @ptrCast(TileType.get(ctx, shape, elem)); + return @ptrCast(TileType.get(ctx, .{ .shape = shape, .elem = elem })); } /// `!cuda_tile.token` — the ordering token every `*_tko` op produces. @@ -92,6 +109,10 @@ pub const TokenType = opaque { pub fn get(ctx: *mlir.Context) *const TokenType { return @ptrCast(typeFromC(c.mlirCudaTileTokenTypeGet(ctx.ptr()))); } + + pub fn type_(self: *const TokenType) *const mlir.Type { + return @ptrCast(self); + } }; pub fn tokenType(ctx: *mlir.Context) *const mlir.Type { @@ -113,10 +134,18 @@ pub const TensorViewType = opaque { return c.mlirCudaTileTensorViewTypeGetDynamicSize(); } - pub fn get(ctx: *mlir.Context, elem: *const mlir.Type, shape: []const i64, strides: []const i64) *const TensorViewType { + pub const InitArgs = struct { + elem: *const mlir.Type, + shape: []const i64, + strides: []const i64, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const TensorViewType { + const shape = args.shape; + const strides = args.strides; return @ptrCast(typeFromC(c.mlirCudaTileTensorViewTypeGet( ctx.ptr(), - elem.ptr(), + args.elem.ptr(), @intCast(shape.len), shape.ptr, @intCast(strides.len), @@ -139,10 +168,14 @@ pub const TensorViewType = opaque { pub fn stride(self: *const TensorViewType, i: usize) i64 { return c.mlirCudaTileTensorViewTypeGetStride(self.ptr(), @intCast(i)); } + + pub fn type_(self: *const TensorViewType) *const mlir.Type { + return @ptrCast(self); + } }; pub fn tensorViewType(ctx: *mlir.Context, elem: *const mlir.Type, shape: []const i64, strides: []const i64) *const mlir.Type { - return @ptrCast(TensorViewType.get(ctx, elem, shape, strides)); + return @ptrCast(TensorViewType.get(ctx, .{ .elem = elem, .shape = shape, .strides = strides })); } const DimMap = stdx.BoundedArray(i32, mlir.ShapedType.MAX_RANK); @@ -171,13 +204,17 @@ pub const PartitionViewType = opaque { /// An empty `dim_map` is the identity; the type requires one entry per /// tile dimension. - pub fn get( - ctx: *mlir.Context, + pub const InitArgs = struct { tile_shape: []const i32, tensor_view: *const mlir.Type, - dim_map: []const i32, - padding: ?PaddingValue, - ) *const PartitionViewType { + dim_map: []const i32 = &.{}, + padding: ?PaddingValue = null, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const PartitionViewType { + const tile_shape = args.tile_shape; + const tensor_view = args.tensor_view; + const dim_map = args.dim_map; const tile_attr: *const mlir.Attribute = .denseArray(ctx, .i32, tile_shape); const map = identityDimMap(dim_map, tile_shape.len); return @ptrCast(typeFromC(c.mlirCudaTilePartitionViewTypeGet( @@ -186,7 +223,7 @@ pub const PartitionViewType = opaque { tensor_view.ptr(), @intCast(map.len), map.constSlice().ptr, - if (padding) |p| p.attribute(ctx).ptr() else null_attr, + if (args.padding) |p| p.attribute(ctx).ptr() else null_attr, ))); } @@ -201,6 +238,10 @@ pub const PartitionViewType = opaque { pub fn viewIndexRank(self: *const PartitionViewType) usize { return @intCast(c.mlirCudaTilePartitionViewTypeGetViewIndexRank(self.ptr())); } + + pub fn type_(self: *const PartitionViewType) *const mlir.Type { + return @ptrCast(self); + } }; /// `!cuda_tile.strided_view` (13.3). @@ -213,14 +254,19 @@ pub const StridedViewType = opaque { pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; - pub fn get( - ctx: *mlir.Context, + pub const InitArgs = struct { tile_shape: []const i32, traversal_strides: []const i32, tensor_view: *const mlir.Type, - dim_map: []const i32, - padding: ?PaddingValue, - ) *const StridedViewType { + dim_map: []const i32 = &.{}, + padding: ?PaddingValue = null, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const StridedViewType { + const tile_shape = args.tile_shape; + const traversal_strides = args.traversal_strides; + const tensor_view = args.tensor_view; + const dim_map = args.dim_map; const tile_attr: *const mlir.Attribute = .denseArray(ctx, .i32, tile_shape); const strides_attr: *const mlir.Attribute = .denseArray(ctx, .i32, traversal_strides); const map = identityDimMap(dim_map, tile_shape.len); @@ -231,7 +277,7 @@ pub const StridedViewType = opaque { tensor_view.ptr(), @intCast(map.len), map.constSlice().ptr, - if (padding) |p| p.attribute(ctx).ptr() else null_attr, + if (args.padding) |p| p.attribute(ctx).ptr() else null_attr, ))); } @@ -242,6 +288,10 @@ pub const StridedViewType = opaque { pub fn viewIndexRank(self: *const StridedViewType) usize { return @intCast(c.mlirCudaTileStridedViewTypeGetViewIndexRank(self.ptr())); } + + pub fn type_(self: *const StridedViewType) *const mlir.Type { + return @ptrCast(self); + } }; /// `!cuda_tile.gather_scatter_view, sparse_dim=N>` @@ -252,11 +302,9 @@ pub const GatherScatterViewType = opaque { pub const isAFn = c.mlirTypeIsACudaTileGatherScatterView; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "gather_scatter_view"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -277,16 +325,16 @@ pub const GatherScatterViewType = opaque { ); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getTileShape(self: *const Self) *const mlir.DenseArrayAttribute(.i32) { + pub fn tileShape(self: *const Self) *const mlir.DenseArrayAttribute(.i32) { return @ptrCast(attrFromC(c.mlirCudaTileGatherScatterViewTypeGetTileShape(self.ptr()))); } - pub fn getTensorView(self: *const Self) *const TensorViewType { + pub fn tensorView(self: *const Self) *const TensorViewType { return @ptrCast(typeFromC(c.mlirCudaTileGatherScatterViewTypeGetTensorView(self.ptr()))); } - pub fn getSparseDim(self: *const Self) u32 { + pub fn sparseDim(self: *const Self) u32 { return c.mlirCudaTileGatherScatterViewTypeGetSparseDim(self.ptr()); } - pub fn getPaddingValue(self: *const Self) ?PaddingValue { + pub fn paddingValue(self: *const Self) ?PaddingValue { const attr = c.mlirCudaTileGatherScatterViewTypeGetPaddingValue(self.ptr()); if (attr.ptr == null) return null; const name = c.mlirCudaTilePaddingValueAttrGetValue(attr); @@ -299,7 +347,9 @@ pub fn gatherScatterViewType( tile_shape: []const i64, tensor_view: *const mlir.Type, sparse_dim: u32, - padding: ?PaddingValue, + opts: struct { + padding: ?PaddingValue = null, + }, ) *const mlir.Type { var shape: stdx.BoundedArray(i32, mlir.ShapedType.MAX_RANK) = .empty; for (tile_shape) |d| shape.appendAssumeCapacity(@intCast(d)); @@ -307,13 +357,13 @@ pub fn gatherScatterViewType( .tileShape = shape.constSlice(), .tensorView = tensor_view.isA(TensorViewType) orelse std.debug.panic("cuda_tile: {f} is not a tensor_view", .{tensor_view}), .sparseDim = sparse_dim, - .paddingValue = padding, + .paddingValue = opts.padding, }) catch std.debug.panic("cuda_tile: invalid gather_scatter_view over {f}", .{tensor_view}); return view.type_(); } /// The `tile<...>` type a load through `view` yields, for any view kind — -/// through the dialect's `TileView` type interface (`cuda_tile_capi.cc`), so +/// through the dialect's `TileView` type interface (`capi.cc`), so /// the gather/scatter view, which has no getters in NVIDIA's C API, works too. pub fn tileTypeOfView(view: *const mlir.Type) *const mlir.Type { if (!c.mlirTypeIsACudaTileTileView(view.ptr())) std.debug.panic("cuda_tile: {f} is not a tiled view", .{view}); @@ -483,11 +533,9 @@ pub const OptimizationHintsAttr = opaque { pub const isAFn = c.mlirCudaTileAttributeIsAOptimizationHintsAttr; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "optimization_hints"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -498,7 +546,7 @@ pub const OptimizationHintsAttr = opaque { const result = c.mlirCudaTileOptimizationHintsAttrGet(ctx.ptr(), args.value.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) *const mlir.DictionaryAttribute { + pub fn value(self: *const Self) *const mlir.DictionaryAttribute { return @ptrCast(attrFromC(c.mlirCudaTileOptimizationHintsAttrGetValue(self.ptr()))); } }; @@ -523,7 +571,7 @@ fn hintDictionary(ctx: *mlir.Context, hint: anytype) *const mlir.Attribute { /// `#cuda_tile.optimization_hints` from a slice of `EntryHint`, /// `LoadStoreHint` or `PtrLoadStoreHint`. -pub fn optimizationHints(ctx: *mlir.Context, hints: anytype) *const mlir.Attribute { +pub fn optimizationHintsAttr(ctx: *mlir.Context, hints: anytype) *const mlir.Attribute { var entries: stdx.BoundedArray(mlir.NamedAttribute, std.meta.fieldNames(Arch).len) = .empty; for (hints) |h| entries.appendAssumeCapacity(.named(ctx, @tagName(h.arch), hintDictionary(ctx, h))); const value: *const mlir.DictionaryAttribute = @ptrCast(mlir.Attribute.dict(ctx, entries.constSlice())); @@ -542,11 +590,9 @@ pub const DivByAttr = opaque { pub const isAFn = c.mlirAttributeIsACudaTileDivBy; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "div_by"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -566,23 +612,30 @@ pub const DivByAttr = opaque { ); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getDivisor(self: *const Self) u64 { + pub fn divisor(self: *const Self) u64 { return c.mlirCudaTileDivByAttrGetDivisor(self.ptr()); } - pub fn getEvery(self: *const Self) ?i64 { + pub fn every(self: *const Self) ?i64 { return if (c.mlirCudaTileDivByAttrHasEvery(self.ptr())) c.mlirCudaTileDivByAttrGetEvery(self.ptr()) else null; } - pub fn getAlong(self: *const Self) ?i64 { + pub fn along(self: *const Self) ?i64 { return if (c.mlirCudaTileDivByAttrHasAlong(self.ptr())) c.mlirCudaTileDivByAttrGetAlong(self.ptr()) else null; } }; /// `along` defaults to 0 when only `every` is given. -pub fn divBy(ctx: *mlir.Context, divisor: u64, every: ?i64, along: ?i64) *const mlir.Attribute { +pub fn divByAttr( + ctx: *mlir.Context, + divisor: u64, + opts: struct { + every: ?i64 = null, + along: ?i64 = null, + }, +) *const mlir.Attribute { const attr = DivByAttr.get(ctx, .{ .divisor = divisor, - .every = every, - .along = if (every != null) along orelse 0 else null, + .every = opts.every, + .along = if (opts.every != null) opts.along orelse 0 else null, }) catch unreachable; return attr.attribute(); } @@ -595,11 +648,9 @@ pub const SameElementsAttr = opaque { pub const isAFn = c.mlirAttributeIsACudaTileSameElements; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "same_elements"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -610,12 +661,12 @@ pub const SameElementsAttr = opaque { const result = c.mlirCudaTileSameElementsAttrGet(ctx.ptr(), @intCast(args.values.len), args.values.ptr); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValues(self: *const Self) *const mlir.DenseArrayAttribute(.i64) { + pub fn values(self: *const Self) *const mlir.DenseArrayAttribute(.i64) { return @ptrCast(attrFromC(c.mlirCudaTileSameElementsAttrGetValues(self.ptr()))); } }; -pub fn sameElements(ctx: *mlir.Context, values: []const i64) *const mlir.Attribute { +pub fn sameElementsAttr(ctx: *mlir.Context, values: []const i64) *const mlir.Attribute { const attr = SameElementsAttr.get(ctx, .{ .values = values }) catch unreachable; return attr.attribute(); } @@ -627,11 +678,9 @@ pub const BoundedAttr = opaque { pub const isAFn = c.mlirAttributeIsACudaTileBounded; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "bounded"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -643,16 +692,22 @@ pub const BoundedAttr = opaque { const result = c.mlirCudaTileBoundedAttrGet(ctx.ptr(), args.lb != null, args.lb orelse 0, args.ub != null, args.ub orelse 0); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getLb(self: *const Self) ?i64 { + pub fn lb(self: *const Self) ?i64 { return if (c.mlirCudaTileBoundedAttrHasLb(self.ptr())) c.mlirCudaTileBoundedAttrGetLb(self.ptr()) else null; } - pub fn getUb(self: *const Self) ?i64 { + pub fn ub(self: *const Self) ?i64 { return if (c.mlirCudaTileBoundedAttrHasUb(self.ptr())) c.mlirCudaTileBoundedAttrGetUb(self.ptr()) else null; } }; -pub fn bounded(ctx: *mlir.Context, lb: ?i64, ub: ?i64) *const mlir.Attribute { - const attr = BoundedAttr.get(ctx, .{ .lb = lb, .ub = ub }) catch unreachable; +pub fn boundedAttr( + ctx: *mlir.Context, + opts: struct { + lb: ?i64 = null, + ub: ?i64 = null, + }, +) *const mlir.Attribute { + const attr = BoundedAttr.get(ctx, .{ .lb = opts.lb, .ub = opts.ub }) catch unreachable; return attr.attribute(); } @@ -662,11 +717,11 @@ pub fn splatAttr(tile_ty: *const mlir.Type, elem_attr: *const mlir.Attribute) *c return attrFromC(c.mlirDenseElementsAttrSplatGet(tile_ty.ptr(), elem_attr.ptr())); } -pub fn intElem(elem_ty: *const mlir.Type, value: i64) *const mlir.Attribute { +pub fn intElemAttr(elem_ty: *const mlir.Type, value: i64) *const mlir.Attribute { return attrFromC(c.mlirIntegerAttrGet(elem_ty.ptr(), value)); } -pub fn floatElem(ctx: *mlir.Context, elem_ty: *const mlir.Type, value: f64) *const mlir.Attribute { +pub fn floatElemAttr(ctx: *mlir.Context, elem_ty: *const mlir.Type, value: f64) *const mlir.Attribute { return attrFromC(c.mlirFloatAttrDoubleGet(ctx.ptr(), elem_ty.ptr(), value)); } @@ -806,9 +861,16 @@ pub fn atan2(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, loc } /// `rounding` null keeps the op's default (13.3 attribute). -pub fn exp(ctx: *mlir.Context, source: *const mlir.Value, rounding: ?RoundingMode, location: *const mlir.Location) *mlir.Operation { +pub fn exp( + ctx: *mlir.Context, + source: *const mlir.Value, + opts: struct { + rounding: ?RoundingMode = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attrs: Attrs = .empty; - if (rounding) |r| attrs.appendAssumeCapacity(.named(ctx, "rounding_mode", r.attribute(ctx))); + if (opts.rounding) |r| attrs.appendAssumeCapacity(.named(ctx, "rounding_mode", r.attribute(ctx))); return unary(ctx, "exp", source, attrs, location); } @@ -855,9 +917,16 @@ pub fn cosh(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir /// `rounding` null keeps the op's default (13.2 attribute). Only `approx` /// and `full` are legal; the upstream verifier reports the others without /// failing, so they are refused here. -pub fn tanh(ctx: *mlir.Context, source: *const mlir.Value, rounding: ?RoundingMode, location: *const mlir.Location) *mlir.Operation { +pub fn tanh( + ctx: *mlir.Context, + source: *const mlir.Value, + opts: struct { + rounding: ?RoundingMode = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attrs: Attrs = .empty; - if (rounding) |r| { + if (opts.rounding) |r| { switch (r) { .approx, .full => {}, else => std.debug.panic("cuda_tile.tanh: rounding mode {s} is not allowed (only approx or full)", .{@tagName(r)}), @@ -928,10 +997,19 @@ pub fn mulhii(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, lo } /// `rounding` null keeps the op's default (truncation). -pub fn divi(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, signedness: Signedness, rounding: ?RoundingMode, location: *const mlir.Location) *mlir.Operation { +pub fn divi( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + signedness: Signedness, + opts: struct { + rounding: ?RoundingMode = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attrs: Attrs = .empty; attrs.appendAssumeCapacity(.named(ctx, "signedness", signedness.attribute(ctx))); - if (rounding) |r| attrs.appendAssumeCapacity(.named(ctx, "rounding", r.attribute(ctx))); + if (opts.rounding) |r| attrs.appendAssumeCapacity(.named(ctx, "rounding", r.attribute(ctx))); return binary(ctx, "divi", lhs, rhs, attrs, location); } @@ -1012,9 +1090,17 @@ pub fn trunci(ctx: *mlir.Context, from: *const mlir.Value, to: *const mlir.Type, } /// `rounding` null keeps the op's default (nearest_even). -pub fn ftof(ctx: *mlir.Context, from: *const mlir.Value, to: *const mlir.Type, rounding: ?RoundingMode, location: *const mlir.Location) *mlir.Operation { +pub fn ftof( + ctx: *mlir.Context, + from: *const mlir.Value, + to: *const mlir.Type, + opts: struct { + rounding: ?RoundingMode = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attrs: Attrs = .empty; - if (rounding) |r| attrs.appendAssumeCapacity(.named(ctx, "rounding_mode", r.attribute(ctx))); + if (opts.rounding) |r| attrs.appendAssumeCapacity(.named(ctx, "rounding_mode", r.attribute(ctx))); return convert(ctx, "ftof", from, to, attrs, location); } @@ -1243,7 +1329,16 @@ pub fn for_(ctx: *mlir.Context, lower: *const mlir.Value, upper: *const mlir.Val /// `if %cond -> (types) { ... } else { ... }`. Branches end with `yield` /// (or `continue`/`break`/`return` when they leave the enclosing scope). -pub fn if_(ctx: *mlir.Context, cond: *const mlir.Value, result_types: []const *const mlir.Type, then_block: *mlir.Block, else_block: ?*mlir.Block, location: *const mlir.Location) *mlir.Operation { +pub fn if_( + ctx: *mlir.Context, + cond: *const mlir.Value, + result_types: []const *const mlir.Type, + then_block: *mlir.Block, + opts: struct { + else_block: ?*mlir.Block = null, + }, + location: *const mlir.Location, +) *mlir.Operation { _ = ctx; var state: mlir.OperationState = .init(opName("if"), location); state.addOperands(&.{cond}); @@ -1251,7 +1346,7 @@ pub fn if_(ctx: *mlir.Context, cond: *const mlir.Value, result_types: []const *c const then_region = mlir.Region.init(); then_region.appendOwnedBlock(then_block); const else_region = mlir.Region.init(); - if (else_block) |b| else_region.appendOwnedBlock(b); + if (opts.else_block) |b| else_region.appendOwnedBlock(b); state.addOwnedRegions(&.{ then_region, else_region }); return mlir.Operation.init(&state) catch @panic("Failed to create cuda_tile.if"); } @@ -1626,8 +1721,16 @@ pub fn join_tokens(ctx: *mlir.Context, tokens: []const *const mlir.Value, locati } /// Device-side printf. One result: the token. -pub fn print_tko(ctx: *mlir.Context, str: []const u8, args: []const *const mlir.Value, token: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { - const tok = optional(token); +pub fn print_tko( + ctx: *mlir.Context, + str: []const u8, + args: []const *const mlir.Value, + opts: struct { + token: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + const tok = optional(opts.token); var segs: Segments = .{}; segs.add(args); segs.add(tok.slice()); @@ -1642,270 +1745,3 @@ pub fn print_tko(ctx: *mlir.Context, str: []const u8, args: []const *const mlir. // ============================================================================= // Tests // ============================================================================= - -test { - std.testing.refAllDecls(@This()); -} - -fn testContext() !*mlir.Context { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - mlir.DialectHandle.fromString(dialect_namespace).insertDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - return ctx; -} - -test "dialect registers and types round-trip" { - const ctx = try testContext(); - defer ctx.deinit(); - - try std.testing.expect(ctx.isRegisteredOperation("cuda_tile.entry")); - try std.testing.expect(ctx.isRegisteredOperation("cuda_tile.load_view_tko")); - try std.testing.expect(ctx.isRegisteredOperation("cuda_tile.mmaf")); - - const f32_ty = mlir.Type.float(ctx, .f32); - const tile = tileType(ctx, &.{ 64, 64 }, f32_ty); - try std.testing.expect(tile.eql(try mlir.Type.parse(ctx, "!cuda_tile.tile<64x64xf32>"))); - try std.testing.expectEqual(@as(usize, 2), tile.isA(TileType).?.rank()); - try std.testing.expectEqual(@as(i64, 64), tile.isA(TileType).?.dimension(1)); - - const ptr = pointerType(ctx, f32_ty); - try std.testing.expect(ptr.eql(try mlir.Type.parse(ctx, "!cuda_tile.ptr"))); - try std.testing.expect(ptr.isA(PointerType).?.pointee().eql(f32_ty)); - - const dyn = TensorViewType.dynamic(); - const tv = tensorViewType(ctx, f32_ty, &.{ dyn, 32 }, &.{ dyn, 1 }); - try std.testing.expect(tv.eql(try mlir.Type.parse(ctx, "!cuda_tile.tensor_view"))); - - const static_tv = tensorViewType(ctx, f32_ty, &.{16}, &.{1}); - const pv: *const mlir.Type = @ptrCast(PartitionViewType.get(ctx, &.{2}, static_tv, &.{}, .zero)); - try std.testing.expect(pv.eql(try mlir.Type.parse(ctx, "!cuda_tile.partition_view>"))); - try std.testing.expect(tileTypeOfView(pv).eql(tileType(ctx, &.{2}, f32_ty))); - try std.testing.expectEqual(@as(usize, 1), indexRankOfView(pv)); - - try std.testing.expect(tokenType(ctx).eql(try mlir.Type.parse(ctx, "!cuda_tile.token"))); - - const gather_tv = tensorViewType(ctx, f32_ty, &.{ 64, 32 }, &.{ 32, 1 }); - const gsv = gatherScatterViewType(ctx, &.{ 8, 16 }, gather_tv, 0, .zero); - try std.testing.expect(gsv.eql(try mlir.Type.parse(ctx, "!cuda_tile.gather_scatter_view, sparse_dim=0>"))); - const typed_gsv = gsv.isA(GatherScatterViewType).?; - const gsv_tile = typed_gsv.getTileShape(); - var gsv_shape: [2]i32 = undefined; - try std.testing.expectEqual(gsv_shape.len, gsv_tile.numElements()); - for (&gsv_shape, 0..) |*d, i| d.* = gsv_tile.element(i); - const rebuilt_gsv = try GatherScatterViewType.get(ctx, .{ - .tileShape = &gsv_shape, - .tensorView = typed_gsv.getTensorView(), - .sparseDim = typed_gsv.getSparseDim(), - .paddingValue = typed_gsv.getPaddingValue(), - }); - try std.testing.expect(rebuilt_gsv.eql(typed_gsv)); - try std.testing.expect(tileTypeOfView(gsv).eql(tileType(ctx, &.{ 8, 16 }, f32_ty))); - try std.testing.expect((try GatherScatterViewType.get(ctx, .{ - .tileShape = &.{ 8, 16 }, - .tensorView = gather_tv.isA(TensorViewType).?, - .sparseDim = 1, - })).getPaddingValue() == null); - try std.testing.expectError(error.InvalidMlir, GatherScatterViewType.get(ctx, .{ - .tileShape = &.{8}, - .tensorView = gather_tv.isA(TensorViewType).?, - .sparseDim = 0, - })); -} - -test "enum and hint attributes" { - const ctx = try testContext(); - defer ctx.deinit(); - - try std.testing.expect(RoundingMode.nearest_even.attribute(ctx).eql(try mlir.Attribute.parse(ctx, "#cuda_tile.rounding"))); - const padding = PaddingValue.neg_inf.attribute(ctx); - const padding_text = c.mlirCudaTilePaddingValueAttrGetValue(padding.ptr()); - try std.testing.expectEqualStrings("neg_inf", padding_text.data[0..padding_text.length]); - - const hints = optimizationHints(ctx, &[_]EntryHint{ - .{ .arch = .sm_100, .num_cta_in_cga = 2 }, - .{ .arch = .sm_120, .num_cta_in_cga = 2, .occupancy = 2 }, - }); - try std.testing.expect(hints.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.optimization_hints"))); - const typed_hints = hints.isA(OptimizationHintsAttr).?; - try std.testing.expect((try OptimizationHintsAttr.get(ctx, .{ .value = typed_hints.getValue() })).eql(typed_hints)); - const ls_hints = optimizationHints(ctx, &[_]LoadStoreHint{.{ .arch = .sm_100, .allow_tma = false, .latency = 3 }}); - try std.testing.expect(ls_hints.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.optimization_hints"))); - const not_a_dict: *const mlir.DictionaryAttribute = @ptrCast(mlir.Attribute.dict(ctx, &.{.named(ctx, "sm_100", .int(ctx, .i64, 1))})); - try std.testing.expectError(error.InvalidMlir, OptimizationHintsAttr.get(ctx, .{ .value = not_a_dict })); - - const div = divBy(ctx, 16, null, null); - try std.testing.expect(div.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.div_by<16>"))); - const div_every = divBy(ctx, 4, 4, 1); - try std.testing.expect(div_every.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.div_by<4, every 4 along 1>"))); - try std.testing.expect(divBy(ctx, 32, 4, null).eql(try mlir.Attribute.parse(ctx, "#cuda_tile.div_by<32, every 4 along 0>"))); - const typed_div = div_every.isA(DivByAttr).?; - try std.testing.expect((try DivByAttr.get(ctx, .{ - .divisor = typed_div.getDivisor(), - .every = typed_div.getEvery(), - .along = typed_div.getAlong(), - })).eql(typed_div)); - try std.testing.expect(div.isA(DivByAttr).?.getEvery() == null); - try std.testing.expectError(error.InvalidMlir, DivByAttr.get(ctx, .{ .divisor = 4, .every = 4 })); - - const same = sameElements(ctx, &.{ 2, 4 }); - try std.testing.expect(same.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.same_elements<[2, 4]>"))); - const same_values = same.isA(SameElementsAttr).?.getValues(); - try std.testing.expectEqual(@as(usize, 2), same_values.numElements()); - try std.testing.expectEqual(@as(i64, 4), same_values.element(1)); - - const lower = bounded(ctx, 0, null); - try std.testing.expect(lower.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.bounded<0, ?>"))); - try std.testing.expect(bounded(ctx, null, 7).eql(try mlir.Attribute.parse(ctx, "#cuda_tile.bounded"))); - try std.testing.expect(bounded(ctx, -3, 7).eql(try mlir.Attribute.parse(ctx, "#cuda_tile.bounded<-3, 7>"))); - const typed_lower = lower.isA(BoundedAttr).?; - try std.testing.expectEqual(@as(?i64, 0), typed_lower.getLb()); - try std.testing.expect(typed_lower.getUb() == null); -} - -test "entry round-trip through print and parse" { - const ctx = try testContext(); - defer ctx.deinit(); - - const loc: *const mlir.Location = .unknown(ctx); - const module_op: *mlir.Module = .init(loc); - defer module_op.deinit(); - - const f32_ty = mlir.Type.float(ctx, .f32); - const ptr_tile = tileType(ctx, &.{}, pointerType(ctx, f32_ty)); - const block = mlir.Block.init(&.{ ptr_tile, ptr_tile }, &.{ loc, loc }); - - // out[i] = in[i] + 1 over 128 elements, the shape of cuda-tile's README example. - const idx_ty = tileType(ctx, &.{128}, .int(ctx, .i32)); - const offsets = iota(ctx, idx_ty, loc).appendTo(block); - const ptr1 = tileType(ctx, &.{1}, pointerType(ctx, f32_ty)); - const ptr128 = tileType(ctx, &.{128}, pointerType(ctx, f32_ty)); - const in_r = reshape(ctx, block.argument(0), ptr1, loc).appendTo(block); - const in_b = broadcast(ctx, in_r.result(0), ptr128, loc).appendTo(block); - const in_p = offset(ctx, in_b.result(0), offsets.result(0), loc).appendTo(block); - const f32_128 = tileType(ctx, &.{128}, f32_ty); - const loaded = load_ptr_tko(ctx, in_p.result(0), f32_128, .{}, loc).appendTo(block); - const one = constant(ctx, splatAttr(f32_128, floatElem(ctx, f32_ty, 1.0)), f32_128, loc).appendTo(block); - const sum = addf(ctx, loaded.result(0), one.result(0), .nearest_even, false, loc).appendTo(block); - const out_r = reshape(ctx, block.argument(1), ptr1, loc).appendTo(block); - const out_b = broadcast(ctx, out_r.result(0), ptr128, loc).appendTo(block); - const out_p = offset(ctx, out_b.result(0), offsets.result(0), loc).appendTo(block); - _ = store_ptr_tko(ctx, out_p.result(0), sum.result(0), .{ .mem = .{ .token = loaded.result(1) } }, loc).appendTo(block); - _ = return_(ctx, &.{}, loc).appendTo(block); - - const ct_body = mlir.Block.init(&.{}, &.{}); - _ = entry(ctx, .{ .name = "add_one", .block = block, .location = loc }).appendTo(ct_body); - _ = module(ctx, "m", ct_body, loc).appendTo(module_op.body()); - - try std.testing.expect(module_op.operation().verify()); - - var al: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer al.deinit(); - try al.writer.print("{f}", .{module_op.operation()}); - const text = al.written(); - try std.testing.expect(std.mem.indexOf(u8, text, "cuda_tile.module @m") != null); - try std.testing.expect(std.mem.indexOf(u8, text, "entry @add_one(") != null); - try std.testing.expect(std.mem.indexOf(u8, text, "load_ptr_tko weak") != null); - try std.testing.expect(std.mem.indexOf(u8, text, "addf") != null); - - const parsed = try mlir.Module.parse(ctx, text); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "views, control flow and mma verify" { - const ctx = try testContext(); - defer ctx.deinit(); - - const loc: *const mlir.Location = .unknown(ctx); - const module_op: *mlir.Module = .init(loc); - defer module_op.deinit(); - - const bf16_ty = mlir.Type.float(ctx, .bf16); - const f32_ty = mlir.Type.float(ctx, .f32); - const i32_ty = mlir.Type.int(ctx, .i32); - const bf16_ptr = tileType(ctx, &.{}, pointerType(ctx, bf16_ty)); - const f32_ptr = tileType(ctx, &.{}, pointerType(ctx, f32_ty)); - const block = mlir.Block.init(&.{ bf16_ptr, bf16_ptr, f32_ptr }, &.{ loc, loc, loc }); - - // C[128x128] = A[128x64] @ B[64x128] in 64-wide K tiles. - const tv_a = make_tensor_view(ctx, block.argument(0), &.{}, &.{}, tensorViewType(ctx, bf16_ty, &.{ 128, 64 }, &.{ 64, 1 }), loc).appendTo(block); - const tv_b = make_tensor_view(ctx, block.argument(1), &.{}, &.{}, tensorViewType(ctx, bf16_ty, &.{ 64, 128 }, &.{ 128, 1 }), loc).appendTo(block); - const tv_c = make_tensor_view(ctx, block.argument(2), &.{}, &.{}, tensorViewType(ctx, f32_ty, &.{ 128, 128 }, &.{ 128, 1 }), loc).appendTo(block); - const pv_a: *const mlir.Type = @ptrCast(PartitionViewType.get(ctx, &.{ 64, 64 }, tv_a.result(0).type_(), &.{}, .zero)); - const pv_b: *const mlir.Type = @ptrCast(PartitionViewType.get(ctx, &.{ 64, 64 }, tv_b.result(0).type_(), &.{}, .zero)); - const pv_c: *const mlir.Type = @ptrCast(PartitionViewType.get(ctx, &.{ 64, 64 }, tv_c.result(0).type_(), &.{}, null)); - const va = make_partition_view(ctx, tv_a.result(0), pv_a, loc).appendTo(block); - const vb = make_partition_view(ctx, tv_b.result(0), pv_b, loc).appendTo(block); - const vc = make_partition_view(ctx, tv_c.result(0), pv_c, loc).appendTo(block); - - const bid = get_tile_block_id(ctx, loc).appendTo(block); - const i32_tile = tileType(ctx, &.{}, i32_ty); - const zero = constant(ctx, splatAttr(i32_tile, intElem(i32_ty, 0)), i32_tile, loc).appendTo(block); - const one = constant(ctx, splatAttr(i32_tile, intElem(i32_ty, 1)), i32_tile, loc).appendTo(block); - const acc_ty = tileType(ctx, &.{ 64, 64 }, f32_ty); - const acc0 = constant(ctx, splatAttr(acc_ty, floatElem(ctx, f32_ty, 0.0)), acc_ty, loc).appendTo(block); - - const body = mlir.Block.init(&.{ i32_tile, acc_ty }, &.{ loc, loc }); - const at = load_view_tko(ctx, va.result(0), &.{ bid.result(0), body.argument(0) }, .{}, loc).appendTo(body); - const bt = load_view_tko(ctx, vb.result(0), &.{ body.argument(0), bid.result(1) }, .{}, loc).appendTo(body); - try std.testing.expect(at.attributeByName("inbounds").?.eql(.denseArray(ctx, .bool, &.{ 0, 0 }))); - const acc = mmaf(ctx, at.result(0), bt.result(0), body.argument(1), false, loc).appendTo(body); - _ = continue_(ctx, &.{acc.result(0)}, loc).appendTo(body); - const loop_op = for_(ctx, zero.result(0), one.result(0), one.result(0), &.{acc0.result(0)}, body, false, loc).appendTo(block); - - // if %cond -> (tile) { yield } else { yield } - const cond = cmpi(ctx, .equal, .signed, bid.result(0), zero.result(0), loc).appendTo(block); - const then_block = mlir.Block.init(&.{}, &.{}); - _ = yield(ctx, &.{loop_op.result(0)}, loc).appendTo(then_block); - const else_block = mlir.Block.init(&.{}, &.{}); - _ = yield(ctx, &.{acc0.result(0)}, loc).appendTo(else_block); - const if_op = if_(ctx, cond.result(0), &.{acc_ty}, then_block, else_block, loc).appendTo(block); - - const store = store_view_tko(ctx, if_op.result(0), vc.result(0), &.{ bid.result(0), bid.result(1) }, .{}, loc).appendTo(block); - try std.testing.expect(store.attributeByName("inbounds").?.eql(.denseArray(ctx, .bool, &.{ 0, 0 }))); - _ = return_(ctx, &.{}, loc).appendTo(block); - - const ct_body = mlir.Block.init(&.{}, &.{}); - _ = entry(ctx, .{ - .name = "gemm", - .block = block, - .optimization_hints = optimizationHints(ctx, &[_]EntryHint{.{ .arch = .sm_120, .num_worker_warps_per_cta = 4 }}), - .location = loc, - }).appendTo(ct_body); - _ = module(ctx, "m", ct_body, loc).appendTo(module_op.body()); - - try std.testing.expect(module_op.operation().verify()); - - var al: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer al.deinit(); - try al.writer.print("{f}", .{module_op.operation()}); - const text = al.written(); - try std.testing.expect(std.mem.indexOf(u8, text, "optimization_hints=") != null); - try std.testing.expect(std.mem.indexOf(u8, text, "load_view_tko weak") != null); - try std.testing.expect(std.mem.indexOf(u8, text, "continue ") != null); - - const parsed = try mlir.Module.parse(ctx, text); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "the verifier rejects what the dialect forbids" { - const ctx = try testContext(); - defer ctx.deinit(); - const loc: *const mlir.Location = .unknown(ctx); - const f32_ty = mlir.Type.float(ctx, .f32); - - // An entry with a rank-1 argument. (Per-op negatives go through - // `Operation.try_make`, whose logged error the test runner counts as a - // failure, so the module-level verifier is the oracle here.) - const module_op: *mlir.Module = .init(loc); - defer module_op.deinit(); - const block = mlir.Block.init(&.{tileType(ctx, &.{4}, f32_ty)}, &.{loc}); - _ = return_(ctx, &.{}, loc).appendTo(block); - const ct_body = mlir.Block.init(&.{}, &.{}); - _ = entry(ctx, .{ .name = "bad", .block = block, .location = loc }).appendTo(ct_body); - _ = module(ctx, "m", ct_body, loc).appendTo(module_op.body()); - try std.testing.expect(!module_op.operation().verify()); -} diff --git a/mlir/dialects/cuda_tile/cuda_tile_capi.h b/mlir/dialects/cuda_tile/cuda_tile_capi.h deleted file mode 100644 index fcce31cb..00000000 --- a/mlir/dialects/cuda_tile/cuda_tile_capi.h +++ /dev/null @@ -1,79 +0,0 @@ -// C entry points for what NVIDIA's cuda_tile C API leaves out: the -// `TileView` type interface, shared by the partition, strided and -// gather/scatter views, the gather/scatter view type, the general -// `optimization_hints` constructor and the `assume` predicates. -#ifndef ZML_MLIR_DIALECTS_CUDA_TILE_CAPI_H_ -#define ZML_MLIR_DIALECTS_CUDA_TILE_CAPI_H_ - -#include -#include - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Constructors return null for invalid parameters; optional attribute -// parameters take null handles, and std::optional integers a `has` flag. -// Getters require a value accepted by the matching IsA. - -bool mlirTypeIsACudaTileTileView(MlirType type); -MlirType mlirCudaTileTileViewTypeGetViewTileType(MlirType type); -intptr_t mlirCudaTileTileViewTypeGetViewIndexRank(MlirType type); - -// `!cuda_tile.gather_scatter_view`. `paddingValue` is a -// `#cuda_tile.padding_value` attribute, or null for none. -bool mlirTypeIsACudaTileGatherScatterView(MlirType type); -MlirType mlirCudaTileGatherScatterViewTypeGet(MlirContext context, - intptr_t rank, - const int32_t *tileShape, - MlirType tensorView, - uint32_t sparseDim, - MlirAttribute paddingValue); -// A DenseI32ArrayAttr. -MlirAttribute mlirCudaTileGatherScatterViewTypeGetTileShape(MlirType type); -MlirType mlirCudaTileGatherScatterViewTypeGetTensorView(MlirType type); -uint32_t mlirCudaTileGatherScatterViewTypeGetSparseDim(MlirType type); -// Null when the view has no padding value. -MlirAttribute mlirCudaTileGatherScatterViewTypeGetPaddingValue(MlirType type); - -// `#cuda_tile.optimization_hints` over its dictionary of per-architecture -// dictionaries (IsA: NVIDIA's mlirCudaTileAttributeIsAOptimizationHintsAttr). -MlirAttribute mlirCudaTileOptimizationHintsAttrGet(MlirContext context, - MlirAttribute value); -MlirAttribute mlirCudaTileOptimizationHintsAttrGetValue(MlirAttribute attr); - -// `#cuda_tile.div_by`. `every` and `along` are both present or both absent. -bool mlirAttributeIsACudaTileDivBy(MlirAttribute attr); -MlirAttribute mlirCudaTileDivByAttrGet(MlirContext context, uint64_t divisor, - bool hasEvery, int64_t every, - bool hasAlong, int64_t along); -uint64_t mlirCudaTileDivByAttrGetDivisor(MlirAttribute attr); -bool mlirCudaTileDivByAttrHasEvery(MlirAttribute attr); -int64_t mlirCudaTileDivByAttrGetEvery(MlirAttribute attr); -bool mlirCudaTileDivByAttrHasAlong(MlirAttribute attr); -int64_t mlirCudaTileDivByAttrGetAlong(MlirAttribute attr); - -// `#cuda_tile.same_elements`. -bool mlirAttributeIsACudaTileSameElements(MlirAttribute attr); -MlirAttribute mlirCudaTileSameElementsAttrGet(MlirContext context, - intptr_t rank, - const int64_t *values); -// A DenseI64ArrayAttr. -MlirAttribute mlirCudaTileSameElementsAttrGetValues(MlirAttribute attr); - -// `#cuda_tile.bounded`, each bound optional. -bool mlirAttributeIsACudaTileBounded(MlirAttribute attr); -MlirAttribute mlirCudaTileBoundedAttrGet(MlirContext context, bool hasLb, - int64_t lb, bool hasUb, int64_t ub); -bool mlirCudaTileBoundedAttrHasLb(MlirAttribute attr); -int64_t mlirCudaTileBoundedAttrGetLb(MlirAttribute attr); -bool mlirCudaTileBoundedAttrHasUb(MlirAttribute attr); -int64_t mlirCudaTileBoundedAttrGetUb(MlirAttribute attr); - -#ifdef __cplusplus -} -#endif - -#endif // ZML_MLIR_DIALECTS_CUDA_TILE_CAPI_H_ diff --git a/mlir/dialects/cuda_tile/test.zig b/mlir/dialects/cuda_tile/test.zig new file mode 100644 index 00000000..52ae9480 --- /dev/null +++ b/mlir/dialects/cuda_tile/test.zig @@ -0,0 +1,391 @@ +const std = @import("std"); + +const c = @import("c"); +const cuda_tile = @import("mlir/dialects/cuda_tile"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "cuda_tile.absf", + "cuda_tile.absi", + "cuda_tile.addf", + "cuda_tile.addi", + "cuda_tile.alloca", + "cuda_tile.andi", + "cuda_tile.assert", + "cuda_tile.assume", + "cuda_tile.atan2", + "cuda_tile.atomic_cas_tko", + "cuda_tile.atomic_red_view_tko", + "cuda_tile.atomic_rmw_tko", + "cuda_tile.bitcast", + "cuda_tile.break", + "cuda_tile.broadcast", + "cuda_tile.cat", + "cuda_tile.ceil", + "cuda_tile.cmpf", + "cuda_tile.cmpi", + "cuda_tile.constant", + "cuda_tile.continue", + "cuda_tile.cos", + "cuda_tile.cosh", + "cuda_tile.divf", + "cuda_tile.divi", + "cuda_tile.entry", + "cuda_tile.exp", + "cuda_tile.exp2", + "cuda_tile.exti", + "cuda_tile.extract", + "cuda_tile.floor", + "cuda_tile.fma", + "cuda_tile.for", + "cuda_tile.fpowf", + "cuda_tile.ftof", + "cuda_tile.ftoi", + "cuda_tile.get_global", + "cuda_tile.get_index_space_shape", + "cuda_tile.get_num_tile_blocks", + "cuda_tile.get_tensor_shape", + "cuda_tile.get_tile_block_id", + "cuda_tile.global", + "cuda_tile.if", + "cuda_tile.int_to_ptr", + "cuda_tile.iota", + "cuda_tile.itof", + "cuda_tile.join_tokens", + "cuda_tile.load_ptr_tko", + "cuda_tile.load_view_tko", + "cuda_tile.log", + "cuda_tile.log2", + "cuda_tile.loop", + "cuda_tile.make_gather_scatter_view", + "cuda_tile.make_partition_view", + "cuda_tile.make_strided_view", + "cuda_tile.make_tensor_view", + "cuda_tile.make_token", + "cuda_tile.maxf", + "cuda_tile.maxi", + "cuda_tile.minf", + "cuda_tile.mini", + "cuda_tile.mmaf", + "cuda_tile.mmaf_scaled", + "cuda_tile.mmai", + "cuda_tile.module", + "cuda_tile.mulf", + "cuda_tile.mulhii", + "cuda_tile.muli", + "cuda_tile.negf", + "cuda_tile.negi", + "cuda_tile.offset", + "cuda_tile.ori", + "cuda_tile.pack", + "cuda_tile.permute", + "cuda_tile.print_tko", + "cuda_tile.ptr_to_int", + "cuda_tile.ptr_to_ptr", + "cuda_tile.reduce", + "cuda_tile.remf", + "cuda_tile.remi", + "cuda_tile.reshape", + "cuda_tile.return", + "cuda_tile.rsqrt", + "cuda_tile.scan", + "cuda_tile.select", + "cuda_tile.shli", + "cuda_tile.shri", + "cuda_tile.sin", + "cuda_tile.sinh", + "cuda_tile.sqrt", + "cuda_tile.store_ptr_tko", + "cuda_tile.store_view_tko", + "cuda_tile.subf", + "cuda_tile.subi", + "cuda_tile.tan", + "cuda_tile.tanh", + "cuda_tile.trunci", + "cuda_tile.unpack", + "cuda_tile.xori", + "cuda_tile.yield", +}; + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + cuda_tile.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(cuda_tile); + inline for (comptime std.meta.declarations(cuda_tile)) |decl| { + const T = @field(cuda_tile, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "dialect registers and types round-trip" { + const ctx = try testContext(); + defer ctx.deinit(); + + try std.testing.expect(ctx.isRegisteredOperation("cuda_tile.entry")); + try std.testing.expect(ctx.isRegisteredOperation("cuda_tile.load_view_tko")); + try std.testing.expect(ctx.isRegisteredOperation("cuda_tile.mmaf")); + + const f32_ty = mlir.Type.float(ctx, .f32); + const tile = cuda_tile.tileType(ctx, &.{ 64, 64 }, f32_ty); + try std.testing.expect(tile.eql(try mlir.Type.parse(ctx, "!cuda_tile.tile<64x64xf32>"))); + try std.testing.expectEqual(@as(usize, 2), tile.isA(cuda_tile.TileType).?.rank()); + try std.testing.expectEqual(@as(i64, 64), tile.isA(cuda_tile.TileType).?.dimension(1)); + + const ptr = cuda_tile.pointerType(ctx, f32_ty); + try std.testing.expect(ptr.eql(try mlir.Type.parse(ctx, "!cuda_tile.ptr"))); + try std.testing.expect(ptr.isA(cuda_tile.PointerType).?.pointee().eql(f32_ty)); + + const dyn = cuda_tile.TensorViewType.dynamic(); + const tv = cuda_tile.tensorViewType(ctx, f32_ty, &.{ dyn, 32 }, &.{ dyn, 1 }); + try std.testing.expect(tv.eql(try mlir.Type.parse(ctx, "!cuda_tile.tensor_view"))); + + const static_tv = cuda_tile.tensorViewType(ctx, f32_ty, &.{16}, &.{1}); + const pv: *const mlir.Type = @ptrCast(cuda_tile.PartitionViewType.get(ctx, .{ .tile_shape = &.{2}, .tensor_view = static_tv, .padding = .zero })); + try std.testing.expect(pv.eql(try mlir.Type.parse(ctx, "!cuda_tile.partition_view>"))); + try std.testing.expect(cuda_tile.tileTypeOfView(pv).eql(cuda_tile.tileType(ctx, &.{2}, f32_ty))); + try std.testing.expectEqual(@as(usize, 1), cuda_tile.indexRankOfView(pv)); + + try std.testing.expect(cuda_tile.tokenType(ctx).eql(try mlir.Type.parse(ctx, "!cuda_tile.token"))); + + const gather_tv = cuda_tile.tensorViewType(ctx, f32_ty, &.{ 64, 32 }, &.{ 32, 1 }); + const gsv = cuda_tile.gatherScatterViewType(ctx, &.{ 8, 16 }, gather_tv, 0, .{ .padding = .zero }); + try std.testing.expect(gsv.eql(try mlir.Type.parse(ctx, "!cuda_tile.gather_scatter_view, sparse_dim=0>"))); + const typed_gsv = gsv.isA(cuda_tile.GatherScatterViewType).?; + const gsv_tile = typed_gsv.tileShape(); + var gsv_shape: [2]i32 = undefined; + try std.testing.expectEqual(gsv_shape.len, gsv_tile.numElements()); + for (&gsv_shape, 0..) |*d, i| d.* = gsv_tile.element(i); + const rebuilt_gsv = try cuda_tile.GatherScatterViewType.get(ctx, .{ + .tileShape = &gsv_shape, + .tensorView = typed_gsv.tensorView(), + .sparseDim = typed_gsv.sparseDim(), + .paddingValue = typed_gsv.paddingValue(), + }); + try std.testing.expect(rebuilt_gsv.eql(typed_gsv)); + try std.testing.expect(cuda_tile.tileTypeOfView(gsv).eql(cuda_tile.tileType(ctx, &.{ 8, 16 }, f32_ty))); + try std.testing.expect((try cuda_tile.GatherScatterViewType.get(ctx, .{ + .tileShape = &.{ 8, 16 }, + .tensorView = gather_tv.isA(cuda_tile.TensorViewType).?, + .sparseDim = 1, + })).paddingValue() == null); + try std.testing.expectError(error.InvalidMlir, cuda_tile.GatherScatterViewType.get(ctx, .{ + .tileShape = &.{8}, + .tensorView = gather_tv.isA(cuda_tile.TensorViewType).?, + .sparseDim = 0, + })); +} + +test "enum and hint attributes" { + const ctx = try testContext(); + defer ctx.deinit(); + + try std.testing.expect(cuda_tile.RoundingMode.nearest_even.attribute(ctx).eql(try mlir.Attribute.parse(ctx, "#cuda_tile.rounding"))); + const padding = cuda_tile.PaddingValue.neg_inf.attribute(ctx); + const padding_text = c.mlirCudaTilePaddingValueAttrGetValue(padding.ptr()); + try std.testing.expectEqualStrings("neg_inf", padding_text.data[0..padding_text.length]); + + const hints = cuda_tile.optimizationHintsAttr(ctx, &[_]cuda_tile.EntryHint{ + .{ .arch = .sm_100, .num_cta_in_cga = 2 }, + .{ .arch = .sm_120, .num_cta_in_cga = 2, .occupancy = 2 }, + }); + try std.testing.expect(hints.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.optimization_hints"))); + const typed_hints = hints.isA(cuda_tile.OptimizationHintsAttr).?; + try std.testing.expect((try cuda_tile.OptimizationHintsAttr.get(ctx, .{ .value = typed_hints.value() })).eql(typed_hints)); + const ls_hints = cuda_tile.optimizationHintsAttr(ctx, &[_]cuda_tile.LoadStoreHint{.{ .arch = .sm_100, .allow_tma = false, .latency = 3 }}); + try std.testing.expect(ls_hints.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.optimization_hints"))); + const not_a_dict: *const mlir.DictionaryAttribute = @ptrCast(mlir.Attribute.dict(ctx, &.{.named(ctx, "sm_100", .int(ctx, .i64, 1))})); + try std.testing.expectError(error.InvalidMlir, cuda_tile.OptimizationHintsAttr.get(ctx, .{ .value = not_a_dict })); + + const div = cuda_tile.divByAttr(ctx, 16, .{}); + try std.testing.expect(div.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.div_by<16>"))); + const div_every = cuda_tile.divByAttr(ctx, 4, .{ .every = 4, .along = 1 }); + try std.testing.expect(div_every.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.div_by<4, every 4 along 1>"))); + try std.testing.expect(cuda_tile.divByAttr(ctx, 32, .{ .every = 4 }).eql(try mlir.Attribute.parse(ctx, "#cuda_tile.div_by<32, every 4 along 0>"))); + const typed_div = div_every.isA(cuda_tile.DivByAttr).?; + try std.testing.expect((try cuda_tile.DivByAttr.get(ctx, .{ + .divisor = typed_div.divisor(), + .every = typed_div.every(), + .along = typed_div.along(), + })).eql(typed_div)); + try std.testing.expect(div.isA(cuda_tile.DivByAttr).?.every() == null); + try std.testing.expectError(error.InvalidMlir, cuda_tile.DivByAttr.get(ctx, .{ .divisor = 4, .every = 4 })); + + const same = cuda_tile.sameElementsAttr(ctx, &.{ 2, 4 }); + try std.testing.expect(same.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.same_elements<[2, 4]>"))); + const same_values = same.isA(cuda_tile.SameElementsAttr).?.values(); + try std.testing.expectEqual(@as(usize, 2), same_values.numElements()); + try std.testing.expectEqual(@as(i64, 4), same_values.element(1)); + + const lower = cuda_tile.boundedAttr(ctx, .{ .lb = 0 }); + try std.testing.expect(lower.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.bounded<0, ?>"))); + try std.testing.expect(cuda_tile.boundedAttr(ctx, .{ .ub = 7 }).eql(try mlir.Attribute.parse(ctx, "#cuda_tile.bounded"))); + try std.testing.expect(cuda_tile.boundedAttr(ctx, .{ .lb = -3, .ub = 7 }).eql(try mlir.Attribute.parse(ctx, "#cuda_tile.bounded<-3, 7>"))); + const typed_lower = lower.isA(cuda_tile.BoundedAttr).?; + try std.testing.expectEqual(@as(?i64, 0), typed_lower.lb()); + try std.testing.expect(typed_lower.ub() == null); +} + +test "entry round-trip through print and parse" { + const ctx = try testContext(); + defer ctx.deinit(); + + const loc: *const mlir.Location = .unknown(ctx); + const module_op: *mlir.Module = .init(loc); + defer module_op.deinit(); + + const f32_ty = mlir.Type.float(ctx, .f32); + const ptr_tile = cuda_tile.tileType(ctx, &.{}, cuda_tile.pointerType(ctx, f32_ty)); + const block = mlir.Block.init(&.{ ptr_tile, ptr_tile }, &.{ loc, loc }); + + // out[i] = in[i] + 1 over 128 elements, the shape of cuda-tile's README example. + const idx_ty = cuda_tile.tileType(ctx, &.{128}, .int(ctx, .i32)); + const offsets = cuda_tile.iota(ctx, idx_ty, loc).appendTo(block); + const ptr1 = cuda_tile.tileType(ctx, &.{1}, cuda_tile.pointerType(ctx, f32_ty)); + const ptr128 = cuda_tile.tileType(ctx, &.{128}, cuda_tile.pointerType(ctx, f32_ty)); + const in_r = cuda_tile.reshape(ctx, block.argument(0), ptr1, loc).appendTo(block); + const in_b = cuda_tile.broadcast(ctx, in_r.result(0), ptr128, loc).appendTo(block); + const in_p = cuda_tile.offset(ctx, in_b.result(0), offsets.result(0), loc).appendTo(block); + const f32_128 = cuda_tile.tileType(ctx, &.{128}, f32_ty); + const loaded = cuda_tile.load_ptr_tko(ctx, in_p.result(0), f32_128, .{}, loc).appendTo(block); + const one = cuda_tile.constant(ctx, cuda_tile.splatAttr(f32_128, cuda_tile.floatElemAttr(ctx, f32_ty, 1.0)), f32_128, loc).appendTo(block); + const sum = cuda_tile.addf(ctx, loaded.result(0), one.result(0), .nearest_even, false, loc).appendTo(block); + const out_r = cuda_tile.reshape(ctx, block.argument(1), ptr1, loc).appendTo(block); + const out_b = cuda_tile.broadcast(ctx, out_r.result(0), ptr128, loc).appendTo(block); + const out_p = cuda_tile.offset(ctx, out_b.result(0), offsets.result(0), loc).appendTo(block); + _ = cuda_tile.store_ptr_tko(ctx, out_p.result(0), sum.result(0), .{ .mem = .{ .token = loaded.result(1) } }, loc).appendTo(block); + _ = cuda_tile.return_(ctx, &.{}, loc).appendTo(block); + + const ct_body = mlir.Block.init(&.{}, &.{}); + _ = cuda_tile.entry(ctx, .{ .name = "add_one", .block = block, .location = loc }).appendTo(ct_body); + _ = cuda_tile.module(ctx, "m", ct_body, loc).appendTo(module_op.body()); + + try std.testing.expect(module_op.operation().verify()); + + var al: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer al.deinit(); + try al.writer.print("{f}", .{module_op.operation()}); + const text = al.written(); + try std.testing.expect(std.mem.indexOf(u8, text, "cuda_tile.module @m") != null); + try std.testing.expect(std.mem.indexOf(u8, text, "entry @add_one(") != null); + try std.testing.expect(std.mem.indexOf(u8, text, "load_ptr_tko weak") != null); + try std.testing.expect(std.mem.indexOf(u8, text, "addf") != null); + + const parsed = try mlir.Module.parse(ctx, text); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "views, control flow and mma verify" { + const ctx = try testContext(); + defer ctx.deinit(); + + const loc: *const mlir.Location = .unknown(ctx); + const module_op: *mlir.Module = .init(loc); + defer module_op.deinit(); + + const bf16_ty = mlir.Type.float(ctx, .bf16); + const f32_ty = mlir.Type.float(ctx, .f32); + const i32_ty = mlir.Type.int(ctx, .i32); + const bf16_ptr = cuda_tile.tileType(ctx, &.{}, cuda_tile.pointerType(ctx, bf16_ty)); + const f32_ptr = cuda_tile.tileType(ctx, &.{}, cuda_tile.pointerType(ctx, f32_ty)); + const block = mlir.Block.init(&.{ bf16_ptr, bf16_ptr, f32_ptr }, &.{ loc, loc, loc }); + + // C[128x128] = A[128x64] @ B[64x128] in 64-wide K tiles. + const tv_a = cuda_tile.make_tensor_view(ctx, block.argument(0), &.{}, &.{}, cuda_tile.tensorViewType(ctx, bf16_ty, &.{ 128, 64 }, &.{ 64, 1 }), loc).appendTo(block); + const tv_b = cuda_tile.make_tensor_view(ctx, block.argument(1), &.{}, &.{}, cuda_tile.tensorViewType(ctx, bf16_ty, &.{ 64, 128 }, &.{ 128, 1 }), loc).appendTo(block); + const tv_c = cuda_tile.make_tensor_view(ctx, block.argument(2), &.{}, &.{}, cuda_tile.tensorViewType(ctx, f32_ty, &.{ 128, 128 }, &.{ 128, 1 }), loc).appendTo(block); + const pv_a: *const mlir.Type = @ptrCast(cuda_tile.PartitionViewType.get(ctx, .{ .tile_shape = &.{ 64, 64 }, .tensor_view = tv_a.result(0).type_(), .padding = .zero })); + const pv_b: *const mlir.Type = @ptrCast(cuda_tile.PartitionViewType.get(ctx, .{ .tile_shape = &.{ 64, 64 }, .tensor_view = tv_b.result(0).type_(), .padding = .zero })); + const pv_c: *const mlir.Type = @ptrCast(cuda_tile.PartitionViewType.get(ctx, .{ .tile_shape = &.{ 64, 64 }, .tensor_view = tv_c.result(0).type_() })); + const va = cuda_tile.make_partition_view(ctx, tv_a.result(0), pv_a, loc).appendTo(block); + const vb = cuda_tile.make_partition_view(ctx, tv_b.result(0), pv_b, loc).appendTo(block); + const vc = cuda_tile.make_partition_view(ctx, tv_c.result(0), pv_c, loc).appendTo(block); + + const bid = cuda_tile.get_tile_block_id(ctx, loc).appendTo(block); + const i32_tile = cuda_tile.tileType(ctx, &.{}, i32_ty); + const zero = cuda_tile.constant(ctx, cuda_tile.splatAttr(i32_tile, cuda_tile.intElemAttr(i32_ty, 0)), i32_tile, loc).appendTo(block); + const one = cuda_tile.constant(ctx, cuda_tile.splatAttr(i32_tile, cuda_tile.intElemAttr(i32_ty, 1)), i32_tile, loc).appendTo(block); + const acc_ty = cuda_tile.tileType(ctx, &.{ 64, 64 }, f32_ty); + const acc0 = cuda_tile.constant(ctx, cuda_tile.splatAttr(acc_ty, cuda_tile.floatElemAttr(ctx, f32_ty, 0.0)), acc_ty, loc).appendTo(block); + + const body = mlir.Block.init(&.{ i32_tile, acc_ty }, &.{ loc, loc }); + const at = cuda_tile.load_view_tko(ctx, va.result(0), &.{ bid.result(0), body.argument(0) }, .{}, loc).appendTo(body); + const bt = cuda_tile.load_view_tko(ctx, vb.result(0), &.{ body.argument(0), bid.result(1) }, .{}, loc).appendTo(body); + try std.testing.expect(at.attributeByName("inbounds").?.eql(.denseArray(ctx, .bool, &.{ 0, 0 }))); + const acc = cuda_tile.mmaf(ctx, at.result(0), bt.result(0), body.argument(1), false, loc).appendTo(body); + _ = cuda_tile.continue_(ctx, &.{acc.result(0)}, loc).appendTo(body); + const loop_op = cuda_tile.for_(ctx, zero.result(0), one.result(0), one.result(0), &.{acc0.result(0)}, body, false, loc).appendTo(block); + + // if %cond -> (tile) { yield } else { yield } + const cond = cuda_tile.cmpi(ctx, .equal, .signed, bid.result(0), zero.result(0), loc).appendTo(block); + const then_block = mlir.Block.init(&.{}, &.{}); + _ = cuda_tile.yield(ctx, &.{loop_op.result(0)}, loc).appendTo(then_block); + const else_block = mlir.Block.init(&.{}, &.{}); + _ = cuda_tile.yield(ctx, &.{acc0.result(0)}, loc).appendTo(else_block); + const if_op = cuda_tile.if_(ctx, cond.result(0), &.{acc_ty}, then_block, .{ .else_block = else_block }, loc).appendTo(block); + + const store = cuda_tile.store_view_tko(ctx, if_op.result(0), vc.result(0), &.{ bid.result(0), bid.result(1) }, .{}, loc).appendTo(block); + try std.testing.expect(store.attributeByName("inbounds").?.eql(.denseArray(ctx, .bool, &.{ 0, 0 }))); + _ = cuda_tile.return_(ctx, &.{}, loc).appendTo(block); + + const ct_body = mlir.Block.init(&.{}, &.{}); + _ = cuda_tile.entry(ctx, .{ + .name = "gemm", + .block = block, + .optimization_hints = cuda_tile.optimizationHintsAttr(ctx, &[_]cuda_tile.EntryHint{.{ .arch = .sm_120, .num_worker_warps_per_cta = 4 }}), + .location = loc, + }).appendTo(ct_body); + _ = cuda_tile.module(ctx, "m", ct_body, loc).appendTo(module_op.body()); + + try std.testing.expect(module_op.operation().verify()); + + var al: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer al.deinit(); + try al.writer.print("{f}", .{module_op.operation()}); + const text = al.written(); + try std.testing.expect(std.mem.indexOf(u8, text, "optimization_hints=") != null); + try std.testing.expect(std.mem.indexOf(u8, text, "load_view_tko weak") != null); + try std.testing.expect(std.mem.indexOf(u8, text, "continue ") != null); + + const parsed = try mlir.Module.parse(ctx, text); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "the verifier rejects what the dialect forbids" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc: *const mlir.Location = .unknown(ctx); + const f32_ty = mlir.Type.float(ctx, .f32); + + // An entry with a rank-1 argument. (Per-op negatives go through + // `Operation.try_make`, whose logged error the test runner counts as a + // failure, so the module-level verifier is the oracle here.) + const module_op: *mlir.Module = .init(loc); + defer module_op.deinit(); + const block = mlir.Block.init(&.{cuda_tile.tileType(ctx, &.{4}, f32_ty)}, &.{loc}); + _ = cuda_tile.return_(ctx, &.{}, loc).appendTo(block); + const ct_body = mlir.Block.init(&.{}, &.{}); + _ = cuda_tile.entry(ctx, .{ .name = "bad", .block = block, .location = loc }).appendTo(ct_body); + _ = cuda_tile.module(ctx, "m", ct_body, loc).appendTo(module_op.body()); + try std.testing.expect(!module_op.operation().verify()); +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + cuda_tile.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/cute_ir/BUILD.bazel b/mlir/dialects/cute_ir/BUILD.bazel index e5354d89..c584509f 100644 --- a/mlir/dialects/cute_ir/BUILD.bazel +++ b/mlir/dialects/cute_ir/BUILD.bazel @@ -1,36 +1,211 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + ":dialect", + ":headers", + "//mlir/dialects/cute_nvgpu:dialect", + "@llvm-project//mlir:CAPIIR", + ], +) + zig_library( name = "cute_ir", - srcs = [ - "cuda.zig", - "cute_nvgpu.zig", - "nvvm.zig", - ], import_name = "mlir/dialects/cute_ir", main = "cute_ir.zig", - visibility = ["//visibility:public"], + tags = ["manual"], deps = [ + ":capi", "//mlir", - "//mlir:c", - "//mlir/dialects/cute_ir/include/cute_ir-c/Dialect:cuda", - "//mlir/dialects/cute_ir/include/cute_ir-c/Dialect:cute", - "//mlir/dialects/cute_ir/include/cute_ir-c/Dialect:cute_nvgpu", - "//mlir/dialects/cute_ir/include/cute_ir-c/Dialect:nvvm", - "//mlir/dialects/cute_ir/lib/CAPI:cuda", - "//mlir/dialects/cute_ir/lib/CAPI:cute", - "//mlir/dialects/cute_ir/lib/CAPI:cute_nvgpu", - "//mlir/dialects/cute_ir/lib/CAPI:nvvm", "//stdx", ], ) zig_test( - name = "zig_test", - deps = [":cute_ir"], + name = "test", + main = "test.zig", + deps = [ + ":cute_ir", + "//mlir", + "//mlir/dialects/cute_nvgpu", + ], ) -test_suite( - name = "test", - tests = [":zig_test"], +td_library( + name = "td_files", + srcs = [ + "CuteAttrsAll.td", + "CuteAttrsCompiler.td", + "CuteEnums.td", + "CuteOps.td", + "CuteTypesAll.td", + "CuteTypesCompiler.td", + ], + includes = ["/"], + deps = [ + "@cute_ir//:cute_ir_dialect_td", + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "enums_inc_gen", + tbl_outs = [ + ( + ["-gen-enum-decls"], + "CuteEnums.h.inc", + ), + ( + ["-gen-enum-defs"], + "CuteEnums.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteEnums.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "attrs_inc_gen", + tbl_outs = [ + ( + [ + "-gen-attrdef-decls", + "-attrdefs-dialect=cute", + ], + "CuteAttrDecls.h.inc", + ), + ( + [ + "-gen-attrdef-defs", + "-attrdefs-dialect=cute", + ], + "CuteAttrs.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteAttrsAll.td", + deps = [":td_files"], +) + +genrule( + name = "attrs_h_inc", + srcs = ["CuteAttrDecls.h.inc"], + outs = ["CuteAttrs.h.inc"], + cmd = "(echo '#ifdef GET_ATTRDEF_CLASSES'; " + + "echo '#include \"cute_ir/Dialect/Cute/IR/CuteEnums.h.inc\"'; " + + "echo '#endif'; cat $<) > $@", +) + +gentbl_cc_library( + name = "types_inc_gen", + tbl_outs = [ + ( + [ + "-gen-typedef-decls", + "-typedefs-dialect=cute", + ], + "CuteTypes.h.inc", + ), + ( + [ + "-gen-typedef-defs", + "-typedefs-dialect=cute", + ], + "CuteTypes.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteTypesAll.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "ops_inc_gen", + tbl_outs = [ + ( + ["-gen-op-decls"], + "CuteOps.h.inc", + ), + ( + ["-gen-op-defs"], + "CuteOps.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteOps.td", + deps = [":td_files"], +) + +cc_library( + name = "headers", + hdrs = [ + "CuteAttrs.h.inc", + "CuteInference.h", + ], + deps = [ + ":attrs_inc_gen", + ":enums_inc_gen", + ":ops_inc_gen", + ":types_inc_gen", + ":upstream_generated_headers", + "@cute_ir//:cute_ir_dialect_header", + "@cute_ir//:cutegen", + "@llvm-project//mlir:DataLayoutInterfaces", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:SideEffectInterfaces", + ], +) + +cc_library( + name = "dialect", + srcs = [ + "CuteInference.cpp", + "CuteOps.cpp", + "CuteTypes.cpp", + "@cute_ir//:cute_ir/lib/Dialect/IR/CuteAttributes.cpp", + "@cute_ir//:cute_ir/lib/Dialect/IR/CuteDialect.cpp", + ], + deps = [ + ":headers", + "//mlir/dialects/cute_nvgpu:headers", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:MathDialect", + "@llvm-project//mlir:NVVMDialect", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:UBDialect", + ], +) + +# Upstream CuTe sources retain their generated-header include paths. +cc_library( + name = "upstream_generated_headers", + hdrs = [ + "CuteAttrDecls.h.inc", + "CuteAttrs.cpp.inc", + "CuteAttrs.h.inc", + "CuteEnums.cpp.inc", + "CuteEnums.h.inc", + "CuteOps.cpp.inc", + "CuteOps.h.inc", + "CuteTypes.cpp.inc", + "CuteTypes.h.inc", + ], + include_prefix = "cute_ir/Dialect/Cute/IR", + strip_include_prefix = ".", ) diff --git a/mlir/dialects/cute_ir/CuteAttrsAll.td b/mlir/dialects/cute_ir/CuteAttrsAll.td new file mode 100644 index 00000000..3f42a1cb --- /dev/null +++ b/mlir/dialects/cute_ir/CuteAttrsAll.td @@ -0,0 +1,14 @@ +//===- CuteAttrsAll.td - all cute attributes ---------------*- tablegen -*-===// +// +// The dialect has one generated attribute parser and printer, so the +// release's attributes, the compiler's and its enum attributes are generated +// together from here. + +#ifndef MLIR_DIALECTS_CUTE_IR_CUTEATTRSALL_TD +#define MLIR_DIALECTS_CUTE_IR_CUTEATTRSALL_TD + +include "cute_ir/Dialect/Cute/IR/CuteAttrs.td" +include "mlir/dialects/cute_ir/CuteAttrsCompiler.td" +include "mlir/dialects/cute_ir/CuteEnums.td" + +#endif // MLIR_DIALECTS_CUTE_IR_CUTEATTRSALL_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteAttrsCompiler.td b/mlir/dialects/cute_ir/CuteAttrsCompiler.td similarity index 98% rename from mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteAttrsCompiler.td rename to mlir/dialects/cute_ir/CuteAttrsCompiler.td index 7ce36eb5..874763a5 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteAttrsCompiler.td +++ b/mlir/dialects/cute_ir/CuteAttrsCompiler.td @@ -12,8 +12,8 @@ // // The reduction_op enum attribute is in CuteEnums.td. -#ifndef CUTE_IR_GENERATED_CUTE_ATTRS_COMPILER_TD -#define CUTE_IR_GENERATED_CUTE_ATTRS_COMPILER_TD +#ifndef MLIR_DIALECTS_CUTE_IR_CUTEATTRSCOMPILER_TD +#define MLIR_DIALECTS_CUTE_IR_CUTEATTRSCOMPILER_TD include "cute_ir/Dialect/Cute/IR/CuteAttrs.td" @@ -217,4 +217,4 @@ def Cute_SparseMxMmaAtomAttr }]; } -#endif // CUTE_IR_GENERATED_CUTE_ATTRS_COMPILER_TD +#endif // MLIR_DIALECTS_CUTE_IR_CUTEATTRSCOMPILER_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteEnums.td b/mlir/dialects/cute_ir/CuteEnums.td similarity index 94% rename from mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteEnums.td rename to mlir/dialects/cute_ir/CuteEnums.td index d3464172..68018cc1 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteEnums.td +++ b/mlir/dialects/cute_ir/CuteEnums.td @@ -4,8 +4,8 @@ // Case values and spellings are the compiler's. An enum the bindings build // as a dialect attribute also has its EnumAttr, printed `#`. -#ifndef CUTE_IR_GENERATED_CUTE_ENUMS_TD -#define CUTE_IR_GENERATED_CUTE_ENUMS_TD +#ifndef MLIR_DIALECTS_CUTE_IR_CUTEENUMS_TD +#define MLIR_DIALECTS_CUTE_IR_CUTEENUMS_TD include "cute_ir/Dialect/Cute/IR/CuteDialect.td" include "mlir/IR/EnumAttr.td" @@ -67,4 +67,4 @@ def Cute_ReductionOpAttr : EnumAttr #include #include @@ -14,13 +23,6 @@ #include #include -#include "cute_ir/Dialect/Cute/IR/CuteInference.h" -#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h" -#include "llvm/ADT/StringExtras.h" -#include "llvm/ADT/TypeSwitch.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/Matchers.h" - using namespace mlir; using namespace mlir::cutlass_compiler::cute; namespace cg = cutegen; @@ -36,22 +38,24 @@ Type layoutType(MLIRContext *ctx, cg::layout l) { return LayoutType::get(ctx, std::move(l)); } Type layoutType(MLIRContext *ctx, cg::composed_layout l) { - if (l.is_normal_layout()) return LayoutType::get(ctx, l.layout_b()); + if (l.is_normal_layout()) + return LayoutType::get(ctx, l.layout_b()); return ComposedLayoutType::get(ctx, std::move(l)); } // `f` runs over the algebra value of a layout or composed layout type; the // result keeps the input's kind (see rebuild for views). -template -Type mapLayout(MLIRContext *ctx, Type layout, F &&f) { +template Type mapLayout(MLIRContext *ctx, Type layout, F &&f) { if (auto l = llvm::dyn_cast(layout)) { auto r = f(l.getRef()); - if (!cg::is_valid(r)) return {}; + if (!cg::is_valid(r)) + return {}; return layoutType(ctx, std::move(r)); } if (auto l = llvm::dyn_cast(layout)) { auto r = f(l.getRef()); - if (!cg::is_valid(r)) return {}; + if (!cg::is_valid(r)) + return {}; return ComposedLayoutType::get(ctx, std::move(r)); } return {}; @@ -61,39 +65,47 @@ Type mapLayout(MLIRContext *ctx, Type layout, F &&f) { // coordinate tensor (arithmetic tuple + layout). struct Carrier { Type layout; - PtrType ptr; // memref - IntTupleType iter; // coord_tensor - Type desc; // cute_nvgpu.smem_desc_view + PtrType ptr; // memref + IntTupleType iter; // coord_tensor + Type desc; // cute_nvgpu.smem_desc_view }; std::optional carrierOf(Type t) { - if (llvm::isa(t)) return Carrier{t, {}, {}, {}}; + if (llvm::isa(t)) + return Carrier{t, {}, {}, {}}; if (auto m = llvm::dyn_cast(t)) return Carrier{m.getLayout(), m.getPtr(), {}, {}}; if (auto c = llvm::dyn_cast(t)) return Carrier{c.getLayout(), {}, c.getArithTuple(), {}}; if (auto v = llvm::dyn_cast(t)) - return Carrier{LayoutType::get(t.getContext(), v.getLayout()), {}, {}, v.getDesc()}; + return Carrier{ + LayoutType::get(t.getContext(), v.getLayout()), {}, {}, v.getDesc()}; return std::nullopt; } Type rebuild(MLIRContext *ctx, const Carrier &c, Type layout) { - if (!layout) return {}; + if (!layout) + return {}; // A view's layout is plain when its composed layout turned out plain. - if (auto l = llvm::dyn_cast(layout); l && (c.ptr || c.iter || c.desc)) + if (auto l = llvm::dyn_cast(layout); + l && (c.ptr || c.iter || c.desc)) layout = layoutType(ctx, l.getRef()); - if (c.ptr) return CuteMemRefType::get(ctx, c.ptr, layout); - if (c.iter) return CoordTensorType::get(ctx, c.iter, layout); + if (c.ptr) + return CuteMemRefType::get(ctx, c.ptr, layout); + if (c.iter) + return CoordTensorType::get(ctx, c.iter, layout); if (c.desc) { auto l = llvm::dyn_cast(layout); - return l ? cute_nvgpu::SmemDescViewType::get(ctx, c.desc, l.getAttr()) : Type(); + return l ? cute_nvgpu::SmemDescViewType::get(ctx, c.desc, l.getAttr()) + : Type(); } return layout; } // The bit width of an element type; tf32 counts as its 32-bit storage. int64_t bitsOfType(Type t) { - if (t.isTF32()) return 32; + if (t.isTF32()) + return 32; return t.isIntOrFloat() ? t.getIntOrFloatBitWidth() : 0; } @@ -108,7 +120,8 @@ uint64_t movedAlignment(PtrType ptr, const cg::int_tuple &offset) { uint64_t known = 0; if (cg::holds_int(offset)) { int64_t v = offset.as_int64(); - if (v == 0) return ptr.getAlignment(); + if (v == 0) + return ptr.getAlignment(); known = static_cast(v < 0 ? -v : v); } else if (cg::holds_dynamic_int(offset)) { known = static_cast( @@ -121,9 +134,9 @@ uint64_t movedAlignment(PtrType ptr, const cg::int_tuple &offset) { } PtrType movedPtr(PtrType ptr, const cg::int_tuple &offset) { - return PtrType::get(ptr.getContext(), ptr.getValueType(), ptr.getAddressSpace(), - movedAlignment(ptr, offset), ptr.getSwizzle(), - ptr.getBitlayout()); + return PtrType::get(ptr.getContext(), ptr.getValueType(), + ptr.getAddressSpace(), movedAlignment(ptr, offset), + ptr.getSwizzle(), ptr.getBitlayout()); } // A coordinate with its underscores read as zero, for evaluating a layout. @@ -140,21 +153,25 @@ Type sliceCarrier(MLIRContext *ctx, const Carrier &c, const cg::coord &coord) { Type layout; if (auto l = llvm::dyn_cast(c.layout)) { auto [sliced, off] = cg::slice_and_offset(coord, l.getRef()); - if (!cg::is_valid(sliced)) return {}; + if (!cg::is_valid(sliced)) + return {}; layout = layoutType(ctx, std::move(sliced)); offset = off; } else if (auto l = llvm::dyn_cast(c.layout)) { // The composed layout's offset takes the move; the iterator stays. auto sliced = cg::slice(coord, l.getRef()); - if (!cg::is_valid(sliced)) return {}; + if (!cg::is_valid(sliced)) + return {}; return rebuild(ctx, c, layoutType(ctx, std::move(sliced))); } else { return {}; } - if (c.ptr) return CuteMemRefType::get(ctx, movedPtr(c.ptr, offset), layout); + if (c.ptr) + return CuteMemRefType::get(ctx, movedPtr(c.ptr, offset), layout); if (c.iter) { auto sum = cg::arith_tuple_sum(c.iter.getRef(), offset); - if (!cg::is_valid(sum)) return {}; + if (!cg::is_valid(sum)) + return {}; return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), layout); } @@ -167,7 +184,8 @@ Type sliceCarrier(MLIRContext *ctx, const Carrier &c, const cg::coord &coord) { cg::layout zeroUnitModes(cg::layout l) { if (cg::holds_leaf(l.shape())) { if (cg::holds_int(l.shape()) && l.shape().as_int64() == 1) { - if (auto unit = cg::from_string("1:0")) return *unit; + if (auto unit = cg::from_string("1:0")) + return *unit; } return l; } @@ -176,7 +194,8 @@ cg::layout zeroUnitModes(cg::layout l) { auto stride = cg::to_string(l.stride()); auto fs = cg::flatten(l.shape()); auto ft = cg::flatten(l.stride()); - if (cg::rank(fs) != cg::rank(ft)) return l; + if (cg::rank(fs) != cg::rank(ft)) + return l; bool changed = false; for (size_t i = 0; i < cg::rank(fs); ++i) { if (cg::holds_int(fs[i]) && fs[i].as_int64() == 1 && @@ -184,7 +203,8 @@ cg::layout zeroUnitModes(cg::layout l) { changed = true; } } - if (!changed) return l; + if (!changed) + return l; // Rewrite the stride text leaf by leaf, in flattened order. std::string out; size_t leaf = 0; @@ -203,8 +223,10 @@ cg::layout zeroUnitModes(cg::layout l) { int depth = 0; while (j < stride.size() && !((stride[j] == ',' || stride[j] == ')') && depth == 0)) { - if (stride[j] == '<' || stride[j] == '{') ++depth; - if (stride[j] == '>' || stride[j] == '}') --depth; + if (stride[j] == '<' || stride[j] == '{') + ++depth; + if (stride[j] == '>' || stride[j] == '}') + --depth; ++j; } out += stridesAreLeaves(leaf) ? std::string("0") : stride.substr(i, j - i); @@ -220,56 +242,67 @@ cg::layout outerOf(const cg::composed_layout &l) { return l.layout_b(); } cg::composed_layout zeroUnitModes(cg::composed_layout l) { if (l.is_a_swizzle()) - return cg::composed_layout(l.swizzle_a(), l.offset(), zeroUnitModes(l.layout_b())); - return cg::composed_layout(l.layout_a(), l.offset(), zeroUnitModes(l.layout_b())); + return cg::composed_layout(l.swizzle_a(), l.offset(), + zeroUnitModes(l.layout_b())); + return cg::composed_layout(l.layout_a(), l.offset(), + zeroUnitModes(l.layout_b())); } template LogicalResult infer(std::optional loc, SmallVectorImpl &out, F &&f) { Type t = f(); - if (!t) return emitOptionalError(loc, "cannot infer the result type"); + if (!t) + return emitOptionalError(loc, "cannot infer the result type"); out.push_back(t); return success(); } // `like`'s nesting with its leaves replaced, in order, by `leaves`. template -std::string nestLike(const T &like, llvm::ArrayRef leaves, size_t &i) { - if (!cg::holds_vector(like)) return i < leaves.size() ? leaves[i++] : "x"; +std::string nestLike(const T &like, llvm::ArrayRef leaves, + size_t &i) { + if (!cg::holds_vector(like)) + return i < leaves.size() ? leaves[i++] : "x"; std::string s = "("; for (size_t k = 0; k < cg::rank(like); ++k) s += (k ? "," : "") + nestLike(like[k], leaves, i); return s + ")"; } -template -void leavesOf(const T &t, std::vector &out) { - if (!cg::holds_vector(t)) return out.push_back(cg::to_string(t)); - for (size_t k = 0; k < cg::rank(t); ++k) leavesOf(t[k], out); +template void leavesOf(const T &t, std::vector &out) { + if (!cg::holds_vector(t)) + return out.push_back(cg::to_string(t)); + for (size_t k = 0; k < cg::rank(t); ++k) + leavesOf(t[k], out); } bool isOne(const std::string &leaf) { return leaf == "1"; } // The plain layout of a layout, or the outer layout of a composed one. std::optional outerLayout(Type t) { - if (auto l = llvm::dyn_cast(t)) return l.getRef(); - if (auto c = llvm::dyn_cast(t)) return c.getRef().layout_b(); + if (auto l = llvm::dyn_cast(t)) + return l.getRef(); + if (auto c = llvm::dyn_cast(t)) + return c.getRef().layout_b(); return std::nullopt; } // The plain layout of a layout or swizzled composed layout. std::optional swizzledOuter(Type t) { - if (auto l = llvm::dyn_cast(t)) return l.getRef(); + if (auto l = llvm::dyn_cast(t)) + return l.getRef(); auto c = llvm::dyn_cast(t); - if (c && c.getRef().is_a_swizzle()) return c.getRef().layout_b(); + if (c && c.getRef().is_a_swizzle()) + return c.getRef().layout_b(); return std::nullopt; } // A dynamic integer's spelling: `?`, with its width when 64 and its // divisibility when above 1. std::string dynamicInt(int64_t div, bool wide) { - std::string props = std::string(wide ? "i64" : "") + (wide && div > 1 ? " " : "") + + std::string props = std::string(wide ? "i64" : "") + + (wide && div > 1 ? " " : "") + (div > 1 ? "div=" + std::to_string(div) : ""); return props.empty() ? "?" : "?{" + props + "}"; } @@ -278,11 +311,13 @@ std::string dynamicInt(int64_t div, bool wide) { // dynamic index's divisibility tells of each coordinate, as the compiler // does (cutegen drops it): a divisible mode's coordinate is 0; a mode whose // extent the divisibility divides keeps it; the rest know nothing. -std::optional splitIndex(const cg::int_tuple &idx, const cg::shape &shape) { +std::optional splitIndex(const cg::int_tuple &idx, + const cg::shape &shape) { std::vector leaves, crd; leavesOf(shape, leaves); bool known = cg::holds_int(idx); - if (!known && !cg::holds_dynamic_int(idx)) return std::nullopt; + if (!known && !cg::holds_dynamic_int(idx)) + return std::nullopt; int64_t v = known ? idx.as_int64() : 0, div = 1; bool wide = false; if (!known) { @@ -295,7 +330,8 @@ std::optional splitIndex(const cg::int_tuple &idx, const cg::shape bool staticExtent = !StringRef(leaves[i]).getAsInteger(10, n) && n > 0; bool last = i + 1 == leaves.size(); if (known) { - if (!staticExtent) return std::nullopt; + if (!staticExtent) + return std::nullopt; crd.push_back(std::to_string(last ? v : v % n)); v /= n; } else if (last) { @@ -311,24 +347,29 @@ std::optional splitIndex(const cg::int_tuple &idx, const cg::shape size_t i = 0; return nestLike(shape, crd, i); } -} // namespace +} // namespace // ---------------------------------------------------------------- cute -LogicalResult AddOffsetOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { +LogicalResult AddOffsetOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { Adaptor a(operands, attrs, props); auto offset = llvm::dyn_cast(a.getOffset().getType()); - if (!offset) return emitOptionalError(loc, "offset must be an int tuple"); + if (!offset) + return emitOptionalError(loc, "offset must be an int tuple"); return infer(loc, out, [&]() -> Type { Type src = a.getSrc().getType(); - if (auto p = llvm::dyn_cast(src)) return movedPtr(p, offset.getRef()); + if (auto p = llvm::dyn_cast(src)) + return movedPtr(p, offset.getRef()); if (auto it = llvm::dyn_cast(src)) { auto sum = cg::arith_tuple_sum(it.getArithTuple().getRef(), offset.getRef()); - if (!cg::is_valid(sum)) return {}; + if (!cg::is_valid(sum)) + return {}; return ArithTupleIteratorType::get(ctx, IntTupleType::get(ctx, sum)); } return {}; @@ -346,51 +387,65 @@ LogicalResult inferToRank(MLIRContext *ctx, std::optional loc, int32_t rank = a.getRank(); Type in = a.getInput().getType(), e = a.getElement().getType(); return infer(loc, out, [&]() -> Type { - if (rank <= 0) return {}; + if (rank <= 0) + return {}; return llvm::TypeSwitch(in) .Case([&](auto ty) -> Type { using T = decltype(ty); // A composed layout is padded with plain layouts. - using E = std::conditional_t, LayoutType, T>; + using E = std::conditional_t, + LayoutType, T>; auto el = llvm::dyn_cast(e); - if (!el) return {}; - auto r = append ? cg::append_to_rank_N(rank, ty.getRef(), el.getRef()) - : cg::prepend_to_rank_N(rank, ty.getRef(), el.getRef()); - if (!cg::is_valid(r)) return {}; + if (!el) + return {}; + auto r = append + ? cg::append_to_rank_N(rank, ty.getRef(), el.getRef()) + : cg::prepend_to_rank_N(rank, ty.getRef(), el.getRef()); + if (!cg::is_valid(r)) + return {}; return T::get(ctx, std::move(r)); }) .Default([](Type) -> Type { return {}; }); }); } -} // namespace +} // namespace -LogicalResult AppendToRankOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { - return inferToRank(ctx, loc, operands, attrs, props, out, true); +LogicalResult AppendToRankOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferToRank(ctx, loc, operands, attrs, props, out, + true); } -LogicalResult PrependToRankOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { - return inferToRank(ctx, loc, operands, attrs, props, out, false); +LogicalResult PrependToRankOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferToRank(ctx, loc, operands, attrs, props, out, + false); } Type mlir::cutlass_compiler::cute::assumeResultType(Type src) { auto constrained = llvm::dyn_cast(src); for (unsigned width : {64u, 32u}) - if (src.isInteger(width) || (constrained && constrained.getWidth() == width)) + if (src.isInteger(width) || + (constrained && constrained.getWidth() == width)) return ConstrainedIntType::get(src.getContext(), 1, width, false); return {}; } -LogicalResult CeilDivOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { +LogicalResult CeilDivOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { Adaptor a(operands, attrs, props); Type in = a.getInput().getType(), tiler = a.getTiler().getType(); return infer(loc, out, [&]() -> Type { @@ -415,26 +470,32 @@ LogicalResult CeilDivOp::inferReturnTypes( } else { return {}; } - if (!cg::is_valid(r)) return {}; - if (tuple) return IntTupleType::get(ctx, cg::rec_var_cast(r)); + if (!cg::is_valid(r)) + return {}; + if (tuple) + return IntTupleType::get(ctx, cg::rec_var_cast(r)); return ShapeType::get(ctx, std::move(r)); }); } -LogicalResult CoalesceOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { +LogicalResult CoalesceOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { Adaptor a(operands, attrs, props); auto c = carrierOf(a.getInput().getType()); - if (!c) return emitOptionalError(loc, "input carries no layout"); + if (!c) + return emitOptionalError(loc, "input carries no layout"); CoordType profile; if (a.getTargetProfile()) profile = llvm::dyn_cast(a.getTargetProfile().getType()); return infer(loc, out, [&]() -> Type { return rebuild(ctx, *c, mapLayout(ctx, c->layout, [&](const auto &l) { - return profile ? cg::coalesce(l, profile.getRef()) : cg::coalesce(l); - })); + return profile ? cg::coalesce(l, profile.getRef()) + : cg::coalesce(l); + })); }); } @@ -443,40 +504,49 @@ LogicalResult ComposedGetOuterOp::inferReturnTypes( DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { return infer(loc, out, [&]() -> Type { auto c = llvm::dyn_cast(operands[0].getType()); - if (!c) return {}; + if (!c) + return {}; return LayoutType::get(ctx, c.getRef().layout_b()); }); } -LogicalResult CompositionOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { +LogicalResult CompositionOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { Adaptor a(operands, attrs, props); auto c = carrierOf(a.getLhs().getType()); - if (!c) return emitOptionalError(loc, "lhs carries no layout"); + if (!c) + return emitOptionalError(loc, "lhs carries no layout"); Type rhs = a.getRhs().getType(); return infer(loc, out, [&]() -> Type { - return rebuild(ctx, *c, llvm::TypeSwitch(rhs) - .Case([&](auto r) { - return mapLayout(ctx, c->layout, [&](const auto &l) { - return cg::composition(l, r.getRef()); - }); - }) - .Default([](Type) -> Type { return {}; })); + return rebuild(ctx, *c, + llvm::TypeSwitch(rhs) + .Case([&](auto r) { + return mapLayout(ctx, c->layout, [&](const auto &l) { + return cg::composition(l, r.getRef()); + }); + }) + .Default([](Type) -> Type { return {}; })); }); } -LogicalResult Crd2IdxOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { +LogicalResult Crd2IdxOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { Adaptor a(operands, attrs, props); auto coord = llvm::dyn_cast(a.getCoord().getType()); - if (!coord) return emitOptionalError(loc, "coord must be a coord"); + if (!coord) + return emitOptionalError(loc, "coord must be a coord"); return infer(loc, out, [&]() -> Type { auto c = zeroUnderscores(coord.getRef()); - if (!c) return {}; + if (!c) + return {}; Type layout = a.getLayout().getType(); cg::int_tuple idx; if (auto l = llvm::dyn_cast(layout)) { @@ -486,7 +556,8 @@ LogicalResult Crd2IdxOp::inferReturnTypes( } else { return {}; } - if (!cg::is_valid(idx)) return {}; + if (!cg::is_valid(idx)) + return {}; return IntTupleType::get(ctx, std::move(idx)); }); } @@ -500,25 +571,29 @@ LogicalResult DereferenceArithTupleIteratorOp::inferReturnTypes( }); } -LogicalResult DiceOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { +LogicalResult DiceOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange, + SmallVectorImpl &out) { Adaptor a(operands, attrs, props); const cg::coord &coord = a.getCoordAttr().getRef(); Type in = a.getInput().getType(); return infer(loc, out, [&]() -> Type { if (auto c = carrierOf(in)) { return rebuild(ctx, *c, mapLayout(ctx, c->layout, [&](const auto &l) { - return cg::dice(coord, l); - })); + return cg::dice(coord, l); + })); } return llvm::TypeSwitch(in) - .Case([&](auto ty) -> Type { - auto r = cg::dice(coord, ty.getRef()); - if (!cg::is_valid(r)) return {}; - return decltype(ty)::get(ctx, std::move(r)); - }) + .Case( + [&](auto ty) -> Type { + auto r = cg::dice(coord, ty.getRef()); + if (!cg::is_valid(r)) + return {}; + return decltype(ty)::get(ctx, std::move(r)); + }) .Default([](Type) -> Type { return {}; }); }); } @@ -528,22 +603,28 @@ Type mlir::cutlass_compiler::cute::fastDivmodComputeResultType(Type divisor) { return d ? IntegerType::get(divisor.getContext(), d.getWidth()) : Type(); } -Type mlir::cutlass_compiler::cute::fastDivmodCreateDivisorResultType(Type divisor) { +Type mlir::cutlass_compiler::cute::fastDivmodCreateDivisorResultType( + Type divisor) { auto i = llvm::dyn_cast(divisor); - return i ? FastDivmodDivisorType::get(divisor.getContext(), i.getWidth(), false) : Type(); + return i ? FastDivmodDivisorType::get(divisor.getContext(), i.getWidth(), + false) + : Type(); } -LogicalResult FilterZerosOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { +LogicalResult FilterZerosOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { Adaptor a(operands, attrs, props); auto c = carrierOf(a.getInput().getType()); - if (!c) return emitOptionalError(loc, "input carries no layout"); + if (!c) + return emitOptionalError(loc, "input carries no layout"); return infer(loc, out, [&]() -> Type { return rebuild(ctx, *c, mapLayout(ctx, c->layout, [&](const auto &l) { - return cg::filter_zeros(l); - })); + return cg::filter_zeros(l); + })); }); } @@ -551,22 +632,29 @@ namespace { // zeroUnitModes on all but the last `keep` modes (of mode 1 when `zipped`): // the modes a divide leaves untouched keep their strides. cg::layout zeroUnitModesHead(const cg::layout &l, size_t keep, bool zipped) { - if (!keep || !cg::holds_vector(l.shape())) return zeroUnitModes(l); + if (!keep || !cg::holds_vector(l.shape())) + return zeroUnitModes(l); if (zipped) { - if (cg::rank(l.shape()) != 2) return zeroUnitModes(l); - return cg::make_layout(std::vector{ - zeroUnitModes(cg::get(l, 0)), zeroUnitModesHead(cg::get(l, 1), keep, false)}); + if (cg::rank(l.shape()) != 2) + return zeroUnitModes(l); + return cg::make_layout( + std::vector{zeroUnitModes(cg::get(l, 0)), + zeroUnitModesHead(cg::get(l, 1), keep, false)}); } size_t n = cg::rank(l.shape()); - if (keep > n) return l; + if (keep > n) + return l; std::vector modes; for (size_t i = 0; i < n; ++i) - modes.push_back(i < n - keep ? zeroUnitModes(cg::get(l, i)) : cg::get(l, i)); + modes.push_back(i < n - keep ? zeroUnitModes(cg::get(l, i)) + : cg::get(l, i)); return cg::make_layout(modes); } -cg::composed_layout zeroUnitModesHead(const cg::composed_layout &l, size_t keep, bool zipped) { +cg::composed_layout zeroUnitModesHead(const cg::composed_layout &l, size_t keep, + bool zipped) { cg::layout b = zeroUnitModesHead(l.layout_b(), keep, zipped); - if (l.is_a_swizzle()) return cg::composed_layout(l.swizzle_a(), l.offset(), b); + if (l.is_a_swizzle()) + return cg::composed_layout(l.swizzle_a(), l.offset(), b); return cg::composed_layout(l.layout_a(), l.offset(), b); } @@ -577,83 +665,110 @@ LogicalResult inferDivide(MLIRContext *ctx, std::optional loc, F &÷, bool zipped = false) { typename Op::Adaptor a(operands, attrs, props); auto c = carrierOf(a.getInput().getType()); - if (!c) return emitOptionalError(loc, "input carries no layout"); + if (!c) + return emitOptionalError(loc, "input carries no layout"); Type tiler = a.getTiler().getType(); return infer(loc, out, [&]() -> Type { - return rebuild(ctx, *c, llvm::TypeSwitch(tiler) - .template Case([&](auto t) { - // Unit modes the tiler passes through (`_` or extent 1) keep their - // nonzero strides, which zeroUnitModes does not model: not covered. - if constexpr (std::is_same_v) { - auto outer = outerLayout(c->layout); - for (size_t i = 0; outer && cg::holds_vector(t.getRef()) && i < cg::rank(t.getRef()) && - cg::holds_vector(outer->shape()) && i < cg::rank(outer->shape()); - ++i) { - std::string m = cg::to_string(t.getRef()[i]); - auto tl = cg::from_string(m); - bool pass = m == "_" || (tl && cg::holds_int(cg::size(*tl)) && - cg::size(*tl).as_int64() == 1 && cg::to_string(tl->stride()) != "0"); - std::vector shapes, strides; - leavesOf(outer->shape()[i], shapes); - leavesOf(outer->stride()[i], strides); - for (size_t k = 0; pass && k < shapes.size() && k < strides.size(); ++k) - if (shapes[k] == "1" && strides[k] != "0") return Type(); - } - } - return mapLayout(ctx, c->layout, [&](const auto &l) { - // A tile or shape tiler leaves the input's trailing modes alone. - size_t keep = 0; - if constexpr (!std::is_same_v) { - size_t in = cg::rank(outerOf(l).shape()), r = cg::rank(t.getRef()); - if (cg::holds_vector(outerOf(l).shape()) && r < in) keep = in - r; - } - return zeroUnitModesHead(divide(l, t.getRef()), keep, zipped); - }); - }) - .Default([](Type) -> Type { return {}; })); + return rebuild( + ctx, *c, + llvm::TypeSwitch(tiler) + .template Case([&](auto t) { + // Unit modes the tiler passes through (`_` or extent 1) keep + // their nonzero strides, which zeroUnitModes does not model: not + // covered. + if constexpr (std::is_same_v) { + auto outer = outerLayout(c->layout); + for (size_t i = 0; outer && cg::holds_vector(t.getRef()) && + i < cg::rank(t.getRef()) && + cg::holds_vector(outer->shape()) && + i < cg::rank(outer->shape()); + ++i) { + std::string m = cg::to_string(t.getRef()[i]); + auto tl = cg::from_string(m); + bool pass = m == "_" || (tl && cg::holds_int(cg::size(*tl)) && + cg::size(*tl).as_int64() == 1 && + cg::to_string(tl->stride()) != "0"); + std::vector shapes, strides; + leavesOf(outer->shape()[i], shapes); + leavesOf(outer->stride()[i], strides); + for (size_t k = 0; + pass && k < shapes.size() && k < strides.size(); ++k) + if (shapes[k] == "1" && strides[k] != "0") + return Type(); + } + } + return mapLayout(ctx, c->layout, [&](const auto &l) { + // A tile or shape tiler leaves the input's trailing modes + // alone. + size_t keep = 0; + if constexpr (!std::is_same_v) { + size_t in = cg::rank(outerOf(l).shape()), + r = cg::rank(t.getRef()); + if (cg::holds_vector(outerOf(l).shape()) && r < in) + keep = in - r; + } + return zeroUnitModesHead(divide(l, t.getRef()), keep, zipped); + }); + }) + .Default([](Type) -> Type { return {}; })); }); } -} // namespace +} // namespace -LogicalResult FlatDivideOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { - return inferDivide(ctx, loc, operands, attrs, props, out, +LogicalResult FlatDivideOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferDivide( + ctx, loc, operands, attrs, props, out, [](const auto &l, const auto &t) { return cg::flat_divide(l, t); }); } -LogicalResult LogicalDivideOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { - return inferDivide(ctx, loc, operands, attrs, props, out, +LogicalResult LogicalDivideOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferDivide( + ctx, loc, operands, attrs, props, out, [](const auto &l, const auto &t) { return cg::logical_divide(l, t); }); } -LogicalResult TiledDivideOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { - return inferDivide(ctx, loc, operands, attrs, props, out, +LogicalResult TiledDivideOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferDivide( + ctx, loc, operands, attrs, props, out, [](const auto &l, const auto &t) { return cg::tiled_divide(l, t); }); } -LogicalResult ZippedDivideOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { - return inferDivide(ctx, loc, operands, attrs, props, out, - [](const auto &l, const auto &t) { return cg::zipped_divide(l, t); }, true); +LogicalResult ZippedDivideOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferDivide( + ctx, loc, operands, attrs, props, out, + [](const auto &l, const auto &t) { return cg::zipped_divide(l, t); }, + true); } -Type mlir::cutlass_compiler::cute::getResultType(Type in, ArrayRef mode) { +Type mlir::cutlass_compiler::cute::getResultType(Type in, + ArrayRef mode) { MLIRContext *ctx = in.getContext(); return llvm::TypeSwitch(in) .Case([&](auto ty) -> Type { auto r = cg::get(ty.getRef(), mode); - if (!cg::is_valid(r)) return {}; + if (!cg::is_valid(r)) + return {}; if constexpr (std::is_same_v) return layoutType(ctx, std::move(r)); else @@ -662,31 +777,39 @@ Type mlir::cutlass_compiler::cute::getResultType(Type in, ArrayRef mode .Default([](Type) -> Type { return {}; }); } -LogicalResult GetIterOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { +LogicalResult GetIterOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { return infer(loc, out, [&]() -> Type { Type src = operands[0].getType(); - if (auto m = llvm::dyn_cast(src)) return m.getPtr(); + if (auto m = llvm::dyn_cast(src)) + return m.getPtr(); if (auto c = llvm::dyn_cast(src)) return ArithTupleIteratorType::get(ctx, c.getArithTuple()); - if (auto v = llvm::dyn_cast(src)) return v.getDesc(); + if (auto v = llvm::dyn_cast(src)) + return v.getDesc(); return {}; }); } -LogicalResult GetLayoutOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { +LogicalResult GetLayoutOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { return infer(loc, out, [&]() -> Type { auto c = carrierOf(operands[0].getType()); return c ? c->layout : Type{}; }); } -LogicalResult GetLeavesOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { +LogicalResult GetLeavesOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { return llvm::TypeSwitch(operands[0].getType()) .Case( [&](auto ty) -> LogicalResult { @@ -714,10 +837,12 @@ LogicalResult GetLeavesOp::inferReturnTypes( }); } -LogicalResult GetScalarsOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { +LogicalResult GetScalarsOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { Adaptor a(operands, attrs, props); bool onlyDynamic = a.getOnlyDynamic(); return llvm::TypeSwitch(a.getCuteValue().getType()) @@ -733,15 +858,18 @@ LogicalResult GetScalarsOp::inferReturnTypes( }); } -LogicalResult GetShapeOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { +LogicalResult GetShapeOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { return infer(loc, out, [&]() -> Type { Type in = operands[0].getType(); if (auto t = llvm::dyn_cast(in)) return ShapeType::get(ctx, cg::extract_shape_from_tile(t.getRef())); auto c = carrierOf(in); - if (!c) return {}; + if (!c) + return {}; if (auto l = llvm::dyn_cast(c->layout)) return ShapeType::get(ctx, l.getRef().shape()); if (auto l = llvm::dyn_cast(c->layout)) @@ -750,31 +878,38 @@ LogicalResult GetShapeOp::inferReturnTypes( }); } -LogicalResult GroupModesOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { +LogicalResult GroupModesOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { Adaptor a(operands, attrs, props); auto c = carrierOf(a.getInput().getType()); - if (!c) return emitOptionalError(loc, "input carries no layout"); - int begin = static_cast(a.getBegin()), end = static_cast(a.getEnd()); + if (!c) + return emitOptionalError(loc, "input carries no layout"); + int begin = static_cast(a.getBegin()), + end = static_cast(a.getEnd()); return infer(loc, out, [&]() -> Type { return rebuild(ctx, *c, mapLayout(ctx, c->layout, [&](const auto &l) { - return cg::group(begin, end, l); - })); + return cg::group(begin, end, l); + })); }); } -LogicalResult LocalTileOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { +LogicalResult LocalTileOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { Adaptor a(operands, attrs, props); auto c = carrierOf(a.getInput().getType()); auto tile = llvm::dyn_cast(a.getTile().getType()); auto coord = llvm::dyn_cast(a.getCoord().getType()); if (!c || !tile || !coord) - return emitOptionalError(loc, "expects a layout carrier, a tile and a coord"); + return emitOptionalError(loc, + "expects a layout carrier, a tile and a coord"); return infer(loc, out, [&]() -> Type { // CuTe's inner_partition: divide by the tile, take the coordinate in the // rest modes, and flatten the two modes into one rank. @@ -785,12 +920,15 @@ LogicalResult LocalTileOp::inferReturnTypes( crd = cg::dice(proj.getRef(), crd); } auto layout = llvm::dyn_cast(c->layout); - if (!layout) return {}; + if (!layout) + return {}; cg::layout zipped = cg::zipped_divide(layout.getRef(), tiler); - if (!cg::is_valid(zipped)) return {}; + if (!cg::is_valid(zipped)) + return {}; cg::layout tiles = cg::get(zipped, 0), rest = cg::get(zipped, 1); auto [sliced, offset] = cg::slice_and_offset(crd, rest); - if (!cg::is_valid(sliced)) return {}; + if (!cg::is_valid(sliced)) + return {}; // One rank: the tile modes, then what is left of the rest modes. std::vector modes; for (size_t i = 0; i < cg::rank(tiles.shape()); ++i) @@ -803,15 +941,19 @@ LogicalResult LocalTileOp::inferReturnTypes( } // The input's modes beyond the tiler that the coordinate keeps are // untouched and keep their strides. - size_t keep = 0, in = cg::rank(layout.getRef().shape()), r = cg::rank(tiler); - for (size_t i = r; cg::holds_vector(crd) && i < in && i < cg::rank(crd); ++i) + size_t keep = 0, in = cg::rank(layout.getRef().shape()), + r = cg::rank(tiler); + for (size_t i = r; cg::holds_vector(crd) && i < in && i < cg::rank(crd); + ++i) keep += cg::to_string(crd[i]) == "_"; cg::layout flat = zeroUnitModesHead(cg::make_layout(modes), keep, false); Type result = layoutType(ctx, std::move(flat)); - if (c->ptr) return CuteMemRefType::get(ctx, movedPtr(c->ptr, offset), result); + if (c->ptr) + return CuteMemRefType::get(ctx, movedPtr(c->ptr, offset), result); if (c->iter) { auto sum = cg::arith_tuple_sum(c->iter.getRef(), offset); - if (!cg::is_valid(sum)) return {}; + if (!cg::is_valid(sum)) + return {}; return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), result); } @@ -836,9 +978,13 @@ LogicalResult MakeComposedLayoutOp::inferReturnTypes( return infer(loc, out, [&]() -> Type { Type inner = a.getInner().getType(); if (auto l = llvm::dyn_cast(inner)) - return ComposedLayoutType::get(ctx, cg::composed_layout(l.getRef(), offset.getRef(), outer.getRef())); + return ComposedLayoutType::get( + ctx, + cg::composed_layout(l.getRef(), offset.getRef(), outer.getRef())); if (auto s = llvm::dyn_cast(inner)) - return ComposedLayoutType::get(ctx, cg::composed_layout(s.getRef(), offset.getRef(), outer.getRef())); + return ComposedLayoutType::get( + ctx, + cg::composed_layout(s.getRef(), offset.getRef(), outer.getRef())); return {}; }); } @@ -848,7 +994,8 @@ LogicalResult MakeIdentityLayoutOp::inferReturnTypes( DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { return infer(loc, out, [&]() -> Type { auto s = llvm::dyn_cast(operands[0].getType()); - if (!s) return {}; + if (!s) + return {}; cg::layout l = cg::make_identity_layout(s.getRef()); return cg::is_valid(l) ? LayoutType::get(ctx, std::move(l)) : Type{}; }); @@ -861,7 +1008,8 @@ LogicalResult MakeOrderedLayoutOp::inferReturnTypes( Adaptor a(operands, attrs, props); auto shape = llvm::dyn_cast(a.getShape().getType()); auto order = llvm::dyn_cast(a.getOrder().getType()); - if (!shape || !order) return emitOptionalError(loc, "expects a shape and an order"); + if (!shape || !order) + return emitOptionalError(loc, "expects a shape and an order"); return infer(loc, out, [&]() -> Type { cg::layout l(shape.getRef(), cg::compact_order(shape.getRef(), order.getRef())); @@ -871,8 +1019,10 @@ LogicalResult MakeOrderedLayoutOp::inferReturnTypes( Type mlir::cutlass_compiler::cute::makeViewResultType(Type iter, Type layout) { MLIRContext *ctx = iter.getContext(); - if (!llvm::isa(layout)) return {}; - if (auto p = llvm::dyn_cast(iter)) return CuteMemRefType::get(ctx, p, layout); + if (!llvm::isa(layout)) + return {}; + if (auto p = llvm::dyn_cast(iter)) + return CuteMemRefType::get(ctx, p, layout); if (auto it = llvm::dyn_cast(iter)) return CoordTensorType::get(ctx, it.getArithTuple(), layout); auto plain = llvm::dyn_cast(layout); @@ -881,81 +1031,101 @@ Type mlir::cutlass_compiler::cute::makeViewResultType(Type iter, Type layout) { return {}; } -LogicalResult MemRefLoadOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { +LogicalResult +MemRefLoadOp::inferReturnTypes(MLIRContext *ctx, std::optional loc, + ValueRange operands, DictionaryAttr, PropertyRef, + RegionRange, SmallVectorImpl &out) { return infer(loc, out, [&]() -> Type { auto m = llvm::dyn_cast(operands[0].getType()); return m ? m.getValueType() : Type{}; }); } -LogicalResult MemRefLoadVecOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { +LogicalResult MemRefLoadVecOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { Adaptor a(operands, attrs, props); return infer(loc, out, [&]() -> Type { auto m = llvm::dyn_cast(a.getSrc().getType()); - if (!m) return {}; + if (!m) + return {}; auto l = llvm::dyn_cast(m.getLayout()); - if (!l) return {}; + if (!l) + return {}; auto n = cg::size(l.getRef()); - if (!cg::holds_int(n)) return {}; + if (!cg::holds_int(n)) + return {}; return VectorType::get({n.as_int64()}, m.getValueType()); }); } -LogicalResult SizeOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { +LogicalResult SizeOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange, + SmallVectorImpl &out) { Adaptor a(operands, attrs, props); ArrayRef mode; - if (auto m = a.getModeAttr()) mode = m.asArrayRef(); + if (auto m = a.getModeAttr()) + mode = m.asArrayRef(); Type in = a.getInput().getType(); - if (auto c = carrierOf(in)) in = c->layout; + if (auto c = carrierOf(in)) + in = c->layout; return infer(loc, out, [&]() -> Type { return llvm::TypeSwitch(in) .Case( [&](auto ty) -> Type { auto r = cg::rec_var_cast( cg::size(cg::get(ty.getRef(), mode))); - if (!cg::is_valid(r)) return {}; + if (!cg::is_valid(r)) + return {}; return IntTupleType::get(ctx, std::move(r)); }) .Default([](Type) -> Type { return {}; }); }); } -LogicalResult SliceOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { +LogicalResult SliceOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange, + SmallVectorImpl &out) { Adaptor a(operands, attrs, props); auto coord = llvm::dyn_cast(a.getCoord().getType()); - if (!coord) return emitOptionalError(loc, "coord must be a coord"); + if (!coord) + return emitOptionalError(loc, "coord must be a coord"); Type in = a.getInput().getType(); return infer(loc, out, [&]() -> Type { - if (auto c = carrierOf(in)) return sliceCarrier(ctx, *c, coord.getRef()); + if (auto c = carrierOf(in)) + return sliceCarrier(ctx, *c, coord.getRef()); return llvm::TypeSwitch(in) - .Case([&](auto ty) -> Type { - auto r = cg::slice(coord.getRef(), ty.getRef()); - if (!cg::is_valid(r)) return {}; - return decltype(ty)::get(ctx, std::move(r)); - }) + .Case( + [&](auto ty) -> Type { + auto r = cg::slice(coord.getRef(), ty.getRef()); + if (!cg::is_valid(r)) + return {}; + return decltype(ty)::get(ctx, std::move(r)); + }) .Default([](Type) -> Type { return {}; }); }); } -LogicalResult TileToShapeOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { +LogicalResult TileToShapeOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { Adaptor a(operands, attrs, props); auto shape = llvm::dyn_cast(a.getTrgShape().getType()); auto order = llvm::dyn_cast(a.getOrdShape().getType()); - if (!shape || !order) return emitOptionalError(loc, "expects a shape and an order"); + if (!shape || !order) + return emitOptionalError(loc, "expects a shape and an order"); return infer(loc, out, [&]() -> Type { return mapLayout(ctx, a.getBlock().getType(), [&](const auto &l) { return cg::tile_to_shape(l, shape.getRef(), order.getRef()); @@ -963,14 +1133,17 @@ LogicalResult TileToShapeOp::inferReturnTypes( }); } -LogicalResult ToIntTupleOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { +LogicalResult +ToIntTupleOp::inferReturnTypes(MLIRContext *ctx, std::optional loc, + ValueRange operands, DictionaryAttr, PropertyRef, + RegionRange, SmallVectorImpl &out) { return infer(loc, out, [&]() -> Type { return llvm::TypeSwitch(operands[0].getType()) - .Case([&](auto ty) -> Type { - return IntTupleType::get(ctx, cg::rec_var_cast(ty.getRef())); - }) + .Case( + [&](auto ty) -> Type { + return IntTupleType::get( + ctx, cg::rec_var_cast(ty.getRef())); + }) .Default([](Type) -> Type { return {}; }); }); } @@ -985,51 +1158,67 @@ LogicalResult inferTupleArith(MLIRContext *ctx, std::optional loc, .Case([&](auto lhs) -> Type { using T = decltype(lhs); auto rhs = llvm::dyn_cast(operands[1].getType()); - if (!rhs) return {}; - auto r = f.template operator()(lhs.getRef(), rhs.getRef()); - if (!cg::is_valid(r)) return {}; + if (!rhs) + return {}; + auto r = f.template operator()(lhs.getRef(), + rhs.getRef()); + if (!cg::is_valid(r)) + return {}; return T::get(ctx, std::move(r)); }) .Default([](Type) -> Type { return {}; }); }); } -} // namespace +} // namespace -LogicalResult TupleAddOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { - return inferTupleArith(ctx, loc, operands, out, [](const auto &a, const auto &b) { - return cg::arith_tuple_sum(a, b); - }); +LogicalResult TupleAddOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return inferTupleArith(ctx, loc, operands, out, + [](const auto &a, const auto &b) { + return cg::arith_tuple_sum(a, b); + }); } -LogicalResult TupleMulOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { - return inferTupleArith(ctx, loc, operands, out, [](const auto &a, const auto &b) { - return cg::arith_tuple_mul(a, b); - }); +LogicalResult TupleMulOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return inferTupleArith(ctx, loc, operands, out, + [](const auto &a, const auto &b) { + return cg::arith_tuple_mul(a, b); + }); } -LogicalResult TupleDivOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { - return inferTupleArith(ctx, loc, operands, out, [](const auto &a, const auto &b) { - return cg::arith_tuple_div(a, b); - }); +LogicalResult TupleDivOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return inferTupleArith(ctx, loc, operands, out, + [](const auto &a, const auto &b) { + return cg::arith_tuple_div(a, b); + }); } // Without a stride the layout is compact column-major, unit modes at 0. -Type mlir::cutlass_compiler::cute::makeLayoutResultType(Type shape, Type stride) { +Type mlir::cutlass_compiler::cute::makeLayoutResultType(Type shape, + Type stride) { auto s = llvm::dyn_cast(shape); - if (!s) return {}; + if (!s) + return {}; MLIRContext *ctx = shape.getContext(); if (stride) { auto st = llvm::dyn_cast(stride); - if (!st) return {}; + if (!st) + return {}; return LayoutType::get(ctx, cg::layout(s.getRef(), st.getRef())); } - cg::layout l(s.getRef(), cg::compact_col_major(s.getRef())); + cg::layout l(s.getRef(), + cg::compact_col_major(s.getRef())); return LayoutType::get(ctx, zeroUnitModes(l)); } @@ -1037,12 +1226,13 @@ namespace { // `MxNxK` and `num_cta` of an SM100 block-scaled or plain mma atom. struct MmaAtom { int64_t m = 0, n = 0, k = 0, ctas = 1; - Type c; // accumulator element - bool aTmem = false; // frag_kind = ts + Type c; // accumulator element + bool aTmem = false; // frag_kind = ts }; std::optional mmaAtomOf(Type tiledMma) { auto tiled = llvm::dyn_cast(tiledMma); - if (!tiled) return std::nullopt; + if (!tiled) + return std::nullopt; // One atom, not permuted: other tilings are not covered here. auto n = cg::size(tiled.getAtomLayout_MNK().getRef()); TileAttr perm = tiled.getPermutation_MNK(); @@ -1051,12 +1241,14 @@ std::optional mmaAtomOf(Type tiledMma) { return std::nullopt; MmaAtom atom; ShapeAttr shape; - if (auto bs = llvm::dyn_cast(tiled.getMmaAtom())) { + if (auto bs = llvm::dyn_cast( + tiled.getMmaAtom())) { shape = bs.getShapeMnk(); atom.ctas = bs.getNumCta(); atom.c = bs.getCType(); atom.aTmem = bs.getAFragKind() == cute_nvgpu::MmaFragKind::tmem; - } else if (auto mma = llvm::dyn_cast(tiled.getMmaAtom())) { + } else if (auto mma = llvm::dyn_cast( + tiled.getMmaAtom())) { shape = mma.getShapeMnk(); atom.ctas = mma.getNumCta(); atom.c = mma.getCType(); @@ -1064,16 +1256,18 @@ std::optional mmaAtomOf(Type tiledMma) { } else { return std::nullopt; } - if (cg::rank(shape.getRef()) != 3) return std::nullopt; + if (cg::rank(shape.getRef()) != 3) + return std::nullopt; int64_t *mnk[] = {&atom.m, &atom.n, &atom.k}; for (int i = 0; i < 3; ++i) { auto e = cg::get(shape.getRef(), i); - if (!cg::holds_int(e)) return std::nullopt; + if (!cg::holds_int(e)) + return std::nullopt; *mnk[i] = e.as_int64(); } return atom; } -} // namespace +} // namespace // partition_A/B/C of the trivial tiled mma: the leading two modes are divided // by the per-CTA atom shape, the CTA's share of the first is taken at the @@ -1088,49 +1282,76 @@ LogicalResult TiledMmaPartitionOp::inferReturnTypes( auto coord = llvm::dyn_cast(a.getCoord().getType()); auto which = llvm::dyn_cast_or_null(a.getOperandIdAttr()); if (!atom || !c || !coord || !which) - return emitOptionalError(loc, "expects a tiled mma, a tensor, a coord and an operand"); + return emitOptionalError( + loc, "expects a tiled mma, a tensor, a coord and an operand"); return infer(loc, out, [&]() -> Type { auto layout = llvm::dyn_cast(c->layout); - if (!layout) return {}; + if (!layout) + return {}; int64_t a0, a1; switch (which.getInt()) { - case 0: a0 = atom->m / atom->ctas; a1 = atom->k; break; // A: (M, K) - case 1: a0 = atom->n / atom->ctas; a1 = atom->k; break; // B: (N, K) - case 2: a0 = atom->m / atom->ctas; a1 = atom->n; break; // C: (M, N) - default: return {}; + case 0: + a0 = atom->m / atom->ctas; + a1 = atom->k; + break; // A: (M, K) + case 1: + a0 = atom->n / atom->ctas; + a1 = atom->k; + break; // B: (N, K) + case 2: + a0 = atom->m / atom->ctas; + a1 = atom->n; + break; // C: (M, N) + default: + return {}; } const cg::layout &l = layout.getRef(); - if (cg::rank(l.shape()) < 2) return {}; + if (cg::rank(l.shape()) < 2) + return {}; auto tile = cg::from_string("[" + std::to_string(a0) + ":1;" + - std::to_string(a1) + ":1]"); - if (!tile) return {}; - cg::layout two = cg::make_layout(std::vector{cg::get(l, 0), cg::get(l, 1)}); + std::to_string(a1) + ":1]"); + if (!tile) + return {}; + cg::layout two = + cg::make_layout(std::vector{cg::get(l, 0), cg::get(l, 1)}); cg::layout divided = cg::zipped_divide(two, *tile); - if (!cg::is_valid(divided)) return {}; + if (!cg::is_valid(divided)) + return {}; cg::layout atoms = cg::get(divided, 0), rest = cg::get(divided, 1); // The first rest mode holds the CTA index innermost. - auto ctaTile = cg::from_string("[" + std::to_string(atom->ctas) + ":1]"); - if (!ctaTile) return {}; + auto ctaTile = + cg::from_string("[" + std::to_string(atom->ctas) + ":1]"); + if (!ctaTile) + return {}; cg::layout rest0 = cg::zipped_divide(cg::get(rest, 0), *ctaTile); - if (!cg::is_valid(rest0) || cg::rank(rest0.shape()) != 2) return {}; + if (!cg::is_valid(rest0) || cg::rank(rest0.shape()) != 2) + return {}; // A lone CTA has nothing to select: its coordinate is 0 whatever the // thread index says. - std::string which_cta = atom->ctas == 1 ? "0" : cg::to_string(coord.getRef()); + std::string which_cta = + atom->ctas == 1 ? "0" : cg::to_string(coord.getRef()); auto [rest0Sliced, offset] = cg::slice_and_offset( *cg::from_string("(" + which_cta + ",_)"), rest0); - if (!cg::is_valid(rest0Sliced)) return {}; + if (!cg::is_valid(rest0Sliced)) + return {}; // What the slice leaves is one mode, not a tuple around it. - while (cg::holds_vector(rest0Sliced.shape()) && cg::rank(rest0Sliced.shape()) == 1) + while (cg::holds_vector(rest0Sliced.shape()) && + cg::rank(rest0Sliced.shape()) == 1) rest0Sliced = cg::get(rest0Sliced, 0); std::vector modes{atoms, rest0Sliced, cg::get(rest, 1)}; - for (size_t i = 2; i < cg::rank(l.shape()); ++i) modes.push_back(cg::get(l, i)); + for (size_t i = 2; i < cg::rank(l.shape()); ++i) + modes.push_back(cg::get(l, i)); // Modes beyond M and K (or N) are untouched. - Type result = layoutType(ctx, zeroUnitModesHead(cg::make_layout(modes), modes.size() - 3, false)); - if (c->ptr) return CuteMemRefType::get(ctx, movedPtr(c->ptr, offset), result); + Type result = layoutType(ctx, zeroUnitModesHead(cg::make_layout(modes), + modes.size() - 3, false)); + if (c->ptr) + return CuteMemRefType::get(ctx, movedPtr(c->ptr, offset), result); if (c->iter) { auto sum = cg::arith_tuple_sum(c->iter.getRef(), offset); - if (!cg::is_valid(sum)) return {}; - return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), result); + if (!cg::is_valid(sum)) + return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), + result); } return result; }); @@ -1148,16 +1369,20 @@ LogicalResult MmaMakeFragmentOp::inferReturnTypes( auto atom = mmaAtomOf(a.getAtom().getType()); // A in tensor memory (frag_kind = ts) is not covered. if (!which || !atom || (which.getInt() == 0 && atom->aTmem)) - return emitOptionalError(loc, "expects an SM100 tiled mma and an operand id"); + return emitOptionalError(loc, + "expects an SM100 tiled mma and an operand id"); Type in = a.getInput().getType(); return infer(loc, out, [&]() -> Type { if (which.getInt() == 2) { auto shape = llvm::dyn_cast(in); - if (!shape) return {}; + if (!shape) + return {}; const cg::shape &s = shape.getRef(); - if (cg::rank(s) < 1 || !cg::holds_vector(cg::get(s, 0))) return {}; + if (cg::rank(s) < 1 || !cg::holds_vector(cg::get(s, 0))) + return {}; auto m = cg::size(cg::get(s, 0), 0), n = cg::size(cg::get(s, 0), 1); - if (!cg::holds_int(m) || !cg::holds_int(n)) return {}; + if (!cg::holds_int(m) || !cg::holds_int(n)) + return {}; // ((M,N), rest...): rows on lanes, columns contiguous, stages by N // columns; unit modes at 0. A 64-row share of a 2-CTA atom puts the // second half of the columns on lanes 64 and up. @@ -1166,7 +1391,8 @@ LogicalResult MmaMakeFragmentOp::inferReturnTypes( std::string text = "((" + cg::to_string(m) + "," + cg::to_string(n) + ")", stride = "((65536,1)"; if (atom->m / atom->ctas == 64) { - if (atom->ctas != 2 || m.as_int64() != 64 || cols % 2) return {}; + if (atom->ctas != 2 || m.as_int64() != 64 || cols % 2) + return {}; cols /= 2; text = "((64,(" + std::to_string(cols) + ",2))"; stride = "((65536,(1,4194304))"; @@ -1177,76 +1403,105 @@ LogicalResult MmaMakeFragmentOp::inferReturnTypes( auto e = cg::size(s, i); text += "," + cg::to_string(cg::get(s, i)); stride += "," + ((cg::holds_int(e) && e.as_int64() == 1) - ? std::string("0") : std::to_string(cols)); + ? std::string("0") + : std::to_string(cols)); } auto l = cg::from_string(text + "):" + stride + ")"); - if (!l) return {}; + if (!l) + return {}; auto ptr = PtrType::get(ctx, atom->c, AddressSpace::tmem, 1, {}, {}); return CuteMemRefType::get(ctx, ptr, LayoutType::get(ctx, *l)); } auto m = llvm::dyn_cast(in); - if (!m) return {}; + if (!m) + return {}; cg::layout l; - if (auto plain = llvm::dyn_cast(m.getLayout())) l = plain.getRef(); - else if (auto composed = llvm::dyn_cast(m.getLayout())) l = composed.getRef().layout_b(); - else return {}; + if (auto plain = llvm::dyn_cast(m.getLayout())) + l = plain.getRef(); + else if (auto composed = llvm::dyn_cast(m.getLayout())) + l = composed.getRef().layout_b(); + else + return {}; // The atom mode collapses to one; the rest modes follow. int64_t bits = bitsOfType(m.getValueType()); std::string shape = "(1", stride = "(0"; for (size_t i = 1; i < cg::rank(l.shape()); ++i) { auto e = cg::size(l.shape(), i); auto st = cg::get(l.stride(), i); - if (!cg::holds_int(e) || !cg::holds_int(st)) return {}; + if (!cg::holds_int(e) || !cg::holds_int(st)) + return {}; shape += "," + std::to_string(e.as_int64()); stride += "," + std::to_string(st.as_int64() * bits / 128); } auto view = cg::from_string(shape + "):" + stride + ")"); - if (!view) return {}; + if (!view) + return {}; return cute_nvgpu::SmemDescViewType::get( ctx, cute_nvgpu::SmemDescType::get(ctx), LayoutAttr::get(ctx, *view)); }); } -LogicalResult TupleSubOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { - return inferTupleArith(ctx, loc, operands, out, [](const auto &a, const auto &b) { - return cg::arith_tuple_sub(a, b); - }); +LogicalResult TupleSubOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return inferTupleArith(ctx, loc, operands, out, + [](const auto &a, const auto &b) { + return cg::arith_tuple_sub(a, b); + }); } -LogicalResult TupleModOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { - if (failed(inferTupleArith(ctx, loc, operands, out, [](const auto &a, const auto &b) { - return cg::arith_tuple_mod(a, b); - }))) +LogicalResult TupleModOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + if (failed(inferTupleArith(ctx, loc, operands, out, + [](const auto &a, const auto &b) { + return cg::arith_tuple_mod(a, b); + }))) return failure(); // A dynamic leaf modulo a static one: 0 when its divisibility is a // multiple, itself when a divisor, else unknown (cutegen drops this). std::vector lhs, rhs, res; auto leaves = [](Type t, std::vector &l) { - if (auto i = llvm::dyn_cast(t)) leavesOf(i.getRef(), l); - if (auto s = llvm::dyn_cast(t)) leavesOf(s.getRef(), l); + if (auto i = llvm::dyn_cast(t)) + leavesOf(i.getRef(), l); + if (auto s = llvm::dyn_cast(t)) + leavesOf(s.getRef(), l); }; leaves(operands[0].getType(), lhs); leaves(operands[1].getType(), rhs); leaves(out[0], res); - if (lhs.size() != rhs.size() || lhs.size() != res.size()) return success(); + if (lhs.size() != rhs.size() || lhs.size() != res.size()) + return success(); for (size_t i = 0; i < res.size(); ++i) { int64_t n, div = 1; - if (lhs[i].front() != '?' || StringRef(rhs[i]).getAsInteger(10, n) || n <= 0) continue; + if (lhs[i].front() != '?' || StringRef(rhs[i]).getAsInteger(10, n) || + n <= 0) + continue; size_t d = lhs[i].find("div="); - if (d != std::string::npos) StringRef(lhs[i]).substr(d + 4).take_while(llvm::isDigit).getAsInteger(10, div); - res[i] = div % n == 0 ? "0" : n % div == 0 ? lhs[i] : dynamicInt(1, lhs[i].find("i64") != std::string::npos); + if (d != std::string::npos) + StringRef(lhs[i]) + .substr(d + 4) + .take_while(llvm::isDigit) + .getAsInteger(10, div); + res[i] = div % n == 0 ? "0" + : n % div == 0 + ? lhs[i] + : dynamicInt(1, lhs[i].find("i64") != std::string::npos); } size_t i = 0; Type t = out[0]; std::string text; - if (auto it = llvm::dyn_cast(t)) text = nestLike(it.getRef(), res, i); - else text = nestLike(llvm::cast(t).getRef(), res, i); + if (auto it = llvm::dyn_cast(t)) + text = nestLike(it.getRef(), res, i); + else + text = nestLike(llvm::cast(t).getRef(), res, i); if (llvm::isa(t)) { - if (auto r = cg::from_string(text)) out[0] = IntTupleType::get(ctx, *r); + if (auto r = cg::from_string(text)) + out[0] = IntTupleType::get(ctx, *r); } else if (auto r = cg::from_string(text)) { out[0] = ShapeType::get(ctx, *r); } @@ -1260,7 +1515,8 @@ namespace { template LogicalResult inferProduct(MLIRContext *ctx, std::optional loc, ValueRange operands, DictionaryAttr attrs, - PropertyRef props, SmallVectorImpl &out, F &&f) { + PropertyRef props, SmallVectorImpl &out, + F &&f) { typename Op::Adaptor a(operands, attrs, props); auto tiler = llvm::dyn_cast(a.getTiler().getType()); if (!tiler || !cg::is_int_or_dynamic_int_only(tiler.getRef().stride())) @@ -1269,9 +1525,11 @@ LogicalResult inferProduct(MLIRContext *ctx, std::optional loc, return infer(loc, out, [&]() -> Type { auto outer = outerLayout(in); if (!outer || !cg::is_static(*outer) || - (!cg::is_int_or_dynamic_int_only(outer->stride()) && !cg::is_static(tiler.getRef()))) + (!cg::is_int_or_dynamic_int_only(outer->stride()) && + !cg::is_static(tiler.getRef()))) return {}; - return mapLayout(ctx, in, [&](const auto &l) { return f(l, tiler.getRef()); }); + return mapLayout(ctx, in, + [&](const auto &l) { return f(l, tiler.getRef()); }); }); } @@ -1279,31 +1537,37 @@ LogicalResult inferProduct(MLIRContext *ctx, std::optional loc, std::optional dynamicCoordLike(const cg::shape &shape) { std::vector leaves; leavesOf(shape, leaves); - for (auto &leaf : leaves) leaf = isOne(leaf) ? "0" : "?"; + for (auto &leaf : leaves) + leaf = isOne(leaf) ? "0" : "?"; size_t i = 0; return cg::from_string(nestLike(shape, leaves, i)); } // Moves a carrier's iterator by `offset` and gives it `layout`. -Type moved(MLIRContext *ctx, const Carrier &c, const cg::int_tuple &offset, Type layout) { - if (c.ptr) return CuteMemRefType::get(ctx, movedPtr(c.ptr, offset), layout); +Type moved(MLIRContext *ctx, const Carrier &c, const cg::int_tuple &offset, + Type layout) { + if (c.ptr) + return CuteMemRefType::get(ctx, movedPtr(c.ptr, offset), layout); if (c.iter) { auto sum = cg::arith_tuple_sum(c.iter.getRef(), offset); - if (!cg::is_valid(sum)) return {}; - return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), layout); + if (!cg::is_valid(sum)) + return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), + layout); } return rebuild(ctx, c, layout); } -} // namespace +} // namespace #define CUTE_PRODUCT(Op, fn) \ LogicalResult Op::inferReturnTypes( \ MLIRContext *ctx, std::optional loc, ValueRange operands, \ DictionaryAttr attrs, PropertyRef props, RegionRange, \ SmallVectorImpl &out) { \ - return inferProduct(ctx, loc, operands, attrs, props, out, \ - [](const auto &l, const auto &t) { return cg::fn(l, t); }); \ + return inferProduct( \ + ctx, loc, operands, attrs, props, out, \ + [](const auto &l, const auto &t) { return cg::fn(l, t); }); \ } CUTE_PRODUCT(LogicalProductOp, logical_product) CUTE_PRODUCT(ZippedProductOp, zipped_product) @@ -1313,11 +1577,16 @@ CUTE_PRODUCT(BlockedProductOp, blocked_product) #undef CUTE_PRODUCT // Unlike the other products, the raked one leaves unit modes at stride 0. -LogicalResult RakedProductOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, SmallVectorImpl &out) { - return inferProduct(ctx, loc, operands, attrs, props, out, - [](const auto &l, const auto &t) { return zeroUnitModes(cg::raked_product(l, t)); }); +LogicalResult RakedProductOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferProduct( + ctx, loc, operands, attrs, props, out, [](const auto &l, const auto &t) { + return zeroUnitModes(cg::raked_product(l, t)); + }); } // The pointer's type is kept: the compiler records no swizzle on it. @@ -1328,23 +1597,29 @@ LogicalResult ApplySwizzleOp::inferReturnTypes( return success(); } -LogicalResult ComplementOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { +LogicalResult ComplementOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { Adaptor a(operands, attrs, props); auto in = llvm::dyn_cast(a.getInput().getType()); Type co = a.getCotarget().getType(); return infer(loc, out, [&]() -> Type { - if (!in) return {}; + if (!in) + return {}; const cg::layout &l = in.getRef(); if (!cg::is_int_or_dynamic_int_only(l.stride()) || (!cg::is_static(l.stride()) && cg::rank(l.shape()) > 1)) return {}; cg::shape shape; - if (auto s = llvm::dyn_cast(co)) shape = s.getRef(); - else if (auto c = llvm::dyn_cast(co)) shape = c.getRef().shape(); - else return {}; + if (auto s = llvm::dyn_cast(co)) + shape = s.getRef(); + else if (auto c = llvm::dyn_cast(co)) + shape = c.getRef().shape(); + else + return {}; cg::layout r = cg::complement(l, shape); return cg::is_valid(r) ? LayoutType::get(ctx, std::move(r)) : Type(); }); @@ -1355,8 +1630,10 @@ LogicalResult ComposedGetInnerOp::inferReturnTypes( DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { return infer(loc, out, [&]() -> Type { auto c = llvm::dyn_cast(operands[0].getType()); - if (!c) return {}; - if (c.getRef().is_a_swizzle()) return SwizzleType::get(ctx, c.getRef().swizzle_a()); + if (!c) + return {}; + if (c.getRef().is_a_swizzle()) + return SwizzleType::get(ctx, c.getRef().swizzle_a()); return LayoutType::get(ctx, c.getRef().layout_a()); }); } @@ -1370,23 +1647,29 @@ LogicalResult ComposedGetOffsetOp::inferReturnTypes( }); } -LogicalResult CosizeOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { +LogicalResult CosizeOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { Adaptor a(operands, attrs, props); ArrayRef mode = a.getMode(); auto c = carrierOf(a.getInput().getType()); return infer(loc, out, [&]() -> Type { - if (!c) return {}; + if (!c) + return {}; cg::int_tuple r; if (auto l = llvm::dyn_cast(c->layout)) { - r = cg::rec_var_cast(cg::cosize(cg::get(l.getRef(), mode))); + r = cg::rec_var_cast( + cg::cosize(cg::get(l.getRef(), mode))); } else { // An affine inner layout has no cosize. auto cl = llvm::cast(c->layout); - if (!cl.getRef().is_a_swizzle()) return {}; - r = cg::rec_var_cast(cg::cosize(cg::get(cl.getRef(), mode))); + if (!cl.getRef().is_a_swizzle()) + return {}; + r = cg::rec_var_cast( + cg::cosize(cg::get(cl.getRef(), mode))); } return cg::is_valid(r) ? IntTupleType::get(ctx, std::move(r)) : Type(); }); @@ -1397,7 +1680,8 @@ LogicalResult FastDivmodGetAuxOp::inferReturnTypes( MLIRContext *ctx, std::optional loc, ValueRange operands, DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { Type i = fastDivmodComputeResultType(operands[0].getType()); - if (!i) return emitOptionalError(loc, "expects a fast divmod divisor"); + if (!i) + return emitOptionalError(loc, "expects a fast divmod divisor"); out.append({i, IntegerType::get(ctx, 8), IntegerType::get(ctx, 8)}); return success(); } @@ -1405,25 +1689,34 @@ LogicalResult FastDivmodGetAuxOp::inferReturnTypes( LogicalResult FastDivmodGetDivisorOp::inferReturnTypes( MLIRContext *, std::optional loc, ValueRange operands, DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { - return infer(loc, out, [&] { return fastDivmodComputeResultType(operands[0].getType()); }); + return infer(loc, out, [&] { + return fastDivmodComputeResultType(operands[0].getType()); + }); } -LogicalResult FilterOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { +LogicalResult FilterOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { auto c = carrierOf(operands[0].getType()); - if (!c) return emitOptionalError(loc, "input carries no layout"); + if (!c) + return emitOptionalError(loc, "input carries no layout"); return infer(loc, out, [&]() -> Type { Type layout; if (auto l = llvm::dyn_cast(c->layout)) { layout = layoutType(ctx, cg::filter(l.getRef())); } else { // The outer layout is filtered; the inner function is kept. - const cg::composed_layout &cl = llvm::cast(c->layout).getRef(); + const cg::composed_layout &cl = + llvm::cast(c->layout).getRef(); cg::layout f = cg::filter(cl.layout_b()); - if (!cg::is_valid(f)) return {}; - layout = layoutType(ctx, cl.is_a_swizzle() ? cg::composed_layout(cl.swizzle_a(), cl.offset(), f) - : cg::composed_layout(cl.layout_a(), cl.offset(), f)); + if (!cg::is_valid(f)) + return {}; + layout = layoutType( + ctx, cl.is_a_swizzle() + ? cg::composed_layout(cl.swizzle_a(), cl.offset(), f) + : cg::composed_layout(cl.layout_a(), cl.offset(), f)); } return rebuild(ctx, *c, layout); }); @@ -1433,11 +1726,13 @@ namespace { // Layout::get_hier_coord: each leaf's coordinate is (index / stride) % shape. // A constant index gives static coordinates where the leaf is static; any // other index gives dynamic ones, 0 on leaves of extent 1. -std::optional> hierCoord(Value index, const cg::layout &l) { +std::optional> hierCoord(Value index, + const cg::layout &l) { std::vector shapes, strides; leavesOf(l.shape(), shapes); leavesOf(l.stride(), strides); - if (shapes.size() != strides.size()) return std::nullopt; + if (shapes.size() != strides.size()) + return std::nullopt; APInt value; bool constant = matchPattern(index, m_ConstantInt(&value)); std::vector crd; @@ -1449,7 +1744,7 @@ std::optional> hierCoord(Value index, const cg::layout } else if (!constant) { crd.push_back("?"); } else if (StringRef(strides[i]).getAsInteger(10, d) || d <= 0) { - return std::nullopt; // not recovered for dynamic or zero strides + return std::nullopt; // not recovered for dynamic or zero strides } else { int64_t q = value.getSExtValue() / d; crd.push_back(staticShape ? std::to_string(q % n) : "?"); @@ -1459,45 +1754,55 @@ std::optional> hierCoord(Value index, const cg::layout } // Column-major index of static coordinates in `shape`, or "?". -std::string colMajor(const std::vector &crd, const std::vector &shapes, - size_t begin, size_t end) { +std::string colMajor(const std::vector &crd, + const std::vector &shapes, size_t begin, + size_t end) { int64_t idx = 0, scale = 1; for (size_t i = begin; i < end; ++i) { int64_t c, n; - if (StringRef(crd[i]).getAsInteger(10, c)) return "?"; + if (StringRef(crd[i]).getAsInteger(10, c)) + return "?"; idx += c * scale; - if (i + 1 < end && StringRef(shapes[i]).getAsInteger(10, n)) return "?"; + if (i + 1 < end && StringRef(shapes[i]).getAsInteger(10, n)) + return "?"; scale *= n; } return std::to_string(idx); } -} // namespace +} // namespace -LogicalResult GetHierCoordOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { +LogicalResult GetHierCoordOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { Adaptor a(operands, attrs, props); auto layout = llvm::dyn_cast(a.getInput().getType()); return infer(loc, out, [&]() -> Type { auto crd = layout ? hierCoord(a.getIndex(), layout.getRef()) : std::nullopt; - if (!crd) return {}; + if (!crd) + return {}; size_t i = 0; - auto c = cg::from_string(nestLike(layout.getRef().shape(), *crd, i)); + auto c = + cg::from_string(nestLike(layout.getRef().shape(), *crd, i)); return c ? CoordType::get(ctx, std::move(*c)) : Type(); }); } // One coordinate per mode: the mode's hierarchical coordinate, column-major. -LogicalResult GetFlatCoordOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { +LogicalResult GetFlatCoordOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { Adaptor a(operands, attrs, props); auto layout = llvm::dyn_cast(a.getInput().getType()); return infer(loc, out, [&]() -> Type { auto crd = layout ? hierCoord(a.getIndex(), layout.getRef()) : std::nullopt; - if (!crd) return {}; + if (!crd) + return {}; const cg::shape &shape = layout.getRef().shape(); std::vector shapes, modes; leavesOf(shape, shapes); @@ -1512,8 +1817,9 @@ LogicalResult GetFlatCoordOp::inferReturnTypes( begin += sub.size(); } } - auto c = cg::from_string(cg::holds_vector(shape) ? "(" + llvm::join(modes, ",") + ")" - : modes[0]); + auto c = cg::from_string(cg::holds_vector(shape) + ? "(" + llvm::join(modes, ",") + ")" + : modes[0]); return c ? CoordType::get(ctx, std::move(*c)) : Type(); }); } @@ -1526,10 +1832,12 @@ LogicalResult GetIntegralCoordOp::inferReturnTypes( auto layout = llvm::dyn_cast(a.getInput().getType()); return infer(loc, out, [&]() -> Type { auto crd = layout ? hierCoord(a.getIndex(), layout.getRef()) : std::nullopt; - if (!crd) return {}; + if (!crd) + return {}; std::vector shapes; leavesOf(layout.getRef().shape(), shapes); - auto c = cg::from_string(colMajor(*crd, shapes, 0, shapes.size())); + auto c = + cg::from_string(colMajor(*crd, shapes, 0, shapes.size())); return c ? CoordType::get(ctx, std::move(*c)) : Type(); }); } @@ -1538,7 +1846,8 @@ LogicalResult GetLayoutsFromTileOp::inferReturnTypes( MLIRContext *ctx, std::optional loc, ValueRange operands, DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { auto tile = llvm::dyn_cast(operands[0].getType()); - if (!tile) return emitOptionalError(loc, "expects a tile"); + if (!tile) + return emitOptionalError(loc, "expects a tile"); auto flat = cg::flatten(tile.getRef()); for (size_t i = 0; i < cg::rank(flat); ++i) { flat[i].visit([&](const auto &e) { @@ -1546,44 +1855,55 @@ LogicalResult GetLayoutsFromTileOp::inferReturnTypes( out.push_back(LayoutType::get(ctx, e)); }); } - if (out.empty()) return emitOptionalError(loc, "the tile holds no layout"); + if (out.empty()) + return emitOptionalError(loc, "the tile holds no layout"); return success(); } -LogicalResult GetStrideOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { +LogicalResult GetStrideOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { return infer(loc, out, [&]() -> Type { auto l = llvm::dyn_cast(operands[0].getType()); return l ? StrideType::get(ctx, l.getRef().stride()) : Type(); }); } -LogicalResult Idx2CrdOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { +LogicalResult Idx2CrdOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { Adaptor a(operands, attrs, props); auto index = llvm::dyn_cast(a.getIndex().getType()); auto shape = llvm::dyn_cast(a.getShape().getType()); return infer(loc, out, [&]() -> Type { - if (!index || !shape) return {}; + if (!index || !shape) + return {}; size_t r = cg::rank(index.getRef()); - if (r > 1 && r != cg::rank(shape.getRef())) return {}; + if (r > 1 && r != cg::rank(shape.getRef())) + return {}; if (auto split = cg::holds_dynamic_int(index.getRef()) - ? splitIndex(index.getRef(), shape.getRef()) : std::nullopt) { + ? splitIndex(index.getRef(), shape.getRef()) + : std::nullopt) { auto c = cg::from_string(*split); return c ? CoordType::get(ctx, std::move(*c)) : Type(); } - auto c = cg::rec_var_cast(cg::idx2crd(index.getRef(), shape.getRef())); + auto c = cg::rec_var_cast( + cg::idx2crd(index.getRef(), shape.getRef())); return cg::is_valid(c) ? CoordType::get(ctx, std::move(c)) : Type(); }); } -LogicalResult IncrementCoordOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { +LogicalResult IncrementCoordOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { Adaptor a(operands, attrs, props); auto coord = llvm::dyn_cast(a.getCoord().getType()); auto shape = llvm::dyn_cast(a.getShape().getType()); @@ -1599,8 +1919,8 @@ LogicalResult IncrementCoordOp::inferReturnTypes( namespace { template LogicalResult inferInverse(MLIRContext *ctx, std::optional loc, - Type input, bool staticStride, SmallVectorImpl &out, - F &&f) { + Type input, bool staticStride, + SmallVectorImpl &out, F &&f) { return infer(loc, out, [&]() -> Type { auto l = llvm::dyn_cast(input); if (!l || !cg::is_static(l.getRef().shape()) || @@ -1610,7 +1930,7 @@ LogicalResult inferInverse(MLIRContext *ctx, std::optional loc, return cg::is_valid(r) ? LayoutType::get(ctx, std::move(r)) : Type(); }); } -} // namespace +} // namespace LogicalResult LeftInverseOp::inferReturnTypes( MLIRContext *ctx, std::optional loc, ValueRange operands, @@ -1635,24 +1955,30 @@ LogicalResult LoadScaledIndexOp::inferReturnTypes( // CuTe's local_partition: the tensor divided by the tiler's shape, sliced at // the thread's (dynamic) coordinate in the tile. -LogicalResult LocalPartitionOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { +LogicalResult LocalPartitionOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { Adaptor a(operands, attrs, props); auto c = carrierOf(a.getInput().getType()); auto tiler = llvm::dyn_cast(a.getTiler().getType()); return infer(loc, out, [&]() -> Type { auto layout = c ? llvm::dyn_cast(c->layout) : LayoutType(); - if (!layout || !tiler || !cg::is_static(tiler.getRef())) return {}; + if (!layout || !tiler || !cg::is_static(tiler.getRef())) + return {}; cg::shape tile = cg::product_each(tiler.getRef().shape()); cg::layout zipped = cg::zipped_divide(layout.getRef(), tile); auto thread = dynamicCoordLike(tile); - if (!cg::is_valid(zipped) || !thread) return {}; + if (!cg::is_valid(zipped) || !thread) + return {}; auto crd = cg::from_string("(" + cg::to_string(*thread) + ",_)"); - if (!crd) return {}; + if (!crd) + return {}; auto [sliced, offset] = cg::slice_and_offset(*crd, zipped); - if (!cg::is_valid(sliced)) return {}; + if (!cg::is_valid(sliced)) + return {}; // The slice keeps the rest mode's own modes, not a tuple around them. if (cg::holds_vector(sliced.shape()) && cg::rank(sliced.shape()) == 1) sliced = cg::get(sliced, 0); @@ -1669,14 +1995,17 @@ LogicalResult MakeFragmentLikeOp::inferReturnTypes( DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { auto src = swizzledOuter(operands[0].getType()); return infer(loc, out, [&]() -> Type { - if (!src) return {}; + if (!src) + return {}; const cg::shape &shape = src->shape(); std::vector shapes, strides; leavesOf(shape, shapes); leavesOf(src->stride(), strides); - if (shapes.size() != strides.size()) return {}; - size_t first = shapes.size(); // leaves of mode 0 - bool ordered = cg::holds_vector(shape) && cg::rank(shape) > 1 && cg::is_static(shape[0]); + if (shapes.size() != strides.size()) + return {}; + size_t first = shapes.size(); // leaves of mode 0 + bool ordered = cg::holds_vector(shape) && cg::rank(shape) > 1 && + cg::is_static(shape[0]); if (ordered) { std::vector m0; leavesOf(shape[0], m0); @@ -1701,11 +2030,13 @@ LogicalResult MakeFragmentLikeOp::inferReturnTypes( } result[i] = cg::to_string(step); auto n = cg::from_string(shapes[i]); - if (!n) return {}; + if (!n) + return {}; step = cg::arith_tuple_mul(step, *n); } size_t i = 0; - auto l = cg::from_string(cg::to_string(shape) + ":" + nestLike(shape, result, i)); + auto l = cg::from_string(cg::to_string(shape) + ":" + + nestLike(shape, result, i)); return l ? LayoutType::get(ctx, std::move(*l)) : Type(); }); } @@ -1715,15 +2046,19 @@ LogicalResult MakeIdentityTensorOp::inferReturnTypes( DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { return infer(loc, out, [&]() -> Type { auto s = llvm::dyn_cast(operands[0].getType()); - if (!s) return {}; + if (!s) + return {}; std::vector leaves; leavesOf(s.getRef(), leaves); - for (auto &leaf : leaves) leaf = "0"; + for (auto &leaf : leaves) + leaf = "0"; size_t i = 0; auto zero = cg::from_string(nestLike(s.getRef(), leaves, i)); cg::layout l = cg::make_identity_layout(s.getRef()); - if (!zero || !cg::is_valid(l)) return {}; - return CoordTensorType::get(ctx, IntTupleType::get(ctx, *zero), LayoutType::get(ctx, l)); + if (!zero || !cg::is_valid(l)) + return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, *zero), + LayoutType::get(ctx, l)); }); } @@ -1734,10 +2069,12 @@ LogicalResult MakeLayoutLikeOp::inferReturnTypes( Type src = operands[0].getType(); cg::layout r; if (auto l = llvm::dyn_cast(src)) { - if (cg::has_scaled_basis(l.getRef().stride())) return {}; + if (cg::has_scaled_basis(l.getRef().stride())) + return {}; r = cg::make_layout_like(l.getRef()); } else if (auto c = llvm::dyn_cast(src)) { - if (!c.getRef().is_a_swizzle() || cg::has_scaled_basis(c.getRef().layout_b().stride())) + if (!c.getRef().is_a_swizzle() || + cg::has_scaled_basis(c.getRef().layout_b().stride())) return {}; r = cg::make_layout_like(c.getRef()); } else { @@ -1747,46 +2084,58 @@ LogicalResult MakeLayoutLikeOp::inferReturnTypes( }); } -LogicalResult PtrLoadOp::inferReturnTypes( - MLIRContext *, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { +LogicalResult PtrLoadOp::inferReturnTypes(MLIRContext *, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { return infer(loc, out, [&]() -> Type { auto p = llvm::dyn_cast(operands[0].getType()); return p ? p.getValueType() : Type(); }); } -LogicalResult RecastLayoutOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { +LogicalResult RecastLayoutOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { Adaptor a(operands, attrs, props); int newBits = a.getNewTypeBits(), oldBits = a.getOldTypeBits(); return infer(loc, out, [&]() -> Type { - if (newBits <= 0 || oldBits <= 0) return {}; + if (newBits <= 0 || oldBits <= 0) + return {}; auto outer = outerLayout(a.getSrc().getType()); - if (!outer || !cg::is_int_or_dynamic_int_only(outer->stride())) return {}; - return mapLayout(ctx, a.getSrc().getType(), - [&](const auto &l) { return cg::recast(newBits, oldBits, l); }); + if (!outer || !cg::is_int_or_dynamic_int_only(outer->stride())) + return {}; + return mapLayout(ctx, a.getSrc().getType(), [&](const auto &l) { + return cg::recast(newBits, oldBits, l); + }); }); } // Without a profile the whole vector reduces to a scalar; with one, the // modes it marks reduce and the others (`_`) remain, as a vector and shape. -LogicalResult ReduceOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { +LogicalResult ReduceOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { Adaptor a(operands, attrs, props); auto vec = llvm::dyn_cast(a.getInput().getType()); - if (!vec) return emitOptionalError(loc, "expects a vector input"); + if (!vec) + return emitOptionalError(loc, "expects a vector input"); if (!a.getReductionProfileAttr()) { out.push_back(vec.getElementType()); return success(); } auto shape = llvm::dyn_cast(a.getShape().getType()); - if (!shape) return emitOptionalError(loc, "expects a shape"); - cg::shape rest = cg::slice(a.getReductionProfileAttr().getRef(), shape.getRef()); + if (!shape) + return emitOptionalError(loc, "expects a shape"); + cg::shape rest = + cg::slice(a.getReductionProfileAttr().getRef(), shape.getRef()); auto n = cg::size(rest); if (!cg::is_valid(rest) || !cg::holds_int(n)) return emitOptionalError(loc, "cannot reduce the shape"); @@ -1795,44 +2144,54 @@ LogicalResult ReduceOp::inferReturnTypes( return success(); } -LogicalResult SelectOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { +LogicalResult SelectOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { Adaptor a(operands, attrs, props); ArrayRef mode = a.getMode(); return infer(loc, out, [&]() -> Type { return llvm::TypeSwitch(a.getInput().getType()) - .Case([&](auto ty) -> Type { + .Case([&](auto ty) -> Type { size_t r = cg::rank(ty.getRef()); for (int32_t m : mode) - if (m < 0 || static_cast(m) >= r) return {}; + if (m < 0 || static_cast(m) >= r) + return {}; auto res = cg::select(ty.getRef(), mode); - if (!cg::is_valid(res)) return {}; + if (!cg::is_valid(res)) + return {}; return decltype(ty)::get(ctx, std::move(res)); }) .Default([](Type) -> Type { return {}; }); }); } -LogicalResult ShapeDivOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { +LogicalResult ShapeDivOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { return infer(loc, out, [&]() -> Type { auto x = llvm::dyn_cast(operands[0].getType()); auto y = llvm::dyn_cast(operands[1].getType()); - if (!x || !y) return {}; + if (!x || !y) + return {}; cg::shape r = cg::shape_div(x.getRef(), y.getRef()); return cg::is_valid(r) ? ShapeType::get(ctx, std::move(r)) : Type(); }); } // The compiler aborts on every input tried (bad_alloc, segfault): no rule. -LogicalResult StencilDivideOp::inferReturnTypes( - MLIRContext *, std::optional loc, ValueRange, DictionaryAttr, - PropertyRef, RegionRange, SmallVectorImpl &) { - return emitOptionalError(loc, "cute.stencil_divide inference is not recovered"); +LogicalResult StencilDivideOp::inferReturnTypes(MLIRContext *, + std::optional loc, + ValueRange, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &) { + return emitOptionalError(loc, + "cute.stencil_divide inference is not recovered"); } // No type of the compiler's registry implements the descriptor iterator @@ -1850,7 +2209,8 @@ LogicalResult TupleProductOp::inferReturnTypes( return llvm::TypeSwitch(operands[0].getType()) .Case([&](auto ty) -> Type { auto r = cg::product(ty.getRef()); - return cg::is_valid(r) ? decltype(ty)::get(ctx, std::move(r)) : Type(); + return cg::is_valid(r) ? decltype(ty)::get(ctx, std::move(r)) + : Type(); }) .Default([](Type) -> Type { return {}; }); }); @@ -1863,7 +2223,8 @@ LogicalResult TupleProductEachOp::inferReturnTypes( return llvm::TypeSwitch(operands[0].getType()) .Case([&](auto ty) -> Type { auto r = cg::product_each(ty.getRef()); - return cg::is_valid(r) ? decltype(ty)::get(ctx, std::move(r)) : Type(); + return cg::is_valid(r) ? decltype(ty)::get(ctx, std::move(r)) + : Type(); }) .Default([](Type) -> Type { return {}; }); }); @@ -1871,16 +2232,19 @@ LogicalResult TupleProductEachOp::inferReturnTypes( // A SIMT copy's fragment is the tensor itself; the rule for other atoms is // not recovered. -LogicalResult CopyMakeFragmentOp::inferReturnTypes( - MLIRContext *, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { +LogicalResult +CopyMakeFragmentOp::inferReturnTypes(MLIRContext *, std::optional loc, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { Adaptor a(operands, attrs, props); auto tiled = llvm::dyn_cast(a.getAtom().getType()); Type in = a.getInput().getType(); - if (!tiled || !llvm::isa(tiled.getCopyAtom()) || + if (!tiled || + !llvm::isa(tiled.getCopyAtom()) || !llvm::isa(in)) - return emitOptionalError(loc, "copy fragment inference covers tiled SIMT copies only"); + return emitOptionalError( + loc, "copy fragment inference covers tiled SIMT copies only"); out.push_back(in); return success(); } @@ -1900,20 +2264,35 @@ LogicalResult TiledMmaPartitionShapeOp::inferReturnTypes( return {}; int64_t full[2], part[2]; switch (which.getInt()) { - case 0: full[0] = atom->m; full[1] = atom->k; part[0] = atom->m / atom->ctas; break; - case 1: full[0] = atom->n; full[1] = atom->k; part[0] = atom->n / atom->ctas; break; - case 2: full[0] = atom->m; full[1] = atom->n; part[0] = atom->m / atom->ctas; break; - default: return {}; + case 0: + full[0] = atom->m; + full[1] = atom->k; + part[0] = atom->m / atom->ctas; + break; + case 1: + full[0] = atom->n; + full[1] = atom->k; + part[0] = atom->n / atom->ctas; + break; + case 2: + full[0] = atom->m; + full[1] = atom->n; + part[0] = atom->m / atom->ctas; + break; + default: + return {}; } part[1] = full[1]; std::string rest; for (int i = 0; i < 2; ++i) { auto n = cg::size(shape.getRef(), i); - if (!cg::holds_int(n) || n.as_int64() % full[i]) return {}; + if (!cg::holds_int(n) || n.as_int64() % full[i]) + return {}; rest += "," + std::to_string(n.as_int64() / full[i]); } - auto r = cg::from_string("((" + std::to_string(part[0]) + "," + - std::to_string(part[1]) + ")" + rest + ")"); + auto r = + cg::from_string("((" + std::to_string(part[0]) + "," + + std::to_string(part[1]) + ")" + rest + ")"); return r ? ShapeType::get(ctx, std::move(*r)) : Type(); }); } @@ -1926,12 +2305,10 @@ std::optional layoutOf(const std::string &text) { return cg::from_string(text); } -template -int64_t asInt(const V &v) { +template int64_t asInt(const V &v) { return cg::holds_int(v) ? v.as_int64() : -1; } - // TMEM addresses count lanes in the upper half-word: 2^21 bits per lane. constexpr int64_t kTmemLaneBits = int64_t(1) << 21; @@ -1949,13 +2326,15 @@ std::optional copyTraitsOf(Type atom) { auto set = [&](std::string src, std::string dst, bool refIsSrc, std::string valId = "") -> std::optional { auto s = layoutOf(src), d = layoutOf(dst); - if (!s || !d) return std::nullopt; + if (!s || !d) + return std::nullopt; t.src = *s; t.dst = *d; t.ref = refIsSrc ? *s : *d; if (!valId.empty()) { auto v = layoutOf(valId); - if (!v) return std::nullopt; + if (!v) + return std::nullopt; t.valId = *v; } return t; @@ -1963,7 +2342,8 @@ std::optional copyTraitsOf(Type atom) { auto n = [](int64_t v) { return std::to_string(v); }; if (auto u = llvm::dyn_cast(atom)) { t.bits = bitsOfType(u.getValType()); - std::string l = "(1," + n(u.getCopyBits() ? u.getCopyBits() : t.bits) + "):(0,1)"; + std::string l = + "(1," + n(u.getCopyBits() ? u.getCopyBits() : t.bits) + "):(0,1)"; return set(l, l, true); } if (auto s = llvm::dyn_cast(atom)) { @@ -1976,24 +2356,45 @@ std::optional copyTraitsOf(Type atom) { std::string ldsmSrc, ldsmDst; if (!transposed) { switch (matrices) { - case 1: ldsmSrc = "((8,4),128):((128,0),1)"; ldsmDst = "(32,32):(32,1)"; break; - case 2: ldsmSrc = "((16,2),128):((128,0),1)"; ldsmDst = "(32,(32,2)):(32,(1,1024))"; break; - case 4: ldsmSrc = "(32,128):(128,1)"; ldsmDst = "(32,(32,4)):(32,(1,1024))"; break; - default: return std::nullopt; + case 1: + ldsmSrc = "((8,4),128):((128,0),1)"; + ldsmDst = "(32,32):(32,1)"; + break; + case 2: + ldsmSrc = "((16,2),128):((128,0),1)"; + ldsmDst = "(32,(32,2)):(32,(1,1024))"; + break; + case 4: + ldsmSrc = "(32,128):(128,1)"; + ldsmDst = "(32,(32,4)):(32,(1,1024))"; + break; + default: + return std::nullopt; } } else { switch (matrices) { - case 1: ldsmSrc = "((8,4),128):((128,0),1)"; ldsmDst = "((4,8),(16,2)):((256,16),(1,128))"; break; - case 2: ldsmSrc = "((16,2),128):((128,0),1)"; ldsmDst = "((4,8),(16,2,2)):((256,16),(1,128,1024))"; break; - case 4: ldsmSrc = "(32,128):(128,1)"; ldsmDst = "((4,8),(16,2,4)):((256,16),(1,128,1024))"; break; - default: return std::nullopt; + case 1: + ldsmSrc = "((8,4),128):((128,0),1)"; + ldsmDst = "((4,8),(16,2)):((256,16),(1,128))"; + break; + case 2: + ldsmSrc = "((16,2),128):((128,0),1)"; + ldsmDst = "((4,8),(16,2,2)):((256,16),(1,128,1024))"; + break; + case 4: + ldsmSrc = "(32,128):(128,1)"; + ldsmDst = "((4,8),(16,2,4)):((256,16),(1,128,1024))"; + break; + default: + return std::nullopt; } } return set(ldsmDst, ldsmSrc, true); } // Tensor memory loads and stores: the tmem side is the reference; a store // is a load with source and destination swapped. - auto tmem = [&](Type val, int64_t dp, int64_t bit, int64_t x, bool load) -> std::optional { + auto tmem = [&](Type val, int64_t dp, int64_t bit, int64_t x, + bool load) -> std::optional { t.bits = bitsOfType(val); t.threads = 32; std::string mem, reg, valId; @@ -2004,7 +2405,8 @@ std::optional copyTraitsOf(Type atom) { } else if (dp == 16 && bit == 256) { mem = "(32," + n(4096 * x) + "):(0,1)"; reg = x == 1 ? "((4,8),(64,2)):((64,256),(1,2048))" - : "((4,8),(64,2," + n(x) + ")):((64," + n(256 * x) + "),(1," + n(2048 * x) + ",256))"; + : "((4,8),(64,2," + n(x) + ")):((64," + n(256 * x) + + "),(1," + n(2048 * x) + ",256))"; valId = "(" + n(256 * x) + ",16):(1," + n(kTmemLaneBits) + ")"; } else { return std::nullopt; @@ -2014,24 +2416,28 @@ std::optional copyTraitsOf(Type atom) { // 16-bit packing and split reductions are not covered. if (auto l = llvm::dyn_cast(atom)) if (!l.getPack16b()) - return tmem(l.getValType(), l.getNumDp(), l.getNumBit(), l.getNumRep(), true); + return tmem(l.getValType(), l.getNumDp(), l.getNumBit(), l.getNumRep(), + true); if (auto l = llvm::dyn_cast(atom)) if (!l.getHalfSplitOff()) - return tmem(l.getValType(), l.getNumDp(), l.getNumBit(), l.getNumRep(), true); + return tmem(l.getValType(), l.getNumDp(), l.getNumBit(), l.getNumRep(), + true); if (auto st = llvm::dyn_cast(atom)) if (!st.getExpand16b()) - return tmem(st.getValType(), st.getNumDp(), st.getNumBit(), st.getNumRep(), false); + return tmem(st.getValType(), st.getNumDp(), st.getNumBit(), + st.getNumRep(), false); if (auto s = llvm::dyn_cast(atom)) { // : SM100_UTCCP_4x32dp128bit_{C}cta. t.bits = bitsOfType(s.getValType()); int64_t dp = s.getNumDp(), bit = s.getNumBit(); t.threads = s.getNumCta(); - if (dp != 32 || bit != 128 || s.getBroadcast() != cute_nvgpu::CopyS2TBroadcast::x4 || - t.threads < 1) + if (dp != 32 || bit != 128 || + s.getBroadcast() != cute_nvgpu::CopyS2TBroadcast::x4 || t.threads < 1) return std::nullopt; return set("(" + n(t.threads) + ",(32,128,4)):(0,(1,32,0))", "(" + n(t.threads) + ",16384):(0,1)", false, - "(32,128,4):(" + n(kTmemLaneBits) + ",1," + n(32 * kTmemLaneBits) + ")"); + "(32,128,4):(" + n(kTmemLaneBits) + ",1," + + n(32 * kTmemLaneBits) + ")"); } return std::nullopt; } @@ -2048,9 +2454,11 @@ struct TiledCopy { std::optional tiledCopyOf(Type t) { auto tiled = llvm::dyn_cast(t); - if (!tiled) return std::nullopt; + if (!tiled) + return std::nullopt; auto traits = copyTraitsOf(tiled.getCopyAtom()); - if (!traits) return std::nullopt; + if (!traits) + return std::nullopt; TiledCopy tc; tc.atom = tiled.getCopyAtom(); tc.traits = *traits; @@ -2074,8 +2482,10 @@ std::optional tiledCopyOf(Type t) { // The modes of a layout, one leaf mode as itself. std::vector modesOf(const cg::layout &l) { std::vector out; - if (!cg::holds_vector(l.shape())) return {l}; - for (size_t i = 0; i < cg::rank(l.shape()); ++i) out.push_back(cg::get(l, i)); + if (!cg::holds_vector(l.shape())) + return {l}; + for (size_t i = 0; i < cg::rank(l.shape()); ++i) + out.push_back(cg::get(l, i)); return out; } @@ -2090,8 +2500,10 @@ cg::layout composeFirst(const cg::layout &l, const cg::layout &f) { // ((a,b),(c,d)) -> ((a,c),(b,d)) cg::layout zip2(const cg::layout &l) { return cg::make_layout(std::vector{ - cg::make_layout(std::vector{cg::get(l, 0, 0), cg::get(l, 1, 0)}), - cg::make_layout(std::vector{cg::get(l, 0, 1), cg::get(l, 1, 1)})}); + cg::make_layout( + std::vector{cg::get(l, 0, 0), cg::get(l, 1, 0)}), + cg::make_layout( + std::vector{cg::get(l, 0, 1), cg::get(l, 1, 1)})}); } cg::coord profileOf(const char *text) { @@ -2102,30 +2514,40 @@ cg::coord profileOf(const char *text) { // through the atom's (src or dst) thread-value map `ref2trg`. cg::layout tile2thrfrg(const TiledCopy &tc, const cg::layout &tiled, const cg::layout &ref2trg, bool flatRest = true) { - auto atomShape = cg::from_string( - "(" + std::to_string(tc.atomThreads) + "," + std::to_string(tc.atomVals) + ")"); - if (!atomShape) return cg::layout(cg::cg_error_t{}); + auto atomShape = + cg::from_string("(" + std::to_string(tc.atomThreads) + "," + + std::to_string(tc.atomVals) + ")"); + if (!atomShape) + return cg::layout(cg::cg_error_t{}); cg::layout atomTV = cg::zipped_divide(tc.tv, *atomShape); - if (!cg::is_valid(atomTV)) return atomTV; + if (!cg::is_valid(atomTV)) + return atomTV; cg::layout trgTV = composeFirst(atomTV, ref2trg); cg::layout thrval2mn = cg::coalesce(zip2(trgTV), profileOf("(1,(1,1))")); - if (!cg::is_valid(thrval2mn)) return thrval2mn; + if (!cg::is_valid(thrval2mn)) + return thrval2mn; cg::layout tv = cg::composition(cg::get(tiled, 0), thrval2mn); - if (!cg::is_valid(tv)) return tv; + if (!cg::is_valid(tv)) + return tv; std::vector modes{cg::get(tv, 0), cg::get(tv, 1)}; // The rest modes follow; a rank-1 tensor keeps its one rest mode whole. - if (!flatRest) modes.push_back(cg::get(tiled, 1)); - else for (const cg::layout &m : modesOf(cg::get(tiled, 1))) modes.push_back(m); + if (!flatRest) + modes.push_back(cg::get(tiled, 1)); + else + for (const cg::layout &m : modesOf(cg::get(tiled, 1))) + modes.push_back(m); return cg::make_layout(modes); } // TiledCopy::tidfrg_S/D: (M,N,...) -> (Thr, (FrgV, FrgX), (RestM, RestN, ...)). cg::layout tidfrg(const TiledCopy &tc, const cg::layout &tensor, bool src) { cg::layout tiled = cg::zipped_divide(tensor, tc.tiler); - if (!cg::is_valid(tiled)) return tiled; + if (!cg::is_valid(tiled)) + return tiled; cg::layout ref2trg = cg::composition(cg::right_inverse(tc.traits.ref), src ? tc.traits.src : tc.traits.dst); - bool flatRest = cg::holds_vector(tensor.shape()) && cg::rank(tensor.shape()) > 1; + bool flatRest = + cg::holds_vector(tensor.shape()) && cg::rank(tensor.shape()) > 1; return tile2thrfrg(tc, tiled, ref2trg, flatRest); } @@ -2133,25 +2555,34 @@ cg::layout tidfrg(const TiledCopy &tc, const cg::layout &tensor, bool src) { Type partitionCopy(MLIRContext *ctx, const TiledCopy &tc, const Carrier &c, const cg::coord &thread, bool src) { auto layout = llvm::dyn_cast(c.layout); - if (!layout) return {}; + if (!layout) + return {}; cg::layout tv = tidfrg(tc, layout.getRef(), src); - if (!cg::is_valid(tv)) return {}; + if (!cg::is_valid(tv)) + return {}; std::string coord = "(" + cg::to_string(thread); - for (size_t i = 1; i < cg::rank(tv.shape()); ++i) coord += ",_"; + for (size_t i = 1; i < cg::rank(tv.shape()); ++i) + coord += ",_"; coord += ")"; auto crd = cg::from_string(coord); - if (!crd) return {}; + if (!crd) + return {}; auto [sliced, offset] = cg::slice_and_offset(*crd, tv); - if (!cg::is_valid(sliced)) return {}; + if (!cg::is_valid(sliced)) + return {}; // The tensor's modes beyond the tiler are untouched and keep their strides. size_t in = cg::rank(layout.getRef().shape()), r = cg::rank(tc.tiler); - size_t keep = cg::holds_vector(layout.getRef().shape()) && in > r ? in - r : 0; + size_t keep = + cg::holds_vector(layout.getRef().shape()) && in > r ? in - r : 0; Type result = LayoutType::get(ctx, zeroUnitModesHead(sliced, keep, false)); - if (c.ptr) return CuteMemRefType::get(ctx, movedPtr(c.ptr, offset), result); + if (c.ptr) + return CuteMemRefType::get(ctx, movedPtr(c.ptr, offset), result); if (c.iter) { auto sum = cg::arith_tuple_sum(c.iter.getRef(), offset); - if (!cg::is_valid(sum)) return {}; - return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), result); + if (!cg::is_valid(sum)) + return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), + result); } return rebuild(ctx, c, result); } @@ -2159,13 +2590,15 @@ Type partitionCopy(MLIRContext *ctx, const TiledCopy &tc, const Carrier &c, // The shapes of a tiler's modes, as one shape. std::optional tilerShape(const cg::tile &tiler) { std::string text = cg::to_string(tiler); - if (text.size() < 2 || text.front() != '[') return std::nullopt; + if (text.size() < 2 || text.front() != '[') + return std::nullopt; SmallVector parts; StringRef(text).drop_front().drop_back().split(parts, ';'); std::string shape = "("; for (size_t i = 0; i < parts.size(); ++i) { auto l = cg::from_string(parts[i].str()); - if (!l) return std::nullopt; + if (!l) + return std::nullopt; shape += (i ? "," : "") + cg::to_string(l->shape()); } return cg::from_string(shape + ")"); @@ -2175,68 +2608,87 @@ std::optional tilerShape(const cg::tile &tiler) { // into the atom's values. Type retileCopy(MLIRContext *ctx, const TiledCopy &tc, const Carrier &c) { auto layout = llvm::dyn_cast(c.layout); - if (!layout) return {}; + if (!layout) + return {}; const cg::layout &l = layout.getRef(); size_t R = cg::rank(l.shape()); int64_t V = asInt(cg::size(l, 0)); auto shape = tilerShape(tc.tiler); - if (V <= 0 || !shape) return {}; + if (V <= 0 || !shape) + return {}; cg::layout inv = cg::right_inverse(tc.tv).with_shape(*shape); cg::layout frgMN = cg::upcast(tc.tiledThreads * V, inv); - if (!cg::is_valid(frgMN)) return {}; + if (!cg::is_valid(frgMN)) + return {}; auto vLayout = layoutOf(std::to_string(V) + ":1"); auto atomVals = layoutOf(std::to_string(tc.atomVals) + ":1"); - if (!vLayout || !atomVals) return {}; + if (!vLayout || !atomVals) + return {}; cg::layout frgV = cg::zipped_divide( cg::logical_product(*vLayout, cg::right_inverse(frgMN)), *atomVals); - if (!cg::is_valid(frgV)) return {}; + if (!cg::is_valid(frgV)) + return {}; std::string tiler = "(" + std::to_string(V); auto each = cg::product_each(frgMN.shape()); - for (size_t i = 0; i < cg::rank(each); ++i) tiler += "," + cg::to_string(each[i]); + for (size_t i = 0; i < cg::rank(each); ++i) + tiler += "," + cg::to_string(each[i]); auto tilerShape = cg::from_string(tiler + ")"); - if (!tilerShape) return {}; + if (!tilerShape) + return {}; cg::layout tiled = cg::zipped_divide(l, *tilerShape); - if (!cg::is_valid(tiled)) return {}; + if (!cg::is_valid(tiled)) + return {}; cg::layout v = composeFirst(tiled, frgV); - if (!cg::is_valid(v)) return {}; + if (!cg::is_valid(v)) + return {}; // v(_, (0, _, _...)): the first rest mode is the one the tile added. std::vector modes{cg::get(v, 0)}; std::vector rest = modesOf(cg::get(v, 1)); - for (size_t i = 1; i < rest.size() && i < R; ++i) modes.push_back(rest[i]); - return rebuild(ctx, c, LayoutType::get(ctx, zeroUnitModes(cg::make_layout(modes)))); + for (size_t i = 1; i < rest.size() && i < R; ++i) + modes.push_back(rest[i]); + return rebuild(ctx, c, + LayoutType::get(ctx, zeroUnitModes(cg::make_layout(modes)))); } // make_cotiled_copy: the (thr,val) -> tile layout and the tiler of a copy // whose atom addresses `data` directly. -std::optional> cotiledCopy(const cg::layout &atomTV, - const cg::layout &data) { +std::optional> +cotiledCopy(const cg::layout &atomTV, const cg::layout &data) { auto unit = layoutOf("1:0"); - cg::layout invData = cg::make_layout(std::vector{cg::left_inverse(data), *unit}); + cg::layout invData = + cg::make_layout(std::vector{cg::left_inverse(data), *unit}); cg::layout tvData = cg::composition(invData, atomTV); - if (!cg::is_valid(tvData)) return std::nullopt; + if (!cg::is_valid(tvData)) + return std::nullopt; auto flat = cg::product_each(data.shape()); size_t R = cg::rank(flat); std::string flatText = cg::to_string(flat); std::string tilerText = "["; for (size_t i = 0; i < R; ++i) { std::string stride = "("; - for (size_t j = 0; j < R; ++j) stride += std::string(j ? "," : "") + (i == j ? "1" : "0"); + for (size_t j = 0; j < R; ++j) + stride += std::string(j ? "," : "") + (i == j ? "1" : "0"); auto pick = layoutOf(flatText + ":" + stride + ")"); - if (!pick) return std::nullopt; + if (!pick) + return std::nullopt; cg::layout mode = cg::filter(cg::composition(*pick, tvData)); - if (!cg::is_valid(mode)) return std::nullopt; + if (!cg::is_valid(mode)) + return std::nullopt; tilerText += (i ? ";" : "") + cg::to_string(mode); } auto tiler = cg::from_string(tilerText + "]"); auto flatShape = cg::from_string(flatText); - if (!tiler || !flatShape) return std::nullopt; + if (!tiler || !flatShape) + return std::nullopt; cg::layout tile2data = cg::composition(cg::layout(*flatShape), *tiler); cg::layout tv = cg::composition(cg::left_inverse(tile2data), tvData); - if (!cg::is_valid(tv)) return std::nullopt; + if (!cg::is_valid(tv)) + return std::nullopt; return std::make_pair(tv, *tiler); } -Type tiledCopyType(MLIRContext *ctx, Type atom, const cg::layout &tv, const cg::tile &tiler) { +Type tiledCopyType(MLIRContext *ctx, Type atom, const cg::layout &tv, + const cg::tile &tiler) { return TiledCopyType::get(ctx, atom, LayoutAttr::get(ctx, tv), TileAttr::get(ctx, tiler)); } @@ -2253,23 +2705,36 @@ std::optional tmaFormatOf(Type elem, Attribute attr) { TmaFormat f; if (attr) { auto format = llvm::dyn_cast(attr); - if (!format) return std::nullopt; + if (!format) + return std::nullopt; f.name = cute_nvgpu::stringifyTmaDataFormat(format.getValue()).str(); StringRef n = f.name; - if (n == "F16_RN" || n == "BF16_RN") f.bits = 16; - else if (n == "F32_RN" || n == "F32_FTZ_RN" || n == "INT32" || n == "TF32") f.bits = 32; - else if (n == "F64_RN" || n == "INT64") f.bits = 64; - else if (!n.drop_until(llvm::isDigit).take_while(llvm::isDigit).getAsInteger(10, f.bits)) {} - else return std::nullopt; + if (n == "F16_RN" || n == "BF16_RN") + f.bits = 16; + else if (n == "F32_RN" || n == "F32_FTZ_RN" || n == "INT32" || n == "TF32") + f.bits = 32; + else if (n == "F64_RN" || n == "INT64") + f.bits = 64; + else if (!n.drop_until(llvm::isDigit) + .take_while(llvm::isDigit) + .getAsInteger(10, f.bits)) { + } else + return std::nullopt; return f; } f.bits = bitsOfType(elem); - if (elem.isTF32()) f.name = "TF32_RN"; - else if (elem.isF16()) f.name = "F16_RN"; - else if (elem.isBF16()) f.name = "BF16_RN"; - else if (elem.isF32()) f.name = "F32_RN"; - else if (elem.isF64()) f.name = "F64_RN"; - else f.name = "U" + std::to_string(f.bits); + if (elem.isTF32()) + f.name = "TF32_RN"; + else if (elem.isF16()) + f.name = "F16_RN"; + else if (elem.isBF16()) + f.name = "BF16_RN"; + else if (elem.isF32()) + f.name = "F32_RN"; + else if (elem.isF64()) + f.name = "F64_RN"; + else + f.name = "U" + std::to_string(f.bits); return f; } @@ -2278,14 +2743,17 @@ std::optional tmaFormatOf(Type elem, Attribute attr) { std::vector> coalesce256Groups(const cg::layout &l) { auto fs = cg::flatten(l.shape()); auto ft = cg::flatten(l.stride()); - if (!cg::holds_vector(fs)) return {{0}}; + if (!cg::holds_vector(fs)) + return {{0}}; std::vector> groups; std::vector cur{0}; int64_t curS = asInt(fs[0]), curD = asInt(ft[0]); for (size_t i = 1; i < cg::rank(fs); ++i) { int64_t s = asInt(fs[i]), d = asInt(ft[i]); if (curS == 1) { - cur = {i}; curS = s; curD = d; + cur = {i}; + curS = s; + curD = d; } else if (s == 1) { continue; } else if (curS > 0 && curD >= 0 && s > 0 && d >= 0 && curS * curD == d && @@ -2294,7 +2762,9 @@ std::vector> coalesce256Groups(const cg::layout &l) { curS *= s; } else { groups.push_back(cur); - cur = {i}; curS = s; curD = d; + cur = {i}; + curS = s; + curD = d; } } groups.push_back(cur); @@ -2303,28 +2773,38 @@ std::vector> coalesce256Groups(const cg::layout &l) { // The layout with `shape`'s leaves regrouped by `groups`, over `stride`'s // leaves: what composing with the box shape does, keeping every basis. -std::optional regroup(const cg::layout &shape, const cg::layout &stride, - const std::vector> &groups) { +std::optional +regroup(const cg::layout &shape, const cg::layout &stride, + const std::vector> &groups) { auto fs = cg::flatten(shape.shape()); auto ft = cg::flatten(stride.stride()); auto leaf = [&](const auto &flat, size_t i) { - return cg::holds_vector(flat) ? cg::to_string(flat[i]) : cg::to_string(flat); + return cg::holds_vector(flat) ? cg::to_string(flat[i]) + : cg::to_string(flat); }; std::string shapes, strides; for (size_t g = 0; g < groups.size(); ++g) { - if (g) { shapes += ","; strides += ","; } + if (g) { + shapes += ","; + strides += ","; + } if (groups[g].size() == 1) { shapes += leaf(fs, groups[g][0]); strides += leaf(ft, groups[g][0]); continue; } - shapes += "("; strides += "("; + shapes += "("; + strides += "("; for (size_t k = 0; k < groups[g].size(); ++k) { - if (k) { shapes += ","; strides += ","; } + if (k) { + shapes += ","; + strides += ","; + } shapes += leaf(fs, groups[g][k]); strides += leaf(ft, groups[g][k]); } - shapes += ")"; strides += ")"; + shapes += ")"; + strides += ")"; } return layoutOf("(" + shapes + "):(" + strides + ")"); } @@ -2339,15 +2819,19 @@ cg::layout composeCoord(const cg::layout &lhs, const cg::layout &rhs) { modes.push_back(m); continue; } - if (!cg::holds_scaled_basis(d)) return cg::layout(cg::cg_error_t{}); + if (!cg::holds_scaled_basis(d)) + return cg::layout(cg::cg_error_t{}); const auto &basis = std::get(d); - if (!basis.value_holds_int()) return cg::layout(cg::cg_error_t{}); + if (!basis.value_holds_int()) + return cg::layout(cg::cg_error_t{}); auto scale = layoutOf(cg::to_string(m.shape()) + ":" + std::to_string(basis.static_integral_value())); - if (!scale) return cg::layout(cg::cg_error_t{}); + if (!scale) + return cg::layout(cg::cg_error_t{}); cg::layout sub = cg::get(lhs, basis.modes()); cg::layout composed = cg::composition(sub, *scale); - if (!cg::is_valid(composed)) return composed; + if (!cg::is_valid(composed)) + return composed; modes.push_back(composed); } return cg::make_layout(modes); @@ -2374,15 +2858,18 @@ bool basisAt(const cg::stride &leaf, const std::vector &path) { } // Which mode of a stride tuple carries the basis `path`, if any. -std::optional modeWithBasis(const cg::stride &strides, const std::vector &path) { +std::optional modeWithBasis(const cg::stride &strides, + const std::vector &path) { for (size_t j = 0; j < cg::rank(strides); ++j) { auto flat = cg::flatten(strides[j]); if (!cg::holds_vector(flat)) { - if (basisAt(flat, path)) return j; + if (basisAt(flat, path)) + return j; continue; } for (size_t k = 0; k < cg::rank(flat); ++k) - if (basisAt(flat[k], path)) return j; + if (basisAt(flat[k], path)) + return j; } return std::nullopt; } @@ -2394,37 +2881,49 @@ cg::layout tmaGbasis(const cg::layout &gmem, int64_t elemBits, int64_t tmaBits, auto fail = [] { return cg::layout(cg::cg_error_t{}); }; cg::layout inv = cg::right_inverse(smem); cg::layout full = cg::coalesce(cg::composition(ctaVMap, inv)); - if (!cg::is_valid(full)) return full; + if (!cg::is_valid(full)) + return full; std::vector fullModes = modesOf(full); // Keep the leading modes that step through gmem one element at a time. size_t smemRank = 0; for (; smemRank < fullModes.size(); ++smemRank) { const auto &d = fullModes[smemRank].stride(); - if (!cg::holds_scaled_basis(d)) break; + if (!cg::holds_scaled_basis(d)) + break; const auto &b = std::get(d); - if (!b.value_holds_int() || b.static_integral_value() != 1) break; + if (!b.value_holds_int() || b.static_integral_value() != 1) + break; } - if (smemRank == 0) return fail(); + if (smemRank == 0) + return fail(); fullModes.resize(smemRank); cg::layout sidx2gmode = cg::make_layout(fullModes); - cg::layout tileGstride = cg::recast(tmaBits, elemBits, composeCoord(gmem, sidx2gmode)); - if (!cg::is_valid(tileGstride)) return tileGstride; + cg::layout tileGstride = + cg::recast(tmaBits, elemBits, composeCoord(gmem, sidx2gmode)); + if (!cg::is_valid(tileGstride)) + return tileGstride; cg::layout gbasis = cg::make_identity_layout(gmem.shape()); cg::layout tileGbasisTmp = composeCoord(gbasis, sidx2gmode); - if (!cg::is_valid(tileGbasisTmp)) return tileGbasisTmp; - auto regrouped = regroup(tileGstride, tileGbasisTmp, coalesce256Groups(tileGstride)); - if (!regrouped) return fail(); + if (!cg::is_valid(tileGbasisTmp)) + return tileGbasisTmp; + auto regrouped = + regroup(tileGstride, tileGbasisTmp, coalesce256Groups(tileGstride)); + if (!regrouped) + return fail(); cg::layout tmaGbasisTile = *regrouped; // Gmem modes the tile misses still shape the descriptor, as size-1 modes. cg::layout gmemT = cg::recast(tmaBits, elemBits, gmem); - if (!cg::is_valid(gmemT)) return gmemT; + if (!cg::is_valid(gmemT)) + return gmemT; std::vector modes = modesOf(tmaGbasisTile); auto flatTile = cg::flatten(tmaGbasisTile.stride()); std::vector path; bool ok = true; walkLeaves(gmemT.shape(), gmemT.stride(), gbasis.stride(), path, - [&](const auto &s, const auto &d, const auto &e, const std::vector &p) { - if (asInt(s) == 1 || asInt(d) == 0) return; + [&](const auto &s, const auto &d, const auto &e, + const std::vector &p) { + if (asInt(s) == 1 || asInt(d) == 0) + return; bool found = false; if (!cg::holds_vector(flatTile)) { found = basisAt(flatTile, p); @@ -2432,33 +2931,41 @@ cg::layout tmaGbasis(const cg::layout &gmem, int64_t elemBits, int64_t tmaBits, for (size_t k = 0; k < cg::rank(flatTile) && !found; ++k) found = basisAt(flatTile[k], p); } - if (found) return; + if (found) + return; auto extra = layoutOf("1:" + cg::to_string(e)); - if (!extra) ok = false; - else modes.push_back(*extra); + if (!extra) + ok = false; + else + modes.push_back(*extra); }); - if (!ok) return fail(); + if (!ok) + return fail(); cg::layout result = cg::make_layout(modes); - if (modes.size() > 4) result = cg::group(4, modes.size(), result); + if (modes.size() > 4) + result = cg::group(4, modes.size(), result); return result; } // The TMA tensor's layout: gmem shape over TMA coordinate bases, scaled to // the descriptor's element. -std::optional tmaTensorLayout(const cg::layout &gmem, int64_t elemBits, - int64_t tmaBits, const cg::layout &gbasis) { +std::optional tmaTensorLayout(const cg::layout &gmem, + int64_t elemBits, int64_t tmaBits, + const cg::layout &gbasis) { cg::layout basis = cg::make_identity_layout(gmem.shape()); const auto &tmaStrides = gbasis.stride(); bool ok = true; std::string text; std::vector path; // Written in the shape's own nesting. - std::function write = - [&](const cg::shape &s, const cg::stride &d, const cg::stride &e) { + std::function + write = [&](const cg::shape &s, const cg::stride &d, + const cg::stride &e) { if (cg::holds_vector(s)) { text += "("; for (size_t i = 0; i < cg::rank(s); ++i) { - if (i) text += ","; + if (i) + text += ","; path.push_back(i); write(s[i], d[i], e[i]); path.pop_back(); @@ -2466,33 +2973,52 @@ std::optional tmaTensorLayout(const cg::layout &gmem, int64_t elemBi text += ")"; return; } - if (asInt(s) == 1 || asInt(d) == 0) { text += "0"; return; } - auto j = cg::holds_vector(tmaStrides) ? modeWithBasis(tmaStrides, path) - : (basisAt(tmaStrides, path) ? std::optional(0) : std::nullopt); - if (!j) { text += "0"; return; } + if (asInt(s) == 1 || asInt(d) == 0) { + text += "0"; + return; + } + auto j = cg::holds_vector(tmaStrides) + ? modeWithBasis(tmaStrides, path) + : (basisAt(tmaStrides, path) ? std::optional(0) + : std::nullopt); + if (!j) { + text += "0"; + return; + } if (*j == 0) { int64_t stride = asInt(d); - if (stride <= 0) { ok = false; return; } + if (stride <= 0) { + ok = false; + return; + } int64_t num = stride * elemBits, den = tmaBits; int64_t g = std::gcd(num, den); - num /= g; den /= g; - text += std::to_string(num) + (den == 1 ? "" : "/" + std::to_string(den)) + "@0"; + num /= g; + den /= g; + text += std::to_string(num) + + (den == 1 ? "" : "/" + std::to_string(den)) + "@0"; return; } auto mode = cg::holds_vector(tmaStrides) ? tmaStrides[*j] : tmaStrides; auto flat = cg::flatten(mode); - if (cg::holds_vector(flat) && cg::rank(flat) != 1) { ok = false; return; } + if (cg::holds_vector(flat) && cg::rank(flat) != 1) { + ok = false; + return; + } text += "1@" + std::to_string(*j); }; write(gmem.shape(), gmem.stride(), basis.stride()); - if (!ok) return std::nullopt; + if (!ok) + return std::nullopt; return layoutOf(cg::to_string(gmem.shape()) + ":" + text); } // The smem tile layout without its swizzle. std::optional plainLayoutOf(Type t) { - if (auto l = llvm::dyn_cast(t)) return l.getRef(); - if (auto c = llvm::dyn_cast(t)) return c.getRef().layout_b(); + if (auto l = llvm::dyn_cast(t)) + return l.getRef(); + if (auto c = llvm::dyn_cast(t)) + return c.getRef().layout_b(); return std::nullopt; } @@ -2502,92 +3028,118 @@ struct TmaAtomInfo { std::optional tmaAtomInfoOf(Type atom) { TmaAtomInfo info; - if (auto l = llvm::dyn_cast(atom)) { + if (auto l = + llvm::dyn_cast(atom)) { info.elemBits = bitsOfType(l.getValType()); info.copyBits = l.getCopyBits(); - } else if (auto s = llvm::dyn_cast(atom)) { + } else if (auto s = + llvm::dyn_cast( + atom)) { info.elemBits = bitsOfType(s.getValType()); info.copyBits = s.getCopyBits(); - } else if (auto r = llvm::dyn_cast(atom)) { + } else if (auto r = + llvm::dyn_cast( + atom)) { info.elemBits = bitsOfType(r.getValType()); info.copyBits = r.getCopyBits(); - } else if (auto l = llvm::dyn_cast(atom)) { + } else if (auto l = + llvm::dyn_cast( + atom)) { info.elemBits = bitsOfType(l.getValType()); info.copyBits = l.getCopyBits(); - } else if (auto s = llvm::dyn_cast(atom)) { + } else if (auto s = + llvm::dyn_cast( + atom)) { info.elemBits = bitsOfType(s.getValType()); info.copyBits = s.getCopyBits(); } else { return std::nullopt; } - if (info.elemBits <= 0 || info.copyBits <= 0) return std::nullopt; + if (info.elemBits <= 0 || info.copyBits <= 0) + return std::nullopt; return info; } - // make_non_exec_tiled_tma_load/store: the atom and the TMA tensor. LogicalResult inferTmaAtom(MLIRContext *ctx, std::optional loc, Type gmemType, Type smemType, Type ctaVMapType, - Attribute kind, Attribute format, SmallVectorImpl &out) { + Attribute kind, Attribute format, + SmallVectorImpl &out) { auto gmem = llvm::dyn_cast(gmemType); auto smem = plainLayoutOf(smemType); auto ctaVMap = llvm::dyn_cast(ctaVMapType); if (!gmem || !smem || !ctaVMap) - return emitOptionalError(loc, "expects a gmem tensor, an smem layout and a CTA value map"); + return emitOptionalError( + loc, "expects a gmem tensor, an smem layout and a CTA value map"); auto glayout = llvm::dyn_cast(gmem.getLayout()); - if (!glayout) return emitOptionalError(loc, "gmem tensor needs a plain layout"); + if (!glayout) + return emitOptionalError(loc, "gmem tensor needs a plain layout"); Type elem = gmem.getValueType(); auto fmt = tmaFormatOf(elem, format); - if (!fmt || fmt->bits <= 0) return emitOptionalError(loc, "unknown TMA data format"); + if (!fmt || fmt->bits <= 0) + return emitOptionalError(loc, "unknown TMA data format"); int64_t elemBits = bitsOfType(elem); - cg::layout gbasis = tmaGbasis(glayout.getRef(), elemBits, fmt->bits, *smem, ctaVMap.getRef()); - if (!cg::is_valid(gbasis)) return emitOptionalError(loc, "cannot build the TMA basis"); + cg::layout gbasis = + tmaGbasis(glayout.getRef(), elemBits, fmt->bits, *smem, ctaVMap.getRef()); + if (!cg::is_valid(gbasis)) + return emitOptionalError(loc, "cannot build the TMA basis"); auto tensor = tmaTensorLayout(glayout.getRef(), elemBits, fmt->bits, gbasis); - if (!tensor) return emitOptionalError(loc, "cannot build the TMA tensor"); + if (!tensor) + return emitOptionalError(loc, "cannot build the TMA tensor"); int64_t copyBits = asInt(cg::size(gbasis)) * fmt->bits; auto gbasisType = LayoutType::get(ctx, gbasis); std::optional dataFormat = cute_nvgpu::symbolizeTmaDataFormat(fmt->name); Type atom; if (auto k = llvm::dyn_cast_or_null(kind)) { - atom = cute_nvgpu::CopyAtomNonExecTiledTmaLoadType::get(ctx, k.getValue(), elem, copyBits, - gbasisType, dataFormat); - } else if (auto k = llvm::dyn_cast_or_null(kind)) { - atom = cute_nvgpu::CopyAtomNonExecTiledTmaReduceType::get(ctx, k.getValue(), elem, copyBits, - gbasisType, dataFormat); + atom = cute_nvgpu::CopyAtomNonExecTiledTmaLoadType::get( + ctx, k.getValue(), elem, copyBits, gbasisType, dataFormat); + } else if (auto k = + llvm::dyn_cast_or_null(kind)) { + atom = cute_nvgpu::CopyAtomNonExecTiledTmaReduceType::get( + ctx, k.getValue(), elem, copyBits, gbasisType, dataFormat); } else { - atom = cute_nvgpu::CopyAtomNonExecTiledTmaStoreType::get(ctx, elem, copyBits, gbasisType, - dataFormat); + atom = cute_nvgpu::CopyAtomNonExecTiledTmaStoreType::get( + ctx, elem, copyBits, gbasisType, dataFormat); } std::string zeros = "("; - for (size_t i = 0; i < cg::rank(gbasis.shape()); ++i) zeros += i ? ",0" : "0"; + for (size_t i = 0; i < cg::rank(gbasis.shape()); ++i) + zeros += i ? ",0" : "0"; auto iter = cg::from_string(zeros + ")"); - if (!iter) return emitOptionalError(loc, "cannot build the TMA coordinate"); + if (!iter) + return emitOptionalError(loc, "cannot build the TMA coordinate"); out.push_back(atom); out.push_back(CoordTensorType::get(ctx, IntTupleType::get(ctx, *iter), LayoutType::get(ctx, *tensor))); return success(); } -} // namespace +} // namespace -// ---------------------------------------------------------------- cute copy ops +// ---------------------------------------------------------------- cute copy +// ops // TiledCopy::get_layoutD_TV over the reference tile, then the tiled copy of // another atom with that layout (make_tiled_copy_D). Type mlir::cutlass_compiler::cute::tiledCopyD(Type atom, TiledCopyType tiled) { auto tc = tiledCopyOf(tiled); - if (!tc) return {}; + if (!tc) + return {}; auto shape = tilerShape(tc->tiler); - if (!shape) return {}; - auto refShape = cg::from_string("(" + cg::to_string(*shape) + ",1)"); - if (!refShape) return {}; + if (!shape) + return {}; + auto refShape = + cg::from_string("(" + cg::to_string(*shape) + ",1)"); + if (!refShape) + return {}; cg::layout refD(*refShape); - cg::layout ref2trg = cg::composition(cg::right_inverse(tc->traits.ref), tc->traits.dst); + cg::layout ref2trg = + cg::composition(cg::right_inverse(tc->traits.ref), tc->traits.dst); cg::layout tv = tile2thrfrg(*tc, refD, ref2trg); - if (!cg::is_valid(tv) || cg::rank(tv.shape()) < 2) return {}; - cg::layout layoutTV = - zeroUnitModes(cg::make_layout(std::vector{cg::get(tv, 0), cg::get(tv, 1)})); + if (!cg::is_valid(tv) || cg::rank(tv.shape()) < 2) + return {}; + cg::layout layoutTV = zeroUnitModes( + cg::make_layout(std::vector{cg::get(tv, 0), cg::get(tv, 1)})); MLIRContext *ctx = atom.getContext(); return TiledCopyType::get(ctx, atom, LayoutAttr::get(ctx, layoutTV), tiled.getTilerMn()); @@ -2602,8 +3154,11 @@ LogicalResult TiledCopyPartitionSOp::inferReturnTypes( auto c = carrierOf(a.getInput().getType()); auto coord = llvm::dyn_cast(a.getCoord().getType()); if (!tc || !c || !coord) - return emitOptionalError(loc, "expects a tiled copy, a tensor and a thread coord"); - return infer(loc, out, [&] { return partitionCopy(ctx, *tc, *c, coord.getRef(), true); }); + return emitOptionalError( + loc, "expects a tiled copy, a tensor and a thread coord"); + return infer(loc, out, [&] { + return partitionCopy(ctx, *tc, *c, coord.getRef(), true); + }); } LogicalResult TiledCopyPartitionDOp::inferReturnTypes( @@ -2615,8 +3170,11 @@ LogicalResult TiledCopyPartitionDOp::inferReturnTypes( auto c = carrierOf(a.getInput().getType()); auto coord = llvm::dyn_cast(a.getCoord().getType()); if (!tc || !c || !coord) - return emitOptionalError(loc, "expects a tiled copy, a tensor and a thread coord"); - return infer(loc, out, [&] { return partitionCopy(ctx, *tc, *c, coord.getRef(), false); }); + return emitOptionalError( + loc, "expects a tiled copy, a tensor and a thread coord"); + return infer(loc, out, [&] { + return partitionCopy(ctx, *tc, *c, coord.getRef(), false); + }); } LogicalResult TiledCopyRetileOp::inferReturnTypes( @@ -2626,13 +3184,13 @@ LogicalResult TiledCopyRetileOp::inferReturnTypes( Adaptor a(operands, attrs, props); auto tc = tiledCopyOf(a.getTiledCopy().getType()); auto c = carrierOf(a.getInput().getType()); - if (!tc || !c) return emitOptionalError(loc, "expects a tiled copy and a tensor"); + if (!tc || !c) + return emitOptionalError(loc, "expects a tiled copy and a tensor"); return infer(loc, out, [&] { return retileCopy(ctx, *tc, *c); }); } // ---------------------------------------------------------------- cute_nvgpu - namespace mlir::cutlass_compiler::cute_nvgpu { LogicalResult AtomSetValueOp::inferReturnTypes( @@ -2652,47 +3210,55 @@ LogicalResult AtomCopyMakeExecTmaOp::inferReturnTypes( bool over = a.getOverride(), noOob = a.getNoFullyOobTile(); Type in = a.getInput().getType(); auto stride = cg::from_string("()"); - if (!stride) return emitOptionalError(loc, "cannot build the TMA stride"); + if (!stride) + return emitOptionalError(loc, "cannot build the TMA stride"); auto gStride = StrideType::get(ctx, *stride); if (auto load = llvm::dyn_cast(in)) { TiledTmaLoad kind = load.getKind(); - bool twoSm = kind == TiledTmaLoad::sm_100_2sm || kind == TiledTmaLoad::sm_100_2sm_multicast; - bool mcast = kind == TiledTmaLoad::sm_90_multicast || kind == TiledTmaLoad::sm_100_2sm_multicast; - out.push_back(CopyAtomTmaLoadType::get(ctx, load.getValType(), /*sparsity=*/1, - load.getCopyBits(), TmaLoadMode::tiled, twoSm ? 2 : 1, - gStride, mcast, load.getTmaGbasis(), over, noOob)); + bool twoSm = kind == TiledTmaLoad::sm_100_2sm || + kind == TiledTmaLoad::sm_100_2sm_multicast; + bool mcast = kind == TiledTmaLoad::sm_90_multicast || + kind == TiledTmaLoad::sm_100_2sm_multicast; + out.push_back(CopyAtomTmaLoadType::get( + ctx, load.getValType(), /*sparsity=*/1, load.getCopyBits(), + TmaLoadMode::tiled, twoSm ? 2 : 1, gStride, mcast, load.getTmaGbasis(), + over, noOob)); return success(); } if (auto store = llvm::dyn_cast(in)) { - out.push_back(CopyAtomTmaStoreType::get(ctx, store.getValType(), /*sparsity=*/1, - store.getCopyBits(), TmaStoreMode::tiled, gStride, - store.getTmaGbasis(), over, noOob)); + out.push_back(CopyAtomTmaStoreType::get( + ctx, store.getValType(), /*sparsity=*/1, store.getCopyBits(), + TmaStoreMode::tiled, gStride, store.getTmaGbasis(), over, noOob)); return success(); } if (auto red = llvm::dyn_cast(in)) { - out.push_back(CopyAtomTmaReduceType::get(ctx, red.getValType(), red.getCopyBits(), - TmaStoreMode::tiled, red.getKind(), gStride, - red.getTmaGbasis())); + out.push_back(CopyAtomTmaReduceType::get( + ctx, red.getValType(), red.getCopyBits(), TmaStoreMode::tiled, + red.getKind(), gStride, red.getTmaGbasis())); return success(); } if (auto load = llvm::dyn_cast(in)) { Im2ColTmaLoad kind = load.getKind(); - bool twoSm = kind == Im2ColTmaLoad::sm_100_2sm || kind == Im2ColTmaLoad::sm_100_2sm_multicast; - bool mcast = kind == Im2ColTmaLoad::sm_90_multicast || kind == Im2ColTmaLoad::sm_100_2sm_multicast; - out.push_back(CopyAtomIm2ColTmaLoadType::get(ctx, load.getValType(), load.getCopyBits(), - twoSm ? 2 : 1, gStride, mcast, load.getTmaGbasis())); + bool twoSm = kind == Im2ColTmaLoad::sm_100_2sm || + kind == Im2ColTmaLoad::sm_100_2sm_multicast; + bool mcast = kind == Im2ColTmaLoad::sm_90_multicast || + kind == Im2ColTmaLoad::sm_100_2sm_multicast; + out.push_back(CopyAtomIm2ColTmaLoadType::get( + ctx, load.getValType(), load.getCopyBits(), twoSm ? 2 : 1, gStride, + mcast, load.getTmaGbasis())); return success(); } if (auto store = llvm::dyn_cast(in)) { - out.push_back(CopyAtomIm2ColTmaStoreType::get(ctx, store.getValType(), store.getCopyBits(), - gStride, store.getTmaGbasis())); + out.push_back(CopyAtomIm2ColTmaStoreType::get(ctx, store.getValType(), + store.getCopyBits(), gStride, + store.getTmaGbasis())); return success(); } // The gather/scatter atoms' executable forms are not recovered. - return emitOptionalError(loc, "expects a non-executable tiled or im2col TMA atom"); + return emitOptionalError(loc, + "expects a non-executable tiled or im2col TMA atom"); } - LogicalResult AtomCopyMakeNonExecTiledTmaLoadOp::inferReturnTypes( MLIRContext *ctx, std::optional loc, ValueRange operands, DictionaryAttr attrs, PropertyRef props, RegionRange, @@ -2735,10 +3301,12 @@ LogicalResult AtomTmaPartitionOp::inferReturnTypes( auto ctaLayout = llvm::dyn_cast(a.getCtaLayout().getType()); auto smem = carrierOf(a.getSmemTensor().getType()); if (!info || !coord || !ctaLayout || !smem) - return emitOptionalError(loc, "expects a TMA atom, a CTA coord and layout, and tensors"); + return emitOptionalError( + loc, "expects a TMA atom, a CTA coord and layout, and tensors"); auto smemLayout = llvm::dyn_cast_or_null(smem->layout); if (!smemLayout || !cg::holds_vector(smemLayout.getRef().shape())) - return emitOptionalError(loc, "smem tensor needs a plain layout of rank >= 1"); + return emitOptionalError(loc, + "smem tensor needs a plain layout of rank >= 1"); int64_t numVal = info->copyBits / info->elemBits; cg::layout inv = cg::right_inverse(cg::get(smemLayout.getRef(), 0)); int64_t b = asInt(cg::size(inv)); @@ -2750,55 +3318,74 @@ LogicalResult AtomTmaPartitionOp::inferReturnTypes( if (n != b) { auto rest = layoutOf(std::to_string(n / b) + ":1"); layoutV = cg::logical_product(inv, *rest); - if (!cg::is_valid(layoutV)) return emitOptionalError(loc, "cannot tile the smem vector"); + if (!cg::is_valid(layoutV)) + return emitOptionalError(loc, "cannot tile the smem vector"); } auto tmaV = layoutOf(std::to_string(numVal) + ":1"); layoutV = cg::logical_divide(layoutV, *tmaV); - if (!cg::is_valid(layoutV)) return emitOptionalError(loc, "cannot divide the smem vector"); + if (!cg::is_valid(layoutV)) + return emitOptionalError(loc, "cannot divide the smem vector"); // The multicast offset inside the instruction's vector. int64_t ctas = asInt(cg::cosize(ctaLayout.getRef())); std::optional multicast; if (ctas > 1) { // The CTA layout scaled by the share: (cosize:share) o cta_layout. - auto share = layoutOf(std::to_string(ctas) + ":" + std::to_string(numVal / ctas)); + auto share = + layoutOf(std::to_string(ctas) + ":" + std::to_string(numVal / ctas)); cg::layout scaled = cg::composition(*share, ctaLayout.getRef()); - if (!cg::is_valid(scaled)) return emitOptionalError(loc, "cannot scale the CTA layout"); + if (!cg::is_valid(scaled)) + return emitOptionalError(loc, "cannot scale the CTA layout"); multicast = cg::layout_eval(coord.getRef(), scaled); } SmallVector results; auto partition = [&](Type tensor) -> Type { auto c = carrierOf(tensor); - if (!c) return {}; + if (!c) + return {}; auto layout = llvm::dyn_cast(c->layout); - if (!layout) return {}; - cg::layout v = cg::coalesce(composeFirst(layout.getRef(), layoutV), profileOf("((1,1))")); - if (!cg::is_valid(v)) return {}; + if (!layout) + return {}; + cg::layout v = cg::coalesce(composeFirst(layout.getRef(), layoutV), + profileOf("((1,1))")); + if (!cg::is_valid(v)) + return {}; // Only the vector's unit modes get a zero stride; the rest keep theirs. - Type result = LayoutType::get(ctx, zeroUnitModesHead(v, cg::rank(v.shape()) - 1, false)); - if (!multicast) return rebuild(ctx, *c, result); + Type result = LayoutType::get( + ctx, zeroUnitModesHead(v, cg::rank(v.shape()) - 1, false)); + if (!multicast) + return rebuild(ctx, *c, result); // The index split over the vector's modes, column-major, keeping what // its divisibility says of each coordinate. auto vec = cg::get(v.shape(), 0); - if (!cg::holds_vector(vec)) return {}; + if (!cg::holds_vector(vec)) + return {}; auto split = splitIndex(*multicast, vec[0]); - if (!split) return {}; + if (!split) + return {}; std::string crd = "((" + *split + ",0)"; - for (size_t i = 1; i < cg::rank(v.shape()); ++i) crd += ",0"; + for (size_t i = 1; i < cg::rank(v.shape()); ++i) + crd += ",0"; auto at = cg::from_string(crd + ")"); - if (!at) return {}; + if (!at) + return {}; cg::int_tuple offset = cg::layout_eval(*at, v); - if (c->ptr) return CuteMemRefType::get(ctx, movedPtr(c->ptr, offset), result); + if (c->ptr) + return CuteMemRefType::get(ctx, movedPtr(c->ptr, offset), result); if (c->iter) { auto sum = cg::arith_tuple_sum(c->iter.getRef(), offset); - if (!cg::is_valid(sum)) return {}; - return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), result); + if (!cg::is_valid(sum)) + return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), + result); } return rebuild(ctx, *c, result); }; results.push_back(partition(a.getSmemTensor().getType())); - for (Value target : a.getTargetTensors()) results.push_back(partition(target.getType())); + for (Value target : a.getTargetTensors()) + results.push_back(partition(target.getType())); for (Type t : results) - if (!t) return emitOptionalError(loc, "cannot partition for TMA"); + if (!t) + return emitOptionalError(loc, "cannot partition for TMA"); out.append(results.begin(), results.end()); return success(); } @@ -2807,20 +3394,26 @@ namespace { // make_cotiled_copy over a tensor memory tensor for an atom whose thread // layout is `atomT`. Type tmemTiledCopy(MLIRContext *ctx, std::optional loc, Type atom, - Type tmem, const std::string &atomT, SmallVectorImpl &out) { + Type tmem, const std::string &atomT, + SmallVectorImpl &out) { auto traits = copyTraitsOf(atom); auto m = llvm::dyn_cast(tmem); auto layout = m ? llvm::dyn_cast(m.getLayout()) : LayoutType(); - if (!traits || !layout || !cg::is_valid(traits->valId)) return {}; + if (!traits || !layout || !cg::is_valid(traits->valId)) + return {}; auto t = layoutOf(atomT); - if (!t) return {}; + if (!t) + return {}; cg::layout v = cg::coalesce(cg::upcast(traits->bits, traits->valId)); - if (!cg::is_valid(v)) return {}; - auto tiled = cotiledCopy(cg::make_layout(std::vector{*t, v}), layout.getRef()); - if (!tiled) return {}; + if (!cg::is_valid(v)) + return {}; + auto tiled = cotiledCopy(cg::make_layout(std::vector{*t, v}), + layout.getRef()); + if (!tiled) + return {}; return tiledCopyType(ctx, atom, tiled->first, tiled->second); } -} // namespace +} // namespace // make_tmem_copy: four warps, each at the same place in its own lane quarter. LogicalResult AtomCopyMakeTmemCopyOp::inferReturnTypes( @@ -2829,10 +3422,12 @@ LogicalResult AtomCopyMakeTmemCopyOp::inferReturnTypes( SmallVectorImpl &out) { Adaptor a(operands, attrs, props); auto traits = copyTraitsOf(a.getAtom().getType()); - if (!traits || traits->bits <= 0) return emitOptionalError(loc, "expects a tmem atom"); + if (!traits || traits->bits <= 0) + return emitOptionalError(loc, "expects a tmem atom"); int64_t lane = kTmemLaneBits / traits->bits; return infer(loc, out, [&] { - return tmemTiledCopy(ctx, loc, a.getAtom().getType(), a.getTmemMemref().getType(), + return tmemTiledCopy(ctx, loc, a.getAtom().getType(), + a.getTmemMemref().getType(), "(32,4):(0," + std::to_string(32 * lane) + ")", out); }); } @@ -2844,7 +3439,8 @@ LogicalResult AtomCopyMakeS2TCopyOp::inferReturnTypes( SmallVectorImpl &out) { Adaptor a(operands, attrs, props); auto traits = copyTraitsOf(a.getCopyS2tAtom().getType()); - if (!traits) return emitOptionalError(loc, "expects an s2t atom"); + if (!traits) + return emitOptionalError(loc, "expects an s2t atom"); return infer(loc, out, [&] { return tmemTiledCopy(ctx, loc, a.getCopyS2tAtom().getType(), a.getTmemMemref().getType(), @@ -2857,7 +3453,8 @@ namespace { // bits, viewed through `desc`. LogicalResult s2tSmemDescView(std::optional loc, Type atom, Type view, Type desc, SmallVectorImpl &out) { - if (auto tiled = llvm::dyn_cast(atom)) atom = tiled.getCopyAtom(); + if (auto tiled = llvm::dyn_cast(atom)) + atom = tiled.getCopyAtom(); auto m = llvm::dyn_cast(view); auto layout = m ? llvm::dyn_cast(m.getLayout()) : LayoutType(); if (!llvm::isa(atom) || !layout) @@ -2865,13 +3462,15 @@ LogicalResult s2tSmemDescView(std::optional loc, Type atom, Type view, // The compiler wants mode 0 as (values, 1). const cg::layout &l = layout.getRef(); auto m0 = cg::get(l.shape(), 0); - if (!cg::holds_vector(m0) || cg::rank(m0) != 2 || !cg::holds_int(cg::size(m0, 1)) || - cg::size(m0, 1).as_int64() != 1) + if (!cg::holds_vector(m0) || cg::rank(m0) != 2 || + !cg::holds_int(cg::size(m0, 1)) || cg::size(m0, 1).as_int64() != 1) return emitOptionalError(loc, "expects mode 0 as (values, 1)"); return infer(loc, out, [&]() -> Type { cg::layout v = cg::recast(128, bitsOfType(m.getValueType()), l); - if (!cg::is_valid(v)) return {}; - return SmemDescViewType::get(atom.getContext(), desc, LayoutAttr::get(atom.getContext(), v)); + if (!cg::is_valid(v)) + return {}; + return SmemDescViewType::get(atom.getContext(), desc, + LayoutAttr::get(atom.getContext(), v)); }); } @@ -2880,7 +3479,7 @@ LogicalResult s2tSmemDescView(std::optional loc, Type atom, Type view, LogicalResult unrecovered(std::optional loc, StringRef op) { return emitOptionalError(loc, op, " result inference is not recovered"); } -} // namespace +} // namespace LogicalResult AtomGetCopyS2TSmemDescViewOp::inferReturnTypes( MLIRContext *ctx, std::optional loc, ValueRange operands, @@ -2907,7 +3506,8 @@ LogicalResult GetGridConstantPointerOp::inferReturnTypes( Type t = operands[0].getType(); bool desc = llvm::isa(t); out.push_back(PtrType::get(ctx, t, AddressSpace::generic, - desc ? 64 : PtrType::getNaturalAlignment(t), {}, {})); + desc ? 64 : PtrType::getNaturalAlignment(t), {}, + {})); return success(); } @@ -2919,40 +3519,52 @@ LogicalResult AtomMakeTmaResidueTensorOp::inferReturnTypes( SmallVectorImpl &out) { Adaptor a(operands, attrs, props); Type atom = a.getTmaAtom().getType(), gbasis; - if (auto l = llvm::dyn_cast(atom)) gbasis = l.getTmaGbasis(); - if (auto s = llvm::dyn_cast(atom)) gbasis = s.getTmaGbasis(); + if (auto l = llvm::dyn_cast(atom)) + gbasis = l.getTmaGbasis(); + if (auto s = llvm::dyn_cast(atom)) + gbasis = s.getTmaGbasis(); auto basis = llvm::dyn_cast_or_null(gbasis); auto m = llvm::dyn_cast(a.getGmemTensor().getType()); auto gmem = m ? llvm::dyn_cast(m.getLayout()) : LayoutType(); return infer(loc, out, [&]() -> Type { - if (!basis || !gmem || !cg::holds_vector(basis.getRef().stride())) return {}; + if (!basis || !gmem || !cg::holds_vector(basis.getRef().stride())) + return {}; const cg::stride &modes = basis.getRef().stride(); // Each box mode's one basis: the gmem leaf it runs over. std::vector> paths; for (size_t j = 0; j < cg::rank(modes); ++j) { auto flat = cg::flatten(modes[j]); - const cg::stride &leaf = cg::holds_vector(flat) && cg::rank(flat) == 1 ? flat[0] : flat; - if (!cg::holds_scaled_basis(leaf)) return {}; + const cg::stride &leaf = + cg::holds_vector(flat) && cg::rank(flat) == 1 ? flat[0] : flat; + if (!cg::holds_scaled_basis(leaf)) + return {}; paths.push_back(std::get(leaf).modes()); } std::vector iter(paths.size()), strides; std::vector path; const cg::layout &g = gmem.getRef(); walkLeaves(g.shape(), g.stride(), g.stride(), path, - [&](const auto &s, const auto &, const auto &, const std::vector &p) { - auto j = std::find(paths.begin(), paths.end(), p) - paths.begin(); - if (j == static_cast(paths.size())) return strides.push_back("0"); + [&](const auto &s, const auto &, const auto &, + const std::vector &p) { + auto j = + std::find(paths.begin(), paths.end(), p) - paths.begin(); + if (j == static_cast(paths.size())) + return strides.push_back("0"); iter[j] = cg::to_string(s); strides.push_back("-1@" + std::to_string(j)); }); for (const std::string &e : iter) - if (e.empty()) return {}; + if (e.empty()) + return {}; size_t i = 0; auto layout = cg::from_string(cg::to_string(g.shape()) + ":" + nestLike(g.shape(), strides, i)); - auto start = cg::from_string("(" + llvm::join(iter, ",") + ")"); - if (!layout || !start) return {}; - return CoordTensorType::get(ctx, IntTupleType::get(ctx, *start), LayoutType::get(ctx, *layout)); + auto start = + cg::from_string("(" + llvm::join(iter, ",") + ")"); + if (!layout || !start) + return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, *start), + LayoutType::get(ctx, *layout)); }); } @@ -2971,12 +3583,18 @@ LogicalResult AtomGetCoordTensorOp::inferReturnTypes( PropertyRef, RegionRange, SmallVectorImpl &) { \ return unrecovered(loc, name); \ } -CUTE_UNRECOVERED(AtomCopyMakeNonExec2DGather4TmaLoadOp, "cute_nvgpu.atom.make_non_exec_2d_gather4_tma_load") -CUTE_UNRECOVERED(AtomCopyMakeNonExec2DScatter4TmaStoreOp, "cute_nvgpu.atom.make_non_exec_2d_scatter4_tma_store") -CUTE_UNRECOVERED(AtomCopyMakeNonExecIm2ColTmaLoadOp, "cute_nvgpu.atom.make_non_exec_im2col_tma_load") -CUTE_UNRECOVERED(AtomCopyMakeNonExecIm2ColTmaStoreOp, "cute_nvgpu.atom.make_non_exec_im2col_tma_store") -CUTE_UNRECOVERED(MakeUMMASmemDescCircularSM103Op, "cute_nvgpu.sm103.make_umma_smem_desc_circular") -CUTE_UNRECOVERED(MakeUMMASmemDescCircularSM107Op, "cute_nvgpu.sm107.make_umma_smem_desc_circular") +CUTE_UNRECOVERED(AtomCopyMakeNonExec2DGather4TmaLoadOp, + "cute_nvgpu.atom.make_non_exec_2d_gather4_tma_load") +CUTE_UNRECOVERED(AtomCopyMakeNonExec2DScatter4TmaStoreOp, + "cute_nvgpu.atom.make_non_exec_2d_scatter4_tma_store") +CUTE_UNRECOVERED(AtomCopyMakeNonExecIm2ColTmaLoadOp, + "cute_nvgpu.atom.make_non_exec_im2col_tma_load") +CUTE_UNRECOVERED(AtomCopyMakeNonExecIm2ColTmaStoreOp, + "cute_nvgpu.atom.make_non_exec_im2col_tma_store") +CUTE_UNRECOVERED(MakeUMMASmemDescCircularSM103Op, + "cute_nvgpu.sm103.make_umma_smem_desc_circular") +CUTE_UNRECOVERED(MakeUMMASmemDescCircularSM107Op, + "cute_nvgpu.sm107.make_umma_smem_desc_circular") #undef CUTE_UNRECOVERED -} // namespace mlir::cutlass_compiler::cute_nvgpu +} // namespace mlir::cutlass_compiler::cute_nvgpu diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteInference.h b/mlir/dialects/cute_ir/CuteInference.h similarity index 71% rename from mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteInference.h rename to mlir/dialects/cute_ir/CuteInference.h index 26fbfb69..c7837b20 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteInference.h +++ b/mlir/dialects/cute_ir/CuteInference.h @@ -1,7 +1,7 @@ -//===- CuteInference.h - result types the compiler infers --------*- C++ -*-===// +//===- CuteInference.h - result types the compiler infers -------*- C++ -*-===// -#ifndef CUTE_IR_DIALECT_CUTE_IR_CUTE_INFERENCE_H -#define CUTE_IR_DIALECT_CUTE_IR_CUTE_INFERENCE_H +#ifndef MLIR_DIALECTS_CUTE_IR_CUTEINFERENCE_H +#define MLIR_DIALECTS_CUTE_IR_CUTEINFERENCE_H #include "cute_ir/Dialect/Cute/IR/CuteDialect.h" @@ -12,13 +12,13 @@ Type tiledCopyD(Type atom, TiledCopyType tiled); // Result types of operations the compiler does not infer (their builders // take the type): what the DSL passes, or null when not computable here. -Type makeLayoutResultType(Type shape, Type stride); // stride may be null +Type makeLayoutResultType(Type shape, Type stride); // stride may be null Type makeViewResultType(Type iter, Type layout); Type getResultType(Type input, ArrayRef mode); Type assumeResultType(Type src); Type makeArithTupleIterResultType(Type tuple); Type fastDivmodCreateDivisorResultType(Type divisor); -Type fastDivmodComputeResultType(Type divisor); // both results +Type fastDivmodComputeResultType(Type divisor); // both results } // namespace mlir::cutlass_compiler::cute -#endif // CUTE_IR_DIALECT_CUTE_IR_CUTE_INFERENCE_H +#endif // MLIR_DIALECTS_CUTE_IR_CUTEINFERENCE_H diff --git a/mlir/dialects/cute_ir/lib/Dialect/IR/CuteOps.cpp b/mlir/dialects/cute_ir/CuteOps.cpp similarity index 69% rename from mlir/dialects/cute_ir/lib/Dialect/IR/CuteOps.cpp rename to mlir/dialects/cute_ir/CuteOps.cpp index e28b9e8a..0c8ddaf3 100644 --- a/mlir/dialects/cute_ir/lib/Dialect/IR/CuteOps.cpp +++ b/mlir/dialects/cute_ir/CuteOps.cpp @@ -2,8 +2,8 @@ #include "cute_ir/Dialect/Cute/IR/CuteDialect.h" // Operand constraints name cute_nvgpu and LLVM types. -#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h" #include "mlir/Dialect/LLVMIR/LLVMTypes.h" +#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h" #define GET_OP_CLASSES -#include "cute_ir/Dialect/Cute/IR/CuteOps.cpp.inc" +#include "mlir/dialects/cute_ir/CuteOps.cpp.inc" diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td b/mlir/dialects/cute_ir/CuteOps.td similarity index 99% rename from mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td rename to mlir/dialects/cute_ir/CuteOps.td index 768fbf9f..d2e342df 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td +++ b/mlir/dialects/cute_ir/CuteOps.td @@ -17,11 +17,11 @@ // cute.deref_desc_iter operand #0: no probe type accepted; any type allowed // cute.make_desc_iter operand #0: no probe type accepted; any type allowed -#ifndef CUTE_IR_GENERATED_CUTE_OPS_TD -#define CUTE_IR_GENERATED_CUTE_OPS_TD +#ifndef MLIR_DIALECTS_CUTE_IR_CUTEOPS_TD +#define MLIR_DIALECTS_CUTE_IR_CUTEOPS_TD include "cute_ir/Dialect/Cute/IR/CuteAttrs.td" -include "cute_ir/Dialect/Cute/IR/CuteEnums.td" +include "mlir/dialects/cute_ir/CuteEnums.td" include "mlir/Interfaces/InferTypeOpInterface.td" include "mlir/Interfaces/SideEffectInterfaces.td" @@ -3792,4 +3792,4 @@ def Cute_ZippedProductOp : Cute_InferredOp<"zipped_product", [Pure]> { ); } -#endif // CUTE_IR_GENERATED_CUTE_OPS_TD +#endif // MLIR_DIALECTS_CUTE_IR_CUTEOPS_TD diff --git a/mlir/dialects/cute_ir/lib/Dialect/IR/CuteTypes.cpp b/mlir/dialects/cute_ir/CuteTypes.cpp similarity index 96% rename from mlir/dialects/cute_ir/lib/Dialect/IR/CuteTypes.cpp rename to mlir/dialects/cute_ir/CuteTypes.cpp index bd37b7bd..604dfbf2 100644 --- a/mlir/dialects/cute_ir/lib/Dialect/IR/CuteTypes.cpp +++ b/mlir/dialects/cute_ir/CuteTypes.cpp @@ -1,3 +1,5 @@ +//===- CuteTypes.cpp - parsers, printers and verifiers of cute types ------===// + // clang-format off /*************************************************************************************************** * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. @@ -32,13 +34,12 @@ // clang-format on #include "cute_ir/Dialect/Cute/IR/CuteDialect.h" -#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h" +#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h" #include "mlir/Dialect/LLVMIR/LLVMTypes.h" #include "mlir/IR/Builders.h" #include "mlir/IR/DialectImplementation.h" #include "mlir/IR/OpImplementation.h" - #include "llvm/ADT/TypeSwitch.h" #include "llvm/Support/FormatVariadic.h" #include "llvm/Support/MathExtras.h" @@ -53,8 +54,7 @@ namespace cg = cutegen; // The compiler's parser of the algebra types reads `<"text">` leniently: a // missing or unreadable payload, including a bare mnemonic, gives the default // value (`()`, `():()`, `[]`, `S<0,4,3>`) instead of an error. -template -static T parseLenientPayload(mlir::AsmParser &parser) { +template static T parseLenientPayload(mlir::AsmParser &parser) { std::string str; if (succeeded(parser.parseOptionalLess()) && succeeded(parser.parseOptionalString(&str)) && @@ -78,7 +78,7 @@ static LogicalResult generatedCuteTypePrinter(Type type, // Emit the tablegen-generated type storage / parsing machinery (also drives // CuteDialect::parseType / printType via useDefaultTypePrinterParser). #define GET_TYPEDEF_CLASSES -#include "cute_ir/Dialect/Cute/IR/CuteTypes.cpp.inc" +#include "mlir/dialects/cute_ir/CuteTypes.cpp.inc" #undef generatedTypeParser #undef generatedTypePrinter @@ -186,7 +186,7 @@ struct CuteOpAsmInterface : public OpAsmDialectInterface { void CuteDialect::registerCuteTypes(CuteDialect *dialect) { dialect->addTypes< #define GET_TYPEDEF_LIST -#include "cute_ir/Dialect/Cute/IR/CuteTypes.cpp.inc" +#include "mlir/dialects/cute_ir/CuteTypes.cpp.inc" >(); dialect->addInterfaces(); } @@ -199,9 +199,10 @@ void CuteDialect::registerCuteTypes(CuteDialect *dialect) { //===----------------------------------------------------------------------===// mlir::Type IntTupleType::parse(mlir::AsmParser &parser) { - return get(parser.getContext(), - IntTupleAttr::get(parser.getContext(), - parseLenientPayload(parser))); + return get( + parser.getContext(), + IntTupleAttr::get(parser.getContext(), + parseLenientPayload(parser))); } void IntTupleType::print(mlir::AsmPrinter &printer) const { @@ -286,7 +287,7 @@ IntTupleType::verifyEntries(mlir::DataLayoutEntryListRef entries, mlir::Type CoordType::parse(mlir::AsmParser &parser) { return get(parser.getContext(), CoordAttr::get(parser.getContext(), - parseLenientPayload(parser))); + parseLenientPayload(parser))); } void CoordType::print(mlir::AsmPrinter &printer) const { @@ -365,7 +366,7 @@ CoordType::verifyEntries(mlir::DataLayoutEntryListRef entries, mlir::Type ShapeType::parse(mlir::AsmParser &parser) { return get(parser.getContext(), ShapeAttr::get(parser.getContext(), - parseLenientPayload(parser))); + parseLenientPayload(parser))); } void ShapeType::print(mlir::AsmPrinter &printer) const { @@ -444,7 +445,7 @@ ShapeType::verifyEntries(mlir::DataLayoutEntryListRef entries, mlir::Type StrideType::parse(mlir::AsmParser &parser) { auto loc = parser.getCurrentLocation(); auto attr = StrideAttr::get(parser.getContext(), - parseLenientPayload(parser)); + parseLenientPayload(parser)); return getChecked([&] { return parser.emitError(loc); }, parser.getContext(), attr); } @@ -534,7 +535,7 @@ StrideType::verifyEntries(mlir::DataLayoutEntryListRef entries, mlir::Type LayoutType::parse(mlir::AsmParser &parser) { auto loc = parser.getCurrentLocation(); auto attr = LayoutAttr::get(parser.getContext(), - parseLenientPayload(parser)); + parseLenientPayload(parser)); return getChecked([&] { return parser.emitError(loc); }, parser.getContext(), attr); } @@ -641,7 +642,7 @@ LayoutType::verifyEntries(mlir::DataLayoutEntryListRef entries, mlir::Type TileType::parse(mlir::AsmParser &parser) { auto loc = parser.getCurrentLocation(); auto attr = TileAttr::get(parser.getContext(), - parseLenientPayload(parser)); + parseLenientPayload(parser)); return getChecked([&] { return parser.emitError(loc); }, parser.getContext(), attr); } @@ -899,7 +900,7 @@ ComposedLayoutType::verifyEntries(mlir::DataLayoutEntryListRef entries, mlir::Type SwizzleType::parse(mlir::AsmParser &parser) { auto loc = parser.getCurrentLocation(); auto attr = SwizzleAttr::get(parser.getContext(), - parseLenientPayload(parser)); + parseLenientPayload(parser)); return getChecked([&] { return parser.emitError(loc); }, parser.getContext(), attr); } @@ -1015,7 +1016,7 @@ SwizzleType::verifyEntries(mlir::DataLayoutEntryListRef entries, // CuteAttrsCompiler.td) //===----------------------------------------------------------------------===// -#include "cute_ir/Dialect/Cute/IR/CuteEnums.cpp.inc" +#include "mlir/dialects/cute_ir/CuteEnums.cpp.inc" static OptionalParseResult generatedCuteTypeParser(DialectAsmParser &parser, StringRef *mnemonic, @@ -1113,8 +1114,9 @@ static std::optional chunkBits(const cutegen::layout &layout) { return bits; } -LogicalResult BitLayoutAttr::verify(function_ref emitError, - LayoutAttr layout) { +LogicalResult +BitLayoutAttr::verify(function_ref emitError, + LayoutAttr layout) { const cutegen::layout &l = layout.getRef(); if (!cutegen::is_static(l)) return emitError() << "bitlayout must be fully static but got " << layout; @@ -1241,13 +1243,13 @@ static FailureOr parsePtrFields(AsmParser &parser, Type *layout) { return failure(); } first = false; - if (auto space = symbolizeAddressSpace(keyword); - space && next <= kSpace) { + if (auto space = symbolizeAddressSpace(keyword); space && next <= kSpace) { fields.space = *space; next = kAlign; } else if (keyword == "align" && next <= kAlign) { uint64_t alignment; - if (failed(parser.parseLess()) || failed(parser.parseInteger(alignment)) || + if (failed(parser.parseLess()) || + failed(parser.parseInteger(alignment)) || failed(parser.parseGreater())) return failure(); fields.alignment = alignment; @@ -1261,7 +1263,8 @@ static FailureOr parsePtrFields(AsmParser &parser, Type *layout) { return failure(); next = kDone; } else { - parser.emitError(parser.getCurrentLocation(), "unexpected pointer field `") + parser.emitError(parser.getCurrentLocation(), + "unexpected pointer field `") << keyword << "`"; return failure(); } @@ -1273,8 +1276,8 @@ static FailureOr parsePtrFields(AsmParser &parser, Type *layout) { // Defaults are left out: they are what the parser fills in. static PtrType buildPtr(AsmParser &parser, const PtrFields &fields) { - SwizzleAttr swizzle = isIdentitySwizzle(fields.swizzle) ? SwizzleAttr() - : fields.swizzle; + SwizzleAttr swizzle = + isIdentitySwizzle(fields.swizzle) ? SwizzleAttr() : fields.swizzle; BitLayoutAttr bitlayout = fields.bitlayout; if (bitlayout && bitlayout == PtrType::getDefaultBitLayout(fields.valueType, fields.space)) @@ -1365,7 +1368,7 @@ static bool isDenseForSparsity(const cutegen::layout &layout, return false; auto shape = cutegen::flatten(layout.shape()); auto stride = cutegen::flatten(layout.stride()); - std::vector> modes; // (stride, extent) + std::vector> modes; // (stride, extent) auto add = [&](const auto &extent, const auto &step) { if (extent.as_int64() != 1 && step.as_int64() != 0) modes.emplace_back(step.as_int64(), extent.as_int64()); @@ -1410,8 +1413,9 @@ Type mlir::cutlass_compiler::cute::MemRefType::parse(AsmParser &parser) { PtrType ptr = buildPtr(parser, *fields); if (!ptr) return {}; - return getChecked([&] { return parser.emitError(parser.getCurrentLocation()); }, - parser.getContext(), ptr, layout); + return getChecked( + [&] { return parser.emitError(parser.getCurrentLocation()); }, + parser.getContext(), ptr, layout); } void mlir::cutlass_compiler::cute::MemRefType::print( @@ -1433,9 +1437,9 @@ LogicalResult mlir::cutlass_compiler::cute::MemRefType::verify( Type valueType = ptr.getValueType(); if (!valueType || !(valueType.isIntOrIndexOrFloat() || llvm::isa(valueType))) - return emitError() - << "expects value type to be int, float, ptr or sparse_elem, but got '" - << valueType << "'"; + return emitError() << "expects value type to be int, float, ptr or " + "sparse_elem, but got '" + << valueType << "'"; if (auto sparse = llvm::dyn_cast(valueType)) { int64_t sparsity = sparse.getNumLogical(); bool dense; @@ -1448,15 +1452,16 @@ LogicalResult mlir::cutlass_compiler::cute::MemRefType::verify( c.offset().as_int64() % sparsity == 0 && isDenseForSparsity(c.layout_b(), sparsity); } else { - dense = isDenseForSparsity(llvm::cast(layout).getRef(), - sparsity); + dense = + isDenseForSparsity(llvm::cast(layout).getRef(), sparsity); } if (!dense) return emitError() << "layout must be dense but cannot be divided by sparsity"; } if (auto composed = llvm::dyn_cast(layout)) { - if (!isIdentitySwizzle(ptr.getSwizzle()) && composed.getRef().is_a_swizzle() && + if (!isIdentitySwizzle(ptr.getSwizzle()) && + composed.getRef().is_a_swizzle() && !composed.getRef().swizzle_a().is_identity()) return emitError() << "cannot have both a swizzle on the pointer and a " "non-identity swizzle in the layout"; @@ -1498,10 +1503,10 @@ Type CoordTensorType::parse(AsmParser &parser) { if (failed(parser.parseComma()) || !(layout = parseLayoutPayload(parser)) || failed(parser.parseGreater())) return {}; - return getChecked([&] { return parser.emitError(parser.getCurrentLocation()); }, - parser.getContext(), - IntTupleType::get(parser.getContext(), std::move(*tuple)), - layout); + return getChecked( + [&] { return parser.emitError(parser.getCurrentLocation()); }, + parser.getContext(), + IntTupleType::get(parser.getContext(), std::move(*tuple)), layout); } void CoordTensorType::print(AsmPrinter &printer) const { @@ -1512,9 +1517,9 @@ void CoordTensorType::print(AsmPrinter &printer) const { printer << '>'; } -LogicalResult CoordTensorType::verify( - function_ref emitError, IntTupleType arithTuple, - Type layout) { +LogicalResult +CoordTensorType::verify(function_ref emitError, + IntTupleType arithTuple, Type layout) { if (!arithTuple) return emitError() << "expects an arithmetic tuple"; if (!llvm::isa(layout)) @@ -1572,9 +1577,9 @@ void FastDivmodDivisorType::print(AsmPrinter &printer) const { printer << '>'; } -LogicalResult SparseElemType::verify( - function_ref emitError, int numLogical, - Type physicalType) { +LogicalResult +SparseElemType::verify(function_ref emitError, + int numLogical, Type physicalType) { if (numLogical <= 0) return emitError() << "sparse element type logical count must be > 0"; Type element = physicalType; diff --git a/mlir/dialects/cute_ir/CuteTypesAll.td b/mlir/dialects/cute_ir/CuteTypesAll.td new file mode 100644 index 00000000..99766dcf --- /dev/null +++ b/mlir/dialects/cute_ir/CuteTypesAll.td @@ -0,0 +1,12 @@ +//===- CuteTypesAll.td - all cute types --------------------*- tablegen -*-===// +// +// The dialect has one generated type parser and printer, so the release's +// types and the compiler's are generated together from here. + +#ifndef MLIR_DIALECTS_CUTE_IR_CUTETYPESALL_TD +#define MLIR_DIALECTS_CUTE_IR_CUTETYPESALL_TD + +include "cute_ir/Dialect/Cute/IR/CuteTypes.td" +include "mlir/dialects/cute_ir/CuteTypesCompiler.td" + +#endif // MLIR_DIALECTS_CUTE_IR_CUTETYPESALL_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesCompiler.td b/mlir/dialects/cute_ir/CuteTypesCompiler.td similarity index 97% rename from mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesCompiler.td rename to mlir/dialects/cute_ir/CuteTypesCompiler.td index 3b240ad2..6ca5588a 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesCompiler.td +++ b/mlir/dialects/cute_ir/CuteTypesCompiler.td @@ -25,12 +25,12 @@ // dense but cannot be divided by sparsity") is inferred from probes: every // probe agrees with it. -#ifndef CUTE_IR_GENERATED_CUTE_TYPES_COMPILER_TD -#define CUTE_IR_GENERATED_CUTE_TYPES_COMPILER_TD +#ifndef MLIR_DIALECTS_CUTE_IR_CUTETYPESCOMPILER_TD +#define MLIR_DIALECTS_CUTE_IR_CUTETYPESCOMPILER_TD include "cute_ir/Dialect/Cute/IR/CuteTypes.td" -include "cute_ir/Dialect/Cute/IR/CuteAttrsCompiler.td" -include "cute_ir/Dialect/Cute/IR/CuteEnums.td" +include "mlir/dialects/cute_ir/CuteAttrsCompiler.td" +include "mlir/dialects/cute_ir/CuteEnums.td" def Cute_PtrType : Cute_Type<"Ptr", "ptr"> { let summary = "Pointer with address space, alignment, swizzle and bit layout"; @@ -266,4 +266,4 @@ def Cute_TupleType : Cute_Type<"Tuple", "tuple"> { let assemblyFormat = "`<` $types `>`"; } -#endif // CUTE_IR_GENERATED_CUTE_TYPES_COMPILER_TD +#endif // MLIR_DIALECTS_CUTE_IR_CUTETYPESCOMPILER_TD diff --git a/mlir/dialects/cute_ir/README.md b/mlir/dialects/cute_ir/README.md index 6a67ad3d..6a667ce5 100644 --- a/mlir/dialects/cute_ir/README.md +++ b/mlir/dialects/cute_ir/README.md @@ -8,33 +8,30 @@ pinned in `third_party/cute_ir`, one patch giving cutegen the the compiler's remaining types, attributes and enums and ratio strides allowed, the compiler's attributes, types and enums beyond the release (`CuteAttrsCompiler.td`, `CuteTypesCompiler.td`, `CuteEnums.td` and the -`CuteNVGPU*.td` files), and every operation of the compiler's registry in -`CuteOps.td` and `CuteNVGPUOps.td`. +`../cute_nvgpu` definitions), and every operation of the compiler's registry in +`CuteOps.td` and the companion `CuteNVGPUOps.td`. -It also carries NVIDIA's `nvvm` and `cuda` dialects as the same compiler -registers them (`include/cute_ir/Dialect/{NVVM,Cuda}`): every operation, -attribute, type and enum, rebuilt from the release's bindings and the live -compiler, since no `.td` ships for them and the compiler's `nvvm` is not -upstream's. The CuTe builder registers them, with the upstream `llvm` -dialect, so its kernels are built without parsing any of their text. +The companion dialects live in `../cute_nvgpu`, `../nvvm`, and `../cuda`, +with separate Bazel targets and Zig modules. NVIDIA's `nvvm` differs from +upstream's. The CuTe builder registers these dialects together with upstream +`llvm`. Updating to a new DSL release means refreshing the compiler entry-point offsets and regenerating the `.td` files from the release's Python bindings -and compiler. The C API (except the algebra attributes, `CuteAttributes.*`) +and compiler. The C API (except the algebra attribute functions in `capi.cc`) and the Zig types, attributes, enums and operation builders are generated from those `.td` files and follow them. ## Zig -Depend on `//mlir/dialects/cute_ir` and import `mlir/dialects/cute_ir`, or use -`@import("mlir/dialects").cute` through the dialect aggregator. +Depend on `//mlir/dialects/cute_ir` and import `mlir/dialects/cute_ir`. +`cute.registerDialect(registry)` registers `cute`. -`cute.registerDialects(registry)` registers `cute`, `cute_nvgpu`, and NVIDIA's -`nvvm` and `cuda` (handles `cute_nvvm` and `cuda`: this `nvvm` and upstream's -cannot share a registry). Cute builders, types, and attributes live directly in -`cute`; CuteNVGPU builders and types live in `cute.nvgpu`, NVVM's in -`cute.nvvm` and CUDA's in `cute.cuda`. `mlir/dialects/nvvm.zig` and -`mlir/dialects/cuda.zig` are convenience builders over the last two. +Import each companion dialect explicitly and add its Bazel dependency: +`mlir/dialects/cute_nvgpu`, `mlir/dialects/nvvm`, and `mlir/dialects/cuda`. +CUDA and NVVM expose generated bindings in `.ops` and convenience builders +at the module root. NVIDIA's NVVM handle is `cute_nvvm`; its dialect cannot +share a registry with upstream NVVM. ```zig const cute = @import("mlir/dialects/cute_ir"); @@ -44,7 +41,10 @@ const pointer = try cute.PtrType.get(ctx, .{ .addressSpace = .smem, .alignment = 16, }); -const layout_attr = try cute.LayoutAttr.get(ctx, "4:1"); +const layout_attr = try cute.LayoutAttr.get(ctx, .{ + .shape = try .getStatic(ctx, 4), + .stride = try .getStatic(ctx, 1), +}); const layout = try cute.LayoutType.get(ctx, .{ .attr = layout_attr.attribute() }); const memref = try cute.MemRefType.get(ctx, .{ .ptr = pointer.type_(), @@ -53,24 +53,25 @@ const memref = try cute.MemRefType.get(ctx, .{ const alloc = cute.memref_alloc_smem(ctx, memref.type_(), loc).appendTo(block); ``` -Type and attribute constructors call the C++ `get`/`getChecked` functions, -not a parser. `InitArgs` fields are the ODS parameter names in lower camel -case; getters are the C++ accessor names without underscores. Optional and defaulted parameters have defaults in `InitArgs`. Enums -are Zig enums with the compiler's values (the C API takes the integer value). -`get` returns `error.InvalidMlir` for invalid parameters or mismatched contexts. -`type_()` converts a typed wrapper to `*const mlir.Type` for operation builders. -Algebra types take an algebra attribute, `cute.LayoutType.get(ctx, .{ .attr = -layout_attr })`; algebra attribute constructors call the C++ algebra parser -with the raw expression and `getValue()` returns the canonical text. NVGPU atom -types take their typed parameters, such as `.valType` and `.copyBits`; -`cute.nvgpu.typeFromPayload(T, ctx, "")` parses one from the text after -its mnemonic. For reading existing IR use MLIR's generic `Type.parse`. +The bindings follow `mlir/dialects/README.md`. Algebra types take an algebra +attribute, `cute.LayoutType.get(ctx, .{ .attr = layout_attr.attribute() })`. +Algebra attributes are built bottom up with named constructors: integer +tuples, coordinates, shapes and strides from `getStatic`, `getDynamic`, +`getBasis` (strides) or `getUnderscore` (coordinates) leaves and `getTuple`; +layouts from a shape and a stride; tiles from `getLayout`, `getUnderscore` and +`getTuple`; swizzles from their three parameters and composed layouts from an +inner swizzle or layout, an offset and an outer layout. `value()` returns the +canonical text. NVGPU atom types take their typed parameters, such +as `.valType` and `.copyBits`. For reading existing IR use MLIR's generic +`Type.parse`. Operation builders take the operands, then the result types, then the attributes, then the location, and call `mlir.Operation.make`. Parameters follow ODS names, with a trailing underscore where needed. Result parameters -append `_type` or `_types`; attributes take `*const mlir.Attribute`. Pass `null` -for absent optional parameters and `&.{}` for empty variadic ones. Dots in +append `_type` or `_types`; attributes take `*const mlir.Attribute`. Optional +operands, attributes and result types go in a trailing `opts` struct whose +fields default to null; pass `.{}` to omit them, and `&.{}` for empty variadic +operands. Dots in mnemonics become underscores. Variable-length operand and result groups go through `Operation.make`'s `.variadic` API, which writes the segment-size attributes. Builders verify the created operations; to defer verification, use @@ -80,14 +81,20 @@ Builders of operations with regions (`cuda.kernel`) and of terminators (`cuda.return`) do not verify on creation, since their blocks are completed afterwards; the module is verified instead. -The C API is also available independently of Zig from `lib/CAPI:cute`, -`lib/CAPI:cute_nvgpu`, `lib/CAPI:nvvm` and `lib/CAPI:cuda`, via -`cute_ir-c/Dialect/Cute.h`, `CuteNVGPU.h`, `NVVM.h` (names prefixed -`CuteNVVM`) and `Cuda.h`; constructors return a null handle for invalid -parameters. +Each dialect exposes a `:capi` target and a package-local `capi.h`, independently +of Zig: `//mlir/dialects/cute_ir:capi`, `//mlir/dialects/cute_nvgpu:capi`, +`//mlir/dialects/nvvm:capi`, and `//mlir/dialects/cuda:capi`. C API declarations +and definitions are grouped by registration, attributes, and types in +`capi.h` and `capi.cc`. Constructors return a null handle for invalid parameters. + +C++ implementations, headers, and TableGen sources live directly in each +dialect package. Generated `.inc` files are emitted into the same package +under Bazel's output tree. Virtual headers preserve upstream CuTe include +paths required by the release sources. ```sh -bazel test //mlir/dialects/cute_ir:test +bazel test //mlir/dialects/cute_ir:test //mlir/dialects/cute_nvgpu:test \ + //mlir/dialects/nvvm:test //mlir/dialects/cuda:test ``` These tests only check the bindings: every bound operation is registered, diff --git a/mlir/dialects/cute_ir/REGENERATING.md b/mlir/dialects/cute_ir/REGENERATING.md index 823973ac..52b49e69 100644 --- a/mlir/dialects/cute_ir/REGENERATING.md +++ b/mlir/dialects/cute_ir/REGENERATING.md @@ -6,12 +6,13 @@ to check the result. How to get there is left to you: combine the sources as you see fit, write throwaway scripts, probe the compiler. Scripts and intermediate tables are not committed; only the definitions are. -Paths below are relative to this directory (the one holding this file). +Bare file names are in this directory, `../` are the sibling packages, +and other paths are relative to the repository root. ## Goal -Every dialect this directory carries (`cute`, `cute_nvgpu`, and NVIDIA's -`nvvm` and `cuda`, see below) must describe the DSL compiler's +The `cute` dialect in this directory and its companions in `../cute_nvgpu`, +`../nvvm`, and `../cuda` must describe the DSL compiler's registry as completely and as precisely as the release allows: every operation, type, attribute and enum the compiler registers, with every fact that can be established about each of them. Nothing is opaque unless it cannot @@ -38,8 +39,8 @@ without it, and so that anything built here is accepted by it unchanged. result type inference; the other dialect `.cpp` files and the `BUILD.bazel` files complete them. Edit them as the new definitions require. For `nvvm` and `cuda` these are `NVVMDialect.{h,cpp}` (with the `#nvvm.target` - verifier) and `CudaDialect.{h,cpp}` under `include/` and `lib/Dialect/`. -- **Generated: what you produce.** Every `.td` file under `include/` that is not + verifier) and `CudaDialect.{h,cpp}` in their dialect package roots. +- **Generated: what you produce.** Every package-local `.td` file that is not a dialect declaration: the operations (`CuteOps.td`, `CuteNVGPUOps.td`, `NVVMOps.td`, `CudaOps.td`), the compiler's types, attributes and enums beyond the release (`CuteTypesCompiler.td`, `CuteAttrsCompiler.td`, @@ -50,28 +51,29 @@ without it, and so that anything built here is accepted by it unchanged. any new `.td` you need to split them sensibly. The dialect declarations (`*Dialect.td`) change only as far as the generated files need. - **Language bindings, if present.** When the tree carries bindings derived - from these definitions (a C API under `lib/CAPI`, builder files next to this + from these definitions (a `capi.cc`/`capi.h` C API in each dialect package, builder files in each directory), regenerate them from the records of the new definitions (`llvm-tblgen --dump-json` gives them all; do not regex the `.td` files). When a class changes shape (opaque to parametric, merged, split, renamed accessors), change the bindings to match, and keep the existing - convenience helpers working where it costs little. + convenience helpers working where it costs little. Names, builder + parameters and `opts` follow `mlir/dialects/README.md`. Consumers outside this directory use the dialect classes and must keep compiling: search the repository for the dialect namespaces (`cutlass_compiler::cute`, `cute_nvgpu`, `nvvm`, `cuda`), for the Zig modules -(`cute.nvgpu`, `cute.nvvm`, `cute.cuda`, and the convenience layers over them -in `mlir/dialects/nvvm.zig` and `mlir/dialects/cuda.zig`) and for the headers -under `include/cute_ir`. When a definition changes shape, update those users in the +(`mlir/dialects/cute_nvgpu`, `mlir/dialects/nvvm`, and +`mlir/dialects/cuda`) and for the headers +in the dialect packages. When a definition changes shape, update those users in the same change. Tighter operand constraints also break consumers that built IR with loose types, and so does an operation that stops inferring its results: fix the consumer, not the constraint. ## NVIDIA's `nvvm` and `cuda` dialects -The kernels the CuTe builder emits (`kernels/cute`) use two more dialects of +The kernels the CuTe builder emits (`dsl/cute`) use two more dialects of the compiler: `nvvm` (special registers, mbarriers, tcgen05, ...) and `cuda` -(kernels, launch configurations, host runtime calls). Both are carried here +(kernels, launch configurations, host runtime calls). Both have sibling packages and must be regenerated with every release, like `cute` and `cute_nvgpu`. - **Neither is upstream's.** The compiler is built on NVIDIA's own LLVM @@ -82,23 +84,22 @@ and must be regenerated with every release, like `cute` and `cute_nvgpu`. compiler. No `.td` ships with the release for either: rebuild both from the bindings and the live compiler, as for the cute dialects. Upstream's NVVM `.td` is neither a source nor a substitute. -- **Where they live.** `include/cute_ir/Dialect/{NVVM,Cuda}/IR` (declarations - and generated `.td`), `lib/Dialect/{NVVM,Cuda}/IR` (dialect and inference - `.cpp`), `include/cute_ir-c/Dialect/{NVVM,Cuda}*.h` and - `lib/CAPI/{NVVM,Cuda}*.cpp` (C API), and `nvvm.zig`, `cuda.zig` (Zig, - `cute.nvvm` and `cute.cuda`). +- **Where they live.** `../nvvm` and `../cuda` contain the declarations, + generated `.td` files, dialect and inference `.cpp` files, and consolidated + `capi.h`/`capi.cc` C APIs. Each package's `ops.zig` contains the generated + bindings; `nvvm.zig` and `cuda.zig` provide the convenience builders. + - **Names.** The C++ namespaces are `mlir::cutlass_compiler::nvvm` and `mlir::cutlass_compiler::cuda`. The dialects are named `nvvm` and `cuda` as in the compiler, so a registry holds this `nvvm` or upstream's, never both; upstream's NVVM C++ is linked into the tree and must stay unregistered where this one is. The C dialect handles are `cute_nvvm` and `cuda`, and the C API - of `nvvm` is prefixed `CuteNVVM` (`mlirAttributeIsACuteNVVM`, - `mlirCuteNVVMAttrGet`) so that neither collides with upstream's + of `nvvm` is prefixed `CuteNVVM` so that neither collides with upstream's `mlirGetDialectHandle__nvvm__` or a future upstream NVVM C API; `cuda`'s is prefixed `Cuda`. - **`llvm` is upstream's.** NVVM operands are `!llvm.ptr` and LLVM structs, and the builder emits `llvm.*` operations. The CuTe context - registers the upstream LLVM dialect, bound by `mlir/dialects/llvm_dialect.zig` + registers the upstream LLVM dialect, bound by `mlir/dialects/llvm/llvm.zig` (its builders, and `mlirLLVMPointerTypeGet` for `!llvm.ptr`); the nvvm and cuda constraints name `::mlir::LLVM` types. NVIDIA's `llvm` is not vendored: its types print and parse the same, and the compiler reads the @@ -144,10 +145,10 @@ Sources specific to them, besides those in the table below: The bindings follow the cute ones: the C API and the Zig enums, typed attribute and type wrappers and operation builders are generated from -`llvm-tblgen --dump-json` of the generated `.td` files, then formatted with -`zig fmt`. Builders of operations with regions, and of terminators, do not -verify on creation (their blocks are filled, or their block is completed, -afterwards). `mlir/dialects/nvvm.zig` and `mlir/dialects/cuda.zig` are the +`llvm-tblgen --dump-json` of the generated `.td` files, following +`mlir/dialects/README.md` (names, wrappers, comments, formatting). Builders of +operations with regions, and of terminators, do not verify on creation (their +blocks are filled, or their block is completed, afterwards). `mlir/dialects/nvvm/nvvm.zig` and `mlir/dialects/cuda/cuda.zig` are the hand-written convenience layer the builder calls; they fix the attributes the kernels do not choose and must keep building no text. @@ -212,8 +213,7 @@ The round-trip corpus: run every example of the archive's `examples/python/CuTeDSL` with `CUTE_DSL_KEEP=ir-debug`, which keeps each compiled module's IR; set `CUTE_DSL_ARCH` for the architecture-restricted ones. This gives hundreds of real modules covering far more operations than -the fixtures in the repository (the testdata under `tools/` and the kernels' -fixtures), which are added to it. +the kernels' fixtures in the repository, which are added to it. When sources disagree, the live compiler wins; record the disagreement in a comment. @@ -313,12 +313,11 @@ A change is done when all of these hold: tests of every consumer found above (kernel emitters, host tools, language bindings), built with the tree's usual Bazel configuration. 5. **NVIDIA's compiler reads what the kernels emit.** Dump the IR of every - kernel the repository's tests emit through `kernels/cute`: the builder's - own tests, the MXFP4 emission tests under `zml/moe/cute_kernels`, and the - FlashMLA program (`zml/attention/cute_kernels/flashmla_sm100.zig`, whose - `Program.emit` needs a temporary test with a few configurations, single-CTA, - clustered and persistent). Dump both with the previous definitions and with - the new ones. In the release's venv, `import cutlass`, then parse and + kernel the repository's tests emit through `dsl/cute`: the builder's + own tests and the CuTe kernels under `zml/kernels/moe/cute`, + `zml/kernels/flash_attention/cute` and `zml/kernels/paged_attention/cute`. + Dump both with the previous definitions and with the new ones. In the + release's venv, `import cutlass`, then parse and verify each module with `cutlass._mlir.ir.Module.parse`, and compare the generic print (`get_asm(print_generic_op_form=True)`) of each module before and after: it must be identical, and a module the compiler rejected before diff --git a/mlir/dialects/cute_ir/capi.cc b/mlir/dialects/cute_ir/capi.cc new file mode 100644 index 00000000..fe1296db --- /dev/null +++ b/mlir/dialects/cute_ir/capi.cc @@ -0,0 +1,1450 @@ +//===- capi.cc - C API for the cute dialect -------------------------------===// + +#include "capi.h" + +#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" +#include "mlir/CAPI/Support.h" +#include "llvm/ADT/STLExtras.h" + +#include + +// Registration + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Cute, cute, + mlir::cutlass_compiler::cute::CuteDialect) + +// Attributes + +namespace cute = mlir::cutlass_compiler::cute; + +// Unwraps `handle` into `out`: false when it is of another context or not a +// T, or when it is null and not optional. +template +static bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { + auto value = unwrap(handle); + if (!value) + return optional; + if (value.getContext() != ctx) + return false; + out = llvm::dyn_cast(value); + return static_cast(out); +} + +template +static MlirAttribute getAlgebraAttr(MlirContext context, + typename Attr::algebra_t algebra) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(Attr::get(ctx, std::move(algebra))); +} + +template +static MlirAttribute getStaticAttr(MlirContext context, int64_t value) { + return getAlgebraAttr( + context, typename Attr::algebra_t(cutegen::static_int_t(value))); +} + +template +static MlirAttribute getDynamicAttr(MlirContext context, int32_t width, + int64_t divisibility) { + if (!llvm::is_contained({1, 8, 16, 32, 64}, width) || divisibility <= 0) + return {nullptr}; + return getAlgebraAttr( + context, typename Attr::algebra_t(cutegen::mlir_dynamic_t( + cutegen::dynamic_int_properties_t(width, divisibility)))); +} + +template +static MlirAttribute getUnderscoreAttr(MlirContext context) { + return getAlgebraAttr( + context, typename Attr::algebra_t(cutegen::underscore_t())); +} + +template +static MlirAttribute getTupleAttr(MlirContext context, intptr_t numElements, + MlirAttribute const *elements) { + auto *ctx = unwrap(context); + if (!ctx || numElements < 0 || (numElements > 0 && !elements)) + return {nullptr}; + typename Attr::algebra_t::vector_t children; + children.reserve(numElements); + for (intptr_t i = 0; i < numElements; ++i) { + Attr element; + if (!unwrapAs(ctx, elements[i], /*optional=*/false, element)) + return {nullptr}; + children.push_back(element.getRef()); + } + return getAlgebraAttr(context, + typename Attr::algebra_t(std::move(children))); +} + +template +static MlirStringRef getAlgebraValue(MlirAttribute attr) { + auto value = llvm::cast(unwrap(attr)); + // Intern the serialization so the returned view outlives this function. + auto text = mlir::StringAttr::get(value.getContext(), + cutegen::to_string(value.getRef())); + return wrap(text.getValue()); +} + +MlirAttribute mlirCuteIntTupleAttrStaticGet(MlirContext context, + int64_t value) { + return getStaticAttr(context, value); +} + +MlirAttribute mlirCuteIntTupleAttrDynamicGet(MlirContext context, int32_t width, + int64_t divisibility) { + return getDynamicAttr(context, width, divisibility); +} + +MlirAttribute mlirCuteIntTupleAttrTupleGet(MlirContext context, + intptr_t numElements, + MlirAttribute const *elements) { + return getTupleAttr(context, numElements, elements); +} + +bool mlirAttributeIsACuteIntTuple(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteIntTupleAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteCoordAttrStaticGet(MlirContext context, int64_t value) { + return getStaticAttr(context, value); +} + +MlirAttribute mlirCuteCoordAttrDynamicGet(MlirContext context, int32_t width, + int64_t divisibility) { + return getDynamicAttr(context, width, divisibility); +} + +MlirAttribute mlirCuteCoordAttrUnderscoreGet(MlirContext context) { + return getUnderscoreAttr(context); +} + +MlirAttribute mlirCuteCoordAttrTupleGet(MlirContext context, + intptr_t numElements, + MlirAttribute const *elements) { + return getTupleAttr(context, numElements, elements); +} + +bool mlirAttributeIsACuteCoord(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteCoordAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteShapeAttrStaticGet(MlirContext context, int64_t value) { + return getStaticAttr(context, value); +} + +MlirAttribute mlirCuteShapeAttrDynamicGet(MlirContext context, int32_t width, + int64_t divisibility) { + return getDynamicAttr(context, width, divisibility); +} + +MlirAttribute mlirCuteShapeAttrTupleGet(MlirContext context, + intptr_t numElements, + MlirAttribute const *elements) { + return getTupleAttr(context, numElements, elements); +} + +bool mlirAttributeIsACuteShape(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteShapeAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteStrideAttrStaticGet(MlirContext context, int64_t value) { + return getStaticAttr(context, value); +} + +MlirAttribute mlirCuteStrideAttrDynamicGet(MlirContext context, int32_t width, + int64_t divisibility) { + return getDynamicAttr(context, width, divisibility); +} + +MlirAttribute mlirCuteStrideAttrBasisGet(MlirContext context, int64_t numerator, + int64_t denominator, intptr_t numModes, + int32_t const *modes) { + if (denominator <= 0 || numModes <= 0 || !modes) + return {nullptr}; + cutegen::scaled_basis::vector_t path(modes, modes + numModes); + auto basis = + denominator == 1 + ? cutegen::scaled_basis(std::move(path), numerator) + : cutegen::scaled_basis(std::move(path), + cutegen::ratio(numerator, denominator)); + return getAlgebraAttr(context, + cutegen::stride(std::move(basis))); +} + +MlirAttribute mlirCuteStrideAttrTupleGet(MlirContext context, + intptr_t numElements, + MlirAttribute const *elements) { + return getTupleAttr(context, numElements, elements); +} + +bool mlirAttributeIsACuteStride(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteStrideAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteLayoutAttrGet(MlirContext context, MlirAttribute shape, + MlirAttribute stride) { + auto *ctx = unwrap(context); + cute::ShapeAttr shapeAttr; + cute::StrideAttr strideAttr; + if (!ctx || !unwrapAs(ctx, shape, /*optional=*/false, shapeAttr) || + !unwrapAs(ctx, stride, /*optional=*/false, strideAttr)) + return {nullptr}; + cutegen::layout layout(shapeAttr.getRef(), strideAttr.getRef()); + if (cutegen::has_error(layout)) + return {nullptr}; + return getAlgebraAttr(context, std::move(layout)); +} + +bool mlirAttributeIsACuteLayout(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteLayoutAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteTileAttrLayoutGet(MlirContext context, + MlirAttribute layout) { + auto *ctx = unwrap(context); + cute::LayoutAttr layoutAttr; + if (!ctx || !unwrapAs(ctx, layout, /*optional=*/false, layoutAttr)) + return {nullptr}; + return getAlgebraAttr(context, + cutegen::tile(layoutAttr.getRef())); +} + +MlirAttribute mlirCuteTileAttrUnderscoreGet(MlirContext context) { + return getUnderscoreAttr(context); +} + +MlirAttribute mlirCuteTileAttrTupleGet(MlirContext context, + intptr_t numElements, + MlirAttribute const *elements) { + return getTupleAttr(context, numElements, elements); +} + +bool mlirAttributeIsACuteTile(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteTileAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteComposedLayoutAttrGet(MlirContext context, + MlirAttribute inner, + MlirAttribute offset, + MlirAttribute outer) { + auto *ctx = unwrap(context); + cute::IntTupleAttr offsetAttr; + cute::LayoutAttr outerAttr; + if (!ctx || !unwrapAs(ctx, offset, /*optional=*/false, offsetAttr) || + !unwrapAs(ctx, outer, /*optional=*/false, outerAttr)) + return {nullptr}; + cute::SwizzleAttr swizzleAttr; + cute::LayoutAttr layoutAttr; + std::optional composed; + if (unwrapAs(ctx, inner, /*optional=*/false, swizzleAttr)) + composed.emplace(swizzleAttr.getRef(), offsetAttr.getRef(), + outerAttr.getRef()); + else if (unwrapAs(ctx, inner, /*optional=*/false, layoutAttr)) + composed.emplace(layoutAttr.getRef(), offsetAttr.getRef(), + outerAttr.getRef()); + if (!composed || cutegen::has_error(*composed)) + return {nullptr}; + return getAlgebraAttr(context, + std::move(*composed)); +} + +bool mlirAttributeIsACuteComposedLayout(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteComposedLayoutAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteSwizzleAttrGet(MlirContext context, uint32_t bits, + uint32_t base, int32_t shift) { + auto swizzle = cutegen::swizzle::get(bits, base, shift); + if (!swizzle) + return {nullptr}; + return getAlgebraAttr(context, *swizzle); +} + +bool mlirAttributeIsACuteSwizzle(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteSwizzleAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +// Compiler attributes + +bool mlirAttributeIsACuteBitLayout(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteBitLayoutAttrGet(MlirContext context, + MlirAttribute layout) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::LayoutAttr layoutValue; + if (!unwrapAs(ctx, layout, false, layoutValue)) + return {nullptr}; + return wrap(cute::BitLayoutAttr::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + layoutValue)); +} +MlirAttribute mlirCuteBitLayoutAttrGetLayout(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayout())); +} + +bool mlirAttributeIsACuteCopyAtom(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteCopyAtomAttrGet(MlirContext context, MlirAttribute thrId, + MlirAttribute layoutSrc, + MlirAttribute layoutDst, + MlirAttribute layoutRef) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; + if (!unwrapAs(ctx, thrId, false, thrIdValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutSrcValue; + if (!unwrapAs(ctx, layoutSrc, false, layoutSrcValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutDstValue; + if (!unwrapAs(ctx, layoutDst, false, layoutDstValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutRefValue; + if (!unwrapAs(ctx, layoutRef, false, layoutRefValue)) + return {nullptr}; + return wrap(cute::CopyAtomAttr::get(ctx, thrIdValue, layoutSrcValue, + layoutDstValue, layoutRefValue)); +} +MlirAttribute mlirCuteCopyAtomAttrGetThrId(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getThrId())); +} +MlirAttribute mlirCuteCopyAtomAttrGetLayoutSrc(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutSrc())); +} +MlirAttribute mlirCuteCopyAtomAttrGetLayoutDst(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutDst())); +} +MlirAttribute mlirCuteCopyAtomAttrGetLayoutRef(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutRef())); +} + +bool mlirAttributeIsACuteCopyAtomV2(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute +mlirCuteCopyAtomV2AttrGet(MlirContext context, MlirAttribute thrId, + MlirAttribute layoutSrcTV, MlirAttribute layoutDstTV, + MlirAttribute frgSrc, MlirAttribute frgDst) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; + if (!unwrapAs(ctx, thrId, false, thrIdValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutSrcTVValue; + if (!unwrapAs(ctx, layoutSrcTV, false, layoutSrcTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutDstTVValue; + if (!unwrapAs(ctx, layoutDstTV, false, layoutDstTVValue)) + return {nullptr}; + mlir::Attribute frgSrcValue; + if (!unwrapAs(ctx, frgSrc, false, frgSrcValue)) + return {nullptr}; + mlir::Attribute frgDstValue; + if (!unwrapAs(ctx, frgDst, false, frgDstValue)) + return {nullptr}; + return wrap(cute::CopyAtomV2Attr::get(ctx, thrIdValue, layoutSrcTVValue, + layoutDstTVValue, frgSrcValue, + frgDstValue)); +} +MlirAttribute mlirCuteCopyAtomV2AttrGetThrId(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getThrId())); +} +MlirAttribute mlirCuteCopyAtomV2AttrGetLayoutSrcTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutSrc_TV())); +} +MlirAttribute mlirCuteCopyAtomV2AttrGetLayoutDstTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutDst_TV())); +} +MlirAttribute mlirCuteCopyAtomV2AttrGetFrgSrc(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgSrc())); +} +MlirAttribute mlirCuteCopyAtomV2AttrGetFrgDst(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgDst())); +} + +bool mlirAttributeIsACuteMmaAtom(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutBTV, MlirAttribute layoutCTV, + MlirAttribute shapeAMK, MlirAttribute shapeBNK, MlirAttribute shapeCMN, + MlirAttribute frgA, MlirAttribute frgB, MlirAttribute frgC) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMNKValue; + if (!unwrapAs(ctx, shapeMNK, false, shapeMNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; + if (!unwrapAs(ctx, thrId, false, thrIdValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutATVValue; + if (!unwrapAs(ctx, layoutATV, false, layoutATVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutBTVValue; + if (!unwrapAs(ctx, layoutBTV, false, layoutBTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutCTVValue; + if (!unwrapAs(ctx, layoutCTV, false, layoutCTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeAMKValue; + if (!unwrapAs(ctx, shapeAMK, false, shapeAMKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeBNKValue; + if (!unwrapAs(ctx, shapeBNK, false, shapeBNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeCMNValue; + if (!unwrapAs(ctx, shapeCMN, false, shapeCMNValue)) + return {nullptr}; + mlir::Attribute frgAValue; + if (!unwrapAs(ctx, frgA, false, frgAValue)) + return {nullptr}; + mlir::Attribute frgBValue; + if (!unwrapAs(ctx, frgB, false, frgBValue)) + return {nullptr}; + mlir::Attribute frgCValue; + if (!unwrapAs(ctx, frgC, false, frgCValue)) + return {nullptr}; + return wrap(cute::MmaAtomAttr::get( + ctx, shapeMNKValue, thrIdValue, layoutATVValue, layoutBTVValue, + layoutCTVValue, shapeAMKValue, shapeBNKValue, shapeCMNValue, frgAValue, + frgBValue, frgCValue)); +} +MlirAttribute mlirCuteMmaAtomAttrGetShapeMNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShape_MNK())); +} +MlirAttribute mlirCuteMmaAtomAttrGetThrId(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getThrId())); +} +MlirAttribute mlirCuteMmaAtomAttrGetLayoutATV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutA_TV())); +} +MlirAttribute mlirCuteMmaAtomAttrGetLayoutBTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutB_TV())); +} +MlirAttribute mlirCuteMmaAtomAttrGetLayoutCTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutC_TV())); +} +MlirAttribute mlirCuteMmaAtomAttrGetShapeAMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeA_MK())); +} +MlirAttribute mlirCuteMmaAtomAttrGetShapeBNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeB_NK())); +} +MlirAttribute mlirCuteMmaAtomAttrGetShapeCMN(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeC_MN())); +} +MlirAttribute mlirCuteMmaAtomAttrGetFrgA(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgA())); +} +MlirAttribute mlirCuteMmaAtomAttrGetFrgB(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgB())); +} +MlirAttribute mlirCuteMmaAtomAttrGetFrgC(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgC())); +} + +bool mlirAttributeIsACuteMxMmaAtom(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteMxMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutSFATV, MlirAttribute layoutBTV, + MlirAttribute layoutSFBTV, MlirAttribute layoutCTV, MlirAttribute shapeAMK, + MlirAttribute shapeSFAMK, MlirAttribute shapeBNK, MlirAttribute shapeSFBNK, + MlirAttribute shapeCMN, MlirAttribute frgA, MlirAttribute frgSFA, + MlirAttribute frgB, MlirAttribute frgSFB, MlirAttribute frgC) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMNKValue; + if (!unwrapAs(ctx, shapeMNK, false, shapeMNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; + if (!unwrapAs(ctx, thrId, false, thrIdValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutATVValue; + if (!unwrapAs(ctx, layoutATV, false, layoutATVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutSFATVValue; + if (!unwrapAs(ctx, layoutSFATV, false, layoutSFATVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutBTVValue; + if (!unwrapAs(ctx, layoutBTV, false, layoutBTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutSFBTVValue; + if (!unwrapAs(ctx, layoutSFBTV, false, layoutSFBTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutCTVValue; + if (!unwrapAs(ctx, layoutCTV, false, layoutCTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeAMKValue; + if (!unwrapAs(ctx, shapeAMK, false, shapeAMKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeSFAMKValue; + if (!unwrapAs(ctx, shapeSFAMK, false, shapeSFAMKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeBNKValue; + if (!unwrapAs(ctx, shapeBNK, false, shapeBNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeSFBNKValue; + if (!unwrapAs(ctx, shapeSFBNK, false, shapeSFBNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeCMNValue; + if (!unwrapAs(ctx, shapeCMN, false, shapeCMNValue)) + return {nullptr}; + mlir::Attribute frgAValue; + if (!unwrapAs(ctx, frgA, false, frgAValue)) + return {nullptr}; + mlir::Attribute frgSFAValue; + if (!unwrapAs(ctx, frgSFA, false, frgSFAValue)) + return {nullptr}; + mlir::Attribute frgBValue; + if (!unwrapAs(ctx, frgB, false, frgBValue)) + return {nullptr}; + mlir::Attribute frgSFBValue; + if (!unwrapAs(ctx, frgSFB, false, frgSFBValue)) + return {nullptr}; + mlir::Attribute frgCValue; + if (!unwrapAs(ctx, frgC, false, frgCValue)) + return {nullptr}; + return wrap(cute::MxMmaAtomAttr::get( + ctx, shapeMNKValue, thrIdValue, layoutATVValue, layoutSFATVValue, + layoutBTVValue, layoutSFBTVValue, layoutCTVValue, shapeAMKValue, + shapeSFAMKValue, shapeBNKValue, shapeSFBNKValue, shapeCMNValue, frgAValue, + frgSFAValue, frgBValue, frgSFBValue, frgCValue)); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetShapeMNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShape_MNK())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetThrId(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getThrId())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutATV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutA_TV())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutSFATV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutSFA_TV())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutBTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutB_TV())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutSFBTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutSFB_TV())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutCTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutC_TV())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetShapeAMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeA_MK())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetShapeSFAMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeSFA_MK())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetShapeBNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeB_NK())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetShapeSFBNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeSFB_NK())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetShapeCMN(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeC_MN())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetFrgA(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgA())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetFrgSFA(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgSFA())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetFrgB(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgB())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetFrgSFB(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgSFB())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetFrgC(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgC())); +} + +bool mlirAttributeIsACuteReductionOp(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteReductionOpAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute::symbolizeReductionOp(value); + if (!valueValue) + return {nullptr}; + return wrap(cute::ReductionOpAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteReductionOpAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteSparseMmaAtom(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutBTV, MlirAttribute layoutCTV, + MlirAttribute layoutETV, MlirAttribute shapeAMK, MlirAttribute shapeBNK, + MlirAttribute shapeCMN, MlirAttribute shapeEMK, MlirAttribute frgA, + MlirAttribute frgB, MlirAttribute frgC, MlirAttribute frgE) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMNKValue; + if (!unwrapAs(ctx, shapeMNK, false, shapeMNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; + if (!unwrapAs(ctx, thrId, false, thrIdValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutATVValue; + if (!unwrapAs(ctx, layoutATV, false, layoutATVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutBTVValue; + if (!unwrapAs(ctx, layoutBTV, false, layoutBTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutCTVValue; + if (!unwrapAs(ctx, layoutCTV, false, layoutCTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutETVValue; + if (!unwrapAs(ctx, layoutETV, false, layoutETVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeAMKValue; + if (!unwrapAs(ctx, shapeAMK, false, shapeAMKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeBNKValue; + if (!unwrapAs(ctx, shapeBNK, false, shapeBNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeCMNValue; + if (!unwrapAs(ctx, shapeCMN, false, shapeCMNValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeEMKValue; + if (!unwrapAs(ctx, shapeEMK, false, shapeEMKValue)) + return {nullptr}; + mlir::Attribute frgAValue; + if (!unwrapAs(ctx, frgA, false, frgAValue)) + return {nullptr}; + mlir::Attribute frgBValue; + if (!unwrapAs(ctx, frgB, false, frgBValue)) + return {nullptr}; + mlir::Attribute frgCValue; + if (!unwrapAs(ctx, frgC, false, frgCValue)) + return {nullptr}; + mlir::Attribute frgEValue; + if (!unwrapAs(ctx, frgE, false, frgEValue)) + return {nullptr}; + return wrap(cute::SparseMmaAtomAttr::get( + ctx, shapeMNKValue, thrIdValue, layoutATVValue, layoutBTVValue, + layoutCTVValue, layoutETVValue, shapeAMKValue, shapeBNKValue, + shapeCMNValue, shapeEMKValue, frgAValue, frgBValue, frgCValue, + frgEValue)); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeMNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShape_MNK())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetThrId(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getThrId())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetLayoutATV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutA_TV())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetLayoutBTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutB_TV())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetLayoutCTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutC_TV())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetLayoutETV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutE_TV())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeAMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeA_MK())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeBNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeB_NK())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeCMN(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeC_MN())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeEMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeE_MK())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetFrgA(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgA())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetFrgB(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgB())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetFrgC(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgC())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetFrgE(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgE())); +} + +bool mlirAttributeIsACuteSparseMxMmaAtom(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutSFATV, MlirAttribute layoutBTV, + MlirAttribute layoutSFBTV, MlirAttribute layoutCTV, MlirAttribute layoutETV, + MlirAttribute shapeAMK, MlirAttribute shapeSFAMK, MlirAttribute shapeBNK, + MlirAttribute shapeSFBNK, MlirAttribute shapeCMN, MlirAttribute shapeEMK, + MlirAttribute frgA, MlirAttribute frgSFA, MlirAttribute frgB, + MlirAttribute frgSFB, MlirAttribute frgC, MlirAttribute frgE) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMNKValue; + if (!unwrapAs(ctx, shapeMNK, false, shapeMNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; + if (!unwrapAs(ctx, thrId, false, thrIdValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutATVValue; + if (!unwrapAs(ctx, layoutATV, false, layoutATVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutSFATVValue; + if (!unwrapAs(ctx, layoutSFATV, false, layoutSFATVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutBTVValue; + if (!unwrapAs(ctx, layoutBTV, false, layoutBTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutSFBTVValue; + if (!unwrapAs(ctx, layoutSFBTV, false, layoutSFBTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutCTVValue; + if (!unwrapAs(ctx, layoutCTV, false, layoutCTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutETVValue; + if (!unwrapAs(ctx, layoutETV, false, layoutETVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeAMKValue; + if (!unwrapAs(ctx, shapeAMK, false, shapeAMKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeSFAMKValue; + if (!unwrapAs(ctx, shapeSFAMK, false, shapeSFAMKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeBNKValue; + if (!unwrapAs(ctx, shapeBNK, false, shapeBNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeSFBNKValue; + if (!unwrapAs(ctx, shapeSFBNK, false, shapeSFBNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeCMNValue; + if (!unwrapAs(ctx, shapeCMN, false, shapeCMNValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeEMKValue; + if (!unwrapAs(ctx, shapeEMK, false, shapeEMKValue)) + return {nullptr}; + mlir::Attribute frgAValue; + if (!unwrapAs(ctx, frgA, false, frgAValue)) + return {nullptr}; + mlir::Attribute frgSFAValue; + if (!unwrapAs(ctx, frgSFA, false, frgSFAValue)) + return {nullptr}; + mlir::Attribute frgBValue; + if (!unwrapAs(ctx, frgB, false, frgBValue)) + return {nullptr}; + mlir::Attribute frgSFBValue; + if (!unwrapAs(ctx, frgSFB, false, frgSFBValue)) + return {nullptr}; + mlir::Attribute frgCValue; + if (!unwrapAs(ctx, frgC, false, frgCValue)) + return {nullptr}; + mlir::Attribute frgEValue; + if (!unwrapAs(ctx, frgE, false, frgEValue)) + return {nullptr}; + return wrap(cute::SparseMxMmaAtomAttr::get( + ctx, shapeMNKValue, thrIdValue, layoutATVValue, layoutSFATVValue, + layoutBTVValue, layoutSFBTVValue, layoutCTVValue, layoutETVValue, + shapeAMKValue, shapeSFAMKValue, shapeBNKValue, shapeSFBNKValue, + shapeCMNValue, shapeEMKValue, frgAValue, frgSFAValue, frgBValue, + frgSFBValue, frgCValue, frgEValue)); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeMNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShape_MNK())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetThrId(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getThrId())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutATV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutA_TV())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutSFATV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutSFA_TV())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutBTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutB_TV())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutSFBTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutSFB_TV())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutCTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutC_TV())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutETV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutE_TV())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeAMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeA_MK())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeSFAMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeSFA_MK())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeBNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeB_NK())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeSFBNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeSFB_NK())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeCMN(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeC_MN())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeEMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeE_MK())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgA(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgA())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgSFA(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgSFA())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgB(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgB())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgSFB(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgSFB())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgC(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgC())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgE(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgE())); +} + +// Types + +bool mlirTypeIsACuteType(MlirType type) { + mlir::Type cppType = unwrap(type); + return cppType && + cppType.getDialect().getNamespace() == + mlir::cutlass_compiler::cute::CuteDialect::getDialectNamespace(); +} + +bool mlirTypeIsACuteArithTupleIterator(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteArithTupleIteratorTypeGet(MlirContext context, + MlirType arithTuple) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::IntTupleType arithTupleValue; + if (!unwrapAs(ctx, arithTuple, false, arithTupleValue)) + return {nullptr}; + return wrap(cute::ArithTupleIteratorType::get(ctx, arithTupleValue)); +} +MlirType mlirCuteArithTupleIteratorTypeGetArithTuple(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getArithTuple())); +} + +bool mlirTypeIsACuteComposedLayout(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteComposedLayoutTypeGet(MlirContext context, + MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ComposedLayoutAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) + return {nullptr}; + return wrap(cute::ComposedLayoutType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + attrValue)); +} +MlirAttribute mlirCuteComposedLayoutTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteConstrainedInt(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteConstrainedIntTypeGet(MlirContext context, + int64_t divisibility, uint32_t width, + bool isPow2) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute::ConstrainedIntType::get(ctx, divisibility, width, isPow2)); +} +int64_t mlirCuteConstrainedIntTypeGetDivisibility(MlirType type) { + return llvm::cast(unwrap(type)).getDivisibility(); +} +uint32_t mlirCuteConstrainedIntTypeGetWidth(MlirType type) { + return llvm::cast(unwrap(type)).getWidth(); +} +bool mlirCuteConstrainedIntTypeGetIsPow2(MlirType type) { + return llvm::cast(unwrap(type)).getIsPow2(); +} + +bool mlirTypeIsACuteCoordTensor(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteCoordTensorTypeGet(MlirContext context, MlirType arithTuple, + MlirType layout) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::IntTupleType arithTupleValue; + if (!unwrapAs(ctx, arithTuple, false, arithTupleValue)) + return {nullptr}; + mlir::Type layoutValue; + if (!unwrapAs(ctx, layout, false, layoutValue)) + return {nullptr}; + return wrap(cute::CoordTensorType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + arithTupleValue, layoutValue)); +} +MlirType mlirCuteCoordTensorTypeGetArithTuple(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getArithTuple())); +} +MlirType mlirCuteCoordTensorTypeGetLayout(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getLayout())); +} + +bool mlirTypeIsACuteCoord(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteCoordTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::CoordAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) + return {nullptr}; + return wrap(cute::CoordType::get(ctx, attrValue)); +} +MlirAttribute mlirCuteCoordTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteFastDivmodDivisor(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteFastDivmodDivisorTypeGet(MlirContext context, uint32_t width, + bool isPow2) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute::FastDivmodDivisorType::get(ctx, width, isPow2)); +} +uint32_t mlirCuteFastDivmodDivisorTypeGetWidth(MlirType type) { + return llvm::cast(unwrap(type)).getWidth(); +} +bool mlirCuteFastDivmodDivisorTypeGetIsPow2(MlirType type) { + return llvm::cast(unwrap(type)).getIsPow_2(); +} + +bool mlirTypeIsACuteIntTuple(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteIntTupleTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::IntTupleAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) + return {nullptr}; + return wrap(cute::IntTupleType::get(ctx, attrValue)); +} +MlirAttribute mlirCuteIntTupleTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteLayout(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteLayoutTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::LayoutAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) + return {nullptr}; + return wrap(cute::LayoutType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + attrValue)); +} +MlirAttribute mlirCuteLayoutTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteMemRef(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteMemRefTypeGet(MlirContext context, MlirType ptr, + MlirType layout) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::PtrType ptrValue; + if (!unwrapAs(ctx, ptr, false, ptrValue)) + return {nullptr}; + mlir::Type layoutValue; + if (!unwrapAs(ctx, layout, false, layoutValue)) + return {nullptr}; + return wrap(cute::MemRefType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + ptrValue, layoutValue)); +} +MlirType mlirCuteMemRefTypeGetPtr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPtr())); +} +MlirType mlirCuteMemRefTypeGetLayout(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getLayout())); +} + +bool mlirTypeIsACutePtr(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCutePtrTypeGet(MlirContext context, MlirType valueType, + uint32_t addressSpace, uint64_t alignment, + MlirAttribute swizzle, MlirAttribute bitlayout) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valueTypeValue; + if (!unwrapAs(ctx, valueType, true, valueTypeValue)) + return {nullptr}; + auto addressSpaceValue = + ::mlir::cutlass_compiler::cute::symbolizeAddressSpace(addressSpace); + if (!addressSpaceValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute::SwizzleAttr swizzleValue; + if (!unwrapAs(ctx, swizzle, true, swizzleValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::BitLayoutAttr bitlayoutValue; + if (!unwrapAs(ctx, bitlayout, true, bitlayoutValue)) + return {nullptr}; + return wrap(cute::PtrType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valueTypeValue, *addressSpaceValue, alignment, swizzleValue, + bitlayoutValue)); +} +MlirType mlirCutePtrTypeGetValueType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValueType())); +} +uint32_t mlirCutePtrTypeGetAddressSpace(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getAddressSpace()); +} +uint64_t mlirCutePtrTypeGetAlignment(MlirType type) { + return llvm::cast(unwrap(type)).getAlignment(); +} +MlirAttribute mlirCutePtrTypeGetSwizzle(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getSwizzle())); +} +MlirAttribute mlirCutePtrTypeGetBitlayout(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBitlayout())); +} + +bool mlirTypeIsACuteShape(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteShapeTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) + return {nullptr}; + return wrap(cute::ShapeType::get(ctx, attrValue)); +} +MlirAttribute mlirCuteShapeTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteSparseElem(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteSparseElemTypeGet(MlirContext context, int numLogical, + MlirType physicalType) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type physicalTypeValue; + if (!unwrapAs(ctx, physicalType, false, physicalTypeValue)) + return {nullptr}; + return wrap(cute::SparseElemType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + numLogical, physicalTypeValue)); +} +int mlirCuteSparseElemTypeGetNumLogical(MlirType type) { + return llvm::cast(unwrap(type)).getNumLogical(); +} +MlirType mlirCuteSparseElemTypeGetPhysicalType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPhysicalType())); +} + +bool mlirTypeIsACuteStride(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteStrideTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::StrideAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) + return {nullptr}; + return wrap(cute::StrideType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + attrValue)); +} +MlirAttribute mlirCuteStrideTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteSwizzle(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteSwizzleTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::SwizzleAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) + return {nullptr}; + return wrap(cute::SwizzleType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + attrValue)); +} +MlirAttribute mlirCuteSwizzleTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteTile(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteTileTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::TileAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) + return {nullptr}; + return wrap(cute::TileType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + attrValue)); +} +MlirAttribute mlirCuteTileTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteTiledCopy(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteTiledCopyTypeGet(MlirContext context, MlirType copyAtom, + MlirAttribute layoutCopyTv, + MlirAttribute tilerMn) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type copyAtomValue; + if (!unwrapAs(ctx, copyAtom, false, copyAtomValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutCopyTvValue; + if (!unwrapAs(ctx, layoutCopyTv, false, layoutCopyTvValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::TileAttr tilerMnValue; + if (!unwrapAs(ctx, tilerMn, false, tilerMnValue)) + return {nullptr}; + return wrap(cute::TiledCopyType::get(ctx, copyAtomValue, layoutCopyTvValue, + tilerMnValue)); +} +MlirType mlirCuteTiledCopyTypeGetCopyAtom(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCopyAtom())); +} +MlirType mlirCuteTiledCopyTypeGetAtom(MlirType type) { + return mlirCuteTiledCopyTypeGetCopyAtom(type); +} +MlirAttribute mlirCuteTiledCopyTypeGetLayoutCopyTv(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getLayoutCopyTv())); +} +MlirAttribute mlirCuteTiledCopyTypeGetTilerMn(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getTilerMn())); +} + +bool mlirTypeIsACuteTiledCopyV2(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteTiledCopyV2TypeGet(MlirContext context, MlirType copyAtom, + MlirAttribute layoutCopyTv, + MlirAttribute tilerMn) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type copyAtomValue; + if (!unwrapAs(ctx, copyAtom, false, copyAtomValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutCopyTvValue; + if (!unwrapAs(ctx, layoutCopyTv, false, layoutCopyTvValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::TileAttr tilerMnValue; + if (!unwrapAs(ctx, tilerMn, false, tilerMnValue)) + return {nullptr}; + return wrap(cute::TiledCopyV2Type::get(ctx, copyAtomValue, layoutCopyTvValue, + tilerMnValue)); +} +MlirType mlirCuteTiledCopyV2TypeGetCopyAtom(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCopyAtom())); +} +MlirAttribute mlirCuteTiledCopyV2TypeGetLayoutCopyTv(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getLayoutCopyTv())); +} +MlirAttribute mlirCuteTiledCopyV2TypeGetTilerMn(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getTilerMn())); +} + +bool mlirTypeIsACuteTiledMma(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteTiledMmaTypeGet(MlirContext context, MlirType mmaAtom, + MlirAttribute atomLayoutMNK, + MlirAttribute permutationMNK) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type mmaAtomValue; + if (!unwrapAs(ctx, mmaAtom, false, mmaAtomValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr atomLayoutMNKValue; + if (!unwrapAs(ctx, atomLayoutMNK, false, atomLayoutMNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::TileAttr permutationMNKValue; + if (!unwrapAs(ctx, permutationMNK, true, permutationMNKValue)) + return {nullptr}; + return wrap(cute::TiledMmaType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + mmaAtomValue, atomLayoutMNKValue, permutationMNKValue)); +} +MlirType mlirCuteTiledMmaTypeGetMmaAtom(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getMmaAtom())); +} +MlirType mlirCuteTiledMmaTypeGetAtom(MlirType type) { + return mlirCuteTiledMmaTypeGetMmaAtom(type); +} +MlirAttribute mlirCuteTiledMmaTypeGetAtomLayoutMNK(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAtomLayout_MNK())); +} +MlirAttribute mlirCuteTiledMmaTypeGetAtomLayoutMnk(MlirType type) { + return mlirCuteTiledMmaTypeGetAtomLayoutMNK(type); +} +MlirAttribute mlirCuteTiledMmaTypeGetPermutationMNK(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPermutation_MNK())); +} +MlirAttribute mlirCuteTiledMmaTypeGetPermutationMnk(MlirType type) { + return mlirCuteTiledMmaTypeGetPermutationMNK(type); +} + +bool mlirTypeIsACuteTuple(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteTupleTypeGet(MlirContext context, intptr_t numTypes, + MlirType const *types) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + llvm::SmallVector typesValue; + for (intptr_t i = 0; i < numTypes; ++i) { + mlir::Type element; + if (!unwrapAs(ctx, types[i], false, element)) + return {nullptr}; + typesValue.push_back(element); + } + return wrap(cute::TupleType::get(ctx, typesValue)); +} +intptr_t mlirCuteTupleTypeGetNumTypes(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getTypes().size()); +} +MlirType mlirCuteTupleTypeGetType(MlirType type, intptr_t pos) { + return wrap(llvm::cast(unwrap(type)).getTypes()[pos]); +} diff --git a/mlir/dialects/cute_ir/capi.h b/mlir/dialects/cute_ir/capi.h new file mode 100644 index 00000000..e9e285d2 --- /dev/null +++ b/mlir/dialects/cute_ir/capi.h @@ -0,0 +1,491 @@ +//===- capi.h - C API for the cute dialect ------------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_CUTE_IR_CAPI_H +#define MLIR_DIALECTS_CUTE_IR_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Registration + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Cute, cute); + +// Attributes + +// Algebra attributes are built bottom up. A leaf is a static integer, a +// dynamic integer of `width` bits (1, 8, 16, 32 or 64) known to be a multiple +// of `divisibility`, a scaled basis (strides) or an underscore (coordinates +// and tiles); a tuple groups attributes of the same kind. A basis `modes` is +// its mode path, outermost first, which CuTe prints innermost first: +// `1@1@0` is modes {0, 1}. Constructors return null for invalid parameters or +// handles of another context. GetValue returns canonical algebra text owned +// by the attribute's context; callers must not free it. Getters require an +// attribute accepted by the corresponding IsA predicate. + +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteIntTupleAttrStaticGet(MlirContext context, int64_t value); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteIntTupleAttrDynamicGet( + MlirContext context, int32_t width, int64_t divisibility); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteIntTupleAttrTupleGet( + MlirContext context, intptr_t numElements, MlirAttribute const *elements); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteIntTuple(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirCuteIntTupleAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCoordAttrStaticGet(MlirContext context, + int64_t value); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCoordAttrDynamicGet( + MlirContext context, int32_t width, int64_t divisibility); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCoordAttrUnderscoreGet(MlirContext context); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCoordAttrTupleGet( + MlirContext context, intptr_t numElements, MlirAttribute const *elements); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteCoord(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirCuteCoordAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteShapeAttrStaticGet(MlirContext context, + int64_t value); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteShapeAttrDynamicGet( + MlirContext context, int32_t width, int64_t divisibility); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteShapeAttrTupleGet( + MlirContext context, intptr_t numElements, MlirAttribute const *elements); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirCuteShapeAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteStrideAttrStaticGet(MlirContext context, int64_t value); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteStrideAttrDynamicGet( + MlirContext context, int32_t width, int64_t divisibility); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteStrideAttrBasisGet( + MlirContext context, int64_t numerator, int64_t denominator, + intptr_t numModes, int32_t const *modes); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteStrideAttrTupleGet( + MlirContext context, intptr_t numElements, MlirAttribute const *elements); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteStride(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirCuteStrideAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteLayoutAttrGet(MlirContext context, + MlirAttribute shape, + MlirAttribute stride); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirCuteLayoutAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTileAttrLayoutGet(MlirContext context, MlirAttribute layout); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTileAttrUnderscoreGet(MlirContext context); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteTileAttrTupleGet( + MlirContext context, intptr_t numElements, MlirAttribute const *elements); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteTile(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirCuteTileAttrGetValue(MlirAttribute attr); + +// `inner` is a swizzle or a layout attribute. +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteComposedLayoutAttrGet(MlirContext context, MlirAttribute inner, + MlirAttribute offset, MlirAttribute outer); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteComposedLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirCuteComposedLayoutAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteSwizzleAttrGet(MlirContext context, + uint32_t bits, + uint32_t base, + int32_t shift); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteSwizzle(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirCuteSwizzleAttrGetValue(MlirAttribute attr); + +// Compiler attributes + +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles, and std::optional enums a +// negative value. Enums are their integer values (the dialect's *Enums.td). +// Getters require a value accepted by the matching IsA. + +// `#cute.bitlayout`: Bit layout of a pointer's elements within a storage chunk. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteBitLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteBitLayoutAttrGet(MlirContext context, + MlirAttribute layout); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteBitLayoutAttrGetLayout(MlirAttribute attr); + +// `#cute.copy_atom`: Thread and value layouts of a copy atom. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteCopyAtom(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCopyAtomAttrGet( + MlirContext context, MlirAttribute thrId, MlirAttribute layoutSrc, + MlirAttribute layoutDst, MlirAttribute layoutRef); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomAttrGetThrId(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomAttrGetLayoutSrc(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomAttrGetLayoutDst(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomAttrGetLayoutRef(MlirAttribute attr); + +// `#cute.copy_atom_v2`: Thread and value layouts and fragments of a V2 copy +// atom. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteCopyAtomV2(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCopyAtomV2AttrGet( + MlirContext context, MlirAttribute thrId, MlirAttribute layoutSrcTV, + MlirAttribute layoutDstTV, MlirAttribute frgSrc, MlirAttribute frgDst); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomV2AttrGetThrId(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomV2AttrGetLayoutSrcTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomV2AttrGetLayoutDstTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomV2AttrGetFrgSrc(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomV2AttrGetFrgDst(MlirAttribute attr); + +// `#cute.mma_atom`: Shapes, thread-value layouts and fragments of an MMA atom. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteMmaAtom(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutBTV, MlirAttribute layoutCTV, + MlirAttribute shapeAMK, MlirAttribute shapeBNK, MlirAttribute shapeCMN, + MlirAttribute frgA, MlirAttribute frgB, MlirAttribute frgC); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetShapeMNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetThrId(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetLayoutATV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetLayoutBTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetLayoutCTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetShapeAMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetShapeBNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetShapeCMN(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMmaAtomAttrGetFrgA(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMmaAtomAttrGetFrgB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMmaAtomAttrGetFrgC(MlirAttribute attr); + +// `#cute.mx_mma_atom`: MMA atom attribute with scale-factor operands. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteMxMmaAtom(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMxMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutSFATV, MlirAttribute layoutBTV, + MlirAttribute layoutSFBTV, MlirAttribute layoutCTV, MlirAttribute shapeAMK, + MlirAttribute shapeSFAMK, MlirAttribute shapeBNK, MlirAttribute shapeSFBNK, + MlirAttribute shapeCMN, MlirAttribute frgA, MlirAttribute frgSFA, + MlirAttribute frgB, MlirAttribute frgSFB, MlirAttribute frgC); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetShapeMNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetThrId(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetLayoutATV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetLayoutSFATV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetLayoutBTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetLayoutSFBTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetLayoutCTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetShapeAMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetShapeSFAMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetShapeBNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetShapeSFBNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetShapeCMN(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetFrgA(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetFrgSFA(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetFrgB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetFrgSFB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetFrgC(MlirAttribute attr); + +// `#cute.reduction_op`: Op for cute reduce operations. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteReductionOp(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteReductionOpAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteReductionOpAttrGetValue(MlirAttribute attr); + +// `#cute.sparse_mma_atom`: MMA atom attribute with a sparsity-metadata operand. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteSparseMmaAtom(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteSparseMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutBTV, MlirAttribute layoutCTV, + MlirAttribute layoutETV, MlirAttribute shapeAMK, MlirAttribute shapeBNK, + MlirAttribute shapeCMN, MlirAttribute shapeEMK, MlirAttribute frgA, + MlirAttribute frgB, MlirAttribute frgC, MlirAttribute frgE); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetShapeMNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetThrId(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetLayoutATV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetLayoutBTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetLayoutCTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetLayoutETV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetShapeAMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetShapeBNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetShapeCMN(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetShapeEMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetFrgA(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetFrgB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetFrgC(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetFrgE(MlirAttribute attr); + +// `#cute.sparse_mx_mma_atom`: MMA atom attribute with scale-factor and metadata +// operands. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteSparseMxMmaAtom(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteSparseMxMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutSFATV, MlirAttribute layoutBTV, + MlirAttribute layoutSFBTV, MlirAttribute layoutCTV, MlirAttribute layoutETV, + MlirAttribute shapeAMK, MlirAttribute shapeSFAMK, MlirAttribute shapeBNK, + MlirAttribute shapeSFBNK, MlirAttribute shapeCMN, MlirAttribute shapeEMK, + MlirAttribute frgA, MlirAttribute frgSFA, MlirAttribute frgB, + MlirAttribute frgSFB, MlirAttribute frgC, MlirAttribute frgE); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeMNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetThrId(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetLayoutATV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetLayoutSFATV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetLayoutBTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetLayoutSFBTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetLayoutCTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetLayoutETV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeAMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeSFAMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeBNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeSFBNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeCMN(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeEMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetFrgA(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetFrgSFA(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetFrgB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetFrgSFB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetFrgC(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetFrgE(MlirAttribute attr); + +// Types + +/// Returns true when `type` belongs to the CuTe dialect. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteType(MlirType type); + +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles, and std::optional enums a +// negative value. Enums are their integer values (the dialect's *Enums.td). +// Getters require a value accepted by the matching IsA. + +// `!cute.arith_tuple_iter`: Iterator over an arithmetic tuple. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteArithTupleIterator(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteArithTupleIteratorTypeGet(MlirContext context, MlirType arithTuple); +MLIR_CAPI_EXPORTED MlirType +mlirCuteArithTupleIteratorTypeGetArithTuple(MlirType type); + +// `!cute.composed_layout`: CuTe composed layout: A ∘ offset ∘ B. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteComposedLayout(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteComposedLayoutTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteComposedLayoutTypeGetAttr(MlirType type); + +// `!cute.i`: Integer of any width with known divisibility and power of two. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteConstrainedInt(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteConstrainedIntTypeGet(MlirContext context, + int64_t divisibility, + uint32_t width, + bool isPow2); +MLIR_CAPI_EXPORTED int64_t +mlirCuteConstrainedIntTypeGetDivisibility(MlirType type); +MLIR_CAPI_EXPORTED uint32_t mlirCuteConstrainedIntTypeGetWidth(MlirType type); +MLIR_CAPI_EXPORTED bool mlirCuteConstrainedIntTypeGetIsPow2(MlirType type); + +// `!cute.coord_tensor`: Coordinate tensor: an arithmetic tuple with a layout. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteCoordTensor(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTensorTypeGet(MlirContext context, + MlirType arithTuple, + MlirType layout); +MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTensorTypeGetArithTuple(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTensorTypeGetLayout(MlirType type); + +// `!cute.coord`: Scalar integer, underscore wildcard, or recursive tuple of +// coordinates. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteCoord(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCoordTypeGetAttr(MlirType type); + +// `!cute.fast_divmod_divisor`: Precomputed divisor of the fast divmod +// operations. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteFastDivmodDivisor(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteFastDivmodDivisorTypeGet( + MlirContext context, uint32_t width, bool isPow2); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteFastDivmodDivisorTypeGetWidth(MlirType type); +MLIR_CAPI_EXPORTED bool mlirCuteFastDivmodDivisorTypeGetIsPow2(MlirType type); + +// `!cute.int_tuple`: Scalar integer or recursive tuple of integers. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteIntTuple(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteIntTupleTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteIntTupleTypeGetAttr(MlirType type); + +// `!cute.layout`: CuTe layout: a shape/stride pair. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteLayout(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteLayoutTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteLayoutTypeGetAttr(MlirType type); + +// `!cute.memref`: Tensor view: a pointer with a layout. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteMemRef(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteMemRefTypeGet(MlirContext context, + MlirType ptr, + MlirType layout); +MLIR_CAPI_EXPORTED MlirType mlirCuteMemRefTypeGetPtr(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteMemRefTypeGetLayout(MlirType type); + +// `!cute.ptr`: Pointer with address space, alignment, swizzle and bit layout. +MLIR_CAPI_EXPORTED bool mlirTypeIsACutePtr(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCutePtrTypeGet( + MlirContext context, MlirType valueType, uint32_t addressSpace, + uint64_t alignment, MlirAttribute swizzle, MlirAttribute bitlayout); +MLIR_CAPI_EXPORTED MlirType mlirCutePtrTypeGetValueType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t mlirCutePtrTypeGetAddressSpace(MlirType type); +MLIR_CAPI_EXPORTED uint64_t mlirCutePtrTypeGetAlignment(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirCutePtrTypeGetSwizzle(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirCutePtrTypeGetBitlayout(MlirType type); + +// `!cute.shape`: Scalar integer or recursive tuple of shape extents. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteShape(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteShapeTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteShapeTypeGetAttr(MlirType type); + +// `!cute.sparse_elem`: Logical sparse element stored in a physical type. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteSparseElem(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteSparseElemTypeGet(MlirContext context, + int numLogical, + MlirType physicalType); +MLIR_CAPI_EXPORTED int mlirCuteSparseElemTypeGetNumLogical(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteSparseElemTypeGetPhysicalType(MlirType type); + +// `!cute.stride`: Scalar or recursive tuple of stride elements. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteStride(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteStrideTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteStrideTypeGetAttr(MlirType type); + +// `!cute.swizzle`: CuTe swizzle: S. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteSwizzle(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteSwizzleTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteSwizzleTypeGetAttr(MlirType type); + +// `!cute.tile`: CuTe tile: a recursive sequence of layouts and underscores. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTile(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteTileTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteTileTypeGetAttr(MlirType type); + +// `!cute.tiled_copy`: Copy atom tiled over threads and values. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTiledCopy(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledCopyTypeGet(MlirContext context, + MlirType copyAtom, + MlirAttribute layoutCopyTv, + MlirAttribute tilerMn); +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledCopyTypeGetCopyAtom(MlirType type); +// Former name of mlirCuteTiledCopyTypeGetCopyAtom. +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledCopyTypeGetAtom(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledCopyTypeGetLayoutCopyTv(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteTiledCopyTypeGetTilerMn(MlirType type); + +// `!cute.tiled_copy_v2`: V2 copy atom tiled over threads and values. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTiledCopyV2(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteTiledCopyV2TypeGet(MlirContext context, MlirType copyAtom, + MlirAttribute layoutCopyTv, MlirAttribute tilerMn); +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledCopyV2TypeGetCopyAtom(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledCopyV2TypeGetLayoutCopyTv(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledCopyV2TypeGetTilerMn(MlirType type); + +// `!cute.tiled_mma`: MMA atom tiled over MNK. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTiledMma(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledMmaTypeGet( + MlirContext context, MlirType mmaAtom, MlirAttribute atomLayoutMNK, + MlirAttribute permutationMNK); +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledMmaTypeGetMmaAtom(MlirType type); +// Former name of mlirCuteTiledMmaTypeGetMmaAtom. +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledMmaTypeGetAtom(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledMmaTypeGetAtomLayoutMNK(MlirType type); +// Former name of mlirCuteTiledMmaTypeGetAtomLayoutMNK. +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledMmaTypeGetAtomLayoutMnk(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledMmaTypeGetPermutationMNK(MlirType type); +// Former name of mlirCuteTiledMmaTypeGetPermutationMNK. +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledMmaTypeGetPermutationMnk(MlirType type); + +// `!cute.tuple`: Non-empty tuple of types. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTuple(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteTupleTypeGet(MlirContext context, + intptr_t numTypes, + MlirType const *types); +MLIR_CAPI_EXPORTED intptr_t mlirCuteTupleTypeGetNumTypes(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteTupleTypeGetType(MlirType type, + intptr_t pos); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_CUTE_IR_CAPI_H diff --git a/mlir/dialects/cute_ir/cute_ir.zig b/mlir/dialects/cute_ir/cute_ir.zig index b36565ac..f22fcf64 100644 --- a/mlir/dialects/cute_ir/cute_ir.zig +++ b/mlir/dialects/cute_ir/cute_ir.zig @@ -3,15 +3,6 @@ const std = @import("std"); const c = @import("c"); const mlir = @import("mlir"); const stdx = @import("stdx"); -const cute = @This(); - -pub const dialect_namespace = "cute"; -pub const nvgpu = @import("cute_nvgpu.zig"); -/// NVIDIA's `nvvm` and `cuda` dialects, as the DSL compiler registers them. -pub const nvvm = @import("nvvm.zig"); -pub const cuda = @import("cuda.zig"); - -pub const AttributeKind = enum { int_tuple, coord, shape, stride, layout, tile, composed_layout, swizzle }; pub const CuteType = opaque { const M = mlir.Methods(CuteType, c.MlirType); @@ -19,71 +10,27 @@ pub const CuteType = opaque { pub const isAFn = c.mlirTypeIsACuteType; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; - - pub fn format(self: *const CuteType, writer: *std.Io.Writer) std.Io.Writer.Error!void { - const t: *const mlir.Type = @ptrCast(self); - return t.format(writer); - } }; -/// Registers `cute`, `cute_nvgpu`, and NVIDIA's `nvvm` and `cuda` in the -/// existing MLIR registry. The `nvvm` here is the compiler's, not upstream's: -/// a registry cannot hold both. -pub fn registerDialects(registry: *mlir.DialectRegistry) void { - mlir.DialectHandle.fromString(dialect_namespace).insertDialect(registry); - mlir.DialectHandle.fromString(nvgpu.dialect_namespace).insertDialect(registry); - mlir.DialectHandle.fromString(nvvm.dialect_handle).insertDialect(registry); - mlir.DialectHandle.fromString(cuda.dialect_handle).insertDialect(registry); -} - -pub const AlgebraKind = AttributeKind; - -/// Constructs the algebra attribute and type through their C APIs. -/// Existing attributes can be passed directly to LayoutType.get, etc. -pub fn algebraType(ctx: *mlir.Context, kind: AlgebraKind, expression: []const u8) !*const mlir.Type { - return switch (kind) { - inline else => |k| blk: { - const T = switch (k) { - .int_tuple => IntTupleType, - .coord => CoordType, - .shape => ShapeType, - .stride => StrideType, - .layout => LayoutType, - .tile => TileType, - .composed_layout => ComposedLayoutType, - .swizzle => SwizzleType, - }; - const attr = try algebraAttribute(ctx, @field(AttributeKind, @tagName(k)), expression); - break :blk (try T.get(ctx, .{ .attr = attr })).type_(); - }, - }; -} - -pub fn algebraAttribute(ctx: *mlir.Context, kind: AttributeKind, expression: []const u8) !*const mlir.Attribute { - return switch (kind) { - inline else => |k| blk: { - const A = switch (k) { - .int_tuple => IntTupleAttr, - .coord => CoordAttr, - .shape => ShapeAttr, - .stride => StrideAttr, - .layout => LayoutAttr, - .tile => TileAttr, - .composed_layout => ComposedLayoutAttr, - .swizzle => SwizzleAttr, - }; - break :blk (try A.get(ctx, expression)).attribute(); - }, - }; +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("cute"); } /// The address space of a pointer or memref. pub const MemorySpace = AddressSpace; -pub fn pointerType(ctx: *mlir.Context, element_type: ?*const mlir.Type, space: MemorySpace, alignment: u64) !*const mlir.Type { +pub fn pointerType( + ctx: *mlir.Context, + space: MemorySpace, + alignment: u64, + opts: struct { + element_type: ?*const mlir.Type = null, + }, +) !*const mlir.Type { return (try PtrType.get(ctx, .{ - .valueType = element_type, + .valueType = opts.element_type, .addressSpace = space, .alignment = alignment, })).type_(); @@ -100,32 +47,38 @@ pub fn constrainedIntType(ctx: *mlir.Context, width: enum { i32, i64 }, divisibl })).type_(); } -/// Constructors take raw CuTe algebra expressions. Getter strings are owned by -/// the MLIR context and remain valid until that context is destroyed. +/// Algebra attributes are built bottom up from leaves and tuples of the same +/// kind. A dynamic leaf has `width` bits (1, 8, 16, 32 or 64) and is known to +/// be a multiple of `divisibility`. Getter strings are owned by the MLIR +/// context and remain valid until that context is destroyed. pub const IntTupleAttr = opaque { const M = mlir.Methods(IntTupleAttr, c.MlirAttribute); pub const isAFn = c.mlirAttributeIsACuteIntTuple; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; - pub fn get(ctx: *mlir.Context, value: []const u8) mlir.Error!*const IntTupleAttr { - const result = c.mlirCuteIntTupleAttrGet(ctx.ptr(), mlir.stringRef(value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); + pub fn getStatic(ctx: *mlir.Context, value_: i64) mlir.Error!*const IntTupleAttr { + return @ptrCast(c.mlirCuteIntTupleAttrStaticGet(ctx.ptr(), value_).ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const IntTupleAttr) []const u8 { + pub fn getDynamic(ctx: *mlir.Context, width: i32, divisibility: i64) mlir.Error!*const IntTupleAttr { + return @ptrCast(c.mlirCuteIntTupleAttrDynamicGet(ctx.ptr(), width, divisibility).ptr orelse return error.InvalidMlir); + } + + pub fn getTuple(ctx: *mlir.Context, elements: []const *const IntTupleAttr) mlir.Error!*const IntTupleAttr { + return @ptrCast(c.mlirCuteIntTupleAttrTupleGet(ctx.ptr(), @intCast(elements.len), @ptrCast(elements)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const IntTupleAttr) []const u8 { return mlir.string(c.mlirCuteIntTupleAttrGetValue(self.ptr())); } pub fn attribute(self: *const IntTupleAttr) *const mlir.Attribute { return @ptrCast(self); } - - pub fn format(self: *const IntTupleAttr, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } }; pub const CoordAttr = opaque { @@ -134,24 +87,32 @@ pub const CoordAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteCoord; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; - pub fn get(ctx: *mlir.Context, value: []const u8) mlir.Error!*const CoordAttr { - const result = c.mlirCuteCoordAttrGet(ctx.ptr(), mlir.stringRef(value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); + pub fn getStatic(ctx: *mlir.Context, value_: i64) mlir.Error!*const CoordAttr { + return @ptrCast(c.mlirCuteCoordAttrStaticGet(ctx.ptr(), value_).ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const CoordAttr) []const u8 { + pub fn getDynamic(ctx: *mlir.Context, width: i32, divisibility: i64) mlir.Error!*const CoordAttr { + return @ptrCast(c.mlirCuteCoordAttrDynamicGet(ctx.ptr(), width, divisibility).ptr orelse return error.InvalidMlir); + } + + pub fn getUnderscore(ctx: *mlir.Context) mlir.Error!*const CoordAttr { + return @ptrCast(c.mlirCuteCoordAttrUnderscoreGet(ctx.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn getTuple(ctx: *mlir.Context, elements: []const *const CoordAttr) mlir.Error!*const CoordAttr { + return @ptrCast(c.mlirCuteCoordAttrTupleGet(ctx.ptr(), @intCast(elements.len), @ptrCast(elements)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const CoordAttr) []const u8 { return mlir.string(c.mlirCuteCoordAttrGetValue(self.ptr())); } pub fn attribute(self: *const CoordAttr) *const mlir.Attribute { return @ptrCast(self); } - - pub fn format(self: *const CoordAttr, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } }; pub const ShapeAttr = opaque { @@ -160,24 +121,28 @@ pub const ShapeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteShape; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; - pub fn get(ctx: *mlir.Context, value: []const u8) mlir.Error!*const ShapeAttr { - const result = c.mlirCuteShapeAttrGet(ctx.ptr(), mlir.stringRef(value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); + pub fn getStatic(ctx: *mlir.Context, value_: i64) mlir.Error!*const ShapeAttr { + return @ptrCast(c.mlirCuteShapeAttrStaticGet(ctx.ptr(), value_).ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const ShapeAttr) []const u8 { + pub fn getDynamic(ctx: *mlir.Context, width: i32, divisibility: i64) mlir.Error!*const ShapeAttr { + return @ptrCast(c.mlirCuteShapeAttrDynamicGet(ctx.ptr(), width, divisibility).ptr orelse return error.InvalidMlir); + } + + pub fn getTuple(ctx: *mlir.Context, elements: []const *const ShapeAttr) mlir.Error!*const ShapeAttr { + return @ptrCast(c.mlirCuteShapeAttrTupleGet(ctx.ptr(), @intCast(elements.len), @ptrCast(elements)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const ShapeAttr) []const u8 { return mlir.string(c.mlirCuteShapeAttrGetValue(self.ptr())); } pub fn attribute(self: *const ShapeAttr) *const mlir.Attribute { return @ptrCast(self); } - - pub fn format(self: *const ShapeAttr, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } }; pub const StrideAttr = opaque { @@ -186,24 +151,34 @@ pub const StrideAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteStride; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; - pub fn get(ctx: *mlir.Context, value: []const u8) mlir.Error!*const StrideAttr { - const result = c.mlirCuteStrideAttrGet(ctx.ptr(), mlir.stringRef(value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); + pub fn getStatic(ctx: *mlir.Context, value_: i64) mlir.Error!*const StrideAttr { + return @ptrCast(c.mlirCuteStrideAttrStaticGet(ctx.ptr(), value_).ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const StrideAttr) []const u8 { + pub fn getDynamic(ctx: *mlir.Context, width: i32, divisibility: i64) mlir.Error!*const StrideAttr { + return @ptrCast(c.mlirCuteStrideAttrDynamicGet(ctx.ptr(), width, divisibility).ptr orelse return error.InvalidMlir); + } + + /// `numerator/denominator` along `modes`, the mode path outermost first; + /// CuTe prints it innermost first, so `1@1@0` is modes `{ 0, 1 }`. + pub fn getBasis(ctx: *mlir.Context, numerator: i64, denominator: i64, modes: []const i32) mlir.Error!*const StrideAttr { + return @ptrCast(c.mlirCuteStrideAttrBasisGet(ctx.ptr(), numerator, denominator, @intCast(modes.len), modes.ptr).ptr orelse return error.InvalidMlir); + } + + pub fn getTuple(ctx: *mlir.Context, elements: []const *const StrideAttr) mlir.Error!*const StrideAttr { + return @ptrCast(c.mlirCuteStrideAttrTupleGet(ctx.ptr(), @intCast(elements.len), @ptrCast(elements)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const StrideAttr) []const u8 { return mlir.string(c.mlirCuteStrideAttrGetValue(self.ptr())); } pub fn attribute(self: *const StrideAttr) *const mlir.Attribute { return @ptrCast(self); } - - pub fn format(self: *const StrideAttr, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } }; pub const LayoutAttr = opaque { @@ -212,24 +187,26 @@ pub const LayoutAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteLayout; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; - pub fn get(ctx: *mlir.Context, value: []const u8) mlir.Error!*const LayoutAttr { - const result = c.mlirCuteLayoutAttrGet(ctx.ptr(), mlir.stringRef(value)); + pub const InitArgs = struct { + shape: *const ShapeAttr, + stride: *const StrideAttr, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const LayoutAttr { + const result = c.mlirCuteLayoutAttrGet(ctx.ptr(), args.shape.ptr(), args.stride.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const LayoutAttr) []const u8 { + pub fn value(self: *const LayoutAttr) []const u8 { return mlir.string(c.mlirCuteLayoutAttrGetValue(self.ptr())); } pub fn attribute(self: *const LayoutAttr) *const mlir.Attribute { return @ptrCast(self); } - - pub fn format(self: *const LayoutAttr, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } }; pub const TileAttr = opaque { @@ -238,24 +215,28 @@ pub const TileAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteTile; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; - pub fn get(ctx: *mlir.Context, value: []const u8) mlir.Error!*const TileAttr { - const result = c.mlirCuteTileAttrGet(ctx.ptr(), mlir.stringRef(value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); + pub fn getLayout(ctx: *mlir.Context, layout: *const LayoutAttr) mlir.Error!*const TileAttr { + return @ptrCast(c.mlirCuteTileAttrLayoutGet(ctx.ptr(), layout.ptr()).ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const TileAttr) []const u8 { + pub fn getUnderscore(ctx: *mlir.Context) mlir.Error!*const TileAttr { + return @ptrCast(c.mlirCuteTileAttrUnderscoreGet(ctx.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn getTuple(ctx: *mlir.Context, elements: []const *const TileAttr) mlir.Error!*const TileAttr { + return @ptrCast(c.mlirCuteTileAttrTupleGet(ctx.ptr(), @intCast(elements.len), @ptrCast(elements)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const TileAttr) []const u8 { return mlir.string(c.mlirCuteTileAttrGetValue(self.ptr())); } pub fn attribute(self: *const TileAttr) *const mlir.Attribute { return @ptrCast(self); } - - pub fn format(self: *const TileAttr, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } }; pub const ComposedLayoutAttr = opaque { @@ -264,24 +245,34 @@ pub const ComposedLayoutAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteComposedLayout; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; - pub fn get(ctx: *mlir.Context, value: []const u8) mlir.Error!*const ComposedLayoutAttr { - const result = c.mlirCuteComposedLayoutAttrGet(ctx.ptr(), mlir.stringRef(value)); + /// `inner ∘ offset ∘ outer`. + pub const InitArgs = struct { + inner: union(enum) { + swizzle: *const SwizzleAttr, + layout: *const LayoutAttr, + }, + offset: *const IntTupleAttr, + outer: *const LayoutAttr, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const ComposedLayoutAttr { + const inner = switch (args.inner) { + inline else => |attr| attr.ptr(), + }; + const result = c.mlirCuteComposedLayoutAttrGet(ctx.ptr(), inner, args.offset.ptr(), args.outer.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const ComposedLayoutAttr) []const u8 { + pub fn value(self: *const ComposedLayoutAttr) []const u8 { return mlir.string(c.mlirCuteComposedLayoutAttrGetValue(self.ptr())); } pub fn attribute(self: *const ComposedLayoutAttr) *const mlir.Attribute { return @ptrCast(self); } - - pub fn format(self: *const ComposedLayoutAttr, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } }; pub const SwizzleAttr = opaque { @@ -290,24 +281,28 @@ pub const SwizzleAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteSwizzle; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; - pub fn get(ctx: *mlir.Context, value: []const u8) mlir.Error!*const SwizzleAttr { - const result = c.mlirCuteSwizzleAttrGet(ctx.ptr(), mlir.stringRef(value)); + /// `S`. + pub const InitArgs = struct { + bits: u32, + base: u32, + shift: i32, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const SwizzleAttr { + const result = c.mlirCuteSwizzleAttrGet(ctx.ptr(), args.bits, args.base, args.shift); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const SwizzleAttr) []const u8 { + pub fn value(self: *const SwizzleAttr) []const u8 { return mlir.string(c.mlirCuteSwizzleAttrGetValue(self.ptr())); } pub fn attribute(self: *const SwizzleAttr) *const mlir.Attribute { return @ptrCast(self); } - - pub fn format(self: *const SwizzleAttr, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } }; // Enums, generated from the dialect's .td files; values are the compiler's. @@ -363,11 +358,9 @@ pub const ArithTupleIteratorType = opaque { pub const isAFn = c.mlirTypeIsACuteArithTupleIterator; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "arith_tuple_iter"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -378,7 +371,7 @@ pub const ArithTupleIteratorType = opaque { const result = c.mlirCuteArithTupleIteratorTypeGet(ctx.ptr(), args.arithTuple.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getArithTuple(self: *const Self) *const mlir.Type { + pub fn arithTuple(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteArithTupleIteratorTypeGetArithTuple(self.ptr()).ptr.?); } }; @@ -390,11 +383,9 @@ pub const ComposedLayoutType = opaque { pub const isAFn = c.mlirTypeIsACuteComposedLayout; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "composed_layout"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -405,7 +396,7 @@ pub const ComposedLayoutType = opaque { const result = c.mlirCuteComposedLayoutTypeGet(ctx.ptr(), args.attr.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getAttr(self: *const Self) *const mlir.Attribute { + pub fn attr(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteComposedLayoutTypeGetAttr(self.ptr()).ptr.?); } }; @@ -417,29 +408,27 @@ pub const ConstrainedIntType = opaque { pub const isAFn = c.mlirTypeIsACuteConstrainedInt; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { divisibility: i64, - width: c_uint, + width: u32, isPow2: bool, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { const result = c.mlirCuteConstrainedIntTypeGet(ctx.ptr(), args.divisibility, args.width, args.isPow2); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getDivisibility(self: *const Self) i64 { + pub fn divisibility(self: *const Self) i64 { return c.mlirCuteConstrainedIntTypeGetDivisibility(self.ptr()); } - pub fn getWidth(self: *const Self) c_uint { + pub fn width(self: *const Self) u32 { return c.mlirCuteConstrainedIntTypeGetWidth(self.ptr()); } - pub fn getIsPow2(self: *const Self) bool { + pub fn isPow2(self: *const Self) bool { return c.mlirCuteConstrainedIntTypeGetIsPow2(self.ptr()); } }; @@ -451,11 +440,9 @@ pub const CoordTensorType = opaque { pub const isAFn = c.mlirTypeIsACuteCoordTensor; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "coord_tensor"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -467,10 +454,10 @@ pub const CoordTensorType = opaque { const result = c.mlirCuteCoordTensorTypeGet(ctx.ptr(), args.arithTuple.ptr(), args.layout.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getArithTuple(self: *const Self) *const mlir.Type { + pub fn arithTuple(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteCoordTensorTypeGetArithTuple(self.ptr()).ptr.?); } - pub fn getLayout(self: *const Self) *const mlir.Type { + pub fn layout(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteCoordTensorTypeGetLayout(self.ptr()).ptr.?); } }; @@ -482,11 +469,9 @@ pub const CoordType = opaque { pub const isAFn = c.mlirTypeIsACuteCoord; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "coord"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -497,7 +482,7 @@ pub const CoordType = opaque { const result = c.mlirCuteCoordTypeGet(ctx.ptr(), args.attr.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getAttr(self: *const Self) *const mlir.Attribute { + pub fn attr(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteCoordTypeGetAttr(self.ptr()).ptr.?); } }; @@ -509,26 +494,24 @@ pub const FastDivmodDivisorType = opaque { pub const isAFn = c.mlirTypeIsACuteFastDivmodDivisor; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "fast_divmod_divisor"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { - width: c_uint, + width: u32, isPow2: bool, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { const result = c.mlirCuteFastDivmodDivisorTypeGet(ctx.ptr(), args.width, args.isPow2); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getWidth(self: *const Self) c_uint { + pub fn width(self: *const Self) u32 { return c.mlirCuteFastDivmodDivisorTypeGetWidth(self.ptr()); } - pub fn getIsPow2(self: *const Self) bool { + pub fn isPow2(self: *const Self) bool { return c.mlirCuteFastDivmodDivisorTypeGetIsPow2(self.ptr()); } }; @@ -540,11 +523,9 @@ pub const IntTupleType = opaque { pub const isAFn = c.mlirTypeIsACuteIntTuple; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "int_tuple"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -555,7 +536,7 @@ pub const IntTupleType = opaque { const result = c.mlirCuteIntTupleTypeGet(ctx.ptr(), args.attr.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getAttr(self: *const Self) *const mlir.Attribute { + pub fn attr(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteIntTupleTypeGetAttr(self.ptr()).ptr.?); } }; @@ -567,11 +548,9 @@ pub const LayoutType = opaque { pub const isAFn = c.mlirTypeIsACuteLayout; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "layout"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -582,7 +561,7 @@ pub const LayoutType = opaque { const result = c.mlirCuteLayoutTypeGet(ctx.ptr(), args.attr.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getAttr(self: *const Self) *const mlir.Attribute { + pub fn attr(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteLayoutTypeGetAttr(self.ptr()).ptr.?); } }; @@ -594,11 +573,9 @@ pub const MemRefType = opaque { pub const isAFn = c.mlirTypeIsACuteMemRef; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "memref"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -610,10 +587,10 @@ pub const MemRefType = opaque { const result = c.mlirCuteMemRefTypeGet(ctx.ptr(), args.ptr.ptr(), args.layout.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getPtr(self: *const Self) *const mlir.Type { + pub fn ptr_(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteMemRefTypeGetPtr(self.ptr()).ptr.?); } - pub fn getLayout(self: *const Self) *const mlir.Type { + pub fn layout(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteMemRefTypeGetLayout(self.ptr()).ptr.?); } }; @@ -625,11 +602,9 @@ pub const PtrType = opaque { pub const isAFn = c.mlirTypeIsACutePtr; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "ptr"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -644,19 +619,19 @@ pub const PtrType = opaque { const result = c.mlirCutePtrTypeGet(ctx.ptr(), if (args.valueType) |v| v.ptr() else c.MlirType{ .ptr = null }, @backingInt(args.addressSpace), args.alignment, if (args.swizzle) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, if (args.bitlayout) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValueType(self: *const Self) ?*const mlir.Type { + pub fn valueType(self: *const Self) ?*const mlir.Type { return @ptrCast(c.mlirCutePtrTypeGetValueType(self.ptr()).ptr); } - pub fn getAddressSpace(self: *const Self) AddressSpace { + pub fn addressSpace(self: *const Self) AddressSpace { return @fromBackingInt(@intCast(c.mlirCutePtrTypeGetAddressSpace(self.ptr()))); } - pub fn getAlignment(self: *const Self) u64 { + pub fn alignment(self: *const Self) u64 { return c.mlirCutePtrTypeGetAlignment(self.ptr()); } - pub fn getSwizzle(self: *const Self) ?*const mlir.Attribute { + pub fn swizzle(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCutePtrTypeGetSwizzle(self.ptr()).ptr); } - pub fn getBitlayout(self: *const Self) ?*const mlir.Attribute { + pub fn bitlayout(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCutePtrTypeGetBitlayout(self.ptr()).ptr); } }; @@ -668,11 +643,9 @@ pub const ShapeType = opaque { pub const isAFn = c.mlirTypeIsACuteShape; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "shape"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -683,7 +656,7 @@ pub const ShapeType = opaque { const result = c.mlirCuteShapeTypeGet(ctx.ptr(), args.attr.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getAttr(self: *const Self) *const mlir.Attribute { + pub fn attr(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteShapeTypeGetAttr(self.ptr()).ptr.?); } }; @@ -695,11 +668,9 @@ pub const SparseElemType = opaque { pub const isAFn = c.mlirTypeIsACuteSparseElem; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "sparse_elem"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -711,10 +682,10 @@ pub const SparseElemType = opaque { const result = c.mlirCuteSparseElemTypeGet(ctx.ptr(), args.numLogical, args.physicalType.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getNumLogical(self: *const Self) c_int { + pub fn numLogical(self: *const Self) c_int { return c.mlirCuteSparseElemTypeGetNumLogical(self.ptr()); } - pub fn getPhysicalType(self: *const Self) *const mlir.Type { + pub fn physicalType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteSparseElemTypeGetPhysicalType(self.ptr()).ptr.?); } }; @@ -726,11 +697,9 @@ pub const StrideType = opaque { pub const isAFn = c.mlirTypeIsACuteStride; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "stride"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -741,7 +710,7 @@ pub const StrideType = opaque { const result = c.mlirCuteStrideTypeGet(ctx.ptr(), args.attr.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getAttr(self: *const Self) *const mlir.Attribute { + pub fn attr(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteStrideTypeGetAttr(self.ptr()).ptr.?); } }; @@ -753,11 +722,9 @@ pub const SwizzleType = opaque { pub const isAFn = c.mlirTypeIsACuteSwizzle; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "swizzle"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -768,7 +735,7 @@ pub const SwizzleType = opaque { const result = c.mlirCuteSwizzleTypeGet(ctx.ptr(), args.attr.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getAttr(self: *const Self) *const mlir.Attribute { + pub fn attr(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSwizzleTypeGetAttr(self.ptr()).ptr.?); } }; @@ -780,11 +747,9 @@ pub const TileType = opaque { pub const isAFn = c.mlirTypeIsACuteTile; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "tile"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -795,7 +760,7 @@ pub const TileType = opaque { const result = c.mlirCuteTileTypeGet(ctx.ptr(), args.attr.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getAttr(self: *const Self) *const mlir.Attribute { + pub fn attr(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteTileTypeGetAttr(self.ptr()).ptr.?); } }; @@ -807,11 +772,9 @@ pub const TiledCopyType = opaque { pub const isAFn = c.mlirTypeIsACuteTiledCopy; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "tiled_copy"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -824,16 +787,16 @@ pub const TiledCopyType = opaque { const result = c.mlirCuteTiledCopyTypeGet(ctx.ptr(), args.copyAtom.ptr(), args.layoutCopyTv.ptr(), args.tilerMn.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getCopyAtom(self: *const Self) *const mlir.Type { + pub fn copyAtom(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteTiledCopyTypeGetCopyAtom(self.ptr()).ptr.?); } - pub fn getLayoutCopyTv(self: *const Self) *const mlir.Attribute { + pub fn layoutCopyTv(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteTiledCopyTypeGetLayoutCopyTv(self.ptr()).ptr.?); } - pub fn getTilerMn(self: *const Self) *const mlir.Attribute { + pub fn tilerMn(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteTiledCopyTypeGetTilerMn(self.ptr()).ptr.?); } - pub const getAtom = getCopyAtom; + pub const atom = copyAtom; }; /// `!cute.tiled_copy_v2`: V2 copy atom tiled over threads and values. @@ -843,11 +806,9 @@ pub const TiledCopyV2Type = opaque { pub const isAFn = c.mlirTypeIsACuteTiledCopyV2; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "tiled_copy_v2"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -860,13 +821,13 @@ pub const TiledCopyV2Type = opaque { const result = c.mlirCuteTiledCopyV2TypeGet(ctx.ptr(), args.copyAtom.ptr(), args.layoutCopyTv.ptr(), args.tilerMn.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getCopyAtom(self: *const Self) *const mlir.Type { + pub fn copyAtom(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteTiledCopyV2TypeGetCopyAtom(self.ptr()).ptr.?); } - pub fn getLayoutCopyTv(self: *const Self) *const mlir.Attribute { + pub fn layoutCopyTv(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteTiledCopyV2TypeGetLayoutCopyTv(self.ptr()).ptr.?); } - pub fn getTilerMn(self: *const Self) *const mlir.Attribute { + pub fn tilerMn(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteTiledCopyV2TypeGetTilerMn(self.ptr()).ptr.?); } }; @@ -878,11 +839,9 @@ pub const TiledMmaType = opaque { pub const isAFn = c.mlirTypeIsACuteTiledMma; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "tiled_mma"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -895,18 +854,18 @@ pub const TiledMmaType = opaque { const result = c.mlirCuteTiledMmaTypeGet(ctx.ptr(), args.mmaAtom.ptr(), args.atomLayoutMNK.ptr(), if (args.permutationMNK) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getMmaAtom(self: *const Self) *const mlir.Type { + pub fn mmaAtom(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteTiledMmaTypeGetMmaAtom(self.ptr()).ptr.?); } - pub fn getAtomLayoutMNK(self: *const Self) *const mlir.Attribute { + pub fn atomLayoutMNK(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteTiledMmaTypeGetAtomLayoutMNK(self.ptr()).ptr.?); } - pub fn getPermutationMNK(self: *const Self) ?*const mlir.Attribute { + pub fn permutationMNK(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteTiledMmaTypeGetPermutationMNK(self.ptr()).ptr); } - pub const getAtom = getMmaAtom; - pub const getAtomLayoutMnk = getAtomLayoutMNK; - pub const getPermutationMnk = getPermutationMNK; + pub const atom = mmaAtom; + pub const getAtomLayoutMnk = atomLayoutMNK; + pub const getPermutationMnk = permutationMNK; }; /// `!cute.tuple`: Non-empty tuple of types. @@ -916,11 +875,9 @@ pub const TupleType = opaque { pub const isAFn = c.mlirTypeIsACuteTuple; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "tuple"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -931,10 +888,10 @@ pub const TupleType = opaque { const result = c.mlirCuteTupleTypeGet(ctx.ptr(), @intCast(args.types.len), @ptrCast(args.types.ptr)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getNumTypes(self: *const Self) usize { + pub fn numTypes(self: *const Self) usize { return @intCast(c.mlirCuteTupleTypeGetNumTypes(self.ptr())); } - pub fn getType(self: *const Self, pos: usize) *const mlir.Type { + pub fn @"type"(self: *const Self, pos: usize) *const mlir.Type { return @ptrCast(c.mlirCuteTupleTypeGetType(self.ptr(), @intCast(pos)).ptr.?); } }; @@ -946,11 +903,9 @@ pub const BitLayoutAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteBitLayout; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "bitlayout"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -961,7 +916,7 @@ pub const BitLayoutAttr = opaque { const result = c.mlirCuteBitLayoutAttrGet(ctx.ptr(), args.layout.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getLayout(self: *const Self) *const mlir.Attribute { + pub fn layout(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteBitLayoutAttrGetLayout(self.ptr()).ptr.?); } }; @@ -973,11 +928,9 @@ pub const CopyAtomAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteCopyAtom; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "copy_atom"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -991,16 +944,16 @@ pub const CopyAtomAttr = opaque { const result = c.mlirCuteCopyAtomAttrGet(ctx.ptr(), args.thrId.ptr(), args.layoutSrc.ptr(), args.layoutDst.ptr(), args.layoutRef.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getThrId(self: *const Self) *const mlir.Attribute { + pub fn thrId(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteCopyAtomAttrGetThrId(self.ptr()).ptr.?); } - pub fn getLayoutSrc(self: *const Self) *const mlir.Attribute { + pub fn layoutSrc(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteCopyAtomAttrGetLayoutSrc(self.ptr()).ptr.?); } - pub fn getLayoutDst(self: *const Self) *const mlir.Attribute { + pub fn layoutDst(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteCopyAtomAttrGetLayoutDst(self.ptr()).ptr.?); } - pub fn getLayoutRef(self: *const Self) *const mlir.Attribute { + pub fn layoutRef(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteCopyAtomAttrGetLayoutRef(self.ptr()).ptr.?); } }; @@ -1012,11 +965,9 @@ pub const CopyAtomV2Attr = opaque { pub const isAFn = c.mlirAttributeIsACuteCopyAtomV2; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "copy_atom_v2"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1031,19 +982,19 @@ pub const CopyAtomV2Attr = opaque { const result = c.mlirCuteCopyAtomV2AttrGet(ctx.ptr(), args.thrId.ptr(), args.layoutSrcTV.ptr(), args.layoutDstTV.ptr(), args.frgSrc.ptr(), args.frgDst.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getThrId(self: *const Self) *const mlir.Attribute { + pub fn thrId(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteCopyAtomV2AttrGetThrId(self.ptr()).ptr.?); } - pub fn getLayoutSrcTV(self: *const Self) *const mlir.Attribute { + pub fn layoutSrcTV(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteCopyAtomV2AttrGetLayoutSrcTV(self.ptr()).ptr.?); } - pub fn getLayoutDstTV(self: *const Self) *const mlir.Attribute { + pub fn layoutDstTV(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteCopyAtomV2AttrGetLayoutDstTV(self.ptr()).ptr.?); } - pub fn getFrgSrc(self: *const Self) *const mlir.Attribute { + pub fn frgSrc(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteCopyAtomV2AttrGetFrgSrc(self.ptr()).ptr.?); } - pub fn getFrgDst(self: *const Self) *const mlir.Attribute { + pub fn frgDst(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteCopyAtomV2AttrGetFrgDst(self.ptr()).ptr.?); } }; @@ -1055,11 +1006,9 @@ pub const MmaAtomAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteMmaAtom; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "mma_atom"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1080,37 +1029,37 @@ pub const MmaAtomAttr = opaque { const result = c.mlirCuteMmaAtomAttrGet(ctx.ptr(), args.shapeMNK.ptr(), args.thrId.ptr(), args.layoutATV.ptr(), args.layoutBTV.ptr(), args.layoutCTV.ptr(), args.shapeAMK.ptr(), args.shapeBNK.ptr(), args.shapeCMN.ptr(), args.frgA.ptr(), args.frgB.ptr(), args.frgC.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getShapeMNK(self: *const Self) *const mlir.Attribute { + pub fn shapeMNK(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMmaAtomAttrGetShapeMNK(self.ptr()).ptr.?); } - pub fn getThrId(self: *const Self) *const mlir.Attribute { + pub fn thrId(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMmaAtomAttrGetThrId(self.ptr()).ptr.?); } - pub fn getLayoutATV(self: *const Self) *const mlir.Attribute { + pub fn layoutATV(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMmaAtomAttrGetLayoutATV(self.ptr()).ptr.?); } - pub fn getLayoutBTV(self: *const Self) *const mlir.Attribute { + pub fn layoutBTV(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMmaAtomAttrGetLayoutBTV(self.ptr()).ptr.?); } - pub fn getLayoutCTV(self: *const Self) *const mlir.Attribute { + pub fn layoutCTV(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMmaAtomAttrGetLayoutCTV(self.ptr()).ptr.?); } - pub fn getShapeAMK(self: *const Self) *const mlir.Attribute { + pub fn shapeAMK(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMmaAtomAttrGetShapeAMK(self.ptr()).ptr.?); } - pub fn getShapeBNK(self: *const Self) *const mlir.Attribute { + pub fn shapeBNK(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMmaAtomAttrGetShapeBNK(self.ptr()).ptr.?); } - pub fn getShapeCMN(self: *const Self) *const mlir.Attribute { + pub fn shapeCMN(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMmaAtomAttrGetShapeCMN(self.ptr()).ptr.?); } - pub fn getFrgA(self: *const Self) *const mlir.Attribute { + pub fn frgA(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMmaAtomAttrGetFrgA(self.ptr()).ptr.?); } - pub fn getFrgB(self: *const Self) *const mlir.Attribute { + pub fn frgB(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMmaAtomAttrGetFrgB(self.ptr()).ptr.?); } - pub fn getFrgC(self: *const Self) *const mlir.Attribute { + pub fn frgC(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMmaAtomAttrGetFrgC(self.ptr()).ptr.?); } }; @@ -1122,11 +1071,9 @@ pub const MxMmaAtomAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteMxMmaAtom; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "mx_mma_atom"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1153,55 +1100,55 @@ pub const MxMmaAtomAttr = opaque { const result = c.mlirCuteMxMmaAtomAttrGet(ctx.ptr(), args.shapeMNK.ptr(), args.thrId.ptr(), args.layoutATV.ptr(), args.layoutSFATV.ptr(), args.layoutBTV.ptr(), args.layoutSFBTV.ptr(), args.layoutCTV.ptr(), args.shapeAMK.ptr(), args.shapeSFAMK.ptr(), args.shapeBNK.ptr(), args.shapeSFBNK.ptr(), args.shapeCMN.ptr(), args.frgA.ptr(), args.frgSFA.ptr(), args.frgB.ptr(), args.frgSFB.ptr(), args.frgC.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getShapeMNK(self: *const Self) *const mlir.Attribute { + pub fn shapeMNK(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeMNK(self.ptr()).ptr.?); } - pub fn getThrId(self: *const Self) *const mlir.Attribute { + pub fn thrId(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMxMmaAtomAttrGetThrId(self.ptr()).ptr.?); } - pub fn getLayoutATV(self: *const Self) *const mlir.Attribute { + pub fn layoutATV(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMxMmaAtomAttrGetLayoutATV(self.ptr()).ptr.?); } - pub fn getLayoutSFATV(self: *const Self) *const mlir.Attribute { + pub fn layoutSFATV(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMxMmaAtomAttrGetLayoutSFATV(self.ptr()).ptr.?); } - pub fn getLayoutBTV(self: *const Self) *const mlir.Attribute { + pub fn layoutBTV(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMxMmaAtomAttrGetLayoutBTV(self.ptr()).ptr.?); } - pub fn getLayoutSFBTV(self: *const Self) *const mlir.Attribute { + pub fn layoutSFBTV(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMxMmaAtomAttrGetLayoutSFBTV(self.ptr()).ptr.?); } - pub fn getLayoutCTV(self: *const Self) *const mlir.Attribute { + pub fn layoutCTV(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMxMmaAtomAttrGetLayoutCTV(self.ptr()).ptr.?); } - pub fn getShapeAMK(self: *const Self) *const mlir.Attribute { + pub fn shapeAMK(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeAMK(self.ptr()).ptr.?); } - pub fn getShapeSFAMK(self: *const Self) *const mlir.Attribute { + pub fn shapeSFAMK(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeSFAMK(self.ptr()).ptr.?); } - pub fn getShapeBNK(self: *const Self) *const mlir.Attribute { + pub fn shapeBNK(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeBNK(self.ptr()).ptr.?); } - pub fn getShapeSFBNK(self: *const Self) *const mlir.Attribute { + pub fn shapeSFBNK(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeSFBNK(self.ptr()).ptr.?); } - pub fn getShapeCMN(self: *const Self) *const mlir.Attribute { + pub fn shapeCMN(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeCMN(self.ptr()).ptr.?); } - pub fn getFrgA(self: *const Self) *const mlir.Attribute { + pub fn frgA(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMxMmaAtomAttrGetFrgA(self.ptr()).ptr.?); } - pub fn getFrgSFA(self: *const Self) *const mlir.Attribute { + pub fn frgSFA(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMxMmaAtomAttrGetFrgSFA(self.ptr()).ptr.?); } - pub fn getFrgB(self: *const Self) *const mlir.Attribute { + pub fn frgB(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMxMmaAtomAttrGetFrgB(self.ptr()).ptr.?); } - pub fn getFrgSFB(self: *const Self) *const mlir.Attribute { + pub fn frgSFB(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMxMmaAtomAttrGetFrgSFB(self.ptr()).ptr.?); } - pub fn getFrgC(self: *const Self) *const mlir.Attribute { + pub fn frgC(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteMxMmaAtomAttrGetFrgC(self.ptr()).ptr.?); } }; @@ -1213,11 +1160,9 @@ pub const ReductionOpAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteReductionOp; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "reduction_op"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1228,7 +1173,7 @@ pub const ReductionOpAttr = opaque { const result = c.mlirCuteReductionOpAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) ReductionOp { + pub fn value(self: *const Self) ReductionOp { return @fromBackingInt(@intCast(c.mlirCuteReductionOpAttrGetValue(self.ptr()))); } }; @@ -1240,11 +1185,9 @@ pub const SparseMmaAtomAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteSparseMmaAtom; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "sparse_mma_atom"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1268,46 +1211,46 @@ pub const SparseMmaAtomAttr = opaque { const result = c.mlirCuteSparseMmaAtomAttrGet(ctx.ptr(), args.shapeMNK.ptr(), args.thrId.ptr(), args.layoutATV.ptr(), args.layoutBTV.ptr(), args.layoutCTV.ptr(), args.layoutETV.ptr(), args.shapeAMK.ptr(), args.shapeBNK.ptr(), args.shapeCMN.ptr(), args.shapeEMK.ptr(), args.frgA.ptr(), args.frgB.ptr(), args.frgC.ptr(), args.frgE.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getShapeMNK(self: *const Self) *const mlir.Attribute { + pub fn shapeMNK(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetShapeMNK(self.ptr()).ptr.?); } - pub fn getThrId(self: *const Self) *const mlir.Attribute { + pub fn thrId(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetThrId(self.ptr()).ptr.?); } - pub fn getLayoutATV(self: *const Self) *const mlir.Attribute { + pub fn layoutATV(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetLayoutATV(self.ptr()).ptr.?); } - pub fn getLayoutBTV(self: *const Self) *const mlir.Attribute { + pub fn layoutBTV(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetLayoutBTV(self.ptr()).ptr.?); } - pub fn getLayoutCTV(self: *const Self) *const mlir.Attribute { + pub fn layoutCTV(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetLayoutCTV(self.ptr()).ptr.?); } - pub fn getLayoutETV(self: *const Self) *const mlir.Attribute { + pub fn layoutETV(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetLayoutETV(self.ptr()).ptr.?); } - pub fn getShapeAMK(self: *const Self) *const mlir.Attribute { + pub fn shapeAMK(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetShapeAMK(self.ptr()).ptr.?); } - pub fn getShapeBNK(self: *const Self) *const mlir.Attribute { + pub fn shapeBNK(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetShapeBNK(self.ptr()).ptr.?); } - pub fn getShapeCMN(self: *const Self) *const mlir.Attribute { + pub fn shapeCMN(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetShapeCMN(self.ptr()).ptr.?); } - pub fn getShapeEMK(self: *const Self) *const mlir.Attribute { + pub fn shapeEMK(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetShapeEMK(self.ptr()).ptr.?); } - pub fn getFrgA(self: *const Self) *const mlir.Attribute { + pub fn frgA(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetFrgA(self.ptr()).ptr.?); } - pub fn getFrgB(self: *const Self) *const mlir.Attribute { + pub fn frgB(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetFrgB(self.ptr()).ptr.?); } - pub fn getFrgC(self: *const Self) *const mlir.Attribute { + pub fn frgC(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetFrgC(self.ptr()).ptr.?); } - pub fn getFrgE(self: *const Self) *const mlir.Attribute { + pub fn frgE(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetFrgE(self.ptr()).ptr.?); } }; @@ -1319,11 +1262,9 @@ pub const SparseMxMmaAtomAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteSparseMxMmaAtom; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "sparse_mx_mma_atom"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1353,64 +1294,64 @@ pub const SparseMxMmaAtomAttr = opaque { const result = c.mlirCuteSparseMxMmaAtomAttrGet(ctx.ptr(), args.shapeMNK.ptr(), args.thrId.ptr(), args.layoutATV.ptr(), args.layoutSFATV.ptr(), args.layoutBTV.ptr(), args.layoutSFBTV.ptr(), args.layoutCTV.ptr(), args.layoutETV.ptr(), args.shapeAMK.ptr(), args.shapeSFAMK.ptr(), args.shapeBNK.ptr(), args.shapeSFBNK.ptr(), args.shapeCMN.ptr(), args.shapeEMK.ptr(), args.frgA.ptr(), args.frgSFA.ptr(), args.frgB.ptr(), args.frgSFB.ptr(), args.frgC.ptr(), args.frgE.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getShapeMNK(self: *const Self) *const mlir.Attribute { + pub fn shapeMNK(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeMNK(self.ptr()).ptr.?); } - pub fn getThrId(self: *const Self) *const mlir.Attribute { + pub fn thrId(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetThrId(self.ptr()).ptr.?); } - pub fn getLayoutATV(self: *const Self) *const mlir.Attribute { + pub fn layoutATV(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutATV(self.ptr()).ptr.?); } - pub fn getLayoutSFATV(self: *const Self) *const mlir.Attribute { + pub fn layoutSFATV(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutSFATV(self.ptr()).ptr.?); } - pub fn getLayoutBTV(self: *const Self) *const mlir.Attribute { + pub fn layoutBTV(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutBTV(self.ptr()).ptr.?); } - pub fn getLayoutSFBTV(self: *const Self) *const mlir.Attribute { + pub fn layoutSFBTV(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutSFBTV(self.ptr()).ptr.?); } - pub fn getLayoutCTV(self: *const Self) *const mlir.Attribute { + pub fn layoutCTV(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutCTV(self.ptr()).ptr.?); } - pub fn getLayoutETV(self: *const Self) *const mlir.Attribute { + pub fn layoutETV(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutETV(self.ptr()).ptr.?); } - pub fn getShapeAMK(self: *const Self) *const mlir.Attribute { + pub fn shapeAMK(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeAMK(self.ptr()).ptr.?); } - pub fn getShapeSFAMK(self: *const Self) *const mlir.Attribute { + pub fn shapeSFAMK(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeSFAMK(self.ptr()).ptr.?); } - pub fn getShapeBNK(self: *const Self) *const mlir.Attribute { + pub fn shapeBNK(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeBNK(self.ptr()).ptr.?); } - pub fn getShapeSFBNK(self: *const Self) *const mlir.Attribute { + pub fn shapeSFBNK(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeSFBNK(self.ptr()).ptr.?); } - pub fn getShapeCMN(self: *const Self) *const mlir.Attribute { + pub fn shapeCMN(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeCMN(self.ptr()).ptr.?); } - pub fn getShapeEMK(self: *const Self) *const mlir.Attribute { + pub fn shapeEMK(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeEMK(self.ptr()).ptr.?); } - pub fn getFrgA(self: *const Self) *const mlir.Attribute { + pub fn frgA(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgA(self.ptr()).ptr.?); } - pub fn getFrgSFA(self: *const Self) *const mlir.Attribute { + pub fn frgSFA(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgSFA(self.ptr()).ptr.?); } - pub fn getFrgB(self: *const Self) *const mlir.Attribute { + pub fn frgB(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgB(self.ptr()).ptr.?); } - pub fn getFrgSFB(self: *const Self) *const mlir.Attribute { + pub fn frgSFB(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgSFB(self.ptr()).ptr.?); } - pub fn getFrgC(self: *const Self) *const mlir.Attribute { + pub fn frgC(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgC(self.ptr()).ptr.?); } - pub fn getFrgE(self: *const Self) *const mlir.Attribute { + pub fn frgE(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgE(self.ptr()).ptr.?); } }; @@ -1438,9 +1379,17 @@ pub fn append_to_rank(ctx: *mlir.Context, input: *const mlir.Value, element: *co } /// `cute.apply_swizzle`. Result types are explicit; attributes use mlir.Attribute. -pub fn apply_swizzle(ctx: *mlir.Context, ptr: *const mlir.Value, swizzled_ptr_type: *const mlir.Type, swizzle: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn apply_swizzle( + ctx: *mlir.Context, + ptr: *const mlir.Value, + swizzled_ptr_type: *const mlir.Type, + opts: struct { + swizzle: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (swizzle) |value| attributes.appendAssumeCapacity(.named(ctx, "swizzle", value)); + if (opts.swizzle) |value| attributes.appendAssumeCapacity(.named(ctx, "swizzle", value)); return mlir.Operation.make(ctx, "cute.apply_swizzle", .{ .operands = .{ .flat = &.{ptr} }, .results = .{ .flat = &.{swizzled_ptr_type} }, @@ -1477,11 +1426,19 @@ pub fn ceil_div(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir } /// `cute.coalesce`. Result types are explicit; attributes use mlir.Attribute. -pub fn coalesce(ctx: *mlir.Context, input: *const mlir.Value, target_profile: ?*const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +pub fn coalesce( + ctx: *mlir.Context, + input: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + target_profile: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cute.coalesce", .{ .operands = .{ .variadic = &.{ &.{input}, - if (target_profile) |value| &.{value} else &.{}, + if (opts.target_profile) |value| &.{value} else &.{}, } }, .results = .{ .flat = &.{result_type} }, .location = location, @@ -1534,13 +1491,22 @@ pub fn composition(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir. } /// `cute.copy`. Result types are explicit; attributes use mlir.Attribute. -pub fn copy(ctx: *mlir.Context, copy_atom: *const mlir.Value, src: []const *const mlir.Value, dst: []const *const mlir.Value, pred: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { +pub fn copy( + ctx: *mlir.Context, + copy_atom: *const mlir.Value, + src: []const *const mlir.Value, + dst: []const *const mlir.Value, + opts: struct { + pred: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cute.copy", .{ .operands = .{ .variadic = &.{ &.{copy_atom}, src, dst, - if (pred) |value| &.{value} else &.{}, + if (opts.pred) |value| &.{value} else &.{}, } }, .location = location, }); @@ -1559,22 +1525,39 @@ pub fn copy_make_fragment(ctx: *mlir.Context, atom: *const mlir.Value, input: *c } /// `cute.copy_atom_call`. Result types are explicit; attributes use mlir.Attribute. -pub fn copy_atom_call(ctx: *mlir.Context, atom: *const mlir.Value, src: []const *const mlir.Value, dst: []const *const mlir.Value, pred: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { +pub fn copy_atom_call( + ctx: *mlir.Context, + atom: *const mlir.Value, + src: []const *const mlir.Value, + dst: []const *const mlir.Value, + opts: struct { + pred: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cute.copy_atom_call", .{ .operands = .{ .variadic = &.{ &.{atom}, src, dst, - if (pred) |value| &.{value} else &.{}, + if (opts.pred) |value| &.{value} else &.{}, } }, .location = location, }); } /// `cute.cosize`. Result types are explicit; attributes use mlir.Attribute. -pub fn cosize(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, mode: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn cosize( + ctx: *mlir.Context, + input: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); return mlir.Operation.make(ctx, "cute.cosize", .{ .operands = .{ .flat = &.{input} }, .results = .{ .flat = &.{result_type} }, @@ -1713,11 +1696,19 @@ pub fn filter(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const } /// `cute.filter_zeros`. Result types are explicit; attributes use mlir.Attribute. -pub fn filter_zeros(ctx: *mlir.Context, input: *const mlir.Value, target_profile: ?*const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +pub fn filter_zeros( + ctx: *mlir.Context, + input: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + target_profile: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cute.filter_zeros", .{ .operands = .{ .variadic = &.{ &.{input}, - if (target_profile) |value| &.{value} else &.{}, + if (opts.target_profile) |value| &.{value} else &.{}, } }, .results = .{ .flat = &.{res_type} }, .location = location, @@ -1757,9 +1748,17 @@ pub fn gemm(ctx: *mlir.Context, mma_atom: *const mlir.Value, d: *const mlir.Valu } /// `cute.get`. Result types are explicit; attributes use mlir.Attribute. -pub fn get(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, mode: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn get( + ctx: *mlir.Context, + input: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); return mlir.Operation.make(ctx, "cute.get", .{ .operands = .{ .flat = &.{input} }, .results = .{ .flat = &.{result_type} }, @@ -1836,9 +1835,17 @@ pub fn get_leaves(ctx: *mlir.Context, input: *const mlir.Value, results__types: } /// `cute.get_scalars`. Result types are explicit; attributes use mlir.Attribute. -pub fn get_scalars(ctx: *mlir.Context, cute_value: *const mlir.Value, scalars_types: []const *const mlir.Type, only_dynamic: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn get_scalars( + ctx: *mlir.Context, + cute_value: *const mlir.Value, + scalars_types: []const *const mlir.Type, + opts: struct { + only_dynamic: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (only_dynamic) |value| attributes.appendAssumeCapacity(.named(ctx, "only_dynamic", value)); + if (opts.only_dynamic) |value| attributes.appendAssumeCapacity(.named(ctx, "only_dynamic", value)); return mlir.Operation.make(ctx, "cute.get_scalars", .{ .operands = .{ .flat = &.{cute_value} }, .results = .{ .variadic = &.{ @@ -1946,9 +1953,19 @@ pub fn local_partition(ctx: *mlir.Context, input: *const mlir.Value, tiler: *con } /// `cute.local_tile`. Result types are explicit; attributes use mlir.Attribute. -pub fn local_tile(ctx: *mlir.Context, input: *const mlir.Value, tile: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, proj: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn local_tile( + ctx: *mlir.Context, + input: *const mlir.Value, + tile: *const mlir.Value, + coord: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + proj: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (proj) |value| attributes.appendAssumeCapacity(.named(ctx, "proj", value)); + if (opts.proj) |value| attributes.appendAssumeCapacity(.named(ctx, "proj", value)); return mlir.Operation.make(ctx, "cute.local_tile", .{ .operands = .{ .flat = &.{ input, tile, coord } }, .results = .{ .flat = &.{result_type} }, @@ -1976,10 +1993,17 @@ pub fn logical_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *con } /// `cute.make_arith_tuple_iter`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_arith_tuple_iter(ctx: *mlir.Context, value_: ?*const mlir.Value, iter_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +pub fn make_arith_tuple_iter( + ctx: *mlir.Context, + iter_type: *const mlir.Type, + opts: struct { + value_: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cute.make_arith_tuple_iter", .{ .operands = .{ .variadic = &.{ - if (value_) |value| &.{value} else &.{}, + if (opts.value_) |value| &.{value} else &.{}, } }, .results = .{ .flat = &.{iter_type} }, .location = location, @@ -2065,11 +2089,19 @@ pub fn make_int_tuple(ctx: *mlir.Context, dynamicElements: []const *const mlir.V } /// `cute.make_layout`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_layout(ctx: *mlir.Context, shape: ?*const mlir.Value, stride: ?*const mlir.Value, layout_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +pub fn make_layout( + ctx: *mlir.Context, + layout_type: *const mlir.Type, + opts: struct { + shape: ?*const mlir.Value = null, + stride: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cute.make_layout", .{ .operands = .{ .variadic = &.{ - if (shape) |value| &.{value} else &.{}, - if (stride) |value| &.{value} else &.{}, + if (opts.shape) |value| &.{value} else &.{}, + if (opts.stride) |value| &.{value} else &.{}, } }, .results = .{ .flat = &.{layout_type} }, .location = location, @@ -2175,11 +2207,19 @@ pub fn make_tuple(ctx: *mlir.Context, values: []const *const mlir.Value, result_ } /// `cute.make_view`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_view(ctx: *mlir.Context, iter: *const mlir.Value, layout: ?*const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +pub fn make_view( + ctx: *mlir.Context, + iter: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + layout: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cute.make_view", .{ .operands = .{ .variadic = &.{ &.{iter}, - if (layout) |value| &.{value} else &.{}, + if (opts.layout) |value| &.{value} else &.{}, } }, .results = .{ .flat = &.{result_type} }, .location = location, @@ -2195,10 +2235,17 @@ pub fn memref_alloc_smem(ctx: *mlir.Context, memref_type: *const mlir.Type, loca } /// `cute.memref.alloca`. Result types are explicit; attributes use mlir.Attribute. -pub fn memref_alloca(ctx: *mlir.Context, layout: ?*const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +pub fn memref_alloca( + ctx: *mlir.Context, + result_type: *const mlir.Type, + opts: struct { + layout: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cute.memref.alloca", .{ .operands = .{ .variadic = &.{ - if (layout) |value| &.{value} else &.{}, + if (opts.layout) |value| &.{value} else &.{}, } }, .results = .{ .flat = &.{result_type} }, .location = location, @@ -2215,12 +2262,21 @@ pub fn memref_load(ctx: *mlir.Context, src: *const mlir.Value, coord: *const mli } /// `cute.memref.load_vec`. Result types are explicit; attributes use mlir.Attribute. -pub fn memref_load_vec(ctx: *mlir.Context, src: *const mlir.Value, mask: ?*const mlir.Value, pass_thru: ?*const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +pub fn memref_load_vec( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + mask: ?*const mlir.Value = null, + pass_thru: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cute.memref.load_vec", .{ .operands = .{ .variadic = &.{ &.{src}, - if (mask) |value| &.{value} else &.{}, - if (pass_thru) |value| &.{value} else &.{}, + if (opts.mask) |value| &.{value} else &.{}, + if (opts.pass_thru) |value| &.{value} else &.{}, } }, .results = .{ .flat = &.{result_type} }, .location = location, @@ -2236,12 +2292,20 @@ pub fn memref_store(ctx: *mlir.Context, dst: *const mlir.Value, coord: *const ml } /// `cute.memref.store_vec`. Result types are explicit; attributes use mlir.Attribute. -pub fn memref_store_vec(ctx: *mlir.Context, value_: *const mlir.Value, dst: *const mlir.Value, mask: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { +pub fn memref_store_vec( + ctx: *mlir.Context, + value_: *const mlir.Value, + dst: *const mlir.Value, + opts: struct { + mask: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cute.memref.store_vec", .{ .operands = .{ .variadic = &.{ &.{value_}, &.{dst}, - if (mask) |value| &.{value} else &.{}, + if (opts.mask) |value| &.{value} else &.{}, } }, .location = location, }); @@ -2308,10 +2372,18 @@ pub fn prepend_to_rank(ctx: *mlir.Context, input: *const mlir.Value, element: *c } /// `cute.print`. Result types are explicit; attributes use mlir.Attribute. -pub fn print(ctx: *mlir.Context, values: []const *const mlir.Value, fmt: ?*const mlir.Attribute, operand_signed: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn print( + ctx: *mlir.Context, + values: []const *const mlir.Value, + opts: struct { + fmt: ?*const mlir.Attribute = null, + operand_signed: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (fmt) |value| attributes.appendAssumeCapacity(.named(ctx, "fmt", value)); - if (operand_signed) |value| attributes.appendAssumeCapacity(.named(ctx, "operand_signed", value)); + if (opts.fmt) |value| attributes.appendAssumeCapacity(.named(ctx, "fmt", value)); + if (opts.operand_signed) |value| attributes.appendAssumeCapacity(.named(ctx, "operand_signed", value)); return mlir.Operation.make(ctx, "cute.print", .{ .operands = .{ .variadic = &.{ values, @@ -2338,14 +2410,23 @@ pub fn print_tma_desc_tiled(ctx: *mlir.Context, input: *const mlir.Value, fd: *c } /// `cute.print_view`. Result types are explicit; attributes use mlir.Attribute. -pub fn print_view(ctx: *mlir.Context, src: *const mlir.Value, coord: ?*const mlir.Value, verbose: ?*const mlir.Attribute, is_signed: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn print_view( + ctx: *mlir.Context, + src: *const mlir.Value, + opts: struct { + coord: ?*const mlir.Value = null, + verbose: ?*const mlir.Attribute = null, + is_signed: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (verbose) |value| attributes.appendAssumeCapacity(.named(ctx, "verbose", value)); - if (is_signed) |value| attributes.appendAssumeCapacity(.named(ctx, "is_signed", value)); + if (opts.verbose) |value| attributes.appendAssumeCapacity(.named(ctx, "verbose", value)); + if (opts.is_signed) |value| attributes.appendAssumeCapacity(.named(ctx, "is_signed", value)); return mlir.Operation.make(ctx, "cute.print_view", .{ .operands = .{ .variadic = &.{ &.{src}, - if (coord) |value| &.{value} else &.{}, + if (opts.coord) |value| &.{value} else &.{}, } }, .attributes = attributes.constSlice(), .location = location, @@ -2417,20 +2498,33 @@ pub fn recast_layout(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *cons } /// `cute.reduce`. Result types are explicit; attributes use mlir.Attribute. -pub fn reduce(ctx: *mlir.Context, input: *const mlir.Value, shape: *const mlir.Value, init_val: *const mlir.Value, predication: ?*const mlir.Value, output_type: *const mlir.Type, reduced_shape_type: ?*const mlir.Type, reduction_op: *const mlir.Attribute, reduction_profile: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn reduce( + ctx: *mlir.Context, + input: *const mlir.Value, + shape: *const mlir.Value, + init_val: *const mlir.Value, + output_type: *const mlir.Type, + reduction_op: *const mlir.Attribute, + opts: struct { + predication: ?*const mlir.Value = null, + reduced_shape_type: ?*const mlir.Type = null, + reduction_profile: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; attributes.appendAssumeCapacity(.named(ctx, "reduction_op", reduction_op)); - if (reduction_profile) |value| attributes.appendAssumeCapacity(.named(ctx, "reduction_profile", value)); + if (opts.reduction_profile) |value| attributes.appendAssumeCapacity(.named(ctx, "reduction_profile", value)); return mlir.Operation.make(ctx, "cute.reduce", .{ .operands = .{ .variadic = &.{ &.{input}, &.{shape}, &.{init_val}, - if (predication) |value| &.{value} else &.{}, + if (opts.predication) |value| &.{value} else &.{}, } }, .results = .{ .variadic = &.{ &.{output_type}, - if (reduced_shape_type) |value| &.{value} else &.{}, + if (opts.reduced_shape_type) |value| &.{value} else &.{}, } }, .attributes = attributes.constSlice(), .location = location, @@ -2468,9 +2562,17 @@ pub fn shape_div(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, } /// `cute.size`. Result types are explicit; attributes use mlir.Attribute. -pub fn size(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, mode: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn size( + ctx: *mlir.Context, + input: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); return mlir.Operation.make(ctx, "cute.size", .{ .operands = .{ .flat = &.{input} }, .results = .{ .flat = &.{result_type} }, @@ -2489,10 +2591,18 @@ pub fn slice(ctx: *mlir.Context, input: *const mlir.Value, coord: *const mlir.Va } /// `cute.smem_partition_size`. Result types are explicit; attributes use mlir.Attribute. -pub fn smem_partition_size(ctx: *mlir.Context, size_type: *const mlir.Type, partition_id: *const mlir.Attribute, cumulative: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn smem_partition_size( + ctx: *mlir.Context, + size_type: *const mlir.Type, + partition_id: *const mlir.Attribute, + opts: struct { + cumulative: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; attributes.appendAssumeCapacity(.named(ctx, "partition_id", partition_id)); - if (cumulative) |value| attributes.appendAssumeCapacity(.named(ctx, "cumulative", value)); + if (opts.cumulative) |value| attributes.appendAssumeCapacity(.named(ctx, "cumulative", value)); return mlir.Operation.make(ctx, "cute.smem_partition_size", .{ .results = .{ .flat = &.{size_type} }, .attributes = attributes.constSlice(), @@ -2509,14 +2619,25 @@ pub fn static(ctx: *mlir.Context, result_type: *const mlir.Type, location: *cons } /// `cute.stencil_divide`. Result types are explicit; attributes use mlir.Attribute. -pub fn stencil_divide(ctx: *mlir.Context, input: *const mlir.Value, stencil: *const mlir.Value, padding_upper: ?*const mlir.Value, padding_lower: ?*const mlir.Value, traversal_stride: ?*const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +pub fn stencil_divide( + ctx: *mlir.Context, + input: *const mlir.Value, + stencil: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + padding_upper: ?*const mlir.Value = null, + padding_lower: ?*const mlir.Value = null, + traversal_stride: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cute.stencil_divide", .{ .operands = .{ .variadic = &.{ &.{input}, &.{stencil}, - if (padding_upper) |value| &.{value} else &.{}, - if (padding_lower) |value| &.{value} else &.{}, - if (traversal_stride) |value| &.{value} else &.{}, + if (opts.padding_upper) |value| &.{value} else &.{}, + if (opts.padding_lower) |value| &.{value} else &.{}, + if (opts.traversal_stride) |value| &.{value} else &.{}, } }, .results = .{ .flat = &.{result_type} }, .location = location, @@ -2712,393 +2833,3 @@ pub fn zipped_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *cons .location = location, }); } - -pub const operation_names: []const []const u8 = &.{ - "cute.add_offset", - "cute.append_to_rank", - "cute.apply_swizzle", - "cute.assume", - "cute.blocked_product", - "cute.ceil_div", - "cute.coalesce", - "cute.complement", - "cute.composed_get_inner", - "cute.composed_get_offset", - "cute.composed_get_outer", - "cute.composition", - "cute.copy", - "cute.copy.make_fragment", - "cute.copy_atom_call", - "cute.cosize", - "cute.crd2idx", - "cute.deref_arith_tuple_iter", - "cute.deref_desc_iter", - "cute.derefine", - "cute.dice", - "cute.elem_less", - "cute.equal", - "cute.fast_divmod.compute", - "cute.fast_divmod.create_divisor", - "cute.fast_divmod.divide", - "cute.fast_divmod.get_aux", - "cute.fast_divmod.get_divisor", - "cute.fast_divmod.make_divisor", - "cute.filter", - "cute.filter_zeros", - "cute.flat_divide", - "cute.flat_product", - "cute.gemm", - "cute.get", - "cute.get_flat_coord", - "cute.get_hier_coord", - "cute.get_integral_coord", - "cute.get_iter", - "cute.get_layout", - "cute.get_layouts_from_tile", - "cute.get_leaves", - "cute.get_scalars", - "cute.get_shape", - "cute.get_stride", - "cute.group_modes", - "cute.idx2crd", - "cute.increment_coord", - "cute.inttoptr", - "cute.kernel_smem_size", - "cute.left_inverse", - "cute.load_scaled_index", - "cute.local_partition", - "cute.local_tile", - "cute.logical_divide", - "cute.logical_product", - "cute.make_arith_tuple_iter", - "cute.make_atom", - "cute.make_composed_layout", - "cute.make_coord", - "cute.make_desc_iter", - "cute.make_fragment_like", - "cute.make_identity_layout", - "cute.make_identity_tensor", - "cute.make_int_tuple", - "cute.make_layout", - "cute.make_layout_like", - "cute.make_ordered_layout", - "cute.make_shape", - "cute.make_sparse_elem", - "cute.make_stride", - "cute.make_tile", - "cute.make_tiled_copy", - "cute.make_tiled_copy_v2", - "cute.make_tiled_mma", - "cute.make_tuple", - "cute.make_view", - "cute.memref.alloc_smem", - "cute.memref.alloca", - "cute.memref.load", - "cute.memref.load_vec", - "cute.memref.store", - "cute.memref.store_vec", - "cute.mma.make_fragment", - "cute.mma_atom_call", - "cute.prefetch", - "cute.prefetch_atom_call", - "cute.prepend_to_rank", - "cute.print", - "cute.print_tma_desc_im2col", - "cute.print_tma_desc_tiled", - "cute.print_view", - "cute.ptr.load", - "cute.ptr.store", - "cute.ptrtoint", - "cute.raked_product", - "cute.random_delay", - "cute.recast_iter", - "cute.recast_layout", - "cute.reduce", - "cute.right_inverse", - "cute.select", - "cute.shape_div", - "cute.size", - "cute.slice", - "cute.smem_partition_size", - "cute.static", - "cute.stencil_divide", - "cute.symbolic", - "cute.tile_to_shape", - "cute.tiled.copy.partition_D", - "cute.tiled.copy.partition_S", - "cute.tiled.copy.retile", - "cute.tiled.mma.partition", - "cute.tiled.mma.partition_shape", - "cute.tiled_divide", - "cute.tiled_product", - "cute.to_int_tuple", - "cute.tuple.product", - "cute.tuple.product_each", - "cute.tuple_add", - "cute.tuple_div", - "cute.tuple_mod", - "cute.tuple_mul", - "cute.tuple_sub", - "cute.unpack_tuple", - "cute.zipped_divide", - "cute.zipped_product", -}; - -fn testContext() !*mlir.Context { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - registerDialects(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - return ctx; -} - -test { - @setEvalBranchQuota(100_000); - std.testing.refAllDecls(@This()); - std.testing.refAllDecls(nvgpu); - std.testing.refAllDecls(nvvm); - std.testing.refAllDecls(cuda); - inline for (.{ nvvm, cuda }) |namespace| { - inline for (comptime std.meta.declarations(namespace)) |decl| { - const T = @field(namespace, decl); - if (@TypeOf(T) == type) std.testing.refAllDecls(T); - } - } - inline for (comptime std.meta.declarations(@This())) |decl| { - const T = @field(@This(), decl); - if (@TypeOf(T) == type) std.testing.refAllDecls(T); - } - inline for (comptime std.meta.declarations(nvgpu)) |decl| { - const T = @field(nvgpu, decl); - if (@TypeOf(T) == type) std.testing.refAllDecls(T); - } -} - -test "every bound operation is registered" { - const ctx = try testContext(); - defer ctx.deinit(); - for (cute.operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - for (nvgpu.operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - for (nvvm.operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - for (cuda.operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - try std.testing.expect(!ctx.allowUnregisteredDialects()); -} - -test "generated types and attributes rebuild from their getters" { - @setEvalBranchQuota(100_000); - const ctx = try testContext(); - defer ctx.deinit(); - inline for (.{ - .{ ArithTupleIteratorType, "!cute.arith_tuple_iter<\"(?,?{div=16})\">" }, - .{ ComposedLayoutType, "!cute.composed_layout<\"(4,5):(1,4) o 2 o (2,3):(1,2)\">" }, - .{ ConstrainedIntType, "!cute.i16" }, - .{ CoordTensorType, "!cute.coord_tensor<\"(?,?,?)\", \"((512,1),1):((1/2@0,0),0)\">" }, - .{ CoordType, "!cute.coord<\"1\">" }, - .{ FastDivmodDivisorType, "!cute.fast_divmod_divisor<32>" }, - .{ IntTupleType, "!cute.int_tuple<\"1\">" }, - .{ LayoutType, "!cute.layout<\"(2,3):(1,2)\">" }, - .{ MemRefType, "!cute.memref, S<2,4,3>, \"((8,16),(32,1),(1,3)):((32,256),(1,0),(0,4096))\">" }, - .{ PtrType, "!cute.ptr, S<2,4,3>>" }, - .{ ShapeType, "!cute.shape<\"(2,3)\">" }, - .{ SparseElemType, "!cute.sparse_elem<2, vector<1xf16>>" }, - .{ StrideType, "!cute.stride<\"1\">" }, - .{ SwizzleType, "!cute.swizzle<\"S<0,4,3>\">" }, - .{ TileType, "!cute.tile<\"[(2,3):(1,2)]\">" }, - .{ TiledCopyType, "!cute.tiled_copy, layout_copy_tv = <\"(32,4):(4,1)\">, tiler_mn = <\"[4:1;32:1]\">>" }, - .{ TiledCopyV2Type, "!cute.tiled_copy_v2, layout_copy_tv = <\"(1,8):(0,1)\">, tiler_mn = <\"[8:1]\">>" }, - .{ TiledMmaType, "!cute.tiled_mma f32 >, atom_layout_MNK = <\"(16,16,1):(16,1,0)\">, permutation_MNK = <\"[(16,4):(4,1);(16,4):(4,1);_]\">>" }, - .{ TupleType, "!cute.tuple" }, - .{ BitLayoutAttr, "#cute.bitlayout<<\"(8,4):(1,8)\">>" }, - .{ CopyAtomAttr, "#cute.copy_atom, layout_tv = (<\"(1,8):(0,1)\">, <\"(1,8):(0,1)\">, <\"(1,8):(0,1)\">)>" }, - .{ CopyAtomV2Attr, "#cute.copy_atom_v2, layout_tv = (<\"(1,8):(0,1)\">, <\"(1,8):(0,1)\">), frg = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, - .{ MmaAtomAttr, "#cute.mma_atom<<\"(16,8,16)\">, <\"32:1\">, layout_TV = (<\"1:0\">, <\"1:0\">, <\"1:0\">), shapes = (<\"(16,16)\">, <\"(8,16)\">, <\"(16,8)\">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, - .{ MxMmaAtomAttr, "#cute.mx_mma_atom<<\"(16,8,16)\">, <\"32:1\">, layout_TV = (<\"1:0\">, <\"1:0\">, <\"1:0\">, <\"1:0\">, <\"1:0\">), shapes = (<\"(16,16)\">, <\"(8,16)\">, <\"(16,8)\">, <\"1\">, <\"2\">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, - .{ SparseMmaAtomAttr, "#cute.sparse_mma_atom<<\"(16,8,16)\">, <\"32:1\">, layout_TV = (<\"1:0\">, <\"1:0\">, <\"1:0\">, <\"1:0\">), shapes = (<\"(16,16)\">, <\"(8,16)\">, <\"(16,8)\">, <\"1\">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, - .{ SparseMxMmaAtomAttr, "#cute.sparse_mx_mma_atom<<\"(16,8,16)\">, <\"32:1\">, layout_TV = (<\"1:0\">, <\"2:0\">, <\"3:0\">, <\"4:0\">, <\"5:0\">, <\"6:0\">), shapes = (<\"1\">, <\"2\">, <\"3\">, <\"4\">, <\"5\">, <\"6\">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, - .{ nvgpu.CopyAtomBulkCopyG2SType, "!cute_nvgpu.atom.bulk_copy_g2s" }, - .{ nvgpu.CopyAtomBulkCopyS2GType, "!cute_nvgpu.atom.bulk_copy_s2g" }, - .{ nvgpu.CopyAtomBulkCopyS2SType, "!cute_nvgpu.atom.bulk_copy_s2s" }, - .{ nvgpu.CopyAtomDsmemStoreType, "!cute_nvgpu.atom.dsmem_store" }, - .{ nvgpu.CopyAtomG2RType, "!cute_nvgpu.atom.g2r" }, - .{ nvgpu.CopyAtomIm2ColTmaLoadType, "!cute_nvgpu.atom.im2col_tma_load mcast = true tma_gbasis = <\"(64,128):(1@1,1@0)\">>" }, - .{ nvgpu.CopyAtomIm2ColTmaStoreType, "!cute_nvgpu.atom.im2col_tma_store tma_gbasis = <\"(32,128):(1@1,1@0)\">>" }, - .{ nvgpu.CopyAtomLdsmType, "!cute_nvgpu.atom.ldsm, sz_pattern = u16, num_matrices = 4, t>" }, - .{ nvgpu.CopyAtomNonExec2DGather4TmaLoadType, "!cute_nvgpu.atom.non_exec_2d_gather4_tma_load, tma_format = F16_RN>" }, - .{ nvgpu.CopyAtomNonExec2DScatter4TmaStoreType, "!cute_nvgpu.atom.non_exec_2d_scatter4_tma_store, tma_format = F16_RN>" }, - .{ nvgpu.CopyAtomNonExecIm2ColTmaLoadType, "!cute_nvgpu.atom.non_exec_im2col_tma_load, tma_format = F16_RN>" }, - .{ nvgpu.CopyAtomNonExecIm2ColTmaStoreType, "!cute_nvgpu.atom.non_exec_im2col_tma_store, tma_format = F16_RN>" }, - .{ nvgpu.CopyAtomNonExecTiledTmaLoadType, "!cute_nvgpu.atom.non_exec_tiled_tma_load, tma_format = BF16_RN>" }, - .{ nvgpu.CopyAtomNonExecTiledTmaReduceType, "!cute_nvgpu.atom.non_exec_tiled_tma_reduce, tma_format = F32_RN>" }, - .{ nvgpu.CopyAtomNonExecTiledTmaStoreType, "!cute_nvgpu.atom.non_exec_tiled_tma_store, tma_format = BF16_RN>" }, - .{ nvgpu.CopyAtomR2GType, "!cute_nvgpu.atom.r2g" }, - .{ nvgpu.CopyAtomR2SType, "!cute_nvgpu.atom.r2s" }, - .{ nvgpu.CopyAtomS2RType, "!cute_nvgpu.atom.s2r" }, - .{ nvgpu.CopyAtomSIMTAsyncCopyType, "!cute_nvgpu.atom.simt_async_copy" }, - .{ nvgpu.CopyAtomSIMTMultimemLdReduceType, "!cute_nvgpu.atom.simt_multimem_ld_reduce, acquire, gpu>" }, - .{ nvgpu.CopyAtomSIMTMultimemRedType, "!cute_nvgpu.atom.simt_multimem_red" }, - .{ nvgpu.CopyAtomSIMTMultimemStType, "!cute_nvgpu.atom.simt_multimem_st" }, - .{ nvgpu.CopyAtomSIMTSyncCopyType, "!cute_nvgpu.atom.universal_copy" }, - .{ nvgpu.CopyAtomSM100CopyS2TType, "!cute_nvgpu.atom.s2t_copy" }, - .{ nvgpu.CopyAtomSM100S2TCopyV2Type, "!cute_nvgpu.atom.sm100_s2t_copy_v2" }, - .{ nvgpu.CopyAtomSM100TmemLoadType, "!cute_nvgpu.atom.tmem_load" }, - .{ nvgpu.CopyAtomSM100TmemStoreType, "!cute_nvgpu.atom.tmem_store" }, - .{ nvgpu.CopyAtomSM107TmemLoadSPCompressType, "!cute_nvgpu.atom.tmem_load_spcompress" }, - .{ nvgpu.CopyAtomSM10xTmemLoadRedType, "!cute_nvgpu.atom.tmem_load_red" }, - .{ nvgpu.CopyAtomStsmType, "!cute_nvgpu.atom.stsm, num_matrices = 4, n>" }, - .{ nvgpu.CopyAtomTmaLoadType, "!cute_nvgpu.atom.tma_load mcast = true tma_gbasis = <\"(64,1):(1@0,1@1)\">>" }, - .{ nvgpu.CopyAtomTmaReduceType, "!cute_nvgpu.atom.tma_reduce tma_gbasis = <\"(32,128):(1@1,1@0)\">>" }, - .{ nvgpu.CopyAtomTmaStoreType, "!cute_nvgpu.atom.tma_store tma_gbasis = <\"(64,16):(1@0,1@1)\">>" }, - .{ nvgpu.MmaAtomSM100UMMABlockScaledSparseType, "!cute_nvgpu.sm100.mma_bs_sp<128x128x64, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, sparse_metadata_format = tid, frag_kind = ss, vec_size = 32>" }, - .{ nvgpu.MmaAtomSM100UMMABlockScaledType, "!cute_nvgpu.sm100.mma_bs<128x128x96, num_cta = 1, ab_major = (k, k), elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E8M0FNU, frag_kind = ss, vec_size = 16, arch_promote = sm_103>" }, - .{ nvgpu.MmaAtomSM100UMMASparseType, "!cute_nvgpu.sm100.mma_sp<128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f16, f16, f32), e_type = i8, sparse_metadata_format = tid, frag_kind = ss, c_scale_exp = 0>" }, - .{ nvgpu.MmaAtomSM100UMMAType, "!cute_nvgpu.sm100.mma<128x128x16, num_cta = 1, ab_major = (k, mn), elem_type = (f16, f16, f32), frag_kind = ss, c_scale_exp = 0>" }, - .{ nvgpu.MmaAtomSM107UMMABlockScaledSparseType, "!cute_nvgpu.sm107.mma_bs_sp<128x128x64, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, sparse_metadata_format = tid, frag_kind = ss, vec_size = 32, ab_collector_op = (discard, discard)>" }, - .{ nvgpu.MmaAtomSM107UMMABlockScaledType, "!cute_nvgpu.sm107.mma_bs<128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, frag_kind = ss, vec_size = 32, ab_collector_op = (discard, use), int_overflow = >" }, - .{ nvgpu.MmaAtomSM107UMMASparseType, "!cute_nvgpu.sm107.mma_sp<128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f16, f16, f32), e_type = i8, sparse_metadata_format = tid, frag_kind = ss, c_scale_exp = 0, ab_collector_op = (discard, discard)>" }, - .{ nvgpu.MmaAtomSM107UMMAType, "!cute_nvgpu.sm107.mma<128x128x16, num_cta = 1, ab_major = (k, mn), elem_type = (f16, f16, f32), frag_kind = ss, c_scale_exp = 0, ab_collector_op = (lastuse, fill)>" }, - .{ nvgpu.MmaAtomSM120BlockScaledType, "!cute_nvgpu.SM120.mma_bs<16x8x64, vec_size = 32, elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E8M0FNU, use_sf_layout_TV = true>" }, - .{ nvgpu.MmaAtomSM80SparseType, "!cute_nvgpu.sm80.sparse_mma<16x8x64, elem_type = (i8, i8, i32), md_format = tid, int_overflow = >" }, - .{ nvgpu.MmaAtomSM80Type, "!cute_nvgpu.sm80.mma<16x8x32, elem_type = (i8, i8, i32), int_overflow = , binary_op = >" }, - .{ nvgpu.MmaAtomSM89Type, "!cute_nvgpu.sm89.mma<16x8x32, elem_type = (f8E4M3FN, f8E4M3FN, f32) >" }, - .{ nvgpu.MmaAtomSM90Type, "!cute_nvgpu.sm90.mma<64x128x32, ab_major = (k, k), elem_type = (i8, i8, i32), frag_kind = ss, int_overflow = , a_neg, b_neg>" }, - .{ nvgpu.SmemDescCircularSM103Type, "!cute_nvgpu.sm103.smem_desc_circular<<\"(8,128,2):(128,1,1024)\">>" }, - .{ nvgpu.SmemDescCircularSM107Type, "!cute_nvgpu.sm107.smem_desc_circular<<\"(8,128,2):(128,1,1024)\">>" }, - .{ nvgpu.SmemDescSM107Type, "!cute_nvgpu.sm107.smem_desc" }, - .{ nvgpu.SmemDescType, "!cute_nvgpu.smem_desc" }, - .{ nvgpu.SmemDescViewType, "!cute_nvgpu.smem_desc_view" }, - .{ nvgpu.TmaDescriptorIm2ColType, "!cute_nvgpu.tma_descriptor_im2col" }, - .{ nvgpu.TmaDescriptorTiledType, "!cute_nvgpu.tma_descriptor_tiled" }, - .{ nvgpu.UniversalFmaAtomType, "!cute_nvgpu.atom.universal_fma<1x1x1, (f32, f32) -> f32 >" }, - .{ nvgpu.WorkIdResponseType, "!cute_nvgpu.workid_response" }, - .{ nvgpu.NotImplementedFrgAttr, "#cute_nvgpu.not_implemented_frg<>" }, - .{ nvgpu.RmemFrgAttr, "#cute_nvgpu.rmem_frg" }, - .{ nvgpu.SM100CircularSmemFrgAttr, "#cute_nvgpu.arch.sm100.circular_smem_frg" }, - .{ nvgpu.SM100SmemFrgAttr, "#cute_nvgpu.arch.sm100.smem_frg" }, - .{ nvgpu.SM100TmemEFrgAttr, "#cute_nvgpu.arch.sm100.tmem_e_frg" }, - .{ nvgpu.SM100TmemFrgAttr, "#cute_nvgpu.arch.sm100.tmem_frg" }, - .{ nvgpu.SM100TmemSfFrgAttr, "#cute_nvgpu.arch.sm100.tmem_sf_frg" }, - .{ nvgpu.SM107SmemFrgAttr, "#cute_nvgpu.arch.sm107.smem_frg" }, - .{ nvgpu.SM90SmemFrgAttr, "#cute_nvgpu.arch.sm90.smem_frg" }, - }) |example| { - const T = example[0]; - const is_type = @hasDecl(T, "type_"); - const parsed = if (is_type) try mlir.Type.parse(ctx, example[1]) else try mlir.Attribute.parse(ctx, example[1]); - const value = parsed.isA(T) orelse return error.TestUnexpectedResult; - try std.testing.expect((try rebuild(T, ctx, value)).eql(value)); - var text: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer text.deinit(); - try text.writer.print("{f}", .{value}); - try std.testing.expect(parsed.eql(if (is_type) try mlir.Type.parse(ctx, text.written()) else try mlir.Attribute.parse(ctx, text.written()))); - } - // Every case of every enum attribute. - inline for (.{ cute, nvgpu }) |namespace| { - inline for (comptime std.meta.declarations(namespace)) |decl| { - const T = @field(namespace, decl); - if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque" and @hasDecl(T, "InitArgs")) { - const fields = @typeInfo(T.InitArgs).@"struct".field_names; - if (fields.len == 1 and comptime std.mem.eql(u8, fields[0], "value")) { - inline for (@typeInfo(@FieldType(T.InitArgs, fields[0])).@"enum".field_names) |case| { - const case_value = @field(@FieldType(T.InitArgs, fields[0]), case); - const attr = try T.get(ctx, .{ .value = case_value }); - try std.testing.expectEqual(@backingInt(case_value), @backingInt(attr.getValue())); - try std.testing.expect((try rebuild(T, ctx, attr)).eql(attr)); - } - } - } - } - } - try std.testing.expect((try nvgpu.SmemDescType.get(ctx, .{})).type_().isA(nvgpu.NVGPUType) != null); -} - -test "nvvm and cuda types and attributes print as the compiler does" { - @setEvalBranchQuota(100_000); - const ctx = try testContext(); - defer ctx.deinit(); - // The compiler's printed forms, which these must print byte for byte. - inline for (.{ - .{ nvvm.MMAShapeAttr, "#nvvm.shape" }, - .{ nvvm.LdStMatrixShapeAttr, "#nvvm.ld_st_matrix_shape" }, - .{ nvvm.TargetAttr, "#nvvm.target" }, - .{ nvvm.TargetAttr, "#nvvm.target" }, - .{ nvvm.TargetAttr, "#nvvm.target" }, - .{ cuda.AssumeKernelAttr, "#cuda.assume_kernel_attr" }, - .{ cuda.ComputeTargetAttr, "#cuda.compute_target" }, - .{ cuda.DevMaxSharedMemoryOptinAttr, "#cuda.dev_max_shared_memory_optin" }, - .{ cuda.ExecutableAttr, "#cuda.executable" }, - .{ cuda.LaunchConfigType, "!cuda.launch_cfg" }, - .{ cuda.StreamType, "!cuda.stream" }, - .{ cuda.ResultType, "!cuda.result" }, - .{ cuda.IntType, "!cuda.int" }, - .{ cuda.RuntimeDim3Type, "!cuda.runtime.dim_3" }, - .{ cuda.RuntimeLaunchAttributeValueSyncPolicyType, "!cuda.runtime.launch_attribute_value.sync_policy" }, - .{ cuda.RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType, "!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference" }, - .{ cuda.RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType, "!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling" }, - .{ cuda.RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType, "!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed" }, - }) |example| { - const T = example[0]; - const is_type = @hasDecl(T, "type_"); - const parsed = if (is_type) try mlir.Type.parse(ctx, example[1]) else try mlir.Attribute.parse(ctx, example[1]); - const value = parsed.isA(T) orelse return error.TestUnexpectedResult; - try std.testing.expect((try rebuildEnums(T, ctx, value)).eql(value)); - var buf: [512]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{value}); - try std.testing.expectEqualStrings(example[1], w.buffered()); - } - // The compiler prints these without the space before the dictionary; it - // parses both. - inline for (.{ - .{ cuda.DeviceAttributesAttr, "#cuda" }, - .{ cuda.FuncAttributesAttr, "#cuda" }, - }) |example| { - const parsed = try mlir.Attribute.parse(ctx, example[1]); - const value = parsed.isA(example[0]) orelse return error.TestUnexpectedResult; - try std.testing.expect((try rebuildEnums(example[0], ctx, value)).eql(value)); - } -} - -test "nvvm target rejects what the compiler rejects" { - const ctx = try testContext(); - defer ctx.deinit(); - try std.testing.expectError(error.InvalidMlir, nvvm.TargetAttr.get(ctx, .{ .O = 4 })); - try std.testing.expectError(error.InvalidMlir, nvvm.TargetAttr.get(ctx, .{ .chip = "" })); - const not_strings = try mlir.Attribute.parse(ctx, "[\"a\", 2]"); - try std.testing.expectError(error.InvalidMlir, nvvm.TargetAttr.get(ctx, .{ .link = not_strings })); -} - -/// `rebuild` for parameters that are slices of enums. -fn rebuildEnums(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T { - var args: T.InitArgs = undefined; - var buffers: [4][16]u32 = undefined; - inline for (@typeInfo(T.InitArgs).@"struct".field_names, 0..) |field, i| { - const name = [_]u8{comptime std.ascii.toUpper(field[0])} ++ field[1..]; - const info = @typeInfo(@FieldType(T.InitArgs, field)); - if (info == .pointer and info.pointer.size == .slice and @typeInfo(info.pointer.child) == .@"enum") { - const E = info.pointer.child; - const count = @field(T, "getNum" ++ name)(value); - const elements: []E = @ptrCast(buffers[i][0..count]); - for (elements, 0..) |*element, pos| element.* = @field(T, "get" ++ name[0 .. name.len - 1])(value, pos); - @field(args, field) = elements; - } else { - @field(args, field) = @field(T, "get" ++ name)(value); - } - } - return T.get(ctx, args); -} - -fn rebuild(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T { - var args: T.InitArgs = undefined; - var elements: [8]*const mlir.Type = undefined; - inline for (@typeInfo(T.InitArgs).@"struct".field_names) |field| { - const name = [_]u8{comptime std.ascii.toUpper(field[0])} ++ field[1..]; - if (@FieldType(T.InitArgs, field) == []const *const mlir.Type) { - const count = @field(T, "getNum" ++ name)(value); - for (elements[0..count], 0..) |*element, i| element.* = @field(T, "get" ++ name[0 .. name.len - 1])(value, i); - @field(args, field) = elements[0..count]; - } else { - @field(args, field) = @field(T, "get" ++ name)(value); - } - } - return T.get(ctx, args); -} diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/BUILD.bazel b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/BUILD.bazel deleted file mode 100644 index e30f2462..00000000 --- a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/BUILD.bazel +++ /dev/null @@ -1,47 +0,0 @@ -load("@rules_cc//cc:defs.bzl", "cc_library") - -package(default_visibility = ["//visibility:public"]) - -cc_library( - name = "cute", - hdrs = [ - "Cute.h", - "CuteAttributes.h", - "CuteCompilerAttributes.h", - "CuteTypes.h", - ], - includes = ["../.."], - deps = ["@llvm-project//mlir:CAPIIR"], -) - -cc_library( - name = "cute_nvgpu", - hdrs = [ - "CuteNVGPU.h", - "CuteNVGPUAttributes.h", - "CuteNVGPUTypes.h", - ], - includes = ["../.."], - deps = ["@llvm-project//mlir:CAPIIR"], -) - -cc_library( - name = "cuda", - hdrs = [ - "Cuda.h", - "CudaAttributes.h", - "CudaTypes.h", - ], - includes = ["../.."], - deps = ["@llvm-project//mlir:CAPIIR"], -) - -cc_library( - name = "nvvm", - hdrs = [ - "NVVM.h", - "NVVMAttributes.h", - ], - includes = ["../.."], - deps = ["@llvm-project//mlir:CAPIIR"], -) diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/Cuda.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/Cuda.h deleted file mode 100644 index fbe099a4..00000000 --- a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/Cuda.h +++ /dev/null @@ -1,22 +0,0 @@ -//===- Cuda.h - C API for NVIDIA's CUDA host dialect of the CuTe DSL -*- C -*-===// -// -// The dialect handle is `cuda`. - -#ifndef CUTE_IR_C_DIALECT_CUDA_H -#define CUTE_IR_C_DIALECT_CUDA_H - -#include "cute_ir-c/Dialect/CudaAttributes.h" -#include "cute_ir-c/Dialect/CudaTypes.h" -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Cuda, cuda); - -#ifdef __cplusplus -} -#endif - -#endif // CUTE_IR_C_DIALECT_CUDA_H diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CudaAttributes.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CudaAttributes.h deleted file mode 100644 index b0d55d2e..00000000 --- a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CudaAttributes.h +++ /dev/null @@ -1,52 +0,0 @@ -// C API for the cuda attributes, generated from the dialect's .td files. -#ifndef CUTE_IR_C_DIALECT_CUDA_ATTRIBUTES_H -#define CUTE_IR_C_DIALECT_CUDA_ATTRIBUTES_H - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Constructors return null for invalid parameters or handles of another -// context; optional parameters take null handles. Enums are their integer -// values (the dialect's *Enums.td). Getters require a value accepted by the -// matching IsA. - -// `#cuda.assume_kernel_attr`: Whether `cuda.launch` may assume its callee is a kernel -MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaAssumeKernel(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCudaAssumeKernelAttrGet(MlirContext context, bool value); -MLIR_CAPI_EXPORTED bool mlirCudaAssumeKernelAttrGetValue(MlirAttribute attr); - -// `#cuda.compute_target`: Representation, portability and architectures of a compiled executable -MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaComputeTarget(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCudaComputeTargetAttrGet(MlirContext context, uint32_t representation, uint32_t portability, intptr_t numArchs, const uint32_t *archs); -MLIR_CAPI_EXPORTED uint32_t mlirCudaComputeTargetAttrGetRepresentation(MlirAttribute attr); -MLIR_CAPI_EXPORTED uint32_t mlirCudaComputeTargetAttrGetPortability(MlirAttribute attr); -MLIR_CAPI_EXPORTED intptr_t mlirCudaComputeTargetAttrGetNumArchs(MlirAttribute attr); -MLIR_CAPI_EXPORTED uint32_t mlirCudaComputeTargetAttrGetArchs(MlirAttribute attr, intptr_t pos); - -// `#cuda.dev_max_shared_memory_optin`: The device's opt-in maximum of shared memory per block -MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaDevMaxSharedMemoryOptin(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCudaDevMaxSharedMemoryOptinAttrGet(MlirContext context); - -// `#cuda.device_attributes`: CUDA device attributes -MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaDeviceAttributes(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCudaDeviceAttributesAttrGet(MlirContext context, MlirAttribute values); -MLIR_CAPI_EXPORTED MlirAttribute mlirCudaDeviceAttributesAttrGetValues(MlirAttribute attr); - -// `#cuda.executable`: Format of a compiled executable -MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaExecutable(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCudaExecutableAttrGet(MlirContext context, uint32_t format); -MLIR_CAPI_EXPORTED uint32_t mlirCudaExecutableAttrGetFormat(MlirAttribute attr); - -// `#cuda.func_attributes`: CUDA function attributes of a kernel -MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaFuncAttributes(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCudaFuncAttributesAttrGet(MlirContext context, MlirAttribute values); -MLIR_CAPI_EXPORTED MlirAttribute mlirCudaFuncAttributesAttrGetValues(MlirAttribute attr); - -#ifdef __cplusplus -} -#endif - -#endif // CUTE_IR_C_DIALECT_CUDA_ATTRIBUTES_H diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CudaTypes.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CudaTypes.h deleted file mode 100644 index c788e0f8..00000000 --- a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CudaTypes.h +++ /dev/null @@ -1,149 +0,0 @@ -// C API for the cuda types, generated from the dialect's .td files. -#ifndef CUTE_IR_C_DIALECT_CUDA_TYPES_H -#define CUTE_IR_C_DIALECT_CUDA_TYPES_H - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Constructors return null for invalid parameters or handles of another -// context; optional parameters take null handles. Enums are their integer -// values (the dialect's *Enums.td). Getters require a value accepted by the -// matching IsA. - -// `!cuda.arch_set`: `!cuda.arch_set` -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaArchSet(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaArchSetTypeGet(MlirContext context); - -// `!cuda.event`: `Event` represents CUDA event -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaEvent(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaEventTypeGet(MlirContext context); - -// `!cuda.graph_device_node`: `GraphDeviceNode` represents CUDA device node handle for device-side node update -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaGraphDeviceNode(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaGraphDeviceNodeTypeGet(MlirContext context); - -// `!cuda.graph_exec`: CUDA Graph Execution -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaGraphExec(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaGraphExecTypeGet(MlirContext context); - -// `!cuda.graph_node`: CUDA Graph Node handle -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaGraphNode(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaGraphNodeTypeGet(MlirContext context); - -// `!cuda.graph`: CUDA Graph -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaGraph(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaGraphTypeGet(MlirContext context); - -// `!cuda.int`: `cuda.int` is a integer type with a bitwidth matching the underlying ABI integer bitwidth -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaInt(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaIntTypeGet(MlirContext context); - -// `!cuda.kernel`: CUDA Kernel handle -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaKernel(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaKernelTypeGet(MlirContext context); - -// `!cuda.launch_cfg`: Representation of CUDA's extensible launch configuration mirroring cuda runtime API -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaLaunchConfig(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaLaunchConfigTypeGet(MlirContext context, uint32_t maxNumAttrs); -MLIR_CAPI_EXPORTED uint32_t mlirCudaLaunchConfigTypeGetMaxNumAttrs(MlirType type); - -// `!cuda.library`: CUDA Library handle -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaLibrary(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaLibraryTypeGet(MlirContext context); - -// `!cuda.module`: CUDA Module handle -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaModule(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaModuleTypeGet(MlirContext context); - -// `!cuda.result`: CUDA Result type -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaResult(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaResultTypeGet(MlirContext context); - -// `!cuda.runtime.device_prop`: CUDA cudaDeviceProp struct type (value-typed). -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeDeviceProp(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeDevicePropTypeGet(MlirContext context); - -// `!cuda.runtime.dim_3`: CUDA dim3 struct type (value-typed). -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeDim3(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeDim3TypeGet(MlirContext context); - -// `!cuda.runtime.extent`: CUDA cudaExtent struct type (value-typed). -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeExtent(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeExtentTypeGet(MlirContext context); - -// `!cuda.runtime.kernel_node_params`: CUDA cudaKernelNodeParams struct type (value-typed). -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeKernelNodeParams(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeKernelNodeParamsTypeGet(MlirContext context); - -// `!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference`: `!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference` -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreference(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceTypeGet(MlirContext context); - -// `!cuda.runtime.launch_attribute_value.cooperative`: `!cuda.runtime.launch_attribute_value.cooperative` -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeLaunchAttributeValueCooperative(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeLaunchAttributeValueCooperativeTypeGet(MlirContext context); - -// `!cuda.runtime.launch_attribute_value.mem_sync_domain`: `!cuda.runtime.launch_attribute_value.mem_sync_domain` -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeLaunchAttributeValueMemSyncDomain(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeLaunchAttributeValueMemSyncDomainTypeGet(MlirContext context); - -// `!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling`: `!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling` -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeLaunchAttributeValueNvlinkUtilCentricScheduling(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingTypeGet(MlirContext context); - -// `!cuda.runtime.launch_attribute_value.portable_cluster_size_mode`: `!cuda.runtime.launch_attribute_value.portable_cluster_size_mode` -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeLaunchAttributeValuePortableClusterSizeMode(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeLaunchAttributeValuePortableClusterSizeModeTypeGet(MlirContext context); - -// `!cuda.runtime.launch_attribute_value.priority`: `!cuda.runtime.launch_attribute_value.priority` -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeLaunchAttributeValuePriority(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeLaunchAttributeValuePriorityTypeGet(MlirContext context); - -// `!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed`: `!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed` -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowed(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedTypeGet(MlirContext context); - -// `!cuda.runtime.launch_attribute_value.shared_mem_carveout`: `!cuda.runtime.launch_attribute_value.shared_mem_carveout` -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemCarveout(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeLaunchAttributeValueSharedMemCarveoutTypeGet(MlirContext context); - -// `!cuda.runtime.launch_attribute_value.shared_memory_mode`: `!cuda.runtime.launch_attribute_value.shared_memory_mode` -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemoryMode(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeLaunchAttributeValueSharedMemoryModeTypeGet(MlirContext context); - -// `!cuda.runtime.launch_attribute_value.sync_policy`: `!cuda.runtime.launch_attribute_value.sync_policy` -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeLaunchAttributeValueSyncPolicy(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeLaunchAttributeValueSyncPolicyTypeGet(MlirContext context); - -// `!cuda.runtime.memcpy_3d_params`: CUDA cudaMemcpy3DParms struct type (value-typed). -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeMemcpy3DParams(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeMemcpy3DParamsTypeGet(MlirContext context); - -// `!cuda.runtime.pitched_ptr`: CUDA cudaPitchedPtr struct type (value-typed). -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimePitchedPtr(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimePitchedPtrTypeGet(MlirContext context); - -// `!cuda.runtime.pos`: CUDA cudaPos struct type (value-typed). -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimePos(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimePosTypeGet(MlirContext context); - -// `!cuda.runtime.uuid`: CUDA cudaUUID_t struct type (value-typed). -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeUuid(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeUuidTypeGet(MlirContext context); - -// `!cuda.stream`: CUDA Stream -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaStream(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaStreamTypeGet(MlirContext context); - -// `!cuda.tensor_map`: CUDA Tensor Map type -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaTensorMap(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaTensorMapTypeGet(MlirContext context); - -#ifdef __cplusplus -} -#endif - -#endif // CUTE_IR_C_DIALECT_CUDA_TYPES_H diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/Cute.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/Cute.h deleted file mode 100644 index dd896824..00000000 --- a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/Cute.h +++ /dev/null @@ -1,24 +0,0 @@ -//===- Cute.h - C API for the CuTe dialect ----------------------*- C -*-===// - -#ifndef CUTE_IR_C_DIALECT_CUTE_H -#define CUTE_IR_C_DIALECT_CUTE_H - -#include "cute_ir-c/Dialect/CuteAttributes.h" -#include "cute_ir-c/Dialect/CuteCompilerAttributes.h" -#include "cute_ir-c/Dialect/CuteTypes.h" -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Cute, cute); - -/// Returns true when `type` belongs to the CuTe dialect. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteType(MlirType type); - -#ifdef __cplusplus -} -#endif - -#endif // CUTE_IR_C_DIALECT_CUTE_H diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteAttributes.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteAttributes.h deleted file mode 100644 index 6aecc8cb..00000000 --- a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteAttributes.h +++ /dev/null @@ -1,65 +0,0 @@ -// C API for CuTe algebra attributes. - -#ifndef CUTE_IR_C_DIALECT_CUTE_ATTRIBUTES_H -#define CUTE_IR_C_DIALECT_CUTE_ATTRIBUTES_H - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Constructors accept raw algebra expressions, not MLIR attribute assembly. -// For example, LayoutAttrGet accepts "(16,32):(32,1)". Invalid expressions -// return a null attribute. GetValue returns canonical algebra text owned by -// the attribute's context; callers must not free it. Getters require an -// attribute accepted by the corresponding IsA predicate. - -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteIntTupleAttrGet(MlirContext context, - MlirStringRef value); -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteIntTuple(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef -mlirCuteIntTupleAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCoordAttrGet(MlirContext context, - MlirStringRef value); -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteCoord(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef mlirCuteCoordAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteShapeAttrGet(MlirContext context, - MlirStringRef value); -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteShape(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef mlirCuteShapeAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteStrideAttrGet(MlirContext context, - MlirStringRef value); -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteStride(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef mlirCuteStrideAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteLayoutAttrGet(MlirContext context, - MlirStringRef value); -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteLayout(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef mlirCuteLayoutAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteTileAttrGet(MlirContext context, - MlirStringRef value); -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteTile(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef mlirCuteTileAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteComposedLayoutAttrGet(MlirContext context, MlirStringRef value); -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteComposedLayout(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef -mlirCuteComposedLayoutAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteSwizzleAttrGet(MlirContext context, - MlirStringRef value); -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteSwizzle(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef -mlirCuteSwizzleAttrGetValue(MlirAttribute attr); - -#ifdef __cplusplus -} -#endif - -#endif // CUTE_IR_C_DIALECT_CUTE_ATTRIBUTES_H diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteCompilerAttributes.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteCompilerAttributes.h deleted file mode 100644 index f3b4375a..00000000 --- a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteCompilerAttributes.h +++ /dev/null @@ -1,225 +0,0 @@ -// C API for the CuTe attributes beyond the algebra, generated from the -// dialect's .td files. -#ifndef CUTE_IR_C_DIALECT_CUTE_COMPILER_ATTRIBUTES_H -#define CUTE_IR_C_DIALECT_CUTE_COMPILER_ATTRIBUTES_H - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Constructors return null for invalid parameters or handles of another -// context; optional parameters take null handles, and std::optional enums a -// negative value. Enums are their integer values (the dialect's *Enums.td). -// Getters require a value accepted by the matching IsA. - -// `#cute.bitlayout`: Bit layout of a pointer's elements within a storage chunk. -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteBitLayout(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteBitLayoutAttrGet(MlirContext context, - MlirAttribute layout); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteBitLayoutAttrGetLayout(MlirAttribute attr); - -// `#cute.copy_atom`: Thread and value layouts of a copy atom. -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteCopyAtom(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCopyAtomAttrGet( - MlirContext context, MlirAttribute thrId, MlirAttribute layoutSrc, - MlirAttribute layoutDst, MlirAttribute layoutRef); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteCopyAtomAttrGetThrId(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteCopyAtomAttrGetLayoutSrc(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteCopyAtomAttrGetLayoutDst(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteCopyAtomAttrGetLayoutRef(MlirAttribute attr); - -// `#cute.copy_atom_v2`: Thread and value layouts and fragments of a V2 copy -// atom. -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteCopyAtomV2(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCopyAtomV2AttrGet( - MlirContext context, MlirAttribute thrId, MlirAttribute layoutSrcTV, - MlirAttribute layoutDstTV, MlirAttribute frgSrc, MlirAttribute frgDst); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteCopyAtomV2AttrGetThrId(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteCopyAtomV2AttrGetLayoutSrcTV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteCopyAtomV2AttrGetLayoutDstTV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteCopyAtomV2AttrGetFrgSrc(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteCopyAtomV2AttrGetFrgDst(MlirAttribute attr); - -// `#cute.mma_atom`: Shapes, thread-value layouts and fragments of an MMA atom. -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteMmaAtom(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMmaAtomAttrGet( - MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, - MlirAttribute layoutATV, MlirAttribute layoutBTV, MlirAttribute layoutCTV, - MlirAttribute shapeAMK, MlirAttribute shapeBNK, MlirAttribute shapeCMN, - MlirAttribute frgA, MlirAttribute frgB, MlirAttribute frgC); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMmaAtomAttrGetShapeMNK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMmaAtomAttrGetThrId(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMmaAtomAttrGetLayoutATV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMmaAtomAttrGetLayoutBTV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMmaAtomAttrGetLayoutCTV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMmaAtomAttrGetShapeAMK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMmaAtomAttrGetShapeBNK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMmaAtomAttrGetShapeCMN(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMmaAtomAttrGetFrgA(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMmaAtomAttrGetFrgB(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMmaAtomAttrGetFrgC(MlirAttribute attr); - -// `#cute.mx_mma_atom`: MMA atom attribute with scale-factor operands. -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteMxMmaAtom(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMxMmaAtomAttrGet( - MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, - MlirAttribute layoutATV, MlirAttribute layoutSFATV, MlirAttribute layoutBTV, - MlirAttribute layoutSFBTV, MlirAttribute layoutCTV, MlirAttribute shapeAMK, - MlirAttribute shapeSFAMK, MlirAttribute shapeBNK, MlirAttribute shapeSFBNK, - MlirAttribute shapeCMN, MlirAttribute frgA, MlirAttribute frgSFA, - MlirAttribute frgB, MlirAttribute frgSFB, MlirAttribute frgC); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetShapeMNK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetThrId(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetLayoutATV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetLayoutSFATV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetLayoutBTV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetLayoutSFBTV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetLayoutCTV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetShapeAMK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetShapeSFAMK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetShapeBNK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetShapeSFBNK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetShapeCMN(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetFrgA(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetFrgSFA(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetFrgB(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetFrgSFB(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetFrgC(MlirAttribute attr); - -// `#cute.reduction_op`: Op for cute reduce operations. -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteReductionOp(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteReductionOpAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteReductionOpAttrGetValue(MlirAttribute attr); - -// `#cute.sparse_mma_atom`: MMA atom attribute with a sparsity-metadata operand. -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteSparseMmaAtom(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteSparseMmaAtomAttrGet( - MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, - MlirAttribute layoutATV, MlirAttribute layoutBTV, MlirAttribute layoutCTV, - MlirAttribute layoutETV, MlirAttribute shapeAMK, MlirAttribute shapeBNK, - MlirAttribute shapeCMN, MlirAttribute shapeEMK, MlirAttribute frgA, - MlirAttribute frgB, MlirAttribute frgC, MlirAttribute frgE); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetShapeMNK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetThrId(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetLayoutATV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetLayoutBTV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetLayoutCTV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetLayoutETV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetShapeAMK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetShapeBNK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetShapeCMN(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetShapeEMK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetFrgA(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetFrgB(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetFrgC(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetFrgE(MlirAttribute attr); - -// `#cute.sparse_mx_mma_atom`: MMA atom attribute with scale-factor and metadata -// operands. -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteSparseMxMmaAtom(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteSparseMxMmaAtomAttrGet( - MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, - MlirAttribute layoutATV, MlirAttribute layoutSFATV, MlirAttribute layoutBTV, - MlirAttribute layoutSFBTV, MlirAttribute layoutCTV, MlirAttribute layoutETV, - MlirAttribute shapeAMK, MlirAttribute shapeSFAMK, MlirAttribute shapeBNK, - MlirAttribute shapeSFBNK, MlirAttribute shapeCMN, MlirAttribute shapeEMK, - MlirAttribute frgA, MlirAttribute frgSFA, MlirAttribute frgB, - MlirAttribute frgSFB, MlirAttribute frgC, MlirAttribute frgE); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetShapeMNK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetThrId(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetLayoutATV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetLayoutSFATV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetLayoutBTV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetLayoutSFBTV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetLayoutCTV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetLayoutETV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetShapeAMK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetShapeSFAMK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetShapeBNK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetShapeSFBNK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetShapeCMN(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetShapeEMK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetFrgA(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetFrgSFA(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetFrgB(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetFrgSFB(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetFrgC(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetFrgE(MlirAttribute attr); - -#ifdef __cplusplus -} -#endif - -#endif // CUTE_IR_C_DIALECT_CUTE_COMPILER_ATTRIBUTES_H diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPU.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPU.h deleted file mode 100644 index 175435f5..00000000 --- a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPU.h +++ /dev/null @@ -1,22 +0,0 @@ -//===- CuteNVGPU.h - C API for the CuTe NVIDIA GPU dialect ------*- C -*-===// - -#ifndef CUTE_IR_C_DIALECT_CUTENVGPU_H -#define CUTE_IR_C_DIALECT_CUTENVGPU_H - -#include "cute_ir-c/Dialect/CuteNVGPUAttributes.h" -#include "cute_ir-c/Dialect/CuteNVGPUTypes.h" -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(CuteNVGPU, cute_nvgpu); - -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUType(MlirType type); - -#ifdef __cplusplus -} -#endif - -#endif // CUTE_IR_C_DIALECT_CUTENVGPU_H diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPUAttributes.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPUAttributes.h deleted file mode 100644 index c382d546..00000000 --- a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPUAttributes.h +++ /dev/null @@ -1,480 +0,0 @@ -// C API for the CuTe NVIDIA GPU attributes, generated from the dialect's .td -// files. -#ifndef CUTE_IR_C_DIALECT_CUTENVGPU_ATTRIBUTES_H -#define CUTE_IR_C_DIALECT_CUTENVGPU_ATTRIBUTES_H - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Constructors return null for invalid parameters or handles of another -// context; optional parameters take null handles, and std::optional enums a -// negative value. Enums are their integer values (the dialect's *Enums.td). -// Getters require a value accepted by the matching IsA. - -// `#cute_nvgpu.atom_copy_field_bulkg2s`: Fields stored in Bulk Load Copy Atom -// type. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyG2S(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGet( - MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_bulks2g`: Fields stored in Bulk Store Copy Atom -// type. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2G(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGet( - MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_bulks2s`: Fields stored in Bulk CTA to Cluster -// Copy Atom type. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2S(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGet( - MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_dsmem_store`: Fields stored in -// CopyAtomDsmemStoreType. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldDsmemStore(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGet( - MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_g2r`: Fields stored in CopyAtomG2RType. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldLoadGlobal(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGet( - MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_non_exec_2d_gather4_tma_load`: Fields stored in -// CopyAtomNonExec2DGather4TmaLoadType. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoad( - MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGetValue( - MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_non_exec_2d_scatter4_tma_store`: Fields stored -// in CopyAtomNonExec2DScatter4TmaStoreType. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStore( - MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGetValue( - MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_non_exec_im2col_tma_load`: Fields stored in -// CopyAtomNonExecIm2ColTmaLoadType. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoad(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_non_exec_im2col_tma_store`: Fields stored in -// CopyAtomNonExecIm2ColTmaStoreType. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaStore(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_non_exec_tma_load`: Fields stored in -// CopyAtomNonExecTiledTmaLoadType. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaLoad(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_non_exec_tma_reduce`: Fields stored in -// CopyAtomNonExecTiledTmaReduceType. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaReduce(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_non_exec_tma_store`: Fields stored in -// CopyAtomNonExecTiledTmaStoreType. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaStore(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_r2g`: Fields stored in CopyAtomR2GType. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldStoreGlobal(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGet( - MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_tmaload`: Fields stored in Tma Load Copy Atom -// type. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldTmaLoad(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_tmareduce`: Fields stored in Tma Reduce Copy -// Atom type. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldTmaReduce(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_tmastore`: Fields stored in Tma Store Copy Atom -// type. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldTmaStore(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_mma_field_sm100`: Fields stored in MMA Atom types for -// SM100. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomMmaFieldSM100(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomMmaFieldSM100AttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomMmaFieldSM100AttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_mma_field_sm100_block_scaled`: Fields stored in MMA Atom -// types for SM100 (block scaled). -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaled(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_mma_field_sm100_block_scaled_sparse`: Fields stored in MMA -// Atom types for SM100 (block scaled). -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaledSparse(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_mma_field_sm100_sparse`: Fields stored in sparse MMA Atom -// types for SM100. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomMmaFieldSM100Sparse(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGet( - MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_mma_field_sm120_block_scaled`: Fields stored in MMA Atom -// types for SM120 (block scaled). -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomMmaFieldSM120BlockScaled(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_mma_field_sm80_sparse`: Fields stored in MMA Atom types for -// SM80. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomMmaFieldSM80Sparse(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_mma_field_sm90`: Fields stored in MMA Atom types for SM90. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomMmaFieldSM90(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomMmaFieldSM90AttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomMmaFieldSM90AttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.bin_op`: Binary operation for single-bit MMA operations. -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVGPUBinaryOp(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUBinaryOpAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUBinaryOpAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.copy_s2t_broadcast_mode`: Broadcast modes for the different -// utccp instructions. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUCopyS2TBroadcast(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyS2TBroadcastAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyS2TBroadcastAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.gather_scatter_tma_load`: The various kinds of TMA loads in -// gather/scatter mode. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUGatherScatterTmaLoad(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUGatherScatterTmaLoadAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUGatherScatterTmaLoadAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.im2col_tma_load`: The various kinds of TMA loads in im2col mode. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUIm2ColTmaLoad(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUIm2ColTmaLoadAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUIm2ColTmaLoadAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.ld_reduce_acc_precision_kind`: multimem ld_reduce accumulation -// precision kind. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPULdReduceAccPrecisionKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPULdReduceAccPrecisionKindAttrGet( - MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPULdReduceAccPrecisionKindAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.ldsm_sz_pattern`: LDSM's sz pattern, describing the bit size. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPULdsmSzPattern(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPULdsmSzPatternAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPULdsmSzPatternAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.load_cache_mode`: Cache modes for the load instructions. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPULoadCacheMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPULoadCacheModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPULoadCacheModeAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.mma_int_overflow`: MMA overflow options. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUMMAIntOverflow(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMMAIntOverflowAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMMAIntOverflowAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.major`: Major mode for MMA operations. -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVGPUMajorMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMajorModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMajorModeAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.mma_collector_op`: Enums for the mma collector op. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUMmaCollectorOp(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaCollectorOpAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaCollectorOpAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.mma_frag_kind`: Enums for the mma frag type. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUMmaFragKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaFragKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaFragKindAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.not_implemented_frg`: Fragment of an atom that has none. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUNotImplementedFrg(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUNotImplementedFrgAttrGet(MlirContext context); - -// `#cute_nvgpu.tma_reduce_kind`: Op for the TMASTORE instruction. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUReductionKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUReductionKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUReductionKindAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.rmem_frg`: Register-memory fragment of an MMA operand. -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVGPURmemFrg(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPURmemFrgAttrGet(MlirContext context, MlirType valueType, - uint32_t operand, bool deriveElemType); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPURmemFrgAttrGetValueType(MlirAttribute attr); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPURmemFrgAttrGetOperand(MlirAttribute attr); -MLIR_CAPI_EXPORTED bool -mlirCuteNVGPURmemFrgAttrGetDeriveElemType(MlirAttribute attr); - -// `#cute_nvgpu.arch.sm100.circular_smem_frg`: SM100 circular shared-memory -// fragment. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUSM100CircularSmemFrg(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUSM100CircularSmemFrgAttrGet( - MlirContext context, uint32_t majorMode); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUSM100CircularSmemFrgAttrGetMajorMode(MlirAttribute attr); - -// `#cute_nvgpu.arch.sm100.smem_frg`: SM100 shared-memory fragment. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUSM100SmemFrg(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUSM100SmemFrgAttrGet(MlirContext context, uint32_t majorMode); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUSM100SmemFrgAttrGetMajorMode(MlirAttribute attr); - -// `#cute_nvgpu.arch.sm100.tmem_e_frg`: SM100 tensor-memory sparse-metadata -// fragment. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUSM100TmemEFrg(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUSM100TmemEFrgAttrGet( - MlirContext context, MlirType aType, MlirType eType); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUSM100TmemEFrgAttrGetAType(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUSM100TmemEFrgAttrGetEType(MlirAttribute attr); - -// `#cute_nvgpu.arch.sm100.tmem_frg`: SM100 tensor-memory fragment. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUSM100TmemFrg(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUSM100TmemFrgAttrGet( - MlirContext context, MlirType dataType, MlirType storageType, int ctaGroup, - uint32_t tmemAllocMode); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUSM100TmemFrgAttrGetDataType(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUSM100TmemFrgAttrGetStorageType(MlirAttribute attr); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUSM100TmemFrgAttrGetCtaGroup(MlirAttribute attr); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUSM100TmemFrgAttrGetTmemAllocMode(MlirAttribute attr); - -// `#cute_nvgpu.arch.sm100.tmem_sf_frg`: SM100 tensor-memory scale-factor -// fragment. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUSM100TmemSfFrg(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUSM100TmemSfFrgAttrGet( - MlirContext context, MlirType sfType, int sfVecSize, int ctaGroup, - bool isSfa, uint32_t tmemAllocMode); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUSM100TmemSfFrgAttrGetSfType(MlirAttribute attr); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUSM100TmemSfFrgAttrGetSfVecSize(MlirAttribute attr); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUSM100TmemSfFrgAttrGetCtaGroup(MlirAttribute attr); -MLIR_CAPI_EXPORTED bool -mlirCuteNVGPUSM100TmemSfFrgAttrGetIsSfa(MlirAttribute attr); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUSM100TmemSfFrgAttrGetTmemAllocMode(MlirAttribute attr); - -// `#cute_nvgpu.arch.sm107.smem_frg`: SM107 shared-memory fragment. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUSM107SmemFrg(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUSM107SmemFrgAttrGet(MlirContext context, uint32_t majorMode); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUSM107SmemFrgAttrGetMajorMode(MlirAttribute attr); - -// `#cute_nvgpu.arch.sm90.smem_frg`: SM90 shared-memory fragment. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUSM90SmemFrg(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUSM90SmemFrgAttrGet(MlirContext context, uint32_t majorMode); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUSM90SmemFrgAttrGetMajorMode(MlirAttribute attr); - -// `#cute_nvgpu.tiled_tma_load`: The various kinds of TMA loads in tiled mode. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUTiledTmaLoad(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUTiledTmaLoadAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUTiledTmaLoadAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.tma_data_format`: Bits 74-71 of the TMA descriptor. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUTmaDataFormat(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUTmaDataFormatAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUTmaDataFormatAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.tma_load_mode`: Modes for the TMA load instruction. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUTmaLoadMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUTmaLoadModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUTmaLoadModeAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.tma_store_mode`: Modes for the TMA store instruction. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUTmaStoreMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUTmaStoreModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUTmaStoreModeAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.tmem_alloc_mode`: Modes for the Tmem allocation. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUTmemAllocMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUTmemAllocModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUTmemAllocModeAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.tmem_load_red_op`: Reduce operation for the Tmem load -// instruction. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUTmemLoadRedOp(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUTmemLoadRedOpAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUTmemLoadRedOpAttrGetValue(MlirAttribute attr); - -#ifdef __cplusplus -} -#endif - -#endif // CUTE_IR_C_DIALECT_CUTENVGPU_ATTRIBUTES_H diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteTypes.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteTypes.h deleted file mode 100644 index 0a4bd93d..00000000 --- a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteTypes.h +++ /dev/null @@ -1,185 +0,0 @@ -// C API for the CuTe types, generated from the dialect's .td files. -#ifndef CUTE_IR_C_DIALECT_CUTE_TYPES_H -#define CUTE_IR_C_DIALECT_CUTE_TYPES_H - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Constructors return null for invalid parameters or handles of another -// context; optional parameters take null handles, and std::optional enums a -// negative value. Enums are their integer values (the dialect's *Enums.td). -// Getters require a value accepted by the matching IsA. - -// `!cute.arith_tuple_iter`: Iterator over an arithmetic tuple. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteArithTupleIterator(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteArithTupleIteratorTypeGet(MlirContext context, MlirType arithTuple); -MLIR_CAPI_EXPORTED MlirType -mlirCuteArithTupleIteratorTypeGetArithTuple(MlirType type); - -// `!cute.composed_layout`: CuTe composed layout: A ∘ offset ∘ B. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteComposedLayout(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteComposedLayoutTypeGet(MlirContext context, - MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteComposedLayoutTypeGetAttr(MlirType type); - -// `!cute.i`: Integer of any width with known divisibility and power of two. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteConstrainedInt(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteConstrainedIntTypeGet(MlirContext context, - int64_t divisibility, - unsigned width, - bool isPow2); -MLIR_CAPI_EXPORTED int64_t -mlirCuteConstrainedIntTypeGetDivisibility(MlirType type); -MLIR_CAPI_EXPORTED unsigned mlirCuteConstrainedIntTypeGetWidth(MlirType type); -MLIR_CAPI_EXPORTED bool mlirCuteConstrainedIntTypeGetIsPow2(MlirType type); - -// `!cute.coord_tensor`: Coordinate tensor: an arithmetic tuple with a layout. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteCoordTensor(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTensorTypeGet(MlirContext context, - MlirType arithTuple, - MlirType layout); -MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTensorTypeGetArithTuple(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTensorTypeGetLayout(MlirType type); - -// `!cute.coord`: Scalar integer, underscore wildcard, or recursive tuple of -// coordinates. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteCoord(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTypeGet(MlirContext context, - MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCoordTypeGetAttr(MlirType type); - -// `!cute.fast_divmod_divisor`: Precomputed divisor of the fast divmod -// operations. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteFastDivmodDivisor(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteFastDivmodDivisorTypeGet( - MlirContext context, unsigned width, bool isPow2); -MLIR_CAPI_EXPORTED unsigned -mlirCuteFastDivmodDivisorTypeGetWidth(MlirType type); -MLIR_CAPI_EXPORTED bool mlirCuteFastDivmodDivisorTypeGetIsPow2(MlirType type); - -// `!cute.int_tuple`: Scalar integer or recursive tuple of integers. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteIntTuple(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteIntTupleTypeGet(MlirContext context, - MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteIntTupleTypeGetAttr(MlirType type); - -// `!cute.layout`: CuTe layout: a shape/stride pair. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteLayout(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteLayoutTypeGet(MlirContext context, - MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteLayoutTypeGetAttr(MlirType type); - -// `!cute.memref`: Tensor view: a pointer with a layout. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteMemRef(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteMemRefTypeGet(MlirContext context, - MlirType ptr, - MlirType layout); -MLIR_CAPI_EXPORTED MlirType mlirCuteMemRefTypeGetPtr(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteMemRefTypeGetLayout(MlirType type); - -// `!cute.ptr`: Pointer with address space, alignment, swizzle and bit layout. -MLIR_CAPI_EXPORTED bool mlirTypeIsACutePtr(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCutePtrTypeGet( - MlirContext context, MlirType valueType, uint32_t addressSpace, - uint64_t alignment, MlirAttribute swizzle, MlirAttribute bitlayout); -MLIR_CAPI_EXPORTED MlirType mlirCutePtrTypeGetValueType(MlirType type); -MLIR_CAPI_EXPORTED uint32_t mlirCutePtrTypeGetAddressSpace(MlirType type); -MLIR_CAPI_EXPORTED uint64_t mlirCutePtrTypeGetAlignment(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute mlirCutePtrTypeGetSwizzle(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute mlirCutePtrTypeGetBitlayout(MlirType type); - -// `!cute.shape`: Scalar integer or recursive tuple of shape extents. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteShape(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteShapeTypeGet(MlirContext context, - MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteShapeTypeGetAttr(MlirType type); - -// `!cute.sparse_elem`: Logical sparse element stored in a physical type. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteSparseElem(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteSparseElemTypeGet(MlirContext context, - int numLogical, - MlirType physicalType); -MLIR_CAPI_EXPORTED int mlirCuteSparseElemTypeGetNumLogical(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteSparseElemTypeGetPhysicalType(MlirType type); - -// `!cute.stride`: Scalar or recursive tuple of stride elements. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteStride(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteStrideTypeGet(MlirContext context, - MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteStrideTypeGetAttr(MlirType type); - -// `!cute.swizzle`: CuTe swizzle: S. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteSwizzle(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteSwizzleTypeGet(MlirContext context, - MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteSwizzleTypeGetAttr(MlirType type); - -// `!cute.tile`: CuTe tile: a recursive sequence of layouts and underscores. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTile(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteTileTypeGet(MlirContext context, - MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteTileTypeGetAttr(MlirType type); - -// `!cute.tiled_copy`: Copy atom tiled over threads and values. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTiledCopy(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteTiledCopyTypeGet(MlirContext context, - MlirType copyAtom, - MlirAttribute layoutCopyTv, - MlirAttribute tilerMn); -MLIR_CAPI_EXPORTED MlirType mlirCuteTiledCopyTypeGetCopyAtom(MlirType type); -// Former name of mlirCuteTiledCopyTypeGetCopyAtom. -MLIR_CAPI_EXPORTED MlirType mlirCuteTiledCopyTypeGetAtom(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteTiledCopyTypeGetLayoutCopyTv(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteTiledCopyTypeGetTilerMn(MlirType type); - -// `!cute.tiled_copy_v2`: V2 copy atom tiled over threads and values. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTiledCopyV2(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteTiledCopyV2TypeGet(MlirContext context, MlirType copyAtom, - MlirAttribute layoutCopyTv, MlirAttribute tilerMn); -MLIR_CAPI_EXPORTED MlirType mlirCuteTiledCopyV2TypeGetCopyAtom(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteTiledCopyV2TypeGetLayoutCopyTv(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteTiledCopyV2TypeGetTilerMn(MlirType type); - -// `!cute.tiled_mma`: MMA atom tiled over MNK. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTiledMma(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteTiledMmaTypeGet( - MlirContext context, MlirType mmaAtom, MlirAttribute atomLayoutMNK, - MlirAttribute permutationMNK); -MLIR_CAPI_EXPORTED MlirType mlirCuteTiledMmaTypeGetMmaAtom(MlirType type); -// Former name of mlirCuteTiledMmaTypeGetMmaAtom. -MLIR_CAPI_EXPORTED MlirType mlirCuteTiledMmaTypeGetAtom(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteTiledMmaTypeGetAtomLayoutMNK(MlirType type); -// Former name of mlirCuteTiledMmaTypeGetAtomLayoutMNK. -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteTiledMmaTypeGetAtomLayoutMnk(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteTiledMmaTypeGetPermutationMNK(MlirType type); -// Former name of mlirCuteTiledMmaTypeGetPermutationMNK. -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteTiledMmaTypeGetPermutationMnk(MlirType type); - -// `!cute.tuple`: Non-empty tuple of types. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTuple(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteTupleTypeGet(MlirContext context, - intptr_t numTypes, - MlirType const *types); -MLIR_CAPI_EXPORTED intptr_t mlirCuteTupleTypeGetNumTypes(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteTupleTypeGetType(MlirType type, - intptr_t pos); - -#ifdef __cplusplus -} -#endif - -#endif // CUTE_IR_C_DIALECT_CUTE_TYPES_H diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/NVVM.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/NVVM.h deleted file mode 100644 index 94d00310..00000000 --- a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/NVVM.h +++ /dev/null @@ -1,21 +0,0 @@ -//===- NVVM.h - C API for NVIDIA's NVVM dialect of the CuTe DSL -*- C -*-===// -// -// The dialect handle is `cute_nvvm`, not the upstream dialect's `nvvm`: both register a dialect named `nvvm`. - -#ifndef CUTE_IR_C_DIALECT_NVVM_H -#define CUTE_IR_C_DIALECT_NVVM_H - -#include "cute_ir-c/Dialect/NVVMAttributes.h" -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(CuteNVVM, cute_nvvm); - -#ifdef __cplusplus -} -#endif - -#endif // CUTE_IR_C_DIALECT_NVVM_H diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/NVVMAttributes.h b/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/NVVMAttributes.h deleted file mode 100644 index e6eac4d4..00000000 --- a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/NVVMAttributes.h +++ /dev/null @@ -1,509 +0,0 @@ -// C API for the nvvm attributes, generated from the dialect's .td files. -#ifndef CUTE_IR_C_DIALECT_NVVM_ATTRIBUTES_H -#define CUTE_IR_C_DIALECT_NVVM_ATTRIBUTES_H - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Constructors return null for invalid parameters or handles of another -// context; optional parameters take null handles. Enums are their integer -// values (the dialect's *Enums.td). Getters require a value accepted by the -// matching IsA. - -// `#nvvm.atomic_op`: operations supported by atom instruction -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMAtomicOpKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMAtomicOpKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMAtomicOpKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.reduction`: NVVM barrier reduction operation -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMBarrierReduction(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMBarrierReductionAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMBarrierReductionAttrGetValue(MlirAttribute attr); - -// `#nvvm.barrier_redux_kind`: NVVM barrier redux kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMBarrierReduxKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMBarrierReduxKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMBarrierReduxKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.block_scale_format`: MMA Block Scale Format -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMBlockScaleFormat(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMBlockScaleFormatAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMBlockScaleFormatAttrGetValue(MlirAttribute attr); - -// `#nvvm.cta_group`: NVVM CTA group kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMCTAGroupKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMCTAGroupKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMCTAGroupKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.packfloat_type`: NVVM CVT Pack Float kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMCVTPackFloatKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMCVTPackFloatKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMCVTPackFloatKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.cache_eviction_priority`: NVVM Cache Eviction Priority -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMCacheEvictionPriority(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMCacheEvictionPriorityAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMCacheEvictionPriorityAttrGetValue(MlirAttribute attr); - -// `#nvvm.cluster_launch_control_query_type`: NVVM ClusterLaunchControlQueryType -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMClusterLaunchControlQueryType(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMClusterLaunchControlQueryTypeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMClusterLaunchControlQueryTypeAttrGetValue(MlirAttribute attr); - -// `#nvvm.op`: Comparison operator encoding -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMCompareOpKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMCompareOpKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMCompareOpKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.convert_fp4_type`: NVVM ConvertFP4Type kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMConvertFP4Type(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMConvertFP4TypeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMConvertFP4TypeAttrGetValue(MlirAttribute attr); - -// `#nvvm.convert_fp8_type`: NVVM ConvertFP8Type kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMConvertFP8Type(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMConvertFP8TypeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMConvertFP8TypeAttrGetValue(MlirAttribute attr); - -// `#nvvm.convert_scale_kind`: NVVM ConvertScale kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMConvertScaleKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMConvertScaleKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMConvertScaleKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.dot_accumulate_type`: NVVM DotAccumulateType -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMDotAccumulateType(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMDotAccumulateTypeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMDotAccumulateTypeAttrGetValue(MlirAttribute attr); - -// `#nvvm.evict_kind`: NVVM L2 Prefetch Size -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMEvictKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMEvictKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMEvictKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.fp_rnd_mode`: NVVM FPRoundingMode kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMFPRoundingMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMFPRoundingModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMFPRoundingModeAttrGetValue(MlirAttribute attr); - -// `#nvvm.grid_dep_action`: Action kind for grid dependency control -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMGridDepActionKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMGridDepActionKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMGridDepActionKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.int_rnd_mode`: NVVM IntegerRoundingMode kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMIntegerRoundingMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMIntegerRoundingModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMIntegerRoundingModeAttrGetValue(MlirAttribute attr); - -// `#nvvm.l2_prefetch`: NVVM L2 Prefetch Size -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVML2PrefetchSize(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVML2PrefetchSizeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVML2PrefetchSizeAttrGetValue(MlirAttribute attr); - -// `#nvvm.ld_st_matrix_elt_type`: Element type for ldmatrix and stmatrix -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMLdStMatrixEltType(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMLdStMatrixEltTypeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMLdStMatrixEltTypeAttrGetValue(MlirAttribute attr); - -// `#nvvm.load_cache_modifier_ext`: NVVM load cache modifier kind(Ext) -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMLoadCacheModifierExtKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMLoadCacheModifierExtKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMLoadCacheModifierExtKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.load_cache_modifier`: NVVM load cache modifier kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMLoadCacheModifierKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMLoadCacheModifierKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMLoadCacheModifierKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.load_shape`: shape attribute for ldmatrix -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMLoadShape(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMLoadShapeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMLoadShapeAttrGetValue(MlirAttribute attr); - -// `#nvvm.load_src_format`: source format for ldmatrix -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMLoadSrcFormat(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMLoadSrcFormatAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMLoadSrcFormatAttrGetValue(MlirAttribute attr); - -// `#nvvm.mbarrier_layout`: NVVM MBarrier Layout -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMBarrierLayout(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMBarrierLayoutAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMBarrierLayoutAttrGetValue(MlirAttribute attr); - -// `#nvvm.mbarrier_phase`: NVVM mbarrier phase type -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMBarrierPhase(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMBarrierPhaseAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMBarrierPhaseAttrGetValue(MlirAttribute attr); - -// `#nvvm.mbar_scope`: NVVM MBarrier scope kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMBarrierScopeKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMBarrierScopeKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMBarrierScopeKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.mbar_space`: NVVM MBarrier space kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMBarrierSpaceKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMBarrierSpaceKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMBarrierSpaceKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.mbar_txn_kind`: NVVM MBarrier Transaction kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMBarrierTxnKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMBarrierTxnKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMBarrierTxnKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.mbar_wait`: NVVM MBarrier wait kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMBarrierWaitKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMBarrierWaitKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMBarrierWaitKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.mma_b1op`: MMA binary operations -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAB1Op(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMAB1OpAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMAB1OpAttrGetValue(MlirAttribute attr); - -// `#nvvm.block_scale_kind`: Block Scale Kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMABlockScaleKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMABlockScaleKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMABlockScaleKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.mma_cta_count`: MMA CTA count -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMACtaCount(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMACtaCountAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMACtaCountAttrGetValue(MlirAttribute attr); - -// `#nvvm.mma_frag`: NVVM MMA frag type -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAFrag(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMAFragAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMAFragAttrGetValue(MlirAttribute attr); - -// `#nvvm.mma_int_overflow`: MMA overflow options -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAIntOverflow(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMAIntOverflowAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMAIntOverflowAttrGetValue(MlirAttribute attr); - -// `#nvvm.mma_kind`: MMA operation kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMAKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMAKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.mma_layout`: NVVM MMA layout -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMALayout(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMALayoutAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMALayoutAttrGetValue(MlirAttribute attr); - -// `#nvvm.mma_type`: NVVM MMA types -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMATypes(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMATypesAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMATypesAttrGetValue(MlirAttribute attr); - -// `#nvvm.match_sync_kind`: NVVM match sync kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMatchSyncKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMatchSyncKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMatchSyncKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.mem_order`: NVVM Memory Ordering kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMemOrderKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMemOrderKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMemOrderKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.mem_scope`: NVVM Memory Scope kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMemScopeKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMemScopeKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMemScopeKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.mul_mode`: multiply mode attribute -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMulMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMulModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMulModeAttrGetValue(MlirAttribute attr); - -// `#nvvm.memory_space`: NVVM Memory Space -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMNVVMMemorySpace(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMNVVMMemorySpaceAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMNVVMMemorySpaceAttrGetValue(MlirAttribute attr); - -// `#nvvm.permute_mode`: NVVM permute mode -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMPermuteMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMPermuteModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMPermuteModeAttrGetValue(MlirAttribute attr); - -// `#nvvm.prefetch_cache_level`: NVVM Prefetch Cache Level -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMPrefetchCacheLevel(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMPrefetchCacheLevelAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMPrefetchCacheLevelAttrGetValue(MlirAttribute attr); - -// `#nvvm.proxy_kind`: Proxy kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMProxyKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMProxyKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMProxyKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.reduction_kind`: NVVM Reduction Kind attribute -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMReductionKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMReductionKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMReductionKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.red_op`: Ops supported by red instruction -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMReductionOp(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMReductionOpAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMReductionOpAttrGetValue(MlirAttribute attr); - -// `#nvvm.red_type`: types supported by red instruction -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMReductionType(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMReductionTypeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMReductionTypeAttrGetValue(MlirAttribute attr); - -// `#nvvm.spcomp_elem_size`: Sparse tensor compression element size -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSPCompressElemSize(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSPCompressElemSizeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSPCompressElemSizeAttrGetValue(MlirAttribute attr); - -// `#nvvm.spcomp_factor`: Sparse tensor compression factor type -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSPCompressFactorType(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSPCompressFactorTypeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSPCompressFactorTypeAttrGetValue(MlirAttribute attr); - -// `#nvvm.spcomp_index_size`: Sparse tensor compression index size -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSPCompressIndexSize(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSPCompressIndexSizeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSPCompressIndexSizeAttrGetValue(MlirAttribute attr); - -// `#nvvm.spcomp_op_kind`: spcompress operation kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSPCompressOpKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSPCompressOpKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSPCompressOpKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.spcomp_rep_factor`: Sparse tensor compression repetition factor -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSPCompressRepFactor(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSPCompressRepFactorAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSPCompressRepFactorAttrGetValue(MlirAttribute attr); - -// `#nvvm.spdecomp_elem_size`: Sparse tensor decompression element size -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSPDecompressElemSize(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSPDecompressElemSizeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSPDecompressElemSizeAttrGetValue(MlirAttribute attr); - -// `#nvvm.spdecomp_factor`: Sparse tensor decompression factor type -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSPDecompressFactorType(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSPDecompressFactorTypeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSPDecompressFactorTypeAttrGetValue(MlirAttribute attr); - -// `#nvvm.spdecomp_index_size`: Sparse tensor decompression index size -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSPDecompressIndexSize(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSPDecompressIndexSizeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSPDecompressIndexSizeAttrGetValue(MlirAttribute attr); - -// `#nvvm.spdecomp_rep_factor`: Sparse tensor decompression repetition factor -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSPDecompressRepFactor(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSPDecompressRepFactorAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSPDecompressRepFactorAttrGetValue(MlirAttribute attr); - -// `#nvvm.sat_mode`: NVVM SaturationMode kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSaturationMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSaturationModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSaturationModeAttrGetValue(MlirAttribute attr); - -// `#nvvm.sat`: NVVM SaturationMode kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSaturationModeKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSaturationModeKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSaturationModeKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.scale_vec_size`: MMA Scale Vector Sizes -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMScaleVecSize(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMScaleVecSizeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMScaleVecSizeAttrGetValue(MlirAttribute attr); - -// `#nvvm.action`: NVVM set max register action -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSetMaxRegisterAction(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSetMaxRegisterActionAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSetMaxRegisterActionAttrGetValue(MlirAttribute attr); - -// `#nvvm.shared_space`: Shared memory space -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSharedSpace(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSharedSpaceAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSharedSpaceAttrGetValue(MlirAttribute attr); - -// `#nvvm.shfl_kind`: NVVM shuffle kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMShflKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMShflKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMShflKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.sparsity_format`: MMA Sparsity Format -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSparsityFormat(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMSparsityFormatAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMSparsityFormatAttrGetValue(MlirAttribute attr); - -// `#nvvm.state_space`: NVVM State Space -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMStateSpace(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMStateSpaceAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMStateSpaceAttrGetValue(MlirAttribute attr); - -// `#nvvm.store_cache_modifier`: NVVM store cache modifier kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMStoreCacheModifierKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMStoreCacheModifierKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMStoreCacheModifierKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.store_shape`: shape attribute for stmatrix -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMStoreShape(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMStoreShapeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMStoreShapeAttrGetValue(MlirAttribute attr); - -// `#nvvm.TCBarParam`: Cluster MMA Barrier Parameter Type -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTCBarParam(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTCBarParamAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTCBarParamAttrGetValue(MlirAttribute attr); - -// `#nvvm.tma_load_mode`: NVVM TMA Load Mode -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTMALoadMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTMALoadModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTMALoadModeAttrGetValue(MlirAttribute attr); - -// `#nvvm.tma_redux_kind`: NVVM TMA redux kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTMAReduxKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTMAReduxKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTMAReduxKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.tma_store_mode`: NVVM TMA Store Mode -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTMAStoreMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTMAStoreModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTMAStoreModeAttrGetValue(MlirAttribute attr); - -// `#nvvm.tcgen05_cp_multicast`: tcgen05 cp multicast -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTcgen05CpMulticast(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTcgen05CpMulticastAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTcgen05CpMulticastAttrGetValue(MlirAttribute attr); - -// `#nvvm.tcgen05_cp_shape`: tcgen05 cp shapes -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTcgen05CpShape(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTcgen05CpShapeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTcgen05CpShapeAttrGetValue(MlirAttribute attr); - -// `#nvvm.tcgen05_cp_src_fmt`: tcgen05 cp source format -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTcgen05CpSrcFormat(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTcgen05CpSrcFormatAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTcgen05CpSrcFormatAttrGetValue(MlirAttribute attr); - -// `#nvvm.tcgen05_fence`: NVVM Tcgen05 fence kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTcgen05FenceKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTcgen05FenceKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTcgen05FenceKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.tcgen05_ldst_shape`: allowed 32-bit signless integer cases: 0, 1, 2, 3, 4 -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTcgen05LdStShape(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTcgen05LdStShapeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTcgen05LdStShapeAttrGetValue(MlirAttribute attr); - -// `#nvvm.tcgen05_mma_block_scale`: tcgen05.mma block scale attribute -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTcgen05MMABlockScale(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTcgen05MMABlockScaleAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTcgen05MMABlockScaleAttrGetValue(MlirAttribute attr); - -// `#nvvm.tcgen05_mma_collectorb`: tcgen05 MMA Collector Buffer B Attribute -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTcgen05MMACollectorBBuffer(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTcgen05MMACollectorBBufferAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTcgen05MMACollectorBBufferAttrGetValue(MlirAttribute attr); - -// `#nvvm.tcgen05_mma_collectorop`: tcgen05.mma Collector Buffer Operation -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTcgen05MMACollectorOp(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTcgen05MMACollectorOpAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTcgen05MMACollectorOpAttrGetValue(MlirAttribute attr); - -// `#nvvm.tcgen05_mma_kind`: tcgen05 MMA Supported Types -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTcgen05MMAKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTcgen05MMAKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTcgen05MMAKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.tcgen05_wait`: NVVM Tcgen05 wait kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTcgen05WaitKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTcgen05WaitKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTcgen05WaitKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.tensormap_elemtype`: NVVM Tensormap Elemtype -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTensormapElemtype(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTensormapElemtypeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTensormapElemtypeAttrGetValue(MlirAttribute attr); - -// `#nvvm.tensormap_field`: NVVM Tensormap Field Kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTensormapField(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTensormapFieldAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTensormapFieldAttrGetValue(MlirAttribute attr); - -// `#nvvm.tensormap_fill_mode`: NVVM Tensormap Fill Mode -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTensormapFillMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTensormapFillModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTensormapFillModeAttrGetValue(MlirAttribute attr); - -// `#nvvm.tensormap_interleave_layout`: NVVM Tensormap Interleave Layout -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTensormapInterleaveLayout(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTensormapInterleaveLayoutAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTensormapInterleaveLayoutAttrGetValue(MlirAttribute attr); - -// `#nvvm.tensormap_swizzle_atomicity`: NVVM Tensormap Swizzle Atomicity -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTensormapSwizzleAtomicity(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTensormapSwizzleAtomicityAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTensormapSwizzleAtomicityAttrGetValue(MlirAttribute attr); - -// `#nvvm.tensormap_swizzle_mode`: NVVM Tensormap Swizzle Mode -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTensormapSwizzleMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTensormapSwizzleModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTensormapSwizzleModeAttrGetValue(MlirAttribute attr); - -// `#nvvm.TmemLayout`: Tensor Memory Layout Enumerated Type -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTmemLayout(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTmemLayoutAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMTmemLayoutAttrGetValue(MlirAttribute attr); - -// `#nvvm.validate_pattern`: NVVM validate data pattern -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMValidatePattern(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMValidatePatternAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMValidatePatternAttrGetValue(MlirAttribute attr); - -// `#nvvm.vote_sync_kind`: NVVM vote sync kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMVoteSyncKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMVoteSyncKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMVoteSyncKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.wgmma_scale_in`: WGMMA overflow options -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMWGMMAScaleIn(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMWGMMAScaleInAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMWGMMAScaleInAttrGetValue(MlirAttribute attr); - -// `#nvvm.wgmma_scale_out`: WGMMA input predicate -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMWGMMAScaleOut(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMWGMMAScaleOutAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMWGMMAScaleOutAttrGetValue(MlirAttribute attr); - -// `#nvvm.wgmma_type`: NVVM WGMMA types -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMWGMMATypes(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMWGMMATypesAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMWGMMATypesAttrGetValue(MlirAttribute attr); - -// `#nvvm.ld_st_matrix_shape`: Matrix shape of ldmatrix, stmatrix and movmatrix -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMLdStMatrixShape(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMLdStMatrixShapeAttrGet(MlirContext context, int m, int n); -MLIR_CAPI_EXPORTED int mlirCuteNVVMLdStMatrixShapeAttrGetM(MlirAttribute attr); -MLIR_CAPI_EXPORTED int mlirCuteNVVMLdStMatrixShapeAttrGetN(MlirAttribute attr); - -// `#nvvm.shape`: Shape of an MMA operation -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAShape(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMAShapeAttrGet(MlirContext context, int m, int n, int k); -MLIR_CAPI_EXPORTED int mlirCuteNVVMMMAShapeAttrGetM(MlirAttribute attr); -MLIR_CAPI_EXPORTED int mlirCuteNVVMMMAShapeAttrGetN(MlirAttribute attr); -MLIR_CAPI_EXPORTED int mlirCuteNVVMMMAShapeAttrGetK(MlirAttribute attr); - -// `#nvvm.target`: GPU target of an NVVM module -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTarget(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTargetAttrGet(MlirContext context, int O, MlirStringRef triple, MlirStringRef chip, MlirStringRef features, MlirAttribute flags, MlirAttribute link, bool verifyTarget); -MLIR_CAPI_EXPORTED int mlirCuteNVVMTargetAttrGetO(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef mlirCuteNVVMTargetAttrGetTriple(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef mlirCuteNVVMTargetAttrGetChip(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef mlirCuteNVVMTargetAttrGetFeatures(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTargetAttrGetFlags(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTargetAttrGetLink(MlirAttribute attr); -MLIR_CAPI_EXPORTED bool mlirCuteNVVMTargetAttrGetVerifyTarget(MlirAttribute attr); - -#ifdef __cplusplus -} -#endif - -#endif // CUTE_IR_C_DIALECT_NVVM_ATTRIBUTES_H diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/BUILD.bazel b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/BUILD.bazel deleted file mode 100644 index ed80bac3..00000000 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/BUILD.bazel +++ /dev/null @@ -1,138 +0,0 @@ -load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") -load("@rules_cc//cc:defs.bzl", "cc_library") - -package(default_visibility = ["//visibility:public"]) - -# The release's dialect, attributes and types come from the archive; the DSL -# compiler's remaining attributes, types and enums are CuteAttrsCompiler.td, -# CuteTypesCompiler.td and CuteEnums.td, and every one of its operations is -# CuteOps.td, generated from the DSL's Python bindings and compiler. -td_library( - name = "tabledefs", - srcs = [ - "CuteAttrsAll.td", - "CuteAttrsCompiler.td", - "CuteEnums.td", - "CuteOps.td", - "CuteTypesAll.td", - "CuteTypesCompiler.td", - ], - includes = ["/mlir/dialects/cute_ir/include"], - deps = ["@cute_ir//:cute_ir_dialect_td"], -) - -gentbl_cc_library( - name = "enums_inc_gen", - includes = ["../../../.."], - tbl_outs = [ - ( - ["-gen-enum-decls"], - "CuteEnums.h.inc", - ), - ( - ["-gen-enum-defs"], - "CuteEnums.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "CuteEnums.td", - deps = [":tabledefs"], -) - -gentbl_cc_library( - name = "attrs_inc_gen", - includes = ["../../../.."], - tbl_outs = [ - ( - [ - "-gen-attrdef-decls", - "-attrdefs-dialect=cute", - ], - "CuteAttrDecls.h.inc", - ), - ( - [ - "-gen-attrdef-defs", - "-attrdefs-dialect=cute", - ], - "CuteAttrs.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "CuteAttrsAll.td", - deps = [":tabledefs"], -) - -# CuteDialect.h includes the attribute declarations first: they bring the -# enums the attributes and types use. -genrule( - name = "attrs_h_inc", - srcs = ["CuteAttrDecls.h.inc"], - outs = ["CuteAttrs.h.inc"], - cmd = "(echo '#ifdef GET_ATTRDEF_CLASSES'; " + - "echo '#include \"cute_ir/Dialect/Cute/IR/CuteEnums.h.inc\"'; " + - "echo '#endif'; cat $<) > $@", -) - -gentbl_cc_library( - name = "types_inc_gen", - includes = ["../../../.."], - tbl_outs = [ - ( - [ - "-gen-typedef-decls", - "-typedefs-dialect=cute", - ], - "CuteTypes.h.inc", - ), - ( - [ - "-gen-typedef-defs", - "-typedefs-dialect=cute", - ], - "CuteTypes.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "CuteTypesAll.td", - deps = [":tabledefs"], -) - -gentbl_cc_library( - name = "ops_inc_gen", - includes = ["../../../.."], - tbl_outs = [ - ( - ["-gen-op-decls"], - "CuteOps.h.inc", - ), - ( - ["-gen-op-defs"], - "CuteOps.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "CuteOps.td", - deps = [":tabledefs"], -) - -cc_library( - name = "headers", - hdrs = [ - "CuteAttrs.h.inc", - "CuteInference.h", - ], - includes = ["../../../.."], - deps = [ - ":attrs_inc_gen", - ":enums_inc_gen", - ":ops_inc_gen", - ":types_inc_gen", - "@cute_ir//:cute_ir_dialect_header", - "@cute_ir//:cutegen", - "@llvm-project//mlir:DataLayoutInterfaces", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:InferTypeOpInterface", - "@llvm-project//mlir:SideEffectInterfaces", - ], -) diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteAttrsAll.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteAttrsAll.td deleted file mode 100644 index ea4264cb..00000000 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteAttrsAll.td +++ /dev/null @@ -1,7 +0,0 @@ -// The dialect has one generated attribute parser and printer, so the -// release's attributes, the compiler's and its enum attributes are generated -// together from here. - -include "cute_ir/Dialect/Cute/IR/CuteAttrs.td" -include "cute_ir/Dialect/Cute/IR/CuteAttrsCompiler.td" -include "cute_ir/Dialect/Cute/IR/CuteEnums.td" diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesAll.td b/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesAll.td deleted file mode 100644 index a8cfdb29..00000000 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteTypesAll.td +++ /dev/null @@ -1,5 +0,0 @@ -// The dialect has one generated type parser and printer, so the release's -// types and the compiler's are generated together from here. - -include "cute_ir/Dialect/Cute/IR/CuteTypes.td" -include "cute_ir/Dialect/Cute/IR/CuteTypesCompiler.td" diff --git a/mlir/dialects/cute_ir/lib/CAPI/BUILD.bazel b/mlir/dialects/cute_ir/lib/CAPI/BUILD.bazel deleted file mode 100644 index b471d7f4..00000000 --- a/mlir/dialects/cute_ir/lib/CAPI/BUILD.bazel +++ /dev/null @@ -1,63 +0,0 @@ -load("@rules_cc//cc:defs.bzl", "cc_library") - -package(default_visibility = ["//visibility:public"]) - -cc_library( - name = "cute", - srcs = [ - "Cute.cpp", - "CuteAttributes.cpp", - "CuteCompilerAttributes.cpp", - "CuteTypes.cpp", - ], - deps = [ - "//mlir/dialects/cute_ir/include/cute_ir-c/Dialect:cute", - "//mlir/dialects/cute_ir/lib/Dialect/IR:dialect", - "@llvm-project//mlir:CAPIIR", - ], - alwayslink = True, -) - -cc_library( - name = "cute_nvgpu", - srcs = [ - "CuteNVGPU.cpp", - "CuteNVGPUAttributes.cpp", - "CuteNVGPUTypes.cpp", - ], - deps = [ - "//mlir/dialects/cute_ir/include/cute_ir-c/Dialect:cute_nvgpu", - "//mlir/dialects/cute_ir/lib/Dialect/CuteNVGPU/IR:dialect", - "@llvm-project//mlir:CAPIIR", - ], - alwayslink = True, -) - -cc_library( - name = "cuda", - srcs = [ - "Cuda.cpp", - "CudaAttributes.cpp", - "CudaTypes.cpp", - ], - deps = [ - "//mlir/dialects/cute_ir/include/cute_ir-c/Dialect:cuda", - "//mlir/dialects/cute_ir/lib/Dialect/Cuda/IR:dialect", - "@llvm-project//mlir:CAPIIR", - ], - alwayslink = True, -) - -cc_library( - name = "nvvm", - srcs = [ - "NVVM.cpp", - "NVVMAttributes.cpp", - ], - deps = [ - "//mlir/dialects/cute_ir/include/cute_ir-c/Dialect:nvvm", - "//mlir/dialects/cute_ir/lib/Dialect/NVVM/IR:dialect", - "@llvm-project//mlir:CAPIIR", - ], - alwayslink = True, -) diff --git a/mlir/dialects/cute_ir/lib/CAPI/Cuda.cpp b/mlir/dialects/cute_ir/lib/CAPI/Cuda.cpp deleted file mode 100644 index eef03db0..00000000 --- a/mlir/dialects/cute_ir/lib/CAPI/Cuda.cpp +++ /dev/null @@ -1,10 +0,0 @@ -//===- Cuda.cpp - C API for NVIDIA's CUDA host dialect of the CuTe DSL ------===// - -#include "cute_ir-c/Dialect/Cuda.h" - -#include "cute_ir/Dialect/Cuda/IR/CudaDialect.h" -#include "mlir/CAPI/IR.h" -#include "mlir/CAPI/Registration.h" - -MLIR_DEFINE_CAPI_DIALECT_REGISTRATION( - Cuda, cuda, mlir::cutlass_compiler::cuda::CudaDialect) diff --git a/mlir/dialects/cute_ir/lib/CAPI/CudaAttributes.cpp b/mlir/dialects/cute_ir/lib/CAPI/CudaAttributes.cpp deleted file mode 100644 index 33bbac33..00000000 --- a/mlir/dialects/cute_ir/lib/CAPI/CudaAttributes.cpp +++ /dev/null @@ -1,136 +0,0 @@ -// C API for the cuda attributes, generated from the dialect's .td -// files. -#include "cute_ir-c/Dialect/CudaAttributes.h" - -#include "cute_ir/Dialect/Cuda/IR/CudaDialect.h" -#include "mlir/CAPI/IR.h" -#include "mlir/CAPI/Support.h" - -namespace cuda = mlir::cutlass_compiler::cuda; - -namespace { -// Unwraps `handle` into `out`: false when it is of another context or not a -// T, or when it is null and not optional. -template -bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { - auto value = unwrap(handle); - if (!value) - return optional; - if (value.getContext() != ctx) - return false; - out = llvm::dyn_cast(value); - return static_cast(out); -} -} // namespace - -bool mlirAttributeIsACudaAssumeKernel(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCudaAssumeKernelAttrGet(MlirContext context, bool value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::AssumeKernelAttr::get(ctx, value)); -} -bool mlirCudaAssumeKernelAttrGetValue(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getValue(); -} - -bool mlirAttributeIsACudaComputeTarget(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCudaComputeTargetAttrGet(MlirContext context, uint32_t representation, uint32_t portability, intptr_t numArchs, const uint32_t *archs) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto representationValue = ::mlir::cutlass_compiler::cuda::symbolizeExecutableRepresentation(representation); - if (!representationValue) - return {nullptr}; - auto portabilityValue = ::mlir::cutlass_compiler::cuda::symbolizeArchPortability(portability); - if (!portabilityValue) - return {nullptr}; - llvm::SmallVector<::mlir::cutlass_compiler::cuda::GpuArchitecture> archsValue; - for (intptr_t i = 0; i < numArchs; ++i) { - auto value = ::mlir::cutlass_compiler::cuda::symbolizeGpuArchitecture(archs[i]); - if (!value) - return {nullptr}; - archsValue.push_back(*value); - } - return wrap(cuda::ComputeTargetAttr::get(ctx, *representationValue, *portabilityValue, archsValue)); -} -uint32_t mlirCudaComputeTargetAttrGetRepresentation(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getRepresentation()); -} -uint32_t mlirCudaComputeTargetAttrGetPortability(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getPortability()); -} -intptr_t mlirCudaComputeTargetAttrGetNumArchs(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getArchs().size(); -} -uint32_t mlirCudaComputeTargetAttrGetArchs(MlirAttribute attr, intptr_t pos) { - return static_cast(llvm::cast(unwrap(attr)).getArchs()[pos]); -} - -bool mlirAttributeIsACudaDevMaxSharedMemoryOptin(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCudaDevMaxSharedMemoryOptinAttrGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::DevMaxSharedMemoryOptinAttr::get(ctx)); -} - -bool mlirAttributeIsACudaDeviceAttributes(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCudaDeviceAttributesAttrGet(MlirContext context, MlirAttribute values) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::DictionaryAttr valuesValue; - if (!unwrapAs(ctx, values, false, valuesValue)) - return {nullptr}; - return wrap(cuda::DeviceAttributesAttr::get(ctx, valuesValue)); -} -MlirAttribute mlirCudaDeviceAttributesAttrGetValues(MlirAttribute attr) { - return wrap(static_cast(llvm::cast(unwrap(attr)).getValues())); -} - -bool mlirAttributeIsACudaExecutable(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCudaExecutableAttrGet(MlirContext context, uint32_t format) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto formatValue = ::mlir::cutlass_compiler::cuda::symbolizeExecutableFormat(format); - if (!formatValue) - return {nullptr}; - return wrap(cuda::ExecutableAttr::get(ctx, *formatValue)); -} -uint32_t mlirCudaExecutableAttrGetFormat(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getFormat()); -} - -bool mlirAttributeIsACudaFuncAttributes(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCudaFuncAttributesAttrGet(MlirContext context, MlirAttribute values) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::DictionaryAttr valuesValue; - if (!unwrapAs(ctx, values, false, valuesValue)) - return {nullptr}; - return wrap(cuda::FuncAttributesAttr::get(ctx, valuesValue)); -} -MlirAttribute mlirCudaFuncAttributesAttrGetValues(MlirAttribute attr) { - return wrap(static_cast(llvm::cast(unwrap(attr)).getValues())); -} diff --git a/mlir/dialects/cute_ir/lib/CAPI/Cute.cpp b/mlir/dialects/cute_ir/lib/CAPI/Cute.cpp deleted file mode 100644 index 6956b2aa..00000000 --- a/mlir/dialects/cute_ir/lib/CAPI/Cute.cpp +++ /dev/null @@ -1,17 +0,0 @@ -//===- Cute.cpp - C API for the CuTe dialect -----------------------------===// - -#include "cute_ir-c/Dialect/Cute.h" - -#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" -#include "mlir/CAPI/IR.h" -#include "mlir/CAPI/Registration.h" - -MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Cute, cute, - mlir::cutlass_compiler::cute::CuteDialect) - -bool mlirTypeIsACuteType(MlirType type) { - mlir::Type cppType = unwrap(type); - return cppType && - cppType.getDialect().getNamespace() == - mlir::cutlass_compiler::cute::CuteDialect::getDialectNamespace(); -} diff --git a/mlir/dialects/cute_ir/lib/CAPI/CuteAttributes.cpp b/mlir/dialects/cute_ir/lib/CAPI/CuteAttributes.cpp deleted file mode 100644 index 4b7b267c..00000000 --- a/mlir/dialects/cute_ir/lib/CAPI/CuteAttributes.cpp +++ /dev/null @@ -1,130 +0,0 @@ -// C++ to C bindings for CuTe algebra attributes. - -#include "cute_ir-c/Dialect/CuteAttributes.h" - -#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" -#include "mlir/CAPI/IR.h" -#include "mlir/CAPI/Support.h" - -namespace cute = mlir::cutlass_compiler::cute; - -namespace { -template -MlirAttribute getAlgebraAttribute(MlirContext context, MlirStringRef value) { - auto *ctx = unwrap(context); - if (!ctx || (!value.data && value.length)) - return {nullptr}; - auto algebra = - cutegen::from_string(unwrap(value).str()); - if (!algebra) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(Attr::get(ctx, std::move(*algebra))); -} - -template MlirStringRef getAlgebraValue(MlirAttribute attr) { - auto value = llvm::cast(unwrap(attr)); - // Intern the serialization so the returned view outlives this function. - auto text = mlir::StringAttr::get(value.getContext(), - cutegen::to_string(value.getRef())); - return wrap(text.getValue()); -} -} // namespace - -MlirAttribute mlirCuteIntTupleAttrGet(MlirContext context, - MlirStringRef value) { - return getAlgebraAttribute(context, value); -} - -bool mlirAttributeIsACuteIntTuple(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirStringRef mlirCuteIntTupleAttrGetValue(MlirAttribute attr) { - return getAlgebraValue(attr); -} - -MlirAttribute mlirCuteCoordAttrGet(MlirContext context, MlirStringRef value) { - return getAlgebraAttribute(context, value); -} - -bool mlirAttributeIsACuteCoord(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirStringRef mlirCuteCoordAttrGetValue(MlirAttribute attr) { - return getAlgebraValue(attr); -} - -MlirAttribute mlirCuteShapeAttrGet(MlirContext context, MlirStringRef value) { - return getAlgebraAttribute(context, value); -} - -bool mlirAttributeIsACuteShape(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirStringRef mlirCuteShapeAttrGetValue(MlirAttribute attr) { - return getAlgebraValue(attr); -} - -MlirAttribute mlirCuteStrideAttrGet(MlirContext context, MlirStringRef value) { - return getAlgebraAttribute(context, value); -} - -bool mlirAttributeIsACuteStride(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirStringRef mlirCuteStrideAttrGetValue(MlirAttribute attr) { - return getAlgebraValue(attr); -} - -MlirAttribute mlirCuteLayoutAttrGet(MlirContext context, MlirStringRef value) { - return getAlgebraAttribute(context, value); -} - -bool mlirAttributeIsACuteLayout(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirStringRef mlirCuteLayoutAttrGetValue(MlirAttribute attr) { - return getAlgebraValue(attr); -} - -MlirAttribute mlirCuteTileAttrGet(MlirContext context, MlirStringRef value) { - return getAlgebraAttribute(context, value); -} - -bool mlirAttributeIsACuteTile(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirStringRef mlirCuteTileAttrGetValue(MlirAttribute attr) { - return getAlgebraValue(attr); -} - -MlirAttribute mlirCuteComposedLayoutAttrGet(MlirContext context, - MlirStringRef value) { - return getAlgebraAttribute(context, value); -} - -bool mlirAttributeIsACuteComposedLayout(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirStringRef mlirCuteComposedLayoutAttrGetValue(MlirAttribute attr) { - return getAlgebraValue(attr); -} - -MlirAttribute mlirCuteSwizzleAttrGet(MlirContext context, MlirStringRef value) { - return getAlgebraAttribute(context, value); -} - -bool mlirAttributeIsACuteSwizzle(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirStringRef mlirCuteSwizzleAttrGetValue(MlirAttribute attr) { - return getAlgebraValue(attr); -} diff --git a/mlir/dialects/cute_ir/lib/CAPI/CuteCompilerAttributes.cpp b/mlir/dialects/cute_ir/lib/CAPI/CuteCompilerAttributes.cpp deleted file mode 100644 index d60e59e5..00000000 --- a/mlir/dialects/cute_ir/lib/CAPI/CuteCompilerAttributes.cpp +++ /dev/null @@ -1,672 +0,0 @@ -// C API for the CuTe attributes beyond the algebra, generated from the -// dialect's .td files. -#include "cute_ir-c/Dialect/CuteCompilerAttributes.h" - -#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" -#include "mlir/CAPI/IR.h" - -namespace cute = mlir::cutlass_compiler::cute; - -namespace { -// Unwraps `handle` into `out`: false when it is of another context or not a -// T, or when it is null and not optional. -template -bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { - auto value = unwrap(handle); - if (!value) - return optional; - if (value.getContext() != ctx) - return false; - out = llvm::dyn_cast(value); - return static_cast(out); -} -} // namespace - -bool mlirAttributeIsACuteBitLayout(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteBitLayoutAttrGet(MlirContext context, - MlirAttribute layout) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::LayoutAttr layoutValue; - if (!unwrapAs(ctx, layout, false, layoutValue)) - return {nullptr}; - return wrap(cute::BitLayoutAttr::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - layoutValue)); -} -MlirAttribute mlirCuteBitLayoutAttrGetLayout(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayout())); -} - -bool mlirAttributeIsACuteCopyAtom(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteCopyAtomAttrGet(MlirContext context, MlirAttribute thrId, - MlirAttribute layoutSrc, - MlirAttribute layoutDst, - MlirAttribute layoutRef) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; - if (!unwrapAs(ctx, thrId, false, thrIdValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutSrcValue; - if (!unwrapAs(ctx, layoutSrc, false, layoutSrcValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutDstValue; - if (!unwrapAs(ctx, layoutDst, false, layoutDstValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutRefValue; - if (!unwrapAs(ctx, layoutRef, false, layoutRefValue)) - return {nullptr}; - return wrap(cute::CopyAtomAttr::get(ctx, thrIdValue, layoutSrcValue, - layoutDstValue, layoutRefValue)); -} -MlirAttribute mlirCuteCopyAtomAttrGetThrId(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getThrId())); -} -MlirAttribute mlirCuteCopyAtomAttrGetLayoutSrc(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutSrc())); -} -MlirAttribute mlirCuteCopyAtomAttrGetLayoutDst(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutDst())); -} -MlirAttribute mlirCuteCopyAtomAttrGetLayoutRef(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutRef())); -} - -bool mlirAttributeIsACuteCopyAtomV2(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute -mlirCuteCopyAtomV2AttrGet(MlirContext context, MlirAttribute thrId, - MlirAttribute layoutSrcTV, MlirAttribute layoutDstTV, - MlirAttribute frgSrc, MlirAttribute frgDst) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; - if (!unwrapAs(ctx, thrId, false, thrIdValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutSrcTVValue; - if (!unwrapAs(ctx, layoutSrcTV, false, layoutSrcTVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutDstTVValue; - if (!unwrapAs(ctx, layoutDstTV, false, layoutDstTVValue)) - return {nullptr}; - mlir::Attribute frgSrcValue; - if (!unwrapAs(ctx, frgSrc, false, frgSrcValue)) - return {nullptr}; - mlir::Attribute frgDstValue; - if (!unwrapAs(ctx, frgDst, false, frgDstValue)) - return {nullptr}; - return wrap(cute::CopyAtomV2Attr::get(ctx, thrIdValue, layoutSrcTVValue, - layoutDstTVValue, frgSrcValue, - frgDstValue)); -} -MlirAttribute mlirCuteCopyAtomV2AttrGetThrId(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getThrId())); -} -MlirAttribute mlirCuteCopyAtomV2AttrGetLayoutSrcTV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutSrc_TV())); -} -MlirAttribute mlirCuteCopyAtomV2AttrGetLayoutDstTV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutDst_TV())); -} -MlirAttribute mlirCuteCopyAtomV2AttrGetFrgSrc(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgSrc())); -} -MlirAttribute mlirCuteCopyAtomV2AttrGetFrgDst(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgDst())); -} - -bool mlirAttributeIsACuteMmaAtom(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteMmaAtomAttrGet( - MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, - MlirAttribute layoutATV, MlirAttribute layoutBTV, MlirAttribute layoutCTV, - MlirAttribute shapeAMK, MlirAttribute shapeBNK, MlirAttribute shapeCMN, - MlirAttribute frgA, MlirAttribute frgB, MlirAttribute frgC) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ShapeAttr shapeMNKValue; - if (!unwrapAs(ctx, shapeMNK, false, shapeMNKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; - if (!unwrapAs(ctx, thrId, false, thrIdValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutATVValue; - if (!unwrapAs(ctx, layoutATV, false, layoutATVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutBTVValue; - if (!unwrapAs(ctx, layoutBTV, false, layoutBTVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutCTVValue; - if (!unwrapAs(ctx, layoutCTV, false, layoutCTVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeAMKValue; - if (!unwrapAs(ctx, shapeAMK, false, shapeAMKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeBNKValue; - if (!unwrapAs(ctx, shapeBNK, false, shapeBNKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeCMNValue; - if (!unwrapAs(ctx, shapeCMN, false, shapeCMNValue)) - return {nullptr}; - mlir::Attribute frgAValue; - if (!unwrapAs(ctx, frgA, false, frgAValue)) - return {nullptr}; - mlir::Attribute frgBValue; - if (!unwrapAs(ctx, frgB, false, frgBValue)) - return {nullptr}; - mlir::Attribute frgCValue; - if (!unwrapAs(ctx, frgC, false, frgCValue)) - return {nullptr}; - return wrap(cute::MmaAtomAttr::get( - ctx, shapeMNKValue, thrIdValue, layoutATVValue, layoutBTVValue, - layoutCTVValue, shapeAMKValue, shapeBNKValue, shapeCMNValue, frgAValue, - frgBValue, frgCValue)); -} -MlirAttribute mlirCuteMmaAtomAttrGetShapeMNK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShape_MNK())); -} -MlirAttribute mlirCuteMmaAtomAttrGetThrId(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getThrId())); -} -MlirAttribute mlirCuteMmaAtomAttrGetLayoutATV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutA_TV())); -} -MlirAttribute mlirCuteMmaAtomAttrGetLayoutBTV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutB_TV())); -} -MlirAttribute mlirCuteMmaAtomAttrGetLayoutCTV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutC_TV())); -} -MlirAttribute mlirCuteMmaAtomAttrGetShapeAMK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeA_MK())); -} -MlirAttribute mlirCuteMmaAtomAttrGetShapeBNK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeB_NK())); -} -MlirAttribute mlirCuteMmaAtomAttrGetShapeCMN(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeC_MN())); -} -MlirAttribute mlirCuteMmaAtomAttrGetFrgA(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgA())); -} -MlirAttribute mlirCuteMmaAtomAttrGetFrgB(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgB())); -} -MlirAttribute mlirCuteMmaAtomAttrGetFrgC(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgC())); -} - -bool mlirAttributeIsACuteMxMmaAtom(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteMxMmaAtomAttrGet( - MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, - MlirAttribute layoutATV, MlirAttribute layoutSFATV, MlirAttribute layoutBTV, - MlirAttribute layoutSFBTV, MlirAttribute layoutCTV, MlirAttribute shapeAMK, - MlirAttribute shapeSFAMK, MlirAttribute shapeBNK, MlirAttribute shapeSFBNK, - MlirAttribute shapeCMN, MlirAttribute frgA, MlirAttribute frgSFA, - MlirAttribute frgB, MlirAttribute frgSFB, MlirAttribute frgC) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ShapeAttr shapeMNKValue; - if (!unwrapAs(ctx, shapeMNK, false, shapeMNKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; - if (!unwrapAs(ctx, thrId, false, thrIdValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutATVValue; - if (!unwrapAs(ctx, layoutATV, false, layoutATVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutSFATVValue; - if (!unwrapAs(ctx, layoutSFATV, false, layoutSFATVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutBTVValue; - if (!unwrapAs(ctx, layoutBTV, false, layoutBTVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutSFBTVValue; - if (!unwrapAs(ctx, layoutSFBTV, false, layoutSFBTVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutCTVValue; - if (!unwrapAs(ctx, layoutCTV, false, layoutCTVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeAMKValue; - if (!unwrapAs(ctx, shapeAMK, false, shapeAMKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeSFAMKValue; - if (!unwrapAs(ctx, shapeSFAMK, false, shapeSFAMKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeBNKValue; - if (!unwrapAs(ctx, shapeBNK, false, shapeBNKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeSFBNKValue; - if (!unwrapAs(ctx, shapeSFBNK, false, shapeSFBNKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeCMNValue; - if (!unwrapAs(ctx, shapeCMN, false, shapeCMNValue)) - return {nullptr}; - mlir::Attribute frgAValue; - if (!unwrapAs(ctx, frgA, false, frgAValue)) - return {nullptr}; - mlir::Attribute frgSFAValue; - if (!unwrapAs(ctx, frgSFA, false, frgSFAValue)) - return {nullptr}; - mlir::Attribute frgBValue; - if (!unwrapAs(ctx, frgB, false, frgBValue)) - return {nullptr}; - mlir::Attribute frgSFBValue; - if (!unwrapAs(ctx, frgSFB, false, frgSFBValue)) - return {nullptr}; - mlir::Attribute frgCValue; - if (!unwrapAs(ctx, frgC, false, frgCValue)) - return {nullptr}; - return wrap(cute::MxMmaAtomAttr::get( - ctx, shapeMNKValue, thrIdValue, layoutATVValue, layoutSFATVValue, - layoutBTVValue, layoutSFBTVValue, layoutCTVValue, shapeAMKValue, - shapeSFAMKValue, shapeBNKValue, shapeSFBNKValue, shapeCMNValue, frgAValue, - frgSFAValue, frgBValue, frgSFBValue, frgCValue)); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetShapeMNK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShape_MNK())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetThrId(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getThrId())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutATV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutA_TV())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutSFATV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutSFA_TV())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutBTV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutB_TV())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutSFBTV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutSFB_TV())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutCTV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutC_TV())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetShapeAMK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeA_MK())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetShapeSFAMK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeSFA_MK())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetShapeBNK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeB_NK())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetShapeSFBNK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeSFB_NK())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetShapeCMN(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeC_MN())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetFrgA(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgA())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetFrgSFA(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgSFA())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetFrgB(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgB())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetFrgSFB(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgSFB())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetFrgC(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgC())); -} - -bool mlirAttributeIsACuteReductionOp(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteReductionOpAttrGet(MlirContext context, uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::cute::symbolizeReductionOp(value); - if (!valueValue) - return {nullptr}; - return wrap(cute::ReductionOpAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteReductionOpAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteSparseMmaAtom(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGet( - MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, - MlirAttribute layoutATV, MlirAttribute layoutBTV, MlirAttribute layoutCTV, - MlirAttribute layoutETV, MlirAttribute shapeAMK, MlirAttribute shapeBNK, - MlirAttribute shapeCMN, MlirAttribute shapeEMK, MlirAttribute frgA, - MlirAttribute frgB, MlirAttribute frgC, MlirAttribute frgE) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ShapeAttr shapeMNKValue; - if (!unwrapAs(ctx, shapeMNK, false, shapeMNKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; - if (!unwrapAs(ctx, thrId, false, thrIdValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutATVValue; - if (!unwrapAs(ctx, layoutATV, false, layoutATVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutBTVValue; - if (!unwrapAs(ctx, layoutBTV, false, layoutBTVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutCTVValue; - if (!unwrapAs(ctx, layoutCTV, false, layoutCTVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutETVValue; - if (!unwrapAs(ctx, layoutETV, false, layoutETVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeAMKValue; - if (!unwrapAs(ctx, shapeAMK, false, shapeAMKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeBNKValue; - if (!unwrapAs(ctx, shapeBNK, false, shapeBNKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeCMNValue; - if (!unwrapAs(ctx, shapeCMN, false, shapeCMNValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeEMKValue; - if (!unwrapAs(ctx, shapeEMK, false, shapeEMKValue)) - return {nullptr}; - mlir::Attribute frgAValue; - if (!unwrapAs(ctx, frgA, false, frgAValue)) - return {nullptr}; - mlir::Attribute frgBValue; - if (!unwrapAs(ctx, frgB, false, frgBValue)) - return {nullptr}; - mlir::Attribute frgCValue; - if (!unwrapAs(ctx, frgC, false, frgCValue)) - return {nullptr}; - mlir::Attribute frgEValue; - if (!unwrapAs(ctx, frgE, false, frgEValue)) - return {nullptr}; - return wrap(cute::SparseMmaAtomAttr::get( - ctx, shapeMNKValue, thrIdValue, layoutATVValue, layoutBTVValue, - layoutCTVValue, layoutETVValue, shapeAMKValue, shapeBNKValue, - shapeCMNValue, shapeEMKValue, frgAValue, frgBValue, frgCValue, - frgEValue)); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeMNK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShape_MNK())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetThrId(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getThrId())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetLayoutATV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutA_TV())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetLayoutBTV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutB_TV())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetLayoutCTV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutC_TV())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetLayoutETV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutE_TV())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeAMK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeA_MK())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeBNK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeB_NK())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeCMN(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeC_MN())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeEMK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeE_MK())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetFrgA(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgA())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetFrgB(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgB())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetFrgC(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgC())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetFrgE(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgE())); -} - -bool mlirAttributeIsACuteSparseMxMmaAtom(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGet( - MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, - MlirAttribute layoutATV, MlirAttribute layoutSFATV, MlirAttribute layoutBTV, - MlirAttribute layoutSFBTV, MlirAttribute layoutCTV, MlirAttribute layoutETV, - MlirAttribute shapeAMK, MlirAttribute shapeSFAMK, MlirAttribute shapeBNK, - MlirAttribute shapeSFBNK, MlirAttribute shapeCMN, MlirAttribute shapeEMK, - MlirAttribute frgA, MlirAttribute frgSFA, MlirAttribute frgB, - MlirAttribute frgSFB, MlirAttribute frgC, MlirAttribute frgE) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ShapeAttr shapeMNKValue; - if (!unwrapAs(ctx, shapeMNK, false, shapeMNKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; - if (!unwrapAs(ctx, thrId, false, thrIdValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutATVValue; - if (!unwrapAs(ctx, layoutATV, false, layoutATVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutSFATVValue; - if (!unwrapAs(ctx, layoutSFATV, false, layoutSFATVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutBTVValue; - if (!unwrapAs(ctx, layoutBTV, false, layoutBTVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutSFBTVValue; - if (!unwrapAs(ctx, layoutSFBTV, false, layoutSFBTVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutCTVValue; - if (!unwrapAs(ctx, layoutCTV, false, layoutCTVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutETVValue; - if (!unwrapAs(ctx, layoutETV, false, layoutETVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeAMKValue; - if (!unwrapAs(ctx, shapeAMK, false, shapeAMKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeSFAMKValue; - if (!unwrapAs(ctx, shapeSFAMK, false, shapeSFAMKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeBNKValue; - if (!unwrapAs(ctx, shapeBNK, false, shapeBNKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeSFBNKValue; - if (!unwrapAs(ctx, shapeSFBNK, false, shapeSFBNKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeCMNValue; - if (!unwrapAs(ctx, shapeCMN, false, shapeCMNValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeEMKValue; - if (!unwrapAs(ctx, shapeEMK, false, shapeEMKValue)) - return {nullptr}; - mlir::Attribute frgAValue; - if (!unwrapAs(ctx, frgA, false, frgAValue)) - return {nullptr}; - mlir::Attribute frgSFAValue; - if (!unwrapAs(ctx, frgSFA, false, frgSFAValue)) - return {nullptr}; - mlir::Attribute frgBValue; - if (!unwrapAs(ctx, frgB, false, frgBValue)) - return {nullptr}; - mlir::Attribute frgSFBValue; - if (!unwrapAs(ctx, frgSFB, false, frgSFBValue)) - return {nullptr}; - mlir::Attribute frgCValue; - if (!unwrapAs(ctx, frgC, false, frgCValue)) - return {nullptr}; - mlir::Attribute frgEValue; - if (!unwrapAs(ctx, frgE, false, frgEValue)) - return {nullptr}; - return wrap(cute::SparseMxMmaAtomAttr::get( - ctx, shapeMNKValue, thrIdValue, layoutATVValue, layoutSFATVValue, - layoutBTVValue, layoutSFBTVValue, layoutCTVValue, layoutETVValue, - shapeAMKValue, shapeSFAMKValue, shapeBNKValue, shapeSFBNKValue, - shapeCMNValue, shapeEMKValue, frgAValue, frgSFAValue, frgBValue, - frgSFBValue, frgCValue, frgEValue)); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeMNK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShape_MNK())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetThrId(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getThrId())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutATV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutA_TV())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutSFATV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutSFA_TV())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutBTV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutB_TV())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutSFBTV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutSFB_TV())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutCTV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutC_TV())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutETV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutE_TV())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeAMK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeA_MK())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeSFAMK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeSFA_MK())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeBNK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeB_NK())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeSFBNK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeSFB_NK())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeCMN(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeC_MN())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeEMK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeE_MK())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgA(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgA())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgSFA(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgSFA())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgB(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgB())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgSFB(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgSFB())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgC(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgC())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgE(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgE())); -} diff --git a/mlir/dialects/cute_ir/lib/CAPI/CuteNVGPU.cpp b/mlir/dialects/cute_ir/lib/CAPI/CuteNVGPU.cpp deleted file mode 100644 index 1de2f1df..00000000 --- a/mlir/dialects/cute_ir/lib/CAPI/CuteNVGPU.cpp +++ /dev/null @@ -1,19 +0,0 @@ -//===- CuteNVGPU.cpp - C API for the CuTe NVIDIA GPU dialect ------------===// - -#include "cute_ir-c/Dialect/CuteNVGPU.h" - -#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h" -#include "mlir/CAPI/IR.h" -#include "mlir/CAPI/Registration.h" - -MLIR_DEFINE_CAPI_DIALECT_REGISTRATION( - CuteNVGPU, cute_nvgpu, mlir::cutlass_compiler::cute_nvgpu::CuteNVGPUDialect) - -bool mlirTypeIsACuteNVGPUType(MlirType type) { - mlir::Type cppType = unwrap(type); - if (!cppType) - return false; - if (auto opaque = mlir::dyn_cast(cppType)) - return opaque.getDialectNamespace() == "cute_nvgpu"; - return cppType.getDialect().getNamespace() == "cute_nvgpu"; -} diff --git a/mlir/dialects/cute_ir/lib/CAPI/CuteNVGPUAttributes.cpp b/mlir/dialects/cute_ir/lib/CAPI/CuteNVGPUAttributes.cpp deleted file mode 100644 index e4633940..00000000 --- a/mlir/dialects/cute_ir/lib/CAPI/CuteNVGPUAttributes.cpp +++ /dev/null @@ -1,1183 +0,0 @@ -// C API for the CuTe NVIDIA GPU attributes, generated from the dialect's .td -// files. -#include "cute_ir-c/Dialect/CuteNVGPUAttributes.h" - -#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h" -#include "mlir/CAPI/IR.h" - -namespace cute_nvgpu = mlir::cutlass_compiler::cute_nvgpu; - -namespace { -// Unwraps `handle` into `out`: false when it is of another context or not a -// T, or when it is null and not optional. -template -bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { - auto value = unwrap(handle); - if (!value) - return optional; - if (value.getContext() != ctx) - return false; - out = llvm::dyn_cast(value); - return static_cast(out); -} -} // namespace - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyG2S(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldBulkCopyG2S( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldBulkCopyG2SAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2G(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldBulkCopyS2G( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldBulkCopyS2GAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2S(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldBulkCopyS2S( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldBulkCopyS2SAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldDsmemStore(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldDsmemStore( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldDsmemStoreAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldLoadGlobal(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldLoadGlobal( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldLoadGlobalAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoad( - MlirAttribute attr) { - return llvm::isa_and_nonnull< - cute_nvgpu::AtomCopyFieldNonExec2DGather4TmaLoadAttr>(unwrap(attr)); -} -MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: - symbolizeAtomCopyFieldNonExec2DGather4TmaLoad(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldNonExec2DGather4TmaLoadAttr::get( - ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGetValue( - MlirAttribute attr) { - return static_cast( - llvm::cast( - unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStore( - MlirAttribute attr) { - return llvm::isa_and_nonnull< - cute_nvgpu::AtomCopyFieldNonExec2DScatter4TmaStoreAttr>(unwrap(attr)); -} -MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: - symbolizeAtomCopyFieldNonExec2DScatter4TmaStore(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldNonExec2DScatter4TmaStoreAttr::get( - ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGetValue( - MlirAttribute attr) { - return static_cast( - llvm::cast( - unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoad( - MlirAttribute attr) { - return llvm::isa_and_nonnull< - cute_nvgpu::AtomCopyFieldNonExecIm2ColTmaLoadAttr>(unwrap(attr)); -} -MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: - symbolizeAtomCopyFieldNonExecIm2ColTmaLoad(value); - if (!valueValue) - return {nullptr}; - return wrap( - cute_nvgpu::AtomCopyFieldNonExecIm2ColTmaLoadAttr::get(ctx, *valueValue)); -} -uint32_t -mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast( - unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaStore( - MlirAttribute attr) { - return llvm::isa_and_nonnull< - cute_nvgpu::AtomCopyFieldNonExecIm2ColTmaStoreAttr>(unwrap(attr)); -} -MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: - symbolizeAtomCopyFieldNonExecIm2ColTmaStore(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldNonExecIm2ColTmaStoreAttr::get( - ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGetValue( - MlirAttribute attr) { - return static_cast( - llvm::cast( - unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaLoad( - MlirAttribute attr) { - return llvm::isa_and_nonnull< - cute_nvgpu::AtomCopyFieldNonExecTiledTmaLoadAttr>(unwrap(attr)); -} -MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: - symbolizeAtomCopyFieldNonExecTiledTmaLoad(value); - if (!valueValue) - return {nullptr}; - return wrap( - cute_nvgpu::AtomCopyFieldNonExecTiledTmaLoadAttr::get(ctx, *valueValue)); -} -uint32_t -mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaReduce( - MlirAttribute attr) { - return llvm::isa_and_nonnull< - cute_nvgpu::AtomCopyFieldNonExecTiledTmaReduceAttr>(unwrap(attr)); -} -MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: - symbolizeAtomCopyFieldNonExecTiledTmaReduce(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldNonExecTiledTmaReduceAttr::get( - ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGetValue( - MlirAttribute attr) { - return static_cast( - llvm::cast( - unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaStore( - MlirAttribute attr) { - return llvm::isa_and_nonnull< - cute_nvgpu::AtomCopyFieldNonExecTiledTmaStoreAttr>(unwrap(attr)); -} -MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: - symbolizeAtomCopyFieldNonExecTiledTmaStore(value); - if (!valueValue) - return {nullptr}; - return wrap( - cute_nvgpu::AtomCopyFieldNonExecTiledTmaStoreAttr::get(ctx, *valueValue)); -} -uint32_t -mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast( - unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldStoreGlobal(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldStoreGlobal( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldStoreGlobalAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldTmaLoad(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldTmaLoad( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldTmaLoadAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldTmaReduce(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldTmaReduce( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldTmaReduceAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldTmaStore(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldTmaStore( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldTmaStoreAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM100(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomMmaFieldSM100AttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomMmaFieldSM100(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomMmaFieldSM100Attr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomMmaFieldSM100AttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaled(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute -mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: - symbolizeAtomMmaFieldSM100BlockScaled(value); - if (!valueValue) - return {nullptr}; - return wrap( - cute_nvgpu::AtomMmaFieldSM100BlockScaledAttr::get(ctx, *valueValue)); -} -uint32_t -mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaledSparse( - MlirAttribute attr) { - return llvm::isa_and_nonnull< - cute_nvgpu::AtomMmaFieldSM100BlockScaledSparseAttr>(unwrap(attr)); -} -MlirAttribute -mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: - symbolizeAtomMmaFieldSM100BlockScaledSparse(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomMmaFieldSM100BlockScaledSparseAttr::get( - ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGetValue( - MlirAttribute attr) { - return static_cast( - llvm::cast( - unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM100Sparse(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomMmaFieldSM100Sparse( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomMmaFieldSM100SparseAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM120BlockScaled(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute -mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: - symbolizeAtomMmaFieldSM120BlockScaled(value); - if (!valueValue) - return {nullptr}; - return wrap( - cute_nvgpu::AtomMmaFieldSM120BlockScaledAttr::get(ctx, *valueValue)); -} -uint32_t -mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM80Sparse(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomMmaFieldSM80Sparse( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomMmaFieldSM80SparseAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM90(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomMmaFieldSM90AttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomMmaFieldSM90(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomMmaFieldSM90Attr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomMmaFieldSM90AttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUBinaryOp(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUBinaryOpAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeBinaryOp(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::BinaryOpAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUBinaryOpAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUCopyS2TBroadcast(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUCopyS2TBroadcastAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeCopyS2TBroadcast(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::CopyS2TBroadcastAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUCopyS2TBroadcastAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUGatherScatterTmaLoad(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUGatherScatterTmaLoadAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeGatherScatterTmaLoad( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::GatherScatterTmaLoadAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUGatherScatterTmaLoadAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUIm2ColTmaLoad(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUIm2ColTmaLoadAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeIm2ColTmaLoad(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::Im2ColTmaLoadAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUIm2ColTmaLoadAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPULdReduceAccPrecisionKind(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPULdReduceAccPrecisionKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLdReduceAccPrecisionKind( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::LdReduceAccPrecisionKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPULdReduceAccPrecisionKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPULdsmSzPattern(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPULdsmSzPatternAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLdsmSzPattern(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::LdsmSzPatternAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPULdsmSzPatternAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPULoadCacheMode(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPULoadCacheModeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLoadCacheMode(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::LoadCacheModeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPULoadCacheModeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUMMAIntOverflow(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUMMAIntOverflowAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMMAIntOverflow(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::MMAIntOverflowAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUMMAIntOverflowAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUMajorMode(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUMajorModeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::MajorModeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUMajorModeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUMmaCollectorOp(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUMmaCollectorOpAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::MmaCollectorOpAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUMmaCollectorOpAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUMmaFragKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUMmaFragKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::MmaFragKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUMmaFragKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUNotImplementedFrg(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUNotImplementedFrgAttrGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cute_nvgpu::NotImplementedFrgAttr::get(ctx)); -} - -bool mlirAttributeIsACuteNVGPUReductionKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUReductionKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeReductionKind(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::ReductionKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUReductionKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPURmemFrg(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPURmemFrgAttrGet(MlirContext context, - MlirType valueType, uint32_t operand, - bool deriveElemType) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valueTypeValue; - if (!unwrapAs(ctx, valueType, false, valueTypeValue)) - return {nullptr}; - auto operandValue = - ::mlir::cutlass_compiler::cute::symbolizeMmaOperand(operand); - if (!operandValue) - return {nullptr}; - return wrap(cute_nvgpu::RmemFrgAttr::get(ctx, valueTypeValue, *operandValue, - deriveElemType)); -} -MlirType mlirCuteNVGPURmemFrgAttrGetValueType(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getValueType())); -} -uint32_t mlirCuteNVGPURmemFrgAttrGetOperand(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getOperand()); -} -bool mlirCuteNVGPURmemFrgAttrGetDeriveElemType(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getDeriveElemType(); -} - -bool mlirAttributeIsACuteNVGPUSM100CircularSmemFrg(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUSM100CircularSmemFrgAttrGet(MlirContext context, - uint32_t majorMode) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto majorModeValue = - ::mlir::cutlass_compiler::cute::symbolizeMajorMode(majorMode); - if (!majorModeValue) - return {nullptr}; - return wrap(cute_nvgpu::SM100CircularSmemFrgAttr::get(ctx, *majorModeValue)); -} -uint32_t mlirCuteNVGPUSM100CircularSmemFrgAttrGetMajorMode(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getMajorMode()); -} - -bool mlirAttributeIsACuteNVGPUSM100SmemFrg(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUSM100SmemFrgAttrGet(MlirContext context, - uint32_t majorMode) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto majorModeValue = - ::mlir::cutlass_compiler::cute::symbolizeMajorMode(majorMode); - if (!majorModeValue) - return {nullptr}; - return wrap(cute_nvgpu::SM100SmemFrgAttr::get(ctx, *majorModeValue)); -} -uint32_t mlirCuteNVGPUSM100SmemFrgAttrGetMajorMode(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getMajorMode()); -} - -bool mlirAttributeIsACuteNVGPUSM100TmemEFrg(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUSM100TmemEFrgAttrGet(MlirContext context, - MlirType aType, - MlirType eType) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type aTypeValue; - if (!unwrapAs(ctx, aType, false, aTypeValue)) - return {nullptr}; - mlir::Type eTypeValue; - if (!unwrapAs(ctx, eType, false, eTypeValue)) - return {nullptr}; - return wrap(cute_nvgpu::SM100TmemEFrgAttr::get(ctx, aTypeValue, eTypeValue)); -} -MlirType mlirCuteNVGPUSM100TmemEFrgAttrGetAType(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getAType())); -} -MlirType mlirCuteNVGPUSM100TmemEFrgAttrGetEType(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getEType())); -} - -bool mlirAttributeIsACuteNVGPUSM100TmemFrg(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUSM100TmemFrgAttrGet(MlirContext context, - MlirType dataType, - MlirType storageType, - int ctaGroup, - uint32_t tmemAllocMode) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type dataTypeValue; - if (!unwrapAs(ctx, dataType, false, dataTypeValue)) - return {nullptr}; - mlir::Type storageTypeValue; - if (!unwrapAs(ctx, storageType, false, storageTypeValue)) - return {nullptr}; - auto tmemAllocModeValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemAllocMode( - tmemAllocMode); - if (!tmemAllocModeValue) - return {nullptr}; - return wrap(cute_nvgpu::SM100TmemFrgAttr::get( - ctx, dataTypeValue, storageTypeValue, ctaGroup, *tmemAllocModeValue)); -} -MlirType mlirCuteNVGPUSM100TmemFrgAttrGetDataType(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getDataType())); -} -MlirType mlirCuteNVGPUSM100TmemFrgAttrGetStorageType(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getStorageType())); -} -int mlirCuteNVGPUSM100TmemFrgAttrGetCtaGroup(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getCtaGroup(); -} -uint32_t mlirCuteNVGPUSM100TmemFrgAttrGetTmemAllocMode(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getTmemAllocMode()); -} - -bool mlirAttributeIsACuteNVGPUSM100TmemSfFrg(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUSM100TmemSfFrgAttrGet(MlirContext context, - MlirType sfType, int sfVecSize, - int ctaGroup, bool isSfa, - uint32_t tmemAllocMode) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type sfTypeValue; - if (!unwrapAs(ctx, sfType, false, sfTypeValue)) - return {nullptr}; - auto tmemAllocModeValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemAllocMode( - tmemAllocMode); - if (!tmemAllocModeValue) - return {nullptr}; - return wrap(cute_nvgpu::SM100TmemSfFrgAttr::get( - ctx, sfTypeValue, sfVecSize, ctaGroup, isSfa, *tmemAllocModeValue)); -} -MlirType mlirCuteNVGPUSM100TmemSfFrgAttrGetSfType(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getSfType())); -} -int mlirCuteNVGPUSM100TmemSfFrgAttrGetSfVecSize(MlirAttribute attr) { - return llvm::cast(unwrap(attr)) - .getSfVecSize(); -} -int mlirCuteNVGPUSM100TmemSfFrgAttrGetCtaGroup(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getCtaGroup(); -} -bool mlirCuteNVGPUSM100TmemSfFrgAttrGetIsSfa(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getIsSfa(); -} -uint32_t mlirCuteNVGPUSM100TmemSfFrgAttrGetTmemAllocMode(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getTmemAllocMode()); -} - -bool mlirAttributeIsACuteNVGPUSM107SmemFrg(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUSM107SmemFrgAttrGet(MlirContext context, - uint32_t majorMode) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto majorModeValue = - ::mlir::cutlass_compiler::cute::symbolizeMajorMode(majorMode); - if (!majorModeValue) - return {nullptr}; - return wrap(cute_nvgpu::SM107SmemFrgAttr::get(ctx, *majorModeValue)); -} -uint32_t mlirCuteNVGPUSM107SmemFrgAttrGetMajorMode(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getMajorMode()); -} - -bool mlirAttributeIsACuteNVGPUSM90SmemFrg(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUSM90SmemFrgAttrGet(MlirContext context, - uint32_t majorMode) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto majorModeValue = - ::mlir::cutlass_compiler::cute::symbolizeMajorMode(majorMode); - if (!majorModeValue) - return {nullptr}; - return wrap(cute_nvgpu::SM90SmemFrgAttr::get(ctx, *majorModeValue)); -} -uint32_t mlirCuteNVGPUSM90SmemFrgAttrGetMajorMode(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getMajorMode()); -} - -bool mlirAttributeIsACuteNVGPUTiledTmaLoad(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUTiledTmaLoadAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTiledTmaLoad(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::TiledTmaLoadAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUTiledTmaLoadAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUTmaDataFormat(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUTmaDataFormatAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::TmaDataFormatAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUTmaDataFormatAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUTmaLoadMode(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUTmaLoadModeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaLoadMode(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::TmaLoadModeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUTmaLoadModeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUTmaStoreMode(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUTmaStoreModeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaStoreMode(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::TmaStoreModeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUTmaStoreModeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUTmemAllocMode(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUTmemAllocModeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemAllocMode(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::TmemAllocModeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUTmemAllocModeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUTmemLoadRedOp(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUTmemLoadRedOpAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemLoadRedOp(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::TmemLoadRedOpAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUTmemLoadRedOpAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} diff --git a/mlir/dialects/cute_ir/lib/CAPI/CuteTypes.cpp b/mlir/dialects/cute_ir/lib/CAPI/CuteTypes.cpp deleted file mode 100644 index 986cbb09..00000000 --- a/mlir/dialects/cute_ir/lib/CAPI/CuteTypes.cpp +++ /dev/null @@ -1,508 +0,0 @@ -// C API for the CuTe types, generated from the dialect's .td files. -#include "cute_ir-c/Dialect/CuteTypes.h" - -#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" -#include "mlir/CAPI/IR.h" - -namespace cute = mlir::cutlass_compiler::cute; - -namespace { -// Unwraps `handle` into `out`: false when it is of another context or not a -// T, or when it is null and not optional. -template -bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { - auto value = unwrap(handle); - if (!value) - return optional; - if (value.getContext() != ctx) - return false; - out = llvm::dyn_cast(value); - return static_cast(out); -} -} // namespace - -bool mlirTypeIsACuteArithTupleIterator(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteArithTupleIteratorTypeGet(MlirContext context, - MlirType arithTuple) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::IntTupleType arithTupleValue; - if (!unwrapAs(ctx, arithTuple, false, arithTupleValue)) - return {nullptr}; - return wrap(cute::ArithTupleIteratorType::get(ctx, arithTupleValue)); -} -MlirType mlirCuteArithTupleIteratorTypeGetArithTuple(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getArithTuple())); -} - -bool mlirTypeIsACuteComposedLayout(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteComposedLayoutTypeGet(MlirContext context, - MlirAttribute attr) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ComposedLayoutAttr attrValue; - if (!unwrapAs(ctx, attr, false, attrValue)) - return {nullptr}; - return wrap(cute::ComposedLayoutType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - attrValue)); -} -MlirAttribute mlirCuteComposedLayoutTypeGetAttr(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getAttr())); -} - -bool mlirTypeIsACuteConstrainedInt(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteConstrainedIntTypeGet(MlirContext context, - int64_t divisibility, unsigned width, - bool isPow2) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cute::ConstrainedIntType::get(ctx, divisibility, width, isPow2)); -} -int64_t mlirCuteConstrainedIntTypeGetDivisibility(MlirType type) { - return llvm::cast(unwrap(type)).getDivisibility(); -} -unsigned mlirCuteConstrainedIntTypeGetWidth(MlirType type) { - return llvm::cast(unwrap(type)).getWidth(); -} -bool mlirCuteConstrainedIntTypeGetIsPow2(MlirType type) { - return llvm::cast(unwrap(type)).getIsPow2(); -} - -bool mlirTypeIsACuteCoordTensor(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteCoordTensorTypeGet(MlirContext context, MlirType arithTuple, - MlirType layout) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::IntTupleType arithTupleValue; - if (!unwrapAs(ctx, arithTuple, false, arithTupleValue)) - return {nullptr}; - mlir::Type layoutValue; - if (!unwrapAs(ctx, layout, false, layoutValue)) - return {nullptr}; - return wrap(cute::CoordTensorType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - arithTupleValue, layoutValue)); -} -MlirType mlirCuteCoordTensorTypeGetArithTuple(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getArithTuple())); -} -MlirType mlirCuteCoordTensorTypeGetLayout(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getLayout())); -} - -bool mlirTypeIsACuteCoord(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteCoordTypeGet(MlirContext context, MlirAttribute attr) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::CoordAttr attrValue; - if (!unwrapAs(ctx, attr, false, attrValue)) - return {nullptr}; - return wrap(cute::CoordType::get(ctx, attrValue)); -} -MlirAttribute mlirCuteCoordTypeGetAttr(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getAttr())); -} - -bool mlirTypeIsACuteFastDivmodDivisor(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteFastDivmodDivisorTypeGet(MlirContext context, unsigned width, - bool isPow2) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cute::FastDivmodDivisorType::get(ctx, width, isPow2)); -} -unsigned mlirCuteFastDivmodDivisorTypeGetWidth(MlirType type) { - return llvm::cast(unwrap(type)).getWidth(); -} -bool mlirCuteFastDivmodDivisorTypeGetIsPow2(MlirType type) { - return llvm::cast(unwrap(type)).getIsPow_2(); -} - -bool mlirTypeIsACuteIntTuple(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteIntTupleTypeGet(MlirContext context, MlirAttribute attr) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::IntTupleAttr attrValue; - if (!unwrapAs(ctx, attr, false, attrValue)) - return {nullptr}; - return wrap(cute::IntTupleType::get(ctx, attrValue)); -} -MlirAttribute mlirCuteIntTupleTypeGetAttr(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getAttr())); -} - -bool mlirTypeIsACuteLayout(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteLayoutTypeGet(MlirContext context, MlirAttribute attr) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::LayoutAttr attrValue; - if (!unwrapAs(ctx, attr, false, attrValue)) - return {nullptr}; - return wrap(cute::LayoutType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - attrValue)); -} -MlirAttribute mlirCuteLayoutTypeGetAttr(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getAttr())); -} - -bool mlirTypeIsACuteMemRef(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteMemRefTypeGet(MlirContext context, MlirType ptr, - MlirType layout) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::PtrType ptrValue; - if (!unwrapAs(ctx, ptr, false, ptrValue)) - return {nullptr}; - mlir::Type layoutValue; - if (!unwrapAs(ctx, layout, false, layoutValue)) - return {nullptr}; - return wrap(cute::MemRefType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - ptrValue, layoutValue)); -} -MlirType mlirCuteMemRefTypeGetPtr(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getPtr())); -} -MlirType mlirCuteMemRefTypeGetLayout(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getLayout())); -} - -bool mlirTypeIsACutePtr(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCutePtrTypeGet(MlirContext context, MlirType valueType, - uint32_t addressSpace, uint64_t alignment, - MlirAttribute swizzle, MlirAttribute bitlayout) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valueTypeValue; - if (!unwrapAs(ctx, valueType, true, valueTypeValue)) - return {nullptr}; - auto addressSpaceValue = - ::mlir::cutlass_compiler::cute::symbolizeAddressSpace(addressSpace); - if (!addressSpaceValue) - return {nullptr}; - ::mlir::cutlass_compiler::cute::SwizzleAttr swizzleValue; - if (!unwrapAs(ctx, swizzle, true, swizzleValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::BitLayoutAttr bitlayoutValue; - if (!unwrapAs(ctx, bitlayout, true, bitlayoutValue)) - return {nullptr}; - return wrap(cute::PtrType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - valueTypeValue, *addressSpaceValue, alignment, swizzleValue, - bitlayoutValue)); -} -MlirType mlirCutePtrTypeGetValueType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getValueType())); -} -uint32_t mlirCutePtrTypeGetAddressSpace(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getAddressSpace()); -} -uint64_t mlirCutePtrTypeGetAlignment(MlirType type) { - return llvm::cast(unwrap(type)).getAlignment(); -} -MlirAttribute mlirCutePtrTypeGetSwizzle(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getSwizzle())); -} -MlirAttribute mlirCutePtrTypeGetBitlayout(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getBitlayout())); -} - -bool mlirTypeIsACuteShape(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteShapeTypeGet(MlirContext context, MlirAttribute attr) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ShapeAttr attrValue; - if (!unwrapAs(ctx, attr, false, attrValue)) - return {nullptr}; - return wrap(cute::ShapeType::get(ctx, attrValue)); -} -MlirAttribute mlirCuteShapeTypeGetAttr(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getAttr())); -} - -bool mlirTypeIsACuteSparseElem(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteSparseElemTypeGet(MlirContext context, int numLogical, - MlirType physicalType) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type physicalTypeValue; - if (!unwrapAs(ctx, physicalType, false, physicalTypeValue)) - return {nullptr}; - return wrap(cute::SparseElemType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - numLogical, physicalTypeValue)); -} -int mlirCuteSparseElemTypeGetNumLogical(MlirType type) { - return llvm::cast(unwrap(type)).getNumLogical(); -} -MlirType mlirCuteSparseElemTypeGetPhysicalType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getPhysicalType())); -} - -bool mlirTypeIsACuteStride(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteStrideTypeGet(MlirContext context, MlirAttribute attr) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::StrideAttr attrValue; - if (!unwrapAs(ctx, attr, false, attrValue)) - return {nullptr}; - return wrap(cute::StrideType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - attrValue)); -} -MlirAttribute mlirCuteStrideTypeGetAttr(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getAttr())); -} - -bool mlirTypeIsACuteSwizzle(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteSwizzleTypeGet(MlirContext context, MlirAttribute attr) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::SwizzleAttr attrValue; - if (!unwrapAs(ctx, attr, false, attrValue)) - return {nullptr}; - return wrap(cute::SwizzleType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - attrValue)); -} -MlirAttribute mlirCuteSwizzleTypeGetAttr(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getAttr())); -} - -bool mlirTypeIsACuteTile(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteTileTypeGet(MlirContext context, MlirAttribute attr) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::TileAttr attrValue; - if (!unwrapAs(ctx, attr, false, attrValue)) - return {nullptr}; - return wrap(cute::TileType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - attrValue)); -} -MlirAttribute mlirCuteTileTypeGetAttr(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getAttr())); -} - -bool mlirTypeIsACuteTiledCopy(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteTiledCopyTypeGet(MlirContext context, MlirType copyAtom, - MlirAttribute layoutCopyTv, - MlirAttribute tilerMn) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type copyAtomValue; - if (!unwrapAs(ctx, copyAtom, false, copyAtomValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutCopyTvValue; - if (!unwrapAs(ctx, layoutCopyTv, false, layoutCopyTvValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::TileAttr tilerMnValue; - if (!unwrapAs(ctx, tilerMn, false, tilerMnValue)) - return {nullptr}; - return wrap(cute::TiledCopyType::get(ctx, copyAtomValue, layoutCopyTvValue, - tilerMnValue)); -} -MlirType mlirCuteTiledCopyTypeGetCopyAtom(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getCopyAtom())); -} -MlirType mlirCuteTiledCopyTypeGetAtom(MlirType type) { - return mlirCuteTiledCopyTypeGetCopyAtom(type); -} -MlirAttribute mlirCuteTiledCopyTypeGetLayoutCopyTv(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getLayoutCopyTv())); -} -MlirAttribute mlirCuteTiledCopyTypeGetTilerMn(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getTilerMn())); -} - -bool mlirTypeIsACuteTiledCopyV2(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteTiledCopyV2TypeGet(MlirContext context, MlirType copyAtom, - MlirAttribute layoutCopyTv, - MlirAttribute tilerMn) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type copyAtomValue; - if (!unwrapAs(ctx, copyAtom, false, copyAtomValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutCopyTvValue; - if (!unwrapAs(ctx, layoutCopyTv, false, layoutCopyTvValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::TileAttr tilerMnValue; - if (!unwrapAs(ctx, tilerMn, false, tilerMnValue)) - return {nullptr}; - return wrap(cute::TiledCopyV2Type::get(ctx, copyAtomValue, layoutCopyTvValue, - tilerMnValue)); -} -MlirType mlirCuteTiledCopyV2TypeGetCopyAtom(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getCopyAtom())); -} -MlirAttribute mlirCuteTiledCopyV2TypeGetLayoutCopyTv(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getLayoutCopyTv())); -} -MlirAttribute mlirCuteTiledCopyV2TypeGetTilerMn(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getTilerMn())); -} - -bool mlirTypeIsACuteTiledMma(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteTiledMmaTypeGet(MlirContext context, MlirType mmaAtom, - MlirAttribute atomLayoutMNK, - MlirAttribute permutationMNK) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type mmaAtomValue; - if (!unwrapAs(ctx, mmaAtom, false, mmaAtomValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr atomLayoutMNKValue; - if (!unwrapAs(ctx, atomLayoutMNK, false, atomLayoutMNKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::TileAttr permutationMNKValue; - if (!unwrapAs(ctx, permutationMNK, true, permutationMNKValue)) - return {nullptr}; - return wrap(cute::TiledMmaType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - mmaAtomValue, atomLayoutMNKValue, permutationMNKValue)); -} -MlirType mlirCuteTiledMmaTypeGetMmaAtom(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getMmaAtom())); -} -MlirType mlirCuteTiledMmaTypeGetAtom(MlirType type) { - return mlirCuteTiledMmaTypeGetMmaAtom(type); -} -MlirAttribute mlirCuteTiledMmaTypeGetAtomLayoutMNK(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getAtomLayout_MNK())); -} -MlirAttribute mlirCuteTiledMmaTypeGetAtomLayoutMnk(MlirType type) { - return mlirCuteTiledMmaTypeGetAtomLayoutMNK(type); -} -MlirAttribute mlirCuteTiledMmaTypeGetPermutationMNK(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getPermutation_MNK())); -} -MlirAttribute mlirCuteTiledMmaTypeGetPermutationMnk(MlirType type) { - return mlirCuteTiledMmaTypeGetPermutationMNK(type); -} - -bool mlirTypeIsACuteTuple(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteTupleTypeGet(MlirContext context, intptr_t numTypes, - MlirType const *types) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - llvm::SmallVector typesValue; - for (intptr_t i = 0; i < numTypes; ++i) { - mlir::Type element; - if (!unwrapAs(ctx, types[i], false, element)) - return {nullptr}; - typesValue.push_back(element); - } - return wrap(cute::TupleType::get(ctx, typesValue)); -} -intptr_t mlirCuteTupleTypeGetNumTypes(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getTypes().size()); -} -MlirType mlirCuteTupleTypeGetType(MlirType type, intptr_t pos) { - return wrap(llvm::cast(unwrap(type)).getTypes()[pos]); -} diff --git a/mlir/dialects/cute_ir/lib/CAPI/NVVM.cpp b/mlir/dialects/cute_ir/lib/CAPI/NVVM.cpp deleted file mode 100644 index e290c183..00000000 --- a/mlir/dialects/cute_ir/lib/CAPI/NVVM.cpp +++ /dev/null @@ -1,10 +0,0 @@ -//===- NVVM.cpp - C API for NVIDIA's NVVM dialect of the CuTe DSL ------===// - -#include "cute_ir-c/Dialect/NVVM.h" - -#include "cute_ir/Dialect/NVVM/IR/NVVMDialect.h" -#include "mlir/CAPI/IR.h" -#include "mlir/CAPI/Registration.h" - -MLIR_DEFINE_CAPI_DIALECT_REGISTRATION( - CuteNVVM, cute_nvvm, mlir::cutlass_compiler::nvvm::NVVMDialect) diff --git a/mlir/dialects/cute_ir/lib/Dialect/Cuda/IR/BUILD.bazel b/mlir/dialects/cute_ir/lib/Dialect/Cuda/IR/BUILD.bazel deleted file mode 100644 index 946266c0..00000000 --- a/mlir/dialects/cute_ir/lib/Dialect/Cuda/IR/BUILD.bazel +++ /dev/null @@ -1,18 +0,0 @@ -load("@rules_cc//cc:defs.bzl", "cc_library") - -package(default_visibility = ["//visibility:public"]) - -cc_library( - name = "dialect", - srcs = [ - "CudaDialect.cpp", - "CudaInference.cpp", - ], - deps = [ - "//mlir/dialects/cute_ir/include/cute_ir/Dialect/Cuda/IR:headers", - "@llvm-project//llvm:Support", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:InferTypeOpInterface", - "@llvm-project//mlir:LLVMDialect", - ], -) diff --git a/mlir/dialects/cute_ir/lib/Dialect/CuteNVGPU/IR/BUILD.bazel b/mlir/dialects/cute_ir/lib/Dialect/CuteNVGPU/IR/BUILD.bazel deleted file mode 100644 index 708508fa..00000000 --- a/mlir/dialects/cute_ir/lib/Dialect/CuteNVGPU/IR/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -load("@rules_cc//cc:defs.bzl", "cc_library") - -package(default_visibility = ["//visibility:public"]) - -cc_library( - name = "dialect", - srcs = ["CuteNVGPUDialect.cpp"], - deps = [ - "//mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR:headers", - "//mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR:headers", - "@llvm-project//llvm:Support", - "@llvm-project//mlir:AsmParser", - "@llvm-project//mlir:IR", - ], -) diff --git a/mlir/dialects/cute_ir/lib/Dialect/IR/BUILD.bazel b/mlir/dialects/cute_ir/lib/Dialect/IR/BUILD.bazel deleted file mode 100644 index ba8a35ff..00000000 --- a/mlir/dialects/cute_ir/lib/Dialect/IR/BUILD.bazel +++ /dev/null @@ -1,30 +0,0 @@ -load("@rules_cc//cc:defs.bzl", "cc_library") - -package(default_visibility = ["//visibility:public"]) - -# CuteTypes.cpp is the release's with the compiler's types, attributes and -# enums added and ratio strides in scaled bases allowed (`1/2@0`, how a 2-CTA -# TMA partition addresses its halves). -cc_library( - name = "dialect", - srcs = [ - "CuteInference.cpp", - "CuteOps.cpp", - "CuteTypes.cpp", - "@cute_ir//:cute_ir/lib/Dialect/IR/CuteAttributes.cpp", - "@cute_ir//:cute_ir/lib/Dialect/IR/CuteDialect.cpp", - ], - deps = [ - "//mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR:headers", - "//mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR:headers", - "@llvm-project//llvm:Support", - "@llvm-project//mlir:ArithDialect", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:InferTypeOpInterface", - "@llvm-project//mlir:LLVMDialect", - "@llvm-project//mlir:MathDialect", - "@llvm-project//mlir:NVVMDialect", - "@llvm-project//mlir:SCFDialect", - "@llvm-project//mlir:UBDialect", - ], -) diff --git a/mlir/dialects/cute_ir/lib/Dialect/NVVM/IR/BUILD.bazel b/mlir/dialects/cute_ir/lib/Dialect/NVVM/IR/BUILD.bazel deleted file mode 100644 index dddca500..00000000 --- a/mlir/dialects/cute_ir/lib/Dialect/NVVM/IR/BUILD.bazel +++ /dev/null @@ -1,18 +0,0 @@ -load("@rules_cc//cc:defs.bzl", "cc_library") - -package(default_visibility = ["//visibility:public"]) - -cc_library( - name = "dialect", - srcs = [ - "NVVMDialect.cpp", - "NVVMInference.cpp", - ], - deps = [ - "//mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR:headers", - "@llvm-project//llvm:Support", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:InferTypeOpInterface", - "@llvm-project//mlir:LLVMDialect", - ], -) diff --git a/mlir/dialects/cute_ir/lib/Dialect/NVVM/IR/NVVMInference.cpp b/mlir/dialects/cute_ir/lib/Dialect/NVVM/IR/NVVMInference.cpp deleted file mode 100644 index b0ae6e5a..00000000 --- a/mlir/dialects/cute_ir/lib/Dialect/NVVM/IR/NVVMInference.cpp +++ /dev/null @@ -1,227 +0,0 @@ -//===- NVVMInference.cpp - result types the compiler infers ------------===// -// -// Generated from the nvidia-cutlass-dsl 4.8.0 compiler (its inference -// probed by varying operand types); do not edit. An operation without a -// recovered rule fails inference, so the result types written stand. - -#include "cute_ir/Dialect/NVVM/IR/NVVMDialect.h" - -using namespace mlir; -using namespace ::mlir::cutlass_compiler::nvvm; - -LogicalResult AddFOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - Adaptor adaptor(operands, attrs, props, regions); - auto group = adaptor.getODSOperands(0); - if (group.empty()) return failure(); - out.push_back(group.front().getType()); - return success(); -} - -LogicalResult AddPackedF32x2Op::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult AtomicRMWOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult BarrierOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult ClusterLaunchControlQueryCancelOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult CompareAndSetOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - Adaptor adaptor(operands, attrs, props, regions); - auto group = adaptor.getODSOperands(0); - if (group.empty()) return failure(); - out.push_back(group.front().getType()); - return success(); -} - -LogicalResult FAbsOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - Adaptor adaptor(operands, attrs, props, regions); - auto group = adaptor.getODSOperands(0); - if (group.empty()) return failure(); - out.push_back(group.front().getType()); - return success(); -} - -LogicalResult FMAPackedF32x2Op::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult FmaOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult InternalMmaBlockScaleOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - Adaptor adaptor(operands, attrs, props, regions); - auto group = adaptor.getODSOperands(8); - if (group.empty()) return failure(); - out.push_back(group.front().getType()); - return success(); -} - -LogicalResult InternalMmaSparseBlockScaleOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - Adaptor adaptor(operands, attrs, props, regions); - auto group = adaptor.getODSOperands(8); - if (group.empty()) return failure(); - out.push_back(group.front().getType()); - return success(); -} - -LogicalResult LdMatrixOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult MBarrierArriveDropExpectTxOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult MBarrierArriveDropOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult MBarrierArriveExpectTxOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult MBarrierArriveOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult MBarrierTestWaitOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult MBarrierTryWaitOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult MatchSyncOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult MulOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult MulPackedF32x2Op::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult ReduxOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult RsqrtOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - Adaptor adaptor(operands, attrs, props, regions); - auto group = adaptor.getODSOperands(0); - if (group.empty()) return failure(); - out.push_back(group.front().getType()); - return success(); -} - -LogicalResult ShflOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult SubFOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - Adaptor adaptor(operands, attrs, props, regions); - auto group = adaptor.getODSOperands(0); - if (group.empty()) return failure(); - out.push_back(group.front().getType()); - return success(); -} - -LogicalResult SubPackedF32x2Op::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult VoteSyncOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} diff --git a/mlir/dialects/cute_ir/test.zig b/mlir/dialects/cute_ir/test.zig new file mode 100644 index 00000000..183c884e --- /dev/null +++ b/mlir/dialects/cute_ir/test.zig @@ -0,0 +1,332 @@ +const std = @import("std"); + +const cute_ir = @import("mlir/dialects/cute_ir"); +const cute_nvgpu = @import("mlir/dialects/cute_nvgpu"); +const mlir = @import("mlir"); + +const ArithTupleIteratorType = cute_ir.ArithTupleIteratorType; +const ComposedLayoutType = cute_ir.ComposedLayoutType; +const ConstrainedIntType = cute_ir.ConstrainedIntType; +const CoordTensorType = cute_ir.CoordTensorType; +const CoordType = cute_ir.CoordType; +const FastDivmodDivisorType = cute_ir.FastDivmodDivisorType; +const IntTupleType = cute_ir.IntTupleType; +const LayoutType = cute_ir.LayoutType; +const MemRefType = cute_ir.MemRefType; +const PtrType = cute_ir.PtrType; +const ShapeType = cute_ir.ShapeType; +const SparseElemType = cute_ir.SparseElemType; +const StrideType = cute_ir.StrideType; +const SwizzleType = cute_ir.SwizzleType; +const TileType = cute_ir.TileType; +const TiledCopyType = cute_ir.TiledCopyType; +const TiledCopyV2Type = cute_ir.TiledCopyV2Type; +const TiledMmaType = cute_ir.TiledMmaType; +const TupleType = cute_ir.TupleType; +const BitLayoutAttr = cute_ir.BitLayoutAttr; +const CopyAtomAttr = cute_ir.CopyAtomAttr; +const CopyAtomV2Attr = cute_ir.CopyAtomV2Attr; +const MmaAtomAttr = cute_ir.MmaAtomAttr; +const MxMmaAtomAttr = cute_ir.MxMmaAtomAttr; +const SparseMmaAtomAttr = cute_ir.SparseMmaAtomAttr; +const SparseMxMmaAtomAttr = cute_ir.SparseMxMmaAtomAttr; + +const operation_names: []const []const u8 = &.{ + "cute.add_offset", + "cute.append_to_rank", + "cute.apply_swizzle", + "cute.assume", + "cute.blocked_product", + "cute.ceil_div", + "cute.coalesce", + "cute.complement", + "cute.composed_get_inner", + "cute.composed_get_offset", + "cute.composed_get_outer", + "cute.composition", + "cute.copy", + "cute.copy.make_fragment", + "cute.copy_atom_call", + "cute.cosize", + "cute.crd2idx", + "cute.deref_arith_tuple_iter", + "cute.deref_desc_iter", + "cute.derefine", + "cute.dice", + "cute.elem_less", + "cute.equal", + "cute.fast_divmod.compute", + "cute.fast_divmod.create_divisor", + "cute.fast_divmod.divide", + "cute.fast_divmod.get_aux", + "cute.fast_divmod.get_divisor", + "cute.fast_divmod.make_divisor", + "cute.filter", + "cute.filter_zeros", + "cute.flat_divide", + "cute.flat_product", + "cute.gemm", + "cute.get", + "cute.get_flat_coord", + "cute.get_hier_coord", + "cute.get_integral_coord", + "cute.get_iter", + "cute.get_layout", + "cute.get_layouts_from_tile", + "cute.get_leaves", + "cute.get_scalars", + "cute.get_shape", + "cute.get_stride", + "cute.group_modes", + "cute.idx2crd", + "cute.increment_coord", + "cute.inttoptr", + "cute.kernel_smem_size", + "cute.left_inverse", + "cute.load_scaled_index", + "cute.local_partition", + "cute.local_tile", + "cute.logical_divide", + "cute.logical_product", + "cute.make_arith_tuple_iter", + "cute.make_atom", + "cute.make_composed_layout", + "cute.make_coord", + "cute.make_desc_iter", + "cute.make_fragment_like", + "cute.make_identity_layout", + "cute.make_identity_tensor", + "cute.make_int_tuple", + "cute.make_layout", + "cute.make_layout_like", + "cute.make_ordered_layout", + "cute.make_shape", + "cute.make_sparse_elem", + "cute.make_stride", + "cute.make_tile", + "cute.make_tiled_copy", + "cute.make_tiled_copy_v2", + "cute.make_tiled_mma", + "cute.make_tuple", + "cute.make_view", + "cute.memref.alloc_smem", + "cute.memref.alloca", + "cute.memref.load", + "cute.memref.load_vec", + "cute.memref.store", + "cute.memref.store_vec", + "cute.mma.make_fragment", + "cute.mma_atom_call", + "cute.prefetch", + "cute.prefetch_atom_call", + "cute.prepend_to_rank", + "cute.print", + "cute.print_tma_desc_im2col", + "cute.print_tma_desc_tiled", + "cute.print_view", + "cute.ptr.load", + "cute.ptr.store", + "cute.ptrtoint", + "cute.raked_product", + "cute.random_delay", + "cute.recast_iter", + "cute.recast_layout", + "cute.reduce", + "cute.right_inverse", + "cute.select", + "cute.shape_div", + "cute.size", + "cute.slice", + "cute.smem_partition_size", + "cute.static", + "cute.stencil_divide", + "cute.symbolic", + "cute.tile_to_shape", + "cute.tiled.copy.partition_D", + "cute.tiled.copy.partition_S", + "cute.tiled.copy.retile", + "cute.tiled.mma.partition", + "cute.tiled.mma.partition_shape", + "cute.tiled_divide", + "cute.tiled_product", + "cute.to_int_tuple", + "cute.tuple.product", + "cute.tuple.product_each", + "cute.tuple_add", + "cute.tuple_div", + "cute.tuple_mod", + "cute.tuple_mul", + "cute.tuple_sub", + "cute.unpack_tuple", + "cute.zipped_divide", + "cute.zipped_product", +}; + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + cute_ir.registerDialect(registry); + cute_nvgpu.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(cute_ir); + inline for (comptime std.meta.declarations(cute_ir)) |decl| { + const T = @field(cute_ir, decl); + if (@TypeOf(T) == type) std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const ctx = try testContext(); + defer ctx.deinit(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "generated types and attributes rebuild from their getters" { + @setEvalBranchQuota(100_000); + const ctx = try testContext(); + defer ctx.deinit(); + inline for (.{ + .{ ArithTupleIteratorType, "!cute.arith_tuple_iter<\"(?,?{div=16})\">" }, + .{ ComposedLayoutType, "!cute.composed_layout<\"(4,5):(1,4) o 2 o (2,3):(1,2)\">" }, + .{ ConstrainedIntType, "!cute.i16" }, + .{ CoordTensorType, "!cute.coord_tensor<\"(?,?,?)\", \"((512,1),1):((1/2@0,0),0)\">" }, + .{ CoordType, "!cute.coord<\"1\">" }, + .{ FastDivmodDivisorType, "!cute.fast_divmod_divisor<32>" }, + .{ IntTupleType, "!cute.int_tuple<\"1\">" }, + .{ LayoutType, "!cute.layout<\"(2,3):(1,2)\">" }, + .{ MemRefType, "!cute.memref, S<2,4,3>, \"((8,16),(32,1),(1,3)):((32,256),(1,0),(0,4096))\">" }, + .{ PtrType, "!cute.ptr, S<2,4,3>>" }, + .{ ShapeType, "!cute.shape<\"(2,3)\">" }, + .{ SparseElemType, "!cute.sparse_elem<2, vector<1xf16>>" }, + .{ StrideType, "!cute.stride<\"1\">" }, + .{ SwizzleType, "!cute.swizzle<\"S<0,4,3>\">" }, + .{ TileType, "!cute.tile<\"[(2,3):(1,2)]\">" }, + .{ TiledCopyType, "!cute.tiled_copy, layout_copy_tv = <\"(32,4):(4,1)\">, tiler_mn = <\"[4:1;32:1]\">>" }, + .{ TiledCopyV2Type, "!cute.tiled_copy_v2, layout_copy_tv = <\"(1,8):(0,1)\">, tiler_mn = <\"[8:1]\">>" }, + .{ TiledMmaType, "!cute.tiled_mma f32 >, atom_layout_MNK = <\"(16,16,1):(16,1,0)\">, permutation_MNK = <\"[(16,4):(4,1);(16,4):(4,1);_]\">>" }, + .{ TupleType, "!cute.tuple" }, + .{ BitLayoutAttr, "#cute.bitlayout<<\"(8,4):(1,8)\">>" }, + .{ CopyAtomAttr, "#cute.copy_atom, layout_tv = (<\"(1,8):(0,1)\">, <\"(1,8):(0,1)\">, <\"(1,8):(0,1)\">)>" }, + .{ CopyAtomV2Attr, "#cute.copy_atom_v2, layout_tv = (<\"(1,8):(0,1)\">, <\"(1,8):(0,1)\">), frg = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, + .{ MmaAtomAttr, "#cute.mma_atom<<\"(16,8,16)\">, <\"32:1\">, layout_TV = (<\"1:0\">, <\"1:0\">, <\"1:0\">), shapes = (<\"(16,16)\">, <\"(8,16)\">, <\"(16,8)\">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, + .{ MxMmaAtomAttr, "#cute.mx_mma_atom<<\"(16,8,16)\">, <\"32:1\">, layout_TV = (<\"1:0\">, <\"1:0\">, <\"1:0\">, <\"1:0\">, <\"1:0\">), shapes = (<\"(16,16)\">, <\"(8,16)\">, <\"(16,8)\">, <\"1\">, <\"2\">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, + .{ SparseMmaAtomAttr, "#cute.sparse_mma_atom<<\"(16,8,16)\">, <\"32:1\">, layout_TV = (<\"1:0\">, <\"1:0\">, <\"1:0\">, <\"1:0\">), shapes = (<\"(16,16)\">, <\"(8,16)\">, <\"(16,8)\">, <\"1\">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, + .{ SparseMxMmaAtomAttr, "#cute.sparse_mx_mma_atom<<\"(16,8,16)\">, <\"32:1\">, layout_TV = (<\"1:0\">, <\"2:0\">, <\"3:0\">, <\"4:0\">, <\"5:0\">, <\"6:0\">), shapes = (<\"1\">, <\"2\">, <\"3\">, <\"4\">, <\"5\">, <\"6\">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, + }) |example| { + const T = example[0]; + const is_type = @hasDecl(T, "type_"); + const parsed = if (is_type) try mlir.Type.parse(ctx, example[1]) else try mlir.Attribute.parse(ctx, example[1]); + const value = parsed.isA(T) orelse return error.TestUnexpectedResult; + try std.testing.expect((try rebuild(T, ctx, value)).eql(value)); + var text: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer text.deinit(); + try text.writer.print("{f}", .{value}); + try std.testing.expect(parsed.eql(if (is_type) try mlir.Type.parse(ctx, text.written()) else try mlir.Attribute.parse(ctx, text.written()))); + } + // Every case of every enum attribute. + inline for (.{cute_ir}) |namespace| { + inline for (comptime std.meta.declarations(namespace)) |decl| { + const T = @field(namespace, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque" and @hasDecl(T, "InitArgs")) { + const fields = @typeInfo(T.InitArgs).@"struct".field_names; + if (fields.len == 1 and comptime std.mem.eql(u8, fields[0], "value")) { + inline for (@typeInfo(@FieldType(T.InitArgs, fields[0])).@"enum".field_names) |case| { + const case_value = @field(@FieldType(T.InitArgs, fields[0]), case); + const attr = try T.get(ctx, .{ .value = case_value }); + try std.testing.expectEqual(@backingInt(case_value), @backingInt(attr.value())); + try std.testing.expect((try rebuild(T, ctx, attr)).eql(attr)); + } + } + } + } + } +} + +test "algebra attributes build from their leaves and tuples" { + const ctx = try testContext(); + defer ctx.deinit(); + + const int_tuple = try cute_ir.IntTupleAttr.getTuple(ctx, &.{ + try .getStatic(ctx, 0), + try .getDynamic(ctx, 32, 1), + try .getDynamic(ctx, 32, 16), + try .getTuple(ctx, &.{ try .getStatic(ctx, 4), try .getDynamic(ctx, 64, 1) }), + }); + try expectType(ctx, "!cute.int_tuple<\"(0,?,?{div=16},(4,?{i64}))\">", (try IntTupleType.get(ctx, .{ .attr = int_tuple.attribute() })).type_()); + try expectType(ctx, "!cute.int_tuple<\"()\">", (try IntTupleType.get(ctx, .{ .attr = (try cute_ir.IntTupleAttr.getTuple(ctx, &.{})).attribute() })).type_()); + + const coord = try cute_ir.CoordAttr.getTuple(ctx, &.{ + try .getUnderscore(ctx), + try .getStatic(ctx, 1), + try .getTuple(ctx, &.{ try .getDynamic(ctx, 32, 1), try .getUnderscore(ctx) }), + }); + try expectType(ctx, "!cute.coord<\"(_,1,(?,_))\">", (try CoordType.get(ctx, .{ .attr = coord.attribute() })).type_()); + + const shape = try cute_ir.ShapeAttr.getTuple(ctx, &.{ + try .getTuple(ctx, &.{ try .getStatic(ctx, 512), try .getStatic(ctx, 1) }), + try .getStatic(ctx, 1), + }); + const stride = try cute_ir.StrideAttr.getTuple(ctx, &.{ + try .getTuple(ctx, &.{ try .getBasis(ctx, 1, 2, &.{0}), try .getStatic(ctx, 0) }), + try .getBasis(ctx, 3, 1, &.{ 0, 1 }), + }); + const layout = try cute_ir.LayoutAttr.get(ctx, .{ .shape = shape, .stride = stride }); + try expectType(ctx, "!cute.layout<\"((512,1),1):((1/2@0,0),3@1@0)\">", (try LayoutType.get(ctx, .{ .attr = layout.attribute() })).type_()); + try std.testing.expectError(error.InvalidMlir, cute_ir.LayoutAttr.get(ctx, .{ .shape = shape, .stride = try .getStatic(ctx, 1) })); + + const row = try cute_ir.LayoutAttr.get(ctx, .{ + .shape = try .getTuple(ctx, &.{ try .getStatic(ctx, 8), try .getStatic(ctx, 64) }), + .stride = try .getTuple(ctx, &.{ try .getStatic(ctx, 64), try .getDynamic(ctx, 32, 8) }), + }); + try expectType(ctx, "!cute.layout<\"(8,64):(64,?{div=8})\">", (try LayoutType.get(ctx, .{ .attr = row.attribute() })).type_()); + + const tile = try cute_ir.TileAttr.getTuple(ctx, &.{ try .getLayout(ctx, row), try .getUnderscore(ctx) }); + try expectType(ctx, "!cute.tile<\"[(8,64):(64,?{div=8});_]\">", (try TileType.get(ctx, .{ .attr = tile.attribute() })).type_()); + + const swizzle = try cute_ir.SwizzleAttr.get(ctx, .{ .bits = 3, .base = 4, .shift = 3 }); + try expectType(ctx, "!cute.swizzle<\"S<3,4,3>\">", (try SwizzleType.get(ctx, .{ .attr = swizzle.attribute() })).type_()); + + const outer = try cute_ir.LayoutAttr.get(ctx, .{ + .shape = try .getTuple(ctx, &.{ try .getStatic(ctx, 8), try .getStatic(ctx, 64) }), + .stride = try .getTuple(ctx, &.{ try .getStatic(ctx, 64), try .getStatic(ctx, 1) }), + }); + const composed = try cute_ir.ComposedLayoutAttr.get(ctx, .{ .inner = .{ .swizzle = swizzle }, .offset = try .getStatic(ctx, 0), .outer = outer }); + try expectType(ctx, "!cute.composed_layout<\"S<3,4,3> o 0 o (8,64):(64,1)\">", (try ComposedLayoutType.get(ctx, .{ .attr = composed.attribute() })).type_()); + + try std.testing.expectError(error.InvalidMlir, cute_ir.IntTupleAttr.getDynamic(ctx, 7, 1)); + try std.testing.expectError(error.InvalidMlir, cute_ir.StrideAttr.getBasis(ctx, 1, 1, &.{})); + try std.testing.expectError(error.InvalidMlir, cute_ir.SwizzleAttr.get(ctx, .{ .bits = 5, .base = 4, .shift = 3 })); +} + +fn expectType(ctx: *mlir.Context, expected: []const u8, actual: *const mlir.Type) !void { + const parsed = try mlir.Type.parse(ctx, expected); + if (!parsed.eql(actual)) { + std.debug.print("expected {s}, built {f}\n", .{ expected, actual }); + return error.TestUnexpectedResult; + } +} + +fn rebuild(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T { + var args: T.InitArgs = undefined; + var elements: [8]*const mlir.Type = undefined; + inline for (@typeInfo(T.InitArgs).@"struct".field_names) |field| { + const name = [_]u8{comptime std.ascii.toUpper(field[0])} ++ field[1..]; + const noun = [_]u8{comptime std.ascii.toLower(field[0])} ++ field[1..]; + const accessor = if (@hasDecl(T, noun ++ "_")) noun ++ "_" else noun; + if (@FieldType(T.InitArgs, field) == []const *const mlir.Type) { + const count = @field(T, "num" ++ name)(value); + for (elements[0..count], 0..) |*element, i| element.* = @field(T, noun[0 .. noun.len - 1])(value, i); + @field(args, field) = elements[0..count]; + } else { + @field(args, field) = @field(T, accessor)(value); + } + } + return T.get(ctx, args); +} diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/BUILD.bazel b/mlir/dialects/cute_nvgpu/BUILD.bazel similarity index 67% rename from mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/BUILD.bazel rename to mlir/dialects/cute_nvgpu/BUILD.bazel index 9cf051ca..07c40ea9 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/BUILD.bazel +++ b/mlir/dialects/cute_nvgpu/BUILD.bazel @@ -1,14 +1,58 @@ load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") package(default_visibility = ["//visibility:public"]) +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + ":dialect", + ":headers", + "//mlir/dialects/cute_ir:dialect", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "cute_nvgpu", + import_name = "mlir/dialects/cute_nvgpu", + main = "cute_nvgpu.zig", + tags = ["manual"], + deps = [ + ":capi", + "//mlir", + "//mlir/dialects/cute_ir", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":cute_nvgpu", + "//mlir", + "//mlir/dialects/cute_ir", + ], +) + td_library( name = "td_files", - srcs = glob(["*.td"]), - includes = ["/mlir/dialects/cute_ir/include"], + srcs = [ + "CuteNVGPUAttrs.td", + "CuteNVGPUDialect.td", + "CuteNVGPUEnums.td", + "CuteNVGPUOps.td", + "CuteNVGPUTypes.td", + ], + includes = ["/"], deps = [ - "//mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR:tabledefs", + "//mlir/dialects/cute_ir:td_files", + "@cute_ir//:cute_ir_dialect_td", "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", "@llvm-project//mlir:OpBaseTdFiles", "@llvm-project//mlir:SideEffectInterfacesTdFiles", @@ -17,7 +61,6 @@ td_library( gentbl_cc_library( name = "dialect_inc_gen", - includes = ["../../../.."], tbl_outs = [ ( [ @@ -41,7 +84,6 @@ gentbl_cc_library( gentbl_cc_library( name = "types_inc_gen", - includes = ["../../../.."], tbl_outs = [ ( [ @@ -63,11 +105,8 @@ gentbl_cc_library( deps = [":td_files"], ) -# The compiler's nvgpu enums (CuteNVGPUEnums.td does not include the cute -# enums, which the cute dialect emits). gentbl_cc_library( name = "enums_inc_gen", - includes = ["../../../.."], tbl_outs = [ ( ["-gen-enum-decls"], @@ -85,7 +124,6 @@ gentbl_cc_library( gentbl_cc_library( name = "attrs_inc_gen", - includes = ["../../../.."], tbl_outs = [ ( [ @@ -107,10 +145,8 @@ gentbl_cc_library( deps = [":td_files"], ) -# Every cute_nvgpu operation of the compiler's registry. gentbl_cc_library( name = "ops_inc_gen", - includes = ["../../../.."], tbl_outs = [ ( ["-gen-op-decls"], @@ -129,14 +165,28 @@ gentbl_cc_library( cc_library( name = "headers", hdrs = ["CuteNVGPUDialect.h"], - includes = ["../../../.."], deps = [ ":attrs_inc_gen", ":dialect_inc_gen", ":enums_inc_gen", ":ops_inc_gen", ":types_inc_gen", - "//mlir/dialects/cute_ir/include/cute_ir/Dialect/Cute/IR:headers", + "//mlir/dialects/cute_ir:headers", + "@llvm-project//mlir:BytecodeOpInterface", + "@llvm-project//mlir:IR", "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:SideEffectInterfaces", + ], +) + +cc_library( + name = "dialect", + srcs = ["CuteNVGPUDialect.cpp"], + deps = [ + ":headers", + "//mlir/dialects/cute_ir:headers", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:AsmParser", + "@llvm-project//mlir:IR", ], ) diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUAttrs.td b/mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.td similarity index 93% rename from mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUAttrs.td rename to mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.td index 94c2cbdf..b4230c61 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUAttrs.td +++ b/mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.td @@ -12,12 +12,12 @@ // //===----------------------------------------------------------------------===// -#ifndef CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_ATTRS_TD -#define CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_ATTRS_TD +#ifndef MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUATTRS_TD +#define MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUATTRS_TD -include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.td" -include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUEnums.td" -include "cute_ir/Dialect/Cute/IR/CuteEnums.td" +include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.td" +include "mlir/dialects/cute_nvgpu/CuteNVGPUEnums.td" +include "mlir/dialects/cute_ir/CuteEnums.td" def CuteNVGPU_MajorModeAttr : EnumAttr { @@ -129,4 +129,4 @@ def CuteNVGPU_NotImplementedFrgAttr let assemblyFormat = "`<` `>`"; } -#endif // CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_ATTRS_TD +#endif // MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUATTRS_TD diff --git a/mlir/dialects/cute_ir/lib/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.cpp b/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.cpp similarity index 63% rename from mlir/dialects/cute_ir/lib/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.cpp rename to mlir/dialects/cute_nvgpu/CuteNVGPUDialect.cpp index 7c2e6bc3..a912714e 100644 --- a/mlir/dialects/cute_ir/lib/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.cpp +++ b/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.cpp @@ -1,43 +1,45 @@ -//===- CuteNVGPUDialect.cpp - CuTe NVIDIA GPU dialect --------------------===// +//===- CuteNVGPUDialect.cpp - CuTe NVIDIA GPU dialect ---------------------===// // // Parsers, printers and verifiers of the cute_nvgpu types that the declarative // format cannot express. The syntax and the verifier messages are the DSL // compiler's (see CuteNVGPUTypes.td). -#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h" +#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h" -#include -#include - -#include "llvm/ADT/StringExtras.h" -#include "llvm/ADT/TypeSwitch.h" #include "mlir/AsmParser/AsmParser.h" #include "mlir/IR/Builders.h" #include "mlir/IR/BuiltinTypes.h" #include "mlir/IR/DialectImplementation.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/TypeSwitch.h" + +#include +#include using namespace mlir; using namespace mlir::cutlass_compiler::cute_nvgpu; namespace cute = mlir::cutlass_compiler::cute; -#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.cpp.inc" +#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.cpp.inc" -#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUEnums.cpp.inc" +#include "mlir/dialects/cute_nvgpu/CuteNVGPUEnums.cpp.inc" #define GET_ATTRDEF_CLASSES -#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUAttrs.cpp.inc" +#include "mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.cpp.inc" #define GET_TYPEDEF_CLASSES -#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUTypes.cpp.inc" +#include "mlir/dialects/cute_nvgpu/CuteNVGPUTypes.cpp.inc" #define GET_OP_CLASSES -#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUOps.cpp.inc" +#include "mlir/dialects/cute_nvgpu/CuteNVGPUOps.cpp.inc" FailureOr mlir::cutlass_compiler::cute_nvgpu::parseBool(AsmParser &p) { SMLoc loc = p.getCurrentLocation(); StringRef kw; - if (p.parseKeyword(&kw)) return failure(); - if (kw == "true" || kw == "false") return kw == "true"; + if (p.parseKeyword(&kw)) + return failure(); + if (kw == "true" || kw == "false") + return kw == "true"; return p.emitError(loc, "expected `true` or `false`"); } @@ -49,10 +51,10 @@ namespace { using EmitFn = llvm::function_ref; -template -ParseResult field(AsmParser &p, T &out) { +template ParseResult field(AsmParser &p, T &out) { FailureOr v = FieldParser::parse(p); - if (failed(v)) return failure(); + if (failed(v)) + return failure(); out = *v; return success(); } @@ -60,7 +62,8 @@ ParseResult field(AsmParser &p, Type &out) { return p.parseType(out); } ParseResult field(AsmParser &p, int &out) { return p.parseInteger(out); } ParseResult field(AsmParser &p, bool &out) { FailureOr v = parseBool(p); - if (failed(v)) return failure(); + if (failed(v)) + return failure(); out = *v; return success(); } @@ -78,7 +81,8 @@ ParseResult nextKeyField(AsmParser &p, StringRef key, T &out) { // An optional ` key = value` group. template ParseResult optKeyField(AsmParser &p, StringRef key, T &out) { - if (failed(p.parseOptionalKeyword(key))) return success(); + if (failed(p.parseOptionalKeyword(key))) + return success(); return failure(p.parseEqual() || field(p, out)); } // `N word`, as in `128 b` or `32 DP`. @@ -86,8 +90,7 @@ ParseResult countOf(AsmParser &p, StringRef word, int &out) { return failure(p.parseInteger(out) || p.parseKeyword(word)); } // `(a, b)`. -template -ParseResult pairOf(AsmParser &p, T &a, T &b) { +template ParseResult pairOf(AsmParser &p, T &a, T &b) { return failure(p.parseLParen() || field(p, a) || p.parseComma() || field(p, b) || p.parseRParen()); } @@ -105,21 +108,26 @@ void printElemTypes(AsmPrinter &p, Type a, Type b, Type c) { ParseResult repeat(AsmParser &p, unsigned &out) { SMLoc loc = p.getCurrentLocation(); StringRef kw; - if (p.parseKeyword(&kw)) return failure(); + if (p.parseKeyword(&kw)) + return failure(); if (!kw.consume_front("x")) - return p.emitError(loc, "failed to parse parameter 'num_rep' which is to start with `x`"); + return p.emitError( + loc, "failed to parse parameter 'num_rep' which is to start with `x`"); if (kw.getAsInteger(10, out)) - return p.emitError(loc, "failed to parse parameter 'num_rep' which is to be a `unsigned`"); + return p.emitError( + loc, "failed to parse parameter 'num_rep' which is to be a `unsigned`"); return success(); } template void printIfNot(AsmPrinter &p, StringRef key, E value, E dflt) { - if (value != dflt) p << ' ' << key << " = " << stringifyEnum(value); + if (value != dflt) + p << ' ' << key << " = " << stringifyEnum(value); } template void printAttrField(AsmPrinter &p, StringRef key, A attr) { - if (!attr) return; + if (!attr) + return; p << ", " << key << " = "; p.printStrippedAttrOrType(attr); } @@ -128,21 +136,28 @@ void printAttrField(AsmPrinter &p, StringRef key, A attr) { // a sparse element's share of its physical storage. int elementBits(Type t) { if (auto s = llvm::dyn_cast(t)) - return s.getNumLogical() ? elementBits(s.getPhysicalType()) / s.getNumLogical() : 0; - if (t.isTF32()) return 32; - if (t.isIntOrFloat()) return t.getIntOrFloatBitWidth(); - if (llvm::isa(t)) return 64; + return s.getNumLogical() + ? elementBits(s.getPhysicalType()) / s.getNumLogical() + : 0; + if (t.isTF32()) + return 32; + if (t.isIntOrFloat()) + return t.getIntOrFloatBitWidth(); + if (llvm::isa(t)) + return 64; return 0; } int orElementBits(int bits, Type t) { return bits ? bits : elementBits(t); } // The integers of a flat tuple `(a,b,c)`. std::optional> flatInts(StringRef text) { - if (!text.consume_front("(") || !text.consume_back(")")) return std::nullopt; + if (!text.consume_front("(") || !text.consume_back(")")) + return std::nullopt; SmallVector out; for (StringRef part : llvm::split(text, ',')) { int64_t v; - if (part.getAsInteger(10, v)) return std::nullopt; + if (part.getAsInteger(10, v)) + return std::nullopt; out.push_back(v); } return out; @@ -152,11 +167,17 @@ std::optional> flatInts(StringRef text) { ParseResult parseShapeMNK(AsmParser &p, cute::ShapeAttr &out) { SMLoc loc = p.getCurrentLocation(); SmallVector dims; - if (p.parseDimensionList(dims, /*allowDynamic=*/false, /*withTrailingX=*/false)) + if (p.parseDimensionList(dims, /*allowDynamic=*/false, + /*withTrailingX=*/false)) return p.emitError(loc, "can not parse shape_MNK"); auto shape = cutegen::from_string( - "(" + llvm::join(llvm::map_range(dims, [](int64_t d) { return std::to_string(d); }), ",") + ")"); - if (!shape) return p.emitError(loc, "can not parse shape_MNK"); + "(" + + llvm::join( + llvm::map_range(dims, [](int64_t d) { return std::to_string(d); }), + ",") + + ")"); + if (!shape) + return p.emitError(loc, "can not parse shape_MNK"); out = cute::ShapeAttr::get(p.getContext(), *shape); return success(); } @@ -165,13 +186,15 @@ void printShapeMNK(AsmPrinter &p, cute::ShapeAttr shape) { StringRef body(text); body.consume_front("("); body.consume_back(")"); - for (char c : body) p << (c == ',' ? 'x' : c); + for (char c : body) + p << (c == ',' ? 'x' : c); } LogicalResult verifyShapeMNK(EmitFn emitError, cute::ShapeAttr shape) { std::string text = cutegen::to_string(shape.getRef()); auto dims = flatInts(text); if (!dims || dims->size() != 3) - return emitError() << "expects flatten rank-3 tuple of shape, but got " << text; + return emitError() << "expects flatten rank-3 tuple of shape, but got " + << text; return success(); } @@ -183,17 +206,25 @@ ParseResult parseFragKind(AsmParser &p, MmaFragKind other, MmaFragKind &out) { bool ts = other == MmaFragKind::tmem; if (p.parseKeyword("frag_kind") || p.parseEqual() || p.parseKeyword(&kw)) return failure(); - if (kw == "ss") out = MmaFragKind::smem_desc; - else if (kw == (ts ? "ts" : "rs")) out = other; - else return p.emitError(loc, ts ? "expects `ts` or `ss`" : "expects `rs` or `ss`"); + if (kw == "ss") + out = MmaFragKind::smem_desc; + else if (kw == (ts ? "ts" : "rs")) + out = other; + else + return p.emitError(loc, + ts ? "expects `ts` or `ss`" : "expects `rs` or `ss`"); return success(); } StringRef fragKindText(MmaFragKind k) { switch (k) { - case MmaFragKind::smem_desc: return "ss"; - case MmaFragKind::rmem: return "rs"; - case MmaFragKind::tmem: return "ts"; - default: return stringifyEnum(k); + case MmaFragKind::smem_desc: + return "ss"; + case MmaFragKind::rmem: + return "rs"; + case MmaFragKind::tmem: + return "ts"; + default: + return stringifyEnum(k); } } @@ -201,15 +232,17 @@ StringRef fragKindText(MmaFragKind k) { ParseResult parseTranspose(AsmParser &p, UnitAttr &out) { SMLoc loc = p.getCurrentLocation(); StringRef kw; - if (p.parseKeyword(&kw)) return failure(); - if (kw != "t" && kw != "n") return p.emitError(loc, "expects `t` or `n`"); + if (p.parseKeyword(&kw)) + return failure(); + if (kw != "t" && kw != "n") + return p.emitError(loc, "expects `t` or `n`"); out = kw == "t" ? UnitAttr::get(p.getContext()) : UnitAttr(); return success(); } bool isPow2AtLeast8(int v) { return v >= 8 && llvm::isPowerOf2_32(v); } -} // namespace +} // namespace //===----------------------------------------------------------------------===// // Descriptors @@ -224,20 +257,28 @@ static LogicalResult verifyCircular(EmitFn emitError, cute::LayoutAttr layout) { return emitError() << "block layout must be fully static but got " << text; auto shape = flatInts(shapeText), stride = flatInts(strideText); if (!shape || shape->size() != 3 || !stride || stride->size() != 3) - return emitError() << "expects block layout to have 3 modes, but got " << text; + return emitError() << "expects block layout to have 3 modes, but got " + << text; if ((*shape)[1] != 128) - return emitError() << "expects block layout to have 128B 1st mode, but got " << text; - if ((*stride)[1] != 1 || (*stride)[0] != 128 || (*stride)[2] != 128 * (*shape)[0]) - return emitError() << "expects mode-1 major compact layout, but got " << text; + return emitError() << "expects block layout to have 128B 1st mode, but got " + << text; + if ((*stride)[1] != 1 || (*stride)[0] != 128 || + (*stride)[2] != 128 * (*shape)[0]) + return emitError() << "expects mode-1 major compact layout, but got " + << text; if ((*shape)[2] < 2) - return emitError() << "Circular SMEM descriptor requires at least 2 128B blocks since " - "MMA may span 2 buffers, but got " << text; + return emitError() + << "Circular SMEM descriptor requires at least 2 128B blocks since " + "MMA may span 2 buffers, but got " + << text; return success(); } -LogicalResult SmemDescCircularSM103Type::verify(EmitFn emitError, cute::LayoutAttr layout) { +LogicalResult SmemDescCircularSM103Type::verify(EmitFn emitError, + cute::LayoutAttr layout) { return verifyCircular(emitError, layout); } -LogicalResult SmemDescCircularSM107Type::verify(EmitFn emitError, cute::LayoutAttr layout) { +LogicalResult SmemDescCircularSM107Type::verify(EmitFn emitError, + cute::LayoutAttr layout) { return verifyCircular(emitError, layout); } @@ -245,12 +286,15 @@ Type SmemDescViewType::parse(AsmParser &p) { SMLoc loc = p.getCurrentLocation(); Type desc; std::string text; - if (p.parseLess() || p.parseType(desc) || p.parseComma()) return {}; + if (p.parseLess() || p.parseType(desc) || p.parseComma()) + return {}; SMLoc layoutLoc = p.getCurrentLocation(); - if (p.parseString(&text) || p.parseGreater()) return {}; + if (p.parseString(&text) || p.parseGreater()) + return {}; auto layout = cutegen::from_string(text); if (!layout) { - p.emitError(layoutLoc, "failed to parse SmemDescViewType parameter 'layout' as `cutegen::layout`"); + p.emitError(layoutLoc, "failed to parse SmemDescViewType parameter " + "'layout' as `cutegen::layout`"); return {}; } return getChecked([&] { return p.emitError(loc); }, p.getContext(), desc, @@ -261,9 +305,10 @@ void SmemDescViewType::print(AsmPrinter &p) const { p.printString(cutegen::to_string(getLayout().getRef())); p << '>'; } -LogicalResult SmemDescViewType::verify(EmitFn emitError, Type desc, cute::LayoutAttr) { - if (!llvm::isa(desc)) +LogicalResult SmemDescViewType::verify(EmitFn emitError, Type desc, + cute::LayoutAttr) { + if (!llvm::isa(desc)) return emitError() << "expected SmemDescType, SmemDescCircularSM103Type, " "SmemDescCircularSM107Type or SmemDescSM107Type"; return success(); @@ -280,37 +325,49 @@ Type CopyAtomSIMTSyncCopyType::parse(AsmParser &p) { Type t; int bits = 0; auto src = cute::AddressSpace::generic, dst = src; - if (p.parseLess() || p.parseType(t)) return {}; + if (p.parseLess() || p.parseType(t)) + return {}; // `src_space` / `dst_space` groups; one alone is what the compiler prints // when the other is generic (or copy_bits is 0), so it is read too. auto spaces = [&]() -> ParseResult { if (succeeded(p.parseOptionalKeyword("dst_space"))) return failure(p.parseEqual() || field(p, dst)); - if (p.parseKeyword("src_space") || p.parseEqual() || field(p, src)) return failure(); - if (succeeded(p.parseOptionalComma())) return keyField(p, "dst_space", dst); + if (p.parseKeyword("src_space") || p.parseEqual() || field(p, src)) + return failure(); + if (succeeded(p.parseOptionalComma())) + return keyField(p, "dst_space", dst); return success(); }; if (succeeded(p.parseOptionalComma())) { OptionalParseResult n = p.parseOptionalInteger(bits); - if (n.has_value() && (failed(*n) || p.parseKeyword("b"))) return {}; - if ((!n.has_value() || succeeded(p.parseOptionalComma())) && spaces()) return {}; + if (n.has_value() && (failed(*n) || p.parseKeyword("b"))) + return {}; + if ((!n.has_value() || succeeded(p.parseOptionalComma())) && spaces()) + return {}; } - if (p.parseGreater()) return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, bits, src, dst); + if (p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, bits, + src, dst); } void CopyAtomSIMTSyncCopyType::print(AsmPrinter &p) const { auto generic = cute::AddressSpace::generic; bool spaces = getSrcSpace() != generic || getDstSpace() != generic; p << '<' << getValType(); - if (getCopyBits() || spaces) p << ", " << getCopyBits() << " b"; - if (spaces) p << ", src_space = " << stringifyEnum(getSrcSpace()); - if (getDstSpace() != generic) p << ", dst_space = " << stringifyEnum(getDstSpace()); + if (getCopyBits() || spaces) + p << ", " << getCopyBits() << " b"; + if (spaces) + p << ", src_space = " << stringifyEnum(getSrcSpace()); + if (getDstSpace() != generic) + p << ", dst_space = " << stringifyEnum(getDstSpace()); p << '>'; } LogicalResult CopyAtomSIMTSyncCopyType::verify(EmitFn emitError, Type, int bits, - cute::AddressSpace, cute::AddressSpace) { + cute::AddressSpace, + cute::AddressSpace) { if (bits != 0 && !isPow2AtLeast8(bits)) - return emitError() << "expected copy_bits to be 0 (auto) or a power of 2 (>= 8), but got " + return emitError() << "expected copy_bits to be 0 (auto) or a power of 2 " + "(>= 8), but got " << bits; return success(); } @@ -320,9 +377,12 @@ Type CopyAtomSIMTAsyncCopyType::parse(AsmParser &p) { Type t; int bits = 0; LoadCacheMode cache; - if (p.parseLess() || p.parseType(t) || nextKeyField(p, "cache", cache)) return {}; - if (succeeded(p.parseOptionalComma()) && countOf(p, "b", bits)) return {}; - if (p.parseGreater()) return {}; + if (p.parseLess() || p.parseType(t) || nextKeyField(p, "cache", cache)) + return {}; + if (succeeded(p.parseOptionalComma()) && countOf(p, "b", bits)) + return {}; + if (p.parseGreater()) + return {}; return get(p.getContext(), t, cache, orElementBits(bits, t)); } void CopyAtomSIMTAsyncCopyType::print(AsmPrinter &p) const { @@ -335,9 +395,12 @@ Type CopyAtomDsmemStoreType::parse(AsmParser &p) { SMLoc loc = p.getCurrentLocation(); Type t; int bits = 0; - if (p.parseLess() || p.parseType(t)) return {}; - if (succeeded(p.parseOptionalComma()) && countOf(p, "b", bits)) return {}; - if (p.parseGreater()) return {}; + if (p.parseLess() || p.parseType(t)) + return {}; + if (succeeded(p.parseOptionalComma()) && countOf(p, "b", bits)) + return {}; + if (p.parseGreater()) + return {}; return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, orElementBits(bits, t)); } @@ -346,7 +409,9 @@ void CopyAtomDsmemStoreType::print(AsmPrinter &p) const { } LogicalResult CopyAtomDsmemStoreType::verify(EmitFn emitError, Type, int bits) { if (bits != 0 && bits != 32 && bits != 64 && bits != 128) - return emitError() << "expects copy_bits to be one of (0, 32, 64, 128), but got " << bits; + return emitError() + << "expects copy_bits to be one of (0, 32, 64, 128), but got " + << bits; return success(); } @@ -362,8 +427,10 @@ Type CopyAtomBulkCopyG2SType::parse(AsmParser &p) { } void CopyAtomBulkCopyG2SType::print(AsmPrinter &p) const { p << '<' << getValType(); - if (getCopyBits()) p << " copy_bits = " << getCopyBits(); - if (getMcast()) p << " mcast = true"; + if (getCopyBits()) + p << " copy_bits = " << getCopyBits(); + if (getMcast()) + p << " mcast = true"; p << '>'; } Type CopyAtomBulkCopyS2GType::parse(AsmParser &p) { @@ -377,8 +444,10 @@ Type CopyAtomBulkCopyS2GType::parse(AsmParser &p) { } void CopyAtomBulkCopyS2GType::print(AsmPrinter &p) const { p << '<' << getValType(); - if (getCopyBits()) p << " copy_bits = " << getCopyBits(); - if (getMask()) p << " mask = true"; + if (getCopyBits()) + p << " copy_bits = " << getCopyBits(); + if (getMask()) + p << " mask = true"; p << '>'; } Type CopyAtomBulkCopyS2SType::parse(AsmParser &p) { @@ -391,14 +460,16 @@ Type CopyAtomBulkCopyS2SType::parse(AsmParser &p) { } void CopyAtomBulkCopyS2SType::print(AsmPrinter &p) const { p << '<' << getValType(); - if (getCopyBits()) p << " copy_bits = " << getCopyBits(); + if (getCopyBits()) + p << " copy_bits = " << getCopyBits(); p << '>'; } // g2r, r2g, r2s, s2r: ``. static LogicalResult verifyCopyBits(EmitFn emitError, int bits) { if (!isPow2AtLeast8(bits)) - return emitError() << "expects copy_bits to be a power of 2 (>= 8), but got " << bits; + return emitError() + << "expects copy_bits to be a power of 2 (>= 8), but got " << bits; return success(); } @@ -414,15 +485,17 @@ Type CopyAtomG2RType::parse(AsmParser &p) { auto shared = SharedSpace::CTA; bool invariant = false; if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || - optKeyField(p, "mem_order", order) || optKeyField(p, "mem_scope", scope) || + optKeyField(p, "mem_order", order) || + optKeyField(p, "mem_scope", scope) || optKeyField(p, "l2_prefetch_size", prefetch) || optKeyField(p, "l1_cache_evict_priority", evict) || - optKeyField(p, "load_cache_mode", cache) || optKeyField(p, "shared_space", shared) || + optKeyField(p, "load_cache_mode", cache) || + optKeyField(p, "shared_space", shared) || optKeyField(p, "invariant", invariant) || p.parseGreater()) return {}; return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, - orElementBits(bits, t), order, scope, prefetch, evict, cache, shared, - invariant); + orElementBits(bits, t), order, scope, prefetch, evict, + cache, shared, invariant); } void CopyAtomG2RType::print(AsmPrinter &p) const { p << '<' << getValType() << " copy_bits = " << getCopyBits(); @@ -433,24 +506,30 @@ void CopyAtomG2RType::print(AsmPrinter &p) const { CacheEvictionPriority::EVICT_NORMAL); printIfNot(p, "load_cache_mode", getLoadCacheMode(), LoadCacheMode::always); printIfNot(p, "shared_space", getSharedSpace(), SharedSpace::CTA); - if (getInvariant()) p << " invariant = true"; + if (getInvariant()) + p << " invariant = true"; p << '>'; } -LogicalResult CopyAtomG2RType::verify(EmitFn emitError, Type, int bits, MemOrderKind order, - MemScopeKind scope, L2PrefetchSize prefetch, - CacheEvictionPriority evict, LoadCacheMode cache, - SharedSpace, bool invariant) { - if (failed(verifyCopyBits(emitError, bits))) return failure(); +LogicalResult CopyAtomG2RType::verify(EmitFn emitError, Type, int bits, + MemOrderKind order, MemScopeKind scope, + L2PrefetchSize prefetch, + CacheEvictionPriority evict, + LoadCacheMode cache, SharedSpace, + bool invariant) { + if (failed(verifyCopyBits(emitError, bits))) + return failure(); if (invariant && (order != MemOrderKind::WEAK || scope != MemScopeKind::CTA || prefetch != L2PrefetchSize::NONE || evict != CacheEvictionPriority::EVICT_NORMAL || cache != LoadCacheMode::always)) - return emitError() << "invariant_load is not allowed when the other non-default memory " - "features (e.g., mem_order) are set to non-default values"; + return emitError() + << "invariant_load is not allowed when the other non-default memory " + "features (e.g., mem_order) are set to non-default values"; if (order != MemOrderKind::WEAK && cache != LoadCacheMode::always) - return emitError() << "expects load_cache_mode to be 'always' when mem_order is not " - "'weak', but got '" - << stringifyEnum(cache) << "' and '" << stringifyEnum(order) << "'"; + return emitError() + << "expects load_cache_mode to be 'always' when mem_order is not " + "'weak', but got '" + << stringifyEnum(cache) << "' and '" << stringifyEnum(order) << "'"; return success(); } @@ -464,10 +543,11 @@ Type CopyAtomR2GType::parse(AsmParser &p) { auto cache = StoreCacheMode::write_back; auto shared = SharedSpace::CTA; if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || - optKeyField(p, "mem_order", order) || optKeyField(p, "mem_scope", scope) || + optKeyField(p, "mem_order", order) || + optKeyField(p, "mem_scope", scope) || optKeyField(p, "l1_cache_evict_priority", evict) || - optKeyField(p, "store_cache_mode", cache) || optKeyField(p, "shared_space", shared) || - p.parseGreater()) + optKeyField(p, "store_cache_mode", cache) || + optKeyField(p, "shared_space", shared) || p.parseGreater()) return {}; return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, orElementBits(bits, t), order, scope, evict, cache, shared); @@ -478,23 +558,26 @@ void CopyAtomR2GType::print(AsmPrinter &p) const { printIfNot(p, "mem_scope", getMemScope(), MemScopeKind::CTA); printIfNot(p, "l1_cache_evict_priority", getL1CacheEvictPriority(), CacheEvictionPriority::EVICT_NORMAL); - printIfNot(p, "store_cache_mode", getStoreCacheMode(), StoreCacheMode::write_back); + printIfNot(p, "store_cache_mode", getStoreCacheMode(), + StoreCacheMode::write_back); printIfNot(p, "shared_space", getSharedSpace(), SharedSpace::CTA); p << '>'; } -LogicalResult CopyAtomR2GType::verify(EmitFn emitError, Type, int bits, MemOrderKind order, - MemScopeKind, CacheEvictionPriority, +LogicalResult CopyAtomR2GType::verify(EmitFn emitError, Type, int bits, + MemOrderKind order, MemScopeKind, + CacheEvictionPriority, StoreCacheMode cache, SharedSpace) { - if (failed(verifyCopyBits(emitError, bits))) return failure(); + if (failed(verifyCopyBits(emitError, bits))) + return failure(); if (order != MemOrderKind::WEAK && cache != StoreCacheMode::write_back) - return emitError() << "expects store_cache_mode to be 'write_back' when mem_order is " - "not 'weak', but got '" - << stringifyEnum(cache) << "' and '" << stringifyEnum(order) << "'"; + return emitError() + << "expects store_cache_mode to be 'write_back' when mem_order is " + "not 'weak', but got '" + << stringifyEnum(cache) << "' and '" << stringifyEnum(order) << "'"; return success(); } -template -static Type parseSharedCopy(AsmParser &p) { +template static Type parseSharedCopy(AsmParser &p) { SMLoc loc = p.getCurrentLocation(); Type t; int bits = 0; @@ -502,30 +585,34 @@ static Type parseSharedCopy(AsmParser &p) { auto scope = MemScopeKind::CTA; auto shared = SharedSpace::CTA; if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || - optKeyField(p, "mem_order", order) || optKeyField(p, "mem_scope", scope) || + optKeyField(p, "mem_order", order) || + optKeyField(p, "mem_scope", scope) || optKeyField(p, "shared_space", shared) || p.parseGreater()) return {}; return T::getChecked([&] { return p.emitError(loc); }, p.getContext(), t, orElementBits(bits, t), order, scope, shared); } -template -static void printSharedCopy(AsmPrinter &p, T type) { +template static void printSharedCopy(AsmPrinter &p, T type) { p << '<' << type.getValType() << " copy_bits = " << type.getCopyBits(); printIfNot(p, "mem_order", type.getMemOrder(), MemOrderKind::WEAK); printIfNot(p, "mem_scope", type.getMemScope(), MemScopeKind::CTA); printIfNot(p, "shared_space", type.getSharedSpace(), SharedSpace::CTA); p << '>'; } -Type CopyAtomR2SType::parse(AsmParser &p) { return parseSharedCopy(p); } +Type CopyAtomR2SType::parse(AsmParser &p) { + return parseSharedCopy(p); +} void CopyAtomR2SType::print(AsmPrinter &p) const { printSharedCopy(p, *this); } -LogicalResult CopyAtomR2SType::verify(EmitFn emitError, Type, int bits, MemOrderKind, - MemScopeKind, SharedSpace) { +LogicalResult CopyAtomR2SType::verify(EmitFn emitError, Type, int bits, + MemOrderKind, MemScopeKind, SharedSpace) { return verifyCopyBits(emitError, bits); } -Type CopyAtomS2RType::parse(AsmParser &p) { return parseSharedCopy(p); } +Type CopyAtomS2RType::parse(AsmParser &p) { + return parseSharedCopy(p); +} void CopyAtomS2RType::print(AsmPrinter &p) const { printSharedCopy(p, *this); } -LogicalResult CopyAtomS2RType::verify(EmitFn emitError, Type, int bits, MemOrderKind, - MemScopeKind, SharedSpace) { +LogicalResult CopyAtomS2RType::verify(EmitFn emitError, Type, int bits, + MemOrderKind, MemScopeKind, SharedSpace) { return verifyCopyBits(emitError, bits); } @@ -544,68 +631,84 @@ struct Multimem { MemScopeKind scope = MemScopeKind::SYS; }; -ParseResult parseMultimem(AsmParser &p, Type &t, Multimem &m, bool hasRed, bool hasAcc) { - if (p.parseLess() || p.parseType(t)) return failure(); +ParseResult parseMultimem(AsmParser &p, Type &t, Multimem &m, bool hasRed, + bool hasAcc) { + if (p.parseLess() || p.parseType(t)) + return failure(); while (succeeded(p.parseOptionalComma())) { SMLoc loc = p.getCurrentLocation(); StringRef kw; OptionalParseResult bits = p.parseOptionalInteger(m.bits); if (bits.has_value()) { - if (failed(*bits) || p.parseKeyword("b")) return failure(); + if (failed(*bits) || p.parseKeyword("b")) + return failure(); continue; } if (hasAcc && succeeded(p.parseOptionalLess())) { LdReduceAccPrecisionKind kind; - if (field(p, kind) || p.parseGreater()) return failure(); + if (field(p, kind) || p.parseGreater()) + return failure(); m.acc = LdReduceAccPrecisionKindAttr::get(p.getContext(), kind); continue; } - if (p.parseKeyword(&kw)) return failure(); - if (auto r = symbolizeReductionKind(kw); r && hasRed) m.red = *r; - else if (auto o = symbolizeMemOrderKind(kw)) m.order = *o; - else if (auto s = symbolizeMemScopeKind(kw)) m.scope = *s; - else return p.emitError(loc, "unexpected `") << kw << "`"; + if (p.parseKeyword(&kw)) + return failure(); + if (auto r = symbolizeReductionKind(kw); r && hasRed) + m.red = *r; + else if (auto o = symbolizeMemOrderKind(kw)) + m.order = *o; + else if (auto s = symbolizeMemScopeKind(kw)) + m.scope = *s; + else + return p.emitError(loc, "unexpected `") << kw << "`"; } return p.parseGreater(); } // Prints the fields up to the last one that is not default. -void printMultimem(AsmPrinter &p, Type t, const Multimem &m, bool bitsSet, bool hasRed, - bool hasAcc) { - int last = m.scope != MemScopeKind::SYS ? 4 - : m.order != MemOrderKind::RELAXED ? 3 - : m.acc ? 2 +void printMultimem(AsmPrinter &p, Type t, const Multimem &m, bool bitsSet, + bool hasRed, bool hasAcc) { + int last = m.scope != MemScopeKind::SYS ? 4 + : m.order != MemOrderKind::RELAXED ? 3 + : m.acc ? 2 : hasRed && m.red != ReductionKind::ADD ? 1 - : bitsSet ? 0 - : -1; + : bitsSet ? 0 + : -1; p << '<' << t; - if (last >= 0) p << ", " << m.bits << " b"; - if (hasRed && last >= 1) p << ", " << stringifyEnum(m.red); + if (last >= 0) + p << ", " << m.bits << " b"; + if (hasRed && last >= 1) + p << ", " << stringifyEnum(m.red); if (hasAcc && last >= 2 && m.acc) { p << ", "; p.printStrippedAttrOrType(m.acc); } - if (last >= 3) p << ", " << stringifyEnum(m.order); - if (last >= 4) p << ", " << stringifyEnum(m.scope); + if (last >= 3) + p << ", " << stringifyEnum(m.order); + if (last >= 4) + p << ", " << stringifyEnum(m.scope); p << '>'; } -} // namespace +} // namespace Type CopyAtomSIMTMultimemStType::parse(AsmParser &p) { Type t; Multimem m; - if (parseMultimem(p, t, m, /*hasRed=*/false, /*hasAcc=*/false)) return {}; + if (parseMultimem(p, t, m, /*hasRed=*/false, /*hasAcc=*/false)) + return {}; return get(p.getContext(), t, m.bits, m.order, m.scope); } void CopyAtomSIMTMultimemStType::print(AsmPrinter &p) const { - Multimem m{getCopyBits(), ReductionKind::ADD, {}, getMemOrder(), getMemScope()}; + Multimem m{ + getCopyBits(), ReductionKind::ADD, {}, getMemOrder(), getMemScope()}; printMultimem(p, getValType(), m, getCopyBits() != 0, false, false); } Type CopyAtomSIMTMultimemRedType::parse(AsmParser &p) { Type t; Multimem m; - if (parseMultimem(p, t, m, /*hasRed=*/true, /*hasAcc=*/false)) return {}; + if (parseMultimem(p, t, m, /*hasRed=*/true, /*hasAcc=*/false)) + return {}; // An unset width is the element's, at least 32 bits (the f16x2 form). int bits = m.bits ? m.bits : std::max(elementBits(t), 32); return get(p.getContext(), t, bits, m.red, m.order, m.scope); @@ -618,12 +721,13 @@ void CopyAtomSIMTMultimemRedType::print(AsmPrinter &p) const { Type CopyAtomSIMTMultimemLdReduceType::parse(AsmParser &p) { Type t; Multimem m; - if (parseMultimem(p, t, m, /*hasRed=*/true, /*hasAcc=*/true)) return {}; + if (parseMultimem(p, t, m, /*hasRed=*/true, /*hasAcc=*/true)) + return {}; return get(p.getContext(), t, m.bits, m.red, m.acc, m.order, m.scope); } void CopyAtomSIMTMultimemLdReduceType::print(AsmPrinter &p) const { - Multimem m{getCopyBits(), getReduction(), getLdReduceAccPrecision(), getMemOrder(), - getMemScope()}; + Multimem m{getCopyBits(), getReduction(), getLdReduceAccPrecision(), + getMemOrder(), getMemScope()}; printMultimem(p, getValType(), m, getCopyBits() != 0, true, true); } @@ -631,39 +735,53 @@ void CopyAtomSIMTMultimemLdReduceType::print(AsmPrinter &p) const { // Tensor-memory atoms: `` //===----------------------------------------------------------------------===// -static ParseResult parseTmemHead(AsmParser &p, Type &t, int &dp, int &bit, unsigned &rep) { +static ParseResult parseTmemHead(AsmParser &p, Type &t, int &dp, int &bit, + unsigned &rep) { return failure(p.parseLess() || p.parseType(t) || p.parseComma() || - countOf(p, "DP", dp) || p.parseComma() || countOf(p, "bit", bit) || - p.parseComma() || repeat(p, rep)); + countOf(p, "DP", dp) || p.parseComma() || + countOf(p, "bit", bit) || p.parseComma() || repeat(p, rep)); } -static void printTmemHead(AsmPrinter &p, Type t, int dp, int bit, unsigned rep) { +static void printTmemHead(AsmPrinter &p, Type t, int dp, int bit, + unsigned rep) { p << '<' << t << ", " << dp << " DP, " << bit << " bit, x" << rep; } // An optional `, word`. static ParseResult parseFlag(AsmParser &p, StringRef word, UnitAttr &out) { - if (failed(p.parseOptionalComma())) return success(); - if (p.parseKeyword(word)) return failure(); + if (failed(p.parseOptionalComma())) + return success(); + if (p.parseKeyword(word)) + return failure(); out = UnitAttr::get(p.getContext()); return success(); } -static LogicalResult verifyTmemCopy(EmitFn emitError, int dp, int bit, unsigned rep) { +static LogicalResult verifyTmemCopy(EmitFn emitError, int dp, int bit, + unsigned rep) { if (dp != 16 && dp != 32) - return emitError() << "expects the number of data paths to be one of [16,32], but got " << dp; + return emitError() + << "expects the number of data paths to be one of [16,32], but got " + << dp; if (bit != 32 && bit != 64 && bit != 128 && bit != 256) - return emitError() << "expects the number of bits to be one of [32,64,128,256], but got " << bit; + return emitError() << "expects the number of bits to be one of " + "[32,64,128,256], but got " + << bit; if (rep < 1 || rep > 128 || !llvm::isPowerOf2_32(rep)) return emitError() << "expects the number of repetitions to be one of " - "[1,2,4,8,16,32,64,128], but got " << rep; + "[1,2,4,8,16,32,64,128], but got " + << rep; if (dp == 32 && bit != 32) - return emitError() << "only 32b patterns are supported when the number of data paths " - "is 32, but got " << bit; + return emitError() + << "only 32b patterns are supported when the number of data paths " + "is 32, but got " + << bit; if (dp == 16 && bit == 256 && rep >= 64) - return emitError() << "a repetition of 64 or 128 is not supported when the number of " - "data paths is 16 and the number of bits is 256"; + return emitError() + << "a repetition of 64 or 128 is not supported when the number of " + "data paths is 16 and the number of bits is 256"; if (dp == 16 && bit == 128 && rep == 128) - return emitError() << "a repetition of 128 is not supported when the number of data " - "paths is 16 and the number of bits is 128"; + return emitError() + << "a repetition of 128 is not supported when the number of data " + "paths is 16 and the number of bits is 128"; return success(); } @@ -673,16 +791,19 @@ Type CopyAtomSM100TmemLoadType::parse(AsmParser &p) { int dp, bit; unsigned rep; UnitAttr pack; - if (parseTmemHead(p, t, dp, bit, rep) || parseFlag(p, "pack16b", pack) || p.parseGreater()) + if (parseTmemHead(p, t, dp, bit, rep) || parseFlag(p, "pack16b", pack) || + p.parseGreater()) return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, bit, rep, pack); + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, + bit, rep, pack); } void CopyAtomSM100TmemLoadType::print(AsmPrinter &p) const { printTmemHead(p, getValType(), getNumDp(), getNumBit(), getNumRep()); p << (getPack16b() ? ", pack16b>" : ">"); } -LogicalResult CopyAtomSM100TmemLoadType::verify(EmitFn emitError, Type, int dp, int bit, - unsigned rep, UnitAttr) { +LogicalResult CopyAtomSM100TmemLoadType::verify(EmitFn emitError, Type, int dp, + int bit, unsigned rep, + UnitAttr) { return verifyTmemCopy(emitError, dp, bit, rep); } @@ -695,14 +816,16 @@ Type CopyAtomSM100TmemStoreType::parse(AsmParser &p) { if (parseTmemHead(p, t, dp, bit, rep) || parseFlag(p, "expand16b", expand) || p.parseGreater()) return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, bit, rep, expand); + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, + bit, rep, expand); } void CopyAtomSM100TmemStoreType::print(AsmPrinter &p) const { printTmemHead(p, getValType(), getNumDp(), getNumBit(), getNumRep()); p << (getExpand16b() ? ", expand16b>" : ">"); } -LogicalResult CopyAtomSM100TmemStoreType::verify(EmitFn emitError, Type, int dp, int bit, - unsigned rep, UnitAttr) { +LogicalResult CopyAtomSM100TmemStoreType::verify(EmitFn emitError, Type, int dp, + int bit, unsigned rep, + UnitAttr) { return verifyTmemCopy(emitError, dp, bit, rep); } @@ -715,41 +838,53 @@ Type CopyAtomSM10xTmemLoadRedType::parse(AsmParser &p) { TmemLoadRedOp op; UnitAttr nan; IntegerAttr split; - if (parseTmemHead(p, t, dp, bit, rep) || p.parseComma() || field(p, op)) return {}; + if (parseTmemHead(p, t, dp, bit, rep) || p.parseComma() || field(p, op)) + return {}; while (succeeded(p.parseOptionalComma())) { if (!nan && !split && succeeded(p.parseOptionalKeyword("nan"))) { nan = UnitAttr::get(p.getContext()); continue; } - if (p.parseKeyword("half_split_off") || p.parseEqual() || p.parseAttribute(split)) + if (p.parseKeyword("half_split_off") || p.parseEqual() || + p.parseAttribute(split)) return {}; } - if (p.parseGreater()) return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, bit, rep, op, nan, - split); + if (p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, + bit, rep, op, nan, split); } void CopyAtomSM10xTmemLoadRedType::print(AsmPrinter &p) const { printTmemHead(p, getValType(), getNumDp(), getNumBit(), getNumRep()); p << ", " << stringifyEnum(getRedOp()); - if (getNan()) p << ", nan"; - if (getHalfSplitOff()) p << ", half_split_off=" << getHalfSplitOff(); + if (getNan()) + p << ", nan"; + if (getHalfSplitOff()) + p << ", half_split_off=" << getHalfSplitOff(); p << '>'; } -LogicalResult CopyAtomSM10xTmemLoadRedType::verify(EmitFn emitError, Type t, int dp, int bit, - unsigned rep, TmemLoadRedOp op, UnitAttr nan, - IntegerAttr split) { +LogicalResult +CopyAtomSM10xTmemLoadRedType::verify(EmitFn emitError, Type t, int dp, int bit, + unsigned rep, TmemLoadRedOp op, + UnitAttr nan, IntegerAttr split) { if (!(t.isF32() || t.isInteger(32))) - return emitError() << "expect val_type to be u32, s32, or f32, but got '" << t << "'"; + return emitError() << "expect val_type to be u32, s32, or f32, but got '" + << t << "'"; if (nan && !t.isF32()) return emitError() << "expect val_type to be f32 when nan is specified"; - if ((op == TmemLoadRedOp::minabs || op == TmemLoadRedOp::maxabs) && !t.isF32()) - return emitError() << "expect val_type to be f32 when red_op is minabs or maxabs"; - if (bit != 32) return emitError() << "only 32b patterns are supported, but got " << bit; + if ((op == TmemLoadRedOp::minabs || op == TmemLoadRedOp::maxabs) && + !t.isF32()) + return emitError() + << "expect val_type to be f32 when red_op is minabs or maxabs"; + if (bit != 32) + return emitError() << "only 32b patterns are supported, but got " << bit; if (rep < 2 || rep > 128 || !llvm::isPowerOf2_32(rep)) return emitError() << "expects the number of repetitions to be one of " - "[2,4,8,16,32,64,128], but got " << rep; + "[2,4,8,16,32,64,128], but got " + << rep; if (split && dp != 16) - return emitError() << "expect half_split_off to be set only when mode is 16dp 32bit"; + return emitError() + << "expect half_split_off to be set only when mode is 16dp 32bit"; return success(); } @@ -762,36 +897,43 @@ Type CopyAtomSM107TmemLoadSPCompressType::parse(AsmParser &p) { TmemLoadRedOp op; UnitAttr red, nan; if (parseTmemHead(p, t, dp, bit, rep) || p.parseComma() || field(p, op) || - parseFlag(p, "red", red) || (red && parseFlag(p, "nan", nan)) || p.parseGreater()) + parseFlag(p, "red", red) || (red && parseFlag(p, "nan", nan)) || + p.parseGreater()) return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, bit, rep, op, red, - nan); + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, + bit, rep, op, red, nan); } void CopyAtomSM107TmemLoadSPCompressType::print(AsmPrinter &p) const { printTmemHead(p, getValType(), getNumDp(), getNumBit(), getNumRep()); p << ", " << stringifyEnum(getRedOp()); - if (getRed()) p << ", red"; - if (getNan()) p << ", nan"; + if (getRed()) + p << ", red"; + if (getNan()) + p << ", nan"; p << '>'; } -LogicalResult CopyAtomSM107TmemLoadSPCompressType::verify(EmitFn emitError, Type t, int dp, - int bit, unsigned rep, TmemLoadRedOp, - UnitAttr red, UnitAttr nan) { +LogicalResult CopyAtomSM107TmemLoadSPCompressType::verify( + EmitFn emitError, Type t, int dp, int bit, unsigned rep, TmemLoadRedOp, + UnitAttr red, UnitAttr nan) { if (dp != 32 || bit != 32) - return emitError() << "expect `num_dp` and `num_bit` to be 32, but got " << dp << " and " - << bit; - if (!t.isF32()) return emitError() << "expect val_type to be f32, but got '" << t << "'"; + return emitError() << "expect `num_dp` and `num_bit` to be 32, but got " + << dp << " and " << bit; + if (!t.isF32()) + return emitError() << "expect val_type to be f32, but got '" << t << "'"; if (rep < 4 || rep > 128 || !llvm::isPowerOf2_32(rep)) return emitError() << "expect the number of repetitions to be one of " - "[4,8,16,32,64,128], but got " << rep; - if (nan && !red) return emitError() << "expect `red` to be true when `nan` is true"; + "[4,8,16,32,64,128], but got " + << rep; + if (nan && !red) + return emitError() << "expect `red` to be true when `nan` is true"; return success(); } // The UTCCP shapes of tcgen05.cp. static bool isUtccp(int dp, int bit, CopyS2TBroadcast b) { using B = CopyS2TBroadcast; - return (b == B::none && (dp == 4 || dp == 128) && (bit == 128 || bit == 256)) || + return (b == B::none && (dp == 4 || dp == 128) && + (bit == 128 || bit == 256)) || (b == B::x4 && dp == 32 && bit == 128) || ((b == B::lw_0213 || b == B::lw_0123) && dp == 64 && bit == 128); } @@ -802,25 +944,33 @@ Type CopyAtomSM100CopyS2TType::parse(AsmParser &p) { Type t; int dp, bit, cta; auto bcast = CopyS2TBroadcast::none; - if (p.parseLess() || p.parseType(t) || p.parseComma() || countOf(p, "DP", dp) || - p.parseComma() || countOf(p, "bit", bit) || p.parseComma() || countOf(p, "cta", cta)) + if (p.parseLess() || p.parseType(t) || p.parseComma() || + countOf(p, "DP", dp) || p.parseComma() || countOf(p, "bit", bit) || + p.parseComma() || countOf(p, "cta", cta)) return {}; - if (succeeded(p.parseOptionalComma()) && field(p, bcast)) return {}; - if (p.parseGreater()) return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, bit, cta, bcast); + if (succeeded(p.parseOptionalComma()) && field(p, bcast)) + return {}; + if (p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, + bit, cta, bcast); } void CopyAtomSM100CopyS2TType::print(AsmPrinter &p) const { - p << '<' << getValType() << ", " << getNumDp() << " DP, " << getNumBit() << " bit, " - << getNumCta() << " cta"; - if (getBroadcast() != CopyS2TBroadcast::none) p << ", " << stringifyEnum(getBroadcast()); + p << '<' << getValType() << ", " << getNumDp() << " DP, " << getNumBit() + << " bit, " << getNumCta() << " cta"; + if (getBroadcast() != CopyS2TBroadcast::none) + p << ", " << stringifyEnum(getBroadcast()); p << '>'; } -LogicalResult CopyAtomSM100CopyS2TType::verify(EmitFn emitError, Type, int dp, int bit, int cta, +LogicalResult CopyAtomSM100CopyS2TType::verify(EmitFn emitError, Type, int dp, + int bit, int cta, CopyS2TBroadcast b) { - if (cta != 1 && cta != 2) return emitError() << "Expect cta is 1/2 but got: " << cta; + if (cta != 1 && cta != 2) + return emitError() << "Expect cta is 1/2 but got: " << cta; if (!isUtccp(dp, bit, b)) - return emitError() << "The given UTCCP params: dp " << dp << ", bits " << bit - << ", broadcast " << stringifyEnum(b) << " is unsupported or illegal."; + return emitError() << "The given UTCCP params: dp " << dp << ", bits " + << bit << ", broadcast " << stringifyEnum(b) + << " is unsupported or illegal."; return success(); } @@ -835,25 +985,31 @@ Type CopyAtomSM100S2TCopyV2Type::parse(AsmParser &p) { nextKeyField(p, "num_bit", bit) || nextKeyField(p, "num_cta", cta) || nextKeyField(p, "smem_major", major)) return {}; - if (succeeded(p.parseOptionalComma()) && keyField(p, "broadcast", bcast)) return {}; - if (p.parseGreater()) return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, bit, cta, major, - bcast); + if (succeeded(p.parseOptionalComma()) && keyField(p, "broadcast", bcast)) + return {}; + if (p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, + bit, cta, major, bcast); } void CopyAtomSM100S2TCopyV2Type::print(AsmPrinter &p) const { - p << '<' << getValType() << ", num_dp = " << getNumDp() << ", num_bit = " << getNumBit() - << ", num_cta = " << getNumCta() << ", smem_major = " << stringifyEnum(getSmemMajor()); + p << '<' << getValType() << ", num_dp = " << getNumDp() + << ", num_bit = " << getNumBit() << ", num_cta = " << getNumCta() + << ", smem_major = " << stringifyEnum(getSmemMajor()); if (getBroadcast() != CopyS2TBroadcast::none) p << ", broadcast = " << stringifyEnum(getBroadcast()); p << '>'; } -LogicalResult CopyAtomSM100S2TCopyV2Type::verify(EmitFn emitError, Type, int dp, int bit, - int cta, cute::MajorMode, +LogicalResult CopyAtomSM100S2TCopyV2Type::verify(EmitFn emitError, Type, int dp, + int bit, int cta, + cute::MajorMode, CopyS2TBroadcast b) { - if (cta != 1 && cta != 2) return emitError() << "expects cta to be 1 or 2, but got " << cta; + if (cta != 1 && cta != 2) + return emitError() << "expects cta to be 1 or 2, but got " << cta; if (!isUtccp(dp, bit, b)) - return emitError() << "expects valid UTCCP params, but got dp " << dp << ", bits " << bit - << ", broadcast " << stringifyEnum(b); + return emitError() << "expects valid UTCCP params, but got dp " << dp + << ", bits " << bit << ", broadcast " + << stringifyEnum(b); return success(); } @@ -867,17 +1023,19 @@ Type CopyAtomLdsmType::parse(AsmParser &p) { LdsmSzPattern sz; int n; UnitAttr trans; - if (p.parseLess() || keyField(p, "val_type", t) || nextKeyField(p, "mode", mode) || - nextKeyField(p, "sz_pattern", sz) || nextKeyField(p, "num_matrices", n) || - p.parseComma() || parseTranspose(p, trans) || p.parseGreater()) + if (p.parseLess() || keyField(p, "val_type", t) || + nextKeyField(p, "mode", mode) || nextKeyField(p, "sz_pattern", sz) || + nextKeyField(p, "num_matrices", n) || p.parseComma() || + parseTranspose(p, trans) || p.parseGreater()) return {}; return get(p.getContext(), t, mode, sz, n, trans); } void CopyAtomLdsmType::print(AsmPrinter &p) const { p << "" : ", n>"); + p << ", sz_pattern = " << stringifyEnum(getSzPattern()) + << ", num_matrices = " << getNumMatrices() + << (getTranspose() ? ", t>" : ", n>"); } Type CopyAtomStsmType::parse(AsmParser &p) { @@ -886,15 +1044,16 @@ Type CopyAtomStsmType::parse(AsmParser &p) { int n; UnitAttr trans; if (p.parseLess() || p.parseType(t) || nextKeyField(p, "mode", mode) || - nextKeyField(p, "num_matrices", n) || p.parseComma() || parseTranspose(p, trans) || - p.parseGreater()) + nextKeyField(p, "num_matrices", n) || p.parseComma() || + parseTranspose(p, trans) || p.parseGreater()) return {}; return get(p.getContext(), t, mode, n, trans); } void CopyAtomStsmType::print(AsmPrinter &p) const { p << '<' << getValType() << ", mode = "; p.printStrippedAttrOrType(getMode()); - p << ", num_matrices = " << getNumMatrices() << (getTranspose() ? ", t>" : ", n>"); + p << ", num_matrices = " << getNumMatrices() + << (getTranspose() ? ", t>" : ", n>"); } //===----------------------------------------------------------------------===// @@ -902,13 +1061,18 @@ void CopyAtomStsmType::print(AsmPrinter &p) const { //===----------------------------------------------------------------------===// static LogicalResult verifySparsity(EmitFn emitError, int sparsity) { - if (sparsity != 1 && sparsity != 2 && sparsity != 4 && sparsity != 8 && sparsity != 16) - return emitError() << "expects sparsity to be one of (1, 2, 4, 8, 16), but got " << sparsity; + if (sparsity != 1 && sparsity != 2 && sparsity != 4 && sparsity != 8 && + sparsity != 16) + return emitError() + << "expects sparsity to be one of (1, 2, 4, 8, 16), but got " + << sparsity; return success(); } // `'; } -LogicalResult CopyAtomTmaLoadType::verify(EmitFn emitError, Type, int sparsity, int, - TmaLoadMode, int, cute::StrideType, bool, +LogicalResult CopyAtomTmaLoadType::verify(EmitFn emitError, Type, int sparsity, + int, TmaLoadMode, int, + cute::StrideType, bool, cute::LayoutType, bool, bool) { return verifySparsity(emitError, sparsity); } @@ -977,12 +1146,13 @@ Type CopyAtomTmaStoreType::parse(AsmParser &p) { cute::LayoutType gbasis; bool override = false, noOob = false; if (parseTmaHead(p, t, &sparsity, bits) || nextKeyField(p, "mode", mode) || - nextKeyField(p, "g_stride", stride) || optKeyField(p, "tma_gbasis", gbasis) || - optKeyField(p, "override", override) || optKeyField(p, "no_fully_oob_tile", noOob) || - p.parseGreater()) + nextKeyField(p, "g_stride", stride) || + optKeyField(p, "tma_gbasis", gbasis) || + optKeyField(p, "override", override) || + optKeyField(p, "no_fully_oob_tile", noOob) || p.parseGreater()) return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, sparsity, bits, mode, - stride, gbasis, override, noOob); + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, + sparsity, bits, mode, stride, gbasis, override, noOob); } void CopyAtomTmaStoreType::print(AsmPrinter &p) const { printTmaHead(p, getValType(), getSparsity(), getCopyBits()); @@ -992,9 +1162,9 @@ void CopyAtomTmaStoreType::print(AsmPrinter &p) const { printTrue(p, "no_fully_oob_tile", getNoFullyOobTile()); p << '>'; } -LogicalResult CopyAtomTmaStoreType::verify(EmitFn emitError, Type, int sparsity, int, - TmaStoreMode, cute::StrideType, cute::LayoutType, - bool, bool) { +LogicalResult CopyAtomTmaStoreType::verify(EmitFn emitError, Type, int sparsity, + int, TmaStoreMode, cute::StrideType, + cute::LayoutType, bool, bool) { return verifySparsity(emitError, sparsity); } @@ -1013,7 +1183,8 @@ Type CopyAtomTmaReduceType::parse(AsmParser &p) { } void CopyAtomTmaReduceType::print(AsmPrinter &p) const { printTmaHead(p, getValType(), 1, getCopyBits()); - p << ", mode = " << stringifyEnum(getMode()) << ", kind = " << stringifyEnum(getKind()); + p << ", mode = " << stringifyEnum(getMode()) + << ", kind = " << stringifyEnum(getKind()); printTmaTail(p, getGStride(), getTmaGbasis()); p << '>'; } @@ -1047,7 +1218,8 @@ Type CopyAtomIm2ColTmaStoreType::parse(AsmParser &p) { int bits; cute::StrideType stride; cute::LayoutType gbasis; - if (parseTmaHead(p, t, nullptr, bits) || nextKeyField(p, "g_stride", stride) || + if (parseTmaHead(p, t, nullptr, bits) || + nextKeyField(p, "g_stride", stride) || optKeyField(p, "tma_gbasis", gbasis) || p.parseGreater()) return {}; return get(p.getContext(), t, bits, stride, gbasis); @@ -1059,21 +1231,34 @@ void CopyAtomIm2ColTmaStoreType::print(AsmPrinter &p) const { } // The TMA data format of an element type when none is given. -std::optional mlir::cutlass_compiler::cute_nvgpu::defaultTmaDataFormat(Type t) { - if (auto s = llvm::dyn_cast(t)) t = s.getPhysicalType(); - if (t.isF16()) return TmaDataFormat::F16_RN; - if (t.isBF16()) return TmaDataFormat::BF16_RN; - if (t.isF32()) return TmaDataFormat::F32_RN; - if (t.isTF32()) return TmaDataFormat::TF32_RN; - if (t.isF64()) return TmaDataFormat::F64_RN; - if (!t.isIntOrFloat()) return std::nullopt; +std::optional +mlir::cutlass_compiler::cute_nvgpu::defaultTmaDataFormat(Type t) { + if (auto s = llvm::dyn_cast(t)) + t = s.getPhysicalType(); + if (t.isF16()) + return TmaDataFormat::F16_RN; + if (t.isBF16()) + return TmaDataFormat::BF16_RN; + if (t.isF32()) + return TmaDataFormat::F32_RN; + if (t.isTF32()) + return TmaDataFormat::TF32_RN; + if (t.isF64()) + return TmaDataFormat::F64_RN; + if (!t.isIntOrFloat()) + return std::nullopt; unsigned bits = t.getIntOrFloatBitWidth(); bool isSigned = t.isSignedInteger(); - if (bits == 4) return TmaDataFormat::U4; - if (bits <= 8) return TmaDataFormat::U8; - if (bits == 16) return TmaDataFormat::U16; - if (bits == 32) return isSigned ? TmaDataFormat::S32 : TmaDataFormat::U32; - if (bits == 64) return isSigned ? TmaDataFormat::S64 : TmaDataFormat::U64; + if (bits == 4) + return TmaDataFormat::U4; + if (bits <= 8) + return TmaDataFormat::U8; + if (bits == 16) + return TmaDataFormat::U16; + if (bits == 32) + return isSigned ? TmaDataFormat::S32 : TmaDataFormat::U32; + if (bits == 64) + return isSigned ? TmaDataFormat::S64 : TmaDataFormat::U64; return std::nullopt; } @@ -1081,83 +1266,93 @@ namespace { // `<[kind, ]T, copy_bits = N, tma_gbasis = L[, tma_format = F]>`. template ParseResult parseNonExecTma(AsmParser &p, Kind *kind, Type &t, int &bits, - cute::LayoutType &gbasis, std::optional &format) { - if (p.parseLess()) return failure(); - if (kind && (field(p, *kind) || p.parseComma())) return failure(); + cute::LayoutType &gbasis, + std::optional &format) { + if (p.parseLess()) + return failure(); + if (kind && (field(p, *kind) || p.parseComma())) + return failure(); if (p.parseType(t) || nextKeyField(p, "copy_bits", bits) || nextKeyField(p, "tma_gbasis", gbasis)) return failure(); format = defaultTmaDataFormat(t); if (succeeded(p.parseOptionalComma())) { // Empty for an element type without a format, as the compiler prints it. - if (p.parseKeyword("tma_format") || p.parseEqual()) return failure(); + if (p.parseKeyword("tma_format") || p.parseEqual()) + return failure(); if (succeeded(p.parseOptionalGreater())) { format = std::nullopt; return success(); } TmaDataFormat f; - if (field(p, f)) return failure(); + if (field(p, f)) + return failure(); format = f; } return p.parseGreater(); } -template -void printNonExecTma(AsmPrinter &p, T type) { - p << type.getValType() << ", copy_bits = " << type.getCopyBits() << ", tma_gbasis = "; +template void printNonExecTma(AsmPrinter &p, T type) { + p << type.getValType() << ", copy_bits = " << type.getCopyBits() + << ", tma_gbasis = "; p.printStrippedAttrOrType(type.getTmaGbasis()); p << ", tma_format = "; - if (type.getTmaFormat()) p << stringifyEnum(*type.getTmaFormat()); + if (type.getTmaFormat()) + p << stringifyEnum(*type.getTmaFormat()); p << '>'; } LogicalResult verifyNonExecTma(EmitFn emitError, int bits) { - if (bits == 0) return emitError() << "invalid copy_bits, got 0"; + if (bits == 0) + return emitError() << "invalid copy_bits, got 0"; return success(); } -} // namespace +} // namespace -#define NON_EXEC_TMA_WITH_KIND(TYPE, KIND) \ - Type TYPE::parse(AsmParser &p) { \ - SMLoc loc = p.getCurrentLocation(); \ - KIND kind; \ - Type t; \ - int bits; \ - cute::LayoutType gbasis; \ - std::optional format; \ - if (parseNonExecTma(p, &kind, t, bits, gbasis, format)) return {}; \ - return getChecked([&] { return p.emitError(loc); }, p.getContext(), kind, t, \ - bits, gbasis, format); \ - } \ - void TYPE::print(AsmPrinter &p) const { \ - p << '<' << stringifyEnum(getKind()) << ", "; \ - printNonExecTma(p, *this); \ - } \ - LogicalResult TYPE::verify(EmitFn emitError, KIND, Type, int bits, \ - cute::LayoutType, std::optional) { \ - return verifyNonExecTma(emitError, bits); \ +#define NON_EXEC_TMA_WITH_KIND(TYPE, KIND) \ + Type TYPE::parse(AsmParser &p) { \ + SMLoc loc = p.getCurrentLocation(); \ + KIND kind; \ + Type t; \ + int bits; \ + cute::LayoutType gbasis; \ + std::optional format; \ + if (parseNonExecTma(p, &kind, t, bits, gbasis, format)) \ + return {}; \ + return getChecked([&] { return p.emitError(loc); }, p.getContext(), kind, \ + t, bits, gbasis, format); \ + } \ + void TYPE::print(AsmPrinter &p) const { \ + p << '<' << stringifyEnum(getKind()) << ", "; \ + printNonExecTma(p, *this); \ + } \ + LogicalResult TYPE::verify(EmitFn emitError, KIND, Type, int bits, \ + cute::LayoutType, std::optional) { \ + return verifyNonExecTma(emitError, bits); \ } -#define NON_EXEC_TMA(TYPE) \ - Type TYPE::parse(AsmParser &p) { \ - SMLoc loc = p.getCurrentLocation(); \ - Type t; \ - int bits; \ - cute::LayoutType gbasis; \ - std::optional format; \ - if (parseNonExecTma(p, nullptr, t, bits, gbasis, format)) return {}; \ - return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, bits, \ - gbasis, format); \ - } \ - void TYPE::print(AsmPrinter &p) const { \ - p << '<'; \ - printNonExecTma(p, *this); \ - } \ - LogicalResult TYPE::verify(EmitFn emitError, Type, int bits, cute::LayoutType, \ - std::optional) { \ - return verifyNonExecTma(emitError, bits); \ +#define NON_EXEC_TMA(TYPE) \ + Type TYPE::parse(AsmParser &p) { \ + SMLoc loc = p.getCurrentLocation(); \ + Type t; \ + int bits; \ + cute::LayoutType gbasis; \ + std::optional format; \ + if (parseNonExecTma(p, nullptr, t, bits, gbasis, format)) \ + return {}; \ + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, \ + bits, gbasis, format); \ + } \ + void TYPE::print(AsmPrinter &p) const { \ + p << '<'; \ + printNonExecTma(p, *this); \ + } \ + LogicalResult TYPE::verify(EmitFn emitError, Type, int bits, \ + cute::LayoutType, std::optional) { \ + return verifyNonExecTma(emitError, bits); \ } NON_EXEC_TMA_WITH_KIND(CopyAtomNonExecTiledTmaLoadType, TiledTmaLoad) NON_EXEC_TMA_WITH_KIND(CopyAtomNonExecTiledTmaReduceType, ReductionKind) NON_EXEC_TMA_WITH_KIND(CopyAtomNonExecIm2ColTmaLoadType, Im2ColTmaLoad) -NON_EXEC_TMA_WITH_KIND(CopyAtomNonExec2DGather4TmaLoadType, GatherScatterTmaLoad) +NON_EXEC_TMA_WITH_KIND(CopyAtomNonExec2DGather4TmaLoadType, + GatherScatterTmaLoad) NON_EXEC_TMA(CopyAtomNonExecTiledTmaStoreType) NON_EXEC_TMA(CopyAtomNonExecIm2ColTmaStoreType) NON_EXEC_TMA(CopyAtomNonExec2DScatter4TmaStoreType) @@ -1172,18 +1367,21 @@ Type UniversalFmaAtomType::parse(AsmParser &p) { SMLoc loc = p.getCurrentLocation(); cute::ShapeAttr shape; Type a, b, c; - if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || p.parseLParen() || - p.parseType(a) || p.parseComma() || p.parseType(b) || p.parseRParen() || - p.parseArrow() || p.parseType(c) || p.parseGreater()) + if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || + p.parseLParen() || p.parseType(a) || p.parseComma() || p.parseType(b) || + p.parseRParen() || p.parseArrow() || p.parseType(c) || p.parseGreater()) return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, a, b, c); + return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, a, + b, c); } void UniversalFmaAtomType::print(AsmPrinter &p) const { p << '<'; printShapeMNK(p, getShapeMnk()); - p << ", (" << getAType() << ", " << getBType() << ") -> " << getCType() << " >"; + p << ", (" << getAType() << ", " << getBType() << ") -> " << getCType() + << " >"; } -LogicalResult UniversalFmaAtomType::verify(EmitFn emitError, cute::ShapeAttr shape, Type, Type, +LogicalResult UniversalFmaAtomType::verify(EmitFn emitError, + cute::ShapeAttr shape, Type, Type, Type) { return verifyShapeMNK(emitError, shape); } @@ -1195,15 +1393,17 @@ Type MmaAtomSM80Type::parse(AsmParser &p) { Type a, b, c; MMAIntOverflowAttr overflow; BinaryOpAttr op; - if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || elemTypes(p, a, b, c)) + if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || + elemTypes(p, a, b, c)) return {}; if (succeeded(p.parseOptionalComma()) && (keyField(p, "int_overflow", overflow) || (succeeded(p.parseOptionalComma()) && keyField(p, "binary_op", op)))) return {}; - if (p.parseGreater()) return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, a, b, c, overflow, - op); + if (p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, a, + b, c, overflow, op); } void MmaAtomSM80Type::print(AsmPrinter &p) const { p << '<'; @@ -1214,8 +1414,9 @@ void MmaAtomSM80Type::print(AsmPrinter &p) const { printAttrField(p, "binary_op", getBinaryOp()); p << " >"; } -LogicalResult MmaAtomSM80Type::verify(EmitFn emitError, cute::ShapeAttr shape, Type, Type, Type, - MMAIntOverflowAttr, BinaryOpAttr) { +LogicalResult MmaAtomSM80Type::verify(EmitFn emitError, cute::ShapeAttr shape, + Type, Type, Type, MMAIntOverflowAttr, + BinaryOpAttr) { return verifyShapeMNK(emitError, shape); } @@ -1226,13 +1427,16 @@ Type MmaAtomSM80SparseType::parse(AsmParser &p) { Type a, b, c; SparseMetadataFormat format; MMAIntOverflowAttr overflow; - if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || elemTypes(p, a, b, c) || - nextKeyField(p, "md_format", format)) + if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || + elemTypes(p, a, b, c) || nextKeyField(p, "md_format", format)) return {}; - if (succeeded(p.parseOptionalComma()) && keyField(p, "int_overflow", overflow)) return {}; - if (p.parseGreater()) return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, a, b, c, format, - overflow); + if (succeeded(p.parseOptionalComma()) && + keyField(p, "int_overflow", overflow)) + return {}; + if (p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, a, + b, c, format, overflow); } void MmaAtomSM80SparseType::print(AsmPrinter &p) const { p << '<'; @@ -1243,8 +1447,10 @@ void MmaAtomSM80SparseType::print(AsmPrinter &p) const { printAttrField(p, "int_overflow", getIntOverflow()); p << " >"; } -LogicalResult MmaAtomSM80SparseType::verify(EmitFn emitError, cute::ShapeAttr shape, Type, Type, - Type, SparseMetadataFormat, MMAIntOverflowAttr) { +LogicalResult MmaAtomSM80SparseType::verify(EmitFn emitError, + cute::ShapeAttr shape, Type, Type, + Type, SparseMetadataFormat, + MMAIntOverflowAttr) { return verifyShapeMNK(emitError, shape); } @@ -1252,10 +1458,11 @@ Type MmaAtomSM89Type::parse(AsmParser &p) { SMLoc loc = p.getCurrentLocation(); cute::ShapeAttr shape; Type a, b, c; - if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || elemTypes(p, a, b, c) || - p.parseGreater()) + if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || + elemTypes(p, a, b, c) || p.parseGreater()) return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, a, b, c); + return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, a, + b, c); } void MmaAtomSM89Type::print(AsmPrinter &p) const { p << '<'; @@ -1264,8 +1471,8 @@ void MmaAtomSM89Type::print(AsmPrinter &p) const { printElemTypes(p, getAType(), getBType(), getCType()); p << " >"; } -LogicalResult MmaAtomSM89Type::verify(EmitFn emitError, cute::ShapeAttr shape, Type, Type, - Type) { +LogicalResult MmaAtomSM89Type::verify(EmitFn emitError, cute::ShapeAttr shape, + Type, Type, Type) { return verifyShapeMNK(emitError, shape); } @@ -1280,67 +1487,79 @@ Type MmaAtomSM90Type::parse(AsmParser &p) { MMAIntOverflowAttr overflow; UnitAttr aNeg, bNeg; if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || - p.parseKeyword("ab_major") || p.parseEqual() || pairOf(p, aMajor, bMajor) || - p.parseComma() || elemTypes(p, a, b, c) || p.parseComma() || - parseFragKind(p, MmaFragKind::rmem, frag)) + p.parseKeyword("ab_major") || p.parseEqual() || + pairOf(p, aMajor, bMajor) || p.parseComma() || elemTypes(p, a, b, c) || + p.parseComma() || parseFragKind(p, MmaFragKind::rmem, frag)) return {}; while (succeeded(p.parseOptionalComma())) { SMLoc at = p.getCurrentLocation(); - if (succeeded(p.parseOptionalKeyword("a_neg"))) aNeg = UnitAttr::get(p.getContext()); - else if (succeeded(p.parseOptionalKeyword("b_neg"))) bNeg = UnitAttr::get(p.getContext()); + if (succeeded(p.parseOptionalKeyword("a_neg"))) + aNeg = UnitAttr::get(p.getContext()); + else if (succeeded(p.parseOptionalKeyword("b_neg"))) + bNeg = UnitAttr::get(p.getContext()); else if (succeeded(p.parseOptionalKeyword("int_overflow"))) { - if (p.parseEqual() || field(p, overflow)) return {}; + if (p.parseEqual() || field(p, overflow)) + return {}; } else { p.emitError(at, "expected `int_overflow`, `a_neg`, or `b_neg`"); return {}; } } - if (p.parseGreater()) return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, aMajor, bMajor, a, - b, c, frag, overflow, aNeg, bNeg); + if (p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, + aMajor, bMajor, a, b, c, frag, overflow, aNeg, bNeg); } void MmaAtomSM90Type::print(AsmPrinter &p) const { p << '<'; printShapeMNK(p, getShapeMnk()); - p << ", ab_major = (" << stringifyEnum(getAMajor()) << ", " << stringifyEnum(getBMajor()) - << "), "; + p << ", ab_major = (" << stringifyEnum(getAMajor()) << ", " + << stringifyEnum(getBMajor()) << "), "; printElemTypes(p, getAType(), getBType(), getCType()); p << ", frag_kind = " << fragKindText(getAFragKind()); printAttrField(p, "int_overflow", getIntOverflow()); - if (getANeg()) p << ", a_neg"; - if (getBNeg()) p << ", b_neg"; + if (getANeg()) + p << ", a_neg"; + if (getBNeg()) + p << ", b_neg"; p << '>'; } -LogicalResult MmaAtomSM90Type::verify(EmitFn emitError, cute::ShapeAttr shape, cute::MajorMode, - cute::MajorMode, Type, Type, Type, MmaFragKind, +LogicalResult MmaAtomSM90Type::verify(EmitFn emitError, cute::ShapeAttr shape, + cute::MajorMode, cute::MajorMode, Type, + Type, Type, MmaFragKind, MMAIntOverflowAttr, UnitAttr, UnitAttr) { return verifyShapeMNK(emitError, shape); } -// ``. +// ``. Type MmaAtomSM120BlockScaledType::parse(AsmParser &p) { SMLoc loc = p.getCurrentLocation(); cute::ShapeAttr shape; int vec; Type a, b, c, sf; bool useTV; - if (p.parseLess() || parseShapeMNK(p, shape) || nextKeyField(p, "vec_size", vec) || - p.parseComma() || elemTypes(p, a, b, c) || nextKeyField(p, "sf_type", sf) || + if (p.parseLess() || parseShapeMNK(p, shape) || + nextKeyField(p, "vec_size", vec) || p.parseComma() || + elemTypes(p, a, b, c) || nextKeyField(p, "sf_type", sf) || nextKeyField(p, "use_sf_layout_TV", useTV) || p.parseGreater()) return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, vec, a, b, c, sf, - useTV); + return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, + vec, a, b, c, sf, useTV); } void MmaAtomSM120BlockScaledType::print(AsmPrinter &p) const { p << '<'; printShapeMNK(p, getShapeMnk()); p << ", vec_size = " << getVecSize() << ", "; printElemTypes(p, getAType(), getBType(), getCType()); - p << ", sf_type = " << getSfType() << ", use_sf_layout_TV = " - << (getUseSfLayout_TV() ? "true" : "false") << '>'; + p << ", sf_type = " << getSfType() + << ", use_sf_layout_TV = " << (getUseSfLayout_TV() ? "true" : "false") + << '>'; } -LogicalResult MmaAtomSM120BlockScaledType::verify(EmitFn emitError, cute::ShapeAttr shape, int, - Type, Type, Type, Type, bool) { +LogicalResult MmaAtomSM120BlockScaledType::verify(EmitFn emitError, + cute::ShapeAttr shape, int, + Type, Type, Type, Type, + bool) { return verifyShapeMNK(emitError, shape); } @@ -1356,48 +1575,61 @@ struct Umma { MmaFragKind frag = MmaFragKind::smem_desc; int scaleOrVec = 0; Arch arch = Arch::sm_100; - MmaCollectorOp aCollector = MmaCollectorOp::discard, bCollector = MmaCollectorOp::discard; + MmaCollectorOp aCollector = MmaCollectorOp::discard, + bCollector = MmaCollectorOp::discard; MMAIntOverflowAttr overflow; }; ParseResult parseUmma(AsmParser &p, Umma &u) { - if (p.parseLess() || parseShapeMNK(p, u.shape) || nextKeyField(p, "num_cta", u.numCta) || - p.parseComma() || p.parseKeyword("ab_major") || p.parseEqual() || - pairOf(p, u.aMajor, u.bMajor) || p.parseComma() || elemTypes(p, u.a, u.b, u.c)) + if (p.parseLess() || parseShapeMNK(p, u.shape) || + nextKeyField(p, "num_cta", u.numCta) || p.parseComma() || + p.parseKeyword("ab_major") || p.parseEqual() || + pairOf(p, u.aMajor, u.bMajor) || p.parseComma() || + elemTypes(p, u.a, u.b, u.c)) + return failure(); + if (u.sf && nextKeyField(p, "sf_type", u.sfType)) + return failure(); + if (u.sp && !u.sf && nextKeyField(p, "e_type", u.eType)) + return failure(); + if (u.sp && nextKeyField(p, "sparse_metadata_format", u.metadata)) return failure(); - if (u.sf && nextKeyField(p, "sf_type", u.sfType)) return failure(); - if (u.sp && !u.sf && nextKeyField(p, "e_type", u.eType)) return failure(); - if (u.sp && nextKeyField(p, "sparse_metadata_format", u.metadata)) return failure(); if (p.parseComma() || parseFragKind(p, MmaFragKind::tmem, u.frag) || nextKeyField(p, u.bs ? "vec_size" : "c_scale_exp", u.scaleOrVec)) return failure(); bool comma = succeeded(p.parseOptionalComma()); - if (comma && u.bs && !u.sm107 && succeeded(p.parseOptionalKeyword("arch_promote"))) { - if (p.parseEqual() || field(p, u.arch)) return failure(); + if (comma && u.bs && !u.sm107 && + succeeded(p.parseOptionalKeyword("arch_promote"))) { + if (p.parseEqual() || field(p, u.arch)) + return failure(); comma = succeeded(p.parseOptionalComma()); } if (u.sm107) { - if (!comma) return p.parseComma(); + if (!comma) + return p.parseComma(); if (p.parseKeyword("ab_collector_op") || p.parseEqual() || pairOf(p, u.aCollector, u.bCollector)) return failure(); comma = succeeded(p.parseOptionalComma()); } - if (comma && keyField(p, "int_overflow", u.overflow)) return failure(); + if (comma && keyField(p, "int_overflow", u.overflow)) + return failure(); return p.parseGreater(); } void printUmma(AsmPrinter &p, const Umma &u) { p << '<'; printShapeMNK(p, u.shape); - p << ", num_cta = " << u.numCta << ", ab_major = (" << stringifyEnum(u.aMajor) << ", " - << stringifyEnum(u.bMajor) << "), "; + p << ", num_cta = " << u.numCta << ", ab_major = (" << stringifyEnum(u.aMajor) + << ", " << stringifyEnum(u.bMajor) << "), "; printElemTypes(p, u.a, u.b, u.c); - if (u.sf) p << ", sf_type = " << u.sfType; - if (u.sp && !u.sf) p << ", e_type = " << u.eType; - if (u.sp) p << ", sparse_metadata_format = " << stringifyEnum(u.metadata); - p << ", frag_kind = " << fragKindText(u.frag) << (u.bs ? ", vec_size = " : ", c_scale_exp = ") - << u.scaleOrVec; + if (u.sf) + p << ", sf_type = " << u.sfType; + if (u.sp && !u.sf) + p << ", e_type = " << u.eType; + if (u.sp) + p << ", sparse_metadata_format = " << stringifyEnum(u.metadata); + p << ", frag_kind = " << fragKindText(u.frag) + << (u.bs ? ", vec_size = " : ", c_scale_exp = ") << u.scaleOrVec; if (u.bs && !u.sm107 && (u.arch != Arch::sm_100 || u.overflow)) p << ", arch_promote = " << stringifyEnum(u.arch); if (u.sm107) @@ -1408,37 +1640,48 @@ void printUmma(AsmPrinter &p, const Umma &u) { } // `sourced` picks the atom's wording of the tensor-memory A rule. -LogicalResult verifyUmma(EmitFn emitError, cute::ShapeAttr shape, int numCta, MmaFragKind frag, - cute::MajorMode aMajor, int cScaleExp = 0, bool sourced = false) { - if (failed(verifyShapeMNK(emitError, shape))) return failure(); +LogicalResult verifyUmma(EmitFn emitError, cute::ShapeAttr shape, int numCta, + MmaFragKind frag, cute::MajorMode aMajor, + int cScaleExp = 0, bool sourced = false) { + if (failed(verifyShapeMNK(emitError, shape))) + return failure(); if (numCta != 1 && numCta != 2) - return emitError() << "expects `num_cta` equals to 1 or 2, but got " << numCta; + return emitError() << "expects `num_cta` equals to 1 or 2, but got " + << numCta; if (cScaleExp > 15) - return emitError() << "expects `c_scale_exp` in range [0, 15], but got " << cScaleExp; + return emitError() << "expects `c_scale_exp` in range [0, 15], but got " + << cScaleExp; if (frag == MmaFragKind::tmem && aMajor != cute::MajorMode::k) - return emitError() << (sourced ? "a operand must be k-major when sourced from tmem" - : "a operand must be of major-mode k when coming from tmem"); + return emitError() + << (sourced + ? "a operand must be k-major when sourced from tmem" + : "a operand must be of major-mode k when coming from tmem"); return success(); } -} // namespace +} // namespace Type MmaAtomSM100UMMAType::parse(AsmParser &p) { SMLoc loc = p.getCurrentLocation(); Umma u{false, false, false, false}; - if (parseUmma(p, u)) return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, u.numCta, - u.aMajor, u.bMajor, u.a, u.b, u.c, u.frag, u.scaleOrVec, u.overflow); + if (parseUmma(p, u)) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, + u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.frag, + u.scaleOrVec, u.overflow); } void MmaAtomSM100UMMAType::print(AsmPrinter &p) const { Umma u{false, false, false, false}; u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), - u.frag = getAFragKind(), u.scaleOrVec = getCScaleExp(), u.overflow = getIntOverflow(); + u.frag = getAFragKind(), u.scaleOrVec = getCScaleExp(), + u.overflow = getIntOverflow(); printUmma(p, u); } -LogicalResult MmaAtomSM100UMMAType::verify(EmitFn emitError, cute::ShapeAttr shape, int numCta, - cute::MajorMode aMajor, cute::MajorMode, Type, Type, - Type, MmaFragKind frag, int cScaleExp, +LogicalResult MmaAtomSM100UMMAType::verify(EmitFn emitError, + cute::ShapeAttr shape, int numCta, + cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, + MmaFragKind frag, int cScaleExp, MMAIntOverflowAttr) { return verifyUmma(emitError, shape, numCta, frag, aMajor, cScaleExp); } @@ -1446,34 +1689,36 @@ LogicalResult MmaAtomSM100UMMAType::verify(EmitFn emitError, cute::ShapeAttr sha Type MmaAtomSM100UMMASparseType::parse(AsmParser &p) { SMLoc loc = p.getCurrentLocation(); Umma u{false, true, false, false}; - if (parseUmma(p, u)) return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, u.numCta, - u.aMajor, u.bMajor, u.a, u.b, u.c, u.eType, u.metadata, u.frag, - u.scaleOrVec, u.overflow); + if (parseUmma(p, u)) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, + u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.eType, + u.metadata, u.frag, u.scaleOrVec, u.overflow); } void MmaAtomSM100UMMASparseType::print(AsmPrinter &p) const { Umma u{false, true, false, false}; u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), - u.eType = getEType(), u.metadata = getSparseMetadataFormat(), u.frag = getAFragKind(), - u.scaleOrVec = getCScaleExp(), u.overflow = getIntOverflow(); + u.eType = getEType(), u.metadata = getSparseMetadataFormat(), + u.frag = getAFragKind(), u.scaleOrVec = getCScaleExp(), + u.overflow = getIntOverflow(); printUmma(p, u); } -LogicalResult MmaAtomSM100UMMASparseType::verify(EmitFn emitError, cute::ShapeAttr shape, - int numCta, cute::MajorMode aMajor, cute::MajorMode, - Type, Type, Type, Type, SparseMetadataFormat, - MmaFragKind frag, int cScaleExp, - MMAIntOverflowAttr) { +LogicalResult MmaAtomSM100UMMASparseType::verify( + EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, Type, SparseMetadataFormat, + MmaFragKind frag, int cScaleExp, MMAIntOverflowAttr) { return verifyUmma(emitError, shape, numCta, frag, aMajor, cScaleExp); } Type MmaAtomSM100UMMABlockScaledType::parse(AsmParser &p) { SMLoc loc = p.getCurrentLocation(); Umma u{true, false, true, false}; - if (parseUmma(p, u)) return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, u.numCta, - u.aMajor, u.bMajor, u.a, u.b, u.c, u.sfType, u.frag, u.scaleOrVec, u.arch, - u.overflow); + if (parseUmma(p, u)) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, + u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.sfType, + u.frag, u.scaleOrVec, u.arch, u.overflow); } void MmaAtomSM100UMMABlockScaledType::print(AsmPrinter &p) const { Umma u{true, false, true, false}; @@ -1483,92 +1728,101 @@ void MmaAtomSM100UMMABlockScaledType::print(AsmPrinter &p) const { u.arch = getArchPromote(), u.overflow = getIntOverflow(); printUmma(p, u); } -LogicalResult MmaAtomSM100UMMABlockScaledType::verify(EmitFn emitError, cute::ShapeAttr shape, - int numCta, cute::MajorMode aMajor, - cute::MajorMode, Type, Type, Type, Type, - MmaFragKind frag, int, Arch, - MMAIntOverflowAttr) { - return verifyUmma(emitError, shape, numCta, frag, aMajor, 0, /*sourced=*/true); +LogicalResult MmaAtomSM100UMMABlockScaledType::verify( + EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, Type, MmaFragKind frag, int, Arch, + MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor, 0, + /*sourced=*/true); } Type MmaAtomSM100UMMABlockScaledSparseType::parse(AsmParser &p) { SMLoc loc = p.getCurrentLocation(); Umma u{true, true, true, false}; - if (parseUmma(p, u)) return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, u.numCta, - u.aMajor, u.bMajor, u.a, u.b, u.c, u.sfType, u.metadata, u.frag, - u.scaleOrVec, u.arch, u.overflow); + if (parseUmma(p, u)) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, + u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.sfType, + u.metadata, u.frag, u.scaleOrVec, u.arch, u.overflow); } void MmaAtomSM100UMMABlockScaledSparseType::print(AsmPrinter &p) const { Umma u{true, true, true, false}; u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), - u.sfType = getSfType(), u.metadata = getSparseMetadataFormat(), u.frag = getAFragKind(), - u.scaleOrVec = getVecSize(), u.arch = getArchPromote(), u.overflow = getIntOverflow(); + u.sfType = getSfType(), u.metadata = getSparseMetadataFormat(), + u.frag = getAFragKind(), u.scaleOrVec = getVecSize(), + u.arch = getArchPromote(), u.overflow = getIntOverflow(); printUmma(p, u); } LogicalResult MmaAtomSM100UMMABlockScaledSparseType::verify( - EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, cute::MajorMode, - Type, Type, Type, Type, SparseMetadataFormat, MmaFragKind frag, int, Arch, - MMAIntOverflowAttr) { + EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, Type, SparseMetadataFormat, + MmaFragKind frag, int, Arch, MMAIntOverflowAttr) { return verifyUmma(emitError, shape, numCta, frag, aMajor); } Type MmaAtomSM107UMMAType::parse(AsmParser &p) { SMLoc loc = p.getCurrentLocation(); Umma u{false, false, false, true}; - if (parseUmma(p, u)) return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, u.numCta, - u.aMajor, u.bMajor, u.a, u.b, u.c, u.frag, u.scaleOrVec, u.aCollector, - u.bCollector, u.overflow); + if (parseUmma(p, u)) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, + u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.frag, + u.scaleOrVec, u.aCollector, u.bCollector, u.overflow); } void MmaAtomSM107UMMAType::print(AsmPrinter &p) const { Umma u{false, false, false, true}; u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), - u.frag = getAFragKind(), u.scaleOrVec = getCScaleExp(), u.aCollector = getACollectorOp(), - u.bCollector = getBCollectorOp(), u.overflow = getIntOverflow(); + u.frag = getAFragKind(), u.scaleOrVec = getCScaleExp(), + u.aCollector = getACollectorOp(), u.bCollector = getBCollectorOp(), + u.overflow = getIntOverflow(); printUmma(p, u); } -LogicalResult MmaAtomSM107UMMAType::verify(EmitFn emitError, cute::ShapeAttr shape, int numCta, - cute::MajorMode aMajor, cute::MajorMode, Type, Type, - Type, MmaFragKind frag, int cScaleExp, - MmaCollectorOp, MmaCollectorOp, MMAIntOverflowAttr) { +LogicalResult MmaAtomSM107UMMAType::verify( + EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, MmaFragKind frag, int cScaleExp, + MmaCollectorOp, MmaCollectorOp, MMAIntOverflowAttr) { return verifyUmma(emitError, shape, numCta, frag, aMajor, cScaleExp); } Type MmaAtomSM107UMMASparseType::parse(AsmParser &p) { SMLoc loc = p.getCurrentLocation(); Umma u{false, true, false, true}; - if (parseUmma(p, u)) return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, u.numCta, - u.aMajor, u.bMajor, u.a, u.b, u.c, u.eType, u.metadata, u.frag, - u.scaleOrVec, u.aCollector, u.bCollector, u.overflow); + if (parseUmma(p, u)) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, + u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.eType, + u.metadata, u.frag, u.scaleOrVec, u.aCollector, + u.bCollector, u.overflow); } void MmaAtomSM107UMMASparseType::print(AsmPrinter &p) const { Umma u{false, true, false, true}; u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), - u.eType = getEType(), u.metadata = getSparseMetadataFormat(), u.frag = getAFragKind(), - u.scaleOrVec = getCScaleExp(), u.aCollector = getACollectorOp(), - u.bCollector = getBCollectorOp(), u.overflow = getIntOverflow(); + u.eType = getEType(), u.metadata = getSparseMetadataFormat(), + u.frag = getAFragKind(), u.scaleOrVec = getCScaleExp(), + u.aCollector = getACollectorOp(), u.bCollector = getBCollectorOp(), + u.overflow = getIntOverflow(); printUmma(p, u); } -LogicalResult MmaAtomSM107UMMASparseType::verify(EmitFn emitError, cute::ShapeAttr shape, - int numCta, cute::MajorMode aMajor, cute::MajorMode, - Type, Type, Type, Type, SparseMetadataFormat, - MmaFragKind frag, int cScaleExp, MmaCollectorOp, - MmaCollectorOp, MMAIntOverflowAttr) { +LogicalResult MmaAtomSM107UMMASparseType::verify( + EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, Type, SparseMetadataFormat, + MmaFragKind frag, int cScaleExp, MmaCollectorOp, MmaCollectorOp, + MMAIntOverflowAttr) { return verifyUmma(emitError, shape, numCta, frag, aMajor, cScaleExp); } Type MmaAtomSM107UMMABlockScaledType::parse(AsmParser &p) { SMLoc loc = p.getCurrentLocation(); Umma u{true, false, true, true}; - if (parseUmma(p, u)) return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, u.numCta, - u.aMajor, u.bMajor, u.a, u.b, u.c, u.sfType, u.frag, u.scaleOrVec, - u.aCollector, u.bCollector, u.overflow); + if (parseUmma(p, u)) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, + u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.sfType, + u.frag, u.scaleOrVec, u.aCollector, u.bCollector, + u.overflow); } void MmaAtomSM107UMMABlockScaledType::print(AsmPrinter &p) const { Umma u{true, false, true, true}; @@ -1579,36 +1833,40 @@ void MmaAtomSM107UMMABlockScaledType::print(AsmPrinter &p) const { u.overflow = getIntOverflow(); printUmma(p, u); } -LogicalResult MmaAtomSM107UMMABlockScaledType::verify(EmitFn emitError, cute::ShapeAttr shape, - int numCta, cute::MajorMode aMajor, - cute::MajorMode, Type, Type, Type, Type, - MmaFragKind frag, int, MmaCollectorOp, - MmaCollectorOp, MMAIntOverflowAttr) { - return verifyUmma(emitError, shape, numCta, frag, aMajor, 0, /*sourced=*/true); +LogicalResult MmaAtomSM107UMMABlockScaledType::verify( + EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, Type, MmaFragKind frag, int, + MmaCollectorOp, MmaCollectorOp, MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor, 0, + /*sourced=*/true); } Type MmaAtomSM107UMMABlockScaledSparseType::parse(AsmParser &p) { SMLoc loc = p.getCurrentLocation(); Umma u{true, true, true, true}; - if (parseUmma(p, u)) return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, u.numCta, - u.aMajor, u.bMajor, u.a, u.b, u.c, u.sfType, u.metadata, u.frag, - u.scaleOrVec, u.aCollector, u.bCollector, u.overflow); + if (parseUmma(p, u)) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, + u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.sfType, + u.metadata, u.frag, u.scaleOrVec, u.aCollector, + u.bCollector, u.overflow); } void MmaAtomSM107UMMABlockScaledSparseType::print(AsmPrinter &p) const { Umma u{true, true, true, true}; u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), - u.sfType = getSfType(), u.metadata = getSparseMetadataFormat(), u.frag = getAFragKind(), - u.scaleOrVec = getVecSize(), u.aCollector = getACollectorOp(), - u.bCollector = getBCollectorOp(), u.overflow = getIntOverflow(); + u.sfType = getSfType(), u.metadata = getSparseMetadataFormat(), + u.frag = getAFragKind(), u.scaleOrVec = getVecSize(), + u.aCollector = getACollectorOp(), u.bCollector = getBCollectorOp(), + u.overflow = getIntOverflow(); printUmma(p, u); } LogicalResult MmaAtomSM107UMMABlockScaledSparseType::verify( - EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, cute::MajorMode, - Type, Type, Type, Type, SparseMetadataFormat, MmaFragKind frag, int, MmaCollectorOp, - MmaCollectorOp, MMAIntOverflowAttr) { - return verifyUmma(emitError, shape, numCta, frag, aMajor, 0, /*sourced=*/true); + EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, Type, SparseMetadataFormat, + MmaFragKind frag, int, MmaCollectorOp, MmaCollectorOp, MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor, 0, + /*sourced=*/true); } //===----------------------------------------------------------------------===// @@ -1621,13 +1879,16 @@ Type CuteNVGPUDialect::parseType(DialectAsmParser &parser) const { StringRef mnemonic; Type type; OptionalParseResult result = generatedTypeParser(parser, &mnemonic, type); - if (result.has_value()) return succeeded(*result) ? type : Type(); + if (result.has_value()) + return succeeded(*result) ? type : Type(); StringRef width = mnemonic; - if (width.consume_front("i") && !width.empty() && llvm::all_of(width, llvm::isDigit)) { + if (width.consume_front("i") && !width.empty() && + llvm::all_of(width, llvm::isDigit)) { std::string spec = ("!cute." + mnemonic).str(); int64_t divisor; if (succeeded(parser.parseOptionalLess())) { - if (parser.parseKeyword("divby") || parser.parseInteger(divisor) || parser.parseGreater()) + if (parser.parseKeyword("divby") || parser.parseInteger(divisor) || + parser.parseGreater()) return {}; spec += ""; } @@ -1639,21 +1900,22 @@ Type CuteNVGPUDialect::parseType(DialectAsmParser &parser) const { } void CuteNVGPUDialect::printType(Type type, DialectAsmPrinter &printer) const { - if (succeeded(generatedTypePrinter(type, printer))) return; + if (succeeded(generatedTypePrinter(type, printer))) + return; llvm_unreachable("unexpected non-CuteNVGPU type"); } void CuteNVGPUDialect::initialize() { addAttributes< #define GET_ATTRDEF_LIST -#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUAttrs.cpp.inc" +#include "mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.cpp.inc" >(); addTypes< #define GET_TYPEDEF_LIST -#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUTypes.cpp.inc" +#include "mlir/dialects/cute_nvgpu/CuteNVGPUTypes.cpp.inc" >(); addOperations< #define GET_OP_LIST -#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUOps.cpp.inc" +#include "mlir/dialects/cute_nvgpu/CuteNVGPUOps.cpp.inc" >(); } diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h b/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h similarity index 59% rename from mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h rename to mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h index 0011c3de..19d8cb14 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h +++ b/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h @@ -1,11 +1,10 @@ -//===- CuteNVGPUDialect.h - CuTe NVIDIA GPU dialect ------------*- C++ -*-===// +//===- CuteNVGPUDialect.h - CuTe NVIDIA GPU dialect -------------*- C++ -*-===// -#ifndef CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_DIALECT_H -#define CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_DIALECT_H - -#include +#ifndef MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUDIALECT_H +#define MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUDIALECT_H #include "cute_ir/Dialect/Cute/IR/CuteDialect.h" + #include "mlir/Bytecode/BytecodeOpInterface.h" #include "mlir/IR/BuiltinAttributes.h" #include "mlir/IR/DialectImplementation.h" @@ -13,9 +12,11 @@ #include "mlir/Interfaces/InferTypeOpInterface.h" #include "mlir/Interfaces/SideEffectInterfaces.h" -#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h.inc" +#include -#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUEnums.h.inc" +#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h.inc" + +#include "mlir/dialects/cute_nvgpu/CuteNVGPUEnums.h.inc" namespace mlir::cutlass_compiler::cute_nvgpu { /// Parses `true` or `false`. @@ -23,15 +24,15 @@ FailureOr parseBool(AsmParser &parser); /// The TMA data format a non-executable TMA atom takes for an element type /// when none is given, if there is one. std::optional defaultTmaDataFormat(Type elementType); -} // namespace mlir::cutlass_compiler::cute_nvgpu +} // namespace mlir::cutlass_compiler::cute_nvgpu #define GET_ATTRDEF_CLASSES -#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUAttrs.h.inc" +#include "mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.h.inc" #define GET_TYPEDEF_CLASSES -#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUTypes.h.inc" +#include "mlir/dialects/cute_nvgpu/CuteNVGPUTypes.h.inc" #define GET_OP_CLASSES -#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUOps.h.inc" +#include "mlir/dialects/cute_nvgpu/CuteNVGPUOps.h.inc" -#endif // CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_DIALECT_H +#endif // MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUDIALECT_H diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.td b/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.td similarity index 86% rename from mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.td rename to mlir/dialects/cute_nvgpu/CuteNVGPUDialect.td index 88fe7fd4..5f6c6680 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.td +++ b/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.td @@ -1,7 +1,7 @@ //===- CuteNVGPUDialect.td - CuTe NVIDIA GPU dialect -------*- tablegen -*-===// -#ifndef CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_DIALECT_TD -#define CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_DIALECT_TD +#ifndef MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUDIALECT_TD +#define MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUDIALECT_TD include "mlir/IR/AttrTypeBase.td" include "mlir/IR/DialectBase.td" @@ -33,4 +33,4 @@ class CuteNVGPU_Type let mnemonic = typeMnemonic; } -#endif // CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_DIALECT_TD +#endif // MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUDIALECT_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUEnums.td b/mlir/dialects/cute_nvgpu/CuteNVGPUEnums.td similarity index 99% rename from mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUEnums.td rename to mlir/dialects/cute_nvgpu/CuteNVGPUEnums.td index 3a350562..a583a57e 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUEnums.td +++ b/mlir/dialects/cute_nvgpu/CuteNVGPUEnums.td @@ -7,10 +7,10 @@ // nvgpu attributes are in CuteNVGPUAttrs.td, so that -gen-enum-decls // over this file emits the nvgpu enums only. -#ifndef CUTE_IR_GENERATED_CUTENVGPU_ENUMS_TD -#define CUTE_IR_GENERATED_CUTENVGPU_ENUMS_TD +#ifndef MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUENUMS_TD +#define MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUENUMS_TD -include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.td" +include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.td" include "mlir/IR/EnumAttr.td" def CuteNVGPU_BinaryOpEnum : I32EnumAttr<"BinaryOp", "Binary operation for single-bit MMA operations", [ @@ -680,4 +680,4 @@ def CuteNVGPU_TmemLoadRedOpAttr : EnumAttr { ); } -#endif // CUTE_IR_GENERATED_CUTENVGPU_OPS_TD +#endif // MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUOPS_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUTypes.td b/mlir/dialects/cute_nvgpu/CuteNVGPUTypes.td similarity index 99% rename from mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUTypes.td rename to mlir/dialects/cute_nvgpu/CuteNVGPUTypes.td index 00fa734e..817af564 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUTypes.td +++ b/mlir/dialects/cute_nvgpu/CuteNVGPUTypes.td @@ -28,10 +28,10 @@ // //===----------------------------------------------------------------------===// -#ifndef CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_TYPES_TD -#define CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_TYPES_TD +#ifndef MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUTYPES_TD +#define MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUTYPES_TD -include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.td" +include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.td" class CuteNVGPU_CustomType : CuteNVGPU_Type { @@ -747,4 +747,4 @@ def CuteNVGPU_MmaAtomSM107UMMABlockScaledSparseType "128x128x64, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, sparse_metadata_format = tid, frag_kind = ss, vec_size = 32, ab_collector_op = (discard, discard)", 1, 1, 1, 1>; -#endif // CUTE_IR_DIALECT_CUTENVGPU_IR_CUTENVGPU_TYPES_TD +#endif // MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUTYPES_TD diff --git a/mlir/dialects/cute_ir/lib/CAPI/CuteNVGPUTypes.cpp b/mlir/dialects/cute_nvgpu/capi.cc similarity index 74% rename from mlir/dialects/cute_ir/lib/CAPI/CuteNVGPUTypes.cpp rename to mlir/dialects/cute_nvgpu/capi.cc index 181419da..73d55bc3 100644 --- a/mlir/dialects/cute_ir/lib/CAPI/CuteNVGPUTypes.cpp +++ b/mlir/dialects/cute_nvgpu/capi.cc @@ -1,16 +1,25 @@ -// C API for the CuTe NVIDIA GPU types, generated from the dialect's .td files. -#include "cute_ir-c/Dialect/CuteNVGPUTypes.h" +//===- capi.cc - C API for the cute_nvgpu dialect -------------------------===// + +#include "capi.h" + +#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h" -#include "cute_ir/Dialect/CuteNVGPU/IR/CuteNVGPUDialect.h" #include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" + +// Registration + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION( + CuteNVGPU, cute_nvgpu, mlir::cutlass_compiler::cute_nvgpu::CuteNVGPUDialect) + +// Attributes namespace cute_nvgpu = mlir::cutlass_compiler::cute_nvgpu; -namespace { // Unwraps `handle` into `out`: false when it is of another context or not a // T, or when it is null and not optional. template -bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { +static bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { auto value = unwrap(handle); if (!value) return optional; @@ -19,7 +28,1177 @@ bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { out = llvm::dyn_cast(value); return static_cast(out); } -} // namespace + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyG2S(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldBulkCopyG2S( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldBulkCopyG2SAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2G(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldBulkCopyS2G( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldBulkCopyS2GAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2S(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldBulkCopyS2S( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldBulkCopyS2SAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldDsmemStore(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldDsmemStore( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldDsmemStoreAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldLoadGlobal(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldLoadGlobal( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldLoadGlobalAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoad( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExec2DGather4TmaLoadAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExec2DGather4TmaLoad(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldNonExec2DGather4TmaLoadAttr::get( + ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGetValue( + MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStore( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExec2DScatter4TmaStoreAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExec2DScatter4TmaStore(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldNonExec2DScatter4TmaStoreAttr::get( + ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGetValue( + MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoad( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExecIm2ColTmaLoadAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExecIm2ColTmaLoad(value); + if (!valueValue) + return {nullptr}; + return wrap( + cute_nvgpu::AtomCopyFieldNonExecIm2ColTmaLoadAttr::get(ctx, *valueValue)); +} +uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaStore( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExecIm2ColTmaStoreAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExecIm2ColTmaStore(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldNonExecIm2ColTmaStoreAttr::get( + ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGetValue( + MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaLoad( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExecTiledTmaLoadAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExecTiledTmaLoad(value); + if (!valueValue) + return {nullptr}; + return wrap( + cute_nvgpu::AtomCopyFieldNonExecTiledTmaLoadAttr::get(ctx, *valueValue)); +} +uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaReduce( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExecTiledTmaReduceAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExecTiledTmaReduce(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldNonExecTiledTmaReduceAttr::get( + ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGetValue( + MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaStore( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExecTiledTmaStoreAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExecTiledTmaStore(value); + if (!valueValue) + return {nullptr}; + return wrap( + cute_nvgpu::AtomCopyFieldNonExecTiledTmaStoreAttr::get(ctx, *valueValue)); +} +uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldStoreGlobal(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldStoreGlobal( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldStoreGlobalAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldTmaLoad(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldTmaLoad( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldTmaLoadAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldTmaReduce(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldTmaReduce( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldTmaReduceAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldTmaStore(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldTmaStore( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldTmaStoreAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM100(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomMmaFieldSM100AttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomMmaFieldSM100(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomMmaFieldSM100Attr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomMmaFieldSM100AttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaled(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomMmaFieldSM100BlockScaled(value); + if (!valueValue) + return {nullptr}; + return wrap( + cute_nvgpu::AtomMmaFieldSM100BlockScaledAttr::get(ctx, *valueValue)); +} +uint32_t +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaledSparse( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomMmaFieldSM100BlockScaledSparseAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomMmaFieldSM100BlockScaledSparse(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomMmaFieldSM100BlockScaledSparseAttr::get( + ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGetValue( + MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM100Sparse(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomMmaFieldSM100Sparse( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomMmaFieldSM100SparseAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM120BlockScaled(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomMmaFieldSM120BlockScaled(value); + if (!valueValue) + return {nullptr}; + return wrap( + cute_nvgpu::AtomMmaFieldSM120BlockScaledAttr::get(ctx, *valueValue)); +} +uint32_t +mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM80Sparse(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomMmaFieldSM80Sparse( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomMmaFieldSM80SparseAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM90(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomMmaFieldSM90AttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomMmaFieldSM90(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomMmaFieldSM90Attr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomMmaFieldSM90AttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUBinaryOp(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUBinaryOpAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeBinaryOp(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::BinaryOpAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUBinaryOpAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUCopyS2TBroadcast(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUCopyS2TBroadcastAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeCopyS2TBroadcast(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyS2TBroadcastAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUCopyS2TBroadcastAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUGatherScatterTmaLoad(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUGatherScatterTmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeGatherScatterTmaLoad( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::GatherScatterTmaLoadAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUGatherScatterTmaLoadAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUIm2ColTmaLoad(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUIm2ColTmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeIm2ColTmaLoad(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::Im2ColTmaLoadAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUIm2ColTmaLoadAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPULdReduceAccPrecisionKind(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPULdReduceAccPrecisionKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLdReduceAccPrecisionKind( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::LdReduceAccPrecisionKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPULdReduceAccPrecisionKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPULdsmSzPattern(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPULdsmSzPatternAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLdsmSzPattern(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::LdsmSzPatternAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPULdsmSzPatternAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPULoadCacheMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPULoadCacheModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLoadCacheMode(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::LoadCacheModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPULoadCacheModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUMMAIntOverflow(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUMMAIntOverflowAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMMAIntOverflow(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::MMAIntOverflowAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUMMAIntOverflowAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUMajorMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUMajorModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::MajorModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUMajorModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUMmaCollectorOp(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUMmaCollectorOpAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::MmaCollectorOpAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUMmaCollectorOpAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUMmaFragKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUMmaFragKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::MmaFragKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUMmaFragKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUNotImplementedFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUNotImplementedFrgAttrGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute_nvgpu::NotImplementedFrgAttr::get(ctx)); +} + +bool mlirAttributeIsACuteNVGPUReductionKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUReductionKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeReductionKind(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::ReductionKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUReductionKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPURmemFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPURmemFrgAttrGet(MlirContext context, + MlirType valueType, uint32_t operand, + bool deriveElemType) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valueTypeValue; + if (!unwrapAs(ctx, valueType, false, valueTypeValue)) + return {nullptr}; + auto operandValue = + ::mlir::cutlass_compiler::cute::symbolizeMmaOperand(operand); + if (!operandValue) + return {nullptr}; + return wrap(cute_nvgpu::RmemFrgAttr::get(ctx, valueTypeValue, *operandValue, + deriveElemType)); +} +MlirType mlirCuteNVGPURmemFrgAttrGetValueType(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getValueType())); +} +uint32_t mlirCuteNVGPURmemFrgAttrGetOperand(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getOperand()); +} +bool mlirCuteNVGPURmemFrgAttrGetDeriveElemType(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getDeriveElemType(); +} + +bool mlirAttributeIsACuteNVGPUSM100CircularSmemFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM100CircularSmemFrgAttrGet(MlirContext context, + uint32_t majorMode) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto majorModeValue = + ::mlir::cutlass_compiler::cute::symbolizeMajorMode(majorMode); + if (!majorModeValue) + return {nullptr}; + return wrap(cute_nvgpu::SM100CircularSmemFrgAttr::get(ctx, *majorModeValue)); +} +uint32_t mlirCuteNVGPUSM100CircularSmemFrgAttrGetMajorMode(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getMajorMode()); +} + +bool mlirAttributeIsACuteNVGPUSM100SmemFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM100SmemFrgAttrGet(MlirContext context, + uint32_t majorMode) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto majorModeValue = + ::mlir::cutlass_compiler::cute::symbolizeMajorMode(majorMode); + if (!majorModeValue) + return {nullptr}; + return wrap(cute_nvgpu::SM100SmemFrgAttr::get(ctx, *majorModeValue)); +} +uint32_t mlirCuteNVGPUSM100SmemFrgAttrGetMajorMode(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getMajorMode()); +} + +bool mlirAttributeIsACuteNVGPUSM100TmemEFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM100TmemEFrgAttrGet(MlirContext context, + MlirType aType, + MlirType eType) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type eTypeValue; + if (!unwrapAs(ctx, eType, false, eTypeValue)) + return {nullptr}; + return wrap(cute_nvgpu::SM100TmemEFrgAttr::get(ctx, aTypeValue, eTypeValue)); +} +MlirType mlirCuteNVGPUSM100TmemEFrgAttrGetAType(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getAType())); +} +MlirType mlirCuteNVGPUSM100TmemEFrgAttrGetEType(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getEType())); +} + +bool mlirAttributeIsACuteNVGPUSM100TmemFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM100TmemFrgAttrGet(MlirContext context, + MlirType dataType, + MlirType storageType, + int ctaGroup, + uint32_t tmemAllocMode) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type dataTypeValue; + if (!unwrapAs(ctx, dataType, false, dataTypeValue)) + return {nullptr}; + mlir::Type storageTypeValue; + if (!unwrapAs(ctx, storageType, false, storageTypeValue)) + return {nullptr}; + auto tmemAllocModeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemAllocMode( + tmemAllocMode); + if (!tmemAllocModeValue) + return {nullptr}; + return wrap(cute_nvgpu::SM100TmemFrgAttr::get( + ctx, dataTypeValue, storageTypeValue, ctaGroup, *tmemAllocModeValue)); +} +MlirType mlirCuteNVGPUSM100TmemFrgAttrGetDataType(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getDataType())); +} +MlirType mlirCuteNVGPUSM100TmemFrgAttrGetStorageType(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getStorageType())); +} +int mlirCuteNVGPUSM100TmemFrgAttrGetCtaGroup(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getCtaGroup(); +} +uint32_t mlirCuteNVGPUSM100TmemFrgAttrGetTmemAllocMode(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getTmemAllocMode()); +} + +bool mlirAttributeIsACuteNVGPUSM100TmemSfFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM100TmemSfFrgAttrGet(MlirContext context, + MlirType sfType, int sfVecSize, + int ctaGroup, bool isSfa, + uint32_t tmemAllocMode) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type sfTypeValue; + if (!unwrapAs(ctx, sfType, false, sfTypeValue)) + return {nullptr}; + auto tmemAllocModeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemAllocMode( + tmemAllocMode); + if (!tmemAllocModeValue) + return {nullptr}; + return wrap(cute_nvgpu::SM100TmemSfFrgAttr::get( + ctx, sfTypeValue, sfVecSize, ctaGroup, isSfa, *tmemAllocModeValue)); +} +MlirType mlirCuteNVGPUSM100TmemSfFrgAttrGetSfType(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getSfType())); +} +int mlirCuteNVGPUSM100TmemSfFrgAttrGetSfVecSize(MlirAttribute attr) { + return llvm::cast(unwrap(attr)) + .getSfVecSize(); +} +int mlirCuteNVGPUSM100TmemSfFrgAttrGetCtaGroup(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getCtaGroup(); +} +bool mlirCuteNVGPUSM100TmemSfFrgAttrGetIsSfa(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getIsSfa(); +} +uint32_t mlirCuteNVGPUSM100TmemSfFrgAttrGetTmemAllocMode(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getTmemAllocMode()); +} + +bool mlirAttributeIsACuteNVGPUSM107SmemFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM107SmemFrgAttrGet(MlirContext context, + uint32_t majorMode) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto majorModeValue = + ::mlir::cutlass_compiler::cute::symbolizeMajorMode(majorMode); + if (!majorModeValue) + return {nullptr}; + return wrap(cute_nvgpu::SM107SmemFrgAttr::get(ctx, *majorModeValue)); +} +uint32_t mlirCuteNVGPUSM107SmemFrgAttrGetMajorMode(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getMajorMode()); +} + +bool mlirAttributeIsACuteNVGPUSM90SmemFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM90SmemFrgAttrGet(MlirContext context, + uint32_t majorMode) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto majorModeValue = + ::mlir::cutlass_compiler::cute::symbolizeMajorMode(majorMode); + if (!majorModeValue) + return {nullptr}; + return wrap(cute_nvgpu::SM90SmemFrgAttr::get(ctx, *majorModeValue)); +} +uint32_t mlirCuteNVGPUSM90SmemFrgAttrGetMajorMode(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getMajorMode()); +} + +bool mlirAttributeIsACuteNVGPUTiledTmaLoad(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUTiledTmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTiledTmaLoad(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::TiledTmaLoadAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUTiledTmaLoadAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUTmaDataFormat(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUTmaDataFormatAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::TmaDataFormatAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUTmaDataFormatAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUTmaLoadMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUTmaLoadModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaLoadMode(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::TmaLoadModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUTmaLoadModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUTmaStoreMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUTmaStoreModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaStoreMode(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::TmaStoreModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUTmaStoreModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUTmemAllocMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUTmemAllocModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemAllocMode(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::TmemAllocModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUTmemAllocModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUTmemLoadRedOp(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUTmemLoadRedOpAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemLoadRedOp(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::TmemLoadRedOpAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUTmemLoadRedOpAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +// Types + +bool mlirTypeIsACuteNVGPUType(MlirType type) { + mlir::Type cppType = unwrap(type); + if (!cppType) + return false; + if (auto opaque = mlir::dyn_cast(cppType)) + return opaque.getDialectNamespace() == "cute_nvgpu"; + return cppType.getDialect().getNamespace() == "cute_nvgpu"; +} bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyG2S(MlirType type) { return llvm::isa_and_nonnull( @@ -1332,7 +2511,7 @@ bool mlirTypeIsACuteNVGPUCopyAtomSM100TmemLoad(MlirType type) { } MlirType mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGet(MlirContext context, MlirType valType, int numDp, - int numBit, unsigned numRep, + int numBit, uint32_t numRep, bool pack16b) { auto *ctx = unwrap(context); if (!ctx) @@ -1359,7 +2538,7 @@ int mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumBit(MlirType type) { return llvm::cast(unwrap(type)) .getNumBit(); } -unsigned mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumRep(MlirType type) { +uint32_t mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumRep(MlirType type) { return llvm::cast(unwrap(type)) .getNumRep(); } @@ -1375,7 +2554,7 @@ bool mlirTypeIsACuteNVGPUCopyAtomSM100TmemStore(MlirType type) { } MlirType mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGet(MlirContext context, MlirType valType, int numDp, - int numBit, unsigned numRep, + int numBit, uint32_t numRep, bool expand16b) { auto *ctx = unwrap(context); if (!ctx) @@ -1402,7 +2581,7 @@ int mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumBit(MlirType type) { return llvm::cast(unwrap(type)) .getNumBit(); } -unsigned mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumRep(MlirType type) { +uint32_t mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumRep(MlirType type) { return llvm::cast(unwrap(type)) .getNumRep(); } @@ -1418,7 +2597,7 @@ bool mlirTypeIsACuteNVGPUCopyAtomSM107TmemLoadSPCompress(MlirType type) { } MlirType mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGet( MlirContext context, MlirType valType, int numDp, int numBit, - unsigned numRep, uint32_t redOp, bool red, bool nan) { + uint32_t numRep, uint32_t redOp, bool red, bool nan) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; @@ -1452,7 +2631,7 @@ int mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumBit(MlirType type) { unwrap(type)) .getNumBit(); } -unsigned +uint32_t mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumRep(MlirType type) { return llvm::cast( unwrap(type)) @@ -1481,7 +2660,7 @@ bool mlirTypeIsACuteNVGPUCopyAtomSM10xTmemLoadRed(MlirType type) { } MlirType mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet( MlirContext context, MlirType valType, int numDp, int numBit, - unsigned numRep, uint32_t redOp, bool nan, MlirAttribute halfSplitOff) { + uint32_t numRep, uint32_t redOp, bool nan, MlirAttribute halfSplitOff) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; @@ -1514,7 +2693,7 @@ int mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumBit(MlirType type) { return llvm::cast(unwrap(type)) .getNumBit(); } -unsigned mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumRep(MlirType type) { +uint32_t mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumRep(MlirType type) { return llvm::cast(unwrap(type)) .getNumRep(); } diff --git a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPUTypes.h b/mlir/dialects/cute_nvgpu/capi.h similarity index 68% rename from mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPUTypes.h rename to mlir/dialects/cute_nvgpu/capi.h index f4060a1a..f1e4e999 100644 --- a/mlir/dialects/cute_ir/include/cute_ir-c/Dialect/CuteNVGPUTypes.h +++ b/mlir/dialects/cute_nvgpu/capi.h @@ -1,6 +1,7 @@ -// C API for the CuTe NVIDIA GPU types, generated from the dialect's .td files. -#ifndef CUTE_IR_C_DIALECT_CUTENVGPU_TYPES_H -#define CUTE_IR_C_DIALECT_CUTENVGPU_TYPES_H +//===- capi.h - C API for the cute_nvgpu dialect ------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_CUTE_NVGPU_CAPI_H +#define MLIR_DIALECTS_CUTE_NVGPU_CAPI_H #include "mlir-c/IR.h" @@ -8,6 +9,480 @@ extern "C" { #endif +// Registration + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(CuteNVGPU, cute_nvgpu); + +// Attributes + +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles, and std::optional enums a +// negative value. Enums are their integer values (the dialect's *Enums.td). +// Getters require a value accepted by the matching IsA. + +// `#cute_nvgpu.atom_copy_field_bulkg2s`: Fields stored in Bulk Load Copy Atom +// type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyG2S(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_bulks2g`: Fields stored in Bulk Store Copy Atom +// type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2G(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_bulks2s`: Fields stored in Bulk CTA to Cluster +// Copy Atom type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2S(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_dsmem_store`: Fields stored in +// CopyAtomDsmemStoreType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldDsmemStore(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_g2r`: Fields stored in CopyAtomG2RType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldLoadGlobal(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_2d_gather4_tma_load`: Fields stored in +// CopyAtomNonExec2DGather4TmaLoadType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoad( + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGetValue( + MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_2d_scatter4_tma_store`: Fields stored +// in CopyAtomNonExec2DScatter4TmaStoreType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStore( + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGetValue( + MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_im2col_tma_load`: Fields stored in +// CopyAtomNonExecIm2ColTmaLoadType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoad(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_im2col_tma_store`: Fields stored in +// CopyAtomNonExecIm2ColTmaStoreType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaStore(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_tma_load`: Fields stored in +// CopyAtomNonExecTiledTmaLoadType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaLoad(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_tma_reduce`: Fields stored in +// CopyAtomNonExecTiledTmaReduceType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaReduce(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_tma_store`: Fields stored in +// CopyAtomNonExecTiledTmaStoreType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaStore(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_r2g`: Fields stored in CopyAtomR2GType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldStoreGlobal(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_tmaload`: Fields stored in Tma Load Copy Atom +// type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldTmaLoad(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_tmareduce`: Fields stored in Tma Reduce Copy +// Atom type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldTmaReduce(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_tmastore`: Fields stored in Tma Store Copy Atom +// type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldTmaStore(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm100`: Fields stored in MMA Atom types for +// SM100. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM100(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM100AttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM100AttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm100_block_scaled`: Fields stored in MMA Atom +// types for SM100 (block scaled). +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaled(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm100_block_scaled_sparse`: Fields stored in MMA +// Atom types for SM100 (block scaled). +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaledSparse(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm100_sparse`: Fields stored in sparse MMA Atom +// types for SM100. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM100Sparse(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm120_block_scaled`: Fields stored in MMA Atom +// types for SM120 (block scaled). +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM120BlockScaled(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm80_sparse`: Fields stored in MMA Atom types for +// SM80. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM80Sparse(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm90`: Fields stored in MMA Atom types for SM90. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM90(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM90AttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM90AttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.bin_op`: Binary operation for single-bit MMA operations. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVGPUBinaryOp(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUBinaryOpAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUBinaryOpAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.copy_s2t_broadcast_mode`: Broadcast modes for the different +// utccp instructions. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUCopyS2TBroadcast(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyS2TBroadcastAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyS2TBroadcastAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.gather_scatter_tma_load`: The various kinds of TMA loads in +// gather/scatter mode. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUGatherScatterTmaLoad(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUGatherScatterTmaLoadAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUGatherScatterTmaLoadAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.im2col_tma_load`: The various kinds of TMA loads in im2col mode. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUIm2ColTmaLoad(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUIm2ColTmaLoadAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUIm2ColTmaLoadAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.ld_reduce_acc_precision_kind`: multimem ld_reduce accumulation +// precision kind. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPULdReduceAccPrecisionKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPULdReduceAccPrecisionKindAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPULdReduceAccPrecisionKindAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.ldsm_sz_pattern`: LDSM's sz pattern, describing the bit size. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPULdsmSzPattern(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPULdsmSzPatternAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPULdsmSzPatternAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.load_cache_mode`: Cache modes for the load instructions. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPULoadCacheMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPULoadCacheModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPULoadCacheModeAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.mma_int_overflow`: MMA overflow options. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUMMAIntOverflow(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMMAIntOverflowAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMMAIntOverflowAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.major`: Major mode for MMA operations. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVGPUMajorMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMajorModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMajorModeAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.mma_collector_op`: Enums for the mma collector op. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUMmaCollectorOp(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaCollectorOpAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaCollectorOpAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.mma_frag_kind`: Enums for the mma frag type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUMmaFragKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaFragKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaFragKindAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.not_implemented_frg`: Fragment of an atom that has none. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUNotImplementedFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUNotImplementedFrgAttrGet(MlirContext context); + +// `#cute_nvgpu.tma_reduce_kind`: Op for the TMASTORE instruction. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUReductionKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUReductionKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUReductionKindAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.rmem_frg`: Register-memory fragment of an MMA operand. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVGPURmemFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPURmemFrgAttrGet(MlirContext context, MlirType valueType, + uint32_t operand, bool deriveElemType); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPURmemFrgAttrGetValueType(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPURmemFrgAttrGetOperand(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPURmemFrgAttrGetDeriveElemType(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm100.circular_smem_frg`: SM100 circular shared-memory +// fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM100CircularSmemFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUSM100CircularSmemFrgAttrGet( + MlirContext context, uint32_t majorMode); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUSM100CircularSmemFrgAttrGetMajorMode(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm100.smem_frg`: SM100 shared-memory fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM100SmemFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUSM100SmemFrgAttrGet(MlirContext context, uint32_t majorMode); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUSM100SmemFrgAttrGetMajorMode(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm100.tmem_e_frg`: SM100 tensor-memory sparse-metadata +// fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM100TmemEFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUSM100TmemEFrgAttrGet( + MlirContext context, MlirType aType, MlirType eType); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUSM100TmemEFrgAttrGetAType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUSM100TmemEFrgAttrGetEType(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm100.tmem_frg`: SM100 tensor-memory fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM100TmemFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUSM100TmemFrgAttrGet( + MlirContext context, MlirType dataType, MlirType storageType, int ctaGroup, + uint32_t tmemAllocMode); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUSM100TmemFrgAttrGetDataType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUSM100TmemFrgAttrGetStorageType(MlirAttribute attr); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUSM100TmemFrgAttrGetCtaGroup(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUSM100TmemFrgAttrGetTmemAllocMode(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm100.tmem_sf_frg`: SM100 tensor-memory scale-factor +// fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM100TmemSfFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUSM100TmemSfFrgAttrGet( + MlirContext context, MlirType sfType, int sfVecSize, int ctaGroup, + bool isSfa, uint32_t tmemAllocMode); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUSM100TmemSfFrgAttrGetSfType(MlirAttribute attr); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUSM100TmemSfFrgAttrGetSfVecSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUSM100TmemSfFrgAttrGetCtaGroup(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUSM100TmemSfFrgAttrGetIsSfa(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUSM100TmemSfFrgAttrGetTmemAllocMode(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm107.smem_frg`: SM107 shared-memory fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM107SmemFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUSM107SmemFrgAttrGet(MlirContext context, uint32_t majorMode); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUSM107SmemFrgAttrGetMajorMode(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm90.smem_frg`: SM90 shared-memory fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM90SmemFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUSM90SmemFrgAttrGet(MlirContext context, uint32_t majorMode); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUSM90SmemFrgAttrGetMajorMode(MlirAttribute attr); + +// `#cute_nvgpu.tiled_tma_load`: The various kinds of TMA loads in tiled mode. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUTiledTmaLoad(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTiledTmaLoadAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUTiledTmaLoadAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.tma_data_format`: Bits 74-71 of the TMA descriptor. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUTmaDataFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTmaDataFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUTmaDataFormatAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.tma_load_mode`: Modes for the TMA load instruction. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUTmaLoadMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTmaLoadModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUTmaLoadModeAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.tma_store_mode`: Modes for the TMA store instruction. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUTmaStoreMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTmaStoreModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUTmaStoreModeAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.tmem_alloc_mode`: Modes for the Tmem allocation. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUTmemAllocMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTmemAllocModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUTmemAllocModeAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.tmem_load_red_op`: Reduce operation for the Tmem load +// instruction. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUTmemLoadRedOp(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTmemLoadRedOpAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUTmemLoadRedOpAttrGetValue(MlirAttribute attr); + +// Types + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUType(MlirType type); + // Constructors return null for invalid parameters or handles of another // context; optional parameters take null handles, and std::optional enums a // negative value. Enums are their integer values (the dialect's *Enums.td). @@ -418,14 +893,14 @@ MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomSM100TmemLoad(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGet( MlirContext context, MlirType valType, int numDp, int numBit, - unsigned numRep, bool pack16b); + uint32_t numRep, bool pack16b); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetValType(MlirType type); MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumDp(MlirType type); MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumBit(MlirType type); -MLIR_CAPI_EXPORTED unsigned +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumRep(MlirType type); MLIR_CAPI_EXPORTED bool mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetPack16b(MlirType type); @@ -435,14 +910,14 @@ MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomSM100TmemStore(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGet( MlirContext context, MlirType valType, int numDp, int numBit, - unsigned numRep, bool expand16b); + uint32_t numRep, bool expand16b); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetValType(MlirType type); MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumDp(MlirType type); MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumBit(MlirType type); -MLIR_CAPI_EXPORTED unsigned +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumRep(MlirType type); MLIR_CAPI_EXPORTED bool mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetExpand16b(MlirType type); @@ -453,14 +928,14 @@ MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomSM107TmemLoadSPCompress(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGet( MlirContext context, MlirType valType, int numDp, int numBit, - unsigned numRep, uint32_t redOp, bool red, bool nan); + uint32_t numRep, uint32_t redOp, bool red, bool nan); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetValType(MlirType type); MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumDp(MlirType type); MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumBit(MlirType type); -MLIR_CAPI_EXPORTED unsigned +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumRep(MlirType type); MLIR_CAPI_EXPORTED uint32_t mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRedOp(MlirType type); @@ -474,14 +949,14 @@ MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomSM10xTmemLoadRed(MlirType type); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet( MlirContext context, MlirType valType, int numDp, int numBit, - unsigned numRep, uint32_t redOp, bool nan, MlirAttribute halfSplitOff); + uint32_t numRep, uint32_t redOp, bool nan, MlirAttribute halfSplitOff); MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetValType(MlirType type); MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumDp(MlirType type); MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumBit(MlirType type); -MLIR_CAPI_EXPORTED unsigned +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumRep(MlirType type); MLIR_CAPI_EXPORTED uint32_t mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetRedOp(MlirType type); @@ -1003,4 +1478,4 @@ mlirCuteNVGPUWorkIdResponseTypeGet(MlirContext context); } #endif -#endif // CUTE_IR_C_DIALECT_CUTENVGPU_TYPES_H +#endif // MLIR_DIALECTS_CUTE_NVGPU_CAPI_H diff --git a/mlir/dialects/cute_ir/cute_nvgpu.zig b/mlir/dialects/cute_nvgpu/cute_nvgpu.zig similarity index 76% rename from mlir/dialects/cute_ir/cute_nvgpu.zig rename to mlir/dialects/cute_nvgpu/cute_nvgpu.zig index 652ff9e1..4c039ecf 100644 --- a/mlir/dialects/cute_ir/cute_nvgpu.zig +++ b/mlir/dialects/cute_nvgpu/cute_nvgpu.zig @@ -4,9 +4,11 @@ const c = @import("c"); const mlir = @import("mlir"); const stdx = @import("stdx"); -const cute = @import("cute_ir.zig"); +const cute_ir = @import("mlir/dialects/cute_ir"); -pub const dialect_namespace = "cute_nvgpu"; +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("cute_nvgpu"); +} pub const NVGPUType = opaque { const M = mlir.Methods(NVGPUType, c.MlirType); @@ -14,22 +16,10 @@ pub const NVGPUType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUType; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; - - pub fn format(self: *const NVGPUType, writer: *std.Io.Writer) std.Io.Writer.Error!void { - const t: *const mlir.Type = @ptrCast(self); - return t.format(writer); - } }; -/// Parses `!cute_nvgpu.`, such as `` for -/// `CopyAtomSIMTSyncCopyType`, and returns it as a `T`. -pub fn typeFromPayload(comptime T: type, ctx: *mlir.Context, payload: []const u8) mlir.Error!*const T { - var buffer: [1024]u8 = undefined; - const text = std.fmt.bufPrint(&buffer, "!" ++ dialect_namespace ++ "." ++ T.mnemonic ++ "{s}", .{payload}) catch return error.InvalidMlir; - return (try mlir.Type.parse(ctx, text)).isA(T) orelse error.InvalidMlir; -} - // Enums, generated from the dialect's .td files; values are the compiler's. /// NVGpu arch number like sm_103. @@ -412,11 +402,9 @@ pub const CopyAtomBulkCopyG2SType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomBulkCopyG2S; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.bulk_copy_g2s"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -429,13 +417,13 @@ pub const CopyAtomBulkCopyG2SType = opaque { const result = c.mlirCuteNVGPUCopyAtomBulkCopyG2STypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.mcast); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetValType(self.ptr()).ptr.?); } - pub fn getCopyBits(self: *const Self) c_int { + pub fn copyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetCopyBits(self.ptr()); } - pub fn getMcast(self: *const Self) bool { + pub fn mcast(self: *const Self) bool { return c.mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetMcast(self.ptr()); } }; @@ -447,11 +435,9 @@ pub const CopyAtomBulkCopyS2GType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2G; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.bulk_copy_s2g"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -464,13 +450,13 @@ pub const CopyAtomBulkCopyS2GType = opaque { const result = c.mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.mask); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetValType(self.ptr()).ptr.?); } - pub fn getCopyBits(self: *const Self) c_int { + pub fn copyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetCopyBits(self.ptr()); } - pub fn getMask(self: *const Self) bool { + pub fn mask(self: *const Self) bool { return c.mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetMask(self.ptr()); } }; @@ -482,11 +468,9 @@ pub const CopyAtomBulkCopyS2SType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2S; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.bulk_copy_s2s"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -498,10 +482,10 @@ pub const CopyAtomBulkCopyS2SType = opaque { const result = c.mlirCuteNVGPUCopyAtomBulkCopyS2STypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetValType(self.ptr()).ptr.?); } - pub fn getCopyBits(self: *const Self) c_int { + pub fn copyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetCopyBits(self.ptr()); } }; @@ -513,11 +497,9 @@ pub const CopyAtomDsmemStoreType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomDsmemStore; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.dsmem_store"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -529,10 +511,10 @@ pub const CopyAtomDsmemStoreType = opaque { const result = c.mlirCuteNVGPUCopyAtomDsmemStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomDsmemStoreTypeGetValType(self.ptr()).ptr.?); } - pub fn getCopyBits(self: *const Self) c_int { + pub fn copyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomDsmemStoreTypeGetCopyBits(self.ptr()); } }; @@ -544,11 +526,9 @@ pub const CopyAtomG2RType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomG2R; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.g2r"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -567,31 +547,31 @@ pub const CopyAtomG2RType = opaque { const result = c.mlirCuteNVGPUCopyAtomG2RTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.l2PrefetchSize), @backingInt(args.l1CacheEvictPriority), @backingInt(args.loadCacheMode), @backingInt(args.sharedSpace), args.invariant); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetValType(self.ptr()).ptr.?); } - pub fn getCopyBits(self: *const Self) c_int { + pub fn copyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomG2RTypeGetCopyBits(self.ptr()); } - pub fn getMemOrder(self: *const Self) MemOrderKind { + pub fn memOrder(self: *const Self) MemOrderKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetMemOrder(self.ptr()))); } - pub fn getMemScope(self: *const Self) MemScopeKind { + pub fn memScope(self: *const Self) MemScopeKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetMemScope(self.ptr()))); } - pub fn getL2PrefetchSize(self: *const Self) L2PrefetchSize { + pub fn l2PrefetchSize(self: *const Self) L2PrefetchSize { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetL2PrefetchSize(self.ptr()))); } - pub fn getL1CacheEvictPriority(self: *const Self) CacheEvictionPriority { + pub fn l1CacheEvictPriority(self: *const Self) CacheEvictionPriority { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetL1CacheEvictPriority(self.ptr()))); } - pub fn getLoadCacheMode(self: *const Self) LoadCacheMode { + pub fn loadCacheMode(self: *const Self) LoadCacheMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetLoadCacheMode(self.ptr()))); } - pub fn getSharedSpace(self: *const Self) SharedSpace { + pub fn sharedSpace(self: *const Self) SharedSpace { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetSharedSpace(self.ptr()))); } - pub fn getInvariant(self: *const Self) bool { + pub fn invariant(self: *const Self) bool { return c.mlirCuteNVGPUCopyAtomG2RTypeGetInvariant(self.ptr()); } }; @@ -603,11 +583,9 @@ pub const CopyAtomIm2ColTmaLoadType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaLoad; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.im2col_tma_load"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -623,22 +601,22 @@ pub const CopyAtomIm2ColTmaLoadType = opaque { const result = c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.numCta, args.gStride.ptr(), args.mcast, if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetValType(self.ptr()).ptr.?); } - pub fn getCopyBits(self: *const Self) c_int { + pub fn copyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetCopyBits(self.ptr()); } - pub fn getNumCta(self: *const Self) c_int { + pub fn numCta(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetNumCta(self.ptr()); } - pub fn getGStride(self: *const Self) *const mlir.Type { + pub fn gStride(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetGStride(self.ptr()).ptr.?); } - pub fn getMcast(self: *const Self) bool { + pub fn mcast(self: *const Self) bool { return c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetMcast(self.ptr()); } - pub fn getTmaGbasis(self: *const Self) ?*const mlir.Type { + pub fn tmaGbasis(self: *const Self) ?*const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetTmaGbasis(self.ptr()).ptr); } }; @@ -650,11 +628,9 @@ pub const CopyAtomIm2ColTmaStoreType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaStore; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.im2col_tma_store"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -668,16 +644,16 @@ pub const CopyAtomIm2ColTmaStoreType = opaque { const result = c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetValType(self.ptr()).ptr.?); } - pub fn getCopyBits(self: *const Self) c_int { + pub fn copyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetCopyBits(self.ptr()); } - pub fn getGStride(self: *const Self) *const mlir.Type { + pub fn gStride(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetGStride(self.ptr()).ptr.?); } - pub fn getTmaGbasis(self: *const Self) ?*const mlir.Type { + pub fn tmaGbasis(self: *const Self) ?*const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetTmaGbasis(self.ptr()).ptr); } }; @@ -689,11 +665,9 @@ pub const CopyAtomLdsmType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomLdsm; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.ldsm"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -708,19 +682,19 @@ pub const CopyAtomLdsmType = opaque { const result = c.mlirCuteNVGPUCopyAtomLdsmTypeGet(ctx.ptr(), args.valType.ptr(), args.mode.ptr(), @backingInt(args.szPattern), args.numMatrices, args.transpose); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomLdsmTypeGetValType(self.ptr()).ptr.?); } - pub fn getMode(self: *const Self) *const mlir.Attribute { + pub fn mode(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUCopyAtomLdsmTypeGetMode(self.ptr()).ptr.?); } - pub fn getSzPattern(self: *const Self) LdsmSzPattern { + pub fn szPattern(self: *const Self) LdsmSzPattern { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomLdsmTypeGetSzPattern(self.ptr()))); } - pub fn getNumMatrices(self: *const Self) c_int { + pub fn numMatrices(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomLdsmTypeGetNumMatrices(self.ptr()); } - pub fn getTranspose(self: *const Self) bool { + pub fn transpose(self: *const Self) bool { return c.mlirCuteNVGPUCopyAtomLdsmTypeGetTranspose(self.ptr()); } }; @@ -732,11 +706,9 @@ pub const CopyAtomNonExec2DGather4TmaLoadType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExec2DGather4TmaLoad; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.non_exec_2d_gather4_tma_load"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -751,19 +723,19 @@ pub const CopyAtomNonExec2DGather4TmaLoadType = opaque { const result = c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getKind(self: *const Self) GatherScatterTmaLoad { + pub fn kind(self: *const Self) GatherScatterTmaLoad { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetKind(self.ptr()))); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetValType(self.ptr()).ptr.?); } - pub fn getCopyBits(self: *const Self) c_int { + pub fn copyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetCopyBits(self.ptr()); } - pub fn getTmaGbasis(self: *const Self) *const mlir.Type { + pub fn tmaGbasis(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaGbasis(self.ptr()).ptr.?); } - pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { + pub fn tmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaFormat(self.ptr()); return if (value < 0) null else @fromBackingInt(@intCast(value)); } @@ -776,11 +748,9 @@ pub const CopyAtomNonExec2DScatter4TmaStoreType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExec2DScatter4TmaStore; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.non_exec_2d_scatter4_tma_store"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -794,16 +764,16 @@ pub const CopyAtomNonExec2DScatter4TmaStoreType = opaque { const result = c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetValType(self.ptr()).ptr.?); } - pub fn getCopyBits(self: *const Self) c_int { + pub fn copyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetCopyBits(self.ptr()); } - pub fn getTmaGbasis(self: *const Self) *const mlir.Type { + pub fn tmaGbasis(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaGbasis(self.ptr()).ptr.?); } - pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { + pub fn tmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaFormat(self.ptr()); return if (value < 0) null else @fromBackingInt(@intCast(value)); } @@ -816,11 +786,9 @@ pub const CopyAtomNonExecIm2ColTmaLoadType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaLoad; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.non_exec_im2col_tma_load"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -835,19 +803,19 @@ pub const CopyAtomNonExecIm2ColTmaLoadType = opaque { const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getKind(self: *const Self) Im2ColTmaLoad { + pub fn kind(self: *const Self) Im2ColTmaLoad { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetKind(self.ptr()))); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetValType(self.ptr()).ptr.?); } - pub fn getCopyBits(self: *const Self) c_int { + pub fn copyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetCopyBits(self.ptr()); } - pub fn getTmaGbasis(self: *const Self) *const mlir.Type { + pub fn tmaGbasis(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaGbasis(self.ptr()).ptr.?); } - pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { + pub fn tmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaFormat(self.ptr()); return if (value < 0) null else @fromBackingInt(@intCast(value)); } @@ -860,11 +828,9 @@ pub const CopyAtomNonExecIm2ColTmaStoreType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaStore; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.non_exec_im2col_tma_store"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -878,16 +844,16 @@ pub const CopyAtomNonExecIm2ColTmaStoreType = opaque { const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetValType(self.ptr()).ptr.?); } - pub fn getCopyBits(self: *const Self) c_int { + pub fn copyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetCopyBits(self.ptr()); } - pub fn getTmaGbasis(self: *const Self) *const mlir.Type { + pub fn tmaGbasis(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaGbasis(self.ptr()).ptr.?); } - pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { + pub fn tmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaFormat(self.ptr()); return if (value < 0) null else @fromBackingInt(@intCast(value)); } @@ -900,11 +866,9 @@ pub const CopyAtomNonExecTiledTmaLoadType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaLoad; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.non_exec_tiled_tma_load"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -919,19 +883,19 @@ pub const CopyAtomNonExecTiledTmaLoadType = opaque { const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getKind(self: *const Self) TiledTmaLoad { + pub fn kind(self: *const Self) TiledTmaLoad { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetKind(self.ptr()))); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetValType(self.ptr()).ptr.?); } - pub fn getCopyBits(self: *const Self) c_int { + pub fn copyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetCopyBits(self.ptr()); } - pub fn getTmaGbasis(self: *const Self) *const mlir.Type { + pub fn tmaGbasis(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaGbasis(self.ptr()).ptr.?); } - pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { + pub fn tmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaFormat(self.ptr()); return if (value < 0) null else @fromBackingInt(@intCast(value)); } @@ -944,11 +908,9 @@ pub const CopyAtomNonExecTiledTmaReduceType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaReduce; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.non_exec_tiled_tma_reduce"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -963,19 +925,19 @@ pub const CopyAtomNonExecTiledTmaReduceType = opaque { const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getKind(self: *const Self) ReductionKind { + pub fn kind(self: *const Self) ReductionKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetKind(self.ptr()))); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetValType(self.ptr()).ptr.?); } - pub fn getCopyBits(self: *const Self) c_int { + pub fn copyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetCopyBits(self.ptr()); } - pub fn getTmaGbasis(self: *const Self) *const mlir.Type { + pub fn tmaGbasis(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaGbasis(self.ptr()).ptr.?); } - pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { + pub fn tmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaFormat(self.ptr()); return if (value < 0) null else @fromBackingInt(@intCast(value)); } @@ -988,11 +950,9 @@ pub const CopyAtomNonExecTiledTmaStoreType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaStore; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.non_exec_tiled_tma_store"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1006,16 +966,16 @@ pub const CopyAtomNonExecTiledTmaStoreType = opaque { const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetValType(self.ptr()).ptr.?); } - pub fn getCopyBits(self: *const Self) c_int { + pub fn copyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetCopyBits(self.ptr()); } - pub fn getTmaGbasis(self: *const Self) *const mlir.Type { + pub fn tmaGbasis(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaGbasis(self.ptr()).ptr.?); } - pub fn getTmaFormat(self: *const Self) ?TmaDataFormat { + pub fn tmaFormat(self: *const Self) ?TmaDataFormat { const value = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaFormat(self.ptr()); return if (value < 0) null else @fromBackingInt(@intCast(value)); } @@ -1028,11 +988,9 @@ pub const CopyAtomR2GType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomR2G; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.r2g"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1049,25 +1007,25 @@ pub const CopyAtomR2GType = opaque { const result = c.mlirCuteNVGPUCopyAtomR2GTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.l1CacheEvictPriority), @backingInt(args.storeCacheMode), @backingInt(args.sharedSpace)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetValType(self.ptr()).ptr.?); } - pub fn getCopyBits(self: *const Self) c_int { + pub fn copyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomR2GTypeGetCopyBits(self.ptr()); } - pub fn getMemOrder(self: *const Self) MemOrderKind { + pub fn memOrder(self: *const Self) MemOrderKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetMemOrder(self.ptr()))); } - pub fn getMemScope(self: *const Self) MemScopeKind { + pub fn memScope(self: *const Self) MemScopeKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetMemScope(self.ptr()))); } - pub fn getL1CacheEvictPriority(self: *const Self) CacheEvictionPriority { + pub fn l1CacheEvictPriority(self: *const Self) CacheEvictionPriority { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetL1CacheEvictPriority(self.ptr()))); } - pub fn getStoreCacheMode(self: *const Self) StoreCacheMode { + pub fn storeCacheMode(self: *const Self) StoreCacheMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetStoreCacheMode(self.ptr()))); } - pub fn getSharedSpace(self: *const Self) SharedSpace { + pub fn sharedSpace(self: *const Self) SharedSpace { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetSharedSpace(self.ptr()))); } }; @@ -1079,11 +1037,9 @@ pub const CopyAtomR2SType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomR2S; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.r2s"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1098,19 +1054,19 @@ pub const CopyAtomR2SType = opaque { const result = c.mlirCuteNVGPUCopyAtomR2STypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.sharedSpace)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomR2STypeGetValType(self.ptr()).ptr.?); } - pub fn getCopyBits(self: *const Self) c_int { + pub fn copyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomR2STypeGetCopyBits(self.ptr()); } - pub fn getMemOrder(self: *const Self) MemOrderKind { + pub fn memOrder(self: *const Self) MemOrderKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2STypeGetMemOrder(self.ptr()))); } - pub fn getMemScope(self: *const Self) MemScopeKind { + pub fn memScope(self: *const Self) MemScopeKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2STypeGetMemScope(self.ptr()))); } - pub fn getSharedSpace(self: *const Self) SharedSpace { + pub fn sharedSpace(self: *const Self) SharedSpace { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2STypeGetSharedSpace(self.ptr()))); } }; @@ -1122,11 +1078,9 @@ pub const CopyAtomS2RType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomS2R; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.s2r"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1141,19 +1095,19 @@ pub const CopyAtomS2RType = opaque { const result = c.mlirCuteNVGPUCopyAtomS2RTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.sharedSpace)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetValType(self.ptr()).ptr.?); } - pub fn getCopyBits(self: *const Self) c_int { + pub fn copyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomS2RTypeGetCopyBits(self.ptr()); } - pub fn getMemOrder(self: *const Self) MemOrderKind { + pub fn memOrder(self: *const Self) MemOrderKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetMemOrder(self.ptr()))); } - pub fn getMemScope(self: *const Self) MemScopeKind { + pub fn memScope(self: *const Self) MemScopeKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetMemScope(self.ptr()))); } - pub fn getSharedSpace(self: *const Self) SharedSpace { + pub fn sharedSpace(self: *const Self) SharedSpace { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetSharedSpace(self.ptr()))); } }; @@ -1165,11 +1119,9 @@ pub const CopyAtomSIMTAsyncCopyType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTAsyncCopy; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.simt_async_copy"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1182,13 +1134,13 @@ pub const CopyAtomSIMTAsyncCopyType = opaque { const result = c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGet(ctx.ptr(), args.valType.ptr(), @backingInt(args.cache), args.copyBits); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetValType(self.ptr()).ptr.?); } - pub fn getCache(self: *const Self) LoadCacheMode { + pub fn cache(self: *const Self) LoadCacheMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCache(self.ptr()))); } - pub fn getCopyBits(self: *const Self) c_int { + pub fn copyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCopyBits(self.ptr()); } }; @@ -1200,11 +1152,9 @@ pub const CopyAtomSIMTMultimemLdReduceType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemLdReduce; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.simt_multimem_ld_reduce"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1220,22 +1170,22 @@ pub const CopyAtomSIMTMultimemLdReduceType = opaque { const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.reduction), if (args.ldReduceAccPrecision) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, @backingInt(args.memOrder), @backingInt(args.memScope)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetValType(self.ptr()).ptr.?); } - pub fn getCopyBits(self: *const Self) c_int { + pub fn copyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetCopyBits(self.ptr()); } - pub fn getReduction(self: *const Self) ReductionKind { + pub fn reduction(self: *const Self) ReductionKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetReduction(self.ptr()))); } - pub fn getLdReduceAccPrecision(self: *const Self) ?*const mlir.Attribute { + pub fn ldReduceAccPrecision(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetLdReduceAccPrecision(self.ptr()).ptr); } - pub fn getMemOrder(self: *const Self) MemOrderKind { + pub fn memOrder(self: *const Self) MemOrderKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemOrder(self.ptr()))); } - pub fn getMemScope(self: *const Self) MemScopeKind { + pub fn memScope(self: *const Self) MemScopeKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemScope(self.ptr()))); } }; @@ -1247,11 +1197,9 @@ pub const CopyAtomSIMTMultimemRedType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemRed; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.simt_multimem_red"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1266,19 +1214,19 @@ pub const CopyAtomSIMTMultimemRedType = opaque { const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.reduction), @backingInt(args.memOrder), @backingInt(args.memScope)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetValType(self.ptr()).ptr.?); } - pub fn getCopyBits(self: *const Self) c_int { + pub fn copyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetCopyBits(self.ptr()); } - pub fn getReduction(self: *const Self) ReductionKind { + pub fn reduction(self: *const Self) ReductionKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetReduction(self.ptr()))); } - pub fn getMemOrder(self: *const Self) MemOrderKind { + pub fn memOrder(self: *const Self) MemOrderKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemOrder(self.ptr()))); } - pub fn getMemScope(self: *const Self) MemScopeKind { + pub fn memScope(self: *const Self) MemScopeKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemScope(self.ptr()))); } }; @@ -1290,11 +1238,9 @@ pub const CopyAtomSIMTMultimemStType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemSt; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.simt_multimem_st"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1308,16 +1254,16 @@ pub const CopyAtomSIMTMultimemStType = opaque { const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetValType(self.ptr()).ptr.?); } - pub fn getCopyBits(self: *const Self) c_int { + pub fn copyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetCopyBits(self.ptr()); } - pub fn getMemOrder(self: *const Self) MemOrderKind { + pub fn memOrder(self: *const Self) MemOrderKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemOrder(self.ptr()))); } - pub fn getMemScope(self: *const Self) MemScopeKind { + pub fn memScope(self: *const Self) MemScopeKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemScope(self.ptr()))); } }; @@ -1329,34 +1275,32 @@ pub const CopyAtomSIMTSyncCopyType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTSyncCopy; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.universal_copy"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { valType: *const mlir.Type, copyBits: c_int = 0, - srcSpace: cute.AddressSpace = .generic, - dstSpace: cute.AddressSpace = .generic, + srcSpace: cute_ir.AddressSpace = .generic, + dstSpace: cute_ir.AddressSpace = .generic, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { const result = c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.srcSpace), @backingInt(args.dstSpace)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetValType(self.ptr()).ptr.?); } - pub fn getCopyBits(self: *const Self) c_int { + pub fn copyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetCopyBits(self.ptr()); } - pub fn getSrcSpace(self: *const Self) cute.AddressSpace { + pub fn srcSpace(self: *const Self) cute_ir.AddressSpace { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetSrcSpace(self.ptr()))); } - pub fn getDstSpace(self: *const Self) cute.AddressSpace { + pub fn dstSpace(self: *const Self) cute_ir.AddressSpace { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetDstSpace(self.ptr()))); } }; @@ -1368,11 +1312,9 @@ pub const CopyAtomSM100CopyS2TType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM100CopyS2T; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.s2t_copy"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1387,19 +1329,19 @@ pub const CopyAtomSM100CopyS2TType = opaque { const result = c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numCta, @backingInt(args.broadcast)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetValType(self.ptr()).ptr.?); } - pub fn getNumDp(self: *const Self) c_int { + pub fn numDp(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumDp(self.ptr()); } - pub fn getNumBit(self: *const Self) c_int { + pub fn numBit(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumBit(self.ptr()); } - pub fn getNumCta(self: *const Self) c_int { + pub fn numCta(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumCta(self.ptr()); } - pub fn getBroadcast(self: *const Self) CopyS2TBroadcast { + pub fn broadcast(self: *const Self) CopyS2TBroadcast { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetBroadcast(self.ptr()))); } }; @@ -1411,11 +1353,9 @@ pub const CopyAtomSM100S2TCopyV2Type = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM100S2TCopyV2; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.sm100_s2t_copy_v2"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1424,29 +1364,29 @@ pub const CopyAtomSM100S2TCopyV2Type = opaque { numDp: c_int, numBit: c_int, numCta: c_int, - smemMajor: cute.MajorMode, + smemMajor: cute_ir.MajorMode, broadcast: CopyS2TBroadcast = .none, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { const result = c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numCta, @backingInt(args.smemMajor), @backingInt(args.broadcast)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetValType(self.ptr()).ptr.?); } - pub fn getNumDp(self: *const Self) c_int { + pub fn numDp(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumDp(self.ptr()); } - pub fn getNumBit(self: *const Self) c_int { + pub fn numBit(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumBit(self.ptr()); } - pub fn getNumCta(self: *const Self) c_int { + pub fn numCta(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumCta(self.ptr()); } - pub fn getSmemMajor(self: *const Self) cute.MajorMode { + pub fn smemMajor(self: *const Self) cute_ir.MajorMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetSmemMajor(self.ptr()))); } - pub fn getBroadcast(self: *const Self) CopyS2TBroadcast { + pub fn broadcast(self: *const Self) CopyS2TBroadcast { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetBroadcast(self.ptr()))); } }; @@ -1458,11 +1398,9 @@ pub const CopyAtomSM100TmemLoadType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM100TmemLoad; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.tmem_load"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1470,26 +1408,26 @@ pub const CopyAtomSM100TmemLoadType = opaque { valType: *const mlir.Type, numDp: c_int, numBit: c_int, - numRep: c_uint, + numRep: u32, pack16b: bool = false, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { const result = c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, args.pack16b); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetValType(self.ptr()).ptr.?); } - pub fn getNumDp(self: *const Self) c_int { + pub fn numDp(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumDp(self.ptr()); } - pub fn getNumBit(self: *const Self) c_int { + pub fn numBit(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumBit(self.ptr()); } - pub fn getNumRep(self: *const Self) c_uint { + pub fn numRep(self: *const Self) u32 { return c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumRep(self.ptr()); } - pub fn getPack16b(self: *const Self) bool { + pub fn pack16b(self: *const Self) bool { return c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetPack16b(self.ptr()); } }; @@ -1501,11 +1439,9 @@ pub const CopyAtomSM100TmemStoreType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM100TmemStore; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.tmem_store"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1513,26 +1449,26 @@ pub const CopyAtomSM100TmemStoreType = opaque { valType: *const mlir.Type, numDp: c_int, numBit: c_int, - numRep: c_uint, + numRep: u32, expand16b: bool = false, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { const result = c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, args.expand16b); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetValType(self.ptr()).ptr.?); } - pub fn getNumDp(self: *const Self) c_int { + pub fn numDp(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumDp(self.ptr()); } - pub fn getNumBit(self: *const Self) c_int { + pub fn numBit(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumBit(self.ptr()); } - pub fn getNumRep(self: *const Self) c_uint { + pub fn numRep(self: *const Self) u32 { return c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumRep(self.ptr()); } - pub fn getExpand16b(self: *const Self) bool { + pub fn expand16b(self: *const Self) bool { return c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetExpand16b(self.ptr()); } }; @@ -1544,11 +1480,9 @@ pub const CopyAtomSM107TmemLoadSPCompressType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM107TmemLoadSPCompress; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.tmem_load_spcompress"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1556,7 +1490,7 @@ pub const CopyAtomSM107TmemLoadSPCompressType = opaque { valType: *const mlir.Type, numDp: c_int, numBit: c_int, - numRep: c_uint, + numRep: u32, redOp: TmemLoadRedOp, red: bool = false, nan: bool = false, @@ -1565,25 +1499,25 @@ pub const CopyAtomSM107TmemLoadSPCompressType = opaque { const result = c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, @backingInt(args.redOp), args.red, args.nan); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetValType(self.ptr()).ptr.?); } - pub fn getNumDp(self: *const Self) c_int { + pub fn numDp(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumDp(self.ptr()); } - pub fn getNumBit(self: *const Self) c_int { + pub fn numBit(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumBit(self.ptr()); } - pub fn getNumRep(self: *const Self) c_uint { + pub fn numRep(self: *const Self) u32 { return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumRep(self.ptr()); } - pub fn getRedOp(self: *const Self) TmemLoadRedOp { + pub fn redOp(self: *const Self) TmemLoadRedOp { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRedOp(self.ptr()))); } - pub fn getRed(self: *const Self) bool { + pub fn red(self: *const Self) bool { return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRed(self.ptr()); } - pub fn getNan(self: *const Self) bool { + pub fn nan(self: *const Self) bool { return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNan(self.ptr()); } }; @@ -1595,11 +1529,9 @@ pub const CopyAtomSM10xTmemLoadRedType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM10xTmemLoadRed; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.tmem_load_red"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1607,7 +1539,7 @@ pub const CopyAtomSM10xTmemLoadRedType = opaque { valType: *const mlir.Type, numDp: c_int, numBit: c_int, - numRep: c_uint, + numRep: u32, redOp: TmemLoadRedOp, nan: bool = false, halfSplitOff: ?*const mlir.Attribute = null, @@ -1616,25 +1548,25 @@ pub const CopyAtomSM10xTmemLoadRedType = opaque { const result = c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, @backingInt(args.redOp), args.nan, if (args.halfSplitOff) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetValType(self.ptr()).ptr.?); } - pub fn getNumDp(self: *const Self) c_int { + pub fn numDp(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumDp(self.ptr()); } - pub fn getNumBit(self: *const Self) c_int { + pub fn numBit(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumBit(self.ptr()); } - pub fn getNumRep(self: *const Self) c_uint { + pub fn numRep(self: *const Self) u32 { return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumRep(self.ptr()); } - pub fn getRedOp(self: *const Self) TmemLoadRedOp { + pub fn redOp(self: *const Self) TmemLoadRedOp { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetRedOp(self.ptr()))); } - pub fn getNan(self: *const Self) bool { + pub fn nan(self: *const Self) bool { return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNan(self.ptr()); } - pub fn getHalfSplitOff(self: *const Self) ?*const mlir.Attribute { + pub fn halfSplitOff(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetHalfSplitOff(self.ptr()).ptr); } }; @@ -1646,11 +1578,9 @@ pub const CopyAtomStsmType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomStsm; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.stsm"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1664,16 +1594,16 @@ pub const CopyAtomStsmType = opaque { const result = c.mlirCuteNVGPUCopyAtomStsmTypeGet(ctx.ptr(), args.valType.ptr(), args.mode.ptr(), args.numMatrices, args.transpose); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomStsmTypeGetValType(self.ptr()).ptr.?); } - pub fn getMode(self: *const Self) *const mlir.Attribute { + pub fn mode(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUCopyAtomStsmTypeGetMode(self.ptr()).ptr.?); } - pub fn getNumMatrices(self: *const Self) c_int { + pub fn numMatrices(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomStsmTypeGetNumMatrices(self.ptr()); } - pub fn getTranspose(self: *const Self) bool { + pub fn transpose(self: *const Self) bool { return c.mlirCuteNVGPUCopyAtomStsmTypeGetTranspose(self.ptr()); } }; @@ -1685,11 +1615,9 @@ pub const CopyAtomTmaLoadType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomTmaLoad; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.tma_load"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1709,34 +1637,34 @@ pub const CopyAtomTmaLoadType = opaque { const result = c.mlirCuteNVGPUCopyAtomTmaLoadTypeGet(ctx.ptr(), args.valType.ptr(), args.sparsity, args.copyBits, @backingInt(args.mode), args.numCta, args.gStride.ptr(), args.mcast, if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }, args.override, args.noFullyOobTile); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetValType(self.ptr()).ptr.?); } - pub fn getSparsity(self: *const Self) c_int { + pub fn sparsity(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetSparsity(self.ptr()); } - pub fn getCopyBits(self: *const Self) c_int { + pub fn copyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetCopyBits(self.ptr()); } - pub fn getMode(self: *const Self) TmaLoadMode { + pub fn mode(self: *const Self) TmaLoadMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetMode(self.ptr()))); } - pub fn getNumCta(self: *const Self) c_int { + pub fn numCta(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetNumCta(self.ptr()); } - pub fn getGStride(self: *const Self) *const mlir.Type { + pub fn gStride(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetGStride(self.ptr()).ptr.?); } - pub fn getMcast(self: *const Self) bool { + pub fn mcast(self: *const Self) bool { return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetMcast(self.ptr()); } - pub fn getTmaGbasis(self: *const Self) ?*const mlir.Type { + pub fn tmaGbasis(self: *const Self) ?*const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetTmaGbasis(self.ptr()).ptr); } - pub fn getOverride(self: *const Self) bool { + pub fn override(self: *const Self) bool { return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetOverride(self.ptr()); } - pub fn getNoFullyOobTile(self: *const Self) bool { + pub fn noFullyOobTile(self: *const Self) bool { return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetNoFullyOobTile(self.ptr()); } }; @@ -1748,11 +1676,9 @@ pub const CopyAtomTmaReduceType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomTmaReduce; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.tma_reduce"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1768,22 +1694,22 @@ pub const CopyAtomTmaReduceType = opaque { const result = c.mlirCuteNVGPUCopyAtomTmaReduceTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.mode), @backingInt(args.kind), args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetValType(self.ptr()).ptr.?); } - pub fn getCopyBits(self: *const Self) c_int { + pub fn copyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetCopyBits(self.ptr()); } - pub fn getMode(self: *const Self) TmaStoreMode { + pub fn mode(self: *const Self) TmaStoreMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetMode(self.ptr()))); } - pub fn getKind(self: *const Self) ReductionKind { + pub fn kind(self: *const Self) ReductionKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetKind(self.ptr()))); } - pub fn getGStride(self: *const Self) *const mlir.Type { + pub fn gStride(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetGStride(self.ptr()).ptr.?); } - pub fn getTmaGbasis(self: *const Self) ?*const mlir.Type { + pub fn tmaGbasis(self: *const Self) ?*const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetTmaGbasis(self.ptr()).ptr); } }; @@ -1795,11 +1721,9 @@ pub const CopyAtomTmaStoreType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomTmaStore; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.tma_store"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -1817,28 +1741,28 @@ pub const CopyAtomTmaStoreType = opaque { const result = c.mlirCuteNVGPUCopyAtomTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.sparsity, args.copyBits, @backingInt(args.mode), args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }, args.override, args.noFullyOobTile); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValType(self: *const Self) *const mlir.Type { + pub fn valType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetValType(self.ptr()).ptr.?); } - pub fn getSparsity(self: *const Self) c_int { + pub fn sparsity(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetSparsity(self.ptr()); } - pub fn getCopyBits(self: *const Self) c_int { + pub fn copyBits(self: *const Self) c_int { return c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetCopyBits(self.ptr()); } - pub fn getMode(self: *const Self) TmaStoreMode { + pub fn mode(self: *const Self) TmaStoreMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetMode(self.ptr()))); } - pub fn getGStride(self: *const Self) *const mlir.Type { + pub fn gStride(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetGStride(self.ptr()).ptr.?); } - pub fn getTmaGbasis(self: *const Self) ?*const mlir.Type { + pub fn tmaGbasis(self: *const Self) ?*const mlir.Type { return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetTmaGbasis(self.ptr()).ptr); } - pub fn getOverride(self: *const Self) bool { + pub fn override(self: *const Self) bool { return c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetOverride(self.ptr()); } - pub fn getNoFullyOobTile(self: *const Self) bool { + pub fn noFullyOobTile(self: *const Self) bool { return c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetNoFullyOobTile(self.ptr()); } }; @@ -1850,19 +1774,17 @@ pub const MmaAtomSM100UMMABlockScaledSparseType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaledSparse; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "sm100.mma_bs_sp"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { shapeMnk: *const mlir.Attribute, numCta: c_int, - aMajor: cute.MajorMode, - bMajor: cute.MajorMode, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, aType: *const mlir.Type, bType: *const mlir.Type, cType: *const mlir.Type, @@ -1877,43 +1799,43 @@ pub const MmaAtomSM100UMMABlockScaledSparseType = opaque { const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.archPromote), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetShapeMnk(self.ptr()).ptr.?); } - pub fn getNumCta(self: *const Self) c_int { + pub fn numCta(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetNumCta(self.ptr()); } - pub fn getAMajor(self: *const Self) cute.MajorMode { + pub fn aMajor(self: *const Self) cute_ir.MajorMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAMajor(self.ptr()))); } - pub fn getBMajor(self: *const Self) cute.MajorMode { + pub fn bMajor(self: *const Self) cute_ir.MajorMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBMajor(self.ptr()))); } - pub fn getAType(self: *const Self) *const mlir.Type { + pub fn aType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAType(self.ptr()).ptr.?); } - pub fn getBType(self: *const Self) *const mlir.Type { + pub fn bType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBType(self.ptr()).ptr.?); } - pub fn getCType(self: *const Self) *const mlir.Type { + pub fn cType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetCType(self.ptr()).ptr.?); } - pub fn getSfType(self: *const Self) *const mlir.Type { + pub fn sfType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSfType(self.ptr()).ptr.?); } - pub fn getSparseMetadataFormat(self: *const Self) SparseMetadataFormat { + pub fn sparseMetadataFormat(self: *const Self) SparseMetadataFormat { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSparseMetadataFormat(self.ptr()))); } - pub fn getAFragKind(self: *const Self) MmaFragKind { + pub fn aFragKind(self: *const Self) MmaFragKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAFragKind(self.ptr()))); } - pub fn getVecSize(self: *const Self) c_int { + pub fn vecSize(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetVecSize(self.ptr()); } - pub fn getArchPromote(self: *const Self) Arch { + pub fn archPromote(self: *const Self) Arch { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetArchPromote(self.ptr()))); } - pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetIntOverflow(self.ptr()).ptr); } }; @@ -1925,19 +1847,17 @@ pub const MmaAtomSM100UMMABlockScaledType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaled; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "sm100.mma_bs"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { shapeMnk: *const mlir.Attribute, numCta: c_int, - aMajor: cute.MajorMode, - bMajor: cute.MajorMode, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, aType: *const mlir.Type, bType: *const mlir.Type, cType: *const mlir.Type, @@ -1951,40 +1871,40 @@ pub const MmaAtomSM100UMMABlockScaledType = opaque { const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.archPromote), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetShapeMnk(self.ptr()).ptr.?); } - pub fn getNumCta(self: *const Self) c_int { + pub fn numCta(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetNumCta(self.ptr()); } - pub fn getAMajor(self: *const Self) cute.MajorMode { + pub fn aMajor(self: *const Self) cute_ir.MajorMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAMajor(self.ptr()))); } - pub fn getBMajor(self: *const Self) cute.MajorMode { + pub fn bMajor(self: *const Self) cute_ir.MajorMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBMajor(self.ptr()))); } - pub fn getAType(self: *const Self) *const mlir.Type { + pub fn aType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAType(self.ptr()).ptr.?); } - pub fn getBType(self: *const Self) *const mlir.Type { + pub fn bType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBType(self.ptr()).ptr.?); } - pub fn getCType(self: *const Self) *const mlir.Type { + pub fn cType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetCType(self.ptr()).ptr.?); } - pub fn getSfType(self: *const Self) *const mlir.Type { + pub fn sfType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetSfType(self.ptr()).ptr.?); } - pub fn getAFragKind(self: *const Self) MmaFragKind { + pub fn aFragKind(self: *const Self) MmaFragKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAFragKind(self.ptr()))); } - pub fn getVecSize(self: *const Self) c_int { + pub fn vecSize(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetVecSize(self.ptr()); } - pub fn getArchPromote(self: *const Self) Arch { + pub fn archPromote(self: *const Self) Arch { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetArchPromote(self.ptr()))); } - pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetIntOverflow(self.ptr()).ptr); } }; @@ -1996,19 +1916,17 @@ pub const MmaAtomSM100UMMASparseType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM100UMMASparse; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "sm100.mma_sp"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { shapeMnk: *const mlir.Attribute, numCta: c_int, - aMajor: cute.MajorMode, - bMajor: cute.MajorMode, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, aType: *const mlir.Type, bType: *const mlir.Type, cType: *const mlir.Type, @@ -2022,40 +1940,40 @@ pub const MmaAtomSM100UMMASparseType = opaque { const result = c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.eType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.cScaleExp, if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetShapeMnk(self.ptr()).ptr.?); } - pub fn getNumCta(self: *const Self) c_int { + pub fn numCta(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetNumCta(self.ptr()); } - pub fn getAMajor(self: *const Self) cute.MajorMode { + pub fn aMajor(self: *const Self) cute_ir.MajorMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAMajor(self.ptr()))); } - pub fn getBMajor(self: *const Self) cute.MajorMode { + pub fn bMajor(self: *const Self) cute_ir.MajorMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBMajor(self.ptr()))); } - pub fn getAType(self: *const Self) *const mlir.Type { + pub fn aType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAType(self.ptr()).ptr.?); } - pub fn getBType(self: *const Self) *const mlir.Type { + pub fn bType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBType(self.ptr()).ptr.?); } - pub fn getCType(self: *const Self) *const mlir.Type { + pub fn cType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCType(self.ptr()).ptr.?); } - pub fn getEType(self: *const Self) *const mlir.Type { + pub fn eType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetEType(self.ptr()).ptr.?); } - pub fn getSparseMetadataFormat(self: *const Self) SparseMetadataFormat { + pub fn sparseMetadataFormat(self: *const Self) SparseMetadataFormat { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetSparseMetadataFormat(self.ptr()))); } - pub fn getAFragKind(self: *const Self) MmaFragKind { + pub fn aFragKind(self: *const Self) MmaFragKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAFragKind(self.ptr()))); } - pub fn getCScaleExp(self: *const Self) c_int { + pub fn cScaleExp(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCScaleExp(self.ptr()); } - pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetIntOverflow(self.ptr()).ptr); } }; @@ -2067,19 +1985,17 @@ pub const MmaAtomSM100UMMAType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM100UMMA; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "sm100.mma"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { shapeMnk: *const mlir.Attribute, numCta: c_int, - aMajor: cute.MajorMode, - bMajor: cute.MajorMode, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, aType: *const mlir.Type, bType: *const mlir.Type, cType: *const mlir.Type, @@ -2091,34 +2007,34 @@ pub const MmaAtomSM100UMMAType = opaque { const result = c.mlirCuteNVGPUMmaAtomSM100UMMATypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.aFragKind), args.cScaleExp, if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetShapeMnk(self.ptr()).ptr.?); } - pub fn getNumCta(self: *const Self) c_int { + pub fn numCta(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetNumCta(self.ptr()); } - pub fn getAMajor(self: *const Self) cute.MajorMode { + pub fn aMajor(self: *const Self) cute_ir.MajorMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAMajor(self.ptr()))); } - pub fn getBMajor(self: *const Self) cute.MajorMode { + pub fn bMajor(self: *const Self) cute_ir.MajorMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetBMajor(self.ptr()))); } - pub fn getAType(self: *const Self) *const mlir.Type { + pub fn aType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAType(self.ptr()).ptr.?); } - pub fn getBType(self: *const Self) *const mlir.Type { + pub fn bType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetBType(self.ptr()).ptr.?); } - pub fn getCType(self: *const Self) *const mlir.Type { + pub fn cType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetCType(self.ptr()).ptr.?); } - pub fn getAFragKind(self: *const Self) MmaFragKind { + pub fn aFragKind(self: *const Self) MmaFragKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAFragKind(self.ptr()))); } - pub fn getCScaleExp(self: *const Self) c_int { + pub fn cScaleExp(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetCScaleExp(self.ptr()); } - pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetIntOverflow(self.ptr()).ptr); } }; @@ -2130,19 +2046,17 @@ pub const MmaAtomSM107UMMABlockScaledSparseType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaledSparse; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "sm107.mma_bs_sp"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { shapeMnk: *const mlir.Attribute, numCta: c_int, - aMajor: cute.MajorMode, - bMajor: cute.MajorMode, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, aType: *const mlir.Type, bType: *const mlir.Type, cType: *const mlir.Type, @@ -2158,46 +2072,46 @@ pub const MmaAtomSM107UMMABlockScaledSparseType = opaque { const result = c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetShapeMnk(self.ptr()).ptr.?); } - pub fn getNumCta(self: *const Self) c_int { + pub fn numCta(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetNumCta(self.ptr()); } - pub fn getAMajor(self: *const Self) cute.MajorMode { + pub fn aMajor(self: *const Self) cute_ir.MajorMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAMajor(self.ptr()))); } - pub fn getBMajor(self: *const Self) cute.MajorMode { + pub fn bMajor(self: *const Self) cute_ir.MajorMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBMajor(self.ptr()))); } - pub fn getAType(self: *const Self) *const mlir.Type { + pub fn aType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAType(self.ptr()).ptr.?); } - pub fn getBType(self: *const Self) *const mlir.Type { + pub fn bType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBType(self.ptr()).ptr.?); } - pub fn getCType(self: *const Self) *const mlir.Type { + pub fn cType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetCType(self.ptr()).ptr.?); } - pub fn getSfType(self: *const Self) *const mlir.Type { + pub fn sfType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSfType(self.ptr()).ptr.?); } - pub fn getSparseMetadataFormat(self: *const Self) SparseMetadataFormat { + pub fn sparseMetadataFormat(self: *const Self) SparseMetadataFormat { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSparseMetadataFormat(self.ptr()))); } - pub fn getAFragKind(self: *const Self) MmaFragKind { + pub fn aFragKind(self: *const Self) MmaFragKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAFragKind(self.ptr()))); } - pub fn getVecSize(self: *const Self) c_int { + pub fn vecSize(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetVecSize(self.ptr()); } - pub fn getACollectorOp(self: *const Self) MmaCollectorOp { + pub fn aCollectorOp(self: *const Self) MmaCollectorOp { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetACollectorOp(self.ptr()))); } - pub fn getBCollectorOp(self: *const Self) MmaCollectorOp { + pub fn bCollectorOp(self: *const Self) MmaCollectorOp { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBCollectorOp(self.ptr()))); } - pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetIntOverflow(self.ptr()).ptr); } }; @@ -2209,19 +2123,17 @@ pub const MmaAtomSM107UMMABlockScaledType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaled; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "sm107.mma_bs"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { shapeMnk: *const mlir.Attribute, numCta: c_int, - aMajor: cute.MajorMode, - bMajor: cute.MajorMode, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, aType: *const mlir.Type, bType: *const mlir.Type, cType: *const mlir.Type, @@ -2236,43 +2148,43 @@ pub const MmaAtomSM107UMMABlockScaledType = opaque { const result = c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetShapeMnk(self.ptr()).ptr.?); } - pub fn getNumCta(self: *const Self) c_int { + pub fn numCta(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetNumCta(self.ptr()); } - pub fn getAMajor(self: *const Self) cute.MajorMode { + pub fn aMajor(self: *const Self) cute_ir.MajorMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAMajor(self.ptr()))); } - pub fn getBMajor(self: *const Self) cute.MajorMode { + pub fn bMajor(self: *const Self) cute_ir.MajorMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBMajor(self.ptr()))); } - pub fn getAType(self: *const Self) *const mlir.Type { + pub fn aType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAType(self.ptr()).ptr.?); } - pub fn getBType(self: *const Self) *const mlir.Type { + pub fn bType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBType(self.ptr()).ptr.?); } - pub fn getCType(self: *const Self) *const mlir.Type { + pub fn cType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetCType(self.ptr()).ptr.?); } - pub fn getSfType(self: *const Self) *const mlir.Type { + pub fn sfType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetSfType(self.ptr()).ptr.?); } - pub fn getAFragKind(self: *const Self) MmaFragKind { + pub fn aFragKind(self: *const Self) MmaFragKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAFragKind(self.ptr()))); } - pub fn getVecSize(self: *const Self) c_int { + pub fn vecSize(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetVecSize(self.ptr()); } - pub fn getACollectorOp(self: *const Self) MmaCollectorOp { + pub fn aCollectorOp(self: *const Self) MmaCollectorOp { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetACollectorOp(self.ptr()))); } - pub fn getBCollectorOp(self: *const Self) MmaCollectorOp { + pub fn bCollectorOp(self: *const Self) MmaCollectorOp { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBCollectorOp(self.ptr()))); } - pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetIntOverflow(self.ptr()).ptr); } }; @@ -2284,19 +2196,17 @@ pub const MmaAtomSM107UMMASparseType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM107UMMASparse; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "sm107.mma_sp"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { shapeMnk: *const mlir.Attribute, numCta: c_int, - aMajor: cute.MajorMode, - bMajor: cute.MajorMode, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, aType: *const mlir.Type, bType: *const mlir.Type, cType: *const mlir.Type, @@ -2312,46 +2222,46 @@ pub const MmaAtomSM107UMMASparseType = opaque { const result = c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.eType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.cScaleExp, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetShapeMnk(self.ptr()).ptr.?); } - pub fn getNumCta(self: *const Self) c_int { + pub fn numCta(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetNumCta(self.ptr()); } - pub fn getAMajor(self: *const Self) cute.MajorMode { + pub fn aMajor(self: *const Self) cute_ir.MajorMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAMajor(self.ptr()))); } - pub fn getBMajor(self: *const Self) cute.MajorMode { + pub fn bMajor(self: *const Self) cute_ir.MajorMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBMajor(self.ptr()))); } - pub fn getAType(self: *const Self) *const mlir.Type { + pub fn aType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAType(self.ptr()).ptr.?); } - pub fn getBType(self: *const Self) *const mlir.Type { + pub fn bType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBType(self.ptr()).ptr.?); } - pub fn getCType(self: *const Self) *const mlir.Type { + pub fn cType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCType(self.ptr()).ptr.?); } - pub fn getEType(self: *const Self) *const mlir.Type { + pub fn eType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetEType(self.ptr()).ptr.?); } - pub fn getSparseMetadataFormat(self: *const Self) SparseMetadataFormat { + pub fn sparseMetadataFormat(self: *const Self) SparseMetadataFormat { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetSparseMetadataFormat(self.ptr()))); } - pub fn getAFragKind(self: *const Self) MmaFragKind { + pub fn aFragKind(self: *const Self) MmaFragKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAFragKind(self.ptr()))); } - pub fn getCScaleExp(self: *const Self) c_int { + pub fn cScaleExp(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCScaleExp(self.ptr()); } - pub fn getACollectorOp(self: *const Self) MmaCollectorOp { + pub fn aCollectorOp(self: *const Self) MmaCollectorOp { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetACollectorOp(self.ptr()))); } - pub fn getBCollectorOp(self: *const Self) MmaCollectorOp { + pub fn bCollectorOp(self: *const Self) MmaCollectorOp { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBCollectorOp(self.ptr()))); } - pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetIntOverflow(self.ptr()).ptr); } }; @@ -2363,19 +2273,17 @@ pub const MmaAtomSM107UMMAType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM107UMMA; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "sm107.mma"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { shapeMnk: *const mlir.Attribute, numCta: c_int, - aMajor: cute.MajorMode, - bMajor: cute.MajorMode, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, aType: *const mlir.Type, bType: *const mlir.Type, cType: *const mlir.Type, @@ -2389,40 +2297,40 @@ pub const MmaAtomSM107UMMAType = opaque { const result = c.mlirCuteNVGPUMmaAtomSM107UMMATypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.aFragKind), args.cScaleExp, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetShapeMnk(self.ptr()).ptr.?); } - pub fn getNumCta(self: *const Self) c_int { + pub fn numCta(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetNumCta(self.ptr()); } - pub fn getAMajor(self: *const Self) cute.MajorMode { + pub fn aMajor(self: *const Self) cute_ir.MajorMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAMajor(self.ptr()))); } - pub fn getBMajor(self: *const Self) cute.MajorMode { + pub fn bMajor(self: *const Self) cute_ir.MajorMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBMajor(self.ptr()))); } - pub fn getAType(self: *const Self) *const mlir.Type { + pub fn aType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAType(self.ptr()).ptr.?); } - pub fn getBType(self: *const Self) *const mlir.Type { + pub fn bType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBType(self.ptr()).ptr.?); } - pub fn getCType(self: *const Self) *const mlir.Type { + pub fn cType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetCType(self.ptr()).ptr.?); } - pub fn getAFragKind(self: *const Self) MmaFragKind { + pub fn aFragKind(self: *const Self) MmaFragKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAFragKind(self.ptr()))); } - pub fn getCScaleExp(self: *const Self) c_int { + pub fn cScaleExp(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetCScaleExp(self.ptr()); } - pub fn getACollectorOp(self: *const Self) MmaCollectorOp { + pub fn aCollectorOp(self: *const Self) MmaCollectorOp { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetACollectorOp(self.ptr()))); } - pub fn getBCollectorOp(self: *const Self) MmaCollectorOp { + pub fn bCollectorOp(self: *const Self) MmaCollectorOp { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBCollectorOp(self.ptr()))); } - pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetIntOverflow(self.ptr()).ptr); } }; @@ -2434,11 +2342,9 @@ pub const MmaAtomSM120BlockScaledType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM120BlockScaled; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "SM120.mma_bs"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -2455,25 +2361,25 @@ pub const MmaAtomSM120BlockScaledType = opaque { const result = c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.vecSize, args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), args.useSfLayoutTV); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetShapeMnk(self.ptr()).ptr.?); } - pub fn getVecSize(self: *const Self) c_int { + pub fn vecSize(self: *const Self) c_int { return c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetVecSize(self.ptr()); } - pub fn getAType(self: *const Self) *const mlir.Type { + pub fn aType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetAType(self.ptr()).ptr.?); } - pub fn getBType(self: *const Self) *const mlir.Type { + pub fn bType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetBType(self.ptr()).ptr.?); } - pub fn getCType(self: *const Self) *const mlir.Type { + pub fn cType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetCType(self.ptr()).ptr.?); } - pub fn getSfType(self: *const Self) *const mlir.Type { + pub fn sfType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetSfType(self.ptr()).ptr.?); } - pub fn getUseSfLayoutTV(self: *const Self) bool { + pub fn useSfLayoutTV(self: *const Self) bool { return c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetUseSfLayoutTV(self.ptr()); } }; @@ -2485,11 +2391,9 @@ pub const MmaAtomSM80SparseType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM80Sparse; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "sm80.sparse_mma"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -2505,22 +2409,22 @@ pub const MmaAtomSM80SparseType = opaque { const result = c.mlirCuteNVGPUMmaAtomSM80SparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.sparseMetadataFormat), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetShapeMnk(self.ptr()).ptr.?); } - pub fn getAType(self: *const Self) *const mlir.Type { + pub fn aType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetAType(self.ptr()).ptr.?); } - pub fn getBType(self: *const Self) *const mlir.Type { + pub fn bType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetBType(self.ptr()).ptr.?); } - pub fn getCType(self: *const Self) *const mlir.Type { + pub fn cType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetCType(self.ptr()).ptr.?); } - pub fn getSparseMetadataFormat(self: *const Self) SparseMetadataFormat { + pub fn sparseMetadataFormat(self: *const Self) SparseMetadataFormat { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetSparseMetadataFormat(self.ptr()))); } - pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetIntOverflow(self.ptr()).ptr); } }; @@ -2532,11 +2436,9 @@ pub const MmaAtomSM80Type = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM80; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "sm80.mma"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -2552,22 +2454,22 @@ pub const MmaAtomSM80Type = opaque { const result = c.mlirCuteNVGPUMmaAtomSM80TypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, if (args.binaryOp) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetShapeMnk(self.ptr()).ptr.?); } - pub fn getAType(self: *const Self) *const mlir.Type { + pub fn aType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetAType(self.ptr()).ptr.?); } - pub fn getBType(self: *const Self) *const mlir.Type { + pub fn bType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetBType(self.ptr()).ptr.?); } - pub fn getCType(self: *const Self) *const mlir.Type { + pub fn cType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetCType(self.ptr()).ptr.?); } - pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetIntOverflow(self.ptr()).ptr); } - pub fn getBinaryOp(self: *const Self) ?*const mlir.Attribute { + pub fn binaryOp(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetBinaryOp(self.ptr()).ptr); } }; @@ -2579,11 +2481,9 @@ pub const MmaAtomSM89Type = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM89; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "sm89.mma"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -2597,16 +2497,16 @@ pub const MmaAtomSM89Type = opaque { const result = c.mlirCuteNVGPUMmaAtomSM89TypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM89TypeGetShapeMnk(self.ptr()).ptr.?); } - pub fn getAType(self: *const Self) *const mlir.Type { + pub fn aType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM89TypeGetAType(self.ptr()).ptr.?); } - pub fn getBType(self: *const Self) *const mlir.Type { + pub fn bType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM89TypeGetBType(self.ptr()).ptr.?); } - pub fn getCType(self: *const Self) *const mlir.Type { + pub fn cType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM89TypeGetCType(self.ptr()).ptr.?); } }; @@ -2618,18 +2518,16 @@ pub const MmaAtomSM90Type = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM90; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "sm90.mma"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } pub const InitArgs = struct { shapeMnk: *const mlir.Attribute, - aMajor: cute.MajorMode, - bMajor: cute.MajorMode, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, aType: *const mlir.Type, bType: *const mlir.Type, cType: *const mlir.Type, @@ -2642,34 +2540,34 @@ pub const MmaAtomSM90Type = opaque { const result = c.mlirCuteNVGPUMmaAtomSM90TypeGet(ctx.ptr(), args.shapeMnk.ptr(), @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.aFragKind), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, args.aNeg, args.bNeg); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetShapeMnk(self.ptr()).ptr.?); } - pub fn getAMajor(self: *const Self) cute.MajorMode { + pub fn aMajor(self: *const Self) cute_ir.MajorMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetAMajor(self.ptr()))); } - pub fn getBMajor(self: *const Self) cute.MajorMode { + pub fn bMajor(self: *const Self) cute_ir.MajorMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetBMajor(self.ptr()))); } - pub fn getAType(self: *const Self) *const mlir.Type { + pub fn aType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetAType(self.ptr()).ptr.?); } - pub fn getBType(self: *const Self) *const mlir.Type { + pub fn bType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetBType(self.ptr()).ptr.?); } - pub fn getCType(self: *const Self) *const mlir.Type { + pub fn cType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetCType(self.ptr()).ptr.?); } - pub fn getAFragKind(self: *const Self) MmaFragKind { + pub fn aFragKind(self: *const Self) MmaFragKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetAFragKind(self.ptr()))); } - pub fn getIntOverflow(self: *const Self) ?*const mlir.Attribute { + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetIntOverflow(self.ptr()).ptr); } - pub fn getANeg(self: *const Self) bool { + pub fn aNeg(self: *const Self) bool { return c.mlirCuteNVGPUMmaAtomSM90TypeGetANeg(self.ptr()); } - pub fn getBNeg(self: *const Self) bool { + pub fn bNeg(self: *const Self) bool { return c.mlirCuteNVGPUMmaAtomSM90TypeGetBNeg(self.ptr()); } }; @@ -2681,11 +2579,9 @@ pub const SmemDescCircularSM103Type = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDescCircularSM103; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "sm103.smem_desc_circular"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -2696,7 +2592,7 @@ pub const SmemDescCircularSM103Type = opaque { const result = c.mlirCuteNVGPUSmemDescCircularSM103TypeGet(ctx.ptr(), args.blockLayoutBytes.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getBlockLayoutBytes(self: *const Self) *const mlir.Attribute { + pub fn blockLayoutBytes(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUSmemDescCircularSM103TypeGetBlockLayoutBytes(self.ptr()).ptr.?); } }; @@ -2708,11 +2604,9 @@ pub const SmemDescCircularSM107Type = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDescCircularSM107; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "sm107.smem_desc_circular"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -2723,7 +2617,7 @@ pub const SmemDescCircularSM107Type = opaque { const result = c.mlirCuteNVGPUSmemDescCircularSM107TypeGet(ctx.ptr(), args.blockLayoutBytes.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getBlockLayoutBytes(self: *const Self) *const mlir.Attribute { + pub fn blockLayoutBytes(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUSmemDescCircularSM107TypeGetBlockLayoutBytes(self.ptr()).ptr.?); } }; @@ -2735,11 +2629,9 @@ pub const SmemDescSM107Type = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDescSM107; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "sm107.smem_desc"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -2758,11 +2650,9 @@ pub const SmemDescType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDesc; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "smem_desc"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -2781,11 +2671,9 @@ pub const SmemDescViewType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDescView; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "smem_desc_view"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -2797,10 +2685,10 @@ pub const SmemDescViewType = opaque { const result = c.mlirCuteNVGPUSmemDescViewTypeGet(ctx.ptr(), args.desc.ptr(), args.layout.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getDesc(self: *const Self) *const mlir.Type { + pub fn desc(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUSmemDescViewTypeGetDesc(self.ptr()).ptr.?); } - pub fn getLayout(self: *const Self) *const mlir.Attribute { + pub fn layout(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUSmemDescViewTypeGetLayout(self.ptr()).ptr.?); } }; @@ -2812,11 +2700,9 @@ pub const TmaDescriptorIm2ColType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUTmaDescriptorIm2Col; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "tma_descriptor_im2col"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -2835,11 +2721,9 @@ pub const TmaDescriptorTiledType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUTmaDescriptorTiled; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "tma_descriptor_tiled"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -2858,11 +2742,9 @@ pub const UniversalFmaAtomType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUUniversalFmaAtom; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "atom.universal_fma"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -2876,16 +2758,16 @@ pub const UniversalFmaAtomType = opaque { const result = c.mlirCuteNVGPUUniversalFmaAtomTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getShapeMnk(self: *const Self) *const mlir.Attribute { + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirCuteNVGPUUniversalFmaAtomTypeGetShapeMnk(self.ptr()).ptr.?); } - pub fn getAType(self: *const Self) *const mlir.Type { + pub fn aType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUUniversalFmaAtomTypeGetAType(self.ptr()).ptr.?); } - pub fn getBType(self: *const Self) *const mlir.Type { + pub fn bType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUUniversalFmaAtomTypeGetBType(self.ptr()).ptr.?); } - pub fn getCType(self: *const Self) *const mlir.Type { + pub fn cType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUUniversalFmaAtomTypeGetCType(self.ptr()).ptr.?); } }; @@ -2897,11 +2779,9 @@ pub const WorkIdResponseType = opaque { pub const isAFn = c.mlirTypeIsACuteNVGPUWorkIdResponse; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "workid_response"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -2920,11 +2800,9 @@ pub const AtomCopyFieldBulkCopyG2SAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyG2S; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "atom_copy_field_bulkg2s"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2935,7 +2813,7 @@ pub const AtomCopyFieldBulkCopyG2SAttr = opaque { const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) AtomCopyFieldBulkCopyG2S { + pub fn value(self: *const Self) AtomCopyFieldBulkCopyG2S { return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGetValue(self.ptr()))); } }; @@ -2947,11 +2825,9 @@ pub const AtomCopyFieldBulkCopyS2GAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2G; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "atom_copy_field_bulks2g"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2962,7 +2838,7 @@ pub const AtomCopyFieldBulkCopyS2GAttr = opaque { const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) AtomCopyFieldBulkCopyS2G { + pub fn value(self: *const Self) AtomCopyFieldBulkCopyS2G { return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGetValue(self.ptr()))); } }; @@ -2974,11 +2850,9 @@ pub const AtomCopyFieldBulkCopyS2SAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2S; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "atom_copy_field_bulks2s"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2989,7 +2863,7 @@ pub const AtomCopyFieldBulkCopyS2SAttr = opaque { const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) AtomCopyFieldBulkCopyS2S { + pub fn value(self: *const Self) AtomCopyFieldBulkCopyS2S { return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGetValue(self.ptr()))); } }; @@ -3001,11 +2875,9 @@ pub const AtomCopyFieldDsmemStoreAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldDsmemStore; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "atom_copy_field_dsmem_store"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3016,7 +2888,7 @@ pub const AtomCopyFieldDsmemStoreAttr = opaque { const result = c.mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) AtomCopyFieldDsmemStore { + pub fn value(self: *const Self) AtomCopyFieldDsmemStore { return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGetValue(self.ptr()))); } }; @@ -3028,11 +2900,9 @@ pub const AtomCopyFieldLoadGlobalAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldLoadGlobal; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "atom_copy_field_g2r"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3043,7 +2913,7 @@ pub const AtomCopyFieldLoadGlobalAttr = opaque { const result = c.mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) AtomCopyFieldLoadGlobal { + pub fn value(self: *const Self) AtomCopyFieldLoadGlobal { return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGetValue(self.ptr()))); } }; @@ -3055,11 +2925,9 @@ pub const AtomCopyFieldNonExec2DGather4TmaLoadAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoad; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "atom_copy_field_non_exec_2d_gather4_tma_load"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3070,7 +2938,7 @@ pub const AtomCopyFieldNonExec2DGather4TmaLoadAttr = opaque { const result = c.mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) AtomCopyFieldNonExec2DGather4TmaLoad { + pub fn value(self: *const Self) AtomCopyFieldNonExec2DGather4TmaLoad { return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGetValue(self.ptr()))); } }; @@ -3082,11 +2950,9 @@ pub const AtomCopyFieldNonExec2DScatter4TmaStoreAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStore; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "atom_copy_field_non_exec_2d_scatter4_tma_store"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3097,7 +2963,7 @@ pub const AtomCopyFieldNonExec2DScatter4TmaStoreAttr = opaque { const result = c.mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) AtomCopyFieldNonExec2DScatter4TmaStore { + pub fn value(self: *const Self) AtomCopyFieldNonExec2DScatter4TmaStore { return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGetValue(self.ptr()))); } }; @@ -3109,11 +2975,9 @@ pub const AtomCopyFieldNonExecIm2ColTmaLoadAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoad; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "atom_copy_field_non_exec_im2col_tma_load"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3124,7 +2988,7 @@ pub const AtomCopyFieldNonExecIm2ColTmaLoadAttr = opaque { const result = c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) AtomCopyFieldNonExecIm2ColTmaLoad { + pub fn value(self: *const Self) AtomCopyFieldNonExecIm2ColTmaLoad { return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGetValue(self.ptr()))); } }; @@ -3136,11 +3000,9 @@ pub const AtomCopyFieldNonExecIm2ColTmaStoreAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaStore; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "atom_copy_field_non_exec_im2col_tma_store"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3151,7 +3013,7 @@ pub const AtomCopyFieldNonExecIm2ColTmaStoreAttr = opaque { const result = c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) AtomCopyFieldNonExecIm2ColTmaStore { + pub fn value(self: *const Self) AtomCopyFieldNonExecIm2ColTmaStore { return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGetValue(self.ptr()))); } }; @@ -3163,11 +3025,9 @@ pub const AtomCopyFieldNonExecTiledTmaLoadAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaLoad; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "atom_copy_field_non_exec_tma_load"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3178,7 +3038,7 @@ pub const AtomCopyFieldNonExecTiledTmaLoadAttr = opaque { const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) AtomCopyFieldNonExecTiledTmaLoad { + pub fn value(self: *const Self) AtomCopyFieldNonExecTiledTmaLoad { return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGetValue(self.ptr()))); } }; @@ -3190,11 +3050,9 @@ pub const AtomCopyFieldNonExecTiledTmaReduceAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaReduce; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "atom_copy_field_non_exec_tma_reduce"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3205,7 +3063,7 @@ pub const AtomCopyFieldNonExecTiledTmaReduceAttr = opaque { const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) AtomCopyFieldNonExecTiledTmaReduce { + pub fn value(self: *const Self) AtomCopyFieldNonExecTiledTmaReduce { return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGetValue(self.ptr()))); } }; @@ -3217,11 +3075,9 @@ pub const AtomCopyFieldNonExecTiledTmaStoreAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaStore; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "atom_copy_field_non_exec_tma_store"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3232,7 +3088,7 @@ pub const AtomCopyFieldNonExecTiledTmaStoreAttr = opaque { const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) AtomCopyFieldNonExecTiledTmaStore { + pub fn value(self: *const Self) AtomCopyFieldNonExecTiledTmaStore { return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGetValue(self.ptr()))); } }; @@ -3244,11 +3100,9 @@ pub const AtomCopyFieldStoreGlobalAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldStoreGlobal; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "atom_copy_field_r2g"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3259,7 +3113,7 @@ pub const AtomCopyFieldStoreGlobalAttr = opaque { const result = c.mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) AtomCopyFieldStoreGlobal { + pub fn value(self: *const Self) AtomCopyFieldStoreGlobal { return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGetValue(self.ptr()))); } }; @@ -3271,11 +3125,9 @@ pub const AtomCopyFieldTmaLoadAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldTmaLoad; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "atom_copy_field_tmaload"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3286,7 +3138,7 @@ pub const AtomCopyFieldTmaLoadAttr = opaque { const result = c.mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) AtomCopyFieldTmaLoad { + pub fn value(self: *const Self) AtomCopyFieldTmaLoad { return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGetValue(self.ptr()))); } }; @@ -3298,11 +3150,9 @@ pub const AtomCopyFieldTmaReduceAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldTmaReduce; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "atom_copy_field_tmareduce"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3313,7 +3163,7 @@ pub const AtomCopyFieldTmaReduceAttr = opaque { const result = c.mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) AtomCopyFieldTmaReduce { + pub fn value(self: *const Self) AtomCopyFieldTmaReduce { return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGetValue(self.ptr()))); } }; @@ -3325,11 +3175,9 @@ pub const AtomCopyFieldTmaStoreAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldTmaStore; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "atom_copy_field_tmastore"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3340,7 +3188,7 @@ pub const AtomCopyFieldTmaStoreAttr = opaque { const result = c.mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) AtomCopyFieldTmaStore { + pub fn value(self: *const Self) AtomCopyFieldTmaStore { return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGetValue(self.ptr()))); } }; @@ -3352,11 +3200,9 @@ pub const AtomMmaFieldSM100Attr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM100; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "atom_mma_field_sm100"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3367,7 +3213,7 @@ pub const AtomMmaFieldSM100Attr = opaque { const result = c.mlirCuteNVGPUAtomMmaFieldSM100AttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) AtomMmaFieldSM100 { + pub fn value(self: *const Self) AtomMmaFieldSM100 { return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100AttrGetValue(self.ptr()))); } }; @@ -3379,11 +3225,9 @@ pub const AtomMmaFieldSM100BlockScaledAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaled; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "atom_mma_field_sm100_block_scaled"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3394,7 +3238,7 @@ pub const AtomMmaFieldSM100BlockScaledAttr = opaque { const result = c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) AtomMmaFieldSM100BlockScaled { + pub fn value(self: *const Self) AtomMmaFieldSM100BlockScaled { return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGetValue(self.ptr()))); } }; @@ -3406,11 +3250,9 @@ pub const AtomMmaFieldSM100BlockScaledSparseAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaledSparse; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "atom_mma_field_sm100_block_scaled_sparse"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3421,7 +3263,7 @@ pub const AtomMmaFieldSM100BlockScaledSparseAttr = opaque { const result = c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) AtomMmaFieldSM100BlockScaledSparse { + pub fn value(self: *const Self) AtomMmaFieldSM100BlockScaledSparse { return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGetValue(self.ptr()))); } }; @@ -3433,11 +3275,9 @@ pub const AtomMmaFieldSM100SparseAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM100Sparse; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "atom_mma_field_sm100_sparse"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3448,7 +3288,7 @@ pub const AtomMmaFieldSM100SparseAttr = opaque { const result = c.mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) AtomMmaFieldSM100Sparse { + pub fn value(self: *const Self) AtomMmaFieldSM100Sparse { return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGetValue(self.ptr()))); } }; @@ -3460,11 +3300,9 @@ pub const AtomMmaFieldSM120BlockScaledAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM120BlockScaled; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "atom_mma_field_sm120_block_scaled"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3475,7 +3313,7 @@ pub const AtomMmaFieldSM120BlockScaledAttr = opaque { const result = c.mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) AtomMmaFieldSM120BlockScaled { + pub fn value(self: *const Self) AtomMmaFieldSM120BlockScaled { return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGetValue(self.ptr()))); } }; @@ -3487,11 +3325,9 @@ pub const AtomMmaFieldSM80SparseAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM80Sparse; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "atom_mma_field_sm80_sparse"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3502,7 +3338,7 @@ pub const AtomMmaFieldSM80SparseAttr = opaque { const result = c.mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) AtomMmaFieldSM80Sparse { + pub fn value(self: *const Self) AtomMmaFieldSM80Sparse { return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGetValue(self.ptr()))); } }; @@ -3514,11 +3350,9 @@ pub const AtomMmaFieldSM90Attr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM90; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "atom_mma_field_sm90"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3529,7 +3363,7 @@ pub const AtomMmaFieldSM90Attr = opaque { const result = c.mlirCuteNVGPUAtomMmaFieldSM90AttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) AtomMmaFieldSM90 { + pub fn value(self: *const Self) AtomMmaFieldSM90 { return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM90AttrGetValue(self.ptr()))); } }; @@ -3541,11 +3375,9 @@ pub const BinaryOpAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUBinaryOp; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "bin_op"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3556,7 +3388,7 @@ pub const BinaryOpAttr = opaque { const result = c.mlirCuteNVGPUBinaryOpAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) BinaryOp { + pub fn value(self: *const Self) BinaryOp { return @fromBackingInt(@intCast(c.mlirCuteNVGPUBinaryOpAttrGetValue(self.ptr()))); } }; @@ -3568,11 +3400,9 @@ pub const CopyS2TBroadcastAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUCopyS2TBroadcast; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "copy_s2t_broadcast_mode"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3583,7 +3413,7 @@ pub const CopyS2TBroadcastAttr = opaque { const result = c.mlirCuteNVGPUCopyS2TBroadcastAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) CopyS2TBroadcast { + pub fn value(self: *const Self) CopyS2TBroadcast { return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyS2TBroadcastAttrGetValue(self.ptr()))); } }; @@ -3595,11 +3425,9 @@ pub const GatherScatterTmaLoadAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUGatherScatterTmaLoad; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "gather_scatter_tma_load"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3610,7 +3438,7 @@ pub const GatherScatterTmaLoadAttr = opaque { const result = c.mlirCuteNVGPUGatherScatterTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) GatherScatterTmaLoad { + pub fn value(self: *const Self) GatherScatterTmaLoad { return @fromBackingInt(@intCast(c.mlirCuteNVGPUGatherScatterTmaLoadAttrGetValue(self.ptr()))); } }; @@ -3622,11 +3450,9 @@ pub const Im2ColTmaLoadAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUIm2ColTmaLoad; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "im2col_tma_load"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3637,7 +3463,7 @@ pub const Im2ColTmaLoadAttr = opaque { const result = c.mlirCuteNVGPUIm2ColTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) Im2ColTmaLoad { + pub fn value(self: *const Self) Im2ColTmaLoad { return @fromBackingInt(@intCast(c.mlirCuteNVGPUIm2ColTmaLoadAttrGetValue(self.ptr()))); } }; @@ -3649,11 +3475,9 @@ pub const LdReduceAccPrecisionKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPULdReduceAccPrecisionKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "ld_reduce_acc_precision_kind"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3664,7 +3488,7 @@ pub const LdReduceAccPrecisionKindAttr = opaque { const result = c.mlirCuteNVGPULdReduceAccPrecisionKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) LdReduceAccPrecisionKind { + pub fn value(self: *const Self) LdReduceAccPrecisionKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPULdReduceAccPrecisionKindAttrGetValue(self.ptr()))); } }; @@ -3676,11 +3500,9 @@ pub const LdsmSzPatternAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPULdsmSzPattern; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "ldsm_sz_pattern"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3691,7 +3513,7 @@ pub const LdsmSzPatternAttr = opaque { const result = c.mlirCuteNVGPULdsmSzPatternAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) LdsmSzPattern { + pub fn value(self: *const Self) LdsmSzPattern { return @fromBackingInt(@intCast(c.mlirCuteNVGPULdsmSzPatternAttrGetValue(self.ptr()))); } }; @@ -3703,11 +3525,9 @@ pub const LoadCacheModeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPULoadCacheMode; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "load_cache_mode"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3718,7 +3538,7 @@ pub const LoadCacheModeAttr = opaque { const result = c.mlirCuteNVGPULoadCacheModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) LoadCacheMode { + pub fn value(self: *const Self) LoadCacheMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPULoadCacheModeAttrGetValue(self.ptr()))); } }; @@ -3730,11 +3550,9 @@ pub const MMAIntOverflowAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUMMAIntOverflow; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "mma_int_overflow"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3745,7 +3563,7 @@ pub const MMAIntOverflowAttr = opaque { const result = c.mlirCuteNVGPUMMAIntOverflowAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) MMAIntOverflow { + pub fn value(self: *const Self) MMAIntOverflow { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMMAIntOverflowAttrGetValue(self.ptr()))); } }; @@ -3757,22 +3575,20 @@ pub const MajorModeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUMajorMode; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "major"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } pub const InitArgs = struct { - value: cute.MajorMode, + value: cute_ir.MajorMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { const result = c.mlirCuteNVGPUMajorModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) cute.MajorMode { + pub fn value(self: *const Self) cute_ir.MajorMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMajorModeAttrGetValue(self.ptr()))); } }; @@ -3784,11 +3600,9 @@ pub const MmaCollectorOpAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUMmaCollectorOp; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "mma_collector_op"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3799,7 +3613,7 @@ pub const MmaCollectorOpAttr = opaque { const result = c.mlirCuteNVGPUMmaCollectorOpAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) MmaCollectorOp { + pub fn value(self: *const Self) MmaCollectorOp { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaCollectorOpAttrGetValue(self.ptr()))); } }; @@ -3811,11 +3625,9 @@ pub const MmaFragKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUMmaFragKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "mma_frag_kind"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3826,7 +3638,7 @@ pub const MmaFragKindAttr = opaque { const result = c.mlirCuteNVGPUMmaFragKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) MmaFragKind { + pub fn value(self: *const Self) MmaFragKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaFragKindAttrGetValue(self.ptr()))); } }; @@ -3838,11 +3650,9 @@ pub const NotImplementedFrgAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUNotImplementedFrg; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "not_implemented_frg"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3861,11 +3671,9 @@ pub const ReductionKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUReductionKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tma_reduce_kind"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3876,7 +3684,7 @@ pub const ReductionKindAttr = opaque { const result = c.mlirCuteNVGPUReductionKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) ReductionKind { + pub fn value(self: *const Self) ReductionKind { return @fromBackingInt(@intCast(c.mlirCuteNVGPUReductionKindAttrGetValue(self.ptr()))); } }; @@ -3888,30 +3696,28 @@ pub const RmemFrgAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPURmemFrg; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "rmem_frg"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } pub const InitArgs = struct { valueType: *const mlir.Type, - operand: cute.MmaOperand, + operand: cute_ir.MmaOperand, deriveElemType: bool = false, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { const result = c.mlirCuteNVGPURmemFrgAttrGet(ctx.ptr(), args.valueType.ptr(), @backingInt(args.operand), args.deriveElemType); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValueType(self: *const Self) *const mlir.Type { + pub fn valueType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPURmemFrgAttrGetValueType(self.ptr()).ptr.?); } - pub fn getOperand(self: *const Self) cute.MmaOperand { + pub fn operand(self: *const Self) cute_ir.MmaOperand { return @fromBackingInt(@intCast(c.mlirCuteNVGPURmemFrgAttrGetOperand(self.ptr()))); } - pub fn getDeriveElemType(self: *const Self) bool { + pub fn deriveElemType(self: *const Self) bool { return c.mlirCuteNVGPURmemFrgAttrGetDeriveElemType(self.ptr()); } }; @@ -3923,22 +3729,20 @@ pub const SM100CircularSmemFrgAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUSM100CircularSmemFrg; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "arch.sm100.circular_smem_frg"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } pub const InitArgs = struct { - majorMode: cute.MajorMode, + majorMode: cute_ir.MajorMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { const result = c.mlirCuteNVGPUSM100CircularSmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getMajorMode(self: *const Self) cute.MajorMode { + pub fn majorMode(self: *const Self) cute_ir.MajorMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100CircularSmemFrgAttrGetMajorMode(self.ptr()))); } }; @@ -3950,22 +3754,20 @@ pub const SM100SmemFrgAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUSM100SmemFrg; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "arch.sm100.smem_frg"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } pub const InitArgs = struct { - majorMode: cute.MajorMode, + majorMode: cute_ir.MajorMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { const result = c.mlirCuteNVGPUSM100SmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getMajorMode(self: *const Self) cute.MajorMode { + pub fn majorMode(self: *const Self) cute_ir.MajorMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100SmemFrgAttrGetMajorMode(self.ptr()))); } }; @@ -3977,11 +3779,9 @@ pub const SM100TmemEFrgAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUSM100TmemEFrg; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "arch.sm100.tmem_e_frg"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3993,10 +3793,10 @@ pub const SM100TmemEFrgAttr = opaque { const result = c.mlirCuteNVGPUSM100TmemEFrgAttrGet(ctx.ptr(), args.aType.ptr(), args.eType.ptr()); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getAType(self: *const Self) *const mlir.Type { + pub fn aType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUSM100TmemEFrgAttrGetAType(self.ptr()).ptr.?); } - pub fn getEType(self: *const Self) *const mlir.Type { + pub fn eType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUSM100TmemEFrgAttrGetEType(self.ptr()).ptr.?); } }; @@ -4008,11 +3808,9 @@ pub const SM100TmemFrgAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUSM100TmemFrg; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "arch.sm100.tmem_frg"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -4026,16 +3824,16 @@ pub const SM100TmemFrgAttr = opaque { const result = c.mlirCuteNVGPUSM100TmemFrgAttrGet(ctx.ptr(), args.dataType.ptr(), args.storageType.ptr(), args.ctaGroup, @backingInt(args.tmemAllocMode)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getDataType(self: *const Self) *const mlir.Type { + pub fn dataType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUSM100TmemFrgAttrGetDataType(self.ptr()).ptr.?); } - pub fn getStorageType(self: *const Self) *const mlir.Type { + pub fn storageType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUSM100TmemFrgAttrGetStorageType(self.ptr()).ptr.?); } - pub fn getCtaGroup(self: *const Self) c_int { + pub fn ctaGroup(self: *const Self) c_int { return c.mlirCuteNVGPUSM100TmemFrgAttrGetCtaGroup(self.ptr()); } - pub fn getTmemAllocMode(self: *const Self) TmemAllocMode { + pub fn tmemAllocMode(self: *const Self) TmemAllocMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100TmemFrgAttrGetTmemAllocMode(self.ptr()))); } }; @@ -4047,11 +3845,9 @@ pub const SM100TmemSfFrgAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUSM100TmemSfFrg; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "arch.sm100.tmem_sf_frg"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -4066,19 +3862,19 @@ pub const SM100TmemSfFrgAttr = opaque { const result = c.mlirCuteNVGPUSM100TmemSfFrgAttrGet(ctx.ptr(), args.sfType.ptr(), args.sfVecSize, args.ctaGroup, args.isSfa, @backingInt(args.tmemAllocMode)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getSfType(self: *const Self) *const mlir.Type { + pub fn sfType(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirCuteNVGPUSM100TmemSfFrgAttrGetSfType(self.ptr()).ptr.?); } - pub fn getSfVecSize(self: *const Self) c_int { + pub fn sfVecSize(self: *const Self) c_int { return c.mlirCuteNVGPUSM100TmemSfFrgAttrGetSfVecSize(self.ptr()); } - pub fn getCtaGroup(self: *const Self) c_int { + pub fn ctaGroup(self: *const Self) c_int { return c.mlirCuteNVGPUSM100TmemSfFrgAttrGetCtaGroup(self.ptr()); } - pub fn getIsSfa(self: *const Self) bool { + pub fn isSfa(self: *const Self) bool { return c.mlirCuteNVGPUSM100TmemSfFrgAttrGetIsSfa(self.ptr()); } - pub fn getTmemAllocMode(self: *const Self) TmemAllocMode { + pub fn tmemAllocMode(self: *const Self) TmemAllocMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100TmemSfFrgAttrGetTmemAllocMode(self.ptr()))); } }; @@ -4090,22 +3886,20 @@ pub const SM107SmemFrgAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUSM107SmemFrg; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "arch.sm107.smem_frg"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } pub const InitArgs = struct { - majorMode: cute.MajorMode, + majorMode: cute_ir.MajorMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { const result = c.mlirCuteNVGPUSM107SmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getMajorMode(self: *const Self) cute.MajorMode { + pub fn majorMode(self: *const Self) cute_ir.MajorMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM107SmemFrgAttrGetMajorMode(self.ptr()))); } }; @@ -4117,22 +3911,20 @@ pub const SM90SmemFrgAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUSM90SmemFrg; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "arch.sm90.smem_frg"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } pub const InitArgs = struct { - majorMode: cute.MajorMode, + majorMode: cute_ir.MajorMode, }; pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { const result = c.mlirCuteNVGPUSM90SmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getMajorMode(self: *const Self) cute.MajorMode { + pub fn majorMode(self: *const Self) cute_ir.MajorMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM90SmemFrgAttrGetMajorMode(self.ptr()))); } }; @@ -4144,11 +3936,9 @@ pub const TiledTmaLoadAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUTiledTmaLoad; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tiled_tma_load"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -4159,7 +3949,7 @@ pub const TiledTmaLoadAttr = opaque { const result = c.mlirCuteNVGPUTiledTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) TiledTmaLoad { + pub fn value(self: *const Self) TiledTmaLoad { return @fromBackingInt(@intCast(c.mlirCuteNVGPUTiledTmaLoadAttrGetValue(self.ptr()))); } }; @@ -4171,11 +3961,9 @@ pub const TmaDataFormatAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUTmaDataFormat; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tma_data_format"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -4186,7 +3974,7 @@ pub const TmaDataFormatAttr = opaque { const result = c.mlirCuteNVGPUTmaDataFormatAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) TmaDataFormat { + pub fn value(self: *const Self) TmaDataFormat { return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmaDataFormatAttrGetValue(self.ptr()))); } }; @@ -4198,11 +3986,9 @@ pub const TmaLoadModeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUTmaLoadMode; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tma_load_mode"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -4213,7 +3999,7 @@ pub const TmaLoadModeAttr = opaque { const result = c.mlirCuteNVGPUTmaLoadModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) TmaLoadMode { + pub fn value(self: *const Self) TmaLoadMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmaLoadModeAttrGetValue(self.ptr()))); } }; @@ -4225,11 +4011,9 @@ pub const TmaStoreModeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUTmaStoreMode; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tma_store_mode"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -4240,7 +4024,7 @@ pub const TmaStoreModeAttr = opaque { const result = c.mlirCuteNVGPUTmaStoreModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) TmaStoreMode { + pub fn value(self: *const Self) TmaStoreMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmaStoreModeAttrGetValue(self.ptr()))); } }; @@ -4252,11 +4036,9 @@ pub const TmemAllocModeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUTmemAllocMode; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tmem_alloc_mode"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -4267,7 +4049,7 @@ pub const TmemAllocModeAttr = opaque { const result = c.mlirCuteNVGPUTmemAllocModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) TmemAllocMode { + pub fn value(self: *const Self) TmemAllocMode { return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmemAllocModeAttrGetValue(self.ptr()))); } }; @@ -4279,11 +4061,9 @@ pub const TmemLoadRedOpAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVGPUTmemLoadRedOp; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tmem_load_red_op"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -4294,7 +4074,7 @@ pub const TmemLoadRedOpAttr = opaque { const result = c.mlirCuteNVGPUTmemLoadRedOpAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) TmemLoadRedOp { + pub fn value(self: *const Self) TmemLoadRedOp { return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmemLoadRedOpAttrGetValue(self.ptr()))); } }; @@ -4335,16 +4115,30 @@ pub fn arch_copy_SM100_copy_s2t(ctx: *mlir.Context, src: *const mlir.Value, dst: } /// `cute_nvgpu.arch.copy.SM100.tma_load`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_copy_SM100_tma_load(ctx: *mlir.Context, src_desc: *const mlir.Value, dsmem_data_addr: *const mlir.Value, dsmem_bar_addr: *const mlir.Value, coord: []const *const mlir.Value, multicast_mask: ?*const mlir.Value, offsets: []const *const mlir.Value, cache_policy: ?*const mlir.Value, mode: *const mlir.Attribute, num_cta: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn arch_copy_SM100_tma_load( + ctx: *mlir.Context, + src_desc: *const mlir.Value, + dsmem_data_addr: *const mlir.Value, + dsmem_bar_addr: *const mlir.Value, + coord: []const *const mlir.Value, + offsets: []const *const mlir.Value, + mode: *const mlir.Attribute, + num_cta: *const mlir.Attribute, + opts: struct { + multicast_mask: ?*const mlir.Value = null, + cache_policy: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.tma_load", .{ .operands = .{ .variadic = &.{ &.{src_desc}, &.{dsmem_data_addr}, &.{dsmem_bar_addr}, coord, - if (multicast_mask) |value| &.{value} else &.{}, + if (opts.multicast_mask) |value| &.{value} else &.{}, offsets, - if (cache_policy) |value| &.{value} else &.{}, + if (opts.cache_policy) |value| &.{value} else &.{}, } }, .attributes = &.{ .named(ctx, "mode", mode), @@ -4355,13 +4149,24 @@ pub fn arch_copy_SM100_tma_load(ctx: *mlir.Context, src_desc: *const mlir.Value, } /// `cute_nvgpu.arch.copy.SM100.tma_reduce`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_copy_SM100_tma_reduce(ctx: *mlir.Context, dst_desc: *const mlir.Value, src_smem_data_addr: *const mlir.Value, coord: []const *const mlir.Value, cache_policy: ?*const mlir.Value, mode: *const mlir.Attribute, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn arch_copy_SM100_tma_reduce( + ctx: *mlir.Context, + dst_desc: *const mlir.Value, + src_smem_data_addr: *const mlir.Value, + coord: []const *const mlir.Value, + mode: *const mlir.Attribute, + kind: *const mlir.Attribute, + opts: struct { + cache_policy: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.tma_reduce", .{ .operands = .{ .variadic = &.{ &.{dst_desc}, &.{src_smem_data_addr}, coord, - if (cache_policy) |value| &.{value} else &.{}, + if (opts.cache_policy) |value| &.{value} else &.{}, } }, .attributes = &.{ .named(ctx, "mode", mode), @@ -4372,13 +4177,23 @@ pub fn arch_copy_SM100_tma_reduce(ctx: *mlir.Context, dst_desc: *const mlir.Valu } /// `cute_nvgpu.arch.copy.SM100.tma_store`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_copy_SM100_tma_store(ctx: *mlir.Context, dst_desc: *const mlir.Value, src_smem_data_addr: *const mlir.Value, coord: []const *const mlir.Value, cache_policy: ?*const mlir.Value, mode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn arch_copy_SM100_tma_store( + ctx: *mlir.Context, + dst_desc: *const mlir.Value, + src_smem_data_addr: *const mlir.Value, + coord: []const *const mlir.Value, + mode: *const mlir.Attribute, + opts: struct { + cache_policy: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.tma_store", .{ .operands = .{ .variadic = &.{ &.{dst_desc}, &.{src_smem_data_addr}, coord, - if (cache_policy) |value| &.{value} else &.{}, + if (opts.cache_policy) |value| &.{value} else &.{}, } }, .attributes = &.{ .named(ctx, "mode", mode), @@ -4388,12 +4203,23 @@ pub fn arch_copy_SM100_tma_store(ctx: *mlir.Context, dst_desc: *const mlir.Value } /// `cute_nvgpu.arch.copy.SM100.tmem_load`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_copy_SM100_tmem_load(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, num_dp: *const mlir.Attribute, num_b: *const mlir.Attribute, num_rep: *const mlir.Attribute, pack_16: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn arch_copy_SM100_tmem_load( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + num_dp: *const mlir.Attribute, + num_b: *const mlir.Attribute, + num_rep: *const mlir.Attribute, + opts: struct { + pack_16: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; attributes.appendAssumeCapacity(.named(ctx, "num_dp", num_dp)); attributes.appendAssumeCapacity(.named(ctx, "num_b", num_b)); attributes.appendAssumeCapacity(.named(ctx, "num_rep", num_rep)); - if (pack_16) |value| attributes.appendAssumeCapacity(.named(ctx, "pack_16", value)); + if (opts.pack_16) |value| attributes.appendAssumeCapacity(.named(ctx, "pack_16", value)); return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.tmem_load", .{ .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{dst_type} }, @@ -4403,12 +4229,23 @@ pub fn arch_copy_SM100_tmem_load(ctx: *mlir.Context, src: *const mlir.Value, dst } /// `cute_nvgpu.arch.copy.SM100.tmem_store`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_copy_SM100_tmem_store(ctx: *mlir.Context, dst: *const mlir.Value, src: *const mlir.Value, num_dp: *const mlir.Attribute, num_b: *const mlir.Attribute, num_rep: *const mlir.Attribute, expand_16: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn arch_copy_SM100_tmem_store( + ctx: *mlir.Context, + dst: *const mlir.Value, + src: *const mlir.Value, + num_dp: *const mlir.Attribute, + num_b: *const mlir.Attribute, + num_rep: *const mlir.Attribute, + opts: struct { + expand_16: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; attributes.appendAssumeCapacity(.named(ctx, "num_dp", num_dp)); attributes.appendAssumeCapacity(.named(ctx, "num_b", num_b)); attributes.appendAssumeCapacity(.named(ctx, "num_rep", num_rep)); - if (expand_16) |value| attributes.appendAssumeCapacity(.named(ctx, "expand_16", value)); + if (opts.expand_16) |value| attributes.appendAssumeCapacity(.named(ctx, "expand_16", value)); return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.tmem_store", .{ .operands = .{ .flat = &.{ dst, src } }, .attributes = attributes.constSlice(), @@ -4417,7 +4254,24 @@ pub fn arch_copy_SM100_tmem_store(ctx: *mlir.Context, dst: *const mlir.Value, sr } /// `cute_nvgpu.arch.copy.SM107.tma_load_override`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_copy_SM107_tma_load_override(ctx: *mlir.Context, src_desc: *const mlir.Value, dsmem_data_addr: *const mlir.Value, dsmem_bar_addr: *const mlir.Value, override_addr: *const mlir.Value, tensor_size: []const *const mlir.Value, lower_stride: []const *const mlir.Value, upper_stride: ?*const mlir.Value, coord: []const *const mlir.Value, multicast_mask: ?*const mlir.Value, cache_policy: ?*const mlir.Value, mode: *const mlir.Attribute, num_cta: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn arch_copy_SM107_tma_load_override( + ctx: *mlir.Context, + src_desc: *const mlir.Value, + dsmem_data_addr: *const mlir.Value, + dsmem_bar_addr: *const mlir.Value, + override_addr: *const mlir.Value, + tensor_size: []const *const mlir.Value, + lower_stride: []const *const mlir.Value, + coord: []const *const mlir.Value, + mode: *const mlir.Attribute, + num_cta: *const mlir.Attribute, + opts: struct { + upper_stride: ?*const mlir.Value = null, + multicast_mask: ?*const mlir.Value = null, + cache_policy: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM107.tma_load_override", .{ .operands = .{ .variadic = &.{ &.{src_desc}, @@ -4426,10 +4280,10 @@ pub fn arch_copy_SM107_tma_load_override(ctx: *mlir.Context, src_desc: *const ml &.{override_addr}, tensor_size, lower_stride, - if (upper_stride) |value| &.{value} else &.{}, + if (opts.upper_stride) |value| &.{value} else &.{}, coord, - if (multicast_mask) |value| &.{value} else &.{}, - if (cache_policy) |value| &.{value} else &.{}, + if (opts.multicast_mask) |value| &.{value} else &.{}, + if (opts.cache_policy) |value| &.{value} else &.{}, } }, .attributes = &.{ .named(ctx, "mode", mode), @@ -4440,7 +4294,21 @@ pub fn arch_copy_SM107_tma_load_override(ctx: *mlir.Context, src_desc: *const ml } /// `cute_nvgpu.arch.copy.SM107.tma_store_override`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_copy_SM107_tma_store_override(ctx: *mlir.Context, dst_desc: *const mlir.Value, src_smem_data_addr: *const mlir.Value, override_addr: *const mlir.Value, tensor_size: []const *const mlir.Value, lower_stride: []const *const mlir.Value, upper_stride: ?*const mlir.Value, coord: []const *const mlir.Value, cache_policy: ?*const mlir.Value, mode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn arch_copy_SM107_tma_store_override( + ctx: *mlir.Context, + dst_desc: *const mlir.Value, + src_smem_data_addr: *const mlir.Value, + override_addr: *const mlir.Value, + tensor_size: []const *const mlir.Value, + lower_stride: []const *const mlir.Value, + coord: []const *const mlir.Value, + mode: *const mlir.Attribute, + opts: struct { + upper_stride: ?*const mlir.Value = null, + cache_policy: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM107.tma_store_override", .{ .operands = .{ .variadic = &.{ &.{dst_desc}, @@ -4448,9 +4316,9 @@ pub fn arch_copy_SM107_tma_store_override(ctx: *mlir.Context, dst_desc: *const m &.{override_addr}, tensor_size, lower_stride, - if (upper_stride) |value| &.{value} else &.{}, + if (opts.upper_stride) |value| &.{value} else &.{}, coord, - if (cache_policy) |value| &.{value} else &.{}, + if (opts.cache_policy) |value| &.{value} else &.{}, } }, .attributes = &.{ .named(ctx, "mode", mode), @@ -4460,14 +4328,25 @@ pub fn arch_copy_SM107_tma_store_override(ctx: *mlir.Context, dst_desc: *const m } /// `cute_nvgpu.arch.copy.SM90.bulk_copy_g2s`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_copy_SM90_bulk_copy_g2s(ctx: *mlir.Context, gmem_data_addr: *const mlir.Value, dsmem_data_addr: *const mlir.Value, dsmem_bar_addr: *const mlir.Value, multicast_mask: ?*const mlir.Value, cache_policy: ?*const mlir.Value, size: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn arch_copy_SM90_bulk_copy_g2s( + ctx: *mlir.Context, + gmem_data_addr: *const mlir.Value, + dsmem_data_addr: *const mlir.Value, + dsmem_bar_addr: *const mlir.Value, + size: *const mlir.Attribute, + opts: struct { + multicast_mask: ?*const mlir.Value = null, + cache_policy: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM90.bulk_copy_g2s", .{ .operands = .{ .variadic = &.{ &.{gmem_data_addr}, &.{dsmem_data_addr}, &.{dsmem_bar_addr}, - if (multicast_mask) |value| &.{value} else &.{}, - if (cache_policy) |value| &.{value} else &.{}, + if (opts.multicast_mask) |value| &.{value} else &.{}, + if (opts.cache_policy) |value| &.{value} else &.{}, } }, .attributes = &.{ .named(ctx, "size", size), @@ -4477,13 +4356,23 @@ pub fn arch_copy_SM90_bulk_copy_g2s(ctx: *mlir.Context, gmem_data_addr: *const m } /// `cute_nvgpu.arch.copy.SM90.bulk_copy_s2g`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_copy_SM90_bulk_copy_s2g(ctx: *mlir.Context, smem_data_addr: *const mlir.Value, gmem_data_addr: *const mlir.Value, byte_mask: ?*const mlir.Value, cache_policy: ?*const mlir.Value, size: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn arch_copy_SM90_bulk_copy_s2g( + ctx: *mlir.Context, + smem_data_addr: *const mlir.Value, + gmem_data_addr: *const mlir.Value, + size: *const mlir.Attribute, + opts: struct { + byte_mask: ?*const mlir.Value = null, + cache_policy: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM90.bulk_copy_s2g", .{ .operands = .{ .variadic = &.{ &.{smem_data_addr}, &.{gmem_data_addr}, - if (byte_mask) |value| &.{value} else &.{}, - if (cache_policy) |value| &.{value} else &.{}, + if (opts.byte_mask) |value| &.{value} else &.{}, + if (opts.cache_policy) |value| &.{value} else &.{}, } }, .attributes = &.{ .named(ctx, "size", size), @@ -4597,12 +4486,32 @@ pub fn arch_mma_SM107_umma_sparse(ctx: *mlir.Context, a: *const mlir.Value, b: * } /// `cute_nvgpu.arch.mma.SM120.block_scaled`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_mma_SM120_block_scaled(ctx: *mlir.Context, opA: []const *const mlir.Value, opB: []const *const mlir.Value, opC: []const *const mlir.Value, sfA: *const mlir.Value, sfB: *const mlir.Value, byte_id_a: ?*const mlir.Value, byte_id_b: ?*const mlir.Value, res_types: []const *const mlir.Type, shape_MNK: *const mlir.Attribute, vec_size: *const mlir.Attribute, thread_id_a: ?*const mlir.Attribute, thread_id_b: ?*const mlir.Attribute, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, sf_type: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn arch_mma_SM120_block_scaled( + ctx: *mlir.Context, + opA: []const *const mlir.Value, + opB: []const *const mlir.Value, + opC: []const *const mlir.Value, + sfA: *const mlir.Value, + sfB: *const mlir.Value, + res_types: []const *const mlir.Type, + shape_MNK: *const mlir.Attribute, + vec_size: *const mlir.Attribute, + a_type: *const mlir.Attribute, + b_type: *const mlir.Attribute, + sf_type: *const mlir.Attribute, + opts: struct { + byte_id_a: ?*const mlir.Value = null, + byte_id_b: ?*const mlir.Value = null, + thread_id_a: ?*const mlir.Attribute = null, + thread_id_b: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; attributes.appendAssumeCapacity(.named(ctx, "shape_MNK", shape_MNK)); attributes.appendAssumeCapacity(.named(ctx, "vec_size", vec_size)); - if (thread_id_a) |value| attributes.appendAssumeCapacity(.named(ctx, "thread_id_a", value)); - if (thread_id_b) |value| attributes.appendAssumeCapacity(.named(ctx, "thread_id_b", value)); + if (opts.thread_id_a) |value| attributes.appendAssumeCapacity(.named(ctx, "thread_id_a", value)); + if (opts.thread_id_b) |value| attributes.appendAssumeCapacity(.named(ctx, "thread_id_b", value)); attributes.appendAssumeCapacity(.named(ctx, "a_type", a_type)); attributes.appendAssumeCapacity(.named(ctx, "b_type", b_type)); attributes.appendAssumeCapacity(.named(ctx, "sf_type", sf_type)); @@ -4613,8 +4522,8 @@ pub fn arch_mma_SM120_block_scaled(ctx: *mlir.Context, opA: []const *const mlir. opC, &.{sfA}, &.{sfB}, - if (byte_id_a) |value| &.{value} else &.{}, - if (byte_id_b) |value| &.{value} else &.{}, + if (opts.byte_id_a) |value| &.{value} else &.{}, + if (opts.byte_id_b) |value| &.{value} else &.{}, } }, .results = .{ .variadic = &.{ res_types, @@ -4633,10 +4542,19 @@ pub fn arch_prefetch_tma_desc(ctx: *mlir.Context, tma_descriptor_ptr: *const mli } /// `cute_nvgpu.arch.sm100.alloc_tmem`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_sm100_alloc_tmem(ctx: *mlir.Context, n_cols: *const mlir.Value, dst_ptr: *const mlir.Value, is_two_cta: ?*const mlir.Attribute, exclusive: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn arch_sm100_alloc_tmem( + ctx: *mlir.Context, + n_cols: *const mlir.Value, + dst_ptr: *const mlir.Value, + opts: struct { + is_two_cta: ?*const mlir.Attribute = null, + exclusive: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (is_two_cta) |value| attributes.appendAssumeCapacity(.named(ctx, "is_two_cta", value)); - if (exclusive) |value| attributes.appendAssumeCapacity(.named(ctx, "exclusive", value)); + if (opts.is_two_cta) |value| attributes.appendAssumeCapacity(.named(ctx, "is_two_cta", value)); + if (opts.exclusive) |value| attributes.appendAssumeCapacity(.named(ctx, "exclusive", value)); return mlir.Operation.make(ctx, "cute_nvgpu.arch.sm100.alloc_tmem", .{ .operands = .{ .flat = &.{ n_cols, dst_ptr } }, .attributes = attributes.constSlice(), @@ -4645,10 +4563,19 @@ pub fn arch_sm100_alloc_tmem(ctx: *mlir.Context, n_cols: *const mlir.Value, dst_ } /// `cute_nvgpu.arch.sm100.dealloc_tmem`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_sm100_dealloc_tmem(ctx: *mlir.Context, tmem_ptr: *const mlir.Value, n_cols: *const mlir.Value, is_two_cta: ?*const mlir.Attribute, exclusive: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn arch_sm100_dealloc_tmem( + ctx: *mlir.Context, + tmem_ptr: *const mlir.Value, + n_cols: *const mlir.Value, + opts: struct { + is_two_cta: ?*const mlir.Attribute = null, + exclusive: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (is_two_cta) |value| attributes.appendAssumeCapacity(.named(ctx, "is_two_cta", value)); - if (exclusive) |value| attributes.appendAssumeCapacity(.named(ctx, "exclusive", value)); + if (opts.is_two_cta) |value| attributes.appendAssumeCapacity(.named(ctx, "is_two_cta", value)); + if (opts.exclusive) |value| attributes.appendAssumeCapacity(.named(ctx, "exclusive", value)); return mlir.Operation.make(ctx, "cute_nvgpu.arch.sm100.dealloc_tmem", .{ .operands = .{ .flat = &.{ tmem_ptr, n_cols } }, .attributes = attributes.constSlice(), @@ -4657,9 +4584,15 @@ pub fn arch_sm100_dealloc_tmem(ctx: *mlir.Context, tmem_ptr: *const mlir.Value, } /// `cute_nvgpu.arch.sm100.relinquish_tmem_alloc_permit`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_sm100_relinquish_tmem_alloc_permit(ctx: *mlir.Context, is_two_cta: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn arch_sm100_relinquish_tmem_alloc_permit( + ctx: *mlir.Context, + opts: struct { + is_two_cta: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (is_two_cta) |value| attributes.appendAssumeCapacity(.named(ctx, "is_two_cta", value)); + if (opts.is_two_cta) |value| attributes.appendAssumeCapacity(.named(ctx, "is_two_cta", value)); return mlir.Operation.make(ctx, "cute_nvgpu.arch.sm100.relinquish_tmem_alloc_permit", .{ .attributes = attributes.constSlice(), .location = location, @@ -4676,13 +4609,23 @@ pub fn arch_sm100_retrieve_tmem_ptr(ctx: *mlir.Context, smem_ptr: *const mlir.Va } /// `cute_nvgpu.atom.get_coord_tensor`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_get_coord_tensor(ctx: *mlir.Context, atom: *const mlir.Value, shape: *const mlir.Value, offset_b: ?*const mlir.Value, layout_b: ?*const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +pub fn atom_get_coord_tensor( + ctx: *mlir.Context, + atom: *const mlir.Value, + shape: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + offset_b: ?*const mlir.Value = null, + layout_b: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cute_nvgpu.atom.get_coord_tensor", .{ .operands = .{ .variadic = &.{ &.{atom}, &.{shape}, - if (offset_b) |value| &.{value} else &.{}, - if (layout_b) |value| &.{value} else &.{}, + if (opts.offset_b) |value| &.{value} else &.{}, + if (opts.layout_b) |value| &.{value} else &.{}, } }, .results = .{ .flat = &.{result_type} }, .location = location, @@ -4711,10 +4654,19 @@ pub fn atom_get_value(ctx: *mlir.Context, atom: *const mlir.Value, result_type: } /// `cute_nvgpu.atom.make_exec_tma`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_make_exec_tma(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, override: ?*const mlir.Attribute, no_fully_oob_tile: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn atom_make_exec_tma( + ctx: *mlir.Context, + input: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + override: ?*const mlir.Attribute = null, + no_fully_oob_tile: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (override) |value| attributes.appendAssumeCapacity(.named(ctx, "override", value)); - if (no_fully_oob_tile) |value| attributes.appendAssumeCapacity(.named(ctx, "no_fully_oob_tile", value)); + if (opts.override) |value| attributes.appendAssumeCapacity(.named(ctx, "override", value)); + if (opts.no_fully_oob_tile) |value| attributes.appendAssumeCapacity(.named(ctx, "no_fully_oob_tile", value)); return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_exec_tma", .{ .operands = .{ .flat = &.{input} }, .results = .{ .flat = &.{result_type} }, @@ -4724,11 +4676,25 @@ pub fn atom_make_exec_tma(ctx: *mlir.Context, input: *const mlir.Value, result_t } /// `cute_nvgpu.atom.make_non_exec_2d_gather4_tma_load`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_make_non_exec_2d_gather4_tma_load(ctx: *mlir.Context, gmem_tensor: *const mlir.Value, gmem_coord_layout: *const mlir.Value, smem_layout: *const mlir.Value, cta_v_map: *const mlir.Value, non_exec_atom_type: *const mlir.Type, tma_tensor_type: *const mlir.Type, kind: *const mlir.Attribute, num_multicast: ?*const mlir.Attribute, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn atom_make_non_exec_2d_gather4_tma_load( + ctx: *mlir.Context, + gmem_tensor: *const mlir.Value, + gmem_coord_layout: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_v_map: *const mlir.Value, + non_exec_atom_type: *const mlir.Type, + tma_tensor_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + num_multicast: ?*const mlir.Attribute = null, + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); - if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_2d_gather4_tma_load", .{ .operands = .{ .flat = &.{ gmem_tensor, gmem_coord_layout, smem_layout, cta_v_map } }, .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, @@ -4738,9 +4704,21 @@ pub fn atom_make_non_exec_2d_gather4_tma_load(ctx: *mlir.Context, gmem_tensor: * } /// `cute_nvgpu.atom.make_non_exec_2d_scatter4_tma_store`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_make_non_exec_2d_scatter4_tma_store(ctx: *mlir.Context, gmem_tensor: *const mlir.Value, gmem_index_layout: *const mlir.Value, smem_layout: *const mlir.Value, cta_v_map: *const mlir.Value, non_exec_atom_type: *const mlir.Type, tma_tensor_type: *const mlir.Type, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn atom_make_non_exec_2d_scatter4_tma_store( + ctx: *mlir.Context, + gmem_tensor: *const mlir.Value, + gmem_index_layout: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_v_map: *const mlir.Value, + non_exec_atom_type: *const mlir.Type, + tma_tensor_type: *const mlir.Type, + opts: struct { + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_2d_scatter4_tma_store", .{ .operands = .{ .flat = &.{ gmem_tensor, gmem_index_layout, smem_layout, cta_v_map } }, .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, @@ -4750,11 +4728,31 @@ pub fn atom_make_non_exec_2d_scatter4_tma_store(ctx: *mlir.Context, gmem_tensor: } /// `cute_nvgpu.atom.make_non_exec_im2col_tma_load`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_make_non_exec_im2col_tma_load(ctx: *mlir.Context, gmem_tensor: *const mlir.Value, smem_layout: *const mlir.Value, cta_v_map: *const mlir.Value, lower_corner_whd: *const mlir.Value, upper_corner_whd: *const mlir.Value, lower_padding_whd: *const mlir.Value, upper_padding_whd: *const mlir.Value, stride_whd: *const mlir.Value, lower_srt: *const mlir.Value, stride_srt: *const mlir.Value, non_exec_atom_type: *const mlir.Type, tma_tensor_type: *const mlir.Type, kind: *const mlir.Attribute, num_multicast: ?*const mlir.Attribute, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn atom_make_non_exec_im2col_tma_load( + ctx: *mlir.Context, + gmem_tensor: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_v_map: *const mlir.Value, + lower_corner_whd: *const mlir.Value, + upper_corner_whd: *const mlir.Value, + lower_padding_whd: *const mlir.Value, + upper_padding_whd: *const mlir.Value, + stride_whd: *const mlir.Value, + lower_srt: *const mlir.Value, + stride_srt: *const mlir.Value, + non_exec_atom_type: *const mlir.Type, + tma_tensor_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + num_multicast: ?*const mlir.Attribute = null, + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); - if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_im2col_tma_load", .{ .operands = .{ .flat = &.{ gmem_tensor, smem_layout, cta_v_map, lower_corner_whd, upper_corner_whd, lower_padding_whd, upper_padding_whd, stride_whd, lower_srt, stride_srt } }, .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, @@ -4764,9 +4762,20 @@ pub fn atom_make_non_exec_im2col_tma_load(ctx: *mlir.Context, gmem_tensor: *cons } /// `cute_nvgpu.atom.make_non_exec_im2col_tma_store`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_make_non_exec_im2col_tma_store(ctx: *mlir.Context, gmem_tensor: *const mlir.Value, smem_layout: *const mlir.Value, cta_v_map: *const mlir.Value, non_exec_atom_type: *const mlir.Type, tma_tensor_type: *const mlir.Type, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn atom_make_non_exec_im2col_tma_store( + ctx: *mlir.Context, + gmem_tensor: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_v_map: *const mlir.Value, + non_exec_atom_type: *const mlir.Type, + tma_tensor_type: *const mlir.Type, + opts: struct { + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_im2col_tma_store", .{ .operands = .{ .flat = &.{ gmem_tensor, smem_layout, cta_v_map } }, .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, @@ -4776,11 +4785,24 @@ pub fn atom_make_non_exec_im2col_tma_store(ctx: *mlir.Context, gmem_tensor: *con } /// `cute_nvgpu.atom.make_non_exec_tiled_tma_load`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_make_non_exec_tiled_tma_load(ctx: *mlir.Context, gmem_tensor: *const mlir.Value, smem_layout: *const mlir.Value, cta_v_map: *const mlir.Value, non_exec_atom_type: *const mlir.Type, tma_tensor_type: *const mlir.Type, kind: *const mlir.Attribute, num_multicast: ?*const mlir.Attribute, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn atom_make_non_exec_tiled_tma_load( + ctx: *mlir.Context, + gmem_tensor: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_v_map: *const mlir.Value, + non_exec_atom_type: *const mlir.Type, + tma_tensor_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + num_multicast: ?*const mlir.Attribute = null, + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); - if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_tiled_tma_load", .{ .operands = .{ .flat = &.{ gmem_tensor, smem_layout, cta_v_map } }, .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, @@ -4790,10 +4812,22 @@ pub fn atom_make_non_exec_tiled_tma_load(ctx: *mlir.Context, gmem_tensor: *const } /// `cute_nvgpu.atom.make_non_exec_tiled_tma_reduce`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_make_non_exec_tiled_tma_reduce(ctx: *mlir.Context, gmem_tensor: *const mlir.Value, smem_layout: *const mlir.Value, cta_v_map: *const mlir.Value, non_exec_atom_type: *const mlir.Type, tma_tensor_type: *const mlir.Type, kind: *const mlir.Attribute, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn atom_make_non_exec_tiled_tma_reduce( + ctx: *mlir.Context, + gmem_tensor: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_v_map: *const mlir.Value, + non_exec_atom_type: *const mlir.Type, + tma_tensor_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_tiled_tma_reduce", .{ .operands = .{ .flat = &.{ gmem_tensor, smem_layout, cta_v_map } }, .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, @@ -4803,9 +4837,20 @@ pub fn atom_make_non_exec_tiled_tma_reduce(ctx: *mlir.Context, gmem_tensor: *con } /// `cute_nvgpu.atom.make_non_exec_tiled_tma_store`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_make_non_exec_tiled_tma_store(ctx: *mlir.Context, gmem_tensor: *const mlir.Value, smem_layout: *const mlir.Value, cta_v_map: *const mlir.Value, non_exec_atom_type: *const mlir.Type, tma_tensor_type: *const mlir.Type, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn atom_make_non_exec_tiled_tma_store( + ctx: *mlir.Context, + gmem_tensor: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_v_map: *const mlir.Value, + non_exec_atom_type: *const mlir.Type, + tma_tensor_type: *const mlir.Type, + opts: struct { + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_tiled_tma_store", .{ .operands = .{ .flat = &.{ gmem_tensor, smem_layout, cta_v_map } }, .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, @@ -4929,10 +4974,28 @@ pub fn make_gmma_smem_desc(ctx: *mlir.Context, src: *const mlir.Value, res_type: } /// `cute_nvgpu.make_tma_desc_im2col`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_tma_desc_im2col(ctx: *mlir.Context, gmem_view: *const mlir.Value, smem_layout: *const mlir.Value, cta_value_tile: *const mlir.Value, lower_corner_whd: *const mlir.Value, upper_corner_whd: *const mlir.Value, lower_padding_whd: *const mlir.Value, upper_padding_whd: *const mlir.Value, stride_whd: *const mlir.Value, lower_srt: *const mlir.Value, stride_srt: *const mlir.Value, result_type: *const mlir.Type, num_multicast: ?*const mlir.Attribute, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn make_tma_desc_im2col( + ctx: *mlir.Context, + gmem_view: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_value_tile: *const mlir.Value, + lower_corner_whd: *const mlir.Value, + upper_corner_whd: *const mlir.Value, + lower_padding_whd: *const mlir.Value, + upper_padding_whd: *const mlir.Value, + stride_whd: *const mlir.Value, + lower_srt: *const mlir.Value, + stride_srt: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + num_multicast: ?*const mlir.Attribute = null, + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); - if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); return mlir.Operation.make(ctx, "cute_nvgpu.make_tma_desc_im2col", .{ .operands = .{ .flat = &.{ gmem_view, smem_layout, cta_value_tile, lower_corner_whd, upper_corner_whd, lower_padding_whd, upper_padding_whd, stride_whd, lower_srt, stride_srt } }, .results = .{ .flat = &.{result_type} }, @@ -4942,10 +5005,28 @@ pub fn make_tma_desc_im2col(ctx: *mlir.Context, gmem_view: *const mlir.Value, sm } /// `cute_nvgpu.make_tma_desc_im2col_at`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_tma_desc_im2col_at(ctx: *mlir.Context, gmem_view: *const mlir.Value, smem_layout: *const mlir.Value, cta_value_tile: *const mlir.Value, lower_corner_whd: *const mlir.Value, upper_corner_whd: *const mlir.Value, lower_padding_whd: *const mlir.Value, upper_padding_whd: *const mlir.Value, stride_whd: *const mlir.Value, lower_srt: *const mlir.Value, stride_srt: *const mlir.Value, tma_desc_addr: *const mlir.Value, num_multicast: ?*const mlir.Attribute, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn make_tma_desc_im2col_at( + ctx: *mlir.Context, + gmem_view: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_value_tile: *const mlir.Value, + lower_corner_whd: *const mlir.Value, + upper_corner_whd: *const mlir.Value, + lower_padding_whd: *const mlir.Value, + upper_padding_whd: *const mlir.Value, + stride_whd: *const mlir.Value, + lower_srt: *const mlir.Value, + stride_srt: *const mlir.Value, + tma_desc_addr: *const mlir.Value, + opts: struct { + num_multicast: ?*const mlir.Attribute = null, + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); - if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); return mlir.Operation.make(ctx, "cute_nvgpu.make_tma_desc_im2col_at", .{ .operands = .{ .flat = &.{ gmem_view, smem_layout, cta_value_tile, lower_corner_whd, upper_corner_whd, lower_padding_whd, upper_padding_whd, stride_whd, lower_srt, stride_srt, tma_desc_addr } }, .attributes = attributes.constSlice(), @@ -4954,17 +5035,30 @@ pub fn make_tma_desc_im2col_at(ctx: *mlir.Context, gmem_view: *const mlir.Value, } /// `cute_nvgpu.make_tma_desc_tiled`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_tma_desc_tiled(ctx: *mlir.Context, gmem_view: *const mlir.Value, smem_layout: *const mlir.Value, cta_value_tile: *const mlir.Value, traversal_stride: ?*const mlir.Value, result_type: *const mlir.Type, mode: *const mlir.Attribute, num_multicast: ?*const mlir.Attribute, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn make_tma_desc_tiled( + ctx: *mlir.Context, + gmem_view: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_value_tile: *const mlir.Value, + result_type: *const mlir.Type, + mode: *const mlir.Attribute, + opts: struct { + traversal_stride: ?*const mlir.Value = null, + num_multicast: ?*const mlir.Attribute = null, + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; attributes.appendAssumeCapacity(.named(ctx, "mode", mode)); - if (num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); - if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); return mlir.Operation.make(ctx, "cute_nvgpu.make_tma_desc_tiled", .{ .operands = .{ .variadic = &.{ &.{gmem_view}, &.{smem_layout}, &.{cta_value_tile}, - if (traversal_stride) |value| &.{value} else &.{}, + if (opts.traversal_stride) |value| &.{value} else &.{}, } }, .results = .{ .flat = &.{result_type} }, .attributes = attributes.constSlice(), @@ -4973,18 +5067,31 @@ pub fn make_tma_desc_tiled(ctx: *mlir.Context, gmem_view: *const mlir.Value, sme } /// `cute_nvgpu.make_tma_desc_tiled_at`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_tma_desc_tiled_at(ctx: *mlir.Context, gmem_view: *const mlir.Value, smem_layout: *const mlir.Value, cta_value_tile: *const mlir.Value, tma_desc_addr: *const mlir.Value, traversal_stride: ?*const mlir.Value, mode: *const mlir.Attribute, num_multicast: ?*const mlir.Attribute, tma_format: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn make_tma_desc_tiled_at( + ctx: *mlir.Context, + gmem_view: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_value_tile: *const mlir.Value, + tma_desc_addr: *const mlir.Value, + mode: *const mlir.Attribute, + opts: struct { + traversal_stride: ?*const mlir.Value = null, + num_multicast: ?*const mlir.Attribute = null, + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; attributes.appendAssumeCapacity(.named(ctx, "mode", mode)); - if (num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); - if (tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); return mlir.Operation.make(ctx, "cute_nvgpu.make_tma_desc_tiled_at", .{ .operands = .{ .variadic = &.{ &.{gmem_view}, &.{smem_layout}, &.{cta_value_tile}, &.{tma_desc_addr}, - if (traversal_stride) |value| &.{value} else &.{}, + if (opts.traversal_stride) |value| &.{value} else &.{}, } }, .attributes = attributes.constSlice(), .location = location, @@ -4992,11 +5099,21 @@ pub fn make_tma_desc_tiled_at(ctx: *mlir.Context, gmem_view: *const mlir.Value, } /// `cute_nvgpu.make_umma_smem_desc`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_umma_smem_desc(ctx: *mlir.Context, src: *const mlir.Value, nextSrc: ?*const mlir.Value, res_type: *const mlir.Type, layout: *const mlir.Attribute, major: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn make_umma_smem_desc( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + layout: *const mlir.Attribute, + major: *const mlir.Attribute, + opts: struct { + nextSrc: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cute_nvgpu.make_umma_smem_desc", .{ .operands = .{ .variadic = &.{ &.{src}, - if (nextSrc) |value| &.{value} else &.{}, + if (opts.nextSrc) |value| &.{value} else &.{}, } }, .results = .{ .flat = &.{res_type} }, .attributes = &.{ @@ -5016,11 +5133,21 @@ pub fn prefetch_tma_desc(ctx: *mlir.Context, atom: *const mlir.Value, location: } /// `cute_nvgpu.sm103.make_umma_smem_desc_circular`. Result types are explicit; attributes use mlir.Attribute. -pub fn sm103_make_umma_smem_desc_circular(ctx: *mlir.Context, src: *const mlir.Value, nextSrc: ?*const mlir.Value, res_type: *const mlir.Type, layout: *const mlir.Attribute, major: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn sm103_make_umma_smem_desc_circular( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + layout: *const mlir.Attribute, + major: *const mlir.Attribute, + opts: struct { + nextSrc: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cute_nvgpu.sm103.make_umma_smem_desc_circular", .{ .operands = .{ .variadic = &.{ &.{src}, - if (nextSrc) |value| &.{value} else &.{}, + if (opts.nextSrc) |value| &.{value} else &.{}, } }, .results = .{ .flat = &.{res_type} }, .attributes = &.{ @@ -5032,11 +5159,21 @@ pub fn sm103_make_umma_smem_desc_circular(ctx: *mlir.Context, src: *const mlir.V } /// `cute_nvgpu.sm107.make_umma_smem_desc`. Result types are explicit; attributes use mlir.Attribute. -pub fn sm107_make_umma_smem_desc(ctx: *mlir.Context, src: *const mlir.Value, nextSrc: ?*const mlir.Value, res_type: *const mlir.Type, layout: *const mlir.Attribute, major: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn sm107_make_umma_smem_desc( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + layout: *const mlir.Attribute, + major: *const mlir.Attribute, + opts: struct { + nextSrc: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cute_nvgpu.sm107.make_umma_smem_desc", .{ .operands = .{ .variadic = &.{ &.{src}, - if (nextSrc) |value| &.{value} else &.{}, + if (opts.nextSrc) |value| &.{value} else &.{}, } }, .results = .{ .flat = &.{res_type} }, .attributes = &.{ @@ -5048,11 +5185,21 @@ pub fn sm107_make_umma_smem_desc(ctx: *mlir.Context, src: *const mlir.Value, nex } /// `cute_nvgpu.sm107.make_umma_smem_desc_circular`. Result types are explicit; attributes use mlir.Attribute. -pub fn sm107_make_umma_smem_desc_circular(ctx: *mlir.Context, src: *const mlir.Value, nextSrc: ?*const mlir.Value, res_type: *const mlir.Type, layout: *const mlir.Attribute, major: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn sm107_make_umma_smem_desc_circular( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + layout: *const mlir.Attribute, + major: *const mlir.Attribute, + opts: struct { + nextSrc: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "cute_nvgpu.sm107.make_umma_smem_desc_circular", .{ .operands = .{ .variadic = &.{ &.{src}, - if (nextSrc) |value| &.{value} else &.{}, + if (opts.nextSrc) |value| &.{value} else &.{}, } }, .results = .{ .flat = &.{res_type} }, .attributes = &.{ @@ -5070,64 +5217,3 @@ pub fn update_tma_desc(ctx: *mlir.Context, tma_atom: *const mlir.Value, gmem_ten .location = location, }); } - -pub const operation_names: []const []const u8 = &.{ - "cute_nvgpu.arch.alloc_rmem", - "cute_nvgpu.arch.alloc_smem", - "cute_nvgpu.arch.copy.SM100.copy_s2t", - "cute_nvgpu.arch.copy.SM100.tma_load", - "cute_nvgpu.arch.copy.SM100.tma_reduce", - "cute_nvgpu.arch.copy.SM100.tma_store", - "cute_nvgpu.arch.copy.SM100.tmem_load", - "cute_nvgpu.arch.copy.SM100.tmem_store", - "cute_nvgpu.arch.copy.SM107.tma_load_override", - "cute_nvgpu.arch.copy.SM107.tma_store_override", - "cute_nvgpu.arch.copy.SM90.bulk_copy_g2s", - "cute_nvgpu.arch.copy.SM90.bulk_copy_s2g", - "cute_nvgpu.arch.copy.SM90.bulk_copy_s2s", - "cute_nvgpu.arch.get_dyn_smem", - "cute_nvgpu.arch.get_dyn_smem_size", - "cute_nvgpu.arch.make_warp_uniform", - "cute_nvgpu.arch.mma.SM107.umma", - "cute_nvgpu.arch.mma.SM107.umma_block_scaled", - "cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse", - "cute_nvgpu.arch.mma.SM107.umma_sparse", - "cute_nvgpu.arch.mma.SM120.block_scaled", - "cute_nvgpu.arch.prefetch_tma_desc", - "cute_nvgpu.arch.sm100.alloc_tmem", - "cute_nvgpu.arch.sm100.dealloc_tmem", - "cute_nvgpu.arch.sm100.relinquish_tmem_alloc_permit", - "cute_nvgpu.arch.sm100.retrieve_tmem_ptr", - "cute_nvgpu.atom.get_coord_tensor", - "cute_nvgpu.atom.get_copy_s2t_smem_desc_view", - "cute_nvgpu.atom.get_value", - "cute_nvgpu.atom.make_exec_tma", - "cute_nvgpu.atom.make_non_exec_2d_gather4_tma_load", - "cute_nvgpu.atom.make_non_exec_2d_scatter4_tma_store", - "cute_nvgpu.atom.make_non_exec_im2col_tma_load", - "cute_nvgpu.atom.make_non_exec_im2col_tma_store", - "cute_nvgpu.atom.make_non_exec_tiled_tma_load", - "cute_nvgpu.atom.make_non_exec_tiled_tma_reduce", - "cute_nvgpu.atom.make_non_exec_tiled_tma_store", - "cute_nvgpu.atom.make_s2t_copy", - "cute_nvgpu.atom.make_tma_residue_tensor", - "cute_nvgpu.atom.make_tmem_copy", - "cute_nvgpu.atom.set_value", - "cute_nvgpu.atom.sm107.get_copy_s2t_smem_desc_view", - "cute_nvgpu.atom.tma_partition", - "cute_nvgpu.cast_tma_desc_to_integer", - "cute_nvgpu.copy_tma_desc", - "cute_nvgpu.get_grid_constant_pointer", - "cute_nvgpu.get_tma_desc_addr", - "cute_nvgpu.make_gmma_smem_desc", - "cute_nvgpu.make_tma_desc_im2col", - "cute_nvgpu.make_tma_desc_im2col_at", - "cute_nvgpu.make_tma_desc_tiled", - "cute_nvgpu.make_tma_desc_tiled_at", - "cute_nvgpu.make_umma_smem_desc", - "cute_nvgpu.prefetch_tma_desc", - "cute_nvgpu.sm103.make_umma_smem_desc_circular", - "cute_nvgpu.sm107.make_umma_smem_desc", - "cute_nvgpu.sm107.make_umma_smem_desc_circular", - "cute_nvgpu.update_tma_desc", -}; diff --git a/mlir/dialects/cute_nvgpu/test.zig b/mlir/dialects/cute_nvgpu/test.zig new file mode 100644 index 00000000..7c6d0fc6 --- /dev/null +++ b/mlir/dialects/cute_nvgpu/test.zig @@ -0,0 +1,210 @@ +const std = @import("std"); + +const cute_ir = @import("mlir/dialects/cute_ir"); +const cute_nvgpu = @import("mlir/dialects/cute_nvgpu"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "cute_nvgpu.arch.alloc_rmem", + "cute_nvgpu.arch.alloc_smem", + "cute_nvgpu.arch.copy.SM100.copy_s2t", + "cute_nvgpu.arch.copy.SM100.tma_load", + "cute_nvgpu.arch.copy.SM100.tma_reduce", + "cute_nvgpu.arch.copy.SM100.tma_store", + "cute_nvgpu.arch.copy.SM100.tmem_load", + "cute_nvgpu.arch.copy.SM100.tmem_store", + "cute_nvgpu.arch.copy.SM107.tma_load_override", + "cute_nvgpu.arch.copy.SM107.tma_store_override", + "cute_nvgpu.arch.copy.SM90.bulk_copy_g2s", + "cute_nvgpu.arch.copy.SM90.bulk_copy_s2g", + "cute_nvgpu.arch.copy.SM90.bulk_copy_s2s", + "cute_nvgpu.arch.get_dyn_smem", + "cute_nvgpu.arch.get_dyn_smem_size", + "cute_nvgpu.arch.make_warp_uniform", + "cute_nvgpu.arch.mma.SM107.umma", + "cute_nvgpu.arch.mma.SM107.umma_block_scaled", + "cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse", + "cute_nvgpu.arch.mma.SM107.umma_sparse", + "cute_nvgpu.arch.mma.SM120.block_scaled", + "cute_nvgpu.arch.prefetch_tma_desc", + "cute_nvgpu.arch.sm100.alloc_tmem", + "cute_nvgpu.arch.sm100.dealloc_tmem", + "cute_nvgpu.arch.sm100.relinquish_tmem_alloc_permit", + "cute_nvgpu.arch.sm100.retrieve_tmem_ptr", + "cute_nvgpu.atom.get_coord_tensor", + "cute_nvgpu.atom.get_copy_s2t_smem_desc_view", + "cute_nvgpu.atom.get_value", + "cute_nvgpu.atom.make_exec_tma", + "cute_nvgpu.atom.make_non_exec_2d_gather4_tma_load", + "cute_nvgpu.atom.make_non_exec_2d_scatter4_tma_store", + "cute_nvgpu.atom.make_non_exec_im2col_tma_load", + "cute_nvgpu.atom.make_non_exec_im2col_tma_store", + "cute_nvgpu.atom.make_non_exec_tiled_tma_load", + "cute_nvgpu.atom.make_non_exec_tiled_tma_reduce", + "cute_nvgpu.atom.make_non_exec_tiled_tma_store", + "cute_nvgpu.atom.make_s2t_copy", + "cute_nvgpu.atom.make_tma_residue_tensor", + "cute_nvgpu.atom.make_tmem_copy", + "cute_nvgpu.atom.set_value", + "cute_nvgpu.atom.sm107.get_copy_s2t_smem_desc_view", + "cute_nvgpu.atom.tma_partition", + "cute_nvgpu.cast_tma_desc_to_integer", + "cute_nvgpu.copy_tma_desc", + "cute_nvgpu.get_grid_constant_pointer", + "cute_nvgpu.get_tma_desc_addr", + "cute_nvgpu.make_gmma_smem_desc", + "cute_nvgpu.make_tma_desc_im2col", + "cute_nvgpu.make_tma_desc_im2col_at", + "cute_nvgpu.make_tma_desc_tiled", + "cute_nvgpu.make_tma_desc_tiled_at", + "cute_nvgpu.make_umma_smem_desc", + "cute_nvgpu.prefetch_tma_desc", + "cute_nvgpu.sm103.make_umma_smem_desc_circular", + "cute_nvgpu.sm107.make_umma_smem_desc", + "cute_nvgpu.sm107.make_umma_smem_desc_circular", + "cute_nvgpu.update_tma_desc", +}; + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + cute_ir.registerDialect(registry); + cute_nvgpu.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(cute_nvgpu); + inline for (comptime std.meta.declarations(cute_nvgpu)) |decl| { + const T = @field(cute_nvgpu, decl); + if (@TypeOf(T) == type) std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const ctx = try testContext(); + defer ctx.deinit(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "generated types and attributes rebuild from their getters" { + @setEvalBranchQuota(100_000); + const ctx = try testContext(); + defer ctx.deinit(); + inline for (.{ + .{ cute_nvgpu.CopyAtomBulkCopyG2SType, "!cute_nvgpu.atom.bulk_copy_g2s" }, + .{ cute_nvgpu.CopyAtomBulkCopyS2GType, "!cute_nvgpu.atom.bulk_copy_s2g" }, + .{ cute_nvgpu.CopyAtomBulkCopyS2SType, "!cute_nvgpu.atom.bulk_copy_s2s" }, + .{ cute_nvgpu.CopyAtomDsmemStoreType, "!cute_nvgpu.atom.dsmem_store" }, + .{ cute_nvgpu.CopyAtomG2RType, "!cute_nvgpu.atom.g2r" }, + .{ cute_nvgpu.CopyAtomIm2ColTmaLoadType, "!cute_nvgpu.atom.im2col_tma_load mcast = true tma_gbasis = <\"(64,128):(1@1,1@0)\">>" }, + .{ cute_nvgpu.CopyAtomIm2ColTmaStoreType, "!cute_nvgpu.atom.im2col_tma_store tma_gbasis = <\"(32,128):(1@1,1@0)\">>" }, + .{ cute_nvgpu.CopyAtomLdsmType, "!cute_nvgpu.atom.ldsm, sz_pattern = u16, num_matrices = 4, t>" }, + .{ cute_nvgpu.CopyAtomNonExec2DGather4TmaLoadType, "!cute_nvgpu.atom.non_exec_2d_gather4_tma_load, tma_format = F16_RN>" }, + .{ cute_nvgpu.CopyAtomNonExec2DScatter4TmaStoreType, "!cute_nvgpu.atom.non_exec_2d_scatter4_tma_store, tma_format = F16_RN>" }, + .{ cute_nvgpu.CopyAtomNonExecIm2ColTmaLoadType, "!cute_nvgpu.atom.non_exec_im2col_tma_load, tma_format = F16_RN>" }, + .{ cute_nvgpu.CopyAtomNonExecIm2ColTmaStoreType, "!cute_nvgpu.atom.non_exec_im2col_tma_store, tma_format = F16_RN>" }, + .{ cute_nvgpu.CopyAtomNonExecTiledTmaLoadType, "!cute_nvgpu.atom.non_exec_tiled_tma_load, tma_format = BF16_RN>" }, + .{ cute_nvgpu.CopyAtomNonExecTiledTmaReduceType, "!cute_nvgpu.atom.non_exec_tiled_tma_reduce, tma_format = F32_RN>" }, + .{ cute_nvgpu.CopyAtomNonExecTiledTmaStoreType, "!cute_nvgpu.atom.non_exec_tiled_tma_store, tma_format = BF16_RN>" }, + .{ cute_nvgpu.CopyAtomR2GType, "!cute_nvgpu.atom.r2g" }, + .{ cute_nvgpu.CopyAtomR2SType, "!cute_nvgpu.atom.r2s" }, + .{ cute_nvgpu.CopyAtomS2RType, "!cute_nvgpu.atom.s2r" }, + .{ cute_nvgpu.CopyAtomSIMTAsyncCopyType, "!cute_nvgpu.atom.simt_async_copy" }, + .{ cute_nvgpu.CopyAtomSIMTMultimemLdReduceType, "!cute_nvgpu.atom.simt_multimem_ld_reduce, acquire, gpu>" }, + .{ cute_nvgpu.CopyAtomSIMTMultimemRedType, "!cute_nvgpu.atom.simt_multimem_red" }, + .{ cute_nvgpu.CopyAtomSIMTMultimemStType, "!cute_nvgpu.atom.simt_multimem_st" }, + .{ cute_nvgpu.CopyAtomSIMTSyncCopyType, "!cute_nvgpu.atom.universal_copy" }, + .{ cute_nvgpu.CopyAtomSM100CopyS2TType, "!cute_nvgpu.atom.s2t_copy" }, + .{ cute_nvgpu.CopyAtomSM100S2TCopyV2Type, "!cute_nvgpu.atom.sm100_s2t_copy_v2" }, + .{ cute_nvgpu.CopyAtomSM100TmemLoadType, "!cute_nvgpu.atom.tmem_load" }, + .{ cute_nvgpu.CopyAtomSM100TmemStoreType, "!cute_nvgpu.atom.tmem_store" }, + .{ cute_nvgpu.CopyAtomSM107TmemLoadSPCompressType, "!cute_nvgpu.atom.tmem_load_spcompress" }, + .{ cute_nvgpu.CopyAtomSM10xTmemLoadRedType, "!cute_nvgpu.atom.tmem_load_red" }, + .{ cute_nvgpu.CopyAtomStsmType, "!cute_nvgpu.atom.stsm, num_matrices = 4, n>" }, + .{ cute_nvgpu.CopyAtomTmaLoadType, "!cute_nvgpu.atom.tma_load mcast = true tma_gbasis = <\"(64,1):(1@0,1@1)\">>" }, + .{ cute_nvgpu.CopyAtomTmaReduceType, "!cute_nvgpu.atom.tma_reduce tma_gbasis = <\"(32,128):(1@1,1@0)\">>" }, + .{ cute_nvgpu.CopyAtomTmaStoreType, "!cute_nvgpu.atom.tma_store tma_gbasis = <\"(64,16):(1@0,1@1)\">>" }, + .{ cute_nvgpu.MmaAtomSM100UMMABlockScaledSparseType, "!cute_nvgpu.sm100.mma_bs_sp<128x128x64, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, sparse_metadata_format = tid, frag_kind = ss, vec_size = 32>" }, + .{ cute_nvgpu.MmaAtomSM100UMMABlockScaledType, "!cute_nvgpu.sm100.mma_bs<128x128x96, num_cta = 1, ab_major = (k, k), elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E8M0FNU, frag_kind = ss, vec_size = 16, arch_promote = sm_103>" }, + .{ cute_nvgpu.MmaAtomSM100UMMASparseType, "!cute_nvgpu.sm100.mma_sp<128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f16, f16, f32), e_type = i8, sparse_metadata_format = tid, frag_kind = ss, c_scale_exp = 0>" }, + .{ cute_nvgpu.MmaAtomSM100UMMAType, "!cute_nvgpu.sm100.mma<128x128x16, num_cta = 1, ab_major = (k, mn), elem_type = (f16, f16, f32), frag_kind = ss, c_scale_exp = 0>" }, + .{ cute_nvgpu.MmaAtomSM107UMMABlockScaledSparseType, "!cute_nvgpu.sm107.mma_bs_sp<128x128x64, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, sparse_metadata_format = tid, frag_kind = ss, vec_size = 32, ab_collector_op = (discard, discard)>" }, + .{ cute_nvgpu.MmaAtomSM107UMMABlockScaledType, "!cute_nvgpu.sm107.mma_bs<128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, frag_kind = ss, vec_size = 32, ab_collector_op = (discard, use), int_overflow = >" }, + .{ cute_nvgpu.MmaAtomSM107UMMASparseType, "!cute_nvgpu.sm107.mma_sp<128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f16, f16, f32), e_type = i8, sparse_metadata_format = tid, frag_kind = ss, c_scale_exp = 0, ab_collector_op = (discard, discard)>" }, + .{ cute_nvgpu.MmaAtomSM107UMMAType, "!cute_nvgpu.sm107.mma<128x128x16, num_cta = 1, ab_major = (k, mn), elem_type = (f16, f16, f32), frag_kind = ss, c_scale_exp = 0, ab_collector_op = (lastuse, fill)>" }, + .{ cute_nvgpu.MmaAtomSM120BlockScaledType, "!cute_nvgpu.SM120.mma_bs<16x8x64, vec_size = 32, elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E8M0FNU, use_sf_layout_TV = true>" }, + .{ cute_nvgpu.MmaAtomSM80SparseType, "!cute_nvgpu.sm80.sparse_mma<16x8x64, elem_type = (i8, i8, i32), md_format = tid, int_overflow = >" }, + .{ cute_nvgpu.MmaAtomSM80Type, "!cute_nvgpu.sm80.mma<16x8x32, elem_type = (i8, i8, i32), int_overflow = , binary_op = >" }, + .{ cute_nvgpu.MmaAtomSM89Type, "!cute_nvgpu.sm89.mma<16x8x32, elem_type = (f8E4M3FN, f8E4M3FN, f32) >" }, + .{ cute_nvgpu.MmaAtomSM90Type, "!cute_nvgpu.sm90.mma<64x128x32, ab_major = (k, k), elem_type = (i8, i8, i32), frag_kind = ss, int_overflow = , a_neg, b_neg>" }, + .{ cute_nvgpu.SmemDescCircularSM103Type, "!cute_nvgpu.sm103.smem_desc_circular<<\"(8,128,2):(128,1,1024)\">>" }, + .{ cute_nvgpu.SmemDescCircularSM107Type, "!cute_nvgpu.sm107.smem_desc_circular<<\"(8,128,2):(128,1,1024)\">>" }, + .{ cute_nvgpu.SmemDescSM107Type, "!cute_nvgpu.sm107.smem_desc" }, + .{ cute_nvgpu.SmemDescType, "!cute_nvgpu.smem_desc" }, + .{ cute_nvgpu.SmemDescViewType, "!cute_nvgpu.smem_desc_view" }, + .{ cute_nvgpu.TmaDescriptorIm2ColType, "!cute_nvgpu.tma_descriptor_im2col" }, + .{ cute_nvgpu.TmaDescriptorTiledType, "!cute_nvgpu.tma_descriptor_tiled" }, + .{ cute_nvgpu.UniversalFmaAtomType, "!cute_nvgpu.atom.universal_fma<1x1x1, (f32, f32) -> f32 >" }, + .{ cute_nvgpu.WorkIdResponseType, "!cute_nvgpu.workid_response" }, + .{ cute_nvgpu.NotImplementedFrgAttr, "#cute_nvgpu.not_implemented_frg<>" }, + .{ cute_nvgpu.RmemFrgAttr, "#cute_nvgpu.rmem_frg" }, + .{ cute_nvgpu.SM100CircularSmemFrgAttr, "#cute_nvgpu.arch.sm100.circular_smem_frg" }, + .{ cute_nvgpu.SM100SmemFrgAttr, "#cute_nvgpu.arch.sm100.smem_frg" }, + .{ cute_nvgpu.SM100TmemEFrgAttr, "#cute_nvgpu.arch.sm100.tmem_e_frg" }, + .{ cute_nvgpu.SM100TmemFrgAttr, "#cute_nvgpu.arch.sm100.tmem_frg" }, + .{ cute_nvgpu.SM100TmemSfFrgAttr, "#cute_nvgpu.arch.sm100.tmem_sf_frg" }, + .{ cute_nvgpu.SM107SmemFrgAttr, "#cute_nvgpu.arch.sm107.smem_frg" }, + .{ cute_nvgpu.SM90SmemFrgAttr, "#cute_nvgpu.arch.sm90.smem_frg" }, + }) |example| { + const T = example[0]; + const is_type = @hasDecl(T, "type_"); + const parsed = if (is_type) try mlir.Type.parse(ctx, example[1]) else try mlir.Attribute.parse(ctx, example[1]); + const value = parsed.isA(T) orelse return error.TestUnexpectedResult; + try std.testing.expect((try rebuild(T, ctx, value)).eql(value)); + var text: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer text.deinit(); + try text.writer.print("{f}", .{value}); + try std.testing.expect(parsed.eql(if (is_type) try mlir.Type.parse(ctx, text.written()) else try mlir.Attribute.parse(ctx, text.written()))); + } + // Every case of every enum attribute. + inline for (.{cute_nvgpu}) |namespace| { + inline for (comptime std.meta.declarations(namespace)) |decl| { + const T = @field(namespace, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque" and @hasDecl(T, "InitArgs")) { + const fields = @typeInfo(T.InitArgs).@"struct".field_names; + if (fields.len == 1 and comptime std.mem.eql(u8, fields[0], "value")) { + inline for (@typeInfo(@FieldType(T.InitArgs, fields[0])).@"enum".field_names) |case| { + const case_value = @field(@FieldType(T.InitArgs, fields[0]), case); + const attr = try T.get(ctx, .{ .value = case_value }); + try std.testing.expectEqual(@backingInt(case_value), @backingInt(attr.value())); + try std.testing.expect((try rebuild(T, ctx, attr)).eql(attr)); + } + } + } + } + } + try std.testing.expect((try cute_nvgpu.SmemDescType.get(ctx, .{})).type_().isA(cute_nvgpu.NVGPUType) != null); +} + +fn rebuild(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T { + var args: T.InitArgs = undefined; + var elements: [8]*const mlir.Type = undefined; + inline for (@typeInfo(T.InitArgs).@"struct".field_names) |field| { + const name = [_]u8{comptime std.ascii.toUpper(field[0])} ++ field[1..]; + const noun = [_]u8{comptime std.ascii.toLower(field[0])} ++ field[1..]; + const accessor = if (@hasDecl(T, noun ++ "_")) noun ++ "_" else noun; + if (@FieldType(T.InitArgs, field) == []const *const mlir.Type) { + const count = @field(T, "num" ++ name)(value); + for (elements[0..count], 0..) |*element, i| element.* = @field(T, noun[0 .. noun.len - 1])(value, i); + @field(args, field) = elements[0..count]; + } else { + @field(args, field) = @field(T, accessor)(value); + } + } + return T.get(ctx, args); +} diff --git a/mlir/dialects/dialects.zig b/mlir/dialects/dialects.zig deleted file mode 100644 index e64b8601..00000000 --- a/mlir/dialects/dialects.zig +++ /dev/null @@ -1,26 +0,0 @@ -const std = @import("std"); - -pub const cuda_tile = @import("mlir/dialects/cuda_tile"); -pub const cute = @import("mlir/dialects/cute_ir"); -pub const fly = @import("mlir/dialects/fly"); -pub const mosaic_tpu = @import("mlir/dialects/mosaic_tpu"); -pub const stablehlo = @import("mlir/dialects/stablehlo"); -pub const ttir = @import("mlir/dialects/ttir"); - -pub const affine = @import("affine.zig"); -pub const arith = @import("arith.zig"); -pub const cf = @import("cf.zig"); -pub const cuda = @import("cuda.zig"); -pub const func = @import("func.zig"); -pub const gpu = @import("gpu.zig"); -pub const llvm = @import("llvm_dialect.zig"); -pub const math = @import("math.zig"); -pub const memref = @import("memref.zig"); -pub const nvvm = @import("nvvm.zig"); -pub const scf = @import("scf.zig"); -pub const shardy = @import("shardy.zig"); -pub const vector = @import("vector.zig"); - -test { - std.testing.refAllDecls(@This()); -} diff --git a/mlir/dialects/fly/BUILD.bazel b/mlir/dialects/fly/BUILD.bazel index 24adcc45..87fe2405 100644 --- a/mlir/dialects/fly/BUILD.bazel +++ b/mlir/dialects/fly/BUILD.bazel @@ -2,20 +2,19 @@ load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") cc_library( - name = "fly_capi", - srcs = ["fly_capi.cc"], - hdrs = ["fly_capi.h"], - strip_include_prefix = ".", + name = "capi", + srcs = [ + "capi.cc", + ], + hdrs = [ + "capi.h", + ], tags = ["manual"], - visibility = ["//visibility:private"], deps = [ "@flydsl//:FlyDialect", - "@flydsl//:FlyROCDLDialect", "@flydsl//:TiledOpTraits", "@llvm-project//mlir:CAPIIR", - "@llvm-project//mlir:UBDialect", ], - alwayslink = True, ) zig_library( @@ -23,8 +22,6 @@ zig_library( srcs = [ "attributes.zig", "ops.zig", - "rocdl.zig", - "rocdl_ops.zig", "types.zig", ], import_name = "mlir/dialects/fly", @@ -32,15 +29,28 @@ zig_library( tags = ["manual"], visibility = ["//visibility:public"], deps = [ - ":fly_capi", + ":capi", "//mlir", - "//mlir:c", - "//mlir/dialects:capi", "//stdx", ], ) zig_test( name = "test", - deps = [":fly"], + main = "test.zig", + deps = [ + ":fly", + "//mlir", + "//mlir/dialects/arith", + "//mlir/dialects/fly_rocdl", + "//mlir/dialects/func", + "//mlir/dialects/gpu", + "//mlir/dialects/llvm", + "//mlir/dialects/math", + "//mlir/dialects/memref", + "//mlir/dialects/rocdl", + "//mlir/dialects/scf", + "//mlir/dialects/ub", + "//mlir/dialects/vector", + ], ) diff --git a/mlir/dialects/fly/REGENERATING.md b/mlir/dialects/fly/REGENERATING.md index 163842d6..b66b7abe 100644 --- a/mlir/dialects/fly/REGENERATING.md +++ b/mlir/dialects/fly/REGENERATING.md @@ -1,7 +1,7 @@ # Regenerating the operation builders These are instructions for an agent. They describe what `ops.zig` and -`rocdl_ops.zig` must contain after a FlyDSL bump, where to derive them from, +`../fly_rocdl/ops.zig` must contain after a FlyDSL bump, where to derive them from, and how to check the result. Scripts and intermediate files are not committed; only the generated builders are. @@ -9,10 +9,10 @@ Paths below are relative to this directory. ## Goal -`ops.zig` binds every operation of the `fly` dialect and `rocdl_ops.zig` every +`ops.zig` binds every operation of the `fly` dialect and `../fly_rocdl/ops.zig` every operation of the `fly_rocdl` dialect, one builder per operation, so that no code builds a FlyDSL operation from its name. The attributes and types -(`attributes.zig`, `types.zig`, `rocdl.zig`, `fly_capi.*`) are maintained by +(`attributes.zig`, `types.zig`, `../fly_rocdl/fly_rocdl.zig`, `capi.*`, `../fly_rocdl/capi.*`) are maintained by hand and are not covered here. ## Sources @@ -23,7 +23,7 @@ patches applied: read the `.td` files from the fetched repository unpatched checkout. - `include/flydsl/Dialect/Fly/IR/FlyOps.td` for `ops.zig`. -- `include/flydsl/Dialect/FlyROCDL/IR/Ops.td` for `rocdl_ops.zig`. +- `include/flydsl/Dialect/FlyROCDL/IR/Ops.td` for `../fly_rocdl/ops.zig`. Dump their records with the LLVM the workspace builds: @@ -39,46 +39,41 @@ operation. Do not regex the `.td` files. ## What each builder must be -- One `pub fn` per operation, named by its mnemonic with dots replaced by - underscores (`fly.tiled_copy.partition_src` → `tiled_copy_partition_src`), - sorted by mnemonic, with a doc comment naming the operation, its `summary`, - and whether result types are inferred, explicit, or absent. -- Parameters, in order: `ctx`, the operands, the result types (only when the - operation does not implement `InferTypeOpInterface`), the attributes, then - `location`. Names are the ODS names in snake case, with a trailing - underscore when they collide with a Zig keyword, a builder name or a - reserved local (`ctx`, `location`, `make`, `operands`, `attributes`, - `present`). -- Operands: single → `*const mlir.Value`, `Optional` → `?*const mlir.Value`, - `Variadic` → `[]const *const mlir.Value`. Operations with - `AttrSizedOperandSegments` pass every operand group through +Builder names, parameter order, `opts`, operand and result types, doc comments +and `operation_names` follow `mlir/dialects/README.md`. Specific to FlyDSL: + +- Builders are sorted by mnemonic. The doc comment gives the `summary` and + whether result types are inferred, explicit or absent. +- Unlike the README rule, parameter names are the ODS names in snake case. The + reserved locals `ctx`, `opts`, `location`, `make`, `operands`, `attributes` + and `present` also take a trailing underscore. +- Operations with `AttrSizedOperandSegments` pass every operand group through `Operation.make`'s `.variadic` operands, which writes `operandSegmentSizes`; others pass a flat list (an optional operand may only be last). -- Results: single → `*const mlir.Type`, `Variadic` → - `[]const *const mlir.Type` named `_types`. Inferred operations set - `.result_type_inference = true` and take no result types. Inference comes +- Inferred operations set `.result_type_inference = true`. Inference comes from the flattened trait list (trait lists such as `Pure` expand to their members); an `OpInterface` record whose `cppInterfaceName` is `InferTypeOpInterface` marks it. -- Attributes: `*const mlir.Attribute`, optional (`?`) when the attribute is - `OptionalAttr`, `UnitAttr` or has a default value; set under its ODS name. +- Attributes are `*const mlir.Attribute`, set under their ODS name. - Builders call `make` from `fly.zig`, which verifies the operation and names the operand types when creation fails. - Each file starts with a header naming the FlyDSL commit and the `.td` it - was generated from, and exports `names`, the list of operation names, for - the registration test. + was generated from. Regions, successors and `VariadicOfVariadic` operands have no use in either dialect today; when one appears, extend the generator and this file. ## Checks -- `zig fmt` the generated files. -- `bazel test //mlir/dialects/fly:test //kernels/fly:test`: every name in - `ops.names` and `rocdl.ops.names` must be a registered operation. +- Format and run the conformance checks of `mlir/dialects/README.md`. +- `bazel test //mlir/dialects/fly:test //dsl/fly:test`, which includes the + registration tests of the `operation_names` lists in both packages' + `test.zig`. - Callers that no longer compile are the point of generating: an operation whose signature changed in FlyDSL now fails at the call site. Fix them in - the same change (`kernels/fly/builder.zig`, `zml/*/fly_kernels`). + the same change (`dsl/fly/builder.zig`, `zml/kernel.zig`, `zml/kernels/*/fly/`, + and downstream llmd's `llmd/kernels/*/fly/`). - Emit the IR of the fly kernels before and after (for instance - `sparse_mla.Main2D.emit` and its siblings with representative configs) and + llmd's `sparse_mla.Main2D.emit` and the `zml/kernels/moe/fly/` kernels with + representative configs) and diff it: a regeneration without an upstream change must be byte-identical. diff --git a/mlir/dialects/fly/attributes.zig b/mlir/dialects/fly/attributes.zig index a30336fc..3fc86ad3 100644 --- a/mlir/dialects/fly/attributes.zig +++ b/mlir/dialects/fly/attributes.zig @@ -1,5 +1,3 @@ -const std = @import("std"); - const c = @import("c"); const mlir = @import("mlir"); @@ -10,24 +8,22 @@ pub const IntAttr = opaque { pub const isAFn = c.mlirAttributeIsAFlyInt; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn getStatic(ctx: *mlir.Context, value: i32) mlir.Error!*const Self { - return @ptrCast(c.mlirFlyIntAttrGetStatic(ctx.ptr(), value).ptr orelse return error.InvalidMlir); + return @ptrCast(c.mlirFlyIntAttrStaticGet(ctx.ptr(), value).ptr orelse return error.InvalidMlir); } pub fn getDynamic(ctx: *mlir.Context, width: i32, divisibility: i32) mlir.Error!*const Self { - return @ptrCast(c.mlirFlyIntAttrGetDynamic(ctx.ptr(), width, divisibility).ptr orelse return error.InvalidMlir); + return @ptrCast(c.mlirFlyIntAttrDynamicGet(ctx.ptr(), width, divisibility).ptr orelse return error.InvalidMlir); } pub fn getNone(ctx: *mlir.Context) mlir.Error!*const Self { - return @ptrCast(c.mlirFlyIntAttrGetNone(ctx.ptr()).ptr orelse return error.InvalidMlir); + return @ptrCast(c.mlirFlyIntAttrNoneGet(ctx.ptr()).ptr orelse return error.InvalidMlir); } }; @@ -38,13 +34,11 @@ pub const IntTupleAttr = opaque { pub const isAFn = c.mlirAttributeIsAFlyIntTuple; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub const InitArgs = struct { value: *const mlir.Attribute, @@ -67,12 +61,12 @@ pub const IntTupleAttr = opaque { } pub fn getBasis(ctx: *mlir.Context, value: *const IntAttr, modes: []const i32) mlir.Error!*const Self { - const result = c.mlirFlyIntTupleAttrGetBasis(ctx.ptr(), value.ptr(), @intCast(modes.len), modes.ptr); + const result = c.mlirFlyIntTupleAttrBasisGet(ctx.ptr(), value.ptr(), @intCast(modes.len), modes.ptr); return @ptrCast(result.ptr orelse return error.InvalidMlir); } pub fn getTuple(ctx: *mlir.Context, elements: []const *const Self) mlir.Error!*const Self { - const result = c.mlirFlyIntTupleAttrGetTuple(ctx.ptr(), @intCast(elements.len), @ptrCast(elements.ptr)); + const result = c.mlirFlyIntTupleAttrTupleGet(ctx.ptr(), @intCast(elements.len), @ptrCast(elements.ptr)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } }; @@ -84,13 +78,11 @@ pub const LayoutAttr = opaque { pub const isAFn = c.mlirAttributeIsAFlyLayout; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub const InitArgs = struct { shape: *const IntTupleAttr, @@ -110,13 +102,11 @@ pub const TileAttr = opaque { pub const isAFn = c.mlirAttributeIsAFlyTile; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub const InitArgs = struct { value: *const mlir.Attribute, @@ -127,7 +117,7 @@ pub const TileAttr = opaque { } pub fn getModes(ctx: *mlir.Context, modes: []const *const mlir.Attribute) mlir.Error!*const Self { - const result = c.mlirFlyTileAttrGetModes(ctx.ptr(), @intCast(modes.len), @ptrCast(modes.ptr)); + const result = c.mlirFlyTileAttrModesGet(ctx.ptr(), @intCast(modes.len), @ptrCast(modes.ptr)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } }; @@ -139,13 +129,11 @@ pub const SwizzleAttr = opaque { pub const isAFn = c.mlirAttributeIsAFlySwizzle; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub const InitArgs = struct { mask: i32, @@ -166,13 +154,11 @@ pub const AlignAttr = opaque { pub const isAFn = c.mlirAttributeIsAFlyAlign; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub const InitArgs = struct { alignment: i32, @@ -193,13 +179,11 @@ pub const AddressSpaceAttr = opaque { pub const isAFn = c.mlirAttributeIsAFlyAddressSpace; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub const InitArgs = struct { addressSpace: AddressSpace, diff --git a/mlir/dialects/fly/capi.cc b/mlir/dialects/fly/capi.cc new file mode 100644 index 00000000..1f4913bd --- /dev/null +++ b/mlir/dialects/fly/capi.cc @@ -0,0 +1,529 @@ +//===- capi.cc - C API for the fly dialect --------------------------------===// + +#include "capi.h" + +#include "TiledOpTraits.h" +#include "flydsl/Dialect/Fly/IR/FlyDialect.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Fly, fly, mlir::fly::FlyDialect) + +using mlir::cast; +using mlir::isa; +using namespace mlir::fly; + +static mlir::MLIRContext *loadFly(MlirContext context) { + mlir::MLIRContext *ctx = unwrap(context); + if (ctx) + ctx->getOrLoadDialect(); + return ctx; +} + +/// `!attr` and `attr == other` are value arithmetic on a fly IntAttr, not a +/// null test: every handle check here goes through these two helpers. +template static bool isNull(T value) { + return !static_cast(value); +} + +template +static T checked(Handle handle, mlir::MLIRContext *ctx) { + auto value = mlir::dyn_cast_if_present(unwrap(handle)); + if (isNull(value) || value.getContext() != ctx) + return T{}; + return value; +} + +#define FLY_ATTR_ISA(Name, Attr) \ + bool mlirAttributeIsAFly##Name(MlirAttribute attr) { \ + return mlir::isa_and_nonnull(unwrap(attr)); \ + } + +FLY_ATTR_ISA(Int, IntAttr) +FLY_ATTR_ISA(IntTuple, IntTupleAttr) +FLY_ATTR_ISA(Layout, LayoutAttr) +FLY_ATTR_ISA(Tile, TileAttr) +FLY_ATTR_ISA(Swizzle, SwizzleAttr) +FLY_ATTR_ISA(Align, AlignAttr) +FLY_ATTR_ISA(AddressSpace, AddressSpaceAttr) + +#undef FLY_ATTR_ISA + +#define FLY_TYPE_ISA(Name, Type) \ + bool mlirTypeIsAFly##Name(MlirType type) { \ + return mlir::isa_and_nonnull(unwrap(type)); \ + } + +FLY_TYPE_ISA(IntTuple, IntTupleType) +FLY_TYPE_ISA(Layout, LayoutType) +FLY_TYPE_ISA(ComposedLayout, ComposedLayoutType) +FLY_TYPE_ISA(Tile, TileType) +FLY_TYPE_ISA(Swizzle, SwizzleType) +FLY_TYPE_ISA(Pointer, PointerType) +FLY_TYPE_ISA(MemRef, mlir::fly::MemRefType) +FLY_TYPE_ISA(CoordTensor, CoordTensorType) +FLY_TYPE_ISA(CopyAtom, CopyAtomType) +FLY_TYPE_ISA(MmaAtom, MmaAtomType) +FLY_TYPE_ISA(TiledCopy, TiledCopyType) +FLY_TYPE_ISA(TiledMma, TiledMmaType) +FLY_TYPE_ISA(CopyOpUniversalCopy, CopyOpUniversalCopyType) + +#undef FLY_TYPE_ISA + +MlirAttribute mlirFlyIntAttrStaticGet(MlirContext context, int32_t value) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast(IntAttr::getStatic(ctx, value))); +} + +MlirAttribute mlirFlyIntAttrDynamicGet(MlirContext context, int32_t width, + int32_t divisibility) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast( + IntAttr::getDynamic(ctx, width, divisibility))); +} + +MlirAttribute mlirFlyIntAttrNoneGet(MlirContext context) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast(IntAttr::getNone(ctx))); +} + +MlirAttribute mlirFlyIntTupleAttrGet(MlirContext context, MlirAttribute value) { + mlir::MLIRContext *ctx = loadFly(context); + mlir::Attribute inner = unwrap(value); + if (!ctx || isNull(inner) || inner.getContext() != ctx) + return {nullptr}; + return wrap(static_cast(IntTupleAttr::get(ctx, inner))); +} + +MlirAttribute mlirFlyIntTupleAttrBasisGet(MlirContext context, + MlirAttribute value, intptr_t nModes, + const int32_t *modes) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + IntAttr intValue = checked(value, ctx); + if (isNull(intValue) || (nModes > 0 && !modes)) + return {nullptr}; + return wrap(static_cast( + IntTupleAttr::get(intValue, llvm::ArrayRef(modes, nModes)))); +} + +MlirAttribute mlirFlyIntTupleAttrTupleGet(MlirContext context, + intptr_t nElements, + MlirAttribute const *elements) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx || (nElements > 0 && !elements)) + return {nullptr}; + llvm::SmallVector values; + values.reserve(nElements); + for (intptr_t i = 0; i < nElements; ++i) { + IntTupleAttr element = checked(elements[i], ctx); + if (isNull(element)) + return {nullptr}; + values.push_back(element); + } + return wrap(static_cast( + IntTupleAttr::get(ctx, mlir::ArrayAttr::get(ctx, values)))); +} + +MlirAttribute mlirFlyLayoutAttrGet(MlirContext context, MlirAttribute shape, + MlirAttribute stride) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + IntTupleAttr shapeAttr = checked(shape, ctx); + IntTupleAttr strideAttr = checked(stride, ctx); + if (isNull(shapeAttr) || isNull(strideAttr)) + return {nullptr}; + return wrap(static_cast( + LayoutAttr::get(ctx, shapeAttr, strideAttr))); +} + +MlirAttribute mlirFlyTileAttrGet(MlirContext context, MlirAttribute value) { + mlir::MLIRContext *ctx = loadFly(context); + mlir::Attribute inner = unwrap(value); + if (!ctx || isNull(inner) || inner.getContext() != ctx) + return {nullptr}; + return wrap(static_cast(TileAttr::get(ctx, inner))); +} + +MlirAttribute mlirFlyTileAttrModesGet(MlirContext context, intptr_t nModes, + MlirAttribute const *modes) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx || (nModes > 0 && !modes)) + return {nullptr}; + llvm::SmallVector values; + values.reserve(nModes); + for (intptr_t i = 0; i < nModes; ++i) { + mlir::Attribute mode = unwrap(modes[i]); + if (isNull(mode) || mode.getContext() != ctx) + return {nullptr}; + values.push_back(mode); + } + return wrap(static_cast( + TileAttr::get(ctx, mlir::ArrayAttr::get(ctx, values)))); +} + +MlirAttribute mlirFlySwizzleAttrGet(MlirContext context, int32_t mask, + int32_t base, int32_t shift) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap( + static_cast(SwizzleAttr::get(ctx, mask, base, shift))); +} + +MlirAttribute mlirFlyAlignAttrGet(MlirContext context, int32_t alignment) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast(AlignAttr::get(ctx, alignment))); +} + +MlirAttribute mlirFlyAddressSpaceAttrGet(MlirContext context, + int32_t addressSpace) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast( + AddressSpaceAttr::get(ctx, static_cast(addressSpace)))); +} + +MlirAttribute mlirFlyMmaOperandAttrGet(MlirContext context, int32_t operand) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast( + MmaOperandAttr::get(ctx, static_cast(operand)))); +} + +MlirAttribute mlirFlyGemmTraversalOrderAttrGet(MlirContext context, + int32_t order) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast(GemmTraversalOrderAttr::get( + ctx, static_cast(order)))); +} + +MlirType mlirFlyIntTupleTypeGet(MlirContext context, MlirAttribute attr) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + IntTupleAttr value = checked(attr, ctx); + if (isNull(value)) + return {nullptr}; + return wrap(static_cast(IntTupleType::get(value))); +} + +MlirAttribute mlirFlyIntTupleTypeGetAttr(MlirType type) { + return wrap( + static_cast(cast(unwrap(type)).getAttr())); +} + +intptr_t mlirFlyIntTupleTypeGetNumElements(MlirType type) { + return cast(unwrap(type)).rank(); +} + +MlirType mlirFlyIntTupleTypeGetElement(MlirType type, intptr_t pos) { + return wrap( + static_cast(cast(unwrap(type)).at(pos))); +} + +bool mlirFlyIntTupleTypeIsLeaf(MlirType type) { + return cast(unwrap(type)).isLeaf(); +} + +bool mlirFlyIntTupleTypeIsStatic(MlirType type) { + return cast(unwrap(type)).isStatic(); +} + +bool mlirFlyIntTupleTypeIsStaticLeaf(MlirType type) { + IntTupleAttr attr = cast(unwrap(type)).getAttr(); + return attr.isLeafInt() && attr.getLeafAsInt().isStatic(); +} + +bool mlirFlyIntTupleTypeIsNoneLeaf(MlirType type) { + return cast(unwrap(type)).getAttr().isLeafNone(); +} + +bool mlirFlyIntTupleTypeIsBasisLeaf(MlirType type) { + return cast(unwrap(type)).getAttr().isLeafBasis(); +} + +int64_t mlirFlyIntTupleTypeGetStaticValue(MlirType type) { + return cast(unwrap(type)).getAttr().getLeafAsInt().getValue(); +} + +MlirType mlirFlyLayoutTypeGet(MlirContext context, MlirAttribute attr) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + LayoutAttr value = checked(attr, ctx); + if (isNull(value)) + return {nullptr}; + return wrap(static_cast(LayoutType::get(value))); +} + +MlirAttribute mlirFlyLayoutTypeGetAttr(MlirType type) { + return wrap( + static_cast(cast(unwrap(type)).getAttr())); +} + +MlirType mlirFlyLayoutTypeGetShape(MlirType type) { + return wrap(static_cast( + IntTupleType::get(cast(unwrap(type)).getAttr().getShape()))); +} + +MlirType mlirFlyLayoutTypeGetStride(MlirType type) { + return wrap(static_cast( + IntTupleType::get(cast(unwrap(type)).getAttr().getStride()))); +} + +MlirAttribute mlirFlyComposedLayoutTypeGetAttr(MlirType type) { + return wrap(static_cast( + cast(unwrap(type)).getAttr())); +} + +MlirType mlirFlyTileTypeGet(MlirContext context, MlirAttribute attr) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + TileAttr value = checked(attr, ctx); + if (isNull(value)) + return {nullptr}; + return wrap(static_cast(TileType::get(value))); +} + +MlirAttribute mlirFlyTileTypeGetAttr(MlirType type) { + return wrap( + static_cast(cast(unwrap(type)).getAttr())); +} + +MlirType mlirFlySwizzleTypeGet(MlirContext context, MlirAttribute attr) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + SwizzleAttr value = checked(attr, ctx); + if (isNull(value)) + return {nullptr}; + return wrap(static_cast(SwizzleType::get(value))); +} + +MlirType mlirFlyPointerTypeGet(MlirContext context, MlirType elemTy, + MlirAttribute addressSpace, + MlirAttribute alignment, MlirAttribute swizzle) { + mlir::MLIRContext *ctx = loadFly(context); + mlir::Type element = unwrap(elemTy); + mlir::Attribute space = unwrap(addressSpace); + if (!ctx || isNull(element) || element.getContext() != ctx) + return {nullptr}; + if (isNull(space) || space.getContext() != ctx) + return {nullptr}; + AlignAttr align = alignment.ptr ? checked(alignment, ctx) + : AlignAttr::getTrivialAlignment(element); + SwizzleAttr sw = swizzle.ptr ? checked(swizzle, ctx) + : SwizzleAttr::getTrivialSwizzle(ctx); + if (isNull(align) || isNull(sw)) + return {nullptr}; + return wrap(static_cast( + PointerType::get(ctx, element, space, align, sw))); +} + +MlirType mlirFlyPointerTypeGetElemTy(MlirType type) { + return wrap(cast(unwrap(type)).getElemTy()); +} + +MlirAttribute mlirFlyPointerTypeGetAddressSpace(MlirType type) { + return wrap(cast(unwrap(type)).getAddressSpace()); +} + +MlirAttribute mlirFlyPointerTypeGetAlignment(MlirType type) { + return wrap(static_cast( + cast(unwrap(type)).getAlignment())); +} + +MlirAttribute mlirFlyPointerTypeGetSwizzle(MlirType type) { + return wrap(static_cast( + cast(unwrap(type)).getSwizzle())); +} + +MlirType mlirFlyMemRefTypeGet(MlirContext context, MlirType elemTy, + MlirAttribute addressSpace, MlirAttribute layout, + MlirAttribute alignment, MlirAttribute swizzle) { + mlir::MLIRContext *ctx = loadFly(context); + mlir::Type element = unwrap(elemTy); + mlir::Attribute space = unwrap(addressSpace); + mlir::Attribute lay = unwrap(layout); + if (!ctx || isNull(element) || element.getContext() != ctx) + return {nullptr}; + if (isNull(space) || space.getContext() != ctx) + return {nullptr}; + if (isNull(lay) || lay.getContext() != ctx) + return {nullptr}; + AlignAttr align = alignment.ptr ? checked(alignment, ctx) + : AlignAttr::getTrivialAlignment(element); + SwizzleAttr sw = swizzle.ptr ? checked(swizzle, ctx) + : SwizzleAttr::getTrivialSwizzle(ctx); + if (isNull(align) || isNull(sw)) + return {nullptr}; + return wrap(static_cast( + mlir::fly::MemRefType::get(ctx, element, space, lay, align, sw))); +} + +MlirType mlirFlyMemRefTypeGetElemTy(MlirType type) { + return wrap(cast(unwrap(type)).getElemTy()); +} + +MlirAttribute mlirFlyMemRefTypeGetAddressSpace(MlirType type) { + return wrap(cast(unwrap(type)).getAddressSpace()); +} + +MlirAttribute mlirFlyMemRefTypeGetLayout(MlirType type) { + return wrap(cast(unwrap(type)).getLayout()); +} + +MlirType mlirFlyCoordTensorTypeGet(MlirContext context, MlirAttribute base, + MlirAttribute layout) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + IntTupleAttr baseAttr = checked(base, ctx); + mlir::Attribute lay = unwrap(layout); + if (isNull(baseAttr) || isNull(lay) || lay.getContext() != ctx) + return {nullptr}; + return wrap( + static_cast(CoordTensorType::get(ctx, baseAttr, lay))); +} + +static mlir::Attribute outerLayoutAttr(mlir::Attribute layout) { + while (auto composed = mlir::dyn_cast(layout)) + layout = composed.getOuter(); + return layout; +} + +MlirType mlirFlyLayoutLikeTypeGetShape(MlirType type) { + mlir::Type ty = unwrap(type); + mlir::Attribute layout; + if (auto l = mlir::dyn_cast(ty)) { + layout = l.getAttr(); + } else if (auto c = mlir::dyn_cast(ty)) { + layout = c.getAttr(); + } else if (auto m = mlir::dyn_cast(ty)) { + layout = m.getLayout(); + } else { + layout = cast(ty).getLayout(); + } + return wrap(static_cast( + IntTupleType::get(cast(outerLayoutAttr(layout)).getShape()))); +} + +MlirType mlirFlyCopyAtomTypeGet(MlirContext context, MlirType copyOp, + int32_t valBits) { + mlir::MLIRContext *ctx = loadFly(context); + mlir::Type op = unwrap(copyOp); + if (!ctx || isNull(op) || op.getContext() != ctx) + return {nullptr}; + return wrap(static_cast(CopyAtomType::get(ctx, op, valBits))); +} + +MlirType mlirFlyMmaAtomTypeGet(MlirContext context, MlirType mmaOp) { + mlir::MLIRContext *ctx = loadFly(context); + mlir::Type op = unwrap(mmaOp); + if (!ctx || isNull(op) || op.getContext() != ctx) + return {nullptr}; + return wrap(static_cast(MmaAtomType::get(ctx, op))); +} + +#define FLY_ATOM_LAYOUT(Name, AtomType, getter) \ + MlirType Name(MlirType type) { \ + return wrap(static_cast(LayoutType::get( \ + cast(cast(unwrap(type)).getter())))); \ + } + +FLY_ATOM_LAYOUT(mlirFlyCopyAtomTypeGetThrLayout, CopyAtomType, getThrLayout) +FLY_ATOM_LAYOUT(mlirFlyCopyAtomTypeGetThrValLayoutSrc, CopyAtomType, + getThrValLayoutSrc) +FLY_ATOM_LAYOUT(mlirFlyCopyAtomTypeGetThrValLayoutDst, CopyAtomType, + getThrValLayoutDst) +FLY_ATOM_LAYOUT(mlirFlyCopyAtomTypeGetThrValLayoutRef, CopyAtomType, + getThrValLayoutRef) +FLY_ATOM_LAYOUT(mlirFlyMmaAtomTypeGetThrLayout, MmaAtomType, getThrLayout) +FLY_ATOM_LAYOUT(mlirFlyMmaAtomTypeGetThrValLayoutA, MmaAtomType, + getThrValLayoutA) +FLY_ATOM_LAYOUT(mlirFlyMmaAtomTypeGetThrValLayoutB, MmaAtomType, + getThrValLayoutB) +FLY_ATOM_LAYOUT(mlirFlyMmaAtomTypeGetThrValLayoutC, MmaAtomType, + getThrValLayoutC) + +#undef FLY_ATOM_LAYOUT + +MlirType mlirFlyMmaAtomTypeGetShapeMNK(MlirType type) { + return wrap(static_cast(IntTupleType::get( + cast(cast(unwrap(type)).getShapeMNK())))); +} + +MlirType mlirFlyTiledCopyTypeGetTiledThrValLayoutSrc(MlirType type) { + auto ty = cast(unwrap(type)); + return wrap( + static_cast(LayoutType::get(tiledCopyGetTiledThrValLayoutSrc( + cast(ty.getCopyAtom()), ty.getLayoutThrVal().getAttr(), + ty.getTileMN().getAttr())))); +} + +MlirType mlirFlyTiledCopyTypeGetTiledThrValLayoutDst(MlirType type) { + auto ty = cast(unwrap(type)); + return wrap( + static_cast(LayoutType::get(tiledCopyGetTiledThrValLayoutDst( + cast(ty.getCopyAtom()), ty.getLayoutThrVal().getAttr(), + ty.getTileMN().getAttr())))); +} + +MlirType mlirFlyTiledMmaTypeGetTileSizeMNK(MlirType type) { + auto ty = cast(unwrap(type)); + return wrap(static_cast(IntTupleType::get(tiledMmaGetTileSizeMNK( + cast(ty.getMmaAtom()), ty.getAtomLayout().getAttr(), + ty.getPermutation().getAttr())))); +} + +MlirType mlirFlyTiledMmaTypeGetThrLayoutVMNK(MlirType type) { + auto ty = cast(unwrap(type)); + return wrap(static_cast(LayoutType::get(tiledMmaGetThrLayoutVMNK( + cast(ty.getMmaAtom()), ty.getAtomLayout().getAttr())))); +} + +static MlirType tiledMmaTiledThrVal(MlirType tiledMma, MmaOperand operand) { + auto ty = cast(unwrap(tiledMma)); + return wrap( + static_cast(LayoutType::get(tiledMmaGetTiledThrValLayout( + cast(ty.getMmaAtom()), ty.getAtomLayout().getAttr(), + ty.getPermutation().getAttr(), operand)))); +} + +MlirType mlirFlyTiledMmaTypeGetTiledThrValLayoutA(MlirType type) { + return tiledMmaTiledThrVal(type, MmaOperand::A); +} + +MlirType mlirFlyTiledMmaTypeGetTiledThrValLayoutB(MlirType type) { + return tiledMmaTiledThrVal(type, MmaOperand::B); +} + +MlirType mlirFlyTiledMmaTypeGetTiledThrValLayoutC(MlirType type) { + return tiledMmaTiledThrVal(type, MmaOperand::C); +} + +MlirType mlirFlyCopyOpUniversalCopyTypeGet(MlirContext context, + int32_t bitSize) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap( + static_cast(CopyOpUniversalCopyType::get(ctx, bitSize))); +} diff --git a/mlir/dialects/fly/capi.h b/mlir/dialects/fly/capi.h new file mode 100644 index 00000000..50fb0f49 --- /dev/null +++ b/mlir/dialects/fly/capi.h @@ -0,0 +1,168 @@ +//===- capi.h - C API for the fly dialect -------------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_FLY_CAPI_H +#define MLIR_DIALECTS_FLY_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Fly, fly); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyInt(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntAttrStaticGet(MlirContext context, + int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntAttrDynamicGet(MlirContext context, + int32_t width, + int32_t divisibility); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntAttrNoneGet(MlirContext context); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyIntTuple(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntTupleAttrGet(MlirContext context, + MlirAttribute value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirFlyIntTupleAttrBasisGet(MlirContext context, MlirAttribute value, + intptr_t nModes, const int32_t *modes); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntTupleAttrTupleGet( + MlirContext context, intptr_t nElements, MlirAttribute const *elements); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyLayoutAttrGet(MlirContext context, + MlirAttribute shape, + MlirAttribute stride); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyTile(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyTileAttrGet(MlirContext context, + MlirAttribute value); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyTileAttrModesGet( + MlirContext context, intptr_t nModes, MlirAttribute const *modes); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlySwizzle(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlySwizzleAttrGet(MlirContext context, + int32_t mask, + int32_t base, + int32_t shift); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyAlign(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyAlignAttrGet(MlirContext context, + int32_t alignment); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyAddressSpace(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirFlyAddressSpaceAttrGet(MlirContext context, int32_t addressSpace); + +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyMmaOperandAttrGet(MlirContext context, + int32_t operand); +MLIR_CAPI_EXPORTED MlirAttribute +mlirFlyGemmTraversalOrderAttrGet(MlirContext context, int32_t order); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyIntTuple(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyIntTupleTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntTupleTypeGetAttr(MlirType type); +MLIR_CAPI_EXPORTED intptr_t mlirFlyIntTupleTypeGetNumElements(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyIntTupleTypeGetElement(MlirType type, + intptr_t pos); +MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsLeaf(MlirType type); +MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsStatic(MlirType type); +MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsStaticLeaf(MlirType type); +MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsNoneLeaf(MlirType type); +MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsBasisLeaf(MlirType type); +MLIR_CAPI_EXPORTED int64_t mlirFlyIntTupleTypeGetStaticValue(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyLayout(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyLayoutTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyLayoutTypeGetAttr(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyLayoutTypeGetShape(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyLayoutTypeGetStride(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyComposedLayout(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirFlyComposedLayoutTypeGetAttr(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyTile(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyTileTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyTileTypeGetAttr(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlySwizzle(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlySwizzleTypeGet(MlirContext context, + MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyPointer(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyPointerTypeGet(MlirContext context, + MlirType elemTy, + MlirAttribute addressSpace, + MlirAttribute alignment, + MlirAttribute swizzle); +MLIR_CAPI_EXPORTED MlirType mlirFlyPointerTypeGetElemTy(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirFlyPointerTypeGetAddressSpace(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyPointerTypeGetAlignment(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyPointerTypeGetSwizzle(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyMemRef(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyMemRefTypeGet( + MlirContext context, MlirType elemTy, MlirAttribute addressSpace, + MlirAttribute layout, MlirAttribute alignment, MlirAttribute swizzle); +MLIR_CAPI_EXPORTED MlirType mlirFlyMemRefTypeGetElemTy(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirFlyMemRefTypeGetAddressSpace(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyMemRefTypeGetLayout(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyCoordTensor(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyCoordTensorTypeGet(MlirContext context, + MlirAttribute base, + MlirAttribute layout); + +MLIR_CAPI_EXPORTED MlirType mlirFlyLayoutLikeTypeGetShape(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyCopyAtom(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyCopyAtomTypeGet(MlirContext context, + MlirType copyOp, + int32_t valBits); +MLIR_CAPI_EXPORTED MlirType mlirFlyCopyAtomTypeGetThrLayout(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyCopyAtomTypeGetThrValLayoutSrc(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyCopyAtomTypeGetThrValLayoutDst(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyCopyAtomTypeGetThrValLayoutRef(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyMmaAtom(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGet(MlirContext context, + MlirType mmaOp); +MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetThrLayout(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetShapeMNK(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetThrValLayoutA(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetThrValLayoutB(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetThrValLayoutC(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyTiledCopy(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyTiledCopyTypeGetTiledThrValLayoutSrc(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyTiledCopyTypeGetTiledThrValLayoutDst(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyTiledMma(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyTiledMmaTypeGetTileSizeMNK(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyTiledMmaTypeGetThrLayoutVMNK(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyTiledMmaTypeGetTiledThrValLayoutA(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyTiledMmaTypeGetTiledThrValLayoutB(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyTiledMmaTypeGetTiledThrValLayoutC(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyCopyOpUniversalCopy(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyCopyOpUniversalCopyTypeGet(MlirContext context, int32_t bitSize); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_FLY_CAPI_H diff --git a/mlir/dialects/fly/fly.zig b/mlir/dialects/fly/fly.zig index 8bfafbce..15530388 100644 --- a/mlir/dialects/fly/fly.zig +++ b/mlir/dialects/fly/fly.zig @@ -2,25 +2,17 @@ const std = @import("std"); const c = @import("c"); const mlir = @import("mlir"); -const stdx = @import("stdx"); pub const attributes = @import("attributes.zig"); pub const ops = @import("ops.zig"); -pub const rocdl = @import("rocdl.zig"); pub const types = @import("types.zig"); pub const AddressSpace = attributes.AddressSpace; pub const GemmTraversalOrder = attributes.GemmTraversalOrder; pub const MmaOperand = attributes.MmaOperand; -/// `gpu`/`rocdl` for `gpu.func` kernels and the ROCDL escape hatches; `ub` -/// because fly ops may fold to `ub.poison`. -pub const dialects_needed = [_][]const u8{ "func", "arith", "scf", "math", "vector", "memref", "llvm", "gpu", "rocdl", "ub", "fly", "fly_rocdl" }; - -pub fn insertDialects(registry: *mlir.DialectRegistry) void { - inline for (dialects_needed) |d| { - mlir.DialectHandle.fromString(d).insertDialect(registry); - } +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("fly"); } /// `Operation.make` that names the op and its operand types on failure: a @@ -105,14 +97,14 @@ pub fn layoutLikeShape(ty: *const mlir.Type) *const types.IntTupleType { } pub fn elemType(ty: *const mlir.Type) *const mlir.Type { - if (ty.isA(types.MemRefType)) |m| return m.getElemTy(); - if (ty.isA(types.PointerType)) |p| return p.getElemTy(); + if (ty.isA(types.MemRefType)) |m| return m.elemTy(); + if (ty.isA(types.PointerType)) |p| return p.elemTy(); std.debug.panic("fly: `{f}` has no element type", .{ty}); } pub fn addressSpace(ty: *const mlir.Type) *const mlir.Attribute { - if (ty.isA(types.MemRefType)) |m| return m.getAddressSpace(); - if (ty.isA(types.PointerType)) |p| return p.getAddressSpace(); + if (ty.isA(types.MemRefType)) |m| return m.addressSpace(); + if (ty.isA(types.PointerType)) |p| return p.addressSpace(); std.debug.panic("fly: `{f}` has no address space", .{ty}); } @@ -121,243 +113,8 @@ pub fn ptrWithElem(ctx: *mlir.Context, ptr: *const mlir.Type, elem: *const mlir. const p = expect(ptr, .ptr); return types.PointerType.get(ctx, .{ .elemTy = elem, - .addressSpace = p.getAddressSpace(), - .alignment = p.getAlignment(), - .swizzle = p.getSwizzle(), + .addressSpace = p.addressSpace(), + .alignment = p.alignment(), + .swizzle = p.swizzle(), }); } - -test { - std.testing.refAllDecls(@This()); - inline for (.{ types, attributes, ops, rocdl, rocdl.ops }) |module| { - std.testing.refAllDecls(module); - inline for (comptime std.meta.declarations(module)) |decl| { - const value = @field(module, decl); - if (@TypeOf(value) == type and @typeInfo(value) == .@"opaque") std.testing.refAllDecls(value); - } - } -} - -fn testContext() !*mlir.Context { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - insertDialects(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - return ctx; -} - -fn expectPrints(expected: []const u8, value: anytype) !void { - var buf: [512]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{value}); - try std.testing.expectEqualStrings(expected, w.buffered()); -} - -test "fly dialects register" { - const ctx = try testContext(); - defer ctx.deinit(); - - try std.testing.expect(ctx.isRegisteredOperation("fly.make_int_tuple")); - try std.testing.expect(ctx.isRegisteredOperation("fly.make_layout")); - try std.testing.expect(ctx.isRegisteredOperation("fly.ptr.load")); - try std.testing.expect(ctx.isRegisteredOperation("fly.copy_atom_call")); - try std.testing.expect(ctx.isRegisteredOperation("gpu.func")); - try std.testing.expect(ctx.isRegisteredOperation("rocdl.sched.barrier")); - try std.testing.expect(ctx.isRegisteredOperation("ub.poison")); -} - -test "every generated operation builder names a registered operation" { - const ctx = try testContext(); - defer ctx.deinit(); - - for (ops.names ++ rocdl.ops.names) |name| { - if (!ctx.isRegisteredOperation(name)) { - std.debug.print("`{s}` is not registered\n", .{name}); - return error.TestUnexpectedResult; - } - } -} - -test "fly types parse and print round-trip" { - const ctx = try testContext(); - defer ctx.deinit(); - - const cases = [_][]const u8{ - "!fly.int_tuple<((2, 4), 8)>", - "!fly.int_tuple<((1, 4), (32, 64))>", - "!fly.layout<((2, 3), 3) : ((57, 69), 19)>", - "!fly.memref", - "!fly.memref", - "!fly.ptr", - "!fly.ptr", - "!fly.ptr", - "!fly.memref", - "!fly_rocdl.cdna3.buffer_copy<128>", - "!fly_rocdl.cdna3.buffer_copy_lds<32>", - "!fly_rocdl.cdna3.mfma<16x16x16, (f16, f16) -> f32>", - "!fly.copy_atom, 32>", - "!fly.tile<[8|64]>", - "!fly.coord_tensor<(0,0), (100,1000):(1E0,1E1)>", - }; - for (cases) |src| { - const ty = mlir.Type.parse(ctx, src) catch |err| { - std.debug.print("failed to parse `{s}`: {}\n", .{ src, err }); - return err; - }; - var buf: [512]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{ty}); - const ty2 = try mlir.Type.parse(ctx, w.buffered()); - try std.testing.expect(ty.eql(ty2)); - } -} - -test "types and attributes are built through the C API" { - const ctx = try testContext(); - defer ctx.deinit(); - - const two = try attributes.IntTupleAttr.getStatic(ctx, 2); - const four = try attributes.IntTupleAttr.getStatic(ctx, 4); - const one = try attributes.IntTupleAttr.getStatic(ctx, 1); - const shape = try attributes.IntTupleAttr.getTuple(ctx, &.{ two, four }); - const stride = try attributes.IntTupleAttr.getTuple(ctx, &.{ four, one }); - const layout = try types.LayoutType.get(ctx, .{ .attr = try .get(ctx, .{ .shape = shape, .stride = stride }) }); - try expectPrints("!fly.layout<(2,4):(4,1)>", layout); - try expectPrints("!fly.int_tuple<(2,4)>", layout.getShape()); - try std.testing.expectEqual(@as(usize, 2), layout.getShape().getNumElements()); - try std.testing.expectEqual(types.IntTupleType.Leaf{ .static = 4 }, layout.getShape().getElement(1).getLeaf()); - - const global = try attributes.AddressSpaceAttr.get(ctx, .{ .addressSpace = .global }); - const memref = try types.MemRefType.get(ctx, .{ - .elemTy = .float(ctx, .f16), - .addressSpace = global.attribute(), - .layout = layout.getAttr().attribute(), - }); - try expectPrints("!fly.memref", memref); - try expectPrints("!fly.int_tuple<(2,4)>", layoutLikeShape(memref.type_())); - - const shared = try attributes.AddressSpaceAttr.get(ctx, .{ .addressSpace = .shared }); - const ptr = try types.PointerType.get(ctx, .{ - .elemTy = .int(ctx, .i8), - .addressSpace = shared.attribute(), - .alignment = try .get(ctx, .{ .alignment = 16 }), - }); - try expectPrints("!fly.ptr>", ptr); - try expectPrints("!fly.ptr>", try ptrWithElem(ctx, ptr.type_(), .float(ctx, .f32))); - - const tile = try types.TileType.get(ctx, .{ .attr = try .getModes(ctx, &.{ - (try attributes.IntAttr.getStatic(ctx, 8)).attribute(), - (try attributes.IntAttr.getStatic(ctx, 64)).attribute(), - }) }); - try expectPrints("!fly.tile<[8|64]>", tile); - - const copy_atom = try types.CopyAtomType.get(ctx, .{ - .copyOp = (try types.CopyOpUniversalCopyType.get(ctx, .{ .bitSize = 128 })).type_(), - .valBits = 32, - }); - try expectPrints("!fly.copy_atom, 32>", copy_atom); - - const buffer_desc = try rocdl.BufferDescAddressAttr.get(ctx); - try expectPrints("!fly.ptr", try types.PointerType.get(ctx, .{ - .elemTy = .float(ctx, .f16), - .addressSpace = buffer_desc.attribute(), - })); -} - -test "fly enum attributes" { - const ctx = try testContext(); - defer ctx.deinit(); - _ = try attributes.mmaOperandAttr(ctx, .a); - _ = try attributes.gemmTraversalOrderAttr(ctx, .kmn); -} - -test "inferred fly ops compute layout algebra" { - const ctx = try testContext(); - defer ctx.deinit(); - const loc: *const mlir.Location = .unknown(ctx); - const module: *mlir.Module = .init(loc); - defer module.deinit(); - const block = module.body(); - - // raked_product((8,16):(16,1), (1,4):(1,1)) — the vectorAdd TV layout. - const thr = ops.static(ctx, try mlir.Type.parse(ctx, "!fly.layout<(8,16):(16,1)>"), loc).appendTo(block); - const val = ops.static(ctx, try mlir.Type.parse(ctx, "!fly.layout<(1,4):(1,1)>"), loc).appendTo(block); - const mn = ops.raked_product(ctx, thr.result(0), val.result(0), loc).appendTo(block); - const shape = ops.get_shape(ctx, mn.result(0), loc).appendTo(block); - const tiler = ops.int_tuple_product_each(ctx, shape.result(0), loc).appendTo(block); - - try expectPrints("!fly.int_tuple<(8,64)>", tiler.result(0).type_()); - try std.testing.expect(module.operation().verify()); -} - -test "atom traits" { - const ctx = try testContext(); - defer ctx.deinit(); - - // MFMA 16x16x4 f32 tiled over a (2,2,1) atom layout. - const tm = expect(try mlir.Type.parse(ctx, "!fly.tiled_mma f32>>, <(2,2,1):(1,2,0)>>"), .tiled_mma); - try expectPrints("!fly.int_tuple<(32,32,4)>", tm.getTileSizeMNK()); - _ = tm.getTiledThrValLayoutA(); - _ = tm.getThrLayoutVMNK(); - - const atom = try types.MmaAtomType.get(ctx, .{ .mmaOp = (try rocdl.MmaOpCDNA3MFMAType.get(ctx, .{ - .m = 16, - .n = 16, - .k = 4, - .elemTyA = .float(ctx, .f32), - .elemTyB = .float(ctx, .f32), - .elemTyAcc = .float(ctx, .f32), - })).type_() }); - try expectPrints("!fly.mma_atom f32>>", atom); - try expectPrints("!fly.int_tuple<(16,16,4)>", atom.getShapeMNK()); - try expectPrints("!fly.layout<64:1>", atom.getThrLayout()); - - const wmma = try types.MmaAtomType.get(ctx, .{ .mmaOp = (try rocdl.MmaOpGFX11WMMAType.get(ctx, .{ - .m = 16, - .n = 16, - .k = 16, - .elemTyA = .float(ctx, .f16), - .elemTyB = .float(ctx, .f16), - .elemTyAcc = .float(ctx, .f32), - })).type_() }); - try expectPrints("!fly.layout<32:1>", wmma.getThrLayout()); - - const ca = expect(try mlir.Type.parse(ctx, "!fly.copy_atom, 32>"), .copy_atom); - _ = ca.getThrValLayoutSrc(); -} - -test "type kinds" { - const ctx = try testContext(); - defer ctx.deinit(); - const cases = .{ - .{ "!fly.memref", TypeKind.memref }, - .{ "!fly.ptr", TypeKind.ptr }, - .{ "!fly.int_tuple<(2,4)>", TypeKind.int_tuple }, - .{ "!fly.layout<(8,4):(4,1)>", TypeKind.layout }, - .{ "!fly.tile<[8|4]>", TypeKind.tile }, - .{ "!fly.copy_atom, 32>", TypeKind.copy_atom }, - .{ "!fly.mma_atom f32>>", TypeKind.mma_atom }, - }; - inline for (cases) |case| { - try std.testing.expectEqual(case[1], typeKind(try mlir.Type.parse(ctx, case[0]))); - } - try std.testing.expectEqual(TypeKind.other, typeKind(.float(ctx, .f32))); -} - -test "structural readers" { - const ctx = try testContext(); - defer ctx.deinit(); - const it = expect(try mlir.Type.parse(ctx, "!fly.int_tuple<((2,4),?)>"), .int_tuple); - try std.testing.expectEqual(@as(usize, 2), it.getNumElements()); - try std.testing.expect(!it.isLeaf()); - try std.testing.expect(!it.isStatic()); - const inner = it.getElement(0); - try std.testing.expectEqual(@as(usize, 2), inner.getNumElements()); - try std.testing.expectEqual(types.IntTupleType.Leaf{ .static = 4 }, inner.getElement(1).getLeaf()); - try std.testing.expectEqual(types.IntTupleType.Leaf.dynamic, it.getElement(1).getLeaf()); - - const m = try mlir.Type.parse(ctx, "!fly.memref o 0 o (64,32):(32,1), align<16>>"); - try expectPrints("!fly.int_tuple<(64,32)>", layoutLikeShape(m)); - try expectPrints("bf16", elemType(m)); -} diff --git a/mlir/dialects/fly/fly_capi.cc b/mlir/dialects/fly/fly_capi.cc deleted file mode 100644 index 0f2f3c11..00000000 --- a/mlir/dialects/fly/fly_capi.cc +++ /dev/null @@ -1,519 +0,0 @@ -#include "fly_capi.h" - -#include "TiledOpTraits.h" -#include "flydsl/Dialect/Fly/IR/FlyDialect.h" -#include "flydsl/Dialect/FlyROCDL/IR/Dialect.h" -#include "mlir/CAPI/IR.h" -#include "mlir/CAPI/Registration.h" -#include "mlir/Dialect/UB/IR/UBOps.h" - -MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Fly, fly, mlir::fly::FlyDialect) -MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(FlyROCDL, fly_rocdl, - mlir::fly_rocdl::FlyROCDLDialect) -MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(UB, ub, mlir::ub::UBDialect) - -using mlir::cast; -using mlir::isa; -using namespace mlir::fly; - -namespace { - -mlir::MLIRContext *loadFly(MlirContext ctx) { - mlir::MLIRContext *context = unwrap(ctx); - if (context) context->getOrLoadDialect(); - return context; -} - -mlir::MLIRContext *loadFlyROCDL(MlirContext ctx) { - mlir::MLIRContext *context = unwrap(ctx); - if (context) context->getOrLoadDialect(); - return context; -} - -/// `!attr` and `attr == other` are value arithmetic on a fly IntAttr, not a -/// null test: every handle check here goes through these two helpers. -template -bool isNull(T value) { - return !static_cast(value); -} - -template -T checked(Handle handle, mlir::MLIRContext *ctx) { - auto value = mlir::dyn_cast_if_present(unwrap(handle)); - if (isNull(value) || value.getContext() != ctx) return T{}; - return value; -} - -template -MlirType mmaOpGet(MlirContext ctx, int32_t m, int32_t n, int32_t k, MlirType elemTyA, MlirType elemTyB, MlirType elemTyAcc, Flags... flags) { - mlir::MLIRContext *context = loadFlyROCDL(ctx); - mlir::Type a = unwrap(elemTyA), b = unwrap(elemTyB), acc = unwrap(elemTyAcc); - if (!context || isNull(a) || isNull(b) || isNull(acc)) return {nullptr}; - if (a.getContext() != context || b.getContext() != context || - acc.getContext() != context) - return {nullptr}; - return wrap(static_cast( - Type::get(context, m, n, k, a, b, acc, flags...))); -} - - -} // namespace - -extern "C" { - -#define FLY_ATTR_ISA(Name, Attr) \ - bool mlirAttributeIsAFly##Name(MlirAttribute a) { \ - return mlir::isa_and_nonnull(unwrap(a)); \ - } - -FLY_ATTR_ISA(Int, IntAttr) -FLY_ATTR_ISA(IntTuple, IntTupleAttr) -FLY_ATTR_ISA(Layout, LayoutAttr) -FLY_ATTR_ISA(Tile, TileAttr) -FLY_ATTR_ISA(Swizzle, SwizzleAttr) -FLY_ATTR_ISA(Align, AlignAttr) -FLY_ATTR_ISA(AddressSpace, AddressSpaceAttr) - -#undef FLY_ATTR_ISA - -#define FLY_TYPE_ISA(Name, Type) \ - bool mlirTypeIsAFly##Name(MlirType t) { \ - return mlir::isa_and_nonnull(unwrap(t)); \ - } - -FLY_TYPE_ISA(IntTuple, IntTupleType) -FLY_TYPE_ISA(Layout, LayoutType) -FLY_TYPE_ISA(ComposedLayout, ComposedLayoutType) -FLY_TYPE_ISA(Tile, TileType) -FLY_TYPE_ISA(Swizzle, SwizzleType) -FLY_TYPE_ISA(Pointer, PointerType) -FLY_TYPE_ISA(MemRef, mlir::fly::MemRefType) -FLY_TYPE_ISA(CoordTensor, CoordTensorType) -FLY_TYPE_ISA(CopyAtom, CopyAtomType) -FLY_TYPE_ISA(MmaAtom, MmaAtomType) -FLY_TYPE_ISA(TiledCopy, TiledCopyType) -FLY_TYPE_ISA(TiledMma, TiledMmaType) -FLY_TYPE_ISA(CopyOpUniversalCopy, CopyOpUniversalCopyType) -FLY_TYPE_ISA(ROCDLCopyOpCDNA3BufferCopy, mlir::fly_rocdl::CopyOpCDNA3BufferCopyType) -FLY_TYPE_ISA(ROCDLCopyOpCDNA3BufferCopyLDS, mlir::fly_rocdl::CopyOpCDNA3BufferCopyLDSType) -FLY_TYPE_ISA(ROCDLMmaOpCDNA3MFMA, mlir::fly_rocdl::MmaOpCDNA3_MFMAType) -FLY_TYPE_ISA(ROCDLMmaOpGFX11WMMA, mlir::fly_rocdl::MmaOpGFX11_WMMAType) -FLY_TYPE_ISA(ROCDLMmaOpGFX120XWMMA, mlir::fly_rocdl::MmaOpGFX120X_WMMAType) - -#undef FLY_TYPE_ISA - -MlirAttribute mlirFlyIntAttrGetStatic(MlirContext ctx, int32_t value) { - mlir::MLIRContext *context = loadFly(ctx); - if (!context) return {nullptr}; - return wrap(static_cast(IntAttr::getStatic(context, value))); -} - -MlirAttribute mlirFlyIntAttrGetDynamic(MlirContext ctx, int32_t width, int32_t divisibility) { - mlir::MLIRContext *context = loadFly(ctx); - if (!context) return {nullptr}; - return wrap(static_cast(IntAttr::getDynamic(context, width, divisibility))); -} - -MlirAttribute mlirFlyIntAttrGetNone(MlirContext ctx) { - mlir::MLIRContext *context = loadFly(ctx); - if (!context) return {nullptr}; - return wrap(static_cast(IntAttr::getNone(context))); -} - -MlirAttribute mlirFlyIntTupleAttrGet(MlirContext ctx, MlirAttribute value) { - mlir::MLIRContext *context = loadFly(ctx); - mlir::Attribute inner = unwrap(value); - if (!context || isNull(inner) || inner.getContext() != context) return {nullptr}; - return wrap(static_cast(IntTupleAttr::get(context, inner))); -} - -MlirAttribute mlirFlyIntTupleAttrGetBasis(MlirContext ctx, MlirAttribute value, intptr_t nModes, const int32_t *modes) { - mlir::MLIRContext *context = loadFly(ctx); - if (!context) return {nullptr}; - IntAttr intValue = checked(value, context); - if (isNull(intValue) || (nModes > 0 && !modes)) return {nullptr}; - return wrap(static_cast( - IntTupleAttr::get(intValue, llvm::ArrayRef(modes, nModes)))); -} - -MlirAttribute mlirFlyIntTupleAttrGetTuple(MlirContext ctx, intptr_t nElements, MlirAttribute const *elements) { - mlir::MLIRContext *context = loadFly(ctx); - if (!context || (nElements > 0 && !elements)) return {nullptr}; - llvm::SmallVector values; - values.reserve(nElements); - for (intptr_t i = 0; i < nElements; ++i) { - IntTupleAttr element = checked(elements[i], context); - if (isNull(element)) return {nullptr}; - values.push_back(element); - } - return wrap(static_cast( - IntTupleAttr::get(context, mlir::ArrayAttr::get(context, values)))); -} - -MlirAttribute mlirFlyLayoutAttrGet(MlirContext ctx, MlirAttribute shape, MlirAttribute stride) { - mlir::MLIRContext *context = loadFly(ctx); - if (!context) return {nullptr}; - IntTupleAttr shapeAttr = checked(shape, context); - IntTupleAttr strideAttr = checked(stride, context); - if (isNull(shapeAttr) || isNull(strideAttr)) return {nullptr}; - return wrap(static_cast(LayoutAttr::get(context, shapeAttr, strideAttr))); -} - -MlirAttribute mlirFlyTileAttrGet(MlirContext ctx, MlirAttribute value) { - mlir::MLIRContext *context = loadFly(ctx); - mlir::Attribute inner = unwrap(value); - if (!context || isNull(inner) || inner.getContext() != context) return {nullptr}; - return wrap(static_cast(TileAttr::get(context, inner))); -} - -MlirAttribute mlirFlyTileAttrGetModes(MlirContext ctx, intptr_t nModes, MlirAttribute const *modes) { - mlir::MLIRContext *context = loadFly(ctx); - if (!context || (nModes > 0 && !modes)) return {nullptr}; - llvm::SmallVector values; - values.reserve(nModes); - for (intptr_t i = 0; i < nModes; ++i) { - mlir::Attribute mode = unwrap(modes[i]); - if (isNull(mode) || mode.getContext() != context) return {nullptr}; - values.push_back(mode); - } - return wrap(static_cast( - TileAttr::get(context, mlir::ArrayAttr::get(context, values)))); -} - -MlirAttribute mlirFlySwizzleAttrGet(MlirContext ctx, int32_t mask, int32_t base, int32_t shift) { - mlir::MLIRContext *context = loadFly(ctx); - if (!context) return {nullptr}; - return wrap(static_cast(SwizzleAttr::get(context, mask, base, shift))); -} - -MlirAttribute mlirFlyAlignAttrGet(MlirContext ctx, int32_t alignment) { - mlir::MLIRContext *context = loadFly(ctx); - if (!context) return {nullptr}; - return wrap(static_cast(AlignAttr::get(context, alignment))); -} - -MlirAttribute mlirFlyAddressSpaceAttrGet(MlirContext ctx, int32_t addressSpace) { - mlir::MLIRContext *context = loadFly(ctx); - if (!context) return {nullptr}; - return wrap(static_cast( - AddressSpaceAttr::get(context, static_cast(addressSpace)))); -} - -MlirAttribute mlirFlyMmaOperandAttrGet(MlirContext ctx, int32_t operand) { - mlir::MLIRContext *context = loadFly(ctx); - if (!context) return {nullptr}; - return wrap(static_cast( - MmaOperandAttr::get(context, static_cast(operand)))); -} - -MlirAttribute mlirFlyGemmTraversalOrderAttrGet(MlirContext ctx, int32_t order) { - mlir::MLIRContext *context = loadFly(ctx); - if (!context) return {nullptr}; - return wrap(static_cast(GemmTraversalOrderAttr::get( - context, static_cast(order)))); -} - -MlirType mlirFlyIntTupleTypeGet(MlirContext ctx, MlirAttribute attr) { - mlir::MLIRContext *context = loadFly(ctx); - if (!context) return {nullptr}; - IntTupleAttr value = checked(attr, context); - if (isNull(value)) return {nullptr}; - return wrap(static_cast(IntTupleType::get(value))); -} - -MlirAttribute mlirFlyIntTupleTypeGetAttr(MlirType type) { - return wrap(static_cast(cast(unwrap(type)).getAttr())); -} - -intptr_t mlirFlyIntTupleTypeGetNumElements(MlirType type) { - return cast(unwrap(type)).rank(); -} - -MlirType mlirFlyIntTupleTypeGetElement(MlirType type, intptr_t pos) { - return wrap(static_cast(cast(unwrap(type)).at(pos))); -} - -bool mlirFlyIntTupleTypeIsLeaf(MlirType type) { - return cast(unwrap(type)).isLeaf(); -} - -bool mlirFlyIntTupleTypeIsStatic(MlirType type) { - return cast(unwrap(type)).isStatic(); -} - -bool mlirFlyIntTupleTypeIsStaticLeaf(MlirType type) { - IntTupleAttr attr = cast(unwrap(type)).getAttr(); - return attr.isLeafInt() && attr.getLeafAsInt().isStatic(); -} - -bool mlirFlyIntTupleTypeIsNoneLeaf(MlirType type) { - return cast(unwrap(type)).getAttr().isLeafNone(); -} - -bool mlirFlyIntTupleTypeIsBasisLeaf(MlirType type) { - return cast(unwrap(type)).getAttr().isLeafBasis(); -} - -int64_t mlirFlyIntTupleTypeGetStaticValue(MlirType type) { - return cast(unwrap(type)).getAttr().getLeafAsInt().getValue(); -} - -MlirType mlirFlyLayoutTypeGet(MlirContext ctx, MlirAttribute attr) { - mlir::MLIRContext *context = loadFly(ctx); - if (!context) return {nullptr}; - LayoutAttr value = checked(attr, context); - if (isNull(value)) return {nullptr}; - return wrap(static_cast(LayoutType::get(value))); -} - -MlirAttribute mlirFlyLayoutTypeGetAttr(MlirType type) { - return wrap(static_cast(cast(unwrap(type)).getAttr())); -} - -MlirType mlirFlyLayoutTypeGetShape(MlirType type) { - return wrap(static_cast( - IntTupleType::get(cast(unwrap(type)).getAttr().getShape()))); -} - -MlirType mlirFlyLayoutTypeGetStride(MlirType type) { - return wrap(static_cast( - IntTupleType::get(cast(unwrap(type)).getAttr().getStride()))); -} - -MlirAttribute mlirFlyComposedLayoutTypeGetAttr(MlirType type) { - return wrap(static_cast(cast(unwrap(type)).getAttr())); -} - -MlirType mlirFlyTileTypeGet(MlirContext ctx, MlirAttribute attr) { - mlir::MLIRContext *context = loadFly(ctx); - if (!context) return {nullptr}; - TileAttr value = checked(attr, context); - if (isNull(value)) return {nullptr}; - return wrap(static_cast(TileType::get(value))); -} - -MlirAttribute mlirFlyTileTypeGetAttr(MlirType type) { - return wrap(static_cast(cast(unwrap(type)).getAttr())); -} - -MlirType mlirFlySwizzleTypeGet(MlirContext ctx, MlirAttribute attr) { - mlir::MLIRContext *context = loadFly(ctx); - if (!context) return {nullptr}; - SwizzleAttr value = checked(attr, context); - if (isNull(value)) return {nullptr}; - return wrap(static_cast(SwizzleType::get(value))); -} - -MlirType mlirFlyPointerTypeGet(MlirContext ctx, MlirType elemTy, MlirAttribute addressSpace, MlirAttribute alignment, MlirAttribute swizzle) { - mlir::MLIRContext *context = loadFly(ctx); - mlir::Type element = unwrap(elemTy); - mlir::Attribute space = unwrap(addressSpace); - if (!context || isNull(element) || element.getContext() != context) return {nullptr}; - if (isNull(space) || space.getContext() != context) return {nullptr}; - AlignAttr align = alignment.ptr ? checked(alignment, context) - : AlignAttr::getTrivialAlignment(element); - SwizzleAttr sw = swizzle.ptr ? checked(swizzle, context) - : SwizzleAttr::getTrivialSwizzle(context); - if (isNull(align) || isNull(sw)) return {nullptr}; - return wrap(static_cast( - PointerType::get(context, element, space, align, sw))); -} - -MlirType mlirFlyPointerTypeGetElemTy(MlirType type) { - return wrap(cast(unwrap(type)).getElemTy()); -} - -MlirAttribute mlirFlyPointerTypeGetAddressSpace(MlirType type) { - return wrap(cast(unwrap(type)).getAddressSpace()); -} - -MlirAttribute mlirFlyPointerTypeGetAlignment(MlirType type) { - return wrap(static_cast(cast(unwrap(type)).getAlignment())); -} - -MlirAttribute mlirFlyPointerTypeGetSwizzle(MlirType type) { - return wrap(static_cast(cast(unwrap(type)).getSwizzle())); -} - -MlirType mlirFlyMemRefTypeGet(MlirContext ctx, MlirType elemTy, MlirAttribute addressSpace, MlirAttribute layout, MlirAttribute alignment, MlirAttribute swizzle) { - mlir::MLIRContext *context = loadFly(ctx); - mlir::Type element = unwrap(elemTy); - mlir::Attribute space = unwrap(addressSpace); - mlir::Attribute lay = unwrap(layout); - if (!context || isNull(element) || element.getContext() != context) return {nullptr}; - if (isNull(space) || space.getContext() != context) return {nullptr}; - if (isNull(lay) || lay.getContext() != context) return {nullptr}; - AlignAttr align = alignment.ptr ? checked(alignment, context) - : AlignAttr::getTrivialAlignment(element); - SwizzleAttr sw = swizzle.ptr ? checked(swizzle, context) - : SwizzleAttr::getTrivialSwizzle(context); - if (isNull(align) || isNull(sw)) return {nullptr}; - return wrap(static_cast( - mlir::fly::MemRefType::get(context, element, space, lay, align, sw))); -} - -MlirType mlirFlyMemRefTypeGetElemTy(MlirType type) { - return wrap(cast(unwrap(type)).getElemTy()); -} - -MlirAttribute mlirFlyMemRefTypeGetAddressSpace(MlirType type) { - return wrap(cast(unwrap(type)).getAddressSpace()); -} - -MlirAttribute mlirFlyMemRefTypeGetLayout(MlirType type) { - return wrap(cast(unwrap(type)).getLayout()); -} - -MlirType mlirFlyCoordTensorTypeGet(MlirContext ctx, MlirAttribute base, MlirAttribute layout) { - mlir::MLIRContext *context = loadFly(ctx); - if (!context) return {nullptr}; - IntTupleAttr baseAttr = checked(base, context); - mlir::Attribute lay = unwrap(layout); - if (isNull(baseAttr) || isNull(lay) || lay.getContext() != context) return {nullptr}; - return wrap(static_cast(CoordTensorType::get(context, baseAttr, lay))); -} - -static mlir::Attribute outerLayoutAttr(mlir::Attribute layout) { - while (auto composed = mlir::dyn_cast(layout)) - layout = composed.getOuter(); - return layout; -} - -MlirType mlirFlyLayoutLikeTypeGetShape(MlirType type) { - mlir::Type ty = unwrap(type); - mlir::Attribute layout; - if (auto l = mlir::dyn_cast(ty)) { - layout = l.getAttr(); - } else if (auto c = mlir::dyn_cast(ty)) { - layout = c.getAttr(); - } else if (auto m = mlir::dyn_cast(ty)) { - layout = m.getLayout(); - } else { - layout = cast(ty).getLayout(); - } - return wrap(static_cast( - IntTupleType::get(cast(outerLayoutAttr(layout)).getShape()))); -} - -MlirType mlirFlyCopyAtomTypeGet(MlirContext ctx, MlirType copyOp, int32_t valBits) { - mlir::MLIRContext *context = loadFly(ctx); - mlir::Type op = unwrap(copyOp); - if (!context || isNull(op) || op.getContext() != context) return {nullptr}; - return wrap(static_cast(CopyAtomType::get(context, op, valBits))); -} - -MlirType mlirFlyMmaAtomTypeGet(MlirContext ctx, MlirType mmaOp) { - mlir::MLIRContext *context = loadFly(ctx); - mlir::Type op = unwrap(mmaOp); - if (!context || isNull(op) || op.getContext() != context) return {nullptr}; - return wrap(static_cast(MmaAtomType::get(context, op))); -} - -#define FLY_ATOM_LAYOUT(Name, AtomType, getter) \ - MlirType Name(MlirType type) { \ - return wrap(static_cast(LayoutType::get( \ - cast(cast(unwrap(type)).getter())))); \ - } - -FLY_ATOM_LAYOUT(mlirFlyCopyAtomTypeGetThrLayout, CopyAtomType, getThrLayout) -FLY_ATOM_LAYOUT(mlirFlyCopyAtomTypeGetThrValLayoutSrc, CopyAtomType, getThrValLayoutSrc) -FLY_ATOM_LAYOUT(mlirFlyCopyAtomTypeGetThrValLayoutDst, CopyAtomType, getThrValLayoutDst) -FLY_ATOM_LAYOUT(mlirFlyCopyAtomTypeGetThrValLayoutRef, CopyAtomType, getThrValLayoutRef) -FLY_ATOM_LAYOUT(mlirFlyMmaAtomTypeGetThrLayout, MmaAtomType, getThrLayout) -FLY_ATOM_LAYOUT(mlirFlyMmaAtomTypeGetThrValLayoutA, MmaAtomType, getThrValLayoutA) -FLY_ATOM_LAYOUT(mlirFlyMmaAtomTypeGetThrValLayoutB, MmaAtomType, getThrValLayoutB) -FLY_ATOM_LAYOUT(mlirFlyMmaAtomTypeGetThrValLayoutC, MmaAtomType, getThrValLayoutC) - -#undef FLY_ATOM_LAYOUT - -MlirType mlirFlyMmaAtomTypeGetShapeMNK(MlirType type) { - return wrap(static_cast(IntTupleType::get( - cast(cast(unwrap(type)).getShapeMNK())))); -} - -MlirType mlirFlyTiledCopyTypeGetTiledThrValLayoutSrc(MlirType type) { - auto ty = cast(unwrap(type)); - return wrap(static_cast(LayoutType::get(tiledCopyGetTiledThrValLayoutSrc( - cast(ty.getCopyAtom()), ty.getLayoutThrVal().getAttr(), - ty.getTileMN().getAttr())))); -} - -MlirType mlirFlyTiledCopyTypeGetTiledThrValLayoutDst(MlirType type) { - auto ty = cast(unwrap(type)); - return wrap(static_cast(LayoutType::get(tiledCopyGetTiledThrValLayoutDst( - cast(ty.getCopyAtom()), ty.getLayoutThrVal().getAttr(), - ty.getTileMN().getAttr())))); -} - -MlirType mlirFlyTiledMmaTypeGetTileSizeMNK(MlirType type) { - auto ty = cast(unwrap(type)); - return wrap(static_cast(IntTupleType::get(tiledMmaGetTileSizeMNK( - cast(ty.getMmaAtom()), ty.getAtomLayout().getAttr(), - ty.getPermutation().getAttr())))); -} - -MlirType mlirFlyTiledMmaTypeGetThrLayoutVMNK(MlirType type) { - auto ty = cast(unwrap(type)); - return wrap(static_cast(LayoutType::get(tiledMmaGetThrLayoutVMNK( - cast(ty.getMmaAtom()), ty.getAtomLayout().getAttr())))); -} - -static MlirType tiledMmaTiledThrVal(MlirType tiledMma, MmaOperand operand) { - auto ty = cast(unwrap(tiledMma)); - return wrap(static_cast(LayoutType::get(tiledMmaGetTiledThrValLayout( - cast(ty.getMmaAtom()), ty.getAtomLayout().getAttr(), - ty.getPermutation().getAttr(), operand)))); -} - -MlirType mlirFlyTiledMmaTypeGetTiledThrValLayoutA(MlirType type) { - return tiledMmaTiledThrVal(type, MmaOperand::A); -} - -MlirType mlirFlyTiledMmaTypeGetTiledThrValLayoutB(MlirType type) { - return tiledMmaTiledThrVal(type, MmaOperand::B); -} - -MlirType mlirFlyTiledMmaTypeGetTiledThrValLayoutC(MlirType type) { - return tiledMmaTiledThrVal(type, MmaOperand::C); -} - -MlirType mlirFlyCopyOpUniversalCopyTypeGet(MlirContext ctx, int32_t bitSize) { - mlir::MLIRContext *context = loadFly(ctx); - if (!context) return {nullptr}; - return wrap(static_cast(CopyOpUniversalCopyType::get(context, bitSize))); -} - -MlirAttribute mlirFlyROCDLBufferDescAddressAttrGet(MlirContext ctx) { - mlir::MLIRContext *context = loadFlyROCDL(ctx); - if (!context) return {nullptr}; - return wrap(static_cast( - mlir::fly_rocdl::BufferDescAddressAttr::get(context))); -} - -MlirType mlirFlyROCDLCopyOpCDNA3BufferCopyTypeGet(MlirContext ctx, int32_t bitSize, int32_t cacheModifier) { - mlir::MLIRContext *context = loadFlyROCDL(ctx); - if (!context) return {nullptr}; - return wrap(static_cast( - mlir::fly_rocdl::CopyOpCDNA3BufferCopyType::get(context, bitSize, cacheModifier))); -} - -MlirType mlirFlyROCDLCopyOpCDNA3BufferCopyLDSTypeGet(MlirContext ctx, int32_t bitSize) { - mlir::MLIRContext *context = loadFlyROCDL(ctx); - if (!context) return {nullptr}; - return wrap(static_cast( - mlir::fly_rocdl::CopyOpCDNA3BufferCopyLDSType::get(context, bitSize))); -} - -MlirType mlirFlyROCDLMmaOpCDNA3MFMATypeGet(MlirContext ctx, int32_t m, int32_t n, int32_t k, MlirType elemTyA, MlirType elemTyB, MlirType elemTyAcc) { - return mmaOpGet(ctx, m, n, k, elemTyA, elemTyB, elemTyAcc); -} - -MlirType mlirFlyROCDLMmaOpGFX11WMMATypeGet(MlirContext ctx, int32_t m, int32_t n, int32_t k, MlirType elemTyA, MlirType elemTyB, MlirType elemTyAcc, bool signA, bool signB, bool clamp) { - return mmaOpGet(ctx, m, n, k, elemTyA, elemTyB, elemTyAcc, signA, signB, clamp); -} - -MlirType mlirFlyROCDLMmaOpGFX120XWMMATypeGet(MlirContext ctx, int32_t m, int32_t n, int32_t k, MlirType elemTyA, MlirType elemTyB, MlirType elemTyAcc, bool signA, bool signB, bool clamp) { - return mmaOpGet(ctx, m, n, k, elemTyA, elemTyB, elemTyAcc, signA, signB, clamp); -} - -} // extern "C" diff --git a/mlir/dialects/fly/fly_capi.h b/mlir/dialects/fly/fly_capi.h deleted file mode 100644 index 6a0e3f39..00000000 --- a/mlir/dialects/fly/fly_capi.h +++ /dev/null @@ -1,143 +0,0 @@ -#ifndef ZML_MLIR_DIALECTS_FLY_FLY_CAPI_H -#define ZML_MLIR_DIALECTS_FLY_FLY_CAPI_H - -#include -#include - -#include "mlir-c/IR.h" -#include "mlir-c/Support.h" - -#ifdef __cplusplus -extern "C" { -#endif - -MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Fly, fly); -MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(FlyROCDL, fly_rocdl); -MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(UB, ub); - -MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyInt(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntAttrGetStatic(MlirContext ctx, int32_t value); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntAttrGetDynamic(MlirContext ctx, int32_t width, int32_t divisibility); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntAttrGetNone(MlirContext ctx); - -MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyIntTuple(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntTupleAttrGet(MlirContext ctx, MlirAttribute value); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntTupleAttrGetBasis(MlirContext ctx, MlirAttribute value, intptr_t nModes, const int32_t *modes); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntTupleAttrGetTuple(MlirContext ctx, intptr_t nElements, MlirAttribute const *elements); - -MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyLayout(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyLayoutAttrGet(MlirContext ctx, MlirAttribute shape, MlirAttribute stride); - -MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyTile(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyTileAttrGet(MlirContext ctx, MlirAttribute value); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyTileAttrGetModes(MlirContext ctx, intptr_t nModes, MlirAttribute const *modes); - -MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlySwizzle(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlySwizzleAttrGet(MlirContext ctx, int32_t mask, int32_t base, int32_t shift); - -MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyAlign(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyAlignAttrGet(MlirContext ctx, int32_t alignment); - -MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyAddressSpace(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyAddressSpaceAttrGet(MlirContext ctx, int32_t addressSpace); - -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyMmaOperandAttrGet(MlirContext ctx, int32_t operand); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyGemmTraversalOrderAttrGet(MlirContext ctx, int32_t order); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyIntTuple(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyIntTupleTypeGet(MlirContext ctx, MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntTupleTypeGetAttr(MlirType type); -MLIR_CAPI_EXPORTED intptr_t mlirFlyIntTupleTypeGetNumElements(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyIntTupleTypeGetElement(MlirType type, intptr_t pos); -MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsLeaf(MlirType type); -MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsStatic(MlirType type); -MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsStaticLeaf(MlirType type); -MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsNoneLeaf(MlirType type); -MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsBasisLeaf(MlirType type); -MLIR_CAPI_EXPORTED int64_t mlirFlyIntTupleTypeGetStaticValue(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyLayout(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyLayoutTypeGet(MlirContext ctx, MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyLayoutTypeGetAttr(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyLayoutTypeGetShape(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyLayoutTypeGetStride(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyComposedLayout(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyComposedLayoutTypeGetAttr(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyTile(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyTileTypeGet(MlirContext ctx, MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyTileTypeGetAttr(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlySwizzle(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlySwizzleTypeGet(MlirContext ctx, MlirAttribute attr); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyPointer(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyPointerTypeGet(MlirContext ctx, MlirType elemTy, MlirAttribute addressSpace, MlirAttribute alignment, MlirAttribute swizzle); -MLIR_CAPI_EXPORTED MlirType mlirFlyPointerTypeGetElemTy(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyPointerTypeGetAddressSpace(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyPointerTypeGetAlignment(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyPointerTypeGetSwizzle(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyMemRef(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyMemRefTypeGet(MlirContext ctx, MlirType elemTy, MlirAttribute addressSpace, MlirAttribute layout, MlirAttribute alignment, MlirAttribute swizzle); -MLIR_CAPI_EXPORTED MlirType mlirFlyMemRefTypeGetElemTy(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyMemRefTypeGetAddressSpace(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyMemRefTypeGetLayout(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyCoordTensor(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyCoordTensorTypeGet(MlirContext ctx, MlirAttribute base, MlirAttribute layout); - -MLIR_CAPI_EXPORTED MlirType mlirFlyLayoutLikeTypeGetShape(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyCopyAtom(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyCopyAtomTypeGet(MlirContext ctx, MlirType copyOp, int32_t valBits); -MLIR_CAPI_EXPORTED MlirType mlirFlyCopyAtomTypeGetThrLayout(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyCopyAtomTypeGetThrValLayoutSrc(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyCopyAtomTypeGetThrValLayoutDst(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyCopyAtomTypeGetThrValLayoutRef(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyMmaAtom(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGet(MlirContext ctx, MlirType mmaOp); -MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetThrLayout(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetShapeMNK(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetThrValLayoutA(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetThrValLayoutB(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetThrValLayoutC(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyTiledCopy(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyTiledCopyTypeGetTiledThrValLayoutSrc(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyTiledCopyTypeGetTiledThrValLayoutDst(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyTiledMma(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyTiledMmaTypeGetTileSizeMNK(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyTiledMmaTypeGetThrLayoutVMNK(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyTiledMmaTypeGetTiledThrValLayoutA(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyTiledMmaTypeGetTiledThrValLayoutB(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyTiledMmaTypeGetTiledThrValLayoutC(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyCopyOpUniversalCopy(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyCopyOpUniversalCopyTypeGet(MlirContext ctx, int32_t bitSize); - -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyROCDLBufferDescAddressAttrGet(MlirContext ctx); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLCopyOpCDNA3BufferCopy(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLCopyOpCDNA3BufferCopyTypeGet(MlirContext ctx, int32_t bitSize, int32_t cacheModifier); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLCopyOpCDNA3BufferCopyLDS(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLCopyOpCDNA3BufferCopyLDSTypeGet(MlirContext ctx, int32_t bitSize); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLMmaOpCDNA3MFMA(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLMmaOpCDNA3MFMATypeGet(MlirContext ctx, int32_t m, int32_t n, int32_t k, MlirType elemTyA, MlirType elemTyB, MlirType elemTyAcc); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLMmaOpGFX11WMMA(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLMmaOpGFX11WMMATypeGet(MlirContext ctx, int32_t m, int32_t n, int32_t k, MlirType elemTyA, MlirType elemTyB, MlirType elemTyAcc, bool signA, bool signB, bool clamp); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLMmaOpGFX120XWMMA(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLMmaOpGFX120XWMMATypeGet(MlirContext ctx, int32_t m, int32_t n, int32_t k, MlirType elemTyA, MlirType elemTyB, MlirType elemTyAcc, bool signA, bool signB, bool clamp); - -#ifdef __cplusplus -} -#endif - -#endif // ZML_MLIR_DIALECTS_FLY_FLY_CAPI_H diff --git a/mlir/dialects/fly/ops.zig b/mlir/dialects/fly/ops.zig index 7ac4adba..b9753136 100644 --- a/mlir/dialects/fly/ops.zig +++ b/mlir/dialects/fly/ops.zig @@ -2,129 +2,19 @@ //! 1941889400621f1fc3d8f03bada1a4e7380cdf7e; do not edit, see REGENERATING.md. //! //! Builders take the operands, then the result types (only for operations -//! that do not infer them), then the attributes, then the location. -//! Parameters are the ODS names in snake case, with a trailing underscore -//! where needed. Pass `null` for absent optional operands or attributes and -//! `&.{}` for empty variadic ones; variable-length operand groups of +//! that do not infer them), then the required attributes, then an `opts` +//! struct of optional operands and attributes (pass `.{}` to omit them), then +//! the location. Parameters are the ODS names in snake case, with a trailing +//! underscore where needed. Pass `&.{}` for empty variadic operands; +//! variable-length operand groups of //! `AttrSizedOperandSegments` operations get their segment sizes from //! `Operation.make`. Attributes use `mlir.Attribute`. -const std = @import("std"); - const mlir = @import("mlir"); const stdx = @import("stdx"); const make = @import("fly.zig").make; -/// Every operation bound here, for the registration test. -pub const names = [_][]const u8{ - "fly.add_offset", - "fly.append", - "fly.apply_swizzle", - "fly.assume", - "fly.atom.set_value", - "fly.blocked_product", - "fly.ceil_div", - "fly.coalesce", - "fly.complement", - "fly.composed_get_inner", - "fly.composed_get_offset", - "fly.composed_get_outer", - "fly.composition", - "fly.coprofile", - "fly.copy", - "fly.copy_atom_call", - "fly.copy_atom_call_ssa", - "fly.coshape", - "fly.cosize", - "fly.crd2idx", - "fly.decomposition", - "fly.dice", - "fly.elem_less", - "fly.equal", - "fly.extract_aligned_pointer_as_index", - "fly.flat_divide", - "fly.flat_product", - "fly.gemm", - "fly.get", - "fly.get_1d_coord", - "fly.get_copy_atom", - "fly.get_dyn_shared", - "fly.get_flat_coord", - "fly.get_iter", - "fly.get_layout", - "fly.get_leaves", - "fly.get_mma_atom", - "fly.get_scalar", - "fly.get_shape", - "fly.get_stride", - "fly.group", - "fly.idx2crd", - "fly.int_tuple_add", - "fly.int_tuple_div", - "fly.int_tuple_mod", - "fly.int_tuple_mul", - "fly.int_tuple_product", - "fly.int_tuple_product_each", - "fly.int_tuple_product_like", - "fly.int_tuple_sub", - "fly.inttoptr", - "fly.left_inverse", - "fly.logical_divide", - "fly.logical_product", - "fly.make_composed_layout", - "fly.make_coord", - "fly.make_copy_atom", - "fly.make_fragment_layout_like", - "fly.make_fragment_like", - "fly.make_identity_layout", - "fly.make_int_tuple", - "fly.make_layout", - "fly.make_layout_like", - "fly.make_mma_atom", - "fly.make_ordered_layout", - "fly.make_ptr", - "fly.make_shape", - "fly.make_stride", - "fly.make_tiled_copy", - "fly.make_tiled_mma", - "fly.make_view", - "fly.memref.alloca", - "fly.memref.load", - "fly.memref.load_vec", - "fly.memref.store", - "fly.memref.store_vec", - "fly.mma.make_fragment", - "fly.mma_atom_call", - "fly.mma_atom_call_ssa", - "fly.prepend", - "fly.print", - "fly.ptr.load", - "fly.ptr.store", - "fly.ptrtoint", - "fly.raked_product", - "fly.recast_iter", - "fly.recast_layout", - "fly.right_inverse", - "fly.select", - "fly.shape_div", - "fly.size", - "fly.slice", - "fly.static", - "fly.take", - "fly.tile_to_shape", - "fly.tiled_copy.partition_dst", - "fly.tiled_copy.partition_src", - "fly.tiled_copy.retile", - "fly.tiled_divide", - "fly.tiled_mma.partition", - "fly.tiled_mma.partition_shape", - "fly.tiled_product", - "fly.to_llvm_ptr", - "fly.zipped_divide", - "fly.zipped_product", -}; - /// `fly.add_offset`. Result types are inferred. pub fn add_offset(ctx: *mlir.Context, ptr: *const mlir.Value, offset: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { return make(ctx, "fly.add_offset", .{ @@ -135,9 +25,17 @@ pub fn add_offset(ctx: *mlir.Context, ptr: *const mlir.Value, offset: *const mli } /// `fly.append`. Result types are inferred. -pub fn append(ctx: *mlir.Context, tuple: *const mlir.Value, elem: *const mlir.Value, n: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn append( + ctx: *mlir.Context, + tuple: *const mlir.Value, + elem: *const mlir.Value, + opts: struct { + n: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (n) |present| attributes.appendAssumeCapacity(.named(ctx, "n", present)); + if (opts.n) |present| attributes.appendAssumeCapacity(.named(ctx, "n", present)); return make(ctx, "fly.append", .{ .operands = .{ .flat = &.{ tuple, elem } }, .attributes = attributes.constSlice(), @@ -195,18 +93,32 @@ pub fn ceil_div(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Val } /// `fly.coalesce`. Result types are inferred. -pub fn coalesce(ctx: *mlir.Context, layout: *const mlir.Value, pattern: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { +pub fn coalesce( + ctx: *mlir.Context, + layout: *const mlir.Value, + opts: struct { + pattern: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return make(ctx, "fly.coalesce", .{ - .operands = .{ .flat = if (pattern) |present| &.{ layout, present } else &.{layout} }, + .operands = .{ .flat = if (opts.pattern) |present| &.{ layout, present } else &.{layout} }, .result_type_inference = true, .location = location, }); } /// `fly.complement`. Result types are inferred. -pub fn complement(ctx: *mlir.Context, layout: *const mlir.Value, codomain_size: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { +pub fn complement( + ctx: *mlir.Context, + layout: *const mlir.Value, + opts: struct { + codomain_size: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return make(ctx, "fly.complement", .{ - .operands = .{ .flat = if (codomain_size) |present| &.{ layout, present } else &.{layout} }, + .operands = .{ .flat = if (opts.codomain_size) |present| &.{ layout, present } else &.{layout} }, .result_type_inference = true, .location = location, }); @@ -258,29 +170,57 @@ pub fn coprofile(ctx: *mlir.Context, layout: *const mlir.Value, location: *const } /// `fly.copy`. No results. -pub fn copy(ctx: *mlir.Context, copy_atom: *const mlir.Value, src: *const mlir.Value, dst: *const mlir.Value, pred: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { +pub fn copy( + ctx: *mlir.Context, + copy_atom: *const mlir.Value, + src: *const mlir.Value, + dst: *const mlir.Value, + opts: struct { + pred: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return make(ctx, "fly.copy", .{ - .operands = .{ .flat = if (pred) |present| &.{ copy_atom, src, dst, present } else &.{ copy_atom, src, dst } }, + .operands = .{ .flat = if (opts.pred) |present| &.{ copy_atom, src, dst, present } else &.{ copy_atom, src, dst } }, .location = location, }); } /// `fly.copy_atom_call`. No results. -pub fn copy_atom_call(ctx: *mlir.Context, copy_atom: *const mlir.Value, src: *const mlir.Value, dst: *const mlir.Value, pred: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { +pub fn copy_atom_call( + ctx: *mlir.Context, + copy_atom: *const mlir.Value, + src: *const mlir.Value, + dst: *const mlir.Value, + opts: struct { + pred: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return make(ctx, "fly.copy_atom_call", .{ - .operands = .{ .flat = if (pred) |present| &.{ copy_atom, src, dst, present } else &.{ copy_atom, src, dst } }, + .operands = .{ .flat = if (opts.pred) |present| &.{ copy_atom, src, dst, present } else &.{ copy_atom, src, dst } }, .location = location, }); } /// `fly.copy_atom_call_ssa`. Result types are explicit. -pub fn copy_atom_call_ssa(ctx: *mlir.Context, copy_atom: *const mlir.Value, src: *const mlir.Value, dst: ?*const mlir.Value, pred: ?*const mlir.Value, results_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +pub fn copy_atom_call_ssa( + ctx: *mlir.Context, + copy_atom: *const mlir.Value, + src: *const mlir.Value, + results_types: []const *const mlir.Type, + opts: struct { + dst: ?*const mlir.Value = null, + pred: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return make(ctx, "fly.copy_atom_call_ssa", .{ .operands = .{ .variadic = &.{ &.{copy_atom}, &.{src}, - if (dst) |present| &.{present} else &.{}, - if (pred) |present| &.{present} else &.{}, + if (opts.dst) |present| &.{present} else &.{}, + if (opts.pred) |present| &.{present} else &.{}, } }, .results = .{ .flat = results_types }, .location = location, @@ -378,9 +318,21 @@ pub fn flat_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *const } /// `fly.gemm`. Fully expand tiled GEMM with variadic A and B operand groups. No results. -pub fn gemm(ctx: *mlir.Context, mma_atom: *const mlir.Value, d: *const mlir.Value, a: []const *const mlir.Value, b: []const *const mlir.Value, c: *const mlir.Value, traversal_layout: ?*const mlir.Value, traversal_order: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn gemm( + ctx: *mlir.Context, + mma_atom: *const mlir.Value, + d: *const mlir.Value, + a: []const *const mlir.Value, + b: []const *const mlir.Value, + c: *const mlir.Value, + opts: struct { + traversal_layout: ?*const mlir.Value = null, + traversal_order: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (traversal_order) |present| attributes.appendAssumeCapacity(.named(ctx, "traversalOrder", present)); + if (opts.traversal_order) |present| attributes.appendAssumeCapacity(.named(ctx, "traversalOrder", present)); return make(ctx, "fly.gemm", .{ .operands = .{ .variadic = &.{ &.{mma_atom}, @@ -388,7 +340,7 @@ pub fn gemm(ctx: *mlir.Context, mma_atom: *const mlir.Value, d: *const mlir.Valu a, b, &.{c}, - if (traversal_layout) |present| &.{present} else &.{}, + if (opts.traversal_layout) |present| &.{present} else &.{}, } }, .attributes = attributes.constSlice(), .location = location, @@ -461,9 +413,16 @@ pub fn get_layout(ctx: *mlir.Context, memref: *const mlir.Value, location: *cons } /// `fly.get_leaves`. Result types are inferred. -pub fn get_leaves(ctx: *mlir.Context, input: *const mlir.Value, dynamic_only: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn get_leaves( + ctx: *mlir.Context, + input: *const mlir.Value, + opts: struct { + dynamic_only: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (dynamic_only) |present| attributes.appendAssumeCapacity(.named(ctx, "dynamicOnly", present)); + if (opts.dynamic_only) |present| attributes.appendAssumeCapacity(.named(ctx, "dynamicOnly", present)); return make(ctx, "fly.get_leaves", .{ .operands = .{ .flat = &.{input} }, .attributes = attributes.constSlice(), @@ -678,9 +637,16 @@ pub fn make_fragment_layout_like(ctx: *mlir.Context, src: *const mlir.Value, loc } /// `fly.make_fragment_like`. Result types are inferred. -pub fn make_fragment_like(ctx: *mlir.Context, src: *const mlir.Value, dtype: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn make_fragment_like( + ctx: *mlir.Context, + src: *const mlir.Value, + opts: struct { + dtype: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (dtype) |present| attributes.appendAssumeCapacity(.named(ctx, "dtype", present)); + if (opts.dtype) |present| attributes.appendAssumeCapacity(.named(ctx, "dtype", present)); return make(ctx, "fly.make_fragment_like", .{ .operands = .{ .flat = &.{src} }, .attributes = attributes.constSlice(), @@ -743,9 +709,17 @@ pub fn make_ordered_layout(ctx: *mlir.Context, shape: *const mlir.Value, order: } /// `fly.make_ptr`. Result types are explicit. -pub fn make_ptr(ctx: *mlir.Context, args: []const *const mlir.Value, result_type: *const mlir.Type, dict_attrs: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn make_ptr( + ctx: *mlir.Context, + args: []const *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + dict_attrs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (dict_attrs) |present| attributes.appendAssumeCapacity(.named(ctx, "dictAttrs", present)); + if (opts.dict_attrs) |present| attributes.appendAssumeCapacity(.named(ctx, "dictAttrs", present)); return make(ctx, "fly.make_ptr", .{ .operands = .{ .flat = args }, .results = .{ .flat = &.{result_type} }, @@ -782,9 +756,17 @@ pub fn make_tiled_copy(ctx: *mlir.Context, copy_atom: *const mlir.Value, layout_ } /// `fly.make_tiled_mma`. Result types are inferred. -pub fn make_tiled_mma(ctx: *mlir.Context, mma_atom: *const mlir.Value, atom_layout: *const mlir.Value, permutation: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { +pub fn make_tiled_mma( + ctx: *mlir.Context, + mma_atom: *const mlir.Value, + atom_layout: *const mlir.Value, + opts: struct { + permutation: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return make(ctx, "fly.make_tiled_mma", .{ - .operands = .{ .flat = if (permutation) |present| &.{ mma_atom, atom_layout, present } else &.{ mma_atom, atom_layout } }, + .operands = .{ .flat = if (opts.permutation) |present| &.{ mma_atom, atom_layout, present } else &.{ mma_atom, atom_layout } }, .result_type_inference = true, .location = location, }); @@ -843,10 +825,19 @@ pub fn memref_store_vec(ctx: *mlir.Context, vector: *const mlir.Value, memref: * } /// `fly.mma.make_fragment`. Result types are inferred. -pub fn mma_make_fragment(ctx: *mlir.Context, tiled_mma: *const mlir.Value, input: *const mlir.Value, operand_id: *const mlir.Attribute, stages: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mma_make_fragment( + ctx: *mlir.Context, + tiled_mma: *const mlir.Value, + input: *const mlir.Value, + operand_id: *const mlir.Attribute, + opts: struct { + stages: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; attributes.appendAssumeCapacity(.named(ctx, "operand_id", operand_id)); - if (stages) |present| attributes.appendAssumeCapacity(.named(ctx, "stages", present)); + if (opts.stages) |present| attributes.appendAssumeCapacity(.named(ctx, "stages", present)); return make(ctx, "fly.mma.make_fragment", .{ .operands = .{ .flat = &.{ tiled_mma, input } }, .attributes = attributes.constSlice(), @@ -870,11 +861,22 @@ pub fn mma_atom_call(ctx: *mlir.Context, mma_atom: *const mlir.Value, d: *const } /// `fly.mma_atom_call_ssa`. Result types are explicit. -pub fn mma_atom_call_ssa(ctx: *mlir.Context, mma_atom: *const mlir.Value, d: ?*const mlir.Value, a: []const *const mlir.Value, b: []const *const mlir.Value, c: *const mlir.Value, results_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +pub fn mma_atom_call_ssa( + ctx: *mlir.Context, + mma_atom: *const mlir.Value, + a: []const *const mlir.Value, + b: []const *const mlir.Value, + c: *const mlir.Value, + results_types: []const *const mlir.Type, + opts: struct { + d: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return make(ctx, "fly.mma_atom_call_ssa", .{ .operands = .{ .variadic = &.{ &.{mma_atom}, - if (d) |present| &.{present} else &.{}, + if (opts.d) |present| &.{present} else &.{}, a, b, &.{c}, @@ -885,9 +887,17 @@ pub fn mma_atom_call_ssa(ctx: *mlir.Context, mma_atom: *const mlir.Value, d: ?*c } /// `fly.prepend`. Result types are inferred. -pub fn prepend(ctx: *mlir.Context, tuple: *const mlir.Value, elem: *const mlir.Value, n: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn prepend( + ctx: *mlir.Context, + tuple: *const mlir.Value, + elem: *const mlir.Value, + opts: struct { + n: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (n) |present| attributes.appendAssumeCapacity(.named(ctx, "n", present)); + if (opts.n) |present| attributes.appendAssumeCapacity(.named(ctx, "n", present)); return make(ctx, "fly.prepend", .{ .operands = .{ .flat = &.{ tuple, elem } }, .attributes = attributes.constSlice(), @@ -1140,7 +1150,3 @@ pub fn zipped_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *cons .location = location, }); } - -test { - std.testing.refAllDecls(@This()); -} diff --git a/mlir/dialects/fly/rocdl_ops.zig b/mlir/dialects/fly/rocdl_ops.zig deleted file mode 100644 index 4a8a53cb..00000000 --- a/mlir/dialects/fly/rocdl_ops.zig +++ /dev/null @@ -1,69 +0,0 @@ -//! `fly_rocdl` operation builders, generated from FlyDSL's `include/flydsl/Dialect/FlyROCDL/IR/Ops.td` at -//! 1941889400621f1fc3d8f03bada1a4e7380cdf7e; do not edit, see REGENERATING.md. -//! -//! Builders take the operands, then the result types (only for operations -//! that do not infer them), then the attributes, then the location. -//! Parameters are the ODS names in snake case, with a trailing underscore -//! where needed. Pass `null` for absent optional operands or attributes and -//! `&.{}` for empty variadic ones; variable-length operand groups of -//! `AttrSizedOperandSegments` operations get their segment sizes from -//! `Operation.make`. Attributes use `mlir.Attribute`. - -const std = @import("std"); - -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -const make = @import("fly.zig").make; - -/// Every operation bound here, for the registration test. -pub const names = [_][]const u8{ - "fly_rocdl.get_buffer_rsrc", - "fly_rocdl.make_tiled_tdm_load_atom", - "fly_rocdl.make_tiled_tdm_store_atom", -}; - -/// `fly_rocdl.get_buffer_rsrc`. Extract the raw ROCDL buffer resource from a buffer-descriptor pointer. Result types are inferred. -pub fn get_buffer_rsrc(ctx: *mlir.Context, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly_rocdl.get_buffer_rsrc", .{ - .operands = .{ .flat = &.{ptr} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly_rocdl.make_tiled_tdm_load_atom`. Build a CDNA5 TDM load atom and the coordinate tensor that addresses it. Result types are inferred. -pub fn make_tiled_tdm_load_atom(ctx: *mlir.Context, tensor: *const mlir.Value, smem_layout: *const mlir.Value, tiler: *const mlir.Value, num_warps: ?*const mlir.Attribute, init_boundary_check: ?*const mlir.Attribute, cache_modifier: ?*const mlir.Attribute, atomic_barrier: ?*const mlir.Attribute, internal_type: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - if (num_warps) |present| attributes.appendAssumeCapacity(.named(ctx, "numWarps", present)); - if (init_boundary_check) |present| attributes.appendAssumeCapacity(.named(ctx, "initBoundaryCheck", present)); - if (cache_modifier) |present| attributes.appendAssumeCapacity(.named(ctx, "cacheModifier", present)); - if (atomic_barrier) |present| attributes.appendAssumeCapacity(.named(ctx, "atomicBarrier", present)); - if (internal_type) |present| attributes.appendAssumeCapacity(.named(ctx, "internalType", present)); - return make(ctx, "fly_rocdl.make_tiled_tdm_load_atom", .{ - .operands = .{ .flat = &.{ tensor, smem_layout, tiler } }, - .attributes = attributes.constSlice(), - .result_type_inference = true, - .location = location, - }); -} - -/// `fly_rocdl.make_tiled_tdm_store_atom`. Build a CDNA5 TDM store atom and the coordinate tensor that addresses it. Result types are inferred. -pub fn make_tiled_tdm_store_atom(ctx: *mlir.Context, tensor: *const mlir.Value, smem_layout: *const mlir.Value, tiler: *const mlir.Value, num_warps: ?*const mlir.Attribute, init_boundary_check: ?*const mlir.Attribute, cache_modifier: ?*const mlir.Attribute, atomic_barrier: ?*const mlir.Attribute, internal_type: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - if (num_warps) |present| attributes.appendAssumeCapacity(.named(ctx, "numWarps", present)); - if (init_boundary_check) |present| attributes.appendAssumeCapacity(.named(ctx, "initBoundaryCheck", present)); - if (cache_modifier) |present| attributes.appendAssumeCapacity(.named(ctx, "cacheModifier", present)); - if (atomic_barrier) |present| attributes.appendAssumeCapacity(.named(ctx, "atomicBarrier", present)); - if (internal_type) |present| attributes.appendAssumeCapacity(.named(ctx, "internalType", present)); - return make(ctx, "fly_rocdl.make_tiled_tdm_store_atom", .{ - .operands = .{ .flat = &.{ tensor, smem_layout, tiler } }, - .attributes = attributes.constSlice(), - .result_type_inference = true, - .location = location, - }); -} - -test { - std.testing.refAllDecls(@This()); -} diff --git a/mlir/dialects/fly/test.zig b/mlir/dialects/fly/test.zig new file mode 100644 index 00000000..a30172ff --- /dev/null +++ b/mlir/dialects/fly/test.zig @@ -0,0 +1,378 @@ +const std = @import("std"); + +const arith = @import("mlir/dialects/arith"); +const fly = @import("mlir/dialects/fly"); +const fly_rocdl = @import("mlir/dialects/fly_rocdl"); +const func = @import("mlir/dialects/func"); +const gpu = @import("mlir/dialects/gpu"); +const llvm = @import("mlir/dialects/llvm"); +const math = @import("mlir/dialects/math"); +const memref = @import("mlir/dialects/memref"); +const mlir = @import("mlir"); +const rocdl = @import("mlir/dialects/rocdl"); +const scf = @import("mlir/dialects/scf"); +const ub = @import("mlir/dialects/ub"); +const vector = @import("mlir/dialects/vector"); + +const attributes = fly.attributes; +const ops = fly.ops; +const types = fly.types; +const TypeKind = fly.TypeKind; +const typeKind = fly.typeKind; +const expect = fly.expect; +const layoutLikeShape = fly.layoutLikeShape; +const elemType = fly.elemType; +const ptrWithElem = fly.ptrWithElem; + +const operation_names: []const []const u8 = &.{ + "fly.add_offset", + "fly.append", + "fly.apply_swizzle", + "fly.assume", + "fly.atom.set_value", + "fly.blocked_product", + "fly.ceil_div", + "fly.coalesce", + "fly.complement", + "fly.composed_get_inner", + "fly.composed_get_offset", + "fly.composed_get_outer", + "fly.composition", + "fly.coprofile", + "fly.copy", + "fly.copy_atom_call", + "fly.copy_atom_call_ssa", + "fly.coshape", + "fly.cosize", + "fly.crd2idx", + "fly.decomposition", + "fly.dice", + "fly.elem_less", + "fly.equal", + "fly.extract_aligned_pointer_as_index", + "fly.flat_divide", + "fly.flat_product", + "fly.gemm", + "fly.get", + "fly.get_1d_coord", + "fly.get_copy_atom", + "fly.get_dyn_shared", + "fly.get_flat_coord", + "fly.get_iter", + "fly.get_layout", + "fly.get_leaves", + "fly.get_mma_atom", + "fly.get_scalar", + "fly.get_shape", + "fly.get_stride", + "fly.group", + "fly.idx2crd", + "fly.int_tuple_add", + "fly.int_tuple_div", + "fly.int_tuple_mod", + "fly.int_tuple_mul", + "fly.int_tuple_product", + "fly.int_tuple_product_each", + "fly.int_tuple_product_like", + "fly.int_tuple_sub", + "fly.inttoptr", + "fly.left_inverse", + "fly.logical_divide", + "fly.logical_product", + "fly.make_composed_layout", + "fly.make_coord", + "fly.make_copy_atom", + "fly.make_fragment_layout_like", + "fly.make_fragment_like", + "fly.make_identity_layout", + "fly.make_int_tuple", + "fly.make_layout", + "fly.make_layout_like", + "fly.make_mma_atom", + "fly.make_ordered_layout", + "fly.make_ptr", + "fly.make_shape", + "fly.make_stride", + "fly.make_tiled_copy", + "fly.make_tiled_mma", + "fly.make_view", + "fly.memref.alloca", + "fly.memref.load", + "fly.memref.load_vec", + "fly.memref.store", + "fly.memref.store_vec", + "fly.mma.make_fragment", + "fly.mma_atom_call", + "fly.mma_atom_call_ssa", + "fly.prepend", + "fly.print", + "fly.ptr.load", + "fly.ptr.store", + "fly.ptrtoint", + "fly.raked_product", + "fly.recast_iter", + "fly.recast_layout", + "fly.right_inverse", + "fly.select", + "fly.shape_div", + "fly.size", + "fly.slice", + "fly.static", + "fly.take", + "fly.tile_to_shape", + "fly.tiled_copy.partition_dst", + "fly.tiled_copy.partition_src", + "fly.tiled_copy.retile", + "fly.tiled_divide", + "fly.tiled_mma.partition", + "fly.tiled_mma.partition_shape", + "fly.tiled_product", + "fly.to_llvm_ptr", + "fly.zipped_divide", + "fly.zipped_product", +}; + +test { + std.testing.refAllDecls(fly); + inline for (.{ types, attributes, ops, fly_rocdl, fly_rocdl.ops }) |module| { + std.testing.refAllDecls(module); + inline for (comptime std.meta.declarations(module)) |decl| { + const value = @field(module, decl); + if (@TypeOf(value) == type and @typeInfo(value) == .@"opaque") std.testing.refAllDecls(value); + } + } +} + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + func.registerDialect(registry); + arith.registerDialect(registry); + scf.registerDialect(registry); + math.registerDialect(registry); + vector.registerDialect(registry); + memref.registerDialect(registry); + llvm.registerDialect(registry); + gpu.registerDialect(registry); + rocdl.registerDialect(registry); + ub.registerDialect(registry); + fly.registerDialect(registry); + fly_rocdl.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectPrints(expected: []const u8, value: anytype) !void { + var buf: [512]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(expected, w.buffered()); +} + +test "fly dialects register" { + const ctx = try testContext(); + defer ctx.deinit(); + + try std.testing.expect(ctx.isRegisteredOperation("fly.make_int_tuple")); + try std.testing.expect(ctx.isRegisteredOperation("fly.make_layout")); + try std.testing.expect(ctx.isRegisteredOperation("fly.ptr.load")); + try std.testing.expect(ctx.isRegisteredOperation("fly.copy_atom_call")); + try std.testing.expect(ctx.isRegisteredOperation("gpu.func")); + try std.testing.expect(ctx.isRegisteredOperation("rocdl.sched.barrier")); + try std.testing.expect(ctx.isRegisteredOperation("ub.poison")); +} + +test "fly types parse and print round-trip" { + const ctx = try testContext(); + defer ctx.deinit(); + + const cases = [_][]const u8{ + "!fly.int_tuple<((2, 4), 8)>", + "!fly.int_tuple<((1, 4), (32, 64))>", + "!fly.layout<((2, 3), 3) : ((57, 69), 19)>", + "!fly.memref", + "!fly.memref", + "!fly.ptr", + "!fly.ptr", + "!fly.ptr", + "!fly.memref", + "!fly_rocdl.cdna3.buffer_copy<128>", + "!fly_rocdl.cdna3.buffer_copy_lds<32>", + "!fly_rocdl.cdna3.mfma<16x16x16, (f16, f16) -> f32>", + "!fly.copy_atom, 32>", + "!fly.tile<[8|64]>", + "!fly.coord_tensor<(0,0), (100,1000):(1E0,1E1)>", + }; + for (cases) |src| { + const ty = mlir.Type.parse(ctx, src) catch |err| { + std.debug.print("failed to parse `{s}`: {}\n", .{ src, err }); + return err; + }; + var buf: [512]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{ty}); + const ty2 = try mlir.Type.parse(ctx, w.buffered()); + try std.testing.expect(ty.eql(ty2)); + } +} + +test "types and attributes are built through the C API" { + const ctx = try testContext(); + defer ctx.deinit(); + + const two = try attributes.IntTupleAttr.getStatic(ctx, 2); + const four = try attributes.IntTupleAttr.getStatic(ctx, 4); + const one = try attributes.IntTupleAttr.getStatic(ctx, 1); + const shape = try attributes.IntTupleAttr.getTuple(ctx, &.{ two, four }); + const stride = try attributes.IntTupleAttr.getTuple(ctx, &.{ four, one }); + const layout = try types.LayoutType.get(ctx, .{ .attr = try .get(ctx, .{ .shape = shape, .stride = stride }) }); + try expectPrints("!fly.layout<(2,4):(4,1)>", layout); + try expectPrints("!fly.int_tuple<(2,4)>", layout.shape()); + try std.testing.expectEqual(@as(usize, 2), layout.shape().numElements()); + try std.testing.expectEqual(types.IntTupleType.Leaf{ .static = 4 }, layout.shape().element(1).leaf()); + + const global = try attributes.AddressSpaceAttr.get(ctx, .{ .addressSpace = .global }); + const memref_type = try types.MemRefType.get(ctx, .{ + .elemTy = .float(ctx, .f16), + .addressSpace = global.attribute(), + .layout = layout.attr().attribute(), + }); + try expectPrints("!fly.memref", memref_type); + try expectPrints("!fly.int_tuple<(2,4)>", layoutLikeShape(memref_type.type_())); + + const shared = try attributes.AddressSpaceAttr.get(ctx, .{ .addressSpace = .shared }); + const ptr = try types.PointerType.get(ctx, .{ + .elemTy = .int(ctx, .i8), + .addressSpace = shared.attribute(), + .alignment = try .get(ctx, .{ .alignment = 16 }), + }); + try expectPrints("!fly.ptr>", ptr); + try expectPrints("!fly.ptr>", try ptrWithElem(ctx, ptr.type_(), .float(ctx, .f32))); + + const tile = try types.TileType.get(ctx, .{ .attr = try .getModes(ctx, &.{ + (try attributes.IntAttr.getStatic(ctx, 8)).attribute(), + (try attributes.IntAttr.getStatic(ctx, 64)).attribute(), + }) }); + try expectPrints("!fly.tile<[8|64]>", tile); + + const copy_atom = try types.CopyAtomType.get(ctx, .{ + .copyOp = (try types.CopyOpUniversalCopyType.get(ctx, .{ .bitSize = 128 })).type_(), + .valBits = 32, + }); + try expectPrints("!fly.copy_atom, 32>", copy_atom); + + const buffer_desc = try fly_rocdl.BufferDescAddressAttr.get(ctx); + try expectPrints("!fly.ptr", try types.PointerType.get(ctx, .{ + .elemTy = .float(ctx, .f16), + .addressSpace = buffer_desc.attribute(), + })); +} + +test "fly enum attributes" { + const ctx = try testContext(); + defer ctx.deinit(); + _ = try attributes.mmaOperandAttr(ctx, .a); + _ = try attributes.gemmTraversalOrderAttr(ctx, .kmn); +} + +test "inferred fly ops compute layout algebra" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc: *const mlir.Location = .unknown(ctx); + const module: *mlir.Module = .init(loc); + defer module.deinit(); + const block = module.body(); + + // raked_product((8,16):(16,1), (1,4):(1,1)) — the vectorAdd TV layout. + const thr = ops.static(ctx, try mlir.Type.parse(ctx, "!fly.layout<(8,16):(16,1)>"), loc).appendTo(block); + const val = ops.static(ctx, try mlir.Type.parse(ctx, "!fly.layout<(1,4):(1,1)>"), loc).appendTo(block); + const mn = ops.raked_product(ctx, thr.result(0), val.result(0), loc).appendTo(block); + const shape = ops.get_shape(ctx, mn.result(0), loc).appendTo(block); + const tiler = ops.int_tuple_product_each(ctx, shape.result(0), loc).appendTo(block); + + try expectPrints("!fly.int_tuple<(8,64)>", tiler.result(0).type_()); + try std.testing.expect(module.operation().verify()); +} + +test "atom traits" { + const ctx = try testContext(); + defer ctx.deinit(); + + // MFMA 16x16x4 f32 tiled over a (2,2,1) atom layout. + const tm = expect(try mlir.Type.parse(ctx, "!fly.tiled_mma f32>>, <(2,2,1):(1,2,0)>>"), .tiled_mma); + try expectPrints("!fly.int_tuple<(32,32,4)>", tm.tileSizeMNK()); + _ = tm.tiledThrValLayoutA(); + _ = tm.thrLayoutVMNK(); + + const atom = try types.MmaAtomType.get(ctx, .{ .mmaOp = (try fly_rocdl.MmaOpCDNA3MFMAType.get(ctx, .{ + .m = 16, + .n = 16, + .k = 4, + .elemTyA = .float(ctx, .f32), + .elemTyB = .float(ctx, .f32), + .elemTyAcc = .float(ctx, .f32), + })).type_() }); + try expectPrints("!fly.mma_atom f32>>", atom); + try expectPrints("!fly.int_tuple<(16,16,4)>", atom.shapeMNK()); + try expectPrints("!fly.layout<64:1>", atom.thrLayout()); + + const wmma = try types.MmaAtomType.get(ctx, .{ .mmaOp = (try fly_rocdl.MmaOpGFX11WMMAType.get(ctx, .{ + .m = 16, + .n = 16, + .k = 16, + .elemTyA = .float(ctx, .f16), + .elemTyB = .float(ctx, .f16), + .elemTyAcc = .float(ctx, .f32), + })).type_() }); + try expectPrints("!fly.layout<32:1>", wmma.thrLayout()); + + const ca = expect(try mlir.Type.parse(ctx, "!fly.copy_atom, 32>"), .copy_atom); + _ = ca.thrValLayoutSrc(); +} + +test "type kinds" { + const ctx = try testContext(); + defer ctx.deinit(); + const cases = .{ + .{ "!fly.memref", TypeKind.memref }, + .{ "!fly.ptr", TypeKind.ptr }, + .{ "!fly.int_tuple<(2,4)>", TypeKind.int_tuple }, + .{ "!fly.layout<(8,4):(4,1)>", TypeKind.layout }, + .{ "!fly.tile<[8|4]>", TypeKind.tile }, + .{ "!fly.copy_atom, 32>", TypeKind.copy_atom }, + .{ "!fly.mma_atom f32>>", TypeKind.mma_atom }, + }; + inline for (cases) |case| { + try std.testing.expectEqual(case[1], typeKind(try mlir.Type.parse(ctx, case[0]))); + } + try std.testing.expectEqual(TypeKind.other, typeKind(.float(ctx, .f32))); +} + +test "structural readers" { + const ctx = try testContext(); + defer ctx.deinit(); + const it = expect(try mlir.Type.parse(ctx, "!fly.int_tuple<((2,4),?)>"), .int_tuple); + try std.testing.expectEqual(@as(usize, 2), it.numElements()); + try std.testing.expect(!it.isLeaf()); + try std.testing.expect(!it.isStatic()); + const inner = it.element(0); + try std.testing.expectEqual(@as(usize, 2), inner.numElements()); + try std.testing.expectEqual(types.IntTupleType.Leaf{ .static = 4 }, inner.element(1).leaf()); + try std.testing.expectEqual(types.IntTupleType.Leaf.dynamic, it.element(1).leaf()); + + const m = try mlir.Type.parse(ctx, "!fly.memref o 0 o (64,32):(32,1), align<16>>"); + try expectPrints("!fly.int_tuple<(64,32)>", layoutLikeShape(m)); + try expectPrints("bf16", elemType(m)); +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + fly.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/fly/types.zig b/mlir/dialects/fly/types.zig index d974588c..27966997 100644 --- a/mlir/dialects/fly/types.zig +++ b/mlir/dialects/fly/types.zig @@ -1,5 +1,3 @@ -const std = @import("std"); - const c = @import("c"); const mlir = @import("mlir"); @@ -16,13 +14,11 @@ pub const IntTupleType = opaque { pub const isAFn = c.mlirTypeIsAFlyIntTuple; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub const InitArgs = struct { attr: *const attributes.IntTupleAttr, @@ -32,16 +28,16 @@ pub const IntTupleType = opaque { return @ptrCast(c.mlirFlyIntTupleTypeGet(ctx.ptr(), args.attr.ptr()).ptr orelse return error.InvalidMlir); } - pub fn getAttr(self: *const Self) *const attributes.IntTupleAttr { + pub fn attr(self: *const Self) *const attributes.IntTupleAttr { return @ptrCast(c.mlirFlyIntTupleTypeGetAttr(self.ptr()).ptr.?); } /// 1 for a leaf. - pub fn getNumElements(self: *const Self) usize { + pub fn numElements(self: *const Self) usize { return @intCast(c.mlirFlyIntTupleTypeGetNumElements(self.ptr())); } - pub fn getElement(self: *const Self, pos: usize) *const Self { + pub fn element(self: *const Self, pos: usize) *const Self { return @ptrCast(c.mlirFlyIntTupleTypeGetElement(self.ptr(), @intCast(pos)).ptr.?); } @@ -56,7 +52,7 @@ pub const IntTupleType = opaque { pub const Leaf = union(enum) { static: i64, dynamic, none, basis }; /// Asserts the tuple is a leaf. - pub fn getLeaf(self: *const Self) Leaf { + pub fn leaf(self: *const Self) Leaf { if (c.mlirFlyIntTupleTypeIsStaticLeaf(self.ptr())) { return .{ .static = c.mlirFlyIntTupleTypeGetStaticValue(self.ptr()) }; } @@ -73,13 +69,11 @@ pub const LayoutType = opaque { pub const isAFn = c.mlirTypeIsAFlyLayout; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub const InitArgs = struct { attr: *const attributes.LayoutAttr, @@ -89,15 +83,15 @@ pub const LayoutType = opaque { return @ptrCast(c.mlirFlyLayoutTypeGet(ctx.ptr(), args.attr.ptr()).ptr orelse return error.InvalidMlir); } - pub fn getAttr(self: *const Self) *const attributes.LayoutAttr { + pub fn attr(self: *const Self) *const attributes.LayoutAttr { return @ptrCast(c.mlirFlyLayoutTypeGetAttr(self.ptr()).ptr.?); } - pub fn getShape(self: *const Self) *const IntTupleType { + pub fn shape(self: *const Self) *const IntTupleType { return @ptrCast(c.mlirFlyLayoutTypeGetShape(self.ptr()).ptr.?); } - pub fn getStride(self: *const Self) *const IntTupleType { + pub fn stride(self: *const Self) *const IntTupleType { return @ptrCast(c.mlirFlyLayoutTypeGetStride(self.ptr()).ptr.?); } }; @@ -109,15 +103,13 @@ pub const ComposedLayoutType = opaque { pub const isAFn = c.mlirTypeIsAFlyComposedLayout; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } - pub fn getAttr(self: *const Self) *const mlir.Attribute { + pub fn attr(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirFlyComposedLayoutTypeGetAttr(self.ptr()).ptr.?); } }; @@ -129,13 +121,11 @@ pub const TileType = opaque { pub const isAFn = c.mlirTypeIsAFlyTile; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub const InitArgs = struct { attr: *const attributes.TileAttr, @@ -145,7 +135,7 @@ pub const TileType = opaque { return @ptrCast(c.mlirFlyTileTypeGet(ctx.ptr(), args.attr.ptr()).ptr orelse return error.InvalidMlir); } - pub fn getAttr(self: *const Self) *const attributes.TileAttr { + pub fn attr(self: *const Self) *const attributes.TileAttr { return @ptrCast(c.mlirFlyTileTypeGetAttr(self.ptr()).ptr.?); } }; @@ -157,13 +147,11 @@ pub const SwizzleType = opaque { pub const isAFn = c.mlirTypeIsAFlySwizzle; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub const InitArgs = struct { attr: *const attributes.SwizzleAttr, @@ -181,13 +169,11 @@ pub const PointerType = opaque { pub const isAFn = c.mlirTypeIsAFlyPointer; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub const InitArgs = struct { elemTy: *const mlir.Type, @@ -201,19 +187,19 @@ pub const PointerType = opaque { return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getElemTy(self: *const Self) *const mlir.Type { + pub fn elemTy(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirFlyPointerTypeGetElemTy(self.ptr()).ptr.?); } - pub fn getAddressSpace(self: *const Self) *const mlir.Attribute { + pub fn addressSpace(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirFlyPointerTypeGetAddressSpace(self.ptr()).ptr.?); } - pub fn getAlignment(self: *const Self) *const attributes.AlignAttr { + pub fn alignment(self: *const Self) *const attributes.AlignAttr { return @ptrCast(c.mlirFlyPointerTypeGetAlignment(self.ptr()).ptr.?); } - pub fn getSwizzle(self: *const Self) *const attributes.SwizzleAttr { + pub fn swizzle(self: *const Self) *const attributes.SwizzleAttr { return @ptrCast(c.mlirFlyPointerTypeGetSwizzle(self.ptr()).ptr.?); } }; @@ -225,13 +211,11 @@ pub const MemRefType = opaque { pub const isAFn = c.mlirTypeIsAFlyMemRef; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub const InitArgs = struct { elemTy: *const mlir.Type, @@ -246,15 +230,15 @@ pub const MemRefType = opaque { return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getElemTy(self: *const Self) *const mlir.Type { + pub fn elemTy(self: *const Self) *const mlir.Type { return @ptrCast(c.mlirFlyMemRefTypeGetElemTy(self.ptr()).ptr.?); } - pub fn getAddressSpace(self: *const Self) *const mlir.Attribute { + pub fn addressSpace(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirFlyMemRefTypeGetAddressSpace(self.ptr()).ptr.?); } - pub fn getLayout(self: *const Self) *const mlir.Attribute { + pub fn layout(self: *const Self) *const mlir.Attribute { return @ptrCast(c.mlirFlyMemRefTypeGetLayout(self.ptr()).ptr.?); } }; @@ -266,13 +250,11 @@ pub const CoordTensorType = opaque { pub const isAFn = c.mlirTypeIsAFlyCoordTensor; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub const InitArgs = struct { base: *const attributes.IntTupleAttr, @@ -292,13 +274,11 @@ pub const CopyAtomType = opaque { pub const isAFn = c.mlirTypeIsAFlyCopyAtom; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub const InitArgs = struct { copyOp: *const mlir.Type, @@ -310,19 +290,19 @@ pub const CopyAtomType = opaque { return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getThrLayout(self: *const Self) *const LayoutType { + pub fn thrLayout(self: *const Self) *const LayoutType { return @ptrCast(c.mlirFlyCopyAtomTypeGetThrLayout(self.ptr()).ptr.?); } - pub fn getThrValLayoutSrc(self: *const Self) *const LayoutType { + pub fn thrValLayoutSrc(self: *const Self) *const LayoutType { return @ptrCast(c.mlirFlyCopyAtomTypeGetThrValLayoutSrc(self.ptr()).ptr.?); } - pub fn getThrValLayoutDst(self: *const Self) *const LayoutType { + pub fn thrValLayoutDst(self: *const Self) *const LayoutType { return @ptrCast(c.mlirFlyCopyAtomTypeGetThrValLayoutDst(self.ptr()).ptr.?); } - pub fn getThrValLayoutRef(self: *const Self) *const LayoutType { + pub fn thrValLayoutRef(self: *const Self) *const LayoutType { return @ptrCast(c.mlirFlyCopyAtomTypeGetThrValLayoutRef(self.ptr()).ptr.?); } }; @@ -334,13 +314,11 @@ pub const MmaAtomType = opaque { pub const isAFn = c.mlirTypeIsAFlyMmaAtom; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub const InitArgs = struct { mmaOp: *const mlir.Type, @@ -350,23 +328,23 @@ pub const MmaAtomType = opaque { return @ptrCast(c.mlirFlyMmaAtomTypeGet(ctx.ptr(), args.mmaOp.ptr()).ptr orelse return error.InvalidMlir); } - pub fn getThrLayout(self: *const Self) *const LayoutType { + pub fn thrLayout(self: *const Self) *const LayoutType { return @ptrCast(c.mlirFlyMmaAtomTypeGetThrLayout(self.ptr()).ptr.?); } - pub fn getShapeMNK(self: *const Self) *const IntTupleType { + pub fn shapeMNK(self: *const Self) *const IntTupleType { return @ptrCast(c.mlirFlyMmaAtomTypeGetShapeMNK(self.ptr()).ptr.?); } - pub fn getThrValLayoutA(self: *const Self) *const LayoutType { + pub fn thrValLayoutA(self: *const Self) *const LayoutType { return @ptrCast(c.mlirFlyMmaAtomTypeGetThrValLayoutA(self.ptr()).ptr.?); } - pub fn getThrValLayoutB(self: *const Self) *const LayoutType { + pub fn thrValLayoutB(self: *const Self) *const LayoutType { return @ptrCast(c.mlirFlyMmaAtomTypeGetThrValLayoutB(self.ptr()).ptr.?); } - pub fn getThrValLayoutC(self: *const Self) *const LayoutType { + pub fn thrValLayoutC(self: *const Self) *const LayoutType { return @ptrCast(c.mlirFlyMmaAtomTypeGetThrValLayoutC(self.ptr()).ptr.?); } }; @@ -378,19 +356,17 @@ pub const TiledCopyType = opaque { pub const isAFn = c.mlirTypeIsAFlyTiledCopy; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } - pub fn getTiledThrValLayoutSrc(self: *const Self) *const LayoutType { + pub fn tiledThrValLayoutSrc(self: *const Self) *const LayoutType { return @ptrCast(c.mlirFlyTiledCopyTypeGetTiledThrValLayoutSrc(self.ptr()).ptr.?); } - pub fn getTiledThrValLayoutDst(self: *const Self) *const LayoutType { + pub fn tiledThrValLayoutDst(self: *const Self) *const LayoutType { return @ptrCast(c.mlirFlyTiledCopyTypeGetTiledThrValLayoutDst(self.ptr()).ptr.?); } }; @@ -402,31 +378,29 @@ pub const TiledMmaType = opaque { pub const isAFn = c.mlirTypeIsAFlyTiledMma; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } - pub fn getTileSizeMNK(self: *const Self) *const IntTupleType { + pub fn tileSizeMNK(self: *const Self) *const IntTupleType { return @ptrCast(c.mlirFlyTiledMmaTypeGetTileSizeMNK(self.ptr()).ptr.?); } - pub fn getThrLayoutVMNK(self: *const Self) *const LayoutType { + pub fn thrLayoutVMNK(self: *const Self) *const LayoutType { return @ptrCast(c.mlirFlyTiledMmaTypeGetThrLayoutVMNK(self.ptr()).ptr.?); } - pub fn getTiledThrValLayoutA(self: *const Self) *const LayoutType { + pub fn tiledThrValLayoutA(self: *const Self) *const LayoutType { return @ptrCast(c.mlirFlyTiledMmaTypeGetTiledThrValLayoutA(self.ptr()).ptr.?); } - pub fn getTiledThrValLayoutB(self: *const Self) *const LayoutType { + pub fn tiledThrValLayoutB(self: *const Self) *const LayoutType { return @ptrCast(c.mlirFlyTiledMmaTypeGetTiledThrValLayoutB(self.ptr()).ptr.?); } - pub fn getTiledThrValLayoutC(self: *const Self) *const LayoutType { + pub fn tiledThrValLayoutC(self: *const Self) *const LayoutType { return @ptrCast(c.mlirFlyTiledMmaTypeGetTiledThrValLayoutC(self.ptr()).ptr.?); } }; @@ -438,13 +412,11 @@ pub const CopyOpUniversalCopyType = opaque { pub const isAFn = c.mlirTypeIsAFlyCopyOpUniversalCopy; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub const InitArgs = struct { bitSize: i32, diff --git a/mlir/dialects/fly_rocdl/BUILD.bazel b/mlir/dialects/fly_rocdl/BUILD.bazel new file mode 100644 index 00000000..334e520b --- /dev/null +++ b/mlir/dialects/fly_rocdl/BUILD.bazel @@ -0,0 +1,43 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = [ + "capi.cc", + ], + hdrs = [ + "capi.h", + ], + tags = ["manual"], + deps = [ + "@flydsl//:FlyROCDLDialect", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "fly_rocdl", + srcs = [ + "ops.zig", + ], + import_name = "mlir/dialects/fly_rocdl", + main = "fly_rocdl.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//mlir/dialects/fly", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":fly_rocdl", + "//mlir", + ], +) diff --git a/mlir/dialects/fly_rocdl/capi.cc b/mlir/dialects/fly_rocdl/capi.cc new file mode 100644 index 00000000..b055e993 --- /dev/null +++ b/mlir/dialects/fly_rocdl/capi.cc @@ -0,0 +1,107 @@ +//===- capi.cc - C API for the fly_rocdl dialect --------------------------===// + +#include "capi.h" + +#include "flydsl/Dialect/FlyROCDL/IR/Dialect.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(FlyROCDL, fly_rocdl, + mlir::fly_rocdl::FlyROCDLDialect) + +static mlir::MLIRContext *loadFlyROCDL(MlirContext context) { + mlir::MLIRContext *ctx = unwrap(context); + if (ctx) + ctx->getOrLoadDialect(); + return ctx; +} + +/// Null test for unwrapped handles, kept in one place as in fly's C API, where +/// `!attr` is value arithmetic on a fly IntAttr rather than a null test. +template static bool isNull(T value) { + return !static_cast(value); +} + +template +static MlirType mmaOpGet(MlirContext context, int32_t m, int32_t n, int32_t k, + MlirType elemTyA, MlirType elemTyB, MlirType elemTyAcc, + Flags... flags) { + mlir::MLIRContext *ctx = loadFlyROCDL(context); + mlir::Type a = unwrap(elemTyA), b = unwrap(elemTyB), acc = unwrap(elemTyAcc); + if (!ctx || isNull(a) || isNull(b) || isNull(acc)) + return {nullptr}; + if (a.getContext() != ctx || b.getContext() != ctx || acc.getContext() != ctx) + return {nullptr}; + return wrap( + static_cast(Type::get(ctx, m, n, k, a, b, acc, flags...))); +} + +#define FLY_ROCDL_TYPE_ISA(Name, Type) \ + bool mlirTypeIsAFlyROCDL##Name(MlirType type) { \ + return mlir::isa_and_nonnull(unwrap(type)); \ + } + +FLY_ROCDL_TYPE_ISA(CopyOpCDNA3BufferCopy, + mlir::fly_rocdl::CopyOpCDNA3BufferCopyType) +FLY_ROCDL_TYPE_ISA(CopyOpCDNA3BufferCopyLDS, + mlir::fly_rocdl::CopyOpCDNA3BufferCopyLDSType) +FLY_ROCDL_TYPE_ISA(MmaOpCDNA3MFMA, mlir::fly_rocdl::MmaOpCDNA3_MFMAType) +FLY_ROCDL_TYPE_ISA(MmaOpGFX11WMMA, mlir::fly_rocdl::MmaOpGFX11_WMMAType) +FLY_ROCDL_TYPE_ISA(MmaOpGFX120XWMMA, mlir::fly_rocdl::MmaOpGFX120X_WMMAType) + +#undef FLY_ROCDL_TYPE_ISA + +MlirAttribute mlirFlyROCDLBufferDescAddressAttrGet(MlirContext context) { + mlir::MLIRContext *ctx = loadFlyROCDL(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast( + mlir::fly_rocdl::BufferDescAddressAttr::get(ctx))); +} + +MlirType mlirFlyROCDLCopyOpCDNA3BufferCopyTypeGet(MlirContext context, + int32_t bitSize, + int32_t cacheModifier) { + mlir::MLIRContext *ctx = loadFlyROCDL(context); + if (!ctx) + return {nullptr}; + return wrap( + static_cast(mlir::fly_rocdl::CopyOpCDNA3BufferCopyType::get( + ctx, bitSize, cacheModifier))); +} + +MlirType mlirFlyROCDLCopyOpCDNA3BufferCopyLDSTypeGet(MlirContext context, + int32_t bitSize) { + mlir::MLIRContext *ctx = loadFlyROCDL(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast( + mlir::fly_rocdl::CopyOpCDNA3BufferCopyLDSType::get(ctx, bitSize))); +} + +MlirType mlirFlyROCDLMmaOpCDNA3MFMATypeGet(MlirContext context, int32_t m, + int32_t n, int32_t k, + MlirType elemTyA, MlirType elemTyB, + MlirType elemTyAcc) { + return mmaOpGet( + context, m, n, k, elemTyA, elemTyB, elemTyAcc); +} + +MlirType mlirFlyROCDLMmaOpGFX11WMMATypeGet(MlirContext context, int32_t m, + int32_t n, int32_t k, + MlirType elemTyA, MlirType elemTyB, + MlirType elemTyAcc, bool signA, + bool signB, bool clamp) { + return mmaOpGet( + context, m, n, k, elemTyA, elemTyB, elemTyAcc, signA, signB, clamp); +} + +MlirType mlirFlyROCDLMmaOpGFX120XWMMATypeGet(MlirContext context, int32_t m, + int32_t n, int32_t k, + MlirType elemTyA, MlirType elemTyB, + MlirType elemTyAcc, bool signA, + bool signB, bool clamp) { + return mmaOpGet( + context, m, n, k, elemTyA, elemTyB, elemTyAcc, signA, signB, clamp); +} diff --git a/mlir/dialects/fly_rocdl/capi.h b/mlir/dialects/fly_rocdl/capi.h new file mode 100644 index 00000000..0ee248bb --- /dev/null +++ b/mlir/dialects/fly_rocdl/capi.h @@ -0,0 +1,45 @@ +//===- capi.h - C API for the fly_rocdl dialect -------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_FLY_ROCDL_CAPI_H +#define MLIR_DIALECTS_FLY_ROCDL_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(FlyROCDL, fly_rocdl); + +MLIR_CAPI_EXPORTED MlirAttribute +mlirFlyROCDLBufferDescAddressAttrGet(MlirContext context); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLCopyOpCDNA3BufferCopy(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLCopyOpCDNA3BufferCopyTypeGet( + MlirContext context, int32_t bitSize, int32_t cacheModifier); + +MLIR_CAPI_EXPORTED bool +mlirTypeIsAFlyROCDLCopyOpCDNA3BufferCopyLDS(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLCopyOpCDNA3BufferCopyLDSTypeGet( + MlirContext context, int32_t bitSize); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLMmaOpCDNA3MFMA(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLMmaOpCDNA3MFMATypeGet( + MlirContext context, int32_t m, int32_t n, int32_t k, MlirType elemTyA, + MlirType elemTyB, MlirType elemTyAcc); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLMmaOpGFX11WMMA(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLMmaOpGFX11WMMATypeGet( + MlirContext context, int32_t m, int32_t n, int32_t k, MlirType elemTyA, + MlirType elemTyB, MlirType elemTyAcc, bool signA, bool signB, bool clamp); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLMmaOpGFX120XWMMA(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLMmaOpGFX120XWMMATypeGet( + MlirContext context, int32_t m, int32_t n, int32_t k, MlirType elemTyA, + MlirType elemTyB, MlirType elemTyAcc, bool signA, bool signB, bool clamp); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_FLY_ROCDL_CAPI_H diff --git a/mlir/dialects/fly/rocdl.zig b/mlir/dialects/fly_rocdl/fly_rocdl.zig similarity index 84% rename from mlir/dialects/fly/rocdl.zig rename to mlir/dialects/fly_rocdl/fly_rocdl.zig index fa8a8a56..7a91a925 100644 --- a/mlir/dialects/fly/rocdl.zig +++ b/mlir/dialects/fly_rocdl/fly_rocdl.zig @@ -1,9 +1,11 @@ -const std = @import("std"); - const c = @import("c"); const mlir = @import("mlir"); -pub const ops = @import("rocdl_ops.zig"); +pub const ops = @import("ops.zig"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("fly_rocdl"); +} /// `#fly_rocdl.buffer_desc`, an address space for buffer-descriptor pointers. pub const BufferDescAddressAttr = opaque { @@ -11,12 +13,10 @@ pub const BufferDescAddressAttr = opaque { const M = mlir.Methods(Self, c.MlirAttribute); pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn get(ctx: *mlir.Context) mlir.Error!*const Self { return @ptrCast(c.mlirFlyROCDLBufferDescAddressAttrGet(ctx.ptr()).ptr orelse return error.InvalidMlir); @@ -30,13 +30,11 @@ pub const CopyOpCDNA3BufferCopyType = opaque { pub const isAFn = c.mlirTypeIsAFlyROCDLCopyOpCDNA3BufferCopy; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub const InitArgs = struct { bitSize: i32, @@ -57,13 +55,11 @@ pub const CopyOpCDNA3BufferCopyLDSType = opaque { pub const isAFn = c.mlirTypeIsAFlyROCDLCopyOpCDNA3BufferCopyLDS; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub const InitArgs = struct { bitSize: i32, @@ -82,13 +78,11 @@ pub const MmaOpCDNA3MFMAType = opaque { pub const isAFn = c.mlirTypeIsAFlyROCDLMmaOpCDNA3MFMA; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub const InitArgs = struct { m: i32, @@ -112,13 +106,11 @@ pub const MmaOpGFX11WMMAType = opaque { pub const isAFn = c.mlirTypeIsAFlyROCDLMmaOpGFX11WMMA; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub const InitArgs = WmmaInitArgs; @@ -135,13 +127,11 @@ pub const MmaOpGFX120XWMMAType = opaque { pub const isAFn = c.mlirTypeIsAFlyROCDLMmaOpGFX120XWMMA; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub const InitArgs = WmmaInitArgs; diff --git a/mlir/dialects/fly_rocdl/ops.zig b/mlir/dialects/fly_rocdl/ops.zig new file mode 100644 index 00000000..6cff2bf6 --- /dev/null +++ b/mlir/dialects/fly_rocdl/ops.zig @@ -0,0 +1,83 @@ +//! `fly_rocdl` operation builders, generated from FlyDSL's `include/flydsl/Dialect/FlyROCDL/IR/Ops.td` at +//! 1941889400621f1fc3d8f03bada1a4e7380cdf7e; do not edit, see ../fly/REGENERATING.md. +//! +//! Builders take the operands, then the result types (only for operations +//! that do not infer them), then the required attributes, then an `opts` +//! struct of optional operands and attributes (pass `.{}` to omit them), then +//! the location. Parameters are the ODS names in snake case, with a trailing +//! underscore where needed. Pass `&.{}` for empty variadic operands; +//! variable-length operand groups of +//! `AttrSizedOperandSegments` operations get their segment sizes from +//! `Operation.make`. Attributes use `mlir.Attribute`. + +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +const make = @import("mlir/dialects/fly").make; + +/// `fly_rocdl.get_buffer_rsrc`. Extract the raw ROCDL buffer resource from a buffer-descriptor pointer. Result types are inferred. +pub fn get_buffer_rsrc(ctx: *mlir.Context, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly_rocdl.get_buffer_rsrc", .{ + .operands = .{ .flat = &.{ptr} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly_rocdl.make_tiled_tdm_load_atom`. Build a CDNA5 TDM load atom and the coordinate tensor that addresses it. Result types are inferred. +pub fn make_tiled_tdm_load_atom( + ctx: *mlir.Context, + tensor: *const mlir.Value, + smem_layout: *const mlir.Value, + tiler: *const mlir.Value, + opts: struct { + num_warps: ?*const mlir.Attribute = null, + init_boundary_check: ?*const mlir.Attribute = null, + cache_modifier: ?*const mlir.Attribute = null, + atomic_barrier: ?*const mlir.Attribute = null, + internal_type: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.num_warps) |present| attributes.appendAssumeCapacity(.named(ctx, "numWarps", present)); + if (opts.init_boundary_check) |present| attributes.appendAssumeCapacity(.named(ctx, "initBoundaryCheck", present)); + if (opts.cache_modifier) |present| attributes.appendAssumeCapacity(.named(ctx, "cacheModifier", present)); + if (opts.atomic_barrier) |present| attributes.appendAssumeCapacity(.named(ctx, "atomicBarrier", present)); + if (opts.internal_type) |present| attributes.appendAssumeCapacity(.named(ctx, "internalType", present)); + return make(ctx, "fly_rocdl.make_tiled_tdm_load_atom", .{ + .operands = .{ .flat = &.{ tensor, smem_layout, tiler } }, + .attributes = attributes.constSlice(), + .result_type_inference = true, + .location = location, + }); +} + +/// `fly_rocdl.make_tiled_tdm_store_atom`. Build a CDNA5 TDM store atom and the coordinate tensor that addresses it. Result types are inferred. +pub fn make_tiled_tdm_store_atom( + ctx: *mlir.Context, + tensor: *const mlir.Value, + smem_layout: *const mlir.Value, + tiler: *const mlir.Value, + opts: struct { + num_warps: ?*const mlir.Attribute = null, + init_boundary_check: ?*const mlir.Attribute = null, + cache_modifier: ?*const mlir.Attribute = null, + atomic_barrier: ?*const mlir.Attribute = null, + internal_type: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.num_warps) |present| attributes.appendAssumeCapacity(.named(ctx, "numWarps", present)); + if (opts.init_boundary_check) |present| attributes.appendAssumeCapacity(.named(ctx, "initBoundaryCheck", present)); + if (opts.cache_modifier) |present| attributes.appendAssumeCapacity(.named(ctx, "cacheModifier", present)); + if (opts.atomic_barrier) |present| attributes.appendAssumeCapacity(.named(ctx, "atomicBarrier", present)); + if (opts.internal_type) |present| attributes.appendAssumeCapacity(.named(ctx, "internalType", present)); + return make(ctx, "fly_rocdl.make_tiled_tdm_store_atom", .{ + .operands = .{ .flat = &.{ tensor, smem_layout, tiler } }, + .attributes = attributes.constSlice(), + .result_type_inference = true, + .location = location, + }); +} diff --git a/mlir/dialects/fly_rocdl/test.zig b/mlir/dialects/fly_rocdl/test.zig new file mode 100644 index 00000000..bb85d6e5 --- /dev/null +++ b/mlir/dialects/fly_rocdl/test.zig @@ -0,0 +1,30 @@ +const std = @import("std"); + +const fly_rocdl = @import("mlir/dialects/fly_rocdl"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "fly_rocdl.get_buffer_rsrc", + "fly_rocdl.make_tiled_tdm_load_atom", + "fly_rocdl.make_tiled_tdm_store_atom", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(fly_rocdl); + inline for (comptime std.meta.declarations(fly_rocdl)) |decl| { + const T = @field(fly_rocdl, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + fly_rocdl.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/func/BUILD.bazel b/mlir/dialects/func/BUILD.bazel new file mode 100644 index 00000000..4524f2dd --- /dev/null +++ b/mlir/dialects/func/BUILD.bazel @@ -0,0 +1,35 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:CAPIIR", + "@llvm-project//mlir:FuncExtensions", + ], +) + +zig_library( + name = "func", + import_name = "mlir/dialects/func", + main = "func.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":func", + "//mlir", + ], +) diff --git a/mlir/dialects/func/capi.cc b/mlir/dialects/func/capi.cc new file mode 100644 index 00000000..11a5e006 --- /dev/null +++ b/mlir/dialects/func/capi.cc @@ -0,0 +1,10 @@ +//===- capi.cc - C API for the func dialect -------------------------------===// + +#include "capi.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/Dialect/Func/Extensions/AllExtensions.h" + +void mlirRegisterFuncExtensions(MlirDialectRegistry registry) { + mlir::func::registerAllExtensions(*unwrap(registry)); +} diff --git a/mlir/dialects/func/capi.h b/mlir/dialects/func/capi.h new file mode 100644 index 00000000..cd522b2d --- /dev/null +++ b/mlir/dialects/func/capi.h @@ -0,0 +1,20 @@ +//===- capi.h - C API for the func dialect ------------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_FUNC_CAPI_H +#define MLIR_DIALECTS_FUNC_CAPI_H + +#include "mlir-c/Dialect/Func.h" +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_CAPI_EXPORTED void +mlirRegisterFuncExtensions(MlirDialectRegistry registry); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_FUNC_CAPI_H diff --git a/mlir/dialects/func.zig b/mlir/dialects/func/func.zig similarity index 92% rename from mlir/dialects/func.zig rename to mlir/dialects/func/func.zig index 93407bd7..45c74674 100644 --- a/mlir/dialects/func.zig +++ b/mlir/dialects/func/func.zig @@ -1,8 +1,12 @@ -const std = @import("std"); +const c = @import("c"); const mlir = @import("mlir"); const stdx = @import("stdx"); +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("func"); +} + const FuncOpArgs = struct { pub const Visibility = enum { public, @@ -17,7 +21,7 @@ const FuncOpArgs = struct { block: *mlir.Block, location: *const mlir.Location, no_inline: bool = false, - visibility: ?Visibility = .public, + visibility: ?Visibility = null, verify: bool = true, extra_attributes: []const mlir.NamedAttribute = &.{}, }; @@ -99,3 +103,7 @@ pub fn returns(ctx: *mlir.Context, values: []const *const mlir.Value, location: .location = location, }); } + +pub fn registerExtensions(registry: *mlir.DialectRegistry) void { + c.mlirRegisterFuncExtensions(registry.ptr()); +} diff --git a/mlir/dialects/func/test.zig b/mlir/dialects/func/test.zig new file mode 100644 index 00000000..7a3b17f9 --- /dev/null +++ b/mlir/dialects/func/test.zig @@ -0,0 +1,41 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "func.call", + "func.func", + "func.return", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(func); + inline for (comptime std.meta.declarations(func)) |decl| { + const T = @field(func, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "func dialect and extensions register" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + func.registerDialect(registry); + func.registerExtensions(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + try std.testing.expect(ctx.isRegisteredOperation("func.func")); +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + func.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/gpu/BUILD.bazel b/mlir/dialects/gpu/BUILD.bazel new file mode 100644 index 00000000..7264487a --- /dev/null +++ b/mlir/dialects/gpu/BUILD.bazel @@ -0,0 +1,35 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:CAPIIR", + "@llvm-project//mlir:GPUDialect", + ], +) + +zig_library( + name = "gpu", + import_name = "mlir/dialects/gpu", + main = "gpu.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":gpu", + "//mlir", + ], +) diff --git a/mlir/dialects/gpu/capi.cc b/mlir/dialects/gpu/capi.cc new file mode 100644 index 00000000..2e215d58 --- /dev/null +++ b/mlir/dialects/gpu/capi.cc @@ -0,0 +1,52 @@ +//===- capi.cc - C API for the gpu dialect --------------------------------===// + +#include "capi.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" +#include "mlir/Dialect/GPU/IR/GPUDialect.h" + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(GPU, gpu, mlir::gpu::GPUDialect) + +static mlir::MLIRContext *loadGPU(MlirContext context) { + mlir::MLIRContext *ctx = unwrap(context); + if (ctx) + ctx->getOrLoadDialect(); + return ctx; +} + +bool mlirAttributeIsAGPUDimension(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirGPUDimensionAttrGet(MlirContext context, uint32_t value) { + mlir::MLIRContext *ctx = loadGPU(context); + std::optional dimension = + mlir::gpu::symbolizeDimension(value); + if (!ctx || !dimension) + return {nullptr}; + return wrap(mlir::gpu::DimensionAttr::get(ctx, *dimension)); +} + +uint32_t mlirGPUDimensionAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAGPUShuffleMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirGPUShuffleModeAttrGet(MlirContext context, uint32_t value) { + mlir::MLIRContext *ctx = loadGPU(context); + std::optional mode = + mlir::gpu::symbolizeShuffleMode(value); + if (!ctx || !mode) + return {nullptr}; + return wrap(mlir::gpu::ShuffleModeAttr::get(ctx, *mode)); +} + +uint32_t mlirGPUShuffleModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} diff --git a/mlir/dialects/gpu_capi.h b/mlir/dialects/gpu/capi.h similarity index 72% rename from mlir/dialects/gpu_capi.h rename to mlir/dialects/gpu/capi.h index 033042e0..670e23a9 100644 --- a/mlir/dialects/gpu_capi.h +++ b/mlir/dialects/gpu/capi.h @@ -1,11 +1,11 @@ +//===- capi.h - C API for the gpu dialect -------------------------*- C -*-===// +// // C entry points for the `gpu` dialect that upstream's C API leaves out: its // registration (outside `CAPIGPU`, which also brings the GPU passes) and the // enum attributes kernels build. -#ifndef ZML_MLIR_DIALECTS_GPU_CAPI_H -#define ZML_MLIR_DIALECTS_GPU_CAPI_H -#include -#include +#ifndef MLIR_DIALECTS_GPU_CAPI_H +#define MLIR_DIALECTS_GPU_CAPI_H #include "mlir-c/IR.h" @@ -20,16 +20,18 @@ MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(GPU, gpu); // #gpu MLIR_CAPI_EXPORTED bool mlirAttributeIsAGPUDimension(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirGPUDimensionAttrGet(MlirContext ctx, uint32_t value); +MLIR_CAPI_EXPORTED MlirAttribute mlirGPUDimensionAttrGet(MlirContext context, + uint32_t value); MLIR_CAPI_EXPORTED uint32_t mlirGPUDimensionAttrGetValue(MlirAttribute attr); // #gpu MLIR_CAPI_EXPORTED bool mlirAttributeIsAGPUShuffleMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirGPUShuffleModeAttrGet(MlirContext ctx, uint32_t value); +MLIR_CAPI_EXPORTED MlirAttribute mlirGPUShuffleModeAttrGet(MlirContext context, + uint32_t value); MLIR_CAPI_EXPORTED uint32_t mlirGPUShuffleModeAttrGetValue(MlirAttribute attr); #ifdef __cplusplus } #endif -#endif // ZML_MLIR_DIALECTS_GPU_CAPI_H +#endif // MLIR_DIALECTS_GPU_CAPI_H diff --git a/mlir/dialects/gpu.zig b/mlir/dialects/gpu/gpu.zig similarity index 78% rename from mlir/dialects/gpu.zig rename to mlir/dialects/gpu/gpu.zig index a6e16ebf..1e3d76fe 100644 --- a/mlir/dialects/gpu.zig +++ b/mlir/dialects/gpu/gpu.zig @@ -1,9 +1,11 @@ -const std = @import("std"); - const c = @import("c"); const mlir = @import("mlir"); const stdx = @import("stdx"); +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("gpu"); +} + /// A dimension, either `x`, `y` or `z`. pub const Dim = enum(u32) { x = 0, @@ -26,11 +28,9 @@ pub const DimensionAttr = opaque { pub const isAFn = c.mlirAttributeIsAGPUDimension; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "dim"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -41,7 +41,7 @@ pub const DimensionAttr = opaque { const result = c.mlirGPUDimensionAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) Dim { + pub fn value(self: *const Self) Dim { return @fromBackingInt(@intCast(c.mlirGPUDimensionAttrGetValue(self.ptr()))); } }; @@ -53,11 +53,9 @@ pub const ShuffleModeAttr = opaque { pub const isAFn = c.mlirAttributeIsAGPUShuffleMode; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "shuffle_mode"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -68,7 +66,7 @@ pub const ShuffleModeAttr = opaque { const result = c.mlirGPUShuffleModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) ShuffleMode { + pub fn value(self: *const Self) ShuffleMode { return @fromBackingInt(@intCast(c.mlirGPUShuffleModeAttrGetValue(self.ptr()))); } }; @@ -158,33 +156,3 @@ pub fn shuffle(ctx: *mlir.Context, value: *const mlir.Value, offset: *const mlir pub fn barrier(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "gpu.barrier", .{ .location = location }); } - -test { - std.testing.refAllDecls(@This()); -} - -fn expectPrints(expected: []const u8, value: anytype) !void { - var buf: [128]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{value}); - try std.testing.expectEqualStrings(expected, w.buffered()); -} - -test "gpu enum attributes are built through the C API" { - const ctx = try mlir.Context.init(.{}); - defer ctx.deinit(); - - inline for (@typeInfo(Dim).@"enum".field_names) |name| { - const value: Dim = @field(Dim, name); - const attr = try DimensionAttr.get(ctx, .{ .value = value }); - try std.testing.expectEqual(value, attr.getValue()); - try std.testing.expect(attr.attribute().isA(DimensionAttr) != null); - try expectPrints("#gpu.dim<" ++ name ++ ">", attr); - } - inline for (@typeInfo(ShuffleMode).@"enum".field_names) |name| { - const value: ShuffleMode = @field(ShuffleMode, name); - const attr = try ShuffleModeAttr.get(ctx, .{ .value = value }); - try std.testing.expectEqual(value, attr.getValue()); - try expectPrints("#gpu.shuffle_mode<" ++ name ++ ">", attr); - } -} diff --git a/mlir/dialects/gpu/test.zig b/mlir/dialects/gpu/test.zig new file mode 100644 index 00000000..bcf37065 --- /dev/null +++ b/mlir/dialects/gpu/test.zig @@ -0,0 +1,62 @@ +const std = @import("std"); + +const gpu = @import("mlir/dialects/gpu"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "gpu.barrier", + "gpu.block_dim", + "gpu.block_id", + "gpu.func", + "gpu.grid_dim", + "gpu.module", + "gpu.return", + "gpu.shuffle", + "gpu.thread_id", +}; + +fn expectPrints(expected: []const u8, value: anytype) !void { + var buf: [128]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(expected, w.buffered()); +} + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(gpu); + inline for (comptime std.meta.declarations(gpu)) |decl| { + const T = @field(gpu, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "gpu enum attributes are built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + inline for (@typeInfo(gpu.Dim).@"enum".field_names) |name| { + const value: gpu.Dim = @field(gpu.Dim, name); + const attr = try gpu.DimensionAttr.get(ctx, .{ .value = value }); + try std.testing.expectEqual(value, attr.value()); + try std.testing.expect(attr.attribute().isA(gpu.DimensionAttr) != null); + try expectPrints("#gpu.dim<" ++ name ++ ">", attr); + } + inline for (@typeInfo(gpu.ShuffleMode).@"enum".field_names) |name| { + const value: gpu.ShuffleMode = @field(gpu.ShuffleMode, name); + const attr = try gpu.ShuffleModeAttr.get(ctx, .{ .value = value }); + try std.testing.expectEqual(value, attr.value()); + try expectPrints("#gpu.shuffle_mode<" ++ name ++ ">", attr); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + gpu.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/gpu_capi.cc b/mlir/dialects/gpu_capi.cc deleted file mode 100644 index 2e8bd4a9..00000000 --- a/mlir/dialects/gpu_capi.cc +++ /dev/null @@ -1,47 +0,0 @@ -#include "gpu_capi.h" - -#include "mlir/CAPI/IR.h" -#include "mlir/CAPI/Registration.h" -#include "mlir/Dialect/GPU/IR/GPUDialect.h" - -MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(GPU, gpu, mlir::gpu::GPUDialect) - -namespace { - -mlir::MLIRContext *loadGPU(MlirContext ctx) { - mlir::MLIRContext *context = unwrap(ctx); - if (context) context->getOrLoadDialect(); - return context; -} - -} // namespace - -bool mlirAttributeIsAGPUDimension(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirGPUDimensionAttrGet(MlirContext ctx, uint32_t value) { - mlir::MLIRContext *context = loadGPU(ctx); - std::optional dimension = mlir::gpu::symbolizeDimension(value); - if (!context || !dimension) return {nullptr}; - return wrap(mlir::gpu::DimensionAttr::get(context, *dimension)); -} - -uint32_t mlirGPUDimensionAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsAGPUShuffleMode(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirGPUShuffleModeAttrGet(MlirContext ctx, uint32_t value) { - mlir::MLIRContext *context = loadGPU(ctx); - std::optional mode = mlir::gpu::symbolizeShuffleMode(value); - if (!context || !mode) return {nullptr}; - return wrap(mlir::gpu::ShuffleModeAttr::get(context, *mode)); -} - -uint32_t mlirGPUShuffleModeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); -} diff --git a/mlir/dialects/llvm/BUILD.bazel b/mlir/dialects/llvm/BUILD.bazel new file mode 100644 index 00000000..352025d2 --- /dev/null +++ b/mlir/dialects/llvm/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "llvm", + srcs = ["ops.zig"], + import_name = "mlir/dialects/llvm", + main = "llvm.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "//stdx", + "@llvm-project//mlir:CAPILLVM", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":llvm", + "//mlir", + ], +) diff --git a/mlir/dialects/llvm/llvm.zig b/mlir/dialects/llvm/llvm.zig new file mode 100644 index 00000000..a5241296 --- /dev/null +++ b/mlir/dialects/llvm/llvm.zig @@ -0,0 +1,17 @@ +// Keep function definitions in ops.zig: LLVM reserves symbols starting with "llvm.". +const ops = @import("ops.zig"); + +pub const NVPTXAddressSpace = ops.NVPTXAddressSpace; +pub const pointerType = ops.pointerType; +pub const structType = ops.structType; +pub const inttoptr = ops.inttoptr; +pub const bitcast = ops.bitcast; +pub const fmul = ops.fmul; +pub const LoadOpts = ops.LoadOpts; +pub const load = ops.load; +pub const StoreOpts = ops.StoreOpts; +pub const store = ops.store; +pub const call_intrinsic = ops.call_intrinsic; +pub const InlineAsmOpts = ops.InlineAsmOpts; +pub const inline_asm = ops.inline_asm; +pub const registerDialect = ops.registerDialect; diff --git a/mlir/dialects/llvm_dialect.zig b/mlir/dialects/llvm/ops.zig similarity index 73% rename from mlir/dialects/llvm_dialect.zig rename to mlir/dialects/llvm/ops.zig index 541c1373..19412dfd 100644 --- a/mlir/dialects/llvm_dialect.zig +++ b/mlir/dialects/llvm/ops.zig @@ -1,7 +1,10 @@ -const std = @import("std"); - const c = @import("c"); const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("llvm"); +} // ============================================================================= // Types @@ -26,6 +29,12 @@ pub fn pointerType(ctx: *mlir.Context, space: anytype) *const mlir.Type { return @ptrCast(c.mlirLLVMPointerTypeGet(ctx.ptr(), @intCast(@backingInt(space))).ptr); } +/// `!llvm.struct<(fields...)>`, a literal struct. +pub fn structType(ctx: *mlir.Context, fields: []const *const mlir.Type) *const mlir.Type { + _ = c.mlirContextGetOrLoadDialect(ctx.ptr(), mlir.stringRef("llvm")); + return @ptrCast(c.mlirLLVMStructTypeLiteralGet(ctx.ptr(), @intCast(fields.len), @ptrCast(fields), false).ptr); +} + // ============================================================================= // Casts and arithmetic // ============================================================================= @@ -61,7 +70,7 @@ pub fn fmul(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, // Memory // ============================================================================= -pub const LoadOptions = struct { +pub const LoadOpts = struct { /// Alignment in bytes (none: the natural alignment of the type). alignment: ?u64 = null, /// The memory is not written while the kernel runs (`!invariant.load`; on NVPTX @@ -70,41 +79,36 @@ pub const LoadOptions = struct { }; /// llvm.load — a value of type `result_type` through `pointer`. -pub fn load(ctx: *mlir.Context, pointer: *const mlir.Value, result_type: *const mlir.Type, opts: LoadOptions, location: *const mlir.Location) *mlir.Operation { - var attrs: [2]mlir.NamedAttribute = undefined; - var len: usize = 0; +pub fn load(ctx: *mlir.Context, pointer: *const mlir.Value, result_type: *const mlir.Type, opts: LoadOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; if (opts.alignment) |alignment| { - attrs[len] = .named(ctx, "alignment", .int(ctx, .i64, alignment)); - len += 1; + attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, alignment))); } if (opts.invariant) { - attrs[len] = .named(ctx, "invariant", .unit(ctx)); - len += 1; + attrs.appendAssumeCapacity(.named(ctx, "invariant", .unit(ctx))); } return mlir.Operation.make(ctx, "llvm.load", .{ .operands = .{ .flat = &.{pointer} }, .results = .{ .flat = &.{result_type} }, - .attributes = attrs[0..len], + .attributes = attrs.constSlice(), .location = location, }); } -pub const StoreOptions = struct { +pub const StoreOpts = struct { /// Alignment in bytes (none: the natural alignment of the type). alignment: ?u64 = null, }; /// llvm.store — `value` through `pointer`. -pub fn store(ctx: *mlir.Context, value: *const mlir.Value, pointer: *const mlir.Value, opts: StoreOptions, location: *const mlir.Location) *mlir.Operation { - var attrs: [1]mlir.NamedAttribute = undefined; - var len: usize = 0; +pub fn store(ctx: *mlir.Context, value: *const mlir.Value, pointer: *const mlir.Value, opts: StoreOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; if (opts.alignment) |alignment| { - attrs[len] = .named(ctx, "alignment", .int(ctx, .i64, alignment)); - len += 1; + attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, alignment))); } return mlir.Operation.make(ctx, "llvm.store", .{ .operands = .{ .flat = &.{ value, pointer } }, - .attributes = attrs[0..len], + .attributes = attrs.constSlice(), .location = location, }); } @@ -119,12 +123,14 @@ pub fn call_intrinsic( ctx: *mlir.Context, name: []const u8, operands: []const *const mlir.Value, - result_type: ?*const mlir.Type, + opts: struct { + result_type: ?*const mlir.Type = null, + }, location: *const mlir.Location, ) *mlir.Operation { return mlir.Operation.make(ctx, "llvm.call_intrinsic", .{ .operands = .{ .flat = operands }, - .results = .{ .flat = if (result_type) |r| &.{r} else &.{} }, + .results = .{ .flat = if (opts.result_type) |r| &.{r} else &.{} }, .attributes = &.{ .named(ctx, "intrin", .string(ctx, name)), .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ @intCast(operands.len), 0 })), @@ -134,7 +140,9 @@ pub fn call_intrinsic( }); } -pub const InlineAsmOptions = struct { +pub const InlineAsmOpts = struct { + result_type: ?*const mlir.Type = null, + /// LLVM constraint string, e.g. `"=r,r"` (outputs first). constraints: []const u8 = "", /// The assembly has effects beyond its outputs (it is never removed or merged). @@ -147,26 +155,19 @@ pub fn inline_asm( ctx: *mlir.Context, asm_string: []const u8, operands: []const *const mlir.Value, - result_type: ?*const mlir.Type, - opts: InlineAsmOptions, + opts: InlineAsmOpts, location: *const mlir.Location, ) *mlir.Operation { - var attrs: [3]mlir.NamedAttribute = undefined; - attrs[0] = .named(ctx, "asm_string", .string(ctx, asm_string)); - attrs[1] = .named(ctx, "constraints", .string(ctx, opts.constraints)); - var len: usize = 2; + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attrs.appendAssumeCapacity(.named(ctx, "asm_string", .string(ctx, asm_string))); + attrs.appendAssumeCapacity(.named(ctx, "constraints", .string(ctx, opts.constraints))); if (opts.has_side_effects) { - attrs[len] = .named(ctx, "has_side_effects", .unit(ctx)); - len += 1; + attrs.appendAssumeCapacity(.named(ctx, "has_side_effects", .unit(ctx))); } return mlir.Operation.make(ctx, "llvm.inline_asm", .{ .operands = .{ .flat = operands }, - .results = .{ .flat = if (result_type) |r| &.{r} else &.{} }, - .attributes = attrs[0..len], + .results = .{ .flat = if (opts.result_type) |r| &.{r} else &.{} }, + .attributes = attrs.constSlice(), .location = location, }); } - -test { - std.testing.refAllDecls(@This()); -} diff --git a/mlir/dialects/llvm/test.zig b/mlir/dialects/llvm/test.zig new file mode 100644 index 00000000..d5c2b282 --- /dev/null +++ b/mlir/dialects/llvm/test.zig @@ -0,0 +1,44 @@ +const std = @import("std"); + +const llvm = @import("mlir/dialects/llvm"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "llvm.bitcast", + "llvm.call_intrinsic", + "llvm.fmul", + "llvm.inline_asm", + "llvm.inttoptr", + "llvm.load", + "llvm.store", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(llvm); + inline for (comptime std.meta.declarations(llvm)) |decl| { + const T = @field(llvm, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "literal struct types build from their fields" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + const struct_type = llvm.structType(ctx, &.{ .int(ctx, .i32), .float(ctx, .f32) }); + try std.testing.expect(struct_type.eql(try mlir.Type.parse(ctx, "!llvm.struct<(i32, f32)>"))); +} diff --git a/mlir/dialects/math/BUILD.bazel b/mlir/dialects/math/BUILD.bazel new file mode 100644 index 00000000..3845c969 --- /dev/null +++ b/mlir/dialects/math/BUILD.bazel @@ -0,0 +1,22 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "math", + import_name = "mlir/dialects/math", + main = "math.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "@llvm-project//mlir:CAPIMath", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":math", + "//mlir", + ], +) diff --git a/mlir/dialects/math.zig b/mlir/dialects/math/math.zig similarity index 98% rename from mlir/dialects/math.zig rename to mlir/dialects/math/math.zig index 39ce5666..4ca6ee6d 100644 --- a/mlir/dialects/math.zig +++ b/mlir/dialects/math/math.zig @@ -1,7 +1,9 @@ -const std = @import("std"); - const mlir = @import("mlir"); +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("math"); +} + // ============================================================================= // Float unary — SameOperandsAndResultType. Result type inferred from operand. // ============================================================================= @@ -206,7 +208,3 @@ pub const isfinite = classifyOp("math.isfinite").call; pub const isinf = classifyOp("math.isinf").call; pub const isnan = classifyOp("math.isnan").call; pub const isnormal = classifyOp("math.isnormal").call; - -test { - std.testing.refAllDecls(@This()); -} diff --git a/mlir/dialects/math/test.zig b/mlir/dialects/math/test.zig new file mode 100644 index 00000000..fe2b7bc4 --- /dev/null +++ b/mlir/dialects/math/test.zig @@ -0,0 +1,73 @@ +const std = @import("std"); + +const math = @import("mlir/dialects/math"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "math.absf", + "math.absi", + "math.acos", + "math.acosh", + "math.asin", + "math.asinh", + "math.atan", + "math.atan2", + "math.atanh", + "math.cbrt", + "math.ceil", + "math.clampf", + "math.copysign", + "math.cos", + "math.cosh", + "math.ctlz", + "math.ctpop", + "math.cttz", + "math.erf", + "math.erfc", + "math.exp", + "math.exp2", + "math.expm1", + "math.floor", + "math.fma", + "math.fpowi", + "math.ipowi", + "math.isfinite", + "math.isinf", + "math.isnan", + "math.isnormal", + "math.log", + "math.log10", + "math.log1p", + "math.log2", + "math.powf", + "math.round", + "math.roundeven", + "math.rsqrt", + "math.sin", + "math.sincos", + "math.sinh", + "math.sqrt", + "math.tan", + "math.tanh", + "math.trunc", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(math); + inline for (comptime std.meta.declarations(math)) |decl| { + const T = @field(math, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + math.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/memref/BUILD.bazel b/mlir/dialects/memref/BUILD.bazel new file mode 100644 index 00000000..be1a16bb --- /dev/null +++ b/mlir/dialects/memref/BUILD.bazel @@ -0,0 +1,23 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "memref", + import_name = "mlir/dialects/memref", + main = "memref.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "//stdx", + "@llvm-project//mlir:CAPIMemRef", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":memref", + "//mlir", + ], +) diff --git a/mlir/dialects/memref.zig b/mlir/dialects/memref/memref.zig similarity index 97% rename from mlir/dialects/memref.zig rename to mlir/dialects/memref/memref.zig index 338d1732..d0b337b9 100644 --- a/mlir/dialects/memref.zig +++ b/mlir/dialects/memref/memref.zig @@ -1,9 +1,12 @@ const std = @import("std"); -const c = @import("c"); const mlir = @import("mlir"); const stdx = @import("stdx"); +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("memref"); +} + /// Sentinel matching `mlir::ShapedType::kDynamic` (== INT64_MIN). /// Use this in static-size/offset/stride arrays to mark a slot as dynamic. pub const kDynamic: i64 = std.math.minInt(i64); @@ -16,6 +19,11 @@ pub const AllocOpts = struct { alignment: ?i64 = null, }; +pub const ReallocOpts = struct { + alignment: ?i64 = null, + dynamic_size: ?*const mlir.Value = null, +}; + /// memref.alloc — heap allocation. `dynamic_sizes` provides one Index value per /// dynamic dim of `result_type` (left-to-right). `symbol_operands` binds to /// the symbols of the layout map (rare; pass &.{} when none). @@ -86,18 +94,17 @@ pub fn dealloc( } /// memref.realloc — grow/shrink an existing allocation. `dynamic_size` is -/// optional; pass null to keep the size encoded in `result_type`. +/// optional; omit `opts.dynamic_size` to keep the size encoded in `result_type`. pub fn realloc( ctx: *mlir.Context, src: *const mlir.Value, - dynamic_size: ?*const mlir.Value, result_type: *const mlir.Type, - opts: AllocOpts, + opts: ReallocOpts, location: *const mlir.Location, ) *mlir.Operation { var operands_buf: stdx.BoundedArray(*const mlir.Value, 2) = .empty; operands_buf.appendAssumeCapacity(src); - if (dynamic_size) |s| operands_buf.appendAssumeCapacity(s); + if (opts.dynamic_size) |s| operands_buf.appendAssumeCapacity(s); var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; if (opts.alignment) |a| { @@ -184,19 +191,20 @@ pub fn store( pub const AtomicRMWKind = enum(i64) { addf = 0, addi = 1, - assign = 2, - maximumf = 3, - maxs = 4, - maxu = 5, - minimumf = 6, - mins = 7, - minu = 8, - mulf = 9, - muli = 10, - ori = 11, - andi = 12, - maxnumf = 13, - minnumf = 14, + andi = 2, + assign = 3, + maximumf = 4, + maxnumf = 5, + maxs = 6, + maxu = 7, + minimumf = 8, + minnumf = 9, + mins = 10, + minu = 11, + mulf = 12, + muli = 13, + ori = 14, + xori = 15, }; /// memref.atomic_rmw — atomic read-modify-write. @@ -658,7 +666,3 @@ pub fn get_global( .location = location, }); } - -test { - std.testing.refAllDecls(@This()); -} diff --git a/mlir/dialects/memref/test.zig b/mlir/dialects/memref/test.zig new file mode 100644 index 00000000..cddb74c5 --- /dev/null +++ b/mlir/dialects/memref/test.zig @@ -0,0 +1,77 @@ +const std = @import("std"); + +const memref = @import("mlir/dialects/memref"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "memref.alloc", + "memref.alloca", + "memref.assume_alignment", + "memref.atomic_rmw", + "memref.cast", + "memref.collapse_shape", + "memref.copy", + "memref.dealloc", + "memref.dim", + "memref.expand_shape", + "memref.extract_aligned_pointer_as_index", + "memref.get_global", + "memref.global", + "memref.load", + "memref.memory_space_cast", + "memref.prefetch", + "memref.rank", + "memref.realloc", + "memref.reinterpret_cast", + "memref.reshape", + "memref.store", + "memref.subview", + "memref.transpose", + "memref.view", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(memref); + inline for (comptime std.meta.declarations(memref)) |decl| { + const T = @field(memref, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + memref.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "atomic_rmw kinds print as their IR spelling" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + memref.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + const loc = mlir.Location.unknown(ctx); + const block = mlir.Block.init(&.{ .float(ctx, .f32), try mlir.Type.parse(ctx, "memref<4xf32>"), .int(ctx, .i32), try mlir.Type.parse(ctx, "memref<4xi32>"), .index(ctx) }, &.{ loc, loc, loc, loc, loc }); + defer block.deinit(); + inline for (comptime std.meta.tags(memref.AtomicRMWKind)) |kind| { + const float = switch (kind) { + .addf, .assign, .maximumf, .maxnumf, .minimumf, .minnumf, .mulf => true, + else => false, + }; + const value = block.argument(if (float) 0 else 2); + const ref = block.argument(if (float) 1 else 3); + const op = memref.atomic_rmw(ctx, kind, value, ref, &.{block.argument(4)}, loc); + defer op.deinit(); + var text: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer text.deinit(); + try text.writer.print("{f}", .{op}); + try std.testing.expect(std.mem.indexOf(u8, text.written(), "memref.atomic_rmw " ++ @tagName(kind) ++ " ") != null); + } +} diff --git a/mlir/dialects/mosaic_tpu/BUILD.bazel b/mlir/dialects/mosaic_tpu/BUILD.bazel index 37933e22..88e67265 100644 --- a/mlir/dialects/mosaic_tpu/BUILD.bazel +++ b/mlir/dialects/mosaic_tpu/BUILD.bazel @@ -3,29 +3,26 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") # The types and attributes Mosaic's C API does not expose. cc_library( - name = "mosaic_tpu_capi", - srcs = ["mosaic_tpu_capi.cc"], - hdrs = ["mosaic_tpu_capi.h"], - strip_include_prefix = ".", + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], tags = ["manual"], - visibility = ["//visibility:private"], deps = [ "@llvm-project//llvm:Support", "@llvm-project//mlir:CAPIIR", "@xla//xla/mosaic:tpu_dialect", ], - alwayslink = True, ) zig_library( name = "mosaic_tpu", import_name = "mlir/dialects/mosaic_tpu", main = "mosaic_tpu.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ - ":mosaic_tpu_capi", + ":capi", "//mlir", - "//mlir:c", "//stdx", "@xla//xla/mosaic:tpu_dialect_capi", ], @@ -33,5 +30,9 @@ zig_library( zig_test( name = "test", - deps = [":mosaic_tpu"], + main = "test.zig", + deps = [ + ":mosaic_tpu", + "//mlir", + ], ) diff --git a/mlir/dialects/mosaic_tpu/mosaic_tpu_capi.cc b/mlir/dialects/mosaic_tpu/capi.cc similarity index 56% rename from mlir/dialects/mosaic_tpu/mosaic_tpu_capi.cc rename to mlir/dialects/mosaic_tpu/capi.cc index 5b9e7e20..22662082 100644 --- a/mlir/dialects/mosaic_tpu/mosaic_tpu_capi.cc +++ b/mlir/dialects/mosaic_tpu/capi.cc @@ -1,53 +1,60 @@ -#include "mosaic_tpu_capi.h" +//===- capi.cc - C API for the Mosaic tpu dialect -------------------------===// + +#include "capi.h" + +#include "xla/mosaic/dialect/tpu/tpu_dialect.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/IR/Diagnostics.h" +#include "mlir/IR/Location.h" +#include "llvm/ADT/ArrayRef.h" #include #include -#include "xla/mosaic/dialect/tpu/tpu_dialect.h" -#include "llvm/ADT/ArrayRef.h" -#include "mlir/CAPI/IR.h" -#include "mlir/IR/Diagnostics.h" -#include "mlir/IR/Location.h" - -namespace { - -mlir::MLIRContext *loadTPU(MlirContext context) { +static mlir::MLIRContext *loadTPU(MlirContext context) { mlir::MLIRContext *ctx = unwrap(context); - if (ctx) ctx->getOrLoadDialect(); + if (ctx) + ctx->getOrLoadDialect(); return ctx; } template -MlirAttribute enumAttrGet(MlirContext context, std::optional value) { +static MlirAttribute enumAttrGet(MlirContext context, + std::optional value) { mlir::MLIRContext *ctx = loadTPU(context); - if (!ctx || !value) return {nullptr}; + if (!ctx || !value) + return {nullptr}; return wrap(AttrT::get(ctx, *value)); } -template -uint32_t enumAttrGetValue(MlirAttribute attr) { +template static uint32_t enumAttrGetValue(MlirAttribute attr) { return static_cast(llvm::cast(unwrap(attr)).getValue()); } -llvm::ArrayRef arrayRef(intptr_t size, const int64_t *data) { +static llvm::ArrayRef arrayRef(intptr_t size, const int64_t *data) { return llvm::ArrayRef(data, static_cast(size)); } -} // namespace - -bool mlirTypeIsATpuSemaphore(MlirType type) { return llvm::isa(unwrap(type)); } +bool mlirTypeIsATpuSemaphore(MlirType type) { + return llvm::isa(unwrap(type)); +} MlirType mlirTpuSemaphoreTypeGet(MlirContext context) { mlir::MLIRContext *ctx = loadTPU(context); - if (!ctx) return {nullptr}; + if (!ctx) + return {nullptr}; return wrap(mlir::tpu::SemaphoreType::get(ctx)); } -bool mlirTypeIsATpuDMASemaphore(MlirType type) { return llvm::isa(unwrap(type)); } +bool mlirTypeIsATpuDMASemaphore(MlirType type) { + return llvm::isa(unwrap(type)); +} MlirType mlirTpuDMASemaphoreTypeGet(MlirContext context) { mlir::MLIRContext *ctx = loadTPU(context); - if (!ctx) return {nullptr}; + if (!ctx) + return {nullptr}; return wrap(mlir::tpu::DMASemaphoreType::get(ctx)); } @@ -56,7 +63,8 @@ bool mlirAttributeIsATpuReductionKind(MlirAttribute attr) { } MlirAttribute mlirTpuReductionKindAttrGet(MlirContext context, uint32_t value) { - return enumAttrGet(context, mlir::tpu::symbolizeReductionKind(value)); + return enumAttrGet( + context, mlir::tpu::symbolizeReductionKind(value)); } uint32_t mlirTpuReductionKindAttrGetValue(MlirAttribute attr) { @@ -67,8 +75,10 @@ bool mlirAttributeIsATpuContractPrecision(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirTpuContractPrecisionAttrGet(MlirContext context, uint32_t value) { - return enumAttrGet(context, mlir::tpu::symbolizeContractPrecision(value)); +MlirAttribute mlirTpuContractPrecisionAttrGet(MlirContext context, + uint32_t value) { + return enumAttrGet( + context, mlir::tpu::symbolizeContractPrecision(value)); } uint32_t mlirTpuContractPrecisionAttrGetValue(MlirAttribute attr) { @@ -80,7 +90,8 @@ bool mlirAttributeIsATpuRoundingMode(MlirAttribute attr) { } MlirAttribute mlirTpuRoundingModeAttrGet(MlirContext context, uint32_t value) { - return enumAttrGet(context, mlir::tpu::symbolizeRoundingMode(value)); + return enumAttrGet( + context, mlir::tpu::symbolizeRoundingMode(value)); } uint32_t mlirTpuRoundingModeAttrGetValue(MlirAttribute attr) { @@ -92,17 +103,22 @@ bool mlirAttributeIsATpuCoreType(MlirAttribute attr) { } MlirAttribute mlirTpuCoreTypeAttrGet(MlirContext context, uint32_t value) { - return enumAttrGet(context, mlir::tpu::symbolizeCoreType(value)); + return enumAttrGet( + context, mlir::tpu::symbolizeCoreType(value)); } -uint32_t mlirTpuCoreTypeAttrGetValue(MlirAttribute attr) { return enumAttrGetValue(attr); } +uint32_t mlirTpuCoreTypeAttrGetValue(MlirAttribute attr) { + return enumAttrGetValue(attr); +} bool mlirAttributeIsATpuDimensionSemantics(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirTpuDimensionSemanticsAttrGet(MlirContext context, uint32_t value) { - return enumAttrGet(context, mlir::tpu::symbolizeDimensionSemantics(value)); +MlirAttribute mlirTpuDimensionSemanticsAttrGet(MlirContext context, + uint32_t value) { + return enumAttrGet( + context, mlir::tpu::symbolizeDimensionSemantics(value)); } uint32_t mlirTpuDimensionSemanticsAttrGetValue(MlirAttribute attr) { @@ -114,7 +130,8 @@ bool mlirAttributeIsATpuPipelineMode(MlirAttribute attr) { } MlirAttribute mlirTpuPipelineModeAttrGet(MlirContext context, uint32_t value) { - return enumAttrGet(context, mlir::tpu::symbolizePipelineMode(value)); + return enumAttrGet( + context, mlir::tpu::symbolizePipelineMode(value)); } uint32_t mlirTpuPipelineModeAttrGetValue(MlirAttribute attr) { @@ -126,7 +143,8 @@ bool mlirAttributeIsATpuRevisitMode(MlirAttribute attr) { } MlirAttribute mlirTpuRevisitModeAttrGet(MlirContext context, uint32_t value) { - return enumAttrGet(context, mlir::tpu::symbolizeRevisitMode(value)); + return enumAttrGet( + context, mlir::tpu::symbolizeRevisitMode(value)); } uint32_t mlirTpuRevisitModeAttrGetValue(MlirAttribute attr) { @@ -137,18 +155,24 @@ bool mlirAttributeIsATpuMemorySpace(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirTpuMemorySpaceAttrGet(MlirContext context, uint32_t value, int64_t coreType) { +MlirAttribute mlirTpuMemorySpaceAttrGet(MlirContext context, uint32_t value, + int64_t coreType) { mlir::MLIRContext *ctx = loadTPU(context); - std::optional memorySpace = mlir::tpu::symbolizeMemorySpace(value); - if (!ctx || !memorySpace) return {nullptr}; + std::optional memorySpace = + mlir::tpu::symbolizeMemorySpace(value); + if (!ctx || !memorySpace) + return {nullptr}; std::optional core; if (coreType >= 0) { - if (coreType > UINT32_MAX) return {nullptr}; + if (coreType > UINT32_MAX) + return {nullptr}; core = mlir::tpu::symbolizeCoreType(static_cast(coreType)); - if (!core) return {nullptr}; + if (!core) + return {nullptr}; } return wrap(mlir::tpu::MemorySpaceAttr::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, *memorySpace, core)); + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + *memorySpace, core)); } uint32_t mlirTpuMemorySpaceAttrGetValue(MlirAttribute attr) { @@ -156,7 +180,8 @@ uint32_t mlirTpuMemorySpaceAttrGetValue(MlirAttribute attr) { } int64_t mlirTpuMemorySpaceAttrGetCoreType(MlirAttribute attr) { - std::optional core = llvm::cast(unwrap(attr)).getCoreType(); + std::optional core = + llvm::cast(unwrap(attr)).getCoreType(); return core ? static_cast(*core) : -1; } @@ -164,54 +189,77 @@ bool mlirAttributeIsATpuElementWindow(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirTpuElementWindowAttrGet(MlirContext context, intptr_t numPadLow, const int64_t *padLow, - intptr_t numPadHigh, const int64_t *padHigh) { +MlirAttribute mlirTpuElementWindowAttrGet(MlirContext context, + intptr_t numPadLow, + const int64_t *padLow, + intptr_t numPadHigh, + const int64_t *padHigh) { mlir::MLIRContext *ctx = loadTPU(context); - if (!ctx) return {nullptr}; - return wrap(mlir::tpu::ElementWindowAttr::get(ctx, arrayRef(numPadLow, padLow), arrayRef(numPadHigh, padHigh))); + if (!ctx) + return {nullptr}; + return wrap(mlir::tpu::ElementWindowAttr::get( + ctx, arrayRef(numPadLow, padLow), arrayRef(numPadHigh, padHigh))); } intptr_t mlirTpuElementWindowAttrGetNumPadLow(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getPadLow().size(); + return llvm::cast(unwrap(attr)) + .getPadLow() + .size(); } int64_t mlirTpuElementWindowAttrGetPadLow(MlirAttribute attr, intptr_t pos) { - return llvm::cast(unwrap(attr)).getPadLow()[pos]; + return llvm::cast(unwrap(attr)) + .getPadLow()[pos]; } intptr_t mlirTpuElementWindowAttrGetNumPadHigh(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getPadHigh().size(); + return llvm::cast(unwrap(attr)) + .getPadHigh() + .size(); } int64_t mlirTpuElementWindowAttrGetPadHigh(MlirAttribute attr, intptr_t pos) { - return llvm::cast(unwrap(attr)).getPadHigh()[pos]; + return llvm::cast(unwrap(attr)) + .getPadHigh()[pos]; } bool mlirAttributeIsATpuDotDimensionNumbers(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); + return llvm::isa_and_nonnull( + unwrap(attr)); } MlirAttribute mlirTpuDotDimensionNumbersAttrGet( - MlirContext context, intptr_t numLhsContractingDims, const int64_t *lhsContractingDims, - intptr_t numRhsContractingDims, const int64_t *rhsContractingDims, intptr_t numLhsNonContractingDims, - const int64_t *lhsNonContractingDims, intptr_t numRhsNonContractingDims, const int64_t *rhsNonContractingDims, - intptr_t numOutputDimOrder, const int64_t *outputDimOrder, intptr_t numLhsBatchDims, const int64_t *lhsBatchDims, - intptr_t numRhsBatchDims, const int64_t *rhsBatchDims) { + MlirContext context, intptr_t numLhsContractingDims, + const int64_t *lhsContractingDims, intptr_t numRhsContractingDims, + const int64_t *rhsContractingDims, intptr_t numLhsNonContractingDims, + const int64_t *lhsNonContractingDims, intptr_t numRhsNonContractingDims, + const int64_t *rhsNonContractingDims, intptr_t numOutputDimOrder, + const int64_t *outputDimOrder, intptr_t numLhsBatchDims, + const int64_t *lhsBatchDims, intptr_t numRhsBatchDims, + const int64_t *rhsBatchDims) { mlir::MLIRContext *ctx = loadTPU(context); - if (!ctx) return {nullptr}; + if (!ctx) + return {nullptr}; return wrap(mlir::tpu::DotDimensionNumbersAttr::get( ctx, arrayRef(numLhsContractingDims, lhsContractingDims), - arrayRef(numRhsContractingDims, rhsContractingDims), arrayRef(numLhsNonContractingDims, lhsNonContractingDims), - arrayRef(numRhsNonContractingDims, rhsNonContractingDims), arrayRef(numOutputDimOrder, outputDimOrder), - arrayRef(numLhsBatchDims, lhsBatchDims), arrayRef(numRhsBatchDims, rhsBatchDims))); + arrayRef(numRhsContractingDims, rhsContractingDims), + arrayRef(numLhsNonContractingDims, lhsNonContractingDims), + arrayRef(numRhsNonContractingDims, rhsNonContractingDims), + arrayRef(numOutputDimOrder, outputDimOrder), + arrayRef(numLhsBatchDims, lhsBatchDims), + arrayRef(numRhsBatchDims, rhsBatchDims))); } -#define ZML_TPU_DOT_DIMS_GETTERS(Name) \ - intptr_t mlirTpuDotDimensionNumbersAttrGetNum##Name(MlirAttribute attr) { \ - return llvm::cast(unwrap(attr)).get##Name().size(); \ - } \ - int64_t mlirTpuDotDimensionNumbersAttrGet##Name(MlirAttribute attr, intptr_t pos) { \ - return llvm::cast(unwrap(attr)).get##Name()[pos]; \ +#define ZML_TPU_DOT_DIMS_GETTERS(Name) \ + intptr_t mlirTpuDotDimensionNumbersAttrGetNum##Name(MlirAttribute attr) { \ + return llvm::cast(unwrap(attr)) \ + .get##Name() \ + .size(); \ + } \ + int64_t mlirTpuDotDimensionNumbersAttrGet##Name(MlirAttribute attr, \ + intptr_t pos) { \ + return llvm::cast(unwrap(attr)) \ + .get##Name()[pos]; \ } ZML_TPU_DOT_DIMS_GETTERS(LhsContractingDims) diff --git a/mlir/dialects/mosaic_tpu/capi.h b/mlir/dialects/mosaic_tpu/capi.h new file mode 100644 index 00000000..2e1cc0bb --- /dev/null +++ b/mlir/dialects/mosaic_tpu/capi.h @@ -0,0 +1,145 @@ +//===- capi.h - C API for the Mosaic tpu dialect ------------------*- C -*-===// +// +// C entry points for the `tpu` dialect that Mosaic's C API +// (`tpu_dialect.h`) leaves out: the semaphore types and the attributes +// kernels build. + +#ifndef MLIR_DIALECTS_MOSAIC_TPU_CAPI_H +#define MLIR_DIALECTS_MOSAIC_TPU_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// !tpu.semaphore +MLIR_CAPI_EXPORTED bool mlirTypeIsATpuSemaphore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirTpuSemaphoreTypeGet(MlirContext context); + +// !tpu.dma_semaphore +MLIR_CAPI_EXPORTED bool mlirTypeIsATpuDMASemaphore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirTpuDMASemaphoreTypeGet(MlirContext context); + +// Enum attributes take and return the C++ enum's integer value (tpu_ops.td); +// `Get` returns a null attribute for a value outside the enum. + +// #tpu.reduction_kind<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuReductionKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTpuReductionKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirTpuReductionKindAttrGetValue(MlirAttribute attr); + +// #tpu.contract_precision<...> +MLIR_CAPI_EXPORTED bool +mlirAttributeIsATpuContractPrecision(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTpuContractPrecisionAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirTpuContractPrecisionAttrGetValue(MlirAttribute attr); + +// #tpu.rounding_mode<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuRoundingMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuRoundingModeAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirTpuRoundingModeAttrGetValue(MlirAttribute attr); + +// #tpu.core_type<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuCoreType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuCoreTypeAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirTpuCoreTypeAttrGetValue(MlirAttribute attr); + +// #tpu.dimension_semantics<...> +MLIR_CAPI_EXPORTED bool +mlirAttributeIsATpuDimensionSemantics(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTpuDimensionSemanticsAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirTpuDimensionSemanticsAttrGetValue(MlirAttribute attr); + +// #tpu.pipeline_mode<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuPipelineMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuPipelineModeAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirTpuPipelineModeAttrGetValue(MlirAttribute attr); + +// #tpu.revisit_mode<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuRevisitMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuRevisitModeAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirTpuRevisitModeAttrGetValue(MlirAttribute attr); + +// #tpu.memory_space: a negative `coreType` leaves the +// optional core type out, and `GetCoreType` returns -1 when it is absent. +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuMemorySpace(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuMemorySpaceAttrGet(MlirContext context, + uint32_t value, + int64_t coreType); +MLIR_CAPI_EXPORTED uint32_t mlirTpuMemorySpaceAttrGetValue(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t +mlirTpuMemorySpaceAttrGetCoreType(MlirAttribute attr); + +// #tpu.element_window<[pad_low], [pad_high]> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuElementWindow(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuElementWindowAttrGet( + MlirContext context, intptr_t numPadLow, const int64_t *padLow, + intptr_t numPadHigh, const int64_t *padHigh); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuElementWindowAttrGetNumPadLow(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirTpuElementWindowAttrGetPadLow(MlirAttribute attr, + intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuElementWindowAttrGetNumPadHigh(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t +mlirTpuElementWindowAttrGetPadHigh(MlirAttribute attr, intptr_t pos); + +// #tpu.dot_dimension_numbers<...>: the dimension lists in ODS order. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsATpuDotDimensionNumbers(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuDotDimensionNumbersAttrGet( + MlirContext context, intptr_t numLhsContractingDims, + const int64_t *lhsContractingDims, intptr_t numRhsContractingDims, + const int64_t *rhsContractingDims, intptr_t numLhsNonContractingDims, + const int64_t *lhsNonContractingDims, intptr_t numRhsNonContractingDims, + const int64_t *rhsNonContractingDims, intptr_t numOutputDimOrder, + const int64_t *outputDimOrder, intptr_t numLhsBatchDims, + const int64_t *lhsBatchDims, intptr_t numRhsBatchDims, + const int64_t *rhsBatchDims); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuDotDimensionNumbersAttrGetNumLhsContractingDims(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetLhsContractingDims( + MlirAttribute attr, intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuDotDimensionNumbersAttrGetNumRhsContractingDims(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetRhsContractingDims( + MlirAttribute attr, intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuDotDimensionNumbersAttrGetNumLhsNonContractingDims(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t +mlirTpuDotDimensionNumbersAttrGetLhsNonContractingDims(MlirAttribute attr, + intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuDotDimensionNumbersAttrGetNumRhsNonContractingDims(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t +mlirTpuDotDimensionNumbersAttrGetRhsNonContractingDims(MlirAttribute attr, + intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuDotDimensionNumbersAttrGetNumOutputDimOrder(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetOutputDimOrder( + MlirAttribute attr, intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuDotDimensionNumbersAttrGetNumLhsBatchDims(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t +mlirTpuDotDimensionNumbersAttrGetLhsBatchDims(MlirAttribute attr, intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuDotDimensionNumbersAttrGetNumRhsBatchDims(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t +mlirTpuDotDimensionNumbersAttrGetRhsBatchDims(MlirAttribute attr, intptr_t pos); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_MOSAIC_TPU_CAPI_H diff --git a/mlir/dialects/mosaic_tpu/mosaic_tpu.zig b/mlir/dialects/mosaic_tpu/mosaic_tpu.zig index b70e3c3e..51bbd36e 100644 --- a/mlir/dialects/mosaic_tpu/mosaic_tpu.zig +++ b/mlir/dialects/mosaic_tpu/mosaic_tpu.zig @@ -9,6 +9,10 @@ const stdx = @import("stdx"); /// the custom-call JSON `serialization_format`, which remains 1. pub const SERDE_VERSION: i32 = 18; +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("tpu"); +} + /// Register the `mosaic-serde` pass so `PassManager.parse` can pick it up by /// name. Idempotent — safe to call repeatedly. Required before running the /// `mosaic-serde{serialize=true}` pipeline that produces the bytecode embedded @@ -28,11 +32,9 @@ pub const SemaphoreType = opaque { pub const isAFn = c.mlirTypeIsATpuSemaphore; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "semaphore"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -57,11 +59,9 @@ pub const DMASemaphoreType = opaque { pub const isAFn = c.mlirTypeIsATpuDMASemaphore; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; pub const mnemonic = "dma_semaphore"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.type_().format(writer); - } pub fn type_(self: *const Self) *const mlir.Type { return @ptrCast(self); } @@ -96,6 +96,10 @@ pub const Float8EXMYType = opaque { pub fn underlyingType(self: *const Float8EXMYType) *const mlir.Type { return @ptrCast(c.mlirTpuFloat8EXMYTypeGetUnderlyingType(self.ptr()).ptr); } + + pub fn type_(self: *const Float8EXMYType) *const mlir.Type { + return @ptrCast(self); + } }; pub fn float8ExmyType(ctx: *mlir.Context, underlying: *const mlir.Type) *const mlir.Type { @@ -146,11 +150,9 @@ pub const ReductionKindAttr = opaque { pub const isAFn = c.mlirAttributeIsATpuReductionKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "reduction_kind"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -161,7 +163,7 @@ pub const ReductionKindAttr = opaque { const result = c.mlirTpuReductionKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) ReductionKind { + pub fn value(self: *const Self) ReductionKind { return @fromBackingInt(@intCast(c.mlirTpuReductionKindAttrGetValue(self.ptr()))); } }; @@ -185,11 +187,9 @@ pub const ContractPrecisionAttr = opaque { pub const isAFn = c.mlirAttributeIsATpuContractPrecision; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "contract_precision"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -200,7 +200,7 @@ pub const ContractPrecisionAttr = opaque { const result = c.mlirTpuContractPrecisionAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) ContractPrecision { + pub fn value(self: *const Self) ContractPrecision { return @fromBackingInt(@intCast(c.mlirTpuContractPrecisionAttrGetValue(self.ptr()))); } }; @@ -223,11 +223,9 @@ pub const RoundingModeAttr = opaque { pub const isAFn = c.mlirAttributeIsATpuRoundingMode; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "rounding_mode"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -238,7 +236,7 @@ pub const RoundingModeAttr = opaque { const result = c.mlirTpuRoundingModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) RoundingMode { + pub fn value(self: *const Self) RoundingMode { return @fromBackingInt(@intCast(c.mlirTpuRoundingModeAttrGetValue(self.ptr()))); } }; @@ -262,11 +260,9 @@ pub const CoreTypeAttr = opaque { pub const isAFn = c.mlirAttributeIsATpuCoreType; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "core_type"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -277,7 +273,7 @@ pub const CoreTypeAttr = opaque { const result = c.mlirTpuCoreTypeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) CoreType { + pub fn value(self: *const Self) CoreType { return @fromBackingInt(@intCast(c.mlirTpuCoreTypeAttrGetValue(self.ptr()))); } }; @@ -302,11 +298,9 @@ pub const DimensionSemanticsAttr = opaque { pub const isAFn = c.mlirAttributeIsATpuDimensionSemantics; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "dimension_semantics"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -317,7 +311,7 @@ pub const DimensionSemanticsAttr = opaque { const result = c.mlirTpuDimensionSemanticsAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) DimensionSemantics { + pub fn value(self: *const Self) DimensionSemantics { return @fromBackingInt(@intCast(c.mlirTpuDimensionSemanticsAttrGetValue(self.ptr()))); } }; @@ -340,11 +334,9 @@ pub const PipelineModeAttr = opaque { pub const isAFn = c.mlirAttributeIsATpuPipelineMode; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "pipeline_mode"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -355,7 +347,7 @@ pub const PipelineModeAttr = opaque { const result = c.mlirTpuPipelineModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) PipelineMode { + pub fn value(self: *const Self) PipelineMode { return @fromBackingInt(@intCast(c.mlirTpuPipelineModeAttrGetValue(self.ptr()))); } }; @@ -378,11 +370,9 @@ pub const RevisitModeAttr = opaque { pub const isAFn = c.mlirAttributeIsATpuRevisitMode; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "revisit_mode"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -393,7 +383,7 @@ pub const RevisitModeAttr = opaque { const result = c.mlirTpuRevisitModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) RevisitMode { + pub fn value(self: *const Self) RevisitMode { return @fromBackingInt(@intCast(c.mlirTpuRevisitModeAttrGetValue(self.ptr()))); } }; @@ -422,11 +412,9 @@ pub const MemorySpaceAttr = opaque { pub const isAFn = c.mlirAttributeIsATpuMemorySpace; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "memory_space"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -439,10 +427,10 @@ pub const MemorySpaceAttr = opaque { const result = c.mlirTpuMemorySpaceAttrGet(ctx.ptr(), @backingInt(args.value), core_type); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) MemorySpace { + pub fn value(self: *const Self) MemorySpace { return @fromBackingInt(@intCast(c.mlirTpuMemorySpaceAttrGetValue(self.ptr()))); } - pub fn getCoreType(self: *const Self) ?CoreType { + pub fn coreType(self: *const Self) ?CoreType { const core_type = c.mlirTpuMemorySpaceAttrGetCoreType(self.ptr()); return if (core_type < 0) null else @fromBackingInt(@intCast(core_type)); } @@ -455,11 +443,9 @@ pub const ElementWindowAttr = opaque { pub const isAFn = c.mlirAttributeIsATpuElementWindow; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "element_window"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -477,22 +463,22 @@ pub const ElementWindowAttr = opaque { ); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getNumPadLow(self: *const Self) usize { + pub fn numPadLow(self: *const Self) usize { return @intCast(c.mlirTpuElementWindowAttrGetNumPadLow(self.ptr())); } - pub fn getPadLow(self: *const Self, pos: usize) i64 { + pub fn padLow(self: *const Self, pos: usize) i64 { return c.mlirTpuElementWindowAttrGetPadLow(self.ptr(), @intCast(pos)); } - pub fn getNumPadHigh(self: *const Self) usize { + pub fn numPadHigh(self: *const Self) usize { return @intCast(c.mlirTpuElementWindowAttrGetNumPadHigh(self.ptr())); } - pub fn getPadHigh(self: *const Self, pos: usize) i64 { + pub fn padHigh(self: *const Self, pos: usize) i64 { return c.mlirTpuElementWindowAttrGetPadHigh(self.ptr(), @intCast(pos)); } }; /// `#tpu.element_window<[pad_low], [pad_high]>` — per-arg window padding. -pub fn elementWindowAttribute( +pub fn elementWindowAttr( ctx: *mlir.Context, pad_low: []const i64, pad_high: []const i64, @@ -508,11 +494,9 @@ pub const DotDimensionNumbersAttr = opaque { pub const isAFn = c.mlirAttributeIsATpuDotDimensionNumbers; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "dot_dimension_numbers"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -547,46 +531,46 @@ pub const DotDimensionNumbersAttr = opaque { ); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getNumLhsContractingDims(self: *const Self) usize { + pub fn numLhsContractingDims(self: *const Self) usize { return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumLhsContractingDims(self.ptr())); } - pub fn getLhsContractingDims(self: *const Self, pos: usize) i64 { + pub fn lhsContractingDims(self: *const Self, pos: usize) i64 { return c.mlirTpuDotDimensionNumbersAttrGetLhsContractingDims(self.ptr(), @intCast(pos)); } - pub fn getNumRhsContractingDims(self: *const Self) usize { + pub fn numRhsContractingDims(self: *const Self) usize { return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumRhsContractingDims(self.ptr())); } - pub fn getRhsContractingDims(self: *const Self, pos: usize) i64 { + pub fn rhsContractingDims(self: *const Self, pos: usize) i64 { return c.mlirTpuDotDimensionNumbersAttrGetRhsContractingDims(self.ptr(), @intCast(pos)); } - pub fn getNumLhsNonContractingDims(self: *const Self) usize { + pub fn numLhsNonContractingDims(self: *const Self) usize { return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumLhsNonContractingDims(self.ptr())); } - pub fn getLhsNonContractingDims(self: *const Self, pos: usize) i64 { + pub fn lhsNonContractingDims(self: *const Self, pos: usize) i64 { return c.mlirTpuDotDimensionNumbersAttrGetLhsNonContractingDims(self.ptr(), @intCast(pos)); } - pub fn getNumRhsNonContractingDims(self: *const Self) usize { + pub fn numRhsNonContractingDims(self: *const Self) usize { return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumRhsNonContractingDims(self.ptr())); } - pub fn getRhsNonContractingDims(self: *const Self, pos: usize) i64 { + pub fn rhsNonContractingDims(self: *const Self, pos: usize) i64 { return c.mlirTpuDotDimensionNumbersAttrGetRhsNonContractingDims(self.ptr(), @intCast(pos)); } - pub fn getNumOutputDimOrder(self: *const Self) usize { + pub fn numOutputDimOrder(self: *const Self) usize { return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumOutputDimOrder(self.ptr())); } - pub fn getOutputDimOrder(self: *const Self, pos: usize) i64 { + pub fn outputDimOrder(self: *const Self, pos: usize) i64 { return c.mlirTpuDotDimensionNumbersAttrGetOutputDimOrder(self.ptr(), @intCast(pos)); } - pub fn getNumLhsBatchDims(self: *const Self) usize { + pub fn numLhsBatchDims(self: *const Self) usize { return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumLhsBatchDims(self.ptr())); } - pub fn getLhsBatchDims(self: *const Self, pos: usize) i64 { + pub fn lhsBatchDims(self: *const Self, pos: usize) i64 { return c.mlirTpuDotDimensionNumbersAttrGetLhsBatchDims(self.ptr(), @intCast(pos)); } - pub fn getNumRhsBatchDims(self: *const Self) usize { + pub fn numRhsBatchDims(self: *const Self) usize { return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumRhsBatchDims(self.ptr())); } - pub fn getRhsBatchDims(self: *const Self, pos: usize) i64 { + pub fn rhsBatchDims(self: *const Self, pos: usize) i64 { return c.mlirTpuDotDimensionNumbersAttrGetRhsBatchDims(self.ptr(), @intCast(pos)); } }; @@ -633,18 +617,22 @@ pub fn reduce_index( }); } +pub const ScanOpts = struct { + mask: ?*const mlir.Value = null, +}; + /// Scan the last dimension; an optional rank-one mask spans that dimension. pub fn scan( ctx: *mlir.Context, input: *const mlir.Value, kind: ReductionKind, - mask: ?*const mlir.Value, result_type: *const mlir.Type, + opts: ScanOpts, location: *const mlir.Location, ) *mlir.Operation { var operands_buf: stdx.BoundedArray(*const mlir.Value, 2) = .empty; operands_buf.appendAssumeCapacity(input); - if (mask) |m| operands_buf.appendAssumeCapacity(m); + if (opts.mask) |m| operands_buf.appendAssumeCapacity(m); return mlir.Operation.make(ctx, "tpu.scan", .{ .operands = .{ .flat = operands_buf.constSlice() }, @@ -658,6 +646,8 @@ pub fn scan( } pub const SortOpts = struct { + mask: ?*const mlir.Value = null, + descending: bool = false, }; @@ -665,16 +655,15 @@ pub fn sort( ctx: *mlir.Context, keys: *const mlir.Value, values: *const mlir.Value, - mask: ?*const mlir.Value, - opts: SortOpts, output_mask_type: *const mlir.Type, sorted_keys_type: *const mlir.Type, sorted_values_type: *const mlir.Type, + opts: SortOpts, location: *const mlir.Location, ) *mlir.Operation { var operands_buf: stdx.BoundedArray(*const mlir.Value, 3) = .empty; operands_buf.appendSliceAssumeCapacity(&.{ keys, values }); - if (mask) |m| operands_buf.appendAssumeCapacity(m); + if (opts.mask) |m| operands_buf.appendAssumeCapacity(m); return mlir.Operation.make(ctx, "tpu.sort", .{ .operands = .{ .flat = operands_buf.constSlice() }, @@ -700,8 +689,8 @@ pub fn load( ctx: *mlir.Context, base: *const mlir.Value, indices: []const *const mlir.Value, - opts: LoadOpts, result_type: *const mlir.Type, + opts: LoadOpts, location: *const mlir.Location, ) *mlir.Operation { var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; @@ -720,6 +709,8 @@ pub fn load( } pub const StoreOpts = struct { + mask: ?*const mlir.Value = null, + sublane_mask: []const bool, sublane_stride: i32 = 1, add: bool = false, @@ -731,14 +722,13 @@ pub fn store( value: *const mlir.Value, base: *const mlir.Value, indices: []const *const mlir.Value, - mask: ?*const mlir.Value, opts: StoreOpts, location: *const mlir.Location, ) *mlir.Operation { var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; operands_buf.appendSliceAssumeCapacity(&.{ value, base }); operands_buf.appendSliceAssumeCapacity(indices); - const mask_len: i32 = if (mask) |m| blk: { + const mask_len: i32 = if (opts.mask) |m| blk: { operands_buf.appendAssumeCapacity(m); break :blk 1; } else 0; @@ -757,6 +747,8 @@ pub fn store( } pub const VectorLoadOpts = struct { + mask: ?*const mlir.Value = null, + /// Per-dim load stride. Pass `&.{}` for unit-strided. strides: []const i32 = &.{}, }; @@ -766,15 +758,14 @@ pub fn vector_load( ctx: *mlir.Context, base: *const mlir.Value, indices: []const *const mlir.Value, - mask: ?*const mlir.Value, - opts: VectorLoadOpts, result_type: *const mlir.Type, + opts: VectorLoadOpts, location: *const mlir.Location, ) *mlir.Operation { var operands_buf: stdx.BoundedArray(*const mlir.Value, 17) = .empty; operands_buf.appendAssumeCapacity(base); operands_buf.appendSliceAssumeCapacity(indices); - const mask_len: i32 = if (mask) |m| blk: { + const mask_len: i32 = if (opts.mask) |m| blk: { operands_buf.appendAssumeCapacity(m); break :blk 1; } else 0; @@ -792,6 +783,8 @@ pub fn vector_load( } pub const VectorStoreOpts = struct { + mask: ?*const mlir.Value = null, + strides: []const i32 = &.{}, add: bool = false, }; @@ -801,14 +794,13 @@ pub fn vector_store( value: *const mlir.Value, base: *const mlir.Value, indices: []const *const mlir.Value, - mask: ?*const mlir.Value, opts: VectorStoreOpts, location: *const mlir.Location, ) *mlir.Operation { var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; operands_buf.appendSliceAssumeCapacity(&.{ value, base }); operands_buf.appendSliceAssumeCapacity(indices); - const mask_len: i32 = if (mask) |m| blk: { + const mask_len: i32 = if (opts.mask) |m| blk: { operands_buf.appendAssumeCapacity(m); break :blk 1; } else 0; @@ -881,7 +873,7 @@ pub const MatmulOpts = struct { precision: ?ContractPrecision = null, /// Optional `#tpu.dot_dimension_numbers<...>` attribute. When omitted the /// canonicalizer derives one. Build with `DotDimensionNumbersAttr` or - /// `dotDimensionNumbers`. + /// `dotDimensionNumbersAttr`. dimension_numbers: ?*const mlir.Attribute = null, }; @@ -890,8 +882,8 @@ pub fn matmul( lhs: *const mlir.Value, rhs: *const mlir.Value, acc: *const mlir.Value, - opts: MatmulOpts, result_type: *const mlir.Type, + opts: MatmulOpts, location: *const mlir.Location, ) *mlir.Operation { var attrs: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; @@ -915,14 +907,18 @@ pub fn matmul( }); } +pub const IotaOpts = struct { + dimensions: ?[]const i32 = null, +}; + pub fn iota( ctx: *mlir.Context, - dimensions: ?[]const i32, result_type: *const mlir.Type, + opts: IotaOpts, location: *const mlir.Location, ) *mlir.Operation { var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (dimensions) |d| { + if (opts.dimensions) |d| { attrs.appendAssumeCapacity(.named(ctx, "dimensions", .denseArray(ctx, .i32, d))); } return mlir.Operation.make(ctx, "tpu.iota", .{ @@ -999,25 +995,6 @@ pub fn reshape( }); } -pub fn repeat( - ctx: *mlir.Context, - src: *const mlir.Value, - dimension: i32, - times: i32, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.repeat", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "dimension", .int(ctx, .i32, dimension)), - .named(ctx, "times", .int(ctx, .i32, times)), - }, - .location = location, - }); -} - pub fn concatenate( ctx: *mlir.Context, sources: []const *const mlir.Value, @@ -1493,7 +1470,7 @@ pub fn trace_stop(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Oper /// Build a `#tpu.dot_dimension_numbers<...>` attribute. Mirrors /// `tpu_ops.td:TPU_DotDimensionNumbersAttr`. Pass empty slices for the /// fields you don't use (most dot operations leave batch dims empty). -pub fn dotDimensionNumbers( +pub fn dotDimensionNumbersAttr( ctx: *mlir.Context, lhs_contracting: []const i64, rhs_contracting: []const i64, @@ -1563,198 +1540,3 @@ fn denseBoolArrayAttribute(ctx: *mlir.Context, values: []const bool) *const mlir for (values) |b| ints.appendAssumeCapacity(@intFromBool(b)); return .denseArray(ctx, .bool, ints.constSlice()); } - -test { - std.testing.refAllDecls(@This()); -} - -fn testContext() !*mlir.Context { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - registry.registerDialect("tpu"); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - return ctx; -} - -fn expectSegments(op: *const mlir.Operation, expected: []const i32) !void { - const segments = op.attributeByName("operandSegmentSizes").?.isA(mlir.DenseArrayAttribute(.i32)).?; - try std.testing.expectEqual(expected.len, segments.numElements()); - for (expected, 0..) |size, i| try std.testing.expectEqual(size, segments.element(i)); -} - -test "DMA and semaphore wrappers satisfy current operand groups" { - const ctx = try testContext(); - defer ctx.deinit(); - const loc = mlir.Location.unknown(ctx); - const args = mlir.Block.init(&.{}, &.{}); - defer args.deinit(); - const source = args.addArgument(try mlir.Type.parse(ctx, "memref<8x128xbf16, #tpu.memory_space>"), loc); - const target = args.addArgument(try mlir.Type.parse(ctx, "memref<8x128xbf16, #tpu.memory_space>"), loc); - const dma_sem = args.addArgument(try mlir.Type.parse(ctx, "memref>"), loc); - const sem = args.addArgument(try mlir.Type.parse(ctx, "memref>"), loc); - const amount = args.addArgument(.int(ctx, .i32), loc); - - const dma = enqueue_dma(ctx, source, target, dma_sem, .{}, loc); - defer dma.deinit(); - try expectSegments(dma, &.{ 1, 0, 1, 1, 0, 0, 0 }); - try std.testing.expect(dma.verify()); - const signal = sem_signal(ctx, sem, amount, .{}, loc); - defer signal.deinit(); - try expectSegments(signal, &.{ 1, 1, 0, 0, 0 }); - try std.testing.expect(signal.verify()); - - const sc_sem = args.addArgument(try mlir.Type.parse(ctx, "memref>"), loc); - const sc_dma_sem = args.addArgument(try mlir.Type.parse(ctx, "memref>"), loc); - const core = args.addArgument(.int(ctx, .i32), loc); - const subcore = args.addArgument(.int(ctx, .i32), loc); - const remote_signal = sem_signal(ctx, sc_sem, amount, .{ .core_id = core, .subcore_id = subcore }, loc); - defer remote_signal.deinit(); - try expectSegments(remote_signal, &.{ 1, 1, 0, 1, 1 }); - try std.testing.expect(remote_signal.operand(3).eql(subcore)); - const remote_dma = enqueue_dma(ctx, source, target, sc_dma_sem, .{ - .source_semaphore = dma_sem, - .core_id = core, - .subcore_id = subcore, - }, loc); - defer remote_dma.deinit(); - try expectSegments(remote_dma, &.{ 1, 1, 1, 1, 0, 1, 1 }); - try std.testing.expect(remote_dma.operand(5).eql(subcore)); -} - -test "matmul and reinterpret cast satisfy current required properties" { - const ctx = try testContext(); - defer ctx.deinit(); - const loc = mlir.Location.unknown(ctx); - const args = mlir.Block.init(&.{}, &.{}); - defer args.deinit(); - const vector_type = mlir.Type.vector(&.{ 8, 8 }, .float(ctx, .f32)); - const lhs = args.addArgument(vector_type, loc); - const rhs = args.addArgument(vector_type, loc); - const acc = args.addArgument(vector_type, loc); - const product = matmul(ctx, lhs, rhs, acc, .{}, vector_type, loc); - defer product.deinit(); - try std.testing.expect(product.verify()); - try std.testing.expect(!product.attributeByName("transpose_lhs_hint").?.isA(mlir.BoolAttribute).?.value()); - - const ref_type = try mlir.Type.parse(ctx, "memref<8x8xf32, #tpu.memory_space>"); - const ref = args.addArgument(ref_type, loc); - const cast = reinterpret_cast(ctx, ref, ref_type, loc); - defer cast.deinit(); - try expectSegments(cast, &.{ 1, 0, 0, 0 }); - try std.testing.expect(cast.verify()); -} - -test "legacy reduction extrema and scan axis preserve their semantics" { - const ctx = try testContext(); - defer ctx.deinit(); - const loc = mlir.Location.unknown(ctx); - const args = mlir.Block.init(&.{}, &.{}); - defer args.deinit(); - const float_type = mlir.Type.vector(&.{ 2, 8 }, .float(ctx, .f32)); - const int_type = mlir.Type.vector(&.{ 2, 8 }, .int(ctx, .i32)); - const floats = args.addArgument(float_type, loc); - const integers = args.addArgument(int_type, loc); - const mask = args.addArgument(mlir.Type.vector(&.{8}, .int(ctx, .i1)), loc); - const float_min = all_reduce(ctx, floats, 1, .min, float_type, loc); - defer float_min.deinit(); - const int_max = all_reduce(ctx, integers, 1, .max, int_type, loc); - defer int_max.deinit(); - try std.testing.expect(float_min.attributeByName("kind").?.eql(try mlir.Attribute.parse(ctx, "#tpu.reduction_kind"))); - try std.testing.expect(int_max.attributeByName("kind").?.eql(try mlir.Attribute.parse(ctx, "#tpu.reduction_kind"))); - - const int_scan = scan(ctx, integers, .max, mask, int_type, loc); - defer int_scan.deinit(); - try std.testing.expect(int_scan.verify()); - try std.testing.expect(int_scan.attributeByName("kind").?.eql(try mlir.Attribute.parse(ctx, "#tpu.reduction_kind"))); - try std.testing.expectEqual(@as(i64, 1), int_scan.attributeByName("dimension").?.isA(mlir.IntegerAttribute).?.value(i64)); - const float_scan = scan(ctx, floats, .min, null, float_type, loc); - defer float_scan.deinit(); - try std.testing.expect(float_scan.attributeByName("kind").?.eql(try mlir.Attribute.parse(ctx, "#tpu.reduction_kind"))); -} - -fn expectPrints(expected: []const u8, value: anytype) !void { - var buf: [256]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{value}); - try std.testing.expectEqualStrings(expected, w.buffered()); -} - -fn expectEnumAttr(comptime Attr: type, ctx: *mlir.Context) !void { - const Enum = @FieldType(Attr.InitArgs, "value"); - inline for (@typeInfo(Enum).@"enum".field_names) |name| { - const value: Enum = @field(Enum, name); - const attr = try Attr.get(ctx, .{ .value = value }); - try std.testing.expectEqual(value, attr.getValue()); - try std.testing.expect(attr.attribute().isA(Attr) != null); - try std.testing.expect(value.attribute(ctx).eql(attr.attribute())); - try expectPrints("#tpu." ++ Attr.mnemonic ++ "<" ++ name ++ ">", attr); - } -} - -test "tpu types are built through the C API" { - const ctx = try mlir.Context.init(.{}); - defer ctx.deinit(); - - const semaphore = try SemaphoreType.get(ctx, .{}); - try std.testing.expect(semaphoreType(ctx).isA(SemaphoreType) != null); - try std.testing.expect(semaphoreType(ctx).isA(DMASemaphoreType) == null); - try expectPrints("!tpu.semaphore", semaphore); - - const dma_semaphore = try DMASemaphoreType.get(ctx, .{}); - try std.testing.expect(dmaSemaphoreType(ctx).isA(DMASemaphoreType) != null); - try expectPrints("!tpu.dma_semaphore", dma_semaphore); -} - -test "tpu enum attributes are built through the C API" { - const ctx = try mlir.Context.init(.{}); - defer ctx.deinit(); - - try expectEnumAttr(ReductionKindAttr, ctx); - try expectEnumAttr(ContractPrecisionAttr, ctx); - try expectEnumAttr(RoundingModeAttr, ctx); - try expectEnumAttr(CoreTypeAttr, ctx); - try expectEnumAttr(DimensionSemanticsAttr, ctx); - try expectEnumAttr(PipelineModeAttr, ctx); - try expectEnumAttr(RevisitModeAttr, ctx); - try expectEnumAttr(MemorySpaceAttr, ctx); - - const smem_tc = try MemorySpaceAttr.get(ctx, .{ .value = .smem, .coreType = .sc_scalar_subcore }); - try std.testing.expectEqual(MemorySpace.smem, smem_tc.getValue()); - try std.testing.expectEqual(CoreType.sc_scalar_subcore, smem_tc.getCoreType().?); - try expectPrints("#tpu.memory_space", smem_tc); - try std.testing.expectEqual(null, (try MemorySpaceAttr.get(ctx, .{ .value = .hbm })).getCoreType()); - // The dialect verifier's pairing rules: no core type on hbm, no scalar subcore on vmem. - try std.testing.expectError(error.InvalidMlir, MemorySpaceAttr.get(ctx, .{ .value = .hbm, .coreType = .tc })); - try std.testing.expectError(error.InvalidMlir, MemorySpaceAttr.get(ctx, .{ .value = .vmem, .coreType = .sc_scalar_subcore })); -} - -test "tpu array attributes are built through the C API" { - const ctx = try mlir.Context.init(.{}); - defer ctx.deinit(); - - const window = try ElementWindowAttr.get(ctx, .{ .padLow = &.{ 0, 2 }, .padHigh = &.{ 1, 3 } }); - try std.testing.expectEqual(2, window.getNumPadLow()); - try std.testing.expectEqual(2, window.getPadLow(1)); - try std.testing.expectEqual(2, window.getNumPadHigh()); - try std.testing.expectEqual(3, window.getPadHigh(1)); - try expectPrints("#tpu.element_window<[0, 2], [1, 3]>", window); - try std.testing.expect(elementWindowAttribute(ctx, &.{ 0, 2 }, &.{ 1, 3 }).eql(window.attribute())); - - const dims = dotDimensionNumbers(ctx, &.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}); - try expectPrints("#tpu.dot_dimension_numbers<[1], [0], [0], [1], [0, 0, 1, 1], [], []>", dims); - - const batched = try DotDimensionNumbersAttr.get(ctx, .{ - .lhsContractingDims = &.{2}, - .rhsContractingDims = &.{1}, - .lhsNonContractingDims = &.{1}, - .outputDimOrder = &.{ 0, 0, 0, 1 }, - .lhsBatchDims = &.{0}, - .rhsBatchDims = &.{0}, - }); - try std.testing.expectEqual(0, batched.getNumRhsNonContractingDims()); - try std.testing.expectEqual(4, batched.getNumOutputDimOrder()); - try std.testing.expectEqual(1, batched.getOutputDimOrder(3)); - try std.testing.expectEqual(0, batched.getRhsBatchDims(0)); - try expectPrints("#tpu.dot_dimension_numbers<[2], [1], [1], [], [0, 0, 0, 1], [0], [0]>", batched); -} diff --git a/mlir/dialects/mosaic_tpu/mosaic_tpu_capi.h b/mlir/dialects/mosaic_tpu/mosaic_tpu_capi.h deleted file mode 100644 index 816d3aa6..00000000 --- a/mlir/dialects/mosaic_tpu/mosaic_tpu_capi.h +++ /dev/null @@ -1,105 +0,0 @@ -// C entry points for the `tpu` dialect that Mosaic's C API -// (`tpu_dialect.h`) leaves out: the semaphore types and the attributes -// kernels build. -#ifndef ZML_MLIR_DIALECTS_MOSAIC_TPU_CAPI_H -#define ZML_MLIR_DIALECTS_MOSAIC_TPU_CAPI_H - -#include -#include - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// !tpu.semaphore -MLIR_CAPI_EXPORTED bool mlirTypeIsATpuSemaphore(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirTpuSemaphoreTypeGet(MlirContext context); - -// !tpu.dma_semaphore -MLIR_CAPI_EXPORTED bool mlirTypeIsATpuDMASemaphore(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirTpuDMASemaphoreTypeGet(MlirContext context); - -// Enum attributes take and return the C++ enum's integer value (tpu_ops.td); -// `Get` returns a null attribute for a value outside the enum. - -// #tpu.reduction_kind<...> -MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuReductionKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirTpuReductionKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirTpuReductionKindAttrGetValue(MlirAttribute attr); - -// #tpu.contract_precision<...> -MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuContractPrecision(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirTpuContractPrecisionAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirTpuContractPrecisionAttrGetValue(MlirAttribute attr); - -// #tpu.rounding_mode<...> -MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuRoundingMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirTpuRoundingModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirTpuRoundingModeAttrGetValue(MlirAttribute attr); - -// #tpu.core_type<...> -MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuCoreType(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirTpuCoreTypeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirTpuCoreTypeAttrGetValue(MlirAttribute attr); - -// #tpu.dimension_semantics<...> -MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuDimensionSemantics(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirTpuDimensionSemanticsAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirTpuDimensionSemanticsAttrGetValue(MlirAttribute attr); - -// #tpu.pipeline_mode<...> -MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuPipelineMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirTpuPipelineModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirTpuPipelineModeAttrGetValue(MlirAttribute attr); - -// #tpu.revisit_mode<...> -MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuRevisitMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirTpuRevisitModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirTpuRevisitModeAttrGetValue(MlirAttribute attr); - -// #tpu.memory_space: a negative `coreType` leaves the -// optional core type out, and `GetCoreType` returns -1 when it is absent. -MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuMemorySpace(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirTpuMemorySpaceAttrGet(MlirContext context, uint32_t value, int64_t coreType); -MLIR_CAPI_EXPORTED uint32_t mlirTpuMemorySpaceAttrGetValue(MlirAttribute attr); -MLIR_CAPI_EXPORTED int64_t mlirTpuMemorySpaceAttrGetCoreType(MlirAttribute attr); - -// #tpu.element_window<[pad_low], [pad_high]> -MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuElementWindow(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirTpuElementWindowAttrGet(MlirContext context, intptr_t numPadLow, const int64_t *padLow, - intptr_t numPadHigh, const int64_t *padHigh); -MLIR_CAPI_EXPORTED intptr_t mlirTpuElementWindowAttrGetNumPadLow(MlirAttribute attr); -MLIR_CAPI_EXPORTED int64_t mlirTpuElementWindowAttrGetPadLow(MlirAttribute attr, intptr_t pos); -MLIR_CAPI_EXPORTED intptr_t mlirTpuElementWindowAttrGetNumPadHigh(MlirAttribute attr); -MLIR_CAPI_EXPORTED int64_t mlirTpuElementWindowAttrGetPadHigh(MlirAttribute attr, intptr_t pos); - -// #tpu.dot_dimension_numbers<...>: the dimension lists in ODS order. -MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuDotDimensionNumbers(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirTpuDotDimensionNumbersAttrGet( - MlirContext context, intptr_t numLhsContractingDims, const int64_t *lhsContractingDims, - intptr_t numRhsContractingDims, const int64_t *rhsContractingDims, intptr_t numLhsNonContractingDims, - const int64_t *lhsNonContractingDims, intptr_t numRhsNonContractingDims, const int64_t *rhsNonContractingDims, - intptr_t numOutputDimOrder, const int64_t *outputDimOrder, intptr_t numLhsBatchDims, const int64_t *lhsBatchDims, - intptr_t numRhsBatchDims, const int64_t *rhsBatchDims); -MLIR_CAPI_EXPORTED intptr_t mlirTpuDotDimensionNumbersAttrGetNumLhsContractingDims(MlirAttribute attr); -MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetLhsContractingDims(MlirAttribute attr, intptr_t pos); -MLIR_CAPI_EXPORTED intptr_t mlirTpuDotDimensionNumbersAttrGetNumRhsContractingDims(MlirAttribute attr); -MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetRhsContractingDims(MlirAttribute attr, intptr_t pos); -MLIR_CAPI_EXPORTED intptr_t mlirTpuDotDimensionNumbersAttrGetNumLhsNonContractingDims(MlirAttribute attr); -MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetLhsNonContractingDims(MlirAttribute attr, intptr_t pos); -MLIR_CAPI_EXPORTED intptr_t mlirTpuDotDimensionNumbersAttrGetNumRhsNonContractingDims(MlirAttribute attr); -MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetRhsNonContractingDims(MlirAttribute attr, intptr_t pos); -MLIR_CAPI_EXPORTED intptr_t mlirTpuDotDimensionNumbersAttrGetNumOutputDimOrder(MlirAttribute attr); -MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetOutputDimOrder(MlirAttribute attr, intptr_t pos); -MLIR_CAPI_EXPORTED intptr_t mlirTpuDotDimensionNumbersAttrGetNumLhsBatchDims(MlirAttribute attr); -MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetLhsBatchDims(MlirAttribute attr, intptr_t pos); -MLIR_CAPI_EXPORTED intptr_t mlirTpuDotDimensionNumbersAttrGetNumRhsBatchDims(MlirAttribute attr); -MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetRhsBatchDims(MlirAttribute attr, intptr_t pos); - -#ifdef __cplusplus -} -#endif - -#endif // ZML_MLIR_DIALECTS_MOSAIC_TPU_CAPI_H diff --git a/mlir/dialects/mosaic_tpu/test.zig b/mlir/dialects/mosaic_tpu/test.zig new file mode 100644 index 00000000..22853676 --- /dev/null +++ b/mlir/dialects/mosaic_tpu/test.zig @@ -0,0 +1,269 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const mosaic_tpu = @import("mlir/dialects/mosaic_tpu"); + +const operation_names: []const []const u8 = &.{ + "tpu.all_reduce", + "tpu.assume_layout", + "tpu.assume_multiple", + "tpu.barrier", + "tpu.bitcast", + "tpu.bitcast_vreg", + "tpu.broadcast_in_sublanes", + "tpu.concatenate", + "tpu.create_mask", + "tpu.delay", + "tpu.device_id", + "tpu.dynamic_rotate", + "tpu.enqueue_dma", + "tpu.erase_memref_layout", + "tpu.extf", + "tpu.fptosi", + "tpu.fptoui", + "tpu.iota", + "tpu.load", + "tpu.mask_cast", + "tpu.matmul", + "tpu.memref_bitcast", + "tpu.memref_reshape", + "tpu.memref_slice", + "tpu.memref_squeeze", + "tpu.reciprocal", + "tpu.reduce_index", + "tpu.region", + "tpu.reinterpret_cast", + "tpu.reshape", + "tpu.scan", + "tpu.sem_alloc", + "tpu.sem_barrier", + "tpu.sem_read", + "tpu.sem_signal", + "tpu.sem_wait", + "tpu.sitofp", + "tpu.sort", + "tpu.store", + "tpu.strided_load", + "tpu.strided_store", + "tpu.trace", + "tpu.trace_start", + "tpu.trace_stop", + "tpu.transpose", + "tpu.truncf", + "tpu.uitofp", + "tpu.vector_load", + "tpu.vector_store", + "tpu.wait_dma2", + "tpu.yield", +}; + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + mosaic_tpu.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectSegments(op: *const mlir.Operation, expected: []const i32) !void { + const segments = op.attributeByName("operandSegmentSizes").?.isA(mlir.DenseArrayAttribute(.i32)).?; + try std.testing.expectEqual(expected.len, segments.numElements()); + for (expected, 0..) |size, i| try std.testing.expectEqual(size, segments.element(i)); +} + +fn expectPrints(expected: []const u8, value: anytype) !void { + var buf: [256]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(expected, w.buffered()); +} + +fn expectEnumAttr(comptime Attr: type, ctx: *mlir.Context) !void { + const Enum = @FieldType(Attr.InitArgs, "value"); + inline for (@typeInfo(Enum).@"enum".field_names) |name| { + const value: Enum = @field(Enum, name); + const attr = try Attr.get(ctx, .{ .value = value }); + try std.testing.expectEqual(value, attr.value()); + try std.testing.expect(attr.attribute().isA(Attr) != null); + try std.testing.expect(value.attribute(ctx).eql(attr.attribute())); + try expectPrints("#tpu." ++ Attr.mnemonic ++ "<" ++ name ++ ">", attr); + } +} + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(mosaic_tpu); + inline for (comptime std.meta.declarations(mosaic_tpu)) |decl| { + const T = @field(mosaic_tpu, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "DMA and semaphore wrappers satisfy current operand groups" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc = mlir.Location.unknown(ctx); + const args = mlir.Block.init(&.{}, &.{}); + defer args.deinit(); + const source = args.addArgument(try mlir.Type.parse(ctx, "memref<8x128xbf16, #tpu.memory_space>"), loc); + const target = args.addArgument(try mlir.Type.parse(ctx, "memref<8x128xbf16, #tpu.memory_space>"), loc); + const dma_sem = args.addArgument(try mlir.Type.parse(ctx, "memref>"), loc); + const sem = args.addArgument(try mlir.Type.parse(ctx, "memref>"), loc); + const amount = args.addArgument(.int(ctx, .i32), loc); + + const dma = mosaic_tpu.enqueue_dma(ctx, source, target, dma_sem, .{}, loc); + defer dma.deinit(); + try expectSegments(dma, &.{ 1, 0, 1, 1, 0, 0, 0 }); + try std.testing.expect(dma.verify()); + const signal = mosaic_tpu.sem_signal(ctx, sem, amount, .{}, loc); + defer signal.deinit(); + try expectSegments(signal, &.{ 1, 1, 0, 0, 0 }); + try std.testing.expect(signal.verify()); + + const sc_sem = args.addArgument(try mlir.Type.parse(ctx, "memref>"), loc); + const sc_dma_sem = args.addArgument(try mlir.Type.parse(ctx, "memref>"), loc); + const core = args.addArgument(.int(ctx, .i32), loc); + const subcore = args.addArgument(.int(ctx, .i32), loc); + const remote_signal = mosaic_tpu.sem_signal(ctx, sc_sem, amount, .{ .core_id = core, .subcore_id = subcore }, loc); + defer remote_signal.deinit(); + try expectSegments(remote_signal, &.{ 1, 1, 0, 1, 1 }); + try std.testing.expect(remote_signal.operand(3).eql(subcore)); + const remote_dma = mosaic_tpu.enqueue_dma(ctx, source, target, sc_dma_sem, .{ + .source_semaphore = dma_sem, + .core_id = core, + .subcore_id = subcore, + }, loc); + defer remote_dma.deinit(); + try expectSegments(remote_dma, &.{ 1, 1, 1, 1, 0, 1, 1 }); + try std.testing.expect(remote_dma.operand(5).eql(subcore)); +} + +test "matmul and reinterpret cast satisfy current required properties" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc = mlir.Location.unknown(ctx); + const args = mlir.Block.init(&.{}, &.{}); + defer args.deinit(); + const vector_type = mlir.Type.vector(&.{ 8, 8 }, .float(ctx, .f32)); + const lhs = args.addArgument(vector_type, loc); + const rhs = args.addArgument(vector_type, loc); + const acc = args.addArgument(vector_type, loc); + const product = mosaic_tpu.matmul(ctx, lhs, rhs, acc, vector_type, .{}, loc); + defer product.deinit(); + try std.testing.expect(product.verify()); + try std.testing.expect(!product.attributeByName("transpose_lhs_hint").?.isA(mlir.BoolAttribute).?.value()); + + const ref_type = try mlir.Type.parse(ctx, "memref<8x8xf32, #tpu.memory_space>"); + const ref = args.addArgument(ref_type, loc); + const cast = mosaic_tpu.reinterpret_cast(ctx, ref, ref_type, loc); + defer cast.deinit(); + try expectSegments(cast, &.{ 1, 0, 0, 0 }); + try std.testing.expect(cast.verify()); +} + +test "legacy reduction extrema and scan axis preserve their semantics" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc = mlir.Location.unknown(ctx); + const args = mlir.Block.init(&.{}, &.{}); + defer args.deinit(); + const float_type = mlir.Type.vector(&.{ 2, 8 }, .float(ctx, .f32)); + const int_type = mlir.Type.vector(&.{ 2, 8 }, .int(ctx, .i32)); + const floats = args.addArgument(float_type, loc); + const integers = args.addArgument(int_type, loc); + const mask = args.addArgument(mlir.Type.vector(&.{8}, .int(ctx, .i1)), loc); + const float_min = mosaic_tpu.all_reduce(ctx, floats, 1, .min, float_type, loc); + defer float_min.deinit(); + const int_max = mosaic_tpu.all_reduce(ctx, integers, 1, .max, int_type, loc); + defer int_max.deinit(); + try std.testing.expect(float_min.attributeByName("kind").?.eql(try mlir.Attribute.parse(ctx, "#tpu.reduction_kind"))); + try std.testing.expect(int_max.attributeByName("kind").?.eql(try mlir.Attribute.parse(ctx, "#tpu.reduction_kind"))); + + const int_scan = mosaic_tpu.scan(ctx, integers, .max, int_type, .{ .mask = mask }, loc); + defer int_scan.deinit(); + try std.testing.expect(int_scan.verify()); + try std.testing.expect(int_scan.attributeByName("kind").?.eql(try mlir.Attribute.parse(ctx, "#tpu.reduction_kind"))); + try std.testing.expectEqual(@as(i64, 1), int_scan.attributeByName("dimension").?.isA(mlir.IntegerAttribute).?.value(i64)); + const float_scan = mosaic_tpu.scan(ctx, floats, .min, float_type, .{}, loc); + defer float_scan.deinit(); + try std.testing.expect(float_scan.attributeByName("kind").?.eql(try mlir.Attribute.parse(ctx, "#tpu.reduction_kind"))); +} + +test "tpu types are built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + const semaphore = try mosaic_tpu.SemaphoreType.get(ctx, .{}); + try std.testing.expect(mosaic_tpu.semaphoreType(ctx).isA(mosaic_tpu.SemaphoreType) != null); + try std.testing.expect(mosaic_tpu.semaphoreType(ctx).isA(mosaic_tpu.DMASemaphoreType) == null); + try expectPrints("!tpu.semaphore", semaphore); + + const dma_semaphore = try mosaic_tpu.DMASemaphoreType.get(ctx, .{}); + try std.testing.expect(mosaic_tpu.dmaSemaphoreType(ctx).isA(mosaic_tpu.DMASemaphoreType) != null); + try expectPrints("!tpu.dma_semaphore", dma_semaphore); +} + +test "tpu enum attributes are built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + try expectEnumAttr(mosaic_tpu.ReductionKindAttr, ctx); + try expectEnumAttr(mosaic_tpu.ContractPrecisionAttr, ctx); + try expectEnumAttr(mosaic_tpu.RoundingModeAttr, ctx); + try expectEnumAttr(mosaic_tpu.CoreTypeAttr, ctx); + try expectEnumAttr(mosaic_tpu.DimensionSemanticsAttr, ctx); + try expectEnumAttr(mosaic_tpu.PipelineModeAttr, ctx); + try expectEnumAttr(mosaic_tpu.RevisitModeAttr, ctx); + try expectEnumAttr(mosaic_tpu.MemorySpaceAttr, ctx); + + const smem_tc = try mosaic_tpu.MemorySpaceAttr.get(ctx, .{ .value = .smem, .coreType = .sc_scalar_subcore }); + try std.testing.expectEqual(mosaic_tpu.MemorySpace.smem, smem_tc.value()); + try std.testing.expectEqual(mosaic_tpu.CoreType.sc_scalar_subcore, smem_tc.coreType().?); + try expectPrints("#tpu.memory_space", smem_tc); + try std.testing.expectEqual(null, (try mosaic_tpu.MemorySpaceAttr.get(ctx, .{ .value = .hbm })).coreType()); + // The dialect verifier's pairing rules: no core type on hbm, no scalar subcore on vmem. + try std.testing.expectError(error.InvalidMlir, mosaic_tpu.MemorySpaceAttr.get(ctx, .{ .value = .hbm, .coreType = .tc })); + try std.testing.expectError(error.InvalidMlir, mosaic_tpu.MemorySpaceAttr.get(ctx, .{ .value = .vmem, .coreType = .sc_scalar_subcore })); +} + +test "tpu array attributes are built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + const window = try mosaic_tpu.ElementWindowAttr.get(ctx, .{ .padLow = &.{ 0, 2 }, .padHigh = &.{ 1, 3 } }); + try std.testing.expectEqual(2, window.numPadLow()); + try std.testing.expectEqual(2, window.padLow(1)); + try std.testing.expectEqual(2, window.numPadHigh()); + try std.testing.expectEqual(3, window.padHigh(1)); + try expectPrints("#tpu.element_window<[0, 2], [1, 3]>", window); + try std.testing.expect(mosaic_tpu.elementWindowAttr(ctx, &.{ 0, 2 }, &.{ 1, 3 }).eql(window.attribute())); + + const dims = mosaic_tpu.dotDimensionNumbersAttr(ctx, &.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}); + try expectPrints("#tpu.dot_dimension_numbers<[1], [0], [0], [1], [0, 0, 1, 1], [], []>", dims); + + const batched = try mosaic_tpu.DotDimensionNumbersAttr.get(ctx, .{ + .lhsContractingDims = &.{2}, + .rhsContractingDims = &.{1}, + .lhsNonContractingDims = &.{1}, + .outputDimOrder = &.{ 0, 0, 0, 1 }, + .lhsBatchDims = &.{0}, + .rhsBatchDims = &.{0}, + }); + try std.testing.expectEqual(0, batched.numRhsNonContractingDims()); + try std.testing.expectEqual(4, batched.numOutputDimOrder()); + try std.testing.expectEqual(1, batched.outputDimOrder(3)); + try std.testing.expectEqual(0, batched.rhsBatchDims(0)); + try expectPrints("#tpu.dot_dimension_numbers<[2], [1], [1], [], [0, 0, 0, 1], [0], [0]>", batched); +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + mosaic_tpu.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/BUILD.bazel b/mlir/dialects/nvvm/BUILD.bazel similarity index 70% rename from mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/BUILD.bazel rename to mlir/dialects/nvvm/BUILD.bazel index bcb3ad97..f6529062 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/BUILD.bazel +++ b/mlir/dialects/nvvm/BUILD.bazel @@ -1,12 +1,54 @@ load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") package(default_visibility = ["//visibility:public"]) +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + ":dialect", + ":headers", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "nvvm", + srcs = [ + "ops.zig", + ], + import_name = "mlir/dialects/nvvm", + main = "nvvm.zig", + tags = ["manual"], + deps = [ + ":capi", + "//mlir", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":nvvm", + "//mlir", + ], +) + td_library( name = "td_files", - srcs = glob(["*.td"]), - includes = ["/mlir/dialects/cute_ir/include"], + srcs = [ + "NVVMAttrs.td", + "NVVMDialect.td", + "NVVMEnums.td", + "NVVMOps.td", + ], + includes = ["/"], deps = [ "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", "@llvm-project//mlir:OpBaseTdFiles", @@ -16,7 +58,6 @@ td_library( gentbl_cc_library( name = "dialect_inc_gen", - includes = ["../../../.."], tbl_outs = [ ( [ @@ -40,7 +81,6 @@ gentbl_cc_library( gentbl_cc_library( name = "enums_inc_gen", - includes = ["../../../.."], tbl_outs = [ ( ["-gen-enum-decls"], @@ -58,7 +98,6 @@ gentbl_cc_library( gentbl_cc_library( name = "attrs_inc_gen", - includes = ["../../../.."], tbl_outs = [ ( [ @@ -80,10 +119,8 @@ gentbl_cc_library( deps = [":td_files"], ) -# Every nvvm operation of the compiler's registry. gentbl_cc_library( name = "ops_inc_gen", - includes = ["../../../.."], tbl_outs = [ ( ["-gen-op-decls"], @@ -102,7 +139,6 @@ gentbl_cc_library( cc_library( name = "headers", hdrs = ["NVVMDialect.h"], - includes = ["../../../.."], deps = [ ":attrs_inc_gen", ":dialect_inc_gen", @@ -115,3 +151,18 @@ cc_library( "@llvm-project//mlir:SideEffectInterfaces", ], ) + +cc_library( + name = "dialect", + srcs = [ + "NVVMDialect.cpp", + "NVVMInference.cpp", + ], + deps = [ + ":headers", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:LLVMDialect", + ], +) diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMAttrs.td b/mlir/dialects/nvvm/NVVMAttrs.td similarity index 89% rename from mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMAttrs.td rename to mlir/dialects/nvvm/NVVMAttrs.td index f6ee1f61..20c3fa51 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMAttrs.td +++ b/mlir/dialects/nvvm/NVVMAttrs.td @@ -9,11 +9,11 @@ // //===----------------------------------------------------------------------===// -#ifndef CUTE_IR_DIALECT_NVVM_IR_NVVM_ATTRS_TD -#define CUTE_IR_DIALECT_NVVM_IR_NVVM_ATTRS_TD +#ifndef MLIR_DIALECTS_NVVM_NVVMATTRS_TD +#define MLIR_DIALECTS_NVVM_NVVMATTRS_TD -include "cute_ir/Dialect/NVVM/IR/NVVMDialect.td" -include "cute_ir/Dialect/NVVM/IR/NVVMEnums.td" +include "mlir/dialects/nvvm/NVVMDialect.td" +include "mlir/dialects/nvvm/NVVMEnums.td" def NVVM_MMAShapeAttr : NVVM_Attr<"MMAShape", "shape"> { let summary = "Shape of an MMA operation"; @@ -49,4 +49,4 @@ def NVVM_TargetAttr : NVVM_Attr<"Target", "target"> { let genVerifyDecl = 1; } -#endif // CUTE_IR_DIALECT_NVVM_IR_NVVM_ATTRS_TD +#endif // MLIR_DIALECTS_NVVM_NVVMATTRS_TD diff --git a/mlir/dialects/cute_ir/lib/Dialect/NVVM/IR/NVVMDialect.cpp b/mlir/dialects/nvvm/NVVMDialect.cpp similarity index 56% rename from mlir/dialects/cute_ir/lib/Dialect/NVVM/IR/NVVMDialect.cpp rename to mlir/dialects/nvvm/NVVMDialect.cpp index 28cc6bfd..518e825d 100644 --- a/mlir/dialects/cute_ir/lib/Dialect/NVVM/IR/NVVMDialect.cpp +++ b/mlir/dialects/nvvm/NVVMDialect.cpp @@ -1,46 +1,51 @@ -//===- NVVMDialect.cpp - NVIDIA's nvvm dialect of the CuTe DSL -----------===// +//===- NVVMDialect.cpp - NVIDIA's nvvm dialect of the CuTe DSL ------------===// -#include "cute_ir/Dialect/NVVM/IR/NVVMDialect.h" +#include "mlir/dialects/nvvm/NVVMDialect.h" -#include "llvm/ADT/TypeSwitch.h" #include "mlir/IR/Builders.h" #include "mlir/IR/DialectImplementation.h" +#include "llvm/ADT/TypeSwitch.h" using namespace mlir; using namespace mlir::cutlass_compiler::nvvm; -#include "cute_ir/Dialect/NVVM/IR/NVVMDialect.cpp.inc" +#include "mlir/dialects/nvvm/NVVMDialect.cpp.inc" -#include "cute_ir/Dialect/NVVM/IR/NVVMEnums.cpp.inc" +#include "mlir/dialects/nvvm/NVVMEnums.cpp.inc" #define GET_ATTRDEF_CLASSES -#include "cute_ir/Dialect/NVVM/IR/NVVMAttrs.cpp.inc" +#include "mlir/dialects/nvvm/NVVMAttrs.cpp.inc" #define GET_OP_CLASSES -#include "cute_ir/Dialect/NVVM/IR/NVVMOps.cpp.inc" +#include "mlir/dialects/nvvm/NVVMOps.cpp.inc" void NVVMDialect::initialize() { addAttributes< #define GET_ATTRDEF_LIST -#include "cute_ir/Dialect/NVVM/IR/NVVMAttrs.cpp.inc" +#include "mlir/dialects/nvvm/NVVMAttrs.cpp.inc" >(); addOperations< #define GET_OP_LIST -#include "cute_ir/Dialect/NVVM/IR/NVVMOps.cpp.inc" +#include "mlir/dialects/nvvm/NVVMOps.cpp.inc" >(); } // The compiler's rules and messages. LogicalResult TargetAttr::verify(function_ref emitError, int O, StringRef triple, StringRef chip, - StringRef, DictionaryAttr, ArrayAttr link, bool) { + StringRef, DictionaryAttr, ArrayAttr link, + bool) { if (O < 0 || O > 3) - return emitError() << "The optimization level must be a number between 0 and 3."; + return emitError() + << "The optimization level must be a number between 0 and 3."; if (triple.empty()) return emitError() << "The target triple cannot be empty."; if (chip.empty()) return emitError() << "The target chip cannot be empty."; - if (link && !llvm::all_of(link, [](Attribute attr) { return attr && isa(attr); })) - return emitError() << "All the elements in the `link` array must be strings."; + if (link && !llvm::all_of(link, [](Attribute attr) { + return attr && isa(attr); + })) + return emitError() + << "All the elements in the `link` array must be strings."; return success(); } diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMDialect.h b/mlir/dialects/nvvm/NVVMDialect.h similarity index 60% rename from mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMDialect.h rename to mlir/dialects/nvvm/NVVMDialect.h index db64d49b..daa1c67e 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMDialect.h +++ b/mlir/dialects/nvvm/NVVMDialect.h @@ -1,7 +1,7 @@ -//===- NVVMDialect.h - NVIDIA's nvvm dialect of the CuTe DSL ------*- C++ -*-===// +//===- NVVMDialect.h - NVIDIA's nvvm dialect of the CuTe DSL ----*- C++ -*-===// -#ifndef CUTE_IR_DIALECT_NVVM_IR_NVVM_DIALECT_H -#define CUTE_IR_DIALECT_NVVM_IR_NVVM_DIALECT_H +#ifndef MLIR_DIALECTS_NVVM_NVVMDIALECT_H +#define MLIR_DIALECTS_NVVM_NVVMDIALECT_H #include "mlir/Bytecode/BytecodeOpInterface.h" #include "mlir/Dialect/LLVMIR/LLVMAttrs.h" @@ -14,14 +14,14 @@ #include "mlir/Interfaces/InferTypeOpInterface.h" #include "mlir/Interfaces/SideEffectInterfaces.h" -#include "cute_ir/Dialect/NVVM/IR/NVVMDialect.h.inc" +#include "mlir/dialects/nvvm/NVVMDialect.h.inc" -#include "cute_ir/Dialect/NVVM/IR/NVVMEnums.h.inc" +#include "mlir/dialects/nvvm/NVVMEnums.h.inc" #define GET_ATTRDEF_CLASSES -#include "cute_ir/Dialect/NVVM/IR/NVVMAttrs.h.inc" +#include "mlir/dialects/nvvm/NVVMAttrs.h.inc" #define GET_OP_CLASSES -#include "cute_ir/Dialect/NVVM/IR/NVVMOps.h.inc" +#include "mlir/dialects/nvvm/NVVMOps.h.inc" -#endif // CUTE_IR_DIALECT_NVVM_IR_NVVM_DIALECT_H +#endif // MLIR_DIALECTS_NVVM_NVVMDIALECT_H diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMDialect.td b/mlir/dialects/nvvm/NVVMDialect.td similarity index 80% rename from mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMDialect.td rename to mlir/dialects/nvvm/NVVMDialect.td index 59f5dfaa..03ce7682 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMDialect.td +++ b/mlir/dialects/nvvm/NVVMDialect.td @@ -1,7 +1,7 @@ -//===- NVVMDialect.td - NVIDIA's NVVM dialect of the CuTe DSL -*- tablegen -*-===// +//===- NVVMDialect.td - NVVM dialect of the CuTe DSL -------*- tablegen -*-===// -#ifndef CUTE_IR_DIALECT_NVVM_IR_NVVM_DIALECT_TD -#define CUTE_IR_DIALECT_NVVM_IR_NVVM_DIALECT_TD +#ifndef MLIR_DIALECTS_NVVM_NVVMDIALECT_TD +#define MLIR_DIALECTS_NVVM_NVVMDIALECT_TD include "mlir/IR/AttrTypeBase.td" include "mlir/IR/DialectBase.td" @@ -30,4 +30,4 @@ class NVVM_Attr let mnemonic = attrMnemonic; } -#endif // CUTE_IR_DIALECT_NVVM_IR_NVVM_DIALECT_TD +#endif // MLIR_DIALECTS_NVVM_NVVMDIALECT_TD diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMEnums.td b/mlir/dialects/nvvm/NVVMEnums.td similarity index 99% rename from mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMEnums.td rename to mlir/dialects/nvvm/NVVMEnums.td index e5487b64..fbff0bdc 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMEnums.td +++ b/mlir/dialects/nvvm/NVVMEnums.td @@ -5,10 +5,10 @@ // as a dialect attribute also has its EnumAttr, printed `#` // or, where the compiler prints it so, `#nvvm`. -#ifndef CUTE_IR_GENERATED_NVVM_ENUMS_TD -#define CUTE_IR_GENERATED_NVVM_ENUMS_TD +#ifndef MLIR_DIALECTS_NVVM_NVVMENUMS_TD +#define MLIR_DIALECTS_NVVM_NVVMENUMS_TD -include "cute_ir/Dialect/NVVM/IR/NVVMDialect.td" +include "mlir/dialects/nvvm/NVVMDialect.td" include "mlir/IR/EnumAttr.td" def NVVM_AtomicOpKindEnum : I32EnumAttr<"AtomicOpKind", "operations supported by atom instruction", [ @@ -1383,4 +1383,4 @@ def NVVM_WGMMATypesAttr : EnumAttr, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(0); + if (group.empty()) + return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult +AddPackedF32x2Op::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +AtomicRMWOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +BarrierOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult ClusterLaunchControlQueryCancelOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +CompareAndSetOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(0); + if (group.empty()) + return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult FAbsOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(0); + if (group.empty()) + return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult +FMAPackedF32x2Op::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult FmaOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult InternalMmaBlockScaleOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(8); + if (group.empty()) + return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult InternalMmaSparseBlockScaleOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(8); + if (group.empty()) + return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult +LdMatrixOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult MBarrierArriveDropExpectTxOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult MBarrierArriveDropOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult MBarrierArriveExpectTxOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +MBarrierArriveOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +MBarrierTestWaitOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +MBarrierTryWaitOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +MatchSyncOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult MulOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +MulPackedF32x2Op::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult ReduxOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult RsqrtOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(0); + if (group.empty()) + return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult ShflOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult SubFOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(0); + if (group.empty()) + return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult +SubPackedF32x2Op::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +VoteSyncOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} diff --git a/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMOps.td b/mlir/dialects/nvvm/NVVMOps.td similarity index 99% rename from mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMOps.td rename to mlir/dialects/nvvm/NVVMOps.td index 908a2dff..d77563d2 100644 --- a/mlir/dialects/cute_ir/include/cute_ir/Dialect/NVVM/IR/NVVMOps.td +++ b/mlir/dialects/nvvm/NVVMOps.td @@ -38,11 +38,11 @@ // Constraints not recovered: // nvvm.convert.bf16x2.to.f8x2 `dstTy`: `fnv8E5M3FNU` has no type here and is not accepted -#ifndef CUTE_IR_GENERATED_NVVM_OPS_TD -#define CUTE_IR_GENERATED_NVVM_OPS_TD +#ifndef MLIR_DIALECTS_NVVM_NVVMOPS_TD +#define MLIR_DIALECTS_NVVM_NVVMOPS_TD -include "cute_ir/Dialect/NVVM/IR/NVVMAttrs.td" -include "cute_ir/Dialect/NVVM/IR/NVVMEnums.td" +include "mlir/dialects/nvvm/NVVMAttrs.td" +include "mlir/dialects/nvvm/NVVMEnums.td" include "mlir/Interfaces/InferTypeOpInterface.td" include "mlir/Interfaces/SideEffectInterfaces.td" @@ -6442,4 +6442,4 @@ def NVVM_WMMAStoreOp : NVVM_Op<"wmma.store"> { let results = (outs); } -#endif // CUTE_IR_GENERATED_NVVM_OPS_TD +#endif // MLIR_DIALECTS_NVVM_NVVMOPS_TD diff --git a/mlir/dialects/cute_ir/lib/CAPI/NVVMAttributes.cpp b/mlir/dialects/nvvm/capi.cc similarity index 67% rename from mlir/dialects/cute_ir/lib/CAPI/NVVMAttributes.cpp rename to mlir/dialects/nvvm/capi.cc index c3dc0042..ff71af81 100644 --- a/mlir/dialects/cute_ir/lib/CAPI/NVVMAttributes.cpp +++ b/mlir/dialects/nvvm/capi.cc @@ -1,18 +1,26 @@ -// C API for the nvvm attributes, generated from the dialect's .td -// files. -#include "cute_ir-c/Dialect/NVVMAttributes.h" +//===- capi.cc - C API for CuTe's nvvm dialect ----------------------------===// + +#include "capi.h" + +#include "mlir/dialects/nvvm/NVVMDialect.h" -#include "cute_ir/Dialect/NVVM/IR/NVVMDialect.h" #include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" #include "mlir/CAPI/Support.h" +// Registration + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(CuteNVVM, cute_nvvm, + mlir::cutlass_compiler::nvvm::NVVMDialect) + +// Attributes + namespace nvvm = mlir::cutlass_compiler::nvvm; -namespace { // Unwraps `handle` into `out`: false when it is of another context or not a // T, or when it is null and not optional. template -bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { +static bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { auto value = unwrap(handle); if (!value) return optional; @@ -21,233 +29,277 @@ bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { out = llvm::dyn_cast(value); return static_cast(out); } -} // namespace bool mlirAttributeIsACuteNVVMAtomicOpKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMAtomicOpKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMAtomicOpKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeAtomicOpKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeAtomicOpKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::AtomicOpKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMAtomicOpKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMBarrierReduction(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMBarrierReductionAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMBarrierReductionAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeBarrierReduction(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeBarrierReduction(value); if (!valueValue) return {nullptr}; return wrap(nvvm::BarrierReductionAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMBarrierReductionAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMBarrierReduxKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMBarrierReduxKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMBarrierReduxKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeBarrierReduxKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeBarrierReduxKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::BarrierReduxKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMBarrierReduxKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMBlockScaleFormat(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMBlockScaleFormatAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMBlockScaleFormatAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeBlockScaleFormat(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeBlockScaleFormat(value); if (!valueValue) return {nullptr}; return wrap(nvvm::BlockScaleFormatAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMBlockScaleFormatAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMCTAGroupKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMCTAGroupKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMCTAGroupKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeCTAGroupKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeCTAGroupKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::CTAGroupKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMCTAGroupKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMCVTPackFloatKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMCVTPackFloatKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMCVTPackFloatKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeCVTPackFloatKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeCVTPackFloatKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::CVTPackFloatKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMCVTPackFloatKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMCacheEvictionPriority(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMCacheEvictionPriorityAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMCacheEvictionPriorityAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeCacheEvictionPriority(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeCacheEvictionPriority(value); if (!valueValue) return {nullptr}; return wrap(nvvm::CacheEvictionPriorityAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMCacheEvictionPriorityAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMClusterLaunchControlQueryType(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); + return llvm::isa_and_nonnull( + unwrap(attr)); } -MlirAttribute mlirCuteNVVMClusterLaunchControlQueryTypeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute +mlirCuteNVVMClusterLaunchControlQueryTypeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeClusterLaunchControlQueryType(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeClusterLaunchControlQueryType( + value); if (!valueValue) return {nullptr}; return wrap(nvvm::ClusterLaunchControlQueryTypeAttr::get(ctx, *valueValue)); } -uint32_t mlirCuteNVVMClusterLaunchControlQueryTypeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); +uint32_t +mlirCuteNVVMClusterLaunchControlQueryTypeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); } bool mlirAttributeIsACuteNVVMCompareOpKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMCompareOpKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMCompareOpKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeCompareOpKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeCompareOpKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::CompareOpKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMCompareOpKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMConvertFP4Type(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMConvertFP4TypeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMConvertFP4TypeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeConvertFP4Type(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeConvertFP4Type(value); if (!valueValue) return {nullptr}; return wrap(nvvm::ConvertFP4TypeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMConvertFP4TypeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMConvertFP8Type(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMConvertFP8TypeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMConvertFP8TypeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeConvertFP8Type(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeConvertFP8Type(value); if (!valueValue) return {nullptr}; return wrap(nvvm::ConvertFP8TypeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMConvertFP8TypeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMConvertScaleKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMConvertScaleKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMConvertScaleKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeConvertScaleKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeConvertScaleKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::ConvertScaleKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMConvertScaleKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMDotAccumulateType(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMDotAccumulateTypeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMDotAccumulateTypeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeDotAccumulateType(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeDotAccumulateType(value); if (!valueValue) return {nullptr}; return wrap(nvvm::DotAccumulateTypeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMDotAccumulateTypeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMEvictKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMEvictKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMEvictKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; @@ -258,132 +310,156 @@ MlirAttribute mlirCuteNVVMEvictKindAttrGet(MlirContext context, uint32_t value) return wrap(nvvm::EvictKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMEvictKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMFPRoundingMode(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMFPRoundingModeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMFPRoundingModeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeFPRoundingMode(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeFPRoundingMode(value); if (!valueValue) return {nullptr}; return wrap(nvvm::FPRoundingModeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMFPRoundingModeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMGridDepActionKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMGridDepActionKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMGridDepActionKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeGridDepActionKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeGridDepActionKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::GridDepActionKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMGridDepActionKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMIntegerRoundingMode(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMIntegerRoundingModeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMIntegerRoundingModeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeIntegerRoundingMode(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeIntegerRoundingMode(value); if (!valueValue) return {nullptr}; return wrap(nvvm::IntegerRoundingModeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMIntegerRoundingModeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVML2PrefetchSize(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVML2PrefetchSizeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVML2PrefetchSizeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeL2PrefetchSize(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeL2PrefetchSize(value); if (!valueValue) return {nullptr}; return wrap(nvvm::L2PrefetchSizeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVML2PrefetchSizeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMLdStMatrixEltType(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMLdStMatrixEltTypeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMLdStMatrixEltTypeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeLdStMatrixEltType(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeLdStMatrixEltType(value); if (!valueValue) return {nullptr}; return wrap(nvvm::LdStMatrixEltTypeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMLdStMatrixEltTypeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMLoadCacheModifierExtKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); + return llvm::isa_and_nonnull( + unwrap(attr)); } -MlirAttribute mlirCuteNVVMLoadCacheModifierExtKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMLoadCacheModifierExtKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeLoadCacheModifierExtKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeLoadCacheModifierExtKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::LoadCacheModifierExtKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMLoadCacheModifierExtKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMLoadCacheModifierKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMLoadCacheModifierKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMLoadCacheModifierKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeLoadCacheModifierKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeLoadCacheModifierKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::LoadCacheModifierKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMLoadCacheModifierKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMLoadShape(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMLoadShapeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMLoadShapeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; @@ -394,126 +470,148 @@ MlirAttribute mlirCuteNVVMLoadShapeAttrGet(MlirContext context, uint32_t value) return wrap(nvvm::LoadShapeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMLoadShapeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMLoadSrcFormat(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMLoadSrcFormatAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMLoadSrcFormatAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeLoadSrcFormat(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeLoadSrcFormat(value); if (!valueValue) return {nullptr}; return wrap(nvvm::LoadSrcFormatAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMLoadSrcFormatAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMMBarrierLayout(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMMBarrierLayoutAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMMBarrierLayoutAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierLayout(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierLayout(value); if (!valueValue) return {nullptr}; return wrap(nvvm::MBarrierLayoutAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMMBarrierLayoutAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMMBarrierPhase(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMMBarrierPhaseAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMMBarrierPhaseAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierPhase(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierPhase(value); if (!valueValue) return {nullptr}; return wrap(nvvm::MBarrierPhaseAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMMBarrierPhaseAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMMBarrierScopeKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMMBarrierScopeKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMMBarrierScopeKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierScopeKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierScopeKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::MBarrierScopeKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMMBarrierScopeKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMMBarrierSpaceKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMMBarrierSpaceKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMMBarrierSpaceKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierSpaceKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierSpaceKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::MBarrierSpaceKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMMBarrierSpaceKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMMBarrierTxnKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMMBarrierTxnKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMMBarrierTxnKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierTxnKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierTxnKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::MBarrierTxnKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMMBarrierTxnKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMMBarrierWaitKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMMBarrierWaitKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMMBarrierWaitKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierWaitKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierWaitKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::MBarrierWaitKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMMBarrierWaitKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMMMAB1Op(MlirAttribute attr) { @@ -530,30 +628,35 @@ MlirAttribute mlirCuteNVVMMMAB1OpAttrGet(MlirContext context, uint32_t value) { return wrap(nvvm::MMAB1OpAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMMMAB1OpAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMMMABlockScaleKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMMMABlockScaleKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMMMABlockScaleKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMABlockScaleKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMMABlockScaleKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::MMABlockScaleKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMMMABlockScaleKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMMMACtaCount(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMMMACtaCountAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMMMACtaCountAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; @@ -564,7 +667,8 @@ MlirAttribute mlirCuteNVVMMMACtaCountAttrGet(MlirContext context, uint32_t value return wrap(nvvm::MMACtaCountAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMMMACtaCountAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMMMAFrag(MlirAttribute attr) { @@ -581,24 +685,28 @@ MlirAttribute mlirCuteNVVMMMAFragAttrGet(MlirContext context, uint32_t value) { return wrap(nvvm::MMAFragAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMMMAFragAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMMMAIntOverflow(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMMMAIntOverflowAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMMMAIntOverflowAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMAIntOverflow(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMMAIntOverflow(value); if (!valueValue) return {nullptr}; return wrap(nvvm::MMAIntOverflowAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMMMAIntOverflowAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMMMAKind(MlirAttribute attr) { @@ -615,13 +723,15 @@ MlirAttribute mlirCuteNVVMMMAKindAttrGet(MlirContext context, uint32_t value) { return wrap(nvvm::MMAKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMMMAKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMMMALayout(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMMMALayoutAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMMMALayoutAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; @@ -632,7 +742,8 @@ MlirAttribute mlirCuteNVVMMMALayoutAttrGet(MlirContext context, uint32_t value) return wrap(nvvm::MMALayoutAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMMMALayoutAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMMMATypes(MlirAttribute attr) { @@ -649,58 +760,68 @@ MlirAttribute mlirCuteNVVMMMATypesAttrGet(MlirContext context, uint32_t value) { return wrap(nvvm::MMATypesAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMMMATypesAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMMatchSyncKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMMatchSyncKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMMatchSyncKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMatchSyncKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMatchSyncKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::MatchSyncKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMMatchSyncKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMMemOrderKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMMemOrderKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMMemOrderKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMemOrderKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMemOrderKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::MemOrderKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMMemOrderKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMMemScopeKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMMemScopeKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMMemScopeKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMemScopeKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMemScopeKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::MemScopeKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMMemScopeKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMMulMode(MlirAttribute attr) { @@ -717,30 +838,35 @@ MlirAttribute mlirCuteNVVMMulModeAttrGet(MlirContext context, uint32_t value) { return wrap(nvvm::MulModeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMMulModeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMNVVMMemorySpace(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMNVVMMemorySpaceAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMNVVMMemorySpaceAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeNVVMMemorySpace(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeNVVMMemorySpace(value); if (!valueValue) return {nullptr}; return wrap(nvvm::NVVMMemorySpaceAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMNVVMMemorySpaceAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMPermuteMode(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMPermuteModeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMPermuteModeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; @@ -751,30 +877,35 @@ MlirAttribute mlirCuteNVVMPermuteModeAttrGet(MlirContext context, uint32_t value return wrap(nvvm::PermuteModeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMPermuteModeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMPrefetchCacheLevel(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMPrefetchCacheLevelAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMPrefetchCacheLevelAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizePrefetchCacheLevel(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizePrefetchCacheLevel(value); if (!valueValue) return {nullptr}; return wrap(nvvm::PrefetchCacheLevelAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMPrefetchCacheLevelAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMProxyKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMProxyKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMProxyKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; @@ -785,30 +916,35 @@ MlirAttribute mlirCuteNVVMProxyKindAttrGet(MlirContext context, uint32_t value) return wrap(nvvm::ProxyKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMProxyKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMReductionKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMReductionKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMReductionKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeReductionKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeReductionKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::ReductionKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMReductionKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMReductionOp(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMReductionOpAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMReductionOpAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; @@ -819,251 +955,295 @@ MlirAttribute mlirCuteNVVMReductionOpAttrGet(MlirContext context, uint32_t value return wrap(nvvm::ReductionOpAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMReductionOpAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMReductionType(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMReductionTypeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMReductionTypeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeReductionType(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeReductionType(value); if (!valueValue) return {nullptr}; return wrap(nvvm::ReductionTypeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMReductionTypeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMSPCompressElemSize(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMSPCompressElemSizeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMSPCompressElemSizeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressElemSize(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressElemSize(value); if (!valueValue) return {nullptr}; return wrap(nvvm::SPCompressElemSizeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMSPCompressElemSizeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMSPCompressFactorType(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMSPCompressFactorTypeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMSPCompressFactorTypeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressFactorType(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressFactorType(value); if (!valueValue) return {nullptr}; return wrap(nvvm::SPCompressFactorTypeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMSPCompressFactorTypeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMSPCompressIndexSize(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMSPCompressIndexSizeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMSPCompressIndexSizeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressIndexSize(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressIndexSize(value); if (!valueValue) return {nullptr}; return wrap(nvvm::SPCompressIndexSizeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMSPCompressIndexSizeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMSPCompressOpKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMSPCompressOpKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMSPCompressOpKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressOpKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressOpKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::SPCompressOpKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMSPCompressOpKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMSPCompressRepFactor(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMSPCompressRepFactorAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMSPCompressRepFactorAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressRepFactor(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressRepFactor(value); if (!valueValue) return {nullptr}; return wrap(nvvm::SPCompressRepFactorAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMSPCompressRepFactorAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMSPDecompressElemSize(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMSPDecompressElemSizeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMSPDecompressElemSizeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSPDecompressElemSize(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPDecompressElemSize(value); if (!valueValue) return {nullptr}; return wrap(nvvm::SPDecompressElemSizeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMSPDecompressElemSizeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMSPDecompressFactorType(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMSPDecompressFactorTypeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMSPDecompressFactorTypeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSPDecompressFactorType(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPDecompressFactorType(value); if (!valueValue) return {nullptr}; return wrap(nvvm::SPDecompressFactorTypeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMSPDecompressFactorTypeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMSPDecompressIndexSize(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMSPDecompressIndexSizeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMSPDecompressIndexSizeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSPDecompressIndexSize(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPDecompressIndexSize(value); if (!valueValue) return {nullptr}; return wrap(nvvm::SPDecompressIndexSizeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMSPDecompressIndexSizeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMSPDecompressRepFactor(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMSPDecompressRepFactorAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMSPDecompressRepFactorAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSPDecompressRepFactor(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPDecompressRepFactor(value); if (!valueValue) return {nullptr}; return wrap(nvvm::SPDecompressRepFactorAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMSPDecompressRepFactorAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMSaturationMode(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMSaturationModeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMSaturationModeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSaturationMode(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSaturationMode(value); if (!valueValue) return {nullptr}; return wrap(nvvm::SaturationModeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMSaturationModeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMSaturationModeKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMSaturationModeKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMSaturationModeKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSaturationModeKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSaturationModeKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::SaturationModeKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMSaturationModeKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMScaleVecSize(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMScaleVecSizeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMScaleVecSizeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeScaleVecSize(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeScaleVecSize(value); if (!valueValue) return {nullptr}; return wrap(nvvm::ScaleVecSizeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMScaleVecSizeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMSetMaxRegisterAction(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMSetMaxRegisterActionAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMSetMaxRegisterActionAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSetMaxRegisterAction(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSetMaxRegisterAction(value); if (!valueValue) return {nullptr}; return wrap(nvvm::SetMaxRegisterActionAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMSetMaxRegisterActionAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMSharedSpace(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMSharedSpaceAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMSharedSpaceAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; @@ -1074,7 +1254,8 @@ MlirAttribute mlirCuteNVVMSharedSpaceAttrGet(MlirContext context, uint32_t value return wrap(nvvm::SharedSpaceAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMSharedSpaceAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMShflKind(MlirAttribute attr) { @@ -1091,30 +1272,35 @@ MlirAttribute mlirCuteNVVMShflKindAttrGet(MlirContext context, uint32_t value) { return wrap(nvvm::ShflKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMShflKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMSparsityFormat(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMSparsityFormatAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMSparsityFormatAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSparsityFormat(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSparsityFormat(value); if (!valueValue) return {nullptr}; return wrap(nvvm::SparsityFormatAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMSparsityFormatAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMStateSpace(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMStateSpaceAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMStateSpaceAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; @@ -1125,30 +1311,35 @@ MlirAttribute mlirCuteNVVMStateSpaceAttrGet(MlirContext context, uint32_t value) return wrap(nvvm::StateSpaceAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMStateSpaceAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMStoreCacheModifierKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMStoreCacheModifierKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMStoreCacheModifierKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeStoreCacheModifierKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeStoreCacheModifierKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::StoreCacheModifierKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMStoreCacheModifierKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMStoreShape(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMStoreShapeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMStoreShapeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; @@ -1159,13 +1350,15 @@ MlirAttribute mlirCuteNVVMStoreShapeAttrGet(MlirContext context, uint32_t value) return wrap(nvvm::StoreShapeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMStoreShapeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMTCBarParam(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMTCBarParamAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMTCBarParamAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; @@ -1176,13 +1369,15 @@ MlirAttribute mlirCuteNVVMTCBarParamAttrGet(MlirContext context, uint32_t value) return wrap(nvvm::TCBarParamAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMTCBarParamAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMTMALoadMode(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMTMALoadModeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMTMALoadModeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; @@ -1193,319 +1388,381 @@ MlirAttribute mlirCuteNVVMTMALoadModeAttrGet(MlirContext context, uint32_t value return wrap(nvvm::TMALoadModeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMTMALoadModeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMTMAReduxKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMTMAReduxKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMTMAReduxKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTMAReduxKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTMAReduxKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::TMAReduxKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMTMAReduxKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMTMAStoreMode(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMTMAStoreModeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMTMAStoreModeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTMAStoreMode(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTMAStoreMode(value); if (!valueValue) return {nullptr}; return wrap(nvvm::TMAStoreModeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMTMAStoreModeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMTcgen05CpMulticast(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMTcgen05CpMulticastAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMTcgen05CpMulticastAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05CpMulticast(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05CpMulticast(value); if (!valueValue) return {nullptr}; return wrap(nvvm::Tcgen05CpMulticastAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMTcgen05CpMulticastAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMTcgen05CpShape(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMTcgen05CpShapeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMTcgen05CpShapeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05CpShape(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05CpShape(value); if (!valueValue) return {nullptr}; return wrap(nvvm::Tcgen05CpShapeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMTcgen05CpShapeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMTcgen05CpSrcFormat(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMTcgen05CpSrcFormatAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMTcgen05CpSrcFormatAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05CpSrcFormat(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05CpSrcFormat(value); if (!valueValue) return {nullptr}; return wrap(nvvm::Tcgen05CpSrcFormatAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMTcgen05CpSrcFormatAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMTcgen05FenceKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMTcgen05FenceKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMTcgen05FenceKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05FenceKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05FenceKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::Tcgen05FenceKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMTcgen05FenceKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMTcgen05LdStShape(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMTcgen05LdStShapeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMTcgen05LdStShapeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05LdStShape(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05LdStShape(value); if (!valueValue) return {nullptr}; return wrap(nvvm::Tcgen05LdStShapeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMTcgen05LdStShapeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMTcgen05MMABlockScale(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMTcgen05MMABlockScaleAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMTcgen05MMABlockScaleAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05MMABlockScale(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05MMABlockScale(value); if (!valueValue) return {nullptr}; return wrap(nvvm::Tcgen05MMABlockScaleAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMTcgen05MMABlockScaleAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMTcgen05MMACollectorBBuffer(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); + return llvm::isa_and_nonnull( + unwrap(attr)); } -MlirAttribute mlirCuteNVVMTcgen05MMACollectorBBufferAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMTcgen05MMACollectorBBufferAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05MMACollectorBBuffer(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05MMACollectorBBuffer( + value); if (!valueValue) return {nullptr}; return wrap(nvvm::Tcgen05MMACollectorBBufferAttr::get(ctx, *valueValue)); } -uint32_t mlirCuteNVVMTcgen05MMACollectorBBufferAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); +uint32_t +mlirCuteNVVMTcgen05MMACollectorBBufferAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); } bool mlirAttributeIsACuteNVVMTcgen05MMACollectorOp(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMTcgen05MMACollectorOpAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMTcgen05MMACollectorOpAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05MMACollectorOp(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05MMACollectorOp(value); if (!valueValue) return {nullptr}; return wrap(nvvm::Tcgen05MMACollectorOpAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMTcgen05MMACollectorOpAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMTcgen05MMAKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMTcgen05MMAKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMTcgen05MMAKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05MMAKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05MMAKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::Tcgen05MMAKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMTcgen05MMAKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMTcgen05WaitKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMTcgen05WaitKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMTcgen05WaitKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05WaitKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05WaitKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::Tcgen05WaitKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMTcgen05WaitKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMTensormapElemtype(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMTensormapElemtypeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMTensormapElemtypeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTensormapElemtype(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTensormapElemtype(value); if (!valueValue) return {nullptr}; return wrap(nvvm::TensormapElemtypeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMTensormapElemtypeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMTensormapField(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMTensormapFieldAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMTensormapFieldAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTensormapField(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTensormapField(value); if (!valueValue) return {nullptr}; return wrap(nvvm::TensormapFieldAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMTensormapFieldAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMTensormapFillMode(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMTensormapFillModeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMTensormapFillModeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTensormapFillMode(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTensormapFillMode(value); if (!valueValue) return {nullptr}; return wrap(nvvm::TensormapFillModeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMTensormapFillModeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMTensormapInterleaveLayout(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); + return llvm::isa_and_nonnull( + unwrap(attr)); } -MlirAttribute mlirCuteNVVMTensormapInterleaveLayoutAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMTensormapInterleaveLayoutAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTensormapInterleaveLayout(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTensormapInterleaveLayout(value); if (!valueValue) return {nullptr}; return wrap(nvvm::TensormapInterleaveLayoutAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMTensormapInterleaveLayoutAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMTensormapSwizzleAtomicity(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); + return llvm::isa_and_nonnull( + unwrap(attr)); } -MlirAttribute mlirCuteNVVMTensormapSwizzleAtomicityAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMTensormapSwizzleAtomicityAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTensormapSwizzleAtomicity(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTensormapSwizzleAtomicity(value); if (!valueValue) return {nullptr}; return wrap(nvvm::TensormapSwizzleAtomicityAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMTensormapSwizzleAtomicityAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMTensormapSwizzleMode(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMTensormapSwizzleModeAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMTensormapSwizzleModeAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTensormapSwizzleMode(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTensormapSwizzleMode(value); if (!valueValue) return {nullptr}; return wrap(nvvm::TensormapSwizzleModeAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMTensormapSwizzleModeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMTmemLayout(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMTmemLayoutAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMTmemLayoutAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; @@ -1516,81 +1773,95 @@ MlirAttribute mlirCuteNVVMTmemLayoutAttrGet(MlirContext context, uint32_t value) return wrap(nvvm::TmemLayoutAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMTmemLayoutAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMValidatePattern(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMValidatePatternAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMValidatePatternAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeValidatePattern(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeValidatePattern(value); if (!valueValue) return {nullptr}; return wrap(nvvm::ValidatePatternAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMValidatePatternAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMVoteSyncKind(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMVoteSyncKindAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMVoteSyncKindAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeVoteSyncKind(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeVoteSyncKind(value); if (!valueValue) return {nullptr}; return wrap(nvvm::VoteSyncKindAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMVoteSyncKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMWGMMAScaleIn(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMWGMMAScaleInAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMWGMMAScaleInAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeWGMMAScaleIn(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeWGMMAScaleIn(value); if (!valueValue) return {nullptr}; return wrap(nvvm::WGMMAScaleInAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMWGMMAScaleInAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMWGMMAScaleOut(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMWGMMAScaleOutAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMWGMMAScaleOutAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeWGMMAScaleOut(value); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeWGMMAScaleOut(value); if (!valueValue) return {nullptr}; return wrap(nvvm::WGMMAScaleOutAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMWGMMAScaleOutAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMWGMMATypes(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMWGMMATypesAttrGet(MlirContext context, uint32_t value) { +MlirAttribute mlirCuteNVVMWGMMATypesAttrGet(MlirContext context, + uint32_t value) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; @@ -1601,13 +1872,15 @@ MlirAttribute mlirCuteNVVMWGMMATypesAttrGet(MlirContext context, uint32_t value) return wrap(nvvm::WGMMATypesAttr::get(ctx, *valueValue)); } uint32_t mlirCuteNVVMWGMMATypesAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); + return static_cast( + llvm::cast(unwrap(attr)).getValue()); } bool mlirAttributeIsACuteNVVMLdStMatrixShape(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMLdStMatrixShapeAttrGet(MlirContext context, int m, int n) { +MlirAttribute mlirCuteNVVMLdStMatrixShapeAttrGet(MlirContext context, int m, + int n) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; @@ -1624,7 +1897,8 @@ int mlirCuteNVVMLdStMatrixShapeAttrGetN(MlirAttribute attr) { bool mlirAttributeIsACuteNVVMMMAShape(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMMMAShapeAttrGet(MlirContext context, int m, int n, int k) { +MlirAttribute mlirCuteNVVMMMAShapeAttrGet(MlirContext context, int m, int n, + int k) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; @@ -1644,7 +1918,12 @@ int mlirCuteNVVMMMAShapeAttrGetK(MlirAttribute attr) { bool mlirAttributeIsACuteNVVMTarget(MlirAttribute attr) { return llvm::isa_and_nonnull(unwrap(attr)); } -MlirAttribute mlirCuteNVVMTargetAttrGet(MlirContext context, int O, MlirStringRef triple, MlirStringRef chip, MlirStringRef features, MlirAttribute flags, MlirAttribute link, bool verifyTarget) { +MlirAttribute mlirCuteNVVMTargetAttrGet(MlirContext context, int O, + MlirStringRef triple, + MlirStringRef chip, + MlirStringRef features, + MlirAttribute flags, MlirAttribute link, + bool verifyTarget) { auto *ctx = unwrap(context); if (!ctx) return {nullptr}; @@ -1656,7 +1935,9 @@ MlirAttribute mlirCuteNVVMTargetAttrGet(MlirContext context, int O, MlirStringRe if (!unwrapAs(ctx, link, true, linkValue)) return {nullptr}; return wrap(nvvm::TargetAttr::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, O, unwrap(triple), unwrap(chip), unwrap(features), flagsValue, linkValue, verifyTarget)); + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, O, + unwrap(triple), unwrap(chip), unwrap(features), flagsValue, linkValue, + verifyTarget)); } int mlirCuteNVVMTargetAttrGetO(MlirAttribute attr) { return llvm::cast(unwrap(attr)).getO(); @@ -1671,10 +1952,12 @@ MlirStringRef mlirCuteNVVMTargetAttrGetFeatures(MlirAttribute attr) { return wrap(llvm::cast(unwrap(attr)).getFeatures()); } MlirAttribute mlirCuteNVVMTargetAttrGetFlags(MlirAttribute attr) { - return wrap(static_cast(llvm::cast(unwrap(attr)).getFlags())); + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFlags())); } MlirAttribute mlirCuteNVVMTargetAttrGetLink(MlirAttribute attr) { - return wrap(static_cast(llvm::cast(unwrap(attr)).getLink())); + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLink())); } bool mlirCuteNVVMTargetAttrGetVerifyTarget(MlirAttribute attr) { return llvm::cast(unwrap(attr)).getVerifyTarget(); diff --git a/mlir/dialects/nvvm/capi.h b/mlir/dialects/nvvm/capi.h new file mode 100644 index 00000000..ee604770 --- /dev/null +++ b/mlir/dialects/nvvm/capi.h @@ -0,0 +1,785 @@ +//===- capi.h - C API for CuTe's nvvm dialect ---------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_NVVM_CAPI_H +#define MLIR_DIALECTS_NVVM_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Registration + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(CuteNVVM, cute_nvvm); + +// Attributes + +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles. Enums are their integer +// values (the dialect's *Enums.td). Getters require a value accepted by the +// matching IsA. + +// `#nvvm.atomic_op`: operations supported by atom instruction +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMAtomicOpKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMAtomicOpKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMAtomicOpKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.reduction`: NVVM barrier reduction operation +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMBarrierReduction(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMBarrierReductionAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMBarrierReductionAttrGetValue(MlirAttribute attr); + +// `#nvvm.barrier_redux_kind`: NVVM barrier redux kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMBarrierReduxKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMBarrierReduxKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMBarrierReduxKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.block_scale_format`: MMA Block Scale Format +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMBlockScaleFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMBlockScaleFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMBlockScaleFormatAttrGetValue(MlirAttribute attr); + +// `#nvvm.cta_group`: NVVM CTA group kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMCTAGroupKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMCTAGroupKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMCTAGroupKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.packfloat_type`: NVVM CVT Pack Float kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMCVTPackFloatKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMCVTPackFloatKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMCVTPackFloatKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.cache_eviction_priority`: NVVM Cache Eviction Priority +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMCacheEvictionPriority(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMCacheEvictionPriorityAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMCacheEvictionPriorityAttrGetValue(MlirAttribute attr); + +// `#nvvm.cluster_launch_control_query_type`: NVVM ClusterLaunchControlQueryType +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMClusterLaunchControlQueryType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMClusterLaunchControlQueryTypeAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMClusterLaunchControlQueryTypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.op`: Comparison operator encoding +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMCompareOpKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMCompareOpKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMCompareOpKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.convert_fp4_type`: NVVM ConvertFP4Type kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMConvertFP4Type(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMConvertFP4TypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMConvertFP4TypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.convert_fp8_type`: NVVM ConvertFP8Type kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMConvertFP8Type(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMConvertFP8TypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMConvertFP8TypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.convert_scale_kind`: NVVM ConvertScale kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMConvertScaleKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMConvertScaleKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMConvertScaleKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.dot_accumulate_type`: NVVM DotAccumulateType +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMDotAccumulateType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMDotAccumulateTypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMDotAccumulateTypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.evict_kind`: NVVM L2 Prefetch Size +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMEvictKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMEvictKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMEvictKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.fp_rnd_mode`: NVVM FPRoundingMode kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMFPRoundingMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMFPRoundingModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMFPRoundingModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.grid_dep_action`: Action kind for grid dependency control +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMGridDepActionKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMGridDepActionKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMGridDepActionKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.int_rnd_mode`: NVVM IntegerRoundingMode kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMIntegerRoundingMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMIntegerRoundingModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMIntegerRoundingModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.l2_prefetch`: NVVM L2 Prefetch Size +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVML2PrefetchSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVML2PrefetchSizeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVML2PrefetchSizeAttrGetValue(MlirAttribute attr); + +// `#nvvm.ld_st_matrix_elt_type`: Element type for ldmatrix and stmatrix +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMLdStMatrixEltType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMLdStMatrixEltTypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMLdStMatrixEltTypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.load_cache_modifier_ext`: NVVM load cache modifier kind(Ext) +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMLoadCacheModifierExtKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMLoadCacheModifierExtKindAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMLoadCacheModifierExtKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.load_cache_modifier`: NVVM load cache modifier kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMLoadCacheModifierKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMLoadCacheModifierKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMLoadCacheModifierKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.load_shape`: shape attribute for ldmatrix +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMLoadShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMLoadShapeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMLoadShapeAttrGetValue(MlirAttribute attr); + +// `#nvvm.load_src_format`: source format for ldmatrix +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMLoadSrcFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMLoadSrcFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMLoadSrcFormatAttrGetValue(MlirAttribute attr); + +// `#nvvm.mbarrier_layout`: NVVM MBarrier Layout +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMBarrierLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMBarrierLayoutAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMBarrierLayoutAttrGetValue(MlirAttribute attr); + +// `#nvvm.mbarrier_phase`: NVVM mbarrier phase type +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMBarrierPhase(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMBarrierPhaseAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMBarrierPhaseAttrGetValue(MlirAttribute attr); + +// `#nvvm.mbar_scope`: NVVM MBarrier scope kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMBarrierScopeKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMBarrierScopeKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMBarrierScopeKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mbar_space`: NVVM MBarrier space kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMBarrierSpaceKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMBarrierSpaceKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMBarrierSpaceKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mbar_txn_kind`: NVVM MBarrier Transaction kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMBarrierTxnKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMBarrierTxnKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMBarrierTxnKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mbar_wait`: NVVM MBarrier wait kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMBarrierWaitKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMBarrierWaitKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMBarrierWaitKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_b1op`: MMA binary operations +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAB1Op(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMAB1OpAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMAB1OpAttrGetValue(MlirAttribute attr); + +// `#nvvm.block_scale_kind`: Block Scale Kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMMABlockScaleKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMMABlockScaleKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMMABlockScaleKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_cta_count`: MMA CTA count +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMACtaCount(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMMACtaCountAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMMACtaCountAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_frag`: NVVM MMA frag type +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAFrag(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMAFragAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMAFragAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_int_overflow`: MMA overflow options +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMMAIntOverflow(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMMAIntOverflowAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMMAIntOverflowAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_kind`: MMA operation kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMAKindAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMAKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_layout`: NVVM MMA layout +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMALayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMMALayoutAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMMALayoutAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_type`: NVVM MMA types +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMATypes(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMMATypesAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMMATypesAttrGetValue(MlirAttribute attr); + +// `#nvvm.match_sync_kind`: NVVM match sync kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMatchSyncKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMatchSyncKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMatchSyncKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mem_order`: NVVM Memory Ordering kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMemOrderKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMemOrderKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMemOrderKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mem_scope`: NVVM Memory Scope kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMemScopeKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMemScopeKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMemScopeKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mul_mode`: multiply mode attribute +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMulMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMulModeAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMulModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.memory_space`: NVVM Memory Space +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMNVVMMemorySpace(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMNVVMMemorySpaceAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMNVVMMemorySpaceAttrGetValue(MlirAttribute attr); + +// `#nvvm.permute_mode`: NVVM permute mode +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMPermuteMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMPermuteModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMPermuteModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.prefetch_cache_level`: NVVM Prefetch Cache Level +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMPrefetchCacheLevel(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMPrefetchCacheLevelAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMPrefetchCacheLevelAttrGetValue(MlirAttribute attr); + +// `#nvvm.proxy_kind`: Proxy kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMProxyKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMProxyKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMProxyKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.reduction_kind`: NVVM Reduction Kind attribute +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMReductionKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMReductionKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMReductionKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.red_op`: Ops supported by red instruction +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMReductionOp(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMReductionOpAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMReductionOpAttrGetValue(MlirAttribute attr); + +// `#nvvm.red_type`: types supported by red instruction +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMReductionType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMReductionTypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMReductionTypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spcomp_elem_size`: Sparse tensor compression element size +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPCompressElemSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPCompressElemSizeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPCompressElemSizeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spcomp_factor`: Sparse tensor compression factor type +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPCompressFactorType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPCompressFactorTypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPCompressFactorTypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spcomp_index_size`: Sparse tensor compression index size +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPCompressIndexSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPCompressIndexSizeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPCompressIndexSizeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spcomp_op_kind`: spcompress operation kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPCompressOpKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPCompressOpKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPCompressOpKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.spcomp_rep_factor`: Sparse tensor compression repetition factor +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPCompressRepFactor(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPCompressRepFactorAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPCompressRepFactorAttrGetValue(MlirAttribute attr); + +// `#nvvm.spdecomp_elem_size`: Sparse tensor decompression element size +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPDecompressElemSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPDecompressElemSizeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPDecompressElemSizeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spdecomp_factor`: Sparse tensor decompression factor type +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPDecompressFactorType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPDecompressFactorTypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPDecompressFactorTypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spdecomp_index_size`: Sparse tensor decompression index size +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPDecompressIndexSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPDecompressIndexSizeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPDecompressIndexSizeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spdecomp_rep_factor`: Sparse tensor decompression repetition factor +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPDecompressRepFactor(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPDecompressRepFactorAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPDecompressRepFactorAttrGetValue(MlirAttribute attr); + +// `#nvvm.sat_mode`: NVVM SaturationMode kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSaturationMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSaturationModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSaturationModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.sat`: NVVM SaturationMode kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSaturationModeKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSaturationModeKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSaturationModeKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.scale_vec_size`: MMA Scale Vector Sizes +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMScaleVecSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMScaleVecSizeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMScaleVecSizeAttrGetValue(MlirAttribute attr); + +// `#nvvm.action`: NVVM set max register action +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSetMaxRegisterAction(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSetMaxRegisterActionAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSetMaxRegisterActionAttrGetValue(MlirAttribute attr); + +// `#nvvm.shared_space`: Shared memory space +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSharedSpace(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSharedSpaceAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSharedSpaceAttrGetValue(MlirAttribute attr); + +// `#nvvm.shfl_kind`: NVVM shuffle kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMShflKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMShflKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMShflKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.sparsity_format`: MMA Sparsity Format +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSparsityFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSparsityFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSparsityFormatAttrGetValue(MlirAttribute attr); + +// `#nvvm.state_space`: NVVM State Space +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMStateSpace(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMStateSpaceAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMStateSpaceAttrGetValue(MlirAttribute attr); + +// `#nvvm.store_cache_modifier`: NVVM store cache modifier kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMStoreCacheModifierKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMStoreCacheModifierKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMStoreCacheModifierKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.store_shape`: shape attribute for stmatrix +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMStoreShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMStoreShapeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMStoreShapeAttrGetValue(MlirAttribute attr); + +// `#nvvm.TCBarParam`: Cluster MMA Barrier Parameter Type +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTCBarParam(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTCBarParamAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTCBarParamAttrGetValue(MlirAttribute attr); + +// `#nvvm.tma_load_mode`: NVVM TMA Load Mode +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTMALoadMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTMALoadModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTMALoadModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.tma_redux_kind`: NVVM TMA redux kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTMAReduxKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTMAReduxKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTMAReduxKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.tma_store_mode`: NVVM TMA Store Mode +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTMAStoreMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTMAStoreModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTMAStoreModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_cp_multicast`: tcgen05 cp multicast +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05CpMulticast(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05CpMulticastAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05CpMulticastAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_cp_shape`: tcgen05 cp shapes +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05CpShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05CpShapeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05CpShapeAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_cp_src_fmt`: tcgen05 cp source format +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05CpSrcFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05CpSrcFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05CpSrcFormatAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_fence`: NVVM Tcgen05 fence kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05FenceKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05FenceKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05FenceKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_ldst_shape`: allowed 32-bit signless integer cases: 0, 1, 2, +// 3, 4 +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05LdStShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05LdStShapeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05LdStShapeAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_mma_block_scale`: tcgen05.mma block scale attribute +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05MMABlockScale(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05MMABlockScaleAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05MMABlockScaleAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_mma_collectorb`: tcgen05 MMA Collector Buffer B Attribute +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05MMACollectorBBuffer(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTcgen05MMACollectorBBufferAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05MMACollectorBBufferAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_mma_collectorop`: tcgen05.mma Collector Buffer Operation +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05MMACollectorOp(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05MMACollectorOpAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05MMACollectorOpAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_mma_kind`: tcgen05 MMA Supported Types +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05MMAKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05MMAKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05MMAKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_wait`: NVVM Tcgen05 wait kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05WaitKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05WaitKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05WaitKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.tensormap_elemtype`: NVVM Tensormap Elemtype +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTensormapElemtype(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTensormapElemtypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTensormapElemtypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.tensormap_field`: NVVM Tensormap Field Kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTensormapField(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTensormapFieldAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTensormapFieldAttrGetValue(MlirAttribute attr); + +// `#nvvm.tensormap_fill_mode`: NVVM Tensormap Fill Mode +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTensormapFillMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTensormapFillModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTensormapFillModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.tensormap_interleave_layout`: NVVM Tensormap Interleave Layout +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTensormapInterleaveLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTensormapInterleaveLayoutAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTensormapInterleaveLayoutAttrGetValue(MlirAttribute attr); + +// `#nvvm.tensormap_swizzle_atomicity`: NVVM Tensormap Swizzle Atomicity +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTensormapSwizzleAtomicity(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTensormapSwizzleAtomicityAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTensormapSwizzleAtomicityAttrGetValue(MlirAttribute attr); + +// `#nvvm.tensormap_swizzle_mode`: NVVM Tensormap Swizzle Mode +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTensormapSwizzleMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTensormapSwizzleModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTensormapSwizzleModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.TmemLayout`: Tensor Memory Layout Enumerated Type +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTmemLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTmemLayoutAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTmemLayoutAttrGetValue(MlirAttribute attr); + +// `#nvvm.validate_pattern`: NVVM validate data pattern +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMValidatePattern(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMValidatePatternAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMValidatePatternAttrGetValue(MlirAttribute attr); + +// `#nvvm.vote_sync_kind`: NVVM vote sync kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMVoteSyncKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMVoteSyncKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMVoteSyncKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.wgmma_scale_in`: WGMMA overflow options +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMWGMMAScaleIn(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMWGMMAScaleInAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMWGMMAScaleInAttrGetValue(MlirAttribute attr); + +// `#nvvm.wgmma_scale_out`: WGMMA input predicate +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMWGMMAScaleOut(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMWGMMAScaleOutAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMWGMMAScaleOutAttrGetValue(MlirAttribute attr); + +// `#nvvm.wgmma_type`: NVVM WGMMA types +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMWGMMATypes(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMWGMMATypesAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMWGMMATypesAttrGetValue(MlirAttribute attr); + +// `#nvvm.ld_st_matrix_shape`: Matrix shape of ldmatrix, stmatrix and movmatrix +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMLdStMatrixShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMLdStMatrixShapeAttrGet(MlirContext context, int m, int n); +MLIR_CAPI_EXPORTED int mlirCuteNVVMLdStMatrixShapeAttrGetM(MlirAttribute attr); +MLIR_CAPI_EXPORTED int mlirCuteNVVMLdStMatrixShapeAttrGetN(MlirAttribute attr); + +// `#nvvm.shape`: Shape of an MMA operation +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMMAShapeAttrGet(MlirContext context, int m, int n, int k); +MLIR_CAPI_EXPORTED int mlirCuteNVVMMMAShapeAttrGetM(MlirAttribute attr); +MLIR_CAPI_EXPORTED int mlirCuteNVVMMMAShapeAttrGetN(MlirAttribute attr); +MLIR_CAPI_EXPORTED int mlirCuteNVVMMMAShapeAttrGetK(MlirAttribute attr); + +// `#nvvm.target`: GPU target of an NVVM module +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTarget(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTargetAttrGet( + MlirContext context, int O, MlirStringRef triple, MlirStringRef chip, + MlirStringRef features, MlirAttribute flags, MlirAttribute link, + bool verifyTarget); +MLIR_CAPI_EXPORTED int mlirCuteNVVMTargetAttrGetO(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirCuteNVVMTargetAttrGetTriple(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirCuteNVVMTargetAttrGetChip(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirCuteNVVMTargetAttrGetFeatures(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTargetAttrGetFlags(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTargetAttrGetLink(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool +mlirCuteNVVMTargetAttrGetVerifyTarget(MlirAttribute attr); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_NVVM_CAPI_H diff --git a/mlir/dialects/nvvm.zig b/mlir/dialects/nvvm/nvvm.zig similarity index 84% rename from mlir/dialects/nvvm.zig rename to mlir/dialects/nvvm/nvvm.zig index 3a986dd4..701b9de6 100644 --- a/mlir/dialects/nvvm.zig +++ b/mlir/dialects/nvvm/nvvm.zig @@ -1,14 +1,17 @@ //! Builders for the NVVM operations CuTe kernels emit, over the bindings of -//! NVIDIA's `nvvm` dialect (`mlir/dialects/cute_ir`, `cute.nvvm`): the CuTe-DSL +//! NVIDIA's `nvvm` dialect (`mlir/dialects/nvvm`, `ops`): the CuTe-DSL //! compiler's own NVVM, which the upstream dialect differs from. Each builder //! fixes the attributes the kernels do not choose. const std = @import("std"); -const cute = @import("mlir/dialects/cute_ir"); const mlir = @import("mlir"); -pub const ops = cute.nvvm; +pub const ops = @import("ops.zig"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("cute_nvvm"); +} fn enumAttribute(comptime T: type, ctx: *mlir.Context, value: @FieldType(T.InitArgs, "value")) *const mlir.Attribute { return (T.get(ctx, .{ .value = value }) catch unreachable).attribute(); @@ -47,12 +50,12 @@ pub fn read_sreg(ctx: *mlir.Context, comptime name: []const u8, location: *const std.mem.replaceScalar(u8, &fn_name, '.', '_'); break :blk "read_ptx_sreg_" ++ fn_name; }; - return @field(ops, builder)(ctx, .int(ctx, .i32), null, location); + return @field(ops, builder)(ctx, .int(ctx, .i32), .{}, location); } /// nvvm.barrier — `bar.sync 0` over the whole CTA. pub fn barrier(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return ops.barrier(ctx, null, null, null, null, null, location); + return ops.barrier(ctx, .{}, location); } /// nvvm.bar.warp.sync — `bar.warp.sync mask` (i32 lane mask, -1 for the full warp). @@ -62,7 +65,7 @@ pub fn bar_warp_sync(ctx: *mlir.Context, mask: *const mlir.Value, location: *con /// nvvm.elect.sync — i1, true in one elected lane of the warp. pub fn elect_sync(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return ops.elect_sync(ctx, null, .int(ctx, .i1), location); + return ops.elect_sync(ctx, .int(ctx, .i1), .{}, location); } // ============================================================================= @@ -72,18 +75,29 @@ pub fn elect_sync(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Oper /// nvvm.mbarrier.init — initialize the mbarrier at `ptr` (`!llvm.ptr<3>`) to expect /// `count` (i32) arrivals per phase. pub fn mbarrier_init(ctx: *mlir.Context, ptr: *const mlir.Value, count: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return ops.mbarrier_init(ctx, ptr, count, null, enumAttribute(ops.MBarrierLayoutAttr, ctx, .v0), location); + return ops.mbarrier_init(ctx, ptr, count, .{ .layout = enumAttribute(ops.MBarrierLayoutAttr, ctx, .v0) }, location); } /// nvvm.mbarrier.arrive — arrive `count` (i32) times (release semantics). pub fn mbarrier_arrive(ctx: *mlir.Context, ptr: *const mlir.Value, count: *const mlir.Value, scope: MemScope, location: *const mlir.Location) *mlir.Operation { - return ops.mbarrier_arrive(ctx, ptr, count, null, null, enumAttribute(ops.MemScopeKindAttr, ctx, scope), .boolean(ctx, false), location); + return ops.mbarrier_arrive( + ctx, + ptr, + .{ .count = count, .scope = enumAttribute(ops.MemScopeKindAttr, ctx, scope), .relaxed = .boolean(ctx, false) }, + location, + ); } /// nvvm.mbarrier.arrive.expect_tx — arrive and expect `bytes` (i32) of asynchronous /// transactions (SM90+). pub fn mbarrier_arrive_expect_tx(ctx: *mlir.Context, ptr: *const mlir.Value, bytes: *const mlir.Value, scope: MemScope, location: *const mlir.Location) *mlir.Operation { - return ops.mbarrier_arrive_expect_tx(ctx, ptr, bytes, null, null, null, enumAttribute(ops.MemScopeKindAttr, ctx, scope), .boolean(ctx, false), location); + return ops.mbarrier_arrive_expect_tx( + ctx, + ptr, + bytes, + .{ .scope = enumAttribute(ops.MemScopeKindAttr, ctx, scope), .relaxed = .boolean(ctx, false) }, + location, + ); } /// nvvm.mbarrier.wait.parity — i1, whether phase `parity` (i32) has completed @@ -102,8 +116,7 @@ pub fn mbarrier_wait_parity( parity, .int(ctx, .i1), enumAttribute(ops.MBarrierWaitKindAttr, ctx, kind), - enumAttribute(ops.MBarrierScopeKindAttr, ctx, scope), - null, + .{ .scope = enumAttribute(ops.MBarrierScopeKindAttr, ctx, scope) }, location, ); } @@ -117,7 +130,7 @@ pub fn mbarrier_try_wait_parity( suspend_time: *const mlir.Value, location: *const mlir.Location, ) *mlir.Operation { - return ops.mbarrier_try_wait_parity(ctx, ptr, parity, suspend_time, .boolean(ctx, false), location); + return ops.mbarrier_try_wait_parity(ctx, ptr, parity, suspend_time, .{ .useIntrinsic = .boolean(ctx, false) }, location); } /// nvvm.fence.mbarrier.init — make mbarrier initialization visible to the cluster. @@ -127,7 +140,7 @@ pub fn fence_mbarrier_init(ctx: *mlir.Context, location: *const mlir.Location) * /// nvvm.fence.proxy — order memory accesses across proxies. pub fn fence_proxy(ctx: *mlir.Context, kind: ProxyKind, space: SharedSpace, location: *const mlir.Location) *mlir.Operation { - return ops.fence_proxy(ctx, enumAttribute(ops.ProxyKindAttr, ctx, kind), enumAttribute(ops.SharedSpaceAttr, ctx, space), location); + return ops.fence_proxy(ctx, enumAttribute(ops.ProxyKindAttr, ctx, kind), .{ .space = enumAttribute(ops.SharedSpaceAttr, ctx, space) }, location); } // ============================================================================= @@ -152,7 +165,7 @@ pub fn cp_async_bulk_commit_group(ctx: *mlir.Context, location: *const mlir.Loca /// nvvm.cp.async.bulk.wait_group — wait until at most `pending` bulk groups are in /// flight; with `read`, only until their sources have been read. pub fn cp_async_bulk_wait_group(ctx: *mlir.Context, pending: u32, read: bool, location: *const mlir.Location) *mlir.Operation { - return ops.cp_async_bulk_wait_group(ctx, .int(ctx, .i32, pending), if (read) .unit(ctx) else null, location); + return ops.cp_async_bulk_wait_group(ctx, .int(ctx, .i32, pending), .{ .read = if (read) .unit(ctx) else null }, location); } // ============================================================================= @@ -162,13 +175,13 @@ pub fn cp_async_bulk_wait_group(ctx: *mlir.Context, pending: u32, read: bool, lo /// nvvm.tcgen05.alloc — allocate `columns` (i32) tensor-memory columns and write the /// address to `holder` (`!llvm.ptr<3>`). pub fn tcgen05_alloc(ctx: *mlir.Context, holder: *const mlir.Value, columns: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return ops.tcgen05_alloc(ctx, holder, columns, null, null, location); + return ops.tcgen05_alloc(ctx, holder, columns, .{}, location); } /// nvvm.tcgen05.dealloc — free `columns` (i32) tensor-memory columns at `taddr` /// (`!llvm.ptr<6>`). pub fn tcgen05_dealloc(ctx: *mlir.Context, taddr: *const mlir.Value, columns: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return ops.tcgen05_dealloc(ctx, taddr, columns, null, null, location); + return ops.tcgen05_dealloc(ctx, taddr, columns, .{}, location); } /// nvvm.tcgen05.fence — order tcgen05 operations around a thread synchronization. @@ -184,7 +197,7 @@ pub fn tcgen05_wait(ctx: *mlir.Context, kind: Tcgen05WaitKind, location: *const /// nvvm.tcgen05.commit — arrive on the mbarrier at `barrier_ptr` when the issued /// tcgen05 operations complete. pub fn tcgen05_commit(ctx: *mlir.Context, barrier_ptr: *const mlir.Value, group: CtaGroup, location: *const mlir.Location) *mlir.Operation { - return ops.tcgen05_commit(ctx, barrier_ptr, null, null, enumAttribute(ops.CTAGroupKindAttr, ctx, group), location); + return ops.tcgen05_commit(ctx, barrier_ptr, .{ .group = enumAttribute(ops.CTAGroupKindAttr, ctx, group) }, location); } /// nvvm.tcgen05.mma — D (tensor memory, `!llvm.ptr<6>`) = A (shared-memory @@ -208,13 +221,9 @@ pub fn tcgen05_mma( b_desc, idesc, accumulate, - null, - null, enumAttribute(ops.Tcgen05MMAKindAttr, ctx, kind), enumAttribute(ops.CTAGroupKindAttr, ctx, cta_group), - null, - null, - null, + .{}, location, ); } @@ -222,12 +231,12 @@ pub fn tcgen05_mma( /// nvvm.tcgen05.ld — load `result_type` (an i32 or vector of i32) from tensor memory /// at `taddr` (`!llvm.ptr<6>`). pub fn tcgen05_ld(ctx: *mlir.Context, taddr: *const mlir.Value, result_type: *const mlir.Type, shape: Tcgen05LdStShape, location: *const mlir.Location) *mlir.Operation { - return ops.tcgen05_ld(ctx, taddr, null, result_type, null, enumAttribute(ops.Tcgen05LdStShapeAttr, ctx, shape), location); + return ops.tcgen05_ld(ctx, taddr, result_type, enumAttribute(ops.Tcgen05LdStShapeAttr, ctx, shape), .{}, location); } /// nvvm.tcgen05.st — store `value` (an i32 or vector of i32) to tensor memory at `taddr`. pub fn tcgen05_st(ctx: *mlir.Context, taddr: *const mlir.Value, value: *const mlir.Value, shape: Tcgen05LdStShape, location: *const mlir.Location) *mlir.Operation { - return ops.tcgen05_st(ctx, taddr, value, null, null, enumAttribute(ops.Tcgen05LdStShapeAttr, ctx, shape), location); + return ops.tcgen05_st(ctx, taddr, value, enumAttribute(ops.Tcgen05LdStShapeAttr, ctx, shape), .{}, location); } // ============================================================================= @@ -241,12 +250,12 @@ pub fn mapa(ctx: *mlir.Context, ptr: *const mlir.Value, rank: *const mlir.Value, /// nvvm.cluster.arrive — arrive on the cluster barrier. pub fn cluster_arrive(ctx: *mlir.Context, aligned: bool, location: *const mlir.Location) *mlir.Operation { - return ops.cluster_arrive(ctx, if (aligned) .unit(ctx) else null, location); + return ops.cluster_arrive(ctx, .{ .aligned = if (aligned) .unit(ctx) else null }, location); } /// nvvm.cluster.wait — wait on the cluster barrier. pub fn cluster_wait(ctx: *mlir.Context, aligned: bool, location: *const mlir.Location) *mlir.Operation { - return ops.cluster_wait(ctx, if (aligned) .unit(ctx) else null, location); + return ops.cluster_wait(ctx, .{ .aligned = if (aligned) .unit(ctx) else null }, location); } // ============================================================================= @@ -255,14 +264,10 @@ pub fn cluster_wait(ctx: *mlir.Context, aligned: bool, location: *const mlir.Loc /// nvvm.ex2 — `ex2.approx[.ftz].f32`. pub fn ex2(ctx: *mlir.Context, value: *const mlir.Value, ftz: bool, location: *const mlir.Location) *mlir.Operation { - return ops.ex2(ctx, value, .float(ctx, .f32), .boolean(ctx, ftz), location); + return ops.ex2(ctx, value, .float(ctx, .f32), .{ .ftz = .boolean(ctx, ftz) }, location); } /// nvvm.rcp.approx.ftz.f — `rcp.approx.ftz.f32`. pub fn rcp_approx_ftz(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { return ops.rcp_approx_ftz_f(ctx, value, .float(ctx, .f32), location); } - -test { - std.testing.refAllDecls(@This()); -} diff --git a/mlir/dialects/cute_ir/nvvm.zig b/mlir/dialects/nvvm/ops.zig similarity index 65% rename from mlir/dialects/cute_ir/nvvm.zig rename to mlir/dialects/nvvm/ops.zig index bfcd16b5..88c1c514 100644 --- a/mlir/dialects/cute_ir/nvvm.zig +++ b/mlir/dialects/nvvm/ops.zig @@ -1,13 +1,10 @@ //! Zig bindings of NVIDIA's NVVM dialect of the CuTe DSL (nvidia-cutlass-dsl 4.8.0), generated //! from the dialect's .td files; do not edit. -const std = @import("std"); - const c = @import("c"); const mlir = @import("mlir"); const stdx = @import("stdx"); -pub const dialect_namespace = "nvvm"; /// The C dialect handle, `mlirGetDialectHandle__cute_nvvm__`. pub const dialect_handle = "cute_nvvm"; @@ -823,11 +820,9 @@ pub const AtomicOpKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMAtomicOpKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "atomic_op"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -838,7 +833,7 @@ pub const AtomicOpKindAttr = opaque { const result = c.mlirCuteNVVMAtomicOpKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) AtomicOpKind { + pub fn value(self: *const Self) AtomicOpKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMAtomicOpKindAttrGetValue(self.ptr()))); } }; @@ -850,11 +845,9 @@ pub const BarrierReductionAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMBarrierReduction; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "reduction"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -865,7 +858,7 @@ pub const BarrierReductionAttr = opaque { const result = c.mlirCuteNVVMBarrierReductionAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) BarrierReduction { + pub fn value(self: *const Self) BarrierReduction { return @fromBackingInt(@intCast(c.mlirCuteNVVMBarrierReductionAttrGetValue(self.ptr()))); } }; @@ -877,11 +870,9 @@ pub const BarrierReduxKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMBarrierReduxKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "barrier_redux_kind"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -892,7 +883,7 @@ pub const BarrierReduxKindAttr = opaque { const result = c.mlirCuteNVVMBarrierReduxKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) BarrierReduxKind { + pub fn value(self: *const Self) BarrierReduxKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMBarrierReduxKindAttrGetValue(self.ptr()))); } }; @@ -904,11 +895,9 @@ pub const BlockScaleFormatAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMBlockScaleFormat; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "block_scale_format"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -919,7 +908,7 @@ pub const BlockScaleFormatAttr = opaque { const result = c.mlirCuteNVVMBlockScaleFormatAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) BlockScaleFormat { + pub fn value(self: *const Self) BlockScaleFormat { return @fromBackingInt(@intCast(c.mlirCuteNVVMBlockScaleFormatAttrGetValue(self.ptr()))); } }; @@ -931,11 +920,9 @@ pub const CTAGroupKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMCTAGroupKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "cta_group"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -946,7 +933,7 @@ pub const CTAGroupKindAttr = opaque { const result = c.mlirCuteNVVMCTAGroupKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) CTAGroupKind { + pub fn value(self: *const Self) CTAGroupKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMCTAGroupKindAttrGetValue(self.ptr()))); } }; @@ -958,11 +945,9 @@ pub const CVTPackFloatKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMCVTPackFloatKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "packfloat_type"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -973,7 +958,7 @@ pub const CVTPackFloatKindAttr = opaque { const result = c.mlirCuteNVVMCVTPackFloatKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) CVTPackFloatKind { + pub fn value(self: *const Self) CVTPackFloatKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMCVTPackFloatKindAttrGetValue(self.ptr()))); } }; @@ -985,11 +970,9 @@ pub const CacheEvictionPriorityAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMCacheEvictionPriority; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "cache_eviction_priority"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1000,7 +983,7 @@ pub const CacheEvictionPriorityAttr = opaque { const result = c.mlirCuteNVVMCacheEvictionPriorityAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) CacheEvictionPriority { + pub fn value(self: *const Self) CacheEvictionPriority { return @fromBackingInt(@intCast(c.mlirCuteNVVMCacheEvictionPriorityAttrGetValue(self.ptr()))); } }; @@ -1012,11 +995,9 @@ pub const ClusterLaunchControlQueryTypeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMClusterLaunchControlQueryType; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "cluster_launch_control_query_type"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1027,7 +1008,7 @@ pub const ClusterLaunchControlQueryTypeAttr = opaque { const result = c.mlirCuteNVVMClusterLaunchControlQueryTypeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) ClusterLaunchControlQueryType { + pub fn value(self: *const Self) ClusterLaunchControlQueryType { return @fromBackingInt(@intCast(c.mlirCuteNVVMClusterLaunchControlQueryTypeAttrGetValue(self.ptr()))); } }; @@ -1039,11 +1020,9 @@ pub const CompareOpKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMCompareOpKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "op"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1054,7 +1033,7 @@ pub const CompareOpKindAttr = opaque { const result = c.mlirCuteNVVMCompareOpKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) CompareOpKind { + pub fn value(self: *const Self) CompareOpKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMCompareOpKindAttrGetValue(self.ptr()))); } }; @@ -1066,11 +1045,9 @@ pub const ConvertFP4TypeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMConvertFP4Type; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "convert_fp4_type"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1081,7 +1058,7 @@ pub const ConvertFP4TypeAttr = opaque { const result = c.mlirCuteNVVMConvertFP4TypeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) ConvertFP4Type { + pub fn value(self: *const Self) ConvertFP4Type { return @fromBackingInt(@intCast(c.mlirCuteNVVMConvertFP4TypeAttrGetValue(self.ptr()))); } }; @@ -1093,11 +1070,9 @@ pub const ConvertFP8TypeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMConvertFP8Type; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "convert_fp8_type"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1108,7 +1083,7 @@ pub const ConvertFP8TypeAttr = opaque { const result = c.mlirCuteNVVMConvertFP8TypeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) ConvertFP8Type { + pub fn value(self: *const Self) ConvertFP8Type { return @fromBackingInt(@intCast(c.mlirCuteNVVMConvertFP8TypeAttrGetValue(self.ptr()))); } }; @@ -1120,11 +1095,9 @@ pub const ConvertScaleKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMConvertScaleKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "convert_scale_kind"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1135,7 +1108,7 @@ pub const ConvertScaleKindAttr = opaque { const result = c.mlirCuteNVVMConvertScaleKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) ConvertScaleKind { + pub fn value(self: *const Self) ConvertScaleKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMConvertScaleKindAttrGetValue(self.ptr()))); } }; @@ -1147,11 +1120,9 @@ pub const DotAccumulateTypeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMDotAccumulateType; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "dot_accumulate_type"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1162,7 +1133,7 @@ pub const DotAccumulateTypeAttr = opaque { const result = c.mlirCuteNVVMDotAccumulateTypeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) DotAccumulateType { + pub fn value(self: *const Self) DotAccumulateType { return @fromBackingInt(@intCast(c.mlirCuteNVVMDotAccumulateTypeAttrGetValue(self.ptr()))); } }; @@ -1174,11 +1145,9 @@ pub const EvictKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMEvictKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "evict_kind"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1189,7 +1158,7 @@ pub const EvictKindAttr = opaque { const result = c.mlirCuteNVVMEvictKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) EvictKind { + pub fn value(self: *const Self) EvictKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMEvictKindAttrGetValue(self.ptr()))); } }; @@ -1201,11 +1170,9 @@ pub const FPRoundingModeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMFPRoundingMode; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "fp_rnd_mode"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1216,7 +1183,7 @@ pub const FPRoundingModeAttr = opaque { const result = c.mlirCuteNVVMFPRoundingModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) FPRoundingMode { + pub fn value(self: *const Self) FPRoundingMode { return @fromBackingInt(@intCast(c.mlirCuteNVVMFPRoundingModeAttrGetValue(self.ptr()))); } }; @@ -1228,11 +1195,9 @@ pub const GridDepActionKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMGridDepActionKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "grid_dep_action"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1243,7 +1208,7 @@ pub const GridDepActionKindAttr = opaque { const result = c.mlirCuteNVVMGridDepActionKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) GridDepActionKind { + pub fn value(self: *const Self) GridDepActionKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMGridDepActionKindAttrGetValue(self.ptr()))); } }; @@ -1255,11 +1220,9 @@ pub const IntegerRoundingModeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMIntegerRoundingMode; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "int_rnd_mode"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1270,7 +1233,7 @@ pub const IntegerRoundingModeAttr = opaque { const result = c.mlirCuteNVVMIntegerRoundingModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) IntegerRoundingMode { + pub fn value(self: *const Self) IntegerRoundingMode { return @fromBackingInt(@intCast(c.mlirCuteNVVMIntegerRoundingModeAttrGetValue(self.ptr()))); } }; @@ -1282,11 +1245,9 @@ pub const L2PrefetchSizeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVML2PrefetchSize; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "l2_prefetch"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1297,7 +1258,7 @@ pub const L2PrefetchSizeAttr = opaque { const result = c.mlirCuteNVVML2PrefetchSizeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) L2PrefetchSize { + pub fn value(self: *const Self) L2PrefetchSize { return @fromBackingInt(@intCast(c.mlirCuteNVVML2PrefetchSizeAttrGetValue(self.ptr()))); } }; @@ -1309,11 +1270,9 @@ pub const LdStMatrixEltTypeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMLdStMatrixEltType; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "ld_st_matrix_elt_type"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1324,7 +1283,7 @@ pub const LdStMatrixEltTypeAttr = opaque { const result = c.mlirCuteNVVMLdStMatrixEltTypeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) LdStMatrixEltType { + pub fn value(self: *const Self) LdStMatrixEltType { return @fromBackingInt(@intCast(c.mlirCuteNVVMLdStMatrixEltTypeAttrGetValue(self.ptr()))); } }; @@ -1336,11 +1295,9 @@ pub const LoadCacheModifierExtKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMLoadCacheModifierExtKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "load_cache_modifier_ext"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1351,7 +1308,7 @@ pub const LoadCacheModifierExtKindAttr = opaque { const result = c.mlirCuteNVVMLoadCacheModifierExtKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) LoadCacheModifierExtKind { + pub fn value(self: *const Self) LoadCacheModifierExtKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMLoadCacheModifierExtKindAttrGetValue(self.ptr()))); } }; @@ -1363,11 +1320,9 @@ pub const LoadCacheModifierKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMLoadCacheModifierKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "load_cache_modifier"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1378,7 +1333,7 @@ pub const LoadCacheModifierKindAttr = opaque { const result = c.mlirCuteNVVMLoadCacheModifierKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) LoadCacheModifierKind { + pub fn value(self: *const Self) LoadCacheModifierKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMLoadCacheModifierKindAttrGetValue(self.ptr()))); } }; @@ -1390,11 +1345,9 @@ pub const LoadShapeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMLoadShape; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "load_shape"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1405,7 +1358,7 @@ pub const LoadShapeAttr = opaque { const result = c.mlirCuteNVVMLoadShapeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) LoadShape { + pub fn value(self: *const Self) LoadShape { return @fromBackingInt(@intCast(c.mlirCuteNVVMLoadShapeAttrGetValue(self.ptr()))); } }; @@ -1417,11 +1370,9 @@ pub const LoadSrcFormatAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMLoadSrcFormat; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "load_src_format"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1432,7 +1383,7 @@ pub const LoadSrcFormatAttr = opaque { const result = c.mlirCuteNVVMLoadSrcFormatAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) LoadSrcFormat { + pub fn value(self: *const Self) LoadSrcFormat { return @fromBackingInt(@intCast(c.mlirCuteNVVMLoadSrcFormatAttrGetValue(self.ptr()))); } }; @@ -1444,11 +1395,9 @@ pub const MBarrierLayoutAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierLayout; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "mbarrier_layout"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1459,7 +1408,7 @@ pub const MBarrierLayoutAttr = opaque { const result = c.mlirCuteNVVMMBarrierLayoutAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) MBarrierLayout { + pub fn value(self: *const Self) MBarrierLayout { return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierLayoutAttrGetValue(self.ptr()))); } }; @@ -1471,11 +1420,9 @@ pub const MBarrierPhaseAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierPhase; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "mbarrier_phase"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1486,7 +1433,7 @@ pub const MBarrierPhaseAttr = opaque { const result = c.mlirCuteNVVMMBarrierPhaseAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) MBarrierPhase { + pub fn value(self: *const Self) MBarrierPhase { return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierPhaseAttrGetValue(self.ptr()))); } }; @@ -1498,11 +1445,9 @@ pub const MBarrierScopeKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierScopeKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "mbar_scope"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1513,7 +1458,7 @@ pub const MBarrierScopeKindAttr = opaque { const result = c.mlirCuteNVVMMBarrierScopeKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) MBarrierScopeKind { + pub fn value(self: *const Self) MBarrierScopeKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierScopeKindAttrGetValue(self.ptr()))); } }; @@ -1525,11 +1470,9 @@ pub const MBarrierSpaceKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierSpaceKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "mbar_space"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1540,7 +1483,7 @@ pub const MBarrierSpaceKindAttr = opaque { const result = c.mlirCuteNVVMMBarrierSpaceKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) MBarrierSpaceKind { + pub fn value(self: *const Self) MBarrierSpaceKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierSpaceKindAttrGetValue(self.ptr()))); } }; @@ -1552,11 +1495,9 @@ pub const MBarrierTxnKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierTxnKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "mbar_txn_kind"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1567,7 +1508,7 @@ pub const MBarrierTxnKindAttr = opaque { const result = c.mlirCuteNVVMMBarrierTxnKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) MBarrierTxnKind { + pub fn value(self: *const Self) MBarrierTxnKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierTxnKindAttrGetValue(self.ptr()))); } }; @@ -1579,11 +1520,9 @@ pub const MBarrierWaitKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierWaitKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "mbar_wait"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1594,7 +1533,7 @@ pub const MBarrierWaitKindAttr = opaque { const result = c.mlirCuteNVVMMBarrierWaitKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) MBarrierWaitKind { + pub fn value(self: *const Self) MBarrierWaitKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierWaitKindAttrGetValue(self.ptr()))); } }; @@ -1606,11 +1545,9 @@ pub const MMAB1OpAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMMMAB1Op; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "mma_b1op"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1621,7 +1558,7 @@ pub const MMAB1OpAttr = opaque { const result = c.mlirCuteNVVMMMAB1OpAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) MMAB1Op { + pub fn value(self: *const Self) MMAB1Op { return @fromBackingInt(@intCast(c.mlirCuteNVVMMMAB1OpAttrGetValue(self.ptr()))); } }; @@ -1633,11 +1570,9 @@ pub const MMABlockScaleKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMMMABlockScaleKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "block_scale_kind"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1648,7 +1583,7 @@ pub const MMABlockScaleKindAttr = opaque { const result = c.mlirCuteNVVMMMABlockScaleKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) MMABlockScaleKind { + pub fn value(self: *const Self) MMABlockScaleKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMMMABlockScaleKindAttrGetValue(self.ptr()))); } }; @@ -1660,11 +1595,9 @@ pub const MMACtaCountAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMMMACtaCount; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "mma_cta_count"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1675,7 +1608,7 @@ pub const MMACtaCountAttr = opaque { const result = c.mlirCuteNVVMMMACtaCountAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) MMACtaCount { + pub fn value(self: *const Self) MMACtaCount { return @fromBackingInt(@intCast(c.mlirCuteNVVMMMACtaCountAttrGetValue(self.ptr()))); } }; @@ -1687,11 +1620,9 @@ pub const MMAFragAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMMMAFrag; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "mma_frag"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1702,7 +1633,7 @@ pub const MMAFragAttr = opaque { const result = c.mlirCuteNVVMMMAFragAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) MMAFrag { + pub fn value(self: *const Self) MMAFrag { return @fromBackingInt(@intCast(c.mlirCuteNVVMMMAFragAttrGetValue(self.ptr()))); } }; @@ -1714,11 +1645,9 @@ pub const MMAIntOverflowAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMMMAIntOverflow; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "mma_int_overflow"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1729,7 +1658,7 @@ pub const MMAIntOverflowAttr = opaque { const result = c.mlirCuteNVVMMMAIntOverflowAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) MMAIntOverflow { + pub fn value(self: *const Self) MMAIntOverflow { return @fromBackingInt(@intCast(c.mlirCuteNVVMMMAIntOverflowAttrGetValue(self.ptr()))); } }; @@ -1741,11 +1670,9 @@ pub const MMAKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMMMAKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "mma_kind"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1756,7 +1683,7 @@ pub const MMAKindAttr = opaque { const result = c.mlirCuteNVVMMMAKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) MMAKind { + pub fn value(self: *const Self) MMAKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMMMAKindAttrGetValue(self.ptr()))); } }; @@ -1768,11 +1695,9 @@ pub const MMALayoutAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMMMALayout; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "mma_layout"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1783,7 +1708,7 @@ pub const MMALayoutAttr = opaque { const result = c.mlirCuteNVVMMMALayoutAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) MMALayout { + pub fn value(self: *const Self) MMALayout { return @fromBackingInt(@intCast(c.mlirCuteNVVMMMALayoutAttrGetValue(self.ptr()))); } }; @@ -1795,11 +1720,9 @@ pub const MMATypesAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMMMATypes; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "mma_type"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1810,7 +1733,7 @@ pub const MMATypesAttr = opaque { const result = c.mlirCuteNVVMMMATypesAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) MMATypes { + pub fn value(self: *const Self) MMATypes { return @fromBackingInt(@intCast(c.mlirCuteNVVMMMATypesAttrGetValue(self.ptr()))); } }; @@ -1822,11 +1745,9 @@ pub const MatchSyncKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMMatchSyncKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "match_sync_kind"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1837,7 +1758,7 @@ pub const MatchSyncKindAttr = opaque { const result = c.mlirCuteNVVMMatchSyncKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) MatchSyncKind { + pub fn value(self: *const Self) MatchSyncKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMMatchSyncKindAttrGetValue(self.ptr()))); } }; @@ -1849,11 +1770,9 @@ pub const MemOrderKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMMemOrderKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "mem_order"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1864,7 +1783,7 @@ pub const MemOrderKindAttr = opaque { const result = c.mlirCuteNVVMMemOrderKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) MemOrderKind { + pub fn value(self: *const Self) MemOrderKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMMemOrderKindAttrGetValue(self.ptr()))); } }; @@ -1876,11 +1795,9 @@ pub const MemScopeKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMMemScopeKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "mem_scope"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1891,7 +1808,7 @@ pub const MemScopeKindAttr = opaque { const result = c.mlirCuteNVVMMemScopeKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) MemScopeKind { + pub fn value(self: *const Self) MemScopeKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMMemScopeKindAttrGetValue(self.ptr()))); } }; @@ -1903,11 +1820,9 @@ pub const MulModeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMMulMode; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "mul_mode"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1918,7 +1833,7 @@ pub const MulModeAttr = opaque { const result = c.mlirCuteNVVMMulModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) MulMode { + pub fn value(self: *const Self) MulMode { return @fromBackingInt(@intCast(c.mlirCuteNVVMMulModeAttrGetValue(self.ptr()))); } }; @@ -1930,11 +1845,9 @@ pub const NVVMMemorySpaceAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMNVVMMemorySpace; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "memory_space"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1945,7 +1858,7 @@ pub const NVVMMemorySpaceAttr = opaque { const result = c.mlirCuteNVVMNVVMMemorySpaceAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) NVVMMemorySpace { + pub fn value(self: *const Self) NVVMMemorySpace { return @fromBackingInt(@intCast(c.mlirCuteNVVMNVVMMemorySpaceAttrGetValue(self.ptr()))); } }; @@ -1957,11 +1870,9 @@ pub const PermuteModeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMPermuteMode; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "permute_mode"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1972,7 +1883,7 @@ pub const PermuteModeAttr = opaque { const result = c.mlirCuteNVVMPermuteModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) PermuteMode { + pub fn value(self: *const Self) PermuteMode { return @fromBackingInt(@intCast(c.mlirCuteNVVMPermuteModeAttrGetValue(self.ptr()))); } }; @@ -1984,11 +1895,9 @@ pub const PrefetchCacheLevelAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMPrefetchCacheLevel; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "prefetch_cache_level"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -1999,7 +1908,7 @@ pub const PrefetchCacheLevelAttr = opaque { const result = c.mlirCuteNVVMPrefetchCacheLevelAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) PrefetchCacheLevel { + pub fn value(self: *const Self) PrefetchCacheLevel { return @fromBackingInt(@intCast(c.mlirCuteNVVMPrefetchCacheLevelAttrGetValue(self.ptr()))); } }; @@ -2011,11 +1920,9 @@ pub const ProxyKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMProxyKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "proxy_kind"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2026,7 +1933,7 @@ pub const ProxyKindAttr = opaque { const result = c.mlirCuteNVVMProxyKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) ProxyKind { + pub fn value(self: *const Self) ProxyKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMProxyKindAttrGetValue(self.ptr()))); } }; @@ -2038,11 +1945,9 @@ pub const ReductionKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMReductionKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "reduction_kind"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2053,7 +1958,7 @@ pub const ReductionKindAttr = opaque { const result = c.mlirCuteNVVMReductionKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) ReductionKind { + pub fn value(self: *const Self) ReductionKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMReductionKindAttrGetValue(self.ptr()))); } }; @@ -2065,11 +1970,9 @@ pub const ReductionOpAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMReductionOp; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "red_op"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2080,7 +1983,7 @@ pub const ReductionOpAttr = opaque { const result = c.mlirCuteNVVMReductionOpAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) ReductionOp { + pub fn value(self: *const Self) ReductionOp { return @fromBackingInt(@intCast(c.mlirCuteNVVMReductionOpAttrGetValue(self.ptr()))); } }; @@ -2092,11 +1995,9 @@ pub const ReductionTypeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMReductionType; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "red_type"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2107,7 +2008,7 @@ pub const ReductionTypeAttr = opaque { const result = c.mlirCuteNVVMReductionTypeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) ReductionType { + pub fn value(self: *const Self) ReductionType { return @fromBackingInt(@intCast(c.mlirCuteNVVMReductionTypeAttrGetValue(self.ptr()))); } }; @@ -2119,11 +2020,9 @@ pub const SPCompressElemSizeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMSPCompressElemSize; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "spcomp_elem_size"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2134,7 +2033,7 @@ pub const SPCompressElemSizeAttr = opaque { const result = c.mlirCuteNVVMSPCompressElemSizeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) SPCompressElemSize { + pub fn value(self: *const Self) SPCompressElemSize { return @fromBackingInt(@intCast(c.mlirCuteNVVMSPCompressElemSizeAttrGetValue(self.ptr()))); } }; @@ -2146,11 +2045,9 @@ pub const SPCompressFactorTypeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMSPCompressFactorType; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "spcomp_factor"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2161,7 +2058,7 @@ pub const SPCompressFactorTypeAttr = opaque { const result = c.mlirCuteNVVMSPCompressFactorTypeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) SPCompressFactorType { + pub fn value(self: *const Self) SPCompressFactorType { return @fromBackingInt(@intCast(c.mlirCuteNVVMSPCompressFactorTypeAttrGetValue(self.ptr()))); } }; @@ -2173,11 +2070,9 @@ pub const SPCompressIndexSizeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMSPCompressIndexSize; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "spcomp_index_size"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2188,7 +2083,7 @@ pub const SPCompressIndexSizeAttr = opaque { const result = c.mlirCuteNVVMSPCompressIndexSizeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) SPCompressIndexSize { + pub fn value(self: *const Self) SPCompressIndexSize { return @fromBackingInt(@intCast(c.mlirCuteNVVMSPCompressIndexSizeAttrGetValue(self.ptr()))); } }; @@ -2200,11 +2095,9 @@ pub const SPCompressOpKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMSPCompressOpKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "spcomp_op_kind"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2215,7 +2108,7 @@ pub const SPCompressOpKindAttr = opaque { const result = c.mlirCuteNVVMSPCompressOpKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) SPCompressOpKind { + pub fn value(self: *const Self) SPCompressOpKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMSPCompressOpKindAttrGetValue(self.ptr()))); } }; @@ -2227,11 +2120,9 @@ pub const SPCompressRepFactorAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMSPCompressRepFactor; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "spcomp_rep_factor"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2242,7 +2133,7 @@ pub const SPCompressRepFactorAttr = opaque { const result = c.mlirCuteNVVMSPCompressRepFactorAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) SPCompressRepFactor { + pub fn value(self: *const Self) SPCompressRepFactor { return @fromBackingInt(@intCast(c.mlirCuteNVVMSPCompressRepFactorAttrGetValue(self.ptr()))); } }; @@ -2254,11 +2145,9 @@ pub const SPDecompressElemSizeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMSPDecompressElemSize; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "spdecomp_elem_size"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2269,7 +2158,7 @@ pub const SPDecompressElemSizeAttr = opaque { const result = c.mlirCuteNVVMSPDecompressElemSizeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) SPDecompressElemSize { + pub fn value(self: *const Self) SPDecompressElemSize { return @fromBackingInt(@intCast(c.mlirCuteNVVMSPDecompressElemSizeAttrGetValue(self.ptr()))); } }; @@ -2281,11 +2170,9 @@ pub const SPDecompressFactorTypeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMSPDecompressFactorType; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "spdecomp_factor"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2296,7 +2183,7 @@ pub const SPDecompressFactorTypeAttr = opaque { const result = c.mlirCuteNVVMSPDecompressFactorTypeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) SPDecompressFactorType { + pub fn value(self: *const Self) SPDecompressFactorType { return @fromBackingInt(@intCast(c.mlirCuteNVVMSPDecompressFactorTypeAttrGetValue(self.ptr()))); } }; @@ -2308,11 +2195,9 @@ pub const SPDecompressIndexSizeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMSPDecompressIndexSize; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "spdecomp_index_size"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2323,7 +2208,7 @@ pub const SPDecompressIndexSizeAttr = opaque { const result = c.mlirCuteNVVMSPDecompressIndexSizeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) SPDecompressIndexSize { + pub fn value(self: *const Self) SPDecompressIndexSize { return @fromBackingInt(@intCast(c.mlirCuteNVVMSPDecompressIndexSizeAttrGetValue(self.ptr()))); } }; @@ -2335,11 +2220,9 @@ pub const SPDecompressRepFactorAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMSPDecompressRepFactor; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "spdecomp_rep_factor"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2350,7 +2233,7 @@ pub const SPDecompressRepFactorAttr = opaque { const result = c.mlirCuteNVVMSPDecompressRepFactorAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) SPDecompressRepFactor { + pub fn value(self: *const Self) SPDecompressRepFactor { return @fromBackingInt(@intCast(c.mlirCuteNVVMSPDecompressRepFactorAttrGetValue(self.ptr()))); } }; @@ -2362,11 +2245,9 @@ pub const SaturationModeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMSaturationMode; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "sat_mode"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2377,7 +2258,7 @@ pub const SaturationModeAttr = opaque { const result = c.mlirCuteNVVMSaturationModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) SaturationMode { + pub fn value(self: *const Self) SaturationMode { return @fromBackingInt(@intCast(c.mlirCuteNVVMSaturationModeAttrGetValue(self.ptr()))); } }; @@ -2389,11 +2270,9 @@ pub const SaturationModeKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMSaturationModeKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "sat"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2404,7 +2283,7 @@ pub const SaturationModeKindAttr = opaque { const result = c.mlirCuteNVVMSaturationModeKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) SaturationModeKind { + pub fn value(self: *const Self) SaturationModeKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMSaturationModeKindAttrGetValue(self.ptr()))); } }; @@ -2416,11 +2295,9 @@ pub const ScaleVecSizeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMScaleVecSize; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "scale_vec_size"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2431,7 +2308,7 @@ pub const ScaleVecSizeAttr = opaque { const result = c.mlirCuteNVVMScaleVecSizeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) ScaleVecSize { + pub fn value(self: *const Self) ScaleVecSize { return @fromBackingInt(@intCast(c.mlirCuteNVVMScaleVecSizeAttrGetValue(self.ptr()))); } }; @@ -2443,11 +2320,9 @@ pub const SetMaxRegisterActionAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMSetMaxRegisterAction; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "action"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2458,7 +2333,7 @@ pub const SetMaxRegisterActionAttr = opaque { const result = c.mlirCuteNVVMSetMaxRegisterActionAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) SetMaxRegisterAction { + pub fn value(self: *const Self) SetMaxRegisterAction { return @fromBackingInt(@intCast(c.mlirCuteNVVMSetMaxRegisterActionAttrGetValue(self.ptr()))); } }; @@ -2470,11 +2345,9 @@ pub const SharedSpaceAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMSharedSpace; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "shared_space"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2485,7 +2358,7 @@ pub const SharedSpaceAttr = opaque { const result = c.mlirCuteNVVMSharedSpaceAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) SharedSpace { + pub fn value(self: *const Self) SharedSpace { return @fromBackingInt(@intCast(c.mlirCuteNVVMSharedSpaceAttrGetValue(self.ptr()))); } }; @@ -2497,11 +2370,9 @@ pub const ShflKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMShflKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "shfl_kind"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2512,7 +2383,7 @@ pub const ShflKindAttr = opaque { const result = c.mlirCuteNVVMShflKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) ShflKind { + pub fn value(self: *const Self) ShflKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMShflKindAttrGetValue(self.ptr()))); } }; @@ -2524,11 +2395,9 @@ pub const SparsityFormatAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMSparsityFormat; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "sparsity_format"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2539,7 +2408,7 @@ pub const SparsityFormatAttr = opaque { const result = c.mlirCuteNVVMSparsityFormatAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) SparsityFormat { + pub fn value(self: *const Self) SparsityFormat { return @fromBackingInt(@intCast(c.mlirCuteNVVMSparsityFormatAttrGetValue(self.ptr()))); } }; @@ -2551,11 +2420,9 @@ pub const StateSpaceAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMStateSpace; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "state_space"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2566,7 +2433,7 @@ pub const StateSpaceAttr = opaque { const result = c.mlirCuteNVVMStateSpaceAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) StateSpace { + pub fn value(self: *const Self) StateSpace { return @fromBackingInt(@intCast(c.mlirCuteNVVMStateSpaceAttrGetValue(self.ptr()))); } }; @@ -2578,11 +2445,9 @@ pub const StoreCacheModifierKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMStoreCacheModifierKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "store_cache_modifier"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2593,7 +2458,7 @@ pub const StoreCacheModifierKindAttr = opaque { const result = c.mlirCuteNVVMStoreCacheModifierKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) StoreCacheModifierKind { + pub fn value(self: *const Self) StoreCacheModifierKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMStoreCacheModifierKindAttrGetValue(self.ptr()))); } }; @@ -2605,11 +2470,9 @@ pub const StoreShapeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMStoreShape; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "store_shape"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2620,7 +2483,7 @@ pub const StoreShapeAttr = opaque { const result = c.mlirCuteNVVMStoreShapeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) StoreShape { + pub fn value(self: *const Self) StoreShape { return @fromBackingInt(@intCast(c.mlirCuteNVVMStoreShapeAttrGetValue(self.ptr()))); } }; @@ -2632,11 +2495,9 @@ pub const TCBarParamAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMTCBarParam; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "TCBarParam"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2647,7 +2508,7 @@ pub const TCBarParamAttr = opaque { const result = c.mlirCuteNVVMTCBarParamAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) TCBarParam { + pub fn value(self: *const Self) TCBarParam { return @fromBackingInt(@intCast(c.mlirCuteNVVMTCBarParamAttrGetValue(self.ptr()))); } }; @@ -2659,11 +2520,9 @@ pub const TMALoadModeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMTMALoadMode; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tma_load_mode"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2674,7 +2533,7 @@ pub const TMALoadModeAttr = opaque { const result = c.mlirCuteNVVMTMALoadModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) TMALoadMode { + pub fn value(self: *const Self) TMALoadMode { return @fromBackingInt(@intCast(c.mlirCuteNVVMTMALoadModeAttrGetValue(self.ptr()))); } }; @@ -2686,11 +2545,9 @@ pub const TMAReduxKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMTMAReduxKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tma_redux_kind"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2701,7 +2558,7 @@ pub const TMAReduxKindAttr = opaque { const result = c.mlirCuteNVVMTMAReduxKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) TMAReduxKind { + pub fn value(self: *const Self) TMAReduxKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMTMAReduxKindAttrGetValue(self.ptr()))); } }; @@ -2713,11 +2570,9 @@ pub const TMAStoreModeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMTMAStoreMode; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tma_store_mode"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2728,7 +2583,7 @@ pub const TMAStoreModeAttr = opaque { const result = c.mlirCuteNVVMTMAStoreModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) TMAStoreMode { + pub fn value(self: *const Self) TMAStoreMode { return @fromBackingInt(@intCast(c.mlirCuteNVVMTMAStoreModeAttrGetValue(self.ptr()))); } }; @@ -2740,11 +2595,9 @@ pub const Tcgen05CpMulticastAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05CpMulticast; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tcgen05_cp_multicast"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2755,7 +2608,7 @@ pub const Tcgen05CpMulticastAttr = opaque { const result = c.mlirCuteNVVMTcgen05CpMulticastAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) Tcgen05CpMulticast { + pub fn value(self: *const Self) Tcgen05CpMulticast { return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05CpMulticastAttrGetValue(self.ptr()))); } }; @@ -2767,11 +2620,9 @@ pub const Tcgen05CpShapeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05CpShape; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tcgen05_cp_shape"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2782,7 +2633,7 @@ pub const Tcgen05CpShapeAttr = opaque { const result = c.mlirCuteNVVMTcgen05CpShapeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) Tcgen05CpShape { + pub fn value(self: *const Self) Tcgen05CpShape { return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05CpShapeAttrGetValue(self.ptr()))); } }; @@ -2794,11 +2645,9 @@ pub const Tcgen05CpSrcFormatAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05CpSrcFormat; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tcgen05_cp_src_fmt"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2809,7 +2658,7 @@ pub const Tcgen05CpSrcFormatAttr = opaque { const result = c.mlirCuteNVVMTcgen05CpSrcFormatAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) Tcgen05CpSrcFormat { + pub fn value(self: *const Self) Tcgen05CpSrcFormat { return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05CpSrcFormatAttrGetValue(self.ptr()))); } }; @@ -2821,11 +2670,9 @@ pub const Tcgen05FenceKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05FenceKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tcgen05_fence"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2836,7 +2683,7 @@ pub const Tcgen05FenceKindAttr = opaque { const result = c.mlirCuteNVVMTcgen05FenceKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) Tcgen05FenceKind { + pub fn value(self: *const Self) Tcgen05FenceKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05FenceKindAttrGetValue(self.ptr()))); } }; @@ -2848,11 +2695,9 @@ pub const Tcgen05LdStShapeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05LdStShape; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tcgen05_ldst_shape"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2863,7 +2708,7 @@ pub const Tcgen05LdStShapeAttr = opaque { const result = c.mlirCuteNVVMTcgen05LdStShapeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) Tcgen05LdStShape { + pub fn value(self: *const Self) Tcgen05LdStShape { return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05LdStShapeAttrGetValue(self.ptr()))); } }; @@ -2875,11 +2720,9 @@ pub const Tcgen05MMABlockScaleAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05MMABlockScale; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tcgen05_mma_block_scale"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2890,7 +2733,7 @@ pub const Tcgen05MMABlockScaleAttr = opaque { const result = c.mlirCuteNVVMTcgen05MMABlockScaleAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) Tcgen05MMABlockScale { + pub fn value(self: *const Self) Tcgen05MMABlockScale { return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05MMABlockScaleAttrGetValue(self.ptr()))); } }; @@ -2902,11 +2745,9 @@ pub const Tcgen05MMACollectorBBufferAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05MMACollectorBBuffer; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tcgen05_mma_collectorb"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2917,7 +2758,7 @@ pub const Tcgen05MMACollectorBBufferAttr = opaque { const result = c.mlirCuteNVVMTcgen05MMACollectorBBufferAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) Tcgen05MMACollectorBBuffer { + pub fn value(self: *const Self) Tcgen05MMACollectorBBuffer { return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05MMACollectorBBufferAttrGetValue(self.ptr()))); } }; @@ -2929,11 +2770,9 @@ pub const Tcgen05MMACollectorOpAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05MMACollectorOp; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tcgen05_mma_collectorop"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2944,7 +2783,7 @@ pub const Tcgen05MMACollectorOpAttr = opaque { const result = c.mlirCuteNVVMTcgen05MMACollectorOpAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) Tcgen05MMACollectorOp { + pub fn value(self: *const Self) Tcgen05MMACollectorOp { return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05MMACollectorOpAttrGetValue(self.ptr()))); } }; @@ -2956,11 +2795,9 @@ pub const Tcgen05MMAKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05MMAKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tcgen05_mma_kind"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2971,7 +2808,7 @@ pub const Tcgen05MMAKindAttr = opaque { const result = c.mlirCuteNVVMTcgen05MMAKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) Tcgen05MMAKind { + pub fn value(self: *const Self) Tcgen05MMAKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05MMAKindAttrGetValue(self.ptr()))); } }; @@ -2983,11 +2820,9 @@ pub const Tcgen05WaitKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05WaitKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tcgen05_wait"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -2998,7 +2833,7 @@ pub const Tcgen05WaitKindAttr = opaque { const result = c.mlirCuteNVVMTcgen05WaitKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) Tcgen05WaitKind { + pub fn value(self: *const Self) Tcgen05WaitKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05WaitKindAttrGetValue(self.ptr()))); } }; @@ -3010,11 +2845,9 @@ pub const TensormapElemtypeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapElemtype; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tensormap_elemtype"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3025,7 +2858,7 @@ pub const TensormapElemtypeAttr = opaque { const result = c.mlirCuteNVVMTensormapElemtypeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) TensormapElemtype { + pub fn value(self: *const Self) TensormapElemtype { return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapElemtypeAttrGetValue(self.ptr()))); } }; @@ -3037,11 +2870,9 @@ pub const TensormapFieldAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapField; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tensormap_field"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3052,7 +2883,7 @@ pub const TensormapFieldAttr = opaque { const result = c.mlirCuteNVVMTensormapFieldAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) TensormapField { + pub fn value(self: *const Self) TensormapField { return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapFieldAttrGetValue(self.ptr()))); } }; @@ -3064,11 +2895,9 @@ pub const TensormapFillModeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapFillMode; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tensormap_fill_mode"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3079,7 +2908,7 @@ pub const TensormapFillModeAttr = opaque { const result = c.mlirCuteNVVMTensormapFillModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) TensormapFillMode { + pub fn value(self: *const Self) TensormapFillMode { return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapFillModeAttrGetValue(self.ptr()))); } }; @@ -3091,11 +2920,9 @@ pub const TensormapInterleaveLayoutAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapInterleaveLayout; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tensormap_interleave_layout"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3106,7 +2933,7 @@ pub const TensormapInterleaveLayoutAttr = opaque { const result = c.mlirCuteNVVMTensormapInterleaveLayoutAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) TensormapInterleaveLayout { + pub fn value(self: *const Self) TensormapInterleaveLayout { return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapInterleaveLayoutAttrGetValue(self.ptr()))); } }; @@ -3118,11 +2945,9 @@ pub const TensormapSwizzleAtomicityAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapSwizzleAtomicity; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tensormap_swizzle_atomicity"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3133,7 +2958,7 @@ pub const TensormapSwizzleAtomicityAttr = opaque { const result = c.mlirCuteNVVMTensormapSwizzleAtomicityAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) TensormapSwizzleAtomicity { + pub fn value(self: *const Self) TensormapSwizzleAtomicity { return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapSwizzleAtomicityAttrGetValue(self.ptr()))); } }; @@ -3145,11 +2970,9 @@ pub const TensormapSwizzleModeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapSwizzleMode; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "tensormap_swizzle_mode"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3160,7 +2983,7 @@ pub const TensormapSwizzleModeAttr = opaque { const result = c.mlirCuteNVVMTensormapSwizzleModeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) TensormapSwizzleMode { + pub fn value(self: *const Self) TensormapSwizzleMode { return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapSwizzleModeAttrGetValue(self.ptr()))); } }; @@ -3172,11 +2995,9 @@ pub const TmemLayoutAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMTmemLayout; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "TmemLayout"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3187,7 +3008,7 @@ pub const TmemLayoutAttr = opaque { const result = c.mlirCuteNVVMTmemLayoutAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) TmemLayout { + pub fn value(self: *const Self) TmemLayout { return @fromBackingInt(@intCast(c.mlirCuteNVVMTmemLayoutAttrGetValue(self.ptr()))); } }; @@ -3199,11 +3020,9 @@ pub const ValidatePatternAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMValidatePattern; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "validate_pattern"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3214,7 +3033,7 @@ pub const ValidatePatternAttr = opaque { const result = c.mlirCuteNVVMValidatePatternAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) ValidatePattern { + pub fn value(self: *const Self) ValidatePattern { return @fromBackingInt(@intCast(c.mlirCuteNVVMValidatePatternAttrGetValue(self.ptr()))); } }; @@ -3226,11 +3045,9 @@ pub const VoteSyncKindAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMVoteSyncKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "vote_sync_kind"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3241,7 +3058,7 @@ pub const VoteSyncKindAttr = opaque { const result = c.mlirCuteNVVMVoteSyncKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) VoteSyncKind { + pub fn value(self: *const Self) VoteSyncKind { return @fromBackingInt(@intCast(c.mlirCuteNVVMVoteSyncKindAttrGetValue(self.ptr()))); } }; @@ -3253,11 +3070,9 @@ pub const WGMMAScaleInAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMWGMMAScaleIn; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "wgmma_scale_in"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3268,7 +3083,7 @@ pub const WGMMAScaleInAttr = opaque { const result = c.mlirCuteNVVMWGMMAScaleInAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) WGMMAScaleIn { + pub fn value(self: *const Self) WGMMAScaleIn { return @fromBackingInt(@intCast(c.mlirCuteNVVMWGMMAScaleInAttrGetValue(self.ptr()))); } }; @@ -3280,11 +3095,9 @@ pub const WGMMAScaleOutAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMWGMMAScaleOut; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "wgmma_scale_out"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3295,7 +3108,7 @@ pub const WGMMAScaleOutAttr = opaque { const result = c.mlirCuteNVVMWGMMAScaleOutAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) WGMMAScaleOut { + pub fn value(self: *const Self) WGMMAScaleOut { return @fromBackingInt(@intCast(c.mlirCuteNVVMWGMMAScaleOutAttrGetValue(self.ptr()))); } }; @@ -3307,11 +3120,9 @@ pub const WGMMATypesAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMWGMMATypes; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "wgmma_type"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3322,7 +3133,7 @@ pub const WGMMATypesAttr = opaque { const result = c.mlirCuteNVVMWGMMATypesAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) WGMMATypes { + pub fn value(self: *const Self) WGMMATypes { return @fromBackingInt(@intCast(c.mlirCuteNVVMWGMMATypesAttrGetValue(self.ptr()))); } }; @@ -3334,11 +3145,9 @@ pub const LdStMatrixShapeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMLdStMatrixShape; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "ld_st_matrix_shape"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3350,10 +3159,10 @@ pub const LdStMatrixShapeAttr = opaque { const result = c.mlirCuteNVVMLdStMatrixShapeAttrGet(ctx.ptr(), args.m, args.n); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getM(self: *const Self) c_int { + pub fn m(self: *const Self) c_int { return c.mlirCuteNVVMLdStMatrixShapeAttrGetM(self.ptr()); } - pub fn getN(self: *const Self) c_int { + pub fn n(self: *const Self) c_int { return c.mlirCuteNVVMLdStMatrixShapeAttrGetN(self.ptr()); } }; @@ -3365,11 +3174,9 @@ pub const MMAShapeAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMMMAShape; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "shape"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3382,13 +3189,13 @@ pub const MMAShapeAttr = opaque { const result = c.mlirCuteNVVMMMAShapeAttrGet(ctx.ptr(), args.m, args.n, args.k); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getM(self: *const Self) c_int { + pub fn m(self: *const Self) c_int { return c.mlirCuteNVVMMMAShapeAttrGetM(self.ptr()); } - pub fn getN(self: *const Self) c_int { + pub fn n(self: *const Self) c_int { return c.mlirCuteNVVMMMAShapeAttrGetN(self.ptr()); } - pub fn getK(self: *const Self) c_int { + pub fn k(self: *const Self) c_int { return c.mlirCuteNVVMMMAShapeAttrGetK(self.ptr()); } }; @@ -3400,11 +3207,9 @@ pub const TargetAttr = opaque { pub const isAFn = c.mlirAttributeIsACuteNVVMTarget; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "target"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -3421,28 +3226,28 @@ pub const TargetAttr = opaque { const result = c.mlirCuteNVVMTargetAttrGet(ctx.ptr(), args.O, mlir.stringRef(args.triple), mlir.stringRef(args.chip), mlir.stringRef(args.features), if (args.flags) |v| v.ptr() else .{ .ptr = null }, if (args.link) |v| v.ptr() else .{ .ptr = null }, args.verifyTarget); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getO(self: *const Self) c_int { + pub fn o(self: *const Self) c_int { return c.mlirCuteNVVMTargetAttrGetO(self.ptr()); } - pub fn getTriple(self: *const Self) []const u8 { + pub fn triple(self: *const Self) []const u8 { const s = c.mlirCuteNVVMTargetAttrGetTriple(self.ptr()); return if (s.length == 0) "" else mlir.string(s); } - pub fn getChip(self: *const Self) []const u8 { + pub fn chip(self: *const Self) []const u8 { const s = c.mlirCuteNVVMTargetAttrGetChip(self.ptr()); return if (s.length == 0) "" else mlir.string(s); } - pub fn getFeatures(self: *const Self) []const u8 { + pub fn features(self: *const Self) []const u8 { const s = c.mlirCuteNVVMTargetAttrGetFeatures(self.ptr()); return if (s.length == 0) "" else mlir.string(s); } - pub fn getFlags(self: *const Self) ?*const mlir.Attribute { + pub fn flags(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVVMTargetAttrGetFlags(self.ptr()).ptr); } - pub fn getLink(self: *const Self) ?*const mlir.Attribute { + pub fn link(self: *const Self) ?*const mlir.Attribute { return @ptrCast(c.mlirCuteNVVMTargetAttrGetLink(self.ptr()).ptr); } - pub fn getVerifyTarget(self: *const Self) bool { + pub fn verifyTarget(self: *const Self) bool { return c.mlirCuteNVVMTargetAttrGetVerifyTarget(self.ptr()); } }; @@ -3467,10 +3272,20 @@ pub fn add_packed_f16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, } /// `nvvm.add.packed.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn add_packed_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, ftz: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn add_packed_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); return mlir.Operation.make(ctx, "nvvm.add.packed.f32x2", .{ .operands = .{ .flat = &.{ srcA, srcB } }, .results = .{ .flat = &.{res_type} }, @@ -3480,9 +3295,18 @@ pub fn add_packed_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *cons } /// `nvvm.add.packed.f32x2.bf16x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn add_packed_f32x2_bf16x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn add_packed_f32x2_bf16x2_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); return mlir.Operation.make(ctx, "nvvm.add.packed.f32x2.bf16x2.f32x2", .{ .operands = .{ .flat = &.{ srcA, srcB } }, .results = .{ .flat = &.{res_type} }, @@ -3492,9 +3316,18 @@ pub fn add_packed_f32x2_bf16x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value } /// `nvvm.add.packed.f32x2.f16x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn add_packed_f32x2_f16x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn add_packed_f32x2_f16x2_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); return mlir.Operation.make(ctx, "nvvm.add.packed.f32x2.f16x2.f32x2", .{ .operands = .{ .flat = &.{ srcA, srcB } }, .results = .{ .flat = &.{res_type} }, @@ -3504,11 +3337,22 @@ pub fn add_packed_f32x2_f16x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, } /// `nvvm.addf`. Result types are explicit; attributes use mlir.Attribute. -pub fn addf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, sat: ?*const mlir.Attribute, ftz: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn addf( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); return mlir.Operation.make(ctx, "nvvm.addf", .{ .operands = .{ .flat = &.{ lhs, rhs } }, .results = .{ .flat = &.{res_type} }, @@ -3518,9 +3362,17 @@ pub fn addf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, } /// `nvvm.applypriority.async.bulk`. Result types are explicit; attributes use mlir.Attribute. -pub fn applypriority_async_bulk(ctx: *mlir.Context, ptr: *const mlir.Value, size: *const mlir.Value, evict: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn applypriority_async_bulk( + ctx: *mlir.Context, + ptr: *const mlir.Value, + size: *const mlir.Value, + opts: struct { + evict: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); return mlir.Operation.make(ctx, "nvvm.applypriority.async.bulk", .{ .operands = .{ .flat = &.{ ptr, size } }, .attributes = attributes.constSlice(), @@ -3529,10 +3381,20 @@ pub fn applypriority_async_bulk(ctx: *mlir.Context, ptr: *const mlir.Value, size } /// `nvvm.applypriority.async.bulk.tensor`. Result types are explicit; attributes use mlir.Attribute. -pub fn applypriority_async_bulk_tensor(ctx: *mlir.Context, tmaDesc: *const mlir.Value, coordinates: []const *const mlir.Value, im2colArgs: []const *const mlir.Value, mode: ?*const mlir.Attribute, evict: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn applypriority_async_bulk_tensor( + ctx: *mlir.Context, + tmaDesc: *const mlir.Value, + coordinates: []const *const mlir.Value, + im2colArgs: []const *const mlir.Value, + opts: struct { + mode: ?*const mlir.Attribute = null, + evict: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); - if (evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); return mlir.Operation.make(ctx, "nvvm.applypriority.async.bulk.tensor", .{ .operands = .{ .variadic = &.{ &.{tmaDesc}, @@ -3545,10 +3407,24 @@ pub fn applypriority_async_bulk_tensor(ctx: *mlir.Context, tmaDesc: *const mlir. } /// `nvvm.applypriority.async.bulk.tensor.override`. Result types are explicit; attributes use mlir.Attribute. -pub fn applypriority_async_bulk_tensor_override(ctx: *mlir.Context, tmaDesc: *const mlir.Value, overrideAdrr: *const mlir.Value, coordinates: []const *const mlir.Value, tensorSize: []const *const mlir.Value, lowerStride: []const *const mlir.Value, upperStride: ?*const mlir.Value, im2colArgs: []const *const mlir.Value, mode: ?*const mlir.Attribute, evict: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn applypriority_async_bulk_tensor_override( + ctx: *mlir.Context, + tmaDesc: *const mlir.Value, + overrideAdrr: *const mlir.Value, + coordinates: []const *const mlir.Value, + tensorSize: []const *const mlir.Value, + lowerStride: []const *const mlir.Value, + im2colArgs: []const *const mlir.Value, + opts: struct { + upperStride: ?*const mlir.Value = null, + mode: ?*const mlir.Attribute = null, + evict: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); - if (evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); return mlir.Operation.make(ctx, "nvvm.applypriority.async.bulk.tensor.override", .{ .operands = .{ .variadic = &.{ &.{tmaDesc}, @@ -3556,7 +3432,7 @@ pub fn applypriority_async_bulk_tensor_override(ctx: *mlir.Context, tmaDesc: *co coordinates, tensorSize, lowerStride, - if (upperStride) |value| &.{value} else &.{}, + if (opts.upperStride) |value| &.{value} else &.{}, im2colArgs, } }, .attributes = attributes.constSlice(), @@ -3565,13 +3441,25 @@ pub fn applypriority_async_bulk_tensor_override(ctx: *mlir.Context, tmaDesc: *co } /// `nvvm.atomicrmw`. Result types are explicit; attributes use mlir.Attribute. -pub fn atomicrmw(ctx: *mlir.Context, ptr: *const mlir.Value, a: *const mlir.Value, b: ?*const mlir.Value, res_type: *const mlir.Type, op: *const mlir.Attribute, memOrder: ?*const mlir.Attribute, syncscope: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn atomicrmw( + ctx: *mlir.Context, + ptr: *const mlir.Value, + a: *const mlir.Value, + res_type: *const mlir.Type, + op: *const mlir.Attribute, + opts: struct { + b: ?*const mlir.Value = null, + memOrder: ?*const mlir.Attribute = null, + syncscope: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; attributes.appendAssumeCapacity(.named(ctx, "op", op)); - if (memOrder) |value| attributes.appendAssumeCapacity(.named(ctx, "memOrder", value)); - if (syncscope) |value| attributes.appendAssumeCapacity(.named(ctx, "syncscope", value)); + if (opts.memOrder) |value| attributes.appendAssumeCapacity(.named(ctx, "memOrder", value)); + if (opts.syncscope) |value| attributes.appendAssumeCapacity(.named(ctx, "syncscope", value)); return mlir.Operation.make(ctx, "nvvm.atomicrmw", .{ - .operands = .{ .flat = if (b) |value| &.{ ptr, a, value } else &.{ ptr, a } }, + .operands = .{ .flat = if (opts.b) |value| &.{ ptr, a, value } else &.{ ptr, a } }, .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), .location = location, @@ -3587,16 +3475,26 @@ pub fn bar_warp_sync(ctx: *mlir.Context, mask: *const mlir.Value, location: *con } /// `nvvm.barrier`. Result types are explicit; attributes use mlir.Attribute. -pub fn barrier(ctx: *mlir.Context, barrierId: ?*const mlir.Value, numberOfThreads: ?*const mlir.Value, reductionPredicate: ?*const mlir.Value, res_type: ?*const mlir.Type, reductionOp: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn barrier( + ctx: *mlir.Context, + opts: struct { + barrierId: ?*const mlir.Value = null, + numberOfThreads: ?*const mlir.Value = null, + reductionPredicate: ?*const mlir.Value = null, + res_type: ?*const mlir.Type = null, + reductionOp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; - if (res_type) |value| results.appendAssumeCapacity(value); + if (opts.res_type) |value| results.appendAssumeCapacity(value); var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (reductionOp) |value| attributes.appendAssumeCapacity(.named(ctx, "reductionOp", value)); + if (opts.reductionOp) |value| attributes.appendAssumeCapacity(.named(ctx, "reductionOp", value)); return mlir.Operation.make(ctx, "nvvm.barrier", .{ .operands = .{ .variadic = &.{ - if (barrierId) |value| &.{value} else &.{}, - if (numberOfThreads) |value| &.{value} else &.{}, - if (reductionPredicate) |value| &.{value} else &.{}, + if (opts.barrierId) |value| &.{value} else &.{}, + if (opts.numberOfThreads) |value| &.{value} else &.{}, + if (opts.reductionPredicate) |value| &.{value} else &.{}, } }, .results = .{ .flat = results.constSlice() }, .attributes = attributes.constSlice(), @@ -3605,17 +3503,32 @@ pub fn barrier(ctx: *mlir.Context, barrierId: ?*const mlir.Value, numberOfThread } /// `nvvm.barrier.arrive`. Result types are explicit; attributes use mlir.Attribute. -pub fn barrier_arrive(ctx: *mlir.Context, barrierId: ?*const mlir.Value, numberOfThreads: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { +pub fn barrier_arrive( + ctx: *mlir.Context, + numberOfThreads: *const mlir.Value, + opts: struct { + barrierId: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "nvvm.barrier.arrive", .{ - .operands = .{ .flat = if (barrierId) |value| &.{ value, numberOfThreads } else &.{numberOfThreads} }, + .operands = .{ .flat = if (opts.barrierId) |value| &.{ value, numberOfThreads } else &.{numberOfThreads} }, .location = location, }); } /// `nvvm.barrier.cta.arrive`. Result types are explicit; attributes use mlir.Attribute. -pub fn barrier_cta_arrive(ctx: *mlir.Context, barrierId: *const mlir.Value, threadCount: *const mlir.Value, aligned: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn barrier_cta_arrive( + ctx: *mlir.Context, + barrierId: *const mlir.Value, + threadCount: *const mlir.Value, + opts: struct { + aligned: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); + if (opts.aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); return mlir.Operation.make(ctx, "nvvm.barrier.cta.arrive", .{ .operands = .{ .flat = &.{ barrierId, threadCount } }, .attributes = attributes.constSlice(), @@ -3624,12 +3537,23 @@ pub fn barrier_cta_arrive(ctx: *mlir.Context, barrierId: *const mlir.Value, thre } /// `nvvm.barrier.cta.red`. Result types are explicit; attributes use mlir.Attribute. -pub fn barrier_cta_red(ctx: *mlir.Context, pred: *const mlir.Value, barrierId: *const mlir.Value, threadCount: ?*const mlir.Value, res_type: *const mlir.Type, kind: *const mlir.Attribute, aligned: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn barrier_cta_red( + ctx: *mlir.Context, + pred: *const mlir.Value, + barrierId: *const mlir.Value, + res_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + threadCount: ?*const mlir.Value = null, + aligned: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); + if (opts.aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); return mlir.Operation.make(ctx, "nvvm.barrier.cta.red", .{ - .operands = .{ .flat = if (threadCount) |value| &.{ pred, barrierId, value } else &.{ pred, barrierId } }, + .operands = .{ .flat = if (opts.threadCount) |value| &.{ pred, barrierId, value } else &.{ pred, barrierId } }, .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), .location = location, @@ -3637,11 +3561,19 @@ pub fn barrier_cta_red(ctx: *mlir.Context, pred: *const mlir.Value, barrierId: * } /// `nvvm.barrier.cta.sync`. Result types are explicit; attributes use mlir.Attribute. -pub fn barrier_cta_sync(ctx: *mlir.Context, barrierId: *const mlir.Value, threadCount: ?*const mlir.Value, aligned: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn barrier_cta_sync( + ctx: *mlir.Context, + barrierId: *const mlir.Value, + opts: struct { + threadCount: ?*const mlir.Value = null, + aligned: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); + if (opts.aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); return mlir.Operation.make(ctx, "nvvm.barrier.cta.sync", .{ - .operands = .{ .flat = if (threadCount) |value| &.{ barrierId, value } else &.{barrierId} }, + .operands = .{ .flat = if (opts.threadCount) |value| &.{ barrierId, value } else &.{barrierId} }, .attributes = attributes.constSlice(), .location = location, }); @@ -3662,9 +3594,19 @@ pub fn breakpoint(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Oper } /// `nvvm.clmad`. Result types are explicit; attributes use mlir.Attribute. -pub fn clmad(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, c_: *const mlir.Value, res_type: *const mlir.Type, high: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn clmad( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c_: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + high: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (high) |value| attributes.appendAssumeCapacity(.named(ctx, "high", value)); + if (opts.high) |value| attributes.appendAssumeCapacity(.named(ctx, "high", value)); return mlir.Operation.make(ctx, "nvvm.clmad", .{ .operands = .{ .flat = &.{ a, b, c_ } }, .results = .{ .flat = &.{res_type} }, @@ -3674,9 +3616,15 @@ pub fn clmad(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, c_: } /// `nvvm.cluster.arrive`. Result types are explicit; attributes use mlir.Attribute. -pub fn cluster_arrive(ctx: *mlir.Context, aligned: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn cluster_arrive( + ctx: *mlir.Context, + opts: struct { + aligned: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); + if (opts.aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); return mlir.Operation.make(ctx, "nvvm.cluster.arrive", .{ .attributes = attributes.constSlice(), .location = location, @@ -3684,9 +3632,15 @@ pub fn cluster_arrive(ctx: *mlir.Context, aligned: ?*const mlir.Attribute, locat } /// `nvvm.cluster.arrive.relaxed`. Result types are explicit; attributes use mlir.Attribute. -pub fn cluster_arrive_relaxed(ctx: *mlir.Context, aligned: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn cluster_arrive_relaxed( + ctx: *mlir.Context, + opts: struct { + aligned: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); + if (opts.aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); return mlir.Operation.make(ctx, "nvvm.cluster.arrive.relaxed", .{ .attributes = attributes.constSlice(), .location = location, @@ -3694,9 +3648,15 @@ pub fn cluster_arrive_relaxed(ctx: *mlir.Context, aligned: ?*const mlir.Attribut } /// `nvvm.cluster.wait`. Result types are explicit; attributes use mlir.Attribute. -pub fn cluster_wait(ctx: *mlir.Context, aligned: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn cluster_wait( + ctx: *mlir.Context, + opts: struct { + aligned: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); + if (opts.aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); return mlir.Operation.make(ctx, "nvvm.cluster.wait", .{ .attributes = attributes.constSlice(), .location = location, @@ -3716,9 +3676,17 @@ pub fn clusterlaunchcontrol_query_cancel(ctx: *mlir.Context, try_cancel_response } /// `nvvm.clusterlaunchcontrol.try.cancel`. Result types are explicit; attributes use mlir.Attribute. -pub fn clusterlaunchcontrol_try_cancel(ctx: *mlir.Context, smemAddress: *const mlir.Value, mbarrier: *const mlir.Value, multicast: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn clusterlaunchcontrol_try_cancel( + ctx: *mlir.Context, + smemAddress: *const mlir.Value, + mbarrier: *const mlir.Value, + opts: struct { + multicast: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "multicast", value)); + if (opts.multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "multicast", value)); return mlir.Operation.make(ctx, "nvvm.clusterlaunchcontrol.try.cancel", .{ .operands = .{ .flat = &.{ smemAddress, mbarrier } }, .attributes = attributes.constSlice(), @@ -3739,12 +3707,23 @@ pub fn compare_and_set(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir. } /// `nvvm.convert.and.pack.integer`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_and_pack_integer(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, srcC: ?*const mlir.Value, dst_type: *const mlir.Type, is_signed: ?*const mlir.Attribute, convert_type: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn convert_and_pack_integer( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + dst_type: *const mlir.Type, + convert_type: *const mlir.Attribute, + opts: struct { + srcC: ?*const mlir.Value = null, + is_signed: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (is_signed) |value| attributes.appendAssumeCapacity(.named(ctx, "is_signed", value)); + if (opts.is_signed) |value| attributes.appendAssumeCapacity(.named(ctx, "is_signed", value)); attributes.appendAssumeCapacity(.named(ctx, "convert_type", convert_type)); return mlir.Operation.make(ctx, "nvvm.convert.and.pack.integer", .{ - .operands = .{ .flat = if (srcC) |value| &.{ srcA, srcB, value } else &.{ srcA, srcB } }, + .operands = .{ .flat = if (opts.srcC) |value| &.{ srcA, srcB, value } else &.{ srcA, srcB } }, .results = .{ .flat = &.{dst_type} }, .attributes = attributes.constSlice(), .location = location, @@ -3752,15 +3731,28 @@ pub fn convert_and_pack_integer(ctx: *mlir.Context, srcA: *const mlir.Value, src } /// `nvvm.convert.bf16x2.to.f4x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_bf16x2_to_f4x2(ctx: *mlir.Context, src: *const mlir.Value, scale_factor: ?*const mlir.Value, dst_type: *const mlir.Type, isPZO: ?*const mlir.Attribute, scale_factor_kind: ?*const mlir.Attribute, rnd: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, dstTy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn convert_bf16x2_to_f4x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - if (isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); - if (scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); return mlir.Operation.make(ctx, "nvvm.convert.bf16x2.to.f4x2", .{ - .operands = .{ .flat = if (scale_factor) |value| &.{ src, value } else &.{src} }, + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ src, value } else &.{src} }, .results = .{ .flat = &.{dst_type} }, .attributes = attributes.constSlice(), .location = location, @@ -3768,15 +3760,28 @@ pub fn convert_bf16x2_to_f4x2(ctx: *mlir.Context, src: *const mlir.Value, scale_ } /// `nvvm.convert.bf16x2.to.f6x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_bf16x2_to_f6x2(ctx: *mlir.Context, src: *const mlir.Value, scale_factor: ?*const mlir.Value, dst_type: *const mlir.Type, isPZO: ?*const mlir.Attribute, scale_factor_kind: ?*const mlir.Attribute, rnd: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, dstTy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn convert_bf16x2_to_f6x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - if (isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); - if (scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); return mlir.Operation.make(ctx, "nvvm.convert.bf16x2.to.f6x2", .{ - .operands = .{ .flat = if (scale_factor) |value| &.{ src, value } else &.{src} }, + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ src, value } else &.{src} }, .results = .{ .flat = &.{dst_type} }, .attributes = attributes.constSlice(), .location = location, @@ -3784,16 +3789,30 @@ pub fn convert_bf16x2_to_f6x2(ctx: *mlir.Context, src: *const mlir.Value, scale_ } /// `nvvm.convert.bf16x2.to.f8x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_bf16x2_to_f8x2(ctx: *mlir.Context, src: *const mlir.Value, scale_factor: ?*const mlir.Value, dst_type: *const mlir.Type, isPZO: ?*const mlir.Attribute, scale_factor_kind: ?*const mlir.Attribute, rnd: ?*const mlir.Attribute, sat: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, dstTy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn convert_bf16x2_to_f8x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; - if (isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); - if (scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); return mlir.Operation.make(ctx, "nvvm.convert.bf16x2.to.f8x2", .{ - .operands = .{ .flat = if (scale_factor) |value| &.{ src, value } else &.{src} }, + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ src, value } else &.{src} }, .results = .{ .flat = &.{dst_type} }, .attributes = attributes.constSlice(), .location = location, @@ -3801,11 +3820,20 @@ pub fn convert_bf16x2_to_f8x2(ctx: *mlir.Context, src: *const mlir.Value, scale_ } /// `nvvm.convert.bf16x2.to.s2f6x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_bf16x2_to_s2f6x2(ctx: *mlir.Context, src: *const mlir.Value, scaleFactor: ?*const mlir.Value, dst_type: *const mlir.Type, relu: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn convert_bf16x2_to_s2f6x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + opts: struct { + scaleFactor: ?*const mlir.Value = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); return mlir.Operation.make(ctx, "nvvm.convert.bf16x2.to.s2f6x2", .{ - .operands = .{ .flat = if (scaleFactor) |value| &.{ src, value } else &.{src} }, + .operands = .{ .flat = if (opts.scaleFactor) |value| &.{ src, value } else &.{src} }, .results = .{ .flat = &.{dst_type} }, .attributes = attributes.constSlice(), .location = location, @@ -3813,15 +3841,28 @@ pub fn convert_bf16x2_to_s2f6x2(ctx: *mlir.Context, src: *const mlir.Value, scal } /// `nvvm.convert.f16x2.to.f4x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f16x2_to_f4x2(ctx: *mlir.Context, src: *const mlir.Value, scale_factor: ?*const mlir.Value, dst_type: *const mlir.Type, isPZO: ?*const mlir.Attribute, scale_factor_kind: ?*const mlir.Attribute, rnd: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, dstTy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn convert_f16x2_to_f4x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - if (isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); - if (scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); return mlir.Operation.make(ctx, "nvvm.convert.f16x2.to.f4x2", .{ - .operands = .{ .flat = if (scale_factor) |value| &.{ src, value } else &.{src} }, + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ src, value } else &.{src} }, .results = .{ .flat = &.{dst_type} }, .attributes = attributes.constSlice(), .location = location, @@ -3829,15 +3870,28 @@ pub fn convert_f16x2_to_f4x2(ctx: *mlir.Context, src: *const mlir.Value, scale_f } /// `nvvm.convert.f16x2.to.f6x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f16x2_to_f6x2(ctx: *mlir.Context, src: *const mlir.Value, scale_factor: ?*const mlir.Value, dst_type: *const mlir.Type, isPZO: ?*const mlir.Attribute, scale_factor_kind: ?*const mlir.Attribute, rnd: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, dstTy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn convert_f16x2_to_f6x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - if (isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); - if (scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); return mlir.Operation.make(ctx, "nvvm.convert.f16x2.to.f6x2", .{ - .operands = .{ .flat = if (scale_factor) |value| &.{ src, value } else &.{src} }, + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ src, value } else &.{src} }, .results = .{ .flat = &.{dst_type} }, .attributes = attributes.constSlice(), .location = location, @@ -3845,16 +3899,30 @@ pub fn convert_f16x2_to_f6x2(ctx: *mlir.Context, src: *const mlir.Value, scale_f } /// `nvvm.convert.f16x2.to.f8x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f16x2_to_f8x2(ctx: *mlir.Context, a: *const mlir.Value, scale_factor: ?*const mlir.Value, dst_type: *const mlir.Type, isPZO: ?*const mlir.Attribute, scale_factor_kind: ?*const mlir.Attribute, rnd: ?*const mlir.Attribute, sat: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, dstTy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn convert_f16x2_to_f8x2( + ctx: *mlir.Context, + a: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; - if (isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); - if (scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); return mlir.Operation.make(ctx, "nvvm.convert.f16x2.to.f8x2", .{ - .operands = .{ .flat = if (scale_factor) |value| &.{ a, value } else &.{a} }, + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ a, value } else &.{a} }, .results = .{ .flat = &.{dst_type} }, .attributes = attributes.constSlice(), .location = location, @@ -3862,13 +3930,25 @@ pub fn convert_f16x2_to_f8x2(ctx: *mlir.Context, a: *const mlir.Value, scale_fac } /// `nvvm.convert.f32x2.to.bf16x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f32x2_to_bf16x2(ctx: *mlir.Context, src_hi: *const mlir.Value, src_lo: *const mlir.Value, random_bits: ?*const mlir.Value, dst_type: *const mlir.Type, rnd: ?*const mlir.Attribute, sat: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn convert_f32x2_to_bf16x2( + ctx: *mlir.Context, + src_hi: *const mlir.Value, + src_lo: *const mlir.Value, + dst_type: *const mlir.Type, + opts: struct { + random_bits: ?*const mlir.Value = null, + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.bf16x2", .{ - .operands = .{ .flat = if (random_bits) |value| &.{ src_hi, src_lo, value } else &.{ src_hi, src_lo } }, + .operands = .{ .flat = if (opts.random_bits) |value| &.{ src_hi, src_lo, value } else &.{ src_hi, src_lo } }, .results = .{ .flat = &.{dst_type} }, .attributes = attributes.constSlice(), .location = location, @@ -3876,13 +3956,25 @@ pub fn convert_f32x2_to_bf16x2(ctx: *mlir.Context, src_hi: *const mlir.Value, sr } /// `nvvm.convert.f32x2.to.f16x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f32x2_to_f16x2(ctx: *mlir.Context, src_hi: *const mlir.Value, src_lo: *const mlir.Value, random_bits: ?*const mlir.Value, dst_type: *const mlir.Type, rnd: ?*const mlir.Attribute, sat: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn convert_f32x2_to_f16x2( + ctx: *mlir.Context, + src_hi: *const mlir.Value, + src_lo: *const mlir.Value, + dst_type: *const mlir.Type, + opts: struct { + random_bits: ?*const mlir.Value = null, + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.f16x2", .{ - .operands = .{ .flat = if (random_bits) |value| &.{ src_hi, src_lo, value } else &.{ src_hi, src_lo } }, + .operands = .{ .flat = if (opts.random_bits) |value| &.{ src_hi, src_lo, value } else &.{ src_hi, src_lo } }, .results = .{ .flat = &.{dst_type} }, .attributes = attributes.constSlice(), .location = location, @@ -3890,15 +3982,29 @@ pub fn convert_f32x2_to_f16x2(ctx: *mlir.Context, src_hi: *const mlir.Value, src } /// `nvvm.convert.f32x2.to.f4x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f32x2_to_f4x2(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, scale_factor: ?*const mlir.Value, dst_type: *const mlir.Type, isPZO: ?*const mlir.Attribute, scale_factor_kind: ?*const mlir.Attribute, rnd: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, dstType: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn convert_f32x2_to_f4x2( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + dst_type: *const mlir.Type, + dstType: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - if (isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); - if (scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); attributes.appendAssumeCapacity(.named(ctx, "dstType", dstType)); return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.f4x2", .{ - .operands = .{ .flat = if (scale_factor) |value| &.{ a, b, value } else &.{ a, b } }, + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ a, b, value } else &.{ a, b } }, .results = .{ .flat = &.{dst_type} }, .attributes = attributes.constSlice(), .location = location, @@ -3906,15 +4012,29 @@ pub fn convert_f32x2_to_f4x2(ctx: *mlir.Context, a: *const mlir.Value, b: *const } /// `nvvm.convert.f32x2.to.f6x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f32x2_to_f6x2(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, scale_factor: ?*const mlir.Value, dst_type: *const mlir.Type, isPZO: ?*const mlir.Attribute, scale_factor_kind: ?*const mlir.Attribute, rnd: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, dstTy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn convert_f32x2_to_f6x2( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - if (isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); - if (scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.f6x2", .{ - .operands = .{ .flat = if (scale_factor) |value| &.{ a, b, value } else &.{ a, b } }, + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ a, b, value } else &.{ a, b } }, .results = .{ .flat = &.{dst_type} }, .attributes = attributes.constSlice(), .location = location, @@ -3922,16 +4042,31 @@ pub fn convert_f32x2_to_f6x2(ctx: *mlir.Context, a: *const mlir.Value, b: *const } /// `nvvm.convert.f32x2.to.f8x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f32x2_to_f8x2(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, scale_factor: ?*const mlir.Value, dst_type: *const mlir.Type, isPZO: ?*const mlir.Attribute, scale_factor_kind: ?*const mlir.Attribute, rnd: ?*const mlir.Attribute, sat: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, dstTy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn convert_f32x2_to_f8x2( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; - if (isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); - if (scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.f8x2", .{ - .operands = .{ .flat = if (scale_factor) |value| &.{ a, b, value } else &.{ a, b } }, + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ a, b, value } else &.{ a, b } }, .results = .{ .flat = &.{dst_type} }, .attributes = attributes.constSlice(), .location = location, @@ -3939,11 +4074,21 @@ pub fn convert_f32x2_to_f8x2(ctx: *mlir.Context, a: *const mlir.Value, b: *const } /// `nvvm.convert.f32x2.to.s2f6x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f32x2_to_s2f6x2(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, scaleFactor: ?*const mlir.Value, dst_type: *const mlir.Type, relu: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn convert_f32x2_to_s2f6x2( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + dst_type: *const mlir.Type, + opts: struct { + scaleFactor: ?*const mlir.Value = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.s2f6x2", .{ - .operands = .{ .flat = if (scaleFactor) |value| &.{ a, b, value } else &.{ a, b } }, + .operands = .{ .flat = if (opts.scaleFactor) |value| &.{ a, b, value } else &.{ a, b } }, .results = .{ .flat = &.{dst_type} }, .attributes = attributes.constSlice(), .location = location, @@ -3951,9 +4096,19 @@ pub fn convert_f32x2_to_s2f6x2(ctx: *mlir.Context, a: *const mlir.Value, b: *con } /// `nvvm.convert.f32x4.to.f4x4`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f32x4_to_f4x4(ctx: *mlir.Context, src: *const mlir.Value, rbits: *const mlir.Value, dst_type: *const mlir.Type, relu: ?*const mlir.Attribute, dstTy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn convert_f32x4_to_f4x4( + ctx: *mlir.Context, + src: *const mlir.Value, + rbits: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); return mlir.Operation.make(ctx, "nvvm.convert.f32x4.to.f4x4", .{ .operands = .{ .flat = &.{ src, rbits } }, @@ -3964,9 +4119,19 @@ pub fn convert_f32x4_to_f4x4(ctx: *mlir.Context, src: *const mlir.Value, rbits: } /// `nvvm.convert.f32x4.to.f6x4`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f32x4_to_f6x4(ctx: *mlir.Context, src: *const mlir.Value, rbits: *const mlir.Value, dst_type: *const mlir.Type, relu: ?*const mlir.Attribute, dstTy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn convert_f32x4_to_f6x4( + ctx: *mlir.Context, + src: *const mlir.Value, + rbits: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); return mlir.Operation.make(ctx, "nvvm.convert.f32x4.to.f6x4", .{ .operands = .{ .flat = &.{ src, rbits } }, @@ -3977,9 +4142,19 @@ pub fn convert_f32x4_to_f6x4(ctx: *mlir.Context, src: *const mlir.Value, rbits: } /// `nvvm.convert.f32x4.to.f8x4`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f32x4_to_f8x4(ctx: *mlir.Context, src: *const mlir.Value, rbits: *const mlir.Value, dst_type: *const mlir.Type, relu: ?*const mlir.Attribute, dstTy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn convert_f32x4_to_f8x4( + ctx: *mlir.Context, + src: *const mlir.Value, + rbits: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); return mlir.Operation.make(ctx, "nvvm.convert.f32x4.to.f8x4", .{ .operands = .{ .flat = &.{ src, rbits } }, @@ -3990,9 +4165,18 @@ pub fn convert_f32x4_to_f8x4(ctx: *mlir.Context, src: *const mlir.Value, rbits: } /// `nvvm.convert.f4x2.to.f16x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f4x2_to_f16x2(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, relu: ?*const mlir.Attribute, srcType: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn convert_f4x2_to_f16x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + srcType: *const mlir.Attribute, + opts: struct { + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); attributes.appendAssumeCapacity(.named(ctx, "srcType", srcType)); return mlir.Operation.make(ctx, "nvvm.convert.f4x2.to.f16x2", .{ .operands = .{ .flat = &.{src} }, @@ -4003,9 +4187,18 @@ pub fn convert_f4x2_to_f16x2(ctx: *mlir.Context, src: *const mlir.Value, dst_typ } /// `nvvm.convert.f6x2.to.f16x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f6x2_to_f16x2(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, relu: ?*const mlir.Attribute, srcType: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn convert_f6x2_to_f16x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + srcType: *const mlir.Attribute, + opts: struct { + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); attributes.appendAssumeCapacity(.named(ctx, "srcType", srcType)); return mlir.Operation.make(ctx, "nvvm.convert.f6x2.to.f16x2", .{ .operands = .{ .flat = &.{src} }, @@ -4016,12 +4209,22 @@ pub fn convert_f6x2_to_f16x2(ctx: *mlir.Context, src: *const mlir.Value, dst_typ } /// `nvvm.convert.f8x2.to.bf16x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f8x2_to_bf16x2(ctx: *mlir.Context, src: *const mlir.Value, scaleFactor: ?*const mlir.Value, dst_type: *const mlir.Type, sat: ?*const mlir.Attribute, srcType: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn convert_f8x2_to_bf16x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + srcType: *const mlir.Attribute, + opts: struct { + scaleFactor: ?*const mlir.Value = null, + sat: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); attributes.appendAssumeCapacity(.named(ctx, "srcType", srcType)); return mlir.Operation.make(ctx, "nvvm.convert.f8x2.to.bf16x2", .{ - .operands = .{ .flat = if (scaleFactor) |value| &.{ src, value } else &.{src} }, + .operands = .{ .flat = if (opts.scaleFactor) |value| &.{ src, value } else &.{src} }, .results = .{ .flat = &.{dst_type} }, .attributes = attributes.constSlice(), .location = location, @@ -4029,9 +4232,18 @@ pub fn convert_f8x2_to_bf16x2(ctx: *mlir.Context, src: *const mlir.Value, scaleF } /// `nvvm.convert.f8x2.to.f16x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f8x2_to_f16x2(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, relu: ?*const mlir.Attribute, srcType: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn convert_f8x2_to_f16x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + srcType: *const mlir.Attribute, + opts: struct { + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); attributes.appendAssumeCapacity(.named(ctx, "srcType", srcType)); return mlir.Operation.make(ctx, "nvvm.convert.f8x2.to.f16x2", .{ .operands = .{ .flat = &.{src} }, @@ -4042,12 +4254,23 @@ pub fn convert_f8x2_to_f16x2(ctx: *mlir.Context, src: *const mlir.Value, dst_typ } /// `nvvm.convert.float.to.integer`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_float_to_integer(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, rnd: ?*const mlir.Attribute, sat: ?*const mlir.Attribute, ftz: ?*const mlir.Attribute, is_signed: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn convert_float_to_integer( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + ftz: ?*const mlir.Attribute = null, + is_signed: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); - if (is_signed) |value| attributes.appendAssumeCapacity(.named(ctx, "is_signed", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (opts.is_signed) |value| attributes.appendAssumeCapacity(.named(ctx, "is_signed", value)); return mlir.Operation.make(ctx, "nvvm.convert.float.to.integer", .{ .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{dst_type} }, @@ -4057,11 +4280,21 @@ pub fn convert_float_to_integer(ctx: *mlir.Context, src: *const mlir.Value, dst_ } /// `nvvm.convert.float.to.tf32`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_float_to_tf32(ctx: *mlir.Context, src: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, sat: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn convert_float_to_tf32( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); return mlir.Operation.make(ctx, "nvvm.convert.float.to.tf32", .{ .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{res_type} }, @@ -4071,12 +4304,22 @@ pub fn convert_float_to_tf32(ctx: *mlir.Context, src: *const mlir.Value, res_typ } /// `nvvm.convert.s2f6x2.to.bf16x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_s2f6x2_to_bf16x2(ctx: *mlir.Context, src: *const mlir.Value, scaleFactor: ?*const mlir.Value, dst_type: *const mlir.Type, sat: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn convert_s2f6x2_to_bf16x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + opts: struct { + scaleFactor: ?*const mlir.Value = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); return mlir.Operation.make(ctx, "nvvm.convert.s2f6x2.to.bf16x2", .{ - .operands = .{ .flat = if (scaleFactor) |value| &.{ src, value } else &.{src} }, + .operands = .{ .flat = if (opts.scaleFactor) |value| &.{ src, value } else &.{src} }, .results = .{ .flat = &.{dst_type} }, .attributes = attributes.constSlice(), .location = location, @@ -4084,9 +4327,17 @@ pub fn convert_s2f6x2_to_bf16x2(ctx: *mlir.Context, src: *const mlir.Value, scal } /// `nvvm.cos`. Result types are explicit; attributes use mlir.Attribute. -pub fn cos(ctx: *mlir.Context, src: *const mlir.Value, res_type: *const mlir.Type, ftz: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn cos( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); return mlir.Operation.make(ctx, "nvvm.cos", .{ .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{res_type} }, @@ -4103,42 +4354,73 @@ pub fn cp_async_bulk_commit_group(ctx: *mlir.Context, location: *const mlir.Loca } /// `nvvm.cp.async.bulk.global.shared.cta`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_global_shared_cta(ctx: *mlir.Context, dstMem: *const mlir.Value, srcMem: *const mlir.Value, size: *const mlir.Value, l2CacheHint: ?*const mlir.Value, byteMask: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { +pub fn cp_async_bulk_global_shared_cta( + ctx: *mlir.Context, + dstMem: *const mlir.Value, + srcMem: *const mlir.Value, + size: *const mlir.Value, + opts: struct { + l2CacheHint: ?*const mlir.Value = null, + byteMask: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.global.shared.cta", .{ .operands = .{ .variadic = &.{ &.{dstMem}, &.{srcMem}, &.{size}, - if (l2CacheHint) |value| &.{value} else &.{}, - if (byteMask) |value| &.{value} else &.{}, + if (opts.l2CacheHint) |value| &.{value} else &.{}, + if (opts.byteMask) |value| &.{value} else &.{}, } }, .location = location, }); } /// `nvvm.cp.async.bulk.prefetch`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_prefetch(ctx: *mlir.Context, srcMem: *const mlir.Value, size: *const mlir.Value, l2CacheHint: ?*const mlir.Value, evict: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn cp_async_bulk_prefetch( + ctx: *mlir.Context, + srcMem: *const mlir.Value, + size: *const mlir.Value, + opts: struct { + l2CacheHint: ?*const mlir.Value = null, + evict: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.prefetch", .{ - .operands = .{ .flat = if (l2CacheHint) |value| &.{ srcMem, size, value } else &.{ srcMem, size } }, + .operands = .{ .flat = if (opts.l2CacheHint) |value| &.{ srcMem, size, value } else &.{ srcMem, size } }, .attributes = attributes.constSlice(), .location = location, }); } /// `nvvm.cp.async.bulk.shared.cluster.global`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_shared_cluster_global(ctx: *mlir.Context, dstMem: *const mlir.Value, srcMem: *const mlir.Value, mbar: *const mlir.Value, size: *const mlir.Value, multicastMask: ?*const mlir.Value, l2CacheHint: ?*const mlir.Value, validatePattern: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn cp_async_bulk_shared_cluster_global( + ctx: *mlir.Context, + dstMem: *const mlir.Value, + srcMem: *const mlir.Value, + mbar: *const mlir.Value, + size: *const mlir.Value, + opts: struct { + multicastMask: ?*const mlir.Value = null, + l2CacheHint: ?*const mlir.Value = null, + validatePattern: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (validatePattern) |value| attributes.appendAssumeCapacity(.named(ctx, "validatePattern", value)); + if (opts.validatePattern) |value| attributes.appendAssumeCapacity(.named(ctx, "validatePattern", value)); return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.shared.cluster.global", .{ .operands = .{ .variadic = &.{ &.{dstMem}, &.{srcMem}, &.{mbar}, &.{size}, - if (multicastMask) |value| &.{value} else &.{}, - if (l2CacheHint) |value| &.{value} else &.{}, + if (opts.multicastMask) |value| &.{value} else &.{}, + if (opts.l2CacheHint) |value| &.{value} else &.{}, } }, .attributes = attributes.constSlice(), .location = location, @@ -4154,16 +4436,27 @@ pub fn cp_async_bulk_shared_cluster_shared_cta(ctx: *mlir.Context, dstMem: *cons } /// `nvvm.cp.async.bulk.tensor.global.shared.cta`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_tensor_global_shared_cta(ctx: *mlir.Context, tmaDescriptor: *const mlir.Value, srcMem: *const mlir.Value, coordinates: []const *const mlir.Value, l2CacheHint: ?*const mlir.Value, predicate: ?*const mlir.Value, mode: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn cp_async_bulk_tensor_global_shared_cta( + ctx: *mlir.Context, + tmaDescriptor: *const mlir.Value, + srcMem: *const mlir.Value, + coordinates: []const *const mlir.Value, + opts: struct { + l2CacheHint: ?*const mlir.Value = null, + predicate: ?*const mlir.Value = null, + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.global.shared.cta", .{ .operands = .{ .variadic = &.{ &.{tmaDescriptor}, &.{srcMem}, coordinates, - if (l2CacheHint) |value| &.{value} else &.{}, - if (predicate) |value| &.{value} else &.{}, + if (opts.l2CacheHint) |value| &.{value} else &.{}, + if (opts.predicate) |value| &.{value} else &.{}, } }, .attributes = attributes.constSlice(), .location = location, @@ -4171,9 +4464,23 @@ pub fn cp_async_bulk_tensor_global_shared_cta(ctx: *mlir.Context, tmaDescriptor: } /// `nvvm.cp.async.bulk.tensor.global.shared.cta.override`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_tensor_global_shared_cta_override(ctx: *mlir.Context, tmaDesc: *const mlir.Value, srcMem: *const mlir.Value, overrideAdrr: *const mlir.Value, coordinates: []const *const mlir.Value, tensorSize: []const *const mlir.Value, lowerStride: []const *const mlir.Value, upperStride: ?*const mlir.Value, l2CacheHint: ?*const mlir.Value, mode: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn cp_async_bulk_tensor_global_shared_cta_override( + ctx: *mlir.Context, + tmaDesc: *const mlir.Value, + srcMem: *const mlir.Value, + overrideAdrr: *const mlir.Value, + coordinates: []const *const mlir.Value, + tensorSize: []const *const mlir.Value, + lowerStride: []const *const mlir.Value, + opts: struct { + upperStride: ?*const mlir.Value = null, + l2CacheHint: ?*const mlir.Value = null, + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.global.shared.cta.override", .{ .operands = .{ .variadic = &.{ &.{tmaDesc}, @@ -4182,8 +4489,8 @@ pub fn cp_async_bulk_tensor_global_shared_cta_override(ctx: *mlir.Context, tmaDe coordinates, tensorSize, lowerStride, - if (upperStride) |value| &.{value} else &.{}, - if (l2CacheHint) |value| &.{value} else &.{}, + if (opts.upperStride) |value| &.{value} else &.{}, + if (opts.l2CacheHint) |value| &.{value} else &.{}, } }, .attributes = attributes.constSlice(), .location = location, @@ -4191,16 +4498,27 @@ pub fn cp_async_bulk_tensor_global_shared_cta_override(ctx: *mlir.Context, tmaDe } /// `nvvm.cp.async.bulk.tensor.prefetch`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_tensor_prefetch(ctx: *mlir.Context, tmaDescriptor: *const mlir.Value, coordinates: []const *const mlir.Value, im2colOffsets: []const *const mlir.Value, l2CacheHint: ?*const mlir.Value, mode: ?*const mlir.Attribute, evict: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn cp_async_bulk_tensor_prefetch( + ctx: *mlir.Context, + tmaDescriptor: *const mlir.Value, + coordinates: []const *const mlir.Value, + im2colOffsets: []const *const mlir.Value, + opts: struct { + l2CacheHint: ?*const mlir.Value = null, + mode: ?*const mlir.Attribute = null, + evict: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); - if (evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.prefetch", .{ .operands = .{ .variadic = &.{ &.{tmaDescriptor}, coordinates, im2colOffsets, - if (l2CacheHint) |value| &.{value} else &.{}, + if (opts.l2CacheHint) |value| &.{value} else &.{}, } }, .attributes = attributes.constSlice(), .location = location, @@ -4208,10 +4526,25 @@ pub fn cp_async_bulk_tensor_prefetch(ctx: *mlir.Context, tmaDescriptor: *const m } /// `nvvm.cp.async.bulk.tensor.prefetch.override`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_tensor_prefetch_override(ctx: *mlir.Context, tmaDesc: *const mlir.Value, overrideAdrr: *const mlir.Value, coordinates: []const *const mlir.Value, tensorSize: []const *const mlir.Value, lowerStride: []const *const mlir.Value, upperStride: ?*const mlir.Value, im2colOffsets: []const *const mlir.Value, l2CacheHint: ?*const mlir.Value, evict: ?*const mlir.Attribute, mode: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn cp_async_bulk_tensor_prefetch_override( + ctx: *mlir.Context, + tmaDesc: *const mlir.Value, + overrideAdrr: *const mlir.Value, + coordinates: []const *const mlir.Value, + tensorSize: []const *const mlir.Value, + lowerStride: []const *const mlir.Value, + im2colOffsets: []const *const mlir.Value, + opts: struct { + upperStride: ?*const mlir.Value = null, + l2CacheHint: ?*const mlir.Value = null, + evict: ?*const mlir.Attribute = null, + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); - if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.prefetch.override", .{ .operands = .{ .variadic = &.{ &.{tmaDesc}, @@ -4219,9 +4552,9 @@ pub fn cp_async_bulk_tensor_prefetch_override(ctx: *mlir.Context, tmaDesc: *cons coordinates, tensorSize, lowerStride, - if (upperStride) |value| &.{value} else &.{}, + if (opts.upperStride) |value| &.{value} else &.{}, im2colOffsets, - if (l2CacheHint) |value| &.{value} else &.{}, + if (opts.l2CacheHint) |value| &.{value} else &.{}, } }, .attributes = attributes.constSlice(), .location = location, @@ -4229,16 +4562,27 @@ pub fn cp_async_bulk_tensor_prefetch_override(ctx: *mlir.Context, tmaDesc: *cons } /// `nvvm.cp.async.bulk.tensor.reduce`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_tensor_reduce(ctx: *mlir.Context, tmaDescriptor: *const mlir.Value, srcMem: *const mlir.Value, coordinates: []const *const mlir.Value, l2CacheHint: ?*const mlir.Value, redKind: *const mlir.Attribute, mode: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn cp_async_bulk_tensor_reduce( + ctx: *mlir.Context, + tmaDescriptor: *const mlir.Value, + srcMem: *const mlir.Value, + coordinates: []const *const mlir.Value, + redKind: *const mlir.Attribute, + opts: struct { + l2CacheHint: ?*const mlir.Value = null, + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; attributes.appendAssumeCapacity(.named(ctx, "redKind", redKind)); - if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.reduce", .{ .operands = .{ .variadic = &.{ &.{tmaDescriptor}, &.{srcMem}, coordinates, - if (l2CacheHint) |value| &.{value} else &.{}, + if (opts.l2CacheHint) |value| &.{value} else &.{}, } }, .attributes = attributes.constSlice(), .location = location, @@ -4246,10 +4590,25 @@ pub fn cp_async_bulk_tensor_reduce(ctx: *mlir.Context, tmaDescriptor: *const mli } /// `nvvm.cp.async.bulk.tensor.reduce.override`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_tensor_reduce_override(ctx: *mlir.Context, tmaDesc: *const mlir.Value, srcMem: *const mlir.Value, overrideAdrr: *const mlir.Value, coordinates: []const *const mlir.Value, tensorSize: []const *const mlir.Value, lowerStride: []const *const mlir.Value, upperStride: ?*const mlir.Value, l2CacheHint: ?*const mlir.Value, redKind: *const mlir.Attribute, mode: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn cp_async_bulk_tensor_reduce_override( + ctx: *mlir.Context, + tmaDesc: *const mlir.Value, + srcMem: *const mlir.Value, + overrideAdrr: *const mlir.Value, + coordinates: []const *const mlir.Value, + tensorSize: []const *const mlir.Value, + lowerStride: []const *const mlir.Value, + redKind: *const mlir.Attribute, + opts: struct { + upperStride: ?*const mlir.Value = null, + l2CacheHint: ?*const mlir.Value = null, + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; attributes.appendAssumeCapacity(.named(ctx, "redKind", redKind)); - if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.reduce.override", .{ .operands = .{ .variadic = &.{ &.{tmaDesc}, @@ -4258,8 +4617,8 @@ pub fn cp_async_bulk_tensor_reduce_override(ctx: *mlir.Context, tmaDesc: *const coordinates, tensorSize, lowerStride, - if (upperStride) |value| &.{value} else &.{}, - if (l2CacheHint) |value| &.{value} else &.{}, + if (opts.upperStride) |value| &.{value} else &.{}, + if (opts.l2CacheHint) |value| &.{value} else &.{}, } }, .attributes = attributes.constSlice(), .location = location, @@ -4267,12 +4626,29 @@ pub fn cp_async_bulk_tensor_reduce_override(ctx: *mlir.Context, tmaDesc: *const } /// `nvvm.cp.async.bulk.tensor.shared.cluster.global`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_tensor_shared_cluster_global(ctx: *mlir.Context, dstMem: *const mlir.Value, tmaDescriptor: *const mlir.Value, coordinates: []const *const mlir.Value, mbar: *const mlir.Value, im2colOffsets: []const *const mlir.Value, multicastMask: ?*const mlir.Value, l2CacheHint: ?*const mlir.Value, predicate: ?*const mlir.Value, validatePattern: ?*const mlir.Attribute, mode: ?*const mlir.Attribute, isCTAOnly: ?*const mlir.Attribute, group: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn cp_async_bulk_tensor_shared_cluster_global( + ctx: *mlir.Context, + dstMem: *const mlir.Value, + tmaDescriptor: *const mlir.Value, + coordinates: []const *const mlir.Value, + mbar: *const mlir.Value, + im2colOffsets: []const *const mlir.Value, + opts: struct { + multicastMask: ?*const mlir.Value = null, + l2CacheHint: ?*const mlir.Value = null, + predicate: ?*const mlir.Value = null, + validatePattern: ?*const mlir.Attribute = null, + mode: ?*const mlir.Attribute = null, + isCTAOnly: ?*const mlir.Attribute = null, + group: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - if (validatePattern) |value| attributes.appendAssumeCapacity(.named(ctx, "validatePattern", value)); - if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); - if (isCTAOnly) |value| attributes.appendAssumeCapacity(.named(ctx, "isCTAOnly", value)); - if (group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + if (opts.validatePattern) |value| attributes.appendAssumeCapacity(.named(ctx, "validatePattern", value)); + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + if (opts.isCTAOnly) |value| attributes.appendAssumeCapacity(.named(ctx, "isCTAOnly", value)); + if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.shared.cluster.global", .{ .operands = .{ .variadic = &.{ &.{dstMem}, @@ -4280,9 +4656,9 @@ pub fn cp_async_bulk_tensor_shared_cluster_global(ctx: *mlir.Context, dstMem: *c coordinates, &.{mbar}, im2colOffsets, - if (multicastMask) |value| &.{value} else &.{}, - if (l2CacheHint) |value| &.{value} else &.{}, - if (predicate) |value| &.{value} else &.{}, + if (opts.multicastMask) |value| &.{value} else &.{}, + if (opts.l2CacheHint) |value| &.{value} else &.{}, + if (opts.predicate) |value| &.{value} else &.{}, } }, .attributes = attributes.constSlice(), .location = location, @@ -4290,11 +4666,30 @@ pub fn cp_async_bulk_tensor_shared_cluster_global(ctx: *mlir.Context, dstMem: *c } /// `nvvm.cp.async.bulk.tensor.shared.cluster.global.override`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_tensor_shared_cluster_global_override(ctx: *mlir.Context, dstMem: *const mlir.Value, tmaDesc: *const mlir.Value, mbar: *const mlir.Value, overrideAddr: *const mlir.Value, coordinates: []const *const mlir.Value, im2colArgs: []const *const mlir.Value, tensorSize: []const *const mlir.Value, lowerStride: []const *const mlir.Value, upperStride: ?*const mlir.Value, multicastMask: ?*const mlir.Value, l2CacheHint: ?*const mlir.Value, group: ?*const mlir.Attribute, validatePattern: ?*const mlir.Attribute, mode: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn cp_async_bulk_tensor_shared_cluster_global_override( + ctx: *mlir.Context, + dstMem: *const mlir.Value, + tmaDesc: *const mlir.Value, + mbar: *const mlir.Value, + overrideAddr: *const mlir.Value, + coordinates: []const *const mlir.Value, + im2colArgs: []const *const mlir.Value, + tensorSize: []const *const mlir.Value, + lowerStride: []const *const mlir.Value, + opts: struct { + upperStride: ?*const mlir.Value = null, + multicastMask: ?*const mlir.Value = null, + l2CacheHint: ?*const mlir.Value = null, + group: ?*const mlir.Attribute = null, + validatePattern: ?*const mlir.Attribute = null, + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); - if (validatePattern) |value| attributes.appendAssumeCapacity(.named(ctx, "validatePattern", value)); - if (mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + if (opts.validatePattern) |value| attributes.appendAssumeCapacity(.named(ctx, "validatePattern", value)); + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.shared.cluster.global.override", .{ .operands = .{ .variadic = &.{ &.{dstMem}, @@ -4305,9 +4700,9 @@ pub fn cp_async_bulk_tensor_shared_cluster_global_override(ctx: *mlir.Context, d im2colArgs, tensorSize, lowerStride, - if (upperStride) |value| &.{value} else &.{}, - if (multicastMask) |value| &.{value} else &.{}, - if (l2CacheHint) |value| &.{value} else &.{}, + if (opts.upperStride) |value| &.{value} else &.{}, + if (opts.multicastMask) |value| &.{value} else &.{}, + if (opts.l2CacheHint) |value| &.{value} else &.{}, } }, .attributes = attributes.constSlice(), .location = location, @@ -4315,10 +4710,17 @@ pub fn cp_async_bulk_tensor_shared_cluster_global_override(ctx: *mlir.Context, d } /// `nvvm.cp.async.bulk.wait_group`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_wait_group(ctx: *mlir.Context, group: *const mlir.Attribute, read: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn cp_async_bulk_wait_group( + ctx: *mlir.Context, + group: *const mlir.Attribute, + opts: struct { + read: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; attributes.appendAssumeCapacity(.named(ctx, "group", group)); - if (read) |value| attributes.appendAssumeCapacity(.named(ctx, "read", value)); + if (opts.read) |value| attributes.appendAssumeCapacity(.named(ctx, "read", value)); return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.wait_group", .{ .attributes = attributes.constSlice(), .location = location, @@ -4333,9 +4735,16 @@ pub fn cp_async_commit_group(ctx: *mlir.Context, location: *const mlir.Location) } /// `nvvm.cp.async.mbarrier.arrive`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_mbarrier_arrive(ctx: *mlir.Context, addr: *const mlir.Value, noinc: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn cp_async_mbarrier_arrive( + ctx: *mlir.Context, + addr: *const mlir.Value, + opts: struct { + noinc: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (noinc) |value| attributes.appendAssumeCapacity(.named(ctx, "noinc", value)); + if (opts.noinc) |value| attributes.appendAssumeCapacity(.named(ctx, "noinc", value)); return mlir.Operation.make(ctx, "nvvm.cp.async.mbarrier.arrive", .{ .operands = .{ .flat = &.{addr} }, .attributes = attributes.constSlice(), @@ -4344,9 +4753,19 @@ pub fn cp_async_mbarrier_arrive(ctx: *mlir.Context, addr: *const mlir.Value, noi } /// `nvvm.cp.async.shared.global`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_shared_global(ctx: *mlir.Context, dst: *const mlir.Value, src: *const mlir.Value, cpSize: ?*const mlir.Value, size: *const mlir.Attribute, modifier: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn cp_async_shared_global( + ctx: *mlir.Context, + dst: *const mlir.Value, + src: *const mlir.Value, + size: *const mlir.Attribute, + modifier: *const mlir.Attribute, + opts: struct { + cpSize: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "nvvm.cp.async.shared.global", .{ - .operands = .{ .flat = if (cpSize) |value| &.{ dst, src, value } else &.{ dst, src } }, + .operands = .{ .flat = if (opts.cpSize) |value| &.{ dst, src, value } else &.{ dst, src } }, .attributes = &.{ .named(ctx, "size", size), .named(ctx, "modifier", modifier), @@ -4366,14 +4785,28 @@ pub fn cp_async_wait_group(ctx: *mlir.Context, n: *const mlir.Attribute, locatio } /// `nvvm.cvt.packfloat`. Result types are explicit; attributes use mlir.Attribute. -pub fn cvt_packfloat(ctx: *mlir.Context, srcA: *const mlir.Value, srcC: *const mlir.Value, res_type: *const mlir.Type, from: *const mlir.Attribute, to: *const mlir.Attribute, rnd: ?*const mlir.Attribute, sat: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, extractHi: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn cvt_packfloat( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcC: *const mlir.Value, + res_type: *const mlir.Type, + from: *const mlir.Attribute, + to: *const mlir.Attribute, + opts: struct { + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + extractHi: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; attributes.appendAssumeCapacity(.named(ctx, "from", from)); attributes.appendAssumeCapacity(.named(ctx, "to", to)); - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - if (extractHi) |value| attributes.appendAssumeCapacity(.named(ctx, "extractHi", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.extractHi) |value| attributes.appendAssumeCapacity(.named(ctx, "extractHi", value)); return mlir.Operation.make(ctx, "nvvm.cvt.packfloat", .{ .operands = .{ .flat = &.{ srcA, srcC } }, .results = .{ .flat = &.{res_type} }, @@ -4383,13 +4816,27 @@ pub fn cvt_packfloat(ctx: *mlir.Context, srcA: *const mlir.Value, srcC: *const m } /// `nvvm.cvt.packfloat.f32`. Result types are explicit; attributes use mlir.Attribute. -pub fn cvt_packfloat_f32(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, srcC: *const mlir.Value, res_type: *const mlir.Type, to: *const mlir.Attribute, rnd: ?*const mlir.Attribute, sat: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, extractHi: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn cvt_packfloat_f32( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + srcC: *const mlir.Value, + res_type: *const mlir.Type, + to: *const mlir.Attribute, + opts: struct { + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + extractHi: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; attributes.appendAssumeCapacity(.named(ctx, "to", to)); - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - if (extractHi) |value| attributes.appendAssumeCapacity(.named(ctx, "extractHi", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.extractHi) |value| attributes.appendAssumeCapacity(.named(ctx, "extractHi", value)); return mlir.Operation.make(ctx, "nvvm.cvt.packfloat.f32", .{ .operands = .{ .flat = &.{ srcA, srcB, srcC } }, .results = .{ .flat = &.{res_type} }, @@ -4399,9 +4846,19 @@ pub fn cvt_packfloat_f32(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *con } /// `nvvm.cvt.to.f4x8.packed`. Result types are explicit; attributes use mlir.Attribute. -pub fn cvt_to_f4x8_packed(ctx: *mlir.Context, srcs: []const *const mlir.Value, dst_type: *const mlir.Type, rnd: ?*const mlir.Attribute, srcType: *const mlir.Attribute, dstType: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn cvt_to_f4x8_packed( + ctx: *mlir.Context, + srcs: []const *const mlir.Value, + dst_type: *const mlir.Type, + srcType: *const mlir.Attribute, + dstType: *const mlir.Attribute, + opts: struct { + rnd: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); attributes.appendAssumeCapacity(.named(ctx, "srcType", srcType)); attributes.appendAssumeCapacity(.named(ctx, "dstType", dstType)); return mlir.Operation.make(ctx, "nvvm.cvt.to.f4x8.packed", .{ @@ -4440,18 +4897,33 @@ pub fn dot_accumulate_4way(ctx: *mlir.Context, a: *const mlir.Value, b: *const m } /// `nvvm.elect.sync`. Result types are explicit; attributes use mlir.Attribute. -pub fn elect_sync(ctx: *mlir.Context, membermask: ?*const mlir.Value, pred_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { +pub fn elect_sync( + ctx: *mlir.Context, + pred_type: *const mlir.Type, + opts: struct { + membermask: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "nvvm.elect.sync", .{ - .operands = .{ .flat = if (membermask) |value| &.{value} else &.{} }, + .operands = .{ .flat = if (opts.membermask) |value| &.{value} else &.{} }, .results = .{ .flat = &.{pred_type} }, .location = location, }); } /// `nvvm.ex2`. Result types are explicit; attributes use mlir.Attribute. -pub fn ex2(ctx: *mlir.Context, src: *const mlir.Value, res_type: *const mlir.Type, ftz: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn ex2( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); return mlir.Operation.make(ctx, "nvvm.ex2", .{ .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{res_type} }, @@ -4468,9 +4940,17 @@ pub fn exit(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation } /// `nvvm.fabs`. Result types are explicit; attributes use mlir.Attribute. -pub fn fabs(ctx: *mlir.Context, value_: *const mlir.Value, res_type: *const mlir.Type, ftz: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn fabs( + ctx: *mlir.Context, + value_: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); return mlir.Operation.make(ctx, "nvvm.fabs", .{ .operands = .{ .flat = &.{value_} }, .results = .{ .flat = &.{res_type} }, @@ -4487,10 +4967,17 @@ pub fn fence_mbarrier_init(ctx: *mlir.Context, location: *const mlir.Location) * } /// `nvvm.fence.proxy`. Result types are explicit; attributes use mlir.Attribute. -pub fn fence_proxy(ctx: *mlir.Context, kind: *const mlir.Attribute, space: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn fence_proxy( + ctx: *mlir.Context, + kind: *const mlir.Attribute, + opts: struct { + space: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (space) |value| attributes.appendAssumeCapacity(.named(ctx, "space", value)); + if (opts.space) |value| attributes.appendAssumeCapacity(.named(ctx, "space", value)); return mlir.Operation.make(ctx, "nvvm.fence.proxy", .{ .attributes = attributes.constSlice(), .location = location, @@ -4498,11 +4985,21 @@ pub fn fence_proxy(ctx: *mlir.Context, kind: *const mlir.Attribute, space: ?*con } /// `nvvm.fence.proxy.acquire`. Result types are explicit; attributes use mlir.Attribute. -pub fn fence_proxy_acquire(ctx: *mlir.Context, addr: *const mlir.Value, size: *const mlir.Value, scope: *const mlir.Attribute, fromProxy: ?*const mlir.Attribute, toProxy: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn fence_proxy_acquire( + ctx: *mlir.Context, + addr: *const mlir.Value, + size: *const mlir.Value, + scope: *const mlir.Attribute, + opts: struct { + fromProxy: ?*const mlir.Attribute = null, + toProxy: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; attributes.appendAssumeCapacity(.named(ctx, "scope", scope)); - if (fromProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "fromProxy", value)); - if (toProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "toProxy", value)); + if (opts.fromProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "fromProxy", value)); + if (opts.toProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "toProxy", value)); return mlir.Operation.make(ctx, "nvvm.fence.proxy.acquire", .{ .operands = .{ .flat = &.{ addr, size } }, .attributes = attributes.constSlice(), @@ -4511,11 +5008,19 @@ pub fn fence_proxy_acquire(ctx: *mlir.Context, addr: *const mlir.Value, size: *c } /// `nvvm.fence.proxy.release`. Result types are explicit; attributes use mlir.Attribute. -pub fn fence_proxy_release(ctx: *mlir.Context, scope: *const mlir.Attribute, fromProxy: ?*const mlir.Attribute, toProxy: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn fence_proxy_release( + ctx: *mlir.Context, + scope: *const mlir.Attribute, + opts: struct { + fromProxy: ?*const mlir.Attribute = null, + toProxy: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; attributes.appendAssumeCapacity(.named(ctx, "scope", scope)); - if (fromProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "fromProxy", value)); - if (toProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "toProxy", value)); + if (opts.fromProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "fromProxy", value)); + if (opts.toProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "toProxy", value)); return mlir.Operation.make(ctx, "nvvm.fence.proxy.release", .{ .attributes = attributes.constSlice(), .location = location, @@ -4523,11 +5028,19 @@ pub fn fence_proxy_release(ctx: *mlir.Context, scope: *const mlir.Attribute, fro } /// `nvvm.fence.proxy.sync_restrict`. Result types are explicit; attributes use mlir.Attribute. -pub fn fence_proxy_sync_restrict(ctx: *mlir.Context, order: *const mlir.Attribute, fromProxy: ?*const mlir.Attribute, toProxy: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn fence_proxy_sync_restrict( + ctx: *mlir.Context, + order: *const mlir.Attribute, + opts: struct { + fromProxy: ?*const mlir.Attribute = null, + toProxy: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; attributes.appendAssumeCapacity(.named(ctx, "order", order)); - if (fromProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "fromProxy", value)); - if (toProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "toProxy", value)); + if (opts.fromProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "fromProxy", value)); + if (opts.toProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "toProxy", value)); return mlir.Operation.make(ctx, "nvvm.fence.proxy.sync_restrict", .{ .attributes = attributes.constSlice(), .location = location, @@ -4552,13 +5065,27 @@ pub fn fence_sync_restrict(ctx: *mlir.Context, order: *const mlir.Attribute, loc } /// `nvvm.fma`. Result types are explicit; attributes use mlir.Attribute. -pub fn fma(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, c_: *const mlir.Value, res_type: *const mlir.Type, rnd: *const mlir.Attribute, sat: ?*const mlir.Attribute, ftz: ?*const mlir.Attribute, relu: ?*const mlir.Attribute, oob: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn fma( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c_: *const mlir.Value, + res_type: *const mlir.Type, + rnd: *const mlir.Attribute, + opts: struct { + sat: ?*const mlir.Attribute = null, + ftz: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + oob: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; attributes.appendAssumeCapacity(.named(ctx, "rnd", rnd)); - if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); - if (relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - if (oob) |value| attributes.appendAssumeCapacity(.named(ctx, "oob", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.oob) |value| attributes.appendAssumeCapacity(.named(ctx, "oob", value)); return mlir.Operation.make(ctx, "nvvm.fma", .{ .operands = .{ .flat = &.{ a, b, c_ } }, .results = .{ .flat = &.{res_type} }, @@ -4568,10 +5095,21 @@ pub fn fma(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, c_: * } /// `nvvm.fma.packed.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn fma_packed_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, srcC: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, ftz: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn fma_packed_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + srcC: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); return mlir.Operation.make(ctx, "nvvm.fma.packed.f32x2", .{ .operands = .{ .flat = &.{ srcA, srcB, srcC } }, .results = .{ .flat = &.{res_type} }, @@ -4581,9 +5119,19 @@ pub fn fma_packed_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *cons } /// `nvvm.fma.packed.f32x2.bf16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn fma_packed_f32x2_bf16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, srcC: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn fma_packed_f32x2_bf16x2_f32x2_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + srcC: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); return mlir.Operation.make(ctx, "nvvm.fma.packed.f32x2.bf16x2.f32x2.f32x2", .{ .operands = .{ .flat = &.{ srcA, srcB, srcC } }, .results = .{ .flat = &.{res_type} }, @@ -4593,9 +5141,19 @@ pub fn fma_packed_f32x2_bf16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir } /// `nvvm.fma.packed.f32x2.f16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn fma_packed_f32x2_f16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, srcC: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn fma_packed_f32x2_f16x2_f32x2_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + srcC: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); return mlir.Operation.make(ctx, "nvvm.fma.packed.f32x2.f16x2.f32x2.f32x2", .{ .operands = .{ .flat = &.{ srcA, srcB, srcC } }, .results = .{ .flat = &.{res_type} }, @@ -4605,13 +5163,25 @@ pub fn fma_packed_f32x2_f16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir. } /// `nvvm.fmax`. Result types are explicit; attributes use mlir.Attribute. -pub fn fmax(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, c_: ?*const mlir.Value, res_type: *const mlir.Type, ftz: ?*const mlir.Attribute, nan: ?*const mlir.Attribute, abs: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn fmax( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + c_: ?*const mlir.Value = null, + ftz: ?*const mlir.Attribute = null, + nan: ?*const mlir.Attribute = null, + abs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); - if (nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); - if (abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (opts.nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); + if (opts.abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); return mlir.Operation.make(ctx, "nvvm.fmax", .{ - .operands = .{ .flat = if (c_) |value| &.{ a, b, value } else &.{ a, b } }, + .operands = .{ .flat = if (opts.c_) |value| &.{ a, b, value } else &.{ a, b } }, .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), .location = location, @@ -4619,13 +5189,25 @@ pub fn fmax(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, c_: } /// `nvvm.fmin`. Result types are explicit; attributes use mlir.Attribute. -pub fn fmin(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, c_: ?*const mlir.Value, res_type: *const mlir.Type, ftz: ?*const mlir.Attribute, nan: ?*const mlir.Attribute, abs: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn fmin( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + c_: ?*const mlir.Value = null, + ftz: ?*const mlir.Attribute = null, + nan: ?*const mlir.Attribute = null, + abs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); - if (nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); - if (abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (opts.nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); + if (opts.abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); return mlir.Operation.make(ctx, "nvvm.fmin", .{ - .operands = .{ .flat = if (c_) |value| &.{ a, b, value } else &.{ a, b } }, + .operands = .{ .flat = if (opts.c_) |value| &.{ a, b, value } else &.{ a, b } }, .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), .location = location, @@ -4643,14 +5225,24 @@ pub fn griddepcontrol(ctx: *mlir.Context, kind: *const mlir.Attribute, location: } /// `nvvm.inline_ptx`. Result types are explicit; attributes use mlir.Attribute. -pub fn inline_ptx(ctx: *mlir.Context, readOnlyArgs: []const *const mlir.Value, readWriteArgs: []const *const mlir.Value, predicate: ?*const mlir.Value, writeOnlyArgs_type: ?*const mlir.Type, ptxCode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn inline_ptx( + ctx: *mlir.Context, + readOnlyArgs: []const *const mlir.Value, + readWriteArgs: []const *const mlir.Value, + ptxCode: *const mlir.Attribute, + opts: struct { + predicate: ?*const mlir.Value = null, + writeOnlyArgs_type: ?*const mlir.Type = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; - if (writeOnlyArgs_type) |value| results.appendAssumeCapacity(value); + if (opts.writeOnlyArgs_type) |value| results.appendAssumeCapacity(value); return mlir.Operation.make(ctx, "nvvm.inline_ptx", .{ .operands = .{ .variadic = &.{ readOnlyArgs, readWriteArgs, - if (predicate) |value| &.{value} else &.{}, + if (opts.predicate) |value| &.{value} else &.{}, } }, .results = .{ .flat = results.constSlice() }, .attributes = &.{ @@ -4661,12 +5253,23 @@ pub fn inline_ptx(ctx: *mlir.Context, readOnlyArgs: []const *const mlir.Value, r } /// `nvvm.ldmatrix`. Result types are explicit; attributes use mlir.Attribute. -pub fn ldmatrix(ctx: *mlir.Context, ptr: *const mlir.Value, res_type: *const mlir.Type, num: *const mlir.Attribute, layout: *const mlir.Attribute, shape: ?*const mlir.Attribute, srcFormat: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn ldmatrix( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + num: *const mlir.Attribute, + layout: *const mlir.Attribute, + opts: struct { + shape: ?*const mlir.Attribute = null, + srcFormat: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; attributes.appendAssumeCapacity(.named(ctx, "num", num)); attributes.appendAssumeCapacity(.named(ctx, "layout", layout)); - if (shape) |value| attributes.appendAssumeCapacity(.named(ctx, "shape", value)); - if (srcFormat) |value| attributes.appendAssumeCapacity(.named(ctx, "srcFormat", value)); + if (opts.shape) |value| attributes.appendAssumeCapacity(.named(ctx, "shape", value)); + if (opts.srcFormat) |value| attributes.appendAssumeCapacity(.named(ctx, "srcFormat", value)); return mlir.Operation.make(ctx, "nvvm.ldmatrix", .{ .operands = .{ .flat = &.{ptr} }, .results = .{ .flat = &.{res_type} }, @@ -4676,17 +5279,32 @@ pub fn ldmatrix(ctx: *mlir.Context, ptr: *const mlir.Value, res_type: *const mli } /// `nvvm.load.ext`. Result types are explicit; attributes use mlir.Attribute. -pub fn load_ext(ctx: *mlir.Context, addr: *const mlir.Value, l2CacheHint: ?*const mlir.Value, res_type: *const mlir.Type, order: ?*const mlir.Attribute, scope: ?*const mlir.Attribute, prefetch_: ?*const mlir.Attribute, evict: ?*const mlir.Attribute, cacheModifier: ?*const mlir.Attribute, sharedSpace: ?*const mlir.Attribute, unified: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn load_ext( + ctx: *mlir.Context, + addr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + l2CacheHint: ?*const mlir.Value = null, + order: ?*const mlir.Attribute = null, + scope: ?*const mlir.Attribute = null, + prefetch_: ?*const mlir.Attribute = null, + evict: ?*const mlir.Attribute = null, + cacheModifier: ?*const mlir.Attribute = null, + sharedSpace: ?*const mlir.Attribute = null, + unified: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; - if (order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); - if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (prefetch_) |value| attributes.appendAssumeCapacity(.named(ctx, "prefetch", value)); - if (evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); - if (cacheModifier) |value| attributes.appendAssumeCapacity(.named(ctx, "cacheModifier", value)); - if (sharedSpace) |value| attributes.appendAssumeCapacity(.named(ctx, "sharedSpace", value)); - if (unified) |value| attributes.appendAssumeCapacity(.named(ctx, "unified", value)); + if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.prefetch_) |value| attributes.appendAssumeCapacity(.named(ctx, "prefetch", value)); + if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + if (opts.cacheModifier) |value| attributes.appendAssumeCapacity(.named(ctx, "cacheModifier", value)); + if (opts.sharedSpace) |value| attributes.appendAssumeCapacity(.named(ctx, "sharedSpace", value)); + if (opts.unified) |value| attributes.appendAssumeCapacity(.named(ctx, "unified", value)); return mlir.Operation.make(ctx, "nvvm.load.ext", .{ - .operands = .{ .flat = if (l2CacheHint) |value| &.{ addr, value } else &.{addr} }, + .operands = .{ .flat = if (opts.l2CacheHint) |value| &.{ addr, value } else &.{addr} }, .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), .location = location, @@ -4694,9 +5312,17 @@ pub fn load_ext(ctx: *mlir.Context, addr: *const mlir.Value, l2CacheHint: ?*cons } /// `nvvm.log2`. Result types are explicit; attributes use mlir.Attribute. -pub fn log2(ctx: *mlir.Context, src: *const mlir.Value, res_type: *const mlir.Type, ftz: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn log2( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); return mlir.Operation.make(ctx, "nvvm.log2", .{ .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{res_type} }, @@ -4727,17 +5353,28 @@ pub fn match_sync(ctx: *mlir.Context, thread_mask: *const mlir.Value, val: *cons } /// `nvvm.mbarrier.arrive`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_arrive(ctx: *mlir.Context, addr: *const mlir.Value, count: ?*const mlir.Value, multicastMask: ?*const mlir.Value, res_type: ?*const mlir.Type, scope: ?*const mlir.Attribute, relaxed: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mbarrier_arrive( + ctx: *mlir.Context, + addr: *const mlir.Value, + opts: struct { + count: ?*const mlir.Value = null, + multicastMask: ?*const mlir.Value = null, + res_type: ?*const mlir.Type = null, + scope: ?*const mlir.Attribute = null, + relaxed: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; - if (res_type) |value| results.appendAssumeCapacity(value); + if (opts.res_type) |value| results.appendAssumeCapacity(value); var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive", .{ .operands = .{ .variadic = &.{ &.{addr}, - if (count) |value| &.{value} else &.{}, - if (multicastMask) |value| &.{value} else &.{}, + if (opts.count) |value| &.{value} else &.{}, + if (opts.multicastMask) |value| &.{value} else &.{}, } }, .results = .{ .flat = results.constSlice() }, .attributes = attributes.constSlice(), @@ -4746,18 +5383,30 @@ pub fn mbarrier_arrive(ctx: *mlir.Context, addr: *const mlir.Value, count: ?*con } /// `nvvm.mbarrier.arrive.expect_tx`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_arrive_expect_tx(ctx: *mlir.Context, addr: *const mlir.Value, txcount: *const mlir.Value, multicastMask: ?*const mlir.Value, predicate: ?*const mlir.Value, res_type: ?*const mlir.Type, scope: ?*const mlir.Attribute, relaxed: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mbarrier_arrive_expect_tx( + ctx: *mlir.Context, + addr: *const mlir.Value, + txcount: *const mlir.Value, + opts: struct { + multicastMask: ?*const mlir.Value = null, + predicate: ?*const mlir.Value = null, + res_type: ?*const mlir.Type = null, + scope: ?*const mlir.Attribute = null, + relaxed: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; - if (res_type) |value| results.appendAssumeCapacity(value); + if (opts.res_type) |value| results.appendAssumeCapacity(value); var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive.expect_tx", .{ .operands = .{ .variadic = &.{ &.{addr}, &.{txcount}, - if (multicastMask) |value| &.{value} else &.{}, - if (predicate) |value| &.{value} else &.{}, + if (opts.multicastMask) |value| &.{value} else &.{}, + if (opts.predicate) |value| &.{value} else &.{}, } }, .results = .{ .flat = results.constSlice() }, .attributes = attributes.constSlice(), @@ -4775,17 +5424,28 @@ pub fn mbarrier_arrive_nocomplete(ctx: *mlir.Context, addr: *const mlir.Value, c } /// `nvvm.mbarrier.arrive_drop`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_arrive_drop(ctx: *mlir.Context, addr: *const mlir.Value, count: ?*const mlir.Value, multicastMask: ?*const mlir.Value, res_type: ?*const mlir.Type, scope: ?*const mlir.Attribute, relaxed: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mbarrier_arrive_drop( + ctx: *mlir.Context, + addr: *const mlir.Value, + opts: struct { + count: ?*const mlir.Value = null, + multicastMask: ?*const mlir.Value = null, + res_type: ?*const mlir.Type = null, + scope: ?*const mlir.Attribute = null, + relaxed: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; - if (res_type) |value| results.appendAssumeCapacity(value); + if (opts.res_type) |value| results.appendAssumeCapacity(value); var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive_drop", .{ .operands = .{ .variadic = &.{ &.{addr}, - if (count) |value| &.{value} else &.{}, - if (multicastMask) |value| &.{value} else &.{}, + if (opts.count) |value| &.{value} else &.{}, + if (opts.multicastMask) |value| &.{value} else &.{}, } }, .results = .{ .flat = results.constSlice() }, .attributes = attributes.constSlice(), @@ -4794,14 +5454,25 @@ pub fn mbarrier_arrive_drop(ctx: *mlir.Context, addr: *const mlir.Value, count: } /// `nvvm.mbarrier.arrive_drop.expect_tx`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_arrive_drop_expect_tx(ctx: *mlir.Context, addr: *const mlir.Value, txcount: *const mlir.Value, multicastMask: ?*const mlir.Value, res_type: ?*const mlir.Type, scope: ?*const mlir.Attribute, relaxed: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mbarrier_arrive_drop_expect_tx( + ctx: *mlir.Context, + addr: *const mlir.Value, + txcount: *const mlir.Value, + opts: struct { + multicastMask: ?*const mlir.Value = null, + res_type: ?*const mlir.Type = null, + scope: ?*const mlir.Attribute = null, + relaxed: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; - if (res_type) |value| results.appendAssumeCapacity(value); + if (opts.res_type) |value| results.appendAssumeCapacity(value); var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive_drop.expect_tx", .{ - .operands = .{ .flat = if (multicastMask) |value| &.{ addr, txcount, value } else &.{ addr, txcount } }, + .operands = .{ .flat = if (opts.multicastMask) |value| &.{ addr, txcount, value } else &.{ addr, txcount } }, .results = .{ .flat = results.constSlice() }, .attributes = attributes.constSlice(), .location = location, @@ -4830,15 +5501,25 @@ pub fn mbarrier_check_layout(ctx: *mlir.Context, addr: *const mlir.Value, res_ty } /// `nvvm.mbarrier.complete_tx`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_complete_tx(ctx: *mlir.Context, addr: *const mlir.Value, txcount: *const mlir.Value, report: ?*const mlir.Value, multicastMask: ?*const mlir.Value, scope: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mbarrier_complete_tx( + ctx: *mlir.Context, + addr: *const mlir.Value, + txcount: *const mlir.Value, + opts: struct { + report: ?*const mlir.Value = null, + multicastMask: ?*const mlir.Value = null, + scope: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); return mlir.Operation.make(ctx, "nvvm.mbarrier.complete_tx", .{ .operands = .{ .variadic = &.{ &.{addr}, &.{txcount}, - if (report) |value| &.{value} else &.{}, - if (multicastMask) |value| &.{value} else &.{}, + if (opts.report) |value| &.{value} else &.{}, + if (opts.multicastMask) |value| &.{value} else &.{}, } }, .attributes = attributes.constSlice(), .location = location, @@ -4846,22 +5527,40 @@ pub fn mbarrier_complete_tx(ctx: *mlir.Context, addr: *const mlir.Value, txcount } /// `nvvm.mbarrier.expect_tx`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_expect_tx(ctx: *mlir.Context, addr: *const mlir.Value, txcount: *const mlir.Value, multicastMask: ?*const mlir.Value, scope: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mbarrier_expect_tx( + ctx: *mlir.Context, + addr: *const mlir.Value, + txcount: *const mlir.Value, + opts: struct { + multicastMask: ?*const mlir.Value = null, + scope: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); return mlir.Operation.make(ctx, "nvvm.mbarrier.expect_tx", .{ - .operands = .{ .flat = if (multicastMask) |value| &.{ addr, txcount, value } else &.{ addr, txcount } }, + .operands = .{ .flat = if (opts.multicastMask) |value| &.{ addr, txcount, value } else &.{ addr, txcount } }, .attributes = attributes.constSlice(), .location = location, }); } /// `nvvm.mbarrier.init`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_init(ctx: *mlir.Context, addr: *const mlir.Value, count: *const mlir.Value, predicate: ?*const mlir.Value, layout: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mbarrier_init( + ctx: *mlir.Context, + addr: *const mlir.Value, + count: *const mlir.Value, + opts: struct { + predicate: ?*const mlir.Value = null, + layout: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (layout) |value| attributes.appendAssumeCapacity(.named(ctx, "layout", value)); + if (opts.layout) |value| attributes.appendAssumeCapacity(.named(ctx, "layout", value)); return mlir.Operation.make(ctx, "nvvm.mbarrier.init", .{ - .operands = .{ .flat = if (predicate) |value| &.{ addr, count, value } else &.{ addr, count } }, + .operands = .{ .flat = if (opts.predicate) |value| &.{ addr, count, value } else &.{ addr, count } }, .attributes = attributes.constSlice(), .location = location, }); @@ -4876,11 +5575,22 @@ pub fn mbarrier_inval(ctx: *mlir.Context, addr: *const mlir.Value, location: *co } /// `nvvm.mbarrier.test.wait`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_test_wait(ctx: *mlir.Context, addr: *const mlir.Value, stateOrPhase: *const mlir.Value, res_type: *const mlir.Type, scope: ?*const mlir.Attribute, relaxed: ?*const mlir.Attribute, mbarrierPhase: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mbarrier_test_wait( + ctx: *mlir.Context, + addr: *const mlir.Value, + stateOrPhase: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + scope: ?*const mlir.Attribute = null, + relaxed: ?*const mlir.Attribute = null, + mbarrierPhase: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); - if (mbarrierPhase) |value| attributes.appendAssumeCapacity(.named(ctx, "mbarrierPhase", value)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); + if (opts.mbarrierPhase) |value| attributes.appendAssumeCapacity(.named(ctx, "mbarrierPhase", value)); return mlir.Operation.make(ctx, "nvvm.mbarrier.test.wait", .{ .operands = .{ .flat = &.{ addr, stateOrPhase } }, .results = .{ .flat = &.{res_type} }, @@ -4890,13 +5600,25 @@ pub fn mbarrier_test_wait(ctx: *mlir.Context, addr: *const mlir.Value, stateOrPh } /// `nvvm.mbarrier.try_wait`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_try_wait(ctx: *mlir.Context, addr: *const mlir.Value, stateOrPhase: *const mlir.Value, ticks: ?*const mlir.Value, res_type: *const mlir.Type, scope: ?*const mlir.Attribute, relaxed: ?*const mlir.Attribute, mbarrierPhase: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mbarrier_try_wait( + ctx: *mlir.Context, + addr: *const mlir.Value, + stateOrPhase: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + ticks: ?*const mlir.Value = null, + scope: ?*const mlir.Attribute = null, + relaxed: ?*const mlir.Attribute = null, + mbarrierPhase: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); - if (mbarrierPhase) |value| attributes.appendAssumeCapacity(.named(ctx, "mbarrierPhase", value)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); + if (opts.mbarrierPhase) |value| attributes.appendAssumeCapacity(.named(ctx, "mbarrierPhase", value)); return mlir.Operation.make(ctx, "nvvm.mbarrier.try_wait", .{ - .operands = .{ .flat = if (ticks) |value| &.{ addr, stateOrPhase, value } else &.{ addr, stateOrPhase } }, + .operands = .{ .flat = if (opts.ticks) |value| &.{ addr, stateOrPhase, value } else &.{ addr, stateOrPhase } }, .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), .location = location, @@ -4904,9 +5626,18 @@ pub fn mbarrier_try_wait(ctx: *mlir.Context, addr: *const mlir.Value, stateOrPha } /// `nvvm.mbarrier.try_wait.parity`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_try_wait_parity(ctx: *mlir.Context, addr: *const mlir.Value, phase: *const mlir.Value, ticks: *const mlir.Value, useIntrinsic: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mbarrier_try_wait_parity( + ctx: *mlir.Context, + addr: *const mlir.Value, + phase: *const mlir.Value, + ticks: *const mlir.Value, + opts: struct { + useIntrinsic: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (useIntrinsic) |value| attributes.appendAssumeCapacity(.named(ctx, "useIntrinsic", value)); + if (opts.useIntrinsic) |value| attributes.appendAssumeCapacity(.named(ctx, "useIntrinsic", value)); return mlir.Operation.make(ctx, "nvvm.mbarrier.try_wait.parity", .{ .operands = .{ .flat = &.{ addr, phase, ticks } }, .attributes = attributes.constSlice(), @@ -4915,10 +5646,21 @@ pub fn mbarrier_try_wait_parity(ctx: *mlir.Context, addr: *const mlir.Value, pha } /// `nvvm.mbarrier.try_wait.parity.timelimit`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_try_wait_parity_timelimit(ctx: *mlir.Context, addr: *const mlir.Value, phase: *const mlir.Value, timeLimit: *const mlir.Value, res_type: *const mlir.Type, scope: ?*const mlir.Attribute, order: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mbarrier_try_wait_parity_timelimit( + ctx: *mlir.Context, + addr: *const mlir.Value, + phase: *const mlir.Value, + timeLimit: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + scope: ?*const mlir.Attribute = null, + order: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); return mlir.Operation.make(ctx, "nvvm.mbarrier.try_wait.parity.timelimit", .{ .operands = .{ .flat = &.{ addr, phase, timeLimit } }, .results = .{ .flat = &.{res_type} }, @@ -4928,10 +5670,21 @@ pub fn mbarrier_try_wait_parity_timelimit(ctx: *mlir.Context, addr: *const mlir. } /// `nvvm.mbarrier.try_wait.timelimit`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_try_wait_timelimit(ctx: *mlir.Context, addr: *const mlir.Value, state: *const mlir.Value, timeLimit: *const mlir.Value, res_type: *const mlir.Type, scope: ?*const mlir.Attribute, order: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mbarrier_try_wait_timelimit( + ctx: *mlir.Context, + addr: *const mlir.Value, + state: *const mlir.Value, + timeLimit: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + scope: ?*const mlir.Attribute = null, + order: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); return mlir.Operation.make(ctx, "nvvm.mbarrier.try_wait.timelimit", .{ .operands = .{ .flat = &.{ addr, state, timeLimit } }, .results = .{ .flat = &.{res_type} }, @@ -4941,25 +5694,48 @@ pub fn mbarrier_try_wait_timelimit(ctx: *mlir.Context, addr: *const mlir.Value, } /// `nvvm.mbarrier.txn`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_txn(ctx: *mlir.Context, addr: *const mlir.Value, count: *const mlir.Value, multicast: ?*const mlir.Value, kind: *const mlir.Attribute, space: ?*const mlir.Attribute, scope: ?*const mlir.Attribute, order: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mbarrier_txn( + ctx: *mlir.Context, + addr: *const mlir.Value, + count: *const mlir.Value, + kind: *const mlir.Attribute, + opts: struct { + multicast: ?*const mlir.Value = null, + space: ?*const mlir.Attribute = null, + scope: ?*const mlir.Attribute = null, + order: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (space) |value| attributes.appendAssumeCapacity(.named(ctx, "space", value)); - if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + if (opts.space) |value| attributes.appendAssumeCapacity(.named(ctx, "space", value)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); return mlir.Operation.make(ctx, "nvvm.mbarrier.txn", .{ - .operands = .{ .flat = if (multicast) |value| &.{ addr, count, value } else &.{ addr, count } }, + .operands = .{ .flat = if (opts.multicast) |value| &.{ addr, count, value } else &.{ addr, count } }, .attributes = attributes.constSlice(), .location = location, }); } /// `nvvm.mbarrier.txn.cta`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_txn_cta(ctx: *mlir.Context, addr: *const mlir.Value, count: *const mlir.Value, res_type: *const mlir.Type, kind: *const mlir.Attribute, scope: ?*const mlir.Attribute, order: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mbarrier_txn_cta( + ctx: *mlir.Context, + addr: *const mlir.Value, + count: *const mlir.Value, + res_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + scope: ?*const mlir.Attribute = null, + order: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); return mlir.Operation.make(ctx, "nvvm.mbarrier.txn.cta", .{ .operands = .{ .flat = &.{ addr, count } }, .results = .{ .flat = &.{res_type} }, @@ -4969,11 +5745,22 @@ pub fn mbarrier_txn_cta(ctx: *mlir.Context, addr: *const mlir.Value, count: *con } /// `nvvm.mbarrier.wait`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_wait(ctx: *mlir.Context, addr: *const mlir.Value, state: *const mlir.Value, res_type: *const mlir.Type, kind: *const mlir.Attribute, scope: ?*const mlir.Attribute, order: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mbarrier_wait( + ctx: *mlir.Context, + addr: *const mlir.Value, + state: *const mlir.Value, + res_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + scope: ?*const mlir.Attribute = null, + order: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); return mlir.Operation.make(ctx, "nvvm.mbarrier.wait", .{ .operands = .{ .flat = &.{ addr, state } }, .results = .{ .flat = &.{res_type} }, @@ -4983,11 +5770,22 @@ pub fn mbarrier_wait(ctx: *mlir.Context, addr: *const mlir.Value, state: *const } /// `nvvm.mbarrier.wait.parity`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_wait_parity(ctx: *mlir.Context, addr: *const mlir.Value, phase: *const mlir.Value, res_type: *const mlir.Type, kind: *const mlir.Attribute, scope: ?*const mlir.Attribute, order: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mbarrier_wait_parity( + ctx: *mlir.Context, + addr: *const mlir.Value, + phase: *const mlir.Value, + res_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + scope: ?*const mlir.Attribute = null, + order: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); return mlir.Operation.make(ctx, "nvvm.mbarrier.wait.parity", .{ .operands = .{ .flat = &.{ addr, phase } }, .results = .{ .flat = &.{res_type} }, @@ -5007,11 +5805,32 @@ pub fn memory_barrier(ctx: *mlir.Context, scope: *const mlir.Attribute, location } /// `nvvm.mma.block_scale`. Result types are explicit; attributes use mlir.Attribute. -pub fn mma_block_scale(ctx: *mlir.Context, operandA: []const *const mlir.Value, operandB: []const *const mlir.Value, operandC: []const *const mlir.Value, scaleAData: *const mlir.Value, byteIdA: *const mlir.Value, threadIdA: *const mlir.Value, scaleBData: *const mlir.Value, byteIdB: *const mlir.Value, threadIdB: *const mlir.Value, res_type: *const mlir.Type, shape: *const mlir.Attribute, multiplicandAPtxType: ?*const mlir.Attribute, multiplicandBPtxType: ?*const mlir.Attribute, scaleVecSize: *const mlir.Attribute, blockScaleFormat: *const mlir.Attribute, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mma_block_scale( + ctx: *mlir.Context, + operandA: []const *const mlir.Value, + operandB: []const *const mlir.Value, + operandC: []const *const mlir.Value, + scaleAData: *const mlir.Value, + byteIdA: *const mlir.Value, + threadIdA: *const mlir.Value, + scaleBData: *const mlir.Value, + byteIdB: *const mlir.Value, + threadIdB: *const mlir.Value, + res_type: *const mlir.Type, + shape: *const mlir.Attribute, + scaleVecSize: *const mlir.Attribute, + blockScaleFormat: *const mlir.Attribute, + kind: *const mlir.Attribute, + opts: struct { + multiplicandAPtxType: ?*const mlir.Attribute = null, + multiplicandBPtxType: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); - if (multiplicandAPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandAPtxType", value)); - if (multiplicandBPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandBPtxType", value)); + if (opts.multiplicandAPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandAPtxType", value)); + if (opts.multiplicandBPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandBPtxType", value)); attributes.appendAssumeCapacity(.named(ctx, "scaleVecSize", scaleVecSize)); attributes.appendAssumeCapacity(.named(ctx, "blockScaleFormat", blockScaleFormat)); attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); @@ -5034,7 +5853,31 @@ pub fn mma_block_scale(ctx: *mlir.Context, operandA: []const *const mlir.Value, } /// `nvvm.mma.block_scale.internal`. Result types are explicit; attributes use mlir.Attribute. -pub fn mma_block_scale_internal(ctx: *mlir.Context, operandA: *const mlir.Value, scaleAData: *const mlir.Value, byteIdA: *const mlir.Value, threadIdA: *const mlir.Value, operandB: *const mlir.Value, scaleBData: *const mlir.Value, byteIdB: *const mlir.Value, threadIdB: *const mlir.Value, operandC: *const mlir.Value, res_type: *const mlir.Type, shape: *const mlir.Attribute, layoutA: *const mlir.Attribute, layoutB: *const mlir.Attribute, aType: *const mlir.Attribute, bType: *const mlir.Attribute, cType: *const mlir.Attribute, scaleVecSize: *const mlir.Attribute, blockScaleFormat: *const mlir.Attribute, kind: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mma_block_scale_internal( + ctx: *mlir.Context, + operandA: *const mlir.Value, + scaleAData: *const mlir.Value, + byteIdA: *const mlir.Value, + threadIdA: *const mlir.Value, + operandB: *const mlir.Value, + scaleBData: *const mlir.Value, + byteIdB: *const mlir.Value, + threadIdB: *const mlir.Value, + operandC: *const mlir.Value, + res_type: *const mlir.Type, + shape: *const mlir.Attribute, + layoutA: *const mlir.Attribute, + layoutB: *const mlir.Attribute, + aType: *const mlir.Attribute, + bType: *const mlir.Attribute, + cType: *const mlir.Attribute, + scaleVecSize: *const mlir.Attribute, + blockScaleFormat: *const mlir.Attribute, + opts: struct { + kind: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 9) = .empty; attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); attributes.appendAssumeCapacity(.named(ctx, "layoutA", layoutA)); @@ -5044,7 +5887,7 @@ pub fn mma_block_scale_internal(ctx: *mlir.Context, operandA: *const mlir.Value, attributes.appendAssumeCapacity(.named(ctx, "cType", cType)); attributes.appendAssumeCapacity(.named(ctx, "scaleVecSize", scaleVecSize)); attributes.appendAssumeCapacity(.named(ctx, "blockScaleFormat", blockScaleFormat)); - if (kind) |value| attributes.appendAssumeCapacity(.named(ctx, "kind", value)); + if (opts.kind) |value| attributes.appendAssumeCapacity(.named(ctx, "kind", value)); return mlir.Operation.make(ctx, "nvvm.mma.block_scale.internal", .{ .operands = .{ .flat = &.{ operandA, scaleAData, byteIdA, threadIdA, operandB, scaleBData, byteIdB, threadIdB, operandC } }, .results = .{ .flat = &.{res_type} }, @@ -5054,15 +5897,39 @@ pub fn mma_block_scale_internal(ctx: *mlir.Context, operandA: *const mlir.Value, } /// `nvvm.mma.sp.block_scale`. Result types are explicit; attributes use mlir.Attribute. -pub fn mma_sp_block_scale(ctx: *mlir.Context, operandA: []const *const mlir.Value, operandB: []const *const mlir.Value, operandC: []const *const mlir.Value, sparseMetadata: *const mlir.Value, sparsitySelector: *const mlir.Value, scaleAData: *const mlir.Value, byteIdA: *const mlir.Value, threadIdA: *const mlir.Value, scaleBData: *const mlir.Value, byteIdB: *const mlir.Value, threadIdB: *const mlir.Value, res_type: *const mlir.Type, shape: *const mlir.Attribute, multiplicandAPtxType: ?*const mlir.Attribute, multiplicandBPtxType: ?*const mlir.Attribute, scaleVecSize: *const mlir.Attribute, blockScaleFormat: *const mlir.Attribute, kind: *const mlir.Attribute, orderedMetadata: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mma_sp_block_scale( + ctx: *mlir.Context, + operandA: []const *const mlir.Value, + operandB: []const *const mlir.Value, + operandC: []const *const mlir.Value, + sparseMetadata: *const mlir.Value, + sparsitySelector: *const mlir.Value, + scaleAData: *const mlir.Value, + byteIdA: *const mlir.Value, + threadIdA: *const mlir.Value, + scaleBData: *const mlir.Value, + byteIdB: *const mlir.Value, + threadIdB: *const mlir.Value, + res_type: *const mlir.Type, + shape: *const mlir.Attribute, + scaleVecSize: *const mlir.Attribute, + blockScaleFormat: *const mlir.Attribute, + kind: *const mlir.Attribute, + opts: struct { + multiplicandAPtxType: ?*const mlir.Attribute = null, + multiplicandBPtxType: ?*const mlir.Attribute = null, + orderedMetadata: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); - if (multiplicandAPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandAPtxType", value)); - if (multiplicandBPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandBPtxType", value)); + if (opts.multiplicandAPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandAPtxType", value)); + if (opts.multiplicandBPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandBPtxType", value)); attributes.appendAssumeCapacity(.named(ctx, "scaleVecSize", scaleVecSize)); attributes.appendAssumeCapacity(.named(ctx, "blockScaleFormat", blockScaleFormat)); attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (orderedMetadata) |value| attributes.appendAssumeCapacity(.named(ctx, "orderedMetadata", value)); + if (opts.orderedMetadata) |value| attributes.appendAssumeCapacity(.named(ctx, "orderedMetadata", value)); return mlir.Operation.make(ctx, "nvvm.mma.sp.block_scale", .{ .operands = .{ .variadic = &.{ operandA, @@ -5084,11 +5951,39 @@ pub fn mma_sp_block_scale(ctx: *mlir.Context, operandA: []const *const mlir.Valu } /// `nvvm.mma.sp.block_scale.internal`. Result types are explicit; attributes use mlir.Attribute. -pub fn mma_sp_block_scale_internal(ctx: *mlir.Context, operandA: *const mlir.Value, scaleAData: *const mlir.Value, byteIdA: *const mlir.Value, threadIdA: *const mlir.Value, operandB: *const mlir.Value, scaleBData: *const mlir.Value, byteIdB: *const mlir.Value, threadIdB: *const mlir.Value, operandC: *const mlir.Value, sp_metadata: *const mlir.Value, selector: *const mlir.Value, res_type: *const mlir.Type, shape: *const mlir.Attribute, b1Op: ?*const mlir.Attribute, intOverflowBehavior: ?*const mlir.Attribute, layoutA: *const mlir.Attribute, layoutB: *const mlir.Attribute, aType: *const mlir.Attribute, bType: *const mlir.Attribute, cType: *const mlir.Attribute, scaleVecSize: *const mlir.Attribute, blockScaleFormat: *const mlir.Attribute, sparsityFormat: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mma_sp_block_scale_internal( + ctx: *mlir.Context, + operandA: *const mlir.Value, + scaleAData: *const mlir.Value, + byteIdA: *const mlir.Value, + threadIdA: *const mlir.Value, + operandB: *const mlir.Value, + scaleBData: *const mlir.Value, + byteIdB: *const mlir.Value, + threadIdB: *const mlir.Value, + operandC: *const mlir.Value, + sp_metadata: *const mlir.Value, + selector: *const mlir.Value, + res_type: *const mlir.Type, + shape: *const mlir.Attribute, + layoutA: *const mlir.Attribute, + layoutB: *const mlir.Attribute, + aType: *const mlir.Attribute, + bType: *const mlir.Attribute, + cType: *const mlir.Attribute, + scaleVecSize: *const mlir.Attribute, + blockScaleFormat: *const mlir.Attribute, + sparsityFormat: *const mlir.Attribute, + opts: struct { + b1Op: ?*const mlir.Attribute = null, + intOverflowBehavior: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 11) = .empty; attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); - if (b1Op) |value| attributes.appendAssumeCapacity(.named(ctx, "b1Op", value)); - if (intOverflowBehavior) |value| attributes.appendAssumeCapacity(.named(ctx, "intOverflowBehavior", value)); + if (opts.b1Op) |value| attributes.appendAssumeCapacity(.named(ctx, "b1Op", value)); + if (opts.intOverflowBehavior) |value| attributes.appendAssumeCapacity(.named(ctx, "intOverflowBehavior", value)); attributes.appendAssumeCapacity(.named(ctx, "layoutA", layoutA)); attributes.appendAssumeCapacity(.named(ctx, "layoutB", layoutB)); attributes.appendAssumeCapacity(.named(ctx, "aType", aType)); @@ -5106,14 +6001,31 @@ pub fn mma_sp_block_scale_internal(ctx: *mlir.Context, operandA: *const mlir.Val } /// `nvvm.mma.sp.sync`. Result types are explicit; attributes use mlir.Attribute. -pub fn mma_sp_sync(ctx: *mlir.Context, operandA: []const *const mlir.Value, operandB: []const *const mlir.Value, operandC: []const *const mlir.Value, sparseMetadata: *const mlir.Value, sparsitySelector: *const mlir.Value, res_type: *const mlir.Type, shape: *const mlir.Attribute, intOverflowBehavior: ?*const mlir.Attribute, multiplicandAPtxType: ?*const mlir.Attribute, multiplicandBPtxType: ?*const mlir.Attribute, orderedMetadata: ?*const mlir.Attribute, kind: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mma_sp_sync( + ctx: *mlir.Context, + operandA: []const *const mlir.Value, + operandB: []const *const mlir.Value, + operandC: []const *const mlir.Value, + sparseMetadata: *const mlir.Value, + sparsitySelector: *const mlir.Value, + res_type: *const mlir.Type, + shape: *const mlir.Attribute, + opts: struct { + intOverflowBehavior: ?*const mlir.Attribute = null, + multiplicandAPtxType: ?*const mlir.Attribute = null, + multiplicandBPtxType: ?*const mlir.Attribute = null, + orderedMetadata: ?*const mlir.Attribute = null, + kind: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); - if (intOverflowBehavior) |value| attributes.appendAssumeCapacity(.named(ctx, "intOverflowBehavior", value)); - if (multiplicandAPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandAPtxType", value)); - if (multiplicandBPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandBPtxType", value)); - if (orderedMetadata) |value| attributes.appendAssumeCapacity(.named(ctx, "orderedMetadata", value)); - if (kind) |value| attributes.appendAssumeCapacity(.named(ctx, "kind", value)); + if (opts.intOverflowBehavior) |value| attributes.appendAssumeCapacity(.named(ctx, "intOverflowBehavior", value)); + if (opts.multiplicandAPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandAPtxType", value)); + if (opts.multiplicandBPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandBPtxType", value)); + if (opts.orderedMetadata) |value| attributes.appendAssumeCapacity(.named(ctx, "orderedMetadata", value)); + if (opts.kind) |value| attributes.appendAssumeCapacity(.named(ctx, "kind", value)); return mlir.Operation.make(ctx, "nvvm.mma.sp.sync", .{ .operands = .{ .variadic = &.{ operandA, @@ -5129,15 +6041,31 @@ pub fn mma_sp_sync(ctx: *mlir.Context, operandA: []const *const mlir.Value, oper } /// `nvvm.mma.sync`. Result types are explicit; attributes use mlir.Attribute. -pub fn mma_sync(ctx: *mlir.Context, operandA: []const *const mlir.Value, operandB: []const *const mlir.Value, operandC: []const *const mlir.Value, res_type: *const mlir.Type, shape: *const mlir.Attribute, b1Op: ?*const mlir.Attribute, intOverflowBehavior: ?*const mlir.Attribute, layoutA: *const mlir.Attribute, layoutB: *const mlir.Attribute, multiplicandAPtxType: ?*const mlir.Attribute, multiplicandBPtxType: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mma_sync( + ctx: *mlir.Context, + operandA: []const *const mlir.Value, + operandB: []const *const mlir.Value, + operandC: []const *const mlir.Value, + res_type: *const mlir.Type, + shape: *const mlir.Attribute, + layoutA: *const mlir.Attribute, + layoutB: *const mlir.Attribute, + opts: struct { + b1Op: ?*const mlir.Attribute = null, + intOverflowBehavior: ?*const mlir.Attribute = null, + multiplicandAPtxType: ?*const mlir.Attribute = null, + multiplicandBPtxType: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); - if (b1Op) |value| attributes.appendAssumeCapacity(.named(ctx, "b1Op", value)); - if (intOverflowBehavior) |value| attributes.appendAssumeCapacity(.named(ctx, "intOverflowBehavior", value)); + if (opts.b1Op) |value| attributes.appendAssumeCapacity(.named(ctx, "b1Op", value)); + if (opts.intOverflowBehavior) |value| attributes.appendAssumeCapacity(.named(ctx, "intOverflowBehavior", value)); attributes.appendAssumeCapacity(.named(ctx, "layoutA", layoutA)); attributes.appendAssumeCapacity(.named(ctx, "layoutB", layoutB)); - if (multiplicandAPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandAPtxType", value)); - if (multiplicandBPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandBPtxType", value)); + if (opts.multiplicandAPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandAPtxType", value)); + if (opts.multiplicandBPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandBPtxType", value)); return mlir.Operation.make(ctx, "nvvm.mma.sync", .{ .operands = .{ .variadic = &.{ operandA, @@ -5151,9 +6079,21 @@ pub fn mma_sync(ctx: *mlir.Context, operandA: []const *const mlir.Value, operand } /// `nvvm.mma_smem_desc`. Result types are explicit; attributes use mlir.Attribute. -pub fn mma_smem_desc(ctx: *mlir.Context, pointer: *const mlir.Value, ldm: *const mlir.Value, stride: *const mlir.Value, baseOffset: *const mlir.Value, swizzle: *const mlir.Value, res_type: *const mlir.Type, mmaDescVersion: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mma_smem_desc( + ctx: *mlir.Context, + pointer: *const mlir.Value, + ldm: *const mlir.Value, + stride: *const mlir.Value, + baseOffset: *const mlir.Value, + swizzle: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + mmaDescVersion: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (mmaDescVersion) |value| attributes.appendAssumeCapacity(.named(ctx, "mmaDescVersion", value)); + if (opts.mmaDescVersion) |value| attributes.appendAssumeCapacity(.named(ctx, "mmaDescVersion", value)); return mlir.Operation.make(ctx, "nvvm.mma_smem_desc", .{ .operands = .{ .flat = &.{ pointer, ldm, stride, baseOffset, swizzle } }, .results = .{ .flat = &.{res_type} }, @@ -5163,10 +6103,20 @@ pub fn mma_smem_desc(ctx: *mlir.Context, pointer: *const mlir.Value, ldm: *const } /// `nvvm.movmatrix`. Result types are explicit; attributes use mlir.Attribute. -pub fn movmatrix(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, shape: *const mlir.Attribute, layout: ?*const mlir.Attribute, eltType: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn movmatrix( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + shape: *const mlir.Attribute, + eltType: *const mlir.Attribute, + opts: struct { + layout: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); - if (layout) |value| attributes.appendAssumeCapacity(.named(ctx, "layout", value)); + if (opts.layout) |value| attributes.appendAssumeCapacity(.named(ctx, "layout", value)); attributes.appendAssumeCapacity(.named(ctx, "eltType", eltType)); return mlir.Operation.make(ctx, "nvvm.movmatrix", .{ .operands = .{ .flat = &.{src} }, @@ -5177,10 +6127,20 @@ pub fn movmatrix(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const ml } /// `nvvm.mul`. Result types are explicit; attributes use mlir.Attribute. -pub fn mul(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, res_type: *const mlir.Type, mode: *const mlir.Attribute, isSigned: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mul( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + res_type: *const mlir.Type, + mode: *const mlir.Attribute, + opts: struct { + isSigned: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; attributes.appendAssumeCapacity(.named(ctx, "mode", mode)); - if (isSigned) |value| attributes.appendAssumeCapacity(.named(ctx, "isSigned", value)); + if (opts.isSigned) |value| attributes.appendAssumeCapacity(.named(ctx, "isSigned", value)); return mlir.Operation.make(ctx, "nvvm.mul", .{ .operands = .{ .flat = &.{ a, b } }, .results = .{ .flat = &.{res_type} }, @@ -5226,10 +6186,20 @@ pub fn mul_packed_f16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, } /// `nvvm.mul.packed.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn mul_packed_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, ftz: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn mul_packed_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); return mlir.Operation.make(ctx, "nvvm.mul.packed.f32x2", .{ .operands = .{ .flat = &.{ srcA, srcB } }, .results = .{ .flat = &.{res_type} }, @@ -5239,12 +6209,23 @@ pub fn mul_packed_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *cons } /// `nvvm.multimem.ld.reduce`. Result types are explicit; attributes use mlir.Attribute. -pub fn multimem_ld_reduce(ctx: *mlir.Context, addr: *const mlir.Value, res_type: *const mlir.Type, ordering: *const mlir.Attribute, op: *const mlir.Attribute, dataType: *const mlir.Attribute, accPrec: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn multimem_ld_reduce( + ctx: *mlir.Context, + addr: *const mlir.Value, + res_type: *const mlir.Type, + ordering: *const mlir.Attribute, + op: *const mlir.Attribute, + dataType: *const mlir.Attribute, + opts: struct { + accPrec: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; attributes.appendAssumeCapacity(.named(ctx, "ordering", ordering)); attributes.appendAssumeCapacity(.named(ctx, "op", op)); attributes.appendAssumeCapacity(.named(ctx, "dataType", dataType)); - if (accPrec) |value| attributes.appendAssumeCapacity(.named(ctx, "accPrec", value)); + if (opts.accPrec) |value| attributes.appendAssumeCapacity(.named(ctx, "accPrec", value)); return mlir.Operation.make(ctx, "nvvm.multimem.ld.reduce", .{ .operands = .{ .flat = &.{addr} }, .results = .{ .flat = &.{res_type} }, @@ -5289,10 +6270,17 @@ pub fn nanosleep(ctx: *mlir.Context, duration: *const mlir.Value, location: *con } /// `nvvm.pmevent`. Result types are explicit; attributes use mlir.Attribute. -pub fn pmevent(ctx: *mlir.Context, maskedEventId: ?*const mlir.Attribute, eventId: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn pmevent( + ctx: *mlir.Context, + opts: struct { + maskedEventId: ?*const mlir.Attribute = null, + eventId: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (maskedEventId) |value| attributes.appendAssumeCapacity(.named(ctx, "maskedEventId", value)); - if (eventId) |value| attributes.appendAssumeCapacity(.named(ctx, "eventId", value)); + if (opts.maskedEventId) |value| attributes.appendAssumeCapacity(.named(ctx, "maskedEventId", value)); + if (opts.eventId) |value| attributes.appendAssumeCapacity(.named(ctx, "eventId", value)); return mlir.Operation.make(ctx, "nvvm.pmevent", .{ .attributes = attributes.constSlice(), .location = location, @@ -5300,24 +6288,46 @@ pub fn pmevent(ctx: *mlir.Context, maskedEventId: ?*const mlir.Attribute, eventI } /// `nvvm.prefetch`. Result types are explicit; attributes use mlir.Attribute. -pub fn prefetch(ctx: *mlir.Context, addr: *const mlir.Value, predicate: ?*const mlir.Value, cacheLevel: ?*const mlir.Attribute, evictPriority: ?*const mlir.Attribute, tensormap: ?*const mlir.Attribute, uniform: ?*const mlir.Attribute, in_param_space: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn prefetch( + ctx: *mlir.Context, + addr: *const mlir.Value, + opts: struct { + predicate: ?*const mlir.Value = null, + cacheLevel: ?*const mlir.Attribute = null, + evictPriority: ?*const mlir.Attribute = null, + tensormap: ?*const mlir.Attribute = null, + uniform: ?*const mlir.Attribute = null, + in_param_space: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - if (cacheLevel) |value| attributes.appendAssumeCapacity(.named(ctx, "cacheLevel", value)); - if (evictPriority) |value| attributes.appendAssumeCapacity(.named(ctx, "evictPriority", value)); - if (tensormap) |value| attributes.appendAssumeCapacity(.named(ctx, "tensormap", value)); - if (uniform) |value| attributes.appendAssumeCapacity(.named(ctx, "uniform", value)); - if (in_param_space) |value| attributes.appendAssumeCapacity(.named(ctx, "in_param_space", value)); + if (opts.cacheLevel) |value| attributes.appendAssumeCapacity(.named(ctx, "cacheLevel", value)); + if (opts.evictPriority) |value| attributes.appendAssumeCapacity(.named(ctx, "evictPriority", value)); + if (opts.tensormap) |value| attributes.appendAssumeCapacity(.named(ctx, "tensormap", value)); + if (opts.uniform) |value| attributes.appendAssumeCapacity(.named(ctx, "uniform", value)); + if (opts.in_param_space) |value| attributes.appendAssumeCapacity(.named(ctx, "in_param_space", value)); return mlir.Operation.make(ctx, "nvvm.prefetch", .{ - .operands = .{ .flat = if (predicate) |value| &.{ addr, value } else &.{addr} }, + .operands = .{ .flat = if (opts.predicate) |value| &.{ addr, value } else &.{addr} }, .attributes = attributes.constSlice(), .location = location, }); } /// `nvvm.prmt`. Result types are explicit; attributes use mlir.Attribute. -pub fn prmt(ctx: *mlir.Context, lo: *const mlir.Value, hi: ?*const mlir.Value, selector: *const mlir.Value, res_type: *const mlir.Type, mode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn prmt( + ctx: *mlir.Context, + lo: *const mlir.Value, + selector: *const mlir.Value, + res_type: *const mlir.Type, + mode: *const mlir.Attribute, + opts: struct { + hi: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { return mlir.Operation.make(ctx, "nvvm.prmt", .{ - .operands = .{ .flat = if (hi) |value| &.{ lo, value, selector } else &.{ lo, selector } }, + .operands = .{ .flat = if (opts.hi) |value| &.{ lo, value, selector } else &.{ lo, selector } }, .results = .{ .flat = &.{res_type} }, .attributes = &.{ .named(ctx, "mode", mode), @@ -5360,9 +6370,16 @@ pub fn read_ptx_sreg_clock64(ctx: *mlir.Context, res_type: *const mlir.Type, loc } /// `nvvm.read.ptx.sreg.cluster.ctaid.x`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_cluster_ctaid_x(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_cluster_ctaid_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.ctaid.x", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5371,9 +6388,16 @@ pub fn read_ptx_sreg_cluster_ctaid_x(ctx: *mlir.Context, res_type: *const mlir.T } /// `nvvm.read.ptx.sreg.cluster.ctaid.y`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_cluster_ctaid_y(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_cluster_ctaid_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.ctaid.y", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5382,9 +6406,16 @@ pub fn read_ptx_sreg_cluster_ctaid_y(ctx: *mlir.Context, res_type: *const mlir.T } /// `nvvm.read.ptx.sreg.cluster.ctaid.z`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_cluster_ctaid_z(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_cluster_ctaid_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.ctaid.z", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5393,9 +6424,16 @@ pub fn read_ptx_sreg_cluster_ctaid_z(ctx: *mlir.Context, res_type: *const mlir.T } /// `nvvm.read.ptx.sreg.cluster.ctarank`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_cluster_ctarank(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_cluster_ctarank( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.ctarank", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5404,9 +6442,16 @@ pub fn read_ptx_sreg_cluster_ctarank(ctx: *mlir.Context, res_type: *const mlir.T } /// `nvvm.read.ptx.sreg.cluster.nctaid.x`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_cluster_nctaid_x(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_cluster_nctaid_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.nctaid.x", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5415,9 +6460,16 @@ pub fn read_ptx_sreg_cluster_nctaid_x(ctx: *mlir.Context, res_type: *const mlir. } /// `nvvm.read.ptx.sreg.cluster.nctaid.y`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_cluster_nctaid_y(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_cluster_nctaid_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.nctaid.y", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5426,9 +6478,16 @@ pub fn read_ptx_sreg_cluster_nctaid_y(ctx: *mlir.Context, res_type: *const mlir. } /// `nvvm.read.ptx.sreg.cluster.nctaid.z`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_cluster_nctaid_z(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_cluster_nctaid_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.nctaid.z", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5437,9 +6496,16 @@ pub fn read_ptx_sreg_cluster_nctaid_z(ctx: *mlir.Context, res_type: *const mlir. } /// `nvvm.read.ptx.sreg.cluster.nctarank`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_cluster_nctarank(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_cluster_nctarank( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.nctarank", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5448,9 +6514,16 @@ pub fn read_ptx_sreg_cluster_nctarank(ctx: *mlir.Context, res_type: *const mlir. } /// `nvvm.read.ptx.sreg.clusterid.x`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_clusterid_x(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_clusterid_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.clusterid.x", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5459,9 +6532,16 @@ pub fn read_ptx_sreg_clusterid_x(ctx: *mlir.Context, res_type: *const mlir.Type, } /// `nvvm.read.ptx.sreg.clusterid.y`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_clusterid_y(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_clusterid_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.clusterid.y", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5470,9 +6550,16 @@ pub fn read_ptx_sreg_clusterid_y(ctx: *mlir.Context, res_type: *const mlir.Type, } /// `nvvm.read.ptx.sreg.clusterid.z`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_clusterid_z(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_clusterid_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.clusterid.z", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5481,9 +6568,16 @@ pub fn read_ptx_sreg_clusterid_z(ctx: *mlir.Context, res_type: *const mlir.Type, } /// `nvvm.read.ptx.sreg.ctaid.x`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_ctaid_x(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_ctaid_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ctaid.x", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5492,9 +6586,16 @@ pub fn read_ptx_sreg_ctaid_x(ctx: *mlir.Context, res_type: *const mlir.Type, ran } /// `nvvm.read.ptx.sreg.ctaid.y`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_ctaid_y(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_ctaid_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ctaid.y", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5503,9 +6604,16 @@ pub fn read_ptx_sreg_ctaid_y(ctx: *mlir.Context, res_type: *const mlir.Type, ran } /// `nvvm.read.ptx.sreg.ctaid.z`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_ctaid_z(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_ctaid_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ctaid.z", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5794,9 +6902,16 @@ pub fn read_ptx_sreg_globaltimer_lo(ctx: *mlir.Context, res_type: *const mlir.Ty } /// `nvvm.read.ptx.sreg.gridid`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_gridid(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_gridid( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.gridid", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5805,9 +6920,16 @@ pub fn read_ptx_sreg_gridid(ctx: *mlir.Context, res_type: *const mlir.Type, rang } /// `nvvm.read.ptx.sreg.laneid`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_laneid(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_laneid( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.laneid", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5856,9 +6978,16 @@ pub fn read_ptx_sreg_lanemask_lt(ctx: *mlir.Context, res_type: *const mlir.Type, } /// `nvvm.read.ptx.sreg.nclusterid.x`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_nclusterid_x(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_nclusterid_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nclusterid.x", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5867,9 +6996,16 @@ pub fn read_ptx_sreg_nclusterid_x(ctx: *mlir.Context, res_type: *const mlir.Type } /// `nvvm.read.ptx.sreg.nclusterid.y`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_nclusterid_y(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_nclusterid_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nclusterid.y", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5878,9 +7014,16 @@ pub fn read_ptx_sreg_nclusterid_y(ctx: *mlir.Context, res_type: *const mlir.Type } /// `nvvm.read.ptx.sreg.nclusterid.z`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_nclusterid_z(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_nclusterid_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nclusterid.z", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5889,9 +7032,16 @@ pub fn read_ptx_sreg_nclusterid_z(ctx: *mlir.Context, res_type: *const mlir.Type } /// `nvvm.read.ptx.sreg.nctaid.x`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_nctaid_x(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_nctaid_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nctaid.x", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5900,9 +7050,16 @@ pub fn read_ptx_sreg_nctaid_x(ctx: *mlir.Context, res_type: *const mlir.Type, ra } /// `nvvm.read.ptx.sreg.nctaid.y`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_nctaid_y(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_nctaid_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nctaid.y", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5911,9 +7068,16 @@ pub fn read_ptx_sreg_nctaid_y(ctx: *mlir.Context, res_type: *const mlir.Type, ra } /// `nvvm.read.ptx.sreg.nctaid.z`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_nctaid_z(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_nctaid_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nctaid.z", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5922,9 +7086,16 @@ pub fn read_ptx_sreg_nctaid_z(ctx: *mlir.Context, res_type: *const mlir.Type, ra } /// `nvvm.read.ptx.sreg.nsmid`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_nsmid(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_nsmid( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nsmid", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5933,9 +7104,16 @@ pub fn read_ptx_sreg_nsmid(ctx: *mlir.Context, res_type: *const mlir.Type, range } /// `nvvm.read.ptx.sreg.ntid.x`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_ntid_x(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_ntid_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ntid.x", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5944,9 +7122,16 @@ pub fn read_ptx_sreg_ntid_x(ctx: *mlir.Context, res_type: *const mlir.Type, rang } /// `nvvm.read.ptx.sreg.ntid.y`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_ntid_y(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_ntid_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ntid.y", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5955,9 +7140,16 @@ pub fn read_ptx_sreg_ntid_y(ctx: *mlir.Context, res_type: *const mlir.Type, rang } /// `nvvm.read.ptx.sreg.ntid.z`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_ntid_z(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_ntid_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ntid.z", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5966,9 +7158,16 @@ pub fn read_ptx_sreg_ntid_z(ctx: *mlir.Context, res_type: *const mlir.Type, rang } /// `nvvm.read.ptx.sreg.nwarpid`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_nwarpid(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_nwarpid( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nwarpid", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5977,9 +7176,16 @@ pub fn read_ptx_sreg_nwarpid(ctx: *mlir.Context, res_type: *const mlir.Type, ran } /// `nvvm.read.ptx.sreg.smid`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_smid(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_smid( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.smid", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5988,9 +7194,16 @@ pub fn read_ptx_sreg_smid(ctx: *mlir.Context, res_type: *const mlir.Type, range: } /// `nvvm.read.ptx.sreg.tid.x`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_tid_x(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_tid_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.tid.x", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -5999,9 +7212,16 @@ pub fn read_ptx_sreg_tid_x(ctx: *mlir.Context, res_type: *const mlir.Type, range } /// `nvvm.read.ptx.sreg.tid.y`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_tid_y(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_tid_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.tid.y", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -6010,9 +7230,16 @@ pub fn read_ptx_sreg_tid_y(ctx: *mlir.Context, res_type: *const mlir.Type, range } /// `nvvm.read.ptx.sreg.tid.z`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_tid_z(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_tid_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.tid.z", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -6029,9 +7256,16 @@ pub fn read_ptx_sreg_total_smem_size(ctx: *mlir.Context, res_type: *const mlir.T } /// `nvvm.read.ptx.sreg.warpid`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_warpid(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_warpid( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.warpid", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -6040,9 +7274,16 @@ pub fn read_ptx_sreg_warpid(ctx: *mlir.Context, res_type: *const mlir.Type, rang } /// `nvvm.read.ptx.sreg.warpsize`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_warpsize(ctx: *mlir.Context, res_type: *const mlir.Type, range: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn read_ptx_sreg_warpsize( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.warpsize", .{ .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), @@ -6051,25 +7292,48 @@ pub fn read_ptx_sreg_warpsize(ctx: *mlir.Context, res_type: *const mlir.Type, ra } /// `nvvm.red`. Result types are explicit; attributes use mlir.Attribute. -pub fn red(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, cacheHint: ?*const mlir.Value, memOrder: ?*const mlir.Attribute, memScope: ?*const mlir.Attribute, op: *const mlir.Attribute, type_: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn red( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + op: *const mlir.Attribute, + type_: *const mlir.Attribute, + opts: struct { + cacheHint: ?*const mlir.Value = null, + memOrder: ?*const mlir.Attribute = null, + memScope: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - if (memOrder) |value| attributes.appendAssumeCapacity(.named(ctx, "memOrder", value)); - if (memScope) |value| attributes.appendAssumeCapacity(.named(ctx, "memScope", value)); + if (opts.memOrder) |value| attributes.appendAssumeCapacity(.named(ctx, "memOrder", value)); + if (opts.memScope) |value| attributes.appendAssumeCapacity(.named(ctx, "memScope", value)); attributes.appendAssumeCapacity(.named(ctx, "op", op)); attributes.appendAssumeCapacity(.named(ctx, "type", type_)); return mlir.Operation.make(ctx, "nvvm.red", .{ - .operands = .{ .flat = if (cacheHint) |value| &.{ a, b, value } else &.{ a, b } }, + .operands = .{ .flat = if (opts.cacheHint) |value| &.{ a, b, value } else &.{ a, b } }, .attributes = attributes.constSlice(), .location = location, }); } /// `nvvm.redux.sync`. Result types are explicit; attributes use mlir.Attribute. -pub fn redux_sync(ctx: *mlir.Context, val: *const mlir.Value, mask_and_clamp: *const mlir.Value, res_type: *const mlir.Type, kind: *const mlir.Attribute, abs: ?*const mlir.Attribute, nan: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn redux_sync( + ctx: *mlir.Context, + val: *const mlir.Value, + mask_and_clamp: *const mlir.Value, + res_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + abs: ?*const mlir.Attribute = null, + nan: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); - if (nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); + if (opts.abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); + if (opts.nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); return mlir.Operation.make(ctx, "nvvm.redux.sync", .{ .operands = .{ .flat = &.{ val, mask_and_clamp } }, .results = .{ .flat = &.{res_type} }, @@ -6079,9 +7343,17 @@ pub fn redux_sync(ctx: *mlir.Context, val: *const mlir.Value, mask_and_clamp: *c } /// `nvvm.rsqrt`. Result types are explicit; attributes use mlir.Attribute. -pub fn rsqrt(ctx: *mlir.Context, src: *const mlir.Value, res_type: *const mlir.Type, ftz: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn rsqrt( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); return mlir.Operation.make(ctx, "nvvm.rsqrt", .{ .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{res_type} }, @@ -6102,10 +7374,22 @@ pub fn setmaxregister(ctx: *mlir.Context, regCount: *const mlir.Attribute, actio } /// `nvvm.shfl.sync`. Result types are explicit; attributes use mlir.Attribute. -pub fn shfl_sync(ctx: *mlir.Context, thread_mask: *const mlir.Value, val: *const mlir.Value, offset: *const mlir.Value, mask_and_clamp: *const mlir.Value, res_type: *const mlir.Type, kind: *const mlir.Attribute, return_value_and_is_valid: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn shfl_sync( + ctx: *mlir.Context, + thread_mask: *const mlir.Value, + val: *const mlir.Value, + offset: *const mlir.Value, + mask_and_clamp: *const mlir.Value, + res_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + return_value_and_is_valid: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (return_value_and_is_valid) |value| attributes.appendAssumeCapacity(.named(ctx, "return_value_and_is_valid", value)); + if (opts.return_value_and_is_valid) |value| attributes.appendAssumeCapacity(.named(ctx, "return_value_and_is_valid", value)); return mlir.Operation.make(ctx, "nvvm.shfl.sync", .{ .operands = .{ .flat = &.{ thread_mask, val, offset, mask_and_clamp } }, .results = .{ .flat = &.{res_type} }, @@ -6115,9 +7399,17 @@ pub fn shfl_sync(ctx: *mlir.Context, thread_mask: *const mlir.Value, val: *const } /// `nvvm.sin`. Result types are explicit; attributes use mlir.Attribute. -pub fn sin(ctx: *mlir.Context, src: *const mlir.Value, res_type: *const mlir.Type, ftz: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn sin( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); return mlir.Operation.make(ctx, "nvvm.sin", .{ .operands = .{ .flat = &.{src} }, .results = .{ .flat = &.{res_type} }, @@ -6157,9 +7449,17 @@ pub fn spdecompress(ctx: *mlir.Context, metadata: *const mlir.Value, compressed_ } /// `nvvm.st.bulk`. Result types are explicit; attributes use mlir.Attribute. -pub fn st_bulk(ctx: *mlir.Context, addr: *const mlir.Value, size: *const mlir.Value, initVal: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn st_bulk( + ctx: *mlir.Context, + addr: *const mlir.Value, + size: *const mlir.Value, + opts: struct { + initVal: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (initVal) |value| attributes.appendAssumeCapacity(.named(ctx, "initVal", value)); + if (opts.initVal) |value| attributes.appendAssumeCapacity(.named(ctx, "initVal", value)); return mlir.Operation.make(ctx, "nvvm.st.bulk", .{ .operands = .{ .flat = &.{ addr, size } }, .attributes = attributes.constSlice(), @@ -6168,13 +7468,22 @@ pub fn st_bulk(ctx: *mlir.Context, addr: *const mlir.Value, size: *const mlir.Va } /// `nvvm.stmatrix`. Result types are explicit; attributes use mlir.Attribute. -pub fn stmatrix(ctx: *mlir.Context, ptr: *const mlir.Value, sources: []const *const mlir.Value, layout: *const mlir.Attribute, shape: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn stmatrix( + ctx: *mlir.Context, + ptr: *const mlir.Value, + sources: []const *const mlir.Value, + layout: *const mlir.Attribute, + opts: struct { + shape: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var operands: stdx.BoundedArray(*const mlir.Value, 256) = .empty; operands.appendAssumeCapacity(ptr); operands.appendSliceAssumeCapacity(sources); var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; attributes.appendAssumeCapacity(.named(ctx, "layout", layout)); - if (shape) |value| attributes.appendAssumeCapacity(.named(ctx, "shape", value)); + if (opts.shape) |value| attributes.appendAssumeCapacity(.named(ctx, "shape", value)); return mlir.Operation.make(ctx, "nvvm.stmatrix", .{ .operands = .{ .flat = operands.constSlice() }, .attributes = attributes.constSlice(), @@ -6191,15 +7500,28 @@ pub fn store_async(ctx: *mlir.Context, addr: *const mlir.Value, value_: *const m } /// `nvvm.store.ext`. Result types are explicit; attributes use mlir.Attribute. -pub fn store_ext(ctx: *mlir.Context, value_: *const mlir.Value, addr: *const mlir.Value, l2CacheHint: ?*const mlir.Value, order: ?*const mlir.Attribute, scope: ?*const mlir.Attribute, evict: ?*const mlir.Attribute, cacheModifier: ?*const mlir.Attribute, sharedSpace: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn store_ext( + ctx: *mlir.Context, + value_: *const mlir.Value, + addr: *const mlir.Value, + opts: struct { + l2CacheHint: ?*const mlir.Value = null, + order: ?*const mlir.Attribute = null, + scope: ?*const mlir.Attribute = null, + evict: ?*const mlir.Attribute = null, + cacheModifier: ?*const mlir.Attribute = null, + sharedSpace: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - if (order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); - if (scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); - if (cacheModifier) |value| attributes.appendAssumeCapacity(.named(ctx, "cacheModifier", value)); - if (sharedSpace) |value| attributes.appendAssumeCapacity(.named(ctx, "sharedSpace", value)); + if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + if (opts.cacheModifier) |value| attributes.appendAssumeCapacity(.named(ctx, "cacheModifier", value)); + if (opts.sharedSpace) |value| attributes.appendAssumeCapacity(.named(ctx, "sharedSpace", value)); return mlir.Operation.make(ctx, "nvvm.store.ext", .{ - .operands = .{ .flat = if (l2CacheHint) |value| &.{ value_, addr, value } else &.{ value_, addr } }, + .operands = .{ .flat = if (opts.l2CacheHint) |value| &.{ value_, addr, value } else &.{ value_, addr } }, .attributes = attributes.constSlice(), .location = location, }); @@ -6224,10 +7546,20 @@ pub fn sub_packed_f16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, } /// `nvvm.sub.packed.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn sub_packed_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, ftz: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn sub_packed_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); return mlir.Operation.make(ctx, "nvvm.sub.packed.f32x2", .{ .operands = .{ .flat = &.{ srcA, srcB } }, .results = .{ .flat = &.{res_type} }, @@ -6237,9 +7569,18 @@ pub fn sub_packed_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *cons } /// `nvvm.sub.packed.f32x2.bf16x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn sub_packed_f32x2_bf16x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn sub_packed_f32x2_bf16x2_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); return mlir.Operation.make(ctx, "nvvm.sub.packed.f32x2.bf16x2.f32x2", .{ .operands = .{ .flat = &.{ srcA, srcB } }, .results = .{ .flat = &.{res_type} }, @@ -6249,9 +7590,18 @@ pub fn sub_packed_f32x2_bf16x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value } /// `nvvm.sub.packed.f32x2.f16x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn sub_packed_f32x2_f16x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn sub_packed_f32x2_f16x2_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); return mlir.Operation.make(ctx, "nvvm.sub.packed.f32x2.f16x2.f32x2", .{ .operands = .{ .flat = &.{ srcA, srcB } }, .results = .{ .flat = &.{res_type} }, @@ -6261,11 +7611,22 @@ pub fn sub_packed_f32x2_f16x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, } /// `nvvm.subf`. Result types are explicit; attributes use mlir.Attribute. -pub fn subf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, res_type: *const mlir.Type, rnd: ?*const mlir.Attribute, sat: ?*const mlir.Attribute, ftz: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn subf( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); return mlir.Operation.make(ctx, "nvvm.subf", .{ .operands = .{ .flat = &.{ lhs, rhs } }, .results = .{ .flat = &.{res_type} }, @@ -6275,10 +7636,19 @@ pub fn subf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, } /// `nvvm.tcgen05.alloc`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_alloc(ctx: *mlir.Context, addr: *const mlir.Value, nCols: *const mlir.Value, isExclusive: ?*const mlir.Attribute, group: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn tcgen05_alloc( + ctx: *mlir.Context, + addr: *const mlir.Value, + nCols: *const mlir.Value, + opts: struct { + isExclusive: ?*const mlir.Attribute = null, + group: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (isExclusive) |value| attributes.appendAssumeCapacity(.named(ctx, "isExclusive", value)); - if (group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + if (opts.isExclusive) |value| attributes.appendAssumeCapacity(.named(ctx, "isExclusive", value)); + if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); return mlir.Operation.make(ctx, "nvvm.tcgen05.alloc", .{ .operands = .{ .flat = &.{ addr, nCols } }, .attributes = attributes.constSlice(), @@ -6287,24 +7657,44 @@ pub fn tcgen05_alloc(ctx: *mlir.Context, addr: *const mlir.Value, nCols: *const } /// `nvvm.tcgen05.commit`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_commit(ctx: *mlir.Context, addr: *const mlir.Value, multicastMask: ?*const mlir.Value, smem_a_read: ?*const mlir.Attribute, group: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn tcgen05_commit( + ctx: *mlir.Context, + addr: *const mlir.Value, + opts: struct { + multicastMask: ?*const mlir.Value = null, + smem_a_read: ?*const mlir.Attribute = null, + group: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (smem_a_read) |value| attributes.appendAssumeCapacity(.named(ctx, "smem_a_read", value)); - if (group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + if (opts.smem_a_read) |value| attributes.appendAssumeCapacity(.named(ctx, "smem_a_read", value)); + if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); return mlir.Operation.make(ctx, "nvvm.tcgen05.commit", .{ - .operands = .{ .flat = if (multicastMask) |value| &.{ addr, value } else &.{addr} }, + .operands = .{ .flat = if (opts.multicastMask) |value| &.{ addr, value } else &.{addr} }, .attributes = attributes.constSlice(), .location = location, }); } /// `nvvm.tcgen05.cp`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_cp(ctx: *mlir.Context, taddr: *const mlir.Value, smem_desc: *const mlir.Value, shape: *const mlir.Attribute, group: ?*const mlir.Attribute, multicast: ?*const mlir.Attribute, srcFormat: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn tcgen05_cp( + ctx: *mlir.Context, + taddr: *const mlir.Value, + smem_desc: *const mlir.Value, + shape: *const mlir.Attribute, + opts: struct { + group: ?*const mlir.Attribute = null, + multicast: ?*const mlir.Attribute = null, + srcFormat: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); - if (group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); - if (multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "multicast", value)); - if (srcFormat) |value| attributes.appendAssumeCapacity(.named(ctx, "srcFormat", value)); + if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + if (opts.multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "multicast", value)); + if (opts.srcFormat) |value| attributes.appendAssumeCapacity(.named(ctx, "srcFormat", value)); return mlir.Operation.make(ctx, "nvvm.tcgen05.cp", .{ .operands = .{ .flat = &.{ taddr, smem_desc } }, .attributes = attributes.constSlice(), @@ -6313,10 +7703,19 @@ pub fn tcgen05_cp(ctx: *mlir.Context, taddr: *const mlir.Value, smem_desc: *cons } /// `nvvm.tcgen05.dealloc`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_dealloc(ctx: *mlir.Context, taddr: *const mlir.Value, nCols: *const mlir.Value, isExclusive: ?*const mlir.Attribute, group: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn tcgen05_dealloc( + ctx: *mlir.Context, + taddr: *const mlir.Value, + nCols: *const mlir.Value, + opts: struct { + isExclusive: ?*const mlir.Attribute = null, + group: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (isExclusive) |value| attributes.appendAssumeCapacity(.named(ctx, "isExclusive", value)); - if (group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + if (opts.isExclusive) |value| attributes.appendAssumeCapacity(.named(ctx, "isExclusive", value)); + if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); return mlir.Operation.make(ctx, "nvvm.tcgen05.dealloc", .{ .operands = .{ .flat = &.{ taddr, nCols } }, .attributes = attributes.constSlice(), @@ -6335,12 +7734,22 @@ pub fn tcgen05_fence(ctx: *mlir.Context, kind: *const mlir.Attribute, location: } /// `nvvm.tcgen05.ld`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_ld(ctx: *mlir.Context, tmemAddr: *const mlir.Value, offset: ?*const mlir.Value, res_type: *const mlir.Type, pack: ?*const mlir.Attribute, shape: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn tcgen05_ld( + ctx: *mlir.Context, + tmemAddr: *const mlir.Value, + res_type: *const mlir.Type, + shape: *const mlir.Attribute, + opts: struct { + offset: ?*const mlir.Value = null, + pack: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (pack) |value| attributes.appendAssumeCapacity(.named(ctx, "pack", value)); + if (opts.pack) |value| attributes.appendAssumeCapacity(.named(ctx, "pack", value)); attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); return mlir.Operation.make(ctx, "nvvm.tcgen05.ld", .{ - .operands = .{ .flat = if (offset) |value| &.{ tmemAddr, value } else &.{tmemAddr} }, + .operands = .{ .flat = if (opts.offset) |value| &.{ tmemAddr, value } else &.{tmemAddr} }, .results = .{ .flat = &.{res_type} }, .attributes = attributes.constSlice(), .location = location, @@ -6348,15 +7757,29 @@ pub fn tcgen05_ld(ctx: *mlir.Context, tmemAddr: *const mlir.Value, offset: ?*con } /// `nvvm.tcgen05.ld.red`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_ld_red(ctx: *mlir.Context, addr: *const mlir.Value, offset: ?*const mlir.Value, data_type: *const mlir.Type, redVal_type: *const mlir.Type, shape: *const mlir.Attribute, op: *const mlir.Attribute, abs: ?*const mlir.Attribute, nan: ?*const mlir.Attribute, isSigned: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn tcgen05_ld_red( + ctx: *mlir.Context, + addr: *const mlir.Value, + data_type: *const mlir.Type, + redVal_type: *const mlir.Type, + shape: *const mlir.Attribute, + op: *const mlir.Attribute, + opts: struct { + offset: ?*const mlir.Value = null, + abs: ?*const mlir.Attribute = null, + nan: ?*const mlir.Attribute = null, + isSigned: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); attributes.appendAssumeCapacity(.named(ctx, "op", op)); - if (abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); - if (nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); - if (isSigned) |value| attributes.appendAssumeCapacity(.named(ctx, "isSigned", value)); + if (opts.abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); + if (opts.nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); + if (opts.isSigned) |value| attributes.appendAssumeCapacity(.named(ctx, "isSigned", value)); return mlir.Operation.make(ctx, "nvvm.tcgen05.ld.red", .{ - .operands = .{ .flat = if (offset) |value| &.{ addr, value } else &.{addr} }, + .operands = .{ .flat = if (opts.offset) |value| &.{ addr, value } else &.{addr} }, .results = .{ .flat = &.{ data_type, redVal_type } }, .attributes = attributes.constSlice(), .location = location, @@ -6364,10 +7787,25 @@ pub fn tcgen05_ld_red(ctx: *mlir.Context, addr: *const mlir.Value, offset: ?*con } /// `nvvm.tcgen05.ld.red.spcompress`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_ld_red_spcompress(ctx: *mlir.Context, tmemAddr: *const mlir.Value, metadata_type: *const mlir.Type, compressedData_type: *const mlir.Type, redVal_type: *const mlir.Type, abs: ?*const mlir.Attribute, nan: ?*const mlir.Attribute, shape: *const mlir.Attribute, opKind: *const mlir.Attribute, compFactor: *const mlir.Attribute, repFactor: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn tcgen05_ld_red_spcompress( + ctx: *mlir.Context, + tmemAddr: *const mlir.Value, + metadata_type: *const mlir.Type, + compressedData_type: *const mlir.Type, + redVal_type: *const mlir.Type, + shape: *const mlir.Attribute, + opKind: *const mlir.Attribute, + compFactor: *const mlir.Attribute, + repFactor: *const mlir.Attribute, + opts: struct { + abs: ?*const mlir.Attribute = null, + nan: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; - if (abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); - if (nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); + if (opts.abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); + if (opts.nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); attributes.appendAssumeCapacity(.named(ctx, "opKind", opKind)); attributes.appendAssumeCapacity(.named(ctx, "compFactor", compFactor)); @@ -6381,9 +7819,22 @@ pub fn tcgen05_ld_red_spcompress(ctx: *mlir.Context, tmemAddr: *const mlir.Value } /// `nvvm.tcgen05.ld.spcompress`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_ld_spcompress(ctx: *mlir.Context, tmemAddr: *const mlir.Value, metadata_type: *const mlir.Type, compressedData_type: *const mlir.Type, abs: ?*const mlir.Attribute, shape: *const mlir.Attribute, opKind: *const mlir.Attribute, compFactor: *const mlir.Attribute, repFactor: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn tcgen05_ld_spcompress( + ctx: *mlir.Context, + tmemAddr: *const mlir.Value, + metadata_type: *const mlir.Type, + compressedData_type: *const mlir.Type, + shape: *const mlir.Attribute, + opKind: *const mlir.Attribute, + compFactor: *const mlir.Attribute, + repFactor: *const mlir.Attribute, + opts: struct { + abs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - if (abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); + if (opts.abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); attributes.appendAssumeCapacity(.named(ctx, "opKind", opKind)); attributes.appendAssumeCapacity(.named(ctx, "compFactor", compFactor)); @@ -6397,13 +7848,30 @@ pub fn tcgen05_ld_spcompress(ctx: *mlir.Context, tmemAddr: *const mlir.Value, me } /// `nvvm.tcgen05.mma`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_mma(ctx: *mlir.Context, matrixD: *const mlir.Value, matrixA: *const mlir.Value, matrixB: *const mlir.Value, idesc: *const mlir.Value, enableInputD: *const mlir.Value, scaleInputD: ?*const mlir.Value, disableOutputLane: ?*const mlir.Value, kind: *const mlir.Attribute, ctaGroup: *const mlir.Attribute, collectorOp: ?*const mlir.Attribute, collectorOpB: ?*const mlir.Attribute, aShift: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn tcgen05_mma( + ctx: *mlir.Context, + matrixD: *const mlir.Value, + matrixA: *const mlir.Value, + matrixB: *const mlir.Value, + idesc: *const mlir.Value, + enableInputD: *const mlir.Value, + kind: *const mlir.Attribute, + ctaGroup: *const mlir.Attribute, + opts: struct { + scaleInputD: ?*const mlir.Value = null, + disableOutputLane: ?*const mlir.Value = null, + collectorOp: ?*const mlir.Attribute = null, + collectorOpB: ?*const mlir.Attribute = null, + aShift: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); - if (collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); - if (collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); - if (aShift) |value| attributes.appendAssumeCapacity(.named(ctx, "aShift", value)); + if (opts.collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); + if (opts.collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); + if (opts.aShift) |value| attributes.appendAssumeCapacity(.named(ctx, "aShift", value)); return mlir.Operation.make(ctx, "nvvm.tcgen05.mma", .{ .operands = .{ .variadic = &.{ &.{matrixD}, @@ -6411,8 +7879,8 @@ pub fn tcgen05_mma(ctx: *mlir.Context, matrixD: *const mlir.Value, matrixA: *con &.{matrixB}, &.{idesc}, &.{enableInputD}, - if (scaleInputD) |value| &.{value} else &.{}, - if (disableOutputLane) |value| &.{value} else &.{}, + if (opts.scaleInputD) |value| &.{value} else &.{}, + if (opts.disableOutputLane) |value| &.{value} else &.{}, } }, .attributes = attributes.constSlice(), .location = location, @@ -6420,13 +7888,30 @@ pub fn tcgen05_mma(ctx: *mlir.Context, matrixD: *const mlir.Value, matrixA: *con } /// `nvvm.tcgen05.mma.block_scale`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_mma_block_scale(ctx: *mlir.Context, matrixD: *const mlir.Value, matrixA: *const mlir.Value, matrixB: *const mlir.Value, idesc: *const mlir.Value, enableInputD: *const mlir.Value, scaleA: *const mlir.Value, scaleB: *const mlir.Value, kind: *const mlir.Attribute, ctaGroup: *const mlir.Attribute, blockScale: ?*const mlir.Attribute, collectorOp: ?*const mlir.Attribute, collectorOpB: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn tcgen05_mma_block_scale( + ctx: *mlir.Context, + matrixD: *const mlir.Value, + matrixA: *const mlir.Value, + matrixB: *const mlir.Value, + idesc: *const mlir.Value, + enableInputD: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + kind: *const mlir.Attribute, + ctaGroup: *const mlir.Attribute, + opts: struct { + blockScale: ?*const mlir.Attribute = null, + collectorOp: ?*const mlir.Attribute = null, + collectorOpB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); - if (blockScale) |value| attributes.appendAssumeCapacity(.named(ctx, "blockScale", value)); - if (collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); - if (collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); + if (opts.blockScale) |value| attributes.appendAssumeCapacity(.named(ctx, "blockScale", value)); + if (opts.collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); + if (opts.collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.block_scale", .{ .operands = .{ .flat = &.{ matrixD, matrixA, matrixB, idesc, enableInputD, scaleA, scaleB } }, .attributes = attributes.constSlice(), @@ -6435,13 +7920,31 @@ pub fn tcgen05_mma_block_scale(ctx: *mlir.Context, matrixD: *const mlir.Value, m } /// `nvvm.tcgen05.mma.block_scale.decompress_b`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_mma_block_scale_decompress_b(ctx: *mlir.Context, matrixD: *const mlir.Value, matrixA: *const mlir.Value, matrixB: *const mlir.Value, idesc: *const mlir.Value, enableInputD: *const mlir.Value, scaleA: *const mlir.Value, scaleB: *const mlir.Value, decompressBMetadata: *const mlir.Value, kind: *const mlir.Attribute, ctaGroup: *const mlir.Attribute, blockScale: ?*const mlir.Attribute, collectorOpA: ?*const mlir.Attribute, collectorOpB: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn tcgen05_mma_block_scale_decompress_b( + ctx: *mlir.Context, + matrixD: *const mlir.Value, + matrixA: *const mlir.Value, + matrixB: *const mlir.Value, + idesc: *const mlir.Value, + enableInputD: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + decompressBMetadata: *const mlir.Value, + kind: *const mlir.Attribute, + ctaGroup: *const mlir.Attribute, + opts: struct { + blockScale: ?*const mlir.Attribute = null, + collectorOpA: ?*const mlir.Attribute = null, + collectorOpB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); - if (blockScale) |value| attributes.appendAssumeCapacity(.named(ctx, "blockScale", value)); - if (collectorOpA) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpA", value)); - if (collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); + if (opts.blockScale) |value| attributes.appendAssumeCapacity(.named(ctx, "blockScale", value)); + if (opts.collectorOpA) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpA", value)); + if (opts.collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.block_scale.decompress_b", .{ .operands = .{ .flat = &.{ matrixD, matrixA, matrixB, idesc, enableInputD, scaleA, scaleB, decompressBMetadata } }, .attributes = attributes.constSlice(), @@ -6450,27 +7953,61 @@ pub fn tcgen05_mma_block_scale_decompress_b(ctx: *mlir.Context, matrixD: *const } /// `nvvm.tcgen05.mma.decompress_b`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_mma_decompress_b(ctx: *mlir.Context, matrixD: *const mlir.Value, matrixA: *const mlir.Value, matrixB: *const mlir.Value, idesc: *const mlir.Value, enableInputD: *const mlir.Value, decompressBMetadata: *const mlir.Value, disableOutputLane: ?*const mlir.Value, kind: *const mlir.Attribute, ctaGroup: *const mlir.Attribute, collectorOpA: ?*const mlir.Attribute, collectorOpB: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn tcgen05_mma_decompress_b( + ctx: *mlir.Context, + matrixD: *const mlir.Value, + matrixA: *const mlir.Value, + matrixB: *const mlir.Value, + idesc: *const mlir.Value, + enableInputD: *const mlir.Value, + decompressBMetadata: *const mlir.Value, + kind: *const mlir.Attribute, + ctaGroup: *const mlir.Attribute, + opts: struct { + disableOutputLane: ?*const mlir.Value = null, + collectorOpA: ?*const mlir.Attribute = null, + collectorOpB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); - if (collectorOpA) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpA", value)); - if (collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); + if (opts.collectorOpA) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpA", value)); + if (opts.collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.decompress_b", .{ - .operands = .{ .flat = if (disableOutputLane) |value| &.{ matrixD, matrixA, matrixB, idesc, enableInputD, decompressBMetadata, value } else &.{ matrixD, matrixA, matrixB, idesc, enableInputD, decompressBMetadata } }, + .operands = .{ .flat = if (opts.disableOutputLane) |value| &.{ matrixD, matrixA, matrixB, idesc, enableInputD, decompressBMetadata, value } else &.{ matrixD, matrixA, matrixB, idesc, enableInputD, decompressBMetadata } }, .attributes = attributes.constSlice(), .location = location, }); } /// `nvvm.tcgen05.mma.sp`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_mma_sp(ctx: *mlir.Context, matrixD: *const mlir.Value, matrixA: *const mlir.Value, matrixB: *const mlir.Value, idesc: *const mlir.Value, enableInputD: *const mlir.Value, sparseMetadata: *const mlir.Value, scaleInputD: ?*const mlir.Value, disableOutputLane: ?*const mlir.Value, kind: *const mlir.Attribute, ctaGroup: *const mlir.Attribute, collectorOp: ?*const mlir.Attribute, collectorOpB: ?*const mlir.Attribute, aShift: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn tcgen05_mma_sp( + ctx: *mlir.Context, + matrixD: *const mlir.Value, + matrixA: *const mlir.Value, + matrixB: *const mlir.Value, + idesc: *const mlir.Value, + enableInputD: *const mlir.Value, + sparseMetadata: *const mlir.Value, + kind: *const mlir.Attribute, + ctaGroup: *const mlir.Attribute, + opts: struct { + scaleInputD: ?*const mlir.Value = null, + disableOutputLane: ?*const mlir.Value = null, + collectorOp: ?*const mlir.Attribute = null, + collectorOpB: ?*const mlir.Attribute = null, + aShift: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); - if (collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); - if (collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); - if (aShift) |value| attributes.appendAssumeCapacity(.named(ctx, "aShift", value)); + if (opts.collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); + if (opts.collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); + if (opts.aShift) |value| attributes.appendAssumeCapacity(.named(ctx, "aShift", value)); return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.sp", .{ .operands = .{ .variadic = &.{ &.{matrixD}, @@ -6479,8 +8016,8 @@ pub fn tcgen05_mma_sp(ctx: *mlir.Context, matrixD: *const mlir.Value, matrixA: * &.{idesc}, &.{enableInputD}, &.{sparseMetadata}, - if (scaleInputD) |value| &.{value} else &.{}, - if (disableOutputLane) |value| &.{value} else &.{}, + if (opts.scaleInputD) |value| &.{value} else &.{}, + if (opts.disableOutputLane) |value| &.{value} else &.{}, } }, .attributes = attributes.constSlice(), .location = location, @@ -6488,13 +8025,31 @@ pub fn tcgen05_mma_sp(ctx: *mlir.Context, matrixD: *const mlir.Value, matrixA: * } /// `nvvm.tcgen05.mma.sp.block_scale`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_mma_sp_block_scale(ctx: *mlir.Context, matrixD: *const mlir.Value, matrixA: *const mlir.Value, matrixB: *const mlir.Value, idesc: *const mlir.Value, enableInputD: *const mlir.Value, sparseMetadata: *const mlir.Value, scaleA: *const mlir.Value, scaleB: *const mlir.Value, kind: *const mlir.Attribute, ctaGroup: *const mlir.Attribute, blockScale: ?*const mlir.Attribute, collectorOp: ?*const mlir.Attribute, collectorOpB: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn tcgen05_mma_sp_block_scale( + ctx: *mlir.Context, + matrixD: *const mlir.Value, + matrixA: *const mlir.Value, + matrixB: *const mlir.Value, + idesc: *const mlir.Value, + enableInputD: *const mlir.Value, + sparseMetadata: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + kind: *const mlir.Attribute, + ctaGroup: *const mlir.Attribute, + opts: struct { + blockScale: ?*const mlir.Attribute = null, + collectorOp: ?*const mlir.Attribute = null, + collectorOpB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); - if (blockScale) |value| attributes.appendAssumeCapacity(.named(ctx, "blockScale", value)); - if (collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); - if (collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); + if (opts.blockScale) |value| attributes.appendAssumeCapacity(.named(ctx, "blockScale", value)); + if (opts.collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); + if (opts.collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.sp.block_scale", .{ .operands = .{ .flat = &.{ matrixD, matrixA, matrixB, idesc, enableInputD, sparseMetadata, scaleA, scaleB } }, .attributes = attributes.constSlice(), @@ -6503,26 +8058,55 @@ pub fn tcgen05_mma_sp_block_scale(ctx: *mlir.Context, matrixD: *const mlir.Value } /// `nvvm.tcgen05.mma.ws`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_mma_ws(ctx: *mlir.Context, matrixD: *const mlir.Value, matrixA: *const mlir.Value, matrixB: *const mlir.Value, idesc: *const mlir.Value, enableInputD: *const mlir.Value, zeroColMask: ?*const mlir.Value, kind: *const mlir.Attribute, collectorBBuffer: ?*const mlir.Attribute, collectorOp: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn tcgen05_mma_ws( + ctx: *mlir.Context, + matrixD: *const mlir.Value, + matrixA: *const mlir.Value, + matrixB: *const mlir.Value, + idesc: *const mlir.Value, + enableInputD: *const mlir.Value, + kind: *const mlir.Attribute, + opts: struct { + zeroColMask: ?*const mlir.Value = null, + collectorBBuffer: ?*const mlir.Attribute = null, + collectorOp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (collectorBBuffer) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorBBuffer", value)); - if (collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); + if (opts.collectorBBuffer) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorBBuffer", value)); + if (opts.collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.ws", .{ - .operands = .{ .flat = if (zeroColMask) |value| &.{ matrixD, matrixA, matrixB, idesc, enableInputD, value } else &.{ matrixD, matrixA, matrixB, idesc, enableInputD } }, + .operands = .{ .flat = if (opts.zeroColMask) |value| &.{ matrixD, matrixA, matrixB, idesc, enableInputD, value } else &.{ matrixD, matrixA, matrixB, idesc, enableInputD } }, .attributes = attributes.constSlice(), .location = location, }); } /// `nvvm.tcgen05.mma.ws.sp`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_mma_ws_sp(ctx: *mlir.Context, matrixD: *const mlir.Value, matrixA: *const mlir.Value, matrixB: *const mlir.Value, idesc: *const mlir.Value, enableInputD: *const mlir.Value, sparseMetadata: *const mlir.Value, zeroColMask: ?*const mlir.Value, kind: *const mlir.Attribute, collectorBBuffer: ?*const mlir.Attribute, collectorOp: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn tcgen05_mma_ws_sp( + ctx: *mlir.Context, + matrixD: *const mlir.Value, + matrixA: *const mlir.Value, + matrixB: *const mlir.Value, + idesc: *const mlir.Value, + enableInputD: *const mlir.Value, + sparseMetadata: *const mlir.Value, + kind: *const mlir.Attribute, + opts: struct { + zeroColMask: ?*const mlir.Value = null, + collectorBBuffer: ?*const mlir.Attribute = null, + collectorOp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (collectorBBuffer) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorBBuffer", value)); - if (collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); + if (opts.collectorBBuffer) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorBBuffer", value)); + if (opts.collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.ws.sp", .{ - .operands = .{ .flat = if (zeroColMask) |value| &.{ matrixD, matrixA, matrixB, idesc, enableInputD, sparseMetadata, value } else &.{ matrixD, matrixA, matrixB, idesc, enableInputD, sparseMetadata } }, + .operands = .{ .flat = if (opts.zeroColMask) |value| &.{ matrixD, matrixA, matrixB, idesc, enableInputD, sparseMetadata, value } else &.{ matrixD, matrixA, matrixB, idesc, enableInputD, sparseMetadata } }, .attributes = attributes.constSlice(), .location = location, }); @@ -6538,9 +8122,15 @@ pub fn tcgen05_mma_smem_desc(ctx: *mlir.Context, startAddr: *const mlir.Value, l } /// `nvvm.tcgen05.relinquish_alloc_permit`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_relinquish_alloc_permit(ctx: *mlir.Context, group: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn tcgen05_relinquish_alloc_permit( + ctx: *mlir.Context, + opts: struct { + group: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); return mlir.Operation.make(ctx, "nvvm.tcgen05.relinquish_alloc_permit", .{ .attributes = attributes.constSlice(), .location = location, @@ -6548,9 +8138,16 @@ pub fn tcgen05_relinquish_alloc_permit(ctx: *mlir.Context, group: ?*const mlir.A } /// `nvvm.tcgen05.shift`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_shift(ctx: *mlir.Context, taddr: *const mlir.Value, group: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn tcgen05_shift( + ctx: *mlir.Context, + taddr: *const mlir.Value, + opts: struct { + group: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); return mlir.Operation.make(ctx, "nvvm.tcgen05.shift", .{ .operands = .{ .flat = &.{taddr} }, .attributes = attributes.constSlice(), @@ -6559,12 +8156,22 @@ pub fn tcgen05_shift(ctx: *mlir.Context, taddr: *const mlir.Value, group: ?*cons } /// `nvvm.tcgen05.st`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_st(ctx: *mlir.Context, tmemAddr: *const mlir.Value, val: *const mlir.Value, offset: ?*const mlir.Value, unpack: ?*const mlir.Attribute, shape: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn tcgen05_st( + ctx: *mlir.Context, + tmemAddr: *const mlir.Value, + val: *const mlir.Value, + shape: *const mlir.Attribute, + opts: struct { + offset: ?*const mlir.Value = null, + unpack: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (unpack) |value| attributes.appendAssumeCapacity(.named(ctx, "unpack", value)); + if (opts.unpack) |value| attributes.appendAssumeCapacity(.named(ctx, "unpack", value)); attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); return mlir.Operation.make(ctx, "nvvm.tcgen05.st", .{ - .operands = .{ .flat = if (offset) |value| &.{ tmemAddr, val, value } else &.{ tmemAddr, val } }, + .operands = .{ .flat = if (opts.offset) |value| &.{ tmemAddr, val, value } else &.{ tmemAddr, val } }, .attributes = attributes.constSlice(), .location = location, }); @@ -6601,29 +8208,50 @@ pub fn tensormap_cp_fenceproxy(ctx: *mlir.Context, dst: *const mlir.Value, src: } /// `nvvm.tensormap.replace`. Result types are explicit; attributes use mlir.Attribute. -pub fn tensormap_replace(ctx: *mlir.Context, addr: *const mlir.Value, new_value: ?*const mlir.Value, field: *const mlir.Attribute, ord: ?*const mlir.Attribute, new_value_attr: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn tensormap_replace( + ctx: *mlir.Context, + addr: *const mlir.Value, + field: *const mlir.Attribute, + opts: struct { + new_value: ?*const mlir.Value = null, + ord: ?*const mlir.Attribute = null, + new_value_attr: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; attributes.appendAssumeCapacity(.named(ctx, "field", field)); - if (ord) |value| attributes.appendAssumeCapacity(.named(ctx, "ord", value)); - if (new_value_attr) |value| attributes.appendAssumeCapacity(.named(ctx, "new_value_attr", value)); + if (opts.ord) |value| attributes.appendAssumeCapacity(.named(ctx, "ord", value)); + if (opts.new_value_attr) |value| attributes.appendAssumeCapacity(.named(ctx, "new_value_attr", value)); return mlir.Operation.make(ctx, "nvvm.tensormap.replace", .{ - .operands = .{ .flat = if (new_value) |value| &.{ addr, value } else &.{addr} }, + .operands = .{ .flat = if (opts.new_value) |value| &.{ addr, value } else &.{addr} }, .attributes = attributes.constSlice(), .location = location, }); } /// `nvvm.trace.mark`. Result types are explicit; attributes use mlir.Attribute. -pub fn trace_mark(ctx: *mlir.Context, payload: ?*const mlir.Value, predicate: ?*const mlir.Value, eventType: *const mlir.Attribute, domain: *const mlir.Attribute, event: *const mlir.Attribute, payloadDescriptor: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn trace_mark( + ctx: *mlir.Context, + eventType: *const mlir.Attribute, + domain: *const mlir.Attribute, + event: *const mlir.Attribute, + opts: struct { + payload: ?*const mlir.Value = null, + predicate: ?*const mlir.Value = null, + payloadDescriptor: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; attributes.appendAssumeCapacity(.named(ctx, "eventType", eventType)); attributes.appendAssumeCapacity(.named(ctx, "domain", domain)); attributes.appendAssumeCapacity(.named(ctx, "event", event)); - if (payloadDescriptor) |value| attributes.appendAssumeCapacity(.named(ctx, "payloadDescriptor", value)); + if (opts.payloadDescriptor) |value| attributes.appendAssumeCapacity(.named(ctx, "payloadDescriptor", value)); return mlir.Operation.make(ctx, "nvvm.trace.mark", .{ .operands = .{ .variadic = &.{ - if (payload) |value| &.{value} else &.{}, - if (predicate) |value| &.{value} else &.{}, + if (opts.payload) |value| &.{value} else &.{}, + if (opts.predicate) |value| &.{value} else &.{}, } }, .attributes = attributes.constSlice(), .location = location, @@ -6657,7 +8285,26 @@ pub fn wgmma_fence_aligned(ctx: *mlir.Context, location: *const mlir.Location) * } /// `nvvm.wgmma.mma_async`. Result types are explicit; attributes use mlir.Attribute. -pub fn wgmma_mma_async(ctx: *mlir.Context, inouts: *const mlir.Value, descriptorA: *const mlir.Value, descriptorB: *const mlir.Value, results__type: *const mlir.Type, shape: *const mlir.Attribute, typeA: *const mlir.Attribute, typeB: *const mlir.Attribute, typeD: *const mlir.Attribute, scaleD: *const mlir.Attribute, scaleA: *const mlir.Attribute, scaleB: *const mlir.Attribute, layoutA: *const mlir.Attribute, layoutB: *const mlir.Attribute, satfinite: ?*const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { +pub fn wgmma_mma_async( + ctx: *mlir.Context, + inouts: *const mlir.Value, + descriptorA: *const mlir.Value, + descriptorB: *const mlir.Value, + results__type: *const mlir.Type, + shape: *const mlir.Attribute, + typeA: *const mlir.Attribute, + typeB: *const mlir.Attribute, + typeD: *const mlir.Attribute, + scaleD: *const mlir.Attribute, + scaleA: *const mlir.Attribute, + scaleB: *const mlir.Attribute, + layoutA: *const mlir.Attribute, + layoutB: *const mlir.Attribute, + opts: struct { + satfinite: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { var attributes: stdx.BoundedArray(mlir.NamedAttribute, 10) = .empty; attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); attributes.appendAssumeCapacity(.named(ctx, "typeA", typeA)); @@ -6668,7 +8315,7 @@ pub fn wgmma_mma_async(ctx: *mlir.Context, inouts: *const mlir.Value, descriptor attributes.appendAssumeCapacity(.named(ctx, "scaleB", scaleB)); attributes.appendAssumeCapacity(.named(ctx, "layoutA", layoutA)); attributes.appendAssumeCapacity(.named(ctx, "layoutB", layoutB)); - if (satfinite) |value| attributes.appendAssumeCapacity(.named(ctx, "satfinite", value)); + if (opts.satfinite) |value| attributes.appendAssumeCapacity(.named(ctx, "satfinite", value)); return mlir.Operation.make(ctx, "nvvm.wgmma.mma_async", .{ .operands = .{ .flat = &.{ inouts, descriptorA, descriptorB } }, .results = .{ .flat = &.{results__type} }, @@ -6740,391 +8387,3 @@ pub fn wmma_store(ctx: *mlir.Context, ptr: *const mlir.Value, args: []const *con .location = location, }); } - -test "every enum attribute prints as the compiler does" { - @setEvalBranchQuota(1_000_000); - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - mlir.DialectHandle.fromString(dialect_handle).insertDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - inline for (.{ - .{ AtomicOpKindAttr, "#nvvm" }, - .{ BarrierReductionAttr, "#nvvm.reduction<{s}>" }, - .{ BarrierReduxKindAttr, "#nvvm.barrier_redux_kind<{s}>" }, - .{ BlockScaleFormatAttr, "#nvvm.block_scale_format<{s}>" }, - .{ CTAGroupKindAttr, "#nvvm.cta_group<{s}>" }, - .{ CVTPackFloatKindAttr, "#nvvm.packfloat_type<{s}>" }, - .{ CacheEvictionPriorityAttr, "#nvvm" }, - .{ ClusterLaunchControlQueryTypeAttr, "#nvvm" }, - .{ CompareOpKindAttr, "#nvvm.op<{s}>" }, - .{ ConvertFP4TypeAttr, "#nvvm.convert_fp4_type<{s}>" }, - .{ ConvertFP8TypeAttr, "#nvvm.convert_fp8_type<{s}>" }, - .{ ConvertScaleKindAttr, "#nvvm.convert_scale_kind<{s}>" }, - .{ DotAccumulateTypeAttr, "#nvvm.dot_accumulate_type<{s}>" }, - .{ EvictKindAttr, "#nvvm.evict_kind<{s}>" }, - .{ FPRoundingModeAttr, "#nvvm.fp_rnd_mode<{s}>" }, - .{ GridDepActionKindAttr, "#nvvm" }, - .{ IntegerRoundingModeAttr, "#nvvm.int_rnd_mode<{s}>" }, - .{ L2PrefetchSizeAttr, "#nvvm.l2_prefetch<{s}>" }, - .{ LdStMatrixEltTypeAttr, "#nvvm.ld_st_matrix_elt_type<{s}>" }, - .{ LoadCacheModifierExtKindAttr, "#nvvm.load_cache_modifier_ext<{s}>" }, - .{ LoadCacheModifierKindAttr, "#nvvm" }, - .{ LoadShapeAttr, "#nvvm.load_shape<{s}>" }, - .{ LoadSrcFormatAttr, "#nvvm.load_src_format<{s}>" }, - .{ MBarrierLayoutAttr, "#nvvm.mbarrier_layout<{s}>" }, - .{ MBarrierPhaseAttr, "#nvvm.mbarrier_phase<{s}>" }, - .{ MBarrierScopeKindAttr, "#nvvm.mbar_scope<{s}>" }, - .{ MBarrierSpaceKindAttr, "#nvvm.mbar_space<{s}>" }, - .{ MBarrierTxnKindAttr, "#nvvm.mbar_txn_kind<{s}>" }, - .{ MBarrierWaitKindAttr, "#nvvm.mbar_wait<{s}>" }, - .{ MMAB1OpAttr, "#nvvm.mma_b1op<{s}>" }, - .{ MMABlockScaleKindAttr, "#nvvm.block_scale_kind<{s}>" }, - .{ MMACtaCountAttr, "#nvvm.mma_cta_count<{s}>" }, - .{ MMAFragAttr, "#nvvm.mma_frag<{s}>" }, - .{ MMAIntOverflowAttr, "#nvvm.mma_int_overflow<{s}>" }, - .{ MMAKindAttr, "#nvvm.mma_kind<{s}>" }, - .{ MMALayoutAttr, "#nvvm.mma_layout<{s}>" }, - .{ MMATypesAttr, "#nvvm.mma_type<{s}>" }, - .{ MatchSyncKindAttr, "#nvvm" }, - .{ MemOrderKindAttr, "#nvvm.mem_order<{s}>" }, - .{ MemScopeKindAttr, "#nvvm.mem_scope<{s}>" }, - .{ MulModeAttr, "#nvvm" }, - .{ NVVMMemorySpaceAttr, "#nvvm.memory_space<{s}>" }, - .{ PermuteModeAttr, "#nvvm.permute_mode<{s}>" }, - .{ PrefetchCacheLevelAttr, "#nvvm" }, - .{ ProxyKindAttr, "#nvvm.proxy_kind<{s}>" }, - .{ ReductionKindAttr, "#nvvm" }, - .{ ReductionOpAttr, "#nvvm.red_op<{s}>" }, - .{ ReductionTypeAttr, "#nvvm.red_type<{s}>" }, - .{ SPCompressElemSizeAttr, "#nvvm.spcomp_elem_size<{s}>" }, - .{ SPCompressFactorTypeAttr, "#nvvm.spcomp_factor<{s}>" }, - .{ SPCompressIndexSizeAttr, "#nvvm.spcomp_index_size<{s}>" }, - .{ SPCompressOpKindAttr, "#nvvm.spcomp_op_kind<{s}>" }, - .{ SPCompressRepFactorAttr, "#nvvm.spcomp_rep_factor<{s}>" }, - .{ SPDecompressElemSizeAttr, "#nvvm.spdecomp_elem_size<{s}>" }, - .{ SPDecompressFactorTypeAttr, "#nvvm.spdecomp_factor<{s}>" }, - .{ SPDecompressIndexSizeAttr, "#nvvm.spdecomp_index_size<{s}>" }, - .{ SPDecompressRepFactorAttr, "#nvvm.spdecomp_rep_factor<{s}>" }, - .{ SaturationModeAttr, "#nvvm.sat_mode<{s}>" }, - .{ SaturationModeKindAttr, "#nvvm.sat<{s}>" }, - .{ ScaleVecSizeAttr, "#nvvm.scale_vec_size<{s}>" }, - .{ SetMaxRegisterActionAttr, "#nvvm" }, - .{ SharedSpaceAttr, "#nvvm.shared_space<{s}>" }, - .{ ShflKindAttr, "#nvvm" }, - .{ SparsityFormatAttr, "#nvvm.sparsity_format<{s}>" }, - .{ StateSpaceAttr, "#nvvm.state_space<{s}>" }, - .{ StoreCacheModifierKindAttr, "#nvvm.store_cache_modifier<{s}>" }, - .{ StoreShapeAttr, "#nvvm.store_shape<{s}>" }, - .{ TCBarParamAttr, "#nvvm.TCBarParam<{s}>" }, - .{ TMALoadModeAttr, "#nvvm.tma_load_mode<{s}>" }, - .{ TMAReduxKindAttr, "#nvvm.tma_redux_kind<{s}>" }, - .{ TMAStoreModeAttr, "#nvvm.tma_store_mode<{s}>" }, - .{ Tcgen05CpMulticastAttr, "#nvvm.tcgen05_cp_multicast<{s}>" }, - .{ Tcgen05CpShapeAttr, "#nvvm.tcgen05_cp_shape<{s}>" }, - .{ Tcgen05CpSrcFormatAttr, "#nvvm.tcgen05_cp_src_fmt<{s}>" }, - .{ Tcgen05FenceKindAttr, "#nvvm.tcgen05_fence<{s}>" }, - .{ Tcgen05LdStShapeAttr, "#nvvm.tcgen05_ldst_shape<{s}>" }, - .{ Tcgen05MMABlockScaleAttr, "#nvvm.tcgen05_mma_block_scale<{s}>" }, - .{ Tcgen05MMACollectorBBufferAttr, "#nvvm.tcgen05_mma_collectorb<{s}>" }, - .{ Tcgen05MMACollectorOpAttr, "#nvvm.tcgen05_mma_collectorop<{s}>" }, - .{ Tcgen05MMAKindAttr, "#nvvm.tcgen05_mma_kind<{s}>" }, - .{ Tcgen05WaitKindAttr, "#nvvm.tcgen05_wait<{s}>" }, - .{ TensormapElemtypeAttr, "#nvvm.tensormap_elemtype<{s}>" }, - .{ TensormapFieldAttr, "#nvvm" }, - .{ TensormapFillModeAttr, "#nvvm.tensormap_fill_mode<{s}>" }, - .{ TensormapInterleaveLayoutAttr, "#nvvm.tensormap_interleave_layout<{s}>" }, - .{ TensormapSwizzleAtomicityAttr, "#nvvm.tensormap_swizzle_atomicity<{s}>" }, - .{ TensormapSwizzleModeAttr, "#nvvm.tensormap_swizzle_mode<{s}>" }, - .{ TmemLayoutAttr, "#nvvm.TmemLayout<{s}>" }, - .{ ValidatePatternAttr, "#nvvm.validate_pattern<{s}>" }, - .{ VoteSyncKindAttr, "#nvvm" }, - .{ WGMMAScaleInAttr, "#nvvm.wgmma_scale_in<{s}>" }, - .{ WGMMAScaleOutAttr, "#nvvm.wgmma_scale_out<{s}>" }, - .{ WGMMATypesAttr, "#nvvm.wgmma_type<{s}>" }, - }) |example| { - const T = example[0]; - const E = @FieldType(T.InitArgs, "value"); - inline for (@typeInfo(E).@"enum".field_names) |name| { - const attr = try T.get(ctx, .{ .value = @field(E, name) }); - try std.testing.expectEqual(@field(E, name), attr.getValue()); - var buf: [256]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{attr}); - try std.testing.expectEqualStrings(std.fmt.comptimePrint(example[1], .{name}), w.buffered()); - } - } -} - -/// Every operation bound here. -pub const operation_names: []const []const u8 = &.{ - "nvvm.add.packed.bf16x2.f32x2.f32x2", - "nvvm.add.packed.f16x2.f32x2.f32x2", - "nvvm.add.packed.f32x2", - "nvvm.add.packed.f32x2.bf16x2.f32x2", - "nvvm.add.packed.f32x2.f16x2.f32x2", - "nvvm.addf", - "nvvm.applypriority.async.bulk", - "nvvm.applypriority.async.bulk.tensor", - "nvvm.applypriority.async.bulk.tensor.override", - "nvvm.atomicrmw", - "nvvm.bar.warp.sync", - "nvvm.barrier", - "nvvm.barrier.arrive", - "nvvm.barrier.cta.arrive", - "nvvm.barrier.cta.red", - "nvvm.barrier.cta.sync", - "nvvm.barrier0", - "nvvm.breakpoint", - "nvvm.clmad", - "nvvm.cluster.arrive", - "nvvm.cluster.arrive.relaxed", - "nvvm.cluster.wait", - "nvvm.clusterlaunchcontrol.query.cancel", - "nvvm.clusterlaunchcontrol.try.cancel", - "nvvm.compare.and.set", - "nvvm.convert.and.pack.integer", - "nvvm.convert.bf16x2.to.f4x2", - "nvvm.convert.bf16x2.to.f6x2", - "nvvm.convert.bf16x2.to.f8x2", - "nvvm.convert.bf16x2.to.s2f6x2", - "nvvm.convert.f16x2.to.f4x2", - "nvvm.convert.f16x2.to.f6x2", - "nvvm.convert.f16x2.to.f8x2", - "nvvm.convert.f32x2.to.bf16x2", - "nvvm.convert.f32x2.to.f16x2", - "nvvm.convert.f32x2.to.f4x2", - "nvvm.convert.f32x2.to.f6x2", - "nvvm.convert.f32x2.to.f8x2", - "nvvm.convert.f32x2.to.s2f6x2", - "nvvm.convert.f32x4.to.f4x4", - "nvvm.convert.f32x4.to.f6x4", - "nvvm.convert.f32x4.to.f8x4", - "nvvm.convert.f4x2.to.f16x2", - "nvvm.convert.f6x2.to.f16x2", - "nvvm.convert.f8x2.to.bf16x2", - "nvvm.convert.f8x2.to.f16x2", - "nvvm.convert.float.to.integer", - "nvvm.convert.float.to.tf32", - "nvvm.convert.s2f6x2.to.bf16x2", - "nvvm.cos", - "nvvm.cp.async.bulk.commit.group", - "nvvm.cp.async.bulk.global.shared.cta", - "nvvm.cp.async.bulk.prefetch", - "nvvm.cp.async.bulk.shared.cluster.global", - "nvvm.cp.async.bulk.shared.cluster.shared.cta", - "nvvm.cp.async.bulk.tensor.global.shared.cta", - "nvvm.cp.async.bulk.tensor.global.shared.cta.override", - "nvvm.cp.async.bulk.tensor.prefetch", - "nvvm.cp.async.bulk.tensor.prefetch.override", - "nvvm.cp.async.bulk.tensor.reduce", - "nvvm.cp.async.bulk.tensor.reduce.override", - "nvvm.cp.async.bulk.tensor.shared.cluster.global", - "nvvm.cp.async.bulk.tensor.shared.cluster.global.override", - "nvvm.cp.async.bulk.wait_group", - "nvvm.cp.async.commit.group", - "nvvm.cp.async.mbarrier.arrive", - "nvvm.cp.async.shared.global", - "nvvm.cp.async.wait.group", - "nvvm.cvt.packfloat", - "nvvm.cvt.packfloat.f32", - "nvvm.cvt.to.f4x8.packed", - "nvvm.dot.accumulate.2way", - "nvvm.dot.accumulate.4way", - "nvvm.elect.sync", - "nvvm.ex2", - "nvvm.exit", - "nvvm.fabs", - "nvvm.fence.mbarrier.init", - "nvvm.fence.proxy", - "nvvm.fence.proxy.acquire", - "nvvm.fence.proxy.release", - "nvvm.fence.proxy.sync_restrict", - "nvvm.fence.sc.cluster", - "nvvm.fence.sync_restrict", - "nvvm.fma", - "nvvm.fma.packed.f32x2", - "nvvm.fma.packed.f32x2.bf16x2.f32x2.f32x2", - "nvvm.fma.packed.f32x2.f16x2.f32x2.f32x2", - "nvvm.fmax", - "nvvm.fmin", - "nvvm.griddepcontrol", - "nvvm.inline_ptx", - "nvvm.ldmatrix", - "nvvm.load.ext", - "nvvm.log2", - "nvvm.mapa", - "nvvm.match.sync", - "nvvm.mbarrier.arrive", - "nvvm.mbarrier.arrive.expect_tx", - "nvvm.mbarrier.arrive.nocomplete", - "nvvm.mbarrier.arrive_drop", - "nvvm.mbarrier.arrive_drop.expect_tx", - "nvvm.mbarrier.arrive_drop.nocomplete", - "nvvm.mbarrier.check.layout", - "nvvm.mbarrier.complete_tx", - "nvvm.mbarrier.expect_tx", - "nvvm.mbarrier.init", - "nvvm.mbarrier.inval", - "nvvm.mbarrier.test.wait", - "nvvm.mbarrier.try_wait", - "nvvm.mbarrier.try_wait.parity", - "nvvm.mbarrier.try_wait.parity.timelimit", - "nvvm.mbarrier.try_wait.timelimit", - "nvvm.mbarrier.txn", - "nvvm.mbarrier.txn.cta", - "nvvm.mbarrier.wait", - "nvvm.mbarrier.wait.parity", - "nvvm.memory.barrier", - "nvvm.mma.block_scale", - "nvvm.mma.block_scale.internal", - "nvvm.mma.sp.block_scale", - "nvvm.mma.sp.block_scale.internal", - "nvvm.mma.sp.sync", - "nvvm.mma.sync", - "nvvm.mma_smem_desc", - "nvvm.movmatrix", - "nvvm.mul", - "nvvm.mul.packed.bf16x2.bf16x2.f16x2", - "nvvm.mul.packed.bf16x2.f32x2.f32x2", - "nvvm.mul.packed.f16x2.f16x2.bf16x2", - "nvvm.mul.packed.f16x2.f32x2.f32x2", - "nvvm.mul.packed.f32x2", - "nvvm.multimem.ld.reduce", - "nvvm.multimem.red", - "nvvm.multimem.st", - "nvvm.nanosleep", - "nvvm.pmevent", - "nvvm.prefetch", - "nvvm.prmt", - "nvvm.rcp.approx.ftz.f", - "nvvm.read.ptx.sreg.aggr.smem.size", - "nvvm.read.ptx.sreg.clock", - "nvvm.read.ptx.sreg.clock64", - "nvvm.read.ptx.sreg.cluster.ctaid.x", - "nvvm.read.ptx.sreg.cluster.ctaid.y", - "nvvm.read.ptx.sreg.cluster.ctaid.z", - "nvvm.read.ptx.sreg.cluster.ctarank", - "nvvm.read.ptx.sreg.cluster.nctaid.x", - "nvvm.read.ptx.sreg.cluster.nctaid.y", - "nvvm.read.ptx.sreg.cluster.nctaid.z", - "nvvm.read.ptx.sreg.cluster.nctarank", - "nvvm.read.ptx.sreg.clusterid.x", - "nvvm.read.ptx.sreg.clusterid.y", - "nvvm.read.ptx.sreg.clusterid.z", - "nvvm.read.ptx.sreg.ctaid.x", - "nvvm.read.ptx.sreg.ctaid.y", - "nvvm.read.ptx.sreg.ctaid.z", - "nvvm.read.ptx.sreg.dynamic.smem.size", - "nvvm.read.ptx.sreg.envreg0", - "nvvm.read.ptx.sreg.envreg1", - "nvvm.read.ptx.sreg.envreg10", - "nvvm.read.ptx.sreg.envreg11", - "nvvm.read.ptx.sreg.envreg12", - "nvvm.read.ptx.sreg.envreg13", - "nvvm.read.ptx.sreg.envreg14", - "nvvm.read.ptx.sreg.envreg15", - "nvvm.read.ptx.sreg.envreg16", - "nvvm.read.ptx.sreg.envreg17", - "nvvm.read.ptx.sreg.envreg18", - "nvvm.read.ptx.sreg.envreg19", - "nvvm.read.ptx.sreg.envreg2", - "nvvm.read.ptx.sreg.envreg20", - "nvvm.read.ptx.sreg.envreg21", - "nvvm.read.ptx.sreg.envreg22", - "nvvm.read.ptx.sreg.envreg23", - "nvvm.read.ptx.sreg.envreg24", - "nvvm.read.ptx.sreg.envreg25", - "nvvm.read.ptx.sreg.envreg26", - "nvvm.read.ptx.sreg.envreg27", - "nvvm.read.ptx.sreg.envreg28", - "nvvm.read.ptx.sreg.envreg29", - "nvvm.read.ptx.sreg.envreg3", - "nvvm.read.ptx.sreg.envreg30", - "nvvm.read.ptx.sreg.envreg31", - "nvvm.read.ptx.sreg.envreg4", - "nvvm.read.ptx.sreg.envreg5", - "nvvm.read.ptx.sreg.envreg6", - "nvvm.read.ptx.sreg.envreg7", - "nvvm.read.ptx.sreg.envreg8", - "nvvm.read.ptx.sreg.envreg9", - "nvvm.read.ptx.sreg.globaltimer", - "nvvm.read.ptx.sreg.globaltimer.lo", - "nvvm.read.ptx.sreg.gridid", - "nvvm.read.ptx.sreg.laneid", - "nvvm.read.ptx.sreg.lanemask.eq", - "nvvm.read.ptx.sreg.lanemask.ge", - "nvvm.read.ptx.sreg.lanemask.gt", - "nvvm.read.ptx.sreg.lanemask.le", - "nvvm.read.ptx.sreg.lanemask.lt", - "nvvm.read.ptx.sreg.nclusterid.x", - "nvvm.read.ptx.sreg.nclusterid.y", - "nvvm.read.ptx.sreg.nclusterid.z", - "nvvm.read.ptx.sreg.nctaid.x", - "nvvm.read.ptx.sreg.nctaid.y", - "nvvm.read.ptx.sreg.nctaid.z", - "nvvm.read.ptx.sreg.nsmid", - "nvvm.read.ptx.sreg.ntid.x", - "nvvm.read.ptx.sreg.ntid.y", - "nvvm.read.ptx.sreg.ntid.z", - "nvvm.read.ptx.sreg.nwarpid", - "nvvm.read.ptx.sreg.smid", - "nvvm.read.ptx.sreg.tid.x", - "nvvm.read.ptx.sreg.tid.y", - "nvvm.read.ptx.sreg.tid.z", - "nvvm.read.ptx.sreg.total.smem.size", - "nvvm.read.ptx.sreg.warpid", - "nvvm.read.ptx.sreg.warpsize", - "nvvm.red", - "nvvm.redux.sync", - "nvvm.rsqrt", - "nvvm.setmaxregister", - "nvvm.shfl.sync", - "nvvm.sin", - "nvvm.spcompress", - "nvvm.spdecompress", - "nvvm.st.bulk", - "nvvm.stmatrix", - "nvvm.store.async", - "nvvm.store.ext", - "nvvm.sub.packed.bf16x2.f32x2.f32x2", - "nvvm.sub.packed.f16x2.f32x2.f32x2", - "nvvm.sub.packed.f32x2", - "nvvm.sub.packed.f32x2.bf16x2.f32x2", - "nvvm.sub.packed.f32x2.f16x2.f32x2", - "nvvm.subf", - "nvvm.tcgen05.alloc", - "nvvm.tcgen05.commit", - "nvvm.tcgen05.cp", - "nvvm.tcgen05.dealloc", - "nvvm.tcgen05.fence", - "nvvm.tcgen05.ld", - "nvvm.tcgen05.ld.red", - "nvvm.tcgen05.ld.red.spcompress", - "nvvm.tcgen05.ld.spcompress", - "nvvm.tcgen05.mma", - "nvvm.tcgen05.mma.block_scale", - "nvvm.tcgen05.mma.block_scale.decompress_b", - "nvvm.tcgen05.mma.decompress_b", - "nvvm.tcgen05.mma.sp", - "nvvm.tcgen05.mma.sp.block_scale", - "nvvm.tcgen05.mma.ws", - "nvvm.tcgen05.mma.ws.sp", - "nvvm.tcgen05.mma_smem_desc", - "nvvm.tcgen05.relinquish_alloc_permit", - "nvvm.tcgen05.shift", - "nvvm.tcgen05.st", - "nvvm.tcgen05.wait", - "nvvm.tcgen05_mma_smem_desc_v2", - "nvvm.tensormap.cp_fenceproxy", - "nvvm.tensormap.replace", - "nvvm.trace.mark", - "nvvm.vote.sync", - "nvvm.wgmma.commit.group.sync.aligned", - "nvvm.wgmma.fence.aligned", - "nvvm.wgmma.mma_async", - "nvvm.wgmma.wait.group.sync.aligned", - "nvvm.wmma.load", - "nvvm.wmma.mma", - "nvvm.wmma.store", -}; diff --git a/mlir/dialects/nvvm/test.zig b/mlir/dialects/nvvm/test.zig new file mode 100644 index 00000000..6129c078 --- /dev/null +++ b/mlir/dialects/nvvm/test.zig @@ -0,0 +1,472 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const nvvm = @import("mlir/dialects/nvvm"); + +const operation_names: []const []const u8 = &.{ + "nvvm.add.packed.bf16x2.f32x2.f32x2", + "nvvm.add.packed.f16x2.f32x2.f32x2", + "nvvm.add.packed.f32x2", + "nvvm.add.packed.f32x2.bf16x2.f32x2", + "nvvm.add.packed.f32x2.f16x2.f32x2", + "nvvm.addf", + "nvvm.applypriority.async.bulk", + "nvvm.applypriority.async.bulk.tensor", + "nvvm.applypriority.async.bulk.tensor.override", + "nvvm.atomicrmw", + "nvvm.bar.warp.sync", + "nvvm.barrier", + "nvvm.barrier.arrive", + "nvvm.barrier.cta.arrive", + "nvvm.barrier.cta.red", + "nvvm.barrier.cta.sync", + "nvvm.barrier0", + "nvvm.breakpoint", + "nvvm.clmad", + "nvvm.cluster.arrive", + "nvvm.cluster.arrive.relaxed", + "nvvm.cluster.wait", + "nvvm.clusterlaunchcontrol.query.cancel", + "nvvm.clusterlaunchcontrol.try.cancel", + "nvvm.compare.and.set", + "nvvm.convert.and.pack.integer", + "nvvm.convert.bf16x2.to.f4x2", + "nvvm.convert.bf16x2.to.f6x2", + "nvvm.convert.bf16x2.to.f8x2", + "nvvm.convert.bf16x2.to.s2f6x2", + "nvvm.convert.f16x2.to.f4x2", + "nvvm.convert.f16x2.to.f6x2", + "nvvm.convert.f16x2.to.f8x2", + "nvvm.convert.f32x2.to.bf16x2", + "nvvm.convert.f32x2.to.f16x2", + "nvvm.convert.f32x2.to.f4x2", + "nvvm.convert.f32x2.to.f6x2", + "nvvm.convert.f32x2.to.f8x2", + "nvvm.convert.f32x2.to.s2f6x2", + "nvvm.convert.f32x4.to.f4x4", + "nvvm.convert.f32x4.to.f6x4", + "nvvm.convert.f32x4.to.f8x4", + "nvvm.convert.f4x2.to.f16x2", + "nvvm.convert.f6x2.to.f16x2", + "nvvm.convert.f8x2.to.bf16x2", + "nvvm.convert.f8x2.to.f16x2", + "nvvm.convert.float.to.integer", + "nvvm.convert.float.to.tf32", + "nvvm.convert.s2f6x2.to.bf16x2", + "nvvm.cos", + "nvvm.cp.async.bulk.commit.group", + "nvvm.cp.async.bulk.global.shared.cta", + "nvvm.cp.async.bulk.prefetch", + "nvvm.cp.async.bulk.shared.cluster.global", + "nvvm.cp.async.bulk.shared.cluster.shared.cta", + "nvvm.cp.async.bulk.tensor.global.shared.cta", + "nvvm.cp.async.bulk.tensor.global.shared.cta.override", + "nvvm.cp.async.bulk.tensor.prefetch", + "nvvm.cp.async.bulk.tensor.prefetch.override", + "nvvm.cp.async.bulk.tensor.reduce", + "nvvm.cp.async.bulk.tensor.reduce.override", + "nvvm.cp.async.bulk.tensor.shared.cluster.global", + "nvvm.cp.async.bulk.tensor.shared.cluster.global.override", + "nvvm.cp.async.bulk.wait_group", + "nvvm.cp.async.commit.group", + "nvvm.cp.async.mbarrier.arrive", + "nvvm.cp.async.shared.global", + "nvvm.cp.async.wait.group", + "nvvm.cvt.packfloat", + "nvvm.cvt.packfloat.f32", + "nvvm.cvt.to.f4x8.packed", + "nvvm.dot.accumulate.2way", + "nvvm.dot.accumulate.4way", + "nvvm.elect.sync", + "nvvm.ex2", + "nvvm.exit", + "nvvm.fabs", + "nvvm.fence.mbarrier.init", + "nvvm.fence.proxy", + "nvvm.fence.proxy.acquire", + "nvvm.fence.proxy.release", + "nvvm.fence.proxy.sync_restrict", + "nvvm.fence.sc.cluster", + "nvvm.fence.sync_restrict", + "nvvm.fma", + "nvvm.fma.packed.f32x2", + "nvvm.fma.packed.f32x2.bf16x2.f32x2.f32x2", + "nvvm.fma.packed.f32x2.f16x2.f32x2.f32x2", + "nvvm.fmax", + "nvvm.fmin", + "nvvm.griddepcontrol", + "nvvm.inline_ptx", + "nvvm.ldmatrix", + "nvvm.load.ext", + "nvvm.log2", + "nvvm.mapa", + "nvvm.match.sync", + "nvvm.mbarrier.arrive", + "nvvm.mbarrier.arrive.expect_tx", + "nvvm.mbarrier.arrive.nocomplete", + "nvvm.mbarrier.arrive_drop", + "nvvm.mbarrier.arrive_drop.expect_tx", + "nvvm.mbarrier.arrive_drop.nocomplete", + "nvvm.mbarrier.check.layout", + "nvvm.mbarrier.complete_tx", + "nvvm.mbarrier.expect_tx", + "nvvm.mbarrier.init", + "nvvm.mbarrier.inval", + "nvvm.mbarrier.test.wait", + "nvvm.mbarrier.try_wait", + "nvvm.mbarrier.try_wait.parity", + "nvvm.mbarrier.try_wait.parity.timelimit", + "nvvm.mbarrier.try_wait.timelimit", + "nvvm.mbarrier.txn", + "nvvm.mbarrier.txn.cta", + "nvvm.mbarrier.wait", + "nvvm.mbarrier.wait.parity", + "nvvm.memory.barrier", + "nvvm.mma.block_scale", + "nvvm.mma.block_scale.internal", + "nvvm.mma.sp.block_scale", + "nvvm.mma.sp.block_scale.internal", + "nvvm.mma.sp.sync", + "nvvm.mma.sync", + "nvvm.mma_smem_desc", + "nvvm.movmatrix", + "nvvm.mul", + "nvvm.mul.packed.bf16x2.bf16x2.f16x2", + "nvvm.mul.packed.bf16x2.f32x2.f32x2", + "nvvm.mul.packed.f16x2.f16x2.bf16x2", + "nvvm.mul.packed.f16x2.f32x2.f32x2", + "nvvm.mul.packed.f32x2", + "nvvm.multimem.ld.reduce", + "nvvm.multimem.red", + "nvvm.multimem.st", + "nvvm.nanosleep", + "nvvm.pmevent", + "nvvm.prefetch", + "nvvm.prmt", + "nvvm.rcp.approx.ftz.f", + "nvvm.read.ptx.sreg.aggr.smem.size", + "nvvm.read.ptx.sreg.clock", + "nvvm.read.ptx.sreg.clock64", + "nvvm.read.ptx.sreg.cluster.ctaid.x", + "nvvm.read.ptx.sreg.cluster.ctaid.y", + "nvvm.read.ptx.sreg.cluster.ctaid.z", + "nvvm.read.ptx.sreg.cluster.ctarank", + "nvvm.read.ptx.sreg.cluster.nctaid.x", + "nvvm.read.ptx.sreg.cluster.nctaid.y", + "nvvm.read.ptx.sreg.cluster.nctaid.z", + "nvvm.read.ptx.sreg.cluster.nctarank", + "nvvm.read.ptx.sreg.clusterid.x", + "nvvm.read.ptx.sreg.clusterid.y", + "nvvm.read.ptx.sreg.clusterid.z", + "nvvm.read.ptx.sreg.ctaid.x", + "nvvm.read.ptx.sreg.ctaid.y", + "nvvm.read.ptx.sreg.ctaid.z", + "nvvm.read.ptx.sreg.dynamic.smem.size", + "nvvm.read.ptx.sreg.envreg0", + "nvvm.read.ptx.sreg.envreg1", + "nvvm.read.ptx.sreg.envreg10", + "nvvm.read.ptx.sreg.envreg11", + "nvvm.read.ptx.sreg.envreg12", + "nvvm.read.ptx.sreg.envreg13", + "nvvm.read.ptx.sreg.envreg14", + "nvvm.read.ptx.sreg.envreg15", + "nvvm.read.ptx.sreg.envreg16", + "nvvm.read.ptx.sreg.envreg17", + "nvvm.read.ptx.sreg.envreg18", + "nvvm.read.ptx.sreg.envreg19", + "nvvm.read.ptx.sreg.envreg2", + "nvvm.read.ptx.sreg.envreg20", + "nvvm.read.ptx.sreg.envreg21", + "nvvm.read.ptx.sreg.envreg22", + "nvvm.read.ptx.sreg.envreg23", + "nvvm.read.ptx.sreg.envreg24", + "nvvm.read.ptx.sreg.envreg25", + "nvvm.read.ptx.sreg.envreg26", + "nvvm.read.ptx.sreg.envreg27", + "nvvm.read.ptx.sreg.envreg28", + "nvvm.read.ptx.sreg.envreg29", + "nvvm.read.ptx.sreg.envreg3", + "nvvm.read.ptx.sreg.envreg30", + "nvvm.read.ptx.sreg.envreg31", + "nvvm.read.ptx.sreg.envreg4", + "nvvm.read.ptx.sreg.envreg5", + "nvvm.read.ptx.sreg.envreg6", + "nvvm.read.ptx.sreg.envreg7", + "nvvm.read.ptx.sreg.envreg8", + "nvvm.read.ptx.sreg.envreg9", + "nvvm.read.ptx.sreg.globaltimer", + "nvvm.read.ptx.sreg.globaltimer.lo", + "nvvm.read.ptx.sreg.gridid", + "nvvm.read.ptx.sreg.laneid", + "nvvm.read.ptx.sreg.lanemask.eq", + "nvvm.read.ptx.sreg.lanemask.ge", + "nvvm.read.ptx.sreg.lanemask.gt", + "nvvm.read.ptx.sreg.lanemask.le", + "nvvm.read.ptx.sreg.lanemask.lt", + "nvvm.read.ptx.sreg.nclusterid.x", + "nvvm.read.ptx.sreg.nclusterid.y", + "nvvm.read.ptx.sreg.nclusterid.z", + "nvvm.read.ptx.sreg.nctaid.x", + "nvvm.read.ptx.sreg.nctaid.y", + "nvvm.read.ptx.sreg.nctaid.z", + "nvvm.read.ptx.sreg.nsmid", + "nvvm.read.ptx.sreg.ntid.x", + "nvvm.read.ptx.sreg.ntid.y", + "nvvm.read.ptx.sreg.ntid.z", + "nvvm.read.ptx.sreg.nwarpid", + "nvvm.read.ptx.sreg.smid", + "nvvm.read.ptx.sreg.tid.x", + "nvvm.read.ptx.sreg.tid.y", + "nvvm.read.ptx.sreg.tid.z", + "nvvm.read.ptx.sreg.total.smem.size", + "nvvm.read.ptx.sreg.warpid", + "nvvm.read.ptx.sreg.warpsize", + "nvvm.red", + "nvvm.redux.sync", + "nvvm.rsqrt", + "nvvm.setmaxregister", + "nvvm.shfl.sync", + "nvvm.sin", + "nvvm.spcompress", + "nvvm.spdecompress", + "nvvm.st.bulk", + "nvvm.stmatrix", + "nvvm.store.async", + "nvvm.store.ext", + "nvvm.sub.packed.bf16x2.f32x2.f32x2", + "nvvm.sub.packed.f16x2.f32x2.f32x2", + "nvvm.sub.packed.f32x2", + "nvvm.sub.packed.f32x2.bf16x2.f32x2", + "nvvm.sub.packed.f32x2.f16x2.f32x2", + "nvvm.subf", + "nvvm.tcgen05.alloc", + "nvvm.tcgen05.commit", + "nvvm.tcgen05.cp", + "nvvm.tcgen05.dealloc", + "nvvm.tcgen05.fence", + "nvvm.tcgen05.ld", + "nvvm.tcgen05.ld.red", + "nvvm.tcgen05.ld.red.spcompress", + "nvvm.tcgen05.ld.spcompress", + "nvvm.tcgen05.mma", + "nvvm.tcgen05.mma.block_scale", + "nvvm.tcgen05.mma.block_scale.decompress_b", + "nvvm.tcgen05.mma.decompress_b", + "nvvm.tcgen05.mma.sp", + "nvvm.tcgen05.mma.sp.block_scale", + "nvvm.tcgen05.mma.ws", + "nvvm.tcgen05.mma.ws.sp", + "nvvm.tcgen05.mma_smem_desc", + "nvvm.tcgen05.relinquish_alloc_permit", + "nvvm.tcgen05.shift", + "nvvm.tcgen05.st", + "nvvm.tcgen05.wait", + "nvvm.tcgen05_mma_smem_desc_v2", + "nvvm.tensormap.cp_fenceproxy", + "nvvm.tensormap.replace", + "nvvm.trace.mark", + "nvvm.vote.sync", + "nvvm.wgmma.commit.group.sync.aligned", + "nvvm.wgmma.fence.aligned", + "nvvm.wgmma.mma_async", + "nvvm.wgmma.wait.group.sync.aligned", + "nvvm.wmma.load", + "nvvm.wmma.mma", + "nvvm.wmma.store", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(nvvm); + std.testing.refAllDecls(nvvm.ops); + inline for (comptime std.meta.declarations(nvvm.ops)) |decl| { + const T = @field(nvvm.ops, decl); + if (@TypeOf(T) == type) std.testing.refAllDecls(T); + } +} + +test "every enum attribute prints as the compiler does" { + @setEvalBranchQuota(1_000_000); + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + nvvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + inline for (.{ + .{ nvvm.ops.AtomicOpKindAttr, "#nvvm" }, + .{ nvvm.ops.BarrierReductionAttr, "#nvvm.reduction<{s}>" }, + .{ nvvm.ops.BarrierReduxKindAttr, "#nvvm.barrier_redux_kind<{s}>" }, + .{ nvvm.ops.BlockScaleFormatAttr, "#nvvm.block_scale_format<{s}>" }, + .{ nvvm.ops.CTAGroupKindAttr, "#nvvm.cta_group<{s}>" }, + .{ nvvm.ops.CVTPackFloatKindAttr, "#nvvm.packfloat_type<{s}>" }, + .{ nvvm.ops.CacheEvictionPriorityAttr, "#nvvm" }, + .{ nvvm.ops.ClusterLaunchControlQueryTypeAttr, "#nvvm" }, + .{ nvvm.ops.CompareOpKindAttr, "#nvvm.op<{s}>" }, + .{ nvvm.ops.ConvertFP4TypeAttr, "#nvvm.convert_fp4_type<{s}>" }, + .{ nvvm.ops.ConvertFP8TypeAttr, "#nvvm.convert_fp8_type<{s}>" }, + .{ nvvm.ops.ConvertScaleKindAttr, "#nvvm.convert_scale_kind<{s}>" }, + .{ nvvm.ops.DotAccumulateTypeAttr, "#nvvm.dot_accumulate_type<{s}>" }, + .{ nvvm.ops.EvictKindAttr, "#nvvm.evict_kind<{s}>" }, + .{ nvvm.ops.FPRoundingModeAttr, "#nvvm.fp_rnd_mode<{s}>" }, + .{ nvvm.ops.GridDepActionKindAttr, "#nvvm" }, + .{ nvvm.ops.IntegerRoundingModeAttr, "#nvvm.int_rnd_mode<{s}>" }, + .{ nvvm.ops.L2PrefetchSizeAttr, "#nvvm.l2_prefetch<{s}>" }, + .{ nvvm.ops.LdStMatrixEltTypeAttr, "#nvvm.ld_st_matrix_elt_type<{s}>" }, + .{ nvvm.ops.LoadCacheModifierExtKindAttr, "#nvvm.load_cache_modifier_ext<{s}>" }, + .{ nvvm.ops.LoadCacheModifierKindAttr, "#nvvm" }, + .{ nvvm.ops.LoadShapeAttr, "#nvvm.load_shape<{s}>" }, + .{ nvvm.ops.LoadSrcFormatAttr, "#nvvm.load_src_format<{s}>" }, + .{ nvvm.ops.MBarrierLayoutAttr, "#nvvm.mbarrier_layout<{s}>" }, + .{ nvvm.ops.MBarrierPhaseAttr, "#nvvm.mbarrier_phase<{s}>" }, + .{ nvvm.ops.MBarrierScopeKindAttr, "#nvvm.mbar_scope<{s}>" }, + .{ nvvm.ops.MBarrierSpaceKindAttr, "#nvvm.mbar_space<{s}>" }, + .{ nvvm.ops.MBarrierTxnKindAttr, "#nvvm.mbar_txn_kind<{s}>" }, + .{ nvvm.ops.MBarrierWaitKindAttr, "#nvvm.mbar_wait<{s}>" }, + .{ nvvm.ops.MMAB1OpAttr, "#nvvm.mma_b1op<{s}>" }, + .{ nvvm.ops.MMABlockScaleKindAttr, "#nvvm.block_scale_kind<{s}>" }, + .{ nvvm.ops.MMACtaCountAttr, "#nvvm.mma_cta_count<{s}>" }, + .{ nvvm.ops.MMAFragAttr, "#nvvm.mma_frag<{s}>" }, + .{ nvvm.ops.MMAIntOverflowAttr, "#nvvm.mma_int_overflow<{s}>" }, + .{ nvvm.ops.MMAKindAttr, "#nvvm.mma_kind<{s}>" }, + .{ nvvm.ops.MMALayoutAttr, "#nvvm.mma_layout<{s}>" }, + .{ nvvm.ops.MMATypesAttr, "#nvvm.mma_type<{s}>" }, + .{ nvvm.ops.MatchSyncKindAttr, "#nvvm" }, + .{ nvvm.ops.MemOrderKindAttr, "#nvvm.mem_order<{s}>" }, + .{ nvvm.ops.MemScopeKindAttr, "#nvvm.mem_scope<{s}>" }, + .{ nvvm.ops.MulModeAttr, "#nvvm" }, + .{ nvvm.ops.NVVMMemorySpaceAttr, "#nvvm.memory_space<{s}>" }, + .{ nvvm.ops.PermuteModeAttr, "#nvvm.permute_mode<{s}>" }, + .{ nvvm.ops.PrefetchCacheLevelAttr, "#nvvm" }, + .{ nvvm.ops.ProxyKindAttr, "#nvvm.proxy_kind<{s}>" }, + .{ nvvm.ops.ReductionKindAttr, "#nvvm" }, + .{ nvvm.ops.ReductionOpAttr, "#nvvm.red_op<{s}>" }, + .{ nvvm.ops.ReductionTypeAttr, "#nvvm.red_type<{s}>" }, + .{ nvvm.ops.SPCompressElemSizeAttr, "#nvvm.spcomp_elem_size<{s}>" }, + .{ nvvm.ops.SPCompressFactorTypeAttr, "#nvvm.spcomp_factor<{s}>" }, + .{ nvvm.ops.SPCompressIndexSizeAttr, "#nvvm.spcomp_index_size<{s}>" }, + .{ nvvm.ops.SPCompressOpKindAttr, "#nvvm.spcomp_op_kind<{s}>" }, + .{ nvvm.ops.SPCompressRepFactorAttr, "#nvvm.spcomp_rep_factor<{s}>" }, + .{ nvvm.ops.SPDecompressElemSizeAttr, "#nvvm.spdecomp_elem_size<{s}>" }, + .{ nvvm.ops.SPDecompressFactorTypeAttr, "#nvvm.spdecomp_factor<{s}>" }, + .{ nvvm.ops.SPDecompressIndexSizeAttr, "#nvvm.spdecomp_index_size<{s}>" }, + .{ nvvm.ops.SPDecompressRepFactorAttr, "#nvvm.spdecomp_rep_factor<{s}>" }, + .{ nvvm.ops.SaturationModeAttr, "#nvvm.sat_mode<{s}>" }, + .{ nvvm.ops.SaturationModeKindAttr, "#nvvm.sat<{s}>" }, + .{ nvvm.ops.ScaleVecSizeAttr, "#nvvm.scale_vec_size<{s}>" }, + .{ nvvm.ops.SetMaxRegisterActionAttr, "#nvvm" }, + .{ nvvm.ops.SharedSpaceAttr, "#nvvm.shared_space<{s}>" }, + .{ nvvm.ops.ShflKindAttr, "#nvvm" }, + .{ nvvm.ops.SparsityFormatAttr, "#nvvm.sparsity_format<{s}>" }, + .{ nvvm.ops.StateSpaceAttr, "#nvvm.state_space<{s}>" }, + .{ nvvm.ops.StoreCacheModifierKindAttr, "#nvvm.store_cache_modifier<{s}>" }, + .{ nvvm.ops.StoreShapeAttr, "#nvvm.store_shape<{s}>" }, + .{ nvvm.ops.TCBarParamAttr, "#nvvm.TCBarParam<{s}>" }, + .{ nvvm.ops.TMALoadModeAttr, "#nvvm.tma_load_mode<{s}>" }, + .{ nvvm.ops.TMAReduxKindAttr, "#nvvm.tma_redux_kind<{s}>" }, + .{ nvvm.ops.TMAStoreModeAttr, "#nvvm.tma_store_mode<{s}>" }, + .{ nvvm.ops.Tcgen05CpMulticastAttr, "#nvvm.tcgen05_cp_multicast<{s}>" }, + .{ nvvm.ops.Tcgen05CpShapeAttr, "#nvvm.tcgen05_cp_shape<{s}>" }, + .{ nvvm.ops.Tcgen05CpSrcFormatAttr, "#nvvm.tcgen05_cp_src_fmt<{s}>" }, + .{ nvvm.ops.Tcgen05FenceKindAttr, "#nvvm.tcgen05_fence<{s}>" }, + .{ nvvm.ops.Tcgen05LdStShapeAttr, "#nvvm.tcgen05_ldst_shape<{s}>" }, + .{ nvvm.ops.Tcgen05MMABlockScaleAttr, "#nvvm.tcgen05_mma_block_scale<{s}>" }, + .{ nvvm.ops.Tcgen05MMACollectorBBufferAttr, "#nvvm.tcgen05_mma_collectorb<{s}>" }, + .{ nvvm.ops.Tcgen05MMACollectorOpAttr, "#nvvm.tcgen05_mma_collectorop<{s}>" }, + .{ nvvm.ops.Tcgen05MMAKindAttr, "#nvvm.tcgen05_mma_kind<{s}>" }, + .{ nvvm.ops.Tcgen05WaitKindAttr, "#nvvm.tcgen05_wait<{s}>" }, + .{ nvvm.ops.TensormapElemtypeAttr, "#nvvm.tensormap_elemtype<{s}>" }, + .{ nvvm.ops.TensormapFieldAttr, "#nvvm" }, + .{ nvvm.ops.TensormapFillModeAttr, "#nvvm.tensormap_fill_mode<{s}>" }, + .{ nvvm.ops.TensormapInterleaveLayoutAttr, "#nvvm.tensormap_interleave_layout<{s}>" }, + .{ nvvm.ops.TensormapSwizzleAtomicityAttr, "#nvvm.tensormap_swizzle_atomicity<{s}>" }, + .{ nvvm.ops.TensormapSwizzleModeAttr, "#nvvm.tensormap_swizzle_mode<{s}>" }, + .{ nvvm.ops.TmemLayoutAttr, "#nvvm.TmemLayout<{s}>" }, + .{ nvvm.ops.ValidatePatternAttr, "#nvvm.validate_pattern<{s}>" }, + .{ nvvm.ops.VoteSyncKindAttr, "#nvvm" }, + .{ nvvm.ops.WGMMAScaleInAttr, "#nvvm.wgmma_scale_in<{s}>" }, + .{ nvvm.ops.WGMMAScaleOutAttr, "#nvvm.wgmma_scale_out<{s}>" }, + .{ nvvm.ops.WGMMATypesAttr, "#nvvm.wgmma_type<{s}>" }, + }) |example| { + const T = example[0]; + const E = @FieldType(T.InitArgs, "value"); + inline for (@typeInfo(E).@"enum".field_names) |name| { + const attr = try T.get(ctx, .{ .value = @field(E, name) }); + try std.testing.expectEqual(@field(E, name), attr.value()); + var buf: [256]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{attr}); + try std.testing.expectEqualStrings(std.fmt.comptimePrint(example[1], .{name}), w.buffered()); + } + } +} + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + nvvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +test "every bound operation is registered" { + const ctx = try testContext(); + defer ctx.deinit(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "nvvm types and attributes print as the compiler does" { + @setEvalBranchQuota(100_000); + const ctx = try testContext(); + defer ctx.deinit(); + // The compiler's printed forms, which these must print byte for byte. + inline for (.{ + .{ nvvm.ops.MMAShapeAttr, "#nvvm.shape" }, + .{ nvvm.ops.LdStMatrixShapeAttr, "#nvvm.ld_st_matrix_shape" }, + .{ nvvm.ops.TargetAttr, "#nvvm.target" }, + .{ nvvm.ops.TargetAttr, "#nvvm.target" }, + .{ nvvm.ops.TargetAttr, "#nvvm.target" }, + }) |example| { + const T = example[0]; + const is_type = @hasDecl(T, "type_"); + const parsed = if (is_type) try mlir.Type.parse(ctx, example[1]) else try mlir.Attribute.parse(ctx, example[1]); + const value = parsed.isA(T) orelse return error.TestUnexpectedResult; + try std.testing.expect((try rebuildEnums(T, ctx, value)).eql(value)); + var buf: [512]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(example[1], w.buffered()); + } +} + +test "nvvm target rejects what the compiler rejects" { + const ctx = try testContext(); + defer ctx.deinit(); + try std.testing.expectError(error.InvalidMlir, nvvm.ops.TargetAttr.get(ctx, .{ .O = 4 })); + try std.testing.expectError(error.InvalidMlir, nvvm.ops.TargetAttr.get(ctx, .{ .chip = "" })); + const not_strings = try mlir.Attribute.parse(ctx, "[\"a\", 2]"); + try std.testing.expectError(error.InvalidMlir, nvvm.ops.TargetAttr.get(ctx, .{ .link = not_strings })); +} + +/// `rebuild` for parameters that are slices of enums. +fn rebuildEnums(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T { + var args: T.InitArgs = undefined; + var buffers: [4][16]u32 = undefined; + inline for (@typeInfo(T.InitArgs).@"struct".field_names, 0..) |field, i| { + const name = [_]u8{comptime std.ascii.toUpper(field[0])} ++ field[1..]; + const noun = [_]u8{comptime std.ascii.toLower(field[0])} ++ field[1..]; + const accessor = if (@hasDecl(T, noun ++ "_")) noun ++ "_" else noun; + const info = @typeInfo(@FieldType(T.InitArgs, field)); + if (info == .pointer and info.pointer.size == .slice and @typeInfo(info.pointer.child) == .@"enum") { + const E = info.pointer.child; + const count = @field(T, "num" ++ name)(value); + const elements: []E = @ptrCast(buffers[i][0..count]); + for (elements, 0..) |*element, pos| element.* = @field(T, noun[0 .. noun.len - 1])(value, pos); + @field(args, field) = elements; + } else { + @field(args, field) = @field(T, accessor)(value); + } + } + return T.get(ctx, args); +} diff --git a/mlir/dialects/rocdl/BUILD.bazel b/mlir/dialects/rocdl/BUILD.bazel new file mode 100644 index 00000000..ef0d7583 --- /dev/null +++ b/mlir/dialects/rocdl/BUILD.bazel @@ -0,0 +1,46 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:CAPIIR", + "@llvm-project//mlir:CAPIROCDL", + "@llvm-project//mlir:ROCDLDialect", + ], +) + +zig_library( + name = "rocdl", + srcs = [ + "annotations.zig", + "attributes.zig", + "enums.zig", + "ops.zig", + "target.zig", + "types.zig", + ], + import_name = "mlir/dialects/rocdl", + main = "rocdl.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//mlir/dialects/llvm", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":capi", + ":rocdl", + "//mlir", + ], +) diff --git a/mlir/dialects/rocdl/README.md b/mlir/dialects/rocdl/README.md new file mode 100644 index 00000000..500e4657 --- /dev/null +++ b/mlir/dialects/rocdl/README.md @@ -0,0 +1,54 @@ +# ROCDL + +Depend on `//mlir/dialects/rocdl` and import `mlir/dialects/rocdl` for LLVM's +AMDGPU intrinsic dialect. The package links LLVM's `ROCDLDialect` and +`CAPIROCDL`. `fly_rocdl` is a separate dialect and package. + +The pinned LLVM revision supplies 374 operations, 15 attributes, and 12 enums. + +- `ops` provides every operation builder. Names replace dots with underscores, + such as `ops.raw_buffer_load` and `ops.mfma_f32_16x16x4f32`. Arguments are + operands, explicit result types, required attributes, an `opts` struct when + there are optional attributes, and the location, preceded by the MLIR context. + Attribute names match ODS. Optional and default-valued attributes are fields + of `opts` with `null` defaults; pass `.{}` to omit them and let MLIR supply + its defaults. Builders verify the resulting operation. +- Enum types and attribute wrappers are available at the module root and in + `enums`/`attributes`. Attribute constructors return `error.InvalidMlir` for + invalid enum values or flag bits. Bit enums support `combine` and `contains`. +- `TargetAttr` constructs and reads `#rocdl.target`, including optimization + level, triple, chip, features, ABI, flags, and link files. Defaults match LLVM. + Invalid fields and attributes from another context are rejected. +- `annotations` supplies function, module, and memory annotations. Apply + function annotations to `llvm.func`, and `xnack`/`sramecc` to LLVM-compatible + modules. A missing module option preserves the target default; `false` + explicitly disables the option. +- `types` supplies every builtin vector type ROCDL's TableGen records use, + named or inline, and LLVM pointers in ROCDL address spaces. ROCDL defines no independent + MLIR types. +- `registerDialect` registers ROCDL; loading it also loads its LLVM dependency. + The existing `cvtPkF32Fp8` convenience builder remains available. + +```zig +const rocdl = @import("mlir/dialects/rocdl"); + +const policy = try rocdl.Gfx942CachePolicyAttr.get(ctx, .{ .value = .combine(&.{ .sc0, .nt }) }); +const load = rocdl.ops.raw_buffer_load( + ctx, resource, offset, scalar_offset, + mlir.Type.float(ctx, .f32), .{ .aux = policy.attribute() }, location, +); +``` + +The `:capi` target exposes attribute constructors, predicates, and getters +through `capi.h`; dialect registration comes from upstream +`mlir-c/Dialect/ROCDL.h`. Declarations and implementations are grouped by +introspection (a test helper), attributes, and target configuration. Getters require a handle +accepted by the matching predicate. Contexts own all returned attributes and +strings. There are no additional C operation wrappers: Zig builders use MLIR's +standard operation API. + +Tests compare the generated operation list with the loaded dialect's registered +operation count, check every operation name, round-trip every named enum value, +reject invalid enum/target inputs, and verify and reparse representative barrier, +scheduling, buffer, and matrix operations in an LLVM function. The tests require +no AMD GPU or ROCm runtime. diff --git a/mlir/dialects/rocdl/REGENERATING.md b/mlir/dialects/rocdl/REGENERATING.md new file mode 100644 index 00000000..526cb17b --- /dev/null +++ b/mlir/dialects/rocdl/REGENERATING.md @@ -0,0 +1,62 @@ +# Regenerating the ROCDL bindings + +These are instructions for an agent. They describe what the generated ROCDL +bindings must contain after an LLVM bump, where to derive them from, and how to +check the result. Scripts and intermediate files are not committed; only the +generated bindings are. + +## Sources + +Use the LLVM revision Bazel pins, and read TableGen's resolved records rather +than the `.td` text: multiclasses instantiate many ROCDL operations. + +```sh +bazel build @llvm-project//llvm:llvm-tblgen +llvm_project="$(bazel info output_base)/external/+llvm+llvm-project" +bazel-bin/external/+llvm+llvm-project/llvm/llvm-tblgen --dump-json \ + -I "$llvm_project/mlir/include" -I "$llvm_project/llvm/include" \ + "$llvm_project/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td" \ + -o /tmp/rocdl-records.json +``` + +Every `Op`, `AttrDef` and `EnumInfo` record whose dialect is `ROCDL_Dialect` +belongs to the bindings. + +## What each file must contain + +Names, builder parameters, `opts`, wrappers, C API names and comments follow +`mlir/dialects/README.md`. Specific to ROCDL: + +- `ops.zig`: one builder per operation, listed in `test.zig`'s + `operation_names`. Follow argument constraints through `Arg` wrappers. +- `enums.zig`: one enum per ROCDL enum, with its bit width and values. Unlike + the README rule, a case whose IR spelling is not an identifier or is a + keyword takes the C++ symbol (`volatile` → `volatile_op`). Bit enums are + non-exhaustive and provide `combine` and `contains`. +- `attributes.zig`: one `Attr` wrapper per enum attribute, and + `enum_attributes` listing them. +- `types.zig`: `AddressSpace`, `pointerType`, and a `v` constructor + for every vector type the records use, named or inline. +- `capi.h`/`capi.cc`: `mlirROCDLGetOperationCount` under `// Introspection` + and, per enum attribute, the predicate, the getter (null for out-of-range + values) and `GetValue` under `// Attributes`. The `// Target` sections are + maintained by hand: in the + pinned revision `ROCDLTargetAttr::getChecked` delegates to `Base::get`, so the + C wrapper verifies target inputs first. + +Each generated Zig file starts with the "Generated from LLVM ROCDL TableGen +records" header. `target.zig`, `annotations.zig` and `rocdl.zig` are maintained +by hand; update their aliases and helpers when LLVM's surface changes. If the +dialect gains variadic operands, regions, successors, its own types, an enum +without an attribute, or new target fields or discardable attributes, extend +the bindings and this file. + +## Checks + +- Format and run the conformance checks of `mlir/dialects/README.md`. +- `bazel test //mlir/dialects/rocdl:test //dsl/fly:test //zml:test`. Beyond the + README's tests, the operation list is compared to + `mlirROCDLGetOperationCount`, and attribute tests exercise every named enum + case, named bit groups and combined bitfields. Add fixtures when a change + introduces a new operand layout or attribute representation. +- A regeneration without an LLVM change is byte-identical. diff --git a/mlir/dialects/rocdl/annotations.zig b/mlir/dialects/rocdl/annotations.zig new file mode 100644 index 00000000..85128473 --- /dev/null +++ b/mlir/dialects/rocdl/annotations.zig @@ -0,0 +1,56 @@ +const std = @import("std"); + +const mlir = @import("mlir"); + +/// Discardable ROCDL attributes for an `llvm.func` operation. +pub const FunctionOpts = struct { + kernel: bool = false, + reqdWorkGroupSize: ?[3]i32 = null, + flatWorkGroupSize: ?[]const u8 = null, + maxFlatWorkGroupSize: ?i32 = null, + wavesPerEu: ?WavesPerEu = null, + unsafeFpAtomics: ?bool = null, + + pub const WavesPerEu = struct { + min: u32, + max: ?u32 = null, + }; +}; + +pub fn annotateFunction(ctx: *mlir.Context, op: *mlir.Operation, opts: FunctionOpts) void { + if (opts.kernel) op.setAttributeByName("rocdl.kernel", .unit(ctx)); + if (opts.reqdWorkGroupSize) |size| op.setAttributeByName("rocdl.reqd_work_group_size", .denseArray(ctx, .i32, &size)); + if (opts.flatWorkGroupSize) |size| op.setAttributeByName("rocdl.flat_work_group_size", .string(ctx, size)); + if (opts.maxFlatWorkGroupSize) |size| op.setAttributeByName("rocdl.max_flat_work_group_size", .int(ctx, .i32, size)); + if (opts.wavesPerEu) |waves| { + var buffer: [24]u8 = undefined; + const value = if (waves.max) |max| std.fmt.bufPrint(&buffer, "{d},{d}", .{ waves.min, max }) else std.fmt.bufPrint(&buffer, "{d}", .{waves.min}); + op.setAttributeByName("rocdl.waves_per_eu", .string(ctx, value catch unreachable)); + } + if (opts.unsafeFpAtomics) |enabled| op.setAttributeByName("rocdl.unsafe_fp_atomics", .boolean(ctx, enabled)); +} + +/// A missing flag preserves the target's default; false explicitly disables it. +pub const ModuleOpts = struct { + xnack: ?bool = null, + sramecc: ?bool = null, +}; + +pub fn annotateModule(ctx: *mlir.Context, op: *mlir.Operation, opts: ModuleOpts) void { + if (opts.xnack) |enabled| op.setAttributeByName("rocdl.xnack", .boolean(ctx, enabled)); + if (opts.sramecc) |enabled| op.setAttributeByName("rocdl.sramecc", .boolean(ctx, enabled)); +} + +pub const MemoryOpts = struct { + lastUse: bool = false, + noRemoteMemory: bool = false, + noFineGrainedMemory: bool = false, + ignoreDenormalMode: bool = false, +}; + +pub fn annotateMemory(ctx: *mlir.Context, op: *mlir.Operation, opts: MemoryOpts) void { + if (opts.lastUse) op.setAttributeByName("rocdl.last_use", .unit(ctx)); + if (opts.noRemoteMemory) op.setAttributeByName("rocdl.no_remote_memory", .unit(ctx)); + if (opts.noFineGrainedMemory) op.setAttributeByName("rocdl.no_fine_grained_memory", .unit(ctx)); + if (opts.ignoreDenormalMode) op.setAttributeByName("rocdl.ignore_denormal_mode", .unit(ctx)); +} diff --git a/mlir/dialects/rocdl/attributes.zig b/mlir/dialects/rocdl/attributes.zig new file mode 100644 index 00000000..9e1b9863 --- /dev/null +++ b/mlir/dialects/rocdl/attributes.zig @@ -0,0 +1,417 @@ +// Generated from LLVM ROCDL TableGen records; see REGENERATING.md. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +const c = @import("c"); +const mlir = @import("mlir"); +const enums = @import("enums.zig"); + +pub const TargetAttr = @import("target.zig").TargetAttr; + +/// `#rocdl.buffer_oob_mode`. +pub const BufferOOBModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.BufferOOBMode; + pub const isAFn = c.mlirAttributeIsAROCDLBufferOOBMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLBufferOOBModeAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLBufferOOBModeAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.buffer_oob_mode_flag`. +pub const BufferOOBModeModuleFlagAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.BufferOOBMode; + pub const isAFn = c.mlirAttributeIsAROCDLBufferOOBModeModuleFlag; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLBufferOOBModeModuleFlagAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLBufferOOBModeModuleFlagAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.gfx12_atomic_cache_policy`. +pub const Gfx12AtomicCachePolicyAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.Gfx12AtomicCachePolicy; + pub const isAFn = c.mlirAttributeIsAROCDLGfx12AtomicCachePolicy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLGfx12AtomicCachePolicyAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLGfx12AtomicCachePolicyAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.gfx12_cache_policy`. +pub const Gfx12CachePolicyAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.Gfx12CachePolicy; + pub const isAFn = c.mlirAttributeIsAROCDLGfx12CachePolicy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLGfx12CachePolicyAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLGfx12CachePolicyAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.gfx942_cache_policy`. +pub const Gfx942CachePolicyAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.Gfx942CachePolicy; + pub const isAFn = c.mlirAttributeIsAROCDLGfx942CachePolicy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLGfx942CachePolicyAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLGfx942CachePolicyAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.mfma_neg_modifier`. +pub const MFMANegModifierAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.MFMANegModifier; + pub const isAFn = c.mlirAttributeIsAROCDLMFMANegModifier; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLMFMANegModifierAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLMFMANegModifierAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.mfma_perm_b`. +pub const MFMAPermBAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.MFMAPermB; + pub const isAFn = c.mlirAttributeIsAROCDLMFMAPermB; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLMFMAPermBAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLMFMAPermBAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.matrix_format`. +pub const MatrixFormatAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.MatrixFormat; + pub const isAFn = c.mlirAttributeIsAROCDLMatrixFormat; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLMatrixFormatAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLMatrixFormatAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.pre_gfx12_cache_policy`. +pub const PreGfx12CachePolicyAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.PreGfx12CachePolicy; + pub const isAFn = c.mlirAttributeIsAROCDLPreGfx12CachePolicy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLPreGfx12CachePolicyAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLPreGfx12CachePolicyAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.sched_group_mask`. +pub const SchedGroupMaskAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.SchedGroupMask; + pub const isAFn = c.mlirAttributeIsAROCDLSchedGroupMask; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLSchedGroupMaskAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLSchedGroupMaskAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.tbuffer_oob_mode_flag`. +pub const TBufferOOBModeModuleFlagAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.BufferOOBMode; + pub const isAFn = c.mlirAttributeIsAROCDLTBufferOOBModeModuleFlag; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLTBufferOOBModeModuleFlagAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLTBufferOOBModeModuleFlagAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.wmma_c_modifier`. +pub const WMMACModifierAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.WMMACModifier; + pub const isAFn = c.mlirAttributeIsAROCDLWMMACModifier; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLWMMACModifierAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLWMMACModifierAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.wmma_matrix_scale`. +pub const WMMAMatrixScaleAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.WMMAMatrixScale; + pub const isAFn = c.mlirAttributeIsAROCDLWMMAMatrixScale; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLWMMAMatrixScaleAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLWMMAMatrixScaleAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.wmma_matrix_scale_format`. +pub const WMMAMatrixScaleFormatAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.WMMAMatrixScaleFormat; + pub const isAFn = c.mlirAttributeIsAROCDLWMMAMatrixScaleFormat; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLWMMAMatrixScaleFormatAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLWMMAMatrixScaleFormatAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub const enum_attributes = .{ + BufferOOBModeAttr, + BufferOOBModeModuleFlagAttr, + Gfx12AtomicCachePolicyAttr, + Gfx12CachePolicyAttr, + Gfx942CachePolicyAttr, + MFMANegModifierAttr, + MFMAPermBAttr, + MatrixFormatAttr, + PreGfx12CachePolicyAttr, + SchedGroupMaskAttr, + TBufferOOBModeModuleFlagAttr, + WMMACModifierAttr, + WMMAMatrixScaleAttr, + WMMAMatrixScaleFormatAttr, +}; diff --git a/mlir/dialects/rocdl/capi.cc b/mlir/dialects/rocdl/capi.cc new file mode 100644 index 00000000..7a01bd56 --- /dev/null +++ b/mlir/dialects/rocdl/capi.cc @@ -0,0 +1,440 @@ +//===- capi.cc - C API for the rocdl dialect ------------------------------===// + +#include "capi.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Support.h" +#include "mlir/Dialect/LLVMIR/ROCDLDialect.h" +#include "mlir/IR/Diagnostics.h" + +// Introspection + +intptr_t mlirROCDLGetOperationCount(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return 0; + ctx->getOrLoadDialect(); + intptr_t count = 0; + for (auto op : ctx->getRegisteredOperations()) + if (op.getDialectNamespace() == "rocdl") + ++count; + return count; +} + +// Attributes + +bool mlirAttributeIsAROCDLBufferOOBMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLBufferOOBModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeBufferOOBMode(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::BufferOOBModeAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLBufferOOBModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLBufferOOBModeModuleFlag(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} + +MlirAttribute mlirROCDLBufferOOBModeModuleFlagAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeBufferOOBMode(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::BufferOOBModeModuleFlagAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLBufferOOBModeModuleFlagAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsAROCDLGfx12AtomicCachePolicy(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} + +MlirAttribute mlirROCDLGfx12AtomicCachePolicyAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = mlir::ROCDL::symbolizeGfx12AtomicCachePolicy( + static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::Gfx12AtomicCachePolicyAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLGfx12AtomicCachePolicyAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsAROCDLGfx12CachePolicy(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLGfx12CachePolicyAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeGfx12CachePolicy(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::Gfx12CachePolicyAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLGfx12CachePolicyAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLGfx942CachePolicy(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} + +MlirAttribute mlirROCDLGfx942CachePolicyAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeGfx942CachePolicy(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::Gfx942CachePolicyAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLGfx942CachePolicyAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLMFMANegModifier(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLMFMANegModifierAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeMFMANegModifier(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::MFMANegModifierAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLMFMANegModifierAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLMFMAPermB(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLMFMAPermBAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = mlir::ROCDL::symbolizeMFMAPermB(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::MFMAPermBAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLMFMAPermBAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLMatrixFormat(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLMatrixFormatAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeMatrixFormat(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::MatrixFormatAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLMatrixFormatAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLPreGfx12CachePolicy(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} + +MlirAttribute mlirROCDLPreGfx12CachePolicyAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizePreGfx12CachePolicy(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::PreGfx12CachePolicyAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLPreGfx12CachePolicyAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsAROCDLSchedGroupMask(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLSchedGroupMaskAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeSchedGroupMask(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::SchedGroupMaskAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLSchedGroupMaskAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLTBufferOOBModeModuleFlag(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} + +MlirAttribute mlirROCDLTBufferOOBModeModuleFlagAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeBufferOOBMode(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::TBufferOOBModeModuleFlagAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLTBufferOOBModeModuleFlagAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsAROCDLWMMACModifier(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLWMMACModifierAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + if (value > 65535) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeWMMACModifier(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::WMMACModifierAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLWMMACModifierAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLWMMAMatrixScale(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLWMMAMatrixScaleAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeWMMAMatrixScale(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::WMMAMatrixScaleAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLWMMAMatrixScaleAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLWMMAMatrixScaleFormat(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} + +MlirAttribute mlirROCDLWMMAMatrixScaleFormatAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeWMMAMatrixScaleFormat(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::WMMAMatrixScaleFormatAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLWMMAMatrixScaleFormatAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +// Target + +bool mlirAttributeIsAROCDLTarget(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLTargetAttrGet(MlirContext context, int optLevel, + MlirStringRef triple, MlirStringRef chip, + MlirStringRef features, MlirStringRef abi, + MlirAttribute flags, MlirAttribute link) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + for (auto s : {triple, chip, features, abi}) + if (!s.data && s.length) + return {nullptr}; + auto flagsValue = + llvm::dyn_cast_if_present(unwrap(flags)); + auto linkValue = llvm::dyn_cast_if_present(unwrap(link)); + if (unwrap(flags) && (!flagsValue || flagsValue.getContext() != ctx)) + return {nullptr}; + if (unwrap(link) && (!linkValue || linkValue.getContext() != ctx)) + return {nullptr}; + ctx->getOrLoadDialect(); + // The upstream custom getChecked builder delegates to unchecked Base::get. + // Verify first so invalid C API inputs return null instead of asserting. + if (mlir::failed(mlir::ROCDL::ROCDLTargetAttr::verifyInvariants( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, optLevel, + unwrap(triple), unwrap(chip), unwrap(features), unwrap(abi), + flagsValue, linkValue))) + return {nullptr}; + return wrap(mlir::ROCDL::ROCDLTargetAttr::get( + ctx, optLevel, unwrap(triple), unwrap(chip), unwrap(features), + unwrap(abi), flagsValue, linkValue)); +} + +int mlirROCDLTargetAttrGetOptLevel(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getO(); +} + +MlirStringRef mlirROCDLTargetAttrGetTriple(MlirAttribute attr) { + return wrap( + llvm::cast(unwrap(attr)).getTriple()); +} + +MlirStringRef mlirROCDLTargetAttrGetChip(MlirAttribute attr) { + return wrap(llvm::cast(unwrap(attr)).getChip()); +} + +MlirStringRef mlirROCDLTargetAttrGetFeatures(MlirAttribute attr) { + return wrap( + llvm::cast(unwrap(attr)).getFeatures()); +} + +MlirStringRef mlirROCDLTargetAttrGetAbi(MlirAttribute attr) { + return wrap(llvm::cast(unwrap(attr)).getAbi()); +} + +MlirAttribute mlirROCDLTargetAttrGetFlags(MlirAttribute attr) { + return wrap( + llvm::cast(unwrap(attr)).getFlags()); +} + +MlirAttribute mlirROCDLTargetAttrGetLink(MlirAttribute attr) { + return wrap(llvm::cast(unwrap(attr)).getLink()); +} + +bool mlirROCDLTargetAttrHasFlag(MlirAttribute attr, MlirStringRef flag) { + return llvm::cast(unwrap(attr)) + .hasFlag(unwrap(flag)); +} + +bool mlirROCDLTargetAttrHasWave64(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).hasWave64(); +} + +bool mlirROCDLTargetAttrHasFastMath(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).hasFastMath(); +} + +bool mlirROCDLTargetAttrHasDaz(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).hasDaz(); +} + +bool mlirROCDLTargetAttrHasFiniteOnly(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).hasFiniteOnly(); +} + +bool mlirROCDLTargetAttrHasUnsafeMath(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).hasUnsafeMath(); +} + +bool mlirROCDLTargetAttrHasCorrectSqrt(MlirAttribute attr) { + return llvm::cast(unwrap(attr)) + .hasCorrectSqrt(); +} diff --git a/mlir/dialects/rocdl/capi.h b/mlir/dialects/rocdl/capi.h new file mode 100644 index 00000000..b73a948d --- /dev/null +++ b/mlir/dialects/rocdl/capi.h @@ -0,0 +1,140 @@ +//===- capi.h - C API for the rocdl dialect -----------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_ROCDL_CAPI_H +#define MLIR_DIALECTS_ROCDL_CAPI_H + +#include "mlir-c/Dialect/ROCDL.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Introspection + +MLIR_CAPI_EXPORTED intptr_t mlirROCDLGetOperationCount(MlirContext context); + +// Attributes + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAROCDLBufferOOBMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLBufferOOBModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLBufferOOBModeAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLBufferOOBModeModuleFlag(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLBufferOOBModeModuleFlagAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLBufferOOBModeModuleFlagAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLGfx12AtomicCachePolicy(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLGfx12AtomicCachePolicyAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLGfx12AtomicCachePolicyAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLGfx12CachePolicy(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLGfx12CachePolicyAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLGfx12CachePolicyAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLGfx942CachePolicy(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLGfx942CachePolicyAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLGfx942CachePolicyAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLMFMANegModifier(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLMFMANegModifierAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLMFMANegModifierAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAROCDLMFMAPermB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirROCDLMFMAPermBAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirROCDLMFMAPermBAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAROCDLMatrixFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLMatrixFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLMatrixFormatAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLPreGfx12CachePolicy(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLPreGfx12CachePolicyAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLPreGfx12CachePolicyAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAROCDLSchedGroupMask(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLSchedGroupMaskAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLSchedGroupMaskAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLTBufferOOBModeModuleFlag(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLTBufferOOBModeModuleFlagAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLTBufferOOBModeModuleFlagAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAROCDLWMMACModifier(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLWMMACModifierAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLWMMACModifierAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLWMMAMatrixScale(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLWMMAMatrixScaleAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLWMMAMatrixScaleAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLWMMAMatrixScaleFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLWMMAMatrixScaleFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLWMMAMatrixScaleFormatAttrGetValue(MlirAttribute attr); + +// Target + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAROCDLTarget(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirROCDLTargetAttrGet( + MlirContext context, int optLevel, MlirStringRef triple, MlirStringRef chip, + MlirStringRef features, MlirStringRef abi, MlirAttribute flags, + MlirAttribute link); +MLIR_CAPI_EXPORTED int mlirROCDLTargetAttrGetOptLevel(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirROCDLTargetAttrGetTriple(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirROCDLTargetAttrGetChip(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirROCDLTargetAttrGetFeatures(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirROCDLTargetAttrGetAbi(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLTargetAttrGetFlags(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirROCDLTargetAttrGetLink(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasFlag(MlirAttribute attr, + MlirStringRef flag); +MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasWave64(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasFastMath(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasDaz(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasFiniteOnly(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasUnsafeMath(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasCorrectSqrt(MlirAttribute attr); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_ROCDL_CAPI_H diff --git a/mlir/dialects/rocdl/enums.zig b/mlir/dialects/rocdl/enums.zig new file mode 100644 index 00000000..fde24ff7 --- /dev/null +++ b/mlir/dialects/rocdl/enums.zig @@ -0,0 +1,208 @@ +// Generated from LLVM ROCDL TableGen records; see REGENERATING.md. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +/// ROCDL buffer out-of-bounds mode +pub const BufferOOBMode = enum(u32) { + any = 0, + relaxed = 1, + strict = 2, +}; + +/// gfx12 atomic cache policy bits +pub const Gfx12AtomicCachePolicy = enum(u32) { + none = 0, + keep_return = 1, + nt = 2, + cascade = 4, + scope_se = 8, + scope_dev = 16, + scope_sys = 24, + nv = 32, + volatile_op = 2147483648, + _, + + pub fn combine(values: []const Gfx12AtomicCachePolicy) Gfx12AtomicCachePolicy { + var bits: u32 = 0; + for (values) |value| bits |= @backingInt(value); + return @fromBackingInt(bits); + } + + pub fn contains(self: Gfx12AtomicCachePolicy, value: Gfx12AtomicCachePolicy) bool { + return @backingInt(self) & @backingInt(value) == @backingInt(value); + } +}; + +/// gfx12 cache policy bits +pub const Gfx12CachePolicy = enum(u32) { + none = 0, + nt = 1, + ht = 2, + lu = 3, + nt_rt = 4, + rt_nt = 5, + nt_ht = 6, + nt_wb = 7, + scope_se = 8, + scope_dev = 16, + scope_sys = 24, + nv = 32, + swz = 64, + scal = 2048, + volatile_op = 2147483648, + _, + + pub fn combine(values: []const Gfx12CachePolicy) Gfx12CachePolicy { + var bits: u32 = 0; + for (values) |value| bits |= @backingInt(value); + return @fromBackingInt(bits); + } + + pub fn contains(self: Gfx12CachePolicy, value: Gfx12CachePolicy) bool { + return @backingInt(self) & @backingInt(value) == @backingInt(value); + } +}; + +/// gfx942 buffer cache policy bits +pub const Gfx942CachePolicy = enum(u32) { + none = 0, + sc0 = 1, + nt = 2, + swz = 8, + sc1 = 16, + volatile_op = 2147483648, + _, + + pub fn combine(values: []const Gfx942CachePolicy) Gfx942CachePolicy { + var bits: u32 = 0; + for (values) |value| bits |= @backingInt(value); + return @fromBackingInt(bits); + } + + pub fn contains(self: Gfx942CachePolicy, value: Gfx942CachePolicy) bool { + return @backingInt(self) & @backingInt(value) == @backingInt(value); + } +}; + +/// negation modifier bitfield for gfx94x double-precision MFMA +pub const MFMANegModifier = enum(u32) { + none = 0, + neg_a = 1, + neg_b = 2, + neg_c = 4, + _, + + pub fn combine(values: []const MFMANegModifier) MFMANegModifier { + var bits: u32 = 0; + for (values) |value| bits |= @backingInt(value); + return @fromBackingInt(bits); + } + + pub fn contains(self: MFMANegModifier, value: MFMANegModifier) bool { + return @backingInt(self) & @backingInt(value) == @backingInt(value); + } +}; + +/// permutations of the lanes storing B in an MFMA +pub const MFMAPermB = enum(u32) { + none = 0, + bcast_first_32 = 1, + bcast_second_32 = 2, + rotate_16_right = 3, + bcast_first_16 = 4, + bcast_second_16 = 5, + bcast_third_16 = 6, + bcast_fourth_16 = 7, +}; + +/// matrix operand formats selected by scaled MFMA/WMMA format fields +pub const MatrixFormat = enum(u32) { + fp8_e4m3 = 0, + fp8_e5m2 = 1, + fp6_e2m3 = 2, + fp6_e3m2 = 3, + fp4_e2m1 = 4, +}; + +/// pre-gfx12 buffer cache policy bits +pub const PreGfx12CachePolicy = enum(u32) { + none = 0, + glc = 1, + slc = 2, + dlc = 4, + swz = 8, + scc = 16, + all = 23, + volatile_op = 2147483648, + _, + + pub fn combine(values: []const PreGfx12CachePolicy) PreGfx12CachePolicy { + var bits: u32 = 0; + for (values) |value| bits |= @backingInt(value); + return @fromBackingInt(bits); + } + + pub fn contains(self: PreGfx12CachePolicy, value: PreGfx12CachePolicy) bool { + return @backingInt(self) & @backingInt(value) == @backingInt(value); + } +}; + +/// instruction type mask for scheduling barriers +pub const SchedGroupMask = enum(u32) { + none = 0, + non_mem_non_sideeffect = 1, + valu = 2, + salu = 4, + mfma_wmma = 8, + all_vmem = 16, + vmem_read = 32, + vmem_write = 64, + all_ds = 128, + ds_read = 256, + ds_write = 512, + transcendental = 1024, + ldsdma = 2048, + all = 4095, + _, + + pub fn combine(values: []const SchedGroupMask) SchedGroupMask { + var bits: u32 = 0; + for (values) |value| bits |= @backingInt(value); + return @fromBackingInt(bits); + } + + pub fn contains(self: SchedGroupMask, value: SchedGroupMask) bool { + return @backingInt(self) & @backingInt(value) == @backingInt(value); + } +}; + +/// WMMA C operand modifiers +pub const WMMACModifier = enum(u16) { + none = 0, + neg = 1, + abs = 2, + neg_abs = 3, + _, + + pub fn combine(values: []const WMMACModifier) WMMACModifier { + var bits: u16 = 0; + for (values) |value| bits |= @backingInt(value); + return @fromBackingInt(bits); + } + + pub fn contains(self: WMMACModifier, value: WMMACModifier) bool { + return @backingInt(self) & @backingInt(value) == @backingInt(value); + } +}; + +/// matrix scale row selector +pub const WMMAMatrixScale = enum(u32) { + row0 = 0, + row1 = 1, +}; + +/// matrix scale exponent formats +pub const WMMAMatrixScaleFormat = enum(u32) { + e8 = 0, + e5m3 = 1, + e4m3 = 2, +}; diff --git a/mlir/dialects/rocdl/ops.zig b/mlir/dialects/rocdl/ops.zig new file mode 100644 index 00000000..b18a33cf --- /dev/null +++ b/mlir/dialects/rocdl/ops.zig @@ -0,0 +1,8600 @@ +// Generated from LLVM ROCDL TableGen records; see REGENERATING.md. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +/// `rocdl.asyncmark`. Mark the end of a group of asynchronous operations +pub fn asyncmark( + ctx: *mlir.Context, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.asyncmark", .{ + .location = location, + }); +} + +/// `rocdl.ballot`. Vote across thread group +pub fn ballot( + ctx: *mlir.Context, + pred: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.ballot", .{ + .operands = .{ .flat = &.{pred} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.barrier`. +pub fn barrier( + ctx: *mlir.Context, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.barrier", .{ + .location = location, + }); +} + +/// `rocdl.cluster.id.x`. +pub fn cluster_id_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.id.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.id.y`. +pub fn cluster_id_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.id.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.id.z`. +pub fn cluster_id_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.id.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.load.async.to.lds.b128`. +pub fn cluster_load_async_to_lds_b128( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + mask: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + cpol: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.cpol) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "cpol", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.load.async.to.lds.b128", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr, mask } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.load.async.to.lds.b32`. +pub fn cluster_load_async_to_lds_b32( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + mask: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + cpol: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.cpol) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "cpol", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.load.async.to.lds.b32", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr, mask } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.load.async.to.lds.b64`. +pub fn cluster_load_async_to_lds_b64( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + mask: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + cpol: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.cpol) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "cpol", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.load.async.to.lds.b64", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr, mask } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.load.async.to.lds.b8`. +pub fn cluster_load_async_to_lds_b8( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + mask: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + cpol: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.cpol) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "cpol", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.load.async.to.lds.b8", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr, mask } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.workgroup.id.x`. +pub fn cluster_workgroup_id_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.workgroup.id.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.workgroup.id.y`. +pub fn cluster_workgroup_id_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.workgroup.id.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.workgroup.id.z`. +pub fn cluster_workgroup_id_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.workgroup.id.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cos`. +pub fn cos( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cos", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.f32.bf8`. Convert bf8 to f32 +pub fn cvt_f32_bf8( + ctx: *mlir.Context, + srcA: *const mlir.Value, + res_type: *const mlir.Type, + byteSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "byteSel", byteSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.f32.bf8", .{ + .operands = .{ .flat = &.{srcA} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.f32.fp8`. Convert fp8 to f32 +pub fn cvt_f32_fp8( + ctx: *mlir.Context, + srcA: *const mlir.Value, + res_type: *const mlir.Type, + byteSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "byteSel", byteSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.f32.fp8", .{ + .operands = .{ .flat = &.{srcA} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.pk.bf8.f32`. Convert two f32's to bf8 +pub fn cvt_pk_bf8_f32( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + old: *const mlir.Value, + res_type: *const mlir.Type, + wordSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "wordSel", wordSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.pk.bf8.f32", .{ + .operands = .{ .flat = &.{ srcA, srcB, old } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.pk.f32.bf8`. Convert packed bf8 to packed f32 +pub fn cvt_pk_f32_bf8( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + wordSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "wordSel", wordSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.pk.f32.bf8", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.pk.f32.fp8`. Convert packed fp8 to packed f32 +pub fn cvt_pk_f32_fp8( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + wordSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "wordSel", wordSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.pk.f32.fp8", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.pk.fp8.f32`. Convert two f32's to fp8 +pub fn cvt_pk_fp8_f32( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + old: *const mlir.Value, + res_type: *const mlir.Type, + wordSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "wordSel", wordSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.pk.fp8.f32", .{ + .operands = .{ .flat = &.{ srcA, srcB, old } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.pkrtz`. Convert two f32 input into a vector<2xf16> +pub fn cvt_pkrtz( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.pkrtz", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk16.bf16.bf6`. Scales 16 bf6 and converts them to 16 bf16. +pub fn cvt_scale_pk16_bf16_bf6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk16.bf16.bf6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk16.bf16.fp6`. Scales 16 fp6 and converts them to 16 bf16. +pub fn cvt_scale_pk16_bf16_fp6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk16.bf16.fp6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk16.f16.bf6`. Scales 16 bf6 and converts them to 16 f16. +pub fn cvt_scale_pk16_f16_bf6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk16.f16.bf6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk16.f16.fp6`. Scales 16 fp6 and converts them to 16 f16. +pub fn cvt_scale_pk16_f16_fp6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk16.f16.fp6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk16.f32.bf6`. Scales 16 bf6 and converts them to 16 f32. +pub fn cvt_scale_pk16_f32_bf6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk16.f32.bf6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk16.f32.fp6`. Scales 16 fp6 and converts them to 16 f32. +pub fn cvt_scale_pk16_f32_fp6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk16.f32.fp6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.bf16.bf8`. Scales 8 bf8 and converts them to 8 bf16. +pub fn cvt_scale_pk8_bf16_bf8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.bf16.bf8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.bf16.fp4`. Scales 8 fp4 and converts them to 8 bf16. +pub fn cvt_scale_pk8_bf16_fp4( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.bf16.fp4", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.bf16.fp8`. Scales 8 fp8 and converts them to 8 bf16. +pub fn cvt_scale_pk8_bf16_fp8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.bf16.fp8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.f16.bf8`. Scales 8 bf8 and converts them to 8 f16. +pub fn cvt_scale_pk8_f16_bf8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.f16.bf8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.f16.fp4`. Scales 8 fp4 and converts them to 8 f16. +pub fn cvt_scale_pk8_f16_fp4( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.f16.fp4", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.f16.fp8`. Scales 8 fp8 and converts them to 8 f16. +pub fn cvt_scale_pk8_f16_fp8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.f16.fp8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.f32.bf8`. Scales 8 bf8 and converts them to 8 f32. +pub fn cvt_scale_pk8_f32_bf8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.f32.bf8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.f32.fp4`. Scales 8 fp4 and converts them to 8 f32. +pub fn cvt_scale_pk8_f32_fp4( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.f32.fp4", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.f32.fp8`. Scales 8 fp8 and converts them to 8 f32. +pub fn cvt_scale_pk8_f32_fp8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.f32.fp8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.2xpk16.bf6.f32`. Scale and convert two vector<16xf32> to 32 packed bf6 +pub fn cvt_scalef32_2xpk16_bf6_f32( + ctx: *mlir.Context, + src0: *const mlir.Value, + src1: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.2xpk16.bf6.f32", .{ + .operands = .{ .flat = &.{ src0, src1, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.2xpk16.fp6.f32`. Scale and convert two vector<16xf32> to 32 packed fp6 +pub fn cvt_scalef32_2xpk16_fp6_f32( + ctx: *mlir.Context, + src0: *const mlir.Value, + src1: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.2xpk16.fp6.f32", .{ + .operands = .{ .flat = &.{ src0, src1, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.f16.bf8`. Scaled convert bf8 from packed vector to f16, updating tied result +pub fn cvt_scalef32_f16_bf8( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcSelIndex: *const mlir.Attribute, + dstLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); + attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.f16.bf8", .{ + .operands = .{ .flat = &.{ oldVdst, src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.f16.fp8`. Scaled convert fp8 from packed vector to f16, updating tied result +pub fn cvt_scalef32_f16_fp8( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcSelIndex: *const mlir.Attribute, + dstLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); + attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.f16.fp8", .{ + .operands = .{ .flat = &.{ oldVdst, src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.f32.bf8`. Scaled convert bf8 from packed vector to f32 +pub fn cvt_scalef32_f32_bf8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.f32.bf8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.f32.fp8`. Scaled convert fp8 from packed vector to f32 +pub fn cvt_scalef32_f32_fp8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.f32.fp8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.bf16.bf8`. Scaled convert two bf8to two bf16 +pub fn cvt_scalef32_pk_bf16_bf8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcLoHiSel", srcLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.bf16.bf8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.bf16.fp4`. Scale and convert two packed fp4 to packed bf16 +pub fn cvt_scalef32_pk_bf16_fp4( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.bf16.fp4", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.bf16.fp8`. Scaled convert two fp8to two bf16 +pub fn cvt_scalef32_pk_bf16_fp8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcLoHiSel", srcLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.bf16.fp8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.bf8.bf16`. Scaled convert two bf16to two bf8, updating packed vector +pub fn cvt_scalef32_pk_bf8_bf16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.bf8.bf16", .{ + .operands = .{ .flat = &.{ oldVdst, src0, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.bf8.f16`. Scaled convert two f16to two bf8, updating packed vector +pub fn cvt_scalef32_pk_bf8_f16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.bf8.f16", .{ + .operands = .{ .flat = &.{ oldVdst, src0, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.bf8.f32`. Scaled convert two f32 to two bf8, updating packed vector +pub fn cvt_scalef32_pk_bf8_f32( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + src1: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.bf8.f32", .{ + .operands = .{ .flat = &.{ oldVdst, src0, src1, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.f16.bf8`. Scaled convert two bf8to two f16 +pub fn cvt_scalef32_pk_f16_bf8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcLoHiSel", srcLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.f16.bf8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.f16.fp4`. Scale and convert two packed fp4 to packed f16 +pub fn cvt_scalef32_pk_f16_fp4( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.f16.fp4", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.f16.fp8`. Scaled convert two fp8to two f16 +pub fn cvt_scalef32_pk_f16_fp8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcLoHiSel", srcLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.f16.fp8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.f32.bf8`. Scaled convert two bf8to two f32 +pub fn cvt_scalef32_pk_f32_bf8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcLoHiSel", srcLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.f32.bf8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.f32.fp4`. Scale and convert two packed fp4 to packed f32 +pub fn cvt_scalef32_pk_f32_fp4( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.f32.fp4", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.f32.fp8`. Scaled convert two fp8to two f32 +pub fn cvt_scalef32_pk_f32_fp8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcLoHiSel", srcLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.f32.fp8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.fp4.bf16`. Scale and convert two bf16 to packed fp4, updating tied vector +pub fn cvt_scalef32_pk_fp4_bf16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.fp4.bf16", .{ + .operands = .{ .flat = &.{ oldVdst, src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.fp4.f16`. Scale and convert two f16 to packed fp4, updating tied vector +pub fn cvt_scalef32_pk_fp4_f16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.fp4.f16", .{ + .operands = .{ .flat = &.{ oldVdst, src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.fp4.f32`. Scale and convert two f32 values to two packed fp4, updating tied vector +pub fn cvt_scalef32_pk_fp4_f32( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + src1: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.fp4.f32", .{ + .operands = .{ .flat = &.{ oldVdst, src0, src1, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.fp8.bf16`. Scaled convert two bf16to two fp8, updating packed vector +pub fn cvt_scalef32_pk_fp8_bf16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.fp8.bf16", .{ + .operands = .{ .flat = &.{ oldVdst, src0, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.fp8.f16`. Scaled convert two f16to two fp8, updating packed vector +pub fn cvt_scalef32_pk_fp8_f16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.fp8.f16", .{ + .operands = .{ .flat = &.{ oldVdst, src0, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.fp8.f32`. Scaled convert two f32 to two fp8, updating packed vector +pub fn cvt_scalef32_pk_fp8_f32( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + src1: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.fp8.f32", .{ + .operands = .{ .flat = &.{ oldVdst, src0, src1, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk16.bf6.bf16`. Scale and convert packed bf16 to packed bf6 +pub fn cvt_scalef32_pk16_bf6_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk16.bf6.bf16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk16.bf6.f16`. Scale and convert packed f16 to packed bf6 +pub fn cvt_scalef32_pk16_bf6_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk16.bf6.f16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk16.bf6.f32`. Scale and convert packed f32 to packed bf6 +pub fn cvt_scalef32_pk16_bf6_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk16.bf6.f32", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk16.fp6.bf16`. Scale and convert packed bf16 to packed fp6 +pub fn cvt_scalef32_pk16_fp6_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk16.fp6.bf16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk16.fp6.f16`. Scale and convert packed f16 to packed fp6 +pub fn cvt_scalef32_pk16_fp6_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk16.fp6.f16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk16.fp6.f32`. Scale and convert packed f32 to packed fp6 +pub fn cvt_scalef32_pk16_fp6_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk16.fp6.f32", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.bf16.bf6`. Scale and convert packed bf6 to packed bf16 +pub fn cvt_scalef32_pk32_bf16_bf6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.bf16.bf6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.bf16.fp6`. Scale and convert packed fp6 to packed bf16 +pub fn cvt_scalef32_pk32_bf16_fp6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.bf16.fp6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.bf6.bf16`. Scale and convert packed bf16 to packed bf6 +pub fn cvt_scalef32_pk32_bf6_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.bf6.bf16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.bf6.f16`. Scale and convert packed f16 to packed bf6 +pub fn cvt_scalef32_pk32_bf6_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.bf6.f16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.f16.bf6`. Scale and convert packed bf6 to packed f16 +pub fn cvt_scalef32_pk32_f16_bf6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.f16.bf6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.f16.fp6`. Scale and convert packed fp6 to packed f16 +pub fn cvt_scalef32_pk32_f16_fp6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.f16.fp6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.f32.bf6`. Scale and convert packed bf6 to packed f32 +pub fn cvt_scalef32_pk32_f32_bf6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.f32.bf6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.f32.fp6`. Scale and convert packed fp6 to packed f32 +pub fn cvt_scalef32_pk32_f32_fp6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.f32.fp6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.fp6.bf16`. Scale and convert packed bf16 to packed fp6 +pub fn cvt_scalef32_pk32_fp6_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.fp6.bf16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.fp6.f16`. Scale and convert packed f16 to packed fp6 +pub fn cvt_scalef32_pk32_fp6_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.fp6.f16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.bf8.bf16`. Scale and convert packed bf16 to packed bf8 +pub fn cvt_scalef32_pk8_bf8_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.bf8.bf16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.bf8.f16`. Scale and convert packed f16 to packed bf8 +pub fn cvt_scalef32_pk8_bf8_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.bf8.f16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.bf8.f32`. Scale and convert packed f32 to packed bf8 +pub fn cvt_scalef32_pk8_bf8_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.bf8.f32", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.fp4.bf16`. Scale and convert packed bf16 to packed fp4 +pub fn cvt_scalef32_pk8_fp4_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.fp4.bf16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.fp4.f16`. Scale and convert packed f16 to packed fp4 +pub fn cvt_scalef32_pk8_fp4_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.fp4.f16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.fp4.f32`. Scale and convert packed f32 to packed fp4 +pub fn cvt_scalef32_pk8_fp4_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.fp4.f32", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.fp8.bf16`. Scale and convert packed bf16 to packed fp8 +pub fn cvt_scalef32_pk8_fp8_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.fp8.bf16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.fp8.f16`. Scale and convert packed f16 to packed fp8 +pub fn cvt_scalef32_pk8_fp8_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.fp8.f16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.fp8.f32`. Scale and convert packed f32 to packed fp8 +pub fn cvt_scalef32_pk8_fp8_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.fp8.f32", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.bf8.bf16`. Scaled convert bf16to bf8 with stochiastic rounding, updating packed vector +pub fn cvt_scalef32_sr_bf8_bf16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.bf8.bf16", .{ + .operands = .{ .flat = &.{ oldVdst, src0, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.bf8.f16`. Scaled convert f16to bf8 with stochiastic rounding, updating packed vector +pub fn cvt_scalef32_sr_bf8_f16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.bf8.f16", .{ + .operands = .{ .flat = &.{ oldVdst, src0, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.bf8.f32`. Scaled convert f32to bf8 with stochiastic rounding, updating packed vector +pub fn cvt_scalef32_sr_bf8_f32( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.bf8.f32", .{ + .operands = .{ .flat = &.{ oldVdst, src0, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.fp8.bf16`. Scaled convert bf16to fp8 with stochiastic rounding, updating packed vector +pub fn cvt_scalef32_sr_fp8_bf16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.fp8.bf16", .{ + .operands = .{ .flat = &.{ oldVdst, src0, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.fp8.f16`. Scaled convert f16to fp8 with stochiastic rounding, updating packed vector +pub fn cvt_scalef32_sr_fp8_f16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.fp8.f16", .{ + .operands = .{ .flat = &.{ oldVdst, src0, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.fp8.f32`. Scaled convert f32to fp8 with stochiastic rounding, updating packed vector +pub fn cvt_scalef32_sr_fp8_f32( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.fp8.f32", .{ + .operands = .{ .flat = &.{ oldVdst, src0, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk.fp4.bf16`. Scale and convert two bf16 to packed fp4 with stochiastic rounding, updating tied vector +pub fn cvt_scalef32_sr_pk_fp4_bf16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk.fp4.bf16", .{ + .operands = .{ .flat = &.{ oldVdst, src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk.fp4.f16`. Scale and convert two f16 to packed fp4 with stochiastic rounding, updating tied vector +pub fn cvt_scalef32_sr_pk_fp4_f16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk.fp4.f16", .{ + .operands = .{ .flat = &.{ oldVdst, src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk.fp4.f32`. Scale and convert two f32 to packed fp4 with stochiastic rounding, updating tied vector +pub fn cvt_scalef32_sr_pk_fp4_f32( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk.fp4.f32", .{ + .operands = .{ .flat = &.{ oldVdst, src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk16.bf6.bf16`. Scale and convert packed bf16 to packed bf6 with stochastic rounding +pub fn cvt_scalef32_sr_pk16_bf6_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk16.bf6.bf16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk16.bf6.f16`. Scale and convert packed f16 to packed bf6 with stochastic rounding +pub fn cvt_scalef32_sr_pk16_bf6_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk16.bf6.f16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk16.bf6.f32`. Scale and convert packed f32 to packed bf6 with stochastic rounding +pub fn cvt_scalef32_sr_pk16_bf6_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk16.bf6.f32", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk16.fp6.bf16`. Scale and convert packed bf16 to packed fp6 with stochastic rounding +pub fn cvt_scalef32_sr_pk16_fp6_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk16.fp6.bf16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk16.fp6.f16`. Scale and convert packed f16 to packed fp6 with stochastic rounding +pub fn cvt_scalef32_sr_pk16_fp6_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk16.fp6.f16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk16.fp6.f32`. Scale and convert packed f32 to packed fp6 with stochastic rounding +pub fn cvt_scalef32_sr_pk16_fp6_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk16.fp6.f32", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk32.bf6.bf16`. Scale and convert packed bf16 to packed bf6 with stochiastic rounding +pub fn cvt_scalef32_sr_pk32_bf6_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk32.bf6.bf16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk32.bf6.f16`. Scale and convert packed f16 to packed bf6 with stochiastic rounding +pub fn cvt_scalef32_sr_pk32_bf6_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk32.bf6.f16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk32.bf6.f32`. Scale and convert packed f32 to packed bf6 with stochiastic rounding +pub fn cvt_scalef32_sr_pk32_bf6_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk32.bf6.f32", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk32.fp6.bf16`. Scale and convert packed bf16 to packed fp6 with stochiastic rounding +pub fn cvt_scalef32_sr_pk32_fp6_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk32.fp6.bf16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk32.fp6.f16`. Scale and convert packed f16 to packed fp6 with stochiastic rounding +pub fn cvt_scalef32_sr_pk32_fp6_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk32.fp6.f16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk32.fp6.f32`. Scale and convert packed f32 to packed fp6 with stochiastic rounding +pub fn cvt_scalef32_sr_pk32_fp6_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk32.fp6.f32", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.bf8.bf16`. Scale and convert packed bf16 to packed bf8 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_bf8_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.bf8.bf16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.bf8.f16`. Scale and convert packed f16 to packed bf8 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_bf8_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.bf8.f16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.bf8.f32`. Scale and convert packed f32 to packed bf8 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_bf8_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.bf8.f32", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.fp4.bf16`. Scale and convert packed bf16 to packed fp4 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_fp4_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.fp4.bf16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.fp4.f16`. Scale and convert packed f16 to packed fp4 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_fp4_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.fp4.f16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.fp4.f32`. Scale and convert packed f32 to packed fp4 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_fp4_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.fp4.f32", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.fp8.bf16`. Scale and convert packed bf16 to packed fp8 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_fp8_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.fp8.bf16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.fp8.f16`. Scale and convert packed f16 to packed fp8 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_fp8_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.fp8.f16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.fp8.f32`. Scale and convert packed f32 to packed fp8 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_fp8_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.fp8.f32", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.sr.bf8.f32`. Convert f32 to bf8, stochiastic rounding +pub fn cvt_sr_bf8_f32( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + old: *const mlir.Value, + res_type: *const mlir.Type, + byteSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "byteSel", byteSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.sr.bf8.f32", .{ + .operands = .{ .flat = &.{ srcA, srcB, old } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.sr.fp8.f32`. Convert f32 to fp8, stochiastic rounding +pub fn cvt_sr_fp8_f32( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + old: *const mlir.Value, + res_type: *const mlir.Type, + byteSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "byteSel", byteSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.sr.fp8.f32", .{ + .operands = .{ .flat = &.{ srcA, srcB, old } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.dot4.f32.bf8.bf8`. +pub fn dot4_f32_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.dot4.f32.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.dot4.f32.bf8.fp8`. +pub fn dot4_f32_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.dot4.f32.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.dot4.f32.fp8.bf8`. +pub fn dot4_f32_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.dot4.f32.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.dot4.f32.fp8.fp8`. +pub fn dot4_f32_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.dot4.f32.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.ds.atomic.async.barrier.arrive.b64`. +pub fn ds_atomic_async_barrier_arrive_b64( + ctx: *mlir.Context, + barrierPtr: *const mlir.Value, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.atomic.async.barrier.arrive.b64", .{ + .operands = .{ .flat = &.{barrierPtr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.atomic.barrier.arrive.rtn.b64`. +pub fn ds_atomic_barrier_arrive_rtn_b64( + ctx: *mlir.Context, + barrierPtr: *const mlir.Value, + val: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.atomic.barrier.arrive.rtn.b64", .{ + .operands = .{ .flat = &.{ barrierPtr, val } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.load.tr16.b128`. Loads and transposes a matrix from ds memory to registers (available in gfx1250+). +pub fn ds_load_tr16_b128( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.load.tr16.b128", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.load.tr4.b64`. Loads and transposes a matrix from ds memory to registers (available in gfx1250+). +pub fn ds_load_tr4_b64( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.load.tr4.b64", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.load.tr6.b96`. Loads and transposes a matrix from ds memory to registers (available in gfx1250+). +pub fn ds_load_tr6_b96( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.load.tr6.b96", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.load.tr8.b64`. Loads and transposes a matrix from ds memory to registers (available in gfx1250+). +pub fn ds_load_tr8_b64( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.load.tr8.b64", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.read.tr16.b64`. +pub fn ds_read_tr16_b64( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.read.tr16.b64", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.read.tr4.b64`. +pub fn ds_read_tr4_b64( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.read.tr4.b64", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.read.tr6.b96`. +pub fn ds_read_tr6_b96( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.read.tr6.b96", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.read.tr8.b64`. +pub fn ds_read_tr8_b64( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.read.tr8.b64", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds_bpermute`. +pub fn ds_bpermute( + ctx: *mlir.Context, + index: *const mlir.Value, + src: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.ds_bpermute", .{ + .operands = .{ .flat = &.{ index, src } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.ds_swizzle`. +pub fn ds_swizzle( + ctx: *mlir.Context, + src: *const mlir.Value, + offset: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.ds_swizzle", .{ + .operands = .{ .flat = &.{ src, offset } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.exp`. +pub fn exp( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.exp", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.exp2`. +pub fn exp2( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.exp2", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.fdot2`. +pub fn fdot2( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.fdot2", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.fdot2.bf16.bf16`. +pub fn fdot2_bf16_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.fdot2.bf16.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.fdot2.f16.f16`. +pub fn fdot2_f16_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.fdot2.f16.f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.fdot2.f32.bf16`. +pub fn fdot2_f32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.fdot2.f32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.flat.prefetch`. +pub fn flat_prefetch( + ctx: *mlir.Context, + ptr: *const mlir.Value, + opts: struct { + cachePolicy: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.cachePolicy) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "cachePolicy", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.flat.prefetch", .{ + .operands = .{ .flat = &.{ptr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.fmed3`. Median of three float/half values +pub fn fmed3( + ctx: *mlir.Context, + src0: *const mlir.Value, + src1: *const mlir.Value, + src2: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.fmed3", .{ + .operands = .{ .flat = &.{ src0, src1, src2 } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.global.load.async.lds`. Version of rocdl.load.async.to.lds specialized to global pointers +pub fn global_load_async_lds( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + size: *const mlir.Attribute, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "size", size)); + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.async.lds", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.async.to.lds.b128`. +pub fn global_load_async_to_lds_b128( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.async.to.lds.b128", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.async.to.lds.b32`. +pub fn global_load_async_to_lds_b32( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.async.to.lds.b32", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.async.to.lds.b64`. +pub fn global_load_async_to_lds_b64( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.async.to.lds.b64", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.async.to.lds.b8`. +pub fn global_load_async_to_lds_b8( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.async.to.lds.b8", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.lds`. +pub fn global_load_lds( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + size: *const mlir.Attribute, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "size", size)); + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.lds", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.tr.b128`. Loads and transposes a matrix from global memory to registers (available in gfx1250+). +pub fn global_load_tr_b128( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.tr.b128", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.tr.b64`. Loads and transposes a matrix from global memory to registers (available in gfx1250+). +pub fn global_load_tr_b64( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.tr.b64", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.tr4.b64`. Loads and transposes a matrix from global memory to registers (available in gfx1250+). +pub fn global_load_tr4_b64( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.tr4.b64", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.tr6.b96`. Loads and transposes a matrix from global memory to registers (available in gfx1250+). +pub fn global_load_tr6_b96( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.tr6.b96", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.prefetch`. +pub fn global_prefetch( + ctx: *mlir.Context, + ptr: *const mlir.Value, + opts: struct { + cachePolicy: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.cachePolicy) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "cachePolicy", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.prefetch", .{ + .operands = .{ .flat = &.{ptr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.store.async.from.lds.b128`. +pub fn global_store_async_from_lds_b128( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.store.async.from.lds.b128", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.store.async.from.lds.b32`. +pub fn global_store_async_from_lds_b32( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.store.async.from.lds.b32", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.store.async.from.lds.b64`. +pub fn global_store_async_from_lds_b64( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.store.async.from.lds.b64", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.store.async.from.lds.b8`. +pub fn global_store_async_from_lds_b8( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.store.async.from.lds.b8", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.iglp.opt`. +pub fn iglp_opt( + ctx: *mlir.Context, + variant: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "variant", variant)); + return mlir.Operation.make(ctx, "rocdl.iglp.opt", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.load.async.to.lds`. Gathering load to LDS that requires explicit async memory tracking +pub fn load_async_to_lds( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + size: *const mlir.Attribute, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "size", size)); + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.load.async.to.lds", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.load.to.lds`. +pub fn load_to_lds( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + size: *const mlir.Attribute, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "size", size)); + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.load.to.lds", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.log`. +pub fn log( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.log", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.make.buffer.rsrc`. +pub fn make_buffer_rsrc( + ctx: *mlir.Context, + base: *const mlir.Value, + stride: *const mlir.Value, + numRecords: *const mlir.Value, + flags: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.make.buffer.rsrc", .{ + .operands = .{ .flat = &.{ base, stride, numRecords, flags } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.mbcnt.hi`. +pub fn mbcnt_hi( + ctx: *mlir.Context, + in0: *const mlir.Value, + in1: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + arg_attrs: ?*const mlir.Attribute = null, + res_attrs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.arg_attrs) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", attribute)); + } + if (opts.res_attrs) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "res_attrs", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.mbcnt.hi", .{ + .operands = .{ .flat = &.{ in0, in1 } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mbcnt.lo`. +pub fn mbcnt_lo( + ctx: *mlir.Context, + in0: *const mlir.Value, + in1: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + arg_attrs: ?*const mlir.Attribute = null, + res_attrs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.arg_attrs) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", attribute)); + } + if (opts.res_attrs) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "res_attrs", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.mbcnt.lo", .{ + .operands = .{ .flat = &.{ in0, in1 } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x16bf16.1k`. +pub fn mfma_f32_16x16x16bf16_1k( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x16bf16.1k", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x16f16`. +pub fn mfma_f32_16x16x16f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x16f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x1f32`. +pub fn mfma_f32_16x16x1f32( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x1f32", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x2bf16`. +pub fn mfma_f32_16x16x2bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x2bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x32.bf16`. +pub fn mfma_f32_16x16x32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x32.bf8.bf8`. +pub fn mfma_f32_16x16x32_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x32.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x32.bf8.fp8`. +pub fn mfma_f32_16x16x32_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x32.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x32.f16`. +pub fn mfma_f32_16x16x32_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x32.f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x32.fp8.bf8`. +pub fn mfma_f32_16x16x32_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x32.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x32.fp8.fp8`. +pub fn mfma_f32_16x16x32_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x32.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x4bf16.1k`. +pub fn mfma_f32_16x16x4bf16_1k( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x4bf16.1k", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x4f16`. +pub fn mfma_f32_16x16x4f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x4f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x4f32`. +pub fn mfma_f32_16x16x4f32( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x4f32", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x8.xf32`. +pub fn mfma_f32_16x16x8_xf32( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x8.xf32", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x8bf16`. +pub fn mfma_f32_16x16x8bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x8bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x16.bf16`. +pub fn mfma_f32_32x32x16_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x16.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x16.bf8.bf8`. +pub fn mfma_f32_32x32x16_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x16.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x16.bf8.fp8`. +pub fn mfma_f32_32x32x16_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x16.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x16.f16`. +pub fn mfma_f32_32x32x16_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x16.f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x16.fp8.bf8`. +pub fn mfma_f32_32x32x16_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x16.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x16.fp8.fp8`. +pub fn mfma_f32_32x32x16_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x16.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x1f32`. +pub fn mfma_f32_32x32x1f32( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x1f32", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x2bf16`. +pub fn mfma_f32_32x32x2bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x2bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x2f32`. +pub fn mfma_f32_32x32x2f32( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x2f32", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x4.xf32`. +pub fn mfma_f32_32x32x4_xf32( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x4.xf32", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x4bf16`. +pub fn mfma_f32_32x32x4bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x4bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x4bf16.1k`. +pub fn mfma_f32_32x32x4bf16_1k( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x4bf16.1k", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x4f16`. +pub fn mfma_f32_32x32x4f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x4f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x8bf16.1k`. +pub fn mfma_f32_32x32x8bf16_1k( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x8bf16.1k", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x8f16`. +pub fn mfma_f32_32x32x8f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x8f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.4x4x1f32`. +pub fn mfma_f32_4x4x1f32( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.4x4x1f32", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.4x4x2bf16`. +pub fn mfma_f32_4x4x2bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.4x4x2bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.4x4x4bf16.1k`. +pub fn mfma_f32_4x4x4bf16_1k( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.4x4x4bf16.1k", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.4x4x4f16`. +pub fn mfma_f32_4x4x4f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.4x4x4f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f64.16x16x4f64`. +pub fn mfma_f64_16x16x4f64( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f64.16x16x4f64", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f64.4x4x4f64`. +pub fn mfma_f64_4x4x4f64( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f64.4x4x4f64", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.16x16x16i8`. +pub fn mfma_i32_16x16x16i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.16x16x16i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.16x16x32.i8`. +pub fn mfma_i32_16x16x32_i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.16x16x32.i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.16x16x4i8`. +pub fn mfma_i32_16x16x4i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.16x16x4i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.16x16x64.i8`. +pub fn mfma_i32_16x16x64_i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.16x16x64.i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.32x32x16.i8`. +pub fn mfma_i32_32x32x16_i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.32x32x16.i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.32x32x32.i8`. +pub fn mfma_i32_32x32x32_i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.32x32x32.i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.32x32x4i8`. +pub fn mfma_i32_32x32x4i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.32x32x4i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.32x32x8i8`. +pub fn mfma_i32_32x32x8i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.32x32x8i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.4x4x4i8`. +pub fn mfma_i32_4x4x4i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.4x4x4i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.scale.f32.16x16x128.f8f6f4`. +pub fn mfma_scale_f32_16x16x128_f8f6f4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + blgp: *const mlir.Attribute, + opselA: *const mlir.Attribute, + opselB: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + attributes.appendAssumeCapacity(.named(ctx, "opselA", opselA)); + attributes.appendAssumeCapacity(.named(ctx, "opselB", opselB)); + return mlir.Operation.make(ctx, "rocdl.mfma.scale.f32.16x16x128.f8f6f4", .{ + .operands = .{ .flat = &.{ a, b, c, scaleA, scaleB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.scale.f32.32x32x64.f8f6f4`. +pub fn mfma_scale_f32_32x32x64_f8f6f4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + blgp: *const mlir.Attribute, + opselA: *const mlir.Attribute, + opselB: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + attributes.appendAssumeCapacity(.named(ctx, "opselA", opselA)); + attributes.appendAssumeCapacity(.named(ctx, "opselB", opselB)); + return mlir.Operation.make(ctx, "rocdl.mfma.scale.f32.32x32x64.f8f6f4", .{ + .operands = .{ .flat = &.{ a, b, c, scaleA, scaleB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.permlane16.swap`. +pub fn permlane16_swap( + ctx: *mlir.Context, + old: *const mlir.Value, + src: *const mlir.Value, + res_type: *const mlir.Type, + fi: *const mlir.Attribute, + boundControl: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "fi", fi)); + attributes.appendAssumeCapacity(.named(ctx, "boundControl", boundControl)); + return mlir.Operation.make(ctx, "rocdl.permlane16.swap", .{ + .operands = .{ .flat = &.{ old, src } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.permlane16.var`. +pub fn permlane16_var( + ctx: *mlir.Context, + old: *const mlir.Value, + src0: *const mlir.Value, + src1: *const mlir.Value, + res_type: *const mlir.Type, + fi: *const mlir.Attribute, + boundControl: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "fi", fi)); + attributes.appendAssumeCapacity(.named(ctx, "boundControl", boundControl)); + return mlir.Operation.make(ctx, "rocdl.permlane16.var", .{ + .operands = .{ .flat = &.{ old, src0, src1 } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.permlane32.swap`. +pub fn permlane32_swap( + ctx: *mlir.Context, + old: *const mlir.Value, + src: *const mlir.Value, + res_type: *const mlir.Type, + fi: *const mlir.Attribute, + boundControl: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "fi", fi)); + attributes.appendAssumeCapacity(.named(ctx, "boundControl", boundControl)); + return mlir.Operation.make(ctx, "rocdl.permlane32.swap", .{ + .operands = .{ .flat = &.{ old, src } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.permlanex16`. +pub fn permlanex16( + ctx: *mlir.Context, + old: *const mlir.Value, + src0: *const mlir.Value, + src1: *const mlir.Value, + src2: *const mlir.Value, + res_type: *const mlir.Type, + fi: *const mlir.Attribute, + boundControl: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "fi", fi)); + attributes.appendAssumeCapacity(.named(ctx, "boundControl", boundControl)); + return mlir.Operation.make(ctx, "rocdl.permlanex16", .{ + .operands = .{ .flat = &.{ old, src0, src1, src2 } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.permlanex16.var`. +pub fn permlanex16_var( + ctx: *mlir.Context, + old: *const mlir.Value, + src0: *const mlir.Value, + src1: *const mlir.Value, + res_type: *const mlir.Type, + fi: *const mlir.Attribute, + boundControl: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "fi", fi)); + attributes.appendAssumeCapacity(.named(ctx, "boundControl", boundControl)); + return mlir.Operation.make(ctx, "rocdl.permlanex16.var", .{ + .operands = .{ .flat = &.{ old, src0, src1 } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ptr.s.buffer.load`. +pub fn ptr_s_buffer_load( + ctx: *mlir.Context, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ptr.s.buffer.load", .{ + .operands = .{ .flat = &.{ rsrc, offset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.buffer.atomic.cmpswap`. +pub fn raw_buffer_atomic_cmpswap( + ctx: *mlir.Context, + src: *const mlir.Value, + cmp: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.buffer.atomic.cmpswap", .{ + .operands = .{ .flat = &.{ src, cmp, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.buffer.atomic.fadd`. +pub fn raw_buffer_atomic_fadd( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.buffer.atomic.fadd", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.buffer.atomic.fmax`. +pub fn raw_buffer_atomic_fmax( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.buffer.atomic.fmax", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.buffer.atomic.smax`. +pub fn raw_buffer_atomic_smax( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.buffer.atomic.smax", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.buffer.atomic.umin`. +pub fn raw_buffer_atomic_umin( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.buffer.atomic.umin", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.buffer.load`. +pub fn raw_buffer_load( + ctx: *mlir.Context, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.buffer.load", .{ + .operands = .{ .flat = &.{ rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.buffer.store`. +pub fn raw_buffer_store( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + opts: struct { + aux: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.buffer.store", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.atomic.cmpswap`. +pub fn raw_ptr_buffer_atomic_cmpswap( + ctx: *mlir.Context, + src: *const mlir.Value, + cmp: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.atomic.cmpswap", .{ + .operands = .{ .flat = &.{ src, cmp, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.atomic.fadd`. +pub fn raw_ptr_buffer_atomic_fadd( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.atomic.fadd", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.atomic.fmax`. +pub fn raw_ptr_buffer_atomic_fmax( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.atomic.fmax", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.atomic.smax`. +pub fn raw_ptr_buffer_atomic_smax( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.atomic.smax", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.atomic.umin`. +pub fn raw_ptr_buffer_atomic_umin( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.atomic.umin", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.load`. +pub fn raw_ptr_buffer_load( + ctx: *mlir.Context, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.load", .{ + .operands = .{ .flat = &.{ rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.load.async.lds`. Async variant of raw.ptr.buffer.load.lds +pub fn raw_ptr_buffer_load_async_lds( + ctx: *mlir.Context, + rsrc: *const mlir.Value, + ldsPtr: *const mlir.Value, + size: *const mlir.Value, + voffset: *const mlir.Value, + soffset: *const mlir.Value, + offset: *const mlir.Value, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.load.async.lds", .{ + .operands = .{ .flat = &.{ rsrc, ldsPtr, size, voffset, soffset, offset } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.load.lds`. +pub fn raw_ptr_buffer_load_lds( + ctx: *mlir.Context, + rsrc: *const mlir.Value, + ldsPtr: *const mlir.Value, + size: *const mlir.Value, + voffset: *const mlir.Value, + soffset: *const mlir.Value, + offset: *const mlir.Value, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.load.lds", .{ + .operands = .{ .flat = &.{ rsrc, ldsPtr, size, voffset, soffset, offset } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.store`. +pub fn raw_ptr_buffer_store( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.store", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.rcp`. +pub fn rcp( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.rcp", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.readfirstlane`. Get the value in first active lane. +pub fn readfirstlane( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.readfirstlane", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.readlane`. Get the value in the specific lane. +pub fn readlane( + ctx: *mlir.Context, + src0: *const mlir.Value, + src1: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.readlane", .{ + .operands = .{ .flat = &.{ src0, src1 } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.rsq`. +pub fn rsq( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.rsq", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.s.barrier`. +pub fn s_barrier( + ctx: *mlir.Context, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.s.barrier", .{ + .location = location, + }); +} + +/// `rocdl.s.barrier.init`. +pub fn s_barrier_init( + ctx: *mlir.Context, + ptr: *const mlir.Value, + memberCnt: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "memberCnt", memberCnt)); + return mlir.Operation.make(ctx, "rocdl.s.barrier.init", .{ + .operands = .{ .flat = &.{ptr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.barrier.join`. +pub fn s_barrier_join( + ctx: *mlir.Context, + ptr: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.s.barrier.join", .{ + .operands = .{ .flat = &.{ptr} }, + .location = location, + }); +} + +/// `rocdl.s.barrier.leave`. +pub fn s_barrier_leave( + ctx: *mlir.Context, + id: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "id", id)); + return mlir.Operation.make(ctx, "rocdl.s.barrier.leave", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.barrier.signal`. +pub fn s_barrier_signal( + ctx: *mlir.Context, + id: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "id", id)); + return mlir.Operation.make(ctx, "rocdl.s.barrier.signal", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.barrier.signal.isfirst`. +pub fn s_barrier_signal_isfirst( + ctx: *mlir.Context, + res_type: *const mlir.Type, + id: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "id", id)); + return mlir.Operation.make(ctx, "rocdl.s.barrier.signal.isfirst", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.barrier.signal.var`. +pub fn s_barrier_signal_var( + ctx: *mlir.Context, + ptr: *const mlir.Value, + memberCnt: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "memberCnt", memberCnt)); + return mlir.Operation.make(ctx, "rocdl.s.barrier.signal.var", .{ + .operands = .{ .flat = &.{ptr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.barrier.wait`. +pub fn s_barrier_wait( + ctx: *mlir.Context, + id: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "id", id)); + return mlir.Operation.make(ctx, "rocdl.s.barrier.wait", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.get.barrier.state`. +pub fn s_get_barrier_state( + ctx: *mlir.Context, + res_type: *const mlir.Type, + id: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "id", id)); + return mlir.Operation.make(ctx, "rocdl.s.get.barrier.state", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.get.named.barrier.state`. +pub fn s_get_named_barrier_state( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.s.get.named.barrier.state", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.s.nop`. +pub fn s_nop( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.s.nop", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.setprio`. +pub fn s_setprio( + ctx: *mlir.Context, + priority: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "priority", priority)); + return mlir.Operation.make(ctx, "rocdl.s.setprio", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.sleep`. +pub fn s_sleep( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.s.sleep", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.wait.asynccnt`. Wait until ASYNCCNT is less than or equal to `count` +pub fn s_wait_asynccnt( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.s.wait.asynccnt", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.wait.dscnt`. Wait until DSCNT is less than or equal to `count` +pub fn s_wait_dscnt( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.s.wait.dscnt", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.wait.expcnt`. Wait until EXPCNT is less than or equal to `count` +pub fn s_wait_expcnt( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.s.wait.expcnt", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.wait.loadcnt`. Wait until LOADCNT is less than or equal to `count` +pub fn s_wait_loadcnt( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.s.wait.loadcnt", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.wait.storecnt`. Wait until STORECNT is less than or equal to `count` +pub fn s_wait_storecnt( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.s.wait.storecnt", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.wait.tensorcnt`. Wait until TENSORCNT is less than or equal to `count` +pub fn s_wait_tensorcnt( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.s.wait.tensorcnt", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.waitcnt`. +pub fn s_waitcnt( + ctx: *mlir.Context, + bitfield: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "bitfield", bitfield)); + return mlir.Operation.make(ctx, "rocdl.s.waitcnt", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.wakeup.barrier`. +pub fn s_wakeup_barrier( + ctx: *mlir.Context, + ptr: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.s.wakeup.barrier", .{ + .operands = .{ .flat = &.{ptr} }, + .location = location, + }); +} + +/// `rocdl.sched.barrier`. +pub fn sched_barrier( + ctx: *mlir.Context, + mask: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "mask", mask)); + return mlir.Operation.make(ctx, "rocdl.sched.barrier", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.sched.group.barrier`. +pub fn sched_group_barrier( + ctx: *mlir.Context, + mask: *const mlir.Attribute, + size: *const mlir.Attribute, + groupId: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "mask", mask)); + attributes.appendAssumeCapacity(.named(ctx, "size", size)); + attributes.appendAssumeCapacity(.named(ctx, "groupId", groupId)); + return mlir.Operation.make(ctx, "rocdl.sched.group.barrier", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.sdot2`. +pub fn sdot2( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.sdot2", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.sdot4`. +pub fn sdot4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.sdot4", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.sdot8`. +pub fn sdot8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.sdot8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.sin`. +pub fn sin( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.sin", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x128.bf8.bf8`. +pub fn smfmac_f32_16x16x128_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x128.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x128.bf8.fp8`. +pub fn smfmac_f32_16x16x128_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x128.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x128.fp8.bf8`. +pub fn smfmac_f32_16x16x128_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x128.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x128.fp8.fp8`. +pub fn smfmac_f32_16x16x128_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x128.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x32.bf16`. +pub fn smfmac_f32_16x16x32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x32.f16`. +pub fn smfmac_f32_16x16x32_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x32.f16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x64.bf16`. +pub fn smfmac_f32_16x16x64_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x64.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x64.bf8.bf8`. +pub fn smfmac_f32_16x16x64_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x64.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x64.bf8.fp8`. +pub fn smfmac_f32_16x16x64_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x64.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x64.f16`. +pub fn smfmac_f32_16x16x64_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x64.f16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x64.fp8.bf8`. +pub fn smfmac_f32_16x16x64_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x64.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x64.fp8.fp8`. +pub fn smfmac_f32_16x16x64_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x64.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x16.bf16`. +pub fn smfmac_f32_32x32x16_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x16.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x16.f16`. +pub fn smfmac_f32_32x32x16_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x16.f16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x32.bf16`. +pub fn smfmac_f32_32x32x32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x32.bf8.bf8`. +pub fn smfmac_f32_32x32x32_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x32.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x32.bf8.fp8`. +pub fn smfmac_f32_32x32x32_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x32.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x32.f16`. +pub fn smfmac_f32_32x32x32_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x32.f16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x32.fp8.bf8`. +pub fn smfmac_f32_32x32x32_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x32.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x32.fp8.fp8`. +pub fn smfmac_f32_32x32x32_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x32.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x64.bf8.bf8`. +pub fn smfmac_f32_32x32x64_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x64.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x64.bf8.fp8`. +pub fn smfmac_f32_32x32x64_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x64.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x64.fp8.bf8`. +pub fn smfmac_f32_32x32x64_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x64.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x64.fp8.fp8`. +pub fn smfmac_f32_32x32x64_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x64.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.i32.16x16x128.i8`. +pub fn smfmac_i32_16x16x128_i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.i32.16x16x128.i8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.i32.16x16x64.i8`. +pub fn smfmac_i32_16x16x64_i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.i32.16x16x64.i8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.i32.32x32x32.i8`. +pub fn smfmac_i32_32x32x32_i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.i32.32x32x32.i8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.i32.32x32x64.i8`. +pub fn smfmac_i32_32x32x64_i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.i32.32x32x64.i8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.sqrt`. +pub fn sqrt( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.sqrt", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.sudot4`. +pub fn sudot4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.sudot4", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.sudot8`. +pub fn sudot8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.sudot8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.bf16.16x16x32.bf16`. +pub fn swmmac_bf16_16x16x32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.swmmac.bf16.16x16x32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.swmmac.bf16.16x16x64.bf16`. +pub fn swmmac_bf16_16x16x64_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.bf16.16x16x64.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.bf16f32.16x16x64.bf16`. +pub fn swmmac_bf16f32_16x16x64_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.bf16f32.16x16x64.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f16.16x16x128.bf8.bf8`. +pub fn swmmac_f16_16x16x128_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f16.16x16x128.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f16.16x16x128.bf8.fp8`. +pub fn swmmac_f16_16x16x128_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f16.16x16x128.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f16.16x16x128.fp8.bf8`. +pub fn swmmac_f16_16x16x128_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f16.16x16x128.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f16.16x16x128.fp8.fp8`. +pub fn swmmac_f16_16x16x128_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f16.16x16x128.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f16.16x16x32.f16`. +pub fn swmmac_f16_16x16x32_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.swmmac.f16.16x16x32.f16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.swmmac.f16.16x16x64.f16`. +pub fn swmmac_f16_16x16x64_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f16.16x16x64.f16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x128.bf8.bf8`. +pub fn swmmac_f32_16x16x128_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x128.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x128.bf8.fp8`. +pub fn swmmac_f32_16x16x128_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x128.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x128.fp8.bf8`. +pub fn swmmac_f32_16x16x128_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x128.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x128.fp8.fp8`. +pub fn swmmac_f32_16x16x128_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x128.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x32.bf16`. +pub fn swmmac_f32_16x16x32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x32.bf8.bf8`. +pub fn swmmac_f32_16x16x32_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x32.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x32.bf8.fp8`. +pub fn swmmac_f32_16x16x32_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x32.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x32.f16`. +pub fn swmmac_f32_16x16x32_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x32.f16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x32.fp8.bf8`. +pub fn swmmac_f32_16x16x32_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x32.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x32.fp8.fp8`. +pub fn swmmac_f32_16x16x32_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x32.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x64.bf16`. +pub fn swmmac_f32_16x16x64_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x64.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x64.f16`. +pub fn swmmac_f32_16x16x64_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x64.f16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.i32.16x16x128.iu8`. +pub fn swmmac_i32_16x16x128_iu8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.i32.16x16x128.iu8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.i32.16x16x32.iu4`. +pub fn swmmac_i32_16x16x32_iu4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.i32.16x16x32.iu4", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.i32.16x16x32.iu8`. +pub fn swmmac_i32_16x16x32_iu8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.i32.16x16x32.iu8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.i32.16x16x64.iu4`. +pub fn swmmac_i32_16x16x64_iu4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.i32.16x16x64.iu4", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.tanh`. +pub fn tanh( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.tanh", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.tensor.load.to.lds`. Base class for ROCDL tensor load/store to/from LDS. +pub fn tensor_load_to_lds( + ctx: *mlir.Context, + dgroup0: *const mlir.Value, + dgroup1: *const mlir.Value, + dgroup2: *const mlir.Value, + dgroup3: *const mlir.Value, + dgroup4: *const mlir.Value, + opts: struct { + cachePolicy: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.cachePolicy) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "cachePolicy", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.tensor.load.to.lds", .{ + .operands = .{ .flat = &.{ dgroup0, dgroup1, dgroup2, dgroup3, dgroup4 } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.tensor.store.from.lds`. Base class for ROCDL tensor load/store to/from LDS. +pub fn tensor_store_from_lds( + ctx: *mlir.Context, + dgroup0: *const mlir.Value, + dgroup1: *const mlir.Value, + dgroup2: *const mlir.Value, + dgroup3: *const mlir.Value, + dgroup4: *const mlir.Value, + opts: struct { + cachePolicy: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.cachePolicy) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "cachePolicy", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.tensor.store.from.lds", .{ + .operands = .{ .flat = &.{ dgroup0, dgroup1, dgroup2, dgroup3, dgroup4 } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.udot2`. +pub fn udot2( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.udot2", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.udot4`. +pub fn udot4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.udot4", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.udot8`. +pub fn udot8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.udot8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.update.dpp`. +pub fn update_dpp( + ctx: *mlir.Context, + old: *const mlir.Value, + src: *const mlir.Value, + res_type: *const mlir.Type, + dppCtrl: *const mlir.Attribute, + rowMask: *const mlir.Attribute, + bankMask: *const mlir.Attribute, + boundCtrl: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dppCtrl", dppCtrl)); + attributes.appendAssumeCapacity(.named(ctx, "rowMask", rowMask)); + attributes.appendAssumeCapacity(.named(ctx, "bankMask", bankMask)); + attributes.appendAssumeCapacity(.named(ctx, "boundCtrl", boundCtrl)); + return mlir.Operation.make(ctx, "rocdl.update.dpp", .{ + .operands = .{ .flat = &.{ old, src } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wait.asyncmark`. Wait until N or fewer async operation groups are unexecuted +pub fn wait_asyncmark( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.wait.asyncmark", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wave.barrier`. +pub fn wave_barrier( + ctx: *mlir.Context, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.wave.barrier", .{ + .location = location, + }); +} + +/// `rocdl.wave.id`. +pub fn wave_id( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wave.id", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wavefrontsize`. +pub fn wavefrontsize( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wavefrontsize", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.bf16.16x16x16.bf16`. +pub fn wmma_bf16_16x16x16_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + opsel: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.opsel) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "opsel", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.bf16.16x16x16.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.bf16.16x16x32.bf16`. +pub fn wmma_bf16_16x16x32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.bf16.16x16x32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.bf16f32.16x16x32.bf16`. +pub fn wmma_bf16f32_16x16x32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.bf16f32.16x16x32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x128.bf8_bf8`. +pub fn wmma_f16_16x16x128_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x128.bf8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x128.bf8_fp8`. +pub fn wmma_f16_16x16x128_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x128.bf8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x128.fp8_bf8`. +pub fn wmma_f16_16x16x128_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x128.fp8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x128.fp8_fp8`. +pub fn wmma_f16_16x16x128_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x128.fp8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x16.f16`. +pub fn wmma_f16_16x16x16_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + opsel: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.opsel) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "opsel", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x16.f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x32.f16`. +pub fn wmma_f16_16x16x32_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x32.f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x64.bf8_bf8`. +pub fn wmma_f16_16x16x64_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x64.bf8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x64.bf8_fp8`. +pub fn wmma_f16_16x16x64_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x64.bf8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x64.fp8_bf8`. +pub fn wmma_f16_16x16x64_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x64.fp8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x64.fp8_fp8`. +pub fn wmma_f16_16x16x64_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x64.fp8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x128.bf8_bf8`. +pub fn wmma_f32_16x16x128_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x128.bf8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x128.bf8_fp8`. +pub fn wmma_f32_16x16x128_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x128.bf8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x128.fp8_bf8`. +pub fn wmma_f32_16x16x128_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x128.fp8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x128.fp8_fp8`. +pub fn wmma_f32_16x16x128_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x128.fp8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x16.bf16`. +pub fn wmma_f32_16x16x16_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x16.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x16.bf8_bf8`. +pub fn wmma_f32_16x16x16_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x16.bf8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x16.bf8_fp8`. +pub fn wmma_f32_16x16x16_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x16.bf8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x16.f16`. +pub fn wmma_f32_16x16x16_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x16.f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x16.fp8_bf8`. +pub fn wmma_f32_16x16x16_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x16.fp8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x16.fp8_fp8`. +pub fn wmma_f32_16x16x16_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x16.fp8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x32.bf16`. +pub fn wmma_f32_16x16x32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x32.f16`. +pub fn wmma_f32_16x16x32_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x32.f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x4.f32`. +pub fn wmma_f32_16x16x4_f32( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x4.f32", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x64.bf8_bf8`. +pub fn wmma_f32_16x16x64_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x64.bf8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x64.bf8_fp8`. +pub fn wmma_f32_16x16x64_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x64.bf8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x64.fp8_bf8`. +pub fn wmma_f32_16x16x64_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x64.fp8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x64.fp8_fp8`. +pub fn wmma_f32_16x16x64_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x64.fp8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.i32.16x16x16.iu4`. +pub fn wmma_i32_16x16x16_iu4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.i32.16x16x16.iu4", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.i32.16x16x16.iu8`. +pub fn wmma_i32_16x16x16_iu8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.i32.16x16x16.iu8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.i32.16x16x32.iu4`. +pub fn wmma_i32_16x16x32_iu4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.i32.16x16x32.iu4", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.i32.16x16x64.iu8`. +pub fn wmma_i32_16x16x64_iu8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.i32.16x16x64.iu8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.scale.f32.16x16x128.f8f6f4`. +pub fn wmma_scale_f32_16x16x128_f8f6f4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + fmtA: ?*const mlir.Attribute = null, + fmtB: ?*const mlir.Attribute = null, + modC: ?*const mlir.Attribute = null, + scaleAType: ?*const mlir.Attribute = null, + fmtScaleA: ?*const mlir.Attribute = null, + scaleBType: ?*const mlir.Attribute = null, + fmtScaleB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 9) = .empty; + if (opts.fmtA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtA", attribute)); + } + if (opts.fmtB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtB", attribute)); + } + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.scaleAType) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "scaleAType", attribute)); + } + if (opts.fmtScaleA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtScaleA", attribute)); + } + if (opts.scaleBType) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "scaleBType", attribute)); + } + if (opts.fmtScaleB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtScaleB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.scale.f32.16x16x128.f8f6f4", .{ + .operands = .{ .flat = &.{ a, b, c, scaleA, scaleB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.scale.f32.32x16x128.f4`. +pub fn wmma_scale_f32_32x16x128_f4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + scaleAType: ?*const mlir.Attribute = null, + fmtScaleA: ?*const mlir.Attribute = null, + scaleBType: ?*const mlir.Attribute = null, + fmtScaleB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.scaleAType) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "scaleAType", attribute)); + } + if (opts.fmtScaleA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtScaleA", attribute)); + } + if (opts.scaleBType) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "scaleBType", attribute)); + } + if (opts.fmtScaleB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtScaleB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.scale.f32.32x16x128.f4", .{ + .operands = .{ .flat = &.{ a, b, c, scaleA, scaleB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.scale16.f32.16x16x128.f8f6f4`. +pub fn wmma_scale16_f32_16x16x128_f8f6f4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + fmtA: ?*const mlir.Attribute = null, + fmtB: ?*const mlir.Attribute = null, + modC: ?*const mlir.Attribute = null, + scaleAType: ?*const mlir.Attribute = null, + fmtScaleA: ?*const mlir.Attribute = null, + scaleBType: ?*const mlir.Attribute = null, + fmtScaleB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 9) = .empty; + if (opts.fmtA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtA", attribute)); + } + if (opts.fmtB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtB", attribute)); + } + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.scaleAType) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "scaleAType", attribute)); + } + if (opts.fmtScaleA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtScaleA", attribute)); + } + if (opts.scaleBType) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "scaleBType", attribute)); + } + if (opts.fmtScaleB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtScaleB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.scale16.f32.16x16x128.f8f6f4", .{ + .operands = .{ .flat = &.{ a, b, c, scaleA, scaleB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.scale16.f32.32x16x128.f4`. +pub fn wmma_scale16_f32_32x16x128_f4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + scaleAType: ?*const mlir.Attribute = null, + fmtScaleA: ?*const mlir.Attribute = null, + scaleBType: ?*const mlir.Attribute = null, + fmtScaleB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.scaleAType) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "scaleAType", attribute)); + } + if (opts.fmtScaleA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtScaleA", attribute)); + } + if (opts.scaleBType) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "scaleBType", attribute)); + } + if (opts.fmtScaleB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtScaleB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.scale16.f32.32x16x128.f4", .{ + .operands = .{ .flat = &.{ a, b, c, scaleA, scaleB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.workgroup.id.x`. +pub fn workgroup_id_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.workgroup.id.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.workgroup.id.y`. +pub fn workgroup_id_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.workgroup.id.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.workgroup.id.z`. +pub fn workgroup_id_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.workgroup.id.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.workitem.id.x`. +pub fn workitem_id_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.workitem.id.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.workitem.id.y`. +pub fn workitem_id_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.workitem.id.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.workitem.id.z`. +pub fn workitem_id_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.workitem.id.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} diff --git a/mlir/dialects/rocdl/rocdl.zig b/mlir/dialects/rocdl/rocdl.zig new file mode 100644 index 00000000..456ad494 --- /dev/null +++ b/mlir/dialects/rocdl/rocdl.zig @@ -0,0 +1,56 @@ +//! LLVM ROCDL operations, AMDGPU attributes, and GPU target configuration. +//! All operation builders from the pinned LLVM revision are available in `ops`. + +const mlir = @import("mlir"); + +pub const types = @import("types.zig"); +pub const annotations = @import("annotations.zig"); +pub const AddressSpace = types.AddressSpace; +pub const pointerType = types.pointerType; + +pub const ops = @import("ops.zig"); +pub const enums = @import("enums.zig"); +pub const attributes = @import("attributes.zig"); + +pub const BufferOOBModeAttr = attributes.BufferOOBModeAttr; +pub const BufferOOBModeModuleFlagAttr = attributes.BufferOOBModeModuleFlagAttr; +pub const Gfx12AtomicCachePolicyAttr = attributes.Gfx12AtomicCachePolicyAttr; +pub const Gfx12CachePolicyAttr = attributes.Gfx12CachePolicyAttr; +pub const Gfx942CachePolicyAttr = attributes.Gfx942CachePolicyAttr; +pub const MFMANegModifierAttr = attributes.MFMANegModifierAttr; +pub const MFMAPermBAttr = attributes.MFMAPermBAttr; +pub const MatrixFormatAttr = attributes.MatrixFormatAttr; +pub const PreGfx12CachePolicyAttr = attributes.PreGfx12CachePolicyAttr; +pub const SchedGroupMaskAttr = attributes.SchedGroupMaskAttr; +pub const TBufferOOBModeModuleFlagAttr = attributes.TBufferOOBModeModuleFlagAttr; +pub const TargetAttr = attributes.TargetAttr; +pub const WMMACModifierAttr = attributes.WMMACModifierAttr; +pub const WMMAMatrixScaleAttr = attributes.WMMAMatrixScaleAttr; +pub const WMMAMatrixScaleFormatAttr = attributes.WMMAMatrixScaleFormatAttr; + +pub const BufferOOBMode = enums.BufferOOBMode; +pub const Gfx12AtomicCachePolicy = enums.Gfx12AtomicCachePolicy; +pub const Gfx12CachePolicy = enums.Gfx12CachePolicy; +pub const Gfx942CachePolicy = enums.Gfx942CachePolicy; +pub const MFMANegModifier = enums.MFMANegModifier; +pub const MFMAPermB = enums.MFMAPermB; +pub const MatrixFormat = enums.MatrixFormat; +pub const PreGfx12CachePolicy = enums.PreGfx12CachePolicy; +pub const SchedGroupMask = enums.SchedGroupMask; +pub const WMMACModifier = enums.WMMACModifier; +pub const WMMAMatrixScale = enums.WMMAMatrixScale; +pub const WMMAMatrixScaleFormat = enums.WMMAMatrixScaleFormat; + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("rocdl"); +} + +/// Converts two packed FP8 values from the selected word to vector<2xf32>. +pub fn cvtPkF32Fp8( + ctx: *mlir.Context, + bits: *const mlir.Value, + high: bool, + location: *const mlir.Location, +) *mlir.Operation { + return ops.cvt_pk_f32_fp8(ctx, bits, .vector(&.{2}, .float(ctx, .f32)), .boolean(ctx, high), location); +} diff --git a/mlir/dialects/rocdl/target.zig b/mlir/dialects/rocdl/target.zig new file mode 100644 index 00000000..30311dc5 --- /dev/null +++ b/mlir/dialects/rocdl/target.zig @@ -0,0 +1,100 @@ +const c = @import("c"); +const mlir = @import("mlir"); + +/// GPU compilation target, `#rocdl.target`. +pub const TargetAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAROCDLTarget; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + optLevel: i32 = 2, + triple: []const u8 = "amdgcn-amd-amdhsa", + chip: []const u8 = "gfx900", + features: []const u8 = "", + abi: []const u8 = "600", + flags: ?*const mlir.Attribute = null, + link: ?*const mlir.Attribute = null, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLTargetAttrGet( + ctx.ptr(), + args.optLevel, + mlir.stringRef(args.triple), + mlir.stringRef(args.chip), + mlir.stringRef(args.features), + mlir.stringRef(args.abi), + .{ .ptr = if (args.flags) |a| @constCast(a.ptr().ptr) else null }, + .{ .ptr = if (args.link) |a| @constCast(a.ptr().ptr) else null }, + ).ptr orelse return error.InvalidMlir); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn optLevel(self: *const Self) i32 { + return c.mlirROCDLTargetAttrGetOptLevel(self.ptr()); + } + + pub fn triple(self: *const Self) []const u8 { + return string(c.mlirROCDLTargetAttrGetTriple(self.ptr())); + } + + pub fn chip(self: *const Self) []const u8 { + return string(c.mlirROCDLTargetAttrGetChip(self.ptr())); + } + + pub fn features(self: *const Self) []const u8 { + return string(c.mlirROCDLTargetAttrGetFeatures(self.ptr())); + } + + pub fn abi(self: *const Self) []const u8 { + return string(c.mlirROCDLTargetAttrGetAbi(self.ptr())); + } + + pub fn flags(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirROCDLTargetAttrGetFlags(self.ptr()).ptr); + } + + pub fn link(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirROCDLTargetAttrGetLink(self.ptr()).ptr); + } + + pub fn hasFlag(self: *const Self, flag: []const u8) bool { + return c.mlirROCDLTargetAttrHasFlag(self.ptr(), mlir.stringRef(flag)); + } + + pub fn hasWave64(self: *const Self) bool { + return c.mlirROCDLTargetAttrHasWave64(self.ptr()); + } + + pub fn hasFastMath(self: *const Self) bool { + return c.mlirROCDLTargetAttrHasFastMath(self.ptr()); + } + + pub fn hasDaz(self: *const Self) bool { + return c.mlirROCDLTargetAttrHasDaz(self.ptr()); + } + + pub fn hasFiniteOnly(self: *const Self) bool { + return c.mlirROCDLTargetAttrHasFiniteOnly(self.ptr()); + } + + pub fn hasUnsafeMath(self: *const Self) bool { + return c.mlirROCDLTargetAttrHasUnsafeMath(self.ptr()); + } + + pub fn hasCorrectSqrt(self: *const Self) bool { + return c.mlirROCDLTargetAttrHasCorrectSqrt(self.ptr()); + } +}; + +fn string(ref: c.MlirStringRef) []const u8 { + return if (ref.length == 0) "" else mlir.string(ref); +} diff --git a/mlir/dialects/rocdl/test.zig b/mlir/dialects/rocdl/test.zig new file mode 100644 index 00000000..81f0b646 --- /dev/null +++ b/mlir/dialects/rocdl/test.zig @@ -0,0 +1,592 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const rocdl = @import("mlir/dialects/rocdl"); + +const operation_names: []const []const u8 = &.{ + "rocdl.asyncmark", + "rocdl.ballot", + "rocdl.barrier", + "rocdl.cluster.id.x", + "rocdl.cluster.id.y", + "rocdl.cluster.id.z", + "rocdl.cluster.load.async.to.lds.b128", + "rocdl.cluster.load.async.to.lds.b32", + "rocdl.cluster.load.async.to.lds.b64", + "rocdl.cluster.load.async.to.lds.b8", + "rocdl.cluster.workgroup.id.x", + "rocdl.cluster.workgroup.id.y", + "rocdl.cluster.workgroup.id.z", + "rocdl.cos", + "rocdl.cvt.f32.bf8", + "rocdl.cvt.f32.fp8", + "rocdl.cvt.pk.bf8.f32", + "rocdl.cvt.pk.f32.bf8", + "rocdl.cvt.pk.f32.fp8", + "rocdl.cvt.pk.fp8.f32", + "rocdl.cvt.pkrtz", + "rocdl.cvt.scale.pk16.bf16.bf6", + "rocdl.cvt.scale.pk16.bf16.fp6", + "rocdl.cvt.scale.pk16.f16.bf6", + "rocdl.cvt.scale.pk16.f16.fp6", + "rocdl.cvt.scale.pk16.f32.bf6", + "rocdl.cvt.scale.pk16.f32.fp6", + "rocdl.cvt.scale.pk8.bf16.bf8", + "rocdl.cvt.scale.pk8.bf16.fp4", + "rocdl.cvt.scale.pk8.bf16.fp8", + "rocdl.cvt.scale.pk8.f16.bf8", + "rocdl.cvt.scale.pk8.f16.fp4", + "rocdl.cvt.scale.pk8.f16.fp8", + "rocdl.cvt.scale.pk8.f32.bf8", + "rocdl.cvt.scale.pk8.f32.fp4", + "rocdl.cvt.scale.pk8.f32.fp8", + "rocdl.cvt.scalef32.2xpk16.bf6.f32", + "rocdl.cvt.scalef32.2xpk16.fp6.f32", + "rocdl.cvt.scalef32.f16.bf8", + "rocdl.cvt.scalef32.f16.fp8", + "rocdl.cvt.scalef32.f32.bf8", + "rocdl.cvt.scalef32.f32.fp8", + "rocdl.cvt.scalef32.pk.bf16.bf8", + "rocdl.cvt.scalef32.pk.bf16.fp4", + "rocdl.cvt.scalef32.pk.bf16.fp8", + "rocdl.cvt.scalef32.pk.bf8.bf16", + "rocdl.cvt.scalef32.pk.bf8.f16", + "rocdl.cvt.scalef32.pk.bf8.f32", + "rocdl.cvt.scalef32.pk.f16.bf8", + "rocdl.cvt.scalef32.pk.f16.fp4", + "rocdl.cvt.scalef32.pk.f16.fp8", + "rocdl.cvt.scalef32.pk.f32.bf8", + "rocdl.cvt.scalef32.pk.f32.fp4", + "rocdl.cvt.scalef32.pk.f32.fp8", + "rocdl.cvt.scalef32.pk.fp4.bf16", + "rocdl.cvt.scalef32.pk.fp4.f16", + "rocdl.cvt.scalef32.pk.fp4.f32", + "rocdl.cvt.scalef32.pk.fp8.bf16", + "rocdl.cvt.scalef32.pk.fp8.f16", + "rocdl.cvt.scalef32.pk.fp8.f32", + "rocdl.cvt.scalef32.pk16.bf6.bf16", + "rocdl.cvt.scalef32.pk16.bf6.f16", + "rocdl.cvt.scalef32.pk16.bf6.f32", + "rocdl.cvt.scalef32.pk16.fp6.bf16", + "rocdl.cvt.scalef32.pk16.fp6.f16", + "rocdl.cvt.scalef32.pk16.fp6.f32", + "rocdl.cvt.scalef32.pk32.bf16.bf6", + "rocdl.cvt.scalef32.pk32.bf16.fp6", + "rocdl.cvt.scalef32.pk32.bf6.bf16", + "rocdl.cvt.scalef32.pk32.bf6.f16", + "rocdl.cvt.scalef32.pk32.f16.bf6", + "rocdl.cvt.scalef32.pk32.f16.fp6", + "rocdl.cvt.scalef32.pk32.f32.bf6", + "rocdl.cvt.scalef32.pk32.f32.fp6", + "rocdl.cvt.scalef32.pk32.fp6.bf16", + "rocdl.cvt.scalef32.pk32.fp6.f16", + "rocdl.cvt.scalef32.pk8.bf8.bf16", + "rocdl.cvt.scalef32.pk8.bf8.f16", + "rocdl.cvt.scalef32.pk8.bf8.f32", + "rocdl.cvt.scalef32.pk8.fp4.bf16", + "rocdl.cvt.scalef32.pk8.fp4.f16", + "rocdl.cvt.scalef32.pk8.fp4.f32", + "rocdl.cvt.scalef32.pk8.fp8.bf16", + "rocdl.cvt.scalef32.pk8.fp8.f16", + "rocdl.cvt.scalef32.pk8.fp8.f32", + "rocdl.cvt.scalef32.sr.bf8.bf16", + "rocdl.cvt.scalef32.sr.bf8.f16", + "rocdl.cvt.scalef32.sr.bf8.f32", + "rocdl.cvt.scalef32.sr.fp8.bf16", + "rocdl.cvt.scalef32.sr.fp8.f16", + "rocdl.cvt.scalef32.sr.fp8.f32", + "rocdl.cvt.scalef32.sr.pk.fp4.bf16", + "rocdl.cvt.scalef32.sr.pk.fp4.f16", + "rocdl.cvt.scalef32.sr.pk.fp4.f32", + "rocdl.cvt.scalef32.sr.pk16.bf6.bf16", + "rocdl.cvt.scalef32.sr.pk16.bf6.f16", + "rocdl.cvt.scalef32.sr.pk16.bf6.f32", + "rocdl.cvt.scalef32.sr.pk16.fp6.bf16", + "rocdl.cvt.scalef32.sr.pk16.fp6.f16", + "rocdl.cvt.scalef32.sr.pk16.fp6.f32", + "rocdl.cvt.scalef32.sr.pk32.bf6.bf16", + "rocdl.cvt.scalef32.sr.pk32.bf6.f16", + "rocdl.cvt.scalef32.sr.pk32.bf6.f32", + "rocdl.cvt.scalef32.sr.pk32.fp6.bf16", + "rocdl.cvt.scalef32.sr.pk32.fp6.f16", + "rocdl.cvt.scalef32.sr.pk32.fp6.f32", + "rocdl.cvt.scalef32.sr.pk8.bf8.bf16", + "rocdl.cvt.scalef32.sr.pk8.bf8.f16", + "rocdl.cvt.scalef32.sr.pk8.bf8.f32", + "rocdl.cvt.scalef32.sr.pk8.fp4.bf16", + "rocdl.cvt.scalef32.sr.pk8.fp4.f16", + "rocdl.cvt.scalef32.sr.pk8.fp4.f32", + "rocdl.cvt.scalef32.sr.pk8.fp8.bf16", + "rocdl.cvt.scalef32.sr.pk8.fp8.f16", + "rocdl.cvt.scalef32.sr.pk8.fp8.f32", + "rocdl.cvt.sr.bf8.f32", + "rocdl.cvt.sr.fp8.f32", + "rocdl.dot4.f32.bf8.bf8", + "rocdl.dot4.f32.bf8.fp8", + "rocdl.dot4.f32.fp8.bf8", + "rocdl.dot4.f32.fp8.fp8", + "rocdl.ds.atomic.async.barrier.arrive.b64", + "rocdl.ds.atomic.barrier.arrive.rtn.b64", + "rocdl.ds.load.tr16.b128", + "rocdl.ds.load.tr4.b64", + "rocdl.ds.load.tr6.b96", + "rocdl.ds.load.tr8.b64", + "rocdl.ds.read.tr16.b64", + "rocdl.ds.read.tr4.b64", + "rocdl.ds.read.tr6.b96", + "rocdl.ds.read.tr8.b64", + "rocdl.ds_bpermute", + "rocdl.ds_swizzle", + "rocdl.exp", + "rocdl.exp2", + "rocdl.fdot2", + "rocdl.fdot2.bf16.bf16", + "rocdl.fdot2.f16.f16", + "rocdl.fdot2.f32.bf16", + "rocdl.flat.prefetch", + "rocdl.fmed3", + "rocdl.global.load.async.lds", + "rocdl.global.load.async.to.lds.b128", + "rocdl.global.load.async.to.lds.b32", + "rocdl.global.load.async.to.lds.b64", + "rocdl.global.load.async.to.lds.b8", + "rocdl.global.load.lds", + "rocdl.global.load.tr.b128", + "rocdl.global.load.tr.b64", + "rocdl.global.load.tr4.b64", + "rocdl.global.load.tr6.b96", + "rocdl.global.prefetch", + "rocdl.global.store.async.from.lds.b128", + "rocdl.global.store.async.from.lds.b32", + "rocdl.global.store.async.from.lds.b64", + "rocdl.global.store.async.from.lds.b8", + "rocdl.iglp.opt", + "rocdl.load.async.to.lds", + "rocdl.load.to.lds", + "rocdl.log", + "rocdl.make.buffer.rsrc", + "rocdl.mbcnt.hi", + "rocdl.mbcnt.lo", + "rocdl.mfma.f32.16x16x16bf16.1k", + "rocdl.mfma.f32.16x16x16f16", + "rocdl.mfma.f32.16x16x1f32", + "rocdl.mfma.f32.16x16x2bf16", + "rocdl.mfma.f32.16x16x32.bf16", + "rocdl.mfma.f32.16x16x32.bf8.bf8", + "rocdl.mfma.f32.16x16x32.bf8.fp8", + "rocdl.mfma.f32.16x16x32.f16", + "rocdl.mfma.f32.16x16x32.fp8.bf8", + "rocdl.mfma.f32.16x16x32.fp8.fp8", + "rocdl.mfma.f32.16x16x4bf16.1k", + "rocdl.mfma.f32.16x16x4f16", + "rocdl.mfma.f32.16x16x4f32", + "rocdl.mfma.f32.16x16x8.xf32", + "rocdl.mfma.f32.16x16x8bf16", + "rocdl.mfma.f32.32x32x16.bf16", + "rocdl.mfma.f32.32x32x16.bf8.bf8", + "rocdl.mfma.f32.32x32x16.bf8.fp8", + "rocdl.mfma.f32.32x32x16.f16", + "rocdl.mfma.f32.32x32x16.fp8.bf8", + "rocdl.mfma.f32.32x32x16.fp8.fp8", + "rocdl.mfma.f32.32x32x1f32", + "rocdl.mfma.f32.32x32x2bf16", + "rocdl.mfma.f32.32x32x2f32", + "rocdl.mfma.f32.32x32x4.xf32", + "rocdl.mfma.f32.32x32x4bf16", + "rocdl.mfma.f32.32x32x4bf16.1k", + "rocdl.mfma.f32.32x32x4f16", + "rocdl.mfma.f32.32x32x8bf16.1k", + "rocdl.mfma.f32.32x32x8f16", + "rocdl.mfma.f32.4x4x1f32", + "rocdl.mfma.f32.4x4x2bf16", + "rocdl.mfma.f32.4x4x4bf16.1k", + "rocdl.mfma.f32.4x4x4f16", + "rocdl.mfma.f64.16x16x4f64", + "rocdl.mfma.f64.4x4x4f64", + "rocdl.mfma.i32.16x16x16i8", + "rocdl.mfma.i32.16x16x32.i8", + "rocdl.mfma.i32.16x16x4i8", + "rocdl.mfma.i32.16x16x64.i8", + "rocdl.mfma.i32.32x32x16.i8", + "rocdl.mfma.i32.32x32x32.i8", + "rocdl.mfma.i32.32x32x4i8", + "rocdl.mfma.i32.32x32x8i8", + "rocdl.mfma.i32.4x4x4i8", + "rocdl.mfma.scale.f32.16x16x128.f8f6f4", + "rocdl.mfma.scale.f32.32x32x64.f8f6f4", + "rocdl.permlane16.swap", + "rocdl.permlane16.var", + "rocdl.permlane32.swap", + "rocdl.permlanex16", + "rocdl.permlanex16.var", + "rocdl.ptr.s.buffer.load", + "rocdl.raw.buffer.atomic.cmpswap", + "rocdl.raw.buffer.atomic.fadd", + "rocdl.raw.buffer.atomic.fmax", + "rocdl.raw.buffer.atomic.smax", + "rocdl.raw.buffer.atomic.umin", + "rocdl.raw.buffer.load", + "rocdl.raw.buffer.store", + "rocdl.raw.ptr.buffer.atomic.cmpswap", + "rocdl.raw.ptr.buffer.atomic.fadd", + "rocdl.raw.ptr.buffer.atomic.fmax", + "rocdl.raw.ptr.buffer.atomic.smax", + "rocdl.raw.ptr.buffer.atomic.umin", + "rocdl.raw.ptr.buffer.load", + "rocdl.raw.ptr.buffer.load.async.lds", + "rocdl.raw.ptr.buffer.load.lds", + "rocdl.raw.ptr.buffer.store", + "rocdl.rcp", + "rocdl.readfirstlane", + "rocdl.readlane", + "rocdl.rsq", + "rocdl.s.barrier", + "rocdl.s.barrier.init", + "rocdl.s.barrier.join", + "rocdl.s.barrier.leave", + "rocdl.s.barrier.signal", + "rocdl.s.barrier.signal.isfirst", + "rocdl.s.barrier.signal.var", + "rocdl.s.barrier.wait", + "rocdl.s.get.barrier.state", + "rocdl.s.get.named.barrier.state", + "rocdl.s.nop", + "rocdl.s.setprio", + "rocdl.s.sleep", + "rocdl.s.wait.asynccnt", + "rocdl.s.wait.dscnt", + "rocdl.s.wait.expcnt", + "rocdl.s.wait.loadcnt", + "rocdl.s.wait.storecnt", + "rocdl.s.wait.tensorcnt", + "rocdl.s.waitcnt", + "rocdl.s.wakeup.barrier", + "rocdl.sched.barrier", + "rocdl.sched.group.barrier", + "rocdl.sdot2", + "rocdl.sdot4", + "rocdl.sdot8", + "rocdl.sin", + "rocdl.smfmac.f32.16x16x128.bf8.bf8", + "rocdl.smfmac.f32.16x16x128.bf8.fp8", + "rocdl.smfmac.f32.16x16x128.fp8.bf8", + "rocdl.smfmac.f32.16x16x128.fp8.fp8", + "rocdl.smfmac.f32.16x16x32.bf16", + "rocdl.smfmac.f32.16x16x32.f16", + "rocdl.smfmac.f32.16x16x64.bf16", + "rocdl.smfmac.f32.16x16x64.bf8.bf8", + "rocdl.smfmac.f32.16x16x64.bf8.fp8", + "rocdl.smfmac.f32.16x16x64.f16", + "rocdl.smfmac.f32.16x16x64.fp8.bf8", + "rocdl.smfmac.f32.16x16x64.fp8.fp8", + "rocdl.smfmac.f32.32x32x16.bf16", + "rocdl.smfmac.f32.32x32x16.f16", + "rocdl.smfmac.f32.32x32x32.bf16", + "rocdl.smfmac.f32.32x32x32.bf8.bf8", + "rocdl.smfmac.f32.32x32x32.bf8.fp8", + "rocdl.smfmac.f32.32x32x32.f16", + "rocdl.smfmac.f32.32x32x32.fp8.bf8", + "rocdl.smfmac.f32.32x32x32.fp8.fp8", + "rocdl.smfmac.f32.32x32x64.bf8.bf8", + "rocdl.smfmac.f32.32x32x64.bf8.fp8", + "rocdl.smfmac.f32.32x32x64.fp8.bf8", + "rocdl.smfmac.f32.32x32x64.fp8.fp8", + "rocdl.smfmac.i32.16x16x128.i8", + "rocdl.smfmac.i32.16x16x64.i8", + "rocdl.smfmac.i32.32x32x32.i8", + "rocdl.smfmac.i32.32x32x64.i8", + "rocdl.sqrt", + "rocdl.sudot4", + "rocdl.sudot8", + "rocdl.swmmac.bf16.16x16x32.bf16", + "rocdl.swmmac.bf16.16x16x64.bf16", + "rocdl.swmmac.bf16f32.16x16x64.bf16", + "rocdl.swmmac.f16.16x16x128.bf8.bf8", + "rocdl.swmmac.f16.16x16x128.bf8.fp8", + "rocdl.swmmac.f16.16x16x128.fp8.bf8", + "rocdl.swmmac.f16.16x16x128.fp8.fp8", + "rocdl.swmmac.f16.16x16x32.f16", + "rocdl.swmmac.f16.16x16x64.f16", + "rocdl.swmmac.f32.16x16x128.bf8.bf8", + "rocdl.swmmac.f32.16x16x128.bf8.fp8", + "rocdl.swmmac.f32.16x16x128.fp8.bf8", + "rocdl.swmmac.f32.16x16x128.fp8.fp8", + "rocdl.swmmac.f32.16x16x32.bf16", + "rocdl.swmmac.f32.16x16x32.bf8.bf8", + "rocdl.swmmac.f32.16x16x32.bf8.fp8", + "rocdl.swmmac.f32.16x16x32.f16", + "rocdl.swmmac.f32.16x16x32.fp8.bf8", + "rocdl.swmmac.f32.16x16x32.fp8.fp8", + "rocdl.swmmac.f32.16x16x64.bf16", + "rocdl.swmmac.f32.16x16x64.f16", + "rocdl.swmmac.i32.16x16x128.iu8", + "rocdl.swmmac.i32.16x16x32.iu4", + "rocdl.swmmac.i32.16x16x32.iu8", + "rocdl.swmmac.i32.16x16x64.iu4", + "rocdl.tanh", + "rocdl.tensor.load.to.lds", + "rocdl.tensor.store.from.lds", + "rocdl.udot2", + "rocdl.udot4", + "rocdl.udot8", + "rocdl.update.dpp", + "rocdl.wait.asyncmark", + "rocdl.wave.barrier", + "rocdl.wave.id", + "rocdl.wavefrontsize", + "rocdl.wmma.bf16.16x16x16.bf16", + "rocdl.wmma.bf16.16x16x32.bf16", + "rocdl.wmma.bf16f32.16x16x32.bf16", + "rocdl.wmma.f16.16x16x128.bf8_bf8", + "rocdl.wmma.f16.16x16x128.bf8_fp8", + "rocdl.wmma.f16.16x16x128.fp8_bf8", + "rocdl.wmma.f16.16x16x128.fp8_fp8", + "rocdl.wmma.f16.16x16x16.f16", + "rocdl.wmma.f16.16x16x32.f16", + "rocdl.wmma.f16.16x16x64.bf8_bf8", + "rocdl.wmma.f16.16x16x64.bf8_fp8", + "rocdl.wmma.f16.16x16x64.fp8_bf8", + "rocdl.wmma.f16.16x16x64.fp8_fp8", + "rocdl.wmma.f32.16x16x128.bf8_bf8", + "rocdl.wmma.f32.16x16x128.bf8_fp8", + "rocdl.wmma.f32.16x16x128.fp8_bf8", + "rocdl.wmma.f32.16x16x128.fp8_fp8", + "rocdl.wmma.f32.16x16x16.bf16", + "rocdl.wmma.f32.16x16x16.bf8_bf8", + "rocdl.wmma.f32.16x16x16.bf8_fp8", + "rocdl.wmma.f32.16x16x16.f16", + "rocdl.wmma.f32.16x16x16.fp8_bf8", + "rocdl.wmma.f32.16x16x16.fp8_fp8", + "rocdl.wmma.f32.16x16x32.bf16", + "rocdl.wmma.f32.16x16x32.f16", + "rocdl.wmma.f32.16x16x4.f32", + "rocdl.wmma.f32.16x16x64.bf8_bf8", + "rocdl.wmma.f32.16x16x64.bf8_fp8", + "rocdl.wmma.f32.16x16x64.fp8_bf8", + "rocdl.wmma.f32.16x16x64.fp8_fp8", + "rocdl.wmma.i32.16x16x16.iu4", + "rocdl.wmma.i32.16x16x16.iu8", + "rocdl.wmma.i32.16x16x32.iu4", + "rocdl.wmma.i32.16x16x64.iu8", + "rocdl.wmma.scale.f32.16x16x128.f8f6f4", + "rocdl.wmma.scale.f32.32x16x128.f4", + "rocdl.wmma.scale16.f32.16x16x128.f8f6f4", + "rocdl.wmma.scale16.f32.32x16x128.f4", + "rocdl.workgroup.id.x", + "rocdl.workgroup.id.y", + "rocdl.workgroup.id.z", + "rocdl.workitem.id.x", + "rocdl.workitem.id.y", + "rocdl.workitem.id.z", +}; + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + rocdl.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectAttributeRoundTrip(ctx: *mlir.Context, attribute: *const mlir.Attribute) !void { + var buffer: [2048]u8 = undefined; + var writer = std.Io.Writer.fixed(&buffer); + try attribute.format(&writer); + try std.testing.expect(attribute.eql(try mlir.Attribute.parse(ctx, writer.buffered()))); +} + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(rocdl); + std.testing.refAllDecls(rocdl.ops); + std.testing.refAllDecls(rocdl.types); + std.testing.refAllDecls(rocdl.annotations); + inline for (rocdl.attributes.enum_attributes) |A| std.testing.refAllDecls(A); + std.testing.refAllDecls(rocdl.TargetAttr); +} + +test "packed FP8 conversion uses the registered LLVM ROCDL dialect" { + const ctx = try testContext(); + defer ctx.deinit(); + try std.testing.expect(ctx.isRegisteredOperation("rocdl.cvt.pk.f32.fp8")); + + const loc = mlir.Location.unknown(ctx); + const block = mlir.Block.init(&.{.int(ctx, .i32)}, &.{loc}); + defer block.deinit(); + for ([_]bool{ false, true }) |high| { + const op = rocdl.cvtPkF32Fp8(ctx, block.argument(0), high, loc); + defer op.deinit(); + try std.testing.expect(op.verify()); + try std.testing.expect(op.result(0).type_().eql(.vector(&.{2}, .float(ctx, .f32)))); + try std.testing.expect(op.attributeByName("wordSel").?.eql(.boolean(ctx, high))); + } +} + +test "every bound operation is registered" { + const ctx = try testContext(); + defer ctx.deinit(); + try std.testing.expectEqual(operation_names.len, @as(usize, @intCast(c.mlirROCDLGetOperationCount(ctx.ptr())))); + for (operation_names, 0..) |name, i| { + try std.testing.expect(ctx.isRegisteredOperation(name)); + for (operation_names[0..i]) |earlier| try std.testing.expect(!std.mem.eql(u8, name, earlier)); + } + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "all ROCDL enum attributes construct and round-trip every named value" { + @setEvalBranchQuota(100_000); + const ctx = try testContext(); + defer ctx.deinit(); + inline for (rocdl.attributes.enum_attributes) |A| { + for (std.enums.values(A.Value)) |value| { + const attribute = try A.get(ctx, .{ .value = value }); + try std.testing.expect(attribute.isA(A) != null); + try std.testing.expectEqual(value, attribute.value()); + try expectAttributeRoundTrip(ctx, attribute.attribute()); + } + } + const mask: rocdl.SchedGroupMask = .combine(&.{ .valu, .mfma_wmma, .ds_read }); + try std.testing.expect(mask.contains(.mfma_wmma)); + try std.testing.expect(!mask.contains(.all_vmem)); + const attribute = try rocdl.SchedGroupMaskAttr.get(ctx, .{ .value = mask }); + try std.testing.expectEqual(mask, attribute.value()); + try expectAttributeRoundTrip(ctx, attribute.attribute()); +} + +test "ROCDL enum constructors reject invalid values and flag bits" { + const ctx = try testContext(); + defer ctx.deinit(); + try std.testing.expect(c.mlirROCDLMatrixFormatAttrGet(ctx.ptr(), 5).ptr == null); + try std.testing.expect(c.mlirROCDLWMMACModifierAttrGet(ctx.ptr(), 65536).ptr == null); + try std.testing.expectError(error.InvalidMlir, rocdl.SchedGroupMaskAttr.get(ctx, .{ .value = @fromBackingInt(1 << 20) })); + try std.testing.expectError(error.InvalidMlir, rocdl.PreGfx12CachePolicyAttr.get(ctx, .{ .value = @fromBackingInt(1 << 6) })); +} + +test "target attributes expose all fields and reject invalid options" { + const ctx = try testContext(); + defer ctx.deinit(); + const defaults = try rocdl.TargetAttr.get(ctx, .{}); + try std.testing.expectEqual(@as(i32, 2), defaults.optLevel()); + try std.testing.expectEqualStrings("amdgcn-amd-amdhsa", defaults.triple()); + try std.testing.expectEqualStrings("gfx900", defaults.chip()); + try std.testing.expectEqualStrings("", defaults.features()); + try std.testing.expectEqualStrings("600", defaults.abi()); + try std.testing.expect(defaults.flags() == null and defaults.link() == null); + try std.testing.expect(defaults.hasWave64()); + try std.testing.expect(!defaults.hasFastMath()); + try std.testing.expect(defaults.hasCorrectSqrt()); + try expectAttributeRoundTrip(ctx, defaults.attribute()); + + const flags: *const mlir.Attribute = @ptrCast(mlir.DictionaryAttribute.init(ctx, &.{ + .named(ctx, "no_wave64", .unit(ctx)), + .named(ctx, "fast", .unit(ctx)), + .named(ctx, "daz", .unit(ctx)), + .named(ctx, "finite_only", .unit(ctx)), + .named(ctx, "unsafe_math", .unit(ctx)), + .named(ctx, "unsafe_sqrt", .unit(ctx)), + })); + const link: *const mlir.Attribute = @ptrCast(mlir.ArrayAttribute.init(ctx, &.{.string(ctx, "device.bc")})); + const target = try rocdl.TargetAttr.get(ctx, .{ .optLevel = 3, .chip = "gfx942", .features = "+xnack", .abi = "500", .flags = flags, .link = link }); + try std.testing.expectEqual(@as(i32, 3), target.optLevel()); + try std.testing.expectEqualStrings("gfx942", target.chip()); + try std.testing.expectEqualStrings("+xnack", target.features()); + try std.testing.expectEqualStrings("500", target.abi()); + try std.testing.expect(target.flags().?.eql(flags)); + try std.testing.expect(target.link().?.eql(link)); + try std.testing.expect(target.hasFlag("fast") and target.hasFastMath()); + try std.testing.expect(!target.hasWave64()); + try std.testing.expect(target.hasDaz() and target.hasFiniteOnly()); + try std.testing.expect(target.hasUnsafeMath() and !target.hasCorrectSqrt()); + try expectAttributeRoundTrip(ctx, target.attribute()); + + try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .optLevel = 4 })); + try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .triple = "" })); + try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .chip = "" })); + try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .abi = "700" })); + try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .flags = .unit(ctx) })); + const badLink: *const mlir.Attribute = @ptrCast(mlir.ArrayAttribute.init(ctx, &.{.unit(ctx)})); + try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .link = badLink })); + const other = try testContext(); + defer other.deinit(); + const otherFlags: *const mlir.Attribute = @ptrCast(mlir.DictionaryAttribute.init(other, &.{})); + try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .flags = otherFlags })); +} + +test "ROCDL barriers buffer loads and matrix operations verify and reparse" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc = mlir.Location.unknown(ctx); + const module = try mlir.Module.parse(ctx, + \\module { + \\ llvm.func @test(%rsrc: vector<4xi32>, %offset: i32, %a: f32, %c: vector<4xf32>) { + \\ llvm.return + \\ } + \\} + ); + defer module.deinit(); + const function = module.body().firstOperation().?; + const block = function.region(0).firstBlock().?; + const terminator = block.terminator().?; + const i32Type: *const mlir.Type = .int(ctx, .i32); + const f32Type: *const mlir.Type = .float(ctx, .f32); + const vec4f32 = rocdl.types.v4f32(ctx); + const zero: *const mlir.Attribute = .int(ctx, .i32, 0); + const opList = .{ + rocdl.ops.s_barrier(ctx, loc), + rocdl.ops.s_sleep(ctx, .int(ctx, .i32, 1), loc), + rocdl.ops.workitem_id_x(ctx, i32Type, .{}, loc), + rocdl.ops.sched_barrier(ctx, (try rocdl.SchedGroupMaskAttr.get(ctx, .{ .value = .combine(&.{ .valu, .mfma_wmma }) })).attribute(), loc), + rocdl.ops.raw_buffer_load(ctx, block.argument(0), block.argument(1), block.argument(1), f32Type, .{}, loc), + rocdl.ops.raw_buffer_load( + ctx, + block.argument(0), + block.argument(1), + block.argument(1), + f32Type, + .{ .aux = (try rocdl.Gfx942CachePolicyAttr.get(ctx, .{ .value = .combine(&.{ .sc0, .nt }) })).attribute() }, + loc, + ), + rocdl.ops.mfma_f32_16x16x4f32(ctx, block.argument(2), block.argument(2), block.argument(3), vec4f32, zero, zero, (try rocdl.MFMAPermBAttr.get(ctx, .{ .value = .none })).attribute(), loc), + }; + inline for (opList) |op| { + try std.testing.expect(op.verify()); + block.insertOwnedOperationBefore(terminator, op); + } + rocdl.annotations.annotateFunction(ctx, function, .{ + .kernel = true, + .reqdWorkGroupSize = .{ 64, 1, 1 }, + .flatWorkGroupSize = "64,64", + .maxFlatWorkGroupSize = 64, + .wavesPerEu = .{ .min = 1, .max = 2 }, + .unsafeFpAtomics = true, + }); + rocdl.annotations.annotateMemory(ctx, opList[4], .{ .lastUse = true, .noRemoteMemory = true, .noFineGrainedMemory = true, .ignoreDenormalMode = true }); + try std.testing.expect(opList[4].attributeByName("aux").?.eql(.int(ctx, .i32, 0))); + try std.testing.expect(opList[5].attributeByName("aux").?.isA(rocdl.Gfx942CachePolicyAttr) != null); + try std.testing.expect(function.attributeByName("rocdl.waves_per_eu").?.eql(.string(ctx, "1,2"))); + try std.testing.expect(module.operation().verify()); + var buffer: [16384]u8 = undefined; + var writer = std.Io.Writer.fixed(&buffer); + try module.format(&writer); + const parsed = try mlir.Module.parse(ctx, writer.buffered()); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "ROCDL annotations preserve module flag defaults and explicit false" { + const ctx = try testContext(); + defer ctx.deinit(); + const module = mlir.Module.init(.unknown(ctx)); + defer module.deinit(); + rocdl.annotations.annotateModule(ctx, module.operation(), .{}); + try std.testing.expect(module.operation().attributeByName("rocdl.xnack") == null); + rocdl.annotations.annotateModule(ctx, module.operation(), .{ .xnack = false, .sramecc = true }); + try std.testing.expect(module.operation().attributeByName("rocdl.xnack").?.eql(.boolean(ctx, false))); + try std.testing.expect(module.operation().attributeByName("rocdl.sramecc").?.eql(.boolean(ctx, true))); + try std.testing.expect(module.operation().verify()); + const pointer = rocdl.pointerType(ctx, .barrier); + try std.testing.expect(pointer.eql(try mlir.Type.parse(ctx, "!llvm.ptr<15>"))); + try std.testing.expect(rocdl.pointerType(ctx, .buffer_resource).eql(try mlir.Type.parse(ctx, "!llvm.ptr<8>"))); +} diff --git a/mlir/dialects/rocdl/types.zig b/mlir/dialects/rocdl/types.zig new file mode 100644 index 00000000..a91ede15 --- /dev/null +++ b/mlir/dialects/rocdl/types.zig @@ -0,0 +1,147 @@ +// Generated from LLVM ROCDL TableGen records; see REGENERATING.md. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +const mlir = @import("mlir"); +const llvm = @import("mlir/dialects/llvm"); + +/// ROCDL address spaces use LLVM pointer types. +pub const AddressSpace = enum(u32) { + generic = 0, + global = 1, + shared = 3, + constant = 4, + private = 5, + buffer_fat_pointer = 7, + buffer_resource = 8, + buffer_strided_pointer = 9, + barrier = 15, +}; + +pub fn pointerType(ctx: *mlir.Context, space: AddressSpace) *const mlir.Type { + return llvm.pointerType(ctx, space); +} + +/// `vector<16xbf16>`, as used by ROCDL operations. +pub fn v16bf16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{16}, .float(ctx, .bf16)); +} + +/// `vector<16xf16>`, as used by ROCDL operations. +pub fn v16f16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{16}, .float(ctx, .f16)); +} + +/// `vector<16xf32>`, as used by ROCDL operations. +pub fn v16f32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{16}, .float(ctx, .f32)); +} + +/// `vector<16xi32>`, as used by ROCDL operations. +pub fn v16i32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{16}, .int(ctx, .i32)); +} + +/// `vector<2xbf16>`, as used by ROCDL operations. +pub fn v2bf16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{2}, .float(ctx, .bf16)); +} + +/// `vector<2xf16>`, as used by ROCDL operations. +pub fn v2f16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{2}, .float(ctx, .f16)); +} + +/// `vector<2xf32>`, as used by ROCDL operations. +pub fn v2f32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{2}, .float(ctx, .f32)); +} + +/// `vector<2xi16>`, as used by ROCDL operations. +pub fn v2i16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{2}, .int(ctx, .i16)); +} + +/// `vector<2xi32>`, as used by ROCDL operations. +pub fn v2i32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{2}, .int(ctx, .i32)); +} + +/// `vector<32xbf16>`, as used by ROCDL operations. +pub fn v32bf16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{32}, .float(ctx, .bf16)); +} + +/// `vector<32xf16>`, as used by ROCDL operations. +pub fn v32f16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{32}, .float(ctx, .f16)); +} + +/// `vector<32xf32>`, as used by ROCDL operations. +pub fn v32f32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{32}, .float(ctx, .f32)); +} + +/// `vector<32xi32>`, as used by ROCDL operations. +pub fn v32i32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{32}, .int(ctx, .i32)); +} + +/// `vector<3xi32>`, as used by ROCDL operations. +pub fn v3i32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{3}, .int(ctx, .i32)); +} + +/// `vector<4xf16>`, as used by ROCDL operations. +pub fn v4f16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{4}, .float(ctx, .f16)); +} + +/// `vector<4xf32>`, as used by ROCDL operations. +pub fn v4f32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{4}, .float(ctx, .f32)); +} + +/// `vector<4xf64>`, as used by ROCDL operations. +pub fn v4f64(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{4}, .float(ctx, .f64)); +} + +/// `vector<4xi16>`, as used by ROCDL operations. +pub fn v4i16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{4}, .int(ctx, .i16)); +} + +/// `vector<4xi32>`, as used by ROCDL operations. +pub fn v4i32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{4}, .int(ctx, .i32)); +} + +/// `vector<6xi32>`, as used by ROCDL operations. +pub fn v6i32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{6}, .int(ctx, .i32)); +} + +/// `vector<8xbf16>`, as used by ROCDL operations. +pub fn v8bf16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{8}, .float(ctx, .bf16)); +} + +/// `vector<8xf16>`, as used by ROCDL operations. +pub fn v8f16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{8}, .float(ctx, .f16)); +} + +/// `vector<8xf32>`, as used by ROCDL operations. +pub fn v8f32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{8}, .float(ctx, .f32)); +} + +/// `vector<8xi16>`, as used by ROCDL operations. +pub fn v8i16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{8}, .int(ctx, .i16)); +} + +/// `vector<8xi32>`, as used by ROCDL operations. +pub fn v8i32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{8}, .int(ctx, .i32)); +} diff --git a/mlir/dialects/scf/BUILD.bazel b/mlir/dialects/scf/BUILD.bazel new file mode 100644 index 00000000..1d7a0b3c --- /dev/null +++ b/mlir/dialects/scf/BUILD.bazel @@ -0,0 +1,23 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "scf", + import_name = "mlir/dialects/scf", + main = "scf.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "//stdx", + "@llvm-project//mlir:CAPISCF", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":scf", + "//mlir", + ], +) diff --git a/mlir/dialects/scf.zig b/mlir/dialects/scf/scf.zig similarity index 97% rename from mlir/dialects/scf.zig rename to mlir/dialects/scf/scf.zig index 2feb5268..1c82a9e0 100644 --- a/mlir/dialects/scf.zig +++ b/mlir/dialects/scf/scf.zig @@ -3,6 +3,10 @@ const std = @import("std"); const mlir = @import("mlir"); const stdx = @import("stdx"); +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("scf"); +} + pub const ForOpts = struct { /// Per td: `scf.for` uses signed comparison by default. Setting this emits /// the `unsignedCmp` UnitAttr so the induction variable advances with @@ -63,7 +67,9 @@ pub fn if_( cond: *const mlir.Value, result_types: []const *const mlir.Type, then_block: *mlir.Block, - else_block: ?*mlir.Block, + opts: struct { + else_block: ?*mlir.Block = null, + }, location: *const mlir.Location, ) *mlir.Operation { _ = ctx; @@ -74,7 +80,7 @@ pub fn if_( const then_region = mlir.Region.init(); then_region.appendOwnedBlock(then_block); const else_region = mlir.Region.init(); - if (else_block) |b| else_region.appendOwnedBlock(b); + if (opts.else_block) |b| else_region.appendOwnedBlock(b); state.addOwnedRegions(&.{ then_region, else_region }); return mlir.Operation.init(&state) catch @panic("Failed to create scf.if"); @@ -251,9 +257,11 @@ pub fn forall( ubs: []const *const mlir.Value, steps: []const *const mlir.Value, shared_outs: []const *const mlir.Value, - mapping: ?*const mlir.Attribute, body_block: *mlir.Block, result_types: []const *const mlir.Type, + opts: struct { + mapping: ?*const mlir.Attribute = null, + }, location: *const mlir.Location, ) *mlir.Operation { std.debug.assert(lbs.len == ubs.len); @@ -289,7 +297,7 @@ pub fn forall( .named(ctx, "staticUpperBound", .denseArray(ctx, .i64, sentinel_slice)), .named(ctx, "staticStep", .denseArray(ctx, .i64, sentinel_slice)), }); - if (mapping) |m| attrs.appendAssumeCapacity(.named(ctx, "mapping", m)); + if (opts.mapping) |m| attrs.appendAssumeCapacity(.named(ctx, "mapping", m)); return mlir.Operation.make(ctx, "scf.forall", .{ .operands = .{ .flat = operands.constSlice() }, @@ -353,7 +361,3 @@ pub fn index_switch( return mlir.Operation.init(&state) catch @panic("Failed to create scf.index_switch"); } - -test { - std.testing.refAllDecls(@This()); -} diff --git a/mlir/dialects/scf/test.zig b/mlir/dialects/scf/test.zig new file mode 100644 index 00000000..dceac66e --- /dev/null +++ b/mlir/dialects/scf/test.zig @@ -0,0 +1,39 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const scf = @import("mlir/dialects/scf"); + +const operation_names: []const []const u8 = &.{ + "scf.condition", + "scf.execute_region", + "scf.for", + "scf.forall", + "scf.forall.in_parallel", + "scf.if", + "scf.index_switch", + "scf.parallel", + "scf.reduce", + "scf.reduce.return", + "scf.while", + "scf.yield", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(scf); + inline for (comptime std.meta.declarations(scf)) |decl| { + const T = @field(scf, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + scf.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/shardy.zig b/mlir/dialects/shardy.zig deleted file mode 100644 index b391d261..00000000 --- a/mlir/dialects/shardy.zig +++ /dev/null @@ -1,459 +0,0 @@ -//! Bindings for OpenXLA shardy dialect -//! -//! - dialect overview: https://openxla.org/shardy/sdy_dialect -//! - C API: https://github.com/openxla/shardy/blob/main/shardy/integrations/c/attributes.h -const std = @import("std"); - -const c = @import("c"); -const mlir = @import("mlir"); - -pub fn dialectHandle() *const mlir.DialectHandle { - return @ptrCast(c.mlirGetDialectHandle__sdy__().ptr); -} - -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - dialectHandle().insertDialect(registry); -} - -pub const attr = struct { - pub fn mesh(ctx: *mlir.Context, mesh_: MeshAttribute.Mesh) *const mlir.Attribute { - return MeshAttribute.init(ctx, mesh_).asAttr(); - } - - pub fn shardingsPerValue(ctx: *mlir.Context, shardings: []const *const TensorShardingAttribute) *const mlir.Attribute { - return TensorShardingPerValueAttribute.init(ctx, shardings).asAttr(); - } -}; - -pub const MeshAxisAttribute = opaque { - const M = mlir.Methods(MeshAxisAttribute, c.MlirAttribute); - - pub const isAFn = c.sdyAttributeIsAMeshAxisAttr; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub fn init(ctx: *mlir.Context, axis_name: []const u8, axis_size: i64) *const MeshAxisAttribute { - return @ptrCast(c.sdyMeshAxisAttrGet(ctx.ptr(), mlir.stringRef(axis_name), axis_size).ptr); - } - - pub fn asAttr(self: *const MeshAxisAttribute) *const mlir.Attribute { - return @ptrCast(self); - } - - pub fn name(self: *const MeshAxisAttribute) []const u8 { - return mlir.string(c.sdyMeshAxisAttrGetName(self.ptr())); - } - - pub fn size(self: *const MeshAxisAttribute) i64 { - return c.sdyMeshAxisAttrGetSize(self.ptr()); - } -}; - -pub const MeshAttribute = opaque { - const M = mlir.Methods(MeshAttribute, c.MlirAttribute); - - pub const isAFn = c.sdyAttributeIsAMeshAttr; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const Mesh = struct { - axes: []const *const MeshAxisAttribute, - device_ids: []const i64 = &.{}, - }; - - pub fn init(ctx: *mlir.Context, mesh: Mesh) *const MeshAttribute { - const axes: []const c.struct_MlirAttribute = @ptrCast(mesh.axes); - return @ptrCast(c.sdyMeshAttrGet( - ctx.ptr(), - @intCast(axes.len), - axes.ptr, - @intCast(mesh.device_ids.len), - mesh.device_ids.ptr, - ).ptr); - } - - pub fn asAttr(self: *const MeshAttribute) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -pub const SubAxisInfoAttribute = opaque { - const M = mlir.Methods(SubAxisInfoAttribute, c.MlirAttribute); - - pub const isAFn = c.sdyAttributeIsASubAxisInfoAttr; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub fn init(ctx: *mlir.Context, pre_size: i64, sub_axis_size: i64) *const SubAxisInfoAttribute { - return @ptrCast(c.sdySubAxisInfoAttrGet(ctx.ptr(), pre_size, sub_axis_size).ptr); - } - - pub fn asAttr(self: *const SubAxisInfoAttribute) *const mlir.Attribute { - return @ptrCast(self); - } - - pub fn preSize(self: *const SubAxisInfoAttribute) i64 { - return c.sdySubAxisInfoAttrGetPreSize(self.ptr()); - } - - pub fn size(self: *const SubAxisInfoAttribute) i64 { - return c.sdySubAxisInfoAttrGetSize(self.ptr()); - } -}; - -pub const AxisRefAttribute = opaque { - const M = mlir.Methods(AxisRefAttribute, c.MlirAttribute); - - pub const isAFn = c.sdyAttributeIsAnAxisRefAttr; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub fn named(ctx: *mlir.Context, axis_name: []const u8) *const AxisRefAttribute { - return @ptrCast(c.sdyAxisRefAttrGet(ctx.ptr(), mlir.stringRef(axis_name), .{ .ptr = null }).ptr); - } - - pub fn subAxis(ctx: *mlir.Context, axis_name: []const u8, pre_size: i64, size: i64) *const AxisRefAttribute { - const sub_axis: *const SubAxisInfoAttribute = .init(ctx, pre_size, size); - return @ptrCast(c.sdyAxisRefAttrGet(ctx.ptr(), mlir.stringRef(axis_name), sub_axis).ptr); - } - - pub fn asAttr(self: *const AxisRefAttribute) *const mlir.Attribute { - return @ptrCast(self); - } - - pub fn name(self: *const AxisRefAttribute) []const u8 { - return mlir.string(c.sdyAxisRefAttrGetName(self.ptr())); - } - - pub fn subAxisInfo(self: *const AxisRefAttribute) ?*const SubAxisInfoAttribute { - const attr_ = c.sdyAxisRefAttrGetSubAxisInfo(self.ptr()); - return if (attr_.ptr == null) null else @ptrCast(attr_.ptr); - } -}; - -pub const DimensionShardingAttribute = opaque { - const M = mlir.Methods(DimensionShardingAttribute, c.MlirAttribute); - - pub const isAFn = c.sdyAttributeIsADimensionShardingAttr; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub fn init( - ctx: *mlir.Context, - axes: []const *const AxisRefAttribute, - is_closed: bool, - priority_: ?i64, - ) *const DimensionShardingAttribute { - const attrs: []const c.struct_MlirAttribute = @ptrCast(axes); - return @ptrCast(c.sdyDimensionShardingAttrGet(ctx.ptr(), @intCast(attrs.len), attrs.ptr, is_closed, priority_ orelse -1).ptr); - } - - pub fn closed(ctx: *mlir.Context, axes: []const *const AxisRefAttribute) *const DimensionShardingAttribute { - return init(ctx, axes, true, null); - } - - pub fn open(ctx: *mlir.Context, axes: []const *const AxisRefAttribute) *const DimensionShardingAttribute { - return init(ctx, axes, false, null); - } - - pub fn replicated(ctx: *mlir.Context) *const DimensionShardingAttribute { - return closed(ctx, &.{}); - } - - pub fn asAttr(self: *const DimensionShardingAttribute) *const mlir.Attribute { - return @ptrCast(self); - } - - pub fn numAxes(self: *const DimensionShardingAttribute) usize { - return @intCast(c.sdyDimensionShardingAttrGetAxesSize(self.ptr())); - } - - pub fn axis(self: *const DimensionShardingAttribute, index: usize) *const AxisRefAttribute { - return @ptrCast(c.sdyDimensionShardingAttrGetAxesElem(self.ptr(), @intCast(index)).ptr); - } - - pub fn isClosed(self: *const DimensionShardingAttribute) bool { - return c.sdyDimensionShardingAttrGetIsClosed(self.ptr()); - } - - pub fn priority(self: *const DimensionShardingAttribute) ?i64 { - const value = c.sdyDimensionShardingAttrGetPriority(self.ptr()); - return if (value == -1) null else value; - } -}; - -pub const Sharding = struct { - mesh: []const u8, - dimensions: []const *const DimensionShardingAttribute, - replicated_axes: []const *const AxisRefAttribute = &.{}, - unreduced_axes: []const *const AxisRefAttribute = &.{}, -}; - -pub const TensorShardingAttribute = opaque { - const M = mlir.Methods(TensorShardingAttribute, c.MlirAttribute); - - pub const isAFn = c.sdyAttributeIsATensorShardingAttr; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub fn init(ctx: *mlir.Context, args: Sharding) *const TensorShardingAttribute { - comptime std.debug.assert(@sizeOf(c.struct_MlirAttribute) == @sizeOf(*const mlir.Attribute)); - const dimensions: []const c.struct_MlirAttribute = @ptrCast(args.dimensions); - const replicated: []const c.struct_MlirAttribute = @ptrCast(args.replicated_axes); - const unreduced: []const c.struct_MlirAttribute = @ptrCast(args.unreduced_axes); - return @ptrCast(c.sdyTensorShardingAttrGet( - ctx.ptr(), - mlir.Attribute.flatSymbolRef(ctx, args.mesh).ptr(), - @intCast(dimensions.len), - dimensions.ptr, - @intCast(replicated.len), - replicated.ptr, - @intCast(unreduced.len), - unreduced.ptr, - 0, - ).ptr); - } - - pub fn asAttr(self: *const TensorShardingAttribute) *const mlir.Attribute { - return @ptrCast(self); - } - - pub fn meshOrRef(self: *const TensorShardingAttribute) *const mlir.Attribute { - return @ptrCast(c.sdyTensorShardingAttrGetMeshOrRef(self.ptr()).ptr); - } - - pub fn numDimensions(self: *const TensorShardingAttribute) usize { - return @intCast(c.sdyTensorShardingAttrGetDimShardingsSize(self.ptr())); - } - - pub fn dimension(self: *const TensorShardingAttribute, index: usize) *const DimensionShardingAttribute { - return @ptrCast(c.sdyTensorShardingAttrGetDimShardingsElem(self.ptr(), @intCast(index)).ptr); - } - - pub fn numReplicatedAxes(self: *const TensorShardingAttribute) usize { - return @intCast(c.sdyTensorShardingAttrGetReplicatedAxesSize(self.ptr())); - } - - pub fn replicatedAxis(self: *const TensorShardingAttribute, index: usize) *const AxisRefAttribute { - return @ptrCast(c.sdyTensorShardingAttrGetReplicatedAxesElem(self.ptr(), @intCast(index)).ptr); - } - - pub fn numUnreducedAxes(self: *const TensorShardingAttribute) usize { - return @intCast(c.sdyTensorShardingAttrGetUnreducedAxesSize(self.ptr())); - } - - pub fn unreducedAxis(self: *const TensorShardingAttribute, index: usize) *const AxisRefAttribute { - return @ptrCast(c.sdyTensorShardingAttrGetUnreducedAxesElem(self.ptr(), @intCast(index)).ptr); - } -}; - -test TensorShardingAttribute { - const registry: *mlir.DialectRegistry = try .init(); - defer registry.deinit(); - registry.registerDialect("sdy"); - mlir.registerFuncExtensions(registry); - - var ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - - var w: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer w.deinit(); - - { - defer w.clearRetainingCapacity(); - const sharding: *const TensorShardingAttribute = .init(ctx, .{ - .mesh = "folded_mesh", - .dimensions = &.{.replicated(ctx)}, - .replicated_axes = &.{ .named(ctx, "link_x"), .named(ctx, "link_y") }, - }); - try sharding.asAttr().format(&w.writer); - try std.testing.expectEqualSlices(u8, "#sdy.sharding<@folded_mesh, [{}], replicated={\"link_x\", \"link_y\"}>", w.written()); - } - { - defer w.clearRetainingCapacity(); - const sharding: *const TensorShardingAttribute = .init(ctx, .{ - .mesh = "suggested_mesh", - .dimensions = &.{ - .closed(ctx, &.{.named(ctx, "link_z")}), - .closed(ctx, &.{.named(ctx, "link_x")}), - }, - .replicated_axes = &.{.named(ctx, "link_y")}, - }); - try sharding.asAttr().format(&w.writer); - try std.testing.expectEqualSlices(u8, "#sdy.sharding<@suggested_mesh, [{\"link_z\"}, {\"link_x\"}], replicated={\"link_y\"}>", w.written()); - } - { - defer w.clearRetainingCapacity(); - const sharding: *const TensorShardingAttribute = .init(ctx, .{ - .mesh = "fold_mesh", - .dimensions = &.{ .closed(ctx, &.{.named(ctx, "link_x")}), .replicated(ctx) }, - }); - try sharding.asAttr().format(&w.writer); - try std.testing.expectEqualSlices(u8, "#sdy.sharding<@fold_mesh, [{\"link_x\"}, {}]>", w.written()); - } - { - defer w.clearRetainingCapacity(); - const sharding: *const TensorShardingAttribute = .init(ctx, .{ - .mesh = "folded_mesh", - .dimensions = &.{.closed(ctx, &.{ .named(ctx, "link_x"), .named(ctx, "link_y") })}, - }); - try sharding.asAttr().format(&w.writer); - try std.testing.expectEqualSlices(u8, "#sdy.sharding<@folded_mesh, [{\"link_x\", \"link_y\"}]>", w.written()); - } - { - defer w.clearRetainingCapacity(); - const sharding: *const TensorShardingAttribute = .init(ctx, .{ - .mesh = "strategy_fold", - .dimensions = &.{.closed(ctx, &.{.named(ctx, "link_x")})}, - .replicated_axes = &.{.named(ctx, "link_y")}, - }); - try sharding.asAttr().format(&w.writer); - try std.testing.expectEqualSlices(u8, "#sdy.sharding<@strategy_fold, [{\"link_x\"}], replicated={\"link_y\"}>", w.written()); - } - { - defer w.clearRetainingCapacity(); - const sharding: *const TensorShardingAttribute = .init(ctx, .{ - .mesh = "mix_mesh", - .dimensions = &.{ .open(ctx, &.{}), .replicated(ctx) }, - .replicated_axes = &.{ .named(ctx, "link_x"), .named(ctx, "link_y") }, - }); - try sharding.asAttr().format(&w.writer); - try std.testing.expectEqualSlices(u8, "#sdy.sharding<@mix_mesh, [{?}, {}], replicated={\"link_x\", \"link_y\"}>", w.written()); - } - { - defer w.clearRetainingCapacity(); - const sharding: *const TensorShardingAttribute = .init(ctx, .{ - .mesh = "3d_mesh", - .dimensions = &.{ - .closed(ctx, &.{.named(ctx, "link_x")}), - .closed(ctx, &.{.named(ctx, "link_y")}), - .closed(ctx, &.{.named(ctx, "link_z")}), - }, - }); - try sharding.asAttr().format(&w.writer); - try std.testing.expectEqualSlices(u8, "#sdy.sharding<@\"3d_mesh\", [{\"link_x\"}, {\"link_y\"}, {\"link_z\"}]>", w.written()); - } -} - -pub const TensorShardingPerValueAttribute = opaque { - const M = mlir.Methods(TensorShardingPerValueAttribute, c.MlirAttribute); - - pub const isAFn = c.sdyAttributeIsATensorShardingPerValueAttr; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub fn init(ctx: *mlir.Context, shardings: []const *const TensorShardingAttribute) *const TensorShardingPerValueAttribute { - const attrs: []const c.struct_MlirAttribute = @ptrCast(shardings); - return @ptrCast(c.sdyTensorShardingPerValueAttrGet(ctx.ptr(), @intCast(attrs.len), attrs.ptr).ptr); - } - - pub fn asAttr(self: *const TensorShardingPerValueAttribute) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -test TensorShardingPerValueAttribute { - const registry: *mlir.DialectRegistry = try .init(); - defer registry.deinit(); - registry.registerDialect("sdy"); - mlir.registerFuncExtensions(registry); - - var ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - - const shardings: [2]*const TensorShardingAttribute = .{ - .init(ctx, .{ - .mesh = "mesh", - .dimensions = &.{.closed(ctx, &.{.named(ctx, "link_x")})}, - }), - .init(ctx, .{ - .mesh = "mesh", - .dimensions = &.{.replicated(ctx)}, - .replicated_axes = &.{.named(ctx, "link_y")}, - }), - }; - - const per_value = TensorShardingPerValueAttribute.init(ctx, &shardings); - - var w: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer w.deinit(); - - try per_value.asAttr().format(&w.writer); - try std.testing.expectEqualSlices( - u8, - "#sdy.sharding_per_value<[<@mesh, [{\"link_x\"}]>, <@mesh, [{}], replicated={\"link_y\"}>]>", - w.written(), - ); -} - -pub const ManualAxesAttribute = opaque { - const M = mlir.Methods(ManualAxesAttribute, c.MlirAttribute); - - pub const isAFn = c.sdyAttributeIsAManualAxesAttr; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub fn init(ctx: *mlir.Context, axes: []const *const mlir.StringAttribute) *const ManualAxesAttribute { - comptime std.debug.assert(@sizeOf(c.struct_MlirAttribute) == @sizeOf(*const mlir.StringAttribute)); - const attrs: []const c.struct_MlirAttribute = @ptrCast(axes); - return @ptrCast(c.sdyManualAxesAttrGet(ctx.ptr(), @intCast(attrs.len), attrs.ptr).ptr); - } - - pub fn asAttr(self: *const ManualAxesAttribute) *const mlir.Attribute { - return @ptrCast(self); - } - - pub fn numAxes(self: *const ManualAxesAttribute) usize { - return @intCast(c.sdyManualAxesAttrGetAxesSize(self.ptr())); - } - - pub fn axis(self: *const ManualAxesAttribute, index: usize) []const u8 { - return mlir.string(c.sdyManualAxesAttrGetAxesElem(self.ptr(), @intCast(index))); - } -}; - -test ManualAxesAttribute { - const registry: *mlir.DialectRegistry = try .init(); - defer registry.deinit(); - registry.registerDialect("sdy"); - mlir.registerFuncExtensions(registry); - - var ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - - const axes: [2]*const mlir.StringAttribute = .{ - .init(ctx, "link_x"), - .init(ctx, "link_y"), - }; - - const manual_axes: *const ManualAxesAttribute = .init(ctx, &axes); - - try std.testing.expectEqual(2, manual_axes.numAxes()); - try std.testing.expectEqualStrings("link_x", manual_axes.axis(0)); - try std.testing.expectEqualStrings("link_y", manual_axes.axis(1)); - - var w: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer w.deinit(); - - try manual_axes.asAttr().format(&w.writer); - try std.testing.expectEqualSlices(u8, "#sdy", w.written()); -} diff --git a/mlir/dialects/shardy/BUILD.bazel b/mlir/dialects/shardy/BUILD.bazel new file mode 100644 index 00000000..4077f902 --- /dev/null +++ b/mlir/dialects/shardy/BUILD.bazel @@ -0,0 +1,23 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "shardy", + import_name = "mlir/dialects/shardy", + main = "shardy.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "@shardy//shardy/integrations/c:sdy_capi", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":shardy", + "//mlir", + "//mlir/dialects/func", + ], +) diff --git a/mlir/dialects/shardy/shardy.zig b/mlir/dialects/shardy/shardy.zig new file mode 100644 index 00000000..e89685a2 --- /dev/null +++ b/mlir/dialects/shardy/shardy.zig @@ -0,0 +1,312 @@ +//! Bindings for OpenXLA shardy dialect +//! +//! - dialect overview: https://openxla.org/shardy/sdy_dialect +//! - C API: https://github.com/openxla/shardy/blob/main/shardy/integrations/c/attributes.h +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("sdy"); +} + +pub fn meshAttr(ctx: *mlir.Context, args: MeshAttr.InitArgs) *const mlir.Attribute { + return MeshAttr.get(ctx, args).attribute(); +} + +pub fn shardingsPerValueAttr(ctx: *mlir.Context, shardings: []const *const TensorShardingAttr) *const mlir.Attribute { + return TensorShardingPerValueAttr.get(ctx, shardings).attribute(); +} + +pub const MeshAxisAttr = opaque { + const M = mlir.Methods(MeshAxisAttr, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsAMeshAxisAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + name: []const u8, + size: i64, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const MeshAxisAttr { + return @ptrCast(c.sdyMeshAxisAttrGet(ctx.ptr(), mlir.stringRef(args.name), args.size).ptr); + } + + pub fn attribute(self: *const MeshAxisAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn name(self: *const MeshAxisAttr) []const u8 { + return mlir.string(c.sdyMeshAxisAttrGetName(self.ptr())); + } + + pub fn size(self: *const MeshAxisAttr) i64 { + return c.sdyMeshAxisAttrGetSize(self.ptr()); + } +}; + +pub const MeshAttr = opaque { + const M = mlir.Methods(MeshAttr, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsAMeshAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + axes: []const *const MeshAxisAttr, + device_ids: []const i64 = &.{}, + }; + + pub fn get(ctx: *mlir.Context, mesh: InitArgs) *const MeshAttr { + const axes: []const c.struct_MlirAttribute = @ptrCast(mesh.axes); + return @ptrCast(c.sdyMeshAttrGet( + ctx.ptr(), + @intCast(axes.len), + axes.ptr, + @intCast(mesh.device_ids.len), + mesh.device_ids.ptr, + ).ptr); + } + + pub fn attribute(self: *const MeshAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub const SubAxisInfoAttr = opaque { + const M = mlir.Methods(SubAxisInfoAttr, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsASubAxisInfoAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + pre_size: i64, + size: i64, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const SubAxisInfoAttr { + return @ptrCast(c.sdySubAxisInfoAttrGet(ctx.ptr(), args.pre_size, args.size).ptr); + } + + pub fn attribute(self: *const SubAxisInfoAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn preSize(self: *const SubAxisInfoAttr) i64 { + return c.sdySubAxisInfoAttrGetPreSize(self.ptr()); + } + + pub fn size(self: *const SubAxisInfoAttr) i64 { + return c.sdySubAxisInfoAttrGetSize(self.ptr()); + } +}; + +pub const AxisRefAttr = opaque { + const M = mlir.Methods(AxisRefAttr, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsAnAxisRefAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn named(ctx: *mlir.Context, axis_name: []const u8) *const AxisRefAttr { + return @ptrCast(c.sdyAxisRefAttrGet(ctx.ptr(), mlir.stringRef(axis_name), .{ .ptr = null }).ptr); + } + + pub fn subAxis(ctx: *mlir.Context, axis_name: []const u8, pre_size: i64, size: i64) *const AxisRefAttr { + const sub_axis: *const SubAxisInfoAttr = .get(ctx, .{ .pre_size = pre_size, .size = size }); + return @ptrCast(c.sdyAxisRefAttrGet(ctx.ptr(), mlir.stringRef(axis_name), sub_axis.ptr()).ptr); + } + + pub fn attribute(self: *const AxisRefAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn name(self: *const AxisRefAttr) []const u8 { + return mlir.string(c.sdyAxisRefAttrGetName(self.ptr())); + } + + pub fn subAxisInfo(self: *const AxisRefAttr) ?*const SubAxisInfoAttr { + const attr_ = c.sdyAxisRefAttrGetSubAxisInfo(self.ptr()); + return if (attr_.ptr == null) null else @ptrCast(attr_.ptr); + } +}; + +pub const DimensionShardingAttr = opaque { + const M = mlir.Methods(DimensionShardingAttr, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsADimensionShardingAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + axes: []const *const AxisRefAttr, + is_closed: bool, + priority: ?i64 = null, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const DimensionShardingAttr { + const attrs: []const c.struct_MlirAttribute = @ptrCast(args.axes); + return @ptrCast(c.sdyDimensionShardingAttrGet(ctx.ptr(), @intCast(attrs.len), attrs.ptr, args.is_closed, args.priority orelse -1).ptr); + } + + pub fn closed(ctx: *mlir.Context, axes: []const *const AxisRefAttr) *const DimensionShardingAttr { + return get(ctx, .{ .axes = axes, .is_closed = true }); + } + + pub fn open(ctx: *mlir.Context, axes: []const *const AxisRefAttr) *const DimensionShardingAttr { + return get(ctx, .{ .axes = axes, .is_closed = false }); + } + + pub fn replicated(ctx: *mlir.Context) *const DimensionShardingAttr { + return closed(ctx, &.{}); + } + + pub fn attribute(self: *const DimensionShardingAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn numAxes(self: *const DimensionShardingAttr) usize { + return @intCast(c.sdyDimensionShardingAttrGetAxesSize(self.ptr())); + } + + pub fn axis(self: *const DimensionShardingAttr, index: usize) *const AxisRefAttr { + return @ptrCast(c.sdyDimensionShardingAttrGetAxesElem(self.ptr(), @intCast(index)).ptr); + } + + pub fn isClosed(self: *const DimensionShardingAttr) bool { + return c.sdyDimensionShardingAttrGetIsClosed(self.ptr()); + } + + pub fn priority(self: *const DimensionShardingAttr) ?i64 { + const value = c.sdyDimensionShardingAttrGetPriority(self.ptr()); + return if (value == -1) null else value; + } +}; + +pub const TensorShardingAttr = opaque { + const M = mlir.Methods(TensorShardingAttr, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsATensorShardingAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + mesh: []const u8, + dimensions: []const *const DimensionShardingAttr, + replicated_axes: []const *const AxisRefAttr = &.{}, + unreduced_axes: []const *const AxisRefAttr = &.{}, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const TensorShardingAttr { + comptime std.debug.assert(@sizeOf(c.struct_MlirAttribute) == @sizeOf(*const mlir.Attribute)); + const dimensions: []const c.struct_MlirAttribute = @ptrCast(args.dimensions); + const replicated: []const c.struct_MlirAttribute = @ptrCast(args.replicated_axes); + const unreduced: []const c.struct_MlirAttribute = @ptrCast(args.unreduced_axes); + return @ptrCast(c.sdyTensorShardingAttrGet( + ctx.ptr(), + mlir.Attribute.flatSymbolRef(ctx, args.mesh).ptr(), + @intCast(dimensions.len), + dimensions.ptr, + @intCast(replicated.len), + replicated.ptr, + @intCast(unreduced.len), + unreduced.ptr, + 0, + ).ptr); + } + + pub fn attribute(self: *const TensorShardingAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn meshOrRef(self: *const TensorShardingAttr) *const mlir.Attribute { + return @ptrCast(c.sdyTensorShardingAttrGetMeshOrRef(self.ptr()).ptr); + } + + pub fn numDimensions(self: *const TensorShardingAttr) usize { + return @intCast(c.sdyTensorShardingAttrGetDimShardingsSize(self.ptr())); + } + + pub fn dimension(self: *const TensorShardingAttr, index: usize) *const DimensionShardingAttr { + return @ptrCast(c.sdyTensorShardingAttrGetDimShardingsElem(self.ptr(), @intCast(index)).ptr); + } + + pub fn numReplicatedAxes(self: *const TensorShardingAttr) usize { + return @intCast(c.sdyTensorShardingAttrGetReplicatedAxesSize(self.ptr())); + } + + pub fn replicatedAxis(self: *const TensorShardingAttr, index: usize) *const AxisRefAttr { + return @ptrCast(c.sdyTensorShardingAttrGetReplicatedAxesElem(self.ptr(), @intCast(index)).ptr); + } + + pub fn numUnreducedAxes(self: *const TensorShardingAttr) usize { + return @intCast(c.sdyTensorShardingAttrGetUnreducedAxesSize(self.ptr())); + } + + pub fn unreducedAxis(self: *const TensorShardingAttr, index: usize) *const AxisRefAttr { + return @ptrCast(c.sdyTensorShardingAttrGetUnreducedAxesElem(self.ptr(), @intCast(index)).ptr); + } +}; + +pub const TensorShardingPerValueAttr = opaque { + const M = mlir.Methods(TensorShardingPerValueAttr, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsATensorShardingPerValueAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn get(ctx: *mlir.Context, shardings: []const *const TensorShardingAttr) *const TensorShardingPerValueAttr { + const attrs: []const c.struct_MlirAttribute = @ptrCast(shardings); + return @ptrCast(c.sdyTensorShardingPerValueAttrGet(ctx.ptr(), @intCast(attrs.len), attrs.ptr).ptr); + } + + pub fn attribute(self: *const TensorShardingPerValueAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub const ManualAxesAttr = opaque { + const M = mlir.Methods(ManualAxesAttr, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsAManualAxesAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn get(ctx: *mlir.Context, axes: []const *const mlir.StringAttribute) *const ManualAxesAttr { + comptime std.debug.assert(@sizeOf(c.struct_MlirAttribute) == @sizeOf(*const mlir.StringAttribute)); + const attrs: []const c.struct_MlirAttribute = @ptrCast(axes); + return @ptrCast(c.sdyManualAxesAttrGet(ctx.ptr(), @intCast(attrs.len), attrs.ptr).ptr); + } + + pub fn attribute(self: *const ManualAxesAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn numAxes(self: *const ManualAxesAttr) usize { + return @intCast(c.sdyManualAxesAttrGetAxesSize(self.ptr())); + } + + pub fn axis(self: *const ManualAxesAttr, index: usize) []const u8 { + return mlir.string(c.sdyManualAxesAttrGetAxesElem(self.ptr(), @intCast(index))); + } +}; diff --git a/mlir/dialects/shardy/test.zig b/mlir/dialects/shardy/test.zig new file mode 100644 index 00000000..0ae161e7 --- /dev/null +++ b/mlir/dialects/shardy/test.zig @@ -0,0 +1,182 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const shardy = @import("mlir/dialects/shardy"); + +const ManualAxesAttr = shardy.ManualAxesAttr; +const TensorShardingAttr = shardy.TensorShardingAttr; +const TensorShardingPerValueAttr = shardy.TensorShardingPerValueAttr; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(shardy); + inline for (comptime std.meta.declarations(shardy)) |decl| { + const T = @field(shardy, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test TensorShardingAttr { + const registry: *mlir.DialectRegistry = try .init(); + defer registry.deinit(); + shardy.registerDialect(registry); + func.registerExtensions(registry); + + var ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + + var w: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer w.deinit(); + + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttr = .get(ctx, .{ + .mesh = "folded_mesh", + .dimensions = &.{.replicated(ctx)}, + .replicated_axes = &.{ .named(ctx, "link_x"), .named(ctx, "link_y") }, + }); + try sharding.attribute().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@folded_mesh, [{}], replicated={\"link_x\", \"link_y\"}>", w.written()); + } + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttr = .get(ctx, .{ + .mesh = "suggested_mesh", + .dimensions = &.{ + .closed(ctx, &.{.named(ctx, "link_z")}), + .closed(ctx, &.{.named(ctx, "link_x")}), + }, + .replicated_axes = &.{.named(ctx, "link_y")}, + }); + try sharding.attribute().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@suggested_mesh, [{\"link_z\"}, {\"link_x\"}], replicated={\"link_y\"}>", w.written()); + } + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttr = .get(ctx, .{ + .mesh = "fold_mesh", + .dimensions = &.{ .closed(ctx, &.{.named(ctx, "link_x")}), .replicated(ctx) }, + }); + try sharding.attribute().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@fold_mesh, [{\"link_x\"}, {}]>", w.written()); + } + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttr = .get(ctx, .{ + .mesh = "folded_mesh", + .dimensions = &.{.closed(ctx, &.{ .named(ctx, "link_x"), .named(ctx, "link_y") })}, + }); + try sharding.attribute().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@folded_mesh, [{\"link_x\", \"link_y\"}]>", w.written()); + } + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttr = .get(ctx, .{ + .mesh = "strategy_fold", + .dimensions = &.{.closed(ctx, &.{.named(ctx, "link_x")})}, + .replicated_axes = &.{.named(ctx, "link_y")}, + }); + try sharding.attribute().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@strategy_fold, [{\"link_x\"}], replicated={\"link_y\"}>", w.written()); + } + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttr = .get(ctx, .{ + .mesh = "mix_mesh", + .dimensions = &.{ .open(ctx, &.{}), .replicated(ctx) }, + .replicated_axes = &.{ .named(ctx, "link_x"), .named(ctx, "link_y") }, + }); + try sharding.attribute().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@mix_mesh, [{?}, {}], replicated={\"link_x\", \"link_y\"}>", w.written()); + } + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttr = .get(ctx, .{ + .mesh = "3d_mesh", + .dimensions = &.{ + .closed(ctx, &.{.named(ctx, "link_x")}), + .closed(ctx, &.{.named(ctx, "link_y")}), + .closed(ctx, &.{.named(ctx, "link_z")}), + }, + }); + try sharding.attribute().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@\"3d_mesh\", [{\"link_x\"}, {\"link_y\"}, {\"link_z\"}]>", w.written()); + } +} + +test TensorShardingPerValueAttr { + const registry: *mlir.DialectRegistry = try .init(); + defer registry.deinit(); + shardy.registerDialect(registry); + func.registerExtensions(registry); + + var ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + + const shardings: [2]*const TensorShardingAttr = .{ + .get(ctx, .{ + .mesh = "mesh", + .dimensions = &.{.closed(ctx, &.{.named(ctx, "link_x")})}, + }), + .get(ctx, .{ + .mesh = "mesh", + .dimensions = &.{.replicated(ctx)}, + .replicated_axes = &.{.named(ctx, "link_y")}, + }), + }; + + const per_value = TensorShardingPerValueAttr.get(ctx, &shardings); + + var w: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer w.deinit(); + + try per_value.attribute().format(&w.writer); + try std.testing.expectEqualSlices( + u8, + "#sdy.sharding_per_value<[<@mesh, [{\"link_x\"}]>, <@mesh, [{}], replicated={\"link_y\"}>]>", + w.written(), + ); +} + +test ManualAxesAttr { + const registry: *mlir.DialectRegistry = try .init(); + defer registry.deinit(); + shardy.registerDialect(registry); + func.registerExtensions(registry); + + var ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + + const axes: [2]*const mlir.StringAttribute = .{ + .init(ctx, "link_x"), + .init(ctx, "link_y"), + }; + + const manual_axes: *const ManualAxesAttr = .get(ctx, &axes); + + try std.testing.expectEqual(2, manual_axes.numAxes()); + try std.testing.expectEqualStrings("link_x", manual_axes.axis(0)); + try std.testing.expectEqualStrings("link_y", manual_axes.axis(1)); + + var w: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer w.deinit(); + + try manual_axes.attribute().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy", w.written()); +} + +test "sub-axis references build" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + shardy.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + const ref = shardy.AxisRefAttr.subAxis(ctx, "x", 2, 4); + try std.testing.expectEqualStrings("x", ref.name()); + try std.testing.expectEqual(@as(i64, 4), ref.subAxisInfo().?.size()); +} diff --git a/mlir/dialects/stablehlo/BUILD.bazel b/mlir/dialects/stablehlo/BUILD.bazel index 338cce95..d7214000 100644 --- a/mlir/dialects/stablehlo/BUILD.bazel +++ b/mlir/dialects/stablehlo/BUILD.bazel @@ -2,13 +2,12 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "stablehlo", - tags = ["manual"], import_name = "mlir/dialects/stablehlo", main = "stablehlo.zig", - visibility = ["//mlir/dialects:__subpackages__"], + tags = ["manual"], + visibility = ["//visibility:public"], deps = [ "//mlir", - "//mlir:c", "//stdx", "@stablehlo//:stablehlo_dialect_capi", ], @@ -16,5 +15,9 @@ zig_library( zig_test( name = "test", - deps = [":stablehlo"], + main = "test.zig", + deps = [ + ":stablehlo", + "//mlir", + ], ) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index f9ea1e7e..5d845934 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -5,6 +5,10 @@ const mlir = @import("mlir"); const stringRef = mlir.stringRef; const stdx = @import("stdx"); +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("stablehlo"); +} + pub const abs = unaryOp("stablehlo.abs").call; pub const cosine = unaryOp("stablehlo.cosine").call; pub const sine = unaryOp("stablehlo.sine").call; @@ -141,7 +145,7 @@ pub const DotPrecision = union(enum) { highest, algorithm: DotAlgorithm, - pub fn precision(self: DotPrecision) PrecisionAttribute.Precision { + pub fn precision(self: DotPrecision) PrecisionAttr.Precision { return switch (self) { .fast => .DEFAULT, .high => .HIGH, @@ -151,9 +155,9 @@ pub const DotPrecision = union(enum) { }; } - pub fn algorithmAttr(self: DotPrecision, ctx: *mlir.Context, operand_type: mlir.RankedTensorType) ?mlir.Attribute { + pub fn algorithmAttr(self: DotPrecision, ctx: *mlir.Context, element_type: *const mlir.Type) ?*const mlir.Attribute { return switch (self) { - .algorithm => |algo| algo.asAttr(ctx, operand_type), + .algorithm => |algo| dotAlgorithmAttr(ctx, algo, element_type), else => null, }; } @@ -177,7 +181,7 @@ pub const DotAlgorithm = struct { }; }; -pub fn dotAlgorithmAttribute(ctx: *mlir.Context, dot_algorithm: DotAlgorithm, element_type: *const mlir.Type) *const mlir.Attribute { +pub fn dotAlgorithmAttr(ctx: *mlir.Context, dot_algorithm: DotAlgorithm, element_type: *const mlir.Type) *const mlir.Attribute { return @ptrCast(c.stablehloDotAlgorithmGet( ctx.ptr(), element_type.ptr(), @@ -206,12 +210,12 @@ pub fn dot_general( }, location: *const mlir.Location, ) *mlir.Operation { - const precisions: [2]*const mlir.Attribute = @splat(precisionAttribute(ctx, opts.dot_precision.precision())); + const precisions: [2]*const mlir.Attribute = @splat(precisionAttr(ctx, opts.dot_precision.precision())); const attributes = [3]mlir.NamedAttribute{ .named( ctx, "dot_dimension_numbers", - dotDimensionNumbersAttribute(ctx, .{ + dotDimensionNumbersAttr(ctx, .{ .lhs_batching_dimensions = opts.lhs_batching_dimensions, .rhs_batching_dimensions = opts.rhs_batching_dimensions, .lhs_contracting_dimensions = opts.lhs_contracting_dimensions, @@ -221,7 +225,7 @@ pub fn dot_general( .named(ctx, "precision_config", .array(ctx, &precisions)), // keep algorithm as the last attribute so we can omit it when it's not set. .named(ctx, "algorithm", switch (opts.dot_precision) { - .algorithm => |v| dotAlgorithmAttribute(ctx, v, lhs.type_()), + .algorithm => |v| dotAlgorithmAttr(ctx, v, lhs.type_()), else => undefined, }), }; @@ -354,7 +358,7 @@ pub fn gather( .operands = .{ .flat = &.{ value, indices } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "dimension_numbers", gatherDimensionNumbersAttribute( + .named(ctx, "dimension_numbers", gatherDimensionNumbersAttr( ctx, args.offset_dims, args.collapsed_slice_dims, @@ -381,7 +385,7 @@ pub const ScatterArgs = struct { indices_are_sorted: bool = false, unique_indices: bool = false, - pub fn getScatterDimensionNumbers(self: ScatterArgs, ctx: *mlir.Context) *const mlir.Attribute { + pub fn scatterDimensionNumbers(self: ScatterArgs, ctx: *mlir.Context) *const mlir.Attribute { return @ptrCast(c.stablehloScatterDimensionNumbersGet( ctx.ptr(), @intCast(self.update_window_dims.len), @@ -415,7 +419,7 @@ pub fn scatter( .operands = .{ .variadic = &.{ inputs, scatter_indices, updates } }, .blocks = &.{update_block}, .attributes = &.{ - .named(ctx, "scatter_dimension_numbers", args.getScatterDimensionNumbers(ctx)), + .named(ctx, "scatter_dimension_numbers", args.scatterDimensionNumbers(ctx)), .named(ctx, "indices_are_sorted", .boolean(ctx, args.indices_are_sorted)), .named(ctx, "unique_indices", .boolean(ctx, args.unique_indices)), }, @@ -451,16 +455,16 @@ pub fn compare( ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, - comparison_direction: ComparisonDirection.Direction, - compare_type: CompareType.Type, + comparison_direction: ComparisonDirectionAttr.Direction, + compare_type: CompareTypeAttr.Type, location: *const mlir.Location, ) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.compare", .{ .operands = .{ .flat = &.{ lhs, rhs } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "comparison_direction", comparisonDirection(ctx, comparison_direction)), - .named(ctx, "compare_type", compareType(ctx, compare_type)), + .named(ctx, "comparison_direction", comparisonDirectionAttr(ctx, comparison_direction)), + .named(ctx, "compare_type", compareTypeAttr(ctx, compare_type)), }, .location = location, }); @@ -489,7 +493,7 @@ pub const PadOpts = struct { interior: []const i64, }; -pub fn pad(ctx: *mlir.Context, value: *const mlir.Value, padding_value: *const mlir.Value, location: *const mlir.Location, opts: PadOpts) *mlir.Operation { +pub fn pad(ctx: *mlir.Context, value: *const mlir.Value, padding_value: *const mlir.Value, opts: PadOpts, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.pad", .{ .operands = .{ .flat = &.{ value, padding_value } }, .result_type_inference = true, @@ -506,7 +510,7 @@ pub const TriangularSolveOpts = struct { left_side: bool, lower: bool, unit_diagonal: bool, - transpose_a: TransposeAttribute.Type, + transpose_a: TransposeAttr.Type, }; pub fn triangular_solve(ctx: *mlir.Context, value: *const mlir.Value, other: *const mlir.Value, opts: TriangularSolveOpts, location: *const mlir.Location) *mlir.Operation { @@ -517,14 +521,14 @@ pub fn triangular_solve(ctx: *mlir.Context, value: *const mlir.Value, other: *co .named(ctx, "left_side", .int(ctx, .i1, @intFromBool(opts.left_side))), .named(ctx, "lower", .int(ctx, .i1, @intFromBool(opts.lower))), .named(ctx, "unit_diagonal", .int(ctx, .i1, @intFromBool(opts.unit_diagonal))), - .named(ctx, "transpose_a", transposeAttribute(ctx, opts.transpose_a)), + .named(ctx, "transpose_a", transposeAttr(ctx, opts.transpose_a)), }, .location = location, }); } pub const FftOpts = struct { - kind: FftType.Type, + kind: FftTypeAttr.Type, length: []const i64, }; @@ -533,15 +537,15 @@ pub fn fft(ctx: *mlir.Context, value: *const mlir.Value, opts: FftOpts, location .operands = .{ .flat = &.{value} }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "fft_type", fftType(ctx, opts.kind)), + .named(ctx, "fft_type", fftTypeAttr(ctx, opts.kind)), .named(ctx, "fft_length", .denseArray(ctx, .i64, opts.length)), }, .location = location, }); } -pub const RngDistribution = opaque { - const M = mlir.Methods(RngDistribution, c.MlirAttribute); +pub const RngDistributionAttr = opaque { + const M = mlir.Methods(RngDistributionAttr, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsARngDistributionAttr; pub const ptr = M.ptr; @@ -553,24 +557,32 @@ pub const RngDistribution = opaque { NORMAL, }; - pub fn get(ctx: *mlir.Context, value_: Type) *const RngDistribution { - return @ptrCast(c.stablehloRngDistributionAttrGet(ctx.ptr(), stringRef(@tagName(value_))).ptr); + pub const InitArgs = struct { + value: Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const RngDistributionAttr { + return @ptrCast(c.stablehloRngDistributionAttrGet(ctx.ptr(), stringRef(@tagName(args.value))).ptr); } - pub fn value(self: *const RngDistribution) Type { + pub fn value(self: *const RngDistributionAttr) Type { return std.meta.stringToEnum( Type, mlir.string(c.stablehloRngDistributionAttrGetValue(self.ptr())), ) orelse unreachable; } + + pub fn attribute(self: *const RngDistributionAttr) *const mlir.Attribute { + return @ptrCast(self); + } }; -pub fn rngDistribution(ctx: *mlir.Context, value: RngDistribution.Type) *const mlir.Attribute { - return @ptrCast(RngDistribution.get(ctx, value)); +pub fn rngDistributionAttr(ctx: *mlir.Context, value: RngDistributionAttr.Type) *const mlir.Attribute { + return RngDistributionAttr.get(ctx, .{ .value = value }).attribute(); } -pub const RngAlgorithm = opaque { - const M = mlir.Methods(RngAlgorithm, c.MlirAttribute); +pub const RngAlgorithmAttr = opaque { + const M = mlir.Methods(RngAlgorithmAttr, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsARngAlgorithmAttr; pub const ptr = M.ptr; @@ -583,28 +595,36 @@ pub const RngAlgorithm = opaque { PHILOX, }; - pub fn get(ctx: *mlir.Context, value_: Type) *const RngAlgorithm { - return @ptrCast(c.stablehloRngAlgorithmAttrGet(ctx.ptr(), stringRef(@tagName(value_))).ptr); + pub const InitArgs = struct { + value: Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const RngAlgorithmAttr { + return @ptrCast(c.stablehloRngAlgorithmAttrGet(ctx.ptr(), stringRef(@tagName(args.value))).ptr); } - pub fn value(self: *const RngAlgorithm) Type { + pub fn value(self: *const RngAlgorithmAttr) Type { return std.meta.stringToEnum( Type, mlir.string(c.stablehloRngAlgorithmAttrGetValue(self.ptr())), ) orelse unreachable; } + + pub fn attribute(self: *const RngAlgorithmAttr) *const mlir.Attribute { + return @ptrCast(self); + } }; -pub fn rngAlgorithm(ctx: *mlir.Context, value: RngAlgorithm.Type) *const mlir.Attribute { - return @ptrCast(RngAlgorithm.get(ctx, value)); +pub fn rngAlgorithmAttr(ctx: *mlir.Context, value: RngAlgorithmAttr.Type) *const mlir.Attribute { + return RngAlgorithmAttr.get(ctx, .{ .value = value }).attribute(); } -pub fn rng(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, shape: *const mlir.Value, rng_distribution: RngDistribution.Type, location: *const mlir.Location) *mlir.Operation { +pub fn rng(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, shape: *const mlir.Value, rng_distribution: RngDistributionAttr.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.rng", .{ .operands = .{ .flat = &.{ a, b, shape } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "rng_distribution", rngDistribution(ctx, rng_distribution)), + .named(ctx, "rng_distribution", rngDistributionAttr(ctx, rng_distribution)), }, .location = location, }); @@ -612,7 +632,7 @@ pub fn rng(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, shape pub fn rng_bit_generator( ctx: *mlir.Context, - rng_algorithm: RngAlgorithm.Type, + rng_algorithm: RngAlgorithmAttr.Type, initial_state: *const mlir.Value, res_state_type: *const mlir.Type, res_type: *const mlir.Type, @@ -622,7 +642,7 @@ pub fn rng_bit_generator( .operands = .{ .flat = &.{initial_state} }, .results = .{ .flat = &.{ res_state_type, res_type } }, .attributes = &.{ - .named(ctx, "rng_algorithm", rngAlgorithm(ctx, rng_algorithm)), + .named(ctx, "rng_algorithm", rngAlgorithmAttr(ctx, rng_algorithm)), }, .location = location, }); @@ -699,7 +719,7 @@ pub const ConvolutionOpts = struct { output_spatial_dimensions: []const i64, feature_group_count: i64, batch_group_count: i64, - precision_config: []const PrecisionAttribute.Precision = &.{}, + precision_config: []const PrecisionAttr.Precision = &.{}, }; pub fn convolution( @@ -712,7 +732,7 @@ pub fn convolution( ) *mlir.Operation { var max_precisions: [2]*const mlir.Attribute = undefined; for (opts.precision_config, 0..) |p, i| { - max_precisions[i] = precisionAttribute(ctx, p); + max_precisions[i] = precisionAttr(ctx, p); } var window_reversal: [3]i32 = undefined; for (opts.window_reversal, 0..) |w, i| { @@ -727,7 +747,7 @@ pub fn convolution( .named(ctx, "lhs_dilation", .denseArray(ctx, .i64, opts.lhs_dilation)), .named(ctx, "rhs_dilation", .denseArray(ctx, .i64, opts.rhs_dilation)), .named(ctx, "window_reversal", .denseArray(ctx, .bool, window_reversal[0..opts.window_reversal.len])), - .named(ctx, "dimension_numbers", convDimensionNumbersAttribute(ctx, .{ + .named(ctx, "dimension_numbers", convDimensionNumbersAttr(ctx, .{ .input_batch_dimension = opts.input_batch_dimension, .input_feature_dimension = opts.input_feature_dimension, .input_spatial_dimensions = opts.input_spatial_dimensions, @@ -822,7 +842,7 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result for (output_operand_aliases) |alias| { const output_tuple_indices = if (result_types.len > 1) &[1]i64{alias.output_index} else &.{}; buffer.appendAssumeCapacity( - outputOperandAliasAttribute(ctx, .{ .operand_index = alias.operand_index, .output_tuple_indices = output_tuple_indices }), + outputOperandAliasAttr(ctx, .{ .operand_index = alias.operand_index, .output_tuple_indices = output_tuple_indices }), ); } attrs.appendAssumeCapacity( @@ -938,8 +958,8 @@ pub fn partition_id(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Op }); } -pub const DotDimensionNumbersAttribute = opaque { - const M = mlir.Methods(DotDimensionNumbersAttribute, mlir.Attribute); +pub const DotDimensionNumbersAttr = opaque { + const M = mlir.Methods(DotDimensionNumbersAttr, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsADotDimensionNumbers; pub const ptr = M.ptr; @@ -953,7 +973,7 @@ pub const DotDimensionNumbersAttribute = opaque { rhs_contracting_dimensions: []const i64, }; - pub fn init(ctx: *mlir.Context, args: InitArgs) *const DotDimensionNumbersAttribute { + pub fn get(ctx: *mlir.Context, args: InitArgs) *const DotDimensionNumbersAttr { return @ptrCast(c.stablehloDotDimensionNumbersGet( ctx.ptr(), @intCast(args.lhs_batching_dimensions.len), @@ -967,52 +987,56 @@ pub const DotDimensionNumbersAttribute = opaque { ).ptr); } - pub fn getLhsBatchingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { + pub fn lhsBatchingDimensionsSize(self: *const DotDimensionNumbersAttr) usize { return @intCast(c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsSize(self.ptr())); } - pub fn getLhsBatchingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { + pub fn lhsBatchingDimensionsElem(self: *const DotDimensionNumbersAttr, pos: usize) i64 { return c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn getRhsBatchingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { + pub fn rhsBatchingDimensionsSize(self: *const DotDimensionNumbersAttr) usize { return @intCast(c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsSize(self.ptr())); } - pub fn getRhsBatchingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { + pub fn rhsBatchingDimensionsElem(self: *const DotDimensionNumbersAttr, pos: usize) i64 { return c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn getLhsContractingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { + pub fn lhsContractingDimensionsSize(self: *const DotDimensionNumbersAttr) usize { return @intCast(c.stablehloDotDimensionNumbersGetLhsContractingDimensionsSize(self.ptr())); } - pub fn getLhsContractingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { + pub fn lhsContractingDimensionsElem(self: *const DotDimensionNumbersAttr, pos: usize) i64 { return c.stablehloDotDimensionNumbersGetLhsContractingDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn getRhsContractingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { + pub fn rhsContractingDimensionsSize(self: *const DotDimensionNumbersAttr) usize { return @intCast(c.stablehloDotDimensionNumbersGetRhsContractingDimensionsSize(self.ptr())); } - pub fn getRhsContractingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { + pub fn rhsContractingDimensionsElem(self: *const DotDimensionNumbersAttr, pos: usize) i64 { return c.stablehloDotDimensionNumbersGetRhsContractingDimensionsElem(self.ptr(), @intCast(pos)); } + + pub fn attribute(self: *const DotDimensionNumbersAttr) *const mlir.Attribute { + return @ptrCast(self); + } }; -pub fn dotDimensionNumbersAttribute(ctx: *mlir.Context, args: DotDimensionNumbersAttribute.InitArgs) *const mlir.Attribute { - return @ptrCast(DotDimensionNumbersAttribute.init(ctx, args)); +pub fn dotDimensionNumbersAttr(ctx: *mlir.Context, args: DotDimensionNumbersAttr.InitArgs) *const mlir.Attribute { + return @ptrCast(DotDimensionNumbersAttr.get(ctx, args)); } -pub const GatherDimensionNumbersAttribute = opaque { - const M = mlir.Methods(GatherDimensionNumbersAttribute, mlir.Attribute); +pub const GatherDimensionNumbersAttr = opaque { + const M = mlir.Methods(GatherDimensionNumbersAttr, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsAGatherDimensionNumbers; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.eql(c.mlirAttributePrint); + pub const format = M.format(c.mlirAttributePrint); - pub fn init( + pub fn get( ctx: *mlir.Context, offset_dims: []const i64, collapsed_slice_dims: []const i64, @@ -1020,7 +1044,7 @@ pub const GatherDimensionNumbersAttribute = opaque { start_indices_batching_dims: []const i64, start_index_map: []const i64, index_vector_dim: i64, - ) *const GatherDimensionNumbersAttribute { + ) *const GatherDimensionNumbersAttr { return @ptrCast(c.stablehloGatherDimensionNumbersGet( ctx.ptr(), @intCast(offset_dims.len), @@ -1037,52 +1061,52 @@ pub const GatherDimensionNumbersAttribute = opaque { ).ptr); } - pub fn getOffsetDimsSize(self: *const GatherDimensionNumbersAttribute) usize { + pub fn offsetDimsSize(self: *const GatherDimensionNumbersAttr) usize { return @intCast(c.stablehloGatherDimensionNumbersGetOffsetDimsSize(self.ptr())); } - pub fn getOffsetDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + pub fn offsetDimsElem(self: *const GatherDimensionNumbersAttr, pos: usize) i64 { return c.stablehloGatherDimensionNumbersGetOffsetDimsElem(self.ptr(), @intCast(pos)); } - pub fn getCollapsedSliceDimsSize(self: *const GatherDimensionNumbersAttribute) usize { + pub fn collapsedSliceDimsSize(self: *const GatherDimensionNumbersAttr) usize { return @intCast(c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsSize(self.ptr())); } - pub fn getCollapsedSliceDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + pub fn collapsedSliceDimsElem(self: *const GatherDimensionNumbersAttr, pos: usize) i64 { return c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsElem(self.ptr(), @intCast(pos)); } - pub fn getStartIndexMapSize(self: *const GatherDimensionNumbersAttribute) usize { + pub fn startIndexMapSize(self: *const GatherDimensionNumbersAttr) usize { return @intCast(c.stablehloGatherDimensionNumbersGetStartIndexMapSize(self.ptr())); } - pub fn getOperandBatchingDimsSize(self: *const GatherDimensionNumbersAttribute) usize { + pub fn operandBatchingDimsSize(self: *const GatherDimensionNumbersAttr) usize { return @intCast(c.stablehloGatherDimensionNumbersGetOperandBatchingDimsSize(self.ptr())); } - pub fn getOperandBatchingDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + pub fn operandBatchingDimsElem(self: *const GatherDimensionNumbersAttr, pos: usize) i64 { return c.stablehloGatherDimensionNumbersGetOperandBatchingDimsElem(self.ptr(), @intCast(pos)); } - pub fn getStartIndicesBatchingDimsSize(self: *const GatherDimensionNumbersAttribute) usize { + pub fn startIndicesBatchingDimsSize(self: *const GatherDimensionNumbersAttr) usize { return @intCast(c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsSize(self.ptr())); } - pub fn getStartIndicesBatchingDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + pub fn startIndicesBatchingDimsElem(self: *const GatherDimensionNumbersAttr, pos: usize) i64 { return c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsElem(self.ptr(), @intCast(pos)); } - pub fn getStartIndexMapElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + pub fn startIndexMapElem(self: *const GatherDimensionNumbersAttr, pos: usize) i64 { return c.stablehloGatherDimensionNumbersGetStartIndexMapElem(self.ptr(), @intCast(pos)); } - pub fn getIndexVectorDim(self: *const GatherDimensionNumbersAttribute) usize { + pub fn indexVectorDim(self: *const GatherDimensionNumbersAttr) usize { return @intCast(c.stablehloGatherDimensionNumbersGetIndexVectorDim(self.ptr())); } }; -pub fn gatherDimensionNumbersAttribute( +pub fn gatherDimensionNumbersAttr( ctx: *mlir.Context, offset_dims: []const i64, collapsed_slice_dims: []const i64, @@ -1091,7 +1115,7 @@ pub fn gatherDimensionNumbersAttribute( start_index_map: []const i64, index_vector_dim: i64, ) *const mlir.Attribute { - return @ptrCast(GatherDimensionNumbersAttribute.init( + return @ptrCast(GatherDimensionNumbersAttr.get( ctx, offset_dims, collapsed_slice_dims, @@ -1102,8 +1126,8 @@ pub fn gatherDimensionNumbersAttribute( )); } -pub const ConvDimensionNumbersAttribute = opaque { - const M = mlir.Methods(ConvDimensionNumbersAttribute, mlir.Attribute); +pub const ConvDimensionNumbersAttr = opaque { + const M = mlir.Methods(ConvDimensionNumbersAttr, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsAConvDimensionNumbers; pub const ptr = M.ptr; @@ -1122,7 +1146,7 @@ pub const ConvDimensionNumbersAttribute = opaque { output_spatial_dimensions: []const i64, }; - pub fn init(ctx: *mlir.Context, args: InitArgs) *const ConvDimensionNumbersAttribute { + pub fn get(ctx: *mlir.Context, args: InitArgs) *const ConvDimensionNumbersAttr { return @ptrCast(c.stablehloConvDimensionNumbersGet( ctx.ptr(), args.input_batch_dimension, @@ -1140,61 +1164,65 @@ pub const ConvDimensionNumbersAttribute = opaque { ).ptr); } - pub fn getInputBatchDimension(self: *const ConvDimensionNumbersAttribute) i64 { + pub fn inputBatchDimension(self: *const ConvDimensionNumbersAttr) i64 { return c.stablehloConvDimensionNumbersGetInputBatchDimension(self.ptr()); } - pub fn getInputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { + pub fn inputFeatureDimension(self: *const ConvDimensionNumbersAttr) i64 { return c.stablehloConvDimensionNumbersGetInputFeatureDimension(self.ptr()); } - pub fn getInputSpatialDimensionsSize(self: *const ConvDimensionNumbersAttribute) usize { + pub fn inputSpatialDimensionsSize(self: *const ConvDimensionNumbersAttr) usize { return @intCast(c.stablehloConvDimensionNumbersGetInputSpatialDimensionsSize(self.ptr())); } - pub fn getInputSpatialDimensionsElem(self: *const ConvDimensionNumbersAttribute, pos: usize) i64 { + pub fn inputSpatialDimensionsElem(self: *const ConvDimensionNumbersAttr, pos: usize) i64 { return c.stablehloConvDimensionNumbersGetInputSpatialDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn getKernelInputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { + pub fn kernelInputFeatureDimension(self: *const ConvDimensionNumbersAttr) i64 { return c.stablehloConvDimensionNumbersGetKernelInputFeatureDimension(self.ptr()); } - pub fn getKernelOutputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { + pub fn kernelOutputFeatureDimension(self: *const ConvDimensionNumbersAttr) i64 { return c.stablehloConvDimensionNumbersGetKernelOutputFeatureDimension(self.ptr()); } - pub fn getKernelSpatialDimensionsSize(self: *const ConvDimensionNumbersAttribute) usize { + pub fn kernelSpatialDimensionsSize(self: *const ConvDimensionNumbersAttr) usize { return @intCast(c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsSize(self.ptr())); } - pub fn getKernelSpatialDimensionsElem(self: *const ConvDimensionNumbersAttribute, pos: usize) i64 { + pub fn kernelSpatialDimensionsElem(self: *const ConvDimensionNumbersAttr, pos: usize) i64 { return c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn getOutputBatchDimension(self: *const ConvDimensionNumbersAttribute) i64 { + pub fn outputBatchDimension(self: *const ConvDimensionNumbersAttr) i64 { return c.stablehloConvDimensionNumbersGetOutputBatchDimension(self.ptr()); } - pub fn getOutputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { + pub fn outputFeatureDimension(self: *const ConvDimensionNumbersAttr) i64 { return c.stablehloConvDimensionNumbersGetOutputFeatureDimension(self.ptr()); } - pub fn getOutputSpatialDimensionsSize(self: *const ConvDimensionNumbersAttribute) usize { + pub fn outputSpatialDimensionsSize(self: *const ConvDimensionNumbersAttr) usize { return @intCast(c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsSize(self.ptr())); } - pub fn getOutputSpatialDimensionsElem(self: *const ConvDimensionNumbersAttribute, pos: usize) i64 { + pub fn outputSpatialDimensionsElem(self: *const ConvDimensionNumbersAttr, pos: usize) i64 { return c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsElem(self.ptr(), @intCast(pos)); } + + pub fn attribute(self: *const ConvDimensionNumbersAttr) *const mlir.Attribute { + return @ptrCast(self); + } }; -pub fn convDimensionNumbersAttribute(ctx: *mlir.Context, args: ConvDimensionNumbersAttribute.InitArgs) *const mlir.Attribute { - return @ptrCast(ConvDimensionNumbersAttribute.init(ctx, args)); +pub fn convDimensionNumbersAttr(ctx: *mlir.Context, args: ConvDimensionNumbersAttr.InitArgs) *const mlir.Attribute { + return @ptrCast(ConvDimensionNumbersAttr.get(ctx, args)); } -pub const OutputOperandAliasAttribute = opaque { - const M = mlir.Methods(OutputOperandAliasAttribute, c.MlirAttribute); +pub const OutputOperandAliasAttr = opaque { + const M = mlir.Methods(OutputOperandAliasAttr, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsAOutputOperandAlias; pub const ptr = M.ptr; @@ -1207,7 +1235,7 @@ pub const OutputOperandAliasAttribute = opaque { operand_tuple_indices: []const i64 = &.{}, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) *const OutputOperandAliasAttribute { + pub fn get(ctx: *mlir.Context, args: InitArgs) *const OutputOperandAliasAttr { return @ptrCast(c.stablehloOutputOperandAliasGet( ctx.ptr(), @intCast(args.output_tuple_indices.len), @@ -1217,14 +1245,18 @@ pub const OutputOperandAliasAttribute = opaque { @ptrCast(args.operand_tuple_indices), ).ptr); } + + pub fn attribute(self: *const OutputOperandAliasAttr) *const mlir.Attribute { + return @ptrCast(self); + } }; -pub fn outputOperandAliasAttribute(ctx: *mlir.Context, args: OutputOperandAliasAttribute.InitArgs) *const mlir.Attribute { - return @ptrCast(OutputOperandAliasAttribute.get(ctx, args)); +pub fn outputOperandAliasAttr(ctx: *mlir.Context, args: OutputOperandAliasAttr.InitArgs) *const mlir.Attribute { + return @ptrCast(OutputOperandAliasAttr.get(ctx, args)); } -pub const PrecisionAttribute = opaque { - const M = mlir.Methods(PrecisionAttribute, mlir.Attribute); +pub const PrecisionAttr = opaque { + const M = mlir.Methods(PrecisionAttr, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsAPrecisionAttr; pub const ptr = M.ptr; @@ -1237,22 +1269,30 @@ pub const PrecisionAttribute = opaque { HIGHEST, }; - pub fn init(ctx: *mlir.Context, value: Precision) *const PrecisionAttribute { - return @ptrCast(c.stablehloPrecisionAttrGet(ctx.ptr(), stringRef(@tagName(value))).ptr); + pub const InitArgs = struct { + value: Precision, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const PrecisionAttr { + return @ptrCast(c.stablehloPrecisionAttrGet(ctx.ptr(), stringRef(@tagName(args.value))).ptr); } - pub fn getValue(self: *const PrecisionAttribute) Precision { - const value = mlir.string(c.stablehloPrecisionAttrGetValue(self.ptr())); - return std.meta.stringToEnum(Precision, value) orelse unreachable; + pub fn value(self: *const PrecisionAttr) Precision { + const name = mlir.string(c.stablehloPrecisionAttrGetValue(self.ptr())); + return std.meta.stringToEnum(Precision, name) orelse unreachable; + } + + pub fn attribute(self: *const PrecisionAttr) *const mlir.Attribute { + return @ptrCast(self); } }; -pub fn precisionAttribute(ctx: *mlir.Context, precision: PrecisionAttribute.Precision) *const mlir.Attribute { - return @ptrCast(PrecisionAttribute.init(ctx, precision)); +pub fn precisionAttr(ctx: *mlir.Context, precision: PrecisionAttr.Precision) *const mlir.Attribute { + return PrecisionAttr.get(ctx, .{ .value = precision }).attribute(); } -pub const ComparisonDirection = opaque { - const M = mlir.Methods(ComparisonDirection, c.MlirAttribute); +pub const ComparisonDirectionAttr = opaque { + const M = mlir.Methods(ComparisonDirectionAttr, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsAComparisonDirectionAttr; pub const ptr = M.ptr; @@ -1268,22 +1308,30 @@ pub const ComparisonDirection = opaque { LT, }; - pub fn init(ctx: *mlir.Context, value: Direction) *const ComparisonDirection { - return @ptrCast(c.stablehloComparisonDirectionAttrGet(ctx.ptr(), mlir.stringRef(@tagName(value))).ptr); + pub const InitArgs = struct { + value: Direction, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const ComparisonDirectionAttr { + return @ptrCast(c.stablehloComparisonDirectionAttrGet(ctx.ptr(), mlir.stringRef(@tagName(args.value))).ptr); } - pub fn getValue(self: *const ComparisonDirection) Direction { - const value = mlir.string(c.stablehloComparisonDirectionAttrGetValue(self.ptr())); - return std.meta.stringToEnum(Direction, value) orelse unreachable; + pub fn value(self: *const ComparisonDirectionAttr) Direction { + const name = mlir.string(c.stablehloComparisonDirectionAttrGetValue(self.ptr())); + return std.meta.stringToEnum(Direction, name) orelse unreachable; + } + + pub fn attribute(self: *const ComparisonDirectionAttr) *const mlir.Attribute { + return @ptrCast(self); } }; -pub fn comparisonDirection(ctx: *mlir.Context, value: ComparisonDirection.Direction) *const mlir.Attribute { - return @ptrCast(ComparisonDirection.init(ctx, value)); +pub fn comparisonDirectionAttr(ctx: *mlir.Context, value: ComparisonDirectionAttr.Direction) *const mlir.Attribute { + return ComparisonDirectionAttr.get(ctx, .{ .value = value }).attribute(); } -pub const CompareType = opaque { - const M = mlir.Methods(CompareType, c.MlirAttribute); +pub const CompareTypeAttr = opaque { + const M = mlir.Methods(CompareTypeAttr, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsAComparisonTypeAttr; pub const ptr = M.ptr; @@ -1297,22 +1345,30 @@ pub const CompareType = opaque { TOTALORDER, }; - pub fn init(ctx: *mlir.Context, value: Type) *const CompareType { - return @ptrCast(c.stablehloComparisonTypeAttrGet(ctx.ptr(), stringRef(@tagName(value))).ptr); + pub const InitArgs = struct { + value: Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const CompareTypeAttr { + return @ptrCast(c.stablehloComparisonTypeAttrGet(ctx.ptr(), stringRef(@tagName(args.value))).ptr); } - pub fn getValue(self: *const CompareType) Type { - const value = mlir.string(c.stablehloComparisonTypeAttrGetValue(self.ptr())); - return std.meta.stringToEnum(Type, value) orelse unreachable; + pub fn value(self: *const CompareTypeAttr) Type { + const name = mlir.string(c.stablehloComparisonTypeAttrGetValue(self.ptr())); + return std.meta.stringToEnum(Type, name) orelse unreachable; + } + + pub fn attribute(self: *const CompareTypeAttr) *const mlir.Attribute { + return @ptrCast(self); } }; -pub fn compareType(ctx: *mlir.Context, value: CompareType.Type) *const mlir.Attribute { - return @ptrCast(CompareType.init(ctx, value)); +pub fn compareTypeAttr(ctx: *mlir.Context, value: CompareTypeAttr.Type) *const mlir.Attribute { + return CompareTypeAttr.get(ctx, .{ .value = value }).attribute(); } -pub const TransposeAttribute = opaque { - const M = mlir.Methods(TransposeAttribute, mlir.Attribute); +pub const TransposeAttr = opaque { + const M = mlir.Methods(TransposeAttr, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsATransposeAttr; pub const ptr = M.ptr; @@ -1325,22 +1381,30 @@ pub const TransposeAttribute = opaque { ADJOINT, }; - pub fn init(ctx: *mlir.Context, value: Type) *const TransposeAttribute { - return @ptrCast(c.stablehloTransposeAttrGet(ctx.ptr(), mlir.stringRef(@tagName(value))).ptr); + pub const InitArgs = struct { + value: Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const TransposeAttr { + return @ptrCast(c.stablehloTransposeAttrGet(ctx.ptr(), mlir.stringRef(@tagName(args.value))).ptr); } - pub fn getValue(self: *const TransposeAttribute) Type { - const value = mlir.string(c.stablehloTransposeAttrGetValue(self.ptr())); - return std.meta.stringToEnum(Type, value) orelse unreachable; + pub fn value(self: *const TransposeAttr) Type { + const name = mlir.string(c.stablehloTransposeAttrGetValue(self.ptr())); + return std.meta.stringToEnum(Type, name) orelse unreachable; + } + + pub fn attribute(self: *const TransposeAttr) *const mlir.Attribute { + return @ptrCast(self); } }; -pub fn transposeAttribute(ctx: *mlir.Context, value: TransposeAttribute.Type) *const mlir.Attribute { - return @ptrCast(TransposeAttribute.init(ctx, value)); +pub fn transposeAttr(ctx: *mlir.Context, value: TransposeAttr.Type) *const mlir.Attribute { + return TransposeAttr.get(ctx, .{ .value = value }).attribute(); } -pub const FftType = opaque { - const M = mlir.Methods(FftType, mlir.Attribute); +pub const FftTypeAttr = opaque { + const M = mlir.Methods(FftTypeAttr, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsAFftTypeAttr; pub const ptr = M.ptr; @@ -1354,18 +1418,26 @@ pub const FftType = opaque { IRFFT, }; - pub fn init(ctx: *mlir.Context, value: Type) *const FftType { - return @ptrCast(c.stablehloFftTypeAttrGet(ctx.ptr(), mlir.stringRef(@tagName(value))).ptr); + pub const InitArgs = struct { + value: Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const FftTypeAttr { + return @ptrCast(c.stablehloFftTypeAttrGet(ctx.ptr(), mlir.stringRef(@tagName(args.value))).ptr); } - pub fn getValue(self: *const FftType) Type { - const value = mlir.string(c.stablehloFftTypeAttrGetValue(self.ptr())); - return std.meta.stringToEnum(Type, value) orelse unreachable; + pub fn value(self: *const FftTypeAttr) Type { + const name = mlir.string(c.stablehloFftTypeAttrGetValue(self.ptr())); + return std.meta.stringToEnum(Type, name) orelse unreachable; + } + + pub fn attribute(self: *const FftTypeAttr) *const mlir.Attribute { + return @ptrCast(self); } }; -pub fn fftType(ctx: *mlir.Context, value: FftType.Type) *const mlir.Attribute { - return @ptrCast(FftType.init(ctx, value)); +pub fn fftTypeAttr(ctx: *mlir.Context, value: FftTypeAttr.Type) *const mlir.Attribute { + return FftTypeAttr.get(ctx, .{ .value = value }).attribute(); } fn stringFromStream(buf: []u8, streamFn: anytype, args: anytype) []const u8 { @@ -1455,7 +1527,7 @@ pub const ChannelType = enum(i64) { send_recv = 1, }; -pub fn channelHandle( +pub fn channelHandleAttr( ctx: *mlir.Context, handle: i64, channel_type: ChannelType, diff --git a/mlir/dialects/stablehlo/test.zig b/mlir/dialects/stablehlo/test.zig new file mode 100644 index 00000000..f1a2267f --- /dev/null +++ b/mlir/dialects/stablehlo/test.zig @@ -0,0 +1,102 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); + +const operation_names: []const []const u8 = &.{ + "stablehlo.abs", + "stablehlo.add", + "stablehlo.all_reduce", + "stablehlo.and", + "stablehlo.atan2", + "stablehlo.bitcast_convert", + "stablehlo.broadcast_in_dim", + "stablehlo.cbrt", + "stablehlo.ceil", + "stablehlo.cholesky", + "stablehlo.clamp", + "stablehlo.compare", + "stablehlo.complex", + "stablehlo.composite", + "stablehlo.concatenate", + "stablehlo.constant", + "stablehlo.convert", + "stablehlo.convolution", + "stablehlo.cosine", + "stablehlo.count_leading_zeros", + "stablehlo.custom_call", + "stablehlo.divide", + "stablehlo.dot_general", + "stablehlo.dynamic_slice", + "stablehlo.dynamic_update_slice", + "stablehlo.exponential", + "stablehlo.exponential_minus_one", + "stablehlo.fft", + "stablehlo.floor", + "stablehlo.gather", + "stablehlo.get_tuple_element", + "stablehlo.imag", + "stablehlo.iota", + "stablehlo.is_finite", + "stablehlo.log", + "stablehlo.log_plus_one", + "stablehlo.logistic", + "stablehlo.maximum", + "stablehlo.minimum", + "stablehlo.multiply", + "stablehlo.negate", + "stablehlo.not", + "stablehlo.optimization_barrier", + "stablehlo.or", + "stablehlo.pad", + "stablehlo.partition_id", + "stablehlo.popcnt", + "stablehlo.power", + "stablehlo.real", + "stablehlo.reduce_precision", + "stablehlo.remainder", + "stablehlo.reshape", + "stablehlo.return", + "stablehlo.reverse", + "stablehlo.rng", + "stablehlo.rng_bit_generator", + "stablehlo.round_nearest_afz", + "stablehlo.round_nearest_even", + "stablehlo.rsqrt", + "stablehlo.scatter", + "stablehlo.select", + "stablehlo.shift_left", + "stablehlo.shift_right_arithmetic", + "stablehlo.shift_right_logical", + "stablehlo.sign", + "stablehlo.sine", + "stablehlo.slice", + "stablehlo.sqrt", + "stablehlo.subtract", + "stablehlo.tanh", + "stablehlo.transpose", + "stablehlo.triangular_solve", + "stablehlo.tuple", + "stablehlo.while", + "stablehlo.xor", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(stablehlo); + inline for (comptime std.meta.declarations(stablehlo)) |decl| { + const T = @field(stablehlo, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + stablehlo.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/ttir/BUILD.bazel b/mlir/dialects/ttir/BUILD.bazel index 0bea003e..8ab5598f 100644 --- a/mlir/dialects/ttir/BUILD.bazel +++ b/mlir/dialects/ttir/BUILD.bazel @@ -2,36 +2,38 @@ load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") cc_library( - name = "ttir_capi", - srcs = ["ttir_capi.cc"], - hdrs = ["ttir_capi.h"], - strip_include_prefix = ".", + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], tags = ["manual"], - visibility = ["//visibility:private"], deps = [ "@llvm-project//mlir:CAPIIR", "@triton//:TritonDialects", ], - # MLIR_DEFINE_CAPI_DIALECT_REGISTRATION expands to a global static - # initializer that the linker would otherwise dead-strip. - alwayslink = True, ) zig_library( name = "ttir", - tags = ["manual"], import_name = "mlir/dialects/ttir", main = "ttir.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ - ":ttir_capi", + ":capi", "//mlir", - "//mlir:c", "//stdx", ], ) zig_test( name = "test", - deps = [":ttir"], + main = "test.zig", + deps = [ + ":ttir", + "//mlir", + "//mlir/dialects/arith", + "//mlir/dialects/func", + "//mlir/dialects/math", + "//mlir/dialects/scf", + ], ) diff --git a/mlir/dialects/ttir/capi.cc b/mlir/dialects/ttir/capi.cc new file mode 100644 index 00000000..0e7afb28 --- /dev/null +++ b/mlir/dialects/ttir/capi.cc @@ -0,0 +1,167 @@ +//===- capi.cc - C API for the Triton tt dialect --------------------------===// + +#include "capi.h" + +#include "triton/Dialect/Triton/IR/Dialect.h" +#include "triton/Dialect/Triton/IR/Types.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" + +// Note: the second arg `tt` controls the generated symbol name +// `mlirGetDialectHandle__tt__`, which `mlir.DialectHandle.fromString("tt")` on +// the Zig side resolves at comptime (see `DialectHandle.fromString` in +// mlir/mlir.zig). Must match the dialect's declared name (`let name = "tt"` in +// TritonDialect.td). +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Triton, tt, mlir::triton::TritonDialect) + +static mlir::MLIRContext *loadTriton(MlirContext context) { + mlir::MLIRContext *ctx = unwrap(context); + if (ctx) + ctx->getOrLoadDialect(); + return ctx; +} + +using mlir::cast; +using mlir::isa; + +MlirType mlirTritonPointerTypeGet(MlirType pointee, int32_t addressSpace) { + return wrap(mlir::triton::PointerType::get( + unwrap(pointee), static_cast(addressSpace))); +} + +MlirType mlirTritonPointerTypeGetPointee(MlirType type) { + return wrap(cast(unwrap(type)).getPointeeType()); +} + +int32_t mlirTritonPointerTypeGetAddressSpace(MlirType type) { + return static_cast( + cast(unwrap(type)).getAddressSpace()); +} + +bool mlirTypeIsATritonPointer(MlirType type) { + return isa(unwrap(type)); +} + +MlirType mlirTritonTensorDescTypeGet(intptr_t rank, const int64_t *shape, + MlirType elementType, + MlirAttribute sharedLayout) { + mlir::Attribute layoutAttr; + if (!mlirAttributeIsNull(sharedLayout)) { + layoutAttr = unwrap(sharedLayout); + } + // TensorDescType::get is a TypeBuilderWithInferredContext: the context is + // inferred from elementType. Signature: (shape, elementType, sharedLayout). + return wrap(mlir::triton::TensorDescType::get( + llvm::ArrayRef(shape, rank), unwrap(elementType), layoutAttr)); +} + +bool mlirTypeIsATritonTensorDesc(MlirType type) { + return isa(unwrap(type)); +} + +MlirAttribute mlirTritonProgramIDDimAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::ProgramIDDimAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonCacheModifierAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::CacheModifierAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonEvictionPolicyAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::EvictionPolicyAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonPaddingOptionAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::PaddingOptionAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonRoundingModeAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::RoundingModeAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonInputPrecisionAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::InputPrecisionAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonRMWOpAttrGet(MlirContext context, int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::RMWOpAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonMemSemanticAttrGet(MlirContext context, int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::MemSemanticAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonMemSyncScopeAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::MemSyncScopeAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonPropagateNanAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::PropagateNanAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonScaleDotElemTypeAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::ScaleDotElemTypeAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonDescriptorReduceKindAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::DescriptorReduceKindAttr::get( + ctx, static_cast(value))); +} diff --git a/mlir/dialects/ttir/capi.h b/mlir/dialects/ttir/capi.h new file mode 100644 index 00000000..efd0aae9 --- /dev/null +++ b/mlir/dialects/ttir/capi.h @@ -0,0 +1,59 @@ +//===- capi.h - C API for the Triton tt dialect -------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_TTIR_CAPI_H +#define MLIR_DIALECTS_TTIR_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Triton, tt); + +// !tt.ptr +MLIR_CAPI_EXPORTED MlirType mlirTritonPointerTypeGet(MlirType pointee, + int32_t addressSpace); +MLIR_CAPI_EXPORTED MlirType mlirTritonPointerTypeGetPointee(MlirType type); +MLIR_CAPI_EXPORTED int32_t mlirTritonPointerTypeGetAddressSpace(MlirType type); +MLIR_CAPI_EXPORTED bool mlirTypeIsATritonPointer(MlirType type); + +// !tt.tensordesc> — result of tt.make_tensor_descriptor. +// `shape` is a `rank`-element array of int64_t block dims; `elementType` is +// the scalar element type; `sharedLayout` may be a null MlirAttribute for none. +MLIR_CAPI_EXPORTED MlirType +mlirTritonTensorDescTypeGet(intptr_t rank, const int64_t *shape, + MlirType elementType, MlirAttribute sharedLayout); +MLIR_CAPI_EXPORTED bool mlirTypeIsATritonTensorDesc(MlirType type); + +// I32EnumAttr constructors +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonProgramIDDimAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonCacheModifierAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonEvictionPolicyAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonPaddingOptionAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonRoundingModeAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonInputPrecisionAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute mlirTritonRMWOpAttrGet(MlirContext context, + int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonMemSemanticAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonMemSyncScopeAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonPropagateNanAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonScaleDotElemTypeAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonDescriptorReduceKindAttrGet(MlirContext context, int32_t value); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_TTIR_CAPI_H diff --git a/mlir/dialects/ttir/test.zig b/mlir/dialects/ttir/test.zig new file mode 100644 index 00000000..390f34b5 --- /dev/null +++ b/mlir/dialects/ttir/test.zig @@ -0,0 +1,163 @@ +const std = @import("std"); + +const arith = @import("mlir/dialects/arith"); +const func = @import("mlir/dialects/func"); +const math = @import("mlir/dialects/math"); +const mlir = @import("mlir"); +const scf = @import("mlir/dialects/scf"); +const ttir = @import("mlir/dialects/ttir"); + +const operation_names: []const []const u8 = &.{ + "tt.addptr", + "tt.assert", + "tt.atomic_cas", + "tt.atomic_rmw", + "tt.bitcast", + "tt.broadcast", + "tt.call", + "tt.clampf", + "tt.descriptor_gather", + "tt.descriptor_load", + "tt.descriptor_reduce", + "tt.descriptor_scatter", + "tt.descriptor_store", + "tt.dot", + "tt.dot_scaled", + "tt.elementwise_inline_asm", + "tt.expand_dims", + "tt.extern_elementwise", + "tt.fp_to_fp", + "tt.func", + "tt.gather", + "tt.get_num_programs", + "tt.get_program_id", + "tt.histogram", + "tt.int_to_ptr", + "tt.join", + "tt.load", + "tt.make_range", + "tt.make_tensor_descriptor", + "tt.map_elementwise", + "tt.map_elementwise.return", + "tt.mulhiui", + "tt.precise_divf", + "tt.precise_sqrt", + "tt.print", + "tt.ptr_to_int", + "tt.reduce", + "tt.reduce.return", + "tt.reshape", + "tt.return", + "tt.scan", + "tt.scan.return", + "tt.splat", + "tt.split", + "tt.store", + "tt.trans", + "tt.unsplat", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(ttir); + inline for (comptime std.meta.declarations(ttir)) |decl| { + const T = @field(ttir, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "tt.func round-trip" { + // Build: module { tt.func public @id(%arg0: !tt.ptr) { tt.return } } + // by constructing it with the Zig wrappers, printing to string, and re-parsing. + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + func.registerDialect(registry); + arith.registerDialect(registry); + scf.registerDialect(registry); + math.registerDialect(registry); + ttir.registerDialect(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + + try std.testing.expect(ctx.isRegisteredOperation("tt.func")); + try std.testing.expect(ctx.isRegisteredOperation("tt.return")); + try std.testing.expect(ctx.isRegisteredOperation("tt.load")); + try std.testing.expect(ctx.isRegisteredOperation("tt.store")); + try std.testing.expect(ctx.isRegisteredOperation("tt.get_program_id")); + try std.testing.expect(ctx.isRegisteredOperation("tt.splat")); + try std.testing.expect(ctx.isRegisteredOperation("tt.addptr")); + try std.testing.expect(ctx.isRegisteredOperation("scf.for")); + try std.testing.expect(ctx.isRegisteredOperation("arith.constant")); + + const loc: *const mlir.Location = .unknown(ctx); + const module: *mlir.Module = .init(loc); + defer module.deinit(); + + const f32_ty = mlir.Type.float(ctx, .f32); + const ptr_ty = ttir.pointerType(f32_ty, 1); + + const entry = mlir.Block.init(&.{ ptr_ty, ptr_ty }, &.{ loc, loc }); + + const arg0 = entry.argument(0); + const arg1 = entry.argument(1); + + // %0 = tt.load %arg0 : !tt.ptr + const loaded = ttir.load(ctx, arg0, f32_ty, .none, .normal, false, .{}, loc); + _ = loaded.appendTo(entry); + + // %1 = arith.constant 1.0 : f32 — built with Operation.make to avoid cross-dialect import + const one_attr: *const mlir.Attribute = .float(ctx, .f32, 1.0); + const one = mlir.Operation.make(ctx, "arith.constant", .{ + .results = .{ .flat = &.{f32_ty} }, + .attributes = &.{.named(ctx, "value", one_attr)}, + .location = loc, + }); + _ = one.appendTo(entry); + + // %2 = arith.addf %0, %1 : f32 + const summed = mlir.Operation.make(ctx, "arith.addf", .{ + .operands = .{ .flat = &.{ loaded.result(0), one.result(0) } }, + .result_type_inference = true, + .location = loc, + }); + _ = summed.appendTo(entry); + + // tt.store %arg1, %2 + _ = ttir.store(ctx, arg1, summed.result(0), .none, .normal, .{}, loc).appendTo(entry); + + // tt.return + _ = ttir.return_(ctx, &.{}, loc).appendTo(entry); + + const f = ttir.func(ctx, .{ + .name = "add_one", + .block = entry, + .location = loc, + }); + _ = f.appendTo(module.body()); + + // Verify the operation. + try std.testing.expect(module.operation().verify()); + + // Print IR and re-parse it to confirm it's valid text. + var al: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer al.deinit(); + try al.writer.print("{f}", .{module.operation()}); + + const parsed = try mlir.Module.parse(ctx, al.written()); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + ttir.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/ttir/ttir.zig b/mlir/dialects/ttir/ttir.zig index ee302f92..b55bd418 100644 --- a/mlir/dialects/ttir/ttir.zig +++ b/mlir/dialects/ttir/ttir.zig @@ -4,6 +4,10 @@ const c = @import("c"); const mlir = @import("mlir"); const stdx = @import("stdx"); +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("tt"); +} + // ============================================================================= // Types — !tt.ptr // ============================================================================= @@ -11,14 +15,19 @@ const stdx = @import("stdx"); pub const PointerType = opaque { const M = mlir.Methods(PointerType, c.MlirType); - pub const isAFn = c.mlirTritonTypeIsAPointer; + pub const isAFn = c.mlirTypeIsATritonPointer; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; - pub fn get(pointee_type: *const mlir.Type, address_space: i32) *const PointerType { - return @ptrCast(c.mlirTritonPointerTypeGet(pointee_type.ptr(), address_space).ptr); + pub const InitArgs = struct { + pointee_type: *const mlir.Type, + address_space: i32, + }; + + pub fn get(args: InitArgs) *const PointerType { + return @ptrCast(c.mlirTritonPointerTypeGet(args.pointee_type.ptr(), args.address_space).ptr); } pub fn pointee(self: *const PointerType) *const mlir.Type { @@ -28,11 +37,15 @@ pub const PointerType = opaque { pub fn addressSpace(self: *const PointerType) i32 { return c.mlirTritonPointerTypeGetAddressSpace(self.ptr()); } + + pub fn type_(self: *const PointerType) *const mlir.Type { + return @ptrCast(self); + } }; /// Convenience alias: `!tt.ptr` as a plain *const mlir.Type. pub fn pointerType(pointee_type: *const mlir.Type, address_space: i32) *const mlir.Type { - return @ptrCast(PointerType.get(pointee_type, address_space)); + return @ptrCast(PointerType.get(.{ .pointee_type = pointee_type, .address_space = address_space })); } // ----------------------------------------------------------------------------- @@ -42,34 +55,42 @@ pub fn pointerType(pointee_type: *const mlir.Type, address_space: i32) *const ml pub const TensorDescType = opaque { const M = mlir.Methods(TensorDescType, c.MlirType); - pub const isAFn = c.mlirTritonTypeIsATensorDesc; + pub const isAFn = c.mlirTypeIsATritonTensorDesc; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); pub const format = M.format(c.mlirTypePrint); pub const isA = M.isA; - /// `shared_layout` may be null for an unlayout'd descriptor. - pub fn get( + pub const InitArgs = struct { shape: []const i64, element_type: *const mlir.Type, - shared_layout: ?*const mlir.Attribute, - ) *const TensorDescType { - const attr_ptr = if (shared_layout) |a| a.ptr() else c.MlirAttribute{ .ptr = null }; + /// May be null for an unlayout'd descriptor. + shared_layout: ?*const mlir.Attribute = null, + }; + + pub fn get(args: InitArgs) *const TensorDescType { + const attr_ptr = if (args.shared_layout) |a| a.ptr() else c.MlirAttribute{ .ptr = null }; return @ptrCast(c.mlirTritonTensorDescTypeGet( - @intCast(shape.len), - shape.ptr, - element_type.ptr(), + @intCast(args.shape.len), + args.shape.ptr, + args.element_type.ptr(), attr_ptr, ).ptr); } + + pub fn type_(self: *const TensorDescType) *const mlir.Type { + return @ptrCast(self); + } }; pub fn tensorDescType( shape: []const i64, element_type: *const mlir.Type, - shared_layout: ?*const mlir.Attribute, + opts: struct { + shared_layout: ?*const mlir.Attribute = null, + }, ) *const mlir.Type { - return @ptrCast(TensorDescType.get(shape, element_type, shared_layout)); + return @ptrCast(TensorDescType.get(.{ .shape = shape, .element_type = element_type, .shared_layout = opts.shared_layout })); } // ============================================================================= @@ -85,9 +106,9 @@ pub const InputPrecision = enum(i32) { tf32 = 0, tf32x3 = 1, ieee = 2, bf16x3 = /// Atomic read-modify-write operations. pub const RMWOp = enum(i32) { - and_ = 1, - or_ = 2, - xor_ = 3, + @"and" = 1, + @"or" = 2, + xor = 3, add = 4, fadd = 5, max = 6, @@ -136,57 +157,57 @@ pub const DescriptorReduceKind = enum(i32) { max = 3, inc = 4, dec = 5, - and_ = 6, - or_ = 7, - xor_ = 8, + @"and" = 6, + @"or" = 7, + xor = 8, }; -pub fn programDim(ctx: *mlir.Context, v: ProgramIDDim) *const mlir.Attribute { - return @ptrCast(c.mlirTritonProgramDimGet(ctx.ptr(), @backingInt(v)).ptr); +pub fn programDimAttr(ctx: *mlir.Context, v: ProgramIDDim) *const mlir.Attribute { + return @ptrCast(c.mlirTritonProgramIDDimAttrGet(ctx.ptr(), @backingInt(v)).ptr); } -pub fn cacheModifier(ctx: *mlir.Context, v: CacheModifier) *const mlir.Attribute { - return @ptrCast(c.mlirTritonCacheModifierGet(ctx.ptr(), @backingInt(v)).ptr); +pub fn cacheModifierAttr(ctx: *mlir.Context, v: CacheModifier) *const mlir.Attribute { + return @ptrCast(c.mlirTritonCacheModifierAttrGet(ctx.ptr(), @backingInt(v)).ptr); } -pub fn evictionPolicy(ctx: *mlir.Context, v: EvictionPolicy) *const mlir.Attribute { - return @ptrCast(c.mlirTritonEvictionPolicyGet(ctx.ptr(), @backingInt(v)).ptr); +pub fn evictionPolicyAttr(ctx: *mlir.Context, v: EvictionPolicy) *const mlir.Attribute { + return @ptrCast(c.mlirTritonEvictionPolicyAttrGet(ctx.ptr(), @backingInt(v)).ptr); } -pub fn paddingOption(ctx: *mlir.Context, v: PaddingOption) *const mlir.Attribute { - return @ptrCast(c.mlirTritonPaddingOptionGet(ctx.ptr(), @backingInt(v)).ptr); +pub fn paddingOptionAttr(ctx: *mlir.Context, v: PaddingOption) *const mlir.Attribute { + return @ptrCast(c.mlirTritonPaddingOptionAttrGet(ctx.ptr(), @backingInt(v)).ptr); } -pub fn roundingMode(ctx: *mlir.Context, v: RoundingMode) *const mlir.Attribute { - return @ptrCast(c.mlirTritonRoundingModeGet(ctx.ptr(), @backingInt(v)).ptr); +pub fn roundingModeAttr(ctx: *mlir.Context, v: RoundingMode) *const mlir.Attribute { + return @ptrCast(c.mlirTritonRoundingModeAttrGet(ctx.ptr(), @backingInt(v)).ptr); } -pub fn inputPrecision(ctx: *mlir.Context, v: InputPrecision) *const mlir.Attribute { - return @ptrCast(c.mlirTritonInputPrecisionGet(ctx.ptr(), @backingInt(v)).ptr); +pub fn inputPrecisionAttr(ctx: *mlir.Context, v: InputPrecision) *const mlir.Attribute { + return @ptrCast(c.mlirTritonInputPrecisionAttrGet(ctx.ptr(), @backingInt(v)).ptr); } -pub fn rmwOp(ctx: *mlir.Context, v: RMWOp) *const mlir.Attribute { - return @ptrCast(c.mlirTritonRMWOpGet(ctx.ptr(), @backingInt(v)).ptr); +pub fn rmwOpAttr(ctx: *mlir.Context, v: RMWOp) *const mlir.Attribute { + return @ptrCast(c.mlirTritonRMWOpAttrGet(ctx.ptr(), @backingInt(v)).ptr); } -pub fn memSemantic(ctx: *mlir.Context, v: MemSemantic) *const mlir.Attribute { - return @ptrCast(c.mlirTritonMemSemanticGet(ctx.ptr(), @backingInt(v)).ptr); +pub fn memSemanticAttr(ctx: *mlir.Context, v: MemSemantic) *const mlir.Attribute { + return @ptrCast(c.mlirTritonMemSemanticAttrGet(ctx.ptr(), @backingInt(v)).ptr); } -pub fn memSyncScope(ctx: *mlir.Context, v: MemSyncScope) *const mlir.Attribute { - return @ptrCast(c.mlirTritonMemSyncScopeGet(ctx.ptr(), @backingInt(v)).ptr); +pub fn memSyncScopeAttr(ctx: *mlir.Context, v: MemSyncScope) *const mlir.Attribute { + return @ptrCast(c.mlirTritonMemSyncScopeAttrGet(ctx.ptr(), @backingInt(v)).ptr); } -pub fn propagateNan(ctx: *mlir.Context, v: PropagateNan) *const mlir.Attribute { - return @ptrCast(c.mlirTritonPropagateNanGet(ctx.ptr(), @backingInt(v)).ptr); +pub fn propagateNanAttr(ctx: *mlir.Context, v: PropagateNan) *const mlir.Attribute { + return @ptrCast(c.mlirTritonPropagateNanAttrGet(ctx.ptr(), @backingInt(v)).ptr); } -pub fn scaleDotElemType(ctx: *mlir.Context, v: ScaleDotElemType) *const mlir.Attribute { - return @ptrCast(c.mlirTritonScaleDotElemTypeGet(ctx.ptr(), @backingInt(v)).ptr); +pub fn scaleDotElemTypeAttr(ctx: *mlir.Context, v: ScaleDotElemType) *const mlir.Attribute { + return @ptrCast(c.mlirTritonScaleDotElemTypeAttrGet(ctx.ptr(), @backingInt(v)).ptr); } -pub fn descriptorReduceKind(ctx: *mlir.Context, v: DescriptorReduceKind) *const mlir.Attribute { - return @ptrCast(c.mlirTritonDescriptorReduceKindGet(ctx.ptr(), @backingInt(v)).ptr); +pub fn descriptorReduceKindAttr(ctx: *mlir.Context, v: DescriptorReduceKind) *const mlir.Attribute { + return @ptrCast(c.mlirTritonDescriptorReduceKindAttrGet(ctx.ptr(), @backingInt(v)).ptr); } // ============================================================================= @@ -272,7 +293,7 @@ pub fn get_program_id(ctx: *mlir.Context, axis: ProgramIDDim, location: *const m return .make(ctx, "tt.get_program_id", .{ .results = .{ .flat = &.{.int(ctx, .i32)} }, .attributes = &.{ - .named(ctx, "axis", programDim(ctx, axis)), + .named(ctx, "axis", programDimAttr(ctx, axis)), }, .location = location, }); @@ -282,7 +303,7 @@ pub fn get_num_programs(ctx: *mlir.Context, axis: ProgramIDDim, location: *const return .make(ctx, "tt.get_num_programs", .{ .results = .{ .flat = &.{.int(ctx, .i32)} }, .attributes = &.{ - .named(ctx, "axis", programDim(ctx, axis)), + .named(ctx, "axis", programDimAttr(ctx, axis)), }, .location = location, }); @@ -420,20 +441,22 @@ pub fn load( ctx: *mlir.Context, ptr_val: *const mlir.Value, result_type: *const mlir.Type, - mask: ?*const mlir.Value, - other: ?*const mlir.Value, cache: CacheModifier, evict: EvictionPolicy, is_volatile: bool, + opts: struct { + mask: ?*const mlir.Value = null, + other: ?*const mlir.Value = null, + }, location: *const mlir.Location, ) *mlir.Operation { var operands_buf: stdx.BoundedArray(*const mlir.Value, 3) = .empty; operands_buf.appendAssumeCapacity(ptr_val); - const mask_len: i32 = if (mask) |m| blk: { + const mask_len: i32 = if (opts.mask) |m| blk: { operands_buf.appendAssumeCapacity(m); break :blk 1; } else 0; - const other_len: i32 = if (other) |o| blk: { + const other_len: i32 = if (opts.other) |o| blk: { operands_buf.appendAssumeCapacity(o); break :blk 1; } else 0; @@ -444,8 +467,8 @@ pub fn load( .results = .{ .flat = &.{result_type} }, .attributes = &.{ .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), - .named(ctx, "cache", cacheModifier(ctx, cache)), - .named(ctx, "evict", evictionPolicy(ctx, evict)), + .named(ctx, "cache", cacheModifierAttr(ctx, cache)), + .named(ctx, "evict", evictionPolicyAttr(ctx, evict)), .named(ctx, "isVolatile", .boolean(ctx, is_volatile)), }, .location = location, @@ -457,19 +480,21 @@ pub fn store( ctx: *mlir.Context, ptr_val: *const mlir.Value, value: *const mlir.Value, - mask: ?*const mlir.Value, cache: CacheModifier, evict: EvictionPolicy, + opts: struct { + mask: ?*const mlir.Value = null, + }, location: *const mlir.Location, ) *mlir.Operation { var buf: stdx.BoundedArray(*const mlir.Value, 3) = .empty; buf.appendSliceAssumeCapacity(&.{ ptr_val, value }); - if (mask) |m| buf.appendAssumeCapacity(m); + if (opts.mask) |m| buf.appendAssumeCapacity(m); return .make(ctx, "tt.store", .{ .operands = .{ .flat = buf.constSlice() }, .attributes = &.{ - .named(ctx, "cache", cacheModifier(ctx, cache)), - .named(ctx, "evict", evictionPolicy(ctx, evict)), + .named(ctx, "cache", cacheModifierAttr(ctx, cache)), + .named(ctx, "evict", evictionPolicyAttr(ctx, evict)), }, .location = location, }); @@ -493,7 +518,7 @@ pub fn dot( .operands = .{ .flat = &.{ a, b, c_acc } }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "inputPrecision", inputPrecision(ctx, input_precision_)), + .named(ctx, "inputPrecision", inputPrecisionAttr(ctx, input_precision_)), .named(ctx, "maxNumImpreciseAcc", .int(ctx, .i32, max_num_imprecise_acc)), }, .location = location, @@ -587,12 +612,14 @@ pub fn fp_to_fp( ctx: *mlir.Context, src: *const mlir.Value, result_type: *const mlir.Type, - rounding: ?RoundingMode, + opts: struct { + rounding: ?RoundingMode = null, + }, location: *const mlir.Location, ) *mlir.Operation { var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (rounding) |rm| { - attrs.appendAssumeCapacity(.named(ctx, "rounding", roundingMode(ctx, rm))); + if (opts.rounding) |rm| { + attrs.appendAssumeCapacity(.named(ctx, "rounding", roundingModeAttr(ctx, rm))); } return .make(ctx, "tt.fp_to_fp", .{ .operands = .{ .flat = &.{src} }, @@ -619,7 +646,7 @@ pub fn clampf( .operands = .{ .flat = &.{ x, min, max } }, .results = .{ .flat = &.{x.type_()} }, .attributes = &.{ - .named(ctx, "propagateNan", propagateNan(ctx, propagate_nan)), + .named(ctx, "propagateNan", propagateNanAttr(ctx, propagate_nan)), }, .location = location, }); @@ -684,21 +711,6 @@ pub fn unsplat( }); } -/// tt.cat — concatenate two tensors; may reorder elements. -pub fn cat( - ctx: *mlir.Context, - lhs: *const mlir.Value, - rhs: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.cat", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - /// tt.join — join two tensors along a new minor dimension (shape gains +1 dim of size 2). pub fn join( ctx: *mlir.Context, @@ -760,13 +772,15 @@ pub fn gather( pub fn histogram( ctx: *mlir.Context, src: *const mlir.Value, - mask: ?*const mlir.Value, result_type: *const mlir.Type, + opts: struct { + mask: ?*const mlir.Value = null, + }, location: *const mlir.Location, ) *mlir.Operation { var buf: stdx.BoundedArray(*const mlir.Value, 2) = .empty; buf.appendAssumeCapacity(src); - if (mask) |m| buf.appendAssumeCapacity(m); + if (opts.mask) |m| buf.appendAssumeCapacity(m); return .make(ctx, "tt.histogram", .{ .operands = .{ .flat = buf.constSlice() }, .results = .{ .flat = &.{result_type} }, @@ -823,22 +837,24 @@ pub fn atomic_rmw( rmw: RMWOp, ptr_val: *const mlir.Value, val: *const mlir.Value, - mask: ?*const mlir.Value, sem: MemSemantic, scope: MemSyncScope, + opts: struct { + mask: ?*const mlir.Value = null, + }, location: *const mlir.Location, ) *mlir.Operation { var operands_buf: stdx.BoundedArray(*const mlir.Value, 3) = .empty; operands_buf.appendSliceAssumeCapacity(&.{ ptr_val, val }); - if (mask) |m| operands_buf.appendAssumeCapacity(m); + if (opts.mask) |m| operands_buf.appendAssumeCapacity(m); return .make(ctx, "tt.atomic_rmw", .{ .operands = .{ .flat = operands_buf.constSlice() }, .results = .{ .flat = &.{val.type_()} }, .attributes = &.{ - .named(ctx, "atomic_rmw_op", rmwOp(ctx, rmw)), - .named(ctx, "sem", memSemantic(ctx, sem)), - .named(ctx, "scope", memSyncScope(ctx, scope)), + .named(ctx, "atomic_rmw_op", rmwOpAttr(ctx, rmw)), + .named(ctx, "sem", memSemanticAttr(ctx, sem)), + .named(ctx, "scope", memSyncScopeAttr(ctx, scope)), }, .location = location, }); @@ -858,8 +874,8 @@ pub fn atomic_cas( .operands = .{ .flat = &.{ ptr_val, cmp, val } }, .results = .{ .flat = &.{val.type_()} }, .attributes = &.{ - .named(ctx, "sem", memSemantic(ctx, sem)), - .named(ctx, "scope", memSyncScope(ctx, scope)), + .named(ctx, "sem", memSemanticAttr(ctx, sem)), + .named(ctx, "scope", memSyncScopeAttr(ctx, scope)), }, .location = location, }); @@ -890,22 +906,25 @@ pub fn assert_( // ============================================================================= /// tt.call — direct call to another tt.func in the same module. -/// arg_attrs / res_attrs propagate hints like tt.divisibility; pass null to omit. +/// `opts.arg_attrs` / `opts.res_attrs` propagate hints like tt.divisibility; +/// leave them unset to omit them. pub fn call( ctx: *mlir.Context, callee: []const u8, operands: []const *const mlir.Value, result_types: []const *const mlir.Type, - arg_attrs: ?[]const *const mlir.Attribute, - res_attrs: ?[]const *const mlir.Attribute, + opts: struct { + arg_attrs: ?[]const *const mlir.Attribute = null, + res_attrs: ?[]const *const mlir.Attribute = null, + }, location: *const mlir.Location, ) *mlir.Operation { var attrs: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; attrs.appendAssumeCapacity(.named(ctx, "callee", .flatSymbolRef(ctx, callee))); - if (arg_attrs) |aa| { + if (opts.arg_attrs) |aa| { attrs.appendAssumeCapacity(.named(ctx, "arg_attrs", .array(ctx, aa))); } - if (res_attrs) |ra| { + if (opts.res_attrs) |ra| { attrs.appendAssumeCapacity(.named(ctx, "res_attrs", .array(ctx, ra))); } return .make(ctx, "tt.call", .{ @@ -927,23 +946,25 @@ pub fn dot_scaled( a: *const mlir.Value, b: *const mlir.Value, c_acc: *const mlir.Value, - a_scale: ?*const mlir.Value, - b_scale: ?*const mlir.Value, a_elem_type: ScaleDotElemType, b_elem_type: ScaleDotElemType, result_type: *const mlir.Type, fast_math: bool, lhs_k_pack: bool, rhs_k_pack: bool, + opts: struct { + a_scale: ?*const mlir.Value = null, + b_scale: ?*const mlir.Value = null, + }, location: *const mlir.Location, ) *mlir.Operation { var buf: stdx.BoundedArray(*const mlir.Value, 5) = .empty; buf.appendSliceAssumeCapacity(&.{ a, b, c_acc }); - const a_scale_len: i32 = if (a_scale) |s| blk: { + const a_scale_len: i32 = if (opts.a_scale) |s| blk: { buf.appendAssumeCapacity(s); break :blk 1; } else 0; - const b_scale_len: i32 = if (b_scale) |s| blk: { + const b_scale_len: i32 = if (opts.b_scale) |s| blk: { buf.appendAssumeCapacity(s); break :blk 1; } else 0; @@ -954,8 +975,8 @@ pub fn dot_scaled( .results = .{ .flat = &.{result_type} }, .attributes = &.{ .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), - .named(ctx, "a_elem_type", scaleDotElemType(ctx, a_elem_type)), - .named(ctx, "b_elem_type", scaleDotElemType(ctx, b_elem_type)), + .named(ctx, "a_elem_type", scaleDotElemTypeAttr(ctx, a_elem_type)), + .named(ctx, "b_elem_type", scaleDotElemTypeAttr(ctx, b_elem_type)), .named(ctx, "fastMath", .boolean(ctx, fast_math)), .named(ctx, "lhs_k_pack", .boolean(ctx, lhs_k_pack)), .named(ctx, "rhs_k_pack", .boolean(ctx, rhs_k_pack)), @@ -1042,7 +1063,7 @@ pub fn make_tensor_descriptor( .operands = .{ .flat = buf.constSlice() }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "padding", paddingOption(ctx, padding)), + .named(ctx, "padding", paddingOptionAttr(ctx, padding)), }, .location = location, }); @@ -1064,8 +1085,8 @@ pub fn descriptor_load( .operands = .{ .flat = buf.constSlice() }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "cache", cacheModifier(ctx, cache)), - .named(ctx, "evict", evictionPolicy(ctx, evict)), + .named(ctx, "cache", cacheModifierAttr(ctx, cache)), + .named(ctx, "evict", evictionPolicyAttr(ctx, evict)), }, .location = location, }); @@ -1101,7 +1122,7 @@ pub fn descriptor_reduce( return .make(ctx, "tt.descriptor_reduce", .{ .operands = .{ .flat = buf.constSlice() }, .attributes = &.{ - .named(ctx, "kind", descriptorReduceKind(ctx, kind)), + .named(ctx, "kind", descriptorReduceKindAttr(ctx, kind)), }, .location = location, }); @@ -1202,90 +1223,3 @@ pub fn elementwise_inline_asm( .location = location, }); } - -test { - std.testing.refAllDecls(@This()); -} - -test "tt.func round-trip" { - // Build: module { tt.func public @id(%arg0: !tt.ptr) { tt.return } } - // by constructing it with the Zig wrappers, printing to string, and re-parsing. - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - inline for (.{ "func", "arith", "scf", "math", "tt" }) |d| { - mlir.DialectHandle.fromString(d).insertDialect(registry); - } - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - - try std.testing.expect(ctx.isRegisteredOperation("tt.func")); - try std.testing.expect(ctx.isRegisteredOperation("tt.return")); - try std.testing.expect(ctx.isRegisteredOperation("tt.load")); - try std.testing.expect(ctx.isRegisteredOperation("tt.store")); - try std.testing.expect(ctx.isRegisteredOperation("tt.get_program_id")); - try std.testing.expect(ctx.isRegisteredOperation("tt.splat")); - try std.testing.expect(ctx.isRegisteredOperation("tt.addptr")); - try std.testing.expect(ctx.isRegisteredOperation("scf.for")); - try std.testing.expect(ctx.isRegisteredOperation("arith.constant")); - - const loc: *const mlir.Location = .unknown(ctx); - const module: *mlir.Module = .init(loc); - defer module.deinit(); - - const f32_ty = mlir.Type.float(ctx, .f32); - const ptr_ty = pointerType(f32_ty, 1); - - const entry = mlir.Block.init(&.{ ptr_ty, ptr_ty }, &.{ loc, loc }); - - const arg0 = entry.argument(0); - const arg1 = entry.argument(1); - - // %0 = tt.load %arg0 : !tt.ptr - const loaded = load(ctx, arg0, f32_ty, null, null, .none, .normal, false, loc); - _ = loaded.appendTo(entry); - - // %1 = arith.constant 1.0 : f32 — built with Operation.make to avoid cross-dialect import - const one_attr: *const mlir.Attribute = .float(ctx, .f32, 1.0); - const one = mlir.Operation.make(ctx, "arith.constant", .{ - .results = .{ .flat = &.{f32_ty} }, - .attributes = &.{.named(ctx, "value", one_attr)}, - .location = loc, - }); - _ = one.appendTo(entry); - - // %2 = arith.addf %0, %1 : f32 - const summed = mlir.Operation.make(ctx, "arith.addf", .{ - .operands = .{ .flat = &.{ loaded.result(0), one.result(0) } }, - .result_type_inference = true, - .location = loc, - }); - _ = summed.appendTo(entry); - - // tt.store %arg1, %2 - _ = store(ctx, arg1, summed.result(0), null, .none, .normal, loc).appendTo(entry); - - // tt.return - _ = return_(ctx, &.{}, loc).appendTo(entry); - - const f = func(ctx, .{ - .name = "add_one", - .block = entry, - .location = loc, - }); - _ = f.appendTo(module.body()); - - // Verify the operation. - try std.testing.expect(module.operation().verify()); - - // Print IR and re-parse it to confirm it's valid text. - var al: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer al.deinit(); - try al.writer.print("{f}", .{module.operation()}); - - const parsed = try mlir.Module.parse(ctx, al.written()); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} diff --git a/mlir/dialects/ttir/ttir_capi.cc b/mlir/dialects/ttir/ttir_capi.cc deleted file mode 100644 index ffd92990..00000000 --- a/mlir/dialects/ttir/ttir_capi.cc +++ /dev/null @@ -1,115 +0,0 @@ -#include "ttir_capi.h" - -#include "mlir/CAPI/IR.h" -#include "mlir/CAPI/Registration.h" -#include "triton/Dialect/Triton/IR/Dialect.h" -#include "triton/Dialect/Triton/IR/Types.h" - -// Note: the second arg `tt` controls the generated symbol name -// `mlirGetDialectHandle__tt__`, which `mlir.DialectHandle.fromString("tt")` on -// the Zig side resolves at comptime (see mlir/mlir.zig:696-698). Must match the -// dialect's declared name (`let name = "tt"` in TritonDialect.td). -MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Triton, tt, mlir::triton::TritonDialect) - -using mlir::cast; -using mlir::isa; - -extern "C" { - -MlirType mlirTritonPointerTypeGet(MlirType pointee, int32_t address_space) { - return wrap(mlir::triton::PointerType::get( - unwrap(pointee), static_cast(address_space))); -} - -MlirType mlirTritonPointerTypeGetPointee(MlirType ptr) { - return wrap(cast(unwrap(ptr)).getPointeeType()); -} - -int32_t mlirTritonPointerTypeGetAddressSpace(MlirType ptr) { - return static_cast( - cast(unwrap(ptr)).getAddressSpace()); -} - -bool mlirTritonTypeIsAPointer(MlirType t) { - return isa(unwrap(t)); -} - -MlirType mlirTritonTensorDescTypeGet(intptr_t rank, const int64_t* shape, - MlirType element_type, - MlirAttribute shared_layout) { - mlir::Attribute layout_attr; - if (!mlirAttributeIsNull(shared_layout)) { - layout_attr = unwrap(shared_layout); - } - // TensorDescType::get is a TypeBuilderWithInferredContext: the context is - // inferred from elementType. Signature: (shape, elementType, sharedLayout). - return wrap(mlir::triton::TensorDescType::get( - llvm::ArrayRef(shape, rank), - unwrap(element_type), layout_attr)); -} - -bool mlirTritonTypeIsATensorDesc(MlirType t) { - return isa(unwrap(t)); -} - -MlirAttribute mlirTritonProgramDimGet(MlirContext ctx, int32_t value) { - return wrap(mlir::triton::ProgramIDDimAttr::get( - unwrap(ctx), static_cast(value))); -} - -MlirAttribute mlirTritonCacheModifierGet(MlirContext ctx, int32_t value) { - return wrap(mlir::triton::CacheModifierAttr::get( - unwrap(ctx), static_cast(value))); -} - -MlirAttribute mlirTritonEvictionPolicyGet(MlirContext ctx, int32_t value) { - return wrap(mlir::triton::EvictionPolicyAttr::get( - unwrap(ctx), static_cast(value))); -} - -MlirAttribute mlirTritonPaddingOptionGet(MlirContext ctx, int32_t value) { - return wrap(mlir::triton::PaddingOptionAttr::get( - unwrap(ctx), static_cast(value))); -} - -MlirAttribute mlirTritonRoundingModeGet(MlirContext ctx, int32_t value) { - return wrap(mlir::triton::RoundingModeAttr::get( - unwrap(ctx), static_cast(value))); -} - -MlirAttribute mlirTritonInputPrecisionGet(MlirContext ctx, int32_t value) { - return wrap(mlir::triton::InputPrecisionAttr::get( - unwrap(ctx), static_cast(value))); -} - -MlirAttribute mlirTritonRMWOpGet(MlirContext ctx, int32_t value) { - return wrap(mlir::triton::RMWOpAttr::get( - unwrap(ctx), static_cast(value))); -} - -MlirAttribute mlirTritonMemSemanticGet(MlirContext ctx, int32_t value) { - return wrap(mlir::triton::MemSemanticAttr::get( - unwrap(ctx), static_cast(value))); -} - -MlirAttribute mlirTritonMemSyncScopeGet(MlirContext ctx, int32_t value) { - return wrap(mlir::triton::MemSyncScopeAttr::get( - unwrap(ctx), static_cast(value))); -} - -MlirAttribute mlirTritonPropagateNanGet(MlirContext ctx, int32_t value) { - return wrap(mlir::triton::PropagateNanAttr::get( - unwrap(ctx), static_cast(value))); -} - -MlirAttribute mlirTritonScaleDotElemTypeGet(MlirContext ctx, int32_t value) { - return wrap(mlir::triton::ScaleDotElemTypeAttr::get( - unwrap(ctx), static_cast(value))); -} - -MlirAttribute mlirTritonDescriptorReduceKindGet(MlirContext ctx, int32_t value) { - return wrap(mlir::triton::DescriptorReduceKindAttr::get( - unwrap(ctx), static_cast(value))); -} - -} // extern "C" diff --git a/mlir/dialects/ttir/ttir_capi.h b/mlir/dialects/ttir/ttir_capi.h deleted file mode 100644 index 7e5813c2..00000000 --- a/mlir/dialects/ttir/ttir_capi.h +++ /dev/null @@ -1,47 +0,0 @@ -#ifndef ZML_MLIR_DIALECTS_TTIR_TTIR_CAPI_H -#define ZML_MLIR_DIALECTS_TTIR_TTIR_CAPI_H - -#include -#include - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Triton, tt); - -// !tt.ptr -MlirType mlirTritonPointerTypeGet(MlirType pointee, int32_t address_space); -MlirType mlirTritonPointerTypeGetPointee(MlirType ptr); -int32_t mlirTritonPointerTypeGetAddressSpace(MlirType ptr); -bool mlirTritonTypeIsAPointer(MlirType t); - -// !tt.tensordesc> — result of tt.make_tensor_descriptor. -// `shape` is a `rank`-element array of int64_t block dims; `element_type` is -// the scalar element type; `shared_layout` may be a null MlirAttribute for none. -MlirType mlirTritonTensorDescTypeGet(intptr_t rank, const int64_t* shape, - MlirType element_type, - MlirAttribute shared_layout); -bool mlirTritonTypeIsATensorDesc(MlirType t); - -// I32EnumAttr getters -MlirAttribute mlirTritonProgramDimGet(MlirContext ctx, int32_t value); -MlirAttribute mlirTritonCacheModifierGet(MlirContext ctx, int32_t value); -MlirAttribute mlirTritonEvictionPolicyGet(MlirContext ctx, int32_t value); -MlirAttribute mlirTritonPaddingOptionGet(MlirContext ctx, int32_t value); -MlirAttribute mlirTritonRoundingModeGet(MlirContext ctx, int32_t value); -MlirAttribute mlirTritonInputPrecisionGet(MlirContext ctx, int32_t value); -MlirAttribute mlirTritonRMWOpGet(MlirContext ctx, int32_t value); -MlirAttribute mlirTritonMemSemanticGet(MlirContext ctx, int32_t value); -MlirAttribute mlirTritonMemSyncScopeGet(MlirContext ctx, int32_t value); -MlirAttribute mlirTritonPropagateNanGet(MlirContext ctx, int32_t value); -MlirAttribute mlirTritonScaleDotElemTypeGet(MlirContext ctx, int32_t value); -MlirAttribute mlirTritonDescriptorReduceKindGet(MlirContext ctx, int32_t value); - -#ifdef __cplusplus -} -#endif - -#endif // ZML_MLIR_DIALECTS_TTIR_TTIR_CAPI_H diff --git a/mlir/dialects/ub/BUILD.bazel b/mlir/dialects/ub/BUILD.bazel new file mode 100644 index 00000000..039ca5ff --- /dev/null +++ b/mlir/dialects/ub/BUILD.bazel @@ -0,0 +1,34 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:CAPIIR", + "@llvm-project//mlir:UBDialect", + ], +) + +zig_library( + name = "ub", + import_name = "mlir/dialects/ub", + main = "ub.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":ub", + "//mlir", + ], +) diff --git a/mlir/dialects/ub/capi.cc b/mlir/dialects/ub/capi.cc new file mode 100644 index 00000000..cf37a2c0 --- /dev/null +++ b/mlir/dialects/ub/capi.cc @@ -0,0 +1,8 @@ +//===- capi.cc - C API for the ub dialect ---------------------------------===// + +#include "capi.h" + +#include "mlir/CAPI/Registration.h" +#include "mlir/Dialect/UB/IR/UBOps.h" + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(UB, ub, mlir::ub::UBDialect) diff --git a/mlir/dialects/ub/capi.h b/mlir/dialects/ub/capi.h new file mode 100644 index 00000000..76765ad9 --- /dev/null +++ b/mlir/dialects/ub/capi.h @@ -0,0 +1,18 @@ +//===- capi.h - C API for the ub dialect --------------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_UB_CAPI_H +#define MLIR_DIALECTS_UB_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(UB, ub); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_UB_CAPI_H diff --git a/mlir/dialects/ub/test.zig b/mlir/dialects/ub/test.zig new file mode 100644 index 00000000..738d1378 --- /dev/null +++ b/mlir/dialects/ub/test.zig @@ -0,0 +1,23 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const ub = @import("mlir/dialects/ub"); + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(ub); + inline for (comptime std.meta.declarations(ub)) |decl| { + const T = @field(ub, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "registerDialect registers ub" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + ub.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + try std.testing.expect(ctx.isRegisteredOperation("ub.poison")); +} diff --git a/mlir/dialects/ub/ub.zig b/mlir/dialects/ub/ub.zig new file mode 100644 index 00000000..b5352257 --- /dev/null +++ b/mlir/dialects/ub/ub.zig @@ -0,0 +1,5 @@ +const mlir = @import("mlir"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("ub"); +} diff --git a/mlir/dialects/vector/BUILD.bazel b/mlir/dialects/vector/BUILD.bazel new file mode 100644 index 00000000..579fa599 --- /dev/null +++ b/mlir/dialects/vector/BUILD.bazel @@ -0,0 +1,36 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:CAPIIR", + "@llvm-project//mlir:VectorDialect", + ], +) + +zig_library( + name = "vector", + import_name = "mlir/dialects/vector", + main = "vector.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//stdx", + "@llvm-project//mlir:CAPIVector", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":vector", + "//mlir", + ], +) diff --git a/mlir/dialects/vector/capi.cc b/mlir/dialects/vector/capi.cc new file mode 100644 index 00000000..f22ad655 --- /dev/null +++ b/mlir/dialects/vector/capi.cc @@ -0,0 +1,51 @@ +//===- capi.cc - C API for the vector dialect -----------------------------===// + +#include "capi.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/Dialect/Vector/IR/VectorOps.h" + +static mlir::MLIRContext *loadVector(MlirContext context) { + mlir::MLIRContext *ctx = unwrap(context); + if (ctx) + ctx->getOrLoadDialect(); + return ctx; +} + +bool mlirAttributeIsAVectorCombiningKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirVectorCombiningKindAttrGet(MlirContext context, + uint32_t value) { + mlir::MLIRContext *ctx = loadVector(context); + std::optional kind = + mlir::vector::symbolizeCombiningKind(value); + if (!ctx || !kind) + return {nullptr}; + return wrap(mlir::vector::CombiningKindAttr::get(ctx, *kind)); +} + +uint32_t mlirVectorCombiningKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAVectorIteratorType(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirVectorIteratorTypeAttrGet(MlirContext context, + uint32_t value) { + mlir::MLIRContext *ctx = loadVector(context); + std::optional type = + mlir::vector::symbolizeIteratorType(value); + if (!ctx || !type) + return {nullptr}; + return wrap(mlir::vector::IteratorTypeAttr::get(ctx, *type)); +} + +uint32_t mlirVectorIteratorTypeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} diff --git a/mlir/dialects/vector/capi.h b/mlir/dialects/vector/capi.h new file mode 100644 index 00000000..75e12cbd --- /dev/null +++ b/mlir/dialects/vector/capi.h @@ -0,0 +1,36 @@ +//===- capi.h - C API for the vector dialect ----------------------*- C -*-===// +// +// C entry points for the `vector` dialect's enum attributes, which upstream's +// C API leaves out. + +#ifndef MLIR_DIALECTS_VECTOR_CAPI_H +#define MLIR_DIALECTS_VECTOR_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Enum attributes take and return the C++ enum's integer value; `Get` returns +// a null attribute for a value outside the enum. + +// #vector.kind<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsAVectorCombiningKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirVectorCombiningKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirVectorCombiningKindAttrGetValue(MlirAttribute attr); + +// #vector.iterator_type<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsAVectorIteratorType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirVectorIteratorTypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirVectorIteratorTypeAttrGetValue(MlirAttribute attr); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_VECTOR_CAPI_H diff --git a/mlir/dialects/vector/test.zig b/mlir/dialects/vector/test.zig new file mode 100644 index 00000000..ca737ae7 --- /dev/null +++ b/mlir/dialects/vector/test.zig @@ -0,0 +1,86 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const vector = @import("mlir/dialects/vector"); + +const operation_names: []const []const u8 = &.{ + "vector.bitcast", + "vector.broadcast", + "vector.compressstore", + "vector.constant_mask", + "vector.contract", + "vector.create_mask", + "vector.deinterleave", + "vector.expandload", + "vector.extract", + "vector.extract_strided_slice", + "vector.fma", + "vector.from_elements", + "vector.gather", + "vector.insert", + "vector.insert_strided_slice", + "vector.interleave", + "vector.load", + "vector.maskedload", + "vector.maskedstore", + "vector.multi_reduction", + "vector.reduction", + "vector.scatter", + "vector.shape_cast", + "vector.shuffle", + "vector.step", + "vector.store", + "vector.to_elements", + "vector.transfer_read", + "vector.transfer_write", + "vector.transpose", + "vector.type_cast", + "vector.vscale", + "vector.yield", +}; + +fn expectPrints(expected: []const u8, value: anytype) !void { + var buf: [128]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(expected, w.buffered()); +} + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(vector); + inline for (comptime std.meta.declarations(vector)) |decl| { + const T = @field(vector, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "vector enum attributes are built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + inline for (@typeInfo(vector.CombiningKind).@"enum".field_names) |name| { + const value: vector.CombiningKind = @field(vector.CombiningKind, name); + const attr = try vector.CombiningKindAttr.get(ctx, .{ .value = value }); + try std.testing.expectEqual(value, attr.value()); + try std.testing.expect(value.attribute(ctx).isA(vector.CombiningKindAttr) != null); + try expectPrints("#vector.kind<" ++ name ++ ">", attr); + } + inline for (@typeInfo(vector.IteratorType).@"enum".field_names) |name| { + const value: vector.IteratorType = @field(vector.IteratorType, name); + const attr = try vector.IteratorTypeAttr.get(ctx, .{ .value = value }); + try std.testing.expectEqual(value, attr.value()); + try expectPrints("#vector.iterator_type<" ++ name ++ ">", attr); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + vector.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/vector.zig b/mlir/dialects/vector/vector.zig similarity index 94% rename from mlir/dialects/vector.zig rename to mlir/dialects/vector/vector.zig index d2cca4f2..1696d04b 100644 --- a/mlir/dialects/vector.zig +++ b/mlir/dialects/vector/vector.zig @@ -4,6 +4,10 @@ const c = @import("c"); const mlir = @import("mlir"); const stdx = @import("stdx"); +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("vector"); +} + // ============================================================================= // Enum attributes // ============================================================================= @@ -48,11 +52,9 @@ pub const CombiningKindAttr = opaque { pub const isAFn = c.mlirAttributeIsAVectorCombiningKind; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "kind"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -63,7 +65,7 @@ pub const CombiningKindAttr = opaque { const result = c.mlirVectorCombiningKindAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) CombiningKind { + pub fn value(self: *const Self) CombiningKind { return @fromBackingInt(@intCast(c.mlirVectorCombiningKindAttrGetValue(self.ptr()))); } }; @@ -75,11 +77,9 @@ pub const IteratorTypeAttr = opaque { pub const isAFn = c.mlirAttributeIsAVectorIteratorType; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); pub const isA = M.isA; pub const mnemonic = "iterator_type"; - pub fn format(self: *const Self, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return self.attribute().format(writer); - } pub fn attribute(self: *const Self) *const mlir.Attribute { return @ptrCast(self); } @@ -90,7 +90,7 @@ pub const IteratorTypeAttr = opaque { const result = c.mlirVectorIteratorTypeAttrGet(ctx.ptr(), @backingInt(args.value)); return @ptrCast(result.ptr orelse return error.InvalidMlir); } - pub fn getValue(self: *const Self) IteratorType { + pub fn value(self: *const Self) IteratorType { return @fromBackingInt(@intCast(c.mlirVectorIteratorTypeAttrGetValue(self.ptr()))); } }; @@ -818,33 +818,3 @@ pub fn create_mask( .location = location, }); } - -test { - std.testing.refAllDecls(@This()); -} - -fn expectPrints(expected: []const u8, value: anytype) !void { - var buf: [128]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{value}); - try std.testing.expectEqualStrings(expected, w.buffered()); -} - -test "vector enum attributes are built through the C API" { - const ctx = try mlir.Context.init(.{}); - defer ctx.deinit(); - - inline for (@typeInfo(CombiningKind).@"enum".field_names) |name| { - const value: CombiningKind = @field(CombiningKind, name); - const attr = try CombiningKindAttr.get(ctx, .{ .value = value }); - try std.testing.expectEqual(value, attr.getValue()); - try std.testing.expect(value.attribute(ctx).isA(CombiningKindAttr) != null); - try expectPrints("#vector.kind<" ++ name ++ ">", attr); - } - inline for (@typeInfo(IteratorType).@"enum".field_names) |name| { - const value: IteratorType = @field(IteratorType, name); - const attr = try IteratorTypeAttr.get(ctx, .{ .value = value }); - try std.testing.expectEqual(value, attr.getValue()); - try expectPrints("#vector.iterator_type<" ++ name ++ ">", attr); - } -} diff --git a/mlir/dialects/vector_capi.cc b/mlir/dialects/vector_capi.cc deleted file mode 100644 index 0b412f66..00000000 --- a/mlir/dialects/vector_capi.cc +++ /dev/null @@ -1,44 +0,0 @@ -#include "vector_capi.h" - -#include "mlir/CAPI/IR.h" -#include "mlir/Dialect/Vector/IR/VectorOps.h" - -namespace { - -mlir::MLIRContext *loadVector(MlirContext ctx) { - mlir::MLIRContext *context = unwrap(ctx); - if (context) context->getOrLoadDialect(); - return context; -} - -} // namespace - -bool mlirAttributeIsAVectorCombiningKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirVectorCombiningKindAttrGet(MlirContext ctx, uint32_t value) { - mlir::MLIRContext *context = loadVector(ctx); - std::optional kind = mlir::vector::symbolizeCombiningKind(value); - if (!context || !kind) return {nullptr}; - return wrap(mlir::vector::CombiningKindAttr::get(context, *kind)); -} - -uint32_t mlirVectorCombiningKindAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsAVectorIteratorType(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirVectorIteratorTypeAttrGet(MlirContext ctx, uint32_t value) { - mlir::MLIRContext *context = loadVector(ctx); - std::optional type = mlir::vector::symbolizeIteratorType(value); - if (!context || !type) return {nullptr}; - return wrap(mlir::vector::IteratorTypeAttr::get(context, *type)); -} - -uint32_t mlirVectorIteratorTypeAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); -} diff --git a/mlir/dialects/vector_capi.h b/mlir/dialects/vector_capi.h deleted file mode 100644 index 7fae8e6f..00000000 --- a/mlir/dialects/vector_capi.h +++ /dev/null @@ -1,32 +0,0 @@ -// C entry points for the `vector` dialect's enum attributes, which upstream's -// C API leaves out. -#ifndef ZML_MLIR_DIALECTS_VECTOR_CAPI_H -#define ZML_MLIR_DIALECTS_VECTOR_CAPI_H - -#include -#include - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Enum attributes take and return the C++ enum's integer value; `Get` returns -// a null attribute for a value outside the enum. - -// #vector.kind<...> -MLIR_CAPI_EXPORTED bool mlirAttributeIsAVectorCombiningKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirVectorCombiningKindAttrGet(MlirContext ctx, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirVectorCombiningKindAttrGetValue(MlirAttribute attr); - -// #vector.iterator_type<...> -MLIR_CAPI_EXPORTED bool mlirAttributeIsAVectorIteratorType(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirVectorIteratorTypeAttrGet(MlirContext ctx, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirVectorIteratorTypeAttrGetValue(MlirAttribute attr); - -#ifdef __cplusplus -} -#endif - -#endif // ZML_MLIR_DIALECTS_VECTOR_CAPI_H diff --git a/mlir/mlir.h b/mlir/mlir.h index 89043c01..b897c82b 100644 --- a/mlir/mlir.h +++ b/mlir/mlir.h @@ -1,29 +1,15 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +//===- mlir.h - MLIR C API headers bound by the mlir module -------*- C -*-===// -#ifdef __cplusplus -extern "C" { -#endif +#ifndef MLIR_MLIR_H +#define MLIR_MLIR_H -void mlirRegisterFuncExtensions(MlirDialectRegistry registry); +#include "mlir-c/AffineExpr.h" +#include "mlir-c/AffineMap.h" +#include "mlir-c/BuiltinAttributes.h" +#include "mlir-c/BuiltinTypes.h" +#include "mlir-c/IR.h" +#include "mlir-c/IntegerSet.h" +#include "mlir-c/Pass.h" +#include "mlir-c/Transforms.h" -MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Affine, affine); - -#ifdef __cplusplus -} -#endif +#endif // MLIR_MLIR_H diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 83bec2f5..6176b2d4 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -42,6 +42,13 @@ test "variadic operation groups preserve order and dense array segment sizes" { try std.testing.expect(op.attributeByName("resultSegmentSizes").?.eql(.denseArray(ctx, .i32, &.{ 0, 2, 1 }))); } +test "nested symbol references print their path" { + const ctx = try Context.init(.{}); + defer ctx.deinit(); + const symbol = Attribute.symbolRef(ctx, "kernels", &.{FlatSymbolRefAttribute.init(ctx, "add")}); + try std.testing.expect(symbol.eql(try Attribute.parse(ctx, "@kernels::@add"))); +} + pub const Error = error{ /// Invalid Mlir was created. InvalidMlir, @@ -83,10 +90,6 @@ pub fn registerPasses(comptime passes: []const u8) void { @field(c, "mlirRegister" ++ passes ++ "Passes")(); } -pub fn registerFuncExtensions(registry: *DialectRegistry) void { - c.mlirRegisterFuncExtensions(registry.ptr()); -} - fn isPtrConst(comptime T: type) bool { return @typeInfo(T).pointer.attrs.@"const"; } @@ -104,7 +107,7 @@ pub fn Methods(comptime T: type, comptime NativeT: type) type { return .{ .ptr = @ptrCast(@constCast(self)) }; } - fn format(f: anytype) fn (self: *const T, writer: *std.Io.Writer) std.Io.Writer.Error!void { + pub fn format(f: anytype) fn (self: *const T, writer: *std.Io.Writer) std.Io.Writer.Error!void { return struct { fn format(self: *const T, writer: *std.Io.Writer) std.Io.Writer.Error!void { var ctx: StringCallbackCtx = .{ .writer = writer }; @@ -734,6 +737,10 @@ pub const Attribute = opaque { return @ptrCast(FlatSymbolRefAttribute.init(ctx, symbol)); } + pub fn symbolRef(ctx: *Context, root: []const u8, nested: []const *const FlatSymbolRefAttribute) *const Attribute { + return @ptrCast(SymbolRefAttribute.init(ctx, root, nested)); + } + pub fn unit(ctx: *Context) *const Attribute { return @ptrCast(UnitAttribute.get(ctx)); } @@ -1780,6 +1787,20 @@ pub const FlatSymbolRefAttribute = opaque { } }; +/// `@root::@nested...`. +pub const SymbolRefAttribute = opaque { + const M = Methods(SymbolRefAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsASymbolRef; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, root: []const u8, nested: []const *const FlatSymbolRefAttribute) *const SymbolRefAttribute { + return @ptrCast(c.mlirSymbolRefAttrGet(ctx.ptr(), stringRef(root), @intCast(nested.len), @ptrCast(nested)).ptr); + } +}; + pub const UnitAttribute = opaque { const M = Methods(UnitAttribute, c.MlirAttribute); diff --git a/mlir/register.cc b/mlir/register.cc deleted file mode 100644 index ee6b5dfe..00000000 --- a/mlir/register.cc +++ /dev/null @@ -1,13 +0,0 @@ -#include "mlir.h" - -#include -#include -#include -#include - -extern "C" void mlirRegisterFuncExtensions(MlirDialectRegistry registry) { - mlir::func::registerAllExtensions(*unwrap(registry)); -} - -MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Affine, affine, - mlir::affine::AffineDialect) diff --git a/platforms/tpu/BUILD.bazel b/platforms/tpu/BUILD.bazel index f3063c7a..a61cb7fa 100644 --- a/platforms/tpu/BUILD.bazel +++ b/platforms/tpu/BUILD.bazel @@ -33,7 +33,7 @@ zig_library( tags = ["manual"], visibility = ["//visibility:public"], deps = [ - "//kernels/mosaic_tpu:mosaic_tpu_builder", + "//dsl/mosaic_tpu:mosaic_tpu_builder", "//mlir", ], ) diff --git a/platforms/tpu/ragged_paged.zig b/platforms/tpu/ragged_paged.zig index ea44e812..ad99acfa 100644 --- a/platforms/tpu/ragged_paged.zig +++ b/platforms/tpu/ragged_paged.zig @@ -1,7 +1,7 @@ const std = @import("std"); const mlir = @import("mlir"); -const mtt = @import("kernels/mosaic_tpu/builder"); +const mtt = @import("dsl/mosaic_tpu/builder"); pub const Cfg = struct { // Shapes that vary at compile-of-kernel time. diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 435afb5c..0c9340f9 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -145,17 +145,20 @@ zig_library( ":xla_compile_options_upb", ":xla_data_upb", "//bazel", + "//dsl/cuda_tile:cuda_tile_builder", + "//dsl/cute:cute_builder", + "//dsl/fly:fly_builder", + "//dsl/mosaic_tpu:mosaic_tpu_builder", + "//dsl/triton:triton_builder", "//ffi:zig", - "//kernels/cuda_tile:cuda_tile_builder", - "//kernels/cute:cute_builder", - "//kernels/fly:fly_builder", - "//kernels/mosaic_tpu:mosaic_tpu_builder", - "//kernels/triton:triton_builder", "//mlir", - "//mlir/dialects", - "//mlir/dialects/cuda_tile", - "//mlir/dialects/fly", + "//mlir/dialects/func", + "//mlir/dialects/llvm", "//mlir/dialects/mosaic_tpu", + "//mlir/dialects/shardy", + "//mlir/dialects/stablehlo", + "//mlir/dialects/ttir", + "//mlir/dialects/vector", "//pjrt", "//platforms", "//platforms/tpu:ragged_paged", @@ -180,18 +183,6 @@ zig_library( }), ) -zig_library( - name = "moe_mosaic_tpu_gmm_ep", - import_name = "zml/moe/mosaic_tpu_kernels/gmm_ep", - main = "moe/mosaic_tpu_kernels/gmm_ep.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - ":zml", - "//kernels/mosaic_tpu:mosaic_tpu_builder", - ], -) - zig_test( name = "test", deps = ["//zml"], diff --git a/zml/Compiler.zig b/zml/Compiler.zig index d04fccec..5ee5e120 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -1,9 +1,11 @@ const std = @import("std"); const c = @import("c"); -const dialects = @import("mlir/dialects"); +const func = @import("mlir/dialects/func"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); +const shardy = @import("mlir/dialects/shardy"); +const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); const upb = @import("upb"); const zio = @import("zio"); @@ -60,10 +62,10 @@ fn mlirRegistry(io: std.Io) *mlir.DialectRegistry { mlir.registerPasses("Transforms"); const mlir_registry = mlir.DialectRegistry.init() catch unreachable; - inline for (.{ "func", "stablehlo", "sdy" }) |d| { - mlir.DialectHandle.fromString(d).insertDialect(mlir_registry); - } - mlir.registerFuncExtensions(mlir_registry); + func.registerDialect(mlir_registry); + stablehlo.registerDialect(mlir_registry); + shardy.registerDialect(mlir_registry); + func.registerExtensions(mlir_registry); mlir_global_registry = mlir_registry; } @@ -397,16 +399,16 @@ pub fn resolveMesh(compiler: *const Compiler, logical_axes: anytype) *const Shar , .{ logical_axes, stdx.fmt.slice(compiler.meshes) }); } -pub fn Typed(comptime func: anytype) type { +pub fn Typed(comptime func_: anytype) type { return struct { pub fn compile( allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: Options, - args: std.meta.ArgsTuple(@TypeOf(func)), + args: std.meta.ArgsTuple(@TypeOf(func_)), ) Error!Exe { - return Compiler.compile(allocator, io, platform, func, args, opts); + return Compiler.compile(allocator, io, platform, func_, args, opts); } }; } @@ -415,15 +417,15 @@ pub fn compile( allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, - comptime func: anytype, - args: std.meta.ArgsTuple(@TypeOf(func)), + comptime func_: anytype, + args: std.meta.ArgsTuple(@TypeOf(func_)), opts: Options, ) Error!Exe { return switch (platform.io_impl) { - .threaded => compileInternal(allocator, io, platform, func, args, opts), + .threaded => compileInternal(allocator, io, platform, func_, args, opts), .zio => try zio.blockInPlace(struct { fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const Platform, args_: @TypeOf(args), opts_: Options) Error!Exe { - return try compileInternal(allocator_, io_, platform_, func, args_, opts_); + return try compileInternal(allocator_, io_, platform_, func_, args_, opts_); } }.call, .{ allocator, io, platform, args, opts }), }; @@ -433,8 +435,8 @@ pub fn compileInternal( allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, - comptime func: anytype, - args: std.meta.ArgsTuple(@TypeOf(func)), + comptime func_: anytype, + args: std.meta.ArgsTuple(@TypeOf(func_)), opts: Options, ) Error!Exe { // TODO: Here we have somewhat of a requirement @@ -460,7 +462,7 @@ pub fn compileInternal( var compiler: Compiler = .init(allocator, io, platform, opts); defer compiler.deinit(); - var result = compiler.emitMlir(func, args) catch |err| switch (err) { + var result = compiler.emitMlir(func_, args) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => unreachable, }; @@ -518,7 +520,6 @@ pub fn compileInternal( /// Declare all logical meshes used by this program fn addPartitionerOperations(ctx: *Compiler) !void { - const allocator = ctx.arena.allocator(); const mlir_ctx = ctx.mlir_ctx; const module = ctx.module; @@ -528,11 +529,8 @@ fn addPartitionerOperations(ctx: *Compiler) !void { }, .shardy => { for (ctx.meshes) |shd| { - const attr_str = try shd.sdyMeshAttr(allocator); - defer allocator.free(attr_str); - const name = shd.name; - const mesh_attr = try mlir.Attribute.parse(mlir_ctx, attr_str); + const mesh_attr = shd.sdyMeshAttr(mlir_ctx); const mesh_op = mlir.Operation.make(mlir_ctx, "sdy.mesh", .{ .attributes = &.{ @@ -595,7 +593,7 @@ pub const TensorInfo = struct { const AttributeList = stdx.BoundedArray(mlir.NamedAttribute, 3); -fn emitMlir(compiler: *Compiler, comptime func: anytype, args: std.meta.ArgsTuple(@TypeOf(func))) !EmitMlirResult { +fn emitMlir(compiler: *Compiler, comptime func_: anytype, args: std.meta.ArgsTuple(@TypeOf(func_))) !EmitMlirResult { const module = mlir.Module.init(.unknown(compiler.mlir_ctx)); errdefer module.deinit(); @@ -611,7 +609,7 @@ fn emitMlir(compiler: *Compiler, comptime func: anytype, args: std.meta.ArgsTupl var result = blk: { compiler.activate(); defer compiler.deactivate(); - break :blk @call(.auto, func, args); + break :blk @call(.auto, func_, args); }; var output_info = try collectOutputInfo(compiler, fn_scope, &result); @@ -705,7 +703,7 @@ fn finalizeMlirFunc(compiler: *Compiler, fn_scope: *Scope, input_info: std.Multi const arena = arena_state.allocator(); const mlir_ctx = compiler.mlir_ctx; - const fn_return = dialects.func.returns(mlir_ctx, output_info.items(.value), .unknown(mlir_ctx)); + const fn_return = func.returns(mlir_ctx, output_info.items(.value), .unknown(mlir_ctx)); _ = fn_return.appendTo(fn_scope.block); // Resolve donations @@ -744,7 +742,7 @@ fn finalizeMlirFunc(compiler: *Compiler, fn_scope: *Scope, input_info: std.Multi const output_types = try arena.alloc(*const mlir.Type, output_info.len); for (output_types, output_info.items(.value)) |*output_type, value| output_type.* = value.type_(); - const mlir_func = dialects.func.func(mlir_ctx, .{ + const mlir_func = func.func(mlir_ctx, .{ .name = "main", .args = input_types, .results = output_types, @@ -767,14 +765,14 @@ fn unpack(compiler: *Compiler, scope: *Scope, og_shape: Shape, tensor_value: *co const elem_per_bytes = @divExact(8, og_shape.dtype().bitSizeOf()); const intermediary_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, og_shape.splitAxis(-1, .{ -1, elem_per_bytes })); - const bit_cast_op = dialects.stablehlo.bitcast_convert( + const bit_cast_op = stablehlo.bitcast_convert( compiler.mlir_ctx, tensor_value, intermediary_type, .unknown(compiler.mlir_ctx), ).appendTo(scope.block); - const reshape_op = dialects.stablehlo.reshape( + const reshape_op = stablehlo.reshape( compiler.mlir_ctx, bit_cast_op.result(0), true_type, @@ -789,14 +787,14 @@ fn repack(compiler: *Compiler, scope: *Scope, og_shape: Shape, value: *const mli const elem_per_bytes = @divExact(8, og_shape.dtype().bitSizeOf()); const intermediary_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, og_shape.splitAxis(-1, .{ -1, elem_per_bytes })); - const reshape_op = dialects.stablehlo.reshape( + const reshape_op = stablehlo.reshape( compiler.mlir_ctx, value, intermediary_type, .unknown(compiler.mlir_ctx), ).appendTo(scope.block); - const bit_cast_op = dialects.stablehlo.bitcast_convert( + const bit_cast_op = stablehlo.bitcast_convert( compiler.mlir_ctx, reshape_op.result(0), packed_type, @@ -841,7 +839,7 @@ pub fn tensorShardingAttr( ) error{OutOfMemory}!*const mlir.Attribute { const mesh_ = shape._sharding.mesh orelse compiler.platform.replicated_mesh; return switch (compiler.partitioner) { - .shardy => (try mesh_.sdyShardingAttrForShape(allocator, compiler.mlir_ctx, shape)).asAttr(), + .shardy => (try mesh_.sdyShardingAttrForShape(allocator, compiler.mlir_ctx, shape)).attribute(), .gspmd => mesh_.gspmdShardingAttrForShape(allocator, compiler.mlir_ctx, shape) catch |err| switch (err) { error.WriteFailed => error.OutOfMemory, // We're writing to memory error.OutOfMemory => error.OutOfMemory, diff --git a/zml/Sharding.zig b/zml/Sharding.zig index bfb70240..f5ca994b 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -2,10 +2,11 @@ const std = @import("std"); const builtin = @import("builtin"); -const dialects = @import("mlir/dialects"); +const func = @import("mlir/dialects/func"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); const platforms = @import("platforms"); +const shardy = @import("mlir/dialects/shardy"); const stdx = @import("stdx"); const Platform = @import("platform.zig").Platform; @@ -194,11 +195,11 @@ pub fn sdyPerValueShardingAttr( shapes: []const Shape, mesh: *const Mesh, ) error{OutOfMemory}!*const mlir.Attribute { - const meshe_attrs = try allocator.alloc(*const dialects.shardy.TensorShardingAttribute, shapes.len); + const meshe_attrs = try allocator.alloc(*const shardy.TensorShardingAttr, shapes.len); for (meshe_attrs, shapes) |*attr, shape| { attr.* = try mesh.sdyShardingAttrForShape(allocator, ctx, shape); } - return dialects.shardy.TensorShardingPerValueAttribute.init(ctx, meshe_attrs).asAttr(); + return shardy.TensorShardingPerValueAttr.get(ctx, meshe_attrs).attribute(); } pub fn sdyManualAxesAttr( @@ -250,7 +251,7 @@ pub fn sdyManualAxesAttr( axes[i] = mlir.StringAttribute.init(ctx, axis_name); } - return dialects.shardy.ManualAxesAttribute.init(ctx, axes).asAttr(); + return shardy.ManualAxesAttr.get(ctx, axes).attribute(); } pub const Partitioning = packed struct { @@ -1636,25 +1637,20 @@ pub const Mesh = struct { return @divExact(self.numDevices(), self.numPartitions()); } - pub fn sdyMeshAttr(self: *const Mesh, allocator: std.mem.Allocator) ![]const u8 { - var out: std.Io.Writer.Allocating = .init(allocator); - errdefer out.deinit(); - - try out.writer.writeAll("#sdy.mesh<["); + pub fn sdyMeshAttr(self: *const Mesh, ctx: *mlir.Context) *const mlir.Attribute { + var axes: stdx.BoundedArray(*const shardy.MeshAxisAttr, MAX_MESH_RANK + 1) = .empty; var tile_count: usize = 1; for (self.logical.axes.slice(), 0..) |axis, i| { - if (i > 0) try out.writer.writeAll(", "); const size = self.grid.axis_sizes[i]; tile_count *= size; - try out.writer.print("\"{s}\"={d}", .{ std.mem.span(axis), size }); + axes.appendAssumeCapacity(.get(ctx, .{ .name = std.mem.span(axis), .size = @intCast(size) })); } const replicas = @divExact(self.grid.device_count, tile_count); if (replicas > 1) { - try out.writer.print(", \"{s}\"={d}", .{ self.sdyReplicaAxis(), replicas }); + axes.appendAssumeCapacity(.get(ctx, .{ .name = self.sdyReplicaAxis(), .size = @intCast(replicas) })); } - try out.writer.writeAll("], device_ids=["); var replica_indices: [Platform.MAX_NUM_DEVICES]usize = @splat(0); - var ids: [Platform.MAX_NUM_DEVICES]usize = undefined; + var ids: [Platform.MAX_NUM_DEVICES]i64 = undefined; for (0..self.grid.device_count) |partition_index| { const coords: [MAX_MESH_RANK]u8 = self.grid.logical_coords[partition_index]; var tile_index: usize = 0; @@ -1662,15 +1658,10 @@ pub const Mesh = struct { tile_index = tile_index * self.grid.axis_sizes[axis] + coords[axis]; } // Device IDs here are executable partition indices, not PJRT device IDs. - ids[tile_index * replicas + replica_indices[tile_index]] = partition_index; + ids[tile_index * replicas + replica_indices[tile_index]] = @intCast(partition_index); replica_indices[tile_index] += 1; } - for (ids[0..self.grid.device_count], 0..) |id, i| { - if (i > 0) try out.writer.writeAll(", "); - try out.writer.print("{d}", .{id}); - } - try out.writer.writeAll("]>"); - return try out.toOwnedSlice(); + return shardy.meshAttr(ctx, .{ .axes = axes.constSlice(), .device_ids = ids[0..self.grid.device_count] }); } fn sdyReplicaAxis(self: *const Mesh) []const u8 { @@ -1690,11 +1681,11 @@ pub const Mesh = struct { parent_allocator: std.mem.Allocator, ctx: *mlir.Context, shape: Shape, - ) error{OutOfMemory}!*const dialects.shardy.TensorShardingAttribute { + ) error{OutOfMemory}!*const shardy.TensorShardingAttr { var arena = try stdx.arenaWithCapacity(parent_allocator, 1024); defer arena.deinit(); const allocator = arena.allocator(); - const dimensions = try allocator.alloc(*const dialects.shardy.DimensionShardingAttribute, shape.rank()); + const dimensions = try allocator.alloc(*const shardy.DimensionShardingAttr, shape.rank()); var used: [MAX_MESH_RANK]bool = @splat(false); for (0.., dimensions) |ax, *d| { d.* = switch (shape.partition(ax)) { @@ -1709,7 +1700,7 @@ pub const Mesh = struct { }; } - var replicated_axes: stdx.BoundedArray(*const dialects.shardy.AxisRefAttribute, MAX_MESH_RANK + 1) = .empty; + var replicated_axes: stdx.BoundedArray(*const shardy.AxisRefAttr, MAX_MESH_RANK + 1) = .empty; var tile_count: usize = 1; for (data.logical.axes.slice(), 0..) |axis, i| { tile_count *= data.grid.axis_sizes[i]; @@ -1719,7 +1710,7 @@ pub const Mesh = struct { replicated_axes.appendAssumeCapacity(.named(ctx, data.sdyReplicaAxis())); } - return .init(ctx, .{ .mesh = data.name, .dimensions = dimensions, .replicated_axes = replicated_axes.slice() }); + return .get(ctx, .{ .mesh = data.name, .dimensions = dimensions, .replicated_axes = replicated_axes.slice() }); } pub fn gspmdShardingAttrForShape(self: *const Mesh, parent_allocator: std.mem.Allocator, ctx: *mlir.Context, shape: Shape) !*const mlir.Attribute { @@ -2447,8 +2438,8 @@ const ShardingTest = struct { errdefer std.log.warn("Expected shardy annotation failed", .{}); const registry: *mlir.DialectRegistry = try .init(); defer registry.deinit(); - registry.registerDialect("sdy"); - mlir.registerFuncExtensions(registry); + shardy.registerDialect(registry); + func.registerExtensions(registry); var ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); defer ctx.deinit(); @@ -2457,7 +2448,7 @@ const ShardingTest = struct { const actual_attr = try mesh.sdyShardingAttrForShape(self.allocator, ctx, s.shape); var writer: std.Io.Writer.Allocating = .init(self.allocator); defer writer.deinit(); - try actual_attr.asAttr().format(&writer.writer); + try actual_attr.attribute().format(&writer.writer); try std.testing.expectEqualStrings(expected_attr, writer.written()); } @@ -3075,15 +3066,16 @@ test "Shardy meshes preserve logical order and unused physical replicas" { const mesh_ = try Mesh.init("logical", &physical, .mesh(.{ .batch = .balanced, .model = .balanced }), .parseBindings(.{ .batch = .link_z, .model = .link_x })); const registry: *mlir.DialectRegistry = try .init(); defer registry.deinit(); - registry.registerDialect("sdy"); + shardy.registerDialect(registry); const ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); defer ctx.deinit(); ctx.loadAllAvailableDialects(); - const mesh_attr = try mesh_.sdyMeshAttr(allocator); - _ = try mlir.Attribute.parse(ctx, mesh_attr); + var text: std.Io.Writer.Allocating = .init(allocator); + defer text.deinit(); + try text.writer.print("{f}", .{mesh_.sdyMeshAttr(ctx)}); try std.testing.expectEqualStrings( "#sdy.mesh<[\"batch\"=2, \"model\"=2, \"__replica_0\"=2], device_ids=[0, 2, 4, 6, 1, 3, 5, 7]>", - mesh_attr, + text.written(), ); } diff --git a/zml/kernel.zig b/zml/kernel.zig index 204f224d..e5b8aa9c 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -1,14 +1,15 @@ const std = @import("std"); -const cuda_tile_builder = @import("kernels/cuda_tile/builder"); -const cute_builder = @import("kernels/cute/builder"); -const dialects = @import("mlir/dialects"); -const fly_builder = @import("kernels/fly/builder"); +const cuda_tile_builder = @import("dsl/cuda_tile/builder"); +const cute_builder = @import("dsl/cute/builder"); +const fly_builder = @import("dsl/fly/builder"); +const func = @import("mlir/dialects/func"); const mlir = @import("mlir"); -const mosaic_tpu_builder = @import("kernels/mosaic_tpu/builder"); +const mosaic_tpu_builder = @import("dsl/mosaic_tpu/builder"); +const mosaic_tpu_dialect = @import("mlir/dialects/mosaic_tpu"); +const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); -const tpu_dialect = @import("mlir/dialects/mosaic_tpu"); -const triton_builder = @import("kernels/triton/builder"); +const triton_builder = @import("dsl/triton/builder"); const Compiler = @import("Compiler.zig"); const DataType = @import("dtype.zig").DataType; @@ -25,17 +26,15 @@ fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); } -fn makeKernelContext(comptime dialects_needed: []const []const u8) std.mem.Allocator.Error!*mlir.Context { +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { mlir.registerPasses("Transforms"); const registry = try mlir.DialectRegistry.init(); defer registry.deinit(); - inline for (dialects_needed) |d| { - mlir.DialectHandle.fromString(d).insertDialect(registry); - } + registerDialects(registry); - mlir.registerFuncExtensions(registry); + func.registerExtensions(registry); const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); ctx.loadAllAvailableDialects(); @@ -46,8 +45,8 @@ fn makeKernelContext(comptime dialects_needed: []const []const u8) std.mem.Alloc fn resolveOutputOperandAliases( aliases_opt: anytype, operand_offset: i64, -) stdx.BoundedArray(dialects.stablehlo.CustomCallOpts.OutputOperandAlias, dialects.stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(dialects.stablehlo.CustomCallOpts.OutputOperandAlias, dialects.stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; if (aliases_opt) |a| { inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { @@ -68,7 +67,7 @@ pub const triton = struct { pub const FinishError = triton_builder.FinishError; pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(&triton_builder.dialects_needed); + return makeKernelContext(triton_builder.registerDialects); } pub fn from(dt: DataType) DType { @@ -181,7 +180,7 @@ pub const mosaic_tpu = struct { const FinishError = mosaic_tpu_builder.FinishError; pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(&mosaic_tpu_builder.dialects_needed); + return makeKernelContext(mosaic_tpu_builder.registerDialects); } pub fn from(dt: DataType) DType { @@ -332,7 +331,7 @@ pub const mosaic_tpu = struct { const module = try mlir.Module.parse(ctx, ir); defer module.deinit(); - tpu_dialect.registerMosaicSerdePass(); + mosaic_tpu_dialect.registerMosaicSerdePass(); const prev_allow_unregistered = ctx.allowUnregisteredDialects(); ctx.setAllowUnregisteredDialects(true); defer ctx.setAllowUnregisteredDialects(prev_allow_unregistered); @@ -376,7 +375,7 @@ pub const mosaic_tpu = struct { inputs: []const *const mlir.Value, result_types: []const *const mlir.Type, backend_config: []const u8, - aliases: []const dialects.stablehlo.CustomCallOpts.OutputOperandAlias, + aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias, dynamic_grid_bounds: []const Tensor, additional_attributes: []const mlir.NamedAttribute, }; @@ -384,11 +383,11 @@ pub const mosaic_tpu = struct { fn callTpuCustomCall(args: TpuCustomCallArgs) *mlir.Operation { const cur = Compiler.current(); - var all_inputs: stdx.BoundedArray(*const mlir.Value, dialects.stablehlo.CustomCallOpts.MAX_OPERANDS) = .empty; + var all_inputs: stdx.BoundedArray(*const mlir.Value, stablehlo.CustomCallOpts.MAX_OPERANDS) = .empty; for (args.dynamic_grid_bounds) |t| all_inputs.appendAssumeCapacity(t.value()); all_inputs.appendSliceAssumeCapacity(args.inputs); - return dialects.stablehlo.custom_call( + return stablehlo.custom_call( cur.mlir_ctx, all_inputs.constSlice(), args.result_types, @@ -414,7 +413,7 @@ pub const cuda_tile = struct { pub const Arch = cuda_tile_builder.Arch; pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(&cuda_tile_builder.dialects_needed); + return makeKernelContext(cuda_tile_builder.registerDialects); } pub fn from(dt: DataType) DType { @@ -527,7 +526,7 @@ pub const cute = struct { pub const BlockScaledMmaConfig = cute_builder.BlockScaledMmaConfig; pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(&cute_builder.dialects_needed); + return makeKernelContext(cute_builder.registerDialects); } pub fn from(dt: DataType) DType { @@ -735,7 +734,7 @@ pub const fly = struct { pub const TiledMma = fly_builder.TiledMma; pub const Arch = fly_builder.Arch; pub const MmaFlavor = fly_builder.MmaFlavor; - pub const rocdl = fly_builder.rocdl; + pub const fly_rocdl = fly_builder.fly_rocdl; /// The matrix atom this device provides, or null when it has none: WMMA /// arrived with RDNA3, so gfx1030 has no matrix instruction at all. @@ -760,7 +759,7 @@ pub const fly = struct { pub const it = fly_builder.it; pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(&fly_builder.dialects_needed); + return makeKernelContext(fly_builder.registerDialects); } pub fn from(dt: DataType) DType { diff --git a/zml/kernels/moe/cute/persistent_mxfp4.zig b/zml/kernels/moe/cute/persistent_mxfp4.zig index 0e18af45..d1677a8a 100644 --- a/zml/kernels/moe/cute/persistent_mxfp4.zig +++ b/zml/kernels/moe/cute/persistent_mxfp4.zig @@ -390,7 +390,7 @@ fn Layouts(comptime tile: Tile) type { fn sfBasis(b: *B) cute.LayoutSpec { return b.layoutSpec( .{ .{ 2, 4, 32 }, 1, 1, 1 }, - .{ .{ b.basis(1, 1, .{ 1, 0, 1 }), b.basis(1, 1, .{ 1, 0, 0 }), b.basis(1, 1, .{ 0, 0, 0 }) }, b.basis(1, 1, .{ 1, 0 }), b.basis(1, 1, .{ 1, 1 }), b.basis(1, 1, .{ 1, 2 }) }, + .{ .{ b.basis(1, 1, .{ 1, 0, 1 }), b.basis(1, 1, .{ 0, 0, 1 }), b.basis(1, 1, .{ 0, 0, 0 }) }, b.basis(1, 1, .{ 0, 1 }), b.basis(1, 1, .{ 1, 1 }), b.basis(1, 1, .{ 2, 1 }) }, ); } fn tmaA(b: *B, cfg: Config) cute.TmaConfig { @@ -507,7 +507,10 @@ fn buildTiledProgram(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: b.setFunctionAttribute("nvvm.minctasm", .int(b.ctx, .i32, 1)); b.setFunctionAttribute("smem.partition_num", .int(b.ctx, .i32, 2)); b.setFunctionAttribute("smem.hint_smem_base_uniform", .unit(b.ctx)); - b.setFunctionAttribute("cu_attrs", b.parseAttribute("{max_dynamic_shared_size_bytes = #cuda.dev_max_shared_memory_optin, non_portable_cluster_size_allowed = 1 : i32}")); + b.setFunctionAttribute("cu_attrs", .dict(b.ctx, &.{ + .named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin()), + .named(b.ctx, "non_portable_cluster_size_allowed", .int(b.ctx, .i32, 1)), + })); // Prefetch every tensor-map descriptor before the specialized warps begin // the persistent pipeline. This is the work PDL buys us: it runs while the @@ -612,18 +615,18 @@ fn HostArgs(comptime epilogue: Epilogue) type { /// consumers see them. Per-stage views are slices of these. fn stagedOperands(b: *B, comptime tile: Tile, storage: StageStorage) StagedOperands { const L = Layouts(tile); - const a_full = b.makeViewTyped( + const a_full = b.makeView( b.recastSwizzledPointer(storage.a, .i8, .smem, 1024, L.ab_swizzle), b.staticLayout(L.aSmem(b, true)), b.memrefTypeFromPointer(b.swizzledPtrTy(.i8, .smem, 1024, L.ab_swizzle) catch @panic("bad A shared pointer"), b.layoutType(L.aSmem(b, true))), ); - const b_full = b.makeViewTyped( + const b_full = b.makeView( b.recastSwizzledPointer(storage.b, .f8e4m3fn, .smem, 1024, L.ab_swizzle), b.staticLayout(L.bSmem(b, true)), b.memrefTypeFromPointer(b.swizzledPtrTy(.f8e4m3fn, .smem, 1024, L.ab_swizzle) catch @panic("bad B shared pointer"), b.layoutType(L.bSmem(b, true))), ); - const sfa_full = b.makeViewTyped(storage.sfa, b.staticLayout(L.sfSmem(b, true)), b.memrefType(.f8e8m0fnu, .smem, 1024, b.layoutType(L.sfSmem(b, true)))); - const sfb_full = b.makeViewTyped(storage.sfb, b.staticLayout(L.sfSmem(b, true)), b.memrefType(.f8e8m0fnu, .smem, 1024, b.layoutType(L.sfSmem(b, true)))); + const sfa_full = b.makeView(storage.sfa, b.staticLayout(L.sfSmem(b, true)), b.memrefType(.f8e8m0fnu, .smem, 1024, b.layoutType(L.sfSmem(b, true)))); + const sfb_full = b.makeView(storage.sfb, b.staticLayout(L.sfSmem(b, true)), b.memrefType(.f8e8m0fnu, .smem, 1024, b.layoutType(L.sfSmem(b, true)))); return .{ .a = a_full, .b = b_full, .sfa = sfa_full, .sfb = sfb_full }; } @@ -635,13 +638,13 @@ fn stageBuffersAt(b: *B, comptime tile: Tile, storage: StageStorage, stage_index // pointer loses the staged descriptor semantics at ring wrap. const full = stagedOperands(b, tile, storage); const all = cute.AlgebraToken.all; - const a = b.sliceTyped(full.a, .{ all, all, all, stage_index }, b.memrefTypeFromPointer(b.swizzledPtrTy(.i8, .smem, 1024, L.ab_swizzle) catch @panic("bad A stage pointer"), b.layoutType(L.aSmem(b, false)))); - const rhs = b.sliceTyped(full.b, .{ all, all, all, stage_index }, b.memrefTypeFromPointer(b.swizzledPtrTy(.f8e4m3fn, .smem, 1024, L.ab_swizzle) catch @panic("bad B stage pointer"), b.layoutType(L.bSmem(b, false)))); + const a = b.slice(full.a, .{ all, all, all, stage_index }, b.memrefTypeFromPointer(b.swizzledPtrTy(.i8, .smem, 1024, L.ab_swizzle) catch @panic("bad A stage pointer"), b.layoutType(L.aSmem(b, false)))); + const rhs = b.slice(full.b, .{ all, all, all, stage_index }, b.memrefTypeFromPointer(b.swizzledPtrTy(.f8e4m3fn, .smem, 1024, L.ab_swizzle) catch @panic("bad B stage pointer"), b.layoutType(L.bSmem(b, false)))); // One scale stage is 512 bytes, so slicing a stage out of the ring keeps // that alignment, not the array's. const sf_stage_align = Tile.sf_stage_bytes; - const sfa = b.sliceTyped(full.sfa, .{ all, all, all, stage_index }, b.memrefType(.f8e8m0fnu, .smem, sf_stage_align, b.layoutType(L.sfSmem(b, false)))); - const sfb = b.sliceTyped(full.sfb, .{ all, all, all, stage_index }, b.memrefType(.f8e8m0fnu, .smem, sf_stage_align, b.layoutType(L.sfSmem(b, false)))); + const sfa = b.slice(full.sfa, .{ all, all, all, stage_index }, b.memrefType(.f8e8m0fnu, .smem, sf_stage_align, b.layoutType(L.sfSmem(b, false)))); + const sfb = b.slice(full.sfb, .{ all, all, all, stage_index }, b.memrefType(.f8e8m0fnu, .smem, sf_stage_align, b.layoutType(L.sfSmem(b, false)))); // `tma_partition` preserves a singleton CTA mode. Keep it in the copy // views: it is part of the executable TMA atom's expected algebra even @@ -681,6 +684,9 @@ fn issueTmaStage( ) void { const n = tile.n; const all = cute.AlgebraToken.all; + const dynamic = cute.AlgebraToken.dynamic; + const div_128: cute.ConstrainedDynamic = .{ .divisible_by = 128 }; + const div_n: cute.ConstrainedDynamic = .{ .divisible_by = @intCast(n) }; const a_copy_layout = b.staticLayout(b.layoutSpec( .{ .{ .{ 128, 128 }, 1 }, .{1} }, .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, .{0} }, @@ -697,47 +703,47 @@ fn issueTmaStage( // partitioning produces a tensor over (tile, M, K, expert/group); first // select the output tile and expert/group, then select the K stage. The // remaining iterator carries the tensor-map coordinates needed by copy. - const a_tile_k = b.sliceTyped( + const a_tile_k = b.slice( partitioned_a_target, .{ all, tile_m, all, expert }, - b.coordTensorTypePayload("(0,?{div=128},?)", b.layoutSpec( + b.coordTensorTypeWithTuple(.{ 0, div_128, dynamic }, b.layoutSpec( .{ .{ .{ 128, 128 }, 1 }, reduction_tiles }, .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(128, 1, .{0}) }, )), ); - const a_tile = b.sliceTyped( + const a_tile = b.slice( a_tile_k, .{ all, k_tile }, - b.coordTensorTypePayload("(?{div=128},?{div=128},?)", b.layoutSpec( + b.coordTensorTypeWithTuple(.{ div_128, div_128, dynamic }, b.layoutSpec( .{.{ .{ 128, 128 }, 1 }}, .{.{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }}, )), ); - const a_source = b.rebuildCoordTensorIterator(3, a_tile, "(?{div=128},?{div=128},?)", .{ 128, 128, 1 }, a_copy_layout); + const a_source = b.rebuildCoordTensorIterator(3, a_tile, .{ div_128, div_128, dynamic }, .{ 128, 128, 1 }, a_copy_layout); // B's tile spans all of a group's rows, so its N-tile index is zero. Pass // it as a value rather than a literal: a statically zero leaf collapses out // of the coordinate tuple, and the TMA lowering then shifts the remaining // tensor-map coordinates, silently reading group zero for every group. const b_tile_n = b.cst(.i32, 0); - const b_tile_k = b.sliceTyped( + const b_tile_k = b.slice( partitioned_b_target, .{ all, b_tile_n, all, group }, - b.coordTensorTypePayload(fmt(b, "(0,?{{div={d}}},?)", .{n}), b.layoutSpec( + b.coordTensorTypeWithTuple(.{ 0, div_n, dynamic }, b.layoutSpec( .{ .{ .{ 128, n }, 1 }, reduction_tiles }, .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(128, 1, .{0}) }, )), ); - const b_tile = b.sliceTyped( + const b_tile = b.slice( b_tile_k, .{ all, k_tile }, - b.coordTensorTypePayload(fmt(b, "(?{{div=128}},?{{div={d}}},?)", .{n}), b.layoutSpec( + b.coordTensorTypeWithTuple(.{ div_128, div_n, dynamic }, b.layoutSpec( .{.{ .{ 128, n }, 1 }}, .{.{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }}, )), ); - const b_source = b.rebuildCoordTensorIterator(3, b_tile, fmt(b, "(?{{div=128}},?{{div={d}}},?)", .{n}), .{ 128, n, 1 }, b_copy_layout); - const sfa_source = b.makeCoordTensor(.{ 0, tile_m, k_tile, expert }, b.staticCoordinateLayout(4, sf_copy_spec)); - const sfb_source = b.makeCoordTensor(.{ 0, b.cst(.i32, 0), k_tile, group }, b.staticCoordinateLayout(4, sf_copy_spec)); + const b_source = b.rebuildCoordTensorIterator(3, b_tile, .{ div_128, div_n, dynamic }, .{ 128, n, 1 }, b_copy_layout); + const sfa_source = b.makeCoordTensor(.{ 0, tile_m, k_tile, expert }, b.staticLayout(sf_copy_spec)); + const sfb_source = b.makeCoordTensor(.{ 0, b.cst(.i32, 0), k_tile, group }, b.staticLayout(sf_copy_spec)); var elected = b.openIf(b.electSync()); b.mbarrierArriveExpectTx(stage.load_barrier, tile.transactionBytes()); @@ -749,7 +755,7 @@ fn issueTmaStage( } fn barrierAt(b: *B, base: V, stage: V) V { - return b.addOffsetTyped( + return b.addOffset( base, b.makeIntTuple(.{stage}), b.ptrTy(.i64, .smem, 8) catch @panic("bad pipeline barrier pointer"), @@ -872,7 +878,7 @@ fn peekBarrierIfRemaining(b: *B, base: V, count: V, limit: usize, state: Pipelin fn accumulatorTmemAt(b: *B, comptime tile: Tile, tmem: V, stage: V) V { // The stage offset is a whole tile width, which `cute.add_offset` needs to // know: it derives the result alignment from what it knows of the offset. - return b.addOffsetTyped( + return b.addOffset( tmem, b.makeIntTupleDivBy(stage.mul(tile.n), @intCast(tile.n)), b.ptrTy(.f32, .tmem, 16) catch @panic("bad accumulator TMEM pointer"), @@ -891,7 +897,7 @@ fn sharedPointerAt( ) V { const moved = comptime movedAlignment(byte_offset); comptime std.debug.assert(moved >= alignment); - const byte_ptr = b.addOffsetTyped( + const byte_ptr = b.addOffset( base, b.makeIntTuple(.{byte_offset}), b.ptrTy(.i8, .smem, moved) catch @panic("bad shared byte pointer"), @@ -1097,44 +1103,45 @@ const StagedMxfp4Mainloop = struct { pub fn atStage(self: StagedMxfp4Mainloop, b: *B, index: V, accumulator_tmem: V) Mxfp4Mainloop { const all = cute.AlgebraToken.all; - const a = b.sliceTyped( + const a = b.slice( self.a, .{ all, all, all, index }, - b.parseType("!cute_nvgpu.smem_desc_view"), + b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4 }, .{ 0, 0, 2 })), ); - const rhs = b.sliceTyped( + const rhs = b.slice( self.b, .{ all, all, all, index }, - b.parseType("!cute_nvgpu.smem_desc_view"), + b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4 }, .{ 0, 0, 2 })), ); - const sfa_stage = b.sliceTyped( + const sfa_stage = b.slice( self.sfa_source, .{ all, all, all, all, index }, - b.parseType("!cute_nvgpu.smem_desc_view"), + b.smemDescViewType(b.layoutSpec(.{ .{ .{ 32, 1, 1, 4 }, 1 }, 1, 1, 1 }, .{ .{ .{ 1, 1, 1, 0 }, 0 }, 0, 0, 0 })), ); - const sfb_stage = b.sliceTyped( + const sfb_stage = b.slice( self.sfb_source, .{ all, all, all, all, index }, - b.parseType("!cute_nvgpu.smem_desc_view"), + b.smemDescViewType(b.layoutSpec(.{ .{ .{ 32, 1, 1, 4 }, 1 }, 1, 1, 1 }, .{ .{ .{ 1, 1, 1, 0 }, 0 }, 0, 0, 0 })), ); - const source_grouped_layout = b.staticValue("!cute.layout<\"(((32,1,1,4),1),(1,1,1)):(((1,1,1,0),0),(0,0,0))\">"); - const source_grouped_ty = "!cute_nvgpu.smem_desc_view"; + const source_grouped_spec = b.layoutSpec(.{ .{ .{ 32, 1, 1, 4 }, 1 }, .{ 1, 1, 1 } }, .{ .{ .{ 1, 1, 1, 0 }, 0 }, .{ 0, 0, 0 } }); + const source_grouped_layout = b.staticLayout(source_grouped_spec); + const source_grouped_ty = b.smemDescViewType(source_grouped_spec); const sfa_source = b.makeView( - b.getIter(sfa_stage, "!cute_nvgpu.smem_desc"), + b.getIter(sfa_stage, b.smemDescType()), source_grouped_layout, source_grouped_ty, ); const sfb_source = b.makeView( - b.getIter(sfb_stage, "!cute_nvgpu.smem_desc"), + b.getIter(sfb_stage, b.smemDescType()), source_grouped_layout, source_grouped_ty, ); - const acc_payload = fmt(b, "((128,{d}),1,1):((65536,1),0,0)", .{self.n}); - const acc_layout = b.staticValue(fmt(b, "!cute.layout<\"{s}\">", .{acc_payload})); + const acc_spec = b.layoutSpec(.{ .{ 128, self.n }, 1, 1 }, .{ .{ 65536, 1 }, 0, 0 }); + const acc_layout = b.staticLayout(acc_spec); const accumulator = b.makeView( accumulator_tmem, acc_layout, - fmt(b, "!cute.memref, \"{s}\">", .{acc_payload}), + b.memrefType(.f32, .tmem, 16, b.layoutType(acc_spec)), ); return .{ .a = a.value(), @@ -1173,51 +1180,78 @@ fn makeMxfp4StagedMainloop( tiled_mma, smem_a, 0, - fmt(b, "!cute_nvgpu.smem_desc_view", .{stages}), + b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4, stages }, .{ 0, 0, 2, 1024 })), ); const frag_b = b.mmaMakeFragment( tiled_mma, smem_b, 1, - fmt(b, "!cute_nvgpu.smem_desc_view", .{ stages, n * 8 }), + b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4, stages }, .{ 0, 0, 2, n * 8 })), ); // TMEM columns: two N-wide accumulator stages, then 16 SFA and 16 SFB // columns, matching num_accumulator_tmem_cols/num_sfa_tmem_cols in Python. - const sf_layout = b.staticValue("!cute.layout<\"((((32,4),4),(32,1)),1,4):((((262144,4),8388608),(0,0)),0,1)\">"); - const sfa_col = b.addOffset(scale_tmem, b.staticValue(fmt(b, "!cute.int_tuple<\"{d}\">", .{2 * n})), "!cute.ptr>"); - const sfb_col = b.addOffset(scale_tmem, b.staticValue(fmt(b, "!cute.int_tuple<\"{d}\">", .{2 * n + 16})), "!cute.ptr>"); - const sfa_ptr = b.recastIter(sfa_col, "!cute.ptr>"); - const sfb_ptr = b.recastIter(sfb_col, "!cute.ptr>"); - const sf_memref = "!cute.memref, \"((((32,4),4),(32,1)),1,4):((((262144,4),8388608),(0,0)),0,1)\">"; + const sf_spec = b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 4 }, .{ 32, 1 } }, 1, 4 }, .{ .{ .{ .{ 262144, 4 }, 8388608 }, .{ 0, 0 } }, 0, 1 }); + const sf_layout = b.staticLayout(sf_spec); + const tmem_f32_ptr = b.ptrTy(.f32, .tmem, 16) catch @panic("bad scale-factor TMEM pointer"); + const tmem_sf_ptr = b.ptrTy(.f8e8m0fnu, .tmem, 16) catch @panic("bad scale-factor TMEM pointer"); + const sfa_col = b.addOffset(scale_tmem, b.staticIntTuple(2 * n), tmem_f32_ptr); + const sfb_col = b.addOffset(scale_tmem, b.staticIntTuple(2 * n + 16), tmem_f32_ptr); + const sfa_ptr = b.recastIter(sfa_col, tmem_sf_ptr); + const sfb_ptr = b.recastIter(sfb_col, tmem_sf_ptr); + const sf_memref = b.memrefType(.f8e8m0fnu, .tmem, 16, b.layoutType(sf_spec)); const sfa = b.makeView(sfa_ptr, sf_layout, sf_memref); const sfb = b.makeView(sfb_ptr, sf_layout, sf_memref); - const smem_filtered_ty = fmt(b, "!cute.memref, \"((((32,4),1),(1,1)),1,4,{d}):((((16,4),0),(0,0)),0,1,512)\">", .{stages}); - const tmem_filtered_ty = "!cute.memref, \"((((32,4),4),(1,1)),1,4):((((262144,4),8388608),(0,0)),0,1)\">"; + const smem_filtered_ty = b.memrefType(.f8e8m0fnu, .smem, 1024, b.layoutType(b.layoutSpec( + .{ .{ .{ .{ 32, 4 }, 1 }, .{ 1, 1 } }, 1, 4, stages }, + .{ .{ .{ .{ 16, 4 }, 0 }, .{ 0, 0 } }, 0, 1, 512 }, + ))); + const tmem_filtered_ty = b.memrefType(.f8e8m0fnu, .tmem, 16, b.layoutType(b.layoutSpec( + .{ .{ .{ .{ 32, 4 }, 4 }, .{ 1, 1 } }, 1, 4 }, + .{ .{ .{ .{ 262144, 4 }, 8388608 }, .{ 0, 0 } }, 0, 1 }, + ))); const sfa_smem_filtered = b.filterZeros(smem_sfa, null, smem_filtered_ty); const sfb_smem_filtered = b.filterZeros(smem_sfb, null, smem_filtered_ty); const sfa_tmem_filtered = b.filterZeros(sfa, null, tmem_filtered_ty); const sfb_tmem_filtered = b.filterZeros(sfb, null, tmem_filtered_ty); - const copy_atom = b.makeAtom("!cute_nvgpu.atom.s2t_copy", .{}); - const copy_ty = "!cute.tiled_copy, layout_copy_tv = <\"(1,(32,(4,4),4)):(0,(1,(512,32),128))\">, tiler_mn = <\"[512:1;1:0;4:1]\">>"; + const copy_atom_ty = b.s2tCopyAtomType(.f8e8m0fnu, .{ .num_dp = 32, .num_bit = 128, .broadcast = .x4 }); + const copy_atom = b.makeAtom(copy_atom_ty, .{}); + const copy_ty = b.tiledCopyType( + copy_atom_ty, + b.layoutSpec(.{ 1, .{ 32, .{ 4, 4 }, 4 } }, .{ 0, .{ 1, .{ 512, 32 }, 128 } }), + &.{ b.layoutSpec(512, 1), b.layoutSpec(1, 0), b.layoutSpec(4, 1) }, + ); const sfa_copy = b.makeS2tCopy(copy_atom, sfa_tmem_filtered, copy_ty); const sfb_copy = b.makeS2tCopy(copy_atom, sfb_tmem_filtered, copy_ty); - const source_partition_ty = fmt(b, "!cute.memref, \"(((32,4,4,4),1),1,1,1,{d}):(((16,1,4,0),0),0,0,0,512)\">", .{stages}); - const source_desc_ty = fmt(b, "!cute_nvgpu.smem_desc_view", .{stages}); - const target_partition_ty = "!cute.memref, \"(((32,4,(4,4)),1),1,1,1):(((262144,1,(4,8388608)),0),0,0,0)\">"; - const target_grouped_ty = "!cute.memref, \"(((32,4,(4,4)),1),(1,1,1)):(((262144,1,(4,8388608)),0),(0,0,0))\">"; + const source_partition_ty = b.memrefType(.f8e8m0fnu, .smem, 1024, b.layoutType(b.layoutSpec( + .{ .{ .{ 32, 4, 4, 4 }, 1 }, 1, 1, 1, stages }, + .{ .{ .{ 16, 1, 4, 0 }, 0 }, 0, 0, 0, 512 }, + ))); + const source_desc_ty = b.smemDescViewType(b.layoutSpec( + .{ .{ .{ 32, 1, 1, 4 }, 1 }, 1, 1, 1, stages }, + .{ .{ .{ 1, 1, 1, 0 }, 0 }, 0, 0, 0, 32 }, + )); + const target_partition_ty = b.memrefType(.f8e8m0fnu, .tmem, 16, b.layoutType(b.layoutSpec( + .{ .{ .{ 32, 4, .{ 4, 4 } }, 1 }, 1, 1, 1 }, + .{ .{ .{ 262144, 1, .{ 4, 8388608 } }, 0 }, 0, 0, 0 }, + ))); + const target_grouped_spec = b.layoutSpec( + .{ .{ .{ 32, 4, .{ 4, 4 } }, 1 }, .{ 1, 1, 1 } }, + .{ .{ .{ 262144, 1, .{ 4, 8388608 } }, 0 }, .{ 0, 0, 0 } }, + ); + const target_grouped_ty = b.memrefType(.f8e8m0fnu, .tmem, 16, b.layoutType(target_grouped_spec)); const sfa_src_partition = b.tiledCopyPartition(sfa_copy, sfa_smem_filtered, .{0}, false, source_partition_ty); const sfb_src_partition = b.tiledCopyPartition(sfb_copy, sfb_smem_filtered, .{0}, false, source_partition_ty); const sfa_desc = b.s2tSmemDescriptor(sfa_copy, sfa_src_partition, source_desc_ty); const sfb_desc = b.s2tSmemDescriptor(sfb_copy, sfb_src_partition, source_desc_ty); const sfa_dst_partition = b.tiledCopyPartition(sfa_copy, sfa_tmem_filtered, .{0}, true, target_partition_ty); const sfb_dst_partition = b.tiledCopyPartition(sfb_copy, sfb_tmem_filtered, .{0}, true, target_partition_ty); - const target_grouped_layout = b.staticValue("!cute.layout<\"(((32,4,(4,4)),1),(1,1,1)):(((262144,1,(4,8388608)),0),(0,0,0))\">"); - const sfa_dst = b.makeView(b.getIter(sfa_dst_partition, "!cute.ptr>"), target_grouped_layout, target_grouped_ty); - const sfb_dst = b.makeView(b.getIter(sfb_dst_partition, "!cute.ptr>"), target_grouped_layout, target_grouped_ty); + const target_grouped_layout = b.staticLayout(target_grouped_spec); + const sfa_dst = b.makeView(b.getIter(sfa_dst_partition, tmem_sf_ptr), target_grouped_layout, target_grouped_ty); + const sfb_dst = b.makeView(b.getIter(sfb_dst_partition, tmem_sf_ptr), target_grouped_layout, target_grouped_ty); return .{ .a = frag_a, @@ -1234,10 +1268,6 @@ fn makeMxfp4StagedMainloop( }; } -fn fmt(b: *B, comptime format: []const u8, args: anytype) []const u8 { - return std.fmt.allocPrint(b.arena.allocator(), format, args) catch @panic("OOM"); -} - /// Warp 4 drains the A/B ring into one of two independent accumulator stages. fn runMmaConsumer( b: *B, @@ -1467,7 +1497,7 @@ fn runEpilogue( b.mbarrierTryWaitParity(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, 10_000_000); const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, stage); - const warp_tmem = b.addOffsetTyped( + const warp_tmem = b.addOffset( accumulator_tmem, b.makeIntTupleDivBy(warp.mul(32 * 65536), 4), b.ptrTy(.f32, .tmem, 16) catch @panic("bad epilogue TMEM pointer"), @@ -1478,7 +1508,7 @@ fn runEpilogue( const chunk_rows: usize = comptime @intCast(tile.chunkRows()); const tile_chunks = comptime tile.chunks(); inline for (0..tile_chunks) |chunk| { - const chunk_tmem = if (chunk == 0) warp_tmem else b.addOffsetTyped( + const chunk_tmem = if (chunk == 0) warp_tmem else b.addOffset( warp_tmem, b.makeIntTuple(.{@as(i32, @intCast(chunk * chunk_rows))}), b.ptrTy(.f32, .tmem, offsetAlignment(16, @intCast(chunk * chunk_rows), 4)) catch @panic("bad epilogue TMEM pointer"), @@ -1599,7 +1629,7 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo const f32_type = cute.DType.f32.toMlir(b.ctx); // The TMA C staging buffer is unused by this epilogue. const scratch_layout = b.layoutSpec(.{ 4, n }, .{ n, 1 }); - const scratch = b.makeViewTyped(storage.c, b.staticLayout(scratch_layout), b.memrefType(.f32, .smem, 1024, b.layoutType(scratch_layout))); + const scratch = b.makeView(storage.c, b.staticLayout(scratch_layout), b.memrefType(.f32, .smem, 1024, b.layoutType(scratch_layout))); const row_chunks = std.math.divCeil(usize, live, swiglu_chunk) catch unreachable; var activations: [@intCast(n)]V = undefined; @@ -1719,7 +1749,7 @@ fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E 1, tile.fields(b), ); - const shared_base = b.getIterTyped( + const shared_base = b.getIter( shared, b.ptrTy(.i8, .smem, 1024) catch @panic("bad shared base pointer"), ).value(); @@ -1759,8 +1789,8 @@ fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), 0, b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, ); const all_tiles = .{ cute.AlgebraToken.all, cute.AlgebraToken.all, cute.AlgebraToken.all }; - const tiled_a = b.localTileTyped(d.gA, b.makeTile(&.{ 128, 128 }), all_tiles, b.coordTensorType(3, a_local_spec), null); - const tiled_b = b.localTileTyped(d.gB, b.makeTile(&.{ n, 128 }), all_tiles, b.coordTensorType(3, b_local_spec), null); + const tiled_a = b.localTile(d.gA, b.makeTile(&.{ 128, 128 }), all_tiles, b.coordTensorType(3, a_local_spec), null); + const tiled_b = b.localTile(d.gB, b.makeTile(&.{ n, 128 }), all_tiles, b.coordTensorType(3, b_local_spec), null); const a_mma_spec = b.layoutSpec( .{ .{ 128, 32 }, 1, 4, m_tiles, k_tiles, cfg.experts }, @@ -1770,8 +1800,8 @@ fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E .{ .{ n, 32 }, 1, 4, 1, k_tiles, cfg.groups }, .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(32, 1, .{0}), 0, b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, ); - const mma_a = b.tiledMmaPartitionTyped(tiled_mma, tiled_a, .{0}, 0, b.coordTensorType(3, a_mma_spec)); - const mma_b = b.tiledMmaPartitionTyped(tiled_mma, tiled_b, .{0}, 1, b.coordTensorType(3, b_mma_spec)); + const mma_a = b.tiledMmaPartition(tiled_mma, tiled_a, .{0}, 0, b.coordTensorType(3, a_mma_spec)); + const mma_b = b.tiledMmaPartition(tiled_mma, tiled_b, .{0}, 1, b.coordTensorType(3, b_mma_spec)); const a_grouped_layout = b.staticLayout(b.layoutSpec( .{ .{ .{ 128, 32 }, 1, 4 }, m_tiles, k_tiles, cfg.experts }, @@ -1801,7 +1831,7 @@ fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E .{ .{ .{ 128, n }, 1 }, 1, k_tiles, cfg.groups }, .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(n, 1, .{1}), b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, ); - const partitioned_a = b.tmaPartitionTyped( + const partitioned_a = b.tmaPartition( .{ .inner = d.tma_a.inner, .kernel = b }, .{0}, one_cta, @@ -1810,7 +1840,7 @@ fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E b.memrefTypeFromPointer(b.swizzledPtrTy(.i8, .smem, 1024, L.ab_swizzle) catch @panic("bad smem pointer"), b.layoutType(a_partition_layout)), &.{b.coordTensorType(3, a_target_layout)}, ); - const partitioned_b = b.tmaPartitionTyped( + const partitioned_b = b.tmaPartition( .{ .inner = d.tma_b.inner, .kernel = b }, .{0}, one_cta, @@ -1820,10 +1850,10 @@ fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E &.{b.coordTensorType(3, b_target_layout)}, ); - const exec_a = b.makeExecTmaTyped(.{ .inner = d.tma_a.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaA(b, cfg))); - const exec_b = b.makeExecTmaTyped(.{ .inner = d.tma_b.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaB(b, cfg))); - const exec_sfa = b.makeExecTmaTyped(.{ .inner = d.tma_sfa.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaSf(b))); - const exec_sfb = b.makeExecTmaTyped(.{ .inner = d.tma_sfb.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaSf(b))); + const exec_a = b.makeExecTma(.{ .inner = d.tma_a.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaA(b, cfg))); + const exec_b = b.makeExecTma(.{ .inner = d.tma_b.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaB(b, cfg))); + const exec_sfa = b.makeExecTma(.{ .inner = d.tma_sfa.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaSf(b))); + const exec_sfb = b.makeExecTma(.{ .inner = d.tma_sfb.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaSf(b))); const expert_ids: cute.View = .{ .inner = d.expert_ids.inner, .kernel = b }; const active_count: cute.View = .{ .inner = d.active_count.inner, .kernel = b }; @@ -1917,7 +1947,7 @@ fn buildHostLaunch(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E // `schedule` is `[group experts | group sizes | active count]`. const schedule = b.arg(host_arg.schedule); const ids = b.makeTensorView(schedule, b.staticLayout(b.layoutSpec(cfg.groups, 1))); - const active_ptr = b.addOffsetTyped( + const active_ptr = b.addOffset( schedule, b.makeIntTuple(.{2 * cfg.groups}), b.ptrTy(.i32, .gmem, offsetAlignment(16, 2 * cfg.groups, 4)) catch @panic("bad active count pointer"), @@ -1934,10 +1964,10 @@ fn buildHostLaunch(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E // The Zig dialect binding exposes the zero-based form of Python CuTe's // `(2, 1, 3)` tile-to-shape order. const sf_order = b.makeIntTuple(.{ 1, 0, 2 }); - const sfa_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_m), b.arg(host_arg.problem_k), b.arg(host_arg.expert_count) }, b.parseType("!cute.shape<\"(?,?,?)\">")); - const sfb_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_n), b.arg(host_arg.problem_k), b.arg(host_arg.problem_l) }, b.parseType("!cute.shape<\"(?,?,?)\">")); - const dynamic_sfa_layout = b.tileToShapeTyped(sf_atom_layout, sfa_shape, sf_order, b.layoutType(dynamic_sf_layout)); - const dynamic_sfb_layout = b.tileToShapeTyped(sf_atom_layout, sfb_shape, sf_order, b.layoutType(dynamic_sf_layout)); + const sfa_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_m), b.arg(host_arg.problem_k), b.arg(host_arg.expert_count) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); + const sfb_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_n), b.arg(host_arg.problem_k), b.arg(host_arg.problem_l) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); + const dynamic_sfa_layout = b.tileToShape(sf_atom_layout, sfa_shape, sf_order, b.layoutType(dynamic_sf_layout)); + const dynamic_sfb_layout = b.tileToShape(sf_atom_layout, sfb_shape, sf_order, b.layoutType(dynamic_sf_layout)); const gSFA = b.makeTensorView(b.recastPointer(b.arg(host_arg.weight_scale), .f8e8m0fnu, .gmem, 16), dynamic_sfa_layout); const gSFB = b.makeTensorView(b.recastPointer(b.arg(host_arg.input_scale), .f8e8m0fnu, .gmem, 16), dynamic_sfb_layout); @@ -1952,13 +1982,13 @@ fn buildHostLaunch(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E const b_map = b.staticLayout(b.layoutSpec(.{ .{ n, 32 }, 1, 4 }, .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0, b.basis(32, 1, .{1}) })); const sf_map = b.staticLayout(b.layoutSpec( .{ .{ .{ 32, 4 }, 32 }, 1, 4 }, - .{ .{ .{ b.basis(1, 1, .{ 0, 0, 0 }), b.basis(1, 1, .{ 1, 0, 0 }) }, b.basis(1, 1, .{ 0, 0, 1 }) }, 0, b.basis(1, 1, .{ 1, 0, 1 }) }, + .{ .{ .{ b.basis(1, 1, .{ 0, 0, 0 }), b.basis(1, 1, .{ 0, 0, 1 }) }, b.basis(1, 1, .{ 1, 0, 0 }) }, 0, b.basis(1, 1, .{ 1, 0, 1 }) }, )); - const tma_a = b.makeTiledTmaLoadAtomTyped(gA.value(), a_smem.value(), a_map.value(), L.tmaA(b, cfg)); - const tma_b = b.makeTiledTmaLoadAtomTyped(gB.value(), b_smem.value(), b_map.value(), L.tmaB(b, cfg)); - const tma_sfa = b.makeTiledTmaLoadAtomTyped(gSFA.value(), sf_smem.value(), sf_map.value(), L.tmaSf(b)); - const tma_sfb = b.makeTiledTmaLoadAtomTyped(gSFB.value(), sf_smem.value(), sf_map.value(), L.tmaSf(b)); + const tma_a = b.makeTiledTmaLoadAtom(gA.value(), a_smem.value(), a_map.value(), L.tmaA(b, cfg)); + const tma_b = b.makeTiledTmaLoadAtom(gB.value(), b_smem.value(), b_map.value(), L.tmaB(b, cfg)); + const tma_sfa = b.makeTiledTmaLoadAtom(gSFA.value(), sf_smem.value(), sf_map.value(), L.tmaSf(b)); + const tma_sfb = b.makeTiledTmaLoadAtom(gSFB.value(), sf_smem.value(), sf_map.value(), L.tmaSf(b)); const one = b.cst(.i32, 1); const tiles_m = b.cst(.i32, @divExact(cfg.m, 128)); @@ -1977,7 +2007,7 @@ fn buildHostLaunch(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E }; const activations = @divExact(cfg.m, 2); const group_sizes = b.makeTensorView( - b.addOffsetTyped(schedule, b.makeIntTuple(.{cfg.groups}), b.ptrTy(.i32, .gmem, offsetAlignment(16, cfg.groups, 4)) catch @panic("bad group size pointer")), + b.addOffset(schedule, b.makeIntTuple(.{cfg.groups}), b.ptrTy(.i32, .gmem, offsetAlignment(16, cfg.groups, 4)) catch @panic("bad group size pointer")), b.staticLayout(b.layoutSpec(cfg.groups, 1)), ); const launch = switch (epilogue) { diff --git a/zml/kernels/moe/fly/moe.zig b/zml/kernels/moe/fly/moe.zig index 01d5e412..96503fa9 100644 --- a/zml/kernels/moe/fly/moe.zig +++ b/zml/kernels/moe/fly/moe.zig @@ -387,7 +387,7 @@ pub const Mfma = struct { }; const initial = loader.load(b, b.constant(.i64, 0), b.constant(.i64, 0)); b.barrier(); - const atom = b.mmaAtom((fly.rocdl.MmaOpCDNA3MFMAType.get(b.ctx, .{ + const atom = b.mmaAtom((fly.fly_rocdl.MmaOpCDNA3MFMAType.get(b.ctx, .{ .m = 16, .n = 16, .k = 16, diff --git a/zml/kernels/moe/mosaic_tpu/gmm_ep.zig b/zml/kernels/moe/mosaic_tpu/gmm_ep.zig index 59638114..06d71820 100644 --- a/zml/kernels/moe/mosaic_tpu/gmm_ep.zig +++ b/zml/kernels/moe/mosaic_tpu/gmm_ep.zig @@ -1,7 +1,7 @@ const std = @import("std"); const zml = @import("../../../zml.zig"); -const mtt = @import("kernels/mosaic_tpu/builder"); +const mtt = @import("dsl/mosaic_tpu/builder"); const pipeline = mtt.pipeline; const mtt_kernel = zml.kernel.mosaic_tpu; diff --git a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig index 8d7807bf..09dc360a 100644 --- a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig @@ -1,6 +1,6 @@ const std = @import("std"); -const mtt = @import("kernels/mosaic_tpu/builder"); +const mtt = @import("dsl/mosaic_tpu/builder"); const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); diff --git a/zml/kernels/paged_attention/cute/sm10x/attention.zig b/zml/kernels/paged_attention/cute/sm10x/attention.zig index 17f5552e..c336a5ae 100644 --- a/zml/kernels/paged_attention/cute/sm10x/attention.zig +++ b/zml/kernels/paged_attention/cute/sm10x/attention.zig @@ -208,7 +208,7 @@ fn attention(b: *B, c: Config) cute.FinishError!void { .out = .{ .ptr = if (c.splits > 1) cute.DType.f32 else c.dtype }, }); std.debug.assert(c.sharedBytes() <= 227 * 1024); - b.setFunctionAttribute("cu_attrs", b.parseAttribute("{max_dynamic_shared_size_bytes = #cuda.dev_max_shared_memory_optin}")); + b.setFunctionAttribute("cu_attrs", .dict(b.ctx, &.{.named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin())})); const seq = b.blockIdx().x.div(c.queryTiles()); const first_row = b.blockIdx().x.rem(c.queryTiles()).mul(c.rows()); const starts = b.ptrToInt(a.starts, .i64); diff --git a/zml/kernels/paged_attention/cute/sm10x/sm103.zig b/zml/kernels/paged_attention/cute/sm10x/sm103.zig index bc1da928..bc3171b5 100644 --- a/zml/kernels/paged_attention/cute/sm10x/sm103.zig +++ b/zml/kernels/paged_attention/cute/sm10x/sm103.zig @@ -1,6 +1,6 @@ const std = @import("std"); const mlir = @import("mlir"); -const llvm = @import("mlir/dialects").llvm; +const llvm = @import("mlir/dialects/llvm"); const cute = @import("../../../../zml.zig").kernel.cute; @@ -16,19 +16,16 @@ pub fn loadScores(b: *cute.Builder, address: cute.Value) Scores { @setEvalBranchQuota(20000); var registers: []const u8 = ""; var constraints: []const u8 = ""; - var fields: []const u8 = ""; for (0..65) |i| { if (i < 64) registers = registers ++ (if (i == 0) "" else ", ") ++ std.fmt.comptimePrint("${d}", .{i}); constraints = constraints ++ (if (i == 0) "" else ",") ++ "=f"; - fields = fields ++ (if (i == 0) "" else ", ") ++ "f32"; } break :blk .{ "tcgen05.ld.red.sync.aligned.32x32b.x64.max.f32 {" ++ registers ++ "}, $64, [$65];", constraints ++ ",r", - "!llvm.struct<(" ++ fields ++ ")>", }; }; - const result = b.emit(llvm.inline_asm(b.ctx, asm_parts[0], &.{address.inner}, b.parseType(asm_parts[2]), .{ .constraints = asm_parts[1], .has_side_effects = true }, b.loc())); + const result = b.emit(llvm.inline_asm(b.ctx, asm_parts[0], &.{address.inner}, .{ .result_type = llvm.structType(b.ctx, &@as([65]*const mlir.Type, @splat(.float(b.ctx, .f32)))), .constraints = asm_parts[1], .has_side_effects = true }, b.loc())); var scores: Scores = undefined; for (0..65) |i| { const value = b.emit(mlir.Operation.make(b.ctx, "llvm.extractvalue", .{ diff --git a/zml/kernels/paged_attention/cute/sm120/attention.zig b/zml/kernels/paged_attention/cute/sm120/attention.zig index 92a7ba30..ec6a509b 100644 --- a/zml/kernels/paged_attention/cute/sm120/attention.zig +++ b/zml/kernels/paged_attention/cute/sm120/attention.zig @@ -1,7 +1,7 @@ const std = @import("std"); const mlir = @import("mlir"); -const llvm = @import("mlir/dialects").llvm; -const vector = @import("mlir/dialects").vector; +const llvm = @import("mlir/dialects/llvm"); +const vector = @import("mlir/dialects/vector"); const zml = @import("../../../../zml.zig"); const cute = zml.kernel.cute; @@ -66,7 +66,7 @@ fn flatten(comptime n: usize, values: [n][4]V) [n * 4]V { } fn packAcc(b: *B, values: [4]V) V { - return b.emit(vector.from_elements(b.ctx, &.{ values[0].inner, values[1].inner, values[2].inner, values[3].inner }, b.parseType("vector<4xf32>"), b.loc())); + return b.emit(vector.from_elements(b.ctx, &.{ values[0].inner, values[1].inner, values[2].inner, values[3].inner }, .vector(&.{4}, .float(b.ctx, .f32)), b.loc())); } fn loadI32(b: *B, ptr: V, index: V) V { @@ -92,14 +92,14 @@ fn unpackStruct(b: *B, comptime n: usize, value: V, dtype: cute.DType) [n]V { fn matrixLoad(b: *B, comptime n: usize, comptime transpose: bool, address: V) [n]V { const assembly = std.fmt.comptimePrint("ldmatrix.sync.aligned.m8n8.x{d}{s}.shared.b16 {{", .{ n, if (transpose) ".trans" else "" }) ++ (if (n == 4) "$0, $1, $2, $3}, [$4];" else "$0, $1}, [$2];"); - const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{address.inner}, b.parseType(if (n == 4) "!llvm.struct<(i32, i32, i32, i32)>" else "!llvm.struct<(i32, i32)>"), .{ .constraints = if (n == 4) "=r,=r,=r,=r,r" else "=r,=r,r", .has_side_effects = true }, b.loc())); + const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{address.inner}, .{ .result_type = llvm.structType(b.ctx, &@as([n]*const mlir.Type, @splat(.int(b.ctx, .i32)))), .constraints = if (n == 4) "=r,=r,=r,=r,r" else "=r,=r,r", .has_side_effects = true }, b.loc())); return unpackStruct(b, n, result, .i32); } fn mma(b: *B, comptime dtype: cute.DType, a: [4]V, k: [2]V, acc: [4]V) [4]V { const assembly = "mma.sync.aligned.m16n8k16.row.col.f32." ++ @tagName(dtype) ++ "." ++ @tagName(dtype) ++ ".f32 " ++ "{$0, $1, $2, $3}, {$4, $5, $6, $7}, {$8, $9}, {$10, $11, $12, $13};"; - const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{ a[0].inner, a[1].inner, a[2].inner, a[3].inner, k[0].inner, k[1].inner, acc[0].inner, acc[1].inner, acc[2].inner, acc[3].inner }, b.parseType("!llvm.struct<(f32, f32, f32, f32)>"), .{ .constraints = "=f,=f,=f,=f,r,r,r,r,r,r,f,f,f,f" }, b.loc())); + const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{ a[0].inner, a[1].inner, a[2].inner, a[3].inner, k[0].inner, k[1].inner, acc[0].inner, acc[1].inner, acc[2].inner, acc[3].inner }, .{ .result_type = llvm.structType(b.ctx, &@as([4]*const mlir.Type, @splat(.float(b.ctx, .f32)))), .constraints = "=f,=f,=f,=f,r,r,r,r,r,r,f,f,f,f" }, b.loc())); return unpackStruct(b, 4, result, .f32); } @@ -231,7 +231,7 @@ fn runTile(b: *B, c: Config, comptime d: usize, comptime w: usize, comptime n: u .sink = .{ .ptr = cute.DType.f32 }, .out = .{ .ptr = if (c.splits > 1) cute.DType.f32 else dt }, }); - b.setFunctionAttribute("cu_attrs", b.parseAttribute("{max_dynamic_shared_size_bytes = #cuda.dev_max_shared_memory_optin}")); + b.setFunctionAttribute("cu_attrs", .dict(b.ctx, &.{.named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin())})); const seq = b.blockIdx().x.div(c.queryTiles()); const first_row = b.blockIdx().x.rem(c.queryTiles()).mul(c.rows()); const starts = b.ptrToInt(a.starts, .i64); diff --git a/zml/kernels/paged_attention/triton/unified_attention.zig b/zml/kernels/paged_attention/triton/unified_attention.zig index ce28905f..f74aabaf 100644 --- a/zml/kernels/paged_attention/triton/unified_attention.zig +++ b/zml/kernels/paged_attention/triton/unified_attention.zig @@ -1,8 +1,7 @@ const std = @import("std"); const mlir = @import("mlir"); -const dialects = @import("mlir/dialects"); -const ttir = dialects.ttir; +const ttir = @import("mlir/dialects/ttir"); const zml = @import("../../../zml.zig"); const tri = zml.kernel.triton; diff --git a/zml/kernels/paged_attention/triton/unified_attention_oneapi.zig b/zml/kernels/paged_attention/triton/unified_attention_oneapi.zig index 3514f00f..3158b53f 100644 --- a/zml/kernels/paged_attention/triton/unified_attention_oneapi.zig +++ b/zml/kernels/paged_attention/triton/unified_attention_oneapi.zig @@ -1,8 +1,7 @@ const std = @import("std"); const mlir = @import("mlir"); -const dialects = @import("mlir/dialects"); -const ttir = dialects.ttir; +const ttir = @import("mlir/dialects/ttir"); const zml = @import("../../../zml.zig"); const tri = zml.kernel.triton; diff --git a/zml/ops.zig b/zml/ops.zig index 54eedda9..9d837e6b 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1,10 +1,11 @@ const std = @import("std"); const builtin = @import("builtin"); -const dialects = @import("mlir/dialects"); +const func = @import("mlir/dialects/func"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); const platforms = @import("platforms"); +const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); const manual_computation = @import("ops/manual_computation.zig"); @@ -30,7 +31,7 @@ const Tensor = @import("tensor.zig").Tensor; const TensorToCustomCallBuffer = @import("pjrtx.zig").TensorToCustomCallBuffer; const log = std.log.scoped(.@"zml/Compiler"); -pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@TypeOf(inputs)) { +pub fn allReduce(inputs: anytype, comptime func_: anytype) AllReduceReturnType(@TypeOf(inputs)) { const ctx = Compiler.current(); const mlir_ctx = ctx.mlir_ctx; @@ -98,7 +99,7 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@T const reducer_scope = ctx.pushBlock(block); defer reducer_scope.pop(); - ctx.pushLocation(@src(), stdx.meta.fnName(func)); + ctx.pushLocation(@src(), stdx.meta.fnName(func_)); defer ctx.popLocation(); inline for (0..input_tensors.len) |i| { @@ -108,10 +109,10 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@T var reduced_values: [input_tensors.len]*const mlir.Value = undefined; if (input_tensors.len == 1) { - const reduced = @call(.auto, func, .{ args[0].left, args[0].right }); + const reduced = @call(.auto, func_, .{ args[0].left, args[0].right }); reduced_values[0] = reduced.value(); } else { - var reduced = @call(.auto, func, args); + var reduced = @call(.auto, func_, args); var reduced_tensors: [input_tensors.len]Tensor = undefined; meta.collectBuf(stdx.meta.identity, {}, &reduced, &reduced_tensors); @@ -120,7 +121,7 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@T } } - _ = dialects.stablehlo.returns(mlir_ctx, &reduced_values, ctx.location).appendTo(block); + _ = stablehlo.returns(mlir_ctx, &reduced_values, ctx.location).appendTo(block); break :b block; }; @@ -135,7 +136,7 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@T ); const handle = ctx.nextChannelId(); - const channel_handle_attr = dialects.stablehlo.channelHandle( + const channel_handle_attr = stablehlo.channelHandleAttr( mlir_ctx, handle, .collective, @@ -148,7 +149,7 @@ pub fn allReduce(inputs: anytype, comptime func: anytype) AllReduceReturnType(@T result_types[i] = input_values[i].type_(); } - const op = dialects.stablehlo.allReduce( + const op = stablehlo.allReduce( mlir_ctx, &input_values, &result_types, @@ -215,7 +216,7 @@ pub const ReduceArgs = struct { right: Tensor, }; -pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func: anytype, context: anytype) stdx.meta.FnReturn(func) { +pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func_: anytype, context: anytype) stdx.meta.FnReturn(func_) { const compiler = Compiler.current(); const caller_scope = compiler.currentScope(); const mlir_ctx = compiler.mlir_ctx; @@ -242,7 +243,7 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func const reduce_scope = compiler.pushBlock(reduce_block); defer reduce_scope.pop(); - compiler.pushLocation(@src(), stdx.meta.fnName(func)); + compiler.pushLocation(@src(), stdx.meta.fnName(func_)); defer compiler.popLocation(); inline for (0..inits.len) |i| { @@ -250,14 +251,14 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func reduce_scope.registerTensorAsBlockArgument(args[i].right.id, i + inits.len); } - var result = @call(.auto, func, args ++ context); + var result = @call(.auto, func_, args ++ context); var result_values: [inits.len]*const mlir.Value = undefined; inline for (0..inits.len) |i| { result_values[i] = result[i].value(); } - const block_result = dialects.stablehlo.returns(mlir_ctx, &result_values, compiler.location); + const block_result = stablehlo.returns(mlir_ctx, &result_values, compiler.location); _ = block_result.appendTo(reduce_block); break :b .{ reduce_block, result }; }; @@ -296,7 +297,7 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func reduced_shape = reduced_shape.setDim(a, 1); } - const broad_op = dialects.stablehlo.broadcast_in_dim( + const broad_op = stablehlo.broadcast_in_dim( mlir_ctx, reduce_op.result(i), broadcasting_axes.slice()[0 .. reduced_shape.rank() - axes_.len], @@ -328,7 +329,7 @@ pub fn ReduceWindowFn(N: comptime_int) type { ); } -pub fn reduceWindow(N: comptime_int, inputs: [N]Tensor, inits: [N]Tensor, opts: ReduceWindowOpts, func: *const ReduceWindowFn(N)) [N]Tensor { +pub fn reduceWindow(N: comptime_int, inputs: [N]Tensor, inits: [N]Tensor, opts: ReduceWindowOpts, func_: *const ReduceWindowFn(N)) [N]Tensor { const compiler = Compiler.current(); const mlir_ctx = compiler.mlir_ctx; @@ -357,14 +358,14 @@ pub fn reduceWindow(N: comptime_int, inputs: [N]Tensor, inits: [N]Tensor, opts: reduce_scope.registerTensorAsBlockArgument(args[i].right.id, i + N); } - var result = @call(.auto, func, args); + var result = @call(.auto, func_, args); var result_values: [N]*const mlir.Value = undefined; inline for (0..N) |i| { result_values[i] = result[i].value(); } - _ = dialects.stablehlo.returns(mlir_ctx, &result_values, compiler.location).appendTo(reduce_block); + _ = stablehlo.returns(mlir_ctx, &result_values, compiler.location).appendTo(reduce_block); break :b .{ reduce_block, result }; }; var input_values: [inputs.len]*const mlir.Value = undefined; @@ -406,7 +407,7 @@ pub const SortArgs = struct { right: Tensor, }; -pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytype, is_stable: bool) [inputs.len]Tensor { +pub fn sort(inputs: anytype, axis_: i64, comptime func_: anytype, context: anytype, is_stable: bool) [inputs.len]Tensor { const compiler = Compiler.current(); const mlir_ctx = compiler.mlir_ctx; @@ -436,9 +437,9 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytyp scope.registerTensorAsBlockArgument(args[i].right.id, 2 * i + 1); } - var result = @call(.auto, func, args ++ context); + var result = @call(.auto, func_, args ++ context); - _ = dialects.stablehlo.return_(mlir_ctx, result.value(), compiler.location).appendTo(sort_block); + _ = stablehlo.return_(mlir_ctx, result.value(), compiler.location).appendTo(sort_block); break :b sort_block; }; @@ -532,7 +533,7 @@ pub fn @"while"( const cond: Tensor = captured_context.cond(initial_state); stdx.debug.assert(cond.rank() == 0 and cond.dtype() == .bool, "zml.ops.while expects cond to return a scalar bool Tensor, got {f}", .{cond}); - _ = dialects.stablehlo.return_(mlir_ctx, cond.value(), comp.location).appendTo(block); + _ = stablehlo.return_(mlir_ctx, cond.value(), comp.location).appendTo(block); break :b block; }; @@ -563,11 +564,11 @@ pub fn @"while"( } const result_values = meta.collectAlloc(Tensor.value, {}, allocator, &result) catch @panic("OOM"); - _ = dialects.stablehlo.returns(mlir_ctx, result_values, comp.location).appendTo(block); + _ = stablehlo.returns(mlir_ctx, result_values, comp.location).appendTo(block); break :b .{ block, result }; }; - const op = dialects.stablehlo.while_( + const op = stablehlo.while_( mlir_ctx, operands_info.items(.value), operands_info.items(.type), @@ -716,7 +717,7 @@ pub fn @"if"( const result = blkctx.onTrue(); const result_values = meta.collectAlloc(Tensor.value, {}, allocator, &result) catch @panic("OOM"); defer allocator.free(result_values); - _ = dialects.stablehlo.returns(mlir_ctx, result_values, loc).appendTo(block); + _ = stablehlo.returns(mlir_ctx, result_values, loc).appendTo(block); break :b .{ result, block }; }; @@ -730,7 +731,7 @@ pub fn @"if"( const result = blkctx.onFalse(); const result_values = meta.collectAlloc(Tensor.value, {}, allocator, &result) catch @panic("OOM"); defer allocator.free(result_values); - _ = dialects.stablehlo.returns(mlir_ctx, result_values, loc).appendTo(block); + _ = stablehlo.returns(mlir_ctx, result_values, loc).appendTo(block); break :b .{ result, block }; }; // TODO check that false branch values matches right branch @@ -807,7 +808,7 @@ pub fn if2( const scope = compiler.pushBlock(block); defer scope.pop(); - _ = dialects.stablehlo.returns(mlir_ctx, true_values, loc).appendTo(block); + _ = stablehlo.returns(mlir_ctx, true_values, loc).appendTo(block); break :b block; }; @@ -820,7 +821,7 @@ pub fn if2( const scope = compiler.pushBlock(block); defer scope.pop(); - _ = dialects.stablehlo.returns(mlir_ctx, false_values, loc).appendTo(block); + _ = stablehlo.returns(mlir_ctx, false_values, loc).appendTo(block); break :b block; }; @@ -894,7 +895,7 @@ pub const TritonOps = struct { num_warps: i32, is_tma_allowed: bool = false, global_scratch_memory_size: i32 = 0, - output_operand_aliases: []const dialects.stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, + output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, }; /// Generate an MLIR call to the given member function with the given tensors. @@ -936,7 +937,7 @@ pub fn triton(inputs: anytype, outputs: anytype, opts: TritonOps) [outputs.len]T results_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(output.rank())).constSlice()) catch unreachable; } - const op = dialects.stablehlo.custom_call( + const op = stablehlo.custom_call( mlir_ctx, &values, &res_types, @@ -975,7 +976,7 @@ pub const FlyOps = struct { /// `inputs.len + i`. A no-op inside HIP graphs: a kernel that must see /// zeroed outputs should zero them itself. zeroed_args: []const i32 = &.{}, - output_operand_aliases: []const dialects.stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, + output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, }; pub fn fly(inputs: anytype, outputs: anytype, opts: FlyOps) [outputs.len]Tensor { @@ -1008,7 +1009,7 @@ pub fn fly(inputs: anytype, outputs: anytype, opts: FlyOps) [outputs.len]Tensor if (opts.waves_per_eu > 0) attrs.appendAssumeCapacity(.named(mlir_ctx, "waves_per_eu", .int(mlir_ctx, .i32, opts.waves_per_eu))); if (opts.shared_mem_bytes > 0) attrs.appendAssumeCapacity(.named(mlir_ctx, "shared_mem_bytes", .int(mlir_ctx, .i64, opts.shared_mem_bytes))); if (opts.zeroed_args.len > 0) { - const Opts = dialects.stablehlo.CustomCallOpts; + const Opts = stablehlo.CustomCallOpts; var zeroed: stdx.BoundedArray(*const mlir.Attribute, Opts.MAX_OPERANDS + Opts.MAX_RESULTS) = .empty; for (opts.zeroed_args) |i| { std.debug.assert(i >= 0 and i < inputs.len + outputs.len); @@ -1028,7 +1029,7 @@ pub fn fly(inputs: anytype, outputs: anytype, opts: FlyOps) [outputs.len]Tensor results_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(output.rank())).constSlice()) catch unreachable; } - const op = dialects.stablehlo.custom_call( + const op = stablehlo.custom_call( mlir_ctx, &values, &res_types, @@ -1146,7 +1147,7 @@ pub const NeuronNkiOps = struct { source_path: []const u8, compiler_target: []const u8, has_side_effect: bool = false, - output_operand_aliases: []const dialects.stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, + output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, }; /// Generate an MLIR call for a Neuron NKI kernel. @@ -1217,7 +1218,7 @@ pub fn neuronNki(inputs: anytype, outputs: anytype, opts: NeuronNkiOps) [outputs }; defer ctx.allocator.free(compiled_backend_config); - const op = dialects.stablehlo.custom_call( + const op = stablehlo.custom_call( mlir_ctx, &values, &res_types, @@ -1316,7 +1317,7 @@ pub const CudaTileOps = struct { /// Result indices XLA zeroes before the launch, ascending. For a kernel /// that accumulates, or writes less than the whole output. zeroed_outputs: []const i32 = &.{}, - output_operand_aliases: []const dialects.stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, + output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, }; /// A `__gpu$xla.gpu.cuda_tile` custom call: XLA assembles `ir` with tileiras @@ -1347,13 +1348,13 @@ pub fn cudaTile(inputs: anytype, outputs: anytype, opts: CudaTileOps) [outputs.l }); if (opts.ir_version) |v| attrs.appendAssumeCapacity(.named(mlir_ctx, "ir_version", .string(mlir_ctx, v))); if (opts.zeroed_outputs.len > 0) { - var zeroed: stdx.BoundedArray(*const mlir.Attribute, dialects.stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + var zeroed: stdx.BoundedArray(*const mlir.Attribute, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; for (opts.zeroed_outputs) |i| zeroed.appendAssumeCapacity(.int(mlir_ctx, .i32, i)); attrs.appendAssumeCapacity(.named(mlir_ctx, "zeroed_outputs", .array(mlir_ctx, zeroed.constSlice()))); } const backend_config: *const mlir.Attribute = .dict(mlir_ctx, attrs.constSlice()); - const op = dialects.stablehlo.custom_call( + const op = stablehlo.custom_call( mlir_ctx, &values, &res_types, @@ -1385,7 +1386,7 @@ pub const CuteOps = struct { block: ?[3]i32 = null, zeroed_outputs: []const i32 = &.{}, scalars: []const i64 = &.{}, - output_operand_aliases: []const dialects.stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, + output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, }; /// A `__gpu$xla.gpu.cute` custom call: XLA compiles `ir` with the CuTe compiler @@ -1420,7 +1421,7 @@ pub fn cute(inputs: anytype, outputs: anytype, opts: CuteOps) [outputs.len]Tenso } } if (opts.zeroed_outputs.len > 0) { - var zeroed: stdx.BoundedArray(*const mlir.Attribute, dialects.stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + var zeroed: stdx.BoundedArray(*const mlir.Attribute, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; for (opts.zeroed_outputs) |i| zeroed.appendAssumeCapacity(.int(mlir_ctx, .i32, i)); attrs.appendAssumeCapacity(.named(mlir_ctx, "zeroed_outputs", .array(mlir_ctx, zeroed.constSlice()))); } @@ -1433,7 +1434,7 @@ pub fn cute(inputs: anytype, outputs: anytype, opts: CuteOps) [outputs.len]Tenso } const backend_config: *const mlir.Attribute = .dict(mlir_ctx, attrs.constSlice()); - const op = dialects.stablehlo.custom_call( + const op = stablehlo.custom_call( mlir_ctx, &values, &res_types, @@ -1724,10 +1725,10 @@ pub fn scatter( inputs: anytype, index_tensors: anytype, updates: anytype, - comptime func: anytype, + comptime func_: anytype, context: anytype, opts: Tensor.ScatterOpts, -) stdx.meta.FnReturn(func) { +) stdx.meta.FnReturn(func_) { const compiler = Compiler.current(); var arena = std.heap.ArenaAllocator.init(compiler.allocator); defer arena.deinit(); @@ -1760,14 +1761,14 @@ pub fn scatter( scope.registerTensorAsBlockArgument(args[i].update.id, i + inputs.len); } - var result = @call(.auto, func, args ++ context); + var result = @call(.auto, func_, args ++ context); var result_values: [inputs.len]*const mlir.Value = undefined; inline for (0..inputs.len) |i| { result_values[i] = result[i].value(); } - _ = dialects.stablehlo.returns(mlir_ctx, &result_values, compiler.location).appendTo(update_block); + _ = stablehlo.returns(mlir_ctx, &result_values, compiler.location).appendTo(update_block); break :b .{ update_block, result }; }; @@ -1819,7 +1820,7 @@ pub fn scatter( input_values[i] = inputs[i].value(); } - const op = dialects.stablehlo.scatter( + const op = stablehlo.scatter( mlir_ctx, &input_values, &.{indices.value()}, @@ -2135,7 +2136,7 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, // TODO: try changing .last by other axis and see the perf impact. const indices = Tensor.stack(indices_per_axis.constSlice(), .last, .coord); // scoped_log.debug("gather --> {} {any}", .{ res_shape, res_kind.constSlice() }); - const gather_op = dialects.stablehlo.gather( + const gather_op = stablehlo.gather( mlir_ctx, self.value(), indices.value(), @@ -2306,7 +2307,7 @@ fn CustomCallResultTypeFromOutputSpec(comptime OutputSpecT: type) type { pub const CustomCallOptions = struct { has_side_effect: bool, - output_operand_aliases: ?[]const dialects.stablehlo.CustomCallOpts.OutputOperandAlias = null, + output_operand_aliases: ?[]const stablehlo.CustomCallOpts.OutputOperandAlias = null, compute_on_host: bool = false, }; @@ -2326,7 +2327,7 @@ pub fn customCallOutputOperandAliases( comptime I: type, comptime O: type, comptime aliases: ?CustomCallOutputOperandAliases(I, O), -) ?[]const dialects.stablehlo.CustomCallOpts.OutputOperandAlias { +) ?[]const stablehlo.CustomCallOpts.OutputOperandAlias { const aliases_ = aliases orelse return null; const output_field_names = @typeInfo(O).@"struct".field_names; @@ -2336,7 +2337,7 @@ pub fn customCallOutputOperandAliases( } const output_operand_aliases = comptime blk: { - var aliases_buffer: [num_aliases]dialects.stablehlo.CustomCallOpts.OutputOperandAlias = undefined; + var aliases_buffer: [num_aliases]stablehlo.CustomCallOpts.OutputOperandAlias = undefined; var i = 0; for (output_field_names, 0..) |field_name, output_index| { if (@field(aliases_, field_name)) |operand| { @@ -2392,21 +2393,21 @@ pub fn composite( var result = @call(.auto, decomposition, .{ arg_tensors, context }); const rtensors = ctx.alloc(Tensor, outputs.len); meta.collectBuf((struct { - pub fn func(t: Tensor) Tensor { + pub fn func_(t: Tensor) Tensor { return t; } - }).func, {}, &result, rtensors); + }).func_, {}, &result, rtensors); const rvals = ctx.alloc(*const mlir.Value, outputs.len); for (rtensors, 0..) |t, i| { rvals[i] = t.value(); } - _ = dialects.func.returns(mlir_ctx, rvals, ctx.location).appendTo(block); + _ = func.returns(mlir_ctx, rvals, ctx.location).appendTo(block); } scope.pop(); - _ = dialects.func.func(mlir_ctx, .{ + _ = func.func(mlir_ctx, .{ .name = decomp_name, .block = block, .location = ctx.location, @@ -2425,7 +2426,7 @@ pub fn composite( result_types[i] = mlirx.Type.rankedTensor(mlir_ctx, s); } - const op = dialects.stablehlo.composite( + const op = stablehlo.composite( mlir_ctx, operand_values, result_types, @@ -2456,10 +2457,10 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, if (!struct_info.is_tuple) @compileError("Expected tuple"); var inputs_: [struct_info.field_names.len]Tensor = undefined; meta.collectBuf((struct { - pub fn func(t: Tensor) Tensor { + pub fn func_(t: Tensor) Tensor { return t; } - }).func, {}, &inputs, &inputs_); + }).func_, {}, &inputs, &inputs_); break :b &inputs_; }, .array => &inputs, @@ -2480,10 +2481,10 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, if (!struct_info.is_tuple) @compileError("Expected tuple"); var output_shapes: [struct_info.field_names.len]Shape = undefined; meta.collectBuf((struct { - pub fn func(t: Shape) Shape { + pub fn func_(t: Shape) Shape { return t; } - }).func, {}, &outputs, &output_shapes); + }).func_, {}, &outputs, &output_shapes); break :b &output_shapes; }, .array => &outputs, @@ -2594,7 +2595,7 @@ pub fn CustomCall( // Receives the custom call buffers for the input and outputs a struct with the same fields as I and O // and the provided attributes. So if I is `{a: zml.Tensor}`, the first argument is `{a: CustomCallBuffer}` // (TensorToCustomCallBuffer(i), ShapeToCustomCallBuffer(O), A) -> !?*pjrt.ffi.Error - comptime func: anytype, + comptime func_: anytype, comptime params: struct { /// Name of the custom call, must be unique. name: [:0]const u8, @@ -2659,7 +2660,7 @@ pub fn CustomCall( if (call_frame.registeringHook()) return null; const input_buffers: TensorToCustomCallBuffer(I), const output_buffers: ShapeToCustomCallBuffer(O), const attributes: A = customCallArgsFromPjrtCallFrame(I, O, A, call_frame); - return func(call_frame, input_buffers, output_buffers, attributes) catch |err| + return func_(call_frame, input_buffers, output_buffers, attributes) catch |err| pjrt.ffi.Error.create(call_frame.api, .unknown, @errorName(err)); } }; @@ -2807,7 +2808,7 @@ pub fn typedCustomCall( result_layouts[i] = allocator.dupe(usize, toUsize(constants.minorToMajor(shape.rank())).constSlice()) catch unreachable; } - const op = dialects.stablehlo.custom_call( + const op = stablehlo.custom_call( ctx.mlir_ctx, values, result_types, @@ -2925,11 +2926,11 @@ test "CustomCallOutputOperandAliases maps named fields to indices" { try std.testing.expectEqual(@as(usize, 2), aliases.len); try std.testing.expectEqualDeep( - dialects.stablehlo.CustomCallOpts.OutputOperandAlias{ .output_index = 0, .operand_index = 0 }, + stablehlo.CustomCallOpts.OutputOperandAlias{ .output_index = 0, .operand_index = 0 }, aliases[0], ); try std.testing.expectEqualDeep( - dialects.stablehlo.CustomCallOpts.OutputOperandAlias{ .output_index = 1, .operand_index = 2 }, + stablehlo.CustomCallOpts.OutputOperandAlias{ .output_index = 1, .operand_index = 2 }, aliases[1], ); } diff --git a/zml/ops/manual_computation.zig b/zml/ops/manual_computation.zig index 49ebe199..5405c6c2 100644 --- a/zml/ops/manual_computation.zig +++ b/zml/ops/manual_computation.zig @@ -1,8 +1,9 @@ const std = @import("std"); const builtin = @import("builtin"); -const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); +const shardy = @import("mlir/dialects/shardy"); +const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); const zml = struct { @@ -181,7 +182,7 @@ noinline fn manualComputationInternal( .location = compiler.location, }).appendTo(manual_block); - const out_mesh_attrs = try arena.alloc(*const dialects.shardy.TensorShardingAttribute, global_output_shapes.len); + const out_mesh_attrs = try arena.alloc(*const shardy.TensorShardingAttr, global_output_shapes.len); for (out_mesh_attrs, global_output_shapes) |*attr, shape| { attr.* = try mesh.sdyShardingAttrForShape(arena, mlir_ctx, shape); } @@ -192,7 +193,7 @@ noinline fn manualComputationInternal( .blocks = &.{manual_block}, .attributes = &.{ .named(mlir_ctx, "in_shardings", try Sharding.sdyPerValueShardingAttr(arena, mlir_ctx, global_input_shapes, mesh)), - .named(mlir_ctx, "out_shardings", dialects.shardy.attr.shardingsPerValue(mlir_ctx, out_mesh_attrs)), + .named(mlir_ctx, "out_shardings", shardy.shardingsPerValueAttr(mlir_ctx, out_mesh_attrs)), .named(mlir_ctx, "manual_axes", try Sharding.sdyManualAxesAttr(arena, mlir_ctx, global_input_shapes, global_output_shapes, mesh)), }, .verify = true, @@ -210,7 +211,7 @@ noinline fn manualComputationInternal( global_values[i], global_types[i] = try gspmd.localToGlobal(compiler, arena, global_scope, local_output_values[i], global_shape); } - break :op dialects.stablehlo.optimizationBarrier( + break :op stablehlo.optimizationBarrier( mlir_ctx, global_values, global_types, @@ -299,21 +300,21 @@ test collectManualAxes { const registry: *mlir.DialectRegistry = try .init(); defer registry.deinit(); - registry.registerDialect("sdy"); + shardy.registerDialect(registry); const ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); defer ctx.deinit(); ctx.loadAllAvailableDialects(); - const expected = dialects.shardy.ManualAxesAttribute.init(ctx, &.{ + const expected = shardy.ManualAxesAttr.get(ctx, &.{ .init(ctx, "data"), .init(ctx, "model"), - }).asAttr(); - try std.testing.expect(shardy.manualAxesAttr(ctx, axes.constSlice()).eql(expected)); + }).attribute(); + try std.testing.expect(shardy_.manualAxesAttr(ctx, axes.constSlice()).eql(expected)); } const gspmd = struct { fn globalToLocal(mlir_ctx: *mlir.Context, scope: *Compiler.Scope, local_tensor: Tensor, local_shape: Shape, location: *const mlir.Location) Tensor { const local_type = mlirx.Type.rankedTensor(mlir_ctx, local_shape); - const full_to_shard = dialects.stablehlo.custom_call( + const full_to_shard = stablehlo.custom_call( mlir_ctx, &.{scope.value(local_tensor)}, &.{local_type}, @@ -334,7 +335,7 @@ const gspmd = struct { const gspmd_attr = try compiler.tensorShardingAttr(arena, global_shape); const global_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, global_shape); - const shard_to_full = dialects.stablehlo.custom_call( + const shard_to_full = stablehlo.custom_call( compiler.mlir_ctx, &.{local_value}, &.{global_type}, @@ -352,11 +353,11 @@ const gspmd = struct { } }; -const shardy = struct { +const shardy_ = struct { pub fn manualAxesAttr(mlir_ctx: *mlir.Context, axes: []const Shape.Tag) *const mlir.Attribute { var attrs: stdx.BoundedArray(*const mlir.StringAttribute, Sharding.MAX_MESH_RANK) = .empty; for (axes) |axis| attrs.appendAssumeCapacity(.init(mlir_ctx, std.mem.span(axis))); - return dialects.shardy.ManualAxesAttribute.init(mlir_ctx, attrs.constSlice()).asAttr(); + return shardy.ManualAxesAttr.get(mlir_ctx, attrs.constSlice()).attribute(); } }; diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 8543b591..824d2bd1 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -1,8 +1,8 @@ const std = @import("std"); -const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); +const stablehlo = @import("mlir/dialects/stablehlo"); pub const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); @@ -17,9 +17,9 @@ pub const Client = opaque { pub fn compile(client: *const pjrt.Client, api: *const pjrt.Api, allocator: std.mem.Allocator, io: std.Io, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*pjrt.LoadedExecutable { const stablehlo_version = blk: { if (api.stablehloCurrentVersion()) |requested_version| { - break :blk dialects.stablehlo.smallerVersion(requested_version, dialects.stablehlo.currentVersion()); + break :blk stablehlo.smallerVersion(requested_version, stablehlo.currentVersion()); } - break :blk dialects.stablehlo.minimumVersion(); + break :blk stablehlo.minimumVersion(); }; const cloned_op = module.operation().clone(); @@ -29,7 +29,7 @@ pub const Client = opaque { var serialized_buffer: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); defer serialized_buffer.deinit(); - dialects.stablehlo.serializePortableArtifact2(cloned_module, stablehlo_version, &serialized_buffer.writer) catch |err| { + stablehlo.serializePortableArtifact2(cloned_module, stablehlo_version, &serialized_buffer.writer) catch |err| { log.err("failed to serialize to portable artifact: {}", .{err}); return switch (err) { std.Io.Writer.Error.WriteFailed => error.OutOfMemory, diff --git a/zml/tensor.zig b/zml/tensor.zig index c4ac5360..5e2e073f 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1,8 +1,8 @@ const std = @import("std"); const builtin = @import("builtin"); -const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); const Compiler = @import("Compiler.zig"); @@ -205,7 +205,7 @@ pub const Tensor = struct { break :blk op.result(0); }, .gspmd => blk: { - const op = dialects.stablehlo.custom_call( + const op = stablehlo.custom_call( ctx.mlir_ctx, &.{self.value()}, &.{self.value().type_()}, @@ -284,7 +284,7 @@ pub const Tensor = struct { ), }); - const op = dialects.stablehlo.custom_call( + const op = stablehlo.custom_call( ctx.mlir_ctx, &.{self.value()}, &.{self.value().type_()}, @@ -521,7 +521,7 @@ pub const Tensor = struct { /// Returns a Tensor containing the absolute value of each element of the input Tensor. pub fn abs(self: Tensor) Tensor { - const op = dialects.stablehlo.abs(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = stablehlo.abs(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); const dt = switch (self.dtype()) { .c64 => .f32, .c128 => .f64, @@ -552,7 +552,7 @@ pub const Tensor = struct { res_shape = res_shape.withDtype(dt); - const op = dialects.stablehlo.bitcast_convert( + const op = stablehlo.bitcast_convert( mlirCtx(), self.value(), mlirx.Type.rankedTensor(mlirCtx(), res_shape), @@ -569,26 +569,26 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise number of leading 0 bits in the input Tensor. pub fn countLeadingZeros(self: Tensor) Tensor { - const op = dialects.stablehlo.count_leading_zeros(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = stablehlo.count_leading_zeros(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing booleans indicating if each element of the input Tensor is finite. pub fn isFinite(self: Tensor) Tensor { - const op = dialects.stablehlo.is_finite(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = stablehlo.is_finite(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape.withDtype(.bool), op.result(0)); } /// Returns a Tensor containing the element-wise number of bits set in the input Tensor. pub fn popcnt(self: Tensor) Tensor { stdx.debug.assert(self.dtype().isInteger(), "popcnt expects tensor type to be an integer, got {}", .{self.dtype()}); - const op = dialects.stablehlo.popcnt(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = stablehlo.popcnt(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing the sign of the input Tensor element-wise. pub fn sign(self: Tensor) Tensor { - const op = dialects.stablehlo.sign(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = stablehlo.sign(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -632,17 +632,17 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise left-shift operation of 'self' by 'other'. pub fn shiftLeft(self: Tensor, other: Tensor) Tensor { - return binaryOp("shiftLeft", dialects.stablehlo.shift_left)(self, other); + return binaryOp("shiftLeft", stablehlo.shift_left)(self, other); } /// Returns a Tensor containing the element-wise arithmetic right-shift operation of 'self' by 'other'. pub fn shiftRightArithmetic(self: Tensor, other: Tensor) Tensor { - return binaryOp("shiftRightArithmetic", dialects.stablehlo.shift_right_arithmetic)(self, other); + return binaryOp("shiftRightArithmetic", stablehlo.shift_right_arithmetic)(self, other); } /// Returns a Tensor containing the element-wise logical right-shift operation of 'self' by 'other'. pub fn shiftRightLogical(self: Tensor, other: Tensor) Tensor { - return binaryOp("shiftRightLogical", dialects.stablehlo.shift_right_logical)(self, other); + return binaryOp("shiftRightLogical", stablehlo.shift_right_logical)(self, other); } /// Returns the Cholesky decomposition of the input Tensor. @@ -652,16 +652,16 @@ pub const Tensor = struct { pub fn cholesky(self: Tensor, lower: bool) Tensor { stdx.debug.assert(self.rank() <= 2, "cholesky expects tensor rank to be <= 2, got {}", .{self.rank()}); - const op = dialects.stablehlo.cholesky(mlirCtx(), self.value(), lower, currentLoc()).appendTo(currentBlock()); + const op = stablehlo.cholesky(mlirCtx(), self.value(), lower, currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Solves the system of linear equations formed by the input tensors. - pub fn triangularSolve(self: Tensor, other: Tensor, opts: dialects.stablehlo.TriangularSolveOpts) Tensor { + pub fn triangularSolve(self: Tensor, other: Tensor, opts: stablehlo.TriangularSolveOpts) Tensor { stdx.debug.assert(self.dtype() == other.dtype(), "triangularSolve expects tensors to be of the same type, got {} and {}", .{ self.dtype(), other.dtype() }); stdx.debug.assert(self.rank() >= 2 and self.rank() == other.rank(), "triangularSolve expects tensors to have the same rank >= 2, got {} and {}", .{ self.rank(), other.rank() }); - const op = dialects.stablehlo.triangular_solve(mlirCtx(), self.value(), other.value(), opts, currentLoc()).appendTo(currentBlock()); + const op = stablehlo.triangular_solve(mlirCtx(), self.value(), other.value(), opts, currentLoc()).appendTo(currentBlock()); return _result(other._shape, op.result(0)); } @@ -669,7 +669,7 @@ pub const Tensor = struct { pub fn roundNearestAfz(self: Tensor) Tensor { stdx.debug.assert(self.dtype().isFloat(), "roundNearestAfz expects tensor type to be a float, got {}", .{self.dtype()}); - const op = dialects.stablehlo.round_nearest_afz(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = stablehlo.round_nearest_afz(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -677,7 +677,7 @@ pub const Tensor = struct { pub fn roundNearestEven(self: Tensor) Tensor { stdx.debug.assert(self.dtype().isFloat(), "roundNearestEven expects tensor type to be a float, got {}", .{self.dtype()}); - const op = dialects.stablehlo.round_nearest_even(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = stablehlo.round_nearest_even(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -686,7 +686,7 @@ pub const Tensor = struct { stdx.debug.assert(re._shape.eql(im._shape), "complex expects tensor shapes to match, got {f} and {f}", .{ re._shape, im._shape }); stdx.debug.assert(re.dtype() == .f32 or re.dtype() == .f64, "complex expects tensors type to be f32 or f64, got {}", .{re.dtype()}); - const op = dialects.stablehlo.complex(mlirCtx(), re.value(), im.value(), currentLoc()).appendTo(currentBlock()); + const op = stablehlo.complex(mlirCtx(), re.value(), im.value(), currentLoc()).appendTo(currentBlock()); const dt: DataType = if (re.dtype() == .f32) .c64 else .c128; return _result(re._shape.withDtype(dt), op.result(0)); } @@ -706,7 +706,7 @@ pub const Tensor = struct { .c128 => .f64, else => unreachable, }; - const op = dialects.stablehlo.real(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = stablehlo.real(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape.withDtype(dt), op.result(0)); } @@ -727,12 +727,12 @@ pub const Tensor = struct { .c128 => .f64, else => unreachable, }; - const op = dialects.stablehlo.imag(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = stablehlo.imag(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape.withDtype(dt), op.result(0)); } /// Returns the Fast Fourier Transform (FFT) of the input Tensor. - pub fn fft(self: Tensor, opts: dialects.stablehlo.FftOpts) Tensor { + pub fn fft(self: Tensor, opts: stablehlo.FftOpts) Tensor { // TODO: support tagged API. stdx.debug.assert(1 <= opts.length.len and opts.length.len <= 3, "fft expects 'opts.length' length to be between 1 and 3 (inclusive), got {}", .{opts.length.len}); @@ -771,7 +771,7 @@ pub const Tensor = struct { }, }; - const op = dialects.stablehlo.fft(mlirCtx(), self.value(), opts, currentLoc()).appendTo(currentBlock()); + const op = stablehlo.fft(mlirCtx(), self.value(), opts, currentLoc()).appendTo(currentBlock()); return _result(sh, op.result(0)); } @@ -809,7 +809,7 @@ pub const Tensor = struct { pub const Rng = struct { _state: Tensor, - algorithm: dialects.stablehlo.RngAlgorithm.Type = .DEFAULT, + algorithm: stablehlo.RngAlgorithmAttr.Type = .DEFAULT, pub const Buffer = mem.Bufferized(Rng); @@ -833,7 +833,7 @@ pub const Tensor = struct { /// The output is guaranteed to be deterministic function of `self` Rng state, /// but it is not guaranteed to be deterministic between implementations. pub fn bitGenerator(self: Rng, sh: Shape) struct { Rng, Tensor } { - const op = dialects.stablehlo.rng_bit_generator( + const op = stablehlo.rng_bit_generator( mlirCtx(), self.algorithm, self._state.value(), @@ -965,7 +965,7 @@ pub const Tensor = struct { const a = Tensor.constant(DataType.Value.init(sh.dtype(), opts.mean)); const b = Tensor.constant(DataType.Value.init(sh.dtype(), opts.stddev)); const res_tensor_shape = Tensor.constantTensor(Shape.init(.{sh.rank()}, .i64), std.mem.sliceAsBytes(sh.dims())); - const op = dialects.stablehlo.rng(mlirCtx(), a.value(), b.value(), res_tensor_shape.value(), .NORMAL, currentLoc()).appendTo(currentBlock()); + const op = stablehlo.rng(mlirCtx(), a.value(), b.value(), res_tensor_shape.value(), .NORMAL, currentLoc()).appendTo(currentBlock()); return _resultPropagateSharding(sh, op.result(0)); } @@ -1074,11 +1074,11 @@ pub const Tensor = struct { stdx.debug.assert(1 <= exponent_bits, "reducePrecision expects 'exponent_bits' to be >= 1, got {}", .{exponent_bits}); stdx.debug.assert(0 <= mantissa_bits, "reducePrecision expects 'mantissa_bits' to be positive, got {}", .{mantissa_bits}); - const op = dialects.stablehlo.reduce_precision(mlirCtx(), self.value(), exponent_bits, mantissa_bits, currentLoc()).appendTo(currentBlock()); + const op = stablehlo.reduce_precision(mlirCtx(), self.value(), exponent_bits, mantissa_bits, currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } - fn convolution(self: Tensor, other: Tensor, opts: dialects.stablehlo.ConvolutionOpts) Tensor { + fn convolution(self: Tensor, other: Tensor, opts: stablehlo.ConvolutionOpts) Tensor { stdx.debug.assert(self.rank() == other.rank(), "convolution expects tensor ranks to match, got {} and {}", .{ self.rank(), other.rank() }); const N = self.rank(); stdx.debug.guard(opts.window_strides.len == N - 2, @src()); @@ -1161,7 +1161,7 @@ pub const Tensor = struct { } // inferred shape '[1, 256, 1, 12008]' is incompatible with return type of operation '[1, 256, 1, 11978]' - const op = dialects.stablehlo.convolution( + const op = stablehlo.convolution( mlirCtx(), self.value(), other.value(), @@ -1262,42 +1262,42 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise addition of the input Tensors. pub fn add(self: Tensor, other: Tensor) Tensor { - return binaryOp("add", dialects.stablehlo.add)(self, other); + return binaryOp("add", stablehlo.add)(self, other); } /// Returns a Tensor containing the element-wise subtraction of the input Tensors. pub fn sub(self: Tensor, other: Tensor) Tensor { - return binaryOp("subtract", dialects.stablehlo.subtract)(self, other); + return binaryOp("subtract", stablehlo.subtract)(self, other); } /// Returns a Tensor containing the element-wise multiplication of the input Tensors. pub fn mul(self: Tensor, other: Tensor) Tensor { - return binaryOp("mul", dialects.stablehlo.multiply)(self, other); + return binaryOp("mul", stablehlo.multiply)(self, other); } /// Returns a Tensor containing the element-wise division of the input Tensors. pub fn div(self: Tensor, other: Tensor) Tensor { - return binaryOp("div", dialects.stablehlo.divide)(self, other); + return binaryOp("div", stablehlo.divide)(self, other); } /// Returns a Tensor containing the element-wise exponentiation of the input Tensors. pub fn pow(self: Tensor, other: Tensor) Tensor { - return binaryOp("pow", dialects.stablehlo.power)(self, other); + return binaryOp("pow", stablehlo.power)(self, other); } /// Returns a Tensor containing the element-wise maximum operation of the input Tensors. pub fn maximum(self: Tensor, other: Tensor) Tensor { - return binaryOp("maximum", dialects.stablehlo.maximum)(self, other); + return binaryOp("maximum", stablehlo.maximum)(self, other); } /// Returns a Tensor containing the element-wise minimum operation of the input Tensors. pub fn minimum(self: Tensor, other: Tensor) Tensor { - return binaryOp("minimum", dialects.stablehlo.minimum)(self, other); + return binaryOp("minimum", stablehlo.minimum)(self, other); } /// Returns a Tensor containing the element-wise remainder of dividend 'self' and divisor 'other'. pub fn remainder(self: Tensor, other: Tensor) Tensor { - return binaryOp("remainder", dialects.stablehlo.remainder)(self, other); + return binaryOp("remainder", stablehlo.remainder)(self, other); } /// Returns a Tensor containing the element-wise substraction of the input Tensor with a constant. @@ -1335,20 +1335,20 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise logical operation of the input Tensors. pub fn logical(self: Tensor, comptime logical_op: LogicalOp, other: Tensor) Tensor { return switch (logical_op) { - .OR => binaryOp("or", dialects.stablehlo.or_)(self, other), - .XOR => binaryOp("xor", dialects.stablehlo.xor)(self, other), - .AND => binaryOp("and", dialects.stablehlo.and_)(self, other), + .OR => binaryOp("or", stablehlo.or_)(self, other), + .XOR => binaryOp("xor", stablehlo.xor)(self, other), + .AND => binaryOp("and", stablehlo.and_)(self, other), }; } /// Returns a Tensor containing the element-wise floor operation of the input Tensor. pub fn floor(self: Tensor) Tensor { - return _result(self._shape, dialects.stablehlo.floor(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()).result(0)); + return _result(self._shape, stablehlo.floor(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()).result(0)); } /// Returns a Tensor containing the element-wise ceil operation of the input Tensor. pub fn ceil(self: Tensor) Tensor { - return _result(self._shape, dialects.stablehlo.ceil(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()).result(0)); + return _result(self._shape, stablehlo.ceil(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()).result(0)); } /// Returns a Tensor containing the element-wise conversion to another type. @@ -1358,7 +1358,7 @@ pub const Tensor = struct { } const res_type = mlirx.Type.rankedTensor(mlirCtx(), self.shape().withDtype(to)); - const op = dialects.stablehlo.convert(mlirCtx(), self.value(), res_type, currentLoc()).appendTo(currentBlock()); + const op = stablehlo.convert(mlirCtx(), self.value(), res_type, currentLoc()).appendTo(currentBlock()); return _result(self._shape.withDtype(to), op.result(0)); } @@ -1488,13 +1488,13 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise rounding operation of the input Tensor. pub fn round(self: Tensor) Tensor { - const round_op = dialects.stablehlo.round_nearest_even(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const round_op = stablehlo.round_nearest_even(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, round_op.result(0)); } /// Returns a Tensor containing the element-wise clamping operation of the input Tensor. pub fn clamp(self: Tensor, min_: Tensor, max_: Tensor) Tensor { - const op = dialects.stablehlo.clamp(mlirCtx(), min_.value(), self.value(), max_.value(), currentLoc()).appendTo(currentBlock()); + const op = stablehlo.clamp(mlirCtx(), min_.value(), self.value(), max_.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -1661,7 +1661,7 @@ pub const Tensor = struct { res_shape = res_shape.appendDim(rhs._shape.dim(r), rhs._shape.tag(r)); } - const op = dialects.stablehlo.dot_general( + const op = stablehlo.dot_general( mlirCtx(), lhs.value(), rhs.value(), @@ -1680,7 +1680,7 @@ pub const Tensor = struct { /// Returns a Tensor containing the sigmoid function applied to each element of the input Tensor. pub fn sigmoid(self: Tensor) Tensor { - const op = dialects.stablehlo.logistic(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = stablehlo.logistic(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -2072,7 +2072,7 @@ pub const Tensor = struct { return self.reshape(res_shape); } - const op = dialects.stablehlo.transpose( + const op = stablehlo.transpose( mlirCtx(), self.value(), mlirx.Type.rankedTensor(mlirCtx(), res_shape), @@ -2104,7 +2104,7 @@ pub const Tensor = struct { const new_dim = std.math.divExact(i64, self.dim(a), n) catch std.debug.panic("unflatten expects chosen dimension to be divisible by 'n' but {} is not divisible by {}", .{ self.dim(a), n }); const new_shape = self._shape.set(a, n).insert(a + 1, .{ ._ = new_dim }); - const reshaped_val = dialects.stablehlo.reshape( + const reshaped_val = stablehlo.reshape( mlirCtx(), self.value(), mlirx.Type.rankedTensor(mlirCtx(), new_shape), @@ -2120,7 +2120,7 @@ pub const Tensor = struct { pub fn splitAxis(self: Tensor, ax: anytype, split_shape: anytype) Tensor { const new_shape = self._shape.splitAxis(ax, split_shape); - const reshaped_val = dialects.stablehlo.reshape( + const reshaped_val = stablehlo.reshape( mlirCtx(), self.value(), mlirx.Type.rankedTensor(mlirCtx(), new_shape), @@ -2271,7 +2271,7 @@ pub const Tensor = struct { if (s.singleton) to_remove.appendAssumeCapacity(@intCast(a)); } - const op = dialects.stablehlo.dynamic_slice( + const op = stablehlo.dynamic_slice( mlirCtx(), self.value(), new_shape.dims(), @@ -2301,7 +2301,7 @@ pub const Tensor = struct { } const ctx = Compiler.current(); - const slice_op = dialects.stablehlo.slice( + const slice_op = stablehlo.slice( ctx.mlir_ctx, self.value(), start_indices[0..self.rank()], @@ -2413,7 +2413,7 @@ pub const Tensor = struct { } const res_shape = tensors[0]._shape.set(a, concatenated_dim); - const op = dialects.stablehlo.concatenate(mlirCtx(), buffer, a, currentLoc()).appendTo(currentBlock()); + const op = stablehlo.concatenate(mlirCtx(), buffer, a, currentLoc()).appendTo(currentBlock()); // log.debug("concatenate({}, {}, {d}) -> {d}", .{ tensors[0], tensors[1], a, res_shape }); return _result(res_shape, op.result(0)); } @@ -2581,55 +2581,55 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise negation of the input Tensor. pub fn negate(self: Tensor) Tensor { - const negate_op = dialects.stablehlo.negate(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const negate_op = stablehlo.negate(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, negate_op.result(0)); } /// Returns a Tensor containing the element-wise cosine of the input Tensor. pub fn cos(self: Tensor) Tensor { - const cosine_op = dialects.stablehlo.cosine(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const cosine_op = stablehlo.cosine(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, cosine_op.result(0)); } /// Returns a Tensor containing the element-wise sine of the input Tensor. pub fn sin(self: Tensor) Tensor { - const sine_op = dialects.stablehlo.sine(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const sine_op = stablehlo.sine(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, sine_op.result(0)); } /// Returns a Tensor containing the element-wise exponential operation of the input Tensor. pub fn exp(self: Tensor) Tensor { - const op = dialects.stablehlo.exponential(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = stablehlo.exponential(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing the element-wise logarithm operation of the input Tensor. pub fn log(self: Tensor) Tensor { - const op = dialects.stablehlo.log(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = stablehlo.log(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing the element-wise square-root of the input Tensor. pub fn sqrt(self: Tensor) Tensor { - const sqrt_op = dialects.stablehlo.sqrt(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const sqrt_op = stablehlo.sqrt(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, sqrt_op.result(0)); } /// Returns a Tensor containing the element-wise reverse square-root of the input Tensor. pub fn rsqrt(self: Tensor) Tensor { - const rsqrt_op = dialects.stablehlo.rsqrt(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const rsqrt_op = stablehlo.rsqrt(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, rsqrt_op.result(0)); } /// Returns a Tensor containing the element-wise hyperbolic tangent of the input Tensor. pub fn tanh(self: Tensor) Tensor { - const tanh_op = dialects.stablehlo.tanh(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const tanh_op = stablehlo.tanh(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, tanh_op.result(0)); } /// Returns a Tensor containing the element-wise exponential minus one operation of the input Tensor. pub fn exponentialMinusOne(self: Tensor) Tensor { - const expm1_op = dialects.stablehlo.exponential_minus_one(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const expm1_op = stablehlo.exponential_minus_one(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, expm1_op.result(0)); } @@ -2646,7 +2646,7 @@ pub const Tensor = struct { const n_steps = std.math.divCeil(i64, args.end - args.start, args.step) catch unreachable; const sh = Shape.init(.{n_steps}, dt); - var op = dialects.stablehlo.iota( + var op = stablehlo.iota( mlirCtx(), 0, mlirx.Type.rankedTensor(mlirCtx(), sh), @@ -2700,7 +2700,7 @@ pub const Tensor = struct { const dt: DataType = if (sh.dim(a) <= std.math.maxInt(i32)) .i32 else .i64; const res_shape = sh.withDtype(dt); - var op = dialects.stablehlo.iota( + var op = stablehlo.iota( mlirCtx(), a, mlirx.Type.rankedTensor(mlirCtx(), res_shape), @@ -2722,7 +2722,7 @@ pub const Tensor = struct { stdx.debug.assert(dt.isFloat(), "linspace expects type to be a float, got {} (hint: use arange instead)", .{dt}); const sh = Shape.init(.{args.steps}, dt); - var iota_op = dialects.stablehlo.iota( + var iota_op = stablehlo.iota( mlirCtx(), 0, mlirx.Type.rankedTensor(mlirCtx(), sh), @@ -2770,7 +2770,7 @@ pub const Tensor = struct { /// Returns a constant Tensor with the given value. pub fn constant(val: DataType.Value) Tensor { - const op = dialects.stablehlo.constant( + const op = stablehlo.constant( mlirCtx(), &.{}, mlirx.Type.fromDType(mlirCtx(), val.dtype()), @@ -2792,9 +2792,9 @@ pub const Tensor = struct { null, null, ); - const buffer = dialects.stablehlo.createBuffer(ctx.mlir_ctx, buffer_type, ctx.location) + const buffer = stablehlo.createBuffer(ctx.mlir_ctx, buffer_type, ctx.location) .appendTo(currentBlock()); - const tensor = dialects.stablehlo.unpin(ctx.mlir_ctx, buffer.result(0), ctx.location) + const tensor = stablehlo.unpin(ctx.mlir_ctx, buffer.result(0), ctx.location) .appendTo(currentBlock()); return _result(sh, tensor.result(0)); } @@ -2803,7 +2803,7 @@ pub const Tensor = struct { pub fn constantTensor(sh: Shape, bytes_: []const u8) Tensor { const elem_type = mlirx.Type.fromDType(mlirCtx(), sh.dtype()); //const elem_type = mlirx.denseElementAttrType(val.dtype()) orelse std.debug.panic("constantTensor expects a dtype that can be serialized to MLIR, like f32 or i32, got {f}", .{val.shape()}); - const constant_op = dialects.stablehlo.constant(mlirCtx(), sh.dims(), elem_type, bytes_, currentLoc()).appendTo(currentBlock()); + const constant_op = stablehlo.constant(mlirCtx(), sh.dims(), elem_type, bytes_, currentLoc()).appendTo(currentBlock()); return _result(sh, constant_op.result(0)); } @@ -2839,7 +2839,7 @@ pub const Tensor = struct { return _result(res_shape, self.value()); } const result_type = mlirx.Type.rankedTensor(mlirCtx(), res_shape); - const broadcast_op = dialects.stablehlo.broadcast_in_dim(mlirCtx(), self.value(), axes_, result_type, currentLoc()).appendTo(currentBlock()); + const broadcast_op = stablehlo.broadcast_in_dim(mlirCtx(), self.value(), axes_, result_type, currentLoc()).appendTo(currentBlock()); return _result(res_shape, broadcast_op.result(0)); } @@ -2895,7 +2895,7 @@ pub const Tensor = struct { pub fn optimizationBarrier(self: Tensor) Tensor { const ctx = Compiler.current(); - const op = dialects.stablehlo.optimizationBarrier( + const op = stablehlo.optimizationBarrier( ctx.mlir_ctx, &.{self.value()}, &.{self.value().type_()}, @@ -2909,7 +2909,7 @@ pub const Tensor = struct { pub fn reshape(self: Tensor, output_shape_: anytype) Tensor { const output_shape = self._shape.reshape(output_shape_); const tensor_type = mlirx.Type.rankedTensor(mlirCtx(), output_shape); - const reshape_value = dialects.stablehlo.reshape(mlirCtx(), self.value(), tensor_type, currentLoc()).appendTo(currentBlock()); + const reshape_value = stablehlo.reshape(mlirCtx(), self.value(), tensor_type, currentLoc()).appendTo(currentBlock()); return _result(output_shape, reshape_value.result(0)); } @@ -2947,12 +2947,12 @@ pub const Tensor = struct { } const rk = self.rank(); - const pad_op = dialects.stablehlo.pad( + const pad_op = stablehlo.pad( mlirCtx(), self.value(), Tensor.scalar(padding_value, self.dtype()).value(), - currentLoc(), .{ .low = low[0..rk], .high = high[0..rk], .interior = interior[0..rk] }, + currentLoc(), ).appendTo(currentBlock()); return _result(res_shape, pad_op.result(0)); @@ -2997,7 +2997,7 @@ pub const Tensor = struct { pub fn reverse(self: Tensor, axes_: anytype) Tensor { const actual_axes = self._shape.axes(axes_); - const reverse_op = dialects.stablehlo.reverse(mlirCtx(), self.value(), toI64(actual_axes.constSlice()).constSlice(), currentLoc()).appendTo(currentBlock()); + const reverse_op = stablehlo.reverse(mlirCtx(), self.value(), toI64(actual_axes.constSlice()).constSlice(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, reverse_op.result(0)); } @@ -3178,7 +3178,7 @@ pub const Tensor = struct { } } - const gather_op = dialects.stablehlo.gather( + const gather_op = stablehlo.gather( mlirCtx(), self.value(), indices.value(), @@ -3684,9 +3684,9 @@ pub const Tensor = struct { pub fn sort(self: Tensor, axis_: anytype, opts: SortOpts) SortRes { const a = self.axis(axis_); const indices = Tensor.arange(.{ .end = self.dim(a) }, .i32).broadcast(self._shape, &.{a}); - const direction: dialects.stablehlo.ComparisonDirection.Direction = if (opts.descending) .GT else .LT; + const direction: stablehlo.ComparisonDirectionAttr.Direction = if (opts.descending) .GT else .LT; const res = ops.sort(.{ self, indices }, a, struct { - fn call(values: ops.SortArgs, _: ops.SortArgs, direction_: dialects.stablehlo.ComparisonDirection.Direction) Tensor { + fn call(values: ops.SortArgs, _: ops.SortArgs, direction_: stablehlo.ComparisonDirectionAttr.Direction) Tensor { return values.left.cmp(direction_, values.right); } }.call, .{direction}, true); @@ -3831,7 +3831,7 @@ pub const Tensor = struct { var backend_config_buf: [32]u8 = undefined; const backend_config = std.fmt.bufPrint(&backend_config_buf, "{d}", .{k}) catch unreachable; - const op = dialects.stablehlo.custom_call( + const op = stablehlo.custom_call( ctx.mlir_ctx, &.{call_input.value()}, &result_types, @@ -4206,7 +4206,7 @@ pub const Tensor = struct { } } - const op = dialects.stablehlo.dynamic_update_slice( + const op = stablehlo.dynamic_update_slice( mlirCtx(), self.value(), update.value(), @@ -4369,7 +4369,7 @@ pub const Tensor = struct { } /// Returns a Tensor containing the element-wise result of the given 'cmp' comparison between the two input Tensors. - pub fn cmp(self: Tensor, direction: dialects.stablehlo.ComparisonDirection.Direction, other: Tensor) Tensor { + pub fn cmp(self: Tensor, direction: stablehlo.ComparisonDirectionAttr.Direction, other: Tensor) Tensor { stdx.debug.assert(self.dtype() == other.dtype(), "cmp expects input tensors to be of the same type, got {t} and {t}", .{ self.dtype(), other.dtype() }); if (self.rank() == 0 and other.rank() != 0) return self.broadcast(other._shape, &.{}).cmp(direction, other); @@ -4377,12 +4377,12 @@ pub const Tensor = struct { stdx.debug.assert(self._shape.eql(other._shape), "cmp expects input tensor shapes to match, got {f} and {f}", .{ self._shape, other._shape }); - const op = dialects.stablehlo.compare( + const op = stablehlo.compare( mlirCtx(), self.value(), other.value(), - dialects.stablehlo.ComparisonDirection.init(mlirCtx(), direction).getValue(), - getComparisonType(mlirCtx(), self.dtype()).getValue(), + stablehlo.ComparisonDirectionAttr.get(mlirCtx(), .{ .value = direction }).value(), + getComparisonType(mlirCtx(), self.dtype()).value(), currentLoc(), ).appendTo(currentBlock()); @@ -4550,7 +4550,7 @@ pub const Tensor = struct { stdx.debug.assert(bool_tensor._shape.eqlDims(on_true._shape), "select expects input tensor and 'on_true' tensor dimensions to match, got {f} and {f}", .{ bool_tensor._shape, on_true._shape }); stdx.debug.assert(bool_tensor._shape.eqlDims(on_false._shape), "select expects input tensor and 'on_false' tensor dimensions to match, got {f} and {f}", .{ bool_tensor._shape, on_false._shape }); - const op = dialects.stablehlo.select( + const op = stablehlo.select( mlirCtx(), bool_tensor.value(), on_true.value(), @@ -4567,7 +4567,7 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise not logical operation of the input Tensor. pub fn not(self: Tensor) Tensor { - const op = dialects.stablehlo.not(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = stablehlo.not(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -4858,13 +4858,13 @@ fn getPoolResDims(dt: DataType, in_dims: []const i64, base_dilations: @Vector(co return Shape.init(dims_arr[0..in_dims.len], dt); } -fn getComparisonType(ctx: *mlir.Context, dtype: DataType) *const dialects.stablehlo.CompareType { - return .init(ctx, switch (dtype.class()) { +fn getComparisonType(ctx: *mlir.Context, dtype: DataType) *const stablehlo.CompareTypeAttr { + return .get(ctx, .{ .value = switch (dtype.class()) { .bool => .UNSIGNED, .integer => if (dtype.isSignedInt()) .SIGNED else .UNSIGNED, .float => .FLOAT, .complex => @panic("Can't compare complex numbers"), - }); + } }); } test "Tensor.maxPool1d" { From c74d5ff637a0361062febcf23063d94051974392 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Mon, 26 Oct 2026 11:41:06 +0100 Subject: [PATCH 822/881] mlir/dialects: add TCL dialect and DSL (#986) Support Furiosa's TCL dialect and add the builder as well to write custom tcl kernels --- dsl/tcl/BUILD.bazel | 23 + dsl/tcl/README.md | 168 +++ dsl/tcl/builder.zig | 1424 +++++++++++++++++++++++++ dsl/tcl/dtype.zig | 57 + mlir/dialects/README.md | 6 + mlir/dialects/tcl/BUILD.bazel | 172 +++ mlir/dialects/tcl/capi.cc | 118 ++ mlir/dialects/tcl/capi.h | 53 + mlir/dialects/tcl/tcl.zig | 342 ++++++ mlir/dialects/tcl/tcl_attrs.cc | 266 +++++ mlir/dialects/tcl/tcl_attrs.h | 37 + mlir/dialects/tcl/tcl_attrs.td | 74 ++ mlir/dialects/tcl/tcl_dialect.cc | 34 + mlir/dialects/tcl/tcl_dialect.h | 9 + mlir/dialects/tcl/tcl_dialect.td | 19 + mlir/dialects/tcl/tcl_ops.cc | 473 ++++++++ mlir/dialects/tcl/tcl_ops.h | 19 + mlir/dialects/tcl/tcl_ops.td | 122 +++ mlir/dialects/tcl/tcl_types.cc | 68 ++ mlir/dialects/tcl/tcl_types.h | 18 + mlir/dialects/tcl/tcl_types.td | 23 + mlir/dialects/tcl/test.zig | 112 ++ mlir/dialects/tcl/ve_instructions.inc | 48 + zml/BUILD.bazel | 1 + zml/kernel.zig | 267 +++++ zml/ops.zig | 29 + 26 files changed, 3982 insertions(+) create mode 100644 dsl/tcl/BUILD.bazel create mode 100644 dsl/tcl/README.md create mode 100644 dsl/tcl/builder.zig create mode 100644 dsl/tcl/dtype.zig create mode 100644 mlir/dialects/tcl/BUILD.bazel create mode 100644 mlir/dialects/tcl/capi.cc create mode 100644 mlir/dialects/tcl/capi.h create mode 100644 mlir/dialects/tcl/tcl.zig create mode 100644 mlir/dialects/tcl/tcl_attrs.cc create mode 100644 mlir/dialects/tcl/tcl_attrs.h create mode 100644 mlir/dialects/tcl/tcl_attrs.td create mode 100644 mlir/dialects/tcl/tcl_dialect.cc create mode 100644 mlir/dialects/tcl/tcl_dialect.h create mode 100644 mlir/dialects/tcl/tcl_dialect.td create mode 100644 mlir/dialects/tcl/tcl_ops.cc create mode 100644 mlir/dialects/tcl/tcl_ops.h create mode 100644 mlir/dialects/tcl/tcl_ops.td create mode 100644 mlir/dialects/tcl/tcl_types.cc create mode 100644 mlir/dialects/tcl/tcl_types.h create mode 100644 mlir/dialects/tcl/tcl_types.td create mode 100644 mlir/dialects/tcl/test.zig create mode 100644 mlir/dialects/tcl/ve_instructions.inc diff --git a/dsl/tcl/BUILD.bazel b/dsl/tcl/BUILD.bazel new file mode 100644 index 00000000..5977491e --- /dev/null +++ b/dsl/tcl/BUILD.bazel @@ -0,0 +1,23 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "tcl_builder", + srcs = [ + "dtype.zig", + ], + import_name = "dsl/tcl/builder", + main = "builder.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "//mlir/dialects/arith", + "//mlir/dialects/func", + "//mlir/dialects/tcl", + ], +) + +zig_test( + name = "test", + deps = [":tcl_builder"], +) diff --git a/dsl/tcl/README.md b/dsl/tcl/README.md new file mode 100644 index 00000000..e41cf25a --- /dev/null +++ b/dsl/tcl/README.md @@ -0,0 +1,168 @@ +# TCL kernels in Zig + +`dsl/tcl` is a Zig front end for Furiosa's TCL (Tensor Contraction +Language), shaped after the Python `furiosa.tcl` DSL: a kernel declares named +axes and typed tensors, opens tensor operations that `fetch`, `contract`, run +VE instructions and `commit`, and returns tensors. The `Builder` emits the +native `tcl` MLIR dialect (`mlir/dialects/tcl`), and +`zml.kernel.tcl.Kernel` turns it into a `furiosa.tcl` custom call that the +Furiosa plugin exports to TCL source and compiles with `furiosa-tcc` as one +native unit. + +## A kernel + +`furiosa.tcl.examples.softmax`, in Zig: + +```zig +const zml = @import("zml"); +const tcl = zml.kernel.tcl; + +const Cfg = struct { b: i64, s: i64, d: i64 }; + +pub const Softmax = tcl.Kernel(Cfg, .{ + .name = "softmax_kernel", + .inputs = &.{"input"}, + .outputs = &.{"out"}, + .run = run, +}); + +fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + const B = b.axis("B", cfg.b); + const S = b.axis("S", cfg.s); + const D = b.axis("D", cfg.d); + const t = try b.declareArgs(.{ .input = .{ .dtype = .bf16, .axes = &.{ B, S, D } } }); + + const op = b.tensorOperation(.{}); + const x = op.fetch(t.input, .{ .typecast_to = .f32 }); + const exped = x.subf(x.reduce(&.{D}, .maxf)).exp(); + const out = try op.commit(exped.divf(exped.reduce(&.{D}, .addf)), .{ .dtype = .bf16 }); + b.ret(&.{out}); +} + +// In a model: +const y = Softmax.call(.{ .input = x }, .{ .out = x.shape() }, .{ + .cfg = .{ .b = 2, .s = 16, .d = 1024 }, +}).out; +``` + +The plugin compiles it to the same TCL the Python DSL prints: + +``` +def softmax_kernel(B: Axis = 2, D: Axis = 1024, S: Axis = 16, v0: [B, S, D]/bf16) -> [B, S, D]/bf16: + v1: [B, S, D]/bf16 = tk.ReduceByVe( + %0 = read(v0, typecast_to=f32) + %1 = ve.exec(max_f t[D] %0) + %2 = ve.exec(%0 - %1) + %3 = ve.exec(exp %2) + %4 = ve.exec(+ t[D] %3) + %5 = ve.exec(%3 / %4) + write.to([B, S, D]/bf16) + ) + return v1 +``` + +## Python to Zig + +| `furiosa.tcl` | `dsl/tcl` | +|------------------------------------------------|-------------------------------------------------------| +| `x: tcl.Tensor[tcl.bf16, [B, S, D]]` | `.x = .{ .dtype = .bf16, .axes = &.{ B, S, D } }` | +| `x: tcl.Dram[tcl.bf16, [tcl.Broadcast], [M, N]]` | `.x = .{ .dtype = .bf16, .axes = &.{ M, N }, .dram = .{} }` | +| composite axis `C: Axis = (A * B)` | `b.axis("C", b.expr(.mul, A, B))` | +| `@tcl.kernel(config={...}, auto_config={...})` | `b.compilerConfig(.{ ... })`, `b.autoConfig(.{ ... })` | +| `@tcl.tensor_operation` | `const op = b.tensorOperation(.{})` | +| `fetch.typecast(x, tcl.f32)` | `op.fetch(x, .{ .typecast_to = .f32 })` | +| `fetch.broadcast(x, ...)`, `fetch.table_lookup` | `.broadcast_to = &.{...}`, `.table_lookup = table` | +| `fetch.pad(x, axis=L, left=1, right=1)` | `.pad = &.{.{ .axis = L, .left = 1, .right = 1 }}` | +| `fetch.slide(x, axis=L, frame_axis=Lf, window_axis=Lw)` | `.slide = &.{.{ .axis = L, .frame_axis = Lf, .window_axis = Lw }}` | +| `operation.contract(lhs, rhs, to=[B, M, N])` | `op.contract(lhs, rhs, &.{ B, M, N })` | +| `operation.subf(a, b)`, `operation.exp(a)` | `a.subf(b)`, `a.exp()`, or `op.ve(.subf, .{ a, b })` | +| `operation.reduce(x, axes=[D], op=tcl.Maxf)` | `x.reduce(&.{D}, .maxf)` | +| `operation.where(x > 0, a, b)` | `op.where(x, .gt, 0, a, b)` | +| `operation.to_fp(x, 15)`, `to_fxp(x, 15)` | `x.toFp(15)`, `x.toFxp(15)` | +| `compiler_hints=tcl.context(layout={Chip: ...})` | `.context = .{ .layout = .{ .chip = .broadcast } }` | +| `commit.typecast(v, tcl.bf16)` / `commit.permute` | `op.commit(v, .{ .dtype = .bf16, .axes = ... })` | +| `tcl.reshape`, `transmute`, `concat`, `slice` | `b.reshape`, `b.transmute`, `b.concat`, `b.slice` | +| `tcl.gather`, `scatter`, `arange`, `vector`, `all_gather` | `b.gather`, `b.scatter`, `b.arange`, `b.vector`, `b.allGather` | +| `for i in tcl.range(0, n):` (carried `acc`) | `var loop = b.openFor(n, .{acc})` ... `try loop.yield(.{new_acc})`, then `loop.results` | +| `x[i]`, `out[i] = v`, `tcl.scratchpad()` | `b.indexRead(x, loop.iv)`, `b.indexWrite(out, loop.iv, v)`, `b.scratchpad` | +| `core.reduce_max_i32(t)`, `core.eval_expr(...)` | `b.reduceMaxI32(t)`, `b.symExpr(.div, n, 64)` | +| `return out` | `b.ret(&.{out})` | + +* **Axes are static.** `b.axis(name, size)` declares `name: Axis = size`, + with an integer or an `Expr` over other axes; axis names start with an + uppercase letter. `furiosa-tcc` needs every axis bound. Kernel inputs and + outputs must match the XLA shapes axis by axis (row-major), which `call` + checks. +* **DRAM tensors.** A `.dram` argument comes back as its `as_logical` view; + `b.asDram(t, .{...})` returns a result in a DRAM mapping. +* **Multi-chip kernels.** A kernel that places data or work on chips (a + chip-split DRAM tensor, a `.context`, `allGather`) is compiled by the plugin + for all the chips of its SPMD program (2 or 4), and runs as one launch + across the partitions; other kernels stay per chip. Its arguments and + results are DRAM tensors: `Dram{ .chip = &.{.{ .axis = C }} }` splits axis + `C`, sized to the chip count, across chips (each partition holds its share), + the default `.broadcast` keeps a copy per chip. Call it inside + `zml.ops.manualComputation` so it sees each partition's shards. +* **Convolutions** pad and slide the input in its read: a slide replaces its + axis by `window_axis, frame_axis`, then `contract` with the filter. On its + own (without a slide) the padding is dropped. +* **The mainstream rule.** As in the Python DSL, instructions are pipelined: + `commit` must take the last instruction, there is at most one contraction + and it comes before VE instructions. Reads are hoisted in front. +* **VE registers are f32 or i32**, decided by the instruction. Scalar + operands become `f32`/`i32` constants (`x.mulf(0.5)`). +* **Tactics are guessed** like `@tcl.tensor_operation` (`EinsumByDpe`, + `ReduceByVe`, `Interleaving`, ...); `b.tensorOperation(.{ .tactic = ... })` + overrides it. +* **Tensors passed directly** to an instruction are read without modifiers. +* **Constant outputs.** `furiosa-tcc` 2026.3 fails ("No producer") on a + function returning two input-independent tensors, such as `arange` and + `vector`. +* **Loops** run on the device. `x[i]` and `acc[i] = v` address the outermost + axis, and the body must index something with `i`: that bounds the trip + count. A dynamic trip count must be computed in the kernel (`reduceMaxI32`, + `symExpr`); one read straight from an argument adds a CPU node the plugin + rejects. + +## Good practices on RNGD + +The TCL CPU interpreter (`furiosa.torch.TclModule`) accepts several programs +that compute garbage on the hardware: check kernels on RNGD. + +* **One VE stream per tensor operation.** Each instruction takes the previous + result and one side operand (a read or a constant). A select whose branches + are both constants must test the operation's first read. +* **Outer broadcasts** (`[B, H, V] * [B, H, K]` to `[B, H, V, K]`, tactic + `EinsumByVe`) need a shared leading axis, and the read with more axes first. +* **Contraction epilogues** read their extra tensors before the contraction + and combine the contraction result first: `contract(q, k).addf(bias)`. +* **Conversions.** `typecast_to` converts floats only; `toFp(n)` of an i32 + yields `x / 2^31` whatever `n`. +* **Update caches in place.** `scatter(rows, idx, axis, .{ .init = cache })` + on a kernel argument shares its buffer with the result (a whole-cache copy + otherwise), so return it as an aliased output. +* **Few axes, few operations.** tcc's tactic search grows with the number and + extents of axes, and more than linearly with the number of operations in a + function: merge axes before contractions (`[N, H, G*Q, C]`, not + `[N, H, G, Q, P, C]`) and set `.lowering_mode = "Heuristic"` above a few + tokens. +* **Prefer one-shot graphs to hand-written online softmax.** Furiosa's + production attention gathers the context, contracts, masks with a select, + runs the softmax as one `ReduceByVe` and contracts again; gathers take + `valid_length` to skip padding, and padding indices are spread + (`idx | (pos % n)`) rather than all pointing at one row. +* **Furiosa's kernels** in the SDK (`furiosa/kernels/`) are the reference for + layouts, tile sizes and compiler configs: `common/attention.py` (attention + and decode/prefill configs), `common/topk.py` and + `common/blockwise_moe_index.py` (routing on the device), + `qwen3_moe/optimized/` (blockwise MoE for prefill, activation-stationary MoE + for decode), `exaone_moe/k_exaone_w4fa16kv16/flash_attention.py` (chunked + softmax over a dynamic trip count). + +## The custom call + +`ops.tcl` emits `stablehlo.custom_call @furiosa.tcl` with the printed module +as its backend config. The plugin parses it, checks the function signature +against the operand and result shapes, exports TCL source and compiles it +with the rest of the program's units. With `--xla_dump_to=`, +`*.furiosa-unit-sources.txt` holds each unit's TCL. diff --git a/dsl/tcl/builder.zig b/dsl/tcl/builder.zig new file mode 100644 index 00000000..2fb6f61e --- /dev/null +++ b/dsl/tcl/builder.zig @@ -0,0 +1,1424 @@ +const std = @import("std"); + +const arith = @import("mlir/dialects/arith"); +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const tcl = @import("mlir/dialects/tcl"); + +pub const dtypes = @import("dtype.zig"); + +pub const DType = dtypes.DType; +pub const Tactic = tcl.Tactic; +pub const ReduceMode = tcl.ReduceMode; +pub const Predicate = tcl.Predicate; + +pub fn registerDialects(registry: *mlir.DialectRegistry) void { + inline for (.{ func, arith, tcl }) |dialect| dialect.registerDialect(registry); +} + +pub const FinishError = error{ InvalidMlir, OutOfMemory } || std.Io.Writer.Error; + +test { + std.testing.refAllDecls(@This()); +} + +/// A named TCL axis with its extent, from `Builder.axis`. +pub const Axis = struct { + name: []const u8, + size: i64, +}; + +/// A TCL expression over axes and integers, for composite axes +/// (`C: Axis = (A * B)`) and DRAM mappings (`D @ Dp`, `F / 2`, `Broadcast`). +/// `Builder.expr` makes the nodes. +pub const Expr = union(enum) { + int: i64, + axis: Axis, + broadcast, + node: *const Node, + + pub const Kind = enum { add, sub, mul, div, rem, exact_div, stride, modulo, padding, resize, pair }; + pub const Node = struct { kind: Kind, lhs: Expr, rhs: Expr }; + + /// The value of integer arithmetic over axes; null for mappings. + pub fn value(self: Expr) ?i64 { + return switch (self) { + .int => |n| n, + .axis => |a| a.size, + .broadcast => null, + .node => |n| { + const l = n.lhs.value() orelse return null; + const r = n.rhs.value() orelse return null; + return switch (n.kind) { + .add => l + r, + .sub => l - r, + .mul => l * r, + .div, .exact_div => if (r == 0) null else @divFloor(l, r), + .rem => if (r == 0) null else @mod(l, r), + else => null, + }; + }, + }; + } +}; + +/// `dram(chip | inner)/T`: a tensor in an explicit DRAM mapping, Python's +/// `tcl.Dram[T, chip, inner, tcl.original[...]]`. `chip` is `.broadcast` +/// (a copy per chip) or the axes split across chips; `inner` defaults to the +/// tensor's other axes. +pub const Dram = struct { + chip: []const Expr = &.{.broadcast}, + inner: ?[]const Expr = null, + original: []const Axis = &.{}, + + fn shards(self: Dram) i64 { + var n: i64 = 1; + for (self.chip) |e| if (e == .axis) { + n *= e.axis.size; + }; + return n; + } + + fn isChipAxis(self: Dram, a: Axis) bool { + for (self.chip) |e| if (e == .axis and std.mem.eql(u8, e.axis.name, a.name)) return true; + return false; + } +}; + +/// `@ context(operator = {...}, heuristic_hint = {...})`, Python's +/// `tcl.context(layout=..., heuristic_hint=...)`. +pub const Context = struct { + layout: ?Layout = null, + heuristic_hint: []const Hint = &.{}, + + /// `{Chip: ..., Cluster: ..., Split: [...]}`: `chip` and `cluster` are an + /// `.axis` or `.broadcast`. + pub const Layout = struct { + chip: Expr, + cluster: ?Expr = null, + split: []const Axis = &.{}, + }; + + pub const Hint = struct { + name: []const u8, + value: union(enum) { int: i64, float: f64, boolean: bool, axis: Axis }, + }; +}; + +/// The VE (vector engine) instructions, named like `furiosa.tcl.operation`. +pub const VeOp = enum { + exp, + neg_exp, + sqrt, + tanh, + sigmoid, + erf, + log, + sin, + cos, + negf, + absf, + addi, + addi_sat, + subi, + subi_sat, + mul_q31, + muli, + shl, + shl_sat, + shr_logical, + shr_arith, + shr_arith_round, + bit_andi, + bit_ori, + bit_xori, + mini, + maxi, + absmini, + absmaxi, + addf, + subf, + mulf, + divf, + bit_andf, + bit_orf, + bit_xorf, + minf, + maxf, + absminf, + absmaxf, + reinterpret_as_float, + reinterpret_as_int, + to_float, + to_int, + + pub fn arity(self: VeOp) usize { + return switch (self) { + .exp, .neg_exp, .sqrt, .tanh, .sigmoid, .erf, .log, .sin, .cos, .negf, .absf, .reinterpret_as_float, .reinterpret_as_int, .to_float, .to_int => 1, + else => 2, + }; + } + + /// VE registers are f32 or i32, according to the instruction. + pub fn result(self: VeOp) DType { + return switch (self) { + .exp, .neg_exp, .sqrt, .tanh, .sigmoid, .erf, .log, .sin, .cos, .negf, .absf, .addf, .subf, .mulf, .divf, .bit_andf, .bit_orf, .bit_xorf, .minf, .maxf, .absminf, .absmaxf, .reinterpret_as_float, .to_float => .f32, + else => .i32, + }; + } +}; + +/// A graph-level tensor: a kernel argument, or the result of a tensor or +/// graph operation (`x: [A, B]/bf16` in TCL). +pub const Tensor = struct { + inner: *const mlir.Value, + dtype: DType, + axes: []const Axis, + /// Chips a DRAM tensor is split across: each SPMD partition holds + /// 1/shards of it. + shards: i64 = 1, +}; + +/// A value inside a tensor operation (`%N` in TCL): the result of a read, +/// the contraction or a VE instruction. +pub const Value = struct { + op: *TensorOperation, + inner: *const mlir.Value, + dtype: DType, + axes: []const Axis, + + pub fn unary(self: Value, comptime opcode: VeOp) Value { + return self.op.ve(opcode, .{self}); + } + + /// `rhs` is a `Value`, a `Tensor` (read on use) or a scalar literal. + pub fn binary(self: Value, comptime opcode: VeOp, rhs: anytype) Value { + return self.op.ve(opcode, .{ self, rhs }); + } + + pub fn reduce(self: Value, axes: []const Axis, mode: ReduceMode) Value { + return self.op.reduce(self, axes, mode); + } + + pub fn cumsum(self: Value, axes: []const Axis) Value { + return self.op.cumsum(self, axes); + } + + pub fn addf(self: Value, rhs: anytype) Value { + return self.binary(.addf, rhs); + } + pub fn subf(self: Value, rhs: anytype) Value { + return self.binary(.subf, rhs); + } + pub fn mulf(self: Value, rhs: anytype) Value { + return self.binary(.mulf, rhs); + } + pub fn divf(self: Value, rhs: anytype) Value { + return self.binary(.divf, rhs); + } + pub fn maxf(self: Value, rhs: anytype) Value { + return self.binary(.maxf, rhs); + } + pub fn minf(self: Value, rhs: anytype) Value { + return self.binary(.minf, rhs); + } + pub fn addi(self: Value, rhs: anytype) Value { + return self.binary(.addi, rhs); + } + pub fn subi(self: Value, rhs: anytype) Value { + return self.binary(.subi, rhs); + } + pub fn muli(self: Value, rhs: anytype) Value { + return self.binary(.muli, rhs); + } + pub fn exp(self: Value) Value { + return self.unary(.exp); + } + pub fn sqrt(self: Value) Value { + return self.unary(.sqrt); + } + pub fn tanh(self: Value) Value { + return self.unary(.tanh); + } + pub fn sigmoid(self: Value) Value { + return self.unary(.sigmoid); + } + pub fn log(self: Value) Value { + return self.unary(.log); + } + pub fn negf(self: Value) Value { + return self.unary(.negf); + } + pub fn toFloat(self: Value) Value { + return self.unary(.to_float); + } + pub fn toInt(self: Value) Value { + return self.unary(.to_int); + } + pub fn toFp(self: Value, int_width: u5) Value { + return self.op.toFp(self, int_width); + } + pub fn toFxp(self: Value, int_width: u5) Value { + return self.op.toFxp(self, int_width); + } +}; + +/// One TCL tensor operation, `tk.(...)`: reads, at most one +/// contraction, VE instructions, then `commit` writes its result tensor. +/// Like `@tcl.tensor_operation`, every instruction after the reads is +/// pipelined: `commit` takes the last one. +pub const TensorOperation = struct { + pub const Options = struct { + /// Guessed from the instructions like the Python DSL when null. + tactic: ?Tactic = null, + context: ?Context = null, + }; + + pub const FetchOptions = struct { + typecast_to: ?DType = null, + broadcast_to: ?[]const Axis = null, + subtraction: ?Tensor = null, + table_lookup: ?Tensor = null, + /// Padding along an axis, applied by a `slide` over it. + pad: []const Pad = &.{}, + /// Sliding windows: each replaces its axis by `window_axis, frame_axis`. + slide: []const Slide = &.{}, + }; + + pub const Slide = struct { + axis: Axis, + frame_axis: Axis, + window_axis: Axis, + /// The window before dilation; `window_axis.size` by default. + undilated_window: ?i64 = null, + stride: i64 = 1, + dilation: i64 = 1, + }; + + pub const Pad = struct { + axis: Axis, + left: i64 = 0, + right: i64 = 0, + fill: f64 = 0, + }; + + pub const CommitOptions = struct { + dtype: ?DType = null, + /// A permutation of the committed value's axes. + axes: ?[]const Axis = null, + }; + + b: *Builder, + tactic: ?Tactic, + context: ?Context, + body: *mlir.Block, + inputs: std.ArrayList(Tensor) = .empty, + plain_reads: std.ArrayList(struct { tensor: *const mlir.Value, value: Value }) = .empty, + first_compute: ?*mlir.Operation = null, + last: ?*const mlir.Value = null, + has_dpe: bool = false, + has_ve: bool = false, + has_reduce: bool = false, + has_select: bool = false, + + /// `read(t, typecast_to=..., broadcast_to=..., ...)`. + pub fn fetch(self: *TensorOperation, t: Tensor, opts: FetchOptions) Value { + const b = self.b; + const ctx = b.ctx; + var operands: [3]*const mlir.Value = .{ self.input(t), undefined, undefined }; + var n: usize = 1; + var fields: [6]mlir.NamedAttribute = undefined; + var nfields: usize = 0; + var dtype = t.dtype; + var axes = t.axes; + if (opts.subtraction) |s| { + operands[n] = self.input(s); + fields[nfields] = .named(ctx, "subtraction", .int(ctx, .i64, n)); + n += 1; + nfields += 1; + } + if (opts.table_lookup) |table| { + operands[n] = self.input(table); + fields[nfields] = .named(ctx, "table_lookup", .int(ctx, .i64, n)); + dtype = table.dtype; + n += 1; + nfields += 1; + } + if (opts.typecast_to) |to| { + fields[nfields] = .named(ctx, "typecast_to", .typeAttr(to.toMlir(ctx))); + dtype = to; + nfields += 1; + } + if (opts.pad.len > 0) { + const pads = b.arena.allocator().alloc(mlir.NamedAttribute, opts.pad.len) catch @panic("OOM"); + for (pads, opts.pad) |*dst, p| dst.* = .named(ctx, p.axis.name, .array(ctx, &.{ + .int(ctx, .i64, p.left), + .int(ctx, .i64, p.right), + .float(ctx, .f64, p.fill), + })); + fields[nfields] = .named(ctx, "pad", .dict(ctx, pads)); + nfields += 1; + } + if (opts.slide.len > 0) { + const slides = b.arena.allocator().alloc(mlir.NamedAttribute, opts.slide.len) catch @panic("OOM"); + var slid: std.ArrayList(Axis) = .empty; + slid.appendSlice(b.arena.allocator(), axes) catch @panic("OOM"); + for (slides, opts.slide) |*dst, sl| { + dst.* = .named(ctx, sl.axis.name, .dict(ctx, &.{ + .named(ctx, "undilated_window", .int(ctx, .i64, sl.undilated_window orelse sl.window_axis.size)), + .named(ctx, "frame_axis", b.axisAttr(sl.frame_axis)), + .named(ctx, "window_axis", b.axisAttr(sl.window_axis)), + .named(ctx, "stride", .int(ctx, .i64, sl.stride)), + .named(ctx, "dilation", .int(ctx, .i64, sl.dilation)), + })); + const at = for (slid.items, 0..) |a, i| { + if (std.mem.eql(u8, a.name, sl.axis.name)) break i; + } else std.debug.panic("tcl: {s}: slide axis {s} is not read", .{ b.name, sl.axis.name }); + slid.items[at] = sl.window_axis; + slid.insert(b.arena.allocator(), at + 1, sl.frame_axis) catch @panic("OOM"); + } + fields[nfields] = .named(ctx, "slide", .dict(ctx, slides)); + nfields += 1; + axes = slid.items; + } + if (opts.broadcast_to) |to| { + fields[nfields] = .named(ctx, "broadcast_to", b.axesAttr(to)); + axes = b.dupeAxes(to); + nfields += 1; + } + const options = if (nfields == 0) null else tcl.readOptionsAttr(ctx, fields[0..nfields]) catch + std.debug.panic("tcl: invalid read options for {s}", .{b.name}); + const op = tcl.read(ctx, operands[0..n], b.logicalType(dtype, axes), .{ .options = options }, b.loc()); + return self.emit(op, dtype, axes, true); + } + + /// `dpe.exec(lhs, rhs -> [to])`: shared axes missing from `to` are + /// contracted. Accumulates in f32 (float inputs) or i32. + pub fn contract(self: *TensorOperation, lhs: anytype, rhs: anytype, to: []const Axis) Value { + if (self.has_dpe or self.has_ve) std.debug.panic("tcl: one contraction per tensor operation, before VE instructions", .{}); + const l = self.lift(lhs, .f32); + const r = self.lift(rhs, .f32); + const dtype: DType = if (l.dtype.isFloat()) .f32 else .i32; + self.has_dpe = true; + const op = tcl.dpe(self.b.ctx, l.inner, r.inner, self.b.logicalType(dtype, to), self.b.loc()); + return self.emit(op, dtype, self.b.dupeAxes(to), false); + } + + /// `ve.exec(...)` of a unary or binary instruction. Operands are + /// `Value`s, `Tensor`s (read on use) or scalar literals. + pub fn ve(self: *TensorOperation, comptime opcode: VeOp, operands: anytype) Value { + if (comptime operands.len != opcode.arity()) @compileError("tcl: wrong operand count for " ++ @tagName(opcode)); + return self.veNamed(@tagName(opcode), opcode.result(), operands); + } + + /// `ve.exec(x to_f)`: Q-format fixed point to f32, Python's + /// `operation.to_fp`. + pub fn toFp(self: *TensorOperation, x: anytype, int_width: u5) Value { + var buf: [8]u8 = undefined; + return self.veNamed(std.fmt.bufPrint(&buf, "to_f{d}", .{int_width}) catch unreachable, .f32, .{x}); + } + + /// `ve.exec(x to_i)`: f32 to Q-format fixed point, Python's + /// `operation.to_fxp`. + pub fn toFxp(self: *TensorOperation, x: anytype, int_width: u5) Value { + var buf: [8]u8 = undefined; + return self.veNamed(std.fmt.bufPrint(&buf, "to_i{d}", .{int_width}) catch unreachable, .i32, .{x}); + } + + fn veNamed(self: *TensorOperation, opcode: []const u8, scalar: DType, operands: anytype) Value { + var values: [2]*const mlir.Value = undefined; + var axes: []const Axis = &.{}; + inline for (operands, 0..) |o, i| { + const v = self.lift(o, scalar); + values[i] = v.inner; + axes = self.b.unionAxes(axes, v.axes); + } + self.has_ve = true; + const ctx = self.b.ctx; + const code = tcl.veOpcodeAttr(ctx, opcode) catch std.debug.panic("tcl: unknown VE instruction {s}", .{opcode}); + const op = tcl.ve(ctx, values[0..operands.len], self.b.logicalType(scalar, axes), code, self.b.loc()); + return self.emit(op, scalar, axes, false); + } + + /// `ve.exec( t[axes] x)`: removes `axes`. + pub fn reduce(self: *TensorOperation, x: anytype, axes: []const Axis, mode: ReduceMode) Value { + if (mode == .cumsum) return self.cumsum(x, axes); + const v = self.lift(x, .f32); + var kept: std.ArrayList(Axis) = .empty; + for (v.axes) |a| { + if (!containsAxis(axes, a)) kept.append(self.b.arena.allocator(), a) catch @panic("OOM"); + } + self.has_ve = true; + self.has_reduce = true; + const op = tcl.ve_reduce(self.b.ctx, v.inner, self.b.logicalType(v.dtype, kept.items), mode, self.b.axisAttrs(axes), self.b.loc()); + return self.emit(op, v.dtype, kept.items, false); + } + + /// `cumsum t[axes] x`, like `operation.cumsum`: the running sum of an i32 + /// value along `axes`, flattened in that order. + pub fn cumsum(self: *TensorOperation, x: anytype, axes: []const Axis) Value { + const v = self.lift(x, .i32); + self.has_ve = true; + self.has_reduce = true; + const op = tcl.ve_reduce(self.b.ctx, v.inner, self.b.logicalType(v.dtype, v.axes), .cumsum, self.b.axisAttrs(axes), self.b.loc()); + return self.emit(op, v.dtype, v.axes, false); + } + + /// `ve.exec(if cond threshold { yes } else { no })`, like + /// `operation.where(cond threshold, yes, no)`. + pub fn where(self: *TensorOperation, cond: anytype, pred: Predicate, threshold: anytype, yes: anytype, no: anytype) Value { + const ctx = self.b.ctx; + const c = self.lift(cond, .f32); + const float = isFloatOperand(yes) or isFloatOperand(no); + const scalar: DType = if (float) .f32 else .i32; + const t = self.lift(yes, scalar); + const f = self.lift(no, scalar); + const threshold_attr: *const mlir.Attribute = if (c.dtype.isFloat()) + .float(ctx, .f32, threshold) + else + .int(ctx, .i32, threshold); + const axes = self.b.unionAxes(self.b.unionAxes(c.axes, t.axes), f.axes); + self.has_ve = true; + self.has_select = true; + const op = tcl.ve_select(ctx, c.inner, t.inner, f.inner, self.b.logicalType(scalar, axes), pred, threshold_attr, self.b.loc()); + return self.emit(op, scalar, axes, false); + } + + /// `write.to(...)`: closes the tensor operation and returns its result. + pub fn commit(self: *TensorOperation, v: Value, opts: CommitOptions) FinishError!Tensor { + const b = self.b; + if (self.last != v.inner) std.debug.panic("tcl: {s}: commit must take the last instruction", .{b.name}); + const axes = if (opts.axes) |a| b.dupeAxes(a) else v.axes; + if (!sameAxisSet(axes, v.axes)) std.debug.panic("tcl: {s}: commit can only permute axes", .{b.name}); + const dtype = opts.dtype orelse v.dtype; + _ = tcl.write(b.ctx, v.inner, b.loc()).appendTo(self.body); + + const inputs = try b.arena.allocator().alloc(*const mlir.Value, self.inputs.items.len); + for (inputs, self.inputs.items) |*dst, t| dst.* = t.inner; + const context = if (self.context) |c| b.contextAttr(c) else null; + const kernel = tcl.kernel(b.ctx, inputs, b.logicalType(dtype, axes), self.tactic orelse self.guessTactic(axes), self.body, .{ .context = context }, b.loc()); + try b.append(kernel); + return .{ .inner = kernel.result(0), .dtype = dtype, .axes = axes }; + } + + /// Python's `BuilderHistory.guess_tactic_kind`. + fn guessTactic(self: *const TensorOperation, output: []const Axis) Tactic { + if (self.has_dpe) return .einsum_by_dpe; + if (!self.has_ve) return .tensor_operation; + if (self.has_reduce) return .reduce_by_ve; + const in = self.inputs.items; + if (in.len > 0 and in[0].axes.len < output.len) return .einsum_by_ve; + if (!self.has_select and in.len == 2 and sameAxisSet(in[0].axes, in[1].axes)) return .interleaving; + return .elementwise; + } + + const Operand = struct { inner: *const mlir.Value, dtype: DType, axes: []const Axis }; + + fn lift(self: *TensorOperation, x: anytype, scalar: DType) Operand { + const T = @TypeOf(x); + if (T == Value) return .{ .inner = x.inner, .dtype = x.dtype, .axes = x.axes }; + if (T == Tensor) { + const r = self.plainRead(x); + return .{ .inner = r.inner, .dtype = r.dtype, .axes = r.axes }; + } + const ctx = self.b.ctx; + const op = switch (@typeInfo(T)) { + .comptime_int, .int => if (scalar == .f32) + arith.constant_float(ctx, @floatFromInt(x), .f32, self.b.loc()) + else + arith.constant_int(ctx, x, .int(ctx, .i32), self.b.loc()), + .comptime_float, .float => blk: { + if (scalar != .f32) std.debug.panic("tcl: float literal for an integer VE instruction", .{}); + break :blk arith.constant_float(ctx, x, .f32, self.b.loc()); + }, + else => @compileError("tcl: expected a Value, a Tensor or a scalar, got " ++ @typeName(T)), + }; + _ = op.appendTo(self.body); + return .{ .inner = op.result(0), .dtype = scalar, .axes = &.{} }; + } + + /// Registers `t` as a kernel operand. + fn input(self: *TensorOperation, t: Tensor) *const mlir.Value { + if (t.inner.type_().isA(tcl.LogicalType) == null) + std.debug.panic("tcl: {s}: tensor operations read logical tensors, not DRAM ones", .{self.b.name}); + for (self.inputs.items) |known| { + if (known.inner == t.inner) return t.inner; + } + self.inputs.append(self.b.arena.allocator(), t) catch @panic("OOM"); + return t.inner; + } + + fn plainRead(self: *TensorOperation, t: Tensor) Value { + for (self.plain_reads.items) |r| if (r.tensor == t.inner) return r.value; + const r = self.fetch(t, .{}); + self.plain_reads.append(self.b.arena.allocator(), .{ .tensor = t.inner, .value = r }) catch @panic("OOM"); + return r; + } + + /// Reads go before every other instruction, as TCL lists them. + fn emit(self: *TensorOperation, op: *mlir.Operation, dtype: DType, axes: []const Axis, is_read: bool) Value { + if (is_read and self.first_compute != null) { + self.body.insertOwnedOperationBefore(self.first_compute.?, op); + } else { + _ = op.appendTo(self.body); + if (!is_read and self.first_compute == null) self.first_compute = op; + self.last = op.result(0); + } + return .{ .op = self, .inner = op.result(0), .dtype = dtype, .axes = axes }; + } +}; + +fn isFloatOperand(x: anytype) bool { + return switch (@typeInfo(@TypeOf(x))) { + .comptime_float, .float => true, + .comptime_int, .int => false, + else => x.dtype.isFloat(), + }; +} + +fn containsAxis(axes: []const Axis, a: Axis) bool { + for (axes) |x| if (std.mem.eql(u8, x.name, a.name)) return true; + return false; +} + +fn sameAxisSet(a: []const Axis, b: []const Axis) bool { + if (a.len != b.len) return false; + for (a) |x| if (!containsAxis(b, x)) return false; + return true; +} + +/// Builds one TCL function, `def name(axes..., args...) -> results:`, as a +/// `func.func` of the native `tcl` dialect. +pub const Builder = struct { + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + ctx: *mlir.Context, + module: *mlir.Module, + name: []const u8, + axes: std.ArrayList(Declared) = .empty, + compiler_config: ?*const mlir.Attribute = null, + auto_config: ?*const mlir.Attribute = null, + fusible: bool = false, + func_op: ?*mlir.Operation = null, + entry: ?*mlir.Block = null, + args: []const Tensor = &.{}, + results: ?[]const Tensor = null, + /// Arguments `update` writes back into, by argument number. + updated: std.ArrayList(usize) = .empty, + /// Blocks being built inside the entry block (loop bodies), innermost last. + block_stack: std.ArrayList(*mlir.Block) = .empty, + + pub fn open(allocator: std.mem.Allocator, ctx: *mlir.Context, name: []const u8) !Builder { + _ = tcl.symbolAttr(ctx, name) catch std.debug.panic("tcl: `{s}` is not a TCL function name", .{name}); + return .{ + .allocator = allocator, + .arena = .init(allocator), + .ctx = ctx, + .module = .init(.unknown(ctx)), + .name = name, + }; + } + + pub fn deinit(self: *Builder) void { + self.module.deinit(); + self.arena.deinit(); + } + + pub fn loc(self: *const Builder) *const mlir.Location { + return .unknown(self.ctx); + } + + const Declared = struct { axis: Axis, def: ?Expr }; + + /// `A: Axis = size`, where `size` is an integer or an `Expr` over other + /// axes (`b.axis("C", b.expr(.mul, A, B))`). Axis names start with an + /// uppercase letter. + pub fn axis(self: *Builder, name: []const u8, size: anytype) Axis { + const def: ?Expr, const n: i64 = if (comptime @TypeOf(size) == Expr) + .{ size, size.value() orelse std.debug.panic("tcl: axis {s} needs an integer size", .{name}) } + else + .{ null, size }; + if (name.len == 0 or !std.ascii.isUpper(name[0]) or n <= 0) + std.debug.panic("tcl: invalid axis {s} = {d}", .{ name, n }); + _ = tcl.symbolAttr(self.ctx, name) catch std.debug.panic("tcl: invalid axis name {s}", .{name}); + for (self.axes.items) |d| if (std.mem.eql(u8, d.axis.name, name)) { + if (d.axis.size != n) std.debug.panic("tcl: axis {s} redeclared as {d} (was {d})", .{ name, n, d.axis.size }); + return d.axis; + }; + const a: Axis = .{ .name = self.arena.allocator().dupe(u8, name) catch @panic("OOM"), .size = n }; + self.axes.append(self.arena.allocator(), .{ .axis = a, .def = def }) catch @panic("OOM"); + return a; + } + + /// An `Expr` node: `lhs` and `rhs` are axes, integers or expressions. + pub fn expr(self: *Builder, kind: Expr.Kind, lhs: anytype, rhs: anytype) Expr { + const node = self.arena.allocator().create(Expr.Node) catch @panic("OOM"); + node.* = .{ .kind = kind, .lhs = toExpr(lhs), .rhs = toExpr(rhs) }; + return .{ .node = node }; + } + + fn toExpr(x: anytype) Expr { + return switch (@TypeOf(x)) { + Expr => x, + Axis => .{ .axis = x }, + else => .{ .int = x }, + }; + } + + /// `@compiler_config({...})`, Python's `@tcl.kernel(config=...)`: an + /// anonymous struct written out as JSON, `.{ .lowering_mode = "Heuristic" }`. + pub fn compilerConfig(self: *Builder, config: anytype) void { + self.compiler_config = tcl.configAttr(self.ctx, self.jsonFields(config)) catch + std.debug.panic("tcl: {s}: invalid compiler config", .{self.name}); + } + + /// Lets the plugin compile the function into the unit of the work around + /// it instead of a unit of its own; the unit's compiler config applies. + pub fn fuse(self: *Builder) void { + self.fusible = true; + } + + /// `@auto_config({...})`, Python's `@tcl.kernel(auto_config=...)`. + pub fn autoConfig(self: *Builder, config: anytype) void { + self.auto_config = tcl.configAttr(self.ctx, self.jsonFields(config)) catch + std.debug.panic("tcl: {s}: invalid auto config", .{self.name}); + } + + /// One field per argument: `.{ .x = .{ .dtype = .bf16, .axes = &.{ M, K } } }`. + /// With `.dram = Dram{...}` (or `.{}`) the argument is DRAM-mapped and the + /// field is its `as_logical` view. + pub fn declareArgs(self: *Builder, spec: anytype) FinishError!ArgsOf(@TypeOf(spec)) { + std.debug.assert(self.entry == null); + const fields = @typeInfo(@TypeOf(spec)).@"struct".field_names; + const scratch = self.arena.allocator(); + var types: [fields.len]*const mlir.Type = undefined; + var locs: [fields.len]*const mlir.Location = undefined; + const args = try scratch.alloc(Tensor, fields.len); + inline for (fields, 0..) |f, i| { + const s = @field(spec, f); + const axes = self.dupeAxes(s.axes); + const dram: ?Dram = if (@hasField(@TypeOf(s), "dram")) toDram(s.dram) else null; + types[i] = if (dram) |d| self.mappedType(s.dtype, axes, d) else self.logicalType(s.dtype, axes); + locs[i] = self.loc(); + args[i] = .{ .inner = undefined, .dtype = s.dtype, .axes = axes, .shards = if (dram) |d| d.shards() else 1 }; + } + const entry = mlir.Block.init(&types, &locs); + // In place from the start: the kernel verifier walks to its inputs' + // regions. `finish` sets the function type. + const func_op = func.func(self.ctx, .{ .name = self.name, .block = entry, .results = &.{}, .location = self.loc(), .visibility = null, .verify = false }); + _ = func_op.appendTo(self.module.body()); + self.func_op = func_op; + self.entry = entry; + var named: ArgsOf(@TypeOf(spec)) = undefined; + inline for (fields, 0..) |f, i| { + args[i].inner = entry.argument(i); + if (@hasField(@TypeOf(@field(spec, f)), "dram")) + args[i].inner = try self.graphOp(.as_logical, &.{entry.argument(i)}, self.logicalType(args[i].dtype, args[i].axes), &.{}, null); + @field(named, f) = args[i]; + } + self.args = args; + return named; + } + + fn toDram(d: anytype) Dram { + const D = @TypeOf(d); + if (D == Dram) return d; + return .{ + .chip = if (@hasField(D, "chip")) d.chip else &.{.broadcast}, + .inner = if (@hasField(D, "inner")) d.inner else null, + .original = if (@hasField(D, "original")) d.original else &.{}, + }; + } + + fn ArgsOf(comptime Spec: type) type { + const in = @typeInfo(Spec).@"struct".field_names; + comptime var names: [in.len][]const u8 = undefined; + inline for (in, 0..) |f, i| names[i] = f; + return @Struct(.auto, null, &names, &@splat(Tensor), &@splat(.{})); + } + + /// Opens a `tk.(...)` tensor operation; `commit` closes it. + pub fn tensorOperation(self: *Builder, opts: TensorOperation.Options) *TensorOperation { + _ = self.entryBlock(); + const op = self.arena.allocator().create(TensorOperation) catch @panic("OOM"); + op.* = .{ .b = self, .tactic = opts.tactic, .context = opts.context, .body = mlir.Block.init(&.{}, &.{}) }; + return op; + } + + pub const GraphOptions = struct { context: ?Context = null }; + + /// `reshape(t)` to `to`, same element count. + pub fn reshape(self: *Builder, t: Tensor, to: []const Axis, opts: GraphOptions) FinishError!Tensor { + if (elements(t.axes) != elements(to)) std.debug.panic("tcl: reshape changes the element count", .{}); + return self.graph(.reshape, &.{t}, t.dtype, to, &.{}, opts.context); + } + + /// `transmute(t)`: the same bits as `dtype` over `to`. + pub fn transmute(self: *Builder, t: Tensor, dtype: DType, to: []const Axis, opts: GraphOptions) FinishError!Tensor { + return self.graph(.transmute, &.{t}, dtype, to, &.{}, opts.context); + } + + /// `concat([inputs])` along the one axis of `to` the inputs lack. + pub fn concat(self: *Builder, inputs: []const Tensor, to: []const Axis, opts: GraphOptions) FinishError!Tensor { + return self.graph(.concat, inputs, inputs[0].dtype, to, &.{}, opts.context); + } + + /// `slice(t, axis, offset)`: the window of `to` starting at `offset` along `axis`. + pub fn slice(self: *Builder, t: Tensor, along: Axis, offset: i64, to: []const Axis, opts: GraphOptions) FinishError!Tensor { + return self.graph(.slice, &.{t}, t.dtype, to, &.{ + .named(self.ctx, "axis", self.axisAttr(along)), + .named(self.ctx, "offset", .int(self.ctx, .i64, offset)), + }, opts.context); + } + + pub const GatherOptions = struct { + batch_axis: ?Axis = null, + valid_length: ?Tensor = null, + context: ?Context = null, + }; + + /// `gather(table, indices, axis)`: rows of `table` along `axis`. + pub fn gather(self: *Builder, table: Tensor, indices: Tensor, along: Axis, to: []const Axis, opts: GatherOptions) FinishError!Tensor { + var options: [2]mlir.NamedAttribute = .{ .named(self.ctx, "axis", self.axisAttr(along)), undefined }; + if (opts.batch_axis) |a| options[1] = .named(self.ctx, "batch_axis", self.axisAttr(a)); + var inputs: [3]Tensor = .{ table, indices, opts.valid_length orelse undefined }; + const n_inputs = @as(usize, 2) + @intFromBool(opts.valid_length != null); + return self.graph(.gather, inputs[0..n_inputs], table.dtype, to, options[0 .. @as(usize, 1) + @intFromBool(opts.batch_axis != null)], opts.context); + } + + pub const ScatterOptions = struct { + /// The tensor scattered into; zeros otherwise. + init: ?Tensor = null, + batch_axis: ?Axis = null, + context: ?Context = null, + }; + + /// `scatter(updates, indices, axis, init=...)`: `updates` written at + /// `indices` along `axis` of the result. + pub fn scatter(self: *Builder, updates: Tensor, indices: Tensor, along: Axis, to: []const Axis, opts: ScatterOptions) FinishError!Tensor { + var options: [2]mlir.NamedAttribute = .{ .named(self.ctx, "axis", self.axisAttr(along)), undefined }; + if (opts.batch_axis) |a| options[1] = .named(self.ctx, "batch_axis", self.axisAttr(a)); + var inputs: [3]Tensor = .{ updates, indices, opts.init orelse undefined }; + const n_inputs = @as(usize, 2) + @intFromBool(opts.init != null); + return self.graph(.scatter, inputs[0..n_inputs], updates.dtype, to, options[0 .. @as(usize, 1) + @intFromBool(opts.batch_axis != null)], opts.context); + } + + pub const ArangeOptions = struct { + start: ?i64 = null, + step: ?i64 = null, + context: ?Context = null, + }; + + /// `arange(start=, end, step=)` over the one axis of `to`. + pub fn arange(self: *Builder, end: i64, dtype: DType, to: []const Axis, opts: ArangeOptions) FinishError!Tensor { + var options: [3]mlir.NamedAttribute = undefined; + var n: usize = 0; + if (opts.start) |v| { + options[n] = .named(self.ctx, "start", .int(self.ctx, .i64, v)); + n += 1; + } + options[n] = .named(self.ctx, "end", .int(self.ctx, .i64, end)); + n += 1; + if (opts.step) |v| { + options[n] = .named(self.ctx, "step", .int(self.ctx, .i64, v)); + n += 1; + } + return self.graph(.arange, &.{}, dtype, to, options[0..n], opts.context); + } + + pub const VectorOptions = struct { + /// Repeats `values` cyclically along this axis. + repeat_to: ?Axis = null, + context: ?Context = null, + }; + + /// `vector([values], repeat_to=)`: a constant tensor. + pub fn vector(self: *Builder, values: []const f64, dtype: DType, to: []const Axis, opts: VectorOptions) FinishError!Tensor { + const attrs = try self.arena.allocator().alloc(*const mlir.Attribute, values.len); + for (attrs, values) |*dst, v| dst.* = if (dtype.isFloat()) .float(self.ctx, .f64, v) else .int(self.ctx, .i64, @as(i64, @intFromFloat(v))); + var options: [2]mlir.NamedAttribute = .{ .named(self.ctx, "values", .array(self.ctx, attrs)), undefined }; + if (opts.repeat_to) |a| options[1] = .named(self.ctx, "repeat_to", self.axisAttr(a)); + return self.graph(.vector, &.{}, dtype, to, options[0 .. @as(usize, 1) + @intFromBool(opts.repeat_to != null)], opts.context); + } + + /// `all_gather(t, axis)` across chips. + pub fn allGather(self: *Builder, t: Tensor, along: Axis, opts: GraphOptions) FinishError!Tensor { + return self.graph(.all_gather, &.{t}, t.dtype, t.axes, &.{ + .named(self.ctx, "axis", self.axisAttr(along)), + }, opts.context); + } + + /// `as_dram(t)`: `t` in the DRAM mapping `dram`, to return it as such. + pub fn asDram(self: *Builder, t: Tensor, dram: Dram) FinishError!Tensor { + const out = try self.graphOp(.as_dram, &.{t.inner}, self.mappedType(t.dtype, t.axes, dram), &.{}, null); + return .{ .inner = out, .dtype = t.dtype, .axes = t.axes, .shards = dram.shards() }; + } + + /// Any `tcl.graph.*` operation; `options` are its TCL keyword fields. + pub fn graph(self: *Builder, comptime op: tcl.GraphOp, inputs: []const Tensor, dtype: DType, to: []const Axis, options: []const mlir.NamedAttribute, context: ?Context) FinishError!Tensor { + const values = try self.arena.allocator().alloc(*const mlir.Value, inputs.len); + for (values, inputs) |*v, t| v.* = t.inner; + const axes = self.dupeAxes(to); + const out = try self.graphOp(op, values, self.logicalType(dtype, axes), options, context); + return .{ .inner = out, .dtype = dtype, .axes = axes }; + } + + fn graphOp(self: *Builder, comptime op: tcl.GraphOp, inputs: []const *const mlir.Value, output: *const mlir.Type, options: []const mlir.NamedAttribute, context: ?Context) FinishError!*const mlir.Value { + const g = tcl.graph(self.ctx, op, inputs, output, .{ .options = options, .context = if (context) |c| self.contextAttr(c) else null }, self.loc()); + try self.append(g); + return g.result(0); + } + + /// `for iv in 0..limit (acc = init, ...):`, closed by `yield`. + pub fn ForScope(comptime N: usize) type { + return struct { + b: *Builder, + op: *mlir.Operation, + body: *mlir.Block, + /// The loop index, a scalar i32 tensor for `indexRead`, + /// `indexWrite` and `symExpr`. + iv: Tensor, + carried: [N]Tensor, + results: [N]Tensor = undefined, + + const Self = @This(); + + /// `yield values;` closes the loop and sets `results`. + pub fn yield(self: *Self, values: [N]Tensor) FinishError!void { + const b = self.b; + var inner: [N]*const mlir.Value = undefined; + for (&inner, values) |*v, t| v.* = t.inner; + _ = tcl.yield(b.ctx, &inner, b.loc()).appendTo(self.body); + b.popBlock(); + if (!self.op.verify()) { + std.log.err("tcl: {s}: invalid loop:\n{f}", .{ b.name, self.op.fmt(.{ .print_generic_op_form = true }) }); + return error.InvalidMlir; + } + for (&self.results, self.carried, 0..) |*r, c, i| r.* = .{ .inner = self.op.result(i), .dtype = c.dtype, .axes = c.axes, .shards = c.shards }; + } + }; + } + + /// Opens a device loop from 0 to `limit`: an `Axis`, an integer, or a + /// scalar i32 `Tensor` computed in the function (`reduceMaxI32`, + /// `symExpr`). The body must index a tensor with `iv`, which bounds the + /// trip count. `inits` is a tuple of the accumulators' initial tensors. + pub fn openFor(self: *Builder, limit: anytype, inits: anytype) ForScope(@typeInfo(@TypeOf(inits)).@"struct".field_names.len) { + const N = @typeInfo(@TypeOf(inits)).@"struct".field_names.len; + var types: [N + 1]*const mlir.Type = undefined; + var locs: [N + 1]*const mlir.Location = @splat(self.loc()); + var values: [N]*const mlir.Value = undefined; + var carried: [N]Tensor = undefined; + types[0] = self.logicalType(.i32, &.{}); + inline for (0..N) |i| { + carried[i] = inits[i]; + values[i] = inits[i].inner; + types[i + 1] = inits[i].inner.type_(); + } + const body = mlir.Block.init(&types, &locs); + const tensor_limit: ?*const mlir.Value, const bound: ?*const mlir.Attribute = switch (@TypeOf(limit)) { + Tensor => .{ limit.inner, null }, + Axis => .{ null, self.axisAttr(limit) }, + else => .{ null, .int(self.ctx, .i64, limit) }, + }; + // Attached before its body is built: the index operations' verifiers + // look for their loop. + const op = tcl.loop(self.ctx, &values, types[1..], body, .{ .limit = tensor_limit, .bound = bound }, self.loc()); + _ = op.appendTo(self.currentBlock()); + self.pushBlock(body); + for (&carried, 0..) |*c, i| c.inner = body.argument(i + 1); + return .{ .b = self, .op = op, .body = body, .iv = .{ .inner = body.argument(0), .dtype = .i32, .axes = &.{} }, .carried = carried }; + } + + /// `t[i]`: the slice of `t`'s outermost axis at loop index `i`. + pub fn indexRead(self: *Builder, t: Tensor, i: Tensor) FinishError!Tensor { + return self.graph(.index_read, &.{ t, i }, t.dtype, t.axes[1..], &.{}, null); + } + + /// `acc[i] = v`, in place: `acc` must have no other use; use the result. + pub fn indexWrite(self: *Builder, acc: Tensor, i: Tensor, v: Tensor) FinishError!Tensor { + return self.graph(.index_write, &.{ acc, i, v }, acc.dtype, acc.axes, &.{}, null); + } + + /// `scratchpad()`: an uninitialized tensor, typically a loop's output that + /// `indexWrite` fills. + pub fn scratchpad(self: *Builder, dtype: DType, to: []const Axis) FinishError!Tensor { + return self.graph(.scratchpad, &.{}, dtype, to, &.{}, null); + } + + /// `full(value)`: a constant tensor, like `tcl.full`. + pub fn full(self: *Builder, value: f64, dtype: DType, to: []const Axis) FinishError!Tensor { + const v: *const mlir.Attribute = if (dtype.isFloat()) .float(self.ctx, .f64, value) else .int(self.ctx, .i64, @as(i64, @intFromFloat(value))); + return self.graph(.full, &.{}, dtype, to, &.{.named(self.ctx, "value", v)}, null); + } + + /// `reduce_max_i32(t)`: the maximum of an i32 tensor, as a scalar. + pub fn reduceMaxI32(self: *Builder, t: Tensor) FinishError!Tensor { + return self.graph(.reduce_max_i32, &.{t}, .i32, &.{}, &.{}, null); + } + + /// `sym_expr(lhs rhs)`: scalar i32 arithmetic on the device, over + /// scalar tensors (a loop index, `reduceMaxI32`) and integers. + pub fn symExpr(self: *Builder, kind: Expr.Kind, lhs: anytype, rhs: anytype) FinishError!Tensor { + var inputs: [2]Tensor = undefined; + var n: usize = 0; + const l = self.symOperand(lhs, &inputs, &n); + const r = self.symOperand(rhs, &inputs, &n); + const e = tcl.exprAttr(self.ctx, @tagName(kind), &.{ l, r }) catch + std.debug.panic("tcl: {s}: invalid {s} expression", .{ self.name, @tagName(kind) }); + return self.graph(.sym_expr, inputs[0..n], .i32, &.{}, &.{.named(self.ctx, "expr", e)}, null); + } + + fn symOperand(self: *Builder, x: anytype, inputs: *[2]Tensor, n: *usize) *const mlir.Attribute { + if (@TypeOf(x) != Tensor) return .int(self.ctx, .i64, x); + inputs[n.*] = x; + n.* += 1; + return tcl.exprAttr(self.ctx, "arg", &.{.int(self.ctx, .i64, @as(i64, @intCast(n.* - 1)))}) catch unreachable; + } + + /// `copy(src=value, dst=argument)`: `value` written back into the + /// DRAM-mapped argument `index` in place. Return the result like any + /// other: the caller sees it as an output aliasing that argument. + pub fn update(self: *Builder, index: usize, value: Tensor) FinishError!Tensor { + const arg = self.entryBlock().argument(index); + const back = try self.graphOp(.as_dram, &.{value.inner}, arg.type_(), &.{}, null); + try self.append(tcl.copy(self.ctx, back, arg, self.loc())); + self.updated.append(self.arena.allocator(), index) catch @panic("OOM"); + return .{ .inner = arg, .dtype = value.dtype, .axes = value.axes, .shards = self.args[index].shards }; + } + + /// The argument each result updates in place, or null for results the + /// function returns. + pub fn updatedArgument(self: *const Builder, result: Tensor) ?usize { + for (self.updated.items) |i| if (self.entryBlock().argument(i) == result.inner) return i; + return null; + } + + /// `return results...`. + pub fn ret(self: *Builder, results: []const Tensor) void { + self.results = self.arena.allocator().dupe(Tensor, results) catch @panic("OOM"); + } + + /// The verified module: one `func.func` carrying its axes in `tcl.axes`. + pub fn finish(self: *Builder) FinishError![:0]const u8 { + const ctx = self.ctx; + const f = self.func_op orelse return error.InvalidMlir; + const results = self.results orelse return error.InvalidMlir; + const scratch = self.arena.allocator(); + var values: std.ArrayList(*const mlir.Value) = .empty; + var result_types: std.ArrayList(*const mlir.Type) = .empty; + const updated_inputs = try scratch.alloc(i64, results.len); + for (updated_inputs, results) |*u, r| { + if (self.updatedArgument(r)) |i| { + u.* = @intCast(i); + continue; + } + u.* = -1; + try values.append(scratch, r.inner); + try result_types.append(scratch, r.inner.type_()); + } + _ = func.returns(ctx, values.items, self.loc()).appendTo(self.entry.?); + + const entry = self.entry.?; + const arg_types = try scratch.alloc(*const mlir.Type, entry.numArguments()); + for (arg_types, 0..) |*t, i| t.* = entry.argument(i).type_(); + f.setAttributeByName("function_type", .typeAttr(.function(ctx, arg_types, result_types.items))); + if (self.updated.items.len > 0) f.setAttributeByName("tcl.updated_inputs", .denseArray(ctx, .i64, updated_inputs)); + + const axes = try scratch.alloc(mlir.NamedAttribute, self.axes.items.len); + for (axes, self.axes.items) |*dst, d| { + dst.* = .named(ctx, d.axis.name, if (d.def) |e| self.exprAttr(e) else .int(ctx, .i64, d.axis.size)); + } + f.setAttributeByName("tcl.axes", .dict(ctx, axes)); + if (self.compiler_config) |c| f.setAttributeByName("tcl.compiler_config", c); + if (self.auto_config) |c| f.setAttributeByName("tcl.auto_config", c); + if (self.fusible) f.setAttributeByName("tcl.fusible", .unit(ctx)); + if (!self.module.operation().verify()) return error.InvalidMlir; + + var printed: std.Io.Writer.Allocating = .init(self.allocator); + defer printed.deinit(); + try printed.writer.print("{f}", .{self.module.operation()}); + return try self.allocator.dupeSentinel(u8, printed.written(), 0); + } + + fn append(self: *Builder, op: *mlir.Operation) FinishError!void { + _ = op.appendTo(self.currentBlock()); + if (!op.verify()) { + std.log.err("tcl: {s}: invalid operation:\n{f}", .{ self.name, op.fmt(.{ .print_generic_op_form = true }) }); + return error.InvalidMlir; + } + } + + fn entryBlock(self: *const Builder) *mlir.Block { + return self.entry orelse std.debug.panic("tcl: {s}: declareArgs first", .{self.name}); + } + + pub fn pushBlock(self: *Builder, block: *mlir.Block) void { + self.block_stack.append(self.arena.allocator(), block) catch @panic("OOM"); + } + + pub fn popBlock(self: *Builder) void { + _ = self.block_stack.pop(); + } + + pub fn currentBlock(self: *const Builder) *mlir.Block { + return self.block_stack.getLastOrNull() orelse self.entryBlock(); + } + + pub fn axisAttr(self: *const Builder, a: Axis) *const mlir.Attribute { + return tcl.symbolAttr(self.ctx, a.name) catch unreachable; + } + + fn axisAttrs(self: *Builder, axes: []const Axis) []const *const mlir.Attribute { + const attrs = self.arena.allocator().alloc(*const mlir.Attribute, axes.len) catch @panic("OOM"); + for (attrs, axes) |*dst, a| dst.* = self.axisAttr(a); + return attrs; + } + + fn axesAttr(self: *Builder, axes: []const Axis) *const mlir.Attribute { + return .array(self.ctx, self.axisAttrs(axes)); + } + + pub fn exprAttr(self: *Builder, e: Expr) *const mlir.Attribute { + return switch (e) { + .int => |n| .int(self.ctx, .i64, n), + .axis => |a| self.axisAttr(a), + .broadcast => tcl.exprAttr(self.ctx, "broadcast", &.{}) catch unreachable, + .node => |n| tcl.exprAttr(self.ctx, @tagName(n.kind), &.{ self.exprAttr(n.lhs), self.exprAttr(n.rhs) }) catch + std.debug.panic("tcl: {s}: invalid {s} expression", .{ self.name, @tagName(n.kind) }), + }; + } + + fn exprAttrs(self: *Builder, exprs: []const Expr) []const *const mlir.Attribute { + const attrs = self.arena.allocator().alloc(*const mlir.Attribute, exprs.len) catch @panic("OOM"); + for (attrs, exprs) |*dst, e| dst.* = self.exprAttr(e); + return attrs; + } + + pub fn contextAttr(self: *Builder, c: Context) *const mlir.Attribute { + const ctx = self.ctx; + var fields: [2]mlir.NamedAttribute = undefined; + var n: usize = 0; + if (c.layout) |l| { + var layout: [3]mlir.NamedAttribute = undefined; + var m: usize = 0; + layout[m] = .named(ctx, "Chip", self.exprAttr(l.chip)); + m += 1; + if (l.cluster) |cl| { + layout[m] = .named(ctx, "Cluster", self.exprAttr(cl)); + m += 1; + } + if (l.split.len > 0) { + layout[m] = .named(ctx, "Split", self.axesAttr(l.split)); + m += 1; + } + fields[n] = .named(ctx, "operator", .dict(ctx, layout[0..m])); + n += 1; + } + if (c.heuristic_hint.len > 0) { + const hints = self.arena.allocator().alloc(mlir.NamedAttribute, c.heuristic_hint.len) catch @panic("OOM"); + for (hints, c.heuristic_hint) |*dst, h| dst.* = .named(ctx, h.name, switch (h.value) { + .int => |v| .int(ctx, .i64, v), + .float => |v| .float(ctx, .f64, v), + .boolean => |v| .boolean(ctx, v), + .axis => |a| self.axisAttr(a), + }); + fields[n] = .named(ctx, "heuristic_hint", .dict(ctx, hints)); + n += 1; + } + return tcl.contextAttr(ctx, fields[0..n]) catch std.debug.panic("tcl: {s}: invalid context", .{self.name}); + } + + fn jsonFields(self: *Builder, config: anytype) []const mlir.NamedAttribute { + const fields = @typeInfo(@TypeOf(config)).@"struct".field_names; + const out = self.arena.allocator().alloc(mlir.NamedAttribute, fields.len) catch @panic("OOM"); + inline for (fields, 0..) |f, i| out[i] = .named(self.ctx, f, self.json(@field(config, f))); + return out; + } + + fn json(self: *Builder, v: anytype) *const mlir.Attribute { + const T = @TypeOf(v); + return switch (@typeInfo(T)) { + .bool => .boolean(self.ctx, v), + .comptime_int, .int => .int(self.ctx, .i64, v), + .comptime_float, .float => .float(self.ctx, .f64, v), + .enum_literal, .@"enum" => .string(self.ctx, @tagName(v)), + .@"struct" => |info| if (info.is_tuple) blk: { + var items: [info.field_names.len]*const mlir.Attribute = undefined; + inline for (info.field_names, 0..) |f, i| items[i] = self.json(@field(v, f)); + break :blk .array(self.ctx, &items); + } else .dict(self.ctx, self.jsonFields(v)), + .pointer => if (std.meta.Elem(T) == u8) .string(self.ctx, v) else blk: { + const items = self.arena.allocator().alloc(*const mlir.Attribute, v.len) catch @panic("OOM"); + for (items, v) |*dst, x| dst.* = self.json(x); + break :blk .array(self.ctx, items); + }, + else => @compileError("tcl: config values are JSON: bool, numbers, strings, structs, tuples"), + }; + } + + fn mappedType(self: *Builder, dtype: DType, axes: []const Axis, dram: Dram) *const mlir.Type { + for (axes) |a| _ = self.axis(a.name, a.size); + const inner = if (dram.inner) |e| self.exprAttrs(e) else blk: { + var rest: std.ArrayList(Axis) = .empty; + for (axes) |a| if (!dram.isChipAxis(a)) rest.append(self.arena.allocator(), a) catch @panic("OOM"); + break :blk self.axisAttrs(rest.items); + }; + const mapping = tcl.dramMappingAttr(self.ctx, self.exprAttrs(dram.chip), inner, self.axisAttrs(dram.original)) catch + std.debug.panic("tcl: {s}: invalid DRAM mapping", .{self.name}); + return tcl.mappedType(self.ctx, dtype.toMlir(self.ctx), mapping) catch unreachable; + } + + pub fn logicalType(self: *Builder, dtype: DType, axes: []const Axis) *const mlir.Type { + for (axes) |a| _ = self.axis(a.name, a.size); + return tcl.logicalType(self.ctx, dtype.toMlir(self.ctx), self.axisAttrs(axes)) catch + std.debug.panic("tcl: {s}: invalid tensor type (repeated axis?)", .{self.name}); + } + + fn dupeAxes(self: *Builder, axes: []const Axis) []const Axis { + return self.arena.allocator().dupe(Axis, axes) catch @panic("OOM"); + } + + fn unionAxes(self: *Builder, a: []const Axis, b: []const Axis) []const Axis { + var out: std.ArrayList(Axis) = .empty; + const long, const short = if (a.len >= b.len) .{ a, b } else .{ b, a }; + out.appendSlice(self.arena.allocator(), long) catch @panic("OOM"); + for (short) |x| if (!containsAxis(out.items, x)) out.append(self.arena.allocator(), x) catch @panic("OOM"); + return out.items; + } +}; + +fn elements(axes: []const Axis) i64 { + var n: i64 = 1; + for (axes) |a| n *= a.size; + return n; +} + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + registerDialects(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectContains(haystack: []const u8, needle: []const u8) !void { + if (std.mem.indexOf(u8, haystack, needle) == null) { + std.debug.print("missing {s} in:\n{s}\n", .{ needle, haystack }); + return error.TestExpectedContains; + } +} + +test "softmax matches furiosa.tcl.examples.softmax" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "softmax_kernel"); + defer b.deinit(); + + const B = b.axis("B", 2); + const S = b.axis("S", 16); + const D = b.axis("D", 1024); + const t = try b.declareArgs(.{ .input = .{ .dtype = .bf16, .axes = &.{ B, S, D } } }); + + const op = b.tensorOperation(.{}); + const x = op.fetch(t.input, .{ .typecast_to = .f32 }); + const centered = x.subf(x.reduce(&.{D}, .maxf)); + const exped = centered.exp(); + const out = try op.commit(exped.divf(exped.reduce(&.{D}, .addf)), .{ .dtype = .bf16 }); + b.ret(&.{out}); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "tcl.axes = {B = 2 : i64, D = 1024 : i64, S = 16 : i64}"); + try expectContains(ir, "options = #tcl.read_options<{typecast_to = f32}>"); + try expectContains(ir, "mode = #tcl.reduce_mode<\"Maxf\">"); + try expectContains(ir, "opcode = #tcl.ve_opcode<\"divf\">"); + try expectContains(ir, "tactic = #tcl.tactic<\"ReduceByVe\">"); + try expectContains(ir, "-> !tcl.logical, #tcl.symbol<\"S\">, #tcl.symbol<\"D\">]>"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "batched matmul, scalar operands and graph reshape" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "bmm_scaled"); + defer b.deinit(); + + const B = b.axis("B", 2); + const M = b.axis("M", 128); + const K = b.axis("K", 64); + const N = b.axis("N", 256); + const t = try b.declareArgs(.{ + .lhs = .{ .dtype = .bf16, .axes = &.{ B, M, K } }, + .rhs = .{ .dtype = .bf16, .axes = &.{ B, K, N } }, + }); + + const mm = b.tensorOperation(.{}); + const acc = mm.contract(t.lhs, t.rhs, &.{ B, M, N }); + const prod = try mm.commit(acc.mulf(0.5), .{ .dtype = .bf16 }); + const flat = try b.reshape(prod, &.{ B, b.axis("MN", 128 * 256) }, .{}); + b.ret(&.{ prod, flat }); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "tactic = #tcl.tactic<\"EinsumByDpe\">"); + try expectContains(ir, "\"tcl.dpe\""); + try expectContains(ir, "arith.constant 5.000000e-01 : f32"); + try expectContains(ir, "\"tcl.graph.reshape\""); +} + +test "conv1d matches furiosa.kernels.convolution" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "conv"); + defer b.deinit(); + + const N = b.axis("N", 2); + const C = b.axis("C", 64); + const L = b.axis("L", 128); + const K = b.axis("K", 128); + const Lw = b.axis("Lw", 3); + const Lf = b.axis("Lf", 128); + const t = try b.declareArgs(.{ + .inp = .{ .dtype = .bf16, .axes = &.{ N, C, L } }, + .flt = .{ .dtype = .bf16, .axes = &.{ K, C, Lw } }, + .bias = .{ .dtype = .bf16, .axes = &.{K} }, + }); + + const op = b.tensorOperation(.{}); + const windows = op.fetch(t.inp, .{ + .pad = &.{.{ .axis = L, .left = 1, .right = 1 }}, + .slide = &.{.{ .axis = L, .frame_axis = Lf, .window_axis = Lw }}, + }); + try std.testing.expectEqualStrings("Lw", windows.axes[2].name); + const out = try op.commit(op.contract(windows, t.flt, &.{ N, K, Lf }).addf(t.bias), .{ .dtype = .bf16 }); + b.ret(&.{out}); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "slide = {L = {dilation = 1 : i64, frame_axis = #tcl.symbol<\"Lf\">, stride = 1 : i64, undilated_window = 3 : i64, window_axis = #tcl.symbol<\"Lw\">}}"); + try expectContains(ir, "tactic = #tcl.tactic<\"EinsumByDpe\">"); +} + +test "composite axes, DRAM, fixed point, context, config and graph ops" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "features"); + defer b.deinit(); + + const M = b.axis("M", 64); + const N = b.axis("N", 128); + const G = b.axis("G", 8); + const MN = b.axis("MN", b.expr(.mul, M, N)); + try std.testing.expectEqual(64 * 128, MN.size); + b.compilerConfig(.{ .lowering_mode = "Heuristic", .max_num_partitioning_axes = 1 }); + const t = try b.declareArgs(.{ + .x = .{ .dtype = .f32, .axes = &.{ M, N }, .dram = .{} }, + .idx = .{ .dtype = .i32, .axes = &.{G} }, + }); + + const op = b.tensorOperation(.{ .context = .{ + .layout = .{ .chip = .broadcast, .split = &.{M} }, + .heuristic_hint = &.{.{ .name = "max_num_partitioning_axes", .value = .{ .int = 2 } }}, + } }); + const sum = try op.commit(op.fetch(t.x, .{}).reduce(&.{N}, .addf).toFxp(15).toFp(15), .{}); + + const rows = try b.gather(t.x, t.idx, M, &.{ G, N }, .{}); + const back = try b.scatter(rows, t.idx, M, &.{ M, N }, .{ .init = t.x }); + const iota = try b.arange(N.size, .i32, &.{N}, .{}); + const pattern = try b.vector(&.{ 1, 2 }, .f32, &.{N}, .{ .repeat_to = N }); + const bits = try b.transmute(back, .i32, &.{ M, N }, .{}); + const flat = try b.asDram(try b.reshape(bits, &.{MN}, .{}), .{}); + b.ret(&.{ sum, iota, pattern, flat }); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "MN = #tcl.expr<\"mul\", [#tcl.symbol<\"M\">, #tcl.symbol<\"N\">]>"); + try expectContains(ir, "tcl.compiler_config = #tcl.config<{lowering_mode = \"Heuristic\", max_num_partitioning_axes = 1 : i64}>"); + try expectContains(ir, "%arg0: !tcl.mapped], [#tcl.symbol<\"M\">, #tcl.symbol<\"N\">], []>>"); + try expectContains(ir, "\"tcl.graph.as_logical\"(%arg0)"); + try expectContains(ir, "opcode = #tcl.ve_opcode<\"to_i15\">"); + try expectContains(ir, "operator = {Chip = #tcl.expr<\"broadcast\", []>, Split = [#tcl.symbol<\"M\">]}"); + inline for (.{ "gather", "scatter", "arange", "vector", "transmute", "reshape", "as_dram" }) |g| + try expectContains(ir, "\"tcl.graph." ++ g ++ "\""); +} + +test "scatter written back into its DRAM argument in place" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "write_rows"); + defer b.deinit(); + + const M = b.axis("M", 64); + const N = b.axis("N", 128); + const G = b.axis("G", 8); + const t = try b.declareArgs(.{ + .cache = .{ .dtype = .bf16, .axes = &.{ M, N }, .dram = .{} }, + .rows = .{ .dtype = .bf16, .axes = &.{ G, N } }, + .idx = .{ .dtype = .i32, .axes = &.{G} }, + }); + const written = try b.update(0, try b.scatter(t.rows, t.idx, M, &.{ M, N }, .{ .init = t.cache })); + try std.testing.expectEqual(@as(?usize, 0), b.updatedArgument(written)); + b.ret(&.{written}); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "\"tcl.graph.copy\""); + try expectContains(ir, "tcl.updated_inputs = array"); +} + +test "loop over a device-computed trip count with index reads and writes" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "prefix"); + defer b.deinit(); + + const T = b.axis("T", 8); + const D = b.axis("D", 64); + const B = b.axis("B", 4); + const t = try b.declareArgs(.{ + .x = .{ .dtype = .f32, .axes = &.{ T, D } }, + .s = .{ .dtype = .f32, .axes = &.{D} }, + .lens = .{ .dtype = .i32, .axes = &.{B} }, + }); + // Running sums of the rows up to the longest length. + const n = try b.symExpr(.add, try b.reduceMaxI32(t.lens), 0); + var loop = b.openFor(n, .{ t.s, try b.scratchpad(.f32, &.{ T, D }) }); + const op = b.tensorOperation(.{}); + const sum = try op.commit(op.fetch(try b.indexRead(t.x, loop.iv), .{}).addf(op.fetch(loop.carried[0], .{})), .{}); + try loop.yield(.{ sum, try b.indexWrite(loop.carried[1], loop.iv, sum) }); + b.ret(&loop.results); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "\"tcl.graph.for\""); + try expectContains(ir, "operandSegmentSizes = array"); + try expectContains(ir, "\"tcl.graph.index_write\""); + try expectContains(ir, "#tcl.expr<\"add\", [#tcl.expr<\"arg\", [0]>, 0]>"); +} + +test "multi-chip kernel over chip-split DRAM tensors" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "split_exp"); + defer b.deinit(); + + const C = b.axis("C", 2); + const A = b.axis("A", 1024); + const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ C, A }, .dram = Dram{ .chip = &.{.{ .axis = C }} } } }); + try std.testing.expectEqual(2, t.x.shards); + + const op = b.tensorOperation(.{ .context = .{ .layout = .{ .chip = .{ .axis = C } } } }); + const local = try op.commit(op.fetch(t.x, .{}).exp(), .{}); + const all = try b.allGather(local, C, .{}); + b.ret(&.{ try b.asDram(local, .{ .chip = &.{.{ .axis = C }} }), try b.asDram(all, .{}) }); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "%arg0: !tcl.mapped], [#tcl.symbol<\"A\">], []>>"); + try expectContains(ir, "operator = {Chip = #tcl.symbol<\"C\">}"); + try expectContains(ir, "\"tcl.graph.all_gather\""); +} diff --git a/dsl/tcl/dtype.zig b/dsl/tcl/dtype.zig new file mode 100644 index 00000000..9faf1e18 --- /dev/null +++ b/dsl/tcl/dtype.zig @@ -0,0 +1,57 @@ +const std = @import("std"); + +const mlir = @import("mlir"); + +/// TCL element types, named as in TCL (`[A]/f8_e4`). +pub const DType = enum { + bool, + i4, + i8, + i16, + i32, + i64, + u8, + u16, + u32, + u64, + f16, + bf16, + f32, + f64, + f4_e2, + f8_e4, + f8_e5, + + pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { + return switch (self) { + .bool => .int(ctx, .i1), + .i4 => .int(ctx, .i4), + .i8 => .int(ctx, .i8), + .i16 => .int(ctx, .i16), + .i32 => .int(ctx, .i32), + .i64 => .int(ctx, .i64), + .u8 => .int(ctx, .u8), + .u16 => .int(ctx, .u16), + .u32 => .int(ctx, .u32), + .u64 => .int(ctx, .u64), + .f16 => .float(ctx, .f16), + .bf16 => .float(ctx, .bf16), + .f32 => .float(ctx, .f32), + .f64 => .float(ctx, .f64), + .f4_e2 => .float(ctx, .f4e2m1fn), + .f8_e4 => .float(ctx, .f8e4m3fn), + .f8_e5 => .float(ctx, .f8e5m2), + }; + } + + pub fn isFloat(self: DType) bool { + return switch (self) { + .f16, .bf16, .f32, .f64, .f4_e2, .f8_e4, .f8_e5 => true, + else => false, + }; + } +}; + +test { + std.testing.refAllDecls(@This()); +} diff --git a/mlir/dialects/README.md b/mlir/dialects/README.md index d95f7a1c..4ecd976f 100644 --- a/mlir/dialects/README.md +++ b/mlir/dialects/README.md @@ -211,6 +211,12 @@ verified. - fly's attributes and cute_ir's algebra attributes have named constructors (`getStatic`, `getDynamic`, `getTuple`, ...); fly's generated builders use snake_case parameters. +- tcl mirrors the Furiosa plugin's file names and C++ style. One + `graph(op)` builder covers the 17 `tcl.graph.*` operations that share a + shape, and `loop`, `yield` and `copy` build `tcl.graph.for`, `yield` and + `copy`. Its `Tactic` and `ReduceMode` enums map to string attribute values, + its C API covers what ZML emits, and its operations are verified through + `dsl/tcl`'s tests. ## Porting checklist diff --git a/mlir/dialects/tcl/BUILD.bazel b/mlir/dialects/tcl/BUILD.bazel new file mode 100644 index 00000000..d57b6f36 --- /dev/null +++ b/mlir/dialects/tcl/BUILD.bazel @@ -0,0 +1,172 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + ":dialect", + ":headers", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "tcl", + import_name = "mlir/dialects/tcl", + main = "tcl.zig", + tags = ["manual"], + deps = [ + ":capi", + "//mlir", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":tcl", + "//mlir", + "//mlir/dialects/func", + ], +) + +# The native subset of the Furiosa XLA plugin's `tcl` dialect +# (xla/backends/furiosa/codegen/tcl/ir): what ZML emits and the plugin's custom +# call parses. Files mirror the plugin's one for one. +td_library( + name = "td_files", + srcs = [ + "tcl_attrs.td", + "tcl_dialect.td", + "tcl_ops.td", + "tcl_types.td", + ], + includes = ["/"], + deps = [ + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "dialect_inc_gen", + tbl_outs = [ + ( + ["-gen-dialect-decls"], + "tcl_dialect.h.inc", + ), + ( + ["-gen-dialect-defs"], + "tcl_dialect.cc.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "tcl_dialect.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "types_inc_gen", + tbl_outs = [ + ( + [ + "-gen-typedef-decls", + "-typedefs-dialect=tcl", + ], + "tcl_types.h.inc", + ), + ( + [ + "-gen-typedef-defs", + "-typedefs-dialect=tcl", + ], + "tcl_types.cc.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "tcl_types.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "attrs_inc_gen", + tbl_outs = [ + ( + [ + "-gen-attrdef-decls", + "-attrdefs-dialect=tcl", + ], + "tcl_attrs.h.inc", + ), + ( + [ + "-gen-attrdef-defs", + "-attrdefs-dialect=tcl", + ], + "tcl_attrs.cc.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "tcl_attrs.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "ops_inc_gen", + tbl_outs = [ + ( + ["-gen-op-decls"], + "tcl_ops.h.inc", + ), + ( + ["-gen-op-defs"], + "tcl_ops.cc.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "tcl_ops.td", + deps = [":td_files"], +) + +cc_library( + name = "headers", + hdrs = [ + "tcl_attrs.h", + "tcl_dialect.h", + "tcl_ops.h", + "tcl_types.h", + ], + deps = [ + ":attrs_inc_gen", + ":dialect_inc_gen", + ":ops_inc_gen", + ":types_inc_gen", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:BytecodeOpInterface", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:SideEffectInterfaces", + ], +) + +cc_library( + name = "dialect", + srcs = [ + "tcl_attrs.cc", + "tcl_dialect.cc", + "tcl_ops.cc", + "tcl_types.cc", + ], + textual_hdrs = ["ve_instructions.inc"], + deps = [ + ":headers", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + ], +) diff --git a/mlir/dialects/tcl/capi.cc b/mlir/dialects/tcl/capi.cc new file mode 100644 index 00000000..70ee09d6 --- /dev/null +++ b/mlir/dialects/tcl/capi.cc @@ -0,0 +1,118 @@ +//===- capi.cc - C API for the Furiosa TCL dialect ------------------------===// + +#include "capi.h" + +#include "mlir/dialects/tcl/tcl_ops.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" +#include "mlir/CAPI/Support.h" + +#include + +using namespace mlir; +using namespace xla::furiosa::tcl; + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Tcl, tcl, TclDialect) + +// `getChecked` with its diagnostic silenced: an invalid parameter is a null +// handle, which the caller reports. +template +static auto checked(MlirContext context, Args &&...args) { + MLIRContext *ctx = unwrap(context); + if (!ctx) + return T(); + ctx->getOrLoadDialect(); + ScopedDiagnosticHandler silence(ctx, [](Diagnostic &) { return success(); }); + return T::getChecked([ctx] { return emitError(UnknownLoc::get(ctx)); }, ctx, + std::forward(args)...); +} + +template static T as(MlirAttribute attr) { + return dyn_cast_if_present(unwrap(attr)); +} + +MlirAttribute mlirTclSymbolAttrGet(MlirContext context, MlirStringRef name) { + return wrap(checked(context, unwrap(name))); +} + +MlirAttribute mlirTclTacticAttrGet(MlirContext context, MlirStringRef value) { + return wrap(checked(context, unwrap(value))); +} + +MlirAttribute mlirTclVeOpcodeAttrGet(MlirContext context, MlirStringRef value) { + return wrap(checked(context, unwrap(value))); +} + +MlirAttribute mlirTclReduceModeAttrGet(MlirContext context, + MlirStringRef value) { + return wrap(checked(context, unwrap(value))); +} + +MlirAttribute mlirTclPredicateAttrGet(MlirContext context, + MlirStringRef value) { + return wrap(checked(context, unwrap(value))); +} + +MlirAttribute mlirTclContextAttrGet(MlirContext context, MlirAttribute fields) { + auto dict = as(fields); + return dict ? wrap(checked(context, dict)) + : MlirAttribute{nullptr}; +} + +MlirAttribute mlirTclReadOptionsAttrGet(MlirContext context, + MlirAttribute fields) { + auto dict = as(fields); + return dict ? wrap(checked(context, dict)) + : MlirAttribute{nullptr}; +} + +MlirAttribute mlirTclConfigAttrGet(MlirContext context, MlirAttribute fields) { + auto dict = as(fields); + return dict ? wrap(checked(context, dict)) + : MlirAttribute{nullptr}; +} + +MlirAttribute mlirTclExprAttrGet(MlirContext context, MlirStringRef kind, + MlirAttribute args) { + auto array = as(args); + return array ? wrap(checked(context, unwrap(kind), array)) + : MlirAttribute{nullptr}; +} + +MlirAttribute mlirTclDramMappingAttrGet(MlirContext context, MlirAttribute chip, + MlirAttribute inner, + MlirAttribute original) { + auto c = as(chip), i = as(inner), + o = as(original); + return c && i && o ? wrap(checked(context, c, i, o)) + : MlirAttribute{nullptr}; +} + +MlirType mlirTclLogicalTypeGet(MlirContext context, MlirType element, + MlirAttribute axes) { + auto array = as(axes); + if (!array || mlirTypeIsNull(element)) + return {nullptr}; + return wrap(checked(context, unwrap(element), array)); +} + +bool mlirTypeIsATclLogical(MlirType type) { + return isa_and_nonnull(unwrap(type)); +} + +MlirType mlirTclLogicalTypeGetElementType(MlirType type) { + return wrap(cast(unwrap(type)).getElementType()); +} + +MlirAttribute mlirTclLogicalTypeGetAxes(MlirType type) { + return wrap(cast(unwrap(type)).getAxes()); +} + +MlirType mlirTclMappedTypeGet(MlirContext context, MlirType element, + MlirAttribute mapping) { + auto m = as(mapping); + if (!m || mlirTypeIsNull(element)) + return {nullptr}; + return wrap(checked(context, unwrap(element), m)); +} diff --git a/mlir/dialects/tcl/capi.h b/mlir/dialects/tcl/capi.h new file mode 100644 index 00000000..31e72c3b --- /dev/null +++ b/mlir/dialects/tcl/capi.h @@ -0,0 +1,53 @@ +//===- capi.h - C API for the Furiosa TCL dialect -----------------*- C -*-===// + +#ifndef MLIR_DIALECTS_TCL_CAPI_H +#define MLIR_DIALECTS_TCL_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Tcl, tcl); + +// Constructors verify their parameters and return a null handle when invalid. +MLIR_CAPI_EXPORTED MlirAttribute mlirTclSymbolAttrGet(MlirContext context, + MlirStringRef name); +MLIR_CAPI_EXPORTED MlirAttribute mlirTclExprAttrGet(MlirContext context, + MlirStringRef kind, + MlirAttribute args); +MLIR_CAPI_EXPORTED MlirAttribute mlirTclTacticAttrGet(MlirContext context, + MlirStringRef value); +MLIR_CAPI_EXPORTED MlirAttribute mlirTclVeOpcodeAttrGet(MlirContext context, + MlirStringRef value); +MLIR_CAPI_EXPORTED MlirAttribute mlirTclReduceModeAttrGet(MlirContext context, + MlirStringRef value); +MLIR_CAPI_EXPORTED MlirAttribute mlirTclPredicateAttrGet(MlirContext context, + MlirStringRef value); +MLIR_CAPI_EXPORTED MlirAttribute mlirTclContextAttrGet(MlirContext context, + MlirAttribute fields); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTclReadOptionsAttrGet(MlirContext context, MlirAttribute fields); +MLIR_CAPI_EXPORTED MlirAttribute mlirTclConfigAttrGet(MlirContext context, + MlirAttribute fields); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTclDramMappingAttrGet(MlirContext context, MlirAttribute chip, + MlirAttribute inner, MlirAttribute original); + +MLIR_CAPI_EXPORTED MlirType mlirTclLogicalTypeGet(MlirContext context, + MlirType element, + MlirAttribute axes); +MLIR_CAPI_EXPORTED bool mlirTypeIsATclLogical(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirTclLogicalTypeGetElementType(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirTclLogicalTypeGetAxes(MlirType type); + +MLIR_CAPI_EXPORTED MlirType mlirTclMappedTypeGet(MlirContext context, + MlirType element, + MlirAttribute mapping); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_TCL_CAPI_H diff --git a/mlir/dialects/tcl/tcl.zig b/mlir/dialects/tcl/tcl.zig new file mode 100644 index 00000000..4c1f857a --- /dev/null +++ b/mlir/dialects/tcl/tcl.zig @@ -0,0 +1,342 @@ +const c = @import("c"); +const mlir = @import("mlir"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("tcl"); +} + +fn stringRef(s: []const u8) c.MlirStringRef { + return .{ .data = s.ptr, .length = s.len }; +} + +fn attribute(a: c.MlirAttribute) mlir.Error!*const mlir.Attribute { + return if (a.ptr) |p| @ptrCast(p) else error.InvalidMlir; +} + +fn type_(t: c.MlirType) mlir.Error!*const mlir.Type { + return if (t.ptr) |p| @ptrCast(p) else error.InvalidMlir; +} + +// ============================================================================= +// Attributes +// ============================================================================= + +pub const Tactic = enum { + einsum_by_dpe, + reduce_by_ve, + interleaving, + elementwise, + tensor_operation, + einsum_by_ve, + filter_compaction, + + pub fn name(self: Tactic) []const u8 { + return switch (self) { + .einsum_by_dpe => "EinsumByDpe", + .reduce_by_ve => "ReduceByVe", + .interleaving => "Interleaving", + .elementwise => "Elementwise", + .tensor_operation => "TensorOperation", + .einsum_by_ve => "EinsumByVe", + .filter_compaction => "FilterCompaction", + }; + } +}; + +pub const ReduceMode = enum { + addi, + addf, + maxi, + maxf, + mini, + minf, + /// A running i32 sum that keeps its axes. + cumsum, + + pub fn name(self: ReduceMode) []const u8 { + return switch (self) { + .addi => "Addi", + .addf => "Addf", + .maxi => "Maxi", + .maxf => "Maxf", + .mini => "Mini", + .minf => "Minf", + .cumsum => "Cumsum", + }; + } +}; + +pub const Predicate = enum { eq, ne, lt, le, gt, ge }; + +/// An axis or another TCL identifier: `#tcl.symbol<"A">`. +pub fn symbolAttr(ctx: *mlir.Context, name: []const u8) mlir.Error!*const mlir.Attribute { + return attribute(c.mlirTclSymbolAttrGet(ctx.ptr(), stringRef(name))); +} + +/// `#tcl.expr`: `add`, `mul`, `stride`, `broadcast`, `arg`, ... +pub fn exprAttr(ctx: *mlir.Context, kind: []const u8, args: []const *const mlir.Attribute) mlir.Error!*const mlir.Attribute { + return attribute(c.mlirTclExprAttrGet(ctx.ptr(), stringRef(kind), mlir.Attribute.array(ctx, args).ptr())); +} + +pub fn tacticAttr(ctx: *mlir.Context, value: Tactic) *const mlir.Attribute { + return attribute(c.mlirTclTacticAttrGet(ctx.ptr(), stringRef(value.name()))) catch unreachable; +} + +/// A VE opcode of the SDK vocabulary (`exp`, `addf`, `to_f16`, ...). +pub fn veOpcodeAttr(ctx: *mlir.Context, opcode: []const u8) mlir.Error!*const mlir.Attribute { + return attribute(c.mlirTclVeOpcodeAttrGet(ctx.ptr(), stringRef(opcode))); +} + +pub fn reduceModeAttr(ctx: *mlir.Context, mode: ReduceMode) *const mlir.Attribute { + return attribute(c.mlirTclReduceModeAttrGet(ctx.ptr(), stringRef(mode.name()))) catch unreachable; +} + +pub fn predicateAttr(ctx: *mlir.Context, value: Predicate) *const mlir.Attribute { + return attribute(c.mlirTclPredicateAttrGet(ctx.ptr(), stringRef(@tagName(value)))) catch unreachable; +} + +/// `@ context(operator = {Chip: ...}, heuristic_hint = {...})`. +pub fn contextAttr(ctx: *mlir.Context, fields: []const mlir.NamedAttribute) mlir.Error!*const mlir.Attribute { + return attribute(c.mlirTclContextAttrGet(ctx.ptr(), mlir.Attribute.dict(ctx, fields).ptr())); +} + +/// `read(x, subtraction=, table_lookup=, broadcast_to=, typecast_to=, pad=)`. +pub fn readOptionsAttr(ctx: *mlir.Context, fields: []const mlir.NamedAttribute) mlir.Error!*const mlir.Attribute { + return attribute(c.mlirTclReadOptionsAttrGet(ctx.ptr(), mlir.Attribute.dict(ctx, fields).ptr())); +} + +/// `@compiler_config({...})` / `@auto_config({...})`, JSON-compatible fields. +pub fn configAttr(ctx: *mlir.Context, fields: []const mlir.NamedAttribute) mlir.Error!*const mlir.Attribute { + return attribute(c.mlirTclConfigAttrGet(ctx.ptr(), mlir.Attribute.dict(ctx, fields).ptr())); +} + +/// `dram(chip | inner)`, with the original axes of a padded mapping. +pub fn dramMappingAttr(ctx: *mlir.Context, chip: []const *const mlir.Attribute, inner: []const *const mlir.Attribute, original: []const *const mlir.Attribute) mlir.Error!*const mlir.Attribute { + return attribute(c.mlirTclDramMappingAttrGet( + ctx.ptr(), + mlir.Attribute.array(ctx, chip).ptr(), + mlir.Attribute.array(ctx, inner).ptr(), + mlir.Attribute.array(ctx, original).ptr(), + )); +} + +// ============================================================================= +// Types +// ============================================================================= + +/// `[A, B]/bf16`: a logical tensor over named axes. +pub fn logicalType(ctx: *mlir.Context, element: *const mlir.Type, axes: []const *const mlir.Attribute) mlir.Error!*const mlir.Type { + return (try LogicalType.get(ctx, .{ .elementType = element, .axes = axes })).type_(); +} + +/// `dram(...)/bf16`: a tensor in an explicit DRAM mapping. +pub fn mappedType(ctx: *mlir.Context, element: *const mlir.Type, mapping: *const mlir.Attribute) mlir.Error!*const mlir.Type { + return type_(c.mlirTclMappedTypeGet(ctx.ptr(), element.ptr(), mapping.ptr())); +} + +pub const LogicalType = opaque { + const M = mlir.Methods(LogicalType, c.MlirType); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const isAFn = c.mlirTypeIsATclLogical; + + pub const InitArgs = struct { + elementType: *const mlir.Type, + axes: []const *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const LogicalType { + return @ptrCast(c.mlirTclLogicalTypeGet(ctx.ptr(), args.elementType.ptr(), mlir.Attribute.array(ctx, args.axes).ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn type_(self: *const LogicalType) *const mlir.Type { + return @ptrCast(self); + } + + pub fn elementType(self: *const LogicalType) *const mlir.Type { + return @ptrCast(c.mlirTclLogicalTypeGetElementType(self.ptr()).ptr); + } + + pub fn axes(self: *const LogicalType) *const mlir.ArrayAttribute { + return @ptrCast(c.mlirTclLogicalTypeGetAxes(self.ptr()).ptr); + } +}; + +// ============================================================================= +// Operations +// ============================================================================= +// +// Builders do not verify: kernel instructions only verify inside a +// `tcl.kernel`, and a kernel or graph operation only once its operands +// dominate it. Verify each graph-level operation after inserting it. + +pub const KernelOpts = struct { + context: ?*const mlir.Attribute = null, +}; + +pub fn kernel(ctx: *mlir.Context, inputs: []const *const mlir.Value, output: *const mlir.Type, tactic: Tactic, body: *mlir.Block, opts: KernelOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [2]mlir.NamedAttribute = undefined; + attrs[0] = .named(ctx, "tactic", tacticAttr(ctx, tactic)); + if (opts.context) |a| attrs[1] = .named(ctx, "context", a); + return mlir.Operation.make(ctx, "tcl.kernel", .{ + .operands = .{ .flat = inputs }, + .results = .{ .flat = &.{output} }, + .attributes = attrs[0 .. @as(usize, 1) + @intFromBool(opts.context != null)], + .blocks = &.{body}, + .verify = false, + .location = location, + }); +} + +pub const ReadOpts = struct { + options: ?*const mlir.Attribute = null, +}; + +pub fn read(ctx: *mlir.Context, inputs: []const *const mlir.Value, output: *const mlir.Type, opts: ReadOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + if (opts.options) |o| attrs[0] = .named(ctx, "options", o); + return mlir.Operation.make(ctx, "tcl.read", .{ + .operands = .{ .flat = inputs }, + .results = .{ .flat = &.{output} }, + .attributes = attrs[0..@intFromBool(opts.options != null)], + .verify = false, + .location = location, + }); +} + +pub fn dpe(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, output: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tcl.dpe", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{output} }, + .verify = false, + .location = location, + }); +} + +pub fn ve(ctx: *mlir.Context, inputs: []const *const mlir.Value, output: *const mlir.Type, opcode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tcl.ve", .{ + .operands = .{ .flat = inputs }, + .results = .{ .flat = &.{output} }, + .attributes = &.{.named(ctx, "opcode", opcode)}, + .verify = false, + .location = location, + }); +} + +pub fn ve_reduce(ctx: *mlir.Context, input: *const mlir.Value, output: *const mlir.Type, mode: ReduceMode, axes: []const *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tcl.ve_reduce", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{output} }, + .attributes = &.{ + .named(ctx, "mode", reduceModeAttr(ctx, mode)), + .named(ctx, "axes", .array(ctx, axes)), + }, + .verify = false, + .location = location, + }); +} + +pub fn ve_select(ctx: *mlir.Context, condition: *const mlir.Value, true_value: *const mlir.Value, false_value: *const mlir.Value, output: *const mlir.Type, predicate: Predicate, threshold: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tcl.ve_select", .{ + .operands = .{ .flat = &.{ condition, true_value, false_value } }, + .results = .{ .flat = &.{output} }, + .attributes = &.{ + .named(ctx, "predicate", predicateAttr(ctx, predicate)), + .named(ctx, "threshold", threshold), + }, + .verify = false, + .location = location, + }); +} + +/// `tcl.filter_compaction`: keeps the elements of `input` along `axes` whose +/// `condition` satisfies `predicate` against `threshold`, packed into at most +/// `max_size` (an integer or axis attribute) elements. +pub fn filter_compaction(ctx: *mlir.Context, input: *const mlir.Value, condition: *const mlir.Value, output: *const mlir.Type, predicate: Predicate, threshold: *const mlir.Attribute, axes: []const *const mlir.Attribute, max_size: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tcl.filter_compaction", .{ + .operands = .{ .flat = &.{ input, condition } }, + .results = .{ .flat = &.{output} }, + .attributes = &.{ + .named(ctx, "predicate", predicateAttr(ctx, predicate)), + .named(ctx, "threshold", threshold), + .named(ctx, "axes", .array(ctx, axes)), + .named(ctx, "max_size", max_size), + }, + .verify = false, + .location = location, + }); +} + +pub fn write(ctx: *mlir.Context, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tcl.write", .{ + .operands = .{ .flat = &.{input} }, + .verify = false, + .location = location, + }); +} + +pub const GraphOp = enum { gather, all_gather, scatter, reshape, transmute, concat, slice, arange, vector, as_logical, as_dram, index_read, index_write, scratchpad, full, reduce_max_i32, sym_expr }; + +pub const GraphOpts = struct { + options: []const mlir.NamedAttribute = &.{}, + context: ?*const mlir.Attribute = null, +}; + +/// `tcl.graph.`: a graph-level TCL operation whose fields go in `options`. +pub fn graph(ctx: *mlir.Context, comptime op: GraphOp, inputs: []const *const mlir.Value, output: *const mlir.Type, opts: GraphOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [2]mlir.NamedAttribute = undefined; + attrs[0] = .named(ctx, "options", .dict(ctx, opts.options)); + if (opts.context) |a| attrs[1] = .named(ctx, "context", a); + return mlir.Operation.make(ctx, "tcl.graph." ++ @tagName(op), .{ + .operands = .{ .flat = inputs }, + .results = .{ .flat = &.{output} }, + .attributes = attrs[0 .. @as(usize, 1) + @intFromBool(opts.context != null)], + .verify = false, + .location = location, + }); +} + +/// `tcl.graph.copy`, `copy(src=value, dst=input)`: writes `src` into the +/// kernel input `dst` in place. +pub fn copy(ctx: *mlir.Context, src: *const mlir.Value, dst: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tcl.graph.copy", .{ + .operands = .{ .flat = &.{ src, dst } }, + .results = .{ .flat = &.{} }, + .attributes = &.{.named(ctx, "options", .dict(ctx, &.{}))}, + .verify = false, + .location = location, + }); +} + +pub const LoopOpts = struct { + limit: ?*const mlir.Value = null, + bound: ?*const mlir.Attribute = null, +}; + +/// `tcl.graph.for`: runs `body` up to `limit` (a scalar i32 tensor) or `bound` +/// (an integer or axis attribute) times. The body's arguments are the index, +/// then one accumulator per `inits`; it ends with `tcl.graph.yield`. +pub fn loop(ctx: *mlir.Context, inits: []const *const mlir.Value, result_types: []const *const mlir.Type, body: *mlir.Block, opts: LoopOpts, location: *const mlir.Location) *mlir.Operation { + const limits: []const *const mlir.Value = if (opts.limit) |l| &.{l} else &.{}; + var attrs: [1]mlir.NamedAttribute = undefined; + if (opts.bound) |b| attrs[0] = .named(ctx, "bound", b); + return mlir.Operation.make(ctx, "tcl.graph.for", .{ + .operands = .{ .variadic = &.{ limits, inits } }, + .results = .{ .flat = result_types }, + .attributes = attrs[0..@intFromBool(opts.bound != null)], + .blocks = &.{body}, + .verify = false, + .location = location, + }); +} + +pub fn yield(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tcl.graph.yield", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} diff --git a/mlir/dialects/tcl/tcl_attrs.cc b/mlir/dialects/tcl/tcl_attrs.cc new file mode 100644 index 00000000..5b2fe575 --- /dev/null +++ b/mlir/dialects/tcl/tcl_attrs.cc @@ -0,0 +1,266 @@ +//===- tcl_attrs.cc - Furiosa TCL attributes ------------------------------===// + +#include "mlir/dialects/tcl/tcl_attrs.h" + +#include "mlir/dialects/tcl/tcl_dialect.h" +#include "mlir/dialects/tcl/tcl_types.h" + +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/StringExtras.h" + +#include +namespace xla::furiosa::tcl { +using namespace mlir; +using Error = llvm::function_ref; +static bool Identifier(llvm::StringRef s) { + if (s.empty() || (!llvm::isAlpha(s.front()) && s.front() != '_')) + return false; + return llvm::all_of(s, [](char c) { return llvm::isAlnum(c) || c == '_'; }); +} +static bool Symbol(llvm::StringRef s) { + auto [base, view] = s.split('.'); + return Identifier(base) && + (view.empty() ? !s.contains('.') : Identifier(view)); +} +static bool Mapping(Attribute a) { + if (!a) + return false; + if (isa(a)) + return IsNumber(a); + if (isa(a)) + return true; + auto e = dyn_cast(a); + return e && + llvm::is_contained( + {"stride", "modulo", "padding", "resize", "pair", "broadcast"}, + e.getKind()) && + llvm::all_of(e.getArgs(), Mapping); +} +static bool Json(Attribute a) { + if (auto n = dyn_cast(a)) + return n.getValue().isFinite(); + if (IsNumber(a) || isa(a)) + return true; + if (auto arr = dyn_cast(a)) + return llvm::all_of(arr, Json); + if (auto dict = dyn_cast(a)) + return llvm::all_of(dict, + [](NamedAttribute f) { return Json(f.getValue()); }); + return false; +} +static LogicalResult Enum(Error error, llvm::StringRef value, + llvm::ArrayRef values) { + return llvm::is_contained(values, value) + ? success() + : error() << "invalid enumerator: " << value; +} + +bool IsNumber(Attribute a) { + if (!a) + return false; + if (auto i = dyn_cast(a)) + return i.getValue().getBitWidth() <= 64; + return isa(a); +} +bool IsExpression(Attribute a) { + if (!a) + return false; + if (isa(a)) + return IsNumber(a); + if (isa(a)) + return true; + if (auto e = dyn_cast(a)) + return e.getKind() != "broadcast" && e.getKind() != "pair" && + e.getKind() != "padding" && e.getKind() != "resize"; + return false; +} +bool IsAxis(Attribute a) { + if (!a) + return false; + auto s = dyn_cast(a); + return s && llvm::isUpper(s.getName().front()); +} +bool IsAxes(Attribute a) { + if (!a) + return false; + auto array = dyn_cast(a); + return array && llvm::all_of(array, IsAxis); +} +LogicalResult VerifyKeys(Error error, DictionaryAttr d, + llvm::ArrayRef keys) { + for (auto f : d) + if (!llvm::is_contained(keys, f.getName().strref())) + return error() << "unknown field: " << f.getName(); + return success(); +} +LogicalResult VerifyAxes(Error error, ArrayAttr axes) { + if (!IsAxes(axes)) + return error() << "axes must be uppercase TCL symbols"; + llvm::SmallDenseSet seen; + for (auto a : axes) + if (!seen.insert(a).second) + return error() << "duplicate logical axis"; + return success(); +} + +// opcode, emitted spelling, arity. Conversion widths are handled below. +static constexpr VeInstructionSpec kVeInstructions[] = { +#define VE(name, text, arity) {name, text, arity}, +#include "mlir/dialects/tcl/ve_instructions.inc" +#undef VE +}; +llvm::ArrayRef VeInstructions() { return kVeInstructions; } + +LogicalResult SymbolAttr::verify(Error error, llvm::StringRef name) { + return Symbol(name) ? success() + : error() << "expected a TCL identifier, optionally with " + "one view suffix"; +} +LogicalResult ExprAttr::verify(Error error, llvm::StringRef kind, + ArrayAttr args) { + if (kind == "broadcast") + return args.empty() ? success() : error() << "broadcast takes no arguments"; + if (kind == "arg") { + if (args.size() != 1 || !isa(args[0]) || !IsNumber(args[0]) || + cast(args[0]).getInt() < 0) + return error() << "arg requires one nonnegative operand index"; + return success(); + } + bool map = llvm::is_contained( + {"stride", "modulo", "padding", "resize", "pair"}, kind); + bool arithmetic = + llvm::is_contained({"add", "sub", "mul", "div", "rem", "exact_div", "eq", + "ne", "lt", "le", "gt", "ge"}, + kind); + if ((!map && !arithmetic) || args.size() != 2) + return error() << "expected a known binary expression"; + for (auto a : args) + if (!(map ? Mapping(a) : IsExpression(a))) + return error() << "invalid expression operand"; + if (llvm::is_contained({"div", "exact_div", "rem", "stride", "modulo"}, kind)) + if (auto n = dyn_cast(args[1]); n && n.getInt() == 0) + return error() << "division by zero"; + return success(); +} +LogicalResult TacticAttr::verify(Error e, llvm::StringRef v) { + return Enum(e, v, + {"EinsumByDpe", "ReduceByVe", "Interleaving", "Elementwise", + "TensorOperation", "EinsumByVe", "FilterCompaction"}); +} +LogicalResult VeOpcodeAttr::verify(Error e, llvm::StringRef v) { + for (const auto &spec : VeInstructions()) + if (spec.name == v) + return success(); + if (v.consume_front("to_f") || v.consume_front("to_i")) { + unsigned width; + if (!v.getAsInteger(10, width) && width <= 31 && std::to_string(width) == v) + return success(); + } + return e() << "unknown VE opcode"; +} +LogicalResult ReduceModeAttr::verify(Error e, llvm::StringRef v) { + return Enum(e, v, {"Addi", "Addf", "Maxi", "Maxf", "Mini", "Minf", "Cumsum"}); +} + +LogicalResult PredicateAttr::verify(Error e, llvm::StringRef v) { + return Enum(e, v, {"eq", "ne", "lt", "le", "gt", "ge"}); +} +LogicalResult ConfigAttr::verify(Error e, DictionaryAttr d) { + return Json(d) ? success() + : e() << "compiler/auto configuration requires " + "JSON-compatible attributes"; +} + +LogicalResult ReadOptionsAttr::verify(Error e, DictionaryAttr d) { + if (failed(VerifyKeys(e, d, + {"subtraction", "table_lookup", "broadcast_to", + "typecast_to", "pad", "slide"}))) + return failure(); + for (auto field : d) { + auto k = field.getName().strref(); + auto v = field.getValue(); + if (k == "subtraction" || k == "table_lookup") { + auto i = dyn_cast(v); + if (!i || !IsNumber(i) || i.getInt() < 1) + return e() << k << " must index an auxiliary read operand (>=1)"; + } else if (k == "broadcast_to") { + if (!IsAxes(v)) + return e() << "broadcast_to requires axes"; + } else if (k == "typecast_to") { + auto t = dyn_cast(v); + if (!t || ElementName(t.getValue()).empty()) + return e() << "invalid typecast element type"; + } else if (k == "slide") { + auto entries = dyn_cast(v); + if (!entries) + return e() << "slide requires an axis dictionary"; + for (auto item : entries) { + auto spec = dyn_cast(item.getValue()); + if (!Symbol(item.getName()) || + !llvm::isUpper(item.getName().strref().front()) || !spec || + failed(VerifyKeys(e, spec, + {"undilated_window", "frame_axis", "window_axis", + "stride", "dilation"})) || + !IsExpression(spec.get("undilated_window")) || + !IsAxis(spec.get("frame_axis")) || !IsAxis(spec.get("window_axis"))) + return e() << "slide requires undilated_window, frame_axis and " + "window_axis"; + for (auto key : {"stride", "dilation"}) + if (auto a = spec.get(key); a && !IsExpression(a)) + return e() << "invalid slide " << key; + } + } else { + auto entries = dyn_cast(v); + if (!entries) + return e() << k << " requires an axis dictionary"; + for (auto item : entries) { + if (!Symbol(item.getName()) || + !llvm::isUpper(item.getName().strref().front())) + return e() << "invalid axis"; + auto p = dyn_cast(item.getValue()); + if (!p || p.size() != 3 || !IsExpression(p[0]) || !IsExpression(p[1]) || + !IsNumber(p[2])) + return e() << "padding requires [left, right, fill]"; + } + } + } + return success(); +} +LogicalResult ContextAttr::verify(Error e, DictionaryAttr d) { + if (failed(VerifyKeys(e, d, {"operator", "heuristic_hint"}))) + return failure(); + if (auto a = d.get("operator")) { + auto layout = dyn_cast(a); + if (!layout || failed(VerifyKeys(e, layout, {"Chip", "Cluster", "Split"}))) + return e() << "invalid operator layout"; + if (!layout.get("Chip")) + return e() << "operator layout requires Chip"; + for (auto f : layout) { + if (f.getName() == "Split") { + if (!IsAxes(f.getValue())) + return e() << "Split requires axes"; + } else if (!IsAxis(f.getValue()) && + !(isa(f.getValue()) && + cast(f.getValue()).getKind() == "broadcast")) + return e() << "Chip/Cluster requires axis or Broadcast"; + } + } + if (auto a = d.get("heuristic_hint")) { + auto hints = dyn_cast(a); + if (!hints) + return e() << "heuristic_hint requires a dictionary"; + for (auto f : hints) + if (!Identifier(f.getName()) || + !(IsNumber(f.getValue()) || IsAxis(f.getValue()))) + return e() << "invalid heuristic hint"; + } + return success(); +} + +LogicalResult DramMappingAttr::verify(Error e, ArrayAttr chip, ArrayAttr inner, + ArrayAttr original) { + if (!llvm::all_of(chip, Mapping) || !llvm::all_of(inner, Mapping)) + return e() << "invalid DRAM mapping"; + return VerifyAxes(e, original); +} +} // namespace xla::furiosa::tcl diff --git a/mlir/dialects/tcl/tcl_attrs.h b/mlir/dialects/tcl/tcl_attrs.h new file mode 100644 index 00000000..816583b9 --- /dev/null +++ b/mlir/dialects/tcl/tcl_attrs.h @@ -0,0 +1,37 @@ +//===- tcl_attrs.h - Furiosa TCL attributes ---------------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_TCL_TCL_ATTRS_H +#define MLIR_DIALECTS_TCL_TCL_ATTRS_H + +#include "mlir/dialects/tcl/tcl_dialect.h" + +#include "mlir/IR/Attributes.h" +#include "mlir/IR/BuiltinAttributes.h" +#include "mlir/IR/Diagnostics.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLFunctionalExtras.h" +#include "llvm/ADT/StringRef.h" +#define GET_ATTRDEF_CLASSES +#include "mlir/dialects/tcl/tcl_attrs.h.inc" +namespace xla::furiosa::tcl { +struct VeInstructionSpec { + llvm::StringRef name; + llvm::StringRef spelling; + unsigned arity; +}; +llvm::ArrayRef VeInstructions(); + +// Schema predicates shared by attribute, type and operation verifiers. +bool IsNumber(mlir::Attribute a); +bool IsExpression(mlir::Attribute a); +bool IsAxis(mlir::Attribute a); +bool IsAxes(mlir::Attribute a); +mlir::LogicalResult +VerifyKeys(llvm::function_ref error, + mlir::DictionaryAttr d, llvm::ArrayRef keys); +mlir::LogicalResult +VerifyAxes(llvm::function_ref error, + mlir::ArrayAttr axes); +} // namespace xla::furiosa::tcl + +#endif // MLIR_DIALECTS_TCL_TCL_ATTRS_H diff --git a/mlir/dialects/tcl/tcl_attrs.td b/mlir/dialects/tcl/tcl_attrs.td new file mode 100644 index 00000000..f85c518e --- /dev/null +++ b/mlir/dialects/tcl/tcl_attrs.td @@ -0,0 +1,74 @@ +//===- tcl_attrs.td - Furiosa TCL attributes ---------------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_TCL_TCL_ATTRS_TD +#define MLIR_DIALECTS_TCL_TCL_ATTRS_TD + +include "mlir/IR/AttrTypeBase.td" +include "mlir/dialects/tcl/tcl_dialect.td" + +def Tcl_Symbol : AttrDef { + let mnemonic = "symbol"; + let parameters = (ins StringRefParameter<>:$name); + let assemblyFormat = "`<` $name `>`"; + let genVerifyDecl = 1; +} +def Tcl_Expr : AttrDef { + let mnemonic = "expr"; + let summary = "Checked symbolic arithmetic or DRAM mapping expression tree"; + let parameters = (ins StringRefParameter<>:$kind, "::mlir::ArrayAttr":$args); + let assemblyFormat = "`<` $kind `,` $args `>`"; + let genVerifyDecl = 1; +} +def Tcl_Tactic : AttrDef { + let mnemonic = "tactic"; + let parameters = (ins StringRefParameter<>:$value); + let assemblyFormat = "`<` $value `>`"; + let genVerifyDecl = 1; +} +def Tcl_VeOpcode : AttrDef { + let mnemonic = "ve_opcode"; + let parameters = (ins StringRefParameter<>:$value); + let assemblyFormat = "`<` $value `>`"; + let genVerifyDecl = 1; +} +def Tcl_ReduceMode : AttrDef { + let mnemonic = "reduce_mode"; + let parameters = (ins StringRefParameter<>:$value); + let assemblyFormat = "`<` $value `>`"; + let genVerifyDecl = 1; +} + +def Tcl_Predicate : AttrDef { + let mnemonic = "predicate"; + let parameters = (ins StringRefParameter<>:$value); + let assemblyFormat = "`<` $value `>`"; + let genVerifyDecl = 1; +} +def Tcl_Context : AttrDef { + let mnemonic = "context"; + let parameters = (ins "::mlir::DictionaryAttr":$fields); + let assemblyFormat = "`<` $fields `>`"; + let genVerifyDecl = 1; +} +def Tcl_ReadOptions : AttrDef { + let mnemonic = "read_options"; + let parameters = (ins "::mlir::DictionaryAttr":$fields); + let assemblyFormat = "`<` $fields `>`"; + let genVerifyDecl = 1; +} + +def Tcl_Config : AttrDef { + let mnemonic = "config"; + let parameters = (ins "::mlir::DictionaryAttr":$fields); + let assemblyFormat = "`<` $fields `>`"; + let genVerifyDecl = 1; +} +def Tcl_DramMapping : AttrDef { + let mnemonic = "dram_mapping"; + let parameters = (ins "::mlir::ArrayAttr":$chip, "::mlir::ArrayAttr":$inner, + "::mlir::ArrayAttr":$original); + let assemblyFormat = "`<` $chip `,` $inner `,` $original `>`"; + let genVerifyDecl = 1; +} + +#endif // MLIR_DIALECTS_TCL_TCL_ATTRS_TD diff --git a/mlir/dialects/tcl/tcl_dialect.cc b/mlir/dialects/tcl/tcl_dialect.cc new file mode 100644 index 00000000..2cea51a1 --- /dev/null +++ b/mlir/dialects/tcl/tcl_dialect.cc @@ -0,0 +1,34 @@ +//===- tcl_dialect.cc - Furiosa TCL dialect -------------------------------===// + +#include "mlir/dialects/tcl/tcl_dialect.h" + +#include "mlir/dialects/tcl/tcl_attrs.h" // IWYU pragma: keep +#include "mlir/dialects/tcl/tcl_ops.h" // IWYU pragma: keep +#include "mlir/dialects/tcl/tcl_types.h" // IWYU pragma: keep + +#include "mlir/IR/Builders.h" // IWYU pragma: keep +#include "mlir/IR/DialectImplementation.h" // IWYU pragma: keep +#include "llvm/ADT/TypeSwitch.h" // IWYU pragma: keep + +// Storage definitions must be complete where the dialect registers them. +#include "mlir/dialects/tcl/tcl_dialect.cc.inc" +#define GET_ATTRDEF_CLASSES +#include "mlir/dialects/tcl/tcl_attrs.cc.inc" +#define GET_TYPEDEF_CLASSES +#include "mlir/dialects/tcl/tcl_types.cc.inc" +namespace xla::furiosa::tcl { +void TclDialect::initialize() { + addAttributes< +#define GET_ATTRDEF_LIST +#include "mlir/dialects/tcl/tcl_attrs.cc.inc" + >(); + addTypes< +#define GET_TYPEDEF_LIST +#include "mlir/dialects/tcl/tcl_types.cc.inc" + >(); + addOperations< +#define GET_OP_LIST +#include "mlir/dialects/tcl/tcl_ops.cc.inc" + >(); +} +} // namespace xla::furiosa::tcl diff --git a/mlir/dialects/tcl/tcl_dialect.h b/mlir/dialects/tcl/tcl_dialect.h new file mode 100644 index 00000000..97267065 --- /dev/null +++ b/mlir/dialects/tcl/tcl_dialect.h @@ -0,0 +1,9 @@ +//===- tcl_dialect.h - Furiosa TCL dialect ----------------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_TCL_TCL_DIALECT_H +#define MLIR_DIALECTS_TCL_TCL_DIALECT_H + +#include "mlir/IR/Dialect.h" // IWYU pragma: keep +#include "mlir/dialects/tcl/tcl_dialect.h.inc" + +#endif // MLIR_DIALECTS_TCL_TCL_DIALECT_H diff --git a/mlir/dialects/tcl/tcl_dialect.td b/mlir/dialects/tcl/tcl_dialect.td new file mode 100644 index 00000000..7b86a825 --- /dev/null +++ b/mlir/dialects/tcl/tcl_dialect.td @@ -0,0 +1,19 @@ +//===- tcl_dialect.td - Furiosa TCL dialect ----------------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_TCL_TCL_DIALECT_TD +#define MLIR_DIALECTS_TCL_TCL_DIALECT_TD + +include "mlir/IR/DialectBase.td" + +// Native TCL: tensors over named axes, `tk.*` tensor operations and graph +// operations, one-to-one with the TCL source the Furiosa compiler takes. The +// subset of xla/backends/furiosa/codegen/tcl/ir that ZML emits. +def Tcl_Dialect : Dialect { + let name = "tcl"; + let cppNamespace = "::xla::furiosa::tcl"; + let summary = "Typed Furiosa Tensor Compute Language graph"; + let useDefaultTypePrinterParser = 1; + let useDefaultAttributePrinterParser = 1; +} + +#endif // MLIR_DIALECTS_TCL_TCL_DIALECT_TD diff --git a/mlir/dialects/tcl/tcl_ops.cc b/mlir/dialects/tcl/tcl_ops.cc new file mode 100644 index 00000000..9d569f0b --- /dev/null +++ b/mlir/dialects/tcl/tcl_ops.cc @@ -0,0 +1,473 @@ +//===- tcl_ops.cc - Furiosa TCL operations --------------------------------===// + +#include "mlir/dialects/tcl/tcl_ops.h" + +#include "mlir/IR/Builders.h" // IWYU pragma: keep +#include "mlir/IR/Matchers.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#define GET_OP_CLASSES +#include "mlir/dialects/tcl/tcl_ops.cc.inc" +namespace xla::furiosa::tcl { +using namespace mlir; +static bool Tensor(Type t) { return isa(t); } +static bool ScalarI32(Type t) { + auto l = dyn_cast(t); + return l && l.getAxes().empty() && l.getElementType().isInteger(32); +} +static bool LoopIndex(Value v) { + auto arg = dyn_cast(v); + return arg && arg.getArgNumber() == 0 && + isa_and_nonnull(arg.getOwner()->getParentOp()); +} +static LogicalResult InKernel(Operation *op) { + if (!op->getParentOfType()) + return op->emitOpError("requires a tcl.kernel region"); + return success(); +} +// Exact graph operation schema. Optional operands occupy documented positions; +// the options dictionary never changes operand ordering. +static LogicalResult VerifyGraph(Operation *op, DictionaryAttr options) { + auto e = [&] { return op->emitOpError(); }; + llvm::StringRef n = op->getName().getStringRef().drop_front(10); + int min = 1, max = 1, results = 1; + llvm::SmallVector fields; + if (n == "all_gather") { + fields = {"axis"}; + } else if (n == "gather" || n == "scatter") { + min = 2; + max = 3; + fields = {"axis", "batch_axis", "table_dram_layout"}; + } else if (n == "concat") { + max = -1; + } else if (n == "slice") { + fields = {"axis", "offset"}; + } else if (n == "arange") { + min = max = 0; + fields = {"start", "end", "step"}; + } else if (n == "vector") { + min = max = 0; + fields = {"values", "repeat_to"}; + } else if (n == "index_read") { + min = max = 2; + } else if (n == "index_write") { + min = max = 3; + } else if (n == "copy") { + min = max = 2; + results = 0; + } else if (n == "scratchpad") { + min = max = 0; + } else if (n == "full") { + min = max = 0; + fields = {"value"}; + } else if (n == "sym_expr") { + min = 0; + max = -1; + fields = {"expr"}; + } + if (op->getNumOperands() < min || (max >= 0 && op->getNumOperands() > max) || + op->getNumResults() != results) + return e() << "incorrect operand or result count"; + if (failed(VerifyKeys(e, options, fields))) + return failure(); + if (n == "gather" && options.get("batch_axis") && op->getNumOperands() == 3) + return e() << "SDK sparse gather cannot also use batch_axis"; + for (Type t : op->getOperandTypes()) + if (!Tensor(t)) + return e() << "expected a native TCL tensor operand"; + for (Type t : op->getResultTypes()) + if (!Tensor(t)) + return e() << "expected a native TCL tensor result"; + for (auto f : options) { + auto k = f.getName().strref(); + auto v = f.getValue(); + bool ok = false; + if (k.ends_with("axis") || k == "repeat_to") + ok = IsAxis(v); + else if (k == "axes") + ok = IsAxes(v); + else if (k == "value") + ok = IsNumber(v); + else if (k == "values") { + auto a = dyn_cast(v); + ok = a && !a.empty() && llvm::all_of(a, IsNumber); + } else if (k == "table_dram_layout") + ok = isa(v) && isa(cast(v).getValue()); + else + ok = IsExpression(v); + if (!ok) + return e() << "invalid field " << k; + } + for (auto k : fields) { + bool required = llvm::is_contained( + {"axis", "offset", "end", "values", "expr", "value"}, k); + if (required && !options.get(k)) + return e() << "missing required field " << k; + } + if (n == "as_logical" && (!isa(op->getOperand(0).getType()) || + !isa(op->getResult(0).getType()))) + return e() << "as_logical requires storage -> logical"; + if (n == "as_dram" && (!isa(op->getOperand(0).getType()) || + !isa(op->getResult(0).getType()))) + return e() << "as_dram requires logical -> mapped"; + if (n == "index_read" || n == "index_write") { + if (!LoopIndex(op->getOperand(1))) + return e() << "requires the index of an enclosing loop"; + auto table = dyn_cast(op->getOperand(0).getType()); + auto slice = dyn_cast( + (n == "index_read" ? op->getResult(0) : op->getOperand(2)).getType()); + if (!table || !slice || table.getAxes().empty() || + table.getElementType() != slice.getElementType() || + table.getAxes().getValue().drop_front() != slice.getAxes().getValue()) + return e() << "slice must be the table without its outermost axis"; + if (n == "index_write" && + (op->getResult(0).getType() != table || !op->getOperand(0).hasOneUse())) + return e() << "writes in place: the table must have no other use"; + } + if (n == "copy" && + (!isa(op->getOperand(1)) || + op->getOperand(0).getType() != op->getOperand(1).getType())) + return e() << "copies a value into a kernel input of its type"; + if ((n == "reduce_max_i32" || n == "sym_expr") && + !ScalarI32(op->getResult(0).getType())) + return e() << "requires a scalar i32 result"; + if (n == "sym_expr") { + for (Value v : op->getOperands()) + if (!ScalarI32(v.getType())) + return e() << "requires scalar i32 operands"; + llvm::SmallVector pending{options.get("expr")}; + while (!pending.empty()) { + auto x = dyn_cast(pending.pop_back_val()); + if (!x) + continue; + if (x.getKind() == "arg") { + if (cast(x.getArgs()[0]).getInt() >= op->getNumOperands()) + return e() << "expression operand out of range"; + continue; + } + pending.append(x.getArgs().begin(), x.getArgs().end()); + } + } + return success(); +} + +LogicalResult KernelOp::verify() { + if (getBody().empty() || getBody().front().empty() || + getBody().front().getNumArguments()) + return emitOpError("requires one body without block arguments"); + auto write = dyn_cast(getBody().front().getTerminator()); + if (!write) + return emitOpError("requires tcl.write terminator"); + for (auto input : getInputs()) + if (!Tensor(input.getType())) + return emitOpError("requires native tensor inputs"); + for (Operation &op : getBody().front()) { + if (!isa(op) && + op.getName().getStringRef() != "arith.constant") + return emitOpError("unsupported kernel instruction ") << op.getName(); + } + WalkResult captured = getBody().walk([&](Operation *op) { + for (Value v : op->getOperands()) { + if (getBody().isAncestor(v.getParentRegion())) + continue; + if (!isa(op) || !llvm::is_contained(getInputs(), v)) { + op->emitOpError( + "external values must be explicit kernel inputs consumed by read"); + return WalkResult::interrupt(); + } + } + return WalkResult::advance(); + }); + if (captured.wasInterrupted()) + return failure(); + return success(); +} +LogicalResult ReadOp::verify() { + if (failed(InKernel(*this))) + return failure(); + if (getInputs().empty() || + !llvm::all_of(getInputs(), [](Value v) { return Tensor(v.getType()); })) + return emitOpError("requires tensor inputs"); + if (getOptions()) + for (auto k : {"subtraction", "table_lookup"}) + if (auto a = getOptions()->getFields().getAs(k); + a && a.getInt() >= getInputs().size()) + return emitOpError("auxiliary read operand index out of bounds"); + if (auto input = dyn_cast(getInputs()[0].getType())) { + SmallVector axes(input.getAxes().begin(), input.getAxes().end()); + Type element = input.getElementType(); + if (getOptions()) { + auto fields = getOptions()->getFields(); + if (auto table = fields.getAs("table_lookup")) { + auto t = dyn_cast(getInputs()[table.getInt()].getType()); + if (!t || t.getAxes().size() != 1) + return emitOpError("lookup table must be a one-axis logical tensor"); + element = t.getElementType(); + } + if (auto cast = fields.getAs("typecast_to")) + element = cast.getValue(); + if (auto pads = fields.getAs("pad")) + for (auto pad : pads) + if (!llvm::is_contained(axes, + SymbolAttr::get(getContext(), pad.getName()))) + return emitOpError("padding axis missing from input"); + // A slide replaces its axis with the window and frame axes. + if (auto slides = fields.getAs("slide")) + for (auto slide : slides) { + auto it = + llvm::find(axes, SymbolAttr::get(getContext(), slide.getName())); + if (it == axes.end()) + return emitOpError("slide axis missing from input"); + auto spec = cast(slide.getValue()); + *it = spec.get("window_axis"); + axes.insert(it + 1, spec.get("frame_axis")); + } + if (auto broadcast = fields.getAs("broadcast_to")) { + for (auto axis : axes) + if (!llvm::is_contained(broadcast, axis)) + return emitOpError("broadcast must retain all read axes"); + axes.assign(broadcast.begin(), broadcast.end()); + } + } + if (getOutput().getType().getAxes().getValue() != + ArrayRef(axes) || + getOutput().getType().getElementType() != element) + return emitOpError("read result disagrees with its modifiers"); + } + return success(); +} +LogicalResult DpeOp::verify() { + if (failed(InKernel(*this))) + return failure(); + auto out = getOutput().getType(); + if (!out.getElementType().isF32() && + !out.getElementType().isSignlessInteger(32)) + return emitOpError("DPE accumulators require f32 or i32"); + for (auto axis : out.getAxes()) + if (!llvm::is_contained(getLhs().getType().getAxes(), axis) && + !llvm::is_contained(getRhs().getType().getAxes(), axis)) + return emitOpError("DPE result axis missing from both inputs"); + return success(); +} +LogicalResult VeOp::verify() { + if (failed(InKernel(*this))) + return failure(); + unsigned arity = 1; + for (auto spec : VeInstructions()) + if (spec.name == getOpcode().getValue()) + arity = spec.arity; + if (getInputs().size() != arity) + return emitOpError("incorrect VE operand count"); + llvm::SmallDenseSet axes; + for (Value v : getInputs()) { + if (auto type = dyn_cast(v.getType())) { + for (auto a : type.getAxes()) + axes.insert(a); + } else if (!isa(v.getType()) || + !matchPattern(v, m_Constant())) + return emitOpError("VE scalar operands must be constants"); + } + auto out = getOutput().getType(); + if (axes.size() != out.getAxes().size() || + !llvm::all_of(out.getAxes(), + [&](Attribute a) { return axes.contains(a); })) + return emitOpError("VE output axes must be the union of operand axes"); + auto code = getOpcode().getValue(); + llvm::StringRef spelling = code; + for (auto spec : VeInstructions()) + if (spec.name == code) + spelling = spec.spelling; + bool fp = spelling.starts_with("to_f") || spelling == "as_f" || + spelling.ends_with("_f") || + llvm::is_contained({"+", "-", "*", "/", "exp", "neg_exp", "sqrt", + "tanh", "sigmoid", "erf", "log", "sin", "cos"}, + spelling); + if (fp ? !out.getElementType().isF32() + : !out.getElementType().isSignlessInteger(32)) + return emitOpError("VE registers produce f32 or i32 according to opcode"); + return success(); +} +LogicalResult VeReduceOp::verify() { + if (failed(InKernel(*this)) || + failed(VerifyAxes([&] { return emitOpError(); }, getAxes()))) + return failure(); + // An integer cumulative sum runs along its axes and keeps them. + if (getMode().getValue() == "Cumsum") { + if (getAxes().empty() || + getInput().getType().getAxes() != getOutput().getType().getAxes() || + !llvm::all_of(getAxes(), + [&](Attribute a) { + return llvm::is_contained( + getInput().getType().getAxes(), a); + }) || + !getInput().getType().getElementType().isInteger(32)) + return emitOpError("cumsum keeps the axes of an i32 value"); + return success(); + } + llvm::SmallVector remaining; + for (Attribute a : getInput().getType().getAxes()) + if (!llvm::is_contained(getAxes(), a)) + remaining.push_back(a); + if (getAxes().empty() || + remaining.size() + getAxes().size() != + getInput().getType().getAxes().size() || + remaining != getOutput().getType().getAxes().getValue()) + return emitOpError("reduction result must remove exactly the reduced axes"); + return success(); +} + +LogicalResult VeSelectOp::verify() { + if (failed(InKernel(*this)) || !IsNumber(getThreshold())) + return emitOpError("requires kernel and numeric threshold"); + llvm::SmallDenseSet axes; + for (Value v : + SmallVector{getCondition(), getTrueValue(), getFalseValue()}) { + if (auto t = dyn_cast(v.getType())) { + for (auto a : t.getAxes()) + axes.insert(a); + } else if (!matchPattern(v, m_Constant())) + return emitOpError("scalar branches require constants"); + } + auto output = getOutput().getType().getAxes(); + if (axes.size() != output.size() || + !llvm::all_of(output, [&](Attribute a) { return axes.contains(a); })) + return emitOpError("select output axes must cover its operands"); + return success(); +} + +LogicalResult FilterCompactionOp::verify() { + if (failed(InKernel(*this)) || + failed(VerifyAxes([&] { return emitOpError(); }, getAxes()))) + return failure(); + auto size = dyn_cast(getMaxSize()); + if (!IsNumber(getThreshold()) || + (!IsAxis(getMaxSize()) && !(size && size.getInt() > 0)) || + getAxes().empty()) + return emitOpError( + "requires numeric threshold, positive maximum size, and axes"); + if (getInput().getType().getAxes() != getCondition().getType().getAxes() || + getInput().getType().getElementType() != + getOutput().getType().getElementType() || + !llvm::all_of(getAxes(), [&](Attribute axis) { + return llvm::is_contained(getInput().getType().getAxes(), axis); + })) + return emitOpError("incompatible compaction input, condition, or axes"); + if (!llvm::all_of(getOutput().getUsers(), + [](Operation *op) { return isa(op); })) + return emitOpError( + "filter_compaction must be the final kernel instruction"); + return success(); +} + +LogicalResult WriteOp::verify() { + auto kernel = dyn_cast((*this)->getParentOp()); + if (!kernel) + return emitOpError("requires a kernel parent"); + auto in = getInput().getType(), out = kernel.getOutput().getType(); + if (in.getAxes().size() != out.getAxes().size()) + return emitOpError("write must preserve the axis set"); + for (auto a : in.getAxes()) + if (!llvm::is_contained(out.getAxes(), a)) + return emitOpError("write must preserve the axis set"); + // TCL write.to implicitly consumes the preceding instruction. + auto previous = (*this)->getPrevNode(); + while (previous && previous->getName().getStringRef() == "arith.constant") + previous = previous->getPrevNode(); + if (getInput().getDefiningOp() != previous) + return emitOpError( + "write must consume the immediately preceding instruction"); + return success(); +} + +LogicalResult GraphGatherOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphAllGatherOp::verify() { + if (failed(VerifyGraph(*this, getOptions()))) + return failure(); + auto input = dyn_cast(getInputs()[0].getType()); + auto output = dyn_cast(getOutputs()[0].getType()); + if (!input || !output || input != output) + return emitOpError("all_gather requires matching logical tensor types"); + if (!llvm::is_contained(input.getAxes(), getOptions().get("axis"))) + return emitOpError("collective axis must be present in the input"); + return success(); +} +LogicalResult GraphScatterOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphReshapeOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphTransmuteOp::verify() { + return VerifyGraph(*this, getOptions()); +} + +LogicalResult GraphConcatOp::verify() { + return VerifyGraph(*this, getOptions()); +} + +LogicalResult GraphSliceOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphArangeOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphVectorOp::verify() { + return VerifyGraph(*this, getOptions()); +} + +LogicalResult GraphAsLogicalOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphAsDramOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphCopyOp::verify() { return VerifyGraph(*this, getOptions()); } +LogicalResult GraphIndexReadOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphIndexWriteOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphScratchpadOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphFullOp::verify() { return VerifyGraph(*this, getOptions()); } +LogicalResult GraphReduceMaxI32Op::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphSymExprOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphForOp::verify() { + if (!getLimit() == !getBound()) + return emitOpError("requires exactly one of a limit tensor or a bound"); + if (getLimit() && !ScalarI32(getLimit().getType())) + return emitOpError("limit must be a scalar i32 tensor"); + if (auto bound = getBoundAttr()) { + auto n = dyn_cast(bound); + if (!IsAxis(bound) && !(n && IsNumber(n) && n.getInt() > 0)) + return emitOpError("bound must be a positive integer or an axis"); + } + Block &body = getBody().front(); + if (body.getNumArguments() != getInits().size() + 1 || + !ScalarI32(body.getArgument(0).getType())) + return emitOpError("requires a scalar i32 index and one accumulator each"); + auto yield = dyn_cast(body.getTerminator()); + if (!yield) + return emitOpError("requires a tcl.graph.yield terminator"); + if (getOutputs().size() != getInits().size() || + yield.getValues().size() != getInits().size()) + return emitOpError("requires one result and one yield per accumulator"); + for (auto [i, init] : llvm::enumerate(getInits())) { + Type t = init.getType(); + if (!Tensor(t) || body.getArgument(i + 1).getType() != t || + getOutputs()[i].getType() != t || yield.getValues()[i].getType() != t) + return emitOpError("accumulator ") << i << " changes type"; + } + return success(); +} + +} // namespace xla::furiosa::tcl diff --git a/mlir/dialects/tcl/tcl_ops.h b/mlir/dialects/tcl/tcl_ops.h new file mode 100644 index 00000000..bce6ccae --- /dev/null +++ b/mlir/dialects/tcl/tcl_ops.h @@ -0,0 +1,19 @@ +//===- tcl_ops.h - Furiosa TCL operations -----------------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_TCL_TCL_OPS_H +#define MLIR_DIALECTS_TCL_TCL_OPS_H + +#include "mlir/dialects/tcl/tcl_attrs.h" // IWYU pragma: export +#include "mlir/dialects/tcl/tcl_dialect.h" // IWYU pragma: export +#include "mlir/dialects/tcl/tcl_types.h" // IWYU pragma: export + +#include "mlir/Bytecode/BytecodeOpInterface.h" // IWYU pragma: keep +#include "mlir/IR/BuiltinAttributes.h" // IWYU pragma: keep +#include "mlir/IR/BuiltinTypes.h" // IWYU pragma: keep +#include "mlir/IR/OpDefinition.h" // IWYU pragma: keep +#include "mlir/IR/OpImplementation.h" // IWYU pragma: keep +#include "mlir/Interfaces/SideEffectInterfaces.h" // IWYU pragma: keep +#define GET_OP_CLASSES +#include "mlir/dialects/tcl/tcl_ops.h.inc" + +#endif // MLIR_DIALECTS_TCL_TCL_OPS_H diff --git a/mlir/dialects/tcl/tcl_ops.td b/mlir/dialects/tcl/tcl_ops.td new file mode 100644 index 00000000..23676ec9 --- /dev/null +++ b/mlir/dialects/tcl/tcl_ops.td @@ -0,0 +1,122 @@ +//===- tcl_ops.td - Furiosa TCL operations -----------------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_TCL_TCL_OPS_TD +#define MLIR_DIALECTS_TCL_TCL_OPS_TD + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "mlir/dialects/tcl/tcl_attrs.td" +include "mlir/dialects/tcl/tcl_dialect.td" +include "mlir/dialects/tcl/tcl_types.td" + +class Tcl_Op traits = []> + : Op; + +// Options contain typed attributes, never snippets of TCL source. Their exact +// per-operation schema is verified in tcl_ops.cc. Generic assembly is +// intentional: all native constructs support standard MLIR parsing, printing +// and bytecode. +class Tcl_GraphOp traits = [MemoryEffects<[MemRead]>]> + : Tcl_Op { + let arguments = (ins Variadic:$inputs, + DefaultValuedAttr:$options, + OptionalAttr:$context); + let results = (outs Variadic:$outputs); + let hasVerifier = 1; +} +def Tcl_GraphGatherOp : Tcl_GraphOp<"gather">; +def Tcl_GraphAllGatherOp : Tcl_GraphOp<"all_gather">; +def Tcl_GraphScatterOp : Tcl_GraphOp<"scatter">; +def Tcl_GraphReshapeOp : Tcl_GraphOp<"reshape">; +def Tcl_GraphTransmuteOp : Tcl_GraphOp<"transmute">; + +def Tcl_GraphConcatOp : Tcl_GraphOp<"concat">; + +def Tcl_GraphSliceOp : Tcl_GraphOp<"slice">; +def Tcl_GraphArangeOp : Tcl_GraphOp<"arange">; +def Tcl_GraphVectorOp : Tcl_GraphOp<"vector">; + +def Tcl_GraphAsLogicalOp : Tcl_GraphOp<"as_logical">; +def Tcl_GraphAsDramOp : Tcl_GraphOp<"as_dram">; +// `copy(src=value, dst=input)` writes a value into a kernel input in place. +def Tcl_GraphCopyOp + : Tcl_GraphOp<"copy", [MemoryEffects<[MemRead, MemWrite]>]>; + +// Loop-level graph constructs. `x[i]` reads and `acc[i] = v` writes the slice +// of the outermost axis at the loop index; `sym_expr` evaluates an integer +// expression (`#tcl.expr` names its operands) on the device. +def Tcl_GraphIndexReadOp : Tcl_GraphOp<"index_read">; +def Tcl_GraphIndexWriteOp : Tcl_GraphOp<"index_write">; +def Tcl_GraphScratchpadOp : Tcl_GraphOp<"scratchpad">; +def Tcl_GraphFullOp : Tcl_GraphOp<"full">; +def Tcl_GraphReduceMaxI32Op : Tcl_GraphOp<"reduce_max_i32">; +def Tcl_GraphSymExprOp : Tcl_GraphOp<"sym_expr">; + +// `outputs = for i in 0..limit (acc = init, ...): body; yield ...;`. The limit +// is a static bound (integer or axis) or a scalar i32 tensor computed in the +// function. The block arguments are the loop index, then the accumulators. +def Tcl_GraphForOp : Tcl_Op<"graph.for", + [AttrSizedOperandSegments, RecursiveMemoryEffects, SingleBlock]> { + let arguments = (ins Optional:$limit, Variadic:$inits, + OptionalAttr:$bound); + let results = (outs Variadic:$outputs); + let regions = (region SizedRegion<1>:$body); + let hasVerifier = 1; +} +def Tcl_GraphYieldOp : Tcl_Op<"graph.yield", + [Pure, Terminator, HasParent<"GraphForOp">]> { + let arguments = (ins Variadic:$values); +} + +def Tcl_KernelOp : Tcl_Op<"kernel", [MemoryEffects<[MemRead]>, SingleBlock]> { + let arguments = (ins Variadic:$inputs, Tcl_Tactic:$tactic, + OptionalAttr:$context); + let results = (outs Tcl_Logical:$output); + let regions = (region AnyRegion:$body); + let hasVerifier = 1; +} +def Tcl_ReadOp : Tcl_Op<"read", [Pure]> { + let arguments = (ins Variadic:$inputs, + OptionalAttr:$options); + let results = (outs Tcl_Logical:$output); + let hasVerifier = 1; +} +def Tcl_DpeOp : Tcl_Op<"dpe", [Pure]> { + let arguments = (ins Tcl_Logical:$lhs, Tcl_Logical:$rhs); + let results = (outs Tcl_Logical:$output); + let hasVerifier = 1; +} +def Tcl_VeOp : Tcl_Op<"ve", [Pure]> { + let arguments = (ins Variadic:$inputs, Tcl_VeOpcode:$opcode); + let results = (outs Tcl_Logical:$output); + let hasVerifier = 1; +} +def Tcl_VeReduceOp : Tcl_Op<"ve_reduce", [Pure]> { + let arguments = (ins Tcl_Logical:$input, Tcl_ReduceMode:$mode, + ArrayAttr:$axes); + let results = (outs Tcl_Logical:$output); + let hasVerifier = 1; +} + +def Tcl_VeSelectOp : Tcl_Op<"ve_select", [Pure]> { + let arguments = (ins Tcl_Logical:$condition, Tcl_Predicate:$predicate, + AnyAttr:$threshold, AnyType:$true_value, + AnyType:$false_value); + let results = (outs Tcl_Logical:$output); + let hasVerifier = 1; +} + +def Tcl_FilterCompactionOp : Tcl_Op<"filter_compaction", [Pure]> { + let arguments = (ins Tcl_Logical:$input, Tcl_Logical:$condition, + Tcl_Predicate:$predicate, AnyAttr:$threshold, + ArrayAttr:$axes, AnyAttr:$max_size); + let results = (outs Tcl_Logical:$output); + let hasVerifier = 1; +} + +def Tcl_WriteOp : Tcl_Op<"write", [Pure, Terminator]> { + let arguments = (ins Tcl_Logical:$input); + let hasVerifier = 1; +} + +#endif // MLIR_DIALECTS_TCL_TCL_OPS_TD diff --git a/mlir/dialects/tcl/tcl_types.cc b/mlir/dialects/tcl/tcl_types.cc new file mode 100644 index 00000000..8b7f919a --- /dev/null +++ b/mlir/dialects/tcl/tcl_types.cc @@ -0,0 +1,68 @@ +//===- tcl_types.cc - Furiosa TCL types -----------------------------------===// + +#include "mlir/dialects/tcl/tcl_types.h" + +#include "mlir/dialects/tcl/tcl_attrs.h" +#include "mlir/dialects/tcl/tcl_dialect.h" + +#include "llvm/ADT/STLExtras.h" +namespace xla::furiosa::tcl { +using namespace mlir; +using Error = llvm::function_ref; +llvm::StringRef ElementName(Type t) { + if (t.isBF16()) + return "bf16"; + if (t.isF16()) + return "f16"; + if (t.isF32()) + return "f32"; + if (t.isF64()) + return "f64"; + if (isa(t)) + return "f4_e2"; + if (isa(t)) + return "f8_e4"; + if (isa(t)) + return "f8_e5"; + if (auto i = dyn_cast(t)) { + if (i.getWidth() == 1 && i.isSignless()) + return "bool"; + if (i.isUnsigned()) { + switch (i.getWidth()) { + case 8: + return "u8"; + case 16: + return "u16"; + case 32: + return "u32"; + case 64: + return "u64"; + } + } else { + switch (i.getWidth()) { + case 4: + return "i4"; + case 8: + return "i8"; + case 16: + return "i16"; + case 32: + return "i32"; + case 64: + return "i64"; + } + } + } + return {}; +} +LogicalResult LogicalType::verify(Error e, Type element, ArrayAttr axes) { + if (ElementName(element).empty()) + return e() << "unsupported TCL element type"; + return VerifyAxes(e, axes); +} +LogicalResult MappedType::verify(Error e, Type element, + DramMappingAttr mapping) { + return ElementName(element).empty() ? e() << "unsupported TCL element type" + : success(); +} +} // namespace xla::furiosa::tcl diff --git a/mlir/dialects/tcl/tcl_types.h b/mlir/dialects/tcl/tcl_types.h new file mode 100644 index 00000000..b1fdd99d --- /dev/null +++ b/mlir/dialects/tcl/tcl_types.h @@ -0,0 +1,18 @@ +//===- tcl_types.h - Furiosa TCL types --------------------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_TCL_TCL_TYPES_H +#define MLIR_DIALECTS_TCL_TCL_TYPES_H + +#include "mlir/dialects/tcl/tcl_attrs.h" +#include "mlir/dialects/tcl/tcl_dialect.h" + +#include "mlir/IR/BuiltinTypes.h" +#include "llvm/ADT/StringRef.h" +#define GET_TYPEDEF_CLASSES +#include "mlir/dialects/tcl/tcl_types.h.inc" +namespace xla::furiosa::tcl { +// Empty name denotes a type outside the SDK's 17 element types. +llvm::StringRef ElementName(mlir::Type type); +} // namespace xla::furiosa::tcl + +#endif // MLIR_DIALECTS_TCL_TCL_TYPES_H diff --git a/mlir/dialects/tcl/tcl_types.td b/mlir/dialects/tcl/tcl_types.td new file mode 100644 index 00000000..1cbe5e7f --- /dev/null +++ b/mlir/dialects/tcl/tcl_types.td @@ -0,0 +1,23 @@ +//===- tcl_types.td - Furiosa TCL types --------------------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_TCL_TCL_TYPES_TD +#define MLIR_DIALECTS_TCL_TCL_TYPES_TD + +include "mlir/IR/AttrTypeBase.td" +include "mlir/dialects/tcl/tcl_dialect.td" + +def Tcl_Logical : TypeDef { + let mnemonic = "logical"; + let parameters = (ins "::mlir::Type":$elementType, "::mlir::ArrayAttr":$axes); + let assemblyFormat = "`<` $elementType `,` $axes `>`"; + let genVerifyDecl = 1; +} +def Tcl_Mapped : TypeDef { + let mnemonic = "mapped"; + let parameters = (ins "::mlir::Type":$elementType, + "::xla::furiosa::tcl::DramMappingAttr":$mapping); + let assemblyFormat = "`<` $elementType `,` $mapping `>`"; + let genVerifyDecl = 1; +} + +#endif // MLIR_DIALECTS_TCL_TCL_TYPES_TD diff --git a/mlir/dialects/tcl/test.zig b/mlir/dialects/tcl/test.zig new file mode 100644 index 00000000..784739ea --- /dev/null +++ b/mlir/dialects/tcl/test.zig @@ -0,0 +1,112 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const func = @import("mlir/dialects/func"); +const tcl = @import("mlir/dialects/tcl"); + +const operation_names: []const []const u8 = &.{ + "tcl.dpe", + "tcl.filter_compaction", + "tcl.graph.all_gather", + "tcl.graph.arange", + "tcl.graph.as_dram", + "tcl.graph.as_logical", + "tcl.graph.concat", + "tcl.graph.copy", + "tcl.graph.for", + "tcl.graph.full", + "tcl.graph.gather", + "tcl.graph.index_read", + "tcl.graph.index_write", + "tcl.graph.reduce_max_i32", + "tcl.graph.reshape", + "tcl.graph.scatter", + "tcl.graph.scratchpad", + "tcl.graph.slice", + "tcl.graph.sym_expr", + "tcl.graph.transmute", + "tcl.graph.vector", + "tcl.graph.yield", + "tcl.kernel", + "tcl.read", + "tcl.ve", + "tcl.ve_reduce", + "tcl.ve_select", + "tcl.write", +}; + +test { + std.testing.refAllDecls(tcl); + std.testing.refAllDecls(tcl.LogicalType); +} + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + inline for (.{ func, tcl }) |dialect| dialect.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +test "every bound operation is registered" { + const ctx = try testContext(); + defer ctx.deinit(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "types and attributes verify" { + const ctx = try testContext(); + defer ctx.deinit(); + + const a = try tcl.symbolAttr(ctx, "A"); + const t = try tcl.logicalType(ctx, .float(ctx, .bf16), &.{a}); + try std.testing.expect(t.isA(tcl.LogicalType) != null); + try std.testing.expectError(error.InvalidMlir, tcl.symbolAttr(ctx, "not an axis")); + try std.testing.expectError(error.InvalidMlir, tcl.logicalType(ctx, .float(ctx, .bf16), &.{ a, a })); + try std.testing.expectError(error.InvalidMlir, tcl.veOpcodeAttr(ctx, "unknown")); + _ = try tcl.veOpcodeAttr(ctx, "to_f16"); +} + +/// `tcl.kernel(input) { read; filter_compaction along `axis`; write }`. +fn compactionKernel(ctx: *mlir.Context, entry: *mlir.Block, axis: *const mlir.Attribute) !*mlir.Operation { + const loc: *const mlir.Location = .unknown(ctx); + const n = try tcl.symbolAttr(ctx, "N"); + const k = try tcl.symbolAttr(ctx, "K"); + const input = try tcl.logicalType(ctx, .int(ctx, .i32), &.{n}); + const output = try tcl.logicalType(ctx, .int(ctx, .i32), &.{k}); + const body = mlir.Block.init(&.{}, &.{}); + const x = tcl.read(ctx, &.{entry.argument(0)}, input, .{}, loc).appendTo(body); + const compacted = tcl.filter_compaction(ctx, x.result(0), x.result(0), output, .ge, .int(ctx, .i32, 0), &.{axis}, k, loc).appendTo(body); + _ = tcl.write(ctx, compacted.result(0), loc).appendTo(body); + return tcl.kernel(ctx, &.{entry.argument(0)}, output, .filter_compaction, body, .{}, loc).appendTo(entry); +} + +test "filter_compaction verifies inside a kernel" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc: *const mlir.Location = .unknown(ctx); + const n = try tcl.symbolAttr(ctx, "N"); + const entry = mlir.Block.init(&.{try tcl.logicalType(ctx, .int(ctx, .i32), &.{n})}, &.{loc}); + // The kernel verifier walks to its inputs' regions. + const function = func.func(ctx, .{ .name = "compact", .block = entry, .results = &.{}, .location = loc, .verify = false }); + defer function.deinit(); + try std.testing.expect((try compactionKernel(ctx, entry, n)).verify()); + try std.testing.expect(!(try compactionKernel(ctx, entry, try tcl.symbolAttr(ctx, "Missing"))).verify()); +} + +test "attributes and types print and reparse" { + const ctx = try testContext(); + defer ctx.deinit(); + const a = try tcl.symbolAttr(ctx, "A"); + const t = try tcl.logicalType(ctx, .float(ctx, .bf16), &.{a}); + var type_text: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer type_text.deinit(); + try type_text.writer.print("{f}", .{t}); + try std.testing.expect((try mlir.Type.parse(ctx, type_text.written())).eql(t)); + var attr_text: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer attr_text.deinit(); + try attr_text.writer.print("{f}", .{a}); + try std.testing.expect((try mlir.Attribute.parse(ctx, attr_text.written())).eql(a)); +} diff --git a/mlir/dialects/tcl/ve_instructions.inc b/mlir/dialects/tcl/ve_instructions.inc new file mode 100644 index 00000000..657eddca --- /dev/null +++ b/mlir/dialects/tcl/ve_instructions.inc @@ -0,0 +1,48 @@ +//===- ve_instructions.inc - Furiosa TCL VE instructions ------------------===// +// +// SDK 2026.3 VE opcode vocabulary; conversion widths 0..31 are parametric. + +VE("exp", "exp", 1) +VE("neg_exp", "neg_exp", 1) +VE("sqrt", "sqrt", 1) +VE("tanh", "tanh", 1) +VE("sigmoid", "sigmoid", 1) +VE("erf", "erf", 1) +VE("log", "log", 1) +VE("sin", "sin", 1) +VE("cos", "cos", 1) +VE("negf", "neg_f", 1) +VE("absf", "abs_f", 1) +VE("addi", "+_fxp", 2) +VE("addi_sat", "+_fxp_sat", 2) +VE("subi", "-_fxp", 2) +VE("subi_sat", "-_fxp_sat", 2) +VE("mul_q31", "*_fxp", 2) +VE("muli", "*_i", 2) +VE("shl", "<<", 2) +VE("shl_sat", "<<_sat", 2) +VE("shr_logical", ">>>", 2) +VE("shr_arith", ">>", 2) +VE("shr_arith_round", ">>_round", 2) +VE("bit_andi", "&_i", 2) +VE("bit_ori", "|_i", 2) +VE("bit_xori", "^_i", 2) +VE("mini", "min_i", 2) +VE("maxi", "max_i", 2) +VE("absmini", "|min|_i", 2) +VE("absmaxi", "|max|_i", 2) +VE("addf", "+", 2) +VE("subf", "-", 2) +VE("mulf", "*", 2) +VE("divf", "/", 2) +VE("bit_andf", "&_f", 2) +VE("bit_orf", "|_f", 2) +VE("bit_xorf", "^_f", 2) +VE("minf", "min_f", 2) +VE("maxf", "max_f", 2) +VE("absminf", "|min|_f", 2) +VE("absmaxf", "|max|_f", 2) +VE("reinterpret_as_float", "as_f", 1) +VE("reinterpret_as_int", "as_i", 1) +VE("to_float", "to_f31", 1) +VE("to_int", "to_i31", 1) diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 0c9340f9..e0684831 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -149,6 +149,7 @@ zig_library( "//dsl/cute:cute_builder", "//dsl/fly:fly_builder", "//dsl/mosaic_tpu:mosaic_tpu_builder", + "//dsl/tcl:tcl_builder", "//dsl/triton:triton_builder", "//ffi:zig", "//mlir", diff --git a/zml/kernel.zig b/zml/kernel.zig index e5b8aa9c..e60736c9 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -9,6 +9,7 @@ const mosaic_tpu_builder = @import("dsl/mosaic_tpu/builder"); const mosaic_tpu_dialect = @import("mlir/dialects/mosaic_tpu"); const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); +const tcl_builder = @import("dsl/tcl/builder"); const triton_builder = @import("dsl/triton/builder"); const Compiler = @import("Compiler.zig"); @@ -705,6 +706,154 @@ pub const cute = struct { } }; +pub const tcl = struct { + pub const Builder = tcl_builder.Builder; + pub const TensorOperation = tcl_builder.TensorOperation; + pub const Value = tcl_builder.Value; + pub const Axis = tcl_builder.Axis; + pub const Expr = tcl_builder.Expr; + pub const Dram = tcl_builder.Dram; + pub const Context = tcl_builder.Context; + pub const DType = tcl_builder.DType; + pub const VeOp = tcl_builder.VeOp; + pub const ReduceMode = tcl_builder.ReduceMode; + pub const Predicate = tcl_builder.Predicate; + pub const Tactic = tcl_builder.Tactic; + pub const FinishError = tcl_builder.FinishError; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(tcl_builder.registerDialects); + } + + pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .bool, + .i4 => .i4, + .i8 => .i8, + .i16 => .i16, + .i32 => .i32, + .i64 => .i64, + .u8 => .u8, + .u16 => .u16, + .u32 => .u32, + .u64 => .u64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f4e2m1 => .f4_e2, + .f8e4m3fn => .f8_e4, + .f8e5m2 => .f8_e5, + else => std.debug.panic("zml.kernel.tcl.from: dtype {s} has no TCL equivalent", .{@tagName(dt)}), + }; + } + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; + } + + /// A TCL function compiled by the Furiosa plugin as one native unit: + /// `declareArgs` declares the inputs in order, `ret` the outputs. + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + return build(allocator, cfg, null, null, null); + } + + /// `updated`, when given, receives the input each output updates + /// in place (`Builder.update`), or null. + fn build(allocator: std.mem.Allocator, cfg: ConfigT, inputs: ?[]const Shape, outputs: ?[]const Shape, updated: ?*[spec.outputs.len]?usize) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try Builder.open(allocator, ctx, name); + defer b.deinit(); + try spec.run(&b, cfg); + + const results = b.results orelse std.debug.panic("zml.kernel.tcl.Kernel({s}): run did not call ret", .{name}); + if (b.args.len != spec.inputs.len or results.len != spec.outputs.len) { + std.debug.panic("zml.kernel.tcl.Kernel({s}): {d} arguments and {d} results for {d} inputs and {d} outputs", .{ name, b.args.len, results.len, spec.inputs.len, spec.outputs.len }); + } + + if (inputs) |shapes| for (b.args, shapes, spec.inputs) |t, s, n| check(t, s, n); + if (outputs) |shapes| for (results, shapes, spec.outputs) |t, s, n| check(t, s, n); + if (updated) |u| for (u, results) |*dst, r| { + dst.* = b.updatedArgument(r); + }; + + return b.finish(); + } + + fn check(t: tcl_builder.Tensor, s: Shape, field: []const u8) void { + var same = from(s.dtype()) == t.dtype; + if (t.shards > 1) { + // A partition holds one chip's share of a chip-split DRAM tensor. + var n: i64 = 1; + for (t.axes) |a| n *= a.size; + same = same and n == @as(i64, @intCast(s.count())) * t.shards; + } else { + same = same and s.rank() == t.axes.len; + if (same) for (t.axes, 0..) |a, i| { + same = same and a.size == s.dim(i); + }; + } + + if (!same) std.debug.panic("zml.kernel.tcl.Kernel({s}): {s} is {f}, the TCL function has {s} over {d} axes", .{ name, field, s, @tagName(t.dtype), t.axes.len }); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + var input_shapes: [spec.inputs.len]Shape = undefined; + inline for (spec.inputs, 0..) |fname, i| { + inputs_arr[i] = @field(inputs, fname); + input_shapes[i] = inputs_arr[i].shape(); + } + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + var updated: [spec.outputs.len]?usize = undefined; + const ir = build(cur.allocator, opts.cfg, &input_shapes, &outputs_arr, &updated) catch |err| + std.debug.panic("zml.kernel.tcl.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; + for (updated, 0..) |input, output| if (input) |i| aliases.appendAssumeCapacity(.{ + .output_index = @intCast(output), + .operand_index = @intCast(i), + }); + const tensor_results = ops.tcl(inputs_arr, outputs_arr, .{ .ir = ir, .output_operand_aliases = aliases.constSlice() }); + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| { + @field(results, fname) = tensor_results[i]; + } + + return results; + } + }; + } +}; + pub const fly = struct { /// Threads per wavefront: 64 on CDNA, 32 on RDNA. pub fn waveSize(p: *const platform_.Platform) i32 { @@ -1041,6 +1190,71 @@ test "fly kernels run on rocm" { }; } +/// `furiosa.tcl.examples.softmax` over f32. +const TclSoftmax = struct { + const Cfg = struct { b: i64, s: i64, d: i64 }; + const K = tcl.Kernel(Cfg, .{ + .name = "softmax_kernel", + .inputs = &.{"x"}, + .outputs = &.{"y"}, + .run = run, + }); + + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + const B = b.axis("B", cfg.b); + const S = b.axis("S", cfg.s); + const D = b.axis("D", cfg.d); + const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ B, S, D } } }); + + const op = b.tensorOperation(.{}); + const x = op.fetch(t.x, .{}); + const e = x.subf(x.reduce(&.{D}, .maxf)).exp(); + b.ret(&.{try op.commit(e.divf(e.reduce(&.{D}, .addf)), .{})}); + } +}; + +test "tcl kernels run on furiosa" { + const zml = @import("zml.zig"); + const allocator = std.testing.allocator; + const cfg: TclSoftmax.Cfg = .{ .b = 2, .s = 16, .d = 256 }; + + const ir = try TclSoftmax.K.emit(allocator, cfg); + defer allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "func.func @softmax_kernel(") != null); + + const platform = zml.testing.env(); + if (platform.target != .furiosa) return error.SkipZigTest; + + const Mod = struct { + pub fn forward(x: Tensor) Tensor { + return TclSoftmax.K.call(.{ .x = x }, .{ .y = x.shape() }, .{ .cfg = cfg }).y; + } + }; + const x: Tensor = .init(.{ cfg.b, cfg.s, cfg.d }, .f32); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{}); + defer exe.deinit(); + + var input: [cfg.b * cfg.s * cfg.d]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 37)) * 0.25 - 4; + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&input)); + defer x_buffer.deinit(); + var result = try exe.eval(allocator, std.testing.io, .{x_buffer}); + defer result.deinit(); + var host = try result.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + + const out = host.items(f32); + var row: usize = 0; + while (row < cfg.b * cfg.s) : (row += 1) { + const in = input[row * cfg.d ..][0..cfg.d]; + var max: f32 = in[0]; + for (in) |v| max = @max(max, v); + var sum: f32 = 0; + for (in) |v| sum += @exp(v - max); + for (in, out[row * cfg.d ..][0..cfg.d]) |v, y| try std.testing.expectApproxEqAbs(@exp(v - max) / sum, y, 1e-4); + } +} + test "cute kernel emits the module the CuTe compiler takes" { const Cfg = struct { n: i64 }; const AddOne = cute.Kernel(Cfg, .{ @@ -1094,3 +1308,56 @@ test "cuda_tile kernel emits a module XLA can parse" { try std.testing.expect(std.mem.indexOf(u8, ir, "load_view_tko") != null); try std.testing.expect(std.mem.indexOf(u8, ir, "store_view_tko") != null); } + +/// exp over a tensor split across two chips in one launch, returning the +/// split result and its all_gather. +const TclSplitExp = struct { + const Cfg = struct {}; + const K = tcl.Kernel(Cfg, .{ .name = "split_exp", .inputs = &.{"x"}, .outputs = &.{ "y", "all" }, .run = run }); + fn run(b: *tcl.Builder, _: Cfg) tcl.FinishError!void { + const C = b.axis("C", 2); + const A = b.axis("A", 1024); + const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ C, A }, .dram = tcl.Dram{ .chip = &.{.{ .axis = C }} } } }); + const op = b.tensorOperation(.{ .context = .{ .layout = .{ .chip = .{ .axis = C } } } }); + const local = try op.commit(op.fetch(t.x, .{}).exp(), .{}); + const all = try b.allGather(local, C, .{}); + b.ret(&.{ try b.asDram(local, .{ .chip = &.{.{ .axis = C }} }), try b.asDram(all, .{}) }); + } + fn body(ctx: struct { x: Tensor }, shapes: [2]Shape) [2]Tensor { + const r = K.call(.{ .x = ctx.x }, .{ .y = shapes[0], .all = shapes[1] }, .{ .cfg = .{} }); + return .{ r.y, r.all }; + } +}; + +// Needs FURIOSA_VISIBLE_DEVICES=0,1. +test "tcl multi-chip kernels run on two furiosa chips" { + const zml = @import("zml.zig"); + const allocator = std.testing.allocator; + const platform = zml.testing.env(); + if (platform.target != .furiosa or platform.devices.len != 2) return error.SkipZigTest; + const model = platform.meshes.get("model").?; + const x_shape = Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(model, .{ .c = .model }); + const Mod = struct { + pub fn forward(x: Tensor) [2]Tensor { + const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), Shape.init(.{ .c = 2, .a = 1024 }, .f32) }); + return .{ r[0], r[1] }; + } + }; + const x: Tensor = .fromShape(x_shape); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{ .meshes = &.{model} }); + defer exe.deinit(); + var hx: [2 * 1024]f32 = undefined; + for (&hx, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 13)) * 0.125 - 0.5; + var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x_shape, std.mem.sliceAsBytes(&hx)); + defer bx.deinit(); + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + var res: [2]zml.Buffer = undefined; + runner.run(std.testing.io, .{bx}, .{ &res[0], &res[1] }, .{ .wait = true }); + defer for (&res) |*r| r.deinit(); + for (res) |r| { + var host = try r.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectApproxEqRel(@exp(hx[i]), v, 1e-4); + } +} diff --git a/zml/ops.zig b/zml/ops.zig index 9d837e6b..d1f78276 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1375,6 +1375,35 @@ pub fn cudaTile(inputs: anytype, outputs: anytype, opts: CudaTileOps) [outputs.l return outputs_; } +pub const TclOps = struct { + ir: []const u8, + /// Outputs the function updates in place in an input's buffer. + output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, +}; + +/// A TCL function the Furiosa plugin compiles as its own native unit. +pub fn tcl(inputs: anytype, outputs: anytype, opts: TclOps) [outputs.len]Tensor { + const compiler = Compiler.current(); + const mlir_ctx = compiler.mlir_ctx; + + var values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| values[i] = inputs[i].value(); + + var res_types: [outputs.len]*const mlir.Type = undefined; + inline for (outputs, 0..) |output, i| res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); + + const op = stablehlo.custom_call(mlir_ctx, &values, &res_types, .{ + .call_target_name = "furiosa.tcl", + .backend_config = .{ .original = opts.ir }, + .has_side_effect = false, + .output_operand_aliases = opts.output_operand_aliases, + }, compiler.location).appendTo(compiler.currentScope().block); + + var outputs_: [outputs.len]Tensor = undefined; + inline for (outputs, 0..) |output, i| outputs_[i] = Tensor._result(output, op.result(i)); + return outputs_; +} + pub const CuteOps = struct { /// The kernel symbol. name: []const u8, From b71982fda7ef733e98362520b7cfe62e33eee985 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Mon, 26 Oct 2026 11:41:06 +0100 Subject: [PATCH 823/881] mlir/dialects: add AIR dialect (#988) Adds Apple's AIR dialect, for writing Metal kernels in MLIR. - `mlir/dialects/air`: the AIR that Apple's Metal compiler emits for MSL. Each op is named after its `air.*` intrinsic without the type suffix (`fast_exp`, `max.u`, `simd_shuffle_xor`, `wg.barrier`, `simdgroup_matrix_8x8_load`, ...), with fastmath flags on float ops. Kernel arguments are described the way that compiler describes them: builtins, buffers with access/noalias, and `air.arg_name`. - `llvm`, `arith`: bind array and named struct types, linkage, loop annotations, and integer overflow flags. --- mlir/dialects/README.md | 9 +- mlir/dialects/air/AIRAttributes.cpp | 133 +++++++ mlir/dialects/air/AIRAttributes.h | 12 + mlir/dialects/air/AIRAttributes.td | 66 ++++ mlir/dialects/air/AIRDialect.cpp | 166 +++++++++ mlir/dialects/air/AIRDialect.h | 9 + mlir/dialects/air/AIRDialect.td | 58 +++ mlir/dialects/air/AIREnums.h | 10 + mlir/dialects/air/AIREnums.td | 53 +++ mlir/dialects/air/AIROps.cpp | 60 ++++ mlir/dialects/air/AIROps.h | 16 + mlir/dialects/air/AIROps.td | 354 ++++++++++++++++++ mlir/dialects/air/BUILD.bazel | 179 ++++++++++ mlir/dialects/air/README.md | 89 +++++ mlir/dialects/air/air.zig | 536 ++++++++++++++++++++++++++++ mlir/dialects/air/capi.cc | 189 ++++++++++ mlir/dialects/air/capi.h | 81 +++++ mlir/dialects/air/test.zig | 414 +++++++++++++++++++++ mlir/dialects/arith/arith.zig | 34 ++ mlir/dialects/arith/capi.cc | 22 ++ mlir/dialects/arith/capi.h | 7 + mlir/dialects/arith/test.zig | 18 + mlir/dialects/llvm/BUILD.bazel | 14 + mlir/dialects/llvm/capi.cc | 55 +++ mlir/dialects/llvm/capi.h | 36 ++ mlir/dialects/llvm/llvm.zig | 9 + mlir/dialects/llvm/ops.zig | 160 +++++++++ mlir/dialects/llvm/test.zig | 59 +++ 28 files changed, 2844 insertions(+), 4 deletions(-) create mode 100644 mlir/dialects/air/AIRAttributes.cpp create mode 100644 mlir/dialects/air/AIRAttributes.h create mode 100644 mlir/dialects/air/AIRAttributes.td create mode 100644 mlir/dialects/air/AIRDialect.cpp create mode 100644 mlir/dialects/air/AIRDialect.h create mode 100644 mlir/dialects/air/AIRDialect.td create mode 100644 mlir/dialects/air/AIREnums.h create mode 100644 mlir/dialects/air/AIREnums.td create mode 100644 mlir/dialects/air/AIROps.cpp create mode 100644 mlir/dialects/air/AIROps.h create mode 100644 mlir/dialects/air/AIROps.td create mode 100644 mlir/dialects/air/BUILD.bazel create mode 100644 mlir/dialects/air/README.md create mode 100644 mlir/dialects/air/air.zig create mode 100644 mlir/dialects/air/capi.cc create mode 100644 mlir/dialects/air/capi.h create mode 100644 mlir/dialects/air/test.zig create mode 100644 mlir/dialects/llvm/capi.cc create mode 100644 mlir/dialects/llvm/capi.h diff --git a/mlir/dialects/README.md b/mlir/dialects/README.md index 4ecd976f..7df47c5c 100644 --- a/mlir/dialects/README.md +++ b/mlir/dialects/README.md @@ -194,12 +194,13 @@ verified. `cute_ir/REGENERATING.md`, `fly_rocdl/` through `fly/REGENERATING.md`. - `cute_ir:capi` and `cute_nvgpu:capi` each depend on the other's `:dialect`: their C++ sources use each other's types. -- `cf`, `llvm`, `math`, `memref`, `scf`, `shardy` and `stablehlo` have no - `capi` of their own and bind upstream's C API target directly. +- `cf`, `math`, `memref`, `scf`, `shardy` and `stablehlo` have no `capi` of + their own and bind upstream's C API target directly; `llvm`'s `capi` only + adds what upstream's leaves out (loop annotations). - Hand-written builders with many parameters take an `Args` struct (`func.func`, `gpu.func`, `ttir.func`, `cuda.kernel`, `cuda_tile.entry`, - `memref.subview`, `vector.transfer_read`, ...); function-like ones carry - `location` inside it. + `air.kernel`, `memref.subview`, `vector.transfer_read`, ...); + function-like ones carry `location` inside it. - Some hand-written `Opts` hold required or non-null defaulted fields where named arguments read better (`stablehlo.PadOpts`, `mosaic_tpu.LoadOpts.sublane_mask`, `affine.ForOpts.step`). diff --git a/mlir/dialects/air/AIRAttributes.cpp b/mlir/dialects/air/AIRAttributes.cpp new file mode 100644 index 00000000..e7a0065d --- /dev/null +++ b/mlir/dialects/air/AIRAttributes.cpp @@ -0,0 +1,133 @@ +//===- AIRAttributes.cpp - Metal AIR attributes ---------------------------===// + +#include "mlir/dialects/air/AIRAttributes.h" + +#include "mlir/dialects/air/AIRDialect.h" + +#include "mlir/Dialect/LLVMIR/LLVMTypes.h" +#include "mlir/IR/Builders.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/DialectImplementation.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/TypeSwitch.h" +#include "llvm/Support/MathExtras.h" + +#include + +using namespace mlir; +using namespace mlir::metal::air; + +#include "mlir/dialects/air/AIREnums.cpp.inc" +#define GET_ATTRDEF_CLASSES +#include "mlir/dialects/air/AIRAttributes.cpp.inc" + +void AIRDialect::registerAttributes() { + addAttributes< +#define GET_ATTRDEF_LIST +#include "mlir/dialects/air/AIRAttributes.cpp.inc" + >(); +} + +static bool isValidName(StringRef name) { + return !name.empty() && !name.contains('\0'); +} + +// The size of a Metal scalar or vector, where 3-element vectors take the +// space of 4. +static std::optional metalSize(Type type) { + uint64_t elements = 1; + if (auto vector = dyn_cast(type)) { + if (vector.isScalable() || vector.getRank() != 1) + return std::nullopt; + elements = vector.getNumElements(); + if (elements < 2 || elements > 4) + return std::nullopt; + if (elements == 3) + elements = 4; + type = vector.getElementType(); + } + if (!type.isIntOrFloat()) + return std::nullopt; + return elements * llvm::divideCeil(type.getIntOrFloatBitWidth(), 8); +} + +LogicalResult +StructFieldAttr::verify(llvm::function_ref emitError, + uint64_t offset, uint64_t size, uint64_t count, + StringRef typeName, StringRef name) { + if (!size) + return emitError() << "expected a positive struct field size"; + if (!isValidName(typeName) || !isValidName(name)) + return emitError() << "expected nonempty field and type names"; + return success(); +} + +static LogicalResult +verifyStructFields(llvm::function_ref emitError, + LLVM::LLVMStructType type, uint64_t typeSize, + ArrayRef fields) { + if (type.isOpaque() || type.isPacked()) + return emitError() << "expected a sized, unpacked struct"; + ArrayRef body = type.getBody(); + if (body.size() != fields.size()) + return emitError() << "expected one field per struct element"; + uint64_t end = 0; + for (auto [member, field] : llvm::zip_equal(body, fields)) { + Type element = member; + uint64_t count = 0; + if (auto array = dyn_cast(element)) { + count = array.getNumElements(); + element = array.getElementType(); + } + std::optional size = metalSize(element); + if (!size || *size != field.getSize() || count != field.getCount()) + return emitError() << "field " << field.getName() + << " does not match the struct element " << element; + if (field.getOffset() < end || field.getOffset() % *size != 0) + return emitError() << "field " << field.getName() + << " is misplaced at offset " << field.getOffset(); + end = field.getOffset() + *size * std::max(count, 1); + } + if (end > typeSize) + return emitError() << "struct fields overflow the buffer type size"; + return success(); +} + +LogicalResult BufferAttr::verify( + llvm::function_ref emitError, unsigned locationIndex, + AddressSpace addressSpace, Access access, Type elementType, + StringRef typeName, uint64_t typeSize, uint64_t typeAlignment, + uint64_t bufferSize, bool noalias, ArrayRef fields) { + if (addressSpace == AddressSpace::constant && access != Access::read) + return emitError() << "constant buffers are read-only"; + if (addressSpace == AddressSpace::threadgroup && access != Access::read_write) + return emitError() << "threadgroup buffers are read_write"; + if (!elementType) + return emitError() << "expected a buffer element type"; + if (!isValidName(typeName)) + return emitError() << "expected a nonempty Metal type name"; + if (!typeSize || !llvm::isPowerOf2_64(typeAlignment) || + typeSize % typeAlignment != 0) + return emitError() << "expected a positive ABI size divisible by a " + "power-of-two alignment"; + if (auto structType = dyn_cast(elementType)) { + if (addressSpace != AddressSpace::constant) + return emitError() << "struct buffers must be constant"; + if (failed(verifyStructFields(emitError, structType, typeSize, fields))) + return failure(); + } else { + std::optional size = metalSize(elementType); + if (!size) + return emitError() << "expected a scalar, vector or struct element type"; + if (*size != typeSize) + return emitError() << "ABI size " << typeSize << " does not match " + << elementType; + if (!fields.empty()) + return emitError() << "only struct buffers have fields"; + } + if (bufferSize && (addressSpace != AddressSpace::constant || + bufferSize < typeSize || bufferSize % typeAlignment)) + return emitError() << "only constant buffers have a buffer size, which " + "must cover the element and respect its alignment"; + return success(); +} diff --git a/mlir/dialects/air/AIRAttributes.h b/mlir/dialects/air/AIRAttributes.h new file mode 100644 index 00000000..4a31eb77 --- /dev/null +++ b/mlir/dialects/air/AIRAttributes.h @@ -0,0 +1,12 @@ +//===- AIRAttributes.h - Metal AIR attributes -------------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_AIR_AIRATTRIBUTES_H +#define MLIR_DIALECTS_AIR_AIRATTRIBUTES_H + +#include "mlir/dialects/air/AIREnums.h" + +#include "mlir/IR/Attributes.h" +#define GET_ATTRDEF_CLASSES +#include "mlir/dialects/air/AIRAttributes.h.inc" + +#endif // MLIR_DIALECTS_AIR_AIRATTRIBUTES_H diff --git a/mlir/dialects/air/AIRAttributes.td b/mlir/dialects/air/AIRAttributes.td new file mode 100644 index 00000000..3e4852d0 --- /dev/null +++ b/mlir/dialects/air/AIRAttributes.td @@ -0,0 +1,66 @@ +//===- AIRAttributes.td - Metal AIR attributes -------------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_AIR_AIRATTRIBUTES_TD +#define MLIR_DIALECTS_AIR_AIRATTRIBUTES_TD + +include "mlir/dialects/air/AIRDialect.td" +include "mlir/dialects/air/AIREnums.td" + +def AIR_BuiltinAttr : EnumAttr { + let summary = "Kernel argument builtin"; + let description = [{ + Attach as `air.builtin` to a kernel argument. The position and size + builtins are `uint` or `ushort` scalars or 2- and 3-element vectors + (`i32`, `i16`, `vector<3xi32>`, ...); the index builtins are `uint` or + `ushort` scalars. + }]; +} + +def AIR_MemFlagsAttr : EnumAttr { + let summary = "Memory a barrier orders"; +} + +def AIR_StructField : AttrDef { + let mnemonic = "struct_field"; + let summary = "Field of a constant buffer struct"; + let description = [{ + One `air.struct_type_info` entry: the byte offset, the element size, + the array length (zero for a non-array field), the Metal type name and + the field name. + }]; + let parameters = (ins "uint64_t":$offset, + "uint64_t":$size, + DefaultValuedParameter<"uint64_t", "0">:$count, + StringRefParameter<>:$typeName, + StringRefParameter<>:$name); + let assemblyFormat = "`<` struct(params) `>`"; + let genVerifyDecl = 1; +} + +def AIR_Buffer : AttrDef { + let mnemonic = "buffer"; + let summary = "Kernel buffer argument"; + let description = [{ + Attach as `air.buffer` to a kernel argument of type `!llvm.ptr` in + `addressSpace`. `elementType` is the pointee (a scalar, a vector, or an + `!llvm.struct` described by `fields`), `typeName` its Metal spelling, + and `typeSize` and `typeAlignment` its size and alignment in bytes. + `locationIndex` is the `[[buffer(n)]]` or `[[threadgroup(n)]]` index. + A constant reference records its size as `bufferSize`. `noalias` is a + `__restrict__` pointer. + }]; + let parameters = (ins "unsigned":$locationIndex, + EnumParameter:$addressSpace, + EnumParameter:$access, + "::mlir::Type":$elementType, + StringRefParameter<>:$typeName, + "uint64_t":$typeSize, + "uint64_t":$typeAlignment, + DefaultValuedParameter<"uint64_t", "0">:$bufferSize, + DefaultValuedParameter<"bool", "false">:$noalias, + OptionalArrayRefParameter<"StructFieldAttr">:$fields); + let assemblyFormat = "`<` struct(params) `>`"; + let genVerifyDecl = 1; +} + +#endif // MLIR_DIALECTS_AIR_AIRATTRIBUTES_TD diff --git a/mlir/dialects/air/AIRDialect.cpp b/mlir/dialects/air/AIRDialect.cpp new file mode 100644 index 00000000..78ccbba0 --- /dev/null +++ b/mlir/dialects/air/AIRDialect.cpp @@ -0,0 +1,166 @@ +//===- AIRDialect.cpp - Metal AIR dialect ---------------------------------===// + +#include "mlir/dialects/air/AIRDialect.h" + +#include "mlir/dialects/air/AIRAttributes.h" +#include "mlir/dialects/air/AIROps.h" + +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/IR/BuiltinOps.h" +#include "mlir/IR/DialectImplementation.h" +#include "mlir/Interfaces/FunctionInterfaces.h" +#include "llvm/ADT/DenseSet.h" + +#include + +using namespace mlir; +using namespace mlir::metal::air; + +#include "mlir/dialects/air/AIRDialect.cpp.inc" + +void AIRDialect::initialize() { + registerAttributes(); + addOperations< +#define GET_OP_LIST +#include "mlir/dialects/air/AIROps.cpp.inc" + >(); +} + +static bool isKernelArgumentAttribute(StringRef name) { + return name == AIRDialect::getBuiltinAttrName() || + name == AIRDialect::getBufferAttrName() || + name == AIRDialect::getArgNameAttrName(); +} + +// Position and size builtins may be 2- or 3-element vectors; index builtins +// are scalars. +static bool isVectorBuiltin(Builtin builtin) { + switch (builtin) { + case Builtin::thread_position_in_grid: + case Builtin::threadgroup_position_in_grid: + case Builtin::thread_position_in_threadgroup: + case Builtin::threads_per_threadgroup: + case Builtin::threadgroups_per_grid: + case Builtin::threads_per_grid: + return true; + default: + return false; + } +} + +static LogicalResult verifyKernelArgument(FunctionOpInterface function, + unsigned index) { + Operation *op = function.getOperation(); + Attribute builtin = + function.getArgAttr(index, AIRDialect::getBuiltinAttrName()); + Attribute buffer = + function.getArgAttr(index, AIRDialect::getBufferAttrName()); + if (bool(builtin) == bool(buffer)) + return op->emitOpError() + << "argument " << index + << " must have exactly one of air.builtin or air.buffer"; + if (Attribute name = + function.getArgAttr(index, AIRDialect::getArgNameAttrName())) + if (auto string = dyn_cast(name); + !string || string.getValue().empty()) + return op->emitOpError("air.arg_name must be a nonempty string"); + Type type = function.getArgumentTypes()[index]; + if (builtin) { + auto metadata = dyn_cast(builtin); + if (!metadata) + return op->emitOpError("air.builtin must be an #air.builtin"); + if (auto vector = dyn_cast(type)) { + if (!isVectorBuiltin(metadata.getValue()) || vector.isScalable() || + vector.getRank() != 1 || + (vector.getNumElements() != 2 && vector.getNumElements() != 3)) + return op->emitOpError() + << "builtin " << stringifyBuiltin(metadata.getValue()) + << " cannot be a " << type; + type = vector.getElementType(); + } + if (!type.isSignlessInteger(32) && !type.isSignlessInteger(16)) + return op->emitOpError("AIR builtins are uint or ushort scalars or " + "vectors"); + return success(); + } + auto metadata = dyn_cast(buffer); + if (!metadata) + return op->emitOpError("air.buffer must be an #air.buffer"); + auto pointer = dyn_cast(type); + if (!pointer || + pointer.getAddressSpace() != unsigned(metadata.getAddressSpace())) + return op->emitOpError() + << "argument " << index + << " must be an !llvm.ptr in the address space of its air.buffer"; + return success(); +} + +LogicalResult AIRDialect::verifyOperationAttribute(Operation *op, + NamedAttribute attr) { + StringRef name = attr.getName().getValue(); + if (isKernelArgumentAttribute(name)) + return op->emitOpError() << name << " belongs on a kernel argument"; + if (name == AIRDialect::getFastMathEnableAttrName()) { + if (!isa(op) || !isa(attr.getValue())) + return op->emitOpError() << name << " is a unit attribute of a module"; + return success(); + } + if (name != AIRDialect::getKernelAttrName()) + return op->emitOpError() << "unknown AIR attribute " << name; + auto function = dyn_cast(op); + if (!function || !isa(attr.getValue())) + return op->emitOpError( + "air.kernel requires a function and a unit attribute"); + if (failed(op->getName().verifyInvariants(op))) + return failure(); + if (function.getNumResults() != 0) + return op->emitOpError("AIR kernels cannot return values"); + if (auto llvmFunction = dyn_cast(op)) + if (llvmFunction.isVarArg()) + return op->emitOpError("AIR kernels cannot be variadic"); + llvm::DenseSet builtins; + llvm::DenseSet> bindings; + for (unsigned i = 0; i < function.getNumArguments(); ++i) { + if (failed(verifyKernelArgument(function, i))) + return failure(); + if (auto builtin = function.getArgAttrOfType( + i, AIRDialect::getBuiltinAttrName())) { + if (!builtins.insert(builtin.getValue()).second) + return op->emitOpError() << "duplicate AIR builtin " + << stringifyBuiltin(builtin.getValue()); + continue; + } + auto buffer = function.getArgAttrOfType( + i, AIRDialect::getBufferAttrName()); + // Device and constant buffers share bindings; threadgroup buffers do not. + auto key = std::make_pair( + unsigned(buffer.getAddressSpace() == AddressSpace::threadgroup), + buffer.getLocationIndex()); + if (!bindings.insert(key).second) + return op->emitOpError() + << "duplicate AIR buffer binding " << buffer.getLocationIndex(); + } + return success(); +} + +LogicalResult AIRDialect::verifyRegionArgAttribute(Operation *op, + unsigned regionIndex, + unsigned argIndex, + NamedAttribute attr) { + StringRef name = attr.getName().getValue(); + if (!isKernelArgumentAttribute(name)) + return op->emitOpError() << name << " is not an AIR argument attribute"; + auto function = dyn_cast(op); + if (!function || regionIndex != 0 || + !op->getAttrOfType(AIRDialect::getKernelAttrName())) + return op->emitOpError() + << name << " requires an air.kernel function argument"; + return verifyKernelArgument(function, argIndex); +} + +LogicalResult AIRDialect::verifyRegionResultAttribute(Operation *op, unsigned, + unsigned, + NamedAttribute attr) { + return op->emitOpError() << attr.getName() + << " cannot annotate a function result"; +} diff --git a/mlir/dialects/air/AIRDialect.h b/mlir/dialects/air/AIRDialect.h new file mode 100644 index 00000000..e28ec0bc --- /dev/null +++ b/mlir/dialects/air/AIRDialect.h @@ -0,0 +1,9 @@ +//===- AIRDialect.h - Metal AIR dialect -------------------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_AIR_AIRDIALECT_H +#define MLIR_DIALECTS_AIR_AIRDIALECT_H + +#include "mlir/IR/Dialect.h" +#include "mlir/dialects/air/AIRDialect.h.inc" + +#endif // MLIR_DIALECTS_AIR_AIRDIALECT_H diff --git a/mlir/dialects/air/AIRDialect.td b/mlir/dialects/air/AIRDialect.td new file mode 100644 index 00000000..2f1c2e97 --- /dev/null +++ b/mlir/dialects/air/AIRDialect.td @@ -0,0 +1,58 @@ +//===- AIRDialect.td - Metal AIR dialect -------------------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_AIR_AIRDIALECT_TD +#define MLIR_DIALECTS_AIR_AIRDIALECT_TD + +include "mlir/IR/AttrTypeBase.td" +include "mlir/IR/OpBase.td" + +def AIR_Dialect : Dialect { + let name = "air"; + let cppNamespace = "::mlir::metal::air"; + let summary = "Apple Metal AIR intrinsics and kernel ABI metadata"; + let description = [{ + The Metal-specific part of the LLVM IR Apple's Metal compiler emits (AIR), + not AMD's asynchronous AIR dialect. Each operation is an `air.*` + intrinsic named as Apple names it, without the type suffix its operand + and result types determine: `air.fast_exp %x : f32` is + `@air.fast_exp.f32`, `air.max.s %a, %b : vector<2xi32>` is + `@air.max.s.v2i32`. Kernels are `func.func` with `air.kernel`, and their + arguments carry the `!air.kernel` descriptors as `air.builtin` or + `air.buffer`. Arithmetic, memory and control flow use the arith, llvm + and scf dialects; pointers are `!llvm.ptr` in AIR's address spaces. + }]; + let dependentDialects = ["::mlir::LLVM::LLVMDialect"]; + let useDefaultAttributePrinterParser = 1; + let hasOperationAttrVerify = 1; + let hasRegionArgAttrVerify = 1; + let hasRegionResultAttrVerify = 1; + let extraClassDeclaration = [{ + void registerAttributes(); + + static constexpr ::llvm::StringLiteral getKernelAttrName() { + return ::llvm::StringLiteral("air.kernel"); + } + static constexpr ::llvm::StringLiteral getBuiltinAttrName() { + return ::llvm::StringLiteral("air.builtin"); + } + static constexpr ::llvm::StringLiteral getBufferAttrName() { + return ::llvm::StringLiteral("air.buffer"); + } + static constexpr ::llvm::StringLiteral getArgNameAttrName() { + return ::llvm::StringLiteral("air.arg_name"); + } + static constexpr ::llvm::StringLiteral getFastMathEnableAttrName() { + return ::llvm::StringLiteral("air.fast_math_enable"); + } + + static constexpr unsigned kPrivateAddressSpace = 0; + static constexpr unsigned kDeviceAddressSpace = 1; + static constexpr unsigned kConstantAddressSpace = 2; + static constexpr unsigned kThreadgroupAddressSpace = 3; + }]; +} + +class AIR_Op traits = []> + : Op; + +#endif // MLIR_DIALECTS_AIR_AIRDIALECT_TD diff --git a/mlir/dialects/air/AIREnums.h b/mlir/dialects/air/AIREnums.h new file mode 100644 index 00000000..385f2041 --- /dev/null +++ b/mlir/dialects/air/AIREnums.h @@ -0,0 +1,10 @@ +//===- AIREnums.h - Metal AIR enums -----------------------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_AIR_AIRENUMS_H +#define MLIR_DIALECTS_AIR_AIRENUMS_H + +#include "mlir/IR/BuiltinAttributes.h" +#include "mlir/dialects/air/AIREnums.h.inc" +#include "llvm/ADT/StringExtras.h" + +#endif // MLIR_DIALECTS_AIR_AIRENUMS_H diff --git a/mlir/dialects/air/AIREnums.td b/mlir/dialects/air/AIREnums.td new file mode 100644 index 00000000..e5f9f938 --- /dev/null +++ b/mlir/dialects/air/AIREnums.td @@ -0,0 +1,53 @@ +//===- AIREnums.td - Metal AIR enums -----------------------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_AIR_AIRENUMS_TD +#define MLIR_DIALECTS_AIR_AIRENUMS_TD + +include "mlir/IR/EnumAttr.td" + +def AIR_Builtin : I32EnumAttr<"Builtin", "Kernel argument builtin", [ + I32EnumAttrCase<"thread_position_in_grid", 0>, + I32EnumAttrCase<"threadgroup_position_in_grid", 1>, + I32EnumAttrCase<"thread_position_in_threadgroup", 2>, + I32EnumAttrCase<"threads_per_threadgroup", 3>, + I32EnumAttrCase<"threadgroups_per_grid", 4>, + I32EnumAttrCase<"threads_per_grid", 5>, + I32EnumAttrCase<"thread_index_in_simdgroup", 6>, + I32EnumAttrCase<"simdgroup_index_in_threadgroup", 7>, + I32EnumAttrCase<"threads_per_simdgroup", 8>, + I32EnumAttrCase<"simdgroups_per_threadgroup", 9>, + I32EnumAttrCase<"thread_index_in_threadgroup", 10>, +]> { + let cppNamespace = "::mlir::metal::air"; + let genSpecializedAttr = 0; +} + +def AIR_AddressSpace : I32EnumAttr<"AddressSpace", "Buffer address space", [ + I32EnumAttrCase<"device", 1>, + I32EnumAttrCase<"constant", 2>, + I32EnumAttrCase<"threadgroup", 3>, +]> { + let cppNamespace = "::mlir::metal::air"; + let genSpecializedAttr = 0; +} + +def AIR_Access : I32EnumAttr<"Access", "Buffer access", [ + I32EnumAttrCase<"read", 0>, + I32EnumAttrCase<"read_write", 1>, +]> { + let cppNamespace = "::mlir::metal::air"; + let genSpecializedAttr = 0; +} + +def AIR_MemFlags : I32BitEnumAttr<"MemFlags", "Barrier memory flags", [ + I32BitEnumAttrCaseNone<"none">, + I32BitEnumAttrCaseBit<"device", 0>, + I32BitEnumAttrCaseBit<"threadgroup", 1>, + I32BitEnumAttrCaseBit<"texture", 2>, +]> { + let separator = "|"; + let cppNamespace = "::mlir::metal::air"; + let genSpecializedAttr = 0; +} + +#endif // MLIR_DIALECTS_AIR_AIRENUMS_TD diff --git a/mlir/dialects/air/AIROps.cpp b/mlir/dialects/air/AIROps.cpp new file mode 100644 index 00000000..e1dfa3ac --- /dev/null +++ b/mlir/dialects/air/AIROps.cpp @@ -0,0 +1,60 @@ +//===- AIROps.cpp - Metal AIR operations ----------------------------------===// + +#include "mlir/dialects/air/AIROps.h" + +#include "mlir/dialects/air/AIRDialect.h" + +#include "mlir/IR/Builders.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/IR/TypeUtilities.h" + +using namespace mlir; +using namespace mlir::metal::air; + +#define GET_OP_CLASSES +#include "mlir/dialects/air/AIROps.cpp.inc" + +LogicalResult DotOp::verify() { + if (getElementTypeOrSelf(getX().getType()) != getResult().getType()) + return emitOpError("expects the operands' element type as result"); + return success(); +} + +static LogicalResult verifyConvert(Operation *op, bool toOrFromInteger) { + Type input = op->getOperand(0).getType(); + Type result = op->getResult(0).getType(); + if (failed(verifyCompatibleShape(input, result)) || + isa(input) != isa(result)) + return op->emitOpError("expects operands and results of the same shape"); + bool inputInt = isa(getElementTypeOrSelf(input)); + bool resultInt = isa(getElementTypeOrSelf(result)); + if (!toOrFromInteger && (inputInt || resultInt || input == result)) + return op->emitOpError("converts between distinct float types"); + if (toOrFromInteger && inputInt == resultInt) + return op->emitOpError("converts between an integer and a float type"); + if (resultInt && cast(op) + .getFastMathFlagsAttr() + .getValue() != arith::FastMathFlags::none) + return op->emitOpError("integer results take no fastmath flags"); + return success(); +} + +LogicalResult ConvertOp::verify() { return verifyConvert(*this, false); } +LogicalResult ConvertSOp::verify() { return verifyConvert(*this, true); } +LogicalResult ConvertUOp::verify() { return verifyConvert(*this, true); } + +LogicalResult SimdgroupMatrixMultiplyAccumulateOp::verify() { + auto element = [](Value value) { + return cast(value.getType()).getElementType(); + }; + Type d = element(getResult()), a = element(getA()), b = element(getB()), + c = element(getC()); + // The combinations Apple's compiler emits for simdgroup_multiply_accumulate. + bool mixed = d.isF32() && c.isF32() && (a.isF32() || a == b) && + (b.isF16() || b.isBF16() || b.isF32()); + bool half = d.isF16() && a.isF16() && b.isF16() && c.isF16(); + if (!mixed && !half) + return emitOpError("unsupported element types ") + << a << " * " << b << " + " << c << " -> " << d; + return success(); +} diff --git a/mlir/dialects/air/AIROps.h b/mlir/dialects/air/AIROps.h new file mode 100644 index 00000000..baa5a517 --- /dev/null +++ b/mlir/dialects/air/AIROps.h @@ -0,0 +1,16 @@ +//===- AIROps.h - Metal AIR operations --------------------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_AIR_AIROPS_H +#define MLIR_DIALECTS_AIR_AIROPS_H + +#include "mlir/dialects/air/AIRAttributes.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/LLVMIR/LLVMTypes.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" +#define GET_OP_CLASSES +#include "mlir/dialects/air/AIROps.h.inc" + +#endif // MLIR_DIALECTS_AIR_AIROPS_H diff --git a/mlir/dialects/air/AIROps.td b/mlir/dialects/air/AIROps.td new file mode 100644 index 00000000..4761c61e --- /dev/null +++ b/mlir/dialects/air/AIROps.td @@ -0,0 +1,354 @@ +//===- AIROps.td - Metal AIR operations --------------------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_AIR_AIROPS_TD +#define MLIR_DIALECTS_AIR_AIROPS_TD + +include "mlir/Dialect/Arith/IR/ArithBase.td" +include "mlir/Dialect/Arith/IR/ArithOpsInterfaces.td" +include "mlir/Dialect/LLVMIR/LLVMOpBase.td" +include "mlir/Interfaces/InferTypeOpInterface.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "mlir/dialects/air/AIRAttributes.td" + +class AIR_ScalarOrVectorOf types> + : AnyTypeOf])>; + +def AIR_FloatLike : AIR_ScalarOrVectorOf<[F16, F32]>; +def AIR_F32Like : AIR_ScalarOrVectorOf<[F32]>; +def AIR_IntLike : AIR_ScalarOrVectorOf<[I16, I32, I64]>; +def AIR_SimdIntLike : AIR_ScalarOrVectorOf<[I16, I32]>; +def AIR_ConvertLike : AIR_ScalarOrVectorOf<[F16, BF16, F32, I16, I32, I64]>; +def AIR_Matrix : VectorOfLengthAndType<[64], [F16, BF16, F32]>; +def AIR_MatrixVector : VectorOfLengthAndType<[2], [I64]> { + let builderCall = "::mlir::VectorType::get({2}, $_builder.getI64Type())"; +} +def AIR_MatrixPointer : AnyTypeOf<[LLVM_PointerInAddressSpace<1>, + LLVM_PointerInAddressSpace<3>]>; + +def AIR_FastMath : DefaultValuedAttr; + +//===----------------------------------------------------------------------===// +// Math +//===----------------------------------------------------------------------===// + +class AIR_ElementwiseOp traits = []> + : AIR_Op { + let summary = "`" # mnemonic # "` on each element"; + let description = "Calls `@air." # mnemonic # ".`."; +} + +class AIR_FloatUnaryOp + : AIR_ElementwiseOp]> { + let arguments = (ins type:$x, AIR_FastMath:$fastmath); + let results = (outs type:$result); + let assemblyFormat = + "$x (`fastmath` `` $fastmath^)? attr-dict `:` type($result)"; +} + +class AIR_FloatBinaryOp + : AIR_ElementwiseOp]> { + let arguments = (ins type:$x, type:$y, AIR_FastMath:$fastmath); + let results = (outs type:$result); + let assemblyFormat = + "$x `,` $y (`fastmath` `` $fastmath^)? attr-dict `:` type($result)"; +} + +class AIR_FloatTernaryOp + : AIR_ElementwiseOp]> { + let arguments = (ins type:$x, type:$y, type:$z, AIR_FastMath:$fastmath); + let results = (outs type:$result); + let assemblyFormat = [{ + $x `,` $y `,` $z (`fastmath` `` $fastmath^)? attr-dict `:` type($result) + }]; +} + +class AIR_IntUnaryOp : AIR_ElementwiseOp { + let arguments = (ins AIR_IntLike:$x); + let results = (outs AIR_IntLike:$result); + let assemblyFormat = "$x attr-dict `:` type($result)"; +} + +class AIR_IntBinaryOp : AIR_ElementwiseOp { + let arguments = (ins AIR_IntLike:$x, AIR_IntLike:$y); + let results = (outs AIR_IntLike:$result); + let assemblyFormat = "$x `,` $y attr-dict `:` type($result)"; +} + +class AIR_IntTernaryOp : AIR_ElementwiseOp { + let arguments = (ins AIR_IntLike:$x, AIR_IntLike:$y, AIR_IntLike:$z); + let results = (outs AIR_IntLike:$result); + let assemblyFormat = "$x `,` $y `,` $z attr-dict `:` type($result)"; +} + +// `exp` is `metal::precise::exp`; `fast_exp` is `metal::fast::exp`, which +// `exp` selects under fast math. Only f32 has fast variants. +def AIR_ExpOp : AIR_FloatUnaryOp<"exp">; +def AIR_FastExpOp : AIR_FloatUnaryOp<"fast_exp", AIR_F32Like>; +def AIR_Exp2Op : AIR_FloatUnaryOp<"exp2">; +def AIR_FastExp2Op : AIR_FloatUnaryOp<"fast_exp2", AIR_F32Like>; +def AIR_LogOp : AIR_FloatUnaryOp<"log">; +def AIR_FastLogOp : AIR_FloatUnaryOp<"fast_log", AIR_F32Like>; +def AIR_Log2Op : AIR_FloatUnaryOp<"log2">; +def AIR_FastLog2Op : AIR_FloatUnaryOp<"fast_log2", AIR_F32Like>; +def AIR_TanhOp : AIR_FloatUnaryOp<"tanh">; +def AIR_FastTanhOp : AIR_FloatUnaryOp<"fast_tanh", AIR_F32Like>; +def AIR_SqrtOp : AIR_FloatUnaryOp<"sqrt">; +def AIR_FastSqrtOp : AIR_FloatUnaryOp<"fast_sqrt", AIR_F32Like>; +def AIR_RsqrtOp : AIR_FloatUnaryOp<"rsqrt">; +def AIR_FastRsqrtOp : AIR_FloatUnaryOp<"fast_rsqrt", AIR_F32Like>; +def AIR_FabsOp : AIR_FloatUnaryOp<"fabs">; +def AIR_FastFabsOp : AIR_FloatUnaryOp<"fast_fabs", AIR_F32Like>; +def AIR_CeilOp : AIR_FloatUnaryOp<"ceil">; +def AIR_FastCeilOp : AIR_FloatUnaryOp<"fast_ceil", AIR_F32Like>; +def AIR_FloorOp : AIR_FloatUnaryOp<"floor">; +def AIR_FastFloorOp : AIR_FloatUnaryOp<"fast_floor", AIR_F32Like>; +def AIR_RoundOp : AIR_FloatUnaryOp<"round">; +def AIR_FastRoundOp : AIR_FloatUnaryOp<"fast_round", AIR_F32Like>; +def AIR_TruncOp : AIR_FloatUnaryOp<"trunc">; +def AIR_FastTruncOp : AIR_FloatUnaryOp<"fast_trunc", AIR_F32Like>; +def AIR_SinOp : AIR_FloatUnaryOp<"sin">; +def AIR_FastSinOp : AIR_FloatUnaryOp<"fast_sin", AIR_F32Like>; +def AIR_CosOp : AIR_FloatUnaryOp<"cos">; +def AIR_FastCosOp : AIR_FloatUnaryOp<"fast_cos", AIR_F32Like>; + +def AIR_FmaxOp : AIR_FloatBinaryOp<"fmax">; +def AIR_FastFmaxOp : AIR_FloatBinaryOp<"fast_fmax", AIR_F32Like>; +def AIR_FminOp : AIR_FloatBinaryOp<"fmin">; +def AIR_FastFminOp : AIR_FloatBinaryOp<"fast_fmin", AIR_F32Like>; +def AIR_PowOp : AIR_FloatBinaryOp<"pow">; +def AIR_FastPowOp : AIR_FloatBinaryOp<"fast_pow", AIR_F32Like>; +def AIR_FmodOp : AIR_FloatBinaryOp<"fmod">; +def AIR_FastFmodOp : AIR_FloatBinaryOp<"fast_fmod", AIR_F32Like>; + +def AIR_FmaOp : AIR_FloatTernaryOp<"fma">; +def AIR_ClampOp : AIR_FloatTernaryOp<"clamp">; +def AIR_FastClampOp : AIR_FloatTernaryOp<"fast_clamp", AIR_F32Like>; + +def AIR_DotOp : AIR_Op<"dot", [Pure, AllTypesMatch<["x", "y"]>, + DeclareOpInterfaceMethods]> { + let summary = "Dot product of two float vectors"; + let description = "Calls `@air.dot.`."; + let arguments = (ins FixedVectorOfLengthAndType<[2, 3, 4], [F16, F32]>:$x, + FixedVectorOfLengthAndType<[2, 3, 4], [F16, F32]>:$y, + AIR_FastMath:$fastmath); + let results = (outs AnyTypeOf<[F16, F32]>:$result); + let assemblyFormat = [{ + $x `,` $y (`fastmath` `` $fastmath^)? attr-dict `:` type($x) `->` + type($result) + }]; + let hasVerifier = 1; +} + +// `.s` and `.u` are the signed and unsigned overloads of MSL's integer +// functions. +def AIR_MaxSOp : AIR_IntBinaryOp<"max.s">; +def AIR_MaxUOp : AIR_IntBinaryOp<"max.u">; +def AIR_MinSOp : AIR_IntBinaryOp<"min.s">; +def AIR_MinUOp : AIR_IntBinaryOp<"min.u">; +def AIR_AbsSOp : AIR_IntUnaryOp<"abs.s">; +def AIR_ClampSOp : AIR_IntTernaryOp<"clamp.s">; +def AIR_ClampUOp : AIR_IntTernaryOp<"clamp.u">; + +//===----------------------------------------------------------------------===// +// Conversions +//===----------------------------------------------------------------------===// + +class AIR_ConvertBaseOp + : AIR_Op]> { + let summary = summaryText; + let description = [{ + Calls `@air.convert....`, + where floats are `f` and the integer side is `s` or `u` as the mnemonic + says. Scalar f32 to and from f16 or bf16, and integer to integer, are + LLVM casts in Apple's compiler. + }]; + let arguments = (ins AIR_ConvertLike:$x, AIR_FastMath:$fastmath); + let results = (outs AIR_ConvertLike:$result); + let assemblyFormat = [{ + $x (`fastmath` `` $fastmath^)? attr-dict `:` type($x) `to` type($result) + }]; + let hasVerifier = 1; +} + +def AIR_ConvertOp : AIR_ConvertBaseOp<"convert", "Float to float conversion">; +def AIR_ConvertSOp : AIR_ConvertBaseOp<"convert.s", + "Conversion between floats and signed integers">; +def AIR_ConvertUOp : AIR_ConvertBaseOp<"convert.u", + "Conversion between floats and unsigned integers">; + +//===----------------------------------------------------------------------===// +// SIMD-group functions +//===----------------------------------------------------------------------===// + +// SIMD-group functions read the other lanes, so they are not Pure: unknown +// effects keep them from being speculated or moved across divergent control. +class AIR_SimdReduceOp traits = []> + : AIR_Op { + let summary = "`" # mnemonic # "` across the SIMD-group"; + let description = "Calls `@air." # mnemonic # ".`."; + let results = (outs type:$result); +} + +class AIR_SimdFloatReduceOp + : AIR_SimdReduceOp]> { + let arguments = (ins AIR_FloatLike:$x, AIR_FastMath:$fastmath); + let assemblyFormat = + "$x (`fastmath` `` $fastmath^)? attr-dict `:` type($result)"; +} + +class AIR_SimdIntReduceOp + : AIR_SimdReduceOp { + let arguments = (ins AIR_SimdIntLike:$x); + let assemblyFormat = "$x attr-dict `:` type($result)"; +} + +class AIR_SimdShuffleOp traits = []> + : AIR_Op]> { + let summary = "`" # mnemonic # "` across the SIMD-group"; + let description = [{ + Calls `@air.}] # mnemonic # [{.`. `lane` is the source lane, the + XOR mask or the lane delta, as for the MSL function. + }]; + let results = (outs type:$result); +} + +class AIR_SimdFloatShuffleOp + : AIR_SimdShuffleOp]> { + let arguments = (ins AIR_FloatLike:$data, I16:$lane, AIR_FastMath:$fastmath); + let assemblyFormat = [{ + $data `,` $lane (`fastmath` `` $fastmath^)? attr-dict `:` type($result) + }]; +} + +class AIR_SimdIntShuffleOp + : AIR_SimdShuffleOp { + let arguments = (ins AIR_SimdIntLike:$data, I16:$lane); + let assemblyFormat = "$data `,` $lane attr-dict `:` type($result)"; +} + +def AIR_SimdSumOp : AIR_SimdFloatReduceOp<"simd_sum">; +def AIR_SimdSumSOp : AIR_SimdIntReduceOp<"simd_sum.s">; +def AIR_SimdSumUOp : AIR_SimdIntReduceOp<"simd_sum.u">; +def AIR_SimdMaxOp : AIR_SimdFloatReduceOp<"simd_max">; +def AIR_SimdMaxSOp : AIR_SimdIntReduceOp<"simd_max.s">; +def AIR_SimdMaxUOp : AIR_SimdIntReduceOp<"simd_max.u">; +def AIR_SimdMinOp : AIR_SimdFloatReduceOp<"simd_min">; +def AIR_SimdMinSOp : AIR_SimdIntReduceOp<"simd_min.s">; +def AIR_SimdMinUOp : AIR_SimdIntReduceOp<"simd_min.u">; +def AIR_SimdPrefixExclusiveSumOp + : AIR_SimdFloatReduceOp<"simd_prefix_exclusive_sum">; + +def AIR_SimdShuffleOp : AIR_SimdFloatShuffleOp<"simd_shuffle">; +def AIR_SimdShuffleSOp : AIR_SimdIntShuffleOp<"simd_shuffle.s">; +def AIR_SimdShuffleUOp : AIR_SimdIntShuffleOp<"simd_shuffle.u">; +def AIR_SimdShuffleXorOp : AIR_SimdFloatShuffleOp<"simd_shuffle_xor">; +def AIR_SimdShuffleXorSOp : AIR_SimdIntShuffleOp<"simd_shuffle_xor.s">; +def AIR_SimdShuffleXorUOp : AIR_SimdIntShuffleOp<"simd_shuffle_xor.u">; +def AIR_SimdShuffleDownOp : AIR_SimdFloatShuffleOp<"simd_shuffle_down">; +def AIR_SimdShuffleDownSOp : AIR_SimdIntShuffleOp<"simd_shuffle_down.s">; +def AIR_SimdShuffleDownUOp : AIR_SimdIntShuffleOp<"simd_shuffle_down.u">; +def AIR_SimdBroadcastOp : AIR_SimdFloatShuffleOp<"simd_broadcast">; +def AIR_SimdBroadcastSOp : AIR_SimdIntShuffleOp<"simd_broadcast.s">; +def AIR_SimdBroadcastUOp : AIR_SimdIntShuffleOp<"simd_broadcast.u">; + +class AIR_SimdVoteOp : AIR_Op { + let summary = summaryText; + let description = "Calls `@air." # mnemonic # "`."; + let arguments = (ins I1:$x); + let results = (outs I1:$result); + let assemblyFormat = "$x attr-dict"; +} + +def AIR_SimdAnyOp : AIR_SimdVoteOp<"simd_any", + "Whether `x` is true in any active lane">; +def AIR_SimdAllOp : AIR_SimdVoteOp<"simd_all", + "Whether `x` is true in every active lane">; + +//===----------------------------------------------------------------------===// +// Barriers +//===----------------------------------------------------------------------===// + +def AIR_WgBarrierOp : AIR_Op<"wg.barrier"> { + let summary = "threadgroup_barrier"; + let description = "Calls `@air.wg.barrier(i32 mem_flags, i32 1)`."; + let arguments = (ins AIR_MemFlagsAttr:$mem_flags); + let assemblyFormat = "$mem_flags attr-dict"; +} + +def AIR_SimdgroupBarrierOp : AIR_Op<"simdgroup.barrier"> { + let summary = "simdgroup_barrier"; + let description = "Calls `@air.simdgroup.barrier(i32 mem_flags, i32 4)`."; + let arguments = (ins AIR_MemFlagsAttr:$mem_flags); + let assemblyFormat = "$mem_flags attr-dict"; +} + +//===----------------------------------------------------------------------===// +// SIMD-group matrices +//===----------------------------------------------------------------------===// + +// A simdgroup_matrix is a vector<64xT>; thread_elements() are its +// elements 0 and 1. +def AIR_SimdgroupMatrixLoadOp : AIR_Op<"simdgroup_matrix_8x8_load", [ + DeclareOpInterfaceMethods]> { + let summary = "simdgroup_load"; + let description = [{ + Calls `@air.simdgroup_matrix_8x8_load.v64.p
`. + `simdgroup_load(m, p, ld, origin, transpose)` passes `extents` (ld, 8) + and `strides` (1, ld), swapped when transposed. + }]; + let arguments = (ins AIR_MatrixPointer:$pointer, + AIR_MatrixVector:$extents, + AIR_MatrixVector:$strides, + AIR_MatrixVector:$origin, + AIR_FastMath:$fastmath); + let results = (outs AIR_Matrix:$result); + let assemblyFormat = [{ + $pointer `,` $extents `,` $strides `,` $origin (`fastmath` `` $fastmath^)? + attr-dict `:` qualified(type($pointer)) `->` type($result) + }]; +} + +def AIR_SimdgroupMatrixStoreOp : AIR_Op<"simdgroup_matrix_8x8_store"> { + let summary = "simdgroup_store"; + let description = [{ + Calls `@air.simdgroup_matrix_8x8_store.v64.p
`, with + the operands of `simdgroup_matrix_8x8_load`. + }]; + let arguments = (ins AIR_Matrix:$value, + AIR_MatrixPointer:$pointer, + AIR_MatrixVector:$extents, + AIR_MatrixVector:$strides, + AIR_MatrixVector:$origin); + let assemblyFormat = [{ + $value `,` $pointer `,` $extents `,` $strides `,` $origin attr-dict `:` + type($value) `,` qualified(type($pointer)) + }]; +} + +def AIR_SimdgroupMatrixMultiplyAccumulateOp + : AIR_Op<"simdgroup_matrix_8x8_multiply_accumulate", [ + DeclareOpInterfaceMethods]> { + let summary = "simdgroup_multiply_accumulate"; + let description = [{ + Calls `@air.simdgroup_matrix_8x8_multiply_accumulate.v64.v64.v64.v64`: + `result = a * b + c`. `simdgroup_multiply` passes a zero `c`. + }]; + let arguments = (ins AIR_Matrix:$a, AIR_Matrix:$b, AIR_Matrix:$c, + AIR_FastMath:$fastmath); + let results = (outs AIR_Matrix:$result); + let assemblyFormat = [{ + $a `,` $b `,` $c (`fastmath` `` $fastmath^)? attr-dict `:` type($a) `,` + type($b) `,` type($c) `->` type($result) + }]; + let hasVerifier = 1; +} + +#endif // MLIR_DIALECTS_AIR_AIROPS_TD diff --git a/mlir/dialects/air/BUILD.bazel b/mlir/dialects/air/BUILD.bazel new file mode 100644 index 00000000..9fb6ed6c --- /dev/null +++ b/mlir/dialects/air/BUILD.bazel @@ -0,0 +1,179 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + ":dialect", + ":headers", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "air", + import_name = "mlir/dialects/air", + main = "air.zig", + tags = ["manual"], + deps = [ + ":capi", + "//mlir", + "//mlir/dialects/arith", + "//mlir/dialects/func", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":air", + "//mlir", + "//mlir/dialects/arith", + "//mlir/dialects/func", + "//mlir/dialects/llvm", + ], +) + +td_library( + name = "td_files", + srcs = [ + "AIRAttributes.td", + "AIRDialect.td", + "AIREnums.td", + "AIROps.td", + ], + includes = ["/"], + deps = [ + "@llvm-project//mlir:ArithOpsInterfacesTdFiles", + "@llvm-project//mlir:ArithOpsTdFiles", + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:LLVMOpsTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "dialect_inc_gen", + tbl_outs = [ + ( + [ + "-gen-dialect-decls", + "-dialect=air", + ], + "AIRDialect.h.inc", + ), + ( + [ + "-gen-dialect-defs", + "-dialect=air", + ], + "AIRDialect.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "AIRDialect.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "enums_inc_gen", + tbl_outs = [ + ( + ["-gen-enum-decls"], + "AIREnums.h.inc", + ), + ( + ["-gen-enum-defs"], + "AIREnums.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "AIREnums.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "attrs_inc_gen", + tbl_outs = [ + ( + [ + "-gen-attrdef-decls", + "-attrdefs-dialect=air", + ], + "AIRAttributes.h.inc", + ), + ( + [ + "-gen-attrdef-defs", + "-attrdefs-dialect=air", + ], + "AIRAttributes.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "AIRAttributes.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "ops_inc_gen", + tbl_outs = [ + ( + ["-gen-op-decls"], + "AIROps.h.inc", + ), + ( + ["-gen-op-defs"], + "AIROps.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "AIROps.td", + deps = [":td_files"], +) + +cc_library( + name = "headers", + hdrs = [ + "AIRAttributes.h", + "AIRDialect.h", + "AIREnums.h", + "AIROps.h", + ], + deps = [ + ":attrs_inc_gen", + ":dialect_inc_gen", + ":enums_inc_gen", + ":ops_inc_gen", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:SideEffectInterfaces", + ], +) + +cc_library( + name = "dialect", + srcs = [ + "AIRAttributes.cpp", + "AIRDialect.cpp", + "AIROps.cpp", + ], + deps = [ + ":headers", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:FunctionInterfaces", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:LLVMDialect", + ], +) diff --git a/mlir/dialects/air/README.md b/mlir/dialects/air/README.md new file mode 100644 index 00000000..c87d45f2 --- /dev/null +++ b/mlir/dialects/air/README.md @@ -0,0 +1,89 @@ +# Metal AIR dialect + +The Metal-specific part of AIR, the LLVM IR Apple's Metal compiler emits (not +AMD's asynchronous AIR dialect). Namespace `air`, C++ namespace +`mlir::metal::air`, C API prefix `mlirMetalAIR`. `dsl/metal` emits it, and +XLA's `__gpu$xla.gpu.air` custom call lowers it, from a vendored copy, to a +metallib. + +## Operations + +Each operation is an `air.*` intrinsic, named as Apple's compiler names it +minus the type suffix its operand and result types determine: + +| Operation | Intrinsic | +|---|---| +| `air.fast_exp %x : vector<2xf32>` | `@air.fast_exp.v2f32` | +| `air.max.u %a, %b : i32` | `@air.max.u.i32` | +| `air.simd_shuffle_xor %x, %lane : f16` | `@air.simd_shuffle_xor.f16` | +| `air.convert.s %x : i32 to f32` | `@air.convert.f.f32.s.i32` | +| `air.simdgroup_matrix_8x8_load %p, ... : !llvm.ptr<3> -> vector<64xbf16>` | `@air.simdgroup_matrix_8x8_load.v64bf16.p3bf16` | +| `air.wg.barrier ` | `@air.wg.barrier(i32 2, i32 1)` | + +- Float math: `exp`, `exp2`, `log`, `log2`, `tanh`, `sqrt`, `rsqrt`, `fabs`, + `ceil`, `floor`, `round`, `trunc`, `sin`, `cos`, `fmax`, `fmin`, `pow`, + `fmod`, `fma`, `clamp` and `dot` on f16 and f32. The `fast_` variants are + f32 only: under fast math (MSL's default) `exp(x)` is `fast_exp`, and + `metal::precise::exp` is `exp`. +- Integer math: `max`, `min`, `clamp` and `abs`, as `.s` and `.u` + overloads. +- SIMD-group functions: `simd_sum`, `simd_max`, `simd_min`, + `simd_prefix_exclusive_sum`, `simd_shuffle`, `simd_shuffle_xor`, + `simd_shuffle_down` and `simd_broadcast` on floats, the integer ones as + `.s` and `.u`, and `simd_any`, `simd_all`. There are no bfloat overloads. +- `convert` between float types, and `convert.s`, `convert.u` between floats + and integers. Apple's compiler uses LLVM casts for scalar f32 to and from + f16 or bf16, and between integers. +- `wg.barrier` and `simdgroup.barrier` (`threadgroup_barrier`, + `simdgroup_barrier`), with `#air.mem_flags`. +- `simdgroup_matrix_8x8_load`, `_store` and `_multiply_accumulate`. A + `simdgroup_matrix` is a `vector<64xT>`, and `thread_elements()` its + elements 0 and 1. + +Float operations carry `fastmath` flags, which become the call's flags (Apple's +compiler marks every float call `fast` under fast math). SIMD-group, barrier +and matrix operations have unknown effects, so they are not speculated or +moved across divergent control flow. Arithmetic, memory and control flow use +the arith, llvm and scf dialects, with pointers as `!llvm.ptr` in AIR's address +spaces: 0 private, 1 device, 2 constant, 3 threadgroup. + +## Kernels + +A kernel is a void `func.func` with `air.kernel`. Each argument carries one +`!air.kernel` descriptor: + +- `air.builtin = #air.builtin`: the position and + size builtins are `uint` or `ushort` scalars or 2- and 3-element vectors, + the index builtins (`thread_index_in_simdgroup`, ...) scalars. +- `air.buffer = #air.buffer<...>` on an `!llvm.ptr`: the `[[buffer(n)]]` or + `[[threadgroup(n)]]` index, the address space, the access (`const` pointers + are `read`), the pointee type with its Metal name, size and alignment, the + size of a constant reference, `noalias` for `__restrict__`, and the fields + of a constant struct (`air.struct_type_info`). Device and constant buffers + share indices; threadgroup buffers have their own. +- `air.arg_name`, optionally, the argument's name. + +`air.fast_math_enable` on the module records that it was compiled with fast +math (`air.compile.fast_math_enable`). + +## Evidence + +Names, signatures, argument descriptors and flags follow what Apple's Metal +compiler 32023.830 (the `@metal_toolchain` repository) emits for MSL probes and +for XLA's Metal kernels, with `metal -std=metal4.0 -S -emit-llvm` (fast math, +-O2, as XLA compiles them). Only the overloads it emits are accepted. + +## Build and test + +```sh +bazel build //mlir/dialects/air //mlir/dialects/air:capi +bazel test //mlir/dialects/air:test +``` + +The Zig library is `:air`, the C++ dialect `:dialect` and its C API `:capi`. +`registerDialect(registry)` registers AIR, which loads the LLVM dialect; +`kernel` builds a `func.func`, so callers also register `func`. Attributes are +built through the C API (`BuiltinAttr`, `MemFlagsAttr`, `StructFieldAttr`, +`BufferAttr`), kernel arguments through `builtinArgumentAttr` and +`bufferArgumentAttr`, and every operation has a builder named after it +(`fast_exp`, `max_u`, `wg_barrier`, `simdgroup_matrix_8x8_load`). diff --git a/mlir/dialects/air/air.zig b/mlir/dialects/air/air.zig new file mode 100644 index 00000000..62195b35 --- /dev/null +++ b/mlir/dialects/air/air.zig @@ -0,0 +1,536 @@ +//! Metal AIR intrinsics and kernel ABI annotations. A builder emits the +//! `air.*` intrinsic it names, minus the type suffix its types select: +//! `fast_exp` on f32 is `@air.fast_exp.f32`, `max_u` on `vector<2xi32>` is +//! `@air.max.u.v2i32`. Arithmetic, memory and control flow use the arith, +//! llvm and scf dialects, with pointers as `!llvm.ptr` in AIR's address spaces. +const std = @import("std"); + +const arith = @import("mlir/dialects/arith"); +const c = @import("c"); +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("air"); +} + +pub const Builtin = enum(u32) { + thread_position_in_grid, + threadgroup_position_in_grid, + thread_position_in_threadgroup, + threads_per_threadgroup, + threadgroups_per_grid, + threads_per_grid, + thread_index_in_simdgroup, + simdgroup_index_in_threadgroup, + threads_per_simdgroup, + simdgroups_per_threadgroup, + thread_index_in_threadgroup, +}; + +pub const AddressSpace = enum(u32) { device = 1, constant = 2, threadgroup = 3 }; + +pub const Access = enum(u32) { read, read_write }; + +pub const MemFlags = packed struct(u32) { + device: bool = false, + threadgroup: bool = false, + texture: bool = false, + _padding: u29 = 0, + + pub const none: MemFlags = .{}; +}; + +/// `#air.builtin`: Kernel argument builtin +pub const BuiltinAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAMetalAIRBuiltin; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { value: Builtin }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirMetalAIRBuiltinAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn value(self: *const Self) Builtin { + return @fromBackingInt(@intCast(c.mlirMetalAIRBuiltinAttrGetValue(self.ptr()))); + } +}; + +/// `#air.mem_flags`: Memory a barrier orders +pub const MemFlagsAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAMetalAIRMemFlags; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { value: MemFlags }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirMetalAIRMemFlagsAttrGet(ctx.ptr(), @bitCast(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn value(self: *const Self) MemFlags { + return @bitCast(c.mlirMetalAIRMemFlagsAttrGetValue(self.ptr())); + } +}; + +/// `#air.struct_field`: Field of a constant buffer struct +pub const StructFieldAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAMetalAIRStructField; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + offset: u64, + size: u64, + /// The array length, zero for a non-array field. + count: u64 = 0, + typeName: []const u8, + name: []const u8, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirMetalAIRStructFieldAttrGet(ctx.ptr(), args.offset, args.size, args.count, mlir.stringRef(args.typeName), mlir.stringRef(args.name)).ptr orelse return error.InvalidMlir); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn offset(self: *const Self) u64 { + return c.mlirMetalAIRStructFieldAttrGetOffset(self.ptr()); + } + + pub fn size(self: *const Self) u64 { + return c.mlirMetalAIRStructFieldAttrGetSize(self.ptr()); + } + + pub fn count(self: *const Self) u64 { + return c.mlirMetalAIRStructFieldAttrGetCount(self.ptr()); + } + + pub fn typeName(self: *const Self) []const u8 { + return mlir.string(c.mlirMetalAIRStructFieldAttrGetTypeName(self.ptr())); + } + + pub fn name(self: *const Self) []const u8 { + return mlir.string(c.mlirMetalAIRStructFieldAttrGetName(self.ptr())); + } +}; + +/// `#air.buffer`: Kernel buffer argument +pub const BufferAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAMetalAIRBuffer; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + locationIndex: u32, + addressSpace: AddressSpace, + access: Access, + elementType: *const mlir.Type, + typeName: []const u8, + typeSize: u64, + typeAlignment: u64, + /// The size of a constant reference; zero omits it. + bufferSize: u64 = 0, + noalias_: bool = false, + fields: []const *const StructFieldAttr = &.{}, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirMetalAIRBufferAttrGet( + ctx.ptr(), + args.locationIndex, + @backingInt(args.addressSpace), + @backingInt(args.access), + args.elementType.ptr(), + mlir.stringRef(args.typeName), + args.typeSize, + args.typeAlignment, + args.bufferSize, + args.noalias_, + @intCast(args.fields.len), + @ptrCast(args.fields.ptr), + ).ptr orelse return error.InvalidMlir); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn locationIndex(self: *const Self) u32 { + return c.mlirMetalAIRBufferAttrGetLocationIndex(self.ptr()); + } + + pub fn addressSpace(self: *const Self) AddressSpace { + return @fromBackingInt(@intCast(c.mlirMetalAIRBufferAttrGetAddressSpace(self.ptr()))); + } + + pub fn access(self: *const Self) Access { + return @fromBackingInt(@intCast(c.mlirMetalAIRBufferAttrGetAccess(self.ptr()))); + } + + pub fn elementType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirMetalAIRBufferAttrGetElementType(self.ptr()).ptr); + } + + pub fn typeName(self: *const Self) []const u8 { + return mlir.string(c.mlirMetalAIRBufferAttrGetTypeName(self.ptr())); + } + + pub fn typeSize(self: *const Self) u64 { + return c.mlirMetalAIRBufferAttrGetTypeSize(self.ptr()); + } + + pub fn typeAlignment(self: *const Self) u64 { + return c.mlirMetalAIRBufferAttrGetTypeAlignment(self.ptr()); + } + + pub fn bufferSize(self: *const Self) u64 { + return c.mlirMetalAIRBufferAttrGetBufferSize(self.ptr()); + } + + pub fn noalias_(self: *const Self) bool { + return c.mlirMetalAIRBufferAttrGetNoalias(self.ptr()); + } + + pub fn numFields(self: *const Self) usize { + return @intCast(c.mlirMetalAIRBufferAttrGetNumFields(self.ptr())); + } + + pub fn field(self: *const Self, i: usize) *const StructFieldAttr { + return @ptrCast(c.mlirMetalAIRBufferAttrGetField(self.ptr(), @intCast(i)).ptr); + } +}; + +pub const ArgumentOpts = struct { + /// `air.arg_name`, the argument's name in reflection. + name: ?[]const u8 = null, +}; + +/// The argument dictionary of a kernel argument bound to `builtin`. +pub fn builtinArgumentAttr(ctx: *mlir.Context, builtin: Builtin, opts: ArgumentOpts) mlir.Error!*const mlir.Attribute { + return argumentAttr(ctx, .named(ctx, "air.builtin", (try BuiltinAttr.get(ctx, .{ .value = builtin })).attribute()), opts); +} + +/// The argument dictionary of a kernel buffer argument. +pub fn bufferArgumentAttr(ctx: *mlir.Context, args: BufferAttr.InitArgs, opts: ArgumentOpts) mlir.Error!*const mlir.Attribute { + return argumentAttr(ctx, .named(ctx, "air.buffer", (try BufferAttr.get(ctx, args)).attribute()), opts); +} + +fn argumentAttr(ctx: *mlir.Context, annotation: mlir.NamedAttribute, opts: ArgumentOpts) *const mlir.Attribute { + if (opts.name) |name| return .dict(ctx, &.{ annotation, .named(ctx, "air.arg_name", .string(ctx, name)) }); + return .dict(ctx, &.{annotation}); +} + +/// Marks `module` as compiled with fast math (`air.compile.fast_math_enable`). +pub fn enableFastMath(ctx: *mlir.Context, module: *mlir.Operation) void { + module.setAttributeByName("air.fast_math_enable", .unit(ctx)); +} + +pub const KernelArgs = struct { + name: []const u8, + block: *mlir.Block, + args_attributes: []const *const mlir.Attribute, + location: *const mlir.Location, + verify: bool = true, +}; + +/// A `func.func` with `air.kernel`, one argument dictionary per block +/// argument. Takes ownership of `block`; set `verify = false` while the body +/// is incomplete. +pub fn kernel(ctx: *mlir.Context, args: KernelArgs) *mlir.Operation { + std.debug.assert(args.args_attributes.len == args.block.numArguments()); + return func.func(ctx, .{ + .name = args.name, + .block = args.block, + .args_attributes = args.args_attributes, + .results = &.{}, + .extra_attributes = &.{.named(ctx, "air.kernel", .unit(ctx))}, + .location = args.location, + .verify = args.verify, + }); +} + +pub const FloatOpts = struct { + fastmath: arith.FastMathFlags = .none, +}; + +/// The `fastmath` attribute of a float op, stored in the caller's `buf`. +fn fastmathAttrs(buf: *[1]mlir.NamedAttribute, ctx: *mlir.Context, fastmath: arith.FastMathFlags) []const mlir.NamedAttribute { + if (fastmath == arith.FastMathFlags.none) return &.{}; + const attr = arith.FastMathAttr.get(ctx, .{ .value = fastmath }) catch unreachable; + buf[0] = .named(ctx, "fastmath", attr.attribute()); + return buf; +} + +fn floatUnary(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, x: *const mlir.Value, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{x} }, + .results = .{ .flat = &.{x.type_()} }, + .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), + .location = location, + }); + } + }; +} + +fn floatBinary(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{ x, y } }, + .results = .{ .flat = &.{x.type_()} }, + .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), + .location = location, + }); + } + }; +} + +fn floatTernary(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, z: *const mlir.Value, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{ x, y, z } }, + .results = .{ .flat = &.{x.type_()} }, + .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), + .location = location, + }); + } + }; +} + +fn intUnary(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, x: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{x} }, + .results = .{ .flat = &.{x.type_()} }, + .location = location, + }); + } + }; +} + +fn intBinary(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{ x, y } }, + .results = .{ .flat = &.{x.type_()} }, + .location = location, + }); + } + }; +} + +fn intTernary(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, z: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{ x, y, z } }, + .results = .{ .flat = &.{x.type_()} }, + .location = location, + }); + } + }; +} + +fn floatShuffle(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, data: *const mlir.Value, lane: *const mlir.Value, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{ data, lane } }, + .results = .{ .flat = &.{data.type_()} }, + .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), + .location = location, + }); + } + }; +} + +fn intShuffle(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, data: *const mlir.Value, lane: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{ data, lane } }, + .results = .{ .flat = &.{data.type_()} }, + .location = location, + }); + } + }; +} + +fn convertOp(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, x: *const mlir.Value, result_type: *const mlir.Type, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{x} }, + .results = .{ .flat = &.{result_type} }, + .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), + .location = location, + }); + } + }; +} + +fn barrier(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, mem_flags: MemFlags, location: *const mlir.Location) *mlir.Operation { + const flags = MemFlagsAttr.get(ctx, .{ .value = mem_flags }) catch unreachable; + return mlir.Operation.make(ctx, name, .{ + .attributes = &.{.named(ctx, "mem_flags", flags.attribute())}, + .location = location, + }); + } + }; +} + +pub const exp = floatUnary("air.exp").call; +pub const fast_exp = floatUnary("air.fast_exp").call; +pub const exp2 = floatUnary("air.exp2").call; +pub const fast_exp2 = floatUnary("air.fast_exp2").call; +pub const log = floatUnary("air.log").call; +pub const fast_log = floatUnary("air.fast_log").call; +pub const log2 = floatUnary("air.log2").call; +pub const fast_log2 = floatUnary("air.fast_log2").call; +pub const tanh = floatUnary("air.tanh").call; +pub const fast_tanh = floatUnary("air.fast_tanh").call; +pub const sqrt = floatUnary("air.sqrt").call; +pub const fast_sqrt = floatUnary("air.fast_sqrt").call; +pub const rsqrt = floatUnary("air.rsqrt").call; +pub const fast_rsqrt = floatUnary("air.fast_rsqrt").call; +pub const fabs = floatUnary("air.fabs").call; +pub const fast_fabs = floatUnary("air.fast_fabs").call; +pub const ceil = floatUnary("air.ceil").call; +pub const fast_ceil = floatUnary("air.fast_ceil").call; +pub const floor = floatUnary("air.floor").call; +pub const fast_floor = floatUnary("air.fast_floor").call; +pub const round = floatUnary("air.round").call; +pub const fast_round = floatUnary("air.fast_round").call; +pub const trunc = floatUnary("air.trunc").call; +pub const fast_trunc = floatUnary("air.fast_trunc").call; +pub const sin = floatUnary("air.sin").call; +pub const fast_sin = floatUnary("air.fast_sin").call; +pub const cos = floatUnary("air.cos").call; +pub const fast_cos = floatUnary("air.fast_cos").call; +pub const fmax = floatBinary("air.fmax").call; +pub const fast_fmax = floatBinary("air.fast_fmax").call; +pub const fmin = floatBinary("air.fmin").call; +pub const fast_fmin = floatBinary("air.fast_fmin").call; +pub const pow = floatBinary("air.pow").call; +pub const fast_pow = floatBinary("air.fast_pow").call; +pub const fmod = floatBinary("air.fmod").call; +pub const fast_fmod = floatBinary("air.fast_fmod").call; +pub const fma = floatTernary("air.fma").call; +pub const clamp = floatTernary("air.clamp").call; +pub const fast_clamp = floatTernary("air.fast_clamp").call; +pub const abs_s = intUnary("air.abs.s").call; +pub const max_s = intBinary("air.max.s").call; +pub const max_u = intBinary("air.max.u").call; +pub const min_s = intBinary("air.min.s").call; +pub const min_u = intBinary("air.min.u").call; +pub const clamp_s = intTernary("air.clamp.s").call; +pub const clamp_u = intTernary("air.clamp.u").call; +pub const simd_sum = floatUnary("air.simd_sum").call; +pub const simd_sum_s = intUnary("air.simd_sum.s").call; +pub const simd_sum_u = intUnary("air.simd_sum.u").call; +pub const simd_max = floatUnary("air.simd_max").call; +pub const simd_max_s = intUnary("air.simd_max.s").call; +pub const simd_max_u = intUnary("air.simd_max.u").call; +pub const simd_min = floatUnary("air.simd_min").call; +pub const simd_min_s = intUnary("air.simd_min.s").call; +pub const simd_min_u = intUnary("air.simd_min.u").call; +pub const simd_prefix_exclusive_sum = floatUnary("air.simd_prefix_exclusive_sum").call; +pub const simd_shuffle = floatShuffle("air.simd_shuffle").call; +pub const simd_shuffle_s = intShuffle("air.simd_shuffle.s").call; +pub const simd_shuffle_u = intShuffle("air.simd_shuffle.u").call; +pub const simd_shuffle_xor = floatShuffle("air.simd_shuffle_xor").call; +pub const simd_shuffle_xor_s = intShuffle("air.simd_shuffle_xor.s").call; +pub const simd_shuffle_xor_u = intShuffle("air.simd_shuffle_xor.u").call; +pub const simd_shuffle_down = floatShuffle("air.simd_shuffle_down").call; +pub const simd_shuffle_down_s = intShuffle("air.simd_shuffle_down.s").call; +pub const simd_shuffle_down_u = intShuffle("air.simd_shuffle_down.u").call; +pub const simd_broadcast = floatShuffle("air.simd_broadcast").call; +pub const simd_broadcast_s = intShuffle("air.simd_broadcast.s").call; +pub const simd_broadcast_u = intShuffle("air.simd_broadcast.u").call; +pub const simd_any = intUnary("air.simd_any").call; +pub const simd_all = intUnary("air.simd_all").call; +pub const convert = convertOp("air.convert").call; +pub const convert_s = convertOp("air.convert.s").call; +pub const convert_u = convertOp("air.convert.u").call; +pub const wg_barrier = barrier("air.wg.barrier").call; +pub const simdgroup_barrier = barrier("air.simdgroup.barrier").call; + +pub fn dot(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + return mlir.Operation.make(ctx, "air.dot", .{ + .operands = .{ .flat = &.{ x, y } }, + .results = .{ .flat = &.{x.type_().isA(mlir.ShapedType).?.elementType()} }, + .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), + .location = location, + }); +} + +/// `simdgroup_load`: `extents`, `strides` and `origin` are `vector<2xi64>`. +pub fn simdgroup_matrix_8x8_load(ctx: *mlir.Context, pointer: *const mlir.Value, extents: *const mlir.Value, strides: *const mlir.Value, origin: *const mlir.Value, result_type: *const mlir.Type, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + return mlir.Operation.make(ctx, "air.simdgroup_matrix_8x8_load", .{ + .operands = .{ .flat = &.{ pointer, extents, strides, origin } }, + .results = .{ .flat = &.{result_type} }, + .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), + .location = location, + }); +} + +pub fn simdgroup_matrix_8x8_store(ctx: *mlir.Context, value: *const mlir.Value, pointer: *const mlir.Value, extents: *const mlir.Value, strides: *const mlir.Value, origin: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "air.simdgroup_matrix_8x8_store", .{ + .operands = .{ .flat = &.{ value, pointer, extents, strides, origin } }, + .location = location, + }); +} + +/// `simdgroup_multiply_accumulate`: `a * b + c`. +pub fn simdgroup_matrix_8x8_multiply_accumulate(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, c_: *const mlir.Value, result_type: *const mlir.Type, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + return mlir.Operation.make(ctx, "air.simdgroup_matrix_8x8_multiply_accumulate", .{ + .operands = .{ .flat = &.{ a, b, c_ } }, + .results = .{ .flat = &.{result_type} }, + .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), + .location = location, + }); +} diff --git a/mlir/dialects/air/capi.cc b/mlir/dialects/air/capi.cc new file mode 100644 index 00000000..64df5d20 --- /dev/null +++ b/mlir/dialects/air/capi.cc @@ -0,0 +1,189 @@ +//===- capi.cc - C API for the Metal AIR dialect --------------------------===// + +#include "capi.h" + +#include "mlir/dialects/air/AIRAttributes.h" +#include "mlir/dialects/air/AIRDialect.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" +#include "mlir/CAPI/Support.h" +#include "mlir/IR/Location.h" +#include "llvm/ADT/SmallVector.h" + +#include + +using namespace mlir; +using namespace mlir::metal::air; + +// Registration + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(MetalAIR, air, AIRDialect) + +static MLIRContext *loadAIR(MlirContext context) { + MLIRContext *ctx = unwrap(context); + if (ctx) + ctx->getOrLoadDialect(); + return ctx; +} + +static MlirAttribute invalid(MLIRContext *ctx, const Twine &message) { + emitError(UnknownLoc::get(ctx), message); + return {nullptr}; +} + +// Attributes + +bool mlirAttributeIsAMetalAIRBuiltin(MlirAttribute attr) { + return isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirMetalAIRBuiltinAttrGet(MlirContext context, uint32_t value) { + MLIRContext *ctx = loadAIR(context); + if (!ctx) + return {nullptr}; + std::optional builtin = symbolizeBuiltin(value); + if (!builtin) + return invalid(ctx, "unknown AIR builtin " + Twine(value)); + return wrap(BuiltinAttr::get(ctx, *builtin)); +} + +uint32_t mlirMetalAIRBuiltinAttrGetValue(MlirAttribute attr) { + return static_cast(cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAMetalAIRMemFlags(MlirAttribute attr) { + return isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirMetalAIRMemFlagsAttrGet(MlirContext context, uint32_t value) { + MLIRContext *ctx = loadAIR(context); + if (!ctx) + return {nullptr}; + std::optional flags = symbolizeMemFlags(value); + if (!flags) + return invalid(ctx, "unknown AIR memory flags " + Twine(value)); + return wrap(MemFlagsAttr::get(ctx, *flags)); +} + +uint32_t mlirMetalAIRMemFlagsAttrGetValue(MlirAttribute attr) { + return static_cast(cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAMetalAIRStructField(MlirAttribute attr) { + return isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirMetalAIRStructFieldAttrGet(MlirContext context, + uint64_t offset, uint64_t size, + uint64_t count, + MlirStringRef typeName, + MlirStringRef name) { + MLIRContext *ctx = loadAIR(context); + if (!ctx) + return {nullptr}; + return wrap(StructFieldAttr::getChecked(UnknownLoc::get(ctx), ctx, offset, + size, count, unwrap(typeName), + unwrap(name))); +} + +uint64_t mlirMetalAIRStructFieldAttrGetOffset(MlirAttribute attr) { + return cast(unwrap(attr)).getOffset(); +} + +uint64_t mlirMetalAIRStructFieldAttrGetSize(MlirAttribute attr) { + return cast(unwrap(attr)).getSize(); +} + +uint64_t mlirMetalAIRStructFieldAttrGetCount(MlirAttribute attr) { + return cast(unwrap(attr)).getCount(); +} + +MlirStringRef mlirMetalAIRStructFieldAttrGetTypeName(MlirAttribute attr) { + return wrap(cast(unwrap(attr)).getTypeName()); +} + +MlirStringRef mlirMetalAIRStructFieldAttrGetName(MlirAttribute attr) { + return wrap(cast(unwrap(attr)).getName()); +} + +bool mlirAttributeIsAMetalAIRBuffer(MlirAttribute attr) { + return isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirMetalAIRBufferAttrGet( + MlirContext context, uint32_t locationIndex, uint32_t addressSpace, + uint32_t access, MlirType elementType, MlirStringRef typeName, + uint64_t typeSize, uint64_t typeAlignment, uint64_t bufferSize, + bool noalias, intptr_t numFields, MlirAttribute const *fields) { + MLIRContext *ctx = loadAIR(context); + if (!ctx) + return {nullptr}; + std::optional space = symbolizeAddressSpace(addressSpace); + std::optional mode = symbolizeAccess(access); + if (!space || !mode) + return invalid(ctx, "unknown AIR buffer address space or access"); + Type element = unwrap(elementType); + if (!element || element.getContext() != ctx) + return invalid(ctx, + "buffer element type must belong to the attribute context"); + if (numFields < 0 || (numFields > 0 && !fields)) + return invalid(ctx, "invalid AIR struct fields"); + SmallVector structFields; + for (intptr_t i = 0; i < numFields; ++i) { + auto field = dyn_cast_or_null(unwrap(fields[i])); + if (!field || field.getContext() != ctx) + return invalid(ctx, "AIR struct fields must be #air.struct_field of " + "the attribute context"); + structFields.push_back(field); + } + return wrap(BufferAttr::getChecked( + UnknownLoc::get(ctx), ctx, locationIndex, *space, *mode, element, + unwrap(typeName), typeSize, typeAlignment, bufferSize, noalias, + ArrayRef(structFields))); +} + +uint32_t mlirMetalAIRBufferAttrGetLocationIndex(MlirAttribute attr) { + return cast(unwrap(attr)).getLocationIndex(); +} + +uint32_t mlirMetalAIRBufferAttrGetAddressSpace(MlirAttribute attr) { + return static_cast( + cast(unwrap(attr)).getAddressSpace()); +} + +uint32_t mlirMetalAIRBufferAttrGetAccess(MlirAttribute attr) { + return static_cast(cast(unwrap(attr)).getAccess()); +} + +MlirType mlirMetalAIRBufferAttrGetElementType(MlirAttribute attr) { + return wrap(cast(unwrap(attr)).getElementType()); +} + +MlirStringRef mlirMetalAIRBufferAttrGetTypeName(MlirAttribute attr) { + return wrap(cast(unwrap(attr)).getTypeName()); +} + +uint64_t mlirMetalAIRBufferAttrGetTypeSize(MlirAttribute attr) { + return cast(unwrap(attr)).getTypeSize(); +} + +uint64_t mlirMetalAIRBufferAttrGetTypeAlignment(MlirAttribute attr) { + return cast(unwrap(attr)).getTypeAlignment(); +} + +uint64_t mlirMetalAIRBufferAttrGetBufferSize(MlirAttribute attr) { + return cast(unwrap(attr)).getBufferSize(); +} + +bool mlirMetalAIRBufferAttrGetNoalias(MlirAttribute attr) { + return cast(unwrap(attr)).getNoalias(); +} + +intptr_t mlirMetalAIRBufferAttrGetNumFields(MlirAttribute attr) { + return cast(unwrap(attr)).getFields().size(); +} + +MlirAttribute mlirMetalAIRBufferAttrGetField(MlirAttribute attr, intptr_t pos) { + return wrap(cast(unwrap(attr)).getFields()[pos]); +} diff --git a/mlir/dialects/air/capi.h b/mlir/dialects/air/capi.h new file mode 100644 index 00000000..f91b9f64 --- /dev/null +++ b/mlir/dialects/air/capi.h @@ -0,0 +1,81 @@ +//===- capi.h - C API for the Metal AIR dialect -------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_AIR_CAPI_H +#define MLIR_DIALECTS_AIR_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Registration + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(MetalAIR, air); + +// Attributes. Invalid parameters emit a diagnostic and return a null handle; +// typed getters require a handle of their attribute kind. + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAMetalAIRBuiltin(MlirAttribute attr); +/// `value` is a `mlir::metal::air::Builtin`. +MLIR_CAPI_EXPORTED MlirAttribute mlirMetalAIRBuiltinAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirMetalAIRBuiltinAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAMetalAIRMemFlags(MlirAttribute attr); +/// `value` is a `mlir::metal::air::MemFlags` bit set. +MLIR_CAPI_EXPORTED MlirAttribute +mlirMetalAIRMemFlagsAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirMetalAIRMemFlagsAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAMetalAIRStructField(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirMetalAIRStructFieldAttrGet( + MlirContext context, uint64_t offset, uint64_t size, uint64_t count, + MlirStringRef typeName, MlirStringRef name); +MLIR_CAPI_EXPORTED uint64_t +mlirMetalAIRStructFieldAttrGetOffset(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint64_t +mlirMetalAIRStructFieldAttrGetSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint64_t +mlirMetalAIRStructFieldAttrGetCount(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirMetalAIRStructFieldAttrGetTypeName(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirMetalAIRStructFieldAttrGetName(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAMetalAIRBuffer(MlirAttribute attr); +/// `addressSpace` is a `mlir::metal::air::AddressSpace` and `access` a +/// `mlir::metal::air::Access`. Sizes are bytes; a zero `bufferSize` omits it. +/// `elementType` and `fields` must belong to `context`. +MLIR_CAPI_EXPORTED MlirAttribute mlirMetalAIRBufferAttrGet( + MlirContext context, uint32_t locationIndex, uint32_t addressSpace, + uint32_t access, MlirType elementType, MlirStringRef typeName, + uint64_t typeSize, uint64_t typeAlignment, uint64_t bufferSize, + bool noalias, intptr_t numFields, MlirAttribute const *fields); +MLIR_CAPI_EXPORTED uint32_t +mlirMetalAIRBufferAttrGetLocationIndex(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t +mlirMetalAIRBufferAttrGetAddressSpace(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t mlirMetalAIRBufferAttrGetAccess(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirType +mlirMetalAIRBufferAttrGetElementType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirMetalAIRBufferAttrGetTypeName(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint64_t +mlirMetalAIRBufferAttrGetTypeSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint64_t +mlirMetalAIRBufferAttrGetTypeAlignment(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint64_t +mlirMetalAIRBufferAttrGetBufferSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirMetalAIRBufferAttrGetNoalias(MlirAttribute attr); +MLIR_CAPI_EXPORTED intptr_t +mlirMetalAIRBufferAttrGetNumFields(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirMetalAIRBufferAttrGetField(MlirAttribute attr, intptr_t pos); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_AIR_CAPI_H diff --git a/mlir/dialects/air/test.zig b/mlir/dialects/air/test.zig new file mode 100644 index 00000000..04ef2dfb --- /dev/null +++ b/mlir/dialects/air/test.zig @@ -0,0 +1,414 @@ +const std = @import("std"); + +const air = @import("mlir/dialects/air"); +const arith = @import("mlir/dialects/arith"); +const func = @import("mlir/dialects/func"); +const llvm = @import("mlir/dialects/llvm"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "air.abs.s", + "air.ceil", + "air.clamp", + "air.clamp.s", + "air.clamp.u", + "air.convert", + "air.convert.s", + "air.convert.u", + "air.cos", + "air.dot", + "air.exp", + "air.exp2", + "air.fabs", + "air.fast_ceil", + "air.fast_clamp", + "air.fast_cos", + "air.fast_exp", + "air.fast_exp2", + "air.fast_fabs", + "air.fast_floor", + "air.fast_fmax", + "air.fast_fmin", + "air.fast_fmod", + "air.fast_log", + "air.fast_log2", + "air.fast_pow", + "air.fast_round", + "air.fast_rsqrt", + "air.fast_sin", + "air.fast_sqrt", + "air.fast_tanh", + "air.fast_trunc", + "air.floor", + "air.fma", + "air.fmax", + "air.fmin", + "air.fmod", + "air.log", + "air.log2", + "air.max.s", + "air.max.u", + "air.min.s", + "air.min.u", + "air.pow", + "air.round", + "air.rsqrt", + "air.simd_all", + "air.simd_any", + "air.simd_broadcast", + "air.simd_broadcast.s", + "air.simd_broadcast.u", + "air.simd_max", + "air.simd_max.s", + "air.simd_max.u", + "air.simd_min", + "air.simd_min.s", + "air.simd_min.u", + "air.simd_prefix_exclusive_sum", + "air.simd_shuffle", + "air.simd_shuffle.s", + "air.simd_shuffle.u", + "air.simd_shuffle_down", + "air.simd_shuffle_down.s", + "air.simd_shuffle_down.u", + "air.simd_shuffle_xor", + "air.simd_shuffle_xor.s", + "air.simd_shuffle_xor.u", + "air.simd_sum", + "air.simd_sum.s", + "air.simd_sum.u", + "air.simdgroup.barrier", + "air.simdgroup_matrix_8x8_load", + "air.simdgroup_matrix_8x8_multiply_accumulate", + "air.simdgroup_matrix_8x8_store", + "air.sin", + "air.sqrt", + "air.tanh", + "air.trunc", + "air.wg.barrier", +}; + +test { + std.testing.refAllDecls(air); + inline for (.{ air.BuiltinAttr, air.MemFlagsAttr, air.StructFieldAttr, air.BufferAttr }) |T| std.testing.refAllDecls(T); +} + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + inline for (.{ air, arith, func, llvm }) |dialect| dialect.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectRoundTrip(ctx: *mlir.Context, value: anytype) !void { + var writer: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer writer.deinit(); + try writer.writer.print("{f}", .{value}); + try std.testing.expect((try mlir.Attribute.parse(ctx, writer.written())).eql(value.attribute())); +} + +test "every bound operation is registered" { + const ctx = try testContext(); + defer ctx.deinit(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "AIR attributes use the C API" { + const ctx = try testContext(); + defer ctx.deinit(); + inline for (std.meta.tags(air.Builtin)) |builtin| { + const attr = try air.BuiltinAttr.get(ctx, .{ .value = builtin }); + try std.testing.expectEqual(builtin, attr.value()); + try expectRoundTrip(ctx, attr); + } + for ([_]air.MemFlags{ .none, .{ .device = true }, .{ .threadgroup = true }, .{ .device = true, .threadgroup = true, .texture = true } }) |flags| { + const attr = try air.MemFlagsAttr.get(ctx, .{ .value = flags }); + try std.testing.expectEqual(flags, attr.value()); + try expectRoundTrip(ctx, attr); + } + try std.testing.expectError(error.InvalidMlir, air.MemFlagsAttr.get(ctx, .{ .value = .{ ._padding = 1 } })); + + const field = try air.StructFieldAttr.get(ctx, .{ .offset = 4, .size = 4, .count = 3, .typeName = "float", .name = "s" }); + try std.testing.expectEqual(4, field.offset()); + try std.testing.expectEqual(4, field.size()); + try std.testing.expectEqual(3, field.count()); + try std.testing.expectEqualStrings("float", field.typeName()); + try std.testing.expectEqualStrings("s", field.name()); + try expectRoundTrip(ctx, field); + try std.testing.expectError(error.InvalidMlir, air.StructFieldAttr.get(ctx, .{ .offset = 0, .size = 0, .typeName = "int", .name = "n" })); + + const i32_type: *const mlir.Type = .int(ctx, .i32); + const f32_type: *const mlir.Type = .float(ctx, .f32); + const device: air.BufferAttr.InitArgs = .{ + .locationIndex = 1, + .addressSpace = .device, + .access = .read, + .elementType = .vector(&.{4}, .float(ctx, .bf16)), + .typeName = "bfloat4", + .typeSize = 8, + .typeAlignment = 8, + .noalias_ = true, + }; + const buffer = try air.BufferAttr.get(ctx, device); + try std.testing.expectEqual(1, buffer.locationIndex()); + try std.testing.expectEqual(air.AddressSpace.device, buffer.addressSpace()); + try std.testing.expectEqual(air.Access.read, buffer.access()); + try std.testing.expect(buffer.elementType().eql(device.elementType)); + try std.testing.expectEqualStrings("bfloat4", buffer.typeName()); + try std.testing.expectEqual(8, buffer.typeSize()); + try std.testing.expectEqual(8, buffer.typeAlignment()); + try std.testing.expectEqual(0, buffer.bufferSize()); + try std.testing.expect(buffer.noalias_()); + try std.testing.expectEqual(0, buffer.numFields()); + try expectRoundTrip(ctx, buffer); + + // `struct A { int n; float s[3]; }`, as Apple's compiler describes it. + const fields = [_]*const air.StructFieldAttr{ + try air.StructFieldAttr.get(ctx, .{ .offset = 0, .size = 4, .typeName = "int", .name = "n" }), + field, + }; + const constant: air.BufferAttr.InitArgs = .{ + .locationIndex = 0, + .addressSpace = .constant, + .access = .read, + .elementType = llvm.structType(ctx, &.{ i32_type, llvm.arrayType(f32_type, 3) }), + .typeName = "A", + .typeSize = 16, + .typeAlignment = 4, + .bufferSize = 16, + .fields = &fields, + }; + const structured = try air.BufferAttr.get(ctx, constant); + try std.testing.expectEqual(2, structured.numFields()); + try std.testing.expect(structured.field(1).eql(field)); + try expectRoundTrip(ctx, structured); + + var invalid = device; + invalid.typeSize = 16; + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); + invalid = device; + invalid.addressSpace = .constant; + invalid.access = .read_write; + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); + invalid = device; + invalid.addressSpace = .threadgroup; + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); + invalid = device; + invalid.bufferSize = 8; + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); + invalid = device; + invalid.fields = fields[0..1]; + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); + invalid = constant; + invalid.fields = fields[0..1]; + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); + invalid = constant; + invalid.addressSpace = .device; + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); + + const other = try testContext(); + defer other.deinit(); + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(other, device)); +} + +test "all AIR operations construct and verify" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc: *const mlir.Location = .unknown(ctx); + const f32x4: *const mlir.Type = .vector(&.{4}, .float(ctx, .f32)); + const i32x2: *const mlir.Type = .vector(&.{2}, .int(ctx, .i32)); + const i64x2: *const mlir.Type = .vector(&.{2}, .int(ctx, .i64)); + const block = mlir.Block.init( + &.{ .float(ctx, .f32), .float(ctx, .f16), f32x4, .int(ctx, .i32), i32x2, .int(ctx, .i16), .int(ctx, .i1), llvm.pointerType(ctx, air.AddressSpace.threadgroup), i64x2, .vector(&.{64}, .float(ctx, .f16)), .vector(&.{64}, .float(ctx, .f32)), .vector(&.{4}, .float(ctx, .bf16)) }, + &.{ loc, loc, loc, loc, loc, loc, loc, loc, loc, loc, loc, loc }, + ); + defer block.deinit(); + const f = block.argument(0); + const h = block.argument(1); + const v = block.argument(2); + const i = block.argument(3); + const iv = block.argument(4); + const lane = block.argument(5); + const b = block.argument(6); + const fast: air.FloatOpts = .{ .fastmath = .fast }; + var ops: std.ArrayList(*mlir.Operation) = .empty; + defer ops.deinit(std.testing.allocator); + inline for (.{ air.exp, air.exp2, air.log, air.log2, air.tanh, air.sqrt, air.rsqrt, air.fabs, air.ceil, air.floor, air.round, air.trunc, air.sin, air.cos, air.simd_sum, air.simd_max, air.simd_min, air.simd_prefix_exclusive_sum }) |op| { + try ops.append(std.testing.allocator, op(ctx, h, .{}, loc).appendTo(block)); + try ops.append(std.testing.allocator, op(ctx, v, fast, loc).appendTo(block)); + } + inline for (.{ air.fast_exp, air.fast_exp2, air.fast_log, air.fast_log2, air.fast_tanh, air.fast_sqrt, air.fast_rsqrt, air.fast_fabs, air.fast_ceil, air.fast_floor, air.fast_round, air.fast_trunc, air.fast_sin, air.fast_cos }) |op| { + try ops.append(std.testing.allocator, op(ctx, f, fast, loc).appendTo(block)); + try ops.append(std.testing.allocator, op(ctx, v, fast, loc).appendTo(block)); + } + inline for (.{ air.fmax, air.fmin, air.pow, air.fmod, air.fast_fmax, air.fast_fmin, air.fast_pow, air.fast_fmod }) |op| { + try ops.append(std.testing.allocator, op(ctx, f, f, fast, loc).appendTo(block)); + } + inline for (.{ air.fma, air.clamp, air.fast_clamp }) |op| { + try ops.append(std.testing.allocator, op(ctx, v, v, v, fast, loc).appendTo(block)); + } + try ops.append(std.testing.allocator, air.dot(ctx, v, v, fast, loc).appendTo(block)); + try ops.append(std.testing.allocator, air.abs_s(ctx, i, loc).appendTo(block)); + inline for (.{ air.max_s, air.max_u, air.min_s, air.min_u }) |op| { + try ops.append(std.testing.allocator, op(ctx, iv, iv, loc).appendTo(block)); + } + inline for (.{ air.clamp_s, air.clamp_u }) |op| { + try ops.append(std.testing.allocator, op(ctx, i, i, i, loc).appendTo(block)); + } + inline for (.{ air.simd_sum_s, air.simd_sum_u, air.simd_max_s, air.simd_max_u, air.simd_min_s, air.simd_min_u }) |op| { + try ops.append(std.testing.allocator, op(ctx, i, loc).appendTo(block)); + try ops.append(std.testing.allocator, op(ctx, lane, loc).appendTo(block)); + } + inline for (.{ air.simd_shuffle, air.simd_shuffle_xor, air.simd_shuffle_down, air.simd_broadcast }) |op| { + try ops.append(std.testing.allocator, op(ctx, v, lane, fast, loc).appendTo(block)); + } + inline for (.{ air.simd_shuffle_s, air.simd_shuffle_u, air.simd_shuffle_xor_s, air.simd_shuffle_xor_u, air.simd_shuffle_down_s, air.simd_shuffle_down_u, air.simd_broadcast_s, air.simd_broadcast_u }) |op| { + try ops.append(std.testing.allocator, op(ctx, iv, lane, loc).appendTo(block)); + } + inline for (.{ air.simd_any, air.simd_all }) |op| { + try ops.append(std.testing.allocator, op(ctx, b, loc).appendTo(block)); + } + try ops.append(std.testing.allocator, air.convert(ctx, block.argument(11), f32x4, fast, loc).appendTo(block)); + try ops.append(std.testing.allocator, air.convert(ctx, h, .float(ctx, .bf16), .{}, loc).appendTo(block)); + try ops.append(std.testing.allocator, air.convert_s(ctx, i, .float(ctx, .f32), fast, loc).appendTo(block)); + try ops.append(std.testing.allocator, air.convert_u(ctx, f, .int(ctx, .i32), .{}, loc).appendTo(block)); + inline for (.{ air.wg_barrier, air.simdgroup_barrier }) |op| { + try ops.append(std.testing.allocator, op(ctx, .{ .threadgroup = true }, loc).appendTo(block)); + } + const shape = block.argument(8); + const matrix = air.simdgroup_matrix_8x8_load(ctx, block.argument(7), shape, shape, shape, .vector(&.{64}, .float(ctx, .f16)), fast, loc).appendTo(block); + const product = air.simdgroup_matrix_8x8_multiply_accumulate(ctx, matrix.result(0), block.argument(9), block.argument(10), .vector(&.{64}, .float(ctx, .f32)), fast, loc).appendTo(block); + const stored = air.simdgroup_matrix_8x8_store(ctx, product.result(0), block.argument(7), shape, shape, shape, loc).appendTo(block); + try ops.appendSlice(std.testing.allocator, &.{ matrix, product, stored }); + for (ops.items) |op| try std.testing.expect(op.verify()); +} + +test "AIR operations round trip through their custom assembly" { + const ctx = try testContext(); + defer ctx.deinit(); + const source = + \\func.func @ops(%f: f32, %v: vector<4xbf16>, %i: i32, %s: i16, %b: i1, %p: !llvm.ptr<3>, %d: vector<2xi64>, %m: vector<64xf32>) { + \\ %0 = air.fast_exp %f fastmath : f32 + \\ %1 = air.tanh %f : f32 + \\ %2 = air.fma %f, %f, %f fastmath : f32 + \\ %3 = air.max.s %i, %i : i32 + \\ %4 = air.simd_shuffle_xor %f, %s fastmath : f32 + \\ %5 = air.simd_sum.u %i : i32 + \\ %6 = air.simd_any %b + \\ %7 = air.convert %v fastmath : vector<4xbf16> to vector<4xf32> + \\ %8 = air.convert.s %i fastmath : i32 to f32 + \\ %9 = air.dot %7, %7 fastmath : vector<4xf32> -> f32 + \\ air.wg.barrier + \\ air.simdgroup.barrier + \\ %10 = air.simdgroup_matrix_8x8_load %p, %d, %d, %d fastmath : !llvm.ptr<3> -> vector<64xf32> + \\ %11 = air.simdgroup_matrix_8x8_multiply_accumulate %10, %m, %m fastmath : vector<64xf32>, vector<64xf32>, vector<64xf32> -> vector<64xf32> + \\ air.simdgroup_matrix_8x8_store %11, %p, %d, %d, %d : vector<64xf32>, !llvm.ptr<3> + \\ return + \\} + ; + const module = try mlir.Module.parse(ctx, source); + defer module.deinit(); + var first: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer first.deinit(); + try first.writer.print("{f}", .{module.operation()}); + try std.testing.expect(std.mem.find(u8, first.written(), "\"air.") == null); + const reparsed = try mlir.Module.parse(ctx, first.written()); + defer reparsed.deinit(); + var second: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer second.deinit(); + try second.writer.print("{f}", .{reparsed.operation()}); + try std.testing.expectEqualStrings(first.written(), second.written()); +} + +test "AIR verifier rejects types Apple's compiler has no intrinsic for" { + const ctx = try testContext(); + defer ctx.deinit(); + const rejected = [_][]const u8{ + "func.func @f(%x: f16) { %0 = air.fast_exp %x : f16\n return }", + "func.func @f(%x: bf16) { %0 = air.exp %x : bf16\n return }", + "func.func @f(%x: f32) { %0 = air.max.s %x, %x : f32\n return }", + "func.func @f(%x: i64) { %0 = air.simd_sum.s %x : i64\n return }", + "func.func @f(%x: vector<8xf32>) { %0 = air.simd_sum %x : vector<8xf32>\n return }", + "func.func @f(%x: f32) { %0 = air.convert %x : f32 to f32\n return }", + "func.func @f(%x: i32) { %0 = air.convert %x : i32 to f32\n return }", + "func.func @f(%x: f32) { %0 = air.convert.s %x : f32 to f16\n return }", + "func.func @f(%x: f32) { %0 = air.convert.u %x fastmath : f32 to i32\n return }", + "func.func @f(%x: vector<4xf32>) { %0 = air.convert %x : vector<4xf32> to vector<2xf16>\n return }", + "func.func @f(%x: vector<4xf32>) { %0 = air.dot %x, %x : vector<4xf32> -> f16\n return }", + "func.func @f(%p: !llvm.ptr<2>, %d: vector<2xi64>) { %0 = air.simdgroup_matrix_8x8_load %p, %d, %d, %d : !llvm.ptr<2> -> vector<64xf32>\n return }", + "func.func @f(%a: vector<64xbf16>, %b: vector<64xf16>, %c: vector<64xf32>) { %0 = air.simdgroup_matrix_8x8_multiply_accumulate %a, %b, %c : vector<64xbf16>, vector<64xf16>, vector<64xf32> -> vector<64xf32>\n return }", + "func.func @f(%a: vector<64xf16>, %c: vector<64xf32>) { %0 = air.simdgroup_matrix_8x8_multiply_accumulate %a, %a, %c : vector<64xf16>, vector<64xf16>, vector<64xf32> -> vector<64xf16>\n return }", + }; + for (rejected) |source| try std.testing.expectError(error.InvalidMlir, mlir.Module.parse(ctx, source)); +} + +test "AIR kernel ABI" { + const ctx = try testContext(); + defer ctx.deinit(); + const device = "#air.buffer"; + const threadgroup = "#air.buffer"; + const accepted = [_][]const u8{ + "func.func @k(%a: !llvm.ptr<1> {air.buffer = " ++ device ++ ", air.arg_name = \"a\"}, %b: !llvm.ptr<3> {air.buffer = " ++ threadgroup ++ "}, %t: vector<3xi32> {air.builtin = #air.builtin}, %l: i16 {air.builtin = #air.builtin}, %s: i32 {air.builtin = #air.builtin}) attributes {air.kernel} { return }", + "module attributes {air.fast_math_enable} {}", + }; + const rejected = [_][]const u8{ + "func.func @k(%a: i32) attributes {air.kernel} { return }", + "func.func @k(%a: !llvm.ptr<1> {air.buffer = " ++ device ++ ", air.builtin = #air.builtin}) attributes {air.kernel} { return }", + "func.func @k(%a: !llvm.ptr<3> {air.buffer = " ++ device ++ "}) attributes {air.kernel} { return }", + "func.func @k(%a: !llvm.ptr<1> {air.buffer = " ++ device ++ "}, %b: !llvm.ptr<1> {air.buffer = " ++ device ++ "}) attributes {air.kernel} { return }", + "func.func @k(%a: vector<3xi32> {air.builtin = #air.builtin}) attributes {air.kernel} { return }", + "func.func @k(%a: vector<4xi32> {air.builtin = #air.builtin}) attributes {air.kernel} { return }", + "func.func @k(%a: i64 {air.builtin = #air.builtin}) attributes {air.kernel} { return }", + "func.func @k(%a: i32 {air.builtin = #air.builtin}, %b: i32 {air.builtin = #air.builtin}) attributes {air.kernel} { return }", + "func.func @k(%a: i32 {air.builtin = #air.builtin, air.arg_name = \"\"}) attributes {air.kernel} { return }", + "func.func @k() -> i32 attributes {air.kernel} { %c = arith.constant 0 : i32\n return %c : i32 }", + "func.func @h(%a: i32 {air.builtin = #air.builtin}) { return }", + "func.func @h() attributes {air.fast_math_enable} { return }", + "func.func @h() attributes {air.unknown} { return }", + }; + for (accepted) |source| { + const module = try mlir.Module.parse(ctx, source); + module.deinit(); + } + for (rejected) |source| try std.testing.expectError(error.InvalidMlir, mlir.Module.parse(ctx, source)); +} + +test "AIR kernel builder" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc: *const mlir.Location = .unknown(ctx); + const module: *mlir.Module = .init(loc); + defer module.deinit(); + const block = mlir.Block.init(&.{ llvm.pointerType(ctx, air.AddressSpace.device), .vector(&.{3}, .int(ctx, .i32)) }, &.{ loc, loc }); + _ = air.wg_barrier(ctx, .{ .threadgroup = true }, loc).appendTo(block); + _ = func.returns(ctx, &.{}, loc).appendTo(block); + _ = air.kernel(ctx, .{ + .name = "example", + .block = block, + .location = loc, + .args_attributes = &.{ + try air.bufferArgumentAttr(ctx, .{ + .locationIndex = 0, + .addressSpace = .device, + .access = .read_write, + .elementType = .float(ctx, .f32), + .typeName = "float", + .typeSize = 4, + .typeAlignment = 4, + }, .{ .name = "out" }), + try air.builtinArgumentAttr(ctx, .thread_position_in_grid, .{}), + }, + }).appendTo(module.body()); + air.enableFastMath(ctx, module.operation()); + try std.testing.expect(module.operation().verify()); + var writer: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer writer.deinit(); + try writer.writer.print("{f}", .{module.operation()}); + const parsed = try mlir.Module.parse(ctx, writer.written()); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); + try std.testing.expect(std.mem.find(u8, writer.written(), "air.arg_name = \"out\"") != null); +} diff --git a/mlir/dialects/arith/arith.zig b/mlir/dialects/arith/arith.zig index 2e52324e..2c52044e 100644 --- a/mlir/dialects/arith/arith.zig +++ b/mlir/dialects/arith/arith.zig @@ -88,6 +88,40 @@ pub const FastMathAttr = opaque { } }; +/// `mlir::arith::IntegerOverflowFlags`, one bit per flag. +pub const IntegerOverflowFlags = packed struct(u32) { + nsw: bool = false, + nuw: bool = false, + _padding: u30 = 0, + + pub const none: IntegerOverflowFlags = .{}; +}; + +/// `#arith.overflow<...>`. +pub const IntegerOverflowAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAArithIntegerOverflow; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "overflow"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: IntegerOverflowFlags, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirArithIntegerOverflowAttrGet(ctx.ptr(), @bitCast(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) IntegerOverflowFlags { + return @bitCast(c.mlirArithIntegerOverflowAttrGetValue(self.ptr())); + } +}; + // ============================================================================= // Constants // ============================================================================= diff --git a/mlir/dialects/arith/capi.cc b/mlir/dialects/arith/capi.cc index 228c1d09..3d7359c2 100644 --- a/mlir/dialects/arith/capi.cc +++ b/mlir/dialects/arith/capi.cc @@ -23,3 +23,25 @@ uint32_t mlirArithFastMathAttrGetValue(MlirAttribute attr) { return static_cast( llvm::cast(unwrap(attr)).getValue()); } + +bool mlirAttributeIsAArithIntegerOverflow(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} + +MlirAttribute mlirArithIntegerOverflowAttrGet(MlirContext context, + uint32_t value) { + mlir::MLIRContext *ctx = unwrap(context); + std::optional flags = + mlir::arith::symbolizeIntegerOverflowFlags(value); + if (!ctx || !flags) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::arith::IntegerOverflowFlagsAttr::get(ctx, *flags)); +} + +uint32_t mlirArithIntegerOverflowAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} diff --git a/mlir/dialects/arith/capi.h b/mlir/dialects/arith/capi.h index 7c87aa22..ec8962af 100644 --- a/mlir/dialects/arith/capi.h +++ b/mlir/dialects/arith/capi.h @@ -19,6 +19,13 @@ MLIR_CAPI_EXPORTED MlirAttribute mlirArithFastMathAttrGet(MlirContext context, uint32_t value); MLIR_CAPI_EXPORTED uint32_t mlirArithFastMathAttrGetValue(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAArithIntegerOverflow(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirArithIntegerOverflowAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirArithIntegerOverflowAttrGetValue(MlirAttribute attr); + #ifdef __cplusplus } #endif diff --git a/mlir/dialects/arith/test.zig b/mlir/dialects/arith/test.zig index ec1cdf29..913f558d 100644 --- a/mlir/dialects/arith/test.zig +++ b/mlir/dialects/arith/test.zig @@ -94,6 +94,24 @@ test "arith fastmath attribute is built through the C API" { try std.testing.expectError(error.InvalidMlir, arith.FastMathAttr.get(ctx, .{ .value = .{ ._padding = 1 } })); } +test "arith overflow attribute is built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + const cases = [_]struct { arith.IntegerOverflowFlags, []const u8 }{ + .{ .none, "#arith.overflow" }, + .{ .{ .nsw = true }, "#arith.overflow" }, + .{ .{ .nsw = true, .nuw = true }, "#arith.overflow" }, + }; + for (cases) |case| { + const attr = try arith.IntegerOverflowAttr.get(ctx, .{ .value = case[0] }); + try std.testing.expectEqual(case[0], attr.value()); + try std.testing.expect(attr.attribute().isA(arith.IntegerOverflowAttr) != null); + try expectPrints(case[1], attr); + } + try std.testing.expectError(error.InvalidMlir, arith.IntegerOverflowAttr.get(ctx, .{ .value = .{ ._padding = 1 } })); +} + test "every bound operation is registered" { const registry = try mlir.DialectRegistry.init(); defer registry.deinit(); diff --git a/mlir/dialects/llvm/BUILD.bazel b/mlir/dialects/llvm/BUILD.bazel index 352025d2..a03d215e 100644 --- a/mlir/dialects/llvm/BUILD.bazel +++ b/mlir/dialects/llvm/BUILD.bazel @@ -1,5 +1,18 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:CAPIIR", + "@llvm-project//mlir:CAPILLVM", + "@llvm-project//mlir:LLVMDialect", + ], +) + zig_library( name = "llvm", srcs = ["ops.zig"], @@ -8,6 +21,7 @@ zig_library( tags = ["manual"], visibility = ["//visibility:public"], deps = [ + ":capi", "//mlir", "//stdx", "@llvm-project//mlir:CAPILLVM", diff --git a/mlir/dialects/llvm/capi.cc b/mlir/dialects/llvm/capi.cc new file mode 100644 index 00000000..85b480e3 --- /dev/null +++ b/mlir/dialects/llvm/capi.cc @@ -0,0 +1,55 @@ +//===- capi.cc - C API for the llvm dialect -------------------------------===// + +#include "capi.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/Dialect/LLVMIR/LLVMAttrs.h" +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" + +using namespace mlir; + +static MLIRContext *loadLLVM(MlirContext context) { + MLIRContext *ctx = unwrap(context); + if (ctx) + ctx->getOrLoadDialect(); + return ctx; +} + +template static T optional(MlirAttribute attr) { + return llvm::dyn_cast_or_null(unwrap(attr)); +} + +bool mlirAttributeIsALLVMLoopUnroll(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirLLVMLoopUnrollAttrGet(MlirContext context, + MlirAttribute disable, + MlirAttribute count, + MlirAttribute runtimeDisable, + MlirAttribute full) { + MLIRContext *ctx = loadLLVM(context); + if (!ctx) + return {nullptr}; + return wrap(LLVM::LoopUnrollAttr::get(ctx, optional(disable), + optional(count), + optional(runtimeDisable), + optional(full), {}, {}, {})); +} + +bool mlirAttributeIsALLVMLoopAnnotation(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirLLVMLoopAnnotationAttrGet(MlirContext context, + MlirAttribute disableNonforced, + MlirAttribute unroll, + MlirAttribute mustProgress) { + MLIRContext *ctx = loadLLVM(context); + if (!ctx) + return {nullptr}; + return wrap(LLVM::LoopAnnotationAttr::get( + ctx, optional(disableNonforced), {}, {}, + optional(unroll), {}, {}, {}, {}, {}, {}, + optional(mustProgress), {}, {}, {}, {})); +} diff --git a/mlir/dialects/llvm/capi.h b/mlir/dialects/llvm/capi.h new file mode 100644 index 00000000..04528ddc --- /dev/null +++ b/mlir/dialects/llvm/capi.h @@ -0,0 +1,36 @@ +//===- capi.h - C API for the llvm dialect ------------------------*- C -*-===// +// +// C entry points for the `llvm` dialect's loop annotations, which upstream's +// C API leaves out. + +#ifndef MLIR_DIALECTS_LLVM_CAPI_H +#define MLIR_DIALECTS_LLVM_CAPI_H + +#include "mlir-c/Dialect/LLVM.h" +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Null parameters are unset. + +MLIR_CAPI_EXPORTED bool mlirAttributeIsALLVMLoopUnroll(MlirAttribute attr); +/// `disable`, `runtimeDisable` and `full` are bool attributes, `count` an +/// integer attribute. +MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMLoopUnrollAttrGet( + MlirContext context, MlirAttribute disable, MlirAttribute count, + MlirAttribute runtimeDisable, MlirAttribute full); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsALLVMLoopAnnotation(MlirAttribute attr); +/// `disableNonforced` and `mustProgress` are bool attributes, `unroll` a +/// loop unroll attribute; the other parameters are unset. +MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMLoopAnnotationAttrGet( + MlirContext context, MlirAttribute disableNonforced, MlirAttribute unroll, + MlirAttribute mustProgress); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_LLVM_CAPI_H diff --git a/mlir/dialects/llvm/llvm.zig b/mlir/dialects/llvm/llvm.zig index a5241296..167099eb 100644 --- a/mlir/dialects/llvm/llvm.zig +++ b/mlir/dialects/llvm/llvm.zig @@ -4,6 +4,12 @@ const ops = @import("ops.zig"); pub const NVPTXAddressSpace = ops.NVPTXAddressSpace; pub const pointerType = ops.pointerType; pub const structType = ops.structType; +pub const identifiedStructType = ops.identifiedStructType; +pub const arrayType = ops.arrayType; +pub const LoopUnrollAttr = ops.LoopUnrollAttr; +pub const LoopAnnotationAttr = ops.LoopAnnotationAttr; +pub const Linkage = ops.Linkage; +pub const linkageAttr = ops.linkageAttr; pub const inttoptr = ops.inttoptr; pub const bitcast = ops.bitcast; pub const fmul = ops.fmul; @@ -11,6 +17,9 @@ pub const LoadOpts = ops.LoadOpts; pub const load = ops.load; pub const StoreOpts = ops.StoreOpts; pub const store = ops.store; +pub const GEPIndex = ops.GEPIndex; +pub const getelementptr = ops.getelementptr; +pub const alloca = ops.alloca; pub const call_intrinsic = ops.call_intrinsic; pub const InlineAsmOpts = ops.InlineAsmOpts; pub const inline_asm = ops.inline_asm; diff --git a/mlir/dialects/llvm/ops.zig b/mlir/dialects/llvm/ops.zig index 19412dfd..0515986f 100644 --- a/mlir/dialects/llvm/ops.zig +++ b/mlir/dialects/llvm/ops.zig @@ -1,3 +1,5 @@ +const std = @import("std"); + const c = @import("c"); const mlir = @import("mlir"); const stdx = @import("stdx"); @@ -35,6 +37,104 @@ pub fn structType(ctx: *mlir.Context, fields: []const *const mlir.Type) *const m return @ptrCast(c.mlirLLVMStructTypeLiteralGet(ctx.ptr(), @intCast(fields.len), @ptrCast(fields), false).ptr); } +/// `!llvm.struct<"name", (fields...)>`, an identified struct; null if `name` +/// already names a struct of other fields. +pub fn identifiedStructType(ctx: *mlir.Context, name: []const u8, fields: []const *const mlir.Type) ?*const mlir.Type { + _ = c.mlirContextGetOrLoadDialect(ctx.ptr(), mlir.stringRef("llvm")); + const t = c.mlirLLVMStructTypeIdentifiedGet(ctx.ptr(), mlir.stringRef(name)); + if (c.mlirLLVMStructTypeSetBody(t, @intCast(fields.len), @ptrCast(fields), false).value == 0) return null; + return @ptrCast(t.ptr); +} + +/// `!llvm.array`. +pub fn arrayType(element_type: *const mlir.Type, count: u32) *const mlir.Type { + return @ptrCast(c.mlirLLVMArrayTypeGet(element_type.ptr(), count).ptr); +} + +// ============================================================================= +// Attributes +// ============================================================================= + +fn optionalBool(ctx: *mlir.Context, value: ?bool) c.MlirAttribute { + return if (value) |v| mlir.Attribute.boolean(ctx, v).ptr() else .{ .ptr = null }; +} + +/// `#llvm.loop_unroll`: the `llvm.loop.unroll.*` loop metadata. +pub const LoopUnrollAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsALLVMLoopUnroll; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + disable: ?bool = null, + count: ?u32 = null, + runtimeDisable: ?bool = null, + full: ?bool = null, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const Self { + const count: c.MlirAttribute = if (args.count) |n| mlir.Attribute.int(ctx, .i32, n).ptr() else .{ .ptr = null }; + return @ptrCast(c.mlirLLVMLoopUnrollAttrGet(ctx.ptr(), optionalBool(ctx, args.disable), count, optionalBool(ctx, args.runtimeDisable), optionalBool(ctx, args.full)).ptr); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// A global's linkage, as `#llvm.linkage` spells it. +pub const Linkage = enum(c_uint) { + external = c.MlirLLVMLinkageExternal, + available_externally = c.MlirLLVMLinkageAvailableExternally, + linkonce = c.MlirLLVMLinkageLinkonce, + linkonce_odr = c.MlirLLVMLinkageLinkonceODR, + weak = c.MlirLLVMLinkageWeak, + weak_odr = c.MlirLLVMLinkageWeakODR, + appending = c.MlirLLVMLinkageAppending, + internal = c.MlirLLVMLinkageInternal, + private = c.MlirLLVMLinkagePrivate, + extern_weak = c.MlirLLVMLinkageExternWeak, + common = c.MlirLLVMLinkageCommon, +}; + +/// `#llvm.linkage`; `llvm.linkage` on a `func.func` sets its +/// linkage once lowered to LLVM. Loads the LLVM dialect into `ctx`, which +/// must have it registered. +pub fn linkageAttr(ctx: *mlir.Context, linkage: Linkage) *const mlir.Attribute { + _ = c.mlirContextGetOrLoadDialect(ctx.ptr(), mlir.stringRef("llvm")); + return @ptrCast(c.mlirLLVMLinkageAttrGet(ctx.ptr(), @backingInt(linkage)).ptr); +} + +/// `#llvm.loop_annotation`: the `llvm.loop` metadata of a loop. +pub const LoopAnnotationAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsALLVMLoopAnnotation; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + disableNonforced: ?bool = null, + unroll: ?*const LoopUnrollAttr = null, + mustProgress: ?bool = null, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const Self { + const unroll: c.MlirAttribute = if (args.unroll) |u| u.ptr() else .{ .ptr = null }; + return @ptrCast(c.mlirLLVMLoopAnnotationAttrGet(ctx.ptr(), optionalBool(ctx, args.disableNonforced), unroll, optionalBool(ctx, args.mustProgress)).ptr); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + // ============================================================================= // Casts and arithmetic // ============================================================================= @@ -113,6 +213,66 @@ pub fn store(ctx: *mlir.Context, value: *const mlir.Value, pointer: *const mlir. }); } +/// An index of `getelementptr`: a constant, or an SSA integer. +pub const GEPIndex = union(enum) { + constant: i32, + value: *const mlir.Value, +}; + +/// llvm.getelementptr — `base` offset by `indices` into `elem_type`, with +/// `base`'s pointer type. +pub fn getelementptr( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const GEPIndex, + elem_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + const dynamic_index = std.math.minInt(i32); + var operands: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + var raw: stdx.BoundedArray(i32, 16) = .empty; + operands.appendAssumeCapacity(base); + for (indices) |index| switch (index) { + .constant => |c_| raw.appendAssumeCapacity(c_), + .value => |v| { + raw.appendAssumeCapacity(dynamic_index); + operands.appendAssumeCapacity(v); + }, + }; + return mlir.Operation.make(ctx, "llvm.getelementptr", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = &.{base.type_()} }, + .attributes = &.{ + .named(ctx, "rawConstantIndices", .denseArray(ctx, .i32, raw.constSlice())), + .named(ctx, "elem_type", .typeAttr(elem_type)), + }, + .location = location, + }); +} + +/// llvm.alloca — stack storage for `array_size` elements of `elem_type`, as a +/// pointer of type `result_type`. +pub fn alloca( + ctx: *mlir.Context, + array_size: *const mlir.Value, + result_type: *const mlir.Type, + elem_type: *const mlir.Type, + opts: struct { + alignment: ?u64 = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attrs.appendAssumeCapacity(.named(ctx, "elem_type", .typeAttr(elem_type))); + if (opts.alignment) |alignment| attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, alignment))); + return mlir.Operation.make(ctx, "llvm.alloca", .{ + .operands = .{ .flat = &.{array_size} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + // ============================================================================= // Intrinsics and inline assembly // ============================================================================= diff --git a/mlir/dialects/llvm/test.zig b/mlir/dialects/llvm/test.zig index d5c2b282..62a191f5 100644 --- a/mlir/dialects/llvm/test.zig +++ b/mlir/dialects/llvm/test.zig @@ -4,8 +4,10 @@ const llvm = @import("mlir/dialects/llvm"); const mlir = @import("mlir"); const operation_names: []const []const u8 = &.{ + "llvm.alloca", "llvm.bitcast", "llvm.call_intrinsic", + "llvm.getelementptr", "llvm.fmul", "llvm.inline_asm", "llvm.inttoptr", @@ -42,3 +44,60 @@ test "literal struct types build from their fields" { const struct_type = llvm.structType(ctx, &.{ .int(ctx, .i32), .float(ctx, .f32) }); try std.testing.expect(struct_type.eql(try mlir.Type.parse(ctx, "!llvm.struct<(i32, f32)>"))); } + +test "identified struct types build from their name and fields" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + const fields: []const *const mlir.Type = &.{ .int(ctx, .i32), .float(ctx, .f32) }; + const struct_type = llvm.identifiedStructType(ctx, "struct.args", fields).?; + try std.testing.expect(struct_type.eql(try mlir.Type.parse(ctx, "!llvm.struct<\"struct.args\", (i32, f32)>"))); + try std.testing.expect(llvm.identifiedStructType(ctx, "struct.args", fields).?.eql(struct_type)); + try std.testing.expect(llvm.identifiedStructType(ctx, "struct.args", fields[0..1]) == null); +} + +test "getelementptr and alloca build verified operations" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + const loc: *const mlir.Location = .unknown(ctx); + const ptr = llvm.pointerType(ctx, llvm.NVPTXAddressSpace.global); + const block = mlir.Block.init(&.{ ptr, .int(ctx, .i64) }, &.{ loc, loc }); + defer block.deinit(); + const array = llvm.arrayType(.float(ctx, .f32), 8); + const gep = llvm.getelementptr(ctx, block.argument(0), &.{ .{ .constant = 0 }, .{ .value = block.argument(1) } }, array, loc).appendTo(block); + try std.testing.expect(gep.verify()); + try std.testing.expect(gep.result(0).type_().eql(ptr)); + const alloca = llvm.alloca(ctx, block.argument(1), llvm.pointerType(ctx, llvm.NVPTXAddressSpace.generic), array, .{ .alignment = 4 }, loc).appendTo(block); + try std.testing.expect(alloca.verify()); +} + +test "loop annotations are built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + const unroll = llvm.LoopUnrollAttr.get(ctx, .{ .full = true }); + try std.testing.expect(unroll.attribute().isA(llvm.LoopUnrollAttr) != null); + const annotation = llvm.LoopAnnotationAttr.get(ctx, .{ .unroll = unroll, .mustProgress = true }); + var writer: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer writer.deinit(); + try writer.writer.print("{f}", .{annotation}); + try std.testing.expectEqualStrings("#llvm.loop_annotation, mustProgress = true>", writer.written()); + try std.testing.expect((try mlir.Attribute.parse(ctx, writer.written())).eql(annotation.attribute())); +} + +test "linkage attributes are built through the C API" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + var writer: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer writer.deinit(); + try writer.writer.print("{f}", .{llvm.linkageAttr(ctx, .weak_odr)}); + try std.testing.expectEqualStrings("#llvm.linkage", writer.written()); +} From 8f40b8b09fd2cd954d527186c12a804364f1a4a5 Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Mon, 26 Oct 2026 15:47:48 +0100 Subject: [PATCH 824/881] zml/kernels: add TCL paged attention and MoE backends for Furiosa (#973) zml: add TCL kernels and Furiosa attention and MoE backends Introduce a typed TCL kernel DSL and MLIR bindings, and select TCL automatically for Furiosa paged attention and BF16 SwiGLU MoE. - Support native kernel calls, fusion, in-place outputs, device loops, compiler configuration, and chip-aware layouts. - Implement standard paged attention for decode, prefill, and mixed batches, including causal/non-causal masks, sliding windows, custom scales, and per-head sinks. - Use BF16 contractions with FP32 normalization and accumulation. Exclude unused KV entries before contractions and return zero for inactive or fully masked queries. - Reuse prepared metadata for the optimized decode path and process longer contexts in page-aligned chunks. - Add activation-stationary and blockwise fused-expert kernels. - Fix StableHLO attention metadata for variable batch sizes and add host-reference, emission, and hardware regression coverage. - Bump the Furiosa sandbox to 202610221403.283.1.d9f4f68759e1. --------- Co-authored-by: Raphael Nahoum --- platforms/furiosa/furiosa.bzl | 2 +- zml/BUILD.bazel | 2 + zml/kernels/moe/moe.zig | 10 +- zml/kernels/moe/tcl/BUILD.bazel | 6 + zml/kernels/moe/tcl/tcl.zig | 348 +++++ .../paged_attention/paged_attention.zig | 26 +- zml/kernels/paged_attention/tcl/BUILD.bazel | 6 + zml/kernels/paged_attention/tcl/tcl.zig | 1293 +++++++++++++++++ 8 files changed, 1683 insertions(+), 10 deletions(-) create mode 100644 zml/kernels/moe/tcl/BUILD.bazel create mode 100644 zml/kernels/moe/tcl/tcl.zig create mode 100644 zml/kernels/paged_attention/tcl/BUILD.bazel create mode 100644 zml/kernels/paged_attention/tcl/tcl.zig diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index 40e711e7..0aa3905d 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,7 +6,7 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202610231457.291.1.799ad5c39a34/zml-furiosa-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610221403.283.1.d9f4f68759e1/zml-furiosa-linux-amd64.tar.zst", sha256 = "f8d6bb03fb4cd5dad9b37856b2f8f12b28ec1a8d46f59ca03c401bea668215e9", ) diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index e0684831..d959df5e 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -118,10 +118,12 @@ zig_library( "//zml/kernels/moe/metal", "//zml/kernels/moe/mosaic_tpu", "//zml/kernels/moe/stablehlo", + "//zml/kernels/moe/tcl", "//zml/kernels/moe/triton", "//zml/kernels/paged_attention/metal", "//zml/kernels/paged_attention/mosaic_tpu", "//zml/kernels/paged_attention/stablehlo", + "//zml/kernels/paged_attention/tcl", "//zml/kernels/paged_attention/triton", "//zml/kernels/paged_attention/cute", ], diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index fcfd269c..bf6a0a37 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -4,6 +4,7 @@ const platforms = @import("platforms"); const zml = @import("../../zml.zig"); const stdx = zml.stdx; +pub const tcl = @import("tcl/tcl.zig"); pub const stablehlo = @import("stablehlo/stablehlo.zig"); pub const cute = @import("cute/cute.zig"); pub const cutlass_flashinfer = @import("cutlass_flashinfer/cutlass_flashinfer.zig"); @@ -83,6 +84,7 @@ pub const Backend = enum { fly, mosaic_tpu, metal, + tcl, pub fn auto(platform: *const zml.Platform, scheme: ?zml.Quantization.Scheme, dtype: zml.DataType) !Backend { // Keep the dtype as an argument because non scheme-specific backends may depend on it later @@ -118,6 +120,7 @@ pub const Backend = enum { }; }, .tpu => if (scheme == null) .mosaic_tpu else error.UnsupportedQuantization, + .furiosa => if (scheme == null) .tcl else error.UnsupportedQuantization, .metal => b: { const s = scheme orelse break :b .metal; break :b switch (s) { @@ -144,6 +147,7 @@ pub const Backend = enum { }, .mosaic_tpu => platform.target == .tpu, .metal => platform.target == .metal, + .tcl => platform.target == .furiosa, }; } @@ -156,7 +160,7 @@ pub const Backend = enum { .nvfp4 => .{ .gate_up = .concatenated, .packing = .flashinfer_nvfp4 }, else => error.UnsupportedQuantization, } else .{ .gate_up = .concatenated, .gate_up_order = .up_gate, .packing = .plain }, - .mosaic_tpu, .metal => .{ .gate_up = .concatenated, .packing = .plain }, + .mosaic_tpu, .metal, .tcl => .{ .gate_up = .concatenated, .packing = .plain }, .triton, .fly => if (scheme == .mxfp4) .{ .gate_up = .interleaved, .packing = .plain } else .{ .gate_up = .concatenated, .packing = .plain }, }; } @@ -169,6 +173,7 @@ pub const Backend = enum { .triton, .fly => {}, .mosaic_tpu => {}, .metal => {}, + .tcl => {}, }; } }; @@ -177,7 +182,7 @@ pub const Options = struct { activation: Activation, /// Quantize activations for Triton FP8 GEMMs; false keeps BF16 activations. quantize_input: bool, - /// Where routing weights are applied; FlashInfer, Mosaic and Metal require after_down. + /// Where routing weights are applied; FlashInfer, Mosaic, Metal and TCL require after_down. routing_weight_placement: RoutingWeightPlacement, }; @@ -211,6 +216,7 @@ pub fn forwardMoe( .fly => fly.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), .mosaic_tpu => mosaic_tpu.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), .metal => metal.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + .tcl => tcl.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), }; } diff --git a/zml/kernels/moe/tcl/BUILD.bazel b/zml/kernels/moe/tcl/BUILD.bazel new file mode 100644 index 00000000..4057eea3 --- /dev/null +++ b/zml/kernels/moe/tcl/BUILD.bazel @@ -0,0 +1,6 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "tcl", + srcs = ["tcl.zig"], +) diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig new file mode 100644 index 00000000..206dcf66 --- /dev/null +++ b/zml/kernels/moe/tcl/tcl.zig @@ -0,0 +1,348 @@ +//! Furiosa RNGD fused experts in TCL, after Furiosa's production kernels. +//! +//! Few tokens (at most `experts / top_k`): activation stationary, each token +//! gathers the weights of its experts and the down projection contracts over +//! the experts and the intermediate axis at once. +//! +//! More tokens: blockwise. The kernel routes on the device: routes are placed +//! in expert-major blocks of `BB` rows (one-hot, cumulative sums), then a +//! device loop over the used chunks of `BGi` blocks gathers each block's +//! expert weights and runs both projections; outputs are gathered back per +//! route and weighted. +const std = @import("std"); + +const builder = @import("dsl/tcl/builder"); + +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const Tensor = zml.Tensor; +const tcl = zml.kernel.tcl; + +/// `input` is `{.t, .d}`, `topk_ids` and `topk_weights` `{.t, .topk}`, `gate_up` `{.expert, .dout = 2 * inter (gate then up), .d}` +/// and `down` `{.expert, .d, .dout = inter}`, all bf16 but the ids (i32). +/// SwiGLU, routing weights applied after the down projection. +pub fn fusedExperts(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: zml.moe.Options) Tensor { + stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE applies routing weights after the down projection, without FP8 input quantization", .{}); + stdx.debug.assert(opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}), "TCL MoE supports SwiGLU without limit, scale or bias", .{}); + stdx.debug.assert(gate_up_linear.quantization == null and down_linear.quantization == null, "TCL MoE takes bf16 experts", .{}); + stdx.debug.assert(gate_up_linear.bias == null and down_linear.bias == null, "TCL MoE takes experts without bias", .{}); + const gate_up = gate_up_linear.weight; + const down = down_linear.weight; + stdx.debug.assert(input.dtype() == .bf16 and gate_up.dtype() == .bf16 and down.dtype() == .bf16 and topk_ids.dtype() == .i32, "TCL MoE takes bf16 activations and experts, i32 ids", .{}); + const t = input.dim(.t); + const k = topk_ids.dim(.topk); + const cfg: Cfg = .{ + .t = t, + .k = k, + .e = gate_up.dim(.expert), + .d = input.dim(.d), + .i = down.dim(.dout), + }; + stdx.debug.assert(gate_up.dim(.dout) == 2 * cfg.i and down.dim(.d) == cfg.d and gate_up.dim(.d) == cfg.d, "TCL MoE: mismatched expert shapes", .{}); + + const x = input.reshape(.{ t, cfg.d }); + const ids = topk_ids.reshape(.{ t, k }); + const weights = topk_weights.convert(.f32).reshape(.{ t, k }); + const out_shape = x.shape(); + const y = if (cfg.t * cfg.k <= cfg.e) + ActivationStationary.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y + else + Blockwise.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y; + return y.reshape(input.shape()); +} + +pub const Cfg = struct { t: i64, k: i64, e: i64, d: i64, i: i64 }; + +const Axes = struct { + T: tcl.Axis, + K: tcl.Axis, + E: tcl.Axis, + D: tcl.Axis, + I: tcl.Axis, + I2: tcl.Axis, + S2: tcl.Axis, + S1: tcl.Axis, + + fn init(b: *tcl.Builder, cfg: Cfg) Axes { + return .{ + .T = b.axis("T", cfg.t), + .K = b.axis("K", cfg.k), + .E = b.axis("E", cfg.e), + .D = b.axis("D", cfg.d), + .I = b.axis("I", cfg.i), + .I2 = b.axis("I2", 2 * cfg.i), + .S2 = b.axis("S2", 2), + .S1 = b.axis("S1", 1), + }; + } +}; + +fn args(b: *tcl.Builder, a: Axes) tcl.FinishError!Args { + const t = try b.declareArgs(.{ + .x = .{ .dtype = .bf16, .axes = &.{ a.T, a.D } }, + .ids = .{ .dtype = .i32, .axes = &.{ a.T, a.K } }, + .w = .{ .dtype = .f32, .axes = &.{ a.T, a.K } }, + .gate_up = .{ .dtype = .bf16, .axes = &.{ a.E, a.I2, a.D } }, + .down = .{ .dtype = .bf16, .axes = &.{ a.E, a.D, a.I } }, + }); + return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down }; +} + +const Args = struct { x: builder.Tensor, ids: builder.Tensor, w: builder.Tensor, gate_up: builder.Tensor, down: builder.Tensor }; + +/// `silu(gate) * up` of `h = [batch..., S2, I]` (gate first), times +/// `weights` (`[batch...]`) unless null. +fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor) tcl.FinishError!builder.Tensor { + var sliced: [4]tcl.Axis = undefined; + var flat: [3]tcl.Axis = undefined; + @memcpy(sliced[0..batch.len], batch); + @memcpy(flat[0..batch.len], batch); + sliced[batch.len] = a.S1; + sliced[batch.len + 1] = a.I; + flat[batch.len] = a.I; + const s = sliced[0 .. batch.len + 2]; + const f = flat[0 .. batch.len + 1]; + const gate = try b.reshape(try b.slice(h, a.S2, 0, s, .{}), f, .{}); + const up = try b.reshape(try b.slice(h, a.S2, 1, s, .{}), f, .{}); + // Interleaving requires one binary instruction; SwiGLU chains several. + const op = b.tensorOperation(.{ .tactic = .elementwise }); + const g = op.fetch(gate, .{}); + var v = g.sigmoid().mulf(g).mulf(op.fetch(up, .{})); + if (weights) |w| v = v.mulf(op.fetch(w, .{})); + return op.commit(v, .{ .dtype = .bf16 }); +} + +/// Furiosa's compiler configuration for MoE kernels (`qwen3_moe/config.py`): +/// IO-bound tactics for activation stationary, compute-bound for blockwise. +/// Keep the compiler's default lowering mode and omit the adaptive-einsum and +/// half-VRF settings removed in TCC 2026.5. +fn configure(b: *tcl.Builder, blockwise: bool) void { + if (blockwise) b.compilerConfig(.{ + .enable_einsum_fusion = true, + .padding_policy = .{ .Small = 1.1 }, + .instruction_mem_budget = 0xB0000, + .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, + .tensor_unit_bridge_threshold_in_page = 12, + .allow_external_operators = false, + .enable_tactic_pruning = false, + .scheduler_beam_search = true, + .allow_reduce_by_ve_cluster_chip_reduce = false, + .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, + .tactic_hint = .{ .ForLlmModelComputeBound = 200 }, + .dma_preference = 1.2, + .propagate_sparse_axis_from_op = "None", + }) else b.compilerConfig(.{ + .enable_einsum_fusion = true, + .padding_policy = .{ .Small = 1.1 }, + .instruction_mem_budget = 0xB0000, + .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, + .tensor_unit_bridge_threshold_in_page = 12, + .allow_external_operators = false, + .enable_tactic_pruning = false, + .scheduler_beam_search = true, + .allow_reduce_by_ve_cluster_chip_reduce = false, + .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, + .tactic_hint = .{ .ForLlmModelIOBound = 200 }, + .num_transaction_simulation_per_pe = 1024, + .dma_preference = 0.8, + .propagate_sparse_axis_from_op = "None", + }); +} + +pub const ActivationStationary = tcl.Kernel(Cfg, .{ + .name = "moe_activation_stationary", + .inputs = &.{ "x", "ids", "w", "gate_up", "down" }, + .outputs = &.{"y"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + configure(b, false); + const a: Axes = .init(b, cfg); + const in = try args(b, a); + const gate_up = try b.reshape(in.gate_up, &.{ a.E, a.S2, a.I, a.D }, .{}); + const gu = try b.gather(gate_up, in.ids, a.E, &.{ a.T, a.K, a.S2, a.I, a.D }, .{}); + const dn = try b.gather(in.down, in.ids, a.E, &.{ a.T, a.K, a.D, a.I }, .{}); + + var op = b.tensorOperation(.{}); + const h = try op.commit(op.contract(in.x, gu, &.{ a.T, a.K, a.S2, a.I }), .{}); + // Routing weights scale the activations, so the down projection + // also sums over the experts. + const act = try gated(b, a, h, &.{ a.T, a.K }, in.w); + op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.contract(act, dn, &.{ a.T, a.D }), .{ .dtype = .bf16 })}); + } + }.run, +}); + +/// Rows per expert block: the next power of two of the routes per expert. +pub fn blockRows(cfg: Cfg) i64 { + const per_expert = std.math.divCeil(i64, cfg.t * cfg.k, cfg.e) catch unreachable; + return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@max(per_expert, 4))) catch unreachable); +} + +const blocks_per_chunk = 32; + +pub const Blockwise = tcl.Kernel(Cfg, .{ + .name = "moe_blockwise", + .inputs = &.{ "x", "ids", "w", "gate_up", "down" }, + .outputs = &.{"y"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + configure(b, true); + const a: Axes = .init(b, cfg); + const in = try args(b, a); + const bb = blockRows(cfg); + // One block per expert at least, then enough for every route. + const max_blocks = std.math.divCeil(i64, @max(cfg.t * cfg.k - cfg.e, 0), bb) catch unreachable; + const chunks = std.math.divCeil(i64, max_blocks + cfg.e, blocks_per_chunk) catch unreachable; + const BB = b.axis("BB", bb); + const BGi = b.axis("BGi", blocks_per_chunk); + const Go = b.axis("Go", chunks); + const BG = b.axis("BG", chunks * blocks_per_chunk); + const GB = b.axis("GB", chunks * blocks_per_chunk * bb); + const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; + + // Routing: route (t, k) goes to row `slot` of block `start[e] + rank`. + const experts = try b.arange(cfg.e, .i32, &.{E}, .{}); + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); + op = b.tensorOperation(.{}); + const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const rank = try op.commit(op.fetch(onehot, .{}).cumsum(&.{ T, K }), .{}); + op = b.tensorOperation(.{}); + const log2_bb: i64 = std.math.log2_int(u64, @intCast(bb)); + const blocks = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi).addi(bb - 1).binary(.shr_arith, log2_bb), .{}); + op = b.tensorOperation(.{}); + const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{E}), .{}); + op = b.tensorOperation(.{}); + const start_rows = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})).muli(bb), .{}); + op = b.tensorOperation(.{}); + const slot = try op.commit(op.fetch(rank, .{}).addi(op.fetch(start_rows, .{})).muli(op.fetch(onehot, .{})).reduce(&.{E}, .addi).subi(1), .{}); + // The expert of block g: how many experts end at or before g. + const block_ids = try b.arange(chunks * blocks_per_chunk, .i32, &.{BG}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const ends = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(block_ids, .{})), .{}); + op = b.tensorOperation(.{}); + const ended = try op.commit(op.where(ends, .le, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + // Padding blocks past the last expert reuse the last one. + const block_expert = try op.commit(op.fetch(ended, .{}).reduce(&.{E}, .addi).binary(.mini, cfg.e - 1), .{}); + const used = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(block_end), blocks_per_chunk - 1), blocks_per_chunk); + + // Rows of unused slots are never read back. + const rows = try b.reshape(try b.scatter(in.x, slot, GB, &.{ GB, D }, .{}), &.{ Go, BGi, BB, D }, .{}); + const chunk_experts = try b.reshape(block_expert, &.{ Go, BGi }, .{}); + const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); + + var loop = b.openFor(used, .{try b.scratchpad(.bf16, &.{ Go, BGi, BB, D })}); + const chunk = try b.indexRead(rows, loop.iv); + const ids = try b.indexRead(chunk_experts, loop.iv); + const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); + const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); + op = b.tensorOperation(.{}); + const h = try op.commit(op.contract(chunk, gu, &.{ BGi, BB, a.S2, I }), .{}); + const act = try gated(b, a, h, &.{ BGi, BB }, null); + op = b.tensorOperation(.{}); + const y = try op.commit(op.contract(act, dn, &.{ BGi, BB, D }), .{ .dtype = .bf16 }); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); + const out = loop.results[0]; + + const per_route = try b.gather(try b.reshape(out, &.{ GB, D }, .{}), slot, GB, &.{ T, K, D }, .{}); + op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(per_route, .{ .typecast_to = .f32 }).mulf(op.fetch(in.w, .{})).reduce(&.{K}, .addf), .{ .dtype = .bf16 })}); + } + }.run, +}); + +test "tcl fused experts run on furiosa" { + const platform = zml.testing.env(); + if (platform.target != .furiosa) return error.SkipZigTest; + const allocator = std.testing.allocator; + const io = std.testing.io; + const bf16 = zml.floats.BFloat16; + + // 2 * 4 <= 8 experts is activation stationary, 16 tokens is blockwise. + for ([_]i64{ 4, 16 }) |t| { + const k = 2; + const e = 8; + const d = 256; + const i = 64; + const x: Tensor = .init(.{ .t = t, .d = d }, .bf16); + const ids: Tensor = .init(.{ .t = t, .topk = k }, .i32); + const w: Tensor = .init(.{ .t = t, .topk = k }, .f32); + const gu: Tensor = .init(.{ .expert = e, .dout = 2 * i, .d = d }, .bf16); + const dn: Tensor = .init(.{ .expert = e, .d = d, .dout = i }, .bf16); + const Mod = struct { + pub fn forward(x_: Tensor, ids_: Tensor, w_: Tensor, gu_: Tensor, dn_: Tensor) Tensor { + return fusedExperts(x_, ids_, w_, .init(gu_, null, .dout), .init(dn_, null, .dout), .{ + .activation = .{ .swiglu = .{} }, + .quantize_input = false, + .routing_weight_placement = .after_down, + }); + } + }; + var exe = try zml.module.compile(allocator, io, Mod.forward, .{ x, ids, w, gu, dn }, platform, .{}); + defer exe.deinit(); + + var prng: std.Random.DefaultPrng = .init(@intCast(t)); + const r = prng.random(); + const hx = try allocator.alloc(bf16, @intCast(t * d)); + defer allocator.free(hx); + const hgu = try allocator.alloc(bf16, e * 2 * i * d); + defer allocator.free(hgu); + const hdn = try allocator.alloc(bf16, e * d * i); + defer allocator.free(hdn); + for (hx) |*v| v.* = .fromF32(r.floatNorm(f32) * 0.5); + for (hgu) |*v| v.* = .fromF32(r.floatNorm(f32) / 16); + for (hdn) |*v| v.* = .fromF32(r.floatNorm(f32) / 8); + const hids = try allocator.alloc(i32, @intCast(t * k)); + defer allocator.free(hids); + const hw = try allocator.alloc(f32, @intCast(t * k)); + defer allocator.free(hw); + for (0..@intCast(t)) |tok| for (0..k) |j| { + hids[tok * k + j] = @intCast((tok * 3 + j * 5) % e); + hw[tok * k + j] = if (j == 0) 0.7 else 0.3; + }; + + var bx: zml.Buffer = try .fromBytes(io, platform, x.shape(), std.mem.sliceAsBytes(hx)); + defer bx.deinit(); + var bids: zml.Buffer = try .fromBytes(io, platform, ids.shape(), std.mem.sliceAsBytes(hids)); + defer bids.deinit(); + var bw: zml.Buffer = try .fromBytes(io, platform, w.shape(), std.mem.sliceAsBytes(hw)); + defer bw.deinit(); + var bgu: zml.Buffer = try .fromBytes(io, platform, gu.shape(), std.mem.sliceAsBytes(hgu)); + defer bgu.deinit(); + var bdn: zml.Buffer = try .fromBytes(io, platform, dn.shape(), std.mem.sliceAsBytes(hdn)); + defer bdn.deinit(); + var result = try exe.eval(allocator, io, .{ bx, bids, bw, bgu, bdn }); + defer result.deinit(); + var host = try result.toSliceAlloc(allocator, io); + defer host.free(allocator); + const out = host.items(bf16); + + for (0..@intCast(t)) |tok| { + var expected: [d]f32 = @splat(0); + for (0..k) |j| { + const ex: usize = @intCast(hids[tok * k + j]); + var act: [i]f32 = undefined; + for (0..i) |n| { + var g: f32 = 0; + var u: f32 = 0; + for (0..d) |m| { + const xv = hx[tok * d + m].toF32(); + g += xv * hgu[(ex * 2 * i + n) * d + m].toF32(); + u += xv * hgu[(ex * 2 * i + i + n) * d + m].toF32(); + } + act[n] = g / (1 + @exp(-g)) * u; + } + for (0..d) |m| { + var y: f32 = 0; + for (0..i) |n| y += act[n] * hdn[(ex * d + m) * i + n].toF32(); + expected[m] += hw[tok * k + j] * y; + } + } + for (expected, out[tok * d ..][0..d]) |want, got| try std.testing.expectApproxEqAbs(want, got.toF32(), 2e-2 + 2e-2 * @abs(want)); + } + } +} diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index 5cc2ba22..a8ec2121 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -7,11 +7,13 @@ const cute = @import("cute/cute.zig"); const metal = @import("metal/metal.zig"); const tpu = @import("mosaic_tpu/mosaic_tpu.zig"); const triton = @import("triton/triton.zig"); +pub const tcl = @import("tcl/tcl.zig"); const stablehlo = @import("stablehlo/stablehlo.zig"); const PagedAttention = @This(); test { + _ = tcl; std.testing.refAllDecls(Backend); std.testing.refAllDecls(Options); std.testing.refAllDecls(Parameters); @@ -24,6 +26,7 @@ pub const Backend = enum { mosaic_tpu, metal, stablehlo, + tcl, pub fn auto(platform: *const zml.Platform) Backend { return switch (platform.target) { @@ -32,7 +35,8 @@ pub const Backend = enum { .oneapi => .triton, .tpu => .mosaic_tpu, .metal => .metal, - .cpu, .furiosa => .stablehlo, + .cpu => .stablehlo, + .furiosa => .tcl, .neuron => stdx.debug.panic("Paged attention is not supported on {s} yet", .{@tagName(platform.target)}), }; } @@ -41,7 +45,8 @@ pub const Backend = enum { return switch (backend) { .cute => cute.isAvailable(platform), .stablehlo => true, - .triton => platform.target != .cpu, + .tcl => platform.target == .furiosa, + .triton => platform.target != .cpu and platform.target != .furiosa, .metal => platform.target == .metal, .mosaic_tpu => platform.target == .tpu, }; @@ -54,6 +59,7 @@ pub const Options = union(Backend) { mosaic_tpu: tpu.mosaic_tpu.Options, metal: metal.Options, stablehlo: triton.paged.Options, + tcl: tcl.paged.Options, const Args = struct { backend: Backend, @@ -84,7 +90,7 @@ pub const Options = union(Backend) { .head_dim = args.head_dim, }, }, - inline .cute, .triton, .metal, .stablehlo => |t| @unionInit(Options, @tagName(t), .{ + inline .cute, .triton, .metal, .stablehlo, .tcl => |t| @unionInit(Options, @tagName(t), .{ .batch_size = args.batch_size, .max_num_pages = args.max_num_pages, .max_seqlen_q = args.max_seqlen_q, @@ -112,6 +118,7 @@ pub const Parameters = union(Backend) { mosaic_tpu: tpu.mosaic_tpu.Parameters, metal: metal.Parameters, stablehlo: triton.paged.Parameters, + tcl: tcl.paged.Parameters, pub fn init(options_: Options) Parameters { return switch (options_) { @@ -196,7 +203,7 @@ pub const KvCache = union(enum) { const kv: KvCache = switch (self) { .split => |split| switch (backend) { - .cute, .triton, .mosaic_tpu, .metal, .stablehlo => .{ + .cute, .triton, .mosaic_tpu, .metal, .stablehlo, .tcl => .{ .split = .{ .k = split.k.scatterSlices( .{ .page = page_index, .k_chunk = offset }, @@ -258,6 +265,10 @@ pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: z .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), }, .stablehlo => |params| stablehlo.pagedAttention(params, q, kv_cache, opts), + .tcl => |params| switch (kv_cache) { + .split => |split| tcl.paged.pagedAttention(params, q, split.k, split.v, opts), + .dense => std.debug.panic("TCL paged attention takes separate K and V pages", .{}), + }, }; } @@ -299,6 +310,7 @@ test "Backend.auto selects triton on oneAPI" { test pagedAttention { const platform = zml.testing.env(); + if (!Backend.triton.isAvailable(platform)) return error.SkipZigTest; const io = std.testing.io; const allocator = std.testing.allocator; var arena_state: std.heap.ArenaAllocator = .init(allocator); @@ -400,7 +412,7 @@ test pagedAttention { const all_backends = std.enums.values(Backend); // stablehlo ignores these options, mosaic_tpu requires a dense // KV cache instead of the split cache used by this test. - const option_sensitive_backends = [_]Backend{ .triton, .cute, .metal }; + const option_sensitive_backends = [_]Backend{ .triton, .cute, .metal, .tcl }; const test_cases = [_]struct { name: []const u8, attention_options: AttentionOptions, @@ -453,14 +465,14 @@ test pagedAttention { const parameters_d: zml.Bufferized(Parameters) = switch (parameters) { .triton => triton_parameters_d, - inline .cute, .metal, .mosaic_tpu, .stablehlo => |_, t| @unionInit(zml.Bufferized(Parameters), @tagName(t), .{ + inline .cute, .metal, .mosaic_tpu, .stablehlo, .tcl => |_, t| @unionInit(zml.Bufferized(Parameters), @tagName(t), .{ .block_table = triton_parameters_d.triton.block_table, .seq_lens = triton_parameters_d.triton.seq_lens, .query_start_len = triton_parameters_d.triton.query_start_len, }), }; - var output_d = try exe.eval(allocator, io, .{ parameters_d, q, new_k, new_v, kv_cache_d, .{} }); + var output_d = try exe.eval(allocator, io, .{ parameters_d, q, new_k, new_v, kv_cache_d, .{ .mask = .{ .standard = {} }, .sink = null } }); defer output_d.deinit(); results_per_backend.set(backend, try output_d.toSliceAlloc(allocator, io)); } diff --git a/zml/kernels/paged_attention/tcl/BUILD.bazel b/zml/kernels/paged_attention/tcl/BUILD.bazel new file mode 100644 index 00000000..4057eea3 --- /dev/null +++ b/zml/kernels/paged_attention/tcl/BUILD.bazel @@ -0,0 +1,6 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "tcl", + srcs = ["tcl.zig"], +) diff --git a/zml/kernels/paged_attention/tcl/tcl.zig b/zml/kernels/paged_attention/tcl/tcl.zig new file mode 100644 index 00000000..f50db773 --- /dev/null +++ b/zml/kernels/paged_attention/tcl/tcl.zig @@ -0,0 +1,1293 @@ +//! Furiosa RNGD paged attention in TCL, using the common/flash attention +//! algorithms from Furiosa's SDK (`furiosa-kernels 2026.4.0` reference). +//! This implementation retains FP32 normalization/accumulation and BF16 +//! contractions. The exact common SDK operation materializes scores in BF16; +//! it fails the required unit-scale tolerance and is not a selectable backend. +//! +//! Queries are cut in blocks of rows of one sequence (one token per block for +//! decode). A context of at most `chunk_keys` keys runs Furiosa's one-shot +//! attention: gather the keys by token slot, contract, mask, one softmax, +//! contract. Longer contexts run Furiosa's flash attention: a device loop over +//! chunks of `chunk_keys` keys up to the longest sequence, gathering whole +//! pages and accumulating the softmax statistics in f32. +//! +//! Decode without sinks or windows, up to `decode_keys` keys, instead runs one +//! fusible kernel per layer over the whole pages each token sees, with the +//! token metadata every layer shares computed once per forward pass. +const std = @import("std"); + +const stdx = @import("stdx"); +const builder = @import("dsl/tcl/builder"); + +const zml = @import("../../../zml.zig"); +const Tensor = zml.Tensor; +const tcl = zml.kernel.tcl; +const paged_attention = @import("../paged_attention.zig"); +const triton = @import("../triton/triton.zig"); + +const chunk_keys = 4096; +const prefill_block = 32; + +pub const paged = struct { + pub const Options = triton.paged.Options; + pub const Parameters = triton.paged.Parameters; + + /// `q` is `{.b = tokens, .hkv, .hg, .hd}`, the caches + /// `{.page, .k_chunk, .hkv, .hd}`. Sinks are one logit per query head. + /// Unwritten cache entries are excluded before the contractions; callers + /// need only populate the tokens covered by their sequence lengths. + pub fn pagedAttention(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: paged_attention.AttentionOptions) Tensor { + if (opts.mask == .indices) @panic("Sparse attention masks are not supported by TCL paged attention"); + stdx.debug.assert(q.rank() == 4 and k_cache.rank() == 4 and v_cache.rank() == 4 and k_cache.shape().eqlDims(v_cache.shape()), "TCL attention requires matching rank-four split K/V caches", .{}); + stdx.debug.assert(q.dtype().isFloat() and k_cache.dtype().isFloat() and v_cache.dtype().isFloat(), "TCL attention requires floating-point queries and caches", .{}); + stdx.debug.assert(q.dim(.hkv) == k_cache.dim(.hkv) and q.dim(.hd) == k_cache.dim(.hd) and q.dim(.hg) > 0, "TCL attention query and cache head dimensions must match", .{}); + stdx.debug.assert(parameters.block_table.dtype() == .i32 and parameters.seq_lens.dtype() == .i32 and parameters.query_start_len.dtype() == .i32, "TCL attention metadata must be I32", .{}); + stdx.debug.assert(parameters.block_table.dim(.b) == parameters.seq_lens.dim(.b) and parameters.query_start_len.dim(.b) == parameters.seq_lens.dim(.b) + 1, "TCL attention metadata batch dimensions must match", .{}); + stdx.debug.assert(parameters.seq_lens.dim(.b) > 0 and parameters.block_table.dim(.p) > 0 and k_cache.dim(.page) > 0 and k_cache.dim(.k_chunk) > 0, "TCL attention requires nonempty sequence and cache capacities", .{}); + stdx.debug.assert(k_cache.dim(.page) * k_cache.dim(.k_chunk) <= std.math.maxInt(i32) and q.dim(.b) <= std.math.maxInt(i32), "TCL attention indices must fit in I32", .{}); + const compiler = zml.Compiler.current(); + stdx.debug.assert(compiler.platform.target == .furiosa and compiler.meshes[0].numDevices() == 1, "TCL attention requires one Furiosa chip", .{}); + const attributes = compiler.platform.devices[0].pjrt_device.attributes(compiler.platform.pjrt_api) catch + std.debug.panic("TCL attention could not verify the Furiosa device's PE range", .{}); + defer attributes.deinit(); + const first_pe = attributes.get("pe_start"); + const last_pe = attributes.get("pe_end"); + stdx.debug.assert(first_pe != null and first_pe.? == .int64 and first_pe.?.int64 == 0 and + last_pe != null and last_pe.? == .int64 and last_pe.?.int64 == 7, "TCL attention requires a complete 8-PE RNGD chip", .{}); + if (q.dim(.b) == 0) return Tensor.zeroes(q.shape()); + const page = k_cache.dim(.k_chunk); + // TCC 2026.5 has no DPE tactic for 32-key attention. Pad to + // at least 64 keys; the extra pages remain invalid and contribute zero. + const keys = @max(parameters.block_table.dim(.p), std.math.divCeil(i64, 64, page) catch unreachable) * page; + // Chunks hold whole pages and a multiple of 64 keys: TCC 2026.5 fails + // to allocate a chunk of 4080 keys (85 pages of 48). + const key_alignment = @divExact(page, @as(i64, @intCast(std.math.gcd(@as(u64, @intCast(page)), 64)))) * 64; + const alignment = if (key_alignment <= chunk_keys) key_alignment else page; + const chunk = if (keys <= chunk_keys) keys else @min(keys, @max(1, @divTrunc(chunk_keys, alignment)) * alignment); + const decode = !parameters.options_.is_prefill and q.dim(.b) == parameters.seq_lens.dim(.b); + const blocks = (std.math.divCeil(i64, q.dim(.b), prefill_block) catch unreachable) + parameters.seq_lens.dim(.b); + // Regular block capacities limit layout-search cost for the fused graph. + // Extra blocks retain zero valid lengths and invalid query positions. + const cfg: Cfg = .{ + .tokens = q.dim(.b), + .sequences = parameters.seq_lens.dim(.b), + .b = if (decode) q.dim(.b) else @intCast(std.math.ceilPowerOfTwo(u64, @intCast(blocks)) catch unreachable), + .i = if (decode) 1 else prefill_block, + .a = (std.math.divCeil(i64, keys, chunk) catch unreachable) * chunk, + .chunk = chunk, + .hkv = q.dim(.hkv), + .g = q.dim(.hg), + .d = q.dim(.hd), + .pages = k_cache.dim(.page), + .pg = page, + .scale = opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))), + .decode = decode, + .use_sink = opts.sink != null, + .is_causal = opts.mask.isCausal(), + .window = @intCast(opts.mask.slidingWindow() orelse 0), + }; + if (decode and !cfg.use_sink and cfg.window == 0 and keys <= decode_keys) { + return decodeAttention(parameters, q, k_cache, v_cache, .{ + .t = cfg.tokens, + .a = keys, + .hkv = cfg.hkv, + .g = cfg.g, + .d = cfg.d, + .pages = cfg.pages, + .pg = page, + .scale = cfg.scale, + .is_causal = cfg.is_causal, + }).convert(q.dtype()); + } + const table = parameters.block_table.pad(-1, .{ .p = Tensor.Pad{ .high = @divExact(cfg.a, page) - parameters.block_table.dim(.p) } }); + const inputs: OneShot.Inputs = .{ + .q = q.convert(.bf16), + .pages = table, + .seq_lens = parameters.seq_lens, + .query_offsets = parameters.query_start_len, + .k_cache = k_cache.convert(.bf16), + .v_cache = v_cache.convert(.bf16), + .sink = if (opts.sink) |s| s.convert(.f32).reshape(.{ .hkv = cfg.hkv, .hg = cfg.g }) else Tensor.zeroes(.init(.{ .hkv = cfg.hkv, .hg = cfg.g }, .f32)), + }; + const outputs: OneShot.Outputs = .{ .out = q.shape().withDtype(.bf16) }; + const out = if (cfg.a == cfg.chunk) + OneShot.call(inputs, outputs, .{ .cfg = cfg }).out + else + Flash.call(inputs, outputs, .{ .cfg = cfg }).out; + return out.convert(q.dtype()); + } +}; + +/// Keys a decode kernel attends to in one pass: TCC 2026.5 fails on the +/// decode metadata for 131072 keys in pages of 16. +const decode_keys = 65536; + +const DecodeCfg = struct { + t: i64, + a: i64, + hkv: i64, + g: i64, + d: i64, + pages: i64, + pg: i64, + scale: f32, + is_causal: bool, +}; + +/// Decode attention in Furiosa's form: the kernel gathers whole pages, then +/// runs one contraction, a masked softmax and a second contraction. It reads +/// past each sequence's end, selecting zero for those unused cache entries. +/// The token metadata depends only on the attention parameters, which every +/// layer shares: one kernel computes it once per forward pass. +fn decodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, cfg: DecodeCfg) Tensor { + const columns = @divExact(cfg.a, cfg.pg); + const table = parameters.block_table.pad(0, .{ .p = Tensor.Pad{ .high = columns - parameters.block_table.dim(.p) } }); + const meta = DecodeMeta.call(.{ .seq_lens = parameters.seq_lens, .query_offsets = parameters.query_start_len, .table = table }, .{ + .pages = zml.Shape.init(.{ .t = cfg.t, .p = columns }, .i32), + .bounds = zml.Shape.init(.{ .t = cfg.t }, .i32), + .mask = zml.Shape.init(.{ .t = cfg.t, .a = cfg.a }, .i32), + .keyed = zml.Shape.init(.{ .t = cfg.t }, .f32), + }, .{ .cfg = .{ .t = cfg.t, .s = parameters.seq_lens.dim(.b), .ap = columns, .pg = cfg.pg, .pages = cfg.pages, .is_causal = cfg.is_causal } }); + return Decode.call(.{ + .q = q.convert(.bf16), + .pages = meta.pages, + .bounds = meta.bounds, + .mask = meta.mask, + .keyed = meta.keyed, + .k_cache = k_cache.convert(.bf16), + .v_cache = v_cache.convert(.bf16), + }, .{ .out = q.shape().withDtype(.bf16) }, .{ .cfg = cfg }).out; +} + +const MetaCfg = struct { t: i64, s: i64, ap: i64, pg: i64, pages: i64, is_causal: bool }; + +/// For each token: the cache page of each column, how many columns hold keys +/// it sees, the keys it sees (below its limit, in pages inside the cache) and +/// whether it sees any. Columns past the sequence's +/// pages read page `p`, so that the masked padding spreads over many pages +/// instead of hammering one. Rows are selected by one-hot products and +/// comparisons follow `attentionMask`'s clamp arithmetic: TCC 2026.5 has no +/// tactics for small integer gathers and fails on the plugin's lowering of the +/// same comparisons for several tokens. +const DecodeMeta = tcl.Kernel(MetaCfg, .{ + .name = "paged_decode_metadata", + .inputs = &.{ "seq_lens", "query_offsets", "table" }, + .outputs = &.{ "pages", "bounds", "mask", "keyed" }, + .run = struct { + fn run(b: *tcl.Builder, cfg: MetaCfg) tcl.FinishError!void { + const T = b.axis("T", cfg.t); + const S = b.axis("S", cfg.s); + const O = b.axis("O", cfg.s + 1); + const AP = b.axis("AP", cfg.ap); + const PG = b.axis("PG", cfg.pg); + const A = b.axis("A", cfg.ap * cfg.pg); + const t = try b.declareArgs(.{ + .seq_lens = .{ .dtype = .i32, .axes = &.{S} }, + .query_offsets = .{ .dtype = .i32, .axes = &.{O} }, + .table = .{ .dtype = .i32, .axes = &.{ S, AP } }, + }); + const ends = try selectMetadata(b, t.query_offsets, try selector(b, try b.arange(cfg.s + 1, .i32, &.{S}, .{ .start = 1 }), O, cfg.s + 1), O); + const tokens = try b.arange(cfg.t, .i32, &.{T}, .{}); + // Padding tokens follow every sequence's end and belong to none. + const seq = try countBefore(b, ends, tokens, S); + const own = try selector(b, seq, S, cfg.s); + var op = b.tensorOperation(.{}); + const active = try op.commit(op.fetch(seq, .{}).muli(-1).addi(cfg.s).binary(.maxi, 0).binary(.mini, 1), .{}); + var visible = try selectMetadata(b, t.seq_lens, own, S); + if (cfg.is_causal) { + // Causal tokens see their sequence up to their own position. + const end = try selectMetadata(b, ends, own, S); + op = b.tensorOperation(.{}); + visible = try op.commit(op.fetch(visible, .{}).subi(op.fetch(end, .{})).addi(op.fetch(tokens, .{})).addi(1), .{}); + } + op = b.tensorOperation(.{}); + const limit = try op.commit(op.fetch(visible, .{}).binary(.maxi, 0).binary(.mini, cfg.ap * cfg.pg).muli(op.fetch(active, .{})), .{}); + const entries = try selectMetadata(b, t.table, own, S); + const pages = try validIndices(b, entries, cfg.pages); + const firsts = try b.arange(cfg.ap * cfg.pg, .i32, &.{AP}, .{ .step = cfg.pg }); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const used = try op.commit(op.fetch(limit, .{}).subi(op.fetch(firsts, .{})).binary(.maxi, 0).binary(.mini, 1), .{}); + op = b.tensorOperation(.{}); + const bounds = try op.commit(op.fetch(used, .{}).reduce(&.{AP}, .addi), .{}); + op = b.tensorOperation(.{}); + const spread = try op.commit(op.fetch(try b.arange(cfg.ap, .i32, &.{AP}, .{}), .{}).binary(.mini, cfg.pages - 1), .{}); + op = b.tensorOperation(.{}); + const read = try op.commit(op.where(op.fetch(used, .{}), .ne, 0, op.fetch(pages.safe, .{}), op.fetch(spread, .{ .broadcast_to = &.{ T, AP } })), .{}); + // Tokens without keys have no defined attention: the attention + // kernel zeroes them. + op = b.tensorOperation(.{}); + const keyed = try op.commit(op.fetch(used, .{}).muli(op.fetch(pages.valid, .{})).reduce(&.{AP}, .maxi).toFloat(), .{}); + op = b.tensorOperation(.{}); + const valid = try op.commit(op.fetch(pages.valid, .{ .broadcast_to = &.{ T, AP, PG } }), .{}); + op = b.tensorOperation(.{}); + const position = try op.commit(op.fetch(try b.arange(cfg.ap * cfg.pg, .i32, &.{A}, .{}), .{ .broadcast_to = &.{ T, A } }), .{}); + op = b.tensorOperation(.{ .tactic = .elementwise }); + const mask = try op.commit(op.fetch(position, .{}).muli(-1).addi(op.fetch(limit, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(try b.reshape(valid, &.{ T, A }, .{}), .{})), .{}); + b.ret(&.{ read, bounds, mask, keyed }); + } + }.run, +}); + +/// Gathers the key and value pages a token sees, contracts with the query, +/// masks keys past the sequence, normalizes in one softmax times `keyed` (zero +/// for tokens without keys, whose softmax is uniform) and contracts with the +/// values. A token's pages are gathered up to its bound, and the operations +/// reading them skip the rest: the work follows the sequence's length, not the +/// table's. Fusible: it joins the layer around it, one launch per layer. +const Decode = tcl.Kernel(DecodeCfg, .{ + .name = "paged_decode_attention", + .inputs = &.{ "q", "pages", "bounds", "mask", "keyed", "k_cache", "v_cache" }, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: DecodeCfg) tcl.FinishError!void { + b.fuse(); + configure(b, .{ .tokens = cfg.t, .sequences = cfg.t, .b = cfg.t, .i = 1, .a = cfg.a, .chunk = cfg.a, .hkv = cfg.hkv, .g = cfg.g, .d = cfg.d, .pages = cfg.pages, .pg = cfg.pg, .scale = cfg.scale, .decode = true }); + const T = b.axis("T", cfg.t); + const A = b.axis("A", cfg.a); + const AP = b.axis("AP", @divExact(cfg.a, cfg.pg)); + const Hkv = b.axis("Hkv", cfg.hkv); + const G = b.axis("G", cfg.g); + const D = b.axis("D", cfg.d); + const P = b.axis("P", cfg.pages); + const PG = b.axis("PG", cfg.pg); + const t = try b.declareArgs(.{ + .q = .{ .dtype = .bf16, .axes = &.{ T, Hkv, G, D } }, + .pages = .{ .dtype = .i32, .axes = &.{ T, AP } }, + .bounds = .{ .dtype = .i32, .axes = &.{T} }, + .mask = .{ .dtype = .i32, .axes = &.{ T, A } }, + .keyed = .{ .dtype = .f32, .axes = &.{T} }, + .k_cache = .{ .dtype = .bf16, .axes = &.{ P, PG, Hkv, D } }, + .v_cache = .{ .dtype = .bf16, .axes = &.{ P, PG, Hkv, D } }, + }); + const mask = try b.reshape(t.mask, &.{ T, AP, PG }, .{}); + const k = try maskCacheValues(b, try b.gather(t.k_cache, t.pages, P, &.{ T, AP, PG, Hkv, D }, .{ .valid_length = t.bounds }), mask); + const v = try maskCacheValues(b, try b.gather(t.v_cache, t.pages, P, &.{ T, AP, PG, Hkv, D }, .{ .valid_length = t.bounds }), mask); + const keys: []const tcl.Axis = &.{ AP, PG }; + var op = b.tensorOperation(.{}); + const scores = try op.commit(op.contract(t.q, k, &.{ Hkv, G, T, AP, PG }).mulf(cfg.scale), .{}); + op = b.tensorOperation(.{}); + const s = op.fetch(scores, .{}); + const masked = try op.commit(op.where(op.fetch(mask, .{}), .ne, 0, s, score_min), .{}); + op = b.tensorOperation(.{}); + const x = op.fetch(masked, .{}); + const e = x.subf(x.reduce(keys, .maxf)).exp(); + const p = try op.commit(e.divf(e.reduce(keys, .addf)).mulf(op.fetch(t.keyed, .{})), .{ .dtype = .bf16 }); + op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.contract(v, p, &.{ T, Hkv, G, D }), .{ .dtype = .bf16 })}); + } + }.run, +}); + +const Cfg = struct { + tokens: i64, + sequences: i64, + /// Query blocks and query rows per block. + b: i64, + i: i64, + /// Keys per block (padded to whole chunks) and keys per chunk. + a: i64, + chunk: i64, + hkv: i64, + g: i64, + d: i64, + pages: i64, + pg: i64, + scale: f32, + decode: bool, + use_sink: bool = false, + is_causal: bool = true, + window: i32 = -1, +}; + +/// Attention tuning inherited from the original TCL backend, informed by +/// `furiosa.kernels.llama.config` and `common.attention`. Keep the compiler's +/// default lowering mode and omit the adaptive-einsum and half-VRF settings +/// removed in TCC 2026.5. +fn configure(b: *tcl.Builder, cfg: Cfg) void { + if (cfg.decode) b.compilerConfig(.{ + .enable_einsum_fusion = true, + .padding_policy = .{ .Small = 1.1 }, + .instruction_mem_budget = 0xB0000, + .reshape_einsum_mode = .{ .Reshape = .{ .permute = false } }, + .tensor_unit_bridge_threshold_in_page = 12, + .allow_external_operators = false, + .enable_tactic_pruning = false, + .scheduler_beam_search = true, + .allow_reduce_by_ve_cluster_chip_reduce = false, + .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, + .propagate_sparse_axis_from_op = "Gather", + .tactic_hint = .{ .ForLlmModelIOBound = 200 }, + .num_transaction_simulation_per_pe = 1024, + .dma_preference = 0.8, + }) else b.compilerConfig(.{ + .enable_einsum_fusion = true, + .padding_policy = .{ .Small = 1.1 }, + .instruction_mem_budget = 0xB0000, + .reshape_einsum_mode = .{ .Reshape = .{ .permute = false } }, + .tensor_unit_bridge_threshold_in_page = 12, + .allow_external_operators = false, + .enable_tactic_pruning = false, + .scheduler_beam_search = true, + .allow_reduce_by_ve_cluster_chip_reduce = false, + .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, + .tactic_hint = .{ .ForLlmModelComputeBound = 200 }, + .dma_preference = 1.2, + }); +} + +const Axes = struct { + T: tcl.Axis, + S: tcl.Axis, + O: tcl.Axis, + AP: tcl.Axis, + B: tcl.Axis, + I: tcl.Axis, + Hkv: tcl.Axis, + G: tcl.Axis, + D: tcl.Axis, + P: tcl.Axis, + PG: tcl.Axis, + + fn init(b: *tcl.Builder, cfg: Cfg) Axes { + return .{ + .T = b.axis("T", cfg.tokens), + .S = b.axis("S", cfg.sequences), + .O = b.axis("O", cfg.sequences + 1), + .AP = b.axis("AP", @divExact(cfg.a, cfg.pg)), + .B = b.axis("B", cfg.b), + .I = b.axis("I", cfg.i), + .Hkv = b.axis("Hkv", cfg.hkv), + .G = b.axis("G", cfg.g), + .D = b.axis("D", cfg.d), + .P = b.axis("P", cfg.pages), + .PG = b.axis("PG", cfg.pg), + }; + } +}; + +const NativeInputs = struct { + q: builder.Tensor, + pages: builder.Tensor, + seq_lens: builder.Tensor, + query_offsets: builder.Tensor, + k_cache: builder.Tensor, + v_cache: builder.Tensor, + sink: builder.Tensor, +}; + +fn declareInputs(b: *tcl.Builder, a: Axes) tcl.FinishError!NativeInputs { + const t = try b.declareArgs(.{ + .q = .{ .dtype = .bf16, .axes = &.{ a.T, a.Hkv, a.G, a.D } }, + .pages = .{ .dtype = .i32, .axes = &.{ a.S, a.AP } }, + .seq_lens = .{ .dtype = .i32, .axes = &.{a.S} }, + .query_offsets = .{ .dtype = .i32, .axes = &.{a.O} }, + .k_cache = .{ .dtype = .bf16, .axes = &.{ a.P, a.PG, a.Hkv, a.D } }, + .v_cache = .{ .dtype = .bf16, .axes = &.{ a.P, a.PG, a.Hkv, a.D } }, + .sink = .{ .dtype = .f32, .axes = &.{ a.Hkv, a.G } }, + }); + return .{ .q = t.q, .pages = t.pages, .seq_lens = t.seq_lens, .query_offsets = t.query_offsets, .k_cache = t.k_cache, .v_cache = t.v_cache, .sink = t.sink }; +} + +const Prepared = struct { + q: builder.Tensor, + pages: builder.Tensor, + qpos: builder.Tensor, + kvlen: builder.Tensor, + row: builder.Tensor, + active: builder.Tensor, +}; + +fn countBefore(b: *tcl.Builder, ends: builder.Tensor, positions: builder.Tensor, axis: tcl.Axis) tcl.FinishError!builder.Tensor { + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const distance = try op.commit(op.fetch(ends, .{}).subi(op.fetch(positions, .{})), .{}); + op = b.tensorOperation(.{}); + const passed = try op.commit(op.where(distance, .le, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + return op.commit(op.fetch(passed, .{}).reduce(&.{axis}, .addi), .{}); +} + +fn selector(b: *tcl.Builder, indices: builder.Tensor, axis: tcl.Axis, count: i64) tcl.FinishError!builder.Tensor { + const source = try b.arange(count, .i32, &.{axis}, .{}); + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const distance = try op.commit(op.fetch(indices, .{}).subi(op.fetch(source, .{})), .{}); + op = b.tensorOperation(.{}); + return op.commit(op.where(distance, .eq, 0, 1, 0), .{}); +} + +// Keep metadata exact in I32. Scalar gathers and four-byte-offset slices do +// not have native tactics for these shapes in TCC 2026.5. +fn selectMetadata(b: *tcl.Builder, values: builder.Tensor, selected: builder.Tensor, axis: tcl.Axis) tcl.FinishError!builder.Tensor { + const op = b.tensorOperation(.{ .tactic = if (values.axes.len > 1) .einsum_by_ve else .reduce_by_ve }); + return op.commit(op.fetch(selected, .{}).muli(op.fetch(values, .{})).reduce(&.{axis}, .addi), .{}); +} + +/// Keep adjacent I32 reads inside the native unit. Exposing offsets[1:] as a +/// separate argument would require alignment staging at every invocation. +fn prepareQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, t: NativeInputs) tcl.FinishError!Prepared { + const start_indices = try b.arange(cfg.sequences, .i32, &.{a.S}, .{}); + const end_indices = try b.arange(cfg.sequences + 1, .i32, &.{a.S}, .{ .start = 1 }); + const starts = try selectMetadata(b, t.query_offsets, try selector(b, start_indices, a.O, cfg.sequences + 1), a.O); + const ends = try selectMetadata(b, t.query_offsets, try selector(b, end_indices, a.O, cfg.sequences + 1), a.O); + var op = b.tensorOperation(.{}); + const qlen = try op.commit(op.fetch(ends, .{}).subi(op.fetch(starts, .{})), .{}); + const tokens = try b.arange(cfg.tokens, .i32, &.{a.T}, .{}); + const seq_of_token = try countBefore(b, ends, tokens, a.S); + // All sequence ends precede a padding token. Reuse this vector instead of + // bringing a symbolic scalar reduction back into a vector operation. + op = b.tensorOperation(.{}); + const token_tail = try op.commit(op.fetch(seq_of_token, .{}).subi(cfg.sequences), .{}); + op = b.tensorOperation(.{}); + const active = try op.commit(op.where(token_tail, .lt, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const token_seq = try op.commit(op.fetch(seq_of_token, .{}).binary(.mini, cfg.sequences - 1), .{}); + if (cfg.decode) { + op = b.tensorOperation(.{}); + const live = try op.commit(op.where(qlen, .gt, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const lengths = try op.commit(op.fetch(t.seq_lens, .{}).muli(op.fetch(live, .{})), .{}); + op = b.tensorOperation(.{}); + const qpos = try op.commit(op.fetch(lengths, .{}).subi(1), .{}); + op = b.tensorOperation(.{}); + const safe_start = try op.commit(op.fetch(starts, .{}).binary(.mini, cfg.tokens - 1), .{}); + const q = if (cfg.tokens == 1) t.q else try b.gather(t.q, safe_start, a.T, &.{ a.S, a.Hkv, a.G, a.D }, .{}); + return .{ + .q = try b.reshape(q, &.{ a.B, a.I, a.Hkv, a.G, a.D }, .{}), + .pages = try b.reshape(t.pages, &.{ a.B, a.AP }, .{}), + .qpos = try b.reshape(qpos, &.{ a.B, a.I }, .{}), + .kvlen = try b.reshape(lengths, &.{a.B}, .{}), + .row = token_seq, + .active = active, + }; + } + + const shift = std.math.log2_int(u64, prefill_block); + op = b.tensorOperation(.{}); + const blocks = try op.commit(op.fetch(qlen, .{}).addi(prefill_block - 1).binary(.shr_arith, shift), .{}); + op = b.tensorOperation(.{}); + const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{a.S}), .{}); + op = b.tensorOperation(.{}); + const block_start = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})), .{}); + const block_ids = try b.arange(cfg.b, .i32, &.{a.B}, .{}); + const seq_of_block = try countBefore(b, block_end, block_ids, a.S); + op = b.tensorOperation(.{}); + const beyond = try op.commit(op.fetch(seq_of_block, .{}).subi(cfg.sequences), .{}); + op = b.tensorOperation(.{}); + const live_block = try op.commit(op.where(beyond, .lt, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const seq = try op.commit(op.fetch(seq_of_block, .{}).binary(.mini, cfg.sequences - 1), .{}); + const pick = try selector(b, seq, a.S, cfg.sequences); + const block_begin = try selectMetadata(b, block_start, pick, a.S); + const begin = try selectMetadata(b, starts, pick, a.S); + const query_length = try selectMetadata(b, qlen, pick, a.S); + const key_length = try selectMetadata(b, t.seq_lens, pick, a.S); + const local_rows = try b.arange(cfg.i, .i32, &.{a.I}, .{}); + op = b.tensorOperation(.{}); + const first_row = try op.commit(op.fetch(block_ids, .{}).subi(op.fetch(block_begin, .{})).muli(prefill_block), .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const qi = try op.commit(op.fetch(first_row, .{}).addi(op.fetch(local_rows, .{})), .{}); + op = b.tensorOperation(.{}); + const remaining = try op.commit(op.fetch(qi, .{}).subi(op.fetch(query_length, .{})), .{}); + op = b.tensorOperation(.{}); + const inside = try op.commit(op.where(remaining, .lt, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const live = try op.commit(op.fetch(inside, .{}).muli(op.fetch(live_block, .{})), .{}); + op = b.tensorOperation(.{}); + const qidx = try op.commit(op.fetch(qi, .{}).addi(op.fetch(begin, .{})).muli(op.fetch(live, .{})), .{}); + const q = try b.gather(t.q, qidx, a.T, &.{ a.B, a.I, a.Hkv, a.G, a.D }, .{}); + op = b.tensorOperation(.{}); + const pos = try op.commit(op.fetch(qi, .{}).addi(op.fetch(key_length, .{})).subi(op.fetch(query_length, .{})), .{}); + op = b.tensorOperation(.{}); + const qpos = try op.commit(op.where(live, .ne, 0, op.fetch(pos, .{}), -1), .{}); + op = b.tensorOperation(.{}); + const kvlen = try op.commit(op.fetch(key_length, .{}).muli(op.fetch(live_block, .{})), .{}); + const table = try selectMetadata(b, t.pages, pick, a.S); + + const token_pick = try selector(b, token_seq, a.S, cfg.sequences); + const token_begin = try selectMetadata(b, starts, token_pick, a.S); + const token_block = try selectMetadata(b, block_start, token_pick, a.S); + op = b.tensorOperation(.{}); + const row = try op.commit(op.fetch(token_block, .{}).muli(prefill_block).addi(op.fetch(tokens, .{})).subi(op.fetch(token_begin, .{})).binary(.maxi, 0).binary(.mini, cfg.b * cfg.i - 1), .{}); + return .{ .q = q, .pages = table, .qpos = qpos, .kvlen = kvlen, .row = row, .active = active }; +} + +fn restoreQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, prepared: Prepared, output: builder.Tensor) tcl.FinishError!builder.Tensor { + if (cfg.decode and cfg.tokens == 1) return b.reshape(output, &.{ a.T, a.Hkv, a.G, a.D }, .{}); + const R = b.axis("R", cfg.b * cfg.i); + const rows = try b.reshape(output, &.{ R, a.Hkv, a.G, a.D }, .{}); + const ordered = try b.gather(rows, prepared.row, R, &.{ a.T, a.Hkv, a.G, a.D }, .{}); + const op = b.tensorOperation(.{ .tactic = .elementwise }); + // Establish the full data layout before broadcasting the I32 row mask. + const values = op.fetch(ordered, .{}); + return op.commit(op.where(op.fetch(prepared.active, .{ .broadcast_to = ordered.axes }), .ne, 0, values, 0.0), .{ .dtype = .bf16 }); +} + +const score_min = -std.math.floatMax(f32); + +fn validIndices(b: *tcl.Builder, indices: builder.Tensor, count: i64) tcl.FinishError!struct { safe: builder.Tensor, valid: builder.Tensor } { + var op = b.tensorOperation(.{}); + const safe = try op.commit(op.fetch(indices, .{}).binary(.maxi, 0).binary(.mini, count - 1), .{}); + op = b.tensorOperation(.{}); + const nonnegative = try op.commit(op.fetch(indices, .{}).addi(1).binary(.maxi, 0).binary(.mini, 1), .{}); + op = b.tensorOperation(.{ .tactic = .elementwise }); + const valid = try op.commit(op.fetch(indices, .{}).muli(-1).addi(count).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(nonnegative, .{})), .{}); + return .{ .safe = safe, .valid = valid }; +} + +/// Cache validity is per sequence, not per query: causal/window masking stays +/// on scores, without replicating K/V for every row of a prefill block. +fn cacheMask(b: *tcl.Builder, valid_pages: builder.Tensor, kvlen: builder.Tensor, position: builder.Tensor) tcl.FinishError!builder.Tensor { + var op = b.tensorOperation(.{}); + const pos = try op.commit(op.fetch(position, .{ .broadcast_to = valid_pages.axes }), .{}); + op = b.tensorOperation(.{ .tactic = .elementwise }); + return op.commit(op.fetch(pos, .{}).muli(-1).addi(op.fetch(kvlen, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(valid_pages, .{})), .{}); +} + +/// A true select excludes NaN/Inf in unwritten slots; multiplying by zero +/// would still propagate them into the contraction. Materialize only BF16. +fn maskCacheValues(b: *tcl.Builder, values: builder.Tensor, valid: builder.Tensor) tcl.FinishError!builder.Tensor { + const op = b.tensorOperation(.{ .tactic = .elementwise }); + const data = op.fetch(values, .{}); + return op.commit(op.where(op.fetch(valid, .{ .broadcast_to = values.axes }), .ne, 0, data, 0.0), .{ .dtype = .bf16 }); +} + +/// Build the sequence, query, page and window mask for the current keys. +fn attentionMask(b: *tcl.Builder, cfg: Cfg, qpos: builder.Tensor, kvlen: builder.Tensor, position: builder.Tensor, valid_keys: builder.Tensor) tcl.FinishError!builder.Tensor { + const axes = &.{ qpos.axes[0], qpos.axes[1], position.axes[0] }; + const read: builder.TensorOperation.FetchOptions = .{ .broadcast_to = axes }; + // Elementwise operations require their first read to cover the output + // axes, and each VE instruction must consume the current accumulator. + var op = b.tensorOperation(.{}); + const pos = try op.commit(op.fetch(position, read), .{}); + op = b.tensorOperation(.{}); + const query = try op.commit(op.fetch(qpos, read), .{}); + op = b.tensorOperation(.{}); + var mask = try op.commit(op.fetch(valid_keys, read), .{}); + op = b.tensorOperation(.{ .tactic = .elementwise }); + mask = try op.commit(op.fetch(pos, .{}).muli(-1).addi(op.fetch(kvlen, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(mask, .{})), .{}); + op = b.tensorOperation(.{ .tactic = .elementwise }); + mask = try op.commit(op.fetch(query, .{}).addi(1).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(mask, .{})), .{}); + if (cfg.is_causal) { + op = b.tensorOperation(.{ .tactic = .elementwise }); + mask = try op.commit(op.fetch(query, .{}).subi(op.fetch(pos, .{})).addi(1).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(mask, .{})), .{}); + } + if (cfg.window > 0) { + op = b.tensorOperation(.{ .tactic = .elementwise }); + mask = try op.commit(op.fetch(pos, .{}).subi(op.fetch(query, .{})).addi(cfg.window).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(mask, .{})), .{}); + } + return mask; +} + +/// Keep the contraction accumulator in f32 through masking and softmax. +/// A finite sentinel also makes entirely masked chunks safe to rescale. +fn maskedScores(b: *tcl.Builder, a: Axes, cfg: Cfg, q: builder.Tensor, k: builder.Tensor, mask: builder.Tensor, keys: tcl.Axis) tcl.FinishError!builder.Tensor { + const scores_axes = &.{ a.Hkv, a.G, a.B, a.I, keys }; + var op = b.tensorOperation(.{}); + const scores = try op.commit(op.contract(q, k, scores_axes).mulf(cfg.scale), .{}); + op = b.tensorOperation(.{}); + const s = op.fetch(scores, .{}); + return op.commit(op.where(op.fetch(mask, .{}), .ne, 0, s, score_min), .{}); +} + +fn maskedExp(b: *tcl.Builder, scores: builder.Tensor, maximum: builder.Tensor, mask: builder.Tensor) tcl.FinishError!builder.Tensor { + const op = b.tensorOperation(.{}); + const e = op.fetch(scores, .{}).subf(op.fetch(maximum, .{})).exp(); + // Subtracting the sentinel from itself gives exp(0), so masking only + // the logits is insufficient for empty rows and fully masked chunks. + return op.commit(op.where(op.fetch(mask, .{}), .ne, 0, e, 0.0), .{}); +} + +fn expDifference(b: *tcl.Builder, x: builder.Tensor, maximum: builder.Tensor) tcl.FinishError!builder.Tensor { + const op = b.tensorOperation(.{ .tactic = .elementwise }); + const m = op.fetch(maximum, .{}); + return op.commit(op.fetch(x, .{}).subf(m).exp(), .{}); +} + +fn safeDenominator(b: *tcl.Builder, sum: builder.Tensor) tcl.FinishError!builder.Tensor { + const op = b.tensorOperation(.{}); + const s = op.fetch(sum, .{}); + return op.commit(op.where(s, .gt, 0, s, 1.0), .{}); +} + +const OneShot = tcl.Kernel(Cfg, .{ + .name = "attention", + .inputs = &.{ "q", "pages", "seq_lens", "query_offsets", "k_cache", "v_cache", "sink" }, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + configure(b, cfg); + const a: Axes = .init(b, cfg); + const A = b.axis("A", cfg.a); + const Nb = b.axis("Nb", cfg.pages * cfg.pg); + const t = try declareInputs(b, a); + const prepared = try prepareQueries(b, cfg, a, t); + const kc = try b.reshape(t.k_cache, &.{ Nb, a.Hkv, a.D }, .{}); + const vc = try b.reshape(t.v_cache, &.{ Nb, a.Hkv, a.D }, .{}); + const page_indices = try validIndices(b, prepared.pages, cfg.pages); + const within_page = try b.arange(cfg.pg, .i32, &.{a.PG}, .{}); + var slot_op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const flat_slots = try slot_op.commit(slot_op.fetch(page_indices.safe, .{}).muli(cfg.pg).addi(slot_op.fetch(within_page, .{})), .{}); + const expanded_slots = try b.reshape(flat_slots, &.{ a.B, A }, .{}); + slot_op = b.tensorOperation(.{}); + const valid = try slot_op.commit(slot_op.fetch(page_indices.valid, .{ .broadcast_to = &.{ a.B, a.AP, a.PG } }), .{}); + const valid_keys = try b.reshape(valid, &.{ a.B, A }, .{}); + // Match the original slot sanitization before reading invalid pages. + slot_op = b.tensorOperation(.{}); + const slots = try slot_op.commit(slot_op.where(valid_keys, .ne, 0, slot_op.fetch(expanded_slots, .{}), 0), .{}); + const valid_length = if (cfg.decode) prepared.kvlen else null; + const position = try b.arange(cfg.a, .i32, &.{A}, .{}); + const valid_cache = try cacheMask(b, valid_keys, prepared.kvlen, position); + const k = try maskCacheValues(b, try b.gather(kc, slots, Nb, &.{ a.B, A, a.Hkv, a.D }, .{ .valid_length = valid_length }), valid_cache); + const v = try maskCacheValues(b, try b.gather(vc, slots, Nb, &.{ a.B, A, a.Hkv, a.D }, .{ .valid_length = valid_length }), valid_cache); + const mask = try attentionMask(b, cfg, prepared.qpos, prepared.kvlen, position, valid_keys); + const masked = try maskedScores(b, a, cfg, prepared.q, k, mask, A); + var op = b.tensorOperation(.{}); + const p = if (cfg.decode) compact: { + // Furiosa's common.softmax keeps normalization inside one VE + // operation. FP32 intermediates only narrow at the DPE input. + const exp_sink = if (cfg.use_sink) sink: { + const maximum = try op.commit(op.fetch(masked, .{}).reduce(&.{A}, .maxf).maxf(op.fetch(t.sink, .{})), .{}); + break :sink try expDifference(b, t.sink, maximum); + } else null; + op = b.tensorOperation(.{}); + const scores = op.fetch(masked, .{}); + var maximum = scores.reduce(&.{A}, .maxf); + if (cfg.use_sink) maximum = maximum.maxf(op.fetch(t.sink, .{})); + const e = scores.subf(maximum).exp(); + var sum = e.reduce(&.{A}, .addf); + if (exp_sink) |s| sum = sum.addf(op.fetch(s, .{})); + const probabilities = try op.commit(e.divf(sum), .{ .dtype = .bf16 }); + // Empty rows have uniform probabilities from finite sentinels. + // Clear masked probabilities before contracting with values. + op = b.tensorOperation(.{}); + break :compact try op.commit(op.where(op.fetch(mask, .{}), .ne, 0, op.fetch(probabilities, .{}), 0.0), .{ .dtype = .bf16 }); + } else tiled: { + // Separate reductions schedule better for multi-query blocks + // with the current compiler; keep their intermediate state FP32. + var m = op.fetch(masked, .{}).reduce(&.{A}, .maxf); + if (cfg.use_sink) m = m.maxf(op.fetch(t.sink, .{})); + const maximum = try op.commit(m, .{}); + const e = try maskedExp(b, masked, maximum, mask); + const exp_sink = if (cfg.use_sink) try expDifference(b, t.sink, maximum) else null; + op = b.tensorOperation(.{}); + var sum = op.fetch(e, .{}).reduce(&.{A}, .addf); + if (exp_sink) |s| sum = sum.addf(op.fetch(s, .{})); + const denominator = try safeDenominator(b, try op.commit(sum, .{})); + op = b.tensorOperation(.{}); + break :tiled try op.commit(op.fetch(e, .{}).divf(op.fetch(denominator, .{})), .{ .dtype = .bf16 }); + }; + op = b.tensorOperation(.{}); + const output = try op.commit(op.contract(v, p, &.{ a.B, a.I, a.Hkv, a.G, a.D }), .{ .dtype = .bf16 }); + b.ret(&.{try restoreQueries(b, cfg, a, prepared, output)}); + } + }.run, +}); + +const Flash = tcl.Kernel(Cfg, .{ + .name = "flash_attention", + .inputs = &.{ "q", "pages", "seq_lens", "query_offsets", "k_cache", "v_cache", "sink" }, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + configure(b, cfg); + const a: Axes = .init(b, cfg); + const C = b.axis("C", @divExact(cfg.a, cfg.chunk)); + const Ck = b.axis("Ck", cfg.chunk); + const Cp = b.axis("Cp", @divExact(cfg.chunk, cfg.pg)); + const t = try declareInputs(b, a); + const prepared = try prepareQueries(b, cfg, a, t); + const by_batch = try b.reshape(prepared.pages, &.{ a.B, C, Cp }, .{}); + const page_op = b.tensorOperation(.{}); + const pages = try page_op.commit(page_op.fetch(by_batch, .{}), .{ .axes = &.{ C, a.B, Cp } }); + const stats = &.{ a.Hkv, a.G, a.B, a.I }; + const out_axes = &.{ a.B, a.I, a.Hkv, a.G, a.D }; + var op = b.tensorOperation(.{}); + const n = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(prepared.kvlen), cfg.chunk - 1), cfg.chunk); + var loop = b.openFor(n, .{ try b.full(0, .f32, stats), try b.full(0, .f32, out_axes), try b.full(score_min, .f32, stats) }); + const sum_acc, const qkv_acc, const max_acc = loop.carried; + const indices = try validIndices(b, try b.indexRead(pages, loop.iv), cfg.pages); + const idx = indices.safe; + op = b.tensorOperation(.{}); + const valid_pages = try op.commit(op.fetch(indices.valid, .{ .broadcast_to = &.{ a.B, Cp, a.PG } }), .{}); + const valid_keys = try b.reshape(valid_pages, &.{ a.B, Ck }, .{}); + const offset = try b.symExpr(.mul, loop.iv, cfg.chunk); + const local_pos = try b.arange(cfg.chunk, .i32, &.{Ck}, .{}); + op = b.tensorOperation(.{}); + const position = try op.commit(op.fetch(local_pos, .{}).addi(op.fetch(offset, .{})), .{}); + const local_pages = try b.arange(@divExact(cfg.chunk, cfg.pg), .i32, &.{Cp}, .{}); + op = b.tensorOperation(.{}); + const first_keys = try op.commit(op.fetch(local_pages, .{}).muli(cfg.pg).addi(op.fetch(offset, .{})), .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const remaining = try op.commit(op.fetch(prepared.kvlen, .{}).subi(op.fetch(first_keys, .{})), .{}); + op = b.tensorOperation(.{}); + const live_pages = try op.commit(op.where(remaining, .gt, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const cvl = try op.commit(op.fetch(live_pages, .{}).reduce(&.{Cp}, .addi), .{}); + const mask = try attentionMask(b, cfg, prepared.qpos, prepared.kvlen, position, valid_keys); + const valid_cache = try cacheMask(b, valid_keys, prepared.kvlen, position); + const k = try maskCacheValues(b, try b.reshape(try b.gather(t.k_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}), valid_cache); + const v = try maskCacheValues(b, try b.reshape(try b.gather(t.v_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}), valid_cache); + const masked = try maskedScores(b, a, cfg, prepared.q, k, mask, Ck); + op = b.tensorOperation(.{}); + const max_cur = try op.commit(op.fetch(masked, .{}).reduce(&.{Ck}, .maxf), .{}); + op = b.tensorOperation(.{ .tactic = .elementwise }); + const max_new = try op.commit(op.fetch(max_cur, .{}).maxf(op.fetch(max_acc, .{})), .{}); + const rescale = try expDifference(b, max_acc, max_new); + const e = try maskedExp(b, masked, max_new, mask); + op = b.tensorOperation(.{}); + const sum_cur = try op.commit(op.fetch(e, .{}).reduce(&.{Ck}, .addf), .{}); + op = b.tensorOperation(.{}); + const sum_new = try op.commit(op.fetch(sum_acc, .{}).mulf(op.fetch(rescale, .{})).addf(op.fetch(sum_cur, .{})), .{}); + op = b.tensorOperation(.{}); + // Materialize the narrowing conversion through the vector engine. + const p = try op.commit(op.fetch(e, .{}).mulf(1.0), .{ .dtype = .bf16 }); + op = b.tensorOperation(.{}); + const qkv_cur = try op.commit(op.contract(p, v, out_axes), .{}); + op = b.tensorOperation(.{}); + const qkv_new = try op.commit(op.fetch(qkv_acc, .{}).mulf(op.fetch(rescale, .{})).addf(op.fetch(qkv_cur, .{})), .{}); + try loop.yield(.{ sum_new, qkv_new, max_new }); + + var sum = loop.results[0]; + var numerator = loop.results[1]; + if (cfg.use_sink) { + // Merge the zero-valued sink once, after all key chunks. + op = b.tensorOperation(.{}); + const maximum = try op.commit(op.fetch(loop.results[2], .{}).maxf(op.fetch(t.sink, .{})), .{}); + const alpha = try expDifference(b, loop.results[2], maximum); + const exp_sink = try expDifference(b, t.sink, maximum); + op = b.tensorOperation(.{}); + sum = try op.commit(op.fetch(sum, .{}).mulf(op.fetch(alpha, .{})).addf(op.fetch(exp_sink, .{})), .{}); + op = b.tensorOperation(.{}); + numerator = try op.commit(op.fetch(numerator, .{}).mulf(op.fetch(alpha, .{})), .{}); + } + const denominator = try safeDenominator(b, sum); + op = b.tensorOperation(.{}); + const output = try op.commit(op.fetch(numerator, .{}).divf(op.fetch(denominator, .{})), .{ .dtype = .bf16 }); + b.ret(&.{try restoreQueries(b, cfg, a, prepared, output)}); + } + }.run, +}); + +test { + std.testing.refAllDecls(attention_test); +} + +const attention_test = struct { + pub const Options = struct { + is_causal: bool = true, + sliding_window: i32 = -1, + scale: ?f32 = null, + sink: ?[]const f32 = null, + }; + + pub fn reference(allocator: std.mem.Allocator, q: zml.Slice, k: zml.Slice, v: zml.Slice, table: []const i32, lengths: []const i32, starts: []const i32, opts: Options) !zml.Slice { + const bf16 = zml.floats.BFloat16; + const hkv: usize = @intCast(q.shape.dim(.hkv)); + const groups: usize = @intCast(q.shape.dim(.hg)); + const d: usize = @intCast(q.shape.dim(.hd)); + const page: usize = @intCast(k.shape.dim(.k_chunk)); + const pages: usize = @intCast(k.shape.dim(.page)); + const max_pages = table.len / lengths.len; + const scale = opts.scale orelse 1.0 / @sqrt(@as(f32, @floatFromInt(d))); + const queries = q.constItems(bf16); + const keys = k.constItems(bf16); + const values = v.constItems(bf16); + const result = try zml.Slice.alloc(allocator, q.shape.withDtype(.f32)); + errdefer result.free(allocator); + const out = result.items(f32); + @memset(out, 0); + const scores = try allocator.alloc(f32, max_pages * page); + defer allocator.free(scores); + + for (lengths, 0..) |length, seq| { + const qlen = starts[seq + 1] - starts[seq]; + for (@intCast(starts[seq])..@intCast(starts[seq + 1])) |token| { + const qpos = length - qlen + @as(i32, @intCast(token)) - starts[seq]; + if (qpos < 0) continue; + for (0..hkv) |h| for (0..groups) |g| { + const offset = ((token * hkv + h) * groups + g) * d; + const sink = if (opts.sink) |s| s[h * groups + g] else -std.math.inf(f32); + var maximum = sink; + @memset(scores, -std.math.inf(f32)); + for (0..@intCast(length)) |key| { + const pos: i32 = @intCast(key); + if (opts.is_causal and pos > qpos) continue; + if (opts.sliding_window > 0 and qpos - pos >= opts.sliding_window) continue; + const physical = table[seq * max_pages + key / page]; + if (physical < 0 or physical >= pages) continue; + const kv_offset = ((@as(usize, @intCast(physical)) * page + key % page) * hkv + h) * d; + var dot: f32 = 0; + for (0..d) |i| dot += queries[offset + i].toF32() * keys[kv_offset + i].toF32(); + scores[key] = dot * scale; + maximum = @max(maximum, scores[key]); + } + if (maximum == -std.math.inf(f32)) continue; + var denominator = @exp(sink - maximum); + for (scores[0..@intCast(length)]) |score| denominator += @exp(score - maximum); + for (0..@intCast(length)) |key| { + if (scores[key] == -std.math.inf(f32)) continue; + const physical: usize = @intCast(table[seq * max_pages + key / page]); + const kv_offset = ((physical * page + key % page) * hkv + h) * d; + const probability = @exp(scores[key] - maximum) / denominator; + for (0..d) |i| out[offset + i] += probability * values[kv_offset + i].toF32(); + } + }; + } + } + return result; + } + + test "paged attention host reference scale and sink normalization" { + const bf16 = zml.floats.BFloat16; + const queries: [3]bf16 = @splat(.fromF32(1)); + const keys = [_]bf16{ .fromF32(0), .fromF32(1) }; + const values = [_]bf16{ .fromF32(2), .fromF32(4) }; + const q = zml.Slice.initConst(.init(.{ .b = 3, .hkv = 1, .hg = 1, .hd = 1 }, .bf16), std.mem.sliceAsBytes(&queries)); + const k = zml.Slice.initConst(.init(.{ .page = 1, .k_chunk = 2, .hkv = 1, .hd = 1 }, .bf16), std.mem.sliceAsBytes(&keys)); + const v = zml.Slice.initConst(k.shape, std.mem.sliceAsBytes(&values)); + const cases = [_]struct { opts: Options, expected: f32 }{ + .{ .opts = .{ .is_causal = false, .scale = 0 }, .expected = 3 }, + .{ .opts = .{ .is_causal = false, .scale = @log(@as(f32, 3)) }, .expected = 3.5 }, + .{ .opts = .{ .is_causal = false, .scale = @log(@as(f32, 3)), .sink = &.{0} }, .expected = 2.8 }, + }; + for (cases) |case| { + const result = try reference(std.testing.allocator, q, k, v, &.{0}, &.{2}, &.{ 0, 2 }, case.opts); + defer result.free(std.testing.allocator); + for (result.constItems(f32)[0..2]) |value| try std.testing.expectApproxEqAbs(case.expected, value, 1e-6); + try std.testing.expectEqual(@as(f32, 0), result.constItems(f32)[2]); + } + } + + pub fn expectClose(expected: zml.Slice, actual: zml.Slice) !void { + try std.testing.expectEqualSlices(i64, expected.shape.dims(), actual.shape.dims()); + for (expected.constItems(f32), 0..) |want, i| { + const got = floatAt(actual, i); + try std.testing.expect(std.math.isFinite(got)); + std.testing.expectApproxEqAbs(want, got, 3e-2 + 3e-2 * @abs(want)) catch |err| { + std.log.err("paged attention mismatch at element {d}: expected {d}, got {d}", .{ i, want, got }); + return err; + }; + } + } + + fn floatAt(slice: zml.Slice, index: usize) f32 { + return switch (slice.shape.dtype()) { + .bf16 => slice.constItems(zml.floats.BFloat16)[index].toF32(), + .f16 => @floatCast(slice.constItems(f16)[index]), + .f32 => slice.constItems(f32)[index], + else => unreachable, + }; + } + + test "paged attention host reference windows sinks and padding" { + const bf16 = zml.floats.BFloat16; + const queries: [3]bf16 = @splat(.fromF32(0)); + const keys: [2]bf16 = @splat(.fromF32(0)); + const values = [_]bf16{ .fromF32(2), .fromF32(4) }; + const q = zml.Slice.initConst(.init(.{ .b = 3, .hkv = 1, .hg = 1, .hd = 1 }, .bf16), std.mem.sliceAsBytes(&queries)); + const k = zml.Slice.initConst(.init(.{ .page = 1, .k_chunk = 2, .hkv = 1, .hd = 1 }, .bf16), std.mem.sliceAsBytes(&keys)); + const v = zml.Slice.initConst(k.shape, std.mem.sliceAsBytes(&values)); + const cases = [_]struct { opts: Options, expected: [3]f32 }{ + .{ .opts = .{}, .expected = .{ 2, 3, 0 } }, + .{ .opts = .{ .sliding_window = 1 }, .expected = .{ 2, 4, 0 } }, + .{ .opts = .{ .is_causal = false, .sliding_window = 1 }, .expected = .{ 3, 4, 0 } }, + .{ .opts = .{ .sink = &.{0} }, .expected = .{ 1, 2, 0 } }, + .{ .opts = .{ .sink = &.{-std.math.inf(f32)} }, .expected = .{ 2, 3, 0 } }, + }; + for (cases) |case| { + const result = try reference(std.testing.allocator, q, k, v, &.{0}, &.{2}, &.{ 0, 2 }, case.opts); + defer result.free(std.testing.allocator); + try std.testing.expectEqualSlices(f32, &case.expected, result.constItems(f32)); + } + } +}; + +const TestCase = struct { + prefill: bool, + seqs: []const [2]i32, + padding: i64 = 0, + opts: attention_test.Options = .{}, + mask_all: bool = false, + padding_page: i32 = -1, + poison: enum { none, k, v, both } = .none, + poison_value: f32 = std.math.nan(f32), +}; + +const TestDims = struct { + batch: usize = 4, + hkv: usize = 2, + g: usize = 4, + d: usize = 64, + page: usize = 16, + uniform: bool = false, + uniform_scores: bool = false, + dtype: zml.DataType = .bf16, +}; + +test "tcl paged attention unit scale and dtype boundaries on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + try runCases(16, 32, &.{.{ .prefill = true, .seqs = &@as([4][2]i32, @splat(.{ 32, 256 })), .opts = .{ .scale = 1.0 } }}, .{}); + inline for (.{ .f16, .f32 }) |dtype| { + try runCases(2, 8, &.{.{ .prefill = false, .seqs = &.{ .{ 1, 17 }, .{ 1, 1 }, .{ 1, 3 }, .{ 0, 0 } }, .padding = 1, .opts = .{ .sink = &.{ 0, -std.math.inf(f32) } } }}, .{ .hkv = 1, .g = 2, .d = 32, .dtype = dtype }); + } +} + +test "tcl paged attention single decode row needs no rearrangement" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + try runCases(16, 32, &.{ + .{ .prefill = false, .seqs = &.{ .{ 1, 251 }, .{ 0, 0 }, .{ 0, 0 }, .{ 0, 0 } } }, + .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 0, 0 })), .padding = 1 }, + }, .{ .batch = 1 }); + inline for (.{ 257, 320 }) |max_pages| { + try runCases(max_pages, 2 * max_pages, &.{.{ + .prefill = false, + .seqs = &.{ .{ 1, @intCast(max_pages * 16 - 1) }, .{ 0, 0 }, .{ 0, 0 }, .{ 0, 0 } }, + }}, .{ .batch = 1, .hkv = 8, .g = 4, .d = 128 }); + } + inline for (.{ 5, 1025 }) |max_pages| { + try runCases(max_pages, 2 * max_pages, &.{.{ + .prefill = false, + .seqs = &.{ .{ 1, @intCast(max_pages * 48 - 1) }, .{ 0, 0 }, .{ 0, 0 }, .{ 0, 0 } }, + }}, .{ .batch = 1, .hkv = 1, .g = 1, .d = 32, .page = 48 }); + } +} + +test "tcl paged attention packs inactive leading sequences and odd batches" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + try runCases(16, 32, &.{ + .{ .prefill = false, .seqs = &.{ .{ 0, 17 }, .{ 1, 63 }, .{ 0, 23 }, .{ 1, 65 } }, .padding = 2 }, + .{ .prefill = true, .seqs = &.{ .{ 0, 17 }, .{ 33, 64 }, .{ 0, 23 }, .{ 2, 3 } }, .padding = 3 }, + }, .{}); + try runCases(2, 8, &.{ + .{ .prefill = false, .seqs = &.{ .{ 0, 9 }, .{ 1, 17 }, .{ 1, 1 }, .{ 0, 0 } }, .padding = 1 }, + }, .{ .batch = 3, .hkv = 1, .g = 2, .d = 32 }); +} + +test "tcl paged attention masks inactive decode rows across all heads" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + try runCases(32, 64, &.{.{ + .prefill = false, + .seqs = &.{ .{ 1, 511 }, .{ 1, 504 }, .{ 1, 497 }, .{ 1, 490 }, .{ 0, 483 }, .{ 1, 476 }, .{ 1, 469 }, .{ 1, 462 } }, + .padding = 1, + }}, .{ .batch = 8 }); +} + +test "tcl paged attention invalid pages cannot wrap into slot zero" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + const cases = [_]TestCase{ + .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 1, 17 })), .mask_all = true, .padding_page = 1 << 28 }, + .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 1, 17 })), .mask_all = true, .padding_page = std.math.maxInt(i32) }, + }; + try runCases(2, 8, &cases, .{ .hkv = 1, .g = 1, .d = 32 }); +} + +test "tcl paged attention emits decode and prefill kernels with and without sinks" { + inline for (.{ false, true }) |decode| { + inline for (.{ false, true }) |use_sink| { + const cfg: Cfg = .{ .tokens = if (decode) 2 else 8, .sequences = 2, .b = if (decode) 2 else 3, .i = if (decode) 1 else prefill_block, .a = 32, .chunk = 16, .hkv = 2, .g = 4, .d = 32, .pages = 8, .pg = 16, .scale = 0.125, .decode = decode, .use_sink = use_sink }; + const one_shot = try OneShot.emit(std.testing.allocator, cfg); + defer std.testing.allocator.free(one_shot); + const flash = try Flash.emit(std.testing.allocator, cfg); + defer std.testing.allocator.free(flash); + for ([_][]const u8{ one_shot, flash }) |ir| { + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"tcl.dpe\"")); + try std.testing.expect(std.mem.indexOf(u8, ir, "!tcl.logical") == null); + try std.testing.expect(std.mem.indexOf(u8, ir, "#tcl.symbol<\"I\">") == null); + + const decode = try Decode.emit(std.testing.allocator, .{ .t = 2, .a = 64, .hkv = 2, .g = 4, .d = 32, .pages = 8, .pg = 16, .scale = 0.125, .is_causal = true }); + defer std.testing.allocator.free(decode); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, decode, "\"tcl.dpe\"")); + try std.testing.expect(std.mem.indexOf(u8, decode, "tcl.fusible") != null); + try std.testing.expect(std.mem.indexOf(u8, decode, "propagate_sparse_axis_from_op = \"Gather\"") != null); +} + +test "tcl paged attention excludes poisoned unused cache entries on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + const seqs = [_][2]i32{ .{ 1, 17 }, .{ 1, 31 }, .{ 0, 0 }, .{ 0, 23 } }; + try runCases(8, 32, &.{ + .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .k }, + .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .v }, + .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .both }, + .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .both, .poison_value = std.math.inf(f32) }, + .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .both, .poison_value = -std.math.inf(f32) }, + .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .both, .poison_value = 1e30 }, + // A sink selects OneShot's compact decode softmax instead of Decode. + .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .both, .opts = .{ .scale = 0.2, .sink = &.{ 0, -std.math.inf(f32) } } }, + .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .both, .mask_all = true, .padding_page = std.math.maxInt(i32) }, + .{ .prefill = false, .seqs = &.{ .{ 0, 0 }, .{ 0, 0 }, .{ 0, 0 }, .{ 0, 0 } }, .padding = 4, .poison = .both }, + }, .{ .hkv = 1, .g = 2, .d = 32 }); + + const mixed = [_][2]i32{ .{ 37, 65 }, .{ 0, 19 }, .{ 1, 17 }, .{ 4, 2 } }; + try runCases(8, 32, &.{ + .{ .prefill = true, .seqs = &mixed, .padding = 3, .poison = .k }, + .{ .prefill = true, .seqs = &mixed, .padding = 3, .poison = .v }, + .{ .prefill = true, .seqs = &mixed, .padding = 3, .poison = .both, .opts = .{ .is_causal = false, .sliding_window = 17, .scale = 0.2, .sink = &.{ 0, -std.math.inf(f32) } } }, + }, .{ .hkv = 1, .g = 2, .d = 32 }); + + const boundary = [_][2]i32{ .{ 37, 4097 }, .{ 0, 0 }, .{ 1, 4096 }, .{ 2, 17 } }; + try runCases(257, 1028, &.{ + .{ .prefill = true, .seqs = &boundary, .padding = 3, .poison = .both }, + .{ .prefill = true, .seqs = &boundary, .padding = 3, .poison = .both, .poison_value = std.math.inf(f32), .opts = .{ .is_causal = false, .sliding_window = 33, .scale = 0.2, .sink = &.{ 0, -std.math.inf(f32) } } }, + .{ .prefill = false, .seqs = &.{ .{ 1, 4095 }, .{ 1, 4096 }, .{ 1, 4097 }, .{ 0, 0 } }, .padding = 1, .poison = .both, .opts = .{ .sliding_window = 17, .sink = &.{ 0, -std.math.inf(f32) } } }, + }, .{ .hkv = 1, .g = 2, .d = 32 }); + try runCases(86, 344, &.{.{ + .prefill = true, + .seqs = &.{ .{ 3, 4033 }, .{ 1, 4032 }, .{ 0, 0 }, .{ 1, 4031 } }, + .padding = 1, + .poison = .both, + .poison_value = -std.math.inf(f32), + .opts = .{ .sliding_window = 49 }, + }}, .{ .hkv = 1, .g = 1, .d = 32, .page = 48 }); +} + +test "tcl paged attention runs on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + inline for (.{ 16, 320 }) |max_pages| { + const cases = if (max_pages == 16) [_]TestCase{ + .{ .prefill = true, .seqs = &.{ .{ 37, 200 }, .{ 1, 77 }, .{ 20, 20 }, .{ 0, 0 } } }, + .{ .prefill = false, .seqs = &.{ .{ 1, 250 }, .{ 1, 3 }, .{ 1, 64 }, .{ 1, 129 } } }, + } else [_]TestCase{ + .{ .prefill = true, .seqs = &.{ .{ 37, 4500 }, .{ 1, 77 }, .{ 70, 70 }, .{ 0, 0 } } }, + .{ .prefill = false, .seqs = &.{ .{ 1, 5000 }, .{ 1, 3 }, .{ 1, 4097 }, .{ 1, 129 } } }, + }; + try runCases(max_pages, 2 * max_pages, &cases, .{}); + } +} + +test "tcl paged attention options and inactive queries on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + const sinks = [_]f32{ -std.math.inf(f32), -5, 0, 2, 8, -1, 1, 20 }; + const disabled: [8]f32 = @splat(-std.math.inf(f32)); + const mixed = [4][2]i32{ .{ 37, 200 }, .{ 0, 0 }, .{ 1, 77 }, .{ 20, 20 } }; + const cases = [_]TestCase{ + .{ .prefill = true, .seqs = &mixed, .padding = 5, .opts = .{ .sliding_window = 17, .scale = 0.2, .sink = &sinks } }, + .{ .prefill = true, .seqs = &mixed, .padding = 5, .opts = .{ .is_causal = false, .sliding_window = 17, .scale = 0.2, .sink = &sinks } }, + .{ .prefill = true, .seqs = &mixed, .padding = 5, .opts = .{ .is_causal = false, .sliding_window = 0 } }, + .{ .prefill = true, .seqs = &mixed, .padding = 5, .opts = .{ .sliding_window = 1, .sink = &disabled } }, + .{ .prefill = false, .seqs = &.{ .{ 1, 251 }, .{ 0, 73 }, .{ 1, 1 }, .{ 0, 17 } }, .padding = 2, .opts = .{ .sink = &sinks } }, + .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 0, 0 })), .padding = 4 }, + .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 1, 17 })), .mask_all = true }, + .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 1, 17 })), .mask_all = true, .opts = .{ .sink = &sinks } }, + .{ .prefill = false, .seqs = &.{ .{ 1, 251 }, .{ 0, 73 }, .{ 1, 20 }, .{ 0, 17 } }, .padding = 2, .opts = .{ .is_causal = false, .sliding_window = 17, .scale = 0.2, .sink = &sinks } }, + .{ .prefill = false, .seqs = &.{ .{ 1, 251 }, .{ 0, 73 }, .{ 1, 20 }, .{ 0, 17 } }, .padding = 2, .opts = .{ .sliding_window = 1, .sink = &disabled } }, + }; + try runCases(16, 32, &cases, .{}); +} + +test "tcl paged attention chunk boundaries on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + const cases = [_]TestCase{ + .{ .prefill = true, .seqs = &.{ .{ 37, 4097 }, .{ 0, 0 }, .{ 1, 4096 }, .{ 2, 4095 } }, .padding = 7, .opts = .{ .is_causal = false, .sliding_window = 33, .scale = 0.2, .sink = &.{ 0, 10 } } }, + .{ .prefill = false, .seqs = &.{ .{ 1, 4095 }, .{ 1, 4096 }, .{ 1, 4097 }, .{ 0, 0 } }, .padding = 1, .opts = .{ .sliding_window = 17, .sink = &.{ -std.math.inf(f32), 0 } } }, + .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 0, 0 })), .padding = 4 }, + .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 1, 4097 })), .mask_all = true, .opts = .{ .sink = &.{ -std.math.inf(f32), 0 } } }, + }; + try runCases(257, 514, &cases, .{ .hkv = 1, .g = 2, .d = 32 }); + try runCases(256, 512, &.{.{ .prefill = false, .seqs = &.{ .{ 1, 4095 }, .{ 1, 4096 }, .{ 0, 0 }, .{ 0, 0 } }, .padding = 2 }}, .{ .hkv = 1, .g = 1, .d = 32 }); + try runCases(86, 172, &.{.{ .prefill = false, .seqs = &.{ .{ 1, 4079 }, .{ 1, 4080 }, .{ 1, 4081 }, .{ 0, 0 } }, .padding = 1, .opts = .{ .sliding_window = 49 } }}, .{ .hkv = 1, .g = 1, .d = 32, .page = 48 }); +} + +test "tcl paged attention preserves constant values across chunks" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + const cases = [_]TestCase{ + .{ .prefill = false, .seqs = &.{ .{ 1, 64 }, .{ 1, 65 }, .{ 1, 66 }, .{ 0, 0 } }, .padding = 1 }, + .{ .prefill = false, .seqs = &.{ .{ 1, 4095 }, .{ 1, 4096 }, .{ 1, 4097 }, .{ 0, 0 } }, .padding = 1 }, + .{ .prefill = false, .seqs = &.{ .{ 1, 4095 }, .{ 1, 4096 }, .{ 1, 4097 }, .{ 0, 0 } }, .padding = 1, .opts = .{ .sliding_window = 1, .sink = &.{0} } }, + .{ .prefill = true, .seqs = &.{ .{ 37, 4097 }, .{ 0, 0 }, .{ 1, 77 }, .{ 2, 2 } }, .padding = 3 }, + }; + try runCases(257, 514, &cases, .{ .hkv = 1, .g = 1, .d = 32, .uniform = true }); +} + +test "tcl paged attention chunked decode on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + const cases = [_]TestCase{ + .{ .prefill = false, .seqs = &.{ .{ 1, 64 }, .{ 1, 65 }, .{ 1, 66 }, .{ 0, 0 } }, .padding = 1 }, + .{ .prefill = false, .seqs = &.{ .{ 1, 4095 }, .{ 1, 4096 }, .{ 1, 4097 }, .{ 0, 0 } }, .padding = 1 }, + }; + try runCases(257, 514, &cases, .{ .hkv = 1, .g = 1, .d = 32 }); +} + +test "tcl paged attention uniform probabilities on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + try runCases(257, 514, &.{.{ .prefill = false, .seqs = &.{ .{ 1, 64 }, .{ 1, 65 }, .{ 1, 66 }, .{ 0, 0 } }, .padding = 1 }}, .{ .hkv = 1, .g = 1, .d = 32, .uniform_scores = true }); +} + +fn runCases(comptime max_pages: usize, comptime num_pages: usize, cases: []const TestCase, comptime dims: TestDims) !void { + const platform = zml.testing.env(); + const allocator = std.testing.allocator; + const io = std.testing.io; + const bf16 = zml.floats.BFloat16; + const hkv = dims.hkv; + const g = dims.g; + const d = dims.d; + const page = dims.page; + const batch = dims.batch; + var prng: std.Random.DefaultPrng = .init(7); + const r = prng.random(); + const kc = try allocator.alloc(bf16, num_pages * page * hkv * d); + defer allocator.free(kc); + const vc = try allocator.alloc(bf16, num_pages * page * hkv * d); + defer allocator.free(vc); + const k_t: Tensor = .init(.{ .page = num_pages, .k_chunk = page, .hkv = hkv, .hd = d }, dims.dtype); + + for (cases) |case| { + var tokens: i64 = case.padding; + for (case.seqs[0..batch]) |s| tokens += s[0]; + const params: paged.Parameters = .init(.{ .batch_size = batch, .max_num_pages = max_pages, .max_seqlen_q = 128, .is_prefill = case.prefill }); + const q_t: Tensor = .init(.{ .b = tokens, .hkv = hkv, .hg = g, .hd = d }, dims.dtype); + const Mod = struct { + pub fn forward(p: paged.Parameters, q: Tensor, k: Tensor, v: Tensor, opts: paged_attention.AttentionOptions) Tensor { + return paged_attention.pagedAttention(.{ .tcl = p }, q, k, v, .{ .split = .{ .k = k, .v = v } }, opts); + } + }; + const opts: paged_attention.AttentionOptions = .{ + .mask = .{ .standard = .{ + .causal = case.opts.is_causal, + .sliding_window = if (case.opts.sliding_window > 0) @intCast(case.opts.sliding_window) else null, + } }, + .scale = case.opts.scale, + .sink = if (case.opts.sink != null) Tensor.init(.{ .h = hkv * g }, .f32) else null, + }; + const v_t: Tensor = .init(k_t.shape(), dims.dtype); + var exe = try zml.module.compile(allocator, io, Mod.forward, .{ params, q_t, k_t, v_t, opts }, platform, .{}); + defer exe.deinit(); + + const q = try allocator.alloc(bf16, @intCast(tokens * hkv * g * d)); + defer allocator.free(q); + for (q) |*v| v.* = .fromF32(if (dims.uniform or dims.uniform_scores) 0 else r.floatNorm(f32)); + var table: [batch][max_pages]i32 = @splat(@splat(case.padding_page)); + var seq_lens: [batch]i32 = undefined; + var starts: [batch + 1]i32 = undefined; + starts[0] = 0; + var next_page: i32 = 0; + for (case.seqs[0..batch], 0..) |s, seq| { + seq_lens[seq] = s[1]; + starts[seq + 1] = starts[seq] + s[0]; + if (!case.mask_all) { + for (0..@intCast(std.math.divCeil(i32, s[1], page) catch unreachable)) |p| { + table[seq][p] = @mod(next_page * 7, @as(i32, num_pages)); + next_page += 1; + } + } + } + + for (kc) |*v| v.* = .fromF32(if (dims.uniform or dims.uniform_scores) 0 else r.floatNorm(f32)); + for (vc, 0..) |*v, i| v.* = .fromF32(if (dims.uniform) @as(f32, @floatFromInt(i % d)) / d else r.floatNorm(f32)); + if (case.poison != .none) { + // Mark the union of live physical slots before poisoning: reordered + // or shared pages must retain every token a live sequence can read. + const live = try allocator.alloc(bool, num_pages * page); + defer allocator.free(live); + @memset(live, false); + for (case.seqs[0..batch], 0..) |s, seq| { + if (s[0] == 0) continue; + for (0..@intCast(s[1])) |key| { + const physical = table[seq][key / page]; + if (physical < 0 or physical >= num_pages) continue; + live[@as(usize, @intCast(physical)) * page + key % page] = true; + } + } + const poison = bf16.fromF32(case.poison_value); + for (live, 0..) |valid, slot| { + if (valid) continue; + const start = slot * hkv * d; + if (case.poison == .k or case.poison == .both) @memset(kc[start..][0 .. hkv * d], poison); + if (case.poison == .v or case.poison == .both) @memset(vc[start..][0 .. hkv * d], poison); + } + } + var bk = try testBuffer(platform, k_t.shape(), kc); + defer bk.deinit(); + var bv = try testBuffer(platform, k_t.shape(), vc); + defer bv.deinit(); + var bq = try testBuffer(platform, q_t.shape(), q); + defer bq.deinit(); + var bp: zml.Bufferized(paged.Parameters) = .{ + .block_table = try .fromBytes(io, platform, params.block_table.shape(), std.mem.sliceAsBytes(&table)), + .seq_lens = try .fromBytes(io, platform, params.seq_lens.shape(), std.mem.sliceAsBytes(&seq_lens)), + .query_start_len = try .fromBytes(io, platform, params.query_start_len.shape(), std.mem.sliceAsBytes(&starts)), + }; + defer zml.Buffer.deinitAll(paged.Parameters, &bp); + var opts_d: zml.Bufferized(paged_attention.AttentionOptions) = .{ + .mask = .{ .standard = {} }, + .sink = if (case.opts.sink) |s| try .fromBytes(io, platform, opts.sink.?.shape(), std.mem.sliceAsBytes(s)) else null, + }; + defer zml.Buffer.deinitAll(paged_attention.AttentionOptions, &opts_d); + var result = try exe.eval(allocator, io, .{ bp, bq, bk, bv, opts_d }); + defer result.deinit(); + const host = try result.toSliceAlloc(allocator, io); + defer host.free(allocator); + const expected = try attention_test.reference( + allocator, + .initConst(q_t.shape().withDtype(.bf16), std.mem.sliceAsBytes(q)), + .initConst(k_t.shape().withDtype(.bf16), std.mem.sliceAsBytes(kc)), + .initConst(k_t.shape().withDtype(.bf16), std.mem.sliceAsBytes(vc)), + @as([*]const i32, @ptrCast(&table))[0 .. batch * max_pages], + &seq_lens, + &starts, + case.opts, + ); + defer expected.free(allocator); + try std.testing.expectEqual(dims.dtype, host.shape.dtype()); + attention_test.expectClose(expected, host) catch |err| { + std.log.err("TCL attention fixture: {any}", .{case}); + return err; + }; + const active_elements = if (case.mask_all) 0 else @as(usize, @intCast(starts[batch])) * hkv * g * d; + for (active_elements..@intCast(tokens * hkv * g * d)) |index| try std.testing.expectEqual(@as(f32, 0), attention_test.floatAt(host, index)); + for (case.seqs[0..batch], 0..) |seq, index| { + const empty_queries: usize = @intCast(@max(0, seq[0] - seq[1])); + const begin: usize = @as(usize, @intCast(starts[index])) * hkv * g * d; + for (begin..begin + empty_queries * hkv * g * d) |element| try std.testing.expectEqual(@as(f32, 0), attention_test.floatAt(host, element)); + } + } +} + +fn testBuffer(platform: *const zml.Platform, shape: zml.Shape, values: []const zml.floats.BFloat16) !zml.Buffer { + const slice = try zml.Slice.alloc(std.testing.allocator, shape); + defer slice.free(std.testing.allocator); + switch (shape.dtype()) { + .bf16 => @memcpy(slice.items(zml.floats.BFloat16), values), + .f16 => for (slice.items(f16), values) |*out, value| { + out.* = @floatCast(value.toF32()); + }, + .f32 => for (slice.items(f32), values) |*out, value| { + out.* = value.toF32(); + }, + else => unreachable, + } + return .fromBytes(std.testing.io, platform, shape, slice.bytes); +} From aa30110932544ff8683ffdf06f26abbd701737c8 Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Tue, 27 Oct 2026 13:01:40 +0100 Subject: [PATCH 825/881] platforms/rocm: default rocprofiler log level to error (#992) By default rocprofiler log level is `warning`, which can produce unnecessary log output. Now, we override this log level to error, unless `ROCPROFILER_LOG_LEVEL` environment variable is already set. --- platforms/rocm/rocm.zig | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/platforms/rocm/rocm.zig b/platforms/rocm/rocm.zig index 59a4de79..ee31bd83 100644 --- a/platforms/rocm/rocm.zig +++ b/platforms/rocm/rocm.zig @@ -32,6 +32,12 @@ fn setupRocmEnv(rocm_data_dir: []const u8) !void { _ = c.setenv("GPU_MAX_HW_QUEUES", "1", 0); // Disable rocprofiler queue interposition to avoid its dispatch overhead outside profiling. _ = c.setenv("ROCPROFILER_QUEUE_INTERPOSITION", "0", 0); + + if (c.getenv("ROCPROFILER_LOG_LEVEL") == null) { + // rocprofiler defaults to the `warning` log level, which can produce unnecessary log output. + // Set the log level to `fatal` to suppress all but fatal error messages. + _ = c.setenv("ROCPROFILER_LOG_LEVEL", "error", 0); + } } pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { From 3736f3dc297b863a000fa38afde8b81cd9a3cab5 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Tue, 27 Oct 2026 17:53:31 +0100 Subject: [PATCH 826/881] dsl: unify the kernel DSLs (#991) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Makes kernels written in the DSLs read alike, without changing what they emit. - Every DSL target is `//dsl/:builder`, and `dsl/common` is imported as common. - Scopes without values end with `close()`. - The scf DSLs (triton, mosaic, cute, fly) share `openFor`/`openIf`/`openIfElse`/`openWhile` in `dsl/common`. cuda_tile keeps its own scopes with the same shapes. - Scopes take tuples or arrays, and result types can be dtypes (.f32), Values, or MLIR types. - Typed constants are `constant(dtype, value)`, alongside constIndex and constantLike. - Comparisons and bitcasts have the same names in every DSL: Fly gains `lt`/`eq`/…, and every DSL spells it `value.bitCast(dtype)`. - One block stack and one `common.verify`, which logs the module that fails to verify. - READMEs updated. --- dsl/common/BUILD.bazel | 8 +- dsl/common/builder.zig | 35 + dsl/common/control_flow.zig | 275 ++++++- dsl/common/dsl.zig | 17 - dsl/cuda_tile/BUILD.bazel | 6 +- dsl/cuda_tile/README.md | 11 +- dsl/cuda_tile/builder.zig | 239 +++--- dsl/cuda_tile/control_flow.zig | 84 +- dsl/cute/BUILD.bazel | 6 +- dsl/cute/README.md | 20 +- dsl/cute/builder.zig | 263 +++--- dsl/fly/BUILD.bazel | 6 +- dsl/fly/PORTING.md | 6 +- dsl/fly/README.md | 24 +- dsl/fly/builder.zig | 309 +++---- dsl/mosaic_tpu/BUILD.bazel | 6 +- dsl/mosaic_tpu/README.md | 90 ++- dsl/mosaic_tpu/builder.zig | 243 ++---- dsl/mosaic_tpu/pipeline.zig | 88 +- dsl/tcl/BUILD.bazel | 5 +- dsl/tcl/README.md | 3 + dsl/tcl/builder.zig | 20 +- dsl/triton/BUILD.bazel | 6 +- dsl/triton/PORTING.md | 70 +- dsl/triton/README.md | 220 +++-- dsl/triton/builder.zig | 333 +++----- platforms/tpu/BUILD.bazel | 2 +- platforms/tpu/ragged_paged.zig | 148 ++-- zml/BUILD.bazel | 12 +- zml/kernel.zig | 8 +- .../flash_attention/cute/attention.zig | 180 +++-- zml/kernels/moe/cute/mxfp4.zig | 72 +- zml/kernels/moe/cute/persistent_mxfp4.zig | 761 +++++++++--------- zml/kernels/moe/fly/moe.zig | 573 +++++++------ zml/kernels/moe/mosaic_tpu/gmm_ep.zig | 66 +- zml/kernels/moe/tcl/tcl.zig | 22 +- zml/kernels/moe/triton/moe.zig | 164 ++-- .../paged_attention/cute/attention.zig | 28 +- .../paged_attention/cute/sm10x/attention.zig | 126 ++- .../paged_attention/cute/sm120/attention.zig | 56 +- zml/kernels/paged_attention/tcl/tcl.zig | 86 +- zml/kernels/paged_attention/triton/README.md | 3 +- .../triton/sparse_unified_attention.zig | 100 +-- .../triton/unified_attention.zig | 38 +- .../triton/unified_attention_oneapi.zig | 22 +- 45 files changed, 2547 insertions(+), 2313 deletions(-) create mode 100644 dsl/common/builder.zig delete mode 100644 dsl/common/dsl.zig diff --git a/dsl/common/BUILD.bazel b/dsl/common/BUILD.bazel index 9631c34e..1476dcfd 100644 --- a/dsl/common/BUILD.bazel +++ b/dsl/common/BUILD.bazel @@ -1,13 +1,13 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( - name = "common", + name = "builder", srcs = [ "args.zig", "control_flow.zig", ], - import_name = "dsl/common", - main = "dsl.zig", + import_name = "dsl/common/builder", + main = "builder.zig", visibility = ["//visibility:public"], deps = [ "//mlir", @@ -17,5 +17,5 @@ zig_library( zig_test( name = "test", - deps = [":common"], + deps = [":builder"], ) diff --git a/dsl/common/builder.zig b/dsl/common/builder.zig new file mode 100644 index 00000000..1addd259 --- /dev/null +++ b/dsl/common/builder.zig @@ -0,0 +1,35 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const scf = @import("mlir/dialects/scf"); + +pub const control_flow = @import("control_flow.zig"); +pub const args = @import("args.zig"); + +pub const arity = control_flow.arity; +pub const BlockStack = control_flow.BlockStack; +pub const forBounds = control_flow.forBounds; +pub const ForOpts = scf.ForOpts; +pub const openFor = control_flow.openFor; +pub const openIf = control_flow.openIf; +pub const openIfElse = control_flow.openIfElse; +pub const openWhile = control_flow.openWhile; +pub const innerValues = control_flow.innerValues; +pub const resultTypes = control_flow.resultTypes; +pub const ForScope = control_flow.ForScope; +pub const IfOnlyScope = control_flow.IfOnlyScope; +pub const IfScope = control_flow.IfScope; +pub const WhileScope = control_flow.WhileScope; + +pub const NamedArgs = args.NamedArgs; + +/// Verifies `module`, logging it when it fails. +pub fn verify(module: *mlir.Module, comptime target: []const u8) error{InvalidMlir}!void { + if (module.operation().verify()) return; + std.log.err(target ++ ": the module does not verify:\n{f}", .{module.operation().fmt(.{ .debug_info = false })}); + return error.InvalidMlir; +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/dsl/common/control_flow.zig b/dsl/common/control_flow.zig index b57c6ece..e557e24b 100644 --- a/dsl/common/control_flow.zig +++ b/dsl/common/control_flow.zig @@ -1,38 +1,89 @@ -//! `scf` scope wrappers, generic over the frontend's `BuilderT` and `ValueT`. +//! `scf` scope wrappers, generic over the frontend's `BuilderT` and `ValueT`, +//! and `BlockStack`, the blocks a builder emits into. //! -//! `BuilderT` must expose `ctx`, `loc()`, `pushBlock`, `popBlock`, `currentBlock`. +//! `BuilderT` must expose `ctx`, `loc()`, `pushBlock`, `popBlock` and +//! `currentBlock`. The openers also call its `lift` on literal inits and its +//! `scalarTy` on dtype result types, and `forBounds` its `constantLike` on +//! literal bounds. //! `ValueT` must have an `inner: *const mlir.Value` field and a `kernel: ?*BuilderT` field. const std = @import("std"); const mlir = @import("mlir"); const scf = @import("mlir/dialects/scf"); -pub fn tupleArity(comptime T: type, comptime what: []const u8) comptime_int { - const info = @typeInfo(T); - if (info != .@"struct" or !info.@"struct".is_tuple) - @compileError(what ++ " must be a tuple literal like `.{ v1, v2 }`"); - return info.@"struct".field_names.len; +/// The blocks a builder emits into: the function's entry block at the bottom, +/// the innermost scope's on top. +pub const BlockStack = struct { + blocks: std.ArrayList(*mlir.Block) = .empty, + + pub fn push(self: *BlockStack, allocator: std.mem.Allocator, block: *mlir.Block) void { + self.blocks.append(allocator, block) catch @panic("BlockStack.push: out of memory"); + } + + /// Closes the innermost scope. + pub fn pop(self: *BlockStack) void { + if (self.blocks.items.len < 2) @panic("BlockStack.pop: no scope open"); // the entry block always stays + _ = self.blocks.pop(); + } + + /// Drops every block, keeping the memory for the next function. + pub fn clear(self: *BlockStack) void { + self.blocks.clearRetainingCapacity(); + } + + /// Returns the top block. + pub fn current(self: *const BlockStack) *mlir.Block { + return self.blocks.getLastOrNull() orelse @panic("BlockStack.current: no block, call declareArgs first"); + } + + /// Replaces the top block with `block`. + pub fn replaceTop(self: *BlockStack, block: *mlir.Block) void { + if (self.blocks.items.len == 0) @panic("BlockStack.replaceTop: no block, call declareArgs first"); + self.blocks.items[self.blocks.items.len - 1] = block; + } +}; + +/// The length of `T`, a tuple like `.{ v1, v2 }` or an array. +pub fn arity(comptime T: type, comptime what: []const u8) comptime_int { + return switch (@typeInfo(T)) { + .array => |a| a.len, + .@"struct" => |s| if (s.is_tuple) s.field_names.len else @compileError(what ++ " must be a tuple like `.{ v1, v2 }` or an array"), + else => @compileError(what ++ " must be a tuple like `.{ v1, v2 }` or an array"), + }; } -pub fn emitScfYield( - comptime ValueT: type, - kernel: anytype, - block: *mlir.Block, - comptime N: usize, - values: anytype, - comptime what: []const u8, -) void { - const info = @typeInfo(@TypeOf(values)); - if (info != .@"struct" or !info.@"struct".is_tuple) - @compileError(what ++ " expects a tuple literal"); - if (info.@"struct".field_names.len != N) - @compileError(what ++ ": yield arity must match the scope's declared arity"); +/// The MLIR values of `values`, `N` `ValueT`s. +pub fn innerValues(comptime ValueT: type, comptime N: usize, values: anytype, comptime what: []const u8) [N]*const mlir.Value { + if (arity(@TypeOf(values), what) != N) + @compileError(what ++ ": the arity must match the scope's"); var buf: [N]*const mlir.Value = undefined; - inline for (info.@"struct".field_names, info.@"struct".field_types, 0..) |field_name, Field, i| { - if (Field != ValueT) - @compileError(what ++ ": every tuple element must be a Value"); - buf[i] = @field(values, field_name).inner; + inline for (0..N) |i| { + if (@TypeOf(values[i]) != ValueT) @compileError(what ++ ": every element must be a Value"); + buf[i] = values[i].inner; } + return buf; +} + +/// `values` as `ValueT`s: values as they are, anything else through `kernel.lift`. +fn liftValues(kernel: anytype, comptime ValueT: type, values: anytype, comptime what: []const u8) [arity(@TypeOf(values), what)]ValueT { + var out: [arity(@TypeOf(values), what)]ValueT = undefined; + inline for (0..out.len) |i| out[i] = if (@TypeOf(values[i]) == ValueT) values[i] else kernel.lift(values[i]); + return out; +} + +/// The types `types` names: `*const mlir.Type`s, `ValueT`s for their types, +/// or dtypes (`.f32`) for `kernel.scalarTy`. +pub fn resultTypes(kernel: anytype, comptime ValueT: type, types: anytype, comptime what: []const u8) [arity(@TypeOf(types), what)]*const mlir.Type { + var out: [arity(@TypeOf(types), what)]*const mlir.Type = undefined; + inline for (0..out.len) |i| { + const T = @TypeOf(types[i]); + out[i] = if (T == *const mlir.Type) types[i] else if (T == ValueT) types[i].inner.type_() else kernel.scalarTy(types[i]); + } + return out; +} + +fn scfYield(comptime ValueT: type, comptime N: usize, kernel: anytype, block: *mlir.Block, values: anytype, comptime what: []const u8) void { + const buf = innerValues(ValueT, N, values, what); _ = scf.yield(kernel.ctx, &buf, kernel.loc()).appendTo(block); } @@ -52,8 +103,18 @@ pub fn ForScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usiz const Self = @This(); pub fn yield(self: *Self, values: anytype) void { + if (N == 0) @compileError("ForScope.yield: the loop carries nothing, end it with close()"); + self.end(values); + } + + pub fn close(self: *Self) void { + if (N != 0) @compileError("ForScope.close: the loop carries values, end it with yield()"); + self.end(.{}); + } + + fn end(self: *Self, values: anytype) void { const k = self.kernel; - emitScfYield(ValueT, k, self.body, N, values, "ForScope.yield"); + scfYield(ValueT, N, k, self.body, values, "ForScope.yield"); k.popBlock(); const for_op = scf.for_( k.ctx, @@ -79,9 +140,9 @@ pub fn IfOnlyScope(comptime BuilderT: type, comptime ValueT: type) type { const Self = @This(); - pub fn yieldThen(self: *Self, values: anytype) void { + pub fn close(self: *Self) void { const k = self.kernel; - emitScfYield(ValueT, k, self.then_block, 0, values, "IfOnlyScope.yieldThen"); + scfYield(ValueT, 0, k, self.then_block, .{}, "IfOnlyScope.close"); k.popBlock(); const if_op = scf.if_(k.ctx, self.cond_inner, &.{}, self.then_block, .{}, k.loc()); _ = if_op.appendTo(k.currentBlock()); @@ -101,15 +162,35 @@ pub fn IfScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize const Self = @This(); pub fn yieldThen(self: *Self, values: anytype) void { + if (N == 0) @compileError("IfScope.yieldThen: the if has no results, switch branches with elseBranch()"); + self.endThen(values); + } + + pub fn yieldElse(self: *Self, values: anytype) void { + if (N == 0) @compileError("IfScope.yieldElse: the if has no results, end it with close()"); + self.endElse(values); + } + + pub fn elseBranch(self: *Self) void { + if (N != 0) @compileError("IfScope.elseBranch: the if has results, switch branches with yieldThen()"); + self.endThen(.{}); + } + + pub fn close(self: *Self) void { + if (N != 0) @compileError("IfScope.close: the if has results, end it with yieldElse()"); + self.endElse(.{}); + } + + fn endThen(self: *Self, values: anytype) void { const k = self.kernel; - emitScfYield(ValueT, k, self.then_block, N, values, "IfScope.yieldThen"); + scfYield(ValueT, N, k, self.then_block, values, "IfScope.yieldThen"); k.popBlock(); k.pushBlock(self.else_block); } - pub fn yieldElse(self: *Self, values: anytype) void { + fn endElse(self: *Self, values: anytype) void { const k = self.kernel; - emitScfYield(ValueT, k, self.else_block, N, values, "IfScope.yieldElse"); + scfYield(ValueT, N, k, self.else_block, values, "IfScope.yieldElse"); k.popBlock(); const if_op = scf.if_(k.ctx, self.cond_inner, &self.result_types, self.then_block, .{ .else_block = self.else_block }, k.loc()); _ = if_op.appendTo(k.currentBlock()); @@ -132,18 +213,8 @@ pub fn WhileScope(comptime BuilderT: type, comptime ValueT: type, comptime N: us const Self = @This(); pub fn yieldBefore(self: *Self, cond: ValueT, forwarded: anytype) void { - const info = @typeInfo(@TypeOf(forwarded)); - if (info != .@"struct" or !info.@"struct".is_tuple) - @compileError("WhileScope.yieldBefore: forwarded must be a tuple literal"); - if (info.@"struct".field_names.len != M) - @compileError("WhileScope.yieldBefore: forwarded arity must match after_types arity"); const k = self.kernel; - var buf: [M]*const mlir.Value = undefined; - inline for (info.@"struct".field_names, info.@"struct".field_types, 0..) |field_name, Field, i| { - if (Field != ValueT) - @compileError("WhileScope.yieldBefore: every forwarded element must be a Value"); - buf[i] = @field(forwarded, field_name).inner; - } + const buf = innerValues(ValueT, M, forwarded, "WhileScope.yieldBefore"); _ = scf.condition(k.ctx, cond.inner, &buf, k.loc()).appendTo(self.before_block); k.popBlock(); k.pushBlock(self.after_block); @@ -152,7 +223,7 @@ pub fn WhileScope(comptime BuilderT: type, comptime ValueT: type, comptime N: us pub fn yieldAfter(self: *Self, values: anytype) void { const k = self.kernel; - emitScfYield(ValueT, k, self.after_block, N, values, "WhileScope.yieldAfter"); + scfYield(ValueT, N, k, self.after_block, values, "WhileScope.yieldAfter"); k.popBlock(); const w = scf.while_( k.ctx, @@ -168,6 +239,126 @@ pub fn WhileScope(comptime BuilderT: type, comptime ValueT: type, comptime N: us }; } +/// `openFor`'s bounds as values. A bound that isn't a value takes the type of +/// the first one that is (`kernel.constantLike`); with none, `lower` becomes +/// `defaultLower(kernel, lower)` and the others take its type. +pub fn forBounds( + kernel: anytype, + comptime ValueT: type, + lower: anytype, + upper: anytype, + step: anytype, + comptime defaultLower: anytype, +) [3]ValueT { + const bounds: [3]ValueT = if (@TypeOf(lower) != ValueT and @TypeOf(upper) != ValueT and @TypeOf(step) != ValueT) blk: { + const lb = defaultLower(kernel, lower); + break :blk .{ lb, kernel.constantLike(upper, lb), kernel.constantLike(step, lb) }; + } else blk: { + const first: ValueT = if (@TypeOf(lower) == ValueT) lower else if (@TypeOf(upper) == ValueT) upper else step; + break :blk .{ kernel.constantLike(lower, first), kernel.constantLike(upper, first), kernel.constantLike(step, first) }; + }; + const t = bounds[0].inner.type_(); + if (!bounds[1].inner.type_().eql(t) or !bounds[2].inner.type_().eql(t)) + std.debug.panic("openFor: bounds of types {f}, {f} and {f}", .{ t, bounds[1].inner.type_(), bounds[2].inner.type_() }); + return bounds; +} + +/// Opens `for iv in [bounds[0], bounds[1]) step bounds[2]` carrying `inits`, +/// values or literals `kernel.lift` takes. +pub fn openFor( + kernel: anytype, + comptime ValueT: type, + bounds: [3]ValueT, + inits: anytype, + opts: scf.ForOpts, +) ForScope(@TypeOf(kernel.*), ValueT, arity(@TypeOf(inits), "openFor: inits")) { + const N = comptime arity(@TypeOf(inits), "openFor: inits"); + const values = liftValues(kernel, ValueT, inits, "openFor: inits"); + var types: [N + 1]*const mlir.Type = undefined; + types[0] = bounds[0].inner.type_(); + var inits_inner: [N]*const mlir.Value = undefined; + for (values, 0..) |v, i| { + types[i + 1] = v.inner.type_(); + inits_inner[i] = v.inner; + } + const locs: [N + 1]*const mlir.Location = @splat(kernel.loc()); + const body = mlir.Block.init(&types, &locs); + kernel.pushBlock(body); + var carried: [N]ValueT = undefined; + for (&carried, 0..) |*c, i| c.* = .{ .inner = body.argument(i + 1), .kernel = kernel }; + return .{ + .kernel = kernel, + .body = body, + .lb_inner = bounds[0].inner, + .ub_inner = bounds[1].inner, + .step_inner = bounds[2].inner, + .inits_inner = inits_inner, + .iv = .{ .inner = body.argument(0), .kernel = kernel }, + .carried = carried, + .opts = opts, + }; +} + +pub fn openIf(kernel: anytype, comptime ValueT: type, cond: ValueT) IfOnlyScope(@TypeOf(kernel.*), ValueT) { + const then_block = mlir.Block.init(&.{}, &.{}); + kernel.pushBlock(then_block); + return .{ .kernel = kernel, .cond_inner = cond.inner, .then_block = then_block }; +} + +/// Opens an `if` with an `else`, of results of `result_types` (see `resultTypes`). +pub fn openIfElse( + kernel: anytype, + comptime ValueT: type, + cond: ValueT, + result_types: anytype, +) IfScope(@TypeOf(kernel.*), ValueT, arity(@TypeOf(result_types), "openIfElse: result_types")) { + const then_block = mlir.Block.init(&.{}, &.{}); + const else_block = mlir.Block.init(&.{}, &.{}); + kernel.pushBlock(then_block); + return .{ + .kernel = kernel, + .cond_inner = cond.inner, + .then_block = then_block, + .else_block = else_block, + .result_types = resultTypes(kernel, ValueT, result_types, "openIfElse: result_types"), + }; +} + +/// Opens an `scf.while` carrying `inits` into its condition region, which +/// forwards values of `after_types` (see `resultTypes`) to its body. +pub fn openWhile( + kernel: anytype, + comptime ValueT: type, + inits: anytype, + after_types: anytype, +) WhileScope(@TypeOf(kernel.*), ValueT, arity(@TypeOf(inits), "openWhile: inits"), arity(@TypeOf(after_types), "openWhile: after_types")) { + const N = comptime arity(@TypeOf(inits), "openWhile: inits"); + const M = comptime arity(@TypeOf(after_types), "openWhile: after_types"); + const values = liftValues(kernel, ValueT, inits, "openWhile: inits"); + var before_types: [N]*const mlir.Type = undefined; + var inits_inner: [N]*const mlir.Value = undefined; + for (values, 0..) |v, i| { + before_types[i] = v.inner.type_(); + inits_inner[i] = v.inner; + } + const before_locs: [N]*const mlir.Location = @splat(kernel.loc()); + const before_block = mlir.Block.init(&before_types, &before_locs); + const after_types_ = resultTypes(kernel, ValueT, after_types, "openWhile: after_types"); + const after_locs: [M]*const mlir.Location = @splat(kernel.loc()); + const after_block = mlir.Block.init(&after_types_, &after_locs); + kernel.pushBlock(before_block); + var before_carried: [N]ValueT = undefined; + for (&before_carried, 0..) |*c, i| c.* = .{ .inner = before_block.argument(i), .kernel = kernel }; + return .{ + .kernel = kernel, + .before_block = before_block, + .after_block = after_block, + .inits_inner = inits_inner, + .after_types = after_types_, + .before_carried = before_carried, + }; +} + test { std.testing.refAllDecls(@This()); } diff --git a/dsl/common/dsl.zig b/dsl/common/dsl.zig deleted file mode 100644 index acbf9e19..00000000 --- a/dsl/common/dsl.zig +++ /dev/null @@ -1,17 +0,0 @@ -const std = @import("std"); - -pub const control_flow = @import("control_flow.zig"); -pub const args = @import("args.zig"); - -pub const tupleArity = control_flow.tupleArity; -pub const emitScfYield = control_flow.emitScfYield; -pub const ForScope = control_flow.ForScope; -pub const IfOnlyScope = control_flow.IfOnlyScope; -pub const IfScope = control_flow.IfScope; -pub const WhileScope = control_flow.WhileScope; - -pub const NamedArgs = args.NamedArgs; - -test { - std.testing.refAllDecls(@This()); -} diff --git a/dsl/cuda_tile/BUILD.bazel b/dsl/cuda_tile/BUILD.bazel index 20ed77ad..527c53e4 100644 --- a/dsl/cuda_tile/BUILD.bazel +++ b/dsl/cuda_tile/BUILD.bazel @@ -1,7 +1,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( - name = "cuda_tile_builder", + name = "builder", srcs = [ "control_flow.zig", "dtype.zig", @@ -11,7 +11,7 @@ zig_library( tags = ["manual"], visibility = ["//visibility:public"], deps = [ - "//dsl/common", + "//dsl/common:builder", "//mlir", "//mlir/dialects/cuda_tile", "//stdx", @@ -20,5 +20,5 @@ zig_library( zig_test( name = "test", - deps = [":cuda_tile_builder"], + deps = [":builder"], ) diff --git a/dsl/cuda_tile/README.md b/dsl/cuda_tile/README.md index ef25428d..d8c5fdb8 100644 --- a/dsl/cuda_tile/README.md +++ b/dsl/cuda_tile/README.md @@ -48,6 +48,11 @@ const out = AddOne.call(.{ .x = x }, .{ .out = x.shape() }, .{ }).out; ``` +Put each scope's body in a Zig `{ ... }` block right after `openFor`, `openIf` +or `openLoop`, ending with its `yield` (or `close()`). `openIfElse` gets one +block per branch, ending with `yieldThen`/`yieldElse` (`elseBranch()`/`close()` +without results). + ## What is different from the Triton DSL * **Rank 0 is the scalar.** Every value is a `tile<...>`; `entry` parameters @@ -117,7 +122,8 @@ awk '/^pub const Value = struct/,/^};/' dsl/cuda_tile/builder.zig | grep 'pub fn awk '/^pub const Builder = struct/,0' dsl/cuda_tile/builder.zig | grep ' pub fn ' # what each option struct holds, and its defaults -grep -n '^pub const [A-Za-z]*Opts = struct' -A 8 dsl/cuda_tile/builder.zig +grep -n '^pub const [A-Za-z]*Opts = struct' -A 8 dsl/cuda_tile/builder.zig \ + dsl/cuda_tile/control_flow.zig # for / if / loop scopes and their terminators grep -n 'pub fn ' dsl/cuda_tile/control_flow.zig @@ -128,5 +134,4 @@ grep -n 'pub const DType = enum' -A 30 dsl/cuda_tile/dtype.zig Every op also has a test: `grep -n '^test "' dsl/cuda_tile/builder.zig` lists them, and each one prints IR that round-trips through the verifier, so -they double as usage examples. A worked, runnable example (saxpy, softmax, -gemm, in-place relu) lives in `examples/cuda_tile`. +they double as usage examples. diff --git a/dsl/cuda_tile/builder.zig b/dsl/cuda_tile/builder.zig index f534a9c8..a8414acc 100644 --- a/dsl/cuda_tile/builder.zig +++ b/dsl/cuda_tile/builder.zig @@ -1,8 +1,7 @@ const std = @import("std"); +const common = @import("dsl/common/builder"); const cuda_tile = @import("mlir/dialects/cuda_tile"); -const dsl = @import("dsl/common"); -const tupleArity = dsl.tupleArity; const mlir = @import("mlir"); const stdx = @import("stdx"); @@ -214,6 +213,10 @@ pub const Value = struct { return self.kern().cast(self, dtype); } + pub fn bitCast(self: Value, dtype: DType) Value { + return self.kern().bitCast(self, dtype); + } + pub fn reshape(self: Value, shape_: []const i64) Value { return self.kern().reshape(self, shape_); } @@ -390,6 +393,8 @@ pub fn ForScope(comptime N: usize) type { return cf.ForScope(Builder, Value, N); } +pub const ForOpts = cf.ForOpts; + pub const IfOnlyScope = cf.IfOnlyScope(Builder, Value); pub fn IfScope(comptime N: usize) type { @@ -454,7 +459,7 @@ pub const Builder = struct { name: []const u8, func_op: ?*mlir.Operation, entry_block: ?*mlir.Block, - block_stack: std.ArrayList(*mlir.Block), + block_stack: common.BlockStack = .{}, ct_module: ?*mlir.Operation = null, ct_body: ?*mlir.Block = null, refined_args: []?Value = &.{}, @@ -501,15 +506,15 @@ pub const Builder = struct { .name = name, .func_op = null, .entry_block = null, - .block_stack = .empty, + .block_stack = .{}, }; } - pub fn declareArgs(self: *Builder, spec: anytype) !dsl.NamedArgs(@TypeOf(spec), Value) { + pub fn declareArgs(self: *Builder, spec: anytype) !common.NamedArgs(@TypeOf(spec), Value) { return self.declareArgsOpts(spec, .{}); } - pub fn declareArgsOpts(self: *Builder, spec: anytype, opts: Opts) !dsl.NamedArgs(@TypeOf(spec), Value) { + pub fn declareArgsOpts(self: *Builder, spec: anytype, opts: Opts) !common.NamedArgs(@TypeOf(spec), Value) { const Spec = @TypeOf(spec); const field_names = @typeInfo(Spec).@"struct".field_names; @@ -533,7 +538,7 @@ pub const Builder = struct { try self.declareArgsLowOpts(&arg_specs, opts); - var named: dsl.NamedArgs(Spec, Value) = undefined; + var named: common.NamedArgs(Spec, Value) = undefined; inline for (0.., field_names) |i, field_name| { @field(named, field_name) = self.arg(i); } @@ -571,6 +576,7 @@ pub const Builder = struct { self.func_op = func_op; self.entry_block = entry; + self.block_stack.push(self.arena.allocator(), entry); self.ct_module = ct_module; self.ct_body = ct_body; @@ -604,18 +610,15 @@ pub const Builder = struct { } pub fn pushBlock(self: *Builder, b: *mlir.Block) void { - self.block_stack.append(self.arena.allocator(), b) catch @panic("Builder.pushBlock OOM"); + self.block_stack.push(self.arena.allocator(), b); } pub fn popBlock(self: *Builder) void { - _ = self.block_stack.pop(); + self.block_stack.pop(); } pub fn currentBlock(self: *Builder) *mlir.Block { - if (self.block_stack.items.len > 0) { - return self.block_stack.items[self.block_stack.items.len - 1]; - } - return self.entry_block orelse @panic("Builder has no current block — call declareArgs first"); + return self.block_stack.current(); } fn innerSlice(self: *Builder, values: []const Value) []const *const mlir.Value { @@ -712,27 +715,33 @@ pub const Builder = struct { }; return cuda_tile.floatElemAttr(self.ctx, elem, v); } - const v: i64 = switch (@typeInfo(T)) { - .comptime_int => blk: { - if (value >= std.math.minInt(i64) and value <= std.math.maxInt(i64)) - break :blk @intCast(value); - if (value >= 0 and value <= std.math.maxInt(u64)) - break :blk @bitCast(@as(u64, value)); - @compileError("Builder: integer literal out of 64-bit range"); - }, - .int => |info| if (info.signedness == .signed) @intCast(value) else @bitCast(@as(u64, @intCast(value))), + const v: i128 = switch (@typeInfo(T)) { + .comptime_int, .int => value, .bool => @intFromBool(value), .comptime_float, .float => @intFromFloat(value), else => @compileError("Builder: unsupported scalar " ++ @typeName(T)), }; - return cuda_tile.intElemAttr(elem, v); + return cuda_tile.intElemAttr(elem, intBits(v, @intCast(intBitwidth(dtype)), @tagName(dtype))); + } + + /// `value` as the `i64` an integer constant of `bits` bits is built from. + /// It may be signed or an unsigned bit pattern, so it must lie in + /// [-2^(bits-1), 2^bits). + fn intBits(value: i128, bits: u7, dtype: []const u8) i64 { + if (value < -(@as(i128, 1) << (bits - 1)) or value >= @as(i128, 1) << bits) + std.debug.panic("Builder: {d} does not fit {s}", .{ value, dtype }); + return @truncate(value); } fn splat(self: *Builder, tile_ty: *const mlir.Type, dtype: DType, value: anytype) Value { return self.emit(cuda_tile.constant(self.ctx, cuda_tile.splatAttr(tile_ty, self.elemAttr(dtype, value)), tile_ty, self.loc())); } - pub fn cst(self: *Builder, dtype: DType, value: anytype) Value { + /// `value` as a rank-0 constant of `dtype`; a Value passes through. An + /// integer is signed or an unsigned bit pattern of `dtype`'s width; + /// anything else panics. + pub fn constant(self: *Builder, dtype: DType, value: anytype) Value { + if (@TypeOf(value) == Value) return value; return self.splat(self.scalarTy(dtype), dtype, value); } @@ -778,46 +787,51 @@ pub const Builder = struct { } /// Lift a Zig scalar to a rank-0 constant. Dtype follows the source type; - /// unsigned ints are bit-cast. Use `liftAs` for an explicit dtype. + /// unsigned ints are bit-cast. Use `constant` for an explicit dtype. pub fn lift(self: *Builder, value: anytype) Value { const T = @TypeOf(value); if (T == Value) return value; return switch (@typeInfo(T)) { .comptime_int => blk: { if (value >= std.math.minInt(i32) and value <= std.math.maxInt(i32)) - break :blk self.cst(.i32, value); - break :blk self.cst(.i64, value); + break :blk self.constant(.i32, value); + break :blk self.constant(.i64, value); }, - .int => |info| if (info.bits > 32) self.cst(.i64, value) else self.cst(.i32, value), - .bool => self.cst(.i1, value), - .comptime_float => self.cst(.f32, value), + .int => |info| if (info.bits > 32) self.constant(.i64, value) else self.constant(.i32, value), + .bool => self.constant(.i1, value), + .comptime_float => self.constant(.f32, value), .float => |info| switch (info.bits) { - 16 => self.cst(.f16, value), - 32 => self.cst(.f32, value), - 64 => self.cst(.f64, value), + 16 => self.constant(.f16, value), + 32 => self.constant(.f32, value), + 64 => self.constant(.f64, value), else => @compileError("Builder.lift: unsupported float bitwidth"), }, else => @compileError("Builder.lift: unsupported type " ++ @typeName(T)), }; } - pub fn liftAs(self: *Builder, value: anytype, dtype: DType) Value { - if (@TypeOf(value) == Value) return value; - return self.cst(dtype, value); - } - fn liftMatching(self: *Builder, value: anytype, ref_elem: *const mlir.Type) Value { const T = @TypeOf(value); if (T == Value) return value; return switch (@typeInfo(T)) { - .comptime_int, .comptime_float => self.liftAs(value, self.mlirElemToDType(ref_elem)), + .comptime_int, .comptime_float => self.constant(self.mlirElemToDType(ref_elem), value), else => self.lift(value), }; } + /// `value` as a constant of `like`'s element type; a Value passes through. + pub fn constantLike(self: *Builder, value: anytype, like: Value) Value { + if (@TypeOf(value) == Value) return value; + return self.constMatching(value, like.type_()); + } + + fn i32Bound(self: *Builder, value: anytype) Value { + return self.constant(.i32, value); + } + fn constMatching(self: *Builder, value: anytype, ref: *const mlir.Type) Value { const elem = if (ref.isA(cuda_tile.TileType)) |t| t.elementType() else ref; - return self.liftAs(value, self.mlirElemToDType(elem)); + return self.constant(self.mlirElemToDType(elem), value); } pub fn broadcastLike(self: *Builder, value: anytype, ref: Value) Value { @@ -1345,7 +1359,7 @@ pub const Builder = struct { return self.select(cv, xv, yv); } - pub fn bitcast(self: *Builder, src: Value, dtype: DType) Value { + pub fn bitCast(self: *Builder, src: Value, dtype: DType) Value { return self.emit(cuda_tile.bitcast(self.ctx, src.inner, self.tileLike(src, dtype.toMlir(self.ctx)), self.loc())); } pub fn exti(self: *Builder, src: Value, dtype: DType, signedness: Signedness) Value { @@ -1415,7 +1429,7 @@ pub const Builder = struct { const tgt_bw = dtypeBitwidth(dtype); if (tgt_bw > cur_bw) return self.exti(src, dtype, if (cur_dtype == .i1) .unsigned else .signed); if (tgt_bw < cur_bw) return self.trunci(src, dtype); - return self.bitcast(src, dtype); + return self.bitCast(src, dtype); } pub fn reshape(self: *Builder, src: Value, shape: []const i64) Value { @@ -1632,40 +1646,42 @@ pub const Builder = struct { } /// `for iv in [lower, upper) step` carrying `inits`; close with - /// `scope.yield(.{...})`, then read `scope.results`. + /// `scope.yield(.{...})` (`scope.close()` without inits), then read + /// `scope.results`. pub fn openFor( self: *Builder, lower: anytype, upper: anytype, step: anytype, inits: anytype, - ) ForScope(tupleArity(@TypeOf(inits), "openFor: inits")) { - const N = comptime tupleArity(@TypeOf(inits), "openFor: inits"); - const fields = @typeInfo(@TypeOf(inits)).@"struct".field_names; + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return self.openForOpts(lower, upper, step, inits, .{}); + } - // Literals take the type of the first bound that is a Value; all - // three must agree (`AllTypesMatch` on the op). - const ref_ty: *const mlir.Type = if (@TypeOf(lower) == Value) lower.type_() else if (@TypeOf(upper) == Value) upper.type_() else if (@TypeOf(step) == Value) step.type_() else self.scalarTy(.i32); - const lb_v: Value = if (@TypeOf(lower) == Value) lower else self.constMatching(lower, ref_ty); - const ub_v: Value = if (@TypeOf(upper) == Value) upper else self.constMatching(upper, ref_ty); - const step_v: Value = if (@TypeOf(step) == Value) step else self.constMatching(step, ref_ty); - if (!lb_v.type_().eql(ub_v.type_()) or !lb_v.type_().eql(step_v.type_())) { - std.debug.panic("Builder.openFor: bounds must share one type, got {f}, {f}, {f}", .{ lb_v.type_(), ub_v.type_(), step_v.type_() }); - } + /// `openFor` with `cuda_tile.for` options. + pub fn openForOpts( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + opts: ForOpts, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + const N = comptime common.arity(@TypeOf(inits), "openFor: inits"); + + // Literals take the type of the first bound that is a Value, else + // `i32`; all three must agree (`AllTypesMatch` on the op). + const lb_v, const ub_v, const step_v = common.forBounds(self, Value, lower, upper, step, i32Bound); var block_types: [N + 1]*const mlir.Type = undefined; - var block_locs: [N + 1]*const mlir.Location = undefined; block_types[0] = lb_v.type_(); - block_locs[0] = self.loc(); var inits_inner: [N]*const mlir.Value = undefined; - inline for (fields, 0..) |f, i| { - const raw = @field(inits, f); - const v: Value = if (@FieldType(@TypeOf(inits), f) == Value) raw else self.lift(raw); + inline for (0..N) |i| { + const v: Value = if (@TypeOf(inits[i]) == Value) inits[i] else self.lift(inits[i]); block_types[i + 1] = v.type_(); - block_locs[i + 1] = self.loc(); inits_inner[i] = v.inner; } - + const block_locs: [N + 1]*const mlir.Location = @splat(self.loc()); const body = mlir.Block.init(&block_types, &block_locs); self.pushBlock(body); @@ -1681,64 +1697,42 @@ pub const Builder = struct { .inits_inner = inits_inner, .iv = .{ .inner = body.argument(0), .kernel = self }, .carried = carried, + .opts = opts, }; } pub fn openIf(self: *Builder, cond: Value) IfOnlyScope { const then_block = mlir.Block.init(&.{}, &.{}); self.pushBlock(then_block); - return .{ - .kernel = self, - .cond_inner = cond.inner, - .then_block = then_block, - }; + return .{ .kernel = self, .cond_inner = cond.inner, .then_block = then_block }; } - pub fn openIfElse( - self: *Builder, - cond: Value, - result_types: anytype, - ) IfScope(tupleArity(@TypeOf(result_types), "openIfElse: result_types")) { - const N = comptime tupleArity(@TypeOf(result_types), "openIfElse: result_types"); - const fields = @typeInfo(@TypeOf(result_types)).@"struct".field_names; - - var types: [N]*const mlir.Type = undefined; - inline for (fields, 0..) |f, i| { - if (@FieldType(@TypeOf(result_types), f) != *const mlir.Type) - @compileError("openIfElse: every result_type must be *const mlir.Type (use b.scalarTy/tileTy)"); - types[i] = @field(result_types, f); - } - + pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(common.arity(@TypeOf(result_types), "openIfElse: result_types")) { const then_block = mlir.Block.init(&.{}, &.{}); const else_block = mlir.Block.init(&.{}, &.{}); self.pushBlock(then_block); - return .{ .kernel = self, .cond_inner = cond.inner, .then_block = then_block, .else_block = else_block, - .result_types = types, + .result_types = common.resultTypes(self, Value, result_types, "openIfElse: result_types"), }; } - /// An unstructured loop carrying `inits`; leave with `scope.breakWith` + /// An unstructured loop carrying `inits` (`scope.close()` ends a body that + /// carries nothing, in place of `scope.yield`); leave with `scope.breakWith` /// inside an `if`, and close the body with `scope.yield`. - pub fn openLoop(self: *Builder, inits: anytype) LoopScope(tupleArity(@TypeOf(inits), "openLoop: inits")) { - const N = comptime tupleArity(@TypeOf(inits), "openLoop: inits"); - const fields = @typeInfo(@TypeOf(inits)).@"struct".field_names; - + pub fn openLoop(self: *Builder, inits: anytype) LoopScope(common.arity(@TypeOf(inits), "openLoop: inits")) { + const N = comptime common.arity(@TypeOf(inits), "openLoop: inits"); var block_types: [N]*const mlir.Type = undefined; - var block_locs: [N]*const mlir.Location = undefined; var inits_inner: [N]*const mlir.Value = undefined; - inline for (fields, 0..) |f, i| { - const raw = @field(inits, f); - const v: Value = if (@FieldType(@TypeOf(inits), f) == Value) raw else self.lift(raw); + inline for (0..N) |i| { + const v: Value = if (@TypeOf(inits[i]) == Value) inits[i] else self.lift(inits[i]); block_types[i] = v.type_(); - block_locs[i] = self.loc(); inits_inner[i] = v.inner; } - + const block_locs: [N]*const mlir.Location = @splat(self.loc()); const body = mlir.Block.init(&block_types, &block_locs); self.pushBlock(body); @@ -1768,9 +1762,7 @@ pub const Builder = struct { _ = cuda_tile.return_(self.ctx, &.{}, self.loc()).appendTo(current); } - if (!self.module.operation().verify()) { - return error.InvalidMlir; - } + try common.verify(self.module, "dsl/cuda_tile"); var al: std.Io.Writer.Allocating = .init(self.allocator); defer al.deinit(); @@ -1970,32 +1962,53 @@ test "if, if-else, loop with break, reduce, scan, cast, select" { // if / if-else carrying a value. const is_first = bid.x.eq(0); var only = b.openIf(is_first); - _ = b.store(picked, v, &.{ bid.x, bid.y }); - only.yieldThen(.{}); + { + _ = b.store(picked, v, &.{ bid.x, bid.y }); + only.close(); + } var branch = b.openIfElse(is_first, .{b.tileTy(&.{ 8, 64 }, .f32)}); - branch.yieldThen(.{picked}); - branch.yieldElse(.{t}); + { + branch.yieldThen(.{picked}); + } + { + branch.yieldElse(.{t}); + } const chosen = branch.results[0]; // loop: count up to n, break when reached. - var lp = b.openLoop(.{b.cst(.i32, 0)}); + var lp = b.openLoop(.{b.constant(.i32, 0)}); { const i = lp.carried[0]; var done = b.openIf(i.ge(n)); - lp.breakWith(.{i}); - done.yieldThen(.{}); + { + lp.breakWith(.{i}); + done.close(); + } lp.yield(.{i.add(1)}); } _ = lp.results[0]; + // a loop carrying nothing ends with close(). + var once = b.openLoop(.{}); + { + var first = b.openIf(is_first); + { + once.breakWith(.{}); + first.close(); + } + once.close(); + } + // for with an early `continue` inside an `if`, and i64 bounds from a Value. - var skip = b.openFor(b.cst(.i64, 0), 16, 1, .{b.cst(.i64, 0)}); + var skip = b.openFor(b.constant(.i64, 0), 16, 1, .{b.constant(.i64, 0)}); { const acc = skip.carried[0]; var odd = b.openIf(skip.iv.bitAnd(1).eq(1)); - skip.continueWith(.{acc}); - odd.yieldThen(.{}); + { + skip.continueWith(.{acc}); + odd.close(); + } skip.yield(.{acc.add(skip.iv)}); } try std.testing.expect(skip.results[0].isScalar()); @@ -2033,7 +2046,7 @@ test "atomics, print, globals, alloca, dynamic views, strided view, tokens" { const tiles = b.indexSpaceShape(v); try std.testing.expectEqual(@as(usize, 2), tiles.len); const sv = b.stridedView(tv, &.{ 8, 64 }, &.{ 2, 1 }, .{}); - _ = b.load(sv, &.{ b.cst(.i32, 0), b.cst(.i32, 0) }); + _ = b.load(sv, &.{ b.constant(.i32, 0), b.constant(.i32, 0) }); const bid = b.tileBlockId(); const loaded = b.loadOpts(v, &.{ bid.x, bid.y }, .{ .hints = &.{.{ .arch = .sm_120, .allow_tma = true }} }); @@ -2050,10 +2063,10 @@ test "atomics, print, globals, alloca, dynamic views, strided view, tokens" { // A gather/scatter view: the sparse index is a rank-1 tile of row ids. const gsv = b.gatherScatterView(tv, &.{ 8, 64 }, 0, .zero); - const gathered = b.load(gsv, &.{ b.iota(8, .i32), b.cst(.i32, 0) }); + const gathered = b.load(gsv, &.{ b.iota(8, .i32), b.constant(.i32, 0) }); try std.testing.expectEqual(@as(i64, 8), gathered.dim(0)); try std.testing.expectEqual(@as(i64, 64), gathered.dim(1)); - _ = b.store(gathered, gsv, &.{ b.iota(8, .i32), b.cst(.i32, 0) }); + _ = b.store(gathered, gsv, &.{ b.iota(8, .i32), b.constant(.i32, 0) }); const lut = b.getGlobal("lut", .f32); const scratch = b.alloca(.f32, 64, .{}); @@ -2156,7 +2169,7 @@ test "every remaining elementwise op verifies" { _ = b.xori(i, j); _ = b.cmpi(.not_equal, i, j, .unsigned); - _ = b.bitcast(f, .i32); + _ = f.bitCast(.i32); _ = b.exti(i, .i64, .unsigned); _ = b.trunci(i, .i16); _ = b.ftof(f, .bf16); @@ -2180,7 +2193,7 @@ test "every remaining elementwise op verifies" { _ = b.cat(f, g, 0); _ = b.permute(f, &.{ 1, 0 }); _ = b.transpose(f); - _ = b.extract(f, &.{ b.cst(.i32, 0), b.cst(.i32, 1) }, &.{ 4, 4 }); + _ = b.extract(f, &.{ b.constant(.i32, 0), b.constant(.i32, 1) }, &.{ 4, 4 }); _ = b.expandDims(f, 0); _ = b.reshape(f, &.{32}); _ = b.cumprod(f, 0); diff --git a/dsl/cuda_tile/control_flow.zig b/dsl/cuda_tile/control_flow.zig index 8d69a764..26979e0a 100644 --- a/dsl/cuda_tile/control_flow.zig +++ b/dsl/cuda_tile/control_flow.zig @@ -2,31 +2,14 @@ //! the dialect's terminators: `continue` ends a `for`/`loop` body, `yield` an //! `if`, `break` leaves a `loop`. An already-terminated branch is left alone. +const common = @import("dsl/common/builder"); const std = @import("std"); const cuda_tile = @import("mlir/dialects/cuda_tile"); -const dsl = @import("dsl/common"); const mlir = @import("mlir"); -pub const tupleArity = dsl.tupleArity; - -fn collect(comptime ValueT: type, comptime N: usize, values: anytype, comptime what: []const u8) [N]*const mlir.Value { - const info = @typeInfo(@TypeOf(values)); - if (info != .@"struct" or !info.@"struct".is_tuple) - @compileError(what ++ " expects a tuple literal"); - if (info.@"struct".field_names.len != N) - @compileError(what ++ ": arity must match the scope's declared arity"); - var buf: [N]*const mlir.Value = undefined; - inline for (info.@"struct".field_names, 0..) |f, i| { - if (@FieldType(@TypeOf(values), f) != ValueT) - @compileError(what ++ ": every tuple element must be a Value"); - buf[i] = @field(values, f).inner; - } - return buf; -} - fn terminate(comptime ValueT: type, comptime N: usize, kernel: anytype, block: *mlir.Block, comptime mnemonic: enum { yield, continue_, break_ }, values: anytype, comptime what: []const u8) void { if (block.terminator() != null) return; - const buf = collect(ValueT, N, values, what); + const buf = common.innerValues(ValueT, N, values, what); const op = switch (mnemonic) { .yield => cuda_tile.yield(kernel.ctx, &buf, kernel.loc()), .continue_ => cuda_tile.continue_(kernel.ctx, &buf, kernel.loc()), @@ -35,7 +18,13 @@ fn terminate(comptime ValueT: type, comptime N: usize, kernel: anytype, block: * _ = op.appendTo(block); } -/// `for %iv in (lb to ub, step) iter_values(...)`; `yield` closes it. +pub const ForOpts = struct { + /// Compare the induction variable unsigned. + unsigned_cmp: bool = false, +}; + +/// `for %iv in (lb to ub, step) iter_values(...)`; `yield` closes it, or +/// `close()` when it carries nothing. pub fn ForScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize) type { return struct { kernel: *BuilderT, @@ -47,7 +36,7 @@ pub fn ForScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usiz iv: ValueT, carried: [N]ValueT, results: [N]ValueT = undefined, - unsigned_cmp: bool = false, + opts: ForOpts = .{}, const Self = @This(); @@ -59,10 +48,20 @@ pub fn ForScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usiz } pub fn yield(self: *Self, values: anytype) void { + if (N == 0) @compileError("ForScope.yield: the loop carries nothing, end it with close()"); + self.end(values); + } + + pub fn close(self: *Self) void { + if (N != 0) @compileError("ForScope.close: the loop carries values, end it with yield()"); + self.end(.{}); + } + + fn end(self: *Self, values: anytype) void { const k = self.kernel; terminate(ValueT, N, k, self.body, .continue_, values, "ForScope.yield"); k.popBlock(); - const for_op = cuda_tile.for_(k.ctx, self.lb_inner, self.ub_inner, self.step_inner, &self.inits_inner, self.body, self.unsigned_cmp, k.loc()); + const for_op = cuda_tile.for_(k.ctx, self.lb_inner, self.ub_inner, self.step_inner, &self.inits_inner, self.body, self.opts.unsigned_cmp, k.loc()); _ = for_op.appendTo(k.currentBlock()); for (0..N) |i| self.results[i] = .{ .inner = for_op.result(i), .kernel = k }; } @@ -77,9 +76,9 @@ pub fn IfOnlyScope(comptime BuilderT: type, comptime ValueT: type) type { const Self = @This(); - pub fn yieldThen(self: *Self, values: anytype) void { + pub fn close(self: *Self) void { const k = self.kernel; - terminate(ValueT, 0, k, self.then_block, .yield, values, "IfOnlyScope.yieldThen"); + terminate(ValueT, 0, k, self.then_block, .yield, .{}, "IfOnlyScope.close"); k.popBlock(); _ = cuda_tile.if_(k.ctx, self.cond_inner, &.{}, self.then_block, .{}, k.loc()).appendTo(k.currentBlock()); } @@ -98,13 +97,33 @@ pub fn IfScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize const Self = @This(); pub fn yieldThen(self: *Self, values: anytype) void { + if (N == 0) @compileError("IfScope.yieldThen: the if has no results, switch branches with elseBranch()"); + self.endThen(values); + } + + pub fn yieldElse(self: *Self, values: anytype) void { + if (N == 0) @compileError("IfScope.yieldElse: the if has no results, end it with close()"); + self.endElse(values); + } + + pub fn elseBranch(self: *Self) void { + if (N != 0) @compileError("IfScope.elseBranch: the if has results, switch branches with yieldThen()"); + self.endThen(.{}); + } + + pub fn close(self: *Self) void { + if (N != 0) @compileError("IfScope.close: the if has results, end it with yieldElse()"); + self.endElse(.{}); + } + + fn endThen(self: *Self, values: anytype) void { const k = self.kernel; terminate(ValueT, N, k, self.then_block, .yield, values, "IfScope.yieldThen"); k.popBlock(); k.pushBlock(self.else_block); } - pub fn yieldElse(self: *Self, values: anytype) void { + fn endElse(self: *Self, values: anytype) void { const k = self.kernel; terminate(ValueT, N, k, self.else_block, .yield, values, "IfScope.yieldElse"); k.popBlock(); @@ -115,8 +134,9 @@ pub fn IfScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize }; } -/// `loop iter_values(...)`: the body ends with `continue` (`yield`) and leaves -/// through `break` (`breakWith`, inside an `if`). `results` are the broken. +/// `loop iter_values(...)`: the body ends with `continue` (`yield`, or +/// `close()` when it carries nothing) and leaves through `break` (`breakWith`, +/// inside an `if`). `results` are the broken. pub fn LoopScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize) type { return struct { kernel: *BuilderT, @@ -140,6 +160,16 @@ pub fn LoopScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usi } pub fn yield(self: *Self, values: anytype) void { + if (N == 0) @compileError("LoopScope.yield: the loop carries nothing, end it with close()"); + self.end(values); + } + + pub fn close(self: *Self) void { + if (N != 0) @compileError("LoopScope.close: the loop carries values, end it with yield()"); + self.end(.{}); + } + + fn end(self: *Self, values: anytype) void { const k = self.kernel; terminate(ValueT, N, k, self.body, .continue_, values, "LoopScope.yield"); k.popBlock(); diff --git a/dsl/cute/BUILD.bazel b/dsl/cute/BUILD.bazel index 5693ed5c..ef5fc6d9 100644 --- a/dsl/cute/BUILD.bazel +++ b/dsl/cute/BUILD.bazel @@ -1,7 +1,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( - name = "cute_builder", + name = "builder", srcs = [ "dtype.zig", ], @@ -10,7 +10,7 @@ zig_library( tags = ["manual"], visibility = ["//visibility:public"], deps = [ - "//dsl/common", + "//dsl/common:builder", "//mlir", "//mlir/dialects/arith", "//mlir/dialects/cf", @@ -30,5 +30,5 @@ zig_library( zig_test( name = "test", - deps = [":cute_builder"], + deps = [":builder"], ) diff --git a/dsl/cute/README.md b/dsl/cute/README.md index d8de2608..c7828abe 100644 --- a/dsl/cute/README.md +++ b/dsl/cute/README.md @@ -64,13 +64,15 @@ const c = NaiveElementwiseAdd.call(.{ .gA = a, .gB = b }, .{ .gC = a.shape() }, ## Python to Zig Put each control-flow region in a Zig `{ ... }` block immediately after -`openFor`, `openIf`, or `openWhile`, with its corresponding `yield` call inside -the block. For `openIfElse`, use separate blocks ending in `yieldThen` and -`yieldElse`; likewise, give `openWhile` separate `yieldBefore` and `yieldAfter` -blocks. Keep the scope variable outside these blocks so its `.results` remain -accessible afterward. The braces make the generated control flow visible and -keep region-local values from escaping their scope; the builder's `open` and -`yield` calls still create and close the actual IR regions. +`openFor`, `openIf`, or `openWhile`, with its corresponding `yield` call (or +`close()` for a scope without values) inside the block. For `openIfElse`, use +separate blocks ending in `yieldThen` and `yieldElse` (`elseBranch()` and +`close()` when it has no results); likewise, give `openWhile` separate +`yieldBefore` and `yieldAfter` blocks. Keep the scope variable outside these +blocks so its `.results` remain accessible afterward. The braces make the +generated control flow visible and keep region-local values from escaping +their scope; the builder's `open` and `yield` calls still create and close +the actual IR regions. | `cutlass.cute` | `dsl/cute` | |--------------------------------------------|-------------------------------------------------| @@ -85,7 +87,7 @@ keep region-local values from escaping their scope; the builder's `open` and | `cute.arch.shuffle_sync_bfly(v, lane)` | `b.shuffleXor(v, lane)` | | `cute.arch.griddepcontrol_launch_dependents()` | `b.launchDependents()` | | `x.to(Float32)` | `x.to(.f32)` | -| `if cond:` / `for i in range(...)` | `b.openIf(cond)` / `b.openFor(lb, ub, step, .{})` | +| `if cond:` / `for i in range(...)` | `b.openIf(cond)` / `b.openFor(0, n, 1, .{})` | | `cpasync.make_tiled_tma_atom(...)` | `b.makeTiledTmaLoadAtom(...)` | | `cpasync.tma_partition(...)` | `b.tmaPartition(...)` | | `cute.copy(atom, src, dst)` | `b.copy(atom, src, dst, pred)` | @@ -103,7 +105,7 @@ keep region-local values from escaping their scope; the builder's `open` and compact left-most stride like `cute.make_layout`; a `tensor` argument defaults to XLA's row-major layout, which is what the buffer holds. * **Scalars are `i32` and `f32` by default**, like `Int32`/`Float32`; - `cst(dtype, v)` picks another type and `coerce` lifts literals to the other + `constant(dtype, v)` picks another type and `coerce` lifts literals to the other operand's type. * **`nvvm` and `cuda` are the compiler's.** The context registers NVIDIA's `nvvm` and `cuda` dialects (`nvvm.ops`, `cuda.ops`, from the diff --git a/dsl/cute/builder.zig b/dsl/cute/builder.zig index 59afe202..87f654c0 100644 --- a/dsl/cute/builder.zig +++ b/dsl/cute/builder.zig @@ -2,10 +2,10 @@ const std = @import("std"); const arith = @import("mlir/dialects/arith"); const cf = @import("mlir/dialects/cf"); +const common = @import("dsl/common/builder"); const cuda = @import("mlir/dialects/cuda"); const cute_ir = @import("mlir/dialects/cute_ir"); const cute_nvgpu = @import("mlir/dialects/cute_nvgpu"); -const dsl = @import("dsl/common"); const func = @import("mlir/dialects/func"); const gpu = @import("mlir/dialects/gpu"); const llvm = @import("mlir/dialects/llvm"); @@ -13,7 +13,6 @@ const math = @import("mlir/dialects/math"); const nvvm = @import("mlir/dialects/nvvm"); const scf = @import("mlir/dialects/scf"); const vector = @import("mlir/dialects/vector"); -const tupleArity = dsl.tupleArity; const mlir = @import("mlir"); const stdx = @import("stdx"); @@ -157,7 +156,7 @@ pub const Value = struct { pub fn neg(self: Value) Value { const k = self.kern(); if (self.isFloat()) return k.emit(arith.negf(k.ctx, self.inner, k.loc())); - return k.cst(self.dtype(), 0).sub(self); + return k.constant(self.dtype(), 0).sub(self); } pub fn abs(self: Value) Value { @@ -373,7 +372,7 @@ pub const Tensor = struct { pub fn set(self: Tensor, coord: anytype, value_: anytype) void { const k = self.kernel; const c = k.makeCoord(coord); - const v = k.liftAs(value_, self.dtype).to(self.dtype); + const v = k.constant(self.dtype, value_).to(self.dtype); _ = cute_ir.memref_store(k.ctx, self.inner, c, v.inner, k.loc()).appendTo(k.currentBlock()); } @@ -549,18 +548,18 @@ pub const LayoutOpts = struct { stride: ?[]const i64 = null, }; -pub const IfOnlyScope = dsl.IfOnlyScope(Builder, Value); +pub const IfOnlyScope = common.IfOnlyScope(Builder, Value); pub fn IfScope(comptime N: usize) type { - return dsl.IfScope(Builder, Value, N); + return common.IfScope(Builder, Value, N); } pub fn ForScope(comptime N: usize) type { - return dsl.ForScope(Builder, Value, N); + return common.ForScope(Builder, Value, N); } pub fn WhileScope(comptime N: usize, comptime M: usize) type { - return dsl.WhileScope(Builder, Value, N, M); + return common.WhileScope(Builder, Value, N, M); } pub const Builder = struct { @@ -575,7 +574,7 @@ pub const Builder = struct { function_kind: FunctionKind = .func_only_kernel, func_op: ?*mlir.Operation = null, entry_block: ?*mlir.Block = null, - block_stack: std.ArrayList(*mlir.Block) = .empty, + block_stack: common.BlockStack = .{}, args: []const ArgSpec = &.{}, tensors: []?Tensor = &.{}, @@ -738,6 +737,7 @@ pub const Builder = struct { self.func_op = func_op; self.entry_block = entry; + self.block_stack.push(self.arena.allocator(), entry); self.args = args; self.tensors = try scratch.alloc(?Tensor, args.len); @@ -758,18 +758,15 @@ pub const Builder = struct { } pub fn pushBlock(self: *Builder, b: *mlir.Block) void { - self.block_stack.append(self.arena.allocator(), b) catch @panic("Builder.pushBlock OOM"); + self.block_stack.push(self.arena.allocator(), b); } pub fn popBlock(self: *Builder) void { - _ = self.block_stack.pop(); + self.block_stack.pop(); } pub fn currentBlock(self: *Builder) *mlir.Block { - if (self.block_stack.items.len > 0) { - return self.block_stack.items[self.block_stack.items.len - 1]; - } - return self.entry_block orelse @panic("Builder has no current block; call declareArgs first"); + return self.block_stack.current(); } pub fn emit(self: *Builder, op: *mlir.Operation) Value { @@ -860,7 +857,7 @@ pub const Builder = struct { /// CUDA stream. The native CuTe host bridge substitutes XLA's current /// stream when it executes the wrapper. pub fn cudaStream(self: *Builder) Value { - return self.emit(cuda.cast(self.ctx, cuda.streamType(self.ctx), self.cst(.i64, 0).inner, self.loc())); + return self.emit(cuda.cast(self.ctx, cuda.streamType(self.ctx), self.constant(.i64, 0).inner, self.loc())); } pub const LaunchOptions = struct { @@ -886,8 +883,8 @@ pub const Builder = struct { if (opts.cluster) |cluster| { self.emitVoid(cuda.launch_cfg_cluster_dim(self.ctx, config.inner, .{ cluster[0].inner, cluster[1].inner, cluster[2].inner }, self.loc())); } - self.emitVoid(cuda.launch_cfg_cooperative(self.ctx, config.inner, self.cst(.i32, @intFromBool(opts.cooperative)).inner, self.loc())); - self.emitVoid(cuda.launch_cfg_programmatic_stream_serialization_allowed(self.ctx, config.inner, self.cst(.i32, @intFromBool(opts.use_pdl)).inner, self.loc())); + self.emitVoid(cuda.launch_cfg_cooperative(self.ctx, config.inner, self.constant(.i32, @intFromBool(opts.cooperative)).inner, self.loc())); + self.emitVoid(cuda.launch_cfg_programmatic_stream_serialization_allowed(self.ctx, config.inner, self.constant(.i32, @intFromBool(opts.use_pdl)).inner, self.loc())); return .{ .inner = config.inner, .kernel = self }; } @@ -930,14 +927,14 @@ pub const Builder = struct { } self.func_op = null; self.entry_block = null; - self.block_stack.clearRetainingCapacity(); + self.block_stack.clear(); self.args = &.{}; self.tensors = &.{}; } pub fn finishProgram(self: *Builder) FinishError![:0]const u8 { std.debug.assert(self.entry_block == null and self.func_op == null); - if (!self.module.operation().verify()) return error.InvalidMlir; + try common.verify(self.module, "dsl/cute"); return self.renderModule(); } @@ -1900,7 +1897,7 @@ pub const Builder = struct { if (cfg.m != 128 or cfg.k != 32 or cfg.n < 8 or cfg.n > 256 or cfg.n % 8 != 0) { std.debug.panic("unsupported SM100 block-scaled MMA shape {d}x{d}x{d}", .{ cfg.m, cfg.n, cfg.k }); } - const disabled = self.cst(.i1, false); + const disabled = self.constant(.i1, false); const atom = self.makeAtom(self.blockScaledAtomType(cfg), .{ disabled, disabled, disabled, scale_ptr, scale_ptr }); const tiled = self.emit(cute_ir.make_tiled_mma(self.ctx, atom.inner, self.blockScaledMmaType(cfg), self.loc())); return .{ .atom = atom, .tiled = .{ .inner = tiled.inner, .kernel = self } }; @@ -1935,7 +1932,7 @@ pub const Builder = struct { pub fn allocTmem(self: *Builder, columns: u16, holding_ptr: anytype, opts: TmemOptions) void { self.emitVoid(cute_nvgpu.arch_sm100_alloc_tmem( self.ctx, - self.cst(.i32, columns).inner, + self.constant(.i32, columns).inner, self.asValue(holding_ptr).inner, .{ .is_two_cta = if (opts.two_cta) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, self.loc(), @@ -1955,7 +1952,7 @@ pub const Builder = struct { self.emitVoid(cute_nvgpu.arch_sm100_dealloc_tmem( self.ctx, self.asValue(pointer).inner, - self.cst(.i32, columns).inner, + self.constant(.i32, columns).inner, .{ .is_two_cta = if (opts.two_cta) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, self.loc(), )); @@ -2225,7 +2222,7 @@ pub const Builder = struct { const shuffled = self.emit(llvm.inline_asm( self.ctx, "shfl.sync.bfly.b32 $0, $1, $2, 31, 0xffffffff;", - &.{ bits.inner, self.cst(.i32, offset).inner }, + &.{ bits.inner, self.constant(.i32, offset).inner }, .{ .constraints = "=r,r,r", .result_type = .int(self.ctx, .i32) }, self.loc(), )); @@ -2241,7 +2238,7 @@ pub const Builder = struct { const shuffled = self.emit(llvm.inline_asm( self.ctx, "shfl.sync.idx.b32 $0, $1, $2, 31, 0xffffffff;", - &.{ bits.inner, self.liftAs(lane, .i32).inner }, + &.{ bits.inner, self.constant(.i32, lane).inner }, .{ .constraints = "=r,r,r", .result_type = .int(self.ctx, .i32) }, self.loc(), )); @@ -2307,17 +2304,17 @@ pub const Builder = struct { /// Initialize a shared-memory mbarrier expecting `arrivals` per phase. pub fn mbarrierInit(self: *Builder, barrier: Value, arrivals: u32) void { - self.emitVoid(nvvm.mbarrier_init(self.ctx, self.sharedBarrierPtr(barrier).inner, self.cst(.i32, arrivals).inner, self.loc())); + self.emitVoid(nvvm.mbarrier_init(self.ctx, self.sharedBarrierPtr(barrier).inner, self.constant(.i32, arrivals).inner, self.loc())); } /// Arrive and expect `bytes` of asynchronous transactions (SM90+), as a /// TMA producer does before issuing its copies. pub fn mbarrierArriveExpectTx(self: *Builder, barrier: Value, bytes: u32) void { - self.emitVoid(nvvm.mbarrier_arrive_expect_tx(self.ctx, self.sharedBarrierPtr(barrier).inner, self.cst(.i32, bytes).inner, .cta, self.loc())); + self.emitVoid(nvvm.mbarrier_arrive_expect_tx(self.ctx, self.sharedBarrierPtr(barrier).inner, self.constant(.i32, bytes).inner, .cta, self.loc())); } pub fn mbarrierArrive(self: *Builder, barrier: Value, count: u32) void { - self.emitVoid(nvvm.mbarrier_arrive(self.ctx, self.sharedBarrierPtr(barrier).inner, self.cst(.i32, count).inner, .cta, self.loc())); + self.emitVoid(nvvm.mbarrier_arrive(self.ctx, self.sharedBarrierPtr(barrier).inner, self.constant(.i32, count).inner, .cta, self.loc())); } /// Test whether phase `phase` has completed, without suspending (SM90+). @@ -2328,7 +2325,7 @@ pub const Builder = struct { /// Wait for phase `phase`, letting the warp sleep for up to /// `suspend_time` cycles per attempt (SM90+). pub fn mbarrierTryWaitParity(self: *Builder, barrier: Value, phase: Value, suspend_time: u32) void { - self.emitVoid(nvvm.mbarrier_try_wait_parity(self.ctx, self.sharedBarrierPtr(barrier).inner, phase.inner, self.cst(.i32, suspend_time).inner, self.loc())); + self.emitVoid(nvvm.mbarrier_try_wait_parity(self.ctx, self.sharedBarrierPtr(barrier).inner, phase.inner, self.constant(.i32, suspend_time).inner, self.loc())); } /// Wait for a pipeline phase with the two-step sequence of CuTe's @@ -2339,8 +2336,10 @@ pub const Builder = struct { pub fn mbarrierWait(self: *Builder, barrier: Value, phase: Value) void { const ready = self.mbarrierWaitParity(barrier, phase); var pending = self.openIf(ready.eq(false)); - self.mbarrierTryWaitParity(barrier, phase, 10_000_000); - pending.yieldThen(.{}); + { + self.mbarrierTryWaitParity(barrier, phase, 10_000_000); + pending.close(); + } } /// Make mbarrier initialization visible to the async proxy (SM90+). @@ -2432,18 +2431,18 @@ pub const Builder = struct { /// `bar.sync id, threads`: named barrier over `threads` threads. pub fn barrierSync(self: *Builder, id: i32, threads: i32) void { - _ = self.callIntrinsic("llvm.nvvm.barrier.cta.sync.aligned.count", &.{ self.cst(.i32, id), self.cst(.i32, threads) }, null); + _ = self.callIntrinsic("llvm.nvvm.barrier.cta.sync.aligned.count", &.{ self.constant(.i32, id), self.constant(.i32, threads) }, null); } /// `bar.red.or.pred`: named barrier over `threads` threads that also returns whether /// `predicate` (i1) is true in any of them. pub fn barrierReduceOr(self: *Builder, id: i32, threads: i32, predicate: Value) Value { - return self.callIntrinsic("llvm.nvvm.barrier.cta.red.or.aligned.count", &.{ self.cst(.i32, id), self.cst(.i32, threads), predicate }, mlir.Type.int(self.ctx, .i1)).?; + return self.callIntrinsic("llvm.nvvm.barrier.cta.red.or.aligned.count", &.{ self.constant(.i32, id), self.constant(.i32, threads), predicate }, mlir.Type.int(self.ctx, .i1)).?; } /// `bar.warp.sync 0xffffffff`. pub fn syncWarp(self: *Builder) void { - self.emitVoid(nvvm.bar_warp_sync(self.ctx, self.cst(.i32, -1).inner, self.loc())); + self.emitVoid(nvvm.bar_warp_sync(self.ctx, self.constant(.i32, -1).inner, self.loc())); } /// Non-blocking test of mbarrier phase `parity` (i1). @@ -2471,7 +2470,7 @@ pub const Builder = struct { const d = self.addressPtr(dst, .shared); const g = self.addressPtr(src, .global); if (valid) |v| { - _ = self.callIntrinsic("llvm.nvvm.cp.async.cg.shared.global.16.s", &.{ d, g, self.select(v, self.cst(.i32, 16), self.cst(.i32, 0)) }, null); + _ = self.callIntrinsic("llvm.nvvm.cp.async.cg.shared.global.16.s", &.{ d, g, self.select(v, self.constant(.i32, 16), self.constant(.i32, 0)) }, null); } else { _ = self.callIntrinsic("llvm.nvvm.cp.async.cg.shared.global.16", &.{ d, g }, null); } @@ -2496,11 +2495,11 @@ pub const Builder = struct { /// Warp-wide TMEM allocation of `columns`; the base address is written to `holder` (i32 smem). pub fn tmemAllocAt(self: *Builder, holder: Value, columns: u32) void { - self.emitVoid(nvvm.tcgen05_alloc(self.ctx, self.addressPtr(holder, .shared).inner, self.cst(.i32, columns).inner, self.loc())); + self.emitVoid(nvvm.tcgen05_alloc(self.ctx, self.addressPtr(holder, .shared).inner, self.constant(.i32, columns).inner, self.loc())); } pub fn tmemDeallocAt(self: *Builder, taddr: Value, columns: u32) void { - self.emitVoid(nvvm.tcgen05_dealloc(self.ctx, self.addressPtr(taddr, .tensor).inner, self.cst(.i32, columns).inner, self.loc())); + self.emitVoid(nvvm.tcgen05_dealloc(self.ctx, self.addressPtr(taddr, .tensor).inner, self.constant(.i32, columns).inner, self.loc())); } pub const Tcgen05Fence = nvvm.Tcgen05FenceKind; @@ -2553,7 +2552,7 @@ pub const Builder = struct { /// Warp-wide maximum of an f32 (`redux.sync.max.f32`, SM100). pub fn warpMaxF32(self: *Builder, value: Value) Value { - return self.callIntrinsic("llvm.nvvm.redux.sync.fmax", &.{ value, self.cst(.i32, -1) }, mlir.Type.float(self.ctx, .f32)).?; + return self.callIntrinsic("llvm.nvvm.redux.sync.fmax", &.{ value, self.constant(.i32, -1) }, mlir.Type.float(self.ctx, .f32)).?; } /// Fast approximate 2^x (`ex2.approx.ftz.f32`). @@ -2637,7 +2636,11 @@ pub const Builder = struct { // ==================== scalars ==================== - pub fn cst(self: *Builder, dt: DType, value: anytype) Value { + /// `value` as a constant of `dt`; a Value passes through. An integer is + /// signed or an unsigned bit pattern of `dt`'s width; anything else + /// panics. + pub fn constant(self: *Builder, dt: DType, value: anytype) Value { + if (@TypeOf(value) == Value) return value; const ctx = self.ctx; const T = @TypeOf(value); if (isFloatDtype(dt)) { @@ -2646,21 +2649,29 @@ pub const Builder = struct { .int => @floatFromInt(value), .bool => @floatFromInt(@intFromBool(value)), .float => @floatCast(value), - else => @compileError("Builder.cst: unsupported value type " ++ @typeName(T)), + else => @compileError("Builder.constant: unsupported value type " ++ @typeName(T)), }; return self.emit(switch (dt) { inline .f16, .bf16, .f32, .f64, .f4e2m1fn, .f8e4m3fn, .f8e5m2, .f8e8m0fnu => |ft| arith.constant_float(ctx, f, @field(mlir.FloatTypes, @tagName(ft)), self.loc()), else => unreachable, }); } - const i: i64 = switch (@typeInfo(T)) { - .comptime_int, .int => @intCast(value), + const i: i128 = switch (@typeInfo(T)) { + .comptime_int, .int => value, .bool => @intFromBool(value), .comptime_float, .float => @intFromFloat(value), - else => @compileError("Builder.cst: unsupported value type " ++ @typeName(T)), + else => @compileError("Builder.constant: unsupported value type " ++ @typeName(T)), }; - if (!fitsInt(i, dt)) std.debug.panic("Builder.cst: {d} does not fit {s}", .{ i, @tagName(dt) }); - return self.emit(arith.constant_int(ctx, i, dt.toMlir(ctx), self.loc())); + return self.emit(arith.constant_int(ctx, intBits(i, @intCast(dtypeBitwidth(dt)), @tagName(dt)), dt.toMlir(ctx), self.loc())); + } + + /// `value` as the `i64` an integer constant of `bits` bits is built from. + /// It may be signed or an unsigned bit pattern, so it must lie in + /// [-2^(bits-1), 2^bits). + fn intBits(value: i128, bits: u7, dtype: []const u8) i64 { + if (value < -(@as(i128, 1) << (bits - 1)) or value >= @as(i128, 1) << bits) + std.debug.panic("Builder.constant: {d} does not fit {s}", .{ value, dtype }); + return @truncate(value); } fn fitsInt(i: i64, dt: DType) bool { @@ -2685,27 +2696,27 @@ pub const Builder = struct { const T = @TypeOf(value); if (T == Value) return value; return switch (@typeInfo(T)) { - .comptime_int, .int => if (literalFits(value, .i32)) self.cst(.i32, value) else self.cst(.i64, value), - .bool => self.cst(.i1, value), - .comptime_float => self.cst(.f32, value), + .comptime_int, .int => if (literalFits(value, .i32)) self.constant(.i32, value) else self.constant(.i64, value), + .bool => self.constant(.i1, value), + .comptime_float => self.constant(.f32, value), .float => |info| switch (info.bits) { - 16 => self.cst(.f16, value), - 32 => self.cst(.f32, value), - 64 => self.cst(.f64, value), + 16 => self.constant(.f16, value), + 32 => self.constant(.f32, value), + 64 => self.constant(.f64, value), else => @compileError("Builder.lift: unsupported float bitwidth"), }, else => @compileError("Builder.lift: unsupported type " ++ @typeName(T)), }; } - pub fn liftAs(self: *Builder, value: anytype, dt: DType) Value { - if (@TypeOf(value) == Value) return value; - return self.cst(dt, value); + /// `value` as a constant of `like`'s element type; a Value passes through. + pub fn constantLike(self: *Builder, value: anytype, like: Value) Value { + return self.constant(like.dtype(), value); } fn liftLike(self: *Builder, value: anytype, ref: Value) Value { const dt = ref.dtype(); - return if (literalFits(value, dt)) self.cst(dt, value) else self.lift(value); + return if (literalFits(value, dt)) self.constant(dt, value) else self.lift(value); } /// Lift a literal to the other operand's dtype when it fits, else to its @@ -2734,56 +2745,41 @@ pub const Builder = struct { // ==================== scf ==================== + pub fn scalarTy(self: *const Builder, dtype: DType) *const mlir.Type { + return dtype.toMlir(self.ctx); + } + pub fn openIf(self: *Builder, cond: Value) IfOnlyScope { - const then_block = mlir.Block.init(&.{}, &.{}); - self.pushBlock(then_block); - return .{ .kernel = self, .cond_inner = cond.inner, .then_block = then_block }; + return common.openIf(self, Value, cond); } - pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(tupleArity(@TypeOf(result_types), "openIfElse: result_types")) { - const N = comptime tupleArity(@TypeOf(result_types), "openIfElse: result_types"); - var types: [N]*const mlir.Type = undefined; - inline for (@typeInfo(@TypeOf(result_types)).@"struct".field_names, 0..) |f, i| { - types[i] = @field(result_types, f); - } - const then_block = mlir.Block.init(&.{}, &.{}); - const else_block = mlir.Block.init(&.{}, &.{}); - self.pushBlock(then_block); - return .{ .kernel = self, .cond_inner = cond.inner, .then_block = then_block, .else_block = else_block, .result_types = types }; + pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(common.arity(@TypeOf(result_types), "openIfElse: result_types")) { + return common.openIfElse(self, Value, cond, result_types); } - /// `for iv in range(lower, upper, step)` with loop-carried `inits`. - pub fn openFor(self: *Builder, lower: anytype, upper: anytype, step: anytype, inits: anytype) ForScope(tupleArity(@TypeOf(inits), "openFor: inits")) { - const N = comptime tupleArity(@TypeOf(inits), "openFor: inits"); - const lb = self.lift(lower); - const ub = self.liftAs(upper, lb.dtype()); - const st = self.liftAs(step, lb.dtype()); + /// `for iv in range(lower, upper, step)` with loop-carried `inits`. Literal + /// bounds take the type of a Value bound, else `lower` is lifted (`i32` + /// when it fits). + pub fn openFor( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return self.openForOpts(lower, upper, step, inits, .{}); + } - var block_types: [N + 1]*const mlir.Type = undefined; - var block_locs: [N + 1]*const mlir.Location = undefined; - block_types[0] = lb.type_(); - block_locs[0] = self.loc(); - var inits_inner: [N]*const mlir.Value = undefined; - inline for (@typeInfo(@TypeOf(inits)).@"struct".field_names, 0..) |f, i| { - const v = self.lift(@field(inits, f)); - block_types[i + 1] = v.type_(); - block_locs[i + 1] = self.loc(); - inits_inner[i] = v.inner; - } - const body = mlir.Block.init(&block_types, &block_locs); - self.pushBlock(body); - var carried: [N]Value = undefined; - for (0..N) |i| carried[i] = .{ .inner = body.argument(i + 1), .kernel = self }; - return .{ - .kernel = self, - .body = body, - .lb_inner = lb.inner, - .ub_inner = ub.inner, - .step_inner = st.inner, - .inits_inner = inits_inner, - .iv = .{ .inner = body.argument(0), .kernel = self }, - .carried = carried, - }; + /// `openFor` with `scf.for` options. + pub fn openForOpts( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + opts: common.ForOpts, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return common.openFor(self, Value, common.forBounds(self, Value, lower, upper, step, Builder.lift), inits, opts); } /// `scf.while` with explicit loop-carried values. The before region sees @@ -2794,43 +2790,8 @@ pub const Builder = struct { self: *Builder, inits: anytype, after_types: anytype, - ) WhileScope( - tupleArity(@TypeOf(inits), "openWhile: inits"), - tupleArity(@TypeOf(after_types), "openWhile: after_types"), - ) { - const N = comptime tupleArity(@TypeOf(inits), "openWhile: inits"); - const M = comptime tupleArity(@TypeOf(after_types), "openWhile: after_types"); - - var init_types: [N]*const mlir.Type = undefined; - var init_locs: [N]*const mlir.Location = undefined; - var inits_inner: [N]*const mlir.Value = undefined; - inline for (@typeInfo(@TypeOf(inits)).@"struct".field_names, 0..) |f, i| { - const v = self.lift(@field(inits, f)); - init_types[i] = v.type_(); - init_locs[i] = self.loc(); - inits_inner[i] = v.inner; - } - - var result_types: [M]*const mlir.Type = undefined; - var result_locs: [M]*const mlir.Location = undefined; - inline for (@typeInfo(@TypeOf(after_types)).@"struct".field_names, 0..) |f, i| { - result_types[i] = @field(after_types, f); - result_locs[i] = self.loc(); - } - - const before = mlir.Block.init(&init_types, &init_locs); - const after = mlir.Block.init(&result_types, &result_locs); - self.pushBlock(before); - var before_carried: [N]Value = undefined; - for (0..N) |i| before_carried[i] = .{ .inner = before.argument(i), .kernel = self }; - return .{ - .kernel = self, - .before_block = before, - .after_block = after, - .inits_inner = inits_inner, - .after_types = result_types, - .before_carried = before_carried, - }; + ) WhileScope(common.arity(@TypeOf(inits), "openWhile: inits"), common.arity(@TypeOf(after_types), "openWhile: after_types")) { + return common.openWhile(self, Value, inits, after_types); } // ==================== module ==================== @@ -2846,7 +2807,7 @@ pub const Builder = struct { } const func_op = self.func_op orelse return error.InvalidMlir; func_op.setAttributeByName("nvvm.reqntid", .denseArray(self.ctx, .i32, &block)); - if (!self.module.operation().verify()) return error.InvalidMlir; + try common.verify(self.module, "dsl/cute"); return self.renderModule(); } @@ -2928,12 +2889,16 @@ test "guard, shared memory, sync, layouts, for loop" { b.namedBarrier(2, 128); b.launchDependents(); b.waitForDependency(); - const j = b.cst(.i32, 127).sub(tid); + const j = b.constant(.i32, 127).sub(tid); var guard = b.openIf(i.lt(4096)); - var acc = b.openFor(0, 4, 1, .{b.cst(.f32, 0.0)}); - acc.yield(.{acc.carried[0].add(smem.get(.{j}).mul(acc.iv.to(.f32)))}); - a.out.set(.{i}, acc.results[0]); - guard.yieldThen(.{}); + { + var acc = b.openFor(0, 4, 1, .{b.constant(.f32, 0.0)}); + { + acc.yield(.{acc.carried[0].add(smem.get(.{j}).mul(acc.iv.to(.f32)))}); + } + a.out.set(.{i}, acc.results[0]); + guard.close(); + } const kernel = try b.finish(.{ 128, 1, 1 }); defer std.testing.allocator.free(kernel); @@ -2998,9 +2963,13 @@ test "casts, select, if-else, integer widening" { try std.testing.expectEqual(DType.i32, bound.to(.i32).dtype()); const picked = b.select(bound, shuffled, 2.0); - var scope = b.openIfElse(tid.lt(4), .{DType.f16.toMlir(ctx)}); - scope.yieldThen(.{picked.to(.f16)}); - scope.yieldElse(.{b.cst(.f16, 1)}); + var scope = b.openIfElse(tid.lt(4), .{.f16}); + { + scope.yieldThen(.{picked.to(.f16)}); + } + { + scope.yieldElse(.{b.constant(.f16, 1)}); + } a.p.set(.{tid}, scope.results[0]); const kernel = try b.finish(.{ 1, 1, 1 }); @@ -3042,10 +3011,10 @@ test "complete program emits a Zig-built CUDA host launch" { .input = .{ .ptr = DType.f32 }, .output = .{ .ptr = DType.f32 }, }); - const one = b.cst(.i32, 1); + const one = b.constant(.i32, 1); const config = b.makeLaunchConfig(.{ .grid = .{ one, one, one }, - .block = .{ b.cst(.i32, 128), one, one }, + .block = .{ b.constant(.i32, 128), one, one }, .dynamic_smem = b.kernelSmemSize("program_add"), .stream = b.cudaStream(), .use_pdl = true, diff --git a/dsl/fly/BUILD.bazel b/dsl/fly/BUILD.bazel index 492134dc..c8bdf854 100644 --- a/dsl/fly/BUILD.bazel +++ b/dsl/fly/BUILD.bazel @@ -1,7 +1,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( - name = "fly_builder", + name = "builder", srcs = [ "dtype.zig", "layout.zig", @@ -11,7 +11,7 @@ zig_library( tags = ["manual"], visibility = ["//visibility:public"], deps = [ - "//dsl/common", + "//dsl/common:builder", "//mlir", "//mlir/dialects/arith", "//mlir/dialects/fly", @@ -31,5 +31,5 @@ zig_library( zig_test( name = "test", - deps = [":fly_builder"], + deps = [":builder"], ) diff --git a/dsl/fly/PORTING.md b/dsl/fly/PORTING.md index 491db667..f7cb19f3 100644 --- a/dsl/fly/PORTING.md +++ b/dsl/fly/PORTING.md @@ -55,8 +55,8 @@ runtime must agree on the dialect version. FlyDSL HEAD may differ from both. | `v.to(fx.Float32)` | `v.to(.f32)` | | `x * rrms` (vector × scalar) | `x.mul(rrms)` | | `fmath.rsqrt(x, fastmath=fast)` | `b.setFastMath(true); b.rsqrt(x)` | -| `lane < RED_SLOTS`, `cond.select(a, b)` | `lane.cmp(.lt, n)`, `cond.select(a, b)` | -| `if tid == 0: ...` | `var s = b.openIf(tid.cmp(.eq, 0)); ...; s.yieldThen(.{});` | +| `lane < RED_SLOTS`, `cond.select(a, b)` | `lane.lt(n)`, `cond.select(a, b)` | +| `if tid == 0: ...` | `var s = b.openIf(tid.eq(0)); ...; s.close();` | | `for i in range_constexpr(n)` | a plain Zig `for (0..n)` (unrolled at emit time) | | `for i in range(lo, hi)` with carried values | `b.openFor(lo, hi, 1, .{ init })` + `.yield(.{ ... })` | | `launch(grid=..., block=(256,1,1))` | `K.call(inputs, outputs, .{ .grid = ..., .threads = 256 })` | @@ -82,7 +82,7 @@ runtime must agree on the dialect version. FlyDSL HEAD may differ from both. kernel `Config`; branch on it with plain `if` in `run`. - Python lists of vectors across an unrolled loop are Zig arrays of `Value`. `v.insert(i, x)` returns a new vector; `v.extract(i)` reads an element. - `v.bitcast(dtype)` preserves total bits and adjusts vector length. + `v.bitCast(dtype)` preserves total bits and adjusts vector length. - Keep unsigned semantics explicit for packed bytes and offsets with `toUnsigned`, `divUnsigned`, `remUnsigned` and `cmpUnsigned`. - `v.shuffleVector(rhs, indices)` rearranges vector elements within a thread; diff --git a/dsl/fly/README.md b/dsl/fly/README.md index 266d643e..b886988b 100644 --- a/dsl/fly/README.md +++ b/dsl/fly/README.md @@ -178,10 +178,11 @@ coordinate tensor); `t.fill(0)` splats. `b.rmemTensor(layout, dtype)` is `fx.make_rmem_tensor`; `b.sharedArray(.f32, n, 16)` is a static LDS array (`fx.SharedAllocator().allocate(fx.Array[...])`). -Scalars and vectors: `add/sub/mul/div/rem` (a scalar broadcasts against a -vector), `to(dtype)`, `cmp(.lt, x)`, `cond.select(a, b)`, `v.reduce(.add)`, -`v.shuffleXor(off, 64)`, `b.rsqrt(x)`, `b.constant(.f32, 1.0)`, -`b.threadId(.x)` / `b.blockId(.x)` (already `i32`), `b.barrier()`. +Scalars and vectors: `add/sub/mul/div/rem` and `lt(x)` (`cmp` for integers, +`cmpf` for floats; `cmpUnsigned`) broadcast a scalar against a vector; +`to(dtype)`, `cond.select(a, b)`, `v.reduce(.add)`, `v.shuffleXor(off, 64)`, +`b.rsqrt(x)`, `b.constant(.f32, 1.0)`, `b.threadId(.x)` / `b.blockId(.x)` +(already `i32`), `b.barrier()`. Integer types are signless in MLIR. `div`, `rem`, `cmp` and `to` use signed integer semantics, except that `to` treats predicates as unsigned. @@ -190,7 +191,7 @@ operations explicit. `bitAnd`, `bitOr`, `bitXor`, `shl` and `shrU` operate on integer bits. `splat` creates a vector, including when its length is one. Numeric casts -preserve vector shape. `bitcast` preserves total bits and adjusts vector +preserve vector shape. `bitCast` preserves total bits and adjusts vector length to the destination element width; scalar bitcasts require equal widths. Use `extract` to obtain a scalar, `insert` to replace an element and `shuffleVector` to select elements from two vectors. Lane `shuffle` @@ -213,8 +214,10 @@ pointers. The ordering is monotonic; callers provide barriers or kernel dependencies before consuming completed counters. Control flow uses `dsl/common`: `b.openFor(lo, hi, step, .{ inits })` -with `.yield(.{ ... })`, `b.openIf(cond)` with `.yieldThen(.{})`, -`b.openIfElse(cond, .{ types })` with `.yieldThen` / `.yieldElse`. +with `.yield(.{ ... })` (`.close()` without inits), `b.openIf(cond)` with +`.close()`, `b.openIfElse(cond, .{ types })` with `.yieldThen` / `.yieldElse` +(`.elseBranch()` / `.close()` without types). Types are dtypes, Values for +their types, or MLIR types; tuples can also be arrays. `openWhile` has separate condition and body regions: `before_carried` feeds `yieldBefore`, `after_carried` feeds `yieldAfter`, and `results` contains the values returned when the condition becomes false. @@ -232,6 +235,13 @@ keep their register state in SSA values: and `results`. Branches and while loops also return SSA values through their scope objects. +Put each scope's body in a Zig `{ ... }` block right after `openFor`, +`openIf` or `openWhile`, ending with its `yield` (or `close()` for a scope +without values). `openIfElse` and `openWhile` get one block per region, ending +with `yieldThen`/`yieldElse` (`elseBranch()`/`close()` without results) and +`yieldBefore`/`yieldAfter`. The scope variable stays outside, so its `results` +stay readable after the blocks. + The MoE and sparse-MLA kernels linked above use this path with the published runtime. Straight-line kernels can still use `rmemTensor`, `makeFragmentLike`, `fill` and `gemm`. diff --git a/dsl/fly/builder.zig b/dsl/fly/builder.zig index 89ff45cc..ab317c38 100644 --- a/dsl/fly/builder.zig +++ b/dsl/fly/builder.zig @@ -1,7 +1,7 @@ const std = @import("std"); const arith = @import("mlir/dialects/arith"); -const dsl = @import("dsl/common"); +const common = @import("dsl/common/builder"); const fly = @import("mlir/dialects/fly"); const func = @import("mlir/dialects/func"); const gpu = @import("mlir/dialects/gpu"); @@ -12,7 +12,6 @@ const rocdl = @import("mlir/dialects/rocdl"); const scf = @import("mlir/dialects/scf"); const ub = @import("mlir/dialects/ub"); const vector = @import("mlir/dialects/vector"); -const tupleArity = dsl.tupleArity; const mlir = @import("mlir"); const stdx = @import("stdx"); @@ -122,7 +121,7 @@ pub const Value = struct { fn binary(self: Value, comptime build: anytype, rhs: anytype) Value { const k = self.kern(); - const r = k.lift(rhs); + const r = k.toStatic(rhs); return k.emit(build(k.ctx, self.inner, r.inner, k.loc())); } @@ -307,7 +306,7 @@ pub const Value = struct { /// Broadcast a scalar against a vector operand. fn coerceArith(self: Value, rhs: anytype) struct { Value, Value } { const k = self.kern(); - const r: Value = if (@TypeOf(rhs) == Value) rhs else k.constLike(rhs, self); + const r: Value = if (@TypeOf(rhs) == Value) rhs else k.constantLike(rhs, self); if (self.type_().eql(r.type_())) return .{ self, r }; if (self.type_().isA(mlir.VectorType) != null and r.type_().isA(mlir.VectorType) == null) return .{ self, k.emit(vector.broadcast(k.ctx, r.inner, self.type_(), k.loc())) }; @@ -433,7 +432,7 @@ pub const Value = struct { } /// Preserve total bits, adjusting vector length to the destination element. - pub fn bitcast(self: Value, dtype: DType) Value { + pub fn bitCast(self: Value, dtype: DType) Value { const k = self.kern(); const src = self.scalarDType(); if (self.type_().isA(mlir.VectorType)) |vec| { @@ -500,10 +499,11 @@ pub const Value = struct { pub const Cmp = enum { eq, ne, lt, le, gt, ge }; - /// Signed integer comparison against a Value or an integer literal. + /// Signed integer comparison against a Value or an integer literal; a + /// scalar broadcasts against a vector. pub fn cmp(self: Value, pred: Cmp, rhs: anytype) Value { const k = self.kern(); - const r: Value = if (@TypeOf(rhs) == Value) rhs else k.constLike(rhs, self); + const l, const r = self.coerceArith(rhs); const p: arith.CmpIPredicate = switch (pred) { .eq => .eq, .ne => .ne, @@ -512,7 +512,7 @@ pub const Value = struct { .gt => .sgt, .ge => .sge, }; - return k.emit(arith.cmpi(k.ctx, p, self.inner, r.inner, k.loc())); + return k.emit(arith.cmpi(k.ctx, p, l.inner, r.inner, k.loc())); } /// Ordered. @@ -529,6 +529,37 @@ pub const Value = struct { return k.emit(arith.cmpf(k.ctx, p, self.inner, rhs.inner, k.loc())); } + pub fn lt(self: Value, rhs: anytype) Value { + return self.compare(.lt, rhs); + } + + pub fn le(self: Value, rhs: anytype) Value { + return self.compare(.le, rhs); + } + + pub fn gt(self: Value, rhs: anytype) Value { + return self.compare(.gt, rhs); + } + + pub fn ge(self: Value, rhs: anytype) Value { + return self.compare(.ge, rhs); + } + + pub fn eq(self: Value, rhs: anytype) Value { + return self.compare(.eq, rhs); + } + + pub fn ne(self: Value, rhs: anytype) Value { + return self.compare(.ne, rhs); + } + + /// `cmpf` for floats, `cmp` (signed) for integers, `index` included. + fn compare(self: Value, pred: Cmp, rhs: anytype) Value { + if (!self.isFloatElem()) return self.cmp(pred, rhs); + const l, const r = self.coerceArith(rhs); + return l.cmpf(pred, r); + } + pub fn select(self: Value, a: Value, b: Value) Value { const k = self.kern(); return k.emit(arith.select(k.ctx, self.inner, a.inner, b.inner, k.loc())); @@ -687,7 +718,7 @@ pub const Builder = struct { name: []const u8, func_op: ?*mlir.Operation = null, entry_block: ?*mlir.Block = null, - block_stack: std.ArrayList(*mlir.Block) = .empty, + block_stack: common.BlockStack = .{}, arg_specs: []const ArgSpec = &.{}, /// The `!fly.memref` view for tensor arguments, the raw `!fly.ptr` otherwise. arg_values: []Value = &.{}, @@ -733,7 +764,7 @@ pub const Builder = struct { /// Arguments by name. Each field is `.{ .ptr = dtype }` or /// `.{ .tensor = .{ .dtype = ..., .dims = &.{ ... } } }`. - pub fn declareArgs(self: *Builder, spec: anytype) !dsl.NamedArgs(@TypeOf(spec), Value) { + pub fn declareArgs(self: *Builder, spec: anytype) !common.NamedArgs(@TypeOf(spec), Value) { const Spec = @TypeOf(spec); const fields = @typeInfo(Spec).@"struct".field_names; var specs: [fields.len]ArgSpec = undefined; @@ -749,7 +780,7 @@ pub const Builder = struct { @compileError("fly.declareArgs: argument `" ++ f ++ "` must be `.{ .ptr = dtype }` or `.{ .tensor = .{ .dtype, .dims } }`"); } try self.declareArgsLow(&specs); - var named: dsl.NamedArgs(Spec, Value) = undefined; + var named: common.NamedArgs(Spec, Value) = undefined; inline for (fields, 0..) |f, i| @field(named, f) = self.arg(i); return named; } @@ -779,6 +810,7 @@ pub const Builder = struct { _ = func_op.appendTo(self.gpu_body); self.func_op = func_op; self.entry_block = entry; + self.block_stack.push(self.arena.allocator(), entry); // The view keeps the storage element type: recasting a predicate // argument back to `!fly.ptr` would make a copy atom's element @@ -801,18 +833,15 @@ pub const Builder = struct { } pub fn pushBlock(self: *Builder, b: *mlir.Block) void { - self.block_stack.append(self.arena.allocator(), b) catch @panic("Builder.pushBlock OOM"); + self.block_stack.push(self.arena.allocator(), b); } pub fn popBlock(self: *Builder) void { - _ = self.block_stack.pop(); + self.block_stack.pop(); } pub fn currentBlock(self: *Builder) *mlir.Block { - if (self.block_stack.items.len > 0) { - return self.block_stack.items[self.block_stack.items.len - 1]; - } - return self.entry_block orelse @panic("Builder has no current block; call declareArgs first"); + return self.block_stack.current(); } pub fn loc(self: *const Builder) *const mlir.Location { @@ -866,6 +895,10 @@ pub const Builder = struct { return result catch |err| std.debug.panic("fly: cannot build type ({})", .{err}); } + pub fn scalarTy(self: *const Builder, dtype: DType) *const mlir.Type { + return dtype.toMlir(self.ctx); + } + pub fn intTupleType(self: *Builder, t: IntTuple) *const mlir.Type { return must(fly.types.IntTupleType.get(self.ctx, .{ .attr = must(t.toAttr(self.ctx)) })).type_(); } @@ -929,8 +962,9 @@ pub const Builder = struct { return self.static(Layout{ .shape = .{ .tup = shape }, .stride = .{ .tup = stride } }); } - /// A Value passes through; a bare tuple literal (`.{ 64, 8 }`) is a tile. - pub fn lift(self: *Builder, v: anytype) Value { + /// A Value passes through; a layout literal becomes a `fly.static`, and a + /// bare tuple literal (`.{ 64, 8 }`) is a tile. + pub fn toStatic(self: *Builder, v: anytype) Value { const T = @TypeOf(v); if (T == Value) return v; if (T == Layout or T == Tile or T == IntTuple) return self.static(v); @@ -978,10 +1012,13 @@ pub const Builder = struct { // ---------------------------------------------------------------- scalars & ids - /// Only the representation `dtype` needs is computed: a float constant - /// outside i64 range (inf, floatMax) must not be forced through + /// `value` as a constant of `dtype`; a Value passes through. An integer is + /// signed or an unsigned bit pattern of `dtype`'s width; anything else + /// panics. Only the representation `dtype` needs is computed: a float + /// constant outside i64 range (inf, floatMax) must not be forced through /// `@intFromFloat`, which traps. pub fn constant(self: *Builder, dtype: DType, value: anytype) Value { + if (@TypeOf(value) == Value) return value; const is_float_lit = switch (@typeInfo(@TypeOf(value))) { .float, .comptime_float => true, else => false, @@ -997,13 +1034,42 @@ pub const Builder = struct { }); }, .i1, .i8, .i16, .i32, .i64 => blk: { - const i: i64 = if (is_float_lit) @intFromFloat(value) else @intCast(value); - break :blk self.emit(arith.constant_int(self.ctx, i, dtype.toMlir(self.ctx), self.loc())); + const i: i128 = if (is_float_lit) @intFromFloat(value) else value; + break :blk self.emit(arith.constant_int(self.ctx, intBits(i, @intCast(dtype.bitWidth()), @tagName(dtype)), dtype.toMlir(self.ctx), self.loc())); }, else => std.debug.panic("fly.constant: no constant for {s}", .{@tagName(dtype)}), }; } + /// `value` as the `i64` an integer constant of `bits` bits is built from. + /// It may be signed or an unsigned bit pattern, so it must lie in + /// [-2^(bits-1), 2^bits). + fn intBits(value: i128, bits: u7, dtype: []const u8) i64 { + if (value < -(@as(i128, 1) << (bits - 1)) or value >= @as(i128, 1) << bits) + std.debug.panic("Builder.constant: {d} does not fit {s}", .{ value, dtype }); + return @truncate(value); + } + + /// A Zig scalar as a constant: a comptime int is `i32`, or `i64` when it + /// doesn't fit; a runtime int is `i64` past 32 bits, else `i32`; a float + /// keeps its width, and a comptime float is `f32`. A Value passes through. + pub fn lift(self: *Builder, value: anytype) Value { + const T = @TypeOf(value); + if (T == Value) return value; + return switch (@typeInfo(T)) { + .comptime_int => self.constant(if (value >= std.math.minInt(i32) and value <= std.math.maxInt(i32)) .i32 else .i64, value), + .int => |info| self.constant(if (info.bits > 32) .i64 else .i32, value), + .comptime_float => self.constant(.f32, value), + .float => |info| switch (info.bits) { + 16 => self.constant(.f16, value), + 32 => self.constant(.f32, value), + 64 => self.constant(.f64, value), + else => @compileError("Builder.lift: unsupported float bitwidth"), + }, + else => @compileError("Builder.lift: unsupported type " ++ @typeName(T)), + }; + } + pub fn constIndex(self: *Builder, value: i64) Value { return self.emit(arith.constant_index(self.ctx, value, self.loc())); } @@ -1221,8 +1287,8 @@ pub const Builder = struct { } pub fn tiledCopy(self: *Builder, atom: Value, layout_tv: anytype, tile_mn: anytype) TiledCopy { - const tv = self.lift(layout_tv); - const tl = self.lift(tile_mn); + const tv = self.toStatic(layout_tv); + const tl = self.toStatic(tile_mn); const v = self.emit(fly.ops.make_tiled_copy(self.ctx, atom.inner, tv.inner, tl.inner, self.loc())); return .{ .value = v, .tile = tl }; } @@ -1302,7 +1368,7 @@ pub const Builder = struct { } pub fn tiledMma(self: *Builder, atom: Value, atom_layout: anytype, permutation: ?Value) TiledMma { - const al = self.lift(atom_layout); + const al = self.toStatic(atom_layout); const v = self.emit(fly.ops.make_tiled_mma(self.ctx, atom.inner, al.inner, .{ .permutation = if (permutation) |p| p.inner else null }, self.loc())); return .{ .value = v }; } @@ -1322,142 +1388,68 @@ pub const Builder = struct { // ---------------------------------------------------------------- control flow pub fn ForScope(comptime N: usize) type { - return dsl.ForScope(Builder, Value, N); + return common.ForScope(Builder, Value, N); } - pub const IfOnlyScope = dsl.IfOnlyScope(Builder, Value); + pub const IfOnlyScope = common.IfOnlyScope(Builder, Value); pub fn IfScope(comptime N: usize) type { - return dsl.IfScope(Builder, Value, N); + return common.IfScope(Builder, Value, N); } pub fn WhileScope(comptime N: usize, comptime M: usize) type { - return dsl.WhileScope(Builder, Value, N, M); + return common.WhileScope(Builder, Value, N, M); } - /// Loop-carried `inits`; the induction variable is `index` unless - /// `lower` is a Value. + /// Loop-carried `inits`. Literal bounds take the type of a Value bound, + /// else they are `index`. pub fn openFor( self: *Builder, lower: anytype, upper: anytype, step: anytype, inits: anytype, - ) ForScope(tupleArity(@TypeOf(inits), "openFor: inits")) { - const N = comptime tupleArity(@TypeOf(inits), "openFor: inits"); - const fields = @typeInfo(@TypeOf(inits)).@"struct".field_names; - - const lb_v: Value = if (@TypeOf(lower) == Value) lower else self.constIndex(lower); - const ub_v: Value = if (@TypeOf(upper) == Value) upper else self.constLike(upper, lb_v); - const step_v: Value = if (@TypeOf(step) == Value) step else self.constLike(step, lb_v); - - var block_types: [N + 1]*const mlir.Type = undefined; - var block_locs: [N + 1]*const mlir.Location = undefined; - block_types[0] = lb_v.type_(); - block_locs[0] = self.loc(); - var inits_inner: [N]*const mlir.Value = undefined; - inline for (fields, 0..) |f, i| { - if (@TypeOf(@field(inits, f)) != Value) @compileError("openFor: every init must be a Value"); - const v: Value = @field(inits, f); - block_types[i + 1] = v.type_(); - block_locs[i + 1] = self.loc(); - inits_inner[i] = v.inner; - } - - const body = mlir.Block.init(&block_types, &block_locs); - self.pushBlock(body); - - var carried: [N]Value = undefined; - for (0..N) |i| carried[i] = .{ .inner = body.argument(i + 1), .kernel = self }; - - return .{ - .kernel = self, - .body = body, - .lb_inner = lb_v.inner, - .ub_inner = ub_v.inner, - .step_inner = step_v.inner, - .inits_inner = inits_inner, - .iv = .{ .inner = body.argument(0), .kernel = self }, - .carried = carried, - }; + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return self.openForOpts(lower, upper, step, inits, .{}); } - pub fn constLike(self: *Builder, value: anytype, like: Value) Value { + /// `openFor` with `scf.for` options. + pub fn openForOpts( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + opts: common.ForOpts, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return common.openFor(self, Value, common.forBounds(self, Value, lower, upper, step, Builder.constIndex), inits, opts); + } + + /// `value` as a scalar constant of `like`'s element type (`index` for an + /// `index` Value); a Value passes through. + pub fn constantLike(self: *Builder, value: anytype, like: Value) Value { + if (@TypeOf(value) == Value) return value; if (like.type_().isA(mlir.IndexType) != null) { - const i: i64 = switch (@typeInfo(@TypeOf(value))) { + const i: i128 = switch (@typeInfo(@TypeOf(value))) { .float, .comptime_float => @intFromFloat(value), - else => @intCast(value), + else => value, }; - return self.constIndex(i); + return self.constIndex(intBits(i, 64, "index")); } return self.constant(like.scalarDType(), value); } pub fn openIf(self: *Builder, cond: Value) IfOnlyScope { - const then_block = mlir.Block.init(&.{}, &.{}); - self.pushBlock(then_block); - return .{ .kernel = self, .cond_inner = cond.inner, .then_block = then_block }; + return common.openIf(self, Value, cond); } - pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(tupleArity(@TypeOf(result_types), "openIfElse: result_types")) { - const N = comptime tupleArity(@TypeOf(result_types), "openIfElse: result_types"); - const fields = @typeInfo(@TypeOf(result_types)).@"struct".field_names; - var types: [N]*const mlir.Type = undefined; - inline for (fields, 0..) |f, i| { - if (@TypeOf(@field(result_types, f)) != *const mlir.Type) @compileError("openIfElse: every result_type must be *const mlir.Type"); - types[i] = @field(result_types, f); - } - const then_block = mlir.Block.init(&.{}, &.{}); - const else_block = mlir.Block.init(&.{}, &.{}); - self.pushBlock(then_block); - return .{ .kernel = self, .cond_inner = cond.inner, .then_block = then_block, .else_block = else_block, .result_types = types }; + pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(common.arity(@TypeOf(result_types), "openIfElse: result_types")) { + return common.openIfElse(self, Value, cond, result_types); } pub fn openWhile( self: *Builder, inits: anytype, after_types: anytype, - ) WhileScope( - tupleArity(@TypeOf(inits), "openWhile: inits"), - tupleArity(@TypeOf(after_types), "openWhile: after_types"), - ) { - const N = comptime tupleArity(@TypeOf(inits), "openWhile: inits"); - const M = comptime tupleArity(@TypeOf(after_types), "openWhile: after_types"); - const init_fields = @typeInfo(@TypeOf(inits)).@"struct".field_names; - const type_fields = @typeInfo(@TypeOf(after_types)).@"struct".field_names; - - var before_types: [N]*const mlir.Type = undefined; - var before_locs: [N]*const mlir.Location = undefined; - var inits_inner: [N]*const mlir.Value = undefined; - inline for (init_fields, 0..) |f, i| { - if (@TypeOf(@field(inits, f)) != Value) - @compileError("openWhile: every init must be a Value"); - const v: Value = @field(inits, f); - before_types[i] = v.type_(); - before_locs[i] = self.loc(); - inits_inner[i] = v.inner; - } - const before_block = mlir.Block.init(&before_types, &before_locs); - - var after_tys: [M]*const mlir.Type = undefined; - var after_locs: [M]*const mlir.Location = undefined; - inline for (type_fields, 0..) |f, i| { - if (@TypeOf(@field(after_types, f)) != *const mlir.Type) - @compileError("openWhile: every after_type must be *const mlir.Type"); - after_tys[i] = @field(after_types, f); - after_locs[i] = self.loc(); - } - const after_block = mlir.Block.init(&after_tys, &after_locs); - - self.pushBlock(before_block); - var before_carried: [N]Value = undefined; - for (0..N) |i| before_carried[i] = .{ .inner = before_block.argument(i), .kernel = self }; - - return .{ - .kernel = self, - .before_block = before_block, - .after_block = after_block, - .inits_inner = inits_inner, - .after_types = after_tys, - .before_carried = before_carried, - }; + ) WhileScope(common.arity(@TypeOf(inits), "openWhile: inits"), common.arity(@TypeOf(after_types), "openWhile: after_types")) { + return common.openWhile(self, Value, inits, after_types); } // ---------------------------------------------------------------- finalization @@ -1467,10 +1459,7 @@ pub const Builder = struct { pub fn finish(self: *Builder) FinishError![:0]const u8 { _ = gpu.return_(self.ctx, &.{}, self.loc()).appendTo(self.currentBlock()); - if (!self.module.operation().verify()) { - std.log.err("fly: module failed to verify:\n{f}", .{self.module.operation().fmt(.{ .debug_info = false })}); - return error.InvalidMlir; - } + try common.verify(self.module, "dsl/fly"); var al: std.Io.Writer.Allocating = .init(self.allocator); defer al.deinit(); @@ -1821,7 +1810,7 @@ test "scalar conversions and constants" { }); // A predicate is unsigned: `true` must convert to 1, never -1. - const pred = b.constant(.i32, 3).cmp(.lt, 7); + const pred = b.constant(.i32, 3).lt(7); _ = pred.to(.f32); _ = pred.to(.i32); // Equal-width float pairs are neither ext nor trunc. @@ -1832,6 +1821,11 @@ test "scalar conversions and constants" { // A `.bool` argument keeps its storage element type end to end. try std.testing.expectEqual(DType.i8, a.p.elemDType()); try std.testing.expectEqual(DType.f16, a.x.elemDType()); + // Literal inits are lifted, and an integer constant may be a bit pattern. + var loop = b.openFor(0, 4, 1, .{ 0, 0.5 }); + { + loop.yield(.{ loop.carried[0].bitXor(b.constant(.i32, 0xd4d0ccc8)), loop.carried[1].mul(2.0) }); + } const ir = try b.finish(); defer std.testing.allocator.free(ir); @@ -1841,6 +1835,8 @@ test "scalar conversions and constants" { "arith.convertf", "arith.constant 0xFF800000 : f32", "!fly.memref", + "-> (i32, f32)", + "arith.constant -724513592 : i32", }) |needle| { if (std.mem.indexOf(u8, ir, needle) == null) { std.debug.print("missing `{s}` in:\n{s}\n", .{ needle, ir }); @@ -1952,3 +1948,42 @@ test "SSA matrix atom builds, verifies and re-parses" { defer reparsed.deinit(); try std.testing.expect(reparsed.operation().verify()); } + +test "comparisons take index ivs and vectors" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "compare"); + defer b.deinit(); + _ = try b.declareArgs(.{}); + + // Literal bounds make an `index` loop. + var loop = b.openFor(0, 4, 1, .{}); + { + _ = loop.iv.lt(2); + loop.close(); + } + // A literal or a scalar broadcasts against a vector. + const v = b.constant(.f32, 1.0).splat(4); + _ = v.lt(0.5); + _ = v.ge(b.constant(.f32, 2.0)); + _ = b.constant(.i32, 3).splat(4).eq(3); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + for ([_][2][]const u8{ + .{ "arith.cmpi slt", ": index" }, + .{ "arith.cmpf olt", ": vector<4xf32>" }, + .{ "arith.cmpf oge", ": vector<4xf32>" }, + .{ "arith.cmpi eq", ": vector<4xi32>" }, + }) |needles| { + var lines = std.mem.splitScalar(u8, ir, '\n'); + const found = while (lines.next()) |line| { + if (std.mem.indexOf(u8, line, needles[0]) != null and std.mem.endsWith(u8, line, needles[1])) break true; + } else false; + if (!found) { + std.debug.print("no `{s} ... {s}` in:\n{s}\n", .{ needles[0], needles[1], ir }); + return error.TestUnexpectedResult; + } + } +} diff --git a/dsl/mosaic_tpu/BUILD.bazel b/dsl/mosaic_tpu/BUILD.bazel index 81a4474d..15510258 100644 --- a/dsl/mosaic_tpu/BUILD.bazel +++ b/dsl/mosaic_tpu/BUILD.bazel @@ -1,7 +1,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( - name = "mosaic_tpu_builder", + name = "builder", srcs = [ "dtype.zig", "pipeline.zig", @@ -11,7 +11,7 @@ zig_library( tags = ["manual"], visibility = ["//visibility:public"], deps = [ - "//dsl/common", + "//dsl/common:builder", "//mlir", "//mlir/dialects/affine", "//mlir/dialects/arith", @@ -28,5 +28,5 @@ zig_library( zig_test( name = "test", - deps = [":mosaic_tpu_builder"], + deps = [":builder"], ) diff --git a/dsl/mosaic_tpu/README.md b/dsl/mosaic_tpu/README.md index 09e5a429..f9499965 100644 --- a/dsl/mosaic_tpu/README.md +++ b/dsl/mosaic_tpu/README.md @@ -2,7 +2,8 @@ A Zig builder for Mosaic TPU IR (the `tpu` dialect in XLA) that tries to read like `pallas.tpu` Python while staying in pure Zig. Used by -`examples/mosaic_gdn` and by anything else that wants to emit a TPU kernel +`zml/kernels/moe/mosaic_tpu/`, `zml/kernels/paged_attention/mosaic_tpu/` +and `platforms/tpu/`, and by anything else that wants to emit a TPU kernel op-for-op against what `pallas_call` produces. The DSL talks to the `tpu` dialect through `mlir.Operation.make(ctx, "tpu.", …)`. @@ -20,9 +21,7 @@ Two layers define the surface: for the final IR string. Pass `Builder.open` / `Builder.init` a `zml.kernel.mosaic_tpu.newContext()` (a throwaway context with the dialects the DSL emits), `defer ctx.deinit()` once you have the string — not a - `Compilation`'s `mlir_ctx`. (`Builder.buildOpts(...)` is a convenience - that bundles `open + declareArgs` into one call and returns a heap-allocated - `*Built(Spec)`; reach for it in escape-hatch code that drives the lifecycle manually.) + `Compilation`'s `mlir_ctx`. - **High layer** — `zml.kernel.mosaic_tpu.Kernel(Config, spec)`: the declarative kernel form, mirroring the Triton DSL. Bundles a config type, a typed spec literal (name + named `inputs` / `outputs` + a typed @@ -170,9 +169,9 @@ updates). ## Argument specs -Each field passed to `b.declareArgsOpts(...)` (or to the lower-layer -`Builder.buildOpts(...)`) is an `ArgSpec.Kind` tagged-union literal. The -four kinds mirror what a Mosaic `func.func` can carry: +Each field passed to `b.declareArgs(...)` / `b.declareArgsOpts(...)` is an +`ArgSpec.Kind` tagged-union literal. The four kinds mirror what a Mosaic +`func.func` can carry: | Kind literal | MLIR type | Use case | |--------------------------------------------------------|--------------------------------------|-----------------------------------------| @@ -355,7 +354,7 @@ default; ordered float predicates for floats. | Method | Effect | |-----------------------|-----------------------------------------------------| | `.to(dtype)` | Numeric cast preserving shape; auto-dispatches int↔float. | -| `.bitcast(dtype)` | `tpu.bitcast` — same-bitwidth element-type swap. | +| `.bitCast(dtype)` | `tpu.bitcast` — same-bitwidth element-type swap. | | `.abs()` | `math.absf` (float) / `math.absi` (int). | `.to(...)` chooses the right MLIR op based on the source vs target kind: @@ -390,18 +389,19 @@ lift rule: If you need to force a specific width for a runtime variable, cast at the call site: `pid.mul(@as(i32, @intCast(block_size)))`. -### When do you need `b.lift` / `b.liftAs` / `b.cIndex`? +### When do you need `b.lift` / `b.constant` / `b.constIndex`? | Helper | Effect | |--------------------------------|----------------------------------------------------------| | `b.lift(value)` | Wrap a Zig scalar; dtype inferred from source. | -| `b.liftAs(value, dtype)` | Wrap a Zig scalar as a specific `DType`. | -| `b.cIndex(value)` | `arith.constant N : index` — for memref offsets. | +| `b.constant(dtype, value)` | Wrap a Zig scalar as a specific `DType`. | +| `b.constantLike(value, like)` | Wrap a Zig scalar as `like`'s element type. | +| `b.constIndex(value)` | `arith.constant N : index` — for memref offsets. | Reach for these when the DSL surface demands an existing `Value` and there's no binop sibling to match against — typically loop seeds for `openWhile(.{inits}, ...)`, `openFor(0, N, 1, .{init})`, or `var x: Value = -b.liftAs(0, .i32);` that a later branch may reassign. +b.constant(.i32, 0);` that a later branch may reassign. If the constant flows through any `.add` / `.mul` / `splat` etc., skip the lift and let auto-lift do it. @@ -495,10 +495,12 @@ Use these for SMEM args (e.g. `cu_seqlens`) where Pallas emits scalar ### Auto-broadcast -There is **no implicit auto-broadcast** in the Mosaic DSL. Mosaic kernels run -inside the TPU's tile semantics and the layout pass is strict about -`vector<…>` ranks/shapes — surprise-broadcasts would be footguns. Build the -broadcasts you need explicitly with `shapeCast` + `broadcastTo`: +The fluent binary ops and comparisons (`add` … `maximum`, `lt` … `ne`), +`b.where`, `b.maximum` and `b.minimum` splat a scalar operand against a +vector one (`vector.broadcast`) and widen the narrower integer side. Nothing +else broadcasts: Mosaic kernels run inside the TPU's tile semantics and the +layout pass is strict about `vector<…>` ranks/shapes, so build rank changes +and size-1 expansions explicitly with `shapeCast` + `broadcastTo`: ```zig // b_k[:, None] broadcast to @@ -506,8 +508,9 @@ const bk_2d = k.shapeCast(b_k, &.{ BK, 1 }); const bk_bc = k.broadcastTo(bk_2d, &.{ BK, BV }); ``` -For scalar→vector splatting, use `b.broadcastTo(scalar, shape)` (it routes -through `vector.broadcast`) or `b.full / b.splat` for comptime constants. +For a vector made from a scalar outside a binary op, use +`b.broadcastTo(scalar, shape)` (it routes through `vector.broadcast`) or +`b.full / b.splat` for comptime constants. --- @@ -636,9 +639,9 @@ For multi-core / async DMA orchestration: SCF regions are built with **scope-based** builders — `b.openFor(...)`, `b.openIf(...)` / `b.openIfElse(...)`, `b.openWhile(...)` return a typed scope value. You emit body ops into a bare `{}` block, terminate with -`yield` / `yieldThen` / `yieldAfter`, and read results from `scope.results` -afterward. No `ctx` struct, no function literal — lexical scope carries -captures. +`yield` / `yieldThen` / `yieldAfter` (or `close()` for a scope without +values), and read results from `scope.results` afterward. No `ctx` struct, +no function literal — lexical scope carries captures. ### `openFor` — scf.for with iter_args @@ -656,13 +659,15 @@ const final_b_h = loop.results[0]; ``` - `lower` / `upper` / `step` are `anytype`: pass Values, comptime ints, or - runtime Zig ints. Use `i32` constants (`b.lift(@as(i32, 0))`) when you - want the loop counter to be `i32` (matching Pallas's `scf.for ... : i32`). - The default for `index`-typed counters is `b.cIndex(N)`. -- `inits` is a tuple literal `.{v1, v2, ...}`. Arity is comptime — `carried`, - `results`, and the `yield(...)` call are all fixed-size arrays of the - same length. -- Empty loops: `b.openFor(0, N, 1, .{})` + `loop.yield(.{})`. + runtime Zig ints. A bound that isn't a Value takes the type of the first one + that is; with none, all three are `index`. Pass `b.constant(.i32, 0)` as + `lower` for an `i32` counter (matching Pallas's `scf.for ... : i32`). +- `inits` is a tuple `.{v1, v2, ...}` or an array. Arity is comptime — + `carried`, `results`, and the `yield(...)` values (also a tuple or an + array) all have the same length. +- Empty loops: `b.openFor(0, N, 1, .{})` + `loop.close()`. +- `b.openForOpts(lower, upper, step, inits, .{ .unsigned_cmp = true })` sets + the loop's options. ### `openIf` — scf.if without else (side-effects only) @@ -670,7 +675,7 @@ const final_b_h = loop.results[0]; var i = k.openIf(cond); { // then body — store, increment, etc. - i.yieldThen(.{}); + i.close(); } ``` @@ -680,7 +685,7 @@ empty. ### `openIfElse` — scf.if with else and optional results ```zig -var i = k.openIfElse(cond, .{ k.scalarTy(.f32) }); +var i = k.openIfElse(cond, .{.f32}); { i.yieldThen(.{ x }); } @@ -690,14 +695,15 @@ var i = k.openIfElse(cond, .{ k.scalarTy(.f32) }); const r = i.results[0]; ``` -- `result_types` is a tuple of `*const mlir.Type` — use `k.scalarTy(dtype)` - / `k.vectorTy(shape, dtype)` / `k.memRefTy(shape, dtype, mem_space)`. - Pass `.{}` for an if/else with no results. +- `result_types` is a tuple or array of dtypes (`.f32`), Values (for their + types) or `*const mlir.Type`s such as `k.vectorTy(shape, dtype)` / + `k.memRefTy(shape, dtype, mem_space)`. Pass `.{}` for an if/else with no + results, then end its branches with `elseBranch()` and `close()`. ### `openWhile` — scf.while ```zig -var w = k.openWhile(.{ i0 }, .{ k.scalarTy(.i32) }); +var w = k.openWhile(.{ i0 }, .{.i32}); { const s = w.before_carried[0]; w.yieldBefore(s.lt(10), .{ s }); @@ -709,9 +715,10 @@ var w = k.openWhile(.{ i0 }, .{ k.scalarTy(.i32) }); const r = w.results[0]; ``` -- `inits` — tuple of `Value`s; before-region arg types = their types. -- `after_types` — tuple of `*const mlir.Type`; after-region arg types + the - scf.while's result types. +- `inits` — a tuple or array of `Value`s; before-region arg types = their types. +- `after_types` — a tuple or array of dtypes (`.f32`), Values (for their + types) or `*const mlir.Type`s; after-region arg types + the scf.while's + result types. - `yieldBefore(cond, forwarded)` — `forwarded` arity must match `after_types`. - `yieldAfter(values)` — arity must match `inits`. @@ -719,7 +726,8 @@ const r = w.results[0]; - Scopes stack naturally: `openFor { openIf { openFor { ... } } }` all just push/pop the kernel's current-block stack. -- Empty yields are legal: `loop.yield(.{})`, `i.yieldThen(.{})`, etc. +- Scopes without values end with `close()`: `loop.close()`, `i.close()`, and + `elseBranch()` then `close()` for an if/else. - After any `yield*`, the scope's `.results` holds the scf op's results as a `[N]Value`. @@ -741,7 +749,7 @@ book-keeping layer, not a walled garden: `kernel` field populated. - `b.scalarTy(dtype)` / `b.vectorTy(shape, dtype)` / `b.memRefTy(shape, dtype, mem_space)` — type constructors used by raw - ops and by `openIfElse.result_types` / `openWhile.after_types`. + ops and by non-scalar `openIfElse.result_types` / `openWhile.after_types`. When you drop to raw ops, remember to still **append them to the current block** — `b.emit(...)` does that for you; `_ = op.appendTo(b.currentBlock())` @@ -761,7 +769,7 @@ is the manual form. | `o_k < KEY_DIM` | `k.cmpi(.slt, o_k, dim_splat)` / `o_k.lt(dim_splat)` | | `mask_k[:, None] & mask_v[None, :]` | `k.andi(k.broadcastTo(k.shapeCast(mask_k, &.{BK,1}), &.{BK,BV}), k.broadcastTo(mask_v, &.{BK,BV}))` | | `jnp.where(mask, x, 0.0)` | `k.select(mask, x, k.zeros(shape, dtype))` | -| `q_ref[0, t, h, :].astype(jnp.float32)` | `k.vectorLoadShape(a.q, &.{c0, t_idx, h_idx, c0}, &.{1,1,1,BK})` then `.shapeCast(&.{BK})` then `.to(.f32)` | +| `q_ref[0, t, h, :].astype(jnp.float32)` | `k.vectorLoadShape(a.q, &.{c0, t_idx, h_idx, c0}, &.{1,1,1,BK})` then `k.shapeCast(v, &.{BK})` then `.to(.f32)` | | `q_ref[0, t, h, :] = b_o.astype(o_ref.dtype)` | `k.vectorStoreAt(a.q, k.shapeCast(b_o, &.{1,1,1,BK}), &.{c0, t_idx, h_idx, c0})` | | `jnp.sqrt(jnp.sum(b_q * b_q) + 1e-6)` | `k.sqrt(k.addf(k.reduceToScalar(.add, k.mulf(b_q, b_q), cst_zero_1), cst_eps))` | | `jnp.exp(b_g)` | `k.exp(b_g)` | @@ -769,7 +777,7 @@ is the manual form. | `b_h * jnp.exp(b_g)` (broadcast to ) | `k.mulf(b_h, k.broadcastTo(exp_g, &.{BK, BV}))` | | `b_beta * (b_v - predicted_v)` | `k.mulf(k.broadcastTo(b_beta, &.{BV}), k.subf(b_v, predicted_v))` | | `lax.fori_loop(0, T, body, init)` | `var loop = k.openFor(c0_i32, T, c1_i32, .{init}); { …; loop.yield(.{…}); }` | -| `if cond: …` (side-effects only) | `var i = k.openIf(cond); { …; i.yieldThen(.{}); }` | +| `if cond: …` (side-effects only) | `var i = k.openIf(cond); { …; i.close(); }` | | `if cond: x = a else: x = b` | `var i = k.openIfElse(cond, .{ty}); { …; i.yieldThen(.{a}); }; { …; i.yieldElse(.{b}); }` | | `while cond:` | `var w = k.openWhile(.{inits}, .{after_types}); { …; w.yieldBefore(cond, .{…}); }; { …; w.yieldAfter(.{…}); }` | | `pltpu.MemorySpace.SMEM` | `.memory_space = .smem` on the `RefSpec` | diff --git a/dsl/mosaic_tpu/builder.zig b/dsl/mosaic_tpu/builder.zig index 5f54315c..c58f5e96 100644 --- a/dsl/mosaic_tpu/builder.zig +++ b/dsl/mosaic_tpu/builder.zig @@ -18,8 +18,7 @@ const vector = @import("mlir/dialects/vector"); /// `vector`-dialect enums re-exported for convenience. /// Used by `vectorReduction` / `multiReduction`. pub const CombiningKind = vector.CombiningKind; -const dsl = @import("dsl/common"); -const tupleArity = dsl.tupleArity; +const common = @import("dsl/common/builder"); const mlir = @import("mlir"); const stdx = @import("stdx"); const mosaic_tpu = @import("mlir/dialects/mosaic_tpu"); @@ -210,8 +209,8 @@ pub const Value = struct { } /// `tpu.bitcast` — same-bitwidth element-type swap. - pub fn bitcast(self: Value, dtype: DType) Value { - return self.kern().bitcastTo(self, dtype); + pub fn bitCast(self: Value, dtype: DType) Value { + return self.kern().bitCast(self, dtype); } /// Absolute value (float→absf, int→absi). @@ -464,7 +463,7 @@ pub const Builder = struct { /// Entry block for the `func.func`. `null` between `open` and the first /// `declareArgs` call; populated thereafter. entry_block: ?*mlir.Block, - block_stack: std.ArrayList(*mlir.Block), + block_stack: common.BlockStack = .{}, /// Cached `arith.constant 0 : index` Value (and the block it lives /// in) so repeat no-index loads / stores share an SSA value. See /// `zeroIndex` for the lookup logic. @@ -588,13 +587,13 @@ pub const Builder = struct { .name = name, .func_op = null, .entry_block = null, - .block_stack = .empty, + .block_stack = .{}, }; } /// Declare the kernel's args from a named-field struct literal and /// return a `NamedArgs(Spec)`. - pub fn declareArgs(self: *Builder, spec: anytype) !dsl.NamedArgs(@TypeOf(spec), Value) { + pub fn declareArgs(self: *Builder, spec: anytype) !common.NamedArgs(@TypeOf(spec), Value) { return self.declareArgsOpts(spec, &.{}, .{}); } @@ -603,7 +602,7 @@ pub const Builder = struct { spec: anytype, result_types: []const *const mlir.Type, opts: Opts, - ) !dsl.NamedArgs(@TypeOf(spec), Value) { + ) !common.NamedArgs(@TypeOf(spec), Value) { const Spec = @TypeOf(spec); const field_names = @typeInfo(Spec).@"struct".field_names; @@ -639,7 +638,7 @@ pub const Builder = struct { try self.declareArgsLowOpts(&arg_specs, result_types, opts); - var named: dsl.NamedArgs(Spec, Value) = undefined; + var named: common.NamedArgs(Spec, Value) = undefined; inline for (field_names, 0..) |field_name, i| { @field(named, field_name) = self.arg(i); } @@ -845,6 +844,7 @@ pub const Builder = struct { self.func_op = func_op; self.entry_block = entry; + self.block_stack.push(self.arena.allocator(), entry); } pub fn deinit(self: *Builder) void { @@ -859,18 +859,15 @@ pub const Builder = struct { } pub fn pushBlock(self: *Builder, b: *mlir.Block) void { - self.block_stack.append(self.arena.allocator(), b) catch @panic("Builder.pushBlock OOM"); + self.block_stack.push(self.arena.allocator(), b); } pub fn popBlock(self: *Builder) void { - _ = self.block_stack.pop(); + self.block_stack.pop(); } pub fn currentBlock(self: *Builder) *mlir.Block { - if (self.block_stack.items.len > 0) { - return self.block_stack.items[self.block_stack.items.len - 1]; - } - return self.entry_block orelse @panic("Builder has no current block — call declareArgs first"); + return self.block_stack.current(); } /// Unpack a slice of `Value`s into an arena-allocated slice of raw MLIR @@ -960,10 +957,15 @@ pub const Builder = struct { } /// Constant `index`-typed value (for memref / loop bounds). - pub fn cIndex(self: *Builder, value: i64) Value { + pub fn constIndex(self: *Builder, value: i64) Value { return self.emit(arith.constant_index(self.ctx, value, self.loc())); } + /// `openFor`'s default lower bound, an `index` constant. + fn indexBound(self: *Builder, value: anytype) Value { + return self.constIndex(intBits(value, 64, "index")); + } + /// Lift a Zig scalar (or pass-through Value) to a DSL constant. pub fn lift(self: *Builder, value: anytype) Value { const T = @TypeOf(value); @@ -998,39 +1000,48 @@ pub const Builder = struct { }; } - /// Lift `value` and pin its dtype to `dtype`. Unsigned ints bit-cast - /// preserving the source bit-pattern. - pub fn liftAs(self: *Builder, value: anytype, dtype: DType) Value { + /// `value` as a constant of `dtype`; a Value passes through. An integer is + /// signed or an unsigned bit pattern of `dtype`'s width; anything else + /// panics. + pub fn constant(self: *Builder, dtype: DType, value: anytype) Value { const T = @TypeOf(value); if (T == Value) return value; if (isFloatDtype(dtype)) { const v_f64: f64 = switch (@typeInfo(T)) { .comptime_int, .int => @floatFromInt(value), .comptime_float, .float => @floatCast(value), - else => @compileError("Builder.liftAs: unsupported scalar " ++ @typeName(T)), + else => @compileError("Builder.constant: unsupported scalar " ++ @typeName(T)), }; return self.emitFloat(dtype, v_f64); } - const v_i64: i64 = switch (@typeInfo(T)) { - .comptime_int => blk: { - if (value >= std.math.minInt(i64) and value <= std.math.maxInt(i64)) - break :blk @intCast(value); - if (value >= 0 and value <= std.math.maxInt(u64)) - break :blk @bitCast(@as(u64, value)); - @compileError("Builder.liftAs: integer literal out of 64-bit range"); - }, - .int => |info| if (info.signedness == .signed) @intCast(value) else @bitCast(@as(u64, @intCast(value))), + const v: i128 = switch (@typeInfo(T)) { + .comptime_int, .int => value, .comptime_float, .float => @intFromFloat(value), - else => @compileError("Builder.liftAs: unsupported scalar " ++ @typeName(T)), + else => @compileError("Builder.constant: unsupported scalar " ++ @typeName(T)), }; - return self.emitInt(dtype, v_i64); + return self.emitInt(dtype, intBits(v, @intCast(intBitwidth(dtype)), @tagName(dtype))); + } + + /// `value` as the `i64` an integer constant of `bits` bits is built from. + /// It may be signed or an unsigned bit pattern, so it must lie in + /// [-2^(bits-1), 2^bits). + fn intBits(value: i128, bits: u7, dtype: []const u8) i64 { + if (value < -(@as(i128, 1) << (bits - 1)) or value >= @as(i128, 1) << bits) + std.debug.panic("Builder.constant: {d} does not fit {s}", .{ value, dtype }); + return @truncate(value); + } + + /// `value` as a constant of `like`'s element type; a Value passes through. + pub fn constantLike(self: *Builder, value: anytype, like: Value) Value { + if (@TypeOf(value) == Value) return value; + return self.constMatching(value, like.elemType()); } /// Build an arith.constant of the given MLIR element type from a Zig /// numeric. Used for matching the type of an existing reference value. - pub fn constMatching(self: *Builder, value: anytype, elem: *const mlir.Type) Value { - if (elem.eql(.index(self.ctx))) return self.cIndex(@intCast(value)); - return self.liftAs(value, self.mlirElemToDType(elem)); + fn constMatching(self: *Builder, value: anytype, elem: *const mlir.Type) Value { + if (elem.eql(.index(self.ctx))) return self.constIndex(intBits(value, 64, "index")); + return self.constant(self.mlirElemToDType(elem), value); } fn liftMatching(self: *Builder, value: anytype, ref_elem: *const mlir.Type) Value { @@ -1305,7 +1316,7 @@ pub const Builder = struct { // ==================== TPU casts ==================== /// `tpu.bitcast` — same-bitwidth element-type swap. Shape preserved. - pub fn bitcastTo(self: *Builder, src: Value, dt: DType) Value { + pub fn bitCast(self: *Builder, src: Value, dt: DType) Value { return self.emit(mosaic_tpu.bitcast(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); } pub fn extf(self: *Builder, src: Value, dt: DType) Value { @@ -1336,7 +1347,7 @@ pub const Builder = struct { return self.castOpts(src, dt, .{}); } pub fn castOpts(self: *Builder, src: Value, dt: DType, opts: CastOpts) Value { - if (opts.bitcast) return self.bitcastTo(src, dt); + if (opts.bitcast) return self.bitCast(src, dt); const cur_elem = src.elemType(); const tgt_elem = dt.toMlir(self.ctx); if (cur_elem.eql(tgt_elem)) return src; @@ -1373,7 +1384,7 @@ pub const Builder = struct { /// + `arith.mulf x, ln2` + `math.exp`. For the raw `math.exp2` op /// (post-2025-07-26 cloud TPUs), use `mathExp2`. pub fn exp2(self: *Builder, x: Value) Value { - const ln2 = self.liftAs(@as(f64, 0.6931471805599453), self.mlirElemToDType(x.elemType())); + const ln2 = self.constant(self.mlirElemToDType(x.elemType()), @as(f64, 0.6931471805599453)); const ln2_bc = if (x.isShaped()) self.broadcastTo(ln2, x.shape().constSlice()) else ln2; return self.exp(self.mulf(x, ln2_bc)); } @@ -1522,7 +1533,7 @@ pub const Builder = struct { if (self.zero_index) |z| { if (self.zero_index_block == block) return z; } - const z = self.cIndex(0); + const z = self.constIndex(0); self.zero_index = z; self.zero_index_block = block; return z; @@ -1746,7 +1757,7 @@ pub const Builder = struct { } pub fn rotate(self: *Builder, value: Value, opts: RotateOpts) Value { - const amount = self.lift(@as(i32, opts.amount)); + const amount = self.constant(.i32, opts.amount); return self.emit(mosaic_tpu.dynamic_rotate(self.ctx, value.inner, amount.inner, .{ .dimension = opts.dimension, .stride = opts.stride, @@ -1889,7 +1900,7 @@ pub const Builder = struct { /// Convenience: 1-sided range mask `(0 <= idx < limit)` — emits two /// `index` constants and a `tpu.create_mask`. pub fn rangeMask(self: *Builder, limits: []const Value, shape: []const i64) Value { - const zero = self.cIndex(0); + const zero = self.constIndex(0); const lows = self.arena.allocator().alloc(Value, limits.len) catch @panic("rangeMask OOM"); for (lows) |*l| l.* = zero; return self.createMask(lows, limits, shape); @@ -2025,142 +2036,44 @@ pub const Builder = struct { // ==================== SCF regions ==================== - /// Open a scoped `scf.for`. `lower`/`upper`/`step` accept Values, comptime - /// ints, or runtime Zig ints — coerced to `index`. `inits` is `.{v1, v2, ...}`. + /// Open a scoped `scf.for`. Literal bounds take the type of a Value bound, + /// else they are `index`. `inits` is `.{v1, v2, ...}` or an array. pub fn openFor( self: *Builder, lower: anytype, upper: anytype, step: anytype, inits: anytype, - ) dsl.ForScope(Builder, Value, dsl.tupleArity(@TypeOf(inits), "openFor: inits")) { - const N = comptime dsl.tupleArity(@TypeOf(inits), "openFor: inits"); - const info = @typeInfo(@TypeOf(inits)).@"struct"; - - // scf.for bounds default to `index`. - const lb_v: Value = if (@TypeOf(lower) == Value) lower else self.cIndex(@intCast(lower)); - const ub_v: Value = if (@TypeOf(upper) == Value) upper else self.cIndex(@intCast(upper)); - const step_v: Value = if (@TypeOf(step) == Value) step else self.cIndex(@intCast(step)); - - var block_types: [N + 1]*const mlir.Type = undefined; - var block_locs: [N + 1]*const mlir.Location = undefined; - block_types[0] = lb_v.type_(); - block_locs[0] = self.loc(); - var inits_inner: [N]*const mlir.Value = undefined; - inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { - const raw = @field(inits, field_name); - const v: Value = if (Field == Value) raw else self.lift(raw); - block_types[i + 1] = v.type_(); - block_locs[i + 1] = self.loc(); - inits_inner[i] = v.inner; - } - - const body = mlir.Block.init(&block_types, &block_locs); - self.pushBlock(body); - - var carried: [N]Value = undefined; - for (0..N) |i| carried[i] = .{ .inner = body.argument(i + 1), .kernel = self }; - - return .{ - .kernel = self, - .body = body, - .lb_inner = lb_v.inner, - .ub_inner = ub_v.inner, - .step_inner = step_v.inner, - .inits_inner = inits_inner, - .iv = .{ .inner = body.argument(0), .kernel = self }, - .carried = carried, - }; + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return self.openForOpts(lower, upper, step, inits, .{}); } - pub fn openIf(self: *Builder, cond: Value) dsl.IfOnlyScope(Builder, Value) { - const then_block = mlir.Block.init(&.{}, &.{}); - self.pushBlock(then_block); - return .{ - .kernel = self, - .cond_inner = cond.inner, - .then_block = then_block, - }; - } - - pub fn openIfElse( + /// `openFor` with `scf.for` options. + pub fn openForOpts( self: *Builder, - cond: Value, - result_types: anytype, - ) dsl.IfScope(Builder, Value, dsl.tupleArity(@TypeOf(result_types), "openIfElse: result_types")) { - const N = comptime dsl.tupleArity(@TypeOf(result_types), "openIfElse: result_types"); - const info = @typeInfo(@TypeOf(result_types)).@"struct"; + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + opts: common.ForOpts, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return common.openFor(self, Value, common.forBounds(self, Value, lower, upper, step, indexBound), inits, opts); + } - var types: [N]*const mlir.Type = undefined; - inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { - if (Field != *const mlir.Type) - @compileError("openIfElse: every result_type must be *const mlir.Type (use k.scalarTy/vectorTy)"); - types[i] = @field(result_types, field_name); - } + pub fn openIf(self: *Builder, cond: Value) IfOnlyScope { + return common.openIf(self, Value, cond); + } - const then_block = mlir.Block.init(&.{}, &.{}); - const else_block = mlir.Block.init(&.{}, &.{}); - self.pushBlock(then_block); - - return .{ - .kernel = self, - .cond_inner = cond.inner, - .then_block = then_block, - .else_block = else_block, - .result_types = types, - }; + pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(common.arity(@TypeOf(result_types), "openIfElse: result_types")) { + return common.openIfElse(self, Value, cond, result_types); } pub fn openWhile( self: *Builder, inits: anytype, after_types: anytype, - ) dsl.WhileScope( - Builder, - Value, - dsl.tupleArity(@TypeOf(inits), "openWhile: inits"), - dsl.tupleArity(@TypeOf(after_types), "openWhile: after_types"), - ) { - const N = comptime dsl.tupleArity(@TypeOf(inits), "openWhile: inits"); - const M = comptime dsl.tupleArity(@TypeOf(after_types), "openWhile: after_types"); - const init_info = @typeInfo(@TypeOf(inits)).@"struct"; - const type_info = @typeInfo(@TypeOf(after_types)).@"struct"; - - var before_types: [N]*const mlir.Type = undefined; - var before_locs: [N]*const mlir.Location = undefined; - var inits_inner: [N]*const mlir.Value = undefined; - inline for (init_info.field_names, init_info.field_types, 0..) |field_name, Field, i| { - if (Field != Value) - @compileError("openWhile: every init must be a Value"); - const v: Value = @field(inits, field_name); - before_types[i] = v.type_(); - before_locs[i] = self.loc(); - inits_inner[i] = v.inner; - } - const before_block = mlir.Block.init(&before_types, &before_locs); - - var after_tys: [M]*const mlir.Type = undefined; - var after_locs: [M]*const mlir.Location = undefined; - inline for (type_info.field_names, type_info.field_types, 0..) |field_name, Field, i| { - if (Field != *const mlir.Type) - @compileError("openWhile: every after_type must be *const mlir.Type"); - after_tys[i] = @field(after_types, field_name); - after_locs[i] = self.loc(); - } - const after_block = mlir.Block.init(&after_tys, &after_locs); - - self.pushBlock(before_block); - var before_carried: [N]Value = undefined; - for (0..N) |i| before_carried[i] = .{ .inner = before_block.argument(i), .kernel = self }; - - return .{ - .kernel = self, - .before_block = before_block, - .after_block = after_block, - .inits_inner = inits_inner, - .after_types = after_tys, - .before_carried = before_carried, - }; + ) WhileScope(common.arity(@TypeOf(inits), "openWhile: inits"), common.arity(@TypeOf(after_types), "openWhile: after_types")) { + return common.openWhile(self, Value, inits, after_types); } // ==================== finalization ==================== @@ -2187,9 +2100,7 @@ pub const Builder = struct { if (self.emit_transform_stubs) self.appendTransformStubs(); - if (!self.module.operation().verify()) { - return error.InvalidMlir; - } + try common.verify(self.module, "dsl/mosaic_tpu"); if (opts.canonicalize) { const pm = mlir.PassManager.init(self.ctx); @@ -2300,17 +2211,17 @@ fn isSkippedWindowSpace(ms: MemorySpace) bool { // ==================== scope-type aliases pinned to Mosaic's Builder ==================== pub fn ForScope(comptime N: usize) type { - return dsl.ForScope(Builder, Value, N); + return common.ForScope(Builder, Value, N); } -pub const IfOnlyScope = dsl.IfOnlyScope(Builder, Value); +pub const IfOnlyScope = common.IfOnlyScope(Builder, Value); pub fn IfScope(comptime N: usize) type { - return dsl.IfScope(Builder, Value, N); + return common.IfScope(Builder, Value, N); } pub fn WhileScope(comptime N: usize, comptime M: usize) type { - return dsl.WhileScope(Builder, Value, N, M); + return common.WhileScope(Builder, Value, N, M); } test { diff --git a/dsl/mosaic_tpu/pipeline.zig b/dsl/mosaic_tpu/pipeline.zig index 5726c764..857d76bd 100644 --- a/dsl/mosaic_tpu/pipeline.zig +++ b/dsl/mosaic_tpu/pipeline.zig @@ -90,7 +90,7 @@ pub const PipeSpec = struct { pub const PipelineOpts = struct { body: BodyFn, body_ctx: ?*anyopaque = null, - /// Dynamic (`i32`) grid bounds. Comptime: pass `b.lift(@as(i32, N))`. + /// Dynamic (`i32`) grid bounds. Comptime: pass `b.constant(.i32, N)`. grid: []const Value, /// One per HBM input ref (slot `null` <-> `None` spec). in_specs: []const ?PipeSpec, @@ -134,8 +134,8 @@ pub fn emitPipeline( for (brefs.items) |*br| br.allocate(b); const num_steps = gridSize(b, opts.grid); - const c0_i32 = b.lift(@as(i32, 0)); - const c1_i32 = b.lift(@as(i32, 1)); + const c0_i32 = b.constant(.i32, 0); + const c1_i32 = b.constant(.i32, 1); { var when = b.openIf(b.cmpi(.sgt, num_steps, c0_i32)); { @@ -206,7 +206,7 @@ pub fn emitPipeline( .input => br.wait_in_slot.?, .output, .input_output => br.copy_out_slot.?, }; - current_refs[br.spec_index] = .{ .buf = br.window_ref, .slot = b.remui(active, b.lift(@as(i32, @intCast(br.buffer_count)))) }; + current_refs[br.spec_index] = .{ .buf = br.window_ref, .slot = b.remui(active, b.constant(.i32, br.buffer_count)) }; } } @@ -251,7 +251,7 @@ pub fn emitPipeline( for (brefs.items) |*br| sched_fin.finalize(b, br); for (brefs.items) |*br| if (br.is_trivial and (br.buffer_type == .output or br.buffer_type == .input_output)) br.syncCopyOut(b); - when.yieldThen(.{}); + when.close(); } } @@ -352,13 +352,13 @@ pub const BufferedRef = struct { } fn appendSlotInits(self: *BufferedRef, b: *Builder, out: *std.ArrayList(*const mlir.Value)) void { - const c0 = b.lift(@as(i32, 0)).inner; + const c0 = b.constant(.i32, 0).inner; switch (self.buffer_type) { .input => { if (self.is_trivial) { out.appendAssumeCapacity(c0); } else { - out.appendAssumeCapacity((self.copy_in_slot orelse b.lift(@as(i32, 0))).inner); + out.appendAssumeCapacity((self.copy_in_slot orelse b.constant(.i32, 0)).inner); out.appendAssumeCapacity(c0); } }, @@ -368,7 +368,7 @@ pub const BufferedRef = struct { }, .input_output => { out.appendAssumeCapacity( - (self.copy_in_slot orelse b.lift(@as(i32, 0))).inner, + (self.copy_in_slot orelse b.constant(.i32, 0)).inner, ); out.appendAssumeCapacity(c0); out.appendAssumeCapacity(c0); @@ -510,7 +510,7 @@ pub const BufferedRef = struct { const tiling: i64 = if (d == bs.len - 1) 128 else if (bs.len >= 2 and d == bs.len - 2) 8 else 1; const clamped = @min(n, dim_size); const sz_const = @divFloor(clamped + tiling - 1, tiling) * tiling; - size[d] = b.emit(mosaic_tpu.assume_multiple(b.ctx, b.lift(@as(i32, @intCast(sz_const))).inner, @as(i32, @intCast(tiling)), b.loc())); + size[d] = b.emit(mosaic_tpu.assume_multiple(b.ctx, b.constant(.i32, sz_const).inner, @as(i32, @intCast(tiling)), b.loc())); } }, .bounded_slice => { @@ -527,18 +527,18 @@ pub const BufferedRef = struct { } // Prologue shortcut: Python-int 0 start ⇒ `_round_up_to_nearest_multiple` folds. if (static_zero_start and sl.start.asConstantInt() == @as(?i64, 0)) { - const is_oob = b.cmpi(.sgt, sl.size, b.lift(@as(i32, @intCast(dim_size)))); - const sz = b.select(is_oob, b.lift(@as(i32, @intCast(dim_size))), sl.size); + const is_oob = b.cmpi(.sgt, sl.size, b.constant(.i32, dim_size)); + const sz = b.select(is_oob, b.constant(.i32, dim_size), sl.size); size[d] = b.emit(mosaic_tpu.assume_multiple(b.ctx, sz.inner, tiling, b.loc())); continue; } - const is_oob = b.cmpi(.sgt, b.addi(sl.start, sl.size), b.lift(@as(i32, @intCast(dim_size)))); + const is_oob = b.cmpi(.sgt, b.addi(sl.start, sl.size), b.constant(.i32, dim_size)); const rounded = blk: { - if (tiling == 1) break :blk b.subi(b.lift(@as(i32, @intCast(dim_size + 1))), sl.start); + if (tiling == 1) break :blk b.subi(b.constant(.i32, dim_size + 1), sl.start); // lax sign-fixup for `jnp.remainder`. const m = b.lift(tiling); - const c0 = b.lift(@as(i32, 0)); - const rem_arg = b.subi(b.lift(@as(i32, @intCast(dim_size))), sl.start); + const c0 = b.constant(.i32, 0); + const rem_arg = b.subi(b.constant(.i32, dim_size), sl.start); const r = b.remsi(rem_arg, m); const r_nz = b.cmpi(.ne, r, c0); const r_neg = b.cmpi(.slt, r, c0); @@ -588,7 +588,7 @@ pub const BufferedRef = struct { fn sliceWindowAt(self: *const BufferedRef, b: *Builder, slot: Value, dma: DmaSlice) Value { const a = b.arena.allocator(); const bs = self.spec.block_shape.?; - const c0 = b.lift(@as(i32, 0)); + const c0 = b.constant(.i32, 0); const nd = 1 + self.block_compact.len; var base = a.alloc(Value, nd) catch unreachable; var result_shape = a.alloc(i64, nd) catch unreachable; @@ -621,14 +621,14 @@ pub const BufferedRef = struct { fn currentRef(self: *BufferedRef, b: *Builder) Value { if (self.is_trivial or !self.isBuffered()) return self.window_ref; const slot = switch (self.buffer_type) { - .output => b.remui(self.copy_out_slot.?, b.lift(@as(i32, @intCast(self.buffer_count)))), - .input => b.remui(self.wait_in_slot.?, b.lift(@as(i32, @intCast(self.buffer_count)))), + .output => b.remui(self.copy_out_slot.?, b.constant(.i32, self.buffer_count)), + .input => b.remui(self.wait_in_slot.?, b.constant(.i32, self.buffer_count)), }; const a = b.arena.allocator(); const nd = 1 + self.block_compact.len; var base = a.alloc(Value, nd) catch unreachable; var result_shape = a.alloc(i64, nd) catch unreachable; - const c0 = b.lift(@as(i32, 0)); + const c0 = b.constant(.i32, 0); base[0] = slot; result_shape[0] = 1; for (self.block_compact, 0..) |n, ci| { @@ -665,7 +665,7 @@ pub const BufferedRef = struct { // before get_dma_slice; wait_in does it after. fn defaultCopyIn(self: *BufferedRef, b: *Builder, indices: []const Value, slot_cumulative: Value, static_zero_start: bool) void { - const slot = b.remui(slot_cumulative, b.lift(@as(i32, @intCast(self.buffer_count)))); + const slot = b.remui(slot_cumulative, b.constant(.i32, self.buffer_count)); const dma = self.getDmaSlice(b, indices, static_zero_start); const src_sl = self.sliceSrc(b, dma); const dst_sl = self.sliceWindowAt(b, slot, dma); @@ -675,7 +675,7 @@ pub const BufferedRef = struct { fn defaultWaitIn(self: *BufferedRef, b: *Builder, indices: []const Value) void { const dma = self.getDmaSlice(b, indices, false); - const slot = b.remui(self.wait_in_slot.?, b.lift(@as(i32, @intCast(self.buffer_count)))); + const slot = b.remui(self.wait_in_slot.?, b.constant(.i32, self.buffer_count)); const src_sl = self.sliceSrc(b, dma); const dst_sl = self.sliceWindowAt(b, slot, dma); const sem = semAt(b, self.sem_recvs.?, slot); @@ -684,7 +684,7 @@ pub const BufferedRef = struct { fn defaultCopyOut(self: *BufferedRef, b: *Builder, indices: []const Value) void { if (self.buffer_count == 1) @panic("emitPipeline: single-buffered output sync_copy_out unsupported (TODO)"); - const slot = b.remui(self.copy_out_slot.?, b.lift(@as(i32, @intCast(self.buffer_count)))); + const slot = b.remui(self.copy_out_slot.?, b.constant(.i32, self.buffer_count)); const dma = self.getDmaSlice(b, indices, false); // Pallas evaluates `src` (VMEM window) before `dst` (HBM). const src_sl = self.sliceWindowAt(b, slot, dma); @@ -695,7 +695,7 @@ pub const BufferedRef = struct { fn defaultWaitOut(self: *BufferedRef, b: *Builder, indices: []const Value) void { if (self.buffer_count <= 1) return; - const slot = b.remui(self.wait_out_slot.?, b.lift(@as(i32, @intCast(self.buffer_count)))); + const slot = b.remui(self.wait_out_slot.?, b.constant(.i32, self.buffer_count)); const dma = self.getDmaSlice(b, indices, false); const src_sl = self.sliceWindowAt(b, slot, dma); const dst_sl = self.sliceSrc(b, dma); @@ -755,11 +755,11 @@ pub const BufferedRef = struct { } pub fn copyInSlot(self: *const BufferedRef, b: *Builder, slot_cumulative: Value) Value { - return b.remui(slot_cumulative, b.lift(@as(i32, @intCast(self.buffer_count)))); + return b.remui(slot_cumulative, b.constant(.i32, self.buffer_count)); } pub fn waitInSlot(self: *const BufferedRef, b: *Builder) Value { - return b.remui(self.wait_in_slot.?, b.lift(@as(i32, @intCast(self.buffer_count)))); + return b.remui(self.wait_in_slot.?, b.constant(.i32, self.buffer_count)); } pub fn receiveSemaphoreAt(self: *const BufferedRef, b: *Builder, slot: Value) Value { @@ -803,8 +803,8 @@ const Scheduler = struct { fn init(b: *Builder, step: Value, grid: []const Value, num_steps: Value, num_stages: usize, trace_scopes: bool) Scheduler { if (num_stages + 1 > 8) @panic("emitPipeline: buffer_count > 7 unsupported"); - const c0 = b.lift(@as(i32, 0)); - const c1 = b.lift(@as(i32, 1)); + const c0 = b.constant(.i32, 0); + const c1 = b.constant(.i32, 1); var s: Scheduler = .{ .step = step, .grid = grid, @@ -867,7 +867,7 @@ const Scheduler = struct { } fn notFirstStep(self: *Scheduler, b: *Builder) Value { - return b.cmpi(.ne, self.step, b.lift(@as(i32, 0))); + return b.cmpi(.ne, self.step, b.constant(.i32, 0)); } fn indicesDiffer(b: *Builder, br: *const BufferedRef, ai: []const Value, ci: []const Value) Value { @@ -900,13 +900,13 @@ const Scheduler = struct { } fn advance(b: *Builder, slot: Value, pred: Value) Value { - return b.select(pred, b.addi(slot, b.lift(@as(i32, 1))), slot); + return b.select(pred, b.addi(slot, b.constant(.i32, 1)), slot); } fn initializeStep(self: *Scheduler, b: *Builder, br: *BufferedRef, step: usize) void { if ((br.buffer_type != .input and br.buffer_type != .input_output) or !br.isBuffered() or br.is_trivial) return; if (step + 1 >= br.buffer_count) return; - if (br.copy_in_slot == null) br.copy_in_slot = b.lift(@as(i32, 0)); + if (br.copy_in_slot == null) br.copy_in_slot = b.constant(.i32, 0); const pred = if (step == 0) self.first_step else blk: { const block_changed = indicesDiffer(b, br, self.fetchIndices(step), self.fetchIndices(step - 1)); @@ -920,7 +920,7 @@ const Scheduler = struct { var when = b.openIf(pred); { br.copyIn(b, fetch_indices, br.copy_in_slot.?, false); - when.yieldThen(.{}); + when.close(); } } br.copy_in_slot = advance(b, br.copy_in_slot.?, pred); @@ -937,7 +937,7 @@ const Scheduler = struct { self.namedScope(b, "ep_copy_in"); br.copyIn(b, self.fetchIndices(br.buffer_count - 1), br.copy_in_slot.?, false); self.endScope(b); - when.yieldThen(.{}); + when.close(); } } br.copy_in_slot = advance(b, br.copy_in_slot.?, pred); @@ -952,7 +952,7 @@ const Scheduler = struct { self.namedScope(b, "ep_wait_in"); if (br.buffer_type == .input or br.buffer_type == .input_output) br.waitIn(b, self.indices()); self.endScope(b); - when.yieldThen(.{}); + when.close(); } } } @@ -966,7 +966,7 @@ const Scheduler = struct { self.namedScope(b, "ep_copy_out"); if (br.buffer_type == .output or br.buffer_type == .input_output) br.copyOut(b, self.indices()); self.endScope(b); - when.yieldThen(.{}); + when.close(); } } br.copy_out_slot = advance(b, br.copy_out_slot.?, pred); @@ -982,7 +982,7 @@ const Scheduler = struct { // copy_out was issued the previous iteration. if (br.buffer_type == .output or br.buffer_type == .input_output) br.waitOut(b, self.prevIndices()); self.endScope(b); - when.yieldThen(.{}); + when.close(); } } br.wait_out_slot = advance(b, br.wait_out_slot.?, pred); @@ -995,7 +995,7 @@ const Scheduler = struct { self.namedScope(b, "ep_finalize"); br.waitOut(b, self.indices()); self.endScope(b); - when.yieldThen(.{}); + when.close(); } } @@ -1047,7 +1047,7 @@ fn gridSize(b: *Builder, grid: []const Value) Value { fn filterIndices(b: *Builder, indices: []const Value, grid: []const Value) [4]Value { var out: [4]Value = undefined; - const c0 = b.lift(@as(i32, 0)); + const c0 = b.constant(.i32, 0); for (indices, grid, 0..) |i, g, pos| { out[pos] = if (g.asConstantInt() == @as(?i64, 1)) c0 else i; } @@ -1057,12 +1057,12 @@ fn filterIndices(b: *Builder, indices: []const Value, grid: []const Value) [4]Va fn nextIndexND(b: *Builder, indices: []const Value, grid: []const Value) []const Value { const a = b.arena.allocator(); const out = a.alloc(Value, indices.len) catch @panic("nextIndexND OOM"); - const c0 = b.lift(@as(i32, 0)); + const c0 = b.constant(.i32, 0); var carry: Value = b.cmpi(.eq, c0, c0); var rev: usize = 0; while (rev < indices.len) : (rev += 1) { const pos = indices.len - 1 - rev; - const inc = b.select(carry, b.addi(indices[pos], b.lift(@as(i32, 1))), indices[pos]); + const inc = b.select(carry, b.addi(indices[pos], b.constant(.i32, 1)), indices[pos]); carry = b.cmpi(.eq, inc, grid[pos]); out[pos] = b.select(carry, c0, inc); } @@ -1074,15 +1074,15 @@ fn nextIndexND(b: *Builder, indices: []const Value, grid: []const Value) []const fn prevIndexND(b: *Builder, indices: []const Value, grid: []const Value) []const Value { const a = b.arena.allocator(); const out = a.alloc(Value, indices.len) catch @panic("prevIndexND OOM"); - const c0 = b.lift(@as(i32, 0)); + const c0 = b.constant(.i32, 0); var borrow: Value = b.cmpi(.eq, c0, c0); - const cm1 = b.lift(@as(i32, -1)); + const cm1 = b.constant(.i32, -1); var rev: usize = 0; while (rev < indices.len) : (rev += 1) { const pos = indices.len - 1 - rev; - const dec = b.select(borrow, b.subi(indices[pos], b.lift(@as(i32, 1))), indices[pos]); + const dec = b.select(borrow, b.subi(indices[pos], b.constant(.i32, 1)), indices[pos]); borrow = b.cmpi(.eq, dec, cm1); - out[pos] = b.select(borrow, b.subi(grid[pos], b.lift(@as(i32, 1))), dec); + out[pos] = b.select(borrow, b.subi(grid[pos], b.constant(.i32, 1)), dec); } const filtered = filterIndices(b, out, grid); for (0..indices.len) |i| out[i] = filtered[i]; @@ -1138,7 +1138,7 @@ test "trivial input-output ref is copied in and out" { try emitPipeline(&b, &.{b.arg(0)}, &.{}, .{ .body = noOpBody, - .grid = &.{b.lift(@as(i32, 1))}, + .grid = &.{b.constant(.i32, 1)}, .in_specs = &.{.{ .full_shape = &.{ 8, 128 }, .dtype = .bf16, diff --git a/dsl/tcl/BUILD.bazel b/dsl/tcl/BUILD.bazel index 5977491e..096a3a67 100644 --- a/dsl/tcl/BUILD.bazel +++ b/dsl/tcl/BUILD.bazel @@ -1,7 +1,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( - name = "tcl_builder", + name = "builder", srcs = [ "dtype.zig", ], @@ -10,6 +10,7 @@ zig_library( tags = ["manual"], visibility = ["//visibility:public"], deps = [ + "//dsl/common:builder", "//mlir", "//mlir/dialects/arith", "//mlir/dialects/func", @@ -19,5 +20,5 @@ zig_library( zig_test( name = "test", - deps = [":tcl_builder"], + deps = [":builder"], ) diff --git a/dsl/tcl/README.md b/dsl/tcl/README.md index e41cf25a..ec1ae61f 100644 --- a/dsl/tcl/README.md +++ b/dsl/tcl/README.md @@ -93,6 +93,9 @@ def softmax_kernel(B: Axis = 2, D: Axis = 1024, S: Axis = 16, v0: [B, S, D]/bf16 uppercase letter. `furiosa-tcc` needs every axis bound. Kernel inputs and outputs must match the XLA shapes axis by axis (row-major), which `call` checks. +* **Loops.** Put the body in a Zig `{ ... }` block right after `openFor`, + ending with `try loop.yield(...)`. `loop` stays outside, so `loop.results` + stays readable after the block. * **DRAM tensors.** A `.dram` argument comes back as its `as_logical` view; `b.asDram(t, .{...})` returns a result in a DRAM mapping. * **Multi-chip kernels.** A kernel that places data or work on chips (a diff --git a/dsl/tcl/builder.zig b/dsl/tcl/builder.zig index 2fb6f61e..3e5a8e47 100644 --- a/dsl/tcl/builder.zig +++ b/dsl/tcl/builder.zig @@ -1,6 +1,7 @@ const std = @import("std"); const arith = @import("mlir/dialects/arith"); +const common = @import("dsl/common/builder"); const func = @import("mlir/dialects/func"); const mlir = @import("mlir"); const tcl = @import("mlir/dialects/tcl"); @@ -606,7 +607,7 @@ pub const Builder = struct { /// Arguments `update` writes back into, by argument number. updated: std.ArrayList(usize) = .empty, /// Blocks being built inside the entry block (loop bodies), innermost last. - block_stack: std.ArrayList(*mlir.Block) = .empty, + block_stack: common.BlockStack = .{}, pub fn open(allocator: std.mem.Allocator, ctx: *mlir.Context, name: []const u8) !Builder { _ = tcl.symbolAttr(ctx, name) catch std.debug.panic("tcl: `{s}` is not a TCL function name", .{name}); @@ -709,6 +710,7 @@ pub const Builder = struct { _ = func_op.appendTo(self.module.body()); self.func_op = func_op; self.entry = entry; + self.block_stack.push(self.arena.allocator(), entry); var named: ArgsOf(@TypeOf(spec)) = undefined; inline for (fields, 0..) |f, i| { args[i].inner = entry.argument(i); @@ -1033,7 +1035,7 @@ pub const Builder = struct { if (self.compiler_config) |c| f.setAttributeByName("tcl.compiler_config", c); if (self.auto_config) |c| f.setAttributeByName("tcl.auto_config", c); if (self.fusible) f.setAttributeByName("tcl.fusible", .unit(ctx)); - if (!self.module.operation().verify()) return error.InvalidMlir; + try common.verify(self.module, "dsl/tcl"); var printed: std.Io.Writer.Allocating = .init(self.allocator); defer printed.deinit(); @@ -1054,15 +1056,15 @@ pub const Builder = struct { } pub fn pushBlock(self: *Builder, block: *mlir.Block) void { - self.block_stack.append(self.arena.allocator(), block) catch @panic("OOM"); + self.block_stack.push(self.arena.allocator(), block); } pub fn popBlock(self: *Builder) void { - _ = self.block_stack.pop(); + self.block_stack.pop(); } pub fn currentBlock(self: *const Builder) *mlir.Block { - return self.block_stack.getLastOrNull() orelse self.entryBlock(); + return self.block_stack.current(); } pub fn axisAttr(self: *const Builder, a: Axis) *const mlir.Attribute { @@ -1387,9 +1389,11 @@ test "loop over a device-computed trip count with index reads and writes" { // Running sums of the rows up to the longest length. const n = try b.symExpr(.add, try b.reduceMaxI32(t.lens), 0); var loop = b.openFor(n, .{ t.s, try b.scratchpad(.f32, &.{ T, D }) }); - const op = b.tensorOperation(.{}); - const sum = try op.commit(op.fetch(try b.indexRead(t.x, loop.iv), .{}).addf(op.fetch(loop.carried[0], .{})), .{}); - try loop.yield(.{ sum, try b.indexWrite(loop.carried[1], loop.iv, sum) }); + { + const op = b.tensorOperation(.{}); + const sum = try op.commit(op.fetch(try b.indexRead(t.x, loop.iv), .{}).addf(op.fetch(loop.carried[0], .{})), .{}); + try loop.yield(.{ sum, try b.indexWrite(loop.carried[1], loop.iv, sum) }); + } b.ret(&loop.results); const ir = try b.finish(); diff --git a/dsl/triton/BUILD.bazel b/dsl/triton/BUILD.bazel index 780454a8..6699694a 100644 --- a/dsl/triton/BUILD.bazel +++ b/dsl/triton/BUILD.bazel @@ -1,7 +1,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( - name = "triton_builder", + name = "builder", srcs = [ "dtype.zig", ], @@ -10,7 +10,7 @@ zig_library( tags = ["manual"], visibility = ["//visibility:public"], deps = [ - "//dsl/common", + "//dsl/common:builder", "//mlir", "//mlir/dialects/arith", "//mlir/dialects/cf", @@ -25,5 +25,5 @@ zig_library( zig_test( name = "test", - deps = [":triton_builder"], + deps = [":builder"], ) diff --git a/dsl/triton/PORTING.md b/dsl/triton/PORTING.md index 7512cfb8..0d328710 100644 --- a/dsl/triton/PORTING.md +++ b/dsl/triton/PORTING.md @@ -5,6 +5,12 @@ to ZML** (port from a Python `@triton.jit` source) and **fixing an existing Zig kernel that diverges from Python** (using the comparison harness in `examples/triton_emitter/`). +> **The harness is not in this repository.** `examples/triton_emitter/` and +> its scripts (`run.sh`, `dump_python_ir.py`, `kernels_zig.zig`, +> `dump_via_xla.zig`, `compare_ir.py`) are not part of ZML; the steps that +> use them describe the comparison it runs. Writing the kernel and its +> caller (Steps 1.2 and 1.3) and the porting bugs of Phase 3 don't need it. + ## Goals (in priority order) 1. **Lowered code matches.** The Zig version produces **the same LLVM IR + @@ -89,21 +95,27 @@ Inside `zml/kernels/moe/triton/` or `zml/kernels/paged_attention/triton/` (or whichever production file fits), add: ```zig -pub const MyKernel = zml.kernel.triton.Kernel(.{ - .name = "my_kernel", // EXACTLY matches the Python `def my_kernel` - .config = struct { - // One field per `tl.constexpr` parameter. No defaults — every - // field is required at call time. Use the same names Python uses, - // converted to snake_case. +const tri = zml.kernel.triton; + +pub const MyKernel = struct { + pub const Cfg = struct { + // One field per `tl.constexpr` parameter, named as in Python. BLOCK_SIZE: usize, // ... - }, -}, struct { - pub fn run(b: *tri.Builder, cfg: anytype) !void { + }; + pub const Kernel = tri.Kernel(Cfg, .{ + .name = "my_kernel", // EXACTLY matches the Python `def my_kernel` + // Call-site names, in `tt.func` argument order: inputs, then outputs. + .inputs = &.{"x_ptr"}, + .outputs = &.{"out_ptr"}, + .run = run, + }); + + fn run(b: *tri.Builder, cfg: Cfg) tri.FinishError!void { const a = try b.declareArgs(.{ // One field per *runtime* parameter (pointers, dims that // arrive as 0-d tensors). Use the same names + order Python - // uses; the cheat-sheet's "argument specs" table maps the + // uses; the README's "Argument specs" table maps the // tagged-union variants. .x_ptr = .{ .ptr = .f32 }, // ... @@ -113,7 +125,7 @@ pub const MyKernel = zml.kernel.triton.Kernel(.{ // README's "Python Triton ↔ Zig DSL cheat-sheet" section is // your best friend here. } -}); +}; ``` ### Step 1.3 — Wire the production caller @@ -121,8 +133,9 @@ pub const MyKernel = zml.kernel.triton.Kernel(.{ Inside the same `.zig` directory, find the function that launches your kernel (e.g. `pagedAttention2d`, `fusedExpertsImpl`). Replace the TTIR-emit + `ops.triton(...)` two-step with a single -`MyKernel.call(inputs, outputs, opts)` — see the [README](./README.md#calling-kernels-from-a-zml-model--kcall) -for the exact shape. +`MyKernel.Kernel.call(inputs, outputs, opts)` — see the +[README](./README.md#calling-kernels-from-a-zml-model--kkernelcall) for the +exact shape. If this is a brand-new kernel with no existing caller, write the caller fresh. @@ -155,16 +168,13 @@ bazel build //examples/triton_emitter:dump_zig_ir # Zig side compiles + verifi bazel build //zml:test # production side compiles ``` -If the Zig DSL `b.finish()` rejects your IR, the most common causes are: -- **Width mismatches** in `arith.muli` / `addi` (e.g. `i32` × `i64`). - Cast one operand with `.to(.i64)` so both match. -- **Tensor-of-pointers `addPtr` requires same shape** — when the offset - is a tensor, splat the pointer first (`scalar_ptr.addPtr(tensor_off)` - auto-splats; `tensor_ptrs.addPtr(scalar)` doesn't — pass - `tensor_ptrs.addPtr(b.splat(s, &.{shape}))`). -- **`scf.yield` arity** — the tuple you pass to `loop.yield(...)` / - `i.yieldThen(...)` must match the loop's `inits` / `result_types` - arity exactly. +If the Zig DSL `b.finish()` rejects your IR, it logs the module. Mixed +integer widths and scalar/tensor operands are not the cause: the fluent ops +widen the narrower integer side (`v_i32.mul(v_i64)` is an i64 multiply), and +`addPtr` splats a scalar pointer or offset against a tensor partner. A common +cause is an **`scf.yield` type mismatch** — the values you pass to +`loop.yield(...)` / `i.yieldThen(...)` must have the types of the loop's +`inits` / the if's `result_types` (their arity is checked at compile time). ### Step 1.6 — First dump @@ -343,12 +353,12 @@ const y_s_ptr_shifted = a.y_s_ptr.addPtr(pid_i32); // i32, not g_id **Fix:** Match Python's promotion direction. Python implicitly promotes the narrower operand to the wider one. If `cols` is i32 and `group_size` is i64, Python emits `extsi cols i32→i64` then compares -at i64. The Zig DSL doesn't auto-promote, so do it explicitly: +at i64. The fluent ops promote the same way, so compare the two as they +are; a `trunci` like the one above comes from narrowing one side by hand: ```zig -const cols_i32 = b.arange(0, BLOCK, .i32); -const cols_i64 = cols_i32.to(.i64); -const mask = cols_i64.lt(group_size); // both i64, no trunc +const cols = b.arange(0, BLOCK, .i32); +const mask = cols.lt(group_size); // extsi cols to i64, no trunc ``` #### 3.2.5 — Op emit order @@ -709,9 +719,9 @@ If a divergence resists all the standard fixes: Every time a divergence makes you suspect the DSL itself is buggy and you reach for `dsl/triton/builder.zig` — **verify against the Triton sources -first** at `/Users/raph/Documents/Git-Repos/triton`. The DSL exists to -mirror what Python's frontend emits; the source of truth is the Python -trace, not what the rule "looks like" or what feels symmetric. +first** in a Triton checkout. The DSL exists to mirror what Python's +frontend emits; the source of truth is the Python trace, not what the rule +"looks like" or what feels symmetric. Concretely: diff --git a/dsl/triton/README.md b/dsl/triton/README.md index 58cc1939..939a1b07 100644 --- a/dsl/triton/README.md +++ b/dsl/triton/README.md @@ -39,7 +39,7 @@ Two layers define the surface: 11. [Reductions and scans](#reductions-and-scans) 12. [Control flow](#control-flow) 13. [Escape hatches](#escape-hatches) -14. [Lower layer — `Builder.build`](#lower-layer--builderbuild) +14. [Lower layer — `Builder.open`](#lower-layer--builderopen) 15. [Python Triton ↔ Zig DSL cheat-sheet](#python-triton--zig-dsl-cheat-sheet) 16. [Pitfalls](#pitfalls) @@ -216,7 +216,7 @@ Signature: `Kernel.call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Result | `.grid` | `[3]i32` | Triton launch grid `(grid_x, grid_y, grid_z)`. | | `.num_warps` | `i32` | Triton launch param. | | `.num_stages` | `i32` | Triton launch param. | -| `.output_operand_aliases` | `[]const ...` (default `&.{}`) | For in-place updates (still indexed by output position — `.{ .output_index = 0, .operand_index = 1 }`). | +| `.output_operand_aliases` | `?zml.ops.CustomCallOutputOperandAliases(Inputs, Outputs)` (default `null`) | For in-place updates: each output field names the input it aliases — `.{ .out = .x_ptr }`. | | `.debug` | `bool` (default false)| Emit `tt.debug = true` on the custom call. | Returns `Kernel.Results` — a struct of `Tensor` named like `spec.outputs`. @@ -361,27 +361,13 @@ between fp64 / fp32 / fp16 / bf16 lowers to `arith.truncf` (rank-independent fp8 is on either side or the caller requested a non-default `fp_downcast_rounding`. -### Constant hoisting and CSE +### Constants -Every `arith.constant` emitted by the DSL — whether through `lift`, `splat` -of a comptime literal, `zeros` / `ones` / `full`, or any internal helper — -is deduped against a per-`Builder` cache keyed by `:`. On a hit -the existing SSA value is reused; on a miss the new op is inserted at the -*top* of the entry block (before any non-constant op already there) so the -final IR has all constants gathered above the function body, in roughly -first-use order. - -This mirrors what Triton's Python frontend produces and keeps the -`make_ttgir` layout pass from seeing duplicate constants — duplicates can -change the layout-slot assignment for kernels with `tt.dot`, since layout -inference walks the IR and the *first* tensor it sees with a given shape -gets `#blocked`, the second gets `#blocked2`, etc. - -`zeros(shape, dtype)`, `ones(shape, dtype)`, and `full(shape, value, dtype)` -emit a single `arith.constant dense:tensor<...>` (instead of the -old scalar-const + `tt.splat` two-op pattern); same for `splat(comptime, -shape)`. `splat(Value, shape)` still emits `tt.splat` since the operand is -a runtime value, not a literal. +There is no constant cache and no hoisting: every lifted literal emits its +own `arith.constant` where it is used (see Pitfall 5). `zeros(shape, dtype)`, +`ones(shape, dtype)`, `full(shape, value, dtype)` and `splat` of a literal +emit a scalar `arith.constant` and a `tt.splat`; `splat(Value, shape)` emits +the `tt.splat` alone. ### Shape / pointer helpers @@ -451,14 +437,14 @@ in all of these — drop it: | `v_f32.mul(1.0 / config.fp8_max)` | already-f32 expression → f32 const | | `v_i64.lt(NUM_QUERY_HEADS)` (Zig i64) | runtime i64 → i64 const | | `v_i32.add(pad_term)` (Zig i32) | runtime i32 → i32 const | -| `b.splat(-1, shape)` / `b.openFor(0, N, 1, ...)` | comptime → matches elem / lower | -| `b.liftAs(0, .i32)` | liftAs takes anytype — wrap nothing | +| `b.splat(-1, shape)` / `b.openFor(0, N, 1, ...)` | comptime → matches elem / the first Value bound | +| `b.constant(.i32, 0)` | constant takes anytype — wrap nothing | `@as(...)` **is** required exactly when you're narrowing a runtime Zig int to a smaller width that `lift` won't pick on its own. The idiom is `@as(i32, @intCast(X_i64))` — you need the `@as` because `@intCast` has no target type to infer, and without the narrowing `lift` would emit an i64 -constant that mismatches the i32 lhs: +constant, which widens the i32 lhs and the result to i64: ```zig // BLOCK_Q is a Zig i64 (from config). lhs is i32. Must narrow. @@ -470,16 +456,16 @@ Python's `.to(tl.int64)` style but costs two extra Value-level casts; prefer the narrowing `@as` or a pre-declared `const FOO_I32: i32 = @intCast(FOO);` when the same width cast recurs. -### When do you need `b.lift` / `b.liftAs`? +### When do you need `b.lift` / `b.constant`? Almost never. The DSL auto-lifts rhs for every fluent binop, `b.splat`, `b.full`, `b.openFor` bounds, `.addPtr` offsets, etc. Reach for `b.lift` / -`b.liftAs` **only** when the DSL surface demands an existing `Value` and +`b.constant` **only** when the DSL surface demands an existing `Value` and there's no binop sibling to match against — typically: - Loop seeds for `openWhile(.{inits}, ...)` where you start with a raw - comptime literal: `const left_init = b.liftAs(0, .i32);`. -- Mutable `var x: Value = b.liftAs(0, .i32);` that a later branch may + comptime literal: `const left_init = b.constant(.i32, 0);`. +- Mutable `var x: Value = b.constant(.i32, 0);` that a later branch may reassign — the var needs an initial Value even if the comptime path would work on its own. @@ -491,8 +477,8 @@ where such names are required. ### Scalar ↔ tensor auto-broadcast Fluent arithmetic and comparison ops (`.add`, `.sub`, `.mul`, `.div`, `.rem`, -`.cdiv`, `.bitAnd`, `.bitOr`, `.min`, `.max`, `.lt`, `.le`, `.gt`, `.ge`, -`.eq`, `.ne`) are symmetric: if one operand is a tensor and the other a +`.cdiv`, `.bitAnd`, `.bitOr`, `.minimum`, `.maximum`, `.lt`, `.le`, `.gt`, +`.ge`, `.eq`, `.ne`) are symmetric: if one operand is a tensor and the other a scalar, the scalar is splatted to match. Either direction works: ```zig @@ -531,7 +517,7 @@ pointer or scalar offset) against a tensor partner, a pre-emptive ```zig // redundant -pid_n.mul(N).splatTo(shape).add(b.arange(0, N, .i64)).rem(n_blocb.splatTo(shape)) +pid_n.mul(N).splatTo(shape).add(b.arange(0, N, .i64)).rem(n_block.splatTo(shape)) // equivalent — scalars auto-splat against the tensor sibling pid_n.mul(N).add(b.arange(0, N, .i64)).rem(n_block) ``` @@ -554,7 +540,7 @@ the same `extsi` + `muli` pair Python would. For the rare case where you want explicit control, you can still cast yourself; the auto-promotion is a no-op when widths already agree. -Recurring example from `triton_kernels.zig`: +Recurring example from `zml/kernels/moe/triton/moe.zig`: ```zig // pid (i32) vs block (i64 because it came from a usize config) @@ -567,7 +553,7 @@ For pointer math it is often cleaner to promote everything to i64 up front: ```zig const pid = b.programId(.x).to(.i64); -const offs = b.arange(BLOCK, .i64); +const offs = b.arange(0, BLOCK, .i64); ``` --- @@ -583,7 +569,8 @@ const offs = b.arange(BLOCK, .i64); | `b.expandDims(v, axis)` / `v.expandDims(axis)` | `tt.expand_dims` — insert a size-1 axis. Mirrors `v[:, None]` (`axis=1`) / `v[None, :]` (`axis=0`). | | `b.mask2d(cond_m, cond_n, _, _)` | `cond_m[:, None] & cond_n[None, :]` — emits the `(m,1) & (1,n)` form; auto-broadcast at the next consumer. | | `b.lift(value)` | Wrap a Zig scalar as a DSL Value; dtype inferred from source. | -| `b.liftAs(value, dtype)` | Wrap a Zig scalar as a DSL Value of a specific `DType`. | +| `b.constant(dtype, value)` | Wrap a Zig scalar as a DSL Value of a specific `DType`. | +| `b.constantLike(value, like)` | Wrap a Zig scalar as a DSL Value of `like`'s element type. | --- @@ -631,7 +618,7 @@ so user code never constructs `*const mlir.Type` manually: | `b.dot(a, b, acc)` / `b.dotOpts(a, b, acc, opts)` | `acc.type_()`. | | `b.dotScaled(...)` / `b.dotScaledOpts(...)` | `acc.type_()`. | | `b.reshape(src, new_shape)` / `b.reshapeOpts(...)` | `new_shape` + `src` element type. | -| `b.gather(src, indices, axis)` / `b.gatherOpts(...)`| `indices` shape + `src` element type. | +| `b.gather(src, indices, axis)` | `indices` shape + `src` element type. | | `b.histogram(src, num_bins)` / `b.histogramOpts(...)` | `tensor`. | | `b.fpToFp(src, out_dtype)` / `b.fpToFpOpts(...)` | `src` shape + `out_dtype`. | | `b.convertf(src, out_dtype)` / `b.convertfOpts(...)`| `src` shape + `out_dtype`. | @@ -643,9 +630,9 @@ a `new_shape`, a `num_bins`, an `out_dtype`, a TMA tile descriptor. The DSL never asks you to restate information it can read off a `Value`. `b.scalarTy(dtype)` / `b.tensorTy(shape, dtype)` are still available but only -needed by `openIf.result_types` / `openWhile.after_types` (where block-arg -types can be arbitrary) and by the raw `externElementwise` / -`inlineAsmElementwise` escape hatches. +needed for tensor result types in `openIfElse` / `openWhile` (scalar ones are +plain dtypes) and by the raw `externElementwise` / `inlineAsmElementwise` +escape hatches. ### `fn X / fn XOpts` pattern @@ -671,11 +658,12 @@ Pairs in place today: `load` / `loadOpts`, `store` / `storeOpts`, `deviceAssert` / `deviceAssertOpts`, `maximum` / `maximumOpts`, `minimum` / `minimumOpts`, `sum` / `sumOpts`, `max` / `maxOpts`, `min` / `minOpts`, -`cumsum` / `cumsumOpts`, `cumprod` / `cumprodOpts`. +`cumsum` / `cumsumOpts`, `cumprod` / `cumprodOpts`, +`openFor` / `openForOpts` (`tl.range(..., num_stages=...)`). -DSL-only pair (no Python kwarg counterpart — `.inline_return` is about IR +DSL-only pairs (no Python kwarg counterpart — `.inline_return` is about IR block layout, see the `returnIf` / `returnIfOpts` section under Control flow): -`returnIf` / `returnIfOpts`. +`returnIf` / `returnIfOpts`, `openReturnIf` / `openReturnIfOpts`. Ops with no kwargs in Python (pure positional): `gather(src, indices, axis)`, `trans(src, order)`, `permute(src, order)`, `cat(lhs, rhs)` (dim=0 default), @@ -741,7 +729,8 @@ closure for `combine` and give you full control. SCF regions are built with **scope-based** builders — `b.openFor(...)`, `b.openIf(...)` / `b.openIfElse(...)`, `b.openWhile(...)` return a typed scope value. You emit body ops into a bare `{}` block, terminate with `yield` / -`yieldThen` / `yieldAfter`, and read results from `scope.results` afterward. +`yieldThen` / `yieldAfter` (or `close()` for a scope without values), and read +results from `scope.results` afterward. No `ctx` struct, no function literal, no `kk`/`k` split — lexical scope carries captures because everything runs in the same function. @@ -761,12 +750,15 @@ const total = loop.results[0]; ``` - `lower`/`upper`/`step` are `anytype`: pass Values, comptime ints, or runtime - Zig ints. Comptime literals adapt to the first operand's type; cast explicitly - to force a specific width (e.g. `@as(i32, @intCast(config.block))`). -- `inits` is a tuple literal `.{v1, v2, ...}`. Arity is comptime — `carried`, - `results`, and the `yield(...)` call are all fixed-size arrays of the same - length. -- Empty loops: `b.openFor(0, N, S, .{})` + `loop.yield(.{})` (no iter_args). + Zig ints. A bound that isn't a Value takes the type of the first one that is; + with none, `lower` is lifted (`i32` when it fits) and the others follow it. + Pass a typed constant (`b.constant(.i64, 0)`) to pick another width. +- `inits` is a tuple `.{v1, v2, ...}` or an array. Arity is comptime — + `carried`, `results`, and the `yield(...)` values (also a tuple or an array) + all have the same length. +- Empty loops: `b.openFor(0, N, S, .{})` + `loop.close()` (no iter_args). +- `b.openForOpts(lower, upper, step, inits, .{ .num_stages = 3 })` sets the + loop's options (`tt.num_stages`, `.unsigned_cmp`). ### `openIf` — scf.if without else (side-effects only) @@ -774,17 +766,17 @@ const total = loop.results[0]; var i = b.openIf(cond); { // then body - i.yieldThen(.{}); // closes scope, builds scf.if with empty else + i.close(); // closes scope, builds scf.if with empty else } ``` -Use for conditional side-effects (no results). There is no `yieldElse` — the +Use for conditional side-effects (no results). There is no else branch — the else block is auto-built empty. ### `openIfElse` — scf.if with else and optional results ```zig -var i = b.openIfElse(cond, .{ b.scalarTy(.f32) }); // result types +var i = b.openIfElse(cond, .{.f32}); // result types { // then body i.yieldThen(.{ x }); @@ -796,16 +788,17 @@ var i = b.openIfElse(cond, .{ b.scalarTy(.f32) }); // result types const r = i.results[0]; ``` -- `result_types` is a tuple of `*const mlir.Type` — use `b.scalarTy(dtype)` or - `b.tensorTy(shape, dtype)`. Pass `.{}` for an if/else with no results but - distinct then/else bodies. +- `result_types` is a tuple or array of dtypes (`.f32`), Values (for their + types) or `*const mlir.Type`s such as `b.tensorTy(shape, dtype)`. Pass + `.{}` for an if/else with no results but distinct then/else bodies, then + end them with `elseBranch()` and `close()`. - `yieldThen` swaps to the else block; `yieldElse` builds the `scf.if` op. - Both branches must yield arity-matching tuples. ### `openWhile` — scf.while ```zig -var w = b.openWhile(.{ i0 }, .{ b.scalarTy(.i32) }); +var w = b.openWhile(.{ i0 }, .{.i32}); { // before region (condition) const s = w.before_carried[0]; @@ -819,9 +812,10 @@ var w = b.openWhile(.{ i0 }, .{ b.scalarTy(.i32) }); const r = w.results[0]; ``` -- `inits` — tuple of `Value`s; before-region arg types = their types. -- `after_types` — tuple of `*const mlir.Type`; after-region arg types + the - scf.while's result types. +- `inits` — a tuple or array of `Value`s; before-region arg types = their types. +- `after_types` — a tuple or array of dtypes (`.f32`), Values (for their + types) or `*const mlir.Type`s; after-region arg types + the scf.while's + result types. - `yieldBefore(cond, forwarded)` — `forwarded` arity must match `after_types`. - `yieldAfter(values)` — arity must match `inits`. @@ -830,7 +824,7 @@ const r = w.results[0]; ```zig // Python: if pid_m * BLOCK >= num_tokens_post_padded: return const out_of_range = pid_m.mul(block).ge(num_tokens); -b.returnIf(out_of_range, .{}); +b.returnIf(out_of_range); // ... ops here live in a fresh fall-through block ... ``` @@ -844,20 +838,19 @@ cf.cond_br %out_of_range, ^ret, ^cont // subsequent ops ``` -- `values` is a tuple of `Value`s matching the `tt.func`'s declared result - types (pass `.{}` for a void kernel). +- Kernels return nothing, so neither form takes return values. - Works **only at the top-level `tt.func` body** — don't call inside an `scf.if`/`scf.for`/`scf.while` region (those must stay structured; cf branches can't escape their parent scf region). - After `returnIf`, the kernel's current block is swapped to a fresh fall-through block — any subsequent `b.*` calls emit into that block, and - `b.finish(...)` appends the final `tt.return` there. + `b.finish()` appends the final `tt.return` there. - Use this to match Python Triton's `if cond: return` lowering exactly. Python emits `cf.cond_br` for early-return idioms while using `scf.for` / `scf.if` for structured loops and value-carrying branches; our DSL does the same. -**`returnIfOpts(cond, values, .{ .inline_return = true })` — match Python's +**`returnIfOpts(cond, .{ .inline_return = true })` — match Python's per-`return` block layout.** By default `returnIf` routes every return path (every early `returnIf` / `openReturnIf` plus the natural fall-through from `b.finish`) through one **shared exit block** holding the single `tt.return`. @@ -868,8 +861,8 @@ but differ in the function-exit block's label and position — which renumbers every `$L__BB` label downstream and blows up the PTX diff. When the Python source contains *any* literal `return` statement (it then has ≥2 `ret` blocks in LLVM — the explicit one(s) plus the implicit function-end one — which SimplifyCFG merges into `common.ret`), pass -`.{ .inline_return = true }` on *every* return site, and set -`scope.inline_return = true` on each `openReturnIf` (see below). Each one +`.{ .inline_return = true }` on *every* return site, including each +`openReturnIfOpts` (see below). Each one then emits `tt.return` inline and SimplifyCFG reproduces Python's `common.ret`. The default shared exit block emits exactly one `tt.return`, so the merge never fires and the function-exit block lands somewhere else. @@ -878,11 +871,10 @@ so the merge never fires and the function-exit block lands somewhere else. ```zig // Python: if off_experts == -1: write_zeros_to_output(...); return -var scope = b.openReturnIf(is_dead); -scope.inline_return = true; // optional — see `returnIfOpts` above +var scope = b.openReturnIf(is_dead); // or openReturnIfOpts, see `returnIfOpts` above { writeZerosToOutput(k, c_ptr, ...); // ops here emit into ^ret - scope.yieldReturn(.{}); // closes ^ret with tt.return (or cf.br ^exit) + scope.close(); // closes ^ret with tt.return (or cf.br ^exit) } // ... ops here emit into ^cont (fall-through) ... ``` @@ -890,15 +882,14 @@ scope.inline_return = true; // optional — see `returnIfOpts` abov Lowers to the same `cf.cond_br ^ret, ^cont` shape as `returnIf`, but the taken branch can carry arbitrary side-effects between the `cf.cond_br` and the `tt.return`. Use this instead of the -`openIf(cond){body}yieldThen; returnIf(cond, values)` pair — that pattern +`openIf(cond){body}close; returnIf(cond)` pair — that pattern duplicates the predicate (emits both `scf.if` and `cf.cond_br` on the same boolean) and drifts from Triton's reference IR. -`scope.inline_return` is the per-scope twin of `returnIfOpts`'s -`.inline_return` (default `false`): set it before `yieldReturn` to make -`^ret` close with `tt.return` directly instead of `cf.br ^exit`. Flip it on -every `returnIf` / `openReturnIf` of a kernel when matching a Python source -that has literal `return` statements. +`openReturnIfOpts(cond, .{ .inline_return = true })` makes `close` end +`^ret` with `tt.return` directly instead of `cf.br ^exit`. Use it on every +`returnIf` / `openReturnIf` of a kernel when matching a Python source that has +literal `return` statements. Same restrictions as `returnIf`: top-level `tt.func` body only. @@ -906,9 +897,9 @@ Same restrictions as `returnIf`: top-level `tt.func` body only. | Python source | Emits | DSL builder | |----------------------------------------|-------|---------------------------------| -| `if cond: return` | `cf` | `b.returnIf(cond, .{})` | -| `if cond: ; return` | `cf` | `var s = b.openReturnIf(cond); {...; s.yieldReturn(.{})}` | -| `if cond: ` | `scf` | `b.openIf(cond) {...; yieldThen(.{})}` | +| `if cond: return` | `cf` | `b.returnIf(cond)` | +| `if cond: ; return` | `cf` | `var s = b.openReturnIf(cond); {...; s.close()}` | +| `if cond: ` | `scf` | `b.openIf(cond) {...; close()}` | | `if cond: x=A else: x=B` (value-carrying) | `scf` | `b.openIfElse(cond, .{ty}) {...; yieldThen(.{a})} {...; yieldElse(.{b})}` | | `for i in range(...)` | `scf` | `b.openFor(lo, hi, step, .{inits})` | | `while cond:` | `scf` | `b.openWhile(.{inits}, .{after_types})` | @@ -935,7 +926,8 @@ built-in reductions cover the common cases. - Scopes stack naturally: `openFor { openIf { openFor { ... } } }` all just push/pop the kernel's current-block stack (same for `openIfElse` / `openWhile`). -- Empty yields are legal: `loop.yield(.{})`, `i.yieldThen(.{})`, etc. +- Scopes without values end with `close()`: `loop.close()`, `i.close()`, and + `elseBranch()` then `close()` for an if/else. - After any `yield*`, the scope's `.results` field holds the scf op's results as a `[N]Value`. @@ -961,12 +953,11 @@ is the manual form. --- -## Lower layer — `Builder.build` +## Lower layer — `Builder.open` The declarative `zml.kernel.triton.Kernel(Config, spec)` form covers most -cases. For existing code, dynamic-arity arg lists, or escape-hatch scenarios -where you want to drive the IR construction yourself, the lower-layer -`Builder.build` API is still available: +cases. For dynamic-arity arg lists, or escape-hatch scenarios where you want +to drive the IR construction yourself, use the `Builder` directly: ```zig const Builder = tri.Builder; @@ -976,38 +967,32 @@ pub fn fusedMoeKernel( ctx: *mlir.Context, config: GenerationConfig, ) ![:0]const u8 { - var spec = try Builder.build(allocator, ctx, "fused_moe_kernel", .{ + var b = try Builder.open(allocator, ctx, "fused_moe_kernel"); + defer b.deinit(); + const a = try b.declareArgs(.{ .a_ptr = .{ .ptr = config.a_dtype }, .b_ptr = .{ .ptr = config.b_dtype }, // ... - }, &.{}); - defer spec.deinit(); - const b = &spec.kernel; - const a = spec.args; + }); // ... emit body via b.* helpers; same DSL as inside `Kernel(...)` ... - return b.finish(&.{}); + return b.finish(); } ``` -`Builder.build(allocator, ctx, name, spec, result_types)` returns a -heap-allocated `*Built(Spec)`: - -- `spec.kernel` — the runtime `Builder`; pass `&spec.kernel` to helpers. -- `spec.args` — the same `NamedArgs(Spec)` struct that the declarative - `b.declareArgs(...)` returns. -- `spec.deinit()` — frees the module + bundle. -- `b.finish(results)` — terminates with `tt.return`, verifies, serializes - to `[:0]const u8`. Pass `&.{}` for kernels with no return values. +- `Builder.open(allocator, ctx, name)` creates an empty builder, and + `b.declareArgs(spec)` creates the `tt.func` and returns its arguments as + the same `NamedArgs(Spec)` struct as inside `Kernel(...)` — exactly what + `zml.kernel.triton.Kernel(...)` does internally. + `b.declareArgsOpts(spec, .{ .@"noinline" = true })` sets the function's + options. +- `b.finish()` terminates with `tt.return`, verifies, and serializes to a + `[:0]const u8` owned by `allocator`. `b.deinit()` frees the module. For dynamic-arg-count construction, `Builder.init(allocator, ctx, name, -args: []const ArgSpec, result_types)` takes a runtime slice of arg specs -instead of a comptime struct literal; you grab block args with `b.arg(i)`. - -You can also use `Builder.open(allocator, ctx, name)` to create an empty -builder and call `b.declareArgs(spec)` mid-stream — that's exactly what -`zml.kernel.triton.Kernel(...)` does internally. +args)` takes a runtime `[]const ArgSpec` instead of a comptime struct +literal; you grab block args with `b.arg(i)`. Pass these constructors a `zml.kernel.triton.newContext()`, not a `Compilation`'s `mlir_ctx`; `defer ctx.deinit()` once you have the IR @@ -1027,7 +1012,7 @@ Prefer `zml.kernel.triton.Kernel(Config, spec)` for the common case. | `x.to(tl.int64)` | `x.to(.i64)` | | `a + b`, `a * b`, `a // b` | `a.add(b)`, `a.mul(b)`, `a.div(b)` | | `a < b` | `a.lt(b)` | -| `mask & other` | `masb.bitAnd(other)` | +| `mask & other` | `mask.bitAnd(other)` | | `tl.where(c, a, b)` | `b.where(c, a, b)` (alias of `b.select`) | | `vec[:, None]` | `vec.expandDims(1)` | | `vec[None, :]` | `vec.expandDims(0)` | @@ -1078,19 +1063,20 @@ Prefer `zml.kernel.triton.Kernel(Config, spec)` for the common case. | `tl.extern_elementwise(...)` | `b.externElementwise(srcs, shape, dtype, lib, path, sym)` | | `for i in range(0, N, S): ...` | `var loop = b.openFor(0, N, S, .{inits...}); { ...; loop.yield(.{...}); }` | | `while cond: ...` | `var w = b.openWhile(.{inits}, .{after_types}); { ...; w.yieldBefore(cond, .{...}); }; { ...; w.yieldAfter(.{...}); }` | -| `if cond: ...` (side-effects, no else) | `var i = b.openIf(cond); { ...; i.yieldThen(.{}); }` | +| `if cond: ...` (side-effects, no else) | `var i = b.openIf(cond); { ...; i.close(); }` | | `if cond: ... else: ...` | `var i = b.openIfElse(cond, .{result_types}); { ...; i.yieldThen(.{...}); }; { ...; i.yieldElse(.{...}); }` | -| `if cond: return` (early exit) | `b.returnIf(cond, .{});` — emits `cf.cond_br` + `tt.return`, continues in fall-through block (add `b.returnIfOpts(cond, .{}, .{ .inline_return = true })` when matching a Python source that has literal `return`s) | -| `if cond: ; return` (early exit, with side-effects) | `var s = b.openReturnIf(cond); { ...; s.yieldReturn(.{}); }` — single `cf.cond_br`; body emits into `^ret` before the `tt.return` (set `s.inline_return = true` to match, alongside `returnIfOpts`) | +| `if cond: return` (early exit) | `b.returnIf(cond);` — emits `cf.cond_br` + `tt.return`, continues in fall-through block (`b.returnIfOpts(cond, .{ .inline_return = true })` when matching a Python source that has literal `return`s) | +| `if cond: ; return` (early exit, with side-effects) | `var s = b.openReturnIf(cond); { ...; s.close(); }` — single `cf.cond_br`; body emits into `^ret` before the `tt.return` (`b.openReturnIfOpts(cond, .{ .inline_return = true })` to match, alongside `returnIfOpts`) | --- ## Pitfalls -1. **Width mismatches between runtime operands.** `v_i32.mul(v_i64)` will hit - `arith.muli requires the same type` at verify time. Either cast one with - `.to(...)` or reshape the kernel to stay within a single int width. Comptime - literals adapt automatically — runtime values don't. +1. **Mixed integer widths widen.** `v_i32.mul(v_i64)` is an i64 multiply + with an `arith.extsi` on the i32 side, and so is `v_i32.mul(x)` for a + runtime Zig `i64` `x`. Narrow with `@as(i32, @intCast(x))` or `.to(.i32)` + when the result must stay i32. Comptime literals take the other operand's + type instead. 2. **Tensor-of-pointers via `.addPtr`.** Auto-splats either side: scalar `!tt.ptr` + tensor offset (`x_ptr.addPtr(tensor_offs)`) and tensor @@ -1099,7 +1085,7 @@ Prefer `zml.kernel.triton.Kernel(Config, spec)` for the common case. 3. **Scope-based control flow — no closures needed.** `openFor` / `openIf` / `openIfElse` / `openWhile` push a block onto the kernel's insertion stack; - body ops emitted in the bare `{}` that follows go into that blocb. Captures + body ops emitted in the bare `{}` that follows go into that block. Captures are plain lexical scope, so you can reference outer Values, sizes, etc. directly — no `ctx` struct, no function literal. @@ -1115,10 +1101,10 @@ Prefer `zml.kernel.triton.Kernel(Config, spec)` for the common case. way to PTX, so the generated code is identical whether you emit one constant or a hundred. Optimize for readability at the builder level. -6. **`finish(results, allocator)` verifies before serializing.** A verifier - failure here almost always means an operand-type mismatch (see #1) or a +6. **`finish()` verifies before serializing.** A verifier + failure here almost always means an operand-type mismatch or a missing block terminator — a common one is forgetting that `scf.yield` - operand count must match the loop's result types. + operand types must match the loop's result types. 7. **Yielding from loop / if / while bodies.** Inside a scope (`openFor`, `openIf`, `openIfElse`, `openWhile`), you *don't* return values — you call @@ -1126,5 +1112,5 @@ Prefer `zml.kernel.triton.Kernel(Config, spec)` for the common case. `yieldAfter`) with a tuple of Values matching the scope's declared arity. Scopes with results (`openFor`, `openIfElse`, `openWhile`) store the scf op's results in `scope.results` as a fixed-size `[N]Value` array. `openIf` - has no results — `yieldThen(.{})` closes and builds scf.if with an empty + has no results — `close()` closes and builds scf.if with an empty else block in one step. diff --git a/dsl/triton/builder.zig b/dsl/triton/builder.zig index ae802c9c..9917ad2c 100644 --- a/dsl/triton/builder.zig +++ b/dsl/triton/builder.zig @@ -2,13 +2,12 @@ const std = @import("std"); const arith = @import("mlir/dialects/arith"); const cf = @import("mlir/dialects/cf"); -const dsl = @import("dsl/common"); +const common = @import("dsl/common/builder"); const func = @import("mlir/dialects/func"); const llvm = @import("mlir/dialects/llvm"); const math = @import("mlir/dialects/math"); const scf = @import("mlir/dialects/scf"); const ttir = @import("mlir/dialects/ttir"); -const tupleArity = dsl.tupleArity; const mlir = @import("mlir"); const stdx = @import("stdx"); @@ -177,6 +176,9 @@ pub const Value = struct { pub fn to(self: Value, dtype: DType) Value { return self.kern().cast(self, dtype); } + pub fn bitCast(self: Value, dtype: DType) Value { + return self.kern().bitCast(self, dtype); + } pub fn toOpts(self: Value, dtype: DType, opts: CastOpts) Value { return self.kern().castOpts(self, dtype, opts); } @@ -381,59 +383,37 @@ pub const ReturnIfOpts = struct { }; pub fn ForScope(comptime N: usize) type { - return dsl.ForScope(Builder, Value, N); + return common.ForScope(Builder, Value, N); } +/// `if (cond) { ...; return; }`: `close` returns at the end of the body, +/// then the kernel continues after it. pub const ReturnIfScope = struct { kernel: *Builder, ret_block: *mlir.Block, cont_block: *mlir.Block, - inline_return: bool = false, + opts: ReturnIfOpts, - pub fn yieldReturn(self: *ReturnIfScope, values: anytype) void { + pub fn close(self: *ReturnIfScope) void { const k = self.kernel; - const info = @typeInfo(@TypeOf(values)).@"struct"; - - if (self.inline_return) { - var ret_operands: [info.field_names.len]*const mlir.Value = undefined; - inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { - if (Field != Value) - @compileError("ReturnIfScope.yieldReturn: every value must be a Value"); - ret_operands[i] = @field(values, field_name).inner; - } - _ = ttir.return_(k.ctx, &ret_operands, k.loc()).appendTo(self.ret_block); + if (self.opts.inline_return) { + _ = ttir.return_(k.ctx, &.{}, k.loc()).appendTo(self.ret_block); } else { - if (k.exit_block == null) { - k.exit_block = mlir.Block.init(&.{}, &.{}); - - var ret_operands: [info.field_names.len]*const mlir.Value = undefined; - inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { - if (Field != Value) - @compileError("ReturnIfScope.yieldReturn: every value must be a Value"); - ret_operands[i] = @field(values, field_name).inner; - } - _ = ttir.return_(k.ctx, &ret_operands, k.loc()).appendTo(k.exit_block.?); - } - _ = cf.br(k.ctx, k.exit_block.?, &.{}, k.loc()).appendTo(self.ret_block); + _ = cf.br(k.ctx, k.exitBlock(), &.{}, k.loc()).appendTo(self.ret_block); } k.popBlock(); - - if (k.block_stack.items.len == 0) { - k.pushBlock(self.cont_block); - } else { - k.block_stack.items[k.block_stack.items.len - 1] = self.cont_block; - } + k.continueIn(self.cont_block); } }; -pub const IfOnlyScope = dsl.IfOnlyScope(Builder, Value); +pub const IfOnlyScope = common.IfOnlyScope(Builder, Value); pub fn IfScope(comptime N: usize) type { - return dsl.IfScope(Builder, Value, N); + return common.IfScope(Builder, Value, N); } pub fn WhileScope(comptime N: usize, comptime M: usize) type { - return dsl.WhileScope(Builder, Value, N, M); + return common.WhileScope(Builder, Value, N, M); } pub fn ReduceArgs(comptime CtxT: type) type { @@ -470,7 +450,7 @@ pub const Builder = struct { func_op: ?*mlir.Operation, entry_block: ?*mlir.Block, exit_block: ?*mlir.Block = null, - block_stack: std.ArrayList(*mlir.Block), + block_stack: common.BlockStack = .{}, /// `noinline` is a Zig keyword — write `.{ .@"noinline" = true }`. pub const Opts = struct { @@ -482,9 +462,8 @@ pub const Builder = struct { ctx: *mlir.Context, name: []const u8, args: []const ArgSpec, - result_types: []const *const mlir.Type, ) !Builder { - return initOpts(allocator, ctx, name, args, result_types, .{}); + return initOpts(allocator, ctx, name, args, .{}); } pub fn initOpts( @@ -492,12 +471,11 @@ pub const Builder = struct { ctx: *mlir.Context, name: []const u8, args: []const ArgSpec, - result_types: []const *const mlir.Type, opts: Opts, ) !Builder { var b = try open(allocator, ctx, name); errdefer b.deinit(); - try b.declareArgsLowOpts(args, result_types, opts); + try b.declareArgsLowOpts(args, opts); return b; } @@ -521,20 +499,19 @@ pub const Builder = struct { .name = name, .func_op = null, .entry_block = null, - .block_stack = .empty, + .block_stack = .{}, }; } - pub fn declareArgs(self: *Builder, spec: anytype) !dsl.NamedArgs(@TypeOf(spec), Value) { - return self.declareArgsOpts(spec, &.{}, .{}); + pub fn declareArgs(self: *Builder, spec: anytype) !common.NamedArgs(@TypeOf(spec), Value) { + return self.declareArgsOpts(spec, .{}); } pub fn declareArgsOpts( self: *Builder, spec: anytype, - result_types: []const *const mlir.Type, opts: Opts, - ) !dsl.NamedArgs(@TypeOf(spec), Value) { + ) !common.NamedArgs(@TypeOf(spec), Value) { const Spec = @TypeOf(spec); const field_names = @typeInfo(Spec).@"struct".field_names; @@ -563,9 +540,9 @@ pub const Builder = struct { arg_specs[i] = .{ .name = field_name, .kind = kind }; } - try self.declareArgsLowOpts(&arg_specs, result_types, opts); + try self.declareArgsLowOpts(&arg_specs, opts); - var named: dsl.NamedArgs(Spec, Value) = undefined; + var named: common.NamedArgs(Spec, Value) = undefined; inline for (field_names, 0..) |field_name, i| { @field(named, field_name) = self.arg(i); } @@ -575,7 +552,6 @@ pub const Builder = struct { fn declareArgsLowOpts( self: *Builder, args: []const ArgSpec, - result_types: []const *const mlir.Type, opts: Opts, ) !void { std.debug.assert(self.entry_block == null); @@ -623,7 +599,7 @@ pub const Builder = struct { .name = self.name, .args = arg_types, .args_attributes = if (any_arg_attr) arg_attrs else null, - .results = result_types, + .results = &.{}, .block = entry, .location = unknown_loc, .no_inline = opts.@"noinline", @@ -632,6 +608,7 @@ pub const Builder = struct { self.func_op = func_op; self.entry_block = entry; + self.block_stack.push(self.arena.allocator(), entry); } pub fn deinit(self: *Builder) void { @@ -645,18 +622,15 @@ pub const Builder = struct { } pub fn pushBlock(self: *Builder, b: *mlir.Block) void { - self.block_stack.append(self.arena.allocator(), b) catch @panic("Builder.pushBlock OOM"); + self.block_stack.push(self.arena.allocator(), b); } pub fn popBlock(self: *Builder) void { - _ = self.block_stack.pop(); + self.block_stack.pop(); } pub fn currentBlock(self: *Builder) *mlir.Block { - if (self.block_stack.items.len > 0) { - return self.block_stack.items[self.block_stack.items.len - 1]; - } - return self.entry_block orelse @panic("Builder has no current block — call declareArgs first"); + return self.block_stack.current(); } fn innerSlice(self: *Builder, values: []const Value) []const *const mlir.Value { @@ -790,7 +764,7 @@ pub const Builder = struct { } /// Lift a Zig scalar to a DSL constant. Dtype follows the source type; - /// unsigned ints are bit-cast (signless MLIR). Use `liftAs` for explicit dtype. + /// unsigned ints are bit-cast (signless MLIR). Use `constant` for an explicit dtype. pub fn lift(self: *Builder, value: anytype) Value { const T = @TypeOf(value); if (T == Value) return value; @@ -908,35 +882,45 @@ pub const Builder = struct { @panic("element type not a recognized DSL DType"); } - /// Lift a Zig scalar to a DSL constant of `dtype`. Unsigned ints are bit-cast. - pub fn liftAs(self: *Builder, value: anytype, dtype: DType) Value { + /// `value` as a constant of `dtype`; a Value passes through. An integer is + /// signed or an unsigned bit pattern of `dtype`'s width; anything else + /// panics. + pub fn constant(self: *Builder, dtype: DType, value: anytype) Value { const T = @TypeOf(value); if (T == Value) return value; if (isFloatDtype(dtype)) { const v_f64: f64 = switch (@typeInfo(T)) { .comptime_int, .int => @floatFromInt(value), .comptime_float, .float => @floatCast(value), - else => @compileError("Builder.liftAs: unsupported scalar " ++ @typeName(T)), + else => @compileError("Builder.constant: unsupported scalar " ++ @typeName(T)), }; return self.emitFloat(dtype, v_f64); } - const v_i64: i64 = switch (@typeInfo(T)) { - .comptime_int => blk: { - if (value >= std.math.minInt(i64) and value <= std.math.maxInt(i64)) - break :blk @intCast(value); - if (value >= 0 and value <= std.math.maxInt(u64)) - break :blk @bitCast(@as(u64, value)); - @compileError("Builder.liftAs: integer literal out of 64-bit range"); - }, - .int => |info| if (info.signedness == .signed) @intCast(value) else @bitCast(@as(u64, @intCast(value))), + const v: i128 = switch (@typeInfo(T)) { + .comptime_int, .int => value, .comptime_float, .float => @intFromFloat(value), - else => @compileError("Builder.liftAs: unsupported scalar " ++ @typeName(T)), + else => @compileError("Builder.constant: unsupported scalar " ++ @typeName(T)), }; - return self.emitInt(dtype, v_i64); + return self.emitInt(dtype, intBits(v, @intCast(intBitwidth(dtype)), @tagName(dtype))); + } + + /// `value` as the `i64` an integer constant of `bits` bits is built from. + /// It may be signed or an unsigned bit pattern, so it must lie in + /// [-2^(bits-1), 2^bits). + fn intBits(value: i128, bits: u7, dtype: []const u8) i64 { + if (value < -(@as(i128, 1) << (bits - 1)) or value >= @as(i128, 1) << bits) + std.debug.panic("Builder.constant: {d} does not fit {s}", .{ value, dtype }); + return @truncate(value); + } + + /// `value` as a constant of `like`'s element type; a Value passes through. + pub fn constantLike(self: *Builder, value: anytype, like: Value) Value { + if (@TypeOf(value) == Value) return value; + return self.constMatching(value, like.elemType()); } fn constMatching(self: *Builder, value: anytype, elem: *const mlir.Type) Value { - return self.liftAs(value, self.mlirElemToDType(elem)); + return self.constant(self.mlirElemToDType(elem), value); } pub fn zeros(self: *Builder, shape: []const i64, dtype: DType) Value { @@ -948,7 +932,7 @@ pub const Builder = struct { } pub fn full(self: *Builder, shape: []const i64, value: anytype, dtype: DType) Value { - return self.splat(self.liftAs(value, dtype), shape); + return self.splat(self.constant(dtype, value), shape); } /// 2-D mask `(m, 1) & (1, n)` — auto-broadcasts to `(m, n)`. @@ -1307,7 +1291,7 @@ pub const Builder = struct { // ==================== Triton-specific ops ==================== /// tt.bitcast — same-bitwidth cast. Shape preserved from src. - pub fn bitcast(self: *Builder, src: Value, out_dtype: DType) Value { + pub fn bitCast(self: *Builder, src: Value, out_dtype: DType) Value { return self.emit(ttir.bitcast(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); } @@ -1322,7 +1306,7 @@ pub const Builder = struct { /// - other fp↔fp wide: extf. /// - i1→fp: uitofp. int↔int: extsi/trunci/bitcast by width. pub fn castOpts(self: *Builder, src: Value, dtype: DType, opts: CastOpts) Value { - if (opts.bitcast) return self.bitcast(src, dtype); + if (opts.bitcast) return self.bitCast(src, dtype); const cur_elem = src.elemType(); const tgt_elem = dtype.toMlir(self.ctx); if (cur_elem.eql(tgt_elem)) return src; @@ -2019,93 +2003,72 @@ pub const Builder = struct { // ==================== SCF regions ==================== + /// `for iv in [lower, upper) step` carrying `inits`. Literal bounds take the + /// type of a Value bound, else `lower` is lifted (`i32` when it fits). pub fn openFor( self: *Builder, lower: anytype, upper: anytype, step: anytype, inits: anytype, - ) ForScope(tupleArity(@TypeOf(inits), "openFor: inits")) { - const N = comptime tupleArity(@TypeOf(inits), "openFor: inits"); - const info = @typeInfo(@TypeOf(inits)).@"struct"; - - const lb_v: Value = if (@TypeOf(lower) == Value) lower else self.lift(lower); - const ub_v: Value = if (@TypeOf(upper) == Value) upper else self.constMatching(upper, lb_v.type_()); - const step_v: Value = if (@TypeOf(step) == Value) step else self.constMatching(step, lb_v.type_()); - - var block_types: [N + 1]*const mlir.Type = undefined; - var block_locs: [N + 1]*const mlir.Location = undefined; - block_types[0] = lb_v.type_(); - block_locs[0] = self.loc(); - var inits_inner: [N]*const mlir.Value = undefined; - inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { - const raw = @field(inits, field_name); - const v: Value = if (Field == Value) raw else self.lift(raw); - block_types[i + 1] = v.type_(); - block_locs[i + 1] = self.loc(); - inits_inner[i] = v.inner; - } - - const body = mlir.Block.init(&block_types, &block_locs); - self.pushBlock(body); - - var carried: [N]Value = undefined; - for (0..N) |i| carried[i] = .{ .inner = body.argument(i + 1), .kernel = self }; - - return .{ - .kernel = self, - .body = body, - .lb_inner = lb_v.inner, - .ub_inner = ub_v.inner, - .step_inner = step_v.inner, - .inits_inner = inits_inner, - .iv = .{ .inner = body.argument(0), .kernel = self }, - .carried = carried, - }; + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return self.openForOpts(lower, upper, step, inits, .{}); } - pub fn returnIf(self: *Builder, cond: Value, values: anytype) void { - self.returnIfOpts(cond, values, .{}); + /// `openFor` with `scf.for` options. + pub fn openForOpts( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + opts: common.ForOpts, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return common.openFor(self, Value, common.forBounds(self, Value, lower, upper, step, Builder.lift), inits, opts); } - pub fn returnIfOpts(self: *Builder, cond: Value, values: anytype, opts: ReturnIfOpts) void { + pub fn returnIf(self: *Builder, cond: Value) void { + self.returnIfOpts(cond, .{}); + } + + pub fn returnIfOpts(self: *Builder, cond: Value, opts: ReturnIfOpts) void { const current = self.currentBlock(); const region = current.parentRegion(); const cont_block = mlir.Block.init(&.{}, &.{}); - const info = @typeInfo(@TypeOf(values)).@"struct"; - var ret_operands: [info.field_names.len]*const mlir.Value = undefined; - inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { - if (Field != Value) - @compileError("returnIfOpts: every value must be a Value"); - ret_operands[i] = @field(values, field_name).inner; - } - const ret_target: *mlir.Block = if (opts.inline_return) blk: { const rb = mlir.Block.init(&.{}, &.{}); - _ = ttir.return_(self.ctx, &ret_operands, self.loc()).appendTo(rb); + _ = ttir.return_(self.ctx, &.{}, self.loc()).appendTo(rb); region.appendOwnedBlock(rb); break :blk rb; - } else blk: { - if (self.exit_block == null) { - self.exit_block = mlir.Block.init(&.{}, &.{}); - _ = ttir.return_(self.ctx, &ret_operands, self.loc()).appendTo(self.exit_block.?); - } - break :blk self.exit_block.?; - }; + } else self.exitBlock(); region.appendOwnedBlock(cont_block); _ = cf.cond_br(self.ctx, cond.inner, ret_target, &.{}, cont_block, &.{}, .{}, self.loc()).appendTo(current); + self.continueIn(cont_block); + } - if (self.block_stack.items.len == 0) { - self.pushBlock(cont_block); - } else { - self.block_stack.items[self.block_stack.items.len - 1] = cont_block; + /// The block every non-inline return branches to, which `finish` appends. + fn exitBlock(self: *Builder) *mlir.Block { + if (self.exit_block == null) { + self.exit_block = mlir.Block.init(&.{}, &.{}); + _ = ttir.return_(self.ctx, &.{}, self.loc()).appendTo(self.exit_block.?); } + return self.exit_block.?; + } + + /// Continues in `block`, the fall-through of an early return, in place of + /// the current block. + fn continueIn(self: *Builder, block: *mlir.Block) void { + self.block_stack.replaceTop(block); } pub fn openReturnIf(self: *Builder, cond: Value) ReturnIfScope { + return self.openReturnIfOpts(cond, .{}); + } + + pub fn openReturnIfOpts(self: *Builder, cond: Value, opts: ReturnIfOpts) ReturnIfScope { const current = self.currentBlock(); const region = current.parentRegion(); @@ -2122,107 +2085,37 @@ pub const Builder = struct { .kernel = self, .ret_block = ret_block, .cont_block = cont_block, + .opts = opts, }; } pub fn openIf(self: *Builder, cond: Value) IfOnlyScope { - const then_block = mlir.Block.init(&.{}, &.{}); - self.pushBlock(then_block); - return .{ - .kernel = self, - .cond_inner = cond.inner, - .then_block = then_block, - }; + return common.openIf(self, Value, cond); } - pub fn openIfElse( - self: *Builder, - cond: Value, - result_types: anytype, - ) IfScope(tupleArity(@TypeOf(result_types), "openIfElse: result_types")) { - const N = comptime tupleArity(@TypeOf(result_types), "openIfElse: result_types"); - const info = @typeInfo(@TypeOf(result_types)).@"struct"; - - var types: [N]*const mlir.Type = undefined; - inline for (info.field_names, info.field_types, 0..) |field_name, Field, i| { - if (Field != *const mlir.Type) - @compileError("openIfElse: every result_type must be *const mlir.Type (use k.scalarTy/tensorTy)"); - types[i] = @field(result_types, field_name); - } - - const then_block = mlir.Block.init(&.{}, &.{}); - const else_block = mlir.Block.init(&.{}, &.{}); - self.pushBlock(then_block); - - return .{ - .kernel = self, - .cond_inner = cond.inner, - .then_block = then_block, - .else_block = else_block, - .result_types = types, - }; + pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(common.arity(@TypeOf(result_types), "openIfElse: result_types")) { + return common.openIfElse(self, Value, cond, result_types); } pub fn openWhile( self: *Builder, inits: anytype, after_types: anytype, - ) WhileScope( - tupleArity(@TypeOf(inits), "openWhile: inits"), - tupleArity(@TypeOf(after_types), "openWhile: after_types"), - ) { - const N = comptime tupleArity(@TypeOf(inits), "openWhile: inits"); - const M = comptime tupleArity(@TypeOf(after_types), "openWhile: after_types"); - const init_info = @typeInfo(@TypeOf(inits)).@"struct"; - const type_info = @typeInfo(@TypeOf(after_types)).@"struct"; - - var before_types: [N]*const mlir.Type = undefined; - var before_locs: [N]*const mlir.Location = undefined; - var inits_inner: [N]*const mlir.Value = undefined; - inline for (init_info.field_names, init_info.field_types, 0..) |field_name, Field, i| { - if (Field != Value) - @compileError("openWhile: every init must be a Value"); - const v: Value = @field(inits, field_name); - before_types[i] = v.type_(); - before_locs[i] = self.loc(); - inits_inner[i] = v.inner; - } - const before_block = mlir.Block.init(&before_types, &before_locs); - - var after_tys: [M]*const mlir.Type = undefined; - var after_locs: [M]*const mlir.Location = undefined; - inline for (type_info.field_names, type_info.field_types, 0..) |field_name, Field, i| { - if (Field != *const mlir.Type) - @compileError("openWhile: every after_type must be *const mlir.Type"); - after_tys[i] = @field(after_types, field_name); - after_locs[i] = self.loc(); - } - const after_block = mlir.Block.init(&after_tys, &after_locs); - - self.pushBlock(before_block); - var before_carried: [N]Value = undefined; - for (0..N) |i| before_carried[i] = .{ .inner = before_block.argument(i), .kernel = self }; - - return .{ - .kernel = self, - .before_block = before_block, - .after_block = after_block, - .inits_inner = inits_inner, - .after_types = after_tys, - .before_carried = before_carried, - }; + ) WhileScope(common.arity(@TypeOf(inits), "openWhile: inits"), common.arity(@TypeOf(after_types), "openWhile: after_types")) { + return common.openWhile(self, Value, inits, after_types); } // ==================== finalization ==================== - /// Append `tt.return` with the given results, verify the module, and - /// serialize to a NUL-terminated TTIR string owned by the kernel's - /// allocator (passed to `init`). Caller must `defer allocator.free(ir)`. - pub fn finish(self: *Builder, results: []const Value) FinishError![:0]const u8 { + /// Append the final `tt.return` (kernels return nothing), verify the + /// module, and serialize to a NUL-terminated TTIR string owned by the + /// kernel's allocator (passed to `init`). Caller must + /// `defer allocator.free(ir)`. + pub fn finish(self: *Builder) FinishError![:0]const u8 { // After any `returnIf` call, the entry block already has a // `cf.cond_br` terminator; the fall-through block sits on the stack // as the new insertion point. `currentBlock()` returns that block - // (or `entry_block` if no cf was emitted), so the final terminator + // (the entry block if no cf was emitted), so the final terminator // goes to the correct "tail" block in either case. const current = self.currentBlock(); if (self.exit_block) |exit| { @@ -2230,12 +2123,10 @@ pub const Builder = struct { region.appendOwnedBlock(exit); _ = cf.br(self.ctx, exit, &.{}, self.loc()).appendTo(current); } else { - _ = ttir.return_(self.ctx, self.innerSlice(results), self.loc()).appendTo(current); + _ = ttir.return_(self.ctx, &.{}, self.loc()).appendTo(current); } - if (!self.module.operation().verify()) { - return error.InvalidMlir; - } + try common.verify(self.module, "dsl/triton"); var al: std.Io.Writer.Allocating = .init(self.allocator); defer al.deinit(); diff --git a/platforms/tpu/BUILD.bazel b/platforms/tpu/BUILD.bazel index a61cb7fa..1aab4031 100644 --- a/platforms/tpu/BUILD.bazel +++ b/platforms/tpu/BUILD.bazel @@ -33,7 +33,7 @@ zig_library( tags = ["manual"], visibility = ["//visibility:public"], deps = [ - "//dsl/mosaic_tpu:mosaic_tpu_builder", + "//dsl/mosaic_tpu:builder", "//mlir", ], ) diff --git a/platforms/tpu/ragged_paged.zig b/platforms/tpu/ragged_paged.zig index ad99acfa..4c1c50ed 100644 --- a/platforms/tpu/ragged_paged.zig +++ b/platforms/tpu/ragged_paged.zig @@ -127,7 +127,7 @@ fn maskedStoreLM( const slot_shape = [_]i64{ 1, shape[0], shape[1] }; const val_3d = k.shapeCast(val, &slot_shape); const mask_3d = k.shapeCast(mask, &slot_shape); - k.refStoreOpts(ref, val_3d, &.{ kv_head_idx_idx, k.cIndex(0), k.cIndex(0) }, .{ .mask = mask_3d, .strides = &.{} }); + k.refStoreOpts(ref, val_3d, &.{ kv_head_idx_idx, k.constIndex(0), k.constIndex(0) }, .{ .mask = mask_3d, .strides = &.{} }); } /// `pltpu.store(acc_ref, val, mask=…)` — group=1 so no floor-div. @@ -144,7 +144,7 @@ fn maskedStoreAcc( const ge = k.cmpi(.sge, iota, k.broadcastTo(start, shape)); const lt = k.cmpi(.slt, iota, k.broadcastTo(end, shape)); const mask = k.andi(ge, lt); - k.refStoreOpts(ref, val, &.{ k.cIndex(0), q_head_offset_idx, k.cIndex(0) }, .{ .mask = mask, .strides = &.{} }); + k.refStoreOpts(ref, val, &.{ k.constIndex(0), q_head_offset_idx, k.constIndex(0) }, .{ .mask = mask, .strides = &.{} }); } /// `MultiPageAsyncCopyDescriptor.__init__` — `tpu.memref_slice → squeeze @@ -158,23 +158,23 @@ fn createKvAsyncCopyDescriptors( kv_blk_idx: mtt.Value, // i32 — the block-relative page index buf_idx: mtt.Value, // i32 ) void { - const start_kv_page_idx = k.muli(kv_blk_idx, k.lift(@as(i32, @intCast(cfg.num_kv_pages_per_block)))); + const start_kv_page_idx = k.muli(kv_blk_idx, k.constant(.i32, cfg.num_kv_pages_per_block)); const kv_at_seq = k.scalarLoad(a.kv_lens, &.{k.toIndex(seq_idx)}); const end_kv_page_idx = k.minsi( - k.divsi(k.addi(kv_at_seq, k.lift(@as(i32, @intCast(cfg.page_size - 1)))), k.lift(@as(i32, @intCast(cfg.page_size)))), - k.lift(@as(i32, @intCast(cfg.pages_per_seq))), + k.divsi(k.addi(kv_at_seq, k.constant(.i32, cfg.page_size - 1)), k.constant(.i32, cfg.page_size)), + k.constant(.i32, cfg.pages_per_seq), ); - const heads_start = k.muli(heads_blk_idx, k.lift(@as(i32, @intCast(cfg.numCombinedKvHeadsPerBlk())))); + const heads_start = k.muli(heads_blk_idx, k.constant(.i32, cfg.numCombinedKvHeadsPerBlk())); const within = k.cmpi(.slt, start_kv_page_idx, end_kv_page_idx); - const page_idx_safe = k.select(within, start_kv_page_idx, k.lift(@as(i32, 0))); + const page_idx_safe = k.select(within, start_kv_page_idx, k.constant(.i32, 0)); const page_table = k.scalarLoad(a.page_indices, &.{ k.toIndex(seq_idx), k.toIndex(page_idx_safe) }); - const src0 = k.memRefSlice(a.kv_pages, &.{ k.lift(@as(i32, 0)), k.lift(@as(i32, 0)), heads_start, k.lift(@as(i32, 0)) }, &.{ cfg.total_num_pages, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); - const src1 = k.memRefSlice(src0, &.{ page_table, k.lift(@as(i32, 0)), k.lift(@as(i32, 0)), k.lift(@as(i32, 0)) }, &.{ 1, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const src0 = k.memRefSlice(a.kv_pages, &.{ k.constant(.i32, 0), k.constant(.i32, 0), heads_start, k.constant(.i32, 0) }, &.{ cfg.total_num_pages, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const src1 = k.memRefSlice(src0, &.{ page_table, k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0) }, &.{ 1, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); const src = k.memRefSqueeze(src1, &.{ cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); - const dst0 = k.memRefSlice(a.kv_bufs, &.{ buf_idx, k.lift(@as(i32, 0)), k.lift(@as(i32, 0)), k.lift(@as(i32, 0)), k.lift(@as(i32, 0)) }, &.{ 1, cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const dst0 = k.memRefSlice(a.kv_bufs, &.{ buf_idx, k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0) }, &.{ 1, cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); const dst1 = k.memRefSqueeze(dst0, &.{ cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); const dst = k.memRefSqueeze(dst1, &.{ cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); @@ -193,22 +193,22 @@ fn waitKvDma( kv_blk_idx: mtt.Value, buf_idx: mtt.Value, ) void { - const start_kv_page_idx = k.muli(kv_blk_idx, k.lift(@as(i32, @intCast(cfg.num_kv_pages_per_block)))); + const start_kv_page_idx = k.muli(kv_blk_idx, k.constant(.i32, cfg.num_kv_pages_per_block)); const kv_at_seq = k.scalarLoad(a.kv_lens, &.{k.toIndex(seq_idx)}); const end_kv_page_idx = k.minsi( - k.divsi(k.addi(kv_at_seq, k.lift(@as(i32, @intCast(cfg.page_size - 1)))), k.lift(@as(i32, @intCast(cfg.page_size)))), - k.lift(@as(i32, @intCast(cfg.pages_per_seq))), + k.divsi(k.addi(kv_at_seq, k.constant(.i32, cfg.page_size - 1)), k.constant(.i32, cfg.page_size)), + k.constant(.i32, cfg.pages_per_seq), ); - const heads_start = k.muli(heads_blk_idx, k.lift(@as(i32, @intCast(cfg.numCombinedKvHeadsPerBlk())))); + const heads_start = k.muli(heads_blk_idx, k.constant(.i32, cfg.numCombinedKvHeadsPerBlk())); const within = k.cmpi(.slt, start_kv_page_idx, end_kv_page_idx); - const page_idx_safe = k.select(within, start_kv_page_idx, k.lift(@as(i32, 0))); + const page_idx_safe = k.select(within, start_kv_page_idx, k.constant(.i32, 0)); const page_table = k.scalarLoad(a.page_indices, &.{ k.toIndex(seq_idx), k.toIndex(page_idx_safe) }); - const src0 = k.memRefSlice(a.kv_pages, &.{ k.lift(@as(i32, 0)), k.lift(@as(i32, 0)), heads_start, k.lift(@as(i32, 0)) }, &.{ cfg.total_num_pages, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); - const src1 = k.memRefSlice(src0, &.{ page_table, k.lift(@as(i32, 0)), k.lift(@as(i32, 0)), k.lift(@as(i32, 0)) }, &.{ 1, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const src0 = k.memRefSlice(a.kv_pages, &.{ k.constant(.i32, 0), k.constant(.i32, 0), heads_start, k.constant(.i32, 0) }, &.{ cfg.total_num_pages, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const src1 = k.memRefSlice(src0, &.{ page_table, k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0) }, &.{ 1, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); const src = k.memRefSqueeze(src1, &.{ cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); - const dst0 = k.memRefSlice(a.kv_bufs, &.{ buf_idx, k.lift(@as(i32, 0)), k.lift(@as(i32, 0)), k.lift(@as(i32, 0)), k.lift(@as(i32, 0)) }, &.{ 1, cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const dst0 = k.memRefSlice(a.kv_bufs, &.{ buf_idx, k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0) }, &.{ 1, cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); const dst1 = k.memRefSqueeze(dst0, &.{ cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); const dst = k.memRefSqueeze(dst1, &.{ cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); @@ -235,15 +235,15 @@ fn loadStridedOrStack( ) mtt.Value { if (out_last_dim <= 128) { if (stride == 1) { - return k.vectorLoadShape(ref, &.{ k.cIndex(start_row), k.cIndex(0) }, &.{ out_rows, out_last_dim }); + return k.vectorLoadShape(ref, &.{ k.constIndex(start_row), k.constIndex(0) }, &.{ out_rows, out_last_dim }); } - return k.stridedLoadShape(ref, &.{ k.cIndex(start_row), k.cIndex(0) }, &.{ stride, 1 }, &.{ out_rows, out_last_dim }); + return k.stridedLoadShape(ref, &.{ k.constIndex(start_row), k.constIndex(0) }, &.{ stride, 1 }, &.{ out_rows, out_last_dim }); } const rows_count: usize = @intCast(out_rows); const rows = k.arena.allocator().alloc(mtt.Value, rows_count) catch @panic("loadStridedOrStack OOM"); for (0..rows_count) |i| { const row_idx: i64 = start_row + @as(i64, @intCast(i)) * @as(i64, stride); - rows[i] = k.vectorLoadShape(ref, &.{ k.cIndex(row_idx), k.cIndex(0) }, &.{ 1, out_last_dim }); + rows[i] = k.vectorLoadShape(ref, &.{ k.constIndex(row_idx), k.constIndex(0) }, &.{ 1, out_last_dim }); } return k.concatenate(rows, 0); } @@ -281,7 +281,7 @@ fn stridedLoadKv( const start: i64 = kv_head_chunk_idx * 2; const step: i32 = @intCast(cfg.numCombinedKvHeadsPerBlk()); - const dst0 = k.memRefSlice(a.kv_bufs, &.{ buf_idx, k.lift(@as(i32, 0)), k.lift(@as(i32, 0)), k.lift(@as(i32, 0)), k.lift(@as(i32, 0)) }, &.{ 1, cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const dst0 = k.memRefSlice(a.kv_bufs, &.{ buf_idx, k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0) }, &.{ 1, cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); const dst1 = k.memRefSqueeze(dst0, &.{ cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); const flat = k.memRefReshape(dst1, &.{ cfg.kvFlatRows(), cfg.head_dim }); @@ -306,8 +306,8 @@ fn stridedLoadKv( if (cfg.kv_dtype == .bf16) { const bk = k.shli(b, k.splat(@as(i32, 16), splat_shape, .i32)); const bv = k.andi(b, k.splat(@as(i32, @bitCast(@as(u32, 0xFFFF0000))), splat_shape, .i32)); - const k_v = k.arithTruncf(k.bitcastTo(bk, .f32), cfg.kv_dtype); - const v_v = k.arithTruncf(k.bitcastTo(bv, .f32), cfg.kv_dtype); + const k_v = k.arithTruncf(bk.bitCast(.f32), cfg.kv_dtype); + const v_v = k.arithTruncf(bv.bitCast(.f32), cfg.kv_dtype); const result = k.arena.allocator().alloc(KvPair, 1) catch @panic("stridedLoadKv OOM"); result[0] = .{ .k = k_v, .v = v_v }; return result; @@ -330,8 +330,8 @@ fn stridedLoadKv( const bk_trunc = k.trunci(bk, trunc_dtype); const bv_trunc = k.trunci(bv, trunc_dtype); result[i] = .{ - .k = k.bitcastTo(bk_trunc, cfg.kv_dtype), - .v = k.bitcastTo(bv_trunc, cfg.kv_dtype), + .k = bk_trunc.bitCast(cfg.kv_dtype), + .v = bv_trunc.bitCast(cfg.kv_dtype), }; } return result; @@ -367,7 +367,7 @@ fn flashAttention( const num_q_per_blk: i64 = cfg.numQPerBlk(); const num_q_heads_per_kv_head: i64 = cfg.numQHeadsPerKvHead(); - const kv_len_start = k.muli(kv_blk_idx, k.lift(@as(i32, @intCast(cfg.numKvPerBlk())))); + const kv_len_start = k.muli(kv_blk_idx, k.constant(.i32, cfg.numKvPerBlk())); const kv_mask = k.cmpi( .slt, @@ -387,8 +387,8 @@ fn flashAttention( k.traceStop(); var qk = k.mulf(qk_raw, k.splat(cfg.sm_scale, &.{ flash_q, flash_k }, .f32)); - const store_start = k.maxsi(k.subi(q_start, q_len_start), k.lift(@as(i32, 0))); - const store_end = k.minsi(k.subi(q_end, q_len_start), k.lift(@as(i32, @intCast(num_q_per_blk)))); + const store_start = k.maxsi(k.subi(q_start, q_len_start), k.constant(.i32, 0)); + const store_end = k.minsi(k.subi(q_end, q_len_start), k.constant(.i32, num_q_per_blk)); const row_offset = k.subi(k.addi(k.subi(kv_len, q_len), q_len_start), q_start); const row_iota = k.iota(&.{ flash_q, flash_k }, .i32, &.{0}); @@ -439,17 +439,17 @@ fn flashAttention( const l_curr_v = k.multiReduction(.add, s_curr, k.zeros(&.{flash_q}, .f32), &.{1}); const l_curr_bc = k.broadcastTo(k.shapeCast(l_curr_v, &.{ flash_q, 1 }), &.{ flash_q, lm_lane }); - const head_l_ref_idx = k.cIndex(kv_head_idx); + const head_l_ref_idx = k.constIndex(kv_head_idx); const m_prev_load = k.shapeCast( - k.vectorLoadShape(a.m_ref, &.{ head_l_ref_idx, k.cIndex(0), k.cIndex(0) }, &.{ 1, flash_q, lm_lane }), + k.vectorLoadShape(a.m_ref, &.{ head_l_ref_idx, k.constIndex(0), k.constIndex(0) }, &.{ 1, flash_q, lm_lane }), &.{ flash_q, lm_lane }, ); - const m_prev = k.select(k.cmpi(.eq, kv_blk_idx, k.lift(@as(i32, 0))), k.splat(@as(f32, -std.math.inf(f32)), &.{ flash_q, lm_lane }, .f32), m_prev_load); + const m_prev = k.select(k.cmpi(.eq, kv_blk_idx, k.constant(.i32, 0)), k.splat(@as(f32, -std.math.inf(f32)), &.{ flash_q, lm_lane }, .f32), m_prev_load); const l_prev_load = k.shapeCast( - k.vectorLoadShape(a.l_ref, &.{ head_l_ref_idx, k.cIndex(0), k.cIndex(0) }, &.{ 1, flash_q, lm_lane }), + k.vectorLoadShape(a.l_ref, &.{ head_l_ref_idx, k.constIndex(0), k.constIndex(0) }, &.{ 1, flash_q, lm_lane }), &.{ flash_q, lm_lane }, ); - const l_prev = k.select(k.cmpi(.eq, kv_blk_idx, k.lift(@as(i32, 0))), k.zeros(&.{ flash_q, lm_lane }, .f32), l_prev_load); + const l_prev = k.select(k.cmpi(.eq, kv_blk_idx, k.constant(.i32, 0)), k.zeros(&.{ flash_q, lm_lane }, .f32), l_prev_load); const m_next = k.maximumf(m_prev, m_curr_bc); maskedStoreLM(k, a.m_ref, m_next, head_l_ref_idx, store_start, store_end, num_q_heads_per_kv_head); @@ -465,10 +465,10 @@ fn flashAttention( ); maskedStoreLM(k, a.l_ref, l_next_safe, head_l_ref_idx, store_start, store_end, num_q_heads_per_kv_head); - const q_head_idx_idx = k.cIndex(q_head_idx); - const o_curr_3d = k.vectorLoadShape(a.acc_ref, &.{ k.cIndex(0), q_head_idx_idx, k.cIndex(0) }, &.{ num_q_per_blk, num_q_heads_per_kv_head, head_dim }); + const q_head_idx_idx = k.constIndex(q_head_idx); + const o_curr_3d = k.vectorLoadShape(a.acc_ref, &.{ k.constIndex(0), q_head_idx_idx, k.constIndex(0) }, &.{ num_q_per_blk, num_q_heads_per_kv_head, head_dim }); const o_curr_pre = k.select( - k.cmpi(.eq, kv_blk_idx, k.lift(@as(i32, 0))), + k.cmpi(.eq, kv_blk_idx, k.constant(.i32, 0)), k.zeros(&.{ num_q_per_blk, num_q_heads_per_kv_head, head_dim }, .f32), o_curr_3d, ); @@ -539,27 +539,27 @@ pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { const k = b; - const num_seqs = k.scalarLoad(a.num_seqs, &.{k.cIndex(0)}); - const init_seq_idx = k.scalarLoad(a.seq_buf_idx, &.{k.cIndex(0)}); - const init_buf_idx = k.scalarLoad(a.seq_buf_idx, &.{k.cIndex(1)}); - const q_len_start = k.muli(a.q_blk_idx, k.lift(@as(i32, @intCast(cfg.numQPerBlk())))); - const q_len_end = k.addi(q_len_start, k.lift(@as(i32, @intCast(cfg.numQPerBlk())))); + const num_seqs = k.scalarLoad(a.num_seqs, &.{k.constIndex(0)}); + const init_seq_idx = k.scalarLoad(a.seq_buf_idx, &.{k.constIndex(0)}); + const init_buf_idx = k.scalarLoad(a.seq_buf_idx, &.{k.constIndex(1)}); + const q_len_start = k.muli(a.q_blk_idx, k.constant(.i32, cfg.numQPerBlk())); + const q_len_end = k.addi(q_len_start, k.constant(.i32, cfg.numQPerBlk())); + var i = k.openIf(k.cmpi(.eq, k.addi(a.heads_blk_idx, a.q_blk_idx), k.constant(.i32, 0))); { - var i = k.openIf(k.cmpi(.eq, k.addi(a.heads_blk_idx, a.q_blk_idx), k.lift(@as(i32, 0)))); const first_seq_idx = nextNonemptyKvSequence(k, cfg, a, init_seq_idx, num_seqs); + var nonempty = k.openIf(k.cmpi(.slt, first_seq_idx, num_seqs)); { - var nonempty = k.openIf(k.cmpi(.slt, first_seq_idx, num_seqs)); - createKvAsyncCopyDescriptors(k, cfg, a, a.heads_blk_idx, first_seq_idx, k.lift(@as(i32, 0)), init_buf_idx); - nonempty.yieldThen(.{}); + createKvAsyncCopyDescriptors(k, cfg, a, a.heads_blk_idx, first_seq_idx, k.constant(.i32, 0), init_buf_idx); + nonempty.close(); } - i.yieldThen(.{}); + i.close(); } // Empty rows skip flashAttention, so their accumulator must already be zero. - k.vectorStoreAt(a.acc_ref, k.zeros(&.{ cfg.numQPerBlk(), cfg.numQHeadsPerBlk(), cfg.head_dim }, .f32), &.{ k.cIndex(0), k.cIndex(0), k.cIndex(0) }); + k.vectorStoreAt(a.acc_ref, k.zeros(&.{ cfg.numQPerBlk(), cfg.numQHeadsPerBlk(), cfg.head_dim }, .f32), &.{ k.constIndex(0), k.constIndex(0), k.constIndex(0) }); - var q_loop = k.openWhile(.{ k.lift(@as(i32, 0)), init_seq_idx, init_buf_idx }, .{ k.scalarTy(.i32), k.scalarTy(.i32), k.scalarTy(.i32) }); + var q_loop = k.openWhile(.{ k.constant(.i32, 0), init_seq_idx, init_buf_idx }, .{ .i32, .i32, .i32 }); { const done = q_loop.before_carried[0]; const cur_seq_idx = q_loop.before_carried[1]; @@ -569,7 +569,7 @@ pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { k.cmpi(.slt, q_len_start, cu_at_num_seqs), k.cmpi(.slt, cur_seq_idx, num_seqs), ); - const cond = k.andi(k.cmpi(.eq, done, k.lift(@as(i32, 0))), should_run); + const cond = k.andi(k.cmpi(.eq, done, k.constant(.i32, 0)), should_run); q_loop.yieldBefore(cond, .{ done, cur_seq_idx, cur_buf_idx }); } { @@ -578,30 +578,30 @@ pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { const cur_buf_idx = q_loop.after_carried[2]; const q_start = k.scalarLoad(a.cu_q_lens, &.{k.toIndex(cur_seq_idx)}); - const q_end = k.scalarLoad(a.cu_q_lens, &.{k.toIndex(k.addi(cur_seq_idx, k.lift(@as(i32, 1))))}); + const q_end = k.scalarLoad(a.cu_q_lens, &.{k.toIndex(k.addi(cur_seq_idx, k.constant(.i32, 1)))}); const q_len = k.subi(q_end, q_start); const kv_len = k.scalarLoad(a.kv_lens, &.{k.toIndex(cur_seq_idx)}); - var kv_loop = k.openWhile(.{ k.lift(@as(i32, 0)), cur_buf_idx }, .{ k.scalarTy(.i32), k.scalarTy(.i32) }); + var kv_loop = k.openWhile(.{ k.constant(.i32, 0), cur_buf_idx }, .{ .i32, .i32 }); { const kv_blk = kv_loop.before_carried[0]; const buf = kv_loop.before_carried[1]; - const cond = k.cmpi(.slt, k.muli(kv_blk, k.lift(@as(i32, @intCast(cfg.numKvPerBlk())))), kv_len); + const cond = k.cmpi(.slt, k.muli(kv_blk, k.constant(.i32, cfg.numKvPerBlk())), kv_len); kv_loop.yieldBefore(cond, .{ kv_blk, buf }); } { const kv_blk = kv_loop.after_carried[0]; const cur_buf = kv_loop.after_carried[1]; - const next_kv_blk_raw = k.addi(kv_blk, k.lift(@as(i32, 1))); - const is_last_kv = k.cmpi(.sge, k.muli(next_kv_blk_raw, k.lift(@as(i32, @intCast(cfg.numKvPerBlk())))), kv_len); - const next_kv_blk = k.select(is_last_kv, k.lift(@as(i32, 0)), next_kv_blk_raw); + const next_kv_blk_raw = k.addi(kv_blk, k.constant(.i32, 1)); + const is_last_kv = k.cmpi(.sge, k.muli(next_kv_blk_raw, k.constant(.i32, cfg.numKvPerBlk())), kv_len); + const next_kv_blk = k.select(is_last_kv, k.constant(.i32, 0), next_kv_blk_raw); const cur_seq_end = k.cmpi(.sle, q_end, q_len_end); const next_nonempty_seq = blk: { // Only sequence transitions need to scan for empty KV rows. - var next_seq = k.openIfElse(k.andi(is_last_kv, cur_seq_end), .{k.scalarTy(.i32)}); + var next_seq = k.openIfElse(k.andi(is_last_kv, cur_seq_end), .{.i32}); { - const start_seq_idx = k.addi(cur_seq_idx, k.lift(@as(i32, 1))); + const start_seq_idx = k.addi(cur_seq_idx, k.constant(.i32, 1)); next_seq.yieldThen(.{nextNonemptyKvSequence(k, cfg, a, start_seq_idx, num_seqs)}); } { @@ -610,14 +610,14 @@ pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { break :blk next_seq.results[0]; }; const is_last_seq = k.cmpi(.eq, next_nonempty_seq, num_seqs); - const next_seq_idx = k.select(is_last_seq, k.lift(@as(i32, 0)), next_nonempty_seq); - const next_heads_blk_idx = k.select(is_last_seq, k.addi(a.heads_blk_idx, k.lift(@as(i32, 1))), a.heads_blk_idx); - const next_buf = k.extui(k.cmpi(.eq, cur_buf, k.lift(@as(i32, 0))), .i32); + const next_seq_idx = k.select(is_last_seq, k.constant(.i32, 0), next_nonempty_seq); + const next_heads_blk_idx = k.select(is_last_seq, k.addi(a.heads_blk_idx, k.constant(.i32, 1)), a.heads_blk_idx); + const next_buf = k.extui(k.cmpi(.eq, cur_buf, k.constant(.i32, 0)), .i32); + var ii = k.openIf(k.cmpi(.slt, next_heads_blk_idx, k.constant(.i32, cfg.numHeadsBlks()))); { - var ii = k.openIf(k.cmpi(.slt, next_heads_blk_idx, k.lift(@as(i32, @intCast(cfg.numHeadsBlks()))))); createKvAsyncCopyDescriptors(k, cfg, a, next_heads_blk_idx, next_seq_idx, next_kv_blk, next_buf); - ii.yieldThen(.{}); + ii.close(); } waitKvDma(k, cfg, a, a.heads_blk_idx, cur_seq_idx, kv_blk, cur_buf); @@ -651,7 +651,7 @@ pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { // 2nd-to-last dim is not a multiple of q's packing // factor, cast q to f32 before the trailing-dim reshape // so the layout pass can fold it cleanly. - var q_block = k.vectorLoadShape(a.q, &.{ k.cIndex(0), k.cIndex(q_head_idx), k.cIndex(0) }, &.{ cfg.numQPerBlk(), cfg.numQHeadsPerKvHead(), cfg.head_dim }); + var q_block = k.vectorLoadShape(a.q, &.{ k.constIndex(0), k.constIndex(q_head_idx), k.constIndex(0) }, &.{ cfg.numQPerBlk(), cfg.numQHeadsPerKvHead(), cfg.head_dim }); if (@mod(cfg.numQHeadsPerKvHead(), cfg.qPackingFactor()) != 0) { q_block = q_block.to(.f32); } @@ -660,21 +660,21 @@ pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { } } - kv_loop.yieldAfter(.{ k.addi(kv_blk, k.lift(@as(i32, 1))), next_buf }); + kv_loop.yieldAfter(.{ k.addi(kv_blk, k.constant(.i32, 1)), next_buf }); } const next_buf_idx = kv_loop.results[1]; - const next_seq_idx_outer = k.select(k.cmpi(.sle, q_end, q_len_end), k.addi(cur_seq_idx, k.lift(@as(i32, 1))), cur_seq_idx); - const next_done = k.select(k.cmpi(.slt, q_end, q_len_end), done, k.lift(@as(i32, 1))); + const next_seq_idx_outer = k.select(k.cmpi(.sle, q_end, q_len_end), k.addi(cur_seq_idx, k.constant(.i32, 1)), cur_seq_idx); + const next_done = k.select(k.cmpi(.slt, q_end, q_len_end), done, k.constant(.i32, 1)); q_loop.yieldAfter(.{ next_done, next_seq_idx_outer, next_buf_idx }); } const seq_idx_out = q_loop.results[1]; const buf_idx_out = q_loop.results[2]; - k.scalarStore(a.seq_buf_idx, k.select(k.cmpi(.slt, seq_idx_out, num_seqs), seq_idx_out, k.lift(@as(i32, 0))), &.{k.cIndex(0)}); - k.scalarStore(a.seq_buf_idx, buf_idx_out, &.{k.cIndex(1)}); + k.scalarStore(a.seq_buf_idx, k.select(k.cmpi(.slt, seq_idx_out, num_seqs), seq_idx_out, k.constant(.i32, 0)), &.{k.constIndex(0)}); + k.scalarStore(a.seq_buf_idx, buf_idx_out, &.{k.constIndex(1)}); - k.vectorStoreAt(a.o, k.refLoad(a.acc_ref).to(cfg.q_dtype), &.{ k.cIndex(0), k.cIndex(0), k.cIndex(0) }); + k.vectorStoreAt(a.o, k.refLoad(a.acc_ref).to(cfg.q_dtype), &.{ k.constIndex(0), k.constIndex(0), k.constIndex(0) }); } // Fixed drafter blocks can have query tokens without visible KV tokens. @@ -686,20 +686,20 @@ fn nextNonemptyKvSequence( start_seq_idx: mtt.Value, num_seqs: mtt.Value, ) mtt.Value { - var loop = k.openWhile(.{start_seq_idx}, .{k.scalarTy(.i32)}); + var loop = k.openWhile(.{start_seq_idx}, .{.i32}); { const seq_idx = loop.before_carried[0]; // The terminal index can equal the metadata array length. - const safe_seq_idx = k.minsi(seq_idx, k.lift(@as(i32, @intCast(cfg.max_num_seqs - 1)))); + const safe_seq_idx = k.minsi(seq_idx, k.constant(.i32, cfg.max_num_seqs - 1)); const kv_len = k.scalarLoad(a.kv_lens, &.{k.toIndex(safe_seq_idx)}); const skip = k.andi( k.cmpi(.slt, seq_idx, num_seqs), - k.cmpi(.sle, kv_len, k.lift(@as(i32, 0))), + k.cmpi(.sle, kv_len, k.constant(.i32, 0)), ); loop.yieldBefore(skip, .{seq_idx}); } { - loop.yieldAfter(.{k.addi(loop.after_carried[0], k.lift(@as(i32, 1)))}); + loop.yieldAfter(.{k.addi(loop.after_carried[0], k.constant(.i32, 1))}); } return loop.results[0]; } diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index d959df5e..66d5422e 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -147,12 +147,12 @@ zig_library( ":xla_compile_options_upb", ":xla_data_upb", "//bazel", - "//dsl/cuda_tile:cuda_tile_builder", - "//dsl/cute:cute_builder", - "//dsl/fly:fly_builder", - "//dsl/mosaic_tpu:mosaic_tpu_builder", - "//dsl/tcl:tcl_builder", - "//dsl/triton:triton_builder", + "//dsl/cuda_tile:builder", + "//dsl/cute:builder", + "//dsl/fly:builder", + "//dsl/mosaic_tpu:builder", + "//dsl/tcl:builder", + "//dsl/triton:builder", "//ffi:zig", "//mlir", "//mlir/dialects/func", diff --git a/zml/kernel.zig b/zml/kernel.zig index e60736c9..d8601760 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -129,7 +129,7 @@ pub const triton = struct { try spec.run(&b, cfg); - return b.finish(&.{}); + return b.finish(); } pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { @@ -1269,8 +1269,10 @@ test "cute kernel emits the module the CuTe compiler takes" { }); const i = b.blockIdx().x.mul(b.blockDim().x).add(b.threadIdx().x); var guard = b.openIf(i.lt(cfg.n)); - a.out.set(.{i}, a.x.get(.{i}).add(1.0)); - guard.yieldThen(.{}); + { + a.out.set(.{i}, a.x.get(.{i}).add(1.0)); + guard.close(); + } } }.run, }); diff --git a/zml/kernels/flash_attention/cute/attention.zig b/zml/kernels/flash_attention/cute/attention.zig index 396537fe..0d01c357 100644 --- a/zml/kernels/flash_attention/cute/attention.zig +++ b/zml/kernels/flash_attention/cute/attention.zig @@ -119,12 +119,6 @@ fn run(b: *B, cfg: Config) cute.FinishError!void { unreachable; } -fn tupleOf(array: anytype) @Tuple(&@as([array.len]type, @splat(Value))) { - var result: @Tuple(&@as([array.len]type, @splat(Value))) = undefined; - inline for (0..array.len) |i| result[i] = array[i]; - return result; -} - /// Four warps own four (query position, query head) rows sharing one KV head. /// Only a KV tile lives in shared memory; maxima, sums, and outputs stay in /// registers. All lanes participate in shuffles, including padded rows/features. @@ -155,17 +149,21 @@ fn runSized(b: *B, cfg: Config, comptime fragments: usize) cute.FinishError!void // Partition the live cache, rather than its capacity, so decode uses all // blocks even when only a prefix of the cache is populated. const chunk = causal_end.add(cfg.splits - 1).div(cfg.splits); - const begin = if (split) b.blockIdx().z.mul(chunk) else b.cst(.i32, 0); + const begin = if (split) b.blockIdx().z.mul(chunk) else b.constant(.i32, 0); const end = if (split) begin.add(chunk).minimum(causal_end) else causal_end; - const zero = b.cst(.f32, 0); - const neg_inf = b.cst(.f32, -std.math.inf(f32)); + const zero = b.constant(.f32, 0); + const neg_inf = b.constant(.f32, -std.math.inf(f32)); var queries: [fragments]Value = undefined; inline for (0..fragments) |f| { const d = lane.add(f * 32); - var valid = b.openIfElse(d.lt(cfg.head_dim), .{cute.DType.f32.toMlir(b.ctx)}); - valid.yieldThen(.{a.q.get(.{ batch, query, head, d }).to(.f32)}); - valid.yieldElse(.{zero}); + var valid = b.openIfElse(d.lt(cfg.head_dim), .{.f32}); + { + valid.yieldThen(.{a.q.get(.{ batch, query, head, d }).to(.f32)}); + } + { + valid.yieldElse(.{zero}); + } queries[f] = valid.results[0]; } @@ -174,71 +172,87 @@ fn runSized(b: *B, cfg: Config, comptime fragments: usize) cute.FinishError!void const values = b.makeTensor(b.allocSmem(cfg.dtype, tile_keys * padded_dim, 16), layout); var initial: [fragments + 2]Value = @splat(zero); initial[0] = neg_inf; - var tiles = b.openFor(begin, end, tile_keys, tupleOf(initial)); + var tiles = b.openFor(begin, end, tile_keys, initial); + { - // Zero-fill partial tiles and feature padding before the CTA-wide barrier. - var copy = b.openFor(tid, tile_keys * padded_dim, 128, .{}); - const tile_key = copy.iv.div(padded_dim); - const d = copy.iv.rem(padded_dim); - const key = tiles.iv.add(tile_key); - var valid_load = b.openIfElse(key.lt(end).bitAnd(d.lt(cfg.head_dim)), .{ cfg.dtype.toMlir(b.ctx), cfg.dtype.toMlir(b.ctx) }); - valid_load.yieldThen(.{ a.k.get(.{ batch, key, kv_head, d }), a.v.get(.{ batch, key, kv_head, d }) }); - valid_load.yieldElse(.{ zero.to(cfg.dtype), zero.to(cfg.dtype) }); - keys.set(.{ tile_key, d }, valid_load.results[0]); - values.set(.{ tile_key, d }, valid_load.results[1]); - copy.yield(.{}); - b.syncThreads(); + // Zero-fill partial tiles and feature padding before the CTA-wide barrier. + var copy = b.openFor(tid, tile_keys * padded_dim, 128, .{}); + { + const tile_key = copy.iv.div(padded_dim); + const d = copy.iv.rem(padded_dim); + const key = tiles.iv.add(tile_key); + var valid_load = b.openIfElse(key.lt(end).bitAnd(d.lt(cfg.head_dim)), .{ cfg.dtype, cfg.dtype }); + { + valid_load.yieldThen(.{ a.k.get(.{ batch, key, kv_head, d }), a.v.get(.{ batch, key, kv_head, d }) }); + } + { + valid_load.yieldElse(.{ zero.to(cfg.dtype), zero.to(cfg.dtype) }); + } + keys.set(.{ tile_key, d }, valid_load.results[0]); + values.set(.{ tile_key, d }, valid_load.results[1]); + copy.close(); + } + b.syncThreads(); - // Each lane keeps one score; dot products reduce across the feature lanes. - var scores = b.openFor(b.cst(.i32, 0), tile_keys, 1, .{neg_inf}); - var dot = zero; - inline for (0..fragments) |f| { - dot = dot.add(queries[f].mul(keys.get(.{ scores.iv, lane.add(f * 32) }).to(.f32))); + // Each lane keeps one score; dot products reduce across the feature lanes. + var scores = b.openFor(0, tile_keys, 1, .{neg_inf}); + { + var dot = zero; + inline for (0..fragments) |f| { + dot = dot.add(queries[f].mul(keys.get(.{ scores.iv, lane.add(f * 32) }).to(.f32))); + } + inline for (.{ 16, 8, 4, 2, 1 }) |offset| dot = dot.add(b.shuffleXor(dot, offset)); + const scale = std.math.log2e / std.math.sqrt(@as(f32, @floatFromInt(cfg.head_dim))); + const score = b.select(tiles.iv.add(scores.iv).le(position).bitAnd(tiles.iv.add(scores.iv).lt(end)), dot.mul(scale), neg_inf); + scores.yield(.{b.select(lane.eq(scores.iv), score, scores.carried[0])}); + } + var maximum = scores.results[0]; + inline for (.{ 16, 8, 4, 2, 1 }) |offset| maximum = maximum.maximum(b.shuffleXor(maximum, offset)); + maximum = maximum.maximum(tiles.carried[0]); + const rescale = b.exp2Approx(tiles.carried[0].sub(maximum)); + const probability = b.exp2Approx(scores.results[0].sub(maximum)); + var sum = probability; + inline for (.{ 16, 8, 4, 2, 1 }) |offset| sum = sum.add(b.shuffleXor(sum, offset)); + sum = sum.add(tiles.carried[1].mul(rescale)); + + var accumulators: [fragments]Value = undefined; + inline for (0..fragments) |f| accumulators[f] = tiles.carried[f + 2].mul(rescale); + var weighted = b.openFor(0, tile_keys, 1, accumulators); + { + const weight = b.shuffleIdx(probability, weighted.iv); + inline for (0..fragments) |f| { + accumulators[f] = weighted.carried[f].add(weight.mul(values.get(.{ weighted.iv, lane.add(f * 32) }).to(.f32))); + } + weighted.yield(accumulators); + } + // No warp may overwrite the tile before every warp has consumed it. + b.syncThreads(); + var next: [fragments + 2]Value = undefined; + next[0] = maximum; + next[1] = sum; + inline for (0..fragments) |f| next[f + 2] = weighted.results[f]; + tiles.yield(next); } - inline for (.{ 16, 8, 4, 2, 1 }) |offset| dot = dot.add(b.shuffleXor(dot, offset)); - const scale = std.math.log2e / std.math.sqrt(@as(f32, @floatFromInt(cfg.head_dim))); - const score = b.select(tiles.iv.add(scores.iv).le(position).bitAnd(tiles.iv.add(scores.iv).lt(end)), dot.mul(scale), neg_inf); - scores.yield(.{b.select(lane.eq(scores.iv), score, scores.carried[0])}); - var maximum = scores.results[0]; - inline for (.{ 16, 8, 4, 2, 1 }) |offset| maximum = maximum.maximum(b.shuffleXor(maximum, offset)); - maximum = maximum.maximum(tiles.carried[0]); - const rescale = b.exp2Approx(tiles.carried[0].sub(maximum)); - const probability = b.exp2Approx(scores.results[0].sub(maximum)); - var sum = probability; - inline for (.{ 16, 8, 4, 2, 1 }) |offset| sum = sum.add(b.shuffleXor(sum, offset)); - sum = sum.add(tiles.carried[1].mul(rescale)); - - var accumulators: [fragments]Value = undefined; - inline for (0..fragments) |f| accumulators[f] = tiles.carried[f + 2].mul(rescale); - var weighted = b.openFor(b.cst(.i32, 0), tile_keys, 1, tupleOf(accumulators)); - const weight = b.shuffleIdx(probability, weighted.iv); - inline for (0..fragments) |f| { - accumulators[f] = weighted.carried[f].add(weight.mul(values.get(.{ weighted.iv, lane.add(f * 32) }).to(.f32))); - } - weighted.yield(tupleOf(accumulators)); - // No warp may overwrite the tile before every warp has consumed it. - b.syncThreads(); - var next: [fragments + 2]Value = undefined; - next[0] = maximum; - next[1] = sum; - inline for (0..fragments) |f| next[f + 2] = weighted.results[f]; - tiles.yield(tupleOf(next)); inline for (0..fragments) |f| { const col = lane.add(f * 32); var valid_store = b.openIf(row.lt(cfg.query_length * group).bitAnd(col.lt(cfg.head_dim))); - if (split) { - a.out.set(.{ batch, b.blockIdx().z, query, head, col }, tiles.results[f + 2]); - } else { - a.out.set(.{ batch, query, head, col }, tiles.results[f + 2].div(tiles.results[1])); + { + if (split) { + a.out.set(.{ batch, b.blockIdx().z, query, head, col }, tiles.results[f + 2]); + } else { + a.out.set(.{ batch, query, head, col }, tiles.results[f + 2].div(tiles.results[1])); + } + valid_store.close(); } - valid_store.yieldThen(.{}); } if (split) { var stats = b.openIf(row.lt(cfg.query_length * group).bitAnd(lane.eq(0))); - a.out.set(.{ batch, b.blockIdx().z, query, head, cfg.head_dim }, tiles.results[0]); - a.out.set(.{ batch, b.blockIdx().z, query, head, cfg.head_dim + 1 }, tiles.results[1]); - stats.yieldThen(.{}); + { + a.out.set(.{ batch, b.blockIdx().z, query, head, cfg.head_dim }, tiles.results[0]); + a.out.set(.{ batch, b.blockIdx().z, query, head, cfg.head_dim + 1 }, tiles.results[1]); + stats.close(); + } } } @@ -259,28 +273,36 @@ fn merge(b: *B, cfg: Config) cute.FinishError!void { const batch = b.blockIdx().y; const head = b.blockIdx().x; const lane = b.threadIdx().x.rem(32); - const zero = b.cst(.f32, 0); - const neg_inf = b.cst(.f32, -std.math.inf(f32)); + const zero = b.constant(.f32, 0); + const neg_inf = b.constant(.f32, -std.math.inf(f32)); // Every warp reduces the partition statistics, then independently handles // a contiguous range of output features. var maxima = b.openFor(lane, cfg.splits, 32, .{neg_inf}); - maxima.yield(.{maxima.carried[0].maximum(a.partials.get(.{ batch, maxima.iv, head, cfg.head_dim }))}); + { + maxima.yield(.{maxima.carried[0].maximum(a.partials.get(.{ batch, maxima.iv, head, cfg.head_dim }))}); + } var maximum = maxima.results[0]; inline for (.{ 16, 8, 4, 2, 1 }) |offset| maximum = maximum.maximum(b.shuffleXor(maximum, offset)); const weights = b.makeTensor(b.allocSmem(.f32, cfg.splits, 16), b.makeLayout(&.{cfg.splits}, .{})); var stats = b.openFor(b.threadIdx().x, cfg.splits, 128, .{}); - weights.set(.{stats.iv}, b.exp2Approx(a.partials.get(.{ batch, stats.iv, head, cfg.head_dim }).sub(maximum))); - stats.yield(.{}); + { + weights.set(.{stats.iv}, b.exp2Approx(a.partials.get(.{ batch, stats.iv, head, cfg.head_dim }).sub(maximum))); + stats.close(); + } b.syncThreads(); var features = b.openFor(b.threadIdx().x, cfg.head_dim, 128, .{}); - var sum = b.openFor(b.cst(.i32, 0), cfg.splits, 1, .{ zero, zero }); - const weight = weights.get(.{sum.iv}); - sum.yield(.{ - sum.carried[0].add(weight.mul(a.partials.get(.{ batch, sum.iv, head, features.iv }))), - sum.carried[1].add(weight.mul(a.partials.get(.{ batch, sum.iv, head, cfg.head_dim + 1 }))), - }); - a.out.set(.{ batch, head, features.iv }, sum.results[0].div(sum.results[1])); - features.yield(.{}); + { + var sum = b.openFor(0, cfg.splits, 1, .{ zero, zero }); + { + const weight = weights.get(.{sum.iv}); + sum.yield(.{ + sum.carried[0].add(weight.mul(a.partials.get(.{ batch, sum.iv, head, features.iv }))), + sum.carried[1].add(weight.mul(a.partials.get(.{ batch, sum.iv, head, cfg.head_dim + 1 }))), + }); + } + a.out.set(.{ batch, head, features.iv }, sum.results[0].div(sum.results[1])); + features.close(); + } } test "CuTe attention emits tiled online softmax through head dimension 512" { diff --git a/zml/kernels/moe/cute/mxfp4.zig b/zml/kernels/moe/cute/mxfp4.zig index caffea33..d9ab0324 100644 --- a/zml/kernels/moe/cute/mxfp4.zig +++ b/zml/kernels/moe/cute/mxfp4.zig @@ -48,10 +48,10 @@ const combine_kernel_name = "mxfp4_cute_combine_kernel"; /// Host launch of a boundary kernel: forward the buffer pointers, with PDL. fn launchBoundary(b: *B, kernel_name: [:0]const u8, grid: [3]i32, args: anytype) cute.FinishError!void { - const one = b.cst(.i32, 1); + const one = b.constant(.i32, 1); const config = b.makeLaunchConfig(.{ - .grid = .{ b.cst(.i32, grid[0]), b.cst(.i32, grid[1]), b.cst(.i32, grid[2]) }, - .block = .{ b.cst(.i32, 128), one, one }, + .grid = .{ b.constant(.i32, grid[0]), b.constant(.i32, grid[1]), b.constant(.i32, grid[2]) }, + .block = .{ b.constant(.i32, 128), one, one }, .dynamic_smem = b.kernelSmemSize(kernel_name), .stream = b.cudaStream(), .cluster = .{ one, one, one }, @@ -140,7 +140,7 @@ fn groupedQuantizeKernel(b: *B, cfg: GroupedQuantConfig) cute.FinishError!void { const token = b.blockIdx().y; const word = b.blockIdx().x.mul(128).add(tid); var x: [4]cute.Value = undefined; - for (&x, 0..) |*value, i| value.* = a.x.get(.{ token, word.mul(4).add(@as(i32, @intCast(i))) }).to(.f32); + for (&x, 0..) |*value, i| value.* = a.x.get(.{ token, word.mul(4).add(i) }).to(.f32); var maximum = x[0].abs().maximum(x[1].abs()).maximum(x[2].abs().maximum(x[3].abs())); inline for (.{ 1, 2, 4 }) |offset| { @@ -148,7 +148,7 @@ fn groupedQuantizeKernel(b: *B, cfg: GroupedQuantConfig) cute.FinishError!void { } const raw_bits = maximum.maximum(1e-4).mul(1.0 / 448.0).bitCast(.i32); const exponent = raw_bits.shrLogical(23).add(raw_bits.bitAnd(0x7fffff).ne(0).to(.i32)); - const inverse_scale = b.cst(.i32, 254).sub(exponent).shl(23).bitCast(.f32); + const inverse_scale = b.constant(.i32, 254).sub(exponent).shl(23).bitCast(.f32); const packed_word = b.packFp8x4(x[0].mul(inverse_scale), x[1].mul(inverse_scale), x[2].mul(inverse_scale), x[3].mul(inverse_scale)); const scale_col = word.div(8); @@ -160,15 +160,17 @@ fn groupedQuantizeKernel(b: *B, cfg: GroupedQuantConfig) cute.FinishError!void { a.q.set(.{ row, word }, packed_word); var leader = b.openIf(tid.rem(8).eq(0)); - const scale_row = row.div(cfg.group_size); - const row_in_group = row.rem(cfg.group_size); - const swizzled_col = scale_col.div(4).mul(512) - .add(row_in_group.rem(32).mul(16)) - .add(row_in_group.div(32).mul(4)) - .add(scale_col.rem(4)); - a.s.set(.{ scale_row, swizzled_col }, exponent.to(.i8)); - leader.yieldThen(.{}); - if (scheduled) |*s| s.yieldThen(.{}); + { + const scale_row = row.div(cfg.group_size); + const row_in_group = row.rem(cfg.group_size); + const swizzled_col = scale_col.div(4).mul(512) + .add(row_in_group.rem(32).mul(16)) + .add(row_in_group.div(32).mul(4)) + .add(scale_col.rem(4)); + a.s.set(.{ scale_row, swizzled_col }, exponent.to(.i8)); + leader.close(); + } + if (scheduled) |*s| s.close(); } // Only now are the rows this CTA owns complete: release the persistent // GEMM, which waits on this signal before its first TMA read. @@ -213,25 +215,31 @@ fn combineKernel(b: *B, cfg: CombineConfig) cute.FinishError!void { for (0..@intCast(@divExact(pairs_per_cta, 128))) |chunk| { const pair = first_pair.add(chunk * 128); var in_bounds = b.openIf(pair.lt(pairs)); - var low = b.cst(.f32, 0); - var high = b.cst(.f32, 0); - for (0..@intCast(cfg.topk)) |slot| { - const route = token.mul(cfg.topk).add(slot); - // Routes of other expert-parallel ranks were never written. - var computed = b.openIfElse(a.routes.get(.{route}).ge(0), .{ cute.DType.f32.toMlir(b.ctx), cute.DType.f32.toMlir(b.ctx) }); - const word = a.d.get(.{ route, pair }); - computed.yieldThen(.{ - low.add(word.shl(16).bitCast(.f32)), - high.add(word.bitAnd(@as(i32, @bitCast(@as(u32, 0xffff0000)))).bitCast(.f32)), - }); - computed.yieldElse(.{ low, high }); - low = computed.results[0]; - high = computed.results[1]; + { + var low = b.constant(.f32, 0); + var high = b.constant(.f32, 0); + for (0..@intCast(cfg.topk)) |slot| { + const route = token.mul(cfg.topk).add(slot); + // Routes of other expert-parallel ranks were never written. + var computed = b.openIfElse(a.routes.get(.{route}).ge(0), .{ .f32, .f32 }); + { + const word = a.d.get(.{ route, pair }); + computed.yieldThen(.{ + low.add(word.shl(16).bitCast(.f32)), + high.add(word.bitAnd(@as(i32, @bitCast(@as(u32, 0xffff0000)))).bitCast(.f32)), + }); + } + { + computed.yieldElse(.{ low, high }); + } + low = computed.results[0]; + high = computed.results[1]; + } + const low_bits = low.to(.bf16).to(.f32).bitCast(.i32).shrLogical(16); + const high_bits = high.to(.bf16).to(.f32).bitCast(.i32).bitAnd(@as(i32, @bitCast(@as(u32, 0xffff0000)))); + a.y.set(.{ token, pair }, low_bits.bitOr(high_bits)); + in_bounds.close(); } - const low_bits = low.to(.bf16).to(.f32).bitCast(.i32).shrLogical(16); - const high_bits = high.to(.bf16).to(.f32).bitCast(.i32).bitAnd(@as(i32, @bitCast(@as(u32, 0xffff0000)))); - a.y.set(.{ token, pair }, low_bits.bitOr(high_bits)); - in_bounds.yieldThen(.{}); } b.launchDependents(); } diff --git a/zml/kernels/moe/cute/persistent_mxfp4.zig b/zml/kernels/moe/cute/persistent_mxfp4.zig index d1677a8a..c4bbe5cc 100644 --- a/zml/kernels/moe/cute/persistent_mxfp4.zig +++ b/zml/kernels/moe/cute/persistent_mxfp4.zig @@ -517,11 +517,13 @@ fn buildTiledProgram(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: // producing grid is still writing the operands. const warp = b.makeWarpUniform(b.makeWarpUniform(b.threadIdx().x.div(32))); var tma_warp = b.openIf(warp.eq(5)); - inline for (.{ d.tma_a, d.tma_b, d.tma_sfa, d.tma_sfb }) |value| { - const descriptor: cute.Atom = .{ .inner = value.inner, .kernel = b }; - b.prefetchTmaDesc(descriptor); + { + inline for (.{ d.tma_a, d.tma_b, d.tma_sfa, d.tma_sfb }) |value| { + const descriptor: cute.Atom = .{ .inner = value.inner, .kernel = b }; + b.prefetchTmaDesc(descriptor); + } + tma_warp.close(); } - tma_warp.yieldThen(.{}); // Everything below reads the schedule and the quantized activations the // preceding grid produces, so the whole block waits here. b.waitForDependency(); @@ -724,7 +726,7 @@ fn issueTmaStage( // it as a value rather than a literal: a statically zero leaf collapses out // of the coordinate tuple, and the TMA lowering then shifts the remaining // tensor-map coordinates, silently reading group zero for every group. - const b_tile_n = b.cst(.i32, 0); + const b_tile_n = b.constant(.i32, 0); const b_tile_k = b.slice( partitioned_b_target, .{ all, b_tile_n, all, group }, @@ -743,11 +745,13 @@ fn issueTmaStage( ); const b_source = b.rebuildCoordTensorIterator(3, b_tile, .{ div_128, div_n, dynamic }, .{ 128, n, 1 }, b_copy_layout); const sfa_source = b.makeCoordTensor(.{ 0, tile_m, k_tile, expert }, b.staticLayout(sf_copy_spec)); - const sfb_source = b.makeCoordTensor(.{ 0, b.cst(.i32, 0), k_tile, group }, b.staticLayout(sf_copy_spec)); + const sfb_source = b.makeCoordTensor(.{ 0, b.constant(.i32, 0), k_tile, group }, b.staticLayout(sf_copy_spec)); var elected = b.openIf(b.electSync()); - b.mbarrierArriveExpectTx(stage.load_barrier, tile.transactionBytes()); - elected.yieldThen(.{}); + { + b.mbarrierArriveExpectTx(stage.load_barrier, tile.transactionBytes()); + elected.close(); + } b.copy(b.setTmaBarrier(exec_a, stage.load_barrier), .{a_source}, .{stage.a_destination}, null); b.copy(b.setTmaBarrier(exec_b, stage.load_barrier), .{b_source}, .{stage.b_destination}, null); b.copy(b.setTmaBarrier(exec_sfa, stage.load_barrier), .{sfa_source}, .{stage.sfa_destination}, null); @@ -790,25 +794,25 @@ const DirectRoutes = struct { routes: usize, fn count(self: DirectRoutes, b: *B) V { - var valid = b.cst(.i32, 0); + var valid = b.constant(.i32, 0); for (0..self.routes) |r| valid = valid.add(self.expertAt(b, r).ge(0).to(.i32)); return valid; } /// Route of the `k`-th valid route. fn nth(self: DirectRoutes, b: *B, k: V) V { - var seen = b.cst(.i32, 0); - var route = b.cst(.i32, 0); + var seen = b.constant(.i32, 0); + var route = b.constant(.i32, 0); for (0..self.routes) |r| { const valid = self.expertAt(b, r).ge(0); - route = b.select(valid.bitAnd(seen.eq(k)), b.cst(.i32, @as(i32, @intCast(r))), route); + route = b.select(valid.bitAnd(seen.eq(k)), b.constant(.i32, r), route); seen = seen.add(valid.to(.i32)); } return route; } fn expertAt(self: DirectRoutes, b: *B, r: usize) V { - return self.expert_ids.get(.{b.cst(.i32, @as(i32, @intCast(r)))}, .i32); + return self.expert_ids.get(.{b.constant(.i32, r)}, .i32); } }; @@ -862,16 +866,24 @@ fn advancePipeline(b: *B, index: V, phase: V, comptime stages: usize) PipelinePo const advanced = index.add(1); const wraps = advanced.eq(stages); const next_index = b.select(wraps, 0, advanced); - var next_phase = b.openIfElse(wraps, .{phase.type_()}); - next_phase.yieldThen(.{phase.bitXor(1)}); - next_phase.yieldElse(.{phase}); + var next_phase = b.openIfElse(wraps, .{phase}); + { + next_phase.yieldThen(.{phase.bitXor(1)}); + } + { + next_phase.yieldElse(.{phase}); + } return .{ .index = next_index, .phase = next_phase.results[0] }; } fn peekBarrierIfRemaining(b: *B, base: V, count: V, limit: usize, state: PipelinePosition) V { - var peek = b.openIfElse(count.lt(limit), .{cute.DType.i1.toMlir(b.ctx)}); - peek.yieldThen(.{b.mbarrierWaitParity(barrierAt(b, base, state.index), state.phase)}); - peek.yieldElse(.{b.cst(.i1, true)}); + var peek = b.openIfElse(count.lt(limit), .{.i1}); + { + peek.yieldThen(.{b.mbarrierWaitParity(barrierAt(b, base, state.index), state.phase)}); + } + { + peek.yieldElse(.{b.constant(.i1, true)}); + } return peek.results[0]; } @@ -938,16 +950,16 @@ fn runTmaProducer( exec_sfa: cute.Atom, exec_sfb: cute.Atom, ) void { - const initial_index = b.cst(.i32, 0); - const initial_phase = b.cst(.i32, 1); - const initial_count = b.cst(.i32, 0); + const initial_index = b.constant(.i32, 0); + const initial_phase = b.constant(.i32, 1); + const initial_count = b.constant(.i32, 0); const persistent_clusters = b.gridDim().z; const cta_m = b.blockIdx().x; const cta_n = b.blockIdx().y; - const cta_l = b.cst(.i32, 0); + const cta_l = b.constant(.i32, 0); const div_m = b.fastDivmodCreate(tiles_m); const div_n = b.fastDivmodCreate(1); - const initial_work = persistentWorkAt(b, b.blockIdx().z, b.cst(.i32, 0), active, tiles_m, div_m, div_n, direct); + const initial_work = persistentWorkAt(b, b.blockIdx().z, b.constant(.i32, 0), active, tiles_m, div_m, div_n, direct); const carried = .{ initial_work.m, initial_work.n, initial_work.group, initial_work.valid, initial_count, initial_index, initial_phase, persistent_clusters, @@ -955,69 +967,77 @@ fn runTmaProducer( initial_work.executed, tiles_m, active, div_m, div_n, }; - var tiles = b.openWhile(carried, typesOf(carried)); - tiles.yieldBefore(tiles.before_carried[3], tupleOf(tiles.before_carried)); - const tile_m = tiles.after_carried[0]; - const group = tiles.after_carried[2]; - const expert = expert_ids.get(.{group}, .i32); - const first_state: PipelinePosition = .{ - .index = tiles.after_carried[5], - .phase = tiles.after_carried[6], - }; - const first_ready = b.mbarrierWaitParity( - barrierAt(b, storage.mma_barriers, first_state.index), - first_state.phase, - ); - var stages = b.openFor( - 0, - k_tiles, - 1, - .{ first_ready, b.cst(.i32, 0), first_state.index, first_state.phase }, - ); - const k = stages.carried[1]; - const state: PipelinePosition = .{ - .index = stages.carried[2], - .phase = stages.carried[3], - }; - const buffers = stageBuffersAt(b, tile, storage, state.index); - var pending = b.openIf(stages.carried[0].eq(false)); - b.mbarrierTryWaitParity(buffers.mma_barrier, state.phase, 10_000_000); - pending.yieldThen(.{}); - issueTmaStage(b, tile, k_tiles, partitioned_a_target, partitioned_b_target, buffers, k, tile_m, group, expert, exec_a, exec_b, exec_sfa, exec_sfb); - const next_count = stages.carried[1].add(1); - const next_state = advancePipeline(b, state.index, state.phase, tile.stages); - const next_ready = peekBarrierIfRemaining(b, storage.mma_barriers, next_count, k_tiles, next_state); - stages.yield(.{ next_ready, next_count, next_state.index, next_state.phase }); - const next_work = advancePersistentWork( - b, - tiles.after_carried[8], - tiles.after_carried[12], - tiles.after_carried[7], - tiles.after_carried[14], - tiles.after_carried[13], - tiles.after_carried[15], - tiles.after_carried[16], - direct, - ); - tiles.yieldAfter(.{ - next_work.m, - next_work.n, - next_work.group, - next_work.valid, - stages.results[1], - stages.results[2], - stages.results[3], - tiles.after_carried[7], - next_work.linear, - tiles.after_carried[9], - tiles.after_carried[10], - tiles.after_carried[11], - next_work.executed, - tiles.after_carried[13], - tiles.after_carried[14], - tiles.after_carried[15], - tiles.after_carried[16], - }); + var tiles = b.openWhile(carried, carried); + { + tiles.yieldBefore(tiles.before_carried[3], tiles.before_carried); + } + { + const tile_m = tiles.after_carried[0]; + const group = tiles.after_carried[2]; + const expert = expert_ids.get(.{group}, .i32); + const first_state: PipelinePosition = .{ + .index = tiles.after_carried[5], + .phase = tiles.after_carried[6], + }; + const first_ready = b.mbarrierWaitParity( + barrierAt(b, storage.mma_barriers, first_state.index), + first_state.phase, + ); + var stages = b.openFor( + 0, + k_tiles, + 1, + .{ first_ready, b.constant(.i32, 0), first_state.index, first_state.phase }, + ); + { + const k = stages.carried[1]; + const state: PipelinePosition = .{ + .index = stages.carried[2], + .phase = stages.carried[3], + }; + const buffers = stageBuffersAt(b, tile, storage, state.index); + var pending = b.openIf(stages.carried[0].eq(false)); + { + b.mbarrierTryWaitParity(buffers.mma_barrier, state.phase, 10_000_000); + pending.close(); + } + issueTmaStage(b, tile, k_tiles, partitioned_a_target, partitioned_b_target, buffers, k, tile_m, group, expert, exec_a, exec_b, exec_sfa, exec_sfb); + const next_count = stages.carried[1].add(1); + const next_state = advancePipeline(b, state.index, state.phase, tile.stages); + const next_ready = peekBarrierIfRemaining(b, storage.mma_barriers, next_count, k_tiles, next_state); + stages.yield(.{ next_ready, next_count, next_state.index, next_state.phase }); + } + const next_work = advancePersistentWork( + b, + tiles.after_carried[8], + tiles.after_carried[12], + tiles.after_carried[7], + tiles.after_carried[14], + tiles.after_carried[13], + tiles.after_carried[15], + tiles.after_carried[16], + direct, + ); + tiles.yieldAfter(.{ + next_work.m, + next_work.n, + next_work.group, + next_work.valid, + stages.results[1], + stages.results[2], + stages.results[3], + tiles.after_carried[7], + next_work.linear, + tiles.after_carried[9], + tiles.after_carried[10], + tiles.after_carried[11], + next_work.executed, + tiles.after_carried[13], + tiles.after_carried[14], + tiles.after_carried[15], + tiles.after_carried[16], + }); + } // PipelineTmaUmma.producer_tail: drain every empty barrier before the TMA // warp exits so no outstanding tcgen05 arrival outlives the CTA. @@ -1037,29 +1057,6 @@ fn runTmaProducer( } } -fn Tuple(comptime T: type, comptime len: usize) type { - return @Tuple(&@as([len]type, @splat(T))); -} - -fn arity(comptime T: type) usize { - return @typeInfo(T).@"struct".field_names.len; -} - -/// The MLIR types of a tuple of values, as `openWhile` expects them. -fn typesOf(values: anytype) Tuple(*const mlir.Type, arity(@TypeOf(values))) { - const len = comptime arity(@TypeOf(values)); - var out: Tuple(*const mlir.Type, len) = undefined; - inline for (0..len) |i| out[i] = values[i].type_(); - return out; -} - -/// Forward a whole loop-carried array, as `yieldBefore` expects a tuple. -fn tupleOf(array: anytype) Tuple(@TypeOf(array[0]), array.len) { - var out: Tuple(@TypeOf(array[0]), array.len) = undefined; - inline for (0..array.len) |i| out[i] = array[i]; - return out; -} - /// SM100 MXFP4 mainloop objects of one pipeline stage, produced by /// `StagedMxfp4Mainloop.atStage`. The dependent CuTe result types are /// centralized here; callers only provide tensors and the allocated @@ -1299,111 +1296,123 @@ fn runMmaConsumer( const div_m = b.fastDivmodCreate(tiles_m); const div_n = b.fastDivmodCreate(1); // The MMA warp never reads the group, only the tile count. - const initial_work = persistentWorkAt(b, b.blockIdx().z, b.cst(.i32, 0), active, tiles_m, div_m, div_n, null); + const initial_work = persistentWorkAt(b, b.blockIdx().z, b.constant(.i32, 0), active, tiles_m, div_m, div_n, null); const carried = .{ - initial_work.m, initial_work.n, initial_work.group, initial_work.valid, - b.cst(.i32, 0), b.cst(.i32, 0), b.cst(.i32, 0), tiled_mma.value(), - b.cst(.i32, 0), b.cst(.i32, 0), b.cst(.i32, 1), persistent_clusters, - initial_work.linear, b.blockIdx().x, b.blockIdx().y, b.cst(.i32, 0), - initial_work.executed, tiles_m, active, div_m, + initial_work.m, initial_work.n, initial_work.group, initial_work.valid, + b.constant(.i32, 0), b.constant(.i32, 0), b.constant(.i32, 0), tiled_mma.value(), + b.constant(.i32, 0), b.constant(.i32, 0), b.constant(.i32, 1), persistent_clusters, + initial_work.linear, b.blockIdx().x, b.blockIdx().y, b.constant(.i32, 0), + initial_work.executed, tiles_m, active, div_m, div_n, }; - var tiles = b.openWhile(carried, typesOf(carried)); - tiles.yieldBefore(tiles.before_carried[3], tupleOf(tiles.before_carried)); - // Python peeks the first AB-full token before it blocks on an available - // accumulator stage. Preserve that ordering so the two independent - // pipelines enter the K loop with the same outstanding operations. - const first_state: PipelinePosition = .{ - .index = tiles.after_carried[5], - .phase = tiles.after_carried[6], - }; - const first_ready = b.mbarrierWaitParity( - barrierAt(b, storage.load_barriers, first_state.index), - first_state.phase, - ); - const accumulator_state: PipelinePosition = .{ - .index = tiles.after_carried[9], - .phase = tiles.after_carried[10], - }; - const accumulator_stage = accumulator_state.index; - const accumulator_empty = barrierAt(b, storage.accumulator_empty_barriers, accumulator_stage); - b.mbarrierTryWaitParity(accumulator_empty, accumulator_state.phase, 10_000_000); + var tiles = b.openWhile(carried, carried); + { + tiles.yieldBefore(tiles.before_carried[3], tiles.before_carried); + } + { + // Python peeks the first AB-full token before it blocks on an available + // accumulator stage. Preserve that ordering so the two independent + // pipelines enter the K loop with the same outstanding operations. + const first_state: PipelinePosition = .{ + .index = tiles.after_carried[5], + .phase = tiles.after_carried[6], + }; + const first_ready = b.mbarrierWaitParity( + barrierAt(b, storage.load_barriers, first_state.index), + first_state.phase, + ); + const accumulator_state: PipelinePosition = .{ + .index = tiles.after_carried[9], + .phase = tiles.after_carried[10], + }; + const accumulator_stage = accumulator_state.index; + const accumulator_empty = barrierAt(b, storage.accumulator_empty_barriers, accumulator_stage); + b.mbarrierTryWaitParity(accumulator_empty, accumulator_state.phase, 10_000_000); - const persistent_mma: cute.Atom = .{ .inner = tiles.after_carried[7].inner, .kernel = b }; - const initial_mma = b.setMmaAccumulate(persistent_mma, b.cst(.i1, false)); - var stages = b.openFor( - 0, - k_tiles, - 1, - // Match PipelineTmaUmma's Python loop-carried ABI exactly. The - // software-pipeline pass recognizes the leading try-wait token and - // keeps the MMA atom immediately after it. - .{ first_ready, initial_mma.value(), b.cst(.i32, 0), first_state.index, first_state.phase }, - ); - const k = stages.carried[2]; - const state: PipelinePosition = .{ .index = stages.carried[3], .phase = stages.carried[4] }; - var pending = b.openIf(stages.carried[0].eq(false)); - b.mbarrierTryWaitParity(barrierAt(b, storage.load_barriers, state.index), state.phase, 10_000_000); - pending.yieldThen(.{}); - const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, accumulator_stage); - const mainloop = staged_mainloop.atStage(b, state.index, accumulator_tmem); - mainloop.copyScales(b); - const loop_mma: cute.Atom = .{ .inner = stages.carried[1].inner, .kernel = b }; - const mma = b.setMmaAccumulate(loop_mma, k.ne(0)); - b.gemm( - mma, - mainloop.accumulator, - .{ mainloop.a, mainloop.sfa }, - .{ mainloop.b, mainloop.sfb }, - mainloop.accumulator, - ); - var commit_lane = b.openIf(b.electSync()); - b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index)); - commit_lane.yieldThen(.{}); - const next_count = stages.carried[2].add(1); - const next_state = advancePipeline(b, state.index, state.phase, tile.stages); - const next_ready = peekBarrierIfRemaining(b, storage.load_barriers, next_count, k_tiles, next_state); - stages.yield(.{ next_ready, mma.value(), next_count, next_state.index, next_state.phase }); + const persistent_mma: cute.Atom = .{ .inner = tiles.after_carried[7].inner, .kernel = b }; + const initial_mma = b.setMmaAccumulate(persistent_mma, b.constant(.i1, false)); + var stages = b.openFor( + 0, + k_tiles, + 1, + // Match PipelineTmaUmma's Python loop-carried ABI exactly. The + // software-pipeline pass recognizes the leading try-wait token and + // keeps the MMA atom immediately after it. + .{ first_ready, initial_mma.value(), b.constant(.i32, 0), first_state.index, first_state.phase }, + ); + { + const k = stages.carried[2]; + const state: PipelinePosition = .{ .index = stages.carried[3], .phase = stages.carried[4] }; + var pending = b.openIf(stages.carried[0].eq(false)); + { + b.mbarrierTryWaitParity(barrierAt(b, storage.load_barriers, state.index), state.phase, 10_000_000); + pending.close(); + } + const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, accumulator_stage); + const mainloop = staged_mainloop.atStage(b, state.index, accumulator_tmem); + mainloop.copyScales(b); + const loop_mma: cute.Atom = .{ .inner = stages.carried[1].inner, .kernel = b }; + const mma = b.setMmaAccumulate(loop_mma, k.ne(0)); + b.gemm( + mma, + mainloop.accumulator, + .{ mainloop.a, mainloop.sfa }, + .{ mainloop.b, mainloop.sfb }, + mainloop.accumulator, + ); + var commit_lane = b.openIf(b.electSync()); + { + b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index)); + commit_lane.close(); + } + const next_count = stages.carried[2].add(1); + const next_state = advancePipeline(b, state.index, state.phase, tile.stages); + const next_ready = peekBarrierIfRemaining(b, storage.load_barriers, next_count, k_tiles, next_state); + stages.yield(.{ next_ready, mma.value(), next_count, next_state.index, next_state.phase }); + } - var full_lane = b.openIf(b.electSync()); - b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage)); - full_lane.yieldThen(.{}); - const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); - const next_acc_count = tiles.after_carried[8].add(1); - const next_work = advancePersistentWork( - b, - tiles.after_carried[12], - tiles.after_carried[16], - tiles.after_carried[11], - tiles.after_carried[18], - tiles.after_carried[17], - tiles.after_carried[19], - tiles.after_carried[20], - null, - ); - tiles.yieldAfter(.{ - next_work.m, - next_work.n, - next_work.group, - next_work.valid, - stages.results[2], - stages.results[3], - stages.results[4], - stages.results[1], - next_acc_count, - next_accumulator_state.index, - next_accumulator_state.phase, - tiles.after_carried[11], - next_work.linear, - tiles.after_carried[13], - tiles.after_carried[14], - tiles.after_carried[15], - next_work.executed, - tiles.after_carried[17], - tiles.after_carried[18], - tiles.after_carried[19], - tiles.after_carried[20], - }); + var full_lane = b.openIf(b.electSync()); + { + b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage)); + full_lane.close(); + } + const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); + const next_acc_count = tiles.after_carried[8].add(1); + const next_work = advancePersistentWork( + b, + tiles.after_carried[12], + tiles.after_carried[16], + tiles.after_carried[11], + tiles.after_carried[18], + tiles.after_carried[17], + tiles.after_carried[19], + tiles.after_carried[20], + null, + ); + tiles.yieldAfter(.{ + next_work.m, + next_work.n, + next_work.group, + next_work.valid, + stages.results[2], + stages.results[3], + stages.results[4], + stages.results[1], + next_acc_count, + next_accumulator_state.index, + next_accumulator_state.phase, + tiles.after_carried[11], + next_work.linear, + tiles.after_carried[13], + tiles.after_carried[14], + tiles.after_carried[15], + next_work.executed, + tiles.after_carried[17], + tiles.after_carried[18], + tiles.after_carried[19], + tiles.after_carried[20], + }); + } // Match PipelineUmmaAsync.producer_tail. The state points at the next // accumulator slot; advance to the last slot used by this producer and @@ -1461,96 +1470,104 @@ fn runEpilogue( const div_m = b.fastDivmodCreate(tiles_m); const div_n = b.fastDivmodCreate(1); - const initial_work = persistentWorkAt(b, b.blockIdx().z, b.cst(.i32, 0), active, tiles_m, div_m, div_n, direct); + const initial_work = persistentWorkAt(b, b.blockIdx().z, b.constant(.i32, 0), active, tiles_m, div_m, div_n, direct); const carried = .{ - initial_work.m, initial_work.n, initial_work.group, initial_work.valid, - b.cst(.i32, 0), b.cst(.i32, 0), b.cst(.i32, 0), initial_work.linear, + initial_work.m, initial_work.n, initial_work.group, initial_work.valid, + b.constant(.i32, 0), b.constant(.i32, 0), b.constant(.i32, 0), initial_work.linear, initial_work.executed, }; var tiles = b.openWhile(carried, .{ - initial_work.m.type_(), - initial_work.n.type_(), - initial_work.group.type_(), - cute.DType.i32.toMlir(b.ctx), - cute.DType.i32.toMlir(b.ctx), - cute.DType.i32.toMlir(b.ctx), - initial_work.linear.type_(), - initial_work.executed.type_(), + initial_work.m, + initial_work.n, + initial_work.group, + .i32, + .i32, + .i32, + initial_work.linear, + initial_work.executed, }); - tiles.yieldBefore(tiles.before_carried[3], .{ - tiles.before_carried[0], - tiles.before_carried[1], - tiles.before_carried[2], - tiles.before_carried[4], - tiles.before_carried[5], - tiles.before_carried[6], - tiles.before_carried[7], - tiles.before_carried[8], - }); - const tile_m = tiles.after_carried[0]; - const group = tiles.after_carried[2]; - const accumulator_state: PipelinePosition = .{ - .index = tiles.after_carried[4], - .phase = tiles.after_carried[5], - }; - const stage = accumulator_state.index; - b.mbarrierTryWaitParity(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, 10_000_000); + { + tiles.yieldBefore(tiles.before_carried[3], .{ + tiles.before_carried[0], + tiles.before_carried[1], + tiles.before_carried[2], + tiles.before_carried[4], + tiles.before_carried[5], + tiles.before_carried[6], + tiles.before_carried[7], + tiles.before_carried[8], + }); + } + { + const tile_m = tiles.after_carried[0]; + const group = tiles.after_carried[2]; + const accumulator_state: PipelinePosition = .{ + .index = tiles.after_carried[4], + .phase = tiles.after_carried[5], + }; + const stage = accumulator_state.index; + b.mbarrierTryWaitParity(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, 10_000_000); - const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, stage); - const warp_tmem = b.addOffset( - accumulator_tmem, - b.makeIntTupleDivBy(warp.mul(32 * 65536), 4), - b.ptrTy(.f32, .tmem, 16) catch @panic("bad epilogue TMEM pointer"), - ).value(); - // A warp lane holds one routed row, so tiles wider than a warp are drained - // 32 rows at a time. The accumulator stage is released once its last chunk - // is in registers. - const chunk_rows: usize = comptime @intCast(tile.chunkRows()); - const tile_chunks = comptime tile.chunks(); - inline for (0..tile_chunks) |chunk| { - const chunk_tmem = if (chunk == 0) warp_tmem else b.addOffset( - warp_tmem, - b.makeIntTuple(.{@as(i32, @intCast(chunk * chunk_rows))}), - b.ptrTy(.f32, .tmem, offsetAlignment(16, @intCast(chunk * chunk_rows), 4)) catch @panic("bad epilogue TMEM pointer"), + const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, stage); + const warp_tmem = b.addOffset( + accumulator_tmem, + b.makeIntTupleDivBy(warp.mul(32 * 65536), 4), + b.ptrTy(.f32, .tmem, 16) catch @panic("bad epilogue TMEM pointer"), ).value(); - // Route metadata does not depend on the accumulator. - const routed = routedLanes(b, tile, operands, group, chunk); - const values = b.tmemLoadVector(chunk_tmem, @intCast(chunk_rows), .{ .num_dp = 32, .num_b = 32, .num_rep = @intCast(chunk_rows) }); - b.fenceTmemLoad(); - if (chunk + 1 == tile_chunks) { - // Registers now own the accumulator values, so all 128 epilogue - // threads release this TMEM stage before the stores. - b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), 1); + // A warp lane holds one routed row, so tiles wider than a warp are drained + // 32 rows at a time. The accumulator stage is released once its last chunk + // is in registers. + const chunk_rows: usize = comptime @intCast(tile.chunkRows()); + const tile_chunks = comptime tile.chunks(); + inline for (0..tile_chunks) |chunk| { + const chunk_tmem = if (chunk == 0) warp_tmem else b.addOffset( + warp_tmem, + b.makeIntTuple(.{@as(i32, @intCast(chunk * chunk_rows))}), + b.ptrTy(.f32, .tmem, offsetAlignment(16, @intCast(chunk * chunk_rows), 4)) catch @panic("bad epilogue TMEM pointer"), + ).value(); + // Route metadata does not depend on the accumulator. + const routed = routedLanes(b, tile, operands, group, chunk); + const values = b.tmemLoadVector(chunk_tmem, @intCast(chunk_rows), .{ .num_dp = 32, .num_b = 32, .num_rep = @intCast(chunk_rows) }); + b.fenceTmemLoad(); + if (chunk + 1 == tile_chunks) { + // Registers now own the accumulator values, so all 128 epilogue + // threads release this TMEM stage before the stores. + b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), 1); + } + var columns: [@intCast(@min(n, 32))]V = undefined; + inline for (&columns, 0..) |*column, i| column.* = b.vectorExtract(values, i, .i32).bitCast(.f32); + switch (epilogue) { + .swiglu_mxfp8 => storeTileSwiglu(b, tile, storage, operands, routed, tile_m, group, &columns, chunk), + .route_rows => storeTileRows(b, tile, operands, routed, tile_m, &columns, chunk), + } } - var columns: [@intCast(@min(n, 32))]V = undefined; - inline for (&columns, 0..) |*column, i| column.* = b.vectorExtract(values, i, .i32).bitCast(.f32); - switch (epilogue) { - .swiglu_mxfp8 => storeTileSwiglu(b, tile, storage, operands, routed, tile_m, group, &columns, chunk), - .route_rows => storeTileRows(b, tile, operands, routed, tile_m, &columns, chunk), - } + const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); + const next_work = advancePersistentWork(b, tiles.after_carried[6], tiles.after_carried[7], b.gridDim().z, active, tiles_m, div_m, div_n, direct); + tiles.yieldAfter(.{ + next_work.m, + next_work.n, + next_work.group, + next_work.valid, + tiles.after_carried[3].add(1), + next_accumulator_state.index, + next_accumulator_state.phase, + next_work.linear, + next_work.executed, + }); } - const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); - const next_work = advancePersistentWork(b, tiles.after_carried[6], tiles.after_carried[7], b.gridDim().z, active, tiles_m, div_m, div_n, direct); - tiles.yieldAfter(.{ - next_work.m, - next_work.n, - next_work.group, - next_work.valid, - tiles.after_carried[3].add(1), - next_accumulator_state.index, - next_accumulator_state.phase, - next_work.linear, - next_work.executed, - }); var release_warp = b.openIf(warp.eq(0)); - // Permit release and deallocation are warp collective as well. - b.relinquishTmemAllocPermit(.{}); - release_warp.yieldThen(.{}); + { + // Permit release and deallocation are warp collective as well. + b.relinquishTmemAllocPermit(.{}); + release_warp.close(); + } b.namedBarrier(1, 128); var dealloc_warp = b.openIf(warp.eq(0)); - b.deallocTmem(tmem, tile.tmem_columns, .{}); - dealloc_warp.yieldThen(.{}); + { + b.deallocTmem(tmem, tile.tmem_columns, .{}); + dealloc_warp.close(); + } } /// E8M0 exponent of the power-of-two MX scale for a block maximum, rounded @@ -1558,7 +1575,7 @@ fn runEpilogue( fn mxScale(b: *B, maximum: V) struct { exponent: V, inverse: V } { const raw_bits = maximum.maximum(1e-4).mul(1.0 / 448.0).bitCast(.i32); const exponent = raw_bits.shrLogical(23).add(raw_bits.bitAnd(0x7fffff).ne(0).to(.i32)); - return .{ .exponent = exponent, .inverse = b.cst(.i32, 254).sub(exponent).shl(23).bitCast(.f32) }; + return .{ .exponent = exponent, .inverse = b.constant(.i32, 254).sub(exponent).shl(23).bitCast(.f32) }; } /// Per-row routing data of one expert group, spread across a warp: lane `l` @@ -1575,7 +1592,7 @@ fn routedLanes(b: *B, comptime tile: Tile, operands: EpilogueOperands, group: V, // Only the up projection applies routing weights. const weights = operands.routing_weights; if (operands.direct) return .{ - .count = b.cst(.i32, 1), + .count = b.constant(.i32, 1), .route = group, .weight = if (weights) |w| w.get(.{group}, .f32) else null, }; @@ -1584,11 +1601,11 @@ fn routedLanes(b: *B, comptime tile: Tile, operands: EpilogueOperands, group: V, // Lanes past the tile width alias row 0, which is always live. Wide tiles // run one 32-row chunk at a time, so lane `l` holds row `32 * chunk + l`. const chunk_rows = comptime tile.chunkRows(); - const in_chunk = b.select(lane.lt(@as(i32, @intCast(chunk_rows))), lane, b.cst(.i32, 0)); - const row_in_group = if (chunk == 0) in_chunk else in_chunk.add(@as(i32, @intCast(@as(i64, @intCast(chunk)) * chunk_rows))); + const in_chunk = b.select(lane.lt(chunk_rows), lane, b.constant(.i32, 0)); + const row_in_group = if (chunk == 0) in_chunk else in_chunk.add(@as(i64, @intCast(chunk)) * chunk_rows); const live = row_in_group.lt(count); const inverse = operands.route_inverse.get(.{group.mul(tile.n).add(row_in_group)}, .i32); - const route = b.select(live, inverse, b.cst(.i32, 0)); + const route = b.select(live, inverse, b.constant(.i32, 0)); const weight = if (weights) |w| w.get(.{route}, .f32) else null; return .{ .count = count, .route = route, .weight = weight }; } @@ -1634,7 +1651,7 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo var activations: [@intCast(n)]V = undefined; // Lane `c` collects the warp maximum of column `c`. - var lane_maximum = b.cst(.f32, 0); + var lane_maximum = b.constant(.f32, 0); for (0..row_chunks) |row_chunk| { const first = row_chunk * swiglu_chunk; const last = @min(first + swiglu_chunk, live); @@ -1642,7 +1659,7 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo const guarded = skip_padding and (row_chunk > 0 or chunk > 0); const types: [swiglu_chunk + 1]*const mlir.Type = @splat(f32_type); // `@TypeOf` does not evaluate its operand. - var guard: ?@TypeOf(b.openIfElse(routed.count, tupleOf(types))) = if (guarded) b.openIfElse(routed.count.gt(chunk_first + @as(i32, @intCast(first))), tupleOf(types)) else null; + var guard: ?@TypeOf(b.openIfElse(routed.count, types)) = if (guarded) b.openIfElse(routed.count.gt(chunk_first + @as(i32, @intCast(first))), types) else null; var chunk_maximum = lane_maximum; for (columns[first..last], activations[first..last], first..) |value, *activation, column| { const c: i32 = @intCast(column); @@ -1661,33 +1678,35 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo var result = up_value.mul(gate).mul(b.unaryF32("rcp.approx.ftz.f32", exp_neg.add(1.0))); result = result.mul(b.shuffleIdx(routed.weight.?, c)).to(.bf16).to(.f32); // Odd threads hold no activation and must not raise the maxima. - activation.* = b.select(is_gate, result, b.cst(.f32, 0)); + activation.* = b.select(is_gate, result, b.constant(.f32, 0)); const maximum = b.warpMaxNonNegative(activation.*.abs()); chunk_maximum = b.select(lane.eq(c), maximum, chunk_maximum); } if (guard) |*g| { var then_values: [swiglu_chunk + 1]V = @splat(chunk_maximum); for (activations[first..last], then_values[0 .. last - first]) |activation, *t| t.* = activation; - g.yieldThen(tupleOf(then_values)); + g.yieldThen(then_values); // Padding chunk: zero activations, maxima unchanged. The zero is // emitted in the else block, where it is used. - var else_values: [swiglu_chunk + 1]V = @splat(b.cst(.f32, 0)); + var else_values: [swiglu_chunk + 1]V = @splat(b.constant(.f32, 0)); else_values[swiglu_chunk] = lane_maximum; - g.yieldElse(tupleOf(else_values)); + g.yieldElse(else_values); for (activations[first..last], g.results[0 .. last - first]) |*activation, result| activation.* = result; lane_maximum = g.results[swiglu_chunk]; } else { lane_maximum = chunk_maximum; } } - var owner = b.openIf(lane.lt(@as(i32, @intCast(n)))); - scratch.set(.{ warp, lane }, lane_maximum); - owner.yieldThen(.{}); + var owner = b.openIf(lane.lt(n)); + { + scratch.set(.{ warp, lane }, lane_maximum); + owner.close(); + } b.namedBarrier(1, 128); // Lane `c` turns the two half-block maxima of column `c` into its scale. const pair = warp.bitAnd(~@as(i32, 1)); - const lane_column = b.select(lane.lt(@as(i32, @intCast(n))), lane, b.cst(.i32, 0)); + const lane_column = b.select(lane.lt(n), lane, b.constant(.i32, 0)); const scale = mxScale(b, scratch.get(.{ pair, lane_column }, .f32).maximum(scratch.get(.{ pair.add(1), lane_column }, .f32))); // The next tile reuses the scratch maxima. b.namedBarrier(1, 128); @@ -1700,20 +1719,24 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo var inverses: [swiglu_chunk]V = undefined; for (inverses[0 .. last - first], first..) |*inverse, column| inverse.* = b.shuffleIdx(scale.inverse, @as(i32, @intCast(column))); var gate_lane = b.openIf(is_gate); - for (activations[first..last], inverses[0 .. last - first], first..) |activation, inverse, column| { - const quantized = activation.mul(inverse).maximum(-448.0).minimum(448.0).to(.f8e4m3fn); - operands.q.?.set(.{ first_row.add(chunk_first + @as(i32, @intCast(column))), activation_index }, quantized); + { + for (activations[first..last], inverses[0 .. last - first], first..) |activation, inverse, column| { + const quantized = activation.mul(inverse).maximum(-448.0).minimum(448.0).to(.f8e4m3fn); + operands.q.?.set(.{ first_row.add(chunk_first + @as(i32, @intCast(column))), activation_index }, quantized); + } + gate_lane.close(); } - gate_lane.yieldThen(.{}); - if (guard) |*g| g.yieldThen(.{}); + if (guard) |*g| g.close(); } // Warps 0 and 2 each own one 32-activation block of this tile. var scale_lane = b.openIf(warp.rem(2).eq(0).bitAnd(lane.add(chunk_first).lt(routed.count))); - const block = tile_m.mul(2).add(warp.div(2)); - const swizzled = block.div(4).mul(512).add(lane.mul(16)).add(chunk * 4).add(block.rem(4)); - operands.s.?.set(.{ group, swizzled }, scale.exponent.to(.i8)); - scale_lane.yieldThen(.{}); + { + const block = tile_m.mul(2).add(warp.div(2)); + const swizzled = block.div(4).mul(512).add(lane.mul(16)).add(chunk * 4).add(block.rem(4)); + operands.s.?.set(.{ group, swizzled }, scale.exponent.to(.i8)); + scale_lane.close(); + } } /// Python `down_epilogue`: store each live routed row at its token/top-k @@ -1730,7 +1753,7 @@ fn storeTileRows(b: *B, comptime tile: Tile, operands: EpilogueOperands, routed: for (columns, routes[0..columns.len], 0..) |value, route, column| { var live: ?@TypeOf(b.openIf(routed.count)) = if (column == 0 and chunk == 0) null else b.openIf(routed.count.gt(chunk_first + @as(i32, @intCast(column)))); operands.rows.?.set(.{ route, output_column }, value.to(.bf16)); - if (live) |*l| l.yieldThen(.{}); + if (live) |*l| l.close(); } } @@ -1866,67 +1889,79 @@ fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E // every lane races on the same shared state and breaks the first phase // transition when the A/B ring wraps. var init_warp = b.openIf(warp.eq(0)); - const barrier_arrays = .{ - .{ storage.load_barriers, stages, 1 }, - .{ storage.mma_barriers, stages, 1 }, - .{ storage.accumulator_full_barriers, accumulator_stages, 1 }, - .{ storage.accumulator_empty_barriers, accumulator_stages, 128 }, - }; - inline for (barrier_arrays) |array| { - var lane = b.openIf(b.electSync()); - inline for (0..array[1]) |stage| { - b.mbarrierInit(barrierAt(b, array[0], b.cst(.i32, stage)), array[2]); + { + const barrier_arrays = .{ + .{ storage.load_barriers, stages, 1 }, + .{ storage.mma_barriers, stages, 1 }, + .{ storage.accumulator_full_barriers, accumulator_stages, 1 }, + .{ storage.accumulator_empty_barriers, accumulator_stages, 128 }, + }; + inline for (barrier_arrays) |array| { + var lane = b.openIf(b.electSync()); + { + inline for (0..array[1]) |stage| { + b.mbarrierInit(barrierAt(b, array[0], b.constant(.i32, stage)), array[2]); + } + lane.close(); + } } - lane.yieldThen(.{}); + b.fenceMbarrierInit(); + init_warp.close(); } - b.fenceMbarrierInit(); - init_warp.yieldThen(.{}); b.syncThreads(); // Each role walks the same persistent tile sequence independently. The // AB and accumulator mbarriers carry all cross-warp dependencies. var tma_warp = b.openIf(warp.eq(5)); - switch (k_tiles) { - inline 16, 18, 32, 40 => |kt| runTmaProducer(b, tile, kt, storage, active, d.tiles_m, expert_ids, direct, partitioned_a.targets[0], partitioned_b.targets[0], exec_a, exec_b, exec_sfa, exec_sfb), - else => std.debug.panic("unsupported MXFP4 reduction tile count {d}", .{k_tiles}), + { + switch (k_tiles) { + inline 16, 18, 32, 40 => |kt| runTmaProducer(b, tile, kt, storage, active, d.tiles_m, expert_ids, direct, partitioned_a.targets[0], partitioned_b.targets[0], exec_a, exec_b, exec_sfa, exec_sfb), + else => std.debug.panic("unsupported MXFP4 reduction tile count {d}", .{k_tiles}), + } + tma_warp.close(); } - tma_warp.yieldThen(.{}); var mma_warp = b.openIf(warp.eq(4)); - // Python's MMA role reaches its own TmemAllocator.wait_for_alloc site. - b.namedBarrier(2, 160); - switch (k_tiles) { - inline 16, 18, 32, 40 => |kt| runMmaConsumer(b, tile, kt, storage, tiled_mma, tmem_holding, active, d.tiles_m), - else => unreachable, + { + // Python's MMA role reaches its own TmemAllocator.wait_for_alloc site. + b.namedBarrier(2, 160); + switch (k_tiles) { + inline 16, 18, 32, 40 => |kt| runMmaConsumer(b, tile, kt, storage, tiled_mma, tmem_holding, active, d.tiles_m), + else => unreachable, + } + mma_warp.close(); } - mma_warp.yieldThen(.{}); var epilogue_warps = b.openIf(warp.lt(4)); - // TmemAllocator.allocate is issued collectively by warp 0. The four - // epilogue warps then meet the MMA warp at named barrier 2. - var alloc_warp = b.openIf(warp.eq(0)); - b.allocTmem(tile.tmem_columns, tmem_holding, .{}); - alloc_warp.yieldThen(.{}); - b.namedBarrier(2, 160); - switch (epilogue) { - .swiglu_mxfp8 => runEpilogue(b, tile, epilogue, storage, tmem_holding, active, d.tiles_m, direct, .{ - .group_sizes = view(b, d.group_sizes), - .route_inverse = view(b, d.route_inverse), - .routing_weights = view(b, d.routing_weights), - .q = view(b, d.q), - .s = view(b, d.s), - .direct = cfg.direct, - .group_rows = cfg.groupRows(), - .swiglu_limit = cfg.swiglu_limit, - }), - .route_rows => runEpilogue(b, tile, epilogue, storage, tmem_holding, active, d.tiles_m, direct, .{ - .group_sizes = view(b, d.group_sizes), - .route_inverse = view(b, d.route_inverse), - .rows = view(b, d.rows), - .direct = cfg.direct, - }), + { + // TmemAllocator.allocate is issued collectively by warp 0. The four + // epilogue warps then meet the MMA warp at named barrier 2. + var alloc_warp = b.openIf(warp.eq(0)); + { + b.allocTmem(tile.tmem_columns, tmem_holding, .{}); + alloc_warp.close(); + } + b.namedBarrier(2, 160); + switch (epilogue) { + .swiglu_mxfp8 => runEpilogue(b, tile, epilogue, storage, tmem_holding, active, d.tiles_m, direct, .{ + .group_sizes = view(b, d.group_sizes), + .route_inverse = view(b, d.route_inverse), + .routing_weights = view(b, d.routing_weights), + .q = view(b, d.q), + .s = view(b, d.s), + .direct = cfg.direct, + .group_rows = cfg.groupRows(), + .swiglu_limit = cfg.swiglu_limit, + }), + .route_rows => runEpilogue(b, tile, epilogue, storage, tmem_holding, active, d.tiles_m, direct, .{ + .group_sizes = view(b, d.group_sizes), + .route_inverse = view(b, d.route_inverse), + .rows = view(b, d.rows), + .direct = cfg.direct, + }), + } + epilogue_warps.close(); } - epilogue_warps.yieldThen(.{}); } fn buildHostLaunch(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: Epilogue) cute.FinishError!void { @@ -1990,12 +2025,12 @@ fn buildHostLaunch(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E const tma_sfa = b.makeTiledTmaLoadAtom(gSFA.value(), sf_smem.value(), sf_map.value(), L.tmaSf(b)); const tma_sfb = b.makeTiledTmaLoadAtom(gSFB.value(), sf_smem.value(), sf_map.value(), L.tmaSf(b)); - const one = b.cst(.i32, 1); - const tiles_m = b.cst(.i32, @divExact(cfg.m, 128)); - const grid_z = b.cst(.i32, @min(@divExact(cfg.m, 128) * cfg.groups, cfg.persistent_ctas)); + const one = b.constant(.i32, 1); + const tiles_m = b.constant(.i32, @divExact(cfg.m, 128)); + const grid_z = b.constant(.i32, @min(@divExact(cfg.m, 128) * cfg.groups, cfg.persistent_ctas)); const config = b.makeLaunchConfig(.{ .grid = .{ one, one, grid_z }, - .block = .{ b.cst(.i32, 192), one, one }, + .block = .{ b.constant(.i32, 192), one, one }, .dynamic_smem = b.kernelSmemSize(deviceName(epilogue)), .stream = b.cudaStream(), .cluster = .{ one, one, one }, diff --git a/zml/kernels/moe/fly/moe.zig b/zml/kernels/moe/fly/moe.zig index 96503fa9..3cb82b96 100644 --- a/zml/kernels/moe/fly/moe.zig +++ b/zml/kernels/moe/fly/moe.zig @@ -79,68 +79,95 @@ pub const Align = struct { const counts = b.sharedArray(.i32, 512, 16); const ends = b.sharedArray(.i32, 512, 16); const waves = b.sharedArray(.i32, 8, 16); - var fill = b.openIfElse(b.blockId(.x).cmp(.ne, 0), .{}); - var positions = b.openFor(tid64, cfg.capacity, 512, .{}); - args.sorted_token_ids.set(positions.iv, b.constant(.i32, cfg.routes)); - positions.yield(.{}); - fill.yieldThen(.{}); - - counts.set(tid, zero); - b.barrier(); - var routes = b.openFor(tid64, cfg.routes, 512, .{}); - const id = args.ids.at(routes.iv); - var valid_id = b.openIf(id.cmp(.ge, 0).bitAnd(id.cmp(.lt, cfg.experts))); - _ = b.ptrAtomicAdd(counts.emitIter().addOffset(id), one, .workgroup); - valid_id.yieldThen(.{}); - routes.yield(.{}); - b.barrier(); - - const padded = counts.at(tid).add(cfg.padding).divUnsigned(cfg.block_size).mul(cfg.block_size); - const lane = tid.remUnsigned(64); - const wave = tid.divUnsigned(64); - var prefix = padded; - inline for (.{ 1, 2, 4, 8, 16, 32 }) |shift| { - const peer = prefix.shuffle(.up, shift, 64); - prefix = prefix.add(lane.cmpUnsigned(.ge, shift).select(peer, zero)); + var fill = b.openIfElse(b.blockId(.x).ne(0), .{}); + { + var positions = b.openFor(tid64, cfg.capacity, 512, .{}); + { + args.sorted_token_ids.set(positions.iv, b.constant(.i32, cfg.routes)); + positions.close(); + } + fill.elseBranch(); } - var last_lane = b.openIf(lane.cmp(.eq, 63)); - waves.set(wave, prefix); - last_lane.yieldThen(.{}); - b.barrier(); + { + counts.set(tid, zero); + b.barrier(); + var routes = b.openFor(tid64, cfg.routes, 512, .{}); + { + const id = args.ids.at(routes.iv); + var valid_id = b.openIf(id.ge(0).bitAnd(id.lt(cfg.experts))); + { + _ = b.ptrAtomicAdd(counts.emitIter().addOffset(id), one, .workgroup); + valid_id.close(); + } + routes.close(); + } + b.barrier(); - var sums = b.openFor(zero, 8, 1, .{ zero, zero }); - const value = waves.at(sums.iv); - const preceding = sums.carried[0].add(sums.iv.cmpUnsigned(.lt, wave).select(value, zero)); - const total = sums.carried[1].add(value); - sums.yield(.{ preceding, total }); - const end = sums.results[0].add(prefix); - const start = end.sub(padded); - ends.set(tid, end); - var has_expert = b.openIf(tid.cmpUnsigned(.lt, cfg.experts)); - args.cumsum.set(tid64, start); - has_expert.yieldThen(.{}); - var first = b.openIf(tid.cmp(.eq, 0)); - args.cumsum.set(b.constant(.i64, cfg.experts), sums.results[1]); - args.num_tokens_post_pad.set(b.constant(.i64, 0), sums.results[1]); - first.yieldThen(.{}); - b.barrier(); + const padded = counts.at(tid).add(cfg.padding).divUnsigned(cfg.block_size).mul(cfg.block_size); + const lane = tid.remUnsigned(64); + const wave = tid.divUnsigned(64); + var prefix = padded; + inline for (.{ 1, 2, 4, 8, 16, 32 }) |shift| { + const peer = prefix.shuffle(.up, shift, 64); + prefix = prefix.add(lane.cmpUnsigned(.ge, shift).select(peer, zero)); + } + var last_lane = b.openIf(lane.eq(63)); + { + waves.set(wave, prefix); + last_lane.close(); + } + b.barrier(); - var blocks = b.openFor(tid64, cfg.blocks, 512, .{}); - const offset = blocks.iv.to(.i32).mul(cfg.block_size); - var block_expert = b.openIfElse(offset.cmpUnsigned(.lt, sums.results[1]), .{zero.type_()}); - // Upper bound of inclusive expert ends handles empty experts without a scan. - var bounds = b.openWhile(.{ zero, experts }, .{ zero.type_(), zero.type_() }); - bounds.yieldBefore(bounds.before_carried[0].cmpUnsigned(.lt, bounds.before_carried[1]), .{ bounds.before_carried[0], bounds.before_carried[1] }); - const lo = bounds.after_carried[0]; - const hi = bounds.after_carried[1]; - const mid = lo.add(hi).shrU(1); - const advance = ends.at(mid).cmpUnsigned(.le, offset); - bounds.yieldAfter(.{ advance.select(mid.add(1), lo), advance.select(hi, mid) }); - block_expert.yieldThen(.{bounds.results[0]}); - block_expert.yieldElse(.{b.constant(.i32, -1)}); - args.expert_ids.set(blocks.iv, block_expert.results[0]); - blocks.yield(.{}); - fill.yieldElse(.{}); + var sums = b.openFor(zero, 8, 1, .{ zero, zero }); + { + const value = waves.at(sums.iv); + const preceding = sums.carried[0].add(sums.iv.cmpUnsigned(.lt, wave).select(value, zero)); + const total = sums.carried[1].add(value); + sums.yield(.{ preceding, total }); + } + const end = sums.results[0].add(prefix); + const start = end.sub(padded); + ends.set(tid, end); + var has_expert = b.openIf(tid.cmpUnsigned(.lt, cfg.experts)); + { + args.cumsum.set(tid64, start); + has_expert.close(); + } + var first = b.openIf(tid.eq(0)); + { + args.cumsum.set(b.constant(.i64, cfg.experts), sums.results[1]); + args.num_tokens_post_pad.set(b.constant(.i64, 0), sums.results[1]); + first.close(); + } + b.barrier(); + + var blocks = b.openFor(tid64, cfg.blocks, 512, .{}); + { + const offset = blocks.iv.to(.i32).mul(cfg.block_size); + var block_expert = b.openIfElse(offset.cmpUnsigned(.lt, sums.results[1]), .{zero}); + { + // Upper bound of inclusive expert ends handles empty experts without a scan. + var bounds = b.openWhile(.{ zero, experts }, .{ zero, zero }); + { + bounds.yieldBefore(bounds.before_carried[0].cmpUnsigned(.lt, bounds.before_carried[1]), .{ bounds.before_carried[0], bounds.before_carried[1] }); + } + { + const lo = bounds.after_carried[0]; + const hi = bounds.after_carried[1]; + const mid = lo.add(hi).shrU(1); + const advance = ends.at(mid).cmpUnsigned(.le, offset); + bounds.yieldAfter(.{ advance.select(mid.add(1), lo), advance.select(hi, mid) }); + } + block_expert.yieldThen(.{bounds.results[0]}); + } + { + block_expert.yieldElse(.{b.constant(.i32, -1)}); + } + args.expert_ids.set(blocks.iv, block_expert.results[0]); + blocks.close(); + } + fill.close(); + } } }; @@ -192,13 +219,17 @@ pub const Sort = struct { const args = Kernel.args(b); const start = b.blockId(.x).to(.i64).mul(256).add(b.threadId(.x).to(.i64)); var routes = b.openFor(start, cfg.routes, cfg.step, .{}); - const id = args.ids.at(routes.iv); - var valid = b.openIf(id.cmp(.ge, 0).bitAnd(id.cmp(.lt, cfg.experts))); - const cursor = args.cumsum.emitIter().addOffset(id.to(.i64)); - const rank = b.ptrAtomicAdd(cursor, b.constant(.i32, 1), .agent); - args.sorted_token_ids.set(rank.to(.i64), routes.iv.to(.i32)); - valid.yieldThen(.{}); - routes.yield(.{}); + { + const id = args.ids.at(routes.iv); + var valid = b.openIf(id.ge(0).bitAnd(id.lt(cfg.experts))); + { + const cursor = args.cumsum.emitIter().addOffset(id.to(.i64)); + const rank = b.ptrAtomicAdd(cursor, b.constant(.i32, 1), .agent); + args.sorted_token_ids.set(rank.to(.i64), routes.iv.to(.i32)); + valid.close(); + } + routes.close(); + } } }; @@ -325,143 +356,175 @@ pub const Mfma = struct { const route_start = block_m.mul(cfg.native_m); const has_rows = if (cfg.naive != 0) b.constant(.i1, 1) else blk: { const padded = args.padded.at(b.constant(.i64, 0)).to(.i64); - var has_block = b.openIfElse(route_start.cmpUnsigned(.lt, padded), .{b.constant(.i1, 0).type_()}); - has_block.yieldThen(.{args.sorted.at(route_start).cmp(.ne, cfg.routes)}); - has_block.yieldElse(.{b.constant(.i1, 0)}); + var has_block = b.openIfElse(route_start.cmpUnsigned(.lt, padded), .{.i1}); + { + has_block.yieldThen(.{args.sorted.at(route_start).ne(cfg.routes)}); + } + { + has_block.yieldElse(.{b.constant(.i1, 0)}); + } break :blk has_block.results[0]; }; var block = b.openIf(has_rows); - const expert_block = if (cfg.naive != 0) block_m else block_m.divUnsigned(cfg.subtiles); - const eid = args.experts.at(expert_block); - const valid_expert = eid.cmp(.ge, 0).bitAnd(eid.cmp(.lt, cfg.experts)); - const thread = b.threadId(.x).to(.i64); - const lane = thread.remUnsigned(64); - const wave = thread.divUnsigned(64); - const mn_lane = lane.remUnsigned(16); - const k_lane = lane.divUnsigned(16); - const out_n = block_n.mul(64).add(wave.mul(16)).add(mn_lane); - // Each expert has a contiguous real prefix before route-count padding. - const second_active = if (cfg.wide != 0) blk: { - const first = args.sorted.at(route_start.add(16)); - break :blk first.cmp(.ge, 0).bitAnd(first.cmp(.lt, cfg.routes)); - } else b.constant(.i1, 0); + { + const expert_block = if (cfg.naive != 0) block_m else block_m.divUnsigned(cfg.subtiles); + const eid = args.experts.at(expert_block); + const valid_expert = eid.ge(0).bitAnd(eid.lt(cfg.experts)); + const thread = b.threadId(.x).to(.i64); + const lane = thread.remUnsigned(64); + const wave = thread.divUnsigned(64); + const mn_lane = lane.remUnsigned(16); + const k_lane = lane.divUnsigned(16); + const out_n = block_n.mul(64).add(wave.mul(16)).add(mn_lane); + // Each expert has a contiguous real prefix before route-count padding. + const second_active = if (cfg.wide != 0) blk: { + const first = args.sorted.at(route_start.add(16)); + break :blk first.ge(0).bitAnd(first.lt(cfg.routes)); + } else b.constant(.i1, 0); - var sums = b.openIfElse(valid_expert, .{ zeros.type_(), zeros.type_() }); - const expert = eid.to(.i64); - const load_row = thread.divUnsigned(16); - const load_chunk = thread.remUnsigned(16); - const a_route = routeForRow(b, args.sorted, block_m, route_start, load_row, cfg.naive != 0); - const a_valid = a_route.cmp(.ge, 0).bitAnd(a_route.cmp(.lt, cfg.routes)); - const a_base = a_route.to(.i64).divUnsigned(cfg.topk).mul(cfg.ka); - var second_route = b.openIfElse(second_active, .{zero.type_()}); - second_route.yieldThen(.{args.sorted.at(route_start.add(load_row).add(16))}); - second_route.yieldElse(.{b.constant(.i32, -1)}); - const a_route_g1 = second_route.results[0]; - const a_valid_g1 = a_route_g1.cmp(.ge, 0).bitAnd(a_route_g1.cmp(.lt, cfg.routes)); - const a_base_g1 = a_route_g1.to(.i64).divUnsigned(cfg.topk).mul(cfg.ka); + var sums = b.openIfElse(valid_expert, .{ zeros, zeros }); + { + const expert = eid.to(.i64); + const load_row = thread.divUnsigned(16); + const load_chunk = thread.remUnsigned(16); + const a_route = routeForRow(b, args.sorted, block_m, route_start, load_row, cfg.naive != 0); + const a_valid = a_route.ge(0).bitAnd(a_route.lt(cfg.routes)); + const a_base = a_route.to(.i64).divUnsigned(cfg.topk).mul(cfg.ka); + var second_route = b.openIfElse(second_active, .{zero}); + { + second_route.yieldThen(.{args.sorted.at(route_start.add(load_row).add(16))}); + } + { + second_route.yieldElse(.{b.constant(.i32, -1)}); + } + const a_route_g1 = second_route.results[0]; + const a_valid_g1 = a_route_g1.ge(0).bitAnd(a_route_g1.lt(cfg.routes)); + const a_base_g1 = a_route_g1.to(.i64).divUnsigned(cfg.topk).mul(cfg.ka); - // One allocation, two disjoint ping-pong regions; never two identical leaves. - const shared = b.sharedArray(.i16, cfg.shared_elements, 16); - const load_a = b.copyAtom(.{ .universal = 128 }, .i64); - const load_b = b.copyAtom(.{ .universal = 128 }, .i32); - const copy_shared = b.copyAtom(.{ .universal = 128 }, .i16); - const loader: TileLoader = .{ - .a = args.a, - .weight = args.b, - .scales = args.scales, - .shared = shared, - .load_a = load_a, - .load_b = load_b, - .copy_shared = copy_shared, - .a_base = a_base, - .a_base_g1 = a_base_g1, - .a_valid = a_valid, - .a_valid_g1 = a_valid_g1, - .second_active = second_active, - .a_col = load_chunk.mul(2), - .a_store = load_row.mul(128).add(load_chunk.bitXor(load_row).mul(8)), - .b_base = expert.mul(cfg.n * cfg.kb).add(out_n.mul(cfg.kb)), - .expert = expert, - .out_n = out_n, - .k_lane = k_lane, - }; - const initial = loader.load(b, b.constant(.i64, 0), b.constant(.i64, 0)); - b.barrier(); - const atom = b.mmaAtom((fly.fly_rocdl.MmaOpCDNA3MFMAType.get(b.ctx, .{ - .m = 16, - .n = 16, - .k = 16, - .elemTyA = fly.DType.bf16.toMlir(b.ctx), - .elemTyB = fly.DType.bf16.toMlir(b.ctx), - .elemTyAcc = fly.DType.f32.toMlir(b.ctx), - }) catch return error.InvalidMlir).type_()); - var k = b.openFor(b.constant(.i64, 0), cfg.k, 128, .{ zeros, zeros, initial.words, initial.scale }); - const read_base = k.iv.divUnsigned(128).bitAnd(1).mul(cfg.stage_elements); - const write_base = k.iv.divUnsigned(128).bitAnd(1).bitXor(1).mul(cfg.stage_elements); - var a_fragments: [4]Value = undefined; - var a_fragments_g1: [4]Value = undefined; - inline for (0..4) |u| { - const offset = read_base.add(mn_lane.mul(128)).add(k_lane.mul(4).add(u).bitXor(mn_lane).mul(8)); - a_fragments[u] = b.copyAtomLoad(copy_shared, vectorView(shared, offset, 8)); - } - inline for (0..4) |u| { - const offset = read_base.add(mn_lane.mul(128)).add(k_lane.mul(4).add(u).bitXor(mn_lane).mul(8)).add(2048); - const azero = b.constant(.i16, 0).splat(8); - var second = b.openIfElse(second_active, .{azero.type_()}); - second.yieldThen(.{b.copyAtomLoad(copy_shared, vectorView(shared, offset, 8))}); - second.yieldElse(.{azero}); - a_fragments_g1[u] = second.results[0]; - } - // Prefetch the other LDS region only after all current A fragments are read. - const next_k = k.iv.add(128); - var next = b.openIfElse(next_k.cmpUnsigned(.lt, cfg.k), .{ initial.words.type_(), initial.scale.type_() }); - const next_tile = loader.load(b, next_k, write_base); - next.yieldThen(.{ next_tile.words, next_tile.scale }); - next.yieldElse(.{ b.constant(.i32, 0).splat(4), fzero }); - var acc = k.carried[0]; - var acc_g1 = k.carried[1]; - inline for (0..4) |u| { - const decoded = decodeWeight(b, k.carried[2].extract(u), k.carried[3]); - const a0 = a_fragments[u].shuffleVector(a_fragments[u], &.{ 0, 1, 2, 3 }).bitcast(.bf16); - const a1 = a_fragments[u].shuffleVector(a_fragments[u], &.{ 4, 5, 6, 7 }).bitcast(.bf16); - const w0 = decoded.shuffleVector(decoded, &.{ 0, 1, 2, 3 }); - const w1 = decoded.shuffleVector(decoded, &.{ 4, 5, 6, 7 }); - acc = b.mmaAtomCall(atom, a0, w0, acc); - acc = b.mmaAtomCall(atom, a1, w1, acc); - var second = b.openIfElse(second_active, .{zeros.type_()}); - const a0_g1 = a_fragments_g1[u].shuffleVector(a_fragments_g1[u], &.{ 0, 1, 2, 3 }).bitcast(.bf16); - const a1_g1 = a_fragments_g1[u].shuffleVector(a_fragments_g1[u], &.{ 4, 5, 6, 7 }).bitcast(.bf16); - const first_acc = b.mmaAtomCall(atom, a0_g1, w0, acc_g1); - second.yieldThen(.{b.mmaAtomCall(atom, a1_g1, w1, first_acc)}); - second.yieldElse(.{acc_g1}); - acc_g1 = second.results[0]; - } - // All current reads finish before a later iteration reuses this LDS region. - b.barrier(); - k.yield(.{ acc, acc_g1, next.results[0], next.results[1] }); - sums.yieldThen(.{ k.results[0], k.results[1] }); - sums.yieldElse(.{ zeros, zeros }); + // One allocation, two disjoint ping-pong regions; never two identical leaves. + const shared = b.sharedArray(.i16, cfg.shared_elements, 16); + const load_a = b.copyAtom(.{ .universal = 128 }, .i64); + const load_b = b.copyAtom(.{ .universal = 128 }, .i32); + const copy_shared = b.copyAtom(.{ .universal = 128 }, .i16); + const loader: TileLoader = .{ + .a = args.a, + .weight = args.b, + .scales = args.scales, + .shared = shared, + .load_a = load_a, + .load_b = load_b, + .copy_shared = copy_shared, + .a_base = a_base, + .a_base_g1 = a_base_g1, + .a_valid = a_valid, + .a_valid_g1 = a_valid_g1, + .second_active = second_active, + .a_col = load_chunk.mul(2), + .a_store = load_row.mul(128).add(load_chunk.bitXor(load_row).mul(8)), + .b_base = expert.mul(cfg.n * cfg.kb).add(out_n.mul(cfg.kb)), + .expert = expert, + .out_n = out_n, + .k_lane = k_lane, + }; + const initial = loader.load(b, b.constant(.i64, 0), b.constant(.i64, 0)); + b.barrier(); + const atom = b.mmaAtom((fly.fly_rocdl.MmaOpCDNA3MFMAType.get(b.ctx, .{ + .m = 16, + .n = 16, + .k = 16, + .elemTyA = fly.DType.bf16.toMlir(b.ctx), + .elemTyB = fly.DType.bf16.toMlir(b.ctx), + .elemTyAcc = fly.DType.f32.toMlir(b.ctx), + }) catch return error.InvalidMlir).type_()); + var k = b.openFor(b.constant(.i64, 0), cfg.k, 128, .{ zeros, zeros, initial.words, initial.scale }); + { + const read_base = k.iv.divUnsigned(128).bitAnd(1).mul(cfg.stage_elements); + const write_base = k.iv.divUnsigned(128).bitAnd(1).bitXor(1).mul(cfg.stage_elements); + var a_fragments: [4]Value = undefined; + var a_fragments_g1: [4]Value = undefined; + inline for (0..4) |u| { + const offset = read_base.add(mn_lane.mul(128)).add(k_lane.mul(4).add(u).bitXor(mn_lane).mul(8)); + a_fragments[u] = b.copyAtomLoad(copy_shared, vectorView(shared, offset, 8)); + } + inline for (0..4) |u| { + const offset = read_base.add(mn_lane.mul(128)).add(k_lane.mul(4).add(u).bitXor(mn_lane).mul(8)).add(2048); + const azero = b.constant(.i16, 0).splat(8); + var second = b.openIfElse(second_active, .{azero}); + { + second.yieldThen(.{b.copyAtomLoad(copy_shared, vectorView(shared, offset, 8))}); + } + { + second.yieldElse(.{azero}); + } + a_fragments_g1[u] = second.results[0]; + } + // Prefetch the other LDS region only after all current A fragments are read. + const next_k = k.iv.add(128); + var next = b.openIfElse(next_k.cmpUnsigned(.lt, cfg.k), .{ initial.words, initial.scale }); + { + const next_tile = loader.load(b, next_k, write_base); + next.yieldThen(.{ next_tile.words, next_tile.scale }); + } + { + next.yieldElse(.{ b.constant(.i32, 0).splat(4), fzero }); + } + var acc = k.carried[0]; + var acc_g1 = k.carried[1]; + inline for (0..4) |u| { + const decoded = decodeWeight(b, k.carried[2].extract(u), k.carried[3]); + const a0 = a_fragments[u].shuffleVector(a_fragments[u], &.{ 0, 1, 2, 3 }).bitCast(.bf16); + const a1 = a_fragments[u].shuffleVector(a_fragments[u], &.{ 4, 5, 6, 7 }).bitCast(.bf16); + const w0 = decoded.shuffleVector(decoded, &.{ 0, 1, 2, 3 }); + const w1 = decoded.shuffleVector(decoded, &.{ 4, 5, 6, 7 }); + acc = b.mmaAtomCall(atom, a0, w0, acc); + acc = b.mmaAtomCall(atom, a1, w1, acc); + var second = b.openIfElse(second_active, .{zeros}); + { + const a0_g1 = a_fragments_g1[u].shuffleVector(a_fragments_g1[u], &.{ 0, 1, 2, 3 }).bitCast(.bf16); + const a1_g1 = a_fragments_g1[u].shuffleVector(a_fragments_g1[u], &.{ 4, 5, 6, 7 }).bitCast(.bf16); + const first_acc = b.mmaAtomCall(atom, a0_g1, w0, acc_g1); + second.yieldThen(.{b.mmaAtomCall(atom, a1_g1, w1, first_acc)}); + } + { + second.yieldElse(.{acc_g1}); + } + acc_g1 = second.results[0]; + } + // All current reads finish before a later iteration reuses this LDS region. + b.barrier(); + k.yield(.{ acc, acc_g1, next.results[0], next.results[1] }); + } + sums.yieldThen(.{ k.results[0], k.results[1] }); + } + { + sums.yieldElse(.{ zeros, zeros }); + } - inline for (0..4) |i| { - storeRow(b, args.output, args.sorted, block_m, route_start, k_lane.mul(4).add(i), out_n, sums.results[0].extract(i), cfg); + inline for (0..4) |i| { + storeRow(b, args.output, args.sorted, block_m, route_start, k_lane.mul(4).add(i), out_n, sums.results[0].extract(i), cfg); + } + var store_second = b.openIf(second_active); + { + inline for (0..4) |i| { + storeRow(b, args.output, args.sorted, block_m, route_start, k_lane.mul(4).add(i + 16), out_n, sums.results[1].extract(i), cfg); + } + store_second.close(); + } + block.close(); } - var store_second = b.openIf(second_active); - inline for (0..4) |i| { - storeRow(b, args.output, args.sorted, block_m, route_start, k_lane.mul(4).add(i + 16), out_n, sums.results[1].extract(i), cfg); - } - store_second.yieldThen(.{}); - block.yieldThen(.{}); } fn routeForRow(b: *Builder, sorted: Value, block: Value, start: Value, row: Value, naive: bool) Value { - return if (naive) row.cmp(.eq, 0).select(block.to(.i32), b.constant(.i32, -1)) else sorted.at(start.add(row)); + return if (naive) row.eq(0).select(block.to(.i32), b.constant(.i32, -1)) else sorted.at(start.add(row)); } fn storeRow(b: *Builder, output: Value, sorted: Value, block: Value, start: Value, row: Value, col: Value, value: Value, cfg: Config) void { const route = routeForRow(b, sorted, block, start, row, cfg.naive != 0); - var valid = b.openIf(route.cmp(.ge, 0).bitAnd(route.cmp(.lt, cfg.routes))); - output.set(.{ route.to(.i64), col }, value.to(.bf16)); - valid.yieldThen(.{}); + var valid = b.openIf(route.ge(0).bitAnd(route.lt(cfg.routes))); + { + output.set(.{ route.to(.i64), col }, value.to(.bf16)); + valid.close(); + } } fn vectorView(tensor: Value, offset: Value, comptime n: i64) Value { @@ -491,23 +554,29 @@ pub const Mfma = struct { fn load(self: TileLoader, b: *Builder, k: Value, write_base: Value) struct { words: Value, scale: Value } { self.storeActivation(b, self.a_base, self.a_valid, k, self.a_store.add(write_base)); var second = b.openIf(self.second_active); - self.storeActivation(b, self.a_base_g1, self.a_valid_g1, k, self.a_store.add(write_base).add(2048)); - second.yieldThen(.{}); + { + self.storeActivation(b, self.a_base_g1, self.a_valid_g1, k, self.a_store.add(write_base).add(2048)); + second.close(); + } const offset = self.b_base.add(k.divUnsigned(8)).add(self.k_lane.mul(4)); const words = b.copyAtomLoad(self.load_b, vectorView(self.weight, offset, 4)); const byte = self.scales.at(.{ self.expert, self.out_n, k.divUnsigned(32).add(self.k_lane) }); const scale = byte.toUnsigned(.i32); - const finite = scale.cmp(.eq, 0).select(b.constant(.i32, 0x00400000), scale.shl(23)); - return .{ .words = words, .scale = scale.cmp(.eq, 255).select(b.constant(.i32, 0x7fc00000), finite).bitcast(.f32) }; + const finite = scale.eq(0).select(b.constant(.i32, 0x00400000), scale.shl(23)); + return .{ .words = words, .scale = scale.eq(255).select(b.constant(.i32, 0x7fc00000), finite).bitCast(.f32) }; } fn storeActivation(self: TileLoader, b: *Builder, base: Value, valid: Value, k: Value, destination: Value) void { const zeros = b.constant(.i64, 0).splat(2); - var guarded = b.openIfElse(valid, .{zeros.type_()}); - const offset = base.add(k.divUnsigned(4)).add(self.a_col); - guarded.yieldThen(.{b.copyAtomLoad(self.load_a, vectorView(self.a, offset, 2))}); - guarded.yieldElse(.{zeros}); - b.copyAtomStore(self.copy_shared, guarded.results[0].bitcast(.i16), vectorView(self.shared, destination, 8)); + var guarded = b.openIfElse(valid, .{zeros}); + { + const offset = base.add(k.divUnsigned(4)).add(self.a_col); + guarded.yieldThen(.{b.copyAtomLoad(self.load_a, vectorView(self.a, offset, 2))}); + } + { + guarded.yieldElse(.{zeros}); + } + b.copyAtomStore(self.copy_shared, guarded.results[0].bitCast(.i16), vectorView(self.shared, destination, 8)); } }; @@ -527,7 +596,7 @@ pub const Mfma = struct { inline for (0..8) |i| shifts = shifts.insert(i, b.constant(.i32, i * 4)); const signs = word.splat(8).shrU(shifts).bitAnd(8).shl(28); // FNUZ has no -0: restore FP4's sign in F32 before scaling and BF16 rounding. - return magnitude.bitcast(.i32).bitOr(signs).bitcast(.f32).mul(scale).to(.bf16); + return magnitude.bitCast(.i32).bitOr(signs).bitCast(.f32).mul(scale).to(.bf16); } }; @@ -606,61 +675,73 @@ const Gemv = struct { const expert32 = args.ids.at(route); const expert = expert32.to(.i64); const token = route.divUnsigned(cfg.topk); - const local = expert32.cmp(.ge, 0).bitAnd(expert32.cmp(.lt, cfg.experts)); + const local = expert32.ge(0).bitAnd(expert32.lt(cfg.experts)); const inrow = row.cmpUnsigned(.lt, cfg.n); const fzero = b.constant(.f32, 0.0); - var sum = b.openIfElse(local.bitAnd(inrow), .{fzero.type_()}); - var k = b.openFor(lane, cfg.k_words, cfg.stride, .{fzero}); - var carried = k.carried[0]; - for (0..cfg.unroll) |u| { - const pos = k.iv.add(u * cfg.lanes); - var inbounds = b.openIfElse(pos.cmpUnsigned(.lt, cfg.k_words), .{fzero.type_()}); - const word = args.b.at(.{ expert, row, pos }); - const scale8 = args.scales.at(.{ expert, row, pos.divUnsigned(4) }); - const scale = decodeScale(b, scale8); - const low = word.bitAnd(0x07070707); - const high = word.shrU(4).bitAnd(0x07070707); - const even_bits = b.perm(b.constant(.i32, 0xd4d0ccc8), b.constant(.i32, 0xc4c0b800), low); - const odd_bits = b.perm(b.constant(.i32, 0xd4d0ccc8), b.constant(.i32, 0xc4c0b800), high); - const even = even_bits.bitAnd(word.shl(4).bitOr(0x7f7f7f7f)); - const odd = odd_bits.bitAnd(word.bitOr(0x7f7f7f7f)); - // Retain the scalar decoder and its FP32 scale -> BF16 -> FP32 boundary. - const weights = [_]Value{ - b.cvtPkF32Fp8(even, false).mul(scale).to(.bf16).to(.f32), - b.cvtPkF32Fp8(odd, false).mul(scale).to(.bf16).to(.f32), - b.cvtPkF32Fp8(even, true).mul(scale).to(.bf16).to(.f32), - b.cvtPkF32Fp8(odd, true).mul(scale).to(.bf16).to(.f32), - }; - var partial = fzero; - inline for (0..4) |pair| { - const x = args.a.at(.{ token, pos.mul(4).add(pair) }); - const xlo = x.to(.i16).bitcast(.bf16).to(.f32); - const xhi = x.shrU(16).to(.i16).bitcast(.bf16).to(.f32); - const wlo = weights[(pair / 2) * 2].extract(pair % 2); - const whi = weights[(pair / 2) * 2 + 1].extract(pair % 2); - // Odd then even FMA, followed by one add to the K-loop accumulator. - partial = b.fma(xlo, wlo, b.fma(xhi, whi, partial)); + var sum = b.openIfElse(local.bitAnd(inrow), .{fzero}); + { + var k = b.openFor(lane, cfg.k_words, cfg.stride, .{fzero}); + { + var carried = k.carried[0]; + for (0..cfg.unroll) |u| { + const pos = k.iv.add(u * cfg.lanes); + var inbounds = b.openIfElse(pos.cmpUnsigned(.lt, cfg.k_words), .{fzero}); + { + const word = args.b.at(.{ expert, row, pos }); + const scale8 = args.scales.at(.{ expert, row, pos.divUnsigned(4) }); + const scale = decodeScale(b, scale8); + const low = word.bitAnd(0x07070707); + const high = word.shrU(4).bitAnd(0x07070707); + const even_bits = b.perm(b.constant(.i32, 0xd4d0ccc8), b.constant(.i32, 0xc4c0b800), low); + const odd_bits = b.perm(b.constant(.i32, 0xd4d0ccc8), b.constant(.i32, 0xc4c0b800), high); + const even = even_bits.bitAnd(word.shl(4).bitOr(0x7f7f7f7f)); + const odd = odd_bits.bitAnd(word.bitOr(0x7f7f7f7f)); + // Retain the scalar decoder and its FP32 scale -> BF16 -> FP32 boundary. + const weights = [_]Value{ + b.cvtPkF32Fp8(even, false).mul(scale).to(.bf16).to(.f32), + b.cvtPkF32Fp8(odd, false).mul(scale).to(.bf16).to(.f32), + b.cvtPkF32Fp8(even, true).mul(scale).to(.bf16).to(.f32), + b.cvtPkF32Fp8(odd, true).mul(scale).to(.bf16).to(.f32), + }; + var partial = fzero; + inline for (0..4) |pair| { + const x = args.a.at(.{ token, pos.mul(4).add(pair) }); + const xlo = x.to(.i16).bitCast(.bf16).to(.f32); + const xhi = x.shrU(16).to(.i16).bitCast(.bf16).to(.f32); + const wlo = weights[(pair / 2) * 2].extract(pair % 2); + const whi = weights[(pair / 2) * 2 + 1].extract(pair % 2); + // Odd then even FMA, followed by one add to the K-loop accumulator. + partial = b.fma(xlo, wlo, b.fma(xhi, whi, partial)); + } + inbounds.yieldThen(.{partial.add(carried)}); + } + { + inbounds.yieldElse(.{carried}); + } + carried = inbounds.results[0]; + } + k.yield(.{carried}); } - inbounds.yieldThen(.{partial.add(carried)}); - inbounds.yieldElse(.{carried}); - carried = inbounds.results[0]; + var reduced = k.results[0]; + var shift: i32 = 1; + while (shift < @as(i32, @intCast(cfg.lanes))) : (shift *= 2) { + reduced = reduced.add(reduced.shuffleXor(shift, 64)); + } + sum.yieldThen(.{reduced}); } - k.yield(.{carried}); - var reduced = k.results[0]; - var shift: i32 = 1; - while (shift < @as(i32, @intCast(cfg.lanes))) : (shift *= 2) { - reduced = reduced.add(reduced.shuffleXor(shift, 64)); + { + sum.yieldElse(.{fzero}); + } + var store = b.openIf(lane.eq(0).bitAnd(inrow)); + { + args.output.set(.{ route, row }, sum.results[0].to(.bf16)); + store.close(); } - sum.yieldThen(.{reduced}); - sum.yieldElse(.{fzero}); - var store = b.openIf(lane.cmp(.eq, 0).bitAnd(inrow)); - args.output.set(.{ route, row }, sum.results[0].to(.bf16)); - store.yieldThen(.{}); } fn decodeScale(b: *Builder, byte: Value) Value { const scale = byte.toUnsigned(.i32); - const finite = scale.cmp(.eq, 0).select(b.constant(.i32, 0x00400000), scale.shl(23)); - return scale.cmp(.eq, 255).select(b.constant(.i32, 0x7fc00000), finite).bitcast(.f32); + const finite = scale.eq(0).select(b.constant(.i32, 0x00400000), scale.shl(23)); + return scale.eq(255).select(b.constant(.i32, 0x7fc00000), finite).bitCast(.f32); } }; diff --git a/zml/kernels/moe/mosaic_tpu/gmm_ep.zig b/zml/kernels/moe/mosaic_tpu/gmm_ep.zig index 06d71820..c30d805b 100644 --- a/zml/kernels/moe/mosaic_tpu/gmm_ep.zig +++ b/zml/kernels/moe/mosaic_tpu/gmm_ep.zig @@ -14,7 +14,7 @@ fn alignTo(x: i64, y: i64) i64 { } fn floorModI32(b: *mtt.Builder, x: mtt.Value, y: mtt.Value) mtt.Value { - const c0 = b.lift(@as(i32, 0)); + const c0 = b.constant(.i32, 0); const r = b.remsi(x, y); const r_nz = b.cmpi(.ne, r, c0); const r_neg = b.cmpi(.slt, r, c0); @@ -22,8 +22,8 @@ fn floorModI32(b: *mtt.Builder, x: mtt.Value, y: mtt.Value) mtt.Value { } fn floorDivPosI32(b: *mtt.Builder, x: mtt.Value, y: mtt.Value) mtt.Value { - const c0 = b.lift(@as(i32, 0)); - const c1 = b.lift(@as(i32, 1)); + const c0 = b.constant(.i32, 0); + const c1 = b.constant(.i32, 1); const q = b.divsi(x, y); const sign = b.subi( b.extui(b.cmpi(.sgt, x, c0), .i32), @@ -35,7 +35,7 @@ fn floorDivPosI32(b: *mtt.Builder, x: mtt.Value, y: mtt.Value) mtt.Value { } fn ceilDivPosI32(b: *mtt.Builder, x: mtt.Value, y: mtt.Value) mtt.Value { - return b.divsi(b.addi(x, b.subi(y, b.lift(@as(i32, 1)))), y); + return b.divsi(b.addi(x, b.subi(y, b.constant(.i32, 1))), y); } pub const Cfg = struct { @@ -129,7 +129,7 @@ fn winRef(b: *mtt.Builder, rw: pipeline.RefWindow, shape: []const i64) mtt.Value base[0] = slot; result_shape[0] = 1; for (shape, 0..) |d, i| { - base[i + 1] = b.lift(@as(i32, 0)); + base[i + 1] = b.constant(.i32, 0); result_shape[i + 1] = d; } const sliced = b.memRefSlice(buf, base, result_shape, &.{}); @@ -147,15 +147,15 @@ fn lhsIndexMap(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopa const k_id = indices[2]; const m_start = metaLoad(b, ctx.meta.m_offset, gm_id); - const m_end = metaLoad(b, ctx.meta.m_offset, b.addi(gm_id, b.lift(@as(i32, 1)))); - const sublane = b.lift(@as(i32, @intCast(cfg.size_lhs_sublane))); + const m_end = metaLoad(b, ctx.meta.m_offset, b.addi(gm_id, b.constant(.i32, 1))); + const sublane = b.constant(.i32, cfg.size_lhs_sublane); const row_start = floorDivPosI32(b, m_start, sublane); const row_end = ceilDivPosI32(b, m_end, sublane); const row_size = b.subi(row_end, row_start); const out = b.arena.allocator().alloc(pipeline.BlockIndex, 3) catch @panic("gmm_ep lhsIndexMap OOM"); out[0] = .{ .slice = .{ .start = row_start, .size = row_size } }; - out[1] = .{ .scalar = b.lift(@as(i32, 0)) }; + out[1] = .{ .scalar = b.constant(.i32, 0) }; out[2] = .{ .scalar = k_id }; return out; } @@ -177,13 +177,13 @@ fn outIndexMap(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopa const n_id = indices[0]; const m_start = metaLoad(b, ctx.meta.m_offset, gm_id); - const m_end = metaLoad(b, ctx.meta.m_offset, b.addi(gm_id, b.lift(@as(i32, 1)))); - const sublane = b.lift(@as(i32, @intCast(cfg.size_lhs_sublane))); + const m_end = metaLoad(b, ctx.meta.m_offset, b.addi(gm_id, b.constant(.i32, 1))); + const sublane = b.constant(.i32, cfg.size_lhs_sublane); const row_start = floorDivPosI32(b, m_start, sublane); const capped_row_end = floorDivPosI32(b, m_end, sublane); const last_row_end = ceilDivPosI32(b, m_end, sublane); const row_end = b.select( - b.cmpi(.eq, gm_id, b.subi(ctx.num_gm, b.lift(@as(i32, 1)))), + b.cmpi(.eq, gm_id, b.subi(ctx.num_gm, b.constant(.i32, 1))), last_row_end, capped_row_end, ); @@ -191,16 +191,16 @@ fn outIndexMap(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopa const out = b.arena.allocator().alloc(pipeline.BlockIndex, 3) catch @panic("gmm_ep outIndexMap OOM"); out[0] = .{ .slice = .{ .start = row_start, .size = row_size } }; - out[1] = .{ .scalar = b.lift(@as(i32, 0)) }; + out[1] = .{ .scalar = b.constant(.i32, 0) }; out[2] = .{ .scalar = n_id }; return out; } fn indices1DTo3D(b: *mtt.Builder, indices: []const mtt.Value) []const mtt.Value { const out = b.arena.allocator().alloc(mtt.Value, 3) catch @panic("gmm_ep indices1DTo3D OOM"); - out[0] = b.lift(@as(i32, 0)); + out[0] = b.constant(.i32, 0); out[1] = indices[0]; - out[2] = b.lift(@as(i32, 0)); + out[2] = b.constant(.i32, 0); return out; } @@ -217,11 +217,11 @@ fn outIndexMap1D(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyo } fn fillMetadata(b: *mtt.Builder, cfg: Cfg, a: anytype) mtt.Value { - const c0 = b.lift(@as(i32, 0)); - const c1 = b.lift(@as(i32, 1)); - const group_offset = b.scalarLoad(a.group_offset, &.{b.cIndex(0)}); - const max_num_group = b.addi(group_offset, b.lift(@as(i32, @intCast(cfg.size_group)))); - b.scalarStore(a.metadata_m_offset, c0, &.{b.cIndex(0)}); + const c0 = b.constant(.i32, 0); + const c1 = b.constant(.i32, 1); + const group_offset = b.scalarLoad(a.group_offset, &.{b.constIndex(0)}); + const max_num_group = b.addi(group_offset, b.constant(.i32, cfg.size_group)); + b.scalarStore(a.metadata_m_offset, c0, &.{b.constIndex(0)}); var outer = b.openFor(c0, max_num_group, c1, .{ c0, c0 }); { @@ -232,10 +232,10 @@ fn fillMetadata(b: *mtt.Builder, cfg: Cfg, a: anytype) mtt.Value { const group_id = b.subi(lhs_group_id, group_offset); const group_size = b.scalarLoad(a.group_sizes, &.{b.toIndex(lhs_group_id)}); const end_m_offset = b.addi(start_m_offset, group_size); - const sublane = b.lift(@as(i32, @intCast(cfg.size_lhs_sublane))); + const sublane = b.constant(.i32, cfg.size_lhs_sublane); const local_offset = floorModI32(b, start_m_offset, sublane); const aligned_group_size = b.addi(group_size, local_offset); - const tile_m = b.lift(@as(i32, @intCast(cfg.tile_m))); + const tile_m = b.constant(.i32, cfg.tile_m); var curr_num_gm = b.divsi(b.addi(aligned_group_size, b.subi(tile_m, c1)), tile_m); const should_process = b.andi(b.cmpi(.sgt, group_size, c0), b.cmpi(.sge, group_id, c0)); curr_num_gm = b.select(should_process, curr_num_gm, c0); @@ -407,7 +407,7 @@ fn innerBody( const lhs = b.refLoad(lhs_ref_2d); const rhs_slot = b.toIndex(refs[1].slot.?); var rhs = b.shapeCast( - b.vectorLoadShape(refs[1].buf.?, &.{ rhs_slot, b.cIndex(0), b.cIndex(0) }, &.{ 1, cfg.tile_k, rhs_tile_n }), + b.vectorLoadShape(refs[1].buf.?, &.{ rhs_slot, b.constIndex(0), b.constIndex(0) }, &.{ 1, cfg.tile_k, rhs_tile_n }), &.{ cfg.tile_k, rhs_tile_n }, ); const valid_k = @mod(cfg.size_k, cfg.tile_k); @@ -420,8 +420,8 @@ fn innerBody( acc = applyFuseAct(b, cfg, acc); const m_start = metaLoad(b, ctx.meta.m_offset, gm_id); - const m_end = metaLoad(b, ctx.meta.m_offset, b.addi(gm_id, b.lift(@as(i32, 1)))); - const sublane = b.lift(@as(i32, @intCast(cfg.size_lhs_sublane))); + const m_end = metaLoad(b, ctx.meta.m_offset, b.addi(gm_id, b.constant(.i32, 1))); + const sublane = b.constant(.i32, cfg.size_lhs_sublane); const m_offset = b.subi(m_start, floorModI32(b, m_start, sublane)); const m_start_local = b.subi(m_start, m_offset); const m_end_local = b.subi(m_end, m_offset); @@ -437,7 +437,7 @@ fn innerBody( b.vectorStoreAt( refs[2].buf.?, b.shapeCast(b.shapeCast(acc, out_shape).to(cfg.out_dtype), &.{ 1, cdiv(cfg.tile_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.tile_n }), - &.{ out_slot, b.cIndex(0), b.cIndex(0), b.cIndex(0) }, + &.{ out_slot, b.constIndex(0), b.constIndex(0), b.constIndex(0) }, ); const out_ref = winRef(b, refs[2], out_shape); @@ -445,7 +445,7 @@ fn innerBody( const out_value = b.shapeCast(b.refLoad(out_ref), &.{ cfg.tile_m, cfg.tile_n }); const partial_zero = b.zeros(&.{ cfg.tile_m, cfg.tile_n }, cfg.out_dtype); const partial_prev = b.select( - b.cmpi(.eq, gm_id, b.lift(@as(i32, 0))), + b.cmpi(.eq, gm_id, b.constant(.i32, 0)), partial_zero, b.refLoad(scratches[0]), ); @@ -454,12 +454,12 @@ fn innerBody( b.refStore(out_ref_for_store, b.shapeCast(out_accum, out_shape)); } else { const first_row = b.shapeCast( - b.vectorLoadShape(out_ref, &.{ b.cIndex(0), b.cIndex(0), b.cIndex(0) }, &.{ 1, cfg.size_lhs_sublane, cfg.tile_n }), + b.vectorLoadShape(out_ref, &.{ b.constIndex(0), b.constIndex(0), b.constIndex(0) }, &.{ 1, cfg.size_lhs_sublane, cfg.tile_n }), &.{ cfg.size_lhs_sublane, cfg.tile_n }, ); const partial_zero = b.zeros(&.{ cfg.size_lhs_sublane, cfg.tile_n }, cfg.out_dtype); const partial_prev = b.select( - b.cmpi(.eq, gm_id, b.lift(@as(i32, 0))), + b.cmpi(.eq, gm_id, b.constant(.i32, 0)), partial_zero, b.refLoad(scratches[0]), ); @@ -468,13 +468,13 @@ fn innerBody( b.vectorStoreAt( out_ref_for_store, b.shapeCast(out_accum, &.{ 1, cfg.size_lhs_sublane, cfg.tile_n }), - &.{ b.cIndex(0), b.cIndex(0), b.cIndex(0) }, + &.{ b.constIndex(0), b.constIndex(0), b.constIndex(0) }, ); } const last_row = floorDivPosI32(b, m_end_local, sublane); const m_end_mod = floorModI32(b, m_end_local, sublane); - const partial_complete = b.cmpi(.eq, m_end_mod, b.lift(@as(i32, 0))); + const partial_complete = b.cmpi(.eq, m_end_mod, b.constant(.i32, 0)); const out_ref_for_partial = winRef(b, refs[2], out_shape); const partial_shape = if (cfg.tile_m == cfg.size_lhs_sublane) &.{ cfg.tile_m, cfg.tile_n } @@ -482,7 +482,7 @@ fn innerBody( &.{ cfg.size_lhs_sublane, cfg.tile_n }; const partial_zero = b.zeros(partial_shape, cfg.out_dtype); const partial_row = b.shapeCast( - b.vectorLoadShape(out_ref_for_partial, &.{ b.toIndex(last_row), b.cIndex(0), b.cIndex(0) }, &.{ 1, cfg.size_lhs_sublane, cfg.tile_n }), + b.vectorLoadShape(out_ref_for_partial, &.{ b.toIndex(last_row), b.constIndex(0), b.constIndex(0) }, &.{ 1, cfg.size_lhs_sublane, cfg.tile_n }), partial_shape, ); b.refStore(scratches[0], b.select(partial_complete, partial_zero, partial_row)); @@ -518,8 +518,8 @@ pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { .scratch_operands = 4, }); - const num_k = b.lift(@as(i32, @intCast(cdiv(cfg.size_k, cfg.tile_k)))); - const num_n = b.lift(@as(i32, @intCast(cdiv(cfg.alignedN(), cfg.tile_n)))); + const num_k = b.constant(.i32, cdiv(cfg.size_k, cfg.tile_k)); + const num_n = b.constant(.i32, cdiv(cfg.alignedN(), cfg.tile_n)); const num_gm = fillMetadata(b, cfg, a); const ctx = b.arena.allocator().create(PipeCtx) catch @panic("gmm_ep PipeCtx OOM"); diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig index 206dcf66..4b7bcfb3 100644 --- a/zml/kernels/moe/tcl/tcl.zig +++ b/zml/kernels/moe/tcl/tcl.zig @@ -236,16 +236,18 @@ pub const Blockwise = tcl.Kernel(Cfg, .{ const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); var loop = b.openFor(used, .{try b.scratchpad(.bf16, &.{ Go, BGi, BB, D })}); - const chunk = try b.indexRead(rows, loop.iv); - const ids = try b.indexRead(chunk_experts, loop.iv); - const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); - const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); - op = b.tensorOperation(.{}); - const h = try op.commit(op.contract(chunk, gu, &.{ BGi, BB, a.S2, I }), .{}); - const act = try gated(b, a, h, &.{ BGi, BB }, null); - op = b.tensorOperation(.{}); - const y = try op.commit(op.contract(act, dn, &.{ BGi, BB, D }), .{ .dtype = .bf16 }); - try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); + { + const chunk = try b.indexRead(rows, loop.iv); + const ids = try b.indexRead(chunk_experts, loop.iv); + const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); + const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); + op = b.tensorOperation(.{}); + const h = try op.commit(op.contract(chunk, gu, &.{ BGi, BB, a.S2, I }), .{}); + const act = try gated(b, a, h, &.{ BGi, BB }, null); + op = b.tensorOperation(.{}); + const y = try op.commit(op.contract(act, dn, &.{ BGi, BB, D }), .{ .dtype = .bf16 }); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); + } const out = loop.results[0]; const per_route = try b.gather(try b.reshape(out, &.{ GB, D }, .{}), slot, GB, &.{ T, K, D }, .{}); diff --git a/zml/kernels/moe/triton/moe.zig b/zml/kernels/moe/triton/moe.zig index 0d2bab5b..3eb457da 100644 --- a/zml/kernels/moe/triton/moe.zig +++ b/zml/kernels/moe/triton/moe.zig @@ -116,10 +116,10 @@ pub const PerTokenGroupQuantFp8 = struct { // It is unclear whether power-of-two scaling is best for this kernel, // but it keeps activation quantization consistent with that function. // Round up from the FP32 bits, including values just above a power of two. - const bits = b.bitcast(scale_raw, .i32); + const bits = scale_raw.bitCast(.i32); const exponent = bits.div(0x800000); const fractional = bits.bitAnd(0x7fffff).ne(0).to(.i32); - const scale = b.bitcast(exponent.add(fractional).mul(0x800000), .f32); + const scale = exponent.add(fractional).mul(0x800000).bitCast(.f32); break :blk .{ scale, scale.to(scale_dt) }; }, .mxfp4, .nvfp4 => unreachable, @@ -315,7 +315,7 @@ pub const FusedMoe = struct { const num_tokens_post_padded = num_tokens_post_padded_i32.to(.i64); const out_of_range = pid_m_block.ge(num_tokens_post_padded); - b.returnIfOpts(out_of_range, .{}, .{ .inline_return = true }); + b.returnIfOpts(out_of_range, .{ .inline_return = true }); const offs_token = if (cfg.naive_block_assignment) b.select( @@ -336,8 +336,7 @@ pub const FusedMoe = struct { const is_dead = off_experts.eq(-1); // Python: if off_experts == -1: write_zeros_to_output(...); return - var dead_ret = b.openReturnIf(is_dead); - dead_ret.inline_return = true; + var dead_ret = b.openReturnIfOpts(is_dead, .{ .inline_return = true }); { writeZerosToOutput( b, @@ -351,7 +350,7 @@ pub const FusedMoe = struct { block_size_n, compute_type, ); - dead_ret.yieldReturn(.{}); + dead_ret.close(); } // CSE `pid_n * BLOCK_SIZE_N` so it reuses across offs_bn / offs_cn. @@ -391,7 +390,7 @@ pub const FusedMoe = struct { // Loop bounds stay i64 because `k_block` is i64. iter_args order // matches Python's TTIR for the matrix and accumulator state. const num_k_iters = k_block.cdiv(block_size_k); - var loop = b.openFor(@as(i64, 0), num_k_iters, @as(i64, 1), .{ a_ptrs_init, b_ptrs_init, acc_init }); + var loop = b.openFor(0, num_k_iters, 1, .{ a_ptrs_init, b_ptrs_init, acc_init }); { const k_iter = loop.iv; const a_ptrs = loop.carried[0]; @@ -682,8 +681,7 @@ pub const MoeAlignBlockSize = struct { const fill_offs = b.arange(0, hist_block, .i32); // if pid == 1: fill sorted_token_ids with NUMEL; return - var fill_branch = b.openReturnIf(pid.eq(1)); - fill_branch.inline_return = true; + var fill_branch = b.openReturnIfOpts(pid.eq(1), .{ .inline_return = true }); { var fill_loop = b.openFor(0, max_num_tokens_padded, hist_block, .{}); { @@ -694,9 +692,9 @@ pub const MoeAlignBlockSize = struct { b.splat(numel, &.{hist_block}), .{ .mask = mask }, ); - fill_loop.yield(.{}); + fill_loop.close(); } - fill_branch.yieldReturn(.{}); + fill_branch.close(); } // pid != 1: histogram + cumsum + block assignment. @@ -742,7 +740,7 @@ pub const MoeAlignBlockSize = struct { const block_ids = starts.div(block_size_m).add(scatter.iv); const mask = expert_mask.bitAnd(block_counts.gt(scatter.iv)); b.storeOpts(a.expert_ids_ptr.addPtr(block_ids), expert_offs, .{ .mask = mask }); - scatter.yield(.{}); + scatter.close(); } } }; @@ -787,7 +785,7 @@ pub const CountAndSortExpertTokens = struct { const token_start_init = pid.mul(block_i32); // while token_start < NUMEL: - var w = b.openWhile(.{token_start_init}, .{b.scalarTy(.i32)}); + var w = b.openWhile(.{token_start_init}, .{.i32}); { const ts = w.before_carried[0]; w.yieldBefore(ts.lt(numel), .{ts}); @@ -843,11 +841,11 @@ fn ar(b: *Builder, n: i64) Value { } fn ld(b: *Builder, p: Value, mask: Value, dt: tri.DType, other: i64) Value { - return b.loadOpts(p, .{ .mask = mask, .other = if (p.isTensor()) b.full(p.shape().constSlice(), other, dt) else b.liftAs(other, dt) }); + return b.loadOpts(p, .{ .mask = mask, .other = if (p.isTensor()) b.full(p.shape().constSlice(), other, dt) else b.constant(dt, other) }); } fn ci(b: *Builder, n: i64) Value { - return b.liftAs(n, .i32); + return b.constant(.i32, n); } fn quant32(b: *Builder, x: Value) struct { q: Value, s: Value } { @@ -863,12 +861,12 @@ fn quant32(b: *Builder, x: Value) struct { q: Value, s: Value } { // SM89 lowers FP32 -> FP8 through FP16 with truncation. Round to odd at // FP16 precision first so that this intermediate cannot turn a value just // above an FP8 midpoint into an exact tie. Direct FP8 conversion is unchanged. - const scaled_bits = b.bitcast(scaled, .i32); + const scaled_bits = scaled.bitCast(.i32); const sticky = b.select(scaled_bits.bitAnd(0x1fff).ne(0), b.full(scaled.shape().constSlice(), 0x2000, .i32), b.zeros(scaled.shape().constSlice(), .i32)); - const prepared = b.bitcast(scaled_bits.bitAnd(-0x2000).bitOr(sticky), .f32); + const prepared = scaled_bits.bitAnd(-0x2000).bitOr(sticky).bitCast(.f32); break :b prepared.to(.f8e4m3fn); }; - const bits = b.shrui(b.bitcast(sf, .i32), b.full(sf.shape().constSlice(), 23, .i32)).bitAnd(255).to(.i8); + const bits = b.shrui(sf.bitCast(.i32), b.full(sf.shape().constSlice(), 23, .i32)).bitAnd(255).to(.i8); return .{ .q = b.reshape(q, &.{ rows, cols }), .s = b.reshape(bits, &.{ rows, groups }) }; } @@ -1000,8 +998,10 @@ pub const Schedule = struct { const ts = counts.cdiv(group); const starts = ts.cumsum().sub(ts); var first = b.openIf(b.programId(.x).eq(0)); - b.store(a.sched.addPtr(2 * nt), ts.sum()); - first.yieldThen(.{}); + { + b.store(a.sched.addPtr(2 * nt), ts.sum()); + first.close(); + } const live = p.lt(cfg.tokens * cfg.topk).bitAnd(e.ge(0)).bitAnd(e.lt(cfg.experts)); const safe = e.maximum(0).minimum(pow2(cfg.experts) - 1); const pos = ld(b, a.pos.addPtr(p), live, .i32, 0); @@ -1109,65 +1109,69 @@ fn gemm(b: *Builder, c: GemmConfig, comptime projection: Projection) tri.FinishE const expert = if (c.tokens == 1) b.load(a.ids.addPtr(tile)).sub(ci(b, c.expert_offset)) else ld(b, a.sched.addPtr(tile), tile.lt(b.load(a.sched.addPtr(2 * nt))), .i32, 0); const valid = if (c.tokens == 1) expert.ge(0).bitAnd(expert.lt(c.experts)) else tile.lt(b.load(a.sched.addPtr(2 * nt))); var live = b.openIf(valid); - const count = if (c.tokens == 1) ci(b, 1) else b.load(a.sched.addPtr(nt).addPtr(tile)); - const pair = if (c.tokens == 1) b.full(&.{c.bn}, 0, .i32).add(tile) else if (is_up) ld(b, a.map.addPtr(tile.mul(c.bn).add(n)), n.lt(count), .i32, 0) else n; - const width: i64 = if (is_up and c.tokens > 8) 256 else 128; - const ksize = if (is_up) c.hidden else c.intermediate; - const rows = if (is_up) 2 * c.intermediate else c.hidden; - const m = block.mul(width).add(ar(b, width)); - const k = ar(b, c.bk); - const ks = ar(b, @divExact(c.bk, 32)); - const row = if (is_up) pair.div(c.topk) else tile.mul(c.bn).add(n); - const wd = b.makeTensorDescriptor(a.w.addPtr(expert.to(.i64).mul(@divTrunc(rows * ksize, 2))), &.{ ci(b, rows), ci(b, @divTrunc(ksize, 2)) }, &.{ b.liftAs(@divTrunc(ksize, 2), .i64), b.liftAs(1, .i64) }, .pad_zero, &.{ width, @divExact(c.bk, 2) }, .i8); - var loop = b.openFor(0, @divTrunc(ksize + c.bk - 1, c.bk), 1, .{b.zeros(&.{ width, c.bn }, .f32)}); - const sk = loop.iv.mul(@divExact(c.bk, 32)).add(ks); - const kk = loop.iv.mul(c.bk).add(k); - const w = b.descriptorLoad(wd, &.{ block.mul(ci(b, width)), loop.iv.mul(ci(b, @divExact(c.bk, 2))) }, &.{ width, @divExact(c.bk, 2) }, .i8); - const scale_offset = m.expandDims(1).mul(c.scale_stride).add(sk.expandDims(0)); - const ws = ld(b, a.ws.addPtr(expert.to(.i64).mul(rows * c.scale_stride)).addPtr(scale_offset), sk.expandDims(0).lt(@divTrunc(ksize, 32)), .i8, 127); - var mask = kk.expandDims(0).lt(ksize); - var smask = sk.expandDims(0).lt(@divTrunc(ksize, 32)); - // The small-batch epilogue leaves padded rows unwritten. - if (is_up or c.tokens <= 8) { - mask = mask.bitAnd(n.lt(count).expandDims(1)); - smask = smask.bitAnd(n.lt(count).expandDims(1)); + { + const count = if (c.tokens == 1) ci(b, 1) else b.load(a.sched.addPtr(nt).addPtr(tile)); + const pair = if (c.tokens == 1) b.full(&.{c.bn}, 0, .i32).add(tile) else if (is_up) ld(b, a.map.addPtr(tile.mul(c.bn).add(n)), n.lt(count), .i32, 0) else n; + const width: i64 = if (is_up and c.tokens > 8) 256 else 128; + const ksize = if (is_up) c.hidden else c.intermediate; + const rows = if (is_up) 2 * c.intermediate else c.hidden; + const m = block.mul(width).add(ar(b, width)); + const k = ar(b, c.bk); + const ks = ar(b, @divExact(c.bk, 32)); + const row = if (is_up) pair.div(c.topk) else tile.mul(c.bn).add(n); + const wd = b.makeTensorDescriptor(a.w.addPtr(expert.to(.i64).mul(@divTrunc(rows * ksize, 2))), &.{ ci(b, rows), ci(b, @divTrunc(ksize, 2)) }, &.{ b.constant(.i64, @divTrunc(ksize, 2)), b.constant(.i64, 1) }, .pad_zero, &.{ width, @divExact(c.bk, 2) }, .i8); + var loop = b.openFor(0, @divTrunc(ksize + c.bk - 1, c.bk), 1, .{b.zeros(&.{ width, c.bn }, .f32)}); + { + const sk = loop.iv.mul(@divExact(c.bk, 32)).add(ks); + const kk = loop.iv.mul(c.bk).add(k); + const w = b.descriptorLoad(wd, &.{ block.mul(ci(b, width)), loop.iv.mul(ci(b, @divExact(c.bk, 2))) }, &.{ width, @divExact(c.bk, 2) }, .i8); + const scale_offset = m.expandDims(1).mul(c.scale_stride).add(sk.expandDims(0)); + const ws = ld(b, a.ws.addPtr(expert.to(.i64).mul(rows * c.scale_stride)).addPtr(scale_offset), sk.expandDims(0).lt(@divTrunc(ksize, 32)), .i8, 127); + var mask = kk.expandDims(0).lt(ksize); + var smask = sk.expandDims(0).lt(@divTrunc(ksize, 32)); + // The small-batch epilogue leaves padded rows unwritten. + if (is_up or c.tokens <= 8) { + mask = mask.bitAnd(n.lt(count).expandDims(1)); + smask = smask.bitAnd(n.lt(count).expandDims(1)); + } + const x = ld(b, a.q.addPtr(row.expandDims(1).mul(ksize).add(kk.expandDims(0))), mask, .f8e4m3fn, 0); + const xs = ld(b, a.s.addPtr(row.expandDims(1).mul(@divTrunc(ksize, 32)).add(sk.expandDims(0))), smask, .i8, 127); + loop.yield(.{b.dotScaled(w, b.permute(x, &.{ 1, 0 }), loop.carried[0], ws, xs, .e2m1, .e4m3)}); + } + const epilogue_n: i64 = if (c.tokens == 1) 1 else c.bn; + const out_n = ar(b, epilogue_n); + const acc = if (c.tokens == 1) live_row: { + // The remaining MMA columns are padding during single-token decode. + // Extract column zero before activation/reduction and output stores. + var v = loop.results[0]; + var cols: i64 = c.bn; + while (cols > 1) : (cols = @divTrunc(cols, 2)) { + v = b.split(b.reshape(v, &.{ width, @divTrunc(cols, 2), 2 }))[0]; + } + break :live_row b.reshape(v, &.{ 1, width }).to(.bf16); + } else b.permute(loop.results[0], &.{ 1, 0 }).to(.bf16); + if (is_up) { + const columns = @divExact(width, 2); + const halves = b.split(b.reshape(acc.to(.f32), &.{ epilogue_n, columns, 2 })); + var g = halves[0]; + var u = halves[1]; + if (c.swiglu_limit > 0) { + g = g.minimum(c.swiglu_limit); + u = u.minimum(c.swiglu_limit).maximum(-c.swiglu_limit); + } + const ex = b.externElementwise(&.{g.mul(-1.0)}, &.{ epilogue_n, columns }, .f32, "libdevice", "", "__nv_expf"); + var mid = u.mul(b.divRn(g, ex.add(1.0))); + if (c.routing_weight_placement == .before_down) { + const rw = if (c.tokens == 1) b.load(a.rw.addPtr(tile)) else ld(b, a.rw.addPtr(pair), n.lt(count), .f32, 0).expandDims(1); + mid = mid.mul(rw); + } + // Compute scales and quantize directly from the FP32 SwiGLU result. + const q = quant32(b, mid); + const col = block.mul(columns).add(ar(b, columns)); + b.store(a.mid.addPtr(tile.mul(c.bn).add(out_n).expandDims(1).mul(c.intermediate).add(col.expandDims(0))), q.q); + const scale_cols = block.mul(@divExact(columns, 32)).add(ar(b, @divExact(columns, 32))); + b.store(a.ms.addPtr(tile.mul(c.bn).add(out_n).expandDims(1).mul(@divTrunc(c.intermediate, 32)).add(scale_cols.expandDims(0))), q.s); + } else b.store(a.d.addPtr(tile.mul(c.bn).add(out_n).expandDims(1).mul(c.hidden).add(m.expandDims(0))), acc); + live.close(); } - const x = ld(b, a.q.addPtr(row.expandDims(1).mul(ksize).add(kk.expandDims(0))), mask, .f8e4m3fn, 0); - const xs = ld(b, a.s.addPtr(row.expandDims(1).mul(@divTrunc(ksize, 32)).add(sk.expandDims(0))), smask, .i8, 127); - loop.yield(.{b.dotScaled(w, b.permute(x, &.{ 1, 0 }), loop.carried[0], ws, xs, .e2m1, .e4m3)}); - const epilogue_n: i64 = if (c.tokens == 1) 1 else c.bn; - const out_n = ar(b, epilogue_n); - const acc = if (c.tokens == 1) live_row: { - // The remaining MMA columns are padding during single-token decode. - // Extract column zero before activation/reduction and output stores. - var v = loop.results[0]; - var cols: i64 = c.bn; - while (cols > 1) : (cols = @divTrunc(cols, 2)) { - v = b.split(b.reshape(v, &.{ width, @divTrunc(cols, 2), 2 }))[0]; - } - break :live_row b.reshape(v, &.{ 1, width }).to(.bf16); - } else b.permute(loop.results[0], &.{ 1, 0 }).to(.bf16); - if (is_up) { - const columns = @divExact(width, 2); - const halves = b.split(b.reshape(acc.to(.f32), &.{ epilogue_n, columns, 2 })); - var g = halves[0]; - var u = halves[1]; - if (c.swiglu_limit > 0) { - g = g.minimum(c.swiglu_limit); - u = u.minimum(c.swiglu_limit).maximum(-c.swiglu_limit); - } - const ex = b.externElementwise(&.{g.mul(-1.0)}, &.{ epilogue_n, columns }, .f32, "libdevice", "", "__nv_expf"); - var mid = u.mul(b.divRn(g, ex.add(1.0))); - if (c.routing_weight_placement == .before_down) { - const rw = if (c.tokens == 1) b.load(a.rw.addPtr(tile)) else ld(b, a.rw.addPtr(pair), n.lt(count), .f32, 0).expandDims(1); - mid = mid.mul(rw); - } - // Compute scales and quantize directly from the FP32 SwiGLU result. - const q = quant32(b, mid); - const col = block.mul(columns).add(ar(b, columns)); - b.store(a.mid.addPtr(tile.mul(c.bn).add(out_n).expandDims(1).mul(c.intermediate).add(col.expandDims(0))), q.q); - const scale_cols = block.mul(@divExact(columns, 32)).add(ar(b, @divExact(columns, 32))); - b.store(a.ms.addPtr(tile.mul(c.bn).add(out_n).expandDims(1).mul(@divTrunc(c.intermediate, 32)).add(scale_cols.expandDims(0))), q.s); - } else b.store(a.d.addPtr(tile.mul(c.bn).add(out_n).expandDims(1).mul(c.hidden).add(m.expandDims(0))), acc); - live.yieldThen(.{}); } diff --git a/zml/kernels/paged_attention/cute/attention.zig b/zml/kernels/paged_attention/cute/attention.zig index 245ef7e8..50eb2531 100644 --- a/zml/kernels/paged_attention/cute/attention.zig +++ b/zml/kernels/paged_attention/cute/attention.zig @@ -29,12 +29,6 @@ pub const Kernel = cute.Kernel(Config, .{ .run = run, }); -fn tuple(comptime T: type, comptime n: usize, values: [n]T) @Tuple(&@as([n]type, @splat(T))) { - var result: @Tuple(&@as([n]type, @splat(T))) = undefined; - inline for (0..n) |i| result[i] = values[i]; - return result; -} - fn run(b: *cute.Builder, c: Config) cute.FinishError!void { switch (c.head_dim) { 32 => try runDim(b, c, 1), @@ -61,13 +55,13 @@ fn runDim(b: *cute.Builder, c: Config, comptime chunks: usize) cute.FinishError! const token = b.blockIdx().x; const head = b.blockIdx().y; const lane = b.threadIdx().x; - const zero = b.cst(.f32, 0); + const zero = b.constant(.f32, 0); // One full warp owns a query/head; all control flow is warp-uniform. // Also define output for unused capacity at the end of the packed queries. for (0..chunks) |i| a.out.set(.{ token, head, lane.add(i * 32) }, zero.to(c.dtype)); var active = b.openIf(token.lt(a.starts.get(.{c.batch}))); { - var find_seq = b.openFor(0, c.batch, 1, .{b.cst(.i32, 0)}); + var find_seq = b.openFor(0, c.batch, 1, .{b.constant(.i32, 0)}); { const owns = token.ge(a.starts.get(.{find_seq.iv})).bitAnd(token.lt(a.starts.get(.{find_seq.iv.add(1)}))); find_seq.yield(.{b.select(owns, find_seq.iv, find_seq.carried[0])}); @@ -77,7 +71,7 @@ fn runDim(b: *cute.Builder, c: Config, comptime chunks: usize) cute.FinishError! const qlen = a.starts.get(.{seq.add(1)}).sub(start); const len = a.lengths.get(.{seq}); const position = len.sub(qlen).add(token.sub(start)); - const begin = if (c.window > 0) position.sub(c.window - 1).maximum(0) else b.cst(.i32, 0); + const begin = if (c.window > 0) position.sub(c.window - 1).maximum(0) else b.constant(.i32, 0); const end = if (c.causal) position.add(1).minimum(len) else len; var query: [chunks]V = undefined; for (&query, 0..) |*q, i| q.* = a.q.get(.{ token, head, lane.add(i * 32) }).to(.f32); @@ -85,15 +79,15 @@ fn runDim(b: *cute.Builder, c: Config, comptime chunks: usize) cute.FinishError! // State is running max, normalizer, then each lane's output features. // A sink is an initial softmax entry with zero value. var initial: [chunks + 2]V = @splat(zero); - initial[0] = if (c.sink) a.sink.get(.{head}).mul(std.math.log2e) else b.cst(.f32, -std.math.inf(f32)); - initial[1] = if (c.sink) b.cst(.f32, 1) else zero; - var keys = b.openFor(if (c.indexed) b.cst(.i32, 0) else begin, if (c.indexed) b.cst(.i32, c.topk) else end, 1, tuple(V, chunks + 2, initial)); + initial[0] = if (c.sink) a.sink.get(.{head}).mul(std.math.log2e) else b.constant(.f32, -std.math.inf(f32)); + initial[1] = if (c.sink) b.constant(.f32, 1) else zero; + var keys = b.openFor(if (c.indexed) b.constant(.i32, 0) else begin, if (c.indexed) b.constant(.i32, c.topk) else end, 1, initial); { const key = if (c.indexed) a.indices.get(.{ token, keys.iv }) else keys.iv; const valid = key.ge(0).bitAnd(key.lt(len)); // Invalid indexed entries must not load the page table, K, or V. Duplicates // deliberately contribute once per occurrence, as independent softmax entries. - var entry = b.openIfElse(valid, tuple(@TypeOf(zero.type_()), chunks + 2, @splat(zero.type_()))); + var entry = b.openIfElse(valid, @as([chunks + 2]V, @splat(zero))); { const page = a.table.get(.{ seq, key.div(c.page_size) }); const offset = key.rem(c.page_size); @@ -109,15 +103,15 @@ fn runDim(b: *cute.Builder, c: Config, comptime chunks: usize) cute.FinishError! next[0] = maximum; next[1] = keys.carried[1].mul(alpha).add(weight); for (0..chunks) |i| next[i + 2] = keys.carried[i + 2].mul(alpha).add(weight.mul(a.v.get(.{ page, offset, kv_head, lane.add(i * 32) }).to(.f32))); - entry.yieldThen(tuple(V, chunks + 2, next)); + entry.yieldThen(next); } { - entry.yieldElse(tuple(V, chunks + 2, keys.carried)); + entry.yieldElse(keys.carried); } - keys.yield(tuple(V, chunks + 2, entry.results)); + keys.yield(entry.results); } const norm = b.select(keys.results[1].gt(0), b.rcpApprox(keys.results[1]), zero); for (0..chunks) |i| a.out.set(.{ token, head, lane.add(i * 32) }, keys.results[i + 2].mul(norm).to(c.dtype)); - active.yieldThen(.{}); + active.close(); } } diff --git a/zml/kernels/paged_attention/cute/sm10x/attention.zig b/zml/kernels/paged_attention/cute/sm10x/attention.zig index c336a5ae..3d5fd107 100644 --- a/zml/kernels/paged_attention/cute/sm10x/attention.zig +++ b/zml/kernels/paged_attention/cute/sm10x/attention.zig @@ -84,12 +84,6 @@ pub const Attention = cute.Kernel(Config, .{ .run = attention, }); -fn tuple(comptime T: type, comptime n: usize, values: [n]T) @Tuple(&@as([n]type, @splat(T))) { - var result: @Tuple(&@as([n]type, @splat(T))) = undefined; - inline for (0..n) |i| result[i] = values[i]; - return result; -} - fn loadInt(b: *B, pointer: V, index: V) V { return b.loadWords(1, pointer.add(index.to(.i64).mul(4)), .global, .{})[0]; } @@ -102,7 +96,7 @@ fn swizzle(row: V, col: V, rows: i64) V { fn descriptor(b: *B, address: V, leading_bytes: i64) V { const bits: u64 = (64 << 32) | (1 << 46) | (@as(u64, 2) << 61) | (@as(u64, @intCast(@divExact(leading_bytes, 16))) << 16); - return address.to(.i64).shrLogical(4).bitAnd(0x3fff).bitOr(b.cst(.i64, @as(i64, @bitCast(bits)))); + return address.to(.i64).shrLogical(4).bitAnd(0x3fff).bitOr(b.constant(.i64, @as(i64, @bitCast(bits)))); } fn instruction(c: Config, qk: bool) u32 { @@ -121,7 +115,7 @@ const Range = struct { first: V, count: V }; fn keyRange(b: *B, c: Config, first_row: V, qlen: V, klen: V) Range { const first_pos = klen.sub(qlen).add(first_row.div(c.group)); const last_pos = klen.sub(qlen).add(first_row.add(c.rows() - 1).div(c.group).minimum(qlen.sub(1))); - const begin = if (c.window > 0) first_pos.sub(c.window - 1).maximum(0) else b.cst(.i32, 0); + const begin = if (c.window > 0) first_pos.sub(c.window - 1).maximum(0) else b.constant(.i32, 0); const end = if (c.causal) last_pos.add(1).minimum(klen) else klen; const first = begin.div(c.keys()); return .{ .first = first, .count = end.add(c.keys() - 1).div(c.keys()).sub(first).maximum(0) }; @@ -187,8 +181,8 @@ const Tile = struct { } fn norm(t: Tile, row: V, maximum: V, denominator: V) V { const b = t.b; - if (t.c.splits > 1) return b.cst(.f32, 1); - if (!t.c.sink) return b.select(denominator.gt(0), b.rcpApprox(denominator), b.cst(.f32, 0)); + if (t.c.splits > 1) return b.constant(.f32, 1); + if (!t.c.sink) return b.select(denominator.gt(0), b.rcpApprox(denominator), b.constant(.f32, 0)); const sink = loadInt(b, t.sink, t.head.mul(t.c.group).add(row.rem(t.c.group))).bitCast(.f32).mul(std.math.log2e); const reference = maximum.maximum(sink).maximum(-std.math.floatMax(f32)); const weight = b.exp2Approx(maximum.sub(reference)); @@ -256,13 +250,13 @@ fn attention(b: *B, c: Config) cute.FinishError!void { } b.mbarrierInit(shared.add(c.completion()), 1); b.fenceMbarrierInit(); - leader.yieldThen(.{}); + leader.close(); } var alloc = b.openIf(tid.lt(32)); { b.tmemAllocAt(shared.add(c.holder()), c.columns()); b.relinquishTmemAllocPermit(.{}); - alloc.yieldThen(.{}); + alloc.close(); } var queries = b.openFor(tid, c.rows() * @divExact(c.paddedDim(), 8), 256, .{}); { @@ -270,10 +264,10 @@ fn attention(b: *B, c: Config) cute.FinishError!void { const col = queries.iv.rem(@divExact(c.paddedDim(), 8)).mul(8); const packed_row = first_row.add(row); const valid = packed_row.lt(qlen.mul(c.group)).bitAnd(col.lt(c.head_dim)); - const token = start.add(b.select(valid, packed_row.div(c.group), b.cst(.i32, 0))); + const token = start.add(b.select(valid, packed_row.div(c.group), b.constant(.i32, 0))); const offset = token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(packed_row.rem(c.group).to(.i64)).mul(c.head_dim).add(col.to(.i64)); b.cpAsync16(shared.add(c.qBase()).add(swizzle(row, col, c.rows())), t.q.add(offset.mul(2)), valid); - queries.yield(.{}); + queries.close(); } b.cpAsyncWaitAll(); b.syncThreads(); @@ -282,23 +276,23 @@ fn attention(b: *B, c: Config) cute.FinishError!void { var producer = b.openIf(tid.ge(128)); { produce(t); - producer.yieldThen(.{}); + producer.close(); } var consumer = b.openIf(tid.lt(128)); { if (c.decode) { if (c.group <= 2) decode(t, 2) else if (c.group <= 4) decode(t, 4) else if (c.group <= 8) decode(t, 8) else decode(t, 16); } else prefill(t); - consumer.yieldThen(.{}); + consumer.close(); } b.tcgen05Fence(.before); b.syncThreads(); var free = b.openIf(tid.lt(32)); { b.tmemDeallocAt(t.tmem, c.columns()); - free.yieldThen(.{}); + free.close(); } - active.yieldThen(.{}); + active.close(); } } @@ -313,28 +307,28 @@ fn produce(t: Tile) void { var offsets: [16]V = undefined; var valid: [16]V = undefined; for (0..@as(usize, @intCast(@divExact(c.keys(), 8)))) |i| { - const key = t.first_key_tile.add(tiles.iv).mul(c.keys()).add(row0).add(@as(i32, @intCast(i * 8))); + const key = t.first_key_tile.add(tiles.iv).mul(c.keys()).add(row0).add(i * 8); valid[i] = key.lt(t.klen).bitAnd(col.lt(c.head_dim)); - const safe = b.select(key.lt(t.klen), key, b.cst(.i32, 0)); + const safe = b.select(key.lt(t.klen), key, b.constant(.i32, 0)); const page = loadInt(b, t.table, t.seq.mul(c.pages_per_seq).add(safe.div(c.page_size))); offsets[i] = page.to(.i64).mul(c.page_size).add(safe.rem(c.page_size).to(.i64)).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.head_dim).add(col.to(.i64)).mul(2); } for (0..@as(usize, @intCast(c.features() * 2))) |stage| { - const id = tiles.iv.mul(c.features() * 2).add(@as(i32, @intCast(stage))); + const id = tiles.iv.mul(c.features() * 2).add(stage); var reuse = b.openIf(id.ge(c.buffers())); { b.mbarrierWait(t.released(id), t.phase(id).bitXor(1)); - reuse.yieldThen(.{}); + reuse.close(); } const source = if (stage < c.features()) t.k else t.v; const feature_offset: i64 = @mod(@as(i64, @intCast(stage)), c.features()) * 256; for (0..@as(usize, @intCast(@divExact(c.keys(), 8)))) |i| { - const row = row0.add(@as(i32, @intCast(i * 8))); + const row = row0.add(i * 8); b.cpAsync16(t.shared.add(id.rem(c.buffers()).mul(c.kvBytes())).add(swizzle(row, col, c.keys())), source.add(offsets[i]).add(feature_offset), valid[i]); } b.cpAsyncMbarrierArrive(t.ready(id), true); } - tiles.yield(.{}); + tiles.close(); } } @@ -356,15 +350,15 @@ fn multiply(t: Tile, tile: V, comptime qk: bool) void { const left = if (c.decode) kv.add(@divExact(kv_offset, 16)) else qp.add(@divExact(qp_offset, 16)); const right = if (c.decode) qp.add(@divExact(qp_offset, 16)) else kv.add(@divExact(kv_offset, 16)); const column: i64 = if (qk) c.scoreColumn() else @as(i64, @intCast(part)) * (if (c.decode) c.rows() else 128); - const accumulate = if (qk) b.cst(.i32, @intFromBool(part > 0 or step > 0)).ne(0) else tile.gt(0).bitOr(b.cst(.i32, @intFromBool(step > 0)).ne(0)); - b.tcgen05MmaF16(t.tmem.add(column), left, right, b.cst(.i32, @as(i32, @bitCast(instruction(c, qk)))), accumulate); + const accumulate = if (qk) b.constant(.i32, @intFromBool(part > 0 or step > 0)).ne(0) else tile.gt(0).bitOr(b.constant(.i32, @intFromBool(step > 0)).ne(0)); + b.tcgen05MmaF16(t.tmem.add(column), left, right, b.constant(.i32, @as(i32, @bitCast(instruction(c, qk)))), accumulate); } b.tcgen05Commit(t.released(id)); } b.tcgen05Commit(t.shared.add(c.completion())); - leader.yieldThen(.{}); + leader.close(); } - b.mbarrierWait(t.shared.add(c.completion()), b.cst(.i32, if (qk) 0 else 1)); + b.mbarrierWait(t.shared.add(c.completion()), b.constant(.i32, if (qk) 0 else 1)); b.tcgen05Fence(.after); // A fast warp must not wrap completion parity before every consumer has // observed the preceding PV completion. @@ -395,8 +389,8 @@ fn reduceHeads(t: Tile, comptime heads: usize, values: [heads]V, comptime maximu inline for (.{ 16, 8, 4, 2, 1 }) |step| reduced = if (maximum) reduced.maximum(b.shuffleXor(reduced, step)) else reduced.add(b.shuffleXor(reduced, step)); var leader = b.openIf(t.lane.eq(0)); { - b.storeWords(1, t.shared.add(t.c.reductions()).add(t.warp.mul(64)).add(@as(i32, @intCast(h * 4))), .shared, .{reduced.bitCast(.i32)}); - leader.yieldThen(.{}); + b.storeWords(1, t.shared.add(t.c.reductions()).add(t.warp.mul(64)).add(h * 4), .shared, .{reduced.bitCast(.i32)}); + leader.close(); } } b.barrierSync(1, 128); @@ -418,21 +412,21 @@ fn decode(t: Tile, comptime heads: usize) void { const c = t.c; const address = t.tmem.add(t.warp.mul(32).shl(16)); const key_row = t.warp.mul(32).add(t.lane); - const zero = b.cst(.f32, 0); - const neg_inf = b.cst(.f32, -std.math.inf(f32)); + const zero = b.constant(.f32, 0); + const neg_inf = b.constant(.f32, -std.math.inf(f32)); var initial: [2 * heads]V = undefined; for (0..heads) |h| { initial[h] = neg_inf; initial[heads + h] = zero; } - var tiles = b.openFor(0, t.tiles, 1, tuple(V, 2 * heads, initial)); + var tiles = b.openFor(0, t.tiles, 1, initial); { multiply(t, tiles.iv, true); const raw = b.tmemLoad32x32b(heads, address.add(c.scoreColumn())); b.fenceTmemLoad(); const key = t.first_key_tile.add(tiles.iv).mul(c.keys()).add(key_row); var scores: [heads]V = undefined; - for (0..heads) |h| scores[h] = b.select(t.valid(t.first_row.add(@as(i32, @intCast(h))), key), raw[h].bitCast(.f32).mul(c.scale * std.math.log2e), neg_inf); + for (0..heads) |h| scores[h] = b.select(t.valid(t.first_row.add(h), key), raw[h].bitCast(.f32).mul(c.scale * std.math.log2e), neg_inf); const maxima = reduceHeads(t, heads, scores, true); var next: [2 * heads]V = undefined; var alpha: [heads]V = undefined; @@ -446,9 +440,9 @@ fn decode(t: Tile, comptime heads: usize) void { const sums = reduceHeads(t, heads, probabilities, false); for (0..heads) |h| { next[heads + h] = tiles.carried[heads + h].mul(alpha[h]).add(sums[h]); - t.putProbability(b.cst(.i32, @as(i32, @intCast(h))), key_row, probabilities[h]); + t.putProbability(b.constant(.i32, h), key_row, probabilities[h]); } - for (heads..@as(usize, @intCast(c.rows()))) |h| t.putProbability(b.cst(.i32, @as(i32, @intCast(h))), key_row, zero); + for (heads..@as(usize, @intCast(c.rows()))) |h| t.putProbability(b.constant(.i32, h), key_row, zero); var rescale = b.openIf(tiles.iv.gt(0)); { for (0..@as(usize, @intCast(c.features()))) |part| { @@ -459,35 +453,35 @@ fn decode(t: Tile, comptime heads: usize) void { b.tmemStore32x32b(heads, addr, out); } b.fenceTmemStore(); - rescale.yieldThen(.{}); + rescale.close(); } publishProbabilities(t); multiply(t, tiles.iv, false); - tiles.yield(tuple(V, 2 * heads, next)); + tiles.yield(next); } for (0..@as(usize, @intCast(c.features()))) |part| { const out = b.tmemLoad32x32b(heads, address.add(@as(i64, @intCast(part)) * c.rows())); b.fenceTmemLoad(); - const dim = key_row.add(@as(i32, @intCast(part * 128))); + const dim = key_row.add(part * 128); for (0..heads) |h| { - const row = t.first_row.add(@as(i32, @intCast(h))); + const row = t.first_row.add(h); var valid = b.openIf(row.lt(t.qlen.mul(c.group))); { var feature = b.openIf(dim.lt(c.head_dim)); { const value = b.select(tiles.results[heads + h].gt(0), out[h].bitCast(.f32).mul(t.norm(row, tiles.results[h], tiles.results[heads + h])), zero); t.put(row, dim, value); - feature.yieldThen(.{}); + feature.close(); } if (c.splits > 1 and part == 0) { var stats = b.openIf(t.tid.eq(0)); { - t.put(row, b.cst(.i32, c.head_dim), tiles.results[h]); - t.put(row, b.cst(.i32, c.head_dim + 1), tiles.results[heads + h]); - stats.yieldThen(.{}); + t.put(row, b.constant(.i32, c.head_dim), tiles.results[h]); + t.put(row, b.constant(.i32, c.head_dim + 1), tiles.results[heads + h]); + stats.close(); } } - valid.yieldThen(.{}); + valid.close(); } } } @@ -500,24 +494,24 @@ fn prefill(t: Tile) void { const local_row = t.warp.mul(16).add(t.lane); const row = t.first_row.add(local_row); const owns = t.lane.lt(16); - const zero = b.cst(.f32, 0); - const neg_inf = b.cst(.f32, -std.math.inf(f32)); + const zero = b.constant(.f32, 0); + const neg_inf = b.constant(.f32, -std.math.inf(f32)); var tiles = b.openFor(0, t.tiles, 1, .{ neg_inf, zero }); { // D512 fills TMEM with O. Preserve its last 64 columns in registers // while QK reuses them for scores, avoiding duplicate QK/softmax CTAs. var saved: [64]V = undefined; if (c.head_dim == 512) { - var preserve = b.openIfElse(tiles.iv.gt(0), tuple(@TypeOf(zero.type_()), 64, @splat(zero.type_()))); + var preserve = b.openIfElse(tiles.iv.gt(0), @as([64]V, @splat(zero))); { const tail = b.tmemLoad32x32b(64, address.add(c.scoreColumn())); b.fenceTmemLoad(); var words: [64]V = undefined; for (tail, 0..) |value, i| words[i] = value.bitCast(.f32); - preserve.yieldThen(tuple(V, 64, words)); + preserve.yieldThen(words); } { - preserve.yieldElse(tuple(V, 64, @as([64]V, @splat(zero)))); + preserve.yieldElse(@as([64]V, @splat(zero))); } @memcpy(&saved, &preserve.results); b.barrierSync(1, 128); @@ -539,10 +533,10 @@ fn prefill(t: Tile) void { // use them; page tails and mask boundaries reduce post-mask values. const full = owns.bitAnd(t.valid(row, key0)).bitAnd(t.valid(row, key0.add(63))); for (0..64) |i| { - scores[i] = b.select(owns.bitAnd(t.valid(row, key0.add(@as(i32, @intCast(i))))), scores[i].mul(c.scale * std.math.log2e), neg_inf); + scores[i] = b.select(owns.bitAnd(t.valid(row, key0.add(i))), scores[i].mul(c.scale * std.math.log2e), neg_inf); } const tile_max = if (c.architecture == .sm103 and c.scale > 0) blk: { - var choose = b.openIfElse(full, .{zero.type_()}); + var choose = b.openIfElse(full, .{zero}); { choose.yieldThen(.{hardware_max.mul(c.scale * std.math.log2e)}); } @@ -561,13 +555,13 @@ fn prefill(t: Tile) void { const denominator = tiles.carried[1].mul(alpha).add(reduceRow(64, probabilities, false)); var write = b.openIf(owns); { - for (0..64) |i| t.putProbability(local_row, b.cst(.i32, @as(i32, @intCast(i))), probabilities[i]); - write.yieldThen(.{}); + for (0..64) |i| t.putProbability(local_row, b.constant(.i32, i), probabilities[i]); + write.close(); } var rescale = b.openIf(tiles.iv.gt(0)); { for (0..@as(usize, @intCast(@divExact(if (c.head_dim == 512) c.scoreColumn() else c.paddedDim(), 32)))) |chunk| { - const addr = address.add(@as(i32, @intCast(chunk * 32))); + const addr = address.add(chunk * 32); var out = b.tmemLoad32x32b(32, addr); b.fenceTmemLoad(); for (&out) |*value| value.* = value.bitCast(.f32).mul(alpha).bitCast(.i32); @@ -578,7 +572,7 @@ fn prefill(t: Tile) void { b.tmemStore32x32b(64, address.add(c.scoreColumn()), saved); } b.fenceTmemStore(); - rescale.yieldThen(.{}); + rescale.close(); } publishProbabilities(t); multiply(t, tiles.iv, false); @@ -586,19 +580,19 @@ fn prefill(t: Tile) void { } const norm = t.norm(row, tiles.results[0], tiles.results[1]); for (0..@as(usize, @intCast(@divExact(c.paddedDim(), 32)))) |chunk| { - const out = b.tmemLoad32x32b(32, address.add(@as(i32, @intCast(chunk * 32)))); + const out = b.tmemLoad32x32b(32, address.add(chunk * 32)); b.fenceTmemLoad(); var valid = b.openIf(owns.bitAnd(row.lt(t.qlen.mul(c.group)))); { for (0..32) |i| { const dim = chunk * 32 + i; - if (dim < c.head_dim) t.put(row, b.cst(.i32, @as(i32, @intCast(dim))), b.select(tiles.results[1].gt(0), out[i].bitCast(.f32).mul(norm), zero)); + if (dim < c.head_dim) t.put(row, b.constant(.i32, dim), b.select(tiles.results[1].gt(0), out[i].bitCast(.f32).mul(norm), zero)); } if (c.splits > 1 and chunk == 0) { - t.put(row, b.cst(.i32, c.head_dim), tiles.results[0]); - t.put(row, b.cst(.i32, c.head_dim + 1), tiles.results[1]); + t.put(row, b.constant(.i32, c.head_dim), tiles.results[0]); + t.put(row, b.constant(.i32, c.head_dim + 1), tiles.results[1]); } - valid.yieldThen(.{}); + valid.close(); } } } @@ -622,7 +616,7 @@ fn merge(b: *B, c: Config) cute.FinishError!void { const head = b.blockIdx().y; var active = b.openIf(token.lt(a.starts.get(.{c.batch}))); { - var find = b.openFor(0, c.batch, 1, .{b.cst(.i32, 0)}); + var find = b.openFor(0, c.batch, 1, .{b.constant(.i32, 0)}); { const owns = token.ge(a.starts.get(.{find.iv})).bitAnd(token.lt(a.starts.get(.{find.iv.add(1)}))); find.yield(.{b.select(owns, find.iv, find.carried[0])}); @@ -633,8 +627,8 @@ fn merge(b: *B, c: Config) cute.FinishError!void { const row = token.sub(start).mul(c.group).add(head.rem(c.group)); const range = keyRange(b, c, row.div(c.rows()).mul(c.rows()), qlen, a.lengths.get(.{seq})); const splits = liveSplits(c, range.count); - const zero = b.cst(.f32, 0); - const sink = if (c.sink) a.sink.get(.{head}).mul(std.math.log2e) else b.cst(.f32, -std.math.inf(f32)); + const zero = b.constant(.f32, 0); + const sink = if (c.sink) a.sink.get(.{head}).mul(std.math.log2e) else b.constant(.f32, -std.math.inf(f32)); var maximum = b.openFor(0, splits, 1, .{sink}); { maximum.yield(.{maximum.carried[0].maximum(a.partials.get(.{ maximum.iv, token, head, c.head_dim }))}); @@ -652,9 +646,9 @@ fn merge(b: *B, c: Config) cute.FinishError!void { } const norm = b.select(sum.results[1].gt(0), b.rcpApprox(sum.results[1]), zero); a.out.set(.{ token, head, dims.iv }, sum.results[0].mul(norm).to(c.dtype)); - dims.yield(.{}); + dims.close(); } - active.yieldThen(.{}); + active.close(); } } diff --git a/zml/kernels/paged_attention/cute/sm120/attention.zig b/zml/kernels/paged_attention/cute/sm120/attention.zig index ec6a509b..ea364488 100644 --- a/zml/kernels/paged_attention/cute/sm120/attention.zig +++ b/zml/kernels/paged_attention/cute/sm120/attention.zig @@ -53,12 +53,6 @@ pub const Kernel = cute.Kernel(Config, .{ .run = run, }); -fn tuple(comptime n: usize, values: [n]V) @Tuple(&@as([n]type, @splat(V))) { - var result: @Tuple(&@as([n]type, @splat(V))) = undefined; - inline for (0..n) |i| result[i] = values[i]; - return result; -} - fn flatten(comptime n: usize, values: [n][4]V) [n * 4]V { var result: [n * 4]V = undefined; for (0..n) |i| @memcpy(result[i * 4 ..][0..4], &values[i]); @@ -153,7 +147,7 @@ const TileContext = struct { { const first_key = key_base.add(pages.iv); const page_valid = first_key.lt(t.klen); - const logical_page = b.select(page_valid, first_key.div(16), b.cst(.i32, 0)); + const logical_page = b.select(page_valid, first_key.div(16), b.constant(.i32, 0)); const page = loadI32(b, t.table, t.seq.mul(c.pages_per_seq).add(logical_page)); const base = page.to(.i64).mul(16 * c.kv_heads * c.head_dim).add(t.head.to(.i64).mul(c.head_dim)); for (0..@as(usize, @intCast(@divExact(c.head_dim, 64)))) |i| { @@ -163,7 +157,7 @@ const TileContext = struct { const offset = base.add(row.to(.i64).mul(c.kv_heads * c.head_dim)).add(col.to(.i64)); b.cpAsync16(t.shared.add(c.kvBase()).add(swizzle(pages.iv.add(row), col, c.head_dim)), source.add(offset.mul(2)), first_key.add(row).lt(t.klen)); } - pages.yield(.{}); + pages.close(); } return; } @@ -173,11 +167,11 @@ const TileContext = struct { const col = copy.iv.rem(@divExact(c.head_dim, 8)).mul(8); const key = key_base.add(row); const valid = key.lt(t.klen); - const safe_key = b.select(valid, key, b.cst(.i32, 0)); + const safe_key = b.select(valid, key, b.constant(.i32, 0)); const page = loadI32(b, t.table, t.seq.mul(c.pages_per_seq).add(safe_key.div(c.page_size))); const offset = page.to(.i64).mul(c.page_size).add(safe_key.rem(c.page_size).to(.i64)).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.head_dim).add(col.to(.i64)); b.cpAsync16(t.shared.add(c.kvBase()).add(swizzle(row, col, c.head_dim)), source.add(offset.mul(2)), valid); - copy.yield(.{}); + copy.close(); } } @@ -189,7 +183,7 @@ const TileContext = struct { var leader = b.openIf(t.lane.rem(4).eq(0)); { b.storeWords(1, address, .shared, .{v.bitCast(.i32)}); - leader.yieldThen(.{}); + leader.close(); } // Callers synchronize once both row halves have been published. } @@ -267,14 +261,14 @@ fn runTile(b: *B, c: Config, comptime d: usize, comptime w: usize, comptime n: u const col = copy_q.iv.rem(d / 8).mul(8); const packed_row = first_row.add(row); const valid = packed_row.lt(qlen.mul(c.group)); - const token = start.add(b.select(valid, packed_row.div(c.group), b.cst(.i32, 0))); + const token = start.add(b.select(valid, packed_row.div(c.group), b.constant(.i32, 0))); const offset = token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(packed_row.rem(c.group).to(.i64)).mul(d).add(col.to(.i64)); b.cpAsync16(t.shared.add(swizzle(row, col, d)), t.q.add(offset.mul(2)), valid); - copy_q.yield(.{}); + copy_q.close(); } const first_pos = t.klen.sub(qlen).add(first_row.div(c.group)); const last_pos = t.klen.sub(qlen).add(first_row.add(c.rows() - 1).div(c.group).minimum(qlen.sub(1))); - const begin = if (c.window > 0) first_pos.sub(c.window - 1).maximum(0) else b.cst(.i32, 0); + const begin = if (c.window > 0) first_pos.sub(c.window - 1).maximum(0) else b.constant(.i32, 0); const end = if (c.causal) last_pos.add(1).minimum(t.klen) else t.klen; const first_tile = begin.div(n); const tiles = end.add(n - 1).div(n).sub(first_tile).maximum(0); @@ -285,8 +279,8 @@ fn runTile(b: *B, c: Config, comptime d: usize, comptime w: usize, comptime n: u const dim_part = t.warp.rem(w).mul(d / w); const row0 = t.lane.div(4); const col0 = t.lane.rem(4).mul(2); - const zero = b.cst(.f32, 0); - const neg_inf = b.cst(.f32, -std.math.inf(f32)); + const zero = b.constant(.f32, 0); + const neg_inf = b.constant(.f32, -std.math.inf(f32)); const nf = n / w / 8; const of = d / w / 8; const count = 4 + of; @@ -296,7 +290,7 @@ fn runTile(b: *B, c: Config, comptime d: usize, comptime w: usize, comptime n: u for (initial[4..]) |*v| v.* = packAcc(b, .{ zero, zero, zero, zero }); b.cpAsyncWaitAll(); b.syncThreads(); - var loop = b.openFor(lo, hi, 1, tuple(count, initial)); + var loop = b.openFor(lo, hi, 1, initial); { const key_base = loop.iv.mul(n); t.loadKV(t.k, key_base); @@ -305,14 +299,14 @@ fn runTile(b: *B, c: Config, comptime d: usize, comptime w: usize, comptime n: u var scores: [nf][4]V = @splat(@splat(zero)); // Keep only MMA operands live in registers; Q remains in shared // memory so D512 does not consume the entire per-thread register file. - var qk = b.openFor(0, d, 16, tuple(nf * 4, @splat(zero))); + var qk = b.openFor(0, d, 16, @as([nf * 4]V, @splat(zero))); { const qa = matrixLoad(b, 4, false, t.shared.add(swizzle(row_base.add(t.lane.rem(16)), qk.iv.add(t.lane.div(16).mul(8)), d))); for (0..nf) |i| { const kb = matrixLoad(b, 2, false, t.shared.add(c.kvBase()).add(swizzle(key_part.add(i * 8).add(t.lane.rem(8)), qk.iv.add(t.lane.div(8).rem(2).mul(8)), d))); scores[i] = mma(b, dt, qa, kb, qk.carried[i * 4 ..][0..4].*); } - qk.yield(tuple(nf * 4, flatten(nf, scores))); + qk.yield(flatten(nf, scores)); } for (0..nf) |i| scores[i] = qk.results[i * 4 ..][0..4].*; var local_max: [2]V = @splat(neg_inf); @@ -366,7 +360,7 @@ fn runTile(b: *B, c: Config, comptime d: usize, comptime w: usize, comptime n: u }; var packed_acc: [of]V = undefined; for (0..of) |i| packed_acc[i] = packAcc(b, acc[i]); - var pv = b.openFor(0, n, 16, tuple(of, packed_acc)); + var pv = b.openFor(0, n, 16, packed_acc); { const pa = matrixLoad(b, 4, false, t.shared.add(c.pBase()).add(swizzle(row_base.add(t.lane.rem(16)), pv.iv.add(t.lane.div(16).mul(8)), n))); for (0..of) |i| { @@ -376,11 +370,11 @@ fn runTile(b: *B, c: Config, comptime d: usize, comptime w: usize, comptime n: u acc[i] = mma(b, dt, pa, vb, prev); packed_acc[i] = packAcc(b, acc[i]); } - pv.yield(tuple(of, packed_acc)); + pv.yield(packed_acc); } @memcpy(next[4..], &pv.results); b.syncThreads(); - loop.yield(tuple(count, next)); + loop.yield(next); } for (0..2) |h| { const row = first_row.add(row_base).add(row0).add(h * 8); @@ -402,15 +396,15 @@ fn runTile(b: *B, c: Config, comptime d: usize, comptime w: usize, comptime n: u if (c.splits > 1) { var leader = b.openIf(t.warp.rem(w).eq(0).bitAnd(t.lane.rem(4).eq(0))); { - t.store(row, b.cst(.i32, d), m); - t.store(row, b.cst(.i32, d + 1), l); - leader.yieldThen(.{}); + t.store(row, b.constant(.i32, d), m); + t.store(row, b.constant(.i32, d + 1), l); + leader.close(); } } - valid.yieldThen(.{}); + valid.close(); } } - active.yieldThen(.{}); + active.close(); } } @@ -432,8 +426,8 @@ fn merge(b: *B, c: Config) cute.FinishError!void { const head = b.blockIdx().y; var valid = b.openIf(token.lt(a.starts.get(.{c.batch}))); { - const zero = b.cst(.f32, 0); - const sink = if (c.sink) a.sink.get(.{head}).mul(std.math.log2e) else b.cst(.f32, -std.math.inf(f32)); + const zero = b.constant(.f32, 0); + const sink = if (c.sink) a.sink.get(.{head}).mul(std.math.log2e) else b.constant(.f32, -std.math.inf(f32)); var maximum = b.openFor(0, c.splits, 1, .{sink}); { maximum.yield(.{maximum.carried[0].maximum(a.partials.get(.{ maximum.iv, token, head, c.head_dim }))}); @@ -450,8 +444,8 @@ fn merge(b: *B, c: Config) cute.FinishError!void { }); } a.out.set(.{ token, head, dims.iv }, sum.results[0].mul(b.select(sum.results[1].gt(0), b.rcpApprox(sum.results[1]), zero)).to(c.dtype)); - dims.yield(.{}); + dims.close(); } - valid.yieldThen(.{}); + valid.close(); } } diff --git a/zml/kernels/paged_attention/tcl/tcl.zig b/zml/kernels/paged_attention/tcl/tcl.zig index f50db773..ce8bdb4f 100644 --- a/zml/kernels/paged_attention/tcl/tcl.zig +++ b/zml/kernels/paged_attention/tcl/tcl.zig @@ -703,48 +703,50 @@ const Flash = tcl.Kernel(Cfg, .{ var op = b.tensorOperation(.{}); const n = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(prepared.kvlen), cfg.chunk - 1), cfg.chunk); var loop = b.openFor(n, .{ try b.full(0, .f32, stats), try b.full(0, .f32, out_axes), try b.full(score_min, .f32, stats) }); - const sum_acc, const qkv_acc, const max_acc = loop.carried; - const indices = try validIndices(b, try b.indexRead(pages, loop.iv), cfg.pages); - const idx = indices.safe; - op = b.tensorOperation(.{}); - const valid_pages = try op.commit(op.fetch(indices.valid, .{ .broadcast_to = &.{ a.B, Cp, a.PG } }), .{}); - const valid_keys = try b.reshape(valid_pages, &.{ a.B, Ck }, .{}); - const offset = try b.symExpr(.mul, loop.iv, cfg.chunk); - const local_pos = try b.arange(cfg.chunk, .i32, &.{Ck}, .{}); - op = b.tensorOperation(.{}); - const position = try op.commit(op.fetch(local_pos, .{}).addi(op.fetch(offset, .{})), .{}); - const local_pages = try b.arange(@divExact(cfg.chunk, cfg.pg), .i32, &.{Cp}, .{}); - op = b.tensorOperation(.{}); - const first_keys = try op.commit(op.fetch(local_pages, .{}).muli(cfg.pg).addi(op.fetch(offset, .{})), .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const remaining = try op.commit(op.fetch(prepared.kvlen, .{}).subi(op.fetch(first_keys, .{})), .{}); - op = b.tensorOperation(.{}); - const live_pages = try op.commit(op.where(remaining, .gt, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const cvl = try op.commit(op.fetch(live_pages, .{}).reduce(&.{Cp}, .addi), .{}); - const mask = try attentionMask(b, cfg, prepared.qpos, prepared.kvlen, position, valid_keys); - const valid_cache = try cacheMask(b, valid_keys, prepared.kvlen, position); - const k = try maskCacheValues(b, try b.reshape(try b.gather(t.k_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}), valid_cache); - const v = try maskCacheValues(b, try b.reshape(try b.gather(t.v_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}), valid_cache); - const masked = try maskedScores(b, a, cfg, prepared.q, k, mask, Ck); - op = b.tensorOperation(.{}); - const max_cur = try op.commit(op.fetch(masked, .{}).reduce(&.{Ck}, .maxf), .{}); - op = b.tensorOperation(.{ .tactic = .elementwise }); - const max_new = try op.commit(op.fetch(max_cur, .{}).maxf(op.fetch(max_acc, .{})), .{}); - const rescale = try expDifference(b, max_acc, max_new); - const e = try maskedExp(b, masked, max_new, mask); - op = b.tensorOperation(.{}); - const sum_cur = try op.commit(op.fetch(e, .{}).reduce(&.{Ck}, .addf), .{}); - op = b.tensorOperation(.{}); - const sum_new = try op.commit(op.fetch(sum_acc, .{}).mulf(op.fetch(rescale, .{})).addf(op.fetch(sum_cur, .{})), .{}); - op = b.tensorOperation(.{}); - // Materialize the narrowing conversion through the vector engine. - const p = try op.commit(op.fetch(e, .{}).mulf(1.0), .{ .dtype = .bf16 }); - op = b.tensorOperation(.{}); - const qkv_cur = try op.commit(op.contract(p, v, out_axes), .{}); - op = b.tensorOperation(.{}); - const qkv_new = try op.commit(op.fetch(qkv_acc, .{}).mulf(op.fetch(rescale, .{})).addf(op.fetch(qkv_cur, .{})), .{}); - try loop.yield(.{ sum_new, qkv_new, max_new }); + { + const sum_acc, const qkv_acc, const max_acc = loop.carried; + const indices = try validIndices(b, try b.indexRead(pages, loop.iv), cfg.pages); + const idx = indices.safe; + op = b.tensorOperation(.{}); + const valid_pages = try op.commit(op.fetch(indices.valid, .{ .broadcast_to = &.{ a.B, Cp, a.PG } }), .{}); + const valid_keys = try b.reshape(valid_pages, &.{ a.B, Ck }, .{}); + const offset = try b.symExpr(.mul, loop.iv, cfg.chunk); + const local_pos = try b.arange(cfg.chunk, .i32, &.{Ck}, .{}); + op = b.tensorOperation(.{}); + const position = try op.commit(op.fetch(local_pos, .{}).addi(op.fetch(offset, .{})), .{}); + const local_pages = try b.arange(@divExact(cfg.chunk, cfg.pg), .i32, &.{Cp}, .{}); + op = b.tensorOperation(.{}); + const first_keys = try op.commit(op.fetch(local_pages, .{}).muli(cfg.pg).addi(op.fetch(offset, .{})), .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const remaining = try op.commit(op.fetch(prepared.kvlen, .{}).subi(op.fetch(first_keys, .{})), .{}); + op = b.tensorOperation(.{}); + const live_pages = try op.commit(op.where(remaining, .gt, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const cvl = try op.commit(op.fetch(live_pages, .{}).reduce(&.{Cp}, .addi), .{}); + const mask = try attentionMask(b, cfg, prepared.qpos, prepared.kvlen, position, valid_keys); + const valid_cache = try cacheMask(b, valid_keys, prepared.kvlen, position); + const k = try maskCacheValues(b, try b.reshape(try b.gather(t.k_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}), valid_cache); + const v = try maskCacheValues(b, try b.reshape(try b.gather(t.v_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}), valid_cache); + const masked = try maskedScores(b, a, cfg, prepared.q, k, mask, Ck); + op = b.tensorOperation(.{}); + const max_cur = try op.commit(op.fetch(masked, .{}).reduce(&.{Ck}, .maxf), .{}); + op = b.tensorOperation(.{ .tactic = .elementwise }); + const max_new = try op.commit(op.fetch(max_cur, .{}).maxf(op.fetch(max_acc, .{})), .{}); + const rescale = try expDifference(b, max_acc, max_new); + const e = try maskedExp(b, masked, max_new, mask); + op = b.tensorOperation(.{}); + const sum_cur = try op.commit(op.fetch(e, .{}).reduce(&.{Ck}, .addf), .{}); + op = b.tensorOperation(.{}); + const sum_new = try op.commit(op.fetch(sum_acc, .{}).mulf(op.fetch(rescale, .{})).addf(op.fetch(sum_cur, .{})), .{}); + op = b.tensorOperation(.{}); + // Materialize the narrowing conversion through the vector engine. + const p = try op.commit(op.fetch(e, .{}).mulf(1.0), .{ .dtype = .bf16 }); + op = b.tensorOperation(.{}); + const qkv_cur = try op.commit(op.contract(p, v, out_axes), .{}); + op = b.tensorOperation(.{}); + const qkv_new = try op.commit(op.fetch(qkv_acc, .{}).mulf(op.fetch(rescale, .{})).addf(op.fetch(qkv_cur, .{})), .{}); + try loop.yield(.{ sum_new, qkv_new, max_new }); + } var sum = loop.results[0]; var numerator = loop.results[1]; diff --git a/zml/kernels/paged_attention/triton/README.md b/zml/kernels/paged_attention/triton/README.md index 68526150..37aba88b 100644 --- a/zml/kernels/paged_attention/triton/README.md +++ b/zml/kernels/paged_attention/triton/README.md @@ -30,7 +30,8 @@ support for every shape or dtype. ## Sparse attention -`.indices` mask dispatches to a dedicated sparse kernel in `unified_attention.zig`. +`.indices` mask dispatches to a dedicated sparse kernel in +`sparse_unified_attention.zig`. The `i32` tensor has shape `{ .b = T, .topk = K }`, with one list per packed query shared across heads. Each entry is a logical position within that query's sequence, including its cached prefix. The kernel translates each selected diff --git a/zml/kernels/paged_attention/triton/sparse_unified_attention.zig b/zml/kernels/paged_attention/triton/sparse_unified_attention.zig index 04f5bb71..79113f1e 100644 --- a/zml/kernels/paged_attention/triton/sparse_unified_attention.zig +++ b/zml/kernels/paged_attention/triton/sparse_unified_attention.zig @@ -56,9 +56,9 @@ fn findSeqIdx( block_q: i64, use_q_block_mode: bool, ) Value { - const left_init = k.liftAs(0, .i32); + const left_init = k.constant(.i32, 0); - var w = k.openWhile(.{ left_init, num_seqs }, .{ k.scalarTy(.i32), k.scalarTy(.i32) }); + var w = k.openWhile(.{ left_init, num_seqs }, .{ .i32, .i32 }); { const left = w.before_carried[0]; const right = w.before_carried[1]; @@ -74,7 +74,7 @@ fn findSeqIdx( else val; - var i = k.openIfElse(mid_val.le(target_idx), .{ k.scalarTy(.i32), k.scalarTy(.i32) }); + var i = k.openIfElse(mid_val.le(target_idx), .{ .i32, .i32 }); { i.yieldThen(.{ mid.add(1), right }); } @@ -142,23 +142,23 @@ fn KernelSparseAttentionPtr(comptime split_k: bool) type { const partition = SparseSegments.init(cfg.indices_count, cfg.num_segments, cfg.tile_size); const program = k.programId(.x); - const num_seqs = k.liftAs(cfg.num_seqs, .i32); + const num_seqs = k.constant(.i32, cfg.num_seqs); if (!cfg.decode_sequence_grid) { const active_queries = k.load(a.query_start_len_ptr.addPtr(num_seqs)); - k.returnIf(program.ge(active_queries), .{}); + k.returnIf(program.ge(active_queries)); } const seq = if (cfg.decode_sequence_grid) program else findSeqIdx(k, a.query_start_len_ptr, program, num_seqs, 1, false); const seq_len = k.load(a.seq_lens_ptr.addPtr(seq)); const query_start = k.load(a.query_start_len_ptr.addPtr(seq)); const query_end = k.load(a.query_start_len_ptr.addPtr(seq).addPtr(1)); - if (cfg.decode_sequence_grid) k.returnIf(query_start.eq(query_end), .{}); + if (cfg.decode_sequence_grid) k.returnIf(query_start.eq(query_end)); const query = if (cfg.decode_sequence_grid) query_start else program; - const kv_head = if (cfg.num_query_heads == cfg.num_queries_per_kv) k.liftAs(0, .i32) else k.programId(.y); + const kv_head = if (cfg.num_query_heads == cfg.num_queries_per_kv) k.constant(.i32, 0) else k.programId(.y); // Launch the partial tail first, without launching unused capacity. - const slot = if (split_k) k.programId(.z) else k.liftAs(0, .i32); - const segment = if (split_k) k.where(slot.eq(0), k.liftAs(@as(i32, @intCast(partition.launched - 1)), .i32), slot.sub(1)) else slot; + const slot = if (split_k) k.programId(.z) else k.constant(.i32, 0); + const segment = if (split_k) k.where(slot.eq(0), k.constant(.i32, partition.launched - 1), slot.sub(1)) else slot; const segment_start = segment.mul(@as(i32, @intCast(partition.span))); const heads = k.arange(0, cfg.block_m, .i32); const head = kv_head.mul(@as(i32, @intCast(cfg.num_queries_per_kv))).add(heads); @@ -186,10 +186,14 @@ fn KernelSparseAttentionPtr(comptime split_k: bool) type { }); const empty = k.sum(scan_mask.bitAnd(positions.ge(0)).bitAnd(positions.lt(seq_len)).to(.i32)).eq(0); var branch = k.openIfElse(empty, .{}); - storeResult(k, cfg, a, query, head, head_mask, segment, dims, dim_mask, initial_max, initial_sum, initial_acc); - branch.yieldThen(.{}); - branch.yieldElse(.{}); - k.returnIf(empty, .{}); + { + storeResult(k, cfg, a, query, head, head_mask, segment, dims, dim_mask, initial_max, initial_sum, initial_acc); + branch.elseBranch(); + } + { + branch.close(); + } + k.returnIf(empty); } const q_offset = query.to(.i64).mul(cfg.query_strides[0]) @@ -201,7 +205,7 @@ fn KernelSparseAttentionPtr(comptime split_k: bool) type { }); const table = a.block_tables_ptr.addPtr(seq.to(.i64).mul(cfg.block_table_stride)); const offsets = k.arange(0, cfg.tile_size, .i32); - var loop = k.openFor(k.liftAs(0, .i32), k.liftAs(@as(i32, @intCast(partition.tiles_per_segment)), .i32), 1, .{ initial_max, initial_sum, initial_acc }); + var loop = k.openFor(0, partition.tiles_per_segment, 1, .{ initial_max, initial_sum, initial_acc }); { const selected = segment_start.add(loop.iv.mul(@as(i32, @intCast(cfg.tile_size)))).add(offsets); const position = k.loadOpts(indices_row.addPtr(selected.to(.i64).mul(cfg.indices_strides[1])), .{ @@ -210,36 +214,40 @@ fn KernelSparseAttentionPtr(comptime split_k: bool) type { }); const valid = selected.lt(cfg.indices_count).bitAnd(position.ge(0)).bitAnd(position.lt(seq_len)); // A bounded single-tile segment was already checked above. - const nonempty = if (partition.tiles_per_segment == 1 and partition.span <= 256) k.liftAs(1, .i1) else k.sum(valid.to(.i32)).gt(0); + const nonempty = if (partition.tiles_per_segment == 1 and partition.span <= 256) k.constant(.i1, 1) else k.sum(valid.to(.i32)).gt(0); var branch = k.openIfElse(nonempty, .{ k.tensorTy(&.{cfg.block_m}, .f32), k.tensorTy(&.{cfg.block_m}, .f32), k.tensorTy(&.{ cfg.block_m, cfg.head_size_padded }, .f32) }); - const safe_position = k.where(valid, position, k.zeros(&.{cfg.tile_size}, .i32)); - const page = k.loadOpts(table.addPtr(safe_position.div(@as(i32, @intCast(cfg.block_size)))), .{ - .mask = valid, - .other = k.zeros(&.{cfg.tile_size}, .i32), - }).to(.i64); - const in_page = safe_position.rem(@as(i32, @intCast(cfg.block_size))).to(.i64); - const key_offset = page.mul(cfg.key_strides[0]).add(in_page.mul(cfg.key_strides[1])).add(kv_head.to(.i64).mul(cfg.key_strides[2])).expandDims(0) - .add(dims.to(.i64).mul(cfg.key_strides[3]).expandDims(1)); - const value_offset = page.mul(cfg.value_strides[0]).add(in_page.mul(cfg.value_strides[1])).add(kv_head.to(.i64).mul(cfg.value_strides[2])).expandDims(1) - .add(dims.to(.i64).mul(cfg.value_strides[3]).expandDims(0)); - const K = k.loadOpts(a.key_ptr.addPtr(key_offset), .{ - .mask = k.mask2d(dim_mask, valid, cfg.head_size_padded, cfg.tile_size), - .other = k.zeros(&.{ cfg.head_size_padded, cfg.tile_size }, cfg.dtype), - }); - const V = k.loadOpts(a.value_ptr.addPtr(value_offset), .{ - .mask = k.mask2d(valid, dim_mask, cfg.tile_size, cfg.head_size_padded), - .other = k.zeros(&.{ cfg.tile_size, cfg.head_size_padded }, cfg.dtype), - }); - const scores = k.dot(Q, K, k.zeros(&.{ cfg.block_m, cfg.tile_size }, .f32)).mul(cfg.scale * RCP_LN2); - const S = k.where(k.mask2d(head_mask, valid, cfg.block_m, cfg.tile_size), scores, k.full(&.{ cfg.block_m, cfg.tile_size }, -std.math.inf(f32), .f32)); - const max_score = loop.carried[0].maximum(k.maxOpts(S, .{ .axis = 1 })); - const next_max = k.where(max_score.gt(-std.math.inf(f32)), max_score, k.zeros(&.{cfg.block_m}, .f32)); - const alpha = k.exp2(loop.carried[0].sub(next_max)); - const P = k.exp2(S.sub(next_max.expandDims(1))); - const sum = loop.carried[1].mul(alpha).add(k.sumOpts(P, .{ .axis = 1 })); - const acc = k.dot(P.to(cfg.dtype), V, loop.carried[2].mul(alpha.expandDims(1))); - branch.yieldThen(.{ next_max, sum, acc }); - branch.yieldElse(.{ loop.carried[0], loop.carried[1], loop.carried[2] }); + { + const safe_position = k.where(valid, position, k.zeros(&.{cfg.tile_size}, .i32)); + const page = k.loadOpts(table.addPtr(safe_position.div(@as(i32, @intCast(cfg.block_size)))), .{ + .mask = valid, + .other = k.zeros(&.{cfg.tile_size}, .i32), + }).to(.i64); + const in_page = safe_position.rem(@as(i32, @intCast(cfg.block_size))).to(.i64); + const key_offset = page.mul(cfg.key_strides[0]).add(in_page.mul(cfg.key_strides[1])).add(kv_head.to(.i64).mul(cfg.key_strides[2])).expandDims(0) + .add(dims.to(.i64).mul(cfg.key_strides[3]).expandDims(1)); + const value_offset = page.mul(cfg.value_strides[0]).add(in_page.mul(cfg.value_strides[1])).add(kv_head.to(.i64).mul(cfg.value_strides[2])).expandDims(1) + .add(dims.to(.i64).mul(cfg.value_strides[3]).expandDims(0)); + const K = k.loadOpts(a.key_ptr.addPtr(key_offset), .{ + .mask = k.mask2d(dim_mask, valid, cfg.head_size_padded, cfg.tile_size), + .other = k.zeros(&.{ cfg.head_size_padded, cfg.tile_size }, cfg.dtype), + }); + const V = k.loadOpts(a.value_ptr.addPtr(value_offset), .{ + .mask = k.mask2d(valid, dim_mask, cfg.tile_size, cfg.head_size_padded), + .other = k.zeros(&.{ cfg.tile_size, cfg.head_size_padded }, cfg.dtype), + }); + const scores = k.dot(Q, K, k.zeros(&.{ cfg.block_m, cfg.tile_size }, .f32)).mul(cfg.scale * RCP_LN2); + const S = k.where(k.mask2d(head_mask, valid, cfg.block_m, cfg.tile_size), scores, k.full(&.{ cfg.block_m, cfg.tile_size }, -std.math.inf(f32), .f32)); + const max_score = loop.carried[0].maximum(k.maxOpts(S, .{ .axis = 1 })); + const next_max = k.where(max_score.gt(-std.math.inf(f32)), max_score, k.zeros(&.{cfg.block_m}, .f32)); + const alpha = k.exp2(loop.carried[0].sub(next_max)); + const P = k.exp2(S.sub(next_max.expandDims(1))); + const sum = loop.carried[1].mul(alpha).add(k.sumOpts(P, .{ .axis = 1 })); + const acc = k.dot(P.to(cfg.dtype), V, loop.carried[2].mul(alpha.expandDims(1))); + branch.yieldThen(.{ next_max, sum, acc }); + } + { + branch.yieldElse(.{ loop.carried[0], loop.carried[1], loop.carried[2] }); + } loop.yield(.{ branch.results[0], branch.results[1], branch.results[2] }); } @@ -291,7 +299,7 @@ pub const SparseReducePtr = struct { .output_ptr = .{ .ptr = cfg.dtype }, }); const query = k.programId(.x); - k.returnIf(query.ge(k.load(a.query_start_len_ptr.addPtr(cfg.num_seqs))), .{}); + k.returnIf(query.ge(k.load(a.query_start_len_ptr.addPtr(cfg.num_seqs)))); const head = k.programId(.y); const segments = k.arange(0, cfg.num_segments, .i32); const segment_mask = segments.lt(cfg.launched_segments); @@ -302,7 +310,7 @@ pub const SparseReducePtr = struct { const sums = k.loadOpts(a.segm_expsum_ptr.addPtr(stats), .{ .mask = segment_mask, .other = k.zeros(&.{cfg.num_segments}, .f32) }); const nonempty = segment_mask.bitAnd(sums.gt(0.0)); const maximum = k.max(k.where(nonempty, maxima, k.full(&.{cfg.num_segments}, -std.math.inf(f32), .f32))); - const safe_maximum = k.where(maximum.gt(-std.math.inf(f32)), maximum, k.liftAs(0.0, .f32)); + const safe_maximum = k.where(maximum.gt(-std.math.inf(f32)), maximum, k.constant(.f32, 0.0)); const rescale = k.exp2(maxima.sub(safe_maximum)); const denominator = k.sum(sums.mul(rescale)); const offset = query.to(.i64).mul(cfg.num_query_heads).add(head.to(.i64)).mul(cfg.num_segments).add(segments.to(.i64)).mul(cfg.head_size_padded).expandDims(1).add(dims.to(.i64).expandDims(0)); @@ -311,7 +319,7 @@ pub const SparseReducePtr = struct { .other = k.zeros(&.{ cfg.num_segments, cfg.head_tile }, .f32), }); const numerator = k.sumOpts(partial.mul(rescale.expandDims(1)), .{ .axis = 0 }); - const output = numerator.div(k.where(denominator.gt(0.0), denominator, k.liftAs(1.0, .f32))); + const output = numerator.div(k.where(denominator.gt(0.0), denominator, k.constant(.f32, 1.0))); const output_offset = query.to(.i64).mul(cfg.output_strides[0]).add(head.to(.i64).mul(cfg.output_strides[1])).add(dims.to(.i64).mul(cfg.output_strides[2])); k.storeOpts(a.output_ptr.addPtr(output_offset), output.to(cfg.dtype), .{ .mask = dim_mask }); } diff --git a/zml/kernels/paged_attention/triton/unified_attention.zig b/zml/kernels/paged_attention/triton/unified_attention.zig index f74aabaf..39018f1b 100644 --- a/zml/kernels/paged_attention/triton/unified_attention.zig +++ b/zml/kernels/paged_attention/triton/unified_attention.zig @@ -347,9 +347,9 @@ fn findSeqIdx( block_q: i64, use_q_block_mode: bool, ) Value { - const left_init = k.liftAs(0, .i32); + const left_init = k.constant(.i32, 0); - var w = k.openWhile(.{ left_init, num_seqs }, .{ k.scalarTy(.i32), k.scalarTy(.i32) }); + var w = k.openWhile(.{ left_init, num_seqs }, .{ .i32, .i32 }); { const left = w.before_carried[0]; const right = w.before_carried[1]; @@ -365,7 +365,7 @@ fn findSeqIdx( else val; - var i = k.openIfElse(mid_val.le(target_idx), .{ k.scalarTy(.i32), k.scalarTy(.i32) }); + var i = k.openIfElse(mid_val.le(target_idx), .{ .i32, .i32 }); { i.yieldThen(.{ mid.add(1), right }); } @@ -418,12 +418,12 @@ fn kernelUnifiedAttention2d( const NUM_QUERIES_PER_KV: i64 = config.num_queries_per_kv; const NUM_QUERY_HEADS: i64 = config.num_query_heads; const SLIDING_WINDOW: i64 = config.sliding_window; - const stride_k_cache_0 = k.liftAs(config.k_strides[0], .i64); - const stride_k_cache_1 = k.liftAs(config.k_strides[1], .i64); - const stride_k_cache_2 = k.liftAs(config.k_strides[2], .i64); - const stride_v_cache_0 = k.liftAs(config.v_strides[0], .i64); - const stride_v_cache_1 = k.liftAs(config.v_strides[1], .i64); - const stride_v_cache_2 = k.liftAs(config.v_strides[2], .i64); + const stride_k_cache_0 = k.constant(.i64, config.k_strides[0]); + const stride_k_cache_1 = k.constant(.i64, config.k_strides[1]); + const stride_k_cache_2 = k.constant(.i64, config.k_strides[2]); + const stride_v_cache_0 = k.constant(.i64, config.v_strides[0]); + const stride_v_cache_1 = k.constant(.i64, config.v_strides[1]); + const stride_v_cache_2 = k.constant(.i64, config.v_strides[2]); const kv_head_idx = k.programId(.x); const q_block_global_idx = k.programId(.y); @@ -444,7 +444,7 @@ fn kernelUnifiedAttention2d( // Python: if q_block_local_idx * BLOCK_Q >= cur_batch_query_len: return const out_of_range = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))).ge(cur_batch_query_len); - k.returnIf(out_of_range, .{}); + k.returnIf(out_of_range); const offs_m = k.arange(0, BLOCK_M, .i32); const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); @@ -526,7 +526,7 @@ fn kernelUnifiedAttention2d( const num_tiles = cdivFn(max_seq_prefix_len, @as(i32, @intCast(TILE_SIZE))); // Sliding-window tile pruning — comptime branch. - var tile_start: Value = k.liftAs(0, .i32); + var tile_start: Value = k.constant(.i32, 0); var tile_end: Value = num_tiles; if (SLIDING_WINDOW > 0) { const qpos_lo = q_local_bq; @@ -725,12 +725,12 @@ fn kernelUnifiedAttention3d( const NUM_QUERY_HEADS: i64 = config.num_query_heads; const NUM_SEGMENTS_PER_SEQ: i64 = config.num_segments_per_seq; const SLIDING_WINDOW: i64 = config.sliding_window; - const stride_k_cache_0 = k.liftAs(config.k_strides[0], .i64); - const stride_k_cache_1 = k.liftAs(config.k_strides[1], .i64); - const stride_k_cache_2 = k.liftAs(config.k_strides[2], .i64); - const stride_v_cache_0 = k.liftAs(config.v_strides[0], .i64); - const stride_v_cache_1 = k.liftAs(config.v_strides[1], .i64); - const stride_v_cache_2 = k.liftAs(config.v_strides[2], .i64); + const stride_k_cache_0 = k.constant(.i64, config.k_strides[0]); + const stride_k_cache_1 = k.constant(.i64, config.k_strides[1]); + const stride_k_cache_2 = k.constant(.i64, config.k_strides[2]); + const stride_v_cache_0 = k.constant(.i64, config.v_strides[0]); + const stride_v_cache_1 = k.constant(.i64, config.v_strides[1]); + const stride_v_cache_2 = k.constant(.i64, config.v_strides[2]); const q_block_global_idx = k.programId(.x); const kv_head_idx = k.programId(.y); @@ -750,14 +750,14 @@ fn kernelUnifiedAttention3d( const cur_batch_query_len = cur_batch_in_all_stop_index.sub(cur_batch_in_all_start_index); const q_out_of_range = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))).ge(cur_batch_query_len); - k.returnIf(q_out_of_range, .{}); + k.returnIf(q_out_of_range); const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); const tiles_per_segment = cdivFn(seq_len, @as(i32, @intCast(NUM_SEGMENTS_PER_SEQ * TILE_SIZE))); const segm_lo = segm_idx.mul(tiles_per_segment).mul(@as(i32, @intCast(TILE_SIZE))); const segm_out_of_range = segm_lo.ge(seq_len); - k.returnIf(segm_out_of_range, .{}); + k.returnIf(segm_out_of_range); const offs_m = k.arange(0, BLOCK_M, .i32); const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); diff --git a/zml/kernels/paged_attention/triton/unified_attention_oneapi.zig b/zml/kernels/paged_attention/triton/unified_attention_oneapi.zig index 3158b53f..b7c724f9 100644 --- a/zml/kernels/paged_attention/triton/unified_attention_oneapi.zig +++ b/zml/kernels/paged_attention/triton/unified_attention_oneapi.zig @@ -44,9 +44,9 @@ fn findSeqIdx( block_q: i64, use_q_block_mode: bool, ) Value { - const left_init = k.liftAs(0, .i32); + const left_init = k.constant(.i32, 0); - var w = k.openWhile(.{ left_init, num_seqs }, .{ k.scalarTy(.i32), k.scalarTy(.i32) }); + var w = k.openWhile(.{ left_init, num_seqs }, .{ .i32, .i32 }); { const left = w.before_carried[0]; const right = w.before_carried[1]; @@ -62,7 +62,7 @@ fn findSeqIdx( else val; - var i = k.openIfElse(mid_val.le(target_idx), .{ k.scalarTy(.i32), k.scalarTy(.i32) }); + var i = k.openIfElse(mid_val.le(target_idx), .{ .i32, .i32 }); { i.yieldThen(.{ mid.add(1), right }); } @@ -169,12 +169,12 @@ pub const KernelUnifiedAttention3dPtr = struct { const query_stride_0 = b.load(a.query_stride_0_ptr); const query_stride_1 = b.load(a.query_stride_1_ptr); const qq_bias_stride_0 = b.load(a.qq_bias_stride_0_ptr); - const stride_k_cache_0 = b.liftAs(cfg.stride_k_cache_0, .i64); - const stride_k_cache_1 = b.liftAs(cfg.stride_k_cache_1, .i64); - const stride_k_cache_2 = b.liftAs(cfg.stride_k_cache_2, .i64); - const stride_v_cache_0 = b.liftAs(cfg.stride_v_cache_0, .i64); - const stride_v_cache_1 = b.liftAs(cfg.stride_v_cache_1, .i64); - const stride_v_cache_2 = b.liftAs(cfg.stride_v_cache_2, .i64); + const stride_k_cache_0 = b.constant(.i64, cfg.stride_k_cache_0); + const stride_k_cache_1 = b.constant(.i64, cfg.stride_k_cache_1); + const stride_k_cache_2 = b.constant(.i64, cfg.stride_k_cache_2); + const stride_v_cache_0 = b.constant(.i64, cfg.stride_v_cache_0); + const stride_v_cache_1 = b.constant(.i64, cfg.stride_v_cache_1); + const stride_v_cache_2 = b.constant(.i64, cfg.stride_v_cache_2); const num_seqs = b.load(a.num_seqs_ptr); kernelUnifiedAttention3d( @@ -271,14 +271,14 @@ fn kernelUnifiedAttention3d( const cur_batch_query_len = cur_batch_in_all_stop_index.sub(cur_batch_in_all_start_index); const q_out_of_range = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))).ge(cur_batch_query_len); - k.returnIf(q_out_of_range, .{}); + k.returnIf(q_out_of_range); const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); const tiles_per_segment = cdivFn(seq_len, @as(i32, @intCast(NUM_SEGMENTS_PER_SEQ * TILE_SIZE))); const segm_lo = segm_idx.mul(tiles_per_segment).mul(@as(i32, @intCast(TILE_SIZE))); const segm_out_of_range = segm_lo.ge(seq_len); - k.returnIf(segm_out_of_range, .{}); + k.returnIf(segm_out_of_range); const offs_m = k.arange(0, BLOCK_M, .i32); const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); From b37ba31500e07b03bb4d99b9cad798be17bab7be Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Tue, 27 Oct 2026 17:53:31 +0100 Subject: [PATCH 827/881] mlir/dialects: build every remaining op and attribute through the dialect packages (#993) The DSLs and the kernels no longer make ops or set dialect attributes by name: each dialect package gets the builder or helper they were missing, and every consumer goes through it. This now fix ZML core files as well --- dsl/cute/builder.zig | 30 +++--- dsl/fly/builder.zig | 17 +-- dsl/mosaic_tpu/README.md | 3 +- dsl/mosaic_tpu/builder.zig | 4 +- dsl/mosaic_tpu/pipeline.zig | 16 +-- dsl/tcl/builder.zig | 11 +- dsl/triton/builder.zig | 18 +--- mlir/dialects/arith/arith.zig | 6 ++ mlir/dialects/arith/test.zig | 9 ++ mlir/dialects/cuda/cuda.zig | 2 +- mlir/dialects/cute_ir/cute_ir.zig | 5 + mlir/dialects/cute_ir/test.zig | 9 ++ mlir/dialects/cute_nvgpu/cute_nvgpu.zig | 6 ++ mlir/dialects/cute_nvgpu/test.zig | 9 ++ mlir/dialects/gpu/gpu.zig | 13 ++- mlir/dialects/gpu/test.zig | 9 ++ mlir/dialects/llvm/llvm.zig | 6 ++ mlir/dialects/llvm/ops.zig | 102 ++++++++++++++++++ mlir/dialects/llvm/test.zig | 38 ++++++- mlir/dialects/memref/memref.zig | 2 + mlir/dialects/mosaic_tpu/mosaic_tpu.zig | 14 +++ mlir/dialects/mosaic_tpu/test.zig | 12 +++ mlir/dialects/nvvm/nvvm.zig | 14 +++ mlir/dialects/nvvm/test.zig | 9 ++ mlir/dialects/shardy/shardy.zig | 68 ++++++++++-- mlir/dialects/shardy/test.zig | 18 ++++ mlir/dialects/stablehlo/stablehlo.zig | 92 ++++++++++++++++ mlir/dialects/stablehlo/test.zig | 4 + mlir/dialects/tcl/tcl.zig | 22 ++++ mlir/dialects/tcl/test.zig | 22 ++++ mlir/dialects/ttir/test.zig | 9 ++ mlir/dialects/ttir/ttir.zig | 6 ++ mlir/mlir.zig | 26 +++++ zml/BUILD.bazel | 3 +- zml/Compiler.zig | 23 +--- zml/Sharding.zig | 12 +-- zml/kernels/moe/cute/persistent_mxfp4.zig | 11 +- .../paged_attention/cute/sm10x/attention.zig | 2 +- .../paged_attention/cute/sm10x/sm103.zig | 7 +- .../paged_attention/cute/sm120/attention.zig | 9 +- zml/ops.zig | 72 +++---------- zml/ops/manual_computation.zig | 28 ++--- zml/tensor.zig | 15 ++- 43 files changed, 608 insertions(+), 205 deletions(-) diff --git a/dsl/cute/builder.zig b/dsl/cute/builder.zig index 87f654c0..0bfeec1b 100644 --- a/dsl/cute/builder.zig +++ b/dsl/cute/builder.zig @@ -597,7 +597,7 @@ pub const Builder = struct { /// `cuda.launch_ex`. pub fn openProgram(allocator: std.mem.Allocator, ctx: *mlir.Context, name: []const u8) !Builder { var self = try open(allocator, ctx, name); - self.module.operation().setAttributeByName("gpu.container_module", .unit(ctx)); + self.module.operation().setAttribute(gpu.containerModule(ctx)); const body = mlir.Block.init(&.{}, &.{}); const gpu_module = gpu.module(ctx, "kernels", body, self.loc()); _ = gpu_module.appendTo(self.module.body()); @@ -688,9 +688,7 @@ pub const Builder = struct { var any_arg_attr = false; for (arg_attrs, args) |*attr, a| { attr.* = switch (a.kind) { - .mlir_type => |typed| if (typed.grid_constant) .dict(ctx, &.{ - .named(ctx, "cute_nvgpu.grid_constant", .unit(ctx)), - }) else empty_dict, + .mlir_type => |typed| if (typed.grid_constant) .dict(ctx, &.{cute_nvgpu.gridConstant(ctx)}) else empty_dict, else => empty_dict, }; if (attr.* != empty_dict) any_arg_attr = true; @@ -705,8 +703,8 @@ pub const Builder = struct { .location = self.loc(), .verify = false, .extra_attributes = &.{ - .named(ctx, "cute.kernel", .unit(ctx)), - .named(ctx, "gpu.kernel", .unit(ctx)), + cute_ir.kernel(ctx), + gpu.kernel(ctx), }, }), .cuda_kernel => cuda.kernel(ctx, .{ @@ -715,8 +713,8 @@ pub const Builder = struct { .body = entry, .arg_attrs = .array(ctx, arg_attrs), .extra_attributes = &.{ - .named(ctx, "cute.kernel", .unit(ctx)), - .named(ctx, "gpu.kernel", .unit(ctx)), + cute_ir.kernel(ctx), + gpu.kernel(ctx), }, .location = self.loc(), }), @@ -727,7 +725,7 @@ pub const Builder = struct { .results = &.{.int(ctx, .i32)}, .location = self.loc(), .verify = false, - .extra_attributes = &.{.named(ctx, "llvm.emit_c_interface", .unit(ctx))}, + .extra_attributes = &.{llvm.emitCInterface(ctx)}, }), }; _ = func_op.appendTo(if (self.function_kind == .cuda_kernel) @@ -832,9 +830,9 @@ pub const Builder = struct { return (cuda.ops.DevMaxSharedMemoryOptinAttr.get(self.ctx, .{}) catch @panic("invalid CUDA attribute")).attribute(); } - pub fn setFunctionAttribute(self: *Builder, name: []const u8, attribute: *const mlir.Attribute) void { + pub fn setFunctionAttribute(self: *Builder, attribute: mlir.NamedAttribute) void { const function = self.func_op orelse @panic("setFunctionAttribute called before declareArgs"); - function.setAttributeByName(name, attribute); + function.setAttribute(attribute); } // ==================== cuda host launch ==================== @@ -923,7 +921,7 @@ pub const Builder = struct { } if (block) |dims| { const function = self.func_op orelse @panic("endFunction before declareArgs"); - function.setAttributeByName("nvvm.reqntid", .denseArray(self.ctx, .i32, &dims)); + function.setAttribute(nvvm.reqntid(self.ctx, dims)); } self.func_op = null; self.entry_block = null; @@ -2295,11 +2293,7 @@ pub const Builder = struct { /// shared-memory pointer or a raw `i32` shared-memory address. fn sharedBarrierPtr(self: *Builder, barrier: Value) Value { if (barrier.isInt()) return self.addressPtr(barrier, .shared); - return self.emit(mlir.Operation.make(self.ctx, "builtin.unrealized_conversion_cast", .{ - .operands = .{ .flat = &.{barrier.inner} }, - .results = .{ .flat = &.{llvm.pointerType(self.ctx, NVPTXAddressSpace.shared)} }, - .location = self.loc(), - })); + return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{barrier.inner}, &.{llvm.pointerType(self.ctx, NVPTXAddressSpace.shared)}, self.loc())); } /// Initialize a shared-memory mbarrier expecting `arrivals` per phase. @@ -2806,7 +2800,7 @@ pub const Builder = struct { _ = func.returns(self.ctx, &.{}, self.loc()).appendTo(current); } const func_op = self.func_op orelse return error.InvalidMlir; - func_op.setAttributeByName("nvvm.reqntid", .denseArray(self.ctx, .i32, &block)); + func_op.setAttribute(nvvm.reqntid(self.ctx, block)); try common.verify(self.module, "dsl/cute"); return self.renderModule(); diff --git a/dsl/fly/builder.zig b/dsl/fly/builder.zig index ab317c38..a083d943 100644 --- a/dsl/fly/builder.zig +++ b/dsl/fly/builder.zig @@ -737,7 +737,7 @@ pub const Builder = struct { const loc_: *const mlir.Location = .unknown(ctx); const module: *mlir.Module = .init(loc_); errdefer module.deinit(); - module.operation().setAttributeByName("gpu.container_module", .unit(ctx)); + module.operation().setAttribute(gpu.containerModule(ctx)); const gpu_body = mlir.Block.init(&.{}, &.{}); const gpu_module = gpu.module(ctx, gpu_module_name, gpu_body, loc_); @@ -1103,8 +1103,7 @@ pub const Builder = struct { pub fn emitFast(self: *Builder, op: *mlir.Operation) Value { if (self.fast_math and op.numResults() > 0 and (Value{ .inner = op.result(0), .kernel = self }).isFloatElem()) { - const fast = arith.FastMathAttr.get(self.ctx, .{ .value = .fast }) catch unreachable; - op.setAttributeByName("fastmath", fast.attribute()); + op.setAttribute(arith.fastmath(self.ctx, .fast) catch unreachable); } return self.emit(op); } @@ -1159,17 +1158,7 @@ pub const Builder = struct { .agent => 1, }; const llvm_ptr = self.emit(fly.ops.to_llvm_ptr(self.ctx, ptr.inner, .int(self.ctx, .i32, space), self.loc())); - return self.emit(fly.make(self.ctx, "llvm.atomicrmw", .{ - .operands = .{ .flat = &.{ llvm_ptr.inner, value.inner } }, - .results = .{ .flat = &.{value.type_()} }, - .attributes = &.{ - .named(self.ctx, "bin_op", .int(self.ctx, .i64, 1)), // LLVM AtomicBinOp::add. - .named(self.ctx, "ordering", .int(self.ctx, .i64, 2)), // LLVM AtomicOrdering::monotonic. - .named(self.ctx, "syncscope", .string(self.ctx, @tagName(scope))), - .named(self.ctx, "alignment", .int(self.ctx, .i64, 4)), - }, - .location = self.loc(), - })); + return self.emit(llvm.atomicrmw(self.ctx, llvm_ptr.inner, value.inner, .add, .monotonic, .{ .syncscope = @tagName(scope), .alignment = 4 }, self.loc())); } /// Static LDS, as an `n:1` tensor in shared memory. Each call is its own diff --git a/dsl/mosaic_tpu/README.md b/dsl/mosaic_tpu/README.md index f9499965..cb1e0bab 100644 --- a/dsl/mosaic_tpu/README.md +++ b/dsl/mosaic_tpu/README.md @@ -6,7 +6,8 @@ to read like `pallas.tpu` Python while staying in pure Zig. Used by and `platforms/tpu/`, and by anything else that wants to emit a TPU kernel op-for-op against what `pallas_call` produces. -The DSL talks to the `tpu` dialect through `mlir.Operation.make(ctx, "tpu.", …)`. +The DSL builds `tpu` operations and attributes through the bindings in +`mlir/dialects/mosaic_tpu`. A Mosaic kernel is a pure IR generator: it builds its MLIR in a throwaway context, canonicalizes it, serializes it to a string, and hands that to a `stablehlo.custom_call` targeting `tpu_custom_call` — it never touches ZML's diff --git a/dsl/mosaic_tpu/builder.zig b/dsl/mosaic_tpu/builder.zig index c58f5e96..cffcd658 100644 --- a/dsl/mosaic_tpu/builder.zig +++ b/dsl/mosaic_tpu/builder.zig @@ -696,7 +696,7 @@ pub const Builder = struct { .iter => |io| { if (io.semantics) |sem| { arg_attrs[i] = .dict(ctx, &.{ - .named(ctx, "tpu.dimension_semantics", sem.attribute(ctx)), + mosaic_tpu.dimensionSemanticsNamedAttr(ctx, sem), }); any_arg_attr = true; } else { @@ -730,7 +730,7 @@ pub const Builder = struct { } extra_attrs.appendAssumeCapacity(.named(ctx, "scratch_operands", .int(ctx, .i64, opts.scratch_operands))); if (opts.core_type) |ct| { - extra_attrs.appendAssumeCapacity(.named(ctx, "tpu.core_type", ct.attribute(ctx))); + extra_attrs.appendAssumeCapacity(mosaic_tpu.coreTypeNamedAttr(ctx, ct)); } // Snapshot the ref args for `finish`-time stub emission. Only diff --git a/dsl/mosaic_tpu/pipeline.zig b/dsl/mosaic_tpu/pipeline.zig index 857d76bd..01264373 100644 --- a/dsl/mosaic_tpu/pipeline.zig +++ b/dsl/mosaic_tpu/pipeline.zig @@ -6,6 +6,7 @@ const std = @import("std"); const func = @import("mlir/dialects/func"); +const memref = @import("mlir/dialects/memref"); const mlir = @import("mlir"); const scf = @import("mlir/dialects/scf"); const builder = @import("builder.zig"); @@ -1014,23 +1015,12 @@ const Scheduler = struct { } }; -/// `verify = false`: `AutomaticAllocationScope` requirement is only satisfied -/// once it's inside `func.func`; full verify at `Builder.finish`. pub fn memRefAlloca(b: *Builder, ty: *const mlir.Type) Value { - return b.emit(mlir.Operation.make(b.ctx, "memref.alloca", .{ - .results = .{ .flat = &.{ty} }, - .attributes = &.{.named(b.ctx, "operandSegmentSizes", .denseArray(b.ctx, .i32, &.{ 0, 0 }))}, - .verify = false, - .location = b.loc(), - })); + return b.emit(memref.alloca(b.ctx, ty, &.{}, &.{}, .{}, b.loc())); } pub fn semAllocOp(b: *Builder, ty: *const mlir.Type) Value { - return b.emit(mlir.Operation.make(b.ctx, "tpu.sem_alloc", .{ - .results = .{ .flat = &.{ty} }, - .verify = false, - .location = b.loc(), - })); + return b.emit(mosaic_tpu.sem_alloc(b.ctx, ty, b.loc())); } /// `pl.SemaphoreType.DMA((n,))`, or rank-0 for `n == null`. diff --git a/dsl/tcl/builder.zig b/dsl/tcl/builder.zig index 3e5a8e47..f86d338b 100644 --- a/dsl/tcl/builder.zig +++ b/dsl/tcl/builder.zig @@ -1025,16 +1025,17 @@ pub const Builder = struct { const arg_types = try scratch.alloc(*const mlir.Type, entry.numArguments()); for (arg_types, 0..) |*t, i| t.* = entry.argument(i).type_(); f.setAttributeByName("function_type", .typeAttr(.function(ctx, arg_types, result_types.items))); - if (self.updated.items.len > 0) f.setAttributeByName("tcl.updated_inputs", .denseArray(ctx, .i64, updated_inputs)); const axes = try scratch.alloc(mlir.NamedAttribute, self.axes.items.len); for (axes, self.axes.items) |*dst, d| { dst.* = .named(ctx, d.axis.name, if (d.def) |e| self.exprAttr(e) else .int(ctx, .i64, d.axis.size)); } - f.setAttributeByName("tcl.axes", .dict(ctx, axes)); - if (self.compiler_config) |c| f.setAttributeByName("tcl.compiler_config", c); - if (self.auto_config) |c| f.setAttributeByName("tcl.auto_config", c); - if (self.fusible) f.setAttributeByName("tcl.fusible", .unit(ctx)); + tcl.annotateFunction(ctx, f, axes, .{ + .updated_inputs = if (self.updated.items.len > 0) updated_inputs else null, + .compiler_config = self.compiler_config, + .auto_config = self.auto_config, + .fusible = self.fusible, + }); try common.verify(self.module, "dsl/tcl"); var printed: std.Io.Writer.Allocating = .init(self.allocator); diff --git a/dsl/triton/builder.zig b/dsl/triton/builder.zig index 9917ad2c..5ccec273 100644 --- a/dsl/triton/builder.zig +++ b/dsl/triton/builder.zig @@ -579,15 +579,13 @@ pub const Builder = struct { var any_arg_attr = false; for (arg_attrs, args) |*attr, a| { attr.* = switch (a.kind) { - .ptr => .dict(ctx, &.{ - .named(ctx, "tt.divisibility", .int(ctx, .i32, 16)), - }), + .ptr => .dict(ctx, &.{ttir.divisibility(ctx, 16)}), .ptr_opts => |p| if (p.divisibility) |v| - .dict(ctx, &.{.named(ctx, "tt.divisibility", .int(ctx, .i32, v))}) + .dict(ctx, &.{ttir.divisibility(ctx, v)}) else empty_dict, .scalar_opts => |s| if (s.divisibility) |v| - .dict(ctx, &.{.named(ctx, "tt.divisibility", .int(ctx, .i32, v))}) + .dict(ctx, &.{ttir.divisibility(ctx, v)}) else empty_dict, else => empty_dict, @@ -708,15 +706,7 @@ pub const Builder = struct { } pub fn assume(self: *Builder, cond: Value) void { - const op = mlir.Operation.make(self.ctx, "llvm.intr.assume", .{ - .operands = .{ .flat = &.{cond.inner} }, - .attributes = &.{ - .named(self.ctx, "op_bundle_sizes", .denseArray(self.ctx, .i32, &.{})), - .named(self.ctx, "op_bundle_tags", .array(self.ctx, &.{})), - }, - .location = self.loc(), - }); - _ = op.appendTo(self.currentBlock()); + _ = llvm.intr_assume(self.ctx, cond.inner, self.loc()).appendTo(self.currentBlock()); } // ==================== constants ==================== diff --git a/mlir/dialects/arith/arith.zig b/mlir/dialects/arith/arith.zig index 2c52044e..8f7f6159 100644 --- a/mlir/dialects/arith/arith.zig +++ b/mlir/dialects/arith/arith.zig @@ -88,6 +88,12 @@ pub const FastMathAttr = opaque { } }; +/// `fastmath = #arith.fastmath`: the fast-math flags of a float +/// operation. +pub fn fastmath(ctx: *mlir.Context, flags: FastMathFlags) mlir.Error!mlir.NamedAttribute { + return .named(ctx, "fastmath", (try FastMathAttr.get(ctx, .{ .value = flags })).attribute()); +} + /// `mlir::arith::IntegerOverflowFlags`, one bit per flag. pub const IntegerOverflowFlags = packed struct(u32) { nsw: bool = false, diff --git a/mlir/dialects/arith/test.zig b/mlir/dialects/arith/test.zig index 913f558d..85dab738 100644 --- a/mlir/dialects/arith/test.zig +++ b/mlir/dialects/arith/test.zig @@ -94,6 +94,15 @@ test "arith fastmath attribute is built through the C API" { try std.testing.expectError(error.InvalidMlir, arith.FastMathAttr.get(ctx, .{ .value = .{ ._padding = 1 } })); } +test "fastmath names the attribute of a float operation" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + var buf: [64]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{try arith.fastmath(ctx, .fast)})}); + try std.testing.expectEqualStrings("{fastmath = #arith.fastmath}", w.buffered()); +} + test "arith overflow attribute is built through the C API" { const ctx = try mlir.Context.init(.{}); defer ctx.deinit(); diff --git a/mlir/dialects/cuda/cuda.zig b/mlir/dialects/cuda/cuda.zig index 5fb18635..2c74c920 100644 --- a/mlir/dialects/cuda/cuda.zig +++ b/mlir/dialects/cuda/cuda.zig @@ -41,7 +41,7 @@ pub const KernelArgs = struct { /// `cuda.kernel`; `body` is filled afterwards. pub fn kernel(ctx: *mlir.Context, args: KernelArgs) *mlir.Operation { const op = ops.kernel(ctx, .string(ctx, args.name), .typeAttr(args.function_type), .{args.body}, .{ .arg_attrs = args.arg_attrs }, args.location); - for (args.extra_attributes) |attr| op.setAttributeByName(attr.name().str(), attr.attribute()); + for (args.extra_attributes) |attr| op.setAttribute(attr); return op; } diff --git a/mlir/dialects/cute_ir/cute_ir.zig b/mlir/dialects/cute_ir/cute_ir.zig index f22fcf64..8a031d0b 100644 --- a/mlir/dialects/cute_ir/cute_ir.zig +++ b/mlir/dialects/cute_ir/cute_ir.zig @@ -18,6 +18,11 @@ pub fn registerDialect(registry: *mlir.DialectRegistry) void { registry.registerDialect("cute"); } +/// `cute.kernel`: marks a function as a CuTe kernel. +pub fn kernel(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "cute.kernel", .unit(ctx)); +} + /// The address space of a pointer or memref. pub const MemorySpace = AddressSpace; diff --git a/mlir/dialects/cute_ir/test.zig b/mlir/dialects/cute_ir/test.zig index 183c884e..c6f44763 100644 --- a/mlir/dialects/cute_ir/test.zig +++ b/mlir/dialects/cute_ir/test.zig @@ -188,6 +188,15 @@ test "every bound operation is registered" { try std.testing.expect(!ctx.allowUnregisteredDialects()); } +test "kernel is a unit attribute of the cute dialect" { + const ctx = try testContext(); + defer ctx.deinit(); + var buf: [64]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{cute_ir.kernel(ctx)})}); + try std.testing.expectEqualStrings("{cute.kernel}", w.buffered()); +} + test "generated types and attributes rebuild from their getters" { @setEvalBranchQuota(100_000); const ctx = try testContext(); diff --git a/mlir/dialects/cute_nvgpu/cute_nvgpu.zig b/mlir/dialects/cute_nvgpu/cute_nvgpu.zig index 4c039ecf..ac92ab5a 100644 --- a/mlir/dialects/cute_nvgpu/cute_nvgpu.zig +++ b/mlir/dialects/cute_nvgpu/cute_nvgpu.zig @@ -10,6 +10,12 @@ pub fn registerDialect(registry: *mlir.DialectRegistry) void { registry.registerDialect("cute_nvgpu"); } +/// `cute_nvgpu.grid_constant`: marks a kernel argument passed as a +/// `__grid_constant__` parameter. +pub fn gridConstant(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "cute_nvgpu.grid_constant", .unit(ctx)); +} + pub const NVGPUType = opaque { const M = mlir.Methods(NVGPUType, c.MlirType); diff --git a/mlir/dialects/cute_nvgpu/test.zig b/mlir/dialects/cute_nvgpu/test.zig index 7c6d0fc6..3c7de6ce 100644 --- a/mlir/dialects/cute_nvgpu/test.zig +++ b/mlir/dialects/cute_nvgpu/test.zig @@ -91,6 +91,15 @@ test "every bound operation is registered" { try std.testing.expect(!ctx.allowUnregisteredDialects()); } +test "grid_constant is a unit attribute of the cute_nvgpu dialect" { + const ctx = try testContext(); + defer ctx.deinit(); + var buf: [64]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{cute_nvgpu.gridConstant(ctx)})}); + try std.testing.expectEqualStrings("{cute_nvgpu.grid_constant}", w.buffered()); +} + test "generated types and attributes rebuild from their getters" { @setEvalBranchQuota(100_000); const ctx = try testContext(); diff --git a/mlir/dialects/gpu/gpu.zig b/mlir/dialects/gpu/gpu.zig index 1e3d76fe..4b9df47f 100644 --- a/mlir/dialects/gpu/gpu.zig +++ b/mlir/dialects/gpu/gpu.zig @@ -71,6 +71,17 @@ pub const ShuffleModeAttr = opaque { } }; +/// `gpu.container_module`: marks a module whose `gpu.module`s hold the kernels +/// it launches. +pub fn containerModule(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "gpu.container_module", .unit(ctx)); +} + +/// `gpu.kernel`: marks a function as a kernel entry point. +pub fn kernel(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "gpu.kernel", .unit(ctx)); +} + pub fn module(ctx: *mlir.Context, name: []const u8, body: *mlir.Block, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "gpu.module", .{ .blocks = &.{body}, @@ -98,7 +109,7 @@ pub fn func(ctx: *mlir.Context, args: FuncArgs) *mlir.Operation { .named(ctx, "sym_name", .string(ctx, args.name)), .named(ctx, "function_type", .typeAttr(.function(ctx, args.args, &.{}))), }); - if (args.kernel) attrs.appendAssumeCapacity(.named(ctx, "gpu.kernel", .unit(ctx))); + if (args.kernel) attrs.appendAssumeCapacity(kernel(ctx)); attrs.appendSliceAssumeCapacity(args.extra_attributes); return mlir.Operation.make(ctx, "gpu.func", .{ .blocks = &.{args.block}, diff --git a/mlir/dialects/gpu/test.zig b/mlir/dialects/gpu/test.zig index bcf37065..d0f4a80d 100644 --- a/mlir/dialects/gpu/test.zig +++ b/mlir/dialects/gpu/test.zig @@ -50,6 +50,15 @@ test "gpu enum attributes are built through the C API" { } } +test "container_module and kernel are unit attributes of the gpu dialect" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + var buf: [128]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{ gpu.containerModule(ctx), gpu.kernel(ctx) })}); + try std.testing.expectEqualStrings("{gpu.container_module, gpu.kernel}", w.buffered()); +} + test "every bound operation is registered" { const registry = try mlir.DialectRegistry.init(); defer registry.deinit(); diff --git a/mlir/dialects/llvm/llvm.zig b/mlir/dialects/llvm/llvm.zig index 167099eb..22c8539f 100644 --- a/mlir/dialects/llvm/llvm.zig +++ b/mlir/dialects/llvm/llvm.zig @@ -10,9 +10,13 @@ pub const LoopUnrollAttr = ops.LoopUnrollAttr; pub const LoopAnnotationAttr = ops.LoopAnnotationAttr; pub const Linkage = ops.Linkage; pub const linkageAttr = ops.linkageAttr; +pub const emitCInterface = ops.emitCInterface; +pub const AtomicBinOp = ops.AtomicBinOp; +pub const AtomicOrdering = ops.AtomicOrdering; pub const inttoptr = ops.inttoptr; pub const bitcast = ops.bitcast; pub const fmul = ops.fmul; +pub const extractvalue = ops.extractvalue; pub const LoadOpts = ops.LoadOpts; pub const load = ops.load; pub const StoreOpts = ops.StoreOpts; @@ -20,6 +24,8 @@ pub const store = ops.store; pub const GEPIndex = ops.GEPIndex; pub const getelementptr = ops.getelementptr; pub const alloca = ops.alloca; +pub const atomicrmw = ops.atomicrmw; +pub const intr_assume = ops.intr_assume; pub const call_intrinsic = ops.call_intrinsic; pub const InlineAsmOpts = ops.InlineAsmOpts; pub const inline_asm = ops.inline_asm; diff --git a/mlir/dialects/llvm/ops.zig b/mlir/dialects/llvm/ops.zig index 0515986f..731d2f55 100644 --- a/mlir/dialects/llvm/ops.zig +++ b/mlir/dialects/llvm/ops.zig @@ -109,6 +109,50 @@ pub fn linkageAttr(ctx: *mlir.Context, linkage: Linkage) *const mlir.Attribute { return @ptrCast(c.mlirLLVMLinkageAttrGet(ctx.ptr(), @backingInt(linkage)).ptr); } +/// `llvm.emit_c_interface`: has the lowering of a `func.func` emit a +/// `_mlir_ciface_` wrapper that takes its memrefs by pointer. +pub fn emitCInterface(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "llvm.emit_c_interface", .unit(ctx)); +} + +/// The operation of `llvm.atomicrmw`, as the LLVM dialect numbers it. +pub const AtomicBinOp = enum(i64) { + xchg = 0, + add = 1, + sub = 2, + _and = 3, + nand = 4, + _or = 5, + _xor = 6, + max = 7, + min = 8, + umax = 9, + umin = 10, + fadd = 11, + fsub = 12, + fmax = 13, + fmin = 14, + uinc_wrap = 15, + udec_wrap = 16, + usub_cond = 17, + usub_sat = 18, + fmaximum = 19, + fminimum = 20, + fmaximumnum = 21, + fminimumnum = 22, +}; + +/// The memory ordering of an atomic operation, as the LLVM dialect numbers it. +pub const AtomicOrdering = enum(i64) { + not_atomic = 0, + unordered = 1, + monotonic = 2, + acquire = 4, + release = 5, + acq_rel = 6, + seq_cst = 7, +}; + /// `#llvm.loop_annotation`: the `llvm.loop` metadata of a loop. pub const LoopAnnotationAttr = opaque { const Self = @This(); @@ -166,6 +210,21 @@ pub fn fmul(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, }); } +// ============================================================================= +// Aggregates +// ============================================================================= + +/// llvm.extractvalue — the element of type `result_type` at `position` in the +/// struct or array `container`. +pub fn extractvalue(ctx: *mlir.Context, container: *const mlir.Value, result_type: *const mlir.Type, position: []const i64, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.extractvalue", .{ + .operands = .{ .flat = &.{container} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{.named(ctx, "position", .denseArray(ctx, .i64, position))}, + .location = location, + }); +} + // ============================================================================= // Memory // ============================================================================= @@ -273,10 +332,53 @@ pub fn alloca( }); } +/// llvm.atomicrmw — combine `val` into the value at `ptr` with `bin_op`, +/// atomically; the result is the previous value, of `val`'s type. +pub fn atomicrmw( + ctx: *mlir.Context, + ptr: *const mlir.Value, + val: *const mlir.Value, + bin_op: AtomicBinOp, + ordering: AtomicOrdering, + opts: struct { + /// Synchronization scope, e.g. `"agent"` (none: the whole system). + syncscope: ?[]const u8 = null, + /// Alignment in bytes (none: the natural alignment of the type). + alignment: ?u64 = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "bin_op", .int(ctx, .i64, @backingInt(bin_op))), + .named(ctx, "ordering", .int(ctx, .i64, @backingInt(ordering))), + }); + if (opts.syncscope) |syncscope| attrs.appendAssumeCapacity(.named(ctx, "syncscope", .string(ctx, syncscope))); + if (opts.alignment) |alignment| attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, alignment))); + return mlir.Operation.make(ctx, "llvm.atomicrmw", .{ + .operands = .{ .flat = &.{ ptr, val } }, + .results = .{ .flat = &.{val.type_()} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + // ============================================================================= // Intrinsics and inline assembly // ============================================================================= +/// llvm.intr.assume — let the optimizer assume that `cond` (i1) holds. +pub fn intr_assume(ctx: *mlir.Context, cond: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.intr.assume", .{ + .operands = .{ .flat = &.{cond} }, + .attributes = &.{ + .named(ctx, "op_bundle_sizes", .denseArray(ctx, .i32, &.{})), + .named(ctx, "op_bundle_tags", .array(ctx, &.{})), + }, + .location = location, + }); +} + /// llvm.call_intrinsic — call the LLVM intrinsic `name` (e.g. `llvm.nvvm.*`), with an /// optional result of type `result_type`. pub fn call_intrinsic( diff --git a/mlir/dialects/llvm/test.zig b/mlir/dialects/llvm/test.zig index 62a191f5..b4f09648 100644 --- a/mlir/dialects/llvm/test.zig +++ b/mlir/dialects/llvm/test.zig @@ -5,11 +5,14 @@ const mlir = @import("mlir"); const operation_names: []const []const u8 = &.{ "llvm.alloca", + "llvm.atomicrmw", "llvm.bitcast", "llvm.call_intrinsic", - "llvm.getelementptr", + "llvm.extractvalue", "llvm.fmul", + "llvm.getelementptr", "llvm.inline_asm", + "llvm.intr.assume", "llvm.inttoptr", "llvm.load", "llvm.store", @@ -77,6 +80,39 @@ test "getelementptr and alloca build verified operations" { try std.testing.expect(alloca.verify()); } +test "extractvalue, atomicrmw and intr.assume build verified operations" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + const loc: *const mlir.Location = .unknown(ctx); + const pair = llvm.structType(ctx, &.{ .int(ctx, .i32), .float(ctx, .f32) }); + const block = mlir.Block.init(&.{ pair, llvm.pointerType(ctx, llvm.NVPTXAddressSpace.global), .int(ctx, .i1) }, &.{ loc, loc, loc }); + defer block.deinit(); + const first = llvm.extractvalue(ctx, block.argument(0), .int(ctx, .i32), &.{0}, loc).appendTo(block); + try std.testing.expect(first.verify()); + const previous = llvm.atomicrmw(ctx, block.argument(1), first.result(0), .add, .monotonic, .{ .syncscope = "agent", .alignment = 4 }, loc).appendTo(block); + try std.testing.expect(previous.verify()); + try std.testing.expect(previous.result(0).type_().eql(.int(ctx, .i32))); + var writer: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer writer.deinit(); + try writer.writer.print("{f}", .{previous}); + try std.testing.expect(std.mem.indexOf(u8, writer.written(), "llvm.atomicrmw add ") != null); + try std.testing.expect(std.mem.indexOf(u8, writer.written(), " monotonic ") != null); + try std.testing.expect(llvm.intr_assume(ctx, block.argument(2), loc).appendTo(block).verify()); +} + +test "emit_c_interface is a unit attribute of the LLVM dialect" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + var writer: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer writer.deinit(); + try writer.writer.print("{f}", .{mlir.Attribute.dict(ctx, &.{llvm.emitCInterface(ctx)})}); + try std.testing.expectEqualStrings("{llvm.emit_c_interface}", writer.written()); +} + test "loop annotations are built through the C API" { const ctx = try mlir.Context.init(.{}); defer ctx.deinit(); diff --git a/mlir/dialects/memref/memref.zig b/mlir/dialects/memref/memref.zig index d0b337b9..6c45eb33 100644 --- a/mlir/dialects/memref/memref.zig +++ b/mlir/dialects/memref/memref.zig @@ -55,6 +55,7 @@ pub fn alloc( } /// memref.alloca — stack allocation; same operand/attribute layout as alloc. +/// Unverified: it only verifies inside an `AutomaticAllocationScope`. pub fn alloca( ctx: *mlir.Context, result_type: *const mlir.Type, @@ -78,6 +79,7 @@ pub fn alloca( .operands = .{ .flat = operands_buf.constSlice() }, .results = .{ .flat = &.{result_type} }, .attributes = attrs.constSlice(), + .verify = false, .location = location, }); } diff --git a/mlir/dialects/mosaic_tpu/mosaic_tpu.zig b/mlir/dialects/mosaic_tpu/mosaic_tpu.zig index 51bbd36e..8425e7b4 100644 --- a/mlir/dialects/mosaic_tpu/mosaic_tpu.zig +++ b/mlir/dialects/mosaic_tpu/mosaic_tpu.zig @@ -575,6 +575,20 @@ pub const DotDimensionNumbersAttr = opaque { } }; +// ============================================================================= +// Discardable attributes — tpu.core_type, tpu.dimension_semantics +// ============================================================================= + +/// `tpu.core_type`, on a kernel's `func.func`: the core it runs on. +pub fn coreTypeNamedAttr(ctx: *mlir.Context, value: CoreType) mlir.NamedAttribute { + return .named(ctx, "tpu.core_type", value.attribute(ctx)); +} + +/// `tpu.dimension_semantics`, on a kernel's iteration index argument. +pub fn dimensionSemanticsNamedAttr(ctx: *mlir.Context, value: DimensionSemantics) mlir.NamedAttribute { + return .named(ctx, "tpu.dimension_semantics", value.attribute(ctx)); +} + // ============================================================================= // Reductions / scan / sort // ============================================================================= diff --git a/mlir/dialects/mosaic_tpu/test.zig b/mlir/dialects/mosaic_tpu/test.zig index 22853676..20a20f42 100644 --- a/mlir/dialects/mosaic_tpu/test.zig +++ b/mlir/dialects/mosaic_tpu/test.zig @@ -257,6 +257,18 @@ test "tpu array attributes are built through the C API" { try expectPrints("#tpu.dot_dimension_numbers<[2], [1], [1], [], [0, 0, 0, 1], [0], [0]>", batched); } +test "kernel attributes carry their tpu names" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + const core_type = mosaic_tpu.coreTypeNamedAttr(ctx, .sc_vector_subcore); + try std.testing.expectEqualStrings("tpu.core_type", core_type.name().str()); + try std.testing.expect(core_type.attribute().eql(mosaic_tpu.CoreType.sc_vector_subcore.attribute(ctx))); + const semantics = mosaic_tpu.dimensionSemanticsNamedAttr(ctx, .arbitrary); + try std.testing.expectEqualStrings("tpu.dimension_semantics", semantics.name().str()); + try std.testing.expect(semantics.attribute().eql(mosaic_tpu.DimensionSemantics.arbitrary.attribute(ctx))); +} + test "every bound operation is registered" { const registry = try mlir.DialectRegistry.init(); defer registry.deinit(); diff --git a/mlir/dialects/nvvm/nvvm.zig b/mlir/dialects/nvvm/nvvm.zig index 701b9de6..d46f3a4f 100644 --- a/mlir/dialects/nvvm/nvvm.zig +++ b/mlir/dialects/nvvm/nvvm.zig @@ -38,6 +38,20 @@ pub const ProxyKind = ops.ProxyKind; /// `#nvvm.shared_space<...>`: `shared::cta` or `shared::cluster`. pub const SharedSpace = ops.SharedSpace; +// ============================================================================= +// Kernel attributes +// ============================================================================= + +/// `nvvm.reqntid`: the CTA shape (x, y, z) a kernel is always launched with. +pub fn reqntid(ctx: *mlir.Context, dims: [3]i32) mlir.NamedAttribute { + return .named(ctx, "nvvm.reqntid", .denseArray(ctx, .i32, &dims)); +} + +/// `nvvm.minctasm`: the number of CTAs a kernel must fit on one SM at once. +pub fn minctasm(ctx: *mlir.Context, ctas: i32) mlir.NamedAttribute { + return .named(ctx, "nvvm.minctasm", .int(ctx, .i32, ctas)); +} + // ============================================================================= // Special registers and thread synchronization // ============================================================================= diff --git a/mlir/dialects/nvvm/test.zig b/mlir/dialects/nvvm/test.zig index 6129c078..b908f6f4 100644 --- a/mlir/dialects/nvvm/test.zig +++ b/mlir/dialects/nvvm/test.zig @@ -440,6 +440,15 @@ test "nvvm types and attributes print as the compiler does" { } } +test "reqntid and minctasm are kernel attributes of the nvvm dialect" { + const ctx = try testContext(); + defer ctx.deinit(); + var buf: [128]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{ nvvm.reqntid(ctx, .{ 128, 1, 1 }), nvvm.minctasm(ctx, 1) })}); + try std.testing.expectEqualStrings("{nvvm.minctasm = 1 : i32, nvvm.reqntid = array}", w.buffered()); +} + test "nvvm target rejects what the compiler rejects" { const ctx = try testContext(); defer ctx.deinit(); diff --git a/mlir/dialects/shardy/shardy.zig b/mlir/dialects/shardy/shardy.zig index e89685a2..d73bf81b 100644 --- a/mlir/dialects/shardy/shardy.zig +++ b/mlir/dialects/shardy/shardy.zig @@ -11,12 +11,64 @@ pub fn registerDialect(registry: *mlir.DialectRegistry) void { registry.registerDialect("sdy"); } -pub fn meshAttr(ctx: *mlir.Context, args: MeshAttr.InitArgs) *const mlir.Attribute { - return MeshAttr.get(ctx, args).attribute(); +/// `sdy.sharding`, the sharding of a function argument or result. +pub fn shardingNamedAttr(ctx: *mlir.Context, sharding: *const TensorShardingAttr) mlir.NamedAttribute { + return .named(ctx, "sdy.sharding", sharding.attribute()); } -pub fn shardingsPerValueAttr(ctx: *mlir.Context, shardings: []const *const TensorShardingAttr) *const mlir.Attribute { - return TensorShardingPerValueAttr.get(ctx, shardings).attribute(); +/// `sdy.mesh`. Not verified: it only verifies inside a module. +pub fn mesh(ctx: *mlir.Context, sym_name: []const u8, mesh_: *const MeshAttr, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "sdy.mesh", .{ + .attributes = &.{ + .named(ctx, "sym_name", .string(ctx, sym_name)), + .named(ctx, "mesh", mesh_.attribute()), + }, + .verify = false, + .location = location, + }); +} + +pub fn sharding_constraint(ctx: *mlir.Context, input: *const mlir.Value, sharding: *const TensorShardingAttr, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "sdy.sharding_constraint", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{input.type_()} }, + .attributes = &.{ + .named(ctx, "sharding", sharding.attribute()), + }, + .location = location, + }); +} + +pub fn manual_computation( + ctx: *mlir.Context, + tensors: []const *const mlir.Value, + result_types: []const *const mlir.Type, + body_block: *mlir.Block, + in_shardings: *const TensorShardingPerValueAttr, + out_shardings: *const TensorShardingPerValueAttr, + manual_axes: *const ManualAxesAttr, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "sdy.manual_computation", .{ + .operands = .{ .flat = tensors }, + .results = .{ .flat = result_types }, + .blocks = &.{body_block}, + .attributes = &.{ + .named(ctx, "in_shardings", in_shardings.attribute()), + .named(ctx, "out_shardings", out_shardings.attribute()), + .named(ctx, "manual_axes", manual_axes.attribute()), + }, + .location = location, + }); +} + +/// `sdy.return`. Not verified: a terminator only verifies inside its parent. +pub fn return_(ctx: *mlir.Context, results: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "sdy.return", .{ + .operands = .{ .flat = results }, + .verify = false, + .location = location, + }); } pub const MeshAxisAttr = opaque { @@ -64,14 +116,14 @@ pub const MeshAttr = opaque { device_ids: []const i64 = &.{}, }; - pub fn get(ctx: *mlir.Context, mesh: InitArgs) *const MeshAttr { - const axes: []const c.struct_MlirAttribute = @ptrCast(mesh.axes); + pub fn get(ctx: *mlir.Context, args: InitArgs) *const MeshAttr { + const axes: []const c.struct_MlirAttribute = @ptrCast(args.axes); return @ptrCast(c.sdyMeshAttrGet( ctx.ptr(), @intCast(axes.len), axes.ptr, - @intCast(mesh.device_ids.len), - mesh.device_ids.ptr, + @intCast(args.device_ids.len), + args.device_ids.ptr, ).ptr); } diff --git a/mlir/dialects/shardy/test.zig b/mlir/dialects/shardy/test.zig index 0ae161e7..a4a757d2 100644 --- a/mlir/dialects/shardy/test.zig +++ b/mlir/dialects/shardy/test.zig @@ -8,6 +8,13 @@ const ManualAxesAttr = shardy.ManualAxesAttr; const TensorShardingAttr = shardy.TensorShardingAttr; const TensorShardingPerValueAttr = shardy.TensorShardingPerValueAttr; +const operation_names: []const []const u8 = &.{ + "sdy.manual_computation", + "sdy.mesh", + "sdy.return", + "sdy.sharding_constraint", +}; + test { @setEvalBranchQuota(100_000); std.testing.refAllDecls(shardy); @@ -17,6 +24,17 @@ test { } } +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + shardy.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + test TensorShardingAttr { const registry: *mlir.DialectRegistry = try .init(); defer registry.deinit(); diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 5d845934..385a95b0 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -690,6 +690,98 @@ pub fn while_( return mlir.Operation.init(&state) catch @panic("Failed to create stablehlo.while operation"); } +/// `stablehlo.if`. Not verified: its branches may use values defined +/// outside them, which only verify once the operation is in place. +pub fn if_( + ctx: *mlir.Context, + pred: *const mlir.Value, + true_branch_block: *mlir.Block, + false_branch_block: *mlir.Block, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.if", .{ + .operands = .{ .flat = &.{pred} }, + .result_type_inference = true, + .blocks = &.{ true_branch_block, false_branch_block }, + .verify = false, + .location = location, + }); +} + +pub fn reduce( + ctx: *mlir.Context, + inputs: []const *const mlir.Value, + init_values: []const *const mlir.Value, + body_block: *mlir.Block, + dimensions: []const i64, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.reduce", .{ + .operands = .{ .variadic = &.{ inputs, init_values } }, + .result_type_inference = true, + .blocks = &.{body_block}, + .attributes = &.{ + .named(ctx, "dimensions", .denseArray(ctx, .i64, dimensions)), + }, + .location = location, + }); +} + +pub fn reduce_window( + ctx: *mlir.Context, + inputs: []const *const mlir.Value, + init_values: []const *const mlir.Value, + body_block: *mlir.Block, + window_dimensions: []const i64, + opts: struct { + window_strides: ?[]const i64 = null, + base_dilations: ?[]const i64 = null, + window_dilations: ?[]const i64 = null, + /// (low, high) padding of each dimension. + padding: ?[]const [2]i64 = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attrs.appendAssumeCapacity(.named(ctx, "window_dimensions", .denseArray(ctx, .i64, window_dimensions))); + if (opts.window_strides) |v| attrs.appendAssumeCapacity(.named(ctx, "window_strides", .denseArray(ctx, .i64, v))); + if (opts.base_dilations) |v| attrs.appendAssumeCapacity(.named(ctx, "base_dilations", .denseArray(ctx, .i64, v))); + if (opts.window_dilations) |v| attrs.appendAssumeCapacity(.named(ctx, "window_dilations", .denseArray(ctx, .i64, v))); + if (opts.padding) |v| attrs.appendAssumeCapacity(.named(ctx, "padding", .denseElements( + .rankedTensor(&.{ @intCast(v.len), 2 }, .int(ctx, .i64)), + @as([]const i64, @ptrCast(v)), + ))); + return mlir.Operation.make(ctx, "stablehlo.reduce_window", .{ + .operands = .{ .variadic = &.{ inputs, init_values } }, + .result_type_inference = true, + .blocks = &.{body_block}, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub fn sort( + ctx: *mlir.Context, + inputs: []const *const mlir.Value, + comparator_block: *mlir.Block, + opts: struct { + dimension: i64 = -1, + is_stable: bool = false, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.sort", .{ + .operands = .{ .variadic = &.{inputs} }, + .result_type_inference = true, + .blocks = &.{comparator_block}, + .attributes = &.{ + .named(ctx, "dimension", .int(ctx, .i64, opts.dimension)), + .named(ctx, "is_stable", .boolean(ctx, opts.is_stable)), + }, + .location = location, + }); +} + pub fn get_tuple_element(ctx: *mlir.Context, tuple_value: *const mlir.Value, index: i64, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.get_tuple_element", .{ .operands = .{ .flat = &.{tuple_value} }, diff --git a/mlir/dialects/stablehlo/test.zig b/mlir/dialects/stablehlo/test.zig index f1a2267f..c22b2ebd 100644 --- a/mlir/dialects/stablehlo/test.zig +++ b/mlir/dialects/stablehlo/test.zig @@ -35,6 +35,7 @@ const operation_names: []const []const u8 = &.{ "stablehlo.floor", "stablehlo.gather", "stablehlo.get_tuple_element", + "stablehlo.if", "stablehlo.imag", "stablehlo.iota", "stablehlo.is_finite", @@ -53,7 +54,9 @@ const operation_names: []const []const u8 = &.{ "stablehlo.popcnt", "stablehlo.power", "stablehlo.real", + "stablehlo.reduce", "stablehlo.reduce_precision", + "stablehlo.reduce_window", "stablehlo.remainder", "stablehlo.reshape", "stablehlo.return", @@ -71,6 +74,7 @@ const operation_names: []const []const u8 = &.{ "stablehlo.sign", "stablehlo.sine", "stablehlo.slice", + "stablehlo.sort", "stablehlo.sqrt", "stablehlo.subtract", "stablehlo.tanh", diff --git a/mlir/dialects/tcl/tcl.zig b/mlir/dialects/tcl/tcl.zig index 4c1f857a..a3eb4673 100644 --- a/mlir/dialects/tcl/tcl.zig +++ b/mlir/dialects/tcl/tcl.zig @@ -120,6 +120,28 @@ pub fn dramMappingAttr(ctx: *mlir.Context, chip: []const *const mlir.Attribute, )); } +/// Discardable TCL attributes of a `func.func`. +pub const FunctionOpts = struct { + /// `tcl.updated_inputs`: per result, the argument it updates in place, or -1. + updated_inputs: ?[]const i64 = null, + /// `tcl.compiler_config`, a `configAttr`. + compiler_config: ?*const mlir.Attribute = null, + /// `tcl.auto_config`, a `configAttr`. + auto_config: ?*const mlir.Attribute = null, + /// `tcl.fusible`: the plugin may compile the function into the unit around it. + fusible: bool = false, +}; + +/// Sets `tcl.axes`, each axis's size or definition by name, and the +/// attributes of `opts` on `function`. +pub fn annotateFunction(ctx: *mlir.Context, function: *mlir.Operation, axes: []const mlir.NamedAttribute, opts: FunctionOpts) void { + if (opts.updated_inputs) |inputs| function.setAttributeByName("tcl.updated_inputs", .denseArray(ctx, .i64, inputs)); + function.setAttributeByName("tcl.axes", .dict(ctx, axes)); + if (opts.compiler_config) |config| function.setAttributeByName("tcl.compiler_config", config); + if (opts.auto_config) |config| function.setAttributeByName("tcl.auto_config", config); + if (opts.fusible) function.setAttributeByName("tcl.fusible", .unit(ctx)); +} + // ============================================================================= // Types // ============================================================================= diff --git a/mlir/dialects/tcl/test.zig b/mlir/dialects/tcl/test.zig index 784739ea..5adc34c0 100644 --- a/mlir/dialects/tcl/test.zig +++ b/mlir/dialects/tcl/test.zig @@ -96,6 +96,28 @@ test "filter_compaction verifies inside a kernel" { try std.testing.expect(!(try compactionKernel(ctx, entry, try tcl.symbolAttr(ctx, "Missing"))).verify()); } +test "function attributes carry their tcl names" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc: *const mlir.Location = .unknown(ctx); + const entry = mlir.Block.init(&.{}, &.{}); + _ = func.returns(ctx, &.{}, loc).appendTo(entry); + const function = func.func(ctx, .{ .name = "f", .block = entry, .location = loc }); + defer function.deinit(); + const config = try tcl.configAttr(ctx, &.{.named(ctx, "lowering_mode", .string(ctx, "Heuristic"))}); + tcl.annotateFunction(ctx, function, &.{.named(ctx, "A", .int(ctx, .i64, 4))}, .{ + .updated_inputs = &.{-1}, + .compiler_config = config, + .auto_config = config, + .fusible = true, + }); + try std.testing.expect(function.verify()); + var text: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer text.deinit(); + try text.writer.print("{f}", .{function}); + try std.testing.expect(std.mem.indexOf(u8, text.written(), "attributes {tcl.auto_config = #tcl.config<{lowering_mode = \"Heuristic\"}>, tcl.axes = {A = 4 : i64}, tcl.compiler_config = #tcl.config<{lowering_mode = \"Heuristic\"}>, tcl.fusible, tcl.updated_inputs = array}") != null); +} + test "attributes and types print and reparse" { const ctx = try testContext(); defer ctx.deinit(); diff --git a/mlir/dialects/ttir/test.zig b/mlir/dialects/ttir/test.zig index 390f34b5..04361e4e 100644 --- a/mlir/dialects/ttir/test.zig +++ b/mlir/dialects/ttir/test.zig @@ -151,6 +151,15 @@ test "tt.func round-trip" { try std.testing.expect(parsed.operation().verify()); } +test "divisibility is an i32 attribute of the tt dialect" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + var buf: [64]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{ttir.divisibility(ctx, 16)})}); + try std.testing.expectEqualStrings("{tt.divisibility = 16 : i32}", w.buffered()); +} + test "every bound operation is registered" { const registry = try mlir.DialectRegistry.init(); defer registry.deinit(); diff --git a/mlir/dialects/ttir/ttir.zig b/mlir/dialects/ttir/ttir.zig index b55bd418..ee7b5307 100644 --- a/mlir/dialects/ttir/ttir.zig +++ b/mlir/dialects/ttir/ttir.zig @@ -210,6 +210,12 @@ pub fn descriptorReduceKindAttr(ctx: *mlir.Context, v: DescriptorReduceKind) *co return @ptrCast(c.mlirTritonDescriptorReduceKindAttrGet(ctx.ptr(), @backingInt(v)).ptr); } +/// `tt.divisibility`: an argument attribute promising that the argument (the +/// address, for a pointer) is a multiple of `divisor`. +pub fn divisibility(ctx: *mlir.Context, divisor: u32) mlir.NamedAttribute { + return .named(ctx, "tt.divisibility", .int(ctx, .i32, divisor)); +} + // ============================================================================= // Function ops — tt.func / tt.return // ============================================================================= diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 6176b2d4..57e57bd2 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -42,6 +42,17 @@ test "variadic operation groups preserve order and dense array segment sizes" { try std.testing.expect(op.attributeByName("resultSegmentSizes").?.eql(.denseArray(ctx, .i32, &.{ 0, 2, 1 }))); } +test "unrealized_conversion_cast builds a verified operation" { + const ctx = try Context.init(.{}); + defer ctx.deinit(); + const loc = Location.unknown(ctx); + const block = Block.init(&.{Type.int(ctx, .i32)}, &.{loc}); + defer block.deinit(); + const cast = unrealized_conversion_cast(ctx, &.{block.argument(0)}, &.{Type.index(ctx)}, loc).appendTo(block); + try std.testing.expect(cast.verify()); + try std.testing.expect(cast.result(0).type_().eql(Type.index(ctx))); +} + test "nested symbol references print their path" { const ctx = try Context.init(.{}); defer ctx.deinit(); @@ -1301,6 +1312,11 @@ pub const Operation = opaque { c.mlirOperationSetAttributeByName(self.ptr(), stringRef(name_), attribute.ptr()); } + /// Sets `attribute` under its name, as a dialect's attribute helper names it. + pub fn setAttribute(self: *Operation, attribute: NamedAttribute) void { + self.setAttributeByName(attribute.name().str(), attribute.attribute()); + } + pub fn attributeByName(self: *const Operation, name_: []const u8) ?*const Attribute { const attr = c.mlirOperationGetAttributeByName(self.ptr(), stringRef(name_)); return if (attr.ptr == null) null else @ptrCast(attr.ptr); @@ -1532,6 +1548,16 @@ pub const Operation = opaque { } }; +/// builtin.unrealized_conversion_cast — `inputs` as values of `outputs_types`, +/// a cast between type systems that a later conversion resolves. +pub fn unrealized_conversion_cast(ctx: *Context, inputs: []const *const Value, outputs_types: []const *const Type, location: *const Location) *Operation { + return Operation.make(ctx, "builtin.unrealized_conversion_cast", .{ + .operands = .{ .flat = inputs }, + .results = .{ .flat = outputs_types }, + .location = location, + }); +} + pub const Region = opaque { const M = Methods(Region, c.MlirRegion); diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 66d5422e..41ae31c1 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -120,12 +120,12 @@ zig_library( "//zml/kernels/moe/stablehlo", "//zml/kernels/moe/tcl", "//zml/kernels/moe/triton", + "//zml/kernels/paged_attention/cute", "//zml/kernels/paged_attention/metal", "//zml/kernels/paged_attention/mosaic_tpu", "//zml/kernels/paged_attention/stablehlo", "//zml/kernels/paged_attention/tcl", "//zml/kernels/paged_attention/triton", - "//zml/kernels/paged_attention/cute", ], data = select({ "//platforms:neuron.enabled": [ @@ -158,6 +158,7 @@ zig_library( "//mlir/dialects/func", "//mlir/dialects/llvm", "//mlir/dialects/mosaic_tpu", + "//mlir/dialects/nvvm", "//mlir/dialects/shardy", "//mlir/dialects/stablehlo", "//mlir/dialects/ttir", diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 5ee5e120..49277368 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -529,19 +529,7 @@ fn addPartitionerOperations(ctx: *Compiler) !void { }, .shardy => { for (ctx.meshes) |shd| { - const name = shd.name; - const mesh_attr = shd.sdyMeshAttr(mlir_ctx); - - const mesh_op = mlir.Operation.make(mlir_ctx, "sdy.mesh", .{ - .attributes = &.{ - .named(mlir_ctx, "sym_name", .string(mlir_ctx, name)), - .named(mlir_ctx, "mesh", mesh_attr), - }, - .location = .unknown(mlir_ctx), - .verify = false, - }); - - _ = mesh_op.appendTo(module.body()); + _ = shardy.mesh(mlir_ctx, shd.name, shd.sdyMeshAttr(mlir_ctx), .unknown(mlir_ctx)).appendTo(module.body()); } }, } @@ -566,11 +554,10 @@ pub const TensorInfo = struct { const mlir_ctx = compiler.mlir_ctx; const mesh_attr = try compiler.tensorShardingAttr(arena, info.shape); - const name = switch (compiler.partitioner) { - .gspmd => "mhlo.sharding", - .shardy => "sdy.sharding", - }; - attrs.appendAssumeCapacity(.named(mlir_ctx, name, mesh_attr)); + attrs.appendAssumeCapacity(switch (compiler.partitioner) { + .gspmd => .named(mlir_ctx, "mhlo.sharding", mesh_attr), + .shardy => shardy.shardingNamedAttr(mlir_ctx, mesh_attr.isA(shardy.TensorShardingAttr).?), + }); const memory = scope.id_to_memory.get(info.id) orelse .device; if (memory != .device) { diff --git a/zml/Sharding.zig b/zml/Sharding.zig index f5ca994b..2637f209 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -194,12 +194,12 @@ pub fn sdyPerValueShardingAttr( ctx: *mlir.Context, shapes: []const Shape, mesh: *const Mesh, -) error{OutOfMemory}!*const mlir.Attribute { +) error{OutOfMemory}!*const shardy.TensorShardingPerValueAttr { const meshe_attrs = try allocator.alloc(*const shardy.TensorShardingAttr, shapes.len); for (meshe_attrs, shapes) |*attr, shape| { attr.* = try mesh.sdyShardingAttrForShape(allocator, ctx, shape); } - return shardy.TensorShardingPerValueAttr.get(ctx, meshe_attrs).attribute(); + return .get(ctx, meshe_attrs); } pub fn sdyManualAxesAttr( @@ -208,7 +208,7 @@ pub fn sdyManualAxesAttr( in_shapes: []const Shape, out_shapes: []const Shape, mesh: *const Mesh, -) error{OutOfMemory}!*const mlir.Attribute { +) error{OutOfMemory}!*const shardy.ManualAxesAttr { var axis_names = std.ArrayList([]const u8).empty; defer axis_names.deinit(allocator); @@ -251,7 +251,7 @@ pub fn sdyManualAxesAttr( axes[i] = mlir.StringAttribute.init(ctx, axis_name); } - return shardy.ManualAxesAttr.get(ctx, axes).attribute(); + return .get(ctx, axes); } pub const Partitioning = packed struct { @@ -1637,7 +1637,7 @@ pub const Mesh = struct { return @divExact(self.numDevices(), self.numPartitions()); } - pub fn sdyMeshAttr(self: *const Mesh, ctx: *mlir.Context) *const mlir.Attribute { + pub fn sdyMeshAttr(self: *const Mesh, ctx: *mlir.Context) *const shardy.MeshAttr { var axes: stdx.BoundedArray(*const shardy.MeshAxisAttr, MAX_MESH_RANK + 1) = .empty; var tile_count: usize = 1; for (self.logical.axes.slice(), 0..) |axis, i| { @@ -1661,7 +1661,7 @@ pub const Mesh = struct { ids[tile_index * replicas + replica_indices[tile_index]] = @intCast(partition_index); replica_indices[tile_index] += 1; } - return shardy.meshAttr(ctx, .{ .axes = axes.constSlice(), .device_ids = ids[0..self.grid.device_count] }); + return .get(ctx, .{ .axes = axes.constSlice(), .device_ids = ids[0..self.grid.device_count] }); } fn sdyReplicaAxis(self: *const Mesh) []const u8 { diff --git a/zml/kernels/moe/cute/persistent_mxfp4.zig b/zml/kernels/moe/cute/persistent_mxfp4.zig index c4bbe5cc..d4f249b8 100644 --- a/zml/kernels/moe/cute/persistent_mxfp4.zig +++ b/zml/kernels/moe/cute/persistent_mxfp4.zig @@ -12,6 +12,7 @@ //! drain the accumulator from tensor memory into one of the two epilogues. const std = @import("std"); const mlir = @import("mlir"); +const nvvm = @import("mlir/dialects/nvvm"); const zml = @import("../../../zml.zig"); const cute = zml.kernel.cute; @@ -504,13 +505,13 @@ fn buildTiledProgram(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: .tiles_m = typed(i32_type), }, }); - b.setFunctionAttribute("nvvm.minctasm", .int(b.ctx, .i32, 1)); - b.setFunctionAttribute("smem.partition_num", .int(b.ctx, .i32, 2)); - b.setFunctionAttribute("smem.hint_smem_base_uniform", .unit(b.ctx)); - b.setFunctionAttribute("cu_attrs", .dict(b.ctx, &.{ + b.setFunctionAttribute(nvvm.minctasm(b.ctx, 1)); + b.setFunctionAttribute(.named(b.ctx, "smem.partition_num", .int(b.ctx, .i32, 2))); + b.setFunctionAttribute(.named(b.ctx, "smem.hint_smem_base_uniform", .unit(b.ctx))); + b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{ .named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin()), .named(b.ctx, "non_portable_cluster_size_allowed", .int(b.ctx, .i32, 1)), - })); + }))); // Prefetch every tensor-map descriptor before the specialized warps begin // the persistent pipeline. This is the work PDL buys us: it runs while the diff --git a/zml/kernels/paged_attention/cute/sm10x/attention.zig b/zml/kernels/paged_attention/cute/sm10x/attention.zig index 3d5fd107..89761bdd 100644 --- a/zml/kernels/paged_attention/cute/sm10x/attention.zig +++ b/zml/kernels/paged_attention/cute/sm10x/attention.zig @@ -202,7 +202,7 @@ fn attention(b: *B, c: Config) cute.FinishError!void { .out = .{ .ptr = if (c.splits > 1) cute.DType.f32 else c.dtype }, }); std.debug.assert(c.sharedBytes() <= 227 * 1024); - b.setFunctionAttribute("cu_attrs", .dict(b.ctx, &.{.named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin())})); + b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{.named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin())}))); const seq = b.blockIdx().x.div(c.queryTiles()); const first_row = b.blockIdx().x.rem(c.queryTiles()).mul(c.rows()); const starts = b.ptrToInt(a.starts, .i64); diff --git a/zml/kernels/paged_attention/cute/sm10x/sm103.zig b/zml/kernels/paged_attention/cute/sm10x/sm103.zig index bc3171b5..fe96916d 100644 --- a/zml/kernels/paged_attention/cute/sm10x/sm103.zig +++ b/zml/kernels/paged_attention/cute/sm10x/sm103.zig @@ -28,12 +28,7 @@ pub fn loadScores(b: *cute.Builder, address: cute.Value) Scores { const result = b.emit(llvm.inline_asm(b.ctx, asm_parts[0], &.{address.inner}, .{ .result_type = llvm.structType(b.ctx, &@as([65]*const mlir.Type, @splat(.float(b.ctx, .f32)))), .constraints = asm_parts[1], .has_side_effects = true }, b.loc())); var scores: Scores = undefined; for (0..65) |i| { - const value = b.emit(mlir.Operation.make(b.ctx, "llvm.extractvalue", .{ - .operands = .{ .flat = &.{result.inner} }, - .results = .{ .flat = &.{cute.DType.f32.toMlir(b.ctx)} }, - .attributes = &.{.named(b.ctx, "position", .denseArray(b.ctx, .i64, &.{@intCast(i)}))}, - .location = b.loc(), - })); + const value = b.emit(llvm.extractvalue(b.ctx, result.inner, cute.DType.f32.toMlir(b.ctx), &.{@intCast(i)}, b.loc())); if (i < 64) scores.values[i] = value else scores.maximum = value; } b.fenceTmemLoad(); diff --git a/zml/kernels/paged_attention/cute/sm120/attention.zig b/zml/kernels/paged_attention/cute/sm120/attention.zig index ea364488..e9675252 100644 --- a/zml/kernels/paged_attention/cute/sm120/attention.zig +++ b/zml/kernels/paged_attention/cute/sm120/attention.zig @@ -74,12 +74,7 @@ fn swizzle(row: V, col: V, width: i64) V { fn unpackStruct(b: *B, comptime n: usize, value: V, dtype: cute.DType) [n]V { var result: [n]V = undefined; - for (&result, 0..) |*r, i| r.* = b.emit(mlir.Operation.make(b.ctx, "llvm.extractvalue", .{ - .operands = .{ .flat = &.{value.inner} }, - .results = .{ .flat = &.{dtype.toMlir(b.ctx)} }, - .attributes = &.{.named(b.ctx, "position", .denseArray(b.ctx, .i64, &.{@intCast(i)}))}, - .location = b.loc(), - })); + for (&result, 0..) |*r, i| r.* = b.emit(llvm.extractvalue(b.ctx, value.inner, dtype.toMlir(b.ctx), &.{@intCast(i)}, b.loc())); return result; } @@ -225,7 +220,7 @@ fn runTile(b: *B, c: Config, comptime d: usize, comptime w: usize, comptime n: u .sink = .{ .ptr = cute.DType.f32 }, .out = .{ .ptr = if (c.splits > 1) cute.DType.f32 else dt }, }); - b.setFunctionAttribute("cu_attrs", .dict(b.ctx, &.{.named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin())})); + b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{.named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin())}))); const seq = b.blockIdx().x.div(c.queryTiles()); const first_row = b.blockIdx().x.rem(c.queryTiles()).mul(c.rows()); const starts = b.ptrToInt(a.starts, .i64); diff --git a/zml/ops.zig b/zml/ops.zig index d1f78276..6622ac19 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -204,10 +204,7 @@ fn AllReduceReturnType(comptime InputsT: type) type { pub fn partitionId() Tensor { const ctx = Compiler.current(); - const op = mlir.Operation.make(ctx.mlir_ctx, "stablehlo.partition_id", .{ - .results = .{ .flat = &.{mlirx.Type.rankedTensor(ctx.mlir_ctx, Shape.scalar(.u32))} }, - .location = ctx.location, - }).appendTo(ctx.currentScope().block); + const op = stablehlo.partition_id(ctx.mlir_ctx, ctx.location).appendTo(ctx.currentScope().block); return Tensor._result(.init(.{}, .u32), op.result(0)); } @@ -268,16 +265,7 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func var init_values: [inputs.len]*const mlir.Value = undefined; inline for (0..inits.len) |i| init_values[i] = inits[i].value(); - const reduce_op = mlir.Operation.make(mlir_ctx, "stablehlo.reduce", .{ - .operands = .{ .variadic = &.{ &input_values, &init_values } }, - .result_type_inference = true, - .blocks = &.{reduce_block}, - .attributes = &.{ - .named(mlir_ctx, "dimensions", .denseArray(mlir_ctx, .i64, axes_)), - }, - .verify = true, - .location = compiler.location, - }).appendTo(caller_scope.block); + const reduce_op = stablehlo.reduce(mlir_ctx, &input_values, &init_values, reduce_block, axes_, compiler.location).appendTo(caller_scope.block); // `stablehlo.reduce` drops axes. We want to avoid that to propagate tags. // So we need to broadcast the output of `stablehlo.reduce` to the input shapes. @@ -376,24 +364,12 @@ pub fn reduceWindow(N: comptime_int, inputs: [N]Tensor, inits: [N]Tensor, opts: var init_values: [inputs.len]*const mlir.Value = undefined; inline for (0..inits.len) |i| init_values[i] = inits[i].value(); - const reduce_op = mlir.Operation.make(mlir_ctx, "stablehlo.reduce_window", .{ - .operands = .{ .variadic = &.{ &input_values, &init_values } }, - .result_type_inference = true, - .blocks = &.{reduce_block}, - .attributes = &.{ - .named(mlir_ctx, "window_dimensions", .denseArray(mlir_ctx, .i64, opts.window_dimensions)), - .named(mlir_ctx, "window_strides", .denseArray(mlir_ctx, .i64, opts.window_strides)), - .named(mlir_ctx, "base_dilations", .denseArray(mlir_ctx, .i64, opts.base_dilations)), - .named(mlir_ctx, "window_dilations", .denseArray(mlir_ctx, .i64, opts.window_dilations)), - // Cast the [][2]i64 to []i64 (safe) - .named(mlir_ctx, "padding", .denseElements( - .rankedTensor(&.{ @intCast(opts.padding.len), 2 }, .int(mlir_ctx, .i64)), - @as([]const i64, @ptrCast(opts.padding)), - )), - }, - .verify = true, - .location = compiler.location, - }).appendTo(compiler.currentScope().block); + const reduce_op = stablehlo.reduce_window(mlir_ctx, &input_values, &init_values, reduce_block, opts.window_dimensions, .{ + .window_strides = opts.window_strides, + .base_dilations = opts.base_dilations, + .window_dilations = opts.window_dilations, + .padding = opts.padding, + }, compiler.location).appendTo(compiler.currentScope().block); inline for (0..result.len) |i| { result[i] = Tensor.fromMlirValue(reduce_op.result(i)).withTags(inputs[i].shape()); @@ -448,17 +424,10 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func_: anytype, context: anyty input_values[i] = inputs[i].value(); } - const sort_op = mlir.Operation.make(mlir_ctx, "stablehlo.sort", .{ - .operands = .{ .variadic = &.{&input_values} }, - .result_type_inference = true, - .blocks = &.{sort_block}, - .attributes = &.{ - .named(mlir_ctx, "dimension", .int(mlir_ctx, .i64, axis_)), - .named(mlir_ctx, "is_stable", .boolean(mlir_ctx, is_stable)), - }, - .verify = true, - .location = compiler.location, - }).appendTo(compiler.currentScope().block); + const sort_op = stablehlo.sort(mlir_ctx, &input_values, sort_block, .{ + .dimension = axis_, + .is_stable = is_stable, + }, compiler.location).appendTo(compiler.currentScope().block); var result: [inputs.len]Tensor = undefined; inline for (0..inputs.len) |i| { @@ -737,14 +706,7 @@ pub fn @"if"( // TODO check that false branch values matches right branch _ = false_branch; - const op = mlir.Operation.make(mlir_ctx, "stablehlo.if", .{ - .operands = .{ .flat = &.{pred.asScalar().value()} }, - .result_type_inference = true, - .blocks = &.{ true_branch_block, false_branch_block }, - // We can't verify right away, cause the weights captured by the if haven't been added yet. - .verify = false, - .location = loc, - }); + const op = stablehlo.if_(mlir_ctx, pred.asScalar().value(), true_branch_block, false_branch_block, loc); _ = op.appendTo(Compiler.current().currentScope().block); return fromMlirOperationWithTags(op, true_branch); @@ -825,13 +787,7 @@ pub fn if2( break :b block; }; - const op = mlir.Operation.make(mlir_ctx, "stablehlo.if", .{ - .operands = .{ .flat = &.{pred.asScalar().value()} }, - .result_type_inference = true, - .blocks = &.{ true_branch_block, false_branch_block }, - .location = loc, - .verify = false, - }); + const op = stablehlo.if_(mlir_ctx, pred.asScalar().value(), true_branch_block, false_branch_block, loc); _ = op.appendTo(compiler.currentScope().block); return fromMlirOperationWithTags(op, on_true); diff --git a/zml/ops/manual_computation.zig b/zml/ops/manual_computation.zig index 5405c6c2..c9596f98 100644 --- a/zml/ops/manual_computation.zig +++ b/zml/ops/manual_computation.zig @@ -176,29 +176,23 @@ noinline fn manualComputationInternal( const op: *const mlir.Operation = switch (compiler.partitioner) { .shardy => op: { // Shardy creates a sdy.manual_computation block. - _ = mlir.Operation.make(mlir_ctx, "sdy.return", .{ - .operands = .{ .flat = local_output_values }, - .verify = false, - .location = compiler.location, - }).appendTo(manual_block); + _ = shardy.return_(mlir_ctx, local_output_values, compiler.location).appendTo(manual_block); const out_mesh_attrs = try arena.alloc(*const shardy.TensorShardingAttr, global_output_shapes.len); for (out_mesh_attrs, global_output_shapes) |*attr, shape| { attr.* = try mesh.sdyShardingAttrForShape(arena, mlir_ctx, shape); } - break :op mlir.Operation.make(mlir_ctx, "sdy.manual_computation", .{ - .operands = .{ .flat = input_values }, - .results = .{ .flat = global_output_types }, - .blocks = &.{manual_block}, - .attributes = &.{ - .named(mlir_ctx, "in_shardings", try Sharding.sdyPerValueShardingAttr(arena, mlir_ctx, global_input_shapes, mesh)), - .named(mlir_ctx, "out_shardings", shardy.shardingsPerValueAttr(mlir_ctx, out_mesh_attrs)), - .named(mlir_ctx, "manual_axes", try Sharding.sdyManualAxesAttr(arena, mlir_ctx, global_input_shapes, global_output_shapes, mesh)), - }, - .verify = true, - .location = compiler.location, - }).appendTo(global_scope.block); + break :op shardy.manual_computation( + mlir_ctx, + input_values, + global_output_types, + manual_block, + try Sharding.sdyPerValueShardingAttr(arena, mlir_ctx, global_input_shapes, mesh), + .get(mlir_ctx, out_mesh_attrs), + try Sharding.sdyManualAxesAttr(arena, mlir_ctx, global_input_shapes, global_output_shapes, mesh), + compiler.location, + ).appendTo(global_scope.block); }, .gspmd => op: { // Gspmd calls SPMDShardToFullShape and wrap into an optimizationBarrier diff --git a/zml/tensor.zig b/zml/tensor.zig index 5e2e073f..8c148867 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -2,6 +2,7 @@ const std = @import("std"); const builtin = @import("builtin"); const mlir = @import("mlir"); +const shardy = @import("mlir/dialects/shardy"); const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); @@ -194,14 +195,12 @@ pub const Tensor = struct { const op_result = switch (ctx.partitioner) { .shardy => blk: { - const op = mlir.Operation.make(ctx.mlir_ctx, "sdy.sharding_constraint", .{ - .operands = .{ .flat = &.{self.value()} }, - .results = .{ .flat = &.{self.value().type_()} }, - .attributes = &.{ - .named(ctx.mlir_ctx, "sharding", attr), - }, - .location = ctx.location, - }).appendTo(currentBlock()); + const op = shardy.sharding_constraint( + ctx.mlir_ctx, + self.value(), + attr.isA(shardy.TensorShardingAttr).?, + ctx.location, + ).appendTo(currentBlock()); break :blk op.result(0); }, .gspmd => blk: { From 47e692c3005472badfbf190ea57c26f017ec21cb Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Thu, 29 Oct 2026 11:15:39 +0100 Subject: [PATCH 828/881] platforms/oneapi: bump to IGC 2.41.15 (#996) Bump to resolve a compiler bug encountered on LFM3. LFM3 broadcasts one sigmoid gate per attention head. This generated code of sigmoid introduced a BF16 conversion and conditional select. The compiler then lower it to a scalar computation which had a register-aligment bug. The fix introduced in 2.41.11 enforces alignment. --- platforms/oneapi/oneapi.bzl | 8 ++++---- platforms/oneapi/packages.lock.json | 18 +++++++++--------- platforms/oneapi/packages.yaml | 8 ++++++-- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index 55075f0a..6016e389 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -68,11 +68,11 @@ filegroup( "libigc2": """ filegroup( name = "libigc_so_2", - srcs = ["lib/x86_64-linux-gnu/libigc.so.2.40.13+0"], + srcs = ["lib/x86_64-linux-gnu/libigc.so.2.41.15+0"], ) filegroup( name = "libiga64_so_2", - srcs = ["lib/x86_64-linux-gnu/libiga64.so.2.40.13+0"], + srcs = ["lib/x86_64-linux-gnu/libiga64.so.2.41.15+0"], ) filegroup( name = "libigc2", @@ -84,11 +84,11 @@ filegroup( "libigdfcl2": """ filegroup( name = "libigdfcl_so_2", - srcs = ["lib/x86_64-linux-gnu/libigdfcl.so.2.40.13+0"], + srcs = ["lib/x86_64-linux-gnu/libigdfcl.so.2.41.15+0"], ) filegroup( name = "libopencl_clang", - srcs = ["lib/x86_64-linux-gnu/libopencl-clang.so.17"], + srcs = ["lib/x86_64-linux-gnu/libopencl-clang.so.22.1"], ) filegroup( name = "libigdfcl2", diff --git a/platforms/oneapi/packages.lock.json b/platforms/oneapi/packages.lock.json index d8747a09..a3e69ad4 100644 --- a/platforms/oneapi/packages.lock.json +++ b/platforms/oneapi/packages.lock.json @@ -212,24 +212,24 @@ { "arch": "amd64", "dependencies": [], - "key": "libigc2_2.40.13-p-ds1-1_24.04_amd64", + "key": "libigc2_2.41.15-1_24.04_amd64", "name": "libigc2", - "sha256": "c57e4ca8c780dc85a309e35b4d3249fc866251fea7bf3ebe68a5ab5ef5f71362", + "sha256": "c3e5f181eecd40dc8b656c3ff5b7e3d4f690e93181255726516416e61ade1d1f", "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z/pool/main/i/intel-graphics-compiler/libigc2_2.40.13+ds1-1~24.04_amd64.deb" + "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-graphics-compiler/libigc2_2.41.15-1~24.04_amd64.deb" ], - "version": "2.40.13+ds1-1~24.04" + "version": "2.41.15-1~24.04" }, { "arch": "amd64", "dependencies": [], - "key": "libigdfcl2_2.40.13-p-ds1-1_24.04_amd64", + "key": "libigdfcl2_2.41.15-1_24.04_amd64", "name": "libigdfcl2", - "sha256": "e71f41b4dd54c9cda7024680528034204cd911a1c2f644ec2bc3c06e968f5afe", + "sha256": "8a2058fd9f5e7d0a4b05001ff861780d2f72529135fa70fb7fbba0f0ff8cd74f", "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z/pool/main/i/intel-graphics-compiler/libigdfcl2_2.40.13+ds1-1~24.04_amd64.deb" + "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-graphics-compiler/libigdfcl2_2.41.15-1~24.04_amd64.deb" ], - "version": "2.40.13+ds1-1~24.04" + "version": "2.41.15-1~24.04" }, { "arch": "amd64", @@ -266,4 +266,4 @@ } ], "version": 1 -} \ No newline at end of file +} diff --git a/platforms/oneapi/packages.yaml b/platforms/oneapi/packages.yaml index 78f8b49e..272bc02c 100644 --- a/platforms/oneapi/packages.yaml +++ b/platforms/oneapi/packages.yaml @@ -15,6 +15,10 @@ sources: - channel: noble main url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z + # IGC 2.41.15 is newer than the graphics snapshot above. + - channel: noble main + url: https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu + archs: - "amd64" @@ -40,8 +44,8 @@ packages: - "libnl-route-3-200" - "libnuma1" - "libigdgmm12" - - "libigc2" - - "libigdfcl2" + - "libigc2 (= 2.41.15-1~24.04)" + - "libigdfcl2 (= 2.41.15-1~24.04)" - "libze-intel-gpu1" - "libze1" - "zlib1g" From b725ff2481dfd9ee108e17826409dc6bca9320bb Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Thu, 29 Oct 2026 11:16:03 +0100 Subject: [PATCH 829/881] zml/Compiler: add XLA flags to improve generated kernels for ROCm (#998) Enable two flags for ROCm: - multi output fusion: allows to search for fusion opportunity generating different output. - latency hiding scheduler: overlap async communication with computation. --- zml/Compiler.zig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 49277368..5de801e3 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -967,6 +967,10 @@ fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.Load // Enable user buffers to assign color 1 and receive the reclaim exemption: // https://github.com/openxla/xla/pull/46029 try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_nccl_user_buffers", true, upb_arena); + // Enables fusion to produce output for multiple separate output. + try setXlaOverrideFlag(overrides_map, "xla_gpu_unsupported_enable_triton_multi_output_fusion", true, upb_arena); + // Rearranges operations so that communication between GPUs happens in the background while the compute units are busy doing computation. + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_latency_hiding_scheduler", true, upb_arena); }, .metal => { try setXlaOverrideFlag(overrides_map, "xla_gpu_metal_fast_math", false, upb_arena); From 70f1bd4af4ee534fe56b3135e85280cce17f01a4 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Thu, 29 Oct 2026 12:52:20 +0100 Subject: [PATCH 830/881] dsl/metal: add the Metal DSL (#989) Adds a Zig DSL for Metal kernels over the AIR dialect, and moves the Metal paged attention kernels onto it. - `mlir/dialects/air`: a `thread` address space. Each op has an `llvmBuilder` that calls its `air.*` builtin with Apple's attributes, and the dialect's translation interface builds the `!air.kernel` metadata, so lowering to LLVM IR is MLIR's own translation. The verifier rejects what Apple's compiler can't take: bool, scalable and n-D vectors, buffers of non-Metal scalars, scalar f32 converts, and `air.kernel` on a declaration. - `dsl/metal`: an MSL-shaped builder. It emits `air` plus the `arith`/`cf`/`vector`/`llvm` IR that clang produces for MSL, and covers MSL's math, SIMD-group, barrier and `simdgroup_matrix` functions. - `zml.kernel.metal` / `ops.metal`: wrap a kernel into a `__gpu$xla.gpu.air` custom call. XLA lowers it, compiles it to a metallib and launches it. --- dsl/metal/BUILD.bazel | 25 + dsl/metal/README.md | 150 ++ dsl/metal/builder.zig | 2320 +++++++++++++++++++++++++++ dsl/metal/types.zig | 291 ++++ mlir/dialects/air/AIRAttributes.cpp | 13 +- mlir/dialects/air/AIRDialect.cpp | 2 + mlir/dialects/air/AIRDialect.td | 5 - mlir/dialects/air/AIREnums.td | 3 +- mlir/dialects/air/AIROps.cpp | 4 + mlir/dialects/air/AIROps.td | 20 +- mlir/dialects/air/README.md | 5 +- mlir/dialects/air/air.zig | 2 +- mlir/dialects/air/test.zig | 25 + mlir/dialects/arith/arith.zig | 6 + mlir/dialects/arith/test.zig | 9 + mlir/dialects/llvm/llvm.zig | 19 + mlir/dialects/llvm/ops.zig | 139 +- mlir/dialects/llvm/test.zig | 70 + platforms/cpu/cpu.bzl | 12 +- platforms/cuda/cuda.bzl | 8 +- platforms/metal/metal.bzl | 4 +- platforms/rocm/rocm.bzl | 4 +- zml/BUILD.bazel | 1 + zml/Compiler.zig | 2 +- zml/kernel.zig | 249 +++ zml/ops.zig | 63 + 26 files changed, 3405 insertions(+), 46 deletions(-) create mode 100644 dsl/metal/BUILD.bazel create mode 100644 dsl/metal/README.md create mode 100644 dsl/metal/builder.zig create mode 100644 dsl/metal/types.zig diff --git a/dsl/metal/BUILD.bazel b/dsl/metal/BUILD.bazel new file mode 100644 index 00000000..d68355c0 --- /dev/null +++ b/dsl/metal/BUILD.bazel @@ -0,0 +1,25 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "builder", + srcs = ["types.zig"], + import_name = "dsl/metal/builder", + main = "builder.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//dsl/common:builder", + "//mlir", + "//mlir/dialects/air", + "//mlir/dialects/arith", + "//mlir/dialects/cf", + "//mlir/dialects/func", + "//mlir/dialects/llvm", + "//mlir/dialects/vector", + ], +) + +zig_test( + name = "test", + deps = [":builder"], +) diff --git a/dsl/metal/README.md b/dsl/metal/README.md new file mode 100644 index 00000000..1cbc40ce --- /dev/null +++ b/dsl/metal/README.md @@ -0,0 +1,150 @@ +# ZML Metal DSL + +A Zig builder for Metal kernels that reads like MSL. It emits the `air` +dialect (`mlir/dialects/air`) and the parts of `arith`, `cf`, `vector` and +`llvm` that MSL code compiles to: values, locals in memory, blocks and +branches. `zml.kernel.metal.Kernel` wraps a kernel into +a `__gpu$xla.gpu.air` custom call, and XLA does the rest: it lowers the module to +Apple's AIR, runs the Metal frontend's optimization pipeline +(`air-opt -O2 -disable-loop-unrolling`) and builds a metallib. Apple's +backend compiles that metallib the way it compiles an MSL kernel. + +## Skeleton of a kernel + +```zig +const metal = @import("zml").kernel.metal; + +pub const Axpy = struct { + pub const Cfg = struct {}; + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "axpy", + .inputs = &.{ "a", "b" }, + .outputs = &.{"out"}, + .constants = &.{ "scale", "n" }, + .run = run, + }); + + fn run(b: *metal.Builder, _: Cfg) metal.FinishError!void { + const a = try b.declareArgs(.{ + .a = .{ .buffer = .{ .type = .float, .const_ = true } }, + .b = .{ .buffer = .{ .type = .float, .const_ = true } }, + .out = .{ .buffer = .{ .type = .float } }, + .scale = .{ .constant = .float }, + .n = .{ .constant = .uint }, + .tid = .{ .thread_position_in_grid = .uint }, + }); + b.returnIf(a.tid.ge(a.n)); + a.out.store(a.tid, b.fma(a.a.load(a.tid), a.scale, a.b.load(a.tid))); + } +}; + +// In a model: +const out = Axpy.Kernel.call(.{ .a = x, .b = y }, .{ .out = x.shape() }, .{ + .cfg = .{}, + .grid = .{ 4, 1, 1 }, + .threadgroup = .{ 256, 1, 1 }, + .constants = .{ .scale = .of(@as(f32, 0.5)), .n = .of(@as(u32, 1000)) }, +}).out; +``` + +`declareArgs` declares the kernel parameters in order, which is the `[[buffer(n)]]` +order. `inputs` and `outputs` bind buffers to tensors by name. An output named +like an input updates that input's buffer in place. `constants` bind +`constant T &` arguments by value. `Constant.of` takes a scalar, or a host +`extern struct` laid out like the MSL struct. `CallOpts.threadgroup_memory` +sizes `[[threadgroup(0)]]`. + +Function constants have no runtime form. A kernel takes them through its `Cfg`, +so each configuration emits its own specialized kernel. + +## Arguments + +| MSL | `ArgSpec` | +|---|---| +| `device T *p`, `const device T *p` | `.buffer = .{ .type = T, .const_ = true }` | +| `constant T &x` | `.constant = T`, read wherever it is used, as C++ reads a reference | +| `constant S &s` | `.constant_struct = S`, read with `s.field("name")` | +| `threadgroup T *p [[threadgroup(0)]]` | `.threadgroup = T` | +| `uint3 tid [[thread_position_in_grid]]` and the other builtins | `.thread_position_in_grid = .uint3`, ... | + +## Values + +A `Value` is a scalar or vector (`Type`, spelled as in MSL: `.float4`, +`.ushort`), a pointer, a private array, a `simdgroup_matrix` or a constant +struct. Operations follow C++: integer promotion and the usual arithmetic +conversions pick the result type, signed arithmetic is `nsw`, and float math +carries the kernel's fast-math flags. A Zig number is the MSL literal it +spells, an `int` (a `long` when it does not fit) or a `float`, and a runtime +Zig integer keeps its width and signedness; the usual conversions follow, so +`h.mul(0.5)` on a `half` is a `float` multiply, as `h * 0.5` is in MSL. A +scalar next to a vector takes its element type. Stores, assignments and the +functions of one type (`max`, `min`, `clamp`, `fma`) convert a number to their +type. `0.5h` is `b.constant(.half, 0.5)`. + +| MSL | DSL | +|---|---| +| `a + b`, `a * b`, `a < b`, `a & b` | `a.add(b)`, `a.mul(b)`, `a.lt(b)`, `a.bitAnd(b)` | +| `(float) x`, `float4(v)` | `x.to(.float)`, `v.to(.float4)` | +| `as_type(x)` | `x.asType(.uint)` | +| `v.x`, `v[i]`, `v[i] = e`, `v.y = e` | `v.x()`, `v.at(i)`, `v = v.insert(i, e)`, `v = v.insert(1, e)`: a subscript takes the index's type (a literal is an `int`) | +| `p[i]`, `p[i] = v`, `p + i` | `p.load(i)`, `p.store(i, v)`, `p.add(i)` | +| `(threadgroup float4 *) p` | `p.to(.float4)` | +| `T x;`, `T x = e;`, `x = e` | `const x = b.local(.T, null)`, `b.local(.T, e)`, `x.assign(e)` | +| `device T *p = e;` that the code moves | `const p = b.localPointer(e)`, `p.assign(p.add(i))` | +| `float s[8];`, `float s[8] = {};` | `const s = b.array(.float, 8)`, `b.zeroedArray(.float, 8)`, then `s.load(i)` and `s.store(i, v)` | +| `&x` of a local | `x.address()` | +| `float4(a, b, c, d)`, `1.0f` | `b.vec(.float4, .{ a, b, c, d })`, `b.constant(.float, 1)` | +| `c ? x : y` | `b.ternary(c, x, y)`, a select of evaluated operands; `var t = b.openIfElse(c, .{.T})`, `x`, `t.yieldThen(.{x})`, `y`, `t.yieldElse(.{y})`, `t.results[0]` | +| `exp(x)`, `fast::exp(x)`, `precise::tanh(x)` | `b.exp(x)`, `b.fast().exp(x)`, `b.precise().tanh(x)` | +| `fma`, `dot`, `max`, `clamp` | `b.fma`, `b.dot`, `b.max`, `b.clamp` | +| `simd_sum(x)`, `simd_shuffle_xor(x, m)` | `b.simdSum(x)`, `b.simdShuffleXor(x, m)` | +| `threadgroup_barrier(mem_flags::mem_threadgroup)` | `b.threadgroupBarrier(.mem_threadgroup)` | +| `simdgroup_load(m, p, ld, origin, transpose)`, `simdgroup_store(m, p, ld)` | `const m = b.simdgroupLoad(p, ld, .{ .origin = ..., .transpose = true })`, `b.simdgroupStore(m, p, ld, .{})` | +| `simdgroup_multiply_accumulate(d, a, b, c)` | `const d = b.simdgroupMultiplyAccumulate(a, b, c)` | +| `m.thread_elements()` read, assigned | `m.threadElements()`; `const m = b.matrix(.T, elements)` | + +## Control flow + +| MSL | DSL | +|---|---| +| `for (T i = lower; i < upper; i += s)`, `++i` | `var l = b.openFor(lower, upper, s)`, `l.iv`, `l.close()`; a step of 1 is `++i` | +| `#pragma unroll`, `#pragma clang loop unroll(full)` | `b.openForOpts(..., .{ .unroll = .enable })`, `.{ .unroll = .full }` | +| `while (c) { ... }`, `for (init; c; inc)`, `for (;;)` | `var l = b.openLoop(.{})`, `l.body(c)`, the body, `l.next()`, the increment, `l.close()` | +| `break;` | `l.breakLoop()` | +| `if (c) { ... }` | `var s = b.openIf(c)`, `s.close()` | +| `if (c) { ... } else { ... }` | `var s = b.openIfElse(c, .{})`, `s.elseBranch()`, `s.close()` | +| `if (a \|\| (b && !x))` | `var c = b.condition()`, `c.or_(a)`, `c.and_(b)`, `var s = c.openIfNot(x)` | +| `x = a && b && c` | `var l = b.openAnd()`, `l.term(a)`, `l.term(b)`, `l.close(c)` | +| `return;` | `b.return_()`, or `b.returnIf(c)` | + +`condition`, `openAnd`, `openOr` and a ternary `openIfElse` skip a term or an +operand, so use them only when one must not run, such as a load past the end +of a buffer. Combine comparisons with `bitAnd` and `bitOr`, and pick between +values that are safe to compute with `b.ternary`. + +Put each `if` or loop body in a Zig `{ ... }` block right after the opener, +one block per region, each ending with the call that ends the region: +`elseBranch()` or `yieldThen`, then `close()` or `yieldElse` for an `if`; +`body(c)`, `next()`, then `close()` for `openLoop`. The scope variable stays +outside, so its `results` stay readable after the blocks. + +A value is evaluated where it is written; a Zig `const` holds it, and an MSL +function is a Zig function that emits its body. A value a loop or branch +changes is a local, which lives in memory until the pipeline promotes it. +The terms of `&&` and `||` run only when the ones before leave the result +open, and an `openIfElse(c, .{T})` operand only when `c` selects it, each in +its own block. Loops follow C++: every +loop is `mustprogress`, and its condition and increment follow C++'s +conversions, so a `short` loop variable compares as an `int` and `++i` steps +it as a `short`. + +A `simdgroup_matrix` is a value of its 64 elements, of which a lane holds the +first two; the load, store and multiply-accumulate call the +`air.simdgroup_matrix_8x8_*` builtins. + +## Checking a port + +Fast math lets Apple's backend reassociate and contract float math, so a port +computes what its MSL kernel computes up to rounding, not always to the bit. +Compare its outputs with the MSL kernel's on the same inputs, and time both +with the same grid, threadgroup size and threadgroup memory. diff --git a/dsl/metal/builder.zig b/dsl/metal/builder.zig new file mode 100644 index 00000000..dea3d48a --- /dev/null +++ b/dsl/metal/builder.zig @@ -0,0 +1,2320 @@ +const std = @import("std"); + +const air = @import("mlir/dialects/air"); +const arith = @import("mlir/dialects/arith"); +const cf = @import("mlir/dialects/cf"); +const common = @import("dsl/common/builder"); +const func = @import("mlir/dialects/func"); +const llvm = @import("mlir/dialects/llvm"); +const mlir = @import("mlir"); +const vector = @import("mlir/dialects/vector"); + +pub const types = @import("types.zig"); + +pub const Scalar = types.Scalar; +pub const Type = types.Type; +pub const Struct = types.Struct; +pub const StructField = types.StructField; +pub const Builtin = air.Builtin; + +pub fn registerDialects(registry: *mlir.DialectRegistry) void { + inline for (.{ air, arith, cf, func, llvm, vector }) |dialect| dialect.registerDialect(registry); +} + +pub const FinishError = error{InvalidMlir} || std.mem.Allocator.Error || std.Io.Writer.Error; + +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(Builder); + std.testing.refAllDecls(Value); +} + +pub const AddressSpace = air.AddressSpace; + +/// `mem_flags`: `.mem_threadgroup`, or `.{ .device = true, .threadgroup = true }` +/// for `mem_device | mem_threadgroup`. +pub const MemFlags = packed struct(u32) { + device: bool = false, + threadgroup: bool = false, + texture: bool = false, + _padding: u29 = 0, + + pub const mem_none: MemFlags = .{}; + pub const mem_device: MemFlags = .{ .device = true }; + pub const mem_threadgroup: MemFlags = .{ .threadgroup = true }; + pub const mem_texture: MemFlags = .{ .texture = true }; +}; + +/// A kernel argument, as `Builder.declareArgs` takes it. Device and constant +/// arguments take the next `[[buffer(n)]]`, threadgroup arguments the next +/// `[[threadgroup(n)]]`, in declaration order. +pub const ArgSpec = union(enum) { + /// `device T *p`, or `const device T *p` with `.const_`. + buffer: Buffer, + /// `constant T &x`; the argument is its value. + constant: Type, + /// `constant S &s`; `Value.field` reads a field. + constant_struct: Struct, + /// `threadgroup T *p`. + threadgroup: Type, + thread_position_in_grid: Type, + threadgroup_position_in_grid: Type, + thread_position_in_threadgroup: Type, + threads_per_threadgroup: Type, + threadgroups_per_grid: Type, + threads_per_grid: Type, + thread_index_in_simdgroup: Type, + simdgroup_index_in_threadgroup: Type, + threads_per_simdgroup: Type, + simdgroups_per_threadgroup: Type, + thread_index_in_threadgroup: Type, + + pub const Buffer = struct { + type: Type, + const_: bool = false, + /// `__restrict__`. + restrict: bool = false, + }; +}; + +/// `raw` as an `ArgSpec`: an `ArgSpec`, or a literal of one of its variants +/// such as `.{ .buffer = .{ .type = .float } }`. +fn argSpec(raw: anytype) ArgSpec { + if (@TypeOf(raw) == ArgSpec) return raw; + const variant = comptime literalVariant(@TypeOf(raw), "ArgSpec"); + const inner = @field(raw, variant); + return switch (@field(std.meta.Tag(ArgSpec), variant)) { + .buffer => blk: { + comptime knownFields(@TypeOf(inner), ArgSpec.Buffer, "ArgSpec.buffer"); + break :blk .{ .buffer = .{ + .type = typeOf(inner.type), + .const_ = if (@hasField(@TypeOf(inner), "const_")) inner.const_ else false, + .restrict = if (@hasField(@TypeOf(inner), "restrict")) inner.restrict else false, + } }; + }, + .constant_struct => .{ .constant_struct = inner }, + inline else => |tag| @unionInit(ArgSpec, @tagName(tag), typeOf(inner)), + }; +} + +/// The variant a spec literal names: one, as a union literal does. +fn literalVariant(Literal: type, what: []const u8) []const u8 { + const names = @typeInfo(Literal).@"struct".field_names; + if (names.len != 1) @compileError("dsl/metal: " ++ what ++ " literals name one variant, not " ++ std.fmt.comptimePrint("{d}", .{names.len})); + return names[0]; +} + +/// Rejects a field of a spec literal that `Options` does not have. +fn knownFields(Literal: type, Options: type, what: []const u8) void { + for (@typeInfo(Literal).@"struct".field_names) |name| { + if (!@hasField(Options, name)) @compileError("dsl/metal: " ++ what ++ " has no field " ++ name); + } +} + +/// A `Type`, or the name of one (`.float`). +fn typeOf(t: anytype) Type { + return if (@TypeOf(t) == @EnumLiteral()) @field(Type, @tagName(t)) else t; +} + +/// A declared device, constant or threadgroup argument, for binding. +pub const Argument = struct { + name: []const u8, + kind: enum { buffer, constant, threadgroup }, + /// The `[[buffer(n)]]` or `[[threadgroup(n)]]` index. + index: u32, + /// The size of a constant argument. + size: u64 = 0, +}; + +/// A pointer: its address space and pointee type. +pub const Pointer = struct { + space: AddressSpace, + pointee: Type, +}; + +/// An SSA value with its MSL type. Operators follow C++: operands take the +/// usual arithmetic conversions, integers narrower than `int` are promoted, +/// and signedness picks the operation (`uint / int` divides unsigned). +pub const Value = struct { + inner: *const mlir.Value, + kernel: ?*Builder = null, + kind: Kind = undefined, + + pub const Kind = union(enum) { + value: Type, + pointer: Pointer, + /// A `constant S &` argument. + constant_struct: Struct, + /// A `constant T &` argument or a local: an lvalue, read where it is + /// used as C++ reads it. + reference: Pointer, + /// A `simdgroup_matrix`: its 64 elements, of which a lane + /// holds the first two. + matrix: Scalar, + /// A pointer local. + pointer_reference: Pointer, + }; + + /// The value itself, an lvalue read. + pub fn rvalue(self: Value) Value { + return switch (self.kind) { + .reference => |p| (Value{ .inner = self.inner, .kernel = self.kernel, .kind = .{ .pointer = p } }).load(0), + .pointer_reference => |p| blk: { + const k = self.kern(); + break :blk k.emit(llvm.load(k.ctx, self.inner, llvm.pointerType(k.ctx, p.space), .{ .alignment = 8 }, k.loc()), .{ .pointer = p }); + }, + else => self, + }; + } + + /// `&x` of a local. + pub fn address(reference: Value) Value { + return switch (reference.kind) { + .reference => |p| .{ .inner = reference.inner, .kernel = reference.kernel, .kind = .{ .pointer = p } }, + else => std.debug.panic("dsl/metal: the address of a {t}", .{reference.kind}), + }; + } + + /// `x = value` of a local, converting `value` to its type. + pub fn assign(self: Value, value: anytype) void { + const k = self.kern(); + switch (self.kind) { + .reference => |p| { + if (p.space != .thread) std.debug.panic("dsl/metal: assigning to a {t} reference", .{p.space}); + (Value{ .inner = self.inner, .kernel = k, .kind = .{ .pointer = p } }).store(0, value); + }, + .pointer_reference => |p| { + const v = k.lift(value); + if (v.kind != .pointer or v.kind.pointer.space != p.space) std.debug.panic("dsl/metal: assigning a {t} to a {t} pointer", .{ v.kind, p.space }); + _ = llvm.store(k.ctx, v.inner, self.inner, .{ .alignment = 8 }, k.loc()).appendTo(k.currentBlock()); + }, + else => std.debug.panic("dsl/metal: assigning to a {t}", .{self.kind}), + } + } + + fn kern(self: Value) *Builder { + return self.kernel orelse @panic("dsl/metal: value has no builder"); + } + + /// The MSL type of a scalar or vector value. + pub fn type_(self: Value) Type { + return switch (self.kind) { + .value => |t| t, + .reference => |p| p.pointee, + else => std.debug.panic("dsl/metal: expected a scalar or vector, got a {t}", .{self.kind}), + }; + } + + pub fn pointer(self: Value) Pointer { + return switch (self.kind) { + .pointer => |p| p, + else => std.debug.panic("dsl/metal: expected a pointer, got a {t}", .{self.kind}), + }; + } + + pub fn add(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + if (self.kind == .pointer) return self.kern().offset(self, self.kern().promoted(rhs), false); + return self.kern().arithmetic(.add, self, rhs); + } + pub fn sub(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + if (self.kind == .pointer) return self.kern().offset(self, self.kern().promoted(rhs), true); + return self.kern().arithmetic(.sub, self, rhs); + } + pub fn mul(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().arithmetic(.mul, self, rhs); + } + pub fn div(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().arithmetic(.div, self, rhs); + } + pub fn rem(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().arithmetic(.rem, self, rhs); + } + pub fn bitAnd(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().arithmetic(.bit_and, self, rhs); + } + pub fn bitOr(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().arithmetic(.bit_or, self, rhs); + } + pub fn bitXor(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().arithmetic(.bit_xor, self, rhs); + } + pub fn shl(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().shift(.left, self, rhs); + } + pub fn shr(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().shift(.right, self, rhs); + } + pub fn neg(reference: Value) Value { + const self = reference.rvalue(); + return self.kern().negate(self); + } + /// `~x`. + pub fn bitNot(reference: Value) Value { + const self = reference.rvalue(); + const k = self.kern(); + const v = k.convert(self, self.type_().withScalar(self.type_().scalar.promoted())); + return v.bitXor(k.constant(v.type_(), -1)); + } + + pub fn lt(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().compare(.lt, self, rhs); + } + pub fn le(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().compare(.le, self, rhs); + } + pub fn gt(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().compare(.gt, self, rhs); + } + pub fn ge(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().compare(.ge, self, rhs); + } + pub fn eq(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().compare(.eq, self, rhs); + } + pub fn ne(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().compare(.ne, self, rhs); + } + + /// `T(x)`, `(T) x` or `static_cast(x)` on a scalar or vector; on a + /// pointer, `(space T *) p`. + pub fn to(reference: Value, t: Type) Value { + const self = reference.rvalue(); + const k = self.kern(); + return switch (self.kind) { + .pointer => |p| .{ .inner = self.inner, .kernel = k, .kind = .{ .pointer = .{ .space = p.space, .pointee = t } } }, + else => k.convert(self, t), + }; + } + + /// `as_type(x)`: the same bits as another type of the same size. + pub fn asType(reference: Value, t: Type) Value { + const self = reference.rvalue(); + const k = self.kern(); + const from = self.type_(); + if (from.size() != t.size() or from.len == 3 or t.len == 3) + std.debug.panic("dsl/metal: as_type<{s}> of a {s}", .{ t.name(), from.name() }); + if (from.eql(t)) return self; + return k.emit(llvm.bitcast(k.ctx, self.inner, t.mlirType(k.ctx), k.loc()), .{ .value = t }); + } + + /// `v.x`. + pub fn x(reference: Value) Value { + const self = reference.rvalue(); + return self.at(0); + } + pub fn y(reference: Value) Value { + const self = reference.rvalue(); + return self.at(1); + } + pub fn z(reference: Value) Value { + const self = reference.rvalue(); + return self.at(2); + } + pub fn w(reference: Value) Value { + const self = reference.rvalue(); + return self.at(3); + } + + /// `v[i]` of a vector; the index keeps its type, a literal is an `int`. + pub fn at(reference: Value, i: anytype) Value { + const self = reference.rvalue(); + const k = self.kern(); + const t = self.type_(); + if (!t.isVector()) std.debug.panic("dsl/metal: indexing a {s}", .{t.name()}); + const element = t.element(); + return k.emit(llvm.extractelement(k.ctx, self.inner, k.subscript(i).inner, element.mlirType(k.ctx), k.loc()), .{ .value = element }); + } + + /// The vector with `v[i] = element`, `i` typed as `at` types it; `v.y = + /// element` is `insert(@as(i64, 1), element)`. + pub fn insert(reference: Value, i: anytype, element: anytype) Value { + const self = reference.rvalue(); + const k = self.kern(); + const t = self.type_(); + const e = k.convert(k.lift(element), t.element()); + return k.emit(llvm.insertelement(k.ctx, self.inner, e.inner, k.subscript(i).inner, k.loc()), .{ .value = t }); + } + + /// `p[i]`. + pub fn load(reference: Value, i: anytype) Value { + const self = reference.rvalue(); + const k = self.kern(); + const p = self.pointer(); + const at_ = k.offset(self, i, false); + return k.loadMemory(at_.inner, p.pointee); + } + + /// `p[i] = value`, converting `value` to the pointee type. + pub fn store(reference: Value, i: anytype, value: anytype) void { + const self = reference.rvalue(); + const k = self.kern(); + const p = self.pointer(); + var v = k.convert(k.lift(value), p.pointee); + if (p.pointee.scalar == .bool) v = k.emit(arith.extui(k.ctx, v.inner, p.pointee.memory().mlirType(k.ctx), k.loc()), .{ .value = p.pointee.memory() }); + const at_ = k.offset(self, i, false); + _ = llvm.store(k.ctx, v.inner, at_.inner, .{ .alignment = v.type_().alignment() }, k.loc()).appendTo(k.currentBlock()); + } + + /// `s.name` of a constant struct argument. + pub fn field(reference: Value, comptime name: []const u8) Value { + const self = reference.rvalue(); + const k = self.kern(); + const s = switch (self.kind) { + .constant_struct => |s| s, + else => std.debug.panic("dsl/metal: .{s} of a {t}", .{ name, self.kind }), + }; + const index = s.fieldIndex(name); + const f = s.fields[index]; + if (f.count != 0) std.debug.panic("dsl/metal: struct array field {s}", .{name}); + const gep = llvm.getelementptr(k.ctx, self.inner, &.{ .{ .constant = 0 }, .{ .constant = @intCast(index) } }, k.structType(s), k.loc()); + _ = gep.appendTo(k.currentBlock()); + return k.loadMemory(gep.result(0), f.type); + } + + /// `m.thread_elements()` of a `simdgroup_matrix`: the lane's two + /// elements. + pub fn threadElements(m: Value) Value { + const k = m.kern(); + const t: Type = .vec(Builder.matrixScalar(m), 2); + return k.emit(llvm.shufflevector(k.ctx, m.inner, m.inner, t.mlirType(k.ctx), &.{ 0, 1 }, k.loc()), .{ .value = t }); + } +}; + +/// A loop: `while (c)` and `for (init; c; inc)` branch to the body with +/// `body(c)`, `for (;;)` has its body right away; `next()` ends the body +/// and starts the increment, and `close()` ends the loop. +/// +/// A `break` and the end of the body meet in a latch that reads whether to +/// leave from a local, as clang's cleanup does: Apple's compiler then keeps +/// the loop's test at the top instead of rotating the loop, which is faster +/// (paged attention's decode loop by 2%). +pub const LoopScope = struct { + kernel: *Builder, + cond_block: *mlir.Block, + end_block: *mlir.Block, + loop_annotation: *const llvm.LoopAnnotationAttr, + /// Where a `break` goes after setting `leave`; null until the first one. + latch: ?*mlir.Block = null, + leave: Value = undefined, + exits: bool = false, + + /// Leaves the loop when `cond` is false. + pub fn body(self: *LoopScope, cond: Value) void { + const k = self.kernel; + const body_block = mlir.Block.init(&.{}, &.{}); + k.condBranch(cond, body_block, self.end_block); + self.exits = true; + k.startBlock(body_block); + } + + /// `break;`. + pub fn breakLoop(self: *LoopScope) void { + if (self.kernel.insert == null) return; + self.toLatch(true); + self.exits = true; + } + + fn toLatch(self: *LoopScope, leave: bool) void { + const k = self.kernel; + if (self.latch == null) { + self.latch = mlir.Block.init(&.{}, &.{}); + self.leave = k.local(.bool, null); + } + self.leave.assign(leave); + _ = k.branch(self.latch.?); + } + + /// Ends the body: the increment of `for (...; ...; inc)` follows. + pub fn next(self: *LoopScope) void { + const k = self.kernel; + const inc = mlir.Block.init(&.{}, &.{}); + if (self.latch) |latch| { + if (k.insert != null) self.toLatch(false); + k.startBlock(latch); + k.condBranch(self.leave, self.end_block, inc); + self.latch = null; + } else _ = k.branch(inc); + k.startBlock(inc); + } + + pub fn close(self: *LoopScope) void { + const k = self.kernel; + if (self.latch != null) self.next(); + k.branchLoop(self.cond_block, self.loop_annotation); + if (self.exits) k.startBlock(self.end_block) else self.end_block.deinit(); + } +}; + +/// `for (T i = lower; i < upper; i += step)`: `i` is the local `iv`, the +/// condition and the increment follow C++'s conversions, and a step of 1 is +/// `++i`. +pub const ForScope = struct { + loop: LoopScope, + iv: Value, + step: Step, + + const Step = union(enum) { one, value: Value, int: i64 }; + + pub fn breakLoop(self: *ForScope) void { + self.loop.breakLoop(); + } + + pub fn close(self: *ForScope) void { + const k = self.loop.kernel; + self.loop.next(); + if (k.insert != null) switch (self.step) { + .one => self.iv.assign(k.increment(self.iv)), + .value => |v| { + const s = v.rvalue(); + self.iv.assign(self.iv.add(s)); + }, + .int => |n| self.iv.assign(self.iv.add(k.constant(if (std.math.cast(i32, n) != null) .int else .long, n))), + }; + self.loop.close(); + } +}; + +/// `if (c) { ... }`, or `if (c) { ... } else { ... }` with `elseBranch()` +/// between the two. +pub const IfScope = struct { + kernel: *Builder, + else_block: ?*mlir.Block, + end_block: *mlir.Block, + joins: bool = false, + + pub fn elseBranch(self: *IfScope) void { + const k = self.kernel; + const else_block = self.else_block orelse @panic("dsl/metal: elseBranch of an if without else (open it with openIfElse(c, .{})), or a second one"); + if (k.branch(self.end_block)) self.joins = true; + k.startBlock(else_block); + self.else_block = null; + } + + pub fn close(self: *IfScope) void { + const k = self.kernel; + if (k.branch(self.end_block)) self.joins = true; + if (self.joins) k.startBlock(self.end_block) else self.end_block.deinit(); + } +}; + +/// A condition of `&&`, `||` and `!` terms, short-circuited: a term's code +/// runs only when the terms before it leave the outcome open. `and_(x)` is +/// `x && ...`, `or_(x)` is `x || ...`; the last term opens what the condition +/// guards. A group of `&&` terms ends a `||` chain and the +/// other way around, as in `a || (b && c)`. +pub const Condition = struct { + kernel: *Builder, + true_block: *mlir.Block, + false_block: *mlir.Block, + + pub fn and_(self: *Condition, x: Value) void { + self.term(x, false, .@"and"); + } + pub fn andNot(self: *Condition, x: Value) void { + self.term(x, true, .@"and"); + } + pub fn or_(self: *Condition, x: Value) void { + self.term(x, false, .@"or"); + } + pub fn orNot(self: *Condition, x: Value) void { + self.term(x, true, .@"or"); + } + + fn term(self: *Condition, x: Value, negate: bool, comptime op: enum { @"and", @"or" }) void { + const k = self.kernel; + const next = mlir.Block.init(&.{}, &.{}); + const on_true, const on_false = switch (op) { + .@"and" => .{ next, self.false_block }, + .@"or" => .{ self.true_block, next }, + }; + if (negate) k.condBranch(x, on_false, on_true) else k.condBranch(x, on_true, on_false); + k.startBlock(next); + } + + fn last(self: *Condition, x: Value, negate: bool) void { + if (negate) self.kernel.condBranch(x, self.false_block, self.true_block) else self.kernel.condBranch(x, self.true_block, self.false_block); + } + + /// `if (... x) { ... }`. + pub fn openIf(self: *Condition, x: Value) IfScope { + return self.opened(x, false, false); + } + /// `if (... !x) { ... }`. + pub fn openIfNot(self: *Condition, x: Value) IfScope { + return self.opened(x, true, false); + } + /// `if (... x) { ... } else { ... }`; a chain has no result types. + pub fn openIfElse(self: *Condition, x: Value, result_types: anytype) IfScope { + noResults(result_types); + return self.opened(x, false, true); + } + /// `if (... !x) { ... } else { ... }`. + pub fn openIfElseNot(self: *Condition, x: Value, result_types: anytype) IfScope { + noResults(result_types); + return self.opened(x, true, true); + } + + fn noResults(result_types: anytype) void { + if (common.arity(@TypeOf(result_types), "Condition.openIfElse: result_types") != 0) + @compileError("dsl/metal: a condition chain opens no `c ? a : b`"); + } + + fn opened(self: *Condition, x: Value, negate: bool, with_else: bool) IfScope { + const k = self.kernel; + self.last(x, negate); + k.startBlock(self.true_block); + if (!with_else) return .{ .kernel = k, .else_block = null, .end_block = self.false_block, .joins = true }; + return .{ .kernel = k, .else_block = self.false_block, .end_block = mlir.Block.init(&.{}, &.{}) }; + } +}; + +/// `a && b && ...` or `a || b || ...` as a bool, short-circuited: a +/// term's code runs only when the terms before it leave the result open. +/// `term(x)` adds a term followed by more, `close(x)` the last one. +pub const Logical = struct { + kernel: *Builder, + op: enum { @"and", @"or" }, + end_block: *mlir.Block, + + pub fn term(self: *Logical, x: Value) void { + const k = self.kernel; + const next = mlir.Block.init(&.{}, &.{}); + const c = k.convert(x, .bool); + const decided: []const *const mlir.Value = &.{k.constant(.bool, self.op == .@"or").inner}; + _ = switch (self.op) { + .@"and" => cf.cond_br(k.ctx, c.inner, next, &.{}, self.end_block, decided, .{}, k.loc()), + .@"or" => cf.cond_br(k.ctx, c.inner, self.end_block, decided, next, &.{}, .{}, k.loc()), + }.appendTo(k.currentBlock()); + k.insert = null; + k.startBlock(next); + } + + pub fn close(self: *Logical, x: Value) Value { + const k = self.kernel; + const c = k.convert(x, .bool); + _ = cf.br(k.ctx, self.end_block, &.{c.inner}, k.loc()).appendTo(k.currentBlock()); + k.insert = null; + k.startBlock(self.end_block); + return .{ .inner = self.end_block.argument(0), .kernel = k, .kind = .{ .value = .bool } }; + } +}; + +/// `c ? a : b` evaluating one operand: the true operand's code, then +/// `yieldThen(.{a})`, the false operand's code, then `yieldElse(.{b})`; +/// `results[0]` is the value. +pub const TernaryScope = struct { + kernel: *Builder, + type: Type, + else_block: *mlir.Block, + end_block: *mlir.Block, + results: [1]Value = undefined, + + pub fn yieldThen(self: *TernaryScope, values: anytype) void { + const k = self.kernel; + self.branchEnd(values, "yieldThen"); + k.startBlock(self.else_block); + } + + pub fn yieldElse(self: *TernaryScope, values: anytype) void { + const k = self.kernel; + self.branchEnd(values, "yieldElse"); + k.startBlock(self.end_block); + self.results = .{.{ .inner = self.end_block.argument(0), .kernel = k, .kind = .{ .value = self.type } }}; + } + + fn branchEnd(self: *TernaryScope, values: anytype, comptime what: []const u8) void { + if (common.arity(@TypeOf(values), "TernaryScope." ++ what) != 1) @compileError("TernaryScope." ++ what ++ " takes one value"); + const k = self.kernel; + const v = k.convert(k.lift(values[0]), self.type); + _ = cf.br(k.ctx, self.end_block, &.{v.inner}, k.loc()).appendTo(k.currentBlock()); + k.insert = null; + } +}; + +/// What `Builder.openIfElse` opens for `n` result types. +fn IfElseScope(comptime n: usize) type { + return switch (n) { + 0 => IfScope, + 1 => TernaryScope, + else => @compileError("dsl/metal: openIfElse has one result at most, as `c ? a : b`"), + }; +} + +/// A loop pragma: `full` is `#pragma clang loop unroll(full)`, `enable` +/// `#pragma unroll`. +pub const Unroll = enum { full, enable }; + +pub const LoopOpts = struct { + unroll: ?Unroll = null, +}; + +/// MSL's `metal::fast` and `metal::precise` functions. +pub const MathMode = enum { default, fast, precise }; + +/// `metal::fast::` or `metal::precise::`. +pub const Namespace = struct { + b: *Builder, + mode: MathMode, + + pub fn exp(self: Namespace, x: Value) Value { + return self.b.floatMath("exp", self.mode, .{x}); + } + pub fn exp2(self: Namespace, x: Value) Value { + return self.b.floatMath("exp2", self.mode, .{x}); + } + pub fn log(self: Namespace, x: Value) Value { + return self.b.floatMath("log", self.mode, .{x}); + } + pub fn log2(self: Namespace, x: Value) Value { + return self.b.floatMath("log2", self.mode, .{x}); + } + pub fn tanh(self: Namespace, x: Value) Value { + return self.b.floatMath("tanh", self.mode, .{x}); + } + pub fn sqrt(self: Namespace, x: Value) Value { + return self.b.floatMath("sqrt", self.mode, .{x}); + } + pub fn rsqrt(self: Namespace, x: Value) Value { + return self.b.floatMath("rsqrt", self.mode, .{x}); + } + pub fn sin(self: Namespace, x: Value) Value { + return self.b.floatMath("sin", self.mode, .{x}); + } + pub fn cos(self: Namespace, x: Value) Value { + return self.b.floatMath("cos", self.mode, .{x}); + } + pub fn max(self: Namespace, a: Value, b: anytype) Value { + return self.b.floatMath("fmax", self.mode, .{ a, b }); + } + pub fn min(self: Namespace, a: Value, b: anytype) Value { + return self.b.floatMath("fmin", self.mode, .{ a, b }); + } +}; + +pub const Builder = struct { + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + ctx: *mlir.Context, + module: *mlir.Module, + name: []const u8, + fast_math: bool, + func_op: ?*mlir.Operation = null, + entry_block: ?*mlir.Block = null, + /// The kernel's body; blocks join it in the order they start. + body: ?*mlir.Region = null, + /// The block code goes to; null after a `break` or a `return`. + insert: ?*mlir.Block = null, + arguments: std.ArrayList(Argument) = .empty, + + pub const Opts = struct { + /// MSL's default; `-fno-fast-math` otherwise. + fast_math: bool = true, + }; + + pub fn open(allocator: std.mem.Allocator, ctx: *mlir.Context, name: []const u8, opts: Opts) Builder { + const module: *mlir.Module = .init(.unknown(ctx)); + return .{ + .allocator = allocator, + .arena = .init(allocator), + .ctx = ctx, + .module = module, + .name = name, + .fast_math = opts.fast_math, + }; + } + + pub fn deinit(self: *Builder) void { + self.module.deinit(); + self.arena.deinit(); + } + + /// Declares the kernel's arguments, in order, from a struct literal of + /// `ArgSpec`s, and returns them by name. + pub fn declareArgs(self: *Builder, spec: anytype) FinishError!common.NamedArgs(@TypeOf(spec), Value) { + const ctx = self.ctx; + const location = self.loc(); + const fields = @typeInfo(@TypeOf(spec)).@"struct".field_names; + const arena = self.arena.allocator(); + var arg_types: [fields.len]*const mlir.Type = undefined; + var arg_attrs: [fields.len]*const mlir.Attribute = undefined; + var kinds: [fields.len]Value.Kind = undefined; + var buffers: u32 = 0; + var threadgroups: u32 = 0; + inline for (fields, 0..) |name, i| { + const arg = argSpec(@field(spec, name)); + const opts: air.ArgumentOpts = .{ .name = name }; + switch (arg) { + .buffer => |buffer| { + arg_types[i] = llvm.pointerType(ctx, AddressSpace.device); + arg_attrs[i] = air.bufferArgumentAttr(ctx, .{ + .locationIndex = buffers, + .addressSpace = .device, + .access = if (buffer.const_) .read else .read_write, + .elementType = buffer.type.memory().mlirType(ctx), + .typeName = buffer.type.name(), + .typeSize = buffer.type.size(), + .typeAlignment = buffer.type.alignment(), + .noalias_ = buffer.restrict, + }, opts) catch return error.InvalidMlir; + kinds[i] = .{ .pointer = .{ .space = .device, .pointee = buffer.type } }; + try self.arguments.append(arena, .{ .name = name, .kind = .buffer, .index = buffers }); + buffers += 1; + }, + .constant => |t| { + arg_types[i] = llvm.pointerType(ctx, AddressSpace.constant); + arg_attrs[i] = air.bufferArgumentAttr(ctx, .{ + .locationIndex = buffers, + .addressSpace = .constant, + .access = .read, + .elementType = t.memory().mlirType(ctx), + .typeName = t.name(), + .typeSize = t.size(), + .typeAlignment = t.alignment(), + .bufferSize = t.size(), + }, opts) catch return error.InvalidMlir; + kinds[i] = .{ .reference = .{ .space = .constant, .pointee = t } }; + try self.arguments.append(arena, .{ .name = name, .kind = .constant, .index = buffers, .size = t.size() }); + buffers += 1; + }, + .constant_struct => |s| { + var struct_fields = try arena.alloc(*const air.StructFieldAttr, s.fields.len); + for (s.fields, 0..) |f, j| struct_fields[j] = air.StructFieldAttr.get(ctx, .{ + .offset = s.offset(j), + .size = f.type.size(), + .count = f.count, + .typeName = f.type.name(), + .name = f.name, + }) catch return error.InvalidMlir; + arg_types[i] = llvm.pointerType(ctx, AddressSpace.constant); + arg_attrs[i] = air.bufferArgumentAttr(ctx, .{ + .locationIndex = buffers, + .addressSpace = .constant, + .access = .read, + .elementType = self.structType(s), + .typeName = s.name, + .typeSize = s.size(), + .typeAlignment = s.alignment(), + .bufferSize = s.size(), + .fields = struct_fields, + }, opts) catch return error.InvalidMlir; + kinds[i] = .{ .constant_struct = s }; + try self.arguments.append(arena, .{ .name = name, .kind = .constant, .index = buffers, .size = s.size() }); + buffers += 1; + }, + .threadgroup => |t| { + arg_types[i] = llvm.pointerType(ctx, AddressSpace.threadgroup); + arg_attrs[i] = air.bufferArgumentAttr(ctx, .{ + .locationIndex = threadgroups, + .addressSpace = .threadgroup, + .access = .read_write, + .elementType = t.memory().mlirType(ctx), + .typeName = t.name(), + .typeSize = t.size(), + .typeAlignment = t.alignment(), + }, opts) catch return error.InvalidMlir; + kinds[i] = .{ .pointer = .{ .space = .threadgroup, .pointee = t } }; + try self.arguments.append(arena, .{ .name = name, .kind = .threadgroup, .index = threadgroups }); + threadgroups += 1; + }, + inline else => |t, builtin| { + arg_types[i] = t.mlirType(ctx); + arg_attrs[i] = air.builtinArgumentAttr(ctx, @field(air.Builtin, @tagName(builtin)), opts) catch return error.InvalidMlir; + kinds[i] = .{ .value = t }; + }, + } + } + var locs: [fields.len]*const mlir.Location = @splat(location); + const block = mlir.Block.init(&arg_types, &locs); + self.entry_block = block; + self.func_op = air.kernel(ctx, .{ + .name = self.name, + .block = block, + .args_attributes = &arg_attrs, + .location = location, + .verify = false, + }).appendTo(self.module.body()); + self.body = block.parentRegion(); + self.insert = block; + + var out: common.NamedArgs(@TypeOf(spec), Value) = undefined; + inline for (fields, 0..) |name, i| @field(out, name) = .{ .inner = block.argument(i), .kernel = self, .kind = kinds[i] }; + return out; + } + + pub fn loc(self: *const Builder) *const mlir.Location { + return .unknown(self.ctx); + } + + /// The block code goes to: a fresh, unreachable one after a `break` or + /// a `return`. + pub fn currentBlock(self: *Builder) *mlir.Block { + if (self.insert == null) self.startBlock(mlir.Block.init(&.{}, &.{})); + return self.insert.?; + } + + fn startBlock(self: *Builder, block: *mlir.Block) void { + self.body.?.appendOwnedBlock(block); + self.insert = block; + } + + /// Branches to `dest` unless the code here is unreachable; true if it did. + fn branch(self: *Builder, dest: *mlir.Block) bool { + const block = self.insert orelse return false; + _ = cf.br(self.ctx, dest, &.{}, self.loc()).appendTo(block); + self.insert = null; + return true; + } + + fn condBranch(self: *Builder, cond: Value, on_true: *mlir.Block, on_false: *mlir.Block) void { + const c = self.convert(cond, .bool); + _ = cf.cond_br(self.ctx, c.inner, on_true, &.{}, on_false, &.{}, .{}, self.loc()).appendTo(self.currentBlock()); + self.insert = null; + } + + /// The back edge of a loop, which carries its `llvm.loop` metadata. + fn branchLoop(self: *Builder, dest: *mlir.Block, annotation: *const llvm.LoopAnnotationAttr) void { + const block = self.insert orelse return; + _ = llvm.br(self.ctx, dest, &.{}, .{ .loop_annotation = annotation }, self.loc()).appendTo(block); + self.insert = null; + } + + /// A thread-memory slot of `t`, at the top of the entry block. + fn allocate(self: *Builder, t: *const mlir.Type, alignment: u64) *const mlir.Value { + const ctx = self.ctx; + const entry = self.entry_block.?; + const size = arith.constant_int(ctx, 1, .int(ctx, .i32), self.loc()); + const alloca = llvm.alloca(ctx, size.result(0), llvm.pointerType(ctx, AddressSpace.thread), t, .{ .alignment = alignment }, self.loc()); + if (entry.firstOperation()) |first| entry.insertOwnedOperationBefore(first, alloca) else _ = alloca.appendTo(entry); + entry.insertOwnedOperationBefore(alloca, size); + return alloca.result(0); + } + + /// `T x;`, or `T x = init;`: a local, read where it is used and written by + /// `assign`. + pub fn local(self: *Builder, t: Type, init: anytype) Value { + const memory = t.memory(); + const x: Value = .{ .inner = self.allocate(memory.mlirType(self.ctx), memory.alignment()), .kernel = self, .kind = .{ .reference = .{ .space = .thread, .pointee = t } } }; + if (@TypeOf(init) != @TypeOf(null)) x.assign(init); + return x; + } + + /// `T *p = init;`: a pointer local. + pub fn localPointer(self: *Builder, init: Value) Value { + const p: Pointer = switch (init.kind) { + .pointer, .pointer_reference => |q| q, + else => std.debug.panic("dsl/metal: a pointer local of a {t}", .{init.kind}), + }; + const x: Value = .{ .inner = self.allocate(llvm.pointerType(self.ctx, p.space), 8), .kernel = self, .kind = .{ .pointer_reference = p } }; + x.assign(init); + return x; + } + + /// A `simdgroup_matrix` whose lane holds `elements`, two values + /// converted to `T`: what assigning `m.thread_elements()` makes. + pub fn matrix(self: *Builder, scalar: Scalar, elements: anytype) Value { + const v = self.convert(self.lift(elements), .vec(scalar, 2)); + var mask: [64]i32 = @splat(-1); + mask[0..2].* = .{ 0, 1 }; + return self.emit(llvm.shufflevector(self.ctx, v.inner, v.inner, self.matrixType(scalar), &mask, self.loc()), .{ .matrix = scalar }); + } + + /// `++i`: a narrow integer steps in its own type, as C++ steps it. + fn increment(self: *Builder, i: Value) Value { + const v = i.rvalue(); + const t = v.type_(); + if (t.scalar.isFloat() or t.scalar.bits() >= 32) return v.add(1); + return self.emit(arith.addi(self.ctx, v.inner, self.constant(t, 1).inner, self.loc()), .{ .value = t }); + } + + fn emit(self: *Builder, op: *mlir.Operation, kind: Value.Kind) Value { + _ = op.appendTo(self.currentBlock()); + return .{ .inner = op.result(0), .kernel = self, .kind = kind }; + } + + fn fastmath(self: *const Builder) air.FloatOpts { + return .{ .fastmath = if (self.fast_math) .fast else .none }; + } + + fn setFastmath(self: *Builder, op: *mlir.Operation) void { + if (!self.fast_math) return; + op.setAttribute(arith.fastmath(self.ctx, .fast) catch unreachable); + } + + fn structType(self: *Builder, s: Struct) *const mlir.Type { + var fields: [32]*const mlir.Type = undefined; + for (s.fields, 0..) |f, i| { + const t = f.type.memory().mlirType(self.ctx); + fields[i] = if (f.count != 0) llvm.arrayType(t, f.count) else t; + } + var name: std.ArrayList(u8) = .empty; + name.print(self.arena.allocator(), "struct.{s}", .{s.name}) catch @panic("dsl/metal: out of memory"); + return llvm.identifiedStructType(self.ctx, name.items, fields[0..s.fields.len]) orelse + std.debug.panic("dsl/metal: two structs named {s}", .{s.name}); + } + + /// The MSL type of a Zig number, as `lift` gives it. + fn literalType(value: anytype) Type { + return switch (@typeInfo(@TypeOf(value))) { + .comptime_int => if (value >= std.math.minInt(i32) and value <= std.math.maxInt(i32)) .int else .long, + .comptime_float => .float, + .int => |info| switch (info.bits) { + 8 => if (info.signedness == .signed) .char else .uchar, + 16 => if (info.signedness == .signed) .short else .ushort, + 32 => if (info.signedness == .signed) .int else .uint, + 64 => if (info.signedness == .signed) .long else .ulong, + else => @compileError("dsl/metal: no MSL type for " ++ @typeName(@TypeOf(value))), + }, + else => @compileError("dsl/metal: expected a literal, got " ++ @typeName(@TypeOf(value))), + }; + } + + /// A Zig value as an MSL literal: integers are `int` (`long` when they do + /// not fit), floats are `float`. + pub fn lift(self: *Builder, value: anytype) Value { + const T = @TypeOf(value); + if (T == Value) return value.rvalue(); + return switch (@typeInfo(T)) { + .comptime_int, .comptime_float, .int => self.constant(literalType(value), value), + .bool => self.constant(.bool, value), + .float => |info| self.constant(switch (info.bits) { + 16 => Type.half, + 32 => Type.float, + else => @compileError("dsl/metal: no MSL type for " ++ @typeName(T)), + }, value), + else => @compileError("dsl/metal: expected a Value or a number, got " ++ @typeName(T)), + }; + } + + /// A constant of type `t`, splat across a vector. + pub fn constant(self: *Builder, t: Type, value: anytype) Value { + const ctx = self.ctx; + const ty = t.mlirType(ctx); + const op = if (t.scalar.isFloat()) blk: { + const v: f64 = switch (@typeInfo(@TypeOf(value))) { + .int, .comptime_int => @floatFromInt(value), + .bool => @floatFromInt(@intFromBool(value)), + else => @floatCast(value), + }; + if (t.isVector()) break :blk arith.constant_dense_splat_f(ctx, v, ty, self.loc()); + break :blk switch (t.scalar) { + .half => arith.constant_float(ctx, v, .f16, self.loc()), + .bfloat => arith.constant_float(ctx, v, .bf16, self.loc()), + else => arith.constant_float(ctx, v, .f32, self.loc()), + }; + } else blk: { + const v: i64 = switch (@typeInfo(@TypeOf(value))) { + .bool => @intFromBool(value), + .int, .comptime_int => if (@TypeOf(value) == u64) @bitCast(value) else value, + else => @intFromFloat(value), + }; + if (t.isVector()) break :blk arith.constant_dense_splat_i(ctx, v, ty, self.loc()); + break :blk arith.constant_int(ctx, v, ty, self.loc()); + }; + return self.emit(op, .{ .value = t }); + } + + /// `T(a, b, ...)` from scalars, or `T(x)` splat from one scalar. + pub fn vec(self: *Builder, t: Type, elements: anytype) Value { + const fields = @typeInfo(@TypeOf(elements)).@"struct".field_names; + if (fields.len == 1) return self.convert(self.lift(@field(elements, fields[0])), t); + if (fields.len != t.len) std.debug.panic("dsl/metal: {d} elements of a {s}", .{ fields.len, t.name() }); + var inner: [fields.len]*const mlir.Value = undefined; + inline for (fields, 0..) |name, i| inner[i] = self.convert(self.lift(@field(elements, name)), t.element()).inner; + return self.emit(vector.from_elements(self.ctx, &inner, t.mlirType(self.ctx), self.loc()), .{ .value = t }); + } + + fn splat(self: *Builder, scalar: Value, t: Type) Value { + return self.emit(vector.broadcast(self.ctx, scalar.inner, t.mlirType(self.ctx), self.loc()), .{ .value = t }); + } + + /// The conversion of `v` to `to`, as C++ converts: integer widths extend + /// by the source's signedness or truncate, integers and floats convert + /// through `air.convert`, and a scalar converts to a vector by splatting. + pub fn convert(self: *Builder, reference: Value, to: Type) Value { + const v = reference.rvalue(); + const ctx = self.ctx; + const from = v.type_(); + if (from.eql(to)) return v; + if (from.len != to.len) { + if (!from.isVector()) return self.splat(self.convert(v, to.element()), to); + std.debug.panic("dsl/metal: cannot convert a {s} to a {s}", .{ from.name(), to.name() }); + } + const fs = from.scalar; + const ts = to.scalar; + const ty = to.mlirType(ctx); + const kind: Value.Kind = .{ .value = to }; + if (ts == .bool) return v.ne(self.constant(from, 0)); + if (fs == .bool) { + if (ts.isFloat()) return self.convert(self.convert(v, to.withScalar(.int)), to); + return self.emit(arith.extui(ctx, v.inner, ty, self.loc()), kind); + } + if (fs.isFloat() and ts.isFloat()) { + if (!from.isVector() and (fs == .float or ts == .float)) { + const op = if (ts == .float) arith.extf(ctx, v.inner, ty, self.loc()) else arith.truncf(ctx, v.inner, ty, self.loc()); + return self.emit(op, kind); + } + return self.emit(air.convert(ctx, v.inner, ty, self.fastmath(), self.loc()), kind); + } + if (fs.isFloat()) { + const op = if (ts.isUnsigned()) air.convert_u(ctx, v.inner, ty, .{}, self.loc()) else air.convert_s(ctx, v.inner, ty, .{}, self.loc()); + return self.emit(op, kind); + } + if (ts.isFloat()) { + const op = if (fs.isUnsigned()) air.convert_u(ctx, v.inner, ty, self.fastmath(), self.loc()) else air.convert_s(ctx, v.inner, ty, self.fastmath(), self.loc()); + return self.emit(op, kind); + } + if (fs.bits() == ts.bits()) return .{ .inner = v.inner, .kernel = self, .kind = kind }; + if (fs.bits() > ts.bits()) return self.emit(arith.trunci(ctx, v.inner, ty, self.loc()), kind); + const op = if (fs.isUnsigned()) arith.extui(ctx, v.inner, ty, self.loc()) else arith.extsi(ctx, v.inner, ty, self.loc()); + return self.emit(op, kind); + } + + /// The operands of a binary operator after the usual arithmetic + /// conversions; a scalar operand of a vector takes its element type. + fn usualConversions(self: *Builder, a: Value, b_: anytype) [2]Value { + const b = self.lift(b_); + const ta = a.type_(); + const tb = b.type_(); + const converted: Type = if (ta.isVector() and tb.isVector()) blk: { + if (!ta.eql(tb)) std.debug.panic("dsl/metal: {s} and {s} operands", .{ ta.name(), tb.name() }); + break :blk ta; + } else if (ta.isVector()) ta else if (tb.isVector()) tb else .{ .scalar = types.commonScalar(ta.scalar, tb.scalar) }; + return .{ self.convert(a, converted), self.convert(b, converted) }; + } + + const ArithmeticOp = enum { add, sub, mul, div, rem, bit_and, bit_or, bit_xor }; + + fn arithmetic(self: *Builder, comptime op: ArithmeticOp, a: Value, b: anytype) Value { + const ctx = self.ctx; + const l, const r = self.usualConversions(a, b); + const t = l.type_(); + const loc_ = self.loc(); + if (t.scalar.isFloat()) { + const result = switch (op) { + .add => arith.addf(ctx, l.inner, r.inner, loc_), + .sub => arith.subf(ctx, l.inner, r.inner, loc_), + .mul => arith.mulf(ctx, l.inner, r.inner, loc_), + .div => arith.divf(ctx, l.inner, r.inner, loc_), + else => std.debug.panic("dsl/metal: {t} of floats", .{op}), + }; + self.setFastmath(result); + return self.emit(result, .{ .value = t }); + } + const unsigned = t.scalar.isUnsigned(); + const result = switch (op) { + .add => arith.addi(ctx, l.inner, r.inner, loc_), + .sub => arith.subi(ctx, l.inner, r.inner, loc_), + .mul => arith.muli(ctx, l.inner, r.inner, loc_), + .div => if (unsigned) arith.divui(ctx, l.inner, r.inner, loc_) else arith.divsi(ctx, l.inner, r.inner, loc_), + .rem => if (unsigned) arith.remui(ctx, l.inner, r.inner, loc_) else arith.remsi(ctx, l.inner, r.inner, loc_), + .bit_and => arith.andi(ctx, l.inner, r.inner, loc_), + .bit_or => arith.ori(ctx, l.inner, r.inner, loc_), + .bit_xor => arith.xori(ctx, l.inner, r.inner, loc_), + }; + // Signed overflow is undefined in C++. + if (!unsigned and (op == .add or op == .sub or op == .mul)) { + result.setAttribute(arith.overflowFlags(ctx, .{ .nsw = true }) catch unreachable); + } + return self.emit(result, .{ .value = t }); + } + + /// `a << b` or `a >> b`. MSL shifts by the amount modulo the width: + /// clang casts the promoted amount to `a`'s promoted type as unsigned, + /// then masks it, which folds away for a number within the width. + fn shift(self: *Builder, comptime direction: enum { left, right }, a: Value, b: anytype) Value { + const t = a.type_().withScalar(a.type_().scalar.promoted()); + const l = self.convert(a, t); + const mask = t.scalar.bits() - 1; + const r = if (@TypeOf(b) == Value) blk: { + const rb = b.rvalue(); + const amount = self.shiftAmount(self.convert(rb, rb.type_().withScalar(rb.type_().scalar.promoted())), t); + break :blk self.emit(arith.andi(self.ctx, amount.inner, self.constant(t, mask).inner, self.loc()), .{ .value = t }); + } else blk: { + if (b < 0 or b > mask) std.debug.panic("dsl/metal: a shift of a {s} by {d}, past its width", .{ t.name(), b }); + const rb = self.lift(b); + break :blk self.convert(self.convert(rb, rb.type_().withScalar(rb.type_().scalar.promoted())), t); + }; + const loc_ = self.loc(); + const op = switch (direction) { + .left => arith.shli(self.ctx, l.inner, r.inner, loc_), + .right => if (t.scalar.isUnsigned()) arith.shrui(self.ctx, l.inner, r.inner, loc_) else arith.shrsi(self.ctx, l.inner, r.inner, loc_), + }; + return self.emit(op, .{ .value = t }); + } + + /// A promoted shift amount `v` as the shifted type `t`: a scalar of a + /// vector converts and splats, as C++ converts it; otherwise the cast is + /// unsigned, as clang casts it. + fn shiftAmount(self: *Builder, v: Value, t: Type) Value { + const from = v.type_(); + if (from.len != t.len) return self.convert(v, t); + if (from.scalar.bits() == t.scalar.bits()) return .{ .inner = v.inner, .kernel = self, .kind = .{ .value = t } }; + const ty = t.mlirType(self.ctx); + const op = if (from.scalar.bits() > t.scalar.bits()) arith.trunci(self.ctx, v.inner, ty, self.loc()) else arith.extui(self.ctx, v.inner, ty, self.loc()); + return self.emit(op, .{ .value = t }); + } + + fn negate(self: *Builder, a: Value) Value { + const t = a.type_(); + if (t.scalar.isFloat()) { + const op = arith.negf(self.ctx, a.inner, self.loc()); + self.setFastmath(op); + return self.emit(op, .{ .value = t }); + } + const x = self.convert(a, t.withScalar(t.scalar.promoted())); + return self.constant(x.type_(), 0).sub(x); + } + + const Comparison = enum { lt, le, gt, ge, eq, ne }; + + fn compare(self: *Builder, comptime op: Comparison, a: Value, b: anytype) Value { + const l, const r = self.usualConversions(a, b); + const t = l.type_(); + const result_type = t.withScalar(.bool); + if (t.scalar.isFloat()) { + const predicate: arith.CmpFPredicate = switch (op) { + .lt => .olt, + .le => .ole, + .gt => .ogt, + .ge => .oge, + .eq => .oeq, + .ne => .une, + }; + const result = arith.cmpf(self.ctx, predicate, l.inner, r.inner, self.loc()); + self.setFastmath(result); + return self.emit(result, .{ .value = result_type }); + } + const unsigned = t.scalar.isUnsigned(); + const predicate: arith.CmpIPredicate = switch (op) { + .lt => if (unsigned) .ult else .slt, + .le => if (unsigned) .ule else .sle, + .gt => if (unsigned) .ugt else .sgt, + .ge => if (unsigned) .uge else .sge, + .eq => .eq, + .ne => .ne, + }; + return self.emit(arith.cmpi(self.ctx, predicate, l.inner, r.inner, self.loc()), .{ .value = result_type }); + } + + /// `i` of `p + i` and `p - i`, promoted as C++ promotes an integer + /// operand; `p[i]` and a vector subscript take `i` as it is. + fn promoted(self: *Builder, i: anytype) Value { + const v = self.subscript(i); + const t = v.type_(); + return self.convert(v, t.withScalar(t.scalar.promoted())); + } + + /// A vector subscript, of its own type as clang keeps it: a literal is an + /// `int`, a `ushort` an `i16`, a `bool` an `i1`. + fn subscript(self: *Builder, i: anytype) Value { + const v = self.lift(i); + const t = v.type_(); + if (t.isVector() or t.scalar.isFloat()) std.debug.panic("dsl/metal: an index of a {s}", .{t.name()}); + return v; + } + + /// A `t` read from `p`, where it is a `t.memory()`. + fn loadMemory(self: *Builder, p: *const mlir.Value, t: Type) Value { + const memory = t.memory(); + const x = self.emit(llvm.load(self.ctx, p, memory.mlirType(self.ctx), .{ .alignment = memory.alignment() }, self.loc()), .{ .value = memory }); + if (t.scalar != .bool) return x; + return self.emit(arith.trunci(self.ctx, x.inner, t.mlirType(self.ctx), self.loc()), .{ .value = t }); + } + + /// `p + i` (`p - i` when `negative`) in elements of the pointee type. + fn offset(self: *Builder, p: Value, i: anytype, negative: bool) Value { + if (@TypeOf(i) == comptime_int and i == 0) return p; + const pointee = switch (p.kind) { + .pointer => |ptr| ptr.pointee, + else => std.debug.panic("dsl/metal: pointer arithmetic on a {t}", .{p.kind}), + }; + var index = self.lift(i); + if (index.type_().scalar.isFloat() or index.type_().isVector()) std.debug.panic("dsl/metal: pointer offset by a {s}", .{index.type_().name()}); + index = self.convert(index, if (index.type_().scalar.isUnsigned()) .ulong else .long); + if (negative) index = self.constant(.long, 0).sub(index); + const gep = llvm.getelementptr(self.ctx, p.inner, &.{.{ .value = index.inner }}, pointee.memory().mlirType(self.ctx), self.loc()); + return self.emit(gep, p.kind); + } + + /// `cond ? a : b` of evaluated operands, a select; + /// `openIfElse(cond, .{T})` evaluates one operand. + pub fn ternary(self: *Builder, cond: Value, a: anytype, b: anytype) Value { + const l, const r = self.usualConversions(self.lift(a), b); + const c = self.convert(cond, if (l.type_().isVector() and cond.type_().isVector()) l.type_().withScalar(.bool) else .bool); + return self.emit(arith.select(self.ctx, c.inner, l.inner, r.inner, self.loc()), l.kind); + } + + fn openTernary(self: *Builder, cond: Value, t: Type) TernaryScope { + const then_block = mlir.Block.init(&.{}, &.{}); + const else_block = mlir.Block.init(&.{}, &.{}); + var loc_ = [1]*const mlir.Location{self.loc()}; + const end_block = mlir.Block.init(&.{t.mlirType(self.ctx)}, &loc_); + self.condBranch(cond, then_block, else_block); + self.startBlock(then_block); + return .{ .kernel = self, .type = t, .else_block = else_block, .end_block = end_block }; + } + + /// `T a[n] = {};`, zeroed as clang zeroes it. + pub fn zeroedArray(self: *Builder, t: Type, n: u32) Value { + const a = self.array(t, n); + const len = self.constant(.long, @as(i64, n) * @as(i64, @intCast(t.size()))); + const operands = [_]*const mlir.Value{ a.inner, self.constant(.char, 0).inner, len.inner, self.constant(.bool, false).inner }; + _ = llvm.call_intrinsic(self.ctx, "llvm.memset.p0.i64", &operands, .{}, self.loc()).appendTo(self.currentBlock()); + return a; + } + + /// `T a[n];` in thread memory, as a pointer to its first element. + pub fn array(self: *Builder, t: Type, n: u32) Value { + return .{ .inner = self.allocate(llvm.arrayType(t.memory().mlirType(self.ctx), n), t.alignment()), .kernel = self, .kind = .{ .pointer = .{ .space = .thread, .pointee = t } } }; + } + + fn floatMath(self: *Builder, comptime name: []const u8, mode: MathMode, args: anytype) Value { + const x = self.lift(args[0]); + const t = x.type_(); + if (!t.scalar.isFloat() or t.scalar == .bfloat) + std.debug.panic("dsl/metal: no {s} of a {s}", .{ name, t.name() }); + const use_fast = switch (mode) { + .default => self.fast_math and t.scalar == .float, + .fast => true, + .precise => false, + }; + if (use_fast and t.scalar != .float) std.debug.panic("dsl/metal: fast::{s} of a {s}", .{ name, t.name() }); + const opts = self.fastmath(); + const loc_ = self.loc(); + const op = switch (args.len) { + 1 => if (use_fast) @field(air, "fast_" ++ name)(self.ctx, x.inner, opts, loc_) else @field(air, name)(self.ctx, x.inner, opts, loc_), + 2 => blk: { + const y = self.convert(self.lift(args[1]), t); + break :blk if (use_fast) @field(air, "fast_" ++ name)(self.ctx, x.inner, y.inner, opts, loc_) else @field(air, name)(self.ctx, x.inner, y.inner, opts, loc_); + }, + 3 => blk: { + const y = self.convert(self.lift(args[1]), t); + const z = self.convert(self.lift(args[2]), t); + break :blk if (use_fast) @field(air, "fast_" ++ name)(self.ctx, x.inner, y.inner, z.inner, opts, loc_) else @field(air, name)(self.ctx, x.inner, y.inner, z.inner, opts, loc_); + }, + else => unreachable, + }; + return self.emit(op, .{ .value = t }); + } + + /// `metal::fast::f`: `b.fast().exp2(x)`. + pub fn fast(self: *Builder) Namespace { + return .{ .b = self, .mode = .fast }; + } + + /// `metal::precise::f`: `b.precise().tanh(x)`. + pub fn precise(self: *Builder) Namespace { + return .{ .b = self, .mode = .precise }; + } + + pub fn exp(self: *Builder, x: Value) Value { + return self.floatMath("exp", .default, .{x}); + } + pub fn exp2(self: *Builder, x: Value) Value { + return self.floatMath("exp2", .default, .{x}); + } + pub fn log(self: *Builder, x: Value) Value { + return self.floatMath("log", .default, .{x}); + } + pub fn log2(self: *Builder, x: Value) Value { + return self.floatMath("log2", .default, .{x}); + } + pub fn tanh(self: *Builder, x: Value) Value { + return self.floatMath("tanh", .default, .{x}); + } + pub fn sqrt(self: *Builder, x: Value) Value { + return self.floatMath("sqrt", .default, .{x}); + } + pub fn rsqrt(self: *Builder, x: Value) Value { + return self.floatMath("rsqrt", .default, .{x}); + } + pub fn ceil(self: *Builder, x: Value) Value { + return self.floatMath("ceil", .default, .{x}); + } + pub fn floor(self: *Builder, x: Value) Value { + return self.floatMath("floor", .default, .{x}); + } + pub fn round(self: *Builder, x: Value) Value { + return self.floatMath("round", .default, .{x}); + } + pub fn trunc(self: *Builder, x: Value) Value { + return self.floatMath("trunc", .default, .{x}); + } + pub fn sin(self: *Builder, x: Value) Value { + return self.floatMath("sin", .default, .{x}); + } + pub fn cos(self: *Builder, x: Value) Value { + return self.floatMath("cos", .default, .{x}); + } + pub fn pow(self: *Builder, x: Value, y: anytype) Value { + return self.floatMath("pow", .default, .{ x, y }); + } + pub fn fmod(self: *Builder, x: Value, y: anytype) Value { + return self.floatMath("fmod", .default, .{ x, y }); + } + + /// `fma(a, b, c)` of floats. + pub fn fma(self: *Builder, a_: Value, b: anytype, c: anytype) Value { + const a = a_.rvalue(); + const t = a.type_(); + if (!t.scalar.isFloat() or t.scalar == .bfloat) std.debug.panic("dsl/metal: fma of a {s}", .{t.name()}); + const y = self.convert(self.lift(b), t); + const z = self.convert(self.lift(c), t); + return self.emit(air.fma(self.ctx, a.inner, y.inner, z.inner, self.fastmath(), self.loc()), .{ .value = t }); + } + + /// `dot(a, b)` of float vectors. + pub fn dot(self: *Builder, a_: Value, b_: Value) Value { + const a = a_.rvalue(); + const b = b_.rvalue(); + const t = a.type_(); + if (!t.eql(b.type_()) or !t.isVector()) std.debug.panic("dsl/metal: dot of {s} and {s}", .{ t.name(), b.type_().name() }); + return self.emit(air.dot(self.ctx, a.inner, b.inner, self.fastmath(), self.loc()), .{ .value = t.element() }); + } + + /// `abs(x)`. + pub fn abs(self: *Builder, x_: Value) Value { + const x = x_.rvalue(); + const t = x.type_(); + if (t.scalar.isFloat()) return self.floatMath("fabs", .default, .{x}); + if (t.scalar.isUnsigned()) return x; + return self.emit(air.abs_s(self.ctx, x.inner, self.loc()), .{ .value = t }); + } + + /// The operands of an MSL function taking several values of one type, + /// read, with literals converted to the other operand's type. + fn sameType(self: *Builder, a: anytype, b: anytype) [2]Value { + if (@TypeOf(a) != Value) { + const r = self.lift(b); + return .{ self.convert(self.lift(a), r.type_()), r }; + } + const l = a.rvalue(); + const r = if (@TypeOf(b) == Value) b.rvalue() else self.convert(self.lift(b), l.type_()); + if (!l.type_().eql(r.type_())) std.debug.panic("dsl/metal: {s} and {s} arguments", .{ l.type_().name(), r.type_().name() }); + return .{ l, r }; + } + + /// `max(a, b)`. + pub fn max(self: *Builder, a: anytype, b: anytype) Value { + const l, const r = self.sameType(a, b); + const t = l.type_(); + if (t.scalar.isFloat()) return self.floatMath("fmax", .default, .{ l, r }); + const op = if (t.scalar.isUnsigned()) air.max_u(self.ctx, l.inner, r.inner, self.loc()) else air.max_s(self.ctx, l.inner, r.inner, self.loc()); + return self.emit(op, .{ .value = t }); + } + + /// `min(a, b)`. + pub fn min(self: *Builder, a: anytype, b: anytype) Value { + const l, const r = self.sameType(a, b); + const t = l.type_(); + if (t.scalar.isFloat()) return self.floatMath("fmin", .default, .{ l, r }); + const op = if (t.scalar.isUnsigned()) air.min_u(self.ctx, l.inner, r.inner, self.loc()) else air.min_s(self.ctx, l.inner, r.inner, self.loc()); + return self.emit(op, .{ .value = t }); + } + + /// `clamp(x, lo, hi)`. + pub fn clamp(self: *Builder, x_: Value, lo: anytype, hi: anytype) Value { + const x = x_.rvalue(); + const t = x.type_(); + const l = self.convert(self.lift(lo), t); + const h = self.convert(self.lift(hi), t); + if (t.scalar.isFloat()) return self.floatMath("clamp", .default, .{ x, l, h }); + const op = if (t.scalar.isUnsigned()) air.clamp_u(self.ctx, x.inner, l.inner, h.inner, self.loc()) else air.clamp_s(self.ctx, x.inner, l.inner, h.inner, self.loc()); + return self.emit(op, .{ .value = t }); + } + + fn simdType(x: Value, comptime name: []const u8) Type { + const t = x.type_(); + switch (t.scalar) { + .half, .float, .char, .uchar, .short, .ushort, .int, .uint => {}, + else => std.debug.panic("dsl/metal: no {s} of a {s}", .{ name, t.name() }), + } + return t; + } + + fn simdReduce(self: *Builder, comptime name: []const u8, x_: Value) Value { + const x = x_.rvalue(); + const t = simdType(x, name); + const op = if (t.scalar.isFloat()) + @field(air, name)(self.ctx, x.inner, self.fastmath(), self.loc()) + else if (t.scalar.isUnsigned()) + @field(air, name ++ "_u")(self.ctx, x.inner, self.loc()) + else + @field(air, name ++ "_s")(self.ctx, x.inner, self.loc()); + return self.emit(op, .{ .value = t }); + } + + /// A function moving `x` across lanes; `x` is frozen, so that a poison + /// lane does not poison the others. + fn shuffle(self: *Builder, comptime name: []const u8, x_: Value, lane: anytype) Value { + const read = x_.rvalue(); + const t = simdType(read, name); + const x = self.emit(llvm.freeze(self.ctx, read.inner, self.loc()), .{ .value = t }); + const l = self.convert(self.lift(lane), .ushort); + const op = if (t.scalar.isFloat()) + @field(air, name)(self.ctx, x.inner, l.inner, self.fastmath(), self.loc()) + else if (t.scalar.isUnsigned()) + @field(air, name ++ "_u")(self.ctx, x.inner, l.inner, self.loc()) + else + @field(air, name ++ "_s")(self.ctx, x.inner, l.inner, self.loc()); + return self.emit(op, .{ .value = t }); + } + + pub fn simdSum(self: *Builder, x: Value) Value { + return self.simdReduce("simd_sum", x); + } + pub fn simdMax(self: *Builder, x: Value) Value { + return self.simdReduce("simd_max", x); + } + pub fn simdMin(self: *Builder, x: Value) Value { + return self.simdReduce("simd_min", x); + } + pub fn simdPrefixExclusiveSum(self: *Builder, x_: Value) Value { + const x = x_.rvalue(); + const t = x.type_(); + if (!t.scalar.isFloat() or t.scalar == .bfloat) std.debug.panic("dsl/metal: simd_prefix_exclusive_sum of a {s}", .{t.name()}); + return self.emit(air.simd_prefix_exclusive_sum(self.ctx, x.inner, self.fastmath(), self.loc()), .{ .value = t }); + } + /// `simd_shuffle(x, lane)`. + pub fn simdShuffle(self: *Builder, x: Value, lane: anytype) Value { + return self.shuffle("simd_shuffle", x, lane); + } + /// `simd_shuffle_xor(x, mask)`. + pub fn simdShuffleXor(self: *Builder, x: Value, mask: anytype) Value { + return self.shuffle("simd_shuffle_xor", x, mask); + } + /// `simd_shuffle_down(x, delta)`. + pub fn simdShuffleDown(self: *Builder, x: Value, delta: anytype) Value { + return self.shuffle("simd_shuffle_down", x, delta); + } + /// `simd_broadcast(x, lane)`. + pub fn simdBroadcast(self: *Builder, x: Value, lane: anytype) Value { + return self.shuffle("simd_broadcast", x, lane); + } + pub fn simdAny(self: *Builder, x: Value) Value { + return self.emit(air.simd_any(self.ctx, self.convert(x, .bool).inner, self.loc()), .{ .value = .bool }); + } + pub fn simdAll(self: *Builder, x: Value) Value { + return self.emit(air.simd_all(self.ctx, self.convert(x, .bool).inner, self.loc()), .{ .value = .bool }); + } + + /// `threadgroup_barrier(flags)`. + pub fn threadgroupBarrier(self: *Builder, flags: MemFlags) void { + _ = air.wg_barrier(self.ctx, @bitCast(flags), self.loc()).appendTo(self.currentBlock()); + } + + /// `simdgroup_barrier(flags)`. + pub fn simdgroupBarrier(self: *Builder, flags: MemFlags) void { + _ = air.simdgroup_barrier(self.ctx, @bitCast(flags), self.loc()).appendTo(self.currentBlock()); + } + + pub const MatrixOpts = struct { + /// `matrix_origin`. + origin: [2]i64 = .{ 0, 0 }, + /// `transpose_matrix`. + transpose: bool = false, + }; + + fn matrixType(self: *Builder, scalar: Scalar) *const mlir.Type { + return .vector(&.{64}, scalar.mlirType(self.ctx)); + } + + fn matrixScalar(m: Value) Scalar { + return switch (m.kind) { + .matrix => |s| s, + else => std.debug.panic("dsl/metal: expected a simdgroup_matrix, got a {t}", .{m.kind}), + }; + } + + /// The extents, strides and origin a matrix load or store passes the + /// builtin, from its `elements_per_row`, `matrix_origin` and + /// `transpose_matrix`. + fn accessLayout(self: *Builder, elements_per_row: anytype, opts: MatrixOpts) [3]*const mlir.Value { + const ulong2: Type = .vec(.ulong, 2); + const epr = self.convert(self.lift(elements_per_row), .ulong); + const eight = self.constant(.ulong, 8); + const one = self.constant(.ulong, 1); + const o0 = self.constant(.ulong, opts.origin[0]); + const o1 = self.constant(.ulong, opts.origin[1]); + const layout: [3]Value = if (opts.transpose) + .{ self.vec(ulong2, .{ eight, epr }), self.vec(ulong2, .{ epr, one }), self.vec(ulong2, .{ o1, o0 }) } + else + .{ self.vec(ulong2, .{ epr, eight }), self.vec(ulong2, .{ one, epr }), self.vec(ulong2, .{ o0, o1 }) }; + return .{ layout[0].inner, layout[1].inner, layout[2].inner }; + } + + /// `simdgroup_load(m, src, elements_per_row, matrix_origin, + /// transpose_matrix)`: the matrix `m` it loads. + pub fn simdgroupLoad(self: *Builder, src_: Value, elements_per_row: anytype, opts: MatrixOpts) Value { + const src = src_.rvalue(); + const t = src.pointer().pointee.scalar; + const layout = self.accessLayout(elements_per_row, opts); + return self.emit(air.simdgroup_matrix_8x8_load(self.ctx, src.inner, layout[0], layout[1], layout[2], self.matrixType(t), self.fastmath(), self.loc()), .{ .matrix = t }); + } + + /// `simdgroup_store(m, dst, elements_per_row, matrix_origin, + /// transpose_matrix)`. + pub fn simdgroupStore(self: *Builder, m: Value, dst_: Value, elements_per_row: anytype, opts: MatrixOpts) void { + const t = matrixScalar(m); + const dst = dst_.rvalue(); + const p = dst.pointer(); + if (p.pointee.scalar != t) std.debug.panic("dsl/metal: simdgroup_store to a {s} pointer", .{p.pointee.name()}); + const layout = self.accessLayout(elements_per_row, opts); + _ = air.simdgroup_matrix_8x8_store(self.ctx, m.inner, dst.inner, layout[0], layout[1], layout[2], self.loc()).appendTo(self.currentBlock()); + } + + /// `simdgroup_multiply_accumulate(d, a, b, c)`: the matrix `d = a * b + c`, + /// of `c`'s type. + pub fn simdgroupMultiplyAccumulate(self: *Builder, a: Value, b: Value, c: Value) Value { + _ = .{ matrixScalar(a), matrixScalar(b) }; + const t = matrixScalar(c); + return self.emit(air.simdgroup_matrix_8x8_multiply_accumulate(self.ctx, a.inner, b.inner, c.inner, self.matrixType(t), self.fastmath(), self.loc()), .{ .matrix = t }); + } + + /// The `llvm.loop` metadata of an MSL loop: C++ loops must make progress. + fn loopAnnotation(self: *Builder, opts: LoopOpts) *const llvm.LoopAnnotationAttr { + const unroll: ?*const llvm.LoopUnrollAttr = if (opts.unroll) |u| switch (u) { + .full => llvm.LoopUnrollAttr.get(self.ctx, .{ .full = true }), + .enable => llvm.LoopUnrollAttr.get(self.ctx, .{ .disable = false }), + } else null; + return llvm.LoopAnnotationAttr.get(self.ctx, .{ .unroll = unroll, .mustProgress = true }); + } + + /// The type `i < upper` compares in, for a loop variable `i` of type `t`: the + /// loop runs in `t`, which a short or char variable may be narrower than. + fn loopComparison(t: Type, upper: Type) Scalar { + const compared = types.commonScalar(t.scalar, upper.scalar); + const narrow = t.scalar.bits() < 32 and compared == .int; + if (t.scalar.isFloat() or t.isVector() or (compared.bits() != t.scalar.bits() and !narrow)) + std.debug.panic("dsl/metal: a {s} loop variable compared to a {s}", .{ t.name(), upper.name() }); + return compared; + } + + /// `for (T i = lower; i < upper; i += step)`, with `T` the type of + /// `lower`; `close` ends the body. + pub fn openFor(self: *Builder, lower: anytype, upper: anytype, step: anytype) ForScope { + return self.openForOpts(lower, upper, step, .{}); + } + + /// `openFor` under a loop pragma. + pub fn openForOpts(self: *Builder, lower: anytype, upper: anytype, step: anytype, opts: LoopOpts) ForScope { + const i = self.local(if (@TypeOf(lower) == Value) lower.type_() else literalType(lower), lower); + var loop = self.openLoop(opts); + _ = loopComparison(i.type_(), if (@TypeOf(upper) == Value) upper.type_() else literalType(upper)); + loop.body(i.lt(upper)); + const step_: ForScope.Step = switch (@typeInfo(@TypeOf(step))) { + .comptime_int, .int => if (step == 1) .one else .{ .int = step }, + else => .{ .value = step }, + }; + return .{ .loop = loop, .iv = i, .step = step_ }; + } + + /// A loop: `while (c)`, `for (init; c; inc)` or `for (;;)`. + pub fn openLoop(self: *Builder, opts: LoopOpts) LoopScope { + const cond_block = mlir.Block.init(&.{}, &.{}); + _ = self.branch(cond_block); + self.startBlock(cond_block); + return .{ .kernel = self, .cond_block = cond_block, .end_block = mlir.Block.init(&.{}, &.{}), .loop_annotation = self.loopAnnotation(opts) }; + } + + /// `if (cond) { ... }`. + pub fn openIf(self: *Builder, cond: Value) IfScope { + var c = self.condition(); + return c.openIf(cond); + } + + /// `if (cond) { ... } else { ... }` with no result types; with one, `T`, + /// `cond ? a : b` of type `T` evaluating one operand. + pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfElseScope(common.arity(@TypeOf(result_types), "openIfElse: result_types")) { + if (comptime common.arity(@TypeOf(result_types), "openIfElse: result_types") == 0) { + var c = self.condition(); + return c.openIfElse(cond, .{}); + } + return self.openTernary(cond, typeOf(result_types[0])); + } + + /// A condition of `&&`, `||` and `!` terms. + pub fn condition(self: *Builder) Condition { + return .{ .kernel = self, .true_block = mlir.Block.init(&.{}, &.{}), .false_block = mlir.Block.init(&.{}, &.{}) }; + } + + /// `a && b && ...` as a bool. + pub fn openAnd(self: *Builder) Logical { + return self.logicalChain(.@"and"); + } + + /// `a || b || ...` as a bool. + pub fn openOr(self: *Builder) Logical { + return self.logicalChain(.@"or"); + } + + fn logicalChain(self: *Builder, op: @FieldType(Logical, "op")) Logical { + var loc_ = [1]*const mlir.Location{self.loc()}; + return .{ .kernel = self, .op = op, .end_block = mlir.Block.init(&.{Type.bool.mlirType(self.ctx)}, &loc_) }; + } + + /// `return;`. + pub fn return_(self: *Builder) void { + const block = self.insert orelse return; + _ = func.returns(self.ctx, &.{}, self.loc()).appendTo(block); + self.insert = null; + } + + /// `if (cond) return;`. + pub fn returnIf(self: *Builder, cond: Value) void { + var scope = self.openIf(cond); + { + self.return_(); + scope.close(); + } + } + + /// Terminates the kernel and returns its module, verified. + pub fn finish(self: *Builder) FinishError![:0]const u8 { + self.return_(); + if (self.fast_math) air.enableFastMath(self.ctx, self.module.operation()); + try common.verify(self.module, "dsl/metal"); + var writer: std.Io.Writer.Allocating = .init(self.allocator); + defer writer.deinit(); + try writer.writer.print("{f}", .{self.module.operation()}); + return try self.allocator.dupeSentinel(u8, writer.written(), 0); + } +}; + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + registerDialects(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectContains(ir: []const u8, needles: []const []const u8) !void { + for (needles) |needle| { + if (std.mem.find(u8, ir, needle) == null) { + std.debug.print("missing `{s}` in:\n{s}\n", .{ needle, ir }); + return error.TestExpectedEqual; + } + } +} + +test "arguments and C++ arithmetic" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .buffer = .{ .type = .bfloat, .const_ = true, .restrict = true } }, + .y = .{ .buffer = .{ .type = .float4 } }, + .n = .{ .constant = .int }, + .shmem = .{ .threadgroup = .half }, + .gid = .{ .threadgroup_position_in_grid = .uint3 }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + // `gid.z / n` divides unsigned, `lane * 2` promotes to int. + const q = a.gid.z().div(a.n); + try std.testing.expect(q.type_().eql(.uint)); + const twice = a.lane.mul(2); + try std.testing.expect(twice.type_().eql(.int)); + const v = a.x.load(q).to(.float).mul(0.5); + try std.testing.expect(v.type_().eql(.float)); + a.y.store(twice, b.vec(.float4, .{v})); + a.shmem.store(a.lane, v); + try std.testing.expectEqual(4, b.arguments.items.len); + try std.testing.expectEqual(1, b.arguments.items[1].index); + try std.testing.expectEqual(4, b.arguments.items[2].size); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "air.fast_math_enable", "air.kernel", "noalias = true" }); + try expectContains(ir, &.{ "arith.divui", "overflow", "arith.extf", "arith.mulf", "fastmath", "arith.truncf", "llvm.getelementptr", "#air.builtin", "air.arg_name = \"lane\"" }); +} + +test "conversions follow Apple's compiler" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .h = .{ .buffer = .{ .type = .half4, .const_ = true } }, + .o = .{ .buffer = .{ .type = .bfloat4 } }, + .i = .{ .buffer = .{ .type = .int } }, + .tid = .{ .thread_position_in_grid = .uint }, + }); + const f4 = a.h.load(a.tid).to(.float4); + a.o.store(a.tid, f4.to(.bfloat4)); + const n = a.i.load(a.tid); + a.i.store(0, n.to(.float).add(n.to(.half).to(.float)).to(.uint).to(.short)); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "air.convert", ": vector<4xf16> to vector<4xf32>", ": vector<4xf32> to vector<4xbf16>", "air.convert.s", ": i32 to f32", ": i32 to f16", "arith.extf", "air.convert.u", "arith.trunci" }); +} + +test "math, SIMD-group and barrier functions" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .buffer = .{ .type = .float } }, + .v = .{ .buffer = .{ .type = .float4 } }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + const x = a.x.load(a.lane); + const v = a.v.load(a.lane); + const m = b.simdMax(b.max(x, b.fast().exp2(x))); + const s = b.simdSum(b.exp(x.sub(m))); + const t = b.precise().tanh(b.fma(s, 2.0, b.dot(v, v))); + const r = b.simdShuffleXor(t, 8); + b.simdgroupBarrier(.mem_threadgroup); + b.threadgroupBarrier(.{ .device = true, .threadgroup = true }); + a.x.store(a.lane, b.ternary(r.gt(0), b.min(a.lane.to(.int), 3).to(.float), r)); + a.x.store(1, b.ternary(r.gt(1), 1.0, 2.0)); + a.v.store(a.lane, b.simdSum(v)); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "air.fast_exp2", "air.fast_fmax", "air.simd_max", "air.fast_exp", "air.simd_sum", "air.dot", "air.fma", "air.tanh", "air.simd_shuffle_xor", "air.simdgroup.barrier ", "air.wg.barrier ", "air.min.s", "arith.select", ": vector<4xf32>" }); + try std.testing.expect(std.mem.find(u8, ir, "cf.cond_br") == null); +} + +test "loops, ifs and early returns" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .buffer = .{ .type = .float } }, + .n = .{ .constant = .int }, + .tid = .{ .thread_position_in_grid = .uint }, + }); + var ret = b.openIf(a.tid.ge(1024)); + { + a.x.store(0, 0.0); + b.return_(); + ret.close(); + } + b.returnIf(a.n.le(0)); + + // float acc = 0; for (int i = 0; i < n; ++i) { if (x[i] < 0) break; acc += x[i]; } + const acc = b.local(.float, 0.0); + var loop = b.openFor(0, a.n, 1); + { + var neg = b.openIf(a.x.load(loop.iv).lt(0.0)); + { + loop.breakLoop(); + neg.close(); + } + acc.assign(acc.add(a.x.load(loop.iv))); + loop.close(); + } + + // int lo = 0, hi = n; while (lo < hi) { int mid = (lo + hi + 1) / 2; ... } + const lo = b.local(.int, 0); + const hi = b.local(.int, a.n); + var search = b.openLoop(.{}); + { + search.body(lo.lt(hi)); + } + { + const mid = b.local(.int, lo.add(hi).add(1).div(2)); + var up = b.openIfElse(a.x.load(mid).le(acc), .{}); + { + lo.assign(mid); + up.elseBranch(); + } + { + hi.assign(mid.sub(1)); + up.close(); + } + search.close(); + } + + // bool m = lo > 0 || (n > 3 && acc < 1); + var m = b.openOr(); + m.term(lo.gt(0)); + var c = b.condition(); + c.and_(a.n.gt(3)); + var only = c.openIfNot(m.close(acc.lt(1.0))); + { + a.x.store(1, b.ternary(lo.gt(0), acc, 2.0)); + only.close(); + } + + const scratch = b.array(.float, 8); + scratch.store(a.tid.rem(8), acc); + a.x.store(2, scratch.load(3)); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "cf.cond_br", "llvm.br", "loop_annotation", "mustProgress = true", "llvm.alloca", "!llvm.array<8 x f32>", "arith.remui" }); + try std.testing.expect(std.mem.find(u8, ir, "scf.") == null); + // The two early returns and the last one, each where it is. + try std.testing.expectEqual(3, std.mem.count(u8, ir, "\n return\n")); + // Every local is allocated in the entry block, before any branch. + try std.testing.expect(std.mem.findLast(u8, ir, "llvm.alloca").? < std.mem.find(u8, ir, "cf.cond_br").?); +} + +test "a return inside a loop and in both branches of an if" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .buffer = .{ .type = .float } }, + .n = .{ .constant = .int }, + }); + // for (;;) { if (n <= 0) break; if (x[0] < 0) return; x[0] -= 1; } + var forever = b.openLoop(.{}); + { + var stop = b.openIf(a.n.le(0)); + { + forever.breakLoop(); + stop.close(); + } + b.returnIf(a.x.load(0).lt(0.0)); + a.x.store(0, a.x.load(0).sub(1.0)); + forever.close(); + } + // if (n > 1) { x[1] = 1; return; } else { x[1] = 2; return; } + var both = b.openIfElse(a.n.gt(1), .{}); + { + a.x.store(1, 1.0); + b.return_(); + both.elseBranch(); + } + { + a.x.store(1, 2.0); + b.return_(); + both.close(); + } + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + // The loop's return and one per branch; `finish` adds none after them. + try std.testing.expectEqual(3, std.mem.count(u8, ir, "\n return\n")); + try expectContains(ir, &.{"loop_annotation"}); +} + +test "a break and the end of the body meet in the latch" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .buffer = .{ .type = .float } }, + .n = .{ .constant = .int }, + }); + // for (int i = 0; i < n; ++i) { if (x[i] < 0) break; x[i] = 1; } + var loop = b.openFor(0, a.n, 1); + { + var neg = b.openIf(a.x.load(loop.iv).lt(0.0)); + { + loop.breakLoop(); + neg.close(); + } + a.x.store(loop.iv, 1.0); + loop.close(); + } + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + // `i`, and the flag the break and the end of the body set for the latch. + try std.testing.expectEqual(2, std.mem.count(u8, ir, "llvm.alloca")); + // `i < n`, `x[i] < 0`, and the latch's choice between leaving and `++i`. + try std.testing.expectEqual(3, std.mem.count(u8, ir, "cf.cond_br")); +} + +test "simdgroup matrices" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .o = .{ .buffer = .{ .type = .float2 } }, + .s = .{ .threadgroup = .bfloat }, + .f = .{ .constant = .int }, + .lane = .{ .thread_index_in_simdgroup = .uint }, + }); + const k = b.simdgroupLoad(a.s.add(8), 40, .{ .transpose = true }); + const q = b.simdgroupLoad(a.s, a.f, .{}); + const acc = b.simdgroupMultiplyAccumulate(q, k, b.matrix(.float, a.o.load(a.lane))); + a.o.store(a.lane, acc.threadElements()); + b.simdgroupStore(acc, a.s.to(.float), 8, .{}); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "air.simdgroup_matrix_8x8_load", "-> vector<64xbf16>", "air.simdgroup_matrix_8x8_multiply_accumulate", "vector<64xbf16>, vector<64xbf16>, vector<64xf32> -> vector<64xf32>", "air.simdgroup_matrix_8x8_store" }); + // `b.matrix` widens two elements to a matrix, `threadElements` takes them back. + try std.testing.expectEqual(2, std.mem.count(u8, ir, "llvm.shufflevector")); + try expectContains(ir, &.{"[0, 1] : vector<64xf32>"}); + try std.testing.expect(std.mem.find(u8, ir, "llvm.alloca") == null); +} + +test "constant struct arguments" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{ .fast_math = false }); + defer b.deinit(); + const Args: Struct = .{ .name = "args_t", .fields = &.{ + .{ .name = "num_heads", .type = .int }, + .{ .name = "scale", .type = .float }, + } }; + const a = try b.declareArgs(.{ + .args = .{ .constant_struct = Args }, + .o = .{ .buffer = .{ .type = .float } }, + }); + a.o.store(0, a.args.field("scale").mul(a.args.field("num_heads"))); + try std.testing.expectEqual(8, b.arguments.items[0].size); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "", "typeName = \"args_t\"", "!llvm.struct<\"struct.args_t\", (i32, f32)>" }); + try std.testing.expect(std.mem.find(u8, ir, "fastmath") == null); +} + +fn expectCount(ir: []const u8, needle: []const u8, expected: usize) !void { + const n = std.mem.count(u8, ir, needle); + if (n != expected) { + std.debug.print("`{s}` {d} times, not {d}, in:\n{s}\n", .{ needle, n, expected, ir }); + return error.TestExpectedEqual; + } +} + +test "subscripts index with the index's type, swizzles with an int" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .v = .{ .buffer = .{ .type = .float4 } }, + .o = .{ .buffer = .{ .type = .float } }, + .n = .{ .constant = .long }, + .gid = .{ .threadgroup_position_in_grid = .uint3 }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + // float4 x = v[0]; + // o[gid.z] = x.x + x.y + x.z + x.w + x[2] + x[lane] + x[(char) lane] + x[n]; + const x = b.local(.float4, a.v.load(0)); + const sum = x.x().add(x.y()).add(x.z()).add(x.w()).add(x.at(2)).add(x.at(a.lane)).add(x.at(a.lane.to(.char))).add(x.at(a.n)); + a.o.store(a.gid.z(), sum); + // x[lane] = 1.0f; x[2] = 2.0f; x.y = 3.0f; v[0] = x; + x.assign(x.insert(a.lane, 1.0)); + x.assign(x.insert(2, 2.0)); + x.assign(x.insert(1, 3.0)); + a.v.store(0, x); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + // `i32` for the swizzles and literals, `i16 %lane`, `i8`, `i64 %n`. + try expectCount(ir, ": i32] : vector<4xf32>", 7); + try expectCount(ir, ": i16] : vector<4xf32>", 2); + try expectCount(ir, ": i8] : vector<4xf32>", 1); + try expectCount(ir, ": i64] : vector<4xf32>", 1); + try expectCount(ir, ": i32] : vector<3xi32>", 1); +} + +test "a pointer local" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .o = .{ .buffer = .{ .type = .float } }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + // device float *p = o + 4; p += lane; *p = 1.0f; + const p = b.localPointer(a.o.add(4)); + p.assign(p.add(a.lane)); + p.store(0, 1.0); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectCount(ir, "llvm.alloca", 1); + try expectCount(ir, "llvm.getelementptr", 2); +} + +test "integer max, min, abs and clamp read locals and constant arguments" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .o = .{ .buffer = .{ .type = .int } }, + .w = .{ .buffer = .{ .type = .uint } }, + .f = .{ .buffer = .{ .type = .float } }, + .n = .{ .constant = .int }, + .u = .{ .constant = .uint }, + }); + // int m = n; o[0] = max(m, n) + min(n, m) + min(m, 7) + abs(m) + clamp(m, 0, 7); + const m = b.local(.int, a.n); + a.o.store(0, b.max(m, a.n).add(b.min(a.n, m)).add(b.min(m, 7)).add(b.abs(m)).add(b.clamp(m, 0, 7))); + // w[0] = max(u, 3u) + min(3u, u) + abs(u) + clamp(u, 1u, 2u); + a.w.store(0, b.max(a.u, 3).add(b.min(3, a.u)).add(b.abs(a.u)).add(b.clamp(a.u, 1, 2))); + // const float x = f[0]; f[1] = clamp(x, 0.0f, 1.0f) + fabs(x) + max(x, x); + const x = a.f.load(0); + a.f.store(1, b.clamp(x, 0.0, 1.0).add(b.abs(x)).add(b.max(x, x))); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "air.max.s", "air.min.s", "air.abs.s", "air.clamp.s", "air.max.u", "air.min.u", "air.clamp.u", "air.fast_clamp", "air.fast_fabs", "air.fast_fmax" }); + // abs of a uint is the value itself. + try std.testing.expect(std.mem.find(u8, ir, "air.abs.u") == null); +} + +test "char and uchar take the same functions as wider integers" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .c = .{ .buffer = .{ .type = .char } }, + .u = .{ .buffer = .{ .type = .uchar } }, + .v = .{ .buffer = .{ .type = .char2 } }, + .f = .{ .buffer = .{ .type = .float } }, + .w = .{ .buffer = .{ .type = .float2 } }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + const c = a.c.load(a.lane); + const u = a.u.load(a.lane); + const v = a.v.load(a.lane); + // f[0] = float(c) + float(u); c[0] = char(f[1]); u[0] = uchar(f[2]); + a.f.store(0, c.to(.float).add(u.to(.float))); + a.c.store(0, a.f.load(1).to(.char)); + a.u.store(0, a.f.load(2).to(.uchar)); + // w[0] = float2(v); v[0] = char2(w[1]); + a.w.store(0, v.to(.float2)); + a.v.store(0, a.w.load(1).to(.char2)); + // c[1] = max(c, c); u[1] = min(u, u); c[2] = abs(c); c[3] = clamp(c, 1, 5); v[1] = max(v, v); + a.c.store(1, b.max(c, c)); + a.u.store(1, b.min(u, u)); + a.c.store(2, b.abs(c)); + a.c.store(3, b.clamp(c, 1, 5)); + a.v.store(1, b.max(v, v)); + // c[4] = simd_shuffle(c, lane); c[5] = simd_sum(c); u[2] = simd_max(u); + a.c.store(4, b.simdShuffle(c, a.lane)); + a.c.store(5, b.simdSum(c)); + a.u.store(2, b.simdMax(u)); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ + "air.convert.s", ": i8 to f32", "air.convert.u", ": f32 to i8", + ": vector<2xi8> to vector<2xf32>", ": vector<2xf32> to vector<2xi8>", "air.max.s", "air.min.u", + "air.abs.s", "air.clamp.s", "air.simd_shuffle.s", "air.simd_sum.s", + "air.simd_max.u", + }); +} + +test "bitwise operators and shifts" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .o = .{ .buffer = .{ .type = .int } }, + .w = .{ .buffer = .{ .type = .uint } }, + .l = .{ .buffer = .{ .type = .long } }, + .n = .{ .constant = .int }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + // int s = n; o[0] = (n << s) ^ (~(n | 3) & -s); w[0] = w[1] >> s; o[1] = lane >> 1; + // l[0] = l[1] << n; + const s = b.local(.int, a.n); + a.o.store(0, a.n.shl(s).bitXor(a.n.bitOr(3).bitNot().bitAnd(s.neg()))); + a.w.store(0, a.w.load(1).shr(s)); + a.o.store(1, a.lane.shr(1)); + a.l.store(0, a.l.load(1).shl(a.n)); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "arith.shli", "arith.shrui", "arith.shrsi", "arith.xori", "arith.ori", "arith.subi" }); + // clang masks an amount to the width (`and i32 %s, 31`), after casting it + // as unsigned (`zext i32 %n to i64`, `and i64 ..., 63`); a number's mask + // folds away. + try expectCount(ir, "arith.andi", 4); + try expectContains(ir, &.{ "arith.constant 31 : i32", "arith.constant 63 : i64" }); + var lines = std.mem.splitScalar(u8, ir, '\n'); + const unsigned_cast = while (lines.next()) |line| { + if (std.mem.find(u8, line, "arith.extui") != null and std.mem.find(u8, line, "i32 to i64") != null) break true; + } else false; + try std.testing.expect(unsigned_cast); +} + +test "a shift reads its amount once" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .o = .{ .buffer = .{ .type = .int } }, + .n = .{ .constant = .int }, + }); + // int s = n; *o = s << s; + const s = b.local(.int, a.n); + a.o.store(0, s.shl(s)); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + // n, and s twice. + try expectCount(ir, "llvm.load", 3); +} + +test "math functions" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .buffer = .{ .type = .float } }, + .h = .{ .buffer = .{ .type = .half } }, + }); + const v = a.x.load(0); + var sum = b.local(.float, 0.0); + for ([_]Value{ + b.exp2(v), b.log(v), b.log2(v), b.tanh(v), b.sqrt(v), + b.rsqrt(v), b.ceil(v), b.floor(v), b.round(v), b.trunc(v), + b.sin(v), b.cos(v), b.pow(v, 2.0), b.fmod(v, 3.0), + }) |term| sum.assign(sum.add(term)); + const fast_ = b.fast(); + for ([_]Value{ + fast_.exp(v), fast_.log(v), fast_.log2(v), fast_.sqrt(v), fast_.rsqrt(v), + fast_.sin(v), fast_.cos(v), fast_.tanh(v), fast_.max(v, 1.0), fast_.min(v, 0.5), + }) |term| sum.assign(sum.add(term)); + const precise_ = b.precise(); + for ([_]Value{ + precise_.exp(v), precise_.exp2(v), precise_.log(v), precise_.log2(v), precise_.sqrt(v), + precise_.rsqrt(v), precise_.sin(v), precise_.cos(v), precise_.max(v, 1.0), precise_.min(v, 0.5), + }) |term| sum.assign(sum.add(term)); + a.x.store(1, sum); + // half has no fast:: functions: exp of a half is precise. + a.h.store(0, b.exp(a.h.load(1))); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ + "air.fast_exp2", "air.fast_log", "air.fast_log2", "air.fast_tanh", "air.fast_sqrt", "air.fast_rsqrt", + "air.fast_ceil", "air.fast_floor", "air.fast_round", "air.fast_trunc", "air.fast_sin", "air.fast_cos", + "air.fast_pow", "air.fast_fmod", "air.fast_exp", "air.fast_fmax", "air.fast_fmin", + }); + try expectContains(ir, &.{ "air.exp ", "air.exp2 ", "air.log ", "air.log2 ", "air.sqrt ", "air.rsqrt ", "air.sin ", "air.cos ", "air.fmax ", "air.fmin " }); + try expectContains(ir, &.{"f16"}); +} + +test "SIMD-group functions" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .buffer = .{ .type = .float } }, + .i = .{ .buffer = .{ .type = .int } }, + .u = .{ .buffer = .{ .type = .uint } }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + const f = a.x.load(a.lane); + const n = a.i.load(a.lane); + const m = a.u.load(a.lane); + a.x.store(0, b.simdMin(f).add(b.simdPrefixExclusiveSum(f)).add(b.simdShuffle(f, a.lane)).add(b.simdShuffleDown(f, 1)).add(b.simdBroadcast(f, 0))); + a.i.store(0, b.simdMin(n).add(b.simdShuffle(n, 3)).add(b.simdShuffleDown(n, a.lane)).add(b.simdBroadcast(n, 0))); + a.u.store(0, b.simdMin(m).add(b.simdShuffleXor(m, 1))); + a.i.store(1, b.ternary(b.simdAny(f.gt(0.0)), 1, 0).add(b.ternary(b.simdAll(n.ne(0)), 2, 0))); + b.threadgroupBarrier(.mem_device); + b.simdgroupBarrier(.mem_texture); + b.threadgroupBarrier(.mem_none); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ + "air.simd_min ", "air.simd_min.s", "air.simd_min.u", "air.simd_prefix_exclusive_sum", + "air.simd_shuffle ", "air.simd_shuffle.s", "air.simd_shuffle_down ", "air.simd_shuffle_down.s", + "air.simd_broadcast ", "air.simd_broadcast.s", "air.simd_shuffle_xor.u", "air.simd_any", + "air.simd_all", "llvm.freeze", + }); + try expectContains(ir, &.{ "air.wg.barrier ", "air.simdgroup.barrier ", "air.wg.barrier " }); +} + +test "pragmas, conditions and casts" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .o = .{ .buffer = .{ .type = .float } }, + .n = .{ .constant = .int }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + + const r = a.o.load(a.lane).mul(2.0); + // float acc[4] = {}; + const acc = b.zeroedArray(.float, 4); + // #pragma clang loop unroll(full) + var full = b.openForOpts(0, 4, 1, .{ .unroll = .full }); + { + acc.store(full.iv, r); + full.close(); + } + // #pragma unroll + var enabled = b.openForOpts(0, a.n, 2, .{ .unroll = .enable }); + { + acc.store(0, acc.load(0).add(enabled.iv.to(.float))); + enabled.close(); + } + // for (int i = 0; i < 4; i += 1) {} + const i = b.local(.int, 0); + var counted = b.openLoop(.{}); + { + counted.body(i.lt(4)); + } + { + counted.next(); + i.assign(i.add(1)); + counted.close(); + } + // const uint bits = as_type(acc[0]); + const bits = acc.load(0).asType(.uint); + // float t = r; *&t = 2.0f; + const t = b.local(.float, r); + t.address().store(0, 2.0); + // bool ok = n > 0 && bits != 0; + var both = b.openAnd(); + both.term(a.n.gt(0)); + const ok = both.close(bits.ne(0)); + // if (!(n > 1) || ok) { ... } else { ... } + var c1 = b.condition(); + c1.orNot(a.n.gt(1)); + var s1 = c1.openIfElse(ok, .{}); + { + a.o.store(1, 1.0); + s1.elseBranch(); + } + { + a.o.store(1, 2.0); + s1.close(); + } + // if (!ok && !(n == 3)) { ... } else { ... } + var c2 = b.condition(); + c2.andNot(ok); + var s2 = c2.openIfElseNot(a.n.eq(3), .{}); + { + a.o.store(2, 1.0); + s2.elseBranch(); + } + { + s2.close(); + } + // o[lane] = n > 2 ? fast::exp(t) : t; + var choice = b.openIfElse(a.n.gt(2), .{.float}); + { + choice.yieldThen(.{b.fast().exp(t)}); + } + { + choice.yieldElse(.{t}); + } + a.o.store(a.lane, choice.results[0]); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "llvm.memset.p0.i64", "full = true", "disable = false", "llvm.bitcast" }); +} diff --git a/dsl/metal/types.zig b/dsl/metal/types.zig new file mode 100644 index 00000000..9c3072bb --- /dev/null +++ b/dsl/metal/types.zig @@ -0,0 +1,291 @@ +const std = @import("std"); + +const mlir = @import("mlir"); + +/// An MSL scalar type. +pub const Scalar = enum { + bool, + char, + uchar, + short, + ushort, + int, + uint, + long, + ulong, + half, + bfloat, + float, + + pub fn isFloat(self: Scalar) bool { + return switch (self) { + .half, .bfloat, .float => true, + else => false, + }; + } + + pub fn isUnsigned(self: Scalar) bool { + return switch (self) { + .bool, .uchar, .ushort, .uint, .ulong => true, + else => false, + }; + } + + pub fn bits(self: Scalar) u16 { + return switch (self) { + .bool => 1, + .char, .uchar => 8, + .short, .ushort, .half, .bfloat => 16, + .int, .uint, .float => 32, + .long, .ulong => 64, + }; + } + + /// The size in memory: a bool takes a byte. + pub fn size(self: Scalar) u64 { + return if (self == .bool) 1 else self.bits() / 8; + } + + /// The C++ integer conversion rank. + fn rank(self: Scalar) u8 { + return switch (self) { + .bool => 0, + .char, .uchar => 1, + .short, .ushort => 2, + .int, .uint => 3, + .long, .ulong => 4, + else => unreachable, + }; + } + + fn unsignedOf(self: Scalar) Scalar { + return switch (self) { + .char => .uchar, + .short => .ushort, + .int => .uint, + .long => .ulong, + else => self, + }; + } + + /// C++ integral promotion: types narrower than int become int. + pub fn promoted(self: Scalar) Scalar { + return switch (self) { + .bool, .char, .uchar, .short, .ushort => .int, + else => self, + }; + } + + pub fn mlirType(self: Scalar, ctx: *mlir.Context) *const mlir.Type { + return switch (self) { + .bool => .int(ctx, .i1), + .char, .uchar => .int(ctx, .i8), + .short, .ushort => .int(ctx, .i16), + .int, .uint => .int(ctx, .i32), + .long, .ulong => .int(ctx, .i64), + .half => .float(ctx, .f16), + .bfloat => .float(ctx, .bf16), + .float => .float(ctx, .f32), + }; + } +}; + +/// The C++ usual arithmetic conversions of two scalar operand types. +pub fn commonScalar(a: Scalar, b: Scalar) Scalar { + if (a.isFloat() or b.isFloat()) { + if (!b.isFloat()) return a; + if (!a.isFloat()) return b; + if (a == b) return a; + if (a == .float or b == .float) return .float; + std.debug.panic("dsl/metal: no common type of {t} and {t}", .{ a, b }); + } + const x = a.promoted(); + const y = b.promoted(); + if (x == y) return x; + if (x.isUnsigned() == y.isUnsigned()) return if (x.rank() >= y.rank()) x else y; + const u = if (x.isUnsigned()) x else y; + const s = if (x.isUnsigned()) y else x; + if (u.rank() >= s.rank()) return u; + if (s.bits() > u.bits()) return s; + return s.unsignedOf(); +} + +/// An MSL scalar or vector type: `float`, `uint3`, `bfloat4`. +pub const Type = struct { + scalar: Scalar, + /// 1 for a scalar, else the vector length (2, 3 or 4). + len: u8 = 1, + + pub const @"bool": Type = .{ .scalar = .bool }; + pub const char: Type = .{ .scalar = .char }; + pub const uchar: Type = .{ .scalar = .uchar }; + pub const short: Type = .{ .scalar = .short }; + pub const ushort: Type = .{ .scalar = .ushort }; + pub const int: Type = .{ .scalar = .int }; + pub const uint: Type = .{ .scalar = .uint }; + pub const long: Type = .{ .scalar = .long }; + pub const ulong: Type = .{ .scalar = .ulong }; + pub const half: Type = .{ .scalar = .half }; + pub const bfloat: Type = .{ .scalar = .bfloat }; + pub const float: Type = .{ .scalar = .float }; + pub const bool2: Type = .vec(.bool, 2); + pub const bool3: Type = .vec(.bool, 3); + pub const bool4: Type = .vec(.bool, 4); + pub const char2: Type = .vec(.char, 2); + pub const char3: Type = .vec(.char, 3); + pub const char4: Type = .vec(.char, 4); + pub const uchar2: Type = .vec(.uchar, 2); + pub const uchar3: Type = .vec(.uchar, 3); + pub const uchar4: Type = .vec(.uchar, 4); + pub const short2: Type = .vec(.short, 2); + pub const short3: Type = .vec(.short, 3); + pub const short4: Type = .vec(.short, 4); + pub const ushort2: Type = .vec(.ushort, 2); + pub const ushort3: Type = .vec(.ushort, 3); + pub const ushort4: Type = .vec(.ushort, 4); + pub const int2: Type = .vec(.int, 2); + pub const int3: Type = .vec(.int, 3); + pub const int4: Type = .vec(.int, 4); + pub const uint2: Type = .vec(.uint, 2); + pub const uint3: Type = .vec(.uint, 3); + pub const uint4: Type = .vec(.uint, 4); + pub const long2: Type = .vec(.long, 2); + pub const long3: Type = .vec(.long, 3); + pub const long4: Type = .vec(.long, 4); + pub const ulong2: Type = .vec(.ulong, 2); + pub const ulong3: Type = .vec(.ulong, 3); + pub const ulong4: Type = .vec(.ulong, 4); + pub const half2: Type = .vec(.half, 2); + pub const half3: Type = .vec(.half, 3); + pub const half4: Type = .vec(.half, 4); + pub const bfloat2: Type = .vec(.bfloat, 2); + pub const bfloat3: Type = .vec(.bfloat, 3); + pub const bfloat4: Type = .vec(.bfloat, 4); + pub const float2: Type = .vec(.float, 2); + pub const float3: Type = .vec(.float, 3); + pub const float4: Type = .vec(.float, 4); + + /// `vec`. + pub fn vec(scalar: Scalar, len: u8) Type { + std.debug.assert(len >= 2 and len <= 4); + return .{ .scalar = scalar, .len = len }; + } + + pub fn isVector(self: Type) bool { + return self.len > 1; + } + + pub fn element(self: Type) Type { + return .{ .scalar = self.scalar }; + } + + /// The same shape with another element type. + pub fn withScalar(self: Type, scalar: Scalar) Type { + return .{ .scalar = scalar, .len = self.len }; + } + + /// The type in memory, where a bool is a byte, as clang stores it. + pub fn memory(self: Type) Type { + return if (self.scalar == .bool) self.withScalar(.uchar) else self; + } + + /// `sizeof`: a 3-element vector takes the space of 4. + pub fn size(self: Type) u64 { + const len: u64 = if (self.len == 3) 4 else self.len; + return len * self.scalar.size(); + } + + /// `alignof`, which is the size for MSL scalars and vectors. + pub fn alignment(self: Type) u64 { + return self.size(); + } + + pub fn eql(self: Type, other: Type) bool { + return self.scalar == other.scalar and self.len == other.len; + } + + /// The MSL spelling, `uint3`. + pub fn name(self: Type) []const u8 { + const names = comptime blk: { + const scalars = @typeInfo(Scalar).@"enum".field_names; + var table: [scalars.len][4][]const u8 = undefined; + for (scalars, 0..) |scalar, i| { + table[i] = .{ scalar, scalar ++ "2", scalar ++ "3", scalar ++ "4" }; + } + break :blk table; + }; + return names[@backingInt(self.scalar)][self.len - 1]; + } + + pub fn mlirType(self: Type, ctx: *mlir.Context) *const mlir.Type { + const scalar = self.scalar.mlirType(ctx); + return if (self.isVector()) .vector(&.{self.len}, scalar) else scalar; + } +}; + +/// A field of a constant buffer struct: `float s[3]` is +/// `.{ .name = "s", .type = .float, .count = 3 }`. +pub const StructField = struct { + name: []const u8, + type: Type, + /// The array length, zero for a non-array field. + count: u32 = 0, +}; + +/// A struct passed by constant reference, laid out as MSL lays it out. +pub const Struct = struct { + name: []const u8, + fields: []const StructField, + + pub fn offset(self: Struct, index: usize) u64 { + var end: u64 = 0; + for (self.fields[0..index]) |field| { + end = std.mem.alignForward(u64, end, field.type.alignment()); + end += field.type.size() * @max(field.count, 1); + } + return std.mem.alignForward(u64, end, self.fields[index].type.alignment()); + } + + pub fn alignment(self: Struct) u64 { + var a: u64 = 1; + for (self.fields) |field| a = @max(a, field.type.alignment()); + return a; + } + + pub fn size(self: Struct) u64 { + const last = self.fields.len - 1; + const end = self.offset(last) + self.fields[last].type.size() * @max(self.fields[last].count, 1); + return std.mem.alignForward(u64, end, self.alignment()); + } + + pub fn fieldIndex(self: Struct, field_name: []const u8) usize { + for (self.fields, 0..) |field, i| if (std.mem.eql(u8, field.name, field_name)) return i; + std.debug.panic("dsl/metal: struct {s} has no field {s}", .{ self.name, field_name }); + } +}; + +test commonScalar { + try std.testing.expectEqual(Scalar.uint, commonScalar(.uint, .int)); + try std.testing.expectEqual(Scalar.int, commonScalar(.ushort, .short)); + try std.testing.expectEqual(Scalar.long, commonScalar(.long, .uint)); + try std.testing.expectEqual(Scalar.ulong, commonScalar(.ulong, .int)); + try std.testing.expectEqual(Scalar.float, commonScalar(.int, .float)); + try std.testing.expectEqual(Scalar.half, commonScalar(.half, .uint)); + try std.testing.expectEqual(Scalar.float, commonScalar(.half, .float)); +} + +test Struct { + const args: Struct = .{ .name = "A", .fields = &.{ + .{ .name = "n", .type = .int }, + .{ .name = "s", .type = .float, .count = 3 }, + .{ .name = "v", .type = .uint2 }, + .{ .name = "t", .type = .short }, + } }; + try std.testing.expectEqual(0, args.offset(0)); + try std.testing.expectEqual(4, args.offset(1)); + try std.testing.expectEqual(16, args.offset(2)); + try std.testing.expectEqual(24, args.offset(3)); + try std.testing.expectEqual(32, args.size()); + try std.testing.expectEqualStrings("bfloat4", Type.bfloat4.name()); + try std.testing.expectEqual(16, Type.float3.size()); +} diff --git a/mlir/dialects/air/AIRAttributes.cpp b/mlir/dialects/air/AIRAttributes.cpp index e7a0065d..287cb26b 100644 --- a/mlir/dialects/air/AIRAttributes.cpp +++ b/mlir/dialects/air/AIRAttributes.cpp @@ -33,11 +33,13 @@ static bool isValidName(StringRef name) { } // The size of a Metal scalar or vector, where 3-element vectors take the -// space of 4. +// space of 4. A bool vector is an i8 vector in memory, as LLVM packs i1 +// vectors into bits. static std::optional metalSize(Type type) { uint64_t elements = 1; if (auto vector = dyn_cast(type)) { - if (vector.isScalable() || vector.getRank() != 1) + if (vector.isScalable() || vector.getRank() != 1 || + vector.getElementType().isInteger(1)) return std::nullopt; elements = vector.getNumElements(); if (elements < 2 || elements > 4) @@ -46,7 +48,10 @@ static std::optional metalSize(Type type) { elements = 4; type = vector.getElementType(); } - if (!type.isIntOrFloat()) + // bool, char, short, int, long, half, bfloat and float. + if (!type.isInteger(1) && !type.isInteger(8) && !type.isInteger(16) && + !type.isInteger(32) && !type.isInteger(64) && !type.isF16() && + !type.isBF16() && !type.isF32()) return std::nullopt; return elements * llvm::divideCeil(type.getIntOrFloatBitWidth(), 8); } @@ -98,6 +103,8 @@ LogicalResult BufferAttr::verify( AddressSpace addressSpace, Access access, Type elementType, StringRef typeName, uint64_t typeSize, uint64_t typeAlignment, uint64_t bufferSize, bool noalias, ArrayRef fields) { + if (addressSpace == AddressSpace::thread) + return emitError() << "buffers are not in thread memory"; if (addressSpace == AddressSpace::constant && access != Access::read) return emitError() << "constant buffers are read-only"; if (addressSpace == AddressSpace::threadgroup && access != Access::read_write) diff --git a/mlir/dialects/air/AIRDialect.cpp b/mlir/dialects/air/AIRDialect.cpp index 78ccbba0..7ad56427 100644 --- a/mlir/dialects/air/AIRDialect.cpp +++ b/mlir/dialects/air/AIRDialect.cpp @@ -113,6 +113,8 @@ LogicalResult AIRDialect::verifyOperationAttribute(Operation *op, "air.kernel requires a function and a unit attribute"); if (failed(op->getName().verifyInvariants(op))) return failure(); + if (function.isExternal()) + return op->emitOpError("AIR kernels need a body"); if (function.getNumResults() != 0) return op->emitOpError("AIR kernels cannot return values"); if (auto llvmFunction = dyn_cast(op)) diff --git a/mlir/dialects/air/AIRDialect.td b/mlir/dialects/air/AIRDialect.td index 2f1c2e97..96264d76 100644 --- a/mlir/dialects/air/AIRDialect.td +++ b/mlir/dialects/air/AIRDialect.td @@ -44,11 +44,6 @@ def AIR_Dialect : Dialect { static constexpr ::llvm::StringLiteral getFastMathEnableAttrName() { return ::llvm::StringLiteral("air.fast_math_enable"); } - - static constexpr unsigned kPrivateAddressSpace = 0; - static constexpr unsigned kDeviceAddressSpace = 1; - static constexpr unsigned kConstantAddressSpace = 2; - static constexpr unsigned kThreadgroupAddressSpace = 3; }]; } diff --git a/mlir/dialects/air/AIREnums.td b/mlir/dialects/air/AIREnums.td index e5f9f938..61e9c25a 100644 --- a/mlir/dialects/air/AIREnums.td +++ b/mlir/dialects/air/AIREnums.td @@ -22,7 +22,8 @@ def AIR_Builtin : I32EnumAttr<"Builtin", "Kernel argument builtin", [ let genSpecializedAttr = 0; } -def AIR_AddressSpace : I32EnumAttr<"AddressSpace", "Buffer address space", [ +def AIR_AddressSpace : I32EnumAttr<"AddressSpace", "Address space", [ + I32EnumAttrCase<"thread", 0>, I32EnumAttrCase<"device", 1>, I32EnumAttrCase<"constant", 2>, I32EnumAttrCase<"threadgroup", 3>, diff --git a/mlir/dialects/air/AIROps.cpp b/mlir/dialects/air/AIROps.cpp index e1dfa3ac..85e48837 100644 --- a/mlir/dialects/air/AIROps.cpp +++ b/mlir/dialects/air/AIROps.cpp @@ -30,6 +30,10 @@ static LogicalResult verifyConvert(Operation *op, bool toOrFromInteger) { bool resultInt = isa(getElementTypeOrSelf(result)); if (!toOrFromInteger && (inputInt || resultInt || input == result)) return op->emitOpError("converts between distinct float types"); + if (!toOrFromInteger && !isa(input) && + (input.isF32() || result.isF32())) + return op->emitOpError("scalar f32 conversions are LLVM casts in Apple's " + "compiler"); if (toOrFromInteger && inputInt == resultInt) return op->emitOpError("converts between an integer and a float type"); if (resultInt && cast(op) diff --git a/mlir/dialects/air/AIROps.td b/mlir/dialects/air/AIROps.td index 4761c61e..6d072437 100644 --- a/mlir/dialects/air/AIROps.td +++ b/mlir/dialects/air/AIROps.td @@ -10,19 +10,18 @@ include "mlir/Interfaces/InferTypeOpInterface.td" include "mlir/Interfaces/SideEffectInterfaces.td" include "mlir/dialects/air/AIRAttributes.td" +class AIR_VectorOf types> + : FixedVectorOfRankAndLengthAndType<[1], [2, 3, 4], types>; class AIR_ScalarOrVectorOf types> - : AnyTypeOf])>; + : AnyTypeOf])>; def AIR_FloatLike : AIR_ScalarOrVectorOf<[F16, F32]>; def AIR_F32Like : AIR_ScalarOrVectorOf<[F32]>; -def AIR_IntLike : AIR_ScalarOrVectorOf<[I16, I32, I64]>; -def AIR_SimdIntLike : AIR_ScalarOrVectorOf<[I16, I32]>; -def AIR_ConvertLike : AIR_ScalarOrVectorOf<[F16, BF16, F32, I16, I32, I64]>; -def AIR_Matrix : VectorOfLengthAndType<[64], [F16, BF16, F32]>; -def AIR_MatrixVector : VectorOfLengthAndType<[2], [I64]> { - let builderCall = "::mlir::VectorType::get({2}, $_builder.getI64Type())"; -} +def AIR_IntLike : AIR_ScalarOrVectorOf<[I8, I16, I32, I64]>; +def AIR_SimdIntLike : AIR_ScalarOrVectorOf<[I8, I16, I32]>; +def AIR_ConvertLike : AIR_ScalarOrVectorOf<[F16, BF16, F32, I8, I16, I32, I64]>; +def AIR_Matrix : FixedVectorOfRankAndLengthAndType<[1], [64], [F16, BF16, F32]>; +def AIR_MatrixVector : FixedVectorOfShapeAndType<[2], I64>; def AIR_MatrixPointer : AnyTypeOf<[LLVM_PointerInAddressSpace<1>, LLVM_PointerInAddressSpace<3>]>; @@ -133,8 +132,7 @@ def AIR_DotOp : AIR_Op<"dot", [Pure, AllTypesMatch<["x", "y"]>, DeclareOpInterfaceMethods]> { let summary = "Dot product of two float vectors"; let description = "Calls `@air.dot.`."; - let arguments = (ins FixedVectorOfLengthAndType<[2, 3, 4], [F16, F32]>:$x, - FixedVectorOfLengthAndType<[2, 3, 4], [F16, F32]>:$y, + let arguments = (ins AIR_VectorOf<[F16, F32]>:$x, AIR_VectorOf<[F16, F32]>:$y, AIR_FastMath:$fastmath); let results = (outs AnyTypeOf<[F16, F32]>:$result); let assemblyFormat = [{ diff --git a/mlir/dialects/air/README.md b/mlir/dialects/air/README.md index c87d45f2..cd8a0531 100644 --- a/mlir/dialects/air/README.md +++ b/mlir/dialects/air/README.md @@ -44,8 +44,9 @@ Float operations carry `fastmath` flags, which become the call's flags (Apple's compiler marks every float call `fast` under fast math). SIMD-group, barrier and matrix operations have unknown effects, so they are not speculated or moved across divergent control flow. Arithmetic, memory and control flow use -the arith, llvm and scf dialects, with pointers as `!llvm.ptr` in AIR's address -spaces: 0 private, 1 device, 2 constant, 3 threadgroup. +the arith, cf, scf, vector and llvm dialects, with pointers as `!llvm.ptr` in +AIR's address spaces: 0 thread, 1 device, 2 constant, 3 threadgroup (a buffer +is never in thread memory). ## Kernels diff --git a/mlir/dialects/air/air.zig b/mlir/dialects/air/air.zig index 62195b35..2cdf2d25 100644 --- a/mlir/dialects/air/air.zig +++ b/mlir/dialects/air/air.zig @@ -28,7 +28,7 @@ pub const Builtin = enum(u32) { thread_index_in_threadgroup, }; -pub const AddressSpace = enum(u32) { device = 1, constant = 2, threadgroup = 3 }; +pub const AddressSpace = enum(u32) { thread = 0, device = 1, constant = 2, threadgroup = 3 }; pub const Access = enum(u32) { read, read_write }; diff --git a/mlir/dialects/air/test.zig b/mlir/dialects/air/test.zig index 04ef2dfb..57f0427e 100644 --- a/mlir/dialects/air/test.zig +++ b/mlir/dialects/air/test.zig @@ -194,6 +194,9 @@ test "AIR attributes use the C API" { invalid.access = .read_write; try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); invalid = device; + invalid.addressSpace = .thread; + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); + invalid = device; invalid.addressSpace = .threadgroup; try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); invalid = device; @@ -345,6 +348,28 @@ test "AIR verifier rejects types Apple's compiler has no intrinsic for" { for (rejected) |source| try std.testing.expectError(error.InvalidMlir, mlir.Module.parse(ctx, source)); } +test "AIR verifier takes the 8-bit integers Apple's compiler emits for char" { + const ctx = try testContext(); + defer ctx.deinit(); + const module = try mlir.Module.parse(ctx, + \\func.func @f(%c: i8, %v: vector<2xi8>, %f: f32, %w: vector<2xf32>, %l: i16) { + \\ %0 = air.convert.s %c fastmath : i8 to f32 + \\ %1 = air.convert.u %f : f32 to i8 + \\ %2 = air.convert.s %v fastmath : vector<2xi8> to vector<2xf32> + \\ %3 = air.convert.s %w : vector<2xf32> to vector<2xi8> + \\ %4 = air.max.s %c, %c : i8 + \\ %5 = air.min.u %v, %v : vector<2xi8> + \\ %6 = air.abs.s %c : i8 + \\ %7 = air.clamp.u %c, %c, %c : i8 + \\ %8 = air.simd_shuffle.s %c, %l : i8 + \\ %9 = air.simd_sum.s %c : i8 + \\ %10 = air.simd_max.u %c : i8 + \\ return + \\} + ); + module.deinit(); +} + test "AIR kernel ABI" { const ctx = try testContext(); defer ctx.deinit(); diff --git a/mlir/dialects/arith/arith.zig b/mlir/dialects/arith/arith.zig index 8f7f6159..c45ce5b5 100644 --- a/mlir/dialects/arith/arith.zig +++ b/mlir/dialects/arith/arith.zig @@ -128,6 +128,12 @@ pub const IntegerOverflowAttr = opaque { } }; +/// `overflowFlags = #arith.overflow`: the overflow flags of an +/// integer operation. +pub fn overflowFlags(ctx: *mlir.Context, flags: IntegerOverflowFlags) mlir.Error!mlir.NamedAttribute { + return .named(ctx, "overflowFlags", (try IntegerOverflowAttr.get(ctx, .{ .value = flags })).attribute()); +} + // ============================================================================= // Constants // ============================================================================= diff --git a/mlir/dialects/arith/test.zig b/mlir/dialects/arith/test.zig index 85dab738..844898ca 100644 --- a/mlir/dialects/arith/test.zig +++ b/mlir/dialects/arith/test.zig @@ -103,6 +103,15 @@ test "fastmath names the attribute of a float operation" { try std.testing.expectEqualStrings("{fastmath = #arith.fastmath}", w.buffered()); } +test "overflowFlags names the attribute of an integer operation" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + var buf: [64]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{try arith.overflowFlags(ctx, .{ .nsw = true })})}); + try std.testing.expectEqualStrings("{overflowFlags = #arith.overflow}", w.buffered()); +} + test "arith overflow attribute is built through the C API" { const ctx = try mlir.Context.init(.{}); defer ctx.deinit(); diff --git a/mlir/dialects/llvm/llvm.zig b/mlir/dialects/llvm/llvm.zig index 22c8539f..78524e0a 100644 --- a/mlir/dialects/llvm/llvm.zig +++ b/mlir/dialects/llvm/llvm.zig @@ -11,6 +11,14 @@ pub const LoopAnnotationAttr = ops.LoopAnnotationAttr; pub const Linkage = ops.Linkage; pub const linkageAttr = ops.linkageAttr; pub const emitCInterface = ops.emitCInterface; +pub const linkage = ops.linkage; +pub const inlineHint = ops.inlineHint; +pub const noundef = ops.noundef; +pub const zeroext = ops.zeroext; +pub const signext = ops.signext; +pub const alignment = ops.alignment; +pub const dereferenceable = ops.dereferenceable; +pub const elementtype = ops.elementtype; pub const AtomicBinOp = ops.AtomicBinOp; pub const AtomicOrdering = ops.AtomicOrdering; pub const inttoptr = ops.inttoptr; @@ -26,6 +34,17 @@ pub const getelementptr = ops.getelementptr; pub const alloca = ops.alloca; pub const atomicrmw = ops.atomicrmw; pub const intr_assume = ops.intr_assume; +pub const intr_lifetime_start = ops.intr_lifetime_start; +pub const intr_lifetime_end = ops.intr_lifetime_end; +pub const extractelement = ops.extractelement; +pub const insertelement = ops.insertelement; +pub const shufflevector = ops.shufflevector; +pub const mlir_undef = ops.mlir_undef; +pub const mlir_poison = ops.mlir_poison; +pub const freeze = ops.freeze; +pub const BrOpts = ops.BrOpts; +pub const br = ops.br; +pub const unreachable_ = ops.unreachable_; pub const call_intrinsic = ops.call_intrinsic; pub const InlineAsmOpts = ops.InlineAsmOpts; pub const inline_asm = ops.inline_asm; diff --git a/mlir/dialects/llvm/ops.zig b/mlir/dialects/llvm/ops.zig index 731d2f55..2102f68a 100644 --- a/mlir/dialects/llvm/ops.zig +++ b/mlir/dialects/llvm/ops.zig @@ -104,9 +104,9 @@ pub const Linkage = enum(c_uint) { /// `#llvm.linkage`; `llvm.linkage` on a `func.func` sets its /// linkage once lowered to LLVM. Loads the LLVM dialect into `ctx`, which /// must have it registered. -pub fn linkageAttr(ctx: *mlir.Context, linkage: Linkage) *const mlir.Attribute { +pub fn linkageAttr(ctx: *mlir.Context, value: Linkage) *const mlir.Attribute { _ = c.mlirContextGetOrLoadDialect(ctx.ptr(), mlir.stringRef("llvm")); - return @ptrCast(c.mlirLLVMLinkageAttrGet(ctx.ptr(), @backingInt(linkage)).ptr); + return @ptrCast(c.mlirLLVMLinkageAttrGet(ctx.ptr(), @backingInt(value)).ptr); } /// `llvm.emit_c_interface`: has the lowering of a `func.func` emit a @@ -115,6 +115,48 @@ pub fn emitCInterface(ctx: *mlir.Context) mlir.NamedAttribute { return .named(ctx, "llvm.emit_c_interface", .unit(ctx)); } +/// `llvm.linkage = #llvm.linkage` on a `func.func`. +pub fn linkage(ctx: *mlir.Context, value: Linkage) mlir.NamedAttribute { + return .named(ctx, "llvm.linkage", linkageAttr(ctx, value)); +} + +/// `llvm.inline_hint` on a `func.func`: `inlinehint` once lowered. +pub fn inlineHint(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "llvm.inline_hint", .unit(ctx)); +} + +/// `llvm.noundef` on an argument: never undef or poison. +pub fn noundef(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "llvm.noundef", .unit(ctx)); +} + +/// `llvm.zeroext` on an argument: a narrow integer passed zero-extended. +pub fn zeroext(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "llvm.zeroext", .unit(ctx)); +} + +/// `llvm.signext` on an argument: a narrow integer passed sign-extended. +pub fn signext(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "llvm.signext", .unit(ctx)); +} + +/// `llvm.align = bytes` on a pointer argument. +pub fn alignment(ctx: *mlir.Context, bytes: u64) mlir.NamedAttribute { + return .named(ctx, "llvm.align", .int(ctx, .i64, @as(i64, @intCast(bytes)))); +} + +/// `llvm.dereferenceable = bytes` on a pointer argument: that many bytes are +/// readable. +pub fn dereferenceable(ctx: *mlir.Context, bytes: u64) mlir.NamedAttribute { + return .named(ctx, "llvm.dereferenceable", .int(ctx, .i64, @as(i64, @intCast(bytes)))); +} + +/// `llvm.elementtype = type` on a pointer argument: its pointee, which +/// typed-pointer IR prints. +pub fn elementtype(ctx: *mlir.Context, pointee: *const mlir.Type) mlir.NamedAttribute { + return .named(ctx, "llvm.elementtype", .typeAttr(pointee)); +} + /// The operation of `llvm.atomicrmw`, as the LLVM dialect numbers it. pub const AtomicBinOp = enum(i64) { xchg = 0, @@ -240,8 +282,8 @@ pub const LoadOpts = struct { /// llvm.load — a value of type `result_type` through `pointer`. pub fn load(ctx: *mlir.Context, pointer: *const mlir.Value, result_type: *const mlir.Type, opts: LoadOpts, location: *const mlir.Location) *mlir.Operation { var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.alignment) |alignment| { - attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, alignment))); + if (opts.alignment) |bytes| { + attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, bytes))); } if (opts.invariant) { attrs.appendAssumeCapacity(.named(ctx, "invariant", .unit(ctx))); @@ -262,8 +304,8 @@ pub const StoreOpts = struct { /// llvm.store — `value` through `pointer`. pub fn store(ctx: *mlir.Context, value: *const mlir.Value, pointer: *const mlir.Value, opts: StoreOpts, location: *const mlir.Location) *mlir.Operation { var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.alignment) |alignment| { - attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, alignment))); + if (opts.alignment) |bytes| { + attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, bytes))); } return mlir.Operation.make(ctx, "llvm.store", .{ .operands = .{ .flat = &.{ value, pointer } }, @@ -323,7 +365,7 @@ pub fn alloca( ) *mlir.Operation { var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; attrs.appendAssumeCapacity(.named(ctx, "elem_type", .typeAttr(elem_type))); - if (opts.alignment) |alignment| attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, alignment))); + if (opts.alignment) |bytes| attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, bytes))); return mlir.Operation.make(ctx, "llvm.alloca", .{ .operands = .{ .flat = &.{array_size} }, .results = .{ .flat = &.{result_type} }, @@ -354,7 +396,7 @@ pub fn atomicrmw( .named(ctx, "ordering", .int(ctx, .i64, @backingInt(ordering))), }); if (opts.syncscope) |syncscope| attrs.appendAssumeCapacity(.named(ctx, "syncscope", .string(ctx, syncscope))); - if (opts.alignment) |alignment| attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, alignment))); + if (opts.alignment) |bytes| attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, bytes))); return mlir.Operation.make(ctx, "llvm.atomicrmw", .{ .operands = .{ .flat = &.{ ptr, val } }, .results = .{ .flat = &.{val.type_()} }, @@ -379,6 +421,87 @@ pub fn intr_assume(ctx: *mlir.Context, cond: *const mlir.Value, location: *const }); } +pub fn intr_lifetime_start(ctx: *mlir.Context, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.intr.lifetime.start", .{ + .operands = .{ .flat = &.{ptr} }, + .location = location, + }); +} + +pub fn intr_lifetime_end(ctx: *mlir.Context, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.intr.lifetime.end", .{ + .operands = .{ .flat = &.{ptr} }, + .location = location, + }); +} + +pub fn extractelement(ctx: *mlir.Context, vector: *const mlir.Value, position: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.extractelement", .{ + .operands = .{ .flat = &.{ vector, position } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn insertelement(ctx: *mlir.Context, vector: *const mlir.Value, value: *const mlir.Value, position: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.insertelement", .{ + .operands = .{ .flat = &.{ vector, value, position } }, + .results = .{ .flat = &.{vector.type_()} }, + .location = location, + }); +} + +pub fn shufflevector(ctx: *mlir.Context, v1: *const mlir.Value, v2: *const mlir.Value, result_type: *const mlir.Type, mask: []const i32, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.shufflevector", .{ + .operands = .{ .flat = &.{ v1, v2 } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{.named(ctx, "mask", .denseArray(ctx, .i32, mask))}, + .location = location, + }); +} + +pub fn mlir_undef(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.mlir.undef", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn mlir_poison(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.mlir.poison", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn freeze(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.freeze", .{ + .operands = .{ .flat = &.{value} }, + .results = .{ .flat = &.{value.type_()} }, + .location = location, + }); +} + +pub const BrOpts = struct { + loop_annotation: ?*const LoopAnnotationAttr = null, +}; + +/// `llvm.br`. Not verified: a terminator only verifies inside its block. +pub fn br(ctx: *mlir.Context, dest: *mlir.Block, dest_operands: []const *const mlir.Value, opts: BrOpts, location: *const mlir.Location) *mlir.Operation { + var state: mlir.OperationState = .init("llvm.br", location); + state.addOperands(dest_operands); + var successors: [1]*const mlir.Block = .{dest}; + state.addSuccessors(&successors); + if (opts.loop_annotation) |annotation| state.addAttributes(&.{.named(ctx, "loop_annotation", annotation.attribute())}); + return mlir.Operation.init(&state) catch @panic("llvm.br: failed to create the operation"); +} + +/// `llvm.unreachable`. Not verified: a terminator only verifies inside its +/// block. +pub fn unreachable_(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.unreachable", .{ .location = location, .verify = false }); +} + /// llvm.call_intrinsic — call the LLVM intrinsic `name` (e.g. `llvm.nvvm.*`), with an /// optional result of type `result_type`. pub fn call_intrinsic( diff --git a/mlir/dialects/llvm/test.zig b/mlir/dialects/llvm/test.zig index b4f09648..28541d44 100644 --- a/mlir/dialects/llvm/test.zig +++ b/mlir/dialects/llvm/test.zig @@ -7,15 +7,25 @@ const operation_names: []const []const u8 = &.{ "llvm.alloca", "llvm.atomicrmw", "llvm.bitcast", + "llvm.br", "llvm.call_intrinsic", + "llvm.extractelement", "llvm.extractvalue", "llvm.fmul", + "llvm.freeze", "llvm.getelementptr", "llvm.inline_asm", + "llvm.insertelement", "llvm.intr.assume", + "llvm.intr.lifetime.end", + "llvm.intr.lifetime.start", "llvm.inttoptr", "llvm.load", + "llvm.mlir.poison", + "llvm.mlir.undef", + "llvm.shufflevector", "llvm.store", + "llvm.unreachable", }; test { @@ -104,6 +114,66 @@ test "extractvalue, atomicrmw and intr.assume build verified operations" { try std.testing.expect(llvm.intr_assume(ctx, block.argument(2), loc).appendTo(block).verify()); } +test "vector, freeze and lifetime operations build verified operations" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + const loc: *const mlir.Location = .unknown(ctx); + const i64x2: *const mlir.Type = .vector(&.{2}, .int(ctx, .i64)); + const block = mlir.Block.init(&.{ i64x2, .int(ctx, .i64), .int(ctx, .i32), llvm.pointerType(ctx, llvm.NVPTXAddressSpace.generic) }, &.{ loc, loc, loc, loc }); + defer block.deinit(); + const v = block.argument(0); + const inserted = llvm.insertelement(ctx, v, block.argument(1), block.argument(2), loc).appendTo(block); + try std.testing.expect(inserted.verify()); + try std.testing.expect(inserted.result(0).type_().eql(i64x2)); + try std.testing.expect(llvm.extractelement(ctx, v, block.argument(2), .int(ctx, .i64), loc).appendTo(block).verify()); + const poison = llvm.mlir_poison(ctx, i64x2, loc).appendTo(block); + try std.testing.expect(poison.verify()); + try std.testing.expect(llvm.mlir_undef(ctx, i64x2, loc).appendTo(block).verify()); + try std.testing.expect(llvm.shufflevector(ctx, v, poison.result(0), i64x2, &.{ 1, 0 }, loc).appendTo(block).verify()); + try std.testing.expect(llvm.freeze(ctx, v, loc).appendTo(block).verify()); + try std.testing.expect(llvm.intr_lifetime_start(ctx, block.argument(3), loc).appendTo(block).verify()); + try std.testing.expect(llvm.intr_lifetime_end(ctx, block.argument(3), loc).appendTo(block).verify()); +} + +test "br carries its loop annotation" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + const loc: *const mlir.Location = .unknown(ctx); + const dest = mlir.Block.init(&.{}, &.{}); + defer dest.deinit(); + const annotation = llvm.LoopAnnotationAttr.get(ctx, .{ .mustProgress = true }); + const op = llvm.br(ctx, dest, &.{}, .{ .loop_annotation = annotation }, loc); + defer op.deinit(); + try std.testing.expect(op.attributeByName("loop_annotation").?.eql(annotation.attribute())); + const end = llvm.unreachable_(ctx, loc); + defer end.deinit(); +} + +test "argument and function attributes are named by the LLVM dialect" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + var buf: [512]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{ + llvm.noundef(ctx), llvm.zeroext(ctx), llvm.signext(ctx), + llvm.alignment(ctx, 16), llvm.dereferenceable(ctx, 64), llvm.elementtype(ctx, .float(ctx, .f32)), + llvm.linkage(ctx, .internal), llvm.inlineHint(ctx), + })}); + try std.testing.expectEqualStrings("{llvm.align = 16 : i64, llvm.dereferenceable = 64 : i64, llvm.elementtype = f32, llvm.inline_hint, llvm.linkage = #llvm.linkage, llvm.noundef, llvm.signext, llvm.zeroext}", w.buffered()); +} + test "emit_c_interface is a unit attribute of the LLVM dialect" { const ctx = try mlir.Context.init(.{}); defer ctx.deinit(); diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 7e2e227c..cee2a1f3 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -42,22 +42,22 @@ def _cpu_plugin_impl(mctx): http_archive( name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "309b737f996024b98b1ee18a44e37df1b328e7c1502a98f5829c10120eb35d63", - url = "https://mirror.zml.ai/plugins/202610231457.291.1.799ad5c39a34/zml-cpu-linux-amd64.tar.zst", + sha256 = "3d6a6d1b047a0943b50fd9e63e5f6dff6b7cf184b186c0cc1052de40f0dbec92", + url = "https://mirror.zml.ai/plugins/202610281508.306.1.6bf391c563e0/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "bd4a67d94401022b1d5ee0677e427d6ffe5140fa57cddfafb8189a5743565662", - url = "https://mirror.zml.ai/plugins/202610231457.291.1.799ad5c39a34/zml-cpu-darwin-amd64.tar.zst", + sha256 = "d6b4b15b13a2d1a3e0b84f143f568cd572bef5db91784f9f7a826173ed626c7e", + url = "https://mirror.zml.ai/plugins/202610281508.306.1.6bf391c563e0/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "6be93f4527ca7634904ca4eaa4c2a197f2d8f017f89c26cd59ffc8753ed5d248", - url = "https://mirror.zml.ai/plugins/202610231457.291.1.799ad5c39a34/zml-cpu-darwin-arm64.tar.zst", + sha256 = "c06babc001edeab313bc63daab3aa3d40e087266a3a29634d182628fb65fad14", + url = "https://mirror.zml.ai/plugins/202610281508.306.1.6bf391c563e0/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 38dc796c..8a5bb169 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -224,12 +224,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "2528d0612a9503ada8b5a46dbe70e52ae0c21bce4ae11f92c9e3bf8f83045cff", - "url": "https://mirror.zml.ai/plugins/202610231457.291.1.799ad5c39a34/zml-cuda-linux-amd64.tar.zst", + "sha256": "41777a3f65755133562ca31f83fadd2352f220dfa8715ad10c2b8f57e90a071a", + "url": "https://mirror.zml.ai/plugins/202610281508.306.1.6bf391c563e0/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "9926b0a66a2e0c95d0c202f3cc82fb4321a6da2cfe903204f353df8e0209c288", - "url": "https://mirror.zml.ai/plugins/202610231457.291.1.799ad5c39a34/zml-cuda-linux-arm64.tar.zst", + "sha256": "de9adf212b10345e94f45889d49daaa15eb4bd0297007028e74c7081faaaa865", + "url": "https://mirror.zml.ai/plugins/202610281508.306.1.6bf391c563e0/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 303b3843..e6f21f06 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "1493fb9a9a2c1e217aa67914a982825cdc298fcb246d406e4e2ea217c66cbeec", - url = "https://mirror.zml.ai/plugins/202610231457.291.1.799ad5c39a34/zml-metal-darwin-arm64.tar.zst", + sha256 = "3c7b5fc6f24a6964dcf68f06e61a616ce50d7830fd0e96a06d9bd4eecf256bc8", + url = "https://mirror.zml.ai/plugins/202610281508.306.1.6bf391c563e0/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 193ad367..dce411da 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610231457.291.1.799ad5c39a34/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "f9c75c7b9b08a7022b8039d02ebf7bb635ed1a65335969f3a0b241d09b2170b8" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610281508.306.1.6bf391c563e0/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "d89325846766f8630164b3b197a28a2388964032919953433cc22608caa3dd51" def _rocm_package_name(name): return name + _ROCM_VERSION diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 41ae31c1..8188c743 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -150,6 +150,7 @@ zig_library( "//dsl/cuda_tile:builder", "//dsl/cute:builder", "//dsl/fly:builder", + "//dsl/metal:builder", "//dsl/mosaic_tpu:builder", "//dsl/tcl:builder", "//dsl/triton:builder", diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 5de801e3..3daba2ec 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -973,7 +973,7 @@ fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.Load try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_latency_hiding_scheduler", true, upb_arena); }, .metal => { - try setXlaOverrideFlag(overrides_map, "xla_gpu_metal_fast_math", false, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_ftz", true, upb_arena); }, .oneapi => { // More efficient for the allgather/broadcast implementation of the collective permute. diff --git a/zml/kernel.zig b/zml/kernel.zig index d8601760..03a64d70 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -4,6 +4,7 @@ const cuda_tile_builder = @import("dsl/cuda_tile/builder"); const cute_builder = @import("dsl/cute/builder"); const fly_builder = @import("dsl/fly/builder"); const func = @import("mlir/dialects/func"); +const metal_builder = @import("dsl/metal/builder"); const mlir = @import("mlir"); const mosaic_tpu_builder = @import("dsl/mosaic_tpu/builder"); const mosaic_tpu_dialect = @import("mlir/dialects/mosaic_tpu"); @@ -854,6 +855,186 @@ pub const tcl = struct { } }; +pub const metal = struct { + pub const Builder = metal_builder.Builder; + pub const Value = metal_builder.Value; + pub const Type = metal_builder.Type; + pub const Struct = metal_builder.Struct; + pub const ArgSpec = metal_builder.ArgSpec; + pub const LoopOpts = metal_builder.LoopOpts; + pub const FinishError = metal_builder.FinishError; + + pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(metal_builder.registerDialects); + } + + pub fn from(dt: DataType) Type { + return switch (dt) { + .bool => .bool, + .i8 => .char, + .i16 => .short, + .i32 => .int, + .i64 => .long, + .u8 => .uchar, + .u16 => .ushort, + .u32 => .uint, + .u64 => .ulong, + .f16 => .half, + .bf16 => .bfloat, + .f32 => .float, + else => std.debug.panic("zml.kernel.metal.from: dtype {s} has no MSL equivalent", .{@tagName(dt)}), + }; + } + + /// The bytes of a `constant T &` argument: `.of(@as(i32, n))`, or a host + /// `extern struct` laid out like a `constant S &` argument. + pub const Constant = struct { + buffer: [256]u8 = undefined, + len: usize = 0, + + pub fn of(value: anytype) Constant { + const raw = std.mem.asBytes(&value); + var c: Constant = .{ .len = raw.len }; + @memcpy(c.buffer[0..raw.len], raw); + return c; + } + + pub fn bytes(self: *const Constant) []const u8 { + return self.buffer[0..self.len]; + } + }; + + fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + /// The arguments bound to input tensors. + inputs: []const [:0]const u8, + /// The arguments bound to output tensors; one named like an + /// input updates it in place. + outputs: []const [:0]const u8, + /// The `constant` arguments given by value at the call. + constants: []const [:0]const u8 = &.{}, + run: *const fn (*Builder, Config) FinishError!void, + fast_math: bool = true, + }; + } + + /// A Metal kernel: `run` declares its arguments with `declareArgs`, by + /// name, and builds its body. + pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), + ) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + pub const Constants = StructOf(spec.constants, Constant); + + pub const CallOpts = struct { + cfg: ConfigT, + /// Threadgroups per grid. + grid: [3]u32, + /// Threads per threadgroup. + threadgroup: [3]u32, + /// Bytes of `[[threadgroup(0)]]` memory. + threadgroup_memory: u64 = 0, + constants: Constants = std.mem.zeroes(Constants), + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + var arguments: std.ArrayList(metal_builder.Argument) = .empty; + defer arguments.deinit(allocator); + return build(allocator, cfg, &arguments); + } + + fn build(allocator: std.mem.Allocator, cfg: ConfigT, arguments: *std.ArrayList(metal_builder.Argument)) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + var b = Builder.open(allocator, ctx, name, .{ .fast_math = spec.fast_math }); + defer b.deinit(); + try spec.run(&b, cfg); + const ir = try b.finish(); + errdefer allocator.free(ir); + try arguments.appendSlice(allocator, b.arguments.items); + return ir; + } + + fn indexOf(names: []const [:0]const u8, arg: []const u8) ?usize { + for (names, 0..) |n, i| if (std.mem.eql(u8, n, arg)) return i; + return null; + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + const allocator = cur.allocator; + + var arguments: std.ArrayList(metal_builder.Argument) = .empty; + defer arguments.deinit(allocator); + const ir = build(allocator, opts.cfg, &arguments) catch |err| + std.debug.panic("zml.kernel.metal.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |field, i| inputs_arr[i] = @field(inputs, field); + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |field, i| outputs_arr[i] = @field(outputs, field); + var constants: [spec.constants.len][]const u8 = undefined; + inline for (spec.constants, 0..) |field, i| constants[i] = @field(opts.constants, field).bytes(); + + var bound: [spec.outputs.len]bool = @splat(false); + var bound_inputs: [spec.inputs.len]bool = @splat(false); + var bound_constants: [spec.constants.len]bool = @splat(false); + var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; + var bindings: std.ArrayList(i32) = .empty; + defer bindings.deinit(allocator); + for (arguments.items) |arg| { + if (arg.kind == .threadgroup) { + // `threadgroup_memory` sizes `[[threadgroup(0)]]` alone. + if (arg.index != 0) std.debug.panic("zml.kernel.metal.Kernel({s}): threadgroup argument {s} is [[threadgroup({d})]], and only [[threadgroup(0)]] gets memory", .{ name, arg.name, arg.index }); + continue; + } + std.debug.assert(arg.index == bindings.items.len); + const input = indexOf(spec.inputs, arg.name); + const output = indexOf(spec.outputs, arg.name); + if (input) |i| bound_inputs[i] = true; + if (output) |o| bound[o] = true; + const binding: i32 = if (input) |i| blk: { + if (output) |o| aliases.appendAssumeCapacity(.{ .output_index = @intCast(o), .operand_index = @intCast(i) }); + break :blk @intCast(i); + } else if (output) |o| + @intCast(spec.inputs.len + o) + else if (indexOf(spec.constants, arg.name)) |k| blk: { + if (constants[k].len != arg.size) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} has {d} bytes, the argument {d}", .{ name, arg.name, constants[k].len, arg.size }); + bound_constants[k] = true; + break :blk -1 - @as(i32, @intCast(k)); + } else std.debug.panic("zml.kernel.metal.Kernel({s}): argument {s} is no input, output or constant", .{ name, arg.name }); + bindings.append(allocator, binding) catch @panic("OOM"); + } + for (bound, spec.outputs) |b, output| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): output {s} is no argument", .{ name, output }); + for (bound_inputs, spec.inputs) |b, input| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): input {s} is no argument", .{ name, input }); + for (bound_constants, spec.constants) |b, constant| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} is no argument", .{ name, constant }); + + const tensor_results = ops.metal(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .threadgroup = opts.threadgroup, + .threadgroup_memory = opts.threadgroup_memory, + .arguments = bindings.items, + .constants = &constants, + .output_operand_aliases = aliases.constSlice(), + }); + var results: Results = undefined; + inline for (spec.outputs, 0..) |field, i| @field(results, field) = tensor_results[i]; + return results; + } + }; + } +}; + pub const fly = struct { /// Threads per wavefront: 64 on CDNA, 32 on RDNA. pub fn waveSize(p: *const platform_.Platform) i32 { @@ -1363,3 +1544,71 @@ test "tcl multi-chip kernels run on two furiosa chips" { for (host.items(f32), 0..) |v, i| try std.testing.expectApproxEqRel(@exp(hx[i]), v, 1e-4); } } + +/// `out[i] = a[i] * scale + b[i]` over `n` floats. +const MetalAxpy = struct { + const Cfg = struct {}; + const K = metal.Kernel(Cfg, .{ + .name = "axpy", + .inputs = &.{ "a", "b" }, + .outputs = &.{"out"}, + .constants = &.{ "scale", "n" }, + .run = run, + }); + + fn run(b: *metal.Builder, _: Cfg) metal.FinishError!void { + const a = try b.declareArgs(.{ + .a = .{ .buffer = .{ .type = .float, .const_ = true } }, + .b = .{ .buffer = .{ .type = .float, .const_ = true } }, + .out = .{ .buffer = .{ .type = .float } }, + .scale = .{ .constant = .float }, + .n = .{ .constant = .uint }, + .tid = .{ .thread_position_in_grid = .uint }, + }); + b.returnIf(a.tid.ge(a.n)); + a.out.store(a.tid, b.fma(a.a.load(a.tid), a.scale, a.b.load(a.tid))); + } +}; + +test "metal kernels run on metal" { + const zml = @import("zml.zig"); + const allocator = std.testing.allocator; + + const ir = try MetalAxpy.K.emit(allocator, .{}); + defer allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "air.fma") != null); + + const platform = zml.testing.env(); + if (platform.target != .metal) return error.SkipZigTest; + + const n = 1000; + const Mod = struct { + pub fn forward(x: Tensor, y: Tensor) Tensor { + return MetalAxpy.K.call(.{ .a = x, .b = y }, .{ .out = x.shape() }, .{ + .cfg = .{}, + .grid = .{ (n + 255) / 256, 1, 1 }, + .threadgroup = .{ 256, 1, 1 }, + .constants = .{ .scale = .of(@as(f32, 0.5)), .n = .of(@as(u32, n)) }, + }).out; + } + }; + const x: Tensor = .init(.{n}, .f32); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, Tensor.init(.{n}, .f32) }, platform, .{}); + defer exe.deinit(); + + var hx: [n]f32 = undefined; + var hy: [n]f32 = undefined; + for (&hx, &hy, 0..) |*a, *b, i| { + a.* = @floatFromInt(i); + b.* = 1; + } + var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hx)); + defer bx.deinit(); + var by: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hy)); + defer by.deinit(); + var result = try exe.eval(allocator, std.testing.io, .{ bx, by }); + defer result.deinit(); + var host = try result.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + for (host.items(f32), hx, hy) |out, a, b| try std.testing.expectEqual(a * 0.5 + b, out); +} diff --git a/zml/ops.zig b/zml/ops.zig index 6622ac19..b1a26d23 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1360,6 +1360,69 @@ pub fn tcl(inputs: anytype, outputs: anytype, opts: TclOps) [outputs.len]Tensor return outputs_; } +pub const MetalOps = struct { + /// The kernel symbol. + name: []const u8, + /// The kernel's module, in the AIR dialect. + ir: []const u8, + /// Threadgroups per grid. + grid: [3]u32, + /// Threads per threadgroup. + threadgroup: [3]u32, + /// Bytes of `[[threadgroup(0)]]` memory. + threadgroup_memory: u64 = 0, + /// What each `[[buffer(n)]]` binds: `k >= 0` the XLA buffer `k` (the + /// operands, then the results), `k < 0` the constant `-1 - k`. + arguments: []const i32, + /// The bytes of the constant arguments. + constants: []const []const u8 = &.{}, + output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, +}; + +/// A `__gpu$xla.gpu.air` custom call: XLA lowers `ir` to AIR, compiles it to +/// a metallib and launches `name`. +pub fn metal(inputs: anytype, outputs: anytype, opts: MetalOps) [outputs.len]Tensor { + const compiler = Compiler.current(); + const mlir_ctx = compiler.mlir_ctx; + + var values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| values[i] = inputs[i].value(); + + var res_types: [outputs.len]*const mlir.Type = undefined; + inline for (outputs, 0..) |output, i| res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); + + const allocator = compiler.allocator; + const constant_attrs = allocator.alloc(*const mlir.Attribute, opts.constants.len) catch @panic("OOM"); + defer allocator.free(constant_attrs); + for (opts.constants, constant_attrs) |bytes, *attribute| attribute.* = .denseArray(mlir_ctx, .i8, @as([]const i8, @ptrCast(bytes))); + + const sizes = [2][3]u32{ opts.grid, opts.threadgroup }; + var dims: [2][3]*const mlir.Attribute = undefined; + for (&dims, sizes) |*elems, dim| { + for (elems, dim) |*e, size| e.* = .int(mlir_ctx, .i32, size); + } + const backend_config: *const mlir.Attribute = .dict(mlir_ctx, &.{ + .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), + .named(mlir_ctx, "ir", .string(mlir_ctx, opts.ir)), + .named(mlir_ctx, "grid", .array(mlir_ctx, &dims[0])), + .named(mlir_ctx, "threadgroup", .array(mlir_ctx, &dims[1])), + .named(mlir_ctx, "threadgroup_memory", .int(mlir_ctx, .i64, @as(i64, @intCast(opts.threadgroup_memory)))), + .named(mlir_ctx, "arguments", .denseArray(mlir_ctx, .i32, opts.arguments)), + .named(mlir_ctx, "constants", .array(mlir_ctx, constant_attrs)), + }); + + const op = stablehlo.custom_call(mlir_ctx, &values, &res_types, .{ + .call_target_name = "__gpu$xla.gpu.air", + .backend_config = .{ .typed_ffi = backend_config }, + .has_side_effect = false, + .output_operand_aliases = opts.output_operand_aliases, + }, compiler.location).appendTo(compiler.currentScope().block); + + var outputs_: [outputs.len]Tensor = undefined; + inline for (outputs, 0..) |output, i| outputs_[i] = Tensor._result(output, op.result(i)); + return outputs_; +} + pub const CuteOps = struct { /// The kernel symbol. name: []const u8, From e4e031b2c59562d655f1e109fdbcfc855a56a8f3 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Thu, 29 Oct 2026 12:52:20 +0100 Subject: [PATCH 831/881] zml/kernels/paged_attention/metal: port the kernels to the Metal DSL (#999) Moves Metal paged attention from the `zml$paged_attn` custom call to Metal DSL ports of the same kernels. - `fa_vec_paged`, with a split-K reduce, handles causal bf16 decode. `paged_attention_tiled` and `zero_padding_rows` handle everything else. - The shared paged attention test now runs on Metal. Triton counts as available only on CUDA, ROCm and oneAPI. StableHLO is the reference on Metal for the causal case. --- zml/kernels/paged_attention/metal/BUILD.bazel | 5 +- zml/kernels/paged_attention/metal/README.md | 20 +- zml/kernels/paged_attention/metal/metal.zig | 154 ++- .../paged_attention/metal/paged_attn.zig | 1003 +++++++++++++++++ .../paged_attention/paged_attention.zig | 18 +- 5 files changed, 1175 insertions(+), 25 deletions(-) create mode 100644 zml/kernels/paged_attention/metal/paged_attn.zig diff --git a/zml/kernels/paged_attention/metal/BUILD.bazel b/zml/kernels/paged_attention/metal/BUILD.bazel index 855bdcca..39d92259 100644 --- a/zml/kernels/paged_attention/metal/BUILD.bazel +++ b/zml/kernels/paged_attention/metal/BUILD.bazel @@ -2,5 +2,8 @@ package(default_visibility = ["//zml:__pkg__"]) filegroup( name = "metal", - srcs = ["metal.zig"], + srcs = [ + "metal.zig", + "paged_attn.zig", + ], ) diff --git a/zml/kernels/paged_attention/metal/README.md b/zml/kernels/paged_attention/metal/README.md index 8e3e5d6e..5624de0e 100644 --- a/zml/kernels/paged_attention/metal/README.md +++ b/zml/kernels/paged_attention/metal/README.md @@ -5,20 +5,22 @@ This backend is automatically selected for Metal. See the metadata, and page addressing contract. [The wrapper](metal.zig) requires split KV caches. It merges query head groups, -transposes queries and caches into the custom call's layouts, invokes -`zml$paged_attn`, and restores the query's tagged layout for the output. +transposes queries and caches into the kernels' layouts, launches the +[Metal DSL kernels](paged_attn.zig), and restores the query's tagged layout for +the output. + +Queries and caches are `bf16` or `f16`; other dtypes panic. Causal `bf16` +decode with head dimension 128, 256 or 512 runs `fa_vec_paged`, which is `bf16` +only, split over four threadgroups and merged by `fa_vec_paged_reduce` at head +dimension 512 past 4096 KV positions. Everything else, `f16` decode included, +runs `paged_attention_tiled` (head dimension 32, 64, 96, 128, 256 or 512, +pages of 8, 16 or 32 positions), then `zero_padding_rows`. Causal and non-causal standard masks, left sliding windows, and a custom scale -are passed to the custom call. Explicit key indices are rejected, and +are passed to the kernels. Explicit key indices are rejected, and `opts.sink` is currently ignored. Fused caches are rejected by ordinary paged attention. When sharding heads, keep complete GQA groups together with matching query and cache head ownership. Each local computation needs the full required key range, features, and matching sequence metadata. - -The shared tests in [paged_attention.zig](../paged_attention.zig) compare this -backend against StableHLO for causal attention and include option-specific -comparisons. They exercise `bf16`, mixed prefill/decode, shared prefix pages, -partial final KV pages, and trailing inactive sequences. These cases do not -establish support for every shape or dtype. diff --git a/zml/kernels/paged_attention/metal/metal.zig b/zml/kernels/paged_attention/metal/metal.zig index b337638a..e2d68398 100644 --- a/zml/kernels/paged_attention/metal/metal.zig +++ b/zml/kernels/paged_attention/metal/metal.zig @@ -5,9 +5,14 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; +const paged_attn = @import("paged_attn.zig"); const log = std.log.scoped(.@"zml/attention/metal"); +test { + _ = paged_attn; +} + pub const Options = struct { batch_size: usize, max_num_pages: usize, @@ -64,18 +69,145 @@ pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor const sliding_window: i32 = if (opts.mask.slidingWindow()) |window| @intCast(window) else -1; - const out = zml.ops.customCall( - "zml$paged_attn", - .{ qh, kc, vc, parameters.block_table, parameters.seq_lens, parameters.query_start_len }, - qh.shape(), - .{ - .is_causal = opts.mask.isCausal(), - .sliding_window = sliding_window, - .scale = opts.scale, - }, - .{ .has_side_effect = false }, - ); + const out = dispatch(qh, kc, vc, parameters.block_table, parameters.seq_lens, parameters.query_start_len, .{ + .is_causal = opts.mask.isCausal(), + .sliding_window = sliding_window, + .scale = opts.scale orelse 1.0 / @sqrt(@as(f32, @floatFromInt(q.dim(.hd)))), + }); // [total_q_tokens, num_heads, head_dim] -> {.b, .hkv, .hg, .hd} (q's order) return out.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = .auto }).transpose(q.shape()); } + +const DispatchOptions = struct { + is_causal: bool, + /// Below 0 for global attention. + sliding_window: i32, + scale: f32, +}; + +/// Launches fa_vec_paged for causal bf16 decode, split over 4 threadgroups at +/// head_dim 512 past 4096 KV positions, and paged_attention_tiled otherwise. +/// `q` and the result are `[tokens, heads, head_dim]`, the caches +/// `[pages, block_size, kv_heads, head_dim]`, `block_table` +/// `[seqs, max_blocks]`, `seq_lens` `[seqs]` and `query_start_len` +/// `[seqs + 1]`, all i32. +fn dispatch(q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, block_table: zml.Tensor, seq_lens: zml.Tensor, query_start_len: zml.Tensor, opts: DispatchOptions) zml.Tensor { + const dtype = q.dtype(); + if (dtype != .bf16 and dtype != .f16) std.debug.panic("paged attention: Metal takes bf16 or f16 queries, not {t}", .{dtype}); + if (k_cache.dtype() != dtype or v_cache.dtype() != dtype) std.debug.panic("paged attention: Metal takes caches of the queries' {t}, not {t} and {t}", .{ dtype, k_cache.dtype(), v_cache.dtype() }); + + const total_q_tokens: i32 = @intCast(q.dim(0)); + const num_heads: i32 = @intCast(q.dim(1)); + const head_dim: i32 = @intCast(q.dim(2)); + const block_size: i32 = @intCast(k_cache.dim(1)); + const num_kv_heads: i32 = @intCast(k_cache.dim(2)); + const num_seqs: i32 = @intCast(block_table.dim(0)); + const max_blocks: i32 = @intCast(block_table.dim(1)); + + const vec_decode = opts.is_causal and total_q_tokens == num_seqs and + (head_dim == 128 or head_dim == 256 or head_dim == 512) and + dtype == .bf16; + + if (vec_decode) { + const args: paged_attn.FaVecPaged.HostArgs = .{ + .num_heads = num_heads, + .n_kv = num_kv_heads, + .max_blocks = max_blocks, + .num_seqs = num_seqs, + .scale = opts.scale, + .sliding_window = opts.sliding_window, + }; + + const inputs: paged_attn.FaVecPaged.Kernel.Inputs = .{ + .q = q, + .k_cache = k_cache, + .v_cache = v_cache, + .block_tables = block_table, + .seq_lens = seq_lens, + .cu_seqlens_q = query_start_len, + }; + + const kv_capacity = max_blocks * block_size; + if (head_dim == 512 and kv_capacity > 4096) { + const nwg = 4; + const nrows = total_q_tokens * num_heads; + + const partial = paged_attn.FaVecPaged.Kernel.call(inputs, .{ .dst = zml.Shape.init(.{nrows * (head_dim * nwg + 2 * nwg)}, .f32) }, .{ + .cfg = .{ .head_dim = head_dim, .nsg = 8, .block_size = block_size, .position_stride = num_kv_heads * head_dim, .nwg = nwg }, + .grid = .{ nwg, @intCast(num_heads), @intCast(total_q_tokens) }, + .threadgroup = .{ 32, 8, 1 }, + .threadgroup_memory = paged_attn.FaVecPaged.threadgroupMemory(head_dim, 8), + .constants = .{ .args = .of(args) }, + }).dst; + + return paged_attn.FaVecPagedReduce.Kernel.call(.{ .htmp = partial }, .{ .dst = q.shape() }, .{ + .cfg = .{ .nwg = nwg, .head_dim = head_dim }, + .grid = .{ @intCast(nrows), 1, 1 }, + .threadgroup = .{ 32, nwg, 1 }, + .constants = .{ .args = .of(@as(i32, nrows)) }, + }).dst; + } + + var idx: usize = if (kv_capacity <= 1024) 0 else if (kv_capacity <= 2048) 1 else 2; + if (head_dim == 512 and idx > 1) idx = 1; + const nsg = ([_]i32{ 4, 8, 16 })[idx]; + + return paged_attn.FaVecPaged.Kernel.call(inputs, .{ .dst = q.shape() }, .{ + .cfg = .{ .head_dim = head_dim, .nsg = nsg, .block_size = block_size, .position_stride = num_kv_heads * head_dim, .nwg = 1 }, + .grid = .{ 1, @intCast(num_heads), @intCast(total_q_tokens) }, + .threadgroup = .{ 32, @intCast(nsg), 1 }, + .threadgroup_memory = paged_attn.FaVecPaged.threadgroupMemory(head_dim, nsg), + .constants = .{ .args = .of(args) }, + }).dst; + } + + const tile = paged_attn.PagedAttentionTiled.tileConfig(head_dim) orelse std.debug.panic("paged attention: head_dim {d} unsupported", .{head_dim}); + + const cfg: paged_attn.PagedAttentionTiled.Cfg = .{ + .dtype = zml.kernel.metal.from(dtype), + .head_size = head_dim, + .block_size = block_size, + .bq = tile[0], + .tile_kv = tile[1], + .num_threads = tile[2], + .causal = opts.is_causal, + }; + + const total_q_blocks = @divTrunc(total_q_tokens, tile[0]) + num_seqs; + + const out = paged_attn.PagedAttentionTiled.Kernel.call(.{ + .q = q, + .k_cache = k_cache, + .v_cache = v_cache, + .block_tables = block_table, + .context_lens = seq_lens, + .cu_seqlens_q = query_start_len, + }, .{ .out = q.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(num_heads), @intCast(total_q_blocks), 1 }, + .threadgroup = .{ @intCast(tile[2]), 1, 1 }, + .threadgroup_memory = paged_attn.PagedAttentionTiled.threadgroupMemory(cfg), + .constants = .{ + .num_kv_heads = .of(num_kv_heads), + .scale = .of(opts.scale), + .softcapping = .of(@as(f32, 0)), + .max_num_blocks_per_seq = .of(max_blocks), + .q_stride = .of(num_heads * head_dim), + .kv_block_stride = .of(block_size * num_kv_heads * head_dim), + .kv_head_stride = .of(head_dim), + .num_seqs = .of(num_seqs), + .sliding_window = .of(opts.sliding_window), + }, + }).out; + + // The tiled kernel leaves the padding rows untouched, so they are zeroed in + // its output. Zeroing a fresh buffer first, from cu_seqlens_q alone, would + // make XLA share that buffer across every call and copy it for each. + return paged_attn.ZeroPaddingRows.Kernel.call(.{ .cu_seqlens_q = query_start_len, .out = out }, .{ .out = q.shape() }, .{ + .cfg = .{ .dtype = cfg.dtype }, + .grid = .{ @intCast(total_q_tokens), 1, 1 }, + .threadgroup = .{ 256, 1, 1 }, + .constants = .{ .num_seqs = .of(num_seqs), .row_stride = .of(num_heads * head_dim) }, + }).out; +} diff --git a/zml/kernels/paged_attention/metal/paged_attn.zig b/zml/kernels/paged_attention/metal/paged_attn.zig new file mode 100644 index 00000000..20429565 --- /dev/null +++ b/zml/kernels/paged_attention/metal/paged_attn.zig @@ -0,0 +1,1003 @@ +//! Metal paged attention kernels: fa_vec_paged and its split-K reduce for +//! decode, and paged_attention_tiled with zero_padding_rows otherwise. + +const std = @import("std"); + +const kernel = @import("../../../kernel.zig"); + +const metal = kernel.metal; +const ArgSpec = metal.ArgSpec; +const Builder = metal.Builder; +const Type = metal.Type; +const Value = metal.Value; + +const flt_max = std.math.floatMax(f32); +/// A masked score: below any real one yet finite, since fast math assumes +/// no infinities. Halved so subtracting a running maximum cannot overflow. +const masked_score = -flt_max / 2.0; +const max_half = 65504.0; +const simd_width = 32; + +fn pad2(x: i32, n: i32) i32 { + return (x + n - 1) & ~(n - 1); +} + +/// fa_vec_paged: decode attention over paged bf16 caches, one threadgroup +/// of `nsg` simdgroups per (position split, query head, query row). +pub const FaVecPaged = struct { + pub const Cfg = struct { + /// The key and value head dimension, DK = DV. + head_dim: i32, + /// Simdgroups splitting the KV positions. + nsg: i32, + /// The page size. + block_size: i32, + /// The in-page position stride, n_kv * head_dim elements. + position_stride: i32, + /// Position-split threadgroups. Above 1 the kernel writes f32 partials + /// for FaVecPagedReduce. + nwg: i32, + }; + + pub const Args: metal.Struct = .{ .name = "fa_vec_paged_args", .fields = &.{ + .{ .name = "num_heads", .type = .int }, + .{ .name = "n_kv", .type = .int }, + .{ .name = "max_blocks", .type = .int }, + .{ .name = "num_seqs", .type = .int }, + .{ .name = "scale", .type = .float }, + .{ .name = "sliding_window", .type = .int }, + } }; + + pub const HostArgs = extern struct { + num_heads: i32, + n_kv: i32, + max_blocks: i32, + num_seqs: i32, + scale: f32, + /// Below 0 for global attention. + sliding_window: i32, + }; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "fa_vec_paged", + .inputs = &.{ "q", "k_cache", "v_cache", "block_tables", "seq_lens", "cu_seqlens_q" }, + .outputs = &.{"dst"}, + .constants = &.{"args"}, + .run = run, + }); + + /// The threadgroup memory of a launch with `nsg` simdgroups. + pub fn threadgroupMemory(head_dim: i32, nsg: i32) u64 { + const padded = pad2(head_dim, 128); + const bytes = (padded + 4 * 32 + 2 * padded) * nsg * 2; + return @intCast(@divFloor(bytes + 15, 16) * 16); + } + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + switch (cfg.head_dim) { + 128, 256, 512 => {}, + else => std.debug.panic("fa_vec_paged: head_dim {d} unsupported (128, 256 or 512)", .{cfg.head_dim}), + } + const char_in: ArgSpec = .{ .buffer = .{ .type = .char, .const_ = true } }; + const int_in: ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; + const a = try b.declareArgs(.{ + .args = .{ .constant_struct = Args }, + .q = char_in, + .k_cache = char_in, + .v_cache = char_in, + .block_tables = int_in, + .seq_lens = int_in, + .cu_seqlens_q = int_in, + .dst = .{ .buffer = .{ .type = .char } }, + .shmem_f16 = .{ .threadgroup = .half }, + .tgpig = .{ .threadgroup_position_in_grid = .uint3 }, + .tgpg = .{ .threadgroups_per_grid = .uint3 }, + .tiisg = .{ .thread_index_in_simdgroup = .ushort }, + .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, + }); + impl(b, cfg, a); + } + + /// The attention body. + fn impl(b: *Builder, cfg: Cfg, a: anytype) void { + const DK = cfg.head_dim; + const DV = cfg.head_dim; + const NE = 1; + const C = 32; + const NSG = cfg.nsg; + const BS = cfg.block_size; + const PS = cfg.position_stride; + const NWG = cfg.nwg; + + const r = a.tgpig.at(2).to(.int); + const iq2 = a.tgpig.at(1).to(.ushort); + const iwg = a.tgpig.at(0).to(.short); + + const DK4 = @divExact(DK, 4); + const DV4 = @divExact(DV, 4); + + const nrows_static = a.tgpg.at(2).to(.long).mul(a.args.field("num_heads").to(.uint)); + + var padding = b.openIf(r.ge(a.cu_seqlens_q.load(a.args.field("num_seqs")))); + { + var first = b.openIf(a.sgitg.eq(0)); + { + const rid = r.to(.long).mul(a.args.field("num_heads").to(.uint)).add(iq2); + if (NWG == 1) { + const dst4 = a.dst.to(.bfloat4); + var loop = b.openFor(a.tiisg.to(.short), DV4, simd_width); + { + const zero = b.constant(.float4, 0).to(.bfloat4); + dst4.store(rid.mul(DV4).add(loop.iv), zero); + loop.close(); + } + } else { + const dst4 = a.dst.to(.float4); + const dst1 = a.dst.to(.float).add(nrows_static.mul(DV).mul(NWG)); + var loop = b.openFor(a.tiisg.to(.short), DV4, simd_width); + { + dst4.store(rid.mul(DV4).mul(NWG).add(b.lift(NWG).mul(loop.iv)).add(iwg), b.constant(.float4, 0)); + loop.close(); + } + var lane0 = b.openIf(a.tiisg.eq(0)); + { + dst1.store(rid.mul(2 * NWG).add(b.lift(2).mul(iwg)).add(0), 0.0); + dst1.store(rid.mul(2 * NWG).add(b.lift(2).mul(iwg)).add(1), -flt_max / 2.0); + lane0.close(); + } + } + first.close(); + } + b.return_(); + padding.close(); + } + + const first_seq = b.local(.int, 0); + const last_seq = b.local(.int, a.args.field("num_seqs").sub(1)); + var search = b.openLoop(.{}); + { + search.body(first_seq.lt(last_seq)); + } + { + const mid = first_seq.add(last_seq).add(1).div(2); + var half = b.openIfElse(a.cu_seqlens_q.load(mid).le(r), .{}); + { + first_seq.assign(mid); + half.elseBranch(); + } + { + last_seq.assign(mid.sub(1)); + half.close(); + } + search.close(); + } + const seq_idx = first_seq.rvalue(); + const q_pos = r.sub(a.cu_seqlens_q.load(seq_idx)); + const query_len = a.cu_seqlens_q.load(seq_idx.add(1)).sub(a.cu_seqlens_q.load(seq_idx)); + const kv_total = a.seq_lens.load(seq_idx); + const kv_len = kv_total.sub(query_len).add(q_pos).add(1); + const kv_lo = b.ternary(a.args.field("sliding_window").lt(0), 0, b.max(0, kv_len.sub(a.args.field("sliding_window")))); + + const pt = a.block_tables.add(seq_idx.to(.ulong).mul(a.args.field("max_blocks").to(.uint))); + + const PK = pad2(DK, 128); + const PK4 = @divExact(PK, 4); + const PV = pad2(DV, 128); + const PV4 = @divExact(PV, 4); + const NW = simd_width; + const NL = @divExact(NW, NE); + const SH = 4 * C; + + const shmem = a.shmem_f16; + const sq4 = shmem.add(0 * PK).to(.half4); + const ss = shmem.add(a.sgitg.mul(SH)).add(NSG * PK).to(.float); + const ss4 = shmem.add(a.sgitg.mul(SH)).add(NSG * PK).to(.float4); + const sm = shmem.add(a.sgitg.mul(SH)).add(2 * C).add(NSG * PK); + const so4 = shmem.add(b.lift(2).mul(a.sgitg).mul(PV)).add(NSG * PK).add(NSG * SH).to(.float4); + // This lane's output, until the simdgroups merge theirs. + const lane_so4 = so4.add(a.tiisg); + + const q = a.q.add(r.to(.ulong).mul(a.args.field("num_heads").to(.uint)).add(iq2).mul(DK).mul(2)); + const ikv2 = iq2.div(a.args.field("num_heads").div(a.args.field("n_kv"))).to(.short); + const k_cache = a.k_cache.add(ikv2.to(.ulong).mul(DK).mul(2)); + const v_cache = a.v_cache.add(ikv2.to(.ulong).mul(DK).mul(2)); + + const q4 = q.to(.bfloat4); + var load_q = b.openFor(a.tiisg.to(.short), PK4, NW); + { + const i = load_q.iv; + var value = b.openIfElse(i.lt(DK4), .{.half4}); + { + value.yieldThen(.{q4.load(i).to(.half4)}); + } + { + value.yieldElse(.{b.constant(.half4, 0)}); + } + sq4.store(i, value.results[0]); + load_q.close(); + } + var zero_o = b.openFor(b.constant(.short, 0), @divExact(DV4, NL), 1); + { + lane_so4.store(zero_o.iv.mul(NL), b.constant(.float4, 0)); + zero_o.close(); + } + var zero_s = b.openFor(a.tiisg.to(.short), SH / 4, NW); + { + ss4.store(zero_s.iv, b.constant(.float4, 0)); + zero_s.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + // When DV4/NL is a multiple of NW every lane updates the output, so the + // `ty == 0` guard is dropped at comptime. + const all_lanes = @rem(@divExact(DV4, NL), NW) == 0; + const full: metal.LoopOpts = .{ .unroll = .full }; + const S = b.local(.float, 0.0); + const M = b.local(.float, -flt_max / 2.0); + + const tx = a.tiisg.rem(NL).to(.short); + const ty = a.tiisg.div(NL).to(.short); + + const ic0_lo = kv_lo.div(C); + const ic0 = b.local(.int, ic0_lo.add(iwg.mul(NSG)).add(a.sgitg)); + var blocks = b.openLoop(.{}); + { + const ic = ic0.mul(C); + var past = b.openIf(ic.ge(kv_len)); + { + blocks.breakLoop(); + past.close(); + } + + const in_kv = ic.add(a.tiisg).lt(kv_len).bitAnd(ic.add(a.tiisg).ge(kv_lo)); + sm.store(a.tiisg, b.ternary(in_kv, b.constant(.half, 0), b.constant(.half, -max_half))); + + const pq4 = sq4.add(tx); + const mqk = b.zeroedArray(.float, C / NE); + var qk = b.openForOpts(b.constant(.short, 0), C / NE, 1, full); + { + const cc = qk.iv; + const p = b.min(ic.add(cc), kv_len.sub(1)); + const pk4 = pagedRow(b, k_cache, pt, p, BS, PS).to(.bfloat4).add(ty.mul(PS).div(4).add(tx)); + var dots = b.openForOpts(b.constant(.short, 0), @divExact(DK4, NL), 1, full); + { + const ii = dots.iv; + const dot = b.dot(pk4.load(ii.mul(NL)).to(.float4), pq4.load(ii.mul(NL)).to(.float4)); + mqk.store(cc, mqk.load(cc).add(dot)); + dots.close(); + } + mqk.store(cc, b.simdSum(mqk.load(cc))); + qk.close(); + } + + const score = b.fma(mqk.load(tx), a.args.field("scale"), sm.load(b.lift(NE).mul(tx).add(ty)).to(.float)); + ss.store(b.lift(NE).mul(tx).add(ty), score); + + b.simdgroupBarrier(.mem_threadgroup); + + const m = M.rvalue(); + const s = ss.load(a.tiisg); + + M.assign(b.simdMax(b.max(M.rvalue(), s))); + + const ms = b.exp(m.sub(M)); + const vs = b.exp(s.sub(M)); + + S.assign(S.mul(ms).add(b.simdSum(vs))); + + ss.store(a.tiisg, vs); + + var rescale_lanes = if (all_lanes) null else b.openIf(ty.eq(0)); + var rescale = b.openForOpts(b.constant(.short, 0), @divExact(DV4, NL), 1, full); + { + const at = rescale.iv.mul(NL); + lane_so4.store(at, lane_so4.load(at).mul(ms)); + rescale.close(); + } + if (rescale_lanes) |*l| l.close(); + + b.simdgroupBarrier(.mem_threadgroup); + + const lo = b.array(.float4, @intCast(@divExact(DV4, NL))); + var zero_lo = b.openForOpts(b.constant(.short, 0), @divExact(DV4, NL), 1, full); + { + lo.store(zero_lo.iv, b.constant(.float4, 0)); + zero_lo.close(); + } + + const sst = ss.add(ty); + + var pv = b.openForOpts(b.constant(.short, 0), C / NE, 1, full); + { + const cc = pv.iv; + const p = b.min(ic.add(cc), kv_len.sub(1)); + const pv4 = pagedRow(b, v_cache, pt, p, BS, PS).to(.bfloat4).add(ty.mul(PS).div(4).add(tx)); + var dims = b.openForOpts(b.constant(.short, 0), @divExact(DV4, NL), 1, full); + { + const ii = dims.iv; + const weighted = pv4.load(ii.mul(NL)).to(.float4).mul(b.vec(.float4, .{sst.load(cc.mul(NE))})); + lo.store(ii, lo.load(ii).add(weighted)); + dims.close(); + } + pv.close(); + } + + var add_lanes = if (all_lanes) null else b.openIf(ty.eq(0)); + var add_o = b.openForOpts(b.constant(.short, 0), @divExact(DV4, NL), 1, full); + { + const ii = add_o.iv; + const partial = lo.load(ii); + lane_so4.store(ii.mul(NL), lane_so4.load(ii.mul(NL)).add(partial)); + add_o.close(); + } + if (add_lanes) |*l| l.close(); + blocks.next(); + ic0.assign(ic0.add(NWG * NSG)); + blocks.close(); + } + + var stats = b.openIf(a.tiisg.eq(0)); + { + ss.store(0, S); + ss.store(1, M); + stats.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + const rr = b.local(.short, @divFloor(NSG, 2)); + var halving = b.openLoop(.{}); + { + halving.body(rr.gt(0)); + } + { + var group = b.openIf(a.sgitg.lt(rr)); + { + const S0 = ss.load(0); + const S1 = ss.load(rr.mul(@divExact(SH, 2)).add(0)); + + const M0 = ss.load(1); + const M1 = ss.load(rr.mul(@divExact(SH, 2)).add(1)); + + const M_ = b.max(M0, M1); + + const ms0 = b.exp(M0.sub(M_)); + const ms1 = b.exp(M1.sub(M_)); + + const S_ = S0.mul(ms0).add(S1.mul(ms1)); + + var lane0 = b.openIf(a.tiisg.eq(0)); + { + ss.store(0, S_); + ss.store(1, M_); + lane0.close(); + } + + var merge = b.openFor(a.tiisg.to(.short), DV4, NW); + { + const i = merge.iv; + so4.store(i, so4.load(i).mul(ms0).add(so4.load(i.add(rr.mul(PV4))).mul(ms1))); + merge.close(); + } + group.close(); + } + b.threadgroupBarrier(.mem_threadgroup); + rr.assign(rr.shr(1)); + halving.close(); + } + + var first = b.openIf(a.sgitg.eq(0)); + { + const rid = r.to(.long).mul(a.args.field("num_heads").to(.uint)).add(iq2); + if (NWG == 1) { + const inv_sum = b.ternary(ss.load(0).eq(0.0), 0.0, b.lift(1.0).div(ss.load(0))); + const dst4 = a.dst.to(.bfloat4); + var loop = b.openFor(a.tiisg.to(.short), DV4, NW); + { + const out = so4.load(loop.iv).mul(inv_sum).to(.bfloat4); + dst4.store(rid.mul(DV4).add(loop.iv), out); + loop.close(); + } + } else { + const dst4 = a.dst.to(.float4); + const dst1 = a.dst.to(.float).add(nrows_static.mul(DV).mul(NWG)); + var loop = b.openFor(a.tiisg.to(.short), DV4, NW); + { + const part = so4.load(loop.iv); + dst4.store(rid.mul(DV4).mul(NWG).add(b.lift(NWG).mul(loop.iv)).add(iwg), part); + loop.close(); + } + var lane0 = b.openIf(a.tiisg.eq(0)); + { + const sum = ss.load(0); + dst1.store(rid.mul(2 * NWG).add(b.lift(2).mul(iwg)).add(0), sum); + const max = ss.load(1); + dst1.store(rid.mul(2 * NWG).add(b.lift(2).mul(iwg)).add(1), max); + lane0.close(); + } + } + first.close(); + } + } + + /// The address of position `p`'s bf16 row in `cache`, through the page + /// table `pt`. + fn pagedRow(b: *Builder, cache: Value, pt: Value, p: Value, BS: i32, PS: i32) Value { + const page = pt.load(p.div(BS)).to(.uint).to(.ulong).mul(b.constant(.uint, BS * PS)); + const row = p.rem(BS).to(.uint).mul(b.constant(.uint, PS)); + return cache.add(page.add(row).mul(2)); + } +}; + +/// fa_vec_paged_reduce: merges the `nwg` partials of a split-K fa_vec_paged, +/// one threadgroup per row and head. +pub const FaVecPagedReduce = struct { + pub const Cfg = struct { + /// The partials to merge, FaVecPaged's `nwg`. + nwg: i32, + /// The head dimension. + head_dim: i32, + }; + + pub const Args: metal.Struct = .{ .name = "fa_vec_paged_reduce_args", .fields = &.{ + .{ .name = "nrows", .type = .int }, + } }; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "fa_vec_paged_reduce", + .inputs = &.{"htmp"}, + .outputs = &.{"dst"}, + .constants = &.{"args"}, + .run = run, + }); + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + const a = try b.declareArgs(.{ + .args = .{ .constant_struct = Args }, + .htmp = .{ .buffer = .{ .type = .char, .const_ = true } }, + .dst = .{ .buffer = .{ .type = .char } }, + .tgpig = .{ .threadgroup_position_in_grid = .uint }, + .tiisg = .{ .thread_index_in_simdgroup = .ushort }, + .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, + }); + const nwg = cfg.nwg; + const DV = cfg.head_dim; + const rid = a.tgpig.to(.ulong); + const iwg = a.tiisg.to(.short); + const ss = a.htmp.to(.float).add(a.args.field("nrows").to(.ulong).mul(DV).mul(nwg)); + + var max = b.openIfElse(iwg.lt(nwg), .{.float}); + { + max.yieldThen(.{ss.load(rid.mul(2 * nwg).add(b.lift(2).mul(iwg)).add(1))}); + } + { + max.yieldElse(.{-flt_max / 2.0}); + } + const M = max.results[0]; + var sum = b.openIfElse(iwg.lt(nwg), .{.float}); + { + sum.yieldThen(.{ss.load(rid.mul(2 * nwg).add(b.lift(2).mul(iwg)).add(0))}); + } + { + sum.yieldElse(.{0.0}); + } + + const m = b.simdMax(M); + const ms = b.exp(M.sub(m)); + const S = b.simdSum(sum.results[0].mul(ms)); + const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); + + const DV4 = @divTrunc(DV, 4); + const htmp4 = a.htmp.to(.float4).add(rid.mul(DV4).mul(nwg)); + const dst4 = a.dst.to(.bfloat4).add(rid.mul(DV4)); + var loop = b.openFor(a.sgitg.to(.int), DV4, nwg); + { + const i = loop.iv; + var part_ = b.openIfElse(iwg.lt(nwg), .{.float4}); + { + part_.yieldThen(.{htmp4.load(i.mul(nwg).add(iwg)).mul(ms)}); + } + { + part_.yieldElse(.{b.constant(.float4, 0)}); + } + const part = part_.results[0]; + const v = b.simdSum(part); + var lane0 = b.openIf(iwg.eq(0)); + { + dst4.store(i, v.mul(inv_sum).to(.bfloat4)); + lane0.close(); + } + loop.close(); + } + } +}; + +/// zero_padding_rows: zeroes the output rows past the last real token. +pub const ZeroPaddingRows = struct { + pub const Cfg = struct { dtype: Type }; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "zero_padding_rows", + .inputs = &.{ "cu_seqlens_q", "out" }, + .outputs = &.{"out"}, + .constants = &.{ "num_seqs", "row_stride" }, + .run = run, + }); + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + switch (cfg.dtype.scalar) { + .bfloat, .half => {}, + else => std.debug.panic("zero_padding_rows: {s} unsupported (half or bfloat)", .{cfg.dtype.name()}), + } + const a = try b.declareArgs(.{ + .out = .{ .buffer = .{ .type = cfg.dtype } }, + .cu_seqlens_q = .{ .buffer = .{ .type = .int, .const_ = true } }, + .num_seqs = .{ .constant = .int }, + .row_stride = .{ .constant = .int }, + .tgp = .{ .threadgroup_position_in_grid = .uint3 }, + .tpt = .{ .thread_position_in_threadgroup = .uint3 }, + .ntg = .{ .threads_per_threadgroup = .uint3 }, + }); + const row = a.tgp.x().to(.int); + b.returnIf(row.lt(a.cu_seqlens_q.load(a.num_seqs))); + const dst = a.out.add(row.to(.ulong).mul(a.row_stride.to(.ulong))); + var loop = b.openFor(a.tpt.x().to(.int), a.row_stride, a.ntg.x().to(.int)); + { + dst.store(loop.iv, b.constant(cfg.dtype, 0)); + loop.close(); + } + } +}; + +/// paged_attention_tiled: FlashAttention-2 over 8x8 simdgroup matrices, causal +/// or bidirectional, one threadgroup per (query head, query block). +pub const PagedAttentionTiled = struct { + pub const Cfg = struct { + /// half or bfloat. + dtype: Type, + head_size: i32, + block_size: i32, + bq: i32 = 32, + tile_kv: i32 = 32, + num_threads: i32 = 128, + /// Bidirectional when false. + causal: bool = true, + }; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "paged_attention_tiled", + .inputs = &.{ "q", "k_cache", "v_cache", "block_tables", "context_lens", "cu_seqlens_q" }, + .outputs = &.{"out"}, + .constants = &.{ "num_kv_heads", "scale", "softcapping", "max_num_blocks_per_seq", "q_stride", "kv_block_stride", "kv_head_stride", "num_seqs", "sliding_window" }, + .run = run, + }); + + /// What the MSL instantiates: half or bfloat, a head size of tileConfig + /// with its tiles, and pages of 8, 16 or 32 positions. + fn supported(cfg: Cfg) bool { + const tile = tileConfig(cfg.head_size) orelse return false; + return (cfg.dtype.eql(.half) or cfg.dtype.eql(.bfloat)) and + (cfg.block_size == 8 or cfg.block_size == 16 or cfg.block_size == 32) and + cfg.bq == tile[0] and cfg.tile_kv == tile[1] and cfg.num_threads == tile[2]; + } + + /// The (bq, tile_kv, num_threads) of a head size, null when unsupported. + pub fn tileConfig(head_size: i64) ?[3]i32 { + return switch (head_size) { + 32, 64, 96, 128 => .{ 32, 32, 128 }, + 256 => .{ 16, 16, 64 }, + 512 => .{ 8, 8, 32 }, + else => null, + }; + } + + pub fn threadgroupMemory(cfg: Cfg) u64 { + const ld = cfg.head_size + @divExact(16, @as(i32, @intCast(cfg.dtype.size()))); + return @intCast((cfg.bq + 2 * cfg.tile_kv) * ld * @as(i32, @intCast(cfg.dtype.size()))); + } + + /// Lane `lane_id`'s (column, row) in an 8x8 simdgroup_matrix fragment. + fn fragCoord(b: *Builder, lane_id: Value) Value { + const qid = lane_id.to(.short).div(4).to(.short); + const fm = qid.bitAnd(4).add(lane_id.to(.short).div(2).rem(4)).to(.short); + const fn_ = qid.bitAnd(2).mul(2).add(lane_id.to(.short).rem(2).mul(2)).to(.short); + return b.vec(.short2, .{ fn_, fm }); + } + + /// Reduces `v` with `op` across the lanes that hold one row of an 8x8 + /// simdgroup_matrix fragment. + fn fragRowReduce(b: *Builder, comptime op: enum { max, sum }, v: Value) Value { + var x = v.rvalue(); + inline for (.{ 1, 8 }) |mask| { + const y = b.simdShuffleXor(x, b.constant(.ushort, mask)); + x = switch (op) { + .max => b.max(x, y), + .sum => x.add(y), + }; + } + return x; + } + + /// Sets lane `lane` of the vector `p[i]` to `x`. + fn setLane(p: Value, i: Value, lane: anytype, x: Value) void { + p.store(i, p.load(i).insert(lane, x)); + } + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + if (!supported(cfg)) std.debug.panic("paged_attention_tiled: {s} at head size {d}, block size {d} (8, 16 or 32) and tiles {d}, {d}, {d} unsupported", .{ cfg.dtype.name(), cfg.head_size, cfg.block_size, cfg.bq, cfg.tile_kv, cfg.num_threads }); + const T = cfg.dtype; + const HEAD_SIZE = cfg.head_size; + const BLOCK_SIZE = cfg.block_size; + const BQ = cfg.bq; + const TILE_KV = cfg.tile_kv; + const NUM_THREADS = cfg.num_threads; + const int_arg: ArgSpec = .{ .constant = .int }; + const float_arg: ArgSpec = .{ .constant = .float }; + const a = try b.declareArgs(.{ + .out = .{ .buffer = .{ .type = T } }, + .q = .{ .buffer = .{ .type = T, .const_ = true } }, + .k_cache = .{ .buffer = .{ .type = T, .const_ = true } }, + .v_cache = .{ .buffer = .{ .type = T, .const_ = true } }, + .num_kv_heads = int_arg, + .scale = float_arg, + .softcapping = float_arg, + .block_tables = .{ .buffer = .{ .type = .uint, .const_ = true } }, + .context_lens = .{ .buffer = .{ .type = .uint, .const_ = true } }, + .max_num_blocks_per_seq = int_arg, + .q_stride = int_arg, + .kv_block_stride = int_arg, + .kv_head_stride = int_arg, + .cu_seqlens_q = .{ .buffer = .{ .type = .int, .const_ = true } }, + .num_seqs = int_arg, + .sliding_window = int_arg, + .shared_mem = .{ .threadgroup = .char }, + .tgp = .{ .threadgroup_position_in_grid = .uint3 }, + .tgpg = .{ .threadgroups_per_grid = .uint3 }, + .tpt = .{ .thread_position_in_threadgroup = .uint3 }, + .sg_idx = .{ .simdgroup_index_in_threadgroup = .uint }, + .lane = .{ .thread_index_in_simdgroup = .uint }, + }); + + const NUM_SIMD_LANES = 32; + const NUM_SG = @divExact(NUM_THREADS, NUM_SIMD_LANES); + const TD = @divExact(HEAD_SIZE, 8); + const TK = @divExact(TILE_KV, 8); + if (@divExact(BQ, NUM_SG) != 8) @panic("paged_attention_tiled: ROWS_PER_SG must be 8"); + const unroll: metal.LoopOpts = .{ .unroll = .enable }; + + const thread_idx = a.tpt.x().to(.int); + const head_idx = a.tgp.x().to(.int); + const q_block_global_idx = a.tgp.y().to(.int); + const num_heads = a.tgpg.x().to(.int); + const num_queries_per_kv = num_heads.div(a.num_kv_heads); + const kv_head_idx = head_idx.div(num_queries_per_kv); + const kv_token_stride = a.num_kv_heads.mul(a.kv_head_stride); + + const first_seq = b.local(.int, 0); + const last_seq = b.local(.int, a.num_seqs); + var search = b.openLoop(.{}); + { + search.body(first_seq.lt(last_seq)); + } + { + const mid = first_seq.add(last_seq).add(1).div(2); + var half = b.openIfElse(a.cu_seqlens_q.load(mid).div(BQ).add(mid).le(q_block_global_idx), .{}); + { + first_seq.assign(mid); + half.elseBranch(); + } + { + last_seq.assign(mid.sub(1)); + half.close(); + } + search.close(); + } + // The grid is an upper bound on the query blocks: the extra ones land past + // the last sequence and clamp to it. + const seq_idx = b.min(first_seq.rvalue(), a.num_seqs.sub(1)); + + const q_seq_start = a.cu_seqlens_q.load(seq_idx); + const cur_batch_query_len = a.cu_seqlens_q.load(seq_idx.add(1)).sub(q_seq_start); + const q_block_start = q_seq_start.div(BQ).add(seq_idx); + const q_block_local = q_block_global_idx.sub(q_block_start); + const q_pos_start = q_block_local.mul(BQ); + + b.returnIf(q_pos_start.ge(cur_batch_query_len)); + + const seq_len = a.context_lens.load(seq_idx).to(.int); + const context_len = seq_len.sub(cur_batch_query_len); + const valid_q = b.min(BQ, cur_batch_query_len.sub(q_pos_start)); + + const SMEM_PAD = @divExact(16, @as(i32, @intCast(T.size()))); + const LD = HEAD_SIZE + SMEM_PAD; + const Q_ELEMS = BQ * LD; + const KV_ELEMS = TILE_KV * LD; + + const Q_smem = a.shared_mem.to(T); + const K_smem = Q_smem.add(Q_ELEMS); + const V_smem = K_smem.add(KV_ELEMS); + + const log2e: f32 = std.math.log2e; + const scale_log2 = a.scale.mul(log2e); + + const q_base = a.q.add(q_seq_start.add(q_pos_start).mul(a.q_stride)).add(head_idx.mul(HEAD_SIZE)); + var load_q = b.openFor(thread_idx, BQ * HEAD_SIZE, NUM_THREADS); + { + const i = load_q.iv; + const r = i.div(HEAD_SIZE); + const d = i.rem(HEAD_SIZE); + var value = b.openIfElse(r.lt(valid_q), .{T}); + { + value.yieldThen(.{q_base.load(r.mul(a.q_stride).add(d))}); + } + { + value.yieldElse(.{b.constant(T, 0)}); + } + Q_smem.store(r.mul(LD).add(d), value.results[0]); + load_q.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + const T2: Type = .vec(T.scalar, 2); + const q_reg = b.array(T2, @intCast(TD)); + var load_q_reg = b.openForOpts(0, TD, 1, unroll); + { + const d = load_q_reg.iv; + const tmp = b.simdgroupLoad(Q_smem.add(a.sg_idx.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); + q_reg.store(d, tmp.threadElements()); + load_q_reg.close(); + } + + const fc = fragCoord(b, a.lane.to(.ushort)); + const fm = fc.y(); + const fn_ = fc.x(); + + const max_score = b.local(.float, masked_score); + const sum_score = b.local(.float, 0.0); + + const o_reg = b.array(.float2, @intCast(TD)); + var zero_o = b.openForOpts(0, TD, 1, unroll); + { + o_reg.store(zero_o.iv, b.constant(.float2, 0)); + zero_o.close(); + } + + const block_table = a.block_tables.add(seq_idx.mul(a.max_num_blocks_per_seq)); + const num_kv_tiles = seq_len.add(TILE_KV).sub(1).div(TILE_KV); + + var tiles = b.openFor(0, num_kv_tiles, 1); + { + const tile_idx = tiles.iv; + const tile_start = tile_idx.mul(TILE_KV); + + if (cfg.causal) { + var past = b.openIf(tile_start.gt(context_len.add(q_pos_start).add(valid_q).sub(1))); + { + tiles.breakLoop(); + past.close(); + } + } + + const VEC_BYTES: i32 = if (HEAD_SIZE >= 256) 16 else 8; + const VEC = @divExact(VEC_BYTES, @as(i32, @intCast(T.size()))); + const LoadUnit: Type = if (VEC_BYTES == 16) .uint4 else .uint2; + std.debug.assert(@rem(LD, VEC) == 0); + var rows = b.openForOpts(0, @divExact(TILE_KV, NUM_SG), 1, unroll); + { + const t = a.sg_idx.mul(@divExact(TILE_KV, NUM_SG)).add(rows.iv).to(.int); + const kv_pos = tile_start.add(t); + var present = b.openIfElse(kv_pos.lt(seq_len), .{}); + { + const pb = block_table.load(kv_pos.div(BLOCK_SIZE)).to(.long); + const off = pb.mul(a.kv_block_stride).add(kv_pos.rem(BLOCK_SIZE).mul(kv_token_stride)).add(kv_head_idx.mul(a.kv_head_stride)); + const k_ptr = a.k_cache.add(off); + const v_ptr = a.v_cache.add(off); + var loop = b.openForOpts(a.lane.mul(VEC).to(.int), HEAD_SIZE, NUM_SIMD_LANES * VEC, unroll); + { + const d = loop.iv; + const k = k_ptr.add(d).to(LoadUnit).load(0); + K_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, k); + const v = v_ptr.add(d).to(LoadUnit).load(0); + V_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, v); + loop.close(); + } + present.elseBranch(); + } + { + const zero = b.constant(LoadUnit, 0); + var loop = b.openForOpts(a.lane.mul(VEC).to(.int), HEAD_SIZE, NUM_SIMD_LANES * VEC, unroll); + { + const d = loop.iv; + K_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, zero); + V_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, zero); + loop.close(); + } + present.close(); + } + rows.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + const s_reg = b.array(.float2, @intCast(TK)); + var zero_s = b.openForOpts(0, TK, 1, unroll); + { + s_reg.store(zero_s.iv, b.constant(.float2, 0)); + zero_s.close(); + } + + var qk_dims = b.openForOpts(0, TD, 1, unroll); + { + const d = qk_dims.iv; + const q_frag = b.matrix(T.scalar, q_reg.load(d)); + var qk_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = qk_keys.iv; + const k_frag = b.simdgroupLoad(K_smem.add(k.mul(8).mul(LD)).add(d.mul(8)), LD, .{ .transpose = true }); + const s_frag = b.matrix(.float, s_reg.load(k)); + s_reg.store(k, b.simdgroupMultiplyAccumulate(q_frag, k_frag, s_frag).threadElements()); + qk_keys.close(); + } + qk_dims.close(); + } + + const q_abs_pos = context_len.add(q_pos_start).add(a.sg_idx.mul(8)).add(fm).to(.int); + const row_masked = a.sg_idx.mul(8).add(fm).ge(valid_q); + const min_q_abs_pos = context_len.add(q_pos_start); + var tile_no_mask = tile_start.add(TILE_KV).le(seq_len).bitAnd(a.softcapping.le(0.0)).bitAnd(a.sliding_window.lt(0)); + if (cfg.causal) tile_no_mask = tile_no_mask.bitAnd(tile_start.add(TILE_KV).sub(1).lt(min_q_abs_pos)); + + var masking = b.openIfElse(tile_no_mask.bitAnd(row_masked.eq(false)), .{}); + { + var scale = b.openForOpts(0, TK, 1, unroll); + { + const k = scale.iv; + setLane(s_reg, k, 0, s_reg.load(k).at(0).mul(scale_log2)); + setLane(s_reg, k, 1, s_reg.load(k).at(1).mul(scale_log2)); + scale.close(); + } + masking.elseBranch(); + } + { + var keys = b.openForOpts(0, TK, 1, unroll); + { + const k = keys.iv; + var pair = b.openForOpts(0, 2, 1, unroll); + { + const jj = pair.iv; + const s = b.local(.float, s_reg.load(k).at(jj).mul(scale_log2)); + var capped = b.openIf(a.softcapping.gt(0.0)); + { + const s_orig = s.div(log2e); + s.assign(a.softcapping.mul(b.precise().tanh(s_orig.div(a.softcapping))).mul(log2e)); + capped.close(); + } + const kv_pos = tile_start.add(k.mul(8)).add(fn_).add(jj); + var masked = row_masked.bitOr(kv_pos.ge(seq_len)); + if (cfg.causal) masked = masked.bitOr(kv_pos.gt(q_abs_pos)); + const before_window = a.sliding_window.ge(0).bitAnd(kv_pos.lt(q_abs_pos.add(1).sub(a.sliding_window))); + masked = masked.bitOr(before_window); + setLane(s_reg, k, jj, b.ternary(masked, masked_score, s)); + pair.close(); + } + keys.close(); + } + masking.close(); + } + + const local_max = b.local(.float, masked_score); + var max_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = max_keys.iv; + local_max.assign(b.max(local_max.rvalue(), b.max(s_reg.load(k).at(0), s_reg.load(k).at(1)))); + max_keys.close(); + } + const row_max = fragRowReduce(b, .max, local_max); + + const prev_max = max_score.rvalue(); + const grown_max = b.max(prev_max, row_max); + const grew = grown_max.gt(prev_max); + const was_empty = prev_max.eq(masked_score); + const factor = b.ternary(grew, b.ternary(was_empty, 0.0, b.fast().exp2(prev_max.sub(grown_max))), 1.0); + const new_max = b.ternary(grew.eq(false).bitAnd(was_empty), 0.0, grown_max); + // A tile masking a whole row leaves the maximum masked: zero would + // pin it, and later scores far below zero would underflow. + max_score.assign(grown_max); + + const local_sum = b.local(.float, 0.0); + var p_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = p_keys.iv; + var pair = b.openForOpts(0, 2, 1, unroll); + { + const jj = pair.iv; + const s = s_reg.load(k).at(jj); + const p = b.ternary(s.eq(masked_score), 0.0, b.fast().exp2(s.sub(new_max))); + setLane(s_reg, k, jj, p); + local_sum.assign(local_sum.add(p)); + pair.close(); + } + p_keys.close(); + } + const row_sum = fragRowReduce(b, .sum, local_sum); + + sum_score.assign(sum_score.mul(factor).add(row_sum)); + + var rescale_o = b.openForOpts(0, TD, 1, unroll); + { + o_reg.store(rescale_o.iv, o_reg.load(rescale_o.iv).mul(factor)); + rescale_o.close(); + } + + var pv_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = pv_keys.iv; + const p_frag = b.matrix(.float, s_reg.load(k)); + var pv_dims = b.openForOpts(0, TD, 1, unroll); + { + const d = pv_dims.iv; + const v_frag = b.simdgroupLoad(V_smem.add(k.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); + const o_frag = b.matrix(.float, o_reg.load(d)); + o_reg.store(d, b.simdgroupMultiplyAccumulate(p_frag, v_frag, o_frag).threadElements()); + pv_dims.close(); + } + pv_keys.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + tiles.close(); + } + + const inv_sum = b.lift(1.0).div(sum_score.add(@as(f32, 1e-6))); + var normalize = b.openForOpts(0, TD, 1, unroll); + { + o_reg.store(normalize.iv, o_reg.load(normalize.iv).mul(inv_sum)); + normalize.close(); + } + + const O_smem = Q_smem.to(.float); + var store_o = b.openForOpts(0, TD, 1, unroll); + { + const d = store_o.iv; + const o_frag = b.matrix(.float, o_reg.load(d)); + b.simdgroupStore(o_frag, O_smem.add(a.sg_idx.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); + store_o.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + const out_base = a.out.add(q_seq_start.add(q_pos_start).mul(a.q_stride)).add(head_idx.mul(HEAD_SIZE)); + const total = valid_q.mul(HEAD_SIZE); + var write_out = b.openFor(thread_idx, total, NUM_THREADS); + { + const i = write_out.iv; + const r = i.div(HEAD_SIZE); + const d = i.rem(HEAD_SIZE); + const value = O_smem.load(r.mul(LD).add(d)).to(T); + out_base.store(r.mul(a.q_stride).add(d), value); + write_out.close(); + } + } +}; + +test "the paged attention kernels emit verified modules" { + const allocator = std.testing.allocator; + for ([_]i32{ 128, 256, 512 }) |head_dim| { + for ([_]i32{ 4, 8, 16 }) |nsg| { + allocator.free(try FaVecPaged.Kernel.emit(allocator, .{ .head_dim = head_dim, .nsg = nsg, .block_size = 16, .position_stride = 2 * head_dim, .nwg = 1 })); + } + } + allocator.free(try FaVecPaged.Kernel.emit(allocator, .{ .head_dim = 512, .nsg = 8, .block_size = 16, .position_stride = 1024, .nwg = 4 })); + allocator.free(try FaVecPagedReduce.Kernel.emit(allocator, .{ .nwg = 4, .head_dim = 512 })); + for ([_]Type{ .half, .bfloat }) |dtype| { + allocator.free(try ZeroPaddingRows.Kernel.emit(allocator, .{ .dtype = dtype })); + for ([_]i32{ 32, 64, 96, 128, 256, 512 }) |head_size| { + const tile = PagedAttentionTiled.tileConfig(head_size).?; + for ([_]bool{ true, false }) |causal| { + allocator.free(try PagedAttentionTiled.Kernel.emit(allocator, .{ .dtype = dtype, .head_size = head_size, .block_size = 16, .bq = tile[0], .tile_kv = tile[1], .num_threads = tile[2], .causal = causal })); + } + } + } +} diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index a8ec2121..99065140 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -46,7 +46,10 @@ pub const Backend = enum { .cute => cute.isAvailable(platform), .stablehlo => true, .tcl => platform.target == .furiosa, - .triton => platform.target != .cpu and platform.target != .furiosa, + .triton => switch (platform.target) { + .cuda, .rocm, .oneapi => true, + .cpu, .tpu, .neuron, .metal, .furiosa => false, + }, .metal => platform.target == .metal, .mosaic_tpu => platform.target == .tpu, }; @@ -310,7 +313,12 @@ test "Backend.auto selects triton on oneAPI" { test pagedAttention { const platform = zml.testing.env(); - if (!Backend.triton.isAvailable(platform)) return error.SkipZigTest; + const reference_backend: Backend = if (Backend.triton.isAvailable(platform)) + .triton + else if (Backend.metal.isAvailable(platform)) + .stablehlo + else + return error.SkipZigTest; const io = std.testing.io; const allocator = std.testing.allocator; var arena_state: std.heap.ArenaAllocator = .init(allocator); @@ -477,8 +485,10 @@ test pagedAttention { results_per_backend.set(backend, try output_d.toSliceAlloc(allocator, io)); } - const reference_backend: Backend = .triton; - const reference = results_per_backend.get(reference_backend) orelse return error.SkipZigTest; + const reference = results_per_backend.get(reference_backend) orelse { + std.log.warn("test pagedAttention {s} has no {t} reference", .{ test_case.name, reference_backend }); + continue; + }; var num_failed: u32 = 0; for (std.enums.values(Backend)) |backend| { From ca39742f14925bf4b676694a6d842fb7bc143303 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Thu, 29 Oct 2026 12:52:21 +0100 Subject: [PATCH 832/881] zml/platform: bump plugins and preallocate Metal memory again (#1001) - Turns preallocation back on by default on Metal, as before #979: not preallocating seems to cause memory issues we haven't looked into yet. - Bumps the PJRT plugins to `202610290951.309.1.785550677e75`. --- platforms/cpu/cpu.bzl | 6 +++--- platforms/cuda/cuda.bzl | 8 ++++---- platforms/metal/metal.bzl | 4 ++-- platforms/rocm/rocm.bzl | 4 ++-- zml/platform.zig | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index cee2a1f3..1105f776 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -43,21 +43,21 @@ def _cpu_plugin_impl(mctx): name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, sha256 = "3d6a6d1b047a0943b50fd9e63e5f6dff6b7cf184b186c0cc1052de40f0dbec92", - url = "https://mirror.zml.ai/plugins/202610281508.306.1.6bf391c563e0/zml-cpu-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "d6b4b15b13a2d1a3e0b84f143f568cd572bef5db91784f9f7a826173ed626c7e", - url = "https://mirror.zml.ai/plugins/202610281508.306.1.6bf391c563e0/zml-cpu-darwin-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "c06babc001edeab313bc63daab3aa3d40e087266a3a29634d182628fb65fad14", - url = "https://mirror.zml.ai/plugins/202610281508.306.1.6bf391c563e0/zml-cpu-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 8a5bb169..ced8ae98 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -224,12 +224,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "41777a3f65755133562ca31f83fadd2352f220dfa8715ad10c2b8f57e90a071a", - "url": "https://mirror.zml.ai/plugins/202610281508.306.1.6bf391c563e0/zml-cuda-linux-amd64.tar.zst", + "sha256": "77d2ddb19566604c827828fe3dcc53e938f3a1b16d431416fdd5c466266f5242", + "url": "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "de9adf212b10345e94f45889d49daaa15eb4bd0297007028e74c7081faaaa865", - "url": "https://mirror.zml.ai/plugins/202610281508.306.1.6bf391c563e0/zml-cuda-linux-arm64.tar.zst", + "sha256": "2cd18393c44816d3d01aa22fbd890632ba46eca3939c644bddef00d590578a80", + "url": "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index e6f21f06..03cdaf02 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "3c7b5fc6f24a6964dcf68f06e61a616ce50d7830fd0e96a06d9bd4eecf256bc8", - url = "https://mirror.zml.ai/plugins/202610281508.306.1.6bf391c563e0/zml-metal-darwin-arm64.tar.zst", + sha256 = "6eee91776397d472ac590413cf44d4ff3f7b31bebb163aa96339c6becaf2defd", + url = "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index dce411da..babd73ab 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610281508.306.1.6bf391c563e0/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "d89325846766f8630164b3b197a28a2388964032919953433cc22608caa3dd51" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "cd6c3c8673ce690a0fbf89dab5a16e6893f0f501bb2b41997c657cf6effa7549" def _rocm_package_name(name): return name + _ROCM_VERSION diff --git a/zml/platform.zig b/zml/platform.zig index e10a47f2..3b52c08e 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -733,7 +733,7 @@ pub const CreateOptions = struct { neuron: struct {} = .{}, furiosa: Furiosa = .{}, oneapi: XlaGpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.90 } } }, - metal: XlaGpu = .{ .allocator = .{ .bfc = .{ .preallocate = false, .memory_fraction = 0.90 } } }, + metal: XlaGpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.90 } } }, io_impl: Platform.IoImpl = .threaded, pub const Furiosa = struct { From d4b413f5f99f08e7862ea05f250663ea1a4271bb Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Thu, 29 Oct 2026 14:10:32 +0100 Subject: [PATCH 833/881] zml/tokenizer: support Mistral's numeric lookahead (#997) Mistral Large 4's Tekken tokenizer uses positive lookahead to split number runs into groups of three from the right. IREE's regex compiler rejects this pattern, preventing the tokenizer from loading. Patch IREE to recognize the exact isolated Split pattern, match complete number runs with its DFA, and emit the original groups using Unicode codepoint counts. Preserve progress when output fills during streaming and finalization. Register the patch in ZML's IREE repository rule. Treat RESOURCE_EXHAUSTED as normal encoder backpressure instead of logging errors before retrying. Add regressions for numeric token IDs, Unicode numbers, byte-at-a-time input, and output overflow. --- .../iree/mistral-right-aligned-numbers.patch | 303 ++++++++++++++++++ third_party/iree/repo.bzl | 1 + zml/tokenizer/iree/tokenizer.zig | 124 ++++++- 3 files changed, 426 insertions(+), 2 deletions(-) create mode 100644 third_party/iree/mistral-right-aligned-numbers.patch diff --git a/third_party/iree/mistral-right-aligned-numbers.patch b/third_party/iree/mistral-right-aligned-numbers.patch new file mode 100644 index 00000000..ca6e3fe5 --- /dev/null +++ b/third_party/iree/mistral-right-aligned-numbers.patch @@ -0,0 +1,303 @@ +diff --git a/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c b/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c +--- a/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c ++++ b/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c +@@ -225,6 +225,29 @@ + return iree_ok_status(); + } + ++ // Tekken groups numbers from the right using a positive lookahead. Match the ++ // entire number run with the DFA, then subdivide it in the Split segmenter. ++ // Restrict this lowering to the exact known pattern and isolated behavior. ++ bool right_aligned_numbers = false; ++ if (behavior == IREE_TOKENIZER_UTIL_REGEX_SPLIT_ISOLATED && !invert && ++ iree_string_view_equal( ++ pattern, ++ IREE_SV("[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*" ++ "[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|" ++ "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+" ++ "[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|" ++ "\\p{N}{1,3}(?=(?:\\p{N}{3})*(?!\\p{N}))|\\p{N}{1,2}|" ++ " ?[^\\s\\p{L}\\p{N}]+[\\r\\n/" ++ "]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"))) { ++ pattern = IREE_SV( ++ "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*" ++ "[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|" ++ "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+" ++ "[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}+|" ++ " ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"); ++ right_aligned_numbers = true; ++ } ++ + // Compile the regex pattern to a DFA. + iree_tokenizer_regex_dfa_t dfa; + uint8_t* dfa_data = NULL; +@@ -239,8 +262,13 @@ + } + + // Create the Split segmenter. On failure, free DFA data. +- status = iree_tokenizer_segmenter_split_allocate( +- dfa, dfa_data, behavior, invert, allocator, out_segmenter); ++ if (right_aligned_numbers) { ++ status = iree_tokenizer_segmenter_split_right_aligned_numbers_allocate( ++ dfa, dfa_data, allocator, out_segmenter); ++ } else { ++ status = iree_tokenizer_segmenter_split_allocate( ++ dfa, dfa_data, behavior, invert, allocator, out_segmenter); ++ } + if (!iree_status_is_ok(status)) { + iree_allocator_free(allocator, dfa_data); + } +diff --git a/runtime/src/iree/tokenizer/segmenter/BUILD.bazel b/runtime/src/iree/tokenizer/segmenter/BUILD.bazel +--- a/runtime/src/iree/tokenizer/segmenter/BUILD.bazel ++++ b/runtime/src/iree/tokenizer/segmenter/BUILD.bazel +@@ -188,6 +188,7 @@ + hdrs = ["split.h"], + deps = [ + "//runtime/src/iree/base", ++ "//runtime/src/iree/base/internal:unicode", + "//runtime/src/iree/tokenizer:segmenter", + "//runtime/src/iree/tokenizer/regex:exec", + ], +diff --git a/runtime/src/iree/tokenizer/segmenter/CMakeLists.txt b/runtime/src/iree/tokenizer/segmenter/CMakeLists.txt +--- a/runtime/src/iree/tokenizer/segmenter/CMakeLists.txt ++++ b/runtime/src/iree/tokenizer/segmenter/CMakeLists.txt +@@ -201,6 +201,7 @@ + "split.c" + DEPS + iree::base ++ iree::base::internal::unicode + iree::tokenizer::regex::exec + iree::tokenizer::segmenter + PUBLIC +diff --git a/runtime/src/iree/tokenizer/segmenter/split.c b/runtime/src/iree/tokenizer/segmenter/split.c +--- a/runtime/src/iree/tokenizer/segmenter/split.c ++++ b/runtime/src/iree/tokenizer/segmenter/split.c +@@ -7,6 +7,8 @@ + #include "iree/tokenizer/segmenter/split.h" + + #include ++ ++#include "iree/base/internal/unicode.h" + + //===----------------------------------------------------------------------===// + // Types +@@ -19,6 +21,7 @@ + bool invert; + // false = regex mode, true = literal mode. + bool is_literal; ++ bool right_aligned_numbers; + union { + struct { + iree_tokenizer_regex_dfa_t dfa; +@@ -361,7 +364,46 @@ + iree_tokenizer_regex_split_behavior_t behavior; + bool invert; + bool absolute; ++ bool right_aligned_numbers; ++ iree_string_view_t input; + } iree_tokenizer_split_callback_context_t; ++ ++// The caller has recognized a number-only match. Count codepoints, rather than ++// bytes, so mixed Unicode number categories retain the same grouping as \p{N}. ++static iree_status_t iree_tokenizer_split_emit_number_match( ++ iree_tokenizer_split_callback_context_t* context, ++ iree_tokenizer_regex_match_t match, iree_string_view_t text) { ++ iree_host_size_t position = 0; ++ iree_host_size_t count = 0; ++ while (position < text.size) { ++ iree_unicode_utf8_decode(text, &position); ++ ++count; ++ } ++ ++ iree_tokenizer_split_emit(context->emitter, context->last_emit_end, ++ match.start, context->absolute); ++ position = 0; ++ iree_host_size_t start = match.start; ++ iree_host_size_t group_size = (count - 1) % 3 + 1; ++ while (position < text.size) { ++ for (iree_host_size_t i = 0; i < group_size; ++i) { ++ iree_unicode_utf8_decode(text, &position); ++ } ++ iree_host_size_t end = match.start + position; ++ // finalize() can retry the same match after partially filling the output. ++ if (end > context->last_emit_end) { ++ iree_tokenizer_split_emit(context->emitter, start, end, ++ context->absolute); ++ if (context->emitter->full) { ++ return iree_status_from_code(IREE_STATUS_RESOURCE_EXHAUSTED); ++ } ++ } ++ start = end; ++ group_size = 3; ++ } ++ context->last_emit_end = match.end; ++ return iree_ok_status(); ++} + + // Processes each regex match inline: computes behavior result, emits segments, + // and commits state. Returns RESOURCE_EXHAUSTED if output capacity is +@@ -370,6 +412,20 @@ + void* user_data, iree_tokenizer_regex_match_t match) { + iree_tokenizer_split_callback_context_t* context = + (iree_tokenizer_split_callback_context_t*)user_data; ++ ++ if (context->right_aligned_numbers && match.end > match.start) { ++ iree_host_size_t base = context->emitter->chunk_base; ++ if (match.start < base || match.end - base > context->input.size) { ++ return iree_make_status(IREE_STATUS_INTERNAL, ++ "number match outside retained input"); ++ } ++ iree_string_view_t text = iree_make_string_view( ++ context->input.data + match.start - base, match.end - match.start); ++ iree_host_size_t position = 0; ++ if (iree_unicode_is_number(iree_unicode_utf8_decode(text, &position))) { ++ return iree_tokenizer_split_emit_number_match(context, match, text); ++ } ++ } + + // Compute behavior result (pure, no side effects). + // Gap is the region between the last emit and the current match start. +@@ -590,6 +646,7 @@ + segmenter->behavior = behavior; + segmenter->invert = invert; + segmenter->is_literal = false; ++ segmenter->right_aligned_numbers = false; + segmenter->regex.dfa = dfa; + segmenter->regex.dfa_data = dfa_data; + +@@ -605,6 +662,17 @@ + } + IREE_TRACE_ZONE_END(z0); + return status; ++} ++ ++iree_status_t iree_tokenizer_segmenter_split_right_aligned_numbers_allocate( ++ iree_tokenizer_regex_dfa_t dfa, uint8_t* dfa_data, ++ iree_allocator_t allocator, iree_tokenizer_segmenter_t** out_segmenter) { ++ IREE_RETURN_IF_ERROR(iree_tokenizer_segmenter_split_allocate( ++ dfa, dfa_data, IREE_TOKENIZER_UTIL_REGEX_SPLIT_ISOLATED, false, allocator, ++ out_segmenter)); ++ ((iree_tokenizer_segmenter_split_t*)*out_segmenter)->right_aligned_numbers = ++ true; ++ return iree_ok_status(); + } + + iree_status_t iree_tokenizer_segmenter_split_literal_allocate( +@@ -641,6 +709,7 @@ + segmenter->behavior = behavior; + segmenter->invert = invert; + segmenter->is_literal = true; ++ segmenter->right_aligned_numbers = false; + segmenter->literal.pattern_data = pattern_data; + segmenter->literal.pattern = + iree_make_string_view((const char*)pattern_data, pattern.size); +@@ -739,6 +808,8 @@ + .behavior = self->behavior, + .invert = self->invert, + .absolute = false, ++ .right_aligned_numbers = segmenter->right_aligned_numbers, ++ .input = input, + }; + + // Feed pattern matcher; matches are processed inline via callback. +@@ -882,6 +953,8 @@ + .behavior = self->behavior, + .invert = self->invert, + .absolute = false, ++ .right_aligned_numbers = segmenter->right_aligned_numbers, ++ .input = remaining_input, + }; + + iree_host_size_t input_end = chunk_base + remaining_input.size; +diff --git a/runtime/src/iree/tokenizer/segmenter/split.h b/runtime/src/iree/tokenizer/segmenter/split.h +--- a/runtime/src/iree/tokenizer/segmenter/split.h ++++ b/runtime/src/iree/tokenizer/segmenter/split.h +@@ -81,6 +81,14 @@ + iree_tokenizer_regex_split_behavior_t behavior, bool invert, + iree_allocator_t allocator, iree_tokenizer_segmenter_t** out_segmenter); + ++// Like ISOLATED splitting, but subdivides number-only matches into groups of ++// three Unicode numbers from the right. The DFA must match entire number runs ++// separately from other text. This implements Tekken's numeric lookahead ++// without requiring general positive lookahead support in the regex compiler. ++iree_status_t iree_tokenizer_segmenter_split_right_aligned_numbers_allocate( ++ iree_tokenizer_regex_dfa_t dfa, uint8_t* dfa_data, ++ iree_allocator_t allocator, iree_tokenizer_segmenter_t** out_segmenter); ++ + // Allocates a segmenter that splits text on literal string matches. + // Unlike the regex version, this uses efficient byte-by-byte substring search. + // The |pattern| is copied and owned by the segmenter. +diff --git a/runtime/src/iree/tokenizer/segmenter/split_test.cc b/runtime/src/iree/tokenizer/segmenter/split_test.cc +--- a/runtime/src/iree/tokenizer/segmenter/split_test.cc ++++ b/runtime/src/iree/tokenizer/segmenter/split_test.cc +@@ -28,7 +28,8 @@ + public: + ScopedSplitSegmenter(const char* pattern, + iree_tokenizer_regex_split_behavior_t behavior, +- bool invert = false) { ++ bool invert = false, ++ bool right_aligned_numbers = false) { + iree_tokenizer_regex_compile_error_t error = {0}; + status_ = iree_tokenizer_regex_compile_and_load( + iree_make_cstring_view(pattern), +@@ -37,8 +38,13 @@ + if (!iree_status_is_ok(status_)) return; + + iree_tokenizer_segmenter_t* raw = nullptr; +- status_ = iree_tokenizer_segmenter_split_allocate( +- dfa_, dfa_storage_, behavior, invert, iree_allocator_system(), &raw); ++ if (right_aligned_numbers) { ++ status_ = iree_tokenizer_segmenter_split_right_aligned_numbers_allocate( ++ dfa_, dfa_storage_, iree_allocator_system(), &raw); ++ } else { ++ status_ = iree_tokenizer_segmenter_split_allocate( ++ dfa_, dfa_storage_, behavior, invert, iree_allocator_system(), &raw); ++ } + if (!iree_status_is_ok(status_)) return; + + segmenter_ = ScopedSegmenter(raw); +@@ -72,6 +78,44 @@ + IREE_TOKENIZER_UTIL_REGEX_SPLIT_ISOLATED); + IREE_ASSERT_OK(segmenter.status()); + ASSERT_NE(segmenter.get(), nullptr); ++} ++ ++TEST(SplitSegmenterTest, RightAlignedNumbers) { ++ ScopedSplitSegmenter segmenter("\\p{N}+|\\p{L}+|\\s+", ++ IREE_TOKENIZER_UTIL_REGEX_SPLIT_ISOLATED, ++ false, true); ++ IREE_ASSERT_OK(segmenter.status()); ++ auto input = IREE_SV("a1 12 123 1234 12345 123456 1234567z"); ++ std::vector expected = { ++ "a", "1", " ", "12", " ", "123", " ", "1", "234", " ", ++ "12", "345", " ", "123", "456", " ", "1", "234", "567", "z"}; ++ TestWithAllChunkSizes(segmenter.get(), input, expected, false); ++ TestLimitedOutputCapacity(segmenter.get(), input, expected); ++ TestZeroCapacityOutput(segmenter.get(), input); ++} ++ ++TEST(SplitSegmenterTest, RightAlignedUnicodeNumbers) { ++ ScopedSplitSegmenter segmenter("\\p{N}+|\\p{L}+|\\s+", ++ IREE_TOKENIZER_UTIL_REGEX_SPLIT_ISOLATED, ++ false, true); ++ IREE_ASSERT_OK(segmenter.status()); ++ // Nd, Nl and No, including mixed UTF-8 lengths within a number run. ++ auto input = IREE_SV("١٢٣٤ 12345 Ⅷ²3४"); ++ std::vector expected = {"١", "٢٣٤", " ", "12", ++ "345", " ", "Ⅷ", "²3४"}; ++ TestWithAllChunkSizesUtf8(segmenter.get(), input, expected, false); ++ TestLimitedOutputCapacity(segmenter.get(), input, expected); ++} ++ ++TEST(SplitSegmenterTest, RightAlignedNumbersLongRun) { ++ ScopedSplitSegmenter segmenter( ++ "\\p{N}+", IREE_TOKENIZER_UTIL_REGEX_SPLIT_ISOLATED, false, true); ++ IREE_ASSERT_OK(segmenter.status()); ++ std::string input(1000, '1'); ++ std::vector expected(334, "111"); ++ expected[0] = "1"; ++ TestLimitedOutputCapacity(segmenter.get(), ++ iree_make_cstring_view(input.c_str()), expected); + } + + TEST(SplitSegmenterTest, StateSizeIsReasonable) { diff --git a/third_party/iree/repo.bzl b/third_party/iree/repo.bzl index 75c4eacb..ee89fb6d 100644 --- a/third_party/iree/repo.bzl +++ b/third_party/iree/repo.bzl @@ -18,6 +18,7 @@ def repo(): "//third_party/iree:match-hf-tokenizer.patch", "//third_party/iree:laguna-tokenizer-compat.patch", "//third_party/iree:fix-normalizer-lifetime-and-boundaries.patch", + "//third_party/iree:mistral-right-aligned-numbers.patch", ], patch_args = ["-p1"], ) diff --git a/zml/tokenizer/iree/tokenizer.zig b/zml/tokenizer/iree/tokenizer.zig index 0ae25c42..b1586afb 100644 --- a/zml/tokenizer/iree/tokenizer.zig +++ b/zml/tokenizer/iree/tokenizer.zig @@ -167,6 +167,16 @@ pub const Tokenizer = struct { transform_buffer: []u8, state: *c.iree_tokenizer_encode_state_t, + fn checkEncodeStatus(status: c.iree_status_t) !void { + // Output exhaustion is normal streaming backpressure. The caller + // commits any produced tokens and retries with more output space. + if (status != null and statusCode(status) == c.IREE_STATUS_RESOURCE_EXHAUSTED) { + _ = c.iree_status_ignore(status); + return Error.ResourceExhausted; + } + try checkOk(status); + } + fn init(tokenizer: *const Tokenizer) !Encoder { var state_size: usize = undefined; try checkOk(c.iree_tokenizer_encode_state_calculate_size(tokenizer.inner, &state_size)); @@ -298,7 +308,7 @@ pub const Tokenizer = struct { var consumed: usize = 0; var produced: usize = 0; - checkOk(c.iree_tokenizer_encode_state_feed( + checkEncodeStatus(c.iree_tokenizer_encode_state_feed( self.state, stringView(remaining), tokenOutput(try writableTokenSliceGreedy(out, min_tokens)), @@ -333,7 +343,7 @@ pub const Tokenizer = struct { while (true) { var produced: usize = 0; - checkOk(c.iree_tokenizer_encode_state_finalize( + checkEncodeStatus(c.iree_tokenizer_encode_state_finalize( self.state, tokenOutput(try writableTokenSliceGreedy(out, min_tokens)), &produced, @@ -939,6 +949,116 @@ test "loads bpe tokenizer with newline lookahead split and added unk" { try std.testing.expectEqualSlices(u32, &.{ 5, 6, 4 }, ids); } +const test_mistral_tokenizer_json = + \\{ + \\ "model": { + \\ "type": "BPE", + \\ "vocab": { + \\ "1": 0, "2": 1, "3": 2, "4": 3, "5": 4, "6": 5, "7": 6, + \\ "12": 7, "23": 8, "34": 9, "45": 10, "56": 11, "67": 12, + \\ "123": 13, "234": 14, "345": 15, "456": 16, "567": 17, + \\ "1234": 18 + \\ }, + \\ "merges": ["1 2", "2 3", "3 4", "4 5", "5 6", "6 7", + \\ "12 3", "23 4", "34 5", "45 6", "56 7", "123 4"] + \\ }, + \\ "pre_tokenizer": { + \\ "type": "Sequence", + \\ "pretokenizers": [ + \\ { + \\ "type": "Split", + \\ "pattern": {"Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}{1,3}(?=(?:\\p{N}{3})*(?!\\p{N}))|\\p{N}{1,2}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"}, + \\ "behavior": "Isolated", + \\ "invert": false + \\ }, + \\ {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false} + \\ ] + \\ }, + \\ "decoder": {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true} + \\} +; + +test "Mistral right-aligned number lookahead" { + const allocator = std.testing.allocator; + var tokenizer = try Tokenizer.fromBytes(allocator, test_mistral_tokenizer_json); + defer tokenizer.deinit(); + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const cases = .{ + .{ "1", &[_]u32{0} }, + .{ "12", &[_]u32{7} }, + .{ "123", &[_]u32{13} }, + .{ "1234", &[_]u32{ 0, 14 } }, + .{ "12345", &[_]u32{ 7, 15 } }, + .{ "123456", &[_]u32{ 13, 16 } }, + .{ "1234567", &[_]u32{ 0, 14, 17 } }, + }; + inline for (cases) |case| { + encoder.reset(); + const ids = try encoder.encodeAlloc(allocator, case[0]); + defer allocator.free(ids); + try std.testing.expectEqualSlices(u32, case[1], ids); + + encoder.reset(); + var streamed: std.Io.Writer.Allocating = .initAligned(allocator, .of(u32)); + defer streamed.deinit(); + for (0..case[0].len) |i| { + try encoder.feed(case[0][i .. i + 1], &streamed.writer); + } + try encoder.finalize(&streamed.writer); + try std.testing.expectEqualSlices(u8, std.mem.sliceAsBytes(case[1]), streamed.written()); + } +} + +test "Mistral Unicode number grouping and output overflow" { + const allocator = std.testing.allocator; + var parsed = try std.json.parseFromSlice(std.json.Value, allocator, test_mistral_tokenizer_json, .{}); + defer parsed.deinit(); + var model = try std.json.parseFromSlice(std.json.Value, allocator, + \\{"type": "WordPiece", "unk_token": "", "continuing_subword_prefix": "", "vocab": { + \\ "": 0, "١": 1, "٢٣٤": 2, "12": 3, "345": 4, + \\ "Ⅷ": 5, "²3४": 6, "1": 7, "111": 8 + \\}} + , .{}); + defer model.deinit(); + const arena = parsed.arena.allocator(); + try parsed.value.object.put(arena, "model", model.value); + // Test the Split segmenter directly, before ByteLevel maps UTF-8 bytes. + const split = parsed.value.object.get("pre_tokenizer").?.object.get("pretokenizers").?.array.items[0]; + try parsed.value.object.put(arena, "pre_tokenizer", split); + const json = try std.json.Stringify.valueAlloc(allocator, parsed.value, .{}); + defer allocator.free(json); + + var tokenizer = try Tokenizer.fromBytes(allocator, json); + defer tokenizer.deinit(); + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const cases = .{ + .{ "١٢٣٤", &[_]u32{ 1, 2 } }, + .{ "12345", &[_]u32{ 3, 4 } }, + .{ "Ⅷ²3४", &[_]u32{ 5, 6 } }, + }; + inline for (cases) |case| { + encoder.reset(); + var streamed: std.Io.Writer.Allocating = .initAligned(allocator, .of(u32)); + defer streamed.deinit(); + for (0..case[0].len) |i| { + try encoder.feed(case[0][i .. i + 1], &streamed.writer); + } + try encoder.finalize(&streamed.writer); + try std.testing.expectEqualSlices(u8, std.mem.sliceAsBytes(case[1]), streamed.written()); + } + + encoder.reset(); + const ids = try encoder.encodeAlloc(allocator, &(@as([1000]u8, @splat('1')))); + defer allocator.free(ids); + try std.testing.expectEqual(@as(usize, 334), ids.len); + try std.testing.expectEqual(@as(u32, 7), ids[0]); + for (ids[1..]) |id| try std.testing.expectEqual(@as(u32, 8), id); +} + test "writer" { const allocator = std.testing.allocator; const json = From 2532589572e9c0891a05c89b6c45f124593c1a53 Mon Sep 17 00:00:00 2001 From: Louis Lechevalier Date: Mon, 2 Nov 2026 14:46:41 +0100 Subject: [PATCH 834/881] zml/moe: CuTe MoE nvfp4 on sm103 (#981) This PR port the CuTe MoE for nvfp4 x nvfp4 from cutlass flashinfer into zml. It uses the CuTe zig DSL. It is the first PR of a serie that aim to replace the covering of flashinfer cutlass in ZML. The final goal is to remove the flashinfer cutlass binding. It begins by removing the binding of the bf16 kernel on sm90 (never used, we have triton). --- dsl/cute/builder.zig | 114 +- .../flashinfer_cutlass_moe.BUILD.bazel | 6 - zml/kernels/moe/cute/BUILD.bazel | 4 +- zml/kernels/moe/cute/cute.zig | 130 +- zml/kernels/moe/cute/mxfp4.zig | 271 --- zml/kernels/moe/cute/sm10x/BUILD.bazel | 14 + zml/kernels/moe/cute/sm10x/common.zig | 227 +++ zml/kernels/moe/cute/{ => sm10x}/moe.zig | 164 +- zml/kernels/moe/cute/sm10x/mxfp4.zig | 139 ++ zml/kernels/moe/cute/sm10x/nvfp4.zig | 176 ++ .../moe/cute/{ => sm10x}/persistent_mxfp4.zig | 194 +- .../moe/cute/sm10x/persistent_nvfp4.zig | 1693 +++++++++++++++++ zml/kernels/moe/cute/sm10x/reduce.zig | 124 ++ .../cutlass_flashinfer/cutlass_flashinfer.zig | 351 +--- zml/kernels/moe/moe.zig | 10 +- zml/kernels/moe/stablehlo/stablehlo.zig | 5 +- zml/kernels/moe/tests.zig | 73 +- 17 files changed, 2843 insertions(+), 852 deletions(-) delete mode 100644 zml/kernels/moe/cute/mxfp4.zig create mode 100644 zml/kernels/moe/cute/sm10x/BUILD.bazel create mode 100644 zml/kernels/moe/cute/sm10x/common.zig rename zml/kernels/moe/cute/{ => sm10x}/moe.zig (58%) create mode 100644 zml/kernels/moe/cute/sm10x/mxfp4.zig create mode 100644 zml/kernels/moe/cute/sm10x/nvfp4.zig rename zml/kernels/moe/cute/{ => sm10x}/persistent_mxfp4.zig (93%) create mode 100644 zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig create mode 100644 zml/kernels/moe/cute/sm10x/reduce.zig diff --git a/dsl/cute/builder.zig b/dsl/cute/builder.zig index 0bfeec1b..d19819fa 100644 --- a/dsl/cute/builder.zig +++ b/dsl/cute/builder.zig @@ -457,6 +457,11 @@ pub const TmaPartition = struct { /// Element encoding of a TMA transfer when it differs from the tensor dtype. pub const TmaFormat = enum { default, + /// Packed 4-bit elements, two per byte in shared memory (NVFP4 operands + /// of the `mxf4nvf4` MMA). + u4, + /// 4-bit elements expanded to one per byte in shared memory (FP4 operands + /// of the `mxf8f6f4` MMA). u4_unpack_u8, u16, }; @@ -477,17 +482,53 @@ pub const BlockScaledMma = struct { tiled: Atom, }; -/// Shape of the SM100 block-scaled MXFP4 x MXFP8 MMA instruction. +/// SM100 block-scaled MMA instruction: one CTA computes a 128 x `n` tile, +/// FP4 weights as A. The instruction `kind` fixes B, the scales and K. pub const BlockScaledMmaConfig = struct { - m: u16 = 128, + kind: Kind = .mxfp4_mxfp8, n: u16, - k: u16 = 32, - num_cta: u8 = 1, - vec_size: u8 = 32, + + pub const Kind = enum { + /// MXFP4 weights x MXFP8 activations: E8M0 scales every 32 + /// elements, K = 32 (PTX `kind::mxf8f6f4`). + mxfp4_mxfp8, + /// NVFP4 x NVFP4: E4M3 scales every 16 elements, K = 64 (PTX + /// `kind::mxf4nvf4.block16`). + nvfp4_nvfp4, + }; + + pub fn k(self: BlockScaledMmaConfig) u16 { + return switch (self.kind) { + .mxfp4_mxfp8 => 32, + .nvfp4_nvfp4 => 64, + }; + } + + fn bType(self: BlockScaledMmaConfig) DType { + return switch (self.kind) { + .mxfp4_mxfp8 => .f8e4m3fn, + .nvfp4_nvfp4 => .f4e2m1fn, + }; + } + + fn scaleType(self: BlockScaledMmaConfig) DType { + return switch (self.kind) { + .mxfp4_mxfp8 => .f8e8m0fnu, + .nvfp4_nvfp4 => .f8e4m3fn, + }; + } + + fn vecSize(self: BlockScaledMmaConfig) u8 { + return switch (self.kind) { + .mxfp4_mxfp8 => 32, + .nvfp4_nvfp4 => 16, + }; + } }; fn tmaDataFormat(format: TmaFormat, dtype: DType) cute_nvgpu.TmaDataFormat { return switch (format) { + .u4 => .U4, .u4_unpack_u8 => .U4_UNPACK_U8, .u16 => .U16, .default => switch (dtype) { @@ -1864,7 +1905,7 @@ pub const Builder = struct { // ==================== cute_nvgpu.tcgen05 (SM100) ==================== - /// Tiled MMA type of the SM100 block-scaled MXFP4 x MXFP8 atom. + /// Tiled MMA type of the block-scaled atom described by `cfg`. pub fn blockScaledMmaType(self: *Builder, cfg: BlockScaledMmaConfig) *const mlir.Type { return (cute_ir.TiledMmaType.get(self.ctx, .{ .mmaAtom = self.blockScaledAtomType(cfg), @@ -1874,27 +1915,24 @@ pub const Builder = struct { } fn blockScaledAtomType(self: *Builder, cfg: BlockScaledMmaConfig) *const mlir.Type { + if (cfg.n < 8 or cfg.n > 256 or cfg.n % 8 != 0) std.debug.panic("unsupported block-scaled MMA N {d}", .{cfg.n}); return (cute_nvgpu.MmaAtomSM100UMMABlockScaledType.get(self.ctx, .{ - .shapeMnk = self.algebraAttr(cute_ir.ShapeAttr, self.expr(.{ cfg.m, cfg.n, cfg.k })).attribute(), - .numCta = cfg.num_cta, + .shapeMnk = self.algebraAttr(cute_ir.ShapeAttr, self.expr(.{ 128, cfg.n, cfg.k() })).attribute(), + .numCta = 1, .aMajor = .k, .bMajor = .k, .aType = DType.f4e2m1fn.toMlir(self.ctx), - .bType = DType.f8e4m3fn.toMlir(self.ctx), + .bType = cfg.bType().toMlir(self.ctx), .cType = DType.f32.toMlir(self.ctx), - .sfType = DType.f8e8m0fnu.toMlir(self.ctx), + .sfType = cfg.scaleType().toMlir(self.ctx), .aFragKind = .smem_desc, - .vecSize = cfg.vec_size, + .vecSize = cfg.vecSize(), }) catch @panic("invalid block-scaled MMA atom")).type_(); } - /// SS, K-major MXFP4 x MXFP8 block-scaled atom and its tiled MMA, as the - /// CuTe DSL builds them. Both scale-factor pointers start at `scale_ptr` - /// and are patched after tensor-memory allocation. + /// Block-scaled atom and its tiled MMA. Both scale-factor pointers start + /// at `scale_ptr` and are patched after tensor-memory allocation. pub fn makeBlockScaledMma(self: *Builder, cfg: BlockScaledMmaConfig, scale_ptr: Value) BlockScaledMma { - if (cfg.m != 128 or cfg.k != 32 or cfg.n < 8 or cfg.n > 256 or cfg.n % 8 != 0) { - std.debug.panic("unsupported SM100 block-scaled MMA shape {d}x{d}x{d}", .{ cfg.m, cfg.n, cfg.k }); - } const disabled = self.constant(.i1, false); const atom = self.makeAtom(self.blockScaledAtomType(cfg), .{ disabled, disabled, disabled, scale_ptr, scale_ptr }); const tiled = self.emit(cute_ir.make_tiled_mma(self.ctx, atom.inner, self.blockScaledMmaType(cfg), self.loc())); @@ -2255,6 +2293,45 @@ pub const Builder = struct { )); } + /// Eight FP32 values rounded to FP4 E2M1 (saturating, nearest-even) and + /// packed into one 32-bit word, `values[0]` in the lowest nibble: the + /// packed NVFP4 storage order. + pub fn packFp4x8(self: *Builder, values: [8]Value) Value { + var operands: [8]*const mlir.Value = undefined; + for (&operands, values) |*operand, value| operand.* = value.to(.f32).inner; + return self.emit(llvm.inline_asm( + self.ctx, + "{ .reg .b8 b0, b1, b2, b3; cvt.rn.satfinite.e2m1x2.f32 b0, $2, $1; cvt.rn.satfinite.e2m1x2.f32 b1, $4, $3; " ++ + "cvt.rn.satfinite.e2m1x2.f32 b2, $6, $5; cvt.rn.satfinite.e2m1x2.f32 b3, $8, $7; mov.b32 $0, {b0, b1, b2, b3}; }", + &operands, + .{ .constraints = "=r,f,f,f,f,f,f,f,f", .result_type = .int(self.ctx, .i32) }, + self.loc(), + )); + } + + /// Two FP32 values rounded to FP4 E2M1 (saturating, nearest-even) in the + /// low byte of a 32-bit word, `a` in the low nibble. + pub fn packFp4x2(self: *Builder, a: Value, b: Value) Value { + return self.emit(llvm.inline_asm( + self.ctx, + "{ .reg .b8 lo; cvt.rn.satfinite.e2m1x2.f32 lo, $2, $1; cvt.u32.u8 $0, lo; }", + &.{ a.to(.f32).inner, b.to(.f32).inner }, + .{ .constraints = "=r,f,f", .result_type = .int(self.ctx, .i32) }, + self.loc(), + )); + } + + /// The FP8 E4M3 byte at the low end of `bits` (an i32), decoded to FP32. + pub fn e4m3ToF32(self: *Builder, bits: Value) Value { + return self.emit(llvm.inline_asm( + self.ctx, + "{ .reg .b16 lo, hl, hh; .reg .b32 pair; cvt.u16.u32 lo, $1; cvt.rn.f16x2.e4m3x2 pair, lo; mov.b32 {hl, hh}, pair; cvt.f32.f16 $0, hl; }", + &.{bits.inner}, + .{ .constraints = "=f,r", .result_type = DType.f32.toMlir(self.ctx) }, + self.loc(), + )); + } + /// One-operand FP32 PTX instruction, e.g. `ex2.approx.ftz.f32`. pub fn unaryF32(self: *Builder, comptime instruction: []const u8, value: Value) Value { return self.emit(llvm.inline_asm( @@ -3048,11 +3125,14 @@ test "SM100 block-scaled MMA atoms use the CuTe DSL instruction shapes" { const sf_ptr = b.intToPtr(0, .f8e8m0fnu, .tmem, 1); _ = b.makeBlockScaledMma(.{ .n = 16 }, sf_ptr); _ = b.makeBlockScaledMma(.{ .n = 128 }, sf_ptr); + const nvfp4_sf_ptr = b.intToPtr(0, .f8e4m3fn, .tmem, 1); + _ = b.makeBlockScaledMma(.{ .kind = .nvfp4_nvfp4, .n = 16 }, nvfp4_sf_ptr); const ir = try b.finish(.{ 192, 1, 1 }); defer std.testing.allocator.free(ir); try std.testing.expect(std.mem.indexOf(u8, ir, "sm100.mma_bs<128x16x32") != null); try std.testing.expect(std.mem.indexOf(u8, ir, "sm100.mma_bs<128x128x32") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "sm100.mma_bs<128x16x64, num_cta = 1, ab_major = (k, k), elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E4M3FN, frag_kind = ss, vec_size = 16>") != null); try std.testing.expect(std.mem.indexOf(u8, ir, "cute.make_tiled_mma") != null); try std.testing.expect(std.mem.indexOf(u8, ir, "cute_nvgpu.grid_constant") != null); diff --git a/third_party/flashinfer_cutlass_moe/flashinfer_cutlass_moe.BUILD.bazel b/third_party/flashinfer_cutlass_moe/flashinfer_cutlass_moe.BUILD.bazel index 037c2b10..780ac0d2 100644 --- a/third_party/flashinfer_cutlass_moe/flashinfer_cutlass_moe.BUILD.bazel +++ b/third_party/flashinfer_cutlass_moe/flashinfer_cutlass_moe.BUILD.bazel @@ -2,11 +2,6 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") package(default_visibility = ["//visibility:public"]) -filegroup( - name = "lib_sm90", - srcs = ["lib/libflashinfer_cutlass_moe_sm90.so"], -) - filegroup( name = "lib_sm100", srcs = ["lib/libflashinfer_cutlass_moe_sm100.so"], @@ -28,7 +23,6 @@ filegroup( ":lib_sm100", ":lib_sm103", ":lib_sm120", - ":lib_sm90", ], ) diff --git a/zml/kernels/moe/cute/BUILD.bazel b/zml/kernels/moe/cute/BUILD.bazel index 1c3e9683..ca87f37a 100644 --- a/zml/kernels/moe/cute/BUILD.bazel +++ b/zml/kernels/moe/cute/BUILD.bazel @@ -4,8 +4,6 @@ filegroup( name = "cute", srcs = [ "cute.zig", - "moe.zig", - "mxfp4.zig", - "persistent_mxfp4.zig", + "//zml/kernels/moe/cute/sm10x", ], ) diff --git a/zml/kernels/moe/cute/cute.zig b/zml/kernels/moe/cute/cute.zig index ed153b5d..c0156720 100644 --- a/zml/kernels/moe/cute/cute.zig +++ b/zml/kernels/moe/cute/cute.zig @@ -1,9 +1,9 @@ -//! Blackwell MXFP4 MoE backend with persistent CuTe GEMMs. +//! Blackwell MXFP4 and NVFP4 MoE backend with persistent CuTe GEMMs. const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -pub const kernels = @import("moe.zig"); +pub const kernels = @import("sm10x/moe.zig"); // `moe.zig`'s `refAllDecls` reaches this file but not its imports, so the // kernel emit tests are only collected if `kernels` is referenced here. @@ -17,12 +17,25 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major == 10; } +/// The NVFP4 kernels are only validated on SM103. +pub fn isNvfp4Supported(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + const cc = zml.platform.cuda.computeCapability(platform) orelse return false; + return cc.major == 10 and cc.minor == 3; +} + pub fn validateOptions(opts: zml.moe.Options) void { stdx.debug.assert(opts.activation == .swiglu, "cute backend only accepts swiglu activation, got {}", .{opts.activation}); stdx.debug.assert(opts.activation.swiglu.bias == null, "cute backend requires swiglu bias to be null", .{}); stdx.debug.assert(opts.activation.swiglu.scale == null, "cute backend requires swiglu scale to be null", .{}); } +/// MXFP4 weights are `u8` (or `i8`) packed FP4 with E8M0 block scales. NVFP4 +/// weights are packed `u8` with E4M3 block scales; their weight global scale +/// is the per-expert multiplier dequantizing each GEMM output (it folds in +/// the inverse of the input's global scale), and their input scale is the +/// multiplier applied to the GEMM's input before NVFP4 quantization. Both +/// expect block scales in the 128x4 tensor-core layout. pub fn fusedExperts( input: zml.Tensor, ids: zml.Tensor, @@ -34,26 +47,42 @@ pub fn fusedExperts( validateOptions(options); stdx.debug.assert(input.dtype() == .bf16, "cute backend only supports bf16 inputs, got {}", .{input.dtype()}); stdx.debug.assert(gate_up.bias == null and down.bias == null, "cute backend expects gate_up bias and down bias to be null", .{}); - const gq = gate_up.quantization orelse @panic("cute backend requires gate_up quantization to be set"); const dq = down.quantization orelse @panic("cute backend requires down quantization to be set"); - stdx.debug.assert(gq.scheme == .mxfp4 and dq.scheme == .mxfp4, "cute expects gate_up and down quantization scheme to be mxfp4, got {} and {}", .{ gq.scheme, dq.scheme }); + stdx.debug.assert(gq.scheme == dq.scheme, "cute expects gate_up and down quantization scheme to match, got {} and {}", .{ gq.scheme, dq.scheme }); - // Weight storage for mxfp4 in HF is expressed as u8 or i8 - stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "cute expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); - stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "cute expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); + const format: Format = switch (gq.scheme) { + .mxfp4 => blk: { + // Weight storage for mxfp4 in HF is expressed as u8 or i8 + stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "cute expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); + stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "cute expects down weight dtype to be u8 or i8, got {}", .{down.weight.dtype()}); + // Check the boundary quantizer and CuTe GEMM tile constraints. + kernels.validateMxfp4Shapes(down.weight.dim(1), down.weight.dim(2) * 2); + // The SwiGLU clamp is a kernel parameter, but the routing weight is + // multiplied in the up epilogue, before the down projection: applying it + // after would have to move into the down epilogue or the reduction. + stdx.debug.assert(options.routing_weight_placement == .before_down, "cute MXFP4 backend only supports routing_weight_placement = .before_down, got {}", .{options.routing_weight_placement}); + break :blk .mxfp4; + }, + .nvfp4 => blk: { + stdx.debug.assert(!options.quantize_input, "cute NVFP4 backend always quantizes its inputs to NVFP4", .{}); + stdx.debug.assert(gq.swizzled_scales and dq.swizzled_scales, "cute NVFP4 backend expects swizzled block scales", .{}); + stdx.debug.assert(gate_up.weight.dtype() == .u8 and down.weight.dtype() == .u8, "cute NVFP4 backend expects packed u8 weights, got {} and {}", .{ gate_up.weight.dtype(), down.weight.dtype() }); + const scale = struct { + fn f(global: ?zml.Quantization.GlobalScale) zml.Tensor { + return (global orelse @panic("cute NVFP4 backend requires global and input scales")).asMultiplier(); + } + }.f; + break :blk .{ .nvfp4 = .{ + .alpha1 = scale(gq.global_scale), + .alpha2 = scale(dq.global_scale), + .act1 = scale(gq.input_scale), + .act2 = scale(dq.input_scale), + } }; + }, + else => stdx.debug.panic("cute backend only supports mxfp4 and nvfp4 quantization, got {}", .{gq.scheme}), + }; - const expert_parallelism = gate_up.weight.shape().partition(.expert).isSharded(); - const hidden = down.weight.dim(1); - const intermediate = down.weight.dim(2) * 2; - - // Check the boundary quantizer and CuTe GEMM tile constraints. - kernels.validateShapes(hidden, intermediate); - - // The SwiGLU clamp is a kernel parameter, but the routing weight is - // multiplied in the up epilogue, before the down projection: applying it - // after would have to move into the down epilogue or the reduction. - stdx.debug.assert(options.routing_weight_placement == .before_down, "cute backend only supports routing_weight_placement = .before_down, got {}", .{options.routing_weight_placement}); const context: Context = .{ .input = input, .ids = ids, @@ -64,14 +93,29 @@ pub fn fusedExperts( .s2 = dq.scales, .topk = ids.dim(.topk), .swiglu_limit = options.activation.swiglu.limit, - .expert_parallel = expert_parallelism, + .routing_weight_placement = options.routing_weight_placement, + .format = format, + .expert_parallel = gate_up.weight.shape().partition(.expert).isSharded(), }; - return if (expert_parallelism) + return if (context.expert_parallel) zml.ops.manualComputation(Context.body, context, input.shape()) else context.body(input.shape()); } +/// The format-specific tensors: NVFP4's per-expert global scales. They stay +/// as given until the sharded body, so they split across ranks like the +/// weights. +const Format = union(enum) { + mxfp4, + nvfp4: struct { + alpha1: zml.Tensor, + alpha2: zml.Tensor, + act1: zml.Tensor, + act2: zml.Tensor, + }, +}; + const Context = struct { input: zml.Tensor, ids: zml.Tensor, @@ -82,15 +126,17 @@ const Context = struct { s2: zml.Tensor, topk: i64, swiglu_limit: ?f32, + routing_weight_placement: zml.moe.RoutingWeightPlacement, + format: Format, expert_parallel: bool, fn body(self: Context, _: zml.Shape) zml.Tensor { const experts = self.w1.dim(.expert); const hidden = self.w2.dim(1); const intermediate = self.w2.dim(2) * 2; - var ids = self.ids.convert(.i32).reshape(.{ .token = .auto, .topk = self.topk }); - const tokens = ids.dim(.token); - const routing_weights = self.weights.convert(.f32).reshape(.{ tokens, self.topk }); + const tokens = self.input.dim(.t); + var ids = self.ids.convert(.i32); + const routing_weights = self.weights.convert(.f32); if (self.expert_parallel) { const partition_id = zml.ops.partitionId().convert(.i32); const expert_start = partition_id.scale(experts); @@ -99,15 +145,35 @@ const Context = struct { const local = ids.cmp(.GE, expert_start).logical(.AND, ids.cmp(.LT, expert_end)); ids = local.select(ids.sub(expert_start), .scalar(-1, .i32)); } - const result = kernels.forward(tokens, hidden, intermediate, experts, self.topk, self.swiglu_limit, .{ - .x = self.input.reshape(.{ tokens, hidden }), - .routing_weights = routing_weights, - .w1 = self.w1, - .s1 = self.s1, - .w2 = self.w2, - .s2 = self.s2, - .ids = ids, - }).reshape(self.input.shape().dims()).withTags(self.input.shape()); + const result = switch (self.format) { + .mxfp4 => kernels.forwardMxfp4(tokens, hidden, intermediate, experts, self.topk, self.swiglu_limit, .{ + .x = self.input, + .routing_weights = routing_weights, + .w1 = self.w1, + .s1 = self.s1, + .w2 = self.w2, + .s2 = self.s2, + .ids = ids, + }, null), + .nvfp4 => |f| kernels.forwardNvfp4(tokens, hidden, intermediate, experts, self.topk, .{ + .x = self.input, + .routing_weights = routing_weights, + .ids = ids, + .w1 = self.w1, + .s1 = self.s1, + .w2 = self.w2, + .s2 = self.s2, + // `alpha` is per expert; `act` is one scale shared by every + // expert, which the kernels read per expert. + .alpha1 = f.alpha1.reshape(.{experts}), + .alpha2 = f.alpha2.reshape(.{experts}), + .act1 = f.act1.reshape(.{}).broad(.init(.{experts}, .f32)), + .act2 = f.act2.reshape(.{}).broad(.init(.{experts}, .f32)), + }, .{ + .swiglu_limit = self.swiglu_limit, + .routing_weight_placement = self.routing_weight_placement, + }), + }.withTags(self.input.shape()); return if (self.expert_parallel) zml.ops.allReduce(result, zml.Tensor.add) diff --git a/zml/kernels/moe/cute/mxfp4.zig b/zml/kernels/moe/cute/mxfp4.zig deleted file mode 100644 index d9ab0324..00000000 --- a/zml/kernels/moe/cute/mxfp4.zig +++ /dev/null @@ -1,271 +0,0 @@ -//! CuTe boundary kernels around the persistent MXFP4 GEMMs of -//! `persistent_mxfp4.zig`: routed MXFP8 input quantization and the top-k -//! reduction. -const std = @import("std"); - -const zml = @import("../../../zml.zig"); -const cute = zml.kernel.cute; -const B = cute.Builder; - -pub const GroupedQuantConfig = struct { - tokens: i64, - hidden: i64, - topk: i64, - /// Rows stored per expert group. - group_size: i64, - capacity: i64, - /// Route `r` goes to row `r * group_size`; `route_map` is not read. - direct: bool = false, -}; - -pub const CombineConfig = struct { - tokens: i64, - hidden: i64, - topk: i64, - columns_per_cta: i64, -}; - -// Both boundary kernels are programs rather than bare kernels so their launch -// carries the programmatic-dependent-launch attribute: they then start while -// the previous kernel drains, and wait on `griddepcontrol` before their first -// read. A bare kernel is launched by XLA without that attribute. -const GroupedQuantize = cute.Program(GroupedQuantConfig, .{ - .name = "mxfp4_cute_grouped_quantize", - .inputs = &.{ "x", "route_map" }, - .outputs = &.{ "q", "s" }, - .run = groupedQuantizeProgram, -}); - -const Combine = cute.Program(CombineConfig, .{ - .name = "mxfp4_cute_combine", - .inputs = &.{ "d", "routes" }, - .outputs = &.{"y"}, - .run = combineProgram, -}); - -const quantize_kernel_name = "mxfp4_cute_grouped_quantize_kernel"; -const combine_kernel_name = "mxfp4_cute_combine_kernel"; - -/// Host launch of a boundary kernel: forward the buffer pointers, with PDL. -fn launchBoundary(b: *B, kernel_name: [:0]const u8, grid: [3]i32, args: anytype) cute.FinishError!void { - const one = b.constant(.i32, 1); - const config = b.makeLaunchConfig(.{ - .grid = .{ b.constant(.i32, grid[0]), b.constant(.i32, grid[1]), b.constant(.i32, grid[2]) }, - .block = .{ b.constant(.i32, 128), one, one }, - .dynamic_smem = b.kernelSmemSize(kernel_name), - .stream = b.cudaStream(), - .cluster = .{ one, one, one }, - .use_pdl = true, - }); - const launch = b.launchEx(kernel_name, config, args); - b.returnHostStatus(b.cudaResultStatus(launch)); -} - -/// Quantize each token once, then scatter it to the physical rows assigned by -/// the routing. Scale bytes use the 128x4 layout consumed by the SM100 -/// block-scaled MMA scale-factor tensor map. -pub fn groupedQuantize(x: zml.Tensor, route_map: zml.Tensor, cfg: GroupedQuantConfig) struct { q: zml.Tensor, s: zml.Tensor } { - std.debug.assert(cfg.group_size == 8 or cfg.group_size == 16 or cfg.group_size == 32 or cfg.group_size == 64 or cfg.group_size == 128); - const result = GroupedQuantize.call( - .{ .x = x, .route_map = route_map }, - .{ - .q = .init(.{ cfg.capacity * cfg.group_size, cfg.hidden }, .f8e4m3fn), - .s = .init(.{ cfg.capacity, @divExact(cfg.hidden, 32) * 128 }, .u8), - }, - .{ .cfg = cfg }, - ); - return .{ .q = result.q, .s = result.s }; -} - -/// Amortize launch overhead for large prefills while retaining enough CTAs -/// to saturate the GPU for decode and small batches. -pub fn combineColumns(tokens: i64, hidden: i64) i64 { - const elements = tokens * hidden; - if (tokens > 1 and elements >= 512 * 2048) return 2048; - if (tokens > 1 and elements >= 512 * 1024) return 1024; - // Two columns per thread is the smallest tile: 128 threads, one 32-bit - // word each. - return 256; -} - -/// Sum the `[tokens * topk, hidden]` BF16 route rows of each token into BF16. -/// Routing weights were already applied before the down projection. -/// `routes` is negative for routes this rank does not compute (expert -/// parallelism): their rows were never written. -pub fn combine(d: zml.Tensor, routes: zml.Tensor, cfg: CombineConfig) zml.Tensor { - return Combine.call( - .{ .d = d, .routes = routes }, - .{ .y = .init(.{ cfg.tokens, cfg.hidden }, .bf16) }, - .{ .cfg = cfg }, - ).y; -} - -/// Python `prepare_input_vector`: each thread quantizes four consecutive -/// values and stores them as one packed word, so eight lanes cover one -/// 32-value MX block. -fn groupedQuantizeProgram(b: *B, cfg: GroupedQuantConfig) cute.FinishError!void { - b.beginFunction(quantize_kernel_name, .cuda_kernel); - try groupedQuantizeKernel(b, cfg); - b.endFunction(.{ 128, 1, 1 }); - - b.beginFunction(GroupedQuantize.name, .host); - // Host parameters carry the custom call's element types; the kernel reads - // four FP8 values per 32-bit word. - const host = try b.declareArgs(.{ - .x = .{ .ptr = cute.DType.bf16 }, - .route_map = .{ .ptr = cute.DType.i32 }, - .q = .{ .ptr = cute.DType.f8e4m3fn }, - .s = .{ .ptr = cute.DType.i8 }, - }); - try launchBoundary(b, quantize_kernel_name, .{ @intCast(@divExact(cfg.hidden, 4 * 128)), @intCast(cfg.tokens), 1 }, .{ - host.x, host.route_map, b.recastPointer(host.q, .i32, .gmem, 16), host.s, - }); - b.endFunction(null); -} - -fn groupedQuantizeKernel(b: *B, cfg: GroupedQuantConfig) cute.FinishError!void { - std.debug.assert(@rem(cfg.hidden, 4 * 128) == 0); - const a = try b.declareArgs(.{ - .x = .{ .tensor = .{ .dtype = .bf16, .shape = &.{ cfg.tokens, cfg.hidden } } }, - .route_map = .{ .tensor = .{ .dtype = .i32, .shape = &.{cfg.tokens * cfg.topk} } }, - // Four FP8 values per word. - .q = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.capacity * cfg.group_size, @divExact(cfg.hidden, 4) } } }, - .s = .{ .tensor = .{ .dtype = .i8, .shape = &.{ cfg.capacity, @divExact(cfg.hidden, 32) * 128 } } }, - }); - - const tid = b.threadIdx().x; - // This grid may have started before its producer finished: the routing and - // the previous layer write what it reads below. - b.waitForDependency(); - const token = b.blockIdx().y; - const word = b.blockIdx().x.mul(128).add(tid); - var x: [4]cute.Value = undefined; - for (&x, 0..) |*value, i| value.* = a.x.get(.{ token, word.mul(4).add(i) }).to(.f32); - - var maximum = x[0].abs().maximum(x[1].abs()).maximum(x[2].abs().maximum(x[3].abs())); - inline for (.{ 1, 2, 4 }) |offset| { - maximum = maximum.maximum(b.shuffleXor(maximum, offset)); - } - const raw_bits = maximum.maximum(1e-4).mul(1.0 / 448.0).bitCast(.i32); - const exponent = raw_bits.shrLogical(23).add(raw_bits.bitAnd(0x7fffff).ne(0).to(.i32)); - const inverse_scale = b.constant(.i32, 254).sub(exponent).shl(23).bitCast(.f32); - const packed_word = b.packFp8x4(x[0].mul(inverse_scale), x[1].mul(inverse_scale), x[2].mul(inverse_scale), x[3].mul(inverse_scale)); - - const scale_col = word.div(8); - for (0..@intCast(cfg.topk)) |slot| { - const route = token.mul(cfg.topk).add(slot); - const row = if (cfg.direct) route.mul(cfg.group_size) else a.route_map.get(.{route}); - // Routes of other expert-parallel ranks are not scheduled. - var scheduled = if (cfg.direct) null else b.openIf(row.ge(0)); - a.q.set(.{ row, word }, packed_word); - - var leader = b.openIf(tid.rem(8).eq(0)); - { - const scale_row = row.div(cfg.group_size); - const row_in_group = row.rem(cfg.group_size); - const swizzled_col = scale_col.div(4).mul(512) - .add(row_in_group.rem(32).mul(16)) - .add(row_in_group.div(32).mul(4)) - .add(scale_col.rem(4)); - a.s.set(.{ scale_row, swizzled_col }, exponent.to(.i8)); - leader.close(); - } - if (scheduled) |*s| s.close(); - } - // Only now are the rows this CTA owns complete: release the persistent - // GEMM, which waits on this signal before its first TMA read. - b.launchDependents(); -} - -/// Each thread owns two adjacent BF16 columns, read and written as one -/// 32-bit word: a warp then moves a full 128-byte line per access. -fn combineProgram(b: *B, cfg: CombineConfig) cute.FinishError!void { - b.beginFunction(combine_kernel_name, .cuda_kernel); - try combineKernel(b, cfg); - b.endFunction(.{ 128, 1, 1 }); - - b.beginFunction(Combine.name, .host); - // The kernel reads and writes two BF16 columns per 32-bit word. - const host = try b.declareArgs(.{ - .d = .{ .ptr = cute.DType.bf16 }, - .routes = .{ .ptr = cute.DType.i32 }, - .y = .{ .ptr = cute.DType.bf16 }, - }); - try launchBoundary(b, combine_kernel_name, .{ - @intCast(@divTrunc(cfg.hidden + cfg.columns_per_cta - 1, cfg.columns_per_cta)), - @intCast(cfg.tokens), - 1, - }, .{ b.recastPointer(host.d, .i32, .gmem, 16), host.routes, b.recastPointer(host.y, .i32, .gmem, 16) }); - b.endFunction(null); -} - -fn combineKernel(b: *B, cfg: CombineConfig) cute.FinishError!void { - const pairs = @divExact(cfg.hidden, 2); - const a = try b.declareArgs(.{ - .d = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.tokens * cfg.topk, pairs } } }, - .routes = .{ .tensor = .{ .dtype = .i32, .shape = &.{cfg.tokens * cfg.topk} } }, - .y = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.tokens, pairs } } }, - }); - - // The down projection's rows arrive from a grid this one may overlap. - b.waitForDependency(); - const token = b.blockIdx().y; - const pairs_per_cta = @divExact(cfg.columns_per_cta, 2); - const first_pair = b.blockIdx().x.mul(pairs_per_cta).add(b.threadIdx().x); - for (0..@intCast(@divExact(pairs_per_cta, 128))) |chunk| { - const pair = first_pair.add(chunk * 128); - var in_bounds = b.openIf(pair.lt(pairs)); - { - var low = b.constant(.f32, 0); - var high = b.constant(.f32, 0); - for (0..@intCast(cfg.topk)) |slot| { - const route = token.mul(cfg.topk).add(slot); - // Routes of other expert-parallel ranks were never written. - var computed = b.openIfElse(a.routes.get(.{route}).ge(0), .{ .f32, .f32 }); - { - const word = a.d.get(.{ route, pair }); - computed.yieldThen(.{ - low.add(word.shl(16).bitCast(.f32)), - high.add(word.bitAnd(@as(i32, @bitCast(@as(u32, 0xffff0000)))).bitCast(.f32)), - }); - } - { - computed.yieldElse(.{ low, high }); - } - low = computed.results[0]; - high = computed.results[1]; - } - const low_bits = low.to(.bf16).to(.f32).bitCast(.i32).shrLogical(16); - const high_bits = high.to(.bf16).to(.f32).bitCast(.i32).bitAnd(@as(i32, @bitCast(@as(u32, 0xffff0000)))); - a.y.set(.{ token, pair }, low_bits.bitOr(high_bits)); - in_bounds.close(); - } - } - b.launchDependents(); -} - -test "MXFP4 CuTe boundary kernels emit for decode and prefill batches" { - inline for (.{ @as(i64, 1), 6, 96, 16_384 }) |tokens| { - const quant_ir = try GroupedQuantize.emit(std.testing.allocator, .{ - .tokens = tokens, - .hidden = 5120, - .topk = 6, - .group_size = 16, - .capacity = @divTrunc(tokens * 6 + 15, 16) + 191, - }); - defer std.testing.allocator.free(quant_ir); - try std.testing.expect(std.mem.indexOf(u8, quant_ir, "griddepcontrol.launch_dependents;") != null); - try std.testing.expect(std.mem.indexOf(u8, quant_ir, "cuda.launch_ex") != null); - try std.testing.expect(std.mem.indexOf(u8, quant_ir, "cvt.rn.satfinite.e4m3x2.f32") != null); - - const combine_ir = try Combine.emit(std.testing.allocator, .{ - .tokens = tokens, - .hidden = 5120, - .topk = 6, - .columns_per_cta = combineColumns(tokens, 5120), - }); - defer std.testing.allocator.free(combine_ir); - try std.testing.expect(std.mem.indexOf(u8, combine_ir, "cuda.launch_ex") != null); - try std.testing.expect(std.mem.indexOf(u8, combine_ir, "scf.if") != null); - } -} diff --git a/zml/kernels/moe/cute/sm10x/BUILD.bazel b/zml/kernels/moe/cute/sm10x/BUILD.bazel new file mode 100644 index 00000000..2b23bfbb --- /dev/null +++ b/zml/kernels/moe/cute/sm10x/BUILD.bazel @@ -0,0 +1,14 @@ +package(default_visibility = ["//zml/kernels/moe/cute:__pkg__"]) + +filegroup( + name = "sm10x", + srcs = [ + "common.zig", + "moe.zig", + "mxfp4.zig", + "nvfp4.zig", + "persistent_mxfp4.zig", + "persistent_nvfp4.zig", + "reduce.zig", + ], +) diff --git a/zml/kernels/moe/cute/sm10x/common.zig b/zml/kernels/moe/cute/sm10x/common.zig new file mode 100644 index 00000000..47e05d37 --- /dev/null +++ b/zml/kernels/moe/cute/sm10x/common.zig @@ -0,0 +1,227 @@ +//! Machinery shared by the MXFP4 and NVFP4 CuTe MoE kernels: pipeline +//! barriers and state, the static persistent tile scheduler, direct routing, +//! shared-memory pointers, boundary kernel launches and small IR helpers. +const std = @import("std"); +const mlir = @import("mlir"); + +const zml = @import("../../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const V = cute.Value; + +pub fn typed(type_: *const mlir.Type) struct { mlir_type: cute.ArgSpec.MlirTypeSpec } { + return .{ .mlir_type = .{ .type_ = type_ } }; +} + +pub fn gridTyped(type_: *const mlir.Type) struct { mlir_type: cute.ArgSpec.MlirTypeSpec } { + return .{ .mlir_type = .{ .type_ = type_, .grid_constant = true } }; +} + +pub fn barrierAt(b: *B, base: V, stage: V) V { + return b.addOffset( + base, + b.makeIntTuple(.{stage}), + b.ptrTy(.i64, .smem, 8) catch @panic("bad pipeline barrier pointer"), + ).value(); +} + +pub const PipelinePosition = struct { + index: V, + phase: V, +}; + +/// Scalar form of CUTLASS's StaticPersistentTileScheduler state for the +/// one-CTA DeepSeek specialization. Python carries the decoded M/N/L +/// coordinate, validity predicate, linear work index, and executed-tile +/// count through every role's `scf.while`. Keeping the same state shape is +/// important to the CuTe software-pipeline passes, even though N is always +/// zero for one tile per group. +pub const PersistentWork = struct { + m: V, + n: V, + group: V, + valid: V, + linear: V, + executed: V, +}; + +/// Direct routing (one token) runs one single-row group per route. Routes +/// owned by other expert-parallel ranks carry expert -1: the persistent tiles +/// cover the valid routes only, in route order, so they do not occupy a wave. +pub const DirectRoutes = struct { + expert_ids: cute.View, + routes: usize, + + pub fn count(self: DirectRoutes, b: *B) V { + var valid = b.constant(.i32, 0); + for (0..self.routes) |r| valid = valid.add(self.expertAt(b, r).ge(0).to(.i32)); + return valid; + } + + /// Route of the `k`-th valid route. + fn nth(self: DirectRoutes, b: *B, k: V) V { + var seen = b.constant(.i32, 0); + var route = b.constant(.i32, 0); + for (0..self.routes) |r| { + const valid = self.expertAt(b, r).ge(0); + route = b.select(valid.bitAnd(seen.eq(k)), b.constant(.i32, r), route); + seen = seen.add(valid.to(.i32)); + } + return route; + } + + fn expertAt(self: DirectRoutes, b: *B, r: usize) V { + return self.expert_ids.get(.{b.constant(.i32, r)}, .i32); + } +}; + +pub fn persistentWorkAt( + b: *B, + linear: V, + executed: V, + active: V, + tiles_m: V, + div_m: V, + div_n: V, + direct: ?DirectRoutes, +) PersistentWork { + // StaticPersistentTileScheduler uses two first-class FastDivmod objects + // to decode the column-major (M, N=1, L) problem layout. Preserve those + // operations even though ordinary rem/div is mathematically equivalent; + // CuTe's scheduler reconstruction and loop canonicalization key off these + // values and carry them through every persistent loop. + const group_and_m = b.fastDivmod(linear, div_m); + const group_and_n = b.fastDivmod(group_and_m[0], div_n); + const group = if (direct) |routes| routes.nth(b, group_and_n[0]) else group_and_n[0]; + return .{ + .m = b.makeWarpUniform(group_and_m[1]), + .n = b.makeWarpUniform(group_and_n[1]), + .group = b.makeWarpUniform(group), + .valid = b.makeWarpUniform(linear.lt(active.mul(tiles_m))), + .linear = linear, + .executed = executed, + }; +} + +pub fn advancePersistentWork( + b: *B, + linear: V, + executed: V, + persistent_clusters: V, + active: V, + tiles_m: V, + div_m: V, + div_n: V, + direct: ?DirectRoutes, +) PersistentWork { + return persistentWorkAt(b, linear.add(persistent_clusters), executed.add(1), active, tiles_m, div_m, div_n, direct); +} + +/// Equivalent to CUTLASS PipelineState.advance(). Keep the index and phase +/// as explicit loop-carried SSA values instead of recovering them with +/// division/remainder; the generated CuTe program then has the same circular +/// state transition as the Python DSL. +pub fn advancePipeline(b: *B, index: V, phase: V, comptime stages: usize) PipelinePosition { + const advanced = index.add(1); + const wraps = advanced.eq(stages); + const next_index = b.select(wraps, 0, advanced); + var next_phase = b.openIfElse(wraps, .{phase}); + { + next_phase.yieldThen(.{phase.bitXor(1)}); + } + { + next_phase.yieldElse(.{phase}); + } + return .{ .index = next_index, .phase = next_phase.results[0] }; +} + +pub fn peekBarrierIfRemaining(b: *B, base: V, count: V, limit: usize, state: PipelinePosition) V { + var peek = b.openIfElse(count.lt(limit), .{.i1}); + { + peek.yieldThen(.{b.mbarrierWaitParity(barrierAt(b, base, state.index), state.phase)}); + } + { + peek.yieldElse(.{b.constant(.i1, true)}); + } + return peek.results[0]; +} + +/// `cute.add_offset` infers the alignment a pointer keeps as the gcd of the +/// base alignment and the offset, and `cute.recast_iter` may not change it, so +/// the field's own alignment has to be what the offset leaves. +pub fn sharedPointerAt( + b: *B, + base: V, + comptime byte_offset: i64, + comptime dtype: cute.DType, + comptime alignment: u64, +) V { + const moved = comptime movedAlignment(byte_offset); + comptime std.debug.assert(moved >= alignment); + const byte_ptr = b.addOffset( + base, + b.makeIntTuple(.{byte_offset}), + b.ptrTy(.i8, .smem, moved) catch @panic("bad shared byte pointer"), + ); + return b.recastPointer(byte_ptr, dtype, .smem, moved); +} + +/// Alignment of `shared_base + byte_offset`, which the shared storage is +/// allocated with. +fn movedAlignment(comptime byte_offset: i64) u64 { + return offsetAlignment(1024, byte_offset, 1); +} + +/// `cute.add_offset` keeps the gcd of the operand alignment and the offset, +/// and the op's result type has to say so. +pub fn offsetAlignment(base_bytes: u64, offset_elements: i64, element_bytes: u64) u64 { + if (offset_elements == 0) return base_bytes; + const moved = @as(u64, @intCast(@abs(offset_elements))) * element_bytes; + return @max(std.math.gcd(base_bytes, moved), element_bytes); +} + +pub fn Tuple(comptime T: type, comptime len: usize) type { + return @Tuple(&@as([len]type, @splat(T))); +} + +pub fn arity(comptime T: type) usize { + return @typeInfo(T).@"struct".field_names.len; +} + +/// The MLIR types of a tuple of values, as `openWhile` expects them. +pub fn typesOf(values: anytype) Tuple(*const mlir.Type, arity(@TypeOf(values))) { + const len = comptime arity(@TypeOf(values)); + var out: Tuple(*const mlir.Type, len) = undefined; + inline for (0..len) |i| out[i] = values[i].type_(); + return out; +} + +/// Forward a whole loop-carried array, as `yieldBefore` expects a tuple. +pub fn tupleOf(array: anytype) Tuple(@TypeOf(array[0]), array.len) { + var out: Tuple(@TypeOf(array[0]), array.len) = undefined; + inline for (0..array.len) |i| out[i] = array[i]; + return out; +} + +pub fn fmt(b: *B, comptime format: []const u8, args: anytype) []const u8 { + return std.fmt.allocPrint(b.arena.allocator(), format, args) catch @panic("OOM"); +} + +pub fn view(b: *B, value: V) cute.View { + return .{ .inner = value.inner, .kernel = b }; +} + +/// Host launch of a boundary kernel: forward the buffer pointers, with PDL. +pub fn launchBoundary(b: *B, kernel_name: [:0]const u8, grid: [3]i32, args: anytype) cute.FinishError!void { + const one = b.constant(.i32, 1); + const config = b.makeLaunchConfig(.{ + .grid = .{ b.constant(.i32, grid[0]), b.constant(.i32, grid[1]), b.constant(.i32, grid[2]) }, + .block = .{ b.constant(.i32, 128), one, one }, + .dynamic_smem = b.kernelSmemSize(kernel_name), + .stream = b.cudaStream(), + .cluster = .{ one, one, one }, + .use_pdl = true, + }); + const launch = b.launchEx(kernel_name, config, args); + b.returnHostStatus(b.cudaResultStatus(launch)); +} diff --git a/zml/kernels/moe/cute/moe.zig b/zml/kernels/moe/cute/sm10x/moe.zig similarity index 58% rename from zml/kernels/moe/cute/moe.zig rename to zml/kernels/moe/cute/sm10x/moe.zig index 809b374e..479e6516 100644 --- a/zml/kernels/moe/cute/moe.zig +++ b/zml/kernels/moe/cute/sm10x/moe.zig @@ -1,17 +1,20 @@ -//! SM100 CuTe MXFP8 x MXFP4 MoE, the Zig port of the CuTe DSL `NativeMoE`: -//! routing, routed MXFP8 input quantization, persistent up GEMM with a fused -//! SwiGLU + MXFP8 epilogue, persistent down GEMM storing rows in route order, -//! and the top-k reduction. Routing weights are applied before the down -//! projection, with the reference model's BF16 rounding. +//! SM100 CuTe MoE, MXFP8 x MXFP4 (the Zig port of the CuTe DSL `NativeMoE`) +//! and NVFP4 x NVFP4: routing, routed input quantization, persistent up GEMM +//! with a fused SwiGLU + quantization epilogue, persistent down GEMM storing +//! rows in route order, and the top-k reduction. MXFP4 applies routing weights +//! before the down projection, with the reference model's BF16 rounding. const std = @import("std"); const stdx = @import("stdx"); -const zml = @import("../../../zml.zig"); -pub const boundary = @import("mxfp4.zig"); -pub const persistent = @import("persistent_mxfp4.zig"); -const triton_kernels = @import("../triton/moe.zig"); +const zml = @import("../../../../zml.zig"); +pub const mxfp4 = @import("mxfp4.zig"); +pub const mxfp4_persistent = @import("persistent_mxfp4.zig"); +pub const nvfp4 = @import("nvfp4.zig"); +pub const nvfp4_persistent = @import("persistent_nvfp4.zig"); +const reduce = @import("reduce.zig"); +const triton_kernels = @import("../../triton/moe.zig"); -pub const Inputs = struct { +pub const Mxfp4Inputs = struct { /// `[tokens, hidden]` BF16. x: zml.Tensor, /// `[tokens, topk]` FP32. @@ -28,7 +31,7 @@ pub const Inputs = struct { /// The boundary quantizer handles four 128-column tiles per CTA. The GEMMs /// support 128-column tiles and a fixed set of reduction tile counts. -pub fn validateShapes(hidden: i64, intermediate: i64) void { +pub fn validateMxfp4Shapes(hidden: i64, intermediate: i64) void { stdx.debug.assert(hidden > 0 and @rem(hidden, 4 * 128) == 0, "cute backend requires hidden dimension to be a positive multiple of 512, got {}", .{hidden}); stdx.debug.assert(intermediate > 0 and @rem(intermediate, 128) == 0, "cute backend requires intermediate dimension to be a positive multiple of 128, got {}", .{intermediate}); stdx.debug.assert(supportedReductionTiles(@divTrunc(hidden, 128)), "cute backend does not support hidden reduction tile count {}, got hidden dimension {}", .{ @divTrunc(hidden, 128), hidden }); @@ -167,20 +170,15 @@ fn route(tokens: i64, experts: i64, topk: i64, n: i64, direct: bool, ids: zml.Te return .{ .groups = capacity, .route_map = s.perm, .route_inverse = s.map, .schedule = s.sched }; } -/// `[tokens, hidden]` BF16 output of the MoE layer. -pub fn forward(tokens: i64, hidden: i64, intermediate: i64, experts: i64, topk: i64, swiglu_limit: ?f32, a: Inputs) zml.Tensor { - return forwardWithTile(tokens, hidden, intermediate, experts, topk, swiglu_limit, a, null); -} - -/// `forward` with an explicit GEMM tile N, for tuning. `null` selects it from -/// the batch size. -pub fn forwardWithTile(tokens: i64, hidden: i64, intermediate: i64, experts: i64, topk: i64, swiglu_limit: ?f32, a: Inputs, tile_n: ?i64) zml.Tensor { +/// `[tokens, hidden]` BF16 output of the MXFP4 MoE layer. `tile_n` is the +/// GEMM tile N, for tuning; null selects it from the batch size. +pub fn forwardMxfp4(tokens: i64, hidden: i64, intermediate: i64, experts: i64, topk: i64, swiglu_limit: ?f32, a: Mxfp4Inputs, tile_n: ?i64) zml.Tensor { const n = tile_n orelse tileN(tokens, experts, topk); const direct = isDirect(tokens); const routing = route(tokens, experts, topk, n, direct, a.ids); const routes = tokens * topk; - const up_cfg: persistent.Config = .{ + const up_cfg: mxfp4_persistent.Config = .{ .experts = experts, .m = 2 * intermediate, .n = n, @@ -197,7 +195,7 @@ pub fn forwardWithTile(tokens: i64, hidden: i64, intermediate: i64, experts: i64 down_cfg.k = intermediate; down_cfg.epilogue = .route_rows; - const q1 = boundary.groupedQuantize(a.x, routing.route_map orelse routing.schedule, .{ + const q1 = mxfp4.groupedQuantize(a.x, routing.route_map orelse routing.schedule, .{ .tokens = tokens, .hidden = hidden, .topk = topk, @@ -205,29 +203,141 @@ pub fn forwardWithTile(tokens: i64, hidden: i64, intermediate: i64, experts: i64 .capacity = routing.groups, .direct = direct, }); - const q2 = persistent.upQuantized(up_cfg, .{ + const q2 = mxfp4_persistent.upQuantized(up_cfg, .{ .weight = a.w1, .schedule = routing.schedule, .input_quant = q1.q, .weight_scale = a.s1.bitCast(.u8), .input_scale = q1.s, }, routing.route_inverse, a.routing_weights.reshape(.{routes})); - const down = persistent.downRows(down_cfg, .{ + const down = mxfp4_persistent.downRows(down_cfg, .{ .weight = a.w2, .schedule = routing.schedule, .input_quant = q2.q, .weight_scale = a.s2.bitCast(.u8), .input_scale = q2.s, }, routing.route_inverse); - return boundary.combine(down, routing.route_map orelse routing.schedule, .{ + return reduce.topk(down, routing.route_map orelse routing.schedule, .{ .tokens = tokens, .hidden = hidden, .topk = topk, - .columns_per_cta = boundary.combineColumns(tokens, hidden), + .columns_per_cta = reduce.columnsPerCta(tokens, hidden), + }); +} + +pub const Nvfp4Inputs = struct { + /// `[tokens, hidden]` BF16. + x: zml.Tensor, + /// `[tokens, topk]` FP32. + routing_weights: zml.Tensor, + /// `[tokens, topk]` local expert ids, -1 for routes of other ranks. + ids: zml.Tensor, + /// `[experts, 2 * intermediate, hidden / 2]` packed FP4, gate/up rows interleaved. + w1: zml.Tensor, + /// `w1` E4M3 scales `[experts, 2 * intermediate, hidden / 16]` in the 128x4 layout. + s1: zml.Tensor, + /// `[experts, hidden, intermediate / 2]` packed FP4. + w2: zml.Tensor, + /// `w2` E4M3 scales `[experts, hidden, intermediate / 16]` in the 128x4 layout. + s2: zml.Tensor, + /// `[experts]` FP32 dequantization scale of each GEMM output, which folds + /// in the inverse of the GEMM input's global scale. + alpha1: zml.Tensor, + alpha2: zml.Tensor, + /// `[experts]` FP32 global scale each GEMM's input is multiplied by + /// before NVFP4 quantization. + act1: zml.Tensor, + act2: zml.Tensor, +}; + +pub const Nvfp4Options = struct { + swiglu_limit: ?f32 = null, + routing_weight_placement: zml.moe.RoutingWeightPlacement = .after_down, + /// GEMM tile N; null selects it from the batch size. + tile_n: ?i64 = null, +}; + +pub fn validateNvfp4Shapes(hidden: i64, intermediate: i64) void { + const k_tile = nvfp4_persistent.k_tile; + stdx.debug.assert(@rem(hidden, k_tile) == 0, "cute NVFP4 backend expects hidden to be a multiple of {}, got {}", .{ k_tile, hidden }); + stdx.debug.assert(@rem(intermediate, k_tile) == 0, "cute NVFP4 backend expects intermediate to be a multiple of {}, got {}", .{ k_tile, intermediate }); +} + +/// NVFP4 GEMM tile N: the expected routes of one expert, so that each +/// expert's weight tiles are read once per group instead of once per few rows. +fn nvfp4TileN(tokens: i64, experts: i64, topk: i64) i64 { + if (isDirect(tokens)) return 8; + const per_expert = std.math.divCeil(i64, tokens * topk, experts) catch unreachable; + return @intCast(std.math.clamp(std.math.ceilPowerOfTwoAssert(u64, @intCast(per_expert)), 16, 128)); +} + +/// `[tokens, hidden]` BF16 output of the NVFP4 MoE layer. +pub fn forwardNvfp4(tokens: i64, hidden: i64, intermediate: i64, experts: i64, topk: i64, a: Nvfp4Inputs, opts: Nvfp4Options) zml.Tensor { + validateNvfp4Shapes(hidden, intermediate); + const n = opts.tile_n orelse nvfp4TileN(tokens, experts, topk); + const direct = isDirect(tokens); + const routes = tokens * topk; + const flat_ids = a.ids.convert(.i32).reshape(.{routes}); + const routing = route(tokens, experts, topk, n, direct, a.ids); + const route_map = routing.route_map orelse flat_ids; + const weights = a.routing_weights.convert(.f32).reshape(.{routes}); + + const up_cfg: nvfp4_persistent.Config = .{ + .experts = experts, + .m = 2 * intermediate, + .n = n, + .k = hidden, + .groups = routing.groups, + .persistent_ctas = persistentCtas(), + .epilogue = .swiglu_nvfp4, + .routes = routes, + .direct = direct, + .swiglu_limit = opts.swiglu_limit, + .routing_weights = opts.routing_weight_placement == .before_down, + }; + var down_cfg = up_cfg; + down_cfg.m = hidden; + down_cfg.k = intermediate; + down_cfg.epilogue = .route_rows; + down_cfg.routing_weights = opts.routing_weight_placement == .after_down; + + const q1 = nvfp4.groupedQuantize(a.x, route_map, flat_ids, a.act1, .{ + .tokens = tokens, + .hidden = hidden, + .topk = topk, + .experts = experts, + .group_size = up_cfg.groupRows(), + .capacity = routing.groups, + .direct = direct, + }); + const q2 = nvfp4_persistent.upQuantized(up_cfg, .{ + .weight = a.w1, + .schedule = routing.schedule, + .input_quant = q1.q, + .weight_scale = a.s1.bitCast(.u8), + .input_scale = q1.s, + .alpha = a.alpha1, + }, routing.route_inverse, weights, a.act2); + const down = nvfp4_persistent.downRows(down_cfg, .{ + .weight = a.w2, + .schedule = routing.schedule, + .input_quant = q2.q, + .weight_scale = a.s2.bitCast(.u8), + .input_scale = q2.s, + .alpha = a.alpha2, + }, routing.route_inverse, weights); + return reduce.topk(down, route_map, .{ + .tokens = tokens, + .hidden = hidden, + .topk = topk, + .columns_per_cta = reduce.columnsPerCta(tokens, hidden), }); } test { - _ = boundary; - _ = persistent; + _ = mxfp4; + _ = mxfp4_persistent; + _ = nvfp4; + _ = nvfp4_persistent; + _ = reduce; } diff --git a/zml/kernels/moe/cute/sm10x/mxfp4.zig b/zml/kernels/moe/cute/sm10x/mxfp4.zig new file mode 100644 index 00000000..29e68977 --- /dev/null +++ b/zml/kernels/moe/cute/sm10x/mxfp4.zig @@ -0,0 +1,139 @@ +//! CuTe input quantization kernel in front of the persistent MXFP4 GEMMs of +//! `persistent_mxfp4.zig`: routed MXFP8 input quantization. +const std = @import("std"); + +const zml = @import("../../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const launchBoundary = @import("common.zig").launchBoundary; + +pub const GroupedQuantConfig = struct { + tokens: i64, + hidden: i64, + topk: i64, + /// Rows stored per expert group. + group_size: i64, + capacity: i64, + /// Route `r` goes to row `r * group_size`; `route_map` is not read. + direct: bool = false, +}; + +// A program rather than a bare kernel so its launch carries the +// programmatic-dependent-launch attribute: it then starts while the previous +// kernel drains, and waits on `griddepcontrol` before its first read. A bare +// kernel is launched by XLA without that attribute. +const GroupedQuantize = cute.Program(GroupedQuantConfig, .{ + .name = "mxfp4_cute_grouped_quantize", + .inputs = &.{ "x", "route_map" }, + .outputs = &.{ "q", "s" }, + .run = groupedQuantizeProgram, +}); + +const quantize_kernel_name = "mxfp4_cute_grouped_quantize_kernel"; + +/// Quantize each token once, then scatter it to the physical rows assigned by +/// the routing. Scale bytes use the 128x4 layout consumed by the SM100 +/// block-scaled MMA scale-factor tensor map. +pub fn groupedQuantize(x: zml.Tensor, route_map: zml.Tensor, cfg: GroupedQuantConfig) struct { q: zml.Tensor, s: zml.Tensor } { + std.debug.assert(cfg.group_size == 8 or cfg.group_size == 16 or cfg.group_size == 32 or cfg.group_size == 64 or cfg.group_size == 128); + const result = GroupedQuantize.call( + .{ .x = x, .route_map = route_map }, + .{ + .q = .init(.{ cfg.capacity * cfg.group_size, cfg.hidden }, .f8e4m3fn), + .s = .init(.{ cfg.capacity, @divExact(cfg.hidden, 32) * 128 }, .u8), + }, + .{ .cfg = cfg }, + ); + return .{ .q = result.q, .s = result.s }; +} + +/// Python `prepare_input_vector`: each thread quantizes four consecutive +/// values and stores them as one packed word, so eight lanes cover one +/// 32-value MX block. +fn groupedQuantizeProgram(b: *B, cfg: GroupedQuantConfig) cute.FinishError!void { + b.beginFunction(quantize_kernel_name, .cuda_kernel); + try groupedQuantizeKernel(b, cfg); + b.endFunction(.{ 128, 1, 1 }); + + b.beginFunction(GroupedQuantize.name, .host); + // Host parameters carry the custom call's element types; the kernel reads + // four FP8 values per 32-bit word. + const host = try b.declareArgs(.{ + .x = .{ .ptr = cute.DType.bf16 }, + .route_map = .{ .ptr = cute.DType.i32 }, + .q = .{ .ptr = cute.DType.f8e4m3fn }, + .s = .{ .ptr = cute.DType.i8 }, + }); + try launchBoundary(b, quantize_kernel_name, .{ @intCast(@divExact(cfg.hidden, 4 * 128)), @intCast(cfg.tokens), 1 }, .{ + host.x, host.route_map, b.recastPointer(host.q, .i32, .gmem, 16), host.s, + }); + b.endFunction(null); +} + +fn groupedQuantizeKernel(b: *B, cfg: GroupedQuantConfig) cute.FinishError!void { + std.debug.assert(@rem(cfg.hidden, 4 * 128) == 0); + const a = try b.declareArgs(.{ + .x = .{ .tensor = .{ .dtype = .bf16, .shape = &.{ cfg.tokens, cfg.hidden } } }, + .route_map = .{ .tensor = .{ .dtype = .i32, .shape = &.{cfg.tokens * cfg.topk} } }, + // Four FP8 values per word. + .q = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.capacity * cfg.group_size, @divExact(cfg.hidden, 4) } } }, + .s = .{ .tensor = .{ .dtype = .i8, .shape = &.{ cfg.capacity, @divExact(cfg.hidden, 32) * 128 } } }, + }); + + const tid = b.threadIdx().x; + // This grid may have started before its producer finished: the routing and + // the previous layer write what it reads below. + b.waitForDependency(); + const token = b.blockIdx().y; + const word = b.blockIdx().x.mul(128).add(tid); + var x: [4]cute.Value = undefined; + for (&x, 0..) |*value, i| value.* = a.x.get(.{ token, word.mul(4).add(@as(i32, @intCast(i))) }).to(.f32); + + var maximum = x[0].abs().maximum(x[1].abs()).maximum(x[2].abs().maximum(x[3].abs())); + inline for (.{ 1, 2, 4 }) |offset| { + maximum = maximum.maximum(b.shuffleXor(maximum, offset)); + } + const raw_bits = maximum.maximum(1e-4).mul(1.0 / 448.0).bitCast(.i32); + const exponent = raw_bits.shrLogical(23).add(raw_bits.bitAnd(0x7fffff).ne(0).to(.i32)); + const inverse_scale = b.constant(.i32, 254).sub(exponent).shl(23).bitCast(.f32); + const packed_word = b.packFp8x4(x[0].mul(inverse_scale), x[1].mul(inverse_scale), x[2].mul(inverse_scale), x[3].mul(inverse_scale)); + + const scale_col = word.div(8); + for (0..@intCast(cfg.topk)) |slot| { + const route = token.mul(cfg.topk).add(slot); + const row = if (cfg.direct) route.mul(cfg.group_size) else a.route_map.get(.{route}); + // Routes of other expert-parallel ranks are not scheduled. + var scheduled = if (cfg.direct) null else b.openIf(row.ge(0)); + a.q.set(.{ row, word }, packed_word); + + var leader = b.openIf(tid.rem(8).eq(0)); + const scale_row = row.div(cfg.group_size); + const row_in_group = row.rem(cfg.group_size); + const swizzled_col = scale_col.div(4).mul(512) + .add(row_in_group.rem(32).mul(16)) + .add(row_in_group.div(32).mul(4)) + .add(scale_col.rem(4)); + a.s.set(.{ scale_row, swizzled_col }, exponent.to(.i8)); + leader.close(); + if (scheduled) |*s| s.close(); + } + // Only now are the rows this CTA owns complete: release the persistent + // GEMM, which waits on this signal before its first TMA read. + b.launchDependents(); +} + +test "MXFP4 CuTe input quantization emits for decode and prefill batches" { + inline for (.{ @as(i64, 1), 6, 96, 16_384 }) |tokens| { + const quant_ir = try GroupedQuantize.emit(std.testing.allocator, .{ + .tokens = tokens, + .hidden = 5120, + .topk = 6, + .group_size = 16, + .capacity = @divTrunc(tokens * 6 + 15, 16) + 191, + }); + defer std.testing.allocator.free(quant_ir); + try std.testing.expect(std.mem.indexOf(u8, quant_ir, "griddepcontrol.launch_dependents;") != null); + try std.testing.expect(std.mem.indexOf(u8, quant_ir, "cuda.launch_ex") != null); + try std.testing.expect(std.mem.indexOf(u8, quant_ir, "cvt.rn.satfinite.e4m3x2.f32") != null); + } +} diff --git a/zml/kernels/moe/cute/sm10x/nvfp4.zig b/zml/kernels/moe/cute/sm10x/nvfp4.zig new file mode 100644 index 00000000..5afee090 --- /dev/null +++ b/zml/kernels/moe/cute/sm10x/nvfp4.zig @@ -0,0 +1,176 @@ +//! CuTe input quantization kernel in front of the persistent NVFP4 GEMMs of +//! `persistent_nvfp4.zig`: routed NVFP4 input quantization. +const std = @import("std"); + +const zml = @import("../../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const V = cute.Value; + +pub const GroupedQuantConfig = struct { + tokens: i64, + hidden: i64, + topk: i64, + experts: i64, + /// Rows stored per expert group. + group_size: i64, + capacity: i64, + /// Route `r` goes to row `r * group_size`; `route_map` is not read. + direct: bool = false, +}; + +const GroupedQuantize = cute.Program(GroupedQuantConfig, .{ + .name = "nvfp4_cute_grouped_quantize", + .inputs = &.{ "x", "route_map", "ids", "global_scale" }, + .outputs = &.{ "q", "s" }, + .run = groupedQuantizeProgram, +}); + +const quantize_kernel_name = "nvfp4_cute_grouped_quantize_kernel"; +/// Packed FP4 words (eight values) a CTA quantizes per token. +const words_per_cta = 128; + +/// NVFP4 E4M3 block scale of a block maximum: `scale = e4m3(global * +/// maximum / 6)` and the quantization multiplier `global / scale`, zero for an +/// all-zero block. +pub fn blockScale(b: *B, maximum: V, global: V) struct { bits: V, multiplier: V } { + const raw = global.mul(maximum.mul(1.0 / 6.0)).minimum(448.0); + const zero = b.constant(.f32, 0); + const bits = b.packFp8x4(raw, zero, zero, zero).bitAnd(0xff); + const scale = b.e4m3ToF32(bits); + const inverse = b.unaryF32("rcp.approx.ftz.f32", scale.mul(b.unaryF32("rcp.approx.ftz.f32", global))); + return .{ .bits = bits, .multiplier = b.select(scale.ne(0.0), inverse, zero) }; +} + +/// Offset of the scale of row `row` (within its 128-row group) and 16-wide +/// block `block` in the 128x4 tensor-core layout. +pub fn swizzledScaleOffset(row: V, block: V) V { + return block.div(4).mul(512) + .add(row.rem(32).mul(16)) + .add(row.div(32).mul(4)) + .add(block.rem(4)); +} + +/// Quantize each token once per routed expert and scatter it to the physical +/// rows assigned by the routing: packed FP4 `[capacity * group, hidden / 2]` +/// and E4M3 scales `[capacity, hidden / 16 * 128]` in the 128x4 layout. +/// `ids` are the `[tokens * topk]` local experts and `global_scale` the +/// `[experts]` FP32 activation global scales. +pub fn groupedQuantize(x: zml.Tensor, route_map: zml.Tensor, ids: zml.Tensor, global_scale: zml.Tensor, cfg: GroupedQuantConfig) struct { q: zml.Tensor, s: zml.Tensor } { + std.debug.assert(@rem(cfg.hidden, 64) == 0); + const result = GroupedQuantize.call( + .{ .x = x, .route_map = route_map, .ids = ids, .global_scale = global_scale }, + .{ + .q = .init(.{ cfg.capacity * cfg.group_size, @divExact(cfg.hidden, 2) }, .u8), + .s = .init(.{ cfg.capacity, @divExact(cfg.hidden, 16) * 128 }, .u8), + }, + .{ .cfg = cfg }, + ); + return .{ .q = result.q, .s = result.s }; +} + +fn groupedQuantizeProgram(b: *B, cfg: GroupedQuantConfig) cute.FinishError!void { + b.beginFunction(quantize_kernel_name, .cuda_kernel); + try groupedQuantizeKernel(b, cfg); + b.endFunction(.{ words_per_cta, 1, 1 }); + + b.beginFunction(GroupedQuantize.name, .host); + const host = try b.declareArgs(.{ + .x = .{ .ptr = cute.DType.bf16 }, + .route_map = .{ .ptr = cute.DType.i32 }, + .ids = .{ .ptr = cute.DType.i32 }, + .global_scale = .{ .ptr = cute.DType.f32 }, + .q = .{ .ptr = cute.DType.i8 }, + .s = .{ .ptr = cute.DType.i8 }, + }); + const one = b.constant(.i32, 1); + const words = @divExact(cfg.hidden, 8); + const config = b.makeLaunchConfig(.{ + .grid = .{ b.constant(.i32, @as(i32, @intCast(@divTrunc(words + words_per_cta - 1, words_per_cta)))), b.constant(.i32, @as(i32, @intCast(cfg.tokens))), one }, + .block = .{ b.constant(.i32, words_per_cta), one, one }, + .dynamic_smem = b.kernelSmemSize(quantize_kernel_name), + .stream = b.cudaStream(), + .cluster = .{ one, one, one }, + .use_pdl = true, + }); + const launch = b.launchEx(quantize_kernel_name, config, .{ + b.recastPointer(host.x, .i32, .gmem, 16), host.route_map, host.ids, host.global_scale, b.recastPointer(host.q, .i32, .gmem, 16), host.s, + }); + b.returnHostStatus(b.cudaResultStatus(launch)); + b.endFunction(null); +} + +/// Each thread quantizes eight consecutive values into one packed word, so +/// two lanes cover one 16-value NVFP4 block. +fn groupedQuantizeKernel(b: *B, cfg: GroupedQuantConfig) cute.FinishError!void { + const words = @divExact(cfg.hidden, 8); + const routes = cfg.tokens * cfg.topk; + const a = try b.declareArgs(.{ + // Two BF16 values per word. + .x = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.tokens, @divExact(cfg.hidden, 2) } } }, + .route_map = .{ .tensor = .{ .dtype = .i32, .shape = &.{routes} } }, + .ids = .{ .tensor = .{ .dtype = .i32, .shape = &.{routes} } }, + .global_scale = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.experts} } }, + // Eight FP4 values per word. + .q = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.capacity * cfg.group_size, words } } }, + .s = .{ .tensor = .{ .dtype = .i8, .shape = &.{ cfg.capacity, @divExact(cfg.hidden, 16) * 128 } } }, + }); + + const tid = b.threadIdx().x; + // The routing and the previous layer write what this grid reads. + b.waitForDependency(); + const token = b.blockIdx().y; + const word = b.blockIdx().x.mul(words_per_cta).add(tid); + // Out-of-range lanes still take part in the pair reduction. + const in_range = word.lt(@as(i32, @intCast(words))); + const safe_word = b.select(in_range, word, b.constant(.i32, 0)); + var x: [8]V = undefined; + for (0..4) |i| { + const pair = a.x.get(.{ token, safe_word.mul(4).add(@as(i32, @intCast(i))) }); + x[2 * i] = pair.shl(16).bitCast(.f32); + x[2 * i + 1] = pair.bitAnd(@as(i32, @bitCast(@as(u32, 0xffff0000)))).bitCast(.f32); + } + var maximum = x[0].abs(); + for (x[1..]) |value| maximum = maximum.maximum(value.abs()); + maximum = maximum.maximum(b.shuffleXor(maximum, 1)); + + const block = safe_word.div(2); + var in_range_scope = b.openIf(in_range); + for (0..@intCast(cfg.topk)) |slot| { + const route = token.mul(cfg.topk).add(slot); + const row = if (cfg.direct) route.mul(cfg.group_size) else a.route_map.get(.{route}); + // Routes of other expert-parallel ranks are not scheduled. + const expert = a.ids.get(.{route}); + var scheduled = b.openIf(row.ge(0).bitAnd(expert.ge(0))); + const scale = blockScale(b, maximum, a.global_scale.get(.{expert})); + var quantized: [8]V = undefined; + for (&quantized, x) |*q, value| q.* = value.mul(scale.multiplier); + a.q.set(.{ row, safe_word }, b.packFp4x8(quantized)); + + var leader = b.openIf(tid.rem(2).eq(0)); + a.s.set(.{ row.div(cfg.group_size), swizzledScaleOffset(row.rem(cfg.group_size), block) }, scale.bits.to(.i8)); + leader.close(); + scheduled.close(); + } + in_range_scope.close(); + // The rows this CTA owns are complete: release the persistent GEMM. + b.launchDependents(); +} + +test "NVFP4 CuTe quantization emits for decode and prefill batches" { + inline for (.{ @as(i64, 1), 6, 96, 16_384 }) |tokens| { + const ir = try GroupedQuantize.emit(std.testing.allocator, .{ + .tokens = tokens, + .hidden = 5120, + .topk = 6, + .experts = 192, + .group_size = 16, + .capacity = @divTrunc(tokens * 6 + 15, 16) + 191, + .direct = tokens == 1, + }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "griddepcontrol.launch_dependents;") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda.launch_ex") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cvt.rn.satfinite.e2m1x2.f32") != null); + } +} diff --git a/zml/kernels/moe/cute/persistent_mxfp4.zig b/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig similarity index 93% rename from zml/kernels/moe/cute/persistent_mxfp4.zig rename to zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig index d4f249b8..e8901fb1 100644 --- a/zml/kernels/moe/cute/persistent_mxfp4.zig +++ b/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig @@ -14,10 +14,27 @@ const std = @import("std"); const mlir = @import("mlir"); const nvvm = @import("mlir/dialects/nvvm"); -const zml = @import("../../../zml.zig"); +const zml = @import("../../../../zml.zig"); const cute = zml.kernel.cute; const B = cute.Builder; const V = cute.Value; +const common = @import("common.zig"); +const typed = common.typed; +const gridTyped = common.gridTyped; +const barrierAt = common.barrierAt; +const PipelinePosition = common.PipelinePosition; +const PersistentWork = common.PersistentWork; +const DirectRoutes = common.DirectRoutes; +const persistentWorkAt = common.persistentWorkAt; +const advancePersistentWork = common.advancePersistentWork; +const advancePipeline = common.advancePipeline; +const peekBarrierIfRemaining = common.peekBarrierIfRemaining; +const sharedPointerAt = common.sharedPointerAt; +const offsetAlignment = common.offsetAlignment; +const typesOf = common.typesOf; +const tupleOf = common.tupleOf; +const fmt = common.fmt; +const view = common.view; const accumulator_stages = 2; @@ -333,14 +350,6 @@ pub fn downRows(cfg: Config, a: Inputs, route_inverse: zml.Tensor) zml.Tensor { }, .{ .output = .init(.{ cfg.routes, cfg.m }, .bf16) }, .{ .cfg = cfg, .scalars = &.{ cfg.m, cfg.n, cfg.k, cfg.groups, cfg.experts } }).output; } -fn typed(type_: *const mlir.Type) struct { mlir_type: cute.ArgSpec.MlirTypeSpec } { - return .{ .mlir_type = .{ .type_ = type_ } }; -} - -fn gridTyped(type_: *const mlir.Type) struct { mlir_type: cute.ArgSpec.MlirTypeSpec } { - return .{ .mlir_type = .{ .type_ = type_, .grid_constant = true } }; -} - /// The two projections share this kernel but get distinct symbols, so a /// profile shows them as separate rows instead of one merged entry. fn deviceName(comptime epilogue: Epilogue) [:0]const u8 { @@ -759,135 +768,6 @@ fn issueTmaStage( b.copy(b.setTmaBarrier(exec_sfb, stage.load_barrier), .{sfb_source}, .{stage.sfb_destination}, null); } -fn barrierAt(b: *B, base: V, stage: V) V { - return b.addOffset( - base, - b.makeIntTuple(.{stage}), - b.ptrTy(.i64, .smem, 8) catch @panic("bad pipeline barrier pointer"), - ).value(); -} - -const PipelinePosition = struct { - index: V, - phase: V, -}; - -/// Scalar form of CUTLASS's StaticPersistentTileScheduler state for the -/// one-CTA DeepSeek specialization. Python carries the decoded M/N/L -/// coordinate, validity predicate, linear work index, and executed-tile -/// count through every role's `scf.while`. Keeping the same state shape is -/// important to the CuTe software-pipeline passes, even though N is always -/// zero for one tile per group. -const PersistentWork = struct { - m: V, - n: V, - group: V, - valid: V, - linear: V, - executed: V, -}; - -/// Direct routing (one token) runs one single-row group per route. Routes -/// owned by other expert-parallel ranks carry expert -1: the persistent tiles -/// cover the valid routes only, in route order, so they do not occupy a wave. -const DirectRoutes = struct { - expert_ids: cute.View, - routes: usize, - - fn count(self: DirectRoutes, b: *B) V { - var valid = b.constant(.i32, 0); - for (0..self.routes) |r| valid = valid.add(self.expertAt(b, r).ge(0).to(.i32)); - return valid; - } - - /// Route of the `k`-th valid route. - fn nth(self: DirectRoutes, b: *B, k: V) V { - var seen = b.constant(.i32, 0); - var route = b.constant(.i32, 0); - for (0..self.routes) |r| { - const valid = self.expertAt(b, r).ge(0); - route = b.select(valid.bitAnd(seen.eq(k)), b.constant(.i32, r), route); - seen = seen.add(valid.to(.i32)); - } - return route; - } - - fn expertAt(self: DirectRoutes, b: *B, r: usize) V { - return self.expert_ids.get(.{b.constant(.i32, r)}, .i32); - } -}; - -fn persistentWorkAt( - b: *B, - linear: V, - executed: V, - active: V, - tiles_m: V, - div_m: V, - div_n: V, - direct: ?DirectRoutes, -) PersistentWork { - // StaticPersistentTileScheduler uses two first-class FastDivmod objects - // to decode the column-major (M, N=1, L) problem layout. Preserve those - // operations even though ordinary rem/div is mathematically equivalent; - // CuTe's scheduler reconstruction and loop canonicalization key off these - // values and carry them through every persistent loop. - const group_and_m = b.fastDivmod(linear, div_m); - const group_and_n = b.fastDivmod(group_and_m[0], div_n); - const group = if (direct) |routes| routes.nth(b, group_and_n[0]) else group_and_n[0]; - return .{ - .m = b.makeWarpUniform(group_and_m[1]), - .n = b.makeWarpUniform(group_and_n[1]), - .group = b.makeWarpUniform(group), - .valid = b.makeWarpUniform(linear.lt(active.mul(tiles_m))), - .linear = linear, - .executed = executed, - }; -} - -fn advancePersistentWork( - b: *B, - linear: V, - executed: V, - persistent_clusters: V, - active: V, - tiles_m: V, - div_m: V, - div_n: V, - direct: ?DirectRoutes, -) PersistentWork { - return persistentWorkAt(b, linear.add(persistent_clusters), executed.add(1), active, tiles_m, div_m, div_n, direct); -} - -/// Equivalent to CUTLASS PipelineState.advance(). Keep the index and phase -/// as explicit loop-carried SSA values instead of recovering them with -/// division/remainder; the generated CuTe program then has the same circular -/// state transition as the Python DSL. -fn advancePipeline(b: *B, index: V, phase: V, comptime stages: usize) PipelinePosition { - const advanced = index.add(1); - const wraps = advanced.eq(stages); - const next_index = b.select(wraps, 0, advanced); - var next_phase = b.openIfElse(wraps, .{phase}); - { - next_phase.yieldThen(.{phase.bitXor(1)}); - } - { - next_phase.yieldElse(.{phase}); - } - return .{ .index = next_index, .phase = next_phase.results[0] }; -} - -fn peekBarrierIfRemaining(b: *B, base: V, count: V, limit: usize, state: PipelinePosition) V { - var peek = b.openIfElse(count.lt(limit), .{.i1}); - { - peek.yieldThen(.{b.mbarrierWaitParity(barrierAt(b, base, state.index), state.phase)}); - } - { - peek.yieldElse(.{b.constant(.i1, true)}); - } - return peek.results[0]; -} - fn accumulatorTmemAt(b: *B, comptime tile: Tile, tmem: V, stage: V) V { // The stage offset is a whole tile width, which `cute.add_offset` needs to // know: it derives the result alignment from what it knows of the offset. @@ -898,40 +778,6 @@ fn accumulatorTmemAt(b: *B, comptime tile: Tile, tmem: V, stage: V) V { ).value(); } -/// `cute.add_offset` infers the alignment a pointer keeps as the gcd of the -/// base alignment and the offset, and `cute.recast_iter` may not change it, so -/// the field's own alignment has to be what the offset leaves. -fn sharedPointerAt( - b: *B, - base: V, - comptime byte_offset: i64, - comptime dtype: cute.DType, - comptime alignment: u64, -) V { - const moved = comptime movedAlignment(byte_offset); - comptime std.debug.assert(moved >= alignment); - const byte_ptr = b.addOffset( - base, - b.makeIntTuple(.{byte_offset}), - b.ptrTy(.i8, .smem, moved) catch @panic("bad shared byte pointer"), - ); - return b.recastPointer(byte_ptr, dtype, .smem, moved); -} - -/// Alignment of `shared_base + byte_offset`, which the shared storage is -/// allocated with. -fn movedAlignment(comptime byte_offset: i64) u64 { - return offsetAlignment(1024, byte_offset, 1); -} - -/// `cute.add_offset` keeps the gcd of the operand alignment and the offset, -/// and the op's result type has to say so. -fn offsetAlignment(base_bytes: u64, offset_elements: i64, element_bytes: u64) u64 { - if (offset_elements == 0) return base_bytes; - const moved = @as(u64, @intCast(@abs(offset_elements))) * element_bytes; - return @max(std.math.gcd(base_bytes, moved), element_bytes); -} - /// Warp 5 runs independently over every persistent output tile. Keep the /// producer index and phase as loop-carried values across tiles, exactly like /// CUTLASS PipelineState. Only the per-tile count is reset to zero. @@ -1432,10 +1278,6 @@ fn runMmaConsumer( ); } -fn view(b: *B, value: V) cute.View { - return .{ .inner = value.inner, .kernel = b }; -} - /// Epilogue operands; which fields are set depends on the `Epilogue`. const EpilogueOperands = struct { group_sizes: cute.View, diff --git a/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig b/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig new file mode 100644 index 00000000..ac59735c --- /dev/null +++ b/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig @@ -0,0 +1,1693 @@ +//! Source-level SM100 NVFP4 grouped GEMM. +//! +//! The NVFP4 counterpart of `persistent_mxfp4.zig`, following the CuTe DSL +//! `Sm100BlockScaledPersistentDenseGemmKernel` with FP4 x FP4 operands, E4M3 +//! scale factors every 16 elements and the `kind::mxf4nvf4.block16` MMA. Both +//! operands are packed FP4 in global and shared memory (TMA format `U4`); one +//! pipeline stage holds a 256-deep K slice, consumed by four K = 64 MMAs. +//! +//! One CTA computes a 128xN output tile of one expert group: warp 5 drives +//! the TMA ring, warp 4 issues the S2T scale copies and UMMA, and warps 0-3 +//! drain the accumulator from tensor memory into one of the two epilogues. +//! The pipeline, scheduler and warp roles are shared with the MXFP4 kernel. +const std = @import("std"); +const mlir = @import("mlir"); +const nvvm = @import("mlir/dialects/nvvm"); + +const zml = @import("../../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const V = cute.Value; +const common = @import("common.zig"); +const nvfp4 = @import("nvfp4.zig"); + +const accumulator_stages = 2; +/// K elements of one pipeline stage: four K = 64 FP4 MMAs. +pub const k_tile = 256; +/// Largest dynamic shared-memory allocation of an SM100 CTA. +const smem_capacity = 232448; +const mma_kind: cute.BlockScaledMmaConfig.Kind = .nvfp4_nvfp4; + +/// Shared-memory plan for one tile width, laid out like the Python +/// `SharedStorage`: barriers, epilogue scratch, then the A, B, SFA and SFB +/// rings. The ring is as deep as the capacity allows. +pub const Tile = struct { + n: i64, + stages: usize, + smem_bytes: i64, + tmem_columns: u16, + empty_barriers: i64, + accumulator_full: i64, + accumulator_empty: i64, + tmem_dealloc: i64, + tmem_holding: i64, + c: i64, + a: i64, + b: i64, + sfa: i64, + sfb: i64, + + /// 128x256 packed FP4. + const a_stage_bytes = 128 * k_tile / 2; + /// One 128x16 E4M3 scale tile: four 512-byte 128x4 atoms. + const sf_stage_bytes = 128 * k_tile / 16; + const c_bytes = 512; + const header_bytes = 1024; + + pub fn of(comptime n: i64) Tile { + comptime { + std.debug.assert(n >= 8 and n <= 128 and @rem(n, 8) == 0); + const per_stage = a_stage_bytes + @divExact(n * k_tile, 2) + 2 * sf_stage_bytes; + const stages: usize = @intCast(@divTrunc(smem_capacity - header_bytes - c_bytes, per_stage)); + const s: i64 = @intCast(stages); + const a = header_bytes + 1024; + const b = a + s * a_stage_bytes; + const sfa = b + @divExact(s * n * k_tile, 2); + const sfb = sfa + s * sf_stage_bytes; + // Accumulators, then 16 SFA and 16 SFB columns. + const columns = std.math.ceilPowerOfTwo(u16, @max(32, accumulator_stages * n + 32)) catch unreachable; + return .{ + .n = n, + .stages = stages, + .smem_bytes = sfb + s * sf_stage_bytes, + .tmem_columns = columns, + .empty_barriers = 8 * s, + .accumulator_full = 16 * s, + .accumulator_empty = 16 * s + 16, + .tmem_dealloc = 16 * s + 32, + .tmem_holding = 16 * s + 40, + .c = header_bytes, + .a = a, + .b = b, + .sfa = sfa, + .sfb = sfb, + }; + } + } + + /// Routed rows an epilogue pass handles: one per warp lane. + pub fn chunkRows(self: Tile) i64 { + return @min(self.n, 32); + } + + /// Epilogue passes over the tile width. + pub fn chunks(self: Tile) usize { + return @intCast(@divExact(self.n, self.chunkRows())); + } + + fn bStageBytes(self: Tile) i64 { + return @divExact(self.n * k_tile, 2); + } + + /// TMA bytes landing in one stage: packed A and B, then both scale tiles. + fn transactionBytes(self: Tile) u32 { + return @intCast(a_stage_bytes + self.bStageBytes() + 2 * sf_stage_bytes); + } + + fn fields(self: Tile, b: *B) []const []const u8 { + const arena = b.arena.allocator(); + const s: i64 = @intCast(self.stages); + const out = arena.alloc([]const u8, 11) catch @panic("OOM"); + const Field = struct { []const u8, i64, i64 }; + const list = [_]Field{ + .{ "ab_full_mbar_ptr", 8 * s, 0 }, + .{ "ab_empty_mbar_ptr", 8 * s, self.empty_barriers }, + .{ "acc_full_mbar_ptr", 16, self.accumulator_full }, + .{ "acc_empty_mbar_ptr", 16, self.accumulator_empty }, + .{ "tmem_dealloc_mbar", 8, self.tmem_dealloc }, + .{ "tmem_holding_buf", 4, self.tmem_holding }, + .{ "sC", c_bytes, self.c }, + .{ "sA", s * a_stage_bytes, self.a }, + .{ "sB", s * self.bStageBytes(), self.b }, + .{ "sSFA", s * sf_stage_bytes, self.sfa }, + .{ "sSFB", s * sf_stage_bytes, self.sfb }, + }; + for (list, out) |field, *text| { + text.* = std.fmt.allocPrint(arena, "{s}:{d}:{d}", .{ field[0], field[1], field[2] }) catch @panic("OOM"); + } + return out; + } +}; + +const StageStorage = struct { + a: V, + b: V, + sfa: V, + sfb: V, + c: V, + load_barriers: V, + mma_barriers: V, + accumulator_full_barriers: V, + accumulator_empty_barriers: V, +}; + +const StageBuffers = struct { + a_destination: cute.View, + b_destination: cute.View, + sfa_destination: cute.View, + sfb_destination: cute.View, + load_barrier: V, + mma_barrier: V, +}; + +const StagedOperands = struct { + a: cute.View, + b: cute.View, + sfa: cute.View, + sfb: cute.View, +}; + +/// What the epilogue warps do with each finished 128xN accumulator tile. +pub const Epilogue = enum { + /// Up projection: per-expert dequantization scale, BF16 rounding, + /// SwiGLU of the interleaved gate/up rows (optionally clamped), the + /// routing weight when it precedes the down projection, BF16 rounding, + /// then NVFP4 quantization (E4M3 scale per 16 activations, scaled by the + /// per-expert activation global scale) straight into the down GEMM's + /// packed input and 128x4 scale layouts. + swiglu_nvfp4, + /// Down projection: per-expert dequantization scale and, when it follows + /// the down projection, the routing weight, then BF16 rows stored in + /// route order `[routes, m]`, skipping padding rows. + route_rows, +}; + +pub const Config = struct { + experts: i64, + /// GEMM output rows (2 x intermediate for gate/up, hidden for down). + m: i64, + /// Tile N: rows in one routed expert group. + n: i64 = 16, + /// Reduction dimension, a multiple of `k_tile`. + k: i64, + groups: i64, + /// Resident CTAs of the static persistent tile scheduler: one per SM. + persistent_ctas: i64 = 148, + epilogue: Epilogue, + /// Token/top-k routes; sizes the route-indexed epilogue operands. + routes: i64, + /// Clamp of both halves of the fused SwiGLU, if any. + swiglu_limit: ?f32 = null, + /// Multiply rows by their routing weight in this projection's epilogue. + routing_weights: bool = false, + /// Ungrouped routing: group `g` is route `g` and holds one live row. + direct: bool = false, + + /// Rows stored per group in the routed operands: one whole N tile. + pub fn groupRows(self: Config) i64 { + return self.n; + } + + pub fn routedRows(self: Config) i64 { + return self.groups * self.groupRows(); + } +}; + +/// `schedule` is the routing schedule `[group experts | group sizes | active +/// group count]`; `alpha` the per-expert FP32 dequantization scale of the +/// GEMM output. +const inputs = [_][:0]const u8{ + "weight", + "schedule", + "input_quant", + "weight_scale", + "input_scale", + "alpha", +}; + +/// `route_inverse` maps each physical routed row to its token/top-k route; +/// `act_scale` is the per-expert global scale of the down GEMM's NVFP4 input. +const UpQuantized = cute.Program(Config, .{ + .name = "zml_nvfp4_up_swiglu_zig", + .inputs = &(inputs ++ [_][:0]const u8{ "route_inverse", "routing_weights", "act_scale" }), + .outputs = &.{ "q", "s" }, + .run = buildProgram, +}); + +const DownRows = cute.Program(Config, .{ + .name = "zml_nvfp4_down_rows_zig", + .inputs = &(inputs ++ [_][:0]const u8{ "route_inverse", "routing_weights" }), + .outputs = &.{"output"}, + .run = buildProgram, +}); + +pub const Inputs = struct { + /// `[experts, m, k / 2]` packed FP4. + weight: zml.Tensor, + schedule: zml.Tensor, + /// `[groups * n, k / 2]` packed FP4. + input_quant: zml.Tensor, + /// `[experts, m, k / 16]` E4M3 bytes in the 128x4 layout. + weight_scale: zml.Tensor, + /// `[groups, k / 16 * 128]` E4M3 bytes in the 128x4 layout. + input_scale: zml.Tensor, + /// `[experts]` FP32. + alpha: zml.Tensor, +}; + +/// Up projection with the SwiGLU + NVFP4 epilogue: the routed packed FP4 +/// rows `[groups * n, m / 4]` and their 128x4 E4M3 scales +/// `[groups, m / 32 * 128]`, ready for the down GEMM. `routing_weights` is +/// `[routes]` FP32 and `act_scale` `[experts]` FP32. +pub fn upQuantized(cfg: Config, a: Inputs, route_inverse: zml.Tensor, routing_weights: zml.Tensor, act_scale: zml.Tensor) struct { q: zml.Tensor, s: zml.Tensor } { + std.debug.assert(cfg.epilogue == .swiglu_nvfp4); + const activations = @divExact(cfg.m, 2); + const result = UpQuantized.call(.{ + .weight = a.weight, + .schedule = a.schedule, + .input_quant = a.input_quant, + .weight_scale = a.weight_scale, + .input_scale = a.input_scale, + .alpha = a.alpha, + .route_inverse = route_inverse, + .routing_weights = routing_weights, + .act_scale = act_scale, + }, .{ + .q = .init(.{ cfg.routedRows(), @divExact(activations, 2) }, .u8), + .s = .init(.{ cfg.groups, @divExact(activations, 16) * 128 }, .u8), + }, .{ .cfg = cfg, .scalars = &.{ cfg.m, cfg.n, cfg.k, cfg.groups, cfg.experts } }); + return .{ .q = result.q, .s = result.s }; +} + +/// Down projection storing BF16 `[routes, m]` in token/top-k order. +pub fn downRows(cfg: Config, a: Inputs, route_inverse: zml.Tensor, routing_weights: zml.Tensor) zml.Tensor { + std.debug.assert(cfg.epilogue == .route_rows); + return DownRows.call(.{ + .weight = a.weight, + .schedule = a.schedule, + .input_quant = a.input_quant, + .weight_scale = a.weight_scale, + .input_scale = a.input_scale, + .alpha = a.alpha, + .route_inverse = route_inverse, + .routing_weights = routing_weights, + }, .{ .output = .init(.{ @max(cfg.routes, 1), cfg.m }, .bf16) }, .{ .cfg = cfg, .scalars = &.{ cfg.m, cfg.n, cfg.k, cfg.groups, cfg.experts } }).output; +} + +const typed = common.typed; +const gridTyped = common.gridTyped; +const fmt = common.fmt; +const view = common.view; +const barrierAt = common.barrierAt; +const PipelinePosition = common.PipelinePosition; +const DirectRoutes = common.DirectRoutes; +const persistentWorkAt = common.persistentWorkAt; +const advancePersistentWork = common.advancePersistentWork; +const advancePipeline = common.advancePipeline; +const peekBarrierIfRemaining = common.peekBarrierIfRemaining; +const sharedPointerAt = common.sharedPointerAt; +const offsetAlignment = common.offsetAlignment; +const typesOf = common.typesOf; +const tupleOf = common.tupleOf; + +fn deviceName(comptime epilogue: Epilogue) [:0]const u8 { + return switch (epilogue) { + .swiglu_nvfp4 => "nvfp4_sm100_persistent_grouped_gemm_up_swiglu", + .route_rows => "nvfp4_sm100_persistent_grouped_gemm_down_rows", + }; +} + +pub const tile_widths = [_]i64{ 8, 16, 32, 64, 128 }; + +fn buildProgram(b: *B, cfg: Config) cute.FinishError!void { + return switch (cfg.n) { + inline 8, 16, 32, 64, 128 => |n| switch (cfg.epilogue) { + inline else => |epilogue| buildTiledProgram(b, cfg, comptime Tile.of(n), epilogue), + }, + else => std.debug.panic("unsupported NVFP4 tile N {d}", .{cfg.n}), + }; +} + +/// Types and layouts shared by the device ABI and the host launch, as the +/// CuTe DSL derives them for FP4 x FP4 with 16-element E4M3 scale vectors. +fn Layouts(comptime tile: Tile) type { + return struct { + const n = tile.n; + const stages = tile.stages; + const ab_swizzle: B.Swizzle = .{ .bits = 3, .base = 4, .shift = 3 }; + + fn aCoord(b: *B, cfg: Config) cute.LayoutSpec { + return b.layoutSpec(.{ cfg.m, cfg.k, cfg.experts }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); + } + fn bCoord(b: *B, cfg: Config) cute.LayoutSpec { + return b.layoutSpec(.{ cfg.groupRows(), cfg.k, cfg.groups }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); + } + /// TMA coordinates of the 128x4 scale layout: byte in the atom, K + /// atom, M block, L. + fn sfCoord(b: *B) cute.LayoutSpec { + return b.layoutSpec( + .{ .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ .{ 16, 4 }, cute.AlgebraToken.dynamic }, .{ 1, cute.AlgebraToken.dynamic } }, + .{ + .{ .{ b.basis(8, 1, .{0}), b.basis(2, 1, .{0}) }, b.basis(1, 1, .{2}) }, + .{ .{ 0, b.basis(1, 2, .{0}) }, b.basis(1, 1, .{1}) }, + .{ 0, b.basis(1, 1, .{3}) }, + }, + ); + } + fn aBasis(b: *B) cute.LayoutSpec { + return b.layoutSpec(.{ k_tile, 128, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); + } + fn bBasis(b: *B) cute.LayoutSpec { + return b.layoutSpec(.{ k_tile, n, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); + } + fn sfBasis(b: *B) cute.LayoutSpec { + return b.layoutSpec( + .{ .{ 2, 4, 32 }, 4, 1, 1 }, + .{ .{ b.basis(1, 1, .{ 1, 0, 1 }), b.basis(1, 1, .{ 0, 0, 1 }), b.basis(1, 1, .{ 0, 0, 0 }) }, b.basis(1, 1, .{ 1, 1 }), b.basis(1, 1, .{ 0, 1 }), b.basis(1, 1, .{ 2, 1 }) }, + ); + } + fn tmaA(b: *B, cfg: Config) cute.TmaConfig { + return .{ .dtype = .f4e2m1fn, .copy_bits = 128 * k_tile * 4, .global_basis = aBasis(b), .coordinate_layout = aCoord(b, cfg), .coordinate_rank = 3, .format = .u4 }; + } + fn tmaB(b: *B, cfg: Config) cute.TmaConfig { + return .{ .dtype = .f4e2m1fn, .copy_bits = @intCast(n * k_tile * 4), .global_basis = bBasis(b), .coordinate_layout = bCoord(b, cfg), .coordinate_rank = 3, .format = .u4 }; + } + fn tmaSf(b: *B) cute.TmaConfig { + return .{ .dtype = .f8e4m3fn, .copy_bits = Tile.sf_stage_bytes * 8, .global_basis = sfBasis(b), .coordinate_layout = sfCoord(b), .coordinate_rank = 4, .format = .u16 }; + } + // Shared-memory layouts in FP4 elements for A/B, bytes for scales. + fn aSmem(b: *B, comptime staged: bool) cute.LayoutSpec { + return if (staged) + b.layoutSpec(.{ .{ 128, 64 }, 1, 4, stages }, .{ .{ k_tile, 1 }, 0, 64, 2 * Tile.a_stage_bytes }) + else + b.layoutSpec(.{ .{ 128, 64 }, 1, 4 }, .{ .{ k_tile, 1 }, 0, 64 }); + } + fn bSmem(b: *B, comptime staged: bool) cute.LayoutSpec { + return if (staged) + b.layoutSpec(.{ .{ n, 64 }, 1, 4, stages }, .{ .{ k_tile, 1 }, 0, 64, 2 * tile.bStageBytes() }) + else + b.layoutSpec(.{ .{ n, 64 }, 1, 4 }, .{ .{ k_tile, 1 }, 0, 64 }); + } + fn sfSmem(b: *B, comptime staged: bool) cute.LayoutSpec { + return if (staged) + b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 1 }, .{ 16, 4 } }, 1, 4, stages }, .{ .{ .{ .{ 16, 4 }, 0 }, .{ 0, 1 } }, 0, 512, Tile.sf_stage_bytes }) + else + b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 1 }, .{ 16, 4 } }, 1, 4 }, .{ .{ .{ .{ 16, 4 }, 0 }, .{ 0, 1 } }, 0, 512 }); + } + fn fp4SmemPtrTy(b: *B) *const mlir.Type { + return b.swizzledPtrTy(.f4e2m1fn, .smem, 1024, ab_swizzle) catch @panic("bad FP4 shared pointer"); + } + }; +} + +/// Device memrefs of the epilogue operands. +const EpilogueTypes = struct { + sizes: *const mlir.Type, + inverse: *const mlir.Type, + weights: *const mlir.Type, + per_expert: *const mlir.Type, + q: *const mlir.Type, + s: *const mlir.Type, + rows: *const mlir.Type, + + fn of(b: *B, cfg: Config) EpilogueTypes { + const activations = @divExact(cfg.m, 2); + return .{ + .sizes = b.memrefType(.i32, .gmem, 4, b.layoutType(b.layoutSpec(cfg.groups, 1))), + .inverse = b.memrefType(.i32, .gmem, 16, b.layoutType(b.layoutSpec(cfg.routedRows(), 1))), + .weights = b.memrefType(.f32, .gmem, 16, b.layoutType(b.layoutSpec(@max(cfg.routes, 1), 1))), + .per_expert = b.memrefType(.f32, .gmem, 16, b.layoutType(b.layoutSpec(cfg.experts, 1))), + // Eight packed FP4 activations per word. + .q = b.memrefType(.i32, .gmem, 16, b.layoutType(b.layoutSpec(.{ cfg.routedRows(), @divExact(activations, 8) }, .{ @divExact(activations, 8), 1 }))), + .s = b.memrefType(.i8, .gmem, 16, b.layoutType(b.layoutSpec(.{ cfg.groups, @divExact(activations, 16) * 128 }, .{ @divExact(activations, 16) * 128, 1 }))), + .rows = b.memrefType(.bf16, .gmem, 16, b.layoutType(b.layoutSpec(.{ @max(cfg.routes, 1), cfg.m }, .{ cfg.m, 1 }))), + }; + } +}; + +fn buildTiledProgram(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: Epilogue) cute.FinishError!void { + @setEvalBranchQuota(20_000); + std.debug.assert(@rem(cfg.m, 128) == 0); + std.debug.assert(@rem(cfg.k, k_tile) == 0); + const L = Layouts(tile); + const public_name = b.name; + + const ids_memref = b.memrefType(.i32, .gmem, 16, b.layoutType(b.layoutSpec(cfg.groups, 1))); + const active_memref = b.memrefType(.i32, .gmem, 4, b.layoutType(b.layoutSpec(1, 1))); + const et = EpilogueTypes.of(b, cfg); + const i32_type = cute.DType.i32.toMlir(b.ctx); + const mma_type = b.blockScaledMmaType(.{ .kind = mma_kind, .n = @intCast(tile.n) }); + const mma_sfb_type = b.blockScaledMmaType(.{ .kind = mma_kind, .n = 128 }); + + b.beginFunction(deviceName(epilogue), .cuda_kernel); + const d = try b.declareArgs(switch (epilogue) { + .swiglu_nvfp4 => .{ + .expert_ids = typed(ids_memref), + .active_count = typed(active_memref), + .mma = typed(mma_type), + .mma_sfb = typed(mma_sfb_type), + .tma_a = gridTyped(b.tmaLoadAtomType(L.tmaA(b, cfg))), + .gA = typed(b.coordTensorType(3, L.aCoord(b, cfg))), + .tma_b = gridTyped(b.tmaLoadAtomType(L.tmaB(b, cfg))), + .gB = typed(b.coordTensorType(3, L.bCoord(b, cfg))), + .tma_sfa = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), + .gSFA = typed(b.coordTensorType(4, L.sfCoord(b))), + .tma_sfb = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), + .gSFB = typed(b.coordTensorType(4, L.sfCoord(b))), + .alpha = typed(et.per_expert), + .group_sizes = typed(et.sizes), + .route_inverse = typed(et.inverse), + .routing_weights = typed(et.weights), + .act_scale = typed(et.per_expert), + .q = typed(et.q), + .s = typed(et.s), + .tiles_m = typed(i32_type), + }, + .route_rows => .{ + .expert_ids = typed(ids_memref), + .active_count = typed(active_memref), + .mma = typed(mma_type), + .mma_sfb = typed(mma_sfb_type), + .tma_a = gridTyped(b.tmaLoadAtomType(L.tmaA(b, cfg))), + .gA = typed(b.coordTensorType(3, L.aCoord(b, cfg))), + .tma_b = gridTyped(b.tmaLoadAtomType(L.tmaB(b, cfg))), + .gB = typed(b.coordTensorType(3, L.bCoord(b, cfg))), + .tma_sfa = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), + .gSFA = typed(b.coordTensorType(4, L.sfCoord(b))), + .tma_sfb = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), + .gSFB = typed(b.coordTensorType(4, L.sfCoord(b))), + .alpha = typed(et.per_expert), + .group_sizes = typed(et.sizes), + .route_inverse = typed(et.inverse), + .routing_weights = typed(et.weights), + .rows = typed(et.rows), + .tiles_m = typed(i32_type), + }, + }); + b.setFunctionAttribute(nvvm.minctasm(b.ctx, 1)); + b.setFunctionAttribute(.named(b.ctx, "smem.partition_num", .int(b.ctx, .i32, 2))); + b.setFunctionAttribute(.named(b.ctx, "smem.hint_smem_base_uniform", .unit(b.ctx))); + b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{ + .named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin()), + .named(b.ctx, "non_portable_cluster_size_allowed", .int(b.ctx, .i32, 1)), + }))); + + // Prefetch every tensor-map descriptor while the producing grid drains. + const warp = b.makeWarpUniform(b.makeWarpUniform(b.threadIdx().x.div(32))); + var tma_warp = b.openIf(warp.eq(5)); + inline for (.{ d.tma_a, d.tma_b, d.tma_sfa, d.tma_sfb }) |value| { + const descriptor: cute.Atom = .{ .inner = value.inner, .kernel = b }; + b.prefetchTmaDesc(descriptor); + } + tma_warp.close(); + b.waitForDependency(); + buildUmmaDevice(b, cfg, tile, epilogue, d); + b.launchDependents(); + b.endFunction(.{ 192, 1, 1 }); + + // Public host ABI: the XLA operands, the outputs, then the problem shape. + b.beginFunction(public_name, .host); + const operands = .{ + .weight = .{ .ptr = cute.DType.i8 }, + .schedule = .{ .ptr = cute.DType.i32 }, + .input_quant = .{ .ptr = cute.DType.i8 }, + .weight_scale = .{ .ptr = cute.DType.i8 }, + .input_scale = .{ .ptr = cute.DType.i8 }, + .alpha = .{ .ptr = cute.DType.f32 }, + .route_inverse = .{ .ptr = cute.DType.i32 }, + .routing_weights = .{ .ptr = cute.DType.f32 }, + }; + const shape = .{ + .problem_m = typed(i32_type), + .problem_n = typed(i32_type), + .problem_k = typed(i32_type), + .problem_l = typed(i32_type), + .expert_count = typed(i32_type), + }; + _ = try b.declareArgs(switch (epilogue) { + .swiglu_nvfp4 => .{ + .weight = operands.weight, + .schedule = operands.schedule, + .input_quant = operands.input_quant, + .weight_scale = operands.weight_scale, + .input_scale = operands.input_scale, + .alpha = operands.alpha, + .route_inverse = operands.route_inverse, + .routing_weights = operands.routing_weights, + .act_scale = .{ .ptr = cute.DType.f32 }, + .q = .{ .ptr = cute.DType.i8 }, + .s = .{ .ptr = cute.DType.i8 }, + .problem_m = shape.problem_m, + .problem_n = shape.problem_n, + .problem_k = shape.problem_k, + .problem_l = shape.problem_l, + .expert_count = shape.expert_count, + }, + .route_rows => .{ + .weight = operands.weight, + .schedule = operands.schedule, + .input_quant = operands.input_quant, + .weight_scale = operands.weight_scale, + .input_scale = operands.input_scale, + .alpha = operands.alpha, + .route_inverse = operands.route_inverse, + .routing_weights = operands.routing_weights, + .output = .{ .ptr = cute.DType.bf16 }, + .problem_m = shape.problem_m, + .problem_n = shape.problem_n, + .problem_k = shape.problem_k, + .problem_l = shape.problem_l, + .expert_count = shape.expert_count, + }, + }); + try buildHostLaunch(b, cfg, tile, epilogue); + b.endFunction(null); +} + +/// Host argument positions: the six mainloop operands, the route operands, +/// the epilogue's extra inputs and outputs, then the five shape scalars. +fn HostArgs(comptime epilogue: Epilogue) type { + return struct { + const weight = 0; + const schedule = 1; + const input_quant = 2; + const weight_scale = 3; + const input_scale = 4; + const alpha = 5; + const route_inverse = 6; + const routing_weights = 7; + const act_scale = 8; + const output = switch (epilogue) { + .swiglu_nvfp4 => 9, + .route_rows => 8, + }; + const scales = 10; + const problem_m = switch (epilogue) { + .swiglu_nvfp4 => 11, + .route_rows => 9, + }; + const problem_n = problem_m + 1; + const problem_k = problem_m + 2; + const problem_l = problem_m + 3; + const expert_count = problem_m + 4; + }; +} + +/// Complete staged shared-memory tensors. Per-stage views are slices. +fn stagedOperands(b: *B, comptime tile: Tile, storage: StageStorage) StagedOperands { + const L = Layouts(tile); + const a_full = b.makeView( + b.recastSwizzledPointer(storage.a, .f4e2m1fn, .smem, 1024, L.ab_swizzle), + b.staticLayout(L.aSmem(b, true)), + b.memrefTypeFromPointer(L.fp4SmemPtrTy(b), b.layoutType(L.aSmem(b, true))), + ); + const b_full = b.makeView( + b.recastSwizzledPointer(storage.b, .f4e2m1fn, .smem, 1024, L.ab_swizzle), + b.staticLayout(L.bSmem(b, true)), + b.memrefTypeFromPointer(L.fp4SmemPtrTy(b), b.layoutType(L.bSmem(b, true))), + ); + const sfa_full = b.makeView(storage.sfa, b.staticLayout(L.sfSmem(b, true)), b.memrefType(.f8e4m3fn, .smem, 1024, b.layoutType(L.sfSmem(b, true)))); + const sfb_full = b.makeView(storage.sfb, b.staticLayout(L.sfSmem(b, true)), b.memrefType(.f8e4m3fn, .smem, 1024, b.layoutType(L.sfSmem(b, true)))); + return .{ .a = a_full, .b = b_full, .sfa = sfa_full, .sfb = sfb_full }; +} + +fn stageBuffersAt(b: *B, comptime tile: Tile, storage: StageStorage, stage_index: V) StageBuffers { + const L = Layouts(tile); + const full = stagedOperands(b, tile, storage); + const all = cute.AlgebraToken.all; + const a = b.slice(full.a, .{ all, all, all, stage_index }, b.memrefTypeFromPointer(L.fp4SmemPtrTy(b), b.layoutType(L.aSmem(b, false)))); + const rhs = b.slice(full.b, .{ all, all, all, stage_index }, b.memrefTypeFromPointer(L.fp4SmemPtrTy(b), b.layoutType(L.bSmem(b, false)))); + const sf_stage_align = 1024; + const sfa = b.slice(full.sfa, .{ all, all, all, stage_index }, b.memrefType(.f8e4m3fn, .smem, sf_stage_align, b.layoutType(L.sfSmem(b, false)))); + const sfb = b.slice(full.sfb, .{ all, all, all, stage_index }, b.memrefType(.f8e4m3fn, .smem, sf_stage_align, b.layoutType(L.sfSmem(b, false)))); + + // `tma_partition` keeps a singleton CTA mode; the copy views keep it too. + const a_partition_layout = b.staticLayout(b.layoutSpec(.{ .{ 2 * Tile.a_stage_bytes, 1 }, .{1} }, .{ .{ 1, 0 }, .{0} })); + const b_partition_layout = b.staticLayout(b.layoutSpec(.{ .{ 2 * tile.bStageBytes(), 1 }, .{1} }, .{ .{ 1, 0 }, .{0} })); + const sf_partition_layout = b.staticLayout(b.layoutSpec(.{ .{ Tile.sf_stage_bytes, 1 }, .{1} }, .{ .{ 1, 0 }, .{0} })); + + return .{ + .a_destination = b.withLayout(a, a_partition_layout), + .b_destination = b.withLayout(rhs, b_partition_layout), + .sfa_destination = b.withLayout(sfa, sf_partition_layout), + .sfb_destination = b.withLayout(sfb, sf_partition_layout), + .load_barrier = barrierAt(b, storage.load_barriers, stage_index), + .mma_barrier = barrierAt(b, storage.mma_barriers, stage_index), + }; +} + +/// The TMA coordinates of one stage: slice the TMA-partitioned operand at +/// its output tile and L index, then at the K stage, and rebuild the +/// iterator as the Python front end does. +fn tmaSource(b: *B, partitioned: cute.View, tile_coord: anytype, k: V, comptime rows: i64, reduction_tiles: usize, copy_layout: cute.View) cute.View { + const all = cute.AlgebraToken.all; + const tile_k = b.slice( + partitioned, + tile_coord, + b.coordTensorTypeWithTuple(.{ 0, cute.ConstrainedDynamic{ .divisible_by = rows }, cute.AlgebraToken.dynamic }, b.layoutSpec( + .{ .{ .{ k_tile, rows }, 1 }, reduction_tiles }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(k_tile, 1, .{0}) }, + )), + ); + const div_k = cute.ConstrainedDynamic{ .divisible_by = k_tile }; + const div_rows = cute.ConstrainedDynamic{ .divisible_by = rows }; + const dynamic = cute.AlgebraToken.dynamic; + const stage = b.slice( + tile_k, + .{ all, k }, + b.coordTensorTypeWithTuple(.{ div_k, div_rows, dynamic }, b.layoutSpec( + .{.{ .{ k_tile, rows }, 1 }}, + .{.{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }}, + )), + ); + return b.rebuildCoordTensorIterator(3, stage, .{ div_k, div_rows, dynamic }, .{ k_tile, rows, 1 }, copy_layout); +} + +/// Issue the four transactions that fill one mainloop stage. +fn issueTmaStage( + b: *B, + comptime tile: Tile, + reduction_tiles: usize, + partitioned_a_target: cute.View, + partitioned_b_target: cute.View, + stage: StageBuffers, + k: V, + tile_m: V, + group: V, + expert: V, + exec_a: cute.Atom, + exec_b: cute.Atom, + exec_sfa: cute.Atom, + exec_sfb: cute.Atom, +) void { + const n = tile.n; + const all = cute.AlgebraToken.all; + const a_copy_layout = b.staticLayout(b.layoutSpec( + .{ .{ .{ k_tile, 128 }, 1 }, .{1} }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, .{0} }, + )); + const b_copy_layout = b.staticLayout(b.layoutSpec( + .{ .{ .{ k_tile, n }, 1 }, .{1} }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, .{0} }, + )); + // One stage moves four 512-byte atoms, one per K = 64 slice. + const sf_copy_spec = b.layoutSpec( + .{ .{ .{ 512, 4 }, 1 }, .{1} }, + .{ .{ .{ b.basis(1, 2, .{0}), b.basis(1, 1, .{1}) }, 0 }, .{0} }, + ); + // B's tile spans all of a group's rows, so its N-tile index is zero. Pass + // it as a value: a static zero leaf collapses out of the coordinate tuple + // and shifts the remaining tensor-map coordinates. + const b_tile_n = b.constant(.i32, 0); + const a_source = tmaSource(b, partitioned_a_target, .{ all, tile_m, all, expert }, k, 128, reduction_tiles, a_copy_layout); + const b_source = tmaSource(b, partitioned_b_target, .{ all, b_tile_n, all, group }, k, n, reduction_tiles, b_copy_layout); + // Scale coordinates: (byte, K atom, M block, L); a stage covers 4 K atoms. + const k_atom = k.mul(4); + const sfa_source = b.makeCoordTensor(.{ 0, k_atom, tile_m, expert }, b.staticLayout(sf_copy_spec)); + const sfb_source = b.makeCoordTensor(.{ 0, k_atom, b.constant(.i32, 0), group }, b.staticLayout(sf_copy_spec)); + + var elected = b.openIf(b.electSync()); + b.mbarrierArriveExpectTx(stage.load_barrier, tile.transactionBytes()); + elected.close(); + b.copy(b.setTmaBarrier(exec_a, stage.load_barrier), .{a_source}, .{stage.a_destination}, null); + b.copy(b.setTmaBarrier(exec_b, stage.load_barrier), .{b_source}, .{stage.b_destination}, null); + b.copy(b.setTmaBarrier(exec_sfa, stage.load_barrier), .{sfa_source}, .{stage.sfa_destination}, null); + b.copy(b.setTmaBarrier(exec_sfb, stage.load_barrier), .{sfb_source}, .{stage.sfb_destination}, null); +} + +fn accumulatorTmemAt(b: *B, comptime tile: Tile, tmem: V, stage: V) V { + return b.addOffset( + tmem, + b.makeIntTupleDivBy(stage.mul(tile.n), @intCast(tile.n)), + b.ptrTy(.f32, .tmem, 16) catch @panic("bad accumulator TMEM pointer"), + ).value(); +} + +/// Warp 5 runs independently over every persistent output tile. +fn runTmaProducer( + b: *B, + comptime tile: Tile, + k_tiles: usize, + storage: StageStorage, + active: V, + tiles_m: V, + expert_ids: cute.View, + direct: ?DirectRoutes, + partitioned_a_target: cute.View, + partitioned_b_target: cute.View, + exec_a: cute.Atom, + exec_b: cute.Atom, + exec_sfa: cute.Atom, + exec_sfb: cute.Atom, +) void { + const initial_index = b.constant(.i32, 0); + const initial_phase = b.constant(.i32, 1); + const initial_count = b.constant(.i32, 0); + const persistent_clusters = b.gridDim().z; + const cta_m = b.blockIdx().x; + const cta_n = b.blockIdx().y; + const cta_l = b.constant(.i32, 0); + const div_m = b.fastDivmodCreate(tiles_m); + const div_n = b.fastDivmodCreate(1); + const initial_work = persistentWorkAt(b, b.blockIdx().z, b.constant(.i32, 0), active, tiles_m, div_m, div_n, direct); + const carried = .{ + initial_work.m, initial_work.n, initial_work.group, initial_work.valid, + initial_count, initial_index, initial_phase, persistent_clusters, + initial_work.linear, cta_m, cta_n, cta_l, + initial_work.executed, tiles_m, active, div_m, + div_n, + }; + var tiles = b.openWhile(carried, typesOf(carried)); + tiles.yieldBefore(tiles.before_carried[3], tupleOf(tiles.before_carried)); + const tile_m = tiles.after_carried[0]; + const group = tiles.after_carried[2]; + const expert = expert_ids.get(.{group}, .i32); + const first_state: PipelinePosition = .{ + .index = tiles.after_carried[5], + .phase = tiles.after_carried[6], + }; + const first_ready = b.mbarrierWaitParity( + barrierAt(b, storage.mma_barriers, first_state.index), + first_state.phase, + ); + var stages = b.openFor( + 0, + @as(i32, @intCast(k_tiles)), + 1, + .{ first_ready, b.constant(.i32, 0), first_state.index, first_state.phase }, + ); + const k = stages.carried[1]; + const state: PipelinePosition = .{ + .index = stages.carried[2], + .phase = stages.carried[3], + }; + const buffers = stageBuffersAt(b, tile, storage, state.index); + var pending = b.openIf(stages.carried[0].eq(false)); + b.mbarrierTryWaitParity(buffers.mma_barrier, state.phase, 10_000_000); + pending.close(); + issueTmaStage(b, tile, k_tiles, partitioned_a_target, partitioned_b_target, buffers, k, tile_m, group, expert, exec_a, exec_b, exec_sfa, exec_sfb); + const next_count = stages.carried[1].add(1); + const next_state = advancePipeline(b, state.index, state.phase, tile.stages); + const next_ready = peekBarrierIfRemaining(b, storage.mma_barriers, next_count, k_tiles, next_state); + stages.yield(.{ next_ready, next_count, next_state.index, next_state.phase }); + const next_work = advancePersistentWork( + b, + tiles.after_carried[8], + tiles.after_carried[12], + tiles.after_carried[7], + tiles.after_carried[14], + tiles.after_carried[13], + tiles.after_carried[15], + tiles.after_carried[16], + direct, + ); + tiles.yieldAfter(.{ + next_work.m, + next_work.n, + next_work.group, + next_work.valid, + stages.results[1], + stages.results[2], + stages.results[3], + tiles.after_carried[7], + next_work.linear, + tiles.after_carried[9], + tiles.after_carried[10], + tiles.after_carried[11], + next_work.executed, + tiles.after_carried[13], + tiles.after_carried[14], + tiles.after_carried[15], + tiles.after_carried[16], + }); + + // PipelineTmaUmma.producer_tail: drain every empty barrier before exit. + var tail_state: PipelinePosition = .{ + .index = tiles.results[5], + .phase = tiles.results[6], + }; + inline for (0..tile.stages) |stage| { + b.mbarrierTryWaitParity( + barrierAt(b, storage.mma_barriers, tail_state.index), + tail_state.phase, + 10_000_000, + ); + if (stage + 1 < tile.stages) { + tail_state = advancePipeline(b, tail_state.index, tail_state.phase, tile.stages); + } + } +} + +/// NVFP4 mainloop objects of one pipeline stage. +const Mainloop = struct { + a: V, + b: V, + accumulator: V, + sfa: V, + sfb: V, + sfa_copy: cute.Atom, + sfb_copy: cute.Atom, + sfa_source: V, + sfb_source: V, + sfa_target: V, + sfb_target: V, + + pub fn copyScales(self: Mainloop, builder: *B) void { + builder.copy(self.sfa_copy, .{self.sfa_source}, .{self.sfa_target}, null); + builder.copy(self.sfb_copy, .{self.sfb_source}, .{self.sfb_target}, null); + } +}; + +/// Pipeline-wide UMMA and S2T descriptors derived from the complete staged +/// tensors once; `atStage` slices them at the current stage index. +const StagedMainloop = struct { + a: cute.View, + b: cute.View, + sfa_source: cute.View, + sfb_source: cute.View, + sfa: V, + sfb: V, + sfa_copy: cute.Atom, + sfb_copy: cute.Atom, + sfa_target: V, + sfb_target: V, + n: i64, + + pub fn atStage(self: StagedMainloop, b: *B, index: V, accumulator_tmem: V) Mainloop { + const all = cute.AlgebraToken.all; + const a = b.slice(self.a, .{ all, all, all, index }, b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4 }, .{ 0, 0, 2 }))); + const rhs = b.slice(self.b, .{ all, all, all, index }, b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4 }, .{ 0, 0, 2 }))); + const sfa_stage = b.slice(self.sfa_source, .{ all, all, all, all, index }, b.smemDescViewType(b.layoutSpec(.{ .{ .{ .{ 32, 1, 1 }, 4 }, 1 }, 1, 1, 4 }, .{ .{ .{ .{ 1, 1, 1 }, 0 }, 0 }, 0, 0, 32 }))); + const sfb_stage = b.slice(self.sfb_source, .{ all, all, all, all, index }, b.smemDescViewType(b.layoutSpec(.{ .{ .{ .{ 32, 1, 1 }, 4 }, 1 }, 1, 1, 4 }, .{ .{ .{ .{ 1, 1, 1 }, 0 }, 0 }, 0, 0, 32 }))); + const source_grouped_spec = b.layoutSpec(.{ .{ .{ .{ 32, 1, 1 }, 4 }, 1 }, .{ 1, 1, 4 } }, .{ .{ .{ .{ 1, 1, 1 }, 0 }, 0 }, .{ 0, 0, 32 } }); + const source_grouped_layout = b.staticLayout(source_grouped_spec); + const source_grouped_ty = b.smemDescViewType(source_grouped_spec); + const sfa_source = b.makeView(b.getIter(sfa_stage, b.smemDescType()), source_grouped_layout, source_grouped_ty); + const sfb_source = b.makeView(b.getIter(sfb_stage, b.smemDescType()), source_grouped_layout, source_grouped_ty); + const acc_spec = b.layoutSpec(.{ .{ 128, self.n }, 1, 1 }, .{ .{ 65536, 1 }, 0, 0 }); + const acc_layout = b.staticLayout(acc_spec); + const accumulator = b.makeView( + accumulator_tmem, + acc_layout, + b.memrefType(.f32, .tmem, 16, b.layoutType(acc_spec)), + ); + return .{ + .a = a.value(), + .b = rhs.value(), + .accumulator = accumulator.value(), + .sfa = self.sfa, + .sfb = self.sfb, + .sfa_copy = self.sfa_copy, + .sfb_copy = self.sfb_copy, + .sfa_source = sfa_source.value(), + .sfb_source = sfb_source.value(), + .sfa_target = self.sfa_target, + .sfb_target = self.sfb_target, + }; + } +}; + +fn makeStagedMainloop( + b: *B, + tiled_mma: cute.Atom, + comptime n: i64, + comptime stages: usize, + smem_a: V, + smem_b: V, + smem_sfa: V, + smem_sfb: V, + scale_tmem: V, +) StagedMainloop { + // Descriptor strides are in 16-byte units: one 64-deep K slice of packed + // FP4 is 32 bytes, a 128x256 A stage 16 KiB and an Nx256 B stage N*128 bytes. + const frag_a = b.mmaMakeFragment( + tiled_mma, + smem_a, + 0, + b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4, stages }, .{ 0, 0, 2, Tile.a_stage_bytes / 16 })), + ); + const frag_b = b.mmaMakeFragment( + tiled_mma, + smem_b, + 1, + b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4, stages }, .{ 0, 0, 2, n * 8 })), + ); + + // TMEM columns: two N-wide accumulator stages, then 16 SFA and 16 SFB + // columns: four K = 64 slices of four E4M3 scales per lane each. + const sf_spec = b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 4 }, .{ 16, 4 } }, 1, 4 }, .{ .{ .{ .{ 262144, 4 }, 8388608 }, .{ 0, 1 } }, 0, 16 }); + const sf_layout = b.staticLayout(sf_spec); + const sfa_col = b.addOffset(scale_tmem, b.staticIntTuple(2 * n), b.ptrTy(.f32, .tmem, 16) catch @panic("bad TMEM pointer")); + const sfb_col = b.addOffset(scale_tmem, b.staticIntTuple(2 * n + 16), b.ptrTy(.f32, .tmem, 16) catch @panic("bad TMEM pointer")); + const sfa_ptr = b.recastIter(sfa_col, b.ptrTy(.f8e4m3fn, .tmem, 16) catch @panic("bad scale-factor TMEM pointer")); + const sfb_ptr = b.recastIter(sfb_col, b.ptrTy(.f8e4m3fn, .tmem, 16) catch @panic("bad scale-factor TMEM pointer")); + const sf_memref = b.memrefType(.f8e4m3fn, .tmem, 16, b.layoutType(sf_spec)); + const sfa = b.makeView(sfa_ptr, sf_layout, sf_memref); + const sfb = b.makeView(sfb_ptr, sf_layout, sf_memref); + + const smem_filtered_ty = b.memrefType(.f8e4m3fn, .smem, 1024, b.layoutType(b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 1 }, .{ 1, 4 } }, 1, 4, stages }, .{ .{ .{ .{ 16, 4 }, 0 }, .{ 0, 1 } }, 0, 512, 2048 }))); + const tmem_filtered_ty = b.memrefType(.f8e4m3fn, .tmem, 16, b.layoutType(b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 4 }, .{ 1, 4 } }, 1, 4 }, .{ .{ .{ .{ 262144, 4 }, 8388608 }, .{ 0, 1 } }, 0, 16 }))); + const sfa_smem_filtered = b.filterZeros(smem_sfa, null, smem_filtered_ty); + const sfb_smem_filtered = b.filterZeros(smem_sfb, null, smem_filtered_ty); + const sfa_tmem_filtered = b.filterZeros(sfa, null, tmem_filtered_ty); + const sfb_tmem_filtered = b.filterZeros(sfb, null, tmem_filtered_ty); + + const copy_atom_ty = b.s2tCopyAtomType(.f8e4m3fn, .{ .num_dp = 32, .num_bit = 128, .broadcast = .x4 }); + const copy_atom = b.makeAtom(copy_atom_ty, .{}); + const copy_ty = b.tiledCopyType(copy_atom_ty, b.layoutSpec(.{ 1, .{ 32, .{ 4, 4 }, 4 } }, .{ 0, .{ 1, .{ 32, 128 }, 512 } }), &.{ b.layoutSpec(.{ 32, 4, 16 }, .{ 1, 512, 32 }), b.layoutSpec(1, 0), b.layoutSpec(1, 0) }); + const sfa_copy = b.makeS2tCopy(copy_atom, sfa_tmem_filtered, copy_ty); + const sfb_copy = b.makeS2tCopy(copy_atom, sfb_tmem_filtered, copy_ty); + + const source_partition_ty = b.memrefType(.f8e4m3fn, .smem, 1024, b.layoutType(b.layoutSpec(.{ .{ .{ .{ 32, 4, 4 }, 4 }, 1 }, 1, 1, 4, stages }, .{ .{ .{ .{ 16, 4, 4 }, 0 }, 0 }, 0, 0, 512, 2048 }))); + const source_desc_ty = b.smemDescViewType(b.layoutSpec(.{ .{ .{ .{ 32, 1, 1 }, 4 }, 1 }, 1, 1, 4, stages }, .{ .{ .{ .{ 1, 1, 1 }, 0 }, 0 }, 0, 0, 32, 128 })); + const target_partition_ty = b.memrefType(.f8e4m3fn, .tmem, 16, b.layoutType(b.layoutSpec(.{ .{ .{ 32, 16, 4 }, 1 }, 1, 1, 4 }, .{ .{ .{ 262144, 1, 8388608 }, 0 }, 0, 0, 16 }))); + const target_grouped_spec = b.layoutSpec(.{ .{ .{ 32, 16, 4 }, 1 }, .{ 1, 1, 4 } }, .{ .{ .{ 262144, 1, 8388608 }, 0 }, .{ 0, 0, 16 } }); + const target_grouped_ty = b.memrefType(.f8e4m3fn, .tmem, 16, b.layoutType(target_grouped_spec)); + const sfa_src_partition = b.tiledCopyPartition(sfa_copy, sfa_smem_filtered, .{0}, false, source_partition_ty); + const sfb_src_partition = b.tiledCopyPartition(sfb_copy, sfb_smem_filtered, .{0}, false, source_partition_ty); + const sfa_desc = b.s2tSmemDescriptor(sfa_copy, sfa_src_partition, source_desc_ty); + const sfb_desc = b.s2tSmemDescriptor(sfb_copy, sfb_src_partition, source_desc_ty); + const sfa_dst_partition = b.tiledCopyPartition(sfa_copy, sfa_tmem_filtered, .{0}, true, target_partition_ty); + const sfb_dst_partition = b.tiledCopyPartition(sfb_copy, sfb_tmem_filtered, .{0}, true, target_partition_ty); + const target_grouped_layout = b.staticLayout(target_grouped_spec); + const sfa_dst = b.makeView(b.getIter(sfa_dst_partition, b.ptrTy(.f8e4m3fn, .tmem, 16) catch @panic("bad scale-factor TMEM pointer")), target_grouped_layout, target_grouped_ty); + const sfb_dst = b.makeView(b.getIter(sfb_dst_partition, b.ptrTy(.f8e4m3fn, .tmem, 16) catch @panic("bad scale-factor TMEM pointer")), target_grouped_layout, target_grouped_ty); + + return .{ + .a = frag_a, + .b = frag_b, + .sfa_source = sfa_desc, + .sfb_source = sfb_desc, + .sfa = sfa.value(), + .sfb = sfb.value(), + .sfa_copy = sfa_copy, + .sfb_copy = sfb_copy, + .sfa_target = sfa_dst.value(), + .sfb_target = sfb_dst.value(), + .n = n, + }; +} + +/// Warp 4 drains the A/B ring into one of two independent accumulator stages. +fn runMmaConsumer( + b: *B, + comptime tile: Tile, + k_tiles: usize, + storage: StageStorage, + tiled_mma: cute.Atom, + tmem_holding: V, + active: V, + tiles_m: V, +) void { + const tmem = b.retrieveTmemPtr(tmem_holding, .f32, 16); + const staged = stagedOperands(b, tile, storage); + const staged_mainloop = makeStagedMainloop( + b, + tiled_mma, + tile.n, + tile.stages, + staged.a.value(), + staged.b.value(), + staged.sfa.value(), + staged.sfb.value(), + tmem, + ); + const persistent_clusters = b.gridDim().z; + const div_m = b.fastDivmodCreate(tiles_m); + const div_n = b.fastDivmodCreate(1); + const initial_work = persistentWorkAt(b, b.blockIdx().z, b.constant(.i32, 0), active, tiles_m, div_m, div_n, null); + const carried = .{ + initial_work.m, initial_work.n, initial_work.group, initial_work.valid, + b.constant(.i32, 0), b.constant(.i32, 0), b.constant(.i32, 0), tiled_mma.value(), + b.constant(.i32, 0), b.constant(.i32, 0), b.constant(.i32, 1), persistent_clusters, + initial_work.linear, b.blockIdx().x, b.blockIdx().y, b.constant(.i32, 0), + initial_work.executed, tiles_m, active, div_m, + div_n, + }; + var tiles = b.openWhile(carried, typesOf(carried)); + tiles.yieldBefore(tiles.before_carried[3], tupleOf(tiles.before_carried)); + const first_state: PipelinePosition = .{ + .index = tiles.after_carried[5], + .phase = tiles.after_carried[6], + }; + const first_ready = b.mbarrierWaitParity( + barrierAt(b, storage.load_barriers, first_state.index), + first_state.phase, + ); + const accumulator_state: PipelinePosition = .{ + .index = tiles.after_carried[9], + .phase = tiles.after_carried[10], + }; + const accumulator_stage = accumulator_state.index; + const accumulator_empty = barrierAt(b, storage.accumulator_empty_barriers, accumulator_stage); + b.mbarrierTryWaitParity(accumulator_empty, accumulator_state.phase, 10_000_000); + + const persistent_mma: cute.Atom = .{ .inner = tiles.after_carried[7].inner, .kernel = b }; + const initial_mma = b.setMmaAccumulate(persistent_mma, b.constant(.i1, false)); + var stages = b.openFor( + 0, + @as(i32, @intCast(k_tiles)), + 1, + .{ first_ready, initial_mma.value(), b.constant(.i32, 0), first_state.index, first_state.phase }, + ); + const k = stages.carried[2]; + const state: PipelinePosition = .{ .index = stages.carried[3], .phase = stages.carried[4] }; + var pending = b.openIf(stages.carried[0].eq(false)); + b.mbarrierTryWaitParity(barrierAt(b, storage.load_barriers, state.index), state.phase, 10_000_000); + pending.close(); + const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, accumulator_stage); + const mainloop = staged_mainloop.atStage(b, state.index, accumulator_tmem); + mainloop.copyScales(b); + const loop_mma: cute.Atom = .{ .inner = stages.carried[1].inner, .kernel = b }; + const mma = b.setMmaAccumulate(loop_mma, k.ne(0)); + b.gemm( + mma, + mainloop.accumulator, + .{ mainloop.a, mainloop.sfa }, + .{ mainloop.b, mainloop.sfb }, + mainloop.accumulator, + ); + var commit_lane = b.openIf(b.electSync()); + b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index)); + commit_lane.close(); + const next_count = stages.carried[2].add(1); + const next_state = advancePipeline(b, state.index, state.phase, tile.stages); + const next_ready = peekBarrierIfRemaining(b, storage.load_barriers, next_count, k_tiles, next_state); + stages.yield(.{ next_ready, mma.value(), next_count, next_state.index, next_state.phase }); + + var full_lane = b.openIf(b.electSync()); + b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage)); + full_lane.close(); + const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); + const next_acc_count = tiles.after_carried[8].add(1); + const next_work = advancePersistentWork( + b, + tiles.after_carried[12], + tiles.after_carried[16], + tiles.after_carried[11], + tiles.after_carried[18], + tiles.after_carried[17], + tiles.after_carried[19], + tiles.after_carried[20], + null, + ); + tiles.yieldAfter(.{ + next_work.m, + next_work.n, + next_work.group, + next_work.valid, + stages.results[2], + stages.results[3], + stages.results[4], + stages.results[1], + next_acc_count, + next_accumulator_state.index, + next_accumulator_state.phase, + tiles.after_carried[11], + next_work.linear, + tiles.after_carried[13], + tiles.after_carried[14], + tiles.after_carried[15], + next_work.executed, + tiles.after_carried[17], + tiles.after_carried[18], + tiles.after_carried[19], + tiles.after_carried[20], + }); + + // PipelineUmmaAsync.producer_tail. + var accumulator_tail: PipelinePosition = .{ + .index = tiles.results[9], + .phase = tiles.results[10], + }; + inline for (0..accumulator_stages - 1) |_| { + accumulator_tail = advancePipeline(b, accumulator_tail.index, accumulator_tail.phase, accumulator_stages); + } + b.mbarrierTryWaitParity( + barrierAt(b, storage.accumulator_empty_barriers, accumulator_tail.index), + accumulator_tail.phase, + 10_000_000, + ); +} + +/// Epilogue operands; which fields are set depends on the `Epilogue`. +const EpilogueOperands = struct { + expert_ids: cute.View, + alpha: cute.View, + group_sizes: cute.View, + route_inverse: cute.View, + routing_weights: ?cute.View = null, + act_scale: ?cute.View = null, + q: ?cute.View = null, + s: ?cute.View = null, + rows: ?cute.View = null, + direct: bool, + group_rows: i64 = 0, + swiglu_limit: ?f32 = null, +}; + +/// Warps 0-3 drain each TMEM accumulator stage into registers, release it to +/// the MMA warp, then write the tile out according to `epilogue`. +fn runEpilogue( + b: *B, + comptime tile: Tile, + comptime epilogue: Epilogue, + storage: StageStorage, + tmem_holding: V, + active: V, + tiles_m: V, + direct: ?DirectRoutes, + operands: EpilogueOperands, +) void { + const n = tile.n; + const tid = b.threadIdx().x; + const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); + const tmem = b.retrieveTmemPtr(tmem_holding, .f32, 16); + + const div_m = b.fastDivmodCreate(tiles_m); + const div_n = b.fastDivmodCreate(1); + const initial_work = persistentWorkAt(b, b.blockIdx().z, b.constant(.i32, 0), active, tiles_m, div_m, div_n, direct); + const carried = .{ + initial_work.m, initial_work.n, initial_work.group, initial_work.valid, + b.constant(.i32, 0), b.constant(.i32, 0), b.constant(.i32, 0), initial_work.linear, + initial_work.executed, + }; + var tiles = b.openWhile(carried, .{ + initial_work.m.type_(), + initial_work.n.type_(), + initial_work.group.type_(), + cute.DType.i32.toMlir(b.ctx), + cute.DType.i32.toMlir(b.ctx), + cute.DType.i32.toMlir(b.ctx), + initial_work.linear.type_(), + initial_work.executed.type_(), + }); + tiles.yieldBefore(tiles.before_carried[3], .{ + tiles.before_carried[0], + tiles.before_carried[1], + tiles.before_carried[2], + tiles.before_carried[4], + tiles.before_carried[5], + tiles.before_carried[6], + tiles.before_carried[7], + tiles.before_carried[8], + }); + const tile_m = tiles.after_carried[0]; + const group = tiles.after_carried[2]; + const accumulator_state: PipelinePosition = .{ + .index = tiles.after_carried[4], + .phase = tiles.after_carried[5], + }; + const stage = accumulator_state.index; + // Per-expert scalars do not depend on the accumulator. + const expert = operands.expert_ids.get(.{group}, .i32); + const alpha = operands.alpha.get(.{expert}, .f32); + const act_scale = if (operands.act_scale) |scales| scales.get(.{expert}, .f32) else null; + b.mbarrierTryWaitParity(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, 10_000_000); + + const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, stage); + const warp_tmem = b.addOffset( + accumulator_tmem, + b.makeIntTupleDivBy(warp.mul(32 * 65536), 4), + b.ptrTy(.f32, .tmem, 16) catch @panic("bad epilogue TMEM pointer"), + ).value(); + const chunk_rows: usize = comptime @intCast(tile.chunkRows()); + const tile_chunks = comptime tile.chunks(); + inline for (0..tile_chunks) |chunk| { + const chunk_tmem = if (chunk == 0) warp_tmem else b.addOffset( + warp_tmem, + b.makeIntTuple(.{@as(i32, @intCast(chunk * chunk_rows))}), + b.ptrTy(.f32, .tmem, offsetAlignment(16, @intCast(chunk * chunk_rows), 4)) catch @panic("bad epilogue TMEM pointer"), + ).value(); + const routed = routedLanes(b, tile, operands, group, chunk); + const values = b.tmemLoadVector(chunk_tmem, @intCast(chunk_rows), .{ .num_dp = 32, .num_b = 32, .num_rep = @intCast(chunk_rows) }); + b.fenceTmemLoad(); + if (chunk + 1 == tile_chunks) { + b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), 1); + } + var columns: [@intCast(@min(n, 32))]V = undefined; + inline for (&columns, 0..) |*column, i| column.* = b.vectorExtract(values, i, .i32).bitCast(.f32).mul(alpha); + switch (epilogue) { + .swiglu_nvfp4 => storeTileSwiglu(b, tile, operands, routed, tile_m, group, act_scale.?, &columns, chunk), + .route_rows => storeTileRows(b, tile, operands, routed, tile_m, &columns, chunk), + } + } + const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); + const next_work = advancePersistentWork(b, tiles.after_carried[6], tiles.after_carried[7], b.gridDim().z, active, tiles_m, div_m, div_n, direct); + tiles.yieldAfter(.{ + next_work.m, + next_work.n, + next_work.group, + next_work.valid, + tiles.after_carried[3].add(1), + next_accumulator_state.index, + next_accumulator_state.phase, + next_work.linear, + next_work.executed, + }); + + var release_warp = b.openIf(warp.eq(0)); + b.relinquishTmemAllocPermit(.{}); + release_warp.close(); + b.namedBarrier(1, 128); + var dealloc_warp = b.openIf(warp.eq(0)); + b.deallocTmem(tmem, tile.tmem_columns, .{}); + dealloc_warp.close(); +} + +/// Per-row routing data of one expert group, spread across a warp: lane `l` +/// holds routed row `l` of the current chunk. +const RoutedLanes = struct { + count: V, + /// Token/top-k route of the lane's row; 0 for padding rows. + route: V, + /// Routing weight of the lane's row, when this projection applies it. + weight: ?V, +}; + +fn routedLanes(b: *B, comptime tile: Tile, operands: EpilogueOperands, group: V, comptime chunk: usize) RoutedLanes { + const weights = operands.routing_weights; + if (operands.direct) return .{ + .count = b.constant(.i32, 1), + .route = group, + .weight = if (weights) |w| w.get(.{group}, .f32) else null, + }; + const lane = b.threadIdx().x.rem(32); + const count = operands.group_sizes.get(.{group}, .i32); + // Lanes past the tile width alias row 0, which is always live. + const chunk_rows = comptime tile.chunkRows(); + const in_chunk = b.select(lane.lt(@as(i32, @intCast(chunk_rows))), lane, b.constant(.i32, 0)); + const row_in_group = if (chunk == 0) in_chunk else in_chunk.add(@as(i32, @intCast(@as(i64, @intCast(chunk)) * chunk_rows))); + const live = row_in_group.lt(count); + const inverse = operands.route_inverse.get(.{group.mul(tile.n).add(row_in_group)}, .i32); + const route = b.select(live, inverse, b.constant(.i32, 0)); + const weight = if (weights) |w| w.get(.{route}, .f32) else null; + return .{ .count = count, .route = route, .weight = weight }; +} + +const swiglu_chunk = 8; + +/// Thread `t` owns output row `128 * tile_m + t` of the interleaved +/// projection: even threads hold a gate and their odd neighbour the matching +/// up value, so each even thread computes one SwiGLU activation per routed +/// row. A warp's 16 activations are exactly one NVFP4 block, whose maximum is +/// a single warp reduction. Activation `a` of row `r` lands in nibble `a` of +/// row `r` of `q`, and its scale at the 128x4 position of `(r, a / 16)`. +fn storeTileSwiglu(b: *B, comptime tile: Tile, operands: EpilogueOperands, routed: RoutedLanes, tile_m: V, group: V, act_scale: V, columns: []const V, comptime chunk: usize) void { + if (operands.direct) + swigluRows(b, tile, operands, routed, tile_m, group, act_scale, columns, 1, false, chunk) + else + swigluRows(b, tile, operands, routed, tile_m, group, act_scale, columns, comptime @intCast(tile.chunkRows()), true, chunk); +} + +fn swigluRows(b: *B, comptime tile: Tile, operands: EpilogueOperands, routed: RoutedLanes, tile_m: V, group: V, act_scale: V, columns: []const V, comptime live: usize, comptime skip_padding: bool, comptime chunk: usize) void { + const n = comptime tile.chunkRows(); + const chunk_first: i32 = comptime @intCast(chunk * @as(usize, @intCast(tile.chunkRows()))); + const tid = b.threadIdx().x; + const lane = tid.rem(32); + const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); + const is_gate = tid.rem(2).eq(0); + const first_row = group.mul(operands.group_rows); + const f32_type = cute.DType.f32.toMlir(b.ctx); + const row_chunks = std.math.divCeil(usize, live, swiglu_chunk) catch unreachable; + + var activations: [@intCast(n)]V = undefined; + // Lane `c` collects the block maximum of column `c`. + var lane_maximum = b.constant(.f32, 0); + for (0..row_chunks) |row_chunk| { + const first = row_chunk * swiglu_chunk; + const last = @min(first + swiglu_chunk, live); + const guarded = skip_padding and (row_chunk > 0 or chunk > 0); + const types: [swiglu_chunk + 1]*const mlir.Type = @splat(f32_type); + var guard: ?@TypeOf(b.openIfElse(routed.count, tupleOf(types))) = if (guarded) b.openIfElse(routed.count.gt(chunk_first + @as(i32, @intCast(first))), tupleOf(types)) else null; + var chunk_maximum = lane_maximum; + for (columns[first..last], activations[first..last], first..) |value, *activation, column| { + const c: i32 = @intCast(column); + // The reference rounds the projection to BF16. + var gate = value.to(.bf16).to(.f32); + var up_value = b.shuffleXor(value, 1).to(.bf16).to(.f32); + if (operands.swiglu_limit) |limit| { + gate = gate.minimum(limit); + up_value = up_value.maximum(-limit).minimum(limit); + } + // up * SiLU(gate), exp(-g) = 2^(-g * log2 e). + const exp_neg = b.unaryF32("ex2.approx.ftz.f32", gate.mul(-std.math.log2e)); + var result = up_value.mul(gate).mul(b.unaryF32("rcp.approx.ftz.f32", exp_neg.add(1.0))); + if (routed.weight) |weight| result = result.mul(b.shuffleIdx(weight, c)); + // The NVFP4 quantizer reads BF16 activations. + result = result.to(.bf16).to(.f32); + // Odd threads hold no activation and must not raise the maxima. + activation.* = b.select(is_gate, result, b.constant(.f32, 0)); + const maximum = b.warpMaxNonNegative(activation.*.abs()); + chunk_maximum = b.select(lane.eq(c), maximum, chunk_maximum); + } + if (guard) |*g| { + var then_values: [swiglu_chunk + 1]V = @splat(chunk_maximum); + for (activations[first..last], then_values[0 .. last - first]) |activation, *t| t.* = activation; + g.yieldThen(tupleOf(then_values)); + var else_values: [swiglu_chunk + 1]V = @splat(b.constant(.f32, 0)); + else_values[swiglu_chunk] = lane_maximum; + g.yieldElse(tupleOf(else_values)); + for (activations[first..last], g.results[0 .. last - first]) |*activation, result| activation.* = result; + lane_maximum = g.results[swiglu_chunk]; + } else { + lane_maximum = chunk_maximum; + } + } + const scale = nvfp4.blockScale(b, lane_maximum, act_scale); + + // Lane `16 h` writes the word of activations `16 * warp + 8 h ..`. + const word = tile_m.mul(8).add(warp.mul(2)).add(lane.div(16)); + const word_lane = lane.rem(16).eq(0); + for (0..row_chunks) |row_chunk| { + const first = row_chunk * swiglu_chunk; + const last = @min(first + swiglu_chunk, live); + var guard: ?@TypeOf(b.openIf(routed.count)) = if (skip_padding and (chunk > 0 or first > 0)) b.openIf(routed.count.gt(chunk_first + @as(i32, @intCast(first)))) else null; + for (activations[first..last], first..) |activation, column| { + const c: i32 = @intCast(column); + const quantized = activation.mul(b.shuffleIdx(scale.multiplier, c)); + // Gate lane `4i` pairs with gate lane `4i + 2`, then pairs of + // bytes and pairs of halves meet at lanes `8i` and `16i`. + const pair = b.packFp4x2(quantized, b.shuffleXor(quantized, 2)); + const half = pair.bitOr(b.shuffleXor(pair, 4).shl(8)); + const packed_word = half.bitOr(b.shuffleXor(half, 8).shl(16)); + var writer = b.openIf(word_lane); + operands.q.?.set(.{ first_row.add(chunk_first + c), word }, packed_word); + writer.close(); + } + if (guard) |*g| g.close(); + } + + // Lane `c` stores the scale of row `c`, block `4 * tile_m + warp`. + var scale_lane = b.openIf(lane.lt(@as(i32, @intCast(n))).bitAnd(lane.add(chunk_first).lt(routed.count))); + const swizzled = tile_m.mul(512).add(lane.mul(16)).add(chunk * 4).add(warp); + operands.s.?.set(.{ group, swizzled }, scale.bits.to(.i8)); + scale_lane.close(); +} + +/// Store each live routed row at its token/top-k route, so the final +/// reduction reads rows in route order. +fn storeTileRows(b: *B, comptime tile: Tile, operands: EpilogueOperands, routed: RoutedLanes, tile_m: V, columns: []const V, comptime chunk: usize) void { + const chunk_first: i32 = comptime @intCast(chunk * @as(usize, @intCast(tile.chunkRows()))); + const output_column = tile_m.mul(128).add(b.threadIdx().x); + if (operands.direct) { + const value = if (routed.weight) |weight| columns[0].mul(weight) else columns[0]; + operands.rows.?.set(.{ routed.route, output_column }, value.to(.bf16)); + return; + } + var routes: [32]V = undefined; + var weights: [32]?V = @splat(null); + for (routes[0..columns.len], weights[0..columns.len], 0..) |*route, *weight, column| { + route.* = b.shuffleIdx(routed.route, @as(i32, @intCast(column))); + if (routed.weight) |w| weight.* = b.shuffleIdx(w, @as(i32, @intCast(column))); + } + for (columns, routes[0..columns.len], weights[0..columns.len], 0..) |value, route, weight, column| { + var live: ?@TypeOf(b.openIf(routed.count)) = if (column == 0 and chunk == 0) null else b.openIf(routed.count.gt(chunk_first + @as(i32, @intCast(column)))); + const scaled = if (weight) |w| value.mul(w) else value; + operands.rows.?.set(.{ route, output_column }, scaled.to(.bf16)); + if (live) |*l| l.close(); + } +} + +fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: Epilogue, d: anytype) void { + const L = Layouts(tile); + const n = tile.n; + const stages = tile.stages; + const shared = b.allocSmemStorage( + .i8, + 1024, + b.layoutType(b.layoutSpec(.{tile.smem_bytes}, .{1})), + 1, + tile.fields(b), + ); + const shared_base = b.getIter( + shared, + b.ptrTy(.i8, .smem, 1024) catch @panic("bad shared base pointer"), + ).value(); + + const tmem_holding = sharedPointerAt(b, shared_base, tile.tmem_holding, .i32, 8); + const storage: StageStorage = .{ + .c = sharedPointerAt(b, shared_base, tile.c, .f32, 1024), + .a = sharedPointerAt(b, shared_base, tile.a, .i8, 1024), + .b = sharedPointerAt(b, shared_base, tile.b, .i8, 1024), + .sfa = sharedPointerAt(b, shared_base, tile.sfa, .f8e4m3fn, 1024), + .sfb = sharedPointerAt(b, shared_base, tile.sfb, .f8e4m3fn, 1024), + .load_barriers = sharedPointerAt(b, shared_base, 0, .i64, 1024), + .mma_barriers = sharedPointerAt(b, shared_base, tile.empty_barriers, .i64, 8), + .accumulator_full_barriers = sharedPointerAt(b, shared_base, tile.accumulator_full, .i64, 16), + .accumulator_empty_barriers = sharedPointerAt(b, shared_base, tile.accumulator_empty, .i64, 16), + }; + + const tid = b.threadIdx().x; + const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); + const tiled_mma: cute.Atom = .{ .inner = d.mma.inner, .kernel = b }; + + // coordinate tensor -> local_tile -> tiled_mma.partition -> group modes + // -> tma_partition, with the result types of the CuTe DSL. + const m_tiles = @divExact(cfg.m, 128); + const k_tiles = @divExact(cfg.k, k_tile); + const m_step = b.basis(128, 1, .{1}); + const k_step = b.basis(k_tile, 1, .{0}); + const a_local_spec = b.layoutSpec( + .{ 128, k_tile, m_tiles, k_tiles, cfg.experts }, + .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), m_step, k_step, b.basis(1, 1, .{2}) }, + ); + const b_local_spec = b.layoutSpec( + .{ n, k_tile, 1, k_tiles, cfg.groups }, + .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), 0, k_step, b.basis(1, 1, .{2}) }, + ); + const all_tiles = .{ cute.AlgebraToken.all, cute.AlgebraToken.all, cute.AlgebraToken.all }; + const tiled_a = b.localTile(d.gA, b.makeTile(&.{ 128, k_tile }), all_tiles, b.coordTensorType(3, a_local_spec), null); + const tiled_b = b.localTile(d.gB, b.makeTile(&.{ n, k_tile }), all_tiles, b.coordTensorType(3, b_local_spec), null); + + const a_mma_spec = b.layoutSpec( + .{ .{ 128, 64 }, 1, 4, m_tiles, k_tiles, cfg.experts }, + .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(64, 1, .{0}), m_step, k_step, b.basis(1, 1, .{2}) }, + ); + const b_mma_spec = b.layoutSpec( + .{ .{ n, 64 }, 1, 4, 1, k_tiles, cfg.groups }, + .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(64, 1, .{0}), 0, k_step, b.basis(1, 1, .{2}) }, + ); + const mma_a = b.tiledMmaPartition(tiled_mma, tiled_a, .{0}, 0, b.coordTensorType(3, a_mma_spec)); + const mma_b = b.tiledMmaPartition(tiled_mma, tiled_b, .{0}, 1, b.coordTensorType(3, b_mma_spec)); + + const a_grouped_layout = b.staticLayout(b.layoutSpec( + .{ .{ .{ 128, 64 }, 1, 4 }, m_tiles, k_tiles, cfg.experts }, + .{ .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(64, 1, .{0}) }, m_step, k_step, b.basis(1, 1, .{2}) }, + )); + const b_grouped_layout = b.staticLayout(b.layoutSpec( + .{ .{ .{ n, 64 }, 1, 4 }, 1, k_tiles, cfg.groups }, + .{ .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(64, 1, .{0}) }, b.basis(n, 1, .{1}), k_step, b.basis(1, 1, .{2}) }, + )); + const staged = stagedOperands(b, tile, storage); + const smem_a_grouped = b.withLayout(staged.a, b.staticLayout(b.layoutSpec( + .{ .{ .{ 128, 64 }, 1, 4 }, stages }, + .{ .{ .{ k_tile, 1 }, 0, 64 }, 2 * Tile.a_stage_bytes }, + ))); + const smem_b_grouped = b.withLayout(staged.b, b.staticLayout(b.layoutSpec( + .{ .{ .{ n, 64 }, 1, 4 }, stages }, + .{ .{ .{ k_tile, 1 }, 0, 64 }, 2 * tile.bStageBytes() }, + ))); + const one_cta = b.staticLayout(b.layoutSpec(.{1}, .{0})); + const a_partition_layout = b.layoutSpec(.{ .{ 2 * Tile.a_stage_bytes, 1 }, stages }, .{ .{ 1, 0 }, 2 * Tile.a_stage_bytes }); + const b_partition_layout = b.layoutSpec(.{ .{ 2 * tile.bStageBytes(), 1 }, stages }, .{ .{ 1, 0 }, 2 * tile.bStageBytes() }); + const a_target_layout = b.layoutSpec( + .{ .{ .{ k_tile, 128 }, 1 }, m_tiles, k_tiles, cfg.experts }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, m_step, k_step, b.basis(1, 1, .{2}) }, + ); + const b_target_layout = b.layoutSpec( + .{ .{ .{ k_tile, n }, 1 }, 1, k_tiles, cfg.groups }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(n, 1, .{1}), k_step, b.basis(1, 1, .{2}) }, + ); + const partitioned_a = b.tmaPartition( + .{ .inner = d.tma_a.inner, .kernel = b }, + .{0}, + one_cta, + smem_a_grouped, + .{b.withLayout(mma_a, a_grouped_layout)}, + b.memrefTypeFromPointer(L.fp4SmemPtrTy(b), b.layoutType(a_partition_layout)), + &.{b.coordTensorType(3, a_target_layout)}, + ); + const partitioned_b = b.tmaPartition( + .{ .inner = d.tma_b.inner, .kernel = b }, + .{0}, + one_cta, + smem_b_grouped, + .{b.withLayout(mma_b, b_grouped_layout)}, + b.memrefTypeFromPointer(L.fp4SmemPtrTy(b), b.layoutType(b_partition_layout)), + &.{b.coordTensorType(3, b_target_layout)}, + ); + + const exec_a = b.makeExecTma(.{ .inner = d.tma_a.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaA(b, cfg))); + const exec_b = b.makeExecTma(.{ .inner = d.tma_b.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaB(b, cfg))); + const exec_sfa = b.makeExecTma(.{ .inner = d.tma_sfa.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaSf(b))); + const exec_sfb = b.makeExecTma(.{ .inner = d.tma_sfb.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaSf(b))); + + const expert_ids: cute.View = .{ .inner = d.expert_ids.inner, .kernel = b }; + const active_count: cute.View = .{ .inner = d.active_count.inner, .kernel = b }; + const direct: ?DirectRoutes = if (cfg.direct) .{ .expert_ids = expert_ids, .routes = @intCast(cfg.groups) } else null; + const active = if (direct) |routes| routes.count(b) else active_count.get(.{0}, .i32); + + // Warp 0 initializes the barriers; one elected lane per barrier array. + var init_warp = b.openIf(warp.eq(0)); + const barrier_arrays = .{ + .{ storage.load_barriers, stages, 1 }, + .{ storage.mma_barriers, stages, 1 }, + .{ storage.accumulator_full_barriers, accumulator_stages, 1 }, + .{ storage.accumulator_empty_barriers, accumulator_stages, 128 }, + }; + inline for (barrier_arrays) |array| { + var lane = b.openIf(b.electSync()); + inline for (0..array[1]) |stage| { + b.mbarrierInit(barrierAt(b, array[0], b.constant(.i32, stage)), array[2]); + } + lane.close(); + } + b.fenceMbarrierInit(); + init_warp.close(); + b.syncThreads(); + + const reduction_tiles: usize = @intCast(k_tiles); + var tma_warp = b.openIf(warp.eq(5)); + runTmaProducer(b, tile, reduction_tiles, storage, active, d.tiles_m, expert_ids, direct, partitioned_a.targets[0], partitioned_b.targets[0], exec_a, exec_b, exec_sfa, exec_sfb); + tma_warp.close(); + + var mma_warp = b.openIf(warp.eq(4)); + b.namedBarrier(2, 160); + runMmaConsumer(b, tile, reduction_tiles, storage, tiled_mma, tmem_holding, active, d.tiles_m); + mma_warp.close(); + + var epilogue_warps = b.openIf(warp.lt(4)); + var alloc_warp = b.openIf(warp.eq(0)); + b.allocTmem(tile.tmem_columns, tmem_holding, .{}); + alloc_warp.close(); + b.namedBarrier(2, 160); + const shared_operands: EpilogueOperands = .{ + .expert_ids = expert_ids, + .alpha = view(b, d.alpha), + .group_sizes = view(b, d.group_sizes), + .route_inverse = view(b, d.route_inverse), + .routing_weights = if (cfg.routing_weights) view(b, d.routing_weights) else null, + .direct = cfg.direct, + .group_rows = cfg.groupRows(), + .swiglu_limit = cfg.swiglu_limit, + }; + var operands = shared_operands; + switch (epilogue) { + .swiglu_nvfp4 => { + operands.act_scale = view(b, d.act_scale); + operands.q = view(b, d.q); + operands.s = view(b, d.s); + }, + .route_rows => operands.rows = view(b, d.rows), + } + runEpilogue(b, tile, epilogue, storage, tmem_holding, active, d.tiles_m, direct, operands); + epilogue_warps.close(); +} + +fn buildHostLaunch(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: Epilogue) cute.FinishError!void { + const L = Layouts(tile); + const host_arg = HostArgs(epilogue); + const n = tile.n; + const a_layout = b.layoutSpec(.{ cfg.m, cfg.k, cfg.experts }, .{ cfg.k, 1, cfg.m * cfg.k }); + const b_layout = b.layoutSpec(.{ cfg.groupRows(), cfg.k, cfg.groups }, .{ cfg.k, 1, cfg.k * cfg.groupRows() }); + // `cute.tile_to_shape` derives that the dynamic strides step whole + // 512-byte scale atoms, and the result type has to carry it. + const sf_stride: cute.ConstrainedDynamic = .{ .divisible_by = 512 }; + const dynamic_sf_layout = b.layoutSpec( + .{ .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ .{ 16, 4 }, cute.AlgebraToken.dynamic }, .{ 1, cute.AlgebraToken.dynamic } }, + .{ .{ .{ 16, 4 }, sf_stride }, .{ .{ 0, 1 }, 512 }, .{ 0, sf_stride } }, + ); + + // XLA owns packed storage as signless bytes; recast to FP4 here. + const gA = b.makeTensorView(b.recastPointer(b.arg(host_arg.weight), .f4e2m1fn, .gmem, 16), b.staticLayout(a_layout)); + const schedule = b.arg(host_arg.schedule); + const ids = b.makeTensorView(schedule, b.staticLayout(b.layoutSpec(cfg.groups, 1))); + const active_ptr = b.addOffset( + schedule, + b.makeIntTuple(.{2 * cfg.groups}), + b.ptrTy(.i32, .gmem, offsetAlignment(16, 2 * cfg.groups, 4)) catch @panic("bad active count pointer"), + ); + const active = b.makeTensorView(active_ptr, b.staticLayout(b.layoutSpec(1, 1))); + const gB = b.makeTensorView(b.recastPointer(b.arg(host_arg.input_quant), .f4e2m1fn, .gmem, 16), b.staticLayout(b_layout)); + const sf_atom_layout = b.staticLayout(b.layoutSpec( + .{ .{ 32, 4 }, .{ 16, 4 } }, + .{ .{ 16, 4 }, .{ 0, 1 } }, + )); + // Zero-based form of Python CuTe's `(2, 1, 3)` tile-to-shape order. + const sf_order = b.makeIntTuple(.{ 1, 0, 2 }); + const sfa_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_m), b.arg(host_arg.problem_k), b.arg(host_arg.expert_count) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); + const sfb_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_n), b.arg(host_arg.problem_k), b.arg(host_arg.problem_l) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); + const dynamic_sfa_layout = b.tileToShape(sf_atom_layout, sfa_shape, sf_order, b.layoutType(dynamic_sf_layout)); + const dynamic_sfb_layout = b.tileToShape(sf_atom_layout, sfb_shape, sf_order, b.layoutType(dynamic_sf_layout)); + const gSFA = b.makeTensorView(b.recastPointer(b.arg(host_arg.weight_scale), .f8e4m3fn, .gmem, 16), dynamic_sfa_layout); + const gSFB = b.makeTensorView(b.recastPointer(b.arg(host_arg.input_scale), .f8e4m3fn, .gmem, 16), dynamic_sfb_layout); + + const zero_tmem = b.intToPtr(0, .f8e4m3fn, .tmem, 1); + const mma = b.makeBlockScaledMma(.{ .kind = mma_kind, .n = @intCast(n) }, zero_tmem); + const mma_sfb = b.makeBlockScaledMma(.{ .kind = mma_kind, .n = 128 }, zero_tmem); + + const a_smem = b.makeComposedLayout(L.ab_swizzle, 0, L.aSmem(b, false)); + const b_smem = b.makeComposedLayout(L.ab_swizzle, 0, L.bSmem(b, false)); + const sf_smem = b.staticLayout(L.sfSmem(b, false)); + const a_map = b.staticLayout(b.layoutSpec(.{ .{ 128, 64 }, 1, 4 }, .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0, b.basis(64, 1, .{1}) })); + const b_map = b.staticLayout(b.layoutSpec(.{ .{ n, 64 }, 1, 4 }, .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0, b.basis(64, 1, .{1}) })); + const sf_map = b.staticLayout(b.layoutSpec( + .{ .{ .{ 32, 4 }, .{ 16, 4 } }, 1, 4 }, + .{ .{ .{ b.basis(1, 1, .{ 0, 0, 0 }), b.basis(1, 1, .{ 0, 0, 1 }) }, .{ b.basis(1, 1, .{ 1, 0, 0 }), b.basis(1, 1, .{ 1, 0, 1 }) } }, 0, b.basis(1, 1, .{ 1, 1 }) }, + )); + + const tma_a = b.makeTiledTmaLoadAtom(gA.value(), a_smem.value(), a_map.value(), L.tmaA(b, cfg)); + const tma_b = b.makeTiledTmaLoadAtom(gB.value(), b_smem.value(), b_map.value(), L.tmaB(b, cfg)); + const tma_sfa = b.makeTiledTmaLoadAtom(gSFA.value(), sf_smem.value(), sf_map.value(), L.tmaSf(b)); + const tma_sfb = b.makeTiledTmaLoadAtom(gSFB.value(), sf_smem.value(), sf_map.value(), L.tmaSf(b)); + + const one = b.constant(.i32, 1); + const tiles_m = b.constant(.i32, @divExact(cfg.m, 128)); + const grid_z = b.constant(.i32, @min(@divExact(cfg.m, 128) * cfg.groups, cfg.persistent_ctas)); + const config = b.makeLaunchConfig(.{ + .grid = .{ one, one, grid_z }, + .block = .{ b.constant(.i32, 192), one, one }, + .dynamic_smem = b.kernelSmemSize(deviceName(epilogue)), + .stream = b.cudaStream(), + .cluster = .{ one, one, one }, + .use_pdl = true, + }); + const group_sizes = b.makeTensorView( + b.addOffset(schedule, b.makeIntTuple(.{cfg.groups}), b.ptrTy(.i32, .gmem, offsetAlignment(16, cfg.groups, 4)) catch @panic("bad group size pointer")), + b.staticLayout(b.layoutSpec(cfg.groups, 1)), + ); + const per_expert = b.staticLayout(b.layoutSpec(cfg.experts, 1)); + const mainloop = .{ + ids, active, mma.tiled, mma_sfb.tiled, tma_a.atom, tma_a.tensor, tma_b.atom, tma_b.tensor, + tma_sfa.atom, tma_sfa.tensor, tma_sfb.atom, tma_sfb.tensor, b.makeTensorView(b.arg(host_arg.alpha), per_expert), group_sizes, b.makeTensorView(b.arg(host_arg.route_inverse), b.staticLayout(b.layoutSpec(cfg.routedRows(), 1))), b.makeTensorView(b.arg(host_arg.routing_weights), b.staticLayout(b.layoutSpec(@max(cfg.routes, 1), 1))), + }; + const activations = @divExact(cfg.m, 2); + const launch = switch (epilogue) { + .swiglu_nvfp4 => b.launchEx(deviceName(epilogue), config, mainloop ++ .{ + b.makeTensorView(b.arg(host_arg.act_scale), per_expert), + b.makeTensorView(b.recastPointer(b.arg(host_arg.output), .i32, .gmem, 16), b.staticLayout(b.layoutSpec(.{ cfg.routedRows(), @divExact(activations, 8) }, .{ @divExact(activations, 8), 1 }))), + b.makeTensorView(b.arg(host_arg.scales), b.staticLayout(b.layoutSpec(.{ cfg.groups, @divExact(activations, 16) * 128 }, .{ @divExact(activations, 16) * 128, 1 }))), + tiles_m, + }), + .route_rows => b.launchEx(deviceName(epilogue), config, mainloop ++ .{ + b.makeTensorView(b.arg(host_arg.output), b.staticLayout(b.layoutSpec(.{ @max(cfg.routes, 1), cfg.m }, .{ cfg.m, 1 }))), + tiles_m, + }), + }; + b.returnHostStatus(b.cudaResultStatus(launch)); +} + +test "NVFP4 GEMM programs emit their host launch for every tile width" { + inline for (tile_widths) |n| { + inline for (.{ .{ UpQuantized, Epilogue.swiglu_nvfp4, 4608, 5120 }, .{ DownRows, Epilogue.route_rows, 5120, 2304 } }) |program| { + const cfg: Config = .{ + .experts = 192, + .m = program[2], + .n = n, + .k = program[3], + .groups = 227, + .epilogue = program[1], + .routes = 576, + .direct = n == 8, + .routing_weights = program[1] == .route_rows, + }; + const ir = try program[0].emit(std.testing.allocator, cfg); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, deviceName(program[1])) != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda.launch_ex") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "tma_format = U4") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, std.fmt.comptimePrint("sm100.mma_bs<128x{d}x64", .{n})) != null); + } + } +} diff --git a/zml/kernels/moe/cute/sm10x/reduce.zig b/zml/kernels/moe/cute/sm10x/reduce.zig new file mode 100644 index 00000000..63244ac4 --- /dev/null +++ b/zml/kernels/moe/cute/sm10x/reduce.zig @@ -0,0 +1,124 @@ +//! CuTe top-k reduction behind the persistent MoE GEMMs: sums each token's +//! BF16 route rows. +const std = @import("std"); + +const zml = @import("../../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const launchBoundary = @import("common.zig").launchBoundary; + +pub const Config = struct { + tokens: i64, + hidden: i64, + topk: i64, + columns_per_cta: i64, +}; + +// A program rather than a bare kernel so its launch carries the +// programmatic-dependent-launch attribute (see `mxfp4.zig`). +const Topk = cute.Program(Config, .{ + .name = "moe_cute_reduce_topk", + .inputs = &.{ "d", "routes" }, + .outputs = &.{"y"}, + .run = topkProgram, +}); + +const topk_kernel_name = "moe_cute_reduce_topk_kernel"; + +/// Amortize launch overhead for large prefills while retaining enough CTAs +/// to saturate the GPU for decode and small batches. +pub fn columnsPerCta(tokens: i64, hidden: i64) i64 { + const elements = tokens * hidden; + if (tokens > 1 and elements >= 512 * 2048) return 2048; + if (tokens > 1 and elements >= 512 * 1024) return 1024; + // Two columns per thread is the smallest tile: 128 threads, one 32-bit + // word each. + return 256; +} + +/// Sum the `[tokens * topk, hidden]` BF16 route rows of each token into BF16. +/// Routing weights were already applied before the down projection. +/// `routes` is negative for routes this rank does not compute (expert +/// parallelism): their rows were never written. +pub fn topk(d: zml.Tensor, routes: zml.Tensor, cfg: Config) zml.Tensor { + return Topk.call( + .{ .d = d, .routes = routes }, + .{ .y = .init(.{ cfg.tokens, cfg.hidden }, .bf16) }, + .{ .cfg = cfg }, + ).y; +} + +/// Each thread owns two adjacent BF16 columns, read and written as one +/// 32-bit word: a warp then moves a full 128-byte line per access. +fn topkProgram(b: *B, cfg: Config) cute.FinishError!void { + b.beginFunction(topk_kernel_name, .cuda_kernel); + try topkKernel(b, cfg); + b.endFunction(.{ 128, 1, 1 }); + + b.beginFunction(Topk.name, .host); + // The kernel reads and writes two BF16 columns per 32-bit word. + const host = try b.declareArgs(.{ + .d = .{ .ptr = cute.DType.bf16 }, + .routes = .{ .ptr = cute.DType.i32 }, + .y = .{ .ptr = cute.DType.bf16 }, + }); + try launchBoundary(b, topk_kernel_name, .{ + @intCast(@divTrunc(cfg.hidden + cfg.columns_per_cta - 1, cfg.columns_per_cta)), + @intCast(cfg.tokens), + 1, + }, .{ b.recastPointer(host.d, .i32, .gmem, 16), host.routes, b.recastPointer(host.y, .i32, .gmem, 16) }); + b.endFunction(null); +} + +fn topkKernel(b: *B, cfg: Config) cute.FinishError!void { + const pairs = @divExact(cfg.hidden, 2); + const a = try b.declareArgs(.{ + .d = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.tokens * cfg.topk, pairs } } }, + .routes = .{ .tensor = .{ .dtype = .i32, .shape = &.{cfg.tokens * cfg.topk} } }, + .y = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.tokens, pairs } } }, + }); + + // The down projection's rows arrive from a grid this one may overlap. + b.waitForDependency(); + const token = b.blockIdx().y; + const pairs_per_cta = @divExact(cfg.columns_per_cta, 2); + const first_pair = b.blockIdx().x.mul(pairs_per_cta).add(b.threadIdx().x); + for (0..@intCast(@divExact(pairs_per_cta, 128))) |chunk| { + const pair = first_pair.add(chunk * 128); + var in_bounds = b.openIf(pair.lt(pairs)); + var low = b.constant(.f32, 0); + var high = b.constant(.f32, 0); + for (0..@intCast(cfg.topk)) |slot| { + const route = token.mul(cfg.topk).add(slot); + // Routes of other expert-parallel ranks were never written. + var computed = b.openIfElse(a.routes.get(.{route}).ge(0), .{ cute.DType.f32.toMlir(b.ctx), cute.DType.f32.toMlir(b.ctx) }); + const word = a.d.get(.{ route, pair }); + computed.yieldThen(.{ + low.add(word.shl(16).bitCast(.f32)), + high.add(word.bitAnd(@as(i32, @bitCast(@as(u32, 0xffff0000)))).bitCast(.f32)), + }); + computed.yieldElse(.{ low, high }); + low = computed.results[0]; + high = computed.results[1]; + } + const low_bits = low.to(.bf16).to(.f32).bitCast(.i32).shrLogical(16); + const high_bits = high.to(.bf16).to(.f32).bitCast(.i32).bitAnd(@as(i32, @bitCast(@as(u32, 0xffff0000)))); + a.y.set(.{ token, pair }, low_bits.bitOr(high_bits)); + in_bounds.close(); + } + b.launchDependents(); +} + +test "CuTe top-k reduction emits for decode and prefill batches" { + inline for (.{ @as(i64, 1), 6, 96, 16_384 }) |tokens| { + const ir = try Topk.emit(std.testing.allocator, .{ + .tokens = tokens, + .hidden = 5120, + .topk = 6, + .columns_per_cta = columnsPerCta(tokens, 5120), + }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda.launch_ex") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "scf.if") != null); + } +} diff --git a/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig b/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig index ffaec26b..2c7ad342 100644 --- a/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig +++ b/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig @@ -54,14 +54,6 @@ const Input = struct { fc2_global: zml.Tensor, }; -const Bf16Input = struct { - hidden_states: zml.Tensor, - fc1_weights: zml.Tensor, - fc2_weights: zml.Tensor, - topk_weights: zml.Tensor, - topk_ids: zml.Tensor, -}; - const Output = struct { output: zml.Shape, workspace: zml.Shape, @@ -87,43 +79,34 @@ const DeviceRunner = struct { runner: *fi_cutlass_moe.Runner, }; -pub const Variant = enum { - bf16xbf16, - nvfp4xnvfp4, -}; - const max_num_devices = zml.platform.Platform.MAX_NUM_DEVICES; pub const Runners = struct { - runners: [std.meta.fieldNames(Variant).len][max_num_devices]?DeviceRunner = - @splat(@splat(null)), + runners: [max_num_devices]?DeviceRunner = @splat(null), pub fn deinit(self: *Runners) void { - for (&self.runners) |*variant_runners| { - for (variant_runners) |device_runner| { - if (device_runner) |runner| { - _ = runner.api.runnerDestroy(runner.runner); - } + for (self.runners) |device_runner| { + if (device_runner) |runner| { + _ = runner.api.runnerDestroy(runner.runner); } } } - fn ensureRunner(self: *Runners, device: i32, variant: Variant) !DeviceRunner { + fn ensureRunner(self: *Runners, device: i32) !DeviceRunner { if (device < 0 or device >= max_num_devices) return error.UnsupportedDevice; const index: usize = @intCast(device); - const variant_index: usize = @backingInt(variant); - if (self.runners[variant_index][index]) |runner| return runner; + if (self.runners[index]) |runner| return runner; const api = try fi_cutlass_moe.apiForDevice(device); - const options = runnerOptions(device, variant); + const options = runnerOptions(device); var runner: ?*fi_cutlass_moe.Runner = null; try checkStatus(api, api.runnerCreate(&options, &runner)); const result: DeviceRunner = .{ .api = api, .runner = runner orelse return error.RunnerInitializationFailed, }; - self.runners[variant_index][index] = result; + self.runners[index] = result; return result; } }; @@ -146,14 +129,11 @@ fn checkStatus(api: *const fi_cutlass_moe.Api, status: fi_cutlass_moe.Status) !v }; } -fn runnerOptions(device: i32, variant: Variant) fi_cutlass_moe.RunnerOptions { +fn runnerOptions(device: i32) fi_cutlass_moe.RunnerOptions { var options = std.mem.zeroes(fi_cutlass_moe.RunnerOptions); options.struct_size = @sizeOf(fi_cutlass_moe.RunnerOptions); options.activation_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; - options.weight_dtype = switch (variant) { - .bf16xbf16 => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16, - .nvfp4xnvfp4 => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_PACKED_FP4, - }; + options.weight_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_PACKED_FP4; options.output_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; options.device = device; options.use_fused_finalize = 1; @@ -208,7 +188,7 @@ fn ffiCallNvfp4( ) !?*zml.pjrt.ffi.Error { const device = try call_frame.ctx.getDeviceOrdinal(call_frame.api); const runners = try runnersFromAttributes(attributes); - const deviceRunner = try runners.ensureRunner(device, .nvfp4xnvfp4); + const deviceRunner = try runners.ensureRunner(device); const context = makeContext(attributes); var io = std.mem.zeroes(fi_cutlass_moe.Io); @@ -257,56 +237,10 @@ const routedNvfp4Call = zml.ops.CustomCall(Input, Output, Attributes, ffiCallNvf .has_side_effect = false, }); -fn ffiCallBf16( - call_frame: *zml.pjrt.ffi.CallFrame, - input: zml.pjrtx.TensorToCustomCallBuffer(Bf16Input), - output: zml.pjrtx.ShapeToCustomCallBuffer(Output), - attributes: Attributes, -) !?*zml.pjrt.ffi.Error { - const device = try call_frame.ctx.getDeviceOrdinal(call_frame.api); - const runners = try runnersFromAttributes(attributes); - const deviceRunner = try runners.ensureRunner(device, .bf16xbf16); - const context = makeContext(attributes); - - var io = std.mem.zeroes(fi_cutlass_moe.Io); - io.struct_size = @sizeOf(fi_cutlass_moe.Io); - io.input = input.hidden_states.ptr; - io.token_selected_experts = @ptrCast(@alignCast(input.topk_ids.ptr)); - io.token_final_scales = @ptrCast(@alignCast(input.topk_weights.ptr)); - io.fc1_expert_weights = input.fc1_weights.ptr; - io.fc2_expert_weights = input.fc2_weights.ptr; - io.output = output.output.ptr; - - var workspace = std.mem.zeroes(fi_cutlass_moe.Workspace); - workspace.struct_size = @sizeOf(fi_cutlass_moe.Workspace); - workspace.data = output.workspace.ptr; - workspace.data_bytes = output.workspace.shape.byteSize(); - - try checkStatus( - deviceRunner.api, - deviceRunner.api.run( - deviceRunner.runner, - &context, - &io, - &workspace, - @ptrCast(call_frame.api.stream(call_frame.ctx)), - attributes.gemm1_tactic, - attributes.gemm2_tactic, - ), - ); - return null; -} - -const routedBf16Call = zml.ops.CustomCall(Bf16Input, Output, Attributes, ffiCallBf16, .{ - .name = "flashinfer_cutlass_bf16_routed_moe", - .sharding_aware = false, - .has_side_effect = false, -}); - fn computeCapability(platform: *const zml.Platform) !u16 { const cc = zml.platform.cuda.computeCapability(platform) orelse return error.UnsupportedPlatform; return switch (cc.sm()) { - 90, 100, 103, 120 => |sm| sm, + 100, 103, 120 => |sm| sm, else => error.UnsupportedArchitecture, }; } @@ -331,7 +265,6 @@ pub fn load( pub fn register(platform: *const zml.Platform) !void { if (comptime platforms.isEnabled(.cuda)) { try routedNvfp4Call.register(platform); - try routedBf16Call.register(platform); return; } return error.UnsupportedPlatform; @@ -356,10 +289,9 @@ pub fn isNvfp4Supported(platform: *const zml.Platform) bool { pub fn tacticCounts( platform: *const zml.Platform, device: i32, - variant: Variant, ) !struct { gemm1: i32, gemm2: i32 } { const runners = platform.state.cuda.fi_cutlass_moe_runners orelse return error.RunnersNotLoaded; - const deviceRunner = try runners.ensureRunner(device, variant); + const deviceRunner = try runners.ensureRunner(device); var gemm1: i32 = 0; var gemm2: i32 = 0; try checkStatus( @@ -532,7 +464,7 @@ pub fn fusedExpertsNvfp4( ) catch |e| stdx.debug.panic("Invalid inputs: {}", .{e}); attributes.runners = @intFromPtr(runners); - const deviceRunner = runners.ensureRunner(options.workspace_query_device, .nvfp4xnvfp4) catch |e| stdx.debug.panic("Failed to ensure runner: {}", .{e}); + const deviceRunner = runners.ensureRunner(options.workspace_query_device) catch |e| stdx.debug.panic("Failed to ensure runner: {}", .{e}); const context = makeContext(attributes); var requirements = std.mem.zeroes(fi_cutlass_moe.WorkspaceRequirements); requirements.struct_size = @sizeOf(fi_cutlass_moe.WorkspaceRequirements); @@ -568,100 +500,6 @@ pub fn fusedExpertsNvfp4( return result.output; } -/// Routed BF16 x BF16 CUTLASS MoE [expert, output, input] weights. -/// Non-quantized FlashInfer path used on Hopper and Blackwell -pub fn fusedExpertsBf16( - hidden_states: zml.Tensor, - fc1_weights: zml.Tensor, - fc2_weights: zml.Tensor, - topk_weights: zml.Tensor, - topk_ids: zml.Tensor, - options: Options, -) zml.Tensor { - if (hidden_states.dtype() != .bf16 or - fc1_weights.dtype() != .bf16 or - fc2_weights.dtype() != .bf16 or - topk_weights.dtype() != .f32 or - topk_ids.dtype() != .i32 or - hidden_states.rank() != 2 or - fc1_weights.rank() != 3 or - fc2_weights.rank() != 3 or - topk_weights.rank() != 2 or - topk_ids.rank() != 2) - { - // TODO(Corentin): Better error message - @panic("InvalidInput"); - } - - const tokens = hidden_states.dim(.t); - const hiddenSize = hidden_states.dim(.d); - const numExperts = fc1_weights.dim(0); - const fc1Rows = fc1_weights.dim(1); - const intermediateSize = switch (options.activation) { - .silu, .gelu, .relu => fc1Rows, - .swiglu, .geglu, .geglu_tanh, .swiglu_step => @divExact(fc1Rows, 2), - }; - const topK = topk_ids.dim(.topk); - if (tokens <= 0 or hiddenSize <= 0 or - numExperts <= 0 or intermediateSize <= 0 or - topK <= 0 or topK > numExperts or - fc1_weights.dim(2) != hiddenSize or - fc2_weights.dim(0) != numExperts or - fc2_weights.dim(1) != hiddenSize or - fc2_weights.dim(2) != intermediateSize or - topk_weights.dim(.t) != tokens or - topk_weights.dim(.topk) != topK or - topk_ids.dim(.t) != tokens) - { - // TODO(Corentin): Better error message - @panic("InvalidShape"); - } - - const runners = currentRunners() catch |e| stdx.debug.panic("Failed to get runners: {}", .{e}); - const attributes: Attributes = .{ - .runners = @intFromPtr(runners), - .num_tokens = tokens, - .hidden_size = hiddenSize, - .intermediate_size = intermediateSize, - .num_experts = @intCast(numExperts), - .top_k = @intCast(topK), - .activation = cutlassActivation(options.activation), - .enable_pdl = options.enable_pdl, - .fc1_act_per_expert = false, - .fc2_act_per_expert = false, - .gemm1_tactic = options.gemm1_tactic, - .gemm2_tactic = options.gemm2_tactic, - }; - const deviceRunner = runners.ensureRunner(options.workspace_query_device, .bf16xbf16) catch |e| stdx.debug.panic("Failed to ensure runner: {}", .{e}); - const context = makeContext(attributes); - var requirements = std.mem.zeroes(fi_cutlass_moe.WorkspaceRequirements); - requirements.struct_size = @sizeOf(fi_cutlass_moe.WorkspaceRequirements); - checkStatus( - deviceRunner.api, - deviceRunner.api.getWorkspaceRequirements( - deviceRunner.runner, - &context, - &requirements, - ), - ) catch |e| stdx.debug.panic("Failed to get workspace requirements: {}", .{e}); - - const result = routedBf16Call.call( - .{ - .hidden_states = hidden_states, - .fc1_weights = fc1_weights, - .fc2_weights = fc2_weights, - .topk_weights = topk_weights, - .topk_ids = topk_ids, - }, - .{ - .output = hidden_states.shape(), - .workspace = .init(.{@as(i64, @intCast(requirements.total_bytes))}, .u8), - }, - attributes, - ); - return result.output; -} - pub fn fusedExperts( input: zml.Tensor, topk_ids: zml.Tensor, @@ -682,126 +520,27 @@ pub fn fusedExperts( const expert_partition = gate_up.weight.shape().partition(.expert); const quant_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; + stdx.debug.assert(quant_scheme == .nvfp4, "FlashInfer CUTLASS MoE only supports NVFP4, got {?}", .{quant_scheme}); - if (quant_scheme != null and quant_scheme == .nvfp4) { - const gate_up_weight_unpacked = zml.moe.unpackedWeight(gate_up); - const down_weight_unpacked = zml.moe.unpackedWeight(down); - - // TODO(Corentin): Do error checking on nvfp4 - // Also, maybe pass `zml.nn.Linear` directly - if (expert_partition.isSharded()) { - return zml.ops.manualComputation( - (struct { - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - gate_up_weight_unpacked: zml.Tensor, - down_weight_unpacked: zml.Tensor, - gate_up_input_scale: zml.Tensor, - gate_up_scales: zml.Tensor, - gate_up_global_scale: zml.Tensor, - down_input_scale: zml.Tensor, - down_scales: zml.Tensor, - down_global_scale: zml.Tensor, - activation: zml.moe.Activation, - enable_pdl: bool, - gemm1_tactic: i32, - gemm2_tactic: i32, - workspace_query_device: i32, - - fn body( - self: @This(), - _: zml.Shape, - ) zml.Tensor { - const local_num_experts = self.gate_up_weight_unpacked.dim(.expert); - const partition_id = zml.ops.partitionId().convert(.i32); - const expert_start = partition_id.scale(local_num_experts).convert(.i32); - const expert_end = expert_start.addConstant(local_num_experts); - - const local_route_mask = self.topk_ids - .cmp(.GE, expert_start) - .logical(.AND, self.topk_ids.cmp(.LT, expert_end)); - const local_topk_ids = local_route_mask.select( - self.topk_ids.sub(expert_start), - zml.Tensor.scalar(-1, .i32), - ); - const local_topk_weights = local_route_mask.select( - self.topk_weights, - zml.Tensor.scalar(-1, self.topk_weights.dtype()), - ); - - const local_output = fusedExpertsNvfp4( - self.input, - self.gate_up_weight_unpacked, - self.down_weight_unpacked, - local_topk_weights, - local_topk_ids, - self.gate_up_input_scale, - self.gate_up_scales, - self.gate_up_global_scale, - self.down_input_scale, - self.down_scales, - self.down_global_scale, - .{ - .workspace_query_device = self.workspace_query_device, - .activation = self.activation, - .enable_pdl = self.enable_pdl, - .gemm1_tactic = self.gemm1_tactic, - .gemm2_tactic = self.gemm2_tactic, - }, - ); - const local_reshaped = local_output - .reshape(self.input.shape().dims()) - .withTags(.{ .t, .d }); - return zml.ops.allReduce(local_reshaped, zml.Tensor.add); - } - }).body, - .{ - .input = input, - .topk_ids = topk_ids, - .topk_weights = topk_weights, - .gate_up_weight_unpacked = gate_up_weight_unpacked, - .down_weight_unpacked = down_weight_unpacked, - .gate_up_input_scale = gate_up.quantization.?.input_scale.?.asMultiplier(), - .gate_up_scales = gate_up.quantization.?.scales, - .gate_up_global_scale = gate_up.quantization.?.global_scale.?.asMultiplier(), - .down_input_scale = down.quantization.?.input_scale.?.asMultiplier(), - .down_scales = down.quantization.?.scales, - .down_global_scale = down.quantization.?.global_scale.?.asMultiplier(), - .activation = runner_options.activation, - .enable_pdl = runner_options.enable_pdl, - .gemm1_tactic = runner_options.gemm1_tactic, - .gemm2_tactic = runner_options.gemm2_tactic, - .workspace_query_device = runner_options.workspace_query_device, - }, - input.shape(), - ); - } - - return fusedExpertsNvfp4( - input, - gate_up_weight_unpacked, - down_weight_unpacked, - topk_weights, - topk_ids, - gate_up.quantization.?.input_scale.?.asMultiplier(), - gate_up.quantization.?.scales, - gate_up.quantization.?.global_scale.?.asMultiplier(), - down.quantization.?.input_scale.?.asMultiplier(), - down.quantization.?.scales, - down.quantization.?.global_scale.?.asMultiplier(), - runner_options, - ); - } + const gate_up_weight_unpacked = zml.moe.unpackedWeight(gate_up); + const down_weight_unpacked = zml.moe.unpackedWeight(down); + // TODO(Corentin): Do error checking on nvfp4 + // Also, maybe pass `zml.nn.Linear` directly if (expert_partition.isSharded()) { return zml.ops.manualComputation( (struct { input: zml.Tensor, topk_ids: zml.Tensor, topk_weights: zml.Tensor, - weights_gate_up: zml.Tensor, - weights_down: zml.Tensor, + gate_up_weight_unpacked: zml.Tensor, + down_weight_unpacked: zml.Tensor, + gate_up_input_scale: zml.Tensor, + gate_up_scales: zml.Tensor, + gate_up_global_scale: zml.Tensor, + down_input_scale: zml.Tensor, + down_scales: zml.Tensor, + down_global_scale: zml.Tensor, activation: zml.moe.Activation, enable_pdl: bool, gemm1_tactic: i32, @@ -812,7 +551,7 @@ pub fn fusedExperts( self: @This(), _: zml.Shape, ) zml.Tensor { - const local_num_experts = self.weights_gate_up.dim(.expert); + const local_num_experts = self.gate_up_weight_unpacked.dim(.expert); const partition_id = zml.ops.partitionId().convert(.i32); const expert_start = partition_id.scale(local_num_experts).convert(.i32); const expert_end = expert_start.addConstant(local_num_experts); @@ -829,12 +568,18 @@ pub fn fusedExperts( zml.Tensor.scalar(-1, self.topk_weights.dtype()), ); - const local_output = fusedExpertsBf16( + const local_output = fusedExpertsNvfp4( self.input, - self.weights_gate_up, - self.weights_down, + self.gate_up_weight_unpacked, + self.down_weight_unpacked, local_topk_weights, local_topk_ids, + self.gate_up_input_scale, + self.gate_up_scales, + self.gate_up_global_scale, + self.down_input_scale, + self.down_scales, + self.down_global_scale, .{ .workspace_query_device = self.workspace_query_device, .activation = self.activation, @@ -853,8 +598,14 @@ pub fn fusedExperts( .input = input, .topk_ids = topk_ids, .topk_weights = topk_weights, - .weights_gate_up = gate_up.weight, - .weights_down = down.weight, + .gate_up_weight_unpacked = gate_up_weight_unpacked, + .down_weight_unpacked = down_weight_unpacked, + .gate_up_input_scale = gate_up.quantization.?.input_scale.?.asMultiplier(), + .gate_up_scales = gate_up.quantization.?.scales, + .gate_up_global_scale = gate_up.quantization.?.global_scale.?.asMultiplier(), + .down_input_scale = down.quantization.?.input_scale.?.asMultiplier(), + .down_scales = down.quantization.?.scales, + .down_global_scale = down.quantization.?.global_scale.?.asMultiplier(), .activation = runner_options.activation, .enable_pdl = runner_options.enable_pdl, .gemm1_tactic = runner_options.gemm1_tactic, @@ -865,12 +616,18 @@ pub fn fusedExperts( ); } - return fusedExpertsBf16( + return fusedExpertsNvfp4( input, - gate_up.weight, - down.weight, + gate_up_weight_unpacked, + down_weight_unpacked, topk_weights, topk_ids, + gate_up.quantization.?.input_scale.?.asMultiplier(), + gate_up.quantization.?.scales, + gate_up.quantization.?.global_scale.?.asMultiplier(), + down.quantization.?.input_scale.?.asMultiplier(), + down.quantization.?.scales, + down.quantization.?.global_scale.?.asMultiplier(), runner_options, ); } diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index bf6a0a37..25ce26d6 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -28,8 +28,6 @@ pub const RoutingWeightPlacement = enum { before_down, after_down }; pub const ExpertsLayout = struct { /// Whether the two projections occupy contiguous halves or alternate. gate_up: ProjectionLayout, - /// Projection order within the concatenated halves or interleaved pairs. - gate_up_order: enum { gate_up, up_gate } = .gate_up, /// Backend specific transformation of the weights and scales. packing: Packing, @@ -97,7 +95,9 @@ pub const Backend = enum { .cute else .triton, - .nvfp4 => if (cutlass_flashinfer.isNvfp4Supported(platform)) + .nvfp4 => if (cute.isNvfp4Supported(platform)) + .cute + else if (cutlass_flashinfer.isNvfp4Supported(platform)) .flashinfer_cutlass else error.UnsupportedQuantization, @@ -155,11 +155,11 @@ pub const Backend = enum { pub fn expertsLayout(backend: Backend, scheme: ?zml.Quantization.Scheme) !ExpertsLayout { return switch (backend) { .stablehlo => .{ .gate_up = .concatenated, .packing = .plain }, - .cute => if (scheme == .mxfp4) .{ .gate_up = .interleaved, .packing = .swizzled_scales } else error.UnsupportedQuantization, + .cute => if (scheme == .mxfp4 or scheme == .nvfp4) .{ .gate_up = .interleaved, .packing = .swizzled_scales } else error.UnsupportedQuantization, .flashinfer_cutlass => if (scheme) |s| switch (s) { .nvfp4 => .{ .gate_up = .concatenated, .packing = .flashinfer_nvfp4 }, else => error.UnsupportedQuantization, - } else .{ .gate_up = .concatenated, .gate_up_order = .up_gate, .packing = .plain }, + } else error.UnsupportedQuantization, .mosaic_tpu, .metal, .tcl => .{ .gate_up = .concatenated, .packing = .plain }, .triton, .fly => if (scheme == .mxfp4) .{ .gate_up = .interleaved, .packing = .plain } else .{ .gate_up = .concatenated, .packing = .plain }, }; diff --git a/zml/kernels/moe/stablehlo/stablehlo.zig b/zml/kernels/moe/stablehlo/stablehlo.zig index f32769e8..e33d2a14 100644 --- a/zml/kernels/moe/stablehlo/stablehlo.zig +++ b/zml/kernels/moe/stablehlo/stablehlo.zig @@ -104,11 +104,12 @@ fn roundedInput(x: Tensor, linear: zml.nn.Linear, opts: Options, dtype: zml.Data if (scheme == .nvfp4) { const global = if (q.input_scale) |scale| scale.asMultiplier() else Tensor.scalar(1, .f32); stdx.debug.assert(global.count() == 1, "reference NVFP4 input scale must be global", .{}); - const grouped = x.convert(.bf16).convert(.f32).div(global.asScalar()) + // Keep both stored rounding boundaries when XLA fuses the quantizer. + const grouped = x.convert(.bf16).optimizationBarrier().convert(.f32).div(global.asScalar()) .splitAxis(.in, .{ .group = @divExact(x.dim(.in), 16), .element = 16 }); const scale = grouped.abs().max(.element).scale(1.0 / 6.0) .minimum(Tensor.scalar(448, .f32)) - .convert(.f8e4m3fn).convert(.f32).broad(grouped.shape()); + .convert(.f8e4m3fn).optimizationBarrier().convert(.f32).broad(grouped.shape()); const divisor = scale.cmp(.NE, Tensor.scalar(0, .f32)).select(scale, Tensor.scalar(1, .f32).broad(scale.shape())); return roundFp4(grouped.mul(Tensor.scalar(1, .f32).div(divisor))).mul(scale) .reshape(x.shape().withDtype(.f32)).mul(global.asScalar()); diff --git a/zml/kernels/moe/tests.zig b/zml/kernels/moe/tests.zig index cfeb05ba..2bdebdfe 100644 --- a/zml/kernels/moe/tests.zig +++ b/zml/kernels/moe/tests.zig @@ -16,6 +16,7 @@ const Case = struct { quantize_input: bool = false, placement: Placement = .after_down, bias: bool = false, + non_unit_scales: bool = false, tokens: i64 = 1, topk: i64 = 1, width: i64 = 128, @@ -55,8 +56,7 @@ const Case = struct { // BF16 intermediate rounding and different GEMM reduction orders. Every // element must pass; near-zero values use the absolute bound. try zml.testing.expectClose(io, output.expected, output.actual, .{ - // The NVFP4 matrix observed a maximum absolute error of 0.07519531. - // Allow a small margin; this is an empirical comparison budget. + // Share the NVFP4 comparison budget across CUDA backends. .absolute_tolerance = if (c.scheme == .nvfp4) 0.078125 else 0.015625, .relative_tolerance = 0.02, .minimum_close_fraction = 1, @@ -148,10 +148,24 @@ fn makeExpertFixture(c: Case) ExpertFixture { .silu, .gelu, .relu => c.intermediate, else => 2 * c.intermediate, }; - return .{ + var fixture: ExpertFixture = .{ .gate_up = linear(.init(.{ .expert = c.experts, .dout = projection_width, .d = c.width }, c.activation_dtype), c.scheme, c.bias, 7), .down = linear(.init(.{ .expert = c.experts, .d = c.width, .dout = c.intermediate }, c.activation_dtype), c.scheme, c.bias, 11), }; + if (c.non_unit_scales) { + for ([_]*zml.nn.Linear{ &fixture.gate_up, &fixture.down }, 0..) |projection, i| { + const q = &projection.quantization.?; + std.debug.assert(q.scheme == .nvfp4); + // Exercise both scale polarity and distinct per-expert multipliers. + q.input_scale = .{ .value = Tensor.scalar(@as(f32, if (i == 0) 2 else 0.5), .f32), .operation = .divide }; + q.global_scale = .{ + .value = Tensor.arange(.{ .end = @intCast(c.experts) }, .i32) + .remainder(Tensor.scalar(2, .i32)).addConstant(1).convert(.f32).withTags(.{.expert}), + .operation = .divide, + }; + } + } + return fixture; } /// Interleave the gate/up halves of the output axis: [E, 2, N/2, K] -> [E, N/2, 2, K]. @@ -176,16 +190,9 @@ fn packFp4Bytes(weight: Tensor) Tensor { } fn prepareGateUp(tensor: Tensor, layout: zml.moe.ExpertsLayout) Tensor { - const ordered = switch (layout.gate_up_order) { - .gate_up => tensor, - .up_gate => blk: { - const mid = @divExact(tensor.dim(1), 2); - break :blk Tensor.concatenate(&.{ tensor.slice(1, .{ .start = mid }), tensor.slice(1, .{ .end = mid }) }, 1); - }, - }; return switch (layout.gate_up) { - .concatenated => ordered, - .interleaved => interleave(ordered), + .concatenated => tensor, + .interleaved => interleave(tensor), }; } @@ -211,7 +218,15 @@ fn prepareForBackend(canonical: ExpertFixture, backend: zml.moe.Backend) ExpertF switch (backend) { .cute => { projection.weight = packFp4Bytes(projection.weight); - projection.quantization.?.scales = projection.quantization.?.scales.bitCast(.u8); + const q = &projection.quantization.?; + q.scales = q.scales.bitCast(.u8); + if (scheme == .nvfp4) { + // CuTe takes the input quantization multiplier and a GEMM + // output multiplier that folds in its inverse. + const input_scale = q.input_scale.?.asMultiplier(); + q.global_scale = .{ .value = q.global_scale.?.asMultiplier().mul(input_scale), .operation = .multiply }; + q.input_scale = .{ .value = Tensor.scalar(1, .f32).div(input_scale), .operation = .multiply }; + } }, .triton => if (scheme == .mxfp4) { projection.weight = packFp4Bytes(projection.weight); @@ -261,7 +276,7 @@ fn forward(input: Tensor, c: Case) Outputs { .routing_weight_placement = c.placement, .input_quantization = switch (c.backend) { .metal => .none, - .cute => .mxfp8_bf16, + .cute => if (c.scheme == .nvfp4) .automatic else .mxfp8_bf16, .triton => if (c.scheme == .mxfp4 and zml.Compiler.current().platform.target == .cuda) .mxfp8 else .automatic, else => .automatic, }, @@ -306,8 +321,13 @@ const specialized_shapes = [_]FixtureShape{ .{ .width = 5120, .intermediate = 2304, .experts = 2, .tokens = 17, .topk = 2 }, .{ .width = 5120, .intermediate = 2304, .experts = 2, .tokens = 34, .topk = 2 }, }; +// Exercise direct decode routing with multiple experts; a single route is unsupported. +const specialized_shapes_cute = [_]FixtureShape{ + .{ .width = 5120, .intermediate = 2304, .experts = 2, .tokens = 1, .topk = 2 }, +} ++ specialized_shapes[1..].*; const Format = struct { + non_unit_scales: bool = false, scheme: ?zml.Quantization.Scheme = null, activation_dtype: zml.DataType = .bf16, }; @@ -345,6 +365,7 @@ const Matrix = struct { .activation = activation, .scheme = format.scheme, .activation_dtype = format.activation_dtype, + .non_unit_scales = format.non_unit_scales, .quantize_input = quantize_input, .placement = placement, .bias = bias, @@ -588,7 +609,7 @@ test "FlashInfer compatibility matrix" { // Only default SwiGLU, no linear bias, and routing after the down projection. try pipeline.run(std.testing.io, std.testing.allocator, .flashinfer_cutlass, .{ - .formats = &.{ .{}, .{ .scheme = .nvfp4 } }, + .formats = &.{.{ .scheme = .nvfp4 }}, .activations = &.{.{ .swiglu = .{} }}, }, .{}); } @@ -603,7 +624,27 @@ test "specialized CuTe MXFP4 compatibility matrix" { .activations = &clipped_swiglu, .quantize_inputs = &.{ false, true }, .placements = &.{.before_down}, - .shapes = &specialized_shapes, + .shapes = &specialized_shapes_cute, + }, .{}); +} + +test "CuTe NVFP4 compatibility matrix" { + const platform = zml.testing.env(); + if (!zml.moe.cute.isNvfp4Supported(platform)) return error.SkipZigTest; + + const pipeline: Pipeline = .{ .platform = platform }; + try pipeline.run(std.testing.io, std.testing.allocator, .cute, .{ + .formats = &.{ + .{ .scheme = .nvfp4 }, + .{ .scheme = .nvfp4, .non_unit_scales = true }, + }, + .activations = &.{ .{ .swiglu = .{} }, .{ .swiglu = .{ .limit = 0.25 } } }, + .placements = &.{ .before_down, .after_down }, + .shapes = &.{ + .{ .width = 5120, .intermediate = 2304, .tokens = 1, .topk = 2 }, + .{ .width = 5120, .intermediate = 2304, .tokens = 17, .topk = 2 }, + .{ .width = 5120, .intermediate = 2304, .tokens = 34, .topk = 3 }, + }, }, .{}); } From 86ab457d0b24d238564e51bc9a057ef1d034b1b1 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Tue, 3 Nov 2026 10:26:31 +0100 Subject: [PATCH 835/881] dsl/cute: add SM100 primitives and CTA-group support (#1007) Add helpers for TMA gather/prefetch, weight-stationary MMA, shared-memory descriptors, packed FP32 arithmetic, async cluster stores, and register limits. Make CTA groups and barrier scopes explicit across TMA, block-scaled MMA, TMEM, and synchronization, including two-CTA operations and multicast completion. Consolidate named CTA barriers and update the MoE and paged-attention callers. Cover instruction operands, descriptor encoding, and MLIR round-trips in builder tests. --- devenv.lock | 18 +- dsl/cute/README.md | 19 +- dsl/cute/builder.zig | 1259 +++++++++++++++-- mlir/dialects/cute_ir/capi.cc | 4 + mlir/dialects/cute_ir/capi.h | 1 + mlir/dialects/cute_ir/cute_ir.zig | 4 + mlir/dialects/llvm/llvm.zig | 1 + mlir/dialects/llvm/ops.zig | 9 + mlir/dialects/llvm/test.zig | 1 + mlir/dialects/nvvm/nvvm.zig | 34 +- zml/kernel.zig | 11 + zml/kernels/moe/cute/sm10x/common.zig | 2 +- .../moe/cute/sm10x/persistent_mxfp4.zig | 58 +- .../moe/cute/sm10x/persistent_nvfp4.zig | 54 +- .../paged_attention/cute/sm10x/attention.zig | 20 +- 15 files changed, 1295 insertions(+), 200 deletions(-) diff --git a/devenv.lock b/devenv.lock index bb3ef660..6c3be332 100644 --- a/devenv.lock +++ b/devenv.lock @@ -3,11 +3,11 @@ "devenv": { "locked": { "dir": "src/modules", - "lastModified": 1779636362, - "narHash": "sha256-Q/hy/ybvbe91fjA1MbulUwPjJH9yKaFZnSaTkwLE8S0=", + "lastModified": 1793199252, + "narHash": "sha256-LRgdAdMHL3uIX+kix/oyMzG+gMxzSYOzLrrRaoZthog=", "owner": "cachix", "repo": "devenv", - "rev": "9d6e68dac81fab1f7f2ec6784220151b4fba4b59", + "rev": "0d6c556e5f862c14a75a543e2ee4fdb7acc005ad", "type": "github" }, "original": { @@ -22,11 +22,11 @@ "nixpkgs-src": "nixpkgs-src" }, "locked": { - "lastModified": 1778507786, - "narHash": "sha256-HzSQCKMsMr8r55LwM1JuzIOB+8bzk0FEv6sItKvsfoY=", + "lastModified": 1793043970, + "narHash": "sha256-m14odYTdkBUEkJOiEYs2zjGjn8fcjvdCJsm45+k084U=", "owner": "cachix", "repo": "devenv-nixpkgs", - "rev": "8f24a228a782e24576b155d1e39f0d914b380691", + "rev": "d97234e58d4e90e3bf77c082fc3b546e51200938", "type": "github" }, "original": { @@ -39,11 +39,11 @@ "nixpkgs-src": { "flake": false, "locked": { - "lastModified": 1778274207, - "narHash": "sha256-I4puXmX1iovcCHZlRmztO3vW0mAbbRvq4F8wgIMQ1MM=", + "lastModified": 1792812153, + "narHash": "sha256-T2+XBfNbf9pecLUWS4+RtuVNZ3W8K5taG5QdV71mWlk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b3da656039dc7a6240f27b2ef8cc6a3ef3bccae7", + "rev": "674a2028661e550a97400c1cc992b9cca9a668a1", "type": "github" }, "original": { diff --git a/dsl/cute/README.md b/dsl/cute/README.md index c7828abe..946c542d 100644 --- a/dsl/cute/README.md +++ b/dsl/cute/README.md @@ -88,19 +88,20 @@ the actual IR regions. | `cute.arch.griddepcontrol_launch_dependents()` | `b.launchDependents()` | | `x.to(Float32)` | `x.to(.f32)` | | `if cond:` / `for i in range(...)` | `b.openIf(cond)` / `b.openFor(0, n, 1, .{})` | -| `cpasync.make_tiled_tma_atom(...)` | `b.makeTiledTmaLoadAtom(...)` | +| `cpasync.make_tiled_tma_atom(...)` | `b.makeTiledTmaAtom(...)` | | `cpasync.tma_partition(...)` | `b.tmaPartition(...)` | -| `cute.copy(atom, src, dst)` | `b.copy(atom, src, dst, pred)` | +| `cute.copy(atom, src, dst)` | `b.copy(atom, src, dst, .{})` | | `cute.make_tiled_mma(atom)` | `b.makeTiledMma(atom, result_type)` | | `tiled_mma.make_fragment_A/B/C(...)` | `b.mmaMakeFragment(...)` | | `cute.gemm(...)` | `b.gemm(...)` | | `cute.arch.alloc_tmem(...)` | `b.allocTmem(...)` | | `cute.arch.retrieve_tmem_ptr(...)` | `b.retrieveTmemPtr(...)` | -* **Layouts are static.** `makeLayout` takes Zig slices and emits one - `cute.static`; the tensor keeps its shape and stride in Zig, so `dim`, - `size` and coordinates need no MLIR queries. Coordinates are tuples of - `Value`s and ints; ints stay static in the `!cute.coord` type. +* **Layouts accept static and dynamic extents.** `makeLayout` takes Zig slices + for a tracked static `Layout`, or nested tuples of integers and SSA `Value`s + for a `View`. Dynamic shape and stride leaves retain their integer widths. + The tracked `Tensor` keeps its shape and stride in Zig; use `makeTensorView` + with dynamic layouts. Integers stay static in coordinate types. * **Stride defaults differ by origin.** `makeLayout` defaults to CuTe's compact left-most stride like `cute.make_layout`; a `tensor` argument defaults to XLA's row-major layout, which is what the buffer holds. @@ -122,7 +123,11 @@ the actual IR regions. copies, tensor-memory fragments, and SM100 block-scaled MMA. CuTe derives their layouts in the compiler, so callers spell out the resulting CuTe type with `View` instead of relying on Python's dynamic type objects. -* **Not covered yet:** dynamic tensor shapes and automatic CuTe layout algebra. +* **TMA tilers follow Python.** `makeTiledTmaAtom` accepts a shape tuple, tile, + or layout, composes it with the GMEM identity layout, and removes the SMEM + stage mode when present. The lower-level `makeTiledTmaLoadAtom` and + `makeTiledTmaStoreAtom` take an already composed coordinate map. `copy` + makes store atoms executable automatically; loads require `tma_bar_ptr`. * **Typed result types.** Operations whose result type CuTe derives (`localTile`, `slice`, `tmaPartition`, ...) take an `*mlir.Type` assembled with `layoutSpec`, `layoutType`, `memrefType`, `coordTensorType`, diff --git a/dsl/cute/builder.zig b/dsl/cute/builder.zig index d19819fa..926b8419 100644 --- a/dsl/cute/builder.zig +++ b/dsl/cute/builder.zig @@ -51,6 +51,8 @@ pub const AlgebraExpr = union(enum) { static: i64, /// A dynamic leaf known to be a multiple of this, 1 when nothing is known. dynamic: i64, + /// A dynamic integer with an explicit bit width and divisibility. + dynamic_int: struct { width: u8, divisible_by: i64 = 1 }, all, basis: ScaledBasis, tuple: []const AlgebraExpr, @@ -466,8 +468,122 @@ pub const TmaFormat = enum { u16, }; -/// Typed description of a tiled TMA atom: the element type, the bits moved -/// per copy, the basis of one box, and the coordinate layout of the tensor. +/// CuTe's 64-bit TMA `.L2::cache_hint` policies for SM90. +pub const CacheHintSm90 = enum(u64) { + evict_normal = 0x1000000000000000, + evict_first = 0x12F0000000000000, + evict_last = 0x14F0000000000000, +}; + +/// CuTe's 64-bit TMA `.L2::cache_hint` policies for SM100. +pub const CacheHintSm100 = enum(u64) { + evict_normal = 0x1000000000000000, + evict_first = 0x12F0000000000000, + evict_last = 0x14F0000000000000, +}; + +pub const CtaGroup = enum(u8) { + one = 1, + two = 2, + + fn toNvvm(self: CtaGroup) nvvm.CtaGroup { + return switch (self) { + .one => .cta_1, + .two => .cta_2, + }; + } +}; +pub const MBarrierScope = nvvm.MBarrierScope; +pub const OperandMajorMode = enum(u1) { k = 0, mn = 1 }; + +/// Number of CTAs cooperating on one instruction, as in CuTe's CtaGroup.value. +pub fn ctaCount(group: CtaGroup) u8 { + return @backingInt(group); +} + +pub const Tcgen05SmemSwizzle = enum(u3) { + none = 0, + swizzle_128b_atom_32b = 1, + swizzle_128b = 2, + swizzle_64b = 4, + swizzle_32b = 6, +}; + +/// CuTe's packed SM100 shared-memory descriptor. All offsets are in bytes. +pub const Tcgen05SmemDesc = struct { + pub const Options = struct { + leading_byte_offset: u32 = 0, + stride_byte_offset: u32 = 0, + version: u2 = 1, + base_offset: u3 = 0, + layout: Tcgen05SmemSwizzle = .none, + }; + + pub fn build(b: *Builder, start_address: Value, opts: Options) Value { + std.debug.assert(opts.leading_byte_offset % 16 == 0 and opts.leading_byte_offset < 0x40000); + std.debug.assert(opts.stride_byte_offset % 16 == 0 and opts.stride_byte_offset < 0x40000); + const fields = (@as(u64, opts.leading_byte_offset >> 4) << 16) | + (@as(u64, opts.stride_byte_offset >> 4) << 32) | + (@as(u64, opts.version) << 46) | + (@as(u64, opts.base_offset) << 49) | + (@as(u64, @backingInt(opts.layout)) << 61); + const address = if (start_address.isInt()) start_address else b.ptrToInt(start_address, .i32); + return address.to(.i64).shrLogical(4).bitAnd(0x3fff).bitOr(b.constant(.i64, @as(i64, @bitCast(fields)))); + } +}; + +/// Dense SM100 instruction descriptor; the MMA kind disambiguates dtype encodings. +pub const Tcgen05InstrDesc = struct { + pub const Options = struct { + a_dtype: DType, + b_dtype: DType, + c_dtype: DType = .f32, + m_dim: u16, + n_dim: u16, + a_major: OperandMajorMode = .k, + b_major: OperandMajorMode = .k, + a_negate: bool = false, + b_negate: bool = false, + }; + + pub fn build(comptime opts: Options) u32 { + comptime std.debug.assert(opts.m_dim > 0 and opts.m_dim <= 256 and opts.m_dim % 16 == 0); + comptime std.debug.assert(opts.n_dim > 0 and opts.n_dim <= 256 and opts.n_dim % 8 == 0); + const half_inputs = (opts.a_dtype == .f16 or opts.a_dtype == .bf16) and (opts.b_dtype == .f16 or opts.b_dtype == .bf16); + const fp8_inputs = (opts.a_dtype == .f8e4m3fn or opts.a_dtype == .f8e5m2) and (opts.b_dtype == .f8e4m3fn or opts.b_dtype == .f8e5m2); + const valid = (half_inputs and opts.c_dtype == .f32) or + (opts.a_dtype == .f16 and opts.b_dtype == .f16 and opts.c_dtype == .f16) or + (opts.a_dtype == .f32 and opts.b_dtype == .f32 and opts.c_dtype == .f32) or + (fp8_inputs and (opts.c_dtype == .f16 or opts.c_dtype == .f32)) or + (opts.a_dtype == .i8 and opts.b_dtype == .i8 and opts.c_dtype == .i32); + if (!valid) @compileError("unsupported dense tcgen05 dtype combination"); + const c: u32 = switch (opts.c_dtype) { + .f16 => 0, + .f32 => 1, + .i32 => 2, + else => @compileError("unsupported tcgen05 accumulator dtype"), + }; + return (c << 4) | (inputFormat(opts.a_dtype) << 7) | (inputFormat(opts.b_dtype) << 10) | + (@as(u32, @intFromBool(opts.a_negate)) << 13) | + (@as(u32, @intFromBool(opts.b_negate)) << 14) | + (@as(u32, @backingInt(opts.a_major)) << 15) | + (@as(u32, @backingInt(opts.b_major)) << 16) | + (@as(u32, opts.n_dim >> 3) << 17) | (@as(u32, opts.m_dim >> 4) << 24); + } + + fn inputFormat(comptime dtype: DType) u32 { + return switch (dtype) { + .f16, .f8e4m3fn => 0, + .bf16, .f8e5m2, .i8 => 1, + .f32 => 2, + .f4e2m1fn => 5, + else => @compileError("unsupported tcgen05 input dtype"), + }; + } +}; + +/// Typed description of a non-multicast tiled TMA atom: the element type, +/// bits moved per copy, basis of one box, and coordinate layout of the tensor. pub const TmaConfig = struct { dtype: DType, copy_bits: u32, @@ -475,18 +591,29 @@ pub const TmaConfig = struct { coordinate_layout: LayoutSpec, coordinate_rank: u8, format: TmaFormat = .default, + /// Load completion group: with `.two`, each CTA's load completes on + /// the mbarrier of the pair's even CTA. Ignored by TMA stores. + cta_group: CtaGroup = .one, + mode: enum { tiled, gather4 } = .tiled, }; +pub const CopyBulkTensorTileG2SOp = struct { cta_group: CtaGroup = .one }; +pub const CopyBulkTensor2DGather4G2SOp = struct { cta_group: CtaGroup = .one }; +pub const CopyBulkTensorTileS2GOp = struct {}; + pub const BlockScaledMma = struct { atom: Atom, tiled: Atom, }; -/// SM100 block-scaled MMA instruction: one CTA computes a 128 x `n` tile, -/// FP4 weights as A. The instruction `kind` fixes B, the scales and K. +/// SS, K-major SM100 block-scaled MMA instruction with FP4 weights as A. +/// The instruction `kind` fixes B, the scales and K. pub const BlockScaledMmaConfig = struct { kind: Kind = .mxfp4_mxfp8, + /// M is 128 for one CTA, or 128 or 256 for a CTA pair. + m: u16 = 128, n: u16, + cta_group: CtaGroup = .one, pub const Kind = enum { /// MXFP4 weights x MXFP8 activations: E8M0 scales every 32 @@ -524,8 +651,23 @@ pub const BlockScaledMmaConfig = struct { .nvfp4_nvfp4 => 16, }; } + + fn validate(self: BlockScaledMmaConfig) void { + const m_ok = self.m == 128 or (self.cta_group == .two and self.m == 256); + const n_multiple: u16 = if (self.cta_group == .two) 16 else 8; + if (!m_ok or self.n < n_multiple or self.n > 256 or self.n % n_multiple != 0) { + std.debug.panic("unsupported SM100 block-scaled MMA shape {d}x{d}x{d} on {d} CTAs", .{ self.m, self.n, self.k(), ctaCount(self.cta_group) }); + } + } }; +fn tmaLoadKind(config: TmaConfig) cute_nvgpu.TiledTmaLoad { + return switch (config.cta_group) { + .one => .sm_90, + .two => .sm_100_2sm, + }; +} + fn tmaDataFormat(format: TmaFormat, dtype: DType) cute_nvgpu.TmaDataFormat { return switch (format) { .u4 => .U4, @@ -1019,7 +1161,8 @@ pub const Builder = struct { /// `cute.ptrtoint`: the integer address of a pointer (`i32` for shared /// memory, `i64` for global memory), for hand-written PTX operands. pub fn ptrToInt(self: *Builder, pointer: anytype, dt: DType) Value { - return self.emit(cute_ir.ptrtoint(self.ctx, self.asValue(pointer).inner, dt.toMlir(self.ctx), self.loc())); + const ptr = self.asValue(pointer); + return self.emit(if (ptr.isPtr()) cute_ir.ptrtoint(self.ctx, ptr.inner, dt.toMlir(self.ctx), self.loc()) else llvm.ptrtoint(self.ctx, ptr.inner, dt.toMlir(self.ctx), self.loc())); } /// Reinterpret a pointer without changing its address. This is the typed @@ -1074,6 +1217,7 @@ pub const Builder = struct { const result = switch (e) { .static => |v| if (@hasDecl(A, "getStatic")) A.getStatic(self.ctx, v) else std.debug.panic("{s} has no integer leaf", .{@typeName(A)}), .dynamic => |divisibility| if (@hasDecl(A, "getDynamic")) A.getDynamic(self.ctx, 32, divisibility) else std.debug.panic("{s} has no dynamic leaf", .{@typeName(A)}), + .dynamic_int => |v| if (@hasDecl(A, "getDynamic")) A.getDynamic(self.ctx, v.width, v.divisible_by) else std.debug.panic("{s} has no dynamic leaf", .{@typeName(A)}), .all => if (@hasDecl(A, "getUnderscore")) A.getUnderscore(self.ctx) else std.debug.panic("{s} has no `_` leaf", .{@typeName(A)}), .basis => |b| if (@hasDecl(A, "getBasis")) blk: { const modes = self.arena.allocator().alloc(i32, b.mode_count) catch @panic("OOM"); @@ -1238,8 +1382,59 @@ pub const Builder = struct { // ==================== cute.core ==================== - /// `cute.make_layout(shape, stride=...)`, static only. - pub fn makeLayout(self: *Builder, shape: []const i64, opts: LayoutOpts) Layout { + fn layoutResult(comptime Shape: type) type { + return switch (@typeInfo(Shape)) { + .pointer => Layout, + else => View, + }; + } + + /// `cute.make_layout(shape, stride=...)`. Slices keep the tracked static + /// Layout API; nested tuples accept mixed integer and SSA shape/stride leaves. + pub fn makeLayout(self: *Builder, shape: anytype, opts: anytype) layoutResult(@TypeOf(shape)) { + if (comptime layoutResult(@TypeOf(shape)) == Layout) return self.makeStaticLayout(shape, .{ + .stride = if (@hasField(@TypeOf(opts), "stride")) opts.stride else null, + }); + var shape_values: stdx.BoundedArray(*const mlir.Value, MAX_RANK) = .empty; + var stride_values: stdx.BoundedArray(*const mlir.Value, MAX_RANK) = .empty; + const shape_expr = self.coordExpr(&shape_values, shape); + const stride_expr = if (@hasField(@TypeOf(opts), "stride") and @TypeOf(opts.stride) != @TypeOf(null)) self.coordExpr(&stride_values, opts.stride) else blk: { + var stride: AlgebraExpr = .{ .static = 1 }; + var stride_value: ?Value = null; + var index: usize = 0; + break :blk self.compactLayoutStride(shape_expr, shape_values.constSlice(), &index, &stride, &stride_value, &stride_values); + }; + const shape_value = self.emit(cute_ir.make_shape(self.ctx, shape_values.constSlice(), self.shapeType(shape_expr), self.loc())); + const stride_type = (cute_ir.StrideType.get(self.ctx, .{ .attr = self.algebraAttr(cute_ir.StrideAttr, stride_expr).attribute() }) catch @panic("invalid stride")).type_(); + const stride_value = self.emit(cute_ir.make_stride(self.ctx, stride_values.constSlice(), stride_type, self.loc())); + const result = self.emit(cute_ir.make_layout(self.ctx, self.layoutType(.{ .shape = shape_expr, .stride = stride_expr }), .{ .shape = shape_value.inner, .stride = stride_value.inner }, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + fn compactLayoutStride(self: *Builder, shape: AlgebraExpr, values: []const *const mlir.Value, index: *usize, stride: *AlgebraExpr, stride_value: *?Value, dynamic: *stdx.BoundedArray(*const mlir.Value, MAX_RANK)) AlgebraExpr { + if (shape == .tuple) { + const elements = self.arena.allocator().alloc(AlgebraExpr, shape.tuple.len) catch @panic("OOM"); + for (shape.tuple, elements) |extent, *element| element.* = self.compactLayoutStride(extent, values, index, stride, stride_value, dynamic); + return .{ .tuple = elements }; + } + if (shape == .static and shape.static == 1) return .{ .static = 0 }; + const result = stride.*; + if (stride_value.*) |v| dynamic.append(v.inner) catch @panic("too many stride leaves"); + if (shape == .static and stride.* == .static) { + stride.static *= shape.static; + } else { + const extent = if (shape == .dynamic or shape == .dynamic_int) blk: { + const v: Value = .{ .inner = values[index.*], .kernel = self }; + index.* += 1; + break :blk v; + } else self.lift(shape.static); + stride_value.* = (stride_value.* orelse self.lift(stride.static)).mul(extent); + stride.* = dynamicExpr(stride_value.*.?); + } + return result; + } + + fn makeStaticLayout(self: *Builder, shape: []const i64, opts: LayoutOpts) Layout { if (shape.len == 0 or shape.len > MAX_RANK) std.debug.panic("makeLayout: rank {d} not in 1..{d}", .{ shape.len, MAX_RANK }); const stride = opts.stride orelse self.leftMost(shape); if (stride.len != shape.len) std.debug.panic("makeLayout: stride rank {d} != shape rank {d}", .{ stride.len, shape.len }); @@ -1262,8 +1457,8 @@ pub const Builder = struct { fn coordExpr(self: *Builder, dyn: *stdx.BoundedArray(*const mlir.Value, MAX_RANK), coord: anytype) AlgebraExpr { const T = @TypeOf(coord); if (T == Value) { - dyn.append(self.lift(coord).to(.i32).inner) catch @panic("makeCoord: too many dynamic leaves"); - return .{ .dynamic = 1 }; + dyn.append(coord.inner) catch @panic("makeCoord: too many dynamic leaves"); + return dynamicExpr(coord); } if (T == AlgebraToken) return switch (coord) { .all => .all, @@ -1282,6 +1477,12 @@ pub const Builder = struct { } } + fn dynamicExpr(value: Value) AlgebraExpr { + std.debug.assert(value.isInt()); + const width = dtypeBitwidth(value.dtype()); + return if (width == 32) .{ .dynamic = 1 } else .{ .dynamic_int = .{ .width = @intCast(width) } }; + } + /// A `!cute.coord` from nested tuples of `Value`s, integers, and `.all` /// placeholders. Integer leaves stay static in the dependent CuTe type. pub fn makeCoord(self: *Builder, coord: anytype) *const mlir.Value { @@ -1730,8 +1931,55 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn copy(self: *Builder, atom: Atom, src: anytype, dst: anytype, pred: ?Value) void { - self.emitVoid(cute_ir.copy(self.ctx, atom.inner, self.innerSlice(src), self.innerSlice(dst), .{ .pred = if (pred) |p| p.inner else null }, self.loc())); + pub const CopyOptions = struct { pred: ?Value = null, tma_bar_ptr: ?Value = null, cache_policy: ?Value = null }; + + pub fn copy(self: *Builder, atom: Atom, src: anytype, dst: anytype, opts: CopyOptions) void { + var exec = atom; + if (atom.type_().isA(cute_nvgpu.CopyAtomNonExecTiledTmaStoreType)) |ty| { + const exec_type = (cute_nvgpu.CopyAtomTmaStoreType.get(self.ctx, .{ + .valType = ty.valType(), + .copyBits = ty.copyBits(), + .mode = .tiled, + .gStride = self.emptyStrideType(), + .tmaGbasis = ty.tmaGbasis(), + }) catch @panic("invalid executable TMA store atom")).type_(); + exec = self.makeExecTma(atom, exec_type); + } + if (opts.tma_bar_ptr) |mbar| { + if (atom.type_().isA(cute_nvgpu.CopyAtomNonExec2DGather4TmaLoadType)) |ty| { + const exec_type = (cute_nvgpu.CopyAtomTmaLoadType.get(self.ctx, .{ + .valType = ty.valType(), + .copyBits = ty.copyBits(), + .mode = .gather4, + .numCta = if (ty.kind() == .sm_100_2sm) 2 else 1, + .gStride = self.emptyStrideType(), + .tmaGbasis = ty.tmaGbasis(), + }) catch @panic("invalid executable gather atom")).type_(); + exec = self.makeExecTma(atom, exec_type); + } else if (atom.type_().isA(cute_nvgpu.CopyAtomNonExecTiledTmaLoadType)) |ty| { + const exec_type = (cute_nvgpu.CopyAtomTmaLoadType.get(self.ctx, .{ + .valType = ty.valType(), + .copyBits = ty.copyBits(), + .mode = .tiled, + .numCta = if (ty.kind() == .sm_100_2sm) 2 else 1, + .gStride = self.emptyStrideType(), + .tmaGbasis = ty.tmaGbasis(), + }) catch @panic("invalid executable TMA atom")).type_(); + exec = self.makeExecTma(atom, exec_type); + } + exec = self.setTmaBarrier(exec, mbar); + } + if (opts.cache_policy) |policy| { + const field = if (exec.type_().isA(cute_nvgpu.CopyAtomTmaLoadType) != null) + (cute_nvgpu.AtomCopyFieldTmaLoadAttr.get(self.ctx, .{ .value = .cache_policy }) catch unreachable).attribute() + else if (exec.type_().isA(cute_nvgpu.CopyAtomTmaStoreType) != null) + (cute_nvgpu.AtomCopyFieldTmaStoreAttr.get(self.ctx, .{ .value = .cache_policy }) catch unreachable).attribute() + else + @panic("cache_policy requires an executable TMA copy atom"); + const updated = self.emit(cute_nvgpu.atom_set_value(self.ctx, exec.inner, policy.to(.i64).inner, exec.type_(), field, self.loc())); + exec = .{ .inner = updated.inner, .kernel = self }; + } + self.emitVoid(cute_ir.copy(self.ctx, exec.inner, self.innerSlice(src), self.innerSlice(dst), .{ .pred = if (opts.pred) |p| p.inner else null }, self.loc())); } pub fn gemm(self: *Builder, tiled: Atom, d: anytype, a: anytype, rhs: anytype, c: anytype) void { @@ -1762,9 +2010,67 @@ pub const Builder = struct { // the kernel receives them as grid-constant arguments and makes them // executable before issuing copies. + pub const TmaAtomOptions = struct { gmem_coord_tensor: ?View = null }; + + fn inferValue(self: *Builder, name: []const u8, operands: []const *const mlir.Value, attributes: []const mlir.NamedAttribute) Value { + return self.emit(mlir.Operation.make(self.ctx, name, .{ + .operands = .{ .flat = operands }, + .attributes = attributes, + .result_type_inference = true, + .location = self.loc(), + })); + } + + fn valueShape(self: *Builder, value: Value) Value { + if (value.type_().isA(cute_ir.ShapeType) != null) return value; + return self.inferValue("cute.get_shape", &.{value.inner}, &.{}); + } + + fn shapeRank(shape: Value) usize { + return shape.type_().isA(cute_ir.ShapeType).?.attr().isA(cute_ir.ShapeAttr).?.rank(); + } + + /// Common CuTe TMA atom constructor. Gather uses the index tensor's layout + /// at construction and passes data/index tensors together to partition/copy. + pub fn makeTiledTmaAtom(self: *Builder, op: anytype, gmem: anytype, smem_layout: anytype, cta_tiler: anytype, config: TmaConfig, opts: TmaAtomOptions) TmaDescriptor { + const Tiler = @TypeOf(cta_tiler); + const tiler = if (Tiler == Value or Tiler == View or Tiler == Layout) + self.asValue(cta_tiler) + else if (@typeInfo(Tiler) == .pointer) + self.makeShape(self.dimsExpr(cta_tiler)).value() + else + self.makeShape(cta_tiler).value(); + var smem = self.asValue(smem_layout); + const tiler_rank = shapeRank(self.valueShape(tiler)); + const smem_rank = shapeRank(self.valueShape(smem)); + std.debug.assert(smem_rank == tiler_rank or smem_rank == tiler_rank + 1); + if (smem_rank == tiler_rank + 1) { + const modes = self.arena.allocator().alloc(i32, tiler_rank) catch @panic("OOM"); + for (modes, 0..) |*mode, i| mode.* = @intCast(i); + smem = self.inferValue("cute.select", &.{smem.inner}, &.{.named(self.ctx, "mode", .denseArray(self.ctx, .i32, modes))}); + } + const shape = self.valueShape(self.asValue(gmem)); + const identity = self.inferValue("cute.make_identity_layout", &.{shape.inner}, &.{}); + const cta_map = self.inferValue("cute.composition", &.{ identity.inner, tiler.inner }, &.{}); + var cfg = config; + cfg.mode = .tiled; + if (@TypeOf(op) == CopyBulkTensorTileS2GOp) return self.makeTiledTmaStoreAtom(gmem, smem, cta_map, cfg); + cfg.cta_group = op.cta_group; + if (@TypeOf(op) == CopyBulkTensorTileG2SOp) return self.makeTiledTmaLoadAtom(gmem, smem, cta_map, cfg); + if (@TypeOf(op) != CopyBulkTensor2DGather4G2SOp) @compileError("unsupported TMA copy operation"); + cfg.mode = .gather4; + const indices = opts.gmem_coord_tensor orelse @panic("gather4 requires gmem_coord_tensor"); + const index_type = indices.type_().isA(cute_ir.MemRefType) orelse @panic("gather4 indices must be a tensor"); + const layout = self.emit(cute_ir.get_layout(self.ctx, indices.inner, index_type.layout(), self.loc())); + const kind: cute_nvgpu.GatherScatterTmaLoad = if (cfg.cta_group == .two) .sm_100_2sm else .sm_100; + const results = self.emitResults(2, cute_nvgpu.atom_make_non_exec_2d_gather4_tma_load(self.ctx, self.asValue(gmem).inner, layout.inner, smem.inner, cta_map.inner, self.tmaLoadAtomType(cfg), self.coordTensorType(cfg.coordinate_rank, cfg.coordinate_layout), (cute_nvgpu.GatherScatterTmaLoadAttr.get(self.ctx, .{ .value = kind }) catch unreachable).attribute(), .{ .num_multicast = .int(self.ctx, .i32, 1), .tma_format = tmaFormatAttribute(self, cfg) }, self.loc())); + return .{ .atom = .{ .inner = results[0].inner, .kernel = self }, .tensor = .{ .inner = results[1].inner, .kernel = self } }; + } + /// Build the host-visible TMA descriptor and its coordinate tensor, whose /// types are assembled from `config`. pub fn makeTiledTmaLoadAtom(self: *Builder, gmem: anytype, smem_layout: anytype, cta_map: anytype, config: TmaConfig) TmaDescriptor { + std.debug.assert(config.mode == .tiled); const results = self.emitResults(2, cute_nvgpu.atom_make_non_exec_tiled_tma_load( self.ctx, self.asValue(gmem).inner, @@ -1772,7 +2078,7 @@ pub const Builder = struct { self.asValue(cta_map).inner, self.tmaLoadAtomType(config), self.coordTensorType(config.coordinate_rank, config.coordinate_layout), - (cute_nvgpu.TiledTmaLoadAttr.get(self.ctx, .{ .value = .sm_90 }) catch @panic("invalid tiled TMA load")).attribute(), + (cute_nvgpu.TiledTmaLoadAttr.get(self.ctx, .{ .value = tmaLoadKind(config) }) catch @panic("invalid tiled TMA load")).attribute(), .{ .num_multicast = .int(self.ctx, .i32, 1), .tma_format = tmaFormatAttribute(self, config) }, self.loc(), )); @@ -1802,8 +2108,15 @@ pub const Builder = struct { /// Host-side tiled TMA load atom type described by `config`. pub fn tmaLoadAtomType(self: *Builder, config: TmaConfig) *const mlir.Type { + if (config.mode == .gather4) return (cute_nvgpu.CopyAtomNonExec2DGather4TmaLoadType.get(self.ctx, .{ + .kind = if (config.cta_group == .two) .sm_100_2sm else .sm_100, + .valType = config.dtype.toMlir(self.ctx), + .copyBits = @intCast(config.copy_bits), + .tmaGbasis = self.layoutType(config.global_basis), + .tmaFormat = if (config.format == .default) null else tmaDataFormat(config.format, config.dtype), + }) catch @panic("invalid gather TMA atom")).type_(); return (cute_nvgpu.CopyAtomNonExecTiledTmaLoadType.get(self.ctx, .{ - .kind = .sm_90, + .kind = tmaLoadKind(config), .valType = config.dtype.toMlir(self.ctx), .copyBits = @intCast(config.copy_bits), .tmaGbasis = self.layoutType(config.global_basis), @@ -1818,8 +2131,8 @@ pub const Builder = struct { return (cute_nvgpu.CopyAtomTmaLoadType.get(self.ctx, .{ .valType = config.dtype.toMlir(self.ctx), .copyBits = @intCast(config.copy_bits), - .mode = .tiled, - .numCta = 1, + .mode = if (config.mode == .gather4) .gather4 else .tiled, + .numCta = ctaCount(config.cta_group), .gStride = self.emptyStrideType(), .tmaGbasis = self.layoutType(config.global_basis), }) catch @panic("invalid TMA load atom")).type_(); @@ -1851,6 +2164,12 @@ pub const Builder = struct { return (cute_ir.StrideType.get(self.ctx, .{ .attr = self.algebraAttr(cute_ir.StrideAttr, .{ .tuple = &.{} }).attribute() }) catch @panic("invalid CuTe stride")).type_(); } + /// Generic address (i64) of the tensor map in an executable tiled TMA atom. + pub fn tmaDescriptorAddress(self: *Builder, descriptor: Atom) Value { + const ptr = self.emit(cute_nvgpu.get_tma_desc_addr(self.ctx, descriptor.inner, (cute_ir.PtrType.get(self.ctx, .{ .valueType = (cute_nvgpu.TmaDescriptorTiledType.get(self.ctx, .{}) catch unreachable).type_(), .addressSpace = .generic, .alignment = 64 }) catch unreachable).type_(), self.loc())); + return self.ptrToInt(ptr, .i64); + } + pub fn prefetchTmaDesc(self: *Builder, descriptor: Atom) void { self.emitVoid(cute_nvgpu.prefetch_tma_desc(self.ctx, descriptor.inner, self.loc())); } @@ -1861,11 +2180,11 @@ pub const Builder = struct { } /// Attach the mbarrier that an executable TMA load atom completes. - pub fn setTmaBarrier(self: *Builder, atom: Atom, barrier: Value) Atom { + pub fn setTmaBarrier(self: *Builder, atom: Atom, mbar: Value) Atom { const result = self.emit(cute_nvgpu.atom_set_value( self.ctx, atom.inner, - barrier.inner, + mbar.inner, atom.type_(), (cute_nvgpu.AtomCopyFieldTmaLoadAttr.get(self.ctx, .{ .value = .tma_bar }) catch @panic("invalid TMA load field")).attribute(), self.loc(), @@ -1915,10 +2234,10 @@ pub const Builder = struct { } fn blockScaledAtomType(self: *Builder, cfg: BlockScaledMmaConfig) *const mlir.Type { - if (cfg.n < 8 or cfg.n > 256 or cfg.n % 8 != 0) std.debug.panic("unsupported block-scaled MMA N {d}", .{cfg.n}); + cfg.validate(); return (cute_nvgpu.MmaAtomSM100UMMABlockScaledType.get(self.ctx, .{ - .shapeMnk = self.algebraAttr(cute_ir.ShapeAttr, self.expr(.{ 128, cfg.n, cfg.k() })).attribute(), - .numCta = 1, + .shapeMnk = self.algebraAttr(cute_ir.ShapeAttr, self.expr(.{ cfg.m, cfg.n, cfg.k() })).attribute(), + .numCta = ctaCount(cfg.cta_group), .aMajor = .k, .bMajor = .k, .aType = DType.f4e2m1fn.toMlir(self.ctx), @@ -1952,10 +2271,27 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - /// Commit the preceding tcgen05 MMA operations to an mbarrier. One elected - /// lane of the MMA warp issues this after each K-stage. - pub fn tcgen05Commit(self: *Builder, barrier: Value) void { - self.emitVoid(nvvm.tcgen05_commit(self.ctx, self.sharedBarrierPtr(barrier).inner, .cta_1, self.loc())); + pub const Tcgen05CommitOptions = struct { + /// Must match the group of the operations being tracked. + cta_group: CtaGroup = .one, + /// Arrive at the same barrier offset in each CTA whose rank bit is set. + /// Without a mask, arrive only on the issuing CTA's barrier. + multicast_mask: ?Value = null, + }; + + /// Track this thread's preceding tcgen05 MMA/copy operations of the + /// selected group, signaling one arrival per destination on completion. + /// The same elected thread that issues those operations must commit them. + /// For `.two`, either CTA in the pair can be the issuer. All tcgen05 + /// operations in a kernel must use the same group. + pub fn tcgen05Commit(self: *Builder, mbar: Value, opts: Tcgen05CommitOptions) void { + self.emitVoid(nvvm.tcgen05_commit( + self.ctx, + self.sharedBarrierPtr(mbar).inner, + opts.cta_group.toNvvm(), + if (opts.multicast_mask) |mask| mask.to(.i16).inner else null, + self.loc(), + )); } /// Wait until asynchronous tensor-memory loads have reached registers. @@ -1963,14 +2299,14 @@ pub const Builder = struct { self.emitVoid(nvvm.tcgen05_wait(self.ctx, .load, self.loc())); } - pub const TmemOptions = struct { two_cta: bool = false, exclusive: bool = false }; + pub const TmemOptions = struct { cta_group: CtaGroup = .one, exclusive: bool = false }; pub fn allocTmem(self: *Builder, columns: u16, holding_ptr: anytype, opts: TmemOptions) void { self.emitVoid(cute_nvgpu.arch_sm100_alloc_tmem( self.ctx, self.constant(.i32, columns).inner, self.asValue(holding_ptr).inner, - .{ .is_two_cta = if (opts.two_cta) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, + .{ .is_two_cta = if (opts.cta_group == .two) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, self.loc(), )); } @@ -1981,7 +2317,7 @@ pub const Builder = struct { } pub fn relinquishTmemAllocPermit(self: *Builder, opts: TmemOptions) void { - self.emitVoid(cute_nvgpu.arch_sm100_relinquish_tmem_alloc_permit(self.ctx, .{ .is_two_cta = if (opts.two_cta) .unit(self.ctx) else null }, self.loc())); + self.emitVoid(cute_nvgpu.arch_sm100_relinquish_tmem_alloc_permit(self.ctx, .{ .is_two_cta = if (opts.cta_group == .two) .unit(self.ctx) else null }, self.loc())); } pub fn deallocTmem(self: *Builder, pointer: anytype, columns: u16, opts: TmemOptions) void { @@ -1989,7 +2325,7 @@ pub const Builder = struct { self.ctx, self.asValue(pointer).inner, self.constant(.i32, columns).inner, - .{ .is_two_cta = if (opts.two_cta) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, + .{ .is_two_cta = if (opts.cta_group == .two) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, self.loc(), )); } @@ -2182,22 +2518,37 @@ pub const Builder = struct { _ = nvvm.barrier(self.ctx, self.loc()).appendTo(self.currentBlock()); } - /// Synchronize a fixed subset of the CTA on a named hardware barrier. - /// `participants` must be a warp multiple and every participating thread - /// must execute the same barrier id. - pub fn namedBarrier(self: *Builder, id: i32, participants: i32) void { - // The embedded CuTe compiler currently reads NVVM through a different - // MLIR revision. A generic `nvvm.barrier` with optional operands is - // accepted, but those operands are discarded and it lowers to - // `bar.sync 0`. Constant inline PTX preserves the named barrier ABI - // used by CUTLASS (`2,160` for TMEM allocation and `1,128` for the - // epilogue pipeline). - const assembly = std.fmt.allocPrint( - self.arena.allocator(), - "bar.sync {d}, {d};", - .{ id, participants }, - ) catch @panic("namedBarrier OOM"); - _ = llvm.inline_asm(self.ctx, assembly, &.{}, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + /// `bar.sync id, participants` (`barrier.cta.sync.aligned`). + /// `id` is an integer or an i32 Value in [0, 15]. `participants` must be + /// a positive warp multiple, and every participating thread must execute + /// the same barrier id. The aligned form requires CTA-uniform control flow. + pub fn barrierCtaSyncAligned(self: *Builder, id: anytype, participants: anytype) void { + if (@TypeOf(participants) == ?Value) { + if (participants) |count| return self.barrierCtaSyncAligned(id, count); + return self.barrierCtaSyncAligned(id, null); + } + if (@TypeOf(participants) != Value and @TypeOf(participants) != @TypeOf(null)) { + std.debug.assert(participants > 0 and participants <= 1024 and @mod(participants, 32) == 0); + } + if (@TypeOf(id) == Value) { + std.debug.assert(id.dtype() == .i32); + } else { + const immediate: i32 = id; + std.debug.assert(immediate >= 0 and immediate < 16); + } + // The LLVM intrinsic preserves convergence and both operands across + // the embedded compiler's NVVM dialect version boundary. + if (@TypeOf(participants) == @TypeOf(null)) { + _ = self.callIntrinsic("llvm.nvvm.barrier.cta.sync.aligned", &.{self.lift(id).to(.i32)}, null); + } else { + _ = self.callIntrinsic("llvm.nvvm.barrier.cta.sync.aligned.count", &.{ self.lift(id).to(.i32), self.lift(participants).to(.i32) }, null); + } + } + + pub const BarrierOptions = struct { barrier_id: ?Value = null, number_of_threads: ?Value = null }; + + pub fn barrier(self: *Builder, opts: BarrierOptions) void { + self.barrierCtaSyncAligned(opts.barrier_id orelse self.constant(.i32, 0), opts.number_of_threads); } /// Build the CuTe first-class divisor used by CUTLASS's persistent tile @@ -2368,48 +2719,101 @@ pub const Builder = struct { /// mbarrier operands are `!llvm.ptr<3>` for NVVM. A barrier is either a CuTe /// shared-memory pointer or a raw `i32` shared-memory address. - fn sharedBarrierPtr(self: *Builder, barrier: Value) Value { - if (barrier.isInt()) return self.addressPtr(barrier, .shared); - return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{barrier.inner}, &.{llvm.pointerType(self.ctx, NVPTXAddressSpace.shared)}, self.loc())); + fn sharedBarrierPtr(self: *Builder, mbar: Value) Value { + if (mbar.isInt()) return self.addressPtr(mbar, .shared); + return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{mbar.inner}, &.{llvm.pointerType(self.ctx, NVPTXAddressSpace.shared)}, self.loc())); } /// Initialize a shared-memory mbarrier expecting `arrivals` per phase. - pub fn mbarrierInit(self: *Builder, barrier: Value, arrivals: u32) void { - self.emitVoid(nvvm.mbarrier_init(self.ctx, self.sharedBarrierPtr(barrier).inner, self.constant(.i32, arrivals).inner, self.loc())); + pub fn mbarrierInit(self: *Builder, mbar: Value, arrivals: u32) void { + self.emitVoid(nvvm.mbarrier_init(self.ctx, self.sharedBarrierPtr(mbar).inner, self.constant(.i32, arrivals).inner, self.loc())); } /// Arrive and expect `bytes` of asynchronous transactions (SM90+), as a /// TMA producer does before issuing its copies. - pub fn mbarrierArriveExpectTx(self: *Builder, barrier: Value, bytes: u32) void { - self.emitVoid(nvvm.mbarrier_arrive_expect_tx(self.ctx, self.sharedBarrierPtr(barrier).inner, self.constant(.i32, bytes).inner, .cta, self.loc())); + pub fn mbarrierArriveExpectTx(self: *Builder, mbar: Value, bytes: u32) void { + self.emitVoid(nvvm.mbarrier_arrive_expect_tx(self.ctx, self.sharedBarrierPtr(mbar).inner, self.constant(.i32, bytes).inner, .cta, self.loc())); } - pub fn mbarrierArrive(self: *Builder, barrier: Value, count: u32) void { - self.emitVoid(nvvm.mbarrier_arrive(self.ctx, self.sharedBarrierPtr(barrier).inner, self.constant(.i32, count).inner, .cta, self.loc())); + pub const MBarrierArriveOptions = struct { + peer_cta_rank_in_cluster: ?Value = null, + arrive_count: ?Value = null, + scope: MBarrierScope = .cta, + relaxed: bool = false, + }; + + /// Arrive locally or at the same offset in a peer CTA. Local arrivals + /// return the opaque state token; remote arrivals have no state result. + pub fn mbarrierArrive(self: *Builder, mbar: Value, opts: MBarrierArriveOptions) ?Value { + var ptr = self.sharedBarrierPtr(mbar); + if (opts.peer_cta_rank_in_cluster) |rank| { + ptr = self.emit(nvvm.mapa(self.ctx, ptr.inner, rank.inner, llvm.pointerType(self.ctx, NVPTXAddressSpace.shared_cluster), self.loc())); + } + const op = nvvm.ops.mbarrier_arrive(self.ctx, ptr.inner, .{ + .count = if (opts.arrive_count) |count| count.to(.i32).inner else null, + .scope = (nvvm.ops.MemScopeKindAttr.get(self.ctx, .{ .value = switch (opts.scope) { + .cta => .cta, + .cluster => .cluster, + } }) catch unreachable).attribute(), + .relaxed = .boolean(self.ctx, opts.relaxed), + .res_type = if (opts.peer_cta_rank_in_cluster == null) mlir.Type.int(self.ctx, .i64) else null, + }, self.loc()); + if (opts.peer_cta_rank_in_cluster == null) return self.emit(op); + self.emitVoid(op); + return null; } - /// Test whether phase `phase` has completed, without suspending (SM90+). - pub fn mbarrierWaitParity(self: *Builder, barrier: Value, phase: Value) Value { - return self.emit(nvvm.mbarrier_wait_parity(self.ctx, self.sharedBarrierPtr(barrier).inner, phase.inner, .@"try", .cta, self.loc())); + pub const MBarrierTryWaitOptions = struct { + scope: MBarrierScope = .cta, + time_limit: ?Value = null, + }; + + /// Try an acquire wait for phase `phase` (i32 parity, 0 or 1). May suspend + /// briefly and return false before completion. `scope` selects which + /// threads this wait can synchronize with; the barrier remains local. + pub fn mbarrierTryWait(self: *Builder, mbar: Value, phase: Value, opts: MBarrierTryWaitOptions) Value { + const ptr = self.sharedBarrierPtr(mbar); + if (opts.time_limit) |ticks| { + return self.emit(nvvm.mbarrier_try_wait_parity_timelimit(self.ctx, ptr.inner, phase.inner, ticks.to(.i32).inner, opts.scope, self.loc())); + } + return self.emit(nvvm.mbarrier_wait_parity(self.ctx, ptr.inner, phase.inner, .@"try", opts.scope, self.loc())); } - /// Wait for phase `phase`, letting the warp sleep for up to - /// `suspend_time` cycles per attempt (SM90+). - pub fn mbarrierTryWaitParity(self: *Builder, barrier: Value, phase: Value, suspend_time: u32) void { - self.emitVoid(nvvm.mbarrier_try_wait_parity(self.ctx, self.sharedBarrierPtr(barrier).inner, phase.inner, self.constant(.i32, suspend_time).inner, self.loc())); + pub const MBarrierWaitOptions = struct { + scope: MBarrierScope = .cta, + /// Hardware suspend hint in nanoseconds per attempt, not a total timeout. + time_limit: u32 = 10_000_000, + /// Probe before suspending, as in CuTe's pipeline consumer wait. + test_first: bool = true, + }; + + fn mbarrierWaitLoop(self: *Builder, mbar: Value, phase: Value, opts: MBarrierWaitOptions) void { + const ptr = self.sharedBarrierPtr(mbar); + const ticks = self.constant(.i32, opts.time_limit); + switch (opts.scope) { + // Preserve the operation recognized by CuTe's pipeline compiler. + .cta => self.emitVoid(nvvm.mbarrier_try_wait_parity(self.ctx, ptr.inner, phase.inner, ticks.inner, self.loc())), + .cluster => { + var pending = self.openWhile(.{}, .{}); + const ready = self.mbarrierTryWait(mbar, phase, .{ .scope = opts.scope, .time_limit = ticks }); + pending.yieldBefore(ready.eq(false), .{}); + pending.yieldAfter(.{}); + }, + } } - /// Wait for a pipeline phase with the two-step sequence of CuTe's - /// `PipelineTmaUmma`: test the barrier without suspending, then suspend - /// only when the phase is still outstanding. The initial test matters when + /// Acquire a pipeline phase at `scope`: test without suspending, then + /// block only when the phase is still outstanding. The initial test matters when /// a producer reaches a full ring and must yield to the consumer that /// recycles its next stage. - pub fn mbarrierWait(self: *Builder, barrier: Value, phase: Value) void { - const ready = self.mbarrierWaitParity(barrier, phase); - var pending = self.openIf(ready.eq(false)); - { - self.mbarrierTryWaitParity(barrier, phase, 10_000_000); + pub fn mbarrierWait(self: *Builder, mbar: Value, phase: Value, opts: MBarrierWaitOptions) void { + if (opts.test_first) { + const ready = self.mbarrierTestWait(mbar, phase, .{ .scope = opts.scope }); + var pending = self.openIf(ready.eq(false)); + self.mbarrierWaitLoop(mbar, phase, opts); pending.close(); + } else { + self.mbarrierWaitLoop(mbar, phase, opts); } } @@ -2438,7 +2842,7 @@ pub const Builder = struct { // Hand-scheduled kernels address shared memory with `i32` offsets in the // shared window, global memory with `i64` addresses and tensor memory with // `i32` TMEM addresses (lane << 16 | column). These helpers emit the NVVM / - // LLVM operations for them directly (no inline PTX). + // LLVM operations for them directly, using inline PTX where needed. pub const NVPTXAddressSpace = llvm.NVPTXAddressSpace; @@ -2500,11 +2904,6 @@ pub const Builder = struct { self.emitVoid(llvm.store(self.ctx, word.to(.i16).inner, self.addressPtr(address, space).inner, .{ .alignment = 2 }, self.loc())); } - /// `bar.sync id, threads`: named barrier over `threads` threads. - pub fn barrierSync(self: *Builder, id: i32, threads: i32) void { - _ = self.callIntrinsic("llvm.nvvm.barrier.cta.sync.aligned.count", &.{ self.constant(.i32, id), self.constant(.i32, threads) }, null); - } - /// `bar.red.or.pred`: named barrier over `threads` threads that also returns whether /// `predicate` (i1) is true in any of them. pub fn barrierReduceOr(self: *Builder, id: i32, threads: i32, predicate: Value) Value { @@ -2516,9 +2915,10 @@ pub const Builder = struct { self.emitVoid(nvvm.bar_warp_sync(self.ctx, self.constant(.i32, -1).inner, self.loc())); } - /// Non-blocking test of mbarrier phase `parity` (i1). - pub fn mbarrierTestParity(self: *Builder, barrier: Value, parity: Value) Value { - return self.emit(nvvm.mbarrier_wait_parity(self.ctx, self.sharedBarrierPtr(barrier).inner, parity.inner, .@"test", .cta, self.loc())); + /// Non-blocking acquire test of mbarrier phase `parity` (i32, 0 or 1). + pub fn mbarrierTestWait(self: *Builder, mbar: Value, parity: Value, opts: MBarrierTryWaitOptions) Value { + std.debug.assert(opts.time_limit == null); + return self.emit(nvvm.mbarrier_wait_parity(self.ctx, self.sharedBarrierPtr(mbar).inner, parity.inner, .@"test", opts.scope, self.loc())); } /// Call an LLVM intrinsic by name (e.g. an `llvm.nvvm.*` intrinsic the NVVM dialect of @@ -2549,8 +2949,8 @@ pub const Builder = struct { /// Arrive on `barrier` once this thread's prior cp.async copies have landed; with /// `noinc`, the arrival is counted in the barrier's expected count. - pub fn cpAsyncMbarrierArrive(self: *Builder, barrier: Value, comptime noinc: bool) void { - _ = self.callIntrinsic(if (noinc) "llvm.nvvm.cp.async.mbarrier.arrive.noinc.shared" else "llvm.nvvm.cp.async.mbarrier.arrive.shared", &.{self.sharedBarrierPtr(barrier)}, null); + pub fn cpAsyncMbarrierArrive(self: *Builder, mbar: Value, comptime noinc: bool) void { + _ = self.callIntrinsic(if (noinc) "llvm.nvvm.cp.async.mbarrier.arrive.noinc.shared" else "llvm.nvvm.cp.async.mbarrier.arrive.shared", &.{self.sharedBarrierPtr(mbar)}, null); } /// Wait until all of this thread's cp.async copies have landed. @@ -2705,13 +3105,271 @@ pub const Builder = struct { return self.gridDim().z.mul(value); } + /// SM100 `tile::gather4` TMA load from a tiled 2-D descriptor address `desc` (i64). + /// Gather four rows `r` (i32) at column `col` into 128-byte-aligned shared `dst` (i32), + /// completing their bytes on an 8-byte-aligned local mbarrier `bar` (i32). + /// The descriptor must have row box extent 1 and no interleaving. `col` and `r` + /// are element coordinates; row length and out-of-bounds fill come from the + /// descriptor (zero or OOB-NaN). Arm the barrier for all four rows' bytes before waiting. + pub fn tmaGather4(self: *Builder, desc: Value, dst: Value, bar: Value, col: Value, r: [4]Value, policy: CacheHintSm100) void { + std.debug.assert(desc.dtype() == .i64 and dst.dtype() == .i32 and bar.dtype() == .i32 and col.dtype() == .i32); + for (r) |row| std.debug.assert(row.dtype() == .i32); + _ = llvm.inline_asm(self.ctx, "cp.async.bulk.tensor.2d.shared::cta.global.tile::gather4.mbarrier::complete_tx::bytes.cta_group::1.L2::cache_hint [$0], [$1, {$2, $3, $4, $5, $6}], [$7], $8;", &.{ dst.inner, desc.inner, col.inner, r[0].inner, r[1].inner, r[2].inner, r[3].inner, bar.inner, self.constant(.i64, policy).inner }, .{ .constraints = "r,l,r,r,r,r,r,r,l,~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + pub const BulkPrefetchOptions = struct { l2_cache_hint: ?Value = null }; + + /// Best-effort byte-range prefetch into L2; size must be a multiple of 16. + pub fn cpAsyncBulkPrefetch(self: *Builder, addr: Value, bytes: anytype, opts: BulkPrefetchOptions) void { + if (@TypeOf(bytes) != Value) std.debug.assert(bytes > 0 and bytes % 16 == 0); + const address = if (addr.isInt()) self.addressPtr(addr, .global) else self.llvmPointer(addr, .global); + self.emitVoid(nvvm.ops.cp_async_bulk_prefetch(self.ctx, address.inner, self.lift(bytes).to(.i32).inner, .{ + .l2CacheHint = if (opts.l2_cache_hint) |hint| hint.to(.i64).inner else null, + }, self.loc())); + } + + pub fn prefetchL1(self: *Builder, addr: Value) void { + const address = if (addr.isInt()) addr else self.ptrToInt(addr, .i64); + _ = llvm.inline_asm(self.ctx, "prefetch.global.L1 [$0];", &.{address.inner}, .{ .constraints = "l", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + pub fn prefetchTensorMap(self: *Builder, desc: Value) void { + const address = if (desc.isInt()) desc else self.ptrToInt(desc, .i64); + _ = llvm.inline_asm(self.ctx, "prefetch.tensormap [$0];", &.{address.inner}, .{ .constraints = "l", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + pub const OperandSource = union(enum) { smem: Value, tmem: Value }; + pub const Tcgen05MmaWsOptions = struct { + collector_b_buffer: ?nvvm.ops.Tcgen05MMACollectorBBuffer = null, + collector_op: ?nvvm.ops.Tcgen05MMACollectorOp = null, + col_b_zero_mask: ?Value = null, + }; + + /// Single-thread, asynchronous weight-stationary MMA, as in Python primitives. + /// The caller elects the issuer and commits/waits before consuming D. + pub fn tcgen05MmaWs(self: *Builder, kind: nvvm.Tcgen05MmaKind, d: Value, a: OperandSource, bdesc: Value, idesc: anytype, enable_input_d: Value, opts: Tcgen05MmaWsOptions) void { + std.debug.assert(kind == .f16 or kind == .tf32 or kind == .f8f6f4 or kind == .i8); + const adesc = switch (a) { + .smem => |desc| self.smemDescriptorToInt(desc), + .tmem => |ptr| if (ptr.isInt()) self.addressPtr(ptr, .tensor) else self.llvmPointer(ptr, .tensor), + }; + const dest = if (d.isInt()) self.addressPtr(d, .tensor) else self.llvmPointer(d, .tensor); + self.emitVoid(nvvm.ops.tcgen05_mma_ws(self.ctx, dest.inner, adesc.inner, self.smemDescriptorToInt(bdesc).inner, self.lift(idesc).to(.i32).inner, enable_input_d.to(.i1).inner, (nvvm.ops.Tcgen05MMAKindAttr.get(self.ctx, .{ .value = kind }) catch unreachable).attribute(), .{ + .collectorBBuffer = if (opts.collector_b_buffer) |v| (nvvm.ops.Tcgen05MMACollectorBBufferAttr.get(self.ctx, .{ .value = v }) catch unreachable).attribute() else null, + .collectorOp = if (opts.collector_op) |v| (nvvm.ops.Tcgen05MMACollectorOpAttr.get(self.ctx, .{ .value = v }) catch unreachable).attribute() else null, + .zeroColMask = if (opts.col_b_zero_mask) |v| v.to(.i64).inner else null, + }, self.loc())); + } + + pub fn makeUmmaSmemDesc(self: *Builder, src: Value, layout: LayoutSpec, major: OperandMajorMode, next_src: ?Value) Value { + return self.emit(cute_nvgpu.make_umma_smem_desc(self.ctx, src.inner, self.smemDescType(), self.layoutAttr(layout).attribute(), (cute_nvgpu.MajorModeAttr.get(self.ctx, .{ .value = switch (major) { + .k => .k, + .mn => .mn, + } }) catch unreachable).attribute(), .{ .nextSrc = if (next_src) |v| v.inner else null }, self.loc())); + } + + pub fn intToSmemDescriptor(self: *Builder, value: anytype) Value { + return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{self.lift(value).to(.i64).inner}, &.{self.smemDescType()}, self.loc())); + } + + pub fn smemDescriptorToInt(self: *Builder, desc: Value) Value { + if (desc.isInt()) return desc.to(.i64); + return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{desc.inner}, &.{mlir.Type.int(self.ctx, .i64)}, self.loc())); + } + + pub fn setMaxRegisterIncrease(self: *Builder, comptime count: u32) void { + self.setMaxRegister(count, .increase); + } + + pub fn setMaxRegisterDecrease(self: *Builder, comptime count: u32) void { + self.setMaxRegister(count, .decrease); + } + + fn setMaxRegister(self: *Builder, comptime count: u32, comptime action: enum { increase, decrease }) void { + comptime std.debug.assert(count >= 24 and count <= 256 and count % 8 == 0); + const text = std.fmt.comptimePrint("setmaxnreg.{s}.sync.aligned.u32 {d};", .{ if (action == .increase) "inc" else "dec", count }); + _ = llvm.inline_asm(self.ctx, text, &.{}, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + /// Specialized busy wait for hand-scheduled kernels; never hardware-suspends. + pub fn mbarrierSpin(self: *Builder, bar: Value, parity: Value, scope: MBarrierScope) void { + std.debug.assert(bar.dtype() == .i32 and parity.dtype() == .i32); + _ = llvm.inline_asm(self.ctx, switch (scope) { + .cta => "{ .reg .pred p; spin${:uid}: mbarrier.test_wait.parity.acquire.cta.shared::cta.b64 p, [$0], $1; @!p bra spin${:uid}; }", + .cluster => "{ .reg .pred p; spin${:uid}: mbarrier.test_wait.parity.acquire.cluster.shared::cta.b64 p, [$0], $1; @!p bra spin${:uid}; }", + }, &.{ bar.inner, parity.inner }, .{ .constraints = "r,r,~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + /// Convert a CuTe or LLVM pointer to the LLVM pointer required by NVVM. + fn llvmPointer(self: *Builder, ptr: Value, comptime space: NVPTXAddressSpace) Value { + if (!ptr.isPtr()) return ptr; + return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{ptr.inner}, &.{llvm.pointerType(self.ctx, space)}, self.loc())); + } + + /// Map a local shared pointer/address to a typed pointer in peer CTA memory. + pub fn mapDsmemPtr(self: *Builder, smem: Value, peer_cta_rank: Value) Value { + const ptr = if (smem.isInt()) self.addressPtr(smem, .shared) else self.llvmPointer(smem, .shared); + const remote = self.emit(nvvm.mapa(self.ctx, ptr.inner, peer_cta_rank.to(.i32).inner, llvm.pointerType(self.ctx, NVPTXAddressSpace.shared_cluster), self.loc())); + if (smem.type_().isA(cute_ir.PtrType)) |ty| { + std.debug.assert(ty.addressSpace() == .smem); + const result_type = (cute_ir.PtrType.get(self.ctx, .{ + .valueType = ty.valueType(), + .addressSpace = .dsmem, + .alignment = ty.alignment(), + .swizzle = ty.swizzle(), + .bitlayout = ty.bitlayout(), + }) catch @panic("invalid DSMEM pointer")).type_(); + return self.emit(cute_ir.inttoptr(self.ctx, self.ptrToInt(remote, .i32).inner, result_type, self.loc())); + } + return remote; + } + + /// Store one, two, or four i32 words in a peer CTA, completing bytes on its + /// mbarrier. Both inputs name local shared memory; both are mapped to the peer. + pub fn storeAsyncDsmem(self: *Builder, smem: Value, value: anytype, mbar: Value, peer_cta_rank: Value) void { + const n = if (@TypeOf(value) == Value) 1 else common.arity(@TypeOf(value), "storeAsyncDsmem: value"); + comptime std.debug.assert(n == 1 or n == 2 or n == 4); + if (smem.type_().isA(cute_ir.PtrType)) |ty| { + std.debug.assert(ty.addressSpace() == .smem and ty.alignment() % (n * 4) == 0); + } + if (mbar.type_().isA(cute_ir.PtrType)) |ty| { + std.debug.assert(ty.addressSpace() == .smem and ty.alignment() % 8 == 0); + } + const packed_value = if (@TypeOf(value) == Value) value else blk: { + var words: [n]Value = undefined; + inline for (0..n) |i| words[i] = self.lift(value[i]).to(.i32); + break :blk self.joinWords(n, words); + }; + self.emitVoid(nvvm.ops.store_async(self.ctx, self.llvmPointer(self.mapDsmemPtr(smem, peer_cta_rank), .shared_cluster).inner, packed_value.inner, self.llvmPointer(self.mapDsmemPtr(mbar, peer_cta_rank), .shared_cluster).inner, self.loc())); + } + + /// Explicit aligned variants retain the convergence requirement of raw PTX. + pub fn barrierClusterArriveRelaxedAligned(self: *Builder) void { + _ = llvm.inline_asm(self.ctx, "barrier.cluster.arrive.relaxed.aligned;", &.{}, .{ .constraints = "~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + pub fn barrierClusterWaitAligned(self: *Builder) void { + self.emitVoid(nvvm.cluster_wait(self.ctx, true, self.loc())); + } + + pub const ClusterArriveOptions = struct { aligned: bool = false }; + + pub fn clusterArriveRelaxed(self: *Builder, opts: ClusterArriveOptions) void { + if (opts.aligned) return self.barrierClusterArriveRelaxedAligned(); + _ = llvm.inline_asm(self.ctx, "barrier.cluster.arrive.relaxed;", &.{}, .{ .constraints = "~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + pub fn clusterWait(self: *Builder) void { + self.emitVoid(nvvm.cluster_wait(self.ctx, false, self.loc())); + } + + /// A null mask selects CuTe's FULL_MASK; integers and SSA masks also work. + pub fn voteAnySync(self: *Builder, pred: Value, mask: anytype) Value { + const active = if (@TypeOf(mask) == @TypeOf(null)) self.constant(.i32, -1) else self.lift(mask).to(.i32); + return self.callIntrinsic("llvm.nvvm.vote.any.sync", &.{ active, pred }, mlir.Type.int(self.ctx, .i1)).?; + } + + /// A null mask selects CuTe's FULL_MASK. + pub fn voteBallotSync(self: *Builder, pred: Value, mask: anytype) Value { + const active = if (@TypeOf(mask) == @TypeOf(null)) self.constant(.i32, -1) else self.lift(mask).to(.i32); + return self.callIntrinsic("llvm.nvvm.vote.ballot.sync", &.{ active, pred }, mlir.Type.int(self.ctx, .i32)).?; + } + + pub const FmaxOptions = struct { abs: bool = false, nan: bool = false, ftz: bool = false }; + + pub fn fmax(self: *Builder, x: Value, y: Value, opts: FmaxOptions) Value { + if (!opts.abs and !opts.nan and !opts.ftz) return self.callIntrinsic("llvm.nvvm.fmax.f", &.{ x, y }, mlir.Type.float(self.ctx, .f32)).?; + return self.emit(nvvm.ops.fmax(self.ctx, x.inner, y.inner, mlir.Type.float(self.ctx, .f32), .{ + .abs = if (opts.abs) .unit(self.ctx) else null, + .nan = if (opts.nan) .unit(self.ctx) else null, + .ftz = if (opts.ftz) .unit(self.ctx) else null, + }, self.loc())); + } + + fn vec2(self: *Builder, x: [2]Value) Value { + const ty = mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)); + return self.emit(vector.from_elements(self.ctx, &.{ x[0].inner, x[1].inner }, ty, self.loc())); + } + + fn unvec2(self: *Builder, v: Value) [2]Value { + return .{ self.vectorExtract(v, 0, .f32), self.vectorExtract(v, 1, .f32) }; + } + + pub const PackedF32x2Options = struct { rnd: nvvm.ops.FPRoundingMode = .rn, ftz: bool = false }; + + fn packedF32x2Attrs(self: *Builder, opts: PackedF32x2Options) struct { rnd: ?*const mlir.Attribute, ftz: ?*const mlir.Attribute } { + return .{ + .rnd = (nvvm.ops.FPRoundingModeAttr.get(self.ctx, .{ .value = opts.rnd }) catch unreachable).attribute(), + .ftz = if (opts.ftz) .unit(self.ctx) else null, + }; + } + + pub fn fmaPackedF32x2(self: *Builder, a: [2]Value, b: [2]Value, c: [2]Value, opts: PackedF32x2Options) [2]Value { + return self.unvec2(self.emit(nvvm.ops.fma_packed_f32x2(self.ctx, self.vec2(a).inner, self.vec2(b).inner, self.vec2(c).inner, mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)), .{ .rnd = self.packedF32x2Attrs(opts).rnd, .ftz = self.packedF32x2Attrs(opts).ftz }, self.loc()))); + } + + pub fn addPackedF32x2(self: *Builder, a: [2]Value, b: [2]Value, opts: PackedF32x2Options) [2]Value { + return self.unvec2(self.emit(nvvm.ops.add_packed_f32x2(self.ctx, self.vec2(a).inner, self.vec2(b).inner, mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)), .{ .rnd = self.packedF32x2Attrs(opts).rnd, .ftz = self.packedF32x2Attrs(opts).ftz }, self.loc()))); + } + + pub fn mulPackedF32x2(self: *Builder, a: [2]Value, b: [2]Value, opts: PackedF32x2Options) [2]Value { + return self.unvec2(self.emit(nvvm.ops.mul_packed_f32x2(self.ctx, self.vec2(a).inner, self.vec2(b).inner, mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)), .{ .rnd = self.packedF32x2Attrs(opts).rnd, .ftz = self.packedF32x2Attrs(opts).ftz }, self.loc()))); + } + + pub const StoreOptions = struct { + /// Address space for raw integer addresses; typed pointers retain theirs. + space: NVPTXAddressSpace = .global, + level1_eviction_priority: nvvm.ops.EvictKind = .evict_normal, + }; + + /// Scalar/vector store. Eight i32 words with no_allocate keep STG.E.256. + pub fn store(self: *Builder, addr: Value, value: anytype, comptime opts: StoreOptions) void { + if (@TypeOf(value) != Value) { + const n = common.arity(@TypeOf(value), "store: value"); + comptime std.debug.assert(n == 1 or n == 2 or n == 4 or n == 8); + var words: [n]Value = undefined; + inline for (0..n) |i| words[i] = self.lift(value[i]).to(.i32); + const is_global = if (addr.type_().isA(cute_ir.PtrType)) |ty| ty.addressSpace() == .gmem else addr.isInt() and opts.space == .global; + if (n == 8 and is_global and opts.level1_eviction_priority == .evict_no_allocate) { + const address = if (addr.isInt()) addr else self.ptrToInt(addr, .i64); + var args: [9]*const mlir.Value = undefined; + args[0] = address.inner; + for (words, 0..) |word, i| args[i + 1] = word.inner; + _ = llvm.inline_asm(self.ctx, "st.global.L1::no_allocate.v8.b32 [$0], {$1, $2, $3, $4, $5, $6, $7, $8};", &args, .{ .constraints = "l,r,r,r,r,r,r,r,r,~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + return; + } + self.store(addr, self.joinWords(n, words), opts); + return; + } + const ptr = if (addr.isInt()) self.addressPtr(addr, opts.space) else if (addr.type_().isA(cute_ir.PtrType)) |ty| blk: { + const space: NVPTXAddressSpace = @fromBackingInt(@intCast(@backingInt(ty.addressSpace()))); + break :blk self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{addr.inner}, &.{llvm.pointerType(self.ctx, space)}, self.loc())); + } else addr; + self.emitVoid(nvvm.ops.store_ext(self.ctx, value.inner, ptr.inner, .{ + .evict = if (opts.level1_eviction_priority == .evict_normal) null else (nvvm.ops.EvictKindAttr.get(self.ctx, .{ .value = opts.level1_eviction_priority }) catch unreachable).attribute(), + }, self.loc())); + } + + pub fn fma(self: *Builder, x: Value, y: Value, z: Value) Value { + return self.callIntrinsic("llvm.fma.f32", &.{ x, y, z }, mlir.Type.float(self.ctx, .f32)).?; + } + + pub fn clock(self: *Builder) Value { + return self.emit(llvm.inline_asm(self.ctx, "mov.u32 $0, %clock;", &.{}, .{ .result_type = mlir.Type.int(self.ctx, .i32), .constraints = "=r", .has_side_effects = true }, self.loc())); + } + + pub fn globalTimer(self: *Builder) Value { + return self.emit(llvm.inline_asm(self.ctx, "mov.u64 $0, %globaltimer;", &.{}, .{ .result_type = mlir.Type.int(self.ctx, .i64), .constraints = "=l", .has_side_effects = true }, self.loc())); + } + // ==================== scalars ==================== - /// `value` as a constant of `dt`; a Value passes through. An integer is - /// signed or an unsigned bit pattern of `dt`'s width; anything else - /// panics. + /// `value` as a constant of `dt`; a Value passes through and enums use + /// their backing integers. An integer is signed or an unsigned bit + /// pattern of `dt`'s width; anything else panics. pub fn constant(self: *Builder, dt: DType, value: anytype) Value { if (@TypeOf(value) == Value) return value; + if (@typeInfo(@TypeOf(value)) == .@"enum") return self.constant(dt, @backingInt(value)); const ctx = self.ctx; const T = @TypeOf(value); if (isFloatDtype(dt)) { @@ -2957,7 +3615,7 @@ test "guard, shared memory, sync, layouts, for loop" { const i = b.blockIdx().x.mul(128).add(tid); smem.set(.{tid}, gx.get(.{i})); b.syncThreads(); - b.namedBarrier(2, 128); + b.barrierCtaSyncAligned(2, 128); b.launchDependents(); b.waitForDependency(); const j = b.constant(.i32, 127).sub(tid); @@ -2976,7 +3634,9 @@ test "guard, shared memory, sync, layouts, for loop" { try expectContains(kernel, "cute_nvgpu.arch.alloc_smem"); try expectContains(kernel, "!cute.ptr>"); try expectContains(kernel, "\"nvvm.barrier\"() <{operandSegmentSizes = array}> : () -> ()"); - try expectContains(kernel, "bar.sync 2, 128;"); + try expectContains(kernel, "llvm.nvvm.barrier.cta.sync.aligned.count"); + try expectContains(kernel, "value = 2 : i32"); + try expectContains(kernel, "value = 128 : i32"); try expectContains(kernel, "griddepcontrol.launch_dependents;"); try expectContains(kernel, "griddepcontrol.wait;"); try expectContains(kernel, "scf.if"); @@ -3113,30 +3773,433 @@ test "SM100 block-scaled MMA atoms use the CuTe DSL instruction shapes" { var b = try Builder.open(std.testing.allocator, ctx, "sm100_blockscaled_atoms"); defer b.deinit(); - const descriptor = try cute_nvgpu.CopyAtomNonExecTiledTmaLoadType.get(ctx, .{ - .kind = .sm_90, - .valType = DType.f8e4m3fn.toMlir(ctx), - .copyBits = 16384, - .tmaGbasis = b.layoutType(b.layoutSpec(.{ 128, 16, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) })), - .tmaFormat = .U8, + const tma: TmaConfig = .{ + .dtype = .f8e4m3fn, + .copy_bits = 16384, + .global_basis = b.layoutSpec(.{ 128, 16, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }), + .coordinate_layout = b.layoutSpec(.{ 16, 128, 8 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}), b.basis(1, 1, .{2}) }), + .coordinate_rank = 3, + }; + var pair_tma = tma; + pair_tma.cta_group = .two; + const args = try b.declareArgs(.{ + .single = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(tma), .grid_constant = true } }, + .pair = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(pair_tma), .grid_constant = true } }, }); - _ = try b.declareArgs(.{ .descriptor = .{ .mlir_type = .{ .type_ = descriptor.type_(), .grid_constant = true } } }); + _ = b.makeExecTma(.{ .inner = args.single.inner, .kernel = &b }, b.tmaLoadExecAtomType(tma)); + _ = b.makeExecTma(.{ .inner = args.pair.inner, .kernel = &b }, b.tmaLoadExecAtomType(pair_tma)); const sf_ptr = b.intToPtr(0, .f8e8m0fnu, .tmem, 1); - _ = b.makeBlockScaledMma(.{ .n = 16 }, sf_ptr); - _ = b.makeBlockScaledMma(.{ .n = 128 }, sf_ptr); + for ([_]CtaGroup{ .one, .two }) |group| { + _ = b.makeBlockScaledMma(.{ .n = 16, .cta_group = group }, sf_ptr); + _ = b.makeBlockScaledMma(.{ .n = 128, .cta_group = group }, sf_ptr); + } + _ = b.makeBlockScaledMma(.{ .m = 256, .n = 16, .cta_group = .two }, sf_ptr); + _ = b.makeBlockScaledMma(.{ .m = 256, .n = 128, .cta_group = .two }, sf_ptr); const nvfp4_sf_ptr = b.intToPtr(0, .f8e4m3fn, .tmem, 1); _ = b.makeBlockScaledMma(.{ .kind = .nvfp4_nvfp4, .n = 16 }, nvfp4_sf_ptr); const ir = try b.finish(.{ 192, 1, 1 }); defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "sm100.mma_bs<128x16x32") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "sm100.mma_bs<128x128x32") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "sm100.mma_bs<128x16x64, num_cta = 1, ab_major = (k, k), elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E4M3FN, frag_kind = ss, vec_size = 16>") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "cute.make_tiled_mma") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "cute_nvgpu.grid_constant") != null); + try expectContains(ir, "sm100.mma_bs<128x16x32, num_cta = 1"); + try expectContains(ir, "sm100.mma_bs<128x128x32, num_cta = 1"); + try expectContains(ir, "sm100.mma_bs<128x16x32, num_cta = 2"); + try expectContains(ir, "sm100.mma_bs<128x128x32, num_cta = 2"); + try expectContains(ir, "sm100.mma_bs<256x16x32, num_cta = 2"); + try expectContains(ir, "sm100.mma_bs<256x128x32, num_cta = 2"); + try expectContains(ir, "sm100.mma_bs<128x16x64, num_cta = 1, ab_major = (k, k), elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E4M3FN, frag_kind = ss, vec_size = 16>"); + try expectContains(ir, "non_exec_tiled_tma_load"); + try expectContains(ir, "value = 3 : i32"); + try expectContains(ir, "value = 32 : i32"); + try expectContains(ir, "!llvm.ptr<7>"); + try expectContains(ir, "#nvvm.mem_scope"); + try expectContains(ir, "#nvvm.mem_scope"); + + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "mbarrier waits preserve acquire scope through polling and blocking retries" { + const ctx = try testContext(); + defer ctx.deinit(); + for ([_]MBarrierScope{ .cta, .cluster }) |scope| { + var b = try Builder.open(std.testing.allocator, ctx, "barrier_waits"); + defer b.deinit(); + _ = try b.declareArgs(.{}); + + const barrier = b.allocSmem(.i64, 1, 8); + const phase = b.constant(.i32, 0); + _ = b.mbarrierTestWait(barrier, phase, .{ .scope = scope }); + _ = b.mbarrierTryWait(barrier, phase, .{ .scope = scope }); + b.mbarrierWait(barrier, phase, .{ .scope = scope, .time_limit = 1234, .test_first = false }); + b.mbarrierWait(barrier, phase, .{ .scope = scope }); + + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "#nvvm.mbar_wait")); + try std.testing.expectEqual(@as(usize, 1), std.mem.count(u8, ir, "#nvvm.mbar_wait")); + try expectContains(ir, "value = 1234 : i32"); + try expectContains(ir, "\"scf.if\""); + switch (scope) { + .cta => { + try std.testing.expectEqual(@as(usize, 3), std.mem.count(u8, ir, "#nvvm.mbar_scope")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"nvvm.mbarrier.try_wait.parity\"")); + try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, ir, "\"scf.while\"")); + }, + .cluster => { + try std.testing.expectEqual(@as(usize, 5), std.mem.count(u8, ir, "#nvvm.mbar_scope")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "#nvvm.mem_order")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"nvvm.mbarrier.try_wait.parity.timelimit\"")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"scf.while\"")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"scf.condition\"")); + try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, ir, "#nvvm.mbar_scope")); + }, + } + + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); + } +} + +test "SM100 raw primitives preserve operand types and round-trip through MLIR" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "sm100_raw_primitives"); + defer b.deinit(); + const cfg: TmaConfig = .{ + .dtype = .f8e4m3fn, + .copy_bits = 1024, + .global_basis = b.layoutSpec(.{ 128, 1 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_layout = b.layoutSpec(.{ 1024, 16 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_rank = 2, + }; + const a = try b.declareArgs(.{ + .gmem = .{ .ptr = DType.i32 }, + .tma = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(cfg), .grid_constant = true } }, + }); + const atom = b.makeExecTma(.{ .inner = a.tma.inner, .kernel = &b }, b.tmaLoadExecAtomType(cfg)); + const desc = b.tmaDescriptorAddress(atom); + const dst = b.constant(.i32, 1024); + const bar = b.constant(.i32, 0); + const zero = b.constant(.i32, 0); + const gmem = b.ptrToInt(a.gmem, .i64); + const pred = b.threadIdx().x.eq(0); + b.tmaGather4(desc, dst, bar, zero, .{ zero, zero.add(1), zero.add(2), zero.add(3) }, .evict_last); + b.cpAsyncBulkPrefetch(gmem, b.constant(.i32, 128), .{}); + b.prefetchL1(gmem); + b.prefetchTensorMap(desc); + const swizzled = Tcgen05SmemDesc.build(&b, dst, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .layout = .swizzle_128b }); + const interleaved = Tcgen05SmemDesc.build(&b, dst, .{ .leading_byte_offset = 1024, .stride_byte_offset = 128 }); + b.tcgen05MmaWs(.f16, zero, .{ .tmem = dst }, swizzled, Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 128, .a_major = .k, .b_major = .k }), pred, .{}); + b.tcgen05MmaWs(.f16, zero, .{ .smem = interleaved }, swizzled, Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 256, .a_major = .k, .b_major = .mn }), pred, .{}); + b.setMaxRegisterIncrease(128); + b.barrierCtaSyncAligned(0, 512); + b.setMaxRegisterDecrease(128); + b.barrierCtaSyncAligned(b.threadIdx().x.div(128), 128); + b.mbarrierSpin(bar, zero, .cta); + b.mbarrierSpin(bar, zero, .cluster); + const state = b.mbarrierArrive(bar, .{}).?; + const remote = b.mapDsmemPtr(dst, zero); + _ = remote; + _ = b.mbarrierArrive(bar, .{ .peer_cta_rank_in_cluster = zero, .arrive_count = b.constant(.i32, 1), .scope = .cluster }); + b.storeAsyncDsmem(dst, .{ zero, zero }, bar, zero); + b.storeAsyncDsmem(dst, .{ zero, zero, zero, zero }, bar, zero); + b.barrierClusterArriveRelaxedAligned(); + b.barrierClusterWaitAligned(); + const any = b.voteAnySync(pred, null); + const mask = b.voteBallotSync(pred, null); + const x = b.constant(.f32, 1.0); + const y = b.constant(.f32, 2.0); + const maximum = b.fmax(x, y, .{}); + const sum = b.addPackedF32x2(.{ x, y }, .{ y, x }, .{}); + const product = b.mulPackedF32x2(sum, .{ x, y }, .{}); + const fused = b.fmaPackedF32x2(product, sum, .{ x, y }, .{}); + const scalar = b.fma(maximum, fused[0], fused[1]); + b.store(gmem, .{ scalar.bitCast(.i32), maximum.bitCast(.i32), mask, any.to(.i32), state.to(.i32), b.clock(), b.globalTimer().to(.i32), zero }, .{ .level1_eviction_priority = .evict_no_allocate }); + + const ir = try b.finish(.{ 512, 1, 1 }); + defer std.testing.allocator.free(ir); + for ([_][]const u8{ + "cute_nvgpu.get_tma_desc_addr", "tile::gather4", "nvvm.cp.async.bulk.prefetch", + "prefetch.global.L1", "prefetch.tensormap", "nvvm.tcgen05.mma.ws", + "nvvm.tcgen05.mma.ws", "setmaxnreg.inc", "setmaxnreg.dec", + "llvm.nvvm.barrier.cta.sync.aligned.count", "mbarrier.test_wait.parity.acquire.cta", "mbarrier.test_wait.parity.acquire.cluster", + "nvvm.mapa", "#nvvm.mem_scope", "nvvm.store.async", + "vector<4xi32>", "barrier.cluster.arrive.relaxed.aligned", "nvvm.cluster.wait", + "llvm.nvvm.vote.any.sync", "llvm.nvvm.vote.ballot.sync", "llvm.nvvm.fmax.f", + "nvvm.add.packed.f32x2", "nvvm.mul.packed.f32x2", "nvvm.fma.packed.f32x2", + "llvm.fma.f32", "st.global.L1::no_allocate.v8.b32", "%clock", + "%globaltimer", + }) |op| try expectContains(ir, op); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "SM100 BF16 weight-stationary descriptor encodes supported shapes and major modes" { + try std.testing.expectEqual(@as(u32, 0x04200490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 128, .a_major = .k, .b_major = .k })); + try std.testing.expectEqual(@as(u32, 0x04410490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 256, .a_major = .k, .b_major = .mn })); + try std.testing.expectEqual(@as(u32, 0x08118490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 128, .n_dim = 64, .a_major = .mn, .b_major = .mn })); + try std.testing.expectEqual(@as(u32, 0x02100490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 32, .n_dim = 64, .a_major = .k, .b_major = .k })); +} + +test "SM100 shared descriptors preserve nonzero swizzle bases and boundary offsets" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "smem_descriptors"); + defer b.deinit(); + const args = try b.declareArgs(.{ .out = .{ .ptr = DType.i64 } }); + const addr = b.constant(.i32, 128); + const fields = [_]u64{ + 0x4000404000010000, // sw128, base 0, leading 16 bytes, stride 1024 bytes + 0x4002404000010000, // sw128, pattern at byte 128: base 1 + 0x400e404000010000, // sw128, pattern at byte 896: base 7 + 0x00007fff3fff0000, // unswizzled, largest representable offsets + }; + const descs = [_]Value{ + Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .base_offset = 0, .layout = .swizzle_128b }), + Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .base_offset = 1, .layout = .swizzle_128b }), + Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .base_offset = 7, .layout = .swizzle_128b }), + Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 0x3fff0, .stride_byte_offset = 0x3fff0 }), + }; + const out = b.ptrToInt(args.out, .i64); + for (descs, 0..) |desc, i| b.storeWords(2, out.add(@as(i64, @intCast(i * 8))), .global, .{ desc.to(.i32), desc.shrLogical(32).to(.i32) }); + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + for (fields) |bits| { + const expected = try std.fmt.allocPrint(std.testing.allocator, "value = {d} : i64", .{bits}); + defer std.testing.allocator.free(expected); + try expectContains(ir, expected); + } + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "makeLayout preserves dynamic singleton and nested shape and stride leaves" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "mixed_layout"); + defer b.deinit(); + _ = try b.declareArgs(.{}); + const n = b.blockIdx().x.add(1); + const singleton = b.constant(.i32, 1); + const compact = b.makeLayout(.{ .{ 4, n }, singleton }, .{}); + const explicit = b.makeLayout(.{ n, 16 }, .{ .stride = .{ 1, n } }); + const static_nested = b.makeLayout(.{ .{ 4, 8 }, 16 }, .{}); + _ = compact; + _ = explicit; + _ = static_nested; + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try expectContains(ir, "((4,?),?):((1,4),?)"); + try expectContains(ir, "(?,16):(1,?)"); + try expectContains(ir, "((4,8),16):((1,4),32)"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "makeLayout preserves 64-bit shape and stride values" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "wide_layout"); + defer b.deinit(); + const args = try b.declareArgs(.{ .extent = .{ .mlir_type = .{ .type_ = DType.i64.toMlir(ctx) } } }); + const wide = args.extent.add(@as(i64, 1) << 32); + const explicit = b.makeLayout(.{ 8, 2 }, .{ .stride = .{ 1, wide } }); + const wide_leaf: AlgebraExpr = .{ .dynamic_int = .{ .width = 64 } }; + try std.testing.expect(explicit.type_().eql(b.layoutType(.{ + .shape = b.expr(.{ 8, 2 }), + .stride = .{ .tuple = &.{ .{ .static = 1 }, wide_leaf } }, + }))); + _ = b.makeLayout(.{ .{ 4, wide }, 1, 8 }, .{ .stride = null }); + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "arith.trunci") == null); + try expectContains(ir, "((4,?{i64}),1,8):((1,4),0,?{i64})"); + try expectContains(ir, "value = 4294967296 : i64"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "TMA constructors normalize tilers and staged layouts and copy executes store atoms" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "tma_stores"); + defer b.deinit(); + const cfg: TmaConfig = .{ + .dtype = .f32, + .copy_bits = 2048, + .global_basis = b.layoutSpec(.{ 16, 4 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_layout = b.layoutSpec(.{ 128, 64 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_rank = 2, + }; + const args = try b.declareArgs(.{ + .data = .{ .tensor = .{ .dtype = .f32, .shape = &.{ 128, 64 }, .stride = &.{ 1, 128 } } }, + }); + const staged = b.makeLayout(.{ 16, 4, 2 }, .{ .stride = .{ 1, 16, 64 } }); + const store = b.makeTiledTmaAtom(CopyBulkTensorTileS2GOp{}, args.data, staged, .{ 16, 4 }, cfg, .{}); + const smem = b.makeTensorView(b.intToPtr(0, .f32, .smem, 128), b.makeLayout(.{ 16, 4 }, .{})); + const tile = b.makeCoordTensor(.{ 0, 0 }, b.staticLayout(cfg.global_basis)); + b.copy(store.atom, .{smem}, .{tile}, .{}); + b.copy(store.atom, .{smem}, .{tile}, .{ .cache_policy = b.constant(.i64, CacheHintSm90.evict_last) }); + const swizzled = b.makeComposedLayout(.{ .bits = 1, .base = 4, .shift = 3 }, 0, b.layoutSpec(.{ 16, 4, 2 }, .{ 1, 16, 64 })); + _ = b.makeTiledTmaAtom(CopyBulkTensorTileG2SOp{}, args.data, swizzled, b.makeTile(&.{ 16, 4 }), cfg, .{}); + _ = b.makeTiledTmaAtom(CopyBulkTensorTileG2SOp{}, args.data, staged, b.makeLayout(.{ 16, 4 }, .{ .stride = .{ 1, 128 } }), cfg, .{}); + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"cute_nvgpu.atom.make_exec_tma\"")); + try std.testing.expectEqual(@as(usize, 3), std.mem.count(u8, ir, "\"cute.select\"")); + try expectContains(ir, "(16,4):(1@0,1@1)"); + try expectContains(ir, "tma_store"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "CuTe options preserve runtime masks, timed try waits, descriptors and store widths" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "cute_options"); + defer b.deinit(); + const a = try b.declareArgs(.{ .out = .{ .ptr = DType.i32 } }); + const tid = b.threadIdx().x; + const mbar = b.allocSmem(.i64, 1, 8); + b.barrier(.{}); + b.barrier(.{ .barrier_id = tid.div(128), .number_of_threads = b.blockDim().x }); + b.tcgen05Commit(mbar, .{ .cta_group = .two, .multicast_mask = b.clusterCtaRank().shl(1).bitOr(1) }); + _ = b.mbarrierTryWait(mbar, b.constant(.i32, 0), .{ .time_limit = tid.add(100), .scope = .cluster }); + const state = b.mbarrierArrive(mbar, .{ .relaxed = true }).?; + b.storeAsyncDsmem(b.constant(.i32, 128), tid, mbar, b.clusterCtaRank().bitXor(1)); + b.clusterArriveRelaxed(.{}); + b.clusterWait(); + b.clusterArriveRelaxed(.{ .aligned = true }); + b.barrierClusterWaitAligned(); + _ = b.voteAnySync(tid.eq(0), tid.bitOr(1)); + _ = b.voteBallotSync(tid.eq(0), 0xff); + const x = tid.to(.f32); + const y = b.fmax(x, b.constant(.f32, 0), .{ .nan = true, .ftz = true, .abs = true }); + const pair = b.addPackedF32x2(.{ x, y }, .{ y, x }, .{ .rnd = .rz, .ftz = true }); + b.store(a.out, state.to(.i32), .{}); + b.store(a.out, .{ pair[0].bitCast(.i32), pair[1].bitCast(.i32) }, .{}); + const desc = b.makeUmmaSmemDesc(b.intToPtr(0, .bf16, .smem, 16), b.layoutSpec(.{ 64, 16 }, .{ 16, 1 }), .k, null); + const raw_desc = b.smemDescriptorToInt(desc); + _ = b.intToSmemDescriptor(raw_desc); + const mapped = b.mapDsmemPtr(b.intToPtr(128, .i32, .smem, 16), b.clusterCtaRank()); + const mapped_ty = mapped.type_().isA(cute_ir.PtrType).?; + try std.testing.expectEqual(MemorySpace.dsmem, mapped_ty.addressSpace()); + try std.testing.expectEqual(@as(u64, 16), mapped_ty.alignment()); + b.store(mapped, tid, .{}); + const swizzles = [_]Tcgen05SmemSwizzle{ .none, .swizzle_32b, .swizzle_64b, .swizzle_128b, .swizzle_128b_atom_32b }; + for (swizzles) |layout| _ = Tcgen05SmemDesc.build(&b, tid.mul(16), .{ .layout = layout }); + const idesc = b.constant(.i32, Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 128 })); + b.tcgen05MmaWs(.f16, tid, .{ .tmem = tid }, desc, idesc, tid.ne(0), .{ .collector_b_buffer = .b2, .collector_op = .use, .col_b_zero_mask = state }); + b.cpAsyncBulkPrefetch(a.out, 128, .{ .l2_cache_hint = state }); + const ir = try b.finish(.{ 128, 1, 1 }); + defer std.testing.allocator.free(ir); + try expectContains(ir, "nvvm.mbarrier.try_wait.parity.timelimit"); + try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, ir, "scf.while")); + try expectContains(ir, "nvvm.store.ext"); + try expectContains(ir, "vector<2xi32>"); + try expectContains(ir, "#nvvm.fp_rnd_mode"); + try expectContains(ir, "cute_nvgpu.make_umma_smem_desc"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "gather4 uses CuTe TMA atom construction, partitioning and copy options" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.openProgram(std.testing.allocator, ctx, "gather4_copy"); + defer b.deinit(); + const cfg: TmaConfig = .{ + .dtype = .f32, + .copy_bits = 2048, + .global_basis = b.layoutSpec(.{ 16, 1 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_layout = b.layoutSpec(.{ 128, 64 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_rank = 2, + .mode = .gather4, + }; + var pair = cfg; + pair.cta_group = .two; + const args = try b.declareArgs(.{ + .indices = .{ .tensor = .{ .dtype = .i32, .shape = &.{ 128, 64 }, .stride = &.{ 0, 1 } } }, + .atom = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(cfg), .grid_constant = true } }, + .pair = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(pair), .grid_constant = true } }, + }); + const smem_layout = b.makeLayout(.{ 16, 4 }, .{ .stride = .{ 1, 16 } }); + const smem = b.makeTensorView(b.intToPtr(0, .f32, .smem, 128), smem_layout); + const coords = b.makeCoordTensor(.{ 0, 0 }, b.staticLayout(cfg.coordinate_layout)); + const atom: Atom = .{ .inner = args.atom.inner, .kernel = &b }; + const partitioned = b.tmaPartition(atom, 0, b.makeLayout(1, .{}), smem, .{ coords, args.indices.view() }, smem.type_(), &.{ coords.type_(), args.indices.inner.type_() }); + const mbar = b.allocSmem(.i64, 1, 8); + b.copy(atom, .{ partitioned.targets[0], partitioned.targets[1] }, .{partitioned.smem}, .{ + .tma_bar_ptr = mbar, + .cache_policy = b.constant(.i64, CacheHintSm100.evict_last), + }); + b.copy(.{ .inner = args.pair.inner, .kernel = &b }, .{ partitioned.targets[0], partitioned.targets[1] }, .{partitioned.smem}, .{ .tma_bar_ptr = mbar }); + b.endFunction(.{ 32, 1, 1 }); + + b.beginFunction("gather4_copy", .host); + const host = try b.declareArgs(.{ + .data = .{ .tensor = .{ .dtype = .f32, .shape = &.{ 128, 64 }, .stride = &.{ 1, 128 } } }, + .indices = .{ .tensor = .{ .dtype = .i32, .shape = &.{ 128, 64 }, .stride = &.{ 0, 1 } } }, + }); + const host_layout = b.makeLayout(.{ 16, 4 }, .{ .stride = .{ 1, 16 } }); + const tiler = b.makeTile(&.{ 16, 4 }); + _ = b.makeTiledTmaAtom(CopyBulkTensor2DGather4G2SOp{}, host.data, host_layout, tiler, cfg, .{ .gmem_coord_tensor = host.indices.view() }); + _ = b.makeTiledTmaAtom(CopyBulkTensor2DGather4G2SOp{ .cta_group = .two }, host.data, host_layout, tiler, pair, .{ .gmem_coord_tensor = host.indices.view() }); + b.returnHostStatus(b.constant(.i32, 0)); + b.endFunction(null); + const ir = try b.finishProgram(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "atom.make_non_exec_2d_gather4_tma_load"); + try expectContains(ir, "atom.tma_partition"); + try expectContains(ir, "mode = gather4, num_cta = 1"); + try expectContains(ir, "mode = gather4, num_cta = 2"); + try expectContains(ir, "non_exec_2d_gather4_tma_load(attr); } +intptr_t mlirCuteShapeAttrGetRank(MlirAttribute attr) { + return cutegen::rank(llvm::cast(unwrap(attr)).getRef()); +} + MlirAttribute mlirCuteStrideAttrStaticGet(MlirContext context, int64_t value) { return getStaticAttr(context, value); } diff --git a/mlir/dialects/cute_ir/capi.h b/mlir/dialects/cute_ir/capi.h index e9e285d2..002f4885 100644 --- a/mlir/dialects/cute_ir/capi.h +++ b/mlir/dialects/cute_ir/capi.h @@ -54,6 +54,7 @@ MLIR_CAPI_EXPORTED MlirAttribute mlirCuteShapeAttrTupleGet( MlirContext context, intptr_t numElements, MlirAttribute const *elements); MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteShape(MlirAttribute attr); MLIR_CAPI_EXPORTED MlirStringRef mlirCuteShapeAttrGetValue(MlirAttribute attr); +MLIR_CAPI_EXPORTED intptr_t mlirCuteShapeAttrGetRank(MlirAttribute attr); MLIR_CAPI_EXPORTED MlirAttribute mlirCuteStrideAttrStaticGet(MlirContext context, int64_t value); diff --git a/mlir/dialects/cute_ir/cute_ir.zig b/mlir/dialects/cute_ir/cute_ir.zig index 8a031d0b..b277d061 100644 --- a/mlir/dialects/cute_ir/cute_ir.zig +++ b/mlir/dialects/cute_ir/cute_ir.zig @@ -145,6 +145,10 @@ pub const ShapeAttr = opaque { return mlir.string(c.mlirCuteShapeAttrGetValue(self.ptr())); } + pub fn rank(self: *const ShapeAttr) usize { + return @intCast(c.mlirCuteShapeAttrGetRank(self.ptr())); + } + pub fn attribute(self: *const ShapeAttr) *const mlir.Attribute { return @ptrCast(self); } diff --git a/mlir/dialects/llvm/llvm.zig b/mlir/dialects/llvm/llvm.zig index 78524e0a..d1197317 100644 --- a/mlir/dialects/llvm/llvm.zig +++ b/mlir/dialects/llvm/llvm.zig @@ -22,6 +22,7 @@ pub const elementtype = ops.elementtype; pub const AtomicBinOp = ops.AtomicBinOp; pub const AtomicOrdering = ops.AtomicOrdering; pub const inttoptr = ops.inttoptr; +pub const ptrtoint = ops.ptrtoint; pub const bitcast = ops.bitcast; pub const fmul = ops.fmul; pub const extractvalue = ops.extractvalue; diff --git a/mlir/dialects/llvm/ops.zig b/mlir/dialects/llvm/ops.zig index 2102f68a..d1fb5771 100644 --- a/mlir/dialects/llvm/ops.zig +++ b/mlir/dialects/llvm/ops.zig @@ -234,6 +234,15 @@ pub fn inttoptr(ctx: *mlir.Context, address: *const mlir.Value, result_type: *co }); } +/// llvm.ptrtoint — the integer address of an LLVM pointer. +pub fn ptrtoint(ctx: *mlir.Context, pointer: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.ptrtoint", .{ + .operands = .{ .flat = &.{pointer} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + /// llvm.bitcast — reinterpret `value` as `result_type` (same bit width). pub fn bitcast(ctx: *mlir.Context, value: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "llvm.bitcast", .{ diff --git a/mlir/dialects/llvm/test.zig b/mlir/dialects/llvm/test.zig index 28541d44..6ba84b4d 100644 --- a/mlir/dialects/llvm/test.zig +++ b/mlir/dialects/llvm/test.zig @@ -20,6 +20,7 @@ const operation_names: []const []const u8 = &.{ "llvm.intr.lifetime.end", "llvm.intr.lifetime.start", "llvm.inttoptr", + "llvm.ptrtoint", "llvm.load", "llvm.mlir.poison", "llvm.mlir.undef", diff --git a/mlir/dialects/nvvm/nvvm.zig b/mlir/dialects/nvvm/nvvm.zig index d46f3a4f..14a0e8e2 100644 --- a/mlir/dialects/nvvm/nvvm.zig +++ b/mlir/dialects/nvvm/nvvm.zig @@ -136,7 +136,8 @@ pub fn mbarrier_wait_parity( } /// nvvm.mbarrier.try_wait.parity — wait (looping in PTX) until phase `parity` (i32) -/// of the mbarrier completes, suspending up to `suspend_time` (i32) cycles per try. +/// of the mbarrier completes, with CTA acquire semantics. `suspend_time` (i32) +/// is a suspend-time hint in nanoseconds per try. pub fn mbarrier_try_wait_parity( ctx: *mlir.Context, ptr: *const mlir.Value, @@ -147,6 +148,27 @@ pub fn mbarrier_try_wait_parity( return ops.mbarrier_try_wait_parity(ctx, ptr, parity, suspend_time, .{ .useIntrinsic = .boolean(ctx, false) }, location); } +/// One potentially suspending parity test with acquire semantics. A false +/// result means the timeout expired; callers that need completion must retry. +pub fn mbarrier_try_wait_parity_timelimit( + ctx: *mlir.Context, + ptr: *const mlir.Value, + parity: *const mlir.Value, + suspend_time: *const mlir.Value, + scope: MBarrierScope, + location: *const mlir.Location, +) *mlir.Operation { + return ops.mbarrier_try_wait_parity_timelimit( + ctx, + ptr, + parity, + suspend_time, + .int(ctx, .i1), + .{ .scope = enumAttribute(ops.MBarrierScopeKindAttr, ctx, scope), .order = enumAttribute(ops.MemOrderKindAttr, ctx, .acquire) }, + location, + ); +} + /// nvvm.fence.mbarrier.init — make mbarrier initialization visible to the cluster. pub fn fence_mbarrier_init(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { return ops.fence_mbarrier_init(ctx, location); @@ -208,10 +230,12 @@ pub fn tcgen05_wait(ctx: *mlir.Context, kind: Tcgen05WaitKind, location: *const return ops.tcgen05_wait(ctx, enumAttribute(ops.Tcgen05WaitKindAttr, ctx, kind), location); } -/// nvvm.tcgen05.commit — arrive on the mbarrier at `barrier_ptr` when the issued -/// tcgen05 operations complete. -pub fn tcgen05_commit(ctx: *mlir.Context, barrier_ptr: *const mlir.Value, group: CtaGroup, location: *const mlir.Location) *mlir.Operation { - return ops.tcgen05_commit(ctx, barrier_ptr, .{ .group = enumAttribute(ops.CTAGroupKindAttr, ctx, group) }, location); +/// nvvm.tcgen05.commit — arrive on the mbarrier at `barrier_ptr` when the +/// tcgen05 MMA/copy operations issued by the executing thread with `group` +/// complete. An optional mask selects CTAs in the cluster whose barriers at +/// the same shared-memory offset receive one arrival each. +pub fn tcgen05_commit(ctx: *mlir.Context, barrier_ptr: *const mlir.Value, group: CtaGroup, multicast_mask: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return ops.tcgen05_commit(ctx, barrier_ptr, .{ .multicastMask = multicast_mask, .group = enumAttribute(ops.CTAGroupKindAttr, ctx, group) }, location); } /// nvvm.tcgen05.mma — D (tensor memory, `!llvm.ptr<6>`) = A (shared-memory diff --git a/zml/kernel.zig b/zml/kernel.zig index 03a64d70..36c9d8d3 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -513,6 +513,7 @@ pub const cuda_tile = struct { }; pub const cute = struct { + pub const AlgebraExpr = cute_builder.AlgebraExpr; pub const Builder = cute_builder.Builder; pub const ArgSpec = cute_builder.ArgSpec; pub const Value = cute_builder.Value; @@ -526,6 +527,16 @@ pub const cute = struct { pub const TmaConfig = cute_builder.TmaConfig; pub const TmaFormat = cute_builder.TmaFormat; pub const BlockScaledMmaConfig = cute_builder.BlockScaledMmaConfig; + pub const CtaGroup = cute_builder.CtaGroup; + pub const ctaCount = cute_builder.ctaCount; + pub const MBarrierScope = cute_builder.MBarrierScope; + pub const OperandMajorMode = cute_builder.OperandMajorMode; + pub const Tcgen05SmemSwizzle = cute_builder.Tcgen05SmemSwizzle; + pub const Tcgen05SmemDesc = cute_builder.Tcgen05SmemDesc; + pub const Tcgen05InstrDesc = cute_builder.Tcgen05InstrDesc; + pub const CopyBulkTensorTileG2SOp = cute_builder.CopyBulkTensorTileG2SOp; + pub const CopyBulkTensor2DGather4G2SOp = cute_builder.CopyBulkTensor2DGather4G2SOp; + pub const CopyBulkTensorTileS2GOp = cute_builder.CopyBulkTensorTileS2GOp; pub fn newContext() std.mem.Allocator.Error!*mlir.Context { return makeKernelContext(cute_builder.registerDialects); diff --git a/zml/kernels/moe/cute/sm10x/common.zig b/zml/kernels/moe/cute/sm10x/common.zig index 47e05d37..da639c75 100644 --- a/zml/kernels/moe/cute/sm10x/common.zig +++ b/zml/kernels/moe/cute/sm10x/common.zig @@ -138,7 +138,7 @@ pub fn advancePipeline(b: *B, index: V, phase: V, comptime stages: usize) Pipeli pub fn peekBarrierIfRemaining(b: *B, base: V, count: V, limit: usize, state: PipelinePosition) V { var peek = b.openIfElse(count.lt(limit), .{.i1}); { - peek.yieldThen(.{b.mbarrierWaitParity(barrierAt(b, base, state.index), state.phase)}); + peek.yieldThen(.{b.mbarrierTryWait(barrierAt(b, base, state.index), state.phase, .{ .scope = .cta })}); } { peek.yieldElse(.{b.constant(.i1, true)}); diff --git a/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig b/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig index e8901fb1..21a76436 100644 --- a/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig +++ b/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig @@ -762,10 +762,10 @@ fn issueTmaStage( b.mbarrierArriveExpectTx(stage.load_barrier, tile.transactionBytes()); elected.close(); } - b.copy(b.setTmaBarrier(exec_a, stage.load_barrier), .{a_source}, .{stage.a_destination}, null); - b.copy(b.setTmaBarrier(exec_b, stage.load_barrier), .{b_source}, .{stage.b_destination}, null); - b.copy(b.setTmaBarrier(exec_sfa, stage.load_barrier), .{sfa_source}, .{stage.sfa_destination}, null); - b.copy(b.setTmaBarrier(exec_sfb, stage.load_barrier), .{sfb_source}, .{stage.sfb_destination}, null); + b.copy(b.setTmaBarrier(exec_a, stage.load_barrier), .{a_source}, .{stage.a_destination}, .{}); + b.copy(b.setTmaBarrier(exec_b, stage.load_barrier), .{b_source}, .{stage.b_destination}, .{}); + b.copy(b.setTmaBarrier(exec_sfa, stage.load_barrier), .{sfa_source}, .{stage.sfa_destination}, .{}); + b.copy(b.setTmaBarrier(exec_sfb, stage.load_barrier), .{sfb_source}, .{stage.sfb_destination}, .{}); } fn accumulatorTmemAt(b: *B, comptime tile: Tile, tmem: V, stage: V) V { @@ -826,10 +826,7 @@ fn runTmaProducer( .index = tiles.after_carried[5], .phase = tiles.after_carried[6], }; - const first_ready = b.mbarrierWaitParity( - barrierAt(b, storage.mma_barriers, first_state.index), - first_state.phase, - ); + const first_ready = b.mbarrierTryWait(barrierAt(b, storage.mma_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); var stages = b.openFor( 0, k_tiles, @@ -845,7 +842,7 @@ fn runTmaProducer( const buffers = stageBuffersAt(b, tile, storage, state.index); var pending = b.openIf(stages.carried[0].eq(false)); { - b.mbarrierTryWaitParity(buffers.mma_barrier, state.phase, 10_000_000); + b.mbarrierWait(buffers.mma_barrier, state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); pending.close(); } issueTmaStage(b, tile, k_tiles, partitioned_a_target, partitioned_b_target, buffers, k, tile_m, group, expert, exec_a, exec_b, exec_sfa, exec_sfb); @@ -893,11 +890,7 @@ fn runTmaProducer( .phase = tiles.results[6], }; inline for (0..tile.stages) |stage| { - b.mbarrierTryWaitParity( - barrierAt(b, storage.mma_barriers, tail_state.index), - tail_state.phase, - 10_000_000, - ); + b.mbarrierWait(barrierAt(b, storage.mma_barriers, tail_state.index), tail_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); if (stage + 1 < tile.stages) { tail_state = advancePipeline(b, tail_state.index, tail_state.phase, tile.stages); } @@ -922,8 +915,8 @@ const Mxfp4Mainloop = struct { sfb_target: V, pub fn copyScales(self: Mxfp4Mainloop, builder: *B) void { - builder.copy(self.sfa_copy, .{self.sfa_source}, .{self.sfa_target}, null); - builder.copy(self.sfb_copy, .{self.sfb_source}, .{self.sfb_target}, null); + builder.copy(self.sfa_copy, .{self.sfa_source}, .{self.sfa_target}, .{}); + builder.copy(self.sfb_copy, .{self.sfb_source}, .{self.sfb_target}, .{}); } }; @@ -1164,17 +1157,14 @@ fn runMmaConsumer( .index = tiles.after_carried[5], .phase = tiles.after_carried[6], }; - const first_ready = b.mbarrierWaitParity( - barrierAt(b, storage.load_barriers, first_state.index), - first_state.phase, - ); + const first_ready = b.mbarrierTryWait(barrierAt(b, storage.load_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); const accumulator_state: PipelinePosition = .{ .index = tiles.after_carried[9], .phase = tiles.after_carried[10], }; const accumulator_stage = accumulator_state.index; const accumulator_empty = barrierAt(b, storage.accumulator_empty_barriers, accumulator_stage); - b.mbarrierTryWaitParity(accumulator_empty, accumulator_state.phase, 10_000_000); + b.mbarrierWait(accumulator_empty, accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); const persistent_mma: cute.Atom = .{ .inner = tiles.after_carried[7].inner, .kernel = b }; const initial_mma = b.setMmaAccumulate(persistent_mma, b.constant(.i1, false)); @@ -1192,7 +1182,7 @@ fn runMmaConsumer( const state: PipelinePosition = .{ .index = stages.carried[3], .phase = stages.carried[4] }; var pending = b.openIf(stages.carried[0].eq(false)); { - b.mbarrierTryWaitParity(barrierAt(b, storage.load_barriers, state.index), state.phase, 10_000_000); + b.mbarrierWait(barrierAt(b, storage.load_barriers, state.index), state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); pending.close(); } const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, accumulator_stage); @@ -1209,7 +1199,7 @@ fn runMmaConsumer( ); var commit_lane = b.openIf(b.electSync()); { - b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index)); + b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index), .{}); commit_lane.close(); } const next_count = stages.carried[2].add(1); @@ -1220,7 +1210,7 @@ fn runMmaConsumer( var full_lane = b.openIf(b.electSync()); { - b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage)); + b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage), .{}); full_lane.close(); } const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); @@ -1271,11 +1261,7 @@ fn runMmaConsumer( inline for (0..accumulator_stages - 1) |_| { accumulator_tail = advancePipeline(b, accumulator_tail.index, accumulator_tail.phase, accumulator_stages); } - b.mbarrierTryWaitParity( - barrierAt(b, storage.accumulator_empty_barriers, accumulator_tail.index), - accumulator_tail.phase, - 10_000_000, - ); + b.mbarrierWait(barrierAt(b, storage.accumulator_empty_barriers, accumulator_tail.index), accumulator_tail.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); } /// Epilogue operands; which fields are set depends on the `Epilogue`. @@ -1349,7 +1335,7 @@ fn runEpilogue( .phase = tiles.after_carried[5], }; const stage = accumulator_state.index; - b.mbarrierTryWaitParity(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, 10_000_000); + b.mbarrierWait(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, stage); const warp_tmem = b.addOffset( @@ -1375,7 +1361,7 @@ fn runEpilogue( if (chunk + 1 == tile_chunks) { // Registers now own the accumulator values, so all 128 epilogue // threads release this TMEM stage before the stores. - b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), 1); + _ = b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), .{}); } var columns: [@intCast(@min(n, 32))]V = undefined; inline for (&columns, 0..) |*column, i| column.* = b.vectorExtract(values, i, .i32).bitCast(.f32); @@ -1405,7 +1391,7 @@ fn runEpilogue( b.relinquishTmemAllocPermit(.{}); release_warp.close(); } - b.namedBarrier(1, 128); + b.barrierCtaSyncAligned(1, 128); var dealloc_warp = b.openIf(warp.eq(0)); { b.deallocTmem(tmem, tile.tmem_columns, .{}); @@ -1545,14 +1531,14 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo scratch.set(.{ warp, lane }, lane_maximum); owner.close(); } - b.namedBarrier(1, 128); + b.barrierCtaSyncAligned(1, 128); // Lane `c` turns the two half-block maxima of column `c` into its scale. const pair = warp.bitAnd(~@as(i32, 1)); const lane_column = b.select(lane.lt(n), lane, b.constant(.i32, 0)); const scale = mxScale(b, scratch.get(.{ pair, lane_column }, .f32).maximum(scratch.get(.{ pair.add(1), lane_column }, .f32))); // The next tile reuses the scratch maxima. - b.namedBarrier(1, 128); + b.barrierCtaSyncAligned(1, 128); const activation_index = tile_m.mul(64).add(tid.div(2)); for (0..row_chunks) |row_chunk| { @@ -1767,7 +1753,7 @@ fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E var mma_warp = b.openIf(warp.eq(4)); { // Python's MMA role reaches its own TmemAllocator.wait_for_alloc site. - b.namedBarrier(2, 160); + b.barrierCtaSyncAligned(2, 160); switch (k_tiles) { inline 16, 18, 32, 40 => |kt| runMmaConsumer(b, tile, kt, storage, tiled_mma, tmem_holding, active, d.tiles_m), else => unreachable, @@ -1784,7 +1770,7 @@ fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E b.allocTmem(tile.tmem_columns, tmem_holding, .{}); alloc_warp.close(); } - b.namedBarrier(2, 160); + b.barrierCtaSyncAligned(2, 160); switch (epilogue) { .swiglu_mxfp8 => runEpilogue(b, tile, epilogue, storage, tmem_holding, active, d.tiles_m, direct, .{ .group_sizes = view(b, d.group_sizes), diff --git a/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig b/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig index ac59735c..0de45007 100644 --- a/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig +++ b/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig @@ -699,10 +699,10 @@ fn issueTmaStage( var elected = b.openIf(b.electSync()); b.mbarrierArriveExpectTx(stage.load_barrier, tile.transactionBytes()); elected.close(); - b.copy(b.setTmaBarrier(exec_a, stage.load_barrier), .{a_source}, .{stage.a_destination}, null); - b.copy(b.setTmaBarrier(exec_b, stage.load_barrier), .{b_source}, .{stage.b_destination}, null); - b.copy(b.setTmaBarrier(exec_sfa, stage.load_barrier), .{sfa_source}, .{stage.sfa_destination}, null); - b.copy(b.setTmaBarrier(exec_sfb, stage.load_barrier), .{sfb_source}, .{stage.sfb_destination}, null); + b.copy(b.setTmaBarrier(exec_a, stage.load_barrier), .{a_source}, .{stage.a_destination}, .{}); + b.copy(b.setTmaBarrier(exec_b, stage.load_barrier), .{b_source}, .{stage.b_destination}, .{}); + b.copy(b.setTmaBarrier(exec_sfa, stage.load_barrier), .{sfa_source}, .{stage.sfa_destination}, .{}); + b.copy(b.setTmaBarrier(exec_sfb, stage.load_barrier), .{sfb_source}, .{stage.sfb_destination}, .{}); } fn accumulatorTmemAt(b: *B, comptime tile: Tile, tmem: V, stage: V) V { @@ -756,10 +756,7 @@ fn runTmaProducer( .index = tiles.after_carried[5], .phase = tiles.after_carried[6], }; - const first_ready = b.mbarrierWaitParity( - barrierAt(b, storage.mma_barriers, first_state.index), - first_state.phase, - ); + const first_ready = b.mbarrierTryWait(barrierAt(b, storage.mma_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); var stages = b.openFor( 0, @as(i32, @intCast(k_tiles)), @@ -773,7 +770,7 @@ fn runTmaProducer( }; const buffers = stageBuffersAt(b, tile, storage, state.index); var pending = b.openIf(stages.carried[0].eq(false)); - b.mbarrierTryWaitParity(buffers.mma_barrier, state.phase, 10_000_000); + b.mbarrierWait(buffers.mma_barrier, state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); pending.close(); issueTmaStage(b, tile, k_tiles, partitioned_a_target, partitioned_b_target, buffers, k, tile_m, group, expert, exec_a, exec_b, exec_sfa, exec_sfb); const next_count = stages.carried[1].add(1); @@ -817,11 +814,7 @@ fn runTmaProducer( .phase = tiles.results[6], }; inline for (0..tile.stages) |stage| { - b.mbarrierTryWaitParity( - barrierAt(b, storage.mma_barriers, tail_state.index), - tail_state.phase, - 10_000_000, - ); + b.mbarrierWait(barrierAt(b, storage.mma_barriers, tail_state.index), tail_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); if (stage + 1 < tile.stages) { tail_state = advancePipeline(b, tail_state.index, tail_state.phase, tile.stages); } @@ -843,8 +836,8 @@ const Mainloop = struct { sfb_target: V, pub fn copyScales(self: Mainloop, builder: *B) void { - builder.copy(self.sfa_copy, .{self.sfa_source}, .{self.sfa_target}, null); - builder.copy(self.sfb_copy, .{self.sfb_source}, .{self.sfb_target}, null); + builder.copy(self.sfa_copy, .{self.sfa_source}, .{self.sfa_target}, .{}); + builder.copy(self.sfb_copy, .{self.sfb_source}, .{self.sfb_target}, .{}); } }; @@ -1020,17 +1013,14 @@ fn runMmaConsumer( .index = tiles.after_carried[5], .phase = tiles.after_carried[6], }; - const first_ready = b.mbarrierWaitParity( - barrierAt(b, storage.load_barriers, first_state.index), - first_state.phase, - ); + const first_ready = b.mbarrierTryWait(barrierAt(b, storage.load_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); const accumulator_state: PipelinePosition = .{ .index = tiles.after_carried[9], .phase = tiles.after_carried[10], }; const accumulator_stage = accumulator_state.index; const accumulator_empty = barrierAt(b, storage.accumulator_empty_barriers, accumulator_stage); - b.mbarrierTryWaitParity(accumulator_empty, accumulator_state.phase, 10_000_000); + b.mbarrierWait(accumulator_empty, accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); const persistent_mma: cute.Atom = .{ .inner = tiles.after_carried[7].inner, .kernel = b }; const initial_mma = b.setMmaAccumulate(persistent_mma, b.constant(.i1, false)); @@ -1043,7 +1033,7 @@ fn runMmaConsumer( const k = stages.carried[2]; const state: PipelinePosition = .{ .index = stages.carried[3], .phase = stages.carried[4] }; var pending = b.openIf(stages.carried[0].eq(false)); - b.mbarrierTryWaitParity(barrierAt(b, storage.load_barriers, state.index), state.phase, 10_000_000); + b.mbarrierWait(barrierAt(b, storage.load_barriers, state.index), state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); pending.close(); const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, accumulator_stage); const mainloop = staged_mainloop.atStage(b, state.index, accumulator_tmem); @@ -1058,7 +1048,7 @@ fn runMmaConsumer( mainloop.accumulator, ); var commit_lane = b.openIf(b.electSync()); - b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index)); + b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index), .{}); commit_lane.close(); const next_count = stages.carried[2].add(1); const next_state = advancePipeline(b, state.index, state.phase, tile.stages); @@ -1066,7 +1056,7 @@ fn runMmaConsumer( stages.yield(.{ next_ready, mma.value(), next_count, next_state.index, next_state.phase }); var full_lane = b.openIf(b.electSync()); - b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage)); + b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage), .{}); full_lane.close(); const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); const next_acc_count = tiles.after_carried[8].add(1); @@ -1113,11 +1103,7 @@ fn runMmaConsumer( inline for (0..accumulator_stages - 1) |_| { accumulator_tail = advancePipeline(b, accumulator_tail.index, accumulator_tail.phase, accumulator_stages); } - b.mbarrierTryWaitParity( - barrierAt(b, storage.accumulator_empty_barriers, accumulator_tail.index), - accumulator_tail.phase, - 10_000_000, - ); + b.mbarrierWait(barrierAt(b, storage.accumulator_empty_barriers, accumulator_tail.index), accumulator_tail.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); } /// Epilogue operands; which fields are set depends on the `Epilogue`. @@ -1193,7 +1179,7 @@ fn runEpilogue( const expert = operands.expert_ids.get(.{group}, .i32); const alpha = operands.alpha.get(.{expert}, .f32); const act_scale = if (operands.act_scale) |scales| scales.get(.{expert}, .f32) else null; - b.mbarrierTryWaitParity(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, 10_000_000); + b.mbarrierWait(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, stage); const warp_tmem = b.addOffset( @@ -1213,7 +1199,7 @@ fn runEpilogue( const values = b.tmemLoadVector(chunk_tmem, @intCast(chunk_rows), .{ .num_dp = 32, .num_b = 32, .num_rep = @intCast(chunk_rows) }); b.fenceTmemLoad(); if (chunk + 1 == tile_chunks) { - b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), 1); + _ = b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), .{}); } var columns: [@intCast(@min(n, 32))]V = undefined; inline for (&columns, 0..) |*column, i| column.* = b.vectorExtract(values, i, .i32).bitCast(.f32).mul(alpha); @@ -1239,7 +1225,7 @@ fn runEpilogue( var release_warp = b.openIf(warp.eq(0)); b.relinquishTmemAllocPermit(.{}); release_warp.close(); - b.namedBarrier(1, 128); + b.barrierCtaSyncAligned(1, 128); var dealloc_warp = b.openIf(warp.eq(0)); b.deallocTmem(tmem, tile.tmem_columns, .{}); dealloc_warp.close(); @@ -1543,7 +1529,7 @@ fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E tma_warp.close(); var mma_warp = b.openIf(warp.eq(4)); - b.namedBarrier(2, 160); + b.barrierCtaSyncAligned(2, 160); runMmaConsumer(b, tile, reduction_tiles, storage, tiled_mma, tmem_holding, active, d.tiles_m); mma_warp.close(); @@ -1551,7 +1537,7 @@ fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E var alloc_warp = b.openIf(warp.eq(0)); b.allocTmem(tile.tmem_columns, tmem_holding, .{}); alloc_warp.close(); - b.namedBarrier(2, 160); + b.barrierCtaSyncAligned(2, 160); const shared_operands: EpilogueOperands = .{ .expert_ids = expert_ids, .alpha = view(b, d.alpha), diff --git a/zml/kernels/paged_attention/cute/sm10x/attention.zig b/zml/kernels/paged_attention/cute/sm10x/attention.zig index 89761bdd..12d6273c 100644 --- a/zml/kernels/paged_attention/cute/sm10x/attention.zig +++ b/zml/kernels/paged_attention/cute/sm10x/attention.zig @@ -317,7 +317,7 @@ fn produce(t: Tile) void { const id = tiles.iv.mul(c.features() * 2).add(stage); var reuse = b.openIf(id.ge(c.buffers())); { - b.mbarrierWait(t.released(id), t.phase(id).bitXor(1)); + b.mbarrierWait(t.released(id), t.phase(id).bitXor(1), .{ .scope = .cta }); reuse.close(); } const source = if (stage < c.features()) t.k else t.v; @@ -339,7 +339,7 @@ fn multiply(t: Tile, tile: V, comptime qk: bool) void { { for (0..@as(usize, @intCast(c.features()))) |part| { const id = tile.mul(c.features() * 2).add(@as(i64, @intCast(part)) + if (qk) @as(i64, 0) else c.features()); - b.mbarrierWait(t.ready(id), t.phase(id)); + b.mbarrierWait(t.ready(id), t.phase(id), .{ .scope = .cta }); b.tcgen05Fence(.after); const kv = descriptor(b, t.shared.add(id.rem(c.buffers()).mul(c.kvBytes())), if (qk) 16 else c.keys() * 128); const qp = descriptor(b, t.shared.add(if (qk) c.qBase() else c.pBase()), 16); @@ -353,22 +353,22 @@ fn multiply(t: Tile, tile: V, comptime qk: bool) void { const accumulate = if (qk) b.constant(.i32, @intFromBool(part > 0 or step > 0)).ne(0) else tile.gt(0).bitOr(b.constant(.i32, @intFromBool(step > 0)).ne(0)); b.tcgen05MmaF16(t.tmem.add(column), left, right, b.constant(.i32, @as(i32, @bitCast(instruction(c, qk)))), accumulate); } - b.tcgen05Commit(t.released(id)); + b.tcgen05Commit(t.released(id), .{}); } - b.tcgen05Commit(t.shared.add(c.completion())); + b.tcgen05Commit(t.shared.add(c.completion()), .{}); leader.close(); } - b.mbarrierWait(t.shared.add(c.completion()), b.constant(.i32, if (qk) 0 else 1)); + b.mbarrierWait(t.shared.add(c.completion()), b.constant(.i32, if (qk) 0 else 1), .{ .scope = .cta }); b.tcgen05Fence(.after); // A fast warp must not wrap completion parity before every consumer has // observed the preceding PV completion. - if (!qk) b.barrierSync(1, 128); + if (!qk) b.barrierCtaSyncAligned(1, 128); } fn publishProbabilities(t: Tile) void { t.b.fenceProxyAsyncShared(); t.b.tcgen05Fence(.before); - t.b.barrierSync(1, 128); + t.b.barrierCtaSyncAligned(1, 128); t.b.tcgen05Fence(.after); } @@ -393,7 +393,7 @@ fn reduceHeads(t: Tile, comptime heads: usize, values: [heads]V, comptime maximu leader.close(); } } - b.barrierSync(1, 128); + b.barrierCtaSyncAligned(1, 128); var result: [heads]V = undefined; for (0..heads) |h| { var value = b.loadWords(1, t.shared.add(t.c.reductions() + @as(i64, @intCast(h * 4))), .shared, .{})[0].bitCast(.f32); @@ -403,7 +403,7 @@ fn reduceHeads(t: Tile, comptime heads: usize, values: [heads]V, comptime maximu } result[h] = value; } - b.barrierSync(1, 128); + b.barrierCtaSyncAligned(1, 128); return result; } @@ -514,7 +514,7 @@ fn prefill(t: Tile) void { preserve.yieldElse(@as([64]V, @splat(zero))); } @memcpy(&saved, &preserve.results); - b.barrierSync(1, 128); + b.barrierCtaSyncAligned(1, 128); } multiply(t, tiles.iv, true); var scores: [64]V = undefined; From 439ba6c0a61990f4a8d4a08217eaa17b751112a7 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Tue, 3 Nov 2026 22:45:23 +0900 Subject: [PATCH 836/881] bazel: register runfiles initialization from C (#1005) ZML initializes runfiles before `main` so its loaders can locate packaged libraries and data. When using `-fno-llvm -fincremental` and linking with the incremental ELF linker, declaring the registration pointer in Zig produces incorrect `.init_array` data. See https://codeberg.org/ziglang/zig/issues/37453 Use a C constructor on Linux and macOS so the C compiler emits the platform-specific startup registration, while Zig exports the existing runfiles initialization function directly. We can revert when https://codeberg.org/ziglang/zig/issues/37453 is fixed. With this fix, we bypass the wrong behavior in incremental ELF linker and get closer to ZML compliance with zig incremental using their incremental linker. The registration is shared by the `use_cc_common_link=true` and `false` paths. Keep the explicit linker retention so the constructor is included even when its object is in a static archive. --- bazel/BUILD.bazel | 4 +++- bazel/bazel.zig | 41 ++++++++++++++++------------------------- bazel/runfiles_init.c | 15 +++++++++++++++ 3 files changed, 34 insertions(+), 26 deletions(-) create mode 100644 bazel/runfiles_init.c diff --git a/bazel/BUILD.bazel b/bazel/BUILD.bazel index 434ecfa9..eb2b2ab2 100644 --- a/bazel/BUILD.bazel +++ b/bazel/BUILD.bazel @@ -3,7 +3,9 @@ load("@rules_zig//zig:defs.bzl", "zig_library") cc_library( name = "retain_runfiles_init", - # Mach-O adds an underscore to the exported Zig symbol. + srcs = ["runfiles_init.c"], + # Retain the constructor even when its object is in a static archive. + # Mach-O adds an underscore to the exported C symbol. linkopts = select({ "@platforms//os:macos": ["-Wl,-u,__bazel_runfiles_init"], "//conditions:default": ["-Wl,-u,_bazel_runfiles_init"], diff --git a/bazel/bazel.zig b/bazel/bazel.zig index 2ae3324a..8af28b00 100644 --- a/bazel/bazel.zig +++ b/bazel/bazel.zig @@ -1,35 +1,26 @@ const std = @import("std"); -const builtin = @import("builtin"); const bazel_runfiles = @import("runfiles"); const stdx = @import("stdx"); var runfiles_global: bazel_runfiles.Runfiles = undefined; -const init_array_section = switch (builtin.object_format) { - .macho => "__DATA,__mod_init_func", - .elf => ".init_array", - else => "", -}; - -export const _bazel_runfiles_init align(@alignOf(usize)) linksection(init_array_section) = &struct { - fn call(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { - _ = argc; - _ = envp; - var threaded: std.Io.Threaded = .init_single_threaded; - runfiles_global = bazel_runfiles.Runfiles.create(.{ - .allocator = std.heap.c_allocator, - .io = threaded.io(), - .argv0 = std.mem.span(argv[0]), - .directory = if (std.c.getenv("RUNFILES_DIR")) |directory| std.mem.span(directory) else null, - .manifest = if (std.c.getenv("RUNFILES_MANIFEST_FILE")) |manifest| std.mem.span(manifest) else null, - }) catch { - std.debug.panic("Unable to find runfiles", .{}); - } orelse { - std.debug.panic("Unable to initialize runfiles", .{}); - }; - } -}.call; +export fn _bazel_runfiles_init_impl(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { + _ = argc; + _ = envp; + var threaded: std.Io.Threaded = .init_single_threaded; + runfiles_global = bazel_runfiles.Runfiles.create(.{ + .allocator = std.heap.c_allocator, + .io = threaded.io(), + .argv0 = std.mem.span(argv[0]), + .directory = if (std.c.getenv("RUNFILES_DIR")) |directory| std.mem.span(directory) else null, + .manifest = if (std.c.getenv("RUNFILES_MANIFEST_FILE")) |manifest| std.mem.span(manifest) else null, + }) catch { + std.debug.panic("Unable to find runfiles", .{}); + } orelse { + std.debug.panic("Unable to initialize runfiles", .{}); + }; +} pub fn runfiles(source_repository: []const u8) !bazel_runfiles.Runfiles.WithSourceRepo { return runfiles_global.withSourceRepo(source_repository); diff --git a/bazel/runfiles_init.c b/bazel/runfiles_init.c new file mode 100644 index 00000000..770a4b91 --- /dev/null +++ b/bazel/runfiles_init.c @@ -0,0 +1,15 @@ +extern void _bazel_runfiles_init_impl(int argc, char **argv, char **envp); + +// Preserve the existing startup ABI. Registration happens in C; +// runfiles discovery and initialization remain in Zig. +// glibc and Darwin's dyld pass argc/argv/envp to constructors; musl does not. +// The original Zig initializer already made this assumption to read argv[0]. +// +// Let C emit the platform's constructor registration. Zig's incremental ELF +// linker does not yet handle .init_array's section type, flags, and size correctly. +// TODO: Revert this workaround once Zig issue #37453 is fixed: +// https://codeberg.org/ziglang/zig/issues/37453 +__attribute__((constructor)) +void _bazel_runfiles_init(int argc, char **argv, char **envp) { + _bazel_runfiles_init_impl(argc, argv, envp); +} From 4c355c61b792017469e96904973bfdfeb0244e09 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 3 Nov 2026 18:11:15 +0100 Subject: [PATCH 837/881] zml/sharding: keep unconstrained graph results open (#1004) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ordinary graph results should have an unspecified layout until constrained or inferred by the partitioner. Previously, we could report replication simply because a tensor had no mesh, even when its stored partition specs were open. Other paths preserved selected operations’ sharding metadata, making the frontend’s interpretation depend on how a tensor was produced. This change gives those results a consistent open state while requiring concrete layouts at executable boundaries. Shardy explicitly defines the intended semantics: [“no sharding attribute on a tensor is equivalent to a fully open tensor sharding.”](https://openxla.org/shardy/sharding_representation#open) Open dimensions remain available for further partitioning; closed dimensions fix the specified layout. Replication is therefore a constraint, whereas an open layout lets the partitioner choose either sharding or replication: This distinction also applies to unused mesh axes. Marking an axis explicitly replicated prevents propagation from using it, even when a tensor dimension is open. For example, `[{?}, {}]` must leave unused axes available; adding replicated={"data", "model"} removes those choices. Keeping intermediates open lets the partitioner use constraints from producers and consumers, with operation-specific rules for transformations such as dot and reshape. --- examples/mnist/mnist_data_parallel.zig | 18 +++- zml/Compiler.zig | 39 +++++---- zml/Sharding.zig | 84 +++++++++++++++++-- zml/kernel.zig | 6 +- zml/kernels/flash_attention/cute/cute.zig | 2 +- zml/kernels/flash_attention/nki/nki.zig | 4 +- .../paged_attention/paged_attention.zig | 19 +++-- zml/ops.zig | 3 +- zml/ops/manual_computation.zig | 18 ++-- zml/shape.zig | 3 + zml/tensor.zig | 76 ++++++++++------- 11 files changed, 190 insertions(+), 82 deletions(-) diff --git a/examples/mnist/mnist_data_parallel.zig b/examples/mnist/mnist_data_parallel.zig index a539ae5a..a45c0534 100644 --- a/examples/mnist/mnist_data_parallel.zig +++ b/examples/mnist/mnist_data_parallel.zig @@ -2,9 +2,11 @@ //! Separated to kept `mnist.zig` minimal const std = @import("std"); + const zml = @import("zml"); const base = @import("mnist.zig"); + const log = std.log.scoped(.mnist); pub const std_options: std.Options = .{ @@ -53,14 +55,20 @@ const Mnist = struct { /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { var x = input.onMemory(.host_pinned).toMemory(.device).merge(.{ .d = .{ .x, .y } }).convert(.f32); + const layers: []const Layer = &.{ self.fc1, self.fc2 }; + const compiler = zml.Compiler.current(); + for (0.., layers) |i, layer| { compiler.pushLocationFmt(@src(), "layer[{d}]", .{i}); defer compiler.popLocation(); x = layer.forward(x); } - return x.argMax(.d).indices.convert(.u8).withPartitioning(input.shape()._sharding.mesh.?, .{ .b = .data }).toMemory(.host_pinned); + + return x.argMax(.d).indices.convert(.u8) + .withPartitioning(.dp_mp, .{ .b = .data, .d = .replicated }) + .toMemory(.host_pinned); } const Layer = struct { @@ -68,7 +76,7 @@ const Mnist = struct { bias: zml.Tensor, pub fn forward(self: Layer, input: zml.Tensor) zml.Tensor { - log.warn("layer(input={f}, w={f}, b={f}) -> {f}", .{ input, self.weight, self.bias, input.dot(self.weight, .d) }); + log.info("Layer(input={f}, w={f}, b={f}) -> {f}", .{ input, self.weight, self.bias, input.dot(self.weight, .d) }); const x = input.dot(self.weight, .d); return x.add(self.bias.broad(x.shape())).relu().rename(.{ .d_out = .d }); } @@ -100,18 +108,20 @@ pub fn main(init: std.process.Init) !void { &.{ .low_bandwidth, .high_bandwidth }, &.{ 2, -1 }, )); - log.info("topology used: {f}", .{dp_mp}); + log.info("{f}", .{dp_mp}); var profiler = try platform.profiler(allocator, io, .{ .session_id = "mnist" }); defer profiler.deinit(); var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{dp_mp}); defer store.deinit(); + const mnist_model: Mnist = .init(store.view()); - // // Compile model + // Compile model const bs: u32 = 4; const input: zml.Tensor = .withPartitioning(.init(.{ .b = bs, .x = 28, .y = 28 }, .u8), dp_mp, .{ .b = .data }); + var exe = exe: { log.info("Compiling model....", .{}); const start: std.Io.Timestamp = .now(io, .awake); diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 3daba2ec..5c6e3933 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -100,7 +100,7 @@ pub const Error = std.mem.Allocator.Error || mlir.Error || upb.SerializeError || pjrtx.Client.CompileError || - error{MissingDeviceInTile}; + error{ MissingDeviceInTile, OpenShardingAtBoundaries }; pub const Donation = union(enum) { implicit: u32, @@ -462,10 +462,7 @@ pub fn compileInternal( var compiler: Compiler = .init(allocator, io, platform, opts); defer compiler.deinit(); - var result = compiler.emitMlir(func_, args) catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - else => unreachable, - }; + var result = try compiler.emitMlir(func_, args); defer result.output_info.deinit(compiler.allocator); defer result.input_info.deinit(compiler.allocator); @@ -593,20 +590,21 @@ fn emitMlir(compiler: *Compiler, comptime func_: anytype, args: std.meta.ArgsTup var input_info = try createBlockArguments(compiler, fn_scope, &args); errdefer input_info.deinit(compiler.allocator); - var result = blk: { + var output_info = blk: { compiler.activate(); defer compiler.deactivate(); - break :blk @call(.auto, func_, args); - }; - var output_info = try collectOutputInfo(compiler, fn_scope, &result); + var result = @call(.auto, func_, args); + // Output collection can emit replication constraints, so it is part of tracing. + break :blk try collectOutputInfo(compiler, fn_scope, &result); + }; errdefer output_info.deinit(compiler.allocator); // Finalize in a separate function that doesn't depend on `func` type. return try finalizeMlirFunc(compiler, fn_scope, input_info, output_info); } -fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{OutOfMemory}!std.MultiArrayList(TensorInfo) { +fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{ OutOfMemory, OpenShardingAtBoundaries }!std.MultiArrayList(TensorInfo) { const CreateBlockArgumentsCb = struct { compiler: *Compiler, scope: *Scope, @@ -614,9 +612,13 @@ fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{Ou current_argument_id: u32 = 0, infos: std.MultiArrayList(TensorInfo) = .empty, - fn cb(ctx: *@This(), tensor: *const Tensor) error{OutOfMemory}!void { + fn cb(ctx: *@This(), tensor: *const Tensor) error{ OutOfMemory, OpenShardingAtBoundaries }!void { // Declare the program argument as a packed u8 const og_shape = tensor._shape; + if (og_shape.containsPartitionSpec(.open)) { + log.err("Executable input {d} has open sharding: {f}. Specify a logical mesh axis or .replicated for every dimension.", .{ ctx.current_argument_id, og_shape }); + return error.OpenShardingAtBoundaries; + } const packed_shape = og_shape.packedShape(); const mlir_type = mlirx.Type.rankedTensor(ctx.compiler.mlir_ctx, packed_shape); var value = ctx.scope.block.addArgument(mlir_type, .unknown(ctx.compiler.mlir_ctx)); @@ -650,7 +652,7 @@ fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{Ou return context.infos; } -fn collectOutputInfo(compiler: *Compiler, scope: *Scope, v: anytype) error{OutOfMemory}!std.MultiArrayList(TensorInfo) { +fn collectOutputInfo(compiler: *Compiler, scope: *Scope, v: anytype) error{ OutOfMemory, OpenShardingAtBoundaries }!std.MultiArrayList(TensorInfo) { const CollectOutputInfoCb = struct { compiler: *Compiler, scope: *Scope, @@ -658,9 +660,16 @@ fn collectOutputInfo(compiler: *Compiler, scope: *Scope, v: anytype) error{OutOf infos: std.MultiArrayList(TensorInfo) = .empty, fn cb(ctx: *@This(), tensor: *const Tensor) !void { - const og_shape = tensor.shape(); - var value = ctx.scope.id_to_argument.get(tensor.id) orelse - tensor._value orelse + // Outputs with no declared sharding default to replication. Constrain the + // returned value so the partitioner produces a replicated output buffer. + const output = if (tensor.shape()._sharding.mesh != null or tensor.shape().isFullyReplicated()) tensor.* else tensor.replicate(); + const og_shape = output.shape(); + if (og_shape.containsPartitionSpec(.open)) { + log.err("Executable output {d} has open sharding: {f}. Specify a logical mesh axis or .replicated for every dimension.", .{ ctx.infos.len, og_shape }); + return error.OpenShardingAtBoundaries; + } + var value = ctx.scope.id_to_argument.get(output.id) orelse + output._value orelse @panic("no value found for output tensor"); if (tensor.dtype().bitSizeOf() < 8) { diff --git a/zml/Sharding.zig b/zml/Sharding.zig index 2637f209..c03dd07e 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -21,10 +21,22 @@ mesh: ?*const Mesh, partition: Partitioning, pub const MAX_MESH_RANK = 4; + +/// Full replication across participating devices. +/// For buffers, each device holds a complete copy. Applied as a graph constraint, +/// this closes every dimension to partitioning. `Shape.init` uses this sharding. +/// Unconstrained graph results use `open` instead. pub const replicated: Sharding = .{ .mesh = null, .partition = .replicated(Shape.MAX_RANK) }; +/// Unspecified sharding for graph results. During compilation, the partitioner +/// may infer sharding from surrounding constraints or leave the value replicated. +/// Executable inputs require closed sharding. An executable output with this +/// unspecified sharding defaults to replication. An explicitly declared output +/// sharding containing open dimensions is rejected. +pub const open: Sharding = .{ .mesh = null, .partition = .open(Shape.MAX_RANK) }; + pub fn get(sharding: Sharding, ax: usize) PartitionSpec { - return if (sharding.mesh == null) .replicated else sharding.partition.get(ax); + return sharding.partition.get(ax); } pub fn set(self: Sharding, ax: usize, spec: PartitionSpec) Sharding { @@ -52,7 +64,6 @@ pub fn toArray(self: Sharding) [Shape.MAX_RANK]PartitionSpec { } pub fn isFullyReplicated(self: Sharding) bool { - if (self.mesh == null) return true; for (self.partition.toArray()) |spec| { if (spec != .replicated and spec != .out_of_bound) return false; } @@ -98,8 +109,18 @@ test isCompatible { } } +/// Formats the declared sharding, not the partitioner's inferred sharding. +/// Without a mesh, prints `{replicated}` for full replication or `{open}` for +/// unspecified graph sharding. +/// +/// With a mesh, entries follow tensor dimension order, e.g. `{data,replicated,?}@dp`: +/// - `data`: split this dimension over the logical mesh axis named `data`. +/// - `replicated`: keep this dimension whole on each shard. +/// - `?`: leave this dimension to partitioner inference during compilation. +/// - `@dp`: the mesh name. +/// Trailing `.out_of_bound` entries are omitted. Any remaining ones print as `!`. pub fn format(self: Sharding, writer: *std.Io.Writer) !void { - const mesh = self.mesh orelse return writer.writeAll("{replicated}"); + const mesh = self.mesh orelse return writer.writeAll(if (self.isFullyReplicated()) "{replicated}" else "{open}"); const specs = self.partition.toArray(); var rank = specs.len; while (rank > 0 and specs[rank - 1] == .out_of_bound) rank -= 1; @@ -117,8 +138,10 @@ pub fn format(self: Sharding, writer: *std.Io.Writer) !void { try writer.print("}}@{s}", .{mesh.name}); } -/// Describes how a given Shape axis behaves inside a mesh. -/// Is it replicated ? sharded along a specific logical axis ? or open to replication ? +/// Describes how one tensor dimension is distributed across a logical mesh. +/// A mesh axis splits the dimension. `.replicated` keeps it whole on each shard. +/// In a graph constraint, `.open` lets the partitioner choose either behavior. +/// Executable buffers require closed sharding specs: a mesh axis or `.replicated`. pub const PartitionSpec = enum(u4) { mesh_axis_0 = 0, mesh_axis_1 = 1, @@ -1676,6 +1699,50 @@ pub const Mesh = struct { unreachable; } + /// Converts `shape`'s declared partitioning into a `#sdy.sharding` attribute + /// referencing this mesh. Entries in `[...]` follow tensor dimension order. + /// Mesh axis indices in the specs resolve to this mesh's logical axis names. + /// The tensor rank and logical mesh rank are independent. + /// + /// Each dimension spec is encoded as follows: + /// - `PartitionSpec.sharded(i)` -> `{"axis_name"}`: split the dimension over + /// logical mesh axis `i`. The dimension is closed to further partitioning. + /// - `.replicated` -> `{}`: keep the dimension whole on each shard. It is + /// closed, so the partitioner cannot assign a mesh axis to it. + /// - `.open` -> `{?}`: let the partitioner infer sharding from the graph. + /// It may acquire available mesh axes or remain replicated. + /// Closed therefore means no additional mesh axes, not necessarily replicated. + /// Every dimension within `shape.rank()` must have a spec other than `.out_of_bound`. + /// + /// Dimension replication (`{}`) applies to one tensor dimension. The optional + /// tensor-wide `replicated={...}` instead names mesh axes: each listed axis + /// must carry identical tensor data across its coordinates and cannot be used + /// to split any dimension, including an open one. + /// + /// For example, take a mesh `@dp_mp` with `data=2`, `model=2`, no extra replicas, + /// and a shape whose dimensions are ordered `batch`, `hidden`: + /// + /// ``` + /// Partitioning: .{ .batch = .data, .hidden = .open } + /// #sdy.sharding<@dp_mp, [{"data"}, {?}]> + /// ``` + /// `batch` is split over `data`. The partitioner may assign `model` to `hidden`. + /// The unused `model` axis stays available because an open dimension exists. + /// + /// ``` + /// Partitioning: .{ .batch = .data, .hidden = .replicated } + /// #sdy.sharding<@dp_mp, [{"data"}, {}], replicated={"model"}> + /// ``` + /// Both dimensions are closed. Unused logical axes are explicitly replicated, + /// so devices differing only in their `model` coordinate hold the same shard. + /// + /// ``` + /// Partitioning: .{ .batch = .open, .hidden = .replicated } + /// #sdy.sharding<@dp_mp, [{?}, {}]> + /// ``` + /// Only `batch` may acquire axes, and both `data` and `model` are available. + /// Adding `replicated={"data", "model"}` would prevent either axis from being + /// inferred for `batch`, despite its `{?}` entry. pub fn sdyShardingAttrForShape( data: *const Mesh, parent_allocator: std.mem.Allocator, @@ -1687,6 +1754,7 @@ pub const Mesh = struct { const allocator = arena.allocator(); const dimensions = try allocator.alloc(*const shardy.DimensionShardingAttr, shape.rank()); var used: [MAX_MESH_RANK]bool = @splat(false); + const has_open_dimension = shape.containsPartitionSpec(.open); for (0.., dimensions) |ax, *d| { d.* = switch (shape.partition(ax)) { .out_of_bound => @panic("undefined"), @@ -1704,7 +1772,9 @@ pub const Mesh = struct { var tile_count: usize = 1; for (data.logical.axes.slice(), 0..) |axis, i| { tile_count *= data.grid.axis_sizes[i]; - if (!used[i]) replicated_axes.appendAssumeCapacity(.named(ctx, std.mem.span(axis))); + // Open dimensions can acquire unused logical axes during inference. + // Only reserve those axes for replication when every dimension is closed. + if (!has_open_dimension and !used[i]) replicated_axes.appendAssumeCapacity(.named(ctx, std.mem.span(axis))); } if (tile_count < data.grid.device_count) { replicated_axes.appendAssumeCapacity(.named(ctx, data.sdyReplicaAxis())); @@ -2741,7 +2811,7 @@ test "mesh: open and replicated dimension mix" { .expected_num_replicas = 1, .mesh = &mesh, .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(&mesh, .{ .batch = .open, .model = .replicated }), - .expected_sdy = "#sdy.sharding<@mix_mesh, [{?}, {}], replicated={\"batch\", \"model\"}>", + .expected_sdy = "#sdy.sharding<@mix_mesh, [{?}, {}]>", .expected_shards = &.{ .{ .device_id = 0, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, .{ .device_id = 1, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, diff --git a/zml/kernel.zig b/zml/kernel.zig index 36c9d8d3..94d6046a 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -1529,8 +1529,8 @@ test "tcl multi-chip kernels run on two furiosa chips" { const allocator = std.testing.allocator; const platform = zml.testing.env(); if (platform.target != .furiosa or platform.devices.len != 2) return error.SkipZigTest; - const model = platform.meshes.get("model").?; - const x_shape = Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(model, .{ .c = .model }); + const tp = zml.testing.tpMesh(); + const x_shape = Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(tp, .{ .c = .model }); const Mod = struct { pub fn forward(x: Tensor) [2]Tensor { const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), Shape.init(.{ .c = 2, .a = 1024 }, .f32) }); @@ -1538,7 +1538,7 @@ test "tcl multi-chip kernels run on two furiosa chips" { } }; const x: Tensor = .fromShape(x_shape); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{ .meshes = &.{model} }); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{ .meshes = &.{tp} }); defer exe.deinit(); var hx: [2 * 1024]f32 = undefined; for (&hx, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 13)) * 0.125 - 0.5; diff --git a/zml/kernels/flash_attention/cute/cute.zig b/zml/kernels/flash_attention/cute/cute.zig index 2958888d..1136dc9a 100644 --- a/zml/kernels/flash_attention/cute/cute.zig +++ b/zml/kernels/flash_attention/cute/cute.zig @@ -41,7 +41,7 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T token_index.convert(.i32).broad(zml.Shape.init(.{ .b = q.dim(.b) }, .i32)) else token_index.convert(.i32); - return zml.ops.manualComputation(Context.body, Context{ .q = q_sharded, .k = k_sharded, .v = v_sharded, .positions = positions }, q_sharded.shape()); + return zml.ops.manualComputation(Context.body, Context{ .q = q_sharded, .k = k_sharded, .v = v_sharded, .positions = positions }, q.shape().withPartitioning(mesh, .{ .h = .model })); } const Context = struct { diff --git a/zml/kernels/flash_attention/nki/nki.zig b/zml/kernels/flash_attention/nki/nki.zig index 282da132..905f3aa0 100644 --- a/zml/kernels/flash_attention/nki/nki.zig +++ b/zml/kernels/flash_attention/nki/nki.zig @@ -82,7 +82,7 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T .token_index = token_index_2d, .parameters = parameters, }, - q_sharded.shape(), + q.shape().withPartitioning(tp, .{ .h = .model }), ).convert(q.dtype()); } @@ -108,6 +108,6 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T } }).body, .{ .q = q_sharded, .k = k_sharded, .v = v_sharded, .parameters = parameters }, - q_sharded.shape(), + q.shape().withPartitioning(tp, .{ .h = .model }), ).convert(q.dtype()); } diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index 99065140..65cf6cea 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -6,9 +6,9 @@ const zml = @import("../../zml.zig"); const cute = @import("cute/cute.zig"); const metal = @import("metal/metal.zig"); const tpu = @import("mosaic_tpu/mosaic_tpu.zig"); -const triton = @import("triton/triton.zig"); -pub const tcl = @import("tcl/tcl.zig"); const stablehlo = @import("stablehlo/stablehlo.zig"); +pub const tcl = @import("tcl/tcl.zig"); +const triton = @import("triton/triton.zig"); const PagedAttention = @This(); @@ -335,21 +335,22 @@ test pagedAttention { const batch_size = active_batch_size + 1; const query_token_count = prefill_token_count + num_decode; const dt: zml.DataType = .bf16; - const model_meshe = platform.meshes.get("model").?; + const tp = zml.testing.tpMesh(); + const num_kv_heads = tp.numPartitionsForLogicalAxis(.model); const partition = .{ .hkv = .model }; const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache } = .{ - .q = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hg = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), - .k = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), - .v = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), + .q = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = num_kv_heads, .hg = 4, .hd = 32 }, dt)).withPartitioning(tp, partition), + .k = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), + .v = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), .kv_cache = .{ .split = .{ - .k = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), - .v = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = 4, .hd = 32 }, dt)).withPartitioning(model_meshe, partition), + .k = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), + .v = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), }, }, }; - const meshes: []const *const zml.Mesh = &.{ platform.replicated_mesh, platform.meshes.get("model").? }; + const meshes: []const *const zml.Mesh = &.{ platform.replicated_mesh, tp }; const rng_q = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.q.shape(), .{} }, .{ .meshes = meshes }); defer rng_q.deinit(); const rng_k = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.k.shape(), .{} }, .{ .meshes = meshes }); diff --git a/zml/ops.zig b/zml/ops.zig index b1a26d23..e598d99c 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -830,10 +830,9 @@ fn fromMlirOperationWithTags(op: *const mlir.Operation, base: anytype) @TypeOf(b fn cb(inner_ctx: *LocalContext, tensor: *Tensor) void { var new = Tensor.fromMlirValue(inner_ctx.op.result(inner_ctx.index)); stdx.debug.internalAssert(new.rank() == tensor.rank(), "expected operand result to have rank {} but got {f}", .{ tensor.rank(), new }); - // copy tags and mesh info over + // Copy tags; the partitioner infers result sharding. // some ops can change dims eg reduceWindow, so we trust mlir here. new._shape._tags = tensor._shape._tags; - new._shape._sharding = tensor._shape._sharding; tensor.* = new; inner_ctx.index += 1; } diff --git a/zml/ops/manual_computation.zig b/zml/ops/manual_computation.zig index c9596f98..170f96b5 100644 --- a/zml/ops/manual_computation.zig +++ b/zml/ops/manual_computation.zig @@ -6,18 +6,19 @@ const shardy = @import("mlir/dialects/shardy"); const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); -const zml = struct { - // Nested to disambiguate from stdx.meta - const meta = @import("../meta.zig"); - const testing = @import("../testing.zig"); -}; - const Compiler = @import("../Compiler.zig"); const DataType = @import("../dtype.zig").DataType; const mlirx = @import("../mlirx.zig"); const Shape = @import("../shape.zig").Shape; const Sharding = @import("../Sharding.zig"); const Tensor = @import("../tensor.zig").Tensor; + +const zml = struct { + // Nested to disambiguate from stdx.meta + const meta = @import("../meta.zig"); + const testing = @import("../testing.zig"); +}; + const log = std.log.scoped(.@"zml/Compiler"); /// ZML operator describes computation across several devices, @@ -214,9 +215,10 @@ noinline fn manualComputationInternal( }, }; - // reuse local_outputs buffer to store the final result. + // Returned tensors describe the operation's global output shapes and sharding. for (0.., local_outputs, global_output_shapes) |o, output_tensor_ptr, output_shape| { - output_tensor_ptr.* = Tensor._resultPropagateSharding(output_shape, op.result(o)); + output_tensor_ptr.* = Tensor._result(output_shape, op.result(o)); + output_tensor_ptr._shape = output_shape; } return local_outputs; } diff --git a/zml/shape.zig b/zml/shape.zig index b0863367..f866d2e2 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -827,6 +827,9 @@ pub const Shape = struct { .grid = undefined, }; + /// Returns shape metadata with the requested sharding on `mesh`. + /// Omitted dimensions in a mapping are replicated, including during compilation. + /// Use `Tensor.withPartitioning` to emit a constraint on a graph value. pub fn withPartitioning(self: Shape, mesh: *const Sharding.Mesh, partitioning: anytype) Shape { if (@TypeOf(partitioning) != Sharding.Partitioning) { return self.withPartitioning(mesh, Sharding.Partitioning.parse(self.tags(), mesh, .replicated, partitioning)); diff --git a/zml/tensor.zig b/zml/tensor.zig index 8c148867..59d32cbd 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -75,14 +75,11 @@ pub const Tensor = struct { return self._shape.byteSize(); } - /// Internal use - /// - /// Creates a tensor from a Shape and an mlir.Value. - /// Note: sharding information is discarded, the output tensor is considered open for replication. - /// Use _resultPropagateSharding when the shape sharding should be propagated. + /// Internal: creates a graph tensor with dimensions, dtype and tags from `sh`, + /// and open sharding for the partitioner to infer from graph constraints. pub fn _result(sh: Shape, val: *const mlir.Value) Tensor { var res: Tensor = .{ ._shape = sh, ._value = val, .id = nextTensorId() }; - res._shape._sharding.partition = .open(sh.rank()); + res._shape._sharding = .open; if (builtin.mode == .debug) { // Check that the MLIR value actually have the same shape. @@ -95,16 +92,16 @@ pub const Tensor = struct { return res; } + /// Internal: creates a constructor result and honors `sh`'s partitioning when + /// it specifies a mesh. pub fn _resultPropagateSharding(sh: Shape, val: *const mlir.Value) Tensor { - var res = _result(sh, val); - res._shape._sharding = sh._sharding; - return res; + const result = _result(sh, val); + const mesh = sh._sharding.mesh orelse return result; + return result.withPartitioning(mesh, sh._sharding.partition); } - /// Creates a Tensor from a mlir.Value - /// - /// The shape is derived from the type of the mlir.Value. - /// mesh is `unknown` + /// Wraps an MLIR tensor value, deriving dimensions and dtype from its type. + /// Dimension tags are unknown and sharding is open for partitioner inference. pub fn fromMlirValue(val: *const mlir.Value) Tensor { const ctx = Compiler.current(); const ranked_tensor = val.type_().isA(mlir.RankedTensorType).?; @@ -116,7 +113,7 @@ pub const Tensor = struct { ._dtype = ctx.dtype(ranked_tensor.elementType()), ._dims = .empty, ._tags = .{ .buffer = @splat(Shape.TagUnknown), .len = n }, - ._sharding = .replicated, + ._sharding = .open, }; for (0..n) |i| { sh._dims.appendAssumeCapacity(ranked_tensor.dimension(i)); @@ -166,12 +163,22 @@ pub const Tensor = struct { return res; } - /// Specify the mesh of the input tensor. - /// * mesh: *const zml.Mesh, but during compilation the `.meshe_name` syntax can used to get - /// a known mesh from the compilation options. - /// * partition spec: a struct where the field names match the axis of the given mesh + /// Declares this tensor's sharding on `mesh_`. + /// Pass a `*const Mesh`, or a mesh name such as `.tp` during compilation. + /// `partitioning` accepts a `Sharding.Partitioning` or a mapping from dimension + /// tags to logical mesh axes, `.replicated`, or `.open`. /// - /// eg `x.withPartitioning(tp, .{ .h = .model }))` or `x.withPartitioning(.tp, .{ .h = .model }))` + /// During compilation, emits a graph constraint. For example, + /// `x.withPartitioning(.tp, .{ .h = .model })` splits dimension `h` over `model` + /// and leaves omitted dimensions open for inference. Use `.replicated` to + /// keep a dimension whole on each shard. + /// + /// Outside compilation, updates the tensor's shape to describe an executable + /// input's sharding. Omitted dimensions are replicated and no graph constraint is + /// emitted. An explicit `Sharding.Partitioning` uses its supplied specs in both contexts. + /// + /// Executable inputs and explicitly annotated outputs must have no open + /// dimensions. An output with no declared sharding defaults to replication. pub fn withPartitioning(self: Tensor, mesh_: anytype, partitioning: anytype) Tensor { if (@TypeOf(mesh_) == @EnumLiteral()) { const compiler = Compiler.currentOrNull() orelse @panic("Out side of compilation, withPartitioning expects an explicit *const zml.Mesh object as input"); @@ -179,7 +186,8 @@ pub const Tensor = struct { } if (@TypeOf(partitioning) != Sharding.Partitioning) { - const parsed = Sharding.Partitioning.parse(self._shape.tags(), mesh_, .open, partitioning); + const default: Sharding.PartitionSpec = if (Compiler.currentOrNull() == null) .replicated else .open; + const parsed = Sharding.Partitioning.parse(self._shape.tags(), mesh_, default, partitioning); return self.withPartitioning(mesh_, parsed); } @@ -222,11 +230,14 @@ pub const Tensor = struct { }, }; - return _resultPropagateSharding(partitioned_shape, op_result); + var res = _result(partitioned_shape, op_result); + res._shape = partitioned_shape; + return res; } - /// Force the input tensor to be fully replicated. - /// This can be detrimental for performance. + /// Emits a graph constraint requiring a complete copy on every device. + /// Requires active compilation. For a replicated input or buffer description, + /// use the default sharding from `Tensor.init` or `Shape.init`. pub fn replicate(self: Tensor) Tensor { const compiler = Compiler.current(); const partitioning: Sharding.Partitioning = .repeat(.replicated, self.rank()); @@ -267,7 +278,7 @@ pub const Tensor = struct { try std.testing.expectEqual(Sharding.PartitionSpec.replicated, x2_replicated.shape().partition(.h)); } - /// Copy the given tensor to the specified memory. + /// Copy the given tensor to the specified memory, preserving its sharding. pub fn toMemory(self: Tensor, kind: Memory.Kind) Tensor { const ctx = Compiler.current(); switch (ctx.platform.target) { @@ -298,7 +309,8 @@ pub const Tensor = struct { ctx.location, ).appendTo(currentBlock()); - const res = _result(self._shape, op.result(0)); + var res = _result(self._shape, op.result(0)); + res._shape._sharding = self._shape._sharding; ctx.currentScope().id_to_memory.putNoClobber(ctx.currentScope().arena.allocator(), res.id, kind) catch @panic("OOM"); return res; } @@ -845,7 +857,7 @@ pub const Tensor = struct { fn update(self: Rng, new_state: *const mlir.Value) Rng { return .{ - ._state = _resultPropagateSharding(self._state._shape, new_state).reuseBuffer(self._state), + ._state = _result(self._state._shape, new_state).reuseBuffer(self._state), .algorithm = self.algorithm, }; } @@ -886,7 +898,8 @@ pub const Tensor = struct { floats = floats.mul(scalar(opts.max - opts.min, dt)).addConstant(opts.min); // Convert back to integer if needed. - return .{ rng, _resultPropagateSharding(shape_, floats.convert(shape_.dtype()).value()) }; + const result = floats.convert(shape_.dtype()); + return .{ rng, _resultPropagateSharding(shape_, result.value()) }; } test uniform { @@ -2705,7 +2718,7 @@ pub const Tensor = struct { mlirx.Type.rankedTensor(mlirCtx(), res_shape), currentLoc(), ).appendTo(currentBlock()); - return _result(res_shape, op.result(0)); + return _resultPropagateSharding(res_shape, op.result(0)); } pub const LinspaceArgs = struct { @@ -2780,7 +2793,8 @@ pub const Tensor = struct { } pub fn zeroes(sh: Shape) Tensor { - return Tensor.constant(sh.dtype().zero()).broad(sh); + const result = Tensor.constant(sh.dtype().zero()).broad(sh); + return _resultPropagateSharding(sh, result.value()); } pub fn uninitialized(sh: Shape) Tensor { @@ -2795,7 +2809,7 @@ pub const Tensor = struct { .appendTo(currentBlock()); const tensor = stablehlo.unpin(ctx.mlir_ctx, buffer.result(0), ctx.location) .appendTo(currentBlock()); - return _result(sh, tensor.result(0)); + return _resultPropagateSharding(sh, tensor.result(0)); } /// Embeds a buffer with concrete values into an Mlir program. @@ -2803,7 +2817,7 @@ pub const Tensor = struct { const elem_type = mlirx.Type.fromDType(mlirCtx(), sh.dtype()); //const elem_type = mlirx.denseElementAttrType(val.dtype()) orelse std.debug.panic("constantTensor expects a dtype that can be serialized to MLIR, like f32 or i32, got {f}", .{val.shape()}); const constant_op = stablehlo.constant(mlirCtx(), sh.dims(), elem_type, bytes_, currentLoc()).appendTo(currentBlock()); - return _result(sh, constant_op.result(0)); + return _resultPropagateSharding(sh, constant_op.result(0)); } /// Returns a Tensor containing the result of the outer product between the input Tensors. From 6a1698f27066bec5e00a13cbd88f72ffcc88a844 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 3 Nov 2026 18:22:59 +0100 Subject: [PATCH 838/881] zml/ops: add manualComputation2 with explicit partitioning (#1013) Accept a logical mesh and typed input/output partition specs at manual computation boundaries. Will replace `zml.ops.manualComputation` in a future PR. --- examples/sharding/main.zig | 19 +- zml/BUILD.bazel | 1 + zml/Sharding.zig | 50 ++-- zml/meta.zig | 18 +- zml/ops.zig | 5 + zml/ops/manual_computation2.zig | 465 ++++++++++++++++++++++++++++++++ zml/tensor.zig | 4 +- 7 files changed, 538 insertions(+), 24 deletions(-) create mode 100644 zml/ops/manual_computation2.zig diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index cc46a1bd..7e519bf0 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -30,15 +30,19 @@ const DemoModel = struct { } pub fn forward(self: DemoModel, input: zml.Tensor) zml.Tensor { - const x = input.convert(.f32); + const compiler = zml.Compiler.current(); + const x = input.convert(.f32); var y = x.dot(self.w, .feature); y = y.add(self.b.broad(y.shape())); y = y.withPartitioning(.dp_mp, .{ .batch = .data, .hidden = .model }); y.print("dense_out"); - const gate = y.scale(0.01).sigmoid().withPartitioning(.dp_mp, .{ .batch = .data, .hidden = .model }); - return zml.ops.manualComputation( + const gate = y.scale(0.01).sigmoid(); + + const spec: zml.Sharding.Partitioning.Logical = .parse(.{ .batch = .data, .hidden = .model }); + + return zml.ops.manualComputation2( (struct { y: zml.Tensor, gate: zml.Tensor, @@ -50,6 +54,11 @@ const DemoModel = struct { }).body, .{ .y = y, .gate = gate }, y.shape(), + compiler.mesh(.dp_mp), + .{ + .in_specs = .{ .y = spec, .gate = spec }, + .out_specs = spec, + }, ); } }; @@ -60,8 +69,8 @@ fn usage() void { \\ \\Simple sharding demo: \\ - explicit partitioning on data/model axes - \\ - shard-local tensor print - \\ - manualComputation with multiple inputs and one output + \\ - tensor print + \\ - manualComputation2 with multiple inputs and one output \\ \\Mesh modes: \\ - auto: physical_mesh=auto, inferred cpu device_count=2 diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 8188c743..dbb2ad33 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -98,6 +98,7 @@ zig_library( "nn.zig", "ops.zig", "ops/manual_computation.zig", + "ops/manual_computation2.zig", "pjrtx.zig", "platform.zig", "profiling/profiler.zig", diff --git a/zml/Sharding.zig b/zml/Sharding.zig index c03dd07e..2f887a8a 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -437,18 +437,22 @@ pub const Partitioning = packed struct { const logical = if (@TypeOf(partitioning) == Logical) partitioning else - // capture all the comptime logic in the Logical struct. + // capture all the comptime logic in Logical. comptime Logical.parse(partitioning); return logical.applyUseAll(tags, mesh, default_value); } - pub const Logical = struct { - src: Shape.TagsArray, - tgt: Shape.TagsArray, + pub const Logical = union(enum) { + /// Replicates every dimension, including when used as a graph constraint. + replicated, + mapping: struct { + src: Shape.TagsArray, + tgt: Shape.TagsArray, + }, - const replicated: Shape.Tag = Shape.toTag(.replicated); - const open: Shape.Tag = Shape.toTag(.open); + const replicated_tag: Shape.Tag = Shape.toTag(.replicated); + const open_tag: Shape.Tag = Shape.toTag(.open); pub fn parse(v: anytype) Logical { const V = @TypeOf(v); @@ -457,7 +461,7 @@ pub const Partitioning = packed struct { var tgt_: Shape.TagsArray = .empty; if (comptime stdx.meta.isStruct(V)) { - const field_names = std.meta.fieldNames(V); + const field_names = comptime std.meta.fieldNames(V); stdx.debug.assertComptime(field_names.len <= @TypeOf(src_).max_len, "Too many tags in struct {} ({d})", .{ V, field_names.len }); inline for (field_names) |field_name| { const fv = @field(v, field_name); @@ -467,7 +471,7 @@ pub const Partitioning = packed struct { src_.appendAssumeCapacity(field_name); } } - return .{ .src = src_, .tgt = tgt_ }; + return .{ .mapping = .{ .src = src_, .tgt = tgt_ } }; } stdx.debug.compileError("Sharding.Partitioning.parse expects a struct of enum literals, eg {{ .b = .data, .d = .model }}, got: {any}", .{V}); @@ -476,23 +480,39 @@ pub const Partitioning = packed struct { test "Logical.parse" { const logical = comptime Logical.parse(.{ .a = .x, .b = .y }); - try std.testing.expectEqualSlices(Shape.Tag, &.{ "a".ptr, "b".ptr }, logical.src.constSlice()); - try std.testing.expectEqualSlices(Shape.Tag, &.{ "x".ptr, "y".ptr }, logical.tgt.constSlice()); + try std.testing.expectEqualSlices(Shape.Tag, &.{ "a".ptr, "b".ptr }, logical.mapping.src.constSlice()); + try std.testing.expectEqualSlices(Shape.Tag, &.{ "x".ptr, "y".ptr }, logical.mapping.tgt.constSlice()); + } + + test "Logical.replicated" { + const mesh: Mesh = .initTest(.logical, .{.model}); + const tags: []const Shape.Tag = &.{ "a", "b" }; + const logical: Logical = .replicated; + for ([_]PartitionSpec{ .open, .replicated }) |default| { + try std.testing.expectEqual(Partitioning.replicated(tags.len), logical.applyUseAll(tags, &mesh, default)); + } + const unspecified = Logical.parse(.{}); + try std.testing.expectEqual(Partitioning.open(tags.len), unspecified.applyUseAll(tags, &mesh, .open)); } pub fn apply(logical: *const Logical, tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec) struct { Partitioning, ?u8 } { + const mapping = switch (logical.*) { + .replicated => return .{ .replicated(tags.len), null }, + .mapping => |*mapping_| mapping_, + }; + var partition_: Partitioning = .repeat(default_value, tags.len); var missing_src_axis: ?u8 = null; - for (0.., logical.src.slice(), logical.tgt.slice()) |i, shape_tag, mesh_tag| { + for (0.., mapping.src.slice(), mapping.tgt.slice()) |i, shape_tag, mesh_tag| { const shape_ax = Shape.axisFromTagMaybe(tags, shape_tag) orelse { missing_src_axis = @intCast(i); continue; }; - const spec: PartitionSpec = if (mesh_tag == Logical.replicated) + const spec: PartitionSpec = if (mesh_tag == Logical.replicated_tag) .replicated - else if (mesh_tag == Logical.open) + else if (mesh_tag == Logical.open_tag) .open else spec: { const mesh_ax = mesh.resolveLogicalAxis(mesh_tag) orelse { @@ -509,7 +529,7 @@ pub const Partitioning = packed struct { const p, const missing_shape_axis = logical.apply(tags, mesh, default_value); if (missing_shape_axis) |ax| { - std.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), logical.src.get(ax) }); + std.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), logical.mapping.src.get(ax) }); } return p; } @@ -2068,7 +2088,7 @@ pub const DeviceGrid = struct { for (strategy.foldSources(part.axis) orelse &.{part.axis}) |source| { if (used.contains(source)) continue; used.insert(source); - const info = mesh_.physical.axisInfo(source) orelse continue; + const info = mesh_.physical.axisInfo(source) orelse return error.InvalidPhysicalAxis; try grid.appendAxis(mesh_, logical_axis, source, 1, @intCast(info.size)); } } diff --git a/zml/meta.zig b/zml/meta.zig index 1eefea11..84fb9d36 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -143,7 +143,7 @@ test MapType { /// The `to` parameter must be passed with mutable pointer, and tensor data need to be mutable if callback needs it. /// `mapAlloc` tries as much as possible to respect the conversions made by Zig itself. /// For example it can convert from a comptime array to a runtime slice. -/// `mapAlloc` can allocate new slices to write the result if the result struct requires it. +/// `mapAlloc` allocates slices and single-item pointers needed to hold mapped values. /// The caller is owning said allocations, using an `ArenaAllocator` might help tracking them. /// /// Note: to avoid infinite loop, mapAlloc doesn't look for `From` fields inside `To` struct. @@ -180,6 +180,11 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam return; } + if (FromStruct == ToStruct and !comptime Contains(FromStruct, From)) { + to.* = from; + return; + } + if (@sizeOf(ToStruct) == 0) return; switch (type_info_to) { @@ -226,7 +231,11 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam } to.* = items; }, - else => try mapAlloc(cb, allocator, ctx, from.*, to.*), + else => { + const item = try allocator.create(ptr_info.child); + try mapAlloc(cb, allocator, ctx, from.*, item); + to.* = item; + }, }, .slice => { const items = try allocator.alloc(@typeInfo(ToStruct).pointer.child, from.len); @@ -267,6 +276,7 @@ test mapAlloc { field: A, array: [2]A, slice: []const A, + pointer: *const A, other: u8, tag: [*:0]const u8, // We want to allow conversion from comptime to runtime, because Zig type inference works like this. @@ -278,6 +288,7 @@ test mapAlloc { field: B, array: [2]B, slice: []const B, + pointer: *const B, other: u8, tag: [*:0]const u8, static_val: u8, @@ -291,6 +302,7 @@ test mapAlloc { .other = 7, .tag = "feature", .slice = &.{ .{ .a = 9 }, .{ .a = 10 } }, + .pointer = &.{ .a = 13 }, .field_with_empty = .{ .{ .a = 9 }, .{} }, }; var bb: BB = undefined; @@ -298,6 +310,7 @@ test mapAlloc { try mapAlloc(A.convert, testing.allocator, {}, aa, &bb); defer testing.allocator.free(bb.slice); defer testing.allocator.free(bb.static_slice); + defer testing.allocator.destroy(bb.pointer); try testing.expectEqual(4, bb.field.b); try testing.expectEqual(5, bb.array[0].b); @@ -309,6 +322,7 @@ test mapAlloc { try testing.expectEqual(10, bb.slice[1].b); try testing.expectEqual(11, bb.static_slice[0].b); try testing.expectEqual(12, bb.static_slice[1].b); + try testing.expectEqual(13, bb.pointer.b); } /// Visit a given type `T` and: diff --git a/zml/ops.zig b/zml/ops.zig index e598d99c..19d15ec3 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -11,8 +11,13 @@ const stdx = @import("stdx"); const manual_computation = @import("ops/manual_computation.zig"); pub const manualComputation = manual_computation.manualComputation; +const manual_computation2 = @import("ops/manual_computation2.zig"); +pub const manualComputation2 = manual_computation2.manualComputation2; +pub const ManualPartitioning = manual_computation2.ManualPartitioning; + test { _ = manual_computation; + _ = manual_computation2; } const Buffer = @import("buffer.zig").Buffer; diff --git a/zml/ops/manual_computation2.zig b/zml/ops/manual_computation2.zig new file mode 100644 index 00000000..2d3f0aec --- /dev/null +++ b/zml/ops/manual_computation2.zig @@ -0,0 +1,465 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const shardy = @import("mlir/dialects/shardy"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../Compiler.zig"); +const meta = @import("../meta.zig"); +const mlirx = @import("../mlirx.zig"); +const Shape = @import("../shape.zig").Shape; +const Sharding = @import("../Sharding.zig"); +const Tensor = @import("../tensor.zig").Tensor; + +const log = std.log.scoped(.@"zml/Compiler"); + +/// Describes how the inputs and outputs of `body_fn` are partitioned across a mesh. +/// Returns the type of the `partitioning` argument to `manualComputation`. +/// +/// * `in_specs` specifies the sharding of each tensor in the body's first argument. +/// It follows the argument's structure, omitting non-tensor metadata. +/// * `out_specs` specifies the sharding of each shape in the body's second argument. +/// It follows the output shapes' structure and can be omitted for a `void` argument. +/// +/// Each spec is a `Sharding.Partitioning.Logical`. A mapping such as +/// `.parse(.{ .hidden = .model })` splits the tensor's `.hidden` dimension across +/// the mesh's `.model` axis. Omitted dimensions are replicated; `.replicated` +/// replicates the entire tensor. The specs must not contain open dimensions. +/// Slice lengths, optional presence and active union tags must match the arguments. +/// +/// For a body taking `struct { x: Tensor, bias: Tensor, scale: f32 }` and a single +/// output `Shape`, the partitioning can be declared as: +/// +/// ```zig +/// const partitioning: ManualPartitioning(body) = .{ +/// .in_specs = .{ .x = .parse(.{ .hidden = .model }), .bias = .replicated }, +/// .out_specs = .parse(.{ .hidden = .model }), +/// }; +/// ``` +pub fn ManualPartitioning(comptime body_fn: anytype) type { + const Logical = Sharding.Partitioning.Logical; + + const Inputs = stdx.meta.FnParam(body_fn, 0); + const OutputShapes = stdx.meta.FnParam(body_fn, 1); + + const InputSpecs = meta.MapRestrict(Tensor, Logical).map(Inputs); + const OutputSpecs = meta.MapRestrict(Shape, Logical).map(OutputShapes); + + if (OutputSpecs == void) return struct { + in_specs: InputSpecs, + out_specs: void = {}, + }; + + return struct { + in_specs: InputSpecs, + out_specs: OutputSpecs, + }; +} + +/// Describes a computation on local shards of tensors distributed across `mesh`. +/// ZML operations normally describe work on global tensors. The partitioner +/// distributes the inputs according to `in_specs`, and `body_fn` describes the work +/// performed on each shard. The results are exposed to the surrounding graph as +/// global tensors with the sharding declared by `out_specs`. +/// +/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R`, where: +/// - `I` contains the local input tensors and any metadata needed by the body. +/// - `O` is `void`, `Shape`, or a container of shapes describing the local outputs. +/// - `R` is `void`, `Tensor`, or a container of tensors matching those local shapes +/// in traversal order. +/// * `inputs` contains the global tensors and metadata. The partitioner reshards +/// the tensors as needed to satisfy `in_specs`; metadata is passed to the body unchanged. +/// * `output_shapes` contains the global output shapes. The body receives their +/// local shard shapes, computed using `out_specs` and the sizes of the mesh axes. +/// * `mesh` is the logical mesh whose axis names are referenced by the specs. +/// * `partitioning` declares the input and output shardings; see `ManualPartitioning`. +/// Each partitioned dimension must be divisible by the size of its mesh axis. +/// +/// For example, splitting a dimension of size 64 across a mesh axis of size 8 gives +/// the body a dimension of size 8. A replicated dimension keeps its global size. +/// The caller supplies the global output shapes because the body only describes +/// local results: their shapes alone do not determine the global tensor shapes. +pub fn manualComputation2( + comptime body_fn: anytype, + inputs: stdx.meta.FnParam(body_fn, 0), + output_shapes: stdx.meta.FnParam(body_fn, 1), + mesh: *const Sharding.Mesh, + partitioning: ManualPartitioning(body_fn), +) stdx.meta.FnReturn(body_fn) { + const Inputs = stdx.meta.FnParam(body_fn, 0); + const OutputShapes = stdx.meta.FnParam(body_fn, 1); + const Outputs = stdx.meta.FnReturn(body_fn); + + const compiler = Compiler.current(); + var arena_state: std.heap.ArenaAllocator = .init(compiler.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + // The body may return containers borrowed from its inputs or output shapes. + const body_arena = compiler.arena.allocator(); + + var owned_inputs: Inputs = undefined; + meta.mapAlloc((struct { + fn copy(_: void, tensor: Tensor) Tensor { + return tensor; + } + }).copy, body_arena, {}, inputs, &owned_inputs) catch @panic("OOM"); + + const flat_input_tensors: []*Tensor = meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); + const in_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.in_specs) catch @panic("OOM"); + stdx.debug.assert(flat_input_tensors.len == in_specs.len, "manualComputation has {} input tensors but {} input specs", .{ flat_input_tensors.len, in_specs.len }); + + for (flat_input_tensors, in_specs) |tensor, spec| { + tensor._shape = tensor.shape().withPartitioning(mesh, spec); + stdx.debug.assert(!tensor.shape().containsPartitionSpec(.open), "manualComputation input sharding must be closed: {f}", .{tensor.shape()}); + } + + var local_output_shapes: OutputShapes = undefined; + meta.mapAlloc((struct { + fn copy(_: void, shape: Shape) Shape { + return shape; + } + }).copy, body_arena, {}, output_shapes, &local_output_shapes) catch @panic("OOM"); + + const flat_local_output_shapes = meta.collectMutPtrs(Shape, arena, &local_output_shapes) catch @panic("OOM"); + const out_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.out_specs) catch @panic("OOM"); + stdx.debug.assert(flat_local_output_shapes.len == out_specs.len, "manualComputation has {} output shapes but {} output specs", .{ flat_local_output_shapes.len, out_specs.len }); + + const flat_output_shapes = arena.alloc(Shape, flat_local_output_shapes.len) catch @panic("OOM"); + for (flat_output_shapes, flat_local_output_shapes, out_specs) |*global, local, spec| { + global.* = local.withPartitioning(mesh, spec); + stdx.debug.assert(!global.containsPartitionSpec(.open), "manualComputation output sharding must be closed: {f}", .{global}); + local.* = Sharding.shardedShape(global.*) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); + } + + const WrappedInputs = struct { + inputs: *const Inputs, + output_shapes: *const OutputShapes, + + fn wrap(allocator: std.mem.Allocator, opaque_wrapper: *const anyopaque, opaque_outputs: *anyopaque) error{OutOfMemory}![]*Tensor { + const wrapped: *const @This() = @ptrCast(@alignCast(opaque_wrapper)); + const outputs_ptr: *Outputs = @ptrCast(@alignCast(opaque_outputs)); + + outputs_ptr.* = body_fn(wrapped.inputs.*, wrapped.output_shapes.*); + + return try meta.collectMutPtrs(Tensor, allocator, outputs_ptr); + } + }; + + var outputs_struct: Outputs = undefined; + var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &local_output_shapes }; + + _ = manualComputationInternal( + compiler, + arena, + mesh, + @typeName(@TypeOf(body_fn)), + &wrapped_inputs, + flat_input_tensors, + @ptrCast(&outputs_struct), + flat_output_shapes, + WrappedInputs.wrap, + ) catch |err| switch (err) { + error.OutOfMemory => @panic("OOM"), + }; + + return outputs_struct; +} + +const LocalComputationFn = *const fn (std.mem.Allocator, *const anyopaque, *anyopaque) error{OutOfMemory}![]*Tensor; + +/// Reified version of manualComputation2. +/// This one is compiled exactly once, user code just need to compile the `collectPtrs` +noinline fn manualComputationInternal( + compiler: *Compiler, + arena: std.mem.Allocator, + mesh: *const Sharding.Mesh, + functor_name: []const u8, + input_struct: *anyopaque, + input_tensors: []*Tensor, + output_struct: *anyopaque, + global_output_shapes: []const Shape, + local_fn: LocalComputationFn, +) error{OutOfMemory}![]*Tensor { + const global_scope = compiler.currentScope(); + const mlir_ctx = compiler.mlir_ctx; + + log.debug("manualComputation2({s}) will use mesh {s}", .{ functor_name, mesh.name }); + + const global_input_shapes = try arena.alloc(Shape, input_tensors.len); + for (global_input_shapes, input_tensors) |*shape, tensor| shape.* = tensor._shape; + + const input_types = try arena.alloc(*const mlir.Type, input_tensors.len); + for (input_types, input_tensors) |*input_ty, tensor_ptr| { + const global = tensor_ptr._shape; + const local = Sharding.shardedShape(global) catch std.debug.panic("can't shard {f} for {f}", .{ global, mesh }); + log.debug("\t- {f} -> {f}", .{ global, local }); + input_ty.* = mlirx.Type.rankedTensor(mlir_ctx, local); + } + const input_locs = try arena.alloc(*const mlir.Location, input_tensors.len); + @memset(input_locs, compiler.unknown_location); + + const input_values = try arena.alloc(*const mlir.Value, input_tensors.len); + for (input_values, input_tensors) |*input_val, tensor_ptr| { + if (compiler.partitioner == .gspmd) { + tensor_ptr.* = tensor_ptr.withPartitioning(mesh, tensor_ptr.shape()._sharding.partition); + } + input_val.* = global_scope.value(tensor_ptr.*); + } + + const manual_block = switch (compiler.partitioner) { + .shardy => mlir.Block.init(input_types, input_locs), + .gspmd => global_scope.block, + }; + errdefer if (compiler.partitioner == .shardy) manual_block.deinit(); + + const local_scope = compiler.pushBlock(manual_block); + defer local_scope.pop(); + for (0.., input_tensors, global_input_shapes) |i, tensor_ptr, global| { + const local = Sharding.shardedShape(global) catch unreachable; // we already computed this above + tensor_ptr.* = switch (compiler.partitioner) { + .shardy => Tensor._result(local, manual_block.argument(i)), + .gspmd => gspmd.globalToLocal(mlir_ctx, global_scope, tensor_ptr.*, local, compiler.location), + }; + } + + compiler.manual_computation_depth += 1; + defer compiler.manual_computation_depth -= 1; + + const local_outputs = try local_fn(arena, input_struct, output_struct); + stdx.debug.assert(local_outputs.len == global_output_shapes.len, "manualComputation2 body returned {} values, expected {}", .{ local_outputs.len, global_output_shapes.len }); + for (0.., local_outputs, global_output_shapes) |i, local_tensor, global_shape| { + const expected_local = Sharding.shardedShape(global_shape) catch std.debug.panic("can't shard output {f} for {f}", .{ global_shape, mesh }); + stdx.debug.assert(expected_local.eql(local_tensor._shape), "manualComputation2 body output {d} expected {f}, got {f}", .{ i, expected_local, local_tensor._shape }); + } + + const global_output_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); + for (global_output_shapes, 0..) |output_shape, i| { + global_output_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output_shape); + } + + const local_output_values = try arena.alloc(*const mlir.Value, local_outputs.len); + for (local_output_values, local_outputs) |*output_val, tensor_ptr| { + output_val.* = local_scope.value(tensor_ptr.*); + } + + const op: *const mlir.Operation = switch (compiler.partitioner) { + .shardy => op: { + // Shardy creates a sdy.manual_computation block. + _ = shardy.return_(mlir_ctx, local_output_values, compiler.location).appendTo(manual_block); + + const out_mesh_attrs = try arena.alloc(*const shardy.TensorShardingAttr, global_output_shapes.len); + for (out_mesh_attrs, global_output_shapes) |*attr, shape| { + attr.* = try mesh.sdyShardingAttrForShape(arena, mlir_ctx, shape); + } + + break :op shardy.manual_computation( + mlir_ctx, + input_values, + global_output_types, + manual_block, + try Sharding.sdyPerValueShardingAttr(arena, mlir_ctx, global_input_shapes, mesh), + .get(mlir_ctx, out_mesh_attrs), + try Sharding.sdyManualAxesAttr(arena, mlir_ctx, global_input_shapes, global_output_shapes, mesh), + compiler.location, + ).appendTo(global_scope.block); + }, + .gspmd => op: { + // Gspmd calls SPMDShardToFullShape and wrap into an optimizationBarrier + // Shortcut to avoid empty optimization barrier + if (global_output_shapes.len == 0) return &.{}; + + const global_values = try arena.alloc(*const mlir.Value, global_output_shapes.len); + const global_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); + for (global_output_shapes, 0..) |global_shape, i| { + global_values[i], global_types[i] = try gspmd.localToGlobal(compiler, arena, global_scope, local_output_values[i], global_shape); + } + + break :op stablehlo.optimizationBarrier( + mlir_ctx, + global_values, + global_types, + compiler.location, + ).appendTo(global_scope.block); + }, + }; + + // Returned tensors describe the operation's global output shapes and sharding. + for (0.., local_outputs, global_output_shapes) |o, output_tensor_ptr, output_shape| { + output_tensor_ptr.* = Tensor._result(output_shape, op.result(o)); + output_tensor_ptr._shape._sharding = output_shape._sharding; + } + return local_outputs; +} + +const gspmd = struct { + fn globalToLocal(mlir_ctx: *mlir.Context, scope: *Compiler.Scope, local_tensor: Tensor, local_shape: Shape, location: *const mlir.Location) Tensor { + const local_type = mlirx.Type.rankedTensor(mlir_ctx, local_shape); + const full_to_shard = stablehlo.custom_call( + mlir_ctx, + &.{scope.value(local_tensor)}, + &.{local_type}, + .{ + .call_target_name = "SPMDFullToShardShape", + .has_side_effect = false, + .backend_config = .{ .original = "" }, + .additional_attributes = &.{ + .named(mlir_ctx, "mhlo.sharding", .string(mlir_ctx, "{manual}")), + }, + }, + location, + ).appendTo(scope.block); + return Tensor._result(local_shape, full_to_shard.result(0)); + } + + fn localToGlobal(compiler: *Compiler, arena: std.mem.Allocator, scope: *Compiler.Scope, local_value: *const mlir.Value, global_shape: Shape) !struct { *const mlir.Value, *const mlir.Type } { + const gspmd_attr = try compiler.tensorShardingAttr(arena, global_shape); + const global_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, global_shape); + + const shard_to_full = stablehlo.custom_call( + compiler.mlir_ctx, + &.{local_value}, + &.{global_type}, + .{ + .call_target_name = "SPMDShardToFullShape", + .has_side_effect = false, + .backend_config = .{ .original = "" }, + .additional_attributes = &.{ + .named(compiler.mlir_ctx, "mhlo.sharding", gspmd_attr), + }, + }, + compiler.location, + ).appendTo(scope.block); + return .{ shard_to_full.result(0), global_type }; + } +}; + +test manualComputation2 { + const zml = @import("../zml.zig"); + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + const mesh = zml.testing.tpMesh(); + + const LocalSum = struct { + x: Tensor, + bias: Tensor, + scale: i32, + + fn body(self: @This(), output_shape: Shape) Tensor { + return self.x.sum(.x).scale(self.scale).add(self.bias).broad(output_shape); + } + + fn forward(x: Tensor, mesh_: *const Sharding.Mesh) Tensor { + return manualComputation2( + body, + .{ .x = x, .bias = .scalar(10, .i32), .scale = 2 }, + x.shape(), + mesh_, + .{ .in_specs = .{ .x = .parse(.{ .x = .model }), .bias = .replicated }, .out_specs = .parse(.{ .x = .model }) }, + ); + } + }; + + // Each shard sums two elements and broadcasts the result to its local output shape. + const shape: Shape = .init(.{ .x = 2 * mesh.numPartitionsForLogicalAxis(.model) }, .i32); + const values = try allocator.alloc(i32, shape.count()); + defer allocator.free(values); + + for (values, 0..) |*value, i| value.* = @intCast(i + 1); + + var input = try zml.Buffer.fromBytes(io, platform, shape, std.mem.sliceAsBytes(values)); + defer input.deinit(); + + for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { + const exe = try platform.compileFn(allocator, io, LocalSum.forward, .{ .fromShape(shape), mesh }, .{ + .meshes = &.{ platform.replicated_mesh, mesh }, + .partitioner = partitioner, + }); + defer exe.deinit(); + + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + + const actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + + for (actual.constItems(i32), 0..) |value, i| { + const pair = values[i / 2 * 2 ..][0..2]; + try std.testing.expectEqual((pair[0] + pair[1]) * 2 + 10, value); + } + + try std.testing.expect(output.shape().eqlWithTags(shape)); + try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); + } +} + +test "manualComputation2 composes across logical and physical meshes" { + const zml = @import("../zml.zig"); + if (comptime !zml.Target.cpu.isEnabled()) return error.SkipZigTest; + const allocator = std.testing.allocator; + const io = std.testing.io; + + const Topologies = enum { flat, torus }; + + const Model = struct { + fn increment(x: Tensor, _: Shape) Tensor { + return x.addConstant(1); + } + + fn forward(x: Tensor, mesh: *const Sharding.Mesh) Tensor { + const partitioning: ManualPartitioning(increment) = .{ + .in_specs = .parse(.{ .x = .model }), + .out_specs = .parse(.{ .x = .model }), + }; + + const first = manualComputation2(increment, x, x.shape(), mesh, partitioning); + const converted = first.convert(.f32); + return manualComputation2(increment, converted, converted.shape(), mesh, partitioning); + } + }; + + const shape: Shape = .init(.{ .x = 16 }, .i32); + + var values: [16]i32 = undefined; + var expected: [16]f32 = undefined; + for (&values, &expected, 0..) |*value, *result, i| { + value.* = @intCast(i); + result.* = @floatFromInt(i + 2); + } + + for ([_]Topologies{ .flat, .torus }) |topology| { + const platform = try zml.Platform.init(allocator, io, .cpu, .{ + .cpu = .{ .device_count = 8 }, + .physical_mesh = if (topology == .torus) .{ .custom = Sharding.PhysicalMesh.torus3d } else .auto, + }); + defer platform.deinit(allocator, io); + + // Fold three physical axes into one logical axis, or split one into three. + const logical: Sharding.LogicalMesh = if (topology == .torus) + try .init(.{.model}, &.{.balanced}, &.{8}) + else + try Sharding.LogicalMesh.init(.{ .data, .model, .pipeline }, &.{ .low_bandwidth, .high_bandwidth, .balanced }, &.{ 2, 2, 2 }); + + const mesh = try platform.registerMesh("manual", logical); + + var input: zml.Buffer = try .fromBytes(io, platform, shape, std.mem.asBytes(&values)); + defer input.deinit(); + + for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { + const exe = try platform.compileFn(allocator, io, Model.forward, .{ Tensor.fromShape(shape), mesh }, .{ + .meshes = &.{ platform.replicated_mesh, mesh }, + .partitioner = partitioner, + }); + defer exe.deinit(); + + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + + try std.testing.expectEqual(expected, try output.getValue([16]f32, io)); + try std.testing.expect(output.shape().eqlWithTags(shape.withDtype(.f32))); + try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); + } + } +} diff --git a/zml/tensor.zig b/zml/tensor.zig index 59d32cbd..890b5c63 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -4813,14 +4813,14 @@ pub const Tensor = struct { defer ctx.arena.allocator().free(full_name); switch (ctx.platform.target) { .cpu, .cuda, .rocm, .tpu, .metal => { - ops.manualComputation((struct { + ops.manualComputation2((struct { input: Tensor, name: []const u8, fn body(body_ctx: @This(), _: void) void { ops.customCall("zml$print", body_ctx.input, {}, .{ .name = body_ctx.name }, .{ .has_side_effect = true }); } - }).body, .{ .input = input, .name = full_name }, {}); + }).body, .{ .input = input, .name = full_name }, {}, ctx.platform.replicated_mesh, .{ .in_specs = .{ .input = .replicated } }); }, .oneapi, .neuron, .furiosa => {}, } From 194175b36d826baf8255ab1c4df2d79d811d625a Mon Sep 17 00:00:00 2001 From: erwanvivien <44021072+erwanvivien@users.noreply.github.com> Date: Wed, 4 Nov 2026 14:35:55 +0100 Subject: [PATCH 839/881] zml/kernels/flash_attn: add prefill flash_attention kernel and add Causal (#1000) Prefill kernel is based on libflashattn.so from zml/flash-attention --- zml/attention/attention.zig | 22 +- .../flash_attention/cute/attention.zig | 504 +++++++++++++++++- zml/kernels/flash_attention/cute/cute.zig | 13 +- .../flash_attention/flash_attention.zig | 111 +++- 4 files changed, 626 insertions(+), 24 deletions(-) diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index 49fe947b..4cfad309 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -93,6 +93,12 @@ pub const Metadata = union(BackendType) { } }; +pub const Options = struct { + /// Query j sees the keys up to token_index + j, as in a decoder. Otherwise every query sees every + /// key, as in an encoder or a diffusion transformer, and token_index is ignored. + causal: bool = true, +}; + /// Causal attention as used in transformers. /// /// **Shapes**: @@ -105,18 +111,26 @@ pub const Metadata = union(BackendType) { /// - .k is the number of keys /// - .hd is the head dimension pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { + return attentionOpts(q, k, v, token_index, metadata, parameters, .{}); +} + +/// `attention`, causal or not depending on `opts`. +pub fn attentionOpts(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options) zml.Tensor { return switch (parameters) { .stablehlo => b: { // Generate the attention mask. const seq_len = k.dim(.k); - var attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, q.dtype(), null); - // Note: in Pytorch it would be very inefficient to generate the full attn_mask, // then slice into it, but XLA is able to optimize this correctly. - attn_mask = attn_mask.gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, attn_mask.dtype()), token_index.appendAxes(.{.coord}), .{}); + const attn_mask: ?zml.Tensor = if (opts.causal) + zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, q.dtype(), null) + .gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, q.dtype()), token_index.appendAxes(.{.coord}), .{}) + else + null; + const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask }); break :b attn_output; }, - .flash_attention => flash_attention.attention(q, k, v, token_index, metadata.flash_attention, parameters.flash_attention), + .flash_attention => flash_attention.attention(q, k, v, token_index, metadata.flash_attention, parameters.flash_attention, opts), }; } diff --git a/zml/kernels/flash_attention/cute/attention.zig b/zml/kernels/flash_attention/cute/attention.zig index 0d01c357..ec0fce16 100644 --- a/zml/kernels/flash_attention/cute/attention.zig +++ b/zml/kernels/flash_attention/cute/attention.zig @@ -1,4 +1,7 @@ const std = @import("std"); +const mlir = @import("mlir"); +const llvm = @import("mlir/dialects/llvm"); +const vector = @import("mlir/dialects/vector"); const zml = @import("../../../zml.zig"); const cute = zml.kernel.cute; @@ -14,6 +17,8 @@ pub const Config = struct { head_dim: i64, dtype: cute.DType, splits: i64 = 1, + /// Query j sees the keys up to positions + j. Otherwise every query sees every key and positions is ignored. + causal: bool = true, }; /// Planning uses local (post-sharding) dimensions. XLA owns the compiled CUDA @@ -144,13 +149,16 @@ fn runSized(b: *B, cfg: Config, comptime fragments: usize) cute.FinishError!void const query = row.div(group).minimum(cfg.query_length - 1); const head = kv_head.mul(group).add(row.rem(group)); const start = a.positions.get(.{batch}); - const position = start.add(query); - const causal_end = start.add(first_row.add(3).div(group).minimum(cfg.query_length - 1)).add(1); + const position = if (cfg.causal) start.add(query) else b.constant(.i32, cfg.key_length - 1); + const live_end = if (cfg.causal) + start.add(first_row.add(3).div(group).minimum(cfg.query_length - 1)).add(1) + else + b.constant(.i32, cfg.key_length); // Partition the live cache, rather than its capacity, so decode uses all // blocks even when only a prefix of the cache is populated. - const chunk = causal_end.add(cfg.splits - 1).div(cfg.splits); + const chunk = live_end.add(cfg.splits - 1).div(cfg.splits); const begin = if (split) b.blockIdx().z.mul(chunk) else b.constant(.i32, 0); - const end = if (split) begin.add(chunk).minimum(causal_end) else causal_end; + const end = if (split) begin.add(chunk).minimum(live_end) else live_end; const zero = b.constant(.f32, 0); const neg_inf = b.constant(.f32, -std.math.inf(f32)); @@ -305,6 +313,494 @@ fn merge(b: *B, cfg: Config) cute.FinishError!void { } } +// ==================== Prefill ==================== +// +// FlashAttention-2 forward for `query_length > 1`, with the tiling FA2's sm80 kernel uses on sm_120 +// (`Flash_fwd_kernel_traits<128, 128, 64, 4>`): a 128-thread block owns `prefillRows` queries of one +// head and every warp owns whole rows (all the keys of a tile, the whole head dimension). Scores and +// output stay in `mma.sync` accumulators, so P is reused from registers as the A operand of P @ V, +// without cross-warp reductions. Q, K and V tiles go through swizzled shared memory: V_j is copied +// while S_j = Q K_j^T is computed, K_{j+1} while O += P_j V_j is. +// +// It is a program, not a bare kernel: its host function launches the kernel with the dynamic shared +// memory the kernel allocates (up to 96 KiB). A bare kernel's shared memory is static, which ptxas +// caps at 48 KiB before sm_120. + +pub const Prefill = cute.Program(Config, .{ + .name = "flash_attention_cute_prefill", + .inputs = &.{ "q", "k", "v", "positions" }, + .outputs = &.{"out"}, + .run = prefillProgram, +}); + +const prefill_kernel_name = "flash_attention_cute_prefill_kernel"; + +/// Keys per tile (FA2's kBlockN). +const prefill_keys = 64; + +/// Whether `Prefill` handles `cfg`; the other configurations use `Kernel`. +/// Same as FA2 we only support 16-bit inputs +pub fn prefillSupported(cfg: Config) bool { + const dtype_ok = cfg.dtype == .bf16 or cfg.dtype == .f16; + const head_dim_ok = cfg.head_dim == 32 or cfg.head_dim == 64 or cfg.head_dim == 128 or cfg.head_dim == 256; + return cfg.query_length > 1 and cfg.splits == 1 and dtype_ok and head_dim_ok; +} + +/// m16 row tiles per warp: two up to head dimension 128, one above so the output accumulators fit in registers. +fn prefillRowTiles(head_dim: i64) i64 { + return if (head_dim <= 128) 2 else 1; +} + +/// Queries per block (FA2's kBlockM). +fn prefillRows(cfg: Config) i64 { + return 4 * 16 * prefillRowTiles(cfg.head_dim); +} + +/// Query tiles on x, so the blocks resident at the same time share the K/V of one head in L2. +fn prefillGrid(cfg: Config) [3]i32 { + return .{ @intCast(@divTrunc(cfg.query_length + prefillRows(cfg) - 1, prefillRows(cfg))), @intCast(cfg.query_heads), @intCast(cfg.batch) }; +} + +fn prefillProgram(b: *B, cfg: Config) cute.FinishError!void { + std.debug.assert(prefillSupported(cfg)); + switch (cfg.head_dim) { + inline 32, 64, 128, 256 => |d| switch (cfg.dtype) { + inline .bf16, .f16 => |dt| { + b.beginFunction(prefill_kernel_name, .cuda_kernel); + try prefillSized(b, cfg, d, dt); + b.endFunction(block); + + b.beginFunction(Prefill.name, .host); + const host = try b.declareArgs(.{ + .q = .{ .ptr = dt }, + .k = .{ .ptr = dt }, + .v = .{ .ptr = dt }, + .positions = .{ .ptr = cute.DType.i32 }, + .out = .{ .ptr = dt }, + }); + const grid = prefillGrid(cfg); + const config = b.makeLaunchConfig(.{ + .grid = .{ b.constant(.i32, grid[0]), b.constant(.i32, grid[1]), b.constant(.i32, grid[2]) }, + .block = .{ b.constant(.i32, block[0]), b.constant(.i32, block[1]), b.constant(.i32, block[2]) }, + .dynamic_smem = b.kernelSmemSize(prefill_kernel_name), + .stream = b.cudaStream(), + }); + const launch = b.launchEx(prefill_kernel_name, config, .{ host.q, host.k, host.v, host.positions, host.out }); + b.returnHostStatus(b.cudaResultStatus(launch)); + b.endFunction(null); + }, + else => unreachable, + }, + else => unreachable, + } +} + +/// PTX `max.f32` (FMNMX). `Value.maximum` and `Value.maxnum` both lower to NaN-handling compare and +/// select sequences (three instructions); prefill scores are never NaN, masked ones are -inf. +fn fmax(b: *B, x: Value, y: Value) Value { + return b.callIntrinsic("llvm.nvvm.fmax.f", &.{ x, y }, .float(b.ctx, .f32)).?; +} + +fn unpackStruct(b: *B, comptime n: usize, value: Value, dtype: cute.DType) [n]Value { + var result: [n]Value = undefined; + for (&result, 0..) |*r, i| { + r.* = b.emit(llvm.extractvalue(b.ctx, value.inner, dtype.toMlir(b.ctx), &.{@intCast(i)}, b.loc())); + } + return result; +} + +/// `ldmatrix.x4`: lanes 8i..8i+7 address the rows of matrix i, the result holds the four 8x8 fragments. +fn matrixLoad(b: *B, comptime transpose: bool, address: Value) [4]Value { + const assembly = "ldmatrix.sync.aligned.m8n8.x4" ++ (if (transpose) ".trans" else "") ++ ".shared.b16 {$0, $1, $2, $3}, [$4];"; + const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{address.inner}, .{ + .result_type = llvm.structType(b.ctx, &@as([4]*const mlir.Type, @splat(.int(b.ctx, .i32)))), + .constraints = "=r,=r,=r,=r,r", + .has_side_effects = true, + }, b.loc())); + return unpackStruct(b, 4, result, .i32); +} + +/// `mma.sync.m16n8k16` with f32 accumulators: acc += a (16x16, row) @ b (16x8, col). +fn mma(b: *B, comptime dtype: cute.DType, a: [4]Value, rhs: [2]Value, acc: [4]Value) [4]Value { + const assembly = "mma.sync.aligned.m16n8k16.row.col.f32." ++ @tagName(dtype) ++ "." ++ @tagName(dtype) ++ ".f32 " ++ + "{$0, $1, $2, $3}, {$4, $5, $6, $7}, {$8, $9}, {$10, $11, $12, $13};"; + const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{ a[0].inner, a[1].inner, a[2].inner, a[3].inner, rhs[0].inner, rhs[1].inner, acc[0].inner, acc[1].inner, acc[2].inner, acc[3].inner }, .{ + .result_type = llvm.structType(b.ctx, &@as([4]*const mlir.Type, @splat(.float(b.ctx, .f32)))), + .constraints = "=f,=f,=f,=f,r,r,r,r,r,r,f,f,f,f", + }, b.loc())); + return unpackStruct(b, 4, result, .f32); +} + +/// The four f32 accumulators of one mma.sync, carried across loop iterations as one value (the loop +/// carries at most 64 values). +fn packAcc(b: *B, values: [4]Value) Value { + return b.emit(vector.from_elements(b.ctx, &.{ values[0].inner, values[1].inner, values[2].inner, values[3].inner }, .vector(&.{4}, .float(b.ctx, .f32)), b.loc())); +} + +fn unpackAcc(b: *B, value: Value) [4]Value { + var result: [4]Value = undefined; + for (&result, 0..) |*r, i| { + r.* = b.vectorExtract(value, @intCast(i), .f32); + } + return result; +} + +/// Comptime layout of one prefill block, shared by the setup and the per-tile steps. +/// +/// Tiles are row major with the 16-byte vector index of every row XORed with row % swizzle_rows, so +/// 16-byte cp.async stores and 8-row ldmatrix reads are free of bank conflicts. Shared memory addresses +/// are folded the way CuTe's static layouts let nvcc fold them: the XORed row bits of every address a +/// thread uses are a per-thread constant, and the k-step part of a column offset is bit-disjoint from +/// the per-thread part, so each ldmatrix / cp.async address is a per-thread base plus a constant +/// (XORed into the 16-byte vector index). +fn PrefillTile(comptime d: i64, comptime dt: cute.DType) type { + return struct { + const row_tiles = prefillRowTiles(d); + const rows = 4 * 16 * row_tiles; + const n = prefill_keys; + const score_tiles = n / 8; + const out_tiles = d / 8; + const row_bytes = d * 2; + /// The XORed vector index must stay in its row: rows of d=32 only have 4 vectors. + const swizzle_rows = @min(8, d / 8); + const q_bytes = rows * row_bytes; + const kv_bytes = n * row_bytes; + /// Rows covered by one 16-byte cp.async per thread of the block. + const copy_rows = @divExact(128, d / 8); + /// Running max and per-thread partial sum of the two rows of every row tile. + const stats = 2 * row_tiles; + /// Carried by the key loop: maxima, sums, then the packed output accumulators of every + /// (row tile, 8 output features). + const carried = 2 * stats + row_tiles * out_tiles; + const scale: f32 = std.math.log2e / std.math.sqrt(@as(f32, @floatFromInt(d))); + + comptime { + // copy_rows rows starting at a multiple of copy_rows never carry into the XORed bits. + std.debug.assert(std.math.isPowerOfTwo(copy_rows)); + std.debug.assert(std.math.isPowerOfTwo(swizzle_rows)); + } + + b: *B, + cfg: Config, + lane: Value, + warp: Value, + first_query: Value, + start: Value, + q_smem: Value, + k_smem: Value, + v_smem: Value, + k_ptr: Value, + v_ptr: Value, + kv_base: Value, + tiles: Value, + /// cp.async: this thread copies the 16-byte vector `copy_col` of rows `copy_row + i * copy_rows`. + copy_row: Value, + copy_col: Value, + /// Byte offset in a tile of this thread's first cp.async destination, without the XORed part. + copy_row_offset: Value, + copy_vector_offset: Value, + /// ldmatrix rows of this thread, as byte offsets, and their XORed 16-byte vector offsets. + q_frag_row: Value, + k_frag_row: Value, + v_frag_row: Value, + qv_frag_vector: Value, + k_frag_vector: Value, + + const Self = @This(); + + fn init(b: *B, cfg: Config, tid: Value, first_query: Value, start: Value, shared: Value, k_ptr: Value, v_ptr: Value, kv_base: Value, tiles: Value) Self { + const lane = tid.rem(32); + const warp = tid.div(32); + const copy_row = tid.div(d / 8); + const copy_col = tid.rem(d / 8); + const lane8 = lane.rem(8); + // The fragment rows a thread reads are lane % 8 plus multiples of 8. + const lane_xor = lane.rem(swizzle_rows); + return .{ + .b = b, + .cfg = cfg, + .lane = lane, + .warp = warp, + .first_query = first_query, + .start = start, + .q_smem = shared, + .k_smem = shared.add(q_bytes), + .v_smem = shared.add(q_bytes + kv_bytes), + .k_ptr = k_ptr, + .v_ptr = v_ptr, + .kv_base = kv_base, + .tiles = tiles, + .copy_row = copy_row, + .copy_col = copy_col, + .copy_row_offset = copy_row.mul(row_bytes), + .copy_vector_offset = copy_col.bitXor(copy_row.rem(swizzle_rows)).mul(16), + // Q (A operand): rows lane % 16, vectors lane / 16 (+ 2 per k-step). + .q_frag_row = shared.add(warp.mul(16 * row_tiles).add(lane.rem(16)).mul(row_bytes)), + // K (B operand): keys lane % 8 + 8 * (lane / 16), vectors (lane / 8) % 2 (+ 2 per k-step). + .k_frag_row = shared.add(q_bytes).add(lane8.add(lane.div(16).mul(8)).mul(row_bytes)), + // V (B operand, transposed): keys lane % 8 + 8 * ((lane / 8) % 2), vectors lane / 16 (+ 2 per 16 features). + .v_frag_row = shared.add(q_bytes + kv_bytes).add(lane8.add(lane.div(8).rem(2).mul(8)).mul(row_bytes)), + .qv_frag_vector = lane.div(16).bitXor(lane_xor).mul(16), + .k_frag_vector = lane.div(8).rem(2).bitXor(lane_xor).mul(16), + }; + } + + /// Destination of the i-th cp.async of this thread in a tile starting at `base`. + fn copyDst(t: Self, base: Value, comptime i: usize) Value { + const xor = ((copy_rows * i) % swizzle_rows) * 16; + const vector_offset = if (xor == 0) t.copy_vector_offset else t.copy_vector_offset.bitXor(xor); + return base.add(t.copy_row_offset).add(vector_offset).add(copy_rows * i * row_bytes); + } + + /// Copy keys `key_base..key_base + n` of `src` into `dst`; keys past the end are zero-filled. + fn loadTile(t: Self, src: Value, dst: Value, key_base: Value) void { + const b = t.b; + const stride = t.cfg.kv_heads * d; + const first_key = key_base.add(t.copy_row); + const safe = src.add(t.kv_base.mul(2)); + const first = safe.add(first_key.to(.i64).mul(stride).add(t.copy_col.mul(8).to(.i64)).mul(2)); + inline for (0..@divExact(n, copy_rows)) |i| { + const valid = first_key.add(copy_rows * i).lt(t.cfg.key_length); + const source = b.select(valid, first.add(copy_rows * i * stride * 2), safe); + b.cpAsync16(t.copyDst(dst, i), source, valid); + } + } + + fn queryOf(t: Self, mi: usize, h: usize) Value { + return t.first_query.add(t.warp.mul(16 * row_tiles)).add(mi * 16).add(t.lane.div(4)).add(h * 8); + } + + fn accIndex(comptime mi: usize, comptime di: usize) usize { + return @intCast(2 * stats + @as(i64, mi) * out_tiles + @as(i64, di)); + } + + /// One key tile. `masked` tiles apply the causal and key length masks; the other ones are + /// fully visible to every query of the block. + fn step(t: Self, comptime masked: bool, tile: Value, carried_in: [carried]Value) [carried]Value { + const b = t.b; + const lane = t.lane; + const zero = b.constant(.f32, 0); + const neg_inf = b.constant(.f32, -std.math.inf(f32)); + const key_base = tile.mul(n); + b.cpAsyncWaitAll(); + b.syncThreads(); + t.loadTile(t.v_ptr, t.v_smem, key_base); + + // S = Q K^T, k-steps of 16 features. + var s: [row_tiles][score_tiles][4]Value = @splat(@splat(@splat(zero))); + inline for (0..d / 16) |ks| { + var qa: [row_tiles][4]Value = undefined; + inline for (0..row_tiles) |mi| { + qa[mi] = matrixLoad(b, false, t.q_frag_row.add(mi * 16 * row_bytes).add(t.qv_frag_vector.bitXor(ks * 32))); + } + const k_vector = t.k_frag_vector.bitXor(ks * 32); + inline for (0..score_tiles / 2) |np| { + const kb = matrixLoad(b, false, t.k_frag_row.add(np * 16 * row_bytes).add(k_vector)); + inline for (0..row_tiles) |mi| { + s[mi][2 * np] = mma(b, dt, qa[mi], .{ kb[0], kb[1] }, s[mi][2 * np]); + s[mi][2 * np + 1] = mma(b, dt, qa[mi], .{ kb[2], kb[3] }, s[mi][2 * np + 1]); + } + } + } + + // Online softmax in base 2 on the raw scores, the scale is folded into the exponent: + // p = exp2(s * scale - m * scale). Thread rows: lane / 4 (accumulators 0, 1) and lane / 4 + 8 (2, 3). + var next: [carried]Value = undefined; + var alpha: [row_tiles][2]Value = undefined; + inline for (0..row_tiles) |mi| { + if (masked) { + inline for (0..score_tiles) |ni| { + inline for (0..4) |j| { + const key = key_base.add(ni * 8).add(lane.rem(4).mul(2)).add(j % 2); + const in_range = key.lt(t.cfg.key_length); + const visible = if (t.cfg.causal) in_range.bitAnd(key.le(t.start.add(t.queryOf(mi, j / 2)))) else in_range; + s[mi][ni][j] = b.select(visible, s[mi][ni][j], neg_inf); + } + } + } + inline for (0..2) |h| { + var row_max = s[mi][0][h * 2]; + inline for (0..score_tiles) |ni| { + inline for (0..2) |j| { + row_max = fmax(b, row_max, s[mi][ni][h * 2 + j]); + } + } + inline for (.{ 2, 1 }) |offset| { + row_max = fmax(b, row_max, b.shuffleXor(row_max, offset)); + } + const old_max = carried_in[mi * 2 + h]; + const new_max = fmax(b, old_max, row_max); + // A row without any visible key so far keeps m = -inf: exponentiate against a finite value. + const neg_scaled_max = fmax(b, new_max, b.constant(.f32, -std.math.floatMax(f32))).mul(-scale); + alpha[mi][h] = b.exp2Approx(old_max.mul(scale).add(neg_scaled_max)); + next[mi * 2 + h] = new_max; + var sum = carried_in[stats + mi * 2 + h].mul(alpha[mi][h]); + inline for (0..score_tiles) |ni| { + inline for (0..2) |j| { + s[mi][ni][h * 2 + j] = b.exp2Approx(s[mi][ni][h * 2 + j].mul(scale).add(neg_scaled_max)); + sum = sum.add(s[mi][ni][h * 2 + j]); + } + } + next[stats + mi * 2 + h] = sum; + } + } + + // All warps are done with K_j once V_j has landed: start copying K_{j+1}. + b.cpAsyncWaitAll(); + b.syncThreads(); + var prefetch = b.openIf(tile.add(1).lt(t.tiles)); + { + t.loadTile(t.k_ptr, t.k_smem, key_base.add(n)); + prefetch.close(); + } + + // O = O * alpha + P V. The accumulators of two adjacent score tiles are the A operand of one k-step. + var o: [row_tiles][out_tiles][4]Value = undefined; + inline for (0..row_tiles) |mi| { + inline for (0..out_tiles) |di| { + const prev = unpackAcc(b, carried_in[accIndex(mi, di)]); + inline for (0..4) |j| { + o[mi][di][j] = prev[j].mul(alpha[mi][j / 2]); + } + } + } + inline for (0..n / 16) |ks| { + var pa: [row_tiles][4]Value = undefined; + inline for (0..row_tiles) |mi| { + pa[mi] = .{ + b.packHalf2(s[mi][2 * ks][0], s[mi][2 * ks][1], dt), + b.packHalf2(s[mi][2 * ks][2], s[mi][2 * ks][3], dt), + b.packHalf2(s[mi][2 * ks + 1][0], s[mi][2 * ks + 1][1], dt), + b.packHalf2(s[mi][2 * ks + 1][2], s[mi][2 * ks + 1][3], dt), + }; + } + const v_row = t.v_frag_row.add(ks * 16 * row_bytes); + inline for (0..out_tiles / 2) |dp| { + const vb = matrixLoad(b, true, v_row.add(t.qv_frag_vector.bitXor(dp * 32))); + inline for (0..row_tiles) |mi| { + o[mi][2 * dp] = mma(b, dt, pa[mi], .{ vb[0], vb[1] }, o[mi][2 * dp]); + o[mi][2 * dp + 1] = mma(b, dt, pa[mi], .{ vb[2], vb[3] }, o[mi][2 * dp + 1]); + } + } + } + inline for (0..row_tiles) |mi| { + inline for (0..out_tiles) |di| { + next[accIndex(mi, di)] = packAcc(b, o[mi][di]); + } + } + return next; + } + }; +} + +fn prefillSized(b: *B, cfg: Config, comptime d: i64, comptime dt: cute.DType) cute.FinishError!void { + const T = PrefillTile(d, dt); + const a = try b.declareArgs(.{ + .q = .{ .ptr = dt }, + .k = .{ .ptr = dt }, + .v = .{ .ptr = dt }, + .positions = .{ .ptr = cute.DType.i32 }, + .out = .{ .ptr = dt }, + }); + b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{.named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin())}))); + + const tid = b.threadIdx().x; + const first_query = b.blockIdx().x.mul(T.rows); + const head = b.blockIdx().y; + const batch = b.blockIdx().z; + const kv_head = head.div(@divExact(cfg.query_heads, cfg.kv_heads)); + // One shared storage object (like a Python CuTe `SharedStorage`): the host compiler sizes the + // launch's dynamic shared memory from it, which `kernelSmemSize` returns. `allocSmem` and + // `allocSmemView` both become static shared memory instead. + b.setFunctionAttribute(.named(b.ctx, "smem.partition_num", .int(b.ctx, .i32, 1))); + const fields = [_][]const u8{ + std.fmt.comptimePrint("sQ:{d}:0", .{T.q_bytes}), + std.fmt.comptimePrint("sK:{d}:{d}", .{ T.kv_bytes, T.q_bytes }), + std.fmt.comptimePrint("sV:{d}:{d}", .{ T.kv_bytes, T.q_bytes + T.kv_bytes }), + }; + const smem = b.allocSmemStorage(.i8, 128, b.layoutType(b.layoutSpec(.{T.q_bytes + 2 * T.kv_bytes}, .{1})), 0, &fields); + const shared = b.ptrToInt(b.getIter(smem, b.ptrTy(.i8, .smem, 128) catch @panic("bad shared memory pointer")).value(), .i32); + const q_ptr = b.ptrToInt(a.q, .i64); + const out_ptr = b.ptrToInt(a.out, .i64); + const start = b.loadWords(1, b.ptrToInt(a.positions, .i64).add(batch.to(.i64).mul(4)), .global, .{})[0]; + + // Element offsets of this (batch, head) in q/out and (batch, kv head) in k/v, and the q/out row stride. + const q_stride = cfg.query_heads * d; + const q_base = batch.to(.i64).mul(cfg.query_length * q_stride).add(head.to(.i64).mul(d)); + const kv_base = batch.to(.i64).mul(cfg.key_length * cfg.kv_heads * d).add(kv_head.to(.i64).mul(d)); + + // Causal: the last query of the block sees keys up to start + last_query, and every query of the + // block sees the keys up to start + first_query. Otherwise only the last tile, past the end of the + // keys, is masked. + const last_query = first_query.add(T.rows).minimum(cfg.query_length).sub(1); + const tiles = if (cfg.causal) + start.add(last_query).add(1).minimum(cfg.key_length).add(T.n - 1).div(T.n) + else + b.constant(.i32, @divTrunc(cfg.key_length + T.n - 1, T.n)); + const full_tiles = if (cfg.causal) + start.add(first_query).add(1).minimum(cfg.key_length).div(T.n).minimum(tiles) + else + b.constant(.i32, @divTrunc(cfg.key_length, T.n)); + + const t: T = .init(b, cfg, tid, first_query, start, shared, b.ptrToInt(a.k, .i64), b.ptrToInt(a.v, .i64), kv_base, tiles); + + // Out of range rows are zero-filled and never read from global memory. + { + const first_row = first_query.add(t.copy_row); + const safe = q_ptr.add(q_base.mul(2)); + const first = safe.add(first_row.to(.i64).mul(q_stride).add(t.copy_col.mul(8).to(.i64)).mul(2)); + inline for (0..@divExact(T.rows, T.copy_rows)) |i| { + const valid = first_row.add(T.copy_rows * i).lt(cfg.query_length); + b.cpAsync16(t.copyDst(t.q_smem, i), b.select(valid, first.add(T.copy_rows * i * q_stride * 2), safe), valid); + } + } + t.loadTile(t.k_ptr, t.k_smem, b.constant(.i32, 0)); + + var initial: [T.carried]Value = undefined; + const zero = b.constant(.f32, 0); + for (initial[0..T.stats]) |*v| { + v.* = b.constant(.f32, -std.math.inf(f32)); + } + for (initial[T.stats .. 2 * T.stats]) |*v| { + v.* = zero; + } + for (initial[2 * T.stats ..]) |*v| { + v.* = packAcc(b, .{ zero, zero, zero, zero }); + } + + var full = b.openFor(0, full_tiles, 1, initial); + { + full.yield(t.step(false, full.iv, full.carried[0..T.carried].*)); + } + var diagonal = b.openFor(full_tiles, tiles, 1, full.results[0..T.carried].*); + { + diagonal.yield(t.step(true, diagonal.iv, diagonal.carried[0..T.carried].*)); + } + const results = diagonal.results[0..T.carried].*; + + // Normalize by the row sums (reduced over the quad) and store pairs of features as one 32-bit word. + inline for (0..T.row_tiles) |mi| { + inline for (0..2) |h| { + var sum = results[T.stats + mi * 2 + h]; + inline for (.{ 2, 1 }) |offset| { + sum = sum.add(b.shuffleXor(sum, offset)); + } + const norm = b.select(sum.gt(0), b.rcpApprox(sum), zero); + const query = t.queryOf(mi, h); + var valid = b.openIf(query.lt(cfg.query_length)); + { + const row_offset = q_base.add(query.to(.i64).mul(q_stride)).add(t.lane.rem(4).mul(2).to(.i64)); + inline for (0..T.out_tiles) |di| { + const acc = unpackAcc(b, results[T.accIndex(mi, di)]); + const word = b.packHalf2(acc[h * 2].mul(norm), acc[h * 2 + 1].mul(norm), dt); + b.storeWords(1, out_ptr.add(row_offset.add(di * 8).mul(2)), .global, .{word}); + } + valid.close(); + } + } + } +} + test "CuTe attention emits tiled online softmax through head dimension 512" { for ([_]i64{ 64, 80, 128, 256, 512 }) |head_dim| { const ir = try Kernel.emit(std.testing.allocator, .{ diff --git a/zml/kernels/flash_attention/cute/cute.zig b/zml/kernels/flash_attention/cute/cute.zig index 1136dc9a..c1ffa58f 100644 --- a/zml/kernels/flash_attention/cute/cute.zig +++ b/zml/kernels/flash_attention/cute/cute.zig @@ -15,7 +15,8 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major >= 8; } -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor) zml.Tensor { +/// `causal`: query j sees the keys up to token_index + j. Otherwise every key, and token_index is ignored. +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, causal: bool) zml.Tensor { stdx.debug.assert(q.shape().hasTags(.{ .q, .h, .hd }) and k.shape().hasTags(.{ .k, .h, .hd }) and v.shape().hasTags(.{ .k, .h, .hd }), "CuTe attention requires q/h/hd and k/h/hd tags", .{}); const batched = q.shape().hasTag(.b) != null; const rank: usize = if (batched) 4 else 3; @@ -24,7 +25,7 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T stdx.debug.assert(q.dtype() == .bf16 or q.dtype() == .f16 or q.dtype() == .f32, "CuTe attention supports bf16, f16, and f32", .{}); stdx.debug.assert(q.dim(.hd) > 0 and q.dim(.hd) <= 512 and q.dim(.hd) == k.dim(.hd) and k.dim(.hd) == v.dim(.hd), "CuTe attention requires matching head dimensions in [1, 512]", .{}); stdx.debug.assert(k.dim(.h) > 0 and q.dim(.h) > 0 and @rem(q.dim(.h), k.dim(.h)) == 0 and k.dim(.h) == v.dim(.h), "CuTe attention requires Hq divisible by Hkv and matching K/V heads", .{}); - stdx.debug.assert(q.dim(.q) > 0 and q.dim(.q) <= k.dim(.k) and k.dim(.k) == v.dim(.k), "CuTe attention requires 0 < Lq <= Lk and matching K/V lengths", .{}); + stdx.debug.assert(q.dim(.q) > 0 and (!causal or q.dim(.q) <= k.dim(.k)) and k.dim(.k) == v.dim(.k), "CuTe attention requires 0 < Lq, Lq <= Lk when causal, and matching K/V lengths", .{}); stdx.debug.assert(token_index.dtype().isInteger(), "CuTe attention requires integer token positions", .{}); if (batched) { stdx.debug.assert(k.shape().hasTag(.b) != null and v.shape().hasTag(.b) != null and q.dim(.b) == k.dim(.b) and q.dim(.b) == v.dim(.b), "CuTe attention requires matching batch dimensions", .{}); @@ -41,7 +42,7 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T token_index.convert(.i32).broad(zml.Shape.init(.{ .b = q.dim(.b) }, .i32)) else token_index.convert(.i32); - return zml.ops.manualComputation(Context.body, Context{ .q = q_sharded, .k = k_sharded, .v = v_sharded, .positions = positions }, q.shape().withPartitioning(mesh, .{ .h = .model })); + return zml.ops.manualComputation(Context.body, Context{ .q = q_sharded, .k = k_sharded, .v = v_sharded, .positions = positions, .causal = causal }, q.shape().withPartitioning(mesh, .{ .h = .model })); } const Context = struct { @@ -49,6 +50,7 @@ const Context = struct { k: zml.Tensor, v: zml.Tensor, positions: zml.Tensor, + causal: bool, fn body(self: Context, output_shape: zml.Shape) zml.Tensor { const batched = self.q.shape().hasTag(.b) != null; @@ -67,9 +69,12 @@ const Context = struct { .head_dim = q.dim(.hd), .dtype = zml.kernel.cute.from(q.dtype()), .splits = splits, + .causal = self.causal, }; - const out = if (cfg.splits > 1) b: { + const out = if (kernel.prefillSupported(cfg)) + kernel.Prefill.call(.{ .q = q, .k = k, .v = v, .positions = self.positions }, .{ .out = q.shape() }, .{ .cfg = cfg }).out + else if (cfg.splits > 1) b: { const partials = kernel.Kernel.call(.{ .q = q, .k = k, .v = v, .positions = self.positions }, .{ .out = zml.Shape.init(.{ cfg.batch, cfg.splits, cfg.query_length, cfg.query_heads, cfg.head_dim + 2 }, .f32), }, .{ diff --git a/zml/kernels/flash_attention/flash_attention.zig b/zml/kernels/flash_attention/flash_attention.zig index c1a27a36..b0a6106b 100644 --- a/zml/kernels/flash_attention/flash_attention.zig +++ b/zml/kernels/flash_attention/flash_attention.zig @@ -126,11 +126,12 @@ pub const Metadata = union(Backend) { /// - .q is the number of queries /// - .k is the number of keys /// - .hd is the head dimension -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: zml.attention.Options) zml.Tensor { return switch (parameters) { - .cute => cute.attention(q, k, v, token_index), - .nki => nki.attention(q, k, v, token_index, parameters.nki), - .metal => metal.attention(q, k, v, token_index, metadata.metal), + .cute => cute.attention(q, k, v, token_index, opts.causal), + // The NKI and Metal kernels are causal only. + .nki => if (opts.causal) nki.attention(q, k, v, token_index, parameters.nki) else zml.nn.sdpa(q, k, v, .{}), + .metal => if (opts.causal) metal.attention(q, k, v, token_index, metadata.metal) else zml.nn.sdpa(q, k, v, .{}), }; } @@ -185,11 +186,95 @@ test "attention: q=8,qh=64,kh=8" { ); } -pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32) !void { - return testAttentionBackends(q_shape, k_shape, token_index_h, std.enums.values(Backend)); +test "attention: prefill, q=k=300,qh=8,kh=4" { + // Neither length is a multiple of the query (128) or key (64) tiles. + try testAttention( + .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), + .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), + &.{0}, + ); } -fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, backends: []const Backend) !void { +test "attention: prefill, q=200,k=1000,qh=4,kh=4,hd=64" { + // Queries start at 800: every query tile sees full key tiles before the diagonal. + try testAttention( + .init(.{ .q = 200, .h = 4, .hd = 64 }, .bf16), + .init(.{ .k = 1000, .h = 4, .hd = 64 }, .bf16), + &.{800}, + ); +} + +test "attention: prefill, q=k=200,qh=4,kh=2,hd=32" { + // Rows of 32 features only hold 4 16-byte vectors: the shared memory swizzle must stay inside them. + try testAttention( + .init(.{ .q = 200, .h = 4, .hd = 32 }, .bf16), + .init(.{ .k = 200, .h = 2, .hd = 32 }, .bf16), + &.{0}, + ); +} + +test "attention: prefill, q=k=130,qh=4,kh=2,hd=256" { + try testAttention( + .init(.{ .q = 130, .h = 4, .hd = 256 }, .bf16), + .init(.{ .k = 130, .h = 2, .hd = 256 }, .bf16), + &.{0}, + ); +} + +test "attention: prefill, b=2,q=150,k=300,qh=4,kh=2" { + try testAttention( + .init(.{ .b = 2, .q = 150, .h = 4, .hd = 128 }, .bf16), + .init(.{ .b = 2, .k = 300, .h = 2, .hd = 128 }, .bf16), + &.{ 150, 100 }, + ); +} + +test "attention: non-causal prefill, q=k=300,qh=8,kh=4" { + try testAttentionOpts( + .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), + .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), + &.{0}, + .{ .causal = false }, + ); +} + +test "attention: non-causal prefill, b=2,q=200,k=1000,qh=4,kh=4,hd=64" { + // Queries are a suffix of the keys, as for the image tokens of a joint text/image attention. + try testAttentionOpts( + .init(.{ .b = 2, .q = 200, .h = 4, .hd = 64 }, .bf16), + .init(.{ .b = 2, .k = 1000, .h = 4, .hd = 64 }, .bf16), + &.{ 0, 0 }, + .{ .causal = false }, + ); +} + +test "attention: non-causal prefill, more queries than keys, q=300,k=100,qh=4,kh=2" { + try testAttentionOpts( + .init(.{ .q = 300, .h = 4, .hd = 128 }, .bf16), + .init(.{ .k = 100, .h = 2, .hd = 128 }, .bf16), + &.{0}, + .{ .causal = false }, + ); +} + +test "attention: non-causal decode ignores token_index, q=1,k=1000,qh=64,kh=8" { + try testAttentionOpts( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 1000, .h = 8, .hd = 64 }, .bf16), + &.{5}, + .{ .causal = false }, + ); +} + +pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32) !void { + return testAttentionOpts(q_shape, k_shape, token_index_h, .{}); +} + +fn testAttentionOpts(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options) !void { + return testAttentionBackends(q_shape, k_shape, token_index_h, opts, std.enums.values(Backend)); +} + +fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options, backends: []const Backend) !void { const io = std.testing.io; const allocator = std.testing.allocator; @@ -206,9 +291,11 @@ fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: const max_k_idx: i64 = k_shape.dim(.k) - 1; const max_q_idx: i64 = q_shape.dim(.q) - 1; - for (token_index_h) |index| { - // Check for out of bound reads - std.debug.assert(index + max_q_idx <= max_k_idx); + if (opts.causal) { + // Check for out of bound reads, token_index is ignored by non-causal attention. + for (token_index_h) |index| { + std.debug.assert(index + max_q_idx <= max_k_idx); + } } const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor } = .{ @@ -230,7 +317,7 @@ fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: const meshes = platform.meshes.values(); const reference_metadata: zml.attention.Metadata = .{ .stablehlo = {} }; const reference_parameters: zml.attention.Parameters = .{ .stablehlo = {} }; - const reference_exe = try platform.compileFn(allocator, io, zml.attention.attention, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, reference_metadata, reference_parameters }, .{ + const reference_exe = try platform.compileFn(allocator, io, zml.attention.attentionOpts, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, reference_metadata, reference_parameters, opts }, .{ .program_name = "attention_stablehlo", .meshes = meshes, }); @@ -251,7 +338,7 @@ fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: allocator, io, attention, - .{ tensors.q, tensors.k, tensors.v, tensors.token_index, metadata, parameters }, + .{ tensors.q, tensors.k, tensors.v, tensors.token_index, metadata, parameters, opts }, .{ .program_name = try std.fmt.allocPrint(arena, "attention_{t}", .{backend}), .meshes = meshes, From 4707e58774409811d10b0aae95edfebab5441fef Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 4 Nov 2026 16:04:17 +0100 Subject: [PATCH 840/881] zml.Compiler: allow to propagate tensor store names (#1018) The name can appear in the IR and thanks to #1003 it shows in the web view of the model making all compute blocks pretty obvious. The screenshot below without names would be quite confusing, but now we can see this is the QKV of first layer of a Llama model. image --- examples/mnist/mnist_data_parallel.zig | 2 +- zml/Compiler.zig | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/examples/mnist/mnist_data_parallel.zig b/examples/mnist/mnist_data_parallel.zig index a45c0534..b7805e49 100644 --- a/examples/mnist/mnist_data_parallel.zig +++ b/examples/mnist/mnist_data_parallel.zig @@ -135,7 +135,7 @@ pub fn main(init: std.process.Init) !void { .{ .program_name = "mnist", .meshes = &.{dp_mp}, - .xla_dump_to = "/tmp/zml/mnist", + .xla_dump_to = "/tmp/zml/mnist_dp", }, ); }; diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 5c6e3933..654d9cba 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -23,6 +23,7 @@ const tracer = @import("profiling/tracer.zig"); const Shape = @import("shape.zig").Shape; const Sharding = @import("Sharding.zig"); const Tensor = @import("tensor.zig").Tensor; +const TensorStore = @import("io.zig").TensorStore; const Compiler = @This(); const log = std.log.scoped(.@"zml/Compiler"); @@ -31,6 +32,7 @@ allocator: std.mem.Allocator, io: std.Io, arena: std.heap.ArenaAllocator, +options: Options, mlir_registry: *mlir.DialectRegistry, mlir_ctx: *mlir.Context, mlir_pass_manager: *mlir.PassManager, @@ -77,6 +79,7 @@ pub const Options = struct { /// Incremented once after successful compilation. progress: std.Progress.Node = .none, meshes: []const *const Sharding.Mesh = &.{}, + store: ?*const TensorStore = null, // If null, will be initialized from the target partitioner: ?Sharding.Partitioner = null, /// HACK: Extra bytes added to the device memory size reported to XLA. @@ -194,6 +197,7 @@ pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, .mlir_known_types = mlir_known_types, .module = module, .platform = platform, + .options = opts, .partitioner = opts.partitioner orelse .fromTarget(platform.target), .meshes = meshes, .location = unknown_location, @@ -288,6 +292,7 @@ test pushLocation { .mlir_pass_manager = undefined, .mlir_known_types = undefined, .module = undefined, + .options = .{}, .platform = undefined, .partitioner = undefined, .meshes = undefined, @@ -620,8 +625,15 @@ fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{ O return error.OpenShardingAtBoundaries; } const packed_shape = og_shape.packedShape(); - const mlir_type = mlirx.Type.rankedTensor(ctx.compiler.mlir_ctx, packed_shape); - var value = ctx.scope.block.addArgument(mlir_type, .unknown(ctx.compiler.mlir_ctx)); + const mlir_ctx = ctx.compiler.mlir_ctx; + const mlir_type = mlirx.Type.rankedTensor(mlir_ctx, packed_shape); + var location: *const mlir.Location = ctx.compiler.unknown_location; + if (ctx.compiler.options.store) |store| { + if (store.id_to_sources.get(tensor.id)) |sources| { + location = location.named(mlir_ctx, sources.tensors[0].name); + } + } + var value = ctx.scope.block.addArgument(mlir_type, location); // But immediately create the MLIR value corresponding to the fp4 data. // That way the user code only sees a proper fp4 tensor. From 350c6c1606398424e554660c52a580e80470e95f Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 4 Nov 2026 16:04:17 +0100 Subject: [PATCH 841/881] zml.Mesh: axis/hasAxis helpers (#1019) --- zml/Sharding.zig | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/zml/Sharding.zig b/zml/Sharding.zig index 2f887a8a..ddb23cf9 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -1515,6 +1515,10 @@ pub const LogicalMesh = struct { return .{ .axes = axes, .intents = intents, .dimensions = @splat(-1) }; } + pub fn hasAxis(m: *const LogicalMesh, logical_ax: Shape.Tag) ?u3 { + return @truncate(std.mem.findScalar(Shape.Tag, m.axes.slice(), logical_ax) orelse return null); + } + pub fn intent(self: LogicalMesh, tag: Shape.Tag) ?LogicalAxisIntent { const target = std.mem.span(tag); for (self.axes.slice(), self.intents.slice()) |t, i| { @@ -1680,13 +1684,17 @@ pub const Mesh = struct { return @divExact(self.numDevices(), self.numPartitions()); } + pub fn axis(mesh: *const Mesh, ax: @EnumLiteral()) u3 { + return mesh.logical.hasAxis(Shape.toTag(ax)) orelse std.debug.panic("Mesh(name={s}) doesn't have axis {}: {f}", .{ mesh.name, ax, mesh.logical }); + } + pub fn sdyMeshAttr(self: *const Mesh, ctx: *mlir.Context) *const shardy.MeshAttr { var axes: stdx.BoundedArray(*const shardy.MeshAxisAttr, MAX_MESH_RANK + 1) = .empty; var tile_count: usize = 1; - for (self.logical.axes.slice(), 0..) |axis, i| { + for (self.logical.axes.slice(), 0..) |ax, i| { const size = self.grid.axis_sizes[i]; tile_count *= size; - axes.appendAssumeCapacity(.get(ctx, .{ .name = std.mem.span(axis), .size = @intCast(size) })); + axes.appendAssumeCapacity(.get(ctx, .{ .name = std.mem.span(ax), .size = @intCast(size) })); } const replicas = @divExact(self.grid.device_count, tile_count); if (replicas > 1) { @@ -1697,8 +1705,8 @@ pub const Mesh = struct { for (0..self.grid.device_count) |partition_index| { const coords: [MAX_MESH_RANK]u8 = self.grid.logical_coords[partition_index]; var tile_index: usize = 0; - for (0..self.logical.axes.len) |axis| { - tile_index = tile_index * self.grid.axis_sizes[axis] + coords[axis]; + for (0..self.logical.axes.len) |ax| { + tile_index = tile_index * self.grid.axis_sizes[ax] + coords[ax]; } // Device IDs here are executable partition indices, not PJRT device IDs. ids[tile_index * replicas + replica_indices[tile_index]] = @intCast(partition_index); @@ -1711,8 +1719,8 @@ pub const Mesh = struct { // There are at most MAX_MESH_RANK logical names, so one candidate is free. inline for (0..MAX_MESH_RANK + 1) |i| { const name = std.fmt.comptimePrint("__replica_{d}", .{i}); - const taken = for (self.logical.axes.slice()) |axis| { - if (std.mem.eql(u8, std.mem.span(axis), name)) break true; + const taken = for (self.logical.axes.slice()) |ax| { + if (std.mem.eql(u8, std.mem.span(ax), name)) break true; } else false; if (!taken) return name; } @@ -1775,26 +1783,26 @@ pub const Mesh = struct { const dimensions = try allocator.alloc(*const shardy.DimensionShardingAttr, shape.rank()); var used: [MAX_MESH_RANK]bool = @splat(false); const has_open_dimension = shape.containsPartitionSpec(.open); - for (0.., dimensions) |ax, *d| { - d.* = switch (shape.partition(ax)) { + for (0.., dimensions) |i, *d| { + d.* = switch (shape.partition(i)) { .out_of_bound => @panic("undefined"), .replicated => .replicated(ctx), .open => .open(ctx, &.{}), else => d: { - const axis = shape.partition(ax).meshAxis().?; - used[axis] = true; - break :d .closed(ctx, &.{.named(ctx, std.mem.span(data.logical.axes.get(axis)))}); + const ax = shape.partition(i).meshAxis().?; + used[ax] = true; + break :d .closed(ctx, &.{.named(ctx, std.mem.span(data.logical.axes.get(ax)))}); }, }; } var replicated_axes: stdx.BoundedArray(*const shardy.AxisRefAttr, MAX_MESH_RANK + 1) = .empty; var tile_count: usize = 1; - for (data.logical.axes.slice(), 0..) |axis, i| { + for (data.logical.axes.slice(), 0..) |ax, i| { tile_count *= data.grid.axis_sizes[i]; // Open dimensions can acquire unused logical axes during inference. // Only reserve those axes for replication when every dimension is closed. - if (!has_open_dimension and !used[i]) replicated_axes.appendAssumeCapacity(.named(ctx, std.mem.span(axis))); + if (!has_open_dimension and !used[i]) replicated_axes.appendAssumeCapacity(.named(ctx, std.mem.span(ax))); } if (tile_count < data.grid.device_count) { replicated_axes.appendAssumeCapacity(.named(ctx, data.sdyReplicaAxis())); @@ -1810,8 +1818,8 @@ pub const Mesh = struct { var tile_shape: stdx.BoundedArray(i64, Shape.MAX_RANK + 1) = .empty; var tile_count: usize = 1; - for (0..shape.rank()) |ax| { - const size = if (shape.partition(ax).meshAxis()) |axis| self.grid.axis_sizes[axis] else 1; + for (0..shape.rank()) |i| { + const size = if (shape.partition(i).meshAxis()) |ax| self.grid.axis_sizes[ax] else 1; tile_shape.appendAssumeCapacity(size); tile_count *= size; } From 17ba49cce828a260ca6e9f017e7818d6eafa6a95 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 4 Nov 2026 16:04:18 +0100 Subject: [PATCH 842/881] zml.ops.manualComputation: fix error message (#1021) use `.format` instead of default Zig formatting --- zml/ops/manual_computation.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/ops/manual_computation.zig b/zml/ops/manual_computation.zig index 170f96b5..45400206 100644 --- a/zml/ops/manual_computation.zig +++ b/zml/ops/manual_computation.zig @@ -113,7 +113,7 @@ noinline fn manualComputationInternal( const mesh = switch (chooseCommonMesh(input_tensors, compiler.platform.replicated_mesh)) { .mesh => |m| m, - .incompatible => |incomp| std.debug.panic("manualComputation expects all inputs and outputs to use the same mesh, got input {}: {} and input {}: {}", .{ incomp[0], input_tensors[incomp[0]], incomp[1], input_tensors[incomp[1]] }), + .incompatible => |incomp| std.debug.panic("manualComputation expects all inputs and outputs to use the same mesh, got input {d}: {f} and input {d}: {f}", .{ incomp[0], input_tensors[incomp[0]], incomp[1], input_tensors[incomp[1]] }), }; const manual_axes = collectManualAxes(mesh, input_tensors); log.debug("manualComputation({s}) will use mesh {s}, manual axes: {f}", .{ functor_name, mesh.name, stdx.fmt.stringsZ(manual_axes.constSlice()) }); From 8992fbe4343a5a1b241695920e494834d66a6554 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 4 Nov 2026 16:04:18 +0100 Subject: [PATCH 843/881] zml.Shape.assertTags (#1022) we often use `withTags` to document the current shape, but it could also hide bug by overriding previous tags. I introduce `assertTags` that should be favored when possible --- zml/shape.zig | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/zml/shape.zig b/zml/shape.zig index f866d2e2..e793b520 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -808,6 +808,19 @@ pub const Shape = struct { } } + pub fn assertTags(self: Shape, tagz: anytype) void { + const T = @TypeOf(tagz); + if (T == Shape) { + return self.assertTags(tagz.tags()); + } else if (T != []const Shape.Tag) { + const parsed_tags: []const Shape.Tag = comptime parseTags(tagz).constSlice(); + return self.assertTags(parsed_tags); + } + + stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {}", .{ self, stdx.fmt.stringsZ(tagz) }); + stdx.debug.assert(std.mem.eql(Shape.Tag, tagz, self.tags()), "Expected tags {f}, got {f}", .{ stdx.fmt.stringsZ(tagz), self }); + } + pub fn withDtype(self: Shape, dt: DataType) Shape { var res = self; res._dtype = dt; From db119b2d3558a1e042ad0c217bedc5180a9adcc6 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 4 Nov 2026 16:04:19 +0100 Subject: [PATCH 844/881] zml.Shape: improve parseTags (#1023) avoid `stdx.BoundedArray` for tags, use `[]const Tag`, this allows to simplify some logic and more explicitly separate comptime work from runtime work. It reduces binary sizes, preventing big functions to be duplicated because of different tags. --- zml/Compiler.zig | 7 ++----- zml/Sharding.zig | 6 +++--- zml/io.zig | 31 ++++++++++++++++++------------- zml/shape.zig | 32 ++++++++++++++++++++++++-------- zml/tensor.zig | 4 ++-- 5 files changed, 49 insertions(+), 31 deletions(-) diff --git a/zml/Compiler.zig b/zml/Compiler.zig index 654d9cba..93eafa8d 100644 --- a/zml/Compiler.zig +++ b/zml/Compiler.zig @@ -366,16 +366,13 @@ pub fn mesh(compiler: *const Compiler, name: @EnumLiteral()) *const Sharding.Mes /// Find a mesh that covers all the given axes pub fn resolveMesh(compiler: *const Compiler, logical_axes: anytype) *const Sharding.Mesh { if (@TypeOf(logical_axes) != []const Shape.Tag) { - const comp_tags = comptime Shape.parseTags(logical_axes); - const parsed_tags: []const Shape.Tag = comptime comp_tags.constSlice(); - return compiler.resolveMesh(parsed_tags); + return compiler.resolveMesh(comptime Shape.parseTags(logical_axes)); } var ok_meshe: ?*const Sharding.Mesh = null; for (compiler.meshes) |shd| { var covers_all: bool = true; - for (logical_axes) |ax| { - const input_axis = Shape.toTag(ax); + for (logical_axes) |input_axis| { var covers_this: bool = false; for (shd.logical.axes.slice()) |existing_axis| { if (std.mem.eql(u8, std.mem.span(existing_axis), std.mem.span(input_axis))) { diff --git a/zml/Sharding.zig b/zml/Sharding.zig index ddb23cf9..040216bc 100644 --- a/zml/Sharding.zig +++ b/zml/Sharding.zig @@ -1478,8 +1478,8 @@ pub const LogicalMesh = struct { /// * dimensions: dimension for each axis, -1 infers the size from underlying topology pub fn init(names: anytype, intents: []const LogicalAxisIntent, dimensions: []const i32) !LogicalMesh { if (@TypeOf(names) != []const Shape.Tag) { - const parsed_names = comptime Shape.parseTags(names); - return .init(@as([]const Shape.Tag, parsed_names.slice()), intents, dimensions); + const parsed_names: []const Shape.Tag = comptime Shape.parseTags(names); + return .init(parsed_names, intents, dimensions); } if (names.len == 0 or names.len > MAX_MESH_RANK or names.len != intents.len or names.len != dimensions.len) @panic("invalid logical mesh"); @@ -1645,7 +1645,7 @@ pub const Mesh = struct { return .{ .name = @tagName(name_), .logical = .{ - .axes = .init(axes_tags.slice()), + .axes = .init(axes_tags), .intents = .{ .buffer = @splat(.high_bandwidth), .len = axes_tags.len }, .dimensions = @splat(-1), }, diff --git a/zml/io.zig b/zml/io.zig index 43bfdf23..1babdcfc 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -184,19 +184,22 @@ pub const TensorStore = struct { stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); } - pub fn maybeCreateTensor(self: View, subkey: []const u8, comptime tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) ?Tensor { - const has_tags: bool = comptime @TypeOf(tags) != @TypeOf(null); - const parsed_tags: Shape.TagsArray = if (comptime has_tags) Shape.parseTags(tags) else undefined; + pub fn maybeCreateTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) ?Tensor { + const parsed_tags: ?[]const Shape.Tag = comptime parseTags(tags); if (mesh_ == .replicated) { - return self.maybeCreateTensorInternal(subkey, if (has_tags) parsed_tags.constSlice() else null, .replicated); + return self.maybeCreateTensorInternal(subkey, parsed_tags, .replicated); } + const resolved_mesh = self.store.mesh(mesh_); + if (@TypeOf(partitioning) == Sharding.Partitioning) { + return self.maybeCreateTensorInternal(subkey, parsed_tags, .{ .mesh = resolved_mesh, .partition = partitioning }); + } // Unspecified tensor axes are treated as replicated - const partition: Sharding.Partitioning = if (has_tags) - .parse(parsed_tags.constSlice(), resolved_mesh, .replicated, partitioning) + const partition: Sharding.Partitioning = if (parsed_tags) |p_tags| + .parse(p_tags, resolved_mesh, .replicated, partitioning) else partitioning; - return self.maybeCreateTensorInternal(subkey, if (has_tags) parsed_tags.constSlice() else null, .{ .mesh = resolved_mesh, .partition = partition }); + return self.maybeCreateTensorInternal(subkey, parsed_tags, .{ .mesh = resolved_mesh, .partition = partition }); } pub fn createHostPinnedTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) Tensor { @@ -204,8 +207,8 @@ pub const TensorStore = struct { stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); } - pub fn maybeCreateHostPinnedTensor(self: View, subkey: []const u8, comptime tags: anytype, mesh_: @EnumLiteral(), comptime partitioning: anytype) ?Tensor { - const tensor = self.maybeCreateTensor(subkey, tags, mesh_, partitioning); + pub fn maybeCreateHostPinnedTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), comptime partitioning: anytype) ?Tensor { + const tensor = self.maybeCreateTensor(subkey, comptime parseTags(tags), mesh_, partitioning); if (tensor) |t| { const storage = self.store.id_to_sources.getPtr(t.id); storage.?.memory = .host_pinned; @@ -214,10 +217,7 @@ pub const TensorStore = struct { } pub fn createReplicatedTensor(self: View, subkey: []const u8, tags: anytype) Tensor { - const has_tags: bool = comptime @TypeOf(tags) != @TypeOf(null); - const parsed_tags: Shape.TagsArray = if (comptime has_tags) Shape.parseTags(tags) else undefined; - - return self.maybeCreateTensorInternal(subkey, if (has_tags) parsed_tags.slice() else null, .replicated) orelse + return self.maybeCreateTensorInternal(subkey, comptime parseTags(tags), .replicated) orelse stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); } @@ -244,6 +244,11 @@ pub const TensorStore = struct { return tensor; } + /// Accepts "null" as tags + fn parseTags(tags: anytype) ?[]const Shape.Tag { + return if (@TypeOf(tags) == @TypeOf(null)) null else Shape.parseTags(tags); + } + pub fn maybeCreateBinding(self: View, sources: []const []const u8, shape: Shape) ?Tensor { const arena = self.store.arena.allocator(); diff --git a/zml/shape.zig b/zml/shape.zig index e793b520..386b594e 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -117,14 +117,31 @@ pub const Shape = struct { stdx.debug.compileError("Wrong type, got {}. Expected .{{.a, .b}}", .{T}); } - pub fn parseTags(v: anytype) TagsArray { + pub fn parseTags(v: anytype) []const Shape.Tag { const T = @TypeOf(v); - stdx.debug.assertComptime(stdx.meta.isTupleOf(T, @EnumLiteral()), "Wrong type, got {}. Expected .{{ .a, .b }}", .{T}); - var tags_: TagsArray = .empty; - inline for (v) |field| { - tags_.appendAssumeCapacity(toTag(field)); + if (T == []const Shape.Tag) return v; + comptime validateTags(T); + return comptime frozen: { + var tagz: [v.len]Shape.Tag = undefined; + for (v, 0..) |field, i| tagz[i] = @tagName(field).ptr; + // Force the creation of a comptime constant, so that we can return a pointer to it. + const frozen_tags = tagz; + break :frozen &frozen_tags; + }; + } + + fn validateTags(T: type) void { + const message = "Expected an enum-literal tuple such as .{ .b, .d }, got " ++ @typeName(T); + switch (@typeInfo(T)) { + .@"struct" => |info| { + if (!info.is_tuple) @compileError(message); + for (info.field_types) |Field| { + if (Field != @EnumLiteral()) @compileError(message); + } + if (info.field_types.len > Shape.MAX_RANK) @compileError("Too many shape tags (maximum 8)"); + }, + else => @compileError(message), } - return tags_; } /// Create a shape from a struct literal, eg: @@ -813,8 +830,7 @@ pub const Shape = struct { if (T == Shape) { return self.assertTags(tagz.tags()); } else if (T != []const Shape.Tag) { - const parsed_tags: []const Shape.Tag = comptime parseTags(tagz).constSlice(); - return self.assertTags(parsed_tags); + return self.assertTags(comptime parseTags(tagz)); } stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {}", .{ self, stdx.fmt.stringsZ(tagz) }); diff --git a/zml/tensor.zig b/zml/tensor.zig index 890b5c63..a7782844 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1528,7 +1528,7 @@ pub const Tensor = struct { pub fn dotAxes(lhs: Tensor, rhs: Tensor, tags: anytype) DotAxes { const T = @TypeOf(tags); if (comptime T == Shape.Tag or T == @EnumLiteral()) { - return lhs.dotAxes(rhs, .{tags}); + return lhs.dotAxes(rhs, @as([]const Shape.Tag, &.{Shape.toTag(tags)})); } const lhs_contracting_axes, _ = lhs.shape().parseAxes(tags); @@ -2983,7 +2983,7 @@ pub const Tensor = struct { var res_shape = self._shape; const ones: [constants.MAX_RANK]i64 = @splat(1); res_shape._dims.insertSlice(ax, ones[0..tags_.len]) catch unreachable; - res_shape._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; + res_shape._tags.insertSlice(ax, tags_) catch unreachable; return self.reshape(res_shape); } From 5764202e5fe3e3af556c4e183fca21ee0e6cd452 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Wed, 4 Nov 2026 18:52:05 +0100 Subject: [PATCH 845/881] zml/kernels/paged_attention/tcl: follow sequence lengths and support several chips (#1026) - **Work follows sequence lengths:** decode's compute no longer runs over the block table's whole width, and flash prefill loops only over the chunks in use. - **One plan per call:** each call picks straight decode, the hybrid loop or flash attention, following a calibration sweep based on what compiles. The kernels might change once the compiler evolves. - **Several chips:** each chip runs the kernels over its own heads inside a manual computation. --- zml/kernels/paged_attention/tcl/tcl.zig | 759 +++++++++++++++++++----- 1 file changed, 624 insertions(+), 135 deletions(-) diff --git a/zml/kernels/paged_attention/tcl/tcl.zig b/zml/kernels/paged_attention/tcl/tcl.zig index ce8bdb4f..a7c2a5f4 100644 --- a/zml/kernels/paged_attention/tcl/tcl.zig +++ b/zml/kernels/paged_attention/tcl/tcl.zig @@ -1,19 +1,6 @@ -//! Furiosa RNGD paged attention in TCL, using the common/flash attention -//! algorithms from Furiosa's SDK (`furiosa-kernels 2026.4.0` reference). -//! This implementation retains FP32 normalization/accumulation and BF16 -//! contractions. The exact common SDK operation materializes scores in BF16; -//! it fails the required unit-scale tolerance and is not a selectable backend. -//! -//! Queries are cut in blocks of rows of one sequence (one token per block for -//! decode). A context of at most `chunk_keys` keys runs Furiosa's one-shot -//! attention: gather the keys by token slot, contract, mask, one softmax, -//! contract. Longer contexts run Furiosa's flash attention: a device loop over -//! chunks of `chunk_keys` keys up to the longest sequence, gathering whole -//! pages and accumulating the softmax statistics in f32. -//! -//! Decode without sinks or windows, up to `decode_keys` keys, instead runs one -//! fusible kernel per layer over the whole pages each token sees, with the -//! token metadata every layer shares computed once per forward pass. +//! Furiosa RNGD paged attention in TCL: f32 softmax and accumulation, BF16 contractions. +//! `defaultPlan` picks fusible BF16 decode kernels, else one-shot attention or a device loop over key chunks. +//! `Repro:` files are in the reproducer repository, platform/furiosa. const std = @import("std"); const stdx = @import("stdx"); @@ -25,17 +12,43 @@ const tcl = zml.kernel.tcl; const paged_attention = @import("../paged_attention.zig"); const triton = @import("../triton/triton.zig"); +/// WORKAROUND (TCC 2026.5.0b0): flash decode with 8192-key chunks fails to +/// lower, so decode chunks hold 4096 keys. Repro: repro_flash_decode_8k_chunks.py. +/// Remove once 8192-key chunks compile. const chunk_keys = 4096; +const flash_chunk_pairs = 512 * 2048; +const prefill_chunk_keys = 2048; const prefill_block = 32; +/// WORKAROUND (TCC 2026.5.0b0): block counts that are not a power of two slow TCC's layout +/// search or panic its allocator. Repro: repro_flash_allocator_block_group_panic.py. +/// Remove once any count compiles. +fn prefillBlocks(tokens: i64, sequences: i64, block: i64) i64 { + const blocks = @divTrunc(tokens + sequences * (block - 1), block); + return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(blocks)) catch unreachable); +} + +fn prefillBlock(tokens: i64, sequences: i64) i64 { + const short = prefill_block / 2; + return if (short * prefillBlocks(tokens, sequences, short) < prefill_block * prefillBlocks(tokens, sequences, prefill_block)) short else prefill_block; +} + +/// A straight-line head over each token's first `head` keys, then a device loop over +/// `chunk`-key chunks (none when zero), with tokens in groups of `group`. +pub const Plan = struct { + head: i64, + chunk: i64, + group: i64, +}; + +pub var calibration_plan: ?Plan = null; + pub const paged = struct { pub const Options = triton.paged.Options; pub const Parameters = triton.paged.Parameters; - /// `q` is `{.b = tokens, .hkv, .hg, .hd}`, the caches - /// `{.page, .k_chunk, .hkv, .hd}`. Sinks are one logit per query head. - /// Unwritten cache entries are excluded before the contractions; callers - /// need only populate the tokens covered by their sequence lengths. + /// `q` is `{.b = tokens, .hkv, .hg, .hd}`, the caches `{.page, .k_chunk, .hkv, .hd}`. + /// Sinks are one logit per query head. pub fn pagedAttention(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: paged_attention.AttentionOptions) Tensor { if (opts.mask == .indices) @panic("Sparse attention masks are not supported by TCL paged attention"); stdx.debug.assert(q.rank() == 4 and k_cache.rank() == 4 and v_cache.rank() == 4 and k_cache.shape().eqlDims(v_cache.shape()), "TCL attention requires matching rank-four split K/V caches", .{}); @@ -46,7 +59,7 @@ pub const paged = struct { stdx.debug.assert(parameters.seq_lens.dim(.b) > 0 and parameters.block_table.dim(.p) > 0 and k_cache.dim(.page) > 0 and k_cache.dim(.k_chunk) > 0, "TCL attention requires nonempty sequence and cache capacities", .{}); stdx.debug.assert(k_cache.dim(.page) * k_cache.dim(.k_chunk) <= std.math.maxInt(i32) and q.dim(.b) <= std.math.maxInt(i32), "TCL attention indices must fit in I32", .{}); const compiler = zml.Compiler.current(); - stdx.debug.assert(compiler.platform.target == .furiosa and compiler.meshes[0].numDevices() == 1, "TCL attention requires one Furiosa chip", .{}); + stdx.debug.assert(compiler.platform.target == .furiosa, "TCL attention requires Furiosa chips", .{}); const attributes = compiler.platform.devices[0].pjrt_device.attributes(compiler.platform.pjrt_api) catch std.debug.panic("TCL attention could not verify the Furiosa device's PE range", .{}); defer attributes.deinit(); @@ -54,25 +67,46 @@ pub const paged = struct { const last_pe = attributes.get("pe_end"); stdx.debug.assert(first_pe != null and first_pe.? == .int64 and first_pe.?.int64 == 0 and last_pe != null and last_pe.? == .int64 and last_pe.?.int64 == 7, "TCL attention requires a complete 8-PE RNGD chip", .{}); + if (compiler.meshes[0].numDevices() > 1) return zml.ops.manualComputation(Shard.body, .{ .parameters = parameters, .q = q, .k_cache = k_cache, .v_cache = v_cache, .opts = opts }, q.shape()); + return attend(parameters, q, k_cache, v_cache, opts); + } + + const Shard = struct { + parameters: Parameters, + q: Tensor, + k_cache: Tensor, + v_cache: Tensor, + opts: paged_attention.AttentionOptions, + + fn body(self: Shard, _: zml.Shape) Tensor { + return attend(self.parameters, self.q, self.k_cache, self.v_cache, self.opts); + } + }; + + fn attend(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: paged_attention.AttentionOptions) Tensor { if (q.dim(.b) == 0) return Tensor.zeroes(q.shape()); const page = k_cache.dim(.k_chunk); - // TCC 2026.5 has no DPE tactic for 32-key attention. Pad to - // at least 64 keys; the extra pages remain invalid and contribute zero. + // WORKAROUND (TCC 2026.5.0b0): no DPE tactic for 32-key attention, so pad to + // at least 64 keys with invalid pages. Repro: repro_decode_32_keys_no_tactic.py. + // Remove once 16- to 48-key decode compiles. const keys = @max(parameters.block_table.dim(.p), std.math.divCeil(i64, 64, page) catch unreachable) * page; - // Chunks hold whole pages and a multiple of 64 keys: TCC 2026.5 fails - // to allocate a chunk of 4080 keys (85 pages of 48). + // WORKAROUND (TCC 2026.5.0b0): TCC fails to allocate a 4080-key chunk (85 + // pages of 48), so chunks hold whole pages and a multiple of 64 keys. Repro: + // repro_flash_allocator_block_group_panic.py. Remove once such chunks compile. const key_alignment = @divExact(page, @as(i64, @intCast(std.math.gcd(@as(u64, @intCast(page)), 64)))) * 64; const alignment = if (key_alignment <= chunk_keys) key_alignment else page; - const chunk = if (keys <= chunk_keys) keys else @min(keys, @max(1, @divTrunc(chunk_keys, alignment)) * alignment); const decode = !parameters.options_.is_prefill and q.dim(.b) == parameters.seq_lens.dim(.b); - const blocks = (std.math.divCeil(i64, q.dim(.b), prefill_block) catch unreachable) + parameters.seq_lens.dim(.b); - // Regular block capacities limit layout-search cost for the fused graph. - // Extra blocks retain zero valid lengths and invalid query positions. - const cfg: Cfg = .{ + const block = if (decode) 1 else prefillBlock(q.dim(.b), parameters.seq_lens.dim(.b)); + const block_count = if (decode) q.dim(.b) else prefillBlocks(q.dim(.b), parameters.seq_lens.dim(.b), block); + const rows = block_count * block; + // A flash chunk's query-key pairs are bounded: the largest chunks do not finish compiling. + const chunk_limit = @max(1, @divTrunc(@min(@as(i64, if (decode) chunk_keys else prefill_chunk_keys), @divTrunc(flash_chunk_pairs, rows)), alignment)) * alignment; + const chunk = @min(keys, chunk_limit); + var cfg: Cfg = .{ .tokens = q.dim(.b), .sequences = parameters.seq_lens.dim(.b), - .b = if (decode) q.dim(.b) else @intCast(std.math.ceilPowerOfTwo(u64, @intCast(blocks)) catch unreachable), - .i = if (decode) 1 else prefill_block, + .b = block_count, + .i = block, .a = (std.math.divCeil(i64, keys, chunk) catch unreachable) * chunk, .chunk = chunk, .hkv = q.dim(.hkv), @@ -86,10 +120,24 @@ pub const paged = struct { .is_causal = opts.mask.isCausal(), .window = @intCast(opts.mask.slidingWindow() orelse 0), }; - if (decode and !cfg.use_sink and cfg.window == 0 and keys <= decode_keys) { - return decodeAttention(parameters, q, k_cache, v_cache, .{ + const plain_decode = decode and !cfg.use_sink and cfg.window == 0; + // WORKAROUND (TCC 2026.5.0b0): an in-unit cache conversion and several tokens + // over a non-power-of-two page count fail to compile, so straight decode reads + // BF16 caches and pads the table. Repro: repro_sparse_propagation_transmute.py, + // repro_metadata_page_count_cover_valid_range.py. Remove once both compile. + const straight = decode and k_cache.dtype() == .bf16 and v_cache.dtype() == .bf16; + const straight_keys = if (straight and cfg.tokens > 1) @as(i64, @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@divExact(keys, page))) catch unreachable)) * page else keys; + const plan = calibration_plan orelse defaultPlan(cfg, straight_keys, parameters.block_table.dim(.p), plain_decode and straight, straight, chunk); + const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{ .hkv = cfg.hkv, .hg = cfg.g }) else Tensor.zeroes(.init(.{ .hkv = cfg.hkv, .hg = cfg.g }, .f32)); + if (plan.head > 0 and plan.chunk > 0) { + stdx.debug.assert(plain_decode and cfg.tokens == 1, "TCL hybrid attention takes one plain decode token", .{}); + return hybridDecodeAttention(parameters, q, k_cache, v_cache, cfg.scale, cfg.is_causal, plan).convert(q.dtype()); + } + if (plan.chunk == 0 and straight) { + return decodeAttention(parameters, q, k_cache, v_cache, sink, .{ .t = cfg.tokens, - .a = keys, + .a = straight_keys, + .group = plan.group, .hkv = cfg.hkv, .g = cfg.g, .d = cfg.d, @@ -97,8 +145,12 @@ pub const paged = struct { .pg = page, .scale = cfg.scale, .is_causal = cfg.is_causal, + .window = cfg.window, + .use_sink = cfg.use_sink, }).convert(q.dtype()); } + cfg.chunk = if (plan.chunk > 0) plan.chunk else keys; + cfg.a = (std.math.divCeil(i64, keys, cfg.chunk) catch unreachable) * cfg.chunk; const table = parameters.block_table.pad(-1, .{ .p = Tensor.Pad{ .high = @divExact(cfg.a, page) - parameters.block_table.dim(.p) } }); const inputs: OneShot.Inputs = .{ .q = q.convert(.bf16), @@ -107,7 +159,7 @@ pub const paged = struct { .query_offsets = parameters.query_start_len, .k_cache = k_cache.convert(.bf16), .v_cache = v_cache.convert(.bf16), - .sink = if (opts.sink) |s| s.convert(.f32).reshape(.{ .hkv = cfg.hkv, .hg = cfg.g }) else Tensor.zeroes(.init(.{ .hkv = cfg.hkv, .hg = cfg.g }, .f32)), + .sink = sink, }; const outputs: OneShot.Outputs = .{ .out = q.shape().withDtype(.bf16) }; const out = if (cfg.a == cfg.chunk) @@ -118,13 +170,55 @@ pub const paged = struct { } }; -/// Keys a decode kernel attends to in one pass: TCC 2026.5 fails on the -/// decode metadata for 131072 keys in pages of 16. +fn defaultPlan(cfg: Cfg, keys: i64, columns: i64, plain_decode: bool, straight: bool, chunk: i64) Plan { + // WORKAROUND (TCC 2026.5.0b0): the hybrid kernel's tail does not compile in + // reasonable time for 2 or 4 KV heads wider than 128, which run flash instead. + // Repro: repro_hybrid_decode_slow_compile.py. Remove once that tail compiles. + if (plain_decode and cfg.tokens == 1 and columns > std.math.divCeil(i64, straight_decode_keys, cfg.pg) catch unreachable) { + if (cfg.d <= 128 or cfg.hkv == 1 or cfg.hkv >= 8) return .{ .head = head_keys, .chunk = tail_chunk_keys, .group = 1 }; + return .{ .head = 0, .chunk = chunk, .group = cfg.b * cfg.i }; + } + if (straight and cfg.window > 0) { + const window_chunk = @max(window_chunk_keys, @as(i64, @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.window)) catch unreachable))); + const chunks: i64 = if (cfg.tokens < chunked_decode_tokens) window_flash_chunks else batch_window_flash_chunks; + if (window_chunk <= chunk and @rem(window_chunk, cfg.pg) == 0 and keys >= chunks * window_chunk) + return .{ .head = 0, .chunk = window_chunk, .group = cfg.tokens }; + } + if (straight and keys <= decode_keys and (cfg.tokens < chunked_decode_tokens or keys <= straight_batch_decode_keys)) + return .{ .head = keys, .chunk = 0, .group = @divExact(cfg.tokens, decodeGroups(cfg.tokens, keys)) }; + if (chunk == keys) return .{ .head = keys, .chunk = 0, .group = cfg.tokens }; + return .{ .head = 0, .chunk = chunk, .group = cfg.b * cfg.i }; +} + +const window_chunk_keys = 1024; +const window_flash_chunks = 16; +const batch_window_flash_chunks = 4; + +/// Widest block table the straight-line decode kernel takes for one token. +/// WORKAROUND (TCC 2026.5.0b0): several tokens do not use it, as TCC cannot +/// propagate gather bounds outside a loop over a token axis. Repro: +/// repro_sparse_propagation_loop.py. Remove once such kernels propagate. +const straight_decode_keys = 32768; + +/// Keys a straight-line decode kernel attends to in one pass. WORKAROUND (TCC +/// 2026.5.0b0): TCC fails on the decode metadata for 131072 keys in pages of 16. +/// Repro: repro_metadata_einsum_by_ve_no_tactic.py. Remove once that compiles. const decode_keys = 65536; +const chunked_decode_tokens = 16; +const straight_batch_decode_keys = 8192; + +/// Decode contracts the gathered cache unmasked: the select gives unused entries (NaN included) zero +/// weight. WORKAROUND (TCC 2026.5.0b0): TCC panics ("slice out of bounds") on score contractions of more +/// tokens times keys than this. Repro: repro_gather_einsum_slice_panic.py. Remove once larger products compile. +const unmasked_decode_keys = 1 << 17; + +const masked_decode_keys = 4096; + const DecodeCfg = struct { t: i64, a: i64, + group: i64, hkv: i64, g: i64, d: i64, @@ -132,14 +226,12 @@ const DecodeCfg = struct { pg: i64, scale: f32, is_causal: bool, + window: i64, + use_sink: bool, }; -/// Decode attention in Furiosa's form: the kernel gathers whole pages, then -/// runs one contraction, a masked softmax and a second contraction. It reads -/// past each sequence's end, selecting zero for those unused cache entries. -/// The token metadata depends only on the attention parameters, which every -/// layer shares: one kernel computes it once per forward pass. -fn decodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, cfg: DecodeCfg) Tensor { +/// The token metadata depends only on parameters every layer shares, so one kernel computes it per forward pass. +fn decodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, sink: Tensor, cfg: DecodeCfg) Tensor { const columns = @divExact(cfg.a, cfg.pg); const table = parameters.block_table.pad(0, .{ .p = Tensor.Pad{ .high = columns - parameters.block_table.dim(.p) } }); const meta = DecodeMeta.call(.{ .seq_lens = parameters.seq_lens, .query_offsets = parameters.query_start_len, .table = table }, .{ @@ -147,7 +239,7 @@ fn decodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_c .bounds = zml.Shape.init(.{ .t = cfg.t }, .i32), .mask = zml.Shape.init(.{ .t = cfg.t, .a = cfg.a }, .i32), .keyed = zml.Shape.init(.{ .t = cfg.t }, .f32), - }, .{ .cfg = .{ .t = cfg.t, .s = parameters.seq_lens.dim(.b), .ap = columns, .pg = cfg.pg, .pages = cfg.pages, .is_causal = cfg.is_causal } }); + }, .{ .cfg = .{ .t = cfg.t, .s = parameters.seq_lens.dim(.b), .ap = columns, .pg = cfg.pg, .pages = cfg.pages, .is_causal = cfg.is_causal, .window = cfg.window } }); return Decode.call(.{ .q = q.convert(.bf16), .pages = meta.pages, @@ -156,19 +248,15 @@ fn decodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_c .keyed = meta.keyed, .k_cache = k_cache.convert(.bf16), .v_cache = v_cache.convert(.bf16), + .sink = sink, }, .{ .out = q.shape().withDtype(.bf16) }, .{ .cfg = cfg }).out; } -const MetaCfg = struct { t: i64, s: i64, ap: i64, pg: i64, pages: i64, is_causal: bool }; +const MetaCfg = struct { t: i64, s: i64, ap: i64, pg: i64, pages: i64, is_causal: bool, window: i64 }; -/// For each token: the cache page of each column, how many columns hold keys -/// it sees, the keys it sees (below its limit, in pages inside the cache) and -/// whether it sees any. Columns past the sequence's -/// pages read page `p`, so that the masked padding spreads over many pages -/// instead of hammering one. Rows are selected by one-hot products and -/// comparisons follow `attentionMask`'s clamp arithmetic: TCC 2026.5 has no -/// tactics for small integer gathers and fails on the plugin's lowering of the -/// same comparisons for several tokens. +/// WORKAROUND (TCC 2026.5.0b0): one-hot row selection and `attentionMask`'s clamp arithmetic, as TCC +/// fails on small integer gathers and the plugin's comparisons. Repro: repro_scalar_gather_no_axis_to_pad.py, +/// repro_row_broadcast_past_4096.py. Remove once both compile. const DecodeMeta = tcl.Kernel(MetaCfg, .{ .name = "paged_decode_metadata", .inputs = &.{ "seq_lens", "query_offsets", "table" }, @@ -188,14 +276,12 @@ const DecodeMeta = tcl.Kernel(MetaCfg, .{ }); const ends = try selectMetadata(b, t.query_offsets, try selector(b, try b.arange(cfg.s + 1, .i32, &.{S}, .{ .start = 1 }), O, cfg.s + 1), O); const tokens = try b.arange(cfg.t, .i32, &.{T}, .{}); - // Padding tokens follow every sequence's end and belong to none. const seq = try countBefore(b, ends, tokens, S); const own = try selector(b, seq, S, cfg.s); var op = b.tensorOperation(.{}); const active = try op.commit(op.fetch(seq, .{}).muli(-1).addi(cfg.s).binary(.maxi, 0).binary(.mini, 1), .{}); var visible = try selectMetadata(b, t.seq_lens, own, S); if (cfg.is_causal) { - // Causal tokens see their sequence up to their own position. const end = try selectMetadata(b, ends, own, S); op = b.tensorOperation(.{}); visible = try op.commit(op.fetch(visible, .{}).subi(op.fetch(end, .{})).addi(op.fetch(tokens, .{})).addi(1), .{}); @@ -213,8 +299,6 @@ const DecodeMeta = tcl.Kernel(MetaCfg, .{ const spread = try op.commit(op.fetch(try b.arange(cfg.ap, .i32, &.{AP}, .{}), .{}).binary(.mini, cfg.pages - 1), .{}); op = b.tensorOperation(.{}); const read = try op.commit(op.where(op.fetch(used, .{}), .ne, 0, op.fetch(pages.safe, .{}), op.fetch(spread, .{ .broadcast_to = &.{ T, AP } })), .{}); - // Tokens without keys have no defined attention: the attention - // kernel zeroes them. op = b.tensorOperation(.{}); const keyed = try op.commit(op.fetch(used, .{}).muli(op.fetch(pages.valid, .{})).reduce(&.{AP}, .maxi).toFloat(), .{}); op = b.tensorOperation(.{}); @@ -222,21 +306,19 @@ const DecodeMeta = tcl.Kernel(MetaCfg, .{ op = b.tensorOperation(.{}); const position = try op.commit(op.fetch(try b.arange(cfg.ap * cfg.pg, .i32, &.{A}, .{}), .{ .broadcast_to = &.{ T, A } }), .{}); op = b.tensorOperation(.{ .tactic = .elementwise }); - const mask = try op.commit(op.fetch(position, .{}).muli(-1).addi(op.fetch(limit, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(try b.reshape(valid, &.{ T, A }, .{}), .{})), .{}); + var mask = try op.commit(op.fetch(position, .{}).muli(-1).addi(op.fetch(limit, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(try b.reshape(valid, &.{ T, A }, .{}), .{})), .{}); + if (cfg.window > 0) { + op = b.tensorOperation(.{ .tactic = .elementwise }); + mask = try op.commit(op.fetch(position, .{}).subi(op.fetch(limit, .{})).addi(cfg.window + 1).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(mask, .{})), .{}); + } b.ret(&.{ read, bounds, mask, keyed }); } }.run, }); -/// Gathers the key and value pages a token sees, contracts with the query, -/// masks keys past the sequence, normalizes in one softmax times `keyed` (zero -/// for tokens without keys, whose softmax is uniform) and contracts with the -/// values. A token's pages are gathered up to its bound, and the operations -/// reading them skip the rest: the work follows the sequence's length, not the -/// table's. Fusible: it joins the layer around it, one launch per layer. const Decode = tcl.Kernel(DecodeCfg, .{ .name = "paged_decode_attention", - .inputs = &.{ "q", "pages", "bounds", "mask", "keyed", "k_cache", "v_cache" }, + .inputs = &.{ "q", "pages", "bounds", "mask", "keyed", "k_cache", "v_cache", "sink" }, .outputs = &.{"out"}, .run = struct { fn run(b: *tcl.Builder, cfg: DecodeCfg) tcl.FinishError!void { @@ -258,26 +340,360 @@ const Decode = tcl.Kernel(DecodeCfg, .{ .keyed = .{ .dtype = .f32, .axes = &.{T} }, .k_cache = .{ .dtype = .bf16, .axes = &.{ P, PG, Hkv, D } }, .v_cache = .{ .dtype = .bf16, .axes = &.{ P, PG, Hkv, D } }, + .sink = .{ .dtype = .f32, .axes = &.{ Hkv, G } }, }); - const mask = try b.reshape(t.mask, &.{ T, AP, PG }, .{}); - const k = try maskCacheValues(b, try b.gather(t.k_cache, t.pages, P, &.{ T, AP, PG, Hkv, D }, .{ .valid_length = t.bounds }), mask); - const v = try maskCacheValues(b, try b.gather(t.v_cache, t.pages, P, &.{ T, AP, PG, Hkv, D }, .{ .valid_length = t.bounds }), mask); - const keys: []const tcl.Axis = &.{ AP, PG }; - var op = b.tensorOperation(.{}); - const scores = try op.commit(op.contract(t.q, k, &.{ Hkv, G, T, AP, PG }).mulf(cfg.scale), .{}); - op = b.tensorOperation(.{}); - const s = op.fetch(scores, .{}); - const masked = try op.commit(op.where(op.fetch(mask, .{}), .ne, 0, s, score_min), .{}); - op = b.tensorOperation(.{}); - const x = op.fetch(masked, .{}); - const e = x.subf(x.reduce(keys, .maxf)).exp(); - const p = try op.commit(e.divf(e.reduce(keys, .addf)).mulf(op.fetch(t.keyed, .{})), .{ .dtype = .bf16 }); - op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.contract(v, p, &.{ T, Hkv, G, D }), .{ .dtype = .bf16 })}); + const sink = if (cfg.use_sink) t.sink else null; + const groups = @divExact(cfg.t, cfg.group); + if (groups == 1) return b.ret(&.{try decodeTokens(b, cfg.scale, t.q, t.pages, t.bounds, t.mask, t.keyed, t.k_cache, t.v_cache, sink)}); + const NG = b.axis("NG", groups); + const TG = b.axis("TG", @divExact(cfg.t, groups)); + var loop = b.openFor(groups, .{try b.scratchpad(.bf16, &.{ NG, TG, Hkv, G, D })}); + { + const q = try b.indexRead(try b.reshape(t.q, &.{ NG, TG, Hkv, G, D }, .{}), loop.iv); + const pages = try b.indexRead(try b.reshape(t.pages, &.{ NG, TG, AP }, .{}), loop.iv); + const bounds = try b.indexRead(try b.reshape(t.bounds, &.{ NG, TG }, .{}), loop.iv); + const mask = try b.indexRead(try b.reshape(t.mask, &.{ NG, TG, A }, .{}), loop.iv); + const keyed = try b.indexRead(try b.reshape(t.keyed, &.{ NG, TG }, .{}), loop.iv); + const out = try decodeTokens(b, cfg.scale, q, pages, bounds, mask, keyed, t.k_cache, t.v_cache, sink); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, out)}); + } + b.ret(&.{try b.reshape(loop.results[0], &.{ T, Hkv, G, D }, .{})}); } }.run, }); +const unmasked_group_tokens = 4; + +/// Groups of at most `unmasked_group_tokens` tokens whose tokens times keys fit `unmasked_decode_keys`. +fn decodeGroups(tokens: i64, keys: i64) i64 { + var groups: i64 = 1; + while ((@divExact(tokens, groups) > unmasked_group_tokens or @divExact(tokens, groups) * keys > unmasked_decode_keys) and groups < tokens) { + groups += 1; + while (@rem(tokens, groups) != 0) groups += 1; + } + return groups; +} + +/// `keyed` scales the softmax: zero for tokens without keys. +fn decodeTokens(b: *tcl.Builder, scale: f32, q: builder.Tensor, pages: builder.Tensor, bounds: builder.Tensor, mask: builder.Tensor, keyed: builder.Tensor, k_cache: builder.Tensor, v_cache: builder.Tensor, sink: ?builder.Tensor) tcl.FinishError!builder.Tensor { + const T = q.axes[0]; + const Hkv = q.axes[1]; + const G = q.axes[2]; + const D = q.axes[3]; + const AP = pages.axes[1]; + const P = k_cache.axes[0]; + const PG = k_cache.axes[1]; + const keys: []const tcl.Axis = &.{ AP, PG }; + const key_mask = try b.reshape(mask, &.{ T, AP, PG }, .{}); + var k = try b.gather(k_cache, pages, P, &.{ T, AP, PG, Hkv, D }, .{ .valid_length = bounds }); + var v = try b.gather(v_cache, pages, P, &.{ T, AP, PG, Hkv, D }, .{ .valid_length = bounds }); + if (T.size == 1 and AP.size * PG.size <= masked_decode_keys) { + k = try maskCacheValues(b, k, key_mask); + v = try maskCacheValues(b, v, key_mask); + } + var op = b.tensorOperation(.{}); + const scores = try op.commit(op.contract(q, k, &.{ Hkv, G, T, AP, PG }).mulf(scale), .{}); + op = b.tensorOperation(.{}); + const s = op.fetch(scores, .{}); + const masked = try op.commit(op.where(op.fetch(key_mask, .{}), .ne, 0, s, score_min), .{}); + op = b.tensorOperation(.{}); + const p = if (sink) |logits| blk: { + const maximum = try op.commit(op.fetch(masked, .{}).reduce(keys, .maxf).maxf(op.fetch(logits, .{ .broadcast_to = &.{ Hkv, G, T } })), .{}); + op = b.tensorOperation(.{}); + const e = try op.commit(op.fetch(masked, .{}).subf(op.fetch(maximum, .{})).exp(), .{}); + op = b.tensorOperation(.{}); + const sink_exp = try op.commit(op.fetch(maximum, .{}).mulf(-1).addf(op.fetch(logits, .{})).exp(), .{}); + op = b.tensorOperation(.{}); + const denominator = try op.commit(op.fetch(e, .{}).reduce(keys, .addf).addf(op.fetch(sink_exp, .{})), .{}); + op = b.tensorOperation(.{}); + break :blk try op.commit(op.fetch(e, .{}).divf(op.fetch(denominator, .{})).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); + } else blk: { + const x = op.fetch(masked, .{}); + const e = x.subf(x.reduce(keys, .maxf)).exp(); + break :blk try op.commit(e.divf(e.reduce(keys, .addf)).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); + }; + op = b.tensorOperation(.{}); + return op.commit(op.contract(v, p, &.{ T, Hkv, G, D }), .{ .dtype = .bf16 }); +} + +const head_keys = 4096; +const tail_chunk_keys = 8192; + +const HybridCfg = struct { + t: i64, + s: i64, + /// Head pages, tail chunks and pages per tail chunk. + hp: i64, + nc: i64, + cp: i64, + pg: i64, + hkv: i64, + g: i64, + d: i64, + pages: i64, + scale: f32, + is_causal: bool, +}; + +fn hybridDecodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, scale: f32, is_causal: bool, plan: Plan) Tensor { + const pg = k_cache.dim(.k_chunk); + const t = q.dim(.b); + const s = parameters.seq_lens.dim(.b); + const hp = std.math.divCeil(i64, plan.head, pg) catch unreachable; + const cp = std.math.divCeil(i64, plan.chunk, pg) catch unreachable; + const nc = std.math.divCeil(i64, parameters.block_table.dim(.p) - hp, cp) catch unreachable; + const cfg: HybridCfg = .{ .t = t, .s = s, .hp = hp, .nc = nc, .cp = cp, .pg = pg, .hkv = q.dim(.hkv), .g = q.dim(.hg), .d = q.dim(.hd), .pages = k_cache.dim(.page), .scale = scale, .is_causal = is_causal }; + const table = parameters.block_table.pad(-1, .{ .p = Tensor.Pad{ .high = hp + nc * cp - parameters.block_table.dim(.p) } }); + const meta = HybridMeta.call(.{ .seq_lens = parameters.seq_lens, .query_offsets = parameters.query_start_len, .table = table }, .{ + .hpages = zml.Shape.init(.{ .t = t, .p = hp }, .i32), + .hmask = zml.Shape.init(.{ .t = t, .a = hp * pg }, .i32), + .hbounds = zml.Shape.init(.{ .t = t }, .i32), + .tpages = zml.Shape.init(.{ .c = nc, .t = t, .p = cp }, .i32), + .tmask = zml.Shape.init(.{ .c = nc, .t = t, .a = cp * pg }, .i32), + .tbounds = zml.Shape.init(.{ .c = nc, .t = t }, .i32), + .limits = zml.Shape.init(.{ .t = t }, .i32), + .keyed = zml.Shape.init(.{ .t = t }, .f32), + }, .{ .cfg = cfg }); + return HybridDecode.call(.{ + .q = q.convert(.bf16), + .hpages = meta.hpages, + .hmask = meta.hmask, + .hbounds = meta.hbounds, + .tpages = meta.tpages, + .tmask = meta.tmask, + .tbounds = meta.tbounds, + .limits = meta.limits, + .keyed = meta.keyed, + .k_cache = k_cache.convert(.bf16), + .v_cache = v_cache.convert(.bf16), + }, .{ .out = q.shape().withDtype(.bf16) }, .{ .cfg = cfg }).out; +} + +/// Per token, once per forward pass. Tail chunks are chunk-major, only those holding keys of the +/// longest token; `limits` is the visible key count. +const HybridMeta = tcl.Kernel(HybridCfg, .{ + .name = "paged_hybrid_metadata", + .inputs = &.{ "seq_lens", "query_offsets", "table" }, + .outputs = &.{ "hpages", "hmask", "hbounds", "tpages", "tmask", "tbounds", "limits", "keyed" }, + .run = struct { + fn run(b: *tcl.Builder, cfg: HybridCfg) tcl.FinishError!void { + const total = (cfg.hp + cfg.nc * cfg.cp) * cfg.pg; + const T = b.axis("T", cfg.t); + const S = b.axis("S", cfg.s); + const O = b.axis("O", cfg.s + 1); + const AP = b.axis("AP", cfg.hp + cfg.nc * cfg.cp); + const HP = b.axis("HP", cfg.hp); + const RP = b.axis("RP", cfg.nc * cfg.cp); + const NC = b.axis("NC", cfg.nc); + const CP = b.axis("CP", cfg.cp); + const HA = b.axis("HA", cfg.hp * cfg.pg); + const CK = b.axis("CK", cfg.cp * cfg.pg); + const t = try b.declareArgs(.{ + .seq_lens = .{ .dtype = .i32, .axes = &.{S} }, + .query_offsets = .{ .dtype = .i32, .axes = &.{O} }, + .table = .{ .dtype = .i32, .axes = &.{ S, AP } }, + }); + const ends = try selectMetadata(b, t.query_offsets, try selector(b, try b.arange(cfg.s + 1, .i32, &.{S}, .{ .start = 1 }), O, cfg.s + 1), O); + const tokens = try b.arange(cfg.t, .i32, &.{T}, .{}); + const seq = try countBefore(b, ends, tokens, S); + const own = try selector(b, seq, S, cfg.s); + var op = b.tensorOperation(.{}); + const active = try op.commit(op.fetch(seq, .{}).muli(-1).addi(cfg.s).binary(.maxi, 0).binary(.mini, 1), .{}); + var visible = try selectMetadata(b, t.seq_lens, own, S); + if (cfg.is_causal) { + const end = try selectMetadata(b, ends, own, S); + op = b.tensorOperation(.{}); + visible = try op.commit(op.fetch(visible, .{}).subi(op.fetch(end, .{})).addi(op.fetch(tokens, .{})).addi(1), .{}); + } + op = b.tensorOperation(.{}); + const limit = try op.commit(op.fetch(visible, .{}).binary(.maxi, 0).binary(.mini, total).muli(op.fetch(active, .{})), .{}); + + // WORKAROUND (TCC 2026.5.0b0): a 3-D one-hot product computes garbage when the one-hot is read + // first, so the selection is 2-D per chunk. Repro: repro_onehot_einsum_by_ve_wrong_values_run.py. + // Remove once it is exact on RNGD. + const rows1 = try b.reshape(try b.slice(t.table, AP, cfg.hp, &.{ S, RP }, .{}), &.{ S, NC, CP }, .{}); + op = b.tensorOperation(.{}); + const rows = try op.commit(op.fetch(rows1, .{}), .{ .axes = &.{ NC, S, CP } }); + // Trips never exceed nc: `limit` is at most the table's keys. + const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(limit), cfg.cp * cfg.pg - 1 - cfg.hp * cfg.pg), cfg.cp * cfg.pg); + var loop = b.openFor(trips, .{ try b.scratchpad(.i32, &.{ NC, T, CP }), try b.scratchpad(.i32, &.{ NC, T, CK }), try b.scratchpad(.i32, &.{ NC, T }), try b.full(0, .i32, &.{T}) }); + { + const reads, const masks, const counts, const seen = loop.carried; + const first = try b.symExpr(.add, try b.symExpr(.mul, loop.iv, cfg.cp * cfg.pg), cfg.hp * cfg.pg); + const chunk = try hybridColumns(b, cfg, own, try b.indexRead(rows, loop.iv), limit, first, CP, CK); + op = b.tensorOperation(.{}); + const any = try op.commit(op.fetch(seen, .{}).binary(.maxi, op.fetch(chunk.any, .{})), .{}); + try loop.yield(.{ + try b.indexWrite(reads, loop.iv, chunk.pages), + try b.indexWrite(masks, loop.iv, chunk.mask), + try b.indexWrite(counts, loop.iv, chunk.bounds), + any, + }); + } + const head = try hybridColumns(b, cfg, own, try b.slice(t.table, AP, 0, &.{ S, HP }, .{}), limit, 0, HP, HA); + op = b.tensorOperation(.{}); + const keyed = try op.commit(op.fetch(head.any, .{}).binary(.maxi, op.fetch(loop.results[3], .{})).toFloat(), .{}); + b.ret(&.{ head.pages, head.mask, head.bounds, loop.results[0], loop.results[1], loop.results[2], limit, keyed }); + } + }.run, +}); + +/// Columns start at key `first`; sparse gathers never read past a token's `bounds`. +fn hybridColumns(b: *tcl.Builder, cfg: HybridCfg, own: builder.Tensor, table: builder.Tensor, limit: builder.Tensor, first: anytype, cols: tcl.Axis, keys: tcl.Axis) tcl.FinishError!struct { pages: builder.Tensor, mask: builder.Tensor, bounds: builder.Tensor, any: builder.Tensor } { + const T = own.axes[0]; + const S = own.axes[1]; + const PG = b.axis("PG", cfg.pg); + const pages = try validIndices(b, try selectMetadata(b, table, own, S), cfg.pages); + var op = b.tensorOperation(.{}); + const firsts = try op.commit(op.fetch(try b.arange(cols.size * cfg.pg, .i32, &.{cols}, .{ .step = cfg.pg }), .{}).addi(if (@TypeOf(first) == builder.Tensor) op.fetch(first, .{}) else first), .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const used = try op.commit(op.fetch(limit, .{}).subi(op.fetch(firsts, .{})).binary(.maxi, 0).binary(.mini, 1), .{}); + op = b.tensorOperation(.{}); + const bounds = try op.commit(op.fetch(used, .{}).reduce(&.{cols}, .addi), .{}); + op = b.tensorOperation(.{}); + const any = try op.commit(op.fetch(used, .{}).muli(op.fetch(pages.valid, .{})).reduce(&.{cols}, .maxi), .{}); + op = b.tensorOperation(.{}); + const key_ids = try op.commit(op.fetch(try b.arange(keys.size, .i32, &.{keys}, .{}), .{}).addi(if (@TypeOf(first) == builder.Tensor) op.fetch(first, .{}) else first), .{}); + const to: []const tcl.Axis = &.{ T, cols, PG }; + op = b.tensorOperation(.{}); + const position = try op.commit(op.fetch(try b.reshape(key_ids, &.{ cols, PG }, .{}), .{ .broadcast_to = to }), .{}); + op = b.tensorOperation(.{}); + const valid = try op.commit(op.fetch(pages.valid, .{ .broadcast_to = to }), .{}); + op = b.tensorOperation(.{ .tactic = .elementwise }); + const mask = try op.commit(op.fetch(position, .{}).muli(-1).addi(op.fetch(limit, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(valid, .{})), .{}); + return .{ .pages = pages.safe, .mask = try b.reshape(mask, &.{ T, keys }, .{}), .bounds = bounds, .any = any }; +} + +const SoftmaxState = struct { max: builder.Tensor, sum: builder.Tensor, out: builder.Tensor }; + +const HybridDecode = tcl.Kernel(HybridCfg, .{ + .name = "paged_hybrid_decode_attention", + .inputs = &.{ "q", "hpages", "hmask", "hbounds", "tpages", "tmask", "tbounds", "limits", "keyed", "k_cache", "v_cache" }, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: HybridCfg) tcl.FinishError!void { + b.fuse(); + const a = hybridAxes(b, cfg); + const t = try b.declareArgs(.{ + .q = .{ .dtype = .bf16, .axes = &.{ a.T, a.Hkv, a.G, a.D } }, + .hpages = .{ .dtype = .i32, .axes = &.{ a.T, a.HP } }, + .hmask = .{ .dtype = .i32, .axes = &.{ a.T, a.HA } }, + .hbounds = .{ .dtype = .i32, .axes = &.{a.T} }, + .tpages = .{ .dtype = .i32, .axes = &.{ a.NC, a.T, a.CP } }, + .tmask = .{ .dtype = .i32, .axes = &.{ a.NC, a.T, a.CK } }, + .tbounds = .{ .dtype = .i32, .axes = &.{ a.NC, a.T } }, + .limits = .{ .dtype = .i32, .axes = &.{a.T} }, + .keyed = .{ .dtype = .f32, .axes = &.{a.T} }, + .k_cache = .{ .dtype = .bf16, .axes = &.{ a.P, a.PG, a.Hkv, a.D } }, + .v_cache = .{ .dtype = .bf16, .axes = &.{ a.P, a.PG, a.Hkv, a.D } }, + }); + const tail = try hybridTail(b, cfg, a, t.q, t.tpages, t.tmask, t.tbounds, t.limits, t.k_cache, t.v_cache); + b.ret(&.{try hybridHead(b, cfg, a, t.q, t.hpages, t.hmask, t.hbounds, t.keyed, t.k_cache, t.v_cache, tail)}); + } + }.run, +}); + +const HybridAxes = struct { T: tcl.Axis, HP: tcl.Axis, HA: tcl.Axis, NC: tcl.Axis, CP: tcl.Axis, CK: tcl.Axis, Hkv: tcl.Axis, G: tcl.Axis, D: tcl.Axis, P: tcl.Axis, PG: tcl.Axis }; + +fn hybridAxes(b: *tcl.Builder, cfg: HybridCfg) HybridAxes { + return .{ + .T = b.axis("T", cfg.t), + .HP = b.axis("HP", cfg.hp), + .HA = b.axis("HA", cfg.hp * cfg.pg), + .NC = b.axis("NC", cfg.nc), + .CP = b.axis("CP", cfg.cp), + .CK = b.axis("CK", cfg.cp * cfg.pg), + .Hkv = b.axis("Hkv", cfg.hkv), + .G = b.axis("G", cfg.g), + .D = b.axis("D", cfg.d), + .P = b.axis("P", cfg.pages), + .PG = b.axis("PG", cfg.pg), + }; +} + +fn hybridTail(b: *tcl.Builder, cfg: HybridCfg, a: HybridAxes, q: builder.Tensor, tpages: builder.Tensor, tmask: builder.Tensor, tbounds: builder.Tensor, limits: builder.Tensor, k_cache: builder.Tensor, v_cache: builder.Tensor) tcl.FinishError!SoftmaxState { + const stats: []const tcl.Axis = &.{ a.Hkv, a.G, a.T }; + const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(limits), cfg.cp * cfg.pg - 1 - cfg.hp * cfg.pg), cfg.cp * cfg.pg); + var loop = b.openFor(trips, .{ try b.full(score_min, .f32, stats), try b.full(0, .f32, stats), try b.full(0, .f32, &.{ a.T, a.Hkv, a.G, a.D }) }); + { + const max, const sum, const out = loop.carried; + const mask = try b.reshape(try b.indexRead(tmask, loop.iv), &.{ a.T, a.CP, a.PG }, .{}); + const bounds = try b.indexRead(tbounds, loop.iv); + const pages = try b.indexRead(tpages, loop.iv); + const k = try b.gather(k_cache, pages, a.P, &.{ a.T, a.CP, a.PG, a.Hkv, a.D }, .{ .valid_length = bounds }); + const v = try b.gather(v_cache, pages, a.P, &.{ a.T, a.CP, a.PG, a.Hkv, a.D }, .{ .valid_length = bounds }); + const merged = try mergeSoftmax(b, .{ .max = max, .sum = sum, .out = out }, try chunkSoftmax(b, cfg, q, k, v, mask)); + try loop.yield(.{ merged.max, merged.sum, merged.out }); + } + return .{ .max = loop.results[0], .sum = loop.results[1], .out = loop.results[2] }; +} + +fn hybridHead(b: *tcl.Builder, cfg: HybridCfg, a: HybridAxes, q: builder.Tensor, hpages: builder.Tensor, hmask: builder.Tensor, hbounds: builder.Tensor, keyed: builder.Tensor, k_cache: builder.Tensor, v_cache: builder.Tensor, tail: SoftmaxState) tcl.FinishError!builder.Tensor { + const head_mask = try b.reshape(hmask, &.{ a.T, a.HP, a.PG }, .{}); + const head_k = try b.gather(k_cache, hpages, a.P, &.{ a.T, a.HP, a.PG, a.Hkv, a.D }, .{ .valid_length = hbounds }); + const head_v = try b.gather(v_cache, hpages, a.P, &.{ a.T, a.HP, a.PG, a.Hkv, a.D }, .{ .valid_length = hbounds }); + const keys: []const tcl.Axis = &.{ a.HP, a.PG }; + var op = b.tensorOperation(.{}); + const scores = try op.commit(op.contract(q, head_k, &.{ a.Hkv, a.G, a.T, a.HP, a.PG }).mulf(cfg.scale), .{}); + op = b.tensorOperation(.{}); + const sv = op.fetch(scores, .{}); + const masked = try op.commit(op.where(op.fetch(head_mask, .{}), .ne, 0, sv, score_min), .{}); + op = b.tensorOperation(.{}); + const max = try op.commit(op.fetch(masked, .{}).reduce(keys, .maxf).maxf(op.fetch(tail.max, .{})), .{}); + const alpha = try expDifference(b, tail.max, max); + op = b.tensorOperation(.{}); + const carried = try op.commit(op.fetch(tail.sum, .{}).mulf(op.fetch(alpha, .{})), .{}); + op = b.tensorOperation(.{}); + const denominator = try op.commit(op.fetch(masked, .{}).subf(op.fetch(max, .{})).exp().reduce(keys, .addf).addf(op.fetch(carried, .{})), .{}); + op = b.tensorOperation(.{}); + const p = try op.commit(op.fetch(masked, .{}).subf(op.fetch(max, .{})).exp().divf(op.fetch(denominator, .{})).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); + op = b.tensorOperation(.{}); + const head = try op.commit(op.contract(head_v, p, &.{ a.T, a.Hkv, a.G, a.D }), .{}); + op = b.tensorOperation(.{}); + const factor = try op.commit(op.fetch(alpha, .{}).divf(op.fetch(denominator, .{})), .{}); + op = b.tensorOperation(.{}); + return op.commit(op.fetch(tail.out, .{}).mulf(op.fetch(factor, .{})).addf(op.fetch(head, .{})).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); +} + +fn chunkSoftmax(b: *tcl.Builder, cfg: HybridCfg, q: builder.Tensor, k: builder.Tensor, v: builder.Tensor, mask: builder.Tensor) tcl.FinishError!SoftmaxState { + const T = q.axes[0]; + const Hkv = q.axes[1]; + const G = q.axes[2]; + const D = q.axes[3]; + const keys = mask.axes[1..]; + var op = b.tensorOperation(.{}); + const scores = try op.commit(op.contract(q, k, &.{ Hkv, G, T, keys[0], keys[1] }).mulf(cfg.scale), .{}); + op = b.tensorOperation(.{}); + const sv = op.fetch(scores, .{}); + const masked = try op.commit(op.where(op.fetch(mask, .{}), .ne, 0, sv, score_min), .{}); + op = b.tensorOperation(.{}); + const max = try op.commit(op.fetch(masked, .{}).reduce(keys, .maxf), .{}); + const exps = try maskedExp(b, masked, max, mask); + op = b.tensorOperation(.{}); + const sum = try op.commit(op.fetch(exps, .{}).reduce(keys, .addf), .{}); + op = b.tensorOperation(.{}); + const p = try op.commit(op.fetch(exps, .{}).mulf(1.0), .{ .dtype = .bf16 }); + op = b.tensorOperation(.{}); + const out = try op.commit(op.contract(v, p, &.{ T, Hkv, G, D }), .{}); + return .{ .max = max, .sum = sum, .out = out }; +} + +fn mergeSoftmax(b: *tcl.Builder, x: SoftmaxState, y: SoftmaxState) tcl.FinishError!SoftmaxState { + var op = b.tensorOperation(.{ .tactic = .elementwise }); + const max = try op.commit(op.fetch(x.max, .{}).maxf(op.fetch(y.max, .{})), .{}); + const alpha = try expDifference(b, x.max, max); + const beta = try expDifference(b, y.max, max); + op = b.tensorOperation(.{}); + const xs = try op.commit(op.fetch(x.sum, .{}).mulf(op.fetch(alpha, .{})), .{}); + op = b.tensorOperation(.{}); + const sum = try op.commit(op.fetch(y.sum, .{}).mulf(op.fetch(beta, .{})).addf(op.fetch(xs, .{})), .{}); + op = b.tensorOperation(.{}); + const xo = try op.commit(op.fetch(x.out, .{}).mulf(op.fetch(alpha, .{})), .{}); + op = b.tensorOperation(.{}); + const out = try op.commit(op.fetch(y.out, .{}).mulf(op.fetch(beta, .{})).addf(op.fetch(xo, .{})), .{}); + return .{ .max = max, .sum = sum, .out = out }; +} + const Cfg = struct { tokens: i64, sequences: i64, @@ -299,11 +715,9 @@ const Cfg = struct { window: i32 = -1, }; -/// Attention tuning inherited from the original TCL backend, informed by -/// `furiosa.kernels.llama.config` and `common.attention`. Keep the compiler's -/// default lowering mode and omit the adaptive-einsum and half-VRF settings -/// removed in TCC 2026.5. +/// Prefill bounds the tactic shapes searched (fewer for narrow heads): the compute-bound hint compiles too slowly. fn configure(b: *tcl.Builder, cfg: Cfg) void { + const shapes: i64 = if (cfg.hkv * cfg.g * cfg.d >= 1024) 16 else 8; if (cfg.decode) b.compilerConfig(.{ .enable_einsum_fusion = true, .padding_policy = .{ .Small = 1.1 }, @@ -326,11 +740,10 @@ fn configure(b: *tcl.Builder, cfg: Cfg) void { .reshape_einsum_mode = .{ .Reshape = .{ .permute = false } }, .tensor_unit_bridge_threshold_in_page = 12, .allow_external_operators = false, - .enable_tactic_pruning = false, .scheduler_beam_search = true, .allow_reduce_by_ve_cluster_chip_reduce = false, .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, - .tactic_hint = .{ .ForLlmModelComputeBound = 200 }, + .tactic_hint = .{ .DefaultWithMaxNumUniqueShapes = shapes }, .dma_preference = 1.2, }); } @@ -390,11 +803,24 @@ fn declareInputs(b: *tcl.Builder, a: Axes) tcl.FinishError!NativeInputs { const Prepared = struct { q: builder.Tensor, + /// Page table rows: one per block for decode, one per sequence for prefill (blocks select theirs with `pick`). pages: builder.Tensor, + pick: ?builder.Tensor, qpos: builder.Tensor, kvlen: builder.Tensor, row: builder.Tensor, active: builder.Tensor, + /// Per page row: the first query position and the keys. + first: builder.Tensor, + keys: builder.Tensor, + + /// WORKAROUND (TCC 2026.5.0b0): TCC has no tactic for selecting whole rows of 8192 pages, so flash + /// attention selects per chunk. Repro: repro_metadata_einsum_by_ve_no_tactic.py. Remove once such + /// products compile. + fn blockPages(self: Prepared, b: *tcl.Builder, a: Axes, pages: builder.Tensor) tcl.FinishError!builder.Tensor { + const pick = self.pick orelse return pages; + return selectMetadata(b, pages, pick, a.S); + } }; fn countBefore(b: *tcl.Builder, ends: builder.Tensor, positions: builder.Tensor, axis: tcl.Axis) tcl.FinishError!builder.Tensor { @@ -414,15 +840,16 @@ fn selector(b: *tcl.Builder, indices: builder.Tensor, axis: tcl.Axis, count: i64 return op.commit(op.where(distance, .eq, 0, 1, 0), .{}); } -// Keep metadata exact in I32. Scalar gathers and four-byte-offset slices do -// not have native tactics for these shapes in TCC 2026.5. +// Keep metadata exact in I32. WORKAROUND (TCC 2026.5.0b0): no native tactics for +// scalar gathers or four-byte-offset slices. Repro: repro_scalar_gather_no_axis_to_pad.py, +// repro_i32_slice_odd_offset.py. Remove once both compile. fn selectMetadata(b: *tcl.Builder, values: builder.Tensor, selected: builder.Tensor, axis: tcl.Axis) tcl.FinishError!builder.Tensor { const op = b.tensorOperation(.{ .tactic = if (values.axes.len > 1) .einsum_by_ve else .reduce_by_ve }); return op.commit(op.fetch(selected, .{}).muli(op.fetch(values, .{})).reduce(&.{axis}, .addi), .{}); } -/// Keep adjacent I32 reads inside the native unit. Exposing offsets[1:] as a -/// separate argument would require alignment staging at every invocation. +/// Reads adjacent I32 offsets inside the native unit: a separate `offsets[1:]` argument would +/// need alignment staging at every invocation. fn prepareQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, t: NativeInputs) tcl.FinishError!Prepared { const start_indices = try b.arange(cfg.sequences, .i32, &.{a.S}, .{}); const end_indices = try b.arange(cfg.sequences + 1, .i32, &.{a.S}, .{ .start = 1 }); @@ -432,8 +859,7 @@ fn prepareQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, t: NativeInputs) tcl.Finis const qlen = try op.commit(op.fetch(ends, .{}).subi(op.fetch(starts, .{})), .{}); const tokens = try b.arange(cfg.tokens, .i32, &.{a.T}, .{}); const seq_of_token = try countBefore(b, ends, tokens, a.S); - // All sequence ends precede a padding token. Reuse this vector instead of - // bringing a symbolic scalar reduction back into a vector operation. + // Padding tokens follow all sequence ends; this vector avoids a symbolic scalar reduction in a vector operation. op = b.tensorOperation(.{}); const token_tail = try op.commit(op.fetch(seq_of_token, .{}).subi(cfg.sequences), .{}); op = b.tensorOperation(.{}); @@ -453,16 +879,19 @@ fn prepareQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, t: NativeInputs) tcl.Finis return .{ .q = try b.reshape(q, &.{ a.B, a.I, a.Hkv, a.G, a.D }, .{}), .pages = try b.reshape(t.pages, &.{ a.B, a.AP }, .{}), + .pick = null, .qpos = try b.reshape(qpos, &.{ a.B, a.I }, .{}), .kvlen = try b.reshape(lengths, &.{a.B}, .{}), .row = token_seq, .active = active, + .first = try b.reshape(qpos, &.{a.B}, .{}), + .keys = try b.reshape(lengths, &.{a.B}, .{}), }; } - const shift = std.math.log2_int(u64, prefill_block); + const shift = std.math.log2_int(u64, @intCast(cfg.i)); op = b.tensorOperation(.{}); - const blocks = try op.commit(op.fetch(qlen, .{}).addi(prefill_block - 1).binary(.shr_arith, shift), .{}); + const blocks = try op.commit(op.fetch(qlen, .{}).addi(cfg.i - 1).binary(.shr_arith, shift), .{}); op = b.tensorOperation(.{}); const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{a.S}), .{}); op = b.tensorOperation(.{}); @@ -482,7 +911,7 @@ fn prepareQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, t: NativeInputs) tcl.Finis const key_length = try selectMetadata(b, t.seq_lens, pick, a.S); const local_rows = try b.arange(cfg.i, .i32, &.{a.I}, .{}); op = b.tensorOperation(.{}); - const first_row = try op.commit(op.fetch(block_ids, .{}).subi(op.fetch(block_begin, .{})).muli(prefill_block), .{}); + const first_row = try op.commit(op.fetch(block_ids, .{}).subi(op.fetch(block_begin, .{})).muli(cfg.i), .{}); op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); const qi = try op.commit(op.fetch(first_row, .{}).addi(op.fetch(local_rows, .{})), .{}); op = b.tensorOperation(.{}); @@ -500,14 +929,15 @@ fn prepareQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, t: NativeInputs) tcl.Finis const qpos = try op.commit(op.where(live, .ne, 0, op.fetch(pos, .{}), -1), .{}); op = b.tensorOperation(.{}); const kvlen = try op.commit(op.fetch(key_length, .{}).muli(op.fetch(live_block, .{})), .{}); - const table = try selectMetadata(b, t.pages, pick, a.S); const token_pick = try selector(b, token_seq, a.S, cfg.sequences); const token_begin = try selectMetadata(b, starts, token_pick, a.S); const token_block = try selectMetadata(b, block_start, token_pick, a.S); op = b.tensorOperation(.{}); - const row = try op.commit(op.fetch(token_block, .{}).muli(prefill_block).addi(op.fetch(tokens, .{})).subi(op.fetch(token_begin, .{})).binary(.maxi, 0).binary(.mini, cfg.b * cfg.i - 1), .{}); - return .{ .q = q, .pages = table, .qpos = qpos, .kvlen = kvlen, .row = row, .active = active }; + const row = try op.commit(op.fetch(token_block, .{}).muli(cfg.i).addi(op.fetch(tokens, .{})).subi(op.fetch(token_begin, .{})).binary(.maxi, 0).binary(.mini, cfg.b * cfg.i - 1), .{}); + op = b.tensorOperation(.{}); + const first = try op.commit(op.fetch(t.seq_lens, .{}).subi(op.fetch(qlen, .{})), .{}); + return .{ .q = q, .pages = t.pages, .pick = pick, .qpos = qpos, .kvlen = kvlen, .row = row, .active = active, .first = first, .keys = t.seq_lens }; } fn restoreQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, prepared: Prepared, output: builder.Tensor) tcl.FinishError!builder.Tensor { @@ -533,8 +963,7 @@ fn validIndices(b: *tcl.Builder, indices: builder.Tensor, count: i64) tcl.Finish return .{ .safe = safe, .valid = valid }; } -/// Cache validity is per sequence, not per query: causal/window masking stays -/// on scores, without replicating K/V for every row of a prefill block. +/// Validity is per sequence, not per query, so K/V are not replicated per row of a prefill block. fn cacheMask(b: *tcl.Builder, valid_pages: builder.Tensor, kvlen: builder.Tensor, position: builder.Tensor) tcl.FinishError!builder.Tensor { var op = b.tensorOperation(.{}); const pos = try op.commit(op.fetch(position, .{ .broadcast_to = valid_pages.axes }), .{}); @@ -542,17 +971,15 @@ fn cacheMask(b: *tcl.Builder, valid_pages: builder.Tensor, kvlen: builder.Tensor return op.commit(op.fetch(pos, .{}).muli(-1).addi(op.fetch(kvlen, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(valid_pages, .{})), .{}); } -/// A true select excludes NaN/Inf in unwritten slots; multiplying by zero -/// would still propagate them into the contraction. Materialize only BF16. +/// A select, not a multiply by zero: NaN/Inf in unwritten slots must not reach the contraction. fn maskCacheValues(b: *tcl.Builder, values: builder.Tensor, valid: builder.Tensor) tcl.FinishError!builder.Tensor { const op = b.tensorOperation(.{ .tactic = .elementwise }); const data = op.fetch(values, .{}); return op.commit(op.where(op.fetch(valid, .{ .broadcast_to = values.axes }), .ne, 0, data, 0.0), .{ .dtype = .bf16 }); } -/// Build the sequence, query, page and window mask for the current keys. fn attentionMask(b: *tcl.Builder, cfg: Cfg, qpos: builder.Tensor, kvlen: builder.Tensor, position: builder.Tensor, valid_keys: builder.Tensor) tcl.FinishError!builder.Tensor { - const axes = &.{ qpos.axes[0], qpos.axes[1], position.axes[0] }; + const axes = &.{ qpos.axes[0], qpos.axes[1], position.axes[position.axes.len - 1] }; const read: builder.TensorOperation.FetchOptions = .{ .broadcast_to = axes }; // Elementwise operations require their first read to cover the output // axes, and each VE instruction must consume the current accumulator. @@ -577,8 +1004,6 @@ fn attentionMask(b: *tcl.Builder, cfg: Cfg, qpos: builder.Tensor, kvlen: builder return mask; } -/// Keep the contraction accumulator in f32 through masking and softmax. -/// A finite sentinel also makes entirely masked chunks safe to rescale. fn maskedScores(b: *tcl.Builder, a: Axes, cfg: Cfg, q: builder.Tensor, k: builder.Tensor, mask: builder.Tensor, keys: tcl.Axis) tcl.FinishError!builder.Tensor { const scores_axes = &.{ a.Hkv, a.G, a.B, a.I, keys }; var op = b.tensorOperation(.{}); @@ -591,8 +1016,7 @@ fn maskedScores(b: *tcl.Builder, a: Axes, cfg: Cfg, q: builder.Tensor, k: builde fn maskedExp(b: *tcl.Builder, scores: builder.Tensor, maximum: builder.Tensor, mask: builder.Tensor) tcl.FinishError!builder.Tensor { const op = b.tensorOperation(.{}); const e = op.fetch(scores, .{}).subf(op.fetch(maximum, .{})).exp(); - // Subtracting the sentinel from itself gives exp(0), so masking only - // the logits is insufficient for empty rows and fully masked chunks. + // A sentinel minus itself gives exp(0), so masking only the logits fails for empty rows and fully masked chunks. return op.commit(op.where(op.fetch(mask, .{}), .ne, 0, e, 0.0), .{}); } @@ -622,7 +1046,7 @@ const OneShot = tcl.Kernel(Cfg, .{ const prepared = try prepareQueries(b, cfg, a, t); const kc = try b.reshape(t.k_cache, &.{ Nb, a.Hkv, a.D }, .{}); const vc = try b.reshape(t.v_cache, &.{ Nb, a.Hkv, a.D }, .{}); - const page_indices = try validIndices(b, prepared.pages, cfg.pages); + const page_indices = try validIndices(b, try prepared.blockPages(b, a, prepared.pages), cfg.pages); const within_page = try b.arange(cfg.pg, .i32, &.{a.PG}, .{}); var slot_op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); const flat_slots = try slot_op.commit(slot_op.fetch(page_indices.safe, .{}).muli(cfg.pg).addi(slot_op.fetch(within_page, .{})), .{}); @@ -630,7 +1054,6 @@ const OneShot = tcl.Kernel(Cfg, .{ slot_op = b.tensorOperation(.{}); const valid = try slot_op.commit(slot_op.fetch(page_indices.valid, .{ .broadcast_to = &.{ a.B, a.AP, a.PG } }), .{}); const valid_keys = try b.reshape(valid, &.{ a.B, A }, .{}); - // Match the original slot sanitization before reading invalid pages. slot_op = b.tensorOperation(.{}); const slots = try slot_op.commit(slot_op.where(valid_keys, .ne, 0, slot_op.fetch(expanded_slots, .{}), 0), .{}); const valid_length = if (cfg.decode) prepared.kvlen else null; @@ -642,8 +1065,6 @@ const OneShot = tcl.Kernel(Cfg, .{ const masked = try maskedScores(b, a, cfg, prepared.q, k, mask, A); var op = b.tensorOperation(.{}); const p = if (cfg.decode) compact: { - // Furiosa's common.softmax keeps normalization inside one VE - // operation. FP32 intermediates only narrow at the DPE input. const exp_sink = if (cfg.use_sink) sink: { const maximum = try op.commit(op.fetch(masked, .{}).reduce(&.{A}, .maxf).maxf(op.fetch(t.sink, .{})), .{}); break :sink try expDifference(b, t.sink, maximum); @@ -656,13 +1077,11 @@ const OneShot = tcl.Kernel(Cfg, .{ var sum = e.reduce(&.{A}, .addf); if (exp_sink) |s| sum = sum.addf(op.fetch(s, .{})); const probabilities = try op.commit(e.divf(sum), .{ .dtype = .bf16 }); - // Empty rows have uniform probabilities from finite sentinels. - // Clear masked probabilities before contracting with values. + // Empty rows have uniform probabilities from finite sentinels: clear masked ones before the value contraction. op = b.tensorOperation(.{}); break :compact try op.commit(op.where(op.fetch(mask, .{}), .ne, 0, op.fetch(probabilities, .{}), 0.0), .{ .dtype = .bf16 }); } else tiled: { - // Separate reductions schedule better for multi-query blocks - // with the current compiler; keep their intermediate state FP32. + // Separate reductions schedule better for multi-query blocks with the current compiler. var m = op.fetch(masked, .{}).reduce(&.{A}, .maxf); if (cfg.use_sink) m = m.maxf(op.fetch(t.sink, .{})); const maximum = try op.commit(m, .{}); @@ -695,28 +1114,66 @@ const Flash = tcl.Kernel(Cfg, .{ const Cp = b.axis("Cp", @divExact(cfg.chunk, cfg.pg)); const t = try declareInputs(b, a); const prepared = try prepareQueries(b, cfg, a, t); - const by_batch = try b.reshape(prepared.pages, &.{ a.B, C, Cp }, .{}); + const rows = prepared.pages.axes[0]; + const by_row = try b.reshape(prepared.pages, &.{ rows, C, Cp }, .{}); const page_op = b.tensorOperation(.{}); - const pages = try page_op.commit(page_op.fetch(by_batch, .{}), .{ .axes = &.{ C, a.B, Cp } }); + const pages = try page_op.commit(page_op.fetch(by_row, .{}), .{ .axes = &.{ C, rows, Cp } }); const stats = &.{ a.Hkv, a.G, a.B, a.I }; const out_axes = &.{ a.B, a.I, a.Hkv, a.G, a.D }; var op = b.tensorOperation(.{}); - const n = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(prepared.kvlen), cfg.chunk - 1), cfg.chunk); + const shift = std.math.log2_int(u64, @intCast(cfg.chunk)); + // With a window, each page row's loop starts at the chunk holding its first visible key. TCL divides + // by shifts, so chunks that are not a power of two run every chunk and the mask applies the window. + const window = if (cfg.window > 0 and std.math.isPowerOfTwo(cfg.chunk)) blk: { + op = b.tensorOperation(.{}); + const start = try op.commit(op.fetch(prepared.first, .{}).subi(cfg.window - 1).binary(.maxi, 0).binary(.shr_arith, shift), .{}); + op = b.tensorOperation(.{}); + const count = try op.commit(op.fetch(prepared.keys, .{}).addi(cfg.chunk - 1).binary(.shr_arith, shift).subi(op.fetch(start, .{})), .{}); + // WORKAROUND (TCC 2026.5.0b0): TCC reads the loop index only as an index, so each trip's chunk per + // row is built before the loop. Repro: repro_loop_index_as_data.py. Remove once the loop index + // compiles as data. + const trip = try b.arange(C.size, .i32, &.{C}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const chunks = try op.commit(op.fetch(trip, .{}).addi(op.fetch(start, .{})), .{ .axes = &.{ C, rows } }); + break :blk .{ .chunks = chunks, .trips = try b.reduceMaxI32(count) }; + } else null; + const n = if (window) |w| w.trips else try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(prepared.kvlen), cfg.chunk - 1), cfg.chunk); var loop = b.openFor(n, .{ try b.full(0, .f32, stats), try b.full(0, .f32, out_axes), try b.full(score_min, .f32, stats) }); { const sum_acc, const qkv_acc, const max_acc = loop.carried; - const indices = try validIndices(b, try b.indexRead(pages, loop.iv), cfg.pages); + const local_pos = try b.arange(cfg.chunk, .i32, &.{Ck}, .{}); + const local_pages = try b.arange(@divExact(cfg.chunk, cfg.pg), .i32, &.{Cp}, .{}); + var row_pages: builder.Tensor = undefined; + var position: builder.Tensor = undefined; + var first_keys: builder.Tensor = undefined; + if (window) |w| { + const chunk_index = try b.indexRead(w.chunks, loop.iv); + op = b.tensorOperation(.{}); + const read_index = try op.commit(op.fetch(chunk_index, .{}).binary(.mini, @divExact(cfg.a, cfg.chunk) - 1), .{}); + // WORKAROUND (TCC 2026.5.0b0): a batched gather of the chunk does not lower + // for some shapes. Repro: repro_batched_gather_all_tactics_failed.py. Remove + // once batched gathers of several rows compile. + row_pages = try selectMetadata(b, by_row, try selector(b, read_index, C, C.size), C); + op = b.tensorOperation(.{}); + const row_offset = try op.commit(op.fetch(chunk_index, .{}).muli(cfg.chunk), .{}); + const offset = if (prepared.pick) |pick| try selectMetadata(b, row_offset, pick, a.S) else row_offset; + op = b.tensorOperation(.{}); + position = try op.commit(op.fetch(local_pos, .{ .broadcast_to = &.{ a.B, Ck } }).addi(op.fetch(offset, .{})), .{}); + op = b.tensorOperation(.{}); + first_keys = try op.commit(op.fetch(local_pages, .{ .broadcast_to = &.{ a.B, Cp } }).muli(cfg.pg).addi(op.fetch(offset, .{})), .{}); + } else { + row_pages = try b.indexRead(pages, loop.iv); + const offset = try b.symExpr(.mul, loop.iv, cfg.chunk); + op = b.tensorOperation(.{}); + position = try op.commit(op.fetch(local_pos, .{}).addi(op.fetch(offset, .{})), .{}); + op = b.tensorOperation(.{}); + first_keys = try op.commit(op.fetch(local_pages, .{}).muli(cfg.pg).addi(op.fetch(offset, .{})), .{}); + } + const indices = try validIndices(b, try prepared.blockPages(b, a, row_pages), cfg.pages); const idx = indices.safe; op = b.tensorOperation(.{}); const valid_pages = try op.commit(op.fetch(indices.valid, .{ .broadcast_to = &.{ a.B, Cp, a.PG } }), .{}); const valid_keys = try b.reshape(valid_pages, &.{ a.B, Ck }, .{}); - const offset = try b.symExpr(.mul, loop.iv, cfg.chunk); - const local_pos = try b.arange(cfg.chunk, .i32, &.{Ck}, .{}); - op = b.tensorOperation(.{}); - const position = try op.commit(op.fetch(local_pos, .{}).addi(op.fetch(offset, .{})), .{}); - const local_pages = try b.arange(@divExact(cfg.chunk, cfg.pg), .i32, &.{Cp}, .{}); - op = b.tensorOperation(.{}); - const first_keys = try op.commit(op.fetch(local_pages, .{}).muli(cfg.pg).addi(op.fetch(offset, .{})), .{}); op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); const remaining = try op.commit(op.fetch(prepared.kvlen, .{}).subi(op.fetch(first_keys, .{})), .{}); op = b.tensorOperation(.{}); @@ -724,9 +1181,8 @@ const Flash = tcl.Kernel(Cfg, .{ op = b.tensorOperation(.{}); const cvl = try op.commit(op.fetch(live_pages, .{}).reduce(&.{Cp}, .addi), .{}); const mask = try attentionMask(b, cfg, prepared.qpos, prepared.kvlen, position, valid_keys); - const valid_cache = try cacheMask(b, valid_keys, prepared.kvlen, position); - const k = try maskCacheValues(b, try b.reshape(try b.gather(t.k_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}), valid_cache); - const v = try maskCacheValues(b, try b.reshape(try b.gather(t.v_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}), valid_cache); + const k = try b.reshape(try b.gather(t.k_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}); + const v = try b.reshape(try b.gather(t.v_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}); const masked = try maskedScores(b, a, cfg, prepared.q, k, mask, Ck); op = b.tensorOperation(.{}); const max_cur = try op.commit(op.fetch(masked, .{}).reduce(&.{Ck}, .maxf), .{}); @@ -751,7 +1207,6 @@ const Flash = tcl.Kernel(Cfg, .{ var sum = loop.results[0]; var numerator = loop.results[1]; if (cfg.use_sink) { - // Merge the zero-valued sink once, after all key chunks. op = b.tensorOperation(.{}); const maximum = try op.commit(op.fetch(loop.results[2], .{}).maxf(op.fetch(t.sink, .{})), .{}); const alpha = try expDifference(b, loop.results[2], maximum); @@ -922,6 +1377,7 @@ const TestDims = struct { page: usize = 16, uniform: bool = false, uniform_scores: bool = false, + spikes: []const usize = &.{}, dtype: zml.DataType = .bf16, }; @@ -1030,7 +1486,7 @@ test "tcl paged attention cache masking emits a BF16 select without query replic try std.testing.expect(std.mem.indexOf(u8, ir, "#tcl.ve_opcode<\"mulf\">") == null); try std.testing.expect(std.mem.indexOf(u8, ir, "#tcl.symbol<\"I\">") == null); - const decode = try Decode.emit(std.testing.allocator, .{ .t = 2, .a = 64, .hkv = 2, .g = 4, .d = 32, .pages = 8, .pg = 16, .scale = 0.125, .is_causal = true }); + const decode = try Decode.emit(std.testing.allocator, .{ .t = 2, .a = 64, .group = 2, .hkv = 2, .g = 4, .d = 32, .pages = 8, .pg = 16, .scale = 0.125, .is_causal = true, .window = 0, .use_sink = false }); defer std.testing.allocator.free(decode); try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, decode, "\"tcl.dpe\"")); try std.testing.expect(std.mem.indexOf(u8, decode, "tcl.fusible") != null); @@ -1143,6 +1599,32 @@ test "tcl paged attention chunked decode on furiosa" { try runCases(257, 514, &cases, .{ .hkv = 1, .g = 1, .d = 32 }); } +test "tcl paged attention hybrid decode on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + try runCases(2049, 2101, &.{ + .{ .prefill = false, .seqs = &.{.{ 1, 300 }}, .poison = .both }, + .{ .prefill = false, .seqs = &.{.{ 1, 4097 }}, .poison = .both }, + .{ .prefill = false, .seqs = &.{.{ 1, 32784 }}, .poison = .both, .poison_value = std.math.inf(f32) }, + .{ .prefill = false, .seqs = &.{.{ 0, 0 }}, .padding = 1, .poison = .both }, + .{ .prefill = false, .seqs = &.{.{ 1, 17 }}, .mask_all = true, .poison = .both }, + }, .{ .batch = 1, .hkv = 1, .g = 2, .d = 32 }); + try runCases(2049, 2101, &.{.{ .prefill = false, .seqs = &.{.{ 1, 32784 }}, .poison = .both }}, .{ .batch = 1, .hkv = 1, .g = 2, .d = 32, .uniform_scores = true, .spikes = &.{ 4095, 4096, 12287, 12288, 32783 } }); +} + +test "tcl paged attention flash decode of many tokens on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + const seqs = [_][2]i32{ .{ 1, 32784 }, .{ 1, 4097 }, .{ 1, 1 }, .{ 0, 0 } } ++ @as([12][2]i32, @splat(.{ 1, 300 })); + try runCases(2049, 2600, &.{ + .{ .prefill = false, .seqs = &seqs, .padding = 1, .poison = .both }, + .{ .prefill = false, .seqs = &seqs, .padding = 1, .poison = .both, .opts = .{ .sliding_window = 1024 } }, + .{ .prefill = false, .seqs = &seqs, .padding = 1, .poison = .both, .opts = .{ .sliding_window = 5000, .sink = &.{ 0, -std.math.inf(f32) } } }, + }, .{ .batch = 16, .hkv = 1, .g = 2, .d = 32 }); + try runCases(2049, 2700, &.{ + .{ .prefill = true, .seqs = &.{ .{ 37, 32784 }, .{ 5, 9000 } }, .padding = 2, .poison = .both, .opts = .{ .sliding_window = 1024 } }, + .{ .prefill = true, .seqs = &.{ .{ 70, 32784 }, .{ 0, 0 } }, .padding = 1, .poison = .both, .opts = .{ .sliding_window = 3000 } }, + }, .{ .batch = 2, .hkv = 1, .g = 2, .d = 32 }); +} + test "tcl paged attention uniform probabilities on furiosa" { if (zml.testing.env().target != .furiosa) return error.SkipZigTest; try runCases(257, 514, &.{.{ .prefill = false, .seqs = &.{ .{ 1, 64 }, .{ 1, 65 }, .{ 1, 66 }, .{ 0, 0 } }, .padding = 1 }}, .{ .hkv = 1, .g = 1, .d = 32, .uniform_scores = true }); @@ -1209,9 +1691,16 @@ fn runCases(comptime max_pages: usize, comptime num_pages: usize, cases: []const for (kc) |*v| v.* = .fromF32(if (dims.uniform or dims.uniform_scores) 0 else r.floatNorm(f32)); for (vc, 0..) |*v, i| v.* = .fromF32(if (dims.uniform) @as(f32, @floatFromInt(i % d)) / d else r.floatNorm(f32)); + if (dims.spikes.len > 0) { + for (case.seqs[0..batch], 0..) |seq, row| { + for (0..@intCast(seq[1])) |key| { + const slot: usize = @intCast(table[row][key / page] * @as(i32, page) + @as(i32, @intCast(key % page))); + const spike = std.mem.indexOfScalar(usize, dims.spikes, key) != null; + @memset(vc[slot * hkv * d ..][0 .. hkv * d], bf16.fromF32(if (spike) 32768 else 0)); + } + } + } if (case.poison != .none) { - // Mark the union of live physical slots before poisoning: reordered - // or shared pages must retain every token a live sequence can read. const live = try allocator.alloc(bool, num_pages * page); defer allocator.free(live); @memset(live, false); From 8898051c7b6b8cd015cdce8e75afc5404396405f Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Wed, 4 Nov 2026 20:48:56 +0100 Subject: [PATCH 846/881] zml/kernels: fix Triton split-K (#1012) The split-K/3D path writes partial results only for active queries, but the reduction kernel processes inactive queries in the padded batch. These queries have no valid results and cause invalid computation/outputs. This fix zeroes the ouput for inactive queries and return before before accessing sequence metadata. Resolves #1006 --- .../triton/unified_attention.zig | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/zml/kernels/paged_attention/triton/unified_attention.zig b/zml/kernels/paged_attention/triton/unified_attention.zig index 39018f1b..7b2b9428 100644 --- a/zml/kernels/paged_attention/triton/unified_attention.zig +++ b/zml/kernels/paged_attention/triton/unified_attention.zig @@ -1020,6 +1020,25 @@ fn reduceSegments( const query_token_idx = k.programId(.x); const query_head_idx = k.programId(.y); + const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); + const dim_mask: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + offs_d.lt(@as(i32, @intCast(HEAD_SIZE))) + else + k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); + const output_offset = query_token_idx.to(.i64).mul(output_stride_0) + .add(query_head_idx.to(.i64).mul(output_stride_1)) + .add(offs_d.to(.i64)); + + // The launch covers the padded query capacity, but the attention pass only + // writes scratch for active tokens. Zero the tail before looking up a + // sequence or dividing by its segment size (zero for inactive sequences). + const active_tokens = k.load(query_start_len_ptr.addPtr(num_seqs)); + const inactive = query_token_idx.ge(active_tokens); + k.storeOpts(output_ptr.addPtr(output_offset), k.zeros(&.{HEAD_SIZE_PADDED}, config.o_dtype), .{ + .mask = dim_mask.bitAnd(inactive), + }); + k.returnIf(inactive); + const seq_idx = findSeqIdx(k, query_start_len_ptr, query_token_idx, num_seqs, BLOCK_Q, false); const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); @@ -1029,12 +1048,6 @@ fn reduceSegments( const seg_range = k.arange(0, NUM_SEGMENTS_PER_SEQ, .i32); const segm_mask = seg_range.lt(act_num_segments); - const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); - const dim_mask: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) - offs_d.lt(@as(i32, @intCast(HEAD_SIZE))) - else - k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); - // segm_offset = query_token_idx*NQH*NSPS + query_head_idx*NSPS + arange(NSPS) const tok_stride: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ; const head_stride: i32 = @intCast(NUM_SEGMENTS_PER_SEQ); @@ -1100,9 +1113,6 @@ fn reduceSegments( ); } - const output_offset = query_token_idx.to(.i64).mul(output_stride_0) - .add(query_head_idx.to(.i64).mul(output_stride_1)) - .add(offs_d.to(.i64)); k.storeOpts( output_ptr.addPtr(output_offset), acc.to(config.o_dtype), From 69a6b2bb227a881b63a53228e87957495a3887d4 Mon Sep 17 00:00:00 2001 From: Louis Lechevalier Date: Thu, 5 Nov 2026 11:44:39 +0100 Subject: [PATCH 847/881] workspace: bump ZML plugins (#1031) --- platforms/cpu/cpu.bzl | 6 +++--- platforms/cuda/cuda.bzl | 8 ++++---- platforms/furiosa/furiosa.bzl | 4 ++-- platforms/metal/metal.bzl | 4 ++-- platforms/rocm/rocm.bzl | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 1105f776..b33ef4df 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -43,21 +43,21 @@ def _cpu_plugin_impl(mctx): name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, sha256 = "3d6a6d1b047a0943b50fd9e63e5f6dff6b7cf184b186c0cc1052de40f0dbec92", - url = "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-cpu-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "d6b4b15b13a2d1a3e0b84f143f568cd572bef5db91784f9f7a826173ed626c7e", - url = "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-cpu-darwin-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "c06babc001edeab313bc63daab3aa3d40e087266a3a29634d182628fb65fad14", - url = "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-cpu-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index ced8ae98..a0c73948 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -224,12 +224,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "77d2ddb19566604c827828fe3dcc53e938f3a1b16d431416fdd5c466266f5242", - "url": "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-cuda-linux-amd64.tar.zst", + "sha256": "b68b407efcd361bbe4b297a16feb080eabbae9132a7ee1a5b105e24a91d99de0", + "url": "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "2cd18393c44816d3d01aa22fbd890632ba46eca3939c644bddef00d590578a80", - "url": "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-cuda-linux-arm64.tar.zst", + "sha256": "9ac41221901eba84641aef35d1f53ff69ce471c5af3844550cd086abc6916f5c", + "url": "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index 0aa3905d..fc2e5352 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202610221403.283.1.d9f4f68759e1/zml-furiosa-linux-amd64.tar.zst", - sha256 = "f8d6bb03fb4cd5dad9b37856b2f8f12b28ec1a8d46f59ca03c401bea668215e9", + url = "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-furiosa-linux-amd64.tar.zst", + sha256 = "85edea12494278a015c1ad29cf57401217e4809bc3a0c6890d0279659bb6209d", ) return mctx.extension_metadata( diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 03cdaf02..2d43a991 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "6eee91776397d472ac590413cf44d4ff3f7b31bebb163aa96339c6becaf2defd", - url = "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-metal-darwin-arm64.tar.zst", + sha256 = "bdd2de3ec8293faa25b358cdf35ee3e75d69be73a4acea87b6e007a18b8f10e8", + url = "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index babd73ab..ee875995 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202610290951.309.1.785550677e75/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "cd6c3c8673ce690a0fbf89dab5a16e6893f0f501bb2b41997c657cf6effa7549" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "02b86fa3a6599f6aa2b89806b8f6c0b16f11765ec04c94517ec1463a59d92175" def _rocm_package_name(name): return name + _ROCM_VERSION From 4baec259241a8293abef0b86a69711c00514e069 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 5 Nov 2026 11:53:31 +0100 Subject: [PATCH 848/881] kernels/moe: remove confusing bitcast (#1030) the bitcast discard partitioning info the API issue is tracked by https://github.com/zml/zml/issues/914 --- zml/kernels/moe/triton/triton.zig | 106 +++++++++++++++++++----------- 1 file changed, 67 insertions(+), 39 deletions(-) diff --git a/zml/kernels/moe/triton/triton.zig b/zml/kernels/moe/triton/triton.zig index 908cd163..9d3a5529 100644 --- a/zml/kernels/moe/triton/triton.zig +++ b/zml/kernels/moe/triton/triton.zig @@ -897,14 +897,17 @@ pub fn fusedExpertsMxfp4( const expert_parallelism = gate_up.weight.shape().partition(.expert).isSharded(); + // Note: we directly forward input sharding to manualComputation + // TODO: improve sharding in attention and moe APIs + // https://github.com/zml/zml/issues/914 const context: Context = .{ .input = input, .ids = ids, .weights = weights, - .w1 = gate_up.weight.bitCast(.u8), - .s1 = gq.scales, - .w2 = down.weight.bitCast(.u8), - .s2 = dq.scales, + .gate_up = gate_up.weight, + .gate_up_scales = gq.scales, + .down = down.weight, + .down_scales = dq.scales, .global_experts = gate_up.weight.dim(.expert), .topk = ids.dim(.topk), .limit = options.activation.swiglu.limit.?, @@ -952,46 +955,71 @@ pub const Config = struct { } }; -pub fn validateInputs( +fn validateInputs( c: Config, input: zml.Tensor, ids: zml.Tensor, - weights: zml.Tensor, - w1: zml.Tensor, - s1: zml.Tensor, - w2: zml.Tensor, - s2: zml.Tensor, -) !void { - if (s1.rank() != 3 or s2.rank() != 3) return error.InvalidInputShape; - if (s1.dim(2) < @divTrunc(c.hidden, 32) or - s2.dim(2) < @divTrunc(c.intermediate, 32)) return error.InvalidInputShape; + expert_weights: zml.Tensor, + gate_up: zml.Tensor, + gate_up_scales: zml.Tensor, + down: zml.Tensor, + down_scales: zml.Tensor, +) void { + stdx.debug.assert(gate_up_scales.rank() == 3 and down_scales.rank() == 3, "Invalid MXFP4 tensor layout, expected gate_up_scales and down_scales to be rank 3, got {f} and {f}", .{ gate_up_scales, down_scales }); + const shapes = .{ - .x = zml.Shape.init(.{ c.tokens, c.hidden }, .bf16), + .input = zml.Shape.init(.{ c.tokens, c.hidden }, .bf16), .ids = zml.Shape.init(.{ c.tokens, c.topk }, .i32), - .scales = zml.Shape.init(.{ c.tokens, c.topk }, .f32), - .w1 = zml.Shape.init(.{ c.experts, 2 * c.intermediate, @divTrunc(c.hidden, 2) }, .u8), - .s1 = zml.Shape.init(.{ c.experts, 2 * c.intermediate, s1.dim(2) }, .f8e8m0), - .w2 = zml.Shape.init(.{ c.experts, c.hidden, @divTrunc(c.intermediate, 2) }, .u8), - .s2 = zml.Shape.init(.{ c.experts, c.hidden, s2.dim(2) }, .f8e8m0), + .expert_weights = zml.Shape.init(.{ c.tokens, c.topk }, .f32), + .gate_up = zml.Shape.init(.{ c.experts, 2 * c.intermediate, @divExact(c.hidden, 2) }, .i8), + .gate_up_scales = zml.Shape.init(.{ c.experts, 2 * c.intermediate, gate_up_scales.dim(2) }, .f8e8m0), + .down = zml.Shape.init(.{ c.experts, c.hidden, @divExact(c.intermediate, 2) }, .i8), + .down_scales = zml.Shape.init(.{ c.experts, c.hidden, down_scales.dim(2) }, .f8e8m0), }; - if (!input.shape().eql(shapes.x)) return error.InvalidInputShape; - if (!ids.shape().eql(shapes.ids)) return error.InvalidInputShape; - if (!weights.shape().eql(shapes.scales)) return error.InvalidInputShape; - if (!w1.shape().eql(shapes.w1)) return error.InvalidInputShape; - if (!s1.shape().eql(shapes.s1)) return error.InvalidInputShape; - if (!w2.shape().eql(shapes.w2)) return error.InvalidInputShape; - if (!s2.shape().eql(shapes.s2)) return error.InvalidInputShape; + stdx.debug.assert(input.shape().eql(shapes.input) and + ids.shape().eql(shapes.ids) and + expert_weights.shape().eql(shapes.expert_weights) and + // Note: we accept both i8 and u8 for gate_up and down + // TODO: we should instead use f4e2m1 + (gate_up.shape().eql(shapes.gate_up) or gate_up.shape().eql(shapes.gate_up.withDtype(.u8))) and + gate_up_scales.shape().eql(shapes.gate_up_scales) and + (down.shape().eql(shapes.down) or down.shape().eql(shapes.down.withDtype(.u8))) and + down_scales.shape().eql(shapes.down_scales), + \\Invalid MXFP4 tensor layout: + \\ - expected input={f}, got {f} + \\ - expected ids={f}, got {f} + \\ - expected expert_weights={f}, got {f} + \\ - expected gate_up={f}, got {f} + \\ - expected gate_up_scales={f}, got {f} + \\ - expected down={f}, got {f} + \\ - expected down_scales={f}, got {f} + , .{ + shapes.input, input, // + shapes.ids, ids, // + shapes.expert_weights, expert_weights, // + shapes.gate_up, gate_up, // + shapes.gate_up_scales, gate_up_scales, // + shapes.down, down, // + shapes.down_scales, down_scales, // + }); + + stdx.debug.assert( + gate_up_scales.dim(2) >= @divTrunc(c.hidden, 32) and + down_scales.dim(2) >= @divTrunc(c.intermediate, 32), + "Invalid MXFP4 tensor layout, not enough scales, expected a ratio of at most 32 for config {}, got {f} and {f}", + .{ c, gate_up_scales, down_scales }, + ); } const Context = struct { input: zml.Tensor, ids: zml.Tensor, weights: zml.Tensor, - w1: zml.Tensor, - s1: zml.Tensor, - w2: zml.Tensor, - s2: zml.Tensor, + gate_up: zml.Tensor, + gate_up_scales: zml.Tensor, + down: zml.Tensor, + down_scales: zml.Tensor, global_experts: i64, topk: i64, limit: f32, @@ -999,9 +1027,9 @@ const Context = struct { expert_parallel: bool, fn body(self: Context, _: zml.Shape) zml.Tensor { - const experts = self.w1.dim(.expert); - const hidden = self.w2.dim(1); - const intermediate = self.w2.dim(2) * 2; + const experts = self.gate_up.dim(.expert); + const hidden = self.down.dim(1); + const intermediate = self.down.dim(2) * 2; var ids = self.ids.convert(.i32).reshape(.{ .token = .auto, .topk = self.topk }); const tokens = ids.dim(.token); if (self.expert_parallel) { @@ -1023,7 +1051,7 @@ const Context = struct { }; cfg.validate() catch @panic("Invalid MXFP4 MoE configuration"); - validateInputs(cfg, x, ids, scales, self.w1, self.s1, self.w2, self.s2) catch @panic("Invalid MXFP4 tensor layout"); + validateInputs(cfg, x, ids, scales, self.gate_up, self.gate_up_scales, self.down, self.down_scales); const bn = 16; const bk = 512; @@ -1087,7 +1115,7 @@ const Context = struct { // Each narrow tile produces 64 activations: two complete scale groups. const up_columns: i64 = if (cfg.tokens <= 8) 64 else 128; const m = kernels.GateUp.Kernel.call( - .{ .q = q.q, .s = q.s, .w = self.w1, .ws = self.s1.bitCast(.u8), .ids = ids, .rw = scales, .map = map, .sched = sched }, + .{ .q = q.q, .s = q.s, .w = self.gate_up.bitCast(.u8), .ws = self.gate_up_scales.bitCast(.u8), .ids = ids, .rw = scales, .map = map, .sched = sched }, .{ .mid = .init(.{ tile_count * bn, cfg.intermediate }, .f8e4m3fn), .ms = .init(.{ tile_count * bn, @divTrunc(cfg.intermediate, 32) }, .u8) }, .{ .cfg = .{ @@ -1098,7 +1126,7 @@ const Context = struct { .topk = cfg.topk, .expert_offset = cfg.expert_offset, .swiglu_limit = cfg.swiglu_limit, - .scale_stride = self.s1.dim(2), + .scale_stride = self.gate_up_scales.dim(2), .routing_weight_placement = cfg.routing_weight_placement, .bn = bn, .bk = bk, @@ -1111,7 +1139,7 @@ const Context = struct { ); const d = kernels.Down.Kernel.call( - .{ .mid = m.mid, .ms = m.ms, .w = self.w2, .ws = self.s2.bitCast(.u8), .ids = ids, .sched = sched }, + .{ .mid = m.mid, .ms = m.ms, .w = self.down.bitCast(.u8), .ws = self.down_scales.bitCast(.u8), .ids = ids, .sched = sched }, .{ .d = .init(.{ tile_count * bn, cfg.hidden }, .bf16) }, .{ .cfg = .{ @@ -1122,7 +1150,7 @@ const Context = struct { .topk = cfg.topk, .expert_offset = cfg.expert_offset, .swiglu_limit = cfg.swiglu_limit, - .scale_stride = self.s2.dim(2), + .scale_stride = self.down_scales.dim(2), .routing_weight_placement = cfg.routing_weight_placement, .bn = bn, .bk = bk, From 47f50667d879dff1f225ef35eb4878dd573bcea4 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 5 Nov 2026 13:00:00 +0100 Subject: [PATCH 849/881] zml/attention: declare manual partitioning in the shared API instead of kernels (#1016) Move manual regions out of flash attention kernels and use `manualComputation2` for every shared attention backend. Also fully replicate the embedding executable outputs in Qwen and Llama. Partial graph constraints leave their batch and sequence dimensions open, which is invalid at executable boundaries. Fix two further blockers exposed by the Qwen run that we forgot to move recently: softplus comparisons must use the input dtype and ROCm DNN runfiles must include the MIOpen kernel archives. --- examples/llm/models/lfm2/model.zig | 6 ++ examples/llm/models/llama/model.zig | 4 +- examples/llm/models/qwen3_5/model.zig | 8 +- examples/llm/models/qwen3_5_moe/model.zig | 8 +- platforms/rocm/rocm.bzl | 1 + zml/attention/attention.zig | 86 +++++++++++++++---- zml/kernels/flash_attention/cute/cute.zig | 85 ++++++++---------- .../flash_attention/flash_attention.zig | 18 +++- zml/kernels/flash_attention/nki/nki.zig | 80 +++-------------- zml/nn.zig | 32 +++---- 10 files changed, 170 insertions(+), 158 deletions(-) diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig index dd21c7f9..9f7cb909 100644 --- a/examples/llm/models/lfm2/model.zig +++ b/examples/llm/models/lfm2/model.zig @@ -571,6 +571,8 @@ pub const Attention = struct { v = new_kv_cache.values(cache_index); stdx.debug.assert(q.dim(.batch) == 1, "LFM attention currently expects batch size 1 for flash attention backend, got {}", .{q.dim(.batch)}); + const mesh = zml.Compiler.current().mesh(.model); + const partitioning: zml.attention.Partitioning = .head(.model); const attn = switch (attention_parameters) { .stablehlo => zml.attention.attention( q, @@ -579,6 +581,8 @@ pub const Attention = struct { tokens_position_offset, attention_metadata, attention_parameters, + mesh, + partitioning, ).merge(.{ .d = .{ .h, .hd } }), else => zml.attention.attention( q.squeeze(.batch), @@ -587,6 +591,8 @@ pub const Attention = struct { tokens_position_offset.squeeze(.batch), attention_metadata, attention_parameters, + mesh, + partitioning, ).merge(.{ .d = .{ .h, .hd } }) .rename(.{ .q = .seq }) .insertAxes(.seq, .{.batch}), diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig index 05610dc1..8a6a09e9 100644 --- a/examples/llm/models/llama/model.zig +++ b/examples/llm/models/llama/model.zig @@ -329,7 +329,7 @@ pub const EmbedTokens = struct { const tokens = input.tokens.withPartialTags(.{.s}); return .{ .hidden = input.embedding.embed_tokens.forward(tokens) .withPartialTags(.{.d}) - .withPartitioning(.model, .{ .d = .replicated }) }; + .replicate() }; } }; @@ -587,6 +587,8 @@ const SelfAttn = struct { token_index, attention_metadata, attention_parameters, + zml.Compiler.current().mesh(.model), + .head(.model), ); const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig index 77d3884e..40174f5c 100644 --- a/examples/llm/models/qwen3_5/model.zig +++ b/examples/llm/models/qwen3_5/model.zig @@ -290,7 +290,7 @@ pub const EmbedTokens = struct { const tokens = input.tokens.withPartialTags(.{.s}); return .{ .hidden = input.embedding.embed_tokens.forward(tokens) .withPartialTags(.{.d}) - .withPartitioning(.model, .{ .d = .replicated }) }; + .replicate() }; } }; @@ -657,6 +657,10 @@ pub const SelfAttn = struct { k = partitionCachedKv(k, kv_head_sharding); v = partitionCachedKv(v, kv_head_sharding); + const head_partitioning: zml.attention.Partitioning = switch (kv_head_sharding) { + .sharded => .head(.model), + .replicated => .head(.replicated), + }; const attn_output = zml.attention.attention( q, k, @@ -664,6 +668,8 @@ pub const SelfAttn = struct { token_index, zml.attention.Metadata.init(.fromBackend(.stablehlo)), zml.attention.Parameters.init(.fromBackend(.stablehlo)), + zml.Compiler.current().mesh(.model), + head_partitioning, ).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); const gated_output = attn_output.mul(gate.sigmoid()); diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 11ce8151..1375a1bd 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -641,6 +641,10 @@ pub const SelfAttn = struct { k = partitionCachedKv(k, kv_head_sharding); v = partitionCachedKv(v, kv_head_sharding); + const head_partitioning: zml.attention.Partitioning = switch (kv_head_sharding) { + .sharded => .head(.model), + .replicated => .head(.replicated), + }; const attn_output = zml.attention.attention( q, k, @@ -648,7 +652,9 @@ pub const SelfAttn = struct { token_index, zml.attention.Metadata.init(.fromBackend(.stablehlo)), zml.attention.Parameters.init(.fromBackend(.stablehlo)), - ).withPartitioning(tp, .{ .q = .replicated, .h = .model, .hd = .replicated }).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); + zml.Compiler.current().mesh(.model), + head_partitioning, + ).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); const gated_output = attn_output.mul(gate.sigmoid()); const projected_output = self.o_proj.forward(gated_output.rename(.{ .d_out_proj = .d }), gated_output.dtype()).rename(.{ .dout = .d }).withPartitioning(tp, .{ .d = .replicated }); diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index ee875995..0c1f7a92 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -73,6 +73,7 @@ def _rocm_runfile_build_files(loaded_packages): for pkg_name in _family_runfile_package_names(loaded_packages, "dnn"): rocm_packages[pkg_name] = _glob_filegroup("runfiles", [ + ".kpack/**", "lib/libMIOpenCKGroupedConv_*.so", "share/miopen/db/**", ]) diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index 4cfad309..039c6628 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -99,6 +99,28 @@ pub const Options = struct { causal: bool = true, }; +pub const Partitioning = struct { + const Logical = zml.Sharding.Partitioning.Logical; + const InputSpecs = @FieldType(zml.ops.ManualPartitioning(Attention.forward), "in_specs"); + + in_specs: struct { + q: Logical, + k: Logical, + v: Logical, + token_index: Logical = .replicated, + /// Omit to replicate backend metadata tensors. + metadata: ?@FieldType(InputSpecs, "metadata") = null, + }, + out_specs: Logical, + + /// Shards the `.h` dimension of Q, K, V and the output over `mesh_axis`. + /// Other dimensions, token indices and backend metadata are replicated. + pub fn head(comptime mesh_axis: @EnumLiteral()) Partitioning { + const heads: Logical = .parse(.{ .h = mesh_axis }); + return .{ .in_specs = .{ .q = heads, .k = heads, .v = heads }, .out_specs = heads }; + } +}; + /// Causal attention as used in transformers. /// /// **Shapes**: @@ -110,27 +132,53 @@ pub const Options = struct { /// - .q is the number of queries /// - .k is the number of keys /// - .hd is the head dimension -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { - return attentionOpts(q, k, v, token_index, metadata, parameters, .{}); +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { + return attentionOpts(q, k, v, token_index, metadata, parameters, .{}, mesh, partitioning); } /// `attention`, causal or not depending on `opts`. -pub fn attentionOpts(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options) zml.Tensor { - return switch (parameters) { - .stablehlo => b: { - // Generate the attention mask. - const seq_len = k.dim(.k); - // Note: in Pytorch it would be very inefficient to generate the full attn_mask, - // then slice into it, but XLA is able to optimize this correctly. - const attn_mask: ?zml.Tensor = if (opts.causal) - zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, q.dtype(), null) - .gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, q.dtype()), token_index.appendAxes(.{.coord}), .{}) - else - null; - - const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask }); - break :b attn_output; +pub fn attentionOpts(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { + const args: Attention = .{ .q = q, .k = k, .v = v, .token_index = token_index, .metadata = metadata, .parameters = parameters, .opts = opts }; + return zml.ops.manualComputation2(Attention.forward, args, q.shape(), mesh, .{ + .in_specs = .{ + .q = partitioning.in_specs.q, + .k = partitioning.in_specs.k, + .v = partitioning.in_specs.v, + .token_index = partitioning.in_specs.token_index, + .metadata = partitioning.in_specs.metadata orelse switch (metadata) { + .stablehlo => .{ .stablehlo = {} }, + .flash_attention => |metadata_| .{ .flash_attention = switch (metadata_) { + .cute => .{ .cute = {} }, + .nki => .{ .nki = {} }, + .metal => .{ .metal = .{ .num_tokens = .replicated } }, + } }, + }, }, - .flash_attention => flash_attention.attention(q, k, v, token_index, metadata.flash_attention, parameters.flash_attention, opts), - }; + .out_specs = partitioning.out_specs, + }); } + +const Attention = struct { + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + token_index: zml.Tensor, + metadata: Metadata, + parameters: Parameters, + opts: Options, + + fn forward(args: @This(), _: zml.Shape) zml.Tensor { + return switch (args.parameters) { + .stablehlo => b: { + const seq_len = args.k.dim(.k); + const mask: ?zml.Tensor = if (args.opts.causal) + zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, args.q.dtype(), null) + .gatherSlices(zml.Shape.init(.{ .q = args.q.dim(.q) }, args.q.dtype()), args.token_index.appendAxes(.{.coord}), .{}) + else + null; + break :b zml.nn.sdpa(args.q, args.k, args.v, .{ .attn_mask = mask }); + }, + .flash_attention => flash_attention.attention(args.q, args.k, args.v, args.token_index, args.metadata.flash_attention, args.parameters.flash_attention, args.opts), + }; + } +}; diff --git a/zml/kernels/flash_attention/cute/cute.zig b/zml/kernels/flash_attention/cute/cute.zig index c1ffa58f..36e0953b 100644 --- a/zml/kernels/flash_attention/cute/cute.zig +++ b/zml/kernels/flash_attention/cute/cute.zig @@ -34,68 +34,51 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T stdx.debug.assert(token_index.rank() == 0, "Unbatched CuTe attention requires a scalar token position", .{}); } - const mesh = zml.Compiler.current().resolveMesh(.{.model}); - const q_sharded = q.withPartitioning(mesh, .{ .h = .model }); - const k_sharded = k.withPartitioning(mesh, .{ .h = .model }); - const v_sharded = v.withPartitioning(mesh, .{ .h = .model }); const positions = if (batched) token_index.convert(.i32).broad(zml.Shape.init(.{ .b = q.dim(.b) }, .i32)) else token_index.convert(.i32); - return zml.ops.manualComputation(Context.body, Context{ .q = q_sharded, .k = k_sharded, .v = v_sharded, .positions = positions, .causal = causal }, q.shape().withPartitioning(mesh, .{ .h = .model })); -} + const q_ = if (batched) q.transpose(.{ .b, .q, .h, .hd }) else q.transpose(.{ .q, .h, .hd }); + const k_ = if (batched) k.transpose(.{ .b, .k, .h, .hd }) else k.transpose(.{ .k, .h, .hd }); + const v_ = if (batched) v.transpose(.{ .b, .k, .h, .hd }) else v.transpose(.{ .k, .h, .hd }); -const Context = struct { - q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, - positions: zml.Tensor, - causal: bool, + const splits = kernel.decodeSplits(q_.shape(), k_.shape(), decodeResources(zml.Compiler.current().platform)); - fn body(self: Context, output_shape: zml.Shape) zml.Tensor { - const batched = self.q.shape().hasTag(.b) != null; - const q = if (batched) self.q.transpose(.{ .b, .q, .h, .hd }) else self.q.transpose(.{ .q, .h, .hd }); - const k = if (batched) self.k.transpose(.{ .b, .k, .h, .hd }) else self.k.transpose(.{ .k, .h, .hd }); - const v = if (batched) self.v.transpose(.{ .b, .k, .h, .hd }) else self.v.transpose(.{ .k, .h, .hd }); + const cfg: kernel.Config = .{ + .batch = if (batched) q_.dim(.b) else 1, + .query_length = q_.dim(.q), + .key_length = k_.dim(.k), + .query_heads = q_.dim(.h), + .kv_heads = k_.dim(.h), + .head_dim = q_.dim(.hd), + .dtype = zml.kernel.cute.from(q_.dtype()), + .splits = splits, + .causal = causal, + }; - const splits = kernel.decodeSplits(q.shape(), k.shape(), decodeResources(zml.Compiler.current().platform)); - - const cfg: kernel.Config = .{ - .batch = if (batched) q.dim(.b) else 1, - .query_length = q.dim(.q), - .key_length = k.dim(.k), - .query_heads = q.dim(.h), - .kv_heads = k.dim(.h), - .head_dim = q.dim(.hd), - .dtype = zml.kernel.cute.from(q.dtype()), - .splits = splits, - .causal = self.causal, - }; - - const out = if (kernel.prefillSupported(cfg)) - kernel.Prefill.call(.{ .q = q, .k = k, .v = v, .positions = self.positions }, .{ .out = q.shape() }, .{ .cfg = cfg }).out - else if (cfg.splits > 1) b: { - const partials = kernel.Kernel.call(.{ .q = q, .k = k, .v = v, .positions = self.positions }, .{ - .out = zml.Shape.init(.{ cfg.batch, cfg.splits, cfg.query_length, cfg.query_heads, cfg.head_dim + 2 }, .f32), - }, .{ - .cfg = cfg, - .grid = .{ @intCast(@divTrunc(@divExact(cfg.query_heads, cfg.kv_heads) + 3, 4)), @intCast(cfg.kv_heads * cfg.batch), @intCast(cfg.splits) }, - .block = kernel.block, - }).out; - break :b kernel.Merge.call(.{ .partials = partials }, .{ .out = q.shape() }, .{ - .cfg = cfg, - .grid = .{ @intCast(cfg.query_heads), @intCast(cfg.batch), 1 }, - .block = kernel.block, - }).out; - } else kernel.Kernel.call(.{ .q = q, .k = k, .v = v, .positions = self.positions }, .{ .out = q.shape() }, .{ + const out = if (kernel.prefillSupported(cfg)) + kernel.Prefill.call(.{ .q = q_, .k = k_, .v = v_, .positions = positions }, .{ .out = q_.shape() }, .{ .cfg = cfg }).out + else if (cfg.splits > 1) b: { + const partials = kernel.Kernel.call(.{ .q = q_, .k = k_, .v = v_, .positions = positions }, .{ + .out = zml.Shape.init(.{ cfg.batch, cfg.splits, cfg.query_length, cfg.query_heads, cfg.head_dim + 2 }, .f32), + }, .{ .cfg = cfg, - .grid = .{ @intCast(@divTrunc(cfg.query_length * @divExact(cfg.query_heads, cfg.kv_heads) + 3, 4)), @intCast(cfg.kv_heads * cfg.batch), 1 }, + .grid = .{ @intCast(@divTrunc(@divExact(cfg.query_heads, cfg.kv_heads) + 3, 4)), @intCast(cfg.kv_heads * cfg.batch), @intCast(cfg.splits) }, .block = kernel.block, }).out; + break :b kernel.Merge.call(.{ .partials = partials }, .{ .out = q_.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(cfg.query_heads), @intCast(cfg.batch), 1 }, + .block = kernel.block, + }).out; + } else kernel.Kernel.call(.{ .q = q_, .k = k_, .v = v_, .positions = positions }, .{ .out = q_.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(@divTrunc(cfg.query_length * @divExact(cfg.query_heads, cfg.kv_heads) + 3, 4)), @intCast(cfg.kv_heads * cfg.batch), 1 }, + .block = kernel.block, + }).out; - return out.transpose(output_shape); - } -}; + return out.transpose(q.shape()); +} /// Returns the minimum number of multiprocessors and shared memory per block across all devices in the platform. /// If there are no devices, returns 0 for both values. diff --git a/zml/kernels/flash_attention/flash_attention.zig b/zml/kernels/flash_attention/flash_attention.zig index b0a6106b..78602bd0 100644 --- a/zml/kernels/flash_attention/flash_attention.zig +++ b/zml/kernels/flash_attention/flash_attention.zig @@ -127,6 +127,7 @@ pub const Metadata = union(Backend) { /// - .k is the number of keys /// - .hd is the head dimension pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: zml.attention.Options) zml.Tensor { + zml.stdx.debug.assert(zml.Compiler.current().manual_computation_depth > 0, "Attention kernels require an explicit manualComputation boundary", .{}); return switch (parameters) { .cute => cute.attention(q, k, v, token_index, opts.causal), // The NKI and Metal kernels are causal only. @@ -315,9 +316,19 @@ fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: defer token_index.deinit(); const meshes = platform.meshes.values(); + const mesh = zml.testing.tpMesh(); + const partitioning: zml.attention.Partitioning = if (@rem(k_shape.dim(.h), mesh.numPartitionsForLogicalAxis(.model)) == 0) .head(.model) else .{ + .in_specs = .{ .q = .replicated, .k = .replicated, .v = .replicated }, + .out_specs = .replicated, + }; + const AttentionCall = struct { + fn forward(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, position: zml.Tensor, metadata: zml.attention.Metadata, parameters: zml.attention.Parameters, opts_: zml.attention.Options, mesh_: *const zml.Mesh, partitioning_: zml.attention.Partitioning) zml.Tensor { + return zml.attention.attentionOpts(q_, k_, v_, position, metadata, parameters, opts_, mesh_, partitioning_); + } + }; const reference_metadata: zml.attention.Metadata = .{ .stablehlo = {} }; const reference_parameters: zml.attention.Parameters = .{ .stablehlo = {} }; - const reference_exe = try platform.compileFn(allocator, io, zml.attention.attentionOpts, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, reference_metadata, reference_parameters, opts }, .{ + const reference_exe = try platform.compileFn(allocator, io, AttentionCall.forward, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, reference_metadata, reference_parameters, opts, mesh, partitioning }, .{ .program_name = "attention_stablehlo", .meshes = meshes, }); @@ -326,6 +337,7 @@ fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: var reference_d = try reference_exe.eval(allocator, io, .{ q, k, v, token_index, reference_metadata }); defer reference_d.deinit(); try reference_d.await(io); + try std.testing.expectEqualDeep(q_shape.withPartitioning(mesh, partitioning.out_specs)._sharding, reference_d.shape()._sharding); const reference_h: zml.Slice = try reference_d.toSliceAlloc(allocator, io); defer reference_h.free(allocator); @@ -337,8 +349,8 @@ fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: const exe = try platform.compileFn( allocator, io, - attention, - .{ tensors.q, tensors.k, tensors.v, tensors.token_index, metadata, parameters, opts }, + AttentionCall.forward, + .{ tensors.q, tensors.k, tensors.v, tensors.token_index, .{ .flash_attention = metadata }, .{ .flash_attention = parameters }, opts, mesh, partitioning }, .{ .program_name = try std.fmt.allocPrint(arena, "attention_{t}", .{backend}), .meshes = meshes, diff --git a/zml/kernels/flash_attention/nki/nki.zig b/zml/kernels/flash_attention/nki/nki.zig index 905f3aa0..f98a545d 100644 --- a/zml/kernels/flash_attention/nki/nki.zig +++ b/zml/kernels/flash_attention/nki/nki.zig @@ -43,71 +43,17 @@ pub const Parameters = struct { /// Attention for Neuron. Both prefill and decode lower to NKI custom calls. pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, parameters: Parameters) zml.Tensor { - // TODO: https://github.com/zml/zml/issues/914 - const tp = zml.Compiler.current().resolveMesh(.{.model}); - const q_sharded = q.withPartitioning(tp, .{ .h = .model }); - const k_sharded = k.withPartitioning(tp, .{ .h = .model }); - const v_sharded = v.withPartitioning(tp, .{ .h = .model }); - - if (q.dim(.q) == 1) { - const token_index_2d = token_index.broad(zml.Shape.init(.{ .row = 1, .col = 1 }, token_index.dtype())) - .withPartitioning(tp, .{ .row = .replicated, .col = .replicated }); - - return zml.ops.manualComputation( - (struct { - q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, - token_index: zml.Tensor, - parameters: Parameters, - - fn body(self: @This(), output: zml.Shape) zml.Tensor { - const kernel = self.parameters.decode_kernel; - return zml.ops.neuronNki( - .{ self.q, self.k, self.v, self.token_index }, - .{output}, - .{ - .name = kernel.name, - .entrypoint = kernel.entrypoint, - .source_path = kernel.source_path, - .compiler_target = self.parameters.compiler_target, - }, - )[0]; - } - }).body, - .{ - .q = q_sharded, - .k = k_sharded, - .v = v_sharded, - .token_index = token_index_2d, - .parameters = parameters, - }, - q.shape().withPartitioning(tp, .{ .h = .model }), - ).convert(q.dtype()); - } - - return zml.ops.manualComputation( - (struct { - q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, - parameters: Parameters, - - fn body(self: @This(), output: zml.Shape) zml.Tensor { - const kernel = self.parameters.prefill_kernel; - return zml.ops.neuronNki( - .{ self.q, self.k, self.v }, - .{output}, - .{ - .name = kernel.name, - .entrypoint = kernel.entrypoint, - .source_path = kernel.source_path, - .compiler_target = self.parameters.compiler_target, - }, - )[0]; - } - }).body, - .{ .q = q_sharded, .k = k_sharded, .v = v_sharded, .parameters = parameters }, - q.shape().withPartitioning(tp, .{ .h = .model }), - ).convert(q.dtype()); + const decode = q.dim(.q) == 1; + const kernel = if (decode) parameters.decode_kernel else parameters.prefill_kernel; + const options: zml.ops.NeuronNkiOps = .{ + .name = kernel.name, + .entrypoint = kernel.entrypoint, + .source_path = kernel.source_path, + .compiler_target = parameters.compiler_target, + }; + const output = if (decode) + zml.ops.neuronNki(.{ q, k, v, token_index.broad(zml.Shape.init(.{ .row = 1, .col = 1 }, token_index.dtype())) }, .{q.shape()}, options)[0] + else + zml.ops.neuronNki(.{ q, k, v }, .{q.shape()}, options)[0]; + return output.convert(q.dtype()); } diff --git a/zml/nn.zig b/zml/nn.zig index 56fd8b51..4569552f 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -2354,8 +2354,8 @@ pub fn softplus( // Using log(1 + exp(x)) directly loses precision for x < ~-16.6, causing strongly negative logits to produce zero scores. const z = x.exp(); const u = z.addConstant(1); - const log1p = u.cmp(.EQ, .scalar(1, .f32)).select(z, z.mul(u.log()).div(u.subConstant(1))); - return x.cmp(.GT, .scalar(opts.threshold, .f32)).select(x, log1p); + const log1p = u.cmp(.EQ, .scalar(1, x.dtype())).select(z, z.mul(u.log()).div(u.subConstant(1))); + return x.cmp(.GT, .scalar(opts.threshold, x.dtype())).select(x, log1p); } test "softplus" { @@ -2365,14 +2365,11 @@ test "softplus" { const platform = zml.testing.env(); const input: Tensor = .init(.{ .x = inputs.len }, .f32); const Local = struct { - fn call(x: Tensor) Tensor { - return softplus(x, .{ .threshold = 2.0 }); + fn call(x: Tensor, dtype: DataType) Tensor { + return softplus(x.convert(dtype), .{ .threshold = 2.0 }).convert(x.dtype()); } }; - var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Local.call, .{input}, .{}); - defer exe.deinit(); - var input_buffer: zml.Buffer = try .fromBytes( std.testing.io, platform, @@ -2381,13 +2378,18 @@ test "softplus" { ); defer input_buffer.deinit(); - var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); - defer result.deinit(); + for ([_]DataType{ .f32, .bf16 }) |dtype| { + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Local.call, .{ input, dtype }, .{}); + defer exe.deinit(); - try zml.testing.expectClose( - std.testing.io, - Slice.init(input.shape(), std.mem.sliceAsBytes(&expected)), - result, - .{}, - ); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); + defer result.deinit(); + + try zml.testing.expectClose( + std.testing.io, + Slice.init(input.shape(), std.mem.sliceAsBytes(&expected)), + result, + .{}, + ); + } } From 2438b9c364799ee12a52ff4b1a9e14e2aa874e54 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 5 Nov 2026 13:19:58 +0100 Subject: [PATCH 850/881] zml/kernels/paged_attention: make mesh and partitioning explicit (#1024) Paged attention currently relies on input sharding metadata or partitioning choices made inside backend kernels. Ordinary graph operations can leave that metadata unspecified, making the resulting sharding depend on the backend and preceding operations. Move the manual computation boundary into the shared `pagedAttention` API using `manualComputation2`. Callers provide a logical mesh and typed input/output partitioning specs. --- .../paged_attention/cute/benchmark.zig | 27 ++- zml/kernels/paged_attention/cute/cute.zig | 93 ++++----- zml/kernels/paged_attention/cute/fixtures.zig | 2 +- .../paged_attention/cute/sm10x/sm10x.zig | 91 ++++----- .../paged_attention/cute/sm120/sm120.zig | 109 +++++------ .../paged_attention/mosaic_tpu/mosaic_tpu.zig | 74 ++----- .../paged_attention/paged_attention.zig | 80 +++++--- zml/kernels/paged_attention/tcl/tcl.zig | 2 +- zml/kernels/paged_attention/triton/triton.zig | 184 ++++++------------ 9 files changed, 278 insertions(+), 384 deletions(-) diff --git a/zml/kernels/paged_attention/cute/benchmark.zig b/zml/kernels/paged_attention/cute/benchmark.zig index 9fd6a355..3e05fed8 100644 --- a/zml/kernels/paged_attention/cute/benchmark.zig +++ b/zml/kernels/paged_attention/cute/benchmark.zig @@ -54,7 +54,7 @@ fn runBenchmark(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, inputs.sinks = inputs.sinks.withPartitioning(mesh, .{ ._0 = .model }); const compile_start: std.Io.Timestamp = .now(io, .awake); - const exe = try platform.compileFn(a, io, fixtures.forward, .{ inputs, global_case }, .{ + const exe = try platform.compileFn(a, io, forward, .{ inputs, global_case, mesh }, .{ .program_name = "paged_attention_harness", .meshes = &.{mesh}, }); @@ -115,6 +115,31 @@ fn runBenchmark(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, }); } +fn forward(inputs: Inputs, c: cases.Case, mesh: *const zml.Mesh) zml.Tensor { + const Call = struct { + inputs: Inputs, + case: cases.Case, + + fn local(self: @This(), _: zml.Shape) zml.Tensor { + return fixtures.forward(self.inputs, self.case); + } + }; + const heads: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); + return zml.ops.manualComputation2(Call.local, .{ .inputs = inputs, .case = c }, inputs.q.shape(), mesh, .{ + .in_specs = .{ .inputs = .{ + .q = heads, + .k = heads, + .v = heads, + .table = .replicated, + .lengths = .replicated, + .starts = .replicated, + .sinks = .parse(.{ ._0 = .model }), + .indices = .replicated, + } }, + .out_specs = heads, + }); +} + fn elapsedUs(start: std.Io.Timestamp, io: std.Io) f64 { return @as(f64, @floatFromInt(start.untilNow(io, .awake).toNanoseconds())) / 1000; } diff --git a/zml/kernels/paged_attention/cute/cute.zig b/zml/kernels/paged_attention/cute/cute.zig index e572cce1..ea4f36ab 100644 --- a/zml/kernels/paged_attention/cute/cute.zig +++ b/zml/kernels/paged_attention/cute/cute.zig @@ -72,57 +72,44 @@ pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: z return sm10x.pagedAttention(parameters, q, k, v, opts); if (sm120.isAvailable(zml.Compiler.current().platform) and opts.mask != .indices) return sm120.pagedAttention(parameters, q, k, v, opts); - return zml.ops.manualComputation(Context.body, Context{ .q = q, .k = k, .v = v, .parameters = parameters, .opts = opts }, q.shape()); + const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); + const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); + stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); + const indices = switch (opts.mask) { + .indices => |indices| indices.transpose(.{ .b, .topk }), + else => zml.Tensor.zeroes(.init(.{ q_.dim(.b), 1 }, .i32)), + }; + const cfg: kernel.Config = .{ + .head_dim = q_.dim(.hd), + .tokens = q_.dim(.b), + .batch = parameters.block_table.dim(.b), + .kv_heads = k_.dim(.hkv), + .group = q_.dim(.hg), + .pages = k_.dim(.page), + .pages_per_seq = parameters.block_table.dim(.p), + .page_size = k_.dim(.k_chunk), + .dtype = zml.kernel.cute.from(q_.dtype()), + .causal = opts.mask.isCausal(), + .window = opts.mask.slidingWindow() orelse 0, + .scale = opts.scale orelse (1 / @sqrt(@as(f32, @floatFromInt(q_.dim(.hd))))), + .sink = opts.sink != null, + .indexed = opts.mask == .indices, + .topk = indices.dim(-1), + }; + const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{cfg.kv_heads * cfg.group}) else zml.Tensor.zeroes(.init(.{cfg.kv_heads * cfg.group}, .f32)); + return kernel.Kernel.call(.{ + .q = q_, + .k = k_, + .v = v_, + .table = parameters.block_table, + .lengths = parameters.seq_lens, + .starts = parameters.query_start_len, + .sink = sink, + .indices = indices, + }, .{ .out = q_.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(cfg.tokens), @intCast(cfg.kv_heads * cfg.group), 1 }, + .block = .{ 32, 1, 1 }, + }).out.transpose(q.shape()); } - -const Context = struct { - q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, - parameters: Parameters, - opts: paged.AttentionOptions, - - fn body(self: Context, output_shape: zml.Shape) zml.Tensor { - const q = self.q.transpose(.{ .b, .hkv, .hg, .hd }); - const k = self.k.transpose(.{ .page, .k_chunk, .hkv, .hd }); - const v = self.v.transpose(.{ .page, .k_chunk, .hkv, .hd }); - stdx.debug.assert(q.dim(.hkv) == k.dim(.hkv) and k.shape().eql(v.shape()) and k.dim(.hd) == q.dim(.hd) and q.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); - const p = self.parameters; - const indices = switch (self.opts.mask) { - .indices => |indices| indices.transpose(.{ .b, .topk }), - else => zml.Tensor.zeroes(.init(.{ q.dim(.b), 1 }, .i32)), - }; - const cfg: kernel.Config = .{ - .head_dim = q.dim(.hd), - .tokens = q.dim(.b), - .batch = p.block_table.dim(.b), - .kv_heads = k.dim(.hkv), - .group = q.dim(.hg), - .pages = k.dim(.page), - .pages_per_seq = p.block_table.dim(.p), - .page_size = k.dim(.k_chunk), - .dtype = zml.kernel.cute.from(q.dtype()), - .causal = self.opts.mask.isCausal(), - .window = self.opts.mask.slidingWindow() orelse 0, - .scale = self.opts.scale orelse (1 / @sqrt(@as(f32, @floatFromInt(q.dim(.hd))))), - .sink = self.opts.sink != null, - .indexed = self.opts.mask == .indices, - .topk = indices.dim(-1), - }; - const sink = if (self.opts.sink) |s| s.convert(.f32).reshape(.{cfg.kv_heads * cfg.group}) else zml.Tensor.zeroes(.init(.{cfg.kv_heads * cfg.group}, .f32)); - return kernel.Kernel.call(.{ - .q = q, - .k = k, - .v = v, - .table = p.block_table, - .lengths = p.seq_lens, - .starts = p.query_start_len, - .sink = sink, - .indices = indices, - }, .{ .out = q.shape() }, .{ - .cfg = cfg, - .grid = .{ @intCast(cfg.tokens), @intCast(cfg.kv_heads * cfg.group), 1 }, - .block = .{ 32, 1, 1 }, - }).out.transpose(output_shape); - } -}; diff --git a/zml/kernels/paged_attention/cute/fixtures.zig b/zml/kernels/paged_attention/cute/fixtures.zig index 8c289b04..aafb8bf1 100644 --- a/zml/kernels/paged_attention/cute/fixtures.zig +++ b/zml/kernels/paged_attention/cute/fixtures.zig @@ -122,7 +122,7 @@ pub fn forward(i: Inputs, c: cases.Case) zml.Tensor { }, }, }; - return zml.paged_attention.pagedAttention(parameters, i.q, i.k, i.v, .{ .split = .{ .k = i.k, .v = i.v } }, i.options(c)); + return zml.paged_attention.cute.pagedAttention(parameters.cute, i.q, i.k, i.v, i.options(c)); } pub fn upload(io: std.Io, platform: *const zml.Platform, inputs: Inputs, f: Fixture) !zml.Bufferized(Inputs) { diff --git a/zml/kernels/paged_attention/cute/sm10x/sm10x.zig b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig index 664720ee..af068bad 100644 --- a/zml/kernels/paged_attention/cute/sm10x/sm10x.zig +++ b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig @@ -20,60 +20,47 @@ pub fn isAvailable(platform: *const zml.Platform) bool { } pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { - return zml.ops.manualComputation(Context.body, Context{ .q = q, .k = k, .v = v, .parameters = parameters, .opts = opts }, q.shape()); + const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); + const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); + stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); + const decode = !parameters.options_.is_prefill; + const cc = zml.platform.cuda.computeCapability(zml.Compiler.current().platform).?; + var c: kernel.Config = .{ + .architecture = if (cc.minor >= 3) .sm103 else .sm100, + .head_dim = q_.dim(.hd), + .tokens = q_.dim(.b), + .batch = parameters.block_table.dim(.b), + .kv_heads = k_.dim(.hkv), + .group = q_.dim(.hg), + .pages_per_seq = parameters.block_table.dim(.p), + .page_size = k_.dim(.k_chunk), + .max_query = if (decode) 1 else @min(q_.dim(.b), @as(i64, @intCast(parameters.options_.max_seqlen_q))), + .dtype = zml.kernel.cute.from(q_.dtype()), + .decode = decode, + .causal = opts.mask.isCausal(), + .window = opts.mask.slidingWindow() orelse 0, + .scale = opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q_.dim(.hd)))), + .sink = opts.sink != null, + }; + const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); + const work = query_tiles * c.kv_heads; + const bound = if (c.window > 0 and c.causal) @min(c.window + @divTrunc(c.rows() + c.group - 1, c.group), c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; + c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(cute.multiprocessorCount(zml.Compiler.current().platform))) * 2 + work - 1, work), @divTrunc(bound + 2 * c.keys() - 1, 2 * c.keys())))); + // An unused scalar stays a constant buffer; broadcasting a dummy for + // every head would add a helper kernel to each attention invocation. + const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.scalar(0, .f32).reshape(.{1}); + const out = kernel.Attention.call(.{ .q = q_, .k = k_, .v = v_, .table = parameters.block_table, .lengths = parameters.seq_lens, .starts = parameters.query_start_len, .sink = sink }, .{ + .out = if (c.splits > 1) zml.Shape.init(.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 }, .f32) else q_.shape(), + }, .{ .cfg = c, .grid = .{ @intCast(c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, .block = kernel.block }).out; + const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .lengths = parameters.seq_lens, .starts = parameters.query_start_len, .sink = sink }, .{ .out = q_.shape() }, .{ + .cfg = c, + .grid = .{ @intCast(c.tokens), @intCast(c.kv_heads * c.group), 1 }, + .block = .{ 128, 1, 1 }, + }).out else out; + return result.transpose(q.shape()); } -const Context = struct { - q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, - parameters: Parameters, - opts: paged.AttentionOptions, - - fn body(self: Context, output_shape: zml.Shape) zml.Tensor { - const q = self.q.transpose(.{ .b, .hkv, .hg, .hd }); - const k = self.k.transpose(.{ .page, .k_chunk, .hkv, .hd }); - const v = self.v.transpose(.{ .page, .k_chunk, .hkv, .hd }); - stdx.debug.assert(q.dim(.hkv) == k.dim(.hkv) and k.shape().eql(v.shape()) and k.dim(.hd) == q.dim(.hd) and q.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); - const p = self.parameters; - const decode = !p.options_.is_prefill; - const cc = zml.platform.cuda.computeCapability(zml.Compiler.current().platform).?; - var c: kernel.Config = .{ - .architecture = if (cc.minor >= 3) .sm103 else .sm100, - .head_dim = q.dim(.hd), - .tokens = q.dim(.b), - .batch = p.block_table.dim(.b), - .kv_heads = k.dim(.hkv), - .group = q.dim(.hg), - .pages_per_seq = p.block_table.dim(.p), - .page_size = k.dim(.k_chunk), - .max_query = if (decode) 1 else @min(q.dim(.b), @as(i64, @intCast(p.options_.max_seqlen_q))), - .dtype = zml.kernel.cute.from(q.dtype()), - .decode = decode, - .causal = self.opts.mask.isCausal(), - .window = self.opts.mask.slidingWindow() orelse 0, - .scale = self.opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q.dim(.hd)))), - .sink = self.opts.sink != null, - }; - const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); - const work = query_tiles * c.kv_heads; - const bound = if (c.window > 0 and c.causal) @min(c.window + @divTrunc(c.rows() + c.group - 1, c.group), c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; - c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(cute.multiprocessorCount(zml.Compiler.current().platform))) * 2 + work - 1, work), @divTrunc(bound + 2 * c.keys() - 1, 2 * c.keys())))); - // An unused scalar stays a constant buffer; broadcasting a dummy for - // every head would add a helper kernel to each attention invocation. - const sink = if (self.opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.scalar(0, .f32).reshape(.{1}); - const out = kernel.Attention.call(.{ .q = q, .k = k, .v = v, .table = p.block_table, .lengths = p.seq_lens, .starts = p.query_start_len, .sink = sink }, .{ - .out = if (c.splits > 1) zml.Shape.init(.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 }, .f32) else q.shape(), - }, .{ .cfg = c, .grid = .{ @intCast(c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, .block = kernel.block }).out; - const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .lengths = p.seq_lens, .starts = p.query_start_len, .sink = sink }, .{ .out = q.shape() }, .{ - .cfg = c, - .grid = .{ @intCast(c.tokens), @intCast(c.kv_heads * c.group), 1 }, - .block = .{ 128, 1, 1 }, - }).out else out; - return result.transpose(output_shape); - } -}; - const ArchitectureCheck = struct { fn forward(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, c: kernel.Config) zml.Tensor { const table_values: [9]i32 = .{ 4, 2, 8, 1, 6, 0, 7, 3, 5 }; diff --git a/zml/kernels/paged_attention/cute/sm120/sm120.zig b/zml/kernels/paged_attention/cute/sm120/sm120.zig index 6dfeb7e2..292e4a9f 100644 --- a/zml/kernels/paged_attention/cute/sm120/sm120.zig +++ b/zml/kernels/paged_attention/cute/sm120/sm120.zig @@ -19,65 +19,52 @@ pub fn isAvailable(platform: *const zml.Platform) bool { } pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { - return zml.ops.manualComputation(Context.body, Context{ .q = q, .k = k, .v = v, .parameters = parameters, .opts = opts }, q.shape()); + const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); + const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); + stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); + const decode = !parameters.options_.is_prefill; + var c: kernel.Config = .{ + .head_dim = q_.dim(.hd), + .tokens = q_.dim(.b), + .batch = parameters.block_table.dim(.b), + .kv_heads = k_.dim(.hkv), + .group = q_.dim(.hg), + .pages_per_seq = parameters.block_table.dim(.p), + .page_size = k_.dim(.k_chunk), + .max_query = if (decode) 1 else @min(q_.dim(.b), @as(i64, @intCast(parameters.options_.max_seqlen_q))), + .dtype = zml.kernel.cute.from(q_.dtype()), + .warps_per_query = if (decode) 4 else 2, + .kv_tile = if (q_.dim(.hd) == 512) 32 else 64, + .causal = opts.mask.isCausal(), + .window = opts.mask.slidingWindow() orelse 0, + .scale = opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q_.dim(.hd)))), + .sink = opts.sink != null, + }; + // Mixed batches also contain single-token rows. Partition their KV + // range so one long decode cannot hold up all the prefill CTAs. + const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); + const work = query_tiles * c.kv_heads; + const bound = if (c.window > 0) @min(c.window, c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; + c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(cute.multiprocessorCount(zml.Compiler.current().platform))) * 2 + work - 1, work), @divTrunc(bound + 127, 128)))); + const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.zeroes(.init(.{c.kv_heads * c.group}, .f32)); + const out = kernel.Kernel.call(.{ + .q = q_, + .k = k_, + .v = v_, + .table = parameters.block_table, + .lengths = parameters.seq_lens, + .starts = parameters.query_start_len, + .sink = sink, + }, .{ .out = if (c.splits > 1) zml.Shape.init(.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 }, .f32) else q_.shape() }, .{ + .cfg = c, + .grid = .{ @intCast(c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, + .block = .{ 128, 1, 1 }, + }).out; + const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .starts = parameters.query_start_len, .sink = sink }, .{ .out = q_.shape() }, .{ + .cfg = c, + .grid = .{ @intCast(c.tokens), @intCast(c.kv_heads * c.group), 1 }, + .block = .{ 128, 1, 1 }, + }).out else out; + return result.transpose(q.shape()); } - -const Context = struct { - q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, - parameters: Parameters, - opts: paged.AttentionOptions, - - fn body(self: Context, output_shape: zml.Shape) zml.Tensor { - const q = self.q.transpose(.{ .b, .hkv, .hg, .hd }); - const k = self.k.transpose(.{ .page, .k_chunk, .hkv, .hd }); - const v = self.v.transpose(.{ .page, .k_chunk, .hkv, .hd }); - stdx.debug.assert(q.dim(.hkv) == k.dim(.hkv) and k.shape().eql(v.shape()) and k.dim(.hd) == q.dim(.hd) and q.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); - const p = self.parameters; - const decode = !p.options_.is_prefill; - var c: kernel.Config = .{ - .head_dim = q.dim(.hd), - .tokens = q.dim(.b), - .batch = p.block_table.dim(.b), - .kv_heads = k.dim(.hkv), - .group = q.dim(.hg), - .pages_per_seq = p.block_table.dim(.p), - .page_size = k.dim(.k_chunk), - .max_query = if (decode) 1 else @min(q.dim(.b), @as(i64, @intCast(p.options_.max_seqlen_q))), - .dtype = zml.kernel.cute.from(q.dtype()), - .warps_per_query = if (decode) 4 else 2, - .kv_tile = if (q.dim(.hd) == 512) 32 else 64, - .causal = self.opts.mask.isCausal(), - .window = self.opts.mask.slidingWindow() orelse 0, - .scale = self.opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q.dim(.hd)))), - .sink = self.opts.sink != null, - }; - // Mixed batches also contain single-token rows. Partition their KV - // range so one long decode cannot hold up all the prefill CTAs. - const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); - const work = query_tiles * c.kv_heads; - const bound = if (c.window > 0) @min(c.window, c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; - c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(cute.multiprocessorCount(zml.Compiler.current().platform))) * 2 + work - 1, work), @divTrunc(bound + 127, 128)))); - const sink = if (self.opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.zeroes(.init(.{c.kv_heads * c.group}, .f32)); - const out = kernel.Kernel.call(.{ - .q = q, - .k = k, - .v = v, - .table = p.block_table, - .lengths = p.seq_lens, - .starts = p.query_start_len, - .sink = sink, - }, .{ .out = if (c.splits > 1) zml.Shape.init(.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 }, .f32) else q.shape() }, .{ - .cfg = c, - .grid = .{ @intCast(c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, - .block = .{ 128, 1, 1 }, - }).out; - const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .starts = p.query_start_len, .sink = sink }, .{ .out = q.shape() }, .{ - .cfg = c, - .grid = .{ @intCast(c.tokens), @intCast(c.kv_heads * c.group), 1 }, - .block = .{ 128, 1, 1 }, - }).out else out; - return result.transpose(output_shape); - } -}; diff --git a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig index ef9e852c..42e1861f 100644 --- a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig @@ -153,32 +153,28 @@ pub const mosaic_tpu = struct { "mosaic_tpu ragged paged attention q head_dim {} exceeds kernel/cache head_dim {}", .{ q.dim(.hd), target_head_dim }, ); - var q_padded = q.pad(0, .{ .hd = zml.Tensor.Pad{ .high = target_head_dim - q.dim(.hd) } }); - q_padded._shape._sharding = q._shape._sharding; - return q_padded; + return q.pad(0, .{ .hd = zml.Tensor.Pad{ .high = target_head_dim - q.dim(.hd) } }); } inline fn prepareInputs(parameters: Parameters, q: zml.Tensor, kv_pages: zml.Tensor) PreparedInputs { - const mesh = zml.Compiler.current().resolveMesh(.{.model}); - const kv_pages_partitioned = kv_pages.withPartitioning(mesh, .{ .hkv = .model }); - const q_merged = q.merge(.{ .h = .{ .hkv, .hg } }).withPartitioning(mesh, .{ .h = .model }); + const q_merged = q.merge(.{ .h = .{ .hkv, .hg } }); const logical_head_dim: usize = @intCast(q_merged.dim(.hd)); - const cache_head_dim: usize = @intCast(kv_pages_partitioned.dim(.hd)); + const cache_head_dim: usize = @intCast(kv_pages.dim(.hd)); stdx.debug.assert( cache_head_dim == alignedKernelHeadDim(logical_head_dim), "mosaic_tpu ragged paged attention expects KV cache head_dim {} to match aligned logical head_dim {}", .{ cache_head_dim, logical_head_dim }, ); - const q_ragged = alignQueryHeadDimForKernel(q_merged, kv_pages_partitioned.dim(.hd)); - const query_start_len = parameters.query_start_len.withPartitioning(mesh, .{ .b = .replicated }); + const q_ragged = alignQueryHeadDimForKernel(q_merged, kv_pages.dim(.hd)); + const query_start_len = parameters.query_start_len; return .{ .q = q_ragged, - .kv_pages = kv_pages_partitioned, - .seq_lens = parameters.seq_lens.withPartitioning(mesh, .{ ._0 = .replicated }), - .block_table = parameters.block_table.withPartitioning(mesh, .{ ._0 = .replicated, ._1 = .replicated }), + .kv_pages = kv_pages, + .seq_lens = parameters.seq_lens, + .block_table = parameters.block_table, .query_start_len = query_start_len, - .num_seqs = activeSequenceCount(query_start_len).withPartitioning(mesh, .{ ._0 = .replicated }), + .num_seqs = activeSequenceCount(query_start_len), }; } @@ -224,57 +220,9 @@ pub const mosaic_tpu = struct { pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, kv_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { if (opts.mask == .indices) @panic("Sparse Attention is not supported on TPU"); - const prepared = prepareInputs(parameters, q, kv_cache); - - const q_out = zml.ops.manualComputation( - (struct { - q: zml.Tensor, - kv_pages: zml.Tensor, - seq_lens: zml.Tensor, - block_table: zml.Tensor, - query_start_len: zml.Tensor, - num_seqs: zml.Tensor, - opts: AttentionOptions, - parameters: Parameters, - - fn body(self: @This(), output: zml.Shape) zml.Tensor { - const prepared_inputs: PreparedInputs = .{ - .q = self.q, - .kv_pages = self.kv_pages, - .seq_lens = self.seq_lens, - .block_table = self.block_table, - .query_start_len = self.query_start_len, - .num_seqs = self.num_seqs, - }; - - const cfg = buildCfg(prepared_inputs, self.parameters, self.opts); - const q_out = raggedPagedKernelCall( - prepared_inputs.q, - prepared_inputs.kv_pages, - prepared_inputs.seq_lens, - prepared_inputs.block_table, - prepared_inputs.query_start_len, - prepared_inputs.num_seqs, - cfg, - ); - stdx.debug.assert(q_out.shape().eql(output), "mosaic_tpu ragged paged attention output shape mismatch, got {f}, expected {f}", .{ q_out.shape(), output }); - return q_out; - } - }).body, - .{ - .q = prepared.q, - .kv_pages = prepared.kv_pages, - .seq_lens = prepared.seq_lens, - .block_table = prepared.block_table, - .query_start_len = prepared.query_start_len, - .num_seqs = prepared.num_seqs, - .opts = opts, - .parameters = parameters, - }, - prepared.q.shape(), - ); - + const cfg = buildCfg(prepared, parameters, opts); + const q_out = raggedPagedKernelCall(prepared.q, prepared.kv_pages, prepared.seq_lens, prepared.block_table, prepared.query_start_len, prepared.num_seqs, cfg); const restored = restoreQueryHeads(q, q_out); stdx.debug.assert(restored.shape().eql(q.shape()), "mosaic_tpu ragged paged attention output shape mismatch, got {f}, expected {f}", .{ restored.shape(), q.shape() }); return restored; diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index 65cf6cea..380b4cc2 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -3,7 +3,7 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../../zml.zig"); -const cute = @import("cute/cute.zig"); +pub const cute = @import("cute/cute.zig"); const metal = @import("metal/metal.zig"); const tpu = @import("mosaic_tpu/mosaic_tpu.zig"); const stablehlo = @import("stablehlo/stablehlo.zig"); @@ -247,34 +247,48 @@ pub const KvCache = union(enum) { } }; -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache, opts: AttentionOptions) zml.Tensor { +pub const Partitioning = zml.ops.ManualPartitioning(Attention.forward); + +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache, opts: AttentionOptions, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { _ = k; _ = v; - return switch (parameters) { - .cute => |p| switch (kv_cache) { - .split => |split| cute.pagedAttention(p, q, split.k, split.v, opts), - .dense => @panic("CuTe paged attention requires split KV pages"), - }, - .triton => |triton_parameters| switch (kv_cache) { - .split => |split| triton.paged.pagedAttention(triton_parameters, q, split.k, split.v, opts), - .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), - }, - .mosaic_tpu => |mosaic_tpu_parameters| switch (kv_cache) { - .split => std.debug.panic("split KV pages is not supported with the mosaic_tpu backend", .{}), - .dense => tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, q, kv_cache.dense, opts), - }, - .metal => |metal_parameters| switch (kv_cache) { - .split => |split| metal.pagedAttention(metal_parameters, q, split.k, split.v, opts), - .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), - }, - .stablehlo => |params| stablehlo.pagedAttention(params, q, kv_cache, opts), - .tcl => |params| switch (kv_cache) { - .split => |split| tcl.paged.pagedAttention(params, q, split.k, split.v, opts), - .dense => std.debug.panic("TCL paged attention takes separate K and V pages", .{}), - }, - }; + const args: Attention = .{ .parameters = parameters, .q = q, .kv_cache = kv_cache, .opts = opts }; + return zml.ops.manualComputation2(Attention.forward, args, q.shape(), mesh, partitioning); } +const Attention = struct { + parameters: Parameters, + q: zml.Tensor, + kv_cache: KvCache, + opts: AttentionOptions, + + fn forward(args: @This(), _: zml.Shape) zml.Tensor { + return switch (args.parameters) { + .cute => |p| switch (args.kv_cache) { + .split => |split| cute.pagedAttention(p, args.q, split.k, split.v, args.opts), + .dense => @panic("CuTe paged attention requires split KV pages"), + }, + .triton => |triton_parameters| switch (args.kv_cache) { + .split => |split| triton.paged.pagedAttention(triton_parameters, args.q, split.k, split.v, args.opts), + .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + }, + .mosaic_tpu => |mosaic_tpu_parameters| switch (args.kv_cache) { + .split => std.debug.panic("split KV pages is not supported with the mosaic_tpu backend", .{}), + .dense => tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, args.q, args.kv_cache.dense, args.opts), + }, + .metal => |metal_parameters| switch (args.kv_cache) { + .split => |split| metal.pagedAttention(metal_parameters, args.q, split.k, split.v, args.opts), + .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + }, + .stablehlo => |params| stablehlo.pagedAttention(params, args.q, args.kv_cache, args.opts), + .tcl => |params| switch (args.kv_cache) { + .split => |split| tcl.paged.pagedAttention(params, args.q, split.k, split.v, args.opts), + .dense => std.debug.panic("TCL paged attention takes separate K and V pages", .{}), + }, + }; + } +}; + test "Backend.auto selects mosaic_tpu on TPU" { const platform: zml.Platform = .{ .arena = undefined, @@ -463,11 +477,25 @@ test pagedAttention { backend_options_args.backend = backend; const parameters: Parameters = .init(.fromBackend(backend_options_args)); + const head_spec: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); + const ParameterSpecs = zml.meta.MapRestrict(zml.Tensor, zml.Sharding.Partitioning.Logical).map(Parameters); + const partitioning: Partitioning = .{ + .in_specs = .{ + .parameters = switch (parameters) { + inline else => |_, tag| @unionInit(ParameterSpecs, @tagName(tag), .{ .block_table = .replicated, .seq_lens = .replicated, .query_start_len = .replicated }), + }, + .q = head_spec, + .kv_cache = .{ .split = .{ .k = head_spec, .v = head_spec } }, + .opts = .{ .mask = .{ .standard = {} }, .sink = null }, + }, + .out_specs = head_spec, + }; + const exe = try platform.compileFn( allocator, io, pagedAttention, - .{ parameters, tensors.q, tensors.k, tensors.v, tensors.kv_cache, test_case.attention_options }, + .{ parameters, tensors.q, tensors.k, tensors.v, tensors.kv_cache, test_case.attention_options, tp, partitioning }, .{ .program_name = try std.fmt.allocPrint(arena, "paged_attention_{s}_{t}", .{ test_case.name, backend }), .meshes = meshes }, ); defer exe.deinit(); diff --git a/zml/kernels/paged_attention/tcl/tcl.zig b/zml/kernels/paged_attention/tcl/tcl.zig index a7c2a5f4..828811f3 100644 --- a/zml/kernels/paged_attention/tcl/tcl.zig +++ b/zml/kernels/paged_attention/tcl/tcl.zig @@ -1655,7 +1655,7 @@ fn runCases(comptime max_pages: usize, comptime num_pages: usize, cases: []const const q_t: Tensor = .init(.{ .b = tokens, .hkv = hkv, .hg = g, .hd = d }, dims.dtype); const Mod = struct { pub fn forward(p: paged.Parameters, q: Tensor, k: Tensor, v: Tensor, opts: paged_attention.AttentionOptions) Tensor { - return paged_attention.pagedAttention(.{ .tcl = p }, q, k, v, .{ .split = .{ .k = k, .v = v } }, opts); + return paged.pagedAttention(p, q, k, v, opts); } }; const opts: paged_attention.AttentionOptions = .{ diff --git a/zml/kernels/paged_attention/triton/triton.zig b/zml/kernels/paged_attention/triton/triton.zig index 1ccbeba8..67416285 100644 --- a/zml/kernels/paged_attention/triton/triton.zig +++ b/zml/kernels/paged_attention/triton/triton.zig @@ -274,92 +274,57 @@ pub const paged = struct { return pagedSparseAttention(parameters, q, k_cache, v_cache, opts); } - const output = zml.ops.manualComputation( - (struct { - q: zml.Tensor, - k_cache: zml.Tensor, - v_cache: zml.Tensor, - block_table: zml.Tensor, - seq_lens: zml.Tensor, - query_start_len: zml.Tensor, - opts: AttentionOptions, - options: Options, + const cu_count = getCuCount(); + const num_heads: usize = @intCast(q.dim(.hkv) * q.dim(.hg)); + const num_kv_heads: usize = @intCast(k_cache.dim(.hkv)); + const num_queries_per_kv: usize = num_heads / num_kv_heads; + // Intel decode: pack exactly one GQA group per tile (block_q == 1) so the + // single decode query token doesn't carry masked-out fp32 acc lanes. + // oneAPI decode keeps one GQA group per tile, padded to a power of two so tt.make_range emits legal Triton IR. + const block_m: usize = if (!parameters.options_.is_prefill and isOneapiTarget()) + std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv) + else if (num_queries_per_kv <= 16) 16 else std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv); + const block_q: usize = block_m / num_queries_per_kv; + const num_tokens: usize = @intCast(q.dim(.b)); + const num_seqs: usize = @intCast(parameters.block_table.dim(.b)); + const total_q_blocks: usize = num_tokens / block_q + num_seqs; + const target_num_prgms: usize = cu_count * 4; + const num_2d_prgms: usize = total_q_blocks * num_kv_heads; - fn body(self: @This(), _: zml.Shape) zml.Tensor { - const parameters_: Parameters = .{ - .block_table = self.block_table, - .seq_lens = self.seq_lens, - .query_start_len = self.query_start_len, - .options_ = self.options, - }; + const sliding_window: usize = opts.mask.slidingWindow() orelse 0; - const cu_count = getCuCount(); - const num_heads: usize = @intCast(self.q.dim(.hkv) * self.q.dim(.hg)); - const num_kv_heads: usize = @intCast(self.k_cache.dim(.hkv)); - const num_queries_per_kv: usize = num_heads / num_kv_heads; - // Intel decode: pack exactly one GQA group per tile (block_q == 1) so the - // single decode query token doesn't carry masked-out fp32 acc lanes. - // oneAPI decode keeps one GQA group per tile, padded to a power of two so tt.make_range emits legal Triton IR. - const block_m: usize = if (!self.options.is_prefill and isOneapiTarget()) - std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv) - else if (num_queries_per_kv <= 16) 16 else std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv); - const block_q: usize = block_m / num_queries_per_kv; - const num_tokens: usize = @intCast(self.q.dim(.b)); - const num_seqs: usize = @intCast(parameters_.block_table.dim(.b)); - const total_q_blocks: usize = num_tokens / block_q + num_seqs; - const target_num_prgms: usize = cu_count * 4; - const num_2d_prgms: usize = total_q_blocks * num_kv_heads; + const paged_attention_opts: PagedAttentionOptions = .{ + .cu_count = getCuCount(), + .all_decode = !parameters.options_.is_prefill, + .num_tokens = num_tokens, + .num_heads = num_heads, + .num_kv_heads = num_kv_heads, + .head_dim = @intCast(q.dim(.hd)), + .batch_size = @intCast(parameters.block_table.dim(.b)), + .block_size = @intCast(k_cache.dim(.k_chunk)), + .num_blocks = @intCast(k_cache.dim(.page)), + .max_num_block_per_seq = @intCast(parameters.block_table.dim(.p)), + .sliding_window = sliding_window, + .block_m = block_m, + .block_q = block_q, + .total_q_blocks = total_q_blocks, + .target_num_prgms = target_num_prgms, + .num_2d_prgms = num_2d_prgms, + .max_seqlen_q = parameters.options_.max_seqlen_q, + .scale = opts.scale, + }; - const sliding_window: usize = self.opts.mask.slidingWindow() orelse 0; - - const paged_attention_opts: PagedAttentionOptions = .{ - .cu_count = getCuCount(), - .all_decode = !self.options.is_prefill, - .num_tokens = num_tokens, - .num_heads = num_heads, - .num_kv_heads = num_kv_heads, - .head_dim = @intCast(self.q.dim(.hd)), - .batch_size = @intCast(parameters_.block_table.dim(.b)), - .block_size = @intCast(self.k_cache.dim(.k_chunk)), - .num_blocks = @intCast(self.k_cache.dim(.page)), - .max_num_block_per_seq = @intCast(parameters_.block_table.dim(.p)), - .sliding_window = sliding_window, - .block_m = block_m, - .block_q = block_q, - .total_q_blocks = total_q_blocks, - .target_num_prgms = target_num_prgms, - .num_2d_prgms = num_2d_prgms, - .max_seqlen_q = self.options.max_seqlen_q, - .scale = self.opts.scale, - }; - - const use_2d_kernel = use2dKernel( - paged_attention_opts.all_decode, - paged_attention_opts.batch_size, - paged_attention_opts.num_kv_heads, - ); - const output = if (use_2d_kernel) - pagedAttention2d(parameters_, self.q, self.k_cache, self.v_cache, self.opts, paged_attention_opts) - else if (isOneapiTarget()) - pagedAttention3dOneapi(parameters_, self.q, self.k_cache, self.v_cache, self.opts, paged_attention_opts) - else - pagedAttention3d(parameters_, self.q, self.k_cache, self.v_cache, self.opts, paged_attention_opts); - - return output; - } - }).body, - .{ - .q = q, - .k_cache = k_cache, - .v_cache = v_cache, - .block_table = parameters.block_table, - .seq_lens = parameters.seq_lens, - .query_start_len = parameters.query_start_len, - .opts = opts, - .options = parameters.options_, - }, - q.shape(), + const use_2d_kernel = use2dKernel( + paged_attention_opts.all_decode, + paged_attention_opts.batch_size, + paged_attention_opts.num_kv_heads, ); + const output = if (use_2d_kernel) + pagedAttention2d(parameters, q, k_cache, v_cache, opts, paged_attention_opts) + else if (isOneapiTarget()) + pagedAttention3dOneapi(parameters, q, k_cache, v_cache, opts, paged_attention_opts) + else + pagedAttention3d(parameters, q, k_cache, v_cache, opts, paged_attention_opts); return output; } @@ -372,51 +337,18 @@ pub const paged = struct { stdx.debug.assert(indices.shape().hasTags(.{ .b, .topk }), "Sparse indices expect to have .b, .topk tags", .{}); stdx.debug.assert(q.dtype() == k_cache.dtype() and q.dtype() == v_cache.dtype(), "Sparse attention requires matching query and KV dtypes", .{}); - return zml.ops.manualComputation( - (struct { - q: zml.Tensor, - k_cache: zml.Tensor, - v_cache: zml.Tensor, - block_table: zml.Tensor, - seq_lens: zml.Tensor, - query_start_len: zml.Tensor, - opts: AttentionOptions, - options: Options, - - fn body(self: @This(), _: zml.Shape) zml.Tensor { - const parameters_: Parameters = .{ - .block_table = self.block_table, - .seq_lens = self.seq_lens, - .query_start_len = self.query_start_len, - .options_ = self.options, - }; - - const config = selectSparseConfig( - @intCast(self.opts.mask.indices.dim(.topk)), - @intCast(self.block_table.dim(.b)), - @intCast(self.q.dim(.hkv)), - getCuCount(), - !self.options.is_prefill, - ); - - return if (config.num_segments == 1) - pagedSparseAttention2d(parameters_, self.q, self.k_cache, self.v_cache, self.opts, config) - else - pagedSparseAttention3d(parameters_, self.q, self.k_cache, self.v_cache, self.opts, config); - } - }).body, - .{ - .q = q, - .k_cache = k_cache, - .v_cache = v_cache, - .block_table = parameters.block_table, - .seq_lens = parameters.seq_lens, - .query_start_len = parameters.query_start_len, - .opts = opts, - .options = parameters.options_, - }, - q.shape(), + const config = selectSparseConfig( + @intCast(opts.mask.indices.dim(.topk)), + @intCast(parameters.block_table.dim(.b)), + @intCast(q.dim(.hkv)), + getCuCount(), + !parameters.options_.is_prefill, ); + + return if (config.num_segments == 1) + pagedSparseAttention2d(parameters, q, k_cache, v_cache, opts, config) + else + pagedSparseAttention3d(parameters, q, k_cache, v_cache, opts, config); } pub fn pagedSparseAttention2d(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, tuning: SparseConfig) zml.Tensor { From 5decef1fcabb00ed653bc547c4c8dcd6a80a4b51 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 5 Nov 2026 14:17:44 +0100 Subject: [PATCH 851/881] zml/kernels/paged_attention: enable cute by default when available (#1017) --- zml/kernels/paged_attention/paged_attention.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index 380b4cc2..81f7ec3e 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -30,7 +30,7 @@ pub const Backend = enum { pub fn auto(platform: *const zml.Platform) Backend { return switch (platform.target) { - .cuda => .triton, + .cuda => if (cute.isAvailable(platform)) .cute else .triton, .rocm => .triton, .oneapi => .triton, .tpu => .mosaic_tpu, From 3285b7e0f42013d6079c70a641ad53da84399f91 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Thu, 5 Nov 2026 17:38:31 +0100 Subject: [PATCH 852/881] zml: FP8 block activations and paged attention query padding on Furiosa and bump plugins (#1033) - Quantize block FP8 activations to E4M3FN on Furiosa, so scaled dots run natively. Other block sizes keep BF16 activations and XLA's fallback. - Pad paged attention query groups to a power of two with zero queries, as TCC fails other group sizes. Marked as a workaround. - Bump to [202611051630.351.1.48d539621544](https://github.com/zml/xla-private/tree/202611051630.351.1.48d539621544) --- platforms/cpu/cpu.bzl | 6 +++--- platforms/cuda/cuda.bzl | 8 ++++---- platforms/furiosa/furiosa.bzl | 4 ++-- platforms/metal/metal.bzl | 2 +- platforms/rocm/rocm.bzl | 2 +- zml/kernels/paged_attention/tcl/tcl.zig | 18 ++++++++++++++---- zml/nn.zig | 2 +- zml/quantization.zig | 2 ++ 8 files changed, 28 insertions(+), 16 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index b33ef4df..a72c4b15 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -43,21 +43,21 @@ def _cpu_plugin_impl(mctx): name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, sha256 = "3d6a6d1b047a0943b50fd9e63e5f6dff6b7cf184b186c0cc1052de40f0dbec92", - url = "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-cpu-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "d6b4b15b13a2d1a3e0b84f143f568cd572bef5db91784f9f7a826173ed626c7e", - url = "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-cpu-darwin-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "c06babc001edeab313bc63daab3aa3d40e087266a3a29634d182628fb65fad14", - url = "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-cpu-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index a0c73948..c72ef3be 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -224,12 +224,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "b68b407efcd361bbe4b297a16feb080eabbae9132a7ee1a5b105e24a91d99de0", - "url": "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-cuda-linux-amd64.tar.zst", + "sha256": "1ae8a7c8fc677e0f10c0032ce9ae35e5b6c3f5eadc3f54e1bbda63c219b0170d", + "url": "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "9ac41221901eba84641aef35d1f53ff69ce471c5af3844550cd086abc6916f5c", - "url": "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-cuda-linux-arm64.tar.zst", + "sha256": "14a3567c629a1ea97f1dcfa2cb47b54ea08d8c4b9e35eae47a7eff33cfb53ce0", + "url": "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index fc2e5352..b623000b 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-furiosa-linux-amd64.tar.zst", - sha256 = "85edea12494278a015c1ad29cf57401217e4809bc3a0c6890d0279659bb6209d", + url = "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-furiosa-linux-amd64.tar.zst", + sha256 = "b4c0202ec27f87b82a92b014c6b58a6bd684a7acf65eebcd8ec357f8580b2451", ) return mctx.extension_metadata( diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 2d43a991..999cedfa 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -5,7 +5,7 @@ def _metal_impl(mctx): name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", sha256 = "bdd2de3ec8293faa25b358cdf35ee3e75d69be73a4acea87b6e007a18b8f10e8", - url = "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-metal-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 0c1f7a92..5b18a9c9 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,7 +11,7 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202611041808.348.1.a83ad22d7ed5/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-rocm-linux-amd64.tar.zst" _PLUGIN_ROCM_SHA256 = "02b86fa3a6599f6aa2b89806b8f6c0b16f11765ec04c94517ec1463a59d92175" def _rocm_package_name(name): diff --git a/zml/kernels/paged_attention/tcl/tcl.zig b/zml/kernels/paged_attention/tcl/tcl.zig index 828811f3..c9993af9 100644 --- a/zml/kernels/paged_attention/tcl/tcl.zig +++ b/zml/kernels/paged_attention/tcl/tcl.zig @@ -152,21 +152,31 @@ pub const paged = struct { cfg.chunk = if (plan.chunk > 0) plan.chunk else keys; cfg.a = (std.math.divCeil(i64, keys, cfg.chunk) catch unreachable) * cfg.chunk; const table = parameters.block_table.pad(-1, .{ .p = Tensor.Pad{ .high = @divExact(cfg.a, page) - parameters.block_table.dim(.p) } }); + // WORKAROUND (TCC 2026.5.0b0): attention over query groups of a size + // other than a power of two fails ("Inconsistent tensor group + // offsets"): pad the groups with zero queries, whose outputs are + // dropped. Repro: + // repro_flash_prefill_group6_inconsistent_offsets.py (zml/reproducer, + // platform/furiosa). Remove once any group size compiles. + const groups: i64 = @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.g)) catch unreachable); + const extra = groups - cfg.g; + cfg.g = groups; + const queries = q.convert(.bf16).pad(0, .{ .hg = Tensor.Pad{ .high = extra } }); const inputs: OneShot.Inputs = .{ - .q = q.convert(.bf16), + .q = queries, .pages = table, .seq_lens = parameters.seq_lens, .query_offsets = parameters.query_start_len, .k_cache = k_cache.convert(.bf16), .v_cache = v_cache.convert(.bf16), - .sink = sink, + .sink = sink.pad(0, .{ .hg = Tensor.Pad{ .high = extra } }), }; - const outputs: OneShot.Outputs = .{ .out = q.shape().withDtype(.bf16) }; + const outputs: OneShot.Outputs = .{ .out = queries.shape() }; const out = if (cfg.a == cfg.chunk) OneShot.call(inputs, outputs, .{ .cfg = cfg }).out else Flash.call(inputs, outputs, .{ .cfg = cfg }).out; - return out.convert(q.dtype()); + return out.slice(.hg, .{ .end = groups - extra }).convert(q.dtype()); } }; diff --git a/zml/nn.zig b/zml/nn.zig index 4569552f..a2b582ee 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -226,7 +226,7 @@ test "block128 scaled dot layouts" { const io = std.testing.io; const platform = zml.testing.env(); // FIXME: Add rocm when we have a PJRT plugin with native fp8 dot support (by the end of Sept 2026) - if (platform.target != .cuda) return error.SkipZigTest; + if (platform.target != .cuda and platform.target != .furiosa) return error.SkipZigTest; const dtype: DataType = if (platform.target == .rocm and @import("platform.zig").rocm.computeCapability(platform) == .gfx942) .f8e4m3fnuz else .f8e4m3fn; const Local = struct { const Outputs = struct { actual: Tensor, expected: Tensor, linear: Tensor, linear_expected: Tensor }; diff --git a/zml/quantization.zig b/zml/quantization.zig index 89e1b210..db20619a 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -129,6 +129,8 @@ pub fn quantizeInput(quantization: Quantization, input: Tensor, axis: Shape.Tag, .fp8_block32, .fp8_block128 => blk: { const dtype: DataType = switch (platform.target) { .cuda => .f8e4m3fn, + // RNGD contracts matching FP8 pairs in 128-wide K blocks. + .furiosa => if (quantization.scheme == .fp8_block128) .f8e4m3fn else return null, .rocm => if (platform_mod.rocm.computeCapability(platform)) |capability| switch (capability.architecture()) { .cdna4, .rdna4 => .f8e4m3fn, .cdna3 => .f8e4m3fnuz, From 630d96726bf0c39547f6106f7b917397be4e312c Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 5 Nov 2026 19:34:17 +0100 Subject: [PATCH 853/881] zml/io: add `consolidated` version of safetensors to defaults (#1035) Some models entrypoints are exposed as `consolidated.safetensors.index.json` or `consolidated.safetensors`. --- zml/safetensors.zig | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/zml/safetensors.zig b/zml/safetensors.zig index 63791ea9..f854f02f 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -595,17 +595,16 @@ pub fn resolveModelRepo( } pub fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!struct { std.Io.File, []const u8 } { - { - const index_name = "model.safetensors.index.json"; - if (repo.openFile(io, index_name, .{ .mode = .read_only })) |index_file| { - return .{ index_file, "model.safetensors.index.json" }; - } else |_| {} - } + const default_entrypoints = .{ + "model.safetensors.index.json", + "consolidated.safetensors.index.json", + "model.safetensors", + "consolidated.safetensors", + }; - { - const model_name = "model.safetensors"; - if (repo.openFile(io, model_name, .{ .mode = .read_only })) |model_file| { - return .{ model_file, "model.safetensors" }; + inline for (default_entrypoints) |entrypoint| { + if (repo.openFile(io, entrypoint, .{ .mode = .read_only })) |index_file| { + return .{ index_file, entrypoint }; } else |_| {} } From 837e45022a2b95a27af846eb106970ec71a5e159 Mon Sep 17 00:00:00 2001 From: erwanvivien <44021072+erwanvivien@users.noreply.github.com> Date: Fri, 6 Nov 2026 09:57:09 +0100 Subject: [PATCH 854/881] zml/kernels/paged_attention: fix stablehlo backend slot_id increment (#1027) If sequences were not 16-aligned we would have sequence leaks. Now we increase slot_id by query_start_len and pad when needed --- .../paged_attention/stablehlo/stablehlo.zig | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/zml/kernels/paged_attention/stablehlo/stablehlo.zig b/zml/kernels/paged_attention/stablehlo/stablehlo.zig index ab0ffee2..7ee4df03 100644 --- a/zml/kernels/paged_attention/stablehlo/stablehlo.zig +++ b/zml/kernels/paged_attention/stablehlo/stablehlo.zig @@ -15,10 +15,11 @@ pub fn pagedAttention( } const page_size = kv_cache.split.k.dim(.k_chunk); + const padded_q = q.pad(0, .{ .b = zml.Tensor.Pad{ .high = page_size } }); const final_state = zml.ops.@"while"( AttentionLoop, - .{ .q = q, .kv_cache = kv_cache, .parameters = parameters, .opts = opts }, + .{ .q = padded_q, .kv_cache = kv_cache, .parameters = parameters, .opts = opts }, .{ // i32 to match triton conventions .slot_id = .scalar(0, .i32), @@ -28,11 +29,11 @@ pub fn pagedAttention( // Note: we do all partial softmax in f32 .partial_softmax_prefill = .zeroes(q.shape().setDim(.b, page_size).withDtype(.f32)), .partial_softmax_decode = .zeroes(q.shape().setDim(.b, 1).withDtype(.f32)), - .out = .zeroes(q.shape()), + .out = .zeroes(padded_q.shape()), }, ); - return final_state.out; + return final_state.out.slice(.b, .{ .end = q.dim(.b) }); } const AttentionLoop = struct { @@ -93,7 +94,7 @@ const AttentionLoop = struct { const sequence_has_more_queries: zml.Tensor = .cmp(state.q_page_idx.addConstant(1).scale(num_slots), .LT, if_ctx.seq_num_queries); const partial_softmax = while_body.attentionOnePage(state, q_offset, num_slots, page_size); - return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax); + return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax, while_body.nextQueryStart(state)); } /// Decode version: num_slots == 1, we multiply one query with a page of keys. @@ -112,7 +113,7 @@ const AttentionLoop = struct { const sequence_has_more_queries: zml.Tensor = .scalar(false, .bool); const partial_softmax = while_body.attentionOnePage(state, q_offset, num_slots, page_size); - return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax); + return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax, while_body.nextQueryStart(state)); } }, .{ @@ -162,7 +163,11 @@ const AttentionLoop = struct { return og_softmax.merge(partial); } - pub fn updateState(state: State, num_slots: u32, query_has_more_keys: zml.Tensor, sequence_has_more_queries: zml.Tensor, softmax: PartialSoftmax) State { + fn nextQueryStart(self: While, state: State) zml.Tensor { + return self.parameters.query_start_len.slice(0, .dynSingle(state.seq_id.addConstant(1))).asScalar(); + } + + pub fn updateState(state: State, num_slots: u32, query_has_more_keys: zml.Tensor, sequence_has_more_queries: zml.Tensor, softmax: PartialSoftmax, next_query_start: zml.Tensor) State { return zml.ops.if2( query_has_more_keys, // If the query has more pages: update partial softmax with attention on current page, delay updating output. @@ -181,7 +186,7 @@ const AttentionLoop = struct { .partial_softmax_prefill = if (num_slots == 1) state.partial_softmax_prefill else .reset(softmax), .partial_softmax_decode = if (num_slots == 1) .reset(softmax) else state.partial_softmax_decode, .out = state.out.dynamicUpdateSlice(.{ .b = state.slot_id }, softmax.finalize().convert(state.out.dtype())), - .slot_id = state.slot_id.addConstant(num_slots), + .slot_id = .select(sequence_has_more_queries, state.slot_id.addConstant(num_slots), next_query_start), // If the sequence has more query chunks: increment q_page_idx, otherwise start over at 0. .q_page_idx = .select(sequence_has_more_queries, state.q_page_idx.addConstant(1), .scalar(0, .i32)), // If the sequence has more query chunks: keep seq_id, otherwise increment seq_id. From 488c21c89945d669f5d1f46be3cbbc2b51fe040a Mon Sep 17 00:00:00 2001 From: PierreLgol Date: Fri, 6 Nov 2026 10:29:31 +0100 Subject: [PATCH 855/881] workspace: add Helix editor configuration for Zig (#1029) adding basic support for the helix editor, to automatically work --- .helix/languages.toml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .helix/languages.toml diff --git a/.helix/languages.toml b/.helix/languages.toml new file mode 100644 index 00000000..529cdcf3 --- /dev/null +++ b/.helix/languages.toml @@ -0,0 +1,19 @@ +[[language]] +auto-format = true +file-types = [ "zig" ] +language-servers = [ "zls" ] +name = "zig" +roots = [ "build.zig", "build.zig.zon", "zls.build.json" ] +scope = "source.zig" + +[language.indent] +tab-width = 4 +unit = " " + +[language-server.zls] +command = "sh" +args = [ + "-c", + "exec \"$(git rev-parse --show-toplevel)/tools/zls.sh\" \"$@\"", + "zls" +] From 5b079127bb3b8f9bc1e5e8ab773b99c274277d87 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Fri, 6 Nov 2026 10:56:05 +0100 Subject: [PATCH 856/881] workspace: bump ZML plugins (#1034) Bump plugins to [202611060925.355.1.2ca211fa5fbd](https://github.com/zml/xla-private/tree/202611060925.355.1.2ca211fa5fbd) --- platforms/cpu/cpu.bzl | 6 +++--- platforms/cuda/cuda.bzl | 4 ++-- platforms/furiosa/furiosa.bzl | 4 ++-- platforms/metal/metal.bzl | 2 +- platforms/rocm/rocm.bzl | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index a72c4b15..fd070b6e 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -43,21 +43,21 @@ def _cpu_plugin_impl(mctx): name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, sha256 = "3d6a6d1b047a0943b50fd9e63e5f6dff6b7cf184b186c0cc1052de40f0dbec92", - url = "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-cpu-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "d6b4b15b13a2d1a3e0b84f143f568cd572bef5db91784f9f7a826173ed626c7e", - url = "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-cpu-darwin-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "c06babc001edeab313bc63daab3aa3d40e087266a3a29634d182628fb65fad14", - url = "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-cpu-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index c72ef3be..24718ee6 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -225,11 +225,11 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { "sha256": "1ae8a7c8fc677e0f10c0032ce9ae35e5b6c3f5eadc3f54e1bbda63c219b0170d", - "url": "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-cuda-linux-amd64.tar.zst", + "url": "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-cuda-linux-amd64.tar.zst", }, "arm64": { "sha256": "14a3567c629a1ea97f1dcfa2cb47b54ea08d8c4b9e35eae47a7eff33cfb53ce0", - "url": "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-cuda-linux-arm64.tar.zst", + "url": "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index b623000b..ba53bfb4 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-furiosa-linux-amd64.tar.zst", - sha256 = "b4c0202ec27f87b82a92b014c6b58a6bd684a7acf65eebcd8ec357f8580b2451", + url = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-furiosa-linux-amd64.tar.zst", + sha256 = "514b1c96a2237380a9b9e011f5b994c6617eb103a6a9dac8ff0a7f1832d1a3bd", ) return mctx.extension_metadata( diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 999cedfa..767ef2f6 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -5,7 +5,7 @@ def _metal_impl(mctx): name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", sha256 = "bdd2de3ec8293faa25b358cdf35ee3e75d69be73a4acea87b6e007a18b8f10e8", - url = "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-metal-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 5b18a9c9..454bf626 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,7 +11,7 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202611051630.351.1.48d539621544/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-rocm-linux-amd64.tar.zst" _PLUGIN_ROCM_SHA256 = "02b86fa3a6599f6aa2b89806b8f6c0b16f11765ec04c94517ec1463a59d92175" def _rocm_package_name(name): From 59a888f7e2f5049e9addb9c2647cb31fc374fd17 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 6 Nov 2026 15:14:38 +0100 Subject: [PATCH 857/881] third_party/zls: allow all 0.17 zig versions (#1036) --- third_party/zls/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/zls/BUILD.bazel b/third_party/zls/BUILD.bazel index d53e20be..69cc6128 100644 --- a/third_party/zls/BUILD.bazel +++ b/third_party/zls/BUILD.bazel @@ -30,7 +30,7 @@ zls_toolchain( toolchain( name = "toolchain", - target_settings = ["@zig_toolchains//:0.17.0"], + target_settings = ["@zig_toolchains//:any_0.17.release"], toolchain = ":zls", toolchain_type = "@rules_zig//zig/zls:toolchain_type", ) From 1042cba3ff15c1a852e3c2cf8f57fce877e8df52 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Mon, 9 Nov 2026 15:26:03 +0100 Subject: [PATCH 858/881] zml/kernels/moe: make mesh and partitioning explicit (#1028) MOE API currently relies on input sharding metadata or partitioning choices made inside backend kernels. Ordinary graph operations can leave that metadata unspecified, making the resulting sharding depend on the backend and preceding operations. Move the manual computation boundary into the shared `zml.kernels.moe.forwardMoe` API using `manualComputation2`. Callers provide a logical mesh and typed input/output partitioning specs. --- examples/llm/models/qwen3_5_moe/model.zig | 2 + zml/kernels/moe/cute/cute.zig | 124 ++--- .../cutlass_flashinfer/cutlass_flashinfer.zig | 101 +--- zml/kernels/moe/fly/fly.zig | 134 ++--- zml/kernels/moe/metal/metal.zig | 94 +--- zml/kernels/moe/moe.zig | 180 ++++++- zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig | 138 +---- zml/kernels/moe/stablehlo/stablehlo.zig | 15 +- zml/kernels/moe/tcl/tcl.zig | 8 +- zml/kernels/moe/tests.zig | 38 +- zml/kernels/moe/triton/triton.zig | 502 +++++++----------- zml/ops.zig | 136 ++++- 12 files changed, 641 insertions(+), 831 deletions(-) diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig index 1375a1bd..2a3b0ba8 100644 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -787,6 +787,8 @@ pub const Moe = struct { self.gate_up_proj, self.down_proj, moe_backend, + zml.Compiler.current().mesh(.experts), + .expert(.experts), .{ .activation = .{ .swiglu = .{} }, .quantize_input = false, diff --git a/zml/kernels/moe/cute/cute.zig b/zml/kernels/moe/cute/cute.zig index c0156720..708ea757 100644 --- a/zml/kernels/moe/cute/cute.zig +++ b/zml/kernels/moe/cute/cute.zig @@ -38,8 +38,7 @@ pub fn validateOptions(opts: zml.moe.Options) void { /// expect block scales in the 128x4 tensor-core layout. pub fn fusedExperts( input: zml.Tensor, - ids: zml.Tensor, - weights: zml.Tensor, + routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, options: zml.moe.Options, @@ -83,24 +82,43 @@ pub fn fusedExperts( else => stdx.debug.panic("cute backend only supports mxfp4 and nvfp4 quantization, got {}", .{gq.scheme}), }; - const context: Context = .{ - .input = input, - .ids = ids, - .weights = weights, - .w1 = gate_up.weight, - .s1 = gq.scales, - .w2 = down.weight, - .s2 = dq.scales, - .topk = ids.dim(.topk), - .swiglu_limit = options.activation.swiglu.limit, - .routing_weight_placement = options.routing_weight_placement, - .format = format, - .expert_parallel = gate_up.weight.shape().partition(.expert).isSharded(), - }; - return if (context.expert_parallel) - zml.ops.manualComputation(Context.body, context, input.shape()) - else - context.body(input.shape()); + const hidden = down.weight.dim(1); + const intermediate = down.weight.dim(2) * 2; + const topk = routing.ids.dim(.topk); + const experts = gate_up.weight.dim(.expert); + const ids = routing.localIds().convert(.i32).reshape(.{ .token = .auto, .topk = topk }); + const tokens = ids.dim(.token); + const routing_weights = routing.weights.convert(.f32).reshape(.{ tokens, topk }); + + return switch (format) { + .mxfp4 => kernels.forwardMxfp4(tokens, hidden, intermediate, experts, topk, options.activation.swiglu.limit, .{ + .x = input.reshape(.{ tokens, hidden }), + .routing_weights = routing_weights, + .w1 = gate_up.weight, + .s1 = gq.scales, + .w2 = down.weight, + .s2 = dq.scales, + .ids = ids, + }, null), + .nvfp4 => |f| kernels.forwardNvfp4(tokens, hidden, intermediate, experts, topk, .{ + .x = input.reshape(.{ tokens, hidden }), + .routing_weights = routing_weights, + .ids = ids, + .w1 = gate_up.weight, + .s1 = gq.scales, + .w2 = down.weight, + .s2 = dq.scales, + // `alpha` is per expert; `act` is one scale shared by every + // expert, which the kernels read per expert. + .alpha1 = f.alpha1.reshape(.{experts}), + .alpha2 = f.alpha2.reshape(.{experts}), + .act1 = f.act1.reshape(.{}).broad(.init(.{experts}, .f32)), + .act2 = f.act2.reshape(.{}).broad(.init(.{experts}, .f32)), + }, .{ + .swiglu_limit = options.activation.swiglu.limit, + .routing_weight_placement = options.routing_weight_placement, + }), + }.reshape(input.shape().dims()).withTags(input.shape()); } /// The format-specific tensors: NVFP4's per-expert global scales. They stay @@ -115,69 +133,3 @@ const Format = union(enum) { act2: zml.Tensor, }, }; - -const Context = struct { - input: zml.Tensor, - ids: zml.Tensor, - weights: zml.Tensor, - w1: zml.Tensor, - s1: zml.Tensor, - w2: zml.Tensor, - s2: zml.Tensor, - topk: i64, - swiglu_limit: ?f32, - routing_weight_placement: zml.moe.RoutingWeightPlacement, - format: Format, - expert_parallel: bool, - - fn body(self: Context, _: zml.Shape) zml.Tensor { - const experts = self.w1.dim(.expert); - const hidden = self.w2.dim(1); - const intermediate = self.w2.dim(2) * 2; - const tokens = self.input.dim(.t); - var ids = self.ids.convert(.i32); - const routing_weights = self.weights.convert(.f32); - if (self.expert_parallel) { - const partition_id = zml.ops.partitionId().convert(.i32); - const expert_start = partition_id.scale(experts); - const expert_end = expert_start.addConstant(experts); - // Routes of other ranks get expert -1 and are not computed here. - const local = ids.cmp(.GE, expert_start).logical(.AND, ids.cmp(.LT, expert_end)); - ids = local.select(ids.sub(expert_start), .scalar(-1, .i32)); - } - const result = switch (self.format) { - .mxfp4 => kernels.forwardMxfp4(tokens, hidden, intermediate, experts, self.topk, self.swiglu_limit, .{ - .x = self.input, - .routing_weights = routing_weights, - .w1 = self.w1, - .s1 = self.s1, - .w2 = self.w2, - .s2 = self.s2, - .ids = ids, - }, null), - .nvfp4 => |f| kernels.forwardNvfp4(tokens, hidden, intermediate, experts, self.topk, .{ - .x = self.input, - .routing_weights = routing_weights, - .ids = ids, - .w1 = self.w1, - .s1 = self.s1, - .w2 = self.w2, - .s2 = self.s2, - // `alpha` is per expert; `act` is one scale shared by every - // expert, which the kernels read per expert. - .alpha1 = f.alpha1.reshape(.{experts}), - .alpha2 = f.alpha2.reshape(.{experts}), - .act1 = f.act1.reshape(.{}).broad(.init(.{experts}, .f32)), - .act2 = f.act2.reshape(.{}).broad(.init(.{experts}, .f32)), - }, .{ - .swiglu_limit = self.swiglu_limit, - .routing_weight_placement = self.routing_weight_placement, - }), - }.withTags(self.input.shape()); - - return if (self.expert_parallel) - zml.ops.allReduce(result, zml.Tensor.add) - else - result; - } -}; diff --git a/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig b/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig index 2c7ad342..e2c31c44 100644 --- a/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig +++ b/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig @@ -20,7 +20,7 @@ pub const Options = struct { /// obtain the first valid GEMM2 index. gemm2_tactic: i32, - pub fn fromGlobalOptions(opts: zml.moe.Options) Options { + pub fn fromOptions(opts: zml.moe.Options) Options { return .{ .workspace_query_device = 0, .activation = opts.activation, @@ -502,8 +502,7 @@ pub fn fusedExpertsNvfp4( pub fn fusedExperts( input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, + routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: zml.moe.Options, @@ -516,8 +515,11 @@ pub fn fusedExperts( @panic("FlashInfer CUTLASS MoE does not support bias in gate_up or down linear layers"); } - const runner_options: Options = .fromGlobalOptions(opts); - const expert_partition = gate_up.weight.shape().partition(.expert); + const topk_ids = routing.localIds(); + // The runner ignores routes whose local expert id is -1. + const topk_weights = topk_ids.cmp(.GE, .scalar(0, .i32)).select(routing.weights, .scalar(-1, routing.weights.dtype())); + + const runner_options: Options = .fromOptions(opts); const quant_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; stdx.debug.assert(quant_scheme == .nvfp4, "FlashInfer CUTLASS MoE only supports NVFP4, got {?}", .{quant_scheme}); @@ -527,95 +529,6 @@ pub fn fusedExperts( // TODO(Corentin): Do error checking on nvfp4 // Also, maybe pass `zml.nn.Linear` directly - if (expert_partition.isSharded()) { - return zml.ops.manualComputation( - (struct { - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - gate_up_weight_unpacked: zml.Tensor, - down_weight_unpacked: zml.Tensor, - gate_up_input_scale: zml.Tensor, - gate_up_scales: zml.Tensor, - gate_up_global_scale: zml.Tensor, - down_input_scale: zml.Tensor, - down_scales: zml.Tensor, - down_global_scale: zml.Tensor, - activation: zml.moe.Activation, - enable_pdl: bool, - gemm1_tactic: i32, - gemm2_tactic: i32, - workspace_query_device: i32, - - fn body( - self: @This(), - _: zml.Shape, - ) zml.Tensor { - const local_num_experts = self.gate_up_weight_unpacked.dim(.expert); - const partition_id = zml.ops.partitionId().convert(.i32); - const expert_start = partition_id.scale(local_num_experts).convert(.i32); - const expert_end = expert_start.addConstant(local_num_experts); - - const local_route_mask = self.topk_ids - .cmp(.GE, expert_start) - .logical(.AND, self.topk_ids.cmp(.LT, expert_end)); - const local_topk_ids = local_route_mask.select( - self.topk_ids.sub(expert_start), - zml.Tensor.scalar(-1, .i32), - ); - const local_topk_weights = local_route_mask.select( - self.topk_weights, - zml.Tensor.scalar(-1, self.topk_weights.dtype()), - ); - - const local_output = fusedExpertsNvfp4( - self.input, - self.gate_up_weight_unpacked, - self.down_weight_unpacked, - local_topk_weights, - local_topk_ids, - self.gate_up_input_scale, - self.gate_up_scales, - self.gate_up_global_scale, - self.down_input_scale, - self.down_scales, - self.down_global_scale, - .{ - .workspace_query_device = self.workspace_query_device, - .activation = self.activation, - .enable_pdl = self.enable_pdl, - .gemm1_tactic = self.gemm1_tactic, - .gemm2_tactic = self.gemm2_tactic, - }, - ); - const local_reshaped = local_output - .reshape(self.input.shape().dims()) - .withTags(.{ .t, .d }); - return zml.ops.allReduce(local_reshaped, zml.Tensor.add); - } - }).body, - .{ - .input = input, - .topk_ids = topk_ids, - .topk_weights = topk_weights, - .gate_up_weight_unpacked = gate_up_weight_unpacked, - .down_weight_unpacked = down_weight_unpacked, - .gate_up_input_scale = gate_up.quantization.?.input_scale.?.asMultiplier(), - .gate_up_scales = gate_up.quantization.?.scales, - .gate_up_global_scale = gate_up.quantization.?.global_scale.?.asMultiplier(), - .down_input_scale = down.quantization.?.input_scale.?.asMultiplier(), - .down_scales = down.quantization.?.scales, - .down_global_scale = down.quantization.?.global_scale.?.asMultiplier(), - .activation = runner_options.activation, - .enable_pdl = runner_options.enable_pdl, - .gemm1_tactic = runner_options.gemm1_tactic, - .gemm2_tactic = runner_options.gemm2_tactic, - .workspace_query_device = runner_options.workspace_query_device, - }, - input.shape(), - ); - } - return fusedExpertsNvfp4( input, gate_up_weight_unpacked, diff --git a/zml/kernels/moe/fly/fly.zig b/zml/kernels/moe/fly/fly.zig index e5c5b6de..59d71dc9 100644 --- a/zml/kernels/moe/fly/fly.zig +++ b/zml/kernels/moe/fly/fly.zig @@ -8,7 +8,6 @@ const Shape = zml.Shape; const DataType = zml.DataType; const fly = @import("moe.zig"); const ProjectionLayout = zml.moe.ProjectionLayout; -const RoutingWeightPlacement = zml.moe.RoutingWeightPlacement; const log = std.log.scoped(.@"zml/moe/fused_experts"); @@ -17,90 +16,19 @@ test { } pub fn fusedExperts( - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - opts: zml.moe.Options, -) zml.Tensor { - const gate_up_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; - const layout: zml.moe.ExpertsLayout = if (gate_up_scheme == .mxfp4) - .{ .gate_up = .interleaved, .packing = .plain } - else - .{ .gate_up = .concatenated, .packing = .plain }; - - const args: FusedExpertsArgs = .{ - .hidden_states = input, - .gate_up = gate_up, - .down = down, - .topk_weights = topk_weights, - .topk_ids = topk_ids, - .activation = opts.activation, - .quantize_input = opts.quantize_input, - .gate_up_layout = layout.gate_up, - .routing_weight_placement = opts.routing_weight_placement, - }; - const expert_partition = gate_up.weight.shape().partition(.expert); - - if (!expert_partition.isSharded()) { - return fusedExpertsImpl(args); - } - - return zml.ops.manualComputation( - (struct { - args: FusedExpertsArgs, - global_num_experts: i64, - - fn call(self: @This(), _: zml.Shape) zml.Tensor { - const local_args = self.args; - const local_num_experts = local_args.gate_up.weight.dim(.expert); - const partition_id = zml.ops.partitionId().convert(.i32); - const expert_start = partition_id.scale(local_num_experts).convert(.i32); - const global_expert_ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); - - // Map global expert ids to local ids, or -1 for experts outside this partition. - const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) - .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); - var mapped_args = local_args; - mapped_args.expert_map = local_expert_mask.select( - global_expert_ids.sub(expert_start), - zml.Tensor.scalar(-1, .i32), - ); - - const local_output = fusedExpertsImpl(mapped_args); - const local_reshaped = local_output.reshape(local_args.hidden_states.shape()).withTags(.{ .t, .d }); - return zml.ops.allReduce(local_reshaped, zml.Tensor.add); - } - }).call, - .{ .args = args, .global_num_experts = gate_up.weight.dim(.expert) }, - input.shape(), - ); -} - -pub const FusedExpertsArgs = struct { hidden_states: Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - topk_weights: Tensor, - topk_ids: Tensor, - activation: zml.moe.Activation, - expert_map: ?Tensor = null, - /// Use FP8 activations for FP8 weights; false keeps BF16 activations. - quantize_input: bool, - gate_up_layout: ProjectionLayout, - routing_weight_placement: RoutingWeightPlacement, -}; - -pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { - const hidden_states = opts.hidden_states; - const topk_weights = opts.topk_weights; - const topk_ids = opts.topk_ids; + routing_: zml.moe.LocalRouting, + gate_up_: zml.nn.Linear, + down_: zml.nn.Linear, + opts: zml.moe.Options, +) Tensor { + const expert_map_ = routing_.expertMap(); + const layout: ProjectionLayout = if (gate_up_.quantizationScheme() == .mxfp4) .interleaved else .concatenated; const num_tokens = hidden_states.dim(.t); var launch_config = launchConfigForTokens(num_tokens); - const gate_up_scheme = opts.gate_up.quantizationScheme(); - const down_scheme = opts.down.quantizationScheme(); + const gate_up_scheme = gate_up_.quantizationScheme(); + const down_scheme = down_.quantizationScheme(); if (gate_up_scheme) |scheme| switch (scheme) { .nvfp4 => stdx.debug.panic("fly backend doesn't support gate_up nvfp4 quantization scheme", .{}), .fp8_block128 => launch_config.block_size_k = 128, @@ -117,10 +45,10 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { }; const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); - const gate_up = opts.gate_up.weight.withTags(.{ .expert, .out, .in }); - const down = opts.down.weight.withTags(.{ .expert, .out, .mid }); - const routing_weights = topk_weights.reshape(.{ .token = num_tokens, .in = topk_weights.dim(.topk) }).withTags(.{ .token, .topk }); - const ids = topk_ids.reshape(.{ .token = num_tokens, .in = topk_ids.dim(.topk) }).withTags(.{ .token, .topk }); + const gate_up = gate_up_.weight.withTags(.{ .expert, .out, .in }); + const down = down_.weight.withTags(.{ .expert, .out, .mid }); + const routing_weights = routing_.weights.reshape(.{ .token = num_tokens, .in = routing_.weights.dim(.topk) }).withTags(.{ .token, .topk }); + const ids = routing_.ids.reshape(.{ .token = num_tokens, .in = routing_.ids.dim(.topk) }).withTags(.{ .token, .topk }); stdx.debug.assert(hidden.dtype() == .bf16, "fly backend expected BF16 hidden states, got {}", .{hidden.dtype()}); if (gate_up_scheme == .mxfp4) { @@ -148,13 +76,13 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "fly backend expected routing ids and weights to match hidden token count, got {}, {} and {}", .{ ids.dim(.token), routing_weights.dim(.token), hidden.dim(.token) }); stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "fly backend expected routing ids and weights to have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), routing_weights.dim(.topk) }); stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "fly backend expected gate/up and down expert counts to match, got {} and {}", .{ gate_up.dim(.expert), down.dim(.expert) }); - if (opts.expert_map) |expert_map| + if (expert_map_) |expert_map| stdx.debug.assert(expert_map.dtype() == .i32 and expert_map.rank() == 1, "fly backend expected a rank-1 i32 expert map, got rank {} and dtype {}", .{ expert_map.rank(), expert_map.dtype() }); - const num_experts = if (opts.expert_map) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); + const num_experts = if (expert_map_) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); const routing = prepareRouting(ids, num_experts, @intCast(launch_config.block_size_m)); - const expert_ids = if (opts.expert_map) |expert_map| + const expert_ids = if (expert_map_) |expert_map| expert_map.gather(.{ .expert = routing.expert_ids }, .{}).withTags(.{.g}) else routing.expert_ids; @@ -164,9 +92,9 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { const gate_up_out = gemm(.{ .input = hidden_quant, .weight = gate_up, - .bias = opts.gate_up.bias, + .bias = gate_up_.bias, .input_scale = input_scale, - .weight_scale = opts.gate_up.quantizationScales(), + .weight_scale = gate_up_.quantizationScales(), .routing = routing, .expert_ids = expert_ids, .launch_config = launch_config, @@ -175,7 +103,7 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), }); - var activated = zml.moe.applyActivation(gate_up_out, opts.activation, opts.gate_up_layout); + var activated = zml.moe.applyActivation(gate_up_out, opts.activation, layout); if (opts.routing_weight_placement == .before_down) { const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); activated = activated.mul(weights.broad(activated.shape())); @@ -185,9 +113,9 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { const down_out = gemm(.{ .input = activated_quant, .weight = down, - .bias = opts.down.bias, + .bias = down_.bias, .input_scale = down_input_scale, - .weight_scale = opts.down.quantizationScales(), + .weight_scale = down_.quantizationScales(), .routing_weights = if (opts.routing_weight_placement == .after_down) routing_weights else null, .routing = routing, .expert_ids = expert_ids, @@ -203,18 +131,18 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { } // TODO(Corentin): unused for now -pub fn supports(opts: FusedExpertsArgs) bool { - if (opts.hidden_states.dtype() != .bf16 or opts.gate_up.quantizationScheme() != .mxfp4 or - opts.down.quantizationScheme() != .mxfp4 or opts.gate_up.bias != null or opts.down.bias != null or +pub fn supports(hidden_states: Tensor, routing: zml.moe.LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: zml.moe.Options) bool { + if (hidden_states.dtype() != .bf16 or gate_up_.quantizationScheme() != .mxfp4 or + down_.quantizationScheme() != .mxfp4 or gate_up_.bias != null or down_.bias != null or opts.routing_weight_placement != .before_down) return false; - const gate_up = opts.gate_up.weight; - const down = opts.down.weight; + const gate_up = gate_up_.weight; + const down = down_.weight; if (gate_up.rank() != 3 or down.rank() != 3) return false; - const gate_up_scale = opts.gate_up.quantizationScales() orelse return false; - const down_scale = opts.down.quantizationScales() orelse return false; - const experts = if (opts.expert_map) |map| map.dim(.expert) else gate_up.dim(0); + const gate_up_scale = gate_up_.quantizationScales() orelse return false; + const down_scale = down_.quantizationScales() orelse return false; + const experts = routing.global_num_experts; if (experts < 1 or experts > 512) return false; - var max_width: i64 = opts.hidden_states.dim(.d); + var max_width: i64 = hidden_states.dim(.d); inline for (.{ .{ gate_up, gate_up_scale }, .{ down, down_scale } }) |projection| { const weight, const scale = projection; const packing_factor: i64 = switch (weight.dtype()) { @@ -233,7 +161,7 @@ pub fn supports(opts: FusedExpertsArgs) bool { else => return false, } } - const routes = opts.hidden_states.dim(.t) * opts.topk_ids.dim(.topk); + const routes = hidden_states.dim(.t) * routing.ids.dim(.topk); return routes > 0 and routes <= @divTrunc(std.math.maxInt(i32), max_width); } diff --git a/zml/kernels/moe/metal/metal.zig b/zml/kernels/moe/metal/metal.zig index e129c76c..9d397a46 100644 --- a/zml/kernels/moe/metal/metal.zig +++ b/zml/kernels/moe/metal/metal.zig @@ -6,15 +6,6 @@ const Tensor = zml.Tensor; const log = std.log.scoped(.@"zml/moe/metal"); -pub const Options = struct { - activation: zml.moe.Activation, - global_num_experts: i64 = -1, - w1_scale: ?Tensor = null, - w2_scale: ?Tensor = null, - w1_global_scale: ?Tensor = null, - w2_global_scale: ?Tensor = null, -}; - fn validateOptions(opts: zml.moe.Options) void { stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); @@ -72,57 +63,30 @@ fn applyDownGlobalScale(output: Tensor, global_scale: ?Tensor, expert_ids: Tenso } pub fn fusedExperts( - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, + hidden_states: zml.Tensor, + routing: zml.moe.LocalRouting, + gate_up_: zml.nn.Linear, + down_: zml.nn.Linear, opts: zml.moe.Options, ) zml.Tensor { validateOptions(opts); - stdx.debug.assert(gate_up.bias == null, "metal backend doesn't support gate_up bias", .{}); - stdx.debug.assert(down.bias == null, "metal backend doesn't support down bias", .{}); + stdx.debug.assert(gate_up_.bias == null, "metal backend doesn't support gate_up bias", .{}); + stdx.debug.assert(down_.bias == null, "metal backend doesn't support down bias", .{}); - const gate_up_scales: ?zml.Tensor = if (gate_up.quantization) |q| q.scales else null; - const gate_up_global_scale: ?zml.Tensor = if (gate_up.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; + const gate_up_scales: ?zml.Tensor = if (gate_up_.quantization) |q| q.scales else null; + const gate_up_global_scale: ?zml.Tensor = if (gate_up_.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; - const down_scales: ?zml.Tensor = if (down.quantization) |q| q.scales else null; - const down_global_scale: ?zml.Tensor = if (down.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; + const down_scales: ?zml.Tensor = if (down_.quantization) |q| q.scales else null; + const down_global_scale: ?zml.Tensor = if (down_.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; - const gate_up_weight_unpacked = zml.moe.unpackedWeight(gate_up); - const down_weight_unpacked = zml.moe.unpackedWeight(down); - - return fusedExpertsImpl( - input, - gate_up_weight_unpacked, - down_weight_unpacked, - topk_weights, - topk_ids, - .{ - .activation = opts.activation, - .global_num_experts = gate_up_weight_unpacked.dim(.expert), - .w1_scale = gate_up_scales, - .w2_scale = down_scales, - .w1_global_scale = gate_up_global_scale, - .w2_global_scale = down_global_scale, - }, - ); -} - -pub fn fusedExpertsImpl( - hidden_states: Tensor, - gate_up: Tensor, - down: Tensor, - topk_weights: Tensor, - topk_ids: Tensor, - opts: Options, -) Tensor { + const gate_up = zml.moe.unpackedWeight(gate_up_); + const down = zml.moe.unpackedWeight(down_); stdx.debug.assert(gate_up.dtype() == down.dtype(), "metal backend expected gate_up and down data type to match, got {} and {}", .{ gate_up.dtype(), down.dtype() }); const mode = quantMode(gate_up.dtype()); switch (mode) { .none => { - stdx.debug.assert(opts.w1_scale == null and opts.w2_scale == null, "metal backend expected gate_up and down scale to be null when the weights are not quantized", .{}); - stdx.debug.assert(opts.w1_global_scale == null and opts.w2_global_scale == null, "metal backend expected gate_up and down global scale to be null when the weights are not quantized", .{}); + stdx.debug.assert(gate_up_scales == null and down_scales == null, "metal backend expected gate_up and down scale to be null when the weights are not quantized", .{}); + stdx.debug.assert(gate_up_global_scale == null and down_global_scale == null, "metal backend expected gate_up and down global scale to be null when the weights are not quantized", .{}); const is_hidden_states_valid_dtype = switch (hidden_states.dtype()) { .bf16, .f16, .f32 => true, else => false, @@ -130,28 +94,28 @@ pub fn fusedExpertsImpl( stdx.debug.assert(is_hidden_states_valid_dtype, "metal backend expected hidden_states data type to be bf16, f16 or f32, got {}", .{hidden_states.dtype()}); }, .fp8 => { - stdx.debug.assert(opts.w1_scale != null and opts.w2_scale != null, "metal backend expected gate_up and down scale to be non null", .{}); - stdx.debug.assert(opts.w1_scale.?.dtype() == .bf16 and opts.w2_scale.?.dtype() == .bf16, "metal backend expected gate_up and down scale data type to be bf16, got {} and {}", .{ opts.w1_scale.?.dtype(), opts.w2_scale.?.dtype() }); - stdx.debug.assert(opts.w1_global_scale == null and opts.w2_global_scale == null, "metal backend expected gate_up and down global scale to be null", .{}); + stdx.debug.assert(gate_up_scales != null and down_scales != null, "metal backend expected gate_up and down scale to be non null", .{}); + stdx.debug.assert(gate_up_scales.?.dtype() == .bf16 and down_scales.?.dtype() == .bf16, "metal backend expected gate_up and down scale data type to be bf16, got {} and {}", .{ gate_up_scales.?.dtype(), down_scales.?.dtype() }); + stdx.debug.assert(gate_up_global_scale == null and down_global_scale == null, "metal backend expected gate_up and down global scale to be null", .{}); stdx.debug.assert(hidden_states.dtype() == .bf16, "metal backend expected hidden_states data type to be bf16, got {}", .{hidden_states.dtype()}); }, .nvfp4 => { - stdx.debug.assert(opts.w1_scale != null and opts.w2_scale != null, "metal backend expected gate_up and down scale to be non null", .{}); - stdx.debug.assert(opts.w1_scale.?.dtype() == .f8e4m3fn and opts.w2_scale.?.dtype() == .f8e4m3fn, "metal backend expected gate_up and down scale data type to be f8e4m3fn, got {} and {}", .{ opts.w1_scale.?.dtype(), opts.w2_scale.?.dtype() }); + stdx.debug.assert(gate_up_scales != null and down_scales != null, "metal backend expected gate_up and down scale to be non null", .{}); + stdx.debug.assert(gate_up_scales.?.dtype() == .f8e4m3fn and down_scales.?.dtype() == .f8e4m3fn, "metal backend expected gate_up and down scale data type to be f8e4m3fn, got {} and {}", .{ gate_up_scales.?.dtype(), down_scales.?.dtype() }); stdx.debug.assert(hidden_states.dtype() == .bf16, "metal backend expected hidden_states data type to be bf16, got {}", .{hidden_states.dtype()}); }, } - const is_topk_weights_valid_dtype = switch (topk_weights.dtype()) { + const is_topk_weights_valid_dtype = switch (routing.weights.dtype()) { .bf16, .f16, .f32 => true, else => false, }; - stdx.debug.assert(is_topk_weights_valid_dtype, "metal backend expected topk_weights data type to be bf16, f16 or f32, got {}", .{topk_weights.dtype()}); - stdx.debug.assert(topk_ids.dtype() == .i32, "metal backend expected topk_ids data type to be i32, got {}", .{topk_ids.dtype()}); + stdx.debug.assert(is_topk_weights_valid_dtype, "metal backend expected topk_weights data type to be bf16, f16 or f32, got {}", .{routing.weights.dtype()}); + stdx.debug.assert(routing.ids.dtype() == .i32, "metal backend expected topk_ids data type to be i32, got {}", .{routing.ids.dtype()}); const d = hidden_states.dim(.d); const num_tokens = hidden_states.dim(.t); - const topk = topk_ids.dim(.topk); + const topk = routing.ids.dim(.topk); const num_routes = num_tokens * topk; const act_dtype: zml.DataType = switch (mode) { @@ -163,29 +127,29 @@ pub fn fusedExpertsImpl( const x_rows = hidden.insertAxes(.d, .{.topk}) .broad(zml.Shape.init(.{ .token = num_tokens, .topk = topk, .d = d }, act_dtype)) .merge(.{ .r = .{ .token, .topk } }); - const expert_ids = topk_ids.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) + const expert_ids = routing.ids.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) .merge(.{ .r = .{ .token, .topk } }).convert(.i32); const gate_up_out = applyGateUpGlobalScale(moeGemm( x_rows, gate_up, - opts.w1_scale, + gate_up_scales, expert_ids, zml.Shape.init(.{ .r = num_routes, .out = gate_up.dim(.dout) }, act_dtype), mode, - ), opts.w1_global_scale, expert_ids); + ), gate_up_global_scale, expert_ids); const activated = zml.moe.applyActivation(gate_up_out, opts.activation, .concatenated).convert(act_dtype); const down_out = applyDownGlobalScale(moeGemm( activated, down, - opts.w2_scale, + down_scales, expert_ids, zml.Shape.init(.{ .r = num_routes, .d = down.dim(.d) }, act_dtype), mode, - ), opts.w2_global_scale, expert_ids); + ), down_global_scale, expert_ids); - const weights = topk_weights.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) + const weights = routing.weights.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) .merge(.{ .r = .{ .token, .topk } }); const weighted = down_out.mul(weights.convert(down_out.dtype()).broad(down_out.shape())); const combined = weighted.splitAxis(.r, .{ .token = num_tokens, .topk = topk }) diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index 25ce26d6..7ad85c20 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -4,14 +4,14 @@ const platforms = @import("platforms"); const zml = @import("../../zml.zig"); const stdx = zml.stdx; -pub const tcl = @import("tcl/tcl.zig"); -pub const stablehlo = @import("stablehlo/stablehlo.zig"); pub const cute = @import("cute/cute.zig"); pub const cutlass_flashinfer = @import("cutlass_flashinfer/cutlass_flashinfer.zig"); +pub const fly = @import("fly/fly.zig"); pub const metal = @import("metal/metal.zig"); pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); +pub const stablehlo = @import("stablehlo/stablehlo.zig"); +pub const tcl = @import("tcl/tcl.zig"); pub const triton = @import("triton/triton.zig"); -pub const fly = @import("fly/fly.zig"); const log = std.log.scoped(.@"zml/moe"); @@ -178,6 +178,38 @@ pub const Backend = enum { } }; +/// Routing for a local token batch and a contiguous range of experts. +/// IDs retain the model's global expert numbering, while the projection weights +/// contain only the local experts. Kernels compute those experts contributions and +/// `forwardMoe` sums them along the expert axis. +pub const LocalRouting = struct { + /// Global expert IDs for local tokens, with shape {.t, .topk} + ids: zml.Tensor, + weights: zml.Tensor, + global_num_experts: u32, + local_num_experts: u32, + /// Scalar i32 containing the first local expert's global ID. + /// Null when all experts are local and global IDs already index the weights. + expert_start: ?zml.Tensor = null, + + /// Maps routing IDs to local weight indices, using -1 for nonlocal experts. + /// For a shard owning experts 4–7, routing IDs [1, 6] become [-1, 2]. + pub fn localIds(self: LocalRouting) zml.Tensor { + const start = self.expert_start orelse return self.ids; + const local = self.ids.cmp(.GE, start).logical(.AND, self.ids.cmp(.LT, start.addConstant(self.local_num_experts))); + return local.select(self.ids.sub(start), .scalar(-1, .i32)); + } + + /// Returns a lookup table from every global expert ID to its local weight + /// index or -1 if nonlocal. Returns null when every expert is local. + pub fn expertMap(self: LocalRouting) ?zml.Tensor { + const start = self.expert_start orelse return null; + const ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); + const local = ids.cmp(.GE, start).logical(.AND, ids.cmp(.LT, start.addConstant(self.local_num_experts))); + return local.select(ids.sub(start), .scalar(-1, .i32)); + } +}; + pub const Options = struct { activation: Activation, /// Quantize activations for Triton FP8 GEMMs; false keeps BF16 activations. @@ -196,6 +228,35 @@ pub fn validateRoutingInputs(input: zml.Tensor, ids: zml.Tensor, weights: zml.Te weights.shape().hasTag(.t) == 0 and weights.shape().hasTag(.topk) == 1, "MoE expects ordered tags {{.t, .d}} for input and {{.t, .topk}} for routing", .{}); } +pub const Partitioning = struct { + const Logical = zml.Sharding.Partitioning.Logical; + + in_specs: struct { + input: Logical, + ids: Logical, + weights: Logical, + gate_up: Logical, + down: Logical, + }, + out_specs: Logical, + + /// Shards the `.expert` dimension of both projections over `mesh_axis`. + /// Other dimensions, input tokens, routing IDs/weights and output are replicated. + pub fn expert(comptime mesh_axis: @EnumLiteral()) Partitioning { + const experts: Logical = .parse(.{ .expert = mesh_axis }); + return .{ + .in_specs = .{ + .input = .replicated, + .ids = .replicated, + .weights = .replicated, + .gate_up = experts, + .down = experts, + }, + .out_specs = .replicated, + }; + } +}; + /// Input has shape {.t, .d}; routing IDs and weights have shape {.t, .topk}. /// Returns {.t, .d}. Flatten batched input with merge(.{ .t = .{ .b, .s } }). pub fn forwardMoe( @@ -205,19 +266,114 @@ pub fn forwardMoe( gate_up: zml.nn.Linear, down: zml.nn.Linear, backend: Backend, + mesh: *const zml.Mesh, + partitioning: Partitioning, opts: Options, ) zml.Tensor { validateRoutingInputs(input, topk_ids, topk_weights); - return switch (backend) { - .stablehlo => stablehlo.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), - .cute => cute.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), - .flashinfer_cutlass => cutlass_flashinfer.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), - .triton => triton.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), - .fly => fly.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), - .mosaic_tpu => mosaic_tpu.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), - .metal => metal.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), - .tcl => tcl.fusedExperts(input, topk_ids, topk_weights, gate_up, down, opts), + + const shapes = .{ + .input = input.shape().withPartitioning(mesh, partitioning.in_specs.input), + .ids = topk_ids.shape().withPartitioning(mesh, partitioning.in_specs.ids), + .weights = topk_weights.shape().withPartitioning(mesh, partitioning.in_specs.weights), + .gate_up = gate_up.weight.shape().withPartitioning(mesh, partitioning.in_specs.gate_up), + .down = down.weight.shape().withPartitioning(mesh, partitioning.in_specs.down), }; + const output_shape = input.shape().withPartitioning(mesh, partitioning.out_specs); + + const tokens = shapes.input.partition(.t); + const experts = shapes.gate_up.partition(.expert); + // Local kernels require complete feature vectors and top-k routing lists. + stdx.debug.assert(shapes.input.partition(.d) == .replicated and shapes.ids.partition(.topk) == .replicated and shapes.weights.partition(.topk) == .replicated, "MoE feature and top-k dimensions must be replicated", .{}); + // The expert-axis reduction must add contributions for the same token rows. + // Sharding tokens along that axis would mix contributions for different tokens. + stdx.debug.assert(shapes.ids.partition(.t) == tokens and shapes.weights.partition(.t) == tokens and output_shape.partition(.t) == tokens and output_shape.partition(.d) == .replicated, "MoE input, routing and output must have matching token partitioning", .{}); + stdx.debug.assert(shapes.down.partition(.expert) == experts and (!experts.isSharded() or experts != tokens), "MoE projections must agree on expert partitioning; tokens must be replicated along the expert axis", .{}); + for ([_]zml.Shape{ shapes.gate_up, shapes.down }) |shape| { + stdx.debug.assert(!shape.containsPartitionSpec(.open), "MoE kernel weight partitioning must be closed", .{}); + for (0..shape.rank()) |axis| { + stdx.debug.assert(axis == shape.axis(.expert) or shape.partition(axis) == .replicated, "MoE kernel weights can only be partitioned along experts", .{}); + } + } + const expert_axis = experts.meshAxis(); + const parallel = if (expert_axis) |axis| mesh.grid.axis_sizes[axis] > 1 else false; + if (backend == .metal or backend == .tcl) stdx.debug.assert(!parallel, "This MoE backend does not support expert parallelism", .{}); + + const Call = struct { + input: zml.Tensor, + ids: zml.Tensor, + weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + backend: Backend, + opts: Options, + mesh: *const zml.Mesh, + expert_axis: ?u3, + global_experts: u32, + + fn forward(args: @This(), _: zml.Shape) zml.Tensor { + const routing: LocalRouting = .{ + .ids = args.ids, + .weights = args.weights, + .global_num_experts = args.global_experts, + .local_num_experts = @intCast(args.gate_up.weight.dim(.expert)), + .expert_start = if (args.expert_axis) |axis| zml.ops.axisIndex(args.mesh, args.mesh.logical.axes.get(axis)).convert(.i32).scale(args.gate_up.weight.dim(.expert)) else null, + }; + const output = switch (args.backend) { + .stablehlo => stablehlo.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .cute => cute.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .flashinfer_cutlass => cutlass_flashinfer.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .triton => triton.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .fly => fly.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .mosaic_tpu => mosaic_tpu.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .metal => metal.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .tcl => tcl.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + }; + return if (args.expert_axis) |axis| zml.ops.allReduceOnAxis(output, zml.Tensor.add, args.mesh, args.mesh.logical.axes.get(axis)) else output; + } + }; + const args: Call = .{ + .input = input, + .ids = topk_ids, + .weights = topk_weights, + .gate_up = gate_up, + .down = down, + .backend = backend, + .opts = opts, + .mesh = mesh, + .expert_axis = if (parallel) expert_axis else null, + .global_experts = @intCast(gate_up.weight.dim(.expert)), + }; + const Logical = zml.Sharding.Partitioning.Logical; + var projection_specs: [2]zml.meta.MapRestrict(zml.Tensor, Logical).map(zml.nn.Linear) = undefined; + for ([_]zml.nn.Linear{ gate_up, down }, &projection_specs) |linear, *specs| { + zml.meta.mapAlloc(struct { + fn spec(ctx: struct { experts: i64, axis: ?zml.Shape.Tag }, tensor: zml.Tensor) Logical { + const axis = ctx.axis orelse return .replicated; + if (tensor.count() == 1) return .replicated; + // Packed scales may be untagged; experts are their leading dimension. + const dim = tensor.shape().hasTag(.expert) orelse 0; + stdx.debug.assert(tensor.dim(dim) == ctx.experts, "MoE tensor must contain {} experts, got {f}", .{ ctx.experts, tensor.shape() }); + return .{ .mapping = .{ + .src = .init(&.{if (tensor.shape().hasTag(.expert) != null) zml.Shape.toTag(.expert) else zml.Shape.toTag(._0)}), + .tgt = .init(&.{axis}), + } }; + } + }.spec, zml.Compiler.current().arena.allocator(), .{ + .experts = linear.weight.dim(.expert), + .axis = if (expert_axis) |axis| mesh.logical.axes.get(axis) else null, + }, linear, specs) catch @panic("OOM"); + } + return zml.ops.manualComputation2(Call.forward, args, output_shape, mesh, .{ + .in_specs = .{ + .input = partitioning.in_specs.input, + .ids = partitioning.in_specs.ids, + .weights = partitioning.in_specs.weights, + .gate_up = projection_specs[0], + .down = projection_specs[1], + }, + .out_specs = partitioning.out_specs, + }); } pub fn unpackedWeight(linear: zml.nn.Linear) zml.Tensor { diff --git a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig index 09dc360a..c583b3e4 100644 --- a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig @@ -12,22 +12,9 @@ const log = std.log.scoped(.@"zml/moe/mosaic_tpu"); var gate_up_transpose: std.atomic.Value(bool) = .init(false); var down_transpose: std.atomic.Value(bool) = .init(false); -pub const Options = struct { - activation: zml.moe.Activation, - global_num_experts: i64 = -1, - expert_map: ?Tensor = null, - w1_scale: ?Tensor = null, - w2_scale: ?Tensor = null, - w1_bias: ?Tensor = null, - w2_bias: ?Tensor = null, -}; - fn validateOptions(opts: zml.moe.Options) void { stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); - // TODO(Corentin): find where to put that - //if (opts.expert_map != null and opts.global_num_experts == -1) return error.InvalidShape; - //if (opts.w1_scale != null or opts.w2_scale != null) return error.UnsupportedQuantization; } fn validateInputs(hidden: Tensor, gate_up: Tensor, down: Tensor, weights: Tensor, ids: Tensor) !void { @@ -224,120 +211,25 @@ pub fn callGmmEp( } pub fn fusedExperts( - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, + hidden_states: zml.Tensor, + routing: zml.moe.LocalRouting, + gate_up_: zml.nn.Linear, + down_: zml.nn.Linear, opts: zml.moe.Options, ) zml.Tensor { validateOptions(opts); + const expert_map_ = routing.expertMap(); - const gate_up_scales: ?zml.Tensor = if (gate_up.quantization) |q| q.scales else null; - const down_scales: ?zml.Tensor = if (down.quantization) |q| q.scales else null; - - const expert_partition = gate_up.weight.shape().partition(.expert); - - if (expert_partition.isSharded()) { - const global_num_experts = down.weight.dim(.expert); - const partial_output = zml.ops.manualComputation( - (struct { - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - weights_gate_up: zml.Tensor, - weights_down: zml.Tensor, - activation: zml.moe.Activation, - global_num_experts: i64, - gate_up_scales: ?zml.Tensor, - bias_gate_up: ?zml.Tensor, - down_scales: ?zml.Tensor, - bias_down: ?zml.Tensor, - - fn body(self: @This(), _: zml.Shape) zml.Tensor { - const local_num_experts = self.weights_gate_up.dim(.expert); - const partition_id = zml.ops.partitionId().convert(.i32); - const expert_start = partition_id.scale(local_num_experts).convert(.i32); - const global_expert_ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); - - const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) - .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); - const expert_map = local_expert_mask.select( - global_expert_ids.sub(expert_start), - zml.Tensor.scalar(-1, .i32), - ); - const local_output = fusedExpertsImpl( - self.input, - self.weights_gate_up, - self.weights_down, - self.topk_weights, - self.topk_ids, - .{ - .activation = self.activation, - .global_num_experts = self.global_num_experts, - .expert_map = expert_map, - .w1_scale = self.gate_up_scales, - .w2_scale = self.down_scales, - .w1_bias = self.bias_gate_up, - .w2_bias = self.bias_down, - }, - ); - return local_output.reshape(self.input.shape()).withTags(.{ .t, .d }); - } - }).body, - .{ - .input = input, - .topk_ids = topk_ids, - .topk_weights = topk_weights, - .weights_gate_up = gate_up.weight, - .weights_down = down.weight, - .activation = opts.activation, - .global_num_experts = global_num_experts, - .gate_up_scales = gate_up_scales, - .bias_gate_up = gate_up.bias, - .down_scales = down_scales, - .bias_down = down.bias, - }, - input.shape(), - ); - return zml.ops.allReduce(partial_output, zml.Tensor.add); - } - - return fusedExpertsImpl( - input, - gate_up.weight, - down.weight, - topk_weights, - topk_ids, - .{ - .activation = opts.activation, - .global_num_experts = gate_up.weight.dim(.expert), - .w1_scale = gate_up_scales, - .w2_scale = down_scales, - .w1_bias = gate_up.bias, - .w2_bias = down.bias, - }, - ); -} - -pub fn fusedExpertsImpl( - hidden_states: Tensor, - w1: Tensor, - w2: Tensor, - topk_weights: Tensor, - topk_ids: Tensor, - opts: Options, -) Tensor { const tokens = hidden_states.dim(.t); const hidden = hidden_states.reshape(.{ .token = tokens, .in = hidden_states.dim(.d) }); // TODO(Corentin): Better error message - const gate_up = canonicalizeGateUp(w1, hidden.dim(.in)) catch |e| stdx.debug.panic("Invalid gate_up weights for fusedExpertsImpl: {}", .{e}); - const down = canonicalizeDown(w2, hidden.dim(.in), gate_up.dim(.out)) catch |e| stdx.debug.panic("Invalid down weights for fusedExpertsImpl: {}", .{e}); - const weights = topk_weights.reshape(.{ .token = tokens, .topk = topk_weights.dim(.topk) }); - const ids = topk_ids.reshape(.{ .token = tokens, .topk = topk_ids.dim(.topk) }); + const gate_up = canonicalizeGateUp(gate_up_.weight, hidden.dim(.in)) catch |e| stdx.debug.panic("Invalid gate_up weights for fusedExperts: {}", .{e}); + const down = canonicalizeDown(down_.weight, hidden.dim(.in), gate_up.dim(.out)) catch |e| stdx.debug.panic("Invalid down weights for fusedExperts: {}", .{e}); + const weights = routing.weights.reshape(.{ .token = tokens, .topk = routing.weights.dim(.topk) }); + const ids = routing.ids.reshape(.{ .token = tokens, .topk = routing.ids.dim(.topk) }); - const num_experts = if (opts.global_num_experts != -1) opts.global_num_experts else gate_up.dim(.expert); - if (opts.expert_map) |expert_map| { + const num_experts = routing.global_num_experts; + if (expert_map_) |expert_map| { if (expert_map.dtype() != .i32) stdx.debug.panic("expert_map must be of dtype i32, got {}", .{expert_map.dtype()}); if (expert_map.rank() != 1 or expert_map.dim(.expert) != num_experts) stdx.debug.panic("expert_map must be a 1D tensor of length num_experts, got shape {}", .{expert_map.shape()}); } @@ -345,7 +237,7 @@ pub fn fusedExpertsImpl( const flat_ids_global = ids.transpose(.{ .topk, .token }).flatten().withTags(.{.route}); const flat_weights = weights.transpose(.{ .topk, .token }).flatten().withTags(.{.route}); - const flat_ids_local, const local_route_mask = if (opts.expert_map) |expert_map| blk: { + const flat_ids_local, const local_route_mask = if (expert_map_) |expert_map| blk: { const mapped = expert_map.gather(.{ .expert = flat_ids_global }, .{}).withTags(.{.route}); const route_mask = mapped.cmp(.GE, Tensor.scalar(0, .i32).broad(mapped.shape())); const safe_ids = route_mask.select(mapped, Tensor.zeroes(mapped.shape())); @@ -356,7 +248,7 @@ pub fn fusedExpertsImpl( }; const sorted_route = flat_ids_local.argsort(.route, .{ .descending = false }).rename(.{ .route = .perm }); - const hidden_sorted = if (opts.expert_map != null) + const hidden_sorted = if (expert_map_ != null) permuteSmallBatch(hidden, sorted_route, ids.dim(.topk)) else blk: { // If not sharded, we can save the sort + gather by just repeating the hidden states for each expert and gathering with the sorted route @@ -379,7 +271,7 @@ pub fn fusedExpertsImpl( .gather(.{ .token = aligned_hidden.positions.rename(.{ .token = .route }) }, .{}) .rename(.{ .route = .token }); - if (opts.w1_bias) |bias| { + if (gate_up_.bias) |bias| { const bias_per_token = bias.withTags(.{ .expert, .out }).gather(.{ .expert = expert_ids_sorted }, .{}); gate_up_out = gate_up_out.add(bias_per_token); } @@ -391,7 +283,7 @@ pub fn fusedExpertsImpl( .gather(.{ .token = aligned_activated.positions.rename(.{ .token = .route }) }, .{}) .rename(.{ .route = .token }); - if (opts.w2_bias) |bias| { + if (down_.bias) |bias| { const bias_per_token = bias.withTags(.{ .expert, .out }).gather(.{ .expert = expert_ids_sorted }, .{}); down_out = down_out.add(bias_per_token); } diff --git a/zml/kernels/moe/stablehlo/stablehlo.zig b/zml/kernels/moe/stablehlo/stablehlo.zig index e33d2a14..25c69c5c 100644 --- a/zml/kernels/moe/stablehlo/stablehlo.zig +++ b/zml/kernels/moe/stablehlo/stablehlo.zig @@ -157,14 +157,19 @@ pub const Options = struct { routing_weight_placement: zml.moe.RoutingWeightPlacement = .after_down, }; +/// Negative expert IDs denote nonlocal routes and contribute zero, including bias. pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: Options) Tensor { + stdx.debug.assert(topk_ids.dim(.topk) > 0, "reference top-k must be positive", .{}); stdx.debug.assert(topk_ids.dtype() == .i32, "reference expects i32 expert ids", .{}); stdx.debug.assert(gate_up.weight.rank() == 3 and down.weight.rank() == 3, "reference expects rank-three expert weights", .{}); stdx.debug.assert(gate_up.weight.dim(.expert) == down.weight.dim(.expert), "reference expert counts must match", .{}); const tokens = input.dim(.t); const k = topk_ids.dim(.topk); - stdx.debug.assert(k > 0 and k <= gate_up.weight.dim(.expert), "reference top-k must fit the expert count", .{}); - const ids = topk_ids.reshape(.{ .route = tokens * k }); + // Nonlocal routes use expert zero for safe gathers; discard their full + // contribution after the down projection, including any projection bias. + const local_routes = topk_ids.cmp(.GE, Tensor.scalar(0, .i32)); + const safe_ids = topk_ids.maximum(Tensor.scalar(0, .i32)); + const ids = safe_ids.reshape(.{ .route = tokens * k }); const weights = topk_weights.reshape(.{ .route = tokens * k }).convert(.f32); const hidden = input.reshape(.{ .token = tokens, .in = input.dim(.d) }) .insertAxes(.in, .{.topk}).broad(Shape.init(.{ .token = tokens, .topk = k, .in = input.dim(.d) }, input.dtype())) @@ -176,13 +181,15 @@ pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: if (opts.routing_weight_placement == .before_down) activated = activated.mul(weights.broad(activated.shape())); var output = project(activated.rename(.{ .out = .in }), down, ids, opts, input.dtype()); if (opts.routing_weight_placement == .after_down) output = output.mul(weights.broad(output.shape())); + output = local_routes.reshape(.{ .route = tokens * k }).broad(output.shape()).select(output, Tensor.scalar(0, output.dtype())); return output.convert(input.dtype()).convert(.f32) .reshape(.{ .t = tokens, .topk = k, .d = output.dim(.out) }) .sum(.topk).squeeze(.topk).convert(input.dtype()); } -pub fn fusedExperts(input: Tensor, ids: Tensor, weights: Tensor, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: zml.moe.Options) Tensor { - return reference(input, ids, weights, gate_up, down, .{ +pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: zml.moe.Options) Tensor { + stdx.debug.assert(routing.ids.dim(.topk) > 0 and routing.ids.dim(.topk) <= routing.global_num_experts, "MoE top-k must fit the global expert count", .{}); + return reference(input, routing.localIds(), routing.weights, gate_up, down, .{ .activation = opts.activation, .quantize_input = opts.quantize_input, .routing_weight_placement = opts.routing_weight_placement, diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig index 4b7bcfb3..991280fd 100644 --- a/zml/kernels/moe/tcl/tcl.zig +++ b/zml/kernels/moe/tcl/tcl.zig @@ -19,10 +19,12 @@ const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; -/// `input` is `{.t, .d}`, `topk_ids` and `topk_weights` `{.t, .topk}`, `gate_up` `{.expert, .dout = 2 * inter (gate then up), .d}` +/// `input` is `{.t, .d}`, routing IDs and weights `{.t, .topk}`, `gate_up` `{.expert, .dout = 2 * inter (gate then up), .d}` /// and `down` `{.expert, .d, .dout = inter}`, all bf16 but the ids (i32). /// SwiGLU, routing weights applied after the down projection. -pub fn fusedExperts(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: zml.moe.Options) Tensor { +pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: zml.moe.Options) Tensor { + const topk_ids = routing.ids; + const topk_weights = routing.weights; stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE applies routing weights after the down projection, without FP8 input quantization", .{}); stdx.debug.assert(opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}), "TCL MoE supports SwiGLU without limit, scale or bias", .{}); stdx.debug.assert(gate_up_linear.quantization == null and down_linear.quantization == null, "TCL MoE takes bf16 experts", .{}); @@ -277,7 +279,7 @@ test "tcl fused experts run on furiosa" { const dn: Tensor = .init(.{ .expert = e, .d = d, .dout = i }, .bf16); const Mod = struct { pub fn forward(x_: Tensor, ids_: Tensor, w_: Tensor, gu_: Tensor, dn_: Tensor) Tensor { - return fusedExperts(x_, ids_, w_, .init(gu_, null, .dout), .init(dn_, null, .dout), .{ + return zml.moe.forwardMoe(x_, ids_, w_, .init(gu_, null, .dout), .init(dn_, null, .dout), .tcl, zml.Compiler.current().platform.replicated_mesh, .expert(.replicated), .{ .activation = .{ .swiglu = .{} }, .quantize_input = false, .routing_weight_placement = .after_down, diff --git a/zml/kernels/moe/tests.zig b/zml/kernels/moe/tests.zig index 2bdebdfe..aa0be86f 100644 --- a/zml/kernels/moe/tests.zig +++ b/zml/kernels/moe/tests.zig @@ -5,8 +5,8 @@ const std = @import("std"); const zml = @import("../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; -const reference = @import("stablehlo/stablehlo.zig"); const Placement = @import("moe.zig").RoutingWeightPlacement; +const reference = @import("stablehlo/stablehlo.zig"); const Case = struct { backend: zml.moe.Backend, @@ -22,15 +22,22 @@ const Case = struct { width: i64 = 128, intermediate: i64 = 128, experts: i64 = 8, + expert_mesh: *const zml.Mesh, + + fn inputShape(c: Case) Shape { + const shape = Shape.init(.{ .t = c.tokens, .d = c.width }, c.activation_dtype); + const spec: zml.Sharding.Partitioning.Logical = if (c.expert_mesh.resolveLogicalAxis("data") != null) .parse(.{ .t = .data }) else .replicated; + return shape.withPartitioning(c.expert_mesh, spec); + } pub fn compile(c: Case, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Exe { std.debug.assert(c.backend.isAvailable(platform)); - const x: Tensor = .init(.{ .t = c.tokens, .d = c.width }, c.activation_dtype); - return platform.compileFn(allocator, io, forward, .{ x, c }, .{}); + const x: Tensor = .fromShape(c.inputShape()); + return platform.compileFn(allocator, io, forward, .{ x, c }, .{ .meshes = &.{c.expert_mesh} }); } pub fn check(c: Case, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, exe: *const zml.Exe) !void { - const x: Tensor = .init(.{ .t = c.tokens, .d = c.width }, c.activation_dtype); + const x: Tensor = .fromShape(c.inputShape()); var host = try zml.Slice.alloc(allocator, x.shape()); defer host.free(allocator); @@ -52,6 +59,7 @@ const Case = struct { var output: zml.Bufferized(Outputs) = undefined; runner.run(io, .{input}, .{&output}, .{ .wait = true }); defer zml.Buffer.deinitAll(Outputs, &output); + try std.testing.expectEqualDeep(c.inputShape()._sharding, output.actual.shape()._sharding); // BF16 intermediate rounding and different GEMM reduction orders. Every // element must pass; near-zero values use the absolute bound. @@ -269,8 +277,20 @@ fn forward(input: Tensor, c: Case) Outputs { .routing_weight_placement = c.placement, }; - const actual = zml.moe.forwardMoe(input, ids, weights, prepared.gate_up, prepared.down, c.backend, options); - const expected = reference.reference(input, ids, weights, canonical.gate_up, canonical.down, .{ + const data_spec: zml.Sharding.Partitioning.Logical = if (c.expert_mesh.resolveLogicalAxis("data") != null) .parse(.{ .t = .data }) else .replicated; + const expert_spec: zml.Sharding.Partitioning.Logical = if (c.expert_mesh.resolveLogicalAxis("experts") != null) .parse(.{ .expert = .experts }) else .replicated; + const actual = zml.moe.forwardMoe(input, ids, weights, prepared.gate_up, prepared.down, c.backend, c.expert_mesh, .{ + .in_specs = .{ + .input = data_spec, + .ids = data_spec, + .weights = data_spec, + .gate_up = expert_spec, + .down = expert_spec, + }, + .out_specs = data_spec, + }, options); + // Keep the oracle input replicated so token sharding does not change its GEMM rounding. + const expected = reference.reference(input.replicate(), ids, weights, canonical.gate_up, canonical.down, .{ .activation = c.activation, .quantize_input = c.quantize_input, .routing_weight_placement = c.placement, @@ -346,7 +366,7 @@ const Matrix = struct { } /// Shapes vary fastest, matching the nested loops in the backend tests. - pub fn caseAt(self: Matrix, backend: zml.moe.Backend, index: usize) Case { + pub fn caseAt(self: Matrix, backend: zml.moe.Backend, expert_mesh: *const zml.Mesh, index: usize) Case { std.debug.assert(index < self.count()); var remaining = index; const shape = self.shapes[remaining % self.shapes.len]; @@ -374,6 +394,7 @@ const Matrix = struct { .width = shape.width, .intermediate = shape.intermediate, .experts = shape.experts, + .expert_mesh = expert_mesh, }; } }; @@ -414,7 +435,7 @@ const Pipeline = struct { var index: usize = 0; for (0..matrix.count()) |local_index| { - const c = matrix.caseAt(backend, local_index); + const c = matrix.caseAt(backend, self.platform.replicated_mesh, local_index); if (available) { case_queue.putOne(io, .{ .index = index, .case = c }) catch |err| return switch (err) { error.Closed => {}, @@ -591,6 +612,7 @@ test "Triton MoE prefill launch buckets fit shared memory" { for ([_]i64{ 8, 32, 96, 128, 256, 512, 1024, 2048, 3072, 4096 }) |tokens| { const case: Case = .{ .backend = .triton, + .expert_mesh = platform.replicated_mesh, .tokens = tokens, .width = 256, .intermediate = 128, diff --git a/zml/kernels/moe/triton/triton.zig b/zml/kernels/moe/triton/triton.zig index 9d3a5529..348f5a90 100644 --- a/zml/kernels/moe/triton/triton.zig +++ b/zml/kernels/moe/triton/triton.zig @@ -35,8 +35,7 @@ test { pub fn fusedExperts( input: zml.Tensor, - ids: zml.Tensor, - weights: zml.Tensor, + routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, options: zml.moe.Options, @@ -45,99 +44,29 @@ pub fn fusedExperts( if (gate_up.quantization) |q| switch (q.scheme) { .mxfp4 => if (zml.Compiler.current().platform.target == .cuda) { - return fusedExpertsMxfp4(input, ids, weights, gate_up, down, options); + return fusedExpertsMxfp4(input, routing, gate_up, down, options); }, else => {}, }; - return fusedExpertsBase(input, ids, weights, gate_up, down, options); + const layout: ProjectionLayout = if (gate_up.quantizationScheme() == .mxfp4) .interleaved else .concatenated; + return fusedExpertsBase(input, routing, gate_up, down, layout, options); } pub fn fusedExpertsBase( - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - opts: zml.moe.Options, -) zml.Tensor { - const gate_up_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; - const layout: zml.moe.ExpertsLayout = if (gate_up_scheme == .mxfp4) - .{ .gate_up = .interleaved, .packing = .plain } - else - .{ .gate_up = .concatenated, .packing = .plain }; - - const args: FusedExpertsArgs = .{ - .hidden_states = input, - .gate_up = gate_up, - .down = down, - .topk_weights = topk_weights, - .topk_ids = topk_ids, - .activation = opts.activation, - .quantize_input = opts.quantize_input, - .gate_up_layout = layout.gate_up, - .routing_weight_placement = opts.routing_weight_placement, - }; - const expert_partition = gate_up.weight.shape().partition(.expert); - - if (!expert_partition.isSharded()) { - return fusedExpertsImpl(args); - } - - return zml.ops.manualComputation( - (struct { - args: FusedExpertsArgs, - global_num_experts: i64, - - fn call(self: @This(), _: zml.Shape) zml.Tensor { - const local_args = self.args; - const local_num_experts = local_args.gate_up.weight.dim(.expert); - const partition_id = zml.ops.partitionId().convert(.i32); - const expert_start = partition_id.scale(local_num_experts).convert(.i32); - const global_expert_ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); - - // Map global expert ids to local ids, or -1 for experts outside this partition. - const local_expert_mask = global_expert_ids.cmp(.GE, expert_start) - .logical(.AND, global_expert_ids.cmp(.LT, expert_start.addConstant(local_num_experts))); - var mapped_args = local_args; - mapped_args.expert_map = local_expert_mask.select( - global_expert_ids.sub(expert_start), - zml.Tensor.scalar(-1, .i32), - ); - - const local_output = fusedExpertsImpl(mapped_args); - const local_reshaped = local_output.reshape(local_args.hidden_states.shape()).withTags(.{ .t, .d }); - return zml.ops.allReduce(local_reshaped, zml.Tensor.add); - } - }).call, - .{ .args = args, .global_num_experts = gate_up.weight.dim(.expert) }, - input.shape(), - ); -} - -pub const FusedExpertsArgs = struct { hidden_states: Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - topk_weights: Tensor, - topk_ids: Tensor, - activation: zml.moe.Activation, - expert_map: ?Tensor = null, - /// Use FP8 activations for FP8 weights; false keeps BF16 activations. - quantize_input: bool, - gate_up_layout: ProjectionLayout, - routing_weight_placement: RoutingWeightPlacement, -}; - -pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { - const hidden_states = opts.hidden_states; - const topk_weights = opts.topk_weights; - const topk_ids = opts.topk_ids; + routing_: zml.moe.LocalRouting, + gate_up_: zml.nn.Linear, + down_: zml.nn.Linear, + layout: ProjectionLayout, + opts: zml.moe.Options, +) Tensor { + const expert_map_ = routing_.expertMap(); const num_tokens = hidden_states.dim(.t); var launch_config = launchConfigForTokens(num_tokens); - const gate_up_scheme = opts.gate_up.quantizationScheme(); - const down_scheme = opts.down.quantizationScheme(); + const gate_up_scheme = gate_up_.quantizationScheme(); + const down_scheme = down_.quantizationScheme(); if (gate_up_scheme) |scheme| switch (scheme) { .nvfp4 => stdx.debug.panic("triton backend doesn't support gate_up nvfp4 quantization scheme", .{}), .fp8_block128 => launch_config.block_size_k = 128, @@ -154,10 +83,10 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { }; const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); - const gate_up = opts.gate_up.weight.withTags(.{ .expert, .out, .in }); - const down = opts.down.weight.withTags(.{ .expert, .out, .mid }); - const routing_weights = topk_weights.reshape(.{ .token = num_tokens, .in = topk_weights.dim(.topk) }).withTags(.{ .token, .topk }); - const ids = topk_ids.reshape(.{ .token = num_tokens, .in = topk_ids.dim(.topk) }).withTags(.{ .token, .topk }); + const gate_up = gate_up_.weight.withTags(.{ .expert, .out, .in }); + const down = down_.weight.withTags(.{ .expert, .out, .mid }); + const routing_weights = routing_.weights.reshape(.{ .token = num_tokens, .in = routing_.weights.dim(.topk) }).withTags(.{ .token, .topk }); + const ids = routing_.ids.reshape(.{ .token = num_tokens, .in = routing_.ids.dim(.topk) }).withTags(.{ .token, .topk }); stdx.debug.assert(hidden.dtype() == .bf16, "triton backend expected BF16 hidden states, got {}", .{hidden.dtype()}); if (gate_up_scheme == .mxfp4) { @@ -185,13 +114,13 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "triton backend expected routing ids and weights to match hidden token count, got {}, {} and {}", .{ ids.dim(.token), routing_weights.dim(.token), hidden.dim(.token) }); stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "triton backend expected routing ids and weights to have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), routing_weights.dim(.topk) }); stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "triton backend expected gate/up and down expert counts to match, got {} and {}", .{ gate_up.dim(.expert), down.dim(.expert) }); - if (opts.expert_map) |expert_map| + if (expert_map_) |expert_map| stdx.debug.assert(expert_map.dtype() == .i32 and expert_map.rank() == 1, "triton backend expected a rank-1 i32 expert map, got rank {} and dtype {}", .{ expert_map.rank(), expert_map.dtype() }); - const num_experts = if (opts.expert_map) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); + const num_experts = if (expert_map_) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); const routing = prepareRouting(ids, num_experts, @intCast(launch_config.block_size_m)); - const expert_ids = if (opts.expert_map) |expert_map| + const expert_ids = if (expert_map_) |expert_map| expert_map.gather(.{ .expert = routing.expert_ids }, .{}).withTags(.{.g}) else routing.expert_ids; @@ -201,9 +130,9 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { const gate_up_out = gemm(.{ .input = hidden_quant, .weight = gate_up, - .bias = opts.gate_up.bias, + .bias = gate_up_.bias, .input_scale = input_scale, - .weight_scale = opts.gate_up.quantizationScales(), + .weight_scale = gate_up_.quantizationScales(), .routing = routing, .expert_ids = expert_ids, .launch_config = launch_config, @@ -212,7 +141,7 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), }); - var activated = zml.moe.applyActivation(gate_up_out, opts.activation, opts.gate_up_layout); + var activated = zml.moe.applyActivation(gate_up_out, opts.activation, layout); if (opts.routing_weight_placement == .before_down) { const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); activated = activated.mul(weights.broad(activated.shape())); @@ -222,9 +151,9 @@ pub fn fusedExpertsImpl(opts: FusedExpertsArgs) Tensor { const down_out = gemm(.{ .input = activated_quant, .weight = down, - .bias = opts.down.bias, + .bias = down_.bias, .input_scale = down_input_scale, - .weight_scale = opts.down.quantizationScales(), + .weight_scale = down_.quantizationScales(), .routing_weights = if (opts.routing_weight_placement == .after_down) routing_weights else null, .routing = routing, .expert_ids = expert_ids, @@ -765,13 +694,12 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { } const route = Tensor.arange(.{ .end = 2 }, .i32).reshape(.{ .t = 1, .topk = 2 }) .broad(Shape.init(.{ .t = x.dim(.t), .topk = 2 }, .i32)); - return fusedExpertsImpl(.{ - .hidden_states = x, - .gate_up = gate_up, - .down = down, - .topk_ids = route.addConstant(2), - .topk_weights = route.convert(.f32).addConstant(1).scale(0.25), - .gate_up_layout = layout, + return fusedExpertsBase(x, .{ + .ids = route.addConstant(2), + .weights = route.convert(.f32).addConstant(1).scale(0.25), + .global_num_experts = 8, + .local_num_experts = 8, + }, gate_up, down, layout, .{ .routing_weight_placement = placement, .quantize_input = false, .activation = .{ .swiglu = .{} }, @@ -873,54 +801,6 @@ pub fn validateOptions(opts: zml.moe.Options) void { stdx.debug.assert(opts.activation.swiglu.scale == null, "triton backend requires swiglu scale to be null", .{}); } -/// Row-major MXFP4 E2M1 weights and linear E8M0 block32 scales. -/// BF16 activations are quantized on the GPU to FP8 with per-32 E8M0 scales. -pub fn fusedExpertsMxfp4( - input: zml.Tensor, - ids: zml.Tensor, - weights: zml.Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - options: zml.moe.Options, -) zml.Tensor { - validateOptions(options); - stdx.debug.assert(input.dtype() == .bf16, "triton backend only supports bf16 inputs, got {}", .{input.dtype()}); - stdx.debug.assert(gate_up.bias == null and down.bias == null, "triton backend expects gate_up bias and down bias to be null", .{}); - - const gq = gate_up.quantization orelse @panic("triton backend requires gate_up quantization to be set"); - const dq = down.quantization orelse @panic("triton backend requires down quantization to be set"); - stdx.debug.assert(gq.scheme == .mxfp4 and dq.scheme == .mxfp4, "triton expects gate_up and down quantization scheme to be mxfp4, got {} and {}", .{ gq.scheme, dq.scheme }); - - // Weight storage for mxfp4 in HF is expressed as u8 or i8 - stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "triton expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); - stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "triton expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); - - const expert_parallelism = gate_up.weight.shape().partition(.expert).isSharded(); - - // Note: we directly forward input sharding to manualComputation - // TODO: improve sharding in attention and moe APIs - // https://github.com/zml/zml/issues/914 - const context: Context = .{ - .input = input, - .ids = ids, - .weights = weights, - .gate_up = gate_up.weight, - .gate_up_scales = gq.scales, - .down = down.weight, - .down_scales = dq.scales, - .global_experts = gate_up.weight.dim(.expert), - .topk = ids.dim(.topk), - .limit = options.activation.swiglu.limit.?, - .routing_weight_placement = options.routing_weight_placement, - .expert_parallel = expert_parallelism, - }; - - return if (expert_parallelism) - zml.ops.manualComputation(Context.body, context, input.shape()) - else - context.body(input.shape()); -} - pub const Config = struct { tokens: i64, hidden: i64, @@ -1012,178 +892,176 @@ fn validateInputs( ); } -const Context = struct { +/// Row-major MXFP4 E2M1 weights and linear E8M0 block32 scales. +/// BF16 activations are quantized on the GPU to FP8 with per-32 E8M0 scales. +pub fn fusedExpertsMxfp4( input: zml.Tensor, - ids: zml.Tensor, - weights: zml.Tensor, - gate_up: zml.Tensor, - gate_up_scales: zml.Tensor, - down: zml.Tensor, - down_scales: zml.Tensor, - global_experts: i64, - topk: i64, - limit: f32, - routing_weight_placement: RoutingWeightPlacement, - expert_parallel: bool, + routing: zml.moe.LocalRouting, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + options: zml.moe.Options, +) zml.Tensor { + validateOptions(options); + stdx.debug.assert(input.dtype() == .bf16, "triton backend only supports bf16 inputs, got {}", .{input.dtype()}); + stdx.debug.assert(gate_up.bias == null and down.bias == null, "triton backend expects gate_up bias and down bias to be null", .{}); - fn body(self: Context, _: zml.Shape) zml.Tensor { - const experts = self.gate_up.dim(.expert); - const hidden = self.down.dim(1); - const intermediate = self.down.dim(2) * 2; - var ids = self.ids.convert(.i32).reshape(.{ .token = .auto, .topk = self.topk }); - const tokens = ids.dim(.token); - if (self.expert_parallel) { - const partition_id = zml.ops.partitionId().convert(.i32); - ids = ids.sub(partition_id.scale(experts)); - } - const x = self.input.reshape(.{ tokens, hidden }); - const scales = self.weights.convert(.f32).reshape(.{ tokens, self.topk }); + const gq = gate_up.quantization orelse @panic("triton backend requires gate_up quantization to be set"); + const dq = down.quantization orelse @panic("triton backend requires down quantization to be set"); + stdx.debug.assert(gq.scheme == .mxfp4 and dq.scheme == .mxfp4, "triton expects gate_up and down quantization scheme to be mxfp4, got {} and {}", .{ gq.scheme, dq.scheme }); - const cfg: Config = .{ - .tokens = tokens, - .hidden = hidden, - .intermediate = intermediate, - .experts = experts, - .global_experts = self.global_experts, - .topk = self.topk, - .swiglu_limit = self.limit, - .routing_weight_placement = self.routing_weight_placement, - }; + // Weight storage for mxfp4 in HF is expressed as u8 or i8 + stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "triton expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); + stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "triton expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); - cfg.validate() catch @panic("Invalid MXFP4 MoE configuration"); - validateInputs(cfg, x, ids, scales, self.gate_up, self.gate_up_scales, self.down, self.down_scales); + const topk = routing.ids.dim(.topk); + const experts = gate_up.weight.dim(.expert); + const hidden = down.weight.dim(1); + const intermediate = down.weight.dim(2) * 2; + const ids = routing.localIds().convert(.i32).reshape(.{ .token = .auto, .topk = topk }); + const tokens = ids.dim(.token); - const bn = 16; - const bk = 512; + const x = input.reshape(.{ tokens, hidden }); + const scales = routing.weights.convert(.f32).reshape(.{ tokens, topk }); - const tile_count = kernels.tiles(cfg.tokens, cfg.topk, cfg.experts, bn); + const cfg: Config = .{ + .tokens = tokens, + .hidden = hidden, + .intermediate = intermediate, + .experts = experts, + .global_experts = routing.global_num_experts, + .topk = topk, + .swiglu_limit = options.activation.swiglu.limit.?, + .routing_weight_placement = options.routing_weight_placement, + }; - // Quantize activations - const q = kernels.Quant.Kernel.call( - .{ .x = x }, + cfg.validate() catch @panic("Invalid MXFP4 MoE configuration"); + validateInputs(cfg, x, ids, scales, gate_up.weight, gq.scales, down.weight, dq.scales); + + const bn = 16; + const bk = 512; + + const tile_count = kernels.tiles(cfg.tokens, cfg.topk, cfg.experts, bn); + + // Quantize activations + const q = kernels.Quant.Kernel.call( + .{ .x = x }, + .{ + .q = .init(.{ cfg.tokens, cfg.hidden }, .f8e4m3fn), + .s = .init(.{ cfg.tokens, @divTrunc(cfg.hidden, 32) }, .u8), + }, + .{ + .cfg = .{ .tokens = cfg.tokens, .hidden = cfg.hidden }, + .grid = .{ @intCast(@divTrunc(cfg.tokens + kernels.quantRows(cfg.tokens) - 1, kernels.quantRows(cfg.tokens))), @intCast(@divTrunc(cfg.hidden, 128)), 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ); + + const dummy = zml.Tensor.scalar(0, .i32); + var map = dummy; + var perm = dummy; + var sched = dummy; + + if (cfg.tokens != 1) { + const r = kernels.Route.Kernel.call( + .{ .ids = ids }, + .{ .counts = .init(.{cfg.experts}, .i32), .pos = .init(.{cfg.tokens * cfg.topk}, .i32) }, + .{ .cfg = .{ + .tokens = cfg.tokens, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + }, .grid = .{ @intCast(cfg.experts), 1, 1 }, .num_warps = 4, .num_stages = 1 }, + ); + + const s = kernels.Schedule.Kernel.call( + .{ .ids = ids, .counts = r.counts, .pos = r.pos }, + .{ .map = .init(.{tile_count * bn}, .i32), .perm = .init(.{cfg.tokens * cfg.topk}, .i32), .sched = .init(.{2 * tile_count + 1}, .i32) }, .{ - .q = .init(.{ cfg.tokens, cfg.hidden }, .f8e4m3fn), - .s = .init(.{ cfg.tokens, @divTrunc(cfg.hidden, 32) }, .u8), - }, - .{ - .cfg = .{ .tokens = cfg.tokens, .hidden = cfg.hidden }, - .grid = .{ @intCast(@divTrunc(cfg.tokens + kernels.quantRows(cfg.tokens) - 1, kernels.quantRows(cfg.tokens))), @intCast(@divTrunc(cfg.hidden, 128)), 1 }, + .cfg = .{ + .tokens = cfg.tokens, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .experts = cfg.experts, + .capacity = tile_count, + .group = bn, + }, + .grid = .{ @intCast(@divTrunc(cfg.tokens * cfg.topk + 127, 128)), 1, 1 }, .num_warps = 4, .num_stages = 1, }, ); - const dummy = zml.Tensor.scalar(0, .i32); - var map = dummy; - var perm = dummy; - var sched = dummy; - - if (cfg.tokens != 1) { - const r = kernels.Route.Kernel.call( - .{ .ids = ids }, - .{ .counts = .init(.{cfg.experts}, .i32), .pos = .init(.{cfg.tokens * cfg.topk}, .i32) }, - .{ .cfg = .{ - .tokens = cfg.tokens, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - }, .grid = .{ @intCast(cfg.experts), 1, 1 }, .num_warps = 4, .num_stages = 1 }, - ); - - const s = kernels.Schedule.Kernel.call( - .{ .ids = ids, .counts = r.counts, .pos = r.pos }, - .{ .map = .init(.{tile_count * bn}, .i32), .perm = .init(.{cfg.tokens * cfg.topk}, .i32), .sched = .init(.{2 * tile_count + 1}, .i32) }, - .{ - .cfg = .{ - .tokens = cfg.tokens, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - .experts = cfg.experts, - .capacity = tile_count, - .group = bn, - }, - .grid = .{ @intCast(@divTrunc(cfg.tokens * cfg.topk + 127, 128)), 1, 1 }, - .num_warps = 4, - .num_stages = 1, - }, - ); - - map = s.map; - perm = s.perm; - sched = s.sched; - } - - // Each narrow tile produces 64 activations: two complete scale groups. - const up_columns: i64 = if (cfg.tokens <= 8) 64 else 128; - const m = kernels.GateUp.Kernel.call( - .{ .q = q.q, .s = q.s, .w = self.gate_up.bitCast(.u8), .ws = self.gate_up_scales.bitCast(.u8), .ids = ids, .rw = scales, .map = map, .sched = sched }, - .{ .mid = .init(.{ tile_count * bn, cfg.intermediate }, .f8e4m3fn), .ms = .init(.{ tile_count * bn, @divTrunc(cfg.intermediate, 32) }, .u8) }, - .{ - .cfg = .{ - .tokens = cfg.tokens, - .hidden = cfg.hidden, - .intermediate = cfg.intermediate, - .experts = cfg.experts, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - .swiglu_limit = cfg.swiglu_limit, - .scale_stride = self.gate_up_scales.dim(2), - .routing_weight_placement = cfg.routing_weight_placement, - .bn = bn, - .bk = bk, - }, - .grid = .{ @intCast(@divTrunc(cfg.intermediate, up_columns)), @intCast(tile_count), 1 }, - .num_warps = if (cfg.tokens <= 8) 4 else 8, - .num_stages = 2, - .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.intermediate, up_columns) * tile_count), - }, - ); - - const d = kernels.Down.Kernel.call( - .{ .mid = m.mid, .ms = m.ms, .w = self.down.bitCast(.u8), .ws = self.down_scales.bitCast(.u8), .ids = ids, .sched = sched }, - .{ .d = .init(.{ tile_count * bn, cfg.hidden }, .bf16) }, - .{ - .cfg = .{ - .tokens = cfg.tokens, - .hidden = cfg.hidden, - .intermediate = cfg.intermediate, - .experts = cfg.experts, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - .swiglu_limit = cfg.swiglu_limit, - .scale_stride = self.down_scales.dim(2), - .routing_weight_placement = cfg.routing_weight_placement, - .bn = bn, - .bk = bk, - }, - .grid = .{ @intCast(@divTrunc(cfg.hidden, 128)), @intCast(tile_count), 1 }, - .num_warps = if (cfg.tokens <= 8) 4 else 8, - .num_stages = 2, - .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.hidden, 128) * tile_count), - }, - ); - - const result = kernels.Combine.Kernel.call( - .{ .d = d.d, .ids = ids, .perm = perm, .rw = scales }, - .{ .y = x.shape() }, - .{ - .cfg = .{ - .tokens = cfg.tokens, - .hidden = cfg.hidden, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - .experts = cfg.experts, - .routing_weight_placement = cfg.routing_weight_placement, - .bn = bn, - }, - .grid = .{ @intCast(cfg.tokens), @intCast(@divTrunc(cfg.hidden + kernels.combineBlock(cfg.tokens, cfg.hidden) - 1, kernels.combineBlock(cfg.tokens, cfg.hidden))), 1 }, - .num_warps = 4, - .num_stages = 1, - }, - ).y.reshape(self.input.shape().dims()).withTags(self.input.shape()); - - return if (self.expert_parallel) - zml.ops.allReduce(result, zml.Tensor.add) - else - result; + map = s.map; + perm = s.perm; + sched = s.sched; } -}; + + // Each narrow tile produces 64 activations: two complete scale groups. + const up_columns: i64 = if (cfg.tokens <= 8) 64 else 128; + const m = kernels.GateUp.Kernel.call( + .{ .q = q.q, .s = q.s, .w = gate_up.weight.bitCast(.u8), .ws = gq.scales.bitCast(.u8), .ids = ids, .rw = scales, .map = map, .sched = sched }, + .{ .mid = .init(.{ tile_count * bn, cfg.intermediate }, .f8e4m3fn), .ms = .init(.{ tile_count * bn, @divTrunc(cfg.intermediate, 32) }, .u8) }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .hidden = cfg.hidden, + .intermediate = cfg.intermediate, + .experts = cfg.experts, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .swiglu_limit = cfg.swiglu_limit, + .scale_stride = gq.scales.dim(2), + .routing_weight_placement = cfg.routing_weight_placement, + .bn = bn, + .bk = bk, + }, + .grid = .{ @intCast(@divTrunc(cfg.intermediate, up_columns)), @intCast(tile_count), 1 }, + .num_warps = if (cfg.tokens <= 8) 4 else 8, + .num_stages = 2, + .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.intermediate, up_columns) * tile_count), + }, + ); + + const d = kernels.Down.Kernel.call( + .{ .mid = m.mid, .ms = m.ms, .w = down.weight.bitCast(.u8), .ws = dq.scales.bitCast(.u8), .ids = ids, .sched = sched }, + .{ .d = .init(.{ tile_count * bn, cfg.hidden }, .bf16) }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .hidden = cfg.hidden, + .intermediate = cfg.intermediate, + .experts = cfg.experts, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .swiglu_limit = cfg.swiglu_limit, + .scale_stride = dq.scales.dim(2), + .routing_weight_placement = cfg.routing_weight_placement, + .bn = bn, + .bk = bk, + }, + .grid = .{ @intCast(@divTrunc(cfg.hidden, 128)), @intCast(tile_count), 1 }, + .num_warps = if (cfg.tokens <= 8) 4 else 8, + .num_stages = 2, + .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.hidden, 128) * tile_count), + }, + ); + + return kernels.Combine.Kernel.call( + .{ .d = d.d, .ids = ids, .perm = perm, .rw = scales }, + .{ .y = x.shape() }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .hidden = cfg.hidden, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .experts = cfg.experts, + .routing_weight_placement = cfg.routing_weight_placement, + .bn = bn, + }, + .grid = .{ @intCast(cfg.tokens), @intCast(@divTrunc(cfg.hidden + kernels.combineBlock(cfg.tokens, cfg.hidden) - 1, kernels.combineBlock(cfg.tokens, cfg.hidden))), 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ).y.reshape(input.shape().dims()).withTags(input.shape()); +} diff --git a/zml/ops.zig b/zml/ops.zig index 19d15ec3..4a189035 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -8,18 +8,6 @@ const platforms = @import("platforms"); const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); -const manual_computation = @import("ops/manual_computation.zig"); -pub const manualComputation = manual_computation.manualComputation; - -const manual_computation2 = @import("ops/manual_computation2.zig"); -pub const manualComputation2 = manual_computation2.manualComputation2; -pub const ManualPartitioning = manual_computation2.ManualPartitioning; - -test { - _ = manual_computation; - _ = manual_computation2; -} - const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("mem.zig").Bufferized; const Compiler = @import("Compiler.zig"); @@ -28,15 +16,113 @@ const CustomCallBuffer = @import("pjrtx.zig").CustomCallBuffer; const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); +const manual_computation = @import("ops/manual_computation.zig"); +pub const manualComputation = manual_computation.manualComputation; +const manual_computation2 = @import("ops/manual_computation2.zig"); +pub const manualComputation2 = manual_computation2.manualComputation2; +pub const ManualPartitioning = manual_computation2.ManualPartitioning; const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const ShapeToCustomCallBuffer = @import("pjrtx.zig").ShapeToCustomCallBuffer; const Sharding = @import("Sharding.zig"); const Tensor = @import("tensor.zig").Tensor; const TensorToCustomCallBuffer = @import("pjrtx.zig").TensorToCustomCallBuffer; + +test { + _ = manual_computation; + _ = manual_computation2; +} + const log = std.log.scoped(.@"zml/Compiler"); +/// Reduces local tensor values across all processes in the computation. +/// For each tensor index, `func_` combines the values at that index from every +/// process. The result is identical on all processes. +/// +/// Inside a manual computation, `inputs` describes the local tensors at each +/// partition. For example, summing `[1, 2]` and `[3, 4]` from two partitions gives +/// `[4, 6]` at each partition. The local tensor dimensions stay unchanged. +/// +/// * `inputs` is a `Tensor`, a nonempty tuple of tensors, or an `[N]Tensor`. +/// Each input must have matching shapes and dtypes across processes. +/// * `func_` describes how to combine two scalar contributions: +/// - For one input, it must be `fn (left: Tensor, right: Tensor) Tensor`, such +/// as `Tensor.add`. This also applies to a one-element tuple or array. +/// - For multiple inputs, it takes one `ReduceArgs` per input and returns a +/// tuple or array with one scalar tensor per input, in the same order. +/// +/// Returns the same container type as `inputs`, preserving tensor attributes. +/// With one process, returns `inputs` unchanged. +/// +/// The reduction uses one group containing all processes, including mesh replicas. +/// To reduce along one logical mesh axis, use `allReduceOnAxis`. pub fn allReduce(inputs: anytype, comptime func_: anytype) AllReduceReturnType(@TypeOf(inputs)) { + const count = Compiler.current().meshes[0].numDevices(); + var devices: [Platform.MAX_NUM_DEVICES]i64 = undefined; + for (devices[0..count], 0..) |*device, i| device.* = @intCast(i); + return allReduceGroups(inputs, func_, devices[0..count], count); +} + +/// Reduces local tensor values within process groups defined by `axis` in `mesh`. +/// A group contains processes with the same coordinates on every other logical +/// mesh axis. Separate replicas of the mesh belong to separate groups. Within +/// each group, `func_` combines corresponding elements and the result is identical +/// on all member processes. +/// +/// Use this inside a manual computation to combine partial results along `axis` +/// while preserving the independent computations along the other mesh axes. +/// +/// * `inputs` and `func_` follow `allReduce`'s tensor and reducer contracts. +/// * `mesh` is the logical mesh used by the manual computation and must be +/// included in the compilation's meshes. +/// * `axis` names an axis of that mesh, for example `.experts`. It can be an enum +/// literal or a mesh axis tag. The axis must exist in `mesh`. +/// +/// Returns the same container type with tensor attributes as +/// `inputs`. An axis of size one returns `inputs` unchanged. +/// +/// For a mesh with `.data = 2` and `.experts = 4`, the following sums four local +/// expert contributions independently for each of the two data shards: +/// +/// ```zig +/// const output = allReduceOnAxis(local_output, Tensor.add, mesh, .experts); +/// ``` +/// +/// Each expert shard now has the complete output for its data shard. To expose +/// this result through `manualComputation2`, an output spec such as +/// `.parse(.{ .t = .data })` keeps tokens split across `.data` and declares the +/// result replicated along `.experts`. +pub fn allReduceOnAxis(inputs: anytype, comptime func_: anytype, mesh: *const Sharding.Mesh, axis: anytype) AllReduceReturnType(@TypeOf(inputs)) { + const logical_axis = mesh.resolveLogicalAxis(Shape.toTag(axis)) orelse @panic("Unknown logical reduction axis"); + const size = mesh.grid.axis_sizes[logical_axis]; + var groups: [Platform.MAX_NUM_DEVICES]i64 = undefined; + var replica_indices: [Platform.MAX_NUM_DEVICES]usize = @splat(0); + const replicas = mesh.numReplicas(); + for (0..mesh.numDevices()) |device| { + const coords = mesh.grid.logical_coords[device]; + var tile: usize = 0; + var group: usize = 0; + for (0..mesh.logical.axes.len) |dim| { + tile = tile * mesh.grid.axis_sizes[dim] + coords[dim]; + if (dim != logical_axis) group = group * mesh.grid.axis_sizes[dim] + coords[dim]; + } + group = group * replicas + replica_indices[tile]; + replica_indices[tile] += 1; + groups[group * size + coords[logical_axis]] = @intCast(device); + } + return allReduceGroups(inputs, func_, groups[0..mesh.numDevices()], size); +} + +/// Reduces corresponding tensor elements independently within each process group. +/// `inputs`, `func_` and the return value follow `allReduce`'s contract. +/// +/// `groups` identifies processes by their executable partition IDs, as returned +/// by `partitionId()`. Each consecutive `group_size` IDs defines one group. +/// +/// For example, `groups = &.{ 0, 2, 1, 3 }` and `group_size = 2` combine partitions +/// 0 and 2 together, and partitions 1 and 3 together. Both members of each group +/// receive that group's result. Groups of size one return `inputs` unchanged. +fn allReduceGroups(inputs: anytype, comptime func_: anytype, groups: []const i64, group_size: usize) AllReduceReturnType(@TypeOf(inputs)) { const ctx = Compiler.current(); const mlir_ctx = ctx.mlir_ctx; @@ -80,8 +166,7 @@ pub fn allReduce(inputs: anytype, comptime func_: anytype) AllReduceReturnType(@ else => @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"), }; - const num_devices = ctx.meshes[0].numDevices(); - if (num_devices <= 1) return inputs; + if (group_size <= 1) return inputs; const reducer_block = b: { const ArgsTypes: [input_tensors.len]type = @splat(ReduceArgs); @@ -130,14 +215,9 @@ pub fn allReduce(inputs: anytype, comptime func_: anytype) AllReduceReturnType(@ break :b block; }; - var replica_group_storage: [constants.MAX_RANK]i64 = undefined; - for (0..@intCast(num_devices)) |i| { - replica_group_storage[i] = @intCast(i); - } - const replica_group_slice = replica_group_storage[0..@intCast(num_devices)]; const replica_groups_attr = mlir.Attribute.denseElements( - .rankedTensor(&.{ 1, num_devices }, .int(mlir_ctx, .i64)), - replica_group_slice, + .rankedTensor(&.{ @intCast(groups.len / group_size), @intCast(group_size) }, .int(mlir_ctx, .i64)), + groups, ); const handle = ctx.nextChannelId(); @@ -207,6 +287,20 @@ fn AllReduceReturnType(comptime InputsT: type) type { }; } +/// Returns the current process's zero based coordinate on `axis` in `mesh` +/// as a scalar u32. +/// +/// Example: for a mesh {.data = 2, .experts = 2}, each data group has expert coordinates +/// 0 and 1. Use this coordinate inside a manual computation to identify a shard +/// along one axis while `partitionId()` identifies a partition across the whole mesh. +pub fn axisIndex(mesh: *const Sharding.Mesh, axis: anytype) Tensor { + const logical_axis = mesh.resolveLogicalAxis(Shape.toTag(axis)) orelse @panic("Unknown logical mesh axis"); + var coords: [Platform.MAX_NUM_DEVICES]u32 = undefined; + for (coords[0..mesh.numDevices()], 0..) |*coord, i| coord.* = mesh.grid.logical_coords[i][logical_axis]; + return Tensor.constantTensor(.init(.{ .device = mesh.numDevices() }, .u32), std.mem.sliceAsBytes(coords[0..mesh.numDevices()])) + .gather(.{ .device = partitionId() }, .{}); +} + pub fn partitionId() Tensor { const ctx = Compiler.current(); const op = stablehlo.partition_id(ctx.mlir_ctx, ctx.location).appendTo(ctx.currentScope().block); From 7fbd15eee48609f095a11005493d5beef059c443 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 9 Nov 2026 19:24:37 +0100 Subject: [PATCH 859/881] [Modularization] zml: break cycle in `zml/attention` (#1038) Break cycle between `zml/attention/attention.zig` and `zml/kernels/flash_attention/flash_attention.zig` --- zml/BUILD.bazel | 2 + zml/attention/Options.zig | 3 + zml/attention/attention.zig | 6 +- zml/kernels/flash_attention/README.md | 2 +- .../flash_attention/flash_attention.zig | 330 +----------------- zml/kernels/flash_attention/tests.zig | 330 ++++++++++++++++++ 6 files changed, 340 insertions(+), 333 deletions(-) create mode 100644 zml/attention/Options.zig create mode 100644 zml/kernels/flash_attention/tests.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index dbb2ad33..4d58fc77 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -78,6 +78,7 @@ zig_library( srcs = [ "Compiler.zig", "Sharding.zig", + "attention/Options.zig", "attention/attention.zig", "buffer.zig", "buffer/HostAccessible.zig", @@ -88,6 +89,7 @@ zig_library( "io.zig", "kernel.zig", "kernels/flash_attention/flash_attention.zig", + "kernels/flash_attention/tests.zig", "kernels/moe/moe.zig", "kernels/moe/tests.zig", "kernels/paged_attention/paged_attention.zig", diff --git a/zml/attention/Options.zig b/zml/attention/Options.zig new file mode 100644 index 00000000..539f753f --- /dev/null +++ b/zml/attention/Options.zig @@ -0,0 +1,3 @@ +/// Query j sees the keys up to token_index + j, as in a decoder. Otherwise every query sees every +/// key, as in an encoder or a diffusion transformer, and token_index is ignored. +causal: bool = true, diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index 039c6628..0ac2a95c 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -93,11 +93,7 @@ pub const Metadata = union(BackendType) { } }; -pub const Options = struct { - /// Query j sees the keys up to token_index + j, as in a decoder. Otherwise every query sees every - /// key, as in an encoder or a diffusion transformer, and token_index is ignored. - causal: bool = true, -}; +pub const Options = @import("Options.zig"); pub const Partitioning = struct { const Logical = zml.Sharding.Partitioning.Logical; diff --git a/zml/kernels/flash_attention/README.md b/zml/kernels/flash_attention/README.md index b5fe7682..81a35cf4 100644 --- a/zml/kernels/flash_attention/README.md +++ b/zml/kernels/flash_attention/README.md @@ -265,7 +265,7 @@ are not uniformly supported by the current kernels; an existing prefill path requires equal query/key lengths and assumes positions begin at zero. Treat these as current conformance gaps, not alternative definitions of causality. -The tests in [flash_attention.zig](flash_attention.zig) compare results +The tests in [tests.zig](tests.zig) compare results against `sdpa` with the gathered causal mask. The original shared cases cover `D = 64`, MHA and GQA, single-token decode at full and partial context lengths, batched decode with diff --git a/zml/kernels/flash_attention/flash_attention.zig b/zml/kernels/flash_attention/flash_attention.zig index 78602bd0..3fb898db 100644 --- a/zml/kernels/flash_attention/flash_attention.zig +++ b/zml/kernels/flash_attention/flash_attention.zig @@ -6,9 +6,11 @@ pub const cute = @import("cute/cute.zig"); pub const metal = @import("metal/metal.zig"); pub const nki = @import("nki/nki.zig"); const zml = @import("../../zml.zig"); +const Options = @import("../../attention/Options.zig"); test { std.testing.refAllDecls(@This()); + _ = @import("tests.zig"); } pub fn isAvailable(platform: *const zml.Platform) bool { @@ -126,7 +128,7 @@ pub const Metadata = union(Backend) { /// - .q is the number of queries /// - .k is the number of keys /// - .hd is the head dimension -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: zml.attention.Options) zml.Tensor { +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options) zml.Tensor { zml.stdx.debug.assert(zml.Compiler.current().manual_computation_depth > 0, "Attention kernels require an explicit manualComputation boundary", .{}); return switch (parameters) { .cute => cute.attention(q, k, v, token_index, opts.causal), @@ -135,329 +137,3 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T .metal => if (opts.causal) metal.attention(q, k, v, token_index, metadata.metal) else zml.nn.sdpa(q, k, v, .{}), }; } - -test "attention: q=1,qh=64,kh=8" { - try testAttention( - .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{63}, - ); - - try testAttention( - .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{36}, - ); -} - -test "attention: b=4,q=1,qh=64,kh=8" { - // Full attention - try testAttention( - .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{ 63, 63, 63, 63 }, - ); - - // Partial attention - try testAttention( - .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{ 61, 57, 23, 63 }, - ); -} - -test "attention: q=1,qh=8,kh=8" { - try testAttention( - .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{62}, - ); - try testAttention( - .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{63}, - ); -} - -test "attention: q=8,qh=64,kh=8" { - try testAttention( - .init(.{ .q = 8, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{56}, - ); -} - -test "attention: prefill, q=k=300,qh=8,kh=4" { - // Neither length is a multiple of the query (128) or key (64) tiles. - try testAttention( - .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), - .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), - &.{0}, - ); -} - -test "attention: prefill, q=200,k=1000,qh=4,kh=4,hd=64" { - // Queries start at 800: every query tile sees full key tiles before the diagonal. - try testAttention( - .init(.{ .q = 200, .h = 4, .hd = 64 }, .bf16), - .init(.{ .k = 1000, .h = 4, .hd = 64 }, .bf16), - &.{800}, - ); -} - -test "attention: prefill, q=k=200,qh=4,kh=2,hd=32" { - // Rows of 32 features only hold 4 16-byte vectors: the shared memory swizzle must stay inside them. - try testAttention( - .init(.{ .q = 200, .h = 4, .hd = 32 }, .bf16), - .init(.{ .k = 200, .h = 2, .hd = 32 }, .bf16), - &.{0}, - ); -} - -test "attention: prefill, q=k=130,qh=4,kh=2,hd=256" { - try testAttention( - .init(.{ .q = 130, .h = 4, .hd = 256 }, .bf16), - .init(.{ .k = 130, .h = 2, .hd = 256 }, .bf16), - &.{0}, - ); -} - -test "attention: prefill, b=2,q=150,k=300,qh=4,kh=2" { - try testAttention( - .init(.{ .b = 2, .q = 150, .h = 4, .hd = 128 }, .bf16), - .init(.{ .b = 2, .k = 300, .h = 2, .hd = 128 }, .bf16), - &.{ 150, 100 }, - ); -} - -test "attention: non-causal prefill, q=k=300,qh=8,kh=4" { - try testAttentionOpts( - .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), - .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), - &.{0}, - .{ .causal = false }, - ); -} - -test "attention: non-causal prefill, b=2,q=200,k=1000,qh=4,kh=4,hd=64" { - // Queries are a suffix of the keys, as for the image tokens of a joint text/image attention. - try testAttentionOpts( - .init(.{ .b = 2, .q = 200, .h = 4, .hd = 64 }, .bf16), - .init(.{ .b = 2, .k = 1000, .h = 4, .hd = 64 }, .bf16), - &.{ 0, 0 }, - .{ .causal = false }, - ); -} - -test "attention: non-causal prefill, more queries than keys, q=300,k=100,qh=4,kh=2" { - try testAttentionOpts( - .init(.{ .q = 300, .h = 4, .hd = 128 }, .bf16), - .init(.{ .k = 100, .h = 2, .hd = 128 }, .bf16), - &.{0}, - .{ .causal = false }, - ); -} - -test "attention: non-causal decode ignores token_index, q=1,k=1000,qh=64,kh=8" { - try testAttentionOpts( - .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 1000, .h = 8, .hd = 64 }, .bf16), - &.{5}, - .{ .causal = false }, - ); -} - -pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32) !void { - return testAttentionOpts(q_shape, k_shape, token_index_h, .{}); -} - -fn testAttentionOpts(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options) !void { - return testAttentionBackends(q_shape, k_shape, token_index_h, opts, std.enums.values(Backend)); -} - -fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options, backends: []const Backend) !void { - const io = std.testing.io; - const allocator = std.testing.allocator; - - const platform = zml.testing.env(); - - var arena_state: std.heap.ArenaAllocator = .init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - const token_index_shape: zml.Shape = if (token_index_h.len > 1) b: { - std.debug.assert(token_index_h.len == q_shape.dim(.b)); - break :b .init(.{ .b = q_shape.dim(.b) }, .u32); - } else .init(.{}, .u32); - - const max_k_idx: i64 = k_shape.dim(.k) - 1; - const max_q_idx: i64 = q_shape.dim(.q) - 1; - if (opts.causal) { - // Check for out of bound reads, token_index is ignored by non-causal attention. - for (token_index_h) |index| { - std.debug.assert(index + max_q_idx <= max_k_idx); - } - } - - const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor } = .{ - .q = .fromShape(q_shape), - .k = .fromShape(k_shape), - .v = .fromShape(k_shape), - .token_index = .fromShape(token_index_shape), - }; - - var q = try attentionTestInput(q_shape, 17); - defer q.deinit(); - var k = try attentionTestInput(k_shape, 29); - defer k.deinit(); - var v = try attentionTestInput(k_shape, 43); - defer v.deinit(); - var token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, @ptrCast(token_index_h)); - defer token_index.deinit(); - - const meshes = platform.meshes.values(); - const mesh = zml.testing.tpMesh(); - const partitioning: zml.attention.Partitioning = if (@rem(k_shape.dim(.h), mesh.numPartitionsForLogicalAxis(.model)) == 0) .head(.model) else .{ - .in_specs = .{ .q = .replicated, .k = .replicated, .v = .replicated }, - .out_specs = .replicated, - }; - const AttentionCall = struct { - fn forward(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, position: zml.Tensor, metadata: zml.attention.Metadata, parameters: zml.attention.Parameters, opts_: zml.attention.Options, mesh_: *const zml.Mesh, partitioning_: zml.attention.Partitioning) zml.Tensor { - return zml.attention.attentionOpts(q_, k_, v_, position, metadata, parameters, opts_, mesh_, partitioning_); - } - }; - const reference_metadata: zml.attention.Metadata = .{ .stablehlo = {} }; - const reference_parameters: zml.attention.Parameters = .{ .stablehlo = {} }; - const reference_exe = try platform.compileFn(allocator, io, AttentionCall.forward, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, reference_metadata, reference_parameters, opts, mesh, partitioning }, .{ - .program_name = "attention_stablehlo", - .meshes = meshes, - }); - defer reference_exe.deinit(); - - var reference_d = try reference_exe.eval(allocator, io, .{ q, k, v, token_index, reference_metadata }); - defer reference_d.deinit(); - try reference_d.await(io); - try std.testing.expectEqualDeep(q_shape.withPartitioning(mesh, partitioning.out_specs)._sharding, reference_d.shape()._sharding); - const reference_h: zml.Slice = try reference_d.toSliceAlloc(allocator, io); - defer reference_h.free(allocator); - - for (backends) |backend| { - if (!backend.isAvailable(platform)) continue; - - const metadata: Metadata = .init(.fromBackend(backend)); - const parameters: Parameters = .init(.fromBackend(backend)); - const exe = try platform.compileFn( - allocator, - io, - AttentionCall.forward, - .{ tensors.q, tensors.k, tensors.v, tensors.token_index, .{ .flash_attention = metadata }, .{ .flash_attention = parameters }, opts, mesh, partitioning }, - .{ - .program_name = try std.fmt.allocPrint(arena, "attention_{t}", .{backend}), - .meshes = meshes, - }, - ); - defer exe.deinit(); - - var metadata_d = try metadata.initBuffer(io, platform); - defer Metadata.deinitBuffer(&metadata_d); - - var output_d = try exe.eval(allocator, io, .{ q, k, v, token_index, metadata_d }); - defer output_d.deinit(); - try output_d.await(io); - const output_h = try output_d.toSliceAlloc(allocator, io); - defer output_h.free(allocator); - - errdefer std.log.err( - \\ Attention test failed, {0t} output doesn't match reference. - \\ - reference: {1d} - \\ - {0t}: {2d} - , .{ backend, reference_h, output_h }); - try zml.testing.expectClose(io, reference_h, output_h, .{ - .absolute_tolerance = 5e-3, - .relative_tolerance = 1e-2, - .epsilon_relative = 1e-3, - .minimum_close_fraction = 0.99, - }); - } -} - -fn attentionTestInput(shape: zml.Shape, seed: u64) !zml.Buffer { - const host = try zml.Slice.alloc(std.testing.allocator, shape); - defer host.free(std.testing.allocator); - var prng: std.Random.DefaultPrng = .init(seed); - const random = prng.random(); - switch (shape.dtype()) { - inline .bf16, .f16, .f32 => |dtype| { - for (host.items(dtype.toZigType())) |*value| { - const x = random.float(f32) * 4 - 2; - value.* = if (dtype == .bf16) .fromF32(x) else @floatCast(x); - } - }, - else => unreachable, - } - return zml.Buffer.fromSlice(std.testing.io, zml.testing.env(), host); -} - -/// Independent f64 oracle: the SDPA reference uses fast dot precision, so its -/// f32 result alone cannot establish the accuracy of the kernel's f32 arithmetic. -fn checkAttentionF32(q: zml.Slice, k: zml.Slice, v: zml.Slice, positions: []const u32, output: zml.Slice) !void { - const Layout = struct { - b: usize, - s: usize, - h: usize, - d: usize, - - fn stride(shape: zml.Shape, tag: anytype) usize { - const axis = shape.hasTag(tag) orelse return 0; - var result: usize = 1; - for (shape.dims()[axis + 1 ..]) |dim| result *= @intCast(dim); - return result; - } - - fn init(shape: zml.Shape) @This() { - return .{ .b = stride(shape, .b), .s = if (shape.hasTag(.q) != null) stride(shape, .q) else stride(shape, .k), .h = stride(shape, .h), .d = stride(shape, .hd) }; - } - - fn offset(self: @This(), b: usize, s: usize, h: usize, d: usize) usize { - return b * self.b + s * self.s + h * self.h + d * self.d; - } - }; - const ql = Layout.init(q.shape); - const kl = Layout.init(k.shape); - const vl = Layout.init(v.shape); - const batch: usize = if (q.shape.hasTag(.b) != null) @intCast(q.shape.dim(.b)) else 1; - const queries: usize = @intCast(q.shape.dim(.q)); - const heads: usize = @intCast(q.shape.dim(.h)); - const dim: usize = @intCast(q.shape.dim(.hd)); - const group: usize = @intCast(@divExact(q.shape.dim(.h), k.shape.dim(.h))); - const scores = try std.testing.allocator.alloc(f64, @intCast(k.shape.dim(.k))); - defer std.testing.allocator.free(scores); - const scale = 1.0 / @sqrt(@as(f64, @floatFromInt(dim))); - for (0..batch) |b| { - const start: usize = positions[if (positions.len == 1) 0 else b]; - for (0..queries) |i| { - for (0..heads) |h| { - var maximum: f64 = -std.math.inf(f64); - const active = scores[0 .. start + i + 1]; - for (active, 0..) |*score, j| { - var dot: f64 = 0; - for (0..dim) |d| dot += @as(f64, q.items(f32)[ql.offset(b, i, h, d)]) * @as(f64, k.items(f32)[kl.offset(b, j, h / group, d)]); - score.* = dot * scale; - maximum = @max(maximum, score.*); - } - var sum: f64 = 0; - for (active) |*score| { - score.* = @exp(score.* - maximum); - sum += score.*; - } - for (0..dim) |d| { - var expected: f64 = 0; - for (active, 0..) |weight, j| expected += weight * @as(f64, v.items(f32)[vl.offset(b, j, h / group, d)]); - expected /= sum; - const actual = output.items(f32)[ql.offset(b, i, h, d)]; - try std.testing.expectApproxEqAbs(expected, @as(f64, actual), 2e-5 + 2e-5 * @abs(expected)); - } - } - } - } -} diff --git a/zml/kernels/flash_attention/tests.zig b/zml/kernels/flash_attention/tests.zig new file mode 100644 index 00000000..d93bd373 --- /dev/null +++ b/zml/kernels/flash_attention/tests.zig @@ -0,0 +1,330 @@ +const std = @import("std"); + +const zml = @import("../../zml.zig"); +const flash_attention = @import("flash_attention.zig"); + +test "attention: q=1,qh=64,kh=8" { + try testAttention( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{63}, + ); + + try testAttention( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{36}, + ); +} + +test "attention: b=4,q=1,qh=64,kh=8" { + // Full attention + try testAttention( + .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{ 63, 63, 63, 63 }, + ); + + // Partial attention + try testAttention( + .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{ 61, 57, 23, 63 }, + ); +} + +test "attention: q=1,qh=8,kh=8" { + try testAttention( + .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{62}, + ); + try testAttention( + .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{63}, + ); +} + +test "attention: q=8,qh=64,kh=8" { + try testAttention( + .init(.{ .q = 8, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{56}, + ); +} + +test "attention: prefill, q=k=300,qh=8,kh=4" { + // Neither length is a multiple of the query (128) or key (64) tiles. + try testAttention( + .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), + .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), + &.{0}, + ); +} + +test "attention: prefill, q=200,k=1000,qh=4,kh=4,hd=64" { + // Queries start at 800: every query tile sees full key tiles before the diagonal. + try testAttention( + .init(.{ .q = 200, .h = 4, .hd = 64 }, .bf16), + .init(.{ .k = 1000, .h = 4, .hd = 64 }, .bf16), + &.{800}, + ); +} + +test "attention: prefill, q=k=200,qh=4,kh=2,hd=32" { + // Rows of 32 features only hold 4 16-byte vectors: the shared memory swizzle must stay inside them. + try testAttention( + .init(.{ .q = 200, .h = 4, .hd = 32 }, .bf16), + .init(.{ .k = 200, .h = 2, .hd = 32 }, .bf16), + &.{0}, + ); +} + +test "attention: prefill, q=k=130,qh=4,kh=2,hd=256" { + try testAttention( + .init(.{ .q = 130, .h = 4, .hd = 256 }, .bf16), + .init(.{ .k = 130, .h = 2, .hd = 256 }, .bf16), + &.{0}, + ); +} + +test "attention: prefill, b=2,q=150,k=300,qh=4,kh=2" { + try testAttention( + .init(.{ .b = 2, .q = 150, .h = 4, .hd = 128 }, .bf16), + .init(.{ .b = 2, .k = 300, .h = 2, .hd = 128 }, .bf16), + &.{ 150, 100 }, + ); +} + +test "attention: non-causal prefill, q=k=300,qh=8,kh=4" { + try testAttentionOpts( + .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), + .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), + &.{0}, + .{ .causal = false }, + ); +} + +test "attention: non-causal prefill, b=2,q=200,k=1000,qh=4,kh=4,hd=64" { + // Queries are a suffix of the keys, as for the image tokens of a joint text/image attention. + try testAttentionOpts( + .init(.{ .b = 2, .q = 200, .h = 4, .hd = 64 }, .bf16), + .init(.{ .b = 2, .k = 1000, .h = 4, .hd = 64 }, .bf16), + &.{ 0, 0 }, + .{ .causal = false }, + ); +} + +test "attention: non-causal prefill, more queries than keys, q=300,k=100,qh=4,kh=2" { + try testAttentionOpts( + .init(.{ .q = 300, .h = 4, .hd = 128 }, .bf16), + .init(.{ .k = 100, .h = 2, .hd = 128 }, .bf16), + &.{0}, + .{ .causal = false }, + ); +} + +test "attention: non-causal decode ignores token_index, q=1,k=1000,qh=64,kh=8" { + try testAttentionOpts( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 1000, .h = 8, .hd = 64 }, .bf16), + &.{5}, + .{ .causal = false }, + ); +} + +pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32) !void { + return testAttentionOpts(q_shape, k_shape, token_index_h, .{}); +} + +fn testAttentionOpts(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options) !void { + return testAttentionBackends(q_shape, k_shape, token_index_h, opts, std.enums.values(flash_attention.Backend)); +} + +fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options, backends: []const flash_attention.Backend) !void { + const io = std.testing.io; + const allocator = std.testing.allocator; + + const platform = zml.testing.env(); + + var arena_state: std.heap.ArenaAllocator = .init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + const token_index_shape: zml.Shape = if (token_index_h.len > 1) b: { + std.debug.assert(token_index_h.len == q_shape.dim(.b)); + break :b .init(.{ .b = q_shape.dim(.b) }, .u32); + } else .init(.{}, .u32); + + const max_k_idx: i64 = k_shape.dim(.k) - 1; + const max_q_idx: i64 = q_shape.dim(.q) - 1; + if (opts.causal) { + // Check for out of bound reads, token_index is ignored by non-causal attention. + for (token_index_h) |index| { + std.debug.assert(index + max_q_idx <= max_k_idx); + } + } + + const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor } = .{ + .q = .fromShape(q_shape), + .k = .fromShape(k_shape), + .v = .fromShape(k_shape), + .token_index = .fromShape(token_index_shape), + }; + + var q = try attentionTestInput(q_shape, 17); + defer q.deinit(); + var k = try attentionTestInput(k_shape, 29); + defer k.deinit(); + var v = try attentionTestInput(k_shape, 43); + defer v.deinit(); + var token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, @ptrCast(token_index_h)); + defer token_index.deinit(); + + const meshes = platform.meshes.values(); + const mesh = zml.testing.tpMesh(); + const partitioning: zml.attention.Partitioning = if (@rem(k_shape.dim(.h), mesh.numPartitionsForLogicalAxis(.model)) == 0) .head(.model) else .{ + .in_specs = .{ .q = .replicated, .k = .replicated, .v = .replicated }, + .out_specs = .replicated, + }; + const AttentionCall = struct { + fn forward(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, position: zml.Tensor, metadata: zml.attention.Metadata, parameters: zml.attention.Parameters, opts_: zml.attention.Options, mesh_: *const zml.Mesh, partitioning_: zml.attention.Partitioning) zml.Tensor { + return zml.attention.attentionOpts(q_, k_, v_, position, metadata, parameters, opts_, mesh_, partitioning_); + } + }; + const reference_metadata: zml.attention.Metadata = .{ .stablehlo = {} }; + const reference_parameters: zml.attention.Parameters = .{ .stablehlo = {} }; + const reference_exe = try platform.compileFn(allocator, io, AttentionCall.forward, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, reference_metadata, reference_parameters, opts, mesh, partitioning }, .{ + .program_name = "attention_stablehlo", + .meshes = meshes, + }); + defer reference_exe.deinit(); + + var reference_d = try reference_exe.eval(allocator, io, .{ q, k, v, token_index, reference_metadata }); + defer reference_d.deinit(); + try reference_d.await(io); + try std.testing.expectEqualDeep(q_shape.withPartitioning(mesh, partitioning.out_specs)._sharding, reference_d.shape()._sharding); + const reference_h: zml.Slice = try reference_d.toSliceAlloc(allocator, io); + defer reference_h.free(allocator); + + for (backends) |backend| { + if (!backend.isAvailable(platform)) continue; + + const metadata: flash_attention.Metadata = .init(.fromBackend(backend)); + const parameters: flash_attention.Parameters = .init(.fromBackend(backend)); + const exe = try platform.compileFn( + allocator, + io, + AttentionCall.forward, + .{ tensors.q, tensors.k, tensors.v, tensors.token_index, .{ .flash_attention = metadata }, .{ .flash_attention = parameters }, opts, mesh, partitioning }, + .{ + .program_name = try std.fmt.allocPrint(arena, "attention_{t}", .{backend}), + .meshes = meshes, + }, + ); + defer exe.deinit(); + + var metadata_d = try metadata.initBuffer(io, platform); + defer flash_attention.Metadata.deinitBuffer(&metadata_d); + + var output_d = try exe.eval(allocator, io, .{ q, k, v, token_index, metadata_d }); + defer output_d.deinit(); + try output_d.await(io); + const output_h = try output_d.toSliceAlloc(allocator, io); + defer output_h.free(allocator); + + errdefer std.log.err( + \\ Attention test failed, {0t} output doesn't match reference. + \\ - reference: {1d} + \\ - {0t}: {2d} + , .{ backend, reference_h, output_h }); + try zml.testing.expectClose(io, reference_h, output_h, .{ + .absolute_tolerance = 5e-3, + .relative_tolerance = 1e-2, + .epsilon_relative = 1e-3, + .minimum_close_fraction = 0.99, + }); + } +} + +fn attentionTestInput(shape: zml.Shape, seed: u64) !zml.Buffer { + const host = try zml.Slice.alloc(std.testing.allocator, shape); + defer host.free(std.testing.allocator); + var prng: std.Random.DefaultPrng = .init(seed); + const random = prng.random(); + switch (shape.dtype()) { + inline .bf16, .f16, .f32 => |dtype| { + for (host.items(dtype.toZigType())) |*value| { + const x = random.float(f32) * 4 - 2; + value.* = if (dtype == .bf16) .fromF32(x) else @floatCast(x); + } + }, + else => unreachable, + } + return zml.Buffer.fromSlice(std.testing.io, zml.testing.env(), host); +} + +/// Independent f64 oracle: the SDPA reference uses fast dot precision, so its +/// f32 result alone cannot establish the accuracy of the kernel's f32 arithmetic. +fn checkAttentionF32(q: zml.Slice, k: zml.Slice, v: zml.Slice, positions: []const u32, output: zml.Slice) !void { + const Layout = struct { + b: usize, + s: usize, + h: usize, + d: usize, + + fn stride(shape: zml.Shape, tag: anytype) usize { + const axis = shape.hasTag(tag) orelse return 0; + var result: usize = 1; + for (shape.dims()[axis + 1 ..]) |dim| result *= @intCast(dim); + return result; + } + + fn init(shape: zml.Shape) @This() { + return .{ .b = stride(shape, .b), .s = if (shape.hasTag(.q) != null) stride(shape, .q) else stride(shape, .k), .h = stride(shape, .h), .d = stride(shape, .hd) }; + } + + fn offset(self: @This(), b: usize, s: usize, h: usize, d: usize) usize { + return b * self.b + s * self.s + h * self.h + d * self.d; + } + }; + const ql = Layout.init(q.shape); + const kl = Layout.init(k.shape); + const vl = Layout.init(v.shape); + const batch: usize = if (q.shape.hasTag(.b) != null) @intCast(q.shape.dim(.b)) else 1; + const queries: usize = @intCast(q.shape.dim(.q)); + const heads: usize = @intCast(q.shape.dim(.h)); + const dim: usize = @intCast(q.shape.dim(.hd)); + const group: usize = @intCast(@divExact(q.shape.dim(.h), k.shape.dim(.h))); + const scores = try std.testing.allocator.alloc(f64, @intCast(k.shape.dim(.k))); + defer std.testing.allocator.free(scores); + const scale = 1.0 / @sqrt(@as(f64, @floatFromInt(dim))); + for (0..batch) |b| { + const start: usize = positions[if (positions.len == 1) 0 else b]; + for (0..queries) |i| { + for (0..heads) |h| { + var maximum: f64 = -std.math.inf(f64); + const active = scores[0 .. start + i + 1]; + for (active, 0..) |*score, j| { + var dot: f64 = 0; + for (0..dim) |d| dot += @as(f64, q.items(f32)[ql.offset(b, i, h, d)]) * @as(f64, k.items(f32)[kl.offset(b, j, h / group, d)]); + score.* = dot * scale; + maximum = @max(maximum, score.*); + } + var sum: f64 = 0; + for (active) |*score| { + score.* = @exp(score.* - maximum); + sum += score.*; + } + for (0..dim) |d| { + var expected: f64 = 0; + for (active, 0..) |weight, j| expected += weight * @as(f64, v.items(f32)[vl.offset(b, j, h / group, d)]); + expected /= sum; + const actual = output.items(f32)[ql.offset(b, i, h, d)]; + try std.testing.expectApproxEqAbs(expected, @as(f64, actual), 2e-5 + 2e-5 * @abs(expected)); + } + } + } + } +} From b73de969ebe10521f8643a930566e218329f581c Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 9 Nov 2026 19:24:37 +0100 Subject: [PATCH 860/881] [Modularization] zml: break cycle in `zml/kernels/paged_attention` (#1039) --- zml/BUILD.bazel | 1 + zml/kernels/paged_attention/cute/cute.zig | 6 +-- zml/kernels/paged_attention/cute/fixtures.zig | 2 + .../paged_attention/cute/sm10x/sm10x.zig | 3 +- .../paged_attention/cute/sm120/sm120.zig | 3 +- zml/kernels/paged_attention/metal/metal.zig | 3 +- .../paged_attention/mosaic_tpu/mosaic_tpu.zig | 2 +- zml/kernels/paged_attention/options.zig | 36 +++++++++++++ .../paged_attention/paged_attention.zig | 54 ++----------------- .../paged_attention/stablehlo/stablehlo.zig | 20 ++++--- zml/kernels/paged_attention/tcl/tcl.zig | 18 +++---- zml/kernels/paged_attention/triton/triton.zig | 4 +- 12 files changed, 74 insertions(+), 78 deletions(-) create mode 100644 zml/kernels/paged_attention/options.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 4d58fc77..e4ce12eb 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -92,6 +92,7 @@ zig_library( "kernels/flash_attention/tests.zig", "kernels/moe/moe.zig", "kernels/moe/tests.zig", + "kernels/paged_attention/options.zig", "kernels/paged_attention/paged_attention.zig", "logo.zig", "mem.zig", diff --git a/zml/kernels/paged_attention/cute/cute.zig b/zml/kernels/paged_attention/cute/cute.zig index ea4f36ab..ef78cc49 100644 --- a/zml/kernels/paged_attention/cute/cute.zig +++ b/zml/kernels/paged_attention/cute/cute.zig @@ -1,10 +1,10 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const paged = @import("../paged_attention.zig"); +pub const AttentionOptions = @import("../options.zig").AttentionOptions; +const kernel = @import("attention.zig"); const sm10x = @import("sm10x/sm10x.zig"); const sm120 = @import("sm120/sm120.zig"); -const kernel = @import("attention.zig"); pub const Options = struct { batch_size: usize, @@ -64,7 +64,7 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major > 8 or (cc.major == 8 and cc.minor >= 9); } -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: AttentionOptions) zml.Tensor { stdx.debug.assert(isAvailable(zml.Compiler.current().platform), "CuTe paged attention requires SM89 or newer", .{}); stdx.debug.assert(q.dtype() == k.dtype() and q.dtype() == v.dtype() and (q.dtype() == .bf16 or q.dtype() == .f16), "CuTe paged attention requires matching BF16 or FP16 Q/K/V", .{}); stdx.debug.assert(q.dim(.hd) == 32 or q.dim(.hd) == 64 or q.dim(.hd) == 128 or q.dim(.hd) == 256 or q.dim(.hd) == 512, "CuTe paged attention supports head dimensions 32, 64, 128, 256 and 512", .{}); diff --git a/zml/kernels/paged_attention/cute/fixtures.zig b/zml/kernels/paged_attention/cute/fixtures.zig index aafb8bf1..c0815d1d 100644 --- a/zml/kernels/paged_attention/cute/fixtures.zig +++ b/zml/kernels/paged_attention/cute/fixtures.zig @@ -1,5 +1,7 @@ const std = @import("std"); + const zml = @import("zml"); + const cases = @import("cases.zig"); pub const Fixture = struct { diff --git a/zml/kernels/paged_attention/cute/sm10x/sm10x.zig b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig index af068bad..afc4298e 100644 --- a/zml/kernels/paged_attention/cute/sm10x/sm10x.zig +++ b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig @@ -2,7 +2,6 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../../../../zml.zig"); -const paged = @import("../../paged_attention.zig"); const cute = @import("../cute.zig"); const kernel = @import("attention.zig"); @@ -19,7 +18,7 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major == 10; } -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: cute.AttentionOptions) zml.Tensor { const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); diff --git a/zml/kernels/paged_attention/cute/sm120/sm120.zig b/zml/kernels/paged_attention/cute/sm120/sm120.zig index 292e4a9f..6645abf1 100644 --- a/zml/kernels/paged_attention/cute/sm120/sm120.zig +++ b/zml/kernels/paged_attention/cute/sm120/sm120.zig @@ -1,7 +1,6 @@ const stdx = @import("stdx"); const zml = @import("../../../../zml.zig"); -const paged = @import("../../paged_attention.zig"); const cute = @import("../cute.zig"); const kernel = @import("attention.zig"); @@ -18,7 +17,7 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major == 12; } -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: paged.AttentionOptions) zml.Tensor { +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: cute.AttentionOptions) zml.Tensor { const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); diff --git a/zml/kernels/paged_attention/metal/metal.zig b/zml/kernels/paged_attention/metal/metal.zig index e2d68398..e8144e3b 100644 --- a/zml/kernels/paged_attention/metal/metal.zig +++ b/zml/kernels/paged_attention/metal/metal.zig @@ -3,8 +3,7 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); - -const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; +const AttentionOptions = @import("../options.zig").AttentionOptions; const paged_attn = @import("paged_attn.zig"); const log = std.log.scoped(.@"zml/attention/metal"); diff --git a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig index 42e1861f..84716379 100644 --- a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig @@ -5,7 +5,7 @@ const ragged_paged = @import("platforms/tpu/ragged_paged"); const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; +const AttentionOptions = @import("../options.zig").AttentionOptions; const ragged_attention = @import("ragged_attention.zig"); const log = std.log.scoped(.tpu_attention); diff --git a/zml/kernels/paged_attention/options.zig b/zml/kernels/paged_attention/options.zig new file mode 100644 index 00000000..35ad4049 --- /dev/null +++ b/zml/kernels/paged_attention/options.zig @@ -0,0 +1,36 @@ +const Tensor = @import("../../tensor.zig").Tensor; + +pub const AttentionMask = union(enum) { + standard: struct { + /// Restrict attention to keys at or before the query position. + causal: bool = true, + /// Optional left window size, including the current token. + /// Non-causal attention also includes every later valid key. + sliding_window: ?u32 = null, + }, + /// Attend to keys at specific positions. + indices: Tensor, + + /// Attend to every valid key in the sequence. + pub const none: AttentionMask = .{ .standard = .{ .causal = false } }; + + pub fn isCausal(mask: AttentionMask) bool { + return switch (mask) { + .standard => |options| options.causal, + .indices => false, + }; + } + + pub fn slidingWindow(mask: AttentionMask) ?u32 { + return switch (mask) { + .standard => |options| options.sliding_window, + .indices => null, + }; + } +}; + +pub const AttentionOptions = struct { + mask: AttentionMask = .{ .standard = .{} }, + scale: ?f32 = null, + sink: ?Tensor = null, +}; diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index 81f7ec3e..92e77888 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -3,6 +3,7 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../../zml.zig"); +pub const AttentionOptions = @import("options.zig").AttentionOptions; pub const cute = @import("cute/cute.zig"); const metal = @import("metal/metal.zig"); const tpu = @import("mosaic_tpu/mosaic_tpu.zig"); @@ -151,41 +152,6 @@ pub const Parameters = union(Backend) { } }; -pub const AttentionMask = union(enum) { - standard: struct { - /// Restrict attention to keys at or before the query position. - causal: bool = true, - /// Optional left window size, including the current token. - /// Non-causal attention also includes every later valid key. - sliding_window: ?u32 = null, - }, - /// Attend to keys at specific positions. - indices: zml.Tensor, - - /// Attend to every valid key in the sequence. - pub const none: AttentionMask = .{ .standard = .{ .causal = false } }; - - pub fn isCausal(mask: AttentionMask) bool { - return switch (mask) { - .standard => |options| options.causal, - .indices => false, - }; - } - - pub fn slidingWindow(mask: AttentionMask) ?u32 { - return switch (mask) { - .standard => |options| options.sliding_window, - .indices => null, - }; - } -}; - -pub const AttentionOptions = struct { - mask: AttentionMask = .{ .standard = .{} }, - scale: ?f32 = null, - sink: ?zml.Tensor = null, -}; - pub const KvCache = union(enum) { split: struct { k: zml.Tensor, @@ -232,19 +198,6 @@ pub const KvCache = union(enum) { const offset = slot_mapping.remainder(.scalar(page_size, slot_mapping.dtype())); return .{ page_index, offset }; } - - pub fn getPage( - self: KvCache, - page_index: zml.Tensor, - ) [2]zml.Tensor { - return switch (self) { - .split => |split| .{ - split.k.slice(split.k.axis(.page), .dynSingle(page_index)), - split.v.slice(split.k.axis(.page), .dynSingle(page_index)), - }, - .dense => @panic("TODO"), - }; - } }; pub const Partitioning = zml.ops.ManualPartitioning(Attention.forward); @@ -280,7 +233,10 @@ const Attention = struct { .split => |split| metal.pagedAttention(metal_parameters, args.q, split.k, split.v, args.opts), .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), }, - .stablehlo => |params| stablehlo.pagedAttention(params, args.q, args.kv_cache, args.opts), + .stablehlo => |params| switch (args.kv_cache) { + .split => |split| stablehlo.pagedAttention(params, args.q, split.k, split.v, args.opts), + .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + }, .tcl => |params| switch (args.kv_cache) { .split => |split| tcl.paged.pagedAttention(params, args.q, split.k, split.v, args.opts), .dense => std.debug.panic("TCL paged attention takes separate K and V pages", .{}), diff --git a/zml/kernels/paged_attention/stablehlo/stablehlo.zig b/zml/kernels/paged_attention/stablehlo/stablehlo.zig index 7ee4df03..55ee01d5 100644 --- a/zml/kernels/paged_attention/stablehlo/stablehlo.zig +++ b/zml/kernels/paged_attention/stablehlo/stablehlo.zig @@ -1,12 +1,12 @@ const zml = @import("../../../zml.zig"); -const KvCache = @import("../paged_attention.zig").KvCache; -const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; +const AttentionOptions = @import("../options.zig").AttentionOptions; const triton = @import("../triton/triton.zig"); pub fn pagedAttention( parameters: triton.paged.Parameters, q: zml.Tensor, - kv_cache: KvCache, + k: zml.Tensor, + v: zml.Tensor, opts: AttentionOptions, ) zml.Tensor { switch (opts.mask) { @@ -14,12 +14,12 @@ pub fn pagedAttention( .indices => @panic("Sparse attention mask is not supported on StableHLO"), } - const page_size = kv_cache.split.k.dim(.k_chunk); + const page_size = k.dim(.k_chunk); const padded_q = q.pad(0, .{ .b = zml.Tensor.Pad{ .high = page_size } }); const final_state = zml.ops.@"while"( AttentionLoop, - .{ .q = padded_q, .kv_cache = kv_cache, .parameters = parameters, .opts = opts }, + .{ .q = padded_q, .k = k, .v = v, .parameters = parameters, .opts = opts }, .{ // i32 to match triton conventions .slot_id = .scalar(0, .i32), @@ -38,7 +38,8 @@ pub fn pagedAttention( const AttentionLoop = struct { q: zml.Tensor, - kv_cache: KvCache, + k: zml.Tensor, + v: zml.Tensor, parameters: triton.paged.Parameters, opts: AttentionOptions, @@ -66,7 +67,7 @@ const AttentionLoop = struct { const query_start = self.parameters.query_start_len.slice(0, .dynSingle(state_.seq_id)); const query_end = self.parameters.query_start_len.slice(0, .dynSingle(state_.seq_id.addConstant(1))); const seq_num_queries_: zml.Tensor = .asScalar(.sub(query_end, query_start)); - const page_size_: u32 = @intCast(self.kv_cache.split.k.dim(.k_chunk)); + const page_size_: u32 = @intCast(self.k.dim(.k_chunk)); return zml.ops.@"if"( struct { @@ -134,7 +135,10 @@ const AttentionLoop = struct { .{ .b, .p }, &.{ .dynSingle(state.seq_id), .dynSingle(state.k_page_idx) }, ); - const active_k, const active_v = self.kv_cache.getPage(active_k_page_id); + + const page_axis = self.k.axis(.page); + const active_k = self.k.slice(page_axis, .dynSingle(active_k_page_id)); + const active_v = self.v.slice(page_axis, .dynSingle(active_k_page_id)); const dtype = self.q.dtype(); const attn_mask: zml.Tensor = attn_mask: { diff --git a/zml/kernels/paged_attention/tcl/tcl.zig b/zml/kernels/paged_attention/tcl/tcl.zig index c9993af9..881593aa 100644 --- a/zml/kernels/paged_attention/tcl/tcl.zig +++ b/zml/kernels/paged_attention/tcl/tcl.zig @@ -3,13 +3,13 @@ //! `Repro:` files are in the reproducer repository, platform/furiosa. const std = @import("std"); -const stdx = @import("stdx"); const builder = @import("dsl/tcl/builder"); +const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; -const paged_attention = @import("../paged_attention.zig"); +const AttentionOptions = @import("../options.zig").AttentionOptions; const triton = @import("../triton/triton.zig"); /// WORKAROUND (TCC 2026.5.0b0): flash decode with 8192-key chunks fails to @@ -49,7 +49,7 @@ pub const paged = struct { /// `q` is `{.b = tokens, .hkv, .hg, .hd}`, the caches `{.page, .k_chunk, .hkv, .hd}`. /// Sinks are one logit per query head. - pub fn pagedAttention(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: paged_attention.AttentionOptions) Tensor { + pub fn pagedAttention(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: AttentionOptions) Tensor { if (opts.mask == .indices) @panic("Sparse attention masks are not supported by TCL paged attention"); stdx.debug.assert(q.rank() == 4 and k_cache.rank() == 4 and v_cache.rank() == 4 and k_cache.shape().eqlDims(v_cache.shape()), "TCL attention requires matching rank-four split K/V caches", .{}); stdx.debug.assert(q.dtype().isFloat() and k_cache.dtype().isFloat() and v_cache.dtype().isFloat(), "TCL attention requires floating-point queries and caches", .{}); @@ -76,14 +76,14 @@ pub const paged = struct { q: Tensor, k_cache: Tensor, v_cache: Tensor, - opts: paged_attention.AttentionOptions, + opts: AttentionOptions, fn body(self: Shard, _: zml.Shape) Tensor { return attend(self.parameters, self.q, self.k_cache, self.v_cache, self.opts); } }; - fn attend(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: paged_attention.AttentionOptions) Tensor { + fn attend(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: AttentionOptions) Tensor { if (q.dim(.b) == 0) return Tensor.zeroes(q.shape()); const page = k_cache.dim(.k_chunk); // WORKAROUND (TCC 2026.5.0b0): no DPE tactic for 32-key attention, so pad to @@ -1664,11 +1664,11 @@ fn runCases(comptime max_pages: usize, comptime num_pages: usize, cases: []const const params: paged.Parameters = .init(.{ .batch_size = batch, .max_num_pages = max_pages, .max_seqlen_q = 128, .is_prefill = case.prefill }); const q_t: Tensor = .init(.{ .b = tokens, .hkv = hkv, .hg = g, .hd = d }, dims.dtype); const Mod = struct { - pub fn forward(p: paged.Parameters, q: Tensor, k: Tensor, v: Tensor, opts: paged_attention.AttentionOptions) Tensor { + pub fn forward(p: paged.Parameters, q: Tensor, k: Tensor, v: Tensor, opts: AttentionOptions) Tensor { return paged.pagedAttention(p, q, k, v, opts); } }; - const opts: paged_attention.AttentionOptions = .{ + const opts: AttentionOptions = .{ .mask = .{ .standard = .{ .causal = case.opts.is_causal, .sliding_window = if (case.opts.sliding_window > 0) @intCast(case.opts.sliding_window) else null, @@ -1742,11 +1742,11 @@ fn runCases(comptime max_pages: usize, comptime num_pages: usize, cases: []const .query_start_len = try .fromBytes(io, platform, params.query_start_len.shape(), std.mem.sliceAsBytes(&starts)), }; defer zml.Buffer.deinitAll(paged.Parameters, &bp); - var opts_d: zml.Bufferized(paged_attention.AttentionOptions) = .{ + var opts_d: zml.Bufferized(AttentionOptions) = .{ .mask = .{ .standard = {} }, .sink = if (case.opts.sink) |s| try .fromBytes(io, platform, opts.sink.?.shape(), std.mem.sliceAsBytes(s)) else null, }; - defer zml.Buffer.deinitAll(paged_attention.AttentionOptions, &opts_d); + defer zml.Buffer.deinitAll(AttentionOptions, &opts_d); var result = try exe.eval(allocator, io, .{ bp, bq, bk, bv, opts_d }); defer result.deinit(); const host = try result.toSliceAlloc(allocator, io); diff --git a/zml/kernels/paged_attention/triton/triton.zig b/zml/kernels/paged_attention/triton/triton.zig index 67416285..2a12e012 100644 --- a/zml/kernels/paged_attention/triton/triton.zig +++ b/zml/kernels/paged_attention/triton/triton.zig @@ -4,10 +4,10 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const triton = zml.kernel.triton; -const AttentionOptions = @import("../paged_attention.zig").AttentionOptions; +const AttentionOptions = @import("../options.zig").AttentionOptions; const kernels = @import("unified_attention.zig"); -const kernels_sparse = @import("sparse_unified_attention.zig"); const kernels_oneapi = @import("unified_attention_oneapi.zig"); +const kernels_sparse = @import("sparse_unified_attention.zig"); const log = std.log.scoped(.@"zml/attention/triton"); From 2c5795920211d7143ba6af7a1571d93910a4f62a Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 9 Nov 2026 19:24:38 +0100 Subject: [PATCH 861/881] [Modularization] zml/kernels/moe: break cycle (#1041) `zml/kernels/moe.zig` is not imported by backend implementation anymore. I introduced `options.zig` to contain the shared options. I also introduced `weights.zig` to expose the unpacking helpers. --- zml/BUILD.bazel | 2 + zml/kernels/moe/cute/cute.zig | 6 +- zml/kernels/moe/cute/sm10x/moe.zig | 3 +- .../cutlass_flashinfer/cutlass_flashinfer.zig | 27 ++-- zml/kernels/moe/fly/fly.zig | 12 +- zml/kernels/moe/metal/metal.zig | 14 ++- zml/kernels/moe/moe.zig | 119 +----------------- zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig | 8 +- zml/kernels/moe/options.zig | 117 +++++++++++++++++ zml/kernels/moe/stablehlo/stablehlo.zig | 12 +- zml/kernels/moe/tcl/tcl.zig | 4 +- zml/kernels/moe/tests.zig | 19 +-- zml/kernels/moe/triton/moe.zig | 5 +- zml/kernels/moe/triton/triton.zig | 25 ++-- zml/kernels/moe/weights.zig | 10 ++ 15 files changed, 212 insertions(+), 171 deletions(-) create mode 100644 zml/kernels/moe/options.zig create mode 100644 zml/kernels/moe/weights.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index e4ce12eb..3c0ace0b 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -91,7 +91,9 @@ zig_library( "kernels/flash_attention/flash_attention.zig", "kernels/flash_attention/tests.zig", "kernels/moe/moe.zig", + "kernels/moe/options.zig", "kernels/moe/tests.zig", + "kernels/moe/weights.zig", "kernels/paged_attention/options.zig", "kernels/paged_attention/paged_attention.zig", "logo.zig", diff --git a/zml/kernels/moe/cute/cute.zig b/zml/kernels/moe/cute/cute.zig index 708ea757..14f1b090 100644 --- a/zml/kernels/moe/cute/cute.zig +++ b/zml/kernels/moe/cute/cute.zig @@ -3,6 +3,8 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); +const Options = @import("../options.zig").Options; +const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; pub const kernels = @import("sm10x/moe.zig"); // `moe.zig`'s `refAllDecls` reaches this file but not its imports, so the @@ -24,7 +26,7 @@ pub fn isNvfp4Supported(platform: *const zml.Platform) bool { return cc.major == 10 and cc.minor == 3; } -pub fn validateOptions(opts: zml.moe.Options) void { +pub fn validateOptions(opts: Options) void { stdx.debug.assert(opts.activation == .swiglu, "cute backend only accepts swiglu activation, got {}", .{opts.activation}); stdx.debug.assert(opts.activation.swiglu.bias == null, "cute backend requires swiglu bias to be null", .{}); stdx.debug.assert(opts.activation.swiglu.scale == null, "cute backend requires swiglu scale to be null", .{}); @@ -41,7 +43,7 @@ pub fn fusedExperts( routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, - options: zml.moe.Options, + options: Options, ) zml.Tensor { validateOptions(options); stdx.debug.assert(input.dtype() == .bf16, "cute backend only supports bf16 inputs, got {}", .{input.dtype()}); diff --git a/zml/kernels/moe/cute/sm10x/moe.zig b/zml/kernels/moe/cute/sm10x/moe.zig index 479e6516..a2eb1792 100644 --- a/zml/kernels/moe/cute/sm10x/moe.zig +++ b/zml/kernels/moe/cute/sm10x/moe.zig @@ -12,6 +12,7 @@ pub const mxfp4_persistent = @import("persistent_mxfp4.zig"); pub const nvfp4 = @import("nvfp4.zig"); pub const nvfp4_persistent = @import("persistent_nvfp4.zig"); const reduce = @import("reduce.zig"); +const RoutingWeightPlacement = @import("../../options.zig").RoutingWeightPlacement; const triton_kernels = @import("../../triton/moe.zig"); pub const Mxfp4Inputs = struct { @@ -252,7 +253,7 @@ pub const Nvfp4Inputs = struct { pub const Nvfp4Options = struct { swiglu_limit: ?f32 = null, - routing_weight_placement: zml.moe.RoutingWeightPlacement = .after_down, + routing_weight_placement: RoutingWeightPlacement = .after_down, /// GEMM tile N; null selects it from the batch size. tile_n: ?i64 = null, }; diff --git a/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig b/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig index e2c31c44..118593a8 100644 --- a/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig +++ b/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig @@ -1,9 +1,14 @@ const std = @import("std"); -const stdx = @import("stdx"); const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); const platforms = @import("platforms"); +const stdx = @import("stdx"); + const zml = @import("../../../zml.zig"); +const max_num_devices = zml.platform.Platform.MAX_NUM_DEVICES; +const GlobalOptions = @import("../options.zig").Options; +const Activation = @import("../options.zig").Activation; +const unpackedWeight = @import("../weights.zig").unpackedWeight; const log = std.log.scoped(.@"zml/moe/cutlass_flashinfer"); @@ -13,14 +18,14 @@ pub const Options = struct { /// Device used at graph-construction time to select the architecture /// library and size the XLA-owned scratch buffer. workspace_query_device: i32, - activation: zml.moe.Activation, + activation: Activation, enable_pdl: bool, gemm1_tactic: i32, /// GEMM2 uses FlashInfer's absolute tactic index. Query tacticCounts() to /// obtain the first valid GEMM2 index. gemm2_tactic: i32, - pub fn fromOptions(opts: zml.moe.Options) Options { + pub fn fromGlobalOptions(opts: GlobalOptions) Options { return .{ .workspace_query_device = 0, .activation = opts.activation, @@ -31,7 +36,7 @@ pub const Options = struct { } }; -pub fn validateOptions(opts: zml.moe.Options) void { +pub fn validateOptions(opts: GlobalOptions) void { stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); stdx.debug.assert(opts.activation == .swiglu, "flashinfer_cutlass backend only accepts swiglu activation, got {}", .{opts.activation}); @@ -79,8 +84,6 @@ const DeviceRunner = struct { runner: *fi_cutlass_moe.Runner, }; -const max_num_devices = zml.platform.Platform.MAX_NUM_DEVICES; - pub const Runners = struct { runners: [max_num_devices]?DeviceRunner = @splat(null), @@ -151,7 +154,7 @@ fn currentRunners() !*Runners { return runners orelse error.RunnersNotLoaded; } -fn cutlassActivation(activation: zml.moe.Activation) i32 { +fn cutlassActivation(activation: Activation) i32 { return switch (activation) { .gelu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GELU, .relu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_RELU, @@ -309,7 +312,7 @@ pub fn fc1BlockScaleShape( num_experts: i64, hidden_size: i64, intermediate_size: i64, - activation: zml.moe.Activation, + activation: Activation, ) zml.Shape { const rows = switch (activation) { .silu, .gelu, .relu => intermediate_size, @@ -505,7 +508,7 @@ pub fn fusedExperts( routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, - opts: zml.moe.Options, + opts: GlobalOptions, ) zml.Tensor { if (comptime !platforms.isEnabled(.cuda)) { @panic("FlashInfer CUTLASS MoE is only supported on CUDA platforms"); @@ -519,13 +522,13 @@ pub fn fusedExperts( // The runner ignores routes whose local expert id is -1. const topk_weights = topk_ids.cmp(.GE, .scalar(0, .i32)).select(routing.weights, .scalar(-1, routing.weights.dtype())); - const runner_options: Options = .fromOptions(opts); + const runner_options: Options = .fromGlobalOptions(opts); const quant_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; stdx.debug.assert(quant_scheme == .nvfp4, "FlashInfer CUTLASS MoE only supports NVFP4, got {?}", .{quant_scheme}); - const gate_up_weight_unpacked = zml.moe.unpackedWeight(gate_up); - const down_weight_unpacked = zml.moe.unpackedWeight(down); + const gate_up_weight_unpacked = unpackedWeight(gate_up); + const down_weight_unpacked = unpackedWeight(down); // TODO(Corentin): Do error checking on nvfp4 // Also, maybe pass `zml.nn.Linear` directly diff --git a/zml/kernels/moe/fly/fly.zig b/zml/kernels/moe/fly/fly.zig index 59d71dc9..e30292cf 100644 --- a/zml/kernels/moe/fly/fly.zig +++ b/zml/kernels/moe/fly/fly.zig @@ -6,8 +6,12 @@ const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; const DataType = zml.DataType; +const ExpertsLayout = @import("../options.zig").ExpertsLayout; +const ProjectionLayout = @import("../options.zig").ProjectionLayout; +const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; +const Activation = @import("../options.zig").Activation; +const Options = @import("../options.zig").Options; const fly = @import("moe.zig"); -const ProjectionLayout = zml.moe.ProjectionLayout; const log = std.log.scoped(.@"zml/moe/fused_experts"); @@ -20,7 +24,7 @@ pub fn fusedExperts( routing_: zml.moe.LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, - opts: zml.moe.Options, + opts: Options, ) Tensor { const expert_map_ = routing_.expertMap(); const layout: ProjectionLayout = if (gate_up_.quantizationScheme() == .mxfp4) .interleaved else .concatenated; @@ -103,7 +107,7 @@ pub fn fusedExperts( .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), }); - var activated = zml.moe.applyActivation(gate_up_out, opts.activation, layout); + var activated = opts.activation.applyActivation(gate_up_out, layout); if (opts.routing_weight_placement == .before_down) { const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); activated = activated.mul(weights.broad(activated.shape())); @@ -131,7 +135,7 @@ pub fn fusedExperts( } // TODO(Corentin): unused for now -pub fn supports(hidden_states: Tensor, routing: zml.moe.LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: zml.moe.Options) bool { +pub fn supports(hidden_states: Tensor, routing: zml.moe.LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: Options) bool { if (hidden_states.dtype() != .bf16 or gate_up_.quantizationScheme() != .mxfp4 or down_.quantizationScheme() != .mxfp4 or gate_up_.bias != null or down_.bias != null or opts.routing_weight_placement != .before_down) return false; diff --git a/zml/kernels/moe/metal/metal.zig b/zml/kernels/moe/metal/metal.zig index 9d397a46..988a4f51 100644 --- a/zml/kernels/moe/metal/metal.zig +++ b/zml/kernels/moe/metal/metal.zig @@ -1,12 +1,16 @@ const std = @import("std"); + const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; +const Activation = @import("../options.zig").Activation; +const GlobalOptions = @import("../options.zig").Options; +const unpackedWeight = @import("../weights.zig").unpackedWeight; const log = std.log.scoped(.@"zml/moe/metal"); -fn validateOptions(opts: zml.moe.Options) void { +fn validateOptions(opts: GlobalOptions) void { stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); } @@ -67,7 +71,7 @@ pub fn fusedExperts( routing: zml.moe.LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, - opts: zml.moe.Options, + opts: GlobalOptions, ) zml.Tensor { validateOptions(opts); stdx.debug.assert(gate_up_.bias == null, "metal backend doesn't support gate_up bias", .{}); @@ -79,8 +83,8 @@ pub fn fusedExperts( const down_scales: ?zml.Tensor = if (down_.quantization) |q| q.scales else null; const down_global_scale: ?zml.Tensor = if (down_.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; - const gate_up = zml.moe.unpackedWeight(gate_up_); - const down = zml.moe.unpackedWeight(down_); + const gate_up = unpackedWeight(gate_up_); + const down = unpackedWeight(down_); stdx.debug.assert(gate_up.dtype() == down.dtype(), "metal backend expected gate_up and down data type to match, got {} and {}", .{ gate_up.dtype(), down.dtype() }); const mode = quantMode(gate_up.dtype()); switch (mode) { @@ -138,7 +142,7 @@ pub fn fusedExperts( zml.Shape.init(.{ .r = num_routes, .out = gate_up.dim(.dout) }, act_dtype), mode, ), gate_up_global_scale, expert_ids); - const activated = zml.moe.applyActivation(gate_up_out, opts.activation, .concatenated).convert(act_dtype); + const activated = opts.activation.applyActivation(gate_up_out, .concatenated).convert(act_dtype); const down_out = applyDownGlobalScale(moeGemm( activated, diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index 7ad85c20..b3ca9814 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -9,6 +9,8 @@ pub const cutlass_flashinfer = @import("cutlass_flashinfer/cutlass_flashinfer.zi pub const fly = @import("fly/fly.zig"); pub const metal = @import("metal/metal.zig"); pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); +const ExpertsLayout = @import("options.zig").ExpertsLayout; +const Options = @import("options.zig").Options; pub const stablehlo = @import("stablehlo/stablehlo.zig"); pub const tcl = @import("tcl/tcl.zig"); pub const triton = @import("triton/triton.zig"); @@ -20,60 +22,6 @@ test { _ = @import("tests.zig"); } -pub const ProjectionLayout = enum { concatenated, interleaved }; -pub const RoutingWeightPlacement = enum { before_down, after_down }; - -/// How a backend expects the expert weights to be stored -/// The packer writes this layout and forwardMoe reads it -pub const ExpertsLayout = struct { - /// Whether the two projections occupy contiguous halves or alternate. - gate_up: ProjectionLayout, - /// Backend specific transformation of the weights and scales. - packing: Packing, - - pub const Packing = enum { - /// Experts stacked as stored in the checkpoint. - plain, - /// Block scales swizzled to the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). - swizzled_scales, - /// Weights, block scales and global scales in the FlashInfer CUTLASS NVFP4 layout. - flashinfer_nvfp4, - }; -}; - -pub const ActivationKind = enum { - /// Gelu activation function - gelu, - /// ReLU activation function - relu, - /// SiLU activation function - silu, - /// SwiGLU activation function (including clamped/clipped/scaled/biased variants) - swiglu, - /// SwiGLU activation function with step function - swiglu_step, - /// GeGlu activation function - geglu, - /// GeGlu activation function with tanh - geglu_tanh, -}; - -pub const Activation = union(ActivationKind) { - gelu: void, - relu: void, - silu: void, - swiglu: struct { - limit: ?f32 = null, - scale: ?f32 = null, - bias: ?f32 = null, - }, - swiglu_step: struct { - limit: ?f32 = null, - }, - geglu: void, - geglu_tanh: void, -}; - pub const Backend = enum { stablehlo, cute, @@ -210,14 +158,6 @@ pub const LocalRouting = struct { } }; -pub const Options = struct { - activation: Activation, - /// Quantize activations for Triton FP8 GEMMs; false keeps BF16 activations. - quantize_input: bool, - /// Where routing weights are applied; FlashInfer, Mosaic, Metal and TCL require after_down. - routing_weight_placement: RoutingWeightPlacement, -}; - /// Check the shared token-wise contract for activations and routing tensors. pub fn validateRoutingInputs(input: zml.Tensor, ids: zml.Tensor, weights: zml.Tensor) void { stdx.debug.assert(input.rank() == 2 and ids.rank() == 2 and weights.rank() == 2, "MoE expects input {{.t, .d}} and routing {{.t, .topk}}, got {f}, {f}, {f}", .{ input.shape(), ids.shape(), weights.shape() }); @@ -375,58 +315,3 @@ pub fn forwardMoe( .out_specs = partitioning.out_specs, }); } - -pub fn unpackedWeight(linear: zml.nn.Linear) zml.Tensor { - const quantization = linear.quantization orelse return linear.weight; - return if (zml.nn.isPackedFp4(quantization.scheme, linear.weight.dtype())) - zml.nn.unpackFp4(linear.weight, linear.tag, linear.tag) - else - linear.weight; -} - -pub fn applyActivation(input: zml.Tensor, activation: zml.moe.Activation, layout: ProjectionLayout) zml.Tensor { - const x = input.convert(.f32); - return switch (activation) { - .gelu => x.gelu(), - .relu => x.relu(), - .silu => x.silu(), - .swiglu, .swiglu_step, .geglu, .geglu_tanh => b: { - const mid = @divFloor(x.dim(.out), 2); - var gate, var up = switch (layout) { - .concatenated => .{ x.slice(.out, .{ .end = mid }), x.slice(.out, .{ .start = mid }) }, - .interleaved => .{ x.slice(.out, .{ .start = 0, .step = 2 }), x.slice(.out, .{ .start = 1, .step = 2 }) }, - }; - - break :b switch (activation) { - .swiglu => |parameters| { - stdx.debug.assert(parameters.scale == null, "triton and fly moe backend don't support swiglu scale", .{}); - - const limit: ?zml.Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; - - // Apply limit on gate and clamp up - gate = if (limit) |l| gate.minimum(l) else gate; - up = if (limit) |l| up.clamp(l.negate(), l) else up; - - // Apply bias - up = if (parameters.bias) |bias| up.addConstant(bias) else up; - - break :b gate.silu().mul(up); - }, - .swiglu_step => |parameters| { - gate = gate.silu(); - - const limit: ?zml.Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; - gate = if (limit) |l| gate.minimum(l) else gate; - up = if (limit) |l| up.clamp(l.negate(), l) else up; - break :b gate.mul(up); - }, - .geglu_tanh => gate.gelu().mul(up), - .geglu => { - log.warn("The geglu activation function was requested but we only support the tanh approximation", .{}); - break :b gate.gelu().mul(up); - }, - else => unreachable, // already treated by the top-level switch - }; - }, - }; -} diff --git a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig index c583b3e4..4b33b585 100644 --- a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig @@ -5,6 +5,8 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; +const Activation = @import("../options.zig").Activation; +const GlobalOptions = @import("../options.zig").Options; pub const gmm_ep = @import("gmm_ep.zig"); const log = std.log.scoped(.@"zml/moe/mosaic_tpu"); @@ -12,7 +14,7 @@ const log = std.log.scoped(.@"zml/moe/mosaic_tpu"); var gate_up_transpose: std.atomic.Value(bool) = .init(false); var down_transpose: std.atomic.Value(bool) = .init(false); -fn validateOptions(opts: zml.moe.Options) void { +fn validateOptions(opts: GlobalOptions) void { stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); } @@ -215,7 +217,7 @@ pub fn fusedExperts( routing: zml.moe.LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, - opts: zml.moe.Options, + opts: GlobalOptions, ) zml.Tensor { validateOptions(opts); const expert_map_ = routing.expertMap(); @@ -276,7 +278,7 @@ pub fn fusedExperts( gate_up_out = gate_up_out.add(bias_per_token); } - const activated = zml.moe.applyActivation(gate_up_out, opts.activation, .concatenated); + const activated = opts.activation.applyActivation(gate_up_out, .concatenated); const aligned_activated = alignSortedRowsByGroup(activated, expert_ids_sorted, group_sizes, tile_m); var down_out = callGmmEp(aligned_activated.rows, down, aligned_activated.group_sizes, hidden.dtype(), .none) diff --git a/zml/kernels/moe/options.zig b/zml/kernels/moe/options.zig new file mode 100644 index 00000000..aa68b06a --- /dev/null +++ b/zml/kernels/moe/options.zig @@ -0,0 +1,117 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const Tensor = @import("../../tensor.zig").Tensor; + +const log = std.log.scoped(.@"zml/moe"); + +pub const Options = struct { + activation: Activation, + /// Quantize activations for Triton FP8 GEMMs; false keeps BF16 activations. + quantize_input: bool, + /// Where routing weights are applied; FlashInfer, Mosaic, Metal and TCL require after_down. + routing_weight_placement: RoutingWeightPlacement, +}; + +pub const ActivationKind = enum { + /// Gelu activation function + gelu, + /// ReLU activation function + relu, + /// SiLU activation function + silu, + /// SwiGLU activation function (including clamped/clipped/scaled/biased variants) + swiglu, + /// SwiGLU activation function with step function + swiglu_step, + /// GeGlu activation function + geglu, + /// GeGlu activation function with tanh + geglu_tanh, +}; + +pub const Activation = union(ActivationKind) { + gelu: void, + relu: void, + silu: void, + swiglu: struct { + limit: ?f32 = null, + scale: ?f32 = null, + bias: ?f32 = null, + }, + swiglu_step: struct { + limit: ?f32 = null, + }, + geglu: void, + geglu_tanh: void, + + pub fn applyActivation(self: Activation, input: Tensor, layout: ProjectionLayout) Tensor { + const x = input.convert(.f32); + return switch (self) { + .gelu => x.gelu(), + .relu => x.relu(), + .silu => x.silu(), + .swiglu, .swiglu_step, .geglu, .geglu_tanh => b: { + const mid = @divFloor(x.dim(.out), 2); + var gate, var up = switch (layout) { + .concatenated => .{ x.slice(.out, .{ .end = mid }), x.slice(.out, .{ .start = mid }) }, + .interleaved => .{ x.slice(.out, .{ .start = 0, .step = 2 }), x.slice(.out, .{ .start = 1, .step = 2 }) }, + }; + + break :b switch (self) { + .swiglu => |parameters| { + stdx.debug.assert(parameters.scale == null, "triton and fly moe backend don't support swiglu scale", .{}); + + const limit: ?Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; + + // Apply limit on gate and clamp up + gate = if (limit) |l| gate.minimum(l) else gate; + up = if (limit) |l| up.clamp(l.negate(), l) else up; + + // Apply bias + up = if (parameters.bias) |bias| up.addConstant(bias) else up; + + break :b gate.silu().mul(up); + }, + .swiglu_step => |parameters| { + gate = gate.silu(); + + const limit: ?Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; + gate = if (limit) |l| gate.minimum(l) else gate; + up = if (limit) |l| up.clamp(l.negate(), l) else up; + break :b gate.mul(up); + }, + .geglu_tanh => gate.gelu().mul(up), + .geglu => { + log.warn("The geglu activation function was requested but we only support the tanh approximation", .{}); + break :b gate.gelu().mul(up); + }, + else => unreachable, // already treated by the top-level switch + }; + }, + }; + } +}; + +pub const RoutingWeightPlacement = enum { before_down, after_down }; + +pub const ProjectionLayout = enum { concatenated, interleaved }; + +/// How a backend expects the expert weights to be stored +/// The packer writes this layout and forwardMoe reads it +pub const ExpertsLayout = struct { + /// Whether the two projections occupy contiguous halves or alternate. + gate_up: ProjectionLayout, + /// Backend specific transformation of the weights and scales. + packing: Packing, + + pub const Packing = enum { + /// Experts stacked as stored in the checkpoint. + plain, + /// Block scales swizzled to the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). + swizzled_scales, + /// Weights, block scales and global scales in the FlashInfer CUTLASS NVFP4 layout. + flashinfer_nvfp4, + }; +}; diff --git a/zml/kernels/moe/stablehlo/stablehlo.zig b/zml/kernels/moe/stablehlo/stablehlo.zig index 25c69c5c..7f84835b 100644 --- a/zml/kernels/moe/stablehlo/stablehlo.zig +++ b/zml/kernels/moe/stablehlo/stablehlo.zig @@ -1,13 +1,17 @@ //! Deliberately simple MoE reference: no routed GEMMs, packing, or custom calls. //! Weights use [expert, output, input] order with concatenated gate/up columns. const std = @import("std"); + const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; +const GlobalOptions = @import("../options.zig").Options; +const Activation = @import("../options.zig").Activation; +const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; -pub fn activate(x: Tensor, activation: zml.moe.Activation) Tensor { +pub fn activate(x: Tensor, activation: Activation) Tensor { return switch (activation) { .silu => x.silu(), .gelu => x.gelu(), @@ -148,13 +152,13 @@ fn project(x: Tensor, linear: zml.nn.Linear, ids: Tensor, opts: Options, dtype: } pub const Options = struct { - activation: zml.moe.Activation, + activation: Activation, quantize_input: bool = false, /// Model fixed backend contracts independently of optional input quantization: /// Metal uses weight-only quantization; specialized MXFP4 GEMMs always use MXFP8 inputs. /// CuTe additionally rounds the intermediate activation to BF16 before quantizing. input_quantization: enum { automatic, none, mxfp8, mxfp8_bf16 } = .automatic, - routing_weight_placement: zml.moe.RoutingWeightPlacement = .after_down, + routing_weight_placement: RoutingWeightPlacement = .after_down, }; /// Negative expert IDs denote nonlocal routes and contribute zero, including bias. @@ -187,7 +191,7 @@ pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: .sum(.topk).squeeze(.topk).convert(input.dtype()); } -pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: zml.moe.Options) Tensor { +pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: GlobalOptions) Tensor { stdx.debug.assert(routing.ids.dim(.topk) > 0 and routing.ids.dim(.topk) <= routing.global_num_experts, "MoE top-k must fit the global expert count", .{}); return reference(input, routing.localIds(), routing.weights, gate_up, down, .{ .activation = opts.activation, diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig index 991280fd..698c4d30 100644 --- a/zml/kernels/moe/tcl/tcl.zig +++ b/zml/kernels/moe/tcl/tcl.zig @@ -12,17 +12,17 @@ const std = @import("std"); const builder = @import("dsl/tcl/builder"); - const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; +const Options = @import("../options.zig").Options; /// `input` is `{.t, .d}`, routing IDs and weights `{.t, .topk}`, `gate_up` `{.expert, .dout = 2 * inter (gate then up), .d}` /// and `down` `{.expert, .d, .dout = inter}`, all bf16 but the ids (i32). /// SwiGLU, routing weights applied after the down projection. -pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: zml.moe.Options) Tensor { +pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: Options) Tensor { const topk_ids = routing.ids; const topk_weights = routing.weights; stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE applies routing weights after the down projection, without FP8 input quantization", .{}); diff --git a/zml/kernels/moe/tests.zig b/zml/kernels/moe/tests.zig index aa0be86f..0abb3462 100644 --- a/zml/kernels/moe/tests.zig +++ b/zml/kernels/moe/tests.zig @@ -5,12 +5,15 @@ const std = @import("std"); const zml = @import("../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; -const Placement = @import("moe.zig").RoutingWeightPlacement; +const ExpertsLayout = @import("options.zig").ExpertsLayout; +const GlobalOptions = @import("options.zig").Options; +const Activation = @import("options.zig").Activation; +const Placement = @import("options.zig").RoutingWeightPlacement; const reference = @import("stablehlo/stablehlo.zig"); const Case = struct { backend: zml.moe.Backend, - activation: zml.moe.Activation = .{ .swiglu = .{} }, + activation: Activation = .{ .swiglu = .{} }, scheme: ?zml.Quantization.Scheme = null, activation_dtype: zml.DataType = .bf16, quantize_input: bool = false, @@ -197,7 +200,7 @@ fn packFp4Bytes(weight: Tensor) Tensor { return weight.reshape(weight.shape().setDim(2, @divExact(weight.dim(2), 2)).append(.{ .nibble = 2 })).bitCast(.u8); } -fn prepareGateUp(tensor: Tensor, layout: zml.moe.ExpertsLayout) Tensor { +fn prepareGateUp(tensor: Tensor, layout: ExpertsLayout) Tensor { return switch (layout.gate_up) { .concatenated => tensor, .interleaved => interleave(tensor), @@ -271,7 +274,7 @@ fn forward(input: Tensor, c: Case) Outputs { const ids = route.mul(Tensor.scalar(5, .i32)).addConstant(2).remainder(Tensor.scalar(@min(c.experts, 6), .i32)); const weights = route.remainder(Tensor.scalar(3, .i32)).convert(.f32).addConstant(1).scale(0.25); - const options: zml.moe.Options = .{ + const options: GlobalOptions = .{ .activation = c.activation, .quantize_input = c.quantize_input, .routing_weight_placement = c.placement, @@ -305,7 +308,7 @@ fn forward(input: Tensor, c: Case) Outputs { return .{ .actual = actual, .expected = expected }; } -const gated_activations = [_]zml.moe.Activation{ +const gated_activations = [_]Activation{ .{ .swiglu = .{} }, .{ .swiglu = .{ .limit = 0.25 } }, .{ .swiglu = .{ .bias = 1 } }, @@ -315,8 +318,8 @@ const gated_activations = [_]zml.moe.Activation{ .geglu, .geglu_tanh, }; -const all_activations = gated_activations ++ [_]zml.moe.Activation{ .relu, .silu, .gelu }; -const clipped_swiglu = [_]zml.moe.Activation{ +const all_activations = gated_activations ++ [_]Activation{ .relu, .silu, .gelu }; +const clipped_swiglu = [_]Activation{ .{ .swiglu = .{ .limit = 0.25 } }, .{ .swiglu = .{ .limit = 7 } }, }; @@ -354,7 +357,7 @@ const Format = struct { const Matrix = struct { formats: []const Format = &.{.{}}, - activations: []const zml.moe.Activation, + activations: []const Activation, quantize_inputs: []const bool = &.{false}, placements: []const Placement = &.{.after_down}, biases: []const bool = &.{false}, diff --git a/zml/kernels/moe/triton/moe.zig b/zml/kernels/moe/triton/moe.zig index 3eb457da..0776a590 100644 --- a/zml/kernels/moe/triton/moe.zig +++ b/zml/kernels/moe/triton/moe.zig @@ -12,6 +12,7 @@ const tri = zml.kernel.triton; const Builder = tri.Builder; const Value = tri.Value; const DType = tri.DType; +const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; const log = std.log.scoped(.@"zml/moe/triton_kernels"); @@ -1026,7 +1027,7 @@ pub const Combine = struct { topk: i64, expert_offset: i64, experts: i64, - routing_weight_placement: zml.moe.RoutingWeightPlacement = .before_down, + routing_weight_placement: RoutingWeightPlacement = .before_down, bn: i64, }; pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_combine", .inputs = &.{ "d", "ids", "perm", "rw" }, .outputs = &.{"y"}, .run = run }); @@ -1092,7 +1093,7 @@ const GemmConfig = struct { // Swiglu clamping. Set to 0 to disable. swiglu_limit: f32, scale_stride: i64, - routing_weight_placement: zml.moe.RoutingWeightPlacement = .before_down, + routing_weight_placement: RoutingWeightPlacement = .before_down, bn: i64, bk: i64, }; diff --git a/zml/kernels/moe/triton/triton.zig b/zml/kernels/moe/triton/triton.zig index 348f5a90..bae76744 100644 --- a/zml/kernels/moe/triton/triton.zig +++ b/zml/kernels/moe/triton/triton.zig @@ -6,10 +6,13 @@ const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; const DataType = zml.DataType; -const kernels = @import("moe.zig"); const toDType = zml.kernel.triton.from; -const ProjectionLayout = zml.moe.ProjectionLayout; -const RoutingWeightPlacement = zml.moe.RoutingWeightPlacement; +const ExpertsLayout = @import("../options.zig").ExpertsLayout; +const ProjectionLayout = @import("../options.zig").ProjectionLayout; +const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; +const Activation = @import("../options.zig").Activation; +const Options = @import("../options.zig").Options; +const kernels = @import("moe.zig"); const log = std.log.scoped(.@"zml/moe/triton"); @@ -38,7 +41,7 @@ pub fn fusedExperts( routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, - options: zml.moe.Options, + options: Options, ) zml.Tensor { // HACK(Corentin): only rely on the scheme of gate_up for now. @@ -59,7 +62,7 @@ pub fn fusedExpertsBase( gate_up_: zml.nn.Linear, down_: zml.nn.Linear, layout: ProjectionLayout, - opts: zml.moe.Options, + opts: Options, ) Tensor { const expert_map_ = routing_.expertMap(); const num_tokens = hidden_states.dim(.t); @@ -141,7 +144,7 @@ pub fn fusedExpertsBase( .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), }); - var activated = zml.moe.applyActivation(gate_up_out, opts.activation, layout); + var activated = opts.activation.applyActivation(gate_up_out, layout); if (opts.routing_weight_placement == .before_down) { const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); activated = activated.mul(weights.broad(activated.shape())); @@ -490,8 +493,8 @@ test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { const io = std.testing.io; const platform = zml.testing.env(); const Local = struct { - fn forward(x: Tensor, layout: ProjectionLayout, activation: zml.moe.Activation) Tensor { - return zml.moe.applyActivation(x, activation, layout); + fn forward(x: Tensor, layout: ProjectionLayout, activation: Activation) Tensor { + return activation.applyActivation(x, layout); } }; const x: Tensor = .init(.{ .token = 1, .out = 6 }, .bf16); @@ -506,7 +509,7 @@ test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { var input = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&values)); defer input.deinit(); for ([_]?f32{ null, 2 }) |threshold| { - const activation: zml.moe.Activation = .{ .swiglu = .{ .limit = threshold } }; + const activation: Activation = .{ .swiglu = .{ .limit = threshold } }; var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, activation }, .{}); defer exe.deinit(); try zml.testing.expectEqualShapes(Shape.init(.{ .token = 1, .out = 3 }, .f32), exe.output_shapes[0]); @@ -794,7 +797,7 @@ pub fn isAvailable(platform: *const zml.Platform) bool { return cc.major == 10; } -pub fn validateOptions(opts: zml.moe.Options) void { +pub fn validateOptions(opts: Options) void { stdx.debug.assert(opts.activation == .swiglu, "triton backend only accepts swiglu activation, got {}", .{opts.activation}); stdx.debug.assert(opts.activation.swiglu.limit != null, "triton backend requires swiglu limit to be set", .{}); stdx.debug.assert(opts.activation.swiglu.bias == null, "triton backend requires swiglu bias to be null", .{}); @@ -899,7 +902,7 @@ pub fn fusedExpertsMxfp4( routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, - options: zml.moe.Options, + options: Options, ) zml.Tensor { validateOptions(options); stdx.debug.assert(input.dtype() == .bf16, "triton backend only supports bf16 inputs, got {}", .{input.dtype()}); diff --git a/zml/kernels/moe/weights.zig b/zml/kernels/moe/weights.zig new file mode 100644 index 00000000..0e288856 --- /dev/null +++ b/zml/kernels/moe/weights.zig @@ -0,0 +1,10 @@ +const nn = @import("../../nn.zig"); +const Tensor = @import("../../tensor.zig").Tensor; + +pub fn unpackedWeight(linear: nn.Linear) Tensor { + const quantization = linear.quantization orelse return linear.weight; + return if (nn.isPackedFp4(quantization.scheme, linear.weight.dtype())) + nn.unpackFp4(linear.weight, linear.tag, linear.tag) + else + linear.weight; +} From b23dbe334ae477bd4887a46ef5eba4b0c9729d69 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Mon, 9 Nov 2026 19:24:38 +0100 Subject: [PATCH 862/881] [Modularization] zml/kernels/moe: break cycle between flashinfer_cutlass and platform (#1042) * Stopped using `zml.ops.CustomCall` because it creates cycles impossible to break. * renamed some `cutlass_flashinfer` to `flashinfer_cutlass` for consistency I didn't replace `CustomCall` because flashinfer_cutlass is the last user and will be removed soon. --- zml/BUILD.bazel | 2 +- .../moe/cutlass_flashinfer/BUILD.bazel | 8 - .../moe/flashinfer_cutlass/BUILD.bazel | 10 + .../moe/flashinfer_cutlass/contract.zig | 40 +++ .../flashinfer_cutlass.zig} | 244 ++---------------- .../moe/flashinfer_cutlass/runtime.zig | 223 ++++++++++++++++ zml/kernels/moe/moe.zig | 12 +- zml/platform.zig | 44 +++- 8 files changed, 336 insertions(+), 247 deletions(-) delete mode 100644 zml/kernels/moe/cutlass_flashinfer/BUILD.bazel create mode 100644 zml/kernels/moe/flashinfer_cutlass/BUILD.bazel create mode 100644 zml/kernels/moe/flashinfer_cutlass/contract.zig rename zml/kernels/moe/{cutlass_flashinfer/cutlass_flashinfer.zig => flashinfer_cutlass/flashinfer_cutlass.zig} (59%) create mode 100644 zml/kernels/moe/flashinfer_cutlass/runtime.zig diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 3c0ace0b..fff3b8a8 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -119,7 +119,7 @@ zig_library( "//zml/kernels/flash_attention/metal", "//zml/kernels/flash_attention/nki", "//zml/kernels/moe/cute", - "//zml/kernels/moe/cutlass_flashinfer", + "//zml/kernels/moe/flashinfer_cutlass", "//zml/kernels/moe/fly", "//zml/kernels/moe/metal", "//zml/kernels/moe/mosaic_tpu", diff --git a/zml/kernels/moe/cutlass_flashinfer/BUILD.bazel b/zml/kernels/moe/cutlass_flashinfer/BUILD.bazel deleted file mode 100644 index 54b8604c..00000000 --- a/zml/kernels/moe/cutlass_flashinfer/BUILD.bazel +++ /dev/null @@ -1,8 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "cutlass_flashinfer", - srcs = [ - "cutlass_flashinfer.zig", - ], -) diff --git a/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel b/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel new file mode 100644 index 00000000..c9c5d9a6 --- /dev/null +++ b/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel @@ -0,0 +1,10 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "flashinfer_cutlass", + srcs = [ + "contract.zig", + "flashinfer_cutlass.zig", + "runtime.zig", + ], +) diff --git a/zml/kernels/moe/flashinfer_cutlass/contract.zig b/zml/kernels/moe/flashinfer_cutlass/contract.zig new file mode 100644 index 00000000..b4b77eaa --- /dev/null +++ b/zml/kernels/moe/flashinfer_cutlass/contract.zig @@ -0,0 +1,40 @@ +pub fn Inputs(comptime T: type) type { + return struct { + hidden_states: T, + fc1_weights: T, + fc2_weights: T, + topk_weights: T, + topk_ids: T, + fc1_act_global: T, + fc1_weight_block: T, + fc1_global: T, + fc2_act_global: T, + fc2_weight_block: T, + fc2_global: T, + }; +} + +pub fn Outputs(comptime T: type) type { + return struct { + output: T, + workspace: T, + }; +} + +pub const Attributes = struct { + runners: u64 = 0, + num_tokens: i64, + hidden_size: i64, + intermediate_size: i64, + num_experts: i32, + top_k: i32, + activation: i32, + enable_pdl: bool, + fc1_act_per_expert: bool, + fc2_act_per_expert: bool, + gemm1_tactic: i32, + gemm2_tactic: i32, +}; + +pub const compute_on_host: bool = false; +pub const name = "flashinfer_cutlass_nvfp4_routed_moe"; diff --git a/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig similarity index 59% rename from zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig rename to zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig index 118593a8..d1a0e1b5 100644 --- a/zml/kernels/moe/cutlass_flashinfer/cutlass_flashinfer.zig +++ b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig @@ -5,12 +5,13 @@ const platforms = @import("platforms"); const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const max_num_devices = zml.platform.Platform.MAX_NUM_DEVICES; const GlobalOptions = @import("../options.zig").Options; const Activation = @import("../options.zig").Activation; const unpackedWeight = @import("../weights.zig").unpackedWeight; +const contract = @import("contract.zig"); +const runtime = @import("runtime.zig"); -const log = std.log.scoped(.@"zml/moe/cutlass_flashinfer"); +const log = std.log.scoped(.@"zml/moe/flashinfer_cutlass"); pub const auto_tactic: i32 = -1; @@ -45,112 +46,15 @@ pub fn validateOptions(opts: GlobalOptions) void { stdx.debug.assert(opts.activation.swiglu.scale == null, "flashinfer_cutlass backend requires swiglu scale to be null", .{}); } -const Input = struct { - hidden_states: zml.Tensor, - fc1_weights: zml.Tensor, - fc2_weights: zml.Tensor, - topk_weights: zml.Tensor, - topk_ids: zml.Tensor, - fc1_act_global: zml.Tensor, - fc1_weight_block: zml.Tensor, - fc1_global: zml.Tensor, - fc2_act_global: zml.Tensor, - fc2_weight_block: zml.Tensor, - fc2_global: zml.Tensor, -}; +const Inputs = contract.Inputs(zml.Tensor); -const Output = struct { - output: zml.Shape, - workspace: zml.Shape, -}; +const Outputs = contract.Outputs(zml.Shape); -const Attributes = struct { - runners: u64 = 0, - num_tokens: i64, - hidden_size: i64, - intermediate_size: i64, - num_experts: i32, - top_k: i32, - activation: i32, - enable_pdl: bool, - fc1_act_per_expert: bool, - fc2_act_per_expert: bool, - gemm1_tactic: i32, - gemm2_tactic: i32, -}; +const Attributes = contract.Attributes; -const DeviceRunner = struct { - api: *fi_cutlass_moe.Api, - runner: *fi_cutlass_moe.Runner, -}; - -pub const Runners = struct { - runners: [max_num_devices]?DeviceRunner = @splat(null), - - pub fn deinit(self: *Runners) void { - for (self.runners) |device_runner| { - if (device_runner) |runner| { - _ = runner.api.runnerDestroy(runner.runner); - } - } - } - - fn ensureRunner(self: *Runners, device: i32) !DeviceRunner { - if (device < 0 or device >= max_num_devices) return error.UnsupportedDevice; - - const index: usize = @intCast(device); - if (self.runners[index]) |runner| return runner; - - const api = try fi_cutlass_moe.apiForDevice(device); - const options = runnerOptions(device); - var runner: ?*fi_cutlass_moe.Runner = null; - try checkStatus(api, api.runnerCreate(&options, &runner)); - const result: DeviceRunner = .{ - .api = api, - .runner = runner orelse return error.RunnerInitializationFailed, - }; - self.runners[index] = result; - return result; - } -}; - -fn checkStatus(api: *const fi_cutlass_moe.Api, status: fi_cutlass_moe.Status) !void { - if (status == fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_SUCCESS) return; - if (api.lastError()) |message| { - log.err("FlashInfer CUTLASS MoE failed (status {d}): {s}", .{ - status, - std.mem.span(message), - }); - } else { - log.err("FlashInfer CUTLASS MoE failed with status {d}", .{status}); - } - return switch (status) { - fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_INVALID_ARGUMENT => error.InvalidArgument, - fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_UNSUPPORTED => error.UnsupportedArchitecture, - fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_CUDA_ERROR => error.Cuda, - else => error.FlashinferCutlassMoe, - }; -} - -fn runnerOptions(device: i32) fi_cutlass_moe.RunnerOptions { - var options = std.mem.zeroes(fi_cutlass_moe.RunnerOptions); - options.struct_size = @sizeOf(fi_cutlass_moe.RunnerOptions); - options.activation_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; - options.weight_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_PACKED_FP4; - options.output_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; - options.device = device; - options.use_fused_finalize = 1; - return options; -} - -fn runnersFromAttributes(attributes: Attributes) !*Runners { - if (attributes.runners == 0) return error.RunnersNotLoaded; - return @ptrFromInt(attributes.runners); -} - -fn currentRunners() !*Runners { +fn currentRunners() !*runtime.Runners { const platform = zml.Compiler.current().platform; - const runners = platform.state.cuda.fi_cutlass_moe_runners; + const runners = platform.state.cuda.flashinfer_cutlass_moe_runners; return runners orelse error.RunnersNotLoaded; } @@ -166,80 +70,22 @@ fn cutlassActivation(activation: Activation) i32 { }; } -fn makeContext(attributes: Attributes) fi_cutlass_moe.Context { - var context = std.mem.zeroes(fi_cutlass_moe.Context); - context.struct_size = @sizeOf(fi_cutlass_moe.Context); - context.num_tokens = attributes.num_tokens; - context.hidden_size = attributes.hidden_size; - context.intermediate_size = attributes.intermediate_size; - context.num_experts = attributes.num_experts; - context.num_experts_on_rank = attributes.num_experts; - context.top_k = attributes.top_k; - context.tp_size = 1; - context.ep_size = 1; - context.activation = @intCast(attributes.activation); - context.enable_pdl = @intFromBool(attributes.enable_pdl); - context.swizzled_input_sf = 1; - return context; -} +pub fn call(input_tensors: Inputs, output_shapes: Outputs, attributes: Attributes) zml.ops.ShapeToTensor(Outputs) { + const opts: zml.ops.CustomCallOptions = .{ + .has_side_effect = false, + .output_operand_aliases = comptime zml.ops.customCallOutputOperandAliases(Inputs, Outputs, null), + .compute_on_host = contract.compute_on_host, + }; -fn ffiCallNvfp4( - call_frame: *zml.pjrt.ffi.CallFrame, - input: zml.pjrtx.TensorToCustomCallBuffer(Input), - output: zml.pjrtx.ShapeToCustomCallBuffer(Output), - attributes: Attributes, -) !?*zml.pjrt.ffi.Error { - const device = try call_frame.ctx.getDeviceOrdinal(call_frame.api); - const runners = try runnersFromAttributes(attributes); - const deviceRunner = try runners.ensureRunner(device); - const context = makeContext(attributes); - - var io = std.mem.zeroes(fi_cutlass_moe.Io); - io.struct_size = @sizeOf(fi_cutlass_moe.Io); - io.input = input.hidden_states.ptr; - io.token_selected_experts = @ptrCast(@alignCast(input.topk_ids.ptr)); - io.token_final_scales = @ptrCast(@alignCast(input.topk_weights.ptr)); - io.fc1_expert_weights = input.fc1_weights.ptr; - io.fc2_expert_weights = input.fc2_weights.ptr; - io.quant_scales[0] = input.fc1_act_global.ptr; - io.quant_scales[1] = input.fc1_weight_block.ptr; - io.quant_scales[2] = input.fc1_global.ptr; - io.quant_scales[3] = input.fc2_act_global.ptr; - io.quant_scales[4] = input.fc2_weight_block.ptr; - io.quant_scales[5] = input.fc2_global.ptr; - io.quant_scale_count = 6; - io.quant_scale_per_expert_mask = - (@as(u32, @intFromBool(attributes.fc1_act_per_expert)) << 0) | - (@as(u32, @intFromBool(attributes.fc2_act_per_expert)) << 3); - io.output = output.output.ptr; - - var workspace = std.mem.zeroes(fi_cutlass_moe.Workspace); - workspace.struct_size = @sizeOf(fi_cutlass_moe.Workspace); - workspace.data = output.workspace.ptr; - workspace.data_bytes = output.workspace.shape.byteSize(); - - try checkStatus( - deviceRunner.api, - deviceRunner.api.run( - deviceRunner.runner, - &context, - &io, - &workspace, - @ptrCast(call_frame.api.stream(call_frame.ctx)), - attributes.gemm1_tactic, - attributes.gemm2_tactic, - ), + return zml.ops.typedCustomCall( + contract.name, + opts, + input_tensors, + output_shapes, + attributes, ); - return null; } -const routedNvfp4Call = zml.ops.CustomCall(Input, Output, Attributes, ffiCallNvfp4, .{ - .name = "flashinfer_cutlass_nvfp4_routed_moe", - // Expert mesh is owned by forwardMoe outer manual computation. - .sharding_aware = false, - .has_side_effect = false, -}); - fn computeCapability(platform: *const zml.Platform) !u16 { const cc = zml.platform.cuda.computeCapability(platform) orelse return error.UnsupportedPlatform; return switch (cc.sm()) { @@ -248,40 +94,15 @@ fn computeCapability(platform: *const zml.Platform) !u16 { }; } -pub fn load( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, -) !void { - if (comptime platforms.isEnabled(.cuda)) { - var cuda_state = &platform.state.cuda; - if (cuda_state.fi_cutlass_moe_runners != null) return; - try fi_cutlass_moe.load(allocator, io, try computeCapability(platform)); - const runners = try allocator.create(Runners); - runners.* = .{}; - cuda_state.fi_cutlass_moe_runners = runners; - return; - } - return error.UnsupportedPlatform; -} - -pub fn register(platform: *const zml.Platform) !void { - if (comptime platforms.isEnabled(.cuda)) { - try routedNvfp4Call.register(platform); - return; - } - return error.UnsupportedPlatform; -} - pub fn isAvailable(platform: *const zml.Platform) bool { if (platform.target != .cuda) return false; - if (platform.state.cuda.fi_cutlass_moe_runners == null) return false; + if (platform.state.cuda.flashinfer_cutlass_moe_runners == null) return false; _ = computeCapability(platform) catch return false; return true; } pub fn isNvfp4Supported(platform: *const zml.Platform) bool { - if (platform.state.cuda.fi_cutlass_moe_runners == null) return false; + if (platform.state.cuda.flashinfer_cutlass_moe_runners == null) return false; const compute_capability = computeCapability(platform) catch return false; return switch (compute_capability) { 100, 103, 120 => true, @@ -289,21 +110,6 @@ pub fn isNvfp4Supported(platform: *const zml.Platform) bool { }; } -pub fn tacticCounts( - platform: *const zml.Platform, - device: i32, -) !struct { gemm1: i32, gemm2: i32 } { - const runners = platform.state.cuda.fi_cutlass_moe_runners orelse return error.RunnersNotLoaded; - const deviceRunner = try runners.ensureRunner(device); - var gemm1: i32 = 0; - var gemm2: i32 = 0; - try checkStatus( - deviceRunner.api, - deviceRunner.api.getTacticCounts(deviceRunner.runner, &gemm1, &gemm2), - ); - return .{ .gemm1 = gemm1, .gemm2 = gemm2 }; -} - fn roundUp(value: i64, alignment: i64) i64 { return @divTrunc(value + alignment - 1, alignment) * alignment; } @@ -468,10 +274,10 @@ pub fn fusedExpertsNvfp4( attributes.runners = @intFromPtr(runners); const deviceRunner = runners.ensureRunner(options.workspace_query_device) catch |e| stdx.debug.panic("Failed to ensure runner: {}", .{e}); - const context = makeContext(attributes); + const context = runtime.makeContext(attributes); var requirements = std.mem.zeroes(fi_cutlass_moe.WorkspaceRequirements); requirements.struct_size = @sizeOf(fi_cutlass_moe.WorkspaceRequirements); - checkStatus( + runtime.checkStatus( deviceRunner.api, deviceRunner.api.getWorkspaceRequirements( deviceRunner.runner, @@ -480,7 +286,7 @@ pub fn fusedExpertsNvfp4( ), ) catch |e| stdx.debug.panic("Failed to get workspace requirements: {}", .{e}); - const result = routedNvfp4Call.call( + const result = call( .{ .hidden_states = hidden_states, .fc1_weights = fc1_weights, diff --git a/zml/kernels/moe/flashinfer_cutlass/runtime.zig b/zml/kernels/moe/flashinfer_cutlass/runtime.zig new file mode 100644 index 00000000..028ef892 --- /dev/null +++ b/zml/kernels/moe/flashinfer_cutlass/runtime.zig @@ -0,0 +1,223 @@ +const std = @import("std"); + +const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); +const pjrt = @import("pjrt"); + +const DataType = @import("../../../dtype.zig").DataType; +const contract = @import("contract.zig"); + +const log = std.log.scoped(.@"zml/kernels/moe/flashinfer_cutlass/runtime"); + +// TODO(Corentin): This was coming from Platform but we are now downstream. +// Find a way to solve that. +// Maybe the max_num_devices could be its own module because it only depends on Target. +const max_num_devices = 64; + +// TODO(Corentin): replace that with Shape when its dependencies are untangled and +// it doesn't depend on the platform anymore. +fn count(buffer: *const pjrt.ffi.Buffer) usize { + if (buffer.rank == 0) return @intCast(buffer.dims()[0]); + + var c: usize = 1; + for (buffer.dims()) |dim| c = c * @as(usize, @intCast(dim)); + + return c; +} + +fn byteSize(buffer: *const pjrt.ffi.Buffer) usize { + const dt: DataType = switch (buffer.dtype) { + .invalid => @panic("Found an invalid pjrt buffer"), + .token => @panic("Found an token pjrt buffer"), + inline else => |tag| @field(DataType, @tagName(tag)), + }; + return std.math.divCeil(usize, count(buffer) * dt.bitSizeOf(), 8) catch unreachable; +} + +pub fn handler(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { + if (call_frame.registeringHook()) return null; + + // TODO(Corentin): Find a place to abstract that + var input_buffers: contract.Inputs(*const pjrt.ffi.Buffer) = undefined; + inline for (@typeInfo(contract.Inputs(*const pjrt.ffi.Buffer)).@"struct".field_names, 0..) |field_name, i| { + const buf = call_frame.args.buffers()[i]; + @field(input_buffers, field_name) = buf; + } + + var output_buffers: contract.Outputs(*const pjrt.ffi.Buffer) = undefined; + inline for (@typeInfo(contract.Outputs(*const pjrt.ffi.Buffer)).@"struct".field_names, 0..) |field_name, i| { + const buf = call_frame.results.buffers()[i]; + @field(output_buffers, field_name) = buf; + } + + const attributes: contract.Attributes = switch (@typeInfo(contract.Attributes)) { + .void => {}, + .@"struct" => blk: { + var attrs: contract.Attributes = undefined; + inline for (@typeInfo(contract.Attributes).@"struct".field_names, @typeInfo(contract.Attributes).@"struct".field_types) |field_name, Field| { + // TODO: Use getByIndex instead? + const attribute = call_frame.attrs.getByName(.scalar, field_name) orelse + @panic("Attribute not found: " ++ field_name); + @field(attrs, field_name) = attribute.get(Field); + } + break :blk attrs; + }, + else => @compileError("Unsupported attributes type: " ++ @typeName(contract.Attributes)), + }; + + return handlerInner(call_frame, input_buffers, output_buffers, attributes) catch |err| + pjrt.ffi.Error.create(call_frame.api, .unknown, @errorName(err)); +} + +fn handlerInner( + call_frame: *pjrt.ffi.CallFrame, + inputs: contract.Inputs(*const pjrt.ffi.Buffer), + outputs: contract.Outputs(*const pjrt.ffi.Buffer), + attributes: contract.Attributes, +) !?*pjrt.ffi.Error { + const device = try call_frame.ctx.getDeviceOrdinal(call_frame.api); + const runners = try runnersFromAttributes(attributes); + const deviceRunner = try runners.ensureRunner(device); + const context = makeContext(attributes); + + var io = std.mem.zeroes(fi_cutlass_moe.Io); + io.struct_size = @sizeOf(fi_cutlass_moe.Io); + io.input = inputs.hidden_states.data; + io.token_selected_experts = @ptrCast(@alignCast(inputs.topk_ids.data)); + io.token_final_scales = @ptrCast(@alignCast(inputs.topk_weights.data)); + io.fc1_expert_weights = inputs.fc1_weights.data; + io.fc2_expert_weights = inputs.fc2_weights.data; + io.quant_scales[0] = inputs.fc1_act_global.data; + io.quant_scales[1] = inputs.fc1_weight_block.data; + io.quant_scales[2] = inputs.fc1_global.data; + io.quant_scales[3] = inputs.fc2_act_global.data; + io.quant_scales[4] = inputs.fc2_weight_block.data; + io.quant_scales[5] = inputs.fc2_global.data; + io.quant_scale_count = 6; + io.quant_scale_per_expert_mask = + (@as(u32, @intFromBool(attributes.fc1_act_per_expert)) << 0) | + (@as(u32, @intFromBool(attributes.fc2_act_per_expert)) << 3); + io.output = outputs.output.data; + + var workspace = std.mem.zeroes(fi_cutlass_moe.Workspace); + workspace.struct_size = @sizeOf(fi_cutlass_moe.Workspace); + workspace.data = outputs.workspace.data; + workspace.data_bytes = byteSize(outputs.workspace); + + try checkStatus( + deviceRunner.api, + deviceRunner.api.run( + deviceRunner.runner, + &context, + &io, + &workspace, + @ptrCast(call_frame.api.stream(call_frame.ctx)), + attributes.gemm1_tactic, + attributes.gemm2_tactic, + ), + ); + return null; +} + +pub fn makeContext(attributes: contract.Attributes) fi_cutlass_moe.Context { + var context = std.mem.zeroes(fi_cutlass_moe.Context); + context.struct_size = @sizeOf(fi_cutlass_moe.Context); + context.num_tokens = attributes.num_tokens; + context.hidden_size = attributes.hidden_size; + context.intermediate_size = attributes.intermediate_size; + context.num_experts = attributes.num_experts; + context.num_experts_on_rank = attributes.num_experts; + context.top_k = attributes.top_k; + context.tp_size = 1; + context.ep_size = 1; + context.activation = @intCast(attributes.activation); + context.enable_pdl = @intFromBool(attributes.enable_pdl); + context.swizzled_input_sf = 1; + return context; +} + +pub fn checkStatus(api: *const fi_cutlass_moe.Api, status: fi_cutlass_moe.Status) !void { + if (status == fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_SUCCESS) return; + if (api.lastError()) |message| { + log.err("FlashInfer CUTLASS MoE failed (status {d}): {s}", .{ + status, + std.mem.span(message), + }); + } else { + log.err("FlashInfer CUTLASS MoE failed with status {d}", .{status}); + } + return switch (status) { + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_INVALID_ARGUMENT => error.InvalidArgument, + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_UNSUPPORTED => error.UnsupportedArchitecture, + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_CUDA_ERROR => error.Cuda, + else => error.FlashinferCutlassMoe, + }; +} + +fn runnersFromAttributes(attributes: contract.Attributes) !*Runners { + if (attributes.runners == 0) return error.RunnersNotLoaded; + return @ptrFromInt(attributes.runners); +} + +pub const Runners = struct { + runners: [max_num_devices]?DeviceRunner = @splat(null), + + pub fn deinit(self: *Runners) void { + for (self.runners) |device_runner| { + if (device_runner) |runner| { + _ = runner.api.runnerDestroy(runner.runner); + } + } + } + + pub fn ensureRunner(self: *Runners, device: i32) !DeviceRunner { + if (device < 0 or device >= max_num_devices) return error.UnsupportedDevice; + + const index: usize = @intCast(device); + if (self.runners[index]) |runner| return runner; + + const api = try fi_cutlass_moe.apiForDevice(device); + const options = runnerOptions(device); + var runner: ?*fi_cutlass_moe.Runner = null; + try checkStatus(api, api.runnerCreate(&options, &runner)); + const result: DeviceRunner = .{ + .api = api, + .runner = runner orelse return error.RunnerInitializationFailed, + }; + self.runners[index] = result; + return result; + } +}; + +const DeviceRunner = struct { + api: *fi_cutlass_moe.Api, + runner: *fi_cutlass_moe.Runner, +}; + +fn runnerOptions(device: i32) fi_cutlass_moe.RunnerOptions { + var options = std.mem.zeroes(fi_cutlass_moe.RunnerOptions); + options.struct_size = @sizeOf(fi_cutlass_moe.RunnerOptions); + options.activation_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; + options.weight_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_PACKED_FP4; + options.output_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; + options.device = device; + options.use_fused_finalize = 1; + return options; +} + +pub fn tacticCounts( + runners: *const Runners, + device: i32, +) !struct { gemm1: i32, gemm2: i32 } { + const deviceRunner = try runners.ensureRunner(device); + var gemm1: i32 = 0; + var gemm2: i32 = 0; + try checkStatus( + deviceRunner.api, + deviceRunner.api.getTacticCounts(deviceRunner.runner, &gemm1, &gemm2), + ); + return .{ .gemm1 = gemm1, .gemm2 = gemm2 }; +} + +pub fn load(allocator: std.mem.Allocator, io: std.Io, sm: u16) !void { + try fi_cutlass_moe.load(allocator, io, sm); +} diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index b3ca9814..32bebac0 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -5,11 +5,11 @@ const platforms = @import("platforms"); const zml = @import("../../zml.zig"); const stdx = zml.stdx; pub const cute = @import("cute/cute.zig"); -pub const cutlass_flashinfer = @import("cutlass_flashinfer/cutlass_flashinfer.zig"); +const ExpertsLayout = @import("options.zig").ExpertsLayout; +pub const flashinfer_cutlass = @import("flashinfer_cutlass/flashinfer_cutlass.zig"); pub const fly = @import("fly/fly.zig"); pub const metal = @import("metal/metal.zig"); pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); -const ExpertsLayout = @import("options.zig").ExpertsLayout; const Options = @import("options.zig").Options; pub const stablehlo = @import("stablehlo/stablehlo.zig"); pub const tcl = @import("tcl/tcl.zig"); @@ -45,7 +45,7 @@ pub const Backend = enum { .triton, .nvfp4 => if (cute.isNvfp4Supported(platform)) .cute - else if (cutlass_flashinfer.isNvfp4Supported(platform)) + else if (flashinfer_cutlass.isNvfp4Supported(platform)) .flashinfer_cutlass else error.UnsupportedQuantization, @@ -84,7 +84,7 @@ pub const Backend = enum { return switch (backend) { .stablehlo => true, .cute => cute.isAvailable(platform), - .flashinfer_cutlass => cutlass_flashinfer.isAvailable(platform), + .flashinfer_cutlass => flashinfer_cutlass.isAvailable(platform), .fly => switch (platform.target) { .rocm => zml.platform.rocm.computeCapability(platform) == .gfx942, else => false, @@ -117,7 +117,7 @@ pub const Backend = enum { return switch (backend) { .stablehlo => {}, .cute => {}, - .flashinfer_cutlass => cutlass_flashinfer.register(platform), + .flashinfer_cutlass => flashinfer_cutlass.register(platform), .triton, .fly => {}, .mosaic_tpu => {}, .metal => {}, @@ -262,7 +262,7 @@ pub fn forwardMoe( const output = switch (args.backend) { .stablehlo => stablehlo.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), .cute => cute.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - .flashinfer_cutlass => cutlass_flashinfer.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .flashinfer_cutlass => flashinfer_cutlass.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), .triton => triton.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), .fly => fly.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), .mosaic_tpu => mosaic_tpu.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), diff --git a/zml/platform.zig b/zml/platform.zig index 3b52c08e..f3510a19 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -8,6 +8,8 @@ pub const Target = @import("platforms").Platform; const constants = @import("constants.zig"); const Exe = @import("exe.zig").Exe; +const flashinfer_cutlass_contract = @import("kernels/moe/flashinfer_cutlass/contract.zig"); +const flashinfer_cutlass_runtime = @import("kernels/moe/flashinfer_cutlass/runtime.zig"); const pjrtx = @import("pjrtx.zig"); const profiler_ = @import("profiling/profiler.zig"); const Sharding = @import("Sharding.zig"); @@ -257,12 +259,12 @@ pub const State = union(Target) { furiosa: void, pub const CudaState = struct { - fi_cutlass_moe_runners: ?*zml.moe.cutlass_flashinfer.Runners = null, + flashinfer_cutlass_moe_runners: ?*flashinfer_cutlass_runtime.Runners = null, fn deinit(self: *CudaState) void { - if (self.fi_cutlass_moe_runners) |runners| { + if (self.flashinfer_cutlass_moe_runners) |runners| { runners.deinit(); - self.fi_cutlass_moe_runners = null; + self.flashinfer_cutlass_moe_runners = null; } } }; @@ -381,16 +383,9 @@ pub const Platform = struct { switch (target) { .cuda => { - if (zml.moe.cutlass_flashinfer.load(arena, io, platform)) { - zml.moe.cutlass_flashinfer.register(platform) catch |err| { - log.warn( - "Failed to register FlashInfer CUTLASS MoE custom calls: {}", - .{err}, - ); - }; - } else |err| { - log.warn("Failed to load FlashInfer CUTLASS MoE: {}", .{err}); - } + cuda.initFlashinferCutlass(platform, io) catch |err| { + log.warn("Failed to initialize Flashinfer CUTLASS MoE: {}", .{err}); + }; }, else => {}, } @@ -877,6 +872,29 @@ pub const cuda = struct { } } else null; } + + fn initFlashinferCutlass(platform: *Platform, io: std.Io) !void { + if (comptime !Target.cuda.isEnabled()) return error.UnsupportedPlatform; + const cc = computeCapability(platform) orelse return error.UnsupportedArchitecture; + + const allocator = platform.arena.allocator(); + const runners = try allocator.create(flashinfer_cutlass_runtime.Runners); + errdefer allocator.destroy(runners); + + try flashinfer_cutlass_runtime.load(allocator, io, cc.sm()); + + runners.* = .{}; + errdefer runners.deinit(); + + try platform.registerFfi(.{ + .name = flashinfer_cutlass_contract.name, + .handler = flashinfer_cutlass_runtime.handler, + .traits = .{ .command_buffer_compatible = true }, + .platform_name = if (flashinfer_cutlass_contract.compute_on_host) "Host" else null, + }); + + platform.state.cuda.flashinfer_cutlass_moe_runners = runners; + } }; pub const rocm = struct { From 7b40b1b0f880ff6230c38abac06daf104e3c1f93 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Mon, 9 Nov 2026 20:48:45 +0100 Subject: [PATCH 863/881] zml/io: honor slices and splat in DirectMemoryWriter.drain (#1050) `DirectMemoryWriter.drain` only wrote the first slice of data and ignored the splat count. The `std.Io.Writer` contract is to write every slice in order, then repeat the last one splat times. - A one-byte splat advanced one byte per drain call. - After a partial write, `splatBytesAll` passes an empty first slice followed by the pattern. The old drain returned 0 for that forever, so the fill never finished. `drain` now writes every slice and repeats the last one splat times. --- zml/io.zig | 48 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/zml/io.zig b/zml/io.zig index 1babdcfc..ab8d8e6e 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -1287,20 +1287,52 @@ pub const DirectMemoryWriter = struct { self.shard_progress = states; } - fn drain(w: *std.Io.Writer, data: []const []const u8, _: usize) std.Io.Writer.Error!usize { + fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); - const chunk = data[0]; try self.commitWindow(); + if (self.interface.end == self.interface.buffer.len) return std.Io.Writer.Error.WriteFailed; - const writable = self.interface.buffer.len - self.interface.end; - if (writable == 0) return std.Io.Writer.Error.WriteFailed; + const consumed = consumed: { + var n: usize = 0; + for (data[0 .. data.len - 1]) |bytes| { + const copied = self.writeWindow(bytes); + n += copied; - const n = @min(writable, chunk.len); - @memcpy(self.interface.buffer[self.interface.end..][0..n], chunk[0..n]); + if (copied < bytes.len) break :consumed n; + } + + const pattern = data[data.len - 1]; + switch (pattern.len) { + 0 => {}, + 1 => { + const dest = self.interface.buffer[self.interface.end..]; + const count = @min(dest.len, splat); + @memset(dest[0..count], pattern[0]); + + self.interface.end += count; + n += count; + }, + else => for (0..splat) |_| { + const copied = self.writeWindow(pattern); + n += copied; + + if (copied < pattern.len) break; + }, + } + + break :consumed n; + }; + + try self.commitWindow(); + return consumed; + } + + fn writeWindow(self: *DirectMemoryWriter, bytes: []const u8) usize { + const dest = self.interface.buffer[self.interface.end..]; + const n = @min(dest.len, bytes.len); + @memcpy(dest[0..n], bytes[0..n]); self.interface.end += n; - try self.commitWindow(); - return n; } From b61eede864e2c757367e78341aa138411822150c Mon Sep 17 00:00:00 2001 From: Louis Lechevalier Date: Tue, 10 Nov 2026 14:23:27 +0100 Subject: [PATCH 864/881] zml/kernels: Enable MegaMoe support in llmd (#1032) This PR enable the MegaMoe kernel support in llmd by : - Making the symmetric workspace an arg of CuteOps - Adding cuda.grid_constant arg to the DSL builder, enabling GPU kernels to receive TMA tensormap - Adapting the existing mxfp4 kernel (non mega) to the new shared swizzling `interleaved_8` --- dsl/cute/builder.zig | 806 +++++++++++++++++- zml/kernel.zig | 2 + zml/kernels/moe/cute/cute.zig | 3 +- zml/kernels/moe/cute/sm10x/moe.zig | 11 +- .../moe/cute/sm10x/persistent_mxfp4.zig | 28 +- zml/kernels/moe/fly/fly.zig | 2 +- zml/kernels/moe/moe.zig | 16 +- zml/kernels/moe/options.zig | 21 +- zml/kernels/moe/tests.zig | 39 +- zml/kernels/moe/triton/triton.zig | 9 +- .../paged_attention/paged_attention.zig | 1 + zml/ops.zig | 17 +- zml/platform.zig | 6 + 13 files changed, 888 insertions(+), 73 deletions(-) diff --git a/dsl/cute/builder.zig b/dsl/cute/builder.zig index 926b8419..8f791f80 100644 --- a/dsl/cute/builder.zig +++ b/dsl/cute/builder.zig @@ -16,6 +16,49 @@ const vector = @import("mlir/dialects/vector"); const mlir = @import("mlir"); const stdx = @import("stdx"); +/// `CUtensorMapDataType`. +pub const TensorMapDataType = enum(u32) { + uint8 = 0, + uint16 = 1, + uint32 = 2, + int32 = 3, + uint64 = 4, + int64 = 5, + float16 = 6, + float32 = 7, + float64 = 8, + bfloat16 = 9, + float32_ftz = 10, + tfloat32 = 11, + tfloat32_ftz = 12, + @"16u4_align8b" = 13, + @"16u4_align16b" = 14, + @"16u6_align16b" = 15, +}; + +/// `CUtensorMapSwizzle`. +pub const TensorMapSwizzle = enum(u32) { none = 0, @"32b" = 1, @"64b" = 2, @"128b" = 3, @"128b_atom_32b" = 4 }; +/// `CUtensorMapL2promotion`. +pub const TensorMapL2Promotion = enum(u32) { none = 0, @"64b" = 1, @"128b" = 2, @"256b" = 3 }; +/// `CUtensorMapInterleave`. +pub const TensorMapInterleave = enum(u32) { none = 0, @"16b" = 1, @"32b" = 2 }; +/// `CUtensorMapFloatOOBfill`. +pub const TensorMapOobFill = enum(u32) { none = 0, nan_request_zero_fma = 1 }; + +pub const TensorMapSpec = struct { + data_type: TensorMapDataType, + rank: u32, + global_dims: [5]u64 = @splat(1), + /// Byte strides of dimensions 1..rank-1. + global_strides: [4]u64 = @splat(0), + box_dims: [5]u32 = @splat(1), + element_strides: [5]u32 = @splat(1), + interleave: TensorMapInterleave = .none, + swizzle: TensorMapSwizzle, + l2_promotion: TensorMapL2Promotion = .none, + oob_fill: TensorMapOobFill = .none, +}; + const dtypes = @import("dtype.zig"); pub const DType = dtypes.DType; const isFloatDtype = dtypes.isFloatDtype; @@ -715,6 +758,7 @@ pub const ArgSpec = struct { pub const MlirTypeSpec = struct { type_: *const mlir.Type, grid_constant: bool = false, + cuda_grid_constant: bool = false, }; pub const TensorSpec = struct { @@ -823,6 +867,7 @@ pub const Builder = struct { .mlir_type => .{ .mlir_type = .{ .type_ = inner.type_, .grid_constant = if (@hasField(@TypeOf(inner), "grid_constant")) inner.grid_constant else false, + .cuda_grid_constant = if (@hasField(@TypeOf(inner), "cuda_grid_constant")) inner.cuda_grid_constant else false, } }, } }; } @@ -849,7 +894,7 @@ pub const Builder = struct { return @Struct(.auto, null, &names, &types, &@splat(.{})); } - fn declareArgsLow(self: *Builder, args: []const ArgSpec) FinishError!void { + pub fn declareArgsLow(self: *Builder, args: []const ArgSpec) FinishError!void { std.debug.assert(self.entry_block == null); const ctx = self.ctx; const scratch = self.arena.allocator(); @@ -871,7 +916,7 @@ pub const Builder = struct { var any_arg_attr = false; for (arg_attrs, args) |*attr, a| { attr.* = switch (a.kind) { - .mlir_type => |typed| if (typed.grid_constant) .dict(ctx, &.{cute_nvgpu.gridConstant(ctx)}) else empty_dict, + .mlir_type => |typed| if (typed.cuda_grid_constant) .dict(ctx, &.{.named(ctx, "cuda.grid_constant", .unit(ctx))}) else if (typed.grid_constant) .dict(ctx, &.{cute_nvgpu.gridConstant(ctx)}) else empty_dict, else => empty_dict, }; if (attr.* != empty_dict) any_arg_attr = true; @@ -1083,6 +1128,75 @@ pub const Builder = struct { })); } + /// `!cuda.tensor_map`: the kernel argument type of an explicitly encoded map. + pub fn tensorMapType(self: *Builder) *const mlir.Type { + return (cuda.ops.TensorMapType.get(self.ctx, .{}) catch unreachable).type_(); + } + + /// A host stack array of `values` (u64 -> i64 slots, u32 -> i32 slots): a `cuuint64_t*` / + /// `cuuint32_t*` argument of `cuTensorMapEncodeTiled`. + fn tensorMapHostArray(self: *Builder, comptime T: type, values: []const T) Value { + const ctx = self.ctx; + const elem: *const mlir.Type = if (T == u64) .int(ctx, .i64) else .int(ctx, .i32); + const ptr_type = llvm.pointerType(ctx, llvm.NVPTXAddressSpace.generic); + const array = self.emit(mlir.Operation.make(ctx, "llvm.alloca", .{ + .operands = .{ .flat = &.{self.constant(.i32, @as(i32, @intCast(@max(values.len, 1)))).inner} }, + .results = .{ .flat = &.{ptr_type} }, + .attributes = &.{ .named(ctx, "elem_type", .typeAttr(elem)), .named(ctx, "alignment", .int(ctx, .i64, 8)) }, + .location = self.loc(), + })); + for (values, 0..) |v, i| { + const slot = self.emit(mlir.Operation.make(ctx, "llvm.getelementptr", .{ + .operands = .{ .flat = &.{array.inner} }, + .results = .{ .flat = &.{ptr_type} }, + .attributes = &.{ .named(ctx, "elem_type", .typeAttr(elem)), .named(ctx, "rawConstantIndices", .denseArray(ctx, .i32, &[_]i32{@intCast(i)})) }, + .location = self.loc(), + })); + const value = if (T == u64) self.constant(.i64, @as(i64, @intCast(v))) else self.constant(.i32, @as(i32, @intCast(v))); + self.emitVoid(llvm.store(ctx, value.inner, slot.inner, .{ .alignment = @sizeOf(T) }, self.loc())); + } + return array; + } + + /// Encode a tiled map in a host function and return both the map and CUDA status. + /// The caller supplies the final device address and handles the returned status. + pub fn encodeTensorMap(self: *Builder, spec: TensorMapSpec, address: Value) struct { map: Value, result: Value } { + const ctx = self.ctx; + // The `cuda` dialect's address operand is a `ptr` dialect pointer, which zml does not register. + ctx.setAllowUnregisteredDialects(true); + const addr = self.emit(cuda.cast(ctx, (mlir.Type.parse(self.ctx, "!ptr.ptr<#ptr.generic_space>") catch @panic("invalid tensor-map IR")), address.inner, self.loc())); + const results = self.emitResults(2, cuda.ops.tensor_map_encode_tiled( + ctx, + self.constant(.i32, @as(i32, @intCast(@backingInt(spec.data_type)))).inner, + self.constant(.i32, @as(i32, @intCast(spec.rank))).inner, + addr.inner, + self.tensorMapHostArray(u64, spec.global_dims[0..spec.rank]).inner, + self.tensorMapHostArray(u64, spec.global_strides[0 .. spec.rank - 1]).inner, + self.tensorMapHostArray(u32, spec.box_dims[0..spec.rank]).inner, + self.tensorMapHostArray(u32, spec.element_strides[0..spec.rank]).inner, + self.constant(.i32, @as(i32, @intCast(@backingInt(spec.interleave)))).inner, + self.constant(.i32, @as(i32, @intCast(@backingInt(spec.swizzle)))).inner, + self.constant(.i32, @as(i32, @intCast(@backingInt(spec.l2_promotion)))).inner, + self.constant(.i32, @as(i32, @intCast(@backingInt(spec.oob_fill)))).inner, + cuda.resultType(ctx), + self.tensorMapType(), + self.loc(), + )); + return .{ .map = results[1], .result = results[0] }; + } + + /// Device side: the generic address (i64) of the `!cuda.tensor_map` kernel argument `arg` + /// (`&tensor_map` for a `__grid_constant__` parameter). + pub fn tensorMapAddress(self: *Builder, descriptor: Value) Value { + const ctx = self.ctx; + const llvm_ptr = self.emit(cuda.cast(ctx, llvm.pointerType(ctx, llvm.NVPTXAddressSpace.generic), descriptor.inner, self.loc())); + return self.emit(mlir.Operation.make(ctx, "llvm.ptrtoint", .{ + .operands = .{ .flat = &.{llvm_ptr.inner} }, + .results = .{ .flat = &.{DType.i64.toMlir(ctx)} }, + .location = self.loc(), + })); + } + pub fn cudaResultStatus(self: *Builder, result: Value) Value { return self.emit(cuda.cast(self.ctx, .int(self.ctx, .i32), result.inner, self.loc())); } @@ -1717,20 +1831,27 @@ pub const Builder = struct { /// `projection`, like Python's `proj`, selects the modes of `tile` that /// apply, such as `b.expr(.{ 1, AlgebraToken.all, 1 })`. - pub fn localTile(self: *Builder, input: anytype, tile: anytype, coord: anytype, result_type: *const mlir.Type, projection: ?AlgebraExpr) View { + pub fn localTile(self: *Builder, input: anytype, tile: anytype, coord: anytype, result_type: ?*const mlir.Type, projection: ?AlgebraExpr) View { + if (result_type == null) return self.inferView("cute.local_tile", &.{ self.asValue(input).inner, self.asValue(tile).inner, if (@TypeOf(coord) == View) coord.inner else self.makeCoord(coord) }, if (projection) |p| &.{.named(self.ctx, "proj", self.algebraAttr(cute_ir.CoordAttr, p).attribute())} else &.{}); const coord_value = if (@TypeOf(coord) == View) coord.inner else self.makeCoord(coord); const result = self.emit(cute_ir.local_tile( self.ctx, self.asValue(input).inner, self.asValue(tile).inner, coord_value, - result_type, + result_type.?, .{ .proj = if (projection) |p| self.algebraAttr(cute_ir.CoordAttr, p).attribute() else null }, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } + // A null result type on the algebra operations below requests dialect inference. + fn inferView(self: *Builder, name: []const u8, operands: []const *const mlir.Value, attrs: []const mlir.NamedAttribute) View { + const value = self.emit(mlir.Operation.make(self.ctx, name, .{ .operands = .{ .flat = operands }, .attributes = attrs, .result_type_inference = true, .location = self.loc() })); + return .{ .inner = value.inner, .kernel = self }; + } + pub fn localPartition(self: *Builder, input: anytype, tiler: anytype, index: anytype, result_type: *const mlir.Type) View { const result = self.emit(cute_ir.local_partition( self.ctx, @@ -1743,12 +1864,13 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn slice(self: *Builder, input: anytype, coord: anytype, result_type: *const mlir.Type) View { + pub fn slice(self: *Builder, input: anytype, coord: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.slice", &.{ self.asValue(input).inner, self.makeCoord(coord) }, &.{}); const result = self.emit(cute_ir.slice( self.ctx, self.asValue(input).inner, self.makeCoord(coord), - result_type, + result_type.?, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; @@ -1767,13 +1889,17 @@ pub const Builder = struct { (cute_ir.MemRefType.get(self.ctx, .{ .ptr = memref.ptr_(), .layout = layout_value.type_() }) catch @panic("invalid regrouped memref")).type_() else if (input_type.isA(cute_ir.CoordTensorType)) |coords| (cute_ir.CoordTensorType.get(self.ctx, .{ .arithTuple = coords.arithTuple(), .layout = layout_value.type_() }) catch @panic("invalid regrouped coordinate tensor")).type_() + else if (input_type.isA(cute_nvgpu.SmemDescViewType)) |desc| + (cute_nvgpu.SmemDescViewType.get(self.ctx, .{ .desc = desc.desc(), .layout = layout_value.type_().isA(cute_ir.LayoutType).?.attr() }) catch @panic("invalid regrouped descriptor")).type_() else - std.debug.panic("withLayout: expected memref or coordinate tensor, got {f}", .{input_type}); + std.debug.panic("withLayout: expected memref, coordinate tensor or shared-memory descriptor view, got {f}", .{input_type}); const iterator_type: *const mlir.Type = if (input_type.isA(cute_ir.MemRefType)) |memref| memref.ptr_() else if (input_type.isA(cute_ir.CoordTensorType)) |coords| (cute_ir.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = coords.arithTuple() }) catch @panic("invalid arithmetic tuple iterator")).type_() + else if (input_type.isA(cute_nvgpu.SmemDescViewType)) |desc| + desc.desc() else unreachable; const iterator = self.emit(cute_ir.get_iter(self.ctx, value.inner, iterator_type, self.loc())); @@ -1781,11 +1907,12 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn groupModes(self: *Builder, input: anytype, begin: u32, end: u32, result_type: *const mlir.Type) View { + pub fn groupModes(self: *Builder, input: anytype, begin: u32, end: u32, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.group_modes", &.{self.asValue(input).inner}, &.{ .named(self.ctx, "begin", .int(self.ctx, .i32, begin)), .named(self.ctx, "end", .int(self.ctx, .i32, end)) }); const result = self.emit(cute_ir.group_modes( self.ctx, self.asValue(input).inner, - result_type, + result_type.?, .int(self.ctx, .i32, begin), .int(self.ctx, .i32, end), self.loc(), @@ -1804,12 +1931,13 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn flatDivide(self: *Builder, input: anytype, tiler: anytype, result_type: *const mlir.Type) View { + pub fn flatDivide(self: *Builder, input: anytype, tiler: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.flat_divide", &.{ self.asValue(input).inner, self.asValue(tiler).inner }, &.{}); const result = self.emit(cute_ir.flat_divide( self.ctx, self.asValue(input).inner, self.asValue(tiler).inner, - result_type, + result_type.?, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; @@ -1826,39 +1954,44 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn filterZeros(self: *Builder, input: anytype, target_profile: ?View, result_type: *const mlir.Type) View { + pub fn filterZeros(self: *Builder, input: anytype, target_profile: ?View, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.filter_zeros", if (target_profile) |target| &.{ self.asValue(input).inner, target.inner } else &.{self.asValue(input).inner}, &.{}); const result = self.emit(cute_ir.filter_zeros( self.ctx, self.asValue(input).inner, - result_type, + result_type.?, .{ .target_profile = if (target_profile) |target| target.inner else null }, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub fn getLayout(self: *Builder, input: anytype, result_type: *const mlir.Type) View { - const result = self.emit(cute_ir.get_layout(self.ctx, self.asValue(input).inner, result_type, self.loc())); + pub fn getLayout(self: *Builder, input: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.get_layout", &.{self.asValue(input).inner}, &.{}); + const result = self.emit(cute_ir.get_layout(self.ctx, self.asValue(input).inner, result_type.?, self.loc())); return .{ .inner = result.inner, .kernel = self }; } /// `cute.get_iter`: the iterator of `input`, of type `result_type`. - pub fn getIter(self: *Builder, input: anytype, result_type: *const mlir.Type) View { - const result = self.emit(cute_ir.get_iter(self.ctx, self.asValue(input).inner, result_type, self.loc())); + pub fn getIter(self: *Builder, input: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.get_iter", &.{self.asValue(input).inner}, &.{}); + const result = self.emit(cute_ir.get_iter(self.ctx, self.asValue(input).inner, result_type.?, self.loc())); return .{ .inner = result.inner, .kernel = self }; } - pub fn getShape(self: *Builder, input: anytype, result_type: *const mlir.Type) View { - const result = self.emit(cute_ir.get_shape(self.ctx, self.asValue(input).inner, result_type, self.loc())); + pub fn getShape(self: *Builder, input: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.get_shape", &.{self.asValue(input).inner}, &.{}); + const result = self.emit(cute_ir.get_shape(self.ctx, self.asValue(input).inner, result_type.?, self.loc())); return .{ .inner = result.inner, .kernel = self }; } - pub fn addOffset(self: *Builder, input: anytype, offset: anytype, result_type: *const mlir.Type) View { + pub fn addOffset(self: *Builder, input: anytype, offset: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.add_offset", &.{ self.asValue(input).inner, self.asValue(offset).inner }, &.{}); const result = self.emit(cute_ir.add_offset( self.ctx, self.asValue(input).inner, self.asValue(offset).inner, - result_type, + result_type.?, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; @@ -1895,36 +2028,39 @@ pub const Builder = struct { return .{ .inner = result.inner, .kernel = self }; } - pub fn tiledCopyRetile(self: *Builder, tiled: Atom, input: anytype, result_type: *const mlir.Type) View { + pub fn tiledCopyRetile(self: *Builder, tiled: Atom, input: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.tiled.copy.retile", &.{ tiled.inner, self.asValue(input).inner }, &.{}); const result = self.emit(cute_ir.tiled_copy_retile( self.ctx, tiled.inner, self.asValue(input).inner, - result_type, + result_type.?, self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub fn tiledMmaPartition(self: *Builder, tiled: Atom, input: anytype, coord: anytype, operand_id: u2, result_type: *const mlir.Type) View { + pub fn tiledMmaPartition(self: *Builder, tiled: Atom, input: anytype, coord: anytype, operand_id: u2, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.tiled.mma.partition", &.{ tiled.inner, self.asValue(input).inner, self.makeCoord(coord) }, &.{.named(self.ctx, "operand_id", .int(self.ctx, .i32, operand_id))}); const result = self.emit(cute_ir.tiled_mma_partition( self.ctx, tiled.inner, self.asValue(input).inner, self.makeCoord(coord), - result_type, + result_type.?, .int(self.ctx, .i32, operand_id), self.loc(), )); return .{ .inner = result.inner, .kernel = self }; } - pub fn mmaMakeFragment(self: *Builder, tiled: Atom, input: anytype, operand_id: u2, result_type: *const mlir.Type) View { + pub fn mmaMakeFragment(self: *Builder, tiled: Atom, input: anytype, operand_id: u2, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.mma.make_fragment", &.{ tiled.inner, self.asValue(input).inner }, &.{.named(self.ctx, "operand_id", .int(self.ctx, .i32, operand_id))}); const result = self.emit(cute_ir.mma_make_fragment( self.ctx, tiled.inner, self.asValue(input).inner, - result_type, + result_type.?, .int(self.ctx, .i32, operand_id), self.loc(), )); @@ -2731,8 +2867,8 @@ pub const Builder = struct { /// Arrive and expect `bytes` of asynchronous transactions (SM90+), as a /// TMA producer does before issuing its copies. - pub fn mbarrierArriveExpectTx(self: *Builder, mbar: Value, bytes: u32) void { - self.emitVoid(nvvm.mbarrier_arrive_expect_tx(self.ctx, self.sharedBarrierPtr(mbar).inner, self.constant(.i32, bytes).inner, .cta, self.loc())); + pub fn mbarrierArriveExpectTx(self: *Builder, mbar: Value, bytes: anytype) void { + self.emitVoid(nvvm.mbarrier_arrive_expect_tx(self.ctx, self.sharedBarrierPtr(mbar).inner, self.lift(bytes).to(.i32).inner, .cta, self.loc())); } pub const MBarrierArriveOptions = struct { @@ -4203,3 +4339,615 @@ test "gather4 uses CuTe TMA atom construction, partitioning and copy options" { defer parsed.deinit(); try std.testing.expect(parsed.operation().verify()); } + +/// Low-level CUDA primitives. Prefer typed operations where they preserve the +/// instruction semantics. Assembly fallbacks retain exact memory scopes, rounding, +/// CTA-group/cache modifiers, and tied-register dependencies for asynchronous loads. +const sm100_mma_peer_bit_mask: u32 = 0xFEFFFFFF; + +/// Side-effecting inline assembly with no result. +pub fn asmVoid(b: *Builder, text: []const u8, args: []const Value, constraints: []const u8) void { + const inner = b.arena.allocator().alloc(*const mlir.Value, args.len) catch @panic("OOM"); + for (args, inner) |a, *x| x.* = a.inner; + b.emitVoid(llvm.inline_asm(b.ctx, text, inner, .{ .constraints = constraints, .has_side_effects = true }, b.loc())); +} + +/// Inline assembly with one result of `dtype`. +pub fn asmValue(b: *Builder, text: []const u8, args: []const Value, dtype: DType, constraints: []const u8, side_effects: bool) Value { + const inner = b.arena.allocator().alloc(*const mlir.Value, args.len) catch @panic("OOM"); + for (args, inner) |a, *x| x.* = a.inner; + return b.emit(llvm.inline_asm(b.ctx, text, inner, .{ .result_type = dtype.toMlir(b.ctx), .constraints = constraints, .has_side_effects = side_effects }, b.loc())); +} + +/// Inline assembly with `n` results of `dtype` (an `!llvm.struct` result split with +/// `llvm.extractvalue`, how LLVM represents multi-output asm). +pub fn asmValues(b: *Builder, comptime n: usize, text: []const u8, args: []const Value, dtype: DType, constraints: []const u8, side_effects: bool) [n]Value { + return asmValuesSlice(b, n, text, args, dtype, constraints, side_effects)[0..n].*; +} + +/// Inline assembly with a host-computed number of results of the same dtype. +pub fn asmValuesSlice(b: *Builder, n: usize, text: []const u8, args: []const Value, dtype: DType, constraints: []const u8, side_effects: bool) []Value { + const inner = b.arena.allocator().alloc(*const mlir.Value, args.len) catch @panic("OOM"); + for (args, inner) |a, *x| x.* = a.inner; + const elem = dtype.toMlir(b.ctx); + const struct_type = (mlir.Type.parse(b.ctx, std.fmt.allocPrint(b.arena.allocator(), "!llvm.struct<({s})>", .{structFields(b, n, elem)}) catch @panic("OOM")) catch @panic("invalid inline assembly result type")); + const packed_ = b.emit(llvm.inline_asm(b.ctx, text, inner, .{ .result_type = struct_type, .constraints = constraints, .has_side_effects = side_effects }, b.loc())); + const out = b.arena.allocator().alloc(Value, n) catch @panic("OOM"); + for (out, 0..) |*o, i| { + o.* = b.emit(mlir.Operation.make(b.ctx, "llvm.extractvalue", .{ + .operands = .{ .flat = &.{packed_.inner} }, + .results = .{ .flat = &.{elem} }, + .attributes = &.{.named(b.ctx, "position", .denseArray(b.ctx, .i64, &[_]i64{@intCast(i)}))}, + .location = b.loc(), + })); + } + return out; +} + +pub fn structFields(b: *Builder, n: usize, elem: *const mlir.Type) []const u8 { + var w: std.Io.Writer.Allocating = .init(b.arena.allocator()); + for (0..n) |i| w.writer.print("{s}{f}", .{ if (i == 0) "" else ", ", elem }) catch @panic("OOM"); + return w.written(); +} + +pub fn ldAcq(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.acquire.gpu.global.b32 $0, [$1];", &.{ptr}, .i32, "=r,l,~{memory}", true); +} + +pub fn ldAcqSysU64(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.acquire.sys.global.b64 $0, [$1];", &.{ptr}, .i64, "=l,l,~{memory}", true); +} + +pub fn ldAcqSysS32(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.acquire.sys.global.s32 $0, [$1];", &.{ptr}, .i32, "=r,l,~{memory}", true); +} + +pub fn ldAcqSysU32(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.acquire.sys.global.u32 $0, [$1];", &.{ptr}, .i32, "=r,l,~{memory}", true); +} + +pub fn ldAcqGpuU64(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.acquire.gpu.global.u64 $0, [$1];", &.{ptr}, .i64, "=l,l,~{memory}", true); +} + +pub fn atomicAddU32(b: *Builder, ptr: Value, value: anytype) Value { + return asmValue(b, "atom.global.add.u32 $0, [$1], $2;", &.{ ptr, liftAs(b, value, .i32) }, .i32, "=r,l,r", true); +} + +pub fn atomicAddU64(b: *Builder, ptr: Value, value: Value) Value { + return asmValue(b, "atom.global.add.u64 $0, [$1], $2;", &.{ ptr, value }, .i64, "=l,l,l", true); +} + +pub fn atomicAddSys(b: *Builder, ptr: Value, value: Value) Value { + return asmValue(b, "atom.sys.global.add.u64 $0, [$1], $2;", &.{ ptr, value }, .i64, "=l,l,l", true); +} + +pub fn atomicAddRel(b: *Builder, ptr: Value, value: anytype) Value { + return asmValue(b, "atom.release.gpu.global.add.u32 $0, [$1], $2;", &.{ ptr, liftAs(b, value, .i32) }, .i32, "=r,l,r,~{memory}", true); +} + +pub fn stRelSys(b: *Builder, ptr: Value, value: Value) void { + asmVoid(b, "st.release.sys.global.u64 [$0], $1;", &.{ ptr, value }, "l,l,~{memory}"); +} + +pub fn redAddS32(b: *Builder, ptr: Value, value: anytype) void { + asmVoid(b, "red.gpu.global.add.s32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r"); +} + +pub fn redAddU32(b: *Builder, ptr: Value, value: anytype) void { + asmVoid(b, "red.gpu.global.add.u32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r"); +} + +pub fn redXorRel(b: *Builder, ptr: Value, value: Value) void { + asmVoid(b, "red.release.gpu.global.xor.b64 [$0], $1;", &.{ ptr, value }, "l,l,~{memory}"); +} + +pub fn redAddRel(b: *Builder, ptr: Value, value: anytype) void { + asmVoid(b, "red.release.gpu.global.add.u32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r,~{memory}"); +} + +pub fn redAddRelSys(b: *Builder, ptr: Value, value: anytype) void { + asmVoid(b, "red.release.sys.global.add.s32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r,~{memory}"); +} + +pub fn atomicAddBlockShared(b: *Builder, smem: Value, value: anytype) Value { + return asmValue(b, "atom.shared.cta.add.u32 $0, [$1], $2;", &.{ smem, liftAs(b, value, .i32) }, .i32, "=r,r,r,~{memory}", true); +} + +pub fn fenceAcqRelCta(b: *Builder) void { + asmVoid(b, "fence.acq_rel.cta;", &.{}, "~{memory}"); +} + +pub fn liftAs(b: *Builder, value: anytype, dt: DType) Value { + if (@TypeOf(value) == Value) return value; + return b.constant(dt, value); +} + +/// `cvta`-style widening of an `i32` shared-memory address to the 64-bit operand +/// `__cvta_generic_to_shared` gives DeepGEMM's `"l"` constraints. +pub fn zext64(b: *Builder, v: Value) Value { + if (v.dtype() == .i64) return v; + return b.emit(arith.extui(b.ctx, v.inner, DType.i64.toMlir(b.ctx), b.loc())); +} + +pub fn i16Of(b: *Builder, v: Value) Value { + return if (v.dtype() == .i16) v else b.emit(arith.trunci(b.ctx, v.inner, DType.i16.toMlir(b.ctx), b.loc())); +} + +pub fn cst32(b: *Builder, v: anytype) Value { + return b.constant(.i32, @as(i32, @bitCast(@as(u32, @truncate(@as(u64, @bitCast(@as(i64, v)))))))); +} + +pub fn cst64(b: *Builder, v: u64) Value { + return b.constant(.i64, @as(i64, @bitCast(v))); +} + +pub fn getSmIdx(b: *Builder) Value { + return asmValue(b, "mov.u32 $0, %smid;", &.{}, .i32, "=r", false); +} + +pub fn getLaneIdx(b: *Builder) Value { + return asmValue(b, "mov.u32 $0, %laneid;", &.{}, .i32, "=r", false); +} + +/// `ptx::get_grid_idx`: the 64-bit `%gridid`, unique per launch. +pub fn getGridIdx(b: *Builder) Value { + return asmValue(b, "mov.u64 $0, %gridid;", &.{}, .i64, "=l", true); +} + +pub fn blockRankInCluster(b: *Builder) Value { + return asmValue(b, "mov.u32 $0, %cluster_ctarank;\n", &.{}, .i32, "=r", true); +} + +/// `clock64()` (`mov.u64 %rd, %clock64;`). +pub fn clock64(b: *Builder) Value { + return asmValue(b, "mov.u64 $0, %clock64;", &.{}, .i64, "=l", true); +} + +/// Keep the original election sequence until typed election matches its small-batch performance. +pub fn electOneSync(b: *Builder) Value { + const pred = asmValue(b, + \\{ + \\.reg .b32 %rx; + \\.reg .pred %px; + \\ mov.u32 $0, 0; + \\ elect.sync %rx|%px, $1; + \\@%px mov.s32 $0, 1; + \\} + , &.{cst32(b, 0xFFFFFFFF)}, .i32, "=r,r", true); + return pred.ne(0); +} + +/// `asm volatile("trap;")` (`DG_TRAP_ONLY_DEVICE_ASSERT`, and this port's timeout). +pub fn trap(b: *Builder) void { + asmVoid(b, "trap;", &.{}, "~{memory}"); +} + +/// `ptx::sync_unaligned(num_threads, barrier_idx)`: `barrier.sync id, n` (threads of a warp may +/// reach it at different program points). +pub fn syncUnaligned(b: *Builder, num_threads: anytype, barrier_idx: anytype) void { + asmVoid(b, "barrier.sync $0, $1;", &.{ liftAs(b, barrier_idx, .i32), liftAs(b, num_threads, .i32) }, "r,r,~{memory}"); +} + +/// `cute::cluster_arrive`. +pub fn clusterArrive(b: *Builder) void { + asmVoid(b, "barrier.cluster.arrive.aligned;\n", &.{}, "~{memory}"); +} + +/// `cudaGridDependencySynchronize()` (`griddepcontrol.wait;`, PDL). +pub fn gridDependencySynchronize(b: *Builder) void { + asmVoid(b, "griddepcontrol.wait;", &.{}, "~{memory}"); +} + +/// `cutlass::arch::warpgroup_reg_alloc`. +pub fn warpgroupRegAlloc(b: *Builder, reg_count: u32) void { + asmVoid(b, std.fmt.allocPrint(b.arena.allocator(), "setmaxnreg.inc.sync.aligned.u32 {d};\n", .{reg_count}) catch @panic("OOM"), &.{}, ""); +} + +/// `cutlass::arch::warpgroup_reg_dealloc`. +pub fn warpgroupRegDealloc(b: *Builder, reg_count: u32) void { + asmVoid(b, std.fmt.allocPrint(b.arena.allocator(), "setmaxnreg.dec.sync.aligned.u32 {d};\n", .{reg_count}) catch @panic("OOM"), &.{}, ""); +} + +/// `ptx::ld_volatile(const uint64_t*)`. +pub fn ldVolatileU64(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.volatile.global.b64 $0, [$1];", &.{ptr}, .i64, "=l,l", true); +} + +/// `ptx::ld_volatile(const uint32_t*)`. +pub fn ldVolatileU32(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.volatile.global.b32 $0, [$1];", &.{ptr}, .i32, "=r,l", true); +} + +/// `ptx::ld_shared(const uint32_t*)`. +pub fn ldSharedU32(b: *Builder, smem: Value) Value { + return asmValue(b, "ld.shared.u32 $0, [$1];", &.{zext64(b, smem)}, .i32, "=r,l", true); +} + +/// `ptx::ld_shared(const float2*)`. +pub fn ldSharedF32x2(b: *Builder, smem: Value) [2]Value { + return asmValues(b, 2, "ld.shared.v2.f32 {$0, $1}, [$2];", &.{zext64(b, smem)}, .f32, "=f,=f,l", true); +} + +/// `ptx::ld_shared(const float4*)`. +pub fn ldSharedF32x4(b: *Builder, smem: Value) [4]Value { + return asmValues(b, 4, "ld.shared.v4.f32 {$0, $1, $2, $3}, [$4];", &.{zext64(b, smem)}, .f32, "=f,=f,=f,=f,l", true); +} + +/// `ptx::ld_shared(const uint4*)`. +pub fn ldSharedU32x4(b: *Builder, smem: Value) [4]Value { + return asmValues(b, 4, "ld.shared.v4.u32 {$0, $1, $2, $3}, [$4];", &.{zext64(b, smem)}, .i32, "=r,=r,=r,=r,l", true); +} + +/// `ptx::st_shared(const void*, x, y, z, w)`. +pub fn stSharedU32x4(b: *Builder, smem: Value, w: [4]Value) void { + asmVoid(b, "st.shared.v4.u32 [$0], {$1, $2, $3, $4};", &.{ zext64(b, smem), w[0], w[1], w[2], w[3] }, "l,r,r,r,r"); +} + +/// `ptx::st_shared(const uint32_t*, uint32_t)`. +pub fn stSharedU32(b: *Builder, smem: Value, value: Value) void { + asmVoid(b, "st.shared.u32 [$0], $1;", &.{ zext64(b, smem), value }, "l,r"); +} + +/// `ptx::st_shared(const float2*, float2)`. +pub fn stSharedF32x2(b: *Builder, smem: Value, v: [2]Value) void { + asmVoid(b, "st.shared.v2.f32 [$0], {$1, $2};", &.{ zext64(b, smem), v[0], v[1] }, "l,f,f"); +} + +/// `ptx::st_shared_bulk(smem, num_bytes)`: zero `num_bytes` (a multiple of 8) of shared memory. +pub fn stSharedBulk(b: *Builder, smem: Value, num_bytes: u32) void { + std.debug.assert(num_bytes % 8 == 0); + asmVoid(b, "st.bulk.weak.shared::cta [$0], $1, 0;", &.{ zext64(b, smem), b.constant(.i64, num_bytes) }, "l,l,~{memory}"); +} + +/// `ptx::mapa_shared(ptr, dst_cta_idx)`: this CTA's shared address in cluster CTA `cta`. +pub fn mapaShared(b: *Builder, smem: Value, dst_cta_idx: Value) Value { + return asmValue(b, "mapa.shared::cluster.u32 $0, $1, $2;", &.{ smem, dst_cta_idx }, .i32, "=r,r,r", true); +} + +/// `ptx::st_async_cluster(dst, src, dst_cta_idx, barrier)` for `words.len / 4` 16-byte +/// chunks: `st.async` into CTA `dst_cta_idx`'s shared memory, completing the bytes on its +/// mbarrier at `barrier` (both local addresses, mapped with `mapa`). +pub fn stAsyncCluster(b: *Builder, dst: Value, words: []const Value, dst_cta_idx: Value, barrier: Value) void { + std.debug.assert(words.len % 4 == 0); + const mapped_barrier = mapaShared(b, barrier, dst_cta_idx); + const mapped_dst = mapaShared(b, dst, dst_cta_idx); + var i: usize = 0; + while (i < words.len) : (i += 4) { + asmVoid( + b, + "st.async.shared::cluster.mbarrier::complete_tx::bytes.u32.v4 [$0], {$1, $2, $3, $4}, [$5];", + &.{ mapped_dst.add(@as(i32, @intCast(i * 4))), words[i], words[i + 1], words[i + 2], words[i + 3], mapped_barrier }, + "r,r,r,r,r,r,~{memory}", + ); + } +} + +/// `ptx::SM100_U8x4_STSM_T::copy(src, smem)`: `stmatrix.m16n8.x1.trans.b8`. +pub fn stmatrixU8x4T(b: *Builder, src: Value, smem: Value) void { + asmVoid(b, "stmatrix.sync.aligned.m16n8.x1.trans.shared.b8 [$0], {$1};\n", &.{ zext64(b, smem), src }, "l,r"); +} + +/// `ptx::SM90_U32x4_STSM_T::copy(s0, s1, s2, s3, smem)`: `stmatrix.x4.m8n8.trans.b16`. +pub fn stmatrixU32x4T(b: *Builder, src: [4]Value, smem: Value) void { + asmVoid(b, "stmatrix.sync.aligned.x4.m8n8.shared.b16.trans [$0], {$1, $2, $3, $4};\n", &.{ zext64(b, smem), src[0], src[1], src[2], src[3] }, "l,r,r,r,r"); +} + +/// `ClusterBarrier::arrive()` on this CTA's barrier. +pub fn mbarrierArrive(b: *Builder, bar: Value) void { + asmVoid(b, "{\n\tmbarrier.arrive.shared::cta.b64 _, [$0];\n\t}", &.{bar}, "r,~{memory}"); +} + +/// `ClusterBarrier::arrive(cta_id, pred = true)`: arrive on the barrier at the same offset of +/// cluster CTA `cta_id`. +pub fn mbarrierArriveCluster(b: *Builder, bar: Value, cta_id: Value) void { + asmVoid( + b, + "{\n\t.reg .b32 remAddr32;\n\tmapa.shared::cluster.u32 remAddr32, $0, $1;\n\tmbarrier.arrive.shared::cluster.b64 _, [remAddr32];\n\t}", + &.{ bar, cta_id }, + "r,r,~{memory}", + ); +} + +/// `ClusterTransactionBarrier::arrive_and_expect_tx(bytes, cta_id, pred)`: on cluster CTA +/// `cta_id`'s barrier, only when `pred == 1`. +pub fn mbarrierArriveAndExpectTxCluster(b: *Builder, bar: Value, transaction_bytes: anytype, cta_id: Value, pred: Value) void { + asmVoid( + b, + "{\n\t.reg .pred p;\n\t.reg .b32 remAddr32;\n\tsetp.eq.u32 p, $2, 1;\n\t@p mapa.shared::cluster.u32 remAddr32, $0, $1;\n\t@p mbarrier.arrive.expect_tx.shared::cluster.b64 _, [remAddr32], $3;\n\t}", + &.{ bar, cta_id, pred, liftAs(b, transaction_bytes, .i32) }, + "r,r,r,r,~{memory}", + ); +} + +/// `cute::prefetch_tma_descriptor(desc)` (generic descriptor address). +pub fn prefetchTmaDescriptor(b: *Builder, desc: Value) void { + asmVoid(b, "prefetch.tensormap [$0];", &.{desc}, "l,~{memory}"); +} + +/// `SM100_TMA_2SM_LOAD_2D::copy(desc, mbar, cache_hint, smem, c0, c1)`: issued by both CTAs of +/// the pair; the barrier address is masked so the bytes complete on the leader's barrier. +pub fn tma2SmLoad2d(b: *Builder, desc: Value, mbar: Value, cache_hint: u64, smem: Value, c0: Value, c1: Value) void { + asmVoid( + b, + "cp.async.bulk.tensor.2d.cta_group::2.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$0], [$1, {$3, $4}], [$2], $5;", + &.{ smem, desc, mbar.bitAnd(cst32(b, sm100_mma_peer_bit_mask)), c0, c1, cst64(b, cache_hint) }, + "r,l,r,r,r,l,~{memory}", + ); +} + +/// `SM100_TMA_2SM_LOAD_4D::copy(desc, mbar, cache_hint, smem, c0, c1, c2, c3)`. +pub fn tma2SmLoad4d(b: *Builder, desc: Value, mbar: Value, cache_hint: u64, smem: Value, c: [4]Value) void { + asmVoid( + b, + "cp.async.bulk.tensor.4d.cta_group::2.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$0], [$1, {$3, $4, $5, $6}], [$2], $7;", + &.{ smem, desc, mbar.bitAnd(cst32(b, sm100_mma_peer_bit_mask)), c[0], c[1], c[2], c[3], cst64(b, cache_hint) }, + "r,l,r,r,r,r,r,l,~{memory}", + ); +} + +/// `ptx::tma_load_1d(dst, src, mbar, num_bytes, hint = EVICT_FIRST)`: non-tensor bulk copy of +/// `num_bytes` from a global (possibly peer) address into this CTA's shared memory. +pub fn tmaLoad1d(b: *Builder, dst: Value, src: Value, mbar: Value, num_bytes: anytype, cache_hint: u64) void { + asmVoid( + b, + "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$0], [$1], $2, [$3], $4;\n", + &.{ dst, src, liftAs(b, num_bytes, .i32), mbar, cst64(b, cache_hint) }, + "r,l,r,r,l,~{memory}", + ); +} + +/// `ptx::tma_store_1d(dst, src, num_bytes, hint = EVICT_NORMAL)`. +pub fn tmaStore1d(b: *Builder, dst: Value, src: Value, num_bytes: anytype, cache_hint: u64) void { + asmVoid( + b, + "cp.async.bulk.global.shared::cta.bulk_group.L2::cache_hint [$0], [$1], $2, $3;\n", + &.{ dst, src, liftAs(b, num_bytes, .i32), cst64(b, cache_hint) }, + "l,r,r,l,~{memory}", + ); +} + +/// `cute::SM90_TMA_STORE_2D::copy(desc, smem, c0, c1)`. +pub fn tmaStore2d(b: *Builder, desc: Value, smem: Value, c0: Value, c1: Value) void { + asmVoid(b, "cp.async.bulk.tensor.2d.global.shared::cta.bulk_group [$0, {$2, $3}], [$1];", &.{ desc, smem, c0, c1 }, "l,r,r,r,~{memory}"); +} + +/// `cute::TMEM::Allocator2Sm::allocate(num_columns, dst)`: write the base column to shared `dst`. +pub fn tmemAllocate2Sm(b: *Builder, num_columns: u32, dst: Value) void { + asmVoid(b, "tcgen05.alloc.cta_group::2.sync.aligned.shared::cta.b32 [$0], $1;", &.{ dst, cst32(b, num_columns) }, "r,r,~{memory}"); +} + +/// `cute::TMEM::Allocator2Sm::free(tmem_ptr, num_columns)`. +pub fn tmemFree2Sm(b: *Builder, tmem_ptr: Value, num_columns: u32) void { + asmVoid(b, "{\n\ttcgen05.dealloc.cta_group::2.sync.aligned.b32 $0, $1; \n\t}", &.{ tmem_ptr, cst32(b, num_columns) }, "r,r,~{memory}"); +} + +/// `ptx::tcgen05_before_thread_sync`. +pub fn tcgen05BeforeThreadSync(b: *Builder) void { + asmVoid(b, "tcgen05.fence::before_thread_sync;", &.{}, "~{memory}"); +} + +/// `ptx::tcgen05_after_thread_sync`. +pub fn tcgen05AfterThreadSync(b: *Builder) void { + asmVoid(b, "tcgen05.fence::after_thread_sync;", &.{}, "~{memory}"); +} + +/// `cutlass::arch::fence_view_async_tmem_load` (`tcgen05.wait::ld`). +pub fn fenceViewAsyncTmemLoad(b: *Builder) void { + asmVoid(b, "{\n\ttcgen05.wait::ld.sync.aligned; \n}", &.{}, "~{memory}"); +} + +/// `fence_view_async_tmem_load` with the values of earlier TMEM loads threaded through it, so +/// LLVM cannot move their uses above the wait (the PTX is the same single instruction). +pub fn fenceViewAsyncTmemLoadValues(b: *Builder, comptime n: usize, values: [n]Value) [n]Value { + comptime var constraints: []const u8 = ""; + inline for (0..n) |i| constraints = constraints ++ (if (i == 0) "" else ",") ++ "=r"; + inline for (0..n) |i| constraints = constraints ++ std.fmt.comptimePrint(",{d}", .{i}); + constraints = constraints ++ ",~{memory}"; + return asmValues(b, n, "{\n\ttcgen05.wait::ld.sync.aligned; \n}", &values, .i32, constraints, true); +} + +/// `fenceViewAsyncTmemLoadValues` for a run-time number of values (the epilogue's store blocks +/// hold `STORE_BLOCK_M / ATOM_M * 8` words): the same single `tcgen05.wait::ld`, every value +/// tied through it. +pub fn fenceViewAsyncTmemLoadSlice(b: *Builder, values: []const Value) []Value { + const n = values.len; + const a = b.arena.allocator(); + var w: std.Io.Writer.Allocating = .init(a); + for (0..n) |i| w.writer.print("{s}=r", .{if (i == 0) "" else ","}) catch @panic("OOM"); + for (0..n) |i| w.writer.print(",{d}", .{i}) catch @panic("OOM"); + w.writer.writeAll(",~{memory}") catch @panic("OOM"); + return asmValuesSlice(b, n, "{\n\ttcgen05.wait::ld.sync.aligned; \n}", values, .i32, w.written(), true); +} + +/// `cute::SM100_TMEM_LOAD_16dp256b1x::copy(src, d0, d1, d2, d3)`: lane t of the warp gets +/// rows `base + t/4` (d0, d1) and `base + 8 + t/4` (d2, d3), columns `2(t%4)`, `2(t%4)+1`. +pub fn tmemLoad16dp256b1x(b: *Builder, taddr: Value) [4]Value { + return asmValues(b, 4, "tcgen05.ld.sync.aligned.16x256b.x1.b32{$0, $1, $2, $3},[$4];\n", &.{taddr}, .i32, "=r,=r,=r,=r,r", true); +} + +/// `cute::SM100_UTCCP_4x32dp128bit_2cta::copy(src_desc, dst_taddr)`. +pub fn utccp4x32dp128bit2cta(b: *Builder, src_desc: Value, dst_taddr: Value) void { + asmVoid(b, "tcgen05.cp.cta_group::2.32x128b.warpx4 [$0], $1;", &.{ dst_taddr, src_desc }, "r,l"); +} + +/// `ptx::SM100_MMA_MXF8F6F4_2x1SM_SS::fma(desc_a, desc_b, tmem_c, scale_c, desc, tmem_sfa, +/// tmem_sfb)`: `desc` is the 64-bit runtime descriptor (`instr << 32`), `scale_c` (i32) +/// nonzero accumulates. +pub fn mmaMxf8f6f42x1SmSs(b: *Builder, desc_a: Value, desc_b: Value, tmem_c: Value, scale_c: Value, desc: Value, tmem_sfa: Value, tmem_sfb: Value) void { + const idesc = b.emit(arith.trunci(b.ctx, desc.shrLogical(32).inner, DType.i32.toMlir(b.ctx), b.loc())); + asmVoid( + b, + "{\n\t.reg .pred p;\n\tsetp.ne.b32 p, $4, 0;\n\ttcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale [$0], $1, $2, $3, [$5], [$6], p; \n\t}\n", + &.{ tmem_c, desc_a, desc_b, idesc, scale_c, tmem_sfa, tmem_sfb }, + "r,l,l,r,r,r,r", + ); +} + +/// `cutlass::arch::umma_arrive_multicast_2x1SM(barrier, cta_mask)`: inside `elect_one_sync`, +/// `tcgen05.commit` arriving on `barrier` in every CTA of `cta_mask`. +pub fn ummaArriveMulticast2x1Sm(b: *Builder, bar: Value, cta_mask: u16) void { + var elected = b.openIf(electOneSync(b)); + { + asmVoid( + b, + "{\n\ttcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [$0], $1; \n\t}", + &.{ bar, b.constant(.i16, @as(i16, @bitCast(cta_mask))) }, + "r,h,~{memory}", + ); + elected.close(); + } +} + +pub fn bits32(v: Value) Value { + return if (v.dtype() == .i32) v else v.bitCast(.i32); +} + +pub fn fromBits32(v: Value, dt: DType) Value { + return if (dt == .i32) v else v.bitCast(dt); +} + +/// `__shfl_sync(0xffffffff, v, src_lane)` / `ptx::exchange` (32-bit). +pub fn shflSync(b: *Builder, v: Value, src_lane: anytype) Value { + const r = asmValue(b, "shfl.sync.idx.b32 $0, $1, $2, 31, -1;", &.{ bits32(v), liftAs(b, src_lane, .i32) }, .i32, "=r,r,r", true); + return fromBits32(r, v.dtype()); +} + +/// `__shfl_xor_sync(0xffffffff, v, lane_mask)`. +pub fn shflXorSync(b: *Builder, v: Value, lane_mask: anytype) Value { + const r = asmValue(b, "shfl.sync.bfly.b32 $0, $1, $2, 31, -1;", &.{ bits32(v), liftAs(b, lane_mask, .i32) }, .i32, "=r,r,r", true); + return fromBits32(r, v.dtype()); +} + +/// `__shfl_up_sync(0xffffffff, v, delta)`. +pub fn shflUpSync(b: *Builder, v: Value, delta: anytype) Value { + const r = asmValue(b, "shfl.sync.up.b32 $0, $1, $2, 0, -1;", &.{ bits32(v), liftAs(b, delta, .i32) }, .i32, "=r,r,r", true); + return fromBits32(r, v.dtype()); +} + +/// `__reduce_add_sync(0xffffffff, v)` (u32). +pub fn reduceAddSync(b: *Builder, v: Value) Value { + return asmValue(b, "redux.sync.add.u32 $0, $1, $2;", &.{ v, cst32(b, 0xFFFFFFFF) }, .i32, "=r,r,r", true); +} + +/// `__reduce_min_sync(0xffffffff, v)` (u32). +pub fn reduceMinSync(b: *Builder, v: Value) Value { + return asmValue(b, "redux.sync.min.u32 $0, $1, $2;", &.{ v, cst32(b, 0xFFFFFFFF) }, .i32, "=r,r,r", true); +} + +/// `__all_sync(0xffffffff, pred)` (`pred` i1; returns i1). +pub fn allSync(b: *Builder, pred: Value) Value { + const r = asmValue(b, "{ .reg .pred p, q; setp.ne.u32 p, $1, 0; vote.sync.all.pred q, p, $2; selp.u32 $0, 1, 0, q; }", &.{ pred.to(.i32), cst32(b, 0xFFFFFFFF) }, .i32, "=r,r,r", true); + return r.ne(0); +} + +pub fn popc(b: *Builder, v: Value) Value { + return b.callIntrinsic("llvm.ctpop.i32", &.{v}, DType.i32.toMlir(b.ctx)).?; +} + +/// `__ffs(v)`: 1-based index of the lowest set bit, 0 for 0 (nvcc: `brev` + `bfind.shiftamt`). +pub fn ffs(b: *Builder, v: Value) Value { + return asmValue(b, "{ .reg .b32 r; brev.b32 r, $1; bfind.shiftamt.u32 $0, r; }", &.{v}, .i32, "=r,r", false).add(1); +} + +/// `__fns(mask, base, offset)`: position of the `offset`-th set bit of `mask` from `base`. +pub fn fns(b: *Builder, mask: Value, base: anytype, offset: Value) Value { + return asmValue(b, "fns.b32 $0, $1, $2, $3;", &.{ mask, liftAs(b, base, .i32), offset }, .i32, "=r,r,r,r", false); +} + +/// `fminf(a, b)` (`min.f32`, NaN-ignoring; `kClampAfterSilu`). +pub fn fminf(b: *Builder, x: Value, y: Value) Value { + return asmValue(b, "min.f32 $0, $1, $2;", &.{ x, y }, .f32, "=f,f,f", false); +} + +/// `fabsf(x)` (`abs.f32`). +pub fn fabsf(b: *Builder, x: Value) Value { + return asmValue(b, "abs.f32 $0, $1;", &.{x}, .f32, "=f,f", false); +} + +/// `math::fast_rcp(x)`. +pub fn fastRcp(b: *Builder, x: Value) Value { + return asmValue(b, "rcp.approx.ftz.f32 $0, $1;", &.{x}, .f32, "=f,f", true); +} + +/// `__expf(x)` without `--use_fast_math`: `ex2.approx.f32(x * log2(e))` (non-FTZ). +pub fn expfFast(b: *Builder, x: Value) Value { + return asmValue(b, "{ .reg .f32 t; mul.f32 t, $1, 0f3FB8AA3B; ex2.approx.f32 $0, t; }", &.{x}, .f32, "=f,f", false); +} + +/// `__expf(-x)`: nvcc folds the negation into the constant (`mul.f32 t, x, -log2(e)`). +pub fn expfFastNeg(b: *Builder, x: Value) Value { + return asmValue(b, "{ .reg .f32 t; mul.f32 t, $1, 0fBFB8AA3B; ex2.approx.f32 $0, t; }", &.{x}, .f32, "=f,f", false); +} + +/// `__float22bfloat162_rn({lo, hi})`: bf16x2 word, `lo` in the low half. +pub fn float22bfloat162Rn(b: *Builder, lo: Value, hi: Value) Value { + return asmValue(b, "{ cvt.rn.bf16x2.f32 $0, $2, $1;}\n", &.{ lo, hi }, .i32, "=r,f,f", false); +} + +/// `__hmin2(a, b)` on bf16x2 words. +pub fn hmin2Bf16(b: *Builder, x: Value, y: Value) Value { + return asmValue(b, "{ min.bf16x2 $0,$1,$2;\n}", &.{ x, y }, .i32, "=r,r,r", false); +} + +/// `__hmax2(a, b)` on bf16x2 words. +pub fn hmax2Bf16(b: *Builder, x: Value, y: Value) Value { + return asmValue(b, "{ max.bf16x2 $0,$1,$2;\n}", &.{ x, y }, .i32, "=r,r,r", false); +} + +/// `__bfloat1622float2(word)`: `{low, high}` f32 (exact). +pub fn bfloat1622float2(b: *Builder, word: Value) [2]Value { + return asmValues(b, 2, "{ .reg .b16 l, h; mov.b32 {l, h}, $2; cvt.f32.bf16 $0, l; cvt.f32.bf16 $1, h; }", &.{word}, .f32, "=f,=f,r", false); +} + +/// `ptx::accumulate(float2& a, nv_bfloat162 b)`: `a.x += bf16(b.x)`, `a.y += bf16(b.y)` with +/// `add.rn.f32.bf16` (one rounding, the bf16 widened exactly). +pub fn accumulateBf16x2(b: *Builder, acc: [2]Value, bf16x2: Value) [2]Value { + const lo = i16Of(b, bf16x2); + const hi = i16Of(b, bf16x2.shrLogical(16)); + return .{ + asmValue(b, "add.rn.f32.bf16 $0, $1, $0;\n", &.{ lo, acc[0] }, .f32, "=f,h,0", false), + asmValue(b, "add.rn.f32.bf16 $0, $1, $0;\n", &.{ hi, acc[1] }, .f32, "=f,h,0", false), + }; +} +/// IEEE operations used when staging quantized inputs; no FTZ or contraction. +pub fn divRn(b: *Builder, x: Value, y: Value) Value { + return asmValue(b, "div.rn.f32 $0, $1, $2;", &.{ x, y }, .f32, "=f,f,f", false); +} +pub fn mulRn(b: *Builder, x: Value, y: Value) Value { + return asmValue(b, "mul.rn.f32 $0, $1, $2;", &.{ x, y }, .f32, "=f,f,f", false); +} + +test "canonical primitives preserve aligned barriers and runtime transaction counts" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "compatibility_primitives"); + defer b.deinit(); + _ = try b.declareArgs(.{}); + const bar = b.constant(.i32, 0); + const phase = b.constant(.i32, 0); + const bytes = b.threadIdx().x.add(16); + b.barrierCtaSyncAligned(1, 32); + b.barrierClusterArriveRelaxedAligned(); + b.barrierClusterWaitAligned(); + b.mbarrierInit(bar, 1); + b.fenceMbarrierInit(); + b.mbarrierArriveExpectTx(bar, bytes); + b.mbarrierArriveExpectTx(bar, 16); + b.mbarrierWait(bar, phase, .{ .test_first = false }); + _ = phase.bitXor(1); + b.fenceProxyAsyncShared(); + b.fenceProxyAsyncShared(); + b.cpAsyncBulkCommitGroup(); + b.cpAsyncBulkWaitGroup(0, false); + _ = b.voteBallotSync(bytes.ne(0), null); + _ = b.fmax(b.constant(.f32, 1), b.constant(.f32, 2), .{}); + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try expectContains(ir, "llvm.nvvm.barrier.cta.sync.aligned.count"); + try expectContains(ir, "barrier.cluster.arrive.relaxed.aligned"); + try expectContains(ir, "nvvm.mbarrier.arrive.expect_tx"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} diff --git a/zml/kernel.zig b/zml/kernel.zig index 94d6046a..d85b6a56 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -676,6 +676,7 @@ pub const cute = struct { cfg: ConfigT, zeroed_outputs: []const i32 = &.{}, scalars: []const i64 = &.{}, + symmetric_workspace: ?ops.CuteOps.SymmetricWorkspace = null, output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, }; @@ -707,6 +708,7 @@ pub const cute = struct { .ir = ir, .zeroed_outputs = opts.zeroed_outputs, .scalars = opts.scalars, + .symmetric_workspace = opts.symmetric_workspace, .output_operand_aliases = aliases.constSlice(), }); diff --git a/zml/kernels/moe/cute/cute.zig b/zml/kernels/moe/cute/cute.zig index 14f1b090..bd482093 100644 --- a/zml/kernels/moe/cute/cute.zig +++ b/zml/kernels/moe/cute/cute.zig @@ -37,7 +37,8 @@ pub fn validateOptions(opts: Options) void { /// is the per-expert multiplier dequantizing each GEMM output (it folds in /// the inverse of the input's global scale), and their input scale is the /// multiplier applied to the GEMM's input before NVFP4 quantization. Both -/// expect block scales in the 128x4 tensor-core layout. +/// expect block scales in the 128x4 tensor-core layout. MXFP4 gate/up rows +/// interleave by 8 and its scale atoms are MN-major packed i32 words. pub fn fusedExperts( input: zml.Tensor, routing: zml.moe.LocalRouting, diff --git a/zml/kernels/moe/cute/sm10x/moe.zig b/zml/kernels/moe/cute/sm10x/moe.zig index a2eb1792..f651813f 100644 --- a/zml/kernels/moe/cute/sm10x/moe.zig +++ b/zml/kernels/moe/cute/sm10x/moe.zig @@ -20,11 +20,14 @@ pub const Mxfp4Inputs = struct { x: zml.Tensor, /// `[tokens, topk]` FP32. routing_weights: zml.Tensor, + /// `[experts, 2 * intermediate, hidden / 2]` packed FP4, gate/up rows interleaved by 8. w1: zml.Tensor, - /// `w1` scales in the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). + /// `[experts, hidden / 128, 2 * intermediate]` i32 words of four E8M0 scales, + /// in MN-major 128x4 atoms, with rows interleaved like `w1`. s1: zml.Tensor, + /// `[experts, hidden, intermediate / 2]` packed FP4. w2: zml.Tensor, - /// `w2` scales in the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). + /// `[experts, intermediate / 128, hidden]` i32 words in MN-major 128x4 atoms. s2: zml.Tensor, /// `[tokens, topk]` local expert ids. ids: zml.Tensor, @@ -208,14 +211,14 @@ pub fn forwardMxfp4(tokens: i64, hidden: i64, intermediate: i64, experts: i64, t .weight = a.w1, .schedule = routing.schedule, .input_quant = q1.q, - .weight_scale = a.s1.bitCast(.u8), + .weight_scale = a.s1, .input_scale = q1.s, }, routing.route_inverse, a.routing_weights.reshape(.{routes})); const down = mxfp4_persistent.downRows(down_cfg, .{ .weight = a.w2, .schedule = routing.schedule, .input_quant = q2.q, - .weight_scale = a.s2.bitCast(.u8), + .weight_scale = a.s2, .input_scale = q2.s, }, routing.route_inverse); return reduce.topk(down, routing.route_map orelse routing.schedule, .{ diff --git a/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig b/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig index 21a76436..30062a3a 100644 --- a/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig +++ b/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig @@ -550,7 +550,7 @@ fn buildTiledProgram(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: .weight = .{ .ptr = cute.DType.i8 }, .schedule = .{ .ptr = cute.DType.i32 }, .input_quant = .{ .ptr = cute.DType.f8e4m3fn }, - .weight_scale = .{ .ptr = cute.DType.i8 }, + .weight_scale = .{ .ptr = cute.DType.i32 }, .input_scale = .{ .ptr = cute.DType.i8 }, }; const shape = .{ @@ -1439,11 +1439,10 @@ fn routedLanes(b: *B, comptime tile: Tile, operands: EpilogueOperands, group: V, return .{ .count = count, .route = route, .weight = weight }; } -/// Python `fused_epilogue`: thread `t` owns output row `128 * tile_m + t` -/// of the interleaved projection, so even threads hold a gate and their odd -/// neighbour the matching up value. Each even thread computes one SwiGLU -/// activation per routed row; 32 consecutive activations span two warps, -/// whose partial maxima meet in shared memory. +/// Thread `t` owns output row `128 * tile_m + t` of the interleave-8 +/// projection. Gate and up lanes are paired by XOR 8. Each gate thread +/// computes one SwiGLU activation per routed row; 32 consecutive activations +/// span two warps, whose partial maxima meet in shared memory. /// /// At decode sizes most rows of a group are padding, whose outputs are never /// read. `count` is uniform across the CTA, so whole 8-row chunks of padding @@ -1470,7 +1469,7 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo const tid = b.threadIdx().x; const lane = tid.rem(32); const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); - const is_gate = tid.rem(2).eq(0); + const is_gate = tid.rem(16).lt(8); const first_row = group.mul(operands.group_rows); const f32_type = cute.DType.f32.toMlir(b.ctx); // The TMA C staging buffer is unused by this epilogue. @@ -1497,7 +1496,7 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo if (operands.swiglu_limit) |limit| { gate = gate.minimum(limit); } - var up_value = b.shuffleXor(value, 1).to(.bf16).to(.f32); + var up_value = b.shuffleXor(value, 8).to(.bf16).to(.f32); if (operands.swiglu_limit) |limit| { up_value = up_value.maximum(-limit).minimum(limit); } @@ -1506,7 +1505,7 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo const exp_neg = b.unaryF32("ex2.approx.ftz.f32", gate.mul(-std.math.log2e)); var result = up_value.mul(gate).mul(b.unaryF32("rcp.approx.ftz.f32", exp_neg.add(1.0))); result = result.mul(b.shuffleIdx(routed.weight.?, c)).to(.bf16).to(.f32); - // Odd threads hold no activation and must not raise the maxima. + // Up lanes hold no activation and must not raise the maxima. activation.* = b.select(is_gate, result, b.constant(.f32, 0)); const maximum = b.warpMaxNonNegative(activation.*.abs()); chunk_maximum = b.select(lane.eq(c), maximum, chunk_maximum); @@ -1540,7 +1539,8 @@ fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: Epilo // The next tile reuses the scratch maxima. b.barrierCtaSyncAligned(1, 128); - const activation_index = tile_m.mul(64).add(tid.div(2)); + const activation_row = tid.div(16).mul(8).add(tid.rem(8)); + const activation_index = tile_m.mul(64).add(activation_row); for (0..row_chunks) |row_chunk| { const first = row_chunk * swiglu_chunk; const last = @min(first + swiglu_chunk, live); @@ -1830,7 +1830,13 @@ fn buildHostLaunch(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: E const sf_order = b.makeIntTuple(.{ 1, 0, 2 }); const sfa_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_m), b.arg(host_arg.problem_k), b.arg(host_arg.expert_count) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); const sfb_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_n), b.arg(host_arg.problem_k), b.arg(host_arg.problem_l) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); - const dynamic_sfa_layout = b.tileToShape(sf_atom_layout, sfa_shape, sf_order, b.layoutType(dynamic_sf_layout)); + // Weight scales step M atoms first; activation scales retain K-major order. + const sfa_order = b.makeIntTuple(.{ 0, 1, 2 }); + const sfa_layout = b.layoutSpec( + .{ .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ 1, cute.AlgebraToken.dynamic } }, + .{ .{ .{ 16, 4 }, 512 }, .{ .{ 0, 1 }, sf_stride }, .{ 0, sf_stride } }, + ); + const dynamic_sfa_layout = b.tileToShape(sf_atom_layout, sfa_shape, sfa_order, b.layoutType(sfa_layout)); const dynamic_sfb_layout = b.tileToShape(sf_atom_layout, sfb_shape, sf_order, b.layoutType(dynamic_sf_layout)); const gSFA = b.makeTensorView(b.recastPointer(b.arg(host_arg.weight_scale), .f8e8m0fnu, .gmem, 16), dynamic_sfa_layout); const gSFB = b.makeTensorView(b.recastPointer(b.arg(host_arg.input_scale), .f8e8m0fnu, .gmem, 16), dynamic_sfb_layout); diff --git a/zml/kernels/moe/fly/fly.zig b/zml/kernels/moe/fly/fly.zig index e30292cf..d041a5ab 100644 --- a/zml/kernels/moe/fly/fly.zig +++ b/zml/kernels/moe/fly/fly.zig @@ -27,7 +27,7 @@ pub fn fusedExperts( opts: Options, ) Tensor { const expert_map_ = routing_.expertMap(); - const layout: ProjectionLayout = if (gate_up_.quantizationScheme() == .mxfp4) .interleaved else .concatenated; + const layout: ProjectionLayout = if (gate_up_.quantizationScheme() == .mxfp4) .row_interleaved_1 else .concatenated; const num_tokens = hidden_states.dim(.t); var launch_config = launchConfigForTokens(num_tokens); diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index 32bebac0..33b6da55 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -5,12 +5,16 @@ const platforms = @import("platforms"); const zml = @import("../../zml.zig"); const stdx = zml.stdx; pub const cute = @import("cute/cute.zig"); -const ExpertsLayout = @import("options.zig").ExpertsLayout; +pub const ExpertsLayout = @import("options.zig").ExpertsLayout; pub const flashinfer_cutlass = @import("flashinfer_cutlass/flashinfer_cutlass.zig"); pub const fly = @import("fly/fly.zig"); pub const metal = @import("metal/metal.zig"); pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); -const Options = @import("options.zig").Options; +pub const Options = @import("options.zig").Options; +pub const Activation = @import("options.zig").Activation; +pub const ActivationKind = @import("options.zig").ActivationKind; +pub const ProjectionLayout = @import("options.zig").ProjectionLayout; +pub const RoutingWeightPlacement = @import("options.zig").RoutingWeightPlacement; pub const stablehlo = @import("stablehlo/stablehlo.zig"); pub const tcl = @import("tcl/tcl.zig"); pub const triton = @import("triton/triton.zig"); @@ -103,13 +107,17 @@ pub const Backend = enum { pub fn expertsLayout(backend: Backend, scheme: ?zml.Quantization.Scheme) !ExpertsLayout { return switch (backend) { .stablehlo => .{ .gate_up = .concatenated, .packing = .plain }, - .cute => if (scheme == .mxfp4 or scheme == .nvfp4) .{ .gate_up = .interleaved, .packing = .swizzled_scales } else error.UnsupportedQuantization, + .cute => if (scheme) |s| switch (s) { + .mxfp4 => .{ .gate_up = .row_interleaved_8, .packing = .mxfp4_interleaved8 }, + .nvfp4 => .{ .gate_up = .row_interleaved_1, .packing = .swizzled_scales }, + else => error.UnsupportedQuantization, + } else error.UnsupportedQuantization, .flashinfer_cutlass => if (scheme) |s| switch (s) { .nvfp4 => .{ .gate_up = .concatenated, .packing = .flashinfer_nvfp4 }, else => error.UnsupportedQuantization, } else error.UnsupportedQuantization, .mosaic_tpu, .metal, .tcl => .{ .gate_up = .concatenated, .packing = .plain }, - .triton, .fly => if (scheme == .mxfp4) .{ .gate_up = .interleaved, .packing = .plain } else .{ .gate_up = .concatenated, .packing = .plain }, + .triton, .fly => if (scheme == .mxfp4) .{ .gate_up = .row_interleaved_1, .packing = .plain } else .{ .gate_up = .concatenated, .packing = .plain }, }; } diff --git a/zml/kernels/moe/options.zig b/zml/kernels/moe/options.zig index aa68b06a..7f8d26de 100644 --- a/zml/kernels/moe/options.zig +++ b/zml/kernels/moe/options.zig @@ -56,7 +56,14 @@ pub const Activation = union(ActivationKind) { const mid = @divFloor(x.dim(.out), 2); var gate, var up = switch (layout) { .concatenated => .{ x.slice(.out, .{ .end = mid }), x.slice(.out, .{ .start = mid }) }, - .interleaved => .{ x.slice(.out, .{ .start = 0, .step = 2 }), x.slice(.out, .{ .start = 1, .step = 2 }) }, + .row_interleaved_1 => .{ x.slice(.out, .{ .start = 0, .step = 2 }), x.slice(.out, .{ .start = 1, .step = 2 }) }, + .row_interleaved_8 => blk: { + const grouped = x.splitAxis(.out, .{ .group = @divExact(mid, 8), .pair = 2, .row = 8 }); + break :blk .{ + grouped.slice(.pair, .{ .end = 1 }).merge(.{ .out = .{ .group, .pair, .row } }), + grouped.slice(.pair, .{ .start = 1 }).merge(.{ .out = .{ .group, .pair, .row } }), + }; + }, }; break :b switch (self) { @@ -96,12 +103,16 @@ pub const Activation = union(ActivationKind) { pub const RoutingWeightPlacement = enum { before_down, after_down }; -pub const ProjectionLayout = enum { concatenated, interleaved }; +pub const ProjectionLayout = enum { + concatenated, // G0 G1 … Gn U0 U1 … Un + row_interleaved_1, // G0 U0 G1 U1 … + row_interleaved_8, // G0…G7 U0…U7 G8…G15 U8…U15 … +}; /// How a backend expects the expert weights to be stored /// The packer writes this layout and forwardMoe reads it pub const ExpertsLayout = struct { - /// Whether the two projections occupy contiguous halves or alternate. + /// Gate/up ordering before applying the backend-specific packing. gate_up: ProjectionLayout, /// Backend specific transformation of the weights and scales. packing: Packing, @@ -113,5 +124,9 @@ pub const ExpertsLayout = struct { swizzled_scales, /// Weights, block scales and global scales in the FlashInfer CUTLASS NVFP4 layout. flashinfer_nvfp4, + /// MXFP4 gate/up rows interleaved by 8. E8M0 scales use 512-byte + /// 128x4 atoms in MN-major order `[expert, k/128, rows/128, 512]`, + /// represented as `[expert, k/128, rows]` i32 words of four scales. + mxfp4_interleaved8, }; }; diff --git a/zml/kernels/moe/tests.zig b/zml/kernels/moe/tests.zig index 0abb3462..f78cd1ef 100644 --- a/zml/kernels/moe/tests.zig +++ b/zml/kernels/moe/tests.zig @@ -179,20 +179,16 @@ fn makeExpertFixture(c: Case) ExpertFixture { return fixture; } -/// Interleave the gate/up halves of the output axis: [E, 2, N/2, K] -> [E, N/2, 2, K]. -fn interleave(tensor: Tensor) Tensor { - const tags = tensor.shape().tags(); - return tensor.withTags(.{ .expert, .out, .in }) - .splitAxis(.out, .{ .projection = 2, .mid = @divExact(tensor.dim(1), 2) }) - .transpose(.{ .expert, .mid, .projection, .in }) - .merge(.{ .out = .{ .mid, .projection } }).withTags(tags); +fn interleave(tensor: Tensor, rows: i64) Tensor { + return tensor.reshape(.{ .expert = tensor.dim(0), .projection = 2, .row_block = @divExact(tensor.dim(1), 2 * rows), .row = rows, .column = tensor.dim(2) }) + .transpose(.{ .expert, .row_block, .projection, .row, .column }).reshape(tensor.shape()); } /// Blackwell block scales: [E, N/128, 4, 32, K/4, 4] -> /// [E, N/128, K/4, 32, 4, 4], retaining the declared [E, N, K] shape. fn swizzleScales(scales: Tensor) Tensor { - return scales.reshape(.{ scales.dim(0), @divExact(scales.dim(1), 128), 4, 32, @divExact(scales.dim(2), 4), 4 }) - .transpose(.{ 0, 1, 4, 3, 2, 5 }).reshape(scales.shape()); + return scales.reshape(.{ .expert = scales.dim(0), .row_block = @divExact(scales.dim(1), 128), .row_quad = 4, .row_lane = 32, .scale_word = @divExact(scales.dim(2), 4), .scale_byte = 4 }) + .transpose(.{ .expert, .row_block, .scale_word, .row_lane, .row_quad, .scale_byte }).reshape(scales.shape()); } fn packFp4Bytes(weight: Tensor) Tensor { @@ -203,7 +199,8 @@ fn packFp4Bytes(weight: Tensor) Tensor { fn prepareGateUp(tensor: Tensor, layout: ExpertsLayout) Tensor { return switch (layout.gate_up) { .concatenated => tensor, - .interleaved => interleave(tensor), + .row_interleaved_1 => interleave(tensor, 1), + .row_interleaved_8 => interleave(tensor, 8), }; } @@ -221,17 +218,29 @@ fn prepareForBackend(canonical: ExpertFixture, backend: zml.moe.Backend) ExpertF } for ([_]*zml.nn.Linear{ &prepared.gate_up, &prepared.down }) |projection| { - if (layout.packing == .swizzled_scales or layout.packing == .flashinfer_nvfp4) { - const q = &projection.quantization.?; - q.scales = swizzleScales(q.scales); - q.swizzled_scales = true; + switch (layout.packing) { + .swizzled_scales, .flashinfer_nvfp4 => { + const q = &projection.quantization.?; + q.scales = swizzleScales(q.scales); + q.swizzled_scales = true; + }, + .mxfp4_interleaved8 => { + const q = &projection.quantization.?; + const sf = q.scales; + q.scales = sf.bitCast(.u8) + .reshape(.{ .expert = sf.dim(0), .row_block = @divExact(sf.dim(1), 128), .row_quad = 4, .row_lane = 32, .scale_word = @divExact(sf.dim(2), 4), .scale_byte = 4 }) + .transpose(.{ .expert, .scale_word, .row_block, .row_lane, .row_quad, .scale_byte }) + .reshape(.{ .expert = sf.dim(0), .scale_word = @divExact(sf.dim(2), 4), .row = sf.dim(1), .scale_byte = 4 }).bitCast(.i32) + .withTags(sf.shape().transpose(.{ 0, 2, 1 })); + }, + .plain => {}, } switch (backend) { .cute => { projection.weight = packFp4Bytes(projection.weight); const q = &projection.quantization.?; - q.scales = q.scales.bitCast(.u8); if (scheme == .nvfp4) { + q.scales = q.scales.bitCast(.u8); // CuTe takes the input quantization multiplier and a GEMM // output multiplier that folds in its inverse. const input_scale = q.input_scale.?.asMultiplier(); diff --git a/zml/kernels/moe/triton/triton.zig b/zml/kernels/moe/triton/triton.zig index bae76744..ed6c9f5f 100644 --- a/zml/kernels/moe/triton/triton.zig +++ b/zml/kernels/moe/triton/triton.zig @@ -52,7 +52,7 @@ pub fn fusedExperts( else => {}, }; - const layout: ProjectionLayout = if (gate_up.quantizationScheme() == .mxfp4) .interleaved else .concatenated; + const layout: ProjectionLayout = if (gate_up.quantizationScheme() == .mxfp4) .row_interleaved_1 else .concatenated; return fusedExpertsBase(input, routing, gate_up, down, layout, options); } @@ -500,7 +500,7 @@ test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { const x: Tensor = .init(.{ .token = 1, .out = 6 }, .bf16); const gates = [_]f32{ 0.75, -1.25, 3.5 }; const ups = [_]f32{ 0.875, -2.25, 4.5 }; - for ([_]ProjectionLayout{ .concatenated, .interleaved }) |layout| { + for ([_]ProjectionLayout{ .concatenated, .row_interleaved_1 }) |layout| { var values: [6]zml.floats.BFloat16 = undefined; for (gates, ups, 0..) |gate, up, i| { values[if (layout == .concatenated) i else 2 * i] = .fromF32(gate); @@ -665,7 +665,8 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { const columns = Tensor.arange(.{ .end = 256 }, .i32).withTags(.{.dout}); const is_gate = switch (layout) { .concatenated => columns.cmp(.LT, Tensor.scalar(128, .i32)), - .interleaved => columns.remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), + .row_interleaved_1 => columns.remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), + .row_interleaved_8 => columns.divByConst(8).remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), }; const dtype: DataType = if (fp4) .u8 else .bf16; // E2M1 codes 1, 2, 3 encode 0.5, 1, 1.5; each byte holds two values. @@ -717,7 +718,7 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { @memset(host.items(zml.floats.BFloat16), .fromF32(1.0 / 128.0)); var input = try zml.Buffer.fromSlice(io, platform, host); defer input.deinit(); - for ([_]ProjectionLayout{ .concatenated, .interleaved }) |layout| { + for ([_]ProjectionLayout{ .concatenated, .row_interleaved_1 }) |layout| { for ([_]RoutingWeightPlacement{ .before_down, .after_down }) |placement| { var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, placement, storage_dtype }, .{}); defer exe.deinit(); diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index 92e77888..aaa42b20 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -3,6 +3,7 @@ const std = @import("std"); const stdx = @import("stdx"); const zml = @import("../../zml.zig"); +pub const AttentionMask = @import("options.zig").AttentionMask; pub const AttentionOptions = @import("options.zig").AttentionOptions; pub const cute = @import("cute/cute.zig"); const metal = @import("metal/metal.zig"); diff --git a/zml/ops.zig b/zml/ops.zig index 4a189035..a9ae0f8d 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1532,7 +1532,15 @@ pub const CuteOps = struct { block: ?[3]i32 = null, zeroed_outputs: []const i32 = &.{}, scalars: []const i64 = &.{}, + symmetric_workspace: ?SymmetricWorkspace = null, output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, + + /// Named per-device workspace shared across CuTe calls and executables. + /// Every rank allocates `bytes`; XLA passes peer addresses and the rank. + pub const SymmetricWorkspace = struct { + name: []const u8, + bytes: i64, + }; }; /// A `__gpu$xla.gpu.cute` custom call: XLA compiles `ir` with the CuTe compiler @@ -1553,7 +1561,7 @@ pub fn cute(inputs: anytype, outputs: anytype, opts: CuteOps) [outputs.len]Tenso res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); } - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 8) = .empty; attrs.appendSliceAssumeCapacity(&.{ .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), .named(mlir_ctx, "kernel_type", .string(mlir_ctx, "cute")), @@ -1578,6 +1586,13 @@ pub fn cute(inputs: anytype, outputs: anytype, opts: CuteOps) [outputs.len]Tenso for (opts.scalars, scalar_attrs) |value, *attribute| attribute.* = .int(mlir_ctx, .i64, value); attrs.appendAssumeCapacity(.named(mlir_ctx, "scalars", .array(mlir_ctx, scalar_attrs))); } + if (opts.symmetric_workspace) |workspace| { + stdx.debug.assert(workspace.name.len > 0 and workspace.bytes > 0, "zml.ops.cute({s}): symmetric_workspace needs a name and a positive size", .{opts.name}); + attrs.appendAssumeCapacity(.named(mlir_ctx, "symmetric_workspace", .dict(mlir_ctx, &.{ + .named(mlir_ctx, "name", .string(mlir_ctx, workspace.name)), + .named(mlir_ctx, "bytes", .int(mlir_ctx, .i64, workspace.bytes)), + }))); + } const backend_config: *const mlir.Attribute = .dict(mlir_ctx, attrs.constSlice()); const op = stablehlo.custom_call( diff --git a/zml/platform.zig b/zml/platform.zig index f3510a19..6c6081aa 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -860,6 +860,12 @@ pub const cuda = struct { } }; + /// SM count of an addressable CUDA device; unavailable attributes return null. + pub fn coreCount(platform: *const zml.Platform, device_index: usize) ?u32 { + const attribute = platform.devices[device_index].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return null; + return std.math.cast(u32, attribute.int64); + } + pub fn computeCapability(platform: *const zml.Platform) ?ComputeCapability { if (platform.target != .cuda) return null; const devices = platform.pjrt_client.devices(platform.pjrt_api); From 6ee4f84ea77244c9deb7fd1ffec7b908f25d3425 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 10 Nov 2026 14:24:41 +0100 Subject: [PATCH 865/881] zml: Exe.memoryStats (#1052) --- zml/exe.zig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zml/exe.zig b/zml/exe.zig index 85baa7e7..32c0ac48 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -387,6 +387,11 @@ pub const Exe = struct { }, } } + + pub fn memoryStats(self: *const Exe) pjrt.ApiError!pjrt.CompiledMemoryStats { + const exe = try self.exe.executable(self.platform.pjrt_api); + return try exe.getCompiledMemoryStats(self.platform.pjrt_api); + } }; /// A typed executable whose input and output buffer structures are derived from From 9f8da93eafd7b924127cda7027266ce7571cccdb Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Tue, 10 Nov 2026 18:51:08 +0100 Subject: [PATCH 866/881] zml/kernels/paged_attention/metal: fit decode in 256 threads and split it over the GPU cores (#1046) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes Metal decode on M1/M2: `fa_vec_paged` launched up to 512 threads per threadgroup, over the 384 those GPUs accept. - A threadgroup holds up to 4 query heads of one KV head times position splits, at most 256 threads. - The split count follows the PJRT core count and the number of rows. - Head dims 32 and 64 now use the vector kernel instead of the tiled one. llmd decode on M5 Max vs master: no change at short context, +4–27% at 8K, +12–74% at 24K, bs16 unchanged. Head-dim-64 models: 2–7.6× at long context. Should fix https://github.com/zml/zml/issues/1037 but needs some testing on a real M1/M2 --- zml/kernels/paged_attention/metal/README.md | 12 +- zml/kernels/paged_attention/metal/metal.zig | 88 ++- .../paged_attention/metal/paged_attn.zig | 564 ++++++++---------- 3 files changed, 300 insertions(+), 364 deletions(-) diff --git a/zml/kernels/paged_attention/metal/README.md b/zml/kernels/paged_attention/metal/README.md index 5624de0e..8ccc6fd9 100644 --- a/zml/kernels/paged_attention/metal/README.md +++ b/zml/kernels/paged_attention/metal/README.md @@ -10,11 +10,13 @@ transposes queries and caches into the kernels' layouts, launches the the output. Queries and caches are `bf16` or `f16`; other dtypes panic. Causal `bf16` -decode with head dimension 128, 256 or 512 runs `fa_vec_paged`, which is `bf16` -only, split over four threadgroups and merged by `fa_vec_paged_reduce` at head -dimension 512 past 4096 KV positions. Everything else, `f16` decode included, -runs `paged_attention_tiled` (head dimension 32, 64, 96, 128, 256 or 512, -pages of 8, 16 or 32 positions), then `zero_padding_rows`. +decode with head dimension 32, 64, 128, 256 or 512 runs `fa_vec_paged`, which +is `bf16` only: a threadgroup holds up to four query heads of a KV head, times +splits of the KV positions, at most 256 threads. Rows split over the positions +are merged by `fa_vec_paged_reduce`; the split count follows the GPU core count +and the number of rows. Everything else, `f16` decode included, runs +`paged_attention_tiled` (head dimension 32, 64, 96, 128, 256 or 512, pages of +8, 16 or 32 positions), then `zero_padding_rows`. Causal and non-causal standard masks, left sliding windows, and a custom scale are passed to the kernels. Explicit key indices are rejected, and diff --git a/zml/kernels/paged_attention/metal/metal.zig b/zml/kernels/paged_attention/metal/metal.zig index e8144e3b..2cec68f5 100644 --- a/zml/kernels/paged_attention/metal/metal.zig +++ b/zml/kernels/paged_attention/metal/metal.zig @@ -85,8 +85,8 @@ const DispatchOptions = struct { scale: f32, }; -/// Launches fa_vec_paged for causal bf16 decode, split over 4 threadgroups at -/// head_dim 512 past 4096 KV positions, and paged_attention_tiled otherwise. +/// Launches fa_vec_paged for causal bf16 decode and paged_attention_tiled +/// otherwise. /// `q` and the result are `[tokens, heads, head_dim]`, the caches /// `[pages, block_size, kv_heads, head_dim]`, `block_table` /// `[seqs, max_blocks]`, `seq_lens` `[seqs]` and `query_start_len` @@ -105,7 +105,7 @@ fn dispatch(q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, block_table const max_blocks: i32 = @intCast(block_table.dim(1)); const vec_decode = opts.is_causal and total_q_tokens == num_seqs and - (head_dim == 128 or head_dim == 256 or head_dim == 512) and + (head_dim == 32 or head_dim == 64 or head_dim == 128 or head_dim == 256 or head_dim == 512) and dtype == .bf16; if (vec_decode) { @@ -128,35 +128,30 @@ fn dispatch(q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, block_table }; const kv_capacity = max_blocks * block_size; - if (head_dim == 512 and kv_capacity > 4096) { - const nwg = 4; - const nrows = total_q_tokens * num_heads; + const span = if (opts.sliding_window < 0) kv_capacity else @min(kv_capacity, opts.sliding_window + 32); + const nrows = total_q_tokens * num_heads; + const launch = vecLaunch(nrows, @divExact(num_heads, num_kv_heads), span, coreCount()); - const partial = paged_attn.FaVecPaged.Kernel.call(inputs, .{ .dst = zml.Shape.init(.{nrows * (head_dim * nwg + 2 * nwg)}, .f32) }, .{ - .cfg = .{ .head_dim = head_dim, .nsg = 8, .block_size = block_size, .position_stride = num_kv_heads * head_dim, .nwg = nwg }, - .grid = .{ nwg, @intCast(num_heads), @intCast(total_q_tokens) }, - .threadgroup = .{ 32, 8, 1 }, - .threadgroup_memory = paged_attn.FaVecPaged.threadgroupMemory(head_dim, 8), - .constants = .{ .args = .of(args) }, - }).dst; + const cfg: paged_attn.FaVecPaged.Cfg = .{ + .head_dim = head_dim, + .block_size = block_size, + .position_stride = num_kv_heads * head_dim, + .heads = launch.heads, + .splits = launch.splits, + }; + const dst = if (launch.splits == 1) q.shape() else zml.Shape.init(.{nrows * (head_dim * launch.splits + 2 * launch.splits)}, .f32); + const out = paged_attn.FaVecPaged.Kernel.call(inputs, .{ .dst = dst }, .{ + .cfg = cfg, + .grid = .{ @intCast(@divExact(launch.splits, launch.splits_per_threadgroup)), @intCast(@divExact(num_heads, launch.heads)), @intCast(total_q_tokens) }, + .threadgroup = .{ 32, @intCast(launch.heads * launch.splits_per_threadgroup), 1 }, + .constants = .{ .args = .of(args) }, + }).dst; + if (launch.splits == 1) return out; - return paged_attn.FaVecPagedReduce.Kernel.call(.{ .htmp = partial }, .{ .dst = q.shape() }, .{ - .cfg = .{ .nwg = nwg, .head_dim = head_dim }, - .grid = .{ @intCast(nrows), 1, 1 }, - .threadgroup = .{ 32, nwg, 1 }, - .constants = .{ .args = .of(@as(i32, nrows)) }, - }).dst; - } - - var idx: usize = if (kv_capacity <= 1024) 0 else if (kv_capacity <= 2048) 1 else 2; - if (head_dim == 512 and idx > 1) idx = 1; - const nsg = ([_]i32{ 4, 8, 16 })[idx]; - - return paged_attn.FaVecPaged.Kernel.call(inputs, .{ .dst = q.shape() }, .{ - .cfg = .{ .head_dim = head_dim, .nsg = nsg, .block_size = block_size, .position_stride = num_kv_heads * head_dim, .nwg = 1 }, - .grid = .{ 1, @intCast(num_heads), @intCast(total_q_tokens) }, - .threadgroup = .{ 32, @intCast(nsg), 1 }, - .threadgroup_memory = paged_attn.FaVecPaged.threadgroupMemory(head_dim, nsg), + return paged_attn.FaVecPagedReduce.Kernel.call(.{ .htmp = out, .seq_lens = seq_lens, .cu_seqlens_q = query_start_len }, .{ .dst = q.shape() }, .{ + .cfg = .{ .splits = launch.splits, .head_dim = head_dim }, + .grid = .{ @intCast(nrows), @intCast(@divExact(head_dim, 4 * paged_attn.FaVecPagedReduce.simdgroups)), 1 }, + .threadgroup = .{ 32, paged_attn.FaVecPagedReduce.simdgroups, 1 }, .constants = .{ .args = .of(args) }, }).dst; } @@ -210,3 +205,36 @@ fn dispatch(q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, block_table .constants = .{ .num_seqs = .of(num_seqs), .row_stride = .of(num_heads * head_dim) }, }).out; } + +const VecLaunch = struct { + heads: i32, + splits: i32, + splits_per_threadgroup: i32, +}; + +/// At most 256 threads, which every Apple GPU pipeline accepts. Splits past a +/// row's length exit early, so `splits` follows the core and row counts. +fn vecLaunch(rows: i32, group: i32, span: i32, cores: i32) VecLaunch { + var heads: i32 = @min(group, 4); + while (@rem(group, heads) != 0) heads -= 1; + const want: i32 = if (rows <= 2 * cores) 128 else 64; + const chunks = @divFloor(span + 31, 32); + var splits: i32 = 1; + while (splits * 2 <= @min(want, chunks)) splits *= 2; + return .{ .heads = heads, .splits = splits, .splits_per_threadgroup = @min(splits, @divFloor(8, heads)) }; +} + +fn coreCount() i32 { + const platform = zml.Compiler.current().platform; + const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse + std.debug.panic("Metal device has no core_count attribute", .{}); + return @intCast(attribute.int64); +} + +test vecLaunch { + try std.testing.expectEqual(VecLaunch{ .heads = 4, .splits = 128, .splits_per_threadgroup = 2 }, vecLaunch(16, 4, 128000, 40)); + try std.testing.expectEqual(VecLaunch{ .heads = 2, .splits = 64, .splits_per_threadgroup = 4 }, vecLaunch(256, 2, 128000, 40)); + try std.testing.expectEqual(VecLaunch{ .heads = 4, .splits = 64, .splits_per_threadgroup = 2 }, vecLaunch(512, 4, 128000, 40)); + try std.testing.expectEqual(VecLaunch{ .heads = 3, .splits = 8, .splits_per_threadgroup = 2 }, vecLaunch(48, 6, 288, 40)); + try std.testing.expectEqual(VecLaunch{ .heads = 1, .splits = 1, .splits_per_threadgroup = 1 }, vecLaunch(8, 1, 20, 40)); +} diff --git a/zml/kernels/paged_attention/metal/paged_attn.zig b/zml/kernels/paged_attention/metal/paged_attn.zig index 20429565..337432e7 100644 --- a/zml/kernels/paged_attention/metal/paged_attn.zig +++ b/zml/kernels/paged_attention/metal/paged_attn.zig @@ -18,25 +18,18 @@ const masked_score = -flt_max / 2.0; const max_half = 65504.0; const simd_width = 32; -fn pad2(x: i32, n: i32) i32 { - return (x + n - 1) & ~(n - 1); -} - -/// fa_vec_paged: decode attention over paged bf16 caches, one threadgroup -/// of `nsg` simdgroups per (position split, query head, query row). +/// fa_vec_paged: bf16 decode over paged caches. Simdgroup s takes query head +/// s % heads of the threadgroup's KV head group and position split s / heads. pub const FaVecPaged = struct { pub const Cfg = struct { - /// The key and value head dimension, DK = DV. + /// The key and value head dimension. head_dim: i32, - /// Simdgroups splitting the KV positions. - nsg: i32, /// The page size. block_size: i32, /// The in-page position stride, n_kv * head_dim elements. position_stride: i32, - /// Position-split threadgroups. Above 1 the kernel writes f32 partials - /// for FaVecPagedReduce. - nwg: i32, + heads: i32, + splits: i32, }; pub const Args: metal.Struct = .{ .name = "fa_vec_paged_args", .fields = &.{ @@ -66,17 +59,10 @@ pub const FaVecPaged = struct { .run = run, }); - /// The threadgroup memory of a launch with `nsg` simdgroups. - pub fn threadgroupMemory(head_dim: i32, nsg: i32) u64 { - const padded = pad2(head_dim, 128); - const bytes = (padded + 4 * 32 + 2 * padded) * nsg * 2; - return @intCast(@divFloor(bytes + 15, 16) * 16); - } - fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { switch (cfg.head_dim) { - 128, 256, 512 => {}, - else => std.debug.panic("fa_vec_paged: head_dim {d} unsupported (128, 256 or 512)", .{cfg.head_dim}), + 32, 64, 128, 256, 512 => {}, + else => std.debug.panic("fa_vec_paged: head_dim {d} unsupported (32, 64, 128, 256 or 512)", .{cfg.head_dim}), } const char_in: ArgSpec = .{ .buffer = .{ .type = .char, .const_ = true } }; const int_in: ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; @@ -89,158 +75,90 @@ pub const FaVecPaged = struct { .seq_lens = int_in, .cu_seqlens_q = int_in, .dst = .{ .buffer = .{ .type = .char } }, - .shmem_f16 = .{ .threadgroup = .half }, .tgpig = .{ .threadgroup_position_in_grid = .uint3 }, .tgpg = .{ .threadgroups_per_grid = .uint3 }, .tiisg = .{ .thread_index_in_simdgroup = .ushort }, .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, + .nsg = .{ .simdgroups_per_threadgroup = .ushort }, }); impl(b, cfg, a); } - /// The attention body. fn impl(b: *Builder, cfg: Cfg, a: anytype) void { - const DK = cfg.head_dim; - const DV = cfg.head_dim; - const NE = 1; - const C = 32; - const NSG = cfg.nsg; + const D = cfg.head_dim; + const D4 = @divExact(D, 4); + // A K/V row spans NL lanes of NI float4s; a simdgroup reads R rows per step. + const NL = @min(D4, simd_width); + const R = @divExact(simd_width, NL); + const NI = @divExact(D4, NL); + const C = simd_width; const BS = cfg.block_size; const PS = cfg.position_stride; - const NWG = cfg.nwg; + const HG = cfg.heads; + const NWG = cfg.splits; + const full: metal.LoopOpts = .{ .unroll = .full }; + const num_heads = a.args.field("num_heads"); + const group = num_heads.div(a.args.field("n_kv")); + const groups = group.div(HG); const r = a.tgpig.at(2).to(.int); - const iq2 = a.tgpig.at(1).to(.ushort); - const iwg = a.tgpig.at(0).to(.short); - - const DK4 = @divExact(DK, 4); - const DV4 = @divExact(DV, 4); - - const nrows_static = a.tgpg.at(2).to(.long).mul(a.args.field("num_heads").to(.uint)); + const tg_head = a.tgpig.at(1).to(.int); + const ikv = tg_head.div(groups); + const iq2 = ikv.mul(group).add(tg_head.rem(groups).mul(HG)).add(a.sgitg.rem(HG).to(.int)); + const iwg = a.tgpig.at(0).to(.int).mul(a.nsg.div(HG).to(.int)).add(a.sgitg.div(HG).to(.int)); + const rid = r.to(.long).mul(num_heads.to(.uint)).add(iq2); + const nrows = a.tgpg.at(2).to(.long).mul(num_heads.to(.uint)); + // Padding rows: FaVecPagedReduce zeroes them when there are splits. var padding = b.openIf(r.ge(a.cu_seqlens_q.load(a.args.field("num_seqs")))); { - var first = b.openIf(a.sgitg.eq(0)); - { - const rid = r.to(.long).mul(a.args.field("num_heads").to(.uint)).add(iq2); - if (NWG == 1) { - const dst4 = a.dst.to(.bfloat4); - var loop = b.openFor(a.tiisg.to(.short), DV4, simd_width); - { - const zero = b.constant(.float4, 0).to(.bfloat4); - dst4.store(rid.mul(DV4).add(loop.iv), zero); - loop.close(); - } - } else { - const dst4 = a.dst.to(.float4); - const dst1 = a.dst.to(.float).add(nrows_static.mul(DV).mul(NWG)); - var loop = b.openFor(a.tiisg.to(.short), DV4, simd_width); - { - dst4.store(rid.mul(DV4).mul(NWG).add(b.lift(NWG).mul(loop.iv)).add(iwg), b.constant(.float4, 0)); - loop.close(); - } - var lane0 = b.openIf(a.tiisg.eq(0)); - { - dst1.store(rid.mul(2 * NWG).add(b.lift(2).mul(iwg)).add(0), 0.0); - dst1.store(rid.mul(2 * NWG).add(b.lift(2).mul(iwg)).add(1), -flt_max / 2.0); - lane0.close(); - } + if (NWG == 1) { + const dst4 = a.dst.to(.bfloat4).add(rid.mul(D4)); + var loop = b.openFor(a.tiisg.to(.short), D4, simd_width); + { + dst4.store(loop.iv, b.constant(.float4, 0).to(.bfloat4)); + loop.close(); } - first.close(); } b.return_(); padding.close(); } - const first_seq = b.local(.int, 0); - const last_seq = b.local(.int, a.args.field("num_seqs").sub(1)); - var search = b.openLoop(.{}); - { - search.body(first_seq.lt(last_seq)); - } - { - const mid = first_seq.add(last_seq).add(1).div(2); - var half = b.openIfElse(a.cu_seqlens_q.load(mid).le(r), .{}); + const seq = sequenceOf(b, a, r); + const kv_len = seq.kv_len; + const kv_lo = seq.kv_lo; + const ic0_lo = kv_lo.div(C); + const live = if (NWG > 1) liveSplits(b, seq, NWG) else b.lift(@as(i32, 1)); + + if (NWG > 1) { + var idle = b.openIf(iwg.ge(live)); { - first_seq.assign(mid); - half.elseBranch(); + b.return_(); + idle.close(); } - { - last_seq.assign(mid.sub(1)); - half.close(); - } - search.close(); } - const seq_idx = first_seq.rvalue(); - const q_pos = r.sub(a.cu_seqlens_q.load(seq_idx)); - const query_len = a.cu_seqlens_q.load(seq_idx.add(1)).sub(a.cu_seqlens_q.load(seq_idx)); - const kv_total = a.seq_lens.load(seq_idx); - const kv_len = kv_total.sub(query_len).add(q_pos).add(1); - const kv_lo = b.ternary(a.args.field("sliding_window").lt(0), 0, b.max(0, kv_len.sub(a.args.field("sliding_window")))); - const pt = a.block_tables.add(seq_idx.to(.ulong).mul(a.args.field("max_blocks").to(.uint))); + const pt = a.block_tables.add(seq.idx.to(.ulong).mul(a.args.field("max_blocks").to(.uint))); + const k_cache = a.k_cache.add(ikv.to(.ulong).mul(D).mul(2)); + const v_cache = a.v_cache.add(ikv.to(.ulong).mul(D).mul(2)); - const PK = pad2(DK, 128); - const PK4 = @divExact(PK, 4); - const PV = pad2(DV, 128); - const PV4 = @divExact(PV, 4); - const NW = simd_width; - const NL = @divExact(NW, NE); - const SH = 4 * C; - - const shmem = a.shmem_f16; - const sq4 = shmem.add(0 * PK).to(.half4); - const ss = shmem.add(a.sgitg.mul(SH)).add(NSG * PK).to(.float); - const ss4 = shmem.add(a.sgitg.mul(SH)).add(NSG * PK).to(.float4); - const sm = shmem.add(a.sgitg.mul(SH)).add(2 * C).add(NSG * PK); - const so4 = shmem.add(b.lift(2).mul(a.sgitg).mul(PV)).add(NSG * PK).add(NSG * SH).to(.float4); - // This lane's output, until the simdgroups merge theirs. - const lane_so4 = so4.add(a.tiisg); - - const q = a.q.add(r.to(.ulong).mul(a.args.field("num_heads").to(.uint)).add(iq2).mul(DK).mul(2)); - const ikv2 = iq2.div(a.args.field("num_heads").div(a.args.field("n_kv"))).to(.short); - const k_cache = a.k_cache.add(ikv2.to(.ulong).mul(DK).mul(2)); - const v_cache = a.v_cache.add(ikv2.to(.ulong).mul(DK).mul(2)); - - const q4 = q.to(.bfloat4); - var load_q = b.openFor(a.tiisg.to(.short), PK4, NW); + const lane = if (R == 1) a.tiisg else a.tiisg.rem(NL); + const sub = a.tiisg.div(NL).to(.int); + const q4 = a.q.add(rid.mul(D).mul(2)).to(.bfloat4).add(lane); + const qr = b.array(.float4, @intCast(NI)); + const o = b.array(.float4, @intCast(NI)); + var init = b.openForOpts(b.constant(.short, 0), NI, 1, full); { - const i = load_q.iv; - var value = b.openIfElse(i.lt(DK4), .{.half4}); - { - value.yieldThen(.{q4.load(i).to(.half4)}); - } - { - value.yieldElse(.{b.constant(.half4, 0)}); - } - sq4.store(i, value.results[0]); - load_q.close(); - } - var zero_o = b.openFor(b.constant(.short, 0), @divExact(DV4, NL), 1); - { - lane_so4.store(zero_o.iv.mul(NL), b.constant(.float4, 0)); - zero_o.close(); - } - var zero_s = b.openFor(a.tiisg.to(.short), SH / 4, NW); - { - ss4.store(zero_s.iv, b.constant(.float4, 0)); - zero_s.close(); + const ii = init.iv; + qr.store(ii, q4.load(ii.mul(NL)).to(.float4)); + o.store(ii, b.constant(.float4, 0)); + init.close(); } - b.threadgroupBarrier(.mem_threadgroup); - - // When DV4/NL is a multiple of NW every lane updates the output, so the - // `ty == 0` guard is dropped at comptime. - const all_lanes = @rem(@divExact(DV4, NL), NW) == 0; - const full: metal.LoopOpts = .{ .unroll = .full }; const S = b.local(.float, 0.0); const M = b.local(.float, -flt_max / 2.0); - const tx = a.tiisg.rem(NL).to(.short); - const ty = a.tiisg.div(NL).to(.short); - - const ic0_lo = kv_lo.div(C); - const ic0 = b.local(.int, ic0_lo.add(iwg.mul(NSG)).add(a.sgitg)); + const ic0 = b.local(.int, ic0_lo.add(iwg)); var blocks = b.openLoop(.{}); { const ic = ic0.mul(C); @@ -250,175 +168,103 @@ pub const FaVecPaged = struct { past.close(); } - const in_kv = ic.add(a.tiisg).lt(kv_len).bitAnd(ic.add(a.tiisg).ge(kv_lo)); - sm.store(a.tiisg, b.ternary(in_kv, b.constant(.half, 0), b.constant(.half, -max_half))); - - const pq4 = sq4.add(tx); - const mqk = b.zeroedArray(.float, C / NE); - var qk = b.openForOpts(b.constant(.short, 0), C / NE, 1, full); + const s = b.local(.float, 0.0); + var qk = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); { const cc = qk.iv; - const p = b.min(ic.add(cc), kv_len.sub(1)); - const pk4 = pagedRow(b, k_cache, pt, p, BS, PS).to(.bfloat4).add(ty.mul(PS).div(4).add(tx)); - var dots = b.openForOpts(b.constant(.short, 0), @divExact(DK4, NL), 1, full); + const p = b.min(if (R == 1) ic.add(cc) else ic.add(cc.to(.int).mul(R)).add(sub), kv_len.sub(1)); + const pk4 = pagedRow(b, k_cache, pt, p, BS, PS).to(.bfloat4).add(lane); + const d = b.local(.float, 0.0); + var dots = b.openForOpts(b.constant(.short, 0), NI, 1, full); { const ii = dots.iv; - const dot = b.dot(pk4.load(ii.mul(NL)).to(.float4), pq4.load(ii.mul(NL)).to(.float4)); - mqk.store(cc, mqk.load(cc).add(dot)); + d.assign(d.add(b.dot(pk4.load(ii.mul(NL)).to(.float4), qr.load(ii)))); dots.close(); } - mqk.store(cc, b.simdSum(mqk.load(cc))); + if (R == 1) { + s.assign(b.ternary(a.tiisg.to(.short).eq(cc), b.simdSum(d.rvalue()), s.rvalue())); + } else { + var x = d.rvalue(); + var mask: i32 = 1; + while (mask < NL) : (mask *= 2) x = x.add(b.simdShuffleXor(x, b.constant(.ushort, mask))); + const t = b.simdShuffle(x, a.tiisg.rem(R).mul(NL)); + s.assign(b.ternary(a.tiisg.div(R).to(.short).eq(cc), t, s.rvalue())); + } qk.close(); } - const score = b.fma(mqk.load(tx), a.args.field("scale"), sm.load(b.lift(NE).mul(tx).add(ty)).to(.float)); - ss.store(b.lift(NE).mul(tx).add(ty), score); - - b.simdgroupBarrier(.mem_threadgroup); + const pos = ic.add(a.tiisg.to(.int)); + const in_kv = pos.lt(kv_len).bitAnd(pos.ge(kv_lo)); + const score = b.ternary(in_kv, s.mul(a.args.field("scale")), b.constant(.float, -max_half)); const m = M.rvalue(); - const s = ss.load(a.tiisg); - - M.assign(b.simdMax(b.max(M.rvalue(), s))); - + M.assign(b.simdMax(b.max(M.rvalue(), score))); const ms = b.exp(m.sub(M)); - const vs = b.exp(s.sub(M)); - + const vs = b.exp(score.sub(M)); S.assign(S.mul(ms).add(b.simdSum(vs))); - ss.store(a.tiisg, vs); - - var rescale_lanes = if (all_lanes) null else b.openIf(ty.eq(0)); - var rescale = b.openForOpts(b.constant(.short, 0), @divExact(DV4, NL), 1, full); + var rescale = b.openForOpts(b.constant(.short, 0), NI, 1, full); { - const at = rescale.iv.mul(NL); - lane_so4.store(at, lane_so4.load(at).mul(ms)); + o.store(rescale.iv, o.load(rescale.iv).mul(ms)); rescale.close(); } - if (rescale_lanes) |*l| l.close(); - b.simdgroupBarrier(.mem_threadgroup); - - const lo = b.array(.float4, @intCast(@divExact(DV4, NL))); - var zero_lo = b.openForOpts(b.constant(.short, 0), @divExact(DV4, NL), 1, full); - { - lo.store(zero_lo.iv, b.constant(.float4, 0)); - zero_lo.close(); - } - - const sst = ss.add(ty); - - var pv = b.openForOpts(b.constant(.short, 0), C / NE, 1, full); + var pv = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); { const cc = pv.iv; - const p = b.min(ic.add(cc), kv_len.sub(1)); - const pv4 = pagedRow(b, v_cache, pt, p, BS, PS).to(.bfloat4).add(ty.mul(PS).div(4).add(tx)); - var dims = b.openForOpts(b.constant(.short, 0), @divExact(DV4, NL), 1, full); + const row = cc.to(.int).mul(R).add(sub); + const p = b.min(if (R == 1) ic.add(cc) else ic.add(row), kv_len.sub(1)); + const pv4 = pagedRow(b, v_cache, pt, p, BS, PS).to(.bfloat4).add(lane); + const w = b.vec(.float4, .{if (R == 1) b.simdBroadcast(vs, cc.to(.ushort)) else b.simdShuffle(vs, row)}); + var dims = b.openForOpts(b.constant(.short, 0), NI, 1, full); { const ii = dims.iv; - const weighted = pv4.load(ii.mul(NL)).to(.float4).mul(b.vec(.float4, .{sst.load(cc.mul(NE))})); - lo.store(ii, lo.load(ii).add(weighted)); + o.store(ii, o.load(ii).add(pv4.load(ii.mul(NL)).to(.float4).mul(w))); dims.close(); } pv.close(); } - var add_lanes = if (all_lanes) null else b.openIf(ty.eq(0)); - var add_o = b.openForOpts(b.constant(.short, 0), @divExact(DV4, NL), 1, full); - { - const ii = add_o.iv; - const partial = lo.load(ii); - lane_so4.store(ii.mul(NL), lane_so4.load(ii.mul(NL)).add(partial)); - add_o.close(); - } - if (add_lanes) |*l| l.close(); blocks.next(); - ic0.assign(ic0.add(NWG * NSG)); + ic0.assign(ic0.add(live)); blocks.close(); } - var stats = b.openIf(a.tiisg.eq(0)); - { - ss.store(0, S); - ss.store(1, M); - stats.close(); - } - - b.threadgroupBarrier(.mem_threadgroup); - - const rr = b.local(.short, @divFloor(NSG, 2)); - var halving = b.openLoop(.{}); - { - halving.body(rr.gt(0)); - } - { - var group = b.openIf(a.sgitg.lt(rr)); + var mask: i32 = NL; + while (mask < simd_width) : (mask *= 2) { + var sum = b.openForOpts(b.constant(.short, 0), NI, 1, full); { - const S0 = ss.load(0); - const S1 = ss.load(rr.mul(@divExact(SH, 2)).add(0)); - - const M0 = ss.load(1); - const M1 = ss.load(rr.mul(@divExact(SH, 2)).add(1)); - - const M_ = b.max(M0, M1); - - const ms0 = b.exp(M0.sub(M_)); - const ms1 = b.exp(M1.sub(M_)); - - const S_ = S0.mul(ms0).add(S1.mul(ms1)); - - var lane0 = b.openIf(a.tiisg.eq(0)); - { - ss.store(0, S_); - ss.store(1, M_); - lane0.close(); - } - - var merge = b.openFor(a.tiisg.to(.short), DV4, NW); - { - const i = merge.iv; - so4.store(i, so4.load(i).mul(ms0).add(so4.load(i.add(rr.mul(PV4))).mul(ms1))); - merge.close(); - } - group.close(); + o.store(sum.iv, o.load(sum.iv).add(b.simdShuffleXor(o.load(sum.iv), b.constant(.ushort, mask)))); + sum.close(); } - b.threadgroupBarrier(.mem_threadgroup); - rr.assign(rr.shr(1)); - halving.close(); } - - var first = b.openIf(a.sgitg.eq(0)); - { - const rid = r.to(.long).mul(a.args.field("num_heads").to(.uint)).add(iq2); - if (NWG == 1) { - const inv_sum = b.ternary(ss.load(0).eq(0.0), 0.0, b.lift(1.0).div(ss.load(0))); - const dst4 = a.dst.to(.bfloat4); - var loop = b.openFor(a.tiisg.to(.short), DV4, NW); - { - const out = so4.load(loop.iv).mul(inv_sum).to(.bfloat4); - dst4.store(rid.mul(DV4).add(loop.iv), out); - loop.close(); - } - } else { - const dst4 = a.dst.to(.float4); - const dst1 = a.dst.to(.float).add(nrows_static.mul(DV).mul(NWG)); - var loop = b.openFor(a.tiisg.to(.short), DV4, NW); - { - const part = so4.load(loop.iv); - dst4.store(rid.mul(DV4).mul(NWG).add(b.lift(NWG).mul(loop.iv)).add(iwg), part); - loop.close(); - } - var lane0 = b.openIf(a.tiisg.eq(0)); - { - const sum = ss.load(0); - dst1.store(rid.mul(2 * NWG).add(b.lift(2).mul(iwg)).add(0), sum); - const max = ss.load(1); - dst1.store(rid.mul(2 * NWG).add(b.lift(2).mul(iwg)).add(1), max); - lane0.close(); - } + var first = if (R > 1) b.openIf(sub.eq(0)) else null; + if (NWG == 1) { + const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); + const dst4 = a.dst.to(.bfloat4).add(rid.mul(D4)).add(lane); + var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + dst4.store(out.iv.mul(NL), o.load(out.iv).mul(inv_sum).to(.bfloat4)); + out.close(); + } + } else { + // [rows][D4][splits] float4 partials, then [rows][splits] (S, M). + const dst4 = a.dst.to(.float4).add(rid.mul(D4).mul(NWG)); + var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const i = lane.to(.int).add(out.iv.to(.int).mul(NL)); + dst4.store(i.mul(NWG).add(iwg), o.load(out.iv)); + out.close(); + } + var lane0 = b.openIf(a.tiisg.eq(0)); + { + const dst1 = a.dst.to(.float).add(nrows.mul(D).mul(NWG)).add(rid.mul(2 * NWG)); + dst1.store(b.lift(2).mul(iwg), S); + dst1.store(b.lift(2).mul(iwg).add(1), M); + lane0.close(); } - first.close(); } + if (first) |*scope| scope.close(); } /// The address of position `p`'s bf16 row in `cache`, through the page @@ -430,85 +276,144 @@ pub const FaVecPaged = struct { } }; -/// fa_vec_paged_reduce: merges the `nwg` partials of a split-K fa_vec_paged, -/// one threadgroup per row and head. +/// The sequence of query row `r` and its causal KV range [kv_lo, kv_len). +fn sequenceOf(b: *Builder, a: anytype, r: Value) struct { idx: Value, kv_len: Value, kv_lo: Value } { + const first_seq = b.local(.int, 0); + const last_seq = b.local(.int, a.args.field("num_seqs").sub(1)); + var search = b.openLoop(.{}); + { + search.body(first_seq.lt(last_seq)); + } + { + const mid = first_seq.add(last_seq).add(1).div(2); + var half = b.openIfElse(a.cu_seqlens_q.load(mid).le(r), .{}); + { + first_seq.assign(mid); + half.elseBranch(); + } + { + last_seq.assign(mid.sub(1)); + half.close(); + } + search.close(); + } + const idx = first_seq.rvalue(); + const q_pos = r.sub(a.cu_seqlens_q.load(idx)); + const query_len = a.cu_seqlens_q.load(idx.add(1)).sub(a.cu_seqlens_q.load(idx)); + const kv_len = a.seq_lens.load(idx).sub(query_len).add(q_pos).add(1); + const kv_lo = b.ternary(a.args.field("sliding_window").lt(0), 0, b.max(0, kv_len.sub(a.args.field("sliding_window")))); + return .{ .idx = idx, .kv_len = kv_len, .kv_lo = kv_lo }; +} + +fn liveSplits(b: *Builder, seq: anytype, splits: i32) Value { + const chunks = seq.kv_len.add(simd_width - 1).div(simd_width).sub(seq.kv_lo.div(simd_width)); + return b.min(splits, chunks); +} + +/// fa_vec_paged_reduce: merges the fa_vec_paged split partials. pub const FaVecPagedReduce = struct { pub const Cfg = struct { - /// The partials to merge, FaVecPaged's `nwg`. - nwg: i32, + splits: i32, /// The head dimension. head_dim: i32, }; - pub const Args: metal.Struct = .{ .name = "fa_vec_paged_reduce_args", .fields = &.{ - .{ .name = "nrows", .type = .int }, - } }; + pub const simdgroups = 8; pub const Kernel = metal.Kernel(Cfg, .{ .name = "fa_vec_paged_reduce", - .inputs = &.{"htmp"}, + .inputs = &.{ "htmp", "seq_lens", "cu_seqlens_q" }, .outputs = &.{"dst"}, .constants = &.{"args"}, .run = run, }); fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + const int_in: ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; const a = try b.declareArgs(.{ - .args = .{ .constant_struct = Args }, + .args = .{ .constant_struct = FaVecPaged.Args }, .htmp = .{ .buffer = .{ .type = .char, .const_ = true } }, + .seq_lens = int_in, + .cu_seqlens_q = int_in, .dst = .{ .buffer = .{ .type = .char } }, - .tgpig = .{ .threadgroup_position_in_grid = .uint }, + .tgpig = .{ .threadgroup_position_in_grid = .uint2 }, + .tgpg = .{ .threadgroups_per_grid = .uint2 }, .tiisg = .{ .thread_index_in_simdgroup = .ushort }, .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, }); - const nwg = cfg.nwg; + const NWG = cfg.splits; + if (NWG > 4 * simd_width) std.debug.panic("fa_vec_paged_reduce: {d} splits, at most {d}", .{ NWG, 4 * simd_width }); + const NP = @divFloor(NWG + simd_width - 1, simd_width); const DV = cfg.head_dim; - const rid = a.tgpig.to(.ulong); - const iwg = a.tiisg.to(.short); - const ss = a.htmp.to(.float).add(a.args.field("nrows").to(.ulong).mul(DV).mul(nwg)); + const DV4 = @divExact(DV, 4); + const full: metal.LoopOpts = .{ .unroll = .full }; - var max = b.openIfElse(iwg.lt(nwg), .{.float}); - { - max.yieldThen(.{ss.load(rid.mul(2 * nwg).add(b.lift(2).mul(iwg)).add(1))}); - } - { - max.yieldElse(.{-flt_max / 2.0}); - } - const M = max.results[0]; - var sum = b.openIfElse(iwg.lt(nwg), .{.float}); - { - sum.yieldThen(.{ss.load(rid.mul(2 * nwg).add(b.lift(2).mul(iwg)).add(0))}); - } - { - sum.yieldElse(.{0.0}); - } - - const m = b.simdMax(M); - const ms = b.exp(M.sub(m)); - const S = b.simdSum(sum.results[0].mul(ms)); - const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); - - const DV4 = @divTrunc(DV, 4); - const htmp4 = a.htmp.to(.float4).add(rid.mul(DV4).mul(nwg)); + const rid = a.tgpig.at(0).to(.ulong); + const r = a.tgpig.at(0).div(a.args.field("num_heads").to(.uint)).to(.int); + const i = a.tgpig.at(1).to(.int).mul(simdgroups).add(a.sgitg.to(.int)); const dst4 = a.dst.to(.bfloat4).add(rid.mul(DV4)); - var loop = b.openFor(a.sgitg.to(.int), DV4, nwg); + + const live = b.local(.int, 0); + var real = b.openIf(r.lt(a.cu_seqlens_q.load(a.args.field("num_seqs")))); { - const i = loop.iv; - var part_ = b.openIfElse(iwg.lt(nwg), .{.float4}); + live.assign(liveSplits(b, sequenceOf(b, a, r), NWG)); + real.close(); + } + var empty = b.openIf(live.le(0)); + { + var lane0 = b.openIf(a.tiisg.eq(0)); { - part_.yieldThen(.{htmp4.load(i.mul(nwg).add(iwg)).mul(ms)}); - } - { - part_.yieldElse(.{b.constant(.float4, 0)}); - } - const part = part_.results[0]; - const v = b.simdSum(part); - var lane0 = b.openIf(iwg.eq(0)); - { - dst4.store(i, v.mul(inv_sum).to(.bfloat4)); + dst4.store(i, b.constant(.float4, 0).to(.bfloat4)); lane0.close(); } - loop.close(); + b.return_(); + empty.close(); + } + + const ss = a.htmp.to(.float).add(a.tgpg.at(0).to(.ulong).mul(DV).mul(NWG)).add(rid.mul(2 * NWG)); + const Mj = b.array(.float, @intCast(NP)); + const Sj = b.array(.float, @intCast(NP)); + const mx = b.local(.float, -flt_max / 2.0); + var load = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = load.iv; + const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); + const ok = idx.lt(live); + Mj.store(j, b.ternary(ok, ss.load(idx.mul(2).add(1)), b.constant(.float, -flt_max / 2.0))); + Sj.store(j, b.ternary(ok, ss.load(idx.mul(2)), b.constant(.float, 0))); + mx.assign(b.max(mx.rvalue(), Mj.load(j))); + load.close(); + } + const m = b.simdMax(mx.rvalue()); + const sum = b.local(.float, 0.0); + var weigh = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = weigh.iv; + Mj.store(j, b.exp(Mj.load(j).sub(m))); + sum.assign(sum.add(Sj.load(j).mul(Mj.load(j)))); + weigh.close(); + } + const S = b.simdSum(sum.rvalue()); + const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); + + const htmp4 = a.htmp.to(.float4).add(rid.mul(DV4).mul(NWG)); + const acc = b.local(.float4, b.constant(.float4, 0)); + var parts = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = parts.iv; + const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); + var ok = b.openIf(idx.lt(live)); + { + acc.assign(acc.add(htmp4.load(i.mul(NWG).add(idx)).mul(Mj.load(j)))); + ok.close(); + } + parts.close(); + } + const v = b.simdSum(acc.rvalue()); + var lane0 = b.openIf(a.tiisg.eq(0)); + { + dst4.store(i, v.mul(inv_sum).to(.bfloat4)); + lane0.close(); } } }; @@ -984,13 +889,14 @@ pub const PagedAttentionTiled = struct { test "the paged attention kernels emit verified modules" { const allocator = std.testing.allocator; - for ([_]i32{ 128, 256, 512 }) |head_dim| { - for ([_]i32{ 4, 8, 16 }) |nsg| { - allocator.free(try FaVecPaged.Kernel.emit(allocator, .{ .head_dim = head_dim, .nsg = nsg, .block_size = 16, .position_stride = 2 * head_dim, .nwg = 1 })); + for ([_]i32{ 32, 64, 128, 256, 512 }) |head_dim| { + for ([_][2]i32{ .{ 1, 1 }, .{ 4, 1 }, .{ 2, 8 }, .{ 4, 128 } }) |launch| { + allocator.free(try FaVecPaged.Kernel.emit(allocator, .{ .head_dim = head_dim, .block_size = 16, .position_stride = 2 * head_dim, .heads = launch[0], .splits = launch[1] })); + } + for ([_]i32{ 8, 128 }) |splits| { + allocator.free(try FaVecPagedReduce.Kernel.emit(allocator, .{ .splits = splits, .head_dim = head_dim })); } } - allocator.free(try FaVecPaged.Kernel.emit(allocator, .{ .head_dim = 512, .nsg = 8, .block_size = 16, .position_stride = 1024, .nwg = 4 })); - allocator.free(try FaVecPagedReduce.Kernel.emit(allocator, .{ .nwg = 4, .head_dim = 512 })); for ([_]Type{ .half, .bfloat }) |dtype| { allocator.free(try ZeroPaddingRows.Kernel.emit(allocator, .{ .dtype = dtype })); for ([_]i32{ 32, 64, 96, 128, 256, 512 }) |head_size| { From 2b604ebfc64eccbcbf3703674b852f2ab9b82e78 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Tue, 10 Nov 2026 18:51:08 +0100 Subject: [PATCH 867/881] zml/kernels/flash_attention/metal: port the kernels to the Metal DSL (#1054) Replaces the `zml$flash_attn` custom call with Metal DSL kernels over contiguous K/V: - `fa_vec` + `fa_vec_reduce` for bf16 decode, at most 256 threads per threadgroup. - `fa_tiled` for prefill and f16. - Batched queries work (the old path panicked). Other dtypes and head dims fall back to sdpa. Bumps the plugins to [202611101651.363.1.d46c7471c419](https://github.com/zml/xla-private/tree/202611101651.363.1.d46c7471c419) --- platforms/cpu/cpu.bzl | 6 +- platforms/cuda/cuda.bzl | 8 +- platforms/furiosa/furiosa.bzl | 4 +- platforms/metal/metal.bzl | 4 +- platforms/rocm/rocm.bzl | 4 +- zml/kernels/flash_attention/metal/BUILD.bazel | 5 +- .../flash_attention/metal/flash_attn.zig | 733 ++++++++++++++++++ zml/kernels/flash_attention/metal/metal.zig | 129 ++- zml/kernels/flash_attention/tests.zig | 16 + 9 files changed, 883 insertions(+), 26 deletions(-) create mode 100644 zml/kernels/flash_attention/metal/flash_attn.zig diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index fd070b6e..08817afe 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -43,21 +43,21 @@ def _cpu_plugin_impl(mctx): name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, sha256 = "3d6a6d1b047a0943b50fd9e63e5f6dff6b7cf184b186c0cc1052de40f0dbec92", - url = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-cpu-linux-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cpu-linux-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "d6b4b15b13a2d1a3e0b84f143f568cd572bef5db91784f9f7a826173ed626c7e", - url = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-cpu-darwin-amd64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cpu-darwin-amd64.tar.zst", ) http_archive( name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, sha256 = "c06babc001edeab313bc63daab3aa3d40e087266a3a29634d182628fb65fad14", - url = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-cpu-darwin-arm64.tar.zst", + url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 24718ee6..8482e52a 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -224,12 +224,12 @@ _UBUNTU_PACKAGES = { _PLUGINS = { "amd64": { - "sha256": "1ae8a7c8fc677e0f10c0032ce9ae35e5b6c3f5eadc3f54e1bbda63c219b0170d", - "url": "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-cuda-linux-amd64.tar.zst", + "sha256": "72b1359384ac50e727b800e81c56d7134f9d6a061a982c4e36b7fef62cf7c331", + "url": "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "14a3567c629a1ea97f1dcfa2cb47b54ea08d8c4b9e35eae47a7eff33cfb53ce0", - "url": "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-cuda-linux-arm64.tar.zst", + "sha256": "caeefd717960fa2acec6304cd8296299e262abdba669ecf4d487e3af76d73a7b", + "url": "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cuda-linux-arm64.tar.zst", }, } diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index ba53bfb4..81518d53 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-furiosa-linux-amd64.tar.zst", - sha256 = "514b1c96a2237380a9b9e011f5b994c6617eb103a6a9dac8ff0a7f1832d1a3bd", + url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-furiosa-linux-amd64.tar.zst", + sha256 = "6da755570140c9f8959ba477fef3df4c48b0619274c97429d204e177b8649e12", ) return mctx.extension_metadata( diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl index 767ef2f6..71e1858a 100644 --- a/platforms/metal/metal.bzl +++ b/platforms/metal/metal.bzl @@ -4,8 +4,8 @@ def _metal_impl(mctx): http_archive( name = "libzml_metal", build_file = "libzml_metal.BUILD.bazel", - sha256 = "bdd2de3ec8293faa25b358cdf35ee3e75d69be73a4acea87b6e007a18b8f10e8", - url = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-metal-darwin-arm64.tar.zst", + sha256 = "89881d1a1db336e3337a9373db2c4e94f71ddfb77619dee75c0c12e2903913ae", + url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-metal-darwin-arm64.tar.zst", ) return mctx.extension_metadata( diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 454bf626..41b01b60 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -11,8 +11,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") _ROCM_VERSION = "10.1" _ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202611060925.355.1.2ca211fa5fbd/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "02b86fa3a6599f6aa2b89806b8f6c0b16f11765ec04c94517ec1463a59d92175" +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "cdc4434f2023842f0e1a34c08aaed87f082df19f92c4ae21c9d4452a9739b604" def _rocm_package_name(name): return name + _ROCM_VERSION diff --git a/zml/kernels/flash_attention/metal/BUILD.bazel b/zml/kernels/flash_attention/metal/BUILD.bazel index 855bdcca..f82ade45 100644 --- a/zml/kernels/flash_attention/metal/BUILD.bazel +++ b/zml/kernels/flash_attention/metal/BUILD.bazel @@ -2,5 +2,8 @@ package(default_visibility = ["//zml:__pkg__"]) filegroup( name = "metal", - srcs = ["metal.zig"], + srcs = [ + "flash_attn.zig", + "metal.zig", + ], ) diff --git a/zml/kernels/flash_attention/metal/flash_attn.zig b/zml/kernels/flash_attention/metal/flash_attn.zig new file mode 100644 index 00000000..4faa1e82 --- /dev/null +++ b/zml/kernels/flash_attention/metal/flash_attn.zig @@ -0,0 +1,733 @@ +//! Metal flash attention over contiguous `[batch, keys, kv_heads, head_dim]` +//! K/V, causal from `token_index`: fa_vec and fa_vec_reduce for decode, +//! fa_tiled otherwise. + +const std = @import("std"); + +const kernel = @import("../../../kernel.zig"); + +const metal = kernel.metal; +const ArgSpec = metal.ArgSpec; +const Builder = metal.Builder; +const Type = metal.Type; +const Value = metal.Value; + +const flt_max = std.math.floatMax(f32); +/// A masked score: below any real one yet finite, since fast math assumes +/// no infinities. Halved so subtracting a running maximum cannot overflow. +const masked_score = -flt_max / 2.0; +const max_half = 65504.0; +const simd_width = 32; + +/// fa_vec: bf16 decode, one query per batch row. Simdgroup s takes query head +/// s % heads of the threadgroup's KV head group and position split s / heads. +pub const FaVec = struct { + pub const Cfg = struct { + head_dim: i32, + heads: i32, + splits: i32, + }; + + pub const Args: metal.Struct = .{ .name = "fa_vec_args", .fields = &.{ + .{ .name = "num_heads", .type = .int }, + .{ .name = "n_kv", .type = .int }, + .{ .name = "k_len", .type = .int }, + .{ .name = "scale", .type = .float }, + } }; + + pub const HostArgs = extern struct { + num_heads: i32, + n_kv: i32, + k_len: i32, + scale: f32, + }; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "fa_vec", + .inputs = &.{ "q", "k", "v", "token_index" }, + .outputs = &.{"dst"}, + .constants = &.{"args"}, + .run = run, + }); + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + switch (cfg.head_dim) { + 32, 64, 128, 256, 512 => {}, + else => std.debug.panic("fa_vec: head_dim {d} unsupported (32, 64, 128, 256 or 512)", .{cfg.head_dim}), + } + const char_in: ArgSpec = .{ .buffer = .{ .type = .char, .const_ = true } }; + const a = try b.declareArgs(.{ + .args = .{ .constant_struct = Args }, + .q = char_in, + .k = char_in, + .v = char_in, + .token_index = .{ .buffer = .{ .type = .int, .const_ = true } }, + .dst = .{ .buffer = .{ .type = .char } }, + .tgpig = .{ .threadgroup_position_in_grid = .uint3 }, + .tgpg = .{ .threadgroups_per_grid = .uint3 }, + .tiisg = .{ .thread_index_in_simdgroup = .ushort }, + .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, + .nsg = .{ .simdgroups_per_threadgroup = .ushort }, + }); + impl(b, cfg, a); + } + + fn impl(b: *Builder, cfg: Cfg, a: anytype) void { + const D = cfg.head_dim; + const D4 = @divExact(D, 4); + // A K/V row spans NL lanes of NI float4s; a simdgroup reads R rows per step. + const NL = @min(D4, simd_width); + const R = @divExact(simd_width, NL); + const NI = @divExact(D4, NL); + const C = simd_width; + const HG = cfg.heads; + const NWG = cfg.splits; + const full: metal.LoopOpts = .{ .unroll = .full }; + + const num_heads = a.args.field("num_heads"); + const n_kv = a.args.field("n_kv"); + const group = num_heads.div(n_kv); + const groups = group.div(HG); + const r = a.tgpig.at(2).to(.int); + const tg_head = a.tgpig.at(1).to(.int); + const ikv = tg_head.div(groups); + const iq2 = ikv.mul(group).add(tg_head.rem(groups).mul(HG)).add(a.sgitg.rem(HG).to(.int)); + const iwg = a.tgpig.at(0).to(.int).mul(a.nsg.div(HG).to(.int)).add(a.sgitg.div(HG).to(.int)); + const rid = r.to(.long).mul(num_heads.to(.uint)).add(iq2); + const nrows = a.tgpg.at(2).to(.long).mul(num_heads.to(.uint)); + + const kv_len = a.token_index.load(r).add(1); + const live = if (NWG > 1) liveSplits(b, kv_len, NWG) else b.lift(@as(i32, 1)); + + if (NWG > 1) { + var idle = b.openIf(iwg.ge(live)); + { + b.return_(); + idle.close(); + } + } + + const position_bytes = n_kv.to(.ulong).mul(D * 2); + const first = r.to(.ulong).mul(a.args.field("k_len").to(.ulong)).mul(position_bytes).add(ikv.to(.ulong).mul(D * 2)); + const k0 = a.k.add(first); + const v0 = a.v.add(first); + + const lane = if (R == 1) a.tiisg else a.tiisg.rem(NL); + const sub = a.tiisg.div(NL).to(.int); + const q4 = a.q.add(rid.mul(D).mul(2)).to(.bfloat4).add(lane); + const qr = b.array(.float4, @intCast(NI)); + const o = b.array(.float4, @intCast(NI)); + var init = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const ii = init.iv; + qr.store(ii, q4.load(ii.mul(NL)).to(.float4)); + o.store(ii, b.constant(.float4, 0)); + init.close(); + } + + const S = b.local(.float, 0.0); + const M = b.local(.float, -flt_max / 2.0); + + const ic0 = b.local(.int, iwg); + var blocks = b.openLoop(.{}); + { + const ic = ic0.mul(C); + var past = b.openIf(ic.ge(kv_len)); + { + blocks.breakLoop(); + past.close(); + } + + const s = b.local(.float, 0.0); + var qk = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); + { + const cc = qk.iv; + const p = b.min(if (R == 1) ic.add(cc) else ic.add(cc.to(.int).mul(R)).add(sub), kv_len.sub(1)); + const pk4 = k0.add(p.to(.ulong).mul(position_bytes)).to(.bfloat4).add(lane); + const d = b.local(.float, 0.0); + var dots = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const ii = dots.iv; + d.assign(d.add(b.dot(pk4.load(ii.mul(NL)).to(.float4), qr.load(ii)))); + dots.close(); + } + if (R == 1) { + s.assign(b.ternary(a.tiisg.to(.short).eq(cc), b.simdSum(d.rvalue()), s.rvalue())); + } else { + var x = d.rvalue(); + var mask: i32 = 1; + while (mask < NL) : (mask *= 2) x = x.add(b.simdShuffleXor(x, b.constant(.ushort, mask))); + const t = b.simdShuffle(x, a.tiisg.rem(R).mul(NL)); + s.assign(b.ternary(a.tiisg.div(R).to(.short).eq(cc), t, s.rvalue())); + } + qk.close(); + } + + const pos = ic.add(a.tiisg.to(.int)); + const score = b.ternary(pos.lt(kv_len), s.mul(a.args.field("scale")), b.constant(.float, -max_half)); + + const m = M.rvalue(); + M.assign(b.simdMax(b.max(M.rvalue(), score))); + const ms = b.exp(m.sub(M)); + const vs = b.exp(score.sub(M)); + S.assign(S.mul(ms).add(b.simdSum(vs))); + + var rescale = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + o.store(rescale.iv, o.load(rescale.iv).mul(ms)); + rescale.close(); + } + + var pv = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); + { + const cc = pv.iv; + const row = cc.to(.int).mul(R).add(sub); + const p = b.min(if (R == 1) ic.add(cc) else ic.add(row), kv_len.sub(1)); + const pv4 = v0.add(p.to(.ulong).mul(position_bytes)).to(.bfloat4).add(lane); + const w = b.vec(.float4, .{if (R == 1) b.simdBroadcast(vs, cc.to(.ushort)) else b.simdShuffle(vs, row)}); + var dims = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const ii = dims.iv; + o.store(ii, o.load(ii).add(pv4.load(ii.mul(NL)).to(.float4).mul(w))); + dims.close(); + } + pv.close(); + } + + blocks.next(); + ic0.assign(ic0.add(live)); + blocks.close(); + } + + var mask: i32 = NL; + while (mask < simd_width) : (mask *= 2) { + var sum = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + o.store(sum.iv, o.load(sum.iv).add(b.simdShuffleXor(o.load(sum.iv), b.constant(.ushort, mask)))); + sum.close(); + } + } + var first_row = if (R > 1) b.openIf(sub.eq(0)) else null; + if (NWG == 1) { + const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); + const dst4 = a.dst.to(.bfloat4).add(rid.mul(D4)).add(lane); + var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + dst4.store(out.iv.mul(NL), o.load(out.iv).mul(inv_sum).to(.bfloat4)); + out.close(); + } + } else { + // [rows][D4][splits] float4 partials, then [rows][splits] (S, M). + const dst4 = a.dst.to(.float4).add(rid.mul(D4).mul(NWG)); + var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const i = lane.to(.int).add(out.iv.to(.int).mul(NL)); + dst4.store(i.mul(NWG).add(iwg), o.load(out.iv)); + out.close(); + } + var lane0 = b.openIf(a.tiisg.eq(0)); + { + const dst1 = a.dst.to(.float).add(nrows.mul(D).mul(NWG)).add(rid.mul(2 * NWG)); + dst1.store(b.lift(2).mul(iwg), S); + dst1.store(b.lift(2).mul(iwg).add(1), M); + lane0.close(); + } + } + if (first_row) |*scope| scope.close(); + } +}; + +fn liveSplits(b: *Builder, kv_len: Value, splits: i32) Value { + return b.min(splits, kv_len.add(simd_width - 1).div(simd_width)); +} + +/// fa_vec_reduce: merges the fa_vec split partials. +pub const FaVecReduce = struct { + pub const Cfg = struct { + splits: i32, + head_dim: i32, + }; + + pub const simdgroups = 8; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "fa_vec_reduce", + .inputs = &.{ "htmp", "token_index" }, + .outputs = &.{"dst"}, + .constants = &.{"args"}, + .run = run, + }); + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + const a = try b.declareArgs(.{ + .args = .{ .constant_struct = FaVec.Args }, + .htmp = .{ .buffer = .{ .type = .char, .const_ = true } }, + .token_index = .{ .buffer = .{ .type = .int, .const_ = true } }, + .dst = .{ .buffer = .{ .type = .char } }, + .tgpig = .{ .threadgroup_position_in_grid = .uint2 }, + .tgpg = .{ .threadgroups_per_grid = .uint2 }, + .tiisg = .{ .thread_index_in_simdgroup = .ushort }, + .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, + }); + const NWG = cfg.splits; + if (NWG > 4 * simd_width) std.debug.panic("fa_vec_reduce: {d} splits, at most {d}", .{ NWG, 4 * simd_width }); + const NP = @divFloor(NWG + simd_width - 1, simd_width); + const DV = cfg.head_dim; + const DV4 = @divExact(DV, 4); + const full: metal.LoopOpts = .{ .unroll = .full }; + + const rid = a.tgpig.at(0).to(.ulong); + const r = a.tgpig.at(0).div(a.args.field("num_heads").to(.uint)).to(.int); + const i = a.tgpig.at(1).to(.int).mul(simdgroups).add(a.sgitg.to(.int)); + const dst4 = a.dst.to(.bfloat4).add(rid.mul(DV4)); + const live = liveSplits(b, a.token_index.load(r).add(1), NWG); + + const ss = a.htmp.to(.float).add(a.tgpg.at(0).to(.ulong).mul(DV).mul(NWG)).add(rid.mul(2 * NWG)); + const Mj = b.array(.float, @intCast(NP)); + const Sj = b.array(.float, @intCast(NP)); + const mx = b.local(.float, -flt_max / 2.0); + var load = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = load.iv; + const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); + const ok = idx.lt(live); + Mj.store(j, b.ternary(ok, ss.load(idx.mul(2).add(1)), b.constant(.float, -flt_max / 2.0))); + Sj.store(j, b.ternary(ok, ss.load(idx.mul(2)), b.constant(.float, 0))); + mx.assign(b.max(mx.rvalue(), Mj.load(j))); + load.close(); + } + const m = b.simdMax(mx.rvalue()); + const sum = b.local(.float, 0.0); + var weigh = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = weigh.iv; + Mj.store(j, b.exp(Mj.load(j).sub(m))); + sum.assign(sum.add(Sj.load(j).mul(Mj.load(j)))); + weigh.close(); + } + const S = b.simdSum(sum.rvalue()); + const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); + + const htmp4 = a.htmp.to(.float4).add(rid.mul(DV4).mul(NWG)); + const acc = b.local(.float4, b.constant(.float4, 0)); + var parts = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = parts.iv; + const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); + var ok = b.openIf(idx.lt(live)); + { + acc.assign(acc.add(htmp4.load(i.mul(NWG).add(idx)).mul(Mj.load(j)))); + ok.close(); + } + parts.close(); + } + const v = b.simdSum(acc.rvalue()); + var lane0 = b.openIf(a.tiisg.eq(0)); + { + dst4.store(i, v.mul(inv_sum).to(.bfloat4)); + lane0.close(); + } + } +}; + +/// fa_tiled: causal FlashAttention-2 over simdgroup matrices. Queries past +/// `live` output zeros. +pub const FaTiled = struct { + pub const Cfg = struct { + dtype: Type, + head_size: i32, + bq: i32 = 32, + tile_kv: i32 = 32, + num_threads: i32 = 128, + }; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "fa_tiled", + .inputs = &.{ "q", "k", "v", "token_index", "live" }, + .outputs = &.{"out"}, + .constants = &.{ "num_kv_heads", "scale", "q_len", "k_len" }, + .run = run, + }); + + fn supported(cfg: Cfg) bool { + const tile = tileConfig(cfg.head_size) orelse return false; + return (cfg.dtype.eql(.half) or cfg.dtype.eql(.bfloat)) and + cfg.bq == tile[0] and cfg.tile_kv == tile[1] and cfg.num_threads == tile[2]; + } + + pub fn tileConfig(head_size: i64) ?[3]i32 { + return switch (head_size) { + 32, 64, 96, 128 => .{ 32, 32, 128 }, + 256 => .{ 16, 16, 64 }, + 512 => .{ 8, 8, 32 }, + else => null, + }; + } + + pub fn threadgroupMemory(cfg: Cfg) u64 { + const ld = cfg.head_size + @divExact(16, @as(i32, @intCast(cfg.dtype.size()))); + return @intCast((cfg.bq + 2 * cfg.tile_kv) * ld * @as(i32, @intCast(cfg.dtype.size()))); + } + + /// Lane `lane_id`'s (column, row) in an 8x8 simdgroup_matrix fragment. + fn fragCoord(b: *Builder, lane_id: Value) Value { + const qid = lane_id.to(.short).div(4).to(.short); + const fm = qid.bitAnd(4).add(lane_id.to(.short).div(2).rem(4)).to(.short); + const fn_ = qid.bitAnd(2).mul(2).add(lane_id.to(.short).rem(2).mul(2)).to(.short); + return b.vec(.short2, .{ fn_, fm }); + } + + fn fragRowReduce(b: *Builder, comptime op: enum { max, sum }, v: Value) Value { + var x = v.rvalue(); + inline for (.{ 1, 8 }) |mask| { + const y = b.simdShuffleXor(x, b.constant(.ushort, mask)); + x = switch (op) { + .max => b.max(x, y), + .sum => x.add(y), + }; + } + return x; + } + + fn setLane(p: Value, i: Value, lane: anytype, x: Value) void { + p.store(i, p.load(i).insert(lane, x)); + } + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + if (!supported(cfg)) std.debug.panic("fa_tiled: {s} at head size {d} and tiles {d}, {d}, {d} unsupported", .{ cfg.dtype.name(), cfg.head_size, cfg.bq, cfg.tile_kv, cfg.num_threads }); + const T = cfg.dtype; + const HEAD_SIZE = cfg.head_size; + const BQ = cfg.bq; + const TILE_KV = cfg.tile_kv; + const NUM_THREADS = cfg.num_threads; + const int_arg: ArgSpec = .{ .constant = .int }; + const int_in: ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; + const a = try b.declareArgs(.{ + .out = .{ .buffer = .{ .type = T } }, + .q = .{ .buffer = .{ .type = T, .const_ = true } }, + .k = .{ .buffer = .{ .type = T, .const_ = true } }, + .v = .{ .buffer = .{ .type = T, .const_ = true } }, + .token_index = int_in, + .live = int_in, + .num_kv_heads = int_arg, + .scale = .{ .constant = .float }, + .q_len = int_arg, + .k_len = int_arg, + .shared_mem = .{ .threadgroup = .char }, + .tgp = .{ .threadgroup_position_in_grid = .uint3 }, + .tgpg = .{ .threadgroups_per_grid = .uint3 }, + .tpt = .{ .thread_position_in_threadgroup = .uint3 }, + .sg_idx = .{ .simdgroup_index_in_threadgroup = .uint }, + .lane = .{ .thread_index_in_simdgroup = .uint }, + }); + + const NUM_SIMD_LANES = 32; + const NUM_SG = @divExact(NUM_THREADS, NUM_SIMD_LANES); + const TD = @divExact(HEAD_SIZE, 8); + const TK = @divExact(TILE_KV, 8); + if (@divExact(BQ, NUM_SG) != 8) @panic("fa_tiled: ROWS_PER_SG must be 8"); + const unroll: metal.LoopOpts = .{ .unroll = .enable }; + + const thread_idx = a.tpt.x().to(.int); + const head_idx = a.tgp.x().to(.int); + const q_pos_start = a.tgp.y().to(.int).mul(BQ); + const batch = a.tgp.z().to(.int); + const num_heads = a.tgpg.x().to(.int); + const kv_head_idx = head_idx.div(num_heads.div(a.num_kv_heads)); + const q_stride = num_heads.mul(HEAD_SIZE); + const kv_token_stride = a.num_kv_heads.mul(HEAD_SIZE); + + const block_rows = b.min(BQ, a.q_len.sub(q_pos_start)); + const live = a.live.load(0); + const valid_q = b.min(block_rows, b.max(0, live.sub(q_pos_start))); + const out_base = a.out.add(batch.mul(a.q_len).add(q_pos_start).to(.long).mul(q_stride).add(head_idx.mul(HEAD_SIZE))); + + var dead = b.openIf(valid_q.le(0)); + { + var zero = b.openFor(thread_idx, block_rows.mul(HEAD_SIZE), NUM_THREADS); + { + const i = zero.iv; + out_base.store(i.div(HEAD_SIZE).mul(q_stride).add(i.rem(HEAD_SIZE)), b.constant(T, 0)); + zero.close(); + } + b.return_(); + dead.close(); + } + + // Query i of the batch row is at position context_len + i. + const context_len = a.token_index.load(batch); + const seq_len = context_len.add(live); + const kv_base = batch.to(.long).mul(a.k_len).mul(kv_token_stride).add(kv_head_idx.mul(HEAD_SIZE)); + + const SMEM_PAD = @divExact(16, @as(i32, @intCast(T.size()))); + const LD = HEAD_SIZE + SMEM_PAD; + const Q_ELEMS = BQ * LD; + const KV_ELEMS = TILE_KV * LD; + + const Q_smem = a.shared_mem.to(T); + const K_smem = Q_smem.add(Q_ELEMS); + const V_smem = K_smem.add(KV_ELEMS); + + const log2e: f32 = std.math.log2e; + const scale_log2 = a.scale.mul(log2e); + + const q_base = a.q.add(batch.mul(a.q_len).add(q_pos_start).to(.long).mul(q_stride).add(head_idx.mul(HEAD_SIZE))); + var load_q = b.openFor(thread_idx, BQ * HEAD_SIZE, NUM_THREADS); + { + const i = load_q.iv; + const r = i.div(HEAD_SIZE); + const d = i.rem(HEAD_SIZE); + var value = b.openIfElse(r.lt(valid_q), .{T}); + { + value.yieldThen(.{q_base.load(r.mul(q_stride).add(d))}); + } + { + value.yieldElse(.{b.constant(T, 0)}); + } + Q_smem.store(r.mul(LD).add(d), value.results[0]); + load_q.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + const T2: Type = .vec(T.scalar, 2); + const q_reg = b.array(T2, @intCast(TD)); + var load_q_reg = b.openForOpts(0, TD, 1, unroll); + { + const d = load_q_reg.iv; + const tmp = b.simdgroupLoad(Q_smem.add(a.sg_idx.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); + q_reg.store(d, tmp.threadElements()); + load_q_reg.close(); + } + + const fc = fragCoord(b, a.lane.to(.ushort)); + const fm = fc.y(); + const fn_ = fc.x(); + + const max_score = b.local(.float, masked_score); + const sum_score = b.local(.float, 0.0); + + const o_reg = b.array(.float2, @intCast(TD)); + var zero_o = b.openForOpts(0, TD, 1, unroll); + { + o_reg.store(zero_o.iv, b.constant(.float2, 0)); + zero_o.close(); + } + + const num_kv_tiles = seq_len.add(TILE_KV).sub(1).div(TILE_KV); + const last_q_pos = context_len.add(q_pos_start).add(valid_q).sub(1); + + var tiles = b.openFor(0, num_kv_tiles, 1); + { + const tile_idx = tiles.iv; + const tile_start = tile_idx.mul(TILE_KV); + + var past = b.openIf(tile_start.gt(last_q_pos)); + { + tiles.breakLoop(); + past.close(); + } + + const VEC_BYTES: i32 = if (HEAD_SIZE >= 256) 16 else 8; + const VEC = @divExact(VEC_BYTES, @as(i32, @intCast(T.size()))); + const LoadUnit: Type = if (VEC_BYTES == 16) .uint4 else .uint2; + std.debug.assert(@rem(LD, VEC) == 0); + var rows = b.openForOpts(0, @divExact(TILE_KV, NUM_SG), 1, unroll); + { + const t = a.sg_idx.mul(@divExact(TILE_KV, NUM_SG)).add(rows.iv).to(.int); + const kv_pos = tile_start.add(t); + var present = b.openIfElse(kv_pos.lt(seq_len), .{}); + { + const off = kv_base.add(kv_pos.to(.long).mul(kv_token_stride)); + const k_ptr = a.k.add(off); + const v_ptr = a.v.add(off); + var loop = b.openForOpts(a.lane.mul(VEC).to(.int), HEAD_SIZE, NUM_SIMD_LANES * VEC, unroll); + { + const d = loop.iv; + K_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, k_ptr.add(d).to(LoadUnit).load(0)); + V_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, v_ptr.add(d).to(LoadUnit).load(0)); + loop.close(); + } + present.elseBranch(); + } + { + const zero = b.constant(LoadUnit, 0); + var loop = b.openForOpts(a.lane.mul(VEC).to(.int), HEAD_SIZE, NUM_SIMD_LANES * VEC, unroll); + { + const d = loop.iv; + K_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, zero); + V_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, zero); + loop.close(); + } + present.close(); + } + rows.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + const s_reg = b.array(.float2, @intCast(TK)); + var zero_s = b.openForOpts(0, TK, 1, unroll); + { + s_reg.store(zero_s.iv, b.constant(.float2, 0)); + zero_s.close(); + } + + var qk_dims = b.openForOpts(0, TD, 1, unroll); + { + const d = qk_dims.iv; + const q_frag = b.matrix(T.scalar, q_reg.load(d)); + var qk_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = qk_keys.iv; + const k_frag = b.simdgroupLoad(K_smem.add(k.mul(8).mul(LD)).add(d.mul(8)), LD, .{ .transpose = true }); + const s_frag = b.matrix(.float, s_reg.load(k)); + s_reg.store(k, b.simdgroupMultiplyAccumulate(q_frag, k_frag, s_frag).threadElements()); + qk_keys.close(); + } + qk_dims.close(); + } + + const q_abs_pos = context_len.add(q_pos_start).add(a.sg_idx.mul(8)).add(fm).to(.int); + const row_masked = a.sg_idx.mul(8).add(fm).ge(valid_q); + const tile_no_mask = tile_start.add(TILE_KV).le(seq_len).bitAnd(tile_start.add(TILE_KV).sub(1).lt(context_len.add(q_pos_start))); + + var masking = b.openIfElse(tile_no_mask.bitAnd(row_masked.eq(false)), .{}); + { + var scale = b.openForOpts(0, TK, 1, unroll); + { + const k = scale.iv; + setLane(s_reg, k, 0, s_reg.load(k).at(0).mul(scale_log2)); + setLane(s_reg, k, 1, s_reg.load(k).at(1).mul(scale_log2)); + scale.close(); + } + masking.elseBranch(); + } + { + var keys = b.openForOpts(0, TK, 1, unroll); + { + const k = keys.iv; + var pair = b.openForOpts(0, 2, 1, unroll); + { + const jj = pair.iv; + const s = s_reg.load(k).at(jj).mul(scale_log2); + const kv_pos = tile_start.add(k.mul(8)).add(fn_).add(jj); + const masked = row_masked.bitOr(kv_pos.ge(seq_len)).bitOr(kv_pos.gt(q_abs_pos)); + setLane(s_reg, k, jj, b.ternary(masked, masked_score, s)); + pair.close(); + } + keys.close(); + } + masking.close(); + } + + const local_max = b.local(.float, masked_score); + var max_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = max_keys.iv; + local_max.assign(b.max(local_max.rvalue(), b.max(s_reg.load(k).at(0), s_reg.load(k).at(1)))); + max_keys.close(); + } + const row_max = fragRowReduce(b, .max, local_max); + + const prev_max = max_score.rvalue(); + const grown_max = b.max(prev_max, row_max); + const grew = grown_max.gt(prev_max); + const was_empty = prev_max.eq(masked_score); + const factor = b.ternary(grew, b.ternary(was_empty, 0.0, b.fast().exp2(prev_max.sub(grown_max))), 1.0); + const new_max = b.ternary(grew.eq(false).bitAnd(was_empty), 0.0, grown_max); + // A tile masking a whole row leaves the maximum masked: zero would + // pin it, and later scores far below zero would underflow. + max_score.assign(grown_max); + + const local_sum = b.local(.float, 0.0); + var p_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = p_keys.iv; + var pair = b.openForOpts(0, 2, 1, unroll); + { + const jj = pair.iv; + const s = s_reg.load(k).at(jj); + const p = b.ternary(s.eq(masked_score), 0.0, b.fast().exp2(s.sub(new_max))); + setLane(s_reg, k, jj, p); + local_sum.assign(local_sum.add(p)); + pair.close(); + } + p_keys.close(); + } + const row_sum = fragRowReduce(b, .sum, local_sum); + + sum_score.assign(sum_score.mul(factor).add(row_sum)); + + var rescale_o = b.openForOpts(0, TD, 1, unroll); + { + o_reg.store(rescale_o.iv, o_reg.load(rescale_o.iv).mul(factor)); + rescale_o.close(); + } + + var pv_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = pv_keys.iv; + const p_frag = b.matrix(.float, s_reg.load(k)); + var pv_dims = b.openForOpts(0, TD, 1, unroll); + { + const d = pv_dims.iv; + const v_frag = b.simdgroupLoad(V_smem.add(k.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); + const o_frag = b.matrix(.float, o_reg.load(d)); + o_reg.store(d, b.simdgroupMultiplyAccumulate(p_frag, v_frag, o_frag).threadElements()); + pv_dims.close(); + } + pv_keys.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + tiles.close(); + } + + const inv_sum = b.lift(1.0).div(sum_score.add(@as(f32, 1e-6))); + var normalize = b.openForOpts(0, TD, 1, unroll); + { + o_reg.store(normalize.iv, o_reg.load(normalize.iv).mul(inv_sum)); + normalize.close(); + } + + const O_smem = Q_smem.to(.float); + var store_o = b.openForOpts(0, TD, 1, unroll); + { + const d = store_o.iv; + const o_frag = b.matrix(.float, o_reg.load(d)); + b.simdgroupStore(o_frag, O_smem.add(a.sg_idx.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); + store_o.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + var write_out = b.openFor(thread_idx, block_rows.mul(HEAD_SIZE), NUM_THREADS); + { + const i = write_out.iv; + const r = i.div(HEAD_SIZE); + const d = i.rem(HEAD_SIZE); + const value = b.ternary(r.lt(valid_q), O_smem.load(r.mul(LD).add(d)).to(T), b.constant(T, 0)); + out_base.store(r.mul(q_stride).add(d), value); + write_out.close(); + } + } +}; + +test "the flash attention kernels emit verified modules" { + const allocator = std.testing.allocator; + for ([_]i32{ 32, 64, 128, 256, 512 }) |head_dim| { + for ([_][2]i32{ .{ 1, 1 }, .{ 4, 1 }, .{ 2, 8 }, .{ 4, 128 } }) |launch| { + allocator.free(try FaVec.Kernel.emit(allocator, .{ .head_dim = head_dim, .heads = launch[0], .splits = launch[1] })); + } + for ([_]i32{ 8, 128 }) |splits| { + allocator.free(try FaVecReduce.Kernel.emit(allocator, .{ .splits = splits, .head_dim = head_dim })); + } + } + for ([_]Type{ .half, .bfloat }) |dtype| { + for ([_]i32{ 32, 64, 96, 128, 256, 512 }) |head_size| { + const tile = FaTiled.tileConfig(head_size).?; + allocator.free(try FaTiled.Kernel.emit(allocator, .{ .dtype = dtype, .head_size = head_size, .bq = tile[0], .tile_kv = tile[1], .num_threads = tile[2] })); + } + } +} diff --git a/zml/kernels/flash_attention/metal/metal.zig b/zml/kernels/flash_attention/metal/metal.zig index 9327c2a1..bdfb00c0 100644 --- a/zml/kernels/flash_attention/metal/metal.zig +++ b/zml/kernels/flash_attention/metal/metal.zig @@ -1,12 +1,10 @@ const std = @import("std"); -const stdx = @import("stdx"); - const zml = @import("../../../zml.zig"); - -const log = std.log.scoped(.@"zml/attention/metal"); +const flash_attn = @import("flash_attn.zig"); pub const Metadata = struct { + /// Live queries of a single-sequence prefill; 0 means all. num_tokens: zml.Tensor, pub fn init() Metadata { @@ -23,16 +21,123 @@ pub const Metadata = struct { } }; +/// Causal attention: fa_vec for bf16 decode, fa_tiled otherwise, sdpa for +/// unsupported dtypes and head sizes. pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata) zml.Tensor { - const qc = q.transpose(.{ .h, .q, .hd }); - const kc = k.transpose(.{ .h, .k, .hd }); - const vc = v.transpose(.{ .h, .k, .hd }); - const tok_i32 = token_index.convert(.i32); + const dtype = q.dtype(); + const head_dim = q.dim(.hd); + if (!((dtype == .bf16 or dtype == .f16) and k.dtype() == dtype and v.dtype() == dtype and flash_attn.FaTiled.tileConfig(head_dim) != null)) { + const seq_len = k.dim(.k); + const mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, dtype, null) + .gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, dtype), token_index.appendAxes(.{.coord}), .{}); + return zml.nn.sdpa(q, k, v, .{ .attn_mask = mask }); + } - const attn = if (q.dim(.q) > 1) - zml.ops.customCall("zml$flash_attn", .{ qc, kc, vc, tok_i32, metadata.num_tokens }, qc.shape(), .{}, .{ .has_side_effect = false }) + const batched = q.shape().hasTag(.b) != null; + const qb = if (batched) q.transpose(.{ .b, .q, .h, .hd }) else q.transpose(.{ .q, .h, .hd }).insertAxes(0, .{.b}); + const kb = if (batched) k.transpose(.{ .b, .k, .h, .hd }) else k.transpose(.{ .k, .h, .hd }).insertAxes(0, .{.b}); + const vb = if (batched) v.transpose(.{ .b, .k, .h, .hd }) else v.transpose(.{ .k, .h, .hd }).insertAxes(0, .{.b}); + const positions = token_index.convert(.i32).broad(.init(.{ .b = qb.dim(.b) }, .i32)); + const scale = 1.0 / @sqrt(@as(f32, @floatFromInt(head_dim))); + + const vec_head_dim = head_dim == 32 or head_dim == 64 or head_dim == 128 or head_dim == 256 or head_dim == 512; + const out = if (qb.dim(.q) == 1 and dtype == .bf16 and vec_head_dim) + decode(qb, kb, vb, positions, scale) else - zml.ops.customCall("zml$flash_attn", .{ qc, kc, vc, tok_i32 }, qc.shape(), .{}, .{ .has_side_effect = false }); + prefill(qb, kb, vb, positions, liveQueries(qb, metadata), scale); + return (if (batched) out else out.squeeze(.b)).transpose(q.shape()); +} - return attn.transpose(q.shape()); +fn liveQueries(q: zml.Tensor, metadata: Metadata) zml.Tensor { + const all: zml.Tensor = .scalar(q.dim(.q), .i32); + if (q.dim(.b) != 1 or q.dim(.q) == 1) return all; + const num_tokens = metadata.num_tokens.convert(.i32); + return num_tokens.cmp(.GE, .scalar(1, .i32)).select(num_tokens.minimum(all), all); +} + +/// `q` is `[b, 1, h, hd]`, `k` and `v` `[b, k, kv_h, hd]`. +fn decode(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, positions: zml.Tensor, scale: f32) zml.Tensor { + const batch: i32 = @intCast(q.dim(.b)); + const num_heads: i32 = @intCast(q.dim(.h)); + const head_dim: i32 = @intCast(q.dim(.hd)); + const num_kv_heads: i32 = @intCast(k.dim(.h)); + const keys: i32 = @intCast(k.dim(.k)); + const nrows = batch * num_heads; + const launch = vecLaunch(nrows, @divExact(num_heads, num_kv_heads), keys, coreCount()); + const args: flash_attn.FaVec.HostArgs = .{ .num_heads = num_heads, .n_kv = num_kv_heads, .k_len = keys, .scale = scale }; + + const dst = if (launch.splits == 1) q.shape() else zml.Shape.init(.{nrows * (head_dim * launch.splits + 2 * launch.splits)}, .f32); + const out = flash_attn.FaVec.Kernel.call(.{ .q = q, .k = k, .v = v, .token_index = positions }, .{ .dst = dst }, .{ + .cfg = .{ .head_dim = head_dim, .heads = launch.heads, .splits = launch.splits }, + .grid = .{ @intCast(@divExact(launch.splits, launch.splits_per_threadgroup)), @intCast(@divExact(num_heads, launch.heads)), @intCast(batch) }, + .threadgroup = .{ 32, @intCast(launch.heads * launch.splits_per_threadgroup), 1 }, + .constants = .{ .args = .of(args) }, + }).dst; + if (launch.splits == 1) return out; + + return flash_attn.FaVecReduce.Kernel.call(.{ .htmp = out, .token_index = positions }, .{ .dst = q.shape() }, .{ + .cfg = .{ .splits = launch.splits, .head_dim = head_dim }, + .grid = .{ @intCast(nrows), @intCast(@divExact(head_dim, 4 * flash_attn.FaVecReduce.simdgroups)), 1 }, + .threadgroup = .{ 32, flash_attn.FaVecReduce.simdgroups, 1 }, + .constants = .{ .args = .of(args) }, + }).dst; +} + +/// `q` is `[b, q, h, hd]`, `k` and `v` `[b, k, kv_h, hd]`. +fn prefill(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, positions: zml.Tensor, live: zml.Tensor, scale: f32) zml.Tensor { + const queries: i32 = @intCast(q.dim(.q)); + const tile = flash_attn.FaTiled.tileConfig(q.dim(.hd)).?; + const cfg: flash_attn.FaTiled.Cfg = .{ + .dtype = zml.kernel.metal.from(q.dtype()), + .head_size = @intCast(q.dim(.hd)), + .bq = tile[0], + .tile_kv = tile[1], + .num_threads = tile[2], + }; + return flash_attn.FaTiled.Kernel.call(.{ .q = q, .k = k, .v = v, .token_index = positions, .live = live }, .{ .out = q.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(q.dim(.h)), @intCast(@divFloor(queries + tile[0] - 1, tile[0])), @intCast(q.dim(.b)) }, + .threadgroup = .{ @intCast(tile[2]), 1, 1 }, + .threadgroup_memory = flash_attn.FaTiled.threadgroupMemory(cfg), + .constants = .{ + .num_kv_heads = .of(@as(i32, @intCast(k.dim(.h)))), + .scale = .of(scale), + .q_len = .of(queries), + .k_len = .of(@as(i32, @intCast(k.dim(.k)))), + }, + }).out; +} + +const VecLaunch = struct { + heads: i32, + splits: i32, + splits_per_threadgroup: i32, +}; + +/// At most 256 threads, which every Apple GPU pipeline accepts. Splits past a +/// row's length exit early, so `splits` follows the core and row counts. +fn vecLaunch(rows: i32, group: i32, span: i32, cores: i32) VecLaunch { + var heads: i32 = @min(group, 4); + while (@rem(group, heads) != 0) heads -= 1; + const want: i32 = if (rows <= 2 * cores) 128 else 64; + const chunks = @divFloor(span + 31, 32); + var splits: i32 = 1; + while (splits * 2 <= @min(want, chunks)) splits *= 2; + return .{ .heads = heads, .splits = splits, .splits_per_threadgroup = @min(splits, @divFloor(8, heads)) }; +} + +fn coreCount() i32 { + const platform = zml.Compiler.current().platform; + const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse + std.debug.panic("Metal device has no core_count attribute", .{}); + return @intCast(attribute.int64); +} + +test vecLaunch { + try std.testing.expectEqual(VecLaunch{ .heads = 4, .splits = 128, .splits_per_threadgroup = 2 }, vecLaunch(16, 4, 128000, 40)); + try std.testing.expectEqual(VecLaunch{ .heads = 1, .splits = 1, .splits_per_threadgroup = 1 }, vecLaunch(8, 1, 20, 40)); +} + +test { + std.testing.refAllDecls(flash_attn); } diff --git a/zml/kernels/flash_attention/tests.zig b/zml/kernels/flash_attention/tests.zig index d93bd373..c3c3dca2 100644 --- a/zml/kernels/flash_attention/tests.zig +++ b/zml/kernels/flash_attention/tests.zig @@ -46,6 +46,22 @@ test "attention: q=1,qh=8,kh=8" { ); } +test "attention: q=1,k=4096,qh=32,kh=8,hd=128" { + try testAttention( + .init(.{ .q = 1, .h = 32, .hd = 128 }, .bf16), + .init(.{ .k = 4096, .h = 8, .hd = 128 }, .bf16), + &.{3000}, + ); +} + +test "attention: q=1,k=1000,qh=8,kh=2,hd=32" { + try testAttention( + .init(.{ .q = 1, .h = 8, .hd = 32 }, .bf16), + .init(.{ .k = 1000, .h = 2, .hd = 32 }, .bf16), + &.{900}, + ); +} + test "attention: q=8,qh=64,kh=8" { try testAttention( .init(.{ .q = 8, .h = 64, .hd = 64 }, .bf16), From 8ebed5402912e5340333d877a077930ec536e3c3 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 12 Nov 2026 13:12:25 +0100 Subject: [PATCH 868/881] zml: refactor safetensors (#1048) Refactor `zml/safetensors.zig` * Drop metadata parsing * refactor code to be more "object oriented", code is owned by the `TensorRegistry` * implement a new format `model.safetensors.full_index.json` that is similar to `index.json` but actually has data inside it: ``` { "metadata": {"num_tensors":96085,"total_size":510286023000}, "weight_map": { "aligner.w1.bias": {"filename":"model-00001-of-00048.safetensors","shape":[5120],"data_offsets":[27384,37624],"dtype":"BF16"}, "aligner.w1.weight": {"filename":"model-00001-of-00048.safetensors","shape":[5120,9216],"data_offsets":[37624,94409464],"dtype":"BF16"}, "aligner.w2.bias": {"filename":"model-00001-of-00048.safetensors","shape":[5120],"data_offsets":[94409464,94419704],"dtype":"BF16"}, "aligner.w2.weight": {"filename":"model-00001-of-00048.safetensors","shape":[5120,5120],"data_offsets":[94419704,146848504],"dtype":"BF16"}, "vision.blocks.0.attn.wo.bias": {"filename":"model-00001-of-00048.safetensors","shape":[1024],"data_offsets":[146848504,146850552],"dtype":"BF16"}, "vision.blocks.0.attn.wo.weight": {"filename":"model-00001-of-00048.safetensors","shape":[1024,1024],"data_offsets":[146850552,148947704],"dtype":"BF16"}, "vision.blocks.0.attn.wqkv.bias": {"filename":"model-00001-of-00048.safetensors","shape":[3072],"data_offsets":[148947704,148953848],"dtype":"BF16"}, ... ``` I followed conventions of `model.saftensors.index.json` --- bazel/bazel.zig | 2 +- examples/io/main.zig | 20 +- tools/safetensors/BUILD.bazel | 13 + tools/safetensors/main.zig | 91 +++ zml/safetensors.zig | 1067 +++++++++++++-------------------- zml/zml.zig | 6 + 6 files changed, 539 insertions(+), 660 deletions(-) create mode 100644 tools/safetensors/BUILD.bazel create mode 100644 tools/safetensors/main.zig diff --git a/bazel/bazel.zig b/bazel/bazel.zig index 8af28b00..67513227 100644 --- a/bazel/bazel.zig +++ b/bazel/bazel.zig @@ -5,7 +5,7 @@ const stdx = @import("stdx"); var runfiles_global: bazel_runfiles.Runfiles = undefined; -export fn _bazel_runfiles_init_impl(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { +pub export fn _bazel_runfiles_init_impl(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { _ = argc; _ = envp; var threaded: std.Io.Threaded = .init_single_threaded; diff --git a/examples/io/main.zig b/examples/io/main.zig index 1cec23b0..c9dc05ea 100644 --- a/examples/io/main.zig +++ b/examples/io/main.zig @@ -132,13 +132,9 @@ pub fn main(init: std.process.Init) !void { const root = try TensorNode.init(init.arena.allocator(), ""); - var registry_it = registry.iterator(); - while (registry_it.next()) |kv| { - const name = kv.key_ptr.*; - const tensor = kv.value_ptr.*; - + for (registry.tensors.values()) |tensor| { var current = root; - var parts = std.mem.tokenizeScalar(u8, name, '.'); + var parts = std.mem.tokenizeScalar(u8, tensor.name, '.'); while (parts.next()) |part| { const gop = try current.children.getOrPut(current.allocator, part); if (!gop.found_existing) { @@ -183,16 +179,16 @@ pub fn main(init: std.process.Init) !void { const tensors = try allocator.alloc(zml.Tensor, tensor_count); defer allocator.free(tensors); - var registry_it = registry.iterator(); var load_count: usize = 0; - while (registry_it.next()) |entry| : (load_count += 1) { + for (registry.tensors.keys(), registry.tensors.values()) |name, tensor| { tensors[load_count] = switch (mesh_type) { - .replicated => store.view().createReplicatedTensor(entry.key_ptr.*, null), - .sharded => if (entry.value_ptr.shape.rank() > 0) - store.view().createTensor(entry.key_ptr.*, null, .tp, tp_partitioning) + .replicated => store.view().createReplicatedTensor(name, null), + .sharded => if (tensor.shape.rank() > 0) + store.view().createTensor(name, null, .tp, tp_partitioning) else - store.view().createReplicatedTensor(entry.key_ptr.*, null), + store.view().createReplicatedTensor(name, null), }; + load_count += 1; } const model: AllTensorsModel = .{ .tensors = tensors }; diff --git a/tools/safetensors/BUILD.bazel b/tools/safetensors/BUILD.bazel new file mode 100644 index 00000000..aff4954d --- /dev/null +++ b/tools/safetensors/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_test") + +zig_binary( + name = "safetensors", + main = "main.zig", + visibility = ["//visibility:public"], + zigopts = ["-lc"], + deps = [ + "//stdx", + "//vfs", + "//zml", + ], +) diff --git a/tools/safetensors/main.zig b/tools/safetensors/main.zig new file mode 100644 index 00000000..cf902eef --- /dev/null +++ b/tools/safetensors/main.zig @@ -0,0 +1,91 @@ +const std = @import("std"); + +const stdx = @import("stdx"); +const VFS = @import("vfs"); +const zml = @import("zml"); + +const log = std.log.scoped(.safetensors_convert); + +const Args = struct { + output: ?[]const u8 = null, + positional: struct { model: []const u8 }, + + pub const help = + \\Usage: [--output=] + \\ + \\Read safetensors headers and write model.safetensors.full_index.json in the working directory + \\Example: bazel run //tools/safetensors -- --output="model.safetensors.full_index.json" hf://deepseek-ai/DeepSeek-V4.1-Flash + ; +}; + +pub fn main(init: std.process.Init) !void { + const args = stdx.flags.parse(init.minimal.args, Args); + const allocator = init.gpa; + + const start = std.Io.Clock.now(.awake, init.io); + var client: std.http.Client = .{ .allocator = allocator, .io = init.io }; + defer client.deinit(); + try client.initDefaultProxies(allocator, init.environ_map); + + const input = args.positional.model; + + var file_vfs: VFS.File = .init(allocator, init.io, .{ .direct_io = false }); + defer file_vfs.deinit(); + var http_vfs: VFS.HTTP = try .init(allocator, init.io, &client, .http); + defer http_vfs.deinit(); + var https_vfs: VFS.HTTP = try .init(allocator, init.io, &client, .https); + defer https_vfs.deinit(); + var hf_vfs: ?VFS.HF = if (std.mem.startsWith(u8, input, "hf://")) + try .auto(allocator, init.io, &client, init.environ_map) + else + null; + defer if (hf_vfs) |*backend| backend.deinit(); + var s3_vfs: ?VFS.S3 = if (std.mem.startsWith(u8, input, "s3://")) + try .auto(allocator, init.io, &client, init.environ_map) + else + null; + defer if (s3_vfs) |*backend| backend.deinit(); + var gcs_vfs: ?VFS.GCS = if (std.mem.startsWith(u8, input, "gs://")) + try .auto(allocator, init.io, &client, init.environ_map) + else + null; + defer if (gcs_vfs) |*backend| backend.deinit(); + + var vfs: VFS = try .init(allocator, init.io); + const io = vfs.io(); + defer vfs.deinit(); + try vfs.register("file", file_vfs.io()); + try vfs.register("http", http_vfs.io()); + try vfs.register("https", https_vfs.io()); + if (hf_vfs) |*backend| try vfs.register("hf", backend.io()); + if (s3_vfs) |*backend| try vfs.register("s3", backend.io()); + if (gcs_vfs) |*backend| try vfs.register("gs", backend.io()); + + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + + const repo = try zml.safetensors.resolveModelRepo(io, input); + var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); + defer registry.deinit(); + const loaded = std.Io.Clock.now(.awake, init.io); + + const index_json_file = if (args.output) |output| + try std.Io.Dir.cwd().createFile(io, output, .{}) + else + try repo.createFile(io, full_index_path, .{}); + const out_buffer = try allocator.alloc(u8, 4096); + defer allocator.free(out_buffer); + var out_writer = index_json_file.writer(io, out_buffer); + try registry.saveAsFullIndexJson(io, allocator, repo, &out_writer.interface); + const done = std.Io.Clock.now(.awake, init.io); + + log.info("Wrote {s}: {d} tensors; load {f}s, write {f}s, total {f}s", .{ + args.output orelse full_index_path, + registry.tensors.count(), + start.durationTo(loaded), + loaded.durationTo(done), + start.durationTo(done), + }); +} + +const full_index_path = "model.safetensors.full_index.json"; diff --git a/zml/safetensors.zig b/zml/safetensors.zig index f854f02f..3827fe27 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -2,68 +2,337 @@ const std = @import("std"); const DataType = @import("dtype.zig").DataType; const Shape = @import("shape.zig").Shape; -const Dims = Shape.DimsArray; - -const StringBuilder = std.ArrayListUnmanaged(u8); - -pub const Tensors = std.StringArrayHashMapUnmanaged(Tensor); -pub const Metadatas = std.StringArrayHashMapUnmanaged(Metadata); const log = std.log.scoped(.@"zml/safetensors"); -const BYTES_HEADER = 8; +// TODO: consider promoting this to the top level struct +// TODO: Feels like we are missing a mapping file -> tensors that would be helfpful for loading weights +/// Maps tensor names to their shape and positions in different sub files. +pub const TensorRegistry = struct { + arena: std.heap.ArenaAllocator, -pub const FileType = enum { - index, - safetensors, - unknown, + tensors: std.StringArrayHashMapUnmanaged(Tensor), + + /// Metadata found in `__metadata__` field of the safetensors header. + /// This is rarely use and the information there is not standardized, so we let user code handle it. + /// Here we just store the plain bytes, reach out if you need this to be already parsed. + metadata: []const u8, + + pub fn init(allocator: std.mem.Allocator) TensorRegistry { + return .{ + .arena = std.heap.ArenaAllocator.init(allocator), + .tensors = .{}, + .metadata = "", + }; + } + + pub fn deinit(self: *TensorRegistry) void { + self.arena.deinit(); + } + + pub fn fromRepo(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir) !TensorRegistry { + const file_type, const file = try resolveModelEntrypoint(io, repo); + defer file.close(io); + + var registry: TensorRegistry = .init(allocator); + switch (file_type) { + .safetensors => try registry.parseSafetensors(io, allocator, file), + .index => try registry.parseIndexJson(io, allocator, repo, file), + .full_index => try registry.parseFullIndexJson(io, allocator, repo, file), + .unknown => return error.InvalidPath, + } + return registry; + } + + pub fn fromPath(allocator: std.mem.Allocator, io: std.Io, path: []const u8) !TensorRegistry { + const file_type: FileType = .resolve(path); + switch (file_type) { + .safetensors => { + // open directly + const file = try std.Io.Dir.openFile(.cwd(), io, path, .{}); + var registry: TensorRegistry = .init(allocator); + try registry.parseSafetensors(io, allocator, file); + return registry; + }, + .full_index, + .index, + => { + // resolve repo as parent folder then open + const repo_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; + const repo = std.Io.Dir.openDir(.cwd(), io, repo_path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; + }; + const file = try std.Io.Dir.openFile(.cwd(), io, path, .{}); + + var registry: TensorRegistry = .init(allocator); + try switch (file_type) { + .index => registry.parseIndexJson(io, allocator, repo, file), + .full_index => registry.parseFullIndexJson(io, allocator, repo, file), + .safetensors, .unknown => unreachable, + }; + return registry; + }, + .unknown => { + // the path is possibly a folder, let's try that + const repo = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; + }; + return try fromRepo(allocator, io, repo); + }, + } + } + + /// Register a tensor, duplicate tensor name and uri. + /// If you create a lot of tensors sharing the same uri, + /// considere calling `registry.tensors` directly and avoid duplicated uris. + pub fn registerTensor(self: *TensorRegistry, tensor: Tensor) !void { + const allocator = self.arena.allocator(); + + const owned_name = try allocator.dupe(u8, tensor.name); + errdefer allocator.free(owned_name); + const owned_uri = try allocator.dupe(u8, tensor.file_uri); + errdefer allocator.free(owned_uri); + + const owned_tensor: Tensor = .{ + .name = owned_name, + .file_uri = owned_uri, + .shape = tensor.shape, + .offset = tensor.offset, + }; + try self.tensors.put(allocator, owned_name, owned_tensor); + } + + pub fn reader( + self: *TensorRegistry, + io: std.Io, + tensor_name: []const u8, + buffer: []u8, + ) !TensorReader { + const tensor = self.tensors.get(tensor_name) orelse { + log.err("Tensor {s} not found in registry", .{tensor_name}); + return error.TensorNotFound; + }; + + return try .init(io, tensor, buffer, .{}); + } + + pub fn totalBytes(self: *TensorRegistry) u64 { + var total: u64 = 0; + + var it = self.tensors.iterator(); + while (it.next()) |entry| { + total += entry.value_ptr.byteSize(); + } + + return total; + } + + /// Parse a given .safetensors file and create entries for each tensor + pub fn parseSafetensors( + self: *TensorRegistry, + io: std.Io, + scratch_allocator: std.mem.Allocator, + file: std.Io.File, + ) !void { + var tmp_state = std.heap.ArenaAllocator.init(scratch_allocator); + defer tmp_state.deinit(); + const tmp_arena = tmp_state.allocator(); + + const self_arena = self.arena.allocator(); + var owned_file_name = try self_arena.alloc(u8, std.fs.max_path_bytes); + const file_name_len = try file.realPath(io, owned_file_name); + // Try to release extra memory, but we don't care if it fails + _ = self_arena.resize(owned_file_name, file_name_len); + owned_file_name = owned_file_name[0..file_name_len]; + + const header_buf = try tmp_arena.alloc(u8, 64 * 1024); + var file_reader: std.Io.File.Reader = file.reader(io, header_buf); + + const json_header_length: u64 = try file_reader.interface.takeInt(u64, .little); + const BYTES_HEADER = 8; + const data_start_offset = BYTES_HEADER + json_header_length; + const json_data = try tmp_arena.alloc(u8, @intCast(json_header_length)); + + try file_reader.interface.readSliceAll(json_data); + const headers: std.json.Value = try std.json.parseFromSliceLeaky( + std.json.Value, + tmp_arena, + json_data, + .{}, + ); + + try self.tensors.ensureUnusedCapacity(self_arena, headers.object.count()); + for (headers.object.keys(), headers.object.values()) |key, value| { + if (std.mem.eql(u8, key, "__metadata__")) { + // value belong to tmp arena, duplicate it into self arena. + var owned_metadata: std.Io.Writer.Allocating = .init(self_arena); + try std.json.fmt(value, .{ .whitespace = .minified }).format(&owned_metadata.writer); + self.metadata = owned_metadata.written(); + continue; + } + + const owned_name = try self_arena.dupe(u8, key); + const tensor: Tensor = try .parse(value.object, owned_file_name, owned_name, data_start_offset); + self.tensors.putAssumeCapacity(owned_name, tensor); + } + } + + pub fn parseIndexJson( + self: *TensorRegistry, + io: std.Io, + gpa: std.mem.Allocator, + repo: std.Io.Dir, + index_json_file: std.Io.File, + ) !void { + var tmp_state: std.heap.ArenaAllocator = .init(gpa); + defer tmp_state.deinit(); + const tmp = tmp_state.allocator(); + + const file_reader_buf = try tmp.alloc(u8, 64 * 1024); + var file_reader: std.Io.File.Reader = index_json_file.reader(io, file_reader_buf); + var json_reader: std.json.Reader = .init(tmp, &file_reader.interface); + const index = try std.json.parseFromTokenSourceLeaky( + std.json.Value, + tmp, + &json_reader, + .{ .allocate = .alloc_if_needed }, + ); + + var safetensors_files: std.StringArrayHashMapUnmanaged(void) = .empty; + try safetensors_files.ensureUnusedCapacity(tmp, 64); + if (index.object.get("weight_map")) |wm| { + for (wm.object.values()) |file_name| { + try safetensors_files.put(tmp, file_name.string, {}); + } + } else { + log.warn("No weight_map attribute found in index", .{}); + } + + for (safetensors_files.keys()) |file_name| { + const file = try repo.openFile(io, file_name, .{}); + defer file.close(io); + try self.parseSafetensors(io, tmp, file); + } + } + + pub fn parseFullIndexJson( + self: *TensorRegistry, + io: std.Io, + gpa: std.mem.Allocator, + repo: std.Io.Dir, + index_json_file: std.Io.File, + ) !void { + var tmp_state: std.heap.ArenaAllocator = .init(gpa); + defer tmp_state.deinit(); + const tmp = tmp_state.allocator(); + + const self_arena = self.arena.allocator(); + + var repo_uri_buf = try tmp.alloc(u8, 1024); + const repo_name = repo_uri_buf[0..try repo.realPath(io, repo_uri_buf)]; + + // Note: this is relatively inefficient, we read the full file, copy most of it's content, + // then reduce to TensorRegistry format. + // We could update TensorRegistry as we stream the json. + // Zig std json API are a bit awkward around streaming though. + const file_reader_buf = try tmp.alloc(u8, 64 * 1024); + var file_reader = index_json_file.reader(io, file_reader_buf); + var json_reader: std.json.Reader = .init(tmp, &file_reader.interface); + const index = try std.json.parseFromTokenSourceLeaky( + std.json.Value, + tmp, + &json_reader, + .{ .allocate = .alloc_if_needed }, + ); + + var realpaths: std.StringArrayHashMapUnmanaged([]const u8) = .empty; + try realpaths.ensureUnusedCapacity(tmp, 64); + const wm = index.object.get("weight_map") orelse return error.InvalidJson; + + try self.tensors.ensureUnusedCapacity(self_arena, wm.object.count()); + for (wm.object.keys(), wm.object.values()) |key, value| { + const tensor_name = try self_arena.dupe(u8, key); + const object = value.object; + const filename = value.object.get("filename").?.string; + const filename_entry = try realpaths.getOrPut(tmp, filename); + const file_uri = if (filename_entry.found_existing) + filename_entry.value_ptr.* + else uri: { + const uri = try std.fs.path.join(self_arena, &.{ repo_name, filename }); + filename_entry.value_ptr.* = uri; + break :uri uri; + }; + + const tensor_info: Tensor = try .parse(object, file_uri, tensor_name, 0); + self.tensors.putAssumeCapacity(tensor_name, tensor_info); + } + + if (index.object.get("metadata")) |metadata| { + // value belong to tmp arena, duplicate it into self arena. + var owned_metadata: std.Io.Writer.Allocating = .init(self_arena); + try std.json.fmt(metadata, .{ .whitespace = .minified }).format(&owned_metadata.writer); + self.metadata = owned_metadata.written(); + } + } + + pub fn saveAsFullIndexJson( + self: *const TensorRegistry, + io: std.Io, + gpa: std.mem.Allocator, + repo: std.Io.Dir, + out: *std.Io.Writer, + ) !void { + var repo_uri_buf = try gpa.alloc(u8, std.fs.max_path_bytes); + defer gpa.free(repo_uri_buf); + const repo_name = repo_uri_buf[0..try repo.realPath(io, repo_uri_buf)]; + var total_size: u64 = 0; + for (self.tensors.values()) |value| total_size += value.byteSize(); + + try out.print( + \\{{ + \\"metadata": {{"num_tensors":{d},"total_size":{d}}}, + \\"weight_map": {{ + \\ + , .{ self.tensors.count(), total_size }); + + const n = self.tensors.count(); + for (0.., self.tensors.keys(), self.tensors.values()) |i, key, value| { + std.debug.assert(std.mem.startsWith(u8, value.file_uri, repo_name)); + const local_name = value.file_uri[repo_name.len + 1 ..]; + const end_offset = value.offset + value.byteSize(); + const safetensors_dtype: []const u8 = dt: { + for (safetensors_dtype_to_zml_dtype.keys(), safetensors_dtype_to_zml_dtype.values()) |st_dt, zml_dt| { + if (zml_dt == value.shape._dtype) break :dt st_dt; + } + // Note: we could bitcast to u8, + // but reaching this panic implies the user didn't loaded this TensorRegistry + // from a safetensors file, and manually created the dict. + // In which case, they know what they are doing, + // and can easily insert the bitcast themselves. + std.debug.panic("Safetensors doesn't support data type {}", .{value.shape._dtype}); + }; + const comma = if (i + 1 == n) "" else ","; + try out.print( + \\ "{s}": {{"filename":"{s}","shape":{f},"data_offsets":[{d},{d}],"dtype":"{s}"}}{s} + \\ + , .{ key, local_name, std.json.fmt(value.shape.dims(), .{ .whitespace = .minified }), value.offset, end_offset, safetensors_dtype, comma }); + } + + try out.writeAll( + \\} + \\} + \\ + ); + try out.flush(); + } }; -pub fn fetchRegistry( - allocator: std.mem.Allocator, - io: std.Io, - repo: std.Io.Dir, - model: std.Io.File, - model_path: []const u8, -) !TensorRegistry { - const file_type = resolveFiletype(model_path); - - return switch (file_type) { - .index => blk: { - const file_reader_buf = try allocator.alloc(u8, 64 * 1024); - defer allocator.free(file_reader_buf); - - var index_reader = model.reader(io, file_reader_buf); - - var safetensors_index = try parseSafetensorsIndex( - allocator, - &index_reader.interface, - ); - defer safetensors_index.deinit(); - - var registry: TensorRegistry = try .initWithMetadata(allocator, safetensors_index.metadata); - - try parseSafetensorsIndexFiles( - allocator, - io, - ®istry, - safetensors_index, - repo, - ); - - break :blk registry; - }, - .safetensors => blk: { - var registry: TensorRegistry = .init(allocator); - - try parseSafetensors(allocator, io, ®istry, model); - - break :blk registry; - }, - else => return error.InvalidPath, - }; -} - pub const TensorReader = struct { tensor: Tensor, file: std.Io.File, @@ -238,6 +507,30 @@ pub const Tensor = struct { return TensorReader.init(io, self, buffer, opts); } + pub fn parse(object: std.json.ObjectMap, file_uri: []const u8, name: []const u8, base_offset: u64) !Tensor { + const shape_field = object.get("shape").?.array; + if (shape_field.items.len > Shape.MAX_RANK) { + log.warn("Can't load tensor {s}, too many dims: {}", .{ name, shape_field.items.len }); + return error.TooManyDimensions; + } + + const offset_field = object.get("data_offsets").?; + const start: u64 = @intCast(offset_field.array.items[0].integer); + const dtype = try stringToDtype(object.get("dtype").?.string); + + var dims: Shape.DimsArray = .empty; + for (shape_field.items) |d| { + dims.appendAssumeCapacity(d.integer); + } + + return .{ + .file_uri = file_uri, + .name = name, + .shape = .init(dims.slice(), dtype), + .offset = base_offset + start, + }; + } + pub fn format(self: Tensor, writer: *std.Io.Writer) !void { try writer.print("Tensor(name={s} shape={f} size={d}, offset={d}, file_uri={s})", .{ self.name, @@ -249,620 +542,100 @@ pub const Tensor = struct { } }; -pub const TensorRegistry = struct { - arena: std.heap.ArenaAllocator, +pub const FileType = enum { + full_index, + index, + safetensors, + unknown, - tensors: Tensors, - metadata: Metadatas, - - mutex: std.Io.Mutex = .init, - - pub fn init(allocator: std.mem.Allocator) TensorRegistry { - return .{ - .arena = std.heap.ArenaAllocator.init(allocator), - .tensors = .{}, - .metadata = .{}, - }; - } - - pub fn initWithMetadata( - allocator: std.mem.Allocator, - metadata: Metadatas, - ) !TensorRegistry { - var self: TensorRegistry = .{ - .arena = std.heap.ArenaAllocator.init(allocator), - .tensors = .{}, - .metadata = .empty, - }; - - try self.mergeMetadata(metadata); - - return self; - } - - pub fn fromRepo( - allocator: std.mem.Allocator, - io: std.Io, - repo: std.Io.Dir, - ) !TensorRegistry { - const entrypoint_file, const entrypoint_path = try resolveModelEntrypoint(io, repo); - defer entrypoint_file.close(io); - return try fetchRegistry(allocator, io, repo, entrypoint_file, entrypoint_path); - } - - pub fn fromPath( - allocator: std.mem.Allocator, - io: std.Io, - path: []const u8, - ) !TensorRegistry { - var repo = try resolveModelRepo(io, path); - defer repo.close(io); - - const entrypoint_file, const entrypoint_path = if (std.mem.endsWith(u8, path, ".safetensors.index.json") or std.mem.endsWith(u8, path, ".safetensors")) - .{ try repo.openFile(io, path, .{ .mode = .read_only }), path } - else - try resolveModelEntrypoint(io, repo); - defer entrypoint_file.close(io); - - return try fetchRegistry(allocator, io, repo, entrypoint_file, entrypoint_path); - } - - pub fn deinit(self: *TensorRegistry) void { - const allocator = self.arena.allocator(); - self.tensors.deinit(allocator); - self.metadata.deinit(allocator); - self.arena.deinit(); - } - - pub fn mergeMetadata( - self: *TensorRegistry, - other: Metadatas, - ) !void { - // self.mutex.lock(); - // defer self.mutex.unlock(); - - const allocator = self.arena.allocator(); - - var it = other.iterator(); - while (it.next()) |entry| { - const key = try allocator.dupe(u8, entry.key_ptr.*); - const value = try entry.value_ptr.*.clone(allocator); - - const gop = try self.metadata.getOrPut(allocator, key); - if (gop.found_existing) { - gop.value_ptr.*.deinit(allocator); - gop.value_ptr.* = value; - log.debug("Overwrote existing metadata key={s} with value={f}", .{ key, value }); - } else { - gop.value_ptr.* = value; - log.debug("Added new metadata key={s} with value={f}", .{ key, value }); - } + pub fn resolve(path: []const u8) FileType { + if (std.mem.endsWith(u8, path, ".safetensors.index.json")) { + return .index; + } else if (std.mem.endsWith(u8, path, ".safetensors.full_index.json")) { + return .full_index; + } else if (std.mem.endsWith(u8, path, ".safetensors")) { + return .safetensors; + } else { + return .unknown; } } - - pub fn registerTensor( - self: *TensorRegistry, - tensor: Tensor, - ) !void { - const allocator = self.arena.allocator(); - - // self.mutex.lock(); - // defer self.mutex.unlock(); - - var tensor_copy = tensor; - - tensor_copy.name = try allocator.dupe(u8, tensor.name); - tensor_copy.file_uri = try allocator.dupe(u8, tensor.file_uri); - - try self.tensors.put(allocator, tensor_copy.name, tensor_copy); - } - - pub fn reader( - self: *TensorRegistry, - io: std.Io, - tensor_name: []const u8, - buffer: []u8, - ) !TensorReader { - const tensor = self.tensors.get(tensor_name) orelse { - log.err("Tensor {s} not found in registry", .{tensor_name}); - return error.TensorNotFound; - }; - - return try .init(io, tensor, buffer, .{}); - } - - pub fn iterator(self: *TensorRegistry) Tensors.Iterator { - return self.tensors.iterator(); - } - - pub fn totalBytes(self: *TensorRegistry) u64 { - var total: u64 = 0; - - var it = self.tensors.iterator(); - while (it.next()) |entry| { - total += entry.value_ptr.byteSize(); - } - - return total; - } }; -pub fn parseSafetensors( - allocator: std.mem.Allocator, - io: std.Io, - registry: *TensorRegistry, - file: std.Io.File, -) !void { - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - - const header_buf = try allocator.alloc(u8, 64 * 1024); - defer allocator.free(header_buf); - - var reader = file.reader(io, header_buf); - - var file_uri_buf: [1024]u8 = undefined; - const file_uri_len = try file.realPath(io, &file_uri_buf); - const file_uri = file_uri_buf[0..file_uri_len]; - - const json_header_length: u64 = try reader.interface.takeInt(u64, .little); - const data_start_offset = BYTES_HEADER + json_header_length; - const json_data = try arena.allocator().alloc(u8, @intCast(json_header_length)); - - try reader.interface.readSliceAll(json_data); - - const headers: std.json.Value = try std.json.parseFromSliceLeaky( - std.json.Value, - arena.allocator(), - json_data, - .{}, - ); - - var it = headers.object.iterator(); - while (it.next()) |entry| { - const key = entry.key_ptr.*; - const value = entry.value_ptr.*; - - if (std.mem.eql(u8, key, "__metadata__")) { - // Parse metadata into a local arena to avoid concurrent allocations - // against registry.arena (which is not thread-safe). - var local_arena = std.heap.ArenaAllocator.init(allocator); - defer local_arena.deinit(); - - const metas = try parseMetadata(local_arena.allocator(), value); - try registry.mergeMetadata(metas); - - continue; - } - - const shape_field = value.object.get("shape").?.array; - - if (shape_field.items.len > Shape.MAX_RANK) { - log.warn("Can't load tensor {s}, too many dims: {}", .{ key, shape_field.items.len }); - return error.TooManyDimensions; - } - - const offset_field = value.object.get("data_offsets").?; - const start: u64 = @intCast(offset_field.array.items[0].integer); - const dtype = try stringToDtype(value.object.get("dtype").?.string); - - var dims: Dims = .empty; - for (shape_field.items) |d| { - dims.appendAssumeCapacity(d.integer); - } - - const tensor: Tensor = .{ - .file_uri = file_uri, - .name = key, - .shape = .init(dims.slice(), dtype), - .offset = data_start_offset + start, - }; - - try registry.registerTensor(tensor); - } -} - -const SafetensorsIndex = struct { - pub const Map = std.StringArrayHashMapUnmanaged(std.ArrayList([]const u8)); - - arena: std.heap.ArenaAllocator, - - map: Map, - metadata: Metadatas, - - pub fn init(allocator: std.mem.Allocator) SafetensorsIndex { - return .{ - .arena = std.heap.ArenaAllocator.init(allocator), - .map = .empty, - .metadata = .empty, - }; - } - - pub fn iterator(self: *SafetensorsIndex) Map.Iterator { - return self.map.iterator(); - } - - pub fn deinit(self: *SafetensorsIndex) void { - const allocator = self.arena.allocator(); - self.map.deinit(allocator); - self.metadata.deinit(allocator); - self.arena.deinit(); - } -}; - -fn parseSafetensorsIndex( - allocator: std.mem.Allocator, - reader: *std.Io.Reader, -) !SafetensorsIndex { - var safetensors_index: SafetensorsIndex = .init(allocator); - errdefer safetensors_index.deinit(); - - const arena_allocator = safetensors_index.arena.allocator(); - var json_reader: std.json.Reader = .init(arena_allocator, reader); - - const index = try std.json.parseFromTokenSourceLeaky( - std.json.Value, - arena_allocator, - &json_reader, - .{ .allocate = .alloc_if_needed }, - ); - - const weight_map = index.object.get("weight_map"); - - if (weight_map) |wm| { - var it = wm.object.iterator(); - - while (it.next()) |entry| { - const weight_name = entry.key_ptr.*; - const filename = entry.value_ptr.string; - - const map_entry = try safetensors_index.map.getOrPut(arena_allocator, filename); - - if (!map_entry.found_existing) { - map_entry.value_ptr.* = .empty; - } - - try map_entry.value_ptr.append(arena_allocator, try arena_allocator.dupe(u8, weight_name)); - } - } else { - log.warn("No weight_map attribute found in index", .{}); - } - - if (index.object.get("metadata")) |metadata_val| { - safetensors_index.metadata = try parseMetadata(arena_allocator, metadata_val); - } - - return safetensors_index; -} - -fn parseSafetensorsIndexFiles( - allocator: std.mem.Allocator, - io: std.Io, - registry: *TensorRegistry, - safetensors_index: SafetensorsIndex, - repo: std.Io.Dir, -) !void { - const filenames = safetensors_index.map.keys(); - for (filenames) |filename| { - const file = try repo.openFile(io, filename, .{}); - defer file.close(io); - try parseSafetensors(allocator, io, registry, file); - } -} - -pub fn resolveFiletype(path: []const u8) FileType { - if (std.mem.endsWith(u8, path, ".safetensors.index.json")) { - return .index; - } else if (std.mem.endsWith(u8, path, ".safetensors")) { - return .safetensors; - } else { - return .unknown; - } -} - pub const ModelPathResolutionError = error{ FileNotFound, InvalidPath, } || std.mem.Allocator.Error; -pub fn resolveModelRepo( - io: std.Io, - path: []const u8, -) ModelPathResolutionError!std.Io.Dir { - if (std.mem.endsWith(u8, path, ".safetensors.index.json") or - std.mem.endsWith(u8, path, ".safetensors")) - { - const dir_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; +pub fn resolveModelRepo(io: std.Io, path: []const u8) ModelPathResolutionError!std.Io.Dir { + switch (FileType.resolve(path)) { + .full_index, .index, .safetensors => { + // The path is a file, let's open it's folder + const dir_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; - const dir = std.Io.Dir.openDir(.cwd(), io, dir_path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, + const dir = std.Io.Dir.openDir(.cwd(), io, dir_path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; }; - }; - return dir; + return dir; + }, + .unknown => { + // The path is probably a directory, let's try to open it as such + const dir = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; + }; + return dir; + }, } - - const dir = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, - }; - }; - - return dir; } -pub fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!struct { std.Io.File, []const u8 } { - const default_entrypoints = .{ - "model.safetensors.index.json", - "consolidated.safetensors.index.json", - "model.safetensors", - "consolidated.safetensors", +pub fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!struct { FileType, std.Io.File } { + const default_entrypoints: []const struct { FileType, []const u8 } = &.{ + .{ .full_index, "model.safetensors.full_index.json" }, + .{ .full_index, "consolidated.safetensors.full_index.json" }, + .{ .index, "model.safetensors.index.json" }, + .{ .index, "consolidated.safetensors.index.json" }, + .{ .safetensors, "model.safetensors" }, + .{ .safetensors, "consolidated.safetensors" }, }; - inline for (default_entrypoints) |entrypoint| { - if (repo.openFile(io, entrypoint, .{ .mode = .read_only })) |index_file| { - return .{ index_file, entrypoint }; - } else |_| {} + for (default_entrypoints) |entrypoint| { + const file_type, const file_name = entrypoint; + const file = repo.openFile(io, file_name, .{ .mode = .read_only }) catch continue; + + return .{ file_type, file }; } - return ModelPathResolutionError.FileNotFound; + return error.FileNotFound; } -const Metadata = union(enum) { - null: void, - int: i64, - float: f64, - bool: bool, - string: []const u8, +const safetensors_dtype_to_zml_dtype = std.StaticStringMap(DataType).initComptime(.{ + .{ "F64", .f64 }, + .{ "F32", .f32 }, + .{ "F16", .f16 }, + .{ "BF16", .bf16 }, + .{ "F8_E8M0", .f8e8m0 }, + .{ "F8_E4M3", .f8e4m3fn }, + .{ "F4_E2M1", .f4e2m1 }, + .{ "I64", .i64 }, + .{ "I32", .i32 }, + .{ "I16", .i16 }, + .{ "I8", .i8 }, + .{ "U64", .u64 }, + .{ "U32", .u32 }, + .{ "U16", .u16 }, + .{ "U8", .u8 }, + .{ "BOOL", .bool }, +}); - array_bool: []const bool, - array_int: []const i64, - array_float: []const f64, - array_string: []const []const u8, - - pub const ItemType = enum { - int, - float, - bool, - string, - - pub fn toZigType(comptime kind: ItemType) type { - return switch (kind) { - .int => i64, - .float => f64, - .bool => bool, - .string => []const u8, - }; - } - }; - - pub fn wrap(x: anytype) Metadata { - return switch (@TypeOf(x)) { - inline u8, i8, u16, i16, u32, i32, u64, i64 => .{ .int = @intCast(x) }, - inline f16, f32, f64 => .{ .float = @floatCast(x) }, - bool => .{ .bool = x }, - []const u8 => .{ .string = x }, - else => @panic("Unsupported type for Value: " ++ @typeName(@TypeOf(x))), - }; - } - - pub fn copySlice(allocator: std.mem.Allocator, any_slice: anytype) !Metadata { - return switch (@TypeOf(any_slice[0])) { - inline u8, i8, u16, i16, u32, i32, u64, i64 => { - const res = try allocator.alloc(i64, any_slice.len); - for (res, any_slice) |*r, val| r.* = @intCast(val); - return .{ .array_int = res }; - }, - inline f16, f32, f64 => { - const res = try allocator.alloc(f64, any_slice.len); - for (res, any_slice) |*r, val| r.* = @floatCast(val); - return .{ .array_float = res }; - }, - bool => .{ .array_bool = try allocator.dupe(bool, any_slice) }, - []const u8 => .{ .array_string = try allocator.dupe([]const u8, @alignCast(any_slice)) }, - else => @panic("Unsupported type for Value: " ++ @typeName(@TypeOf(any_slice))), - }; - } - - pub fn format( - self: Metadata, - writer: *std.Io.Writer, - ) !void { - switch (self) { - .null => _ = try writer.write("null"), - .string => |s| try writer.print("{s}", .{s}), - .bool => |b| try writer.print("{}", .{b}), - .int => |i| try writer.print("{d}", .{i}), - .float => |f| try writer.print("{d}", .{f}), - .array_bool => |arr| { - try writer.writeByte('['); - for (arr, 0..) |v, i| { - if (i > 0) try writer.writeAll(", "); - try writer.print("{}", .{v}); - } - try writer.writeByte(']'); - }, - .array_int => |arr| { - try writer.writeByte('['); - for (arr, 0..) |v, i| { - if (i > 0) try writer.writeAll(", "); - try writer.print("{d}", .{v}); - } - try writer.writeByte(']'); - }, - .array_float => |arr| { - try writer.writeByte('['); - for (arr, 0..) |v, i| { - if (i > 0) try writer.writeAll(", "); - try writer.print("{d}", .{v}); - } - try writer.writeByte(']'); - }, - .array_string => |arr| { - try writer.writeByte('['); - for (arr, 0..) |v, i| { - if (i > 0) try writer.writeAll(", "); - try writer.print("\"{s}\"", .{v}); - } - try writer.writeByte(']'); - }, - } - } - - pub fn deinit(self: Metadata, allocator: std.mem.Allocator) void { - switch (self) { - .string => |s| allocator.free(s), - .array_bool => |s| allocator.free(s), - .array_int => |s| allocator.free(s), - .array_float => |s| allocator.free(s), - .array_string => |s| { - for (s) |str| allocator.free(str); - allocator.free(s); - }, - else => {}, - } - } - - pub fn clone(self: Metadata, allocator: std.mem.Allocator) !Metadata { - return switch (self) { - .null => .null, - .int => |v| .{ .int = v }, - .float => |v| .{ .float = v }, - .bool => |v| .{ .bool = v }, - .string => |s| .{ .string = try allocator.dupe(u8, s) }, - .array_bool => |s| .{ .array_bool = try allocator.dupe(bool, s) }, - .array_int => |s| .{ .array_int = try allocator.dupe(i64, s) }, - .array_float => |s| .{ .array_float = try allocator.dupe(f64, s) }, - .array_string => |s| blk: { - const new_slice = try allocator.alloc([]const u8, s.len); - errdefer allocator.free(new_slice); - for (s, 0..) |str, i| { - new_slice[i] = try allocator.dupe(u8, str); - } - break :blk .{ .array_string = new_slice }; - }, - }; - } -}; - -fn stringToDtype(safetensor_type: []const u8) !DataType { - const map = std.StaticStringMap(DataType).initComptime(.{ - .{ "F64", .f64 }, - .{ "F32", .f32 }, - .{ "F16", .f16 }, - .{ "BF16", .bf16 }, - .{ "F8_E8M0", .f8e8m0 }, - .{ "F8_E4M3", .f8e4m3fn }, - .{ "F4_E2M1", .f4e2m1 }, - .{ "I64", .i64 }, - .{ "I32", .i32 }, - .{ "I16", .i16 }, - .{ "I8", .i8 }, - .{ "U64", .u64 }, - .{ "U32", .u32 }, - .{ "U16", .u16 }, - .{ "U8", .u8 }, - .{ "BOOL", .bool }, - }); - - return map.get(safetensor_type) orelse { - log.err("Unsupported safetensor data type: {s}", .{safetensor_type}); +fn stringToDtype(safetensor_dtype: []const u8) !DataType { + return safetensors_dtype_to_zml_dtype.get(safetensor_dtype) orelse { + log.err("Unsupported safetensor data type: {s}", .{safetensor_dtype}); return error.UnsupportedDataType; }; } - -fn parseMetadata(allocator: std.mem.Allocator, val: std.json.Value) !Metadatas { - var metadatas: Metadatas = .{}; - var prefix_buf: [1024]u8 = undefined; - var prefix = StringBuilder.initBuffer(&prefix_buf); - - try populateMetadata(allocator, &prefix, val, &metadatas); - - return metadatas; -} - -fn populateMetadata(allocator: std.mem.Allocator, prefix: *StringBuilder, val: std.json.Value, metadatas: *Metadatas) !void { - const key = prefix.items; - return switch (val) { - .null => try metadatas.put(allocator, try allocator.dupe(u8, key), .null), - .bool => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .bool = v }), - .integer => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .int = v }), - .float => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .float = v }), - .number_string, .string => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .string = try allocator.dupe(u8, v) }), - .array => |v| { - if (v.items.len == 0) return; - if (validSlice(v)) |item_type| { - const data: Metadata = switch (item_type) { - .bool => blk: { - const values = try allocator.alloc(bool, v.items.len); - for (v.items, 0..) |item, i| values[i] = item.bool; - break :blk .{ .array_bool = values }; - }, - .integer => blk: { - const values = try allocator.alloc(i64, v.items.len); - for (v.items, 0..) |item, i| values[i] = item.integer; - break :blk .{ .array_int = values }; - }, - .float => blk: { - const values = try allocator.alloc(f64, v.items.len); - for (v.items, 0..) |item, i| values[i] = item.float; - break :blk .{ .array_float = values }; - }, - inline .string, .number_string => |tag| blk: { - const values = try allocator.alloc([]const u8, v.items.len); - for (v.items, 0..) |item, i| { - values[i] = try allocator.dupe(u8, @field(item, @tagName(tag))); - } - break :blk .{ .array_string = values }; - }, - .null, .array, .object => unreachable, - }; - try metadatas.put(allocator, try allocator.dupe(u8, key), data); - } else { - for (v.items, 0..) |item, i| { - const old_len = prefix.items.len; - if (prefix.items.len > 0) { - prefix.appendAssumeCapacity('.'); - } - prefix.items.len += std.fmt.printInt(prefix.unusedCapacitySlice(), i, 10, .lower, .{}); - try populateMetadata(allocator, prefix, item, metadatas); - prefix.items.len = old_len; - } - } - }, - .object => |v| { - var obj_iter = v.iterator(); - while (obj_iter.next()) |entry| { - const old_len = prefix.items.len; - if (prefix.items.len > 0) { - prefix.appendAssumeCapacity('.'); - } - prefix.appendSliceAssumeCapacity(entry.key_ptr.*); - try populateMetadata(allocator, prefix, entry.value_ptr.*, metadatas); - prefix.items.len = old_len; - } - }, - }; -} - -/// We can only create a Zig slice out of json array, if all values -/// in the array have the same type. -fn validSlice(v: std.json.Array) ?std.meta.Tag(std.json.Value) { - if (v.items.len == 0) return null; - - const item_type: std.meta.Tag(std.json.Value) = v.items[0]; - switch (item_type) { - .null, .array, .object => return null, - else => {}, - } - - for (v.items[1..]) |item| { - if (item != item_type) - return null; - } - - return item_type; -} diff --git a/zml/zml.zig b/zml/zml.zig index 23d12e75..35c2f9df 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -81,3 +81,9 @@ pub const KiB = 1024; pub const MiB = 1024 * KiB; pub const GiB = 1024 * MiB; pub const TiB = 1024 * GiB; + +comptime { + // Forces zig to compile `@zml//bazel` module even when users don't depend on it even indirectly. + // Otherwise exported functions declared in `bazel` aren't compiled. + std.testing.refAllDecls(bazel); +} From c40fa52d2db5ca40a6a4500c94d5f04d06ae3e7d Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Thu, 12 Nov 2026 13:47:48 +0100 Subject: [PATCH 869/881] zml/moe [Platform:tcl]: support block-128 FP8 SwiGLU experts (#1053) - Select TCL for plain E4M3FN block-128 experts with matching BF16 or F32 scales. Preserve rejection of unsupported quantization. - Quantize activations per block and apply checkpoint scales during native projections. Keep BF16 materialization and F32 down and routing accumulation. - Reuse local expert routing and AllReduce for expert parallelism. Choose expert-stationary for small FP8 token counts and use the existing activation-stationary and blockwise recipes for larger counts. - Share FP8 and BF16 dispatch preparation while retaining the token padding and BF16 SiLU rounding workarounds. --- platforms/furiosa/furiosa.bzl | 4 +- zml/kernels/moe/moe.zig | 12 +- zml/kernels/moe/tcl/tcl.zig | 451 ++++++++++++++++++++++++++-------- 3 files changed, 363 insertions(+), 104 deletions(-) diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index 81518d53..623c4f0d 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-furiosa-linux-amd64.tar.zst", - sha256 = "6da755570140c9f8959ba477fef3df4c48b0619274c97429d204e177b8649e12", + url = "https://mirror.zml.ai/plugins/202611121233.371.1.9ff7ddf1e19f/zml-furiosa-linux-amd64.tar.zst", + sha256 = "d732b56832b1f009cbe4d808c70c6f8d1dc3857a83eb06804fc53b39bef9dd6b", ) return mctx.extension_metadata( diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index 33b6da55..2a0aa8d7 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -37,8 +37,6 @@ pub const Backend = enum { tcl, pub fn auto(platform: *const zml.Platform, scheme: ?zml.Quantization.Scheme, dtype: zml.DataType) !Backend { - // Keep the dtype as an argument because non scheme-specific backends may depend on it later - _ = dtype; return switch (platform.target) { .cuda => b: { const s = scheme orelse break :b .triton; @@ -72,7 +70,13 @@ pub const Backend = enum { }; }, .tpu => if (scheme == null) .mosaic_tpu else error.UnsupportedQuantization, - .furiosa => if (scheme == null) .tcl else error.UnsupportedQuantization, + .furiosa => b: { + const s = scheme orelse break :b .tcl; + break :b switch (s) { + .fp8_block128 => if (dtype == .f8e4m3fn) .tcl else error.UnsupportedQuantization, + .mxfp4, .nvfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block32 => error.UnsupportedQuantization, + }; + }, .metal => b: { const s = scheme orelse break :b .metal; break :b switch (s) { @@ -245,7 +249,7 @@ pub fn forwardMoe( } const expert_axis = experts.meshAxis(); const parallel = if (expert_axis) |axis| mesh.grid.axis_sizes[axis] > 1 else false; - if (backend == .metal or backend == .tcl) stdx.debug.assert(!parallel, "This MoE backend does not support expert parallelism", .{}); + if (backend == .metal) stdx.debug.assert(!parallel, "This MoE backend does not support expert parallelism", .{}); const Call = struct { input: zml.Tensor, diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig index 698c4d30..e7d7a01b 100644 --- a/zml/kernels/moe/tcl/tcl.zig +++ b/zml/kernels/moe/tcl/tcl.zig @@ -9,9 +9,17 @@ //! device loop over the used chunks of `BGi` blocks gathers each block's //! expert weights and runs both projections; outputs are gathered back per //! route and weighted. +//! +//! FP8 experts split across devices (expert parallel): each device runs its own +//! experts for the routes that reach them and an all-reduce sums the devices' +//! outputs. Up to `expert_stationary_tokens` tokens, expert stationary: a +//! device loop over the used experts, a few at a time, reads each once and +//! runs it for every token, scaled by the token's routing weight (zero for +//! the tokens it does not serve); more tokens take blockwise. const std = @import("std"); const builder = @import("dsl/tcl/builder"); + const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); @@ -19,34 +27,78 @@ const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; const Options = @import("../options.zig").Options; -/// `input` is `{.t, .d}`, routing IDs and weights `{.t, .topk}`, `gate_up` `{.expert, .dout = 2 * inter (gate then up), .d}` -/// and `down` `{.expert, .d, .dout = inter}`, all bf16 but the ids (i32). +/// `input` is `{.t, .d}`, `topk_ids` and `topk_weights` `{.t, .topk}`, `gate_up` `{.expert, .dout = 2 * inter (gate then up), .d}` +/// and `down` `{.expert, .d, .dout = inter}`. Experts are BF16 or block-128 FP8 with BF16/F32 scales. /// SwiGLU, routing weights applied after the down projection. pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: Options) Tensor { - const topk_ids = routing.ids; - const topk_weights = routing.weights; - stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE applies routing weights after the down projection, without FP8 input quantization", .{}); + const topk_ids = routing.localIds(); + stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE owns input precision and applies routing weights after the down projection", .{}); stdx.debug.assert(opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}), "TCL MoE supports SwiGLU without limit, scale or bias", .{}); - stdx.debug.assert(gate_up_linear.quantization == null and down_linear.quantization == null, "TCL MoE takes bf16 experts", .{}); stdx.debug.assert(gate_up_linear.bias == null and down_linear.bias == null, "TCL MoE takes experts without bias", .{}); const gate_up = gate_up_linear.weight; const down = down_linear.weight; - stdx.debug.assert(input.dtype() == .bf16 and gate_up.dtype() == .bf16 and down.dtype() == .bf16 and topk_ids.dtype() == .i32, "TCL MoE takes bf16 activations and experts, i32 ids", .{}); - const t = input.dim(.t); - const k = topk_ids.dim(.topk); - const cfg: Cfg = .{ - .t = t, - .k = k, + const quantized = gate_up_linear.quantization != null or down_linear.quantization != null; + const scales: ?Scales = if (quantized) .{ .gate_up = fp8Scales(gate_up_linear), .down = fp8Scales(down_linear) } else null; + stdx.debug.assert(input.dtype() == .bf16 and topk_ids.dtype() == .i32 and (quantized or (gate_up.dtype() == .bf16 and down.dtype() == .bf16)), "TCL MoE takes BF16 activations, BF16/block-128 FP8 experts and i32 ids", .{}); + const weights = routing.weights.convert(.f32); + const local_weights = if (scales != null) topk_ids.cmp(.GE, .scalar(0, .i32)).select(weights, .scalar(0, .f32)) else weights; + return runExperts(input, topk_ids, local_weights, gate_up, down, scales); +} + +const Scales = struct { gate_up: Tensor, down: Tensor }; + +fn fp8Scales(linear: zml.nn.Linear) Tensor { + const q = linear.quantization orelse std.debug.panic("TCL MoE requires both FP8 projections", .{}); + stdx.debug.assert(q.scheme == .fp8_block128 and linear.weight.dtype() == .f8e4m3fn and !q.swizzled_scales and q.global_scale == null and q.input_scale == null, "TCL MoE takes plain E4M3FN block-128 weights", .{}); + stdx.debug.assert(q.scales.dtype() == .bf16 or q.scales.dtype() == .f32, "TCL MoE FP8 scales must be BF16 or F32", .{}); + const w = linear.weight; + const scale = q.scales; + stdx.debug.assert(@mod(w.dim(.d), 128) == 0 and @mod(w.dim(.dout), 128) == 0 and scale.dim(.expert) == w.dim(.expert) and scale.dim(.d) * 128 == w.dim(.d) and scale.dim(.dout) * 128 == w.dim(.dout), "TCL MoE FP8 scales must cover 128x128 weight blocks", .{}); + return scale; +} + +/// Tokens up to which a device runs its experts stationary. +pub const expert_stationary_tokens = 64; + +fn runExperts(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: Tensor, down: Tensor, scales: ?Scales) Tensor { + var cfg: Cfg = .{ + .t = input.dim(.t), + .k = topk_ids.dim(.topk), .e = gate_up.dim(.expert), .d = input.dim(.d), .i = down.dim(.dout), }; stdx.debug.assert(gate_up.dim(.dout) == 2 * cfg.i and down.dim(.d) == cfg.d and gate_up.dim(.d) == cfg.d, "TCL MoE: mismatched expert shapes", .{}); + if (scales) |s| { + cfg.scale_dtype = scaleDtype(s); + if (cfg.t <= expert_stationary_tokens) { + // WORKAROUND (TCC 2026.5.0b0): a token count other than a power of two + // panics its adder tree ("Fail to cover valid range", 5 tokens padded + // to 8): pad with unrouted tokens. + var padded = cfg; + padded.t = @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.t)) catch unreachable); + const pad = padded.t - cfg.t; + const x = input.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.t, cfg.d }); + const ids = topk_ids.pad(-1, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.t, cfg.k }); + const weights = topk_weights.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.t, cfg.k }); + const y = ExpertStationaryFp8.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down, .gate_up_scale = s.gate_up, .down_scale = s.down }, .{ .y = x.shape() }, .{ .cfg = padded }).y; + return y.slice(0, .{ .end = cfg.t }).reshape(input.shape()); + } + } - const x = input.reshape(.{ t, cfg.d }); - const ids = topk_ids.reshape(.{ t, k }); - const weights = topk_weights.convert(.f32).reshape(.{ t, k }); + const x = input.reshape(.{ cfg.t, cfg.d }); + const ids = topk_ids.reshape(.{ cfg.t, cfg.k }); + const weights = topk_weights.reshape(.{ cfg.t, cfg.k }); const out_shape = x.shape(); + if (scales) |s| { + const valid = ids.cmp(.GE, Tensor.scalar(0, .i32)).logical(.AND, ids.cmp(.LT, Tensor.scalar(cfg.e, .i32))); + const inputs: ActivationStationaryFp8.Inputs = .{ .x = x, .ids = valid.select(ids, Tensor.scalar(0, .i32)), .w = valid.select(weights, Tensor.scalar(0, .f32)), .gate_up = gate_up, .down = down, .gate_up_scale = s.gate_up, .down_scale = s.down }; + const y = if (cfg.t * cfg.k <= cfg.e) + ActivationStationaryFp8.call(inputs, .{ .y = out_shape }, .{ .cfg = cfg }).y + else + BlockwiseFp8.call(inputs, .{ .y = out_shape }, .{ .cfg = cfg }).y; + return y.reshape(input.shape()); + } const y = if (cfg.t * cfg.k <= cfg.e) ActivationStationary.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y else @@ -54,7 +106,12 @@ pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up_linear return y.reshape(input.shape()); } -pub const Cfg = struct { t: i64, k: i64, e: i64, d: i64, i: i64 }; +fn scaleDtype(scales: Scales) tcl.DType { + stdx.debug.assert(scales.gate_up.dtype() == scales.down.dtype(), "TCL MoE projection scales must have matching types", .{}); + return if (scales.gate_up.dtype() == .bf16) .bf16 else .f32; +} + +pub const Cfg = struct { t: i64, k: i64, e: i64, d: i64, i: i64, scale_dtype: tcl.DType = .bf16 }; const Axes = struct { T: tcl.Axis, @@ -80,22 +137,112 @@ const Axes = struct { } }; -fn args(b: *tcl.Builder, a: Axes) tcl.FinishError!Args { - const t = try b.declareArgs(.{ +fn args(b: *tcl.Builder, a: Axes, cfg: Cfg, comptime fp8: bool) tcl.FinishError!Args { + const spec = .{ .x = .{ .dtype = .bf16, .axes = &.{ a.T, a.D } }, .ids = .{ .dtype = .i32, .axes = &.{ a.T, a.K } }, .w = .{ .dtype = .f32, .axes = &.{ a.T, a.K } }, - .gate_up = .{ .dtype = .bf16, .axes = &.{ a.E, a.I2, a.D } }, - .down = .{ .dtype = .bf16, .axes = &.{ a.E, a.D, a.I } }, + .gate_up = .{ .dtype = if (fp8) .f8_e4 else .bf16, .axes = &.{ a.E, a.I2, a.D } }, + .down = .{ .dtype = if (fp8) .f8_e4 else .bf16, .axes = &.{ a.E, a.D, a.I } }, + }; + if (!fp8) { + const t = try b.declareArgs(spec); + return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down }; + } + const IG = b.axis("IG", @divExact(cfg.i, 128)); + const DG = b.axis("DG", @divExact(cfg.d, 128)); + const I2G = b.axis("I2G", 2 * @divExact(cfg.i, 128)); + const t = try b.declareArgs(.{ + .x = spec.x, + .ids = spec.ids, + .w = spec.w, + .gate_up = spec.gate_up, + .down = spec.down, + .gate_up_scale = .{ .dtype = cfg.scale_dtype, .axes = &.{ a.E, I2G, DG } }, + .down_scale = .{ .dtype = cfg.scale_dtype, .axes = &.{ a.E, DG, IG } }, }); - return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down }; + return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down, .gate_up_scale = t.gate_up_scale, .down_scale = t.down_scale }; } -const Args = struct { x: builder.Tensor, ids: builder.Tensor, w: builder.Tensor, gate_up: builder.Tensor, down: builder.Tensor }; +const Args = struct { + x: builder.Tensor, + ids: builder.Tensor, + w: builder.Tensor, + gate_up: builder.Tensor, + down: builder.Tensor, + gate_up_scale: ?builder.Tensor = null, + down_scale: ?builder.Tensor = null, +}; -/// `silu(gate) * up` of `h = [batch..., S2, I]` (gate first), times -/// `weights` (`[batch...]`) unless null. -fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor) tcl.FinishError!builder.Tensor { +const GatheredScales = struct { gate_up: ?builder.Tensor = null, down: ?builder.Tensor = null }; + +fn gatherScales(b: *tcl.Builder, a: Axes, in: Args, ids: builder.Tensor, batch: []const tcl.Axis) tcl.FinishError!GatheredScales { + const gu = in.gate_up_scale orelse return .{}; + const IG = b.axis("IG", @divExact(a.I.size, 128)); + const DG = b.axis("DG", @divExact(a.D.size, 128)); + const gate_up = try b.reshape(gu, &.{ a.E, a.S2, IG, DG }, .{}); + var gu_axes: [8]tcl.Axis = undefined; + var dn_axes: [8]tcl.Axis = undefined; + @memcpy(gu_axes[0..batch.len], batch); + @memcpy(dn_axes[0..batch.len], batch); + @memcpy(gu_axes[batch.len..][0..3], &[_]tcl.Axis{ a.S2, IG, DG }); + @memcpy(dn_axes[batch.len..][0..2], &[_]tcl.Axis{ DG, IG }); + return .{ + .gate_up = try b.gather(gate_up, ids, a.E, gu_axes[0 .. batch.len + 3], .{}), + .down = try b.gather(in.down_scale.?, ids, a.E, dn_axes[0 .. batch.len + 2], .{}), + }; +} + +fn blockedAxes(axes: []const tcl.Axis, axis: tcl.Axis, group: tcl.Axis, lane: tcl.Axis, storage: *[16]tcl.Axis) []const tcl.Axis { + var n: usize = 0; + for (axes) |a| { + if (std.mem.eql(u8, a.name, axis.name)) { + storage[n] = group; + n += 1; + storage[n] = lane; + } else storage[n] = a; + n += 1; + } + return storage[0..n]; +} + +/// Block scales multiply each K-block partial before K-block reduction. +/// Only the requested experts are gathered; their bank remains FP8 in HBM. +fn project(b: *tcl.Builder, x: builder.Tensor, weight: builder.Tensor, scale: ?builder.Tensor, contracting: tcl.Axis, output: tcl.Axis, to: []const tcl.Axis, dtype: ?tcl.DType) tcl.FinishError!builder.Tensor { + if (scale == null) { + const op = b.tensorOperation(.{}); + return op.commit(op.contract(x, weight, to), .{ .dtype = dtype }); + } + const CG = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DG" else "IG", @divExact(contracting.size, 128)); + const CL = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DB" else "IB", 128); + const OG = b.axis(if (std.mem.eql(u8, output.name, "D")) "DG" else "IG", @divExact(output.size, 128)); + const OL = b.axis(if (std.mem.eql(u8, output.name, "D")) "DB" else "IB", 128); + var x_axes: [16]tcl.Axis = undefined; + var w_axes: [16]tcl.Axis = undefined; + var w2_axes: [16]tcl.Axis = undefined; + var out_axes: [16]tcl.Axis = undefined; + var partial_axes: [16]tcl.Axis = undefined; + const xb = try b.reshape(x, blockedAxes(x.axes, contracting, CG, CL, &x_axes), .{}); + const wb = try b.reshape(weight, blockedAxes(blockedAxes(weight.axes, output, OG, OL, &w_axes), contracting, CG, CL, &w2_axes), .{}); + const out = blockedAxes(to, output, OG, OL, &out_axes); + @memcpy(partial_axes[0..out.len], out); + partial_axes[out.len] = CG; + + // Furiosa's published FP8 recipe uses an additive epsilon, including zero blocks. + var op = b.tensorOperation(.{}); + const qscale = try op.commit(op.fetch(xb, .{}).unary(.absf).reduce(&.{CL}, .maxf).divf(448.0).addf(1e-12), .{ .dtype = .f32 }); + op = b.tensorOperation(.{}); + // Native F32 reciprocal accuracy is qualified separately (within one ULP). + const reciprocal = op.ve(.divf, .{ 1.0, op.fetch(qscale, .{}) }); + const quantized = try op.commit(reciprocal.mulf(op.fetch(xb, .{})), .{ .dtype = .f8_e4 }); + op = b.tensorOperation(.{}); + const scaled = op.contract(quantized, wb, partial_axes[0 .. out.len + 1]).mulf(op.fetch(qscale, .{})).mulf(op.fetch(scale.?, .{})).reduce(&.{CG}, .addf); + return b.reshape(try op.commit(scaled, .{ .dtype = dtype, .axes = out }), to, .{}); +} + +/// `silu(gate) * up` or `gelu_tanh(gate) * up` of `h = [batch..., S2, I]` +/// (gate first), times `weights` (`[batch...]`) unless null. +fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor, comptime fp8: bool) tcl.FinishError!builder.Tensor { var sliced: [4]tcl.Axis = undefined; var flat: [3]tcl.Axis = undefined; @memcpy(sliced[0..batch.len], batch); @@ -108,9 +255,22 @@ fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, w const gate = try b.reshape(try b.slice(h, a.S2, 0, s, .{}), f, .{}); const up = try b.reshape(try b.slice(h, a.S2, 1, s, .{}), f, .{}); // Interleaving requires one binary instruction; SwiGLU chains several. - const op = b.tensorOperation(.{ .tactic = .elementwise }); + var op = b.tensorOperation(.{ .tactic = .elementwise }); const g = op.fetch(gate, .{}); - var v = g.sigmoid().mulf(g).mulf(op.fetch(up, .{})); + const activated = if (fp8) blk: { + // WORKAROUND (TCC 2026.5.0b0): native fusion elides an internal + // BF16 SiLU write, even through as_dram/as_logical. Make its RNE + // rounding observable as integer bits before multiplying by up. + // Remove once the native reproducer preserves the BF16 boundary. + const bits = try op.commit(g.sigmoid().mulf(g).unary(.reinterpret_as_int), .{ .dtype = .i32 }); + op = b.tensorOperation(.{ .tactic = .elementwise }); + const v = op.fetch(bits, .{}); + const rounded = v.binary(.shr_logical, 16).binary(.bit_andi, 1).addi(32767).addi(v).binary(.bit_andi, -65536).unary(.reinterpret_as_float); + const silu = try op.commit(rounded, .{ .dtype = .bf16 }); + op = b.tensorOperation(.{ .tactic = .elementwise }); + break :blk op.fetch(silu, .{}); + } else g.sigmoid().mulf(g); + var v = activated.mulf(op.fetch(up, .{})); if (weights) |w| v = v.mulf(op.fetch(w, .{})); return op.commit(v, .{ .dtype = .bf16 }); } @@ -152,29 +312,39 @@ fn configure(b: *tcl.Builder, blockwise: bool) void { }); } -pub const ActivationStationary = tcl.Kernel(Cfg, .{ - .name = "moe_activation_stationary", - .inputs = &.{ "x", "ids", "w", "gate_up", "down" }, - .outputs = &.{"y"}, - .run = struct { - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - configure(b, false); - const a: Axes = .init(b, cfg); - const in = try args(b, a); - const gate_up = try b.reshape(in.gate_up, &.{ a.E, a.S2, a.I, a.D }, .{}); - const gu = try b.gather(gate_up, in.ids, a.E, &.{ a.T, a.K, a.S2, a.I, a.D }, .{}); - const dn = try b.gather(in.down, in.ids, a.E, &.{ a.T, a.K, a.D, a.I }, .{}); +pub const ActivationStationary = activationStationaryKernel(false); +pub const ActivationStationaryFp8 = activationStationaryKernel(true); - var op = b.tensorOperation(.{}); - const h = try op.commit(op.contract(in.x, gu, &.{ a.T, a.K, a.S2, a.I }), .{}); - // Routing weights scale the activations, so the down projection - // also sums over the experts. - const act = try gated(b, a, h, &.{ a.T, a.K }, in.w); - op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.contract(act, dn, &.{ a.T, a.D }), .{ .dtype = .bf16 })}); - } - }.run, -}); +fn activationStationaryKernel(comptime fp8: bool) type { + return tcl.Kernel(Cfg, .{ + .name = if (fp8) "moe_activation_stationary_fp8" else "moe_activation_stationary", + .inputs = if (fp8) &.{ "x", "ids", "w", "gate_up", "down", "gate_up_scale", "down_scale" } else &.{ "x", "ids", "w", "gate_up", "down" }, + .outputs = &.{"y"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + configure(b, false); + const a: Axes = .init(b, cfg); + const in = try args(b, a, cfg, fp8); + const gate_up = try b.reshape(in.gate_up, &.{ a.E, a.S2, a.I, a.D }, .{}); + const gu = try b.gather(gate_up, in.ids, a.E, &.{ a.T, a.K, a.S2, a.I, a.D }, .{}); + const dn = try b.gather(in.down, in.ids, a.E, &.{ a.T, a.K, a.D, a.I }, .{}); + + const scales = try gatherScales(b, a, in, in.ids, &.{ a.T, a.K }); + const h = try project(b, in.x, gu, scales.gate_up, a.D, a.I, &.{ a.T, a.K, a.S2, a.I }, if (fp8) .bf16 else null); + const act = try gated(b, a, h, &.{ a.T, a.K }, if (fp8) null else in.w, fp8); + if (fp8) { + const y = try project(b, act, dn, scales.down, a.I, a.D, &.{ a.T, a.K, a.D }, .f32); + const op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(y, .{}).mulf(op.fetch(in.w, .{})).reduce(&.{a.K}, .addf), .{ .dtype = .bf16 })}); + } else { + // The BF16 recipe weights activations and contracts experts together. + const op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.contract(act, dn, &.{ a.T, a.D }), .{ .dtype = .bf16 })}); + } + } + }.run, + }); +} /// Rows per expert block: the next power of two of the routes per expert. pub fn blockRows(cfg: Cfg) i64 { @@ -184,77 +354,162 @@ pub fn blockRows(cfg: Cfg) i64 { const blocks_per_chunk = 32; -pub const Blockwise = tcl.Kernel(Cfg, .{ - .name = "moe_blockwise", - .inputs = &.{ "x", "ids", "w", "gate_up", "down" }, +pub const Blockwise = blockwiseKernel(false); +pub const BlockwiseFp8 = blockwiseKernel(true); + +fn blockwiseKernel(comptime fp8: bool) type { + return tcl.Kernel(Cfg, .{ + .name = if (fp8) "moe_blockwise_fp8" else "moe_blockwise", + .inputs = if (fp8) &.{ "x", "ids", "w", "gate_up", "down", "gate_up_scale", "down_scale" } else &.{ "x", "ids", "w", "gate_up", "down" }, + .outputs = &.{"y"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + configure(b, true); + const a: Axes = .init(b, cfg); + const in = try args(b, a, cfg, fp8); + const bb = blockRows(cfg); + // One block per expert at least, then enough for every route. + const max_blocks = std.math.divCeil(i64, @max(cfg.t * cfg.k - cfg.e, 0), bb) catch unreachable; + const chunks = std.math.divCeil(i64, max_blocks + cfg.e, blocks_per_chunk) catch unreachable; + const BB = b.axis("BB", bb); + const BGi = b.axis("BGi", blocks_per_chunk); + const Go = b.axis("Go", chunks); + const BG = b.axis("BG", chunks * blocks_per_chunk); + const GB = b.axis("GB", chunks * blocks_per_chunk * bb); + const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; + + // Routing: route (t, k) goes to row `slot` of block `start[e] + rank`. + const experts = try b.arange(cfg.e, .i32, &.{E}, .{}); + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); + op = b.tensorOperation(.{}); + const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const rank = try op.commit(op.fetch(onehot, .{}).cumsum(&.{ T, K }), .{}); + op = b.tensorOperation(.{}); + const log2_bb: i64 = std.math.log2_int(u64, @intCast(bb)); + const blocks = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi).addi(bb - 1).binary(.shr_arith, log2_bb), .{}); + op = b.tensorOperation(.{}); + const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{E}), .{}); + op = b.tensorOperation(.{}); + const start_rows = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})).muli(bb), .{}); + op = b.tensorOperation(.{}); + const slot = try op.commit(op.fetch(rank, .{}).addi(op.fetch(start_rows, .{})).muli(op.fetch(onehot, .{})).reduce(&.{E}, .addi).subi(1), .{}); + // The expert of block g: how many experts end at or before g. + const block_ids = try b.arange(chunks * blocks_per_chunk, .i32, &.{BG}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const ends = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(block_ids, .{})), .{}); + op = b.tensorOperation(.{}); + const ended = try op.commit(op.where(ends, .le, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + // Padding blocks past the last expert reuse the last one. + const block_expert = try op.commit(op.fetch(ended, .{}).reduce(&.{E}, .addi).binary(.mini, cfg.e - 1), .{}); + const used = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(block_end), blocks_per_chunk - 1), blocks_per_chunk); + + // Rows of unused slots are never read back. + const rows = try b.reshape(try b.scatter(in.x, slot, GB, &.{ GB, D }, .{}), &.{ Go, BGi, BB, D }, .{}); + const chunk_experts = try b.reshape(block_expert, &.{ Go, BGi }, .{}); + const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); + + var loop = b.openFor(used, .{try b.scratchpad(if (fp8) .f32 else .bf16, &.{ Go, BGi, BB, D })}); + { + const chunk = try b.indexRead(rows, loop.iv); + const ids = try b.indexRead(chunk_experts, loop.iv); + const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); + const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); + const scales = try gatherScales(b, a, in, ids, &.{BGi}); + const h = try project(b, chunk, gu, scales.gate_up, D, I, &.{ BGi, BB, a.S2, I }, if (fp8) .bf16 else null); + const act = try gated(b, a, h, &.{ BGi, BB }, null, fp8); + const y = try project(b, act, dn, scales.down, I, D, &.{ BGi, BB, D }, if (fp8) .f32 else .bf16); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); + } + const out = loop.results[0]; + + const per_route = try b.gather(try b.reshape(out, &.{ GB, D }, .{}), slot, GB, &.{ T, K, D }, .{}); + op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(per_route, .{ .typecast_to = .f32 }).mulf(op.fetch(in.w, .{})).reduce(&.{K}, .addf), .{ .dtype = .bf16 })}); + } + }.run, + }); +} + +/// Experts per `ExpertStationary` loop step: more as the routes, so the +/// experts they use, grow. +pub fn stationaryChunk(cfg: Cfg) i64 { + const routes: u64 = @intCast(std.math.divCeil(i64, cfg.t * cfg.k, 16) catch unreachable); + return @intCast(std.math.clamp(std.math.ceilPowerOfTwo(u64, routes) catch unreachable, 2, 16)); +} + +/// The experts that `ids` use (-1 for none), `stationaryChunk` at a time in a +/// device loop: each is read once and runs for every token. Routing weights +/// scale the F32 down-projection results before summing the experts and steps. +pub const ExpertStationaryFp8 = tcl.Kernel(Cfg, .{ + .name = "moe_expert_stationary_fp8", + .inputs = &.{ "x", "ids", "w", "gate_up", "down", "gate_up_scale", "down_scale" }, .outputs = &.{"y"}, .run = struct { fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - configure(b, true); + configure(b, false); const a: Axes = .init(b, cfg); - const in = try args(b, a); - const bb = blockRows(cfg); - // One block per expert at least, then enough for every route. - const max_blocks = std.math.divCeil(i64, @max(cfg.t * cfg.k - cfg.e, 0), bb) catch unreachable; - const chunks = std.math.divCeil(i64, max_blocks + cfg.e, blocks_per_chunk) catch unreachable; - const BB = b.axis("BB", bb); - const BGi = b.axis("BGi", blocks_per_chunk); - const Go = b.axis("Go", chunks); - const BG = b.axis("BG", chunks * blocks_per_chunk); - const GB = b.axis("GB", chunks * blocks_per_chunk * bb); + const in = try args(b, a, cfg, true); + const chunk = stationaryChunk(cfg); + const steps = std.math.divCeil(i64, @min(cfg.e, cfg.t * cfg.k), chunk) catch unreachable; + const C = b.axis("C", chunk); + const Go = b.axis("Go", steps); + const J = b.axis("J", steps * chunk); const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; - // Routing: route (t, k) goes to row `slot` of block `start[e] + rank`. + // Each token's routing weight per expert, and the used experts. const experts = try b.arange(cfg.e, .i32, &.{E}, .{}); var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); op = b.tensorOperation(.{}); + const gate = try op.commit(op.where(op.fetch(dist, .{}), .eq, 0, op.fetch(in.w, .{ .broadcast_to = &.{ T, K, E } }), 0.0).reduce(&.{K}, .addf), .{}); + op = b.tensorOperation(.{}); const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); op = b.tensorOperation(.{}); - const rank = try op.commit(op.fetch(onehot, .{}).cumsum(&.{ T, K }), .{}); + const counts = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi), .{}); op = b.tensorOperation(.{}); - const log2_bb: i64 = std.math.log2_int(u64, @intCast(bb)); - const blocks = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi).addi(bb - 1).binary(.shr_arith, log2_bb), .{}); + const used = try op.commit(op.where(counts, .gt, 0, 1, 0), .{}); op = b.tensorOperation(.{}); - const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{E}), .{}); + const position = try op.commit(op.fetch(used, .{}).cumsum(&.{E}), .{}); + // A used expert's rank among them, -1 for the others. op = b.tensorOperation(.{}); - const start_rows = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})).muli(bb), .{}); - op = b.tensorOperation(.{}); - const slot = try op.commit(op.fetch(rank, .{}).addi(op.fetch(start_rows, .{})).muli(op.fetch(onehot, .{})).reduce(&.{E}, .addi).subi(1), .{}); - // The expert of block g: how many experts end at or before g. - const block_ids = try b.arange(chunks * blocks_per_chunk, .i32, &.{BG}, .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const ends = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(block_ids, .{})), .{}); - op = b.tensorOperation(.{}); - const ended = try op.commit(op.where(ends, .le, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - // Padding blocks past the last expert reuse the last one. - const block_expert = try op.commit(op.fetch(ended, .{}).reduce(&.{E}, .addi).binary(.mini, cfg.e - 1), .{}); - const used = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(block_end), blocks_per_chunk - 1), blocks_per_chunk); + const rank = try op.commit(op.fetch(position, .{}).muli(op.fetch(used, .{})).subi(1), .{}); + const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(position), chunk - 1), chunk); - // Rows of unused slots are never read back. - const rows = try b.reshape(try b.scatter(in.x, slot, GB, &.{ GB, D }, .{}), &.{ Go, BGi, BB, D }, .{}); - const chunk_experts = try b.reshape(block_expert, &.{ Go, BGi }, .{}); + // Slot j holds the used expert of rank j and its tokens' weights; + // slots past the last hold expert 0 with no weight. + const ranks = try b.arange(steps * chunk, .i32, &.{J}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const offset = try op.commit(op.fetch(rank, .{}).subi(op.fetch(ranks, .{})), .{}); + op = b.tensorOperation(.{}); + const slot_experts = try op.commit(op.where(op.fetch(offset, .{}), .eq, 0, op.fetch(experts, .{ .broadcast_to = &.{ E, J } }), 0).reduce(&.{E}, .addi), .{ .dtype = .i32 }); + // A reduction's condition is read plainly, so broadcast it first. + op = b.tensorOperation(.{}); + const token_offset = try op.commit(op.fetch(offset, .{ .broadcast_to = &.{ T, E, J } }), .{}); + op = b.tensorOperation(.{}); + const slot_gate = try op.commit(op.where(op.fetch(token_offset, .{}), .eq, 0, op.fetch(gate, .{ .broadcast_to = &.{ T, E, J } }), 0.0).reduce(&.{E}, .addf), .{ .dtype = .f32, .axes = &.{ J, T } }); + const step_experts = try b.reshape(slot_experts, &.{ Go, C }, .{}); + const step_gate = try b.reshape(slot_gate, &.{ Go, C, T }, .{}); const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); - var loop = b.openFor(used, .{try b.scratchpad(.bf16, &.{ Go, BGi, BB, D })}); + var loop = b.openFor(trips, .{try b.full(0, .f32, &.{ Go, T, D })}); { - const chunk = try b.indexRead(rows, loop.iv); - const ids = try b.indexRead(chunk_experts, loop.iv); - const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); - const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); + const ids = try b.indexRead(step_experts, loop.iv); + const weights = try b.indexRead(step_gate, loop.iv); + const gu = try b.gather(gate_up, ids, E, &.{ C, a.S2, I, D }, .{}); + const dn = try b.gather(in.down, ids, E, &.{ C, D, I }, .{}); + const scales = try gatherScales(b, a, in, ids, &.{C}); + const h = try project(b, in.x, gu, scales.gate_up, D, I, &.{ C, T, a.S2, I }, .bf16); + const act = try gated(b, a, h, &.{ C, T }, null, true); + const per_expert = try project(b, act, dn, scales.down, I, D, &.{ C, T, D }, .f32); op = b.tensorOperation(.{}); - const h = try op.commit(op.contract(chunk, gu, &.{ BGi, BB, a.S2, I }), .{}); - const act = try gated(b, a, h, &.{ BGi, BB }, null); - op = b.tensorOperation(.{}); - const y = try op.commit(op.contract(act, dn, &.{ BGi, BB, D }), .{ .dtype = .bf16 }); + const y = try op.commit(op.fetch(per_expert, .{}).mulf(op.fetch(weights, .{})).reduce(&.{C}, .addf), .{ .dtype = .f32 }); try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); } - const out = loop.results[0]; - - const per_route = try b.gather(try b.reshape(out, &.{ GB, D }, .{}), slot, GB, &.{ T, K, D }, .{}); op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(per_route, .{ .typecast_to = .f32 }).mulf(op.fetch(in.w, .{})).reduce(&.{K}, .addf), .{ .dtype = .bf16 })}); + b.ret(&.{try op.commit(op.fetch(loop.results[0], .{}).reduce(&.{Go}, .addf), .{ .dtype = .bf16 })}); } }.run, }); @@ -286,7 +541,7 @@ test "tcl fused experts run on furiosa" { }); } }; - var exe = try zml.module.compile(allocator, io, Mod.forward, .{ x, ids, w, gu, dn }, platform, .{}); + var exe = try platform.compileFn(allocator, io, Mod.forward, .{ x, ids, w, gu, dn }, .{}); defer exe.deinit(); var prng: std.Random.DefaultPrng = .init(@intCast(t)); From dc829cfb7d052f4929524261329d92daa4f30018 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 13 Nov 2026 10:50:15 +0100 Subject: [PATCH 870/881] [Modularization] zml/kernels/moe: replace `options.zig` with `api.zig` (#1059) Also moved `LocalRouting` to `api.zig` --- zml/BUILD.bazel | 2 +- zml/kernels/moe/{options.zig => api.zig} | 32 +++++++++++++ zml/kernels/moe/cute/cute.zig | 7 +-- zml/kernels/moe/cute/sm10x/moe.zig | 5 ++- .../flashinfer_cutlass/flashinfer_cutlass.zig | 10 ++--- zml/kernels/moe/fly/fly.zig | 15 ++++--- zml/kernels/moe/metal/metal.zig | 7 +-- zml/kernels/moe/moe.zig | 45 +++---------------- zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig | 7 +-- zml/kernels/moe/stablehlo/stablehlo.zig | 9 ++-- zml/kernels/moe/tcl/tcl.zig | 6 +-- zml/kernels/moe/tests.zig | 8 ++-- zml/kernels/moe/triton/moe.zig | 2 +- zml/kernels/moe/triton/triton.zig | 17 +++---- 14 files changed, 90 insertions(+), 82 deletions(-) rename zml/kernels/moe/{options.zig => api.zig} (75%) diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index fff3b8a8..b47d4f59 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -90,8 +90,8 @@ zig_library( "kernel.zig", "kernels/flash_attention/flash_attention.zig", "kernels/flash_attention/tests.zig", + "kernels/moe/api.zig", "kernels/moe/moe.zig", - "kernels/moe/options.zig", "kernels/moe/tests.zig", "kernels/moe/weights.zig", "kernels/paged_attention/options.zig", diff --git a/zml/kernels/moe/options.zig b/zml/kernels/moe/api.zig similarity index 75% rename from zml/kernels/moe/options.zig rename to zml/kernels/moe/api.zig index 7f8d26de..b66593c1 100644 --- a/zml/kernels/moe/options.zig +++ b/zml/kernels/moe/api.zig @@ -130,3 +130,35 @@ pub const ExpertsLayout = struct { mxfp4_interleaved8, }; }; + +/// Routing for a local token batch and a contiguous range of experts. +/// IDs retain the model's global expert numbering, while the projection weights +/// contain only the local experts. Kernels compute those experts contributions and +/// `forwardMoe` sums them along the expert axis. +pub const LocalRouting = struct { + /// Global expert IDs for local tokens, with shape {.t, .topk} + ids: Tensor, + weights: Tensor, + global_num_experts: u32, + local_num_experts: u32, + /// Scalar i32 containing the first local expert's global ID. + /// Null when all experts are local and global IDs already index the weights. + expert_start: ?Tensor = null, + + /// Maps routing IDs to local weight indices, using -1 for nonlocal experts. + /// For a shard owning experts 4–7, routing IDs [1, 6] become [-1, 2]. + pub fn localIds(self: LocalRouting) Tensor { + const start = self.expert_start orelse return self.ids; + const local = self.ids.cmp(.GE, start).logical(.AND, self.ids.cmp(.LT, start.addConstant(self.local_num_experts))); + return local.select(self.ids.sub(start), .scalar(-1, .i32)); + } + + /// Returns a lookup table from every global expert ID to its local weight + /// index or -1 if nonlocal. Returns null when every expert is local. + pub fn expertMap(self: LocalRouting) ?Tensor { + const start = self.expert_start orelse return null; + const ids = Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); + const local = ids.cmp(.GE, start).logical(.AND, ids.cmp(.LT, start.addConstant(self.local_num_experts))); + return local.select(ids.sub(start), .scalar(-1, .i32)); + } +}; diff --git a/zml/kernels/moe/cute/cute.zig b/zml/kernels/moe/cute/cute.zig index bd482093..8bc03c9c 100644 --- a/zml/kernels/moe/cute/cute.zig +++ b/zml/kernels/moe/cute/cute.zig @@ -3,8 +3,9 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const Options = @import("../options.zig").Options; -const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; +const Options = @import("../api.zig").Options; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const LocalRouting = @import("../api.zig").LocalRouting; pub const kernels = @import("sm10x/moe.zig"); // `moe.zig`'s `refAllDecls` reaches this file but not its imports, so the @@ -41,7 +42,7 @@ pub fn validateOptions(opts: Options) void { /// interleave by 8 and its scale atoms are MN-major packed i32 words. pub fn fusedExperts( input: zml.Tensor, - routing: zml.moe.LocalRouting, + routing: LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, options: Options, diff --git a/zml/kernels/moe/cute/sm10x/moe.zig b/zml/kernels/moe/cute/sm10x/moe.zig index f651813f..ff6edec2 100644 --- a/zml/kernels/moe/cute/sm10x/moe.zig +++ b/zml/kernels/moe/cute/sm10x/moe.zig @@ -4,16 +4,17 @@ //! rows in route order, and the top-k reduction. MXFP4 applies routing weights //! before the down projection, with the reference model's BF16 rounding. const std = @import("std"); + const stdx = @import("stdx"); const zml = @import("../../../../zml.zig"); +const RoutingWeightPlacement = @import("../../api.zig").RoutingWeightPlacement; +const triton_kernels = @import("../../triton/moe.zig"); pub const mxfp4 = @import("mxfp4.zig"); pub const mxfp4_persistent = @import("persistent_mxfp4.zig"); pub const nvfp4 = @import("nvfp4.zig"); pub const nvfp4_persistent = @import("persistent_nvfp4.zig"); const reduce = @import("reduce.zig"); -const RoutingWeightPlacement = @import("../../options.zig").RoutingWeightPlacement; -const triton_kernels = @import("../../triton/moe.zig"); pub const Mxfp4Inputs = struct { /// `[tokens, hidden]` BF16. diff --git a/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig index d1a0e1b5..92317c7c 100644 --- a/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig +++ b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig @@ -5,10 +5,12 @@ const platforms = @import("platforms"); const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); -const GlobalOptions = @import("../options.zig").Options; -const Activation = @import("../options.zig").Activation; +const GlobalOptions = @import("../api.zig").Options; +const Activation = @import("../api.zig").Activation; +const LocalRouting = @import("../api.zig").LocalRouting; const unpackedWeight = @import("../weights.zig").unpackedWeight; const contract = @import("contract.zig"); +const Attributes = contract.Attributes; const runtime = @import("runtime.zig"); const log = std.log.scoped(.@"zml/moe/flashinfer_cutlass"); @@ -50,8 +52,6 @@ const Inputs = contract.Inputs(zml.Tensor); const Outputs = contract.Outputs(zml.Shape); -const Attributes = contract.Attributes; - fn currentRunners() !*runtime.Runners { const platform = zml.Compiler.current().platform; const runners = platform.state.cuda.flashinfer_cutlass_moe_runners; @@ -311,7 +311,7 @@ pub fn fusedExpertsNvfp4( pub fn fusedExperts( input: zml.Tensor, - routing: zml.moe.LocalRouting, + routing: LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: GlobalOptions, diff --git a/zml/kernels/moe/fly/fly.zig b/zml/kernels/moe/fly/fly.zig index d041a5ab..ba3d3c00 100644 --- a/zml/kernels/moe/fly/fly.zig +++ b/zml/kernels/moe/fly/fly.zig @@ -6,11 +6,12 @@ const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; const DataType = zml.DataType; -const ExpertsLayout = @import("../options.zig").ExpertsLayout; -const ProjectionLayout = @import("../options.zig").ProjectionLayout; -const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; -const Activation = @import("../options.zig").Activation; -const Options = @import("../options.zig").Options; +const ExpertsLayout = @import("../api.zig").ExpertsLayout; +const ProjectionLayout = @import("../api.zig").ProjectionLayout; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const Activation = @import("../api.zig").Activation; +const Options = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; const fly = @import("moe.zig"); const log = std.log.scoped(.@"zml/moe/fused_experts"); @@ -21,7 +22,7 @@ test { pub fn fusedExperts( hidden_states: Tensor, - routing_: zml.moe.LocalRouting, + routing_: LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: Options, @@ -135,7 +136,7 @@ pub fn fusedExperts( } // TODO(Corentin): unused for now -pub fn supports(hidden_states: Tensor, routing: zml.moe.LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: Options) bool { +pub fn supports(hidden_states: Tensor, routing: LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: Options) bool { if (hidden_states.dtype() != .bf16 or gate_up_.quantizationScheme() != .mxfp4 or down_.quantizationScheme() != .mxfp4 or gate_up_.bias != null or down_.bias != null or opts.routing_weight_placement != .before_down) return false; diff --git a/zml/kernels/moe/metal/metal.zig b/zml/kernels/moe/metal/metal.zig index 988a4f51..f835c889 100644 --- a/zml/kernels/moe/metal/metal.zig +++ b/zml/kernels/moe/metal/metal.zig @@ -4,8 +4,9 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; -const Activation = @import("../options.zig").Activation; -const GlobalOptions = @import("../options.zig").Options; +const Activation = @import("../api.zig").Activation; +const GlobalOptions = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; const unpackedWeight = @import("../weights.zig").unpackedWeight; const log = std.log.scoped(.@"zml/moe/metal"); @@ -68,7 +69,7 @@ fn applyDownGlobalScale(output: Tensor, global_scale: ?Tensor, expert_ids: Tenso pub fn fusedExperts( hidden_states: zml.Tensor, - routing: zml.moe.LocalRouting, + routing: LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: GlobalOptions, diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index 2a0aa8d7..45f41b2a 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -4,17 +4,18 @@ const platforms = @import("platforms"); const zml = @import("../../zml.zig"); const stdx = zml.stdx; +pub const Activation = @import("api.zig").Activation; +pub const ActivationKind = @import("api.zig").ActivationKind; pub const cute = @import("cute/cute.zig"); -pub const ExpertsLayout = @import("options.zig").ExpertsLayout; +pub const ExpertsLayout = @import("api.zig").ExpertsLayout; pub const flashinfer_cutlass = @import("flashinfer_cutlass/flashinfer_cutlass.zig"); pub const fly = @import("fly/fly.zig"); +pub const LocalRouting = @import("api.zig").LocalRouting; pub const metal = @import("metal/metal.zig"); pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); -pub const Options = @import("options.zig").Options; -pub const Activation = @import("options.zig").Activation; -pub const ActivationKind = @import("options.zig").ActivationKind; -pub const ProjectionLayout = @import("options.zig").ProjectionLayout; -pub const RoutingWeightPlacement = @import("options.zig").RoutingWeightPlacement; +pub const Options = @import("api.zig").Options; +pub const ProjectionLayout = @import("api.zig").ProjectionLayout; +pub const RoutingWeightPlacement = @import("api.zig").RoutingWeightPlacement; pub const stablehlo = @import("stablehlo/stablehlo.zig"); pub const tcl = @import("tcl/tcl.zig"); pub const triton = @import("triton/triton.zig"); @@ -138,38 +139,6 @@ pub const Backend = enum { } }; -/// Routing for a local token batch and a contiguous range of experts. -/// IDs retain the model's global expert numbering, while the projection weights -/// contain only the local experts. Kernels compute those experts contributions and -/// `forwardMoe` sums them along the expert axis. -pub const LocalRouting = struct { - /// Global expert IDs for local tokens, with shape {.t, .topk} - ids: zml.Tensor, - weights: zml.Tensor, - global_num_experts: u32, - local_num_experts: u32, - /// Scalar i32 containing the first local expert's global ID. - /// Null when all experts are local and global IDs already index the weights. - expert_start: ?zml.Tensor = null, - - /// Maps routing IDs to local weight indices, using -1 for nonlocal experts. - /// For a shard owning experts 4–7, routing IDs [1, 6] become [-1, 2]. - pub fn localIds(self: LocalRouting) zml.Tensor { - const start = self.expert_start orelse return self.ids; - const local = self.ids.cmp(.GE, start).logical(.AND, self.ids.cmp(.LT, start.addConstant(self.local_num_experts))); - return local.select(self.ids.sub(start), .scalar(-1, .i32)); - } - - /// Returns a lookup table from every global expert ID to its local weight - /// index or -1 if nonlocal. Returns null when every expert is local. - pub fn expertMap(self: LocalRouting) ?zml.Tensor { - const start = self.expert_start orelse return null; - const ids = zml.Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); - const local = ids.cmp(.GE, start).logical(.AND, ids.cmp(.LT, start.addConstant(self.local_num_experts))); - return local.select(ids.sub(start), .scalar(-1, .i32)); - } -}; - /// Check the shared token-wise contract for activations and routing tensors. pub fn validateRoutingInputs(input: zml.Tensor, ids: zml.Tensor, weights: zml.Tensor) void { stdx.debug.assert(input.rank() == 2 and ids.rank() == 2 and weights.rank() == 2, "MoE expects input {{.t, .d}} and routing {{.t, .topk}}, got {f}, {f}, {f}", .{ input.shape(), ids.shape(), weights.shape() }); diff --git a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig index 4b33b585..bb60594b 100644 --- a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig @@ -5,8 +5,9 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; -const Activation = @import("../options.zig").Activation; -const GlobalOptions = @import("../options.zig").Options; +const Activation = @import("../api.zig").Activation; +const GlobalOptions = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; pub const gmm_ep = @import("gmm_ep.zig"); const log = std.log.scoped(.@"zml/moe/mosaic_tpu"); @@ -214,7 +215,7 @@ pub fn callGmmEp( pub fn fusedExperts( hidden_states: zml.Tensor, - routing: zml.moe.LocalRouting, + routing: LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: GlobalOptions, diff --git a/zml/kernels/moe/stablehlo/stablehlo.zig b/zml/kernels/moe/stablehlo/stablehlo.zig index 7f84835b..ba723234 100644 --- a/zml/kernels/moe/stablehlo/stablehlo.zig +++ b/zml/kernels/moe/stablehlo/stablehlo.zig @@ -7,9 +7,10 @@ const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; -const GlobalOptions = @import("../options.zig").Options; -const Activation = @import("../options.zig").Activation; -const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; +const GlobalOptions = @import("../api.zig").Options; +const Activation = @import("../api.zig").Activation; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const LocalRouting = @import("../api.zig").LocalRouting; pub fn activate(x: Tensor, activation: Activation) Tensor { return switch (activation) { @@ -191,7 +192,7 @@ pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: .sum(.topk).squeeze(.topk).convert(input.dtype()); } -pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: GlobalOptions) Tensor { +pub fn fusedExperts(input: Tensor, routing: LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: GlobalOptions) Tensor { stdx.debug.assert(routing.ids.dim(.topk) > 0 and routing.ids.dim(.topk) <= routing.global_num_experts, "MoE top-k must fit the global expert count", .{}); return reference(input, routing.localIds(), routing.weights, gate_up, down, .{ .activation = opts.activation, diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig index e7d7a01b..6238d5f6 100644 --- a/zml/kernels/moe/tcl/tcl.zig +++ b/zml/kernels/moe/tcl/tcl.zig @@ -19,18 +19,18 @@ const std = @import("std"); const builder = @import("dsl/tcl/builder"); - const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; -const Options = @import("../options.zig").Options; +const Options = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; /// `input` is `{.t, .d}`, `topk_ids` and `topk_weights` `{.t, .topk}`, `gate_up` `{.expert, .dout = 2 * inter (gate then up), .d}` /// and `down` `{.expert, .d, .dout = inter}`. Experts are BF16 or block-128 FP8 with BF16/F32 scales. /// SwiGLU, routing weights applied after the down projection. -pub fn fusedExperts(input: Tensor, routing: zml.moe.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: Options) Tensor { +pub fn fusedExperts(input: Tensor, routing: LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: Options) Tensor { const topk_ids = routing.localIds(); stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE owns input precision and applies routing weights after the down projection", .{}); stdx.debug.assert(opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}), "TCL MoE supports SwiGLU without limit, scale or bias", .{}); diff --git a/zml/kernels/moe/tests.zig b/zml/kernels/moe/tests.zig index f78cd1ef..c74406c3 100644 --- a/zml/kernels/moe/tests.zig +++ b/zml/kernels/moe/tests.zig @@ -5,10 +5,10 @@ const std = @import("std"); const zml = @import("../../zml.zig"); const Tensor = zml.Tensor; const Shape = zml.Shape; -const ExpertsLayout = @import("options.zig").ExpertsLayout; -const GlobalOptions = @import("options.zig").Options; -const Activation = @import("options.zig").Activation; -const Placement = @import("options.zig").RoutingWeightPlacement; +const Activation = @import("api.zig").Activation; +const ExpertsLayout = @import("api.zig").ExpertsLayout; +const GlobalOptions = @import("api.zig").Options; +const Placement = @import("api.zig").RoutingWeightPlacement; const reference = @import("stablehlo/stablehlo.zig"); const Case = struct { diff --git a/zml/kernels/moe/triton/moe.zig b/zml/kernels/moe/triton/moe.zig index 0776a590..6e4006fa 100644 --- a/zml/kernels/moe/triton/moe.zig +++ b/zml/kernels/moe/triton/moe.zig @@ -12,7 +12,7 @@ const tri = zml.kernel.triton; const Builder = tri.Builder; const Value = tri.Value; const DType = tri.DType; -const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; const log = std.log.scoped(.@"zml/moe/triton_kernels"); diff --git a/zml/kernels/moe/triton/triton.zig b/zml/kernels/moe/triton/triton.zig index ed6c9f5f..8ba799a5 100644 --- a/zml/kernels/moe/triton/triton.zig +++ b/zml/kernels/moe/triton/triton.zig @@ -7,11 +7,12 @@ const Tensor = zml.Tensor; const Shape = zml.Shape; const DataType = zml.DataType; const toDType = zml.kernel.triton.from; -const ExpertsLayout = @import("../options.zig").ExpertsLayout; -const ProjectionLayout = @import("../options.zig").ProjectionLayout; -const RoutingWeightPlacement = @import("../options.zig").RoutingWeightPlacement; -const Activation = @import("../options.zig").Activation; -const Options = @import("../options.zig").Options; +const ExpertsLayout = @import("../api.zig").ExpertsLayout; +const ProjectionLayout = @import("../api.zig").ProjectionLayout; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const Activation = @import("../api.zig").Activation; +const Options = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; const kernels = @import("moe.zig"); const log = std.log.scoped(.@"zml/moe/triton"); @@ -38,7 +39,7 @@ test { pub fn fusedExperts( input: zml.Tensor, - routing: zml.moe.LocalRouting, + routing: LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, options: Options, @@ -58,7 +59,7 @@ pub fn fusedExperts( pub fn fusedExpertsBase( hidden_states: Tensor, - routing_: zml.moe.LocalRouting, + routing_: LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, layout: ProjectionLayout, @@ -900,7 +901,7 @@ fn validateInputs( /// BF16 activations are quantized on the GPU to FP8 with per-32 E8M0 scales. pub fn fusedExpertsMxfp4( input: zml.Tensor, - routing: zml.moe.LocalRouting, + routing: LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, options: Options, From 2d45ef0804132eb4309b71f9b261b52de4ddd02c Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Fri, 13 Nov 2026 13:52:07 +0100 Subject: [PATCH 871/881] zml/platform: fix `zml$print` leak (#1060) --- zml/platform.zig | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/zml/platform.zig b/zml/platform.zig index 6c6081aa..54fb90e4 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -1015,12 +1015,15 @@ fn printCallback(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { if (call_frame.registeringHook()) return null; + const buffers = call_frame.args.buffers(); + if (buffers.len <= 0) return null; + const pjrt_api: *pjrt.Api = @ptrFromInt(getScalarAttributeAs(u64, call_frame, "pjrt_api").?); const pjrt_client: *pjrt.Client = @ptrFromInt(getScalarAttributeAs(u64, call_frame, "pjrt_client").?); const device_ordinal: usize = @intCast(try call_frame.ctx.getDeviceOrdinal(call_frame.api)); - const buffer = call_frame.args.buffers()[0]; + const buffer = buffers[0]; const shape = shapeFromFfiBuffer(buffer); // NOTE(Corentin): This is a hack. We take the first non device memory, hoping that it's host visible, @@ -1044,9 +1047,13 @@ fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { }, }, }); + defer pjrt_buffer.deinit(pjrt_api); pjrt_buffer = try pjrt_buffer.copyToMemory(pjrt_api, first_non_device_memory); - try pjrt_buffer.readyEvent(pjrt_api).awaitRaw(pjrt_api); + + const event = pjrt_buffer.readyEvent(pjrt_api); + defer event.deinit(pjrt_api); + try event.awaitRaw(pjrt_api); const host_visible_data: [*]u8 = @ptrCast(@alignCast(try pjrt_buffer.opaqueDeviceMemoryDataPointer(pjrt_api))); From 8c41b898019a2511d05da9d90e54188569ef962d Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Fri, 13 Nov 2026 13:52:58 +0100 Subject: [PATCH 872/881] zml/ops: manualComputation v2 (#1045) Last PR for manualComputation v2 which promote as default and only one API for manual computation. Custom calls now require to be wrapped in a manual computation too. --- dsl/tcl/README.md | 3 +- examples/sharding/main.zig | 4 +- zml/BUILD.bazel | 1 - zml/attention/attention.zig | 14 +- zml/kernel.zig | 9 +- zml/kernels/moe/moe.zig | 4 +- .../paged_attention/cute/benchmark.zig | 2 +- .../paged_attention/paged_attention.zig | 7 +- zml/kernels/paged_attention/tcl/tcl.zig | 13 - zml/nn.zig | 26 +- zml/ops.zig | 94 +--- zml/ops/manual_computation.zig | 494 +++++++++++------- zml/ops/manual_computation2.zig | 465 ----------------- zml/tensor.zig | 2 +- 14 files changed, 376 insertions(+), 762 deletions(-) delete mode 100644 zml/ops/manual_computation2.zig diff --git a/dsl/tcl/README.md b/dsl/tcl/README.md index ec1ae61f..b01b7189 100644 --- a/dsl/tcl/README.md +++ b/dsl/tcl/README.md @@ -104,8 +104,7 @@ def softmax_kernel(B: Axis = 2, D: Axis = 1024, S: Axis = 16, v0: [B, S, D]/bf16 across the partitions; other kernels stay per chip. Its arguments and results are DRAM tensors: `Dram{ .chip = &.{.{ .axis = C }} }` splits axis `C`, sized to the chip count, across chips (each partition holds its share), - the default `.broadcast` keeps a copy per chip. Call it inside - `zml.ops.manualComputation` so it sees each partition's shards. + the default `.broadcast` keeps a copy per chip. * **Convolutions** pad and slide the input in its read: a slide replaces its axis by `window_axis, frame_axis`, then `contract` with the filter. On its own (without a slide) the padding is dropped. diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index 7e519bf0..34a78bfe 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -42,7 +42,7 @@ const DemoModel = struct { const spec: zml.Sharding.Partitioning.Logical = .parse(.{ .batch = .data, .hidden = .model }); - return zml.ops.manualComputation2( + return zml.ops.manualComputation( (struct { y: zml.Tensor, gate: zml.Tensor, @@ -70,7 +70,7 @@ fn usage() void { \\Simple sharding demo: \\ - explicit partitioning on data/model axes \\ - tensor print - \\ - manualComputation2 with multiple inputs and one output + \\ - manualComputation with multiple inputs and one output \\ \\Mesh modes: \\ - auto: physical_mesh=auto, inferred cpu device_count=2 diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index b47d4f59..2b14c79f 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -103,7 +103,6 @@ zig_library( "nn.zig", "ops.zig", "ops/manual_computation.zig", - "ops/manual_computation2.zig", "pjrtx.zig", "platform.zig", "profiling/profiler.zig", diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index 0ac2a95c..edd9b4ca 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -97,7 +97,6 @@ pub const Options = @import("Options.zig"); pub const Partitioning = struct { const Logical = zml.Sharding.Partitioning.Logical; - const InputSpecs = @FieldType(zml.ops.ManualPartitioning(Attention.forward), "in_specs"); in_specs: struct { q: Logical, @@ -105,7 +104,7 @@ pub const Partitioning = struct { v: Logical, token_index: Logical = .replicated, /// Omit to replicate backend metadata tensors. - metadata: ?@FieldType(InputSpecs, "metadata") = null, + metadata: ?zml.ops.PartitionSpecs(Metadata) = null, }, out_specs: Logical, @@ -135,20 +134,13 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T /// `attention`, causal or not depending on `opts`. pub fn attentionOpts(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { const args: Attention = .{ .q = q, .k = k, .v = v, .token_index = token_index, .metadata = metadata, .parameters = parameters, .opts = opts }; - return zml.ops.manualComputation2(Attention.forward, args, q.shape(), mesh, .{ + return zml.ops.manualComputation(Attention.forward, args, q.shape(), mesh, .{ .in_specs = .{ .q = partitioning.in_specs.q, .k = partitioning.in_specs.k, .v = partitioning.in_specs.v, .token_index = partitioning.in_specs.token_index, - .metadata = partitioning.in_specs.metadata orelse switch (metadata) { - .stablehlo => .{ .stablehlo = {} }, - .flash_attention => |metadata_| .{ .flash_attention = switch (metadata_) { - .cute => .{ .cute = {} }, - .nki => .{ .nki = {} }, - .metal => .{ .metal = .{ .num_tokens = .replicated } }, - } }, - }, + .metadata = partitioning.in_specs.metadata orelse zml.ops.partitionSpecs(metadata, .replicated), }, .out_specs = partitioning.out_specs, }); diff --git a/zml/kernel.zig b/zml/kernel.zig index d85b6a56..f111f721 100644 --- a/zml/kernel.zig +++ b/zml/kernel.zig @@ -1534,13 +1534,16 @@ test "tcl multi-chip kernels run on two furiosa chips" { const tp = zml.testing.tpMesh(); const x_shape = Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(tp, .{ .c = .model }); const Mod = struct { - pub fn forward(x: Tensor) [2]Tensor { - const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), Shape.init(.{ .c = 2, .a = 1024 }, .f32) }); + pub fn forward(x: Tensor, mesh: *const zml.Sharding.Mesh) [2]Tensor { + const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), Shape.init(.{ .c = 2, .a = 1024 }, .f32) }, mesh, .{ + .in_specs = .{ .x = .parse(.{ .c = .model }) }, + .out_specs = .{ .parse(.{ .c = .model }), .replicated }, + }); return .{ r[0], r[1] }; } }; const x: Tensor = .fromShape(x_shape); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{ .meshes = &.{tp} }); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, tp }, platform, .{ .meshes = &.{tp} }); defer exe.deinit(); var hx: [2 * 1024]f32 = undefined; for (&hx, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 13)) * 0.125 - 0.5; diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index 45f41b2a..eb697f36 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -266,7 +266,7 @@ pub fn forwardMoe( .global_experts = @intCast(gate_up.weight.dim(.expert)), }; const Logical = zml.Sharding.Partitioning.Logical; - var projection_specs: [2]zml.meta.MapRestrict(zml.Tensor, Logical).map(zml.nn.Linear) = undefined; + var projection_specs: [2]zml.ops.PartitionSpecs(zml.nn.Linear) = undefined; for ([_]zml.nn.Linear{ gate_up, down }, &projection_specs) |linear, *specs| { zml.meta.mapAlloc(struct { fn spec(ctx: struct { experts: i64, axis: ?zml.Shape.Tag }, tensor: zml.Tensor) Logical { @@ -285,7 +285,7 @@ pub fn forwardMoe( .axis = if (expert_axis) |axis| mesh.logical.axes.get(axis) else null, }, linear, specs) catch @panic("OOM"); } - return zml.ops.manualComputation2(Call.forward, args, output_shape, mesh, .{ + return zml.ops.manualComputation(Call.forward, args, output_shape, mesh, .{ .in_specs = .{ .input = partitioning.in_specs.input, .ids = partitioning.in_specs.ids, diff --git a/zml/kernels/paged_attention/cute/benchmark.zig b/zml/kernels/paged_attention/cute/benchmark.zig index 3e05fed8..587af69f 100644 --- a/zml/kernels/paged_attention/cute/benchmark.zig +++ b/zml/kernels/paged_attention/cute/benchmark.zig @@ -125,7 +125,7 @@ fn forward(inputs: Inputs, c: cases.Case, mesh: *const zml.Mesh) zml.Tensor { } }; const heads: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); - return zml.ops.manualComputation2(Call.local, .{ .inputs = inputs, .case = c }, inputs.q.shape(), mesh, .{ + return zml.ops.manualComputation(Call.local, .{ .inputs = inputs, .case = c }, inputs.q.shape(), mesh, .{ .in_specs = .{ .inputs = .{ .q = heads, .k = heads, diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index aaa42b20..4a97a38e 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -207,7 +207,7 @@ pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: z _ = k; _ = v; const args: Attention = .{ .parameters = parameters, .q = q, .kv_cache = kv_cache, .opts = opts }; - return zml.ops.manualComputation2(Attention.forward, args, q.shape(), mesh, partitioning); + return zml.ops.manualComputation(Attention.forward, args, q.shape(), mesh, partitioning); } const Attention = struct { @@ -435,12 +435,9 @@ test pagedAttention { const parameters: Parameters = .init(.fromBackend(backend_options_args)); const head_spec: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); - const ParameterSpecs = zml.meta.MapRestrict(zml.Tensor, zml.Sharding.Partitioning.Logical).map(Parameters); const partitioning: Partitioning = .{ .in_specs = .{ - .parameters = switch (parameters) { - inline else => |_, tag| @unionInit(ParameterSpecs, @tagName(tag), .{ .block_table = .replicated, .seq_lens = .replicated, .query_start_len = .replicated }), - }, + .parameters = try zml.ops.partitionSpecsAlloc(arena, parameters, .replicated), .q = head_spec, .kv_cache = .{ .split = .{ .k = head_spec, .v = head_spec } }, .opts = .{ .mask = .{ .standard = {} }, .sink = null }, diff --git a/zml/kernels/paged_attention/tcl/tcl.zig b/zml/kernels/paged_attention/tcl/tcl.zig index 881593aa..d04b684e 100644 --- a/zml/kernels/paged_attention/tcl/tcl.zig +++ b/zml/kernels/paged_attention/tcl/tcl.zig @@ -67,22 +67,9 @@ pub const paged = struct { const last_pe = attributes.get("pe_end"); stdx.debug.assert(first_pe != null and first_pe.? == .int64 and first_pe.?.int64 == 0 and last_pe != null and last_pe.? == .int64 and last_pe.?.int64 == 7, "TCL attention requires a complete 8-PE RNGD chip", .{}); - if (compiler.meshes[0].numDevices() > 1) return zml.ops.manualComputation(Shard.body, .{ .parameters = parameters, .q = q, .k_cache = k_cache, .v_cache = v_cache, .opts = opts }, q.shape()); return attend(parameters, q, k_cache, v_cache, opts); } - const Shard = struct { - parameters: Parameters, - q: Tensor, - k_cache: Tensor, - v_cache: Tensor, - opts: AttentionOptions, - - fn body(self: Shard, _: zml.Shape) Tensor { - return attend(self.parameters, self.q, self.k_cache, self.v_cache, self.opts); - } - }; - fn attend(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: AttentionOptions) Tensor { if (q.dim(.b) == 0) return Tensor.zeroes(q.shape()); const page = k_cache.dim(.k_chunk); diff --git a/zml/nn.zig b/zml/nn.zig index a2b582ee..eceb316b 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -2,11 +2,13 @@ const std = @import("std"); const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); +const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); const quantization = @import("quantization.zig"); pub const Quantization = quantization.Quantization; @@ -320,13 +322,23 @@ fn onesGrid(shape: Shape, dt: DataType) Shape { } fn scaledDotReference(in: []const Tensor, out_shape: Shape) Tensor { - return ops.customCall( - "zml$scaled_dot_unmatched", - .{ in[0], in[1], in[2], in[3] }, - out_shape, - {}, - .{ .has_side_effect = false }, - ); + // Keep the unmatched-composite marker in the graph for backend lowering. + const ctx = zml.Compiler.current(); + const op = stablehlo.custom_call( + ctx.mlir_ctx, + &.{ in[0].value(), in[1].value(), in[2].value(), in[3].value() }, + &.{mlirx.Type.rankedTensor(ctx.mlir_ctx, out_shape)}, + .{ + .call_target_name = "zml$scaled_dot_unmatched", + .has_side_effect = false, + .backend_config = .{ .typed_ffi = .dict(ctx.mlir_ctx, &.{}) }, + }, + ctx.location, + ).appendTo(ctx.currentScope().block); + if (ctx.manual_computation_depth > 0 and ctx.partitioner == .gspmd) { + op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); + } + return Tensor._result(out_shape, op.result(0)); } fn applyGlobalScale(acc: Tensor, igs: ?Tensor, wgs: ?Tensor) Tensor { diff --git a/zml/ops.zig b/zml/ops.zig index a9ae0f8d..f8fdba7e 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -18,9 +18,10 @@ const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); const manual_computation = @import("ops/manual_computation.zig"); pub const manualComputation = manual_computation.manualComputation; -const manual_computation2 = @import("ops/manual_computation2.zig"); -pub const manualComputation2 = manual_computation2.manualComputation2; -pub const ManualPartitioning = manual_computation2.ManualPartitioning; +pub const ManualPartitioning = manual_computation.ManualPartitioning; +pub const PartitionSpecs = manual_computation.PartitionSpecs; +pub const partitionSpecs = manual_computation.partitionSpecs; +pub const partitionSpecsAlloc = manual_computation.partitionSpecsAlloc; const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const ShapeToCustomCallBuffer = @import("pjrtx.zig").ShapeToCustomCallBuffer; @@ -30,7 +31,6 @@ const TensorToCustomCallBuffer = @import("pjrtx.zig").TensorToCustomCallBuffer; test { _ = manual_computation; - _ = manual_computation2; } const log = std.log.scoped(.@"zml/Compiler"); @@ -89,7 +89,7 @@ pub fn allReduce(inputs: anytype, comptime func_: anytype) AllReduceReturnType(@ /// ``` /// /// Each expert shard now has the complete output for its data shard. To expose -/// this result through `manualComputation2`, an output spec such as +/// this result through `manualComputation`, an output spec such as /// `.parse(.{ .t = .data })` keeps tokens split across `.data` and declares the /// result replicated along `.experts`. pub fn allReduceOnAxis(inputs: anytype, comptime func_: anytype, mesh: *const Sharding.Mesh, axis: anytype) AllReduceReturnType(@TypeOf(inputs)) { @@ -2760,11 +2760,6 @@ pub fn CustomCall( comptime params: struct { /// Name of the custom call, must be unique. name: [:0]const u8, - /// Custom call will see the data of a single device. So for a sharded buffer, it will only see a single partition. This applies - /// for both the input and output buffers. By default, the partitioning algorithm will assume nothing on the custom call input/output, - /// so everything will be forced as replicated. If do you know / ensure that the custom call is consistent with the provided input/output - /// sharding, this setting will avoid the shuffling of the buffer's data. - sharding_aware: bool, /// Whether the function has any side-effect, meaning that XLA cannot re-order it/optimize it out. A typical example is print. has_side_effect: bool, /// Similar to reuseBuffer on tensors, tells XLA that the custom call re-uses an input buffer for its output. @@ -2798,23 +2793,7 @@ pub fn CustomCall( .output_operand_aliases = comptime customCallOutputOperandAliases(I, O, params.output_operand_aliases), .compute_on_host = params.compute_on_host, }; - if (params.sharding_aware) { - return shardingAwareTypedCustomCall( - params.name, - opts, - input_tensors, - output_shapes, - attributes, - ); - } else { - return typedCustomCall( - params.name, - opts, - input_tensors, - output_shapes, - attributes, - ); - } + return typedCustomCall(params.name, opts, input_tensors, output_shapes, attributes); } fn handler(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { @@ -2842,44 +2821,6 @@ pub fn CustomCallOutputOperandAliases(I: type, O: type) type { ); } -pub fn shardingAwareTypedCustomCall( - comptime target_name: [:0]const u8, - comptime opts: CustomCallOptions, - input: anytype, - output: anytype, - attributes: anytype, -) ShapeToTensor(@TypeOf(output)) { - const Input = @TypeOf(input); - const Output = @TypeOf(output); - const Attributes = @TypeOf(attributes); - - var output_shapes: [@typeInfo(Output).@"struct".field_names.len]Shape = undefined; - inline for (@typeInfo(Output).@"struct".field_names, 0..) |field_name, i| { - output_shapes[i] = @field(output, field_name); - } - - const Handler = struct { - input: Input, - attributes: Attributes, - - fn body(self: @This(), sharded_output_shapes: []const Shape) []const Tensor { - return typedCustomCall(target_name, opts, self.input, sharded_output_shapes, self.attributes); - } - }; - const output_tensors = manualComputation( - Handler.body, - .{ .input = input, .attributes = attributes }, - @as([]const Shape, &output_shapes), - ); - - // Convert the slice back to a struct - var out: ShapeToTensor(Output) = undefined; - inline for (@typeInfo(Output).@"struct".field_names, 0..) |field_name, i| { - @field(out, field_name) = output_tensors[i]; - } - return out; -} - pub fn typedCustomCall( target_name: [:0]const u8, opts: CustomCallOptions, @@ -2894,7 +2835,7 @@ pub fn typedCustomCall( const ctx = Compiler.current(); const allocator = ctx.arena.allocator(); - stdx.debug.assert(!opts.has_side_effect or ctx.manual_computation_depth > 0, "side-effect customCall '{s}' must be emitted inside manualComputation", .{target_name}); + stdx.debug.assert(ctx.manual_computation_depth > 0, "customCall '{s}' must be emitted inside manualComputation", .{target_name}); const input_tensors: []const Tensor = switch (@typeInfo(Input)) { .@"struct" => |struct_info| b: { @@ -2904,7 +2845,6 @@ pub fn typedCustomCall( } break :b &input_tensors; }, - // Extra case to support []const Tensor from mesheAwareTypedCustomCall .pointer => |pointer_info| b: { if (pointer_info.size != .slice) @compileError("Expected input slice"); break :b input; @@ -2920,7 +2860,6 @@ pub fn typedCustomCall( } break :b &output_shapes; }, - // Extra case to support []const Shape from mesheAwareTypedCustomCall .pointer => |pointer_info| b: { if (pointer_info.size != .slice) @compileError("Expected input slice"); break :b output; @@ -2985,7 +2924,7 @@ pub fn typedCustomCall( ctx.location, ).appendTo(ctx.currentScope().block); - if (ctx.manual_computation_depth > 0 and ctx.partitioner == .gspmd) { + if (ctx.partitioner == .gspmd) { op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); } @@ -2997,7 +2936,6 @@ pub fn typedCustomCall( } return out; }, - // Extra case to support []const Shape from mesheAwareTypedCustomCall .pointer => |pointer_info| { if (pointer_info.size != .slice) @compileError("Expected input slice"); var out: []Tensor = allocator.alloc(Tensor, output_shapes.len) catch unreachable; @@ -3050,7 +2988,7 @@ test customCall { const platform = zml.testing.env(); const tp = zml.testing.tpMesh(); - var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{ .meshes = &.{tp} }); defer comp.deinit(); comp.activate(); defer comp.deactivate(); @@ -3061,9 +2999,17 @@ test customCall { const shape = zml.Shape.init(.{128}, .bf16).withPartitioning(tp, .{ ._0 = .model }); const input = Tensor.constant(zml.DataType.bf16.constant(0)).broad(shape); - const output = customCall("my_custom_call", .{input}, .{zml.Shape.init(.{128}, .bf16)}, .{}, .{ - .has_side_effect = false, - .output_operand_aliases = &.{.{ .output_index = 0, .operand_index = 0 }}, + const Local = struct { + fn body(x: Tensor, output_shape: Shape) Tensor { + return customCall("my_custom_call", .{x}, .{output_shape}, .{}, .{ + .has_side_effect = false, + .output_operand_aliases = &.{.{ .output_index = 0, .operand_index = 0 }}, + }); + } + }; + const output = manualComputation(Local.body, input, shape, tp, .{ + .in_specs = .parse(.{ ._0 = .model }), + .out_specs = .parse(.{ ._0 = .model }), }); try zml.testing.expectEqualShapes(input.shape(), output.shape()); diff --git a/zml/ops/manual_computation.zig b/zml/ops/manual_computation.zig index 45400206..a0ec0037 100644 --- a/zml/ops/manual_computation.zig +++ b/zml/ops/manual_computation.zig @@ -1,5 +1,4 @@ const std = @import("std"); -const builtin = @import("builtin"); const mlir = @import("mlir"); const shardy = @import("mlir/dialects/shardy"); @@ -7,44 +6,126 @@ const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); const Compiler = @import("../Compiler.zig"); -const DataType = @import("../dtype.zig").DataType; +const meta = @import("../meta.zig"); const mlirx = @import("../mlirx.zig"); const Shape = @import("../shape.zig").Shape; const Sharding = @import("../Sharding.zig"); const Tensor = @import("../tensor.zig").Tensor; -const zml = struct { - // Nested to disambiguate from stdx.meta - const meta = @import("../meta.zig"); - const testing = @import("../testing.zig"); -}; - const log = std.log.scoped(.@"zml/Compiler"); -/// ZML operator describes computation across several devices, -/// but `manualComputation` allows to partition the devices in small group, -/// split the `Tensor` on those partitions, -/// and describes sub-computation that work inside a partition. +/// Partitioning declarations for the tensors in `T`. +/// Each `Tensor` becomes a `Sharding.Partitioning.Logical`. +/// For example, `struct { x: Tensor, bias: ?Tensor, scale: f32 }` becomes +/// `struct { x: Logical, bias: ?Logical }`. /// -/// See: -/// https://github.com/openxla/shardy/blob/main/docs/sdy_dialect.md#sdymanual_computation-sdymanualcomputationop +/// Initialize individual fields to declare different shardings, or use +/// `partitionSpecs` to apply one spec to every tensor in a value. +pub fn PartitionSpecs(comptime T: type) type { + return meta.MapRestrict(Tensor, Sharding.Partitioning.Logical).map(T); +} + +/// Declares the same sharding for a group of tensors at a `manualComputation` +/// boundary. /// -/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R` where: -/// - `I` is user defined and contains inputs tensors and any metadata required by the computation -/// - `O` contains the partionned output shapes, types must be `void`, `Shape` or `[]const Shape` for 0, 1 or many outputs -/// - `R` must be `void`, `Tensor, or `[]const Tensor` for 0, 1 or many outputs -/// * `inputs` must be using the same mesh (using .withPartitioning), or be fully replicated -/// * `outputs` contains the global output shapes, -/// and `manualComputation` will pass to `body_fn` the shapes of each local output shard. +/// ```zig +/// .in_specs = .{ +/// .query = .parse(.{ .h = .model }), +/// .cache = partitionSpecs(cache, .parse(.{ .h = .model })), +/// .metadata = partitionSpecs(metadata, .replicated), +/// } +/// ``` /// -/// This depart from the rest of ZML where output shapes are generally inferred, -/// but in this case since `body_fn` explains how to compute parts of the result, -/// it's not clear how to put them back together, -/// hence why the user is asked to explicitly provided the expected output shape. +/// Requires an active compiler and uses its arena for allocations. Use +/// `partitionSpecsAlloc` outside graph construction. Use named struct fields +/// when mixing tensors with non-tensor metadata. +pub fn partitionSpecs(tensors: anytype, spec: Sharding.Partitioning.Logical) PartitionSpecs(@TypeOf(tensors)) { + return partitionSpecsAlloc(Compiler.current().arena.allocator(), tensors, spec) catch @panic("OOM"); +} + +/// Builds the same declarations as `partitionSpecs` using an explicit allocator. +pub fn partitionSpecsAlloc(allocator: std.mem.Allocator, tensors: anytype, spec: Sharding.Partitioning.Logical) std.mem.Allocator.Error!PartitionSpecs(@TypeOf(tensors)) { + var specs: PartitionSpecs(@TypeOf(tensors)) = undefined; + try meta.mapAlloc(struct { + fn assign(spec_: Sharding.Partitioning.Logical, _: Tensor) Sharding.Partitioning.Logical { + return spec_; + } + }.assign, allocator, spec, tensors, &specs); + return specs; +} + +/// Describes how the inputs and outputs of `body_fn` are partitioned across a mesh. +/// Returns the type of the `partitioning` argument to `manualComputation`. +/// +/// * `in_specs` specifies the sharding of each tensor in the body's first argument. +/// It follows the argument's structure, omitting non-tensor metadata. +/// * `out_specs` specifies the sharding of each shape in the body's second argument. +/// It follows the output shapes' structure and can be omitted for a `void` argument. +/// +/// Each spec is a `Sharding.Partitioning.Logical`. A mapping such as +/// `.parse(.{ .hidden = .model })` splits the tensor's `.hidden` dimension across +/// the mesh's `.model` axis. Omitted dimensions are replicated; `.replicated` +/// replicates the entire tensor. The specs must not contain open dimensions. +/// Slice lengths, optional presence and active union tags must match the arguments. +/// +/// For a body taking `struct { x: Tensor, bias: Tensor, scale: f32 }` and a single +/// output `Shape`, the partitioning can be declared as: +/// +/// ```zig +/// const partitioning: ManualPartitioning(body) = .{ +/// .in_specs = .{ .x = .parse(.{ .hidden = .model }), .bias = .replicated }, +/// .out_specs = .parse(.{ .hidden = .model }), +/// }; +/// ``` +pub fn ManualPartitioning(comptime body_fn: anytype) type { + const Logical = Sharding.Partitioning.Logical; + + const Inputs = stdx.meta.FnParam(body_fn, 0); + const OutputShapes = stdx.meta.FnParam(body_fn, 1); + + const InputSpecs = PartitionSpecs(Inputs); + const OutputSpecs = meta.MapRestrict(Shape, Logical).map(OutputShapes); + + if (OutputSpecs == void) return struct { + in_specs: InputSpecs, + out_specs: void = {}, + }; + + return struct { + in_specs: InputSpecs, + out_specs: OutputSpecs, + }; +} + +/// Describes a computation on local shards of tensors distributed across `mesh`. +/// ZML operations normally describe work on global tensors. The partitioner +/// distributes the inputs according to `in_specs`, and `body_fn` describes the work +/// performed on each shard. The results are exposed to the surrounding graph as +/// global tensors with the sharding declared by `out_specs`. +/// +/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R`, where: +/// - `I` contains the local input tensors and any metadata needed by the body. +/// - `O` is `void`, `Shape`, or a container of shapes describing the local outputs. +/// - `R` is `void`, `Tensor`, or a container of tensors matching those local shapes +/// in traversal order. +/// * `inputs` contains the global tensors and metadata. The partitioner reshards +/// the tensors as needed to satisfy `in_specs`; metadata is passed to the body unchanged. +/// * `output_shapes` contains the global output shapes. The body receives their +/// local shard shapes, computed using `out_specs` and the sizes of the mesh axes. +/// * `mesh` is the logical mesh whose axis names are referenced by the specs. +/// * `partitioning` declares the input and output shardings; see `ManualPartitioning`. +/// Each partitioned dimension must be divisible by the size of its mesh axis. +/// +/// For example, splitting a dimension of size 64 across a mesh axis of size 8 gives +/// the body a dimension of size 8. A replicated dimension keeps its global size. +/// The caller supplies the global output shapes because the body only describes +/// local results: their shapes alone do not determine the global tensor shapes. pub fn manualComputation( comptime body_fn: anytype, inputs: stdx.meta.FnParam(body_fn, 0), output_shapes: stdx.meta.FnParam(body_fn, 1), + mesh: *const Sharding.Mesh, + partitioning: ManualPartitioning(body_fn), ) stdx.meta.FnReturn(body_fn) { const Inputs = stdx.meta.FnParam(body_fn, 0); const OutputShapes = stdx.meta.FnParam(body_fn, 1); @@ -55,13 +136,42 @@ pub fn manualComputation( defer arena_state.deinit(); const arena = arena_state.allocator(); - var owned_inputs: Inputs = inputs; - const flat_input_tensors: []*Tensor = zml.meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); - defer arena.free(flat_input_tensors); + // The body may return containers borrowed from its inputs or output shapes. + const body_arena = compiler.arena.allocator(); - var owned_output_shapes: OutputShapes = output_shapes; - const flat_output_shapes: []*Shape = zml.meta.collectMutPtrs(Shape, arena, &owned_output_shapes) catch @panic("OOM"); - defer arena.free(flat_output_shapes); + var owned_inputs: Inputs = undefined; + meta.mapAlloc((struct { + fn copy(_: void, tensor: Tensor) Tensor { + return tensor; + } + }).copy, body_arena, {}, inputs, &owned_inputs) catch @panic("OOM"); + + const flat_input_tensors: []*Tensor = meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); + const in_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.in_specs) catch @panic("OOM"); + stdx.debug.assert(flat_input_tensors.len == in_specs.len, "manualComputation has {} input tensors but {} input specs", .{ flat_input_tensors.len, in_specs.len }); + + for (flat_input_tensors, in_specs) |tensor, spec| { + tensor._shape = tensor.shape().withPartitioning(mesh, spec); + stdx.debug.assert(!tensor.shape().containsPartitionSpec(.open), "manualComputation input sharding must be closed: {f}", .{tensor.shape()}); + } + + var local_output_shapes: OutputShapes = undefined; + meta.mapAlloc((struct { + fn copy(_: void, shape: Shape) Shape { + return shape; + } + }).copy, body_arena, {}, output_shapes, &local_output_shapes) catch @panic("OOM"); + + const flat_local_output_shapes = meta.collectMutPtrs(Shape, arena, &local_output_shapes) catch @panic("OOM"); + const out_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.out_specs) catch @panic("OOM"); + stdx.debug.assert(flat_local_output_shapes.len == out_specs.len, "manualComputation has {} output shapes but {} output specs", .{ flat_local_output_shapes.len, out_specs.len }); + + const flat_output_shapes = arena.alloc(Shape, flat_local_output_shapes.len) catch @panic("OOM"); + for (flat_output_shapes, flat_local_output_shapes, out_specs) |*global, local, spec| { + global.* = local.withPartitioning(mesh, spec); + stdx.debug.assert(!global.containsPartitionSpec(.open), "manualComputation output sharding must be closed: {f}", .{global}); + local.* = Sharding.shardedShape(global.*) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); + } const WrappedInputs = struct { inputs: *const Inputs, @@ -73,15 +183,17 @@ pub fn manualComputation( outputs_ptr.* = body_fn(wrapped.inputs.*, wrapped.output_shapes.*); - return try zml.meta.collectMutPtrs(Tensor, allocator, outputs_ptr); + return try meta.collectMutPtrs(Tensor, allocator, outputs_ptr); } }; var outputs_struct: Outputs = undefined; - var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &owned_output_shapes }; + var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &local_output_shapes }; + _ = manualComputationInternal( compiler, arena, + mesh, @typeName(@TypeOf(body_fn)), &wrapped_inputs, flat_input_tensors, @@ -91,6 +203,7 @@ pub fn manualComputation( ) catch |err| switch (err) { error.OutOfMemory => @panic("OOM"), }; + return outputs_struct; } @@ -101,27 +214,21 @@ const LocalComputationFn = *const fn (std.mem.Allocator, *const anyopaque, *anyo noinline fn manualComputationInternal( compiler: *Compiler, arena: std.mem.Allocator, + mesh: *const Sharding.Mesh, functor_name: []const u8, input_struct: *anyopaque, input_tensors: []*Tensor, output_struct: *anyopaque, - output_shapes: []*Shape, + global_output_shapes: []const Shape, local_fn: LocalComputationFn, ) error{OutOfMemory}![]*Tensor { const global_scope = compiler.currentScope(); const mlir_ctx = compiler.mlir_ctx; - const mesh = switch (chooseCommonMesh(input_tensors, compiler.platform.replicated_mesh)) { - .mesh => |m| m, - .incompatible => |incomp| std.debug.panic("manualComputation expects all inputs and outputs to use the same mesh, got input {d}: {f} and input {d}: {f}", .{ incomp[0], input_tensors[incomp[0]], incomp[1], input_tensors[incomp[1]] }), - }; - const manual_axes = collectManualAxes(mesh, input_tensors); - log.debug("manualComputation({s}) will use mesh {s}, manual axes: {f}", .{ functor_name, mesh.name, stdx.fmt.stringsZ(manual_axes.constSlice()) }); + log.debug("manualComputation({s}) will use mesh {s}", .{ functor_name, mesh.name }); const global_input_shapes = try arena.alloc(Shape, input_tensors.len); for (global_input_shapes, input_tensors) |*shape, tensor| shape.* = tensor._shape; - const global_output_shapes = try arena.alloc(Shape, output_shapes.len); - for (global_output_shapes, output_shapes) |*shape, shape_ptr| shape.* = shape_ptr.*; const input_types = try arena.alloc(*const mlir.Type, input_tensors.len); for (input_types, input_tensors) |*input_ty, tensor_ptr| { @@ -135,11 +242,17 @@ noinline fn manualComputationInternal( const input_values = try arena.alloc(*const mlir.Value, input_tensors.len); for (input_values, input_tensors) |*input_val, tensor_ptr| { + if (compiler.partitioner == .gspmd) { + tensor_ptr.* = tensor_ptr.withPartitioning(mesh, tensor_ptr.shape()._sharding.partition); + } input_val.* = global_scope.value(tensor_ptr.*); } - const manual_block = mlir.Block.init(input_types, input_locs); - errdefer manual_block.deinit(); + const manual_block = switch (compiler.partitioner) { + .shardy => mlir.Block.init(input_types, input_locs), + .gspmd => global_scope.block, + }; + errdefer if (compiler.partitioner == .shardy) manual_block.deinit(); const local_scope = compiler.pushBlock(manual_block); defer local_scope.pop(); @@ -151,17 +264,14 @@ noinline fn manualComputationInternal( }; } - for (output_shapes, global_output_shapes) |shape_ptr, global| { - shape_ptr.* = Sharding.shardedShape(global) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); - } - compiler.manual_computation_depth += 1; defer compiler.manual_computation_depth -= 1; const local_outputs = try local_fn(arena, input_struct, output_struct); stdx.debug.assert(local_outputs.len == global_output_shapes.len, "manualComputation body returned {} values, expected {}", .{ local_outputs.len, global_output_shapes.len }); - for (0.., local_outputs, output_shapes) |i, local_tensor, local_shape| { - stdx.debug.assert(local_shape.eql(local_tensor._shape), "manualComputation body output {d} expected {f}, got {f}", .{ i, local_shape.*, local_tensor._shape }); + for (0.., local_outputs, global_output_shapes) |i, local_tensor, global_shape| { + const expected_local = Sharding.shardedShape(global_shape) catch std.debug.panic("can't shard output {f} for {f}", .{ global_shape, mesh }); + stdx.debug.assert(expected_local.eql(local_tensor._shape), "manualComputation body output {d} expected {f}, got {f}", .{ i, expected_local, local_tensor._shape }); } const global_output_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); @@ -218,95 +328,11 @@ noinline fn manualComputationInternal( // Returned tensors describe the operation's global output shapes and sharding. for (0.., local_outputs, global_output_shapes) |o, output_tensor_ptr, output_shape| { output_tensor_ptr.* = Tensor._result(output_shape, op.result(o)); - output_tensor_ptr._shape = output_shape; + output_tensor_ptr._shape._sharding = output_shape._sharding; } return local_outputs; } -const ChosenMesh = union(enum) { - mesh: *const Sharding.Mesh, - incompatible: [2]u32, -}; - -fn chooseCommonMesh(inputs: []const *const Tensor, fallback: *const Sharding.Mesh) ChosenMesh { - var mesh: ?*const Sharding.Mesh = null; - var mesh_source_id: u32 = @intCast(inputs.len); - for (0.., inputs) |i, input| { - if (input.shape()._sharding.mesh) |shape_mesh| { - if (mesh) |m| { - if (m != shape_mesh) - return .{ .incompatible = .{ mesh_source_id, @intCast(i) } }; - } else { - mesh = shape_mesh; - mesh_source_id = @intCast(i); - } - } - } - return .{ .mesh = mesh orelse fallback }; -} - -test chooseCommonMesh { - const fallback: Sharding.Mesh = .initTest(.fallback, .{.all}); - const first: Sharding.Mesh = .initTest(.first, .{ .data, .model }); - const second: Sharding.Mesh = .initTest(.second, .{ .experts, .model }); - const replicated = Tensor.init(.{ .a = 8 }, .f32); - const sharded = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&first, .{ .a = .data }); - const other = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&second, .{ .a = .experts }); - - try std.testing.expectEqual(ChosenMesh{ .mesh = &fallback }, chooseCommonMesh(&.{}, &fallback)); - try std.testing.expectEqual(ChosenMesh{ .mesh = &fallback }, chooseCommonMesh(&.{&replicated}, &fallback)); - try std.testing.expectEqual(ChosenMesh{ .mesh = &first }, chooseCommonMesh(&.{&sharded}, &fallback)); - try std.testing.expectEqual(ChosenMesh{ .mesh = &first }, chooseCommonMesh(&.{ &replicated, &sharded, &sharded }, &fallback)); - try std.testing.expectEqual(ChosenMesh{ .incompatible = .{ 0, 1 } }, chooseCommonMesh(&.{ &sharded, &other }, &fallback)); -} - -fn collectManualAxes(mesh: *const Sharding.Mesh, inputs: []const *const Tensor) stdx.BoundedArray(Shape.Tag, Sharding.MAX_MESH_RANK) { - var used: [Sharding.MAX_MESH_RANK]bool = @splat(false); - for (inputs) |input| { - const sharding = input.shape()._sharding; - if (sharding.mesh == null) continue; - std.debug.assert(sharding.mesh.? == mesh); - for (0..input.rank()) |dim| { - if (sharding.partition.get(dim).meshAxis()) |axis| used[axis] = true; - } - } - - var axes: stdx.BoundedArray(Shape.Tag, Sharding.MAX_MESH_RANK) = .empty; - for (mesh.logical.axes.constSlice(), 0..) |axis, i| { - if (used[i]) axes.appendAssumeCapacity(axis); - } - return axes; -} - -test collectManualAxes { - const mesh: Sharding.Mesh = .initTest(.manual, .{ .data, .model, .unused }); - var data = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .data }); - var model = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .model }); - const replicated = Tensor.init(.{ .a = 8 }, .f32); - const open = Tensor.init(.{ .a = 8 }, .f32).withPartitioning(&mesh, .{ .a = .open }); - - const axes = collectManualAxes(&mesh, &.{ &model, &replicated, &data, &model, &open }); - data = replicated; - model = replicated; - try std.testing.expectEqual(@as(usize, 2), axes.len); - try std.testing.expectEqualStrings("data", std.mem.span(axes.get(0))); - try std.testing.expectEqualStrings("model", std.mem.span(axes.get(1))); - try std.testing.expectEqual(@as(usize, 0), collectManualAxes(&mesh, &.{ &replicated, &open }).len); - try std.testing.expectEqual(@as(usize, 0), collectManualAxes(&mesh, &.{}).len); - - const registry: *mlir.DialectRegistry = try .init(); - defer registry.deinit(); - shardy.registerDialect(registry); - const ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - const expected = shardy.ManualAxesAttr.get(ctx, &.{ - .init(ctx, "data"), - .init(ctx, "model"), - }).attribute(); - try std.testing.expect(shardy_.manualAxesAttr(ctx, axes.constSlice()).eql(expected)); -} - const gspmd = struct { fn globalToLocal(mlir_ctx: *mlir.Context, scope: *Compiler.Scope, local_tensor: Tensor, local_shape: Shape, location: *const mlir.Location) Tensor { const local_type = mlirx.Type.rankedTensor(mlir_ctx, local_shape); @@ -349,54 +375,172 @@ const gspmd = struct { } }; -const shardy_ = struct { - pub fn manualAxesAttr(mlir_ctx: *mlir.Context, axes: []const Shape.Tag) *const mlir.Attribute { - var attrs: stdx.BoundedArray(*const mlir.StringAttribute, Sharding.MAX_MESH_RANK) = .empty; - for (axes) |axis| attrs.appendAssumeCapacity(.init(mlir_ctx, std.mem.span(axis))); - return shardy.ManualAxesAttr.get(mlir_ctx, attrs.constSlice()).attribute(); - } -}; - -test "manualComputation passes local output shapes to the callback" { +test manualComputation { + const zml = @import("../zml.zig"); const allocator = std.testing.allocator; const io = std.testing.io; const platform = zml.testing.env(); const mesh = zml.testing.tpMesh(); - const partitions = mesh.numPartitionsForLogicalAxis(.model); - if (partitions == 1) return error.SkipZigTest; - var compiler: Compiler = .init(allocator, io, platform, .{ .meshes = &.{mesh} }); - defer compiler.deinit(); - compiler.activate(); - defer compiler.deactivate(); - - const block = mlir.Block.init(&.{}, &.{}); - defer block.deinit(); - const scope = compiler.pushBlock(block); - defer scope.pop(); - - const global_shape = Shape.init(.{ .b = 4, .h = 3 * partitions, .hd = 128 }, .bf16) - .withPartitioning(mesh, .{ .h = .model }); - const local_shape = Shape.init(.{ .b = 4, .h = 3, .hd = 128 }, .bf16); - const input = Tensor.zeroes(global_shape).withPartitioning(mesh, .{ .h = .model }); - var received_shape: ?Shape = null; - const Context = struct { - input: Tensor, - received_shape: *?Shape, + const LocalSum = struct { + x: Tensor, + bias: Tensor, + scale: i32, fn body(self: @This(), output_shape: Shape) Tensor { - self.received_shape.* = output_shape; - return self.input; + return self.x.sum(.x).scale(self.scale).add(self.bias).broad(output_shape); + } + + fn forward(x: Tensor, mesh_: *const Sharding.Mesh) Tensor { + return manualComputation( + body, + .{ .x = x, .bias = .scalar(10, .i32), .scale = 2 }, + x.shape(), + mesh_, + .{ .in_specs = .{ .x = .parse(.{ .x = .model }), .bias = .replicated }, .out_specs = .parse(.{ .x = .model }) }, + ); } }; - const output = manualComputation( - Context.body, - .{ .input = input, .received_shape = &received_shape }, - global_shape, - ); - try std.testing.expect(received_shape != null); - try std.testing.expect(received_shape.?.eql(local_shape)); - try std.testing.expect(output.shape().eql(global_shape)); - try std.testing.expectEqualDeep(global_shape._sharding, output.shape()._sharding); + // Each shard sums two elements and broadcasts the result to its local output shape. + const shape: Shape = .init(.{ .x = 2 * mesh.numPartitionsForLogicalAxis(.model) }, .i32); + const values = try allocator.alloc(i32, shape.count()); + defer allocator.free(values); + + for (values, 0..) |*value, i| value.* = @intCast(i + 1); + + var input = try zml.Buffer.fromBytes(io, platform, shape, std.mem.sliceAsBytes(values)); + defer input.deinit(); + + for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { + const exe = try platform.compileFn(allocator, io, LocalSum.forward, .{ .fromShape(shape), mesh }, .{ + .meshes = &.{ platform.replicated_mesh, mesh }, + .partitioner = partitioner, + }); + defer exe.deinit(); + + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + + const actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + + for (actual.constItems(i32), 0..) |value, i| { + const pair = values[i / 2 * 2 ..][0..2]; + try std.testing.expectEqual((pair[0] + pair[1]) * 2 + 10, value); + } + + try std.testing.expect(output.shape().eqlWithTags(shape)); + try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); + } +} + +test "manualComputation composes across logical and physical meshes" { + const zml = @import("../zml.zig"); + if (comptime !zml.Target.cpu.isEnabled()) return error.SkipZigTest; + const allocator = std.testing.allocator; + const io = std.testing.io; + + const Topologies = enum { flat, torus }; + + const Model = struct { + fn increment(x: Tensor, _: Shape) Tensor { + return x.addConstant(1); + } + + fn forward(x: Tensor, mesh: *const Sharding.Mesh) Tensor { + const partitioning: ManualPartitioning(increment) = .{ + .in_specs = partitionSpecs(x, .parse(.{ .x = .model })), + .out_specs = .parse(.{ .x = .model }), + }; + + const first = manualComputation(increment, x, x.shape(), mesh, partitioning); + const converted = first.convert(.f32); + return manualComputation(increment, converted, converted.shape(), mesh, partitioning); + } + }; + + const shape: Shape = .init(.{ .x = 16 }, .i32); + + var values: [16]i32 = undefined; + var expected: [16]f32 = undefined; + for (&values, &expected, 0..) |*value, *result, i| { + value.* = @intCast(i); + result.* = @floatFromInt(i + 2); + } + + for ([_]Topologies{ .flat, .torus }) |topology| { + const platform = try zml.Platform.init(allocator, io, .cpu, .{ + .cpu = .{ .device_count = 8 }, + .physical_mesh = if (topology == .torus) .{ .custom = Sharding.PhysicalMesh.torus3d } else .auto, + }); + defer platform.deinit(allocator, io); + + // Fold three physical axes into one logical axis, or split one into three. + const logical: Sharding.LogicalMesh = if (topology == .torus) + try .init(.{.model}, &.{.balanced}, &.{8}) + else + try Sharding.LogicalMesh.init(.{ .data, .model, .pipeline }, &.{ .low_bandwidth, .high_bandwidth, .balanced }, &.{ 2, 2, 2 }); + + const mesh = try platform.registerMesh("manual", logical); + + var input: zml.Buffer = try .fromBytes(io, platform, shape, std.mem.asBytes(&values)); + defer input.deinit(); + + for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { + const exe = try platform.compileFn(allocator, io, Model.forward, .{ Tensor.fromShape(shape), mesh }, .{ + .meshes = &.{ platform.replicated_mesh, mesh }, + .partitioner = partitioner, + }); + defer exe.deinit(); + + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + + try std.testing.expectEqual(expected, try output.getValue([16]f32, io)); + try std.testing.expect(output.shape().eqlWithTags(shape.withDtype(.f32))); + try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); + } + } +} + +test "partitionSpecs preserves optional tensors and active cache variants" { + const zml = @import("../zml.zig"); + const Logical = Sharding.Partitioning.Logical; + const Inputs = struct { + layers: []const Tensor, + bias: ?Tensor, + cache: union(enum) { disabled, values: [2]Tensor }, + scale: f32, + }; + + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + + const tensor: Tensor = .fromShape(.init(.{ .h = 8, .d = 16 }, .f32)); + var inputs: Inputs = .{ + .layers = &.{ tensor, tensor }, + .bias = null, + .cache = .{ .values = .{ tensor, tensor } }, + .scale = 2, + }; + const heads: Logical = .parse(.{ .h = .model }); + const specs = try zml.ops.partitionSpecsAlloc(arena.allocator(), inputs, heads); + const expected: zml.ops.PartitionSpecs(Inputs) = .{ + .layers = &.{ heads, heads }, + .bias = null, + .cache = .{ .values = .{ heads, heads } }, + }; + try std.testing.expectEqualDeep(expected, specs); + + inputs.layers = &.{}; + inputs.bias = tensor; + inputs.cache = .disabled; + const replicated = try zml.ops.partitionSpecsAlloc(arena.allocator(), inputs, .replicated); + const expected_replicated: zml.ops.PartitionSpecs(Inputs) = .{ + .layers = &.{}, + .bias = .replicated, + .cache = .disabled, + }; + try std.testing.expectEqualDeep(expected_replicated, replicated); } diff --git a/zml/ops/manual_computation2.zig b/zml/ops/manual_computation2.zig deleted file mode 100644 index 2d3f0aec..00000000 --- a/zml/ops/manual_computation2.zig +++ /dev/null @@ -1,465 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); -const shardy = @import("mlir/dialects/shardy"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); - -const Compiler = @import("../Compiler.zig"); -const meta = @import("../meta.zig"); -const mlirx = @import("../mlirx.zig"); -const Shape = @import("../shape.zig").Shape; -const Sharding = @import("../Sharding.zig"); -const Tensor = @import("../tensor.zig").Tensor; - -const log = std.log.scoped(.@"zml/Compiler"); - -/// Describes how the inputs and outputs of `body_fn` are partitioned across a mesh. -/// Returns the type of the `partitioning` argument to `manualComputation`. -/// -/// * `in_specs` specifies the sharding of each tensor in the body's first argument. -/// It follows the argument's structure, omitting non-tensor metadata. -/// * `out_specs` specifies the sharding of each shape in the body's second argument. -/// It follows the output shapes' structure and can be omitted for a `void` argument. -/// -/// Each spec is a `Sharding.Partitioning.Logical`. A mapping such as -/// `.parse(.{ .hidden = .model })` splits the tensor's `.hidden` dimension across -/// the mesh's `.model` axis. Omitted dimensions are replicated; `.replicated` -/// replicates the entire tensor. The specs must not contain open dimensions. -/// Slice lengths, optional presence and active union tags must match the arguments. -/// -/// For a body taking `struct { x: Tensor, bias: Tensor, scale: f32 }` and a single -/// output `Shape`, the partitioning can be declared as: -/// -/// ```zig -/// const partitioning: ManualPartitioning(body) = .{ -/// .in_specs = .{ .x = .parse(.{ .hidden = .model }), .bias = .replicated }, -/// .out_specs = .parse(.{ .hidden = .model }), -/// }; -/// ``` -pub fn ManualPartitioning(comptime body_fn: anytype) type { - const Logical = Sharding.Partitioning.Logical; - - const Inputs = stdx.meta.FnParam(body_fn, 0); - const OutputShapes = stdx.meta.FnParam(body_fn, 1); - - const InputSpecs = meta.MapRestrict(Tensor, Logical).map(Inputs); - const OutputSpecs = meta.MapRestrict(Shape, Logical).map(OutputShapes); - - if (OutputSpecs == void) return struct { - in_specs: InputSpecs, - out_specs: void = {}, - }; - - return struct { - in_specs: InputSpecs, - out_specs: OutputSpecs, - }; -} - -/// Describes a computation on local shards of tensors distributed across `mesh`. -/// ZML operations normally describe work on global tensors. The partitioner -/// distributes the inputs according to `in_specs`, and `body_fn` describes the work -/// performed on each shard. The results are exposed to the surrounding graph as -/// global tensors with the sharding declared by `out_specs`. -/// -/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R`, where: -/// - `I` contains the local input tensors and any metadata needed by the body. -/// - `O` is `void`, `Shape`, or a container of shapes describing the local outputs. -/// - `R` is `void`, `Tensor`, or a container of tensors matching those local shapes -/// in traversal order. -/// * `inputs` contains the global tensors and metadata. The partitioner reshards -/// the tensors as needed to satisfy `in_specs`; metadata is passed to the body unchanged. -/// * `output_shapes` contains the global output shapes. The body receives their -/// local shard shapes, computed using `out_specs` and the sizes of the mesh axes. -/// * `mesh` is the logical mesh whose axis names are referenced by the specs. -/// * `partitioning` declares the input and output shardings; see `ManualPartitioning`. -/// Each partitioned dimension must be divisible by the size of its mesh axis. -/// -/// For example, splitting a dimension of size 64 across a mesh axis of size 8 gives -/// the body a dimension of size 8. A replicated dimension keeps its global size. -/// The caller supplies the global output shapes because the body only describes -/// local results: their shapes alone do not determine the global tensor shapes. -pub fn manualComputation2( - comptime body_fn: anytype, - inputs: stdx.meta.FnParam(body_fn, 0), - output_shapes: stdx.meta.FnParam(body_fn, 1), - mesh: *const Sharding.Mesh, - partitioning: ManualPartitioning(body_fn), -) stdx.meta.FnReturn(body_fn) { - const Inputs = stdx.meta.FnParam(body_fn, 0); - const OutputShapes = stdx.meta.FnParam(body_fn, 1); - const Outputs = stdx.meta.FnReturn(body_fn); - - const compiler = Compiler.current(); - var arena_state: std.heap.ArenaAllocator = .init(compiler.allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - // The body may return containers borrowed from its inputs or output shapes. - const body_arena = compiler.arena.allocator(); - - var owned_inputs: Inputs = undefined; - meta.mapAlloc((struct { - fn copy(_: void, tensor: Tensor) Tensor { - return tensor; - } - }).copy, body_arena, {}, inputs, &owned_inputs) catch @panic("OOM"); - - const flat_input_tensors: []*Tensor = meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); - const in_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.in_specs) catch @panic("OOM"); - stdx.debug.assert(flat_input_tensors.len == in_specs.len, "manualComputation has {} input tensors but {} input specs", .{ flat_input_tensors.len, in_specs.len }); - - for (flat_input_tensors, in_specs) |tensor, spec| { - tensor._shape = tensor.shape().withPartitioning(mesh, spec); - stdx.debug.assert(!tensor.shape().containsPartitionSpec(.open), "manualComputation input sharding must be closed: {f}", .{tensor.shape()}); - } - - var local_output_shapes: OutputShapes = undefined; - meta.mapAlloc((struct { - fn copy(_: void, shape: Shape) Shape { - return shape; - } - }).copy, body_arena, {}, output_shapes, &local_output_shapes) catch @panic("OOM"); - - const flat_local_output_shapes = meta.collectMutPtrs(Shape, arena, &local_output_shapes) catch @panic("OOM"); - const out_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.out_specs) catch @panic("OOM"); - stdx.debug.assert(flat_local_output_shapes.len == out_specs.len, "manualComputation has {} output shapes but {} output specs", .{ flat_local_output_shapes.len, out_specs.len }); - - const flat_output_shapes = arena.alloc(Shape, flat_local_output_shapes.len) catch @panic("OOM"); - for (flat_output_shapes, flat_local_output_shapes, out_specs) |*global, local, spec| { - global.* = local.withPartitioning(mesh, spec); - stdx.debug.assert(!global.containsPartitionSpec(.open), "manualComputation output sharding must be closed: {f}", .{global}); - local.* = Sharding.shardedShape(global.*) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); - } - - const WrappedInputs = struct { - inputs: *const Inputs, - output_shapes: *const OutputShapes, - - fn wrap(allocator: std.mem.Allocator, opaque_wrapper: *const anyopaque, opaque_outputs: *anyopaque) error{OutOfMemory}![]*Tensor { - const wrapped: *const @This() = @ptrCast(@alignCast(opaque_wrapper)); - const outputs_ptr: *Outputs = @ptrCast(@alignCast(opaque_outputs)); - - outputs_ptr.* = body_fn(wrapped.inputs.*, wrapped.output_shapes.*); - - return try meta.collectMutPtrs(Tensor, allocator, outputs_ptr); - } - }; - - var outputs_struct: Outputs = undefined; - var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &local_output_shapes }; - - _ = manualComputationInternal( - compiler, - arena, - mesh, - @typeName(@TypeOf(body_fn)), - &wrapped_inputs, - flat_input_tensors, - @ptrCast(&outputs_struct), - flat_output_shapes, - WrappedInputs.wrap, - ) catch |err| switch (err) { - error.OutOfMemory => @panic("OOM"), - }; - - return outputs_struct; -} - -const LocalComputationFn = *const fn (std.mem.Allocator, *const anyopaque, *anyopaque) error{OutOfMemory}![]*Tensor; - -/// Reified version of manualComputation2. -/// This one is compiled exactly once, user code just need to compile the `collectPtrs` -noinline fn manualComputationInternal( - compiler: *Compiler, - arena: std.mem.Allocator, - mesh: *const Sharding.Mesh, - functor_name: []const u8, - input_struct: *anyopaque, - input_tensors: []*Tensor, - output_struct: *anyopaque, - global_output_shapes: []const Shape, - local_fn: LocalComputationFn, -) error{OutOfMemory}![]*Tensor { - const global_scope = compiler.currentScope(); - const mlir_ctx = compiler.mlir_ctx; - - log.debug("manualComputation2({s}) will use mesh {s}", .{ functor_name, mesh.name }); - - const global_input_shapes = try arena.alloc(Shape, input_tensors.len); - for (global_input_shapes, input_tensors) |*shape, tensor| shape.* = tensor._shape; - - const input_types = try arena.alloc(*const mlir.Type, input_tensors.len); - for (input_types, input_tensors) |*input_ty, tensor_ptr| { - const global = tensor_ptr._shape; - const local = Sharding.shardedShape(global) catch std.debug.panic("can't shard {f} for {f}", .{ global, mesh }); - log.debug("\t- {f} -> {f}", .{ global, local }); - input_ty.* = mlirx.Type.rankedTensor(mlir_ctx, local); - } - const input_locs = try arena.alloc(*const mlir.Location, input_tensors.len); - @memset(input_locs, compiler.unknown_location); - - const input_values = try arena.alloc(*const mlir.Value, input_tensors.len); - for (input_values, input_tensors) |*input_val, tensor_ptr| { - if (compiler.partitioner == .gspmd) { - tensor_ptr.* = tensor_ptr.withPartitioning(mesh, tensor_ptr.shape()._sharding.partition); - } - input_val.* = global_scope.value(tensor_ptr.*); - } - - const manual_block = switch (compiler.partitioner) { - .shardy => mlir.Block.init(input_types, input_locs), - .gspmd => global_scope.block, - }; - errdefer if (compiler.partitioner == .shardy) manual_block.deinit(); - - const local_scope = compiler.pushBlock(manual_block); - defer local_scope.pop(); - for (0.., input_tensors, global_input_shapes) |i, tensor_ptr, global| { - const local = Sharding.shardedShape(global) catch unreachable; // we already computed this above - tensor_ptr.* = switch (compiler.partitioner) { - .shardy => Tensor._result(local, manual_block.argument(i)), - .gspmd => gspmd.globalToLocal(mlir_ctx, global_scope, tensor_ptr.*, local, compiler.location), - }; - } - - compiler.manual_computation_depth += 1; - defer compiler.manual_computation_depth -= 1; - - const local_outputs = try local_fn(arena, input_struct, output_struct); - stdx.debug.assert(local_outputs.len == global_output_shapes.len, "manualComputation2 body returned {} values, expected {}", .{ local_outputs.len, global_output_shapes.len }); - for (0.., local_outputs, global_output_shapes) |i, local_tensor, global_shape| { - const expected_local = Sharding.shardedShape(global_shape) catch std.debug.panic("can't shard output {f} for {f}", .{ global_shape, mesh }); - stdx.debug.assert(expected_local.eql(local_tensor._shape), "manualComputation2 body output {d} expected {f}, got {f}", .{ i, expected_local, local_tensor._shape }); - } - - const global_output_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); - for (global_output_shapes, 0..) |output_shape, i| { - global_output_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output_shape); - } - - const local_output_values = try arena.alloc(*const mlir.Value, local_outputs.len); - for (local_output_values, local_outputs) |*output_val, tensor_ptr| { - output_val.* = local_scope.value(tensor_ptr.*); - } - - const op: *const mlir.Operation = switch (compiler.partitioner) { - .shardy => op: { - // Shardy creates a sdy.manual_computation block. - _ = shardy.return_(mlir_ctx, local_output_values, compiler.location).appendTo(manual_block); - - const out_mesh_attrs = try arena.alloc(*const shardy.TensorShardingAttr, global_output_shapes.len); - for (out_mesh_attrs, global_output_shapes) |*attr, shape| { - attr.* = try mesh.sdyShardingAttrForShape(arena, mlir_ctx, shape); - } - - break :op shardy.manual_computation( - mlir_ctx, - input_values, - global_output_types, - manual_block, - try Sharding.sdyPerValueShardingAttr(arena, mlir_ctx, global_input_shapes, mesh), - .get(mlir_ctx, out_mesh_attrs), - try Sharding.sdyManualAxesAttr(arena, mlir_ctx, global_input_shapes, global_output_shapes, mesh), - compiler.location, - ).appendTo(global_scope.block); - }, - .gspmd => op: { - // Gspmd calls SPMDShardToFullShape and wrap into an optimizationBarrier - // Shortcut to avoid empty optimization barrier - if (global_output_shapes.len == 0) return &.{}; - - const global_values = try arena.alloc(*const mlir.Value, global_output_shapes.len); - const global_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); - for (global_output_shapes, 0..) |global_shape, i| { - global_values[i], global_types[i] = try gspmd.localToGlobal(compiler, arena, global_scope, local_output_values[i], global_shape); - } - - break :op stablehlo.optimizationBarrier( - mlir_ctx, - global_values, - global_types, - compiler.location, - ).appendTo(global_scope.block); - }, - }; - - // Returned tensors describe the operation's global output shapes and sharding. - for (0.., local_outputs, global_output_shapes) |o, output_tensor_ptr, output_shape| { - output_tensor_ptr.* = Tensor._result(output_shape, op.result(o)); - output_tensor_ptr._shape._sharding = output_shape._sharding; - } - return local_outputs; -} - -const gspmd = struct { - fn globalToLocal(mlir_ctx: *mlir.Context, scope: *Compiler.Scope, local_tensor: Tensor, local_shape: Shape, location: *const mlir.Location) Tensor { - const local_type = mlirx.Type.rankedTensor(mlir_ctx, local_shape); - const full_to_shard = stablehlo.custom_call( - mlir_ctx, - &.{scope.value(local_tensor)}, - &.{local_type}, - .{ - .call_target_name = "SPMDFullToShardShape", - .has_side_effect = false, - .backend_config = .{ .original = "" }, - .additional_attributes = &.{ - .named(mlir_ctx, "mhlo.sharding", .string(mlir_ctx, "{manual}")), - }, - }, - location, - ).appendTo(scope.block); - return Tensor._result(local_shape, full_to_shard.result(0)); - } - - fn localToGlobal(compiler: *Compiler, arena: std.mem.Allocator, scope: *Compiler.Scope, local_value: *const mlir.Value, global_shape: Shape) !struct { *const mlir.Value, *const mlir.Type } { - const gspmd_attr = try compiler.tensorShardingAttr(arena, global_shape); - const global_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, global_shape); - - const shard_to_full = stablehlo.custom_call( - compiler.mlir_ctx, - &.{local_value}, - &.{global_type}, - .{ - .call_target_name = "SPMDShardToFullShape", - .has_side_effect = false, - .backend_config = .{ .original = "" }, - .additional_attributes = &.{ - .named(compiler.mlir_ctx, "mhlo.sharding", gspmd_attr), - }, - }, - compiler.location, - ).appendTo(scope.block); - return .{ shard_to_full.result(0), global_type }; - } -}; - -test manualComputation2 { - const zml = @import("../zml.zig"); - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = zml.testing.env(); - const mesh = zml.testing.tpMesh(); - - const LocalSum = struct { - x: Tensor, - bias: Tensor, - scale: i32, - - fn body(self: @This(), output_shape: Shape) Tensor { - return self.x.sum(.x).scale(self.scale).add(self.bias).broad(output_shape); - } - - fn forward(x: Tensor, mesh_: *const Sharding.Mesh) Tensor { - return manualComputation2( - body, - .{ .x = x, .bias = .scalar(10, .i32), .scale = 2 }, - x.shape(), - mesh_, - .{ .in_specs = .{ .x = .parse(.{ .x = .model }), .bias = .replicated }, .out_specs = .parse(.{ .x = .model }) }, - ); - } - }; - - // Each shard sums two elements and broadcasts the result to its local output shape. - const shape: Shape = .init(.{ .x = 2 * mesh.numPartitionsForLogicalAxis(.model) }, .i32); - const values = try allocator.alloc(i32, shape.count()); - defer allocator.free(values); - - for (values, 0..) |*value, i| value.* = @intCast(i + 1); - - var input = try zml.Buffer.fromBytes(io, platform, shape, std.mem.sliceAsBytes(values)); - defer input.deinit(); - - for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { - const exe = try platform.compileFn(allocator, io, LocalSum.forward, .{ .fromShape(shape), mesh }, .{ - .meshes = &.{ platform.replicated_mesh, mesh }, - .partitioner = partitioner, - }); - defer exe.deinit(); - - var output = try exe.eval(allocator, io, .{input}); - defer output.deinit(); - - const actual = try output.toSliceAlloc(allocator, io); - defer actual.free(allocator); - - for (actual.constItems(i32), 0..) |value, i| { - const pair = values[i / 2 * 2 ..][0..2]; - try std.testing.expectEqual((pair[0] + pair[1]) * 2 + 10, value); - } - - try std.testing.expect(output.shape().eqlWithTags(shape)); - try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); - } -} - -test "manualComputation2 composes across logical and physical meshes" { - const zml = @import("../zml.zig"); - if (comptime !zml.Target.cpu.isEnabled()) return error.SkipZigTest; - const allocator = std.testing.allocator; - const io = std.testing.io; - - const Topologies = enum { flat, torus }; - - const Model = struct { - fn increment(x: Tensor, _: Shape) Tensor { - return x.addConstant(1); - } - - fn forward(x: Tensor, mesh: *const Sharding.Mesh) Tensor { - const partitioning: ManualPartitioning(increment) = .{ - .in_specs = .parse(.{ .x = .model }), - .out_specs = .parse(.{ .x = .model }), - }; - - const first = manualComputation2(increment, x, x.shape(), mesh, partitioning); - const converted = first.convert(.f32); - return manualComputation2(increment, converted, converted.shape(), mesh, partitioning); - } - }; - - const shape: Shape = .init(.{ .x = 16 }, .i32); - - var values: [16]i32 = undefined; - var expected: [16]f32 = undefined; - for (&values, &expected, 0..) |*value, *result, i| { - value.* = @intCast(i); - result.* = @floatFromInt(i + 2); - } - - for ([_]Topologies{ .flat, .torus }) |topology| { - const platform = try zml.Platform.init(allocator, io, .cpu, .{ - .cpu = .{ .device_count = 8 }, - .physical_mesh = if (topology == .torus) .{ .custom = Sharding.PhysicalMesh.torus3d } else .auto, - }); - defer platform.deinit(allocator, io); - - // Fold three physical axes into one logical axis, or split one into three. - const logical: Sharding.LogicalMesh = if (topology == .torus) - try .init(.{.model}, &.{.balanced}, &.{8}) - else - try Sharding.LogicalMesh.init(.{ .data, .model, .pipeline }, &.{ .low_bandwidth, .high_bandwidth, .balanced }, &.{ 2, 2, 2 }); - - const mesh = try platform.registerMesh("manual", logical); - - var input: zml.Buffer = try .fromBytes(io, platform, shape, std.mem.asBytes(&values)); - defer input.deinit(); - - for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { - const exe = try platform.compileFn(allocator, io, Model.forward, .{ Tensor.fromShape(shape), mesh }, .{ - .meshes = &.{ platform.replicated_mesh, mesh }, - .partitioner = partitioner, - }); - defer exe.deinit(); - - var output = try exe.eval(allocator, io, .{input}); - defer output.deinit(); - - try std.testing.expectEqual(expected, try output.getValue([16]f32, io)); - try std.testing.expect(output.shape().eqlWithTags(shape.withDtype(.f32))); - try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); - } - } -} diff --git a/zml/tensor.zig b/zml/tensor.zig index a7782844..a09a5d38 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -4813,7 +4813,7 @@ pub const Tensor = struct { defer ctx.arena.allocator().free(full_name); switch (ctx.platform.target) { .cpu, .cuda, .rocm, .tpu, .metal => { - ops.manualComputation2((struct { + ops.manualComputation((struct { input: Tensor, name: []const u8, From a0533da7b87c83ff57f24faae015d0140ffc777b Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Fri, 13 Nov 2026 14:13:10 +0100 Subject: [PATCH 873/881] zml/kernel: introduce packages per DSL (#1061) Split `kernel.zig` into one package per DSL --- dsl/fly/README.md | 2 +- mlir/dialects/fly/REGENERATING.md | 2 +- platforms/tpu/ragged_paged.zig | 2 +- zml/BUILD.bazel | 2 +- zml/kernel.zig | 1630 ----------------- zml/kernel/BUILD.bazel | 15 + zml/kernel/cuda_tile/BUILD.bazel | 9 + zml/kernel/cuda_tile/cuda_tile.zig | 163 ++ zml/kernel/cuda_tile/test.zig | 28 + zml/kernel/cute/BUILD.bazel | 9 + zml/kernel/cute/cute.zig | 264 +++ zml/kernel/cute/test.zig | 34 + zml/kernel/fly/BUILD.bazel | 9 + zml/kernel/fly/fly.zig | 248 +++ zml/kernel/fly/test.zig | 149 ++ zml/kernel/kernel.zig | 12 + zml/kernel/metal/BUILD.bazel | 9 + zml/kernel/metal/metal.zig | 237 +++ zml/kernel/metal/test.zig | 71 + zml/kernel/mosaic_tpu/BUILD.bazel | 6 + zml/kernel/mosaic_tpu/mosaic_tpu.zig | 284 +++ zml/kernel/tcl/BUILD.bazel | 9 + zml/kernel/tcl/tcl.zig | 206 +++ zml/kernel/tcl/test.zig | 123 ++ zml/kernel/triton/BUILD.bazel | 6 + zml/kernel/triton/triton.zig | 165 ++ .../flash_attention/metal/flash_attn.zig | 3 +- .../paged_attention/metal/paged_attn.zig | 3 +- zml/zml.zig | 2 +- 29 files changed, 2063 insertions(+), 1639 deletions(-) delete mode 100644 zml/kernel.zig create mode 100644 zml/kernel/BUILD.bazel create mode 100644 zml/kernel/cuda_tile/BUILD.bazel create mode 100644 zml/kernel/cuda_tile/cuda_tile.zig create mode 100644 zml/kernel/cuda_tile/test.zig create mode 100644 zml/kernel/cute/BUILD.bazel create mode 100644 zml/kernel/cute/cute.zig create mode 100644 zml/kernel/cute/test.zig create mode 100644 zml/kernel/fly/BUILD.bazel create mode 100644 zml/kernel/fly/fly.zig create mode 100644 zml/kernel/fly/test.zig create mode 100644 zml/kernel/kernel.zig create mode 100644 zml/kernel/metal/BUILD.bazel create mode 100644 zml/kernel/metal/metal.zig create mode 100644 zml/kernel/metal/test.zig create mode 100644 zml/kernel/mosaic_tpu/BUILD.bazel create mode 100644 zml/kernel/mosaic_tpu/mosaic_tpu.zig create mode 100644 zml/kernel/tcl/BUILD.bazel create mode 100644 zml/kernel/tcl/tcl.zig create mode 100644 zml/kernel/tcl/test.zig create mode 100644 zml/kernel/triton/BUILD.bazel create mode 100644 zml/kernel/triton/triton.zig diff --git a/dsl/fly/README.md b/dsl/fly/README.md index b886988b..687c99dc 100644 --- a/dsl/fly/README.md +++ b/dsl/fly/README.md @@ -81,7 +81,7 @@ const c = VectorAdd.call(.{ .a = a, .b = bt }, .{ .c = a.shape() }, .{ ``` Complete vector-add and tiled-MMA kernels are the tests in `builder.zig` -and `zml/kernel.zig`. +and `zml/kernel/fly/test.zig`. Model kernels live beside their consumers: diff --git a/mlir/dialects/fly/REGENERATING.md b/mlir/dialects/fly/REGENERATING.md index b66b7abe..9673870f 100644 --- a/mlir/dialects/fly/REGENERATING.md +++ b/mlir/dialects/fly/REGENERATING.md @@ -71,7 +71,7 @@ dialect today; when one appears, extend the generator and this file. `test.zig`. - Callers that no longer compile are the point of generating: an operation whose signature changed in FlyDSL now fails at the call site. Fix them in - the same change (`dsl/fly/builder.zig`, `zml/kernel.zig`, `zml/kernels/*/fly/`, + the same change (`dsl/fly/builder.zig`, `zml/kernel/fly/fly.zig`, `zml/kernels/*/fly/`, and downstream llmd's `llmd/kernels/*/fly/`). - Emit the IR of the fly kernels before and after (for instance llmd's `sparse_mla.Main2D.emit` and the `zml/kernels/moe/fly/` kernels with diff --git a/platforms/tpu/ragged_paged.zig b/platforms/tpu/ragged_paged.zig index 4c1c50ed..6a0fdfae 100644 --- a/platforms/tpu/ragged_paged.zig +++ b/platforms/tpu/ragged_paged.zig @@ -482,7 +482,7 @@ fn flashAttention( } /// Emit the kernel body into a pre-opened `mtt.Builder`. The factory in -/// `zml/kernel.zig` opens the builder, calls this `run`, and finalizes the +/// `zml/kernel/mosaic_tpu/mosaic_tpu.zig` opens the builder, calls this `run`, and finalizes the /// IR with `b.finishOpts(&.{}, .{ .canonicalize = true })`. pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { const q_transform: []const mtt.ArgSpec.TransformReturn = &.{ diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 2b14c79f..f52ecdd5 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -87,7 +87,6 @@ zig_library( "exe.zig", "floats.zig", "io.zig", - "kernel.zig", "kernels/flash_attention/flash_attention.zig", "kernels/flash_attention/tests.zig", "kernels/moe/api.zig", @@ -114,6 +113,7 @@ zig_library( "tensor.zig", "testing.zig", "zml.zig", + "//zml/kernel", "//zml/kernels/flash_attention/cute", "//zml/kernels/flash_attention/metal", "//zml/kernels/flash_attention/nki", diff --git a/zml/kernel.zig b/zml/kernel.zig deleted file mode 100644 index f111f721..00000000 --- a/zml/kernel.zig +++ /dev/null @@ -1,1630 +0,0 @@ -const std = @import("std"); - -const cuda_tile_builder = @import("dsl/cuda_tile/builder"); -const cute_builder = @import("dsl/cute/builder"); -const fly_builder = @import("dsl/fly/builder"); -const func = @import("mlir/dialects/func"); -const metal_builder = @import("dsl/metal/builder"); -const mlir = @import("mlir"); -const mosaic_tpu_builder = @import("dsl/mosaic_tpu/builder"); -const mosaic_tpu_dialect = @import("mlir/dialects/mosaic_tpu"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); -const tcl_builder = @import("dsl/tcl/builder"); -const triton_builder = @import("dsl/triton/builder"); - -const Compiler = @import("Compiler.zig"); -const DataType = @import("dtype.zig").DataType; -const mlirx = @import("mlirx.zig"); -const ops = @import("ops.zig"); -const platform_ = @import("platform.zig"); -const Shape = @import("shape.zig").Shape; -const Tensor = @import("tensor.zig").Tensor; - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -pub const triton = struct { - pub const Builder = triton_builder.Builder; - pub const Value = triton_builder.Value; - pub const DType = triton_builder.DType; - pub const FinishError = triton_builder.FinishError; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(triton_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e4m3fnuz => .f8e4m3fnuz, - .f8e5m2 => .f8e5m2, - else => std.debug.panic("zml.kernel.triton.from: dtype {s} has no Triton equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - num_stages: i32, - num_warps: i32, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - debug: bool = false, - /// Bytes for Triton's implicit global scratch argument, across all CTAs. - global_scratch_memory_size: i32 = 0, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try triton_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finish(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ttir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.triton.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ttir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const call_opts: ops.TritonOps = .{ - .debug = opts.debug, - .name = name, - .ir = ttir, - .grid = opts.grid, - .num_stages = opts.num_stages, - .num_warps = opts.num_warps, - .output_operand_aliases = aliases.constSlice(), - .is_tma_allowed = opts.global_scratch_memory_size > 0, - .global_scratch_memory_size = opts.global_scratch_memory_size, - }; - - const triton_results = if (opts.global_scratch_memory_size > 0) blk: { - // XLA maps the final custom-call result to Triton's implicit - // scratch pointer; it must not appear in the TTIR signature. - const extended = outputs_arr ++ [_]Shape{.init(.{opts.global_scratch_memory_size}, .u8)}; - const res = ops.triton(inputs_arr, extended, call_opts); - break :blk res[0..spec.outputs.len].*; - } else ops.triton(inputs_arr, outputs_arr, call_opts); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = triton_results[i]; - return results; - } - }; - } -}; - -pub const mosaic_tpu = struct { - const Builder = mosaic_tpu_builder.Builder; - const DType = mosaic_tpu_builder.DType; - const FinishError = mosaic_tpu_builder.FinishError; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(mosaic_tpu_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - else => std.debug.panic("zml.kernel.mosaic_tpu.from: dtype {s} has no Mosaic-TPU equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallExtras = struct { - vmem_limit_bytes: ?i64 = null, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, - has_communication: ?bool = null, - additional_attributes: []const mlir.NamedAttribute = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - dynamic_grid_bounds: []const Tensor = &.{}, - }; - - pub const CallOpts = struct { - cfg: ConfigT, - extras: CallExtras = .{}, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try mosaic_tpu_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finishOpts(&.{}, .{ .canonicalize = true }); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - const backend_config = buildBackendConfig(cur.allocator, ir, .{ - .vmem_limit_bytes = opts.extras.vmem_limit_bytes, - .disable_bounds_checks = opts.extras.disable_bounds_checks, - .disable_semaphore_checks = opts.extras.disable_semaphore_checks, - .has_communication = opts.extras.has_communication, - }) catch |err| - std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: backend_config build failed: {}", .{ name, err }); - defer cur.allocator.free(backend_config); - - var values: [spec.inputs.len]*const mlir.Value = undefined; - inline for (spec.inputs, 0..) |fname, i| values[i] = @field(inputs, fname).value(); - - var res_types: [spec.outputs.len]*const mlir.Type = undefined; - inline for (&res_types, spec.outputs) |*r, fname| { - r.* = mlirx.Type.rankedTensor(cur.mlir_ctx, @field(outputs, fname)); - } - - const aliases = resolveOutputOperandAliases(opts.extras.output_operand_aliases, @intCast(opts.extras.dynamic_grid_bounds.len)); - - const op = callTpuCustomCall(.{ - .inputs = &values, - .result_types = &res_types, - .backend_config = backend_config, - .aliases = aliases.constSlice(), - .dynamic_grid_bounds = opts.extras.dynamic_grid_bounds, - .additional_attributes = opts.extras.additional_attributes, - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| { - const out = @field(outputs, fname); - @field(results, fname) = Tensor._result(out, op.result(i)); - } - return results; - } - }; - } - - const CustomCallConfig = struct { - body: []const u8, - has_communication: ?bool = null, - serialization_format: i64 = 1, - needs_layout_passes: bool = true, - shape_invariant_numerics: bool = false, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, - }; - - const ScopedMemoryConfig = struct { - memory_space: i64 = 1, - offset: i64 = 0, - size: i64, - }; - - const BackendConfig = struct { - custom_call_config: CustomCallConfig, - scoped_memory_configs: ?[]const ScopedMemoryConfig = null, - }; - - const BackendConfigOpts = struct { - vmem_limit_bytes: ?i64 = null, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, - has_communication: ?bool = null, - }; - - fn buildBackendConfig( - allocator: std.mem.Allocator, - ir: [:0]const u8, - opts: BackendConfigOpts, - ) ![]u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - const module = try mlir.Module.parse(ctx, ir); - defer module.deinit(); - - mosaic_tpu_dialect.registerMosaicSerdePass(); - const prev_allow_unregistered = ctx.allowUnregisteredDialects(); - ctx.setAllowUnregisteredDialects(true); - defer ctx.setAllowUnregisteredDialects(prev_allow_unregistered); - - const pm = mlir.PassManager.init(ctx); - defer pm.deinit(); - try pm.asOpPassManager().addPipeline("mosaic-serde{serialize=true}"); - try pm.runOnOp(module.operation()); - - const Encoder = std.base64.standard.Encoder; - const b64 = b64: { - var bytecode: std.Io.Writer.Allocating = .init(allocator); - defer bytecode.deinit(); - try module.operation().writeBytecode(.{ .desired_emit_version = 0 }, &bytecode.writer); - const buf = try allocator.alloc(u8, Encoder.calcSize(bytecode.written().len)); - break :b64 Encoder.encode(buf, bytecode.written()); - }; - defer allocator.free(b64); - - const config = BackendConfig{ - .custom_call_config = .{ - .body = b64, - .has_communication = opts.has_communication, - .disable_bounds_checks = opts.disable_bounds_checks, - .disable_semaphore_checks = opts.disable_semaphore_checks, - }, - .scoped_memory_configs = if (opts.vmem_limit_bytes) |vmem| - &.{.{ .size = vmem }} - else - null, - }; - - var json: std.Io.Writer.Allocating = .init(allocator); - errdefer json.deinit(); - - try json.writer.print("{f}", .{std.json.fmt(config, .{ .emit_null_optional_fields = false })}); - return try json.toOwnedSlice(); - } - - const TpuCustomCallArgs = struct { - inputs: []const *const mlir.Value, - result_types: []const *const mlir.Type, - backend_config: []const u8, - aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias, - dynamic_grid_bounds: []const Tensor, - additional_attributes: []const mlir.NamedAttribute, - }; - - fn callTpuCustomCall(args: TpuCustomCallArgs) *mlir.Operation { - const cur = Compiler.current(); - - var all_inputs: stdx.BoundedArray(*const mlir.Value, stablehlo.CustomCallOpts.MAX_OPERANDS) = .empty; - for (args.dynamic_grid_bounds) |t| all_inputs.appendAssumeCapacity(t.value()); - all_inputs.appendSliceAssumeCapacity(args.inputs); - - return stablehlo.custom_call( - cur.mlir_ctx, - all_inputs.constSlice(), - args.result_types, - .{ - .call_target_name = "tpu_custom_call", - .backend_config = .{ .original = args.backend_config }, - .has_side_effect = false, - .api_version = .original, - .additional_attributes = args.additional_attributes, - .output_operand_aliases = args.aliases, - }, - cur.location, - ).appendTo(cur.currentScope().block); - } -}; - -pub const cuda_tile = struct { - pub const Builder = cuda_tile_builder.Builder; - pub const Value = cuda_tile_builder.Value; - pub const DType = cuda_tile_builder.DType; - pub const FinishError = cuda_tile_builder.FinishError; - pub const EntryHint = cuda_tile_builder.EntryHint; - pub const Arch = cuda_tile_builder.Arch; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(cuda_tile_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i4, .u4 => .i4, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - .f8e8m0 => .f8e8m0fnu, - .f4e2m1 => .f4e2m1fn, - else => std.debug.panic("zml.kernel.cuda_tile.from: dtype {s} has no CUDA Tile IR equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - /// No `num_warps`/`num_stages`: warp and CTA tuning is - /// `optimization_hints` inside the IR (`Opts.hints`). - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - /// Outputs XLA zeroes before the launch, by position in - /// `spec.outputs`, ascending. - zeroed_outputs: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cuda_tile_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finish(&.{}); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.cuda_tile.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.cudaTile(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .zeroed_outputs = opts.zeroed_outputs, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; - } -}; - -pub const cute = struct { - pub const AlgebraExpr = cute_builder.AlgebraExpr; - pub const Builder = cute_builder.Builder; - pub const ArgSpec = cute_builder.ArgSpec; - pub const Value = cute_builder.Value; - pub const View = cute_builder.View; - pub const Atom = cute_builder.Atom; - pub const DType = cute_builder.DType; - pub const AlgebraToken = cute_builder.AlgebraToken; - pub const LayoutSpec = cute_builder.LayoutSpec; - pub const ConstrainedDynamic = cute_builder.ConstrainedDynamic; - pub const FinishError = cute_builder.FinishError; - pub const TmaConfig = cute_builder.TmaConfig; - pub const TmaFormat = cute_builder.TmaFormat; - pub const BlockScaledMmaConfig = cute_builder.BlockScaledMmaConfig; - pub const CtaGroup = cute_builder.CtaGroup; - pub const ctaCount = cute_builder.ctaCount; - pub const MBarrierScope = cute_builder.MBarrierScope; - pub const OperandMajorMode = cute_builder.OperandMajorMode; - pub const Tcgen05SmemSwizzle = cute_builder.Tcgen05SmemSwizzle; - pub const Tcgen05SmemDesc = cute_builder.Tcgen05SmemDesc; - pub const Tcgen05InstrDesc = cute_builder.Tcgen05InstrDesc; - pub const CopyBulkTensorTileG2SOp = cute_builder.CopyBulkTensorTileG2SOp; - pub const CopyBulkTensor2DGather4G2SOp = cute_builder.CopyBulkTensor2DGather4G2SOp; - pub const CopyBulkTensorTileS2GOp = cute_builder.CopyBulkTensorTileS2GOp; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(cute_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f4e2m1 => .f4e2m1fn, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - .f8e8m0 => .f8e8m0fnu, - else => std.debug.panic("zml.kernel.cute.from: dtype {s} has no CuTe equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - /// `kernel.launch(grid=, block=)` in the Python DSL; dynamic shared - /// memory is what the kernel declares. - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - block: [3]i32, - zeroed_outputs: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, block: [3]i32) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cute_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - // XLA passes one pointer per input then per output; it does - // not check the count against the kernel. - const expected = spec.inputs.len + spec.outputs.len; - if (b.args.len != expected) { - std.debug.panic("zml.kernel.cute.Kernel({s}): declareArgs declared {d} arguments, inputs + outputs are {d}", .{ name, b.args.len, expected }); - } - - return b.finish(block); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg, opts.block) catch |err| - std.debug.panic("zml.kernel.cute.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .block = opts.block, - .zeroed_outputs = opts.zeroed_outputs, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; - } - - fn ProgramSpec(comptime Config: type) type { - return struct { - /// Public host entry selected by XLA. The callback may give the - /// nested CUDA kernel a different (private) symbol. - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - /// A complete CuTe program: one or more `cuda.kernel` operations inside - /// a `gpu.module`, followed by a public `func.func` which constructs the - /// launch-time CuTe objects and calls `cuda.launch_ex`. - /// - /// This is the source-level equivalent of invoking a Python `@cute.jit` - /// object whose `__call__` builds TMA descriptors and launches an - /// `@cute.kernel`. XLA executes the generated host function on its CUDA - /// stream, so the custom call deliberately carries no external grid or - /// block dimensions. - pub fn Program( - comptime ConfigT: type, - comptime spec: ProgramSpec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - zeroed_outputs: []const i32 = &.{}, - scalars: []const i64 = &.{}, - symmetric_workspace: ?ops.CuteOps.SymmetricWorkspace = null, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cute_builder.Builder.openProgram(allocator, ctx, name); - defer b.deinit(); - try spec.run(&b, cfg); - return b.finishProgram(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.cute.Program({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .zeroed_outputs = opts.zeroed_outputs, - .scalars = opts.scalars, - .symmetric_workspace = opts.symmetric_workspace, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; - } -}; - -pub const tcl = struct { - pub const Builder = tcl_builder.Builder; - pub const TensorOperation = tcl_builder.TensorOperation; - pub const Value = tcl_builder.Value; - pub const Axis = tcl_builder.Axis; - pub const Expr = tcl_builder.Expr; - pub const Dram = tcl_builder.Dram; - pub const Context = tcl_builder.Context; - pub const DType = tcl_builder.DType; - pub const VeOp = tcl_builder.VeOp; - pub const ReduceMode = tcl_builder.ReduceMode; - pub const Predicate = tcl_builder.Predicate; - pub const Tactic = tcl_builder.Tactic; - pub const FinishError = tcl_builder.FinishError; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(tcl_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .bool, - .i4 => .i4, - .i8 => .i8, - .i16 => .i16, - .i32 => .i32, - .i64 => .i64, - .u8 => .u8, - .u16 => .u16, - .u32 => .u32, - .u64 => .u64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f4e2m1 => .f4_e2, - .f8e4m3fn => .f8_e4, - .f8e5m2 => .f8_e5, - else => std.debug.panic("zml.kernel.tcl.from: dtype {s} has no TCL equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - /// A TCL function compiled by the Furiosa plugin as one native unit: - /// `declareArgs` declares the inputs in order, `ret` the outputs. - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - return build(allocator, cfg, null, null, null); - } - - /// `updated`, when given, receives the input each output updates - /// in place (`Builder.update`), or null. - fn build(allocator: std.mem.Allocator, cfg: ConfigT, inputs: ?[]const Shape, outputs: ?[]const Shape, updated: ?*[spec.outputs.len]?usize) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try Builder.open(allocator, ctx, name); - defer b.deinit(); - try spec.run(&b, cfg); - - const results = b.results orelse std.debug.panic("zml.kernel.tcl.Kernel({s}): run did not call ret", .{name}); - if (b.args.len != spec.inputs.len or results.len != spec.outputs.len) { - std.debug.panic("zml.kernel.tcl.Kernel({s}): {d} arguments and {d} results for {d} inputs and {d} outputs", .{ name, b.args.len, results.len, spec.inputs.len, spec.outputs.len }); - } - - if (inputs) |shapes| for (b.args, shapes, spec.inputs) |t, s, n| check(t, s, n); - if (outputs) |shapes| for (results, shapes, spec.outputs) |t, s, n| check(t, s, n); - if (updated) |u| for (u, results) |*dst, r| { - dst.* = b.updatedArgument(r); - }; - - return b.finish(); - } - - fn check(t: tcl_builder.Tensor, s: Shape, field: []const u8) void { - var same = from(s.dtype()) == t.dtype; - if (t.shards > 1) { - // A partition holds one chip's share of a chip-split DRAM tensor. - var n: i64 = 1; - for (t.axes) |a| n *= a.size; - same = same and n == @as(i64, @intCast(s.count())) * t.shards; - } else { - same = same and s.rank() == t.axes.len; - if (same) for (t.axes, 0..) |a, i| { - same = same and a.size == s.dim(i); - }; - } - - if (!same) std.debug.panic("zml.kernel.tcl.Kernel({s}): {s} is {f}, the TCL function has {s} over {d} axes", .{ name, field, s, @tagName(t.dtype), t.axes.len }); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - var input_shapes: [spec.inputs.len]Shape = undefined; - inline for (spec.inputs, 0..) |fname, i| { - inputs_arr[i] = @field(inputs, fname); - input_shapes[i] = inputs_arr[i].shape(); - } - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - var updated: [spec.outputs.len]?usize = undefined; - const ir = build(cur.allocator, opts.cfg, &input_shapes, &outputs_arr, &updated) catch |err| - std.debug.panic("zml.kernel.tcl.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; - for (updated, 0..) |input, output| if (input) |i| aliases.appendAssumeCapacity(.{ - .output_index = @intCast(output), - .operand_index = @intCast(i), - }); - const tensor_results = ops.tcl(inputs_arr, outputs_arr, .{ .ir = ir, .output_operand_aliases = aliases.constSlice() }); - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| { - @field(results, fname) = tensor_results[i]; - } - - return results; - } - }; - } -}; - -pub const metal = struct { - pub const Builder = metal_builder.Builder; - pub const Value = metal_builder.Value; - pub const Type = metal_builder.Type; - pub const Struct = metal_builder.Struct; - pub const ArgSpec = metal_builder.ArgSpec; - pub const LoopOpts = metal_builder.LoopOpts; - pub const FinishError = metal_builder.FinishError; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(metal_builder.registerDialects); - } - - pub fn from(dt: DataType) Type { - return switch (dt) { - .bool => .bool, - .i8 => .char, - .i16 => .short, - .i32 => .int, - .i64 => .long, - .u8 => .uchar, - .u16 => .ushort, - .u32 => .uint, - .u64 => .ulong, - .f16 => .half, - .bf16 => .bfloat, - .f32 => .float, - else => std.debug.panic("zml.kernel.metal.from: dtype {s} has no MSL equivalent", .{@tagName(dt)}), - }; - } - - /// The bytes of a `constant T &` argument: `.of(@as(i32, n))`, or a host - /// `extern struct` laid out like a `constant S &` argument. - pub const Constant = struct { - buffer: [256]u8 = undefined, - len: usize = 0, - - pub fn of(value: anytype) Constant { - const raw = std.mem.asBytes(&value); - var c: Constant = .{ .len = raw.len }; - @memcpy(c.buffer[0..raw.len], raw); - return c; - } - - pub fn bytes(self: *const Constant) []const u8 { - return self.buffer[0..self.len]; - } - }; - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - /// The arguments bound to input tensors. - inputs: []const [:0]const u8, - /// The arguments bound to output tensors; one named like an - /// input updates it in place. - outputs: []const [:0]const u8, - /// The `constant` arguments given by value at the call. - constants: []const [:0]const u8 = &.{}, - run: *const fn (*Builder, Config) FinishError!void, - fast_math: bool = true, - }; - } - - /// A Metal kernel: `run` declares its arguments with `declareArgs`, by - /// name, and builds its body. - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - pub const Constants = StructOf(spec.constants, Constant); - - pub const CallOpts = struct { - cfg: ConfigT, - /// Threadgroups per grid. - grid: [3]u32, - /// Threads per threadgroup. - threadgroup: [3]u32, - /// Bytes of `[[threadgroup(0)]]` memory. - threadgroup_memory: u64 = 0, - constants: Constants = std.mem.zeroes(Constants), - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - var arguments: std.ArrayList(metal_builder.Argument) = .empty; - defer arguments.deinit(allocator); - return build(allocator, cfg, &arguments); - } - - fn build(allocator: std.mem.Allocator, cfg: ConfigT, arguments: *std.ArrayList(metal_builder.Argument)) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - var b = Builder.open(allocator, ctx, name, .{ .fast_math = spec.fast_math }); - defer b.deinit(); - try spec.run(&b, cfg); - const ir = try b.finish(); - errdefer allocator.free(ir); - try arguments.appendSlice(allocator, b.arguments.items); - return ir; - } - - fn indexOf(names: []const [:0]const u8, arg: []const u8) ?usize { - for (names, 0..) |n, i| if (std.mem.eql(u8, n, arg)) return i; - return null; - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - const allocator = cur.allocator; - - var arguments: std.ArrayList(metal_builder.Argument) = .empty; - defer arguments.deinit(allocator); - const ir = build(allocator, opts.cfg, &arguments) catch |err| - std.debug.panic("zml.kernel.metal.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |field, i| inputs_arr[i] = @field(inputs, field); - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |field, i| outputs_arr[i] = @field(outputs, field); - var constants: [spec.constants.len][]const u8 = undefined; - inline for (spec.constants, 0..) |field, i| constants[i] = @field(opts.constants, field).bytes(); - - var bound: [spec.outputs.len]bool = @splat(false); - var bound_inputs: [spec.inputs.len]bool = @splat(false); - var bound_constants: [spec.constants.len]bool = @splat(false); - var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; - var bindings: std.ArrayList(i32) = .empty; - defer bindings.deinit(allocator); - for (arguments.items) |arg| { - if (arg.kind == .threadgroup) { - // `threadgroup_memory` sizes `[[threadgroup(0)]]` alone. - if (arg.index != 0) std.debug.panic("zml.kernel.metal.Kernel({s}): threadgroup argument {s} is [[threadgroup({d})]], and only [[threadgroup(0)]] gets memory", .{ name, arg.name, arg.index }); - continue; - } - std.debug.assert(arg.index == bindings.items.len); - const input = indexOf(spec.inputs, arg.name); - const output = indexOf(spec.outputs, arg.name); - if (input) |i| bound_inputs[i] = true; - if (output) |o| bound[o] = true; - const binding: i32 = if (input) |i| blk: { - if (output) |o| aliases.appendAssumeCapacity(.{ .output_index = @intCast(o), .operand_index = @intCast(i) }); - break :blk @intCast(i); - } else if (output) |o| - @intCast(spec.inputs.len + o) - else if (indexOf(spec.constants, arg.name)) |k| blk: { - if (constants[k].len != arg.size) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} has {d} bytes, the argument {d}", .{ name, arg.name, constants[k].len, arg.size }); - bound_constants[k] = true; - break :blk -1 - @as(i32, @intCast(k)); - } else std.debug.panic("zml.kernel.metal.Kernel({s}): argument {s} is no input, output or constant", .{ name, arg.name }); - bindings.append(allocator, binding) catch @panic("OOM"); - } - for (bound, spec.outputs) |b, output| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): output {s} is no argument", .{ name, output }); - for (bound_inputs, spec.inputs) |b, input| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): input {s} is no argument", .{ name, input }); - for (bound_constants, spec.constants) |b, constant| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} is no argument", .{ name, constant }); - - const tensor_results = ops.metal(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .threadgroup = opts.threadgroup, - .threadgroup_memory = opts.threadgroup_memory, - .arguments = bindings.items, - .constants = &constants, - .output_operand_aliases = aliases.constSlice(), - }); - var results: Results = undefined; - inline for (spec.outputs, 0..) |field, i| @field(results, field) = tensor_results[i]; - return results; - } - }; - } -}; - -pub const fly = struct { - /// Threads per wavefront: 64 on CDNA, 32 on RDNA. - pub fn waveSize(p: *const platform_.Platform) i32 { - const cc = platform_.rocm.computeCapability(p) orelse return 64; - return switch (cc.architecture()) { - .cdna1, .cdna2, .cdna3, .cdna4 => 64, - .rdna2, .rdna3, .rdna3_5, .rdna4 => 32, - }; - } - - /// The plugin sizes a block in wavefronts, so a thread count has to divide - /// evenly: a kernel's layouts are built for an exact number of threads and - /// spare ones would partition out of range. - fn warpsFor(p: *const platform_.Platform, threads: i32) i32 { - const wave = waveSize(p); - if (@rem(threads, wave) != 0) { - std.debug.panic("zml.kernel.fly: {d} threads is not a multiple of the {d}-wide wavefront", .{ threads, wave }); - } - return @divExact(threads, wave); - } - - pub const Builder = fly_builder.Builder; - pub const Value = fly_builder.Value; - pub const DType = fly_builder.DType; - pub const FinishError = fly_builder.FinishError; - pub const TiledCopy = fly_builder.TiledCopy; - pub const TiledMma = fly_builder.TiledMma; - pub const Arch = fly_builder.Arch; - pub const MmaFlavor = fly_builder.MmaFlavor; - pub const fly_rocdl = fly_builder.fly_rocdl; - - /// The matrix atom this device provides, or null when it has none: WMMA - /// arrived with RDNA3, so gfx1030 has no matrix instruction at all. - pub fn mmaFlavor(p: *const platform_.Platform) ?MmaFlavor { - const cc = platform_.rocm.computeCapability(p) orelse return .cdna3_mfma; - return switch (cc.architecture()) { - .cdna1, .cdna2, .cdna3, .cdna4 => .cdna3_mfma, - .rdna3, .rdna3_5 => .gfx11_wmma, - .rdna4 => .gfx120x_wmma, - .rdna2 => null, - }; - } - pub const layout = fly_builder.layout; - pub const Layout = fly_builder.Layout; - pub const Tile = fly_builder.Tile; - pub const IntTuple = fly_builder.IntTuple; - pub const L = fly_builder.L; - pub const rowMajor = fly_builder.rowMajor; - pub const colMajor = fly_builder.colMajor; - pub const ordered = fly_builder.ordered; - pub const tile = fly_builder.tile; - pub const it = fly_builder.it; - - pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(fly_builder.registerDialects); - } - - pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e4m3fnuz => .f8e4m3fnuz, - .f8e5m2 => .f8e5m2, - .f8e5m2fnuz => .f8e5m2fnuz, - else => std.debug.panic("zml.kernel.fly.from: dtype {s} has no Fly equivalent", .{@tagName(dt)}), - }; - } - - fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; - } - - pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), - ) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - pub const arg_names = spec.inputs ++ spec.outputs; - /// By name, as `!fly.memref` views. - pub const Args = StructOf(arg_names, Value); - - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - /// Threads per block. Converted to the wavefronts the plugin - /// wants using the device's wave size, which is 64 on CDNA and - /// 32 on RDNA, so a kernel states the thread count its layouts - /// were built for and runs on both. - threads: i32, - /// > 0 sets `amdgpu-waves-per-eu = "N, N"`: a hard occupancy clamp. - waves_per_eu: i32 = 0, - /// Required when the body uses `fly.get_dyn_shared`. - shared_mem_bytes: i64 = 0, - /// Inputs first, then outputs; see `zeroedOutput`. A no-op - /// inside command buffers. - zeroed_args: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - /// The arguments `run` was declared with. - pub fn args(b: *Builder) Args { - var out: Args = undefined; - inline for (arg_names, 0..) |fname, i| @field(out, fname) = b.arg(i); - return out; - } - - /// `zeroed_args` index of an output. - pub fn zeroedOutput(comptime f: std.meta.FieldEnum(Outputs)) i32 { - return @intCast(spec.inputs.len + @backingInt(f)); - } - - /// `shapes` is inputs, then outputs. - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, shapes: []const Shape) ![:0]const u8 { - std.debug.assert(shapes.len == arg_names.len); - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try fly_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - var specs: [arg_names.len]Builder.ArgSpec = undefined; - inline for (arg_names, 0..) |fname, i| { - const shape = shapes[i]; - specs[i] = .{ - .name = fname, - .dtype = from(shape.dtype()), - .dims = if (shape.rank() > 0) shape.dims() else null, - }; - } - try b.declareArgsLow(&specs); - - try spec.run(&b, cfg); - - return b.finish(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - var shapes: [arg_names.len]Shape = undefined; - inline for (0..spec.inputs.len) |i| shapes[i] = inputs_arr[i].shape(); - inline for (0..spec.outputs.len) |i| shapes[spec.inputs.len + i] = outputs_arr[i]; - - const ir = emit(cur.allocator, opts.cfg, &shapes) catch |err| - std.debug.panic("zml.kernel.fly.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.fly(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .num_warps = warpsFor(cur.platform, opts.threads), - .waves_per_eu = opts.waves_per_eu, - .shared_mem_bytes = opts.shared_mem_bytes, - .zeroed_args = opts.zeroed_args, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; - } -}; - -/// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the -/// ragged edge. -const FlyVectorAdd = struct { - const Cfg = struct { - thr: fly.Layout = fly.ordered(.{ 8, 16 }, .{ 1, 0 }), - val: fly.Layout = fly.ordered(.{ 1, 4 }, .{ 0, 1 }), - }; - - const K = fly.Kernel(Cfg, .{ - .name = "vector_add", - .inputs = &.{ "a", "b" }, - .outputs = &.{"c"}, - .run = run, - }); - - fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { - const a = K.args(b); - fly_builder.emitVectorAdd(b, cfg.thr, cfg.val, a.a, a.b, a.c); - } -}; - -test "fly kernel emits a module XLA can parse" { - const shapes = [_]Shape{ .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32) }; - const ir = try FlyVectorAdd.K.emit(std.testing.allocator, .{}, &shapes); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.func @vector_add(%arg0: !fly.ptr, %arg1: !fly.ptr, %arg2: !fly.ptr) kernel") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "fly.static : !fly.tile<[8|64]>") != null); -} - -/// C = A @ B^T with one MFMA-tiled block. -const FlyTiledMma = struct { - const Cfg = struct { flavor: fly.MmaFlavor }; - const K = fly.Kernel(Cfg, .{ - .name = "tiled_mma", - .inputs = &.{ "a", "b" }, - .outputs = &.{"c"}, - .run = run, - }); - - fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { - const a = K.args(b); - fly_builder.emitTiledMma(b, cfg.flavor, a.a, a.b, a.c); - } -}; - -test "fly kernels run on rocm" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - if (platform.target != .rocm) return error.SkipZigTest; - - const allocator = std.testing.allocator; - - // Compile `forward`, upload both inputs, run, and bring the result back. - const call = struct { - fn f(comptime forward: anytype, p: *const zml.Platform, ta: Tensor, ha: anytype, tb: Tensor, hb: anytype) !zml.Slice { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ ta, tb }, p, .{}); - defer exe.deinit(); - var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), std.mem.sliceAsBytes(ha)); - defer ba.deinit(); - var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), std.mem.sliceAsBytes(hb)); - defer bb.deinit(); - var out = try exe.eval(std.testing.allocator, std.testing.io, .{ ba, bb }); - defer out.deinit(); - return out.toSliceAlloc(std.testing.allocator, std.testing.io); - } - }.f; - - // vector add: elementwise over a ragged (100, 1000), predicated at the edge. - { - const M, const N = .{ 100, 1000 }; - const Mod = struct { - pub fn forward(a: Tensor, b: Tensor) Tensor { - return FlyVectorAdd.K.call(.{ .a = a, .b = b }, .{ .c = a.shape() }, .{ - .cfg = .{}, - .grid = .{ (M + 8 - 1) / 8, (N + 64 - 1) / 64, 1 }, - .threads = 8 * 16, - }).c; - } - }; - - const ha = try allocator.alloc(f32, M * N); - defer allocator.free(ha); - const hb = try allocator.alloc(f32, M * N); - defer allocator.free(hb); - for (ha, hb, 0..) |*va, *vb, i| { - va.* = @floatFromInt(i % 977); - vb.* = @floatFromInt(1000 + (i % 313)); - } - - var host = try call(Mod.forward, platform, .init(.{ M, N }, .f32), ha, .init(.{ M, N }, .f32), hb); - defer host.free(allocator); - for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(ha[i] + hb[i], v); - } - - // tiled mma: C = A @ B^T in one block of 256 threads, over whatever matrix - // atom this device has. CDNA multiplies f32 through MFMA; RDNA3+ has WMMA, - // whose verifier rejects f32 operands, so the operand type and K come from - // the flavour rather than being written into the test. - if (fly.mmaFlavor(platform)) |flavor| switch (flavor) { - inline else => |f| { - const Elem = switch (comptime f.operandDType()) { - .f32 => f32, - .f16 => f16, - else => @compileError("unhandled MMA operand type"), - }; - const M, const N = .{ 64, 64 }; - const Kd = comptime f.blockK(); - const operand_dtype: DataType = switch (comptime f.operandDType()) { - .f32 => .f32, - .f16 => .f16, - else => unreachable, - }; - - const Mod = struct { - pub fn forward(a: Tensor, b: Tensor) Tensor { - return FlyTiledMma.K.call(.{ .a = a, .b = b }, .{ .c = Shape.init(.{ M, N }, .f32) }, .{ - .cfg = .{ .flavor = f }, - .grid = .{ 1, 1, 1 }, - .threads = comptime f.blockThreads(), - }).c; - } - }; - - var ha: [M * Kd]Elem = undefined; - var hb: [N * Kd]Elem = undefined; - for (&ha, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 7) % 13)) - 6; - for (&hb, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 5) % 11)) - 5; - - var host = try call(Mod.forward, platform, .init(.{ M, Kd }, operand_dtype), &ha, .init(.{ N, Kd }, operand_dtype), &hb); - defer host.free(allocator); - - // f16 operands accumulate in f32, so the error is set by the input - // rounding rather than the sum. - const tol: f32 = if (Elem == f16) 1e-2 else 1e-3; - const out = host.items(f32); - for (0..M) |m| for (0..N) |n| { - var acc: f32 = 0; - for (0..Kd) |k| acc += @as(f32, ha[m * Kd + k]) * @as(f32, hb[n * Kd + k]); - try std.testing.expectApproxEqAbs(acc, out[m * N + n], tol); - }; - }, - }; -} - -/// `furiosa.tcl.examples.softmax` over f32. -const TclSoftmax = struct { - const Cfg = struct { b: i64, s: i64, d: i64 }; - const K = tcl.Kernel(Cfg, .{ - .name = "softmax_kernel", - .inputs = &.{"x"}, - .outputs = &.{"y"}, - .run = run, - }); - - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - const B = b.axis("B", cfg.b); - const S = b.axis("S", cfg.s); - const D = b.axis("D", cfg.d); - const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ B, S, D } } }); - - const op = b.tensorOperation(.{}); - const x = op.fetch(t.x, .{}); - const e = x.subf(x.reduce(&.{D}, .maxf)).exp(); - b.ret(&.{try op.commit(e.divf(e.reduce(&.{D}, .addf)), .{})}); - } -}; - -test "tcl kernels run on furiosa" { - const zml = @import("zml.zig"); - const allocator = std.testing.allocator; - const cfg: TclSoftmax.Cfg = .{ .b = 2, .s = 16, .d = 256 }; - - const ir = try TclSoftmax.K.emit(allocator, cfg); - defer allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "func.func @softmax_kernel(") != null); - - const platform = zml.testing.env(); - if (platform.target != .furiosa) return error.SkipZigTest; - - const Mod = struct { - pub fn forward(x: Tensor) Tensor { - return TclSoftmax.K.call(.{ .x = x }, .{ .y = x.shape() }, .{ .cfg = cfg }).y; - } - }; - const x: Tensor = .init(.{ cfg.b, cfg.s, cfg.d }, .f32); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{}); - defer exe.deinit(); - - var input: [cfg.b * cfg.s * cfg.d]f32 = undefined; - for (&input, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 37)) * 0.25 - 4; - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&input)); - defer x_buffer.deinit(); - var result = try exe.eval(allocator, std.testing.io, .{x_buffer}); - defer result.deinit(); - var host = try result.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - - const out = host.items(f32); - var row: usize = 0; - while (row < cfg.b * cfg.s) : (row += 1) { - const in = input[row * cfg.d ..][0..cfg.d]; - var max: f32 = in[0]; - for (in) |v| max = @max(max, v); - var sum: f32 = 0; - for (in) |v| sum += @exp(v - max); - for (in, out[row * cfg.d ..][0..cfg.d]) |v, y| try std.testing.expectApproxEqAbs(@exp(v - max) / sum, y, 1e-4); - } -} - -test "cute kernel emits the module the CuTe compiler takes" { - const Cfg = struct { n: i64 }; - const AddOne = cute.Kernel(Cfg, .{ - .name = "add_one", - .inputs = &.{"x"}, - .outputs = &.{"out"}, - .run = struct { - fn run(b: *cute.Builder, cfg: Cfg) cute.FinishError!void { - const a = try b.declareArgs(.{ - .x = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, - .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, - }); - const i = b.blockIdx().x.mul(b.blockDim().x).add(b.threadIdx().x); - var guard = b.openIf(i.lt(cfg.n)); - { - a.out.set(.{i}, a.x.get(.{i}).add(1.0)); - guard.close(); - } - } - }.run, - }); - - const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1000 }, .{ 128, 1, 1 }); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "\"func.func\"() <{function_type = (") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "sym_name = \"add_one\"") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "nvvm.reqntid = array") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "cute.memref.load") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); -} - -test "cuda_tile kernel emits a module XLA can parse" { - const Cfg = struct { n: i64 }; - const AddOne = cuda_tile.Kernel(Cfg, .{ - .name = "add_one", - .inputs = &.{"x"}, - .outputs = &.{"out"}, - .run = struct { - fn run(b: *cuda_tile.Builder, cfg: Cfg) cuda_tile.FinishError!void { - const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .out = .{ .ptr = .f32 } }); - const vx = b.partitionView(b.tensorView(a.x, &.{cfg.n}, &.{1}), &.{128}, .{}); - const vo = b.partitionView(b.tensorView(a.out, &.{cfg.n}, &.{1}), &.{128}, .{}); - const bid = b.tileBlockId(); - _ = b.store(b.load(vx, &.{bid.x}).add(1.0), vo, &.{bid.x}); - } - }.run, - }); - - const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1024 }); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.module @add_one") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "entry @add_one(") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "load_view_tko") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "store_view_tko") != null); -} - -/// exp over a tensor split across two chips in one launch, returning the -/// split result and its all_gather. -const TclSplitExp = struct { - const Cfg = struct {}; - const K = tcl.Kernel(Cfg, .{ .name = "split_exp", .inputs = &.{"x"}, .outputs = &.{ "y", "all" }, .run = run }); - fn run(b: *tcl.Builder, _: Cfg) tcl.FinishError!void { - const C = b.axis("C", 2); - const A = b.axis("A", 1024); - const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ C, A }, .dram = tcl.Dram{ .chip = &.{.{ .axis = C }} } } }); - const op = b.tensorOperation(.{ .context = .{ .layout = .{ .chip = .{ .axis = C } } } }); - const local = try op.commit(op.fetch(t.x, .{}).exp(), .{}); - const all = try b.allGather(local, C, .{}); - b.ret(&.{ try b.asDram(local, .{ .chip = &.{.{ .axis = C }} }), try b.asDram(all, .{}) }); - } - fn body(ctx: struct { x: Tensor }, shapes: [2]Shape) [2]Tensor { - const r = K.call(.{ .x = ctx.x }, .{ .y = shapes[0], .all = shapes[1] }, .{ .cfg = .{} }); - return .{ r.y, r.all }; - } -}; - -// Needs FURIOSA_VISIBLE_DEVICES=0,1. -test "tcl multi-chip kernels run on two furiosa chips" { - const zml = @import("zml.zig"); - const allocator = std.testing.allocator; - const platform = zml.testing.env(); - if (platform.target != .furiosa or platform.devices.len != 2) return error.SkipZigTest; - const tp = zml.testing.tpMesh(); - const x_shape = Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(tp, .{ .c = .model }); - const Mod = struct { - pub fn forward(x: Tensor, mesh: *const zml.Sharding.Mesh) [2]Tensor { - const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), Shape.init(.{ .c = 2, .a = 1024 }, .f32) }, mesh, .{ - .in_specs = .{ .x = .parse(.{ .c = .model }) }, - .out_specs = .{ .parse(.{ .c = .model }), .replicated }, - }); - return .{ r[0], r[1] }; - } - }; - const x: Tensor = .fromShape(x_shape); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, tp }, platform, .{ .meshes = &.{tp} }); - defer exe.deinit(); - var hx: [2 * 1024]f32 = undefined; - for (&hx, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 13)) * 0.125 - 0.5; - var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x_shape, std.mem.sliceAsBytes(&hx)); - defer bx.deinit(); - var runner = try exe.runner(allocator); - defer runner.deinit(allocator); - var res: [2]zml.Buffer = undefined; - runner.run(std.testing.io, .{bx}, .{ &res[0], &res[1] }, .{ .wait = true }); - defer for (&res) |*r| r.deinit(); - for (res) |r| { - var host = try r.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - for (host.items(f32), 0..) |v, i| try std.testing.expectApproxEqRel(@exp(hx[i]), v, 1e-4); - } -} - -/// `out[i] = a[i] * scale + b[i]` over `n` floats. -const MetalAxpy = struct { - const Cfg = struct {}; - const K = metal.Kernel(Cfg, .{ - .name = "axpy", - .inputs = &.{ "a", "b" }, - .outputs = &.{"out"}, - .constants = &.{ "scale", "n" }, - .run = run, - }); - - fn run(b: *metal.Builder, _: Cfg) metal.FinishError!void { - const a = try b.declareArgs(.{ - .a = .{ .buffer = .{ .type = .float, .const_ = true } }, - .b = .{ .buffer = .{ .type = .float, .const_ = true } }, - .out = .{ .buffer = .{ .type = .float } }, - .scale = .{ .constant = .float }, - .n = .{ .constant = .uint }, - .tid = .{ .thread_position_in_grid = .uint }, - }); - b.returnIf(a.tid.ge(a.n)); - a.out.store(a.tid, b.fma(a.a.load(a.tid), a.scale, a.b.load(a.tid))); - } -}; - -test "metal kernels run on metal" { - const zml = @import("zml.zig"); - const allocator = std.testing.allocator; - - const ir = try MetalAxpy.K.emit(allocator, .{}); - defer allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "air.fma") != null); - - const platform = zml.testing.env(); - if (platform.target != .metal) return error.SkipZigTest; - - const n = 1000; - const Mod = struct { - pub fn forward(x: Tensor, y: Tensor) Tensor { - return MetalAxpy.K.call(.{ .a = x, .b = y }, .{ .out = x.shape() }, .{ - .cfg = .{}, - .grid = .{ (n + 255) / 256, 1, 1 }, - .threadgroup = .{ 256, 1, 1 }, - .constants = .{ .scale = .of(@as(f32, 0.5)), .n = .of(@as(u32, n)) }, - }).out; - } - }; - const x: Tensor = .init(.{n}, .f32); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, Tensor.init(.{n}, .f32) }, platform, .{}); - defer exe.deinit(); - - var hx: [n]f32 = undefined; - var hy: [n]f32 = undefined; - for (&hx, &hy, 0..) |*a, *b, i| { - a.* = @floatFromInt(i); - b.* = 1; - } - var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hx)); - defer bx.deinit(); - var by: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hy)); - defer by.deinit(); - var result = try exe.eval(allocator, std.testing.io, .{ bx, by }); - defer result.deinit(); - var host = try result.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - for (host.items(f32), hx, hy) |out, a, b| try std.testing.expectEqual(a * 0.5 + b, out); -} diff --git a/zml/kernel/BUILD.bazel b/zml/kernel/BUILD.bazel new file mode 100644 index 00000000..61ab44f8 --- /dev/null +++ b/zml/kernel/BUILD.bazel @@ -0,0 +1,15 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "kernel", + srcs = [ + "kernel.zig", + "//zml/kernel/cuda_tile", + "//zml/kernel/cute", + "//zml/kernel/fly", + "//zml/kernel/metal", + "//zml/kernel/mosaic_tpu", + "//zml/kernel/tcl", + "//zml/kernel/triton", + ], +) diff --git a/zml/kernel/cuda_tile/BUILD.bazel b/zml/kernel/cuda_tile/BUILD.bazel new file mode 100644 index 00000000..89b7977e --- /dev/null +++ b/zml/kernel/cuda_tile/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "cuda_tile", + srcs = [ + "cuda_tile.zig", + "test.zig", + ], +) diff --git a/zml/kernel/cuda_tile/cuda_tile.zig b/zml/kernel/cuda_tile/cuda_tile.zig new file mode 100644 index 00000000..3f6d8734 --- /dev/null +++ b/zml/kernel/cuda_tile/cuda_tile.zig @@ -0,0 +1,163 @@ +const std = @import("std"); + +const cuda_tile_builder = @import("dsl/cuda_tile/builder"); +pub const Builder = cuda_tile_builder.Builder; +pub const Value = cuda_tile_builder.Value; +pub const DType = cuda_tile_builder.DType; +pub const FinishError = cuda_tile_builder.FinishError; +pub const EntryHint = cuda_tile_builder.EntryHint; +pub const Arch = cuda_tile_builder.Arch; +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(cuda_tile_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i4, .u4 => .i4, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + .f8e8m0 => .f8e8m0fnu, + .f4e2m1 => .f4e2m1fn, + else => std.debug.panic("zml.kernel.cuda_tile.from: dtype {s} has no CUDA Tile IR equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + /// No `num_warps`/`num_stages`: warp and CTA tuning is + /// `optimization_hints` inside the IR (`Opts.hints`). + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + /// Outputs XLA zeroes before the launch, by position in + /// `spec.outputs`, ascending. + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cuda_tile_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finish(&.{}); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.cuda_tile.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.cudaTile(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .zeroed_outputs = opts.zeroed_outputs, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/cuda_tile/test.zig b/zml/kernel/cuda_tile/test.zig new file mode 100644 index 00000000..c929b0b9 --- /dev/null +++ b/zml/kernel/cuda_tile/test.zig @@ -0,0 +1,28 @@ +const std = @import("std"); + +const cuda_tile = @import("cuda_tile.zig"); + +test "cuda_tile kernel emits a module XLA can parse" { + const Cfg = struct { n: i64 }; + const AddOne = cuda_tile.Kernel(Cfg, .{ + .name = "add_one", + .inputs = &.{"x"}, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *cuda_tile.Builder, cfg: Cfg) cuda_tile.FinishError!void { + const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .out = .{ .ptr = .f32 } }); + const vx = b.partitionView(b.tensorView(a.x, &.{cfg.n}, &.{1}), &.{128}, .{}); + const vo = b.partitionView(b.tensorView(a.out, &.{cfg.n}, &.{1}), &.{128}, .{}); + const bid = b.tileBlockId(); + _ = b.store(b.load(vx, &.{bid.x}).add(1.0), vo, &.{bid.x}); + } + }.run, + }); + + const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1024 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.module @add_one") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "entry @add_one(") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "load_view_tko") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "store_view_tko") != null); +} diff --git a/zml/kernel/cute/BUILD.bazel b/zml/kernel/cute/BUILD.bazel new file mode 100644 index 00000000..7a613072 --- /dev/null +++ b/zml/kernel/cute/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "cute", + srcs = [ + "cute.zig", + "test.zig", + ], +) diff --git a/zml/kernel/cute/cute.zig b/zml/kernel/cute/cute.zig new file mode 100644 index 00000000..2e8c919f --- /dev/null +++ b/zml/kernel/cute/cute.zig @@ -0,0 +1,264 @@ +const std = @import("std"); + +const cute_builder = @import("dsl/cute/builder"); +pub const AlgebraExpr = cute_builder.AlgebraExpr; +pub const Builder = cute_builder.Builder; +pub const ArgSpec = cute_builder.ArgSpec; +pub const Value = cute_builder.Value; +pub const View = cute_builder.View; +pub const Atom = cute_builder.Atom; +pub const DType = cute_builder.DType; +pub const AlgebraToken = cute_builder.AlgebraToken; +pub const LayoutSpec = cute_builder.LayoutSpec; +pub const ConstrainedDynamic = cute_builder.ConstrainedDynamic; +pub const FinishError = cute_builder.FinishError; +pub const TmaConfig = cute_builder.TmaConfig; +pub const TmaFormat = cute_builder.TmaFormat; +pub const BlockScaledMmaConfig = cute_builder.BlockScaledMmaConfig; +pub const CtaGroup = cute_builder.CtaGroup; +pub const ctaCount = cute_builder.ctaCount; +pub const MBarrierScope = cute_builder.MBarrierScope; +pub const OperandMajorMode = cute_builder.OperandMajorMode; +pub const Tcgen05SmemSwizzle = cute_builder.Tcgen05SmemSwizzle; +pub const Tcgen05SmemDesc = cute_builder.Tcgen05SmemDesc; +pub const Tcgen05InstrDesc = cute_builder.Tcgen05InstrDesc; +pub const CopyBulkTensorTileG2SOp = cute_builder.CopyBulkTensorTileG2SOp; +pub const CopyBulkTensor2DGather4G2SOp = cute_builder.CopyBulkTensor2DGather4G2SOp; +pub const CopyBulkTensorTileS2GOp = cute_builder.CopyBulkTensorTileS2GOp; +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(cute_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f4e2m1 => .f4e2m1fn, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + .f8e8m0 => .f8e8m0fnu, + else => std.debug.panic("zml.kernel.cute.from: dtype {s} has no CuTe equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + /// `kernel.launch(grid=, block=)` in the Python DSL; dynamic shared + /// memory is what the kernel declares. + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + block: [3]i32, + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, block: [3]i32) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cute_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + // XLA passes one pointer per input then per output; it does + // not check the count against the kernel. + const expected = spec.inputs.len + spec.outputs.len; + if (b.args.len != expected) { + std.debug.panic("zml.kernel.cute.Kernel({s}): declareArgs declared {d} arguments, inputs + outputs are {d}", .{ name, b.args.len, expected }); + } + + return b.finish(block); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg, opts.block) catch |err| + std.debug.panic("zml.kernel.cute.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .block = opts.block, + .zeroed_outputs = opts.zeroed_outputs, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} + +fn ProgramSpec(comptime Config: type) type { + return struct { + /// Public host entry selected by XLA. The callback may give the + /// nested CUDA kernel a different (private) symbol. + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +/// A complete CuTe program: one or more `cuda.kernel` operations inside +/// a `gpu.module`, followed by a public `func.func` which constructs the +/// launch-time CuTe objects and calls `cuda.launch_ex`. +/// +/// This is the source-level equivalent of invoking a Python `@cute.jit` +/// object whose `__call__` builds TMA descriptors and launches an +/// `@cute.kernel`. XLA executes the generated host function on its CUDA +/// stream, so the custom call deliberately carries no external grid or +/// block dimensions. +pub fn Program( + comptime ConfigT: type, + comptime spec: ProgramSpec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + zeroed_outputs: []const i32 = &.{}, + scalars: []const i64 = &.{}, + symmetric_workspace: ?ops.CuteOps.SymmetricWorkspace = null, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cute_builder.Builder.openProgram(allocator, ctx, name); + defer b.deinit(); + try spec.run(&b, cfg); + return b.finishProgram(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.cute.Program({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .zeroed_outputs = opts.zeroed_outputs, + .scalars = opts.scalars, + .symmetric_workspace = opts.symmetric_workspace, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/cute/test.zig b/zml/kernel/cute/test.zig new file mode 100644 index 00000000..f4d457a3 --- /dev/null +++ b/zml/kernel/cute/test.zig @@ -0,0 +1,34 @@ +const std = @import("std"); + +const cute = @import("cute.zig"); + +test "cute kernel emits the module the CuTe compiler takes" { + const Cfg = struct { n: i64 }; + const AddOne = cute.Kernel(Cfg, .{ + .name = "add_one", + .inputs = &.{"x"}, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *cute.Builder, cfg: Cfg) cute.FinishError!void { + const a = try b.declareArgs(.{ + .x = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, + .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, + }); + const i = b.blockIdx().x.mul(b.blockDim().x).add(b.threadIdx().x); + var guard = b.openIf(i.lt(cfg.n)); + { + a.out.set(.{i}, a.x.get(.{i}).add(1.0)); + guard.close(); + } + } + }.run, + }); + + const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1000 }, .{ 128, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "\"func.func\"() <{function_type = (") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "sym_name = \"add_one\"") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "nvvm.reqntid = array") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cute.memref.load") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); +} diff --git a/zml/kernel/fly/BUILD.bazel b/zml/kernel/fly/BUILD.bazel new file mode 100644 index 00000000..57203521 --- /dev/null +++ b/zml/kernel/fly/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "fly", + srcs = [ + "fly.zig", + "test.zig", + ], +) diff --git a/zml/kernel/fly/fly.zig b/zml/kernel/fly/fly.zig new file mode 100644 index 00000000..b7d6c943 --- /dev/null +++ b/zml/kernel/fly/fly.zig @@ -0,0 +1,248 @@ +const std = @import("std"); + +const fly_builder = @import("dsl/fly/builder"); +pub const Builder = fly_builder.Builder; +pub const Value = fly_builder.Value; +pub const DType = fly_builder.DType; +pub const FinishError = fly_builder.FinishError; +pub const TiledCopy = fly_builder.TiledCopy; +pub const TiledMma = fly_builder.TiledMma; +pub const Arch = fly_builder.Arch; +pub const MmaFlavor = fly_builder.MmaFlavor; +pub const fly_rocdl = fly_builder.fly_rocdl; +pub const layout = fly_builder.layout; +pub const Layout = fly_builder.Layout; +pub const Tile = fly_builder.Tile; +pub const IntTuple = fly_builder.IntTuple; +pub const L = fly_builder.L; +pub const rowMajor = fly_builder.rowMajor; +pub const colMajor = fly_builder.colMajor; +pub const ordered = fly_builder.ordered; +pub const tile = fly_builder.tile; +pub const it = fly_builder.it; +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +/// Threads per wavefront: 64 on CDNA, 32 on RDNA. +pub fn waveSize(p: *const platform_.Platform) i32 { + const cc = platform_.rocm.computeCapability(p) orelse return 64; + return switch (cc.architecture()) { + .cdna1, .cdna2, .cdna3, .cdna4 => 64, + .rdna2, .rdna3, .rdna3_5, .rdna4 => 32, + }; +} + +/// The plugin sizes a block in wavefronts, so a thread count has to divide +/// evenly: a kernel's layouts are built for an exact number of threads and +/// spare ones would partition out of range. +fn warpsFor(p: *const platform_.Platform, threads: i32) i32 { + const wave = waveSize(p); + if (@rem(threads, wave) != 0) { + std.debug.panic("zml.kernel.fly: {d} threads is not a multiple of the {d}-wide wavefront", .{ threads, wave }); + } + return @divExact(threads, wave); +} + +/// The matrix atom this device provides, or null when it has none: WMMA +/// arrived with RDNA3, so gfx1030 has no matrix instruction at all. +pub fn mmaFlavor(p: *const platform_.Platform) ?MmaFlavor { + const cc = platform_.rocm.computeCapability(p) orelse return .cdna3_mfma; + return switch (cc.architecture()) { + .cdna1, .cdna2, .cdna3, .cdna4 => .cdna3_mfma, + .rdna3, .rdna3_5 => .gfx11_wmma, + .rdna4 => .gfx120x_wmma, + .rdna2 => null, + }; +} +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(fly_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3fnuz => .f8e4m3fnuz, + .f8e5m2 => .f8e5m2, + .f8e5m2fnuz => .f8e5m2fnuz, + else => std.debug.panic("zml.kernel.fly.from: dtype {s} has no Fly equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + pub const arg_names = spec.inputs ++ spec.outputs; + /// By name, as `!fly.memref` views. + pub const Args = StructOf(arg_names, Value); + + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + /// Threads per block. Converted to the wavefronts the plugin + /// wants using the device's wave size, which is 64 on CDNA and + /// 32 on RDNA, so a kernel states the thread count its layouts + /// were built for and runs on both. + threads: i32, + /// > 0 sets `amdgpu-waves-per-eu = "N, N"`: a hard occupancy clamp. + waves_per_eu: i32 = 0, + /// Required when the body uses `fly.get_dyn_shared`. + shared_mem_bytes: i64 = 0, + /// Inputs first, then outputs; see `zeroedOutput`. A no-op + /// inside command buffers. + zeroed_args: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + /// The arguments `run` was declared with. + pub fn args(b: *Builder) Args { + var out: Args = undefined; + inline for (arg_names, 0..) |fname, i| @field(out, fname) = b.arg(i); + return out; + } + + /// `zeroed_args` index of an output. + pub fn zeroedOutput(comptime f: std.meta.FieldEnum(Outputs)) i32 { + return @intCast(spec.inputs.len + @backingInt(f)); + } + + /// `shapes` is inputs, then outputs. + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, shapes: []const Shape) ![:0]const u8 { + std.debug.assert(shapes.len == arg_names.len); + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try fly_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + var specs: [arg_names.len]Builder.ArgSpec = undefined; + inline for (arg_names, 0..) |fname, i| { + const shape = shapes[i]; + specs[i] = .{ + .name = fname, + .dtype = from(shape.dtype()), + .dims = if (shape.rank() > 0) shape.dims() else null, + }; + } + try b.declareArgsLow(&specs); + + try spec.run(&b, cfg); + + return b.finish(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + var shapes: [arg_names.len]Shape = undefined; + inline for (0..spec.inputs.len) |i| shapes[i] = inputs_arr[i].shape(); + inline for (0..spec.outputs.len) |i| shapes[spec.inputs.len + i] = outputs_arr[i]; + + const ir = emit(cur.allocator, opts.cfg, &shapes) catch |err| + std.debug.panic("zml.kernel.fly.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.fly(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .num_warps = warpsFor(cur.platform, opts.threads), + .waves_per_eu = opts.waves_per_eu, + .shared_mem_bytes = opts.shared_mem_bytes, + .zeroed_args = opts.zeroed_args, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/fly/test.zig b/zml/kernel/fly/test.zig new file mode 100644 index 00000000..ae788d96 --- /dev/null +++ b/zml/kernel/fly/test.zig @@ -0,0 +1,149 @@ +const std = @import("std"); + +const fly_builder = @import("dsl/fly/builder"); + +const fly = @import("fly.zig"); +const zml = @import("../../zml.zig"); + +/// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the +/// ragged edge. +const FlyVectorAdd = struct { + const Cfg = struct { + thr: fly.Layout = fly.ordered(.{ 8, 16 }, .{ 1, 0 }), + val: fly.Layout = fly.ordered(.{ 1, 4 }, .{ 0, 1 }), + }; + + const K = fly.Kernel(Cfg, .{ + .name = "vector_add", + .inputs = &.{ "a", "b" }, + .outputs = &.{"c"}, + .run = run, + }); + + fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { + const a = K.args(b); + fly_builder.emitVectorAdd(b, cfg.thr, cfg.val, a.a, a.b, a.c); + } +}; + +/// C = A @ B^T with one MFMA-tiled block. +const FlyTiledMma = struct { + const Cfg = struct { flavor: fly.MmaFlavor }; + const K = fly.Kernel(Cfg, .{ + .name = "tiled_mma", + .inputs = &.{ "a", "b" }, + .outputs = &.{"c"}, + .run = run, + }); + + fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { + const a = K.args(b); + fly_builder.emitTiledMma(b, cfg.flavor, a.a, a.b, a.c); + } +}; + +test "fly kernel emits a module XLA can parse" { + const shapes = [_]zml.Shape{ .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32) }; + const ir = try FlyVectorAdd.K.emit(std.testing.allocator, .{}, &shapes); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.func @vector_add(%arg0: !fly.ptr, %arg1: !fly.ptr, %arg2: !fly.ptr) kernel") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "fly.static : !fly.tile<[8|64]>") != null); +} + +test "fly kernels run on rocm" { + const platform = zml.testing.env(); + if (platform.target != .rocm) return error.SkipZigTest; + + const allocator = std.testing.allocator; + + // Compile `forward`, upload both inputs, run, and bring the result back. + const call = struct { + fn f(comptime forward: anytype, p: *const zml.Platform, ta: zml.Tensor, ha: anytype, tb: zml.Tensor, hb: anytype) !zml.Slice { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ ta, tb }, p, .{}); + defer exe.deinit(); + var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), std.mem.sliceAsBytes(ha)); + defer ba.deinit(); + var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), std.mem.sliceAsBytes(hb)); + defer bb.deinit(); + var out = try exe.eval(std.testing.allocator, std.testing.io, .{ ba, bb }); + defer out.deinit(); + return out.toSliceAlloc(std.testing.allocator, std.testing.io); + } + }.f; + + // vector add: elementwise over a ragged (100, 1000), predicated at the edge. + { + const M, const N = .{ 100, 1000 }; + const Mod = struct { + pub fn forward(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + return FlyVectorAdd.K.call(.{ .a = a, .b = b }, .{ .c = a.shape() }, .{ + .cfg = .{}, + .grid = .{ (M + 8 - 1) / 8, (N + 64 - 1) / 64, 1 }, + .threads = 8 * 16, + }).c; + } + }; + + const ha = try allocator.alloc(f32, M * N); + defer allocator.free(ha); + const hb = try allocator.alloc(f32, M * N); + defer allocator.free(hb); + for (ha, hb, 0..) |*va, *vb, i| { + va.* = @floatFromInt(i % 977); + vb.* = @floatFromInt(1000 + (i % 313)); + } + + var host = try call(Mod.forward, platform, .init(.{ M, N }, .f32), ha, .init(.{ M, N }, .f32), hb); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(ha[i] + hb[i], v); + } + + // tiled mma: C = A @ B^T in one block of 256 threads, over whatever matrix + // atom this device has. CDNA multiplies f32 through MFMA; RDNA3+ has WMMA, + // whose verifier rejects f32 operands, so the operand type and K come from + // the flavour rather than being written into the test. + if (fly.mmaFlavor(platform)) |flavor| switch (flavor) { + inline else => |f| { + const Elem = switch (comptime f.operandDType()) { + .f32 => f32, + .f16 => f16, + else => @compileError("unhandled MMA operand type"), + }; + const M, const N = .{ 64, 64 }; + const Kd = comptime f.blockK(); + const operand_dtype: zml.DataType = switch (comptime f.operandDType()) { + .f32 => .f32, + .f16 => .f16, + else => unreachable, + }; + + const Mod = struct { + pub fn forward(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + return FlyTiledMma.K.call(.{ .a = a, .b = b }, .{ .c = zml.Shape.init(.{ M, N }, .f32) }, .{ + .cfg = .{ .flavor = f }, + .grid = .{ 1, 1, 1 }, + .threads = comptime f.blockThreads(), + }).c; + } + }; + + var ha: [M * Kd]Elem = undefined; + var hb: [N * Kd]Elem = undefined; + for (&ha, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 7) % 13)) - 6; + for (&hb, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 5) % 11)) - 5; + + var host = try call(Mod.forward, platform, .init(.{ M, Kd }, operand_dtype), &ha, .init(.{ N, Kd }, operand_dtype), &hb); + defer host.free(allocator); + + // f16 operands accumulate in f32, so the error is set by the input + // rounding rather than the sum. + const tol: f32 = if (Elem == f16) 1e-2 else 1e-3; + const out = host.items(f32); + for (0..M) |m| for (0..N) |n| { + var acc: f32 = 0; + for (0..Kd) |k| acc += @as(f32, ha[m * Kd + k]) * @as(f32, hb[n * Kd + k]); + try std.testing.expectApproxEqAbs(acc, out[m * N + n], tol); + }; + }, + }; +} diff --git a/zml/kernel/kernel.zig b/zml/kernel/kernel.zig new file mode 100644 index 00000000..204c6204 --- /dev/null +++ b/zml/kernel/kernel.zig @@ -0,0 +1,12 @@ +pub const cuda_tile = @import("cuda_tile/cuda_tile.zig"); +pub const cute = @import("cute/cute.zig"); +pub const fly = @import("fly/fly.zig"); +pub const metal = @import("metal/metal.zig"); +pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); +pub const tcl = @import("tcl/tcl.zig"); +pub const triton = @import("triton/triton.zig"); + +test { + const std = @import("std"); + std.testing.refAllDecls(@This()); +} diff --git a/zml/kernel/metal/BUILD.bazel b/zml/kernel/metal/BUILD.bazel new file mode 100644 index 00000000..4007beb4 --- /dev/null +++ b/zml/kernel/metal/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "metal", + srcs = [ + "metal.zig", + "test.zig", + ], +) diff --git a/zml/kernel/metal/metal.zig b/zml/kernel/metal/metal.zig new file mode 100644 index 00000000..9626a1de --- /dev/null +++ b/zml/kernel/metal/metal.zig @@ -0,0 +1,237 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const metal_builder = @import("dsl/metal/builder"); +pub const Builder = metal_builder.Builder; +pub const Value = metal_builder.Value; +pub const Type = metal_builder.Type; +pub const Struct = metal_builder.Struct; +pub const ArgSpec = metal_builder.ArgSpec; +pub const LoopOpts = metal_builder.LoopOpts; +pub const FinishError = metal_builder.FinishError; +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(metal_builder.registerDialects); +} + +pub fn from(dt: DataType) Type { + return switch (dt) { + .bool => .bool, + .i8 => .char, + .i16 => .short, + .i32 => .int, + .i64 => .long, + .u8 => .uchar, + .u16 => .ushort, + .u32 => .uint, + .u64 => .ulong, + .f16 => .half, + .bf16 => .bfloat, + .f32 => .float, + else => std.debug.panic("zml.kernel.metal.from: dtype {s} has no MSL equivalent", .{@tagName(dt)}), + }; +} + +/// The bytes of a `constant T &` argument: `.of(@as(i32, n))`, or a host +/// `extern struct` laid out like a `constant S &` argument. +pub const Constant = struct { + buffer: [256]u8 = undefined, + len: usize = 0, + + pub fn of(value: anytype) Constant { + const raw = std.mem.asBytes(&value); + var c: Constant = .{ .len = raw.len }; + @memcpy(c.buffer[0..raw.len], raw); + return c; + } + + pub fn bytes(self: *const Constant) []const u8 { + return self.buffer[0..self.len]; + } +}; + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + /// The arguments bound to input tensors. + inputs: []const [:0]const u8, + /// The arguments bound to output tensors; one named like an + /// input updates it in place. + outputs: []const [:0]const u8, + /// The `constant` arguments given by value at the call. + constants: []const [:0]const u8 = &.{}, + run: *const fn (*Builder, Config) FinishError!void, + fast_math: bool = true, + }; +} + +/// A Metal kernel: `run` declares its arguments with `declareArgs`, by +/// name, and builds its body. +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + pub const Constants = StructOf(spec.constants, Constant); + + pub const CallOpts = struct { + cfg: ConfigT, + /// Threadgroups per grid. + grid: [3]u32, + /// Threads per threadgroup. + threadgroup: [3]u32, + /// Bytes of `[[threadgroup(0)]]` memory. + threadgroup_memory: u64 = 0, + constants: Constants = std.mem.zeroes(Constants), + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + var arguments: std.ArrayList(metal_builder.Argument) = .empty; + defer arguments.deinit(allocator); + return build(allocator, cfg, &arguments); + } + + fn build(allocator: std.mem.Allocator, cfg: ConfigT, arguments: *std.ArrayList(metal_builder.Argument)) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + var b = Builder.open(allocator, ctx, name, .{ .fast_math = spec.fast_math }); + defer b.deinit(); + try spec.run(&b, cfg); + const ir = try b.finish(); + errdefer allocator.free(ir); + try arguments.appendSlice(allocator, b.arguments.items); + return ir; + } + + fn indexOf(names: []const [:0]const u8, arg: []const u8) ?usize { + for (names, 0..) |n, i| if (std.mem.eql(u8, n, arg)) return i; + return null; + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + const allocator = cur.allocator; + + var arguments: std.ArrayList(metal_builder.Argument) = .empty; + defer arguments.deinit(allocator); + const ir = build(allocator, opts.cfg, &arguments) catch |err| + std.debug.panic("zml.kernel.metal.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |field, i| inputs_arr[i] = @field(inputs, field); + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |field, i| outputs_arr[i] = @field(outputs, field); + var constants: [spec.constants.len][]const u8 = undefined; + inline for (spec.constants, 0..) |field, i| constants[i] = @field(opts.constants, field).bytes(); + + var bound: [spec.outputs.len]bool = @splat(false); + var bound_inputs: [spec.inputs.len]bool = @splat(false); + var bound_constants: [spec.constants.len]bool = @splat(false); + var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; + var bindings: std.ArrayList(i32) = .empty; + defer bindings.deinit(allocator); + for (arguments.items) |arg| { + if (arg.kind == .threadgroup) { + // `threadgroup_memory` sizes `[[threadgroup(0)]]` alone. + if (arg.index != 0) std.debug.panic("zml.kernel.metal.Kernel({s}): threadgroup argument {s} is [[threadgroup({d})]], and only [[threadgroup(0)]] gets memory", .{ name, arg.name, arg.index }); + continue; + } + std.debug.assert(arg.index == bindings.items.len); + const input = indexOf(spec.inputs, arg.name); + const output = indexOf(spec.outputs, arg.name); + if (input) |i| bound_inputs[i] = true; + if (output) |o| bound[o] = true; + const binding: i32 = if (input) |i| blk: { + if (output) |o| aliases.appendAssumeCapacity(.{ .output_index = @intCast(o), .operand_index = @intCast(i) }); + break :blk @intCast(i); + } else if (output) |o| + @intCast(spec.inputs.len + o) + else if (indexOf(spec.constants, arg.name)) |k| blk: { + if (constants[k].len != arg.size) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} has {d} bytes, the argument {d}", .{ name, arg.name, constants[k].len, arg.size }); + bound_constants[k] = true; + break :blk -1 - @as(i32, @intCast(k)); + } else std.debug.panic("zml.kernel.metal.Kernel({s}): argument {s} is no input, output or constant", .{ name, arg.name }); + bindings.append(allocator, binding) catch @panic("OOM"); + } + for (bound, spec.outputs) |b, output| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): output {s} is no argument", .{ name, output }); + for (bound_inputs, spec.inputs) |b, input| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): input {s} is no argument", .{ name, input }); + for (bound_constants, spec.constants) |b, constant| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} is no argument", .{ name, constant }); + + const tensor_results = ops.metal(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .threadgroup = opts.threadgroup, + .threadgroup_memory = opts.threadgroup_memory, + .arguments = bindings.items, + .constants = &constants, + .output_operand_aliases = aliases.constSlice(), + }); + var results: Results = undefined; + inline for (spec.outputs, 0..) |field, i| @field(results, field) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/metal/test.zig b/zml/kernel/metal/test.zig new file mode 100644 index 00000000..0a342af0 --- /dev/null +++ b/zml/kernel/metal/test.zig @@ -0,0 +1,71 @@ +const std = @import("std"); + +const metal = @import("metal.zig"); +const zml = @import("../../zml.zig"); + +/// `out[i] = a[i] * scale + b[i]` over `n` floats. +const MetalAxpy = struct { + const Cfg = struct {}; + const K = metal.Kernel(Cfg, .{ + .name = "axpy", + .inputs = &.{ "a", "b" }, + .outputs = &.{"out"}, + .constants = &.{ "scale", "n" }, + .run = run, + }); + + fn run(b: *metal.Builder, _: Cfg) metal.FinishError!void { + const a = try b.declareArgs(.{ + .a = .{ .buffer = .{ .type = .float, .const_ = true } }, + .b = .{ .buffer = .{ .type = .float, .const_ = true } }, + .out = .{ .buffer = .{ .type = .float } }, + .scale = .{ .constant = .float }, + .n = .{ .constant = .uint }, + .tid = .{ .thread_position_in_grid = .uint }, + }); + b.returnIf(a.tid.ge(a.n)); + a.out.store(a.tid, b.fma(a.a.load(a.tid), a.scale, a.b.load(a.tid))); + } +}; + +test "metal kernels run on metal" { + const allocator = std.testing.allocator; + + const ir = try MetalAxpy.K.emit(allocator, .{}); + defer allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "air.fma") != null); + + const platform = zml.testing.env(); + if (platform.target != .metal) return error.SkipZigTest; + + const n = 1000; + const Mod = struct { + pub fn forward(x: zml.Tensor, y: zml.Tensor) zml.Tensor { + return MetalAxpy.K.call(.{ .a = x, .b = y }, .{ .out = x.shape() }, .{ + .cfg = .{}, + .grid = .{ (n + 255) / 256, 1, 1 }, + .threadgroup = .{ 256, 1, 1 }, + .constants = .{ .scale = .of(@as(f32, 0.5)), .n = .of(@as(u32, n)) }, + }).out; + } + }; + const x: zml.Tensor = .init(.{n}, .f32); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, zml.Tensor.init(.{n}, .f32) }, platform, .{}); + defer exe.deinit(); + + var hx: [n]f32 = undefined; + var hy: [n]f32 = undefined; + for (&hx, &hy, 0..) |*a, *b, i| { + a.* = @floatFromInt(i); + b.* = 1; + } + var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hx)); + defer bx.deinit(); + var by: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hy)); + defer by.deinit(); + var result = try exe.eval(allocator, std.testing.io, .{ bx, by }); + defer result.deinit(); + var host = try result.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + for (host.items(f32), hx, hy) |out, a, b| try std.testing.expectEqual(a * 0.5 + b, out); +} diff --git a/zml/kernel/mosaic_tpu/BUILD.bazel b/zml/kernel/mosaic_tpu/BUILD.bazel new file mode 100644 index 00000000..ba3b6b66 --- /dev/null +++ b/zml/kernel/mosaic_tpu/BUILD.bazel @@ -0,0 +1,6 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "mosaic_tpu", + srcs = ["mosaic_tpu.zig"], +) diff --git a/zml/kernel/mosaic_tpu/mosaic_tpu.zig b/zml/kernel/mosaic_tpu/mosaic_tpu.zig new file mode 100644 index 00000000..a09e8368 --- /dev/null +++ b/zml/kernel/mosaic_tpu/mosaic_tpu.zig @@ -0,0 +1,284 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const mosaic_tpu_builder = @import("dsl/mosaic_tpu/builder"); +const Builder = mosaic_tpu_builder.Builder; +const DType = mosaic_tpu_builder.DType; +const FinishError = mosaic_tpu_builder.FinishError; +const mosaic_tpu_dialect = @import("mlir/dialects/mosaic_tpu"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(mosaic_tpu_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + else => std.debug.panic("zml.kernel.mosaic_tpu.from: dtype {s} has no Mosaic-TPU equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallExtras = struct { + vmem_limit_bytes: ?i64 = null, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + has_communication: ?bool = null, + additional_attributes: []const mlir.NamedAttribute = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + dynamic_grid_bounds: []const Tensor = &.{}, + }; + + pub const CallOpts = struct { + cfg: ConfigT, + extras: CallExtras = .{}, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try mosaic_tpu_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finishOpts(&.{}, .{ .canonicalize = true }); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + const backend_config = buildBackendConfig(cur.allocator, ir, .{ + .vmem_limit_bytes = opts.extras.vmem_limit_bytes, + .disable_bounds_checks = opts.extras.disable_bounds_checks, + .disable_semaphore_checks = opts.extras.disable_semaphore_checks, + .has_communication = opts.extras.has_communication, + }) catch |err| + std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: backend_config build failed: {}", .{ name, err }); + defer cur.allocator.free(backend_config); + + var values: [spec.inputs.len]*const mlir.Value = undefined; + inline for (spec.inputs, 0..) |fname, i| values[i] = @field(inputs, fname).value(); + + var res_types: [spec.outputs.len]*const mlir.Type = undefined; + inline for (&res_types, spec.outputs) |*r, fname| { + r.* = mlirx.Type.rankedTensor(cur.mlir_ctx, @field(outputs, fname)); + } + + const aliases = resolveOutputOperandAliases(opts.extras.output_operand_aliases, @intCast(opts.extras.dynamic_grid_bounds.len)); + + const op = callTpuCustomCall(.{ + .inputs = &values, + .result_types = &res_types, + .backend_config = backend_config, + .aliases = aliases.constSlice(), + .dynamic_grid_bounds = opts.extras.dynamic_grid_bounds, + .additional_attributes = opts.extras.additional_attributes, + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| { + const out = @field(outputs, fname); + @field(results, fname) = Tensor._result(out, op.result(i)); + } + return results; + } + }; +} + +const CustomCallConfig = struct { + body: []const u8, + has_communication: ?bool = null, + serialization_format: i64 = 1, + needs_layout_passes: bool = true, + shape_invariant_numerics: bool = false, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, +}; + +const ScopedMemoryConfig = struct { + memory_space: i64 = 1, + offset: i64 = 0, + size: i64, +}; + +const BackendConfig = struct { + custom_call_config: CustomCallConfig, + scoped_memory_configs: ?[]const ScopedMemoryConfig = null, +}; + +const BackendConfigOpts = struct { + vmem_limit_bytes: ?i64 = null, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + has_communication: ?bool = null, +}; + +fn buildBackendConfig( + allocator: std.mem.Allocator, + ir: [:0]const u8, + opts: BackendConfigOpts, +) ![]u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + const module = try mlir.Module.parse(ctx, ir); + defer module.deinit(); + + mosaic_tpu_dialect.registerMosaicSerdePass(); + const prev_allow_unregistered = ctx.allowUnregisteredDialects(); + ctx.setAllowUnregisteredDialects(true); + defer ctx.setAllowUnregisteredDialects(prev_allow_unregistered); + + const pm = mlir.PassManager.init(ctx); + defer pm.deinit(); + try pm.asOpPassManager().addPipeline("mosaic-serde{serialize=true}"); + try pm.runOnOp(module.operation()); + + const Encoder = std.base64.standard.Encoder; + const b64 = b64: { + var bytecode: std.Io.Writer.Allocating = .init(allocator); + defer bytecode.deinit(); + try module.operation().writeBytecode(.{ .desired_emit_version = 0 }, &bytecode.writer); + const buf = try allocator.alloc(u8, Encoder.calcSize(bytecode.written().len)); + break :b64 Encoder.encode(buf, bytecode.written()); + }; + defer allocator.free(b64); + + const config = BackendConfig{ + .custom_call_config = .{ + .body = b64, + .has_communication = opts.has_communication, + .disable_bounds_checks = opts.disable_bounds_checks, + .disable_semaphore_checks = opts.disable_semaphore_checks, + }, + .scoped_memory_configs = if (opts.vmem_limit_bytes) |vmem| + &.{.{ .size = vmem }} + else + null, + }; + + var json: std.Io.Writer.Allocating = .init(allocator); + errdefer json.deinit(); + + try json.writer.print("{f}", .{std.json.fmt(config, .{ .emit_null_optional_fields = false })}); + return try json.toOwnedSlice(); +} + +const TpuCustomCallArgs = struct { + inputs: []const *const mlir.Value, + result_types: []const *const mlir.Type, + backend_config: []const u8, + aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias, + dynamic_grid_bounds: []const Tensor, + additional_attributes: []const mlir.NamedAttribute, +}; + +fn callTpuCustomCall(args: TpuCustomCallArgs) *mlir.Operation { + const cur = Compiler.current(); + + var all_inputs: stdx.BoundedArray(*const mlir.Value, stablehlo.CustomCallOpts.MAX_OPERANDS) = .empty; + for (args.dynamic_grid_bounds) |t| all_inputs.appendAssumeCapacity(t.value()); + all_inputs.appendSliceAssumeCapacity(args.inputs); + + return stablehlo.custom_call( + cur.mlir_ctx, + all_inputs.constSlice(), + args.result_types, + .{ + .call_target_name = "tpu_custom_call", + .backend_config = .{ .original = args.backend_config }, + .has_side_effect = false, + .api_version = .original, + .additional_attributes = args.additional_attributes, + .output_operand_aliases = args.aliases, + }, + cur.location, + ).appendTo(cur.currentScope().block); +} diff --git a/zml/kernel/tcl/BUILD.bazel b/zml/kernel/tcl/BUILD.bazel new file mode 100644 index 00000000..bc7a0aa2 --- /dev/null +++ b/zml/kernel/tcl/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "tcl", + srcs = [ + "tcl.zig", + "test.zig", + ], +) diff --git a/zml/kernel/tcl/tcl.zig b/zml/kernel/tcl/tcl.zig new file mode 100644 index 00000000..023e862f --- /dev/null +++ b/zml/kernel/tcl/tcl.zig @@ -0,0 +1,206 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const metal_builder = @import("dsl/metal/builder"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); +const tcl_builder = @import("dsl/tcl/builder"); +pub const Builder = tcl_builder.Builder; +pub const TensorOperation = tcl_builder.TensorOperation; +pub const Value = tcl_builder.Value; +pub const Axis = tcl_builder.Axis; +pub const Expr = tcl_builder.Expr; +pub const Dram = tcl_builder.Dram; +pub const Context = tcl_builder.Context; +pub const DType = tcl_builder.DType; +pub const VeOp = tcl_builder.VeOp; +pub const ReduceMode = tcl_builder.ReduceMode; +pub const Predicate = tcl_builder.Predicate; +pub const Tactic = tcl_builder.Tactic; +pub const FinishError = tcl_builder.FinishError; + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(tcl_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .bool, + .i4 => .i4, + .i8 => .i8, + .i16 => .i16, + .i32 => .i32, + .i64 => .i64, + .u8 => .u8, + .u16 => .u16, + .u32 => .u32, + .u64 => .u64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f4e2m1 => .f4_e2, + .f8e4m3fn => .f8_e4, + .f8e5m2 => .f8_e5, + else => std.debug.panic("zml.kernel.tcl.from: dtype {s} has no TCL equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +/// A TCL function compiled by the Furiosa plugin as one native unit: +/// `declareArgs` declares the inputs in order, `ret` the outputs. +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + return build(allocator, cfg, null, null, null); + } + + /// `updated`, when given, receives the input each output updates + /// in place (`Builder.update`), or null. + fn build(allocator: std.mem.Allocator, cfg: ConfigT, inputs: ?[]const Shape, outputs: ?[]const Shape, updated: ?*[spec.outputs.len]?usize) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try Builder.open(allocator, ctx, name); + defer b.deinit(); + try spec.run(&b, cfg); + + const results = b.results orelse std.debug.panic("zml.kernel.tcl.Kernel({s}): run did not call ret", .{name}); + if (b.args.len != spec.inputs.len or results.len != spec.outputs.len) { + std.debug.panic("zml.kernel.tcl.Kernel({s}): {d} arguments and {d} results for {d} inputs and {d} outputs", .{ name, b.args.len, results.len, spec.inputs.len, spec.outputs.len }); + } + + if (inputs) |shapes| for (b.args, shapes, spec.inputs) |t, s, n| check(t, s, n); + if (outputs) |shapes| for (results, shapes, spec.outputs) |t, s, n| check(t, s, n); + if (updated) |u| for (u, results) |*dst, r| { + dst.* = b.updatedArgument(r); + }; + + return b.finish(); + } + + fn check(t: tcl_builder.Tensor, s: Shape, field: []const u8) void { + var same = from(s.dtype()) == t.dtype; + if (t.shards > 1) { + // A partition holds one chip's share of a chip-split DRAM tensor. + var n: i64 = 1; + for (t.axes) |a| n *= a.size; + same = same and n == @as(i64, @intCast(s.count())) * t.shards; + } else { + same = same and s.rank() == t.axes.len; + if (same) for (t.axes, 0..) |a, i| { + same = same and a.size == s.dim(i); + }; + } + + if (!same) std.debug.panic("zml.kernel.tcl.Kernel({s}): {s} is {f}, the TCL function has {s} over {d} axes", .{ name, field, s, @tagName(t.dtype), t.axes.len }); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + var input_shapes: [spec.inputs.len]Shape = undefined; + inline for (spec.inputs, 0..) |fname, i| { + inputs_arr[i] = @field(inputs, fname); + input_shapes[i] = inputs_arr[i].shape(); + } + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + var updated: [spec.outputs.len]?usize = undefined; + const ir = build(cur.allocator, opts.cfg, &input_shapes, &outputs_arr, &updated) catch |err| + std.debug.panic("zml.kernel.tcl.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; + for (updated, 0..) |input, output| if (input) |i| aliases.appendAssumeCapacity(.{ + .output_index = @intCast(output), + .operand_index = @intCast(i), + }); + const tensor_results = ops.tcl(inputs_arr, outputs_arr, .{ .ir = ir, .output_operand_aliases = aliases.constSlice() }); + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| { + @field(results, fname) = tensor_results[i]; + } + + return results; + } + }; +} diff --git a/zml/kernel/tcl/test.zig b/zml/kernel/tcl/test.zig new file mode 100644 index 00000000..b9ae1875 --- /dev/null +++ b/zml/kernel/tcl/test.zig @@ -0,0 +1,123 @@ +const std = @import("std"); + +const zml = @import("../../zml.zig"); +const tcl = @import("tcl.zig"); + +/// `furiosa.tcl.examples.softmax` over f32. +const TclSoftmax = struct { + const Cfg = struct { b: i64, s: i64, d: i64 }; + const K = tcl.Kernel(Cfg, .{ + .name = "softmax_kernel", + .inputs = &.{"x"}, + .outputs = &.{"y"}, + .run = run, + }); + + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + const B = b.axis("B", cfg.b); + const S = b.axis("S", cfg.s); + const D = b.axis("D", cfg.d); + const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ B, S, D } } }); + + const op = b.tensorOperation(.{}); + const x = op.fetch(t.x, .{}); + const e = x.subf(x.reduce(&.{D}, .maxf)).exp(); + b.ret(&.{try op.commit(e.divf(e.reduce(&.{D}, .addf)), .{})}); + } +}; + +test "tcl kernels run on furiosa" { + const allocator = std.testing.allocator; + const cfg: TclSoftmax.Cfg = .{ .b = 2, .s = 16, .d = 256 }; + + const ir = try TclSoftmax.K.emit(allocator, cfg); + defer allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "func.func @softmax_kernel(") != null); + + const platform = zml.testing.env(); + if (platform.target != .furiosa) return error.SkipZigTest; + + const Mod = struct { + pub fn forward(x: zml.Tensor) zml.Tensor { + return TclSoftmax.K.call(.{ .x = x }, .{ .y = x.shape() }, .{ .cfg = cfg }).y; + } + }; + const x: zml.Tensor = .init(.{ cfg.b, cfg.s, cfg.d }, .f32); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{}); + defer exe.deinit(); + + var input: [cfg.b * cfg.s * cfg.d]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 37)) * 0.25 - 4; + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&input)); + defer x_buffer.deinit(); + var result = try exe.eval(allocator, std.testing.io, .{x_buffer}); + defer result.deinit(); + var host = try result.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + + const out = host.items(f32); + var row: usize = 0; + while (row < cfg.b * cfg.s) : (row += 1) { + const in = input[row * cfg.d ..][0..cfg.d]; + var max: f32 = in[0]; + for (in) |v| max = @max(max, v); + var sum: f32 = 0; + for (in) |v| sum += @exp(v - max); + for (in, out[row * cfg.d ..][0..cfg.d]) |v, y| try std.testing.expectApproxEqAbs(@exp(v - max) / sum, y, 1e-4); + } +} + +/// exp over a tensor split across two chips in one launch, returning the +/// split result and its all_gather. +const TclSplitExp = struct { + const Cfg = struct {}; + const K = tcl.Kernel(Cfg, .{ .name = "split_exp", .inputs = &.{"x"}, .outputs = &.{ "y", "all" }, .run = run }); + fn run(b: *tcl.Builder, _: Cfg) tcl.FinishError!void { + const C = b.axis("C", 2); + const A = b.axis("A", 1024); + const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ C, A }, .dram = tcl.Dram{ .chip = &.{.{ .axis = C }} } } }); + const op = b.tensorOperation(.{ .context = .{ .layout = .{ .chip = .{ .axis = C } } } }); + const local = try op.commit(op.fetch(t.x, .{}).exp(), .{}); + const all = try b.allGather(local, C, .{}); + b.ret(&.{ try b.asDram(local, .{ .chip = &.{.{ .axis = C }} }), try b.asDram(all, .{}) }); + } + fn body(ctx: struct { x: zml.Tensor }, shapes: [2]zml.Shape) [2]zml.Tensor { + const r = K.call(.{ .x = ctx.x }, .{ .y = shapes[0], .all = shapes[1] }, .{ .cfg = .{} }); + return .{ r.y, r.all }; + } +}; + +// Needs FURIOSA_VISIBLE_DEVICES=0,1. +test "tcl multi-chip kernels run on two furiosa chips" { + const allocator = std.testing.allocator; + const platform = zml.testing.env(); + if (platform.target != .furiosa or platform.devices.len != 2) return error.SkipZigTest; + const tp = zml.testing.tpMesh(); + const x_shape = zml.Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(tp, .{ .c = .model }); + const Mod = struct { + pub fn forward(x: zml.Tensor, mesh: *const zml.Sharding.Mesh) [2]zml.Tensor { + const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), zml.Shape.init(.{ .c = 2, .a = 1024 }, .f32) }, mesh, .{ + .in_specs = .{ .x = .parse(.{ .c = .model }) }, + .out_specs = .{ .parse(.{ .c = .model }), .replicated }, + }); + return .{ r[0], r[1] }; + } + }; + const x: zml.Tensor = .fromShape(x_shape); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, tp }, platform, .{ .meshes = &.{tp} }); + defer exe.deinit(); + var hx: [2 * 1024]f32 = undefined; + for (&hx, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 13)) * 0.125 - 0.5; + var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x_shape, std.mem.sliceAsBytes(&hx)); + defer bx.deinit(); + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + var res: [2]zml.Buffer = undefined; + runner.run(std.testing.io, .{bx}, .{ &res[0], &res[1] }, .{ .wait = true }); + defer for (&res) |*r| r.deinit(); + for (res) |r| { + var host = try r.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectApproxEqRel(@exp(hx[i]), v, 1e-4); + } +} diff --git a/zml/kernel/triton/BUILD.bazel b/zml/kernel/triton/BUILD.bazel new file mode 100644 index 00000000..1e992352 --- /dev/null +++ b/zml/kernel/triton/BUILD.bazel @@ -0,0 +1,6 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "triton", + srcs = ["triton.zig"], +) diff --git a/zml/kernel/triton/triton.zig b/zml/kernel/triton/triton.zig new file mode 100644 index 00000000..6a91a03f --- /dev/null +++ b/zml/kernel/triton/triton.zig @@ -0,0 +1,165 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); +const triton_builder = @import("dsl/triton/builder"); +pub const Builder = triton_builder.Builder; +pub const Value = triton_builder.Value; +pub const DType = triton_builder.DType; +pub const FinishError = triton_builder.FinishError; + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const ops = @import("../../ops.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(triton_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3fnuz => .f8e4m3fnuz, + .f8e5m2 => .f8e5m2, + else => std.debug.panic("zml.kernel.triton.from: dtype {s} has no Triton equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + num_stages: i32, + num_warps: i32, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + debug: bool = false, + /// Bytes for Triton's implicit global scratch argument, across all CTAs. + global_scratch_memory_size: i32 = 0, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try triton_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finish(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ttir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.triton.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ttir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const call_opts: ops.TritonOps = .{ + .debug = opts.debug, + .name = name, + .ir = ttir, + .grid = opts.grid, + .num_stages = opts.num_stages, + .num_warps = opts.num_warps, + .output_operand_aliases = aliases.constSlice(), + .is_tma_allowed = opts.global_scratch_memory_size > 0, + .global_scratch_memory_size = opts.global_scratch_memory_size, + }; + + const triton_results = if (opts.global_scratch_memory_size > 0) blk: { + // XLA maps the final custom-call result to Triton's implicit + // scratch pointer; it must not appear in the TTIR signature. + const extended = outputs_arr ++ [_]Shape{.init(.{opts.global_scratch_memory_size}, .u8)}; + const res = ops.triton(inputs_arr, extended, call_opts); + break :blk res[0..spec.outputs.len].*; + } else ops.triton(inputs_arr, outputs_arr, call_opts); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = triton_results[i]; + return results; + } + }; +} diff --git a/zml/kernels/flash_attention/metal/flash_attn.zig b/zml/kernels/flash_attention/metal/flash_attn.zig index 4faa1e82..de4b68da 100644 --- a/zml/kernels/flash_attention/metal/flash_attn.zig +++ b/zml/kernels/flash_attention/metal/flash_attn.zig @@ -4,9 +4,8 @@ const std = @import("std"); -const kernel = @import("../../../kernel.zig"); +const metal = @import("../../../kernel/kernel.zig").metal; -const metal = kernel.metal; const ArgSpec = metal.ArgSpec; const Builder = metal.Builder; const Type = metal.Type; diff --git a/zml/kernels/paged_attention/metal/paged_attn.zig b/zml/kernels/paged_attention/metal/paged_attn.zig index 337432e7..04f4c1ac 100644 --- a/zml/kernels/paged_attention/metal/paged_attn.zig +++ b/zml/kernels/paged_attention/metal/paged_attn.zig @@ -3,9 +3,8 @@ const std = @import("std"); -const kernel = @import("../../../kernel.zig"); +const metal = @import("../../../kernel/kernel.zig").metal; -const metal = kernel.metal; const ArgSpec = metal.ArgSpec; const Builder = metal.Builder; const Type = metal.Type; diff --git a/zml/zml.zig b/zml/zml.zig index 35c2f9df..dafc3a0f 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -26,7 +26,7 @@ pub const Exe = exe.Exe; pub const FnExe = exe.FnExe; pub const floats = @import("floats.zig"); pub const io = @import("io.zig"); -pub const kernel = @import("kernel.zig"); +pub const kernel = @import("kernel/kernel.zig"); pub const logo = @import("logo.zig"); pub const mem = @import("mem.zig"); pub const Bufferized = mem.Bufferized; From 4af281a0bed2bb9c6f4de262b36ed635f5663f0a Mon Sep 17 00:00:00 2001 From: erwanvivien <44021072+erwanvivien@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:38:46 +0100 Subject: [PATCH 874/881] stdx/flags: cleanup flag_name using mem.replaceScalar (#1068) Just porting 5253ea1362d04d976734e7fdc6e417422e6e693d from #1065 --- stdx/flags.zig | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/stdx/flags.zig b/stdx/flags.zig index 52a9e6a4..d0c4352d 100644 --- a/stdx/flags.zig +++ b/stdx/flags.zig @@ -591,25 +591,19 @@ fn fields_to_comma_list(comptime E: type) []const u8 { } } +/// Maps a field name to its CLI flag, e.g. `model_path` to `--model-path`. pub fn flag_name(comptime field_name: []const u8) []const u8 { - // TODO(Zig): Cleanup when this is fixed after Zig 0.11. - // Without comptime blk, the compiler thinks the result is a runtime slice returning a UAF. - return comptime blk: { - assert(!std.mem.eql(u8, field_name, "positional")); - - var result: []const u8 = "--"; - var index = 0; - while (std.mem.indexOf(u8, field_name[index..], "_")) |i| { - result = result ++ field_name[index..][0..i] ++ "-"; - index = index + i + 1; - } - result = result ++ field_name[index..]; - break :blk result; - }; + comptime assert(!std.mem.eql(u8, field_name, "positional")); + comptime var name = field_name[0..field_name.len].*; + comptime std.mem.replaceScalar(u8, &name, '_', '-'); + const dashed = name; + return "--" ++ dashed; } test flag_name { - try std.testing.expectEqualStrings(flag_name("statsd"), "--statsd"); + try std.testing.expectEqualStrings("--statsd", flag_name("statsd")); + try std.testing.expectEqualStrings("--model-path", flag_name("model_path")); + try std.testing.expectEqualStrings("--a-b-c", flag_name("a_b_c")); } fn flag_name_positional(comptime field_name: []const u8) []const u8 { From 3ecf1ff68be97b79519a096719666c0d4fcbb54e Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Fri, 13 Nov 2026 17:34:08 +0100 Subject: [PATCH 875/881] zml/moe: mask nonlocal BF16 routes on Furiosa (#1062) Replace nonlocal expert IDs with an in-range placeholder and zero their routing weights before dispatch. This prevents the activation-stationary and blockwise kernels from using -1 in gathers and scatters. Verified BF16 outputs against the CPU reference and coherent Qwen3.5-35B-A3B generation on two and four RNGD cards. --- zml/kernels/moe/tcl/tcl.zig | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig index 6238d5f6..c26fff1b 100644 --- a/zml/kernels/moe/tcl/tcl.zig +++ b/zml/kernels/moe/tcl/tcl.zig @@ -89,10 +89,14 @@ fn runExperts(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: Te const x = input.reshape(.{ cfg.t, cfg.d }); const ids = topk_ids.reshape(.{ cfg.t, cfg.k }); const weights = topk_weights.reshape(.{ cfg.t, cfg.k }); + // These kernels gather every route, and blockwise also scatters each route + // into an expert slot. Give nonlocal routes a valid slot and zero weight. + const valid = ids.cmp(.GE, Tensor.scalar(0, .i32)).logical(.AND, ids.cmp(.LT, Tensor.scalar(cfg.e, .i32))); + const safe_ids = valid.select(ids, Tensor.scalar(0, .i32)); + const safe_weights = valid.select(weights, Tensor.scalar(0, .f32)); const out_shape = x.shape(); if (scales) |s| { - const valid = ids.cmp(.GE, Tensor.scalar(0, .i32)).logical(.AND, ids.cmp(.LT, Tensor.scalar(cfg.e, .i32))); - const inputs: ActivationStationaryFp8.Inputs = .{ .x = x, .ids = valid.select(ids, Tensor.scalar(0, .i32)), .w = valid.select(weights, Tensor.scalar(0, .f32)), .gate_up = gate_up, .down = down, .gate_up_scale = s.gate_up, .down_scale = s.down }; + const inputs: ActivationStationaryFp8.Inputs = .{ .x = x, .ids = safe_ids, .w = safe_weights, .gate_up = gate_up, .down = down, .gate_up_scale = s.gate_up, .down_scale = s.down }; const y = if (cfg.t * cfg.k <= cfg.e) ActivationStationaryFp8.call(inputs, .{ .y = out_shape }, .{ .cfg = cfg }).y else @@ -100,9 +104,9 @@ fn runExperts(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: Te return y.reshape(input.shape()); } const y = if (cfg.t * cfg.k <= cfg.e) - ActivationStationary.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y + ActivationStationary.call(.{ .x = x, .ids = safe_ids, .w = safe_weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y else - Blockwise.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y; + Blockwise.call(.{ .x = x, .ids = safe_ids, .w = safe_weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y; return y.reshape(input.shape()); } From 4c8f520251d587889d88123604a505c2b5644791 Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Fri, 13 Nov 2026 17:34:09 +0100 Subject: [PATCH 876/881] zml/moe: support BF16 GEGLU-tanh on Furiosa (#1063) Pass the requested activation to both BF16 TCL MoE kernels and compute tanh GELU on the gate before multiplying by the up projection. This lets Gemma 4 compile and run with its required activation. Verified both kernel paths against a CPU reference and confirmed coherent Gemma 4 generation on two and four RNGD cards. --- zml/kernels/moe/tcl/tcl.zig | 62 ++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 15 deletions(-) diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig index c26fff1b..71653f8e 100644 --- a/zml/kernels/moe/tcl/tcl.zig +++ b/zml/kernels/moe/tcl/tcl.zig @@ -27,22 +27,26 @@ const tcl = zml.kernel.tcl; const Options = @import("../api.zig").Options; const LocalRouting = @import("../api.zig").LocalRouting; +const Activation = enum { swiglu, geglu_tanh }; + /// `input` is `{.t, .d}`, `topk_ids` and `topk_weights` `{.t, .topk}`, `gate_up` `{.expert, .dout = 2 * inter (gate then up), .d}` /// and `down` `{.expert, .d, .dout = inter}`. Experts are BF16 or block-128 FP8 with BF16/F32 scales. -/// SwiGLU, routing weights applied after the down projection. +/// Supports plain SwiGLU and, for BF16 experts, GEGLU with tanh GELU. pub fn fusedExperts(input: Tensor, routing: LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: Options) Tensor { const topk_ids = routing.localIds(); stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE owns input precision and applies routing weights after the down projection", .{}); - stdx.debug.assert(opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}), "TCL MoE supports SwiGLU without limit, scale or bias", .{}); stdx.debug.assert(gate_up_linear.bias == null and down_linear.bias == null, "TCL MoE takes experts without bias", .{}); const gate_up = gate_up_linear.weight; const down = down_linear.weight; const quantized = gate_up_linear.quantization != null or down_linear.quantization != null; + const swiglu = opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}); + stdx.debug.assert(swiglu or (!quantized and opts.activation == .geglu_tanh), "TCL MoE supports plain SwiGLU or GEGLU-tanh with BF16 experts, got {s}", .{@tagName(opts.activation)}); + const activation: Activation = if (swiglu) .swiglu else .geglu_tanh; const scales: ?Scales = if (quantized) .{ .gate_up = fp8Scales(gate_up_linear), .down = fp8Scales(down_linear) } else null; stdx.debug.assert(input.dtype() == .bf16 and topk_ids.dtype() == .i32 and (quantized or (gate_up.dtype() == .bf16 and down.dtype() == .bf16)), "TCL MoE takes BF16 activations, BF16/block-128 FP8 experts and i32 ids", .{}); const weights = routing.weights.convert(.f32); const local_weights = if (scales != null) topk_ids.cmp(.GE, .scalar(0, .i32)).select(weights, .scalar(0, .f32)) else weights; - return runExperts(input, topk_ids, local_weights, gate_up, down, scales); + return runExperts(input, topk_ids, local_weights, gate_up, down, scales, activation); } const Scales = struct { gate_up: Tensor, down: Tensor }; @@ -60,13 +64,14 @@ fn fp8Scales(linear: zml.nn.Linear) Tensor { /// Tokens up to which a device runs its experts stationary. pub const expert_stationary_tokens = 64; -fn runExperts(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: Tensor, down: Tensor, scales: ?Scales) Tensor { +fn runExperts(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: Tensor, down: Tensor, scales: ?Scales, activation: Activation) Tensor { var cfg: Cfg = .{ .t = input.dim(.t), .k = topk_ids.dim(.topk), .e = gate_up.dim(.expert), .d = input.dim(.d), .i = down.dim(.dout), + .activation = activation, }; stdx.debug.assert(gate_up.dim(.dout) == 2 * cfg.i and down.dim(.d) == cfg.d and gate_up.dim(.d) == cfg.d, "TCL MoE: mismatched expert shapes", .{}); if (scales) |s| { @@ -115,7 +120,15 @@ fn scaleDtype(scales: Scales) tcl.DType { return if (scales.gate_up.dtype() == .bf16) .bf16 else .f32; } -pub const Cfg = struct { t: i64, k: i64, e: i64, d: i64, i: i64, scale_dtype: tcl.DType = .bf16 }; +pub const Cfg = struct { + t: i64, + k: i64, + e: i64, + d: i64, + i: i64, + scale_dtype: tcl.DType = .bf16, + activation: Activation = .swiglu, +}; const Axes = struct { T: tcl.Axis, @@ -246,7 +259,7 @@ fn project(b: *tcl.Builder, x: builder.Tensor, weight: builder.Tensor, scale: ?b /// `silu(gate) * up` or `gelu_tanh(gate) * up` of `h = [batch..., S2, I]` /// (gate first), times `weights` (`[batch...]`) unless null. -fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor, comptime fp8: bool) tcl.FinishError!builder.Tensor { +fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor, activation: Activation, comptime fp8: bool) tcl.FinishError!builder.Tensor { var sliced: [4]tcl.Axis = undefined; var flat: [3]tcl.Axis = undefined; @memcpy(sliced[0..batch.len], batch); @@ -258,7 +271,7 @@ fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, w const f = flat[0 .. batch.len + 1]; const gate = try b.reshape(try b.slice(h, a.S2, 0, s, .{}), f, .{}); const up = try b.reshape(try b.slice(h, a.S2, 1, s, .{}), f, .{}); - // Interleaving requires one binary instruction; SwiGLU chains several. + // Interleaving requires one binary instruction; these activations chain several. var op = b.tensorOperation(.{ .tactic = .elementwise }); const g = op.fetch(gate, .{}); const activated = if (fp8) blk: { @@ -273,12 +286,21 @@ fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, w const silu = try op.commit(rounded, .{ .dtype = .bf16 }); op = b.tensorOperation(.{ .tactic = .elementwise }); break :blk op.fetch(silu, .{}); - } else g.sigmoid().mulf(g); + } else switch (activation) { + .swiglu => g.sigmoid().mulf(g), + .geglu_tanh => geluTanh(g), + }; var v = activated.mulf(op.fetch(up, .{})); if (weights) |w| v = v.mulf(op.fetch(w, .{})); return op.commit(v, .{ .dtype = .bf16 }); } +fn geluTanh(g: builder.Value) builder.Value { + const cube = g.mulf(g).mulf(g); + const inner = g.addf(cube.mulf(0.044715)).mulf(@sqrt(2.0 / std.math.pi)); + return inner.tanh().addf(1.0).mulf(g).mulf(0.5); +} + /// Furiosa's compiler configuration for MoE kernels (`qwen3_moe/config.py`): /// IO-bound tactics for activation stationary, compute-bound for blockwise. /// Keep the compiler's default lowering mode and omit the adaptive-einsum and @@ -335,7 +357,7 @@ fn activationStationaryKernel(comptime fp8: bool) type { const scales = try gatherScales(b, a, in, in.ids, &.{ a.T, a.K }); const h = try project(b, in.x, gu, scales.gate_up, a.D, a.I, &.{ a.T, a.K, a.S2, a.I }, if (fp8) .bf16 else null); - const act = try gated(b, a, h, &.{ a.T, a.K }, if (fp8) null else in.w, fp8); + const act = try gated(b, a, h, &.{ a.T, a.K }, if (fp8) null else in.w, cfg.activation, fp8); if (fp8) { const y = try project(b, act, dn, scales.down, a.I, a.D, &.{ a.T, a.K, a.D }, .f32); const op = b.tensorOperation(.{}); @@ -423,7 +445,7 @@ fn blockwiseKernel(comptime fp8: bool) type { const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); const scales = try gatherScales(b, a, in, ids, &.{BGi}); const h = try project(b, chunk, gu, scales.gate_up, D, I, &.{ BGi, BB, a.S2, I }, if (fp8) .bf16 else null); - const act = try gated(b, a, h, &.{ BGi, BB }, null, fp8); + const act = try gated(b, a, h, &.{ BGi, BB }, null, cfg.activation, fp8); const y = try project(b, act, dn, scales.down, I, D, &.{ BGi, BB, D }, if (fp8) .f32 else .bf16); try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); } @@ -506,7 +528,7 @@ pub const ExpertStationaryFp8 = tcl.Kernel(Cfg, .{ const dn = try b.gather(in.down, ids, E, &.{ C, D, I }, .{}); const scales = try gatherScales(b, a, in, ids, &.{C}); const h = try project(b, in.x, gu, scales.gate_up, D, I, &.{ C, T, a.S2, I }, .bf16); - const act = try gated(b, a, h, &.{ C, T }, null, true); + const act = try gated(b, a, h, &.{ C, T }, null, .swiglu, true); const per_expert = try project(b, act, dn, scales.down, I, D, &.{ C, T, D }, .f32); op = b.tensorOperation(.{}); const y = try op.commit(op.fetch(per_expert, .{}).mulf(op.fetch(weights, .{})).reduce(&.{C}, .addf), .{ .dtype = .f32 }); @@ -519,6 +541,11 @@ pub const ExpertStationaryFp8 = tcl.Kernel(Cfg, .{ }); test "tcl fused experts run on furiosa" { + try testFusedExperts(.{ .swiglu = .{} }); + try testFusedExperts(.geglu_tanh); +} + +fn testFusedExperts(activation: zml.moe.Activation) !void { const platform = zml.testing.env(); if (platform.target != .furiosa) return error.SkipZigTest; const allocator = std.testing.allocator; @@ -537,15 +564,15 @@ test "tcl fused experts run on furiosa" { const gu: Tensor = .init(.{ .expert = e, .dout = 2 * i, .d = d }, .bf16); const dn: Tensor = .init(.{ .expert = e, .d = d, .dout = i }, .bf16); const Mod = struct { - pub fn forward(x_: Tensor, ids_: Tensor, w_: Tensor, gu_: Tensor, dn_: Tensor) Tensor { + pub fn forward(x_: Tensor, ids_: Tensor, w_: Tensor, gu_: Tensor, dn_: Tensor, act: zml.moe.Activation) Tensor { return zml.moe.forwardMoe(x_, ids_, w_, .init(gu_, null, .dout), .init(dn_, null, .dout), .tcl, zml.Compiler.current().platform.replicated_mesh, .expert(.replicated), .{ - .activation = .{ .swiglu = .{} }, + .activation = act, .quantize_input = false, .routing_weight_placement = .after_down, }); } }; - var exe = try platform.compileFn(allocator, io, Mod.forward, .{ x, ids, w, gu, dn }, .{}); + var exe = try platform.compileFn(allocator, io, Mod.forward, .{ x, ids, w, gu, dn, activation }, .{}); defer exe.deinit(); var prng: std.Random.DefaultPrng = .init(@intCast(t)); @@ -597,7 +624,12 @@ test "tcl fused experts run on furiosa" { g += xv * hgu[(ex * 2 * i + n) * d + m].toF32(); u += xv * hgu[(ex * 2 * i + i + n) * d + m].toF32(); } - act[n] = g / (1 + @exp(-g)) * u; + const gate = switch (activation) { + .swiglu => g / (1 + @exp(-g)), + .geglu_tanh => 0.5 * g * (1 + std.math.tanh(@sqrt(2.0 / std.math.pi) * (g + 0.044715 * g * g * g))), + else => unreachable, + }; + act[n] = gate * u; } for (0..d) |m| { var y: f32 = 0; From d2a69b9718e5127d9247491c4b5816d279d78095 Mon Sep 17 00:00:00 2001 From: Kevin Barre Date: Fri, 13 Nov 2026 17:34:09 +0100 Subject: [PATCH 877/881] zml/moe: [refacto] consolidate Furiosa TCL kernel setup (#1066) Separate dispatch from kernel generation. Use one factory to select BF16 and block-128 FP8 kernels by format and schedule, and reuse the shared ActivationKind. --- platforms/furiosa/furiosa.bzl | 4 +- zml/kernels/moe/tcl/BUILD.bazel | 5 +- zml/kernels/moe/tcl/moe.zig | 457 +++++++++++++++++++++++ zml/kernels/moe/tcl/tcl.zig | 623 ++++++-------------------------- 4 files changed, 578 insertions(+), 511 deletions(-) create mode 100644 zml/kernels/moe/tcl/moe.zig diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl index 623c4f0d..afb26864 100644 --- a/platforms/furiosa/furiosa.bzl +++ b/platforms/furiosa/furiosa.bzl @@ -6,8 +6,8 @@ def _furiosa_impl(mctx): http_archive( name = "libzml_furiosa", build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202611121233.371.1.9ff7ddf1e19f/zml-furiosa-linux-amd64.tar.zst", - sha256 = "d732b56832b1f009cbe4d808c70c6f8d1dc3857a83eb06804fc53b39bef9dd6b", + url = "https://mirror.zml.ai/plugins/202611131503.373.1.be246d178d7d/zml-furiosa-linux-amd64.tar.zst", + sha256 = "c7cdf0120818589aee3b53d8a6813e2b5e244447701acc34eb697b30f46e22f9", ) return mctx.extension_metadata( diff --git a/zml/kernels/moe/tcl/BUILD.bazel b/zml/kernels/moe/tcl/BUILD.bazel index 4057eea3..44b56e08 100644 --- a/zml/kernels/moe/tcl/BUILD.bazel +++ b/zml/kernels/moe/tcl/BUILD.bazel @@ -2,5 +2,8 @@ package(default_visibility = ["//zml:__pkg__"]) filegroup( name = "tcl", - srcs = ["tcl.zig"], + srcs = [ + "moe.zig", + "tcl.zig", + ], ) diff --git a/zml/kernels/moe/tcl/moe.zig b/zml/kernels/moe/tcl/moe.zig new file mode 100644 index 00000000..23fb2274 --- /dev/null +++ b/zml/kernels/moe/tcl/moe.zig @@ -0,0 +1,457 @@ +//! TCL kernel schedules for Furiosa RNGD fused experts. +const std = @import("std"); + +const builder = @import("dsl/tcl/builder"); + +const zml = @import("../../../zml.zig"); +const tcl = zml.kernel.tcl; +const ActivationKind = @import("../api.zig").ActivationKind; + +pub const Format = enum { bf16, fp8_block128 }; + +/// Dispatch in `tcl.zig` uses the first matching schedule. +/// +/// - `.small_batch` for FP8 up to 64 tokens before padding. Run each selected +/// expert on the whole batch to reuse its weights, with zero routing weight +/// for unrouted pairs. +/// - `.per_route` when `num_tokens * top_k <= local_num_experts`. Gather +/// weights for each route to avoid the overhead of grouping routes. +/// - `.blockwise` otherwise. Pack routed tokens into blocks by expert to +/// reuse weights within each block, then gather results back by route. +pub const Schedule = enum { per_route, blockwise, small_batch }; + +pub const Cfg = struct { + num_tokens: i64, + top_k: i64, + local_num_experts: i64, + hidden_size: i64, + expert_intermediate_size: i64, + weight_scale_dtype: tcl.DType = .bf16, + activation: ActivationKind = .swiglu, +}; + +const base_inputs: []const [:0]const u8 = &.{ "x", "ids", "w", "gate_up", "down" }; +const scale_inputs: []const [:0]const u8 = &.{ "gate_up_scale", "down_scale" }; + +pub fn Kernel(comptime format: Format, comptime schedule: Schedule) type { + if (schedule == .small_batch and format != .fp8_block128) { + @compileError("TCL MoE FP8 small-batch schedule requires block-128 FP8"); + } + return tcl.Kernel(Cfg, .{ + .name = "moe_" ++ @tagName(schedule) ++ switch (format) { + .bf16 => "", + .fp8_block128 => "_fp8", + }, + .inputs = switch (format) { + .bf16 => base_inputs, + .fp8_block128 => base_inputs ++ scale_inputs, + }, + .outputs = &.{"y"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + configure(b, schedule); + const a: Axes = .init(b, cfg); + const in = try args(b, a, cfg, format); + switch (schedule) { + .per_route => try perRoute(b, a, in, cfg.activation, format), + .blockwise => try blockwise(b, a, in, cfg, format), + .small_batch => try fp8SmallBatch(b, a, in, cfg), + } + } + }.run, + }); +} + +const Axes = struct { + T: tcl.Axis, + K: tcl.Axis, + E: tcl.Axis, + D: tcl.Axis, + I: tcl.Axis, + I2: tcl.Axis, + S2: tcl.Axis, + S1: tcl.Axis, + + fn init(b: *tcl.Builder, cfg: Cfg) Axes { + return .{ + .T = b.axis("T", cfg.num_tokens), + .K = b.axis("K", cfg.top_k), + .E = b.axis("E", cfg.local_num_experts), + .D = b.axis("D", cfg.hidden_size), + .I = b.axis("I", cfg.expert_intermediate_size), + .I2 = b.axis("I2", 2 * cfg.expert_intermediate_size), + .S2 = b.axis("S2", 2), + .S1 = b.axis("S1", 1), + }; + } +}; + +fn args(b: *tcl.Builder, a: Axes, cfg: Cfg, comptime format: Format) tcl.FinishError!Args { + const weight_dtype: tcl.DType = switch (format) { + .bf16 => .bf16, + .fp8_block128 => .f8_e4, + }; + const spec = .{ + .x = .{ .dtype = .bf16, .axes = &.{ a.T, a.D } }, + .ids = .{ .dtype = .i32, .axes = &.{ a.T, a.K } }, + .w = .{ .dtype = .f32, .axes = &.{ a.T, a.K } }, + .gate_up = .{ .dtype = weight_dtype, .axes = &.{ a.E, a.I2, a.D } }, + .down = .{ .dtype = weight_dtype, .axes = &.{ a.E, a.D, a.I } }, + }; + if (format == .bf16) { + const t = try b.declareArgs(spec); + return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down }; + } + const IG = b.axis("IG", @divExact(cfg.expert_intermediate_size, 128)); + const DG = b.axis("DG", @divExact(cfg.hidden_size, 128)); + const I2G = b.axis("I2G", 2 * @divExact(cfg.expert_intermediate_size, 128)); + const t = try b.declareArgs(.{ + .x = spec.x, + .ids = spec.ids, + .w = spec.w, + .gate_up = spec.gate_up, + .down = spec.down, + .gate_up_scale = .{ .dtype = cfg.weight_scale_dtype, .axes = &.{ a.E, I2G, DG } }, + .down_scale = .{ .dtype = cfg.weight_scale_dtype, .axes = &.{ a.E, DG, IG } }, + }); + return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down, .gate_up_scale = t.gate_up_scale, .down_scale = t.down_scale }; +} + +const Args = struct { + x: builder.Tensor, + ids: builder.Tensor, + w: builder.Tensor, + gate_up: builder.Tensor, + down: builder.Tensor, + gate_up_scale: ?builder.Tensor = null, + down_scale: ?builder.Tensor = null, +}; + +const GatheredScales = struct { gate_up: ?builder.Tensor = null, down: ?builder.Tensor = null }; + +fn gatherScales(b: *tcl.Builder, a: Axes, in: Args, ids: builder.Tensor, batch: []const tcl.Axis, comptime format: Format) tcl.FinishError!GatheredScales { + if (format == .bf16) return .{}; + const IG = b.axis("IG", @divExact(a.I.size, 128)); + const DG = b.axis("DG", @divExact(a.D.size, 128)); + const gate_up = try b.reshape(in.gate_up_scale.?, &.{ a.E, a.S2, IG, DG }, .{}); + var gu_axes: [8]tcl.Axis = undefined; + var dn_axes: [8]tcl.Axis = undefined; + @memcpy(gu_axes[0..batch.len], batch); + @memcpy(dn_axes[0..batch.len], batch); + @memcpy(gu_axes[batch.len..][0..3], &[_]tcl.Axis{ a.S2, IG, DG }); + @memcpy(dn_axes[batch.len..][0..2], &[_]tcl.Axis{ DG, IG }); + return .{ + .gate_up = try b.gather(gate_up, ids, a.E, gu_axes[0 .. batch.len + 3], .{}), + .down = try b.gather(in.down_scale.?, ids, a.E, dn_axes[0 .. batch.len + 2], .{}), + }; +} + +fn blockedAxes(axes: []const tcl.Axis, axis: tcl.Axis, group: tcl.Axis, lane: tcl.Axis, storage: *[16]tcl.Axis) []const tcl.Axis { + var n: usize = 0; + for (axes) |a| { + if (std.mem.eql(u8, a.name, axis.name)) { + storage[n] = group; + n += 1; + storage[n] = lane; + } else storage[n] = a; + n += 1; + } + return storage[0..n]; +} + +/// Block scales multiply each K-block partial before K-block reduction. +/// Only the requested experts are gathered; their bank remains FP8 in HBM. +fn project(b: *tcl.Builder, x: builder.Tensor, weight: builder.Tensor, scale: ?builder.Tensor, contracting: tcl.Axis, output: tcl.Axis, to: []const tcl.Axis, dtype: ?tcl.DType, comptime format: Format) tcl.FinishError!builder.Tensor { + if (format == .bf16) { + const op = b.tensorOperation(.{}); + return op.commit(op.contract(x, weight, to), .{ .dtype = dtype }); + } + const CG = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DG" else "IG", @divExact(contracting.size, 128)); + const CL = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DB" else "IB", 128); + const OG = b.axis(if (std.mem.eql(u8, output.name, "D")) "DG" else "IG", @divExact(output.size, 128)); + const OL = b.axis(if (std.mem.eql(u8, output.name, "D")) "DB" else "IB", 128); + var x_axes: [16]tcl.Axis = undefined; + var w_axes: [16]tcl.Axis = undefined; + var w2_axes: [16]tcl.Axis = undefined; + var out_axes: [16]tcl.Axis = undefined; + var partial_axes: [16]tcl.Axis = undefined; + const xb = try b.reshape(x, blockedAxes(x.axes, contracting, CG, CL, &x_axes), .{}); + const wb = try b.reshape(weight, blockedAxes(blockedAxes(weight.axes, output, OG, OL, &w_axes), contracting, CG, CL, &w2_axes), .{}); + const out = blockedAxes(to, output, OG, OL, &out_axes); + @memcpy(partial_axes[0..out.len], out); + partial_axes[out.len] = CG; + + // Furiosa's published FP8 recipe uses an additive epsilon, including zero blocks. + var op = b.tensorOperation(.{}); + const qscale = try op.commit(op.fetch(xb, .{}).unary(.absf).reduce(&.{CL}, .maxf).divf(448.0).addf(1e-12), .{ .dtype = .f32 }); + op = b.tensorOperation(.{}); + // Native F32 reciprocal accuracy is qualified separately (within one ULP). + const reciprocal = op.ve(.divf, .{ 1.0, op.fetch(qscale, .{}) }); + const quantized = try op.commit(reciprocal.mulf(op.fetch(xb, .{})), .{ .dtype = .f8_e4 }); + op = b.tensorOperation(.{}); + const scaled = op.contract(quantized, wb, partial_axes[0 .. out.len + 1]).mulf(op.fetch(qscale, .{})).mulf(op.fetch(scale.?, .{})).reduce(&.{CG}, .addf); + return b.reshape(try op.commit(scaled, .{ .dtype = dtype, .axes = out }), to, .{}); +} + +/// `silu(gate) * up` or `gelu_tanh(gate) * up` of `h = [batch..., S2, I]` +/// (gate first), times `weights` (`[batch...]`) unless null. +fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor, activation: ActivationKind, comptime format: Format) tcl.FinishError!builder.Tensor { + var sliced: [4]tcl.Axis = undefined; + var flat: [3]tcl.Axis = undefined; + @memcpy(sliced[0..batch.len], batch); + @memcpy(flat[0..batch.len], batch); + sliced[batch.len] = a.S1; + sliced[batch.len + 1] = a.I; + flat[batch.len] = a.I; + const s = sliced[0 .. batch.len + 2]; + const f = flat[0 .. batch.len + 1]; + const gate = try b.reshape(try b.slice(h, a.S2, 0, s, .{}), f, .{}); + const up = try b.reshape(try b.slice(h, a.S2, 1, s, .{}), f, .{}); + // The builder can infer the TCL `Interleaving` tactic for two inputs with + // matching axes, such as `gate` and `up`. That tactic allows exactly one + // binary instruction, so force `.elementwise` for this activation chain. + var op = b.tensorOperation(.{ .tactic = .elementwise }); + const g = op.fetch(gate, .{}); + const activated = if (format == .fp8_block128) blk: { + if (activation != .swiglu) std.debug.panic("TCL MoE block-128 FP8 requires SwiGLU, got {s}", .{@tagName(activation)}); + + // WORKAROUND (TCC 2026.5.0b0): native fusion elides an internal + // BF16 SiLU write, even through as_dram/as_logical. Make its RNE + // rounding observable as integer bits before multiplying by up. + // Remove once the native reproducer preserves the BF16 boundary. + const bits = try op.commit(g.sigmoid().mulf(g).unary(.reinterpret_as_int), .{ .dtype = .i32 }); + op = b.tensorOperation(.{ .tactic = .elementwise }); + const v = op.fetch(bits, .{}); + const rounded = v.binary(.shr_logical, 16).binary(.bit_andi, 1).addi(32767).addi(v).binary(.bit_andi, -65536).unary(.reinterpret_as_float); + const silu = try op.commit(rounded, .{ .dtype = .bf16 }); + op = b.tensorOperation(.{ .tactic = .elementwise }); + break :blk op.fetch(silu, .{}); + } else switch (activation) { + .swiglu => g.sigmoid().mulf(g), + .geglu_tanh => geluTanh(g), + else => std.debug.panic("TCL MoE does not support activation {s}", .{@tagName(activation)}), + }; + var v = activated.mulf(op.fetch(up, .{})); + if (weights) |w| v = v.mulf(op.fetch(w, .{})); + return op.commit(v, .{ .dtype = .bf16 }); +} + +fn geluTanh(g: builder.Value) builder.Value { + const cube = g.mulf(g).mulf(g); + const inner = g.addf(cube.mulf(0.044715)).mulf(@sqrt(2.0 / std.math.pi)); + return inner.tanh().addf(1.0).mulf(g).mulf(0.5); +} + +/// Furiosa's compiler configuration for MoE kernels (`qwen3_moe/config.py`): +/// Use IO-bound tactics for per-route and FP8 small-batch schedules and +/// compute-bound tactics for blockwise. +/// Keep the compiler's default lowering mode and omit the adaptive-einsum and +/// half-VRF settings removed in TCC 2026.5. +fn configure(b: *tcl.Builder, comptime schedule: Schedule) void { + const config = comptime blk: { + const common = .{ + .enable_einsum_fusion = true, + .padding_policy = .{ .Small = 1.1 }, + .instruction_mem_budget = 0xB0000, + .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, + .tensor_unit_bridge_threshold_in_page = 12, + .allow_external_operators = false, + .enable_tactic_pruning = false, + .scheduler_beam_search = true, + .allow_reduce_by_ve_cluster_chip_reduce = false, + .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, + .propagate_sparse_axis_from_op = "None", + }; + const tactics = switch (schedule) { + .blockwise => .{ + .tactic_hint = .{ .ForLlmModelComputeBound = 200 }, + .dma_preference = 1.2, + }, + .per_route, .small_batch => .{ + .tactic_hint = .{ .ForLlmModelIOBound = 200 }, + .num_transaction_simulation_per_pe = 1024, + .dma_preference = 0.8, + }, + }; + // The transaction-simulation setting is only emitted for IO-bound schedules. + const common_info = @typeInfo(@TypeOf(common)).@"struct"; + const tactics_info = @typeInfo(@TypeOf(tactics)).@"struct"; + const Config = @Struct( + .auto, + null, + common_info.field_names ++ tactics_info.field_names, + common_info.field_types ++ tactics_info.field_types, + common_info.field_attrs ++ tactics_info.field_attrs, + ); + var merged: Config = undefined; + for (common_info.field_names) |field| @field(merged, field) = @field(common, field); + for (tactics_info.field_names) |field| @field(merged, field) = @field(tactics, field); + break :blk merged; + }; + b.compilerConfig(config); +} + +/// Gather the selected experts for every token. +fn perRoute(b: *tcl.Builder, a: Axes, in: Args, activation: ActivationKind, comptime format: Format) tcl.FinishError!void { + const gate_up = try b.reshape(in.gate_up, &.{ a.E, a.S2, a.I, a.D }, .{}); + const gu = try b.gather(gate_up, in.ids, a.E, &.{ a.T, a.K, a.S2, a.I, a.D }, .{}); + const dn = try b.gather(in.down, in.ids, a.E, &.{ a.T, a.K, a.D, a.I }, .{}); + + const scales = try gatherScales(b, a, in, in.ids, &.{ a.T, a.K }, format); + const h = try project(b, in.x, gu, scales.gate_up, a.D, a.I, &.{ a.T, a.K, a.S2, a.I }, if (format == .fp8_block128) .bf16 else null, format); + const act = try gated(b, a, h, &.{ a.T, a.K }, if (format == .fp8_block128) null else in.w, activation, format); + if (format == .fp8_block128) { + const y = try project(b, act, dn, scales.down, a.I, a.D, &.{ a.T, a.K, a.D }, .f32, format); + const op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(y, .{}).mulf(op.fetch(in.w, .{})).reduce(&.{a.K}, .addf), .{ .dtype = .bf16 })}); + } else { + // The BF16 recipe weights activations and contracts experts together. + const op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.contract(act, dn, &.{ a.T, a.D }), .{ .dtype = .bf16 })}); + } +} + +/// Rows per expert block: the next power of two of the routes per expert. +fn blockRows(cfg: Cfg) i64 { + const per_expert = std.math.divCeil(i64, cfg.num_tokens * cfg.top_k, cfg.local_num_experts) catch unreachable; + return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@max(per_expert, 4))) catch unreachable); +} + +const blocks_per_chunk = 32; + +/// Group routes into expert-major blocks, then run both projections in a +/// device loop over chunks of blocks. +fn blockwise(b: *tcl.Builder, a: Axes, in: Args, cfg: Cfg, comptime format: Format) tcl.FinishError!void { + const bb = blockRows(cfg); + // One block per expert at least, then enough for every route. + const max_blocks = std.math.divCeil(i64, @max(cfg.num_tokens * cfg.top_k - cfg.local_num_experts, 0), bb) catch unreachable; + const chunks = std.math.divCeil(i64, max_blocks + cfg.local_num_experts, blocks_per_chunk) catch unreachable; + const BB = b.axis("BB", bb); + const BGi = b.axis("BGi", blocks_per_chunk); + const Go = b.axis("Go", chunks); + const BG = b.axis("BG", chunks * blocks_per_chunk); + const GB = b.axis("GB", chunks * blocks_per_chunk * bb); + const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; + + // Routing: route (t, k) goes to row `slot` of block `start[e] + rank`. + const experts = try b.arange(cfg.local_num_experts, .i32, &.{E}, .{}); + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); + op = b.tensorOperation(.{}); + const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const rank = try op.commit(op.fetch(onehot, .{}).cumsum(&.{ T, K }), .{}); + op = b.tensorOperation(.{}); + const log2_bb: i64 = std.math.log2_int(u64, @intCast(bb)); + const blocks = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi).addi(bb - 1).binary(.shr_arith, log2_bb), .{}); + op = b.tensorOperation(.{}); + const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{E}), .{}); + op = b.tensorOperation(.{}); + const start_rows = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})).muli(bb), .{}); + op = b.tensorOperation(.{}); + const slot = try op.commit(op.fetch(rank, .{}).addi(op.fetch(start_rows, .{})).muli(op.fetch(onehot, .{})).reduce(&.{E}, .addi).subi(1), .{}); + // The expert of block g: how many experts end at or before g. + const block_ids = try b.arange(chunks * blocks_per_chunk, .i32, &.{BG}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const ends = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(block_ids, .{})), .{}); + op = b.tensorOperation(.{}); + const ended = try op.commit(op.where(ends, .le, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + // Padding blocks past the last expert reuse the last one. + const block_expert = try op.commit(op.fetch(ended, .{}).reduce(&.{E}, .addi).binary(.mini, cfg.local_num_experts - 1), .{}); + const used = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(block_end), blocks_per_chunk - 1), blocks_per_chunk); + + // Rows of unused slots are never read back. + const rows = try b.reshape(try b.scatter(in.x, slot, GB, &.{ GB, D }, .{}), &.{ Go, BGi, BB, D }, .{}); + const chunk_experts = try b.reshape(block_expert, &.{ Go, BGi }, .{}); + const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); + + var loop = b.openFor(used, .{try b.scratchpad(if (format == .fp8_block128) .f32 else .bf16, &.{ Go, BGi, BB, D })}); + { + const chunk = try b.indexRead(rows, loop.iv); + const ids = try b.indexRead(chunk_experts, loop.iv); + const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); + const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); + const scales = try gatherScales(b, a, in, ids, &.{BGi}, format); + const h = try project(b, chunk, gu, scales.gate_up, D, I, &.{ BGi, BB, a.S2, I }, if (format == .fp8_block128) .bf16 else null, format); + const act = try gated(b, a, h, &.{ BGi, BB }, null, cfg.activation, format); + const y = try project(b, act, dn, scales.down, I, D, &.{ BGi, BB, D }, if (format == .fp8_block128) .f32 else .bf16, format); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); + } + const out = loop.results[0]; + + const per_route = try b.gather(try b.reshape(out, &.{ GB, D }, .{}), slot, GB, &.{ T, K, D }, .{}); + op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(per_route, .{ .typecast_to = .f32 }).mulf(op.fetch(in.w, .{})).reduce(&.{K}, .addf), .{ .dtype = .bf16 })}); +} + +/// Number of experts processed per device-loop iteration, based on the +/// batch's route count (`cfg.num_tokens * cfg.top_k`). +fn expertsPerChunk(cfg: Cfg) i64 { + const routes: u64 = @intCast(std.math.divCeil(i64, cfg.num_tokens * cfg.top_k, 16) catch unreachable); + return @intCast(std.math.clamp(std.math.ceilPowerOfTwo(u64, routes) catch unreachable, 2, 16)); +} + +/// Process the FP8 experts selected by the batch in groups sized by `expertsPerChunk`. +/// Load each selected expert once and run it for every token. Routing weights +/// scale the F32 down-projection results before summing the experts and steps. +fn fp8SmallBatch(b: *tcl.Builder, a: Axes, in: Args, cfg: Cfg) tcl.FinishError!void { + const chunk = expertsPerChunk(cfg); + const steps = std.math.divCeil(i64, @min(cfg.local_num_experts, cfg.num_tokens * cfg.top_k), chunk) catch unreachable; + const C = b.axis("C", chunk); + const Go = b.axis("Go", steps); + const J = b.axis("J", steps * chunk); + const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; + + // Each token's routing weight per expert, and the used experts. + const experts = try b.arange(cfg.local_num_experts, .i32, &.{E}, .{}); + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); + op = b.tensorOperation(.{}); + const gate = try op.commit(op.where(op.fetch(dist, .{}), .eq, 0, op.fetch(in.w, .{ .broadcast_to = &.{ T, K, E } }), 0.0).reduce(&.{K}, .addf), .{}); + op = b.tensorOperation(.{}); + const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const counts = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi), .{}); + op = b.tensorOperation(.{}); + const used = try op.commit(op.where(counts, .gt, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const position = try op.commit(op.fetch(used, .{}).cumsum(&.{E}), .{}); + // A used expert's rank among them, -1 for the others. + op = b.tensorOperation(.{}); + const rank = try op.commit(op.fetch(position, .{}).muli(op.fetch(used, .{})).subi(1), .{}); + const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(position), chunk - 1), chunk); + + // Slot j holds the used expert of rank j and its tokens' weights; + // slots past the last hold expert 0 with no weight. + const ranks = try b.arange(steps * chunk, .i32, &.{J}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const offset = try op.commit(op.fetch(rank, .{}).subi(op.fetch(ranks, .{})), .{}); + op = b.tensorOperation(.{}); + const slot_experts = try op.commit(op.where(op.fetch(offset, .{}), .eq, 0, op.fetch(experts, .{ .broadcast_to = &.{ E, J } }), 0).reduce(&.{E}, .addi), .{ .dtype = .i32 }); + // A reduction's condition is read plainly, so broadcast it first. + op = b.tensorOperation(.{}); + const token_offset = try op.commit(op.fetch(offset, .{ .broadcast_to = &.{ T, E, J } }), .{}); + op = b.tensorOperation(.{}); + const slot_gate = try op.commit(op.where(op.fetch(token_offset, .{}), .eq, 0, op.fetch(gate, .{ .broadcast_to = &.{ T, E, J } }), 0.0).reduce(&.{E}, .addf), .{ .dtype = .f32, .axes = &.{ J, T } }); + const step_experts = try b.reshape(slot_experts, &.{ Go, C }, .{}); + const step_gate = try b.reshape(slot_gate, &.{ Go, C, T }, .{}); + const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); + + var loop = b.openFor(trips, .{try b.full(0, .f32, &.{ Go, T, D })}); + { + const ids = try b.indexRead(step_experts, loop.iv); + const weights = try b.indexRead(step_gate, loop.iv); + const gu = try b.gather(gate_up, ids, E, &.{ C, a.S2, I, D }, .{}); + const dn = try b.gather(in.down, ids, E, &.{ C, D, I }, .{}); + const scales = try gatherScales(b, a, in, ids, &.{C}, .fp8_block128); + const h = try project(b, in.x, gu, scales.gate_up, D, I, &.{ C, T, a.S2, I }, .bf16, .fp8_block128); + const act = try gated(b, a, h, &.{ C, T }, null, cfg.activation, .fp8_block128); + const per_expert = try project(b, act, dn, scales.down, I, D, &.{ C, T, D }, .f32, .fp8_block128); + op = b.tensorOperation(.{}); + const y = try op.commit(op.fetch(per_expert, .{}).mulf(op.fetch(weights, .{})).reduce(&.{C}, .addf), .{ .dtype = .f32 }); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); + } + op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(loop.results[0], .{}).reduce(&.{Go}, .addf), .{ .dtype = .bf16 })}); +} diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig index 71653f8e..f05727ce 100644 --- a/zml/kernels/moe/tcl/tcl.zig +++ b/zml/kernels/moe/tcl/tcl.zig @@ -1,59 +1,88 @@ //! Furiosa RNGD fused experts in TCL, after Furiosa's production kernels. -//! -//! Few tokens (at most `experts / top_k`): activation stationary, each token -//! gathers the weights of its experts and the down projection contracts over -//! the experts and the intermediate axis at once. -//! -//! More tokens: blockwise. The kernel routes on the device: routes are placed -//! in expert-major blocks of `BB` rows (one-hot, cumulative sums), then a -//! device loop over the used chunks of `BGi` blocks gathers each block's -//! expert weights and runs both projections; outputs are gathered back per -//! route and weighted. -//! -//! FP8 experts split across devices (expert parallel): each device runs its own -//! experts for the routes that reach them and an all-reduce sums the devices' -//! outputs. Up to `expert_stationary_tokens` tokens, expert stationary: a -//! device loop over the used experts, a few at a time, reads each once and -//! runs it for every token, scaled by the token's routing weight (zero for -//! the tokens it does not serve); more tokens take blockwise. const std = @import("std"); -const builder = @import("dsl/tcl/builder"); const stdx = @import("stdx"); const zml = @import("../../../zml.zig"); const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; -const Options = @import("../api.zig").Options; -const LocalRouting = @import("../api.zig").LocalRouting; +const kernels = @import("moe.zig"); +const api = @import("../api.zig"); -const Activation = enum { swiglu, geglu_tanh }; - -/// `input` is `{.t, .d}`, `topk_ids` and `topk_weights` `{.t, .topk}`, `gate_up` `{.expert, .dout = 2 * inter (gate then up), .d}` -/// and `down` `{.expert, .d, .dout = inter}`. Experts are BF16 or block-128 FP8 with BF16/F32 scales. -/// Supports plain SwiGLU and, for BF16 experts, GEGLU with tanh GELU. -pub fn fusedExperts(input: Tensor, routing: LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: Options) Tensor { +/// `input` is `{.t, .d}`, routing IDs and weights are `{.t, .topk}`, `gate_up` +/// is `{.expert, .dout = 2 * inter (gate then up), .d}`, and `down` is +/// `{.expert, .d, .dout = inter}`. Experts are BF16 or block-128 FP8 with +/// BF16/F32 scales. Supports plain SwiGLU and BF16 GEGLU with tanh GELU. +pub fn fusedExperts(input: Tensor, routing: api.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: api.Options) Tensor { const topk_ids = routing.localIds(); - stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE owns input precision and applies routing weights after the down projection", .{}); + stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE requires quantize_input=false and routing_weight_placement=after_down", .{}); stdx.debug.assert(gate_up_linear.bias == null and down_linear.bias == null, "TCL MoE takes experts without bias", .{}); + stdx.debug.assert(input.dtype() == .bf16 and topk_ids.dtype() == .i32, "TCL MoE takes BF16 activations and i32 ids", .{}); + const format = expertFormat(gate_up_linear, down_linear); + const swiglu = opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}); + stdx.debug.assert(swiglu or (format == .bf16 and opts.activation == .geglu_tanh), "TCL MoE supports plain SwiGLU or GEGLU-tanh with BF16 experts, got {s}", .{@tagName(opts.activation)}); + const gate_up = gate_up_linear.weight; const down = down_linear.weight; - const quantized = gate_up_linear.quantization != null or down_linear.quantization != null; - const swiglu = opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}); - stdx.debug.assert(swiglu or (!quantized and opts.activation == .geglu_tanh), "TCL MoE supports plain SwiGLU or GEGLU-tanh with BF16 experts, got {s}", .{@tagName(opts.activation)}); - const activation: Activation = if (swiglu) .swiglu else .geglu_tanh; - const scales: ?Scales = if (quantized) .{ .gate_up = fp8Scales(gate_up_linear), .down = fp8Scales(down_linear) } else null; - stdx.debug.assert(input.dtype() == .bf16 and topk_ids.dtype() == .i32 and (quantized or (gate_up.dtype() == .bf16 and down.dtype() == .bf16)), "TCL MoE takes BF16 activations, BF16/block-128 FP8 experts and i32 ids", .{}); + const cfg: kernels.Cfg = .{ + .num_tokens = input.dim(.t), + .top_k = topk_ids.dim(.topk), + .local_num_experts = gate_up.dim(.expert), + .hidden_size = input.dim(.d), + .expert_intermediate_size = down.dim(.dout), + .weight_scale_dtype = switch (format) { + .bf16 => .bf16, + .fp8_block128 => |s| tcl.from(s.gate_up.dtype()), + }, + .activation = std.meta.activeTag(opts.activation), + }; + stdx.debug.assert(gate_up.dim(.dout) == 2 * cfg.expert_intermediate_size and down.dim(.d) == cfg.hidden_size and gate_up.dim(.d) == cfg.hidden_size, "TCL MoE: mismatched expert shapes", .{}); + const weights = routing.weights.convert(.f32); - const local_weights = if (scales != null) topk_ids.cmp(.GE, .scalar(0, .i32)).select(weights, .scalar(0, .f32)) else weights; - return runExperts(input, topk_ids, local_weights, gate_up, down, scales, activation); + const local_weights = switch (format) { + .bf16 => weights, + .fp8_block128 => topk_ids.cmp(.GE, .scalar(0, .i32)).select(weights, .scalar(0, .f32)), + }; + return switch (format) { + .bf16 => selectScheduleAndRun(.bf16, cfg, .{ .x = input, .ids = topk_ids, .w = local_weights, .gate_up = gate_up, .down = down }), + .fp8_block128 => |s| selectScheduleAndRun(.fp8_block128, cfg, .{ + .x = input, + .ids = topk_ids, + .w = local_weights, + .gate_up = gate_up, + .down = down, + .gate_up_scale = s.gate_up, + .down_scale = s.down, + }), + }; } const Scales = struct { gate_up: Tensor, down: Tensor }; +const Format = union(kernels.Format) { + bf16: void, + fp8_block128: Scales, +}; + +fn expertFormat(gate_up: zml.nn.Linear, down: zml.nn.Linear) Format { + const scheme = gate_up.quantizationScheme(); + stdx.debug.assert(scheme == down.quantizationScheme(), "TCL MoE projections must have matching quantization schemes", .{}); + return switch (scheme orelse { + stdx.debug.assert(gate_up.weight.dtype() == .bf16 and down.weight.dtype() == .bf16, "TCL MoE unquantized expert weights must be BF16", .{}); + return .bf16; + }) { + .fp8_block128 => blk: { + const scales: Scales = .{ .gate_up = fp8Scales(gate_up), .down = fp8Scales(down) }; + stdx.debug.assert(scales.gate_up.dtype() == scales.down.dtype(), "TCL MoE projection scales must have matching types", .{}); + break :blk .{ .fp8_block128 = scales }; + }, + else => |unsupported| stdx.debug.panic("TCL MoE does not support quantization scheme {s}", .{@tagName(unsupported)}), + }; +} + fn fp8Scales(linear: zml.nn.Linear) Tensor { - const q = linear.quantization orelse std.debug.panic("TCL MoE requires both FP8 projections", .{}); - stdx.debug.assert(q.scheme == .fp8_block128 and linear.weight.dtype() == .f8e4m3fn and !q.swizzled_scales and q.global_scale == null and q.input_scale == null, "TCL MoE takes plain E4M3FN block-128 weights", .{}); + const q = linear.quantization.?; + stdx.debug.assert(linear.weight.dtype() == .f8e4m3fn and !q.swizzled_scales and q.global_scale == null and q.input_scale == null, "TCL MoE takes plain E4M3FN block-128 weights", .{}); stdx.debug.assert(q.scales.dtype() == .bf16 or q.scales.dtype() == .f32, "TCL MoE FP8 scales must be BF16 or F32", .{}); const w = linear.weight; const scale = q.scales; @@ -61,485 +90,62 @@ fn fp8Scales(linear: zml.nn.Linear) Tensor { return scale; } -/// Tokens up to which a device runs its experts stationary. -pub const expert_stationary_tokens = 64; +// Maximum total input tokens per MoE call before padding for the FP8 +// small-batch schedule. It loads each used expert's weights once and reuses +// them across the batch. +// Larger batches fall back to per-route or blockwise dispatch. +const fp8_small_batch_max_tokens = 64; -fn runExperts(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: Tensor, down: Tensor, scales: ?Scales, activation: Activation) Tensor { - var cfg: Cfg = .{ - .t = input.dim(.t), - .k = topk_ids.dim(.topk), - .e = gate_up.dim(.expert), - .d = input.dim(.d), - .i = down.dim(.dout), - .activation = activation, - }; - stdx.debug.assert(gate_up.dim(.dout) == 2 * cfg.i and down.dim(.d) == cfg.d and gate_up.dim(.d) == cfg.d, "TCL MoE: mismatched expert shapes", .{}); - if (scales) |s| { - cfg.scale_dtype = scaleDtype(s); - if (cfg.t <= expert_stationary_tokens) { - // WORKAROUND (TCC 2026.5.0b0): a token count other than a power of two - // panics its adder tree ("Fail to cover valid range", 5 tokens padded - // to 8): pad with unrouted tokens. - var padded = cfg; - padded.t = @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.t)) catch unreachable); - const pad = padded.t - cfg.t; - const x = input.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.t, cfg.d }); - const ids = topk_ids.pad(-1, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.t, cfg.k }); - const weights = topk_weights.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.t, cfg.k }); - const y = ExpertStationaryFp8.call(.{ .x = x, .ids = ids, .w = weights, .gate_up = gate_up, .down = down, .gate_up_scale = s.gate_up, .down_scale = s.down }, .{ .y = x.shape() }, .{ .cfg = padded }).y; - return y.slice(0, .{ .end = cfg.t }).reshape(input.shape()); - } +fn selectScheduleAndRun(comptime format: kernels.Format, cfg: kernels.Cfg, inputs: kernels.Kernel(format, .per_route).Inputs) Tensor { + if (format == .fp8_block128 and cfg.num_tokens <= fp8_small_batch_max_tokens) { + return run(format, .small_batch, cfg, inputs); } + if (cfg.num_tokens * cfg.top_k <= cfg.local_num_experts) { + return run(format, .per_route, cfg, inputs); + } + return run(format, .blockwise, cfg, inputs); +} - const x = input.reshape(.{ cfg.t, cfg.d }); - const ids = topk_ids.reshape(.{ cfg.t, cfg.k }); - const weights = topk_weights.reshape(.{ cfg.t, cfg.k }); +fn run(comptime format: kernels.Format, comptime schedule: kernels.Schedule, cfg: kernels.Cfg, inputs: kernels.Kernel(format, .per_route).Inputs) Tensor { + const call_cfg, const prepared = if (schedule == .small_batch) + prepareFp8SmallBatch(cfg, inputs) + else + .{ cfg, prepareRoutedInputs(cfg, inputs) }; + + const shape = prepared.x.shape(); + var y = kernels.Kernel(format, schedule).call(prepared, .{ .y = shape }, .{ .cfg = call_cfg }).y; + + if (schedule == .small_batch) y = y.slice(0, .{ .end = cfg.num_tokens }); + return y.reshape(inputs.x.shape()); +} + +fn prepareRoutedInputs(cfg: kernels.Cfg, inputs: anytype) @TypeOf(inputs) { + var prepared = inputs; + prepared.x = inputs.x.reshape(.{ cfg.num_tokens, cfg.hidden_size }); + const ids = inputs.ids.reshape(.{ cfg.num_tokens, cfg.top_k }); + const w = inputs.w.reshape(.{ cfg.num_tokens, cfg.top_k }); // These kernels gather every route, and blockwise also scatters each route // into an expert slot. Give nonlocal routes a valid slot and zero weight. - const valid = ids.cmp(.GE, Tensor.scalar(0, .i32)).logical(.AND, ids.cmp(.LT, Tensor.scalar(cfg.e, .i32))); - const safe_ids = valid.select(ids, Tensor.scalar(0, .i32)); - const safe_weights = valid.select(weights, Tensor.scalar(0, .f32)); - const out_shape = x.shape(); - if (scales) |s| { - const inputs: ActivationStationaryFp8.Inputs = .{ .x = x, .ids = safe_ids, .w = safe_weights, .gate_up = gate_up, .down = down, .gate_up_scale = s.gate_up, .down_scale = s.down }; - const y = if (cfg.t * cfg.k <= cfg.e) - ActivationStationaryFp8.call(inputs, .{ .y = out_shape }, .{ .cfg = cfg }).y - else - BlockwiseFp8.call(inputs, .{ .y = out_shape }, .{ .cfg = cfg }).y; - return y.reshape(input.shape()); - } - const y = if (cfg.t * cfg.k <= cfg.e) - ActivationStationary.call(.{ .x = x, .ids = safe_ids, .w = safe_weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y - else - Blockwise.call(.{ .x = x, .ids = safe_ids, .w = safe_weights, .gate_up = gate_up, .down = down }, .{ .y = out_shape }, .{ .cfg = cfg }).y; - return y.reshape(input.shape()); + const valid = ids.cmp(.GE, Tensor.scalar(0, .i32)).logical(.AND, ids.cmp(.LT, Tensor.scalar(cfg.local_num_experts, .i32))); + prepared.ids = valid.select(ids, Tensor.scalar(0, .i32)); + prepared.w = valid.select(w, Tensor.scalar(0, .f32)); + return prepared; } -fn scaleDtype(scales: Scales) tcl.DType { - stdx.debug.assert(scales.gate_up.dtype() == scales.down.dtype(), "TCL MoE projection scales must have matching types", .{}); - return if (scales.gate_up.dtype() == .bf16) .bf16 else .f32; +fn prepareFp8SmallBatch(cfg: kernels.Cfg, inputs: kernels.Kernel(.fp8_block128, .small_batch).Inputs) struct { kernels.Cfg, @TypeOf(inputs) } { + // WORKAROUND (TCC 2026.5.0b0): a token count other than a power of two + // panics its adder tree ("Fail to cover valid range", 5 tokens padded + // to 8): pad with unrouted tokens. + var padded = cfg; + padded.num_tokens = @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.num_tokens)) catch unreachable); + const pad = padded.num_tokens - cfg.num_tokens; + var prepared = inputs; + prepared.x = inputs.x.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.hidden_size }); + prepared.ids = inputs.ids.pad(-1, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.top_k }); + prepared.w = inputs.w.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.top_k }); + return .{ padded, prepared }; } -pub const Cfg = struct { - t: i64, - k: i64, - e: i64, - d: i64, - i: i64, - scale_dtype: tcl.DType = .bf16, - activation: Activation = .swiglu, -}; - -const Axes = struct { - T: tcl.Axis, - K: tcl.Axis, - E: tcl.Axis, - D: tcl.Axis, - I: tcl.Axis, - I2: tcl.Axis, - S2: tcl.Axis, - S1: tcl.Axis, - - fn init(b: *tcl.Builder, cfg: Cfg) Axes { - return .{ - .T = b.axis("T", cfg.t), - .K = b.axis("K", cfg.k), - .E = b.axis("E", cfg.e), - .D = b.axis("D", cfg.d), - .I = b.axis("I", cfg.i), - .I2 = b.axis("I2", 2 * cfg.i), - .S2 = b.axis("S2", 2), - .S1 = b.axis("S1", 1), - }; - } -}; - -fn args(b: *tcl.Builder, a: Axes, cfg: Cfg, comptime fp8: bool) tcl.FinishError!Args { - const spec = .{ - .x = .{ .dtype = .bf16, .axes = &.{ a.T, a.D } }, - .ids = .{ .dtype = .i32, .axes = &.{ a.T, a.K } }, - .w = .{ .dtype = .f32, .axes = &.{ a.T, a.K } }, - .gate_up = .{ .dtype = if (fp8) .f8_e4 else .bf16, .axes = &.{ a.E, a.I2, a.D } }, - .down = .{ .dtype = if (fp8) .f8_e4 else .bf16, .axes = &.{ a.E, a.D, a.I } }, - }; - if (!fp8) { - const t = try b.declareArgs(spec); - return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down }; - } - const IG = b.axis("IG", @divExact(cfg.i, 128)); - const DG = b.axis("DG", @divExact(cfg.d, 128)); - const I2G = b.axis("I2G", 2 * @divExact(cfg.i, 128)); - const t = try b.declareArgs(.{ - .x = spec.x, - .ids = spec.ids, - .w = spec.w, - .gate_up = spec.gate_up, - .down = spec.down, - .gate_up_scale = .{ .dtype = cfg.scale_dtype, .axes = &.{ a.E, I2G, DG } }, - .down_scale = .{ .dtype = cfg.scale_dtype, .axes = &.{ a.E, DG, IG } }, - }); - return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down, .gate_up_scale = t.gate_up_scale, .down_scale = t.down_scale }; -} - -const Args = struct { - x: builder.Tensor, - ids: builder.Tensor, - w: builder.Tensor, - gate_up: builder.Tensor, - down: builder.Tensor, - gate_up_scale: ?builder.Tensor = null, - down_scale: ?builder.Tensor = null, -}; - -const GatheredScales = struct { gate_up: ?builder.Tensor = null, down: ?builder.Tensor = null }; - -fn gatherScales(b: *tcl.Builder, a: Axes, in: Args, ids: builder.Tensor, batch: []const tcl.Axis) tcl.FinishError!GatheredScales { - const gu = in.gate_up_scale orelse return .{}; - const IG = b.axis("IG", @divExact(a.I.size, 128)); - const DG = b.axis("DG", @divExact(a.D.size, 128)); - const gate_up = try b.reshape(gu, &.{ a.E, a.S2, IG, DG }, .{}); - var gu_axes: [8]tcl.Axis = undefined; - var dn_axes: [8]tcl.Axis = undefined; - @memcpy(gu_axes[0..batch.len], batch); - @memcpy(dn_axes[0..batch.len], batch); - @memcpy(gu_axes[batch.len..][0..3], &[_]tcl.Axis{ a.S2, IG, DG }); - @memcpy(dn_axes[batch.len..][0..2], &[_]tcl.Axis{ DG, IG }); - return .{ - .gate_up = try b.gather(gate_up, ids, a.E, gu_axes[0 .. batch.len + 3], .{}), - .down = try b.gather(in.down_scale.?, ids, a.E, dn_axes[0 .. batch.len + 2], .{}), - }; -} - -fn blockedAxes(axes: []const tcl.Axis, axis: tcl.Axis, group: tcl.Axis, lane: tcl.Axis, storage: *[16]tcl.Axis) []const tcl.Axis { - var n: usize = 0; - for (axes) |a| { - if (std.mem.eql(u8, a.name, axis.name)) { - storage[n] = group; - n += 1; - storage[n] = lane; - } else storage[n] = a; - n += 1; - } - return storage[0..n]; -} - -/// Block scales multiply each K-block partial before K-block reduction. -/// Only the requested experts are gathered; their bank remains FP8 in HBM. -fn project(b: *tcl.Builder, x: builder.Tensor, weight: builder.Tensor, scale: ?builder.Tensor, contracting: tcl.Axis, output: tcl.Axis, to: []const tcl.Axis, dtype: ?tcl.DType) tcl.FinishError!builder.Tensor { - if (scale == null) { - const op = b.tensorOperation(.{}); - return op.commit(op.contract(x, weight, to), .{ .dtype = dtype }); - } - const CG = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DG" else "IG", @divExact(contracting.size, 128)); - const CL = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DB" else "IB", 128); - const OG = b.axis(if (std.mem.eql(u8, output.name, "D")) "DG" else "IG", @divExact(output.size, 128)); - const OL = b.axis(if (std.mem.eql(u8, output.name, "D")) "DB" else "IB", 128); - var x_axes: [16]tcl.Axis = undefined; - var w_axes: [16]tcl.Axis = undefined; - var w2_axes: [16]tcl.Axis = undefined; - var out_axes: [16]tcl.Axis = undefined; - var partial_axes: [16]tcl.Axis = undefined; - const xb = try b.reshape(x, blockedAxes(x.axes, contracting, CG, CL, &x_axes), .{}); - const wb = try b.reshape(weight, blockedAxes(blockedAxes(weight.axes, output, OG, OL, &w_axes), contracting, CG, CL, &w2_axes), .{}); - const out = blockedAxes(to, output, OG, OL, &out_axes); - @memcpy(partial_axes[0..out.len], out); - partial_axes[out.len] = CG; - - // Furiosa's published FP8 recipe uses an additive epsilon, including zero blocks. - var op = b.tensorOperation(.{}); - const qscale = try op.commit(op.fetch(xb, .{}).unary(.absf).reduce(&.{CL}, .maxf).divf(448.0).addf(1e-12), .{ .dtype = .f32 }); - op = b.tensorOperation(.{}); - // Native F32 reciprocal accuracy is qualified separately (within one ULP). - const reciprocal = op.ve(.divf, .{ 1.0, op.fetch(qscale, .{}) }); - const quantized = try op.commit(reciprocal.mulf(op.fetch(xb, .{})), .{ .dtype = .f8_e4 }); - op = b.tensorOperation(.{}); - const scaled = op.contract(quantized, wb, partial_axes[0 .. out.len + 1]).mulf(op.fetch(qscale, .{})).mulf(op.fetch(scale.?, .{})).reduce(&.{CG}, .addf); - return b.reshape(try op.commit(scaled, .{ .dtype = dtype, .axes = out }), to, .{}); -} - -/// `silu(gate) * up` or `gelu_tanh(gate) * up` of `h = [batch..., S2, I]` -/// (gate first), times `weights` (`[batch...]`) unless null. -fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor, activation: Activation, comptime fp8: bool) tcl.FinishError!builder.Tensor { - var sliced: [4]tcl.Axis = undefined; - var flat: [3]tcl.Axis = undefined; - @memcpy(sliced[0..batch.len], batch); - @memcpy(flat[0..batch.len], batch); - sliced[batch.len] = a.S1; - sliced[batch.len + 1] = a.I; - flat[batch.len] = a.I; - const s = sliced[0 .. batch.len + 2]; - const f = flat[0 .. batch.len + 1]; - const gate = try b.reshape(try b.slice(h, a.S2, 0, s, .{}), f, .{}); - const up = try b.reshape(try b.slice(h, a.S2, 1, s, .{}), f, .{}); - // Interleaving requires one binary instruction; these activations chain several. - var op = b.tensorOperation(.{ .tactic = .elementwise }); - const g = op.fetch(gate, .{}); - const activated = if (fp8) blk: { - // WORKAROUND (TCC 2026.5.0b0): native fusion elides an internal - // BF16 SiLU write, even through as_dram/as_logical. Make its RNE - // rounding observable as integer bits before multiplying by up. - // Remove once the native reproducer preserves the BF16 boundary. - const bits = try op.commit(g.sigmoid().mulf(g).unary(.reinterpret_as_int), .{ .dtype = .i32 }); - op = b.tensorOperation(.{ .tactic = .elementwise }); - const v = op.fetch(bits, .{}); - const rounded = v.binary(.shr_logical, 16).binary(.bit_andi, 1).addi(32767).addi(v).binary(.bit_andi, -65536).unary(.reinterpret_as_float); - const silu = try op.commit(rounded, .{ .dtype = .bf16 }); - op = b.tensorOperation(.{ .tactic = .elementwise }); - break :blk op.fetch(silu, .{}); - } else switch (activation) { - .swiglu => g.sigmoid().mulf(g), - .geglu_tanh => geluTanh(g), - }; - var v = activated.mulf(op.fetch(up, .{})); - if (weights) |w| v = v.mulf(op.fetch(w, .{})); - return op.commit(v, .{ .dtype = .bf16 }); -} - -fn geluTanh(g: builder.Value) builder.Value { - const cube = g.mulf(g).mulf(g); - const inner = g.addf(cube.mulf(0.044715)).mulf(@sqrt(2.0 / std.math.pi)); - return inner.tanh().addf(1.0).mulf(g).mulf(0.5); -} - -/// Furiosa's compiler configuration for MoE kernels (`qwen3_moe/config.py`): -/// IO-bound tactics for activation stationary, compute-bound for blockwise. -/// Keep the compiler's default lowering mode and omit the adaptive-einsum and -/// half-VRF settings removed in TCC 2026.5. -fn configure(b: *tcl.Builder, blockwise: bool) void { - if (blockwise) b.compilerConfig(.{ - .enable_einsum_fusion = true, - .padding_policy = .{ .Small = 1.1 }, - .instruction_mem_budget = 0xB0000, - .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, - .tensor_unit_bridge_threshold_in_page = 12, - .allow_external_operators = false, - .enable_tactic_pruning = false, - .scheduler_beam_search = true, - .allow_reduce_by_ve_cluster_chip_reduce = false, - .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, - .tactic_hint = .{ .ForLlmModelComputeBound = 200 }, - .dma_preference = 1.2, - .propagate_sparse_axis_from_op = "None", - }) else b.compilerConfig(.{ - .enable_einsum_fusion = true, - .padding_policy = .{ .Small = 1.1 }, - .instruction_mem_budget = 0xB0000, - .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, - .tensor_unit_bridge_threshold_in_page = 12, - .allow_external_operators = false, - .enable_tactic_pruning = false, - .scheduler_beam_search = true, - .allow_reduce_by_ve_cluster_chip_reduce = false, - .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, - .tactic_hint = .{ .ForLlmModelIOBound = 200 }, - .num_transaction_simulation_per_pe = 1024, - .dma_preference = 0.8, - .propagate_sparse_axis_from_op = "None", - }); -} - -pub const ActivationStationary = activationStationaryKernel(false); -pub const ActivationStationaryFp8 = activationStationaryKernel(true); - -fn activationStationaryKernel(comptime fp8: bool) type { - return tcl.Kernel(Cfg, .{ - .name = if (fp8) "moe_activation_stationary_fp8" else "moe_activation_stationary", - .inputs = if (fp8) &.{ "x", "ids", "w", "gate_up", "down", "gate_up_scale", "down_scale" } else &.{ "x", "ids", "w", "gate_up", "down" }, - .outputs = &.{"y"}, - .run = struct { - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - configure(b, false); - const a: Axes = .init(b, cfg); - const in = try args(b, a, cfg, fp8); - const gate_up = try b.reshape(in.gate_up, &.{ a.E, a.S2, a.I, a.D }, .{}); - const gu = try b.gather(gate_up, in.ids, a.E, &.{ a.T, a.K, a.S2, a.I, a.D }, .{}); - const dn = try b.gather(in.down, in.ids, a.E, &.{ a.T, a.K, a.D, a.I }, .{}); - - const scales = try gatherScales(b, a, in, in.ids, &.{ a.T, a.K }); - const h = try project(b, in.x, gu, scales.gate_up, a.D, a.I, &.{ a.T, a.K, a.S2, a.I }, if (fp8) .bf16 else null); - const act = try gated(b, a, h, &.{ a.T, a.K }, if (fp8) null else in.w, cfg.activation, fp8); - if (fp8) { - const y = try project(b, act, dn, scales.down, a.I, a.D, &.{ a.T, a.K, a.D }, .f32); - const op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(y, .{}).mulf(op.fetch(in.w, .{})).reduce(&.{a.K}, .addf), .{ .dtype = .bf16 })}); - } else { - // The BF16 recipe weights activations and contracts experts together. - const op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.contract(act, dn, &.{ a.T, a.D }), .{ .dtype = .bf16 })}); - } - } - }.run, - }); -} - -/// Rows per expert block: the next power of two of the routes per expert. -pub fn blockRows(cfg: Cfg) i64 { - const per_expert = std.math.divCeil(i64, cfg.t * cfg.k, cfg.e) catch unreachable; - return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@max(per_expert, 4))) catch unreachable); -} - -const blocks_per_chunk = 32; - -pub const Blockwise = blockwiseKernel(false); -pub const BlockwiseFp8 = blockwiseKernel(true); - -fn blockwiseKernel(comptime fp8: bool) type { - return tcl.Kernel(Cfg, .{ - .name = if (fp8) "moe_blockwise_fp8" else "moe_blockwise", - .inputs = if (fp8) &.{ "x", "ids", "w", "gate_up", "down", "gate_up_scale", "down_scale" } else &.{ "x", "ids", "w", "gate_up", "down" }, - .outputs = &.{"y"}, - .run = struct { - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - configure(b, true); - const a: Axes = .init(b, cfg); - const in = try args(b, a, cfg, fp8); - const bb = blockRows(cfg); - // One block per expert at least, then enough for every route. - const max_blocks = std.math.divCeil(i64, @max(cfg.t * cfg.k - cfg.e, 0), bb) catch unreachable; - const chunks = std.math.divCeil(i64, max_blocks + cfg.e, blocks_per_chunk) catch unreachable; - const BB = b.axis("BB", bb); - const BGi = b.axis("BGi", blocks_per_chunk); - const Go = b.axis("Go", chunks); - const BG = b.axis("BG", chunks * blocks_per_chunk); - const GB = b.axis("GB", chunks * blocks_per_chunk * bb); - const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; - - // Routing: route (t, k) goes to row `slot` of block `start[e] + rank`. - const experts = try b.arange(cfg.e, .i32, &.{E}, .{}); - var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); - op = b.tensorOperation(.{}); - const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const rank = try op.commit(op.fetch(onehot, .{}).cumsum(&.{ T, K }), .{}); - op = b.tensorOperation(.{}); - const log2_bb: i64 = std.math.log2_int(u64, @intCast(bb)); - const blocks = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi).addi(bb - 1).binary(.shr_arith, log2_bb), .{}); - op = b.tensorOperation(.{}); - const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{E}), .{}); - op = b.tensorOperation(.{}); - const start_rows = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})).muli(bb), .{}); - op = b.tensorOperation(.{}); - const slot = try op.commit(op.fetch(rank, .{}).addi(op.fetch(start_rows, .{})).muli(op.fetch(onehot, .{})).reduce(&.{E}, .addi).subi(1), .{}); - // The expert of block g: how many experts end at or before g. - const block_ids = try b.arange(chunks * blocks_per_chunk, .i32, &.{BG}, .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const ends = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(block_ids, .{})), .{}); - op = b.tensorOperation(.{}); - const ended = try op.commit(op.where(ends, .le, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - // Padding blocks past the last expert reuse the last one. - const block_expert = try op.commit(op.fetch(ended, .{}).reduce(&.{E}, .addi).binary(.mini, cfg.e - 1), .{}); - const used = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(block_end), blocks_per_chunk - 1), blocks_per_chunk); - - // Rows of unused slots are never read back. - const rows = try b.reshape(try b.scatter(in.x, slot, GB, &.{ GB, D }, .{}), &.{ Go, BGi, BB, D }, .{}); - const chunk_experts = try b.reshape(block_expert, &.{ Go, BGi }, .{}); - const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); - - var loop = b.openFor(used, .{try b.scratchpad(if (fp8) .f32 else .bf16, &.{ Go, BGi, BB, D })}); - { - const chunk = try b.indexRead(rows, loop.iv); - const ids = try b.indexRead(chunk_experts, loop.iv); - const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); - const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); - const scales = try gatherScales(b, a, in, ids, &.{BGi}); - const h = try project(b, chunk, gu, scales.gate_up, D, I, &.{ BGi, BB, a.S2, I }, if (fp8) .bf16 else null); - const act = try gated(b, a, h, &.{ BGi, BB }, null, cfg.activation, fp8); - const y = try project(b, act, dn, scales.down, I, D, &.{ BGi, BB, D }, if (fp8) .f32 else .bf16); - try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); - } - const out = loop.results[0]; - - const per_route = try b.gather(try b.reshape(out, &.{ GB, D }, .{}), slot, GB, &.{ T, K, D }, .{}); - op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(per_route, .{ .typecast_to = .f32 }).mulf(op.fetch(in.w, .{})).reduce(&.{K}, .addf), .{ .dtype = .bf16 })}); - } - }.run, - }); -} - -/// Experts per `ExpertStationary` loop step: more as the routes, so the -/// experts they use, grow. -pub fn stationaryChunk(cfg: Cfg) i64 { - const routes: u64 = @intCast(std.math.divCeil(i64, cfg.t * cfg.k, 16) catch unreachable); - return @intCast(std.math.clamp(std.math.ceilPowerOfTwo(u64, routes) catch unreachable, 2, 16)); -} - -/// The experts that `ids` use (-1 for none), `stationaryChunk` at a time in a -/// device loop: each is read once and runs for every token. Routing weights -/// scale the F32 down-projection results before summing the experts and steps. -pub const ExpertStationaryFp8 = tcl.Kernel(Cfg, .{ - .name = "moe_expert_stationary_fp8", - .inputs = &.{ "x", "ids", "w", "gate_up", "down", "gate_up_scale", "down_scale" }, - .outputs = &.{"y"}, - .run = struct { - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - configure(b, false); - const a: Axes = .init(b, cfg); - const in = try args(b, a, cfg, true); - const chunk = stationaryChunk(cfg); - const steps = std.math.divCeil(i64, @min(cfg.e, cfg.t * cfg.k), chunk) catch unreachable; - const C = b.axis("C", chunk); - const Go = b.axis("Go", steps); - const J = b.axis("J", steps * chunk); - const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; - - // Each token's routing weight per expert, and the used experts. - const experts = try b.arange(cfg.e, .i32, &.{E}, .{}); - var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); - op = b.tensorOperation(.{}); - const gate = try op.commit(op.where(op.fetch(dist, .{}), .eq, 0, op.fetch(in.w, .{ .broadcast_to = &.{ T, K, E } }), 0.0).reduce(&.{K}, .addf), .{}); - op = b.tensorOperation(.{}); - const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const counts = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi), .{}); - op = b.tensorOperation(.{}); - const used = try op.commit(op.where(counts, .gt, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const position = try op.commit(op.fetch(used, .{}).cumsum(&.{E}), .{}); - // A used expert's rank among them, -1 for the others. - op = b.tensorOperation(.{}); - const rank = try op.commit(op.fetch(position, .{}).muli(op.fetch(used, .{})).subi(1), .{}); - const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(position), chunk - 1), chunk); - - // Slot j holds the used expert of rank j and its tokens' weights; - // slots past the last hold expert 0 with no weight. - const ranks = try b.arange(steps * chunk, .i32, &.{J}, .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const offset = try op.commit(op.fetch(rank, .{}).subi(op.fetch(ranks, .{})), .{}); - op = b.tensorOperation(.{}); - const slot_experts = try op.commit(op.where(op.fetch(offset, .{}), .eq, 0, op.fetch(experts, .{ .broadcast_to = &.{ E, J } }), 0).reduce(&.{E}, .addi), .{ .dtype = .i32 }); - // A reduction's condition is read plainly, so broadcast it first. - op = b.tensorOperation(.{}); - const token_offset = try op.commit(op.fetch(offset, .{ .broadcast_to = &.{ T, E, J } }), .{}); - op = b.tensorOperation(.{}); - const slot_gate = try op.commit(op.where(op.fetch(token_offset, .{}), .eq, 0, op.fetch(gate, .{ .broadcast_to = &.{ T, E, J } }), 0.0).reduce(&.{E}, .addf), .{ .dtype = .f32, .axes = &.{ J, T } }); - const step_experts = try b.reshape(slot_experts, &.{ Go, C }, .{}); - const step_gate = try b.reshape(slot_gate, &.{ Go, C, T }, .{}); - const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); - - var loop = b.openFor(trips, .{try b.full(0, .f32, &.{ Go, T, D })}); - { - const ids = try b.indexRead(step_experts, loop.iv); - const weights = try b.indexRead(step_gate, loop.iv); - const gu = try b.gather(gate_up, ids, E, &.{ C, a.S2, I, D }, .{}); - const dn = try b.gather(in.down, ids, E, &.{ C, D, I }, .{}); - const scales = try gatherScales(b, a, in, ids, &.{C}); - const h = try project(b, in.x, gu, scales.gate_up, D, I, &.{ C, T, a.S2, I }, .bf16); - const act = try gated(b, a, h, &.{ C, T }, null, .swiglu, true); - const per_expert = try project(b, act, dn, scales.down, I, D, &.{ C, T, D }, .f32); - op = b.tensorOperation(.{}); - const y = try op.commit(op.fetch(per_expert, .{}).mulf(op.fetch(weights, .{})).reduce(&.{C}, .addf), .{ .dtype = .f32 }); - try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); - } - op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(loop.results[0], .{}).reduce(&.{Go}, .addf), .{ .dtype = .bf16 })}); - } - }.run, -}); - test "tcl fused experts run on furiosa" { try testFusedExperts(.{ .swiglu = .{} }); try testFusedExperts(.geglu_tanh); @@ -552,7 +158,8 @@ fn testFusedExperts(activation: zml.moe.Activation) !void { const io = std.testing.io; const bf16 = zml.floats.BFloat16; - // 2 * 4 <= 8 experts is activation stationary, 16 tokens is blockwise. + // With two routes per token and eight experts, four tokens use per-route + // dispatch and sixteen use blockwise dispatch. for ([_]i64{ 4, 16 }) |t| { const k = 2; const e = 8; From 9c9161bac6fd8b6dbbfd285120c6a68b69af2844 Mon Sep 17 00:00:00 2001 From: Benjamin Rabier Date: Fri, 13 Nov 2026 17:59:02 +0100 Subject: [PATCH 878/881] platforms/oneapi: update pjrt plugin and bundle libmkl_def (#1070) Update PJRT OneAPI plugin with fixes for Triton --- platforms/oneapi/oneapi.bzl | 5 +++-- platforms/oneapi/packages.lock.json | 6 +++--- platforms/oneapi/packages.yaml | 10 ++++++---- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index 6016e389..5484bec8 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -1,8 +1,8 @@ load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") -_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202610011939.137.1.0d59bb389ee2/zml-oneapi-linux-amd64.tar.zst" -_PLUGIN_ONEAPI_SHA256 = "9ca5e2c377d938b3d855c46c685d193e90424e44b1c1ea8d0a8e1c924b0d11f1" +_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202611131511.374.1.f393d1632b4f/zml-oneapi-linux-amd64.tar.zst" +_PLUGIN_ONEAPI_SHA256 = "374ebcad90493fb311a65d8954120b7f2e8a85d52b8641329ee712ef1cb5b23a" ONEAPI_VERSION = "2026.1" @@ -248,6 +248,7 @@ filegroup( name = "mkl_core_runtime", srcs = [ "{ONEAPI_MKL_LIB}/libmkl_core.so.3", + "{ONEAPI_MKL_LIB}/libmkl_def.so.3", "{ONEAPI_MKL_LIB}/libmkl_intel_ilp64.so.3", "{ONEAPI_MKL_LIB}/libmkl_sequential.so.3", ], diff --git a/platforms/oneapi/packages.lock.json b/platforms/oneapi/packages.lock.json index a3e69ad4..140ce102 100644 --- a/platforms/oneapi/packages.lock.json +++ b/platforms/oneapi/packages.lock.json @@ -216,7 +216,7 @@ "name": "libigc2", "sha256": "c3e5f181eecd40dc8b656c3ff5b7e3d4f690e93181255726516416e61ade1d1f", "urls": [ - "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-graphics-compiler/libigc2_2.41.15-1~24.04_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/i/intel-graphics-compiler/libigc2_2.41.15-1~24.04_amd64.deb" ], "version": "2.41.15-1~24.04" }, @@ -227,7 +227,7 @@ "name": "libigdfcl2", "sha256": "8a2058fd9f5e7d0a4b05001ff861780d2f72529135fa70fb7fbba0f0ff8cd74f", "urls": [ - "https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/pool/main/i/intel-graphics-compiler/libigdfcl2_2.41.15-1~24.04_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/i/intel-graphics-compiler/libigdfcl2_2.41.15-1~24.04_amd64.deb" ], "version": "2.41.15-1~24.04" }, @@ -249,7 +249,7 @@ "name": "libze1", "sha256": "d4001dc474d3cb74ded50a449b9f646cfdfbb31e69e70b8af38f2d18273dfce1", "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z/pool/main/l/level-zero-loader/libze1_1.32.0-1~24.04~ppa1_amd64.deb" + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/l/level-zero-loader/libze1_1.32.0-1~24.04~ppa1_amd64.deb" ], "version": "1.32.0-1~24.04~ppa1" }, diff --git a/platforms/oneapi/packages.yaml b/platforms/oneapi/packages.yaml index 272bc02c..5a829ac5 100644 --- a/platforms/oneapi/packages.yaml +++ b/platforms/oneapi/packages.yaml @@ -15,9 +15,11 @@ sources: - channel: noble main url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z - # IGC 2.41.15 is newer than the graphics snapshot above. + # IGC 2.41.15 is newer than the graphics snapshot above; the other graphics + # packages are pinned below to keep their versions from it. Use snapshots, + # not the live PPA: the PPA deletes superseded packages from its pool. - channel: noble main - url: https://ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu + url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z archs: - "amd64" @@ -43,9 +45,9 @@ packages: - "libnl-genl-3-200" - "libnl-route-3-200" - "libnuma1" - - "libigdgmm12" + - "libigdgmm12 (= 22.10.1-1~24.04~ppa1)" - "libigc2 (= 2.41.15-1~24.04)" - "libigdfcl2 (= 2.41.15-1~24.04)" - - "libze-intel-gpu1" + - "libze-intel-gpu1 (= 26.31.39395.14-1~24.04~ppa1)" - "libze1" - "zlib1g" From f65072a740fee2b415649f0c41e58935b55410e6 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Fri, 13 Nov 2026 22:09:27 +0100 Subject: [PATCH 879/881] platforms/furiosa: use all RNGD chips by default (#1083) --- platforms/furiosa/furiosa.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/platforms/furiosa/furiosa.zig b/platforms/furiosa/furiosa.zig index ebbd9e3c..ee33fd58 100644 --- a/platforms/furiosa/furiosa.zig +++ b/platforms/furiosa/furiosa.zig @@ -50,7 +50,6 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; try setEnv("FURIOSA_SDK_ROOT", try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{root}), 1); - try setEnv("FURIOSA_VISIBLE_DEVICES", "0", 0); const library = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ root, "lib", "libzml_furiosa.so" }); log.info("Loading Furiosa plugin: {s}", .{library}); From 8fcff8c462070d06056f8a85ba0c8a81f0f3a636 Mon Sep 17 00:00:00 2001 From: Raphael Nahoum Date: Fri, 13 Nov 2026 22:27:25 +0100 Subject: [PATCH 880/881] workspace: bump ZML plugins (#1082) Bump plugins to [202611132003.375.1.7d765f57a407](https://github.com/zml/xla-private/tree/202611132003.375.1.7d765f57a407) oneAPI: weight-only FP8 and FP4 (NVFP4, MXFP4) scaled dots now run in a Triton kernel that reads the weights in their stored format. --- platforms/oneapi/oneapi.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl index 5484bec8..a3c968d3 100644 --- a/platforms/oneapi/oneapi.bzl +++ b/platforms/oneapi/oneapi.bzl @@ -1,8 +1,8 @@ load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") -_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202611131511.374.1.f393d1632b4f/zml-oneapi-linux-amd64.tar.zst" -_PLUGIN_ONEAPI_SHA256 = "374ebcad90493fb311a65d8954120b7f2e8a85d52b8641329ee712ef1cb5b23a" +_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202611132003.375.1.7d765f57a407/zml-oneapi-linux-amd64.tar.zst" +_PLUGIN_ONEAPI_SHA256 = "294722670832c543c8ac4c074c30f636f2a3b0854820f7c5a070fa82c0d6f3d5" ONEAPI_VERSION = "2026.1" From 1a26a051a50ed8293e8c9edb539a93ef0858ca7a Mon Sep 17 00:00:00 2001 From: erwanvivien <44021072+erwanvivien@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:17:19 +0100 Subject: [PATCH 881/881] zml/tensor: return a distinct tensor from toMemory on single-memory platforms (#1087) On cpu, neuron and metal, `toMemory` returned self. Donating a result to a device buffer and its `toMemory` copy to a host buffer then called `reuseBuffer` twice on the same tensor id: the second donation replaced the first, and compilation panicked because both outputs aliased the host buffer. This broke llmd async scheduling on Metal and CPU. Return the same value under a new tensor id instead. --- zml/tensor.zig | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/zml/tensor.zig b/zml/tensor.zig index a09a5d38..ee084835 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -282,7 +282,13 @@ pub const Tensor = struct { pub fn toMemory(self: Tensor, kind: Memory.Kind) Tensor { const ctx = Compiler.current(); switch (ctx.platform.target) { - .cpu, .neuron, .metal => return self, + .cpu, .neuron, .metal => { + // A single memory needs no copy, but the result must still be a distinct + // tensor: `reuseBuffer` donations are keyed by tensor id. + var res = _result(self._shape, self.value()); + res._shape._sharding = self._shape._sharding; + return res; + }, .cuda, .rocm, .tpu, .oneapi, .furiosa => {}, }